diff --git a/.config/ci-slow-tests.txt b/.config/ci-slow-tests.txt new file mode 100644 index 000000000..f8f3b14ec --- /dev/null +++ b/.config/ci-slow-tests.txt @@ -0,0 +1,327 @@ +# Tests GitHub CI skips: every test that took more than 45 s in one prover shard of GitHub run 40517634531 +# (a69ccd64f), then every test still run that took more than 42 s in run 40588562061 (d99e180f8, the first run under +# this list: one shard took 43 min); nextest, one test at a time, on a 4-CPU runner. They still run: the full CI +# reproduction on the project's own GPU machines runs every test, and so do `cargo test` and `cargo nextest run` +# locally. Only the workflow's prover shards read this file (scripts/ci_slow_tests.py builds their filter from it). +# +# One test a line: ` `, then its seconds in that run. Exact names only: a test that is not +# listed runs on GitHub, so a new slow test stays in CI until it is listed here, and a renamed test runs again +# (the build job prints the listed names it no longer finds). Remove a line once its test is fast again. + +lambda-vm-prover tests::block_whir_tests::cpu_and_memw_r_filled_from_records_move_no_byte # 2222 s +lambda-vm-prover tests::block_whir_tests::phase_b_in_any_order_proves_the_same_bytes # 1452 s +lambda-vm-prover tests::block_whir_tests::counting_the_hand_out_on_its_own_thread_moves_no_byte # 1117 s (box CI, 8 runners) +lambda-vm-prover lfm::algebraic_transcript::tests::the_machine_regroups_bytes_exactly_as_the_host_cellifies_them # 982 s +lambda-vm-prover tests::block_whir_tests::split_keccak_and_ecsm_calls_are_keyed_on_the_bus # 832 s +lambda-vm-prover lfm::algebraic_commit::tests::the_emitted_leaf_and_parent_equal_the_host_backend # 822 s +lambda-vm-prover tests::block_whir_tests::live_regeneration_proves_the_same_bytes # 769 s +lambda-vm-prover tests::block_whir_tests::the_rest_in_waves_and_the_finish_packed_move_no_byte # 717 s +lambda-vm-prover tests::block_whir_tests::the_shadow_regenerates_every_streamed_chunk_and_moves_no_byte # 640 s +lambda-vm-prover tests::block_whir_tests::gpack_moves_no_byte # 629 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_with_dropped_ops_proves_and_verifies # 627 s +lambda-vm-prover tests::block_whir_tests::streamed_records_chunks_drop_as_records_and_come_back # 597 s +lambda-vm-prover tests::block_whir_tests::a_deeper_group_queue_moves_no_byte_of_the_proof # 532 s +lambda-vm-prover tests::block_whir_tests::parked_records_need_live_regeneration # 483 s +lambda-vm-prover tests::block_whir_tests::without_live_regeneration_every_table_is_held # not timed: two of the three proves of the 470 s test it replaced +lambda-vm-prover tests::block_whir_tests::streamed_records_chunks_kept_are_never_dropped # 460 s +lambda-vm-prover tests::block_whir_tests::the_shadow_stays_off_where_it_cannot_rebuild_the_chunks # 455 s +lambda-vm-prover tests::block_whir_tests::row_families_filled_from_their_rows_move_no_byte # 418 s +lambda-vm-prover tests::block_batched_tests::a_batched_group_argued_wrongly_is_refused # 414 s +lambda-vm-prover tests::block_whir_tests::keccak_rnd_filled_from_records_moves_no_byte # 358 s +lambda-vm-prover tests::block_whir_tests::regenerators_started_at_the_finishs_end_prove_the_same_bytes # 356 s +lambda-vm-prover tests::block_whir_tests::row_records_chunks_drop_as_records_and_come_back # 352 s +lambda-vm-prover tests::block_whir_tests::phase_b_overlap_failures_are_the_proves_error_and_a_refusal_turns_it_serial # 351 s +lambda-vm-prover tests::block_whir_tests::phase_b_led_by_a_leaf_proves_the_same_bytes # not timed: four proves of the streamed keccak block, as the 351 s test above +lambda-vm-prover lfm::machine_tests::tampered_program_id_inputs_change_the_id # 346 s +lambda-vm-prover tests::block_whir_tests::keccak_and_ecsm_split_into_chunks_prove_and_verify # 324 s +lambda-vm-prover tests::block_whir_tests::a_block_with_keccak256_calls_across_keccak_tables_proves_and_verifies # not timed on GitHub: 21 s on a 9950X box, a block of KECCAK tables cut to a few rows, as the 324 s test above +lambda-vm-prover tests::block_whir_tests::ecrec_chains_are_keyed_apart_on_the_bus # not timed on GitHub: the two ECREC block tests and two others took 102 s together on a 9950X box; ECDAS cut to a few rows, as the 324 s test above +lambda-vm-prover tests::block_whir_tests::ecrec_chains_swapped_between_keys_are_refused # not timed on GitHub: three block proves of test_ecrec (two honest, one forged), as the test above +lambda-vm-prover tests::block_whir_tests::a_wrong_regeneration_is_reported_never_proved # 320 s +lambda-vm-prover tests::block_whir_tests::a_split_ecdas_call_is_keyed_on_the_bus # 317 s +lambda-vm-prover tests::block_onechain_tests::an_indexed_block_round_trips_and_refuses_the_other_mode # 305 s +lambda-vm-prover tests::block_whir_tests::copy_split_into_chunks_proves_and_verifies # 304 s +lambda-vm-prover tests::block_whir_tests::a_wrong_row_records_table_is_refused_by_the_verifier # 302 s +lambda-vm-prover tests::block_whir_tests::a_regenerator_up_before_the_drops_proves_the_same_bytes # 300 s +lambda-vm-prover tests::block_whir_tests::one_byte_windows_take_each_class_in_its_order # 291 s +lambda-vm-prover tests::block_whir_tests::the_kept_tree_depth_moves_no_byte_of_the_proof # 291 s +lambda-vm-prover tables::trace_builder::finish::stream_tests::the_stream_is_the_monolithic_finish_in_air_order # 289 s +lambda-vm-prover tests::is_wh_tests::test_keccak_continuation_carries_is_wh_in_its_epoch # 278 s +lambda-vm-prover lfm::machine_tests::keccak_sponge_reference_lengths_prove_and_verify # 274 s +lambda-vm-prover tests::block_whir_tests::narrow_storage_moves_no_byte_of_the_proof # 273 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_with_keccak_and_ecsm_chunks_proves_and_verifies # 270 s +lambda-vm-prover tests::block_whir_tests::keccak_rnd_held_as_records_is_never_dropped # 270 s +lambda-vm-prover tests::block_whir_tests::a_hand_off_at_the_walk_arms_live_regeneration_and_moves_no_byte # not timed: three proves, as the 269 s test it replaced +lambda-vm-prover tests::is_wh_tests::test_is_wh_noncanonical_word_is_rejected # 258 s +lambda-vm-prover tests::block_whir_tests::uploading_ahead_moves_no_byte_of_the_proof # 256 s +lambda-vm-prover lfm::blake3_socket_tests::tampering_with_the_witness_is_not_accepted # 250 s +lambda-vm-prover tests::block_whir_tests::bitwise_from_tables_commits_bitwise_lasts_tables_and_verifies # 248 s +lambda-vm-prover lfm::epoch_tests::the_from_proof_constructor_matches_the_session_harness # 239 s +lambda-vm-prover lfm::blake3_chip_tests::the_blake3_chip_proves_and_verifies # 235 s +lambda-vm-prover continuation::tests::test_classify_only_matches_byte_level_classification # 224 s +lambda-vm-prover tests::block_whir_tests::the_rest_regenerator_refuses_what_it_did_not_rebuild # 222 s +lambda-vm-prover continuation::tests::prep_ahead_proves_what_the_prover_thread_schedule_proves # 217 s +lambda-vm-prover tests::block_whir_tests::live_regeneration_refuses_what_it_did_not_rebuild # 215 s +lambda-vm-prover tests::block_whir_tests::a_block_without_its_is_wh_table_is_refused # 208 s +lambda-vm-prover lfm::blake3_chip_tests::the_reversed_digest_send_proves_and_verifies # 208 s +lambda-vm-prover tests::block_whir_tests::a_chunked_table_stated_over_its_cap_is_refused # 204 s +lambda-vm-prover lfm::machine_tests::tampered_merkle_opening_rejects # 201 s +lambda-vm-prover tests::block_batched_tests::the_bus_balance_binds_the_batched_block # 198 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_laid_out_on_three_threads_packs_the_same_groups # 197 s +lambda-vm-prover tests::recursion_smoke_test::test_recursion_continuation_blob_decodes_and_verifies_on_host # 196 s +lambda-vm-prover tests::block_whir_tests::bitwise_handed_on_last_moves_the_partition_and_verifies # 192 s +lambda-vm-prover tests::block_whir_tests::ecdas_split_into_chunks_proves_and_verifies # 190 s +lambda-vm-prover lfm::machine_tests::machine_proves_the_sample_replay # 190 s +lambda-vm-prover tests::recursion_smoke_test::test_recursion_accepts_a_bundle_whose_epochs_carry_different_tables # 190 s +lambda-vm-prover tests::block_whir_tests::a_wrong_cpu_records_table_is_refused_by_the_verifier # 189 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_dropped_last_epoch # 188 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_dropped_touched_page_base # 187 s +lambda-vm-prover tests::block_whir_tests::a_block_missing_an_instance_is_refused_on_the_bus # 186 s +lambda-vm-prover continuation::tests::test_continuation_blob_rejects_tampered_l2g_root # 186 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_reordered_epochs # 186 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_tampered_l2g_root # 186 s +lambda-vm-prover continuation::tests::table_presence_is_decided_per_epoch # 184 s +lambda-vm-prover lfm::machine_tests::tampered_statement_or_root_rejects # 181 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_an_epoch_whose_counts_overflow # 179 s +lambda-vm-prover continuation::tests::test_split_verify_tolerates_reordered_touched_page_bases # 179 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_tampered_register_fini # 179 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_non_page_aligned_touched_page_base # 179 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_malformed_register_fini_length # 178 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_oversized_num_private_input_pages # 178 s +lambda-vm-prover tests::block_whir_tests::a_wrongly_committed_prepared_stack_is_refused # 174 s +lambda-vm-prover lfm::epoch_tests::a_lead_in_prologue_is_the_one_built_from_the_bundle # 170 s +lambda-vm-prover tests::prove_elfs_tests::test_continuation_pipeline_end_to_end # 168 s +lambda-vm-prover lfm::machine_tests::tampered_transcript_absorb_half_rejects # 168 s +lambda-vm-prover lfm::blake3_chip_tests::blake3_chunking_does_not_change_what_is_proved # 167 s +lambda-vm-prover tests::block_batched_tests::a_block_argued_batched_proves_and_verifies # 163 s +lambda-vm-prover continuation::tests::test_dma_memcpy_across_continuation_epochs # 155 s +lambda-vm-prover continuation::tests::test_commit_across_epochs_verifies # 151 s +lambda-vm-prover tests::block_onechain_tests::an_indexed_coset_block_is_refused_by_a_natural_verifier # 144 s +lambda-vm-prover lfm::machine_tests::the_merkle_walk_authenticates_a_real_opening # 142 s +lambda-vm-prover tests::block_whir_tests::the_prover_refuses_a_partition_over_the_group_maximum # 142 s +lambda-vm-prover lfm::machine_tests::tampering_the_register_files_moves_the_derived_root # 142 s +lambda-vm-prover tests::block_whir_tests::split_copy_chains_are_keyed_on_the_bus # 140 s +lambda-vm-prover continuation::tests::test_continuation_blob_rejects_global_proof_from_a_different_run # 136 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_global_proof_from_a_different_run # 135 s +lambda-vm-prover tests::block_whir_tests::a_block_at_blowup_2_is_refused_by_a_verifier_at_blowup_4 # 134 s +lambda-vm-prover lfm::machine_tests::program_id_folds_pages_in_the_production_layout # 134 s +lambda-vm-prover tests::block_whir_tests::a_block_ground_at_18_bits_is_refused_by_a_verifier_at_20 # 132 s +lambda-vm-prover lfm::machine_tests::tampered_stream_half_rejects # 132 s +lambda-vm-prover lfm::leaf_tests::fri_toy_proves_and_verifies_under_every_hasher # 131 s +lambda-vm-prover lfm::blake3_chip_tests::both_blake3_surfaces_in_one_machine_balance_bitwise # 131 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_uploading_ahead_proves_and_verifies # 130 s +lambda-vm-prover lfm::blake3_chip_tests::the_two_wrap_hashes_produce_mutually_unverifiable_proofs # 130 s +lambda-vm-prover tests::is_wh_tests::test_keccak_noncanonical_state_ptr_is_rejected # 128 s +lambda-vm-prover continuation::halt_continuation_poc::continuation_continue_past_exit_in_final_of_two_epochs_is_rejected # 128 s +lambda-vm-prover tests::block_batched_tests::prepared_openings_settle_under_the_batched_argue # 128 s +lambda-vm-prover lfm::wrap_tests::the_fixture_epoch_wraps # 122 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_deflated_num_private_input_pages # 119 s +lambda-vm-prover lfm::algebraic_transcript::tests::the_host_transcript_and_the_machine_replay_derive_the_same_challenges # 119 s +lambda-vm-prover continuation::tests::test_continuation_epoch_size_log2 # 119 s +lambda-vm-prover tests::prove_elfs_tests::test_prove_ef_io_demo_concatenates # 119 s +lambda-vm-prover tests::cpu_timestamp_tests::hostile_cpu_heights_are_rejected_by_the_verifier # 118 s +lambda-vm-prover continuation::tests::test_split_verify_roundtrip # 117 s +lambda-vm-prover tests::block_whir_tests::the_bounded_layout_holds_at_most_k_plus_one_chunks_unpacked # 115 s +lambda-vm-prover tests::block_whir_tests::a_memory_log_moves_no_byte_and_its_terms_close # 114 s +lambda-vm-prover tests::local_to_global_bus_tests::test_continuation_private_input_spans_epochs # 113 s +lambda-vm-prover continuation::tests::test_continuation_private_input_verifies_without_bytes # 113 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_inflated_num_private_input_pages # 113 s +lambda-vm-prover continuation::tests::test_split_verify_rejects_undercounted_epoch_table_count # 112 s +lambda-vm-prover continuation::tests::test_prove_and_verify_continuation # 111 s +lambda-vm-prover tests::page_offset_forgery_poc::dup_duplicate_page_forgery_is_rejected # 111 s +lambda-vm-prover lfm::program_shape_tests::dropping_bitwise_under_a_byte_lookup_family_is_refused # 109 s +lambda-vm-prover tests::cpu_timestamp_tests::split_cpu_tables_verify # 108 s +lambda-vm-prover lfm::algebraic_transcript::tests::the_emitter_replay_derives_the_host_transcripts_challenges_and_state # 107 s +lambda-vm-prover lfm::transcript_tests::the_transcript_proves_and_verifies_under_every_hasher # 106 s +lambda-vm-prover tests::zf_vm_one_row_tests::a_vm_proof_with_public_output_round_trips_at_one_row # 105 s +lambda-vm-prover lfm::machine_tests::keccak_merkle_opening_cost # 105 s +lambda-vm-prover continuation::halt_continuation_poc::poc_control_patched_program_two_epochs_verifies # 104 s +lambda-vm-prover tests::multiplicity_forgery_poc::dvrm_negative_mu_r_forgery_is_rejected # 104 s +lambda-vm-prover tests::block_batched_tests::a_tampered_batched_argue_is_refused # 103 s +lambda-vm-prover lfm::machine_tests::real_opening_is_a_usable_tamper_target # 102 s +lambda-vm-prover tests::page_offset_forgery_poc::poc_control_honest_harness_verifies # 102 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_with_keccak_chunks_proves_and_verifies # 101 s +lambda-vm-prover lfm::program_shape_tests::a_program_without_byte_lookups_proves_and_verifies_without_bitwise # 99 s +lambda-vm-prover lfm::algebraic_transcript::tests::phase_a_absorbs_derive_the_hosts_shared_pair # 97 s +lambda-vm-prover tests::multiplicity_forgery_poc::mul_unanswered_forged_claim_is_rejected # 97 s +lambda-vm-prover tests::cpu_timestamp_tests::cpu_proof_start_matches_air_refs_layout # 96 s +lambda-vm-prover tests::block_onechain_tests::an_indexed_block_round_trips_under_the_batched_argue # 96 s +lambda-vm-prover tests::multiplicity_forgery_poc::load_unanswered_forged_claim_is_rejected # 95 s +lambda-vm-prover tests::multiplicity_forgery_poc::mul_negative_mu_hi_forgery_is_rejected # 95 s +lambda-vm-prover tests::cpu_timestamp_tests::swapped_cpu_tables_are_rejected # 95 s +lambda-vm-prover tables::trace_builder::finish::stream_tests::with_bitwise_from_tables_the_stream_is_the_monolithic_finish # 94 s +lambda-vm-prover tables::trace_builder::finish::stream_tests::with_bitwise_last_the_sliding_stream_hands_bitwise_on_last # 94 s +lambda-vm-prover tests::block_whir_tests::a_wrong_records_table_is_refused_by_the_verifier # 93 s +lambda-vm-prover tests::is_wh_tests::test_is_wh_canonical_word_rewrite_verifies # 93 s +lambda-vm-prover tests::multiplicity_forgery_poc::shift_unanswered_forged_claim_is_rejected # 93 s +lambda-vm-prover tests::static_commitments_tests::a_p1_static_is_taken_at_its_own_key_only # 93 s +lambda-vm-prover lfm::algebraic_commit::tests::the_emitted_grinding_check_agrees_with_the_host_on_the_incident_nonces # 93 s +lambda-vm-prover tests::is_wh_tests::test_is_wh_negative_mu_is_rejected # 93 s +lambda-vm-prover tests::zf_vm_one_row_tests::a_vm_proof_round_trips_at_one_row # 93 s +lambda-vm-prover tests::multiplicity_forgery_poc::dvrm_honest_control_verifies # 92 s +lambda-vm-prover tests::page_offset_forgery_poc::poc_negative_control_forged_run_without_repointed_row_fails # 92 s +lambda-vm-prover tests::is_wh_tests::test_keccak_state_across_64k_boundary_verifies # 92 s +lambda-vm-prover tests::prove_elfs_tests::test_verify_rejects_tampered_num_private_input_pages_zero # 92 s +lambda-vm-prover tests::is_wh_tests::test_keccak_lane_on_its_own_is_wh_row_verifies # 91 s +lambda-vm-prover tests::zf_vm_one_row_tests::a_vm_proof_round_trips_at_one_row_auto_with_dp # 91 s +lambda-vm-prover tests::block_onechain_tests::an_indexed_block_at_blowup_2_is_refused_by_a_verifier_at_blowup_4 # 90 s +lambda-vm-prover tables::trace_builder::finish::stream_tests::the_sliding_stream_is_the_monolithic_finish_in_air_order # 90 s +lambda-vm-prover lfm::blake3_chip_tests::a_tampered_non_first_blake3_chunk_rejects # 89 s +lambda-vm-prover tests::multiplicity_forgery_poc::load_honest_control_verifies # 87 s +lambda-vm-prover lfm::machine_tests::chunking_does_not_change_what_is_proved # 87 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_held_narrow_proves_and_verifies # 86 s +lambda-vm-prover lfm::machine_tests::arena_filler_reads_real_committed_roots # 86 s +lambda-vm-prover lfm::machine_tests::continuation_fixture_generates_two_epochs # 86 s +lambda-vm-prover tests::multiplicity_forgery_poc::lt_honest_control_verifies # 86 s +lambda-vm-prover lfm::machine_tests::the_derivation_extends_a_non_u32_register_value_demonstrating_hazard # 85 s +lambda-vm-prover lfm::epoch_tests::the_from_proof_constructor_rejects_a_tampered_bundle # 85 s +lambda-vm-prover tests::windowed_builder_tests::dropping_the_streamed_ops_builds_the_same_tables # 84 s +lambda-vm-prover tests::block_onechain_tests::a_coset_block_at_a_card_sized_stack_encodes_no_successor_on_the_host # 84 s +lambda-vm-prover tests::multiplicity_forgery_poc::lt_unanswered_forged_claim_is_rejected # 84 s +lambda-vm-prover tests::multiplicity_forgery_poc::mul_honest_control_verifies # 84 s +lambda-vm-prover lfm::machine_tests::preprocessed_tags_close_the_output_swap_hazard # 83 s +lambda-vm-prover lfm::machine_tests::tampered_keccak_input_half_rejects # 83 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_with_ecdas_chunks_proves_and_verifies # 83 s +lambda-vm-prover tests::prove_elfs_tests::test_verify_rejects_num_private_input_pages_exceeds_max # 83 s +lambda-vm-prover tests::page_offset_forgery_poc::dup_structural_duplicate_page_coverage_is_rejected # 83 s +lambda-vm-prover lfm::blake3_chip_tests::chunked_blake3_proves_and_verifies # 83 s +lambda-vm-prover tests::page_offset_forgery_poc::dup_negative_control_without_compensating_row_fails # 82 s +lambda-vm-prover tests::prove_elfs_tests::test_verify_rejects_private_input_with_tampered_public_output # 81 s +lambda-vm-prover tests::pc_double_read_poc::preprocessed_timestamp_alone_rejects_the_forgery # 80 s +lambda-vm-prover tests::halt_pc_forgery_poc::continue_past_exit_via_halt_pc_is_rejected # 80 s +lambda-vm-prover tests::prove_elfs_tests::test_verify_rejects_inflated_num_private_input_pages # 80 s +lambda-vm-prover continuation::tests::test_prover_panic_mid_pipeline_returns_err # 78 s +lambda-vm-prover tests::multiplicity_forgery_poc::shift_zbs_two_padding_row_forgery_is_rejected # 78 s +lambda-vm-prover tests::halt_pc_forgery_poc::continue_past_exit_with_pinned_fini_is_rejected # 78 s +lambda-vm-prover tables::trace_builder::finish_tests::a_build_cut_at_the_last_real_row_is_the_oracles_without_its_padding_tables # 77 s +lambda-vm-prover lfm::machine_tests::permute_row_cannot_substitute_the_permuted_state # 77 s +lambda-vm-prover tests::skip_empty_tables_tests::no_present_table_contributes_zero_to_the_bus # 77 s +lambda-vm-prover tables::trace_builder::finish_tests::the_plan_builds_the_oracles_windowed_tables # 77 s +lambda-vm-prover lfm::machine_tests::permutations_may_be_reassigned_across_chunk_boundaries # 76 s +lambda-vm-prover tests::block_onechain_tests::a_composed_block_is_refused_one_setting_away # 76 s +lambda-vm-prover tests::pc_double_read_poc::poc_control_honest_exit_five_is_rejected # 76 s +lambda-vm-prover tests::block_whir_tests::a_streamed_block_proves_and_verifies # 76 s +lambda-vm-prover tests::multiplicity_forgery_poc::shift_honest_control_verifies # 76 s +lambda-vm-prover lfm::machine_tests::statement_replay_proves_and_verifies # 75 s +lambda-vm-prover lfm::algebraic_transcript::tests::the_statement_replay_absorbs_the_hosts_calls_one_for_one # 75 s +lambda-vm-prover tests::pc_double_read_poc::final_pc_pin_alone_rejects_the_forgery # 75 s +lambda-vm-prover tests::decode_tests::decode_commitment_compile_time_const_accepts # 75 s +lambda-vm-prover lfm::machine_tests::splice_proves_and_verifies # 75 s +lambda-vm-prover lfm::machine_tests::append_ext_proves_and_verifies # 74 s +lambda-vm-prover tests::prove_elfs_tests::test_proof_does_not_contain_private_input_field # 74 s +lambda-vm-prover continuation::tests::test_continuation_rejects_hostile_epoch_cpu_heights # 74 s +lambda-vm-prover lfm::machine_tests::verify_rejects_a_chunk_count_that_does_not_match_the_proof # 74 s +lambda-vm-prover lfm::blake3_chip_tests::tampering_with_the_blake3_witness_is_not_accepted # 73 s +lambda-vm-prover tests::halt_pc_forgery_poc::poc_control_patched_program_verifies # 73 s +lambda-vm-prover lfm::algebraic_commit::tests::the_emitted_grinding_check_accepts_exactly_the_hosts_nonces # 73 s +lambda-vm-prover tests::decode_tests::decode_commitment_some_matches_default_path # 73 s +lambda-vm-prover lfm::blake3_chip_tests::keccak_still_proves_and_verifies_through_the_switched_emitters # 73 s +lambda-vm-prover lfm::blake3_chip_tests::verify_rejects_a_blake3_chunk_list_that_does_not_match_the_proof # 72 s +lambda-vm-prover tests::pc_double_read_poc::poc_control_patched_program_verifies # 72 s +lambda-vm-prover tests::block_whir_p1_tests::the_p1_statement_tag_is_bound # 71 s +lambda-vm-prover tests::block_whir_tests::a_tampered_prepared_opening_is_refused # 71 s +lambda-vm-prover continuation::tests::test_continuation_rkyv_roundtrip # 70 s +lambda-vm-prover lfm::blake3_socket_tests::a_blake3_proof_does_not_verify_under_another_hasher # 70 s +lambda-vm-prover lfm::machine_tests::keccak_sponge_proves_and_verifies # 70 s +lambda-vm-prover tests::halt_pc_forgery_poc::poc_control_honest_exit_one_is_rejected # 70 s +lambda-vm-prover lfm::machine_tests::tampered_second_chunk_permutation_rejects # 68 s +lambda-vm-prover tests::page_tests::page_commitments_zero_bytes_rejects # 68 s +lambda-vm-prover tests::block_whir_tests::a_group_over_the_stack_budget_is_refused # 68 s +lambda-vm-prover tests::block_whir_tests::a_block_proof_survives_serialization # 68 s +lambda-vm-prover lfm::machine_tests::program_id_matches_production_on_the_real_fixture # 68 s +lambda-vm-prover lfm::transcript_tests::the_transcript_proves_and_verifies_under_blake3 # 67 s +lambda-vm-prover tests::pc_double_read_poc::poc_forgery_verifies_against_prover_chosen_pins # 66 s +lambda-vm-prover tests::block_whir_tests::a_program_spread_over_many_groups_proves_and_verifies # 66 s +lambda-vm-prover lfm::machine_tests::tampered_absorb_xor_rejects # 66 s +lambda-vm-prover tests::block_whir_tests::a_restated_cpu_table_height_is_refused # 66 s +lambda-vm-prover lfm::machine_tests::a_blake3_authenticating_program_carries_no_keccak_family # 66 s +lambda-vm-prover lfm::leaf_tests::fri_toy_proves_and_verifies_under_blake3 # 65 s +lambda-vm-prover tests::block_batched_tests::a_batched_group_on_another_fork_is_refused # 65 s +lambda-vm-prover tests::block_whir_tests::a_tampered_column_claim_is_refused # 64 s +lambda-vm-prover tests::block_whir_tests::a_host_only_build_drains_the_groups_the_prove_commits # 64 s +lambda-vm-prover tests::block_whir_p1_tests::a_block_proves_under_its_family_base_and_no_other_verifies_it # 64 s +lambda-vm-prover lfm::blake3_chip_tests::the_env_knob_value_chunks_a_real_prove # 63 s +lambda-vm-prover tests::prove_elfs_tests::test_prove_dma_memmove_cases_rust_guest # 63 s +lambda-vm-prover tests::block_whir_tests::a_partition_over_the_group_maximum_is_refused # 62 s +lambda-vm-prover tests::block_whir_tests::a_group_proved_on_another_fork_is_refused # 62 s +lambda-vm-prover tests::block_whir_tests::a_block_with_another_final_register_file_is_refused # 62 s +lambda-vm-prover lfm::machine_tests::transcript_replay_proves_and_verifies # 62 s +lambda-vm-prover lfm::leaf_tests::fri_toy_rejects_a_fixture_built_under_another_hasher # 61 s +lambda-vm-prover tests::block_whir_tests::a_tampered_bus_output_is_refused # 61 s +lambda-vm-prover lfm::machine_tests::tampered_keccak_output_half_rejects # 61 s +lambda-vm-prover tests::constraint_set_tests_a::ecsm::ecsm_constraint_set_folder_capture_agree # 60 s +lambda-vm-prover continuation::halt_continuation_poc::poc_control_patched_program_continuation_verifies # 58 s +lambda-vm-prover tests::block_whir_tests::a_block_with_a_negative_keccak_multiplicity_is_refused # 58 s +lambda-vm-prover tests::block_whir_tests::the_statement_bytes_are_what_absorb_block_appends # 58 s +lambda-vm-prover tests::block_batched_tests::keccak_chunks_argue_batched # 58 s +lambda-vm-prover tests::block_whir_tests::a_table_stated_over_the_table_cap_is_refused # 58 s +lambda-vm-prover tests::is_wh_tests::test_recursion_blob_with_is_wh_verifies_on_host # 58 s +lambda-vm-prover tests::block_whir_p1_tests::the_statement_calls_are_absorb_blocks_own # 57 s +lambda-vm-prover tests::block_whir_tests::a_partition_missing_a_table_is_refused # 57 s +lambda-vm-prover lfm::blake3_socket_tests::the_trivial_program_proves_and_verifies_under_blake3 # 56 s +lambda-vm-prover tests::recursion_soundness_gap_poc::test_supplied_decode_root_not_bound_to_inner_elf # 56 s +lambda-vm-prover tests::is_wh_tests::test_is_wh_positive_split_verifies # 56 s +lambda-vm-prover continuation::halt_continuation_poc::continuation_continue_past_exit_is_rejected # 56 s +lambda-vm-prover tests::block_whir_tests::two_groups_prepared_openings_swapped_are_refused # 56 s +lambda-vm-prover lfm::machine_tests::dropping_the_second_chunks_permutation_rejects # 56 s +lambda-vm-prover tests::is_wh_tests::test_is_wh_honest_keccak_verifies # 56 s +lambda-vm-prover lfm::keccak_probe::adapter_probe_proves_real_permutations # 55 s +lambda-vm-prover lfm::whir_p1_census_tests::the_median_mix_leaves_are_pinned_under_each_query_check # 55 s +lambda-vm-prover continuation::tests::test_continuation_multipage_private_input # 55 s +lambda-vm-prover tests::block_whir_tests::keccak_rnd_split_into_chunks_proves_and_verifies # 54 s +lambda-vm-prover tests::recursion_smoke_test::test_recursion_rejects_corrupted_commitment # 54 s +lambda-vm-prover continuation::halt_continuation_poc::poc_control_honest_exit_one_continuation_is_rejected # 54 s +lambda-vm-prover tests::skip_empty_tables_tests::dropping_a_used_table_through_the_real_verifier_is_rejected # 54 s +lambda-vm-prover tests::recursion_soundness_gap_poc::test_custom_prover_is_not_vacuous # 53 s +lambda-vm-prover lfm::program_shape_tests::a_split_hash_proves_and_verifies_as_its_own_program # 53 s +lambda-vm-prover tests::multiplicity_forgery_poc::dvrm_unanswered_forged_claim_is_rejected # 53 s +lambda-vm-prover lfm::p1_family_tests::a_p1_lfm_proof_verifies_and_the_families_do_not_cross # 52 s +lambda-vm-prover tests::prove_elfs_tests::test_prove_dma_memcpy_cases_rust_guest # 52 s +lambda-vm-prover tests::skip_empty_tables_tests::counts_the_proof_cannot_back_are_refused_before_verification # 51 s +lambda-vm-prover tests::prove_elfs_tests::test_verify_rejects_undercounted_table_count # 51 s +lambda-vm-prover tests::multiplicity_forgery_poc::load_negative_mu_sign_extension_forgery_is_rejected # 51 s +lambda-vm-prover tables::trace_builder::finish::stream_tests::bitwise_counted_beside_the_rest_builds_the_same_tables # 51 s +lambda-vm-prover tests::recursion_smoke_test::test_recursion_blob_decodes_and_verifies_on_host # 51 s +lambda-vm-prover tests::decode_tests::decode_commitment_wrong_value_rejects # 51 s +lambda-vm-prover continuation::tests::test_verify_continuation_with_supplied_roots # 51 s +lambda-vm-prover lfm::keccak_probe::padding_row_multiplicity_rejects # 50 s +lambda-vm-prover lfm::machine_tests::different_arena_values_change_the_public_output_not_the_program # 50 s +lambda-vm-prover continuation::tests::test_ecsm_across_epochs_verifies # 50 s +lambda-vm-prover tests::block_whir_tests::a_page_opened_at_another_tables_point_is_refused # 50 s +lambda-vm-prover lfm::keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard # 49 s +lambda-vm-prover tests::prove_elfs_tests::test_verify_rejects_zero_cpu_count # 49 s +lambda-vm-prover tests::multiplicity_forgery_poc::lt_negative_mu_twin_forgery_is_rejected # 48 s +lambda-vm-prover lfm::keccak_probe::tampered_input_byte_rejects # 48 s +lambda-vm-prover lfm::machine_tests::keccak_chain_proves_and_verifies # 47 s +lambda-vm-prover lfm::keccak_probe::tampered_output_byte_rejects # 47 s +lambda-vm-prover lfm::blake3_probe::falsification_control_the_untampered_proof_verifies # 47 s +lambda-vm-prover lfm::blake3_probe::the_hosted_chip_proves_and_verifies # 47 s +lambda-vm-prover lfm::machine_tests::chunked_sponge_proves_and_verifies # 47 s +lambda-vm-prover lfm::blake3_probe::a_tampered_message_byte_rejects # 47 s +lambda-vm-prover tests::prove_elfs_tests::test_prove_first_epoch_without_halt # 47 s +lambda-vm-prover tests::block_whir_tests::a_mixed_boundary_group_drops_one_class_only # 47 s +lambda-vm-prover tests::prove_elfs_tests::test_prove_second_epoch_from_snapshot # 47 s +lambda-vm-prover tests::decode_tests::decode_commitment_zero_bytes_rejects # 46 s +lambda-vm-prover continuation::tests::test_prove_error_mid_pipeline_returns_err # 46 s +lambda-vm-prover continuation::tests::test_continuation_with_several_cpu_tables_per_epoch_verifies # 46 s +lambda-vm-prover tests::halt_pc_forgery_poc::poc_control_continue_without_halt_pc_tamper_is_rejected # 45 s +lambda-vm-prover lfm::machine_tests::the_register_derivation_proves_and_verifies # 110 s in run 40588562061 +lambda-vm-prover lfm::machine_tests::l2g_binding_proves_and_verifies # 102 s in run 40588562061 +lambda-vm-prover lfm::machine_tests::l2g_binding_holds_on_the_real_bundle # 85 s in run 40588562061 +lambda-vm-prover lfm::machine_tests::the_register_derivation_matches_production # 71 s in run 40588562061 +lambda-vm-prover lfm::algebraic_commit::tests::the_host_grinding_digest_and_the_machine_agree # 66 s in run 40588562061 +lambda-vm-prover tests::page_tests::page_commitments_some_matches_default_path # 63 s in run 40588562061 +lambda-vm-prover tests::page_tests::page_commitments_empty_list_matches_none # 63 s in run 40588562061 +lambda-vm-prover lfm::blake3_socket_tests::the_blake3_socket_proves_and_verifies # 61 s in run 40588562061 +lambda-vm-prover tests::block_whir_tests::a_tampered_partition_is_refused # 58 s in run 40588562061 +lambda-vm-prover tests::pc_double_read_poc::poc_control_forgery_without_double_read_is_rejected # 57 s in run 40588562061 +lambda-vm-prover tests::block_whir_tests::a_block_with_a_moved_cpu_timestamp_is_refused # 56 s in run 40588562061 +lambda-vm-prover tests::prove_elfs_tests::test_prove_with_input_empty # 56 s in run 40588562061 +lambda-vm-prover tests::page_tests::page_commitments_wrong_value_rejects # 56 s in run 40588562061 +lambda-vm-prover tests::block_whir_tests::a_block_proved_without_the_provers_column_check_verifies # 55 s in run 40588562061 +lambda-vm-prover tests::static_commitments_tests::bitwise_static_matches_recompute_for_all_blowups # 52 s in run 40588562061 +lambda-vm-prover tests::prove_elfs_tests::test_verify_rejects_tampered_public_output # 48 s in run 40588562061 +lambda-vm-prover lfm::blake3_socket_tests::the_lane_range_check_is_load_bearing_on_its_own # 48 s in run 40588562061 +lambda-vm-prover lfm::blake3_probe::a_tampered_output_byte_rejects # 45 s in run 40588562061 +lambda-vm-prover tests::prove_elfs_tests::test_epoch_proof_commits_l2g # 44 s in run 40588562061 +lambda-vm-prover tests::block_whir_tests::the_dense_fixture_prepares_decode_and_two_pages # 43 s in run 40588562061 diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 56fa23e5b..4324f7f77 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -64,14 +64,15 @@ env: # from one real one. Only the headline screen was dropped. # # Cycle counts below are for the ELF THIS JOB BUILDS (see "Build ethrex guest ELF"), - # as of main @ 9ccdaf2 with clang 21. They move with guest optimisation (#861 gave the - # guest thin LTO) and ~2% with the clang major on PATH, so a count quoted against a + # as of the guest built with fat LTO and its own memcpy. They move with guest + # optimisation (#861 gave the guest thin LTO, fat LTO and the runtime memcpy took + # another 20 %) and ~2% with the clang major on PATH, so a count quoted against a # different ELF reads as a regression — always pin the ELF when repeating one. ELF: executor/program_artifacts/rust/ethrex.elf # The workload: a real Ethereum block. WHICH block lives in # the Makefile and nowhere else — nothing in this file names one, so a repoint # moves this job without editing it. At the current default and pinned ethrex rev that - # is 30,498,818 cycles, 10,659 keccak calls and 116 ecsm calls — several times the work + # is 23,935,913 cycles, 10,659 keccak calls and 116 ecsm calls — several times the work # of the synthetic block, at a very different keccak:ecrecover mix. That is the whole # point: a prover change can move the synthetic number and the real one in opposite # directions. All three figures move with the ethrex rev, so re-measure on a bump diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml index 89bff2b03..6f5c246b7 100644 --- a/.github/workflows/pr_main.yaml +++ b/.github/workflows/pr_main.yaml @@ -96,6 +96,12 @@ jobs: run: | make compile-programs-asm + # The key hashes every source a Rust guest build reads: the guest crates, the guest + # runtime (syscalls), the crypto crates guests depend on by path (ethrex-crypto for the + # ethrex guest; crypto and math for keccak_transcript_pattern), the target spec and the + # Makefile's recipe. An exact hit skips the compile step below, so a change confined to + # one of them that the key missed would run every test on a stale ELF. The same key + # appears in each job that restores these ELFs. - name: Cache compiled Rust ELF artifacts and build cache id: cache-rust-elfs uses: actions/cache@v4 @@ -103,7 +109,7 @@ jobs: path: | executor/program_artifacts/rust executor/shared_target - key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} + key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'crypto/ethrex-crypto/src/**', 'crypto/ethrex-crypto/Cargo.toml', 'crypto/crypto/src/**', 'crypto/crypto/Cargo.toml', 'crypto/math/src/**', 'crypto/math/Cargo.toml', 'Makefile') }} restore-keys: | rust-elf-artifacts- @@ -197,11 +203,57 @@ jobs: run: make test-ethrex-crypto # "Test" is a required check — keep this name to avoid branch protection changes. - # This gate job passes only when CLI, executor, disk-spill, and prover tests succeed. + # This gate job passes only when CLI, executor, and prover tests succeed. + host-kat: + # The device kernels' known-answer gates. `make test-*-host-kat` compiles the + # REAL `.cu` sources as host C++ through + # `crypto/math-cuda/tests/host_kat/cuda_host_shim.h` and runs them against + # external vectors (the official BLAKE3 vectors, miden-crypto's RPO vectors) + # and the committed oracle tables — g++ only, no CUDA, no cargo, seconds. It + # is the only per-PR check on the kernel arithmetic: the GPU parity suites + # (gpu-tests.yml) run on merge_group alone, and until this job existed the + # host-KAT targets lived in no workflow at all. Arithmetic only — nvcc + # acceptance and everything about execution stay with the GPU tests. + name: Host KAT (kernel pins) + runs-on: ubuntu-latest + if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]' + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: BLAKE3 host-KAT (7-round and 6-round arms) + run: make test-blake3-host-kat + + - name: RPX host-KAT (permutation, leaf sponge, parent, leaf and tree kernels) + run: make test-rpx-host-kat + + - name: Column-major LDE engine host-KAT (NTT passes, coset LDE) + run: make test-ntt-cm-host-kat + + - name: WHIR host-KAT (fused round fold, lean first rounds) + run: make test-whir-host-kat + + - name: GKR host-KAT (layer rounds over an input layer kept without its padding) + run: make test-gkr-host-kat + + - name: Commit-time answers host-KAT (columns' values at one point, one pass) + run: make test-eq-dot-host-kat + + - name: Round-slot host-KAT (a round's sums reduced by its last block) + run: make test-round-slot-host-kat + + # Second source for the TABLES the BLAKE3 host-KAT trusts: upstream + # BLAKE3's portable C with its round loop parameterised, against the + # committed 6-round vectors and the Python oracle. A ~1 s C compile plus a + # few seconds of Python (cc + python3, both on the runner); it too had no + # workflow before this job. + - name: BLAKE3 second-source check of the 6-round tables + run: make test-blake3-second-source + test: name: Test if: always() - needs: [test-executor, test-cli, test-prover, test-disk-spill, test-stark-cuda-lib] + needs: [test-executor, test-cli, test-prover, test-stark-cuda-lib, host-kat] runs-on: ubuntu-latest steps: - name: Check results @@ -209,14 +261,14 @@ jobs: executor="${{ needs.test-executor.result }}" cli="${{ needs.test-cli.result }}" prover="${{ needs.test-prover.result }}" - disk_spill="${{ needs.test-disk-spill.result }}" stark_cuda_lib="${{ needs.test-stark-cuda-lib.result }}" + host_kat="${{ needs.host-kat.result }}" echo "test-executor: $executor" echo "test-cli: $cli" echo "test-prover: $prover" - echo "test-disk-spill: $disk_spill" echo "test-stark-cuda-lib: $stark_cuda_lib" + echo "host-kat: $host_kat" # Allow "success" or "skipped" (skipped on merge queue pushes) if [[ "$executor" != "success" && "$executor" != "skipped" ]]; then @@ -228,72 +280,13 @@ jobs: if [[ "$prover" != "success" && "$prover" != "skipped" ]]; then exit 1 fi - if [[ "$disk_spill" != "success" && "$disk_spill" != "skipped" ]]; then + if [[ "$stark_cuda_lib" != "success" && "$stark_cuda_lib" != "skipped" ]]; then exit 1 fi - if [[ "$stark_cuda_lib" != "success" && "$stark_cuda_lib" != "skipped" ]]; then + if [[ "$host_kat" != "success" && "$host_kat" != "skipped" ]]; then exit 1 fi - test-disk-spill: - name: Disk-spill tests - runs-on: ubuntu-latest - if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]' - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - - name: Setup Rust Environment - uses: ./.github/actions/setup-rust - - - name: Cache cargo build artifacts - uses: Swatinem/rust-cache@v2 - with: - shared-key: "lambda-vm-disk-spill" - cache-all-crates: "true" - - - name: Cache compiled ASM ELF artifacts - id: cache-asm-elfs - uses: actions/cache@v4 - with: - path: executor/program_artifacts/asm - key: asm-elf-artifacts-${{ hashFiles('executor/programs/asm/**') }} - - - name: Install clang and lld - if: steps.cache-asm-elfs.outputs.cache-hit != 'true' - run: sudo apt-get update && sudo apt-get install -y clang lld - - - name: Compile ASM programs to ELF - if: steps.cache-asm-elfs.outputs.cache-hit != 'true' - run: | - make compile-programs-asm - - - name: Cache compiled Rust ELF artifacts and build cache - id: cache-rust-elfs - uses: actions/cache@v4 - with: - path: | - executor/program_artifacts/rust - executor/shared_target - key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} - restore-keys: | - rust-elf-artifacts- - - - name: Compile Rust programs to ELF - if: steps.cache-rust-elfs.outputs.cache-hit != 'true' - run: | - make compile-programs-rust - - - name: Run stark disk-spill tests - run: | - cargo test --release -p stark --features disk-spill disk_spill - - - name: Run prover disk-spill tests - env: - FORCE_DISK_SPILL: "1" - run: | - cargo test --release -p lambda-vm-prover --features disk-spill -- disk_spill count_table_lengths - test-stark-cuda-lib: name: Stark cuda-feature lib tests runs-on: ubuntu-latest @@ -349,6 +342,11 @@ jobs: -p lambda-vm-prover -p stark -p crypto -p ecsm \ --archive-file prover-tests.tar.zst + # The prover shards skip the tests listed in .config/ci-slow-tests.txt (they run in full on the project's own + # machines). Fails unless the filter skips exactly the listed tests and every test binary keeps a test. + - name: Check the slow-test list against the archive + run: python3 scripts/ci_slow_tests.py check prover-tests.tar.zst + - name: Upload test archive uses: actions/upload-artifact@v4 with: @@ -391,7 +389,7 @@ jobs: path: | executor/program_artifacts/rust executor/shared_target - key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} + key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'crypto/ethrex-crypto/src/**', 'crypto/ethrex-crypto/Cargo.toml', 'crypto/crypto/src/**', 'crypto/crypto/Cargo.toml', 'crypto/math/src/**', 'crypto/math/Cargo.toml', 'Makefile') }} restore-keys: | rust-elf-artifacts- @@ -440,13 +438,16 @@ jobs: # the concurrent path or VramGate's blocking path on a PR. The other # three shards keep the default-k coverage. An empty value on those # fails to parse and falls back to the default, so this is inert there. + # The tests listed in .config/ci-slow-tests.txt are skipped here: each took more than 45 s on a runner, and + # the project's own machines run them (scripts/ci_slow_tests.py; the build job checks the list). env: TABLE_PARALLELISM: ${{ matrix.partition == 1 && '6' || '' }} run: | cargo nextest run \ --archive-file prover-tests.tar.zst \ --partition hash:${{ matrix.partition }}/4 \ - --test-threads=1 + --test-threads=1 \ + -E "$(python3 scripts/ci_slow_tests.py expr)" test-prover-comprehensive: name: Prover comprehensive tests @@ -480,7 +481,7 @@ jobs: path: | executor/program_artifacts/rust executor/shared_target - key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} + key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'crypto/ethrex-crypto/src/**', 'crypto/ethrex-crypto/Cargo.toml', 'crypto/crypto/src/**', 'crypto/crypto/Cargo.toml', 'crypto/math/src/**', 'crypto/math/Cargo.toml', 'Makefile') }} restore-keys: | rust-elf-artifacts- @@ -564,7 +565,7 @@ jobs: path: | executor/program_artifacts/rust executor/shared_target - key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} + key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'crypto/ethrex-crypto/src/**', 'crypto/ethrex-crypto/Cargo.toml', 'crypto/crypto/src/**', 'crypto/crypto/Cargo.toml', 'crypto/math/src/**', 'crypto/math/Cargo.toml', 'Makefile') }} restore-keys: | rust-elf-artifacts- diff --git a/Cargo.lock b/Cargo.lock index 56cf0e6a4..b2dd1b50a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "atty" version = "0.2.14" @@ -126,6 +138,20 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -278,6 +304,7 @@ checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" name = "cli" version = "0.1.0" dependencies = [ + "blake3", "clap 4.5.53", "env_logger", "executor", @@ -286,6 +313,7 @@ dependencies = [ "stark", "tempfile", "tikv-jemalloc-ctl", + "tikv-jemalloc-sys", "tikv-jemallocator", ] @@ -302,10 +330,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "constant_time_eq" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "cpufeatures" @@ -316,6 +344,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "criterion" version = "0.4.0" @@ -418,17 +455,16 @@ name = "crypto" version = "0.1.0" dependencies = [ "bincode", + "blake3", "digest", "lambda-vm-syscalls", - "libc", "math", - "memmap2", + "math-cuda", "rayon", "rkyv", "serde", "sha2", "sha3", - "tempfile", ] [[package]] @@ -455,9 +491,9 @@ dependencies = [ [[package]] name = "cudarc" -version = "0.19.4" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f071cd6a7b5d51607df76aa2d426aaabc7a74bc6bdb885b8afa63a880572ad9b" +checksum = "7359bf1de037ddada72729c3acec9b040f3468400c6e9e740b01ba0cd2e89562" dependencies = [ "libloading 0.9.0", ] @@ -796,27 +832,33 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] name = "lambda-vm-prover" version = "0.1.0" dependencies = [ + "blake3", "criterion 0.5.1", "crypto", "digest", "ecsm", "env_logger", "executor", + "libc", "log", "math", "math-cuda", + "multilinear", "rayon", "rkyv", + "serde", + "serde_json", + "sha2", "stark", - "sysinfo", "tikv-jemalloc-ctl", + "tikv-jemalloc-sys", "tikv-jemallocator", "tiny-keccak", ] @@ -907,10 +949,12 @@ dependencies = [ name = "math-cuda" version = "0.1.0" dependencies = [ + "blake3", "crypto", "cudarc", "libloading 0.8.9", "math", + "multilinear", "rand 0.8.5", "rand_chacha 0.3.1", "rayon", @@ -925,12 +969,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] -name = "memmap2" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +name = "multilinear" +version = "0.1.0" dependencies = [ - "libc", + "crypto", + "math", + "math-cuda", + "rayon", + "rkyv", + "serde", + "thiserror", ] [[package]] @@ -953,15 +1001,6 @@ dependencies = [ "syn", ] -[[package]] -name = "ntapi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" -dependencies = [ - "winapi", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1484,7 +1523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1529,11 +1568,10 @@ dependencies = [ "digest", "env_logger", "itertools 0.11.0", - "libc", "log", "math", "math-cuda", - "memmap2", + "multilinear", "rand 0.8.5", "rand_chacha 0.3.1", "rayon", @@ -1541,7 +1579,7 @@ dependencies = [ "serde", "serde-wasm-bindgen", "serde_cbor", - "tempfile", + "sha3", "test-log", "thiserror", "wasm-bindgen", @@ -1571,19 +1609,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sysinfo" -version = "0.31.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355dbe4f8799b304b05e1b0f05fc59b2a18d36645cf169607da45bde2f69a1be" -dependencies = [ - "core-foundation-sys", - "libc", - "memchr", - "ntapi", - "windows", -] - [[package]] name = "tempfile" version = "3.23.0" @@ -1923,65 +1948,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" -dependencies = [ - "windows-core", - "windows-targets", -] - -[[package]] -name = "windows-core" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-targets", -] - -[[package]] -name = "windows-implement" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-result" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -1991,70 +1963,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/Cargo.toml b/Cargo.toml index 8f9bbe7d3..26071b181 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crypto/ecsm", "crypto/math", "crypto/math-cuda", + "crypto/multilinear", "bin/cli", ] # Riscv-only bare-metal crate, path-dependent from crypto/crypto (target-gated), diff --git a/Makefile b/Makefile index cf794e081..b94bfee4a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-s clean-recursion-elfs clean test test-asm \ test-rust test-ethrex test-ethrex-offline test-executor test-syscalls test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \ test-profile-recursion-block recursion-profile-block-input \ -test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-cuda-integration test-cuda-d1 test-cuda-fallback \ +test-fast test-prover test-prover-all test-prover-debug test-math-cuda test-blake3-host-kat test-rpx-host-kat test-p1w16-host-kat test-ntt-cm-host-kat test-whir-host-kat test-gkr-host-kat test-eq-dot-host-kat test-round-slot-host-kat test-fill-host-kat test-blake3-second-source test-cuda-integration test-cuda-d1 test-cuda-fallback \ test-prover-cuda test-prover-comprehensive-cuda \ bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \ update-ethrex-fixture-checksums check-ethrex-fixture-checksums ethrex-real-block-fixture \ @@ -54,9 +54,16 @@ BENCH_ARTIFACTS := $(addprefix $(BENCH_ARTIFACTS_DIR)/, $(addsuffix .elf, $(BENC # Recursion smoke-test guests, in bench_vs/lambda/ (shared with bench_vs/run.sh) # rather than executor/programs/. The recursion guest is the in-VM STARK verifier. +# +# `continuation-fixture` is the exception and the reason this comment grew: it has +# no benchmark arm and run.sh does not build it. It lives here anyway because +# `proof_fixture::read_inner_elf` reads the recursion artifact directory, and +# because it is a sibling of `fibonacci` in everything but its tail — which is +# precisely what must not be added to the benchmarked one, whose whole job is to +# run the same program as bench_vs/sp1/fibonacci. See its own main.rs. RECURSION_GUESTS_DIR=./bench_vs/lambda RECURSION_ARTIFACTS_DIR=./executor/program_artifacts/recursion -RECURSION_GUESTS := empty fibonacci +RECURSION_GUESTS := empty fibonacci continuation-fixture RECURSION_ARTIFACTS := $(addprefix $(RECURSION_ARTIFACTS_DIR)/, $(addsuffix .elf, $(RECURSION_GUESTS))) # The recursion verifier itself (bench_vs/lambda/recursion) requires picking @@ -234,9 +241,16 @@ $(RECURSION_ARTIFACTS_DIR)/%.elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_ # dir "recursion") or copy-paste (presets list is the single source of truth). # $(1) is the preset; the recipe uses $$ so `$$(call build_guest_elf,...)` # expands at recipe-run time (where $@ is defined). +# +# `absorb` rides along with every preset: it routes the verifier legs' coalesced +# leaf hashing through the chained-absorb ecall, which the prover's BLAKE3 table +# answers with its absorb mode. It is a feature rather than a default of the +# guest crate so the no-absorb leg stays buildable for A/B cycle measurement — +# drop it from RECURSION_GUEST_FEATURES to rebuild that leg. +RECURSION_GUEST_FEATURES := absorb define recursion_verifier_rule $(RECURSION_ARTIFACTS_DIR)/recursion-$(1).elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR) - $$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-$(1)-bench,--features $(1)) + $$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-$(1)-bench,--features "$(1) $$(RECURSION_GUEST_FEATURES)") endef $(foreach preset,$(RECURSION_VERIFIER_PRESETS),$(eval $(call recursion_verifier_rule,$(preset)))) @@ -244,7 +258,7 @@ $(foreach preset,$(RECURSION_VERIFIER_PRESETS),$(eval $(call recursion_verifier_ # feature -> recursion-cont--bench -> recursion-cont-.elf. define recursion_cont_verifier_rule $(RECURSION_ARTIFACTS_DIR)/recursion-cont-$(1).elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR) - $$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-cont-$(1)-bench,--features "continuation $(1)") + $$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-cont-$(1)-bench,--features "continuation $(1) $$(RECURSION_GUEST_FEATURES)") endef $(foreach preset,$(RECURSION_CONT_PRESETS),$(eval $(call recursion_cont_verifier_rule,$(preset)))) @@ -545,6 +559,25 @@ test-ethrex-crypto: test: compile-programs test-syscalls test-ethrex-crypto cargo test + # The hash counters compile to nothing unless the feature is on, so their + # own tests only execute here. See the `lint` target for why an instrument + # nobody runs is worth a line in the build. + cargo test -p crypto --features hash-metrics + # The transcript counters answer "which sponge ran". Their own integration + # binary, because the counters are process-global and a parallel neighbour's + # reset lands inside another test's measurement window — moving them out of + # the lib binary left four of five failing until they also took a lock. + cargo test -p crypto --features hash-metrics --test transcript_counters + # And the system test that reads them through a real prove: it is the one + # that says the PROVER picked the configuration's sponge, which the + # type-level test next to it cannot observe. + cargo test -p lambda-vm-prover --features hash-metrics --test whir_transcript_configuration + $(MAKE) test-rpx-host-kat + $(MAKE) test-ntt-cm-host-kat + $(MAKE) test-whir-host-kat + $(MAKE) test-gkr-host-kat + $(MAKE) test-eq-dot-host-kat + $(MAKE) test-round-slot-host-kat # === Quick test shortcuts === @@ -569,11 +602,6 @@ test-prover-all: compile-recursion-elfs test-prover-debug: compile-recursion-elfs cargo test -p lambda-vm-prover --features debug-checks -- --nocapture -# Disk-spill tests (stark + prover). FORCE_DISK_SPILL is required by the prover tests. -test-disk-spill: - cargo test --release -p stark --features disk-spill disk_spill - FORCE_DISK_SPILL=1 cargo test --release -p lambda-vm-prover --features disk-spill -- disk_spill count_table_lengths - # Per-target wall clock for the GPU prover targets below. A panic on a device-only # cliff assert can leave the prover hung rather than aborting — the panicking thread # unwinds while its siblings stay parked in CUDA driver waits, and the process never @@ -587,6 +615,188 @@ GPU_TEST_TIMEOUT := timeout -k 30 2700 test-math-cuda: $(GPU_TEST_TIMEOUT) cargo test -p math-cuda --release +# Known-answer tests for the BLAKE3 device kernels, run on the HOST. No GPU, no +# nvcc, no cargo — a couple of seconds. +# +# This exists because `test-math-cuda` above, which is the authority on these +# kernels, runs only where a GPU does, and the per-PR CI runners have none (GPU +# jobs are merge_group-only). Without this the kernels have no per-PR gate: an +# edit to blake3.cu that broke the hash would reach the merge queue before +# anything caught it. `crypto/math-cuda/tests/host_kat/` compiles the real kernel +# source as host C++ through a shim and runs the official BLAKE3 vectors, the +# canonical 6-round table, the official multi-block vectors against the +# `Blake3Chain` construction, and every leaf kernel's byte stream through it. +# +# BOTH ROUND COUNTS are built and run. The 6-round arm is the one the campaign +# ships and the one no other CI job compiles (risk R10), and the round count is a +# compile-time knob, so a single-arm run would leave the shipping configuration +# ungated. The two arms differ only in `-DBLAKE3_ROUNDS`, exactly as build.rs +# drives the cubin from the `blake3-6round` feature. +# +# It checks arithmetic ONLY. Whether nvcc accepts the file, and everything about +# execution rather than arithmetic — grid indexing, the Merkle tail's barriers, +# device alignment, register pressure — stays with `test-math-cuda`. Necessary, +# never sufficient. +HOST_KAT_DIR := crypto/math-cuda/tests/host_kat +HOST_KAT_CXXFLAGS := -std=c++17 -O2 -Wall -Wno-unknown-pragmas \ + -I$(HOST_KAT_DIR) -Icrypto/math-cuda/kernels +test-blake3-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/blake3_host_kat $(HOST_KAT_DIR)/blake3_host_kat.cpp + ./target/host_kat/blake3_host_kat + @echo + @echo "=== rebuilding for the 6-round arm (BLAKE3_ROUNDS=6) ===" + $(CXX) $(HOST_KAT_CXXFLAGS) -DBLAKE3_ROUNDS=6 \ + -o target/host_kat/blake3_host_kat_6r $(HOST_KAT_DIR)/blake3_host_kat.cpp + ./target/host_kat/blake3_host_kat_6r + +# Known-answer tests for the RPX256 device kernel source, run on the HOST through +# the Track G shim (no GPU, no nvcc): the permutation, the rate-8 overwrite-duplex +# leaf and the parent compress against vectors printed from the Rust oracle +# (`prover/tests/rpx_host_kat_vectors.rs`), plus miden-crypto's 19 RPO vectors +# through the shared FB round. +# +# ★ It also covers the two WHIR COSET kernels, which exist nowhere else and +# which GPU CI would not see until merge_group. That is why this target is a +# per-PR gate and not a GPU one. +# +# The second binary replays the kernels whose lanes cooperate (the half-warp +# permutation and Merkle kernels, the work-queue grind) with every lane a real +# thread (`cuda_host_simt_shim.h`), against the shipped kernels. +test-rpx-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/rpx_host_kat $(HOST_KAT_DIR)/rpx_host_kat.cpp + ./target/host_kat/rpx_host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) -pthread \ + -o target/host_kat/rpx_simt_host_kat $(HOST_KAT_DIR)/rpx_simt_host_kat.cpp + ./target/host_kat/rpx_simt_host_kat + +# Known-answer tests for the Poseidon1 width-16 MEASUREMENT kernels +# (`p1w16.cu`, D-HASH stage 1), run on the HOST through the same shim: the +# permutation, the rate-12 coset leaves (base and ext3), the 4-ary node and the +# grind, at both multiply variants, against the vectors the host reference and +# Plonky3 agree on. Nothing on a proving path uses these kernels. +test-p1w16-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/p1w16_host_kat $(HOST_KAT_DIR)/p1w16_host_kat.cpp + ./target/host_kat/p1w16_host_kat + +# Known-answer tests for the column-major LDE engine's passes (`ntt_cm.cu`), run +# on the HOST through the same shim: every pass shape against the +# textbook radix-2 levels, whole transforms against the naive DFT, and the LDE +# the driver runs (DIF iNTT, bit-reversed weights, DIT with the fused coset +# spread) against direct interpolation and evaluation on the coset. Arithmetic +# and index math only; execution stays with `tests/ntt_cm_parity.rs`. +test-ntt-cm-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/ntt_cm_host_kat $(HOST_KAT_DIR)/ntt_cm_host_kat.cpp + ./target/host_kat/ntt_cm_host_kat + +# Known-answer tests for the WHIR round kernels (`whir_fold.cu`), run on the HOST +# through the same shim: the fused fold against the level-by-level kernels it +# replaces (raw-identical, base and extension, k = 1..6), and the lean first +# rounds — factors in share form — against the materialised opening (six +# rounds and the bound tables). Arithmetic only; execution stays with +# `test-math-cuda`. +test-whir-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/whir_host_kat $(HOST_KAT_DIR)/whir_host_kat.cpp + ./target/host_kat/whir_host_kat + +# Known-answer tests for the GKR layer rounds over an input layer kept without +# its padding (`sumcheck.cu`, `LAMBDA_VM_ARGUE_INPUT_REAL`), run on the HOST +# through the SIMT shim, every thread of a block a real thread so the block +# reductions run as written: the kept layer's rounds and finish against today's +# kernels over the whole layer (raw-identical), today's kernels against a copy +# of their source from before the change, and both against a written-out +# reference. Arithmetic and indexing only; execution stays with +# `test-math-cuda`. +test-gkr-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) -pthread \ + -o target/host_kat/gkr_host_kat $(HOST_KAT_DIR)/gkr_host_kat.cpp + ./target/host_kat/gkr_host_kat + +# Known-answer tests for the commit-time out-of-domain answers' column values +# (`sumcheck.cu`'s `eq_dot_base_ext3_many` and the `sum_partials_ext3` over its +# partials), run on the HOST through the SIMT shim, against a written-out +# reference: every column's value at the point, over shapes with blocks fewer +# than, as many as and more than a column's hi rows. Arithmetic and indexing +# only; execution stays with `multilinear/tests/whir_commit_ood_device.rs`. +test-eq-dot-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) -pthread \ + -o target/host_kat/eq_dot_host_kat $(HOST_KAT_DIR)/eq_dot_host_kat.cpp + ./target/host_kat/eq_dot_host_kat + +# Known-answer tests for the round kernels whose sums leave through a slot in +# pinned host memory (`sumcheck.cu`'s `slot_sums` and `_slot` kernels, +# `LAMBDA_VM_ARGUE_ROUND_SLOT`), on the HOST through the SIMT shim: the last +# block's sums against today's second launch (`sum_partials_ext3`), raw, with +# the ticket's reset and the order of the sums, the fence and `seq` checked. +# Arithmetic and indexing only; the card's memory model stays with +# `test-math-cuda`. +test-round-slot-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) -pthread \ + -o target/host_kat/round_slot_host_kat $(HOST_KAT_DIR)/round_slot_host_kat.cpp + ./target/host_kat/round_slot_host_kat + +# Known-answer tests for the table fills from records (`fill.cu`), run on the +# HOST through the same shim: MEMW_R, CPU (every op class, the decode search +# across runs, a missing pc, the padding rows) and KECCAK_RND (a table cut at +# both ends, every round, zero rows) against the prover's host fill +# (`fill_kat_vectors.h`, written and pinned by the prover's +# `tables::card_fill_inputs`). Arithmetic and indexing only; execution stays +# with the card test (`tables::card_fill_tests`). +test-fill-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/fill_host_kat $(HOST_KAT_DIR)/fill_host_kat.cpp + ./target/host_kat/fill_host_kat + +# SECOND-SOURCE validation of the 6-round vectors the KAT above trusts. +# +# `test-blake3-host-kat` checks the KERNEL against the committed tables. This +# checks the TABLES, against upstream BLAKE3's own portable C with its round loop +# parameterised (`tooling/blake3-second-source/reference-impl/`, a 2 KB reviewable diff in +# PARAMETERISATION.diff). That reference is a different language and author and — +# the part that matters — a different message-schedule CONSTRUCTION: it indexes a +# precomputed MSG_SCHEDULE table where our Rust and CUDA compose one permutation +# between rounds. A bug in the iterative composition is exactly what a single +# source cannot catch, and check [C] compares the two constructions directly. +# +# It exists because 6-round BLAKE3 is computed by nothing else in the world +# (assumption A6R), so the 6-round column of every table here rests on oracle +# agreement rather than on a published vector. +# +# ⚠ Checks [D] and [E] were SILENTLY DEAD from P-a Stage 1 — which moved +# CANONICAL_VECTORS out of `prover/src/lfm/blake3.rs` into `crypto` — until +# 2026-08-15, because nothing ever ran this: it had no target. That is why it has +# one now. A ~1 second C compile plus a few seconds of Python; no cargo, no GPU. +test-blake3-second-source: + tooling/blake3-second-source/reference-impl/build.sh + python3 tooling/blake3-second-source/reference-impl/check.py + +# z3/SMT gate for the BLAKE3 chained-absorb mode's row-local constraints: the +# compression equivalence under the absorb framing, the flags schedule, the mode +# gating, the countdown/END logic, and the byte-width assumptions. +# +# Read `formal_verification/blake3_absorb/README.md` before extending it. The 6 +# rounds are verified ONE AT A TIME because the composed query does not close — +# commit 89aeeb8c measured 145 minutes of `unknown` — and the negative controls +# are what make a green board mean anything. +# +# z3's Python bindings are the only dependency. No cargo, no GPU, ~25 seconds. +test-blake3-absorb-fv: + python3 formal_verification/blake3_absorb/test_ref.py + python3 formal_verification/blake3_absorb/z3_absorb_verify.py + # End-to-end cuda dispatch coverage (requires NVIDIA GPU + nvcc). # Asserts the R1-R4 GPU dispatch counters fired on a real prove. # --test-threads=1: these tests reset and assert on process-global GPU call @@ -675,7 +885,33 @@ check: clippy: cargo clippy --workspace --all-targets -- -D warnings -A clippy::op_ref cargo clippy --workspace --all-targets --no-default-features --features lambda-vm-prover/debug-checks -- -D warnings -A clippy::op_ref - cargo clippy --workspace --all-targets --features lambda-vm-prover/disk-spill -- -D warnings -A clippy::op_ref + # BLAKE3 at SEVEN rounds — the standard, externally anchored arm. + # + # This pass used to be the six-round one. Six is the default now (P-a Stage 6 + # put the commitment aliases on BLAKE3, and every blessed constant is six), so + # the pass that needs writing down is the other one: without this, nothing in + # the matrix compiles `hash::blake3` at seven and the arm the KATs anchor + # against could rot unnoticed. + # + # Scoped to `crypto`, and it has to be. A workspace-level + # `--no-default-features` does NOT turn six off: `lambda-vm-prover` depends on + # `crypto` with default features, and feature unification takes the union, so + # the flag applies to the packages and the dependency edge puts it back. Only + # building `crypto` alone leaves the edge out of the graph. + # + # The lockstep between `crypto`'s knob and `math-cuda`'s (host primitive vs + # compiled cubin) still holds and is now expressed as both defaulting to six, + # which `math_cuda::blake3::device_rounds` makes assertable rather than + # discoverable as a wrong root. + cargo clippy -p crypto --all-targets --no-default-features --features std,asm -- -D warnings -A clippy::op_ref + # The chained-absorb guest arm's feature graph. Nothing else resolves it: it is + # not a default and no dependency names it, so a typo in the forwarding chain + # would go unnoticed. ⚠ This does NOT compile the guest absorb arm — that arm + # is `cfg(all(target_arch = "riscv64", feature = "blake3-absorb"))` and this is + # a host pass, so what compiles is its `cfg(not(...))` stub. The riscv64 arm + # has no CI coverage at all (RESUME-PA-STAGE4 §4.2); closing that needs a + # guest-target build, not a clippy feature. + cargo clippy --workspace --all-targets --features lambda-vm-prover/blake3-absorb -- -D warnings -A clippy::op_ref fmt: cargo fmt --all @@ -685,11 +921,55 @@ lint: cargo fmt --check --all cargo clippy --workspace --all-targets -- -D warnings -A clippy::op_ref cargo clippy --workspace --all-targets --no-default-features --features lambda-vm-prover/debug-checks -- -D warnings -A clippy::op_ref - cargo clippy --workspace --all-targets --features lambda-vm-prover/disk-spill -- -D warnings -A clippy::op_ref + # BLAKE3 at SEVEN rounds — the standard, externally anchored arm. + # + # This pass used to be the six-round one. Six is the default now (P-a Stage 6 + # put the commitment aliases on BLAKE3, and every blessed constant is six), so + # the pass that needs writing down is the other one: without this, nothing in + # the matrix compiles `hash::blake3` at seven and the arm the KATs anchor + # against could rot unnoticed. + # + # Scoped to `crypto`, and it has to be. A workspace-level + # `--no-default-features` does NOT turn six off: `lambda-vm-prover` depends on + # `crypto` with default features, and feature unification takes the union, so + # the flag applies to the packages and the dependency edge puts it back. Only + # building `crypto` alone leaves the edge out of the graph. + # + # The lockstep between `crypto`'s knob and `math-cuda`'s (host primitive vs + # compiled cubin) still holds and is now expressed as both defaulting to six, + # which `math_cuda::blake3::device_rounds` makes assertable rather than + # discoverable as a wrong root. + cargo clippy -p crypto --all-targets --no-default-features --features std,asm -- -D warnings -A clippy::op_ref + # The chained-absorb guest arm's feature graph. Nothing else resolves it: it is + # not a default and no dependency names it, so a typo in the forwarding chain + # would go unnoticed. ⚠ This does NOT compile the guest absorb arm — that arm + # is `cfg(all(target_arch = "riscv64", feature = "blake3-absorb"))` and this is + # a host pass, so what compiles is its `cfg(not(...))` stub. The riscv64 arm + # has no CI coverage at all (RESUME-PA-STAGE4 §4.2); closing that needs a + # guest-target build, not a clippy feature. + cargo clippy --workspace --all-targets --features lambda-vm-prover/blake3-absorb -- -D warnings -A clippy::op_ref # The cuda feature gates whole modules + cuda-only integration tests. build.rs emits empty # cubin stubs when nvcc is absent, so this checks on a GPU-less host (CI lint runner, dev laptop) # too — no GPU required. Catches cuda-gated breakage that the non-cuda passes above miss. cargo clippy --workspace --all-targets --features lambda-vm-prover/cuda -- -D warnings -A clippy::op_ref + # `hash-metrics` is host-only and off by default, so no pass above compiles it. + # Without this line the feature can rot untouched — which is how its Merkle + # counters stayed keccak-only after a second hash arrived, reporting ZERO for + # the arm whose whole purpose was to change the hashing. Lints, does not run: + # its tests are in the `test` target. + cargo clippy -p crypto --all-targets --features hash-metrics -- -D warnings -A clippy::op_ref + # The prover's own `hash-metrics` passthrough gates the per-arm transcript + # line and the system test that reads it; without this line neither compiles + # in any pass, which is how an instrument rots. + cargo clippy -p lambda-vm-prover --all-targets --features hash-metrics -- -D warnings -A clippy::op_ref + # ⛔ BOTH AT ONCE, because neither pass above is both. The device pin — + # `check_device_pins` and `transcript_pin::commits`, which model the GPU + # commit counter against the bundle's own chains — is `cfg(all(cuda, + # hash-metrics))`: the cuda pass has no `hash-metrics` and the two + # `hash-metrics` passes have no cuda, so until this line the whole model was + # compiled by nothing and a box run was its first compiler. No GPU needed; + # cuda clippy builds against the cubin stubs like the pass above. + cargo clippy -p lambda-vm-prover --all-targets --features cuda,hash-metrics -- -D warnings -A clippy::op_ref flamegraph-prover: cd crypto/stark && samply record cargo bench --bench profile_prover --features parallel diff --git a/bench_vs/lambda/continuation-fixture/.cargo/config.toml b/bench_vs/lambda/continuation-fixture/.cargo/config.toml new file mode 100644 index 000000000..be730c3ec --- /dev/null +++ b/bench_vs/lambda/continuation-fixture/.cargo/config.toml @@ -0,0 +1,6 @@ +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "-C", "link-arg=-e", + "-C", "link-arg=main", + "-C", "passes=lower-atomic" +] diff --git a/bench_vs/lambda/continuation-fixture/Cargo.lock b/bench_vs/lambda/continuation-fixture/Cargo.lock new file mode 100644 index 000000000..127f25e97 --- /dev/null +++ b/bench_vs/lambda/continuation-fixture/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "continuation-fixture-bench" +version = "0.1.0" diff --git a/bench_vs/lambda/continuation-fixture/Cargo.toml b/bench_vs/lambda/continuation-fixture/Cargo.toml new file mode 100644 index 000000000..127530df5 --- /dev/null +++ b/bench_vs/lambda/continuation-fixture/Cargo.toml @@ -0,0 +1,8 @@ +[workspace] + +[package] +name = "continuation-fixture-bench" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/bench_vs/lambda/continuation-fixture/src/main.rs b/bench_vs/lambda/continuation-fixture/src/main.rs new file mode 100644 index 000000000..e4e74fdec --- /dev/null +++ b/bench_vs/lambda/continuation-fixture/src/main.rs @@ -0,0 +1,149 @@ +//! ★ The continuation fixture's inner guest: it COMMITS, and then keeps going. +//! +//! `bench_vs/lambda/fibonacci` cannot be this guest, for two separate reasons. +//! +//! The first is what the fixture needs. `epoch_tests::EpochFront::build` asserts +//! `!is_final` — it deliberately harvests an INTERMEDIATE epoch — and a guest +//! that commits immediately before `halt()` puts its public output in the FINAL +//! one, always. So `e.public_output` is empty for every epoch that harness can +//! reach, and `the_closure_rejects_a_moved_index_or_output`'s second half (move +//! a half of the public output, watch the COMMIT bus refuse to close) has no +//! subject. Its anti-vacuity guard says so rather than passing quietly. This +//! guest commits and then does bounded work, so the committing epoch is an +//! intermediate one by CONSTRUCTION rather than by arithmetic luck. +//! +//! The second is that `fibonacci` is not ours to change: `bench_vs/run.sh` builds +//! it as the Lambda VM arm of a cross-prover benchmark against `bench_vs/sp1/ +//! fibonacci`, and a tail loop that one arm runs and the other does not stops the +//! two measuring the same program. +//! +//! ⚠ **The cycle count is a property of the compiled ELF**, as +//! `proof_fixture::FIXTURE_EPOCH_LOG2` has recorded since the toolchain last +//! moved it. What keeps that from biting here is MARGIN, not luck — see +//! [`TAIL_STEPS`]. Re-measure rather than guess: run this ELF to completion under +//! `Executor::resume_with_limit` and count the logs, one per cycle. + +#![no_std] +#![no_main] + +use core::arch::asm; +use core::panic::PanicInfo; + +const PRIVATE_INPUT_START: usize = 0xFF000000; +const SYSCALL_COMMIT: u64 = 64; +const SYSCALL_HALT: u64 = 93; + +/// ★ Iterations of post-commit work, and the number the fixture's whole shape +/// rests on. +/// +/// Measured on the ELF this source builds, by counting logs: the commit lands at +/// cycle **13** and the run is **48** cycles. One step costs TWO cycles — 10 +/// steps gave 36 and 16 gave 48 — so this is a dial, not a guess. +/// `FIXTURE_EPOCH_LOG2 = 5` puts the epoch boundary at 32, which leaves +/// +/// - **19 cycles** between the commit and the boundary — epoch 0 CARRIES the +/// public output, which is what the whole guest is for; +/// - **16 cycles** between the boundary and the halt — epoch 0 is INTERMEDIATE, +/// so `EpochFront::build`'s `!is_final` holds; +/// - **16 cycles** between the halt and 64 — there are exactly TWO epochs, which +/// is what `continuation_fixture_generates_two_epochs` is the canary for. +/// +/// Three margins of sixteen cycles or more, against the two-instruction codegen +/// drift that crossed a boundary the last time this was a four-cycle question. A +/// tail sized to land just past the boundary would have reproduced that +/// fragility rather than fixed it, which is why this is not the smallest number +/// that works — 10 steps also "works", with four cycles to spare, and that is +/// the state this guest exists to get out of. Raising it eats the third margin +/// and runs out at 24 steps (total 64); lowering it eats the second. +/// +/// ★ `proof_fixture::tests::the_fixture_guest_commits_in_an_intermediate_epoch` +/// asserts all three, so none of this has to be believed on the strength of a +/// comment. +const TAIL_STEPS: u64 = 16; + +#[panic_handler] +fn panic(_info: &PanicInfo) -> ! { + loop {} +} + +fn read_n() -> u64 { + // Layout matches `syscalls::get_private_input`: 4-byte LE length prefix at + // PRIVATE_INPUT_START, payload at +4. We only need the first 8 bytes (u64). + let mut n_bytes = [0u8; 8]; + + debug_assert!( + unsafe { core::ptr::read_volatile(PRIVATE_INPUT_START as *const u32) } >= 8, + "private input too short to contain a u64" + ); + + let input_data = (PRIVATE_INPUT_START + 4) as *const u8; + n_bytes.copy_from_slice(unsafe { core::slice::from_raw_parts(input_data, 8) }); + + u64::from_le_bytes(n_bytes) +} + +fn commit(bytes: &[u8]) { + unsafe { + asm!( + "ecall", + in("a0") 1u64, + in("a1") bytes.as_ptr(), + in("a2") bytes.len(), + in("a7") SYSCALL_COMMIT, + ); + } +} + +fn halt() -> ! { + unsafe { + asm!( + "ecall", + in("a0") 0u64, + in("a7") SYSCALL_HALT, + options(noreturn), + ); + } +} + +#[unsafe(no_mangle)] +pub fn main() -> ! { + let n = read_n(); + let mut a: u64 = 0; + let mut b: u64 = 1; + for _ in 0..n { + let c = a.wrapping_add(b); + a = b; + b = c; + } + + commit(&b.to_le_bytes()); + + // ★ The tail. Two constraints shape it, and neither is obvious. + // + // A VOLATILE store is what makes it survive: the value is never read back, + // so anything weaker is dead code the optimiser is entitled to delete, and + // deleting it would silently restore the very shape this guest exists to + // avoid. The sink is a stack local rather than a `static mut` because the + // stack is mapped under every image builder the harnesses use. + // + // ⚠ And the arithmetic is MORE FIBONACCI rather than anything with a + // constant in it, because `recursion::precomputed_commitments` lists every + // ELF page whose `init_values.is_some()` — a pure function of the BINARY, + // not of the run. A 64-bit LCG multiplier is not materialisable inline on + // rv64, so LLVM spills it to a constant pool, the guest grows a `.rodata` + // section fibonacci does not have, and the fixture starts carrying a page + // commitment. `program_id_matches_production_on_the_real_fixture` asserts + // it carries none. Wrapping adds of values already in registers keep the + // section table identical to `fibonacci`'s: `.text` and nothing else. + let mut sink: u64 = 0; + let mut step: u64 = 0; + while step < TAIL_STEPS { + let c = a.wrapping_add(b); + a = b; + b = c; + unsafe { core::ptr::write_volatile(&raw mut sink, b) }; + step += 1; + } + + halt() +} diff --git a/bench_vs/lambda/recursion/Cargo.lock b/bench_vs/lambda/recursion/Cargo.lock index 7af687454..d8500f17a 100644 --- a/bench_vs/lambda/recursion/Cargo.lock +++ b/bench_vs/lambda/recursion/Cargo.lock @@ -64,12 +64,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -359,11 +353,12 @@ dependencies = [ "digest", "ecsm", "executor", + "libc", "log", "math", + "multilinear", "rkyv", "stark", - "sysinfo", ] [[package]] @@ -415,6 +410,17 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "multilinear" +version = "0.1.0" +dependencies = [ + "crypto", + "math", + "rkyv", + "serde", + "thiserror", +] + [[package]] name = "munge" version = "0.4.7" @@ -435,15 +441,6 @@ dependencies = [ "syn", ] -[[package]] -name = "ntapi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" -dependencies = [ - "winapi", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -791,6 +788,7 @@ dependencies = [ "itertools", "log", "math", + "multilinear", "rkyv", "serde", "serde_cbor", @@ -814,19 +812,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sysinfo" -version = "0.31.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355dbe4f8799b304b05e1b0f05fc59b2a18d36645cf169607da45bde2f69a1be" -dependencies = [ - "core-foundation-sys", - "libc", - "memchr", - "ntapi", - "windows", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -940,145 +925,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" -dependencies = [ - "windows-core", - "windows-targets", -] - -[[package]] -name = "windows-core" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-targets", -] - -[[package]] -name = "windows-implement" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-result" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/bench_vs/lambda/recursion/Cargo.toml b/bench_vs/lambda/recursion/Cargo.toml index cc4d00a70..009bf98fa 100644 --- a/bench_vs/lambda/recursion/Cargo.toml +++ b/bench_vs/lambda/recursion/Cargo.toml @@ -21,6 +21,19 @@ blowup8 = [] # memory-bounded inner prove) instead of a monolithic VmProof. Selects the # `recursion-cont--bench` bins below. continuation = [] +# Also orthogonal: let the guest's BLAKE3 hashing reach the chained-absorb +# ecall, so a leaf costs one ecall per run of blocks instead of one per block. +# +# The prover's BLAKE3 table answers this ecall with its absorb mode, so a guest +# built with it is provable as well as cheaper. Every `recursion-*.elf` the +# Makefile builds turns it on; it stays a feature rather than a default so the +# no-absorb leg stays buildable for A/B cycle measurement. +# +# ⚠ Compile-time ONLY, and it must stay that way. Which hash a guest computes, +# and how it reaches the accelerator, are part of what a proof means — a guest +# that could pick either from its private input would let the input decide what +# was proved. Same reason the ProofOptions presets are hardcoded. +absorb = ["lambda-vm-prover/blake3-absorb"] # One distinctly named binary per preset (selected by its feature) so a parallel # `make -j` builds them to different filenames — structurally race-free, no cp diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml index b9140e34c..c8814f9b6 100644 --- a/bin/cli/Cargo.toml +++ b/bin/cli/Cargo.toml @@ -13,12 +13,22 @@ clap = { version = "4.3.10", features = ["derive"] } rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } tempfile = "3" tikv-jemallocator = "0.6" -tikv-jemalloc-ctl = { version = "0.6", features = ["stats"], optional = true } +# The allocator's statistics and `arena..purge`, installed into the +# prover as its `alloc_purge` hooks, and the never-purge posture this binary +# compiles in, read back (`prove-block`'s BLOCK POSTURE SET line and its test). +tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] } +tikv-jemalloc-sys = { version = "0.6", default-features = false } +# `prove-block --digest`: the proof bytes' blake3, the digest the box gates +# compare (the prover's tests print the same). +blake3 = { version = "1.8.5", default-features = false, features = ["std", "pure"] } env_logger = "0.11" [features] -jemalloc-stats = ["dep:tikv-jemalloc-ctl"] -disk-spill = ["prover/disk-spill"] +# The GPU prover: `prove-block` needs it (the W3 block tree proves on the card). +cuda = ["prover/cuda"] +jemalloc-stats = [] instruments = ["prover/instruments", "stark/instruments"] +# Emit a segment-shape profile per proof (see LAMBDA_VM_SHAPE_PROFILE). +shape-profile = ["prover/shape-profile"] # GPU profiling build (Nsight): CUDA prover + instruments spans + NVTX ranges. nvtx = ["prover/nvtx", "instruments"] diff --git a/bin/cli/README.md b/bin/cli/README.md index 733cbd6d7..b8f472ae8 100644 --- a/bin/cli/README.md +++ b/bin/cli/README.md @@ -79,6 +79,43 @@ cargo run -p cli --release -- verify [flags] Returns exit code `0` on successful verification, `1` on failure. `--blowup` must match the value used during proving. +### Prove a block + +Prove a whole block as a recursion tree: the block's WHIR base proof, the leaves that verify its groups, the nodes +and the top. A consumer verifies the one top proof against the ELF with `verify-block`. Needs a CUDA build +(`cargo build --release -p cli --features cuda`). + +```sh +cli prove-block --input -o block.proof [--time] [--digest] +``` + +| Flag | Description | +|---|---| +| `--input ` | The block's private input (the guest reads it with `get_private_input()`). | +| `-o, --output ` | Output path for the block proof. Required. | +| `--time` | Print the whole run's wall, base to top node. | +| `--digest` | Print the blake3 digest of the top proof's bytes. | + +stdout carries the public output and the requested numbers; the run's progress lines go to stderr. + +The command runs the production posture: every knob in `POSTURE` (`prover/src/lfm/whir_block_tree.rs`) that the +environment leaves unset is set to its posture value: the table parallelism, the row cap, the WHIR retention, the +WHIR hash (RPX), the tree cache cap, the executor schedule and the grind search. The `BLOCK POSTURE` lines on stderr +say what was set and what came from the environment. The WHIR hash is a format knob, so `verify-block` sets it too: +prove and verify under the same environment. The binary compiles in jemalloc's never-purge posture. The tree's +leaf count, fan-in and argue format are the block verifier's; `W3_LEAVES`, `W3_FAN_IN` and `BLOCK_WHIR_ARGUE` +(harness arms) are refused. + +### Verify a block + +```sh +cli verify-block [--time] +``` + +Derives the block's recursion plan and top program from the ELF and the proof's claimed statement under the block +presets, verifies the top proof against that program, and checks it claims the ELF's id and the output. Prints the +public output. Returns exit code `0` on success, `1` on failure. + ### Count Elements Build traces and print main-trace and aux-trace field element counts **without** running the proof step. Useful for sizing. diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs index df1b1e033..f95cbc719 100644 --- a/bin/cli/src/main.rs +++ b/bin/cli/src/main.rs @@ -51,7 +51,9 @@ static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; // "_rjem_malloc_conf")]`, its `src/lib.rs`). None of that is compiler-checked. // `prover/tests/jemalloc_conf.rs` reads both options back out of jemalloc, but // it carries its own copy of this block and reads its own process — it pins the -// pattern, not this export. Deleting the lines below turns nothing red. +// pattern. This binary's own unit tests read this export back +// (`the_binary_runs_its_never_purge_posture`): deleting the lines below turns +// that test red. const NEVER_PURGE: &[u8] = b"dirty_decay_ms:-1,muzzy_decay_ms:-1\0"; #[allow(non_upper_case_globals)] @@ -59,6 +61,66 @@ const NEVER_PURGE: &[u8] = b"dirty_decay_ms:-1,muzzy_decay_ms:-1\0"; pub static malloc_conf: Option<&'static core::ffi::c_char> = Some(unsafe { &*(NEVER_PURGE.as_ptr() as *const core::ffi::c_char) }); +/// This binary's jemalloc, as the prover asks it (`prover::alloc_purge`): its +/// statistics, and a purge of every arena's dirty pages. Installed at the top +/// of `main`, so the block's purge points (`LAMBDA_VM_ALLOC_PURGE`, `auto` by +/// default) run here as they do in the prover's own tests. It also reads back +/// the never-purge posture it runs for the `BLOCK POSTURE SET` line and its +/// test. +mod allocator { + use prover::alloc_purge::{AllocStats, AllocatorHooks}; + + pub const HOOKS: AllocatorHooks = AllocatorHooks { stats, purge_all }; + + /// jemalloc's statistics, the epoch turned first (they are cached until + /// it turns). + fn stats() -> Option { + use tikv_jemalloc_ctl::{epoch, stats}; + epoch::advance().ok()?; + Some(AllocStats { + allocated: stats::allocated::read().ok()?, + active: stats::active::read().ok()?, + resident: stats::resident::read().ok()?, + mapped: stats::mapped::read().ok()?, + retained: stats::retained::read().ok()?, + }) + } + + /// `arena..purge` (4096, jemalloc's "every arena" + /// index): a control that neither reads nor writes, so every pointer is + /// null. + fn purge_all() -> bool { + // SAFETY: the name is NUL-terminated, and a control that takes no + // value is called with null old and new pointers and zero lengths, as + // jemalloc's `NEITHER_READ_NOR_WRITE` requires. + let rc = unsafe { + tikv_jemalloc_sys::mallctl( + c"arena.4096.purge".as_ptr(), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + 0, + ) + }; + if rc != 0 { + eprintln!("ALLOC PURGE: arena.4096.purge returned {rc}"); + } + rc == 0 + } + + /// The running `opt.dirty_decay_ms` and `opt.muzzy_decay_ms`: `-1` is the + /// never-purge posture this binary compiles in. + pub fn decay_ms() -> Option<(isize, isize)> { + // SAFETY: both names are NUL-terminated `ssize_t` options. + unsafe { + Some(( + tikv_jemalloc_ctl::raw::read(b"opt.dirty_decay_ms\0").ok()?, + tikv_jemalloc_ctl::raw::read(b"opt.muzzy_decay_ms\0").ok()?, + )) + } + } +} + use executor::vm::instruction::decoding::Instruction; use executor::vm::instruction::execution::{Accelerator, SyscallNumbers}; use executor::{elf::Elf, flamegraph::FlamegraphGenerator, vm::execution::Executor}; @@ -236,6 +298,14 @@ enum Commands { #[arg(long)] continuations: bool, + /// Prove with the multilinear backend (WHIR) instead of FRI. + /// + /// A different proof format: verify it with `verify --whir`. The + /// multilinear path derives its own rate and query count from the + /// trace, so `--blowup` does not apply to it. + #[arg(long, conflicts_with = "continuations")] + whir: bool, + /// Continuation epoch size as log2(cycles); e.g. 20 means 1,048,576 cycles. #[arg( long, @@ -268,6 +338,136 @@ enum Commands { /// Verify a continuation proof bundle (produced by `prove --continuations`) #[arg(long)] continuations: bool, + + /// Verify a multilinear proof (produced by `prove --whir`) + #[arg(long, conflicts_with = "continuations")] + whir: bool, + }, + + /// Prove a whole block as a recursion tree, base to top node + /// + /// The block's no-epoch WHIR base proof, the leaves that verify its + /// groups, the nodes and the top: one proof a consumer verifies with + /// `verify-block` against the ELF. Needs a `--features cuda` build. The + /// production posture (table parallelism, retention, the grind's search, + /// ...) is set for every knob the environment leaves unset; the `BLOCK + /// POSTURE` lines on stderr say which. The base hash is the build's + /// (`hash_pin::BLOCK_BASE`), a constant `verify-block` shares. + ProveBlock { + /// Path to the guest ELF + #[arg(value_parser, value_hint = ValueHint::FilePath)] + elf: PathBuf, + + /// The block's private input (the guest reads it with `get_private_input()`) + #[arg(long, value_hint = ValueHint::FilePath)] + input: Option, + + /// Output path for the block proof + #[arg(short, long, value_hint = ValueHint::FilePath)] + output: PathBuf, + + /// Print the whole run's wall (base to top node) + #[arg(long)] + time: bool, + + /// Print the blake3 digest of the top proof's bytes + #[arg(long)] + digest: bool, + + /// How each group's stacked polynomials are opened: one chain per run + /// of up to four (`indexed`, production) or one per polynomial + /// (`per-poly`). A format choice: `verify-block` must be given the + /// same one; the proof file does not carry it. + #[arg(long, value_parser = ["per-poly", "indexed"], default_value = "indexed")] + group: String, + + /// The WHIR base code's blowup: 2 (production) or 4 (the legacy rate), + /// its query count from the code's own rule. A format choice: + /// `verify-block` must be given the same one; the proof file does not + /// carry it. + #[arg(long, value_parser = ["2", "4"], default_value = "2")] + blowup: String, + + /// Where the WHIR round-0 trees put their leaves: each query's coset + /// together (`coset`, production) or in the codeword's order + /// (`natural`). A format choice: `verify-block` must be given the same + /// one; the proof file does not carry it. + #[arg(long, value_parser = ["natural", "coset"], default_value = "coset")] + leaves: String, + + /// The recursion's leaf scale: `one-top` (production) doubles every + /// leaf and puts at most three under one top; a count `T` doubles the + /// leaves' chip heights when the single heights take `T` leaves or + /// more; `off` keeps every leaf at the single heights. A format choice: + /// `verify-block` must be given the same one; the proof file does not + /// carry it. + #[arg(long, value_parser = leaf_double_from_word, default_value = "one-top")] + leaf_double_from: String, + + /// How the recursion's leaves check each chain's queries: `lean` + /// (production: round 0's values four to a word, paths walked on + /// rotated hints, and the tail rounds wholly in their caps checked at + /// every leaf once) or `v1` (every query opened and folded one by + /// one). A format choice: `verify-block` must be given the same one; + /// the proof file does not carry it. + #[arg(long, value_parser = ["v1", "lean"], default_value = "lean")] + leaf_query: String, + + /// How the base's batched argue lays a table's interactions into GKR + /// trees: a price `τ` in 0..=40, a table cut into equal chunk trees + /// where that saves more input cells than the 2^τ each extra tree is + /// priced at (production: 22), or `off` (one tree a table). A format + /// choice: `verify-block` must be given the same one; the proof file + /// does not carry it. + #[arg(long, value_parser = argue_pack_word, default_value = "22")] + argue_pack: String, + }, + + /// Verify a block proof produced by `prove-block` + VerifyBlock { + /// Path to the block proof + #[arg(value_parser, value_hint = ValueHint::FilePath)] + proof: PathBuf, + + /// Path to the guest ELF the block ran (trusted) + #[arg(value_parser, value_hint = ValueHint::FilePath)] + elf: PathBuf, + + /// Print verification time + #[arg(long)] + time: bool, + + /// The group opening the block was proved with (`prove-block + /// --group`): the verifier's constant, never read from the proof. + #[arg(long, value_parser = ["per-poly", "indexed"], default_value = "indexed")] + group: String, + + /// The WHIR blowup the block was proved with (`prove-block + /// --blowup`): the verifier's constant, never read from the proof. + #[arg(long, value_parser = ["2", "4"], default_value = "2")] + blowup: String, + + /// The round-0 leaf order the block was proved with (`prove-block + /// --leaves`): the verifier's constant, never read from the proof. + #[arg(long, value_parser = ["natural", "coset"], default_value = "coset")] + leaves: String, + + /// The leaf scale the block was proved with (`prove-block + /// --leaf-double-from`): the verifier's constant, never read from the + /// proof. + #[arg(long, value_parser = leaf_double_from_word, default_value = "one-top")] + leaf_double_from: String, + + /// The leaves' query check the block was proved with (`prove-block + /// --leaf-query`): the verifier's constant, never read from the proof. + #[arg(long, value_parser = ["v1", "lean"], default_value = "lean")] + leaf_query: String, + + /// The GKR packing the block was proved with (`prove-block + /// --argue-pack`, production 22): the verifier's constant, never read + /// from the proof. + #[arg(long, value_parser = argue_pack_word, default_value = "22")] + argue_pack: String, }, /// Count main-trace and aux-trace field elements without proving @@ -284,7 +484,18 @@ enum Commands { fn main() -> ExitCode { env_logger::init(); + prover::alloc_purge::install(allocator::HOOKS); let cli = Cli::parse(); + // Still single-threaded here: clap and env_logger spawn nothing. + if matches!( + cli.command, + Commands::ProveBlock { .. } | Commands::VerifyBlock { .. } + ) { + // SAFETY: no thread but this one exists yet. + for line in unsafe { apply_posture() } { + eprintln!("{line}"); + } + } match cli.command { Commands::Execute { @@ -316,6 +527,7 @@ fn main() -> ExitCode { elements, continuations, epoch_size_log2, + whir, } => { if continuations { cmd_prove_continuation( @@ -327,6 +539,8 @@ fn main() -> ExitCode { time, cycles, ) + } else if whir { + cmd_prove_whir(elf, output, private_input, blowup, time) } else { cmd_prove(elf, output, private_input, blowup, time, cycles, elements) } @@ -337,17 +551,551 @@ fn main() -> ExitCode { blowup, time, continuations, + whir, } => { if continuations { cmd_verify_continuation(proof, elf, blowup, time) + } else if whir { + cmd_verify_whir(proof, elf, blowup, time) } else { cmd_verify(proof, elf, blowup, time) } } + Commands::ProveBlock { + elf, + input, + output, + time, + digest, + group, + blowup, + leaves, + leaf_double_from, + leaf_query, + argue_pack, + } => cmd_prove_block( + elf, + input, + output, + time, + digest, + BlockChoices { + group: &group, + blowup: &blowup, + leaves: &leaves, + leaf_double_from: &leaf_double_from, + leaf_query: &leaf_query, + argue_pack: &argue_pack, + }, + ), + Commands::VerifyBlock { + proof, + elf, + time, + group, + blowup, + leaves, + leaf_double_from, + leaf_query, + argue_pack, + } => cmd_verify_block( + proof, + elf, + time, + BlockChoices { + group: &group, + blowup: &blowup, + leaves: &leaves, + leaf_double_from: &leaf_double_from, + leaf_query: &leaf_query, + argue_pack: &argue_pack, + }, + ), Commands::CountElements { elf, private_input } => cmd_count_elements(elf, private_input), } } +/// What the posture does in an environment: the knobs it sets (each posture +/// knob the environment leaves unset), the ones the environment already sets, +/// and notes on the knobs it set to something other than the table's value. +struct PosturePlan { + set: Vec<(&'static str, &'static str)>, + env: Vec, + notes: Vec, +} + +/// The posture against the environment `env` and the host `target` (bytes): +/// the precomputed-tree cache's cap follows the target +/// ([`prover::lfm::whir_block_tree::posture_tree_cache_cap`]). +fn posture_plan(env: impl Fn(&str) -> Option, target: impl Fn() -> u64) -> PosturePlan { + use prover::lfm::whir_block_tree::{POSTURE, POSTURE_TREE_CACHE_KNOB, posture_tree_cache_cap}; + let mut plan = PosturePlan { + set: Vec::new(), + env: Vec::new(), + notes: Vec::new(), + }; + for &(name, value) in POSTURE { + if let Some(v) = env(name) { + plan.env.push(format!("{name}={v}")); + continue; + } + if name == POSTURE_TREE_CACHE_KNOB { + let target = target(); + let cap = posture_tree_cache_cap(target); + if cap != value { + plan.notes.push(format!( + "BLOCK POSTURE: {name}={cap}, not {value}: the host target is {:.1} GiB \ + (LAMBDA_VM_BLOCK_MEMORY_TARGET_GIB, else the cgroup or MemTotal less 10 GiB)", + target as f64 / (1u64 << 30) as f64 + )); + } + plan.set.push((name, cap)); + continue; + } + plan.set.push((name, value)); + } + plan +} + +/// The block's production posture ([`prover::lfm::whir_block_tree::POSTURE`]), +/// set for every knob the environment leaves unset ([`posture_plan`]), for +/// `verify-block` too. The base hash is not posture: it is the build's +/// (`hash_pin::BLOCK_BASE`). +/// Returns the lines that say what it set and what it left. +/// +/// # Safety +/// +/// Sets environment variables: no other thread may exist. +unsafe fn apply_posture() -> Vec { + let plan = posture_plan( + |name| std::env::var(name).ok(), + prover::block_whir::memory_target_bytes, + ); + for (name, value) in &plan.set { + // SAFETY: the caller's: no other thread exists. + unsafe { std::env::set_var(name, value) }; + } + let decay = match allocator::decay_ms() { + Some((-1, -1)) => "never purge (dirty -1, muzzy -1)".to_string(), + Some((d, m)) => format!("dirty_decay_ms {d}, muzzy_decay_ms {m} (_RJEM_MALLOC_CONF)"), + None => "unread".to_string(), + }; + let words = |w: Vec| { + if w.is_empty() { + "none".to_string() + } else { + w.join(" ") + } + }; + let mut lines = vec![format!( + "BLOCK POSTURE SET: {} · from the environment: {} · jemalloc: {decay}", + words(plan.set.iter().map(|(n, v)| format!("{n}={v}")).collect()), + words(plan.env), + )]; + lines.extend(plan.notes); + lines +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// The block commands' format choices, the words of `--group`, `--blowup`, +/// `--leaves`, `--leaf-double-from` and `--leaf-query`: `verify-block` must be +/// given the ones `prove-block` was, and the proof file carries none of them. +struct BlockChoices<'a> { + group: &'a str, + blowup: &'a str, + leaves: &'a str, + leaf_double_from: &'a str, + leaf_query: &'a str, + argue_pack: &'a str, +} + +/// `--argue-pack`'s words: `off` or a price in 0..=40 +/// (`whir_block_tree::pack_setting`). +fn argue_pack_word(word: &str) -> Result { + prover::lfm::whir_block_tree::pack_setting(Some(word))?; + Ok(word.to_string()) +} + +/// `--leaf-double-from`'s words: `off`, `one-top` or a leaf count +/// (`whir_block::leaf_scale_setting`). +fn leaf_double_from_word(word: &str) -> Result { + prover::lfm::whir_block::leaf_scale_setting(Some(word))?; + Ok(word.to_string()) +} + +/// The leaf scale of `--leaf-double-from`: the caller's, so the format's own +/// environment knob is refused rather than silently outranked. +fn block_leaf_scale(word: &str) -> Result { + use prover::lfm::whir_block::{LEAF_DOUBLE_FROM_ENV, leaf_scale_setting}; + if std::env::var_os(LEAF_DOUBLE_FROM_ENV).is_some() { + return Err(format!( + "{LEAF_DOUBLE_FROM_ENV} is the harness's knob: give the CLI --leaf-double-from instead" + )); + } + leaf_scale_setting(Some(word)) +} + +/// The leaves' query check of `--leaf-query`: the caller's, so the format's +/// own environment knob is refused rather than silently outranked. +fn block_leaf_query(word: &str) -> Result { + use prover::lfm::whir_block::{LEAF_QUERY_ENV, leaf_query_setting}; + if std::env::var_os(LEAF_QUERY_ENV).is_some() { + return Err(format!( + "{LEAF_QUERY_ENV} is the harness's knob: give the CLI --leaf-query instead" + )); + } + leaf_query_setting(Some(word)) +} + +fn cmd_prove_block( + elf_path: PathBuf, + input_path: Option, + output_path: PathBuf, + time: bool, + digest: bool, + choices: BlockChoices<'_>, +) -> ExitCode { + use prover::lfm::whir_block_tree::{ + StderrSink, WhirBlockTreeProof, WhirTreeConfig, group_setting, prove_whir_block_tree, + }; + let BlockChoices { + group, + blowup, + leaves, + leaf_double_from, + leaf_query, + argue_pack, + } = choices; + + // The group opening is the caller's constant, given on the command line + // and to `verify-block` alike; the harness's environment word for it is + // refused here rather than silently outranked. + if std::env::var_os("LAMBDA_VM_WHIR_GROUP").is_some() { + eprintln!("LAMBDA_VM_WHIR_GROUP is the harness's knob: give the CLI --group instead"); + return ExitCode::FAILURE; + } + // So is the GKR packing. + if std::env::var_os("BLOCK_WHIR_ARGUE_PACK").is_some() { + eprintln!("BLOCK_WHIR_ARGUE_PACK is the harness's knob: give the CLI --argue-pack instead"); + return ExitCode::FAILURE; + } + let pack = match prover::lfm::whir_block_tree::pack_setting(Some(argue_pack)) { + Ok(pack) => pack, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + // So are the WHIR blowup and the round-0 leaf order, installed as the + // process format before anything reads it: the banner then states the + // format this block is proved under. + let (blowup, order) = match install_block_format(blowup, leaves) { + Ok(choices) => choices, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + eprintln!("BLOCK WHIR BLOWUP: {blowup}"); + eprintln!("BLOCK WHIR LEAVES: {leaves}"); + // So is the leaves' scale: the tree's format below takes the flag's. + let leaf_scale = match block_leaf_scale(leaf_double_from) { + Ok(scale) => scale, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + eprintln!("BLOCK LEAF SCALE: {leaf_scale}"); + // So is the leaves' query check. + let leaf_query = match block_leaf_query(leaf_query) { + Ok(query) => query, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + eprintln!("BLOCK LEAF QUERY: {leaf_query:?}"); + let elf_data = match std::fs::read(&elf_path) { + Ok(data) => data, + Err(e) => { + eprintln!("Failed to read ELF file: {e}"); + return ExitCode::FAILURE; + } + }; + let input = match read_private_input(input_path.as_ref()) { + Ok(input) => input, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + let mut cfg = match WhirTreeConfig::from_env() { + Ok(cfg) => cfg, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + cfg.format.group = match group_setting(Some(group)) { + Ok(group) => group, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + eprintln!("BLOCK GROUP OPENING: {group}"); + cfg.format.leaf_scale = leaf_scale; + cfg.format.leaf_query = leaf_query; + cfg.format = cfg.format.with_pack(pack); + cfg.argue = cfg.format.argue; + eprintln!("BLOCK ARGUE PACK: {argue_pack} ({pack:?})"); + assert_eq!( + (cfg.format.zf.whir_blowup, cfg.format.zf.whir_leaves), + (blowup.factor(), order), + "the tree's format is the installed one" + ); + // A forced leaf count, another fan-in or another argue is another tree than + // the one `verify-block` derives: harness arms, not a production proof. + if !cfg.at_presets() { + eprintln!( + "W3_LEAVES, W3_FAN_IN and BLOCK_WHIR_ARGUE make another tree than the block \ + verifier derives: unset them to prove a block the CLI can verify" + ); + return ExitCode::FAILURE; + } + let mut run = match prove_whir_block_tree(&elf_data, &input, &cfg, &StderrSink) { + Ok(run) => run, + Err(e) => { + eprintln!("Block proof failed: {e}"); + return ExitCode::FAILURE; + } + }; + let whole = run.times.whole; + let Some((_, top)) = run.proofs.pop() else { + eprintln!("Block proof failed: the tree has no top"); + return ExitCode::FAILURE; + }; + let top_digest = digest.then(|| match rkyv::to_bytes::(&top.proof) { + Ok(bytes) => format!( + "BLOCK TOP DIGEST: {} (blake3 of the top proof's {} bytes)", + &blake3::hash(&bytes).to_hex()[..32], + bytes.len() + ), + Err(e) => format!("BLOCK TOP DIGEST: none ({e})"), + }); + let output_hex = hex(&run.statement.public_output); + let file = WhirBlockTreeProof::new(run.statement, top); + let bytes = match file.to_bytes() { + Ok(bytes) => bytes, + Err(e) => { + eprintln!("Failed to serialize the block proof: {e}"); + return ExitCode::FAILURE; + } + }; + if let Err(e) = std::fs::write(&output_path, &bytes) { + eprintln!("Failed to write {output_path:?}: {e}"); + return ExitCode::FAILURE; + } + eprintln!( + "Block proof written to {output_path:?} ({} bytes)", + bytes.len() + ); + println!("Output: {output_hex}"); + if let Some(line) = top_digest { + println!("{line}"); + } + if time { + println!("Proving time: {whole:.3}s"); + } + ExitCode::SUCCESS +} + +fn cmd_verify_block( + proof_path: PathBuf, + elf_path: PathBuf, + time: bool, + choices: BlockChoices<'_>, +) -> ExitCode { + use prover::lfm::whir_block_tree::{ + WhirBlockTreeProof, base_line, group_setting, verify_whir_block_tree_proof_at, + }; + let BlockChoices { + group, + blowup, + leaves, + leaf_double_from, + leaf_query, + argue_pack, + } = choices; + + let fold = match group_setting(Some(group)) { + Ok(fold) => fold, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + // The verifier is told the rate and the leaf order as it is told the group + // opening, and both are installed as the process format before anything + // reads it. + let (blowup, order) = match install_verify_format(blowup, leaves) { + Ok(choices) => choices, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + // So is the leaves' scale, before the production format below reads the + // environment's knob, which this refuses. + let leaf_scale = match block_leaf_scale(leaf_double_from) { + Ok(scale) => scale, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + let leaf_query = match block_leaf_query(leaf_query) { + Ok(query) => query, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + let pack = match prover::lfm::whir_block_tree::pack_setting(Some(argue_pack)) { + Ok(pack) => pack, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + let zf = prover::block_whir::BlockFormat::production_with(fold).zf; + assert_eq!( + (zf.whir_blowup, zf.whir_leaves), + (blowup.factor(), order), + "the verifier's process format is the installed one" + ); + + let elf_data = match std::fs::read(&elf_path) { + Ok(data) => data, + Err(e) => { + eprintln!("Failed to read ELF file: {e}"); + return ExitCode::FAILURE; + } + }; + let bytes = match read_aligned_file(&proof_path) { + Ok(b) => b, + Err(e) => { + eprintln!("Failed to read proof file: {e}"); + return ExitCode::FAILURE; + } + }; + let proof = match WhirBlockTreeProof::from_bytes(&bytes) { + Ok(p) => p, + Err(e) => { + eprintln!("Failed to read the block proof: {e}"); + return ExitCode::FAILURE; + } + }; + drop(bytes); + // The block's base is its proof family's: the verifier's constant, never + // read from the file. + eprintln!("{}", base_line(&prover::hash_pin::BLOCK_BASE)); + // So is the group opening: the caller's, never the file's. + eprintln!("BLOCK GROUP OPENING: {group}"); + eprintln!("BLOCK WHIR BLOWUP: {blowup}"); + eprintln!("BLOCK WHIR LEAVES: {leaves}"); + eprintln!("BLOCK LEAF SCALE: {leaf_scale}"); + eprintln!("BLOCK LEAF QUERY: {leaf_query:?}"); + eprintln!("BLOCK ARGUE PACK: {argue_pack} ({pack:?})"); + eprintln!("Verifying block proof..."); + let start = Instant::now(); + let result = verify_whir_block_tree_proof_at( + &elf_data, &proof, fold, blowup, order, leaf_scale, leaf_query, pack, + ); + let verify_elapsed = start.elapsed(); + match result { + Ok(()) => { + eprintln!("Verification succeeded!"); + println!("Output: {}", hex(&proof.public_output)); + if time { + println!("Verification time: {:.3}s", verify_elapsed.as_secs_f64()); + } + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("Verification failed: {e}"); + ExitCode::FAILURE + } + } +} + +/// `prove-block`'s process format: the environment's ZF knobs with the +/// WHIR blowup of `--blowup` (`blowup`) and the round-0 leaf order of +/// `--leaves` (`leaves`), installed before anything reads the process format, +/// so its banner — printed once, on stdout — and every reader of it state the +/// blowup and the order the block is proved under. The format's +/// own environment knobs for both are refused: the flags are the caller's. +fn install_block_format( + blowup: &str, + leaves: &str, +) -> Result<(prover::zf_format::WhirBlowup, prover::zf_format::LeafOrder), String> { + use prover::zf_format::{ENV_WHIR_BLOWUP, ENV_WHIR_LEAVES, ZfFormat}; + for (knob, flag) in [(ENV_WHIR_BLOWUP, "--blowup"), (ENV_WHIR_LEAVES, "--leaves")] { + if std::env::var_os(knob).is_some() { + return Err(format!( + "{knob} is the harness's knob: give the CLI {flag} instead" + )); + } + } + let blowup = prover::lfm::whir_block_tree::blowup_setting(Some(blowup))?; + let leaves = prover::lfm::whir_block_tree::leaves_setting(Some(leaves))?; + let format = ZfFormat { + whir_blowup: blowup.factor(), + whir_leaves: leaves, + ..ZfFormat::from_env()? + }; + format.check_leaves()?; + ZfFormat::install(format)?; + Ok((blowup, leaves)) +} + +/// `verify-block`'s process format: [`ZfFormat::DEFAULT`] with the WHIR blowup +/// of `--blowup` and the round-0 leaf order of `--leaves`, installed before +/// anything reads the process format. The verifier pins every other field +/// (`verify_whir_block_tree_proof_at` reads none of them from the process), so +/// any ZF knob in the environment is refused rather than silently ignored. +fn install_verify_format( + blowup: &str, + leaves: &str, +) -> Result<(prover::zf_format::WhirBlowup, prover::zf_format::LeafOrder), String> { + use prover::zf_format::{ENV_KNOBS, ENV_WHIR_BLOWUP, ENV_WHIR_LEAVES, ZfFormat}; + if let Some(&knob) = ENV_KNOBS.iter().find(|k| std::env::var_os(k).is_some()) { + return Err(match knob { + ENV_WHIR_BLOWUP => { + format!("{knob} is the harness's knob: give the CLI --blowup instead") + } + ENV_WHIR_LEAVES => { + format!("{knob} is the harness's knob: give the CLI --leaves instead") + } + _ => format!("{knob} is set: the verifier pins the format to its default; unset it"), + }); + } + let blowup = prover::lfm::whir_block_tree::blowup_setting(Some(blowup))?; + let leaves = prover::lfm::whir_block_tree::leaves_setting(Some(leaves))?; + let format = ZfFormat { + whir_blowup: blowup.factor(), + whir_leaves: leaves, + ..ZfFormat::DEFAULT + }; + format.check_leaves()?; + ZfFormat::install(format)?; + Ok((blowup, leaves)) +} + fn read_private_input(path: Option<&PathBuf>) -> Result, String> { match path { Some(path) => { @@ -414,8 +1162,14 @@ struct FlamegraphCliOptions { /// One tally per [`Accelerator`] variant, printed by `execute --cycles`. #[derive(Default)] struct AccelCounts { + /// KECCAK rows: one per `keccak_permute` call, `⌊len / 136⌋ + 1` per + /// Keccak-256 call. keccak: u64, + /// Keccak-256 (buffer) calls; their permutations are counted in `keccak`. + keccak256: u64, + blake3: u64, ecsm: u64, + ecrec: u64, dma: u64, /// Bytes copied and DMA table rows those copies consume. Keccak and ECSM /// cost the same per call, so DMA is the only accelerator whose report needs @@ -436,7 +1190,15 @@ impl AccelCounts { fn tally(&mut self, accelerator: Accelerator, dst_val: u64, rows: u64) { match accelerator { Accelerator::Keccak => self.keccak += 1, + // `rows` is the call's permutation count (the executor logs it in + // `src2_val`). + Accelerator::Keccak256 => { + self.keccak256 += 1; + self.keccak += rows; + } + Accelerator::Blake3 => self.blake3 += 1, Accelerator::Ecsm => self.ecsm += 1, + Accelerator::Ecrec => self.ecrec += 1, Accelerator::Dma => { self.dma += 1; self.dma_bytes += dst_val; @@ -624,7 +1386,10 @@ fn cmd_execute( println!("Cycles: {}", cycle_count); if let Some(counts) = accel_counts { println!("Keccak calls: {}", counts.keccak); + println!("Keccak256 calls: {}", counts.keccak256); + println!("Blake3 calls: {}", counts.blake3); println!("Ecsm calls: {}", counts.ecsm); + println!("Ecrec calls: {}", counts.ecrec); println!("Dma calls: {}", counts.dma); println!("Dma bytes: {}", counts.dma_bytes); println!("Dma rows: {}", counts.dma_rows); @@ -824,6 +1589,159 @@ fn cmd_verify(proof_path: PathBuf, elf_path: PathBuf, blowup: u8, time: bool) -> } } +/// `prove --whir`: the multilinear backend. +/// +/// Same executor, same traces, same AIRs as [`cmd_prove`]; each table is argued +/// in one sumcheck against a WHIR commitment instead of a composition +/// polynomial committed by FRI. The proof is a different type, so `verify` +/// needs `--whir` too. +/// +/// `blowup` reaches the AIRs (it sets the preprocessed commitments) but not the +/// multilinear argument, which derives its own rate and query count from the +/// tallest stacked polynomial in the proof. +fn cmd_prove_whir( + elf_path: PathBuf, + output_path: PathBuf, + private_input_path: Option, + blowup: u8, + time: bool, +) -> ExitCode { + eprintln!("Reading ELF file..."); + let elf_data = match std::fs::read(&elf_path) { + Ok(data) => data, + Err(e) => { + eprintln!("Failed to read ELF file: {}", e); + return ExitCode::FAILURE; + } + }; + + let private_inputs = match read_private_input(private_input_path.as_ref()) { + Ok(inputs) => inputs, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + + let opts = match GoldilocksCubicProofOptions::with_blowup(blowup) { + Ok(opts) => opts, + Err(e) => { + eprintln!("Invalid proof options: {e}"); + return ExitCode::FAILURE; + } + }; + + eprintln!("Generating proof (multilinear/WHIR)..."); + let start = Instant::now(); + let proof = prover::multilinear_prove::prove_with_options_and_inputs( + &elf_data, + &private_inputs, + &opts, + &Default::default(), + ); + let prove_elapsed = start.elapsed(); + let proof = match proof { + Ok(proof) => proof, + Err(e) => { + eprintln!("Proof generation failed: {}", e); + return ExitCode::FAILURE; + } + }; + + eprintln!("Writing proof..."); + let bytes = match rkyv::to_bytes::(&proof) { + Ok(b) => b, + Err(e) => { + eprintln!("Failed to serialize proof: {}", e); + return ExitCode::FAILURE; + } + }; + let file = match File::create(&output_path) { + Ok(f) => f, + Err(e) => { + eprintln!("Failed to create output file: {}", e); + return ExitCode::FAILURE; + } + }; + if let Err(e) = BufWriter::new(file).write_all(&bytes) { + eprintln!("Failed to write proof: {}", e); + return ExitCode::FAILURE; + } + + eprintln!("Proof written to {:?}", output_path); + println!("Tables: {}", proof.proof.tables.len()); + println!("Commitments: {}", proof.proof.roots.len()); + if time { + println!("Proving time: {:.3}s", prove_elapsed.as_secs_f64()); + } + ExitCode::SUCCESS +} + +/// `verify --whir`: the counterpart of [`cmd_prove_whir`]. +fn cmd_verify_whir(proof_path: PathBuf, elf_path: PathBuf, blowup: u8, time: bool) -> ExitCode { + eprintln!("Reading ELF file..."); + let elf_data = match std::fs::read(&elf_path) { + Ok(data) => data, + Err(e) => { + eprintln!("Failed to read ELF file: {}", e); + return ExitCode::FAILURE; + } + }; + + eprintln!("Reading proof..."); + let proof_bytes = match read_aligned_file(&proof_path) { + Ok(b) => b, + Err(e) => { + eprintln!("Failed to read proof file: {}", e); + return ExitCode::FAILURE; + } + }; + + let proof = match rkyv::from_bytes::< + prover::multilinear_prove::MultilinearVmProof, + rkyv::rancor::Error, + >(&proof_bytes) + { + Ok(p) => p, + Err(e) => { + eprintln!("Failed to deserialize proof: {e}"); + eprintln!("(a proof produced without `--whir` needs `verify` without it)"); + return ExitCode::FAILURE; + } + }; + + let opts = match GoldilocksCubicProofOptions::with_blowup(blowup) { + Ok(opts) => opts, + Err(e) => { + eprintln!("Invalid proof options: {e}"); + return ExitCode::FAILURE; + } + }; + + eprintln!("Verifying proof (multilinear/WHIR)..."); + let start = Instant::now(); + let result = prover::multilinear_prove::verify_with_options(&proof, &elf_data, &opts); + let verify_elapsed = start.elapsed(); + let result = match result { + Ok(valid) => valid, + Err(e) => { + eprintln!("Verification error: {}", e); + return ExitCode::FAILURE; + } + }; + + if result { + eprintln!("Verification succeeded!"); + if time { + println!("Verification time: {:.3}s", verify_elapsed.as_secs_f64()); + } + ExitCode::SUCCESS + } else { + eprintln!("Verification failed! Ensure --blowup matches the value used for proving."); + ExitCode::FAILURE + } +} + fn cmd_prove_continuation( elf_path: PathBuf, output_path: PathBuf, @@ -1190,6 +2108,370 @@ mod tests { assert!(!help.contains("--epoch-size <")); } + /// `prove-block` takes the ELF, an optional input and a required output; + /// `verify-block` the proof and the ELF. + #[test] + fn the_block_commands_take_their_arguments() { + let ok = |args: &[&str]| Cli::command().try_get_matches_from(args).is_ok(); + assert!(ok(&[ + "cli", + "prove-block", + "guest.elf", + "-o", + "block.proof" + ])); + assert!(ok(&[ + "cli", + "prove-block", + "guest.elf", + "--input", + "block.bin", + "-o", + "block.proof", + "--time", + "--digest" + ])); + assert!( + !ok(&["cli", "prove-block", "guest.elf"]), + "an output is required" + ); + assert!(ok(&[ + "cli", + "verify-block", + "block.proof", + "guest.elf", + "--time" + ])); + assert!( + !ok(&["cli", "verify-block", "block.proof"]), + "the ELF is required" + ); + } + + /// Both block commands take the group opening, one chain per run unless + /// told otherwise, and nothing but its two words. + #[test] + fn the_block_commands_take_the_group_opening() { + let parse = |args: &[&str]| Cli::try_parse_from(args); + let prove = ["cli", "prove-block", "guest.elf", "-o", "block.proof"]; + let verify = ["cli", "verify-block", "block.proof", "guest.elf"]; + for cmd in [&prove[..], &verify[..]] { + let group = |args: &[&str]| match parse(args).map(|cli| cli.command) { + Ok(Commands::ProveBlock { group, .. } | Commands::VerifyBlock { group, .. }) => { + Some(group) + } + _ => None, + }; + assert_eq!(group(cmd).as_deref(), Some("indexed"), "the default"); + for word in ["per-poly", "indexed"] { + let args = [cmd, &["--group", word]].concat(); + assert_eq!(group(&args).as_deref(), Some(word)); + assert!( + prover::lfm::whir_block_tree::group_setting(Some(word)).is_ok(), + "{word} reads" + ); + } + let args = [cmd, &["--group", "index"]].concat(); + assert!(parse(&args).is_err(), "another word is refused"); + } + } + + /// Both block commands take the WHIR blowup, 2 unless told otherwise, and + /// nothing but its two words; each word reads to the blowup it names. + #[test] + fn the_block_commands_take_the_blowup() { + let parse = |args: &[&str]| Cli::try_parse_from(args); + let prove = ["cli", "prove-block", "guest.elf", "-o", "block.proof"]; + let verify = ["cli", "verify-block", "block.proof", "guest.elf"]; + for cmd in [&prove[..], &verify[..]] { + let blowup = |args: &[&str]| match parse(args).map(|cli| cli.command) { + Ok(Commands::ProveBlock { blowup, .. } | Commands::VerifyBlock { blowup, .. }) => { + Some(blowup) + } + _ => None, + }; + assert_eq!(blowup(cmd).as_deref(), Some("2"), "the default"); + for (word, want) in [ + ("4", prover::zf_format::WhirBlowup::Four), + ("2", prover::zf_format::WhirBlowup::Two), + ] { + let args = [cmd, &["--blowup", word]].concat(); + assert_eq!(blowup(&args).as_deref(), Some(word)); + assert_eq!( + prover::lfm::whir_block_tree::blowup_setting(Some(word)), + Ok(want), + "{word} reads" + ); + } + for word in ["1", "8", "half"] { + let args = [cmd, &["--blowup", word]].concat(); + assert!(parse(&args).is_err(), "--blowup {word} is refused"); + } + } + assert_eq!( + prover::lfm::whir_block_tree::blowup_setting(None).map(|b| b.factor()), + Ok(prover::block_whir::BlockFormat::production().zf.whir_blowup), + "unset is the production blowup" + ); + } + + /// Both block commands take the round-0 leaf order, the coset order unless told + /// otherwise, and nothing but its two words; each word reads to the order + /// it names. + #[test] + fn the_block_commands_take_the_leaf_order() { + use prover::zf_format::LeafOrder; + let parse = |args: &[&str]| Cli::try_parse_from(args); + let prove = ["cli", "prove-block", "guest.elf", "-o", "block.proof"]; + let verify = ["cli", "verify-block", "block.proof", "guest.elf"]; + for cmd in [&prove[..], &verify[..]] { + let leaves = |args: &[&str]| match parse(args).map(|cli| cli.command) { + Ok(Commands::ProveBlock { leaves, .. } | Commands::VerifyBlock { leaves, .. }) => { + Some(leaves) + } + _ => None, + }; + assert_eq!(leaves(cmd).as_deref(), Some("coset"), "the default"); + for (word, want) in [("natural", LeafOrder::Natural), ("coset", LeafOrder::Coset)] { + let args = [cmd, &["--leaves", word]].concat(); + assert_eq!(leaves(&args).as_deref(), Some(word)); + assert_eq!( + prover::lfm::whir_block_tree::leaves_setting(Some(word)), + Ok(want), + "{word} reads" + ); + } + for word in ["Coset", "bit-reversed", "1"] { + let args = [cmd, &["--leaves", word]].concat(); + assert!(parse(&args).is_err(), "--leaves {word} is refused"); + } + } + assert_eq!( + prover::lfm::whir_block_tree::leaves_setting(None), + Ok(prover::zf_format::ZfFormat::DEFAULT.whir_leaves), + "unset is the production order" + ); + } + + /// Both block commands take the leaf scale, production's threshold of 10 + /// unless told otherwise, and nothing but `off` or a leaf count; each word + /// reads to the scale it names. + #[test] + fn the_block_commands_take_the_leaf_scale() { + use prover::lfm::whir_block::LeafScale; + let parse = |args: &[&str]| Cli::try_parse_from(args); + let prove = ["cli", "prove-block", "guest.elf", "-o", "block.proof"]; + let verify = ["cli", "verify-block", "block.proof", "guest.elf"]; + for cmd in [&prove[..], &verify[..]] { + let scale = |args: &[&str]| match parse(args).map(|cli| cli.command) { + Ok( + Commands::ProveBlock { + leaf_double_from, .. + } + | Commands::VerifyBlock { + leaf_double_from, .. + }, + ) => Some(leaf_double_from), + _ => None, + }; + assert_eq!(scale(cmd).as_deref(), Some("one-top"), "the default"); + for (word, want) in [ + ("off", LeafScale::Fixed), + ("12", LeafScale::DoubleFrom(12)), + ("10", LeafScale::DoubleFrom(10)), + ("one-top", LeafScale::DoubledOneTop), + ] { + let args = [cmd, &["--leaf-double-from", word]].concat(); + assert_eq!(scale(&args).as_deref(), Some(word)); + assert_eq!( + prover::lfm::whir_block::leaf_scale_setting(Some(word)), + Ok(want), + "{word} reads" + ); + } + for word in ["0", "on", "twelve", "-3"] { + let args = [cmd, &["--leaf-double-from", word]].concat(); + assert!( + parse(&args).is_err(), + "--leaf-double-from {word} is refused" + ); + } + } + assert_eq!( + prover::lfm::whir_block::leaf_scale_setting(Some("one-top")), + Ok(LeafScale::DEFAULT), + "the flag's default is production's" + ); + } + + /// Both block commands take the leaves' query check, production's `lean` + /// unless told otherwise, and nothing but `v1` or `lean`; each word reads + /// to the check it names. + #[test] + fn the_block_commands_take_the_leaf_query() { + use prover::lfm::whir_chain::LeafQuery; + let parse = |args: &[&str]| Cli::try_parse_from(args); + let prove = ["cli", "prove-block", "guest.elf", "-o", "block.proof"]; + let verify = ["cli", "verify-block", "block.proof", "guest.elf"]; + for cmd in [&prove[..], &verify[..]] { + let query = |args: &[&str]| match parse(args).map(|cli| cli.command) { + Ok( + Commands::ProveBlock { leaf_query, .. } + | Commands::VerifyBlock { leaf_query, .. }, + ) => Some(leaf_query), + _ => None, + }; + assert_eq!(query(cmd).as_deref(), Some("lean"), "the default"); + for (word, want) in [("v1", LeafQuery::V1), ("lean", LeafQuery::Lean)] { + let args = [cmd, &["--leaf-query", word]].concat(); + assert_eq!(query(&args).as_deref(), Some(word)); + assert_eq!( + prover::lfm::whir_block::leaf_query_setting(Some(word)), + Ok(want), + "{word} reads" + ); + } + for word in ["", "V1", "lean2", "on"] { + let args = [cmd, &["--leaf-query", word]].concat(); + assert!(parse(&args).is_err(), "--leaf-query {word:?} is refused"); + } + } + assert_eq!( + prover::lfm::whir_block::leaf_query_setting(Some("lean")), + Ok(LeafQuery::default()), + "the flag's default is production's" + ); + } + + /// Both block commands take the GKR packing as `--argue-pack`, `off` or a + /// price; unset, both read the production packing (the harness's unset + /// reading). + #[test] + fn the_block_commands_take_the_argue_pack() { + use prover::lfm::whir_block_tree::pack_setting; + let parse = |args: &[&str]| Cli::try_parse_from(args); + let prove = ["cli", "prove-block", "guest.elf", "-o", "block.proof"]; + let verify = ["cli", "verify-block", "block.proof", "guest.elf"]; + for cmd in [&prove[..], &verify[..]] { + let pack = |args: &[&str]| match parse(args).map(|cli| cli.command) { + Ok( + Commands::ProveBlock { argue_pack, .. } + | Commands::VerifyBlock { argue_pack, .. }, + ) => Some(argue_pack), + _ => None, + }; + let default = pack(cmd).expect("the default parses"); + assert_eq!( + pack_setting(Some(&default)), + pack_setting(None), + "the default is production's" + ); + assert_ne!(pack_setting(Some("off")), pack_setting(None)); + for word in ["off", "22", "0", "40"] { + let args = [cmd, &["--argue-pack", word]].concat(); + assert_eq!(pack(&args).as_deref(), Some(word)); + } + for word in ["", "on", "41", "-1", "OFF"] { + let args = [cmd, &["--argue-pack", word]].concat(); + assert!(parse(&args).is_err(), "--argue-pack {word:?} is refused"); + } + } + } + + /// The block commands take no base hash: the block's base is its proof + /// family's, a constant of the build, so `--base-hash` is refused on both. + #[test] + fn the_block_commands_take_no_base_hash() { + let ok = |args: &[&str]| Cli::try_parse_from(args).is_ok(); + let prove = ["cli", "prove-block", "guest.elf", "-o", "block.proof"]; + let verify = ["cli", "verify-block", "block.proof", "guest.elf"]; + for cmd in [&prove[..], &verify[..]] { + assert!(ok(cmd)); + for word in ["rpx", "p1"] { + let args = [cmd, &["--base-hash", word]].concat(); + assert!(!ok(&args), "--base-hash {word} is refused"); + } + } + assert_eq!( + prover::block_whir::BlockFormat::production().zf.base, + stark::proof::options::BaseFormat::P1_WHIR, + "the production base is the family's" + ); + } + + /// The posture sets exactly the knobs the environment leaves unset. + #[test] + fn the_posture_sets_only_unset_knobs() { + use prover::lfm::whir_block_tree::POSTURE; + let roomy = || 110u64 << 30; + let all = posture_plan(|_| None, roomy); + assert_eq!(all.set, POSTURE.to_vec()); + assert!(all.env.is_empty() && all.notes.is_empty()); + let mine = posture_plan( + |n| (n == "TABLE_PARALLELISM").then(|| "8".to_string()), + roomy, + ); + assert!(mine.set.iter().all(|(n, _)| *n != "TABLE_PARALLELISM")); + assert_eq!(mine.env, vec!["TABLE_PARALLELISM=8".to_string()]); + } + + /// The precomputed-tree cache's posture follows the host target: 64 + /// entries from a 64 GiB target up, 16 below it (with a note), and a value + /// the environment sets is left alone. + #[test] + fn the_tree_cache_posture_follows_the_host_target() { + use prover::lfm::whir_block_tree::POSTURE_TREE_CACHE_KNOB as CAP; + let none = |_: &str| None; + let cap = |plan: &PosturePlan| plan.set.iter().find(|(n, _)| *n == CAP).map(|(_, v)| *v); + let big = posture_plan(none, || 64u64 << 30); + assert_eq!(cap(&big), Some("64")); + assert!(big.notes.is_empty()); + let small = posture_plan(none, || 38u64 << 30); + assert_eq!(cap(&small), Some("16")); + assert_eq!(small.notes.len(), 1, "the note says why"); + let set = posture_plan(|n| (n == CAP).then(|| "32".to_string()), || 38u64 << 30); + assert_eq!(cap(&set), None, "the environment's value stays"); + assert_eq!(set.env, vec![format!("{CAP}=32")]); + } + + /// The binary runs the allocator posture it compiles in: jemalloc never + /// purges (`opt.dirty_decay_ms` and `opt.muzzy_decay_ms` are -1), unless + /// `_RJEM_MALLOC_CONF` says otherwise. Deleting the `malloc_conf` export + /// turns this red. + #[test] + fn the_binary_runs_its_never_purge_posture() { + if std::env::var_os("_RJEM_MALLOC_CONF").is_some() { + println!("_RJEM_MALLOC_CONF is set: the compiled posture is overridden, not read"); + return; + } + assert_eq!(allocator::decay_ms(), Some((-1, -1))); + } + + /// The hooks the binary installs reach its jemalloc: the prover reads the + /// statistics through them, and the purge hands back the pages 512 freed + /// buffers of 1 MiB left behind (under the never-purge posture they stay + /// resident until then). + #[test] + fn the_installed_hooks_read_and_purge_this_binarys_jemalloc() { + prover::alloc_purge::install(allocator::HOOKS); + let resident = || { + prover::alloc_purge::stats() + .expect("the prover reads the installed hooks") + .resident + }; + let buffers: Vec> = (0..512).map(|_| vec![1u8; 1 << 20]).collect(); + drop(buffers); + let retained = resident(); + assert!((allocator::HOOKS.purge_all)(), "the purge runs"); + let purged = resident(); + assert!( + retained >= purged + (256 << 20), + "resident {} → {} MiB: the purge returned less than half of 512 MiB freed", + retained >> 20, + purged >> 20 + ); + } + #[test] fn continuation_epoch_size_rejects_tiny_cli_values() { assert!(continuation_epoch_size(17).is_err()); @@ -1216,7 +2498,11 @@ mod tests { /// until it gets a row here. const EXPECTED_ACCELERATORS: &[(SyscallNumbers, Option)] = &[ (SyscallNumbers::KeccakPermute, Some(Accelerator::Keccak)), + (SyscallNumbers::Keccak256, Some(Accelerator::Keccak256)), + (SyscallNumbers::Blake3Compress, Some(Accelerator::Blake3)), + (SyscallNumbers::Blake3Absorb, Some(Accelerator::Blake3)), (SyscallNumbers::Ecsm, Some(Accelerator::Ecsm)), + (SyscallNumbers::Ecrec, Some(Accelerator::Ecrec)), (SyscallNumbers::DmaMemcpy, Some(Accelerator::Dma)), (SyscallNumbers::DmaMemset, Some(Accelerator::Dma)), (SyscallNumbers::Print, None), @@ -1276,13 +2562,21 @@ mod tests { let mut counts = AccelCounts::default(); counts.tally(accelerator, 0, 0); assert_eq!( - counts.keccak + counts.ecsm + counts.dma, + counts.keccak + + counts.keccak256 + + counts.blake3 + + counts.ecsm + + counts.ecrec + + counts.dma, 1, "{accelerator:?} must increment exactly one counter" ); let expected = match accelerator { Accelerator::Keccak => counts.keccak, + Accelerator::Keccak256 => counts.keccak256, + Accelerator::Blake3 => counts.blake3, Accelerator::Ecsm => counts.ecsm, + Accelerator::Ecrec => counts.ecrec, Accelerator::Dma => counts.dma, }; assert_eq!( diff --git a/bin/cli/tests/block_banner.rs b/bin/cli/tests/block_banner.rs new file mode 100644 index 000000000..33e0ce224 --- /dev/null +++ b/bin/cli/tests/block_banner.rs @@ -0,0 +1,163 @@ +//! The block commands' `ZF FORMAT` banner states the format they run under: +//! `--blowup 4` prints `whir_blowup=4` on the one banner line (and Q 114 on the +//! schedule line), the default and `--blowup 2` name no blowup (Q 229), and the +//! format's environment knob for the blowup is refused before any banner. So +//! for the round-0 leaf order: `--leaves natural` prints `whir_leaves=natural` +//! (after the blowup when both are set), the default and `--leaves coset` name +//! no order, and its environment knob is refused before any banner. The +//! banner is printed before the files are read, so a missing file is enough. + +use std::process::Command; + +fn run(args: &[&str], extra: &[(&str, &str)]) -> (bool, String, String) { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cli")); + cmd.args(args); + for (name, _) in std::env::vars() { + if name.starts_with("LAMBDA_VM_ZF_") || name == "LAMBDA_VM_WHIR_GROUP" { + cmd.env_remove(name); + } + } + for (name, value) in extra { + cmd.env(name, value); + } + let out = cmd.output().expect("the CLI runs"); + ( + out.status.success(), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +fn lines<'a>(stdout: &'a str, prefix: &str) -> Vec<&'a str> { + stdout.lines().filter(|l| l.starts_with(prefix)).collect() +} + +#[test] +fn the_block_commands_banner_states_the_blowup_they_run_under() { + let dir = tempfile::tempdir().expect("a scratch directory"); + let missing = dir.path().join("missing"); + let missing = missing.to_str().expect("a UTF-8 path"); + let proof = dir.path().join("block.proof"); + let proof = proof.to_str().expect("a UTF-8 path"); + let commands: [(&str, Vec<&str>); 2] = [ + ("prove-block", vec!["prove-block", missing, "-o", proof]), + ("verify-block", vec!["verify-block", missing, missing]), + ]; + for (name, base) in &commands { + for (flag, blowup_word, q) in [ + (None, None, " q=229 "), + (Some("2"), None, " q=229 "), + (Some("4"), Some(" whir_blowup=4"), " q=114 "), + ] { + let mut args = base.clone(); + if let Some(f) = flag { + args.extend(["--blowup", f]); + } + let (ok, stdout, stderr) = run(&args, &[]); + let tag = format!("{name} --blowup {flag:?}"); + assert!(!ok, "{tag}: a missing file fails the command"); + let banners = lines(&stdout, "ZF FORMAT:"); + assert_eq!(banners.len(), 1, "{tag}: one banner\n{stdout}\n{stderr}"); + match blowup_word { + Some(word) => assert!(banners[0].ends_with(word), "{tag}: {}", banners[0]), + None => assert!(!banners[0].contains("whir_blowup"), "{tag}: {}", banners[0]), + } + let schedules = lines(&stdout, "ZF WHIR SCHEDULES:"); + assert_eq!(schedules.len(), 1, "{tag}: one schedule line\n{stdout}"); + assert!(schedules[0].contains(q), "{tag}: {}", schedules[0]); + } + let (ok, stdout, stderr) = run(base, &[("LAMBDA_VM_ZF_WHIR_BLOWUP", "4")]); + assert!(!ok, "{name}: the environment knob is refused"); + assert!( + lines(&stdout, "ZF FORMAT:").is_empty(), + "{name}: no banner before the refusal\n{stdout}" + ); + assert!( + stderr.contains("give the CLI --blowup instead"), + "{name}: the refusal names the flag\n{stderr}" + ); + } +} + +#[test] +fn the_block_commands_banner_states_the_leaf_order_they_run_under() { + let dir = tempfile::tempdir().expect("a scratch directory"); + let missing = dir.path().join("missing"); + let missing = missing.to_str().expect("a UTF-8 path"); + let proof = dir.path().join("block.proof"); + let proof = proof.to_str().expect("a UTF-8 path"); + let commands: [(&str, Vec<&str>); 2] = [ + ("prove-block", vec!["prove-block", missing, "-o", proof]), + ("verify-block", vec!["verify-block", missing, missing]), + ]; + for (name, base) in &commands { + for (flags, tail) in [ + (&[][..], None), + (&["--leaves", "coset"][..], None), + (&["--leaves", "natural"][..], Some(" whir_leaves=natural")), + ( + &["--leaves", "natural", "--blowup", "4"][..], + Some(" whir_blowup=4 whir_leaves=natural"), + ), + ] { + let mut args = base.clone(); + args.extend(flags); + let (ok, stdout, stderr) = run(&args, &[]); + let tag = format!("{name} {flags:?}"); + assert!(!ok, "{tag}: a missing file fails the command"); + let banners = lines(&stdout, "ZF FORMAT:"); + assert_eq!(banners.len(), 1, "{tag}: one banner\n{stdout}\n{stderr}"); + match tail { + Some(tail) => assert!(banners[0].ends_with(tail), "{tag}: {}", banners[0]), + None => assert!(!banners[0].contains("whir_leaves"), "{tag}: {}", banners[0]), + } + } + let (ok, stdout, stderr) = run(base, &[("LAMBDA_VM_ZF_WHIR_LEAVES", "coset")]); + assert!(!ok, "{name}: the environment knob is refused"); + assert!( + lines(&stdout, "ZF FORMAT:").is_empty(), + "{name}: no banner before the refusal\n{stdout}" + ); + assert!( + stderr.contains("give the CLI --leaves instead"), + "{name}: the refusal names the flag\n{stderr}" + ); + } +} + +/// The verifier pins the WHIR format: `verify-block` refuses every ZF knob in +/// its environment before any banner (the grind bits and the folds here, which +/// `prove-block` still takes as the prover's choice), naming it. +#[test] +fn verify_block_refuses_every_format_knob_in_its_environment() { + let dir = tempfile::tempdir().expect("a scratch directory"); + let missing = dir.path().join("missing"); + let missing = missing.to_str().expect("a UTF-8 path"); + let proof = dir.path().join("block.proof"); + let proof = proof.to_str().expect("a UTF-8 path"); + for (knob, value) in [ + ("LAMBDA_VM_ZF_WHIR_GRIND_BITS", "20"), + ("LAMBDA_VM_ZF_WHIR_FOLDS", "uniform4"), + ] { + let (ok, stdout, stderr) = run(&["verify-block", missing, missing], &[(knob, value)]); + assert!(!ok, "verify-block under {knob}"); + assert!( + lines(&stdout, "ZF FORMAT:").is_empty(), + "verify-block under {knob}: no banner before the refusal\n{stdout}" + ); + assert!( + stderr.contains(knob) && stderr.contains("pins the format"), + "verify-block under {knob}: the refusal names it\n{stderr}" + ); + let (ok, stdout, _) = run(&["prove-block", missing, "-o", proof], &[(knob, value)]); + assert!( + !ok, + "prove-block under {knob}: a missing file fails the command" + ); + assert_eq!( + lines(&stdout, "ZF FORMAT:").len(), + 1, + "prove-block under {knob}: the prover takes it\n{stdout}" + ); + } +} diff --git a/crypto/crypto/Cargo.toml b/crypto/crypto/Cargo.toml index a0bbd5f67..27903c2f4 100644 --- a/crypto/crypto/Cargo.toml +++ b/crypto/crypto/Cargo.toml @@ -17,9 +17,6 @@ serde = { version = "1.0", default-features = false, features = [ "alloc", ], optional = true } rayon = { version = "1.8.0", optional = true } -memmap2 = { version = "0.9", optional = true } -tempfile = { version = "3", optional = true } -libc = { version = "0.2", optional = true } # pointer_width_64: proof-format pointer width — see prover/Cargo.toml. rkyv = { version = "0.8.10", default-features = false, features = [ "alloc", @@ -28,6 +25,9 @@ rkyv = { version = "0.8.10", default-features = false, features = [ "pointer_width_64", ], optional = true } +# GPU dispatch for the proof-of-work grind — only linked when `cuda` is enabled. +math-cuda = { path = "../math-cuda", optional = true } + [target.'cfg(target_arch = "riscv64")'.dependencies] lambda-vm-syscalls = { path = "../../syscalls" } @@ -35,16 +35,64 @@ lambda-vm-syscalls = { path = "../../syscalls" } math = { path = "../math", features = ["test-utils"] } sha2 = { version = "0.10", default-features = false } bincode = "1" +# The external anchor for `hash::blake3`: at 7 rounds the compression function +# and `Blake3Chain` up to 1024 bytes are both bit-for-bit this crate. Test-only +# on purpose, and 7-round-only, so it CANNOT become the implementation — the +# variant behind `blake3-6round` is computed by nothing else in the world. Same +# dev-only role it has in `prover` and `math-cuda`. +blake3 = { version = "1.8.5", default-features = false, features = [ + "std", + "pure", +] } [features] -default = ["asm", "std"] +default = ["asm", "std", "blake3-6round"] asm = ["sha3/asm"] std = ["math/std", "sha3/std", "serde?/std"] serde = ["dep:serde"] parallel = ["dep:rayon"] -disk-spill = ["std", "dep:memmap2", "dep:tempfile", "dep:libc"] alloc = [] rkyv = ["dep:rkyv", "math/rkyv"] +# `hash::blake3` at the 6-round internal variant instead of the 7-round standard +# one. Off = 7 rounds: standard BLAKE3, externally anchored, carrying no +# unratified assumption. DO NOT INVERT THE POLARITY — every existing measurement +# and the A6R sign-off read "7-round instantiated baseline, 6 behind the feature". +# +# ★ IT IS NOW ON BY DEFAULT, and the polarity above is unchanged by that: the +# feature still means "6 rounds", it is simply what the default build selects. +# The reason is P-a Stage 6 — the commitment aliases moved to BLAKE3, and the +# shipping round count is 6 (PA-PLAN §1.5), so a default build that did not +# carry this would commit seven-round roots that no blessed constant matches. +# `stark::config`'s round-count assertion turns that into a compile error rather +# than a drift failure, so the two cannot come apart quietly. +# +# What this costs: a dependent using `default-features = false` on THIS crate +# silently drops to 7 rounds. That is the standard additive-features hazard and +# the assertion is what catches it; nothing in this workspace does it (the +# recursion guest disables `lambda-vm-prover`'s defaults, not this crate's). +# +# This is the host tree's single round-count knob: `lambda-vm-prover`'s feature +# of the same name forwards to it, so the `LFM_BLAKE3` chip, the `LFM_HASH` +# socket and the commitment backends move together and one build cannot produce +# two hashes. `math-cuda`'s is necessarily separate (it compiles a cubin) and has +# to be set in lockstep with this one; `make lint` has a combined pass that +# exercises both, and `math_cuda::blake3::device_rounds` makes the match +# assertable rather than discoverable as a wrong root. +blake3-6round = [] +# Let `Blake3Chain::update` reach the chained-absorb ecall — one call per run of +# whole blocks instead of one per block. Implies `blake3-6round`: the absorb +# accelerator, like the compression one, implements six rounds and nothing else. +# +# ⚠ SEPARATE FROM `blake3-6round` ON PURPOSE, and not for tuning. A guest that +# emits this ecall puts an `Ecall` bus tuple on the bus that only the prover's +# BLAKE3 absorb mode can receive. Until that mode lands, a guest built with this +# feature produces a trace whose LogUp argument cannot balance — a proof that +# fails with no algebraic pointer to the cause. Keeping the guest arm behind its +# own switch means flipping `blake3-6round` to a default cannot turn the absorb +# path on by accident. +blake3-absorb = ["blake3-6round"] +# The device nonce search behind `generate_nonce_maybe_gpu`. +cuda = ["dep:math-cuda", "std"] # Host-only diagnostic: count keccak finalizes during verify (see `hash_metrics`). # Off by default → `PlatformKeccak256 = sha3::Keccak256`, provably unchanged. -hash-metrics = [] \ No newline at end of file +hash-metrics = [] diff --git a/crypto/crypto/src/fiat_shamir/default_transcript.rs b/crypto/crypto/src/fiat_shamir/default_transcript.rs index d64f805a2..60f3919c2 100644 --- a/crypto/crypto/src/fiat_shamir/default_transcript.rs +++ b/crypto/crypto/src/fiat_shamir/default_transcript.rs @@ -1,6 +1,8 @@ use crate::fiat_shamir::is_transcript::{IsStarkTranscript, IsTranscript}; +use crate::fiat_shamir::transcript_hash::{ + Blake3TranscriptHash, KeccakTranscriptHash, TranscriptHash, +}; -use crate::hash::platform_keccak::PlatformKeccak256 as Keccak256; use core::marker::PhantomData; use digest::Digest; use math::{ @@ -16,8 +18,12 @@ use math::{ /// per squeeze). const SQUEEZE_LEN: usize = 32; -/// Keccak-sponge Fiat-Shamir transcript with a Plonky3-style duplex output -/// buffer. +/// Bytes in a field element, for the window alignment the counter reports. +#[cfg(feature = "hash-metrics")] +const FELT_BYTES: usize = 8; + +/// Sponge Fiat-Shamir transcript with a Plonky3-style duplex output buffer, +/// over the hash `T` names. /// /// Challenges are derived by squeezing the sponge and rejection-sampling field /// coordinates directly from those bytes — there is **no CSPRNG**. Earlier this @@ -28,8 +34,13 @@ const SQUEEZE_LEN: usize = 32; /// free. The output buffer amortizes one squeeze across up to `SQUEEZE_LEN / 8` /// 64-bit candidates, so a cubic-extension element (3 coordinates) usually costs /// a single squeeze. -pub struct DefaultTranscript { - hasher: Keccak256, +/// +/// `T` defaults to [`KeccakTranscriptHash`], so `DefaultTranscript::::new(..)` +/// still names exactly the transcript this system has always produced — every +/// method body below is hash-agnostic, and the keccak configuration selects the +/// unbounded rejection schedule, so its bytes do not move. +pub struct DefaultTranscript { + hasher: T::Digest, /// Duplex output buffer: bytes squeezed from the sponge, consumed 8 at a /// time by field/`u64` sampling. Positions `[out_pos, SQUEEZE_LEN)` are the /// bytes not yet handed out; `out_pos == SQUEEZE_LEN` means "empty, squeeze @@ -37,52 +48,138 @@ pub struct DefaultTranscript { /// squeeze can never reflect input appended after it was produced. out_buf: [u8; SQUEEZE_LEN], out_pos: usize, - phantom: PhantomData, + /// ★ Bytes absorbed since the sponge was last reset by a squeeze — the + /// WINDOW an in-guest verifier re-slices into field elements every 8 bytes. + /// A value absorbed at an offset that is not a multiple of 8 straddles two + /// of them, which is what the statement padding exists to prevent and what + /// `Counts::transcript_misaligned_absorbs_after_statement` counts — from the + /// statement's end, because only past that boundary is the padding making a + /// promise. See `counting` below. + /// + /// The definition is `WindowRecorder`'s, not a second one: opens at 0, + /// becomes `SQUEEZE_LEN` after a squeeze (which finalize-resets and + /// re-absorbs its own 32-byte output, so a window never opens empty), and + /// does not move on `state()`, which finalizes a clone. + /// + /// ⚠ Gated, because this module promises a normal build compiles every + /// counter call to nothing and is provably unchanged. An unconditional + /// field and an add per absorb would make that sentence false to save a few + /// `cfg` lines. + #[cfg(feature = "hash-metrics")] + window: usize, + /// ★ Whether this transcript is PAST a statement, and therefore whether a + /// misaligned absorb is a finding. + /// + /// False until `mark_statement_end`. A statement's own fields are + /// misaligned by nature and counting them made the counter a number nobody + /// could pre-register — the EQ fixture read 25 at `public_output = 0` and 24 + /// at two bytes, of which 22 and 21 were felt-sized, and ZERO fell after the + /// statement. Counting only past the mark is what makes zero the reading a + /// correct pad produces. + /// + /// ⚠ A transcript that never marks counts nothing, which is right: it has + /// no "after a statement". The byte gate's raw fixture is one. + #[cfg(feature = "hash-metrics")] + counting: bool, + phantom: PhantomData<(F, T)>, } -impl Clone for DefaultTranscript { +impl Clone for DefaultTranscript { fn clone(&self) -> Self { Self { hasher: self.hasher.clone(), out_buf: self.out_buf, out_pos: self.out_pos, + // The window travels with the clone: a fork replays the same stream + // from the same offset, so its absorbs are aligned exactly when the + // original's are. + #[cfg(feature = "hash-metrics")] + window: self.window, + // …and so does the mark. A replay of a stream that is past its + // statement is past it too — the `owed` fork is exactly that. + #[cfg(feature = "hash-metrics")] + counting: self.counting, phantom: PhantomData, } } } -impl DefaultTranscript +impl DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { pub fn new(data: &[u8]) -> Self { let mut res = Self { - hasher: Keccak256::new(), + hasher: T::Digest::new(), out_buf: [0u8; SQUEEZE_LEN], // Empty: the first sample forces a squeeze. out_pos: SQUEEZE_LEN, + #[cfg(feature = "hash-metrics")] + window: 0, + #[cfg(feature = "hash-metrics")] + counting: false, phantom: PhantomData, }; + // The seed goes through `append_bytes`, so a non-empty one advances the + // window like any other absorb — which is why the WHIR byte gate's own + // first window, 13 seed bytes then a root, is NOT aligned. res.append_bytes(data); res } /// Raw squeeze: finalize the current sponge state, advance the hash chain by - /// absorbing the (reversed) output, and return it. Also invalidates the - /// duplex output buffer, so interleaving raw `sample()` calls with buffered - /// field/`u64` sampling can never reuse stale squeeze bytes. + /// absorbing the output, and return it. Also invalidates the duplex output + /// buffer, so interleaving raw `sample()` calls with buffered field/`u64` + /// sampling can never reuse stale squeeze bytes. + /// + /// ★ The byte order is the configuration's, via + /// [`TranscriptHash::REVERSES_SQUEEZE`] — `true` for keccak, which is the + /// convention every proof on this branch has been produced under, and + /// `false` for an algebraic sponge, whose squeeze is already four canonical + /// felts and whose consumer is a field-native verifier that would otherwise + /// spend rows undoing the reversal. + /// + /// ⚠ The returned bytes and the chained bytes are the SAME value, and that + /// is deliberate: a replaying verifier reproducing this chain would + /// otherwise have two byte conventions to carry instead of none. Whichever + /// order the constant selects applies to both. + /// + /// The constant is associated, so each configuration monomorphises to + /// straight-line code — the keccak arm keeps the instruction sequence it + /// had before this became a choice. pub fn sample(&mut self) -> [u8; 32] { + // ★ Hash-agnostic, and deliberately here rather than inside a digest: + // a counter that lives in keccak reads ZERO for an algebraic + // transcript, which is indistinguishable from "no transcript ran". + crate::hash_metrics::count_transcript_squeeze::(); let mut result_hash: [u8; 32] = self.hasher.finalize_reset().into(); - result_hash.reverse(); + if T::REVERSES_SQUEEZE { + result_hash.reverse(); + } self.hasher.update(result_hash); self.out_pos = SQUEEZE_LEN; + // A new window, holding the 32 bytes just re-absorbed. + #[cfg(feature = "hash-metrics")] + { + self.window = SQUEEZE_LEN; + } result_hash } /// Next 64-bit candidate from the duplex output buffer, refilling with one /// squeeze when fewer than 8 bytes remain. Big-endian, matching the byte /// order `sample_u64` used when it read directly from `sample()`. + /// + /// ★ `SQUEEZE_LEN` is 32 and every read is 8, so `out_pos` only ever takes + /// the values `0, 8, 16, 24, 32` and a candidate is always a whole 8-byte + /// group — never two halves of adjacent ones. That is what lets a + /// configuration whose squeeze is four canonical felts promise + /// `CANDIDATES_PER_COORDINATE = Some(1)`: the felt boundaries and the read + /// boundaries are the same boundaries. `append_bytes` invalidates the + /// buffer wholesale rather than partially, so the alignment survives + /// interleaved absorbs. fn next_sample_u64(&mut self) -> u64 { if self.out_pos + 8 > SQUEEZE_LEN { self.out_buf = self.sample(); @@ -93,11 +190,63 @@ where self.out_pos += 8; u64::from_be_bytes(bytes) } + + /// One base coordinate's worth of candidates under a FIXED schedule: draw + /// exactly `n`, hand back the first that `F` would accept. + /// + /// All `n` are drawn whichever one lands in range — that is the entire + /// point. Returning early on the first hit would restore the data-dependent + /// schedule this exists to remove. + /// + /// The value handed back is one `F::sample_field_element_from` accepts, so + /// its own rejection loop exits after a single call and consumption is + /// exactly `n` per coordinate. When every candidate misses (≈ 2⁻³²ⁿ) the + /// last one is returned, `F` rejects it, and the loop draws another `n` — + /// the schedule stays a multiple of `n` and the distribution stays exactly + /// uniform, because nothing is ever reduced into range. + fn next_candidate_fixed(&mut self, n: usize) -> u64 { + candidate_under_fixed_schedule::(n, || self.next_sample_u64()) + } } -impl Default for DefaultTranscript +/// One base coordinate's worth of candidates under a FIXED schedule: pull +/// exactly `n` from `next`, hand back the first that `F` would accept. +/// +/// Free-standing rather than a method so the schedule can be driven by a +/// counting closure in a test — "consumes exactly `n`" is the whole property, +/// and it is not observable from the transcript's outputs. +pub(crate) fn candidate_under_fixed_schedule( + n: usize, + mut next: impl FnMut() -> u64, +) -> u64 { + let mut chosen: Option = None; + let mut last = 0u64; + for _ in 0..n { + let candidate = next(); + last = candidate; + if chosen.is_none() && F::candidate_in_range(candidate) { + chosen = Some(candidate); + } + } + chosen.unwrap_or(last) +} + +/// The BLAKE3 Fiat-Shamir transcript: `Blake3Chain` in the sponge, and rider +/// 1's constant-consumption sampling. +pub type Blake3Transcript = DefaultTranscript; + +impl crate::fiat_shamir::transcript_hash::HasTranscriptHash for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, +{ + type Hash = T; +} + +impl Default for DefaultTranscript +where + F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { fn default() -> Self { @@ -105,9 +254,10 @@ where } } -impl IsTranscript for DefaultTranscript +impl IsTranscript for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { fn append_bytes(&mut self, new_bytes: &[u8]) { @@ -115,24 +265,81 @@ where // subsequent challenge must depend on this input, so drop the bytes // squeezed before it. self.out_pos = SQUEEZE_LEN; + crate::hash_metrics::count_transcript_absorb::(); + #[cfg(feature = "hash-metrics")] + { + if self.counting && !self.window.is_multiple_of(FELT_BYTES) { + crate::hash_metrics::count_transcript_misaligned_absorb(); + } + self.window += new_bytes.len(); + } self.hasher.update(new_bytes); } fn append_field_element(&mut self, element: &FieldElement) { // Absorb, same invalidation as `append_bytes` (the field element's bytes // are streamed straight into the sponge with no intermediate `Vec`). + // + // ⚠ Counted PER `update` rather than once per call, because that is the + // unit `absorb_calls` has always used and the dimension a block- + // absorption change moves. Today the degree-3 extension writes one + // 24-byte buffer and calls the sink once, so the two happen to agree — + // a field or a serialisation that streams in pieces would not, and the + // counter should follow the sponge rather than the argument list. self.out_pos = SQUEEZE_LEN; - element.stream_bytes(&mut |b| self.hasher.update(b)); + // ⚠ Per `update`, the unit the absorb counter already uses and the one + // `WindowRecorder` mirrors: an element streamed in pieces is several + // absorbs in both instruments or in neither. + #[cfg(feature = "hash-metrics")] + let window = &mut self.window; + #[cfg(feature = "hash-metrics")] + let counting = self.counting; + let hasher = &mut self.hasher; + element.stream_bytes(&mut |b| { + crate::hash_metrics::count_transcript_absorb::(); + #[cfg(feature = "hash-metrics")] + { + if counting && !window.is_multiple_of(FELT_BYTES) { + crate::hash_metrics::count_transcript_misaligned_absorb(); + } + *window += b.len(); + } + hasher.update(b); + }); + } + + fn mark_statement_end(&mut self) { + // The window is DELIBERATELY untouched: see the trait's documentation. + // If the statement ended off a boundary the next absorb is counted, and + // that is the whole point of the mark existing at all. + #[cfg(feature = "hash-metrics")] + { + self.counting = true; + } } fn state(&self) -> [u8; 32] { + // ★ Counted, and NOT as a squeeze. This finalizes a CLONE: no reset and + // no re-absorb, so the chain does not advance and a counter hooked to + // `sample` cannot see it. There is one per grind check — 2,996 on a + // block proof against 182,734 squeezes — so a counter that reported + // only their sum could be checked against neither. + crate::hash_metrics::count_transcript_state::(); self.hasher.clone().finalize().into() } fn sample_field_element(&mut self) -> FieldElement { - F::sample_field_element_from(|| self.next_sample_u64()) + match T::CANDIDATES_PER_COORDINATE { + None => F::sample_field_element_from(|| self.next_sample_u64()), + Some(n) => F::sample_field_element_from(|| self.next_candidate_fixed(n.get())), + } } + /// Note this loop is already fixed-consumption where it matters. Its only + /// production caller samples query indices against `domain_size >> 1`, a + /// power of two, and for `upper_bound = 2^k` the threshold is + /// `(-2^k) mod 2^k = 0` — so no candidate is ever rejected. The loop is here + /// for non-power-of-two bounds, which the protocol does not use. fn sample_u64(&mut self, upper_bound: u64) -> u64 { assert!(upper_bound > 0, "upper_bound must be greater than 0"); let threshold = upper_bound.wrapping_neg() % upper_bound; @@ -145,9 +352,10 @@ where } } -impl IsStarkTranscript for DefaultTranscript +impl IsStarkTranscript for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, S: IsField + IsSubFieldOf, { diff --git a/crypto/crypto/src/fiat_shamir/is_transcript.rs b/crypto/crypto/src/fiat_shamir/is_transcript.rs index 316d9a742..dafafc50f 100644 --- a/crypto/crypto/src/fiat_shamir/is_transcript.rs +++ b/crypto/crypto/src/fiat_shamir/is_transcript.rs @@ -9,6 +9,27 @@ pub trait IsTranscript { fn append_field_element(&mut self, element: &FieldElement); /// Appends a bytes to the transcript. fn append_bytes(&mut self, new_bytes: &[u8]); + /// Tells the transcript that a STATEMENT has just ended, so the alignment + /// counter can report what happens after one. + /// + /// ★ A METRICS CONCERN IN A PRODUCTION TRAIT, and it is here rather than + /// beside the counter because only the transcript knows the window. What + /// the padding promises is not "no absorb is ever misaligned" — a statement + /// is variable-length by nature, so its own fields straddle boundaries + /// constantly — but "whatever follows a statement starts aligned", and a + /// transcript cannot tell where a statement ends unless it is told. + /// + /// ⚠ IT MUST NOT TOUCH THE WINDOW. Resetting the offset to zero here would + /// make the next absorb aligned BY CONSTRUCTION, including when the padding + /// is wrong — a check that cannot fail, wearing the name of the one the pad + /// owes. It only starts the counting; + /// `a_statement_that_ends_off_a_boundary_is_still_counted` is what holds + /// that line. + /// + /// The default is a no-op, and without `hash-metrics` the override is one + /// too, so a normal build compiles every call to nothing. + fn mark_statement_end(&mut self) {} + /// Returns a digest of everything absorbed so far (the sponge state). /// /// This binds the absorbed input stream, but it does NOT capture any diff --git a/crypto/crypto/src/fiat_shamir/mod.rs b/crypto/crypto/src/fiat_shamir/mod.rs index a16f61b62..033b3b73d 100644 --- a/crypto/crypto/src/fiat_shamir/mod.rs +++ b/crypto/crypto/src/fiat_shamir/mod.rs @@ -6,3 +6,5 @@ pub mod default_transcript; pub mod is_transcript; +pub mod p1_transcript; +pub mod transcript_hash; diff --git a/crypto/crypto/src/fiat_shamir/p1_transcript.rs b/crypto/crypto/src/fiat_shamir/p1_transcript.rs new file mode 100644 index 000000000..d9860272d --- /dev/null +++ b/crypto/crypto/src/fiat_shamir/p1_transcript.rs @@ -0,0 +1,373 @@ +//! ZisK's Poseidon1 Fiat–Shamir configuration: the transcript sponge, the +//! grinding digest and the [`TranscriptHash`] that names them. +//! +//! The sponge is [`crate::hash::poseidon1_stark::Transcript`] (checked against +//! ZisK's own code); this module fixes the encoding at the transcript's seams: +//! +//! | seam | encoding | +//! |---|---| +//! | `append_bytes(b)` | `len(b)`, then `b` as 8-byte big-endian felts, the last zero-padded | +//! | `append_field_element(x)` | `x`'s three coefficients | +//! | `state()` | lanes 0..4 of the sponge after a flush, big-endian — the grinding seed | +//! | `sample_field_element()` | three squeezed felts | +//! | `sample_u64(n)` | one squeezed felt, canonical, masked to `n − 1` (one felt per draw) | +//! | grinding | one width-8 permutation of the input's 8-byte felts, zero-padded; digest = lanes 0..4 | +//! +//! Grinding keeps the two-level construction of [`crate::grinding`]: the +//! per-nonce hash is `[inner0..3, nonce, 0, 0, 0]` through the width-8 +//! permutation, ZisK's cost (one width-8 permutation per nonce) over our seed +//! layout (ZisK's is `[c0, c1, c2, nonce, 0, 0, 0, 0]`). +//! +//! ⚠ The sponge has no framing between absorbs (`[a]` then `[b]` is `[a, b]`): +//! two transcripts agree only if they make the same calls. Every caller absorbs +//! a schedule fixed by its statement, so prover and verifier stay in step. + +use core::num::NonZeroUsize; + +use alloc::vec::Vec; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsPrimeField; + +use crate::fiat_shamir::is_transcript::{IsStarkTranscript, IsTranscript}; +use crate::fiat_shamir::transcript_hash::{HasTranscriptHash, TranscriptHash}; +use crate::grinding::{DeviceGrindKey, GrindDigest}; +use crate::hash::poseidon1_stark::{self as p1, linear_hash}; +use crate::hash::poseidon1_w8; +use crate::hash::rpx::{BYTES_PER_FELT, Fp, digest_to_commitment, felts_from_bytes}; + +type Ext = FieldElement; + +/// The grinding digest: one width-8 permutation of the absorbed bytes read +/// as 8-byte big-endian felts, zero-padded to eight lanes; the digest is +/// lanes 0..4. Both grinding hashes fit: the inner input is 41 bytes (6 +/// felts), the per-nonce input 40 (5 felts). A longer input, which grinding +/// never produces, takes the width-16 leaf hash so the digest stays total. +#[derive(Clone, Default)] +pub struct P1GrindDigest { + buf: Vec, +} + +impl P1GrindDigest { + /// The digest of everything absorbed so far. + pub fn finalize_digest(&self) -> [u8; 32] { + let felts = felts_from_bytes(&self.buf); + if felts.len() > poseidon1_w8::STATE_FELTS { + return digest_to_commitment(&linear_hash(&felts)); + } + let mut s = [Fp::zero(); poseidon1_w8::STATE_FELTS]; + s[..felts.len()].copy_from_slice(&felts); + let out = poseidon1_w8::permute(s); + digest_to_commitment(&[out[0], out[1], out[2], out[3]]) + } +} + +impl GrindDigest for P1GrindDigest { + /// `math_cuda::grinding::generate_nonce_p1_gpu` (`p1s_grind_w8`): one + /// width-8 permutation of `[inner0..3, nonce, 0, 0, 0]` per nonce, the + /// per-nonce half of [`Self::finalize_digest`] over 40 bytes. + const DEVICE_GRIND: Option = Some(DeviceGrindKey::Poseidon1); +} + +impl digest::HashMarker for P1GrindDigest {} + +impl digest::OutputSizeUser for P1GrindDigest { + type OutputSize = digest::typenum::U32; +} + +impl digest::Update for P1GrindDigest { + fn update(&mut self, data: &[u8]) { + self.buf.extend_from_slice(data); + } +} + +impl digest::FixedOutput for P1GrindDigest { + fn finalize_into(self, out: &mut digest::Output) { + out.copy_from_slice(&self.finalize_digest()); + } +} + +impl digest::Reset for P1GrindDigest { + fn reset(&mut self) { + self.buf.clear(); + } +} + +impl digest::FixedOutputReset for P1GrindDigest { + fn finalize_into_reset(&mut self, out: &mut digest::Output) { + out.copy_from_slice(&self.finalize_digest()); + self.buf.clear(); + } +} + +/// The Fiat–Shamir configuration: what grinding computes over. The challenge +/// stream is [`P1Transcript`], which callers build and hand to the prover. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct P1TranscriptHash; + +impl TranscriptHash for P1TranscriptHash { + type Digest = P1GrindDigest; + const REVERSES_SQUEEZE: bool = false; + /// Structural: a squeeze is canonical felts. + const CANDIDATES_PER_COORDINATE: Option = NonZeroUsize::new(1); + const NAME: &'static str = "poseidon1-w16"; +} + +/// ZisK's transcript sponge under the encodings of the module table. +#[derive(Clone)] +pub struct P1Transcript { + sponge: p1::Transcript, +} + +impl Default for P1Transcript { + fn default() -> Self { + Self::new() + } +} + +impl P1Transcript { + /// An empty transcript. + pub fn new() -> Self { + Self { + sponge: p1::Transcript::new(), + } + } + + /// A fresh transcript with `seed` absorbed as its first `append_bytes`. + pub fn with_seed(seed: &[u8]) -> Self { + let mut t = Self::new(); + >::append_bytes(&mut t, seed); + t + } + + /// The felts `append_bytes(bytes)` absorbs: the length, then the bytes in + /// 8-byte big-endian groups, the last zero-padded. + pub fn append_bytes_felts(bytes: &[u8]) -> Vec { + let mut felts = Vec::with_capacity(1 + bytes.len().div_ceil(BYTES_PER_FELT)); + felts.push(Fp::from(bytes.len() as u64)); + felts.extend(felts_from_bytes(bytes)); + felts + } +} + +impl HasTranscriptHash for P1Transcript { + type Hash = P1TranscriptHash; +} + +impl IsTranscript for P1Transcript { + fn append_field_element(&mut self, element: &Ext) { + self.sponge.put(element.value()); + } + + fn append_bytes(&mut self, new_bytes: &[u8]) { + self.sponge.put(&Self::append_bytes_felts(new_bytes)); + } + + /// Lanes 0..4 after a flush, canonical big-endian. Flushes a copy, so the + /// stream itself is unchanged: prover and verifier read it at the same + /// points and agree. + fn state(&self) -> [u8; 32] { + let s = self.sponge.clone().state(); + digest_to_commitment(&[s[0], s[1], s[2], s[3]]) + } + + fn sample_field_element(&mut self) -> Ext { + Ext::new(self.sponge.field()) + } + + /// Constant consumption: one felt per draw, masked to a power-of-two bound. + /// Any other bound (zero included) is refused in every build: the mask + /// would bias it, and no caller may draw that way (REV-P1W-JUDGE V1). + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + assert!( + upper_bound.is_power_of_two(), + "sample_u64 is masked, so a non-power-of-two bound ({upper_bound}) would be biased" + ); + GoldilocksField::canonical(self.sponge.squeeze1().value()) & (upper_bound.wrapping_sub(1)) + } +} + +impl IsStarkTranscript for P1Transcript {} + +#[cfg(test)] +mod tests { + use super::*; + + fn fe(i: u64) -> Fp { + Fp::from(i.wrapping_mul(0x9E37_79B9_7F4A_7C15)) + } + + fn fee(i: u64) -> Ext { + Ext::new([fe(3 * i), fe(3 * i + 1), fe(3 * i + 2)]) + } + + fn state(t: &P1Transcript) -> [u8; 32] { + >::state(t) + } + + fn sample_u64(t: &mut P1Transcript, bound: u64) -> u64 { + >::sample_u64(t, bound) + } + + #[test] + fn the_grinding_hash_is_one_width_8_permutation_per_nonce() { + let seed = [7u8; 32]; + let factor = 8u8; + let inner = crate::grinding::inner_hash_felts::(&seed, factor); + let nonce = crate::grinding::generate_nonce_smallest::(&seed, factor) + .expect("a nonce at 8 bits"); + assert!(crate::grinding::is_valid_nonce::( + &seed, nonce, factor + )); + // The per-nonce hash, by hand: [inner0..3, nonce, 0, 0, 0] through W8. + let mut s = [Fp::zero(); 8]; + for (lane, v) in s.iter_mut().zip(inner) { + *lane = Fp::from(v); + } + s[4] = Fp::from(nonce); + let lane0 = GoldilocksField::canonical(poseidon1_w8::permute(s)[0].value()); + assert!(lane0 < 1u64 << (64 - factor), "lane 0 meets the factor"); + if nonce > 0 { + assert!(!crate::grinding::is_valid_nonce::( + &seed, + nonce - 1, + factor + )); + } + } + + #[test] + fn the_transcript_binds_lengths_bytes_and_field_elements() { + let first = |f: &dyn Fn(&mut P1Transcript)| { + let mut t = P1Transcript::new(); + f(&mut t); + >::sample_field_element( + &mut t, + ) + }; + let root = [9u8; 32]; + let a = first(&|t| t.append_bytes(&root)); + // A 32-byte root whose tail is zero, against the 8-byte integer it starts with. + let mut short_root = [0u8; 32]; + short_root[..8].copy_from_slice(&root[..8]); + assert_ne!( + first(&|t| t.append_bytes(&short_root)), + first(&|t| t.append_bytes(&root[..8])) + ); + for i in 0..root.len() { + let mut bad = root; + bad[i] ^= 1; + assert_ne!(first(&|t| t.append_bytes(&bad)), a, "byte {i}"); + } + let x = fee(5); + let b = first(&|t| t.append_field_element(&x)); + for k in 0..3 { + let mut v = *x.value(); + v[k] += Fp::one(); + assert_ne!( + first(&|t| t.append_field_element(&Ext::new(v))), + b, + "coefficient {k}" + ); + } + } + + #[test] + fn reading_the_state_does_not_move_the_stream() { + let mut a = P1Transcript::with_seed(b"seed"); + let mut b = P1Transcript::with_seed(b"seed"); + a.append_bytes(&[1, 2, 3]); + b.append_bytes(&[1, 2, 3]); + let s = state(&a); + assert_eq!(s, state(&b)); + for _ in 0..20 { + assert_eq!(sample_u64(&mut a, 1 << 20), sample_u64(&mut b, 1 << 20)); + } + assert_ne!(state(&a), s, "squeezing moves the state"); + } + + #[test] + fn append_bytes_absorbs_the_length_then_the_padded_felts() { + // The encoding, checked against the raw sponge. + let bytes: Vec = (1..=13u8).collect(); + let mut t = P1Transcript::new(); + t.append_bytes(&bytes); + let mut raw = p1::Transcript::new(); + raw.put(&[Fp::from(13u64)]); + raw.put(&felts_from_bytes(&bytes)); + assert_eq!(t.sponge.squeeze1(), raw.squeeze1()); + assert_eq!(P1Transcript::append_bytes_felts(&bytes).len(), 1 + 2); + } + + #[test] + fn a_squeezed_index_stays_below_its_bound() { + let mut t = P1Transcript::with_seed(b"bound"); + for log in [0u32, 1, 5, 20, 63] { + let bound = 1u64 << log; + for _ in 0..8 { + assert!(sample_u64(&mut t, bound) < bound); + } + } + } + + /// The width-8 grind credits its bits (REV-P1-B): `is_valid_nonce` under + /// the P1 digest is exactly "lane 0 of W8(inner ‖ nonce ‖ 0³), canonical, + /// below 2^(64 − g)", the inner digest binds the seed and the factor, and + /// the pass rate over nonces is 2^−g. + #[test] + fn the_width8_grind_credits_its_bits() { + use crate::grinding::is_valid_nonce; + use crate::hash::rpx::commitment_to_digest; + use digest::Digest; + + let seed: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(37) ^ 0x5a); + let g = 6u8; + // The definition, recomputed from the permutation. + let inner: [u8; 32] = { + let mut d = P1GrindDigest::new(); + Digest::update(&mut d, [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xed]); + Digest::update(&mut d, seed); + Digest::update(&mut d, [g]); + d.finalize().into() + }; + let lane0 = |nonce: u64| { + let i = commitment_to_digest(&inner); + let mut s = [Fp::zero(); poseidon1_w8::STATE_FELTS]; + s[..4].copy_from_slice(&i); + s[4] = Fp::from(nonce); + GoldilocksField::canonical(poseidon1_w8::permute(s)[0].value()) + }; + let n = 1u64 << 14; + let mut passes = 0u64; + for nonce in 0..n { + let ok = is_valid_nonce::(&seed, nonce, g); + assert_eq!(ok, lane0(nonce) < 1u64 << (64 - g), "nonce {nonce}"); + passes += u64::from(ok); + } + // Binomial(2^14, 2^-6): mean 256, sd ≈ 15.9; ±6 sd. + assert!((160..=352).contains(&passes), "{passes} passes of {n}"); + // The inner digest binds the seed: one flipped seed bit moves the + // passing set. + let mut other = seed; + other[31] ^= 1; + assert!( + (0..1024u64).any(|k| is_valid_nonce::(&other, k, g) + != is_valid_nonce::(&seed, k, g)), + "the seed moves the passing set" + ); + } + + #[test] + fn the_configuration_names_its_grind_digest() { + assert_eq!( + ::NAME, + "poseidon1-w16", + "the name P1 artifacts carry" + ); + assert!(matches!( + ::DEVICE_GRIND, + Some(DeviceGrindKey::Poseidon1) + )); + } +} diff --git a/crypto/crypto/src/fiat_shamir/transcript_hash.rs b/crypto/crypto/src/fiat_shamir/transcript_hash.rs new file mode 100644 index 000000000..2334550d2 --- /dev/null +++ b/crypto/crypto/src/fiat_shamir/transcript_hash.rs @@ -0,0 +1,213 @@ +//! The hash a Fiat-Shamir transcript runs on. +//! +//! [`DefaultTranscript`](super::default_transcript::DefaultTranscript) is a thin +//! `digest::Digest` wrapper, so swapping the hash is a type substitution. What +//! this trait adds beyond naming a digest is that the name travels with the +//! proof: a transcript is part of what a replaying verifier has to reproduce, +//! so the configuration has to be something a call site can state rather than +//! something a default decides. +//! +//! ★ **The two constants, and why they arrived together.** A straight-line +//! machine replaying the transcript needs a draw count that does not depend on +//! the data, which is what [`TranscriptHash::CANDIDATES_PER_COORDINATE`] states. +//! For an algebraic sponge the count is one, because a squeeze is already field +//! elements — but only if the sampler sees them as the sponge produced them. +//! +//! An earlier revision of this file argued the opposite, and the argument was +//! correct at the time: [`DefaultTranscript::sample`] reversed all 32 bytes of +//! every squeeze, so the first eight bytes a sampler read were the LAST felt's +//! canonical bytes backwards — a number with no canonicality property, for +//! which a one-candidate schedule would have been a claim no test could reach. +//! [`TranscriptHash::REVERSES_SQUEEZE`] is what removed that obstacle, so the +//! two constants are one change: the schedule is a consequence of the byte +//! order, not an independent decision. + +use core::num::NonZeroUsize; +use digest::{Digest, FixedOutputReset, OutputSizeUser, typenum::U32}; + +use crate::hash::blake3::chain::Blake3Chain; +use crate::hash::platform_keccak::PlatformKeccak256; +use crate::hash::rpx::Rpx256Digest; + +/// ★★ Which [`TranscriptHash`] a concrete transcript type is running on. +/// +/// The type-level answer to "what hash is this transcript", so a caller that +/// needs a transcript to match something else can say so in a `where` clause +/// and have the compiler check it. +/// +/// # Why this exists +/// +/// `DefaultTranscript`'s hash parameter has a default, so `DefaultTranscript::` +/// is a keccak transcript and looks like it names no hash at all. Every WHIR +/// call site wrote exactly that, under a dispatch that selects the hash for the +/// Merkle backend and the grind — so the RPX configuration ran an RPX backend, +/// an RPX grind and a KECCAK transcript, for four measured A/Bs, without one +/// instrument disagreeing. Nothing failed: the proofs were valid and the two +/// arms genuinely differed. +/// +/// Naming the hash at every call site would not have prevented it — a site can +/// name the wrong one as easily as it can take a default. What prevents it is +/// an equality the compiler checks, which is what this trait makes sayable: +/// `T: HasTranscriptHash::Transcript>` on the WHIR entry +/// points turns a mismatched transcript into a build error, and leaves the +/// several hundred STARK call sites — for which keccak is not a default but the +/// answer — untouched. +pub trait HasTranscriptHash { + /// The configuration this transcript's sponge runs on. + type Hash: TranscriptHash; +} + +/// One Fiat-Shamir configuration: the digest the sponge runs on. +pub trait TranscriptHash: 'static { + /// The sponge's hash. + /// + /// `Clone` because the transcript is snapshotted (the GPU FRI fallback + /// restores it) and because `state()` finalizes a clone. `FixedOutputReset` + /// because the squeeze is `finalize_reset`. The 32-byte output size is + /// pinned rather than left associated: `state()` returns `[u8; 32]`, and + /// that is what seeds grinding, so a configuration with a different digest + /// width would not be a drop-in anywhere it is consumed. `'static` because + /// the GPU grinding dispatch keys the device search on the concrete digest + /// by `TypeId`, the way the Merkle backends' keccak fast paths do. + /// ⚠ `GrindDigest` is part of the bound, not an afterthought. The + /// proof-of-work hash IS this digest, and which device arm it takes has to + /// travel with it: a configuration that could not say would fall to the + /// host search silently, which is worth thousands of seconds a block and + /// fails nothing. Requiring it here is what makes "a configuration with no + /// declared arm" unconstructible rather than merely unlikely. + type Digest: Digest + + FixedOutputReset + + OutputSizeUser + + Clone + + crate::grinding::GrindDigest + + 'static; + + /// Name for KATs, banners and diagnostics. + const NAME: &'static str; + + /// ★ Whether [`DefaultTranscript::sample`] reverses the 32 bytes of a + /// squeeze before handing them out and chaining them back in. + /// + /// A byte convention, not a security parameter: reversing 32 bytes is a + /// bijection, so the distribution a sampler draws from is the digest's + /// either way. What it decides is *which* bijection sits between the digest + /// and the sampler, and for an algebraic sponge that is the whole question + /// — see [`CANDIDATES_PER_COORDINATE`](Self::CANDIDATES_PER_COORDINATE). + /// + /// ⚠ It is `true` for keccak because that is the convention this system's + /// proofs have always been produced under, and moving it would change every + /// keccak proof on this branch for no gain. It is not `true` for any reason + /// a new configuration should copy. + const REVERSES_SQUEEZE: bool; + + /// ★ How many `u64` candidates a base-field coordinate needs, when that is + /// a fixed number. + /// + /// `None` means unbounded: the sampler rejects candidates `>= p` and draws + /// again, however many times that takes. That is the honest description of + /// a byte-oriented hash, whose squeeze is 32 uniform bytes with no relation + /// to the field — there is no bound, only a probability. + /// + /// `Some(n)` is a promise that `n` candidates always suffice, and it exists + /// for a replaying verifier that cannot branch on how many it needed. A + /// configuration may only claim it if the claim is structural. `Some(1)` + /// here means a squeeze IS field elements, canonically encoded, so the + /// rejection test is unreachable rather than merely unlikely. + /// + /// ⚠ Scope: this governs `sample_field_element` alone. Query indices come + /// from `sample_u64`, whose rejection tests `candidate >= 2^64 mod bound` — + /// a different test, about which canonicality says nothing. Those draws are + /// single-candidate on this branch for an unrelated reason (every WHIR + /// query bound is a power of two, making that threshold zero), which is + /// hash-independent and pinned separately. Two facts, two reasons; a + /// verifier that needs both must not take this constant as evidence of the + /// other. + /// ⚠ `NonZeroUsize`, not `usize`: `Some(0)` would be a schedule that draws + /// no candidates and returns an uninitialised one, and the representation + /// that cannot spell it is better than the check that rejects it. + const CANDIDATES_PER_COORDINATE: Option; +} + +/// The keccak-256 configuration — what every `DefaultTranscript` is unless a +/// caller says otherwise, and byte-for-byte the transcript this system has +/// always produced. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct KeccakTranscriptHash; + +impl TranscriptHash for KeccakTranscriptHash { + type Digest = PlatformKeccak256; + + const NAME: &'static str = "keccak256"; + + /// The convention every keccak proof on this branch was produced under. + const REVERSES_SQUEEZE: bool = true; + + /// Unbounded, and it has to be: a keccak squeeze is 32 uniform bytes, so a + /// candidate lands in `[p, 2^64)` with probability about `2^-32` per draw + /// and the number of draws has no ceiling. Small is not fixed. + const CANDIDATES_PER_COORDINATE: Option = None; +} + +/// The RPX256 configuration — the algebraic sponge, for a transcript a +/// field-native verifier has to replay. +/// +/// ★ **Why this one does not reverse, and what that buys.** A squeeze here is +/// `digest_to_commitment(sponge_leaf_bytes(..))` — four canonical felts, each +/// eight big-endian bytes. Handed out in that order, every 8-byte group a +/// sampler reads is a field element by construction, so the rejection test is +/// unreachable and one candidate per coordinate is exact rather than typical. +/// Reversed, the first group is the LAST felt's bytes backwards, a number with +/// no canonicality property at all — which is why an earlier revision of this +/// file argued `Some(1)` could not be claimed. It could not, then. +/// +/// The reversal had no security role to lose: it is a bijection on 32 bytes, so +/// challenges are the digest's distribution before and after. What it cost was +/// the LFM replay — four byte-reversals and four modular reductions per squeeze +/// that a field-native verifier has to pay in rows to undo an encoding the host +/// had no reason to apply. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct RpxTranscriptHash; + +impl TranscriptHash for RpxTranscriptHash { + type Digest = Rpx256Digest; + + const NAME: &'static str = "rpx256"; + + const REVERSES_SQUEEZE: bool = false; + + /// Structural, not probabilistic: see the type's documentation. + const CANDIDATES_PER_COORDINATE: Option = NonZeroUsize::new(1); +} + +/// The BLAKE3 configuration — `Blake3Chain` over the same sponge, with rider +/// 1's constant-consumption sampling adopted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Blake3TranscriptHash; + +impl TranscriptHash for Blake3TranscriptHash { + type Digest = Blake3Chain; + + const NAME: &'static str = "blake3-chain"; + + /// ⚠ **`true`, and it is not a preference.** Every BLAKE3 proof this + /// repository has produced came from a `sample()` that reversed + /// unconditionally. The reversal became a per-configuration choice so the + /// WHIR RPX transcript could drop it; dropping it anywhere else would move + /// the challenges of a pipeline that is not changing, which is the one + /// thing this merge must not do. + const REVERSES_SQUEEZE: bool = true; + + /// Two candidates per coordinate. + /// + /// One would be free — it is what the current schedule costs in the modal + /// case — but a single candidate that misses has nowhere to go, so the tail + /// would sit at ≈ 2⁻³² per coordinate, i.e. once in a few hundred thousand + /// proofs at production draw counts. That is not negligible enough to call + /// the schedule fixed. Two puts the tail at ≈ 2⁻⁶⁴ per coordinate, for one + /// extra candidate per coordinate — see the cost note in PA-PLAN §2.3. + /// + /// ⚠ Unlike the RPX configuration's `Some(1)`, this is probabilistic: a + /// BLAKE3 squeeze is uniform bytes, so both candidates can miss. Two makes + /// the tail negligible; it does not make it unreachable. + const CANDIDATES_PER_COORDINATE: Option = NonZeroUsize::new(2); +} diff --git a/crypto/crypto/src/grinding.rs b/crypto/crypto/src/grinding.rs new file mode 100644 index 000000000..14a928278 --- /dev/null +++ b/crypto/crypto/src/grinding.rs @@ -0,0 +1,459 @@ +//! Proof of work over a transcript state: the prover finds a nonce whose hash +//! has `grinding_factor` leading zeros, and absorbs it before the next +//! challenge is drawn. +//! +//! Retrying that challenge then costs `2^grinding_factor` hashes, which is what +//! lets a query count buy more soundness than its own bits. +//! +//! Lives here rather than in a proof-system crate because both the univariate +//! prover and the multilinear one grind against the same primitive, and so does +//! the device dispatch below: `multilinear` cannot reach `stark`, which depends +//! on it. +//! +//! # The hash is a parameter, with NO default +//! +//! The construction is two hashes of one block each — 41 bytes inner, 40 bytes +//! outer — so it costs two compressions whichever hash `D` is, and the seed and +//! the digest are `[u8; 32]` on both sides. Swapping the hash is therefore a +//! type substitution that changes the shape of nothing. +//! +//! `D` is deliberately a parameter rather than a defaulted one: the +//! proof-of-work hash has to be the proof's hash, and a defaulted `D` would +//! silently keep grinding on keccak for a configuration that had moved +//! everything else — self-consistent between prover and verifier, and therefore +//! silent. Every call site states its hash. + +use digest::{Digest, OutputSizeUser, typenum::U32}; +#[cfg(feature = "parallel")] +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; + +const PREFIX: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xed]; + +/// Checks if the bit-string `Hash(Hash(prefix || seed || grinding_factor) || nonce)` +/// has at least `grinding_factor` zeros to the left. +/// `prefix` is the bit-string `0x123456789abcded` +/// +/// # Parameters +/// +/// * `seed`: the input seed, +/// * `nonce`: the value to be tested, +/// * `grinding_factor`: the number of leading zeros needed; must be in `1..=64`. +/// +/// # Returns +/// +/// `true` if the number of leading zeros is at least `grinding_factor`, and `false` otherwise. +pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool +where + D: Digest + OutputSizeUser + 'static, +{ + debug_assert!( + (1..=64).contains(&grinding_factor), + "grinding_factor must be in 1..=64, got {grinding_factor}" + ); + let inner_hash = get_inner_hash::(seed, grinding_factor); + let limit = 1 << (64 - grinding_factor); + is_valid_nonce_for_inner_hash::(&inner_hash, nonce, limit) +} + +/// ⚠ **A ground proof is NOT byte-reproducible, and the reason is worse than +/// it looks.** +/// +/// [`generate_nonce`] returns *a* valid nonce, not *the* valid nonce: under +/// `parallel` it is rayon's `find_any`, which hands back whichever worker +/// finished first, and on the device arm it is whatever the kernel's scan +/// reached. That much is a known property. What makes it load-bearing for any +/// byte gate is what happens next: **the nonce is absorbed into the +/// transcript** (`multilinear::whir_chain::grind`, and the univariate prover +/// likewise), so every challenge drawn after the first grind depends on which +/// valid nonce the search happened to return. Two honest runs therefore differ +/// in every Merkle root, every out-of-domain value and every opening from the +/// first grind onward — and **no post-hoc normalisation of the nonce fields can +/// recover the agreement**, because the divergence is not in the nonce fields. +/// +/// [`deterministic`] is the escape hatch a byte gate needs: with +/// `LAMBDA_VM_DETERMINISTIC_GRIND` set, the search returns the SMALLEST valid +/// nonce, which is a function of the seed alone, so the whole proof becomes +/// reproducible. It is off by default and changes nothing about validity — the +/// verifier accepts any nonce passing [`is_valid_nonce`] — only about which of +/// them is chosen. Prove time rises: the smallest-first search cannot stop at +/// the first hit any worker finds. +/// +/// Read once, cached, presence-based — the convention `LAMBDA_VM_NO_GPU_GRIND` +/// uses. +#[cfg(feature = "std")] +pub fn deterministic() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("LAMBDA_VM_DETERMINISTIC_GRIND").is_some()) +} + +#[cfg(not(feature = "std"))] +pub fn deterministic() -> bool { + false +} + +/// Performs grinding, returning a new nonce for the proof. +/// The nonce generated is such that: +/// Hash(Hash(prefix || seed || grinding_factor) || nonce) has at least `grinding_factor` zeros +/// to the left. +/// `prefix` is the bit-string `0x123456789abcded` +/// +/// Which valid nonce comes back is NOT a contract — see [`deterministic`] for +/// the one case where it is, and for why that matters to a byte gate. +/// +/// # Parameters +/// +/// * `seed`: the input seed, +/// * `grinding_factor`: the number of leading zeros needed; must be in `1..=64`. +/// +/// # Returns +/// +/// A `nonce` satisfying the required condition. +pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: Digest + OutputSizeUser + 'static, +{ + debug_assert!( + (1..=64).contains(&grinding_factor), + "grinding_factor must be in 1..=64, got {grinding_factor}" + ); + if deterministic() { + return generate_nonce_smallest::(seed, grinding_factor); + } + let inner_hash = get_inner_hash::(seed, grinding_factor); + let limit = 1 << (64 - grinding_factor); + + #[cfg(not(feature = "parallel"))] + return (0..u64::MAX).find(|&candidate_nonce| { + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) + }); + + #[cfg(feature = "parallel")] + return (0..u64::MAX).into_par_iter().find_any(|&candidate_nonce| { + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) + }); +} + +/// ★ The SMALLEST valid nonce — a function of the seed and the factor alone, +/// and therefore the thing a byte gate can reproduce. +/// +/// Exposed as its own entry point rather than reachable only through the +/// environment, so the property "this is reproducible, and it is genuinely the +/// smallest" is testable without a process-global switch. `find_first` rather +/// than `find_any`: rayon prunes candidates above the best hit so far, so the +/// cost is bounded by the smallest hit's index rather than by the whole range, +/// but it cannot stop as early as `find_any` and that is the price of the +/// property. +pub fn generate_nonce_smallest(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: Digest + OutputSizeUser + 'static, +{ + debug_assert!( + (1..=64).contains(&grinding_factor), + "grinding_factor must be in 1..=64, got {grinding_factor}" + ); + let inner_hash = get_inner_hash::(seed, grinding_factor); + let limit = 1 << (64 - grinding_factor); + + #[cfg(not(feature = "parallel"))] + return (0..u64::MAX).find(|&candidate_nonce| { + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) + }); + + #[cfg(feature = "parallel")] + return (0..u64::MAX) + .into_par_iter() + .find_first(|&candidate_nonce| { + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) + }); +} + +/// Successful KECCAK GPU grind dispatches — one per nonce search that ran on +/// device and produced a nonce the host check accepted (a device miss or an +/// invalid kernel result falls back to the CPU search and is not counted). +#[cfg(feature = "cuda")] +static GPU_GRIND_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// ★ The same for RPX, counted SEPARATELY. +/// +/// Two counters rather than one, because the question an assertion needs to +/// answer is not "did a grind reach the device" but "did the RIGHT kernel run". +/// A single counter is satisfied by the keccak arm firing under an RPX +/// configuration — which is the precise failure this dispatch exists to make +/// impossible, so it must not also be the failure the test cannot see. +#[cfg(feature = "cuda")] +static GPU_GRIND_CALLS_RPX: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +#[cfg(feature = "cuda")] +pub fn gpu_grind_calls() -> u64 { + GPU_GRIND_CALLS.load(core::sync::atomic::Ordering::Relaxed) +} + +/// Successful RPX device grinds. Zero under a keccak configuration. +#[cfg(feature = "cuda")] +pub fn gpu_grind_calls_rpx() -> u64 { + GPU_GRIND_CALLS_RPX.load(core::sync::atomic::Ordering::Relaxed) +} + +/// The same for ZisK's Poseidon1 (`p1/*` exploration branch), counted apart +/// for the reason the RPX counter is. +#[cfg(feature = "cuda")] +static GPU_GRIND_CALLS_P1: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// Successful Poseidon1 device grinds. +#[cfg(feature = "cuda")] +pub fn gpu_grind_calls_p1() -> u64 { + GPU_GRIND_CALLS_P1.load(core::sync::atomic::Ordering::Relaxed) +} + +/// Zeroes BOTH counters — a measuring caller resets once and reads both, so an +/// arm cannot inherit the previous arm's count. +#[cfg(feature = "cuda")] +pub fn reset_gpu_grind_calls() { + GPU_GRIND_CALLS.store(0, core::sync::atomic::Ordering::Relaxed); + GPU_GRIND_CALLS_RPX.store(0, core::sync::atomic::Ordering::Relaxed); + GPU_GRIND_CALLS_P1.store(0, core::sync::atomic::Ordering::Relaxed); +} + +/// Which device grind kernel a digest takes, if it has one. +/// +/// The two arms read the SAME 32-byte inner hash in different byte orders — +/// keccak's kernel as four LITTLE-endian lanes, RPX's as four BIG-endian felts +/// — so this names the arm AND the encoding together. Crossing them compiles, +/// runs, and searches for a nonce under a message the host never hashes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeviceGrindKey { + /// `math_cuda::grinding::generate_nonce_gpu`, little-endian lanes. + Keccak256, + /// `math_cuda::grinding::generate_nonce_rpx_gpu`, big-endian felts. + Rpx256, + /// `math_cuda::grinding::generate_nonce_p1_gpu`, big-endian felts (ZisK's + /// width-8 Poseidon1, `p1/*` exploration branch). + Poseidon1, +} + +/// ★ A grinding digest, and the device arm it takes — stated BY THE DIGEST. +/// +/// ⚠ **This replaced a central list, and the reason is the merge that created +/// this file.** The dispatch used to be `has_device_kernel::()`, a `TypeId` +/// comparison against the two digests `crypto` happens to define. That is +/// correct exactly while every grinding hash lives here. It stopped being true +/// the moment the per-table branch merged in: its block path grinds with +/// `prover::lfm::algebraic_commit::AlgebraicDigest`, which `crypto` +/// cannot name at all — `prover` depends on this crate, not the reverse — so +/// the list returned false, the RPX grind fell to the host rayon search, and +/// nothing failed. It compiles, the proofs stay valid, and the only signal is a +/// counter that does not increment. +/// +/// As an associated constant the arm travels with the configuration instead: +/// there is no list to be missing from, and +/// [`TranscriptHash::Digest`](crate::fiat_shamir::transcript_hash::TranscriptHash::Digest) +/// requires this bound, so a configuration that does not declare an arm does +/// not compile rather than silently taking the slow one. +/// +/// `None` is a statement too — BLAKE3 has no grind kernel, and the CPU search +/// is where it belongs. +pub trait GrindDigest: Digest + OutputSizeUser + 'static { + /// The device arm, or `None` for "the host search is the only one". + const DEVICE_GRIND: Option; +} + +impl GrindDigest for crate::hash::platform_keccak::PlatformKeccak256 { + const DEVICE_GRIND: Option = Some(DeviceGrindKey::Keccak256); +} + +impl GrindDigest for crate::hash::rpx::Rpx256Digest { + const DEVICE_GRIND: Option = Some(DeviceGrindKey::Rpx256); +} + +impl GrindDigest for crate::hash::blake3::chain::Blake3Chain { + /// No BLAKE3 grind kernel exists. `math-cuda` ships BLAKE3 leaf and parent + /// kernels, but not a nonce search, so this is `None` rather than an + /// omission. + const DEVICE_GRIND: Option = None; +} + +/// Grind on the GPU when a CUDA backend is up, falling back to the CPU search +/// otherwise (or on any device error). Which valid nonce comes back depends on +/// the arm: the device search returns the smallest in the range it scanned, +/// while the CPU's `find_any` returns an arbitrary one. Neither is a contract — +/// the verifier accepts any nonce passing [`is_valid_nonce`], and nothing +/// downstream depends on the choice. +/// +/// ★ The dispatch is keyed on [`GrindDigest::DEVICE_GRIND`] — the arm the +/// digest's own configuration names, the host twin of the `DeviceHash` key the +/// commit path carries. +/// +/// Two arms, and the endianness differs between them: keccak's kernel takes the +/// inner hash as four LITTLE-endian lanes, RPX's as four BIG-endian felts. A +/// hash with no kernel takes the CPU search and is correct there, rather than +/// being handed a nonce another hash's kernel found. +/// +/// ⚠ **This guard is load-bearing on the measurement, not only on correctness.** +/// While it read "is `D` keccak", an RPX configuration fell to the host search: +/// ~2^20 RPX permutations per grind, thousands of grinds per block proof. A +/// measured WHIR block arm came in at 571 s against keccak's 39 s, and ~510 s of +/// that was this line — the device idle, 31 host threads at 90%, with a +/// correct, KAT-pinned `rpx_grind_search` sitting in the cubin unused. +#[cfg(feature = "cuda")] +pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: GrindDigest, +{ + debug_assert!( + (1..=64).contains(&grinding_factor), + "grinding_factor must be in 1..=64, got {grinding_factor}" + ); + // Kill switch (presence-based, matching `LAMBDA_VM_NO_GPU_LOGUP`): + // `LAMBDA_VM_NO_GPU_GRIND` forces the CPU search — a production escape hatch + // and fallback-path coverage. Cached; read once. + static GPU_DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *GPU_DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_GRIND").is_some()) { + return generate_nonce::(seed, grinding_factor); + } + // The device search returns the smallest nonce in the range IT scanned, + // which is not the same promise as the smallest that exists. Under the + // deterministic knob the host search is the only one that makes the + // promise, so the device steps aside rather than being trusted to keep it. + if deterministic() { + return generate_nonce::(seed, grinding_factor); + } + // A hash with no device kernel takes the CPU search. + let Some(key) = D::DEVICE_GRIND else { + return generate_nonce::(seed, grinding_factor); + }; + + // Each arm reads the SAME 32 bytes in its own byte order — see + // `math_cuda::grinding`'s header for what crossing them does. + let found = match key { + DeviceGrindKey::Keccak256 => math_cuda::grinding::generate_nonce_gpu( + &inner_hash_lanes::(seed, grinding_factor), + grinding_factor, + ), + DeviceGrindKey::Rpx256 => math_cuda::grinding::generate_nonce_rpx_gpu( + &inner_hash_felts::(seed, grinding_factor), + grinding_factor, + ), + DeviceGrindKey::Poseidon1 => math_cuda::grinding::generate_nonce_p1_gpu( + &inner_hash_felts::(seed, grinding_factor), + grinding_factor, + ), + }; + + if let Some(nonce) = found { + // Validate unconditionally (one host hash against the ~2^grinding_factor + // device search): a kernel/driver defect must degrade to the CPU search, + // never append an unverifiable nonce to the transcript. This is also + // what would catch the two byte orders being crossed — the nonce would + // be valid under a message the host never hashed. + if is_valid_nonce::(seed, nonce, grinding_factor) { + match key { + DeviceGrindKey::Keccak256 => { + GPU_GRIND_CALLS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + } + DeviceGrindKey::Rpx256 => { + GPU_GRIND_CALLS_RPX.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + } + DeviceGrindKey::Poseidon1 => { + GPU_GRIND_CALLS_P1.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + } + }; + return Some(nonce); + } + // eprintln, not log::warn: the CLI initialises env_logger with no + // default filter, so a warn-level line is invisible unless RUST_LOG is + // set — and this is the only signal that the kernel has started + // returning garbage and the feature has silently reverted to the CPU + // search. + eprintln!( + "[gpu] grind returned an invalid nonce ({nonce}); falling back to the CPU search" + ); + } + generate_nonce::(seed, grinding_factor) +} + +#[cfg(not(feature = "cuda"))] +pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: GrindDigest, +{ + generate_nonce::(seed, grinding_factor) +} + +/// Checks if the leftmost 8 bytes of `Hash(inner_hash || candidate_nonce)` are less than `limit` +/// when interpreted as `u64`. +#[inline(always)] +fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool +where + D: Digest + OutputSizeUser + 'static, +{ + // Tag this finalize as grinding so a verify-hash metric can report it apart + // (see `crate::hash_metrics`); no-op unless the `hash-metrics` feature is on. + // The metric is keccak-only by construction, so a non-keccak `D` leaves it + // at zero rather than reporting another hash's work as keccak's. + crate::hash_metrics::count_grinding(); + let mut data = [0; 40]; + data[..32].copy_from_slice(inner_hash); + data[32..].copy_from_slice(&candidate_nonce.to_be_bytes()); + + let digest = D::digest(data); + + let seed_head = u64::from_be_bytes(digest[..8].try_into().unwrap()); + seed_head < limit +} + +/// Returns the bit-string constructed as +/// Hash(prefix || seed || grinding_factor) +/// `prefix` is the bit-string `0x123456789abcded` +fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] +where + D: Digest + OutputSizeUser + 'static, +{ + // Grinding finalize (see `crate::hash_metrics`); no-op unless enabled. + crate::hash_metrics::count_grinding(); + let mut inner_data = [0u8; 41]; + inner_data[0..8].copy_from_slice(&PREFIX); + inner_data[8..40].copy_from_slice(seed); + inner_data[40] = grinding_factor; + + let digest = D::digest(inner_data); + digest[..32].try_into().unwrap() +} + +/// The inner hash as the four little-endian u64 lanes Keccak absorbs it into — +/// the form the device nonce search takes as input. +/// +/// The GPU dispatch and its test both go through here rather than each doing +/// their own byte-to-lane conversion: a second copy would let this one drift +/// (`from_le_bytes` → `from_be_bytes` reads identically at a glance) with every +/// test still green, while at runtime `is_valid_nonce` rejected every device +/// nonce and the search silently sat on the CPU fallback forever. +pub fn inner_hash_lanes(seed: &[u8; 32], grinding_factor: u8) -> [u64; 4] +where + D: Digest + OutputSizeUser + 'static, +{ + let inner_hash = get_inner_hash::(seed, grinding_factor); + core::array::from_fn(|i| u64::from_le_bytes(inner_hash[i * 8..i * 8 + 8].try_into().unwrap())) +} + +/// ★ The inner hash as the four BIG-endian `u64`s an ALGEBRAIC sponge absorbs — +/// the form the RPX device search takes as input. +/// +/// ⚠ The endianness is the whole difference from [`inner_hash_lanes`], and it is +/// not cosmetic. `felts_from_bytes` reads consecutive eight-byte groups +/// big-endian, so these four `u64`s ARE the felts the host sponge absorbs; +/// keccak reads its lanes little-endian. Crossing the two compiles and runs, and +/// produces a device search for a nonce under a message the host never hashes — +/// every returned nonce rejected, the fallback taken on every grind, and nothing +/// louder than one warning line to say so. That is why there are two named +/// functions and not one with a flag. +/// +/// The four values are already canonical: the inner hash is an algebraic +/// digest's own output, which `digest_to_commitment` writes as four canonical +/// big-endian `u64`s. Nothing here reduces them, and the device does not either. +pub fn inner_hash_felts(seed: &[u8; 32], grinding_factor: u8) -> [u64; 4] +where + D: Digest + OutputSizeUser + 'static, +{ + let inner_hash = get_inner_hash::(seed, grinding_factor); + core::array::from_fn(|i| u64::from_be_bytes(inner_hash[i * 8..i * 8 + 8].try_into().unwrap())) +} diff --git a/crypto/crypto/src/hash/blake3/chain.rs b/crypto/crypto/src/hash/blake3/chain.rs new file mode 100644 index 000000000..de05ea9e3 --- /dev/null +++ b/crypto/crypto/src/hash/blake3/chain.rs @@ -0,0 +1,1239 @@ +//! [`Blake3Chain`] — the byte hash the RV64 prover's commitments are built from. +//! +//! **Specified in `thoughts/shared/block-compression/PA-PLAN.md` §1.7, which is +//! the normative text; this is the implementation of it.** The construction is +//! DRAFT: it is the working default by standing decision, and formally awaits +//! ratification. §1.7.3 lists the forks that are still open. +//! +//! # The construction, in one sentence +//! +//! **Standard BLAKE3 restricted to a single chunk that never ends.** The message +//! is split into 64-byte blocks, the last zero-padded; the chaining value starts +//! at [`BLAKE3_IV`] and each block compresses it forward with `t = 0`; the first +//! block carries `CHUNK_START`, the last carries `CHUNK_END | ROOT` and the true +//! byte count as its `block_len`. The digest is the low 8 output words, +//! little-endian. +//! +//! # Why this shape +//! +//! It is chosen so that two other things are true by construction rather than by +//! agreement, which is the whole reason to prefer it to a bare chain with one +//! flag constant: +//! +//! - **For any message of at most 1024 bytes, at 7 rounds, this IS +//! `blake3::hash`.** Standard BLAKE3's first chunk is exactly this chain, and a +//! message of at most one chunk has that chunk's output as its root — so `ROOT` +//! lands on the same compression. The official crate is therefore a direct +//! known-answer test for the *framing*, not merely for the round function, over +//! the entire range that matters: leaves, FRI pairs and parents are all far +//! inside it. `seven_round_chain_is_the_blake3_crate` is that test. +//! - **A 64-byte message is exactly a Merkle parent.** One block, first and last, +//! so `flags = 0x0B`, `block_len = 64`, `h = IV`, `t = 0` — which is what +//! `hash_new_parent` compresses and what the device kernel +//! (`math-cuda/kernels/blake3.cu`) implements. So the `StarkHash` invariant +//! that `Batched::hash_data(&vec![a, b]) == Pair::hash_data(&[a, b])` holds +//! because both are the same 64 bytes through this one function. +//! +//! Above 1024 bytes it deliberately leaves the standard: BLAKE3 would start a +//! second chunk (`t = 1`, chaining value reset to `IV`) and build a tree over +//! chunk chaining values. Keeping one unbounded chunk costs nothing at 6 rounds, +//! where no external verifier exists in any case, and saves both a chunk-tree +//! state machine in every CUDA kernel and the same state machine again in the +//! wrap's eDSL emitter. `past_one_chunk_leaves_the_blake3_crate` pins that the +//! divergence is real, so the claim is falsifiable rather than decorative. + +use digest::{FixedOutput, FixedOutputReset, HashMarker, Output, OutputSizeUser, Reset, Update}; + +use super::{BLAKE3_IV, BLAKE3_ROUNDS, blake3_compress_rounds}; + +/// Bytes in one BLAKE3 message block. +pub const BLOCK_LEN: usize = 64; + +/// Dwords in the accelerator's state region: `h[8] | m[16] | t | +/// (block_len, flags) | out[16]`, two little-endian `u32` words per dword. +/// Mirrors `BLAKE3_STATE_BYTES / 8` in +/// `executor::vm::instruction::execution`. +pub const SYSCALL_STATE_DWORDS: usize = 22; + +/// First dword of the output region — the accelerator reads dwords `0..14` and +/// writes `out[0..16]` into `14..22`. Mirrors the executor's +/// `BLAKE3_OUT_DWORDS`. +pub const SYSCALL_OUT_DWORD: usize = 14; + +/// This block begins the chunk. Set on the first block only. +const CHUNK_START: u32 = 1; +/// This block ends the chunk. Set on the last block only. +const CHUNK_END: u32 = 2; +/// This compression produces the root output. Set on the last block only — +/// a single-chunk message's chunk output *is* its root output. +const ROOT: u32 = 8; + +/// The flags of a message that is one block long: first and last at once. Equal +/// to `CHUNK_START | CHUNK_END | ROOT`, and the framing every Merkle parent uses. +pub const FLAGS_ONE_BLOCK: u32 = CHUNK_START | CHUNK_END | ROOT; + +// ------------------------------------------------------------------------- +// The framing, in closed form +// ------------------------------------------------------------------------- +// +// [`Blake3Chain`] below expresses the same schedule against its streaming state +// (`started`, `is_final`), which is the right shape for a hasher fed one byte at +// a time and the wrong shape for anything that knows the whole length up front. +// The Lambda VM's straight-line eDSL emitter is exactly that: shapes are +// compile-time, so it emits block `i` of `n` with the flags and `block_len` +// already decided. +// +// These three functions are hoisted so the emitter READS the schedule rather +// than restating it. A second statement of "which block carries CHUNK_START" is +// the single most likely way for an in-machine hash to differ from the host's by +// one compression's flags — a difference that produces a perfectly valid proof +// of the wrong digest. `the_streaming_chain_follows_the_closed_form_schedule` +// drives the streaming hasher against these and is what keeps them one +// definition rather than two that happen to agree. + +/// Blocks a message of `len_bytes` compresses into. +/// +/// The empty message is **one** block, not zero: `finalize_digest` always +/// compresses, so `blake3_chain(b"")` is one compression over an all-zero block +/// with `block_len = 0`. +pub const fn num_blocks(len_bytes: usize) -> usize { + if len_bytes == 0 { + 1 + } else { + len_bytes.div_ceil(BLOCK_LEN) + } +} + +/// The flags of block `index` of an `n`-block message. +/// +/// `CHUNK_START` on the first, `CHUNK_END | ROOT` on the last, both on a +/// single-block message — which is why a 64-byte Merkle parent carries +/// [`FLAGS_ONE_BLOCK`] and an interior block carries nothing at all. +pub const fn block_flags(index: usize, num_blocks: usize) -> u32 { + let start = if index == 0 { CHUNK_START } else { 0 }; + let end = if index + 1 == num_blocks { + CHUNK_END | ROOT + } else { + 0 + }; + start | end +} + +/// The `block_len` of block `index` of a `len_bytes` message: a full block +/// except on the last, which carries the true remaining byte count. +/// +/// Zero only for the empty message, whose single block is entirely padding. +pub const fn block_len_of(index: usize, len_bytes: usize) -> u32 { + if index + 1 == num_blocks(len_bytes) { + (len_bytes - BLOCK_LEN * index) as u32 + } else { + BLOCK_LEN as u32 + } +} + +// ------------------------------------------------------------------------- +// The bulk-absorb accelerator path +// ------------------------------------------------------------------------- + +/// Dwords in the absorb accelerator's control region: `cv_in[8 words] | +/// cv_out[8 words]`, two little-endian `u32` words per dword. Mirrors +/// `BLAKE3_ABSORB_CTRL_DWORDS` in `executor::vm::instruction::execution`. +pub const ABSORB_CTRL_DWORDS: usize = 8; + +/// First dword of `cv_out` inside the control region — the accelerator reads +/// dwords `0..4` and writes the outgoing chaining value into `4..8`. +pub const ABSORB_CV_OUT_DWORD: usize = 4; + +/// Largest block run one absorb ecall accepts — 1 024 blocks, 64 KiB. Mirrors +/// the executor's `BLAKE3_ABSORB_MAX_BLOCKS` (a test pins them equal); a longer +/// message simply takes several calls, because the chaining value travels +/// through the control region. +/// +/// The binding reason for the value is trace height, not guest cost: the ecall +/// is one CPU cycle and up to this many chip rows, which breaks the +/// one-cycle-to-one-row ratio every other accelerator keeps and that epoch +/// splitting relies on. See the executor constant for the full argument. It +/// costs the guest nothing — absorb cost is flat in message length past about +/// four blocks. +pub const ABSORB_MAX_BLOCKS: usize = 1 << 10; + +/// How many whole blocks of `input` a bulk absorb may take, given `pending` +/// bytes already held in the block buffer and whether `input` is 8-byte aligned. +/// +/// Zero means "take the ordinary buffered path". Three conditions have to hold +/// for the accelerator to be usable at all, and each is a correctness condition +/// rather than a tuning choice: +/// +/// - **`pending == 0`** — the ecall reads whole blocks straight out of the +/// caller's slice, so a half-filled buffer would put the block boundary +/// somewhere the slice does not start. +/// - **`aligned`** — the accelerator reads the message as doublewords. +/// - **at least one byte left over** — the LAST block of a message carries +/// `CHUNK_END | ROOT` and the true byte count, and which block is last is not +/// known until the message ends. Leaving a byte behind keeps that block in +/// `finalize_digest`'s hands, so the ecall never has to know the final +/// framing. It is why `input_len` of exactly one block absorbs nothing. +/// +/// Compiled on every target and unit-tested on the host, so the schedule this +/// picks is checkable without a guest — the riscv64 arm below is then only the +/// ecall around it. +/// +/// # ⚠ Read this before routing NEW data through the absorb ecall +/// +/// The cost of an absorb is asymmetric between the guest and the prover, and the +/// asymmetry is large. One absorb ecall is **one CPU cycle** but up to **1,025 +/// BLAKE3 chip rows and ~8,196 MEMW operations** — against keccak's one cycle to +/// one row and 25 lane accesses. A loop of capped absorbs therefore inflates an +/// epoch's BLAKE3 row count by roughly **170×** over what the cycle-keyed epoch +/// splitter believes it is scheduling: the splitter counts cycles, and this is +/// the one accelerator whose row count is not a function of them. +/// +/// Nothing enforces a per-epoch absorb budget today. That is a bet on the CALL +/// SITES, not a proof of safety, and it is worth stating which bet: +/// +/// * Absorb ecalls are emitted only by OUR guest code — the recursion verifier's +/// leaf hashing and runtime paths hashing our own buffers. The block counts +/// are functions of proof shape and internal buffer sizes, not of anything an +/// adversary supplies. +/// * Untrusted input executed INSIDE a guest (EVM bytecode and calldata under +/// ethrex, say) never reaches this function; that code hashes through the +/// ordinary path and cannot name this syscall. +/// +/// So adversarial exposure is bounded today by construction of the call graph. +/// +/// ★ **The condition that makes a hard budget REQUIRED rather than optional:** +/// any new call site where untrusted input can drive the absorb COUNT or the +/// blocks per absorb. From that moment an attacker picks an epoch's BLAKE3 table +/// height independently of its cycle budget, and the splitter's assumption stops +/// being an approximation and becomes exploitable. If you are adding that +/// surface, add the budget in the same change — a per-epoch cap on absorbed +/// blocks, enforced where epochs are cut, not here. +pub const fn bulk_absorb_blocks(pending: usize, input_len: usize, aligned: bool) -> usize { + if pending != 0 || !aligned { + return 0; + } + let n = input_len.saturating_sub(1) / BLOCK_LEN; + if n > ABSORB_MAX_BLOCKS { + ABSORB_MAX_BLOCKS + } else { + n + } +} + +/// Lay a chaining value out as the absorb accelerator's control region. +/// +/// The guest side of the absorb ABI, and the only place this crate encodes it. +/// `cv_out` is left zero; the accelerator fills it. +pub fn pack_absorb_ctrl(cv: &[u32; 8]) -> [u64; ABSORB_CTRL_DWORDS] { + let mut ctrl = [0u64; ABSORB_CTRL_DWORDS]; + for (k, dword) in ctrl[..ABSORB_CV_OUT_DWORD].iter_mut().enumerate() { + *dword = (cv[2 * k] as u64) | ((cv[2 * k + 1] as u64) << 32); + } + ctrl +} + +/// Read the chaining value the accelerator wrote into [`pack_absorb_ctrl`]'s +/// region. The inverse of that layout, over dwords `4..8`. +pub fn unpack_absorb_cv(ctrl: &[u64; ABSORB_CTRL_DWORDS]) -> [u32; 8] { + core::array::from_fn(|i| { + let dword = ctrl[ABSORB_CV_OUT_DWORD + i / 2]; + if i.is_multiple_of(2) { + dword as u32 + } else { + (dword >> 32) as u32 + } + }) +} + +/// [`Blake3Chain`] as a one-shot over a byte slice. +/// +/// The streaming type and this agree by construction — this *is* the streaming +/// type, fed once. +pub fn blake3_chain(data: &[u8]) -> [u8; 32] { + blake3_chain_rounds(data, BLAKE3_ROUNDS) +} + +/// [`blake3_chain`] with the round count as an argument. +/// +/// The round count is the only parameter, exactly as in +/// [`blake3_compress_rounds`] and for the same reason: at +/// [`BLAKE3_STANDARD_ROUNDS`](super::BLAKE3_STANDARD_ROUNDS) the result is the +/// `blake3` crate's, so that arm certifies this whole code path — the block +/// splitting, the padding, the flag schedule, the `block_len` — and the 6-round +/// arm differs from it by a loop bound alone. +pub fn blake3_chain_rounds(data: &[u8], rounds: usize) -> [u8; 32] { + let mut chain = Blake3Chain::with_rounds(rounds); + chain.update(data); + chain.finalize_digest() +} + +/// Lay a compression's inputs out as the accelerator's 22-dword state region. +/// +/// The guest side of the syscall ABI, and the *only* place this crate encodes +/// it. The output dwords are left zero; the accelerator fills `14..22`. +/// +/// Compiled on every target although only the riscv64 arm of [`compress_block`] +/// calls it, so the packing can be checked on the host against the executor's +/// handler — which is ordinary host code — rather than only inside a guest. The +/// executor's unpacking is the mirror of this, and +/// `prover::tables::blake3::executor_syscall_packing` drives the two against +/// each other through a real `EcallEbreak`. +pub fn pack_syscall_state( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) -> [u64; SYSCALL_STATE_DWORDS] { + let mut words = [0u32; 2 * SYSCALL_OUT_DWORD]; + words[0..8].copy_from_slice(h); + words[8..24].copy_from_slice(m); + words[24] = t as u32; + words[25] = (t >> 32) as u32; + words[26] = block_len; + words[27] = flags; + + let mut state = [0u64; SYSCALL_STATE_DWORDS]; + for (k, dword) in state[..SYSCALL_OUT_DWORD].iter_mut().enumerate() { + *dword = (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32); + } + state +} + +/// Read the 16 output words the accelerator wrote into [`pack_syscall_state`]'s +/// region. The inverse of the low half of that layout, over dwords `14..22`. +pub fn unpack_syscall_out(state: &[u64; SYSCALL_STATE_DWORDS]) -> [u32; 16] { + core::array::from_fn(|i| { + let dword = state[SYSCALL_OUT_DWORD + i / 2]; + if i.is_multiple_of(2) { + dword as u32 + } else { + (dword >> 32) as u32 + } + }) +} + +/// ★ The chain's single entry into the compression function. +/// +/// Both the interior step and the finalization go through here, so there is one +/// place where a guest reaches the accelerator and one framing above it. Adding +/// a second call to [`blake3_compress_rounds`] in this file would put a +/// compression outside the accelerator's reach on the guest and split the two +/// paths silently — the trap PA-PLAN §1.4 names. +/// +/// `t` is not a parameter: the construction is a single chunk that never ends, +/// so the counter is 0 at every block (§1.7). The syscall ABI still carries a +/// full 64-bit counter, and this is where it is pinned to zero. +#[cfg(all(target_arch = "riscv64", feature = "blake3-6round"))] +fn compress_block( + cv: &[u32; 8], + block: &[u32; 16], + block_len: u32, + flags: u32, + rounds: usize, +) -> [u32; 16] { + // `with_rounds` can hand this any count — it exists so the 7-round anchor + // is reachable from one build — while the accelerator implements six and + // nothing else. Anything but the crate-global count takes the software + // path, so the anchor constructor cannot be answered at the wrong round + // count by a machine that has the precompile. + if rounds != BLAKE3_ROUNDS { + return blake3_compress_rounds(cv, block, 0, block_len, flags, rounds); + } + let mut state = pack_syscall_state(cv, block, 0, block_len, flags); + lambda_vm_syscalls::syscalls::blake3_compress_6round(&mut state); + unpack_syscall_out(&state) +} + +/// The chain's single entry into the compression function, in software. +/// +/// See the riscv64 arm above for what this is one of two of. Every host build +/// takes this path, and so does a guest built without `blake3-6round`: the +/// accelerator is 6-round only, so at 7 rounds there is nothing to dispatch to. +#[cfg(not(all(target_arch = "riscv64", feature = "blake3-6round")))] +fn compress_block( + cv: &[u32; 8], + block: &[u32; 16], + block_len: u32, + flags: u32, + rounds: usize, +) -> [u32; 16] { + blake3_compress_rounds(cv, block, 0, block_len, flags, rounds) +} + +/// The accelerator is **six rounds, hard-coded**: `BLAKE3_ROUNDS` in +/// `executor::vm::instruction::execution` is a plain `6` with no feature behind +/// it, and the chip's columns are laid out for that width. So the syscall arm +/// above computes the host prover's hash only while this crate is at six rounds +/// too, and the coupling is compile-time rather than a comment: inverting +/// `blake3-6round`'s polarity must fail the build, not surface later as a root +/// the verifier rejects. +/// +/// Gated on the feature alone, not on the target, although only the riscv64 arm +/// dispatches to the accelerator: a `target_arch` gate would put it out of reach +/// of every host build, including `make lint`'s `blake3-6round` pass, which is +/// the one place CI compiles this feature at all. +#[cfg(feature = "blake3-6round")] +const _: () = assert!( + BLAKE3_ROUNDS == super::BLAKE3_SIX_ROUNDS, + "the BLAKE3 accelerator implements 6 rounds only, but `blake3-6round` did \ + not select 6 — the guest would hash differently from the host prover" +); + +/// The single-chunk BLAKE3 chain as an incremental hasher. +/// +/// Implements the `digest` traits, so it drops into the Merkle backends and the +/// transcript anywhere a `D: Digest` is expected — the same way +/// [`PlatformKeccak256`](crate::hash::platform_keccak::PlatformKeccak256) does. +/// That is what lets the batched and paired backends be one hash rather than two +/// implementations that have to be shown to coincide. +/// +/// A full block is held rather than compressed until more input arrives, because +/// the final block's flags and `block_len` differ from every other block's and +/// whether a block is final is not known until the message ends. +#[derive(Clone)] +pub struct Blake3Chain { + /// The chaining value: `IV`, then the truncated output of each compressed + /// block. Never reset — that is the "single chunk" of the construction. + cv: [u32; 8], + /// The pending block, zero-padded. Zeroing on reset is what pads the final + /// partial block. + block: [u8; BLOCK_LEN], + /// Bytes of `block` that are message, `0..=BLOCK_LEN`. + block_len: usize, + /// Whether any block has been compressed yet — i.e. whether the pending + /// block still carries `CHUNK_START`. + started: bool, + /// Rounds. [`BLAKE3_ROUNDS`] for every production instance; see + /// [`Self::with_rounds`]. + rounds: usize, +} + +impl Default for Blake3Chain { + fn default() -> Self { + Self::with_rounds(BLAKE3_ROUNDS) + } +} + +impl Blake3Chain { + /// A hasher at the crate-global [`BLAKE3_ROUNDS`]. The only constructor any + /// production path uses; [`Default`] and `Digest::new` are this. + pub fn new() -> Self { + Self::default() + } + + /// A hasher at an explicit round count. + /// + /// **For anchoring and known-answer tests only.** The production round count + /// is a compile-time crate-global on purpose — a per-instance one would let a + /// single build commit under two different hashes, which is the failure the + /// `SOCKET_ROUNDS == BLAKE3_ROUNDS` assertion in `prover` exists to prevent. + /// It is exposed because the 7-round arm is the external anchor for the + /// 6-round one, so both must be reachable from one build's tests. + pub fn with_rounds(rounds: usize) -> Self { + Self { + cv: BLAKE3_IV, + block: [0u8; BLOCK_LEN], + block_len: 0, + started: false, + rounds, + } + } + + /// Resume the chain from a chaining value computed outside it — what the + /// absorb accelerator returns after folding a run of whole blocks. + /// + /// `started` is set, because a resumed chain has already compressed at + /// least one block: `CHUNK_START` is spent and the next block is interior. + /// ★ This is the single statement of that fact. The riscv64 bulk-absorb arm + /// resumes through here, and so does the parity test that gates the + /// accelerator against this hasher — so the test cannot agree with the + /// guest about the flag schedule by restating it. + pub fn resume_with_rounds(cv: [u32; 8], rounds: usize) -> Self { + Self { + cv, + block: [0u8; BLOCK_LEN], + block_len: 0, + started: true, + rounds, + } + } + + /// The pending block as 16 little-endian message words. + fn block_words(&self) -> [u32; 16] { + core::array::from_fn(|i| { + u32::from_le_bytes([ + self.block[4 * i], + self.block[4 * i + 1], + self.block[4 * i + 2], + self.block[4 * i + 3], + ]) + }) + } + + /// The pending block's flags. `CHUNK_START` while nothing has been + /// compressed yet; `CHUNK_END | ROOT` when this is the message's last block. + /// + /// The streaming form of [`block_flags`] — `started` is "index > 0" and + /// `is_final` is "index + 1 == n" — reached through it rather than beside + /// it, so the closed form the eDSL emitter reads cannot drift from the one + /// this hasher runs. + fn flags(&self, is_final: bool) -> u32 { + let index = usize::from(self.started); + let num_blocks = if is_final { index + 1 } else { index + 2 }; + block_flags(index, num_blocks) + } + + /// Fold the pending block — known not to be the last — into the chaining + /// value, and clear the block so the next one is zero-padded. + fn compress_pending(&mut self) { + let out = compress_block( + &self.cv, + &self.block_words(), + BLOCK_LEN as u32, + self.flags(false), + self.rounds, + ); + self.cv.copy_from_slice(&out[..8]); + self.block = [0u8; BLOCK_LEN]; + self.block_len = 0; + self.started = true; + } + + /// Fold whole blocks of `input` into the chaining value with ONE ecall, + /// reading them in place. Returns the bytes consumed, `0` if the + /// accelerator was not usable here ([`bulk_absorb_blocks`] says when). + /// + /// This is the only reason the absorb ABI exists: the block-at-a-time path + /// below marshals 112 bytes and pays an ecall per 64 bytes of message, + /// which is what makes a long BLAKE3 absorb cost more in-guest than the + /// same absorb under keccak. + /// + /// The flag word handed over is `self.flags(false)` — the streaming + /// hasher's own schedule, evaluated before `started` moves — so the run's + /// first block carries exactly what it would have carried block by block. + /// The accelerator supplies `0` for every later block, and *that* half of + /// the schedule is the executor's restatement, gated by the parity tests + /// rather than trusted. + #[cfg(all(target_arch = "riscv64", feature = "blake3-absorb"))] + fn bulk_absorb(&mut self, input: &[u8]) -> usize { + // Same reasoning as `compress_block`: the accelerator is 6-round only, + // so an anchoring instance at another count takes the software path. + if self.rounds != BLAKE3_ROUNDS { + return 0; + } + let aligned = (input.as_ptr() as usize).is_multiple_of(8); + let blocks = bulk_absorb_blocks(self.block_len, input.len(), aligned); + if blocks == 0 { + return 0; + } + let taken = blocks * BLOCK_LEN; + let mut ctrl = pack_absorb_ctrl(&self.cv); + lambda_vm_syscalls::syscalls::blake3_absorb(&mut ctrl, &input[..taken], self.flags(false)); + // The pending block is empty here (`bulk_absorb_blocks` required it), so + // resuming replaces the whole state rather than only the chaining value. + *self = Self::resume_with_rounds(unpack_absorb_cv(&ctrl), self.rounds); + taken + } + + /// The bulk-absorb path, absent. See the riscv64 arm above for what this is + /// one of two of: every host build, and any guest without the accelerator, + /// hashes block at a time. Returning `0` makes the caller's fast path + /// vanish, so the streaming semantics are identical on every target — which + /// is what lets the host tests cover the result of taking it. + #[cfg(not(all(target_arch = "riscv64", feature = "blake3-absorb")))] + fn bulk_absorb(&mut self, _input: &[u8]) -> usize { + 0 + } + + /// Absorb more message. Identical results for any split of the same bytes — + /// `streaming_splits_agree_with_one_shot`. + pub fn update(&mut self, mut input: &[u8]) { + while !input.is_empty() { + // Only now is the pending block known not to be the last one. + if self.block_len == BLOCK_LEN { + self.compress_pending(); + } + // With an empty pending block, whole blocks of `input` are blocks + // of the message and the accelerator can take them in one call. + // + // The length test is what keeps the parent path free. A 64-byte + // message — a Merkle parent, the shape FRI query paths are made of + // — can never absorb anything, because the run always leaves the + // final block behind; without this guard it would still pay for the + // call and its checks, which measured at 22 cycles on 432, a 5% + // regression on the hash this accelerator is best at. `>` and not + // `>=`: at exactly one block there is nothing to take. + if input.len() > BLOCK_LEN { + let bulk = self.bulk_absorb(input); + if bulk != 0 { + input = &input[bulk..]; + continue; + } + } + let take = (BLOCK_LEN - self.block_len).min(input.len()); + self.block[self.block_len..self.block_len + take].copy_from_slice(&input[..take]); + self.block_len += take; + input = &input[take..]; + } + } + + /// The 32-byte digest: one final compression over the pending block, with + /// the true byte count as `block_len` and `CHUNK_END | ROOT` set. + /// + /// The empty message takes this path with an all-zero block and + /// `block_len = 0`, which is one compression, not zero — and is what + /// `blake3::hash(b"")` is at 7 rounds. + pub fn finalize_digest(&self) -> [u8; 32] { + let out = compress_block( + &self.cv, + &self.block_words(), + self.block_len as u32, + self.flags(true), + self.rounds, + ); + let mut digest = [0u8; 32]; + for i in 0..8 { + digest[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + digest + } +} + +/// ★ A 64-byte Merkle parent in ONE compression, with no sponge around it. +/// +/// **P2 is what makes this exact** (PA-PLAN §1.7.2): a 64-byte message is a +/// single block, first and last at once, so `Blake3Chain::hash(left ‖ right)` +/// IS `compress(IV, left‖right, t = 0, block_len = 64, FLAGS_ONE_BLOCK)`. This +/// computes that directly — no `Blake3Chain` object, no incremental absorb, no +/// per-byte copy into a pending block, no runtime decision about which flags a +/// block carries. +/// +/// It is worth its own function because of what the guest measurement showed: +/// at 64 bytes the accelerated hash costs ~556 cycles of which only ~248 is the +/// compression, so roughly 55% is digest-layer plumbing around one ecall. A +/// Merkle parent is the shape a FRI query path is made of, so that overhead is +/// paid per path step per query. +/// +/// ⚠ **ONE FRAMING.** It goes through [`compress_block`], the same dispatch +/// every other block in this file uses, so on a guest with the accelerator this +/// is the ecall and at any other round count it is the software path. Writing a +/// second call to `blake3_compress_rounds` here would put a compression outside +/// the accelerator's reach and split the two paths silently — the trap the +/// dispatch's own doc names. +pub fn blake3_parent_rounds(left: &[u8; 32], right: &[u8; 32], rounds: usize) -> [u8; 32] { + // Same little-endian word packing as `block_words`, over the concatenation. + let byte = |i: usize| if i < 32 { left[i] } else { right[i - 32] }; + let block: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes([ + byte(4 * i), + byte(4 * i + 1), + byte(4 * i + 2), + byte(4 * i + 3), + ]) + }); + let out = compress_block( + &BLAKE3_IV, + &block, + BLOCK_LEN as u32, + FLAGS_ONE_BLOCK, + rounds, + ); + let mut digest = [0u8; 32]; + for i in 0..8 { + digest[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + digest +} + +/// [`blake3_parent_rounds`] at the crate-global round count. +pub fn blake3_parent(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + blake3_parent_rounds(left, right, BLAKE3_ROUNDS) +} + +impl HashMarker for Blake3Chain {} + +impl OutputSizeUser for Blake3Chain { + type OutputSize = digest::typenum::U32; +} + +impl Update for Blake3Chain { + fn update(&mut self, data: &[u8]) { + Blake3Chain::update(self, data); + } +} + +impl FixedOutput for Blake3Chain { + fn finalize_into(self, out: &mut Output) { + out.copy_from_slice(&self.finalize_digest()); + } +} + +impl Reset for Blake3Chain { + fn reset(&mut self) { + *self = Self::with_rounds(self.rounds); + } +} + +impl FixedOutputReset for Blake3Chain { + fn finalize_into_reset(&mut self, out: &mut Output) { + out.copy_from_slice(&self.finalize_digest()); + Reset::reset(self); + } +} + +/// The message the KAT table is taken over, at a given length. +/// +/// Byte `i` is `37i + 11 (mod 256)`: every length is a different message, no +/// byte value repeats within a block, and it is the same generator the existing +/// compression-level anchor in `prover::lfm::blake3` uses. +pub const fn kat_message_byte(i: usize) -> u8 { + (i as u8).wrapping_mul(37).wrapping_add(11) +} + +/// The lengths [`CHAIN_KAT_6ROUND`] covers, in order. PA-PLAN §1.7.4 says what +/// each one discriminates: the empty message is one block (0); `block_len` is +/// the true length and the tail is zero-padded (1, 31, 63); a 64-byte message is +/// the parent form (64); the chain's first step moves `CHUNK_END | ROOT` off +/// block 0 (65); an exact multiple of 64 emits no spurious final block (128); +/// interior blocks carry no flags (192, 256, 1024); and 1088 is the first length +/// past one chunk, where this construction leaves standard BLAKE3 (1088). +pub const CHAIN_KAT_LENS: [usize; 12] = [0, 1, 31, 63, 64, 65, 127, 128, 192, 256, 1024, 1088]; + +/// [`blake3_chain`] at **6 rounds** over `kat_message_byte` messages of each +/// [`CHAIN_KAT_LENS`] length. +/// +/// # What this table is, and how strong its provenance actually is +/// +/// It is a regression pin — generated from this implementation and committed, so +/// a later refactor cannot change the construction silently. But it is more than +/// that, and the difference is worth stating precisely. +/// +/// It is not "stronger than the compression vectors next door" either: the two +/// cover different axes. This table pins the FRAMING across blocks — the flag +/// schedule, the chaining value, the final block's `block_len`. It cannot pin the +/// counter split, because every message here is hashed with `t = 0`; only +/// [`CANONICAL_VECTORS`](super::CANONICAL_VECTORS), whose ten vectors all carry +/// `t >= 2^32`, does that. Neither table is redundant with the other. +/// +/// Every entry from length 0 to 1024 was **independently reproduced** by #903's +/// Python oracle (`tooling/blake3-second-source/blake3-oracle/blake3_ref.py`) evaluated at +/// `rounds = 6`, on 2026-08-14. That oracle is a full standard-BLAKE3 +/// implementation with the round count as a parameter, written by another author +/// for a different purpose, and at `rounds = 7` it reproduces the official +/// `blake3` package bit-for-bit at every length checked — including the +/// multi-chunk ones. So for the whole ≤1-chunk range these digests are not a +/// self-consistency check: two implementations that share no code agree, and +/// the conventions they agree on are pinned to the published hash from outside. +/// +/// Length 1088 is where they part, and that is the point of including it: the +/// oracle stays standard past one chunk and this construction does not (P3). +/// Being able to say the divergence is *the chunking* rather than the round +/// count needs a reference that is standard at 6 rounds too, which is exactly +/// what the oracle is. +/// +/// Everything that cross-check needs is tracked, so it is reproducible rather +/// than merely recorded. `tooling/blake3-second-source/blake3-oracle/` holds the +/// round-parameterized reference (`blake3_ref.py`, vendored at commit +/// `65025095`), which exposes raw compression entry points — `compress`, +/// `compress_cv`, `compress_6round` — as well as `blake3_hash`, alongside +/// `canonical_6round_vectors.json`, `official_test_vectors.json` and +/// `test_oracle.py`. +/// +/// A **second source** sits beside it, and it is the stronger of the two: +/// `tooling/blake3-second-source/reference-impl/` is upstream BLAKE3 1.8.5's own portable C +/// with its round loop parameterized, the whole edit being +/// `PARAMETERISATION.diff`. It reproduces this table at 6 rounds over every +/// length up to one chunk, and it encodes the message schedule as an indexed +/// `MSG_SCHEDULE[r]` table where this crate composes a single permutation +/// between rounds. Those are structurally different expressions of the same +/// convention, so its agreement cross-validates the schedule instead of +/// restating it — a bug in the iterative composition is precisely what one +/// source cannot catch. `make test-blake3-second-source` runs both against these +/// digests: a ~1 second C compile plus a randomised differential, no cargo and +/// no GPU. +/// +/// The 7-round arm remains the primary anchor and needs none of this: +/// `blake3_chain_rounds(m, 7)` is checked directly against the `blake3` crate +/// over all 1025 lengths, with no table in between. +pub const CHAIN_KAT_6ROUND: [[u8; 32]; 12] = [ + // len 0 + [ + 0x3C, 0x3B, 0xBB, 0x1F, 0x33, 0x5A, 0x31, 0xEA, 0x86, 0x46, 0x4B, 0x65, 0x1C, 0x02, 0x06, + 0xFC, 0x81, 0xD3, 0x32, 0x62, 0xAE, 0x00, 0xEA, 0x1A, 0x65, 0xF3, 0xD1, 0xD0, 0x4A, 0xFA, + 0xEF, 0xC9, + ], + // len 1 + [ + 0x2A, 0x50, 0xE4, 0x5B, 0x89, 0x21, 0xF9, 0xEF, 0xA0, 0x08, 0xD9, 0xF3, 0x9F, 0x71, 0x65, + 0x60, 0x0C, 0xF4, 0x8A, 0x7F, 0x0E, 0x85, 0x9C, 0x21, 0x22, 0xE3, 0xCC, 0xB6, 0xB9, 0x67, + 0x7E, 0xE5, + ], + // len 31 + [ + 0xC3, 0x8B, 0xF6, 0x2F, 0x50, 0x60, 0x40, 0xB2, 0x60, 0x02, 0x73, 0x77, 0x8D, 0x28, 0x1B, + 0x89, 0x43, 0x62, 0x1E, 0x2B, 0x8A, 0x9F, 0x59, 0xE2, 0x37, 0x9F, 0x8F, 0xD7, 0xE5, 0xC8, + 0x51, 0x25, + ], + // len 63 + [ + 0xC3, 0x73, 0xF5, 0x1A, 0x5E, 0xB8, 0xB2, 0x7E, 0xA0, 0x5B, 0xB1, 0xF6, 0xF4, 0xE6, 0x2E, + 0x92, 0x4F, 0xF4, 0xD8, 0xA2, 0x79, 0xF0, 0xD0, 0x5A, 0xFA, 0x5C, 0xD5, 0x19, 0x39, 0x1D, + 0x63, 0x89, + ], + // len 64 + [ + 0x59, 0x00, 0xA1, 0xE3, 0x98, 0xBB, 0x2B, 0xF6, 0xD3, 0xBA, 0x7F, 0x1A, 0x29, 0x19, 0x7B, + 0x79, 0xC8, 0x6B, 0x71, 0xAD, 0x2C, 0x26, 0x31, 0xF4, 0xAC, 0x73, 0x6C, 0x82, 0xDB, 0x04, + 0x3C, 0xB5, + ], + // len 65 + [ + 0x53, 0x95, 0x3F, 0xCA, 0xDC, 0x39, 0xB8, 0x62, 0x39, 0x01, 0xAF, 0x7B, 0x53, 0x4F, 0x2F, + 0x69, 0x33, 0xE3, 0x12, 0xF5, 0x02, 0x99, 0x33, 0x13, 0x34, 0xE6, 0xC0, 0xA7, 0xC9, 0xDB, + 0xC2, 0xBE, + ], + // len 127 + [ + 0x9E, 0x0D, 0xD8, 0x16, 0x8D, 0x19, 0x9A, 0x04, 0x59, 0x0C, 0x2C, 0xBA, 0x43, 0x9B, 0x27, + 0x07, 0x76, 0xE4, 0x27, 0x15, 0xD5, 0x18, 0xF6, 0x86, 0x55, 0xE5, 0x66, 0x92, 0x48, 0x3E, + 0x50, 0x5E, + ], + // len 128 + [ + 0x5C, 0xAF, 0xFC, 0x87, 0x84, 0xE8, 0x17, 0xBB, 0xBA, 0x99, 0x1B, 0x21, 0x08, 0xC2, 0x6A, + 0x3D, 0xFD, 0xF8, 0x04, 0x24, 0x5E, 0xF6, 0x3A, 0xE1, 0x04, 0x0A, 0x3C, 0x34, 0xF1, 0xB3, + 0x62, 0xFF, + ], + // len 192 + [ + 0x39, 0x9D, 0x6B, 0x9A, 0xDE, 0xB2, 0xF8, 0x84, 0x50, 0x77, 0x5F, 0x77, 0x3E, 0x9D, 0xEC, + 0x08, 0x83, 0x6C, 0x13, 0x57, 0x13, 0xC2, 0xC5, 0xDD, 0x09, 0xF4, 0xCE, 0xCE, 0xB0, 0xED, + 0x38, 0x88, + ], + // len 256 + [ + 0xFB, 0xCA, 0xB3, 0x69, 0x9A, 0x49, 0x59, 0xFA, 0x37, 0x19, 0x0E, 0x98, 0xCA, 0x51, 0x42, + 0xDD, 0xBC, 0x88, 0x33, 0x0F, 0x2E, 0x7D, 0x12, 0x33, 0x5D, 0xB9, 0xC6, 0xC8, 0x88, 0x1A, + 0x0B, 0x87, + ], + // len 1024 + [ + 0xF3, 0x95, 0xE7, 0xE2, 0x15, 0x03, 0x63, 0xB6, 0xD2, 0x00, 0x48, 0x75, 0x15, 0x42, 0x5B, + 0x02, 0x04, 0xEE, 0xA4, 0x24, 0x07, 0x21, 0x83, 0xB7, 0x01, 0x17, 0x6E, 0xCC, 0xBE, 0x0F, + 0xFE, 0x1B, + ], + // len 1088 + [ + 0xB4, 0x73, 0x8E, 0xDE, 0x77, 0xA6, 0xEC, 0x16, 0x6E, 0xE9, 0x76, 0x67, 0x11, 0x8D, 0x47, + 0x93, 0xCB, 0xF2, 0xB0, 0x8B, 0x45, 0xAA, 0xC7, 0xC6, 0xD5, 0x29, 0x43, 0xB5, 0xD2, 0x98, + 0xC6, 0x88, + ], +]; + +#[cfg(test)] +mod tests { + use super::*; + use crate::hash::blake3::{BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS}; + + /// ★ The parent specialization is the streaming path, byte for byte, at + /// BOTH round counts. + /// + /// This is the gate the specialization needs: it exists only to skip the + /// sponge, so the one thing that must never differ is the answer. Driven + /// over inputs that move every word — including the all-zero and all-0xff + /// corners, where a wrong flag or a swapped operand is easiest to miss. + #[test] + fn the_parent_specialization_equals_the_streaming_path() { + let pattern = |seed: u8| -> [u8; 32] { + core::array::from_fn(|i| (i as u8).wrapping_mul(31).wrapping_add(seed)) + }; + let cases: [([u8; 32], [u8; 32]); 6] = [ + ([0u8; 32], [0u8; 32]), + ([0xffu8; 32], [0xffu8; 32]), + ([0u8; 32], [0xffu8; 32]), + (pattern(0), pattern(1)), + (pattern(7), pattern(200)), + (pattern(1), pattern(0)), + ]; + for rounds in [BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS] { + for (i, (l, r)) in cases.iter().enumerate() { + let mut streamed = Blake3Chain::with_rounds(rounds); + streamed.update(l); + streamed.update(r); + assert_eq!( + blake3_parent_rounds(l, r, rounds), + streamed.finalize_digest(), + "rounds {rounds}, case {i}: the specialization must be the \ + streaming digest of the two concatenated nodes" + ); + } + } + + // NON-VACUITY: swapping the operands must move the digest, or the test + // above would pass for a function that ignored its inputs. + let (l, r) = (pattern(3), pattern(9)); + assert_ne!( + blake3_parent_rounds(&l, &r, BLAKE3_SIX_ROUNDS), + blake3_parent_rounds(&r, &l, BLAKE3_SIX_ROUNDS), + "the parent must not be symmetric in its operands" + ); + // And the two round counts must disagree, or "both" means one. + assert_ne!( + blake3_parent_rounds(&l, &r, BLAKE3_SIX_ROUNDS), + blake3_parent_rounds(&l, &r, BLAKE3_STANDARD_ROUNDS), + ); + } + use alloc::vec::Vec; + + fn message(len: usize) -> Vec { + (0..len).map(kat_message_byte).collect() + } + + /// ★ **The external anchor.** At 7 rounds this construction is the `blake3` + /// crate's hash, for every message length up to one full chunk — no oracle, + /// no JSON, no transcription. + /// + /// The range is what makes it worth more than a compression-level anchor: + /// it pins the block splitting, the zero padding, the `block_len` of the + /// final block, the `CHUNK_START`/`CHUNK_END`/`ROOT` schedule and the + /// little-endian digest read-back, at every boundary those can be wrong at. + #[test] + fn seven_round_chain_is_the_blake3_crate() { + for len in 0..=1024usize { + let msg = message(len); + assert_eq!( + blake3_chain_rounds(&msg, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&msg).as_bytes(), + "the 7-round chain must equal the blake3 crate at length {len}" + ); + } + } + + /// NEGATIVE CONTROL for the anchor: at 6 rounds it must not match, or the + /// test above would pass just as well with `rounds` ignored — the one bug + /// that would make the whole external-anchor argument vacuous. + #[test] + fn six_round_chain_is_not_the_blake3_crate() { + for len in [0usize, 1, 64, 65, 128, 1024] { + let msg = message(len); + assert_ne!( + blake3_chain_rounds(&msg, BLAKE3_SIX_ROUNDS), + *blake3::hash(&msg).as_bytes(), + "length {len}" + ); + } + } + + /// ★ **P3, stated as a test.** Past one chunk the construction deliberately + /// leaves standard BLAKE3 — the standard would start chunk 1 and build a + /// tree, this keeps chaining. Without this, "we implement the single-chunk + /// chain" would be an unfalsifiable claim: the anchor above would pass + /// identically if we had implemented the whole chunk tree instead. + /// + /// 1024 is the last length where they agree and 1088 the first block past + /// it, so the two assertions together locate the divergence exactly. + #[test] + fn past_one_chunk_leaves_the_blake3_crate() { + let last_agreeing = message(1024); + assert_eq!( + blake3_chain_rounds(&last_agreeing, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&last_agreeing).as_bytes(), + "1024 bytes is still one chunk and must agree" + ); + for len in [1025usize, 1088, 2048] { + let msg = message(len); + assert_ne!( + blake3_chain_rounds(&msg, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&msg).as_bytes(), + "past one chunk the constructions must differ, at length {len}" + ); + } + } + + /// ★ **P2** — a 64-byte message is exactly the Merkle parent form: one + /// compression, `h = IV`, `t = 0`, `block_len = 64`, `flags = 0x0B`. + /// + /// This is what makes the `StarkHash` two-element invariant hold by + /// construction, and it is the framing the device kernel implements. Written + /// out as an explicit compression rather than as "whatever the code does", + /// so it fails if the flag schedule or the counter moves. + #[test] + fn a_sixty_four_byte_message_is_the_parent_form() { + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(7)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(31).wrapping_add(3)); + let mut msg = [0u8; 64]; + msg[..32].copy_from_slice(&left); + msg[32..].copy_from_slice(&right); + + for rounds in [BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS] { + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(msg[4 * i..4 * i + 4].try_into().unwrap()) + }); + let out = blake3_compress_rounds(&BLAKE3_IV, &words, 0, 64, FLAGS_ONE_BLOCK, rounds); + let mut expected = [0u8; 32]; + for i in 0..8 { + expected[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_eq!( + blake3_chain_rounds(&msg, rounds), + expected, + "a 64-byte message must be one parent compression, at {rounds} rounds" + ); + } + } + + /// The `Update` contract: the digest depends on the bytes, not on how they + /// were handed over. Splits are taken at and either side of every block + /// boundary, which is where a mis-set `CHUNK_START` or a prematurely + /// compressed final block would show. + #[test] + fn streaming_splits_agree_with_one_shot() { + for len in [0usize, 1, 63, 64, 65, 127, 128, 129, 200] { + let msg = message(len); + let want = blake3_chain(&msg); + for split in 0..=len { + let mut chain = Blake3Chain::new(); + chain.update(&msg[..split]); + chain.update(&msg[split..]); + assert_eq!( + chain.finalize_digest(), + want, + "length {len} split at {split}" + ); + } + // Byte at a time, which crosses every boundary in the smallest + // possible increments. + let mut chain = Blake3Chain::new(); + for b in &msg { + chain.update(&[*b]); + } + assert_eq!(chain.finalize_digest(), want, "length {len} byte at a time"); + } + } + + /// The closed form of the framing, evaluated block by block — what an + /// emitter that knows the length up front produces. + fn closed_form_digest(msg: &[u8], flag_schedule: fn(usize, usize) -> u32) -> [u8; 32] { + let len = msg.len(); + let n = num_blocks(len); + let mut cv = BLAKE3_IV; + let mut out = [0u32; 16]; + for i in 0..n { + let mut block = [0u8; BLOCK_LEN]; + let start = i * BLOCK_LEN; + let take = len.saturating_sub(start).min(BLOCK_LEN); + block[..take].copy_from_slice(&msg[start..start + take]); + let words: [u32; 16] = core::array::from_fn(|k| { + u32::from_le_bytes(block[4 * k..4 * k + 4].try_into().unwrap()) + }); + out = blake3_compress_rounds( + &cv, + &words, + 0, + block_len_of(i, len), + flag_schedule(i, n), + BLAKE3_ROUNDS, + ); + cv.copy_from_slice(&out[..8]); + } + let mut digest = [0u8; 32]; + for i in 0..8 { + digest[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + digest + } + + /// ★ The closed-form schedule IS the streaming hasher's, at every length a + /// block boundary can be wrong at. + /// + /// The Lambda VM's eDSL emitter cannot use [`Blake3Chain`] — it has no + /// runtime, only a compile-time length — so it emits [`num_blocks`] + /// compressions parameterized by [`block_flags`] and [`block_len_of`]. This + /// is the statement that doing so is this hash and not a neighbour of it. + #[test] + fn the_streaming_chain_follows_the_closed_form_schedule() { + for len in [ + 0usize, 1, 4, 31, 32, 63, 64, 65, 100, 127, 128, 129, 192, 200, 256, 1024, 1088, + ] { + let msg = message(len); + assert_eq!( + closed_form_digest(&msg, block_flags), + blake3_chain(&msg), + "the closed-form framing must be the chain's, at length {len}" + ); + } + } + + /// NEGATIVE CONTROL for the closed form: a schedule that puts + /// `CHUNK_START` on every block, or `CHUNK_END | ROOT` on the first, must + /// disagree — otherwise the test above would pass for a construction that + /// ignores the flags entirely, which is the failure it exists to catch. + #[test] + fn a_different_flag_schedule_is_a_different_hash() { + const fn start_on_every_block(_index: usize, num_blocks: usize) -> u32 { + block_flags(0, num_blocks + 1) | block_flags(num_blocks - 1, num_blocks) + } + const fn end_on_the_first_block(index: usize, num_blocks: usize) -> u32 { + block_flags(num_blocks - 1 - index, num_blocks) + } + // At one block every schedule here coincides, so the controls start at + // two — which is also the first length where a schedule exists at all. + for len in [65usize, 128, 200, 1024] { + let msg = message(len); + let honest = blake3_chain(&msg); + assert_eq!(closed_form_digest(&msg, block_flags), honest); + assert_ne!( + closed_form_digest(&msg, start_on_every_block), + honest, + "CHUNK_START on every block must not be this hash, at length {len}" + ); + assert_ne!( + closed_form_digest(&msg, end_on_the_first_block), + honest, + "a reversed flag schedule must not be this hash, at length {len}" + ); + } + } + + /// The committed 6-round regression pin. See [`CHAIN_KAT_6ROUND`] for what + /// this does and does not establish. + #[test] + fn six_round_chain_matches_the_committed_table() { + for (i, &len) in CHAIN_KAT_LENS.iter().enumerate() { + assert_eq!( + blake3_chain_rounds(&message(len), BLAKE3_SIX_ROUNDS), + CHAIN_KAT_6ROUND[i], + "6-round chain KAT at length {len}" + ); + } + } + + /// NEGATIVE CONTROL for the table: the entries must be distinct data, or a + /// generation bug that wrote one digest twelve times would leave the test + /// above passing and pinning nothing. + #[test] + fn the_committed_table_entries_are_distinct() { + for (i, a) in CHAIN_KAT_6ROUND.iter().enumerate() { + for (j, b) in CHAIN_KAT_6ROUND.iter().enumerate().skip(i + 1) { + assert_ne!(a, b, "KAT entries {i} and {j} are the same digest"); + } + } + } + + /// **P4** in its cheapest observable form: lengths that share a padded block + /// must not share a digest. A construction that ignored `block_len` would + /// collide 31 with 32, and one that ignored the flag schedule would collide + /// 64 with 65's first block. + #[test] + fn lengths_sharing_a_padded_block_do_not_collide() { + let mut seen: Vec<[u8; 32]> = Vec::new(); + for len in 0..=130usize { + let digest = blake3_chain(&message(len)); + assert!( + !seen.contains(&digest), + "length {len} collides with a shorter message" + ); + seen.push(digest); + } + } + + /// `Reset` really returns to the initial state, including the pending block + /// and the `CHUNK_START` flag — a reset that kept `started` set would hash + /// the next message under the wrong flags. + #[test] + fn reset_returns_to_the_initial_state() { + let mut chain = Blake3Chain::new(); + chain.update(&message(100)); + Reset::reset(&mut chain); + chain.update(&message(7)); + assert_eq!(chain.finalize_digest(), blake3_chain(&message(7))); + } + + /// ★ The bulk-absorb schedule always leaves a block for `finalize_digest`. + /// + /// This is the property the whole absorb path rests on: the accelerator + /// never sees the message's last block, so it never has to know + /// `CHUNK_END | ROOT` or the true byte count, and the final framing stays + /// where `Blake3Chain` already had it. If this could ever return every + /// block, a full-block message would finalize on a block that had already + /// been compressed under interior flags — a different hash. + #[test] + fn a_bulk_absorb_always_leaves_the_final_block() { + for len in 0..=600usize { + let blocks = bulk_absorb_blocks(0, len, true); + let taken = blocks * BLOCK_LEN; + assert!(taken < len || len == 0, "length {len} absorbed everything"); + // What is left is a whole message tail, never a negative remainder. + assert!(taken <= len.saturating_sub(1), "length {len}"); + // And it is maximal: one more block would not have fit. + assert!(taken + BLOCK_LEN > len.saturating_sub(1), "length {len}"); + } + } + + /// The two conditions that make the ecall unusable, and the cap. + #[test] + fn a_bulk_absorb_declines_what_it_cannot_read() { + // A half-filled pending block: whole blocks of `input` are not blocks + // of the message. + for pending in 1..BLOCK_LEN { + assert_eq!( + bulk_absorb_blocks(pending, 4096, true), + 0, + "pending {pending}" + ); + } + // Unaligned: the accelerator reads doublewords. + assert_eq!(bulk_absorb_blocks(0, 4096, false), 0); + // One block exactly is the message's last block; nothing to absorb. + assert_eq!(bulk_absorb_blocks(0, BLOCK_LEN, true), 0); + assert_eq!(bulk_absorb_blocks(0, BLOCK_LEN + 1, true), 1); + // The cap holds, and a longer message just takes another call. + let huge = (ABSORB_MAX_BLOCKS + 10) * BLOCK_LEN; + assert_eq!(bulk_absorb_blocks(0, huge, true), ABSORB_MAX_BLOCKS); + } + + /// The control region round-trips a chaining value through the layout the + /// executor reads and writes — the absorb ABI's counterpart to + /// `pack_syscall_state`/`unpack_syscall_out`, checkable without a guest. + #[test] + fn the_absorb_control_region_round_trips() { + let cv: [u32; 8] = core::array::from_fn(|i| (i as u32).wrapping_mul(0x9E37_79B9) ^ 0xA5A5); + let mut ctrl = pack_absorb_ctrl(&cv); + // `cv_in` occupies the low half and `cv_out` is left for the accelerator. + assert_eq!(&ctrl[ABSORB_CV_OUT_DWORD..], &[0u64; 4]); + // Simulate the accelerator writing the outgoing value. + for k in 0..4 { + ctrl[ABSORB_CV_OUT_DWORD + k] = ctrl[k]; + } + assert_eq!(unpack_absorb_cv(&ctrl), cv); + } + + /// NEGATIVE CONTROL: the round-trip discriminates word ORDER within a + /// dword, which is the likeliest thing to transpose and the one a + /// symmetric test would miss. + #[test] + fn the_absorb_control_region_is_word_order_sensitive() { + let cv: [u32; 8] = core::array::from_fn(|i| i as u32 + 1); + let ctrl = pack_absorb_ctrl(&cv); + let mut swapped = [0u64; ABSORB_CTRL_DWORDS]; + for k in 0..4 { + swapped[ABSORB_CV_OUT_DWORD + k] = ctrl[k].rotate_left(32); + } + assert_ne!(unpack_absorb_cv(&swapped), cv); + } + + /// The `digest` route and the free function are the same hash — the backends + /// reach this type through `Digest`, the KATs above through `blake3_chain`. + #[test] + fn the_digest_trait_route_agrees_with_the_free_function() { + use digest::Digest; + for len in [0usize, 1, 64, 65, 200] { + let msg = message(len); + let mut hasher = ::new(); + Digest::update(&mut hasher, &msg); + let via_digest: [u8; 32] = Digest::finalize(hasher).into(); + assert_eq!(via_digest, blake3_chain(&msg), "length {len}"); + } + } +} diff --git a/crypto/crypto/src/hash/blake3/mod.rs b/crypto/crypto/src/hash/blake3/mod.rs new file mode 100644 index 000000000..a79dd9c96 --- /dev/null +++ b/crypto/crypto/src/hash/blake3/mod.rs @@ -0,0 +1,216 @@ +//! The BLAKE3 compression function with the round count as a parameter, and the +//! byte hash the RV64 prover's commitments are built from. +//! +//! # Why this lives in `crypto` +//! +//! It has three callers that cannot share a copy any other way. The Merkle +//! backends in [`crate::merkle_tree::backends`] are in this crate; the +//! `LFM_BLAKE3` chip and the `LFM_HASH` socket are in `prover`, which depends on +//! this crate; and the CUDA kernels are checked against it from `math-cuda`, +//! which `prover` depends on. `crypto` is the only place all three can reach, so +//! the compression function is defined here once and re-exported upward — +//! `prover::lfm::blake3` is a re-export of this module, not a second +//! implementation. A chip and a commitment backend that hash identically because +//! they call one function is a different claim from two that agree today. +//! +//! # The round count +//! +//! [`BLAKE3_ROUNDS`] is 7 — standard BLAKE3 — unless the crate's +//! `blake3-6round` feature is on, and then it is 6. The polarity is deliberate +//! and must not be inverted: every existing measurement and sign-off reads +//! "7-round instantiated baseline, 6 behind the feature". +//! +//! The knob is crate-global rather than a generic parameter so that one build +//! cannot produce two hashes. Crates above re-export it rather than defining +//! their own, and `prover`'s `blake3-6round` feature forwards to this one, so +//! the chip's round count and the commitment's round count are the same symbol. +//! `math-cuda` necessarily has its own (it compiles a cubin), which is why it +//! exports the compiled-in count for a caller to assert instead of discover. +//! +//! # Provenance of the primitive, and why no external KAT exists at 6 rounds +//! +//! Vendored from PR #903 (`yetanotherco/lambda_vm`, head +//! `89aeeb8c2b0389e9d21a861c9e3a10a7b1b5704e`). Standing-decisions rule 9 +//! requires pinning a new primitive against an external known-answer vector that +//! nothing in this repository produced. That is *impossible in the usual form* +//! for the 6-round variant: it is not standard BLAKE3, so no published vector +//! and no crate exposes it. The provenance chain #903 supplies instead: +//! +//! 1. A z3-proved model of the compression dataflow +//! (`thoughts/blake3/blake3-chip/z3_blake_verify.py`). +//! 2. A Python oracle (`tooling/blake3-second-source/blake3-oracle/blake3_ref.py`) whose +//! **7-round** instantiation is pinned against the official `blake3` crate's +//! published test vectors — so the oracle's G-function, message schedule, +//! counter split and feed-forward are all externally validated; only the +//! round count is varied. +//! 3. That oracle at `rounds = 6` emitted the 10 canonical vectors in +//! [`CANONICAL_VECTORS`], which pin this port. +//! +//! So the external anchor is one step removed: the *conventions* are pinned by +//! the official crate through the oracle, and the round count is the single +//! degree of freedom the canonical vectors add. That is weaker than a direct +//! KAT and is recorded as such — but [`CANONICAL_VECTORS`] still discriminates +//! every convention a wrong port could get wrong, which the falsification tests +//! in `prover::lfm::blake3` demonstrate one convention at a time. +//! +//! [`chain`] extends the anchor considerably: at 7 rounds [`Blake3Chain`] is the +//! `blake3` crate's full hash for every message up to 1024 bytes, so the framing +//! — not just the round function — is externally checked. +//! +//! ⚠ Security assumption **A6R**: collision resistance of the 6-round variant +//! is a named, unratified assumption (#903's `IMPLEMENTATION.md`). Nothing here +//! ratifies it. + +pub mod chain; +mod vectors; + +pub use chain::{Blake3Chain, blake3_chain}; +pub use vectors::{CANONICAL_OUT_7ROUND, CANONICAL_VECTORS, Vector}; + +/// The BLAKE3 IV (identical to SHA-256's initial state). `IV[0..4]` seeds +/// `v[8..12]` of the compression working state. +pub const BLAKE3_IV: [u32; 8] = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +]; + +/// The BLAKE3 message-schedule permutation, applied between rounds +/// (`m'[i] = m[MSG_PERMUTATION[i]]`). +pub const BLAKE3_MSG_PERMUTATION: [usize; 16] = + [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8]; + +/// Rounds of *standard* BLAKE3. At this value [`blake3_compress_rounds`] is +/// bit-for-bit the published compression function — the property the whole +/// external-anchor argument rests on, pinned by +/// `tests::seven_rounds_is_the_blake3_crate`. +pub const BLAKE3_STANDARD_ROUNDS: usize = 7; + +/// Rounds of the 6-round internal variant. Reachable only through the +/// `blake3-6round` feature; [`CANONICAL_VECTORS`] pin it unconditionally. +pub const BLAKE3_SIX_ROUNDS: usize = 6; + +/// The round count every BLAKE3 chip in this tree is compiled for — the +/// standalone `LFM_BLAKE3` probe and the `LFM_HASH` socket arm alike. They share +/// one knob deliberately: two would let a sweep leave the two chips describing +/// different hashes. +/// +/// **7 by default**, i.e. standard BLAKE3, which is what the A6R sign-off +/// instantiates. At 7 rounds the `blake3` crate is a direct known-answer test +/// for the primitive *and* for the socket, and no unratified assumption is +/// carried. `--features blake3-6round` selects the 6-round internal variant: +/// the measured performance variant, resting on **A6R**. +/// +/// It is a compile-time constant rather than a parameter because both chips' +/// column layouts are `8 · rounds` G-blocks wide and their width functions are +/// `const fn`. The round count is the ONLY thing it varies — the G function, the +/// message schedule, the counter split and the feed-forward are fixed — which is +/// what lets the 7-round anchor certify the whole code path rather than a +/// separate 7-round copy of it. +#[cfg(not(feature = "blake3-6round"))] +pub const BLAKE3_ROUNDS: usize = BLAKE3_STANDARD_ROUNDS; +#[cfg(feature = "blake3-6round")] +pub const BLAKE3_ROUNDS: usize = BLAKE3_SIX_ROUNDS; + +/// The BLAKE3 quarter-round G (spec §2.1). +#[inline] +fn blake3_g(v: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, mx: u32, my: u32) { + v[a] = v[a].wrapping_add(v[b]).wrapping_add(mx); + v[d] = (v[d] ^ v[a]).rotate_right(16); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(12); + v[a] = v[a].wrapping_add(v[b]).wrapping_add(my); + v[d] = (v[d] ^ v[a]).rotate_right(8); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(7); +} + +/// The BLAKE3 compression function `f` at 6 rounds (spec §2.2, oracle §2.4). +/// +/// State init: `v[0..8] = h`, `v[8..12] = IV[0..4]`, `v[12] = t as u32`, +/// `v[13] = (t >> 32) as u32`, `v[14] = block_len`, `v[15] = flags`. Six rounds +/// of 8 G-calls (4 columns then 4 diagonals), permuting the message schedule +/// between rounds (`r < rounds - 1`, i.e. 5 permutes — the trailing permute is +/// never consumed). Feed-forward: `out[i] = v[i] ^ v[i+8]`, +/// `out[i+8] = v[i+8] ^ h[i]`. The truncated chaining value is `out[0..8]`. +pub fn blake3_compress_6round( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) -> [u32; 16] { + blake3_compress_rounds(h, m, t, block_len, flags, BLAKE3_SIX_ROUNDS) +} + +/// [`blake3_compress_6round`] with the round count as an argument. +/// +/// The round count is the *only* parameter: everything else — the G function, +/// the message schedule, the counter split, the feed-forward — is fixed. That +/// is what makes `rounds = BLAKE3_STANDARD_ROUNDS` an external anchor for the +/// whole code path rather than for a separate 7-round copy of it, and it is why +/// this is one function with a loop bound instead of two functions. +pub fn blake3_compress_rounds( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, + rounds: usize, +) -> [u32; 16] { + let mut v: [u32; 16] = [ + h[0], + h[1], + h[2], + h[3], + h[4], + h[5], + h[6], + h[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + t as u32, + (t >> 32) as u32, + block_len, + flags, + ]; + + let mut m = *m; + for r in 0..rounds { + // Mix the columns. + blake3_g(&mut v, 0, 4, 8, 12, m[0], m[1]); + blake3_g(&mut v, 1, 5, 9, 13, m[2], m[3]); + blake3_g(&mut v, 2, 6, 10, 14, m[4], m[5]); + blake3_g(&mut v, 3, 7, 11, 15, m[6], m[7]); + // Mix the diagonals. + blake3_g(&mut v, 0, 5, 10, 15, m[8], m[9]); + blake3_g(&mut v, 1, 6, 11, 12, m[10], m[11]); + blake3_g(&mut v, 2, 7, 8, 13, m[12], m[13]); + blake3_g(&mut v, 3, 4, 9, 14, m[14], m[15]); + // Permute between rounds; the permute after the last round is never + // consumed (oracle: `r < rounds - 1`). + if r < rounds - 1 { + let prev = m; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + m[i] = prev[p]; + } + } + } + + let mut out = [0u32; 16]; + for i in 0..8 { + out[i] = v[i] ^ v[i + 8]; + out[i + 8] = v[i + 8] ^ h[i]; + } + out +} + +/// The 16-word output of `CANONICAL_VECTORS[i]` at the compiled-in +/// [`BLAKE3_ROUNDS`] — what a chip built from this module must produce. +pub const fn canonical_expected_out(i: usize) -> [u32; 16] { + if BLAKE3_ROUNDS == BLAKE3_STANDARD_ROUNDS { + CANONICAL_OUT_7ROUND[i] + } else { + CANONICAL_VECTORS[i].out + } +} diff --git a/crypto/crypto/src/hash/blake3/vectors.rs b/crypto/crypto/src/hash/blake3/vectors.rs new file mode 100644 index 000000000..41e2ca52c --- /dev/null +++ b/crypto/crypto/src/hash/blake3/vectors.rs @@ -0,0 +1,285 @@ +//! The canonical known-answer vectors for [`super::blake3_compress_rounds`], +//! at both round counts. +//! +//! Provenance is recorded in the parent module's header: the 7-round table is +//! what the official `blake3` crate produces (and is checked against it +//! directly), and the 6-round table came from #903's Python oracle, whose +//! conventions the 7-round arm pins from outside. + +/// One canonical 6-round vector: the oracle's inputs and its 16-word output. +#[derive(Debug, Clone, Copy)] +pub struct Vector { + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, + pub out: [u32; 16], +} + +/// The 10 canonical 6-round vectors, transcribed verbatim from #903's +/// `tooling/blake3-second-source/blake3-oracle/canonical_6round_vectors.json` (head +/// `89aeeb8c`). Seeds 0..9 of the oracle's generator; `t` is full-width, which +/// matters — the counter-split order is load-bearing and was behaviourally +/// verified against the official crate. +pub const CANONICAL_VECTORS: [Vector; 10] = [ + Vector { + h: [ + 0xD82C07CD, 0x6BAA9455, 0x82E2E662, 0x7A024204, 0xE87A1613, 0x81332876, 0x48268673, + 0xC17C6279, + ], + m: [ + 0xE6F4590B, 0x4F65D4D9, 0xBAD640FB, 0xAF19922A, 0x19C78DF4, 0x6F25E2A2, 0xE9BB17BC, + 0x7A1D5006, 0x42AF9FC3, 0x03983CA8, 0xDE1B372A, 0xDED733E8, 0x9148624F, 0xF7B0B7D2, + 0x72AE2244, 0xEECE328B, + ], + t: 0xB4E1357D4A84EB03, + block_len: 42, + flags: 52, + out: [ + 0xCED9D1FF, 0xC248EEAB, 0xBD109B7F, 0x911B48F6, 0x923D62C0, 0xD804903F, 0x5974223E, + 0xAA4F0C80, 0xAD61007F, 0xB50B8DDB, 0xE7372BE1, 0x33D3D6C3, 0x42AA284B, 0xC5A25F28, + 0x79AC8370, 0xB75F3915, + ], + }, + Vector { + h: [ + 0xC386BBC4, 0x414C343C, 0x7311D8A3, 0xA6CECC1B, 0xC9E9C616, 0x18072E8C, 0xD5F4B3B2, + 0x7204E52D, + ], + m: [ + 0xF1FD42A2, 0xE6C3F339, 0x07D4BEDC, 0x8A9A021E, 0x3BAB6C39, 0x05805975, 0xA46D6753, + 0xDC2574BD, 0xAB99254A, 0x4DA98F1D, 0xE1EA24C4, 0x815A47C5, 0x08D6AF57, 0xCC22AF58, + 0x2C4A3698, 0x5FEC898F, + ], + t: 0xC74803E31BA16215, + block_len: 50, + flags: 94, + out: [ + 0xF2A972E9, 0x81FDB8EC, 0x40C50EBC, 0x4BA1CAF9, 0x9EE9E930, 0x6B1A16B2, 0xE9156F47, + 0xA89FB436, 0xA2F616B3, 0x12874C12, 0x30768035, 0xE01A17D9, 0xBEE5C17C, 0xD61C0BE0, + 0x3041FF46, 0xDFB91125, + ], + }, + Vector { + h: [ + 0x0E7A269F, 0x15BA2BDD, 0xD5E34124, 0x4EE207F8, 0x9B1F282E, 0x9B575BD1, 0xF30B94FA, + 0x0706A045, + ], + m: [ + 0x6148A86F, 0x8697BBD0, 0x8F7D9B78, 0x3C729578, 0x061B9030, 0x533C9135, 0x829E07B0, + 0xE4C11AB2, 0xCBF87544, 0xC34C769F, 0x5A91C89B, 0xF63F23D0, 0xC1066932, 0x87C56473, + 0x7D718D73, 0xECC1CB63, + ], + t: 0x7604E4B4E73695C3, + block_len: 58, + flags: 124, + out: [ + 0x5AA6B114, 0xC9D6740C, 0x8738CAF4, 0xAC5F4B72, 0x9FC6B9DE, 0x3F2EFB8F, 0x8CB7A912, + 0xF497A285, 0x3D062266, 0x7F22380C, 0xAFD468FA, 0x122CBA80, 0x446B156D, 0xB239D8C2, + 0xC3EAB2CF, 0x775F2F92, + ], + }, + Vector { + h: [ + 0x8B529B4A, 0x9A9A80FD, 0xD6645FA9, 0x3BFD1D33, 0x79F248B0, 0x268ECC45, 0xA2863A7F, + 0x85EF3430, + ], + m: [ + 0xBDC2AE99, 0x10645D51, 0x97524D6A, 0xDD933160, 0xE0F9E038, 0xEBCD1F5E, 0xEF829C88, + 0xE0FD67DD, 0x18F2C41C, 0x22CEDAFB, 0x378C74DC, 0x4D100D8F, 0x95C76AB4, 0x95918694, + 0xE779C470, 0xEDCF6109, + ], + t: 0x92D3043AFCF249F3, + block_len: 36, + flags: 31, + out: [ + 0xEED92FAB, 0x138D9358, 0x915BFE3C, 0x13718B01, 0xB506E277, 0xBE4007CD, 0x35847E06, + 0xCE1C6896, 0x52FA01B5, 0x4AA26AF8, 0xB1078A61, 0x2C517AED, 0xA08867A0, 0xEA6ECFEA, + 0x6D33D3B0, 0xDC293166, + ], + }, + Vector { + h: [ + 0x3C6DA5D7, 0x656412A9, 0x27AC435A, 0x11072231, 0xEAFF1A09, 0xC3E1B258, 0x8963DC6E, + 0x1B2ED40E, + ], + m: [ + 0xED6F0B09, 0xCE80C4B0, 0xCCEA2645, 0x3184FF27, 0x4F5253A0, 0xE14B0190, 0x9B191BF4, + 0xABF4A07C, 0x81862FC9, 0x2D83A823, 0x793D0E45, 0x4CDCE7A6, 0xE8ABB93F, 0xE1DF8AF9, + 0x8224B122, 0x69F85E31, + ], + t: 0x49C7B59B995253FD, + block_len: 57, + flags: 41, + out: [ + 0xCA00BDA3, 0x84239A3A, 0xE7C88E6D, 0x33A8A3D6, 0x09DCD1CE, 0xA1B10212, 0xF48E1156, + 0x8F039915, 0x8A055EAA, 0xFF5B11D5, 0xB725085B, 0x2E1AB267, 0x6AE7323D, 0xB2FF6FA8, + 0x7102C8A1, 0x7561EB37, + ], + }, + Vector { + h: [ + 0x9F767C45, 0xBDE5C099, 0xF17FD374, 0xA6233255, 0xE6A16A3B, 0x1CFB10F6, 0x3F1F65A8, + 0x8B33E968, + ], + m: [ + 0x92EDCF45, 0x377B9AA2, 0x478C281D, 0xC4069545, 0xCC11D357, 0x9E115E4B, 0x206F5C66, + 0xDF1461AA, 0xFB7FF337, 0xDF561D80, 0x4A0FE75D, 0xF6236BF2, 0x346C6E2B, 0xB0CDE917, + 0xE4CC4132, 0x4C7D6DF0, + ], + t: 0x6A3753915C76F18A, + block_len: 18, + flags: 67, + out: [ + 0x14A9F66F, 0x101BDFE8, 0x9B0A50DD, 0xEE4BB45B, 0x7A914502, 0x77B3486B, 0x59BFC114, + 0xA1AD2AFD, 0xC194DDE6, 0x894EC54D, 0xAD36C805, 0x9018F3F5, 0x165AF5D8, 0x3E85B598, + 0x78E76653, 0xBB7A485D, + ], + }, + Vector { + h: [ + 0xD26B9496, 0x42F9A039, 0x001D9A88, 0x5F877031, 0xC527E279, 0x45CF8AA4, 0xCD4A5557, + 0xAE9AF169, + ], + m: [ + 0xAF895F5B, 0xD822E2F9, 0x17D7AB26, 0xCCDF540B, 0xCE06294D, 0x4A8B0188, 0xF38D2E64, + 0x5C41D5C5, 0xE8D5B9E3, 0x5C832A51, 0x9A0C1B76, 0x4DE8344E, 0x96D2F9E0, 0x8677A5F2, + 0xA9A967C1, 0x323BBEAF, + ], + t: 0x390567C27BD6AA42, + block_len: 26, + flags: 3, + out: [ + 0x32A6FF70, 0xC30560BC, 0xD1C777C8, 0xF1871821, 0x7207AB54, 0x9F5B83C7, 0xB6561C5D, + 0x991E738F, 0xB38B62B9, 0x0EF6D156, 0x994BECB1, 0x09A85D0E, 0x32221741, 0xADA3CC5F, + 0x5B654ED6, 0x2A7A62B2, + ], + }, + Vector { + h: [ + 0x269E0D37, 0xA6A3A450, 0x892F902B, 0x81E74EF5, 0x099950D8, 0x6F03675A, 0x11E20B8F, + 0x6CAD4A26, + ], + m: [ + 0xF29D0DA9, 0x658CDA14, 0xF9EBDACC, 0xDBC496CB, 0x4A23D596, 0x2E44158B, 0xA38FD547, + 0x5F557203, 0x34B9B5DF, 0x506BF2EF, 0x7403E430, 0x4CBD87AD, 0xCB5C7427, 0x3E7D1BFB, + 0x930D6EAF, 0x86734721, + ], + t: 0x12BD4ACEFAECBD38, + block_len: 53, + flags: 42, + out: [ + 0xA632AD45, 0x12CE41F4, 0xD21B2CBD, 0x76795C62, 0x6BEC36C1, 0xDAFAFCDE, 0x53CA87B7, + 0x92E8465B, 0x7B424F5D, 0xE1E6AD7F, 0x753BA387, 0xCCC50824, 0x69AEDF6D, 0xBBBBF253, + 0x78D04883, 0xF3F33689, + ], + }, + Vector { + h: [ + 0x3A096533, 0xF658F7A7, 0x205738D1, 0xB46EE1DA, 0x15CEB3A1, 0x359B1548, 0xA4517D6C, + 0x7589CA4A, + ], + m: [ + 0x74007CB4, 0xD49D0AC1, 0x16EDC5D4, 0x685CA8AF, 0x4223AA56, 0x10269470, 0x60908405, + 0xA92D04A3, 0x56A3E957, 0xB0F91306, 0xE6C08269, 0xF2306D4A, 0x31A06A7C, 0x9436D6F6, + 0xE18692E2, 0xE0C99F3E, + ], + t: 0x329911DA9FBD8735, + block_len: 19, + flags: 91, + out: [ + 0x913B2AE1, 0xC7F73082, 0x45E1C023, 0x6F1F3F82, 0x20AEE6F5, 0xDAF21D94, 0xF2C1E4AF, + 0xD4F7D4AC, 0x44A45F87, 0xF4C40CE5, 0x613E9B94, 0x08CE53DE, 0x4FF07AA4, 0x456BF2E2, + 0x2066EA7F, 0x3C5A654B, + ], + }, + Vector { + h: [ + 0x5F915EF0, 0x237751AA, 0x01A5BA50, 0x80B65386, 0x14B044D7, 0x61076DC3, 0xB99DE255, + 0x283B73A6, + ], + m: [ + 0x3CEE5E2C, 0x1C670EA9, 0x972651DA, 0x4A8AA593, 0xAC9ABB0C, 0x35BB5C11, 0x47FBB3B4, + 0xCF3C17E5, 0xE2EB17C8, 0xE11E99FB, 0x7DE0D208, 0x0602FE0C, 0x98CAE043, 0x9425B3E2, + 0x33FB4B4F, 0x15607DF9, + ], + t: 0xEAEB999B8A2E547E, + block_len: 64, + flags: 21, + out: [ + 0xF5EE9114, 0x856CABB8, 0x29BE2CF1, 0x603BE91C, 0x94A7DD0E, 0x28FC3E27, 0xB64E2CC8, + 0x2D2C67FF, 0x69FAC1BA, 0x0C949090, 0xD68DE435, 0xCE91A527, 0xE80C1815, 0x6D44EFE6, + 0x87C7B175, 0xD18A8B94, + ], + }, +]; + +/// The same ten inputs as [`CANONICAL_VECTORS`], at **7 rounds** — that is, +/// under standard BLAKE3's compression function. +/// +/// Provenance, and it is a rung stronger than the 6-round table's: these were +/// emitted by the gate-oracle's independently-written Python reference +/// (`thoughts/shared/lfm-real-hash/gate-oracle/blake3_oracle.py`) at +/// `rounds = 7` and cross-checked word-for-word against the second in-repo +/// reference (`tooling/blake3-second-source/blake3-oracle/blake3_ref.py`) — two +/// implementations, agreeing on all ten. Both references' 7-round paths are +/// themselves pinned by the OFFICIAL BLAKE3 test vectors, so unlike +/// [`CANONICAL_VECTORS`] this table has an external anchor rather than one a +/// step removed. The same generation run re-derived the 6-round table and +/// reproduced it 10/10, which is what ties the two together. +/// +/// Only the outputs are stored: the inputs are [`CANONICAL_VECTORS`]'s, and +/// duplicating them would be a second place for them to drift. +pub const CANONICAL_OUT_7ROUND: [[u32; 16]; 10] = [ + [ + 0xEE79E5DC, 0xEA647B8C, 0x964C097E, 0xE2F3383A, 0xFE2E6D00, 0x78EE613A, 0xC33C8572, + 0xCD444391, 0x0C890604, 0xC3209591, 0x45633FF8, 0xCB171C6A, 0x760247AE, 0xF6D0FC1E, + 0xCD550F20, 0xCD54BF83, + ], + [ + 0xD68593D0, 0xDBC8157A, 0xF6E1687C, 0x52A60555, 0xB56D418A, 0x0CCBB863, 0xADBFB51E, + 0x8BF7D125, 0x75C23432, 0xF484D7A6, 0x06E85F4A, 0x2771FE96, 0x00F6E24D, 0x48368A3E, + 0x04EE7E88, 0x501D8539, + ], + [ + 0xBC92D7C4, 0x56542092, 0x3490E2CB, 0x2E3328CD, 0x13E3746F, 0xA5B88E66, 0x2B5FE530, + 0x92C7AD52, 0xFF502AE5, 0x1F088FBF, 0x9163752F, 0x8A0C8B4D, 0xB557B0E8, 0xE76F23CB, + 0xD054C959, 0x74813CFD, + ], + [ + 0xCF4FB929, 0x1DBADE2A, 0x70E63AAF, 0x2E0FFB48, 0x60123045, 0x798AEAE8, 0x5A911D30, + 0x15977C61, 0x6F7C8334, 0x5EB0BCE2, 0xAB240F17, 0x66B7A3CD, 0xA9064E0B, 0x6AC4747B, + 0x1206F62B, 0x9F3E91EC, + ], + [ + 0xFF525F0F, 0xD892E3D2, 0xFB566B40, 0x3BDF4ED0, 0x78B961CD, 0x9CB86B48, 0x6AB54F3D, + 0x3EF5F695, 0xBD896ED8, 0x6265AC08, 0xF6695D78, 0x9F3795EA, 0x943E0342, 0xD1437B3B, + 0x4F6BAF78, 0x85DFD2C9, + ], + [ + 0xD22912BB, 0x627F992C, 0xE883AF5D, 0x50E58A48, 0xF3D071C6, 0xB20D47A4, 0x29011151, + 0xFE50E232, 0x594B76A3, 0x8706296B, 0x2C1D1E31, 0x6A478D0D, 0x64004E61, 0xA072DA1E, + 0xAB3FCA42, 0x09BB269E, + ], + [ + 0xA101CEAB, 0x9232E0EC, 0x2FE4B24E, 0x35F7F4FE, 0x61A5AB42, 0xBE417503, 0xEB740D5E, + 0x8BB2FE96, 0xC6863DA9, 0x1F31FF5D, 0x5763EA12, 0xDC862699, 0x1A60ADE2, 0x9E3E6745, + 0xE3C8F87E, 0xD3EFB0EA, + ], + [ + 0x318604BE, 0x22A35843, 0x6CA63195, 0xA2E7E2F8, 0x48769A04, 0xC462F1E3, 0x5CF053C7, + 0xFD1EE629, 0x69366332, 0x0ACC819B, 0xBBD2456A, 0xF1DA9DB6, 0x4A7B7D68, 0x6DD1A843, + 0x61555466, 0xBDA36F28, + ], + [ + 0x87584719, 0x15C73090, 0x851C1A4A, 0x99D21014, 0x821A82A8, 0xC7307CD5, 0x6797EFE2, + 0xCF38CEDF, 0x777C177D, 0x202BE3EA, 0x19421985, 0x3176132D, 0x7BB8BC22, 0x65C9804B, + 0x22C68EA3, 0x92504162, + ], + [ + 0xDC60D189, 0xE6311F18, 0x9DC3E078, 0x304BB43E, 0x5C616E7D, 0xE168D00F, 0x2E197872, + 0x175B9188, 0x5A99C462, 0xEF311A88, 0xC61836FD, 0x9FFD4DE3, 0x36AE4940, 0x4D813D81, + 0x9B058DA9, 0x9017D38C, + ], +]; diff --git a/crypto/crypto/src/hash/mod.rs b/crypto/crypto/src/hash/mod.rs index 78f89fca3..92f1cc673 100644 --- a/crypto/crypto/src/hash/mod.rs +++ b/crypto/crypto/src/hash/mod.rs @@ -1,3 +1,9 @@ +pub mod blake3; +pub mod platform_blake3; pub mod platform_keccak; pub mod poseidon; +pub mod poseidon1_stark; +pub mod poseidon1_w16; +pub mod poseidon1_w8; +pub mod rpx; pub mod sha3; diff --git a/crypto/crypto/src/hash/platform_blake3.rs b/crypto/crypto/src/hash/platform_blake3.rs new file mode 100644 index 000000000..731f4334e --- /dev/null +++ b/crypto/crypto/src/hash/platform_blake3.rs @@ -0,0 +1,35 @@ +//! [`PlatformBlake3`] — the BLAKE3 byte hash, under the name shape +//! [`PlatformKeccak256`](crate::hash::platform_keccak::PlatformKeccak256) +//! established for a hash that is accelerated on the riscv64 guest and software +//! everywhere else. +//! +//! # This is a re-export, and that is the design +//! +//! `platform_keccak` needs an adapter because the thing it selects between is +//! two different types: `sha3::Keccak256` on the host and a syscall-backed +//! sponge from the syscall crate on the guest. Those two carry their own +//! framing, so something has to give them one `digest` interface. +//! +//! BLAKE3 has no such pair. [`Blake3Chain`](crate::hash::blake3::Blake3Chain) is +//! one type on every target; the accelerator is reached from *inside* it, at the +//! compression function, where `compress_block` selects the syscall on riscv64 +//! and software otherwise. The framing — single chunk, 64-byte blocks, +//! `CHUNK_START` / `CHUNK_END | ROOT`, `t = 0`, the true byte count as the final +//! block's `block_len` (PA-PLAN §1.7) — is above that seam and is therefore the +//! same code on host and guest by construction. +//! +//! INVARIANT (load-bearing): this must remain a **PURE PASSTHROUGH** — a +//! re-export and nothing else. A wrapper type here would be a second place the +//! framing is expressed, which is exactly what PA-PLAN §1.4 forbids and what +//! `executor::vm::instruction::execution`'s duplicate compression already cost +//! us one gating test to contain. It would also break the argument the `TypeId` +//! specializations in `merkle_tree::backends::field_element_vector` rest on: +//! they dispatch on the concrete `PlatformKeccak256` type, so a BLAKE3 `D` +//! reaches the generic `D::new()/update/finalize` path — correct only while +//! `PlatformBlake3` *is* `Blake3Chain` and hashes identically through both. +//! +//! The round count is not selected here either. It is +//! [`BLAKE3_ROUNDS`](crate::hash::blake3::BLAKE3_ROUNDS), one crate-global knob, +//! so a build cannot commit under two hashes. + +pub use crate::hash::blake3::chain::Blake3Chain as PlatformBlake3; diff --git a/crypto/crypto/src/hash/platform_keccak.rs b/crypto/crypto/src/hash/platform_keccak.rs index 2fab3902c..746bc0c95 100644 --- a/crypto/crypto/src/hash/platform_keccak.rs +++ b/crypto/crypto/src/hash/platform_keccak.rs @@ -84,8 +84,10 @@ mod imp { impl Update for PlatformKeccak256 { #[inline(always)] fn update(&mut self, data: &[u8]) { - // Track absorbed bytes (for the finalize permutation count) and the - // host-side absorption stats. + // Absorption — the guest's dominant keccak cost (many small + // `stream_bytes` absorbs), which no finalize counter would see. The + // running byte total is what lets `finalize` report a permutation + // count as well. self.1 += data.len(); crate::hash_metrics::count_absorb(data.len()); Update::update(&mut self.0, data); diff --git a/crypto/crypto/src/hash/poseidon1_stark/mod.rs b/crypto/crypto/src/hash/poseidon1_stark/mod.rs new file mode 100644 index 000000000..76ba65f59 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_stark/mod.rs @@ -0,0 +1,322 @@ +//! The hashing of a Poseidon1 base STARK, constructed exactly as ZisK +//! (pil2-proofman v1.3.0-alpha, `Poseidon1` hash family) constructs it. +//! +//! ⚠ **Not a shipped hash**: an exploration-branch instance for a like-for-like +//! comparison with ZisK. Nothing commits with it by default. +//! +//! # The instance (every item checked against ZisK's own code, `zisk_kat.rs`) +//! +//! | primitive | construction | +//! |---|---| +//! | permutation | [`super::poseidon1_w16::permute`]: Goldilocks, `x^7`, `R_F = 8`, `R_P = 22`, Grain constants, Plonky3's small circulant MDS | +//! | leaf | [`linear_hash`]: rate 12; the first block's capacity is the width tag [`leaf_capacity`] `[len, LEAF_DOMAIN, 0, 0]`, every later block sees the previous output's lanes `0..4` there. ZisK's own leaf has a zero first capacity ([`zisk_linear_hash`], the KATs' reference) | +//! | node | [`super::poseidon1_w16::compress4`]: the four child digests in child order, permuted, truncated to lanes `0..4` | +//! | tree | arity 4; a level whose length is not a multiple of 4 is padded with zero digests ([`Merkle4`]) | +//! | transcript | [`Transcript`]: an overwrite sponge, rate 12, the previous state's lanes `0..4` as capacity, every lane squeezed | +//! | grinding | [`grinding_lane0`]: the WIDTH-8 permutation of `[c0, c1, c2, nonce, 0, 0, 0, 0]`, lane 0 below `2^(64 − bits)` | +//! +//! The leaf is the one place this instance departs from ZisK's: its first block's +//! capacity carries the leaf's width and a leaf domain (the review's width tag, +//! REV-P1-JUDGE §3.4), as RPX's leaf capacity does. So a leaf binds its own +//! width: a leaf of `n` felts and the same leaf zero-padded hash apart, and a +//! 12-felt leaf is not a node whose fourth child is zero. It costs no +//! permutation (the first block's capacity lanes are otherwise zero). The node +//! and the transcript carry no tag: the verifier fixes every tree's depth and +//! every absorb schedule. + +#[cfg(test)] +mod tests; +#[cfg(test)] +mod zisk_kat; + +use alloc::vec::Vec; + +use super::poseidon1_w8; +use super::poseidon1_w16::{self, ARITY, DIGEST_FELTS, Digest, Fp, compress4, permute}; + +const WIDTH: usize = poseidon1_w16::STATE_FELTS; +const RATE: usize = poseidon1_w16::RATE_FELTS; + +/// Felts of one Merkle path level: the three siblings, in child order. +pub const SIBLING_FELTS: usize = (ARITY - 1) * DIGEST_FELTS; + +/// The leaf domain a leaf's first-block capacity carries: `"P1WL"` +/// ([`poseidon1_w16::DOMAIN_LEAF`]), distinct from RPX's. +pub const LEAF_DOMAIN: u64 = poseidon1_w16::DOMAIN_LEAF; + +/// ★ The width tag: the first block's capacity of a leaf of `num_felts` felts, +/// `[num_felts, LEAF_DOMAIN, 0, 0]`. It mirrors RPX's `leaf_capacity` with the +/// whole length rather than its residue, so two leaf widths never share a +/// first block. +pub fn leaf_capacity(num_felts: usize) -> Digest { + [ + Fp::from(num_felts as u64), + Fp::from(LEAF_DOMAIN), + Fp::zero(), + Fp::zero(), + ] +} + +/// The leaf hash: ZisK's (`linear_hash_seq` at width 16) with the width tag. +/// +/// Each block of up to 12 felts overwrites the rate (zero-filled past its end); +/// the capacity is [`leaf_capacity`] for the first block and the previous +/// output's digest lanes `0..4` for every later one. The digest is lanes `0..4` +/// of the last output. An empty input never permutes and hashes to the zero +/// digest. +pub fn linear_hash(felts: &[Fp]) -> Digest { + linear_hash_from(felts, leaf_capacity(felts.len())) +} + +/// ZisK's own leaf hash: [`linear_hash`] with a zero first capacity. The +/// reference ZisK's known-answer vectors pin; no commitment uses it. +pub fn zisk_linear_hash(felts: &[Fp]) -> Digest { + linear_hash_from(felts, [Fp::zero(); DIGEST_FELTS]) +} + +/// The rate-12 chain with `first` as the first block's capacity. +fn linear_hash_from(felts: &[Fp], first: Digest) -> Digest { + let mut state = [Fp::zero(); WIDTH]; + state[RATE..].copy_from_slice(&first); + for (k, block) in felts.chunks(RATE).enumerate() { + if k > 0 { + let carry = [state[0], state[1], state[2], state[3]]; + state[RATE..].copy_from_slice(&carry); + } + state[..RATE].fill(Fp::zero()); + state[..block.len()].copy_from_slice(block); + state = permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// A 4-ary Merkle tree over leaf digests, every level kept. +pub struct Merkle4 { + /// Bottom first; every level but the root padded with zero digests to a + /// multiple of 4. + levels: Vec>, + /// Leaves before padding. + leaves: usize, +} + +impl Merkle4 { + /// Builds the tree. An empty leaf set has no tree. + pub fn new(leaves: &[Digest]) -> Option { + if leaves.is_empty() { + return None; + } + let mut levels = vec![leaves.to_vec()]; + loop { + let level = levels.last_mut().expect("levels starts non-empty"); + if level.len() == 1 { + break; + } + level.resize( + level.len().div_ceil(ARITY) * ARITY, + [Fp::zero(); DIGEST_FELTS], + ); + let parents = level + .chunks_exact(ARITY) + .map(|c| compress4(&[c[0], c[1], c[2], c[3]])) + .collect(); + levels.push(parents); + } + Some(Self { + levels, + leaves: leaves.len(), + }) + } + + /// The root. + pub fn root(&self) -> Digest { + self.levels.last().expect("a tree has a root")[0] + } + + /// Levels from leaf to root, i.e. the length of every path. + pub fn depth(&self) -> usize { + self.levels.len() - 1 + } + + /// The path of leaf `index`: per level, its three siblings in child order. + /// A padding position is not a leaf. + pub fn path(&self, index: usize) -> Option> { + if index >= self.leaves { + return None; + } + let mut i = index; + let path = self.levels[..self.depth()] + .iter() + .map(|level| { + let first = i / ARITY * ARITY; + let mut siblings = [Fp::zero(); SIBLING_FELTS]; + let mut slots = siblings.chunks_exact_mut(DIGEST_FELTS); + for c in (first..first + ARITY).filter(|&c| c != i) { + slots + .next() + .expect("three siblings") + .copy_from_slice(&level[c]); + } + i /= ARITY; + siblings + }) + .collect(); + Some(path) + } +} + +/// Recomputes the root from leaf `index`'s digest and path, as ZisK's +/// `calculate_root_from_proof` does, and compares it with `root`. +/// +/// Unlike ZisK's `verify_mt` it also rejects an index past the tree, so one leaf +/// has one path. +pub fn verify_path4( + root: &Digest, + leaf: &Digest, + index: usize, + path: &[[Fp; SIBLING_FELTS]], +) -> bool { + let mut node = *leaf; + let mut i = index; + for siblings in path { + let at = i % ARITY; + let mut children = [[Fp::zero(); DIGEST_FELTS]; ARITY]; + let mut sib = siblings.chunks_exact(DIGEST_FELTS); + for (c, child) in children.iter_mut().enumerate() { + if c == at { + *child = node; + } else { + child.copy_from_slice(sib.next().expect("three siblings")); + } + } + node = compress4(&children); + i /= ARITY; + } + i == 0 && node == *root +} + +/// ZisK's Fiat–Shamir transcript over the width-16 permutation +/// (`Transcript`). +/// +/// Absorbed felts collect in a 12-felt buffer; a full buffer, or a squeeze after +/// an absorb, permutes `buffer (zero-filled) ‖ state[0..4]`. A squeeze reads the +/// output's lanes `0, 1, …, 15` in order and permutes again once all sixteen are +/// read. +#[derive(Clone)] +pub struct Transcript { + state: [Fp; WIDTH], + pending: [Fp; RATE], + pending_len: usize, + /// Output lanes not yet squeezed; 0 forces a permutation first. + unread: usize, +} + +impl Default for Transcript { + fn default() -> Self { + Self::new() + } +} + +impl Transcript { + /// An empty transcript. + pub fn new() -> Self { + Self { + state: [Fp::zero(); WIDTH], + pending: [Fp::zero(); RATE], + pending_len: 0, + unread: 0, + } + } + + fn update(&mut self) { + let mut input = [Fp::zero(); WIDTH]; + input[..self.pending_len].copy_from_slice(&self.pending[..self.pending_len]); + input[RATE..].copy_from_slice(&self.state[..WIDTH - RATE]); + self.state = permute(input); + self.pending_len = 0; + self.unread = WIDTH; + } + + /// Absorbs one felt. + pub fn put1(&mut self, x: Fp) { + self.pending[self.pending_len] = x; + self.pending_len += 1; + self.unread = 0; + if self.pending_len == RATE { + self.update(); + } + } + + /// Absorbs felts in order. + pub fn put(&mut self, xs: &[Fp]) { + for x in xs { + self.put1(*x); + } + } + + /// Squeezes one felt. + pub fn squeeze1(&mut self) -> Fp { + if self.unread == 0 { + self.update(); + } + let x = self.state[WIDTH - self.unread]; + self.unread -= 1; + x + } + + /// Squeezes an extension-field challenge: three consecutive felts. + pub fn field(&mut self) -> [Fp; 3] { + [self.squeeze1(), self.squeeze1(), self.squeeze1()] + } + + /// ZisK's `get_state`: flushes a non-empty buffer, then returns the state. + /// It does not reset the squeeze position. + pub fn state(&mut self) -> [Fp; WIDTH] { + if self.pending_len > 0 { + self.update(); + } + self.state + } + + /// ZisK's `get_permutations`: `n` integers of `n_bits` bits each, read + /// least-significant bit first from the low 63 bits of consecutive squeezed + /// felts. `n_bits` must be at most 63. + pub fn indices(&mut self, n: usize, n_bits: usize) -> Vec { + assert!( + n > 0 && (1..=63).contains(&n_bits), + "indices: n {n}, n_bits {n_bits}" + ); + let felts: Vec = (0..(n * n_bits - 1) / 63 + 1) + .map(|_| self.squeeze1().canonical()) + .collect(); + let mut bit = 0usize; + (0..n) + .map(|_| { + let mut v = 0u64; + for j in 0..n_bits { + v |= ((felts[bit / 63] >> (bit % 63)) & 1) << j; + bit += 1; + } + v + }) + .collect() + } +} + +/// Lane 0 (canonical) of the width-8 permutation of `[c0, c1, c2, nonce, 0, 0, +/// 0, 0]`: ZisK's proof-of-work hash. +pub fn grinding_lane0(challenge: &[Fp; 3], nonce: &Fp) -> u64 { + let mut s = [Fp::zero(); poseidon1_w8::STATE_FELTS]; + s[..3].copy_from_slice(challenge); + s[3] = *nonce; + let out = poseidon1_w8::permute(s); + out[0].canonical() +} + +/// Does `nonce` meet `bits` of proof of work for `challenge`? +pub fn grinding_ok(challenge: &[Fp; 3], nonce: &Fp, bits: u32) -> bool { + match bits { + 0 => true, + 1..=63 => grinding_lane0(challenge, nonce) < 1u64 << (64 - bits), + _ => false, + } +} diff --git a/crypto/crypto/src/hash/poseidon1_stark/tests.rs b/crypto/crypto/src/hash/poseidon1_stark/tests.rs new file mode 100644 index 000000000..b4df71344 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_stark/tests.rs @@ -0,0 +1,303 @@ +//! Every construction against ZisK's own implementation ([`super::zisk_kat`], +//! generated by `scripts/poseidon1/zisk_crosscheck` from pil2-proofman's +//! `proofman-fields`), and the checks that make the vectors bind. + +use alloc::vec::Vec; + +use super::zisk_kat::{ + GRIND_BITS, GRIND_LANE0, GRIND_SMALLEST_NONCE, LINEAR_HASH, MERKLE4_ROOTS, PATH_ROOT, + PATH_ROWS, PATH_WIDTH, PATHS, PERM8, PERM16, SEED_GRIND, SEED_LEAF, SEED_ROWS, SEED_TRANSCRIPT, + SEED_TREE, TRANSCRIPT_OUT, TRANSCRIPT_QUERIES, +}; +use super::*; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// The generator's LCG (`p1_params.py kat_inputs`), reduced mod `p`. +pub(super) fn felts(seed: u64, n: usize) -> Vec { + let mut x = seed; + (0..n) + .map(|_| { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + Fp::from(x % P) + }) + .collect() +} + +fn canon(v: &[Fp]) -> Vec { + v.iter().map(|f| f.canonical()).collect() +} + +fn digests(seed: u64, n: usize) -> Vec { + felts(seed, 4 * n) + .chunks_exact(4) + .map(|c| [c[0], c[1], c[2], c[3]]) + .collect() +} + +#[test] +fn the_width_16_permutation_is_zisks() { + for (input, expected) in PERM16.iter() { + let s = core::array::from_fn(|i| Fp::from(input[i])); + assert_eq!(canon(&permute(s)), expected.to_vec()); + } +} + +#[test] +fn the_width_8_permutation_is_zisks() { + for (input, expected) in PERM8.iter() { + let s = core::array::from_fn(|i| Fp::from(input[i])); + assert_eq!(canon(&poseidon1_w8::permute(s)), expected.to_vec()); + } +} + +#[test] +fn the_leaf_hash_is_zisks() { + for (len, expected) in LINEAR_HASH.iter() { + assert_eq!( + canon(&zisk_linear_hash(&felts(SEED_LEAF, *len))), + expected.to_vec(), + "len {len}" + ); + } +} + +#[test] +fn the_leaf_vectors_pin_the_digest_carry() { + // Mutation: a textbook sponge carries the capacity lanes 12..16 between + // blocks; ZisK carries the digest lanes 0..4 into them. Every multi-block + // vector must tell the two apart, and every one-block vector must not. + let sponge = |felts: &[Fp]| { + let mut state = [Fp::zero(); WIDTH]; + for block in felts.chunks(RATE) { + state[..RATE].fill(Fp::zero()); + state[..block.len()].copy_from_slice(block); + state = permute(state); + } + [state[0], state[1], state[2], state[3]] + }; + let mut multi = 0; + for (len, expected) in LINEAR_HASH.iter() { + let differs = canon(&sponge(&felts(SEED_LEAF, *len))) != expected.to_vec(); + assert_eq!(differs, *len > RATE, "len {len}"); + multi += usize::from(*len > RATE); + } + assert!(multi >= 5, "too few multi-block vectors: {multi}"); +} + +#[test] +fn the_tree_roots_are_zisks() { + for (n, expected) in MERKLE4_ROOTS.iter() { + let tree = Merkle4::new(&digests(SEED_TREE, *n)).expect("non-empty"); + assert_eq!(canon(&tree.root()), expected.to_vec(), "{n} leaves"); + } + assert!(Merkle4::new(&[]).is_none()); +} + +#[test] +fn the_tree_vectors_pin_the_zero_padding() { + // Mutation: padding a short level by repeating its last digest instead of + // zero digests must change every root that pads. + for (n, expected) in MERKLE4_ROOTS.iter() { + let mut level = digests(SEED_TREE, *n); + while level.len() > 1 { + while !level.len().is_multiple_of(ARITY) { + level.push(*level.last().expect("non-empty")); + } + level = level + .chunks_exact(ARITY) + .map(|c| compress4(&[c[0], c[1], c[2], c[3]])) + .collect(); + } + let no_padding = n.is_power_of_two() && n.trailing_zeros() % 2 == 0; + assert_eq!( + canon(&level[0]) == expected.to_vec(), + no_padding, + "{n} leaves" + ); + } +} + +#[test] +fn the_paths_are_zisks_and_verify() { + let rows: Vec> = felts(SEED_ROWS, PATH_ROWS * PATH_WIDTH) + .chunks_exact(PATH_WIDTH) + .map(<[Fp]>::to_vec) + .collect(); + let leaves: Vec = rows.iter().map(|r| zisk_linear_hash(r)).collect(); + let tree = Merkle4::new(&leaves).expect("non-empty"); + assert_eq!(canon(&tree.root()), PATH_ROOT.to_vec()); + let root = tree.root(); + for (index, expected) in PATHS.iter() { + let path = tree.path(*index).expect("in range"); + assert_eq!(path.len(), expected.len()); + for (level, want) in path.iter().zip(expected.iter()) { + assert_eq!(canon(level), want.to_vec(), "index {index}"); + } + assert!(verify_path4(&root, &leaves[*index], *index, &path)); + } +} + +#[test] +fn a_tampered_path_is_rejected() { + let leaves = digests(SEED_TREE, 21); + let tree = Merkle4::new(&leaves).expect("non-empty"); + let root = tree.root(); + for index in [0, 7, 20] { + let path = tree.path(index).expect("in range"); + assert!(verify_path4(&root, &leaves[index], index, &path)); + for level in 0..path.len() { + for felt in 0..SIBLING_FELTS { + let mut bad = path.clone(); + bad[level][felt] += Fp::one(); + assert!(!verify_path4(&root, &leaves[index], index, &bad)); + } + } + let mut leaf = leaves[index]; + leaf[3] += Fp::one(); + assert!(!verify_path4(&root, &leaf, index, &path)); + // The same path cannot open a neighbouring index, nor one past the + // tree that aliases this one modulo 4^depth. + assert!(!verify_path4(&root, &leaves[index], index ^ 1, &path)); + let alias = index + ARITY.pow(tree.depth() as u32); + assert!(!verify_path4(&root, &leaves[index], alias, &path)); + assert!(!verify_path4(&root, &leaves[index], index, &path[1..])); + } + assert!(tree.path(21).is_none()); +} + +#[test] +fn the_transcript_is_zisks() { + let input = felts(SEED_TRANSCRIPT, 4 + 4 + 4 + 12 + 30); + let (a, rest) = input.split_at(4); + let (b, rest) = rest.split_at(4); + let (c, rest) = rest.split_at(4); + let (d, e) = rest.split_at(12); + let mut t = Transcript::new(); + let mut out = Vec::new(); + t.put(a); + t.put(b); + out.extend(t.field()); + out.extend(t.field()); + t.put(c); + out.extend(t.field()); + t.put(d); + out.extend(t.field()); + t.put(e); + for _ in 0..6 { + out.extend(t.field()); + } + out.extend(t.state()); + assert_eq!(canon(&out), TRANSCRIPT_OUT.to_vec()); + + let last = &out[out.len() - 16 - 3..out.len() - 16]; + let mut q = Transcript::new(); + q.put(last); + q.put1(Fp::from(12345u64)); + assert_eq!(q.indices(10, 21), TRANSCRIPT_QUERIES.to_vec()); +} + +#[test] +fn the_transcript_binds_every_absorbed_felt() { + // Changing any one absorbed felt changes the first challenge squeezed. + let input = felts(SEED_TRANSCRIPT, 30); + let first = |xs: &[Fp]| { + let mut t = Transcript::new(); + t.put(xs); + t.field() + }; + let honest = first(&input); + for i in 0..input.len() { + let mut bad = input.clone(); + bad[i] += Fp::one(); + assert_ne!(first(&bad), honest, "felt {i}"); + } +} + +#[test] +fn the_grinding_hash_is_zisks() { + let ch = felts(SEED_GRIND, 3); + let ch = [ch[0], ch[1], ch[2]]; + for (nonce, lane0) in GRIND_LANE0.iter() { + assert_eq!(grinding_lane0(&ch, &Fp::from(*nonce)), *lane0); + } + let smallest = (0u64..) + .find(|n| grinding_ok(&ch, &Fp::from(*n), GRIND_BITS)) + .expect("a nonce exists"); + assert_eq!(smallest, GRIND_SMALLEST_NONCE); + assert!(!grinding_ok( + &ch, + &Fp::from(GRIND_SMALLEST_NONCE - 1), + GRIND_BITS + )); + assert!(grinding_ok(&ch, &Fp::from(0u64), 0)); + assert!(!grinding_ok(&ch, &Fp::from(GRIND_SMALLEST_NONCE), 64)); +} + +/// The width tag (REV-P1-JUDGE §3.4): the leaf hash is ZisK's chain with the +/// first block's capacity `[len, LEAF_DOMAIN, 0, 0]`, computed here from the bare +/// permutation, and it differs from ZisK's untagged hash at every width. Pinned +/// at two widths. +#[test] +fn the_leaf_hash_carries_the_width_tag() { + for len in [1usize, 5, 11, 12, 13, 24, 25, 36, 41] { + let x = felts(SEED_LEAF ^ 0x7a6, len); + let mut state = [Fp::zero(); WIDTH]; + state[RATE] = Fp::from(len as u64); + state[RATE + 1] = Fp::from(LEAF_DOMAIN); + for (k, block) in x.chunks(RATE).enumerate() { + if k > 0 { + let carry = [state[0], state[1], state[2], state[3]]; + state[RATE..].copy_from_slice(&carry); + } + state[..RATE].fill(Fp::zero()); + state[..block.len()].copy_from_slice(block); + state = permute(state); + } + let want = [state[0], state[1], state[2], state[3]]; + assert_eq!(linear_hash(&x), want, "len {len}"); + assert_ne!( + linear_hash(&x), + zisk_linear_hash(&x), + "len {len}: the tag moves it" + ); + } + assert_eq!( + linear_hash(&[]), + [Fp::zero(); DIGEST_FELTS], + "an empty leaf never permutes" + ); + assert_eq!(LEAF_DOMAIN, u64::from(u32::from_le_bytes(*b"P1WL"))); + for (len, want) in TAGGED_LEAF { + assert_eq!( + canon(&linear_hash(&felts(SEED_LEAF, len))), + want.to_vec(), + "len {len}" + ); + } +} + +/// [`the_leaf_hash_carries_the_width_tag`]'s pinned digests over +/// `felts(SEED_LEAF, len)`. +const TAGGED_LEAF: [(usize, [u64; 4]); 2] = [ + ( + 12, + [ + 4458286375898503949, + 6509762154028324312, + 350313438890380232, + 5326189538293941466, + ], + ), + ( + 25, + [ + 14038525980236200321, + 10177624454212016161, + 9232529760025479834, + 85431238162842744, + ], + ), +]; diff --git a/crypto/crypto/src/hash/poseidon1_stark/zisk_kat.rs b/crypto/crypto/src/hash/poseidon1_stark/zisk_kat.rs new file mode 100644 index 000000000..eefbf0eb6 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_stark/zisk_kat.rs @@ -0,0 +1,766 @@ +//! Known answers computed by ZisK's own Rust implementation of its Poseidon1 +//! instance: `proofman-fields` of pil2-proofman v1.3.0-alpha (d485fac2, MIT OR +//! Apache-2.0). +//! +//! GENERATED by `scripts/poseidon1/zisk_crosscheck`; do not edit by hand. Inputs are +//! the LCG stream of `tests::felts`, so only the outputs are stored. + +/// `(input, permute(input))` at width 16: `[0..16)`, zeros, `p − 1`, LCG. +pub const PERM16: [([u64; 16], [u64; 16]); 4] = [ + ( + [ + 0x0000000000000000, + 0x0000000000000001, + 0x0000000000000002, + 0x0000000000000003, + 0x0000000000000004, + 0x0000000000000005, + 0x0000000000000006, + 0x0000000000000007, + 0x0000000000000008, + 0x0000000000000009, + 0x000000000000000a, + 0x000000000000000b, + 0x000000000000000c, + 0x000000000000000d, + 0x000000000000000e, + 0x000000000000000f, + ], + [ + 0x81c2ff551d3dd1a3, + 0xa6f3ddabab7998e2, + 0x4372186243233825, + 0xd2bd8442c6cc6df7, + 0x051a796f67578f23, + 0x3b597e26481062ca, + 0x19c3c48645baaabb, + 0x7e142fc8bf48c2ce, + 0x599ca659bfbf033f, + 0x84e132ca4afd703d, + 0xb758d5776f5185c3, + 0xaf58bfc9cb74204e, + 0x7015309157ec7e9c, + 0xe57e7f42acfff2e0, + 0x57043250e11a11bb, + 0x656c21727540ab90, + ], + ), + ( + [ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + [ + 0x078a54a5999c1f89, + 0xba0c619c6e9a0ff0, + 0xe2f9a11694354835, + 0xd2c8e968b04cc9a6, + 0x81d0e75f2327654b, + 0x20912557baff31b0, + 0x69bc2fc6d13e33cd, + 0x032c5a376d7cfc13, + 0xecef36c7bccf4d56, + 0x80b4817b062829dc, + 0xf93659d1793a7952, + 0x1f0dc30f44cb3138, + 0x8b564149bfa10efa, + 0xc7b30100325f4879, + 0xf694a1841608a1e8, + 0xc595e9d1f1914be9, + ], + ), + ( + [ + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + ], + [ + 0x491ef464b2792bd3, + 0x28450225e7342b0e, + 0x1299abb383c26dbe, + 0x96056d60d5b031ca, + 0xb6efaa51f392fb67, + 0x8e428e525552bc22, + 0xcda509fb6d800175, + 0x4b641572c6984696, + 0x01f93fc0af917f75, + 0xf28ed5d932aeee76, + 0x9d2467d3ac8a6c3a, + 0x6c6dc438a4757fd1, + 0xc927d1ffd408297a, + 0x4ed8f3f9228f45ee, + 0x469eb77a91504639, + 0x1e5a53837d729653, + ], + ), + ( + [ + 0x2ceaee21bf46bc00, + 0xaa80754d1a1a8d4f, + 0xb3c4904a6d278932, + 0xbc69cf4276846d19, + 0x377b2fd56a5b15b4, + 0x64d815deeaf29df3, + 0xf66e100db2d7d206, + 0x1069e6a57e06665d, + 0x7be902917b70a2a8, + 0x68901baf16ad70d7, + 0xf2caa7c38002001a, + 0x5a55db213cf06be1, + 0x4c88c014892416dc, + 0xa0c1ac9a9822a9fb, + 0x9e5c77e75ae9e76e, + 0x37e83672575ac1a5, + ], + [ + 0xb077e251291e3c50, + 0x802ac7ee5069af14, + 0x3b40e1590405fd9e, + 0xcfa2bc2aec5966c6, + 0x554dd87f58bdb066, + 0xc11dcdf422d22bb0, + 0x6b6bcf24fa68d8bf, + 0x9cafadebd618a6c3, + 0xf2fd3eacc96f8243, + 0x6fd97ccffe5b67f1, + 0xfbc70f8784230214, + 0x6d96b2b2a82fd02e, + 0x4c73dc266ce6f482, + 0x83d58cae06543d83, + 0x326b8c8355c4bf52, + 0x63ad58b9fcdcb60d, + ], + ), +]; + +/// `(input, permute(input))` at width 8 (the grinding permutation). +pub const PERM8: [([u64; 8], [u64; 8]); 4] = [ + ( + [ + 0x0000000000000000, + 0x0000000000000001, + 0x0000000000000002, + 0x0000000000000003, + 0x0000000000000004, + 0x0000000000000005, + 0x0000000000000006, + 0x0000000000000007, + ], + [ + 0x21bd740749dd30d7, + 0x82d56e9f77070d02, + 0x5ec0547e3dbc8dcc, + 0xeaa429642119fae9, + 0x8c9b30f79af9b031, + 0xf224d564607084bd, + 0x7dd2af74636e87d6, + 0xa7daeb4dc21c22d7, + ], + ), + ( + [ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + [ + 0x967b858fdabc48f5, + 0xad350bc2b4a0e728, + 0x2df1898fef94687a, + 0x03615aabe4616577, + 0x964840faf7231998, + 0x2c23d347f3c7cddb, + 0xc3c52c0cb7d2776c, + 0xa1021e181fe6ea58, + ], + ), + ( + [ + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + ], + [ + 0x00f79505ba6ae588, + 0xa94c3a530e38c062, + 0xc33cd2c7bad60fa5, + 0x9eab666a9418b97c, + 0xba59588e735134f7, + 0xe20003ce19565758, + 0xdd6e988c7e479219, + 0xa7f53d8b009dd2e7, + ], + ), + ( + [ + 0x2ceaee21bf46bc00, + 0xaa80754d1a1a8d4f, + 0xb3c4904a6d278932, + 0xbc69cf4276846d19, + 0x377b2fd56a5b15b4, + 0x64d815deeaf29df3, + 0xf66e100db2d7d206, + 0x1069e6a57e06665d, + ], + [ + 0x6f4b8bcff6c46360, + 0x55fa4e327ba9a074, + 0x64f61d05b00bb015, + 0x503249cfdf8949a2, + 0xee0c105d8e205184, + 0xa296e0f105460d27, + 0x2a1dd0e592a7148b, + 0x3796cef45525bcec, + ], + ), +]; + +/// Seed of the leaf inputs: `zisk_linear_hash(felts(SEED_LEAF, len))`. +pub const SEED_LEAF: u64 = 0x1eaf; +/// `(len, digest)`: the width-16 linear hash (ZisK's leaf hash). +pub const LINEAR_HASH: [(usize, [u64; 4]); 15] = [ + ( + 1, + [ + 0x7d03e575421c5b23, + 0xe7205b8b16d02cae, + 0xe7403af7b190c9b8, + 0x33dce167235a4161, + ], + ), + ( + 3, + [ + 0x6640aacb19b99fef, + 0x8cfa8c6eded38947, + 0x983df120f9673f0e, + 0x1310238c565b68c3, + ], + ), + ( + 4, + [ + 0x5bbe522545759068, + 0x8be0c21c95272122, + 0x4c1d411056f08119, + 0xfd4884cefaae238e, + ], + ), + ( + 5, + [ + 0xf9e35072a2cbb968, + 0x00eecaa4551e4bc6, + 0x2967536c85b1e981, + 0xcc0e4f974a68a925, + ], + ), + ( + 8, + [ + 0x37699b63c7c41ff8, + 0x9b77ac0ed9cfa6ba, + 0xe8e79112528274fa, + 0x0c97bd2a1719f7a9, + ], + ), + ( + 11, + [ + 0x2fb335f8ba81a625, + 0xc60103e3cd048002, + 0x6ad161bff910fe7f, + 0xda3990535144a065, + ], + ), + ( + 12, + [ + 0x56d3b861d55fd314, + 0xa5a31a15748ecf02, + 0x7b8a70ffc050b14a, + 0x1ea870b034c04761, + ], + ), + ( + 13, + [ + 0x970e5d8788b6ff13, + 0x662a81f429cf5db9, + 0x4b0b0ab73c68d61d, + 0x9f39ab57a883f85a, + ], + ), + ( + 23, + [ + 0xb24ab0a82ce7788f, + 0x01e278923e6f556a, + 0x1d5fcb3b6c26b2df, + 0xb854b08d821532a7, + ], + ), + ( + 24, + [ + 0x94f5a3dd9e653ad2, + 0x3170c14426de671b, + 0x2b2be28f1e5de7e6, + 0x121dc669632d4af2, + ], + ), + ( + 25, + [ + 0x92f4915f35848d9f, + 0x23d820916e1a5313, + 0xde776805d1a21bf7, + 0x9e2ecd73e3fcced1, + ], + ), + ( + 36, + [ + 0x042694abd8c60e18, + 0x64e6ea0ecb0e3dac, + 0x089c1f4d45e9b9b7, + 0xa13674c9fe4c3eed, + ], + ), + ( + 64, + [ + 0x8794e906c2322944, + 0x0e3e180c1411cf2a, + 0xd4f728e29cf2838f, + 0x5972df3e1ef71b06, + ], + ), + ( + 100, + [ + 0x61ca36b7bf6d6dd3, + 0x54822101148cfe34, + 0x019fda1d3ba125c2, + 0xff29312691d434eb, + ], + ), + ( + 245, + [ + 0x7c9f951fe2d785d0, + 0x1a234f39f5f1997d, + 0xfa9e991c9153c45c, + 0x17e9375cff057d09, + ], + ), +]; + +/// Seed of the tree inputs: leaf digest `i` is `felts(SEED_TREE, 4·n)[4i..4i+4]`. +pub const SEED_TREE: u64 = 0x7eee; +/// `(leaves, root)`: `partial_merkle_tree` at arity 4. +pub const MERKLE4_ROOTS: [(usize, [u64; 4]); 9] = [ + ( + 1, + [ + 0x8664c94dbefbe325, + 0x30089173afe6c9d0, + 0x04f90fe597272adf, + 0x0b7edc901f59ab82, + ], + ), + ( + 2, + [ + 0x2eb364ea6cdadf19, + 0xccb753f846d8a40a, + 0xf4f07ab8162431e4, + 0x3c5ee032d67fdf59, + ], + ), + ( + 3, + [ + 0x86e1acde79585807, + 0x9abd8ea8b7cbe033, + 0x11362fa741f967a5, + 0x84b6f84e20992320, + ], + ), + ( + 4, + [ + 0xe4439b945904eb77, + 0x0cd8326d455ffdb1, + 0x54b2f128a7109cf2, + 0xcf984c5f8ad6848f, + ], + ), + ( + 5, + [ + 0x20ac5f24df34a743, + 0x3087e607ce719605, + 0x5db55fcbca8a11e2, + 0x983403c51793fa1a, + ], + ), + ( + 16, + [ + 0x5cb2c8680856be1b, + 0xcacd01bd16ca97bd, + 0xa929172d0b2e569f, + 0x0f0511a671a5dd95, + ], + ), + ( + 17, + [ + 0x199efcaf762e1698, + 0xa6a300a7f88b0629, + 0xa495edf09b591039, + 0x9fcfb670ed6704b9, + ], + ), + ( + 21, + [ + 0x28690a55266d917a, + 0xeb6bec957c51abc5, + 0x571cbe2b6f5c05f2, + 0xf101f6580769df1f, + ], + ), + ( + 64, + [ + 0x37efafed6fd1bf30, + 0xc1369a094e1dc939, + 0x5cea7a4244a6e116, + 0xdf140e80cc1b026e, + ], + ), +]; + +/// Seed of the path rows: `21` rows of `7` felts, `felts(SEED_ROWS, ·)` row-major. +pub const SEED_ROWS: u64 = 0x2075; +pub const PATH_ROWS: usize = 21; +pub const PATH_WIDTH: usize = 7; +/// The root over the rows' linear hashes. +pub const PATH_ROOT: [u64; 4] = [ + 0x23f761a0cb6ff00c, + 0x052127d1e609285b, + 0x78609f05b5b25a69, + 0x364c6d50e3240899, +]; +/// `(index, siblings)`: per level, the three other children in child order (12 felts), +/// as ZisK's `verify_mt` reads them; every path here passes it. +pub const PATHS: [(usize, [[u64; 12]; 3]); 4] = [ + ( + 0, + [ + [ + 0x6ebf87658577973b, + 0xb2af426aa2df3d12, + 0x646457bf9de8f505, + 0xbd423e46c0b7c70e, + 0xdc3e47b347cdde6d, + 0x8c557853b1ae4858, + 0x25a1799325aa3076, + 0xec5f1caa799417d2, + 0x690cf267be58acff, + 0xed5b61bdf0c7addc, + 0xda3abba88dc03526, + 0x06ef8ab52eeaae95, + ], + [ + 0xdaf7006956ca017a, + 0x097dd24a97ba5646, + 0xad5994585e0cd6e2, + 0x1ba415ba1aa8761d, + 0xd2b7f30720d8ee9f, + 0x89a016d3ff144dac, + 0x9517c5102664d681, + 0x173fdf02efed4246, + 0x95761bd58616b352, + 0x2cf97fe34ea40dc2, + 0x5106d057dcfce9cd, + 0xf767e484fb091b81, + ], + [ + 0x0035d153eb31324d, + 0x37c538e270be601a, + 0x21f73928c5344f95, + 0x0b0e4f442e8de209, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + ], + ), + ( + 3, + [ + [ + 0x281e019d55d54de1, + 0xf0ab5f7cc3764d6f, + 0x2cb6b556780ba645, + 0xaa6d879899c5cf80, + 0x6ebf87658577973b, + 0xb2af426aa2df3d12, + 0x646457bf9de8f505, + 0xbd423e46c0b7c70e, + 0xdc3e47b347cdde6d, + 0x8c557853b1ae4858, + 0x25a1799325aa3076, + 0xec5f1caa799417d2, + ], + [ + 0xdaf7006956ca017a, + 0x097dd24a97ba5646, + 0xad5994585e0cd6e2, + 0x1ba415ba1aa8761d, + 0xd2b7f30720d8ee9f, + 0x89a016d3ff144dac, + 0x9517c5102664d681, + 0x173fdf02efed4246, + 0x95761bd58616b352, + 0x2cf97fe34ea40dc2, + 0x5106d057dcfce9cd, + 0xf767e484fb091b81, + ], + [ + 0x0035d153eb31324d, + 0x37c538e270be601a, + 0x21f73928c5344f95, + 0x0b0e4f442e8de209, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + ], + ), + ( + 13, + [ + [ + 0x40dd5da5d4363562, + 0x6dff0c226f49789f, + 0x07147d0a82d4ef3a, + 0xa3c79042cf6dddcf, + 0x1b6988f938cd9dda, + 0xa195a6dc7990340e, + 0x1bb1c4cd027f331e, + 0xb3814bb7304c2ae4, + 0x268ebd7c101b16dd, + 0xc8701b039ebf6a61, + 0xcc3c5195235401e8, + 0x86d620a4dc0a8a4a, + ], + [ + 0xc4a0fe4b7c48ac77, + 0x6380dc66613847b3, + 0xb0aa8463bff724e2, + 0x4f2d28e9d19d26a3, + 0xdaf7006956ca017a, + 0x097dd24a97ba5646, + 0xad5994585e0cd6e2, + 0x1ba415ba1aa8761d, + 0xd2b7f30720d8ee9f, + 0x89a016d3ff144dac, + 0x9517c5102664d681, + 0x173fdf02efed4246, + ], + [ + 0x0035d153eb31324d, + 0x37c538e270be601a, + 0x21f73928c5344f95, + 0x0b0e4f442e8de209, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + ], + ), + ( + 20, + [ + [ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + [ + 0x5e833b6c497a8a05, + 0xee643677f38238bc, + 0xb9a53292f76ab34d, + 0x08409edcc0a6b39e, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + [ + 0x889e95bbeee05d35, + 0xd6b76c8d22d89bd6, + 0x795cc749c96b7575, + 0xd9684e68cd1b09e3, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + ], + ), +]; + +/// Seed of the transcript inputs (4 + 4 + 4 + 12 + 30 felts, in call order). +pub const SEED_TRANSCRIPT: u64 = 0x75c1; +/// The script: put 4, put 4, field, field, put 4, field, put 12, field, put 30, +/// field ×6, then `get_state` (16): every squeezed felt in order. +pub const TRANSCRIPT_OUT: [u64; 46] = [ + 0x3ea84d42004c332e, + 0x2d2a63eee7ccf903, + 0xe785089b51e261d0, + 0xfd29e7055aa668c0, + 0x35dfbb7e71089efc, + 0x5c9448b18853c597, + 0xbef8d934ee63a9cf, + 0x17cf0a5aebe9200a, + 0x3f98d75ad78fc43a, + 0x803fc48bb52b8ba0, + 0xb6be73eb1b9a40ce, + 0xa18852d2699385e1, + 0x08606f3620c92a14, + 0x61a13339f326f109, + 0xb9ad7e923ab6b27c, + 0x371db12ad2cc3f37, + 0xedf74f1f951056eb, + 0x204e7f507df77811, + 0x25c7d715cebf474f, + 0xc570ae1334e3e5ee, + 0x6a5aeeb90c87728e, + 0xf656e1753d450439, + 0xd73110514396f2c3, + 0x99248bb4d4c0712e, + 0xe321f5120ac728f2, + 0x4accd7befca5c116, + 0xb4a94524bc62d8a3, + 0x1e0e9dd5c4856e97, + 0x553d0cbf5e2c56db, + 0x89d5929b6c321680, + 0x553d0cbf5e2c56db, + 0x89d5929b6c321680, + 0xe931b69af1e1dfd6, + 0x64e2d546ad92b45d, + 0x62a777619d1efd78, + 0x8eecb8c10e12e51a, + 0x8d998033941a27fc, + 0xed146a0aca6b8495, + 0x38a2038cf64f9e55, + 0x3d4872fbf1879226, + 0xe4b0b9d937a04d20, + 0xd57839674d359806, + 0x5eb4c21e24415598, + 0xfdd6315d52657776, + 0x63a9d68ac5e2db29, + 0x56ba8abfc7849090, +]; +/// A fresh transcript fed the last field squeezed above and the nonce 12345: +/// `get_permutations(10, 21)` (ZisK's query indices). +pub const TRANSCRIPT_QUERIES: [u64; 10] = [ + 0x00000000000e8d95, + 0x0000000000017fd2, + 0x00000000001089c3, + 0x00000000001d6614, + 0x0000000000150753, + 0x00000000001d15e6, + 0x000000000009f50b, + 0x00000000000ed95f, + 0x00000000001e0009, + 0x000000000010eafa, +]; + +/// The grinding challenge, `felts(SEED_GRIND, 3)`. +pub const SEED_GRIND: u64 = 0x6f1d; +/// `(nonce, lane 0)` of the width-8 permutation of `[c0, c1, c2, nonce, 0, 0, 0, 0]`. +pub const GRIND_LANE0: [(u64, u64); 4] = [ + (0, 0xc82edce9946878a9), + (1, 0xa97fc9da5e04914e), + (2, 0x4d160560aa29bea0), + (3, 0x03007779902fd6eb), +]; +/// The smallest nonce whose lane 0 is below `2^(64 − GRIND_BITS)`. +pub const GRIND_BITS: u32 = 10; +pub const GRIND_SMALLEST_NONCE: u64 = 2408; diff --git a/crypto/crypto/src/hash/poseidon1_w16/constants.rs b/crypto/crypto/src/hash/poseidon1_w16/constants.rs new file mode 100644 index 000000000..38bbf2177 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_w16/constants.rs @@ -0,0 +1,849 @@ +//! Poseidon1 over Goldilocks at width 16: round constants and the MDS row. +//! +//! GENERATED by `scripts/poseidon1/p1_params.py rust`; do not edit by hand. +//! Round constants: the Poseidon paper's Grain LFSR (Appendix E) with +//! `field_type=1, sbox=0, n=64, t=16, R_F=8, R_P=22`; reproduced by Plonky3's +//! independent generator (`poseidon1/generate_constants.py`, MIT/Apache). +//! MDS: Plonky3's `MATRIX_CIRC_MDS_16_SML_ROW` (`goldilocks/src/mds.rs`, +//! MIT/Apache), checked MDS over Goldilocks by `scripts/poseidon1/mds_check.rs`. + +/// First ROW of the circulant MDS matrix: `M[i][j] = MDS_CIRC_ROW[(j - i) mod 16]`. +pub const MDS_CIRC_ROW: [u64; 16] = [1, 1, 51, 1, 11, 17, 2, 1, 101, 63, 15, 2, 67, 22, 13, 3]; + +/// Round constants `[round][lane]`: 4 initial full, 22 partial, 4 terminal full. +pub const ROUND_CONSTANTS: [[u64; 16]; 30] = [ + [ + 0x15ebea3fc73397c3, + 0xd73cd9fbfe8e275c, + 0x8c096bfce77f6c26, + 0x4e128f68b53d8fea, + 0x29b779a36b2763f6, + 0xfe2adc6fb65acd08, + 0x8d2520e725ad0955, + 0x1c2392b214624d2a, + 0x37482118206dcc6e, + 0x2f829bed19be019a, + 0x2fe298cb6f8159b0, + 0x2bbad982deccdbbf, + 0xbad568b8cc60a81e, + 0xb86a814265baad10, + 0xbec2005513b3acb3, + 0x6bf89b59a07c2a94, + ], + [ + 0xa25deeb835e230f5, + 0x3c5bad8512b8b12a, + 0x7230f73c3cb7a4f2, + 0xa70c87f095c74d0f, + 0x6b7606b830bb2e80, + 0x6cd467cfc4f24274, + 0xfeed794df42a9b0a, + 0x8cf7cf6163b7dbd3, + 0x9a6e9dda597175a0, + 0xaa52295a684faf7b, + 0x017b811cc3589d8d, + 0x55bfb699b6181648, + 0xc2ccaf71501c2421, + 0x1707950327596402, + 0xdd2fcdcd42a8229f, + 0x8b9d7d5b27778a21, + ], + [ + 0xac9a05525f9cf512, + 0x2ba125c58627b5e8, + 0xc74e91250a8147a5, + 0xa3e64b640d5bb384, + 0xf53047d18d1f9292, + 0xbaaeddacae3a6374, + 0xf2d0914a808b3db1, + 0x18af1a3742bfa3b0, + 0x9a621ef50c55bdb8, + 0xc615f4d1cc5466f3, + 0xb7fbac19a35cf793, + 0xd2b1a15ba517e46d, + 0x4a290c4d7fd26f6f, + 0x4f0cf1bb1770c4c4, + 0x548345386cd377f5, + 0x33978d2789fddd42, + ], + [ + 0xab78c59deb77e211, + 0xc485b2a933d2be7f, + 0xbde3792c00c03c53, + 0xab4cefe8f893d247, + 0xc5c0e752eab7f85f, + 0xdbf5a76f893bafea, + 0xa91f6003e3d984de, + 0x099539077f311e87, + 0x097ec52232f9559e, + 0x53641bdf8991e48c, + 0x2afe9711d5ed9d7c, + 0xa7b13d3661b5d117, + 0x5a0e243fe7af6556, + 0x1076fae8932d5f00, + 0x9b53a83d434934e3, + 0xed3fd595a3c0344a, + ], + [ + 0x28eff4b01103d100, + 0x60400ca3e2685a45, + 0x1c8636beb3389b84, + 0xac1332b60e13eff0, + 0x2adafcc364e20f87, + 0x79ffc2b14054ea0b, + 0x3f98e4c0908f0a05, + 0xcdb230bc4e8a06c4, + 0x1bcaf7705b152a74, + 0xd9bca249a82a7470, + 0x91e24af19bf82551, + 0xa62b43ba5cb78858, + 0xb4898117472e797f, + 0xb3228bca606cdaa0, + 0x844461051bca39c9, + 0xf3411581f6617d68, + ], + [ + 0xf7fd50646782b533, + 0x6ca664253c18fb48, + 0x2d2fcdec0886a08f, + 0x29da00dd799b575e, + 0x47d966cc3b6e1e93, + 0xde884e9a17ced59e, + 0xdacf46dc1c31a045, + 0x5d2e3c121eb387f2, + 0x51f8b0658b124499, + 0x1e7dbd1daa72167d, + 0x8275015a25c55b88, + 0xe8521c24ac7a70b3, + 0x6521d121c40b3f67, + 0xac12de797de135b0, + 0xafa28ead79f6ed6a, + 0x685174a7a8d26f0b, + ], + [ + 0xeff92a08d35d9874, + 0x3058734b76dd123a, + 0xfa55dcfba429f79c, + 0x559294d4324c7728, + 0x7a770f53012dc178, + 0xedd8f7c408f3883b, + 0x39b533cf8d795fa5, + 0x160ef9de243a8c0a, + 0x431d52da6215fe3f, + 0x54c51a2a2ef6d528, + 0x9b13892b46ff9d16, + 0x263c46fcee210289, + 0xb738c96d25aabdc4, + 0x5c33a5203996d38f, + 0x2626496e7c98d8dd, + 0xc669e0a52785903a, + ], + [ + 0xaecde726c8ae1f47, + 0x039343ef3a81e999, + 0x2615ceaf044a54f9, + 0x7e41e834662b66e1, + 0x4ca5fd4895335783, + 0x64b334d02916f2b0, + 0x87268837389a6981, + 0x034b75bcb20a6274, + 0x58e658296cc2cd6e, + 0xe2d0f759acc31df4, + 0x81a652e435093e20, + 0x0b72b6e0172eaf47, + 0x4aec43cec577d66d, + 0xde78365b028a84e6, + 0x444e19569adc0ee4, + 0x942b2451fa40d1da, + ], + [ + 0xe24506623ea5bd6c, + 0x082854bf2ef7c743, + 0x69dbbc566f59d62e, + 0x248c38d02a7b5cb2, + 0x4f4e8f8c09d15edb, + 0xd96682f188d310cf, + 0x6f9a25d56818b54c, + 0xb6cefed606546cd9, + 0x5bc07523da38a67b, + 0x7df5a3c35b8111cf, + 0xaaa2cc5d4db34bb0, + 0x9e673ff22a4653f8, + 0xbd8b278d60739c62, + 0xe10d20f6925b8815, + 0xf6c87b91dd4da2bf, + 0xfed623e2f71b6f1a, + ], + [ + 0xa0f02fa52a94d0d3, + 0xbb5794711b39fa16, + 0xd3b94fba9d005c7f, + 0x15a26e89fad946c9, + 0xf3cb87db8a67cf49, + 0x400d2bf56aa2a577, + 0x56045ba28fd1dbb1, + 0x06851d67ed5acacd, + 0xf7019bc57980d178, + 0x0b56dd650d860d28, + 0xcd095759d842636d, + 0x17180482b913fb6d, + 0xf085ee1e1911b728, + 0x44549837566a0bfd, + 0xadd0271c8a246c59, + 0x4292bcfb6e12c8ce, + ], + [ + 0xb9335e2407759a82, + 0x08eb37a7111d5618, + 0x2b7c80b429d54278, + 0x53ea5d3e1999f4ce, + 0xedee2a45b06fa613, + 0xacaf4772a36459c2, + 0xcc0b4aa757746b2f, + 0x52536a16f47129c0, + 0x9b0abd0b07ccc510, + 0x3d3b86ee7806a3ee, + 0x533072e923c588e5, + 0x14555d5ea03b4428, + 0xd9e7702629648bd3, + 0x273c5dc3b8b6ed83, + 0x26315afb96c27395, + 0xbabc1eb9506c3767, + ], + [ + 0x0d0281a2fb57b4c7, + 0x2c709a3b40cce2bd, + 0x9ab9a14cafb72112, + 0x189aed3cd14d0f5e, + 0xc941490f292f3566, + 0x29d339bc4c4c5ed1, + 0x51886b29d3b0d26e, + 0x26ecf673ad01666f, + 0x58f20c05df6136e4, + 0xcaae02f891d9a693, + 0x99ea2de13df560c7, + 0xfb7e604032a6c89b, + 0x9484e6a84078ffab, + 0x33eb1f78ea2bf882, + 0x305fba6c763df8a4, + 0x855a940575a2f5cc, + ], + [ + 0x3bbe96637669d8a9, + 0xe0bf92ec5aa783b5, + 0xd2d8f7be3c581792, + 0x25d1dbdc95699756, + 0x6811562dd7d380e7, + 0xd2d0c07c8cb96126, + 0xed7de2464aab8093, + 0x28e68f8ff765cddd, + 0x000f4302468decf6, + 0xf408a8da2ec86476, + 0xa910d923652cbc51, + 0x61ff0fdd666285f9, + 0x0d35de1fb1d92032, + 0xc33f743297168e6b, + 0x5280166634968f2f, + 0x8029701a169b8c58, + ], + [ + 0x442b1659f53fc92c, + 0xb80d1df09583aae7, + 0x3f68f6f41e84d283, + 0x2dfd848230eeab8e, + 0x464724542108bcce, + 0x9cbd9af019eb40d8, + 0x9cd056128537dca6, + 0x0fdc70699871ec1f, + 0x91b4ed0840036aa9, + 0x0426a03ab3b941ae, + 0x3620ce1d1da9d9ea, + 0xa909c10f133b12df, + 0xdff78cb9ce5e72af, + 0x355a0507ea3d39e9, + 0x33ab02012ebb8e90, + 0x66e681125ad48d09, + ], + [ + 0x754e7875a4cba209, + 0xb4f4092cd96fe585, + 0xe0beb61c71cd590b, + 0x289d6191348815af, + 0xc7f9310eebd89190, + 0xc404c95833496b5d, + 0x67cd987a2e477666, + 0x83a966eaaba6f2d0, + 0xa843095304ddb411, + 0xab9bd2ffaf5ce005, + 0x4b8b976e23ed8d5e, + 0xe853009d6815ce32, + 0x30b462f67decb8bd, + 0x4fbe35dfc8561dde, + 0xb182e2119eb521e0, + 0x9a37c025983b008b, + ], + [ + 0x9a8036d5c0b0ddff, + 0x530e4069d0629e2f, + 0xb5d7149987ef9062, + 0x6a7120ccc56ca201, + 0xd4fb060e7aed5a56, + 0x9d01fca4ebb8856b, + 0xe4bafe3340849d38, + 0x0632a3df994f0c09, + 0xf084664f0b9c8659, + 0xbed5c89cfd8f9a70, + 0x2732cb3f2103948c, + 0x56b9259a2307cb16, + 0xe9c504996e7d3273, + 0x1c8a11e45261ef10, + 0xdd694710e5643a96, + 0xf84538050c75ba9c, + ], + [ + 0x6d21207c2dd2ffdd, + 0x398156c24d3ee292, + 0x5dfe3b4a26dfdb31, + 0x5264c0dce0fc508d, + 0x68db82e30df8c0b6, + 0x0bb69f951e7dae8f, + 0xa4a00a693a6a9dc6, + 0x7d2ed77fb67e1edb, + 0x3a29153665b5deb6, + 0xde05f000f7734b79, + 0x2d3298b50e9b8ee3, + 0x0a2801c5da77fe8d, + 0xe4dae42bd80dc286, + 0x2a92a223212d9bb1, + 0x4a9e772dc0e7ae65, + 0xf91ad425467514a6, + ], + [ + 0xc947db43432f981c, + 0xfab35dc94da6ff86, + 0x30e86b98128ca1d8, + 0x73045f05de00acdb, + 0x877a90860890a505, + 0x904b699fae24adf6, + 0x2fd36112b8db92ea, + 0x37c8e7b709efcfee, + 0x117825ee6efb194d, + 0xef084994ba70e740, + 0x3bbbb3e35ddff9ff, + 0x94599a930c7d397e, + 0x3c962b6062d37a0b, + 0x8cedf279060e80cf, + 0xe3d6fee148661835, + 0xac6dc576dcf69a1b, + ], + [ + 0x4395a68ba78162dc, + 0x1509b345505d0cfc, + 0x78ea2a2c0ff44da3, + 0x3b8453ac216d9dcc, + 0xeeea1ecb6e9739da, + 0x998a5a4578f042a1, + 0x85b4d1176041379e, + 0xf1c548623858b53e, + 0x84f76ecd462db219, + 0x7d478acf677e40c9, + 0x78bd49e6248a3f8b, + 0xbaa176467eac8dc0, + 0xa1d87308ce5f171b, + 0x6ecdc783e58e6ded, + 0x0a27470d8a54c073, + 0x303dedb68be27aff, + ], + [ + 0x1283fa6b9fab0360, + 0x09b9238f8990d3c6, + 0x783fe452c827ce70, + 0x51053b0a0b311026, + 0x913af1029780fd02, + 0x2768db16e909ab50, + 0xe2711e4680cd49d4, + 0xae38b891ffd47d23, + 0x747a4123904482ef, + 0x7590a5536989e07f, + 0x597bcf699cb8f029, + 0xcf1022c720e9c768, + 0x6be4d569436457d0, + 0x1ea4854422964af6, + 0x9ae378da59b64449, + 0x01111bdcbee5f71a, + ], + [ + 0x3e5b8b050fbb6ffe, + 0x5a14f21fc4493541, + 0x1ce4f54dfef557a1, + 0x5cd4599991cc8d4f, + 0xc564b336f4eb42a2, + 0xdcb6c46e887ac22e, + 0x8a75e51054cd014f, + 0xb0ff306616755d5e, + 0x4dfee54c8dd19e47, + 0x7ca27c634f5d6f20, + 0x377a3f686a605c43, + 0x942f96fe69242828, + 0x63363826c868c690, + 0xf622c7bbff21a41e, + 0x7413fe06ae65a228, + 0xf27e12c8f2b63a12, + ], + [ + 0xcb9cdc2782a601b6, + 0x4c355b11ae9d1501, + 0x97b3caa2ca1983c8, + 0x6f253eb18a69a3d6, + 0x9daaad35dbe278d0, + 0x9b154395fc9479de, + 0xb88a4c10b121859b, + 0x23f574023668b146, + 0xe014dd76f0daeb2a, + 0x47ca3455c99abbfc, + 0xe15bcce7d6626f93, + 0xe785e1d29b697a2a, + 0x72f6e6335f2779b2, + 0x63354be802a8d806, + 0xe8680400a8b7da56, + 0xa57878f6cd7dc1f5, + ], + [ + 0xa3242e1666842e56, + 0x9d4763931e5c80b8, + 0xf5ad9042f9c2a2ea, + 0xfde9819dcf522ccf, + 0x2c1b7126142f99b2, + 0x512a49e97331bcdf, + 0xdc36f6f22179f9a4, + 0x1287293218e2af1d, + 0x9547747ddf7cd847, + 0x27e9d7e39b288667, + 0xf53ed68c44662e1c, + 0xccd3913a837bf56d, + 0xa54b0c67cee88c45, + 0xc9204bc4f678e860, + 0x3b2b3332135618af, + 0x3bfe057360941901, + ], + [ + 0x7ea1c40408898d03, + 0x65735bc04a989af5, + 0xc0d0f9741e308f84, + 0x274505af11e8561c, + 0x00c01a31a2a8adf7, + 0xae16a9b5d19bd9eb, + 0xeb47bf1cf2f8c536, + 0x6281eff11a965684, + 0x16fbec6cb544e37b, + 0x43661c50ecbaf61d, + 0x48ededeefe678215, + 0x41454e2ac92e020f, + 0x677ab8b3c9ba50e2, + 0x7fe3981c7c556ff9, + 0x35c8b8589248bedd, + 0x8466ae507b53ba99, + ], + [ + 0x3c1ce7a6557cd52c, + 0xb547ce487a0afa54, + 0x05edb6bc9e4c7961, + 0x50573b9cb7beeefe, + 0xb26c40ecb1535571, + 0x4a9d2fb483061eb1, + 0x9738a691d84dd72c, + 0xedfb4e4efb79d5f8, + 0xbfdb2e3c4d600b54, + 0xa44a4620bb87c023, + 0xedfd03c8933b6509, + 0x6409ecdd7113b38f, + 0xbc7dd042ff7eb60b, + 0x1c41de22a36b7a75, + 0x177fdf40e9646ded, + 0x8eeff87cd83ea10d, + ], + [ + 0xf4ba0b7ed08bafeb, + 0xe5b3ede57adb7f0d, + 0xd8bf37d380f76ba9, + 0xb5e07f4e8f36d2f7, + 0xf2014366c2e22b88, + 0x876f5726bb600720, + 0x2fdd356780329918, + 0xa60294429a5d0408, + 0x0fd714f70b67a879, + 0xd3efd294503511e7, + 0x9ae10ed6c78b28c6, + 0xae221f0e399fa644, + 0x7d75772d7d8bf011, + 0x0a6321da6233b471, + 0x9bb7b0e6658281be, + 0x7b16cc63c27fe8f4, + ], + [ + 0xbf53f47f87f56a6d, + 0xe635bf90172d71e6, + 0xccf02f48250d3fb5, + 0x3d36aa7d16141aaf, + 0xae1eb04558b772b4, + 0x1d58ab28e8083c4f, + 0x1d9bda2e12d3e534, + 0x423dd55f154482ef, + 0x72b65b610654133b, + 0x487d3548f56a85f0, + 0xd8c52e8889a97c5a, + 0x505ec42c4db39362, + 0xc217b41e7d0ef6e7, + 0xa6fd6c3cda5c8473, + 0xff5dd9dfcd788211, + 0x303ff3284e6f7308, + ], + [ + 0xa1aa11255463d565, + 0x78e87d336ccc6981, + 0xcbe6f13680f9259e, + 0x43ed60b3f0605305, + 0x285bb9012ad02b65, + 0x5e3be4930ba77f91, + 0x7d0b7f76259514de, + 0x196f1021318d4d14, + 0x32aeee488c159236, + 0x7042b7b5a3393989, + 0xefe7a9290c7ff77c, + 0x429dc22cfde71457, + 0x70a0a8b0cd2e4e92, + 0x2a66e414a2e69a2d, + 0xeb6d94e92221051c, + 0x704157837371e9be, + ], + [ + 0x8a085f6f2d59f8bb, + 0xd195560e9297a989, + 0x8ef6701b17a6930c, + 0xbdb996211667ba57, + 0xe240f4c1c4b2ad06, + 0xfc2f101d588e0907, + 0x791d9d059d498e68, + 0xd21b9dfad1421e14, + 0x3f40701396692e53, + 0x837e836ab5a382ca, + 0x6b7358c4631537d0, + 0x7e3b633af145a310, + 0xbac53ee64fd8b153, + 0xb5e07a6e5b2b5242, + 0xe754f45f540247cb, + 0x23ada0ccd4a9e8e9, + ], + [ + 0x5bc4109d42b4c5c8, + 0x754fa15d43d39e06, + 0x9a99178df6fb2088, + 0xe8fca9a16c030c7a, + 0xbb9f10f3f43aef47, + 0x11350f1b54ae9adc, + 0x665e172a588ee1db, + 0x08302ce9f0ee5277, + 0x494e7e049554756c, + 0xded40cca27f6c2ae, + 0xfa62692d0d41fc3c, + 0x7b2f0fdcde013e9b, + 0x0998315134e1d6a9, + 0x5e59c08deb844be8, + 0x3a37c9734e09cfb2, + 0x31719e2d681d7414, + ], +]; + +/// The paper's MDS alternative: the Grain Cauchy matrix `M[i][j] = 1/(x_i + y_j)`, drawn from +/// the same Grain stream after the round constants (the reference procedure; its first +/// candidate passes the subspace-trail checks, and Plonky3's generator picks the same matrix). +pub const CAUCHY_MDS: [[u64; 16]; 16] = [ + [ + 0x5988b85748e01d64, + 0x6ce5b2a51c028cd5, + 0x97151726a74103eb, + 0x48da805da22db860, + 0xab527bddf567ce5d, + 0xa62c7a8802d557bf, + 0x682f001eff55d953, + 0x739f833269079c91, + 0x52f36cc427b3c2d1, + 0x207f3edd31fec0c1, + 0xe861a14803653f31, + 0x7f75391531d1351a, + 0x9c1ff22b7b6519c0, + 0x33b1b3b9f6243b4d, + 0x39a21454890d9e9f, + 0x5153a55d83d32ad0, + ], + [ + 0x2c7fa8fb1e89be4a, + 0x53aaf9e104e4e906, + 0x5466ca73c94286db, + 0x01533ab42def6283, + 0xa1d91119905de8f2, + 0xc4a70e83b3da8175, + 0x9c4fa46f80a875b7, + 0x2ab64e023d7a4574, + 0x95870ac3b43221ad, + 0x8ec4c7dfe0b1e529, + 0x29af3c3d9caab861, + 0x0f5e03a41985622f, + 0x6e351f8b80d25233, + 0x34c392b28480b011, + 0xa2e21b3c21902702, + 0x98c40b93bfa7d635, + ], + [ + 0x3669f4cbc4539d7e, + 0xf8fee0879daec673, + 0xb732c1f23b761aee, + 0xceee88ee79daf441, + 0x86c99724e2cef313, + 0x1cf1f3ab213a0d3c, + 0xd5ea051783a3122a, + 0x22352d4ea73f6045, + 0xf383c05e03cd7c62, + 0x81dfe9c8a0051bbe, + 0x3f07ffcdd9266723, + 0xc93603d00f5dc893, + 0xe69d2645321274ab, + 0xd4d06b40d3b2458d, + 0x3c3b9bf934fc9fd7, + 0x6c3ed734df9cf691, + ], + [ + 0x3c3edb0d8e367cbe, + 0x4d141f636fb94d24, + 0x593dc28f61908b6d, + 0x51429b67ced5b2ea, + 0xb8ad92e65afe1948, + 0x7de771ac1136a634, + 0xf2bc3d4d9f8c12b4, + 0xa00899fee1163f9a, + 0xefd9361aba69c5f9, + 0x692af10ed750d360, + 0x203a34814f5fdba1, + 0x22aa4b0fc8e4f127, + 0x8404c6ef28881b6d, + 0x332c23191cb610fd, + 0xe03b41f9d0a5d63f, + 0xf6bc920a03f7752b, + ], + [ + 0x35accf0a901fcb8d, + 0x8dcff51324521386, + 0x57e239c58d0c8e89, + 0xfaed102b92f35770, + 0x3e4bfd2caf5c21f7, + 0x3cd53e002583952a, + 0xea2a4a22eb681b2c, + 0x670a2ee21d7014eb, + 0xdde0a9aa6df4b771, + 0x95b1fbc56e1a6c39, + 0x6346cd06cc929757, + 0x6ae4bda639af92e0, + 0x77314dfd51e7d92c, + 0x0391e3978ff24b17, + 0xe27611f117d6d97f, + 0x04883758be9f54e3, + ], + [ + 0xe6d3dda85e6395c2, + 0x0cd42c05f66b3c1c, + 0x698359374266b60b, + 0x9e9eaa8893e52d50, + 0x034432c8f57c7559, + 0x4674cb4bc8006eae, + 0x92010504468f2681, + 0x182baea6c583b9c6, + 0xfd348e1d8e95d693, + 0xc2f2fd9b9078ece9, + 0x1aba1a45a6419bf9, + 0xbcc2b6584c1af690, + 0xad0b58758270f4ca, + 0xd4e24b7e35851406, + 0x9572342e0aec74bd, + 0xf681bb067f7ac76e, + ], + [ + 0x30f21444e981732e, + 0x0a4b4c356cf81323, + 0xe0ddde574b7bf32b, + 0xc3f99cc853e76784, + 0xe1aa2dd912c46532, + 0x54531f9f67108f80, + 0x8639b004a91e6a28, + 0x8cd1ddb584f7a9ac, + 0xd78dad0f1337bba4, + 0xc481f8f3a4b87f91, + 0x4c24f9db5e94ab02, + 0x1c62f47a8c19eddb, + 0xbf16cec4fba38913, + 0x4ed589b4fd6bbaf0, + 0x86cff751dac92c05, + 0x86249af3a8eff0f6, + ], + [ + 0xe1033c0d774c285d, + 0x2f8586c1921d8fb4, + 0x7441a10af45e5808, + 0x8d8feb70aded8628, + 0x7da45cc544d9ae35, + 0x767f12ba8a5a0c14, + 0x620efed7f8d3a459, + 0xdd7ff646c99f70ad, + 0x3678cc601846b44a, + 0x6755b4209c3401cc, + 0x8e0ae4aa7035b0eb, + 0xba9b1267dd354322, + 0xfadd3d5f0683e9b6, + 0xbc5c35ae68f2dfe2, + 0x73ae2e3c9c090886, + 0xcc0c1c1e8d9bef49, + ], + [ + 0x8671921f7a0a1794, + 0x9cd5f1eac6037499, + 0x3e527bd0148ade83, + 0xaa717d8659a564c3, + 0x548583860ccd6913, + 0x0df5a873cf6cef5f, + 0x6b1a27e8227f6df8, + 0xdb3e44aa830df1c7, + 0x71353fc324da6e34, + 0x519ed83e6503d741, + 0x81d74e3914f76769, + 0x6c8a3d26ef84b973, + 0x48033d71b77fe8c1, + 0x0b845bc4926fb555, + 0x9e07dbc3ba174490, + 0x0ec39acc7f68d8ce, + ], + [ + 0x7677a79600252463, + 0xf3e2c73584829d44, + 0x554b023d8621a4de, + 0xf8afcc7e5c120419, + 0x385e05151f117f7c, + 0x6f349729b45b5be6, + 0x3a98b59f73759ee1, + 0xd98152e584082edb, + 0x795f0382526cb465, + 0x3d7631494b438675, + 0x1003b04cc5497839, + 0x0f224853d28542fe, + 0xf83d612bda7234b5, + 0xdbfc370b88104851, + 0xc1dd2e22be7c3a16, + 0x17ca5948341a933b, + ], + [ + 0xa9d70e4e8c01ad89, + 0x47bb5a0c5a8da9b0, + 0xc7b5e1da10d895f7, + 0x5f0cc2c2aaa7b03e, + 0xbc8097f44e63863f, + 0x70c8e880581bd43e, + 0x758cfe2ffa079581, + 0x839395f285d3ece8, + 0xa2f27080ac482121, + 0x3f62d1fcb8f9d784, + 0x4b2c662ed6ae100c, + 0x14f3fbce248827f0, + 0xbda2a9225eda8792, + 0x1ed0e69117ef3c36, + 0x3af8405a89b5c1a5, + 0x3429a7904ea79c9d, + ], + [ + 0x9d8119220744a23b, + 0x730bf95139f665b1, + 0xf71f347cf06eb11b, + 0x1e16d46ef71b8e86, + 0xffb5517c741183a7, + 0x9de7bdf6e7763c76, + 0x641535f22ebbb8b5, + 0xcc9a2ac6e3525ff2, + 0x6ca74aadfd1870b2, + 0x26c7b78d1a569b93, + 0x6771174eb61e5363, + 0xfece43af74186a6c, + 0xe2d04ce9d92aac6c, + 0x069bce14795ab3ee, + 0x454663cdef4ad361, + 0xc9d1d1b13aa366ba, + ], + [ + 0x2a14bb5529f46c5e, + 0x9284dc3ec5a3298e, + 0xc32a357d30aa5298, + 0x6378633155977074, + 0xd42fbe0d1987c1d4, + 0x812a7402da911fc3, + 0x0aa377c56139e86d, + 0xf738501437323e35, + 0x716814c20125dd1a, + 0x4b8f884ea190466e, + 0x73af1afa89a8b0f8, + 0x581257e0a6514156, + 0x2a3e9650920f7c21, + 0x3a4b286082ce7d23, + 0xaa89fb9040aed6fa, + 0xcbcd108576e6d1ba, + ], + [ + 0xa0e6af946333f9cc, + 0x5b9fa025d5f91de4, + 0xa37f3398a3ff534c, + 0x5d807c5491b334cc, + 0xdde0491f392f54b3, + 0xd31f9255564584e5, + 0xb682b5175a329122, + 0x36ac21cacdf0a097, + 0xbd69c3e422624ed0, + 0xfd03026489b74c0b, + 0xb2b862a95c6f5fc9, + 0x7a6ec2a715549e03, + 0x6a3cf97409861804, + 0xdd31032efb7843cc, + 0x080d82ec2393f0c3, + 0x5ca5e139dcf07d41, + ], + [ + 0xf57bae2bb3999ef3, + 0x1b3241817e6f40c5, + 0x12f1d5f21156e7b5, + 0x44261d736bf454a8, + 0xe1194d1a1cd17a51, + 0x9394f656c1246dc8, + 0x050fe6e2bb26bab4, + 0xcb184bfb067f33a3, + 0x87b5f94c9d2c7907, + 0x456daf135d96f358, + 0x116eba266dcc0cb3, + 0xa2dbf3c480582332, + 0x1302f5ef13049a4d, + 0xd2ac7ba5adc5269a, + 0x43b32aedb14c08cc, + 0xe00837b7287de52d, + ], + [ + 0xd9553f671aad6e14, + 0x4bb5271bb83f98a0, + 0x2c84ca7932ff0d9e, + 0x0ac13c2ec615ff05, + 0xfd066e3f8f82e427, + 0x6cddcb7116f97320, + 0xce298f1a321e7087, + 0x74c9ceb2944dc1b5, + 0x6afbdadac658ec67, + 0x8c32f317e710b3bd, + 0xc380701ec43dc56d, + 0x96b0a79e387441a4, + 0xcac77034b3035048, + 0x015fd2a7e0a193ac, + 0x8f848885347161d7, + 0x110276a5157b853a, + ], +]; diff --git a/crypto/crypto/src/hash/poseidon1_w16/kat.rs b/crypto/crypto/src/hash/poseidon1_w16/kat.rs new file mode 100644 index 000000000..fe82d47a6 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_w16/kat.rs @@ -0,0 +1,470 @@ +//! Known-answer vectors for the W16 host reference. +//! +//! GENERATED by `scripts/poseidon1/p1_params.py rust` from the Python +//! reference; the permutation vectors are also reproduced by Plonky3's +//! generator's reference permutation and by Plonky3's Rust `Poseidon1` +//! (sparse partial rounds) fed these constants. + +/// `(input, permute(input))`. +pub const PERMUTATION_VECTORS: [([u64; 16], [u64; 16]); 4] = [ + ( + [ + 0x0000000000000000, + 0x0000000000000001, + 0x0000000000000002, + 0x0000000000000003, + 0x0000000000000004, + 0x0000000000000005, + 0x0000000000000006, + 0x0000000000000007, + 0x0000000000000008, + 0x0000000000000009, + 0x000000000000000a, + 0x000000000000000b, + 0x000000000000000c, + 0x000000000000000d, + 0x000000000000000e, + 0x000000000000000f, + ], + [ + 0x81c2ff551d3dd1a3, + 0xa6f3ddabab7998e2, + 0x4372186243233825, + 0xd2bd8442c6cc6df7, + 0x051a796f67578f23, + 0x3b597e26481062ca, + 0x19c3c48645baaabb, + 0x7e142fc8bf48c2ce, + 0x599ca659bfbf033f, + 0x84e132ca4afd703d, + 0xb758d5776f5185c3, + 0xaf58bfc9cb74204e, + 0x7015309157ec7e9c, + 0xe57e7f42acfff2e0, + 0x57043250e11a11bb, + 0x656c21727540ab90, + ], + ), + ( + [ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + [ + 0x078a54a5999c1f89, + 0xba0c619c6e9a0ff0, + 0xe2f9a11694354835, + 0xd2c8e968b04cc9a6, + 0x81d0e75f2327654b, + 0x20912557baff31b0, + 0x69bc2fc6d13e33cd, + 0x032c5a376d7cfc13, + 0xecef36c7bccf4d56, + 0x80b4817b062829dc, + 0xf93659d1793a7952, + 0x1f0dc30f44cb3138, + 0x8b564149bfa10efa, + 0xc7b30100325f4879, + 0xf694a1841608a1e8, + 0xc595e9d1f1914be9, + ], + ), + ( + [ + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + ], + [ + 0x491ef464b2792bd3, + 0x28450225e7342b0e, + 0x1299abb383c26dbe, + 0x96056d60d5b031ca, + 0xb6efaa51f392fb67, + 0x8e428e525552bc22, + 0xcda509fb6d800175, + 0x4b641572c6984696, + 0x01f93fc0af917f75, + 0xf28ed5d932aeee76, + 0x9d2467d3ac8a6c3a, + 0x6c6dc438a4757fd1, + 0xc927d1ffd408297a, + 0x4ed8f3f9228f45ee, + 0x469eb77a91504639, + 0x1e5a53837d729653, + ], + ), + ( + [ + 0x2ceaee21bf46bc00, + 0xaa80754d1a1a8d4f, + 0xb3c4904a6d278932, + 0xbc69cf4276846d19, + 0x377b2fd56a5b15b4, + 0x64d815deeaf29df3, + 0xf66e100db2d7d206, + 0x1069e6a57e06665d, + 0x7be902917b70a2a8, + 0x68901baf16ad70d7, + 0xf2caa7c38002001a, + 0x5a55db213cf06be1, + 0x4c88c014892416dc, + 0xa0c1ac9a9822a9fb, + 0x9e5c77e75ae9e76e, + 0x37e83672575ac1a5, + ], + [ + 0xb077e251291e3c50, + 0x802ac7ee5069af14, + 0x3b40e1590405fd9e, + 0xcfa2bc2aec5966c6, + 0x554dd87f58bdb066, + 0xc11dcdf422d22bb0, + 0x6b6bcf24fa68d8bf, + 0x9cafadebd618a6c3, + 0xf2fd3eacc96f8243, + 0x6fd97ccffe5b67f1, + 0xfbc70f8784230214, + 0x6d96b2b2a82fd02e, + 0x4c73dc266ce6f482, + 0x83d58cae06543d83, + 0x326b8c8355c4bf52, + 0x63ad58b9fcdcb60d, + ], + ), +]; + +/// `(leaf length, digest)`; felt `i` of a length-`n` leaf is +/// `(i * 0x0123456789abcdef + n) mod p`. +pub const LEAF_VECTORS: [(usize, [u64; 4]); 7] = [ + ( + 0, + [ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + ), + ( + 1, + [ + 0xe415de404a2953af, + 0x9572871df89532fb, + 0x422e0e2b1e010d7c, + 0x8d52c08bd5433b76, + ], + ), + ( + 11, + [ + 0x08859e42f65a9057, + 0x540336d531267ee1, + 0x824830ddc5b71850, + 0x16c2f0a23db5e110, + ], + ), + ( + 12, + [ + 0xaecd04f747eff7f8, + 0xdd201f145b1f681f, + 0x1b451c69be18046d, + 0xc914cef09f53f0df, + ], + ), + ( + 13, + [ + 0xad8b01cd8c471f14, + 0x8b56e5f65391eb7b, + 0xc4d5a161ef1186a5, + 0x5b1bce9e120cb1a1, + ], + ), + ( + 48, + [ + 0x80a3635a372a2313, + 0xeab7a5d2c2bc5a03, + 0xd24c114e223f3ec9, + 0xd0942016f536e335, + ], + ), + ( + 64, + [ + 0x964b4d22a7976fc3, + 0x6ea18f3764f7e566, + 0xb86810f36a0e7ce4, + 0xdc3c8124425aa320, + ], + ), +]; + +/// `(four children, node)`; child `c` lane `l` is the KAT input `[0..16)`'s +/// permutation output lane `4c + l`, i.e. `PERMUTATION_VECTORS[0].1`. +pub const NODE4_VECTOR: [u64; 4] = [ + 0x1ddf6497ce8bd322, + 0xe04a80a11f0adc0b, + 0xa6a017e08ba8470a, + 0x082b6f6c76fd7bde, +]; + +/// `(input, permute_cauchy(input))` for the inputs of [`PERMUTATION_VECTORS`]. +pub const CAUCHY_PERMUTATION_VECTORS: [([u64; 16], [u64; 16]); 4] = [ + ( + [ + 0x0000000000000000, + 0x0000000000000001, + 0x0000000000000002, + 0x0000000000000003, + 0x0000000000000004, + 0x0000000000000005, + 0x0000000000000006, + 0x0000000000000007, + 0x0000000000000008, + 0x0000000000000009, + 0x000000000000000a, + 0x000000000000000b, + 0x000000000000000c, + 0x000000000000000d, + 0x000000000000000e, + 0x000000000000000f, + ], + [ + 0x6a84bf02be1f328d, + 0xec14d274b936a21a, + 0xc0539d7bd4eb66de, + 0xb317ecf41fa8d55b, + 0x80b0d36f66671f8a, + 0x74a1592b9a16e832, + 0x65e53afadfadc8c3, + 0xa0007e5ee96ee4b2, + 0x6dd5661a877003a8, + 0xc36a09c2dc25cd6e, + 0xcbda3d58f7cf85f4, + 0x34cb1d63c35596cf, + 0x4fcd09b24769e281, + 0x6c514f906998c65d, + 0xc447035d8d71952b, + 0x591863454267826f, + ], + ), + ( + [ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + [ + 0x6739c9388c4e2ef0, + 0xf30b2e3b622598ad, + 0xe439dd54cf0a515f, + 0x7e44fb022e6afd5e, + 0x6de7468bd2bc085d, + 0xd436b57bc7acc226, + 0xdd008c3a59bda176, + 0x58b4a5d48066e997, + 0x156c5b75c7cac11e, + 0x1c76a3e41d07970e, + 0x839518ae672fd49e, + 0x82f2ac09ce6b0b1b, + 0xf8217338ffb9bdb0, + 0x3ed3402e1f33629d, + 0x8b62125da646935f, + 0x3b27d003425f5481, + ], + ), + ( + [ + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + ], + [ + 0x7a6e22f23f763ef2, + 0x30e26031b6b94bb4, + 0x269b80f1cbb79100, + 0x2c724492563f5f30, + 0xb7d98baf70824ef1, + 0x99b7c1939dc54660, + 0x6a12ad6e9d7969b3, + 0x460f0a2fd8afc57b, + 0x68c84cb2bdce79fd, + 0x8330e46629523758, + 0xfcdbe2677bd08018, + 0xff0ba6fdd3cda119, + 0xee98d495f18bf150, + 0xeb2db2f0b53ca5c2, + 0xf6b4a4d35797d71d, + 0xd90df4ca56b64d46, + ], + ), + ( + [ + 0x2ceaee21bf46bc00, + 0xaa80754d1a1a8d4f, + 0xb3c4904a6d278932, + 0xbc69cf4276846d19, + 0x377b2fd56a5b15b4, + 0x64d815deeaf29df3, + 0xf66e100db2d7d206, + 0x1069e6a57e06665d, + 0x7be902917b70a2a8, + 0x68901baf16ad70d7, + 0xf2caa7c38002001a, + 0x5a55db213cf06be1, + 0x4c88c014892416dc, + 0xa0c1ac9a9822a9fb, + 0x9e5c77e75ae9e76e, + 0x37e83672575ac1a5, + ], + [ + 0x558ecb3ef284e7c9, + 0xf5e658c0d4210d0e, + 0xd156f6ecf4a2dc39, + 0x87398c68e9b1f36c, + 0x2ac68e0cd15c0e64, + 0x63de5d8e79208b7a, + 0xb0c975579f5615e2, + 0x6762becbdc2ac612, + 0x48b6834353d9ba1d, + 0xb82339f848872615, + 0x9bba130306252a7f, + 0xc7e1d0c57fb3b2de, + 0x784ad3867f9ba9b1, + 0x35c0d82ec7544ec2, + 0x014c39294b4f3bee, + 0x361200fea1f613fc, + ], + ), +]; + +/// [`LEAF_VECTORS`]' leaves under the Cauchy MDS. +pub const CAUCHY_LEAF_VECTORS: [(usize, [u64; 4]); 7] = [ + ( + 0, + [ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ], + ), + ( + 1, + [ + 0x0f5a8171129f0e7c, + 0x369de019fc631f9a, + 0x11bcc39be3795feb, + 0x938bda6a985e2cc0, + ], + ), + ( + 11, + [ + 0xaace96db56764328, + 0x77b54185481ffa22, + 0x6bb417ebc6dab7fb, + 0x7f0d29c8a4287602, + ], + ), + ( + 12, + [ + 0xe29c1b20e2c3b82c, + 0x0912444adbf7e934, + 0xdd36f632aab73791, + 0xd4f5efd18bbc032d, + ], + ), + ( + 13, + [ + 0xd54b8304dd89f2ac, + 0xff1de01d10fc30a2, + 0x22dae57424c724d4, + 0x7aa829fb45c98951, + ], + ), + ( + 48, + [ + 0x20f9f35b98a59030, + 0x2923be0334af9725, + 0x2b737bdff4fa2692, + 0x0208ffa011fdb210, + ], + ), + ( + 64, + [ + 0x70a38779ef6e8fca, + 0x6b8f35f6286ba1ff, + 0xdf914b7e3a041f39, + 0xbb45d9892022e65d, + ], + ), +]; + +/// The 4-ary node over `CAUCHY_PERMUTATION_VECTORS[0].1`'s four digests. +pub const CAUCHY_NODE4_VECTOR: [u64; 4] = [ + 0x9aecf43b4bf5d329, + 0x02c7d6eb8e9cc56a, + 0xecd69de566b23205, + 0xf109da81efb2a483, +]; diff --git a/crypto/crypto/src/hash/poseidon1_w16/mod.rs b/crypto/crypto/src/hash/poseidon1_w16/mod.rs new file mode 100644 index 000000000..8a56e08b2 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_w16/mod.rs @@ -0,0 +1,253 @@ +//! Poseidon1 over Goldilocks at width 16 — the MEASUREMENT instance for the +//! D-HASH track (rate 12, capacity 4, 4-ary Merkle nodes). +//! +//! ⚠ **Not a shipped hash.** Nothing in the prover commits with it. It exists +//! so the base-hash candidate can be measured (chip census, device kernels) +//! against RPX; choosing to ship it is a separate decision. +//! +//! ZisK runs this same instance (pil2-proofman's `Poseidon1_16`): its own +//! implementation reproduces the known answers here, and [`super::poseidon1_stark`] +//! builds ZisK's leaf, tree, transcript and grinding on it. +//! +//! # The instance, and where each number comes from +//! +//! Plonky3 publishes Goldilocks Poseidon1 constants and known answers only at +//! widths 8 and 12; at width 16 it has an MDS matrix but no round constants. +//! So the width-16 instance is DERIVED here by the Poseidon paper's procedure +//! (eprint 2019/458), and every step is checked by a second implementation: +//! +//! - **S-box** `x^7`: the smallest exponent coprime to `p − 1` for Goldilocks. +//! - **Rounds** `R_F = 8`, `R_P = 22`: the paper's inequalities at `t = 16`, +//! `α = 7`, `M = 128` (statistical, interpolation, three Gröbner bounds, plus +//! eprint 2023/537's binomial bound) give the minimum `R_F = 6`, `R_P = 20`; +//! the paper's margin (`R_F + 2`, `⌈1.075·R_P⌉`) gives 8 and 22. The binding +//! terms are the statistical bound (`R_F ≥ 6`) and interpolation +//! (`R_F + R_P ≥ 1 + ⌈64·log_7 2⌉ + ⌈log_7 16⌉ = 26`). Same counts as Plonky3's +//! widths 8 and 12. `scripts/poseidon1/p1_params.py rounds --width 16` prints +//! every inequality. +//! - **Round constants**: the paper's Grain LFSR (Appendix E) with +//! `field_type=1, sbox=0, n=64, t=16, R_F=8, R_P=22`. Our generator +//! reproduces all 240/360 of Plonky3's width-8/12 constants and their known +//! answers first; Plonky3's own generator (its width whitelist lifted) +//! produces the same 480 width-16 constants. +//! - **MDS**: Plonky3's circulant `MATRIX_CIRC_MDS_16_SML_ROW` (small entries, +//! so the product is shifts and adds). Checked MDS over Goldilocks +//! exhaustively (all 3.0e8 square submatrices up to the circulant shift, +//! `scripts/poseidon1/mds_check.rs`), and passes Algorithms 1–3 of eprint +//! 2020/500 (no invariant subspace trails) in Plonky3's implementation. +//! ⚠ It is NOT the paper's Grain-derived Cauchy matrix; see the lane note +//! for why that choice is open (eprint 2026/1760 attacks an MDS chosen after +//! the constants). The Cauchy alternative is here too ([`permute_cauchy`], +//! same rounds and constants), so both options can be measured. +//! +//! # Lane conventions (the measurement's, not a frozen format) +//! +//! Rate `0..12`, capacity `12..16`, digest `0..4` — RPX's rule at width 16. +//! A leaf is the overwrite duplex with the padding flag `len mod 12` in +//! capacity lane 0 and the domain [`DOMAIN_LEAF`] in lane 1. A 4-ary node +//! permutes the four child digests (all 16 lanes) and truncates to lanes +//! `0..4`: there is no capacity and hence no domain lane (D-HASH §5). + +pub mod constants; +#[cfg(test)] +mod kat; +#[cfg(test)] +mod tests; + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +use constants::{CAUCHY_MDS, MDS_CIRC_ROW, ROUND_CONSTANTS}; + +/// A Goldilocks field element. +pub type Fp = FieldElement; + +/// Lanes in the permutation's state. +pub const STATE_FELTS: usize = 16; +/// Lanes a block of input overwrites — the sponge's rate. +pub const RATE_FELTS: usize = 12; +/// Felts in a digest. +pub const DIGEST_FELTS: usize = 4; +/// Children per Merkle node. +pub const ARITY: usize = STATE_FELTS / DIGEST_FELTS; + +/// The S-box exponent. +pub const ALPHA: u64 = 7; +/// Full rounds per half; `R_F = 8`. +pub const HALF_FULL_ROUNDS: usize = 4; +/// Partial rounds, S-box on lane 0 only. +pub const PARTIAL_ROUNDS: usize = 22; +/// All rounds. +pub const NUM_ROUNDS: usize = 2 * HALF_FULL_ROUNDS + PARTIAL_ROUNDS; + +/// Capacity lane carrying the padding flag. +pub const CAPACITY_PAD_LANE: usize = 0; +/// Capacity lane carrying the domain tag. +pub const CAPACITY_DOMAIN_LANE: usize = 1; +/// The leaf domain, `"P1WL"` as a little-endian `u32` — distinct from RPX's. +pub const DOMAIN_LEAF: u64 = u32::from_le_bytes(*b"P1WL") as u64; + +/// A four-felt digest. +pub type Digest = [Fp; DIGEST_FELTS]; + +/// Does round `r` S-box every lane? +pub const fn is_full_round(r: usize) -> bool { + r < HALF_FULL_ROUNDS || r >= HALF_FULL_ROUNDS + PARTIAL_ROUNDS +} + +/// `x^7` as `x²`, `x³ = x²·x`, `x^7 = (x³)²·x` — the AIR's association. +pub fn sbox(x: &Fp) -> Fp { + let x2 = x * x; + let x3 = &x2 * x; + let x6 = &x3 * &x3; + &x6 * x +} + +/// [`MDS_CIRC_ROW`] expanded at compile time: `MDS_MATRIX[i][j] = row[(j − i) mod 16]`. +const MDS_MATRIX: [[u64; STATE_FELTS]; STATE_FELTS] = { + let mut m = [[0; STATE_FELTS]; STATE_FELTS]; + let mut i = 0; + while i < STATE_FELTS { + let mut j = 0; + while j < STATE_FELTS { + m[i][j] = MDS_CIRC_ROW[(j + STATE_FELTS - i) % STATE_FELTS]; + j += 1; + } + i += 1; + } + m +}; + +/// The circulant MDS product, one `u128` accumulation and one reduction per +/// lane. The row sums to 371, so each lane's sum is below `371·2^64 < 2^73` +/// and `hi < 2^9`; `hi·EPSILON < 2^41` needs no reduction of its own. +pub fn mds(state: &[Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + /// `2^64 ≡ 2^32 − 1 (mod p)`. + const EPSILON: u64 = 0xFFFF_FFFF; + + let mut out = [Fp::zero(); STATE_FELTS]; + for (row, o) in MDS_MATRIX.iter().zip(out.iter_mut()) { + let mut acc: u128 = 0; + for (c, s) in row.iter().zip(state) { + acc += (*s.value() as u128) * (*c as u128); + } + let lo = acc as u64; + let hi = (acc >> 64) as u64; + *o = Fp::from(lo) + Fp::from(hi * EPSILON); + } + out +} + +/// The dense product by the Grain Cauchy matrix [`CAUCHY_MDS`] — the MDS +/// alternative (full-field entries, so one field multiplication per entry). +pub fn mds_cauchy(state: &[Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let mut out = [Fp::zero(); STATE_FELTS]; + for (row, o) in CAUCHY_MDS.iter().zip(out.iter_mut()) { + for (c, s) in row.iter().zip(state) { + *o += Fp::from(*c) * s; + } + } + out +} + +/// A linear layer: [`mds`] or [`mds_cauchy`]. +type LinearLayer = fn(&[Fp; STATE_FELTS]) -> [Fp; STATE_FELTS]; + +/// One round: add constants, S-box (every lane in a full round, lane 0 in a +/// partial one), then the MDS. +pub fn round(state: &[Fp; STATE_FELTS], r: usize) -> [Fp; STATE_FELTS] { + round_with(&ROUND_CONSTANTS, mds, state, r) +} + +fn round_with( + rc: &[[u64; STATE_FELTS]; NUM_ROUNDS], + linear: LinearLayer, + state: &[Fp; STATE_FELTS], + r: usize, +) -> [Fp; STATE_FELTS] { + let mut s = *state; + for (v, c) in s.iter_mut().zip(rc[r].iter()) { + *v += Fp::from(*c); + } + if is_full_round(r) { + for v in s.iter_mut() { + *v = sbox(v); + } + } else { + s[0] = sbox(&s[0]); + } + linear(&s) +} + +/// ★ The permutation: 4 full, 22 partial, 4 full rounds. +/// +/// The textbook form (a dense round each time), not Plonky3's sparse +/// partial-round decomposition — the known answers are reproduced by both. +pub fn permute(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + permute_with(&ROUND_CONSTANTS, mds, state) +} + +/// The same permutation with the Grain Cauchy MDS — the alternative instance +/// (same rounds, same round constants). +pub fn permute_cauchy(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + permute_with(&ROUND_CONSTANTS, mds_cauchy, state) +} + +/// A permutation over a given constant table and linear layer — the tests' +/// mutation seam. +fn permute_with( + rc: &[[u64; STATE_FELTS]; NUM_ROUNDS], + linear: LinearLayer, + state: [Fp; STATE_FELTS], +) -> [Fp; STATE_FELTS] { + let mut s = state; + for r in 0..NUM_ROUNDS { + s = round_with(rc, linear, &s, r); + } + s +} + +/// The rate-12 overwrite duplex over a felt stream (a Merkle leaf). +pub fn sponge_leaf(felts: &[Fp]) -> Digest { + sponge_leaf_with(felts, permute) +} + +/// [`sponge_leaf`] under [`permute_cauchy`]. +pub fn sponge_leaf_cauchy(felts: &[Fp]) -> Digest { + sponge_leaf_with(felts, permute_cauchy) +} + +fn sponge_leaf_with(felts: &[Fp], perm: fn([Fp; STATE_FELTS]) -> [Fp; STATE_FELTS]) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + state[RATE_FELTS + CAPACITY_PAD_LANE] = Fp::from((felts.len() % RATE_FELTS) as u64); + state[RATE_FELTS + CAPACITY_DOMAIN_LANE] = Fp::from(DOMAIN_LEAF); + for block in felts.chunks(RATE_FELTS) { + for (lane, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + *slot = block.get(lane).copied().unwrap_or_else(Fp::zero); + } + state = perm(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// A 4-ary Merkle node: permute the four children, truncate to four felts. +pub fn compress4(children: &[Digest; ARITY]) -> Digest { + compress4_with(children, permute) +} + +/// [`compress4`] under [`permute_cauchy`]. +pub fn compress4_cauchy(children: &[Digest; ARITY]) -> Digest { + compress4_with(children, permute_cauchy) +} + +fn compress4_with( + children: &[Digest; ARITY], + perm: fn([Fp; STATE_FELTS]) -> [Fp; STATE_FELTS], +) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + for (c, child) in children.iter().enumerate() { + state[c * DIGEST_FELTS..(c + 1) * DIGEST_FELTS].copy_from_slice(child); + } + let out = perm(state); + [out[0], out[1], out[2], out[3]] +} diff --git a/crypto/crypto/src/hash/poseidon1_w16/tests.rs b/crypto/crypto/src/hash/poseidon1_w16/tests.rs new file mode 100644 index 000000000..88614ee95 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_w16/tests.rs @@ -0,0 +1,129 @@ +//! The width-16 instance's known answers, and the checks that make them bind. +//! +//! The vectors in [`super::kat`] come from `scripts/poseidon1/p1_params.py`, +//! a Python reference written from the Poseidon paper, and the permutation +//! vectors are reproduced by two implementations we did not write: Plonky3's +//! generator's reference permutation and Plonky3's Rust `Poseidon1` (sparse +//! partial rounds) fed these constants (lane note `I-HASH.md` §1). + +use super::kat::{ + CAUCHY_LEAF_VECTORS, CAUCHY_NODE4_VECTOR, CAUCHY_PERMUTATION_VECTORS, LEAF_VECTORS, + NODE4_VECTOR, PERMUTATION_VECTORS, +}; +use super::*; + +fn fp16(v: &[u64; STATE_FELTS]) -> [Fp; STATE_FELTS] { + core::array::from_fn(|i| Fp::from(v[i])) +} + +fn canon(v: &[Fp]) -> alloc::vec::Vec { + v.iter().map(|f| f.canonical()).collect() +} + +#[test] +fn the_permutation_matches_the_reference_vectors() { + for (input, expected) in PERMUTATION_VECTORS.iter() { + assert_eq!(canon(&permute(fp16(input))), expected.to_vec()); + } +} + +#[test] +fn a_one_bit_change_in_one_round_constant_breaks_every_vector() { + // Mutation: the vectors must pin the LAST constant of the LAST round (the + // one a truncated or off-by-one table would lose), not only the early ones. + let mut rc = ROUND_CONSTANTS; + rc[NUM_ROUNDS - 1][STATE_FELTS - 1] ^= 1; + for (input, expected) in PERMUTATION_VECTORS.iter() { + assert_ne!( + canon(&permute_with(&rc, mds, fp16(input))), + expected.to_vec() + ); + } +} + +#[test] +fn the_leaf_sponge_matches_the_reference_vectors() { + for (n, expected) in LEAF_VECTORS.iter() { + let felts: alloc::vec::Vec = (0..*n as u64) + // i < 64, so i·c + n < p and the reference's `mod p` is a no-op. + .map(|i| Fp::from(i * 0x0123_4567_89ab_cdef + *n as u64)) + .collect(); + assert_eq!( + canon(&sponge_leaf(&felts)), + expected.to_vec(), + "leaf of {n} felts" + ); + } +} + +#[test] +fn the_4ary_node_matches_the_reference_vector() { + let out = permute(fp16(&PERMUTATION_VECTORS[0].0)); + let children: [Digest; ARITY] = + core::array::from_fn(|c| core::array::from_fn(|l| out[c * DIGEST_FELTS + l])); + assert_eq!(canon(&compress4(&children)), NODE4_VECTOR.to_vec()); +} + +/// The Cauchy alternative's known answers — from the Python reference, whose +/// permutation vectors Plonky3's generator (and its Cauchy matrix) reproduce. +#[test] +fn the_cauchy_alternative_matches_its_reference_vectors() { + for (input, expected) in CAUCHY_PERMUTATION_VECTORS.iter() { + assert_eq!(canon(&permute_cauchy(fp16(input))), expected.to_vec()); + } + for (n, expected) in CAUCHY_LEAF_VECTORS.iter() { + let felts: alloc::vec::Vec = (0..*n as u64) + .map(|i| Fp::from(i * 0x0123_4567_89ab_cdef + *n as u64)) + .collect(); + assert_eq!( + canon(&sponge_leaf_cauchy(&felts)), + expected.to_vec(), + "leaf of {n} felts" + ); + } + let out = permute_cauchy(fp16(&CAUCHY_PERMUTATION_VECTORS[0].0)); + let children: [Digest; ARITY] = + core::array::from_fn(|c| core::array::from_fn(|l| out[c * DIGEST_FELTS + l])); + assert_eq!( + canon(&compress4_cauchy(&children)), + CAUCHY_NODE4_VECTOR.to_vec() + ); + // The two instances differ everywhere they should. + assert_ne!(CAUCHY_PERMUTATION_VECTORS[0].1, PERMUTATION_VECTORS[0].1); +} + +#[test] +fn the_node_depends_on_child_order() { + let out = permute(fp16(&PERMUTATION_VECTORS[0].0)); + let mut children: [Digest; ARITY] = + core::array::from_fn(|c| core::array::from_fn(|l| out[c * DIGEST_FELTS + l])); + children.swap(1, 2); + assert_ne!(canon(&compress4(&children)), NODE4_VECTOR.to_vec()); +} + +#[test] +fn the_mds_matches_a_dense_product_on_extreme_inputs() { + let s = [Fp::from(GOLDILOCKS_P - 1); STATE_FELTS]; + let fast = mds(&s); + for (i, f) in fast.iter().enumerate() { + let mut acc = Fp::zero(); + for (j, v) in s.iter().enumerate() { + acc += Fp::from(MDS_CIRC_ROW[(j + STATE_FELTS - i) % STATE_FELTS]) * v; + } + assert_eq!(*f, acc); + } +} + +#[test] +fn the_round_counts_meet_the_papers_bounds_with_the_margin() { + // Minimum before the margin: R_F = 6 (statistical, M = 128 <= (64 - 3)·17), + // R_F + R_P >= 1 + ceil(64·log_7 2) + ceil(log_7 16) = 1 + 23 + 2 (interpolation). + let log7_2 = core::f64::consts::LN_2 / 7f64.ln(); + let interp = 1 + (64.0 * log7_2).ceil() as usize + (16f64.ln() / 7f64.ln()).ceil() as usize; + assert_eq!(interp, 26); + let (rf_min, rp_min) = (6usize, interp - 6); + assert_eq!(2 * HALF_FULL_ROUNDS, rf_min + 2); + assert_eq!(PARTIAL_ROUNDS, (rp_min as f64 * 1.075).ceil() as usize); +} + +const GOLDILOCKS_P: u64 = 0xFFFF_FFFF_0000_0001; diff --git a/crypto/crypto/src/hash/poseidon1_w8/constants.rs b/crypto/crypto/src/hash/poseidon1_w8/constants.rs new file mode 100644 index 000000000..0b8eac8db --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_w8/constants.rs @@ -0,0 +1,314 @@ +//! Poseidon1 over Goldilocks at width 8: round constants and the MDS row. +//! +//! GENERATED by `scripts/poseidon1/p1_params.py rust8`; do not edit by hand. +//! Round constants: the Poseidon paper's Grain LFSR (Appendix E) with +//! `field_type=1, sbox=0, n=64, t=8, R_F=8, R_P=22`, equal to Plonky3's published +//! width-8 table (`p1_params.py selftest`). MDS: Plonky3's +//! `MATRIX_CIRC_MDS_8_SML_ROW` (`goldilocks/src/mds.rs`, MIT/Apache). + +/// First ROW of the circulant MDS matrix: `M[i][j] = MDS_CIRC_ROW[(j - i) mod 8]`. +pub const MDS_CIRC_ROW: [u64; 8] = [7, 1, 3, 8, 8, 3, 4, 9]; + +/// Round constants `[round][lane]`: 4 initial full, 22 partial, 4 terminal full. +pub const ROUND_CONSTANTS: [[u64; 8]; 30] = [ + [ + 0xdd5743e7f2a5a5d9, + 0xcb3a864e58ada44b, + 0xffa2449ed32f8cdc, + 0x42025f65d6bd13ee, + 0x7889175e25506323, + 0x34b98bb03d24b737, + 0xbdcc535ecc4faa2a, + 0x5b20ad869fc0d033, + ], + [ + 0xf1dda5b9259dfcb4, + 0x27515210be112d59, + 0x4227d1718c766c3f, + 0x26d333161a5bd794, + 0x49b938957bf4b026, + 0x4a56b5938b213669, + 0x1120426b48c8353d, + 0x6b323c3f10a56cad, + ], + [ + 0xce57d6245ddca6b2, + 0xb1fc8d402bba1eb1, + 0xb5c5096ca959bd04, + 0x6db55cd306d31f7f, + 0xc49d293a81cb9641, + 0x1ce55a4fe979719f, + 0xa92e60a9d178a4d1, + 0x002cc64973bcfd8c, + ], + [ + 0xcea721cce82fb11b, + 0xe5b55eb8098ece81, + 0x4e30525c6f1ddd66, + 0x43c6702827070987, + 0xaca68430a7b5762a, + 0x3674238634df9c93, + 0x88cee1c825e33433, + 0xde99ae8d74b57176, + ], + [ + 0x488897d85ff51f56, + 0x1140737ccb162218, + 0xa7eeb9215866ed35, + 0x9bd2976fee49fcc9, + 0xc0c8f0de580a3fcc, + 0x4fb2dae6ee8fc793, + 0x343a89f35f37395b, + 0x223b525a77ca72c8, + ], + [ + 0x56ccb62574aaa918, + 0xc4d507d8027af9ed, + 0xa080673cf0b7e95c, + 0xf0184884eb70dcf8, + 0x044f10b0cb3d5c69, + 0xe9e3f7993938f186, + 0x1b761c80e772f459, + 0x606cec607a1b5fac, + ], + [ + 0x14a0c2e1d45f03cd, + 0x4eace8855398574f, + 0xf905ca7103eff3e6, + 0xf8c8f8d20862c059, + 0xb524fe8bdd678e5a, + 0xfbb7865901a1ec41, + 0x014ef1197d341346, + 0x9725e20825d07394, + ], + [ + 0xfdb25aef2c5bae3b, + 0xbe5402dc598c971e, + 0x93a5711f04cdca3d, + 0xc45a9a5b2f8fb97b, + 0xfe8946a924933545, + 0x2af997a27369091c, + 0xaa62c88e0b294011, + 0x058eb9d810ce9f74, + ], + [ + 0xb3cb23eced349ae4, + 0xa3648177a77b4a84, + 0x43153d905992d95d, + 0xf4e2a97cda44aa4b, + 0x5baa2702b908682f, + 0x082923bdf4f750d1, + 0x98ae09a325893803, + 0xf8a6475077968838, + ], + [ + 0xceb0735bf00b2c5f, + 0x0a1a5d953888e072, + 0x2fcb190489f94475, + 0xb5be06270dec69fc, + 0x739cb934b09acf8b, + 0x537750b75ec7f25b, + 0xe9dd318bae1f3961, + 0xf7462137299efe1a, + ], + [ + 0xb1f6b8eee9adb940, + 0xbdebcc8a809dfe6b, + 0x40fc1f791b178113, + 0x3ac1c3362d014864, + 0x9a016184bdb8aeba, + 0x95f2394459fbc25e, + 0xe3f34a07a76a66c2, + 0x8df25f9ad98b1b96, + ], + [ + 0x85ffc27171439d9d, + 0xddcb9a2dcfd26910, + 0x26b5ba4bf3afb94e, + 0xffff9cc7c7651e2f, + 0x8c88364698280b55, + 0xebc114167b910501, + 0x2d77b4d89ecfb516, + 0x332e0828eba151f2, + ], + [ + 0x46fa6a6450dd4735, + 0xd00db7dd92384a33, + 0x5fd4fb751f3a5fc5, + 0x496fb90c0bb65ea2, + 0xf3baec0bb87cc5c7, + 0x862a3c0a7d4c7713, + 0xbf5f38336a3f47d8, + 0x41ad9dbc1394a20c, + ], + [ + 0xcc535945b7dbf0f7, + 0x82af2bc93685bcec, + 0x8e4c8d0c8cebfccd, + 0x17cb39417e84597e, + 0xd4a965a8c749b232, + 0xa2cab040f33f3ee5, + 0xa98811a1fed4e3a6, + 0x1cc48b54f377e2a1, + ], + [ + 0xe40cd4f6c5609a27, + 0x11de79ebca97a4a4, + 0x9177c73d8b7e929d, + 0x2a6fe8085797e792, + 0x3de6e93329f8d5ae, + 0x3f7af9125da962ff, + 0xd710682cfc77d3ac, + 0x48faf05f3b053cf4, + ], + [ + 0x287db8630da89c8b, + 0x4d0de32053cb30e9, + 0x8b37a4f20c5ada7b, + 0xe7cc6ebe78c84ecf, + 0x240bdc0a66a2610d, + 0x8299e7f02caa1650, + 0x380a53fefb6e754e, + 0x684a1d8cf8eb6810, + ], + [ + 0xe839452eb4b8a5e1, + 0xb03fa62e90626af4, + 0x11a688602fbc5efc, + 0x30dda75c355a2d62, + 0x0f712adcb73810de, + 0xffdc1102187f1ae1, + 0x40c34f398254b99c, + 0xede021b9dc289a4a, + ], + [ + 0x8b7b05225c4e7dad, + 0x3bc794346f9d9ff9, + 0xfccb5a57f2ca86ff, + 0xbb1502015a7da9d4, + 0xd7e0a35d4352a015, + 0x27af7a44f8160931, + 0xc37442f6782f4615, + 0xbdf392a9bd095dcb, + ], + [ + 0xc17f55037cf00de9, + 0xbcffedd34c71a874, + 0x5eb45d2a8133d1f2, + 0xbabe251e1612ebdf, + 0x3efeb9fbe438c536, + 0x2d7cef97b4afe1cf, + 0xe5de1b4660016c0b, + 0xcdcc26c332f5657c, + ], + [ + 0xe01dd653daf15809, + 0xb0a6bdd4b41094b5, + 0x27eac858b0b03a05, + 0x51d43b5e93adbdc0, + 0x8b89a23b0fea5fc9, + 0xdc8ac3b14f7f2fc1, + 0xe793f82f1efec039, + 0x9f6f2cf8969e7b80, + ], + [ + 0x49d45382e0f21d4a, + 0x5f4ad1797cd72786, + 0x4dc3dbebfd45f795, + 0x03a3ef84dba6e1bc, + 0x204bc9b3d3fc4c01, + 0x9ad706081e89b9ba, + 0x638bfb4d840e9f89, + 0x5ef2938cd095ae35, + ], + [ + 0x42cca18ebeb265c8, + 0xb7b2ec5c29aecbf8, + 0x0d84f9535dc78f0f, + 0x04e64ad942e77b8c, + 0xb4880dffffc9da0b, + 0x16db16d9c29adeb1, + 0x09bbaf2a0590cd1e, + 0x76460e74961fcf8d, + ], + [ + 0xed12a2276dfa1553, + 0x0b5acec5de0436fd, + 0x3c6cfea033a1f0a8, + 0x2b5ecefe546cac15, + 0x6e2d82884cd3bf6f, + 0xc134878d1add7b83, + 0x997963422eb7a280, + 0x5e834537ac648cf6, + ], + [ + 0x89e779214737c0b7, + 0x1a8c05e8581ad95b, + 0x8d18b72796437cf7, + 0xe7252c949e04b106, + 0x53267c4fd174585a, + 0xa16ef5d9c81dad47, + 0xda65191937270a46, + 0xcb2a5b55f2df664c, + ], + [ + 0x854aee2dc1924137, + 0xf37013c9d479ece6, + 0x0e163bc0630c4696, + 0x384ee64955048f76, + 0xf65d814e28ee4ec5, + 0xe57bc564fd82f1b1, + 0x4b338937b6876614, + 0x66ee0b04ed43cd8d, + ], + [ + 0x49884bf25f4ef15d, + 0xeb51fe28de1c6f54, + 0x2cd64e84fce8dfcc, + 0x29164a96a541a013, + 0x173ce7558f4cacb8, + 0xeb5b1ce5877c89e9, + 0x5faff4b0f5217bf6, + 0xac42d0b1c20f205e, + ], + [ + 0xfb1d6bf0ca43221b, + 0x97b0a1b01d6a2955, + 0x08c60bd622952b30, + 0x43f2be0f9e24147c, + 0xfa7268b7d3730f5d, + 0x43a6c419a23983bb, + 0xcd77c1f7b29b113c, + 0xcfa43c9db8eec29f, + ], + [ + 0xcaaa95a6c7365dec, + 0x0a91193f798f3be0, + 0x1104497652735dc6, + 0x35aecb93663b515e, + 0x8dbc9916065aa858, + 0xada8f7a0266579ed, + 0x524dee7bec1ea789, + 0xa93aee9dd5af9521, + ], + [ + 0x9d1f1b54750d707e, + 0x7c9feab87096d5dc, + 0xa2e1fb19f9d4261b, + 0xb714deb448de6346, + 0x225d1f0d011c5403, + 0x1549b7f1d28cedc0, + 0xaef3e46f97d43942, + 0x6dfc7ffe0b38bf08, + ], + [ + 0x7de853fdc542b663, + 0xa68ecc96610657b2, + 0xe88bb5428af289b1, + 0xd7cfa1504c5569f5, + 0x78a9aad0d642d30a, + 0xd68315f2353dce52, + 0x46e56300f86fcfd5, + 0x323d95332b145fd6, + ], +]; diff --git a/crypto/crypto/src/hash/poseidon1_w8/mod.rs b/crypto/crypto/src/hash/poseidon1_w8/mod.rs new file mode 100644 index 000000000..85d696bc2 --- /dev/null +++ b/crypto/crypto/src/hash/poseidon1_w8/mod.rs @@ -0,0 +1,78 @@ +//! Poseidon1 over Goldilocks at width 8 — the grinding permutation of the +//! Poseidon1 base STARK (pil2-proofman's `Poseidon1_8`). +//! +//! ⚠ **Not a shipped hash**: an exploration-branch instance, like +//! [`super::poseidon1_w16`]. +//! +//! Same recipe as the width-16 instance: `x^7`, `R_F = 8`, `R_P = 22`, Grain +//! round constants (Plonky3's published width-8 table), and Plonky3's small +//! circulant `MATRIX_CIRC_MDS_8_SML_ROW`. ZisK's own implementation reproduces +//! the known answers ([`super::poseidon1_stark`]'s `PERM8` vectors). + +pub mod constants; + +use constants::{MDS_CIRC_ROW, ROUND_CONSTANTS}; + +use super::poseidon1_w16::{Fp, sbox}; + +/// Lanes in the permutation's state. +pub const STATE_FELTS: usize = 8; +/// Full rounds per half; `R_F = 8`. +pub const HALF_FULL_ROUNDS: usize = 4; +/// Partial rounds, S-box on lane 0 only. +pub const PARTIAL_ROUNDS: usize = 22; +/// All rounds. +pub const NUM_ROUNDS: usize = 2 * HALF_FULL_ROUNDS + PARTIAL_ROUNDS; + +/// [`MDS_CIRC_ROW`] expanded at compile time: `MDS_MATRIX[i][j] = row[(j − i) mod 8]`. +const MDS_MATRIX: [[u64; STATE_FELTS]; STATE_FELTS] = { + let mut m = [[0; STATE_FELTS]; STATE_FELTS]; + let mut i = 0; + while i < STATE_FELTS { + let mut j = 0; + while j < STATE_FELTS { + m[i][j] = MDS_CIRC_ROW[(j + STATE_FELTS - i) % STATE_FELTS]; + j += 1; + } + i += 1; + } + m +}; + +/// The circulant MDS product. The row sums to 43, so each lane's `u128` sum is +/// below `2^70` and `hi·EPSILON < 2^38` needs no reduction of its own. +fn mds(state: &[Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + /// `2^64 ≡ 2^32 − 1 (mod p)`. + const EPSILON: u64 = 0xFFFF_FFFF; + + let mut out = [Fp::zero(); STATE_FELTS]; + for (row, o) in MDS_MATRIX.iter().zip(out.iter_mut()) { + let mut acc: u128 = 0; + for (c, s) in row.iter().zip(state) { + acc += (*s.value() as u128) * (*c as u128); + } + let lo = acc as u64; + let hi = (acc >> 64) as u64; + *o = Fp::from(lo) + Fp::from(hi * EPSILON); + } + out +} + +/// The permutation: 4 full, 22 partial, 4 full rounds (textbook form). +pub fn permute(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let mut s = state; + for (r, rc) in ROUND_CONSTANTS.iter().enumerate() { + for (v, c) in s.iter_mut().zip(rc) { + *v += Fp::from(*c); + } + if !(HALF_FULL_ROUNDS..HALF_FULL_ROUNDS + PARTIAL_ROUNDS).contains(&r) { + for v in s.iter_mut() { + *v = sbox(v); + } + } else { + s[0] = sbox(&s[0]); + } + s = mds(&s); + } + s +} diff --git a/crypto/crypto/src/hash/rpx/constants.rs b/crypto/crypto/src/hash/rpx/constants.rs new file mode 100644 index 000000000..24a3c89e3 --- /dev/null +++ b/crypto/crypto/src/hash/rpx/constants.rs @@ -0,0 +1,254 @@ +//! The round constants and the MDS row of RPO256/RPX256 at width 12 — +//! **transcribed verbatim, and deliberately carrying no prose of their own.** +//! +//! # Provenance: two independent sources, checked against each other +//! +//! These are not this project's numbers and nothing here derives them: +//! +//! 1. the spec's own generator ([eprint 2022/1577](https://eprint.iacr.org/2022/1577), +//! reference implementation `github.com/ASDiscreteMathematics/rpo`) — +//! `SHAKE256("RPO(18446744069414584321,12,4,128)", 9*2*12*7)` cut into +//! nine-byte little-endian chunks reduced mod `p`; +//! 2. `miden-crypto`'s shipped `ARK1` / `ARK2` tables +//! (`src/hash/algebraic_sponge/rescue/mod.rs`), production code since 2022. +//! +//! The SHAKE256 derivation was re-run outside this repository and reproduces +//! miden's 168 constants exactly. [`MDS_CIRC_ROW`] is likewise the spec's +//! `get_mds(12)` and miden's `MDS` first row, identically. RPO's security +//! argument is MDS-AGNOSTIC (spec §4.1: "Rescue-Prime is secure when +//! instantiated with any MDS matrix"), so the row is a speed choice — it is +//! NTT-friendly — and not a security parameter. +//! +//! ⚠ **RPX shares these tables with RPO, byte for byte.** RPX is a round-SCHEDULE +//! swap on RPO's geometry, not a redesign: same width, same rate 8 / capacity 4, +//! same digest width, same MDS and literally the same `ARK1`/`ARK2`. That is +//! what lets the nineteen external RPO known-answer vectors pin RPX's constants +//! too — see the module header of [`super`]. +//! +//! ⚠ **The same numbers appear in the CUDA kernel** (`math-cuda/kernels/rpx.cu`, +//! `__constant__ ARK1`/`ARK2`/`MDS_CIRC_ROW2`). They are pinned against each +//! other by the host known-answer harness, not by being edited together, so a +//! divergence is caught rather than merely discouraged. + +use super::STATE_FELTS; + +/// The forward S-box exponent. Like Poseidon's, 7 is forced by Goldilocks: +/// `p - 1 = 2^32 * 3 * 5 * 17 * 257 * 65537`, so neither 3 nor 5 is coprime to +/// it and neither `x^3` nor `x^5` is a permutation. +pub const ALPHA: u32 = 7; + +/// The inverse S-box exponent, `ALPHA^-1 mod (p - 1)`. +/// +/// ~2^63, and that is the point: the map is cheap in one direction and +/// astronomically dense in the other. `tests::the_inverse_exponent_inverts_alpha` +/// re-derives it rather than trusting the literal. +pub const INV_ALPHA: u64 = 10540996611094048183; + +/// Rounds. The spec's own formula gives 8; RPO ships 7 and defends the 12.5% +/// shave in §4.2 with a 1.5x margin argument and Gröbner estimates above twice +/// the security level. +pub const NUM_ROUNDS: usize = 7; + +pub const MDS_CIRC_ROW: [u64; STATE_FELTS] = [7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8]; + +pub const ARK1: [[u64; STATE_FELTS]; NUM_ROUNDS] = [ + [ + 5789762306288267392, + 6522564764413701783, + 17809893479458208203, + 107145243989736508, + 6388978042437517382, + 15844067734406016715, + 9975000513555218239, + 3344984123768313364, + 9959189626657347191, + 12960773468763563665, + 9602914297752488475, + 16657542370200465908, + ], + [ + 12987190162843096997, + 653957632802705281, + 4441654670647621225, + 4038207883745915761, + 5613464648874830118, + 13222989726778338773, + 3037761201230264149, + 16683759727265180203, + 8337364536491240715, + 3227397518293416448, + 8110510111539674682, + 2872078294163232137, + ], + [ + 18072785500942327487, + 6200974112677013481, + 17682092219085884187, + 10599526828986756440, + 975003873302957338, + 8264241093196931281, + 10065763900435475170, + 2181131744534710197, + 6317303992309418647, + 1401440938888741532, + 8884468225181997494, + 13066900325715521532, + ], + [ + 5674685213610121970, + 5759084860419474071, + 13943282657648897737, + 1352748651966375394, + 17110913224029905221, + 1003883795902368422, + 4141870621881018291, + 8121410972417424656, + 14300518605864919529, + 13712227150607670181, + 17021852944633065291, + 6252096473787587650, + ], + [ + 4887609836208846458, + 3027115137917284492, + 9595098600469470675, + 10528569829048484079, + 7864689113198939815, + 17533723827845969040, + 5781638039037710951, + 17024078752430719006, + 109659393484013511, + 7158933660534805869, + 2955076958026921730, + 7433723648458773977, + ], + [ + 16308865189192447297, + 11977192855656444890, + 12532242556065780287, + 14594890931430968898, + 7291784239689209784, + 5514718540551361949, + 10025733853830934803, + 7293794580341021693, + 6728552937464861756, + 6332385040983343262, + 13277683694236792804, + 2600778905124452676, + ], + [ + 7123075680859040534, + 1034205548717903090, + 7717824418247931797, + 3019070937878604058, + 11403792746066867460, + 10280580802233112374, + 337153209462421218, + 13333398568519923717, + 3596153696935337464, + 8104208463525993784, + 14345062289456085693, + 17036731477169661256, + ], +]; + +pub const ARK2: [[u64; STATE_FELTS]; NUM_ROUNDS] = [ + [ + 6077062762357204287, + 15277620170502011191, + 5358738125714196705, + 14233283787297595718, + 13792579614346651365, + 11614812331536767105, + 14871063686742261166, + 10148237148793043499, + 4457428952329675767, + 15590786458219172475, + 10063319113072092615, + 14200078843431360086, + ], + [ + 6202948458916099932, + 17690140365333231091, + 3595001575307484651, + 373995945117666487, + 1235734395091296013, + 14172757457833931602, + 707573103686350224, + 15453217512188187135, + 219777875004506018, + 17876696346199469008, + 17731621626449383378, + 2897136237748376248, + ], + [ + 8023374565629191455, + 15013690343205953430, + 4485500052507912973, + 12489737547229155153, + 9500452585969030576, + 2054001340201038870, + 12420704059284934186, + 355990932618543755, + 9071225051243523860, + 12766199826003448536, + 9045979173463556963, + 12934431667190679898, + ], + [ + 18389244934624494276, + 16731736864863925227, + 4440209734760478192, + 17208448209698888938, + 8739495587021565984, + 17000774922218161967, + 13533282547195532087, + 525402848358706231, + 16987541523062161972, + 5466806524462797102, + 14512769585918244983, + 10973956031244051118, + ], + [ + 6982293561042362913, + 14065426295947720331, + 16451845770444974180, + 7139138592091306727, + 9012006439959783127, + 14619614108529063361, + 1394813199588124371, + 4635111139507788575, + 16217473952264203365, + 10782018226466330683, + 6844229992533662050, + 7446486531695178711, + ], + [ + 3736792340494631448, + 577852220195055341, + 6689998335515779805, + 13886063479078013492, + 14358505101923202168, + 7744142531772274164, + 16135070735728404443, + 12290902521256031137, + 12059913662657709804, + 16456018495793751911, + 4571485474751953524, + 17200392109565783176, + ], + [ + 17130398059294018733, + 519782857322261988, + 9625384390925085478, + 1664893052631119222, + 7629576092524553570, + 3485239601103661425, + 9755891797164033838, + 15218148195153269027, + 16460604813734957368, + 9643968136937729763, + 3611348709641382851, + 18256379591337759196, + ], +]; diff --git a/crypto/crypto/src/hash/rpx/mod.rs b/crypto/crypto/src/hash/rpx/mod.rs new file mode 100644 index 000000000..c9576b936 --- /dev/null +++ b/crypto/crypto/src/hash/rpx/mod.rs @@ -0,0 +1,586 @@ +//! Rescue-Prime eXtended (RPX256 / XHash12) over Goldilocks at width 12. +//! +//! The algebraic hash the WHIR recursion arm commits, transcripts and grinds +//! with. Ported from `prover::lfm::{rpo, rpx, algebraic_commit}` on the +//! per-table branch, where it is the production-candidate tenant of the +//! `LFM_HASH` socket; the permutation, the leaf construction, the parent and +//! every constant are byte-for-byte the same, because the CUDA kernel and its +//! known-answer tables are pinned to exactly those. +//! +//! # Why an algebraic hash at all +//! +//! Only for a proof that is going to be VERIFIED INSIDE a proof. In software a +//! keccak-f is far cheaper than this. In a field-native verifier the ratio +//! inverts by two orders of magnitude: a keccak-f[1600] costs ~73,700 trace +//! cells against RPX's 325, which is the difference between a WHIR wrap that is +//! twice today's and one that is a third of it. Nothing about this hash is an +//! improvement on keccak for a host prover, and the seam that selects it says +//! so. +//! +//! # What it is, and what it shares with RPO +//! +//! RPX is a **round-function swap on RPO's geometry**, not a redesign +//! ([eprint 2023/1045](https://eprint.iacr.org/2023/1045)): the same state +//! width 12, the same rate 8 / capacity 4, the same four-felt digest, the same +//! MDS and literally the same `ARK1`/`ARK2` tables. What changes is the +//! seven-round schedule: +//! +//! | round | kind | content | +//! |---|---|---| +//! | 0, 2, 4 | **FB** | MDS → +ARK1 → `x^7` → MDS → +ARK2 → `x^{1/7}` — RPO's round exactly | +//! | 1, 3, 5 | **E** | +ARK1 → `x^7` in the degree-3 EXTENSION, on four lane-triples. **No MDS.** | +//! | 6 | **M** | MDS → +ARK1. A linear finish, no S-box. | +//! +//! The E round has no linear layer: its only mixing is the extension +//! multiplication inside each triple, and diffusion across triples is the FB +//! rounds' job. That is the design, not an omission (✓ miden's +//! `Rpx256::apply_ext_round_ref`). +//! +//! # ⚠ PROVENANCE — WEAKER THAN RPO'S, AND THAT MUST BE SAID +//! +//! **miden publishes no RPX known-answer table** — ✓ VERIFIED, its `rpx/tests.rs` +//! carries only structural tests (consistency, determinism, padding, no-panic), +//! no oracle. So RPX cannot be anchored end to end the way RPO is, and this +//! module does not pretend otherwise. What it anchors instead: +//! +//! 1. **The shared half is externally anchored through RPO.** Seven `fb_round`s +//! compose to RPO256, and [`tests`] replays that composition over +//! miden-crypto's nineteen `hash_elements` vectors — numbers nothing in this +//! repository produced. They pin `ARK1`/`ARK2`, the MDS row and its +//! orientation, both S-box chains and the lane convention at once. +//! 2. **The new half is pinned to INDEPENDENT algorithms.** The cubic +//! extension's product against naive polynomial multiplication reduced mod +//! `φ³ − φ − 1`; `power7` against generic square-and-multiply in that +//! extension; the inverse S-box chain against `pow(INV_ALPHA)`. Different +//! algorithms for the same functions, not a second transcription. +//! 3. **The schedule** is the one miden's `Rpx256::apply_permutation` runs. +//! +//! ⚖ Net: strong on arithmetic, weaker on end-to-end identity than RPO. A +//! deployment decision should treat "no published KAT" as a real cost. RPX is +//! also a 2023 design and carries a young-design discount BLAKE3 and keccak do +//! not. +//! +//! # ⚠ NOT XHash8 +//! +//! XHash8 is the faster sibling and is deliberately not built here. Its extra +//! speed comes from a PARTIAL S-box layer (8 lanes of 12), and a partial layer +//! is one of the structural footholds the 2026 Poseidon collapse used — +//! eprint 2026/1692's S-box-skipping gadget restricts into the affine +//! complement of the un-S-boxed lanes, independent of round constants and MDS +//! choice. XHash8's S-boxes are not Poseidon's and eprint 2024/605 analyses +//! XHASH8/12 directly, so this is a flag rather than a verdict — but it is not +//! a thing to adopt quietly for the speed. +//! +//! # Lane convention, domains, and the rules that must not drift +//! +//! Lanes follow **miden's**: rate `0..8`, capacity `8..12`, digest `0..4`. +//! Capacity lane 0 carries the sponge's padding flag `len mod 8`; capacity lane +//! 1 carries a DOMAIN tag, which is miden's `merge_in_domain` mechanism. The +//! security argument is the RPX spec's Appendix C: setting a capacity element +//! to a domain tag degrades only pre-image resistance, by at most the log2 of +//! the domain space, and pre-image is not the sponge's binding term until it +//! falls under 2^128. +//! +//! ⚠ **The domain VALUES are pinned by the device kernel and its KAT tables** +//! (`rpx.cu`'s `DOMAIN_COMPRESS = 0`, `DOMAIN_LEAF = 0x4C4D464C`). The `LFM` +//! spelling of [`DOMAIN_LEAF`] is a historical name — it is `"LFML"` read as a +//! little-endian `u32` — and renaming the constant is free while **changing its +//! value forks the hash** from the kernel, from the KAT header and from every +//! root the per-table branch produced. + +pub mod constants; +#[cfg(test)] +mod tests; + +use alloc::vec::Vec; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::{IsField, IsPrimeField}; +use math::traits::AsBytes; + +use constants::{ARK1, ARK2, MDS_CIRC_ROW, NUM_ROUNDS}; + +/// A Goldilocks field element — the only field this hash is defined over. +pub type Fp = FieldElement; + +/// Lanes in the permutation's state. +pub const STATE_FELTS: usize = 12; +/// Lanes a block of input overwrites — the sponge's rate. +pub const RATE_FELTS: usize = 8; +/// Felts in a digest, hence a 32-byte commitment. +pub const DIGEST_FELTS: usize = 4; +/// Bytes one Goldilocks felt serialises to. +pub const BYTES_PER_FELT: usize = 8; + +/// Capacity lane carrying the sponge padding flag — reserved, never a domain. +pub const CAPACITY_PAD_LANE: usize = 0; +/// Capacity lane carrying the DOMAIN identifier — miden's `merge_in_domain` slot. +pub const CAPACITY_DOMAIN_LANE: usize = 1; + +/// The Merkle-parent domain: ZERO, deliberately. +/// +/// A parent is then bit-identical to `Rpo256::merge`/`Rpx256::merge`, so a +/// parent this code produces is checkable against miden's shipped +/// implementation without knowing anything about this codebase. +pub const DOMAIN_COMPRESS: u64 = 0; + +/// The Merkle LEAF domain — `"LFML"` as a little-endian `u32`, i.e. +/// `0x4C4D464C`. See the module header: the name is historical, the VALUE is +/// pinned by the device kernel and the KAT tables. +pub const DOMAIN_LEAF: u64 = u32::from_le_bytes(*b"LFML") as u64; + +/// Lanes per extension element: the RPX E round reads the state as FOUR triples. +pub const EXT_DEGREE: usize = 3; +/// Extension elements per E round. +pub const EXT_ELEMENTS: usize = STATE_FELTS / EXT_DEGREE; + +/// A four-felt digest. +pub type Digest = [Fp; DIGEST_FELTS]; + +/// The capacity cell for a domain: `[0, domain, 0, 0]`. +/// +/// One rule, stated once, so nothing can disagree about which lane the tag +/// lives in. +pub const fn domain_iv(domain: u64) -> [u64; DIGEST_FELTS] { + let mut iv = [0u64; DIGEST_FELTS]; + iv[CAPACITY_DOMAIN_LANE] = domain; + iv +} + +/// Is round `r` an **FB** round — MDS, forward S-box, MDS, inverse S-box? +pub const fn is_fb_round(r: usize) -> bool { + r.is_multiple_of(2) && r + 1 < NUM_ROUNDS +} + +/// Is round `r` an **E** round — constants then `x^7` in the cubic extension, +/// with NO linear layer? +pub const fn is_ext_round(r: usize) -> bool { + !r.is_multiple_of(2) +} + +/// Is round `r` the **M** round — MDS then constants, and nothing else? +pub const fn is_final_round(r: usize) -> bool { + r + 1 == NUM_ROUNDS +} + +/// Arithmetic in `GF(p³) = GF(p)[φ] / (φ³ − φ − 1)`. +/// +/// ⚠ **Not the VM's own extension**, which is built on `w³ = 2`. Mixing them +/// would be a wrong hash that still type-checks, so this carries its own +/// arithmetic explicitly and never reaches for the VM's. +pub mod cubic_ext { + use super::{EXT_DEGREE, Fp}; + + /// An extension element `a0 + a1·φ + a2·φ²`. + pub type Ext = [Fp; EXT_DEGREE]; + + /// The product, reduced by `φ³ = φ + 1` and `φ⁴ = φ² + φ`. + /// + /// The closed form rather than miden's Karatsuba arrangement, so the three + /// coefficients read as the definition. + /// `tests::the_extension_product_matches_naive_polynomial_arithmetic` pins + /// it against an independent algorithm. + pub fn mul(a: &Ext, b: &Ext) -> Ext { + [ + &(&a[0] * &b[0]) + &(&(&a[1] * &b[2]) + &(&a[2] * &b[1])), + &(&(&a[0] * &b[1]) + &(&a[1] * &b[0])) + + &(&(&(&a[1] * &b[2]) + &(&a[2] * &b[1])) + &(&a[2] * &b[2])), + &(&(&a[0] * &b[2]) + &(&a[1] * &b[1])) + &(&(&a[2] * &b[0]) + &(&a[2] * &b[2])), + ] + } + + /// The square. One function, so a squaring and a product cannot disagree. + pub fn square(a: &Ext) -> Ext { + mul(a, a) + } + + /// `a^7` by the chain `a² → a³ → a⁶ → a⁷`. + pub fn power7(a: &Ext) -> Ext { + let a2 = square(a); + let a3 = mul(&a2, a); + let a6 = square(&a3); + mul(&a6, a) + } +} + +/// `x^7`, in exactly the association the AIR's degree-3 lowering uses +/// (`x²`, `x³ = x²·x`, `x^7 = (x³)²·x`). +pub fn sbox(x: &Fp) -> Fp { + let x2 = x * x; + let x3 = &x2 * x; + let x6 = &x3 * &x3; + &x6 * x +} + +/// `x^{1/7}` over the WHOLE STATE, by miden-crypto's documented addition chain +/// (72 multiplications for a ~2^63 exponent, against ~93 for naive +/// square-and-multiply). +/// +/// ★ **Whole-state rather than per-element, and that is a measurement.** The +/// chain is 72 multiplications each depending on the last, so a single lane is +/// LATENCY-bound and the multiplier pipeline sits idle between them. The twelve +/// lanes are independent, so running them in lockstep interleaves twelve chains +/// and fills it. This layer is the dominant cost of the permutation. +pub fn inv_sbox_layer(state: &mut [Fp; STATE_FELTS]) { + /// `base^(2^m) · tail`, lane-wise — the chain's one building block. + fn exp_acc(base: &[Fp; STATE_FELTS], tail: &[Fp; STATE_FELTS], m: usize) -> [Fp; STATE_FELTS] { + let mut acc = *base; + for _ in 0..m { + for a in acc.iter_mut() { + *a = a.square(); + } + } + core::array::from_fn(|i| &acc[i] * &tail[i]) + } + + let t1: [Fp; STATE_FELTS] = core::array::from_fn(|i| state[i].square()); + let t2: [Fp; STATE_FELTS] = core::array::from_fn(|i| t1[i].square()); + let t3 = exp_acc(&t2, &t2, 3); + let t4 = exp_acc(&t3, &t3, 6); + let t5 = exp_acc(&t4, &t4, 12); + let t6 = exp_acc(&t5, &t3, 6); + let t7 = exp_acc(&t6, &t6, 31); + for (i, s) in state.iter_mut().enumerate() { + let a = (&t7[i].square() * &t6[i]).square().square(); + let b = &(&t1[i] * &t2[i]) * &*s; + *s = &a * &b; + } +} + +/// [`inv_sbox_layer`] for a single element — the same chain, not a second +/// transcription of it. +pub fn inv_sbox(x: &Fp) -> Fp { + let mut state = [*x; STATE_FELTS]; + inv_sbox_layer(&mut state); + state[0] +} + +/// The circulant MDS product, `out_i = Σ_j MDS_CIRC_ROW[(j − i) mod 12]·s_j`. +/// +/// ★ **One `u128` accumulation and one reduction per lane, not twelve field +/// multiplications.** The constants are all ≤ 26, so every term `c·s_j` fits in +/// 70 bits and the twelve-term row sum fits in 73 — comfortably inside a +/// `u128`. The row is accumulated with no reduction and reduced once at the end +/// using `2^64 ≡ EPSILON (mod p)`: `hi·2^64 + lo ≡ lo + hi·EPSILON`, and with +/// `hi < 2^9` the correction `hi·EPSILON < 2^41` needs no reduction of its own. +/// `tests::the_mds_row_sum_cannot_overflow_a_u128` asserts the bound. +/// +/// ⚠ **Plain loops over [`MDS_MATRIX`], not a `core::array::from_fn` closure.** +/// The closure's `from_fn` wrapper is a generic instance that rustc places in a +/// codegen unit of its own choosing, and it is inlined here only when that unit +/// happens to be this function's. When it is not, every lane becomes an +/// out-of-line call that re-derives `(j − i) mod 12` with a 64-bit multiply per +/// term — about a fifth more instructions per permutation, decided by unrelated +/// edits elsewhere in the crate. Loops over a compile-time matrix compile the +/// same way in every build. +pub fn mds(state: &[Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + /// `2^32 − 1`, and `2^64 ≡ EPSILON (mod p)` for the Goldilocks prime. + /// Written here rather than imported because the field crate keeps its own + /// copy private; `tests::the_epsilon_identity_holds` re-derives it. + const EPSILON: u64 = 0xFFFF_FFFF; + + let mut out = [Fp::zero(); STATE_FELTS]; + for (row, o) in MDS_MATRIX.iter().zip(out.iter_mut()) { + let mut acc: u128 = 0; + for (c, s) in row.iter().zip(state) { + acc += (*s.value() as u128) * (*c as u128); + } + let lo = acc as u64; + let hi = (acc >> 64) as u64; + // hi < 2^9, so hi·EPSILON < 2^41 and neither `from` reduces twice. + *o = Fp::from(lo) + Fp::from(hi * EPSILON); + } + out +} + +/// [`MDS_CIRC_ROW`] expanded to the whole circulant at compile time, +/// `MDS_MATRIX[i][j] = MDS_CIRC_ROW[(j − i) mod 12]`, so [`mds`] reads every row +/// as constants instead of rotating an index at run time. +const MDS_MATRIX: [[u64; STATE_FELTS]; STATE_FELTS] = { + let mut m = [[0; STATE_FELTS]; STATE_FELTS]; + let mut i = 0; + while i < STATE_FELTS { + let mut j = 0; + while j < STATE_FELTS { + m[i][j] = MDS_CIRC_ROW[(j + STATE_FELTS - i) % STATE_FELTS]; + j += 1; + } + i += 1; + } + m +}; + +/// ★ The RPX permutation: `FB E FB E FB E M`. +pub fn permute(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let mut s = state; + // Over ARK1 rather than over `0..NUM_ROUNDS`: the round index is still what + // `fb_round` takes, but the E and M rounds read ARK1 and only ARK1, and + // iterating it says so. + for (r, ark1) in ARK1.iter().enumerate() { + if is_fb_round(r) { + s = fb_round(s, r); + } else if is_ext_round(r) { + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ark1[lane]); + } + let mut next = [Fp::zero(); STATE_FELTS]; + for e in 0..EXT_ELEMENTS { + let base = e * EXT_DEGREE; + let x: cubic_ext::Ext = core::array::from_fn(|k| s[base + k]); + let p = cubic_ext::power7(&x); + next[base..base + EXT_DEGREE].copy_from_slice(&p); + } + s = next; + } else { + debug_assert!(is_final_round(r)); + s = mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ark1[lane]); + } + } + } + s +} + +/// One **FB** round: `MDS → +ARK1 → x^7 → MDS → +ARK2 → x^{1/7}`. +/// +/// ★ Exported because it is RPO's round EXACTLY, and seven of them composed are +/// RPO256 — which is how the nineteen external miden vectors reach RPX's +/// constants. [`tests::seven_fb_rounds_are_rpo256`] is that bridge. +pub fn fb_round(state: [Fp; STATE_FELTS], r: usize) -> [Fp; STATE_FELTS] { + let mut s = mds(&state); + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ARK1[r][lane]); + } + for v in s.iter_mut() { + *v = sbox(v); + } + s = mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ARK2[r][lane]); + } + inv_sbox_layer(&mut s); + s +} + +// ========================================================================= +// The sponge: leaves, parents, and the felt/byte conventions +// ========================================================================= + +/// ★ **THE LEAF CAPACITY RULE, stated once.** +/// +/// Lane 0 is the padding flag `len mod 8` — zero when the length divides the +/// rate, which is why no trailing block is spent on an exact multiple — and +/// lane 1 the LEAF domain. +pub fn leaf_capacity(num_felts: usize) -> Digest { + let iv = domain_iv(DOMAIN_LEAF); + let mut cap: Digest = core::array::from_fn(|k| Fp::from(iv[k])); + cap[CAPACITY_PAD_LANE] = Fp::from((num_felts % RATE_FELTS) as u64); + cap +} + +/// ★ The rate-8 OVERWRITE duplex over a felt stream — the leaf construction. +/// +/// Each block OVERWRITES the eight rate lanes (RPO spec §2.6), so absorption +/// costs no field arithmetic outside the permutation; the tail block is +/// zero-padded. It absorbs eight fresh felts per permutation where a +/// four-felt chain absorbs four. +pub fn sponge_leaf(felts: &[Fp]) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + let cap = leaf_capacity(felts.len()); + state[RATE_FELTS..].copy_from_slice(&cap); + + if felts.is_empty() { + return [state[0], state[1], state[2], state[3]]; + } + for block in felts.chunks(RATE_FELTS) { + for (lane, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + *slot = block.get(lane).copied().unwrap_or_else(Fp::zero); + } + state = permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// `sponge_leaf(&felts_from_bytes(bytes))`, without materialising the felts. +/// +/// ⚠ Equivalent to the two-step form BY TEST +/// (`tests::sponge_leaf_bytes_matches_the_felt_form`), not by construction: the +/// trailing partial group is zero-extended on the LOW side here, which is what +/// [`felts_from_bytes`] does and is easy to get backwards. +pub fn sponge_leaf_bytes(bytes: &[u8]) -> Digest { + let num_felts = bytes.len().div_ceil(BYTES_PER_FELT); + let mut state = [Fp::zero(); STATE_FELTS]; + let cap = leaf_capacity(num_felts); + state[RATE_FELTS..].copy_from_slice(&cap); + + if bytes.is_empty() { + return [state[0], state[1], state[2], state[3]]; + } + // One rate block is eight felts, i.e. 64 bytes. + for block in bytes.chunks(RATE_FELTS * BYTES_PER_FELT) { + for (lane, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + let start = lane * BYTES_PER_FELT; + *slot = if start >= block.len() { + Fp::zero() + } else { + let end = (start + BYTES_PER_FELT).min(block.len()); + let mut b = [0u8; BYTES_PER_FELT]; + b[..end - start].copy_from_slice(&block[start..end]); + Fp::from(u64::from_be_bytes(b)) + }; + } + state = permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// ★ A Merkle parent: ONE permutation of `[left ‖ right ‖ capacity]` with the +/// compress domain, which is zero — so a parent is literally `Rpx256::merge` +/// and externally checkable against miden. +pub fn compress(left: &Digest, right: &Digest) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + state[..DIGEST_FELTS].copy_from_slice(left); + state[DIGEST_FELTS..RATE_FELTS].copy_from_slice(right); + let iv = domain_iv(DOMAIN_COMPRESS); + for (k, slot) in state[RATE_FELTS..].iter_mut().enumerate() { + *slot = Fp::from(iv[k]); + } + let out = permute(state); + [out[0], out[1], out[2], out[3]] +} + +/// Four felts as 32 canonical BIG-endian bytes. +pub fn digest_to_commitment(d: &Digest) -> [u8; 32] { + let mut out = [0u8; 32]; + for (i, f) in d.iter().enumerate() { + let v = GoldilocksField::canonical(f.value()); + out[i * BYTES_PER_FELT..(i + 1) * BYTES_PER_FELT].copy_from_slice(&v.to_be_bytes()); + } + out +} + +/// 32 bytes back to four felts. +pub fn commitment_to_digest(c: &[u8; 32]) -> Digest { + core::array::from_fn(|i| { + let mut b = [0u8; BYTES_PER_FELT]; + b.copy_from_slice(&c[i * BYTES_PER_FELT..(i + 1) * BYTES_PER_FELT]); + Fp::from(u64::from_be_bytes(b)) + }) +} + +/// Every 8-byte big-endian group of `bytes` as a felt. +/// +/// The inverse of the serialisation `ByteConversion::write_bytes_be` performs, +/// which is how leaves reach a Merkle backend. A trailing partial group is +/// zero-extended on the LOW side, matching how a short write would land. +pub fn felts_from_bytes(bytes: &[u8]) -> Vec { + bytes + .chunks(BYTES_PER_FELT) + .map(|c| { + let mut b = [0u8; BYTES_PER_FELT]; + b[..c.len()].copy_from_slice(c); + Fp::from(u64::from_be_bytes(b)) + }) + .collect() +} + +/// Decompose a field element — base or extension — into its base felts, by the +/// same serialisation the STARK uses. +/// +/// ★ Through `AsBytes::stream_bytes` rather than `ByteConversion::write_bytes_be`, +/// and the two are the SAME bytes. The reason for the weaker trait is not +/// style: a Merkle backend generic over `F` has `FieldElement: AsBytes` and +/// nothing more, so a decomposition that required `ByteConversion` could not be +/// used there at all. +pub fn element_felts(e: &FieldElement, out: &mut Vec) +where + F: IsField, + FieldElement: AsBytes, +{ + let mut buf = [0u8; 64]; + let mut len = 0usize; + e.stream_bytes(&mut |bytes| { + debug_assert!( + len + bytes.len() <= buf.len(), + "a field element must fit the scratch" + ); + buf[len..len + bytes.len()].copy_from_slice(bytes); + len += bytes.len(); + }); + out.extend(felts_from_bytes(&buf[..len])); +} + +// ========================================================================= +// The `digest::Digest` adapter — what a transcript and the grind consume +// ========================================================================= + +/// RPX256 as a `digest::Digest`, for the two places that take one: the +/// Fiat-Shamir sponge and the proof-of-work grind. +/// +/// # The construction is the LEAF one, deliberately +/// +/// Grinding hashes a byte string — `state ‖ nonce`, 40 bytes, five felts — +/// which is DATA, exactly what a leaf is. It therefore reuses [`sponge_leaf`] +/// and the LEAF domain rather than inventing a fourth. The reuse is not +/// exploitable: the grinding check tests leading zeros of a hash whose preimage +/// is transcript-bound, so colliding it with some leaf digest buys an adversary +/// nothing. +/// +/// # Why it buffers +/// +/// [`sponge_leaf`]'s padding flag is `len mod 8`, needed in the capacity before +/// the FIRST permutation, so an incremental sponge cannot start until the total +/// length is known. Inventing a length-free padding rule instead would be a +/// cryptographic decision this port does not get to make. +#[derive(Default, Clone)] +pub struct Rpx256Digest { + buf: Vec, +} + +impl Rpx256Digest { + /// The digest of everything absorbed so far. + pub fn finalize_digest(&self) -> [u8; 32] { + digest_to_commitment(&sponge_leaf_bytes(&self.buf)) + } +} + +impl digest::HashMarker for Rpx256Digest {} + +impl digest::OutputSizeUser for Rpx256Digest { + type OutputSize = digest::typenum::U32; +} + +impl digest::Update for Rpx256Digest { + fn update(&mut self, data: &[u8]) { + // ⚠ The GENERIC counters were keccak-only: `count_absorb` is bumped + // from the keccak wrapper's `update` and nothing bumped it here, so + // `absorb_calls` read ZERO for an RPX proof and `total`'s own + // documentation — which says it counts transcript squeezes — was false + // for this sponge. The same trap this module's sibling header describes + // for Merkle, which the transcript and the absorb counters never got. + crate::hash_metrics::count_absorb(data.len()); + self.buf.extend_from_slice(data); + } +} + +impl digest::FixedOutput for Rpx256Digest { + fn finalize_into(self, out: &mut digest::Output) { + crate::hash_metrics::count_total(); + out.copy_from_slice(&self.finalize_digest()); + } +} + +impl digest::Reset for Rpx256Digest { + fn reset(&mut self) { + self.buf.clear(); + } +} + +impl digest::FixedOutputReset for Rpx256Digest { + fn finalize_into_reset(&mut self, out: &mut digest::Output) { + crate::hash_metrics::count_total(); + out.copy_from_slice(&self.finalize_digest()); + self.buf.clear(); + } +} diff --git a/crypto/crypto/src/hash/rpx/tests.rs b/crypto/crypto/src/hash/rpx/tests.rs new file mode 100644 index 000000000..58d5b9ac1 --- /dev/null +++ b/crypto/crypto/src/hash/rpx/tests.rs @@ -0,0 +1,1108 @@ +//! The oracles RPX256 rests on, and the independent algorithms that hold up +//! the half no oracle covers. +//! +//! # Two anchors, and they are not the same strength +//! +//! **RPO's half is EXTERNAL.** Seven `fb_round`s composed ARE RPO256 — RPX is a +//! schedule swap on RPO's geometry with literally the same constants — so +//! [`MIDEN_HASH_ELEMENTS`], nineteen `hash_elements` vectors published by +//! miden-crypto, reach into this file from outside. They pin `ARK1`, `ARK2`, +//! the MDS row AND its orientation, both S-box chains and the lane convention +//! at once. Nothing in this repository produced those seventy-six numbers. +//! +//! **RPX's own half is NOT externally anchored, and this says so rather than +//! implying otherwise.** miden publishes no RPX known-answer table (✓ VERIFIED: +//! its `rpx/tests.rs` carries only structural tests — consistency, determinism, +//! padding, no-panic). So the RPX tables below are the per-table branch's host +//! implementation speaking: [`RPX_PERMUTATION_VECTORS`], [`RPX_LEAF_VECTORS`] +//! and [`RPX_PARENT_VECTORS`] are transcribed from +//! `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h` on `per-table-gpu` +//! (introduced by `50c633e1`, the tables printed by +//! `prover/tests/rpx_host_kat_vectors.rs` from `prover::lfm::rpx::Rpx256`, +//! `73ee2a64`). That is worth having for a reason beyond "someone else agrees": +//! **the CUDA kernel is pinned to those same tables**, so a port that +//! reproduces them is byte-compatible with both the other branch's host and its +//! device, which is the property H2's device half will need. +//! +//! What still has no oracle at all is the E round and the schedule. Those rest +//! on layer 2: the cubic extension's product against naive polynomial +//! arithmetic mod `φ³ − φ − 1`, `power7` against square-and-multiply, the +//! inverse S-box against `pow(INV_ALPHA)` — different algorithms for the same +//! functions, not second transcriptions of the same one. + +use super::constants::{ALPHA, ARK1, ARK2, INV_ALPHA, MDS_CIRC_ROW, NUM_ROUNDS}; +use super::*; +use alloc::vec::Vec; + +/// The Goldilocks prime. +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn fe(v: u64) -> Fp { + Fp::from(v) +} + +fn felts(vs: &[u64]) -> Vec { + vs.iter().copied().map(fe).collect() +} + +fn state_of(vs: &[u64; STATE_FELTS]) -> [Fp; STATE_FELTS] { + core::array::from_fn(|i| fe(vs[i])) +} + +fn raw(state: &[Fp; STATE_FELTS]) -> [u64; STATE_FELTS] { + core::array::from_fn(|i| GoldilocksField::canonical(state[i].value())) +} + +fn digest_of(vs: &[u64; DIGEST_FELTS]) -> Digest { + core::array::from_fn(|i| fe(vs[i])) +} + +fn raw_digest(d: &Digest) -> [u64; DIGEST_FELTS] { + core::array::from_fn(|i| GoldilocksField::canonical(d[i].value())) +} + +// ========================================================================= +// LAYER 1 — the EXTERNAL anchor: miden's RPO256 vectors, through `fb_round` +// ========================================================================= + +/// miden-crypto's own `hash_elements` known-answer table — an EXTERNAL oracle. +/// +/// Source: `miden-crypto/src/hash/algebraic_sponge/rescue/rpo/tests.rs`, +/// `EXPECTED` / `hash_test_vectors`. Entry `n` is the digest of the field +/// elements `[0, 1, …, n]`. +/// +/// Entries 1–7 and 9–19 exercise the padding path (`len % 8 ≠ 0`), entries 8 +/// and 16 the exact-block path, and everything above 8 chains two permutations +/// through the capacity — so the table pins the sponge's carry, not only one +/// permutation. +const MIDEN_HASH_ELEMENTS: [[u64; 4]; 19] = [ + [ + 8563248028282119176, + 14757918088501470722, + 14042820149444308297, + 7607140247535155355, + ], + [ + 8762449007102993687, + 4386081033660325954, + 5000814629424193749, + 8171580292230495897, + ], + [ + 16710087681096729759, + 10808706421914121430, + 14661356949236585983, + 5683478730832134441, + ], + [ + 5309818427047650994, + 17172251659920546244, + 8288476618870804357, + 18080473279382182941, + ], + [ + 3647545403045515695, + 3358383208908083302, + 8797161010298072910, + 2412100201132087248, + ], + [ + 8409780526028662686, + 214479528340808320, + 13626616722984122219, + 13991752159726061594, + ], + [ + 4800410126693035096, + 8293686005479024958, + 16849389505608627981, + 12129312715917897796, + ], + [ + 5421234586123900205, + 9738602082989433872, + 7017816005734536787, + 8635896173743411073, + ], + [ + 11707446879505873182, + 7588005580730590001, + 4664404372972250366, + 17613162115550587316, + ], + [ + 6991094187713033844, + 10140064581418506488, + 1235093741254112241, + 16755357411831959519, + ], + [ + 18007834547781860956, + 5262789089508245576, + 4752286606024269423, + 15626544383301396533, + ], + [ + 5419895278045886802, + 10747737918518643252, + 14861255521757514163, + 3291029997369465426, + ], + [ + 16916426112258580265, + 8714377345140065340, + 14207246102129706649, + 6226142825442954311, + ], + [ + 7320977330193495928, + 15630435616748408136, + 10194509925259146809, + 15938750299626487367, + ], + [ + 9872217233988117092, + 5336302253150565952, + 9650742686075483437, + 8725445618118634861, + ], + [ + 12539853708112793207, + 10831674032088582545, + 11090804155187202889, + 105068293543772992, + ], + [ + 7287113073032114129, + 6373434548664566745, + 8097061424355177769, + 14780666619112596652, + ], + [ + 17147873541222871127, + 17350918081193545524, + 5785390176806607444, + 12480094913955467088, + ], + [ + 17273934282489765074, + 8007352780590012415, + 16690624932024962846, + 8137543572359747206, + ], +]; + +/// ★ RPO256's permutation, built from RPX's OWN `fb_round`. +/// +/// This is the bridge. RPX's FB round is RPO's round exactly, so seven of them +/// composed must be RPO256 — and if they are, miden's vectors have pinned +/// RPX's constants, its MDS orientation and both its S-box chains from outside +/// this repository. Composed here rather than imported, because importing an +/// RPO implementation would only pin this file against another copy of the same +/// numbers. +fn rpo256_permute(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let mut s = state; + for r in 0..NUM_ROUNDS { + s = fb_round(s, r); + } + s +} + +/// miden's `hash_elements` in this module's lane convention: capacity lane 8 +/// takes `len % 8`, the rate is OVERWRITTEN, the tail is zero-padded, the +/// digest is lanes 0–3. +fn rpo_hash_elements(elements: &[u64]) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + state[RATE_FELTS + CAPACITY_PAD_LANE] = fe((elements.len() % RATE_FELTS) as u64); + let mut i = 0; + for e in elements { + state[i] = fe(*e); + i += 1; + if i == RATE_FELTS { + state = rpo256_permute(state); + i = 0; + } + } + if i > 0 { + while i < RATE_FELTS { + state[i] = Fp::zero(); + i += 1; + } + state = rpo256_permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// ★★ The differential the whole module rests on: seven FB rounds are RPO256, +/// and RPO256 is what miden published. +#[test] +fn seven_fb_rounds_are_rpo256() { + for (n, want) in MIDEN_HASH_ELEMENTS.iter().enumerate() { + let input: Vec = (0..=n as u64).collect(); + let got = raw_digest(&rpo_hash_elements(&input)); + assert_eq!(got, *want, "hash_elements of 0..={n} must match miden"); + } +} + +/// The compress geometry, pinned to the EXTERNAL table rather than to our own +/// permutation: merging two four-felt cells is the same thing as hashing the +/// eight felts they hold, and the eight-element vector is +/// `MIDEN_HASH_ELEMENTS[7]`. +/// +/// This is what makes the zero compress domain a checkable claim instead of a +/// convention — under RPO, any implementation anywhere computes this digest for +/// this parent. RPX's own parent differs only in the permutation, which is +/// covered by [`RPX_PARENT_VECTORS`]. +#[test] +fn the_compress_geometry_is_a_standard_merge_under_rpo() { + let mut state = [Fp::zero(); STATE_FELTS]; + for (i, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + *slot = fe(i as u64); + } + let iv = domain_iv(DOMAIN_COMPRESS); + for (k, slot) in state[RATE_FELTS..].iter_mut().enumerate() { + *slot = fe(iv[k]); + } + let out = rpo256_permute(state); + let got: [u64; DIGEST_FELTS] = + core::array::from_fn(|i| GoldilocksField::canonical(out[i].value())); + assert_eq!( + got, MIDEN_HASH_ELEMENTS[7], + "compress(0..4, 4..8) must be the standard merge" + ); +} + +/// ✓ RPX is NOT RPO — a negative control, so "seven FB rounds are RPO256" is +/// not accidentally a statement about `permute` too. +#[test] +fn rpx_is_not_rpo_on_the_same_state() { + let s = state_of(&core::array::from_fn(|i| i as u64 + 1)); + assert_ne!( + raw(&permute(s)), + raw(&rpo256_permute(s)), + "the two schedules must not agree" + ); +} + +// ========================================================================= +// LAYER 2 — the RPX oracle: the per-table host implementation's own tables +// ========================================================================= + +/// The bare permutation on eleven states. See the module header for what this +/// is and is not: the per-table branch's host `Rpx256`, and the numbers the +/// CUDA kernel is checked against — NOT an external publication. +#[allow(clippy::type_complexity)] +const RPX_PERMUTATION_VECTORS: [(&str, [u64; 12], [u64; 12]); 11] = [ + ( + "all-zero", + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ + 8760086638283468260, + 18228666152919569253, + 4041825754230271128, + 16906183286731764961, + 4664375192219530269, + 271590372761485506, + 5612474514543166805, + 8933101171974180471, + 1556877437237031065, + 7026397410864970258, + 15101742939622740655, + 4524429088483979565, + ], + ), + ( + "all-(p-1)", + [ + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + ], + [ + 7040074528728887770, + 10474261017970959672, + 6160748039461781206, + 9121740959127811013, + 7259505444118573102, + 6771278935515018093, + 18386914479072470354, + 17160039764143535473, + 1815780993504974800, + 17309055307915657636, + 5977169316478634398, + 4250629519753691035, + ], + ), + ( + "lanes 0..12", + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + [ + 3614697924784493998, + 4917065433670799835, + 12893407190838344317, + 16769932886818781879, + 17010299523770013195, + 9826755761378503206, + 1872785960340665977, + 7783788981462778586, + 45778307605882514, + 7437259891664617628, + 17010253034795346176, + 6863075881906649113, + ], + ), + ( + "alternating 0 / p-1", + [ + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + ], + [ + 12839024277220712229, + 1805658617972785851, + 11708832562581917975, + 2207339757364837492, + 457975798096500050, + 15656130651128894835, + 3485815494872446363, + 10687968103458402677, + 10384294655078062232, + 1487178939946482695, + 12310600107129561463, + 18388841767871832735, + ], + ), + ( + "one-hot lane 0", + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ + 8423002511501289529, + 6761734748202534392, + 17987336675889252592, + 14012777376234247391, + 15293807115397414812, + 15290017247514670316, + 10548590320248089637, + 9459855167724924903, + 10549768014422457033, + 13045952392708592140, + 3310663857881768756, + 7584810783597460418, + ], + ), + ( + "one-hot lane 11", + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + [ + 18436166275486246010, + 14000894557392395452, + 10767551609857089912, + 12516698445112165012, + 13131066481882004069, + 9858979976142754244, + 11402636824743634507, + 10600727647028701714, + 11200928220555719329, + 7317761145158236061, + 16857331551667002769, + 16879508045812612150, + ], + ), + ( + "random #1", + [ + 303661977215735624, + 5244312915552057691, + 9817756985327366386, + 15550273871372065883, + 5764353057648779642, + 16198122637140758912, + 7462824619408935181, + 3819703627846067891, + 10378249170554155646, + 11473525795005675318, + 8246620909628934680, + 4793144044164964625, + ], + [ + 15068850129045079395, + 15287067578585128518, + 13369562146120321575, + 10561395445440413441, + 9652992371859647144, + 4276856065313043669, + 5527444075954724606, + 7786060382866009904, + 16451772069079981395, + 198876956612152837, + 15815343923951857286, + 16122126005548441717, + ], + ), + ( + "random #2", + [ + 5204068831683694011, + 601380814908431653, + 258667317409904638, + 8486618912357792900, + 16418043790810515027, + 10319906524521615844, + 8286207029444254408, + 17770698039797916230, + 12310900488678790115, + 11195649432216834664, + 13332813278057623446, + 16898620073423657296, + ], + [ + 9523479656024648568, + 5510889535488554715, + 8599619832581755346, + 3318619196771576895, + 12581966946741818379, + 12200018864226225973, + 4385075405488142149, + 8051813774684357414, + 3019406547981393239, + 7453667634993074437, + 9864259903669275905, + 6156796699962990553, + ], + ), + ( + "random #3", + [ + 13533914130435405040, + 15234815373149021432, + 10183913914233800905, + 9526239132464493568, + 5375977297676405297, + 5765388458641153407, + 4908125521970473579, + 4421030864271922041, + 15641279279696351384, + 16893076439662162884, + 7253714011824234117, + 14616467593891397000, + ], + [ + 15514260962038810700, + 190255547175148079, + 15766300047716671382, + 10145444481310349528, + 6135237967701788176, + 11361125511081474273, + 9927005018743801106, + 17211086950078547559, + 10833199580085782023, + 13634008743082439065, + 6687522208929839355, + 3545879585555314384, + ], + ), + ( + "random #4", + [ + 389113379214421922, + 1947929307647562990, + 667333451960644926, + 3487966933876559811, + 4195385248066926332, + 2153180418459341747, + 2727969323864685845, + 29633526854483411, + 990649808851061115, + 1355410330370587755, + 11605520071788416946, + 4884409355120715354, + ], + [ + 7025469669435110295, + 17270957437800346011, + 13702589935335807876, + 3666927270871270796, + 16666721215101099684, + 531487850530305024, + 15550553335698242665, + 8959489596577675281, + 11020601500923732075, + 16110845767020565054, + 4778394010005480449, + 7715575140819562371, + ], + ), + ( + "canonicalisation witness", + [ + 15055324559807314153, + 10242425218814686878, + 9326602342065331773, + 15451135068213333861, + 17942679252967467289, + 9284164080268346300, + 5090350781253234438, + 9328738269791029498, + 18385380985273671691, + 3238854716908013220, + 5495049682105235955, + 15773368383738726538, + ], + [ + 1, + 9023883145409261355, + 5839950281880325605, + 5697668523532261268, + 13033383890974728246, + 14801658261553133914, + 3025695522291518949, + 12907720598453111556, + 14827640614007773288, + 14642633917625231592, + 3090884930034198616, + 2894057710100710233, + ], + ), +]; + +/// The rate-8 overwrite-duplex leaf at seven lengths: empty, a partial block, +/// one under a block, an exact block (which spends no trailing permutation), +/// one over, two exact blocks, and two plus one. +#[allow(clippy::type_complexity)] +const RPX_LEAF_VECTORS: [(usize, &[u64], [u64; 4]); 7] = [ + (0, &[], [0, 0, 0, 0]), + ( + 1, + &[14681136968691612469], + [ + 16400186102935428425, + 12817983163740802970, + 13449009006350391325, + 2209445548780258712, + ], + ), + ( + 7, + &[ + 2664695409302073823, + 17298518342786888931, + 17367242851809685948, + 13566833943477212382, + 6789339537410032387, + 5202847705797706501, + 6869254230765949416, + ], + [ + 2289345357069865559, + 8509266780934512918, + 13810958145049281723, + 5769431894700133303, + ], + ), + ( + 8, + &[ + 3521541860211663897, + 5585621328801039182, + 3314063895810834828, + 6286715337571703139, + 9272399501810688383, + 17378448552699642502, + 9663403628134293866, + 8225575178453385283, + ], + [ + 14052993739410942603, + 8384701950754250190, + 11473922331550289114, + 16644313465254305812, + ], + ), + ( + 9, + &[ + 15923052634311126246, + 10423360080185943333, + 4604695570423031111, + 15959212651715575539, + 4341333374822801132, + 3169961389438585383, + 7059846953207312362, + 6231597079039193598, + 14413065529971692326, + ], + [ + 15453186885173297365, + 11395279108043639065, + 15954005188014354330, + 2854892578083306874, + ], + ), + ( + 16, + &[ + 9660685076555889599, + 4027567791223379602, + 11432600011703367870, + 6441517771629429252, + 8272264386868866348, + 16565648022353132158, + 16844837242675693755, + 12942506659476152817, + 11839051358503478840, + 1846358602548732379, + 118703897581348635, + 14480592082795401517, + 12015885875590073011, + 7433808365622677077, + 13247077855319202624, + 17837888200692576115, + ], + [ + 18135965004560326100, + 1948492279228612931, + 17772968542724134453, + 12116464713281646840, + ], + ), + ( + 17, + &[ + 14169068543591784110, + 12906798066534908639, + 1898134805181953282, + 3700382130787856361, + 10455317549184205797, + 1564511190292879407, + 5954886065046464361, + 10320234224067579215, + 17095047743397986079, + 8434180870595516882, + 17706992797230203878, + 813257427175065251, + 13312284969041468023, + 15899260221184366980, + 5770785055252949875, + 11176385994046687487, + 8142444693260481147, + ], + [ + 430819886588247494, + 10400188655761849356, + 3003730485848167815, + 13484379440855863704, + ], + ), +]; + +/// The Merkle parent. +#[allow(clippy::type_complexity)] +const RPX_PARENT_VECTORS: [(&str, [u64; 4], [u64; 4], [u64; 4]); 2] = [ + ( + "digits 0..8", + [0, 1, 2, 3], + [4, 5, 6, 7], + [ + 10386438340626196987, + 10820383641790274229, + 5711121060683785078, + 11046870009967209474, + ], + ), + ( + "random", + [ + 10430052842846219471, + 4016318112082366688, + 17186674839268073878, + 16606021345024473049, + ], + [ + 1405896845186672283, + 13799610513837549656, + 17571522367612218822, + 18082329703565322844, + ], + [ + 18019606657308693634, + 10494109104368286361, + 7943124261980338770, + 17971490172695632899, + ], + ), +]; + +/// ★ The permutation reproduces the per-table host's outputs, lane for lane. +/// +/// Includes the "canonicalisation witness" row, whose output lane 0 is `1` — +/// the canonical twin of a raw `p + 1`. A port that forgot to reduce would +/// disagree there and nowhere else, which is exactly why that row exists. +#[test] +fn the_permutation_matches_the_per_table_host_vectors() { + for (name, input, want) in RPX_PERMUTATION_VECTORS { + let got = raw(&permute(state_of(&input))); + assert_eq!(got, want, "permutation vector {name}"); + } +} + +/// Every output lane is canonical (`< p`) — what the device kernel's final +/// reduction loop is pinned on, and a property the raw comparison above assumes. +#[test] +fn the_permutation_leaves_every_lane_canonical() { + for (name, input, _) in RPX_PERMUTATION_VECTORS { + for (lane, v) in raw(&permute(state_of(&input))).iter().enumerate() { + assert!(*v < P, "vector {name}, lane {lane}: {v} is not canonical"); + } + } +} + +/// ✓ Every input lane reaches the output — a diffusion control, so a +/// permutation that ignored half its state could not pass the vectors by luck. +#[test] +fn every_input_lane_changes_the_output() { + let base = state_of(&core::array::from_fn(|i| i as u64 * 7 + 1)); + let want = raw(&permute(base)); + for lane in 0..STATE_FELTS { + let mut moved = base; + moved[lane] += Fp::one(); + assert_ne!( + raw(&permute(moved)), + want, + "lane {lane} does not reach the output" + ); + } +} + +/// ★ The leaf sponge reproduces the per-table host's digests at every length +/// the padding rule distinguishes. +#[test] +fn the_leaf_sponge_matches_the_per_table_host_vectors() { + for (len, input, want) in RPX_LEAF_VECTORS { + assert_eq!(input.len(), len, "vector for length {len} is malformed"); + let got = raw_digest(&sponge_leaf(&felts(input))); + assert_eq!(got, want, "leaf of {len} felts"); + } +} + +/// ★ The parent reproduces the per-table host's digests. +#[test] +fn the_parent_matches_the_per_table_host_vectors() { + for (name, l, r, want) in RPX_PARENT_VECTORS { + let got = raw_digest(&compress(&digest_of(&l), &digest_of(&r))); + assert_eq!(got, want, "parent vector {name}"); + } +} + +/// ✓ A parent is order-sensitive — otherwise a Merkle tree would not bind a +/// sibling's side, and the two vectors above would not distinguish it. +#[test] +fn a_parent_depends_on_the_order_of_its_children() { + let (_, l, r, _) = RPX_PARENT_VECTORS[1]; + let (a, b) = (digest_of(&l), digest_of(&r)); + assert_ne!( + raw_digest(&compress(&a, &b)), + raw_digest(&compress(&b, &a)), + "compress must not be symmetric" + ); +} + +// ========================================================================= +// LAYER 3 — independent algorithms for the half no oracle covers +// ========================================================================= + +/// `ALPHA · INV_ALPHA ≡ 1 (mod p − 1)`, re-derived rather than trusted. +#[test] +fn the_inverse_exponent_inverts_alpha() { + const P_MINUS_ONE: u128 = (P as u128) - 1; + assert_eq!( + (ALPHA as u128 * INV_ALPHA as u128) % P_MINUS_ONE, + 1, + "INV_ALPHA is not alpha's inverse mod p-1" + ); +} + +/// `2^64 ≡ EPSILON (mod p)` — the identity the MDS reduction rests on. +#[test] +fn the_epsilon_identity_holds() { + const EPSILON: u128 = 0xFFFF_FFFF; + assert_eq!((1u128 << 64) % (P as u128), EPSILON % (P as u128)); +} + +/// The MDS row sum bounds the accumulator below `2^73`, so a `u128` cannot +/// overflow and the single-reduction shortcut is sound. +#[test] +fn the_mds_row_sum_cannot_overflow_a_u128() { + let sum: u128 = MDS_CIRC_ROW.iter().map(|c| *c as u128).sum(); + assert_eq!(sum, 160, "the MDS row sums to 160"); + let bound = sum * ((P as u128) - 1); + assert!(bound < (1u128 << 73), "the row sum needs {bound} < 2^73"); + assert!(bound < u128::MAX); +} + +/// [`mds`] reads a precomputed matrix; this pins it to the definition it +/// replaces, `out_i = Σ_j MDS_CIRC_ROW[(j − i) mod 12]·s_j` in field arithmetic, +/// on states that reach the top of the field. +#[test] +fn the_mds_is_the_circulant_of_its_first_row() { + for seed in 0..64u64 { + let state: [Fp; STATE_FELTS] = core::array::from_fn(|j| { + let x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) + ^ (j as u64).wrapping_mul(0xD1B5_4A32_D192_ED03); + if seed % 4 == 0 { + fe(P - 1 - j as u64) + } else { + fe(x) + } + }); + let want: [Fp; STATE_FELTS] = core::array::from_fn(|i| { + (0..STATE_FELTS).fold(Fp::zero(), |acc, j| { + acc + fe(MDS_CIRC_ROW[(j + STATE_FELTS - i) % STATE_FELTS]) * state[j] + }) + }); + assert_eq!(mds(&state), want, "seed {seed}"); + } +} + +/// The forward and inverse S-boxes invert each other — the property that +/// actually matters, checked on values neither chain was tuned for. +#[test] +fn the_inverse_sbox_inverts_the_forward_sbox() { + for v in [0u64, 1, 2, 7, 12345, P - 1, P - 2, 0x1234_5678_9abc_def0] { + let x = fe(v); + assert_eq!(inv_sbox(&sbox(&x)), x, "x = {v}"); + assert_eq!(sbox(&inv_sbox(&x)), x, "x = {v}"); + } +} + +/// ★ The inverse S-box's addition chain against generic exponentiation — a +/// different algorithm for the same number. +#[test] +fn the_inverse_sbox_chain_agrees_with_the_exponent() { + for v in [0u64, 1, 3, 99, 1 << 40, P - 5] { + let x = fe(v); + assert_eq!(inv_sbox(&x), x.pow(INV_ALPHA), "x = {v}"); + } +} + +/// ★ The cubic extension's product against naive polynomial multiplication +/// reduced by `φ³ = φ + 1`, written out term by term. +#[test] +fn the_extension_product_matches_naive_polynomial_arithmetic() { + fn naive(a: &cubic_ext::Ext, b: &cubic_ext::Ext) -> cubic_ext::Ext { + // The full degree-4 product, then reduce with φ³ = φ + 1, φ⁴ = φ² + φ. + let mut c = [Fp::zero(); 5]; + for (i, ai) in a.iter().enumerate() { + for (j, bj) in b.iter().enumerate() { + c[i + j] += ai * bj; + } + } + // φ³ → φ + 1 + let c3 = c[3]; + c[1] += c3; + c[0] += c3; + // φ⁴ → φ² + φ + let c4 = c[4]; + c[2] += c4; + c[1] += c4; + [c[0], c[1], c[2]] + } + + let mut seed = 0x243f_6a88_85a3_08d3u64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + fe(seed % P) + }; + for _ in 0..64 { + let a: cubic_ext::Ext = [next(), next(), next()]; + let b: cubic_ext::Ext = [next(), next(), next()]; + assert_eq!(cubic_ext::mul(&a, &b), naive(&a, &b)); + } +} + +/// ★ `power7` against generic square-and-multiply in the same extension. +#[test] +fn the_extension_power7_matches_square_and_multiply() { + fn pow(a: &cubic_ext::Ext, mut e: u32) -> cubic_ext::Ext { + let mut acc: cubic_ext::Ext = [Fp::one(), Fp::zero(), Fp::zero()]; + let mut base = *a; + while e > 0 { + if e & 1 == 1 { + acc = cubic_ext::mul(&acc, &base); + } + base = cubic_ext::square(&base); + e >>= 1; + } + acc + } + + let mut seed = 0x1357_9bdf_0246_8aceu64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + fe(seed % P) + }; + for _ in 0..32 { + let a: cubic_ext::Ext = [next(), next(), next()]; + assert_eq!(cubic_ext::power7(&a), pow(&a, 7)); + } +} + +/// The round-kind predicates partition `0..NUM_ROUNDS` exactly once each — the +/// schedule `FB E FB E FB E M`, said as a property rather than by reading it. +#[test] +fn the_round_schedule_is_fb_e_fb_e_fb_e_m() { + let kinds: Vec<&str> = (0..NUM_ROUNDS) + .map(|r| { + let k = [is_fb_round(r), is_ext_round(r), is_final_round(r)]; + assert_eq!( + k.iter().filter(|b| **b).count(), + 1, + "round {r} is {k:?}, which is not exactly one kind" + ); + if k[0] { + "FB" + } else if k[1] { + "E" + } else { + "M" + } + }) + .collect(); + assert_eq!(kinds, ["FB", "E", "FB", "E", "FB", "E", "M"]); +} + +/// The constants have the shape the permutation indexes them at. +#[test] +fn the_constant_tables_have_the_shape_the_rounds_index() { + assert_eq!(ARK1.len(), NUM_ROUNDS); + assert_eq!(ARK2.len(), NUM_ROUNDS); + assert!(ARK1.iter().all(|r| r.len() == STATE_FELTS)); + assert!(ARK2.iter().all(|r| r.len() == STATE_FELTS)); + assert_eq!(MDS_CIRC_ROW.len(), STATE_FELTS); + assert!( + ARK1.iter().chain(ARK2.iter()).flatten().all(|c| *c < P), + "every round constant must be canonical" + ); +} + +// ========================================================================= +// The byte / felt conventions +// ========================================================================= + +/// ⚠ The two leaf entry points must agree. Checked, not assumed: the trailing +/// partial group is zero-extended on the LOW side in both, which is easy to get +/// backwards. +#[test] +fn sponge_leaf_bytes_matches_the_felt_form() { + for len in [0usize, 1, 7, 8, 9, 15, 16, 17, 64, 65] { + let bytes: Vec = (0..len as u64).map(|i| (i * 37 + 11) as u8).collect(); + assert_eq!( + sponge_leaf_bytes(&bytes), + sponge_leaf(&felts_from_bytes(&bytes)), + "len {len}" + ); + } +} + +/// A digest survives the round trip through its 32 canonical big-endian bytes. +#[test] +fn a_digest_round_trips_through_its_commitment_bytes() { + let d: Digest = [fe(0), fe(1), fe(P - 1), fe(0x0123_4567_89ab_cdef)]; + assert_eq!(commitment_to_digest(&digest_to_commitment(&d)), d); +} + +/// ✓ The commitment bytes are BIG-endian — stated as a literal, because an +/// endianness flip round-trips perfectly and would pass the test above. +#[test] +fn the_commitment_bytes_are_big_endian() { + let d: Digest = [fe(1), fe(0), fe(0), fe(0)]; + let c = digest_to_commitment(&d); + assert_eq!( + c[7], 1, + "felt 0 = 1 must land in the LAST byte of its group" + ); + assert_eq!(c[0], 0); +} + +/// `element_felts` on a base element is the felt itself; on a cubic extension +/// element it is its three components in order. Pinned because the Merkle leaf +/// layout depends on it and `AsBytes` is a weaker contract than it looks. +#[test] +fn element_felts_decomposes_base_and_extension_the_same_way_the_stark_serialises() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + let mut out = Vec::new(); + element_felts(&fe(12345), &mut out); + assert_eq!(out, alloc::vec![fe(12345)]); + + let mut out = Vec::new(); + let e = FieldElement::::new([fe(7), fe(8), fe(9)]); + element_felts(&e, &mut out); + assert_eq!(out, alloc::vec![fe(7), fe(8), fe(9)]); +} + +/// The empty leaf is the capacity's own rate lanes — zero — which is what the +/// device kernel's `finalize` returns when nothing was absorbed. +#[test] +fn the_empty_leaf_spends_no_permutation() { + assert_eq!(raw_digest(&sponge_leaf(&[])), [0, 0, 0, 0]); + assert_eq!(raw_digest(&sponge_leaf_bytes(&[])), [0, 0, 0, 0]); +} + +/// ⚠ The domain constants are pinned by the device kernel and the KAT header; +/// changing a VALUE forks the hash. Asserted as literals so the fork is a test +/// failure rather than a silent divergence. +#[test] +fn the_domain_values_are_the_ones_the_kernel_carries() { + assert_eq!(DOMAIN_COMPRESS, 0); + assert_eq!(DOMAIN_LEAF, 0x4C4D_464C, "rpx.cu carries 0x4C4D464C"); + assert_eq!(domain_iv(DOMAIN_LEAF), [0, DOMAIN_LEAF, 0, 0]); +} + +/// The `digest::Digest` adapter computes the leaf construction over its +/// buffered bytes, and resets. +#[test] +fn the_digest_adapter_is_the_leaf_construction() { + use digest::{Digest as _, FixedOutputReset, Update}; + + let msg: Vec = (0..40u8).collect(); + let want = digest_to_commitment(&sponge_leaf_bytes(&msg)); + + let mut d = Rpx256Digest::default(); + Update::update(&mut d, &msg[..17]); + Update::update(&mut d, &msg[17..]); + let got: [u8; 32] = d.clone().finalize().into(); + assert_eq!(got, want, "streamed in two pieces must equal one call"); + + let mut out = digest::Output::::default(); + FixedOutputReset::finalize_into_reset(&mut d, &mut out); + assert_eq!(<[u8; 32]>::from(out), want); + let empty: [u8; 32] = d.finalize().into(); + assert_eq!( + empty, + digest_to_commitment(&sponge_leaf_bytes(&[])), + "reset must clear the buffer" + ); +} diff --git a/crypto/crypto/src/hash_metrics.rs b/crypto/crypto/src/hash_metrics.rs index 42b63438f..7d671b124 100644 --- a/crypto/crypto/src/hash_metrics.rs +++ b/crypto/crypto/src/hash_metrics.rs @@ -1,23 +1,31 @@ -//! Host-only keccak-hash counters for measuring the cost of VERIFYING a proof -//! (a proxy for the recursion guest's dominant work: keccak hashing). +//! Host-only hash counters for measuring the cost of VERIFYING a proof — a +//! proxy for a recursive verifier's dominant work, which is hashing. +//! +//! ⚠ **The counters follow the proof's CONFIGURATION, not one hash.** They were +//! keccak-only when keccak was the only hash on the multilinear path. Under +//! [`crate::hash::rpx`] every Merkle counter would then have read ZERO — a +//! measurement that cannot fail, reporting "no hashing" for the arm whose whole +//! purpose is to change the hashing. The algebraic backend bumps them through +//! [`count_merkle_direct`] / [`count_merkle_node_direct`] instead, which also +//! bump `total`, because unlike the byte backends nothing downstream will. //! //! Behind the `hash-metrics` cargo feature: a normal build keeps //! `PlatformKeccak256 = sha3::Keccak256` and every counter call compiles to //! nothing, so the prover is provably unchanged. With the feature on (host only), //! the host `PlatformKeccak256` wrapper counts, per keccak op: -//! * `perms` — keccak-f permutations, THE unit the guest pays (one `keccak_permute` -//! syscall each): `sum over hashes of (absorbed_bytes / 136 + 1)`. Guest-faithful -//! regardless of host call shape — a 64-byte Merkle parent is 1 perm here (2 -//! updates + finalize) and 1 on the guest (`keccak256_pair`) — so it is -//! self-validatable against a measured `keccak_permute` census; -//! * `total` — every finalize (leaf / node / transcript squeeze / program-id fold). -//! `merkle` and `grinding` are disjoint keccak-only subsets of it (the remainder -//! is the transcript / program-id fold); `merkle_nodes ⊆ merkle`, so -//! leaves = `merkle − merkle_nodes`; -//! * `absorb_calls` / `absorb_bytes` — every `Update::update`. HOST-side absorption: -//! the guest takes `keccak256_pair` for Merkle parents (0 updates), so its absorb -//! calls ≈ `absorb_calls − 2·merkle_nodes`. Useful for spotting a block-absorption -//! change (fewer, larger updates — same `perms`, fewer `absorb_calls`). +//! * `perms` — keccak-f permutations, THE unit the recursion guest pays (one +//! `keccak_permute` syscall each): `sum over hashes of (absorbed_bytes / 136 + 1)`. +//! Guest-faithful regardless of host call shape — a 64-byte Merkle parent is 1 perm +//! here (2 updates + finalize) and 1 on the guest (`keccak256_pair`) — so it is +//! self-validatable against a measured `keccak_permute` census. Keccak only: the +//! algebraic sponge's finalizes arrive through [`count_total`] and bump nothing here; +//! * `total` — every finalize (leaf / node / transcript squeeze / program-id fold); +//! `merkle`/`merkle_nodes`/`grinding` split it (keccak-only, disjoint subsets); +//! * `absorb_calls` / `absorb_bytes` — every `Update::update` (ABSORPTION). This is +//! the guest's DOMINANT keccak cost — the many 8-byte `stream_bytes` absorbs in +//! opening verification, not the finalize — so it is the dimension a block- +//! absorption optimization moves. A finalize-only number would report such a +//! change as zero improvement; `absorb_*` is what makes it visible. //! //! No enable/disable toggle and nothing in the verifier: counting is always on //! under the feature, and a measuring caller just [`reset`]s before the verify @@ -25,10 +33,11 @@ //! at a call site, so there is no cross-thread race under a parallel verify. /// Snapshot of the verify-hash counters (all zero without the `hash-metrics` -/// feature / on the guest). +/// feature / on the guest). `total` is finalizes; `absorb_*` is absorption. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Counts { /// keccak-f permutations — the unit the guest pays (one `keccak_permute` each). + /// Keccak-only, so it is zero on an all-algebraic configuration. pub perms: u64, /// Every keccak-256 finalize. pub total: u64, @@ -38,10 +47,109 @@ pub struct Counts { pub merkle_nodes: u64, /// Grinding proof-of-work finalizes (subset of `total`). pub grinding: u64, - /// Keccak absorb (`Update::update`) invocations (host-side). + /// Keccak absorb (`Update::update`) invocations. pub absorb_calls: u64, - /// Bytes fed through absorb (sum of `data.len()`). + /// Bytes fed through absorb (`Sum of data.len()`). pub absorb_bytes: u64, + /// ★★ Fiat-Shamir absorbs, ALL configurations. Counted in + /// `DefaultTranscript`'s own append methods, not in a hash. + pub transcript_absorbs: u64, + /// Of those, the ones whose sponge is keccak. + pub transcript_absorbs_keccak: u64, + /// Of those, the ones whose sponge is RPX256. + pub transcript_absorbs_rpx: u64, + /// ★★ Fiat-Shamir SQUEEZES, ALL configurations — `finalize_reset`, which + /// advances the chain (the output is re-absorbed). + pub transcript_squeezes: u64, + /// Of those, the ones whose sponge is keccak. + pub transcript_squeezes_keccak: u64, + /// Of those, the ones whose sponge is RPX256. + pub transcript_squeezes_rpx: u64, + /// ★★ Fiat-Shamir STATE reads, ALL configurations — `state()`, a finalize + /// on a CLONE. No reset and no re-absorb, so it does NOT advance the chain. + /// + /// ⚠ A DIFFERENT OPERATION from a squeeze, counted separately because + /// conflating them makes a number unfalsifiable. On a block proof the two + /// are 182,734 and 2,996 (lane V1's closed form, pinned to a measured + /// verify with difference 0): a counter hooked only to `finalize_reset` + /// misses every one of the 2,996, and a counter reporting their sum — + /// 185,730 — cannot be checked against either. + /// + /// ★ The state reads are a CONTROL that costs nothing: there is exactly one + /// per grind check, so this must equal the grind count the same run prints. + /// Two independent instruments on one quantity, and a disagreement names + /// which of them is wrong. + pub transcript_states: u64, + /// Of those, the ones whose sponge is keccak. + pub transcript_states_keccak: u64, + /// Of those, the ones whose sponge is RPX256. + pub transcript_states_rpx: u64, + /// ★★ Fiat-Shamir absorbs AFTER A STATEMENT that began at a byte offset + /// which is NOT a multiple of 8, measured within the current WINDOW — the + /// bytes absorbed since the sponge was last reset by a squeeze. + /// + /// An in-guest verifier re-slices a window into field elements every 8 + /// bytes, so a value that starts off a boundary straddles two of them and + /// costs the machine the arithmetic of putting it back together. The + /// statement padding exists to keep this at zero on a real proof, and this + /// is what says whether it did. + /// + /// NOT split by sponge, unlike the three above: alignment is a property of + /// the byte stream, and the same stream under two hashes is misaligned in + /// the same places or in neither. + /// + /// # ⛔ WHY "AFTER A STATEMENT" IS IN THE NAME + /// + /// The first version of this field counted EVERY misaligned absorb, and a + /// zero was pre-registered for a real proof. That zero was a wish. A + /// STATEMENT is variable-length by nature — a tag, one byte per table + /// count, a three-byte grind trailer — so absorbs inside it start off + /// boundaries constantly, and the padding never promised otherwise: what it + /// promises is that whatever FOLLOWS a statement starts aligned. Measured on + /// the EQ fixture under the old definition, one epoch statement plus its + /// proof read 25 misaligned absorbs at `public_output = 0` and 24 at two + /// bytes, of which 22 and 21 were felt-sized — and ZERO fell after the + /// statement. Filtering by payload size recovered nothing; only the boundary + /// did. + /// + /// So the transcript is TOLD where a statement ends + /// ([`IsTranscript::mark_statement_end`], called by the prover's statement + /// padding) and counts from there. A transcript that is never told counts + /// nothing, which is correct — it has no "after a statement". + /// + /// ⚠ A zero is only a measurement if something can make it non-zero, and + /// two tests hold that. `a_window_that_opens_off_a_boundary_is_counted` + /// marks a transcript whose window is already at 13 — the WHIR byte gate's + /// own first window, documented as not aligned — and requires exactly two. + /// `a_statement_that_ends_off_a_boundary_is_still_counted` is the sharper + /// one: the mark does NOT reset the window, so a statement that ended badly + /// is reported rather than forgiven. Without that, the mark would be a way + /// of switching off the check it was built to sharpen. + /// + /// The cross-check on a real prove is + /// `statement_alignment_tests::the_counter_and_the_recorder_agree_on_one_prove`, + /// which measures the same prove with an independent recorder and compares + /// this counter against the recorder's own after-the-statement count. + pub transcript_misaligned_absorbs_after_statement: u64, +} + +impl Counts { + /// Transcript work this build could not attribute to a known sponge. + /// + /// ★ Zero on every configuration that exists, and it is REPORTED rather + /// than assumed: the failure this whole group of counters exists to catch + /// is a hash nobody instrumented reading as a zero that looks like + /// "nothing ran". A third configuration arriving un-instrumented shows up + /// here instead of being silently folded into one of the two above. + pub fn transcript_unattributed(&self) -> (u64, u64, u64) { + ( + self.transcript_absorbs - self.transcript_absorbs_keccak - self.transcript_absorbs_rpx, + self.transcript_squeezes + - self.transcript_squeezes_keccak + - self.transcript_squeezes_rpx, + self.transcript_states - self.transcript_states_keccak - self.transcript_states_rpx, + ) + } } #[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))] @@ -59,21 +167,46 @@ mod imp { static GRINDING: AtomicU64 = AtomicU64::new(0); static ABSORB_CALLS: AtomicU64 = AtomicU64::new(0); static ABSORB_BYTES: AtomicU64 = AtomicU64::new(0); + static T_ABSORBS: AtomicU64 = AtomicU64::new(0); + static T_ABSORBS_KECCAK: AtomicU64 = AtomicU64::new(0); + static T_ABSORBS_RPX: AtomicU64 = AtomicU64::new(0); + static T_SQUEEZES: AtomicU64 = AtomicU64::new(0); + static T_SQUEEZES_KECCAK: AtomicU64 = AtomicU64::new(0); + static T_SQUEEZES_RPX: AtomicU64 = AtomicU64::new(0); + static T_STATES: AtomicU64 = AtomicU64::new(0); + static T_STATES_KECCAK: AtomicU64 = AtomicU64::new(0); + static T_STATES_RPX: AtomicU64 = AtomicU64::new(0); + static T_MISALIGNED: AtomicU64 = AtomicU64::new(0); - /// A keccak-256 finalize of a hash that absorbed `nbytes`. Bumps the finalize - /// count and the permutation count (`nbytes / RATE + 1` — the full blocks - /// absorbed plus the padded final block), the unit the guest pays. + /// Which known sponge `D` is, if any: `Some(true)` keccak, `Some(false)` + /// RPX256, `None` a configuration nobody has instrumented. + fn sponge() -> Option { + let id = core::any::TypeId::of::(); + if id == core::any::TypeId::of::() { + Some(true) + } else if id == core::any::TypeId::of::() { + Some(false) + } else { + None + } + } + + /// Every keccak-256 finalize, from any site (host `PlatformKeccak256`). #[inline(always)] - pub fn count_finalize(nbytes: usize) { + pub fn count_total() { TOTAL.fetch_add(1, Ordering::Relaxed); - PERMS.fetch_add((nbytes / RATE + 1) as u64, Ordering::Relaxed); } /// A Merkle finalize (leaf or node), counted ONLY when the backend digest is /// the platform keccak wrapper — the one whose `finalize` also bumps - /// [`count_finalize`]. This keeps `merkle` a strict subset of `total` for ANY + /// [`count_total`]. This keeps `merkle` a strict subset of `total` for ANY /// `D` (a non-keccak backend, as in the crypto tests, does not go through the /// counted wrapper, so counting it here would let `merkle` exceed `total`). + /// + /// A hash that does not route its Merkle work through a `digest::Digest` at + /// all — the algebraic backend sponges felts directly — uses + /// [`count_merkle_direct`] instead, which keeps the same invariant by + /// bumping both counters itself. #[inline(always)] pub fn count_merkle() { if core::any::TypeId::of::() @@ -85,7 +218,9 @@ mod imp { /// A Merkle parent (auth-path) compression. Subset of [`count_merkle`]; /// same keccak-only guard. Every parent must ALSO call [`count_merkle`] so - /// `merkle_nodes ⊆ merkle`. + /// `merkle_nodes ⊆ merkle` — on the byte path that is the backend's job (see + /// `merkle_tree::backends::field_element`), on the algebraic path + /// [`count_merkle_node_direct`] does both itself. #[inline(always)] pub fn count_merkle_node() { if core::any::TypeId::of::() @@ -95,21 +230,126 @@ mod imp { } } - /// A grinding (proof-of-work) finalize. Subset of [`count_finalize`]; a caller + /// A grinding (proof-of-work) finalize. Subset of [`count_total`]; a caller /// reports `total - grinding` to exclude the PoW check. #[inline(always)] pub fn count_grinding() { GRINDING.fetch_add(1, Ordering::Relaxed); } - /// A keccak absorb (`Update::update`) of `nbytes` (host-side). Bumps the call - /// count and the byte total. + /// A KECCAK finalize of a hash that absorbed `nbytes`. Bumps the finalize + /// count and the permutation count (`nbytes / RATE + 1` — the full blocks + /// absorbed plus the padded final block), the unit the guest pays. + /// + /// ⚠ Why this and [`count_total`] both exist. Only the byte sponges know how + /// many bytes went into the hash they are finalizing: the host + /// `PlatformKeccak256` wrapper accumulates them and calls this. The algebraic + /// backend sponges FELTS and has no byte count to report, so it calls + /// `count_total` and leaves `perms` alone — which is correct, since `perms` + /// counts keccak-f. Collapsing the two would either invent a byte count for + /// RPX or lose the permutation count for keccak. + #[inline(always)] + pub fn count_finalize(nbytes: usize) { + TOTAL.fetch_add(1, Ordering::Relaxed); + PERMS.fetch_add((nbytes / RATE + 1) as u64, Ordering::Relaxed); + } + + /// A keccak absorb (`Update::update`) of `nbytes` — the guest's dominant + /// keccak cost, and the dimension a block-absorption optimization moves + /// (finalizes do not change). Bumps the call count and the byte total. #[inline(always)] pub fn count_absorb(nbytes: usize) { ABSORB_CALLS.fetch_add(1, Ordering::Relaxed); ABSORB_BYTES.fetch_add(nbytes as u64, Ordering::Relaxed); } + /// ★ A Merkle LEAF finalize by a hash whose Merkle work does not pass + /// through a `digest::Digest` — the algebraic backend, which sponges felts + /// directly and never builds a digest object. + /// + /// Bumps `total` as well as `merkle`, because nothing downstream will: for + /// the byte backends `total` comes from the digest's own `finalize`, and + /// there is no such call here. Doing both in one function is what keeps + /// `merkle ⊆ total` true by construction rather than by two call sites + /// agreeing. + #[inline(always)] + pub fn count_merkle_direct() { + TOTAL.fetch_add(1, Ordering::Relaxed); + MERKLE.fetch_add(1, Ordering::Relaxed); + } + + /// ★ A Merkle PARENT by such a hash. Bumps `total`, `merkle` and + /// `merkle_nodes`, so `merkle - merkle_nodes` is the leaf count on this path + /// exactly as it is on the byte path. + #[inline(always)] + pub fn count_merkle_node_direct() { + TOTAL.fetch_add(1, Ordering::Relaxed); + MERKLE.fetch_add(1, Ordering::Relaxed); + MERKLE_NODES.fetch_add(1, Ordering::Relaxed); + } + + /// ★★ One Fiat-Shamir ABSORB, tagged by the sponge that will consume it. + /// + /// Called from `DefaultTranscript`'s append methods — the transcript, not + /// the hash. Two reasons, and the second is the one that matters: + /// + /// 1. It is hash-agnostic by construction. A counter living inside keccak + /// reads ZERO for an algebraic transcript, which is indistinguishable + /// from "no transcript ran" — the trap this module's header describes + /// for Merkle, which the transcript never got. + /// 2. It counts TRANSCRIPT absorbs only. [`count_absorb`] is bumped from a + /// digest's `update`, so it mixes Merkle leaf bytes with Fiat-Shamir + /// bytes and cannot answer "how much did the transcript absorb" for + /// either hash. + #[inline(always)] + pub fn count_transcript_absorb() { + T_ABSORBS.fetch_add(1, Ordering::Relaxed); + match sponge::() { + Some(true) => T_ABSORBS_KECCAK.fetch_add(1, Ordering::Relaxed), + Some(false) => T_ABSORBS_RPX.fetch_add(1, Ordering::Relaxed), + None => 0, + }; + } + + /// ★★ One Fiat-Shamir absorb, PAST A STATEMENT, that began off a field + /// element boundary. + /// + /// Untagged by sponge on purpose — see the field's documentation. The + /// caller decides both halves: `DefaultTranscript` is the only one, because + /// it is the only place that knows the window AND the only one that has + /// been told where the statement ended. + #[inline(always)] + pub fn count_transcript_misaligned_absorb() { + T_MISALIGNED.fetch_add(1, Ordering::Relaxed); + } + + /// ★★ One Fiat-Shamir SQUEEZE, tagged the same way. + #[inline(always)] + pub fn count_transcript_squeeze() { + T_SQUEEZES.fetch_add(1, Ordering::Relaxed); + match sponge::() { + Some(true) => T_SQUEEZES_KECCAK.fetch_add(1, Ordering::Relaxed), + Some(false) => T_SQUEEZES_RPX.fetch_add(1, Ordering::Relaxed), + None => 0, + }; + } + + /// ★★ One `state()` — a finalize on a CLONE of the sponge. + /// + /// Not a squeeze: no reset, no re-absorb, the chain does not advance. It is + /// its own counter because the two are different operations and a sum + /// cannot be checked against either. One of these per grind check, which is + /// what makes it a free control against the grind count. + #[inline(always)] + pub fn count_transcript_state() { + T_STATES.fetch_add(1, Ordering::Relaxed); + match sponge::() { + Some(true) => T_STATES_KECCAK.fetch_add(1, Ordering::Relaxed), + Some(false) => T_STATES_RPX.fetch_add(1, Ordering::Relaxed), + None => 0, + }; + } + /// Zero all counters. pub fn reset() { PERMS.store(0, Ordering::Relaxed); @@ -119,6 +359,16 @@ mod imp { GRINDING.store(0, Ordering::Relaxed); ABSORB_CALLS.store(0, Ordering::Relaxed); ABSORB_BYTES.store(0, Ordering::Relaxed); + T_ABSORBS.store(0, Ordering::Relaxed); + T_ABSORBS_KECCAK.store(0, Ordering::Relaxed); + T_ABSORBS_RPX.store(0, Ordering::Relaxed); + T_SQUEEZES.store(0, Ordering::Relaxed); + T_SQUEEZES_KECCAK.store(0, Ordering::Relaxed); + T_SQUEEZES_RPX.store(0, Ordering::Relaxed); + T_STATES.store(0, Ordering::Relaxed); + T_STATES_KECCAK.store(0, Ordering::Relaxed); + T_STATES_RPX.store(0, Ordering::Relaxed); + T_MISALIGNED.store(0, Ordering::Relaxed); } pub fn snapshot() -> Counts { @@ -130,6 +380,16 @@ mod imp { grinding: GRINDING.load(Ordering::Relaxed), absorb_calls: ABSORB_CALLS.load(Ordering::Relaxed), absorb_bytes: ABSORB_BYTES.load(Ordering::Relaxed), + transcript_absorbs: T_ABSORBS.load(Ordering::Relaxed), + transcript_absorbs_keccak: T_ABSORBS_KECCAK.load(Ordering::Relaxed), + transcript_absorbs_rpx: T_ABSORBS_RPX.load(Ordering::Relaxed), + transcript_squeezes: T_SQUEEZES.load(Ordering::Relaxed), + transcript_squeezes_keccak: T_SQUEEZES_KECCAK.load(Ordering::Relaxed), + transcript_squeezes_rpx: T_SQUEEZES_RPX.load(Ordering::Relaxed), + transcript_states: T_STATES.load(Ordering::Relaxed), + transcript_states_keccak: T_STATES_KECCAK.load(Ordering::Relaxed), + transcript_states_rpx: T_STATES_RPX.load(Ordering::Relaxed), + transcript_misaligned_absorbs_after_statement: T_MISALIGNED.load(Ordering::Relaxed), } } } @@ -142,13 +402,27 @@ mod imp { #[inline(always)] pub fn count_finalize(_nbytes: usize) {} #[inline(always)] + pub fn count_total() {} + #[inline(always)] pub fn count_merkle() {} #[inline(always)] pub fn count_merkle_node() {} #[inline(always)] pub fn count_grinding() {} #[inline(always)] + pub fn count_merkle_direct() {} + #[inline(always)] + pub fn count_merkle_node_direct() {} + #[inline(always)] pub fn count_absorb(_nbytes: usize) {} + #[inline(always)] + pub fn count_transcript_absorb() {} + #[inline(always)] + pub fn count_transcript_squeeze() {} + #[inline(always)] + pub fn count_transcript_misaligned_absorb() {} + #[inline(always)] + pub fn count_transcript_state() {} pub fn reset() {} pub fn snapshot() -> Counts { Counts::default() @@ -156,5 +430,8 @@ mod imp { } pub use imp::{ - count_absorb, count_finalize, count_grinding, count_merkle, count_merkle_node, reset, snapshot, + count_absorb, count_finalize, count_grinding, count_merkle, count_merkle_direct, + count_merkle_node, count_merkle_node_direct, count_total, count_transcript_absorb, + count_transcript_misaligned_absorb, count_transcript_squeeze, count_transcript_state, reset, + snapshot, }; diff --git a/crypto/crypto/src/lib.rs b/crypto/crypto/src/lib.rs index a70ac977e..c8227a9eb 100644 --- a/crypto/crypto/src/lib.rs +++ b/crypto/crypto/src/lib.rs @@ -1,18 +1,14 @@ #![allow(clippy::op_ref)] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(all(target_arch = "wasm32", feature = "disk-spill"))] -compile_error!("the `disk-spill` feature requires memmap2, which does not compile on wasm32"); - #[macro_use] extern crate alloc; pub mod fiat_shamir; +pub mod grinding; pub mod hash; pub mod hash_metrics; pub mod merkle_tree; -#[cfg(feature = "disk-spill")] -pub mod mmap_util; #[cfg(test)] pub mod tests; diff --git a/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs b/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs index ccd5b3f4d..d7659d0cc 100644 --- a/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs +++ b/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs @@ -1,7 +1,7 @@ use core::marker::PhantomData; use crate::hash::poseidon::Poseidon; -use crate::merkle_tree::traits::IsMerkleTreeBackend; +use crate::merkle_tree::traits::{IsLeafHasher, IsMerkleTreeBackend, IsStreamingLeafBackend}; use alloc::vec::Vec; use digest::{Digest, Output}; use math::{ @@ -9,6 +9,8 @@ use math::{ traits::AsBytes, }; +#[cfg(target_arch = "riscv64")] +use crate::hash::blake3::chain::{Blake3Chain, blake3_parent}; #[cfg(target_arch = "riscv64")] use crate::hash::platform_keccak::PlatformKeccak256; #[cfg(target_arch = "riscv64")] @@ -92,6 +94,28 @@ fn hash_new_parent_bytes( return result; } + // The BLAKE3 twin of the branch above, and it is the same trade: a 64-byte + // parent is ONE compression (P2), so the sponge around it is pure plumbing. + // In-guest that plumbing measured ~55% of a 64-byte hash — ~556 cycles for + // ~248 of compression — and a Merkle parent is the shape every FRI query + // path step is made of. + // + // Keyed on the CONCRETE digest, exactly like keccak's: `Blake3Chain` is the + // type `BatchBlake3Backend`/`PairBlake3Backend` instantiate, so no other + // digest can fall into this branch. `blake3_parent` goes through + // `chain.rs`'s single `compress_block` dispatch, so this is the accelerator + // ecall on a guest that has it and the software path otherwise — one + // framing, not a second transcription. + #[cfg(target_arch = "riscv64")] + if NUM_BYTES == 32 && TypeId::of::() == TypeId::of::() { + let l: &[u8; 32] = left[..].try_into().unwrap(); + let r: &[u8; 32] = right[..].try_into().unwrap(); + let hash = blake3_parent(l, r); + let mut result = [0u8; NUM_BYTES]; + result.copy_from_slice(&hash); + return result; + } + hash_streamed::(|sink| { sink(left); sink(right); @@ -209,6 +233,153 @@ where } } +/// Exposes the streaming leaf routes to callers that reach this backend through +/// a commitment configuration rather than by name. Both bodies go through +/// [`hash_streamed`], which is where the absorbed byte layout is defined, so +/// they agree with `hash_data` by construction. +impl IsStreamingLeafBackend + for FieldElementVectorBackend +where + F: IsField, + FieldElement: AsBytes, + [u8; NUM_BYTES]: From>, + Vec>: Sync + Send, +{ + fn hash_bytes(data: &[u8]) -> [u8; NUM_BYTES] { + hash_streamed::(|sink| sink(data)) + } + + fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> [u8; NUM_BYTES] { + // A size threshold below which this streams straight through was tried + // and MEASURED NEUTRAL-TO-WORSE (963.56M vs 963.28M cycles on a blowup8 + // verify, with `verify_fri` unmoved to the cycle). The 6.9M `verify_fri` + // rise that this change costs is NOT the staging buffer — gating the + // buffer away does not recover it — so it is not worth a branch here. + // Do not re-add one without a measurement. + hash_streamed::(|sink| { + let mut stage = LeafStage::new(); + for element in a.iter().chain(b.iter()) { + element.stream_bytes(&mut |bytes| stage.push(bytes, sink)); + } + stage.flush(sink); + }) + } + + type LeafHasher = DigestLeafHasher; + + fn leaf_hasher() -> Self::LeafHasher { + DigestLeafHasher { + hasher: D::new(), + phantom: PhantomData, + } + } +} + +/// [`IsLeafHasher`] over the same digest the one-shot routes use. +/// +/// The split-invariance the trait demands is inherited rather than argued: +/// `hash_streamed` opens a fresh `D`, feeds it every element's `stream_bytes` +/// and finalizes, with no length prefix, padding or framing of its own — so +/// absorbing the same elements across several `update` calls presents `D` with +/// the identical byte stream. There is no place for a split to show. +/// +/// This is a PROVER-side construct: the guest verifier authenticates leaves it +/// receives whole, through `hash_data_from_slices`. That is why this does not +/// carry `hash_streamed`'s riscv64 syscall-sponge fast path — it would be dead +/// code on the only target that has it. +pub struct DigestLeafHasher { + hasher: D, + /// `fn() -> F` rather than `F`: the field is a type-level label here, never a + /// value, and the function-pointer form is unconditionally `Send`/`Sync`. The + /// bare `PhantomData` would make every leaf hasher's thread-safety hinge on + /// a marker type nobody ever moves. + phantom: PhantomData F>, +} + +impl IsLeafHasher for DigestLeafHasher +where + F: IsField, + FieldElement: AsBytes, + [u8; NUM_BYTES]: From>, +{ + type Node = [u8; NUM_BYTES]; + + fn update(&mut self, data: &[FieldElement]) { + for element in data { + element.stream_bytes(&mut |bytes| self.hasher.update(bytes)); + } + } + + fn finalize(self) -> [u8; NUM_BYTES] { + let mut result = [0u8; NUM_BYTES]; + result.copy_from_slice(&self.hasher.finalize()); + result + } +} + +/// Bytes of the leaf staging buffer. A multiple of the BLAKE3 block so a full +/// flush is a whole number of blocks, and large enough that the run reaching the +/// hasher is worth batching — 16 blocks. +const LEAF_STAGE_BYTES: usize = 1024; + +/// Coalesces a leaf's field elements into large aligned runs before they reach +/// the hasher. +/// +/// ★ Why this exists. A leaf arrives one field element at a time — eight bytes +/// per `stream_bytes` call — so the hasher only ever saw eight bytes at a time, +/// and `Blake3Chain`'s bulk-absorb path (which needs more than one block in hand +/// to batch anything) could never engage. The leaf therefore paid one ecall and +/// one 112-byte marshal per 64 bytes, which is the cost the absorb ecall exists +/// to remove and, measured, the dominant term in a real in-guest verify. +/// +/// The staging buffer is `align(8)` because the accelerator reads the message as +/// doublewords and declines an unaligned slice — an unaligned buffer here would +/// silently give back the per-block path with no diagnostic. +/// +/// **This cannot change any digest.** The same bytes reach the hasher in the +/// same order; only the call boundaries move, and `Blake3Chain::update` is +/// split-invariant by construction (`streaming_splits_agree_with_one_shot` +/// checks every split of every length through two blocks). It is also hash- +/// agnostic: keccak's sponge is equally split-invariant and simply sees fewer, +/// larger `update` calls. +#[repr(align(8))] +struct LeafStage { + buf: [u8; LEAF_STAGE_BYTES], + len: usize, +} + +impl LeafStage { + #[inline] + fn new() -> Self { + Self { + buf: [0u8; LEAF_STAGE_BYTES], + len: 0, + } + } + + #[inline] + fn push(&mut self, mut bytes: &[u8], sink: &mut dyn FnMut(&[u8])) { + while !bytes.is_empty() { + if self.len == LEAF_STAGE_BYTES { + sink(&self.buf[..LEAF_STAGE_BYTES]); + self.len = 0; + } + let take = (LEAF_STAGE_BYTES - self.len).min(bytes.len()); + self.buf[self.len..self.len + take].copy_from_slice(&bytes[..take]); + self.len += take; + bytes = &bytes[take..]; + } + } + + #[inline] + fn flush(&mut self, sink: &mut dyn FnMut(&[u8])) { + if self.len > 0 { + sink(&self.buf[..self.len]); + self.len = 0; + } + } +} + #[derive(Clone, Default)] pub struct BatchPoseidonTree { _poseidon: PhantomData

, diff --git a/crypto/crypto/src/merkle_tree/backends/mod.rs b/crypto/crypto/src/merkle_tree/backends/mod.rs index 431e6597b..27cab1276 100644 --- a/crypto/crypto/src/merkle_tree/backends/mod.rs +++ b/crypto/crypto/src/merkle_tree/backends/mod.rs @@ -1,5 +1,7 @@ pub mod field_element; pub mod field_element_vector; +pub mod p1; +pub mod rpx; /// Configurations for merkle trees /// Setting generics to some value pub mod types; diff --git a/crypto/crypto/src/merkle_tree/backends/p1.rs b/crypto/crypto/src/merkle_tree/backends/p1.rs new file mode 100644 index 000000000..2b5d588bb --- /dev/null +++ b/crypto/crypto/src/merkle_tree/backends/p1.rs @@ -0,0 +1,281 @@ +//! ZisK's Poseidon1 Merkle backends (Goldilocks, width 16, rate 12; arity-4 +//! trees) — the algebraic sibling of [`RpxVectorBackend`](super::rpx::RpxVectorBackend). +//! +//! The primitives are [`crate::hash::poseidon1_stark`] (checked against ZisK's +//! own code). What this module fixes is the encoding at the backend's seams, +//! the conventions [`super::rpx`] fixes for RPX: +//! +//! | seam | encoding | +//! |---|---| +//! | leaf | the elements' base felts in order ([`element_felts`]), then [`linear_hash`]: ZisK's chain with the width tag `[len, LEAF_DOMAIN, 0, 0]` as the first capacity | +//! | node | [`poseidon1_w16::compress4`] over four children; a node is four felts as 32 canonical big-endian bytes ([`digest_to_commitment`]) | +//! | padding child | the zero digest, i.e. 32 zero bytes | +//! +//! Nothing proves with these backends unless a caller names them; RPX stays +//! the default everywhere. + +use core::marker::PhantomData; + +use alloc::vec::Vec; +use math::{ + field::{element::FieldElement, traits::IsField}, + traits::AsBytes, +}; + +use crate::hash::poseidon1_stark::linear_hash; +use crate::hash::poseidon1_w16; +use crate::hash::rpx::{ + Fp, commitment_to_digest, digest_to_commitment, element_felts, felts_from_bytes, +}; +use crate::merkle_tree::traits::{IsLeafHasher, IsMerkleTreeBackend, IsStreamingLeafBackend}; + +/// ZisK's tagged leaf hash over felts, as a node. +pub fn leaf(felts: &[Fp]) -> [u8; 32] { + digest_to_commitment(&linear_hash(felts)) +} + +/// ZisK's 4-ary node over four children. +pub fn node4(children: &[[u8; 32]; 4]) -> [u8; 32] { + digest_to_commitment(&poseidon1_w16::compress4( + &children.map(|c| commitment_to_digest(&c)), + )) +} + +/// The batched leaf backend: one leaf per vector of field elements. +#[derive(Clone, Debug)] +pub struct P1BatchBackend { + /// `fn() -> F`, so the marker is `Send` and `Sync` whatever `F` is. + _marker: PhantomData F>, +} + +impl Default for P1BatchBackend { + fn default() -> Self { + Self { + _marker: PhantomData, + } + } +} + +/// The pair backend: one leaf per fixed pair, the batched leaf of the +/// two-element vector. +#[derive(Clone, Debug)] +pub struct P1PairBackend { + _marker: PhantomData F>, +} + +impl Default for P1PairBackend { + fn default() -> Self { + Self { + _marker: PhantomData, + } + } +} + +impl IsMerkleTreeBackend for P1BatchBackend +where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + Vec>: Sync + Send, +{ + type Node = [u8; 32]; + type Data = Vec>; + const ARITY: usize = 4; + + fn hash_data(input: &Vec>) -> [u8; 32] { + >::hash_data_from_slices(input, &[]) + } + + /// A 4-ary tree has no binary parent; the two-child form is the 4-ary + /// node over the pair and two padding digests (the shape of an odd-depth + /// tree's top), so the method is defined everywhere. + fn hash_new_parent(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + node4(&[*left, *right, [0u8; 32], [0u8; 32]]) + } + + fn hash_four(children: &[[u8; 32]; 4]) -> [u8; 32] { + node4(children) + } + + fn padding_node() -> Option<[u8; 32]> { + Some([0u8; 32]) + } +} + +impl IsStreamingLeafBackend for P1BatchBackend +where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + Vec>: Sync + Send, +{ + /// Equals [`IsMerkleTreeBackend::hash_data`] on the elements `data` + /// encodes (`tests::hash_bytes_agrees_with_hash_data`). + fn hash_bytes(data: &[u8]) -> [u8; 32] { + leaf(&felts_from_bytes(data)) + } + + fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> [u8; 32] { + let mut felts = Vec::with_capacity(3 * (a.len() + b.len())); + for e in a.iter().chain(b.iter()) { + element_felts(e, &mut felts); + } + leaf(&felts) + } + + type LeafHasher = P1LeafHasher; + + fn leaf_hasher() -> Self::LeafHasher { + P1LeafHasher { + felts: Vec::new(), + _marker: PhantomData, + } + } +} + +impl IsMerkleTreeBackend for P1PairBackend +where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send, +{ + type Node = [u8; 32]; + type Data = [FieldElement; 2]; + const ARITY: usize = 4; + + fn hash_data(input: &[FieldElement; 2]) -> [u8; 32] { + let mut felts = Vec::with_capacity(6); + element_felts(&input[0], &mut felts); + element_felts(&input[1], &mut felts); + leaf(&felts) + } + + /// As [`P1BatchBackend`]'s: the 4-ary node over the pair and two padding + /// digests. + fn hash_new_parent(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + node4(&[*left, *right, [0u8; 32], [0u8; 32]]) + } + + fn hash_four(children: &[[u8; 32]; 4]) -> [u8; 32] { + node4(children) + } + + fn padding_node() -> Option<[u8; 32]> { + Some([0u8; 32]) + } +} + +/// The incremental leaf hasher. The width tag needs the felt count, and the +/// row arrives in slices of field elements, so it buffers their felts. +pub struct P1LeafHasher { + felts: Vec, + _marker: PhantomData F>, +} + +impl IsLeafHasher for P1LeafHasher +where + F: IsField, + FieldElement: AsBytes, +{ + type Node = [u8; 32]; + + fn update(&mut self, data: &[FieldElement]) { + for e in data { + element_felts(e, &mut self.felts); + } + } + + fn finalize(self) -> [u8; 32] { + leaf(&self.felts) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hash::poseidon1_stark::Merkle4; + use crate::merkle_tree::merkle::MerkleTree; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Base; + use math::field::traits::IsPrimeField; + + type B = P1BatchBackend; + type E = P1BatchBackend; + + fn fe(i: u64) -> FieldElement { + FieldElement::from(i.wrapping_mul(0x9E37_79B9_7F4A_7C15)) + } + + fn fee(i: u64) -> FieldElement { + FieldElement::::new([fe(3 * i), fe(3 * i + 1), fe(3 * i + 2)]) + } + + #[test] + fn hash_bytes_agrees_with_hash_data() { + for w in [1usize, 5, 12, 13, 40, 48, 64] { + let row: Vec> = (0..w as u64).map(fe).collect(); + let bytes: Vec = row + .iter() + .flat_map(|f| Base::canonical(f.value()).to_be_bytes()) + .collect(); + assert_eq!(B::hash_bytes(&bytes), B::hash_data(&row), "width {w}"); + assert_eq!(B::hash_data(&row), leaf(&row)); + } + } + + #[test] + fn an_ext3_leaf_is_its_coefficients_in_order() { + for w in [1usize, 2, 16] { + let row: Vec> = (0..w as u64).map(fee).collect(); + let felts: Vec = row.iter().flat_map(|e| e.value().to_vec()).collect(); + assert_eq!(E::hash_data(&row), leaf(&felts), "width {w}"); + } + } + + #[test] + fn the_leaf_hasher_is_the_batched_leaf() { + let row: Vec> = (0..16u64).map(fee).collect(); + let mut h = E::leaf_hasher(); + h.update(&row[..5]); + h.update(&row[5..]); + assert_eq!(h.finalize(), E::hash_data(&row)); + assert_eq!( + E::hash_data_from_slices(&row[..7], &row[7..]), + E::hash_data(&row) + ); + } + + #[test] + fn the_pair_and_batched_families_agree_on_a_two_element_leaf() { + // The univariate prover commits FRI layers with `Pair` and the + // verifier authenticates them with `Batched`. + for i in 0..8 { + let (a, b) = (fee(2 * i), fee(2 * i + 1)); + assert_eq!( + P1PairBackend::::hash_data(&[a, b]), + E::hash_data(&vec![a, b]) + ); + } + } + + #[test] + fn a_tree_of_rows_is_the_oracles_tree() { + for n in [1usize, 2, 8, 32, 128] { + let rows: Vec>> = (0..n as u64) + .map(|r| (0..7).map(|c| fe(r * 7 + c)).collect()) + .collect(); + let tree = MerkleTree::::build(&rows).expect("non-empty"); + let leaves: Vec<_> = rows.iter().map(|r| linear_hash(r)).collect(); + let oracle = Merkle4::new(&leaves).expect("non-empty"); + assert_eq!(tree.root, digest_to_commitment(&oracle.root()), "{n} rows"); + let path = tree.get_proof_by_pos(n - 1).expect("in range"); + assert!(path.verify::(&tree.root, n - 1, &rows[n - 1])); + } + } + + #[test] + fn the_binary_parent_is_the_node_over_a_padded_pair() { + let (a, b) = ([1u8; 32], [2u8; 32]); + assert_eq!( + B::hash_new_parent(&a, &b), + B::hash_four(&[a, b, [0u8; 32], [0u8; 32]]) + ); + } +} diff --git a/crypto/crypto/src/merkle_tree/backends/rpx.rs b/crypto/crypto/src/merkle_tree/backends/rpx.rs new file mode 100644 index 000000000..c07297cb9 --- /dev/null +++ b/crypto/crypto/src/merkle_tree/backends/rpx.rs @@ -0,0 +1,260 @@ +//! The RPX256 Merkle backend — the algebraic sibling of +//! [`FieldElementVectorBackend`](super::field_element_vector::FieldElementVectorBackend). +//! +//! A sibling type rather than a reparameterisation of the byte backend, for the +//! reason the per-table branch gives: `FieldElementVectorBackend` is built +//! around a `digest::Digest` fed a byte stream, while this hashes FELTS with a +//! rate-8 overwrite duplex. Routing felts through bytes and back would work and +//! would be slower and less obvious; keeping them apart is what leaves the +//! keccak path untouched by this work. +//! +//! # What a leaf and a parent are +//! +//! **A leaf** is [`sponge_leaf`](crate::hash::rpx::sponge_leaf) over the felt +//! sequence the leaf's elements decompose to, in order — eight fresh felts per +//! permutation, capacity lane 0 the padding flag `len mod 8`, lane 1 the LEAF +//! domain. +//! +//! **A parent** is [`compress`](crate::hash::rpx::compress) — ONE permutation of +//! `[left ‖ right ‖ 0⁴]` at the zero domain, which makes it literally +//! `Rpx256::merge` and externally checkable against miden. +//! +//! **A node** is four canonical felts as 32 big-endian bytes, so +//! `IsMerkleTreeBackend::Node` is the same `[u8; 32]` every other backend in +//! this crate uses and no proof type changes width. +//! +//! # ⚠ The one contract a caller has to know +//! +//! `hash_data` on `&[a, b]` must equal what a FRI-style pair backend would +//! compute for the pair `[a, b]`, because the univariate prover commits layers +//! one way and verifies them the other. That invariant is why there is a single +//! implementation here rather than a "batched" and a "pair" one that could be +//! edited apart: both shapes go through [`hash_data_from_slices`], so there are +//! not two encodings to be shown equal. `tests::a_pair_leaf_is_the_two_element_vector` +//! pins it anyway, because "holds by construction" is a claim about today's +//! code. + +use core::marker::PhantomData; + +use alloc::vec::Vec; +use math::{ + field::{element::FieldElement, traits::IsField}, + traits::AsBytes, +}; + +use crate::hash::rpx::{ + Fp, commitment_to_digest, compress, digest_to_commitment, element_felts, felts_from_bytes, + sponge_leaf, +}; +use crate::merkle_tree::traits::IsMerkleTreeBackend; + +/// RPX256 over vectors of field elements. +#[derive(Clone, Debug)] +pub struct RpxVectorBackend { + /// `fn() -> F` rather than `F`, so the marker is unconditionally `Send` and + /// `Sync` without an `unsafe impl`: a real epoch's base layer has millions + /// of leaves, hashed in parallel. + _marker: PhantomData F>, +} + +impl Default for RpxVectorBackend { + fn default() -> Self { + Self { + _marker: PhantomData, + } + } +} + +impl RpxVectorBackend +where + F: IsField, + FieldElement: AsBytes, +{ + /// Leaf-hash the concatenation `a ‖ b` without materialising it. + /// + /// The single source of truth for the leaf's felt sequence: a plain leaf is + /// the concatenation with an empty second slice, so the two shapes cannot + /// disagree. + pub fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> [u8; 32] { + // Metric: a Merkle leaf finalize. `_direct` because this path builds no + // `digest::Digest`, so it owes `total` as well — see `crate::hash_metrics`. + crate::hash_metrics::count_merkle_direct(); + let mut felts: Vec = Vec::with_capacity(a.len() + b.len()); + for e in a.iter().chain(b.iter()) { + element_felts(e, &mut felts); + } + digest_to_commitment(&sponge_leaf(&felts)) + } + + /// Leaf-hash a byte buffer, rebuilding the felts it encodes. + /// + /// ⚠ Must equal [`hash_data`](IsMerkleTreeBackend::hash_data) on the + /// elements those bytes encode — the one place an algebraic backend can + /// silently disagree with itself, because the byte route has to rebuild + /// what the felt route was handed. + /// `tests::hash_bytes_agrees_with_hash_data` is the gate. + pub fn hash_bytes(data: &[u8]) -> [u8; 32] { + // Metric: a Merkle leaf finalize, as `hash_data_from_slices` is — the + // two must agree on the digest, so they must agree on the count. + crate::hash_metrics::count_merkle_direct(); + digest_to_commitment(&sponge_leaf(&felts_from_bytes(data))) + } +} + +impl IsMerkleTreeBackend for RpxVectorBackend +where + F: IsField, + FieldElement: AsBytes + Sync + Send, + Vec>: Sync + Send, +{ + type Node = [u8; 32]; + type Data = Vec>; + + fn hash_data(input: &Vec>) -> [u8; 32] { + Self::hash_data_from_slices(input, &[]) + } + + fn hash_new_parent(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + // Metric: a Merkle parent. One call, not a leaf count plus a node count: + // this path does not flow through the leaf helper the way the byte + // backend's parent flows through `hash_streamed`. + crate::hash_metrics::count_merkle_node_direct(); + digest_to_commitment(&compress( + &commitment_to_digest(left), + &commitment_to_digest(right), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Base; + + type B = RpxVectorBackend; + type E = RpxVectorBackend; + + fn base(n: usize) -> Vec> { + (0..n as u64) + .map(|i| FieldElement::from(i * 7 + 1)) + .collect() + } + + fn ext(n: usize) -> Vec> { + (0..n as u64) + .map(|i| { + FieldElement::::new([ + FieldElement::from(i + 1), + FieldElement::from(i + 2), + FieldElement::from(i + 3), + ]) + }) + .collect() + } + + /// ★★ The `hash_bytes` / `hash_data` contract, on both fields and at the + /// lengths the padding rule distinguishes. + #[test] + fn hash_bytes_agrees_with_hash_data() { + for n in [0usize, 1, 7, 8, 9, 16, 17] { + let leaf = base(n); + let mut bytes = Vec::new(); + for e in &leaf { + e.stream_bytes(&mut |b| bytes.extend_from_slice(b)); + } + assert_eq!(B::hash_bytes(&bytes), B::hash_data(&leaf), "base, n = {n}"); + + let leaf = ext(n); + let mut bytes = Vec::new(); + for e in &leaf { + e.stream_bytes(&mut |b| bytes.extend_from_slice(b)); + } + assert_eq!(E::hash_bytes(&bytes), E::hash_data(&leaf), "ext3, n = {n}"); + } + } + + /// ⚠ The invariant the univariate FRI path depends on: a two-element leaf + /// is a pair. + #[test] + fn a_pair_leaf_is_the_two_element_vector() { + let a = FieldElement::::from(11u64); + let b = FieldElement::::from(22u64); + assert_eq!( + B::hash_data(&alloc::vec![a, b]), + B::hash_data_from_slices(&[a], &[b]) + ); + } + + /// ✓ A leaf is order-sensitive and length-sensitive — so the equalities + /// above are not equalities between constants. + #[test] + fn a_leaf_depends_on_the_order_and_the_length_of_its_elements() { + let leaf = base(5); + let mut swapped = leaf.clone(); + swapped.swap(0, 1); + assert_ne!(B::hash_data(&leaf), B::hash_data(&swapped)); + + let mut longer = leaf.clone(); + longer.push(FieldElement::from(0u64)); + assert_ne!( + B::hash_data(&leaf), + B::hash_data(&longer), + "a trailing zero must not be invisible" + ); + } + + /// ✓ An exact rate multiple spends no trailing permutation, so the eighth + /// and ninth felts are not interchangeable at the block boundary. + #[test] + fn the_block_boundary_is_not_a_collision() { + assert_ne!(B::hash_data(&base(8)), B::hash_data(&base(9))); + assert_ne!(B::hash_data(&base(16)), B::hash_data(&base(17))); + } + + /// ✓ A parent is order-sensitive. + #[test] + fn a_parent_depends_on_the_order_of_its_children() { + let l = B::hash_data(&base(3)); + let r = B::hash_data(&base(4)); + assert_ne!(B::hash_new_parent(&l, &r), B::hash_new_parent(&r, &l)); + } + + /// ✓ A parent is NOT a leaf of the eight felts its children hold: the + /// domains differ, which is the whole point of the capacity tag. + #[test] + fn a_parent_is_domain_separated_from_a_leaf() { + use crate::hash::rpx::{commitment_to_digest, sponge_leaf}; + + let l = B::hash_data(&base(3)); + let r = B::hash_data(&base(4)); + let parent = B::hash_new_parent(&l, &r); + + let mut felts = Vec::new(); + felts.extend_from_slice(&commitment_to_digest(&l)); + felts.extend_from_slice(&commitment_to_digest(&r)); + let as_leaf = digest_to_commitment(&sponge_leaf(&felts)); + + assert_ne!( + parent, as_leaf, + "the LEAF domain must separate a leaf from a parent over the same felts" + ); + } + + /// ✓ An extension leaf decomposes to three felts per element — checked + /// against the felt sequence rather than against another backend call, so a + /// decomposition that dropped a component would show. + #[test] + fn an_extension_leaf_absorbs_three_felts_per_element() { + let leaf = ext(4); + let mut felts = Vec::new(); + for e in &leaf { + element_felts(e, &mut felts); + } + assert_eq!(felts.len(), 12, "four ext3 elements are twelve felts"); + assert_eq!( + E::hash_data(&leaf), + digest_to_commitment(&sponge_leaf(&felts)) + ); + } +} diff --git a/crypto/crypto/src/merkle_tree/backends/types.rs b/crypto/crypto/src/merkle_tree/backends/types.rs index 2384fda3a..3886690a9 100644 --- a/crypto/crypto/src/merkle_tree/backends/types.rs +++ b/crypto/crypto/src/merkle_tree/backends/types.rs @@ -1,15 +1,38 @@ +use crate::hash::blake3::Blake3Chain; use crate::hash::platform_keccak::PlatformKeccak256 as Keccak256; -use super::{ - field_element::FieldElementBackend, - field_element_vector::{FieldElementPairBackend, FieldElementVectorBackend}, -}; - -// Field element backend definitions -pub type Keccak256Backend = FieldElementBackend; +use super::field_element_vector::{FieldElementPairBackend, FieldElementVectorBackend}; // Vector of field elements backend definitions pub type BatchKeccak256Backend = FieldElementVectorBackend; // Fixed-size pair backends (more efficient for FRI layers) pub type PairKeccak256Backend = FieldElementPairBackend; + +/// The BLAKE3 batched-leaf backend, over [`Blake3Chain`] — the single-chunk +/// chain specified in PA-PLAN §1.7. +/// +/// It is the *same* generic backend the keccak alias is, with the digest +/// swapped, and that is load-bearing rather than an economy. The leaf byte +/// layout, both streaming routes and the parent framing then have one definition +/// each (`field_element_vector.rs`), so the batched and paired families cannot +/// encode a two-element leaf differently: the invariant `stark::config::StarkHash` +/// requires holds because they are one function, not because two implementations +/// were shown to coincide. +/// +/// A parent is `Blake3Chain` over the two concatenated 32-byte nodes — 64 bytes, +/// so one compression with `h = IV`, `t = 0`, `block_len = 64`, `flags = +/// CHUNK_START | CHUNK_END | ROOT`. That is bit-for-bit what the device kernel +/// computes (`math-cuda/kernels/blake3.cu`, `blake3_hash_merkle_parent`), which +/// is what will let a GPU tree and a CPU tree be the same tree once the device +/// leaf kernels land. +pub type BatchBlake3Backend = FieldElementVectorBackend; + +/// The FRI-layer twin of [`BatchBlake3Backend`] — one leaf per fixed pair, no +/// `Vec` per leaf. See there. +pub type PairBlake3Backend = FieldElementPairBackend; +/// RPX256 over a vector of field elements — the algebraic backend, for a proof +/// that is going to be verified inside another proof. See +/// [`crate::hash::rpx`] for why an algebraic hash is worth its host cost, and +/// only there. +pub type BatchRpx256Backend = super::rpx::RpxVectorBackend; diff --git a/crypto/crypto/src/merkle_tree/cap.rs b/crypto/crypto/src/merkle_tree/cap.rs new file mode 100644 index 000000000..128e21df4 --- /dev/null +++ b/crypto/crypto/src/merkle_tree/cap.rs @@ -0,0 +1,1453 @@ +//! Merkle caps: authentication paths that stop `c` levels below the root. +//! +//! A tree of depth `D` (so `2^D` padded leaves) has, at height `c`, the `2^c` +//! nodes that sit `c` levels below its root — its **cap**. A proof can carry a +//! tree's cap once and cut every authentication path of that tree to its first +//! `D − c` siblings: the verifier folds a path up to the cap node +//! `cap[index >> (D − c)]` instead of the root, and checks once per tree that +//! the cap hashes up to the committed root (`2^c − 1` compressions). +//! +//! **The root stays the commitment.** Nothing about the transcript changes: +//! the root is what is absorbed, and a second cap with the same root is a +//! compression collision. Any capped acceptance extends to a full-path +//! acceptance (append the cap-to-root computation above the cap node), so the +//! query-phase bound is the one the full paths had. +//! +//! **Four checks are load-bearing** — dropping any one is a soundness break: +//! 1. `cap.len() == 2^c` and `cap_root(cap) == root`, once per tree +//! ([`verify_cap`], run by [`CappedRoot::from_owner`]); +//! 2. every path is exactly `D − c` siblings long, and the owner path exactly +//! `D − c + 2^c` ([`verify_merkle_path_to_cap_from_leaf_hash`], +//! [`split_owner_path`]); +//! 3. the cap node is `cap[index >> (D − c)]` with `index < 2^D`, the index +//! being the transcript's; +//! 4. `c` itself is a verifier constant ([`CapPolicy::height`] of public shape +//! data), never read from the proof. +//! +//! At `c = 0` the cap is `[root]` and the capped check is exactly +//! [`verify_merkle_path_from_leaf_hash`] plus the two exact-length checks. +//! +//! **Arity 4** (a backend with [`IsMerkleTreeBackend::ARITY`] = 4). `depth` +//! stays the binary depth `D = log2(leaves)` the shapes speak; the tree has +//! `h = ⌈D/2⌉` levels of four children ([`tree_levels`]) and `c` counts those +//! levels. The cap is the level `c` below the root, real nodes only: +//! `2^(D − 2(h − c))` of them ([`cap_len`]: `4^c` at even `D`, `2·4^(c−1)` at +//! odd `D`, whose top group is two real nodes and two padding digests), and +//! [`cap_root`] folds it by four with the backend's padding. A path carries +//! three siblings per level, so an opening keeps `3(h − c)` of them and lands +//! on `cap[index >> 2(h − c)]`; the owner path is `3(h − c) + cap_len`. The +//! four checks above hold with those lengths. +//! +//! **Wire encoding (the owner path).** A tree's cap rides at the end of the +//! authentication path of that tree's first opening in proof order +//! ([`embed_cap`] / [`split_owner_path`]); every other opening of the tree +//! carries exactly `D − c` siblings. At `c = 0` nothing moves, so the default +//! proof bytes are today's by construction. + +use alloc::vec::Vec; +use core::fmt; +use core::str::FromStr; + +use super::proof::{Proof, verify_merkle_path_from_leaf_hash}; +use super::traits::IsMerkleTreeBackend; + +/// The tallest cap any policy may ask for. A proof-size guard: a cap costs +/// `2^c` digests per tree. The `Auto` policy never exceeds 3. +pub const MAX_CAP_HEIGHT: usize = 16; + +/// Why a cap operation refused its input. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CapError { + /// The cap height exceeds the tree depth, or [`MAX_CAP_HEIGHT`]. + CapTooTall { cap_height: usize, depth: usize }, + /// A path did not have the exact length the shape requires. + PathLength { expected: usize, got: usize }, + /// A cap whose length is not a power of two. + CapLength(usize), + /// A cap with no opening to carry it. + NoOwner, +} + +impl fmt::Display for CapError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CapTooTall { cap_height, depth } => write!( + f, + "cap height {cap_height} exceeds the tree depth {depth} or the maximum {MAX_CAP_HEIGHT}" + ), + Self::PathLength { expected, got } => { + write!( + f, + "authentication path has {got} nodes, expected {expected}" + ) + } + Self::CapLength(len) => write!(f, "cap of {len} nodes is not a power of two"), + Self::NoOwner => write!(f, "a cap needs at least one opening to carry it"), + } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for CapError {} + +/// `log2(cap.len())` when the cap is a non-empty power of two no taller than +/// [`MAX_CAP_HEIGHT`]. +fn cap_height_of(cap: &[N]) -> Option { + let len = cap.len(); + if !len.is_power_of_two() { + return None; + } + let c = len.ilog2() as usize; + (c <= MAX_CAP_HEIGHT).then_some(c) +} + +/// `c ≤ depth`, `c ≤ MAX_CAP_HEIGHT`, and `depth` small enough that `2^depth` +/// is a `usize`. +fn shape_ok(depth: usize, cap_height: usize) -> bool { + cap_height <= depth && cap_height <= MAX_CAP_HEIGHT && depth < usize::BITS as usize +} + +/// Levels of a tree of `arity` children per node over `2^depth` leaves: `depth` +/// at arity 2, `⌈depth / 2⌉` at arity 4. Cap heights count these levels. +pub fn tree_levels(depth: usize, arity: usize) -> usize { + if arity == 4 { depth.div_ceil(2) } else { depth } +} + +/// Nodes in the height-`cap_height` cap of a tree of `arity` children per node +/// over `2^depth` leaves: `2^c` at arity 2; at arity 4 the real nodes of the +/// level `c` below the root, `2^(depth − 2(h − c))` (`1` at `c = 0`, the root). +/// `None` when the cap does not fit the tree, or when it would hold more than +/// `2^MAX_CAP_HEIGHT` nodes. +pub fn cap_len(depth: usize, cap_height: usize, arity: usize) -> Option { + if arity != 4 { + return shape_ok(depth, cap_height).then(|| 1usize << cap_height); + } + let h = tree_levels(depth, 4); + if depth >= usize::BITS as usize || cap_height > h || 2 * cap_height > MAX_CAP_HEIGHT { + return None; + } + if cap_height == 0 { + return Some(1); + } + Some(1usize << (depth - 2 * (h - cap_height))) +} + +/// The arity-4 cap height whose cap has `len` nodes in a tree over `2^depth` +/// leaves ([`cap_len`]'s inverse), if one has. +fn cap_height4_of(len: usize, depth: usize) -> Option { + (0..=tree_levels(depth, 4)).find(|&c| cap_len(depth, c, 4) == Some(len)) +} + +/// Siblings an opening keeps under a height-`cap_height` cap. +fn kept_siblings(depth: usize, cap_height: usize, arity: usize) -> usize { + if arity == 4 { + 3 * (tree_levels(depth, 4) - cap_height) + } else { + depth - cap_height + } +} + +impl Proof { + /// Keep the first `depth − cap_height` siblings of a full path. + /// + /// Refuses unless the path is exactly `depth` long and the cap fits the + /// tree, so a path that was already cut, or one of another tree, is not + /// silently cut again. + pub fn truncate_to_cap(&mut self, depth: usize, cap_height: usize) -> Result<(), CapError> { + if !shape_ok(depth, cap_height) { + return Err(CapError::CapTooTall { cap_height, depth }); + } + if self.merkle_path.len() != depth { + return Err(CapError::PathLength { + expected: depth, + got: self.merkle_path.len(), + }); + } + self.merkle_path.truncate(depth - cap_height); + Ok(()) + } +} + +/// The root of a cap: the standard bottom-up build over the cap as leaves, +/// `2^c − 1` compressions (none at `c = 0`). `None` unless `cap.len()` is a +/// power of two `≥ 1` no taller than [`MAX_CAP_HEIGHT`]. +pub fn cap_root(cap: &[B::Node]) -> Option { + cap_height_of(cap)?; + let mut level: Vec = cap.to_vec(); + if B::ARITY == 4 { + // Groups of four, a short group padded (`build4`'s rule): a cap of + // `2·4^(c−1)` nodes folds to two, then to the root with two paddings. + while level.len() > 1 { + let mut up = Vec::with_capacity(level.len().div_ceil(4)); + for group in level.chunks(4) { + let child = |i: usize| match group.get(i) { + Some(node) => Some(node.clone()), + None => B::padding_node(), + }; + up.push(B::hash_four(&[child(0)?, child(1)?, child(2)?, child(3)?])); + } + level = up; + } + return level.pop(); + } + while level.len() > 1 { + level = level + .chunks_exact(2) + .map(|pair| B::hash_new_parent(&pair[0], &pair[1])) + .collect(); + } + level.pop() +} + +/// `cap.len() == 2^cap_height` and the cap hashes up to `root`. +pub fn verify_cap( + cap: &[B::Node], + root: &B::Node, + cap_height: usize, +) -> bool { + if cap_height > MAX_CAP_HEIGHT || cap.len() != 1usize << cap_height { + return false; + } + cap_root::(cap).is_some_and(|r| &r == root) +} + +/// [`verify_cap`] for a tree over `2^depth` leaves at the backend's arity: +/// `cap.len()` must be [`cap_len`]`(depth, cap_height, B::ARITY)` (at arity 4 +/// that depends on `depth`'s parity, not on `cap_height` alone) and the cap +/// must hash up to `root`. +pub fn verify_cap_shaped( + cap: &[B::Node], + root: &B::Node, + depth: usize, + cap_height: usize, +) -> bool { + if B::ARITY != 4 { + return shape_ok(depth, cap_height) && verify_cap::(cap, root, cap_height); + } + cap_len(depth, cap_height, 4) == Some(cap.len()) + && cap_root::(cap).is_some_and(|r| &r == root) +} + +/// The capped inclusion check for one opening. +/// +/// `c = log2(cap.len())`. Accepts iff +/// `siblings.len() == depth − c`, `index < 2^depth`, `cap.len() == 2^c ≤ 2^depth`, +/// and the existing fold ([`verify_merkle_path_from_leaf_hash`], unchanged) +/// of `leaf_hash` along `siblings` lands on `cap[index >> (depth − c)]`. +/// +/// It does NOT check the cap against a root — that is [`verify_cap`], once per +/// tree. [`CappedRoot`] ties the two together. +pub fn verify_merkle_path_to_cap_from_leaf_hash( + siblings: &[B::Node], + cap: &[B::Node], + depth: usize, + index: usize, + leaf_hash: B::Node, +) -> bool { + if B::ARITY == 4 { + // `depth` stays the binary depth `log2(leaves)` the shapes speak; the + // walk has `h − c` levels of three siblings (`h = ⌈depth / 2⌉`) and + // lands on `cap[index >> 2(h − c)]` (module docs). + let Some(c) = cap_height4_of(cap.len(), depth) else { + return false; + }; + let walked = tree_levels(depth, 4) - c; + // An uncapped odd-depth tree's top group is two real children and two + // padding nodes, and the walk reaches it: those two siblings are the + // padding, never proof data (the in-guest walk supplies them as + // constants; REV-P1-JUDGE F4). A capped tree's top is in its cap. + if c == 0 && depth % 2 == 1 && siblings.len() == 3 * walked && walked > 0 { + let Some(pad) = B::padding_node() else { + return false; + }; + if siblings[3 * walked - 2] != pad || siblings[3 * walked - 1] != pad { + return false; + } + } + return index >> depth == 0 + && siblings.len() == 3 * walked + && verify_merkle_path_from_leaf_hash::( + siblings, + // `2·walked ≤ 2·⌈depth / 2⌉` can reach the word size. + &cap[index.checked_shr(2 * walked as u32).unwrap_or(0)], + index, + leaf_hash, + ); + } + let Some(c) = cap_height_of(cap) else { + return false; + }; + if !shape_ok(depth, c) || siblings.len() != depth - c || index >> depth != 0 { + return false; + } + verify_merkle_path_from_leaf_hash::(siblings, &cap[index >> (depth - c)], index, leaf_hash) +} + +/// Split an owner path (the wire encoding) into `(siblings, cap)`. +/// +/// At `c = 0` the whole path is siblings (its length must be `depth`) and the +/// cap is empty — the caller uses the root. At `c ≥ 1` the length must be +/// exactly `depth − c + 2^c`. `None` on any other length or shape. +pub fn split_owner_path(path: &[N], depth: usize, cap_height: usize) -> Option<(&[N], &[N])> { + if !shape_ok(depth, cap_height) { + return None; + } + let siblings = depth - cap_height; + let expected = if cap_height == 0 { + depth + } else { + siblings + (1usize << cap_height) + }; + (path.len() == expected).then(|| path.split_at(siblings)) +} + +/// [`split_owner_path`] for a tree of `arity` children per node over +/// `2^depth` leaves, `cap_height` counting that arity's levels: the owner path +/// is the kept siblings (`3(h − c)` at arity 4) followed by the +/// [`cap_len`]-node cap, or the full path at `c = 0`. +pub fn split_owner_path_arity( + path: &[N], + depth: usize, + cap_height: usize, + arity: usize, +) -> Option<(&[N], &[N])> { + if arity != 4 { + return split_owner_path(path, depth, cap_height); + } + let len = cap_len(depth, cap_height, 4)?; + let siblings = kept_siblings(depth, cap_height, 4); + let expected = if cap_height == 0 { + siblings + } else { + siblings + len + }; + (path.len() == expected).then(|| path.split_at(siblings)) +} + +/// [`embed_cap`] for a tree of `arity` children per node over `2^depth` +/// leaves. At arity 4 every path must be its full `3·⌈depth / 2⌉` siblings; +/// the cap's height is the one whose [`cap_len`] is `cap.len()`, every path is +/// cut to `3(h − c)` siblings and the cap is appended to `paths[0]`. A cap of +/// one node (the root) changes nothing. +pub fn embed_cap_arity( + paths: &mut [&mut Vec], + depth: usize, + cap: &[N], + arity: usize, +) -> Result<(), CapError> { + if arity != 4 { + return embed_cap(paths, depth, cap); + } + let c = cap_height4_of(cap.len(), depth).ok_or(CapError::CapLength(cap.len()))?; + let expected = 3 * tree_levels(depth, 4); + if let Some(p) = paths.iter().find(|p| p.len() != expected) { + return Err(CapError::PathLength { + expected, + got: p.len(), + }); + } + if c == 0 { + return Ok(()); + } + let keep = kept_siblings(depth, c, 4); + let Some((owner, rest)) = paths.split_first_mut() else { + return Err(CapError::NoOwner); + }; + owner.truncate(keep); + owner.extend_from_slice(cap); + for path in rest { + path.truncate(keep); + } + Ok(()) +} + +/// Prover side of the owner-path encoding: cut every path of one tree to +/// `depth − c` siblings and append the cap to `paths[0]`, the tree's first +/// opening in proof order. `c = log2(cap.len())`. +/// +/// Every path must be a full `depth`-long path (checked). At `c = 0` (a cap of +/// one node, the root) it changes nothing. +pub fn embed_cap( + paths: &mut [&mut Vec], + depth: usize, + cap: &[N], +) -> Result<(), CapError> { + let c = cap_height_of(cap).ok_or(CapError::CapLength(cap.len()))?; + if !shape_ok(depth, c) { + return Err(CapError::CapTooTall { + cap_height: c, + depth, + }); + } + for path in paths.iter() { + if path.len() != depth { + return Err(CapError::PathLength { + expected: depth, + got: path.len(), + }); + } + } + if c == 0 { + return Ok(()); + } + let Some((owner, rest)) = paths.split_first_mut() else { + return Err(CapError::NoOwner); + }; + owner.truncate(depth - c); + owner.extend_from_slice(cap); + for path in rest { + path.truncate(depth - c); + } + Ok(()) +} + +/// One tree's authenticated cap: built once per tree, then used for every +/// opening of that tree. +/// +/// The only constructors are [`CappedRoot::uncapped`] (`c = 0`, the cap is the +/// root itself) and [`CappedRoot::from_owner`], which runs [`verify_cap`] +/// against the root before it hands the cap out — so a `CappedRoot` never +/// holds an unauthenticated cap. +#[derive(Debug, Clone, Copy)] +pub struct CappedRoot<'a, N> { + depth: usize, + cap_height: usize, + cap: &'a [N], +} + +impl<'a, N: PartialEq + Eq + Clone> CappedRoot<'a, N> { + /// `c = 0`: every path must be exactly `depth` long and fold to `root`. + pub fn uncapped(root: &'a N, depth: usize) -> Self { + Self { + depth, + cap_height: 0, + cap: core::slice::from_ref(root), + } + } + + /// Split the owner path, authenticate its cap against `root` once, and + /// return the owner's own siblings. + /// + /// Only the cap is checked here. The owner's opening is still an opening: + /// the caller must run [`verify`](Self::verify) on the returned siblings + /// like on any other path. `None` on a wrong length or a cap that does not + /// hash to `root`. + pub fn from_owner>( + root: &'a N, + owner_path: &'a [N], + depth: usize, + cap_height: usize, + ) -> Option<(Self, &'a [N])> { + let (siblings, cap) = split_owner_path_arity(owner_path, depth, cap_height, B::ARITY)?; + if cap_height == 0 { + return Some((Self::uncapped(root, depth), siblings)); + } + if !verify_cap_shaped::(cap, root, depth, cap_height) { + return None; + } + Some(( + Self { + depth, + cap_height, + cap, + }, + siblings, + )) + } + + /// Check one opening: exactly `depth − c` siblings folding `leaf_hash` at + /// `index` onto its cap node. + pub fn verify>( + &self, + siblings: &[N], + index: usize, + leaf_hash: N, + ) -> bool { + verify_merkle_path_to_cap_from_leaf_hash::( + siblings, self.cap, self.depth, index, leaf_hash, + ) + } + + pub fn depth(&self) -> usize { + self.depth + } + + pub fn cap_height(&self) -> usize { + self.cap_height + } + + /// The authenticated cap (`[root]` at `c = 0`). + pub fn cap(&self) -> &'a [N] { + self.cap + } +} + +// =========================================================================== +// The cap-height policy +// =========================================================================== + +/// How tall a cap each tree gets. A proof-format parameter: the prover and +/// every verifier (host and in-guest) derive the same height from public +/// shape data through [`CapPolicy::height`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum CapPolicy { + /// No cap: every path runs to the root. Today's format. + #[default] + Off, + /// The height that minimises the in-guest verifier's cost-law price + /// ([`AUTO_WEIGHTS`]); 3 for a tree opened ≥ 20 times, 2 for 4–19, 0 + /// below, clamped to the tree depth. + Auto, + /// This height for every opened tree, clamped to its depth and to + /// [`MAX_CAP_HEIGHT`]. `Fixed(0)` is `Off`. + Fixed(u8), +} + +/// Per-row prices (ns) of the in-guest verifier operations a cap trades, from +/// the node cost law (421 ns/instruction + 5.63 ns/cell) and the committed +/// widths of the chips that execute them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CapWeights { + /// One two-to-one compression (an `LFM_HASH` row). + pub compress: u64, + /// One two-way `Select`. + pub select: u64, + /// One `Unpack` (a digest compared as lanes). + pub unpack: u64, + /// One hinted word. + pub hint: u64, + /// One digest-equals-root comparison. + pub compare: u64, +} + +/// The weights [`CapPolicy::Auto`] optimises. ⚠ A FORMAT CONSTANT: changing +/// any of them changes the cap heights, and so the proofs, of every tree under +/// `Auto`. Pinned by the policy tests. +pub const AUTO_WEIGHTS: CapWeights = CapWeights { + compress: 2251, + select: 567, + unpack: 528, + hint: 460, + compare: 3789, +}; + +/// The in-guest saving (ns, cost-law units) of a height-`c` cap on a tree +/// opened `openings` times. Integer arithmetic only, so every verifier +/// reproduces it exactly. +/// +/// Signed and bounded: the gain is negative for few openings, so it is an +/// `i128`, and every term fits it for any `usize` opening count because +/// `cap_height` is refused past [`MAX_CAP_HEIGHT`] (the result is then +/// `i128::MIN`, a height no argmax picks) — no shift or product can overflow on +/// any target, 32-bit `wasm` included. `gain(o, 0) = 0`; for `c ≥ 1`: +/// +/// ```text +/// o·( c·(compress + select) − (2^c − 1)·select − unpack ) +/// − ( (2^c − 1)·compress + 2^c·hint + compare ) +/// ``` +/// +/// Per opening the walk loses `c` levels (a `Select` and a compression each), +/// the cap mux adds `2^c − 1` selects and the variable-cell compare one +/// `Unpack`; per tree the cap costs `2^c − 1` compressions to its root, `2^c` +/// hints and one root compare. +pub fn cap_gain(weights: &CapWeights, openings: usize, cap_height: usize) -> i128 { + if cap_height == 0 { + return 0; + } + if cap_height > MAX_CAP_HEIGHT { + return i128::MIN; + } + let o = openings as i128; + let c = cap_height as i128; + let nodes = 1i128 << cap_height; + let w = |x: u64| x as i128; + let per_opening = c * (w(weights.compress) + w(weights.select)) + - (nodes - 1) * w(weights.select) + - w(weights.unpack); + let per_tree = (nodes - 1) * w(weights.compress) + nodes * w(weights.hint) + w(weights.compare); + o * per_opening - per_tree +} + +impl CapPolicy { + /// True when this policy caps nothing (`Off` or `Fixed(0)`). + pub const fn is_off(self) -> bool { + matches!(self, Self::Off | Self::Fixed(0)) + } + + /// The cap height of a tree of `depth` levels opened `openings` times. + /// Always `≤ depth` and `≤ MAX_CAP_HEIGHT`, and 0 for an unopened tree. + /// + /// `Auto` is a fixed table, stated directly — 3 for a tree opened at + /// least [`AUTO_CAP3_MIN_OPENINGS`] times, 2 from + /// [`AUTO_CAP2_MIN_OPENINGS`], 0 below — then clamped to the depth. No + /// arithmetic runs at all, so no verifier can disagree on an overflow. + /// The table is the argmax of [`cap_gain`] under [`AUTO_WEIGHTS`] for + /// every opening count (pinned by a test over all counts up to 10^6 and + /// at the `usize` extremes). + pub fn height(self, openings: usize, depth: usize) -> usize { + if openings == 0 { + return 0; + } + let limit = depth.min(MAX_CAP_HEIGHT); + match self { + Self::Off => 0, + Self::Fixed(c) => (c as usize).min(limit), + Self::Auto => { + let c = if openings >= AUTO_CAP3_MIN_OPENINGS { + 3 + } else if openings >= AUTO_CAP2_MIN_OPENINGS { + 2 + } else { + 0 + }; + c.min(limit) + } + } + } +} + +/// `Auto` gives a height-3 cap to a tree opened at least this many times +/// ⚠ A FORMAT CONSTANT, like [`AUTO_WEIGHTS`]. +pub const AUTO_CAP3_MIN_OPENINGS: usize = 20; + +/// `Auto` gives a height-2 cap to a tree opened at least this many times and +/// fewer than [`AUTO_CAP3_MIN_OPENINGS`]. ⚠ A FORMAT CONSTANT. +pub const AUTO_CAP2_MIN_OPENINGS: usize = 4; + +impl fmt::Display for CapPolicy { + /// `off`, `auto`, or the fixed height (`Fixed(0)` prints `off`) — the + /// spelling the `LAMBDA_VM_ZF_*CAP` knobs accept. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Off | Self::Fixed(0) => f.write_str("off"), + Self::Auto => f.write_str("auto"), + Self::Fixed(c) => write!(f, "{c}"), + } + } +} + +/// A cap-policy spelling that is none of `off`, `auto`, `0..=16`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseCapPolicyError; + +impl fmt::Display for ParseCapPolicyError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "expected `off`, `auto`, or a cap height 0..={MAX_CAP_HEIGHT}" + ) + } +} + +impl FromStr for CapPolicy { + type Err = ParseCapPolicyError; + + /// `off` | `auto` | an integer `0..=MAX_CAP_HEIGHT` (`0` is `off`). + /// Exact spellings only: no case folding, no whitespace. + fn from_str(s: &str) -> Result { + match s { + "off" => Ok(Self::Off), + "auto" => Ok(Self::Auto), + _ => { + // `u8::from_str` accepts a leading `+`; the knob does not. + if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) { + return Err(ParseCapPolicyError); + } + match s.parse::() { + Ok(0) => Ok(Self::Off), + Ok(c) if c as usize <= MAX_CAP_HEIGHT => Ok(Self::Fixed(c)), + _ => Err(ParseCapPolicyError), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::merkle_tree::backends::types::BatchKeccak256Backend; + use crate::merkle_tree::merkle::MerkleTree; + use alloc::string::ToString; + use math::field::{element::FieldElement, goldilocks::GoldilocksField}; + + type F = GoldilocksField; + type Fe = FieldElement; + type K = BatchKeccak256Backend; + type Node = [u8; 32]; + + fn leaves(n: usize, salt: u64) -> Vec> { + (0..n as u64) + .map(|i| vec![Fe::from(i * 7 + salt), Fe::from(i ^ 0x55 ^ salt)]) + .collect() + } + + fn tree(n: usize, salt: u64) -> MerkleTree { + MerkleTree::::build(&leaves(n, salt)).expect("non-empty") + } + + /// The leaf data at position `p` of the padded tree (padding repeats the + /// last leaf). + fn leaf_at(data: &[Vec], p: usize) -> &Vec { + &data[p.min(data.len() - 1)] + } + + const LEAF_COUNTS: &[usize] = &[1, 2, 3, 4, 5, 8, 16, 32, 64, 128, 256, 512, 1024]; + + // ---------------------------------------------------------------- primitive + + #[test] + fn cap_is_the_heap_slice_and_hashes_to_the_root() { + for &n in LEAF_COUNTS { + let t = tree(n, 1); + let d = t.depth().unwrap(); + assert_eq!(1usize << d, n.next_power_of_two(), "n={n}"); + for c in 0..=d { + let cap = t.cap(c).unwrap(); + assert_eq!(cap.len(), 1 << c); + assert_eq!( + &cap[..], + &t.nodes()[(1 << c) - 1..(2 << c) - 1], + "n={n} c={c}" + ); + assert_eq!(cap_root::(&cap), Some(t.root), "n={n} c={c}"); + assert!(verify_cap::(&cap, &t.root, c), "n={n} c={c}"); + } + assert_eq!(t.cap(0).unwrap(), vec![t.root]); + assert!(t.cap(d + 1).is_none(), "c > depth must be None (n={n})"); + } + } + + #[test] + fn root_only_tree_has_no_depth_and_no_cap() { + let t = MerkleTree::::from_root([7u8; 32]); + assert_eq!(t.depth(), None); + assert!(t.cap(0).is_none()); + } + + /// Every leaf of every tree verifies against its cap node at every height, + /// and at `c = 0` the capped check agrees with the full-path check. + fn every_leaf_verifies(verify: impl Fn(&[Node], &[Node], usize, usize, Node) -> bool) -> bool { + for &n in LEAF_COUNTS { + let data = leaves(n, 2); + let t = MerkleTree::::build(&data).unwrap(); + let d = t.depth().unwrap(); + for c in 0..=d { + let cap = t.cap(c).unwrap(); + for p in 0..(1usize << d) { + let mut proof = t.get_proof_by_pos(p).unwrap(); + let full = proof.merkle_path.clone(); + proof.truncate_to_cap(d, c).unwrap(); + assert_eq!(proof.merkle_path.len(), d - c); + let leaf = K::hash_data(leaf_at(&data, p)); + if !verify(&proof.merkle_path, &cap, d, p, leaf) { + return false; + } + if c == 0 { + assert!(verify_merkle_path_from_leaf_hash::( + &full, &t.root, p, leaf + )); + } + } + } + } + true + } + + fn real_verify(s: &[Node], cap: &[Node], d: usize, i: usize, l: Node) -> bool { + verify_merkle_path_to_cap_from_leaf_hash::(s, cap, d, i, l) + } + + #[test] + fn every_leaf_verifies_against_its_cap_node() { + assert!(every_leaf_verifies(real_verify)); + } + + #[test] + fn cap_taller_than_the_tree_is_refused_everywhere() { + let t = tree(8, 3); + let d = 3; + let full = t.get_proof_by_pos(0).unwrap(); + let mut p = full.clone(); + assert_eq!( + p.truncate_to_cap(d, d + 1), + Err(CapError::CapTooTall { + cap_height: 4, + depth: 3 + }) + ); + let big_cap = vec![[0u8; 32]; 16]; + assert!(!verify_merkle_path_to_cap_from_leaf_hash::( + &[], + &big_cap, + d, + 0, + [0u8; 32] + )); + assert!(split_owner_path(&big_cap, d, d + 1).is_none()); + let mut a = full.merkle_path.clone(); + assert!(embed_cap(&mut [&mut a], d, &big_cap).is_err()); + assert!(!verify_cap::(&big_cap, &t.root, MAX_CAP_HEIGHT + 1)); + } + + #[test] + fn truncate_refuses_a_path_that_is_not_full_length() { + let t = tree(16, 4); + let mut p = t.get_proof_by_pos(5).unwrap(); + p.truncate_to_cap(4, 2).unwrap(); + // Already cut: a second cut must not silently shorten it further. + assert_eq!( + p.truncate_to_cap(4, 2), + Err(CapError::PathLength { + expected: 4, + got: 2 + }) + ); + } + + #[test] + fn cap_root_needs_a_power_of_two() { + assert!(cap_root::(&[]).is_none()); + assert!(cap_root::(&[[1u8; 32]; 3]).is_none()); + assert_eq!(cap_root::(&[[1u8; 32]]), Some([1u8; 32])); + } + + // ------------------------------------------------------ owner-path encoding + + #[test] + fn split_owner_path_takes_exact_lengths_only() { + let d: usize = 6; + for c in 0..=d { + let want = if c == 0 { d } else { d - c + (1 << c) }; + for len in want.saturating_sub(1)..=want + 1 { + let path = vec![0u8; len]; + let got = split_owner_path(&path, d, c); + if len == want { + let (s, cap) = got.unwrap(); + assert_eq!(s.len(), d - c); + assert_eq!(cap.len(), if c == 0 { 0 } else { 1 << c }); + } else { + assert!(got.is_none(), "c={c} len={len}"); + } + } + } + } + + #[test] + fn embed_then_split_round_trips_and_every_opening_verifies() { + let data = leaves(64, 5); + let t = MerkleTree::::build(&data).unwrap(); + let d = 6; + let positions = [17usize, 3, 63, 0, 17]; + for c in 0..=d { + let cap = t.cap(c).unwrap(); + let mut paths: Vec> = positions + .iter() + .map(|&p| t.get_proof_by_pos(p).unwrap().merkle_path) + .collect(); + let full0 = paths[0].clone(); + { + let mut refs: Vec<&mut Vec> = paths.iter_mut().collect(); + embed_cap(&mut refs, d, &cap).unwrap(); + } + if c == 0 { + assert_eq!(paths[0], full0, "c = 0 must be a no-op"); + } else { + assert_eq!(paths[0].len(), d - c + (1 << c)); + assert_eq!(&paths[0][d - c..], &cap[..]); + } + for p in &paths[1..] { + assert_eq!(p.len(), d - c); + } + let (check, owner) = CappedRoot::from_owner::(&t.root, &paths[0], d, c).unwrap(); + assert_eq!(check.cap_height(), c); + assert_eq!(owner.len(), d - c); + assert!(check.verify::(owner, positions[0], K::hash_data(&data[positions[0]]))); + for (path, &pos) in paths[1..].iter().zip(&positions[1..]) { + assert!(check.verify::(path, pos, K::hash_data(&data[pos]))); + } + } + } + + #[test] + fn embed_refuses_short_paths_and_an_empty_owner_list() { + let t = tree(16, 6); + let cap = t.cap(2).unwrap(); + let mut short = vec![[0u8; 32]; 3]; + assert_eq!( + embed_cap(&mut [&mut short], 4, &cap), + Err(CapError::PathLength { + expected: 4, + got: 3 + }) + ); + assert_eq!(embed_cap::(&mut [], 4, &cap), Err(CapError::NoOwner)); + assert_eq!( + embed_cap(&mut [&mut vec![[0u8; 32]; 4]], 4, &cap[..3]), + Err(CapError::CapLength(3)) + ); + } + + // ------------------------------------------------------------- tamper tests + + struct Fixture { + data: Vec>, + t: MerkleTree, + d: usize, + c: usize, + } + + fn fixture() -> Fixture { + let data = leaves(256, 9); + let t = MerkleTree::::build(&data).unwrap(); + Fixture { + data, + t, + d: 8, + c: 3, + } + } + + impl Fixture { + fn owner_path(&self, pos: usize) -> Vec { + let mut p = self.t.get_proof_by_pos(pos).unwrap().merkle_path; + embed_cap(&mut [&mut p], self.d, &self.t.cap(self.c).unwrap()).unwrap(); + p + } + fn path(&self, pos: usize) -> Vec { + let mut p = self.t.get_proof_by_pos(pos).unwrap(); + p.truncate_to_cap(self.d, self.c).unwrap(); + p.merkle_path + } + fn leaf(&self, pos: usize) -> Node { + K::hash_data(&self.data[pos]) + } + } + + #[test] + fn a_flipped_cap_byte_is_rejected() { + let f = fixture(); + let honest = f.owner_path(10); + assert!(CappedRoot::from_owner::(&f.t.root, &honest, f.d, f.c).is_some()); + for k in 0..(1 << f.c) { + let mut owner = honest.clone(); + owner[f.d - f.c + k][0] ^= 1; + assert!( + CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).is_none(), + "k={k}" + ); + assert!(!verify_cap::(&owner[f.d - f.c..], &f.t.root, f.c)); + } + } + + #[test] + fn a_flipped_path_node_is_rejected() { + let f = fixture(); + let owner = f.owner_path(10); + let (check, _) = CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).unwrap(); + let honest = f.path(77); + assert!(check.verify::(&honest, 77, f.leaf(77))); + for k in 0..honest.len() { + let mut p = honest.clone(); + p[k][31] ^= 0x80; + assert!(!check.verify::(&p, 77, f.leaf(77)), "k={k}"); + } + } + + #[test] + fn swapped_cap_nodes_are_rejected() { + let f = fixture(); + let mut owner = f.owner_path(10); + let base = f.d - f.c; + owner.swap(base, base + 5); + assert!(CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).is_none()); + } + + #[test] + fn a_cap_from_another_tree_is_rejected() { + let f = fixture(); + let other = tree(256, 1234); + let mut owner = f.path(10); + owner.extend(other.cap(f.c).unwrap()); + assert!(CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).is_none()); + // And another tree's cap cannot vouch for this tree's openings even + // when paired with that tree's own root. + let (check, _) = CappedRoot::from_owner::(&other.root, &owner, f.d, f.c).unwrap(); + assert!(!check.verify::(&f.path(77), 77, f.leaf(77))); + } + + #[test] + fn an_index_with_a_flipped_top_bit_is_rejected() { + let f = fixture(); + let owner = f.owner_path(10); + let (check, _) = CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).unwrap(); + let pos = 77usize; + let path = f.path(pos); + assert!(check.verify::(&path, pos, f.leaf(pos))); + for bit in 0..f.d { + let wrong = pos ^ (1 << bit); + assert!(!check.verify::(&path, wrong, f.leaf(pos)), "bit={bit}"); + } + // Past the tree: an index ≥ 2^D is refused, not wrapped. + assert!(!check.verify::(&path, pos + (1 << f.d), f.leaf(pos))); + } + + #[test] + fn a_path_one_node_too_long_or_short_is_rejected() { + let f = fixture(); + let owner = f.owner_path(10); + let (check, _) = CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).unwrap(); + let path = f.path(77); + assert!(!check.verify::(&path[..path.len() - 1], 77, f.leaf(77))); + let mut long = path.clone(); + long.push(path[0]); + assert!(!check.verify::(&long, 77, f.leaf(77))); + // The full, uncut path is also refused under a cap. + let full = f.t.get_proof_by_pos(77).unwrap().merkle_path; + assert!(!check.verify::(&full, 77, f.leaf(77))); + // And the owner path one node short or long. + assert!(CappedRoot::from_owner::(&f.t.root, &owner[1..], f.d, f.c).is_none()); + let mut owner_long = owner.clone(); + owner_long.push(owner[0]); + assert!(CappedRoot::from_owner::(&f.t.root, &owner_long, f.d, f.c).is_none()); + } + + #[test] + fn a_cap_moved_to_the_second_opening_is_rejected() { + let f = fixture(); + let cap = f.t.cap(f.c).unwrap(); + // Query 0 carries a plain path, query 1 the cap: the wrong owner. + let q0 = f.path(10); + let mut q1 = f.path(77); + q1.extend(cap.iter().copied()); + assert!(CappedRoot::from_owner::(&f.t.root, &q0, f.d, f.c).is_none()); + // Even with a correctly authenticated cap in hand, query 1's path + // (siblings + cap) is the wrong length for a non-owner. + let owner = f.owner_path(10); + let (check, _) = CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).unwrap(); + assert!(!check.verify::(&q1, 77, f.leaf(77))); + } + + #[test] + fn a_proof_capped_at_one_height_fails_at_another() { + let f = fixture(); + let owner = f.owner_path(10); // c = 3 + for other in [0, 1, 2, 4] { + assert!( + CappedRoot::from_owner::(&f.t.root, &owner, f.d, other).is_none(), + "c=3 proof accepted at c={other}" + ); + } + } + + #[test] + fn uncapped_is_the_full_path_check_plus_exact_length() { + let data = leaves(32, 11); + let t = MerkleTree::::build(&data).unwrap(); + let check = CappedRoot::uncapped(&t.root, 5); + for (p, value) in data.iter().enumerate() { + let path = t.get_proof_by_pos(p).unwrap().merkle_path; + let leaf = K::hash_data(value); + assert!(check.verify::(&path, p, leaf)); + assert!(!check.verify::(&path[..4], p, leaf)); + let mut long = path.clone(); + long.push(path[0]); + assert!(!check.verify::(&long, p, leaf)); + } + } + + // ----------------------------------------------------------- mutation tests + // + // Each load-bearing check has a property function that the named test runs + // against the real primitive, and a mutation test that runs the SAME + // function against a copy with that one check removed and asserts it + // fails. So removing the check from the real code makes the named test + // fail: the check is shown to carry the property, not just to be present. + + /// A toy backend whose leaf hash is the identity, so an internal node can + /// be presented as a leaf — the forgery a missing length check admits. + struct IdentityLeaf; + impl IsMerkleTreeBackend for IdentityLeaf { + type Node = u64; + type Data = u64; + fn hash_data(leaf: &u64) -> u64 { + *leaf + } + fn hash_new_parent(a: &u64, b: &u64) -> u64 { + a.wrapping_mul(0x9E37_79B9_7F4A_7C15).rotate_left(17) ^ b.wrapping_add(0x0123_4567_89AB) + } + } + + type U64Verify = fn(&[u64], &[u64], usize, usize, u64) -> bool; + + /// A path one level short, whose "leaf" is really the internal node over + /// leaves 0 and 1, must not verify at index 0. + fn rejects_short_path_forgery(verify: U64Verify) -> bool { + let d = 6; + let c = 2; + let data: Vec = (0..64u64).map(|i| i * 1_000_003 + 17).collect(); + let t = MerkleTree::::build(&data).unwrap(); + let cap = t.cap(c).unwrap(); + let full = t.get_proof_by_pos(0).unwrap().merkle_path; + // Honest opening of leaf 0 verifies. + assert!(verify(&full[..d - c], &cap, d, 0, data[0])); + // The forgery: claim the parent of leaves 0 and 1 as the value at + // index 0, with the path from that parent up to the cap. + let internal = IdentityLeaf::hash_new_parent(&data[0], &data[1]); + assert_ne!( + internal, data[0], + "the forged value must not be the real leaf" + ); + !verify(&full[1..d - c], &cap, d, 0, internal) + } + + #[test] + fn exact_length_check_rejects_a_short_path_forgery() { + assert!(rejects_short_path_forgery( + verify_merkle_path_to_cap_from_leaf_hash:: + )); + } + + #[test] + fn mutation_without_the_length_check_admits_the_forgery() { + fn mutant(s: &[u64], cap: &[u64], d: usize, i: usize, l: u64) -> bool { + let c = cap.len().ilog2() as usize; + // Mutation: no `siblings.len() == depth − c` check. + verify_merkle_path_from_leaf_hash::(s, &cap[i >> (d - c)], i, l) + } + assert!(!rejects_short_path_forgery(mutant)); + } + + #[test] + fn mutation_with_the_wrong_cap_index_fails_honest_openings() { + fn mutant(s: &[Node], cap: &[Node], d: usize, i: usize, l: Node) -> bool { + let c = cap.len().ilog2() as usize; + if s.len() != d - c || c == d { + // `d − c − 1` would underflow; keep the mutant defined there. + return verify_merkle_path_to_cap_from_leaf_hash::(s, cap, d, i, l); + } + // Mutation: `index >> (D − c − 1)` instead of `index >> (D − c)`. + cap.get(i >> (d - c - 1)) + .is_some_and(|node| verify_merkle_path_from_leaf_hash::(s, node, i, l)) + } + assert!(!every_leaf_verifies(mutant)); + } + + /// A cap made from another tree, carried by the owner path next to the + /// honest root, must not let a leaf of that other tree verify. + /// `CappedRoot::from_owner`'s shape, so a mutant can stand in for it. + type FromOwner = for<'a> fn( + &'a Node, + &'a [Node], + usize, + usize, + ) -> Option<(CappedRoot<'a, Node>, &'a [Node])>; + + fn rejects_forged_cap(from_owner: FromOwner) -> bool { + let d = 8; + let c = 3; + let honest = tree(256, 21); + let forged_data = leaves(256, 99); + let forged = MerkleTree::::build(&forged_data).unwrap(); + let mut owner = forged.get_proof_by_pos(40).unwrap().merkle_path; + embed_cap(&mut [&mut owner], d, &forged.cap(c).unwrap()).unwrap(); + match from_owner(&honest.root, &owner, d, c) { + None => true, + Some((check, siblings)) => { + !check.verify::(siblings, 40, K::hash_data(&forged_data[40])) + } + } + } + + #[test] + fn cap_to_root_check_rejects_a_forged_cap() { + assert!(rejects_forged_cap( + |r, p, d, c| CappedRoot::from_owner::(r, p, d, c) + )); + } + + #[test] + fn mutation_without_the_cap_to_root_check_admits_a_forged_cap() { + fn mutant<'a>( + _root: &'a Node, + path: &'a [Node], + d: usize, + c: usize, + ) -> Option<(CappedRoot<'a, Node>, &'a [Node])> { + let (siblings, cap) = split_owner_path(path, d, c)?; + // Mutation: no `verify_cap(cap, root)`. + Some(( + CappedRoot { + depth: d, + cap_height: c, + cap, + }, + siblings, + )) + } + assert!(!rejects_forged_cap(mutant)); + } + + // ------------------------------------------- the only-rejecting-check fixtures + // + // A tamper that some OTHER check also rejects cannot show a + // check is load-bearing — removing it leaves the test green. These two + // fixtures are built so that exactly one check rejects them, on the real + // keccak backend (no toy hash): delete that check and the test fails. + + /// Heap index of the ancestor at `height` levels above leaf `pos` in a tree + /// of depth `d` (`height = 0` is the leaf itself). + fn ancestor(d: usize, pos: usize, height: usize) -> usize { + (1usize << (d - height)) - 1 + (pos >> height) + } + + /// M1(a). The real internal node one level above leaf `pos`, presented as a + /// "leaf hash" with the path from that node upward — one sibling short. + /// At `pos = 0` and `pos = 2^D − 1` the index bits the fold consumes stay + /// consistent after the shift (all 0 / all 1), so the length-agnostic fold + /// ACCEPTS: only `siblings.len() == D − c` rejects it. Hash-agnostic — the + /// node is read out of the tree, not forged — and at `c = 0` it is exactly + /// the uncapped exact-length case. + #[test] + fn an_internal_node_as_leaf_hash_is_rejected_only_by_the_length_check() { + let t = tree(64, 5); + let d = t.depth().unwrap(); + assert_eq!(d, 6); + for c in 0..d { + let cap = t.cap(c).unwrap(); + for pos in [0usize, (1 << d) - 1] { + let full = t.get_proof_by_pos(pos).unwrap().merkle_path; + let node = t.nodes()[ancestor(d, pos, 1)]; + let forged = &full[1..d - c]; + // The length-agnostic fold accepts the forgery: no other check + // stands between it and acceptance. + assert!( + verify_merkle_path_from_leaf_hash::(forged, &cap[pos >> (d - c)], pos, node), + "c={c} pos={pos}: fixture precondition, the fold alone accepts" + ); + // The real check refuses it. + assert!( + !verify_merkle_path_to_cap_from_leaf_hash::(forged, &cap, d, pos, node), + "c={c} pos={pos}: an internal node passed for a leaf" + ); + if c == 0 { + assert!(!CappedRoot::uncapped(&t.root, d).verify::(forged, pos, node)); + } + } + } + } + + /// M1(b). Few openings under a tall cap: with 3 queries and `c = 3`, at + /// least 5 of the 8 cap nodes are reached by no query. Flipping one of those + /// leaves every per-query check green, so only the cap-to-root check + /// (`verify_cap`, run by `from_owner`) rejects it. + #[test] + fn an_unreached_cap_node_is_rejected_only_by_the_cap_to_root_check() { + let f = fixture(); + let queries = [10usize, 20, 30]; // all under cap node 0 (pos >> 5 == 0) + let reached: Vec = queries.iter().map(|q| q >> (f.d - f.c)).collect(); + let unreached = (0..1usize << f.c) + .find(|k| !reached.contains(k)) + .expect("some cap node is unreached"); + let mut owner = f.owner_path(queries[0]); + owner[f.d - f.c + unreached][0] ^= 1; + let (siblings0, tampered_cap) = split_owner_path(&owner, f.d, f.c).unwrap(); + // Every query still verifies against the tampered cap: no per-query + // check sees the unreached node. + assert!(verify_merkle_path_to_cap_from_leaf_hash::( + siblings0, + tampered_cap, + f.d, + queries[0], + f.leaf(queries[0]) + )); + for &q in &queries[1..] { + assert!( + verify_merkle_path_to_cap_from_leaf_hash::( + &f.path(q), + tampered_cap, + f.d, + q, + f.leaf(q) + ), + "q={q}: fixture precondition, per-query checks pass" + ); + } + // Only the cap-to-root check rejects it. + assert!(!verify_cap::(tampered_cap, &f.t.root, f.c)); + assert!(CappedRoot::from_owner::(&f.t.root, &owner, f.d, f.c).is_none()); + } + + // ------------------------------------------------------------ policy pins + + #[test] + fn auto_heights_are_pinned() { + let deep = 30; + for (openings, want) in [ + (0, 0), + (1, 0), + (3, 0), + (4, 2), + (19, 2), + (20, 3), + (110, 3), + (112, 3), + (224, 3), + (10_000, 3), + ] { + assert_eq!(CapPolicy::Auto.height(openings, deep), want, "o={openings}"); + } + } + + #[test] + fn auto_never_goes_past_three_under_the_pinned_weights() { + for o in 0..5_000 { + assert!(CapPolicy::Auto.height(o, 40) <= 3, "o={o}"); + } + // c = 4 loses to c = 3 on both the per-opening and the per-tree term. + assert!(cap_gain(&AUTO_WEIGHTS, 1_000_000, 4) < cap_gain(&AUTO_WEIGHTS, 1_000_000, 3)); + } + + /// The argmax of the cost law over every height `0..=MAX_CAP_HEIGHT`, ties + /// to the smaller height (`gain(·, 0) = 0`). + fn cost_law_argmax(openings: usize, limit: usize) -> usize { + let mut best = (0usize, 0i128); + for c in 1..=limit { + let g = cap_gain(&AUTO_WEIGHTS, openings, c); + if g > best.1 { + best = (c, g); + } + } + best.0 + } + + /// `Auto` is a fixed table; this pins that the table is + /// the cost-law argmax for every opening count, so the table and the + /// weights cannot drift apart. + #[test] + fn the_auto_table_is_the_cost_law_argmax_at_every_opening_count() { + for o in 0..=1_000_000usize { + assert_eq!( + CapPolicy::Auto.height(o, MAX_CAP_HEIGHT), + cost_law_argmax(o, MAX_CAP_HEIGHT), + "o={o}" + ); + } + for o in [usize::MAX, usize::MAX / 2, 1 << 40, u32::MAX as usize] { + assert_eq!(CapPolicy::Auto.height(o, 64), 3, "o={o}"); + assert_eq!(cost_law_argmax(o, MAX_CAP_HEIGHT), 3, "o={o}"); + } + } + + /// Clamping the table to the depth is not the same function as + /// an argmax bounded by the depth, at exactly one point: 4 openings of a + /// depth-1 tree, where the table says 1 and the bounded argmax 0 (a c = 1 + /// cap loses 68 ns there). The table is the rule; this pins the one + /// difference so any other one is a failure. + #[test] + fn the_depth_clamped_table_differs_from_a_bounded_argmax_at_one_point() { + let mut diffs = Vec::new(); + for d in 0..=6usize { + for o in 0..5_000usize { + if CapPolicy::Auto.height(o, d) != cost_law_argmax(o, d.min(MAX_CAP_HEIGHT)) { + diffs.push((o, d)); + } + } + } + assert_eq!(diffs, vec![(4, 1)]); + } + + #[test] + fn the_cost_law_is_bounded_for_every_input() { + // No overflow at the `usize` extremes and the tallest height. + let top = cap_gain(&AUTO_WEIGHTS, usize::MAX, MAX_CAP_HEIGHT); + assert!(top < 0, "a height-16 cap loses at any opening count"); + assert!(cap_gain(&AUTO_WEIGHTS, usize::MAX, 3) > 0); + // A height past the maximum is refused, never shifted. + for c in [MAX_CAP_HEIGHT + 1, 127, 128, usize::MAX] { + assert_eq!(cap_gain(&AUTO_WEIGHTS, 1_000, c), i128::MIN, "c={c}"); + } + } + + #[test] + fn heights_clamp_to_the_depth() { + for d in 0..6 { + assert_eq!(CapPolicy::Auto.height(110, d), d.min(3), "d={d}"); + } + assert_eq!(CapPolicy::Fixed(5).height(110, 2), 2); + assert_eq!(CapPolicy::Fixed(5).height(110, 9), 5); + assert_eq!(CapPolicy::Fixed(16).height(1, 40), 16); + assert_eq!(CapPolicy::Fixed(200).height(1, 40), MAX_CAP_HEIGHT); + assert_eq!( + CapPolicy::Fixed(5).height(0, 9), + 0, + "an unopened tree has no cap" + ); + } + + #[test] + fn off_and_fixed_zero_are_zero_everywhere() { + for o in 0..300 { + for d in 0..24 { + assert_eq!(CapPolicy::Off.height(o, d), 0); + assert_eq!(CapPolicy::Fixed(0).height(o, d), 0); + } + } + assert!(CapPolicy::Off.is_off()); + assert!(CapPolicy::Fixed(0).is_off()); + assert!(!CapPolicy::Auto.is_off()); + assert!(!CapPolicy::Fixed(1).is_off()); + assert_eq!(CapPolicy::default(), CapPolicy::Off); + } + + #[test] + fn auto_weights_are_pinned() { + assert_eq!( + AUTO_WEIGHTS, + CapWeights { + compress: 2251, + select: 567, + unpack: 528, + hint: 460, + compare: 3789, + } + ); + // The gains the pinned heights rest on. + assert_eq!(cap_gain(&AUTO_WEIGHTS, 20, 2), 55_758); + assert_eq!(cap_gain(&AUTO_WEIGHTS, 20, 3), 55_914); + assert_eq!(cap_gain(&AUTO_WEIGHTS, 19, 2), 52_351); + assert_eq!(cap_gain(&AUTO_WEIGHTS, 19, 3), 51_957); + assert_eq!(cap_gain(&AUTO_WEIGHTS, 4, 1), -68); + assert_eq!(cap_gain(&AUTO_WEIGHTS, 4, 2), 1_246); + } + + #[test] + fn policy_spellings_parse_and_print() { + for (s, want) in [ + ("off", CapPolicy::Off), + ("auto", CapPolicy::Auto), + ("0", CapPolicy::Off), + ("1", CapPolicy::Fixed(1)), + ("16", CapPolicy::Fixed(16)), + ] { + assert_eq!(s.parse::(), Ok(want), "{s}"); + } + for bad in [ + "", "17", "256", "-1", "+3", " 3", "3 ", "Auto", "OFF", "on", "3.0", "x", + ] { + assert!(bad.parse::().is_err(), "{bad:?} must be refused"); + } + assert_eq!(CapPolicy::Off.to_string(), "off"); + assert_eq!(CapPolicy::Fixed(0).to_string(), "off"); + assert_eq!(CapPolicy::Auto.to_string(), "auto"); + assert_eq!(CapPolicy::Fixed(7).to_string(), "7"); + for p in [ + CapPolicy::Off, + CapPolicy::Auto, + CapPolicy::Fixed(1), + CapPolicy::Fixed(16), + ] { + assert_eq!(p.to_string().parse::(), Ok(p)); + } + } +} diff --git a/crypto/crypto/src/merkle_tree/merkle.rs b/crypto/crypto/src/merkle_tree/merkle.rs index 447654907..f15603eb8 100644 --- a/crypto/crypto/src/merkle_tree/merkle.rs +++ b/crypto/crypto/src/merkle_tree/merkle.rs @@ -4,19 +4,20 @@ use crate::merkle_tree::proof::BatchProof; use super::{proof::Proof, traits::IsMerkleTreeBackend, utils::*}; use alloc::{collections::BTreeSet, vec::Vec}; -#[cfg(feature = "disk-spill")] -use math::spill_safe::SpillSafe; #[derive(Debug)] pub enum Error { OutOfBounds, EmptyPositionList, + /// The operation exists for binary trees only. + Arity, } impl Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Error::OutOfBounds => write!(f, "Accessed node was out of bound"), Error::EmptyPositionList => write!(f, "Position list cannot be empty"), + Error::Arity => write!(f, "Operation not supported at this tree arity"), } } } @@ -24,16 +25,6 @@ impl Display for Error { #[cfg(feature = "std")] impl std::error::Error for Error {} -/// File-backed mmap storage for Merkle tree nodes. -/// -/// After `spill_nodes_to_disk()`, the in-memory node vector is freed and -/// node access goes through this mmap instead. -#[cfg(feature = "disk-spill")] -pub(crate) struct MmapNodeBacking { - mmap: memmap2::Mmap, - node_count: usize, -} - /// The struct for the Merkle tree, consisting of the root and the nodes. /// A typical tree would look like this /// root @@ -43,75 +34,22 @@ pub(crate) struct MmapNodeBacking { /// leaf 1 leaf 2 leaf 3 leaf 4 /// The bottom leafs correspond to the hashes of the elements, while each upper /// layer contains the hash of the concatenation of the daughter nodes. -#[cfg_attr(not(feature = "disk-spill"), derive(Clone))] -#[cfg_attr( - all(feature = "serde", not(feature = "disk-spill")), - derive(serde::Serialize, serde::Deserialize) -)] -#[cfg_attr( - all(feature = "serde", feature = "disk-spill"), - derive(serde::Deserialize) -)] +/// +/// # Arity 4 ([`IsMerkleTreeBackend::ARITY`] = 4) +/// +/// Every inner node hashes four children ([`IsMerkleTreeBackend::hash_four`]); +/// a level whose length is not a multiple of four is padded with +/// [`IsMerkleTreeBackend::padding_node`] when its parents are hashed (ZisK's +/// rule: 2^odd leaves end in a two-node level). `nodes` stores only real nodes, +/// level by level top-down (root at 0, leaves last; `utils::level_offsets4`). +/// A path carries, per level from the leaves up, the three other children of +/// the node's group in child order — a padding digest where the group is +/// short — so it is `3 · depth` nodes long, `depth = ⌈log2(leaves) / 2⌉`. +#[derive(Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct MerkleTree { pub root: B::Node, nodes: Vec, - #[cfg(feature = "disk-spill")] - #[cfg_attr(feature = "serde", serde(skip))] - mmap_backing: Option, -} - -#[cfg(all(test, feature = "disk-spill"))] -impl MerkleTree { - pub(crate) fn has_mmap_backing(&self) -> bool { - self.mmap_backing.is_some() - } -} - -// `mmap_backing` is `#[serde(skip)]` and `spill_nodes_to_disk` empties `nodes`, -// so the default derive would emit `{root, nodes: []}` and lose the tree. -// -// Output matches the non-disk-spill derive byte-for-byte, so a proof from either -// storage mode deserializes with the same `Deserialize` impl. -#[cfg(all(feature = "serde", feature = "disk-spill"))] -impl serde::Serialize for MerkleTree -where - B::Node: serde::Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - use serde::ser::SerializeStruct; - let mut s = serializer.serialize_struct("MerkleTree", 2)?; - s.serialize_field("root", &self.root)?; - if self.mmap_backing.is_some() { - s.serialize_field("nodes", &MmapNodesSeq(self))?; - } else { - s.serialize_field("nodes", &self.nodes)?; - } - s.end() - } -} - -#[cfg(all(feature = "serde", feature = "disk-spill"))] -struct MmapNodesSeq<'a, B: IsMerkleTreeBackend>(&'a MerkleTree); - -#[cfg(all(feature = "serde", feature = "disk-spill"))] -impl serde::Serialize for MmapNodesSeq<'_, B> -where - B::Node: serde::Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - use serde::ser::SerializeSeq; - let backing = self - .0 - .mmap_backing - .as_ref() - .expect("MmapNodesSeq is only constructed when mmap_backing is Some"); - let n = backing.node_count; - let mut seq = serializer.serialize_seq(Some(n))?; - for i in 0..n { - seq.serialize_element(self.0.node_get(i).expect("index in bounds"))?; - } - seq.end() - } } const ROOT: usize = 0; @@ -142,6 +80,12 @@ where if nodes.is_empty() { return None; } + if B::ARITY == 4 { + // The arity-4 layout (type docs): its node count names its leaf count. + leaves_len4(nodes.len())?; + let root = nodes[ROOT].clone(); + return Some(MerkleTree { root, nodes }); + } // Validate (cheap) that (nodes.len() + 1) is a power of two: there // must be `leaves_len - 1 + leaves_len = 2*leaves_len - 1` entries. let total = nodes.len(); @@ -160,25 +104,13 @@ where ); } let root = nodes[ROOT].clone(); - Some(MerkleTree { - root, - nodes, - #[cfg(feature = "disk-spill")] - mmap_backing: None, - }) + Some(MerkleTree { root, nodes }) } /// True when this tree carries only its root (the nodes live elsewhere, /// e.g. device-resident): openings must not walk this tree. pub fn is_root_only(&self) -> bool { - #[cfg(feature = "disk-spill")] - { - self.nodes.is_empty() && self.mmap_backing.is_none() - } - #[cfg(not(feature = "disk-spill"))] - { - self.nodes.is_empty() - } + self.nodes.is_empty() } /// Create a root only Merkle tree placeholder: stores the commitment root @@ -189,8 +121,6 @@ where MerkleTree { root, nodes: Vec::new(), - #[cfg(feature = "disk-spill")] - mmap_backing: None, } } @@ -207,6 +137,17 @@ where let hashed_leaves = complete_until_power_of_two(hashed_leaves); let leaves_len = hashed_leaves.len(); + if B::ARITY == 4 { + let inner: usize = level_sizes4(leaves_len)[1..].iter().sum(); + let mut nodes = vec![hashed_leaves[0].clone(); inner]; + nodes.extend(hashed_leaves); + build4::(&mut nodes, leaves_len)?; + return Some(MerkleTree { + root: nodes[ROOT].clone(), + nodes, + }); + } + //The length of leaves minus one inner node in the merkle tree //The first elements are overwritten by build function, it doesn't matter what it's there let mut nodes = vec![hashed_leaves[0].clone(); leaves_len - 1]; @@ -218,17 +159,11 @@ where Some(MerkleTree { root: nodes[ROOT].clone(), nodes, - #[cfg(feature = "disk-spill")] - mmap_backing: None, }) } /// Total number of nodes in the tree (inner + leaves). fn node_count(&self) -> usize { - #[cfg(feature = "disk-spill")] - if let Some(ref backing) = self.mmap_backing { - return backing.node_count; - } self.nodes.len() } @@ -236,24 +171,51 @@ where /// /// Returns `None` if `idx` is out of bounds. fn node_get(&self, idx: usize) -> Option<&B::Node> { - #[cfg(feature = "disk-spill")] - if let Some(ref backing) = self.mmap_backing { - if idx < backing.node_count { - // SAFETY: spill_nodes_to_disk is the only function that populates - // mmap_backing, and its where-clause requires B::Node: SpillSafe. - // Reaching this branch means that bound was checked at construction, - // so B::Node carries no padding and every bit pattern is valid. - // - // Alignment: the mmap base is page-aligned (>= 4096), spill_slice_to_mmap - // asserts align_of::() <= 4096, and Rust guarantees - // size_of:: is a multiple of align_of::, so every - // offset idx * node_size lands on an aligned address. - let ptr = unsafe { backing.mmap.as_ptr().add(idx * size_of::()) }; - return Some(unsafe { &*(ptr as *const B::Node) }); - } + self.nodes.get(idx) + } + + /// `log2` of the padded leaf count: the number of siblings on a full + /// authentication path. `None` on a root-only tree + /// ([`from_root`](Self::from_root)), whose shape is not known here. + pub fn depth(&self) -> Option { + if self.is_root_only() { return None; } - self.nodes.get(idx) + if B::ARITY == 4 { + return Some(level_sizes4(self.leaves_len()).len() - 1); + } + // `node_count = 2·leaves − 1` with `leaves` a power of two (every + // constructor guarantees it), so `leaves = (node_count + 1) / 2`. + let leaves = self.node_count().div_ceil(2); + Some(leaves.ilog2() as usize) + } + + /// The Merkle cap at height `cap_height`: the `2^cap_height` nodes that + /// sit `cap_height` levels below the root, left to right (heap indices + /// `[2^c − 1, 2^{c+1} − 1)`). Height 0 is `[root]`; height `depth` is the + /// leaf-hash layer. + /// + /// `None` on a root-only tree, and when `cap_height > depth` — a cap taller + /// than the tree is not representable, and a caller asking for one has a + /// policy bug that must fail closed rather than be clamped here. + pub fn cap(&self, cap_height: usize) -> Option> { + let depth = self.depth()?; + if cap_height > depth { + return None; + } + if B::ARITY == 4 { + // The real nodes of the level `cap_height` below the root. + let sizes = level_sizes4(self.leaves_len()); + let offsets = level_offsets4(&sizes); + let level = depth - cap_height; + return (offsets[level]..offsets[level] + sizes[level]) + .map(|i| self.node_get(i).cloned()) + .collect(); + } + let start = (1usize << cap_height) - 1; + (start..2 * start + 1) + .map(|i| self.node_get(i).cloned()) + .collect() } /// Read-only access to the full node buffer in standard layout: @@ -274,6 +236,11 @@ where if self.is_root_only() { return None; } + if B::ARITY == 4 { + return self + .build_merkle_path4(pos) + .map(|merkle_path| Proof { merkle_path }); + } let pos = pos + self.node_count() / 2; let Ok(merkle_path) = self.build_merkle_path(pos) else { return None; @@ -287,6 +254,41 @@ where Some(Proof { merkle_path }) } + /// The leaf count: `(nodes + 1) / 2` for a binary tree, the inverse of + /// the level-size sum at arity 4. Not meaningful on a root-only tree. + fn leaves_len(&self) -> usize { + if B::ARITY == 4 { + leaves_len4(self.node_count()).unwrap_or(0) + } else { + self.node_count().div_ceil(2) + } + } + + /// The arity-4 path of leaf `pos` (type docs): per level from the leaves + /// up, the three other children of its group in child order. `None` past + /// the last leaf. + fn build_merkle_path4(&self, pos: usize) -> Option> { + let sizes = level_sizes4(self.leaves_len()); + if pos >= sizes[0] { + return None; + } + let offsets = level_offsets4(&sizes); + let mut path = Vec::with_capacity(3 * (sizes.len() - 1)); + let mut i = pos; + for level in 0..sizes.len() - 1 { + let first = i / 4 * 4; + for c in (first..first + 4).filter(|&c| c != i) { + path.push(if c < sizes[level] { + self.node_get(offsets[level] + c)?.clone() + } else { + B::padding_node()? + }); + } + i /= 4; + } + Some(path) + } + /// Returns the Merkle path for the element/s for the leaf at position pos fn build_merkle_path(&self, pos: usize) -> Result, Error> { // Pre-allocate based on tree depth (log2 of tree size) @@ -329,6 +331,9 @@ where if pos_list.is_empty() { return Err(Error::EmptyPositionList); } + if B::ARITY != 2 { + return Err(Error::Arity); + } let num_leaves = (self.node_count() + 1).div_ceil(2); @@ -410,23 +415,4 @@ where // This makes the proof ordered from bottom (nodes closer to leaves) to top (nodes loser to root). auth_path_set.into_iter().rev().collect() } - - /// Spill the node vector to a temp-file-backed mmap and free the heap - /// allocation. Node access methods read from the mmap after this call. - #[cfg(feature = "disk-spill")] - pub fn spill_nodes_to_disk(&mut self) -> std::io::Result<()> - where - B::Node: SpillSafe, - { - if self.nodes.is_empty() || self.mmap_backing.is_some() { - return Ok(()); - } - - let node_count = self.nodes.len(); - let mmap = crate::mmap_util::spill_slice_to_mmap(&self.nodes)?; - self.nodes = Vec::new(); - self.mmap_backing = Some(MmapNodeBacking { mmap, node_count }); - - Ok(()) - } } diff --git a/crypto/crypto/src/merkle_tree/mod.rs b/crypto/crypto/src/merkle_tree/mod.rs index 99ea82dea..363ccaa7c 100644 --- a/crypto/crypto/src/merkle_tree/mod.rs +++ b/crypto/crypto/src/merkle_tree/mod.rs @@ -1,4 +1,5 @@ pub mod backends; +pub mod cap; pub mod merkle; pub mod proof; pub mod traits; diff --git a/crypto/crypto/src/merkle_tree/proof.rs b/crypto/crypto/src/merkle_tree/proof.rs index 6534938e0..48e93c972 100644 --- a/crypto/crypto/src/merkle_tree/proof.rs +++ b/crypto/crypto/src/merkle_tree/proof.rs @@ -37,6 +37,27 @@ pub fn verify_merkle_path_from_leaf_hash( where B: IsMerkleTreeBackend, { + if B::ARITY == 4 { + // Three siblings per level, in child order around the node's own slot + // (`MerkleTree`'s arity-4 docs). + if !merkle_path.len().is_multiple_of(3) { + return false; + } + for siblings in merkle_path.chunks_exact(3) { + // The node's own slot, then its three siblings around it in order. + let at = index % 4; + let [s0, s1, s2] = [&siblings[0], &siblings[1], &siblings[2]].map(Clone::clone); + let children = match at { + 0 => [hashed_value, s0, s1, s2], + 1 => [s0, hashed_value, s1, s2], + 2 => [s0, s1, hashed_value, s2], + _ => [s0, s1, s2, hashed_value], + }; + hashed_value = B::hash_four(&children); + index /= 4; + } + return root_hash == &hashed_value; + } for sibling_node in merkle_path.iter() { if index.is_multiple_of(2) { hashed_value = B::hash_new_parent(&hashed_value, sibling_node); diff --git a/crypto/crypto/src/merkle_tree/traits.rs b/crypto/crypto/src/merkle_tree/traits.rs index c09cff9d0..a431c85dc 100644 --- a/crypto/crypto/src/merkle_tree/traits.rs +++ b/crypto/crypto/src/merkle_tree/traits.rs @@ -1,4 +1,7 @@ use alloc::vec::Vec; +use math::field::element::FieldElement; +use math::field::traits::IsField; +use math::traits::AsBytes; #[cfg(feature = "parallel")] use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; @@ -26,4 +29,95 @@ pub trait IsMerkleTreeBackend { /// This function takes to children nodes and builds a new parent node. /// It will be used in the construction of the Merkle tree. fn hash_new_parent(child_1: &Self::Node, child_2: &Self::Node) -> Self::Node; + + /// Children per inner node: 2, or 4 for a backend that overrides + /// [`Self::hash_four`] and [`Self::padding_node`]. A tree's layout, its + /// paths and its caps all follow this constant (`MerkleTree`'s arity-4 + /// docs), so it is a FORMAT constant of the backend. + const ARITY: usize = 2; + + /// An arity-4 parent over its four children in order. Called only when + /// [`Self::ARITY`] is 4; a binary backend's default is the two binary + /// levels over them, so it is defined everywhere. + fn hash_four(children: &[Self::Node; 4]) -> Self::Node { + Self::hash_new_parent( + &Self::hash_new_parent(&children[0], &children[1]), + &Self::hash_new_parent(&children[2], &children[3]), + ) + } + + /// The digest an arity-4 level is padded with up to a multiple of four + /// children (ZisK's rule: zero). `None` for a backend with none (every + /// binary backend): an arity-4 build, path or cap over it is refused. + fn padding_node() -> Option { + None + } +} + +/// A leaf backend that can hash a leaf without being handed one. +/// +/// [`IsMerkleTreeBackend::hash_data`] takes a `&Self::Data`, which for the +/// batched backends is a `Vec>`. Building one per leaf costs an +/// allocation per leaf — millions on a real trace — so the prover and verifier +/// never do: they serialize into a reused buffer, or hold two slices they want +/// hashed as if concatenated. These are the two shapes they use. +/// +/// Both must agree with `hash_data` on the bytes they absorb, so a leaf hashed +/// through either route is the leaf the tree was built from. That is the whole +/// contract, and it is why these live on a trait rather than staying inherent +/// methods on one concrete backend: a commitment configuration that names its +/// leaf backend generically still has to reach them. +pub trait IsStreamingLeafBackend: IsMerkleTreeBackend +where + F: IsField, + FieldElement: AsBytes, +{ + /// Hash a pre-serialized leaf buffer. Equals `hash_data` applied to the + /// elements `data` encodes, in that order. + fn hash_bytes(data: &[u8]) -> Self::Node; + + /// Hash `a ‖ b` without materializing the concatenation. Equals + /// `hash_data(&[a, b].concat())`. + fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> Self::Node; + + /// The incremental form of the same leaf hash. See [`IsLeafHasher`]. + /// + /// `Send` because there is one of these per leaf and the base layer of a real + /// epoch has millions: absorbing them is parallel across leaves, exactly as + /// the one-shot leaf hashing is. + type LeafHasher: IsLeafHasher + Send; + + /// A leaf hasher that has absorbed nothing yet. + fn leaf_hasher() -> Self::LeafHasher; +} + +/// One leaf's hash, absorbed in an arbitrary number of updates. +/// +/// [`IsStreamingLeafBackend::hash_data_from_slices`] covers the two-slice case, +/// which is every leaf the per-table trees hash. A mixed-height MMCS leaf is +/// different: it concatenates one row pair per matrix at that height, and a +/// prover that wants to produce those matrices ONE AT A TIME — absorbing each +/// into the leaves and dropping its buffer — cannot hand over all the slices at +/// once. This is the API that lets it, and the memory it costs is one hasher +/// state per leaf rather than one LDE per matrix. +/// +/// # Contract +/// +/// Splitting is free: for any partition of a leaf's elements into consecutive +/// chunks, updating with each chunk in order and finalizing must equal +/// [`IsMerkleTreeBackend::hash_data`] over the whole. A backend whose framing +/// depended on where the updates fell would produce leaves no verifier could +/// re-derive, since the verifier only ever sees the concatenation. +pub trait IsLeafHasher +where + F: IsField, + FieldElement: AsBytes, +{ + type Node; + + /// Absorb the next consecutive run of the leaf's elements. + fn update(&mut self, data: &[FieldElement]); + + /// Finish the leaf. + fn finalize(self) -> Self::Node; } diff --git a/crypto/crypto/src/merkle_tree/utils.rs b/crypto/crypto/src/merkle_tree/utils.rs index 7cc64166b..978072f60 100644 --- a/crypto/crypto/src/merkle_tree/utils.rs +++ b/crypto/crypto/src/merkle_tree/utils.rs @@ -94,3 +94,89 @@ where level_begin_index = new_level_begin_index; } } + +/// Level sizes of an arity-4 tree over `leaves_len ≥ 1` leaves, leaves first, +/// root (size 1) last: each level holds `⌈below / 4⌉` nodes. Only real nodes +/// are counted; a level whose length is not a multiple of four is padded with +/// the backend's padding digest when its parents are hashed, and those +/// padding children are never stored. +pub fn level_sizes4(leaves_len: usize) -> Vec { + let mut sizes = vec![leaves_len]; + while let Some(&n) = sizes.last().filter(|&&n| n > 1) { + sizes.push(n.div_ceil(4)); + } + sizes +} + +/// Where each level of [`level_sizes4`] starts in an arity-4 tree's node +/// vector, which stores the levels top-down (root at 0, leaves last). +pub fn level_offsets4(sizes: &[usize]) -> Vec { + let mut offsets = vec![0; sizes.len()]; + for j in (0..sizes.len().saturating_sub(1)).rev() { + offsets[j] = offsets[j + 1] + sizes[j + 1]; + } + offsets +} + +/// The leaf count of an arity-4 tree with `total` stored nodes, if `total` is +/// the node count of a tree over a power-of-two number of leaves. The count +/// is strictly increasing in the leaf count, so the answer is unique. +pub fn leaves_len4(total: usize) -> Option { + let mut leaves = 1usize; + loop { + let n: usize = level_sizes4(leaves).iter().sum(); + if n == total { + return Some(leaves); + } + if n > total { + return None; + } + leaves = leaves.checked_mul(2)?; + } +} + +/// Builds every inner level of an arity-4 tree in place: `nodes` holds the +/// levels top-down per [`level_offsets4`], the leaf level already filled. +/// +/// `None` (nothing built) when a level needs padding and the backend has no +/// padding digest ([`IsMerkleTreeBackend::padding_node`]). +pub fn build4(nodes: &mut [B::Node], leaves_len: usize) -> Option<()> +where + B::Node: Clone, +{ + let sizes = level_sizes4(leaves_len); + let offsets = level_offsets4(&sizes); + // A level of `n > 1` nodes not a multiple of four ends in a short group, + // which takes the padding digest; with no short group the placeholder is + // never read. + let pad = match B::padding_node() { + Some(pad) => pad, + None if sizes.iter().any(|&n| n > 1 && !n.is_multiple_of(4)) => return None, + None => nodes.first()?.clone(), + }; + for j in 0..sizes.len() - 1 { + let (above, below) = nodes.split_at_mut(offsets[j]); + let children = &below[..sizes[j]]; + let parents = &mut above[offsets[j + 1]..offsets[j + 1] + sizes[j + 1]]; + let parent_of = |p: usize| { + let child = |c: usize| { + children + .get(4 * p + c) + .cloned() + .unwrap_or_else(|| pad.clone()) + }; + B::hash_four(&[child(0), child(1), child(2), child(3)]) + }; + #[cfg(feature = "parallel")] + parents + .par_iter_mut() + .enumerate() + .for_each(|(p, out)| *out = parent_of(p)); + #[cfg(not(feature = "parallel"))] + parents + .iter_mut() + .enumerate() + .for_each(|(p, out)| *out = parent_of(p)); + } + Some(()) +} diff --git a/crypto/crypto/src/mmap_util.rs b/crypto/crypto/src/mmap_util.rs deleted file mode 100644 index c5600c5d9..000000000 --- a/crypto/crypto/src/mmap_util.rs +++ /dev/null @@ -1,71 +0,0 @@ -use core::slice; -use math::spill_safe::SpillSafe; -use memmap2::{Mmap, MmapOptions}; -use std::fs::File; -use std::io::{Error, ErrorKind, Result}; - -/// Mmap a fresh temp file, copy `slice` into the mapping, downgrade to -/// read-only, and return it. -/// -/// Alignment: the mmap base is page-aligned (>= 4096), this function -/// asserts `align_of::() <= 4096`, and Rust guarantees `size_of::()` -/// is a multiple of `align_of::()`, so every element offset is aligned. -pub fn spill_slice_to_mmap(slice: &[T]) -> Result { - const { - assert!( - align_of::() <= 4096, - "T alignment must fit within mmap page alignment" - ) - } - - let elem_size = size_of::(); - let total_bytes = (slice.len() as u64) - .checked_mul(elem_size as u64) - .ok_or_else(|| { - Error::new( - ErrorKind::InvalidInput, - "spill_slice_to_mmap: byte count overflows u64", - ) - })?; - - let file = tempfile::tempfile()?; - reserve_file_blocks(&file, total_bytes)?; - - // SAFETY: tempfile() creates an anonymous file with no filesystem path, - // so no other process can open or modify it. - let mut mmap_mut = unsafe { MmapOptions::new().map_mut(&file)? }; - // SAFETY: SpillSafe's safety contract requires no padding on T, so - // `slice`'s bytes are initialized and reading them as &[u8] is sound. - let bytes: &[u8] = - unsafe { slice::from_raw_parts(slice.as_ptr() as *const u8, size_of_val(slice)) }; - mmap_mut.copy_from_slice(bytes); - mmap_mut.make_read_only() -} - -/// Reserve disk blocks up front so this call fails on a full disk. -/// Without reservation, the kernel sends SIGBUS during the later mmap write. -/// -/// Linux only, using `posix_fallocate`. On other platforms we only call -/// `set_len` and skip reservation, so the kernel can still send SIGBUS if -/// the disk fills mid-write. -/// -/// `/tmp` is often tmpfs (RAM-backed) on systemd-default distros; set -/// `TMPDIR` to a disk-backed path so spill files actually live on disk. -fn reserve_file_blocks(file: &File, total_bytes: u64) -> Result<()> { - file.set_len(total_bytes)?; - #[cfg(target_os = "linux")] - { - use std::os::unix::io::AsRawFd; - let len = i64::try_from(total_bytes).map_err(|_| { - Error::new( - ErrorKind::InvalidInput, - "spill file too large for posix_fallocate", - ) - })?; - let ret = unsafe { libc::posix_fallocate(file.as_raw_fd(), 0, len) }; - if ret != 0 { - return Err(Error::from_raw_os_error(ret)); - } - } - Ok(()) -} diff --git a/crypto/crypto/src/tests/blake3_transcript_tests.rs b/crypto/crypto/src/tests/blake3_transcript_tests.rs new file mode 100644 index 000000000..78523120d --- /dev/null +++ b/crypto/crypto/src/tests/blake3_transcript_tests.rs @@ -0,0 +1,268 @@ +//! The BLAKE3 Fiat-Shamir configuration, and rider 1's fixed consumption +//! schedule. +//! +//! Two things are under test and they are independent: that the transcript's +//! sponge is `Blake3Chain` in the framing the keccak one has always used, and +//! that a field draw under this configuration consumes a *fixed* number of +//! candidates. The second is what a straight-line machine needs — the LFM +//! transcript replay encodes one consumption schedule, so a draw whose count +//! varies with the bytes it happened to see is unprovable against it. + +use alloc::vec::Vec; +use digest::Digest; +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField, + goldilocks::GoldilocksField, traits::HasDefaultTranscript, +}; + +use crate::fiat_shamir::default_transcript::{ + Blake3Transcript, DefaultTranscript, candidate_under_fixed_schedule, +}; +use crate::fiat_shamir::is_transcript::IsTranscript; +use crate::fiat_shamir::transcript_hash::{ + Blake3TranscriptHash, KeccakTranscriptHash, TranscriptHash, +}; +use crate::hash::blake3::chain::Blake3Chain; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; + +const SEED: &[u8] = b"lambda-vm-blake3-transcript-kat-v1"; + +// ========================================================================= +// The sponge is Blake3Chain, in the framing the keccak transcript uses. +// ========================================================================= + +/// ★ Anchor: the transcript's state is the chain hash of exactly what was +/// absorbed. +/// +/// `state()` is not an internal detail — it is the grinding seed. Pinning it +/// against `Blake3Chain` computed directly is what says the transcript absorbs +/// what it claims to, and `Blake3Chain` is in turn anchored from outside (its +/// 7-round arm reproduces the `blake3` crate at all 1025 lengths ≤ 1 chunk, and +/// its 6-round arm has the committed `CHAIN_KAT_6ROUND` table). +#[test] +fn the_blake3_transcript_state_is_the_chain_of_what_was_absorbed() { + let mut t = Blake3Transcript::::new(SEED); + assert_eq!( + t.state(), + <[u8; 32]>::from(Blake3Chain::digest(SEED)), + "a fresh transcript's state must be the chain hash of its seed" + ); + + t.append_bytes(b"a-merkle-root"); + let mut expected = Vec::from(SEED); + expected.extend_from_slice(b"a-merkle-root"); + assert_eq!( + t.state(), + <[u8; 32]>::from(Blake3Chain::digest(&expected)), + "absorbing must concatenate into the same chain, not reset it" + ); +} + +/// The duplex squeeze is the same construction under the new digest: finalize +/// **and reset**, reverse, absorb the reversed output. +/// +/// Reimplemented here rather than compared against itself — this is the one +/// place where a hash swap could silently drop the reverse-and-reabsorb step, +/// and prover and verifier would still agree with each other while producing a +/// transcript nobody else can reproduce. +/// +/// Note the reset: the squeeze is `finalize_reset`, so squeeze `k+1` hashes the +/// reversed output of squeeze `k` **alone**, not the whole absorbed history. +/// That is what makes the sponge a chain rather than a growing buffer, and it +/// is the detail this test exists to pin. +#[test] +fn the_blake3_squeeze_chain_matches_the_construction() { + let mut t = Blake3Transcript::::new(SEED); + + let mut pending = Vec::from(SEED); + for k in 0..3 { + let mut expected = <[u8; 32]>::from(Blake3Chain::digest(&pending)); + expected.reverse(); + assert_eq!(t.sample(), expected, "squeeze {k}"); + pending = Vec::from(expected); + } +} + +/// CONTROL: the two configurations are actually different transcripts. +/// +/// Without this, every test here would pass just as well if `Blake3Transcript` +/// had been left resolving to keccak. +#[test] +fn the_blake3_and_keccak_transcripts_diverge() { + let mut blake3 = Blake3Transcript::::new(SEED); + let mut keccak = DefaultTranscript::::new(SEED); + assert_ne!(blake3.state(), keccak.state()); + assert_ne!(blake3.sample(), keccak.sample()); +} + +// ========================================================================= +// Rider 1 — the consumption schedule. +// ========================================================================= + +/// The configurations' schedules, as a fact rather than as prose. +/// +/// The keccak arm MUST stay `None`. Rider 1 is adopted for BLAKE3 only, because +/// changing the keccak schedule would move every existing proof's challenges — +/// the one thing P-a's staging holds still until the flip. +#[test] +fn only_the_blake3_configuration_takes_the_fixed_schedule() { + assert!( + KeccakTranscriptHash::CANDIDATES_PER_COORDINATE.is_none(), + "the keccak schedule must not move before the flip" + ); + assert_eq!( + Blake3TranscriptHash::CANDIDATES_PER_COORDINATE.map(|n| n.get()), + Some(2) + ); +} + +/// ★ Rider 1's whole content: the draw consumes exactly `n` candidates, +/// wherever the acceptable one sits — including when there is none. +/// +/// Counting the calls is the only way to see this; the returned value cannot +/// distinguish "took the first and stopped" from "took the first and kept +/// drawing", and it is the *stopping* that a straight-line machine cannot +/// follow. +#[test] +fn a_fixed_schedule_draw_consumes_exactly_n_candidates() { + // `p = 2^64 - 2^32 + 1`, so anything ≥ p is rejected. `u64::MAX` is. + let out_of_range = u64::MAX; + assert!(!F::candidate_in_range(out_of_range)); + let in_range = [7u64, 11, 13, 17]; + for c in in_range { + assert!(F::candidate_in_range(c)); + } + + for n in 1..=4usize { + for hit in 0..n { + // A stream whose only in-range value sits at position `hit`. + let stream: Vec = (0..n) + .map(|i| if i == hit { in_range[0] } else { out_of_range }) + .collect(); + let mut calls = 0usize; + let mut it = stream.iter(); + let got = candidate_under_fixed_schedule::(n, || { + calls += 1; + *it.next().expect("the schedule must not overdraw") + }); + assert_eq!( + calls, n, + "n={n}, acceptable candidate at {hit}: the draw must consume exactly n" + ); + assert_eq!(got, in_range[0], "it must return the acceptable candidate"); + } + + // No acceptable candidate: still exactly `n`, and the value handed back + // is one the field rejects, so its own loop draws another full `n`. + let mut calls = 0usize; + let got = candidate_under_fixed_schedule::(n, || { + calls += 1; + out_of_range + }); + assert_eq!(calls, n, "n={n}, no acceptable candidate: still exactly n"); + assert!( + !F::candidate_in_range(got), + "the fallback must NOT be reduced into range — a modular fallback \ + would bias challenges by ~2^-32, which at this system's security \ + level would dominate the soundness error" + ); + } +} + +/// ★ The schedule as the transcript actually runs it: an extension-field draw +/// takes SIX candidates from the squeeze stream, two per coordinate. +/// +/// Reconstructed from the raw squeezes, so it distinguishes the fixed schedule +/// from the unbounded one: under `None` the coordinates would be candidates +/// 0, 1, 2 of the stream, and under `Some(2)` they are the first acceptable of +/// (0,1), (2,3), (4,5). +#[test] +fn an_extension_draw_consumes_two_candidates_per_coordinate() { + // The raw candidate stream this transcript will hand out, taken from a + // clone so the transcript under test is untouched. + let candidates: Vec = { + let mut probe = Blake3Transcript::::new(SEED); + let mut out = Vec::new(); + for _ in 0..2 { + let squeeze = probe.sample(); + for chunk in squeeze.chunks_exact(8) { + out.push(u64::from_be_bytes(chunk.try_into().unwrap())); + } + } + out + }; + assert_eq!(candidates.len(), 8); + + let pick = |a: u64, b: u64| { + if F::candidate_in_range(a) { a } else { b } + }; + let expected = [ + pick(candidates[0], candidates[1]), + pick(candidates[2], candidates[3]), + pick(candidates[4], candidates[5]), + ]; + + let mut t = Blake3Transcript::::new(SEED); + let drawn = t.sample_field_element(); + let coords: Vec = drawn.value().iter().map(|c| *c.value()).collect(); + assert_eq!( + coords, + expected.to_vec(), + "each coordinate must be the first acceptable of its OWN pair" + ); + + // NEGATIVE CONTROL: it is not the unbounded schedule, which would take one + // candidate per coordinate and so read 0, 1, 2. + let unbounded = [candidates[0], candidates[1], candidates[2]]; + assert_ne!( + coords, + unbounded.to_vec(), + "the fixed schedule must be distinguishable from the unbounded one" + ); +} + +/// The keccak configuration still draws one candidate per coordinate. +/// +/// The honest-path partner of the test above: it says the branch is a branch, +/// and that the default side of it did not move. +#[test] +fn the_keccak_extension_draw_still_takes_one_candidate_per_coordinate() { + let candidates: Vec = { + let mut probe = DefaultTranscript::::new(SEED); + let squeeze = probe.sample(); + squeeze + .chunks_exact(8) + .map(|c| u64::from_be_bytes(c.try_into().unwrap())) + .collect() + }; + + let mut t = DefaultTranscript::::new(SEED); + let coords: Vec = t + .sample_field_element() + .value() + .iter() + .map(|c| *c.value()) + .collect(); + assert_eq!( + coords, + candidates[..3].to_vec(), + "the keccak draw must still be one candidate per coordinate" + ); +} + +/// A transcript is still deterministic under the fixed schedule — the property +/// every replaying verifier depends on. +#[test] +fn the_blake3_transcript_replays_identically() { + let mut a = Blake3Transcript::::new(SEED); + let mut b = Blake3Transcript::::new(SEED); + a.append_bytes(b"round-1"); + b.append_bytes(b"round-1"); + + let draw_a: Vec> = (0..8).map(|_| a.sample_field_element()).collect(); + let draw_b: Vec> = (0..8).map(|_| b.sample_field_element()).collect(); + assert_eq!(draw_a, draw_b); + assert_eq!(a.sample_u64(1 << 20), b.sample_u64(1 << 20)); +} diff --git a/crypto/crypto/src/tests/grinding_determinism_tests.rs b/crypto/crypto/src/tests/grinding_determinism_tests.rs new file mode 100644 index 000000000..cbf477a30 --- /dev/null +++ b/crypto/crypto/src/tests/grinding_determinism_tests.rs @@ -0,0 +1,129 @@ +//! ★ The reproducible nonce search, and the reason it had to exist. +//! +//! `generate_nonce` returns *a* valid nonce. Which one is not a contract, and +//! under `parallel` it is rayon's `find_any` — whichever worker got there +//! first. That would be harmless if the nonce were merely recorded, but it is +//! **absorbed into the transcript**, so every challenge drawn after the first +//! grind depends on it. Two honest runs of the same prover therefore produce +//! different Merkle roots, different out-of-domain values and different +//! openings, and no normalisation of the nonce fields can undo that, because +//! the divergence is not in those fields. +//! +//! [`generate_nonce_smallest`] is the fix a byte gate needs: the smallest valid +//! nonce is a function of the seed and the factor alone. These tests pin the +//! three things that makes it worth anything — it is reproducible, it really is +//! the smallest, and it is valid — and they are written so each can fail. + +use digest::Digest; + +use crate::grinding::{generate_nonce, generate_nonce_smallest, is_valid_nonce}; +use crate::hash::platform_keccak::PlatformKeccak256 as Keccak; + +/// A factor small enough that the exhaustive scan below is instant and large +/// enough that the answer is not zero on most seeds. +const FACTOR: u8 = 12; + +fn seed(tag: u8) -> [u8; 32] { + let mut out = [0u8; 32]; + out[0] = tag; + for (i, b) in out.iter_mut().enumerate().skip(1) { + *b = (i as u8).wrapping_mul(37).wrapping_add(tag); + } + out +} + +/// It is reproducible: the same inputs give the same nonce. +#[test] +fn the_smallest_nonce_is_reproducible() { + for tag in 0..4u8 { + let s = seed(tag); + let a = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + let b = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + assert_eq!(a, b, "seed {tag}: the smallest nonce must not vary"); + } +} + +/// ★ It really is the smallest — checked against an exhaustive scan, which is a +/// different algorithm from the one under test. +/// +/// This is the assertion that can fail if `find_first` is ever swapped back to +/// `find_any` for speed, which is exactly the regression the deterministic knob +/// exists to prevent. +#[test] +fn it_is_the_smallest_valid_nonce_and_not_merely_a_valid_one() { + // The exhaustive loop below is vacuous when the answer is zero, so at least + // one seed has to land above it for the test to be testing anything. + let mut scanned = 0u64; + for tag in 0..4u8 { + let s = seed(tag); + let n = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + scanned += n; + + assert!( + is_valid_nonce::(&s, n, FACTOR), + "seed {tag}: the chosen nonce {n} does not pass the verifier's own check" + ); + assert!( + (0..n).all(|candidate| !is_valid_nonce::(&s, candidate, FACTOR)), + "seed {tag}: a nonce below {n} is also valid, so {n} is not the smallest" + ); + } + assert!( + scanned > 0, + "every seed's smallest nonce was zero, so the minimality scan never ran" + ); +} + +/// The unpinned search is still correct — it just promises less. +/// +/// Stated as "valid, and never below the smallest" rather than "different": +/// asserting a difference would be a coin flip, and a test that fails at random +/// teaches nothing. +#[test] +fn the_unpinned_search_returns_a_valid_nonce_no_smaller_than_the_smallest() { + for tag in 0..4u8 { + let s = seed(tag); + let smallest = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + let any = generate_nonce::(&s, FACTOR).expect("a nonce exists"); + + assert!( + is_valid_nonce::(&s, any, FACTOR), + "seed {tag}: the unpinned search returned an invalid nonce" + ); + assert!( + any >= smallest, + "seed {tag}: {any} is below the exhaustively-checked smallest {smallest}" + ); + } +} + +/// The construction itself, against the spec in the module doc: the outer hash +/// of `inner ‖ nonce` must have `FACTOR` leading zero bits. +/// +/// An independent reading of the same predicate — `is_valid_nonce` compares a +/// big-endian `u64` against a limit; this counts the bits — so the two cannot +/// be one transcription of the other. +#[test] +fn a_valid_nonce_really_does_have_the_leading_zeros() { + let s = seed(1); + let n = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + + // Rebuild the inner hash the way the module documents it. + const PREFIX: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xed]; + let mut inner_data = [0u8; 41]; + inner_data[0..8].copy_from_slice(&PREFIX); + inner_data[8..40].copy_from_slice(&s); + inner_data[40] = FACTOR; + let inner = Keccak::digest(inner_data); + + let mut outer_data = [0u8; 40]; + outer_data[..32].copy_from_slice(&inner); + outer_data[32..].copy_from_slice(&n.to_be_bytes()); + let outer = Keccak::digest(outer_data); + + let leading = u64::from_be_bytes(outer[..8].try_into().unwrap()).leading_zeros(); + assert!( + leading >= FACTOR as u32, + "nonce {n} gives only {leading} leading zero bits, needed {FACTOR}" + ); +} diff --git a/crypto/crypto/src/tests/hash_metrics_tests.rs b/crypto/crypto/src/tests/hash_metrics_tests.rs new file mode 100644 index 000000000..2b82dedb1 --- /dev/null +++ b/crypto/crypto/src/tests/hash_metrics_tests.rs @@ -0,0 +1,148 @@ +//! ★ The verify-hash counters count BOTH hash families — the test that would +//! have caught the false zero. +//! +//! `hash_metrics` was keccak-only: `count_merkle` compared `TypeId::of::()` +//! against `PlatformKeccak256` and did nothing otherwise. That was correct while +//! keccak was the only hash on the multilinear path, and it became a check that +//! cannot fail the moment a second one arrived — under RPX every Merkle counter +//! would have read ZERO, reporting "no hashing" for precisely the arm whose +//! purpose is to change the hashing, and a reader comparing the two arms would +//! have concluded the algebraic hash was free. +//! +//! So these tests are written the way that failure would have been caught: +//! **non-zero after the RPX arm, non-zero after the keccak arm, and the subset +//! invariant intact for both.** Every one of them fails on the pre-extension +//! code. +//! +//! ⚠ These run only under `--features hash-metrics`. That is the same reason the +//! module exists at all — the counters compile to nothing otherwise — but it +//! does mean a default `cargo test` does not execute them. `make lint`'s passes +//! do not enable the feature either; the gate is `cargo test -p crypto +//! --features hash-metrics`. + +#![cfg(feature = "hash-metrics")] + +use alloc::vec::Vec; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as Fp; + +use crate::hash_metrics::{Counts, reset, snapshot}; +use crate::merkle_tree::backends::rpx::RpxVectorBackend; +use crate::merkle_tree::backends::types::BatchKeccak256Backend; +use crate::merkle_tree::traits::IsMerkleTreeBackend; + +type Rpx = RpxVectorBackend; +type Keccak = BatchKeccak256Backend; + +/// The counters are process-global, so the cases take turns rather than +/// running concurrently. A mutex rather than `--test-threads=1`, so the +/// property does not depend on how the suite is invoked. +/// +/// ⚠ Poisoning is IGNORED, and that is not laziness. A failing case panics +/// while holding this lock, and `unwrap()` would then panic every later case on +/// the poisoned mutex — turning one real failure into five, four of them +/// cascades. That was observed while mutation-testing this file: removing the +/// RPX backend's counter calls failed the two cases that assert it AND the +/// keccak case, which is a lie about the keccak path. The counters are reset at +/// the top of every measurement, so a poisoned lock carries no stale state. +static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serialise() -> std::sync::MutexGuard<'static, ()> { + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +fn leaf(n: usize) -> Vec> { + (0..n as u64) + .map(|i| FieldElement::from(i * 7 + 1)) + .collect() +} + +/// Hash one leaf and one parent under `B`, and report what the counters saw. +fn measure() -> Counts +where + B: IsMerkleTreeBackend>>, +{ + reset(); + let a = B::hash_data(&leaf(5)); + let b = B::hash_data(&leaf(6)); + let _ = B::hash_new_parent(&a, &b); + snapshot() +} + +/// ★★ The RPX arm is COUNTED — the proposition the extension exists for. +#[test] +fn the_algebraic_backend_is_counted() { + let _g = serialise(); + let c = measure::(); + + assert_eq!(c.merkle, 3, "two leaves and one parent"); + assert_eq!(c.merkle_nodes, 1, "one parent"); + assert_eq!( + c.merkle - c.merkle_nodes, + 2, + "merkle - nodes must be the leaf count, as it is on the byte path" + ); + assert!( + c.total >= c.merkle, + "total {} must cover merkle {}", + c.total, + c.merkle + ); +} + +/// The keccak arm still is, unchanged — so the extension did not move the +/// number the existing instrument reports. +#[test] +fn the_byte_backend_is_still_counted() { + let _g = serialise(); + let c = measure::(); + + assert_eq!(c.merkle, 3, "two leaves and one parent"); + assert_eq!(c.merkle_nodes, 1, "one parent"); + assert!(c.total >= c.merkle); +} + +/// ★ The two arms agree on the COUNT while differing in the hash — which is +/// what makes a cross-arm comparison of these numbers meaningful at all. +/// +/// If they disagreed, a difference in the counters would not distinguish "this +/// hash does more work" from "this backend is instrumented differently". +#[test] +fn the_two_backends_report_the_same_shape_for_the_same_tree() { + let _g = serialise(); + let rpx = measure::(); + let keccak = measure::(); + + assert_eq!(rpx.merkle, keccak.merkle); + assert_eq!(rpx.merkle_nodes, keccak.merkle_nodes); +} + +/// ⚠ **The subset invariant, on the path that could break it.** `merkle` must +/// never exceed `total`: the byte backends get `total` from the digest's own +/// `finalize`, and the algebraic backend has no digest to finalize, so it owes +/// the bump itself. A `count_merkle_direct` that forgot `TOTAL` would land here. +#[test] +fn merkle_never_exceeds_total_on_the_algebraic_path() { + let _g = serialise(); + reset(); + for n in 0..16 { + let _ = Rpx::hash_data(&leaf(n)); + } + let c = snapshot(); + assert_eq!(c.merkle, 16); + assert!( + c.total >= c.merkle, + "total {} < merkle {} — the algebraic path did not bump total", + c.total, + c.merkle + ); +} + +/// ✓ `reset` really resets, so one case cannot read another's counts. +#[test] +fn reset_clears_every_counter() { + let _g = serialise(); + let _ = Rpx::hash_data(&leaf(4)); + reset(); + assert_eq!(snapshot(), Counts::default()); +} diff --git a/crypto/crypto/src/tests/merkle_arity4_tests.rs b/crypto/crypto/src/tests/merkle_arity4_tests.rs new file mode 100644 index 000000000..89a68f3d1 --- /dev/null +++ b/crypto/crypto/src/tests/merkle_arity4_tests.rs @@ -0,0 +1,449 @@ +//! `MerkleTree` at arity 4 against the Poseidon1 host oracle +//! ([`crate::hash::poseidon1_stark::Merkle4`], whose roots and paths ZisK's own +//! code reproduces), and the binary layout's invariants left alone. + +use alloc::vec::Vec; + +use crate::hash::poseidon1_stark::{self as p1, Merkle4}; +use crate::hash::poseidon1_w16::{Digest, Fp}; +use crate::merkle_tree::merkle::MerkleTree; +use crate::merkle_tree::proof::verify_merkle_path_from_leaf_hash; +use crate::merkle_tree::traits::IsMerkleTreeBackend; +use crate::merkle_tree::utils::{leaves_len4, level_sizes4}; + +/// ZisK's leaf hash and 4-ary node over canonical digests. +struct P1Backend; + +fn canon(d: &Digest) -> [u64; 4] { + core::array::from_fn(|i| d[i].canonical()) +} + +fn digest(n: &[u64; 4]) -> Digest { + core::array::from_fn(|i| Fp::from(n[i])) +} + +impl IsMerkleTreeBackend for P1Backend { + type Node = [u64; 4]; + type Data = Vec; + const ARITY: usize = 4; + + fn hash_data(leaf: &Vec) -> [u64; 4] { + canon(&p1::linear_hash(leaf)) + } + + fn hash_new_parent(left: &[u64; 4], right: &[u64; 4]) -> [u64; 4] { + Self::hash_four(&[*left, *right, [0; 4], [0; 4]]) + } + + fn hash_four(children: &[[u64; 4]; 4]) -> [u64; 4] { + canon(&crate::hash::poseidon1_w16::compress4( + &children.map(|c| digest(&c)), + )) + } + + fn padding_node() -> Option<[u64; 4]> { + Some([0; 4]) + } +} + +fn rows(n: usize) -> Vec> { + (0..n) + .map(|r| { + (0..5) + .map(|c| Fp::from((r * 7 + c) as u64 * 0x9E37_79B9)) + .collect() + }) + .collect() +} + +#[test] +fn arity_4_trees_match_the_oracle_root_paths_and_caps() { + for k in 0..=9 { + let n = 1usize << k; + let data = rows(n); + let tree = MerkleTree::::build(&data).expect("non-empty"); + let leaves: Vec = data.iter().map(|r| p1::linear_hash(r)).collect(); + let oracle = Merkle4::new(&leaves).expect("non-empty"); + assert_eq!(tree.root, canon(&oracle.root()), "{n} leaves"); + assert_eq!(tree.depth(), Some(oracle.depth()), "{n} leaves"); + for pos in [0, n / 3, n - 1] { + let proof = tree.get_proof_by_pos(pos).expect("in range"); + let want: Vec<[u64; 4]> = oracle + .path(pos) + .expect("in range") + .iter() + .flat_map(|level| { + level + .chunks_exact(4) + .map(|d| canon(&[d[0], d[1], d[2], d[3]])) + .collect::>() + }) + .collect(); + assert_eq!(proof.merkle_path, want, "{n} leaves, pos {pos}"); + assert!(proof.verify::(&tree.root, pos, &data[pos])); + } + assert!(tree.get_proof_by_pos(n).is_none()); + // The cap one level down is the oracle's level below the root. + if let Some(depth) = tree.depth().filter(|&d| d >= 1) { + let cap = tree.cap(1).expect("depth ≥ 1"); + assert!( + cap.len() == 4 || (cap.len() == 2 && k % 2 == 1), + "{n} leaves: cap {}", + cap.len() + ); + assert_eq!(tree.cap(0), Some(alloc::vec![tree.root])); + assert_eq!(tree.cap(depth).map(|c| c.len()), Some(n)); + assert!(tree.cap(depth + 1).is_none()); + } + } +} + +#[test] +fn an_arity_4_tree_round_trips_through_its_node_vector() { + for k in 0..=8 { + let tree = MerkleTree::::build(&rows(1 << k)).expect("non-empty"); + let nodes = tree.nodes().to_vec(); + assert_eq!(nodes.len(), level_sizes4(1 << k).iter().sum::()); + assert_eq!(leaves_len4(nodes.len()), Some(1 << k)); + let back = MerkleTree::::from_precomputed_nodes(nodes).expect("a layout"); + assert_eq!(back.root, tree.root); + assert_eq!( + back.get_proof_by_pos(0).map(|p| p.merkle_path), + tree.get_proof_by_pos(0).map(|p| p.merkle_path) + ); + } + // A node count no power-of-two tree has. + assert!(MerkleTree::::from_precomputed_nodes(alloc::vec![[0; 4]; 4]).is_none()); + assert!(MerkleTree::::from_precomputed_nodes(alloc::vec![[0; 4]; 7]).is_none()); +} + +#[test] +fn a_tampered_arity_4_path_is_rejected() { + let data = rows(32); + let tree = MerkleTree::::build(&data).expect("non-empty"); + for pos in [0usize, 13, 31] { + let path = tree.get_proof_by_pos(pos).expect("in range").merkle_path; + let leaf = P1Backend::hash_data(&data[pos]); + assert!(verify_merkle_path_from_leaf_hash::( + &path, &tree.root, pos, leaf + )); + for i in 0..path.len() { + let mut bad = path.clone(); + bad[i][i % 4] ^= 1; + assert!( + !verify_merkle_path_from_leaf_hash::(&bad, &tree.root, pos, leaf), + "sibling {i}" + ); + } + let mut leaf_bad = leaf; + leaf_bad[0] ^= 1; + assert!(!verify_merkle_path_from_leaf_hash::( + &path, &tree.root, pos, leaf_bad + )); + assert!(!verify_merkle_path_from_leaf_hash::( + &path, + &tree.root, + pos ^ 1, + leaf + )); + // A path that is not whole levels, and one a level short. + assert!(!verify_merkle_path_from_leaf_hash::( + &path[..path.len() - 1], + &tree.root, + pos, + leaf + )); + assert!(!verify_merkle_path_from_leaf_hash::( + &path[..path.len() - 3], + &tree.root, + pos, + leaf + )); + } + assert!(tree.get_batch_proof(&[0, 1]).is_err()); +} + +#[test] +fn arity_4_cap_lengths_are_pinned() { + use crate::merkle_tree::cap::{MAX_CAP_HEIGHT, cap_len, tree_levels}; + // Odd depth: the top group is two real nodes and two paddings. + assert_eq!(tree_levels(3, 4), 2); + assert_eq!( + (0..=2).map(|c| cap_len(3, c, 4)).collect::>(), + [Some(1), Some(2), Some(8)] + ); + assert_eq!(cap_len(3, 3, 4), None); + // Even depth: powers of four. + assert_eq!(tree_levels(4, 4), 2); + assert_eq!( + (0..=2).map(|c| cap_len(4, c, 4)).collect::>(), + [Some(1), Some(4), Some(16)] + ); + assert_eq!(cap_len(22, 4, 4), Some(256)); + assert_eq!(cap_len(21, 4, 4), Some(128)); + // No cap holds more than 2^MAX_CAP_HEIGHT nodes; the binary rule is unchanged. + assert_eq!( + cap_len(40, MAX_CAP_HEIGHT / 2, 4), + Some(1 << MAX_CAP_HEIGHT) + ); + assert_eq!(cap_len(40, MAX_CAP_HEIGHT / 2 + 1, 4), None); + assert_eq!(cap_len(22, 3, 2), Some(8)); + assert_eq!(cap_len(2, 3, 2), None); +} + +/// Every opening of every leaf position class, at every cap height of trees +/// over `2^k` leaves, `k` odd and even: the cap is the tree's level, hashes to +/// the root, rides on the owner path and every opening verifies against it. +#[test] +fn arity_4_caps_round_trip_at_every_height() { + use crate::merkle_tree::cap::{ + CappedRoot, cap_len, cap_root, embed_cap_arity, split_owner_path_arity, tree_levels, + }; + for k in 0..=9usize { + let n = 1usize << k; + let data = rows(n); + let tree = MerkleTree::::build(&data).expect("non-empty"); + let h = tree_levels(k, 4); + for c in 0..=h { + let cap = tree.cap(c).expect("c ≤ h"); + assert_eq!(Some(cap.len()), cap_len(k, c, 4), "k {k} c {c}"); + assert_eq!(cap_root::(&cap), Some(tree.root), "k {k} c {c}"); + let positions: Vec = [0, n / 3, n / 2, n - 1].into_iter().collect(); + let mut paths: Vec> = positions + .iter() + .map(|&p| tree.get_proof_by_pos(p).expect("in range").merkle_path) + .collect(); + let mut refs: Vec<&mut Vec<[u64; 4]>> = paths.iter_mut().collect(); + embed_cap_arity(&mut refs, k, &cap, 4).expect("full paths"); + let keep = 3 * (h - c); + let owner_len = if c == 0 { keep } else { keep + cap.len() }; + assert_eq!(paths[0].len(), owner_len, "k {k} c {c}"); + let (split_siblings, split_cap) = + split_owner_path_arity(&paths[0], k, c, 4).expect("owner length"); + assert_eq!(split_siblings.len(), keep); + if c > 0 { + assert_eq!(split_cap, &cap[..]); + } + let (check, owner) = + CappedRoot::from_owner::(&tree.root, &paths[0], k, c).expect("honest"); + let root_cap = [tree.root]; + assert_eq!(check.cap(), if c == 0 { &root_cap[..] } else { &cap[..] }); + for (q, &pos) in positions.iter().enumerate() { + let siblings = if q == 0 { owner } else { &paths[q][..] }; + assert_eq!(siblings.len(), keep, "k {k} c {c} q {q}"); + let leaf = P1Backend::hash_data(&data[pos]); + assert!( + check.verify::(siblings, pos, leaf), + "k {k} c {c} pos {pos}" + ); + } + } + assert!(tree.cap(h + 1).is_none()); + } +} + +/// A tree, its rows, the owner path of one opening (cap embedded), the binary +/// depth, the cap height and the opened position. +type CappedFixture = ( + MerkleTree, + Vec>, + Vec<[u64; 4]>, + usize, + usize, + usize, +); + +/// The whole capped check of one owner opening: `(root, owner path, depth, c, +/// index, leaf hash)`. +type AcceptOwner = fn(&[u64; 4], &[[u64; 4]], usize, usize, usize, [u64; 4]) -> bool; + +/// [`crate::merkle_tree::cap::verify_merkle_path_to_cap_from_leaf_hash`]'s shape. +type VerifyToCap = fn(&[[u64; 4]], &[[u64; 4]], usize, usize, [u64; 4]) -> bool; + +/// One capped tree (`k = 9`, `h = 5`, `c = 3`: 32 cap nodes, 6 kept siblings) +/// and one honest opening, for the tampers below. +fn capped_fixture() -> CappedFixture { + use crate::merkle_tree::cap::embed_cap_arity; + let (k, c, pos) = (9usize, 3usize, 300usize); + let data = rows(1 << k); + let tree = MerkleTree::::build(&data).expect("non-empty"); + let mut owner = tree.get_proof_by_pos(pos).expect("in range").merkle_path; + embed_cap_arity(&mut [&mut owner], k, &tree.cap(c).expect("c ≤ h"), 4).expect("full"); + (tree, data, owner, k, c, pos) +} + +#[test] +fn a_tampered_capped_arity_4_opening_is_rejected() { + use crate::merkle_tree::cap::CappedRoot; + let (tree, data, owner, k, c, pos) = capped_fixture(); + let leaf = P1Backend::hash_data(&data[pos]); + let (check, siblings) = + CappedRoot::from_owner::(&tree.root, &owner, k, c).expect("honest"); + assert!(check.verify::(siblings, pos, leaf)); + let keep = siblings.len(); + // Every cap node: the cap no longer hashes to the root. + for i in keep..owner.len() { + let mut bad = owner.clone(); + bad[i][i % 4] ^= 1; + assert!( + CappedRoot::from_owner::(&tree.root, &bad, k, c).is_none(), + "cap node {}", + i - keep + ); + } + // Every kept sibling. + for i in 0..keep { + let mut bad = siblings.to_vec(); + bad[i][0] ^= 1; + assert!(!check.verify::(&bad, pos, leaf), "sibling {i}"); + } + // The owner path one node long or short, and at another height. + let mut long = owner.clone(); + long.push([0; 4]); + assert!(CappedRoot::from_owner::(&tree.root, &long, k, c).is_none()); + assert!( + CappedRoot::from_owner::(&tree.root, &owner[..owner.len() - 1], k, c).is_none() + ); + assert!(CappedRoot::from_owner::(&tree.root, &owner, k, c - 1).is_none()); + assert!(CappedRoot::from_owner::(&tree.root, &owner, k, c + 1).is_none()); + // An opening a node short or long, another index, an index past the tree, + // another leaf. + assert!(!check.verify::(&siblings[..keep - 1], pos, leaf)); + assert!(!check.verify::(&[siblings, &[[0; 4]]].concat(), pos, leaf)); + assert!(!check.verify::(siblings, pos ^ 1, leaf)); + assert!(!check.verify::(siblings, pos + (1 << k), leaf)); + assert!(!check.verify::(siblings, pos, P1Backend::hash_data(&data[pos + 1]))); +} + +/// The owner path of a FORGED tree's opening, carried next to the honest +/// root: its cap is consistent with its own siblings, so only the cap-to-root +/// check stands between it and acceptance. `accept` is the whole capped check +/// of one owner opening: `(root, owner path, depth, c, index, leaf hash)`. +fn admits_forged_cap(accept: AcceptOwner) -> bool { + use crate::merkle_tree::cap::embed_cap_arity; + let (honest, _, _, k, c, pos) = capped_fixture(); + let forged_data: Vec> = rows((1 << k) + 1)[1..].to_vec(); + let forged = MerkleTree::::build(&forged_data).expect("non-empty"); + assert_ne!(forged.root, honest.root); + let mut owner = forged.get_proof_by_pos(pos).expect("in range").merkle_path; + embed_cap_arity(&mut [&mut owner], k, &forged.cap(c).expect("c ≤ h"), 4).expect("full"); + accept( + &honest.root, + &owner, + k, + c, + pos, + P1Backend::hash_data(&forged_data[pos]), + ) +} + +#[test] +fn the_cap_to_root_check_rejects_a_forged_arity_4_cap() { + assert!(!admits_forged_cap(|root, owner, d, c, pos, leaf| { + crate::merkle_tree::cap::CappedRoot::from_owner::(root, owner, d, c) + .is_some_and(|(check, siblings)| check.verify::(siblings, pos, leaf)) + })); +} + +#[test] +fn mutation_without_the_cap_to_root_check_admits_a_forged_arity_4_cap() { + assert!(admits_forged_cap(|_root, owner, d, c, pos, leaf| { + use crate::merkle_tree::cap::{ + split_owner_path_arity, verify_merkle_path_to_cap_from_leaf_hash, + }; + // Mutation: the split and the walk to the cap, no cap-to-root check. + split_owner_path_arity(owner, d, c, 4).is_some_and(|(siblings, cap)| { + verify_merkle_path_to_cap_from_leaf_hash::(siblings, cap, d, pos, leaf) + }) + })); +} + +/// The real node one 4-ary level above leaf 0 presented as the leaf hash, with +/// the path from it upward: three siblings short. Index 0 keeps every slot the +/// fold reads at 0, so only the exact-length check rejects it. +fn admits_internal_node(verify: VerifyToCap) -> bool { + let (tree, data, owner, k, c, _) = capped_fixture(); + let cap = &owner[owner.len() - tree.cap(c).expect("c ≤ h").len()..]; + let full = tree.get_proof_by_pos(0).expect("in range").merkle_path; + let parent = P1Backend::hash_four(&[P1Backend::hash_data(&data[0]), full[0], full[1], full[2]]); + let keep = 3 * (crate::merkle_tree::cap::tree_levels(k, 4) - c); + verify(&full[3..keep], cap, k, 0, parent) +} + +#[test] +fn the_exact_length_check_rejects_an_internal_node_at_arity_4() { + assert!(!admits_internal_node( + crate::merkle_tree::cap::verify_merkle_path_to_cap_from_leaf_hash:: + )); +} + +#[test] +fn mutation_without_the_length_check_admits_an_internal_node_at_arity_4() { + fn mutant(s: &[[u64; 4]], cap: &[[u64; 4]], d: usize, i: usize, l: [u64; 4]) -> bool { + // Mutation: no `siblings.len() == 3(h − c)` check; the cap node read + // as the honest verifier reads it. + let h = crate::merkle_tree::cap::tree_levels(d, 4); + let c = (0..=h) + .find(|&c| crate::merkle_tree::cap::cap_len(d, c, 4) == Some(cap.len())) + .expect("a cap length"); + let walked = h - c; + verify_merkle_path_from_leaf_hash::(s, &cap[i >> (2 * walked)], i, l) + } + assert!(admits_internal_node(mutant)); +} + +#[test] +fn an_embed_or_split_of_the_wrong_shape_is_refused() { + use crate::merkle_tree::cap::{embed_cap_arity, split_owner_path_arity}; + let data = rows(16); + let tree = MerkleTree::::build(&data).expect("non-empty"); + let full = tree.get_proof_by_pos(5).expect("in range").merkle_path; + // A path of another depth, a cap of no height (two nodes at even depth), + // no owner. + assert!(embed_cap_arity(&mut [&mut full.clone()], 6, &tree.cap(1).unwrap(), 4).is_err()); + assert!(embed_cap_arity(&mut [&mut full.clone()], 4, &[tree.root, tree.root], 4).is_err()); + assert!(embed_cap_arity::<[u64; 4]>(&mut [], 4, &tree.cap(1).unwrap(), 4).is_err()); + // A full path is not an owner path of height 1. + assert!(split_owner_path_arity(&full, 4, 1, 4).is_none()); + assert!(split_owner_path_arity(&full, 4, 0, 4).is_some()); + assert!(split_owner_path_arity(&full, 4, 3, 4).is_none()); +} + +/// [`P1Backend`] without a padding digest: an arity-4 tree that needs one +/// (a level not a multiple of four) is refused, not built with a guess. +struct NoPadBackend; + +impl IsMerkleTreeBackend for NoPadBackend { + type Node = [u64; 4]; + type Data = Vec; + const ARITY: usize = 4; + + fn hash_data(leaf: &Vec) -> [u64; 4] { + P1Backend::hash_data(leaf) + } + + fn hash_new_parent(left: &[u64; 4], right: &[u64; 4]) -> [u64; 4] { + P1Backend::hash_new_parent(left, right) + } + + fn hash_four(children: &[[u64; 4]; 4]) -> [u64; 4] { + P1Backend::hash_four(children) + } +} + +#[test] +fn an_arity_4_backend_without_padding_refuses_what_needs_it() { + use crate::merkle_tree::cap::cap_root; + // 8 leaves: levels 8, 2, 1 — the two-node level needs padding. + assert!(MerkleTree::::build(&rows(8)).is_none()); + // 16 leaves: levels 16, 4, 1 — no padding, the same tree as P1's. + let tree = MerkleTree::::build(&rows(16)).expect("no padding needed"); + assert_eq!( + tree.root, + MerkleTree::::build(&rows(16)).unwrap().root + ); + // A two-node cap (odd depth) needs padding to reach its root. + let odd = MerkleTree::::build(&rows(8)).unwrap(); + assert!(cap_root::(&odd.cap(1).unwrap()).is_none()); + assert_eq!(cap_root::(&odd.cap(1).unwrap()), Some(odd.root)); +} diff --git a/crypto/crypto/src/tests/merkle_tests.rs b/crypto/crypto/src/tests/merkle_tests.rs index a4be838b1..18f853083 100644 --- a/crypto/crypto/src/tests/merkle_tests.rs +++ b/crypto/crypto/src/tests/merkle_tests.rs @@ -136,39 +136,3 @@ fn batch_proof_len_is_expected_for_long_pos_list() { let batch_proof = merkle_tree.get_batch_proof(&pos_list).unwrap(); assert_eq!(batch_proof.path.len(), 2); } - -#[cfg(all(feature = "serde", feature = "disk-spill"))] -mod disk_spill_serde_tests { - use crate::merkle_tree::backends::field_element::FieldElementBackend; - use crate::merkle_tree::merkle::MerkleTree; - use math::field::{element::FieldElement, goldilocks::GoldilocksField}; - use sha3::Keccak256; - - type F = GoldilocksField; - type FE = FieldElement; - type Backend = FieldElementBackend; - - /// Serializing a spilled MerkleTree must produce identical bytes to - /// serializing the same tree before spilling, and round-trip back to an - /// equal tree. - #[test] - fn test_serialize_spilled_merkle_tree_matches_unspilled() { - let values: Vec = (1..17).map(FE::from).collect(); - let unspilled = MerkleTree::::build(&values).expect("build merkle tree"); - let unspilled_bytes = bincode::serialize(&unspilled).expect("serialize unspilled"); - - let mut spilled = MerkleTree::::build(&values).expect("build merkle tree"); - spilled.spill_nodes_to_disk().expect("spill_nodes_to_disk"); - let spilled_bytes = bincode::serialize(&spilled).expect("serialize spilled"); - - assert_eq!( - spilled_bytes, unspilled_bytes, - "spilled and unspilled trees must serialize to identical bytes" - ); - - let restored: MerkleTree = - bincode::deserialize(&spilled_bytes).expect("deserialize spilled bytes"); - assert!(!restored.has_mmap_backing()); - assert_eq!(restored.root, unspilled.root); - } -} diff --git a/crypto/crypto/src/tests/mod.rs b/crypto/crypto/src/tests/mod.rs index 96bf36e92..3df798c7f 100644 --- a/crypto/crypto/src/tests/mod.rs +++ b/crypto/crypto/src/tests/mod.rs @@ -1,6 +1,14 @@ +pub mod blake3_transcript_tests; pub mod default_transcript_tests; pub mod field_element_tests; pub mod field_element_vector_tests; +pub mod grinding_determinism_tests; +pub mod hash_metrics_tests; +pub mod merkle_arity4_tests; pub mod merkle_proof_tests; pub mod merkle_tests; pub mod merkle_utils_tests; +pub mod p1_leaf_width_tag_tests; +pub mod rp1a_review_tests; +pub mod rpx_grind_tests; +pub mod rpx_transcript_tests; diff --git a/crypto/crypto/src/tests/p1_leaf_width_tag_tests.rs b/crypto/crypto/src/tests/p1_leaf_width_tag_tests.rs new file mode 100644 index 000000000..3a106cbf5 --- /dev/null +++ b/crypto/crypto/src/tests/p1_leaf_width_tag_tests.rs @@ -0,0 +1,200 @@ +//! A P1 root binds its shape (REV-P1-JUDGE §3, F3): under ZisK's untagged leaf +//! (`zisk_linear_hash`) a family of leaf shapes, a narrow tree and a wide tree +//! whose leaves carry the narrow tree's nodes, reach one root; the STARK's leaf +//! (`linear_hash`, ZisK's chain with the width tag `[len, LEAF_DOMAIN, 0, 0]` as +//! its first capacity) parts every member of the family, and the 12-felt-leaf / +//! zero-fourth-child and in-block zero-padding identities with it. + +use alloc::vec::Vec; + +use crate::hash::poseidon1_stark::{linear_hash, zisk_linear_hash}; +use crate::hash::poseidon1_w16::{Digest, Fp, compress4}; +use crate::merkle_tree::cap::{ + tree_levels, verify_cap_shaped, verify_merkle_path_to_cap_from_leaf_hash, +}; +use crate::merkle_tree::merkle::MerkleTree; +use crate::merkle_tree::traits::IsMerkleTreeBackend; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn canon(d: &Digest) -> [u64; 4] { + core::array::from_fn(|i| d[i].canonical()) +} +fn digest(n: &[u64; 4]) -> Digest { + core::array::from_fn(|i| Fp::from(n[i])) +} +fn lcg(seed: u64, n: usize) -> Vec { + let mut x = seed; + (0..n) + .map(|_| { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + Fp::from(x % P) + }) + .collect() +} + +/// ZisK's untagged leaf (`zisk_linear_hash`) over 4-ary nodes: the reference +/// the family is built against. +struct Today; +impl IsMerkleTreeBackend for Today { + type Node = [u64; 4]; + type Data = Vec; + const ARITY: usize = 4; + fn hash_data(leaf: &Vec) -> [u64; 4] { + canon(&zisk_linear_hash(leaf)) + } + fn hash_new_parent(l: &[u64; 4], r: &[u64; 4]) -> [u64; 4] { + Self::hash_four(&[*l, *r, [0; 4], [0; 4]]) + } + fn hash_four(c: &[[u64; 4]; 4]) -> [u64; 4] { + canon(&compress4(&c.map(|x| digest(&x)))) + } + fn padding_node() -> Option<[u64; 4]> { + Some([0; 4]) + } +} + +/// The STARK's leaf (`linear_hash`, width-tagged) over the same 4-ary nodes. +struct Tagged; +impl IsMerkleTreeBackend for Tagged { + type Node = [u64; 4]; + type Data = Vec; + const ARITY: usize = 4; + fn hash_data(leaf: &Vec) -> [u64; 4] { + canon(&linear_hash(leaf)) + } + fn hash_new_parent(l: &[u64; 4], r: &[u64; 4]) -> [u64; 4] { + Self::hash_four(&[*l, *r, [0; 4], [0; 4]]) + } + fn hash_four(c: &[[u64; 4]; 4]) -> [u64; 4] { + Today::hash_four(c) + } + fn padding_node() -> Option<[u64; 4]> { + Some([0; 4]) + } +} + +/// Wide leaves over a narrow tree of `w`-felt rows, `m` 4-ary levels up: +/// the wide leaf for narrow block `i` (4^m narrow rows) is the last narrow +/// row, zero-extended to whole 12-felt blocks, followed by three digests per +/// level, bottom level first. Returns (narrow rows, wide leaves). +fn dual_family>>( + w: usize, + m: u32, + n_wide: usize, +) -> (Vec>, Vec>) { + let span = 4usize.pow(m); + let narrow: Vec> = (0..n_wide * span) + .map(|k| lcg(0xD0A1 + k as u64, w)) + .collect(); + // Level digests of the narrow tree, bottom (leaf digests) first. + let mut levels: Vec> = vec![narrow.iter().map(B::hash_data).collect()]; + for _ in 1..m { + let up = levels + .last() + .unwrap() + .chunks_exact(4) + .map(|c| B::hash_four(&[c[0], c[1], c[2], c[3]])) + .collect(); + levels.push(up); + } + let wide = (0..n_wide) + .map(|i| { + let mut leaf = narrow[i * span + span - 1].clone(); + leaf.resize(w.div_ceil(12) * 12, Fp::zero()); + for (lvl, ds) in levels.iter().enumerate() { + // the last group of 4 at this level under wide leaf i + let per = span / 4usize.pow(lvl as u32); // nodes at this level under i + let base = i * per + per - 4; + for k in 0..3 { + leaf.extend_from_slice(&digest(&ds[base + k])); + } + } + leaf + }) + .collect(); + (narrow, wide) +} + +fn opens_all>>( + data: &[Vec], + depth: usize, + tree: &MerkleTree, + c: usize, +) -> bool { + let levels = tree_levels(depth, 4); + let cap = if c == 0 { + vec![tree.root] + } else { + tree.cap(c).expect("cap") + }; + verify_cap_shaped::(&cap, &tree.root, depth, c) + && data.iter().enumerate().all(|(pos, row)| { + let full = tree.get_proof_by_pos(pos).expect("in range").merkle_path; + verify_merkle_path_to_cap_from_leaf_hash::( + &full[..3 * (levels - c)], + &cap, + depth, + pos, + B::hash_data(row), + ) + }) +} + +/// For every width w ≥ 12 with w ≡ 0 (mod 6), a tree of w-felt leaves and a tree +/// of 2w-felt leaves built from its nodes reach one root under ZisK's untagged +/// leaf, and both open at every cap height. The STARK's width-tagged leaf parts +/// every one of them: a P1 root binds its leaf width. w = 12, 18, 24 and 30. +#[test] +fn the_width_tag_parts_the_shape_dual_family() { + // (w, m): 2w = 12·(⌈w/12⌉ + m) felts. + for (w, m) in [(12usize, 1u32), (18, 1), (24, 2), (30, 2)] { + assert_eq!(2 * w, 12 * (w.div_ceil(12) + m as usize), "w {w}"); + let n_wide = 16usize; + let (narrow, wide) = dual_family::(w, m, n_wide); + assert!(wide.iter().all(|l| l.len() == 2 * w)); + let tn = MerkleTree::::build(&narrow).unwrap(); + let tw = MerkleTree::::build(&wide).unwrap(); + assert_eq!(tn.root, tw.root, "w {w}: one root, two shapes"); + let dn = (narrow.len()).trailing_zeros() as usize; + let dw = (wide.len()).trailing_zeros() as usize; + assert_eq!(dn - dw, 2 * m as usize); + for c in 0..=tree_levels(dw, 4) { + assert!(opens_all(&narrow, dn, &tn, c), "w {w} narrow c {c}"); + assert!(opens_all(&wide, dw, &tw, c), "w {w} wide c {c}"); + if c > 0 { + assert_eq!(tn.cap(c), tw.cap(c), "w {w} c {c}: one cap too"); + } + } + + // The width tag: the same construction lands on two roots. + let (narrow, wide) = dual_family::(w, m, n_wide); + let tn = MerkleTree::::build(&narrow).unwrap(); + let tw = MerkleTree::::build(&wide).unwrap(); + assert_ne!(tn.root, tw.root, "w {w}: the width tag parts the shapes"); + } + // Widths that are not ≡ 0 (mod 6), or below 12, have no dual: the wide + // leaf's last block would need zero digest lanes. + for w in [6usize, 10, 13, 16, 17, 19] { + assert!(w < 12 || w % 6 != 0); + } +} + +/// The width tag also removes the 12-felt-leaf / zero-fourth-child identity +/// (node(a, b, c, 0) = leaf(a ‖ b ‖ c)). +#[test] +fn the_width_tag_parts_a_short_leaf_from_a_padded_node() { + let row = lcg(77, 12); + let d = |k: usize| -> Digest { core::array::from_fn(|j| row[4 * k + j]) }; + let node = compress4(&[d(0), d(1), d(2), [Fp::zero(); 4]]); + assert_eq!(node, zisk_linear_hash(&row), "ZisK's untagged leaf"); + assert_ne!(node, linear_hash(&row), "the STARK's leaf"); + // ... and the in-block zero-padding identity (REV-P1-B M1's leaf half). + let x = lcg(78, 5); + let mut x0 = x.clone(); + x0.push(Fp::zero()); + assert_eq!(zisk_linear_hash(&x), zisk_linear_hash(&x0)); + assert_ne!(linear_hash(&x), linear_hash(&x0)); +} diff --git a/crypto/crypto/src/tests/rp1a_review_tests.rs b/crypto/crypto/src/tests/rp1a_review_tests.rs new file mode 100644 index 000000000..92aa36235 --- /dev/null +++ b/crypto/crypto/src/tests/rp1a_review_tests.rs @@ -0,0 +1,242 @@ +//! The adversarial review REV-P1-A's crypto tests (the subset the judge, +//! REV-P1-JUDGE §7, keeps): the node compression's collisions under the inverse +//! permutation (the documented assumption: a tree binds through its anchored +//! leaves), and the uncapped odd-depth top's padding, which the host now binds. + +use alloc::vec::Vec; + +use crate::hash::poseidon1_stark::linear_hash; +use crate::hash::poseidon1_w16::{ + self as w16, Digest, Fp, STATE_FELTS, compress4, constants::MDS_CIRC_ROW, + constants::ROUND_CONSTANTS, is_full_round, +}; +use crate::merkle_tree::cap::CappedRoot; +use crate::merkle_tree::traits::IsMerkleTreeBackend; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// The P1 leaf hash and 4-ary node over canonical digests (the same test +/// backend `merkle_arity4_tests` uses; `prover`'s `P1BatchBackend` hashes the +/// same felts). +struct P1Backend; + +fn canon(d: &Digest) -> [u64; 4] { + core::array::from_fn(|i| d[i].canonical()) +} + +fn digest(n: &[u64; 4]) -> Digest { + core::array::from_fn(|i| Fp::from(n[i])) +} + +impl IsMerkleTreeBackend for P1Backend { + type Node = [u64; 4]; + type Data = Vec; + const ARITY: usize = 4; + + fn hash_data(leaf: &Vec) -> [u64; 4] { + canon(&linear_hash(leaf)) + } + + fn hash_new_parent(left: &[u64; 4], right: &[u64; 4]) -> [u64; 4] { + Self::hash_four(&[*left, *right, [0; 4], [0; 4]]) + } + + fn hash_four(children: &[[u64; 4]; 4]) -> [u64; 4] { + canon(&compress4(&children.map(|c| digest(&c)))) + } + + fn padding_node() -> Option<[u64; 4]> { + Some([0; 4]) + } +} + +fn lcg(seed: u64, n: usize) -> Vec { + let mut x = seed; + (0..n) + .map(|_| { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + Fp::from(x % P) + }) + .collect() +} + +// ------------------------------------------------------------------ inverse + +/// `7^{-1} mod (p − 1)`: the inverse S-box exponent. +fn inv7() -> u64 { + let m = (P - 1) as i128; + let (mut a, mut b, mut x0, mut x1) = (7i128, m, 1i128, 0i128); + while b != 0 { + let q = a / b; + (a, b) = (b, a - q * b); + (x0, x1) = (x1, x0 - q * x1); + } + assert_eq!(a, 1); + (((x0 % m) + m) % m) as u64 +} + +/// The circulant MDS as a dense matrix (`poseidon1_w16`'s `MDS_MATRIX`). +fn mds_matrix() -> [[Fp; STATE_FELTS]; STATE_FELTS] { + core::array::from_fn(|i| { + core::array::from_fn(|j| Fp::from(MDS_CIRC_ROW[(j + STATE_FELTS - i) % STATE_FELTS])) + }) +} + +/// Solve `M · x = y` by Gauss–Jordan elimination. +fn mds_solve(y: &[Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let mut a: Vec> = mds_matrix() + .iter() + .zip(y) + .map(|(row, v)| { + let mut r = row.to_vec(); + r.push(*v); + r + }) + .collect(); + let n = STATE_FELTS; + for col in 0..n { + let piv = (col..n) + .find(|&r| a[r][col] != Fp::zero()) + .expect("the MDS is invertible"); + a.swap(col, piv); + let inv = a[col][col].inv().expect("nonzero pivot"); + for v in a[col][col..].iter_mut() { + *v *= inv; + } + let pivot_row = a[col].clone(); + for (r, row) in a.iter_mut().enumerate() { + if r != col && row[col] != Fp::zero() { + let f = row[col]; + for (v, p) in row[col..].iter_mut().zip(&pivot_row[col..]) { + *v = *v - *p * f; + } + } + } + } + core::array::from_fn(|i| a[i][n]) +} + +/// `poseidon1_w16::permute`'s inverse: per round, MDS⁻¹, S-box⁻¹, minus the +/// round constants. +fn permute_inv(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let d = inv7(); + let mut s = state; + for r in (0..w16::NUM_ROUNDS).rev() { + s = mds_solve(&s); + if is_full_round(r) { + for v in s.iter_mut() { + *v = v.pow(d); + } + } else { + s[0] = s[0].pow(d); + } + for (v, c) in s.iter_mut().zip(ROUND_CONSTANTS[r].iter()) { + *v = *v - Fp::from(*c); + } + } + s +} + +fn split4(x: &[Fp; STATE_FELTS]) -> [Digest; 4] { + core::array::from_fn(|c| core::array::from_fn(|j| x[4 * c + j])) +} + +/// The 4-ary node is a TRUNCATED permutation of sixteen free lanes, so with +/// the inverse permutation (public, cheap) it has collisions for free: any two +/// outputs agreeing on lanes 0..4 pull back to two node inputs with one +/// digest. What still binds a tree is the leaf: its first block's capacity is +/// the width tag, so a pulled-back input is a leaf only if its four capacity +/// lanes are that tag. +#[test] +fn rp1a_the_node_compression_collides_under_the_inverse_permutation() { + let mut fwd = [Fp::zero(); STATE_FELTS]; + for (i, v) in fwd.iter_mut().enumerate() { + *v = Fp::from(i as u64 * 1_000_003 + 17); + } + assert_eq!(permute_inv(w16::permute(fwd)), fwd, "the inverse is exact"); + + let target: Digest = core::array::from_fn(|i| Fp::from(0xC0FF_EE00 + i as u64)); + let pull = |tail_seed: u64| { + let tail = lcg(tail_seed, 12); + let mut out = [Fp::zero(); STATE_FELTS]; + out[..4].copy_from_slice(&target); + out[4..].copy_from_slice(&tail); + permute_inv(out) + }; + let (x, y) = (pull(1), pull(2)); + assert_ne!(x, y); + let (cx, cy) = (split4(&x), split4(&y)); + assert_eq!(compress4(&cx), target); + assert_eq!(compress4(&cy), target); + assert_ne!( + cx, cy, + "two different child quadruples, one node: a collision" + ); + + // The anchor: a pulled-back input is a one-block leaf only if its lanes + // 12..16 (the leaf's zero capacity) vanish, which they do not. + assert!(x[12..].iter().any(|v| *v != Fp::zero())); +} + +// ----------------------------------------------- the uncapped odd-depth top + +/// At an UNCAPPED odd-depth tree the walk reaches the top group, two real +/// children and two padding nodes. The host checker requires the path's two +/// padding siblings to BE the padding (REV-P1-JUDGE F4), as the in-guest walk +/// (`p1w16_emit::walk4`) supplies them as constants: a root over non-zero +/// padding opens on no index, the honest root on every one. At c ≥ 1 the +/// host's `cap_root` pads with the backend's padding node itself. +#[test] +fn rp1a_uncapped_odd_top_padding_is_bound_on_the_host() { + let depth = 3; // 8 leaves: one full 4-ary level, then a two-node top + let rows: Vec> = (0..8).map(|k| lcg(0xABC + k, 5)).collect(); + let leaves: Vec<[u64; 4]> = rows.iter().map(P1Backend::hash_data).collect(); + let n0 = P1Backend::hash_four(&[leaves[0], leaves[1], leaves[2], leaves[3]]); + let n1 = P1Backend::hash_four(&[leaves[4], leaves[5], leaves[6], leaves[7]]); + let junk = [1u64, 2, 3, 4]; + let forged_root = P1Backend::hash_four(&[n0, n1, junk, junk]); + let honest_root = P1Backend::hash_four(&[n0, n1, [0; 4], [0; 4]]); + assert_ne!(forged_root, honest_root); + + for pos in 0..8 { + let base = pos / 4 * 4; + let mut path: Vec<[u64; 4]> = (base..base + 4) + .filter(|&c| c != pos) + .map(|c| leaves[c]) + .collect(); + // Top level: the partner, then the two padding slots, in child order. + let partner = if pos < 4 { n1 } else { n0 }; + let mut forged = path.clone(); + forged.extend([partner, junk, junk]); + assert!( + !CappedRoot::uncapped(&forged_root, depth).verify::( + &forged, + pos, + leaves[pos] + ), + "the host refuses non-zero padding siblings at pos {pos}" + ); + path.extend([partner, [0; 4], [0; 4]]); + assert!( + CappedRoot::uncapped(&honest_root, depth).verify::(&path, pos, leaves[pos]), + "the honest opening at pos {pos}" + ); + } + + // At cap height 1 the cap is [n0, n1] and its root pads with zeros + // itself: the forged root is not that cap's root. + assert!(!crate::merkle_tree::cap::verify_cap_shaped::( + &[n0, n1], + &forged_root, + depth, + 1 + )); + assert!(crate::merkle_tree::cap::verify_cap_shaped::( + &[n0, n1], + &honest_root, + depth, + 1 + )); +} diff --git a/crypto/crypto/src/tests/rpx_grind_tests.rs b/crypto/crypto/src/tests/rpx_grind_tests.rs new file mode 100644 index 000000000..be3ca5798 --- /dev/null +++ b/crypto/crypto/src/tests/rpx_grind_tests.rs @@ -0,0 +1,185 @@ +//! ★ The RPX grind's host/device contract, and the endianness that decides it. +//! +//! The device search takes the 32-byte inner hash as four `u64`s, and the two +//! arms read those bytes in OPPOSITE orders — little-endian lanes for keccak, +//! big-endian felts for RPX. Crossing them compiles, runs, and searches for a +//! nonce under a message the host never hashes; the host check then rejects +//! every nonce the device returns and the prover falls back to the CPU forever, +//! which is a performance cliff with no error attached to it. +//! +//! So the mapping is pinned here against the ORACLE TABLE the CUDA kernel is +//! itself checked against — `RPX_GRIND_VECTORS` in +//! `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h`, three rows printed by +//! the per-table branch's host implementation. Each row carries the four +//! big-endian `inner_felts`, the smallest valid nonce, and `le_nonce`: what the +//! same kernel answers on the LITTLE-endian reading, which the header records as +//! `u64::MAX` — nothing found. That last column is the endianness control, and +//! it is why these tests need no GPU: they check the HOST side of an agreement +//! whose device side is pinned to the same numbers. + +use alloc::vec::Vec; +use digest::Digest; + +use crate::grinding::{inner_hash_felts, inner_hash_lanes, is_valid_nonce}; +use crate::hash::rpx::Rpx256Digest; + +/// One row of `RPX_GRIND_VECTORS`: the seed byte (repeated 32 times), the +/// grinding factor, the four BIG-endian inner felts, and the smallest valid +/// nonce. +const RPX_GRIND_VECTORS: [(u8, u8, [u64; 4], u64); 3] = [ + ( + 90, + 12, + [ + 17047917526726690733, + 2027278666509702433, + 4678289907902145381, + 4242003890993108442, + ], + 1342, + ), + ( + 17, + 13, + [ + 3807340077325453675, + 129745844021573959, + 15014385560057355003, + 944573484564438641, + ], + 300, + ), + ( + 32, + 14, + [ + 5597071933014793605, + 8702110216523445336, + 2882478612521280078, + 9429844132731097150, + ], + 705, + ), +]; + +fn seed_of(byte: u8) -> [u8; 32] { + [byte; 32] +} + +/// ★★ `inner_hash_felts` reproduces the oracle's four felts, on every row. +/// +/// This is the mapping itself: the four `u64`s the device search is handed. +/// Nothing in this repository produced these numbers — they are the per-table +/// branch's host implementation, and the CUDA kernel is checked against the +/// same table. +#[test] +fn the_inner_felts_match_the_device_oracle_table() { + for (byte, factor, want, _) in RPX_GRIND_VECTORS { + let got = inner_hash_felts::(&seed_of(byte), factor); + assert_eq!(got, want, "seed 0x{byte:02x}, factor {factor}"); + } +} + +/// ★ And the oracle's nonce passes the HOST predicate — the agreement the +/// device dispatch rests on, checked without a device. +#[test] +fn the_oracle_nonce_passes_the_host_predicate() { + for (byte, factor, _, nonce) in RPX_GRIND_VECTORS { + assert!( + is_valid_nonce::(&seed_of(byte), nonce, factor), + "seed 0x{byte:02x}, factor {factor}: nonce {nonce} rejected" + ); + } +} + +/// ★ …and it really is the SMALLEST, by exhaustive scan — a different algorithm +/// from the one that produced it. +#[test] +fn the_oracle_nonce_is_the_smallest_valid_one() { + for (byte, factor, _, nonce) in RPX_GRIND_VECTORS { + let seed = seed_of(byte); + assert!(nonce > 0, "the scan below is vacuous at nonce 0"); + assert!( + (0..nonce).all(|n| !is_valid_nonce::(&seed, n, factor)), + "seed 0x{byte:02x}, factor {factor}: a nonce below {nonce} is also valid" + ); + } +} + +/// ⚠⚠ **THE ENDIANNESS CONTROL.** The little-endian reading of the same inner +/// hash is a DIFFERENT message. +/// +/// Without this, `inner_hash_felts` could be `inner_hash_lanes` with a new name +/// and every test above would still pass — they would simply all be about the +/// wrong four `u64`s together. The header records `le_nonce = u64::MAX` for all +/// three rows: on the little-endian reading the kernel finds nothing at all in +/// its scanned block. +#[test] +fn the_little_endian_reading_is_a_different_message() { + for (byte, factor, felts, _) in RPX_GRIND_VECTORS { + let seed = seed_of(byte); + let lanes = inner_hash_lanes::(&seed, factor); + assert_ne!( + lanes, felts, + "seed 0x{byte:02x}: the two readings must differ, or there is nothing to get wrong" + ); + } +} + +/// ✓ The two readings are byte-reversals of each other, lane for lane — so the +/// difference above is exactly the endianness and not a hash that moved. +#[test] +fn the_two_readings_are_byte_reversals_of_one_another() { + for (byte, factor, _, _) in RPX_GRIND_VECTORS { + let seed = seed_of(byte); + let lanes = inner_hash_lanes::(&seed, factor); + let felts = inner_hash_felts::(&seed, factor); + for (i, (l, f)) in lanes.iter().zip(&felts).enumerate() { + assert_eq!(l.swap_bytes(), *f, "lane {i} of seed 0x{byte:02x}"); + } + } +} + +/// ★ The preimage is ONE rate-8 block, which is what makes the kernel's +/// `init(5)` right: `inner_hash ‖ nonce` is 40 bytes, five felts, padding flag +/// `5 mod 8 = 5`. +/// +/// Checked by computing the digest the long way — felts in, sponge out — and +/// requiring it to equal what the production predicate hashes from bytes. +#[test] +fn the_grind_preimage_is_five_felts_in_one_block() { + use crate::hash::rpx::{Fp, RATE_FELTS, digest_to_commitment, sponge_leaf}; + + for (byte, factor, felts, nonce) in RPX_GRIND_VECTORS { + // What the kernel absorbs: the four inner felts, then the nonce. + let mut block: Vec = felts.iter().map(|v| Fp::from(*v)).collect(); + block.push(Fp::from(nonce)); + assert_eq!(block.len(), 5, "the grind preimage is five felts"); + assert!(block.len() <= RATE_FELTS, "…and therefore one rate block"); + + let by_felts = digest_to_commitment(&sponge_leaf(&block)); + + // What the host predicate hashes: the 32 inner bytes then the nonce, + // big-endian, through the production digest. + let mut inner = [0u8; 32]; + for (i, v) in felts.iter().enumerate() { + inner[i * 8..(i + 1) * 8].copy_from_slice(&v.to_be_bytes()); + } + let mut data = [0u8; 40]; + data[..32].copy_from_slice(&inner); + data[32..].copy_from_slice(&nonce.to_be_bytes()); + let by_bytes: [u8; 32] = Rpx256Digest::digest(data).into(); + + assert_eq!( + by_felts, by_bytes, + "seed 0x{byte:02x}: the felt form and the byte form must be one hash" + ); + + // And that digest's leading u64 is what `limit` is compared against. + let head = u64::from_be_bytes(by_bytes[..8].try_into().unwrap()); + assert!( + head < 1u64 << (64 - factor), + "seed 0x{byte:02x}: the oracle nonce must clear its own limit" + ); + } +} diff --git a/crypto/crypto/src/tests/rpx_transcript_tests.rs b/crypto/crypto/src/tests/rpx_transcript_tests.rs new file mode 100644 index 000000000..018030595 --- /dev/null +++ b/crypto/crypto/src/tests/rpx_transcript_tests.rs @@ -0,0 +1,441 @@ +//! ★★ W1-A — the RPX transcript hands out canonical felts, and the keccak one +//! is untouched. +//! +//! Two facts with two reasons, kept apart on purpose because a verifier that +//! needs both must not take one as evidence of the other: +//! +//! 1. **A field coordinate is one candidate under RPX** — because a squeeze IS +//! four canonical felts once the byte reversal is gone. That is W1-A, and +//! [`TranscriptHash::CANDIDATES_PER_COORDINATE`] states it. +//! 2. **A query index is one draw** — because every WHIR query bound is a power +//! of two, so `sample_u64`'s rejection threshold is zero. That is true today, +//! hash-independent, and has nothing to do with (1). +//! +//! And one finding, which is the strongest reason the reversal had to go and is +//! not a cost argument at all: +//! [`the_reversal_would_have_biased_the_rpx_sampler`] — a reversed candidate is +//! `>= p` for about `2^32` canonical felts, so the old sampler drew uniformly +//! from a subset of the field. +//! +//! # ⛔ Why none of this is tested by sampling, and what that cost +//! +//! The obvious control — "show keccak needing more than one candidate" — is +//! unreachable. A keccak squeeze is 32 uniform bytes, so an 8-byte group lands +//! in `[p, 2^64)` with probability about **2^-32**: observing one takes on the +//! order of a billion squeezes. The same goes for `sample_u64`, whose rejection +//! region is `2^64 mod bound` wide — under `2^-32` of the range for any bound +//! this system uses, power of two or not. +//! +//! So a statistical test cannot tell the two configurations apart, and one that +//! appeared to would be measuring noise. **The difference is structural**: under +//! RPX every group is canonical BY CONSTRUCTION; under keccak every group is +//! canonical WITH HIGH PROBABILITY. The tests below pin the construction — +//! round-trips, the arithmetic of the threshold, and a rejection driven by a +//! candidate constructed to be rejected — rather than waiting for an event that +//! will not arrive. +//! +//! That is also why the rejection path is exercised explicitly: `Some(1)` is a +//! claim about the sampler's INPUTS, and it would be worth nothing if the +//! branch it bypasses had quietly stopped working. +//! +//! # ⚠ WHICH TEST CATCHES WHICH MUTATION — and which do not +//! +//! Run, not assumed. Putting the reversal back (either by flipping +//! `RpxTranscriptHash::REVERSES_SQUEEZE` or by deleting the `if` in `sample`) +//! fails **exactly one** test below: +//! [`an_rpx_squeeze_is_the_unreversed_digest_of_what_was_absorbed`]. +//! +//! [`every_group_of_an_rpx_squeeze_is_a_canonical_felt`] and +//! [`a_cubic_element_costs_exactly_three_draws_under_rpx`] both still PASS with +//! the reversal restored, and that is not a defect in them — it is the same +//! 2^-32 again from the other side. A canonical felt's bytes read backwards are +//! a number below `p` unless the top bytes conspire, so 2048 reversed groups +//! look exactly like 2048 canonical ones. Those two tests guard the +//! canonicalisation inside `digest_to_commitment`, which is a real regression +//! mode; they do **not** guard the byte order, and reading them as if they did +//! would leave the seam covered by nothing. +//! +//! One test guards the byte order. It is the one with the construction in it. + +use digest::Digest; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::{GOLDILOCKS_PRIME, GoldilocksField as Fp}; +use math::field::traits::HasDefaultTranscript; + +use crate::fiat_shamir::default_transcript::DefaultTranscript; +use crate::fiat_shamir::is_transcript::IsTranscript; +use crate::fiat_shamir::transcript_hash::{ + KeccakTranscriptHash, RpxTranscriptHash, TranscriptHash, +}; +use crate::hash::platform_keccak::PlatformKeccak256; +use crate::hash::rpx::{commitment_to_digest, digest_to_commitment, sponge_leaf_bytes}; + +/// The 8-byte big-endian groups a sampler reads out of a squeeze, in the order +/// [`DefaultTranscript::next_sample_u64`] reads them. +fn groups(squeeze: &[u8; 32]) -> [u64; 4] { + core::array::from_fn(|i| { + let mut b = [0u8; 8]; + b.copy_from_slice(&squeeze[i * 8..(i + 1) * 8]); + u64::from_be_bytes(b) + }) +} + +/// ★★ K5 (RPX half). The squeeze is the digest, in the digest's own byte order. +/// +/// Against the CONSTRUCTION — `digest_to_commitment(sponge_leaf_bytes(..))` — +/// not against the other arm. Two transcripts agreeing tells you nothing; they +/// agree on a wrong hash too. +#[test] +fn an_rpx_squeeze_is_the_unreversed_digest_of_what_was_absorbed() { + let absorbed = b"W1-A: the squeeze is the digest"; + let mut transcript = DefaultTranscript::::new(absorbed); + + let squeeze = transcript.sample(); + let expected = digest_to_commitment(&sponge_leaf_bytes(absorbed)); + + assert_eq!( + squeeze, expected, + "the RPX squeeze is not the digest of what was absorbed" + ); + + // …and NOT the reversed one, or the constant is decorative. + let mut reversed = expected; + reversed.reverse(); + assert_ne!( + squeeze, reversed, + "the RPX squeeze is still reversed — REVERSES_SQUEEZE is not being read" + ); + // ⚠ No `assert!(!RpxTranscriptHash::REVERSES_SQUEEZE)` here. Clippy is right + // that it cannot fail at runtime, and it would add nothing: the two + // assertions above test the BEHAVIOUR the constant is supposed to cause, + // which is what a wrong constant would break. Asserting the constant's own + // value would only restate the source line that sets it. +} + +/// ★★ K5 (keccak half). THE CONTROL. The keccak squeeze is still the reversed +/// digest, byte for byte. +/// +/// This is the half that must not move: every proof this system has produced +/// was produced under this convention, and W1-A is only allowed to touch the +/// other arm. +#[test] +fn a_keccak_squeeze_is_still_the_reversed_digest() { + let absorbed = b"W1-A: the squeeze is the digest"; + let mut transcript = DefaultTranscript::::new(absorbed); + + let squeeze = transcript.sample(); + + let mut expected: [u8; 32] = PlatformKeccak256::digest(absorbed).into(); + expected.reverse(); + + assert_eq!( + squeeze, expected, + "the keccak squeeze moved: this commit changed every proof on this branch" + ); +} + +/// ★★ THE PROPERTY BEHIND `Some(1)`. Every group of every RPX squeeze is a +/// canonical felt — by round-trip, not by luck. +/// +/// The chain is exercised, not one squeeze: `sample()` absorbs its own output, +/// so squeeze `n+1` is a function of squeeze `n`, and a canonicality that held +/// only for the first would be an accident of the seed. +#[test] +fn every_group_of_an_rpx_squeeze_is_a_canonical_felt() { + let mut transcript = DefaultTranscript::::new(b"chain"); + + for round in 0..512 { + let squeeze = transcript.sample(); + + for (i, g) in groups(&squeeze).iter().enumerate() { + assert!( + *g < GOLDILOCKS_PRIME, + "round {round}, group {i}: {g} is not a canonical felt, \ + so a one-candidate schedule would miss here" + ); + } + + // The structural statement the bound above is a consequence of: the 32 + // bytes ARE four felts, and reading them back gives the same four. + let digest = commitment_to_digest(&squeeze); + assert_eq!( + digest_to_commitment(&digest), + squeeze, + "round {round}: the squeeze does not round-trip through its felts" + ); + } +} + +/// ★★ ONE CANDIDATE, COMPOSED. The real sampler, fed the real squeeze, +/// consumes exactly three draws for a cubic element. +/// +/// [`Ext::sample_field_element_from`] is the production body; the closure is +/// the production byte source. Only the counter is the test's. +#[test] +fn a_cubic_element_costs_exactly_three_draws_under_rpx() { + let mut transcript = DefaultTranscript::::new(b"three draws"); + let squeeze = transcript.sample(); + let mut supply = groups(&squeeze).into_iter(); + + let mut draws = 0usize; + let element = Ext::sample_field_element_from(|| { + draws += 1; + supply + .next() + .expect("a fourth draw means a rejection occurred") + }); + + assert_eq!( + draws, + 3, + "a cubic element took {draws} draws, so CANDIDATES_PER_COORDINATE = {:?} is wrong", + RpxTranscriptHash::CANDIDATES_PER_COORDINATE + ); + // `NonZeroUsize`, not `usize`: the schedule's type on this branch cannot + // spell `Some(0)`, which is a draw count that returns an uninitialised + // candidate. Same assertion, one constructor deeper. + assert_eq!( + RpxTranscriptHash::CANDIDATES_PER_COORDINATE, + core::num::NonZeroUsize::new(1) + ); + + // The element is the first three groups, in order — which is what makes the + // draw count meaningful rather than a count of a loop that did nothing. + let expected: Vec> = groups(&squeeze)[..3] + .iter() + .map(|g| FieldElement::from(*g)) + .collect(); + assert_eq!(element.value().to_vec(), expected); +} + +/// ★★ …AND THE REJECTION BRANCH IS STILL ALIVE. +/// +/// `Some(1)` is a claim about the sampler's INPUTS. If the rejection test had +/// been deleted, every test above would still pass and the constant would be +/// true for the wrong reason — so the branch is driven by a candidate +/// constructed to be rejected. This is the only way to reach it: waiting for a +/// keccak squeeze to produce one is a 2^-32 event per group. +#[test] +fn a_non_canonical_candidate_is_rejected_and_redrawn() { + // p itself is the smallest non-canonical u64, and `p + 7` is inside the + // window a uniform draw can land in. + let supply = [GOLDILOCKS_PRIME, GOLDILOCKS_PRIME + 7, 42u64]; + let mut it = supply.into_iter(); + + let mut draws = 0usize; + let element = Fp::sample_field_element_from(|| { + draws += 1; + it.next().expect("the sampler drew more than the supply") + }); + + assert_eq!( + draws, 3, + "the sampler accepted a candidate >= p: the rejection test is gone, and \ + CANDIDATES_PER_COORDINATE = Some(1) would then be true of nothing" + ); + assert_eq!(element, FieldElement::::from(42u64)); +} + +/// ★★★ WHAT THE REVERSAL WAS ACTUALLY DOING: biasing the RPX sampler. +/// +/// This is the strongest reason to remove it, and it is not a cost argument. +/// +/// A candidate under the old code was `byteswap(canonical(felt))`. That is +/// `>= p` exactly when the felt's low four bytes are all `0xFF` — reversing +/// puts them in the top four, and `p`'s top four bytes are `0xFFFFFFFF`. The +/// rejection sampler then drew uniformly from a SUBSET of `[0, p)` missing +/// about `2^32` elements: statistical distance ~`2^-32` per coordinate, and +/// over an epoch verify's ~3e4 coordinate draws a loose hybrid bound of +/// ~`2^-17` of added soundness error. +/// +/// Not a demonstrated attack — the excluded set is fixed and public and no +/// prover steers into it — and never exercised, because the RPX transcript was +/// not wired to any prover (see the commit). But it is exactly the kind of +/// unquoted term an audit names, and it was inherited rather than chosen: +/// harmless under keccak, whose 8-byte groups are uniform on 64 bits and whose +/// sampler is therefore exactly uniform. It exists only in the +/// RPX-under-`DefaultTranscript` combination. +/// +/// The test exhibits the witness rather than describing it, and characterises +/// the whole excluded set so the claim is a statement about all of it. +#[test] +fn the_reversal_would_have_biased_the_rpx_sampler() { + let byteswap = |v: u64| { + u64::from_be_bytes({ + let mut b = v.to_be_bytes(); + b.reverse(); + b + }) + }; + + // V1's witness: a canonical felt whose reversed bytes are NOT canonical, so + // the old code would have rejected this felt every time it appeared. + let witness: u64 = 0x0000_0001_ffff_ffff; + assert!( + witness < GOLDILOCKS_PRIME, + "the witness must be a real felt" + ); + assert!( + byteswap(witness) >= GOLDILOCKS_PRIME, + "the witness's reversed bytes are canonical, so it is not a witness" + ); + + // …and the excluded set is exactly the felts whose low four bytes are all + // `0xFF`, save the one whose high four bytes are zero. Checked over the + // whole set rather than sampled: it is generated, not searched for. + for high in 0..4096u64 { + let v = (high << 32) | 0xFFFF_FFFF; + if v >= GOLDILOCKS_PRIME { + continue; + } + let excluded = byteswap(v) >= GOLDILOCKS_PRIME; + assert_eq!( + excluded, + high != 0, + "felt {v:#018x} is misclassified: the excluded set is not what the \ + bias argument says it is" + ); + } + + // The far larger complement: nothing OUTSIDE that set was excluded, so the + // bias is precisely the one described and not a larger one. + let mut checked = 0u32; + for v in (0..1u64 << 24).map(|i| i.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 1) { + if v >= GOLDILOCKS_PRIME || (v & 0xFFFF_FFFF) == 0xFFFF_FFFF { + continue; + } + assert!( + byteswap(v) < GOLDILOCKS_PRIME, + "felt {v:#018x} was excluded but is not in the described set" + ); + checked += 1; + } + assert!( + checked > 1_000_000, + "only {checked} felts were actually checked" + ); + + // And the fix: a live squeeze's groups are canonical, so no candidate is + // excluded and the distribution is the digest's, entire. + let mut transcript = DefaultTranscript::::new(b"bias"); + for _ in 0..64 { + for g in groups(&transcript.sample()) { + assert!(g < GOLDILOCKS_PRIME); + } + } +} + +/// ★ The keccak configuration makes no such claim, and says so. +#[test] +fn keccak_does_not_claim_a_bounded_candidate_count() { + assert_eq!( + KeccakTranscriptHash::CANDIDATES_PER_COORDINATE, + None, + "keccak's squeeze is uniform bytes: the draw count has a distribution, not a bound" + ); +} + +/// ★★ THE SECOND FACT, WITH ITS OWN REASON. A query index is one draw because +/// the bound is a power of two — not because of anything W1-A did. +/// +/// `sample_u64` rejects `candidate < 2^64 mod bound`. For a power of two that +/// region is EMPTY, so the loop cannot turn, whatever the hash. Every WHIR +/// query bound is `num_leaves = 1 << (log_domain_size - log_folding)` +/// (`multilinear::whir_commit::CodewordCommitment::num_leaves`), so this covers +/// all of them. +/// +/// ⚠ The non-power-of-two half is the point of the test. Without it this would +/// assert that zero equals zero for 64 values and pass on any implementation. +#[test] +fn a_power_of_two_bound_has_no_rejection_region_and_a_ragged_one_does() { + let threshold = |bound: u64| bound.wrapping_neg() % bound; + + for k in 0..64 { + let bound = 1u64 << k; + assert_eq!( + threshold(bound), + 0, + "bound 2^{k} has a rejection region, so a query index is not one draw" + ); + } + + // Constructed counter-examples: a bound that is NOT a power of two must + // have a non-empty rejection region, or the expression above is not + // computing what this test claims it computes. + for bound in [3u64, 5, 6, 100, (1 << 20) + 1, u64::MAX] { + assert_ne!( + threshold(bound), + 0, + "bound {bound} is not a power of two yet shows no rejection region" + ); + } +} + +/// ★ …and the draw count that follows from it, observed through the transcript +/// rather than asserted. +/// +/// Four `sample_u64` calls at a power-of-two bound must consume exactly one +/// squeeze. The witness is the transcript's own state: every squeeze chains its +/// output back in, so a fifth draw would leave `a` somewhere `b` is not. +#[test] +fn four_query_indices_cost_one_squeeze() { + for hash_is_rpx in [false, true] { + let (state_a, state_b) = if hash_is_rpx { + let mut a = DefaultTranscript::::new(b"queries"); + let mut b = a.clone(); + for _ in 0..4 { + a.sample_u64(1 << 20); + } + let _ = b.sample(); + ( + IsTranscript::::state(&a), + IsTranscript::::state(&b), + ) + } else { + let mut a = DefaultTranscript::::new(b"queries"); + let mut b = a.clone(); + for _ in 0..4 { + a.sample_u64(1 << 20); + } + let _ = b.sample(); + ( + IsTranscript::::state(&a), + IsTranscript::::state(&b), + ) + }; + + assert_eq!( + state_a, state_b, + "four query draws consumed more than one squeeze (rpx = {hash_is_rpx})" + ); + } +} + +/// ★ The duplex buffer hands out whole groups, which is what makes the felt +/// boundaries and the read boundaries the same boundaries. +/// +/// Sampling 4 `u64`s consumes exactly one squeeze; the 5th forces the next. If +/// a read ever straddled two groups, `Some(1)` would be false even with a +/// canonical squeeze — the claim depends on this alignment, so it is pinned. +#[test] +fn the_buffer_is_consumed_in_whole_felt_groups() { + let mut transcript = DefaultTranscript::::new(b"alignment"); + let mut reference = transcript.clone(); + + let first = reference.sample(); + let second = reference.sample(); + + let drawn: Vec = (0..8).map(|_| transcript.sample_u64(u64::MAX)).collect(); + + let expected: Vec = groups(&first) + .into_iter() + .chain(groups(&second)) + .map(|g| g % u64::MAX) + .collect(); + + assert_eq!( + drawn, expected, + "the buffer is not being handed out as whole 8-byte groups in order" + ); +} diff --git a/crypto/crypto/tests/transcript_counters.rs b/crypto/crypto/tests/transcript_counters.rs new file mode 100644 index 000000000..d349bceff --- /dev/null +++ b/crypto/crypto/tests/transcript_counters.rs @@ -0,0 +1,625 @@ +//! ★★ The Fiat-Shamir counters answer "which sponge ran", on BOTH arms. +//! +//! ```text +//! cargo test -p crypto --features hash-metrics --test transcript_counters +//! ``` +//! +//! # The trap these exist to close +//! +//! `hash_metrics`'s header already tells this story for Merkle: a counter that +//! compares `TypeId` against keccak and does nothing otherwise "became a check +//! that cannot fail the moment a second hash arrived — under RPX every Merkle +//! counter would have read ZERO, reporting *no hashing* for precisely the arm +//! whose purpose is to change the hashing". +//! +//! The transcript never got that treatment. `count_absorb` was bumped from one +//! place, the keccak wrapper's `update`; `Rpx256Digest::update` was a bare +//! `Vec::extend`; and `total`'s documentation claimed to count transcript +//! squeezes while nothing on the RPX side bumped it. So an RPX proof read zero +//! absorbs and zero transcript finalizes, and zero is exactly what a +//! *correctly instrumented* keccak-free run would also read. The measurement +//! could not distinguish "the other hash ran" from "nobody instrumented it". +//! +//! That is not hypothetical here: for four measured A/Bs the RPX arm ran a +//! KECCAK transcript, and no instrument disagreed. +//! +//! # So every assertion below is two-sided +//! +//! Each arm asserts both that its own counters MOVED and that the other arm's +//! are ZERO. One half alone is worthless: "rpx > 0, keccak 0" is equally true +//! of a run where the keccak transcript was never instrumented, which is the +//! state this file is about. +//! +//! # ⚠ Its own binary, AND a lock +//! +//! The counters are process-global, so both are needed and neither is enough. +//! +//! The binary, because `crypto`'s lib-test binary runs tests in parallel and +//! several of them hash: an exact assertion there is an assertion about +//! whatever else happened to be running — a keccak arm read 215 squeezes of +//! which 200 were keccak, purely from neighbours. +//! +//! The lock, because an integration test binary ALSO runs its own tests in +//! parallel. Moving the file and stopping there left four of five failing, one +//! reading `left: 2, right: 0` where a sibling had reset the counters between +//! this test's `reset` and its `snapshot`. Every test below takes +//! [`serialise`] for its whole reset-measure-assert window. + +#![cfg(feature = "hash-metrics")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::transcript_hash::{KeccakTranscriptHash, RpxTranscriptHash}; +use crypto::hash::rpx::Rpx256Digest; +use crypto::hash_metrics; +use digest::Update; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + +/// ★ Taken by every test here: `reset` and `snapshot` address one global pair +/// of counters, so a measurement is only this test's while it holds this. +/// +/// Poisoning is ignored so one failure does not cascade into unrelated tests. +static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serialise() -> std::sync::MutexGuard<'static, ()> { + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Four absorbs and two squeezes, driven identically on either configuration. +/// +/// `new` absorbs once, so the count is `1 + 3`. +fn drive(seed: &[u8]) -> (u64, u64) +where + T: crypto::fiat_shamir::transcript_hash::TranscriptHash, +{ + let mut t = DefaultTranscript::::new(seed); + t.append_bytes(b"one"); + t.append_bytes(b"two"); + t.append_bytes(b"three"); + let _ = t.sample(); + let _ = t.sample(); + (4, 2) +} + +#[test] +fn a_keccak_transcript_counts_as_keccak_and_nothing_else() { + let _serialised = serialise(); + hash_metrics::reset(); + let (absorbs, squeezes) = drive::(b"seed"); + let c = hash_metrics::snapshot(); + + assert_eq!(c.transcript_absorbs_keccak, absorbs); + assert_eq!(c.transcript_squeezes_keccak, squeezes); + // The other arm must be silent — and it must be silent because nothing RPX + // ran, which the totals below are what establish. + assert_eq!(c.transcript_absorbs_rpx, 0); + assert_eq!(c.transcript_squeezes_rpx, 0); + assert_eq!(c.transcript_absorbs, absorbs); + assert_eq!(c.transcript_squeezes, squeezes); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +#[test] +fn an_rpx_transcript_counts_as_rpx_and_nothing_else() { + let _serialised = serialise(); + hash_metrics::reset(); + let (absorbs, squeezes) = drive::(b"seed"); + let c = hash_metrics::snapshot(); + + assert_eq!( + c.transcript_absorbs_rpx, absorbs, + "the RPX transcript absorbed {} times and the counter saw {} — an \ + un-instrumented sponge reads zero, which is indistinguishable from \ + one that never ran", + absorbs, c.transcript_absorbs_rpx + ); + assert_eq!(c.transcript_squeezes_rpx, squeezes); + assert_eq!( + c.transcript_absorbs_keccak, 0, + "a keccak absorb during an RPX-only run: the transcript is not the \ + configuration's" + ); + assert_eq!(c.transcript_squeezes_keccak, 0); + assert_eq!(c.transcript_absorbs, absorbs); + assert_eq!(c.transcript_squeezes, squeezes); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +/// ★ The two configurations do the SAME amount of transcript work. +/// +/// The counters must differ only in which bucket they land in. If a swap +/// changed the absorb count, the per-arm line would be reporting a protocol +/// difference as a hash difference — and a reader comparing arms would draw +/// the wrong conclusion about what the hash costs. +#[test] +fn the_two_configurations_do_the_same_transcript_work() { + let _serialised = serialise(); + hash_metrics::reset(); + drive::(b"same"); + let k = hash_metrics::snapshot(); + + hash_metrics::reset(); + drive::(b"same"); + let r = hash_metrics::snapshot(); + + assert_eq!(k.transcript_absorbs, r.transcript_absorbs); + assert_eq!(k.transcript_squeezes, r.transcript_squeezes); +} + +/// ★★ The transcript counter counts `Update::update` calls, tied to the +/// counter that already did. +/// +/// On a keccak-only run `absorb_calls` sees every `update` the sponge receives +/// and this file's counter sees every one the TRANSCRIPT issues. They differ by +/// exactly the chaining re-absorb — `sample` feeds its own output back in, once +/// per squeeze, which is part of squeezing and not an absorb anyone asked for. +/// So: +/// +/// ```text +/// absorb_calls == transcript_absorbs + transcript_squeezes +/// ``` +/// +/// This is the assertion that makes the new counters non-vacuous against +/// something that predates them, and it fails if either counter is moved, +/// double-counted, or attached to the wrong call. +/// +/// ⚠ SCOPE: state reads are deliberately NOT a term here, and adding one would +/// be wrong rather than more complete. `state()` finalizes a clone; it issues +/// no `update`, so it cannot move `absorb_calls` and has no business in an +/// identity about absorbs. It is counted separately by +/// [`a_state_read_is_counted_separately_from_a_squeeze`], where its own control +/// is the grind count. +/// +/// Lane V1's closed form for the block, in these terms and satisfying this +/// identity by construction: 582,703 absorbs + 182,734 squeezes == 765,437 +/// `absorb_calls`, with the 2,996 state finalizes outside all three. +/// +/// ⚠ An earlier version of this test asserted a cubic element absorbs in +/// "several chunks". It does not: `stream_bytes` for the degree-3 extension +/// writes one 24-byte buffer and calls the sink ONCE +/// (`extensions_goldilocks.rs:567-571`). The test failed, which is how the +/// claim — and a comment in `default_transcript.rs` repeating it — got fixed. +#[test] +fn the_transcript_absorbs_agree_with_the_generic_absorb_counter() { + let _serialised = serialise(); + hash_metrics::reset(); + let (absorbs, squeezes) = drive::(b"tie"); + let c = hash_metrics::snapshot(); + + assert_eq!( + c.absorb_calls, + c.transcript_absorbs + c.transcript_squeezes, + "the keccak sponge saw {} updates; the transcript issued {} absorbs and \ + {} squeezes, and a squeeze chains exactly one update", + c.absorb_calls, + c.transcript_absorbs, + c.transcript_squeezes + ); + assert_eq!( + (c.transcript_absorbs, c.transcript_squeezes), + (absorbs, squeezes) + ); +} + +/// ★ A field element costs the same absorbs on either arm. +/// +/// The counters must differ only in which bucket they land in: if a hash swap +/// changed the absorb count, the per-arm line would report a protocol +/// difference as a hash difference. +#[test] +fn a_field_element_absorb_costs_the_same_on_both_arms() { + let _serialised = serialise(); + let element = FieldElement::::from(7u64); + + hash_metrics::reset(); + let mut k = DefaultTranscript::::new(&[]); + k.append_field_element(&element); + let k = hash_metrics::snapshot(); + + hash_metrics::reset(); + let mut r = DefaultTranscript::::new(&[]); + r.append_field_element(&element); + let r = hash_metrics::snapshot(); + + assert_eq!( + k.transcript_absorbs_keccak, r.transcript_absorbs_rpx, + "the two arms disagree on how much one element absorbs" + ); + assert_eq!(k.transcript_absorbs_rpx, 0); + assert_eq!(r.transcript_absorbs_keccak, 0); +} + +/// ★★ …and the GENERIC counters are no longer keccak-only either. +/// +/// `count_absorb` had exactly one call site — the keccak wrapper's `update` — +/// so `absorb_calls` and `absorb_bytes` read ZERO for an RPX proof, and +/// `total`'s documentation ("every finalize … transcript squeeze") was false +/// for this sponge. This drives the digest directly, so it fails if either call +/// is removed. +#[test] +fn the_rpx_digest_bumps_the_generic_counters() { + let _serialised = serialise(); + hash_metrics::reset(); + let mut d = Rpx256Digest::default(); + Update::update(&mut d, b"twelve bytes"); + Update::update(&mut d, b"and more"); + let after_absorbs = hash_metrics::snapshot(); + + assert_eq!( + after_absorbs.absorb_calls, 2, + "absorbing into an RPX digest moved `absorb_calls` to {} — it read 0 \ + before this was instrumented, for every RPX proof ever measured", + after_absorbs.absorb_calls + ); + assert_eq!(after_absorbs.absorb_bytes, 12 + 8); + assert_eq!( + after_absorbs.total, 0, + "nothing has been finalized yet, so `total` must not have moved" + ); + + use digest::FixedOutputReset; + let mut out = digest::Output::::default(); + d.finalize_into_reset(&mut out); + assert_eq!( + hash_metrics::snapshot().total, + 1, + "an RPX finalize did not reach `total`, whose own doc says it counts \ + every finalize" + ); +} + +/// ★★★ A `state()` IS COUNTED, AND IT IS NOT A SQUEEZE. +/// +/// The distinction lane V1's closed form turns on. `sample` is a +/// `finalize_reset` whose output is chained back in, so it advances the +/// transcript; `state` finalizes a CLONE and changes nothing. On a block proof +/// they are 182,734 and 2,996 — a counter hooked only to `finalize_reset` +/// misses every one of the 2,996, which is precisely what this file's first +/// version did. +/// +/// Both directions are asserted, because either conflation is a live failure +/// mode: a state must not appear as a squeeze, AND a squeeze must not appear as +/// a state. Counting their sum would satisfy neither of V1's two numbers. +#[test] +fn a_state_read_is_counted_separately_from_a_squeeze() { + let _serialised = serialise(); + + hash_metrics::reset(); + let mut t = DefaultTranscript::::new(b"state"); + let before = t.state(); + let c = hash_metrics::snapshot(); + assert_eq!( + (c.transcript_states, c.transcript_states_rpx), + (1, 1), + "a state() read was not counted" + ); + assert_eq!( + c.transcript_squeezes, 0, + "a state() read was counted as a squeeze — it finalizes a clone and \ + advances nothing, so a squeeze count including it cannot be checked \ + against a closed form" + ); + + // …and it really did not advance the chain, which is why it is a different + // number rather than a different name for the same one. + let again = t.state(); + assert_eq!(before, again, "state() advanced the transcript"); + assert_eq!(hash_metrics::snapshot().transcript_states, 2); + + // The converse: a squeeze is not counted as a state. + hash_metrics::reset(); + let _ = t.sample(); + let c = hash_metrics::snapshot(); + assert_eq!( + (c.transcript_squeezes, c.transcript_squeezes_rpx), + (1, 1), + "a squeeze was not counted" + ); + assert_eq!( + c.transcript_states, 0, + "a squeeze was counted as a state read" + ); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +/// ★ …and the keccak arm tags its state reads too. +/// +/// One side is not evidence: "rpx states > 0, keccak 0" is equally true when +/// the keccak path was never instrumented. +#[test] +fn a_keccak_state_read_is_tagged_as_keccak() { + let _serialised = serialise(); + + hash_metrics::reset(); + let t = DefaultTranscript::::new(b"state"); + let _ = t.state(); + let c = hash_metrics::snapshot(); + + assert_eq!((c.transcript_states, c.transcript_states_keccak), (1, 1)); + assert_eq!(c.transcript_states_rpx, 0); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +// ========================================================================= +// W1-B: where the derived root goes +// ========================================================================= + +/// ★★★ THE ABSORB POSITION — the orderings are distinguishable, and the +/// challenge sees it. +/// +/// W1-B adds one root the proof does not carry: the verifier derives it from +/// the ELF and absorbs it AFTER the roots the proof does carry, and BEFORE any +/// challenge. Two guards were proposed for that ordering and neither reaches +/// it: +/// +/// * a transcript-count gate is **blind to position** — an absorb lands at +/// `out_pos == SQUEEZE_LEN` wherever it goes, so first and last cost an +/// identical `+1 absorb, +0 squeezes`. Demonstrated at the end of this test +/// rather than asserted in prose; +/// * the WHIR byte gate is **blind to the feature** — its fixture is a single +/// EQ air (`whir_byte_gate.rs:167,190`), so it commits no DECODE table, no +/// preprocessed group and no derived root at all. +/// +/// Nothing downstream catches it either: prover and verifier share the +/// position, so a consistently wrong one still verifies. **A round-trip between +/// two halves that share code is not evidence about either.** +/// +/// # WHAT THIS TEST DOES NOT DO +/// +/// It does **not** call the production roots block, so it cannot tell you the +/// verifier absorbs anything at all. It pins that the specified order is +/// *distinguishable* from every wrong one — including from omitting the root +/// entirely — which is the property the count gate lacks and which has to hold +/// before a wiring guard can mean anything. +/// +/// The wiring guard arrives at **step 3**, when `multi_verify`'s roots block +/// exists to be called; its mutation is deleting the absorb, and that is the +/// failure this branch has produced twice — the transcript defaulting to keccak +/// while everything else moved, and the grind dispatching keccak-only with a +/// correct RPX kernel sitting unused. **Both were the feature not wired, not +/// the feature wired wrongly.** Until step 3, this file pins the specification +/// and the design note's ordering section is the normative statement. +/// What one run of the roots-block script leaves behind: the sponge state, and +/// the first challenge drawn from it. Named because the two travel together — +/// a failure in the state and a failure in the challenge mean different things. +type Draw = ([u8; 32], FieldElement); + +#[test] +fn the_derived_root_is_absorbed_after_the_carried_ones() { + let _serialised = serialise(); + + // Stand-ins: what matters is the ORDER, not the values. + let carried: [[u8; 32]; 3] = [[0x11; 32], [0x22; 32], [0x33; 32]]; + let derived = [0xAB; 32]; + + // The script runs past the absorbs to the FIRST CHALLENGE, because that is + // where the property has teeth: "every root is in the transcript before the + // first challenge". Pinning the state alone would pin the absorbs' order + // among themselves and leave "absorbed AFTER z" — the genuinely dangerous + // variant — unrepresentable. + // + // `z` comes back beside the state so a failure localises: "the challenge + // moved" reads differently from "the state moved". + let script = |order: &[&[u8; 32]], after_z: Option<&[u8; 32]>| -> Draw { + let mut t = DefaultTranscript::::new(b"w1b-roots"); + for root in order { + t.append_bytes(*root); + } + let z: FieldElement = t.sample_field_element(); + if let Some(late) = after_z { + t.append_bytes(late); + } + (IsTranscript::::state(&t), z) + }; + + let all = [&carried[0], &carried[1], &carried[2], &derived]; + let (specified, z_specified) = script(&all, None); + + // Every wrong placement, including the two that are not reorderings. + let wrong: [(&str, Draw); 4] = [ + ( + "first", + script(&[&derived, &carried[0], &carried[1], &carried[2]], None), + ), + ( + "middle", + script(&[&carried[0], &derived, &carried[1], &carried[2]], None), + ), + // OMITTED — the failure mode that is not a reordering at all. + ( + "omitted", + script(&[&carried[0], &carried[1], &carried[2]], None), + ), + // AFTER the challenge — unrepresentable without the `z` draw above. + ( + "after z", + script(&[&carried[0], &carried[1], &carried[2]], Some(&derived)), + ), + ]; + + for (name, (state, z)) in &wrong { + assert_ne!( + specified, *state, + "placing the derived root {name} gives the same sponge state as the \ + specified order — the transcript cannot distinguish them, so this \ + test pins nothing" + ); + if *name != "after z" { + assert_ne!( + z_specified, *z, + "placing the derived root {name} draws the SAME challenge — the \ + root is not binding z, which is the whole point of absorbing it \ + before any challenge" + ); + } + } + + // ★★ THE DANGER, STATED AS AN EQUALITY: absorbing after the challenge is + // indistinguishable IN THE CHALLENGE from not absorbing at all. Both draw + // `z` from a transcript holding only the carried roots, so the derived root + // binds nothing — while the counters happily see it and the sponge state + // afterwards differs from the omitted case, which is what makes it look + // wired when it is not. + // + // ⚠ An earlier draft asserted this against `z_specified` with the comment + // "the same z BY CONSTRUCTION". That was wrong and the test said so: the + // specified order absorbs FOUR roots before drawing, the late order absorbs + // three, so the two draws differ. The equality that holds — and the one + // worth pinning — is against the OMITTED case. + assert_eq!( + wrong[3].1.1, wrong[2].1.1, + "absorbing after the challenge draws a different z from omitting the \ + root entirely — then a late absorb would be detectable in z, and the \ + reason this ordering is dangerous is not the one stated here" + ); + + // The count gate's blindness, demonstrated rather than asserted. + hash_metrics::reset(); + let _ = script(&all, None); + let last = hash_metrics::snapshot(); + hash_metrics::reset(); + let _ = script(&[&derived, &carried[0], &carried[1], &carried[2]], None); + let first = hash_metrics::snapshot(); + assert_eq!( + (last.transcript_absorbs, last.transcript_squeezes), + (first.transcript_absorbs, first.transcript_squeezes), + "the two orders differ in counters after all — then a count gate DOES see \ + position and this test's premise is wrong" + ); +} + +// ------------------------------------------------------------------------- +// Window alignment +// ------------------------------------------------------------------------- + +/// ★★ A ZERO IS ONLY A MEASUREMENT IF SOMETHING CAN MAKE IT NON-ZERO. +/// +/// `transcript_misaligned_absorbs_after_statement` is expected to read 0 over a +/// real proof, which is exactly the shape of a counter nobody bumps. So this +/// drives a transcript to a KNOWN misalignment and requires a non-zero count. +/// +/// The fixture is not invented: 13 seed bytes then a 32-byte root is the WHIR +/// byte gate's own first window, and its header already records that window as +/// NOT aligned and warns against quoting it as a witness that a real proof's +/// is. Here that same stream is the positive control. +/// +/// The count is an EQUALITY rather than `> 0`, so a counter that fired on every +/// absorb rather than on the misaligned ones fails too: of the three absorbs +/// below, the seed opens the window at 0 and is aligned, and the two roots +/// after it start at 13 and 45 and are not. +#[test] +fn a_window_that_opens_off_a_boundary_is_counted() { + let _serialised = serialise(); + hash_metrics::reset(); + + let mut t = DefaultTranscript::::new(b"whir-identity"); + // The counter reports what happens AFTER a statement, so a fixture with no + // statement has to say where one ended. The 13 seed bytes are already + // absorbed, and the mark does not move the window, so the two roots are + // still the misaligned pair this test is about. + t.mark_statement_end(); + t.append_bytes(&[7u8; 32]); + t.append_bytes(&[9u8; 32]); + let c = hash_metrics::snapshot(); + + assert_eq!( + c.transcript_absorbs, 3, + "the seed absorbs once, then two roots" + ); + assert_eq!( + c.transcript_misaligned_absorbs_after_statement, 2, + "13 seed bytes leave the window at 13, so both roots start off a field \ + element boundary; only the seed itself, at offset 0, is aligned" + ); +} + +/// ★★ THE MARK DOES NOT FORGIVE A BAD PAD, which is the line between a sharper +/// counter and a switched-off one. +/// +/// `mark_statement_end` could have reset the window to zero, and then the first +/// absorb after any statement would be aligned BY CONSTRUCTION — including when +/// the padding is broken, which is the one case the counter exists for. It sets +/// a flag and leaves the offset alone, so a statement that ended at 3 mod 8 is +/// reported. +#[test] +fn a_statement_that_ends_off_a_boundary_is_still_counted() { + let _serialised = serialise(); + hash_metrics::reset(); + + let mut t = DefaultTranscript::::new(&[]); + // A "statement" that forgot its pad: three bytes, and the window is at 3. + t.append_bytes(&[1u8; 3]); + t.mark_statement_end(); + t.append_bytes(&[2u8; 32]); + let c = hash_metrics::snapshot(); + + assert_eq!( + c.transcript_misaligned_absorbs_after_statement, 1, + "the statement ended at 3 mod 8 and the absorb after it straddles two \ + field elements; a mark that zeroed the window would report none" + ); +} + +/// ★★ ...and a stream that IS aligned reads zero, so the counter is not simply +/// counting absorbs. +/// +/// Eight-byte absorbs from an empty seed keep the window at a multiple of 8 +/// throughout. Both halves are needed: the test above alone is satisfied by a +/// counter that fires on everything, and this one alone by a counter that fires +/// on nothing. +#[test] +fn an_aligned_stream_counts_no_misalignment() { + let _serialised = serialise(); + hash_metrics::reset(); + + let mut t = DefaultTranscript::::new(&[]); + // Empty seed, so the window is at 0 and the mark changes nothing about + // alignment — it only switches the counting on. + t.mark_statement_end(); + for _ in 0..4 { + t.append_bytes(&[1u8; 8]); + } + let c = hash_metrics::snapshot(); + + assert_eq!( + c.transcript_absorbs, 5, + "the empty seed absorbs once, then four values" + ); + assert_eq!( + c.transcript_misaligned_absorbs_after_statement, 0, + "every absorb started on a multiple of 8" + ); +} + +/// ★★ A SQUEEZE OPENS A NEW WINDOW, and it opens it ALIGNED. +/// +/// `sample` finalize-resets the sponge and re-absorbs its own 32-byte output, +/// so the next window starts at 32 — a multiple of 8. A misaligned window is +/// therefore repaired by the next squeeze, which is why the padding only has to +/// fix the window the roots land in and not the whole transcript. +/// +/// Without this, the window field could carry a stale offset across a squeeze +/// and every post-squeeze absorb of an odd-length stream would be reported +/// misaligned forever. +#[test] +fn a_squeeze_opens_an_aligned_window() { + let _serialised = serialise(); + hash_metrics::reset(); + + let mut t = DefaultTranscript::::new(b"odd"); + t.mark_statement_end(); + // Off a boundary: 3 seed bytes, so this one is counted. + t.append_bytes(&[5u8; 32]); + let _ = t.sample(); + // The window reopened at 32; these are not. + t.append_bytes(&[6u8; 32]); + t.append_bytes(&[7u8; 8]); + let c = hash_metrics::snapshot(); + + assert_eq!( + c.transcript_misaligned_absorbs_after_statement, 1, + "only the absorb before the squeeze started off a boundary; the squeeze \ + reopens the window at 32, which is aligned" + ); +} diff --git a/crypto/ecsm/src/ecrec.rs b/crypto/ecsm/src/ecrec.rs new file mode 100644 index 000000000..eee70137c --- /dev/null +++ b/crypto/ecsm/src/ecrec.rs @@ -0,0 +1,510 @@ +//! ECDSA public-key recovery in one call: the reference the executor runs and the +//! witness the ECREC chip proves. +//! +//! The call takes a signature `(r, s)`, a message hash `z` (all 32-byte big-endian, +//! as Ethereum carries them) and the parity of `R`'s `y`, and returns the public key +//! `Q = r⁻¹·(s·R − z·G) = u1·G + u2·R` with `u1 = −z·r⁻¹`, `u2 = s·r⁻¹ (mod N)`, or +//! *bails*: it reports that it computed nothing and the guest recovers the key in +//! software. A bail is always sound (the guest's software path decides), so the chip +//! only has to prove the key when it reports one. The honest executor bails exactly +//! when the chip cannot represent the answer: +//! +//! - `r` or `s` is not in `[1, N)`; +//! - `r` is not the x-coordinate of a curve point (`r³ + 7` has no square root); +//! - `u1 = 0` (`z ≡ 0 mod N`), which the first chain cannot take as a scalar; +//! - `x(u1·G) = x(u2·R)`, where the final chord addition is undefined (this covers +//! `Q` at infinity). +//! +//! The chip proves the key as two scalar multiplications through the ECSM machinery's +//! `Ecdas` chains (`A = u1·G` keyed at the call's timestamp `T`, `B = u2·R` at +//! `T + 1`) and one ECDAS addition row `Q = A + B` at `T + 2`, plus the scalar +//! arithmetic mod `N` that turns `(r, s, z)` into `(u1, u2)` and the decompression +//! of `R`. + +use k256::elliptic_curve::PrimeField; +use k256::elliptic_curve::ops::Reduce; +use k256::elliptic_curve::point::DecompressPoint; +use k256::elliptic_curve::sec1::ToEncodedPoint; +use num_bigint::{BigInt, BigUint}; + +use crate::curve::{AffinePoint, StepPts, msb_position}; +use crate::witness::{ + build_step, carries_x2, carries_yg, conv, ext64, limb_carries, shifted_quotient, to_le_33, + witness_for_point, +}; +use crate::{B, EcdasStep, P_BYTES, R_BYTES, n, p, to_le_32}; + +/// Status the call writes when it computed the key. +pub const ECREC_STATUS_KEY: u8 = 0; +/// Status the call writes when it bailed (the key bytes are zero). +pub const ECREC_STATUS_BAIL: u8 = 1; + +/// The values one non-bailing recovery uses. +struct Recovered { + /// `r⁻¹ mod N` + w: BigUint, + /// `u1 = −z·w mod N`, in `[1, N)` + u1: BigUint, + /// `u2 = s·w mod N`, in `[1, N)` + u2: BigUint, + /// `R = (r, y)` with the requested parity + r_point: AffinePoint, + /// `u1·G` + a: AffinePoint, + /// `u2·R` + b: AffinePoint, + /// `A + B` + q: AffinePoint, +} + +fn affine(p: &k256::AffinePoint) -> Option { + let ep = p.to_encoded_point(false); + Some(AffinePoint { + x: BigUint::from_bytes_be(ep.x()?), + y: BigUint::from_bytes_be(ep.y()?), + }) +} + +fn scalar_big(s: &k256::Scalar) -> BigUint { + BigUint::from_bytes_be(&s.to_bytes()) +} + +/// The recovery, or `None` where the call bails (see the module documentation). +fn recover(sig_be: &[u8; 64], msg_be: &[u8; 32], parity: u8) -> Option { + let mut r_be = [0u8; 32]; + let mut s_be = [0u8; 32]; + r_be.copy_from_slice(&sig_be[..32]); + s_be.copy_from_slice(&sig_be[32..]); + let r = Option::::from(k256::Scalar::from_repr(r_be.into()))?; + let s = Option::::from(k256::Scalar::from_repr(s_be.into()))?; + if bool::from(r.is_zero()) || bool::from(s.is_zero()) { + return None; + } + // r < N < p, so r is a canonical x; the parity selects the root. + let r_point = Option::::from(k256::AffinePoint::decompress( + &r_be.into(), + (parity & 1).into(), + ))?; + let z = >::reduce_bytes(&(*msg_be).into()); + let w = Option::::from(r.invert())?; + let u1 = -(z * w); + let u2 = s * w; + if bool::from(u1.is_zero()) { + return None; + } + let a = (k256::ProjectivePoint::GENERATOR * u1).to_affine(); + let b = (k256::ProjectivePoint::from(r_point) * u2).to_affine(); + let (a, b) = (affine(&a)?, affine(&b)?); + if a.x == b.x { + return None; + } + let q = (k256::ProjectivePoint::from(r_point) * u2 + k256::ProjectivePoint::GENERATOR * u1) + .to_affine(); + Some(Recovered { + w: scalar_big(&w), + u1: scalar_big(&u1), + u2: scalar_big(&u2), + r_point: affine(&r_point)?, + a, + b, + q: affine(&q)?, + }) +} + +/// 32 big-endian bytes of a value below 2^256. +fn be32(v: &BigUint) -> [u8; 32] { + let mut out = to_le_32(v); + out.reverse(); + out +} + +/// The executor's entry point: the key `(Q.x, Q.y)` as 32-byte big-endian +/// coordinates, or `None` where the call bails. `parity` is the requested parity of +/// `R`'s y (the recovery id's low bit). +pub fn ecrecover(sig_be: &[u8; 64], msg_be: &[u8; 32], parity: u8) -> Option<([u8; 32], [u8; 32])> { + let rec = recover(sig_be, msg_be, parity)?; + Some((be32(&rec.q.x), be32(&rec.q.y))) +} + +/// Full ECREC-chip witness for one call (one ECREC row) plus its ECDAS rows. +/// Every 256-bit quantity is 32 little-endian bytes. On a bail (`ok == false`) only +/// the inputs are set; every other field is zero and every list empty. +#[derive(Debug, Clone)] +pub struct EcrecWitness { + /// `r`, `s` and `z` as read from memory, little-endian limbs. + pub r: [u8; 32], + pub s: [u8; 32], + pub z: [u8; 32], + /// The requested parity of `R`'s y. + pub parity: u8, + /// Whether the call computed the key (status 0) or bailed (status 1). + pub ok: bool, + /// `w = r⁻¹ mod N`; `r·w − 1 − qw·N = 0`. + pub w: [u8; 32], + pub qw: [u8; 32], + pub cw: [i64; 64], + /// `u1 = −z·w mod N`; `z·w + u1 − qz·N = 0`. + pub u1: [u8; 32], + pub qz: [u8; 32], + pub cz: [i64; 64], + /// `u2 = s·w mod N`; `s·w − u2 − qu·N = 0`. + pub u2: [u8; 32], + pub qu: [u8; 32], + pub cu: [i64; 64], + /// Positions of the most significant set bits of `u1` and `u2`. + pub len1: u8, + pub len2: u8, + /// `(v − N) mod 2^256` for the four `< N` checks. + pub r_sub_n: [u8; 32], + pub s_sub_n: [u8; 32], + pub u1_sub_n: [u8; 32], + pub u2_sub_n: [u8; 32], + /// `R`'s y and the curve relations on `(r, y_R)`, as in ECSM. + pub y_r: [u8; 32], + pub x2: [u8; 32], + pub q0: [u8; 32], + pub c0: [i64; 64], + pub q1: [u8; 33], + pub c1: [i64; 64], + pub y_r_sub_p: [u8; 32], + /// `y_R[0] = 2·h + parity`. + pub h: u8, + /// The two chains' results `A = u1·G`, `B = u2·R`. + pub x_a: [u8; 32], + pub y_a: [u8; 32], + pub x_b: [u8; 32], + pub y_b: [u8; 32], + pub x_a_sub_p: [u8; 32], + pub x_b_sub_p: [u8; 32], + /// The key `Q = A + B`. + pub x_q: [u8; 32], + pub y_q: [u8; 32], + pub x_q_sub_p: [u8; 32], + pub y_q_sub_p: [u8; 32], + /// ECDAS rows: the first chain (timestamp `T`), the second (`T + 1`), the final + /// addition (`T + 2`). + pub chain1: Vec, + pub chain2: Vec, + pub add: Option, +} + +impl EcrecWitness { + fn bail(r: [u8; 32], s: [u8; 32], z: [u8; 32], parity: u8) -> Self { + Self { + r, + s, + z, + parity, + ok: false, + w: [0; 32], + qw: [0; 32], + cw: [0; 64], + u1: [0; 32], + qz: [0; 32], + cz: [0; 64], + u2: [0; 32], + qu: [0; 32], + cu: [0; 64], + len1: 0, + len2: 0, + r_sub_n: [0; 32], + s_sub_n: [0; 32], + u1_sub_n: [0; 32], + u2_sub_n: [0; 32], + y_r: [0; 32], + x2: [0; 32], + q0: [0; 32], + c0: [0; 64], + q1: [0; 33], + c1: [0; 64], + y_r_sub_p: [0; 32], + h: 0, + x_a: [0; 32], + y_a: [0; 32], + x_b: [0; 32], + y_b: [0; 32], + x_a_sub_p: [0; 32], + x_b_sub_p: [0; 32], + x_q: [0; 32], + y_q: [0; 32], + x_q_sub_p: [0; 32], + y_q_sub_p: [0; 32], + chain1: Vec::new(), + chain2: Vec::new(), + add: None, + } + } + + /// The status byte the call writes. + pub fn status(&self) -> u8 { + if self.ok { + ECREC_STATUS_KEY + } else { + ECREC_STATUS_BAIL + } + } +} + +/// The generator's coordinates, little-endian. +pub fn generator_le() -> ([u8; 32], [u8; 32]) { + let g = affine(&k256::AffinePoint::GENERATOR).expect("G is not the identity"); + (to_le_32(&g.x), to_le_32(&g.y)) +} + +/// The ECDAS rows of the double-and-add chain of `k·P` for a full point `P = (x, y)` +/// (little-endian), and its result; `None` unless `0 < k < N` and `P` is a canonical +/// curve point. +pub fn ecdas_chain( + k_le: &[u8; 32], + x_le: &[u8; 32], + y_le: &[u8; 32], +) -> Option<(Vec, [u8; 32], [u8; 32])> { + let k = BigUint::from_bytes_le(k_le); + if k == BigUint::from(0u8) || k >= n() { + return None; + } + let pt = AffinePoint { + x: BigUint::from_bytes_le(x_le), + y: BigUint::from_bytes_le(y_le), + }; + let pp = p(); + if pt.x >= pp || pt.y >= pp || (&pt.y * &pt.y) % &pp != (&pt.x * &pt.x * &pt.x + B) % &pp { + return None; + } + let w = witness_for_point(&k, &pt, k_le); + Some((w.steps, w.x_r, w.y_r)) +} + +/// The ECDAS addition row `A + B` (round 0, `op` 1, `next_op` 0) for two curve +/// points given little-endian, with its result in `x_r`, `y_r`; `None` when +/// `xA = xB` (the chord is undefined). +pub fn ecdas_add_step( + xa: &[u8; 32], + ya: &[u8; 32], + xb: &[u8; 32], + yb: &[u8; 32], +) -> Option { + let pp = p(); + let big = |b: &[u8; 32]| BigUint::from_bytes_le(b); + let (a, b) = ( + AffinePoint { + x: big(xa), + y: big(ya), + }, + AffinePoint { + x: big(xb), + y: big(yb), + }, + ); + if a.x == b.x { + return None; + } + let dy = (&pp + &b.y - &a.y) % &pp; + let dx = (&pp + &b.x - &a.x) % &pp; + let lambda = dy * dx.modpow(&(&pp - BigUint::from(2u8)), &pp) % &pp; + let x = (&lambda * &lambda + &pp * 2u8 - &a.x - &b.x) % &pp; + let y = (&lambda * ((&pp + &a.x - &x) % &pp) + &pp - &a.y) % &pp; + let q = AffinePoint { x, y }; + Some(build_step( + &StepPts { + a, + g: b, + round: 0, + op: 1, + next_op: 0, + r: q, + lambda, + }, + &BigInt::from(pp.clone()), + &BigInt::from(BigUint::from_bytes_le(&R_BYTES)), + &ext64(&R_BYTES), + &ext64(&P_BYTES), + )) +} + +/// Little-endian limbs of a big-endian 32-byte value. +fn le_of_be(be: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + for (i, b) in be.iter().rev().enumerate() { + out[i] = *b; + } + out +} + +/// `(v − m) mod 2^256` as 32 little-endian bytes. +fn sub_mod_2_256(v: &BigUint, m: &BigUint) -> [u8; 32] { + let two_256 = BigUint::from(1u8) << 256u32; + to_le_32(&((two_256 + v) - m)) +} + +/// Witness of `a·b + sign·c − q·m = 0` over 32-byte limbs (`sign` = ±1): the quotient +/// `q` (asserted to be a non-negative integer that fits 32 bytes) and the 64 carries. +fn product_relation( + relation: &str, + a: &BigUint, + b: &BigUint, + c: &BigUint, + sign: i8, + m: &BigUint, +) -> ([u8; 32], [i64; 64]) { + let num = if sign > 0 { a * b + c } else { a * b - c }; + assert!( + (&num % m) == BigUint::from(0u8), + "ECREC witness {relation}: not divisible by the modulus" + ); + let q = &num / m; + assert!( + q.bits() <= 256, + "ECREC witness {relation}: quotient exceeds 32 bytes" + ); + let (ae, be, ce, qe, me) = ( + ext64(&to_le_32(a)), + ext64(&to_le_32(b)), + ext64(&to_le_32(c)), + ext64(&to_le_32(&q)), + ext64(&to_le_32(m)), + ); + let mut terms = [0i128; 64]; + for (i, t) in terms.iter_mut().enumerate() { + *t = conv(&ae, &be, i) + i128::from(sign) * ce[i] - conv(&qe, &me, i); + } + (to_le_32(&q), limb_carries(relation, &terms)) +} + +/// Computes the ECREC witness for one call: the inputs exactly as read from memory +/// (`r ‖ s` and `z` big-endian), and `parity` (0 or 1). +pub fn compute_ecrec_witness(sig_be: &[u8; 64], msg_be: &[u8; 32], parity: u8) -> EcrecWitness { + let r_le = le_of_be(&sig_be[..32]); + let s_le = le_of_be(&sig_be[32..]); + let z_le = le_of_be(msg_be); + let Some(rec) = recover(sig_be, msg_be, parity) else { + return EcrecWitness::bail(r_le, s_le, z_le, parity); + }; + let (nn, pp_big) = (n(), p()); + let r = BigUint::from_bytes_le(&r_le); + let s = BigUint::from_bytes_le(&s_le); + let z = BigUint::from_bytes_le(&z_le); + let one = BigUint::from(1u8); + + // --- scalars mod N --- + let (qw, cw) = product_relation("w", &r, &rec.w, &one, -1, &nn); + let (qz, cz) = product_relation("u1", &z, &rec.w, &rec.u1, 1, &nn); + let (qu, cu) = product_relation("u2", &s, &rec.w, &rec.u2, -1, &nn); + + // --- R on the curve, as ECSM proves (xG, yG): x2 = r² mod p, yR² + p² − r·x2 − b − q1·p = 0 --- + let rp = &rec.r_point; + let x_sq = &rp.x * &rp.x; + let x2_big = &x_sq % &pp_big; + let q0_big = (&x_sq - &x2_big) / &pp_big; + let p_ext = ext64(&P_BYTES); + let (x_b, x2_b, q0_b, yr_b) = ( + to_le_32(&rp.x), + to_le_32(&x2_big), + to_le_32(&q0_big), + to_le_32(&rp.y), + ); + let c0 = carries_x2(&ext64(&x_b), &ext64(&x2_b), &ext64(&q0_b), &p_ext); + let p_signed = BigInt::from(pp_big.clone()); + let num_yg = BigInt::from(&rp.y * &rp.y) - BigInt::from(&rp.x * &x2_big) - BigInt::from(B); + let q1_big = shifted_quotient("ecrec yR", &num_yg, &p_signed, &p_signed); + let q1_b = to_le_33("ecrec yR", &q1_big); + let b_ext = { + let mut a = [0u8; 32]; + a[0] = B as u8; + ext64(&a) + }; + let c1 = carries_yg( + &ext64(&yr_b), + &p_ext, + &ext64(&x2_b), + &ext64(&x_b), + &ext64(&q1_b), + &b_ext, + ); + assert_eq!(yr_b[0] & 1, parity & 1, "R's y has the requested parity"); + + // --- the two chains --- + let g = affine(&k256::AffinePoint::GENERATOR).expect("G is not the identity"); + let u1_le = to_le_32(&rec.u1); + let u2_le = to_le_32(&rec.u2); + let chain1 = witness_for_point(&rec.u1, &g, &u1_le); + let chain2 = witness_for_point(&rec.u2, rp, &u2_le); + assert_eq!( + (chain1.x_r, chain1.y_r), + (to_le_32(&rec.a.x), to_le_32(&rec.a.y)), + "the first chain ends at u1·G" + ); + assert_eq!( + (chain2.x_r, chain2.y_r), + (to_le_32(&rec.b.x), to_le_32(&rec.b.y)), + "the second chain ends at u2·R" + ); + + // --- the final addition, one ECDAS add row --- + let (a, b, q) = (&rec.a, &rec.b, &rec.q); + let lambda = { + let dy = (&pp_big + &b.y - &a.y) % &pp_big; + let dx = (&pp_big + &b.x - &a.x) % &pp_big; + let inv = dx.modpow(&(&pp_big - BigUint::from(2u8)), &pp_big); + dy * inv % &pp_big + }; + let add = build_step( + &StepPts { + a: a.clone(), + g: b.clone(), + round: 0, + op: 1, + next_op: 0, + r: q.clone(), + lambda, + }, + &p_signed, + &BigInt::from(BigUint::from_bytes_le(&R_BYTES)), + &ext64(&R_BYTES), + &p_ext, + ); + + EcrecWitness { + r: r_le, + s: s_le, + z: z_le, + parity: parity & 1, + ok: true, + w: to_le_32(&rec.w), + qw, + cw, + u1: u1_le, + qz, + cz, + u2: u2_le, + qu, + cu, + len1: msb_position(&rec.u1) as u8, + len2: msb_position(&rec.u2) as u8, + r_sub_n: sub_mod_2_256(&r, &nn), + s_sub_n: sub_mod_2_256(&s, &nn), + u1_sub_n: sub_mod_2_256(&rec.u1, &nn), + u2_sub_n: sub_mod_2_256(&rec.u2, &nn), + y_r: yr_b, + x2: x2_b, + q0: q0_b, + c0, + q1: q1_b, + c1, + y_r_sub_p: sub_mod_2_256(&rp.y, &pp_big), + h: yr_b[0] >> 1, + x_a: to_le_32(&a.x), + y_a: to_le_32(&a.y), + x_b: to_le_32(&b.x), + y_b: to_le_32(&b.y), + x_a_sub_p: sub_mod_2_256(&a.x, &pp_big), + x_b_sub_p: sub_mod_2_256(&b.x, &pp_big), + x_q: to_le_32(&q.x), + y_q: to_le_32(&q.y), + x_q_sub_p: sub_mod_2_256(&q.x, &pp_big), + y_q_sub_p: sub_mod_2_256(&q.y, &pp_big), + chain1: chain1.steps, + chain2: chain2.steps, + add: Some(add), + } +} diff --git a/crypto/ecsm/src/lib.rs b/crypto/ecsm/src/lib.rs index e3a5e3a33..8f3cea69f 100644 --- a/crypto/ecsm/src/lib.rs +++ b/crypto/ecsm/src/lib.rs @@ -16,6 +16,7 @@ //! Curve: secp256k1, `y^2 = x^3 + 7 mod p`, `p = 2^256 - 2^32 - 977`, order `N`. pub mod curve; +pub mod ecrec; pub mod witness; #[cfg(test)] @@ -24,6 +25,10 @@ mod tests; use num_bigint::BigUint; pub use curve::{AffinePoint, recover_y_canonical, replay_double_and_add}; +pub use ecrec::{ + ECREC_STATUS_BAIL, ECREC_STATUS_KEY, EcrecWitness, compute_ecrec_witness, ecdas_add_step, + ecdas_chain, ecrecover, generator_le, +}; pub use witness::{EcdasStep, EcsmWitness, compute_witness}; /// secp256k1 curve coefficient `b`. diff --git a/crypto/ecsm/src/tests/ecrec_tests.rs b/crypto/ecsm/src/tests/ecrec_tests.rs new file mode 100644 index 000000000..a4645b3de --- /dev/null +++ b/crypto/ecsm/src/tests/ecrec_tests.rs @@ -0,0 +1,217 @@ +//! Tests for the ECDSA-recovery call's reference (`ecrecover`) and witness +//! (`compute_ecrec_witness`): k256-signed signatures recover their key on both +//! parities, every input class the call bails on bails, and the witness agrees with +//! the reference and satisfies its relations (the witness builder asserts every +//! limb identity it writes). + +use k256::elliptic_curve::PrimeField; +use k256::elliptic_curve::ops::Reduce; +use k256::elliptic_curve::sec1::ToEncodedPoint; +use k256::{ProjectivePoint, Scalar, U256}; +use num_bigint::BigUint; + +use crate::ecrec::{ECREC_STATUS_BAIL, ECREC_STATUS_KEY, compute_ecrec_witness, ecrecover}; +use crate::{n, p, to_le_32}; + +/// A deterministic xorshift stream. +fn rng(seed: u64) -> impl FnMut() -> u64 { + let mut state = seed; + move || { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + state.wrapping_mul(0x2545_F491_4F6C_DD1D) + } +} + +fn scalar(next: &mut impl FnMut() -> u64) -> Scalar { + Scalar::from(next()) * Scalar::from(next()) * Scalar::from(next()) + Scalar::from(next()) +} + +/// `(x, y)` of a point, big-endian. +fn xy(q: &ProjectivePoint) -> ([u8; 32], [u8; 32]) { + let e = q.to_affine().to_encoded_point(false); + let (mut x, mut y) = ([0u8; 32], [0u8; 32]); + x.copy_from_slice(e.x().unwrap()); + y.copy_from_slice(e.y().unwrap()); + (x, y) +} + +/// A signature by key `d` with nonce `k` on hash `msg`: `(r ‖ s, parity of R's y)`, +/// or `None` for the ~2^-128 nonces whose R.x exceeds N. +fn sign(d: Scalar, k: Scalar, msg: &[u8; 32]) -> Option<([u8; 64], u8)> { + let (rx, ry) = xy(&(ProjectivePoint::GENERATOR * k)); + let r = Option::::from(Scalar::from_repr(rx.into()))?; + let z = >::reduce_bytes(&(*msg).into()); + let s = Option::::from(k.invert())? * (z + r * d); + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&r.to_bytes()); + sig[32..].copy_from_slice(&s.to_bytes()); + Some((sig, ry[31] & 1)) +} + +/// Valid signatures recover their signer, and the witness says the same key. +#[test] +fn signatures_recover_their_key() { + let mut next = rng(0x9E37_79B9_7F4A_7C15); + let mut checked = 0; + while checked < 60 { + let (d, k) = (scalar(&mut next), scalar(&mut next)); + let mut msg = [0u8; 32]; + for c in msg.chunks_mut(8) { + c.copy_from_slice(&next().to_le_bytes()); + } + let Some((sig, parity)) = sign(d, k, &msg) else { + continue; + }; + let want = xy(&(ProjectivePoint::GENERATOR * d)); + assert_eq!(ecrecover(&sig, &msg, parity), Some(want), "round {checked}"); + let w = compute_ecrec_witness(&sig, &msg, parity); + assert!(w.ok); + assert_eq!(w.status(), ECREC_STATUS_KEY); + let (mut qx, mut qy) = (w.x_q, w.y_q); + qx.reverse(); + qy.reverse(); + assert_eq!((qx, qy), want, "round {checked}: the witness's key"); + // The other parity is the other R: a valid recovery of a different key. + let other = ecrecover(&sig, &msg, parity ^ 1); + assert!(other.is_some() && other != Some(want)); + checked += 1; + } +} + +/// Every input class the call refuses to compute bails, in the reference and the +/// witness alike. +#[test] +fn invalid_inputs_bail() { + let mut next = rng(7); + let (d, k) = (scalar(&mut next), scalar(&mut next)); + let msg = [0x5Au8; 32]; + let (sig, _) = sign(d, k, &msg).expect("a valid signature"); + let n_be = { + let mut b = to_le_32(&n()); + b.reverse(); + b + }; + let with = |r: Option<[u8; 32]>, s: Option<[u8; 32]>| { + let mut out = sig; + if let Some(r) = r { + out[..32].copy_from_slice(&r); + } + if let Some(s) = s { + out[32..].copy_from_slice(&s); + } + out + }; + let mut cases: Vec<(&str, [u8; 64], [u8; 32])> = vec![ + ("r = 0", with(Some([0; 32]), None), msg), + ("r = N", with(Some(n_be), None), msg), + ("r = 2^256 - 1", with(Some([0xFF; 32]), None), msg), + ("s = 0", with(None, Some([0; 32])), msg), + ("s = N", with(None, Some(n_be)), msg), + ("s = 2^256 - 1", with(None, Some([0xFF; 32])), msg), + ("z = 0", sig, [0; 32]), + ("z = N", sig, n_be), + ]; + // An r with no curve point: scan small values for a non-residue r^3 + 7. + let pp = p(); + let nonres = (2u64..200) + .find(|&x| { + let rhs = (BigUint::from(x).pow(3) + 7u8) % &pp; + rhs.modpow(&((&pp - 1u8) / 2u8), &pp) != BigUint::from(1u8) + }) + .expect("a non-residue below 200"); + let mut r_nonres = [0u8; 32]; + r_nonres[24..].copy_from_slice(&nonres.to_be_bytes()); + cases.push(("r not an x-coordinate", with(Some(r_nonres), None), msg)); + for (name, sig, msg) in cases { + for parity in [0, 1] { + assert_eq!( + ecrecover(&sig, &msg, parity), + None, + "{name}, parity {parity}" + ); + let w = compute_ecrec_witness(&sig, &msg, parity); + assert!(!w.ok, "{name}"); + assert_eq!(w.status(), ECREC_STATUS_BAIL); + assert!(w.chain1.is_empty() && w.chain2.is_empty() && w.add.is_none()); + assert_eq!((w.x_q, w.y_q), ([0; 32], [0; 32])); + } + } +} + +/// `u1·G = ±u2·R`: the chord is undefined (`A = B`) or the key is the point at +/// infinity (`A = −B`); both bail. With `R = t·G`, `A = −B` exactly when +/// `z = s·t` and `A = B` when `z = −s·t` (mod N). +#[test] +fn a_degenerate_final_addition_bails() { + let mut next = rng(11); + let mut done = 0; + while done < 4 { + let (t, s) = (scalar(&mut next), scalar(&mut next)); + let (rx, ry) = xy(&(ProjectivePoint::GENERATOR * t)); + if Option::::from(Scalar::from_repr(rx.into())).is_none() { + continue; + } + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&rx); + sig[32..].copy_from_slice(&s.to_bytes()); + for z in [s * t, -(s * t)] { + let msg: [u8; 32] = z.to_bytes().into(); + assert_eq!(ecrecover(&sig, &msg, ry[31] & 1), None); + assert!(!compute_ecrec_witness(&sig, &msg, ry[31] & 1).ok); + } + done += 1; + } +} + +/// A message hash at or above N is reduced as the recovery reduces it: `z` and +/// `z − N` recover the same key (the witness's `u1` relation takes `z` unreduced). +#[test] +fn a_hash_above_n_is_reduced() { + let mut next = rng(13); + let mut z_small = [0u8; 32]; + z_small[31] = 5; // z = 5; z + N < 2^256 + let z_big = { + let mut b = to_le_32(&(n() + 5u8)); + b.reverse(); + b + }; + let (d, k) = (scalar(&mut next), scalar(&mut next)); + let (sig, parity) = sign(d, k, &z_small).expect("valid"); + let want = xy(&(ProjectivePoint::GENERATOR * d)); + assert_eq!(ecrecover(&sig, &z_small, parity), Some(want)); + assert_eq!(ecrecover(&sig, &z_big, parity), Some(want)); + let w = compute_ecrec_witness(&sig, &z_big, parity); + assert!(w.ok); + assert_eq!( + w.z, + to_le_32(&(n() + 5u8)), + "z is the hash as read, unreduced" + ); +} + +/// The witness's chains are the double-and-add of u1 and u2: one ECDAS row per +/// double and per add below the top bit. +#[test] +fn the_chains_have_one_row_per_step() { + let mut next = rng(17); + let (d, k) = (scalar(&mut next), scalar(&mut next)); + let msg = [0x33u8; 32]; + let (sig, parity) = sign(d, k, &msg).expect("valid"); + let w = compute_ecrec_witness(&sig, &msg, parity); + for (u, len, chain) in [(&w.u1, w.len1, &w.chain1), (&w.u2, w.len2, &w.chain2)] { + let ones: u32 = u.iter().map(|b| b.count_ones()).sum(); + assert_eq!(chain.len(), len as usize + ones as usize - 1); + assert_eq!(BigUint::from_bytes_le(u).bits() as u8 - 1, len); + } + let add = w.add.as_ref().expect("the final addition"); + assert_eq!((add.op, add.next_op, add.round), (1, 0, 0)); + assert_eq!( + (add.x_a, add.y_a, add.x_g, add.y_g), + (w.x_a, w.y_a, w.x_b, w.y_b) + ); + assert_eq!((add.x_r, add.y_r), (w.x_q, w.y_q)); + assert_eq!(w.y_r[0] & 1, parity); + assert_eq!(w.y_r[0], 2 * w.h + parity); +} diff --git a/crypto/ecsm/src/tests/mod.rs b/crypto/ecsm/src/tests/mod.rs index 74e5080c0..85c2479fa 100644 --- a/crypto/ecsm/src/tests/mod.rs +++ b/crypto/ecsm/src/tests/mod.rs @@ -11,5 +11,6 @@ pub mod reference; pub mod reference_field; mod curve_tests; +mod ecrec_tests; mod lib_tests; mod witness_tests; diff --git a/crypto/ecsm/src/witness.rs b/crypto/ecsm/src/witness.rs index 28b971383..22f7ad818 100644 --- a/crypto/ecsm/src/witness.rs +++ b/crypto/ecsm/src/witness.rs @@ -22,7 +22,7 @@ use num_traits::{Signed, Zero}; #[cfg(feature = "parallel")] use rayon::prelude::*; -use crate::curve::{StepPts, replay_double_and_add}; +use crate::curve::{AffinePoint, StepPts, replay_double_and_add}; use crate::{B, EcsmError, P_BYTES, R_BYTES, n, p, prepare, to_le_32}; /// Full ECSM-chip witness for one scalar multiplication (one ECSM row). @@ -86,7 +86,7 @@ pub struct EcdasStep { // ========================================================================= /// Zero-extends a little-endian byte slice (≤ 64 bytes) to 64 `i128` limbs. -fn ext64(bytes: &[u8]) -> [i128; 64] { +pub(crate) fn ext64(bytes: &[u8]) -> [i128; 64] { let mut a = [0i128; 64]; for (i, &b) in bytes.iter().enumerate() { a[i] = b as i128; @@ -95,7 +95,7 @@ fn ext64(bytes: &[u8]) -> [i128; 64] { } /// Convolution `Σ_{j=0}^{i} a[j]·b[i-j]`. -fn conv(a: &[i128; 64], b: &[i128; 64], i: usize) -> i128 { +pub(crate) fn conv(a: &[i128; 64], b: &[i128; 64], i: usize) -> i128 { let mut s = 0i128; for j in 0..=i { s += a[j] * b[i - j]; @@ -108,7 +108,7 @@ fn conv(a: &[i128; 64], b: &[i128; 64], i: usize) -> i128 { /// /// These asserts catch any transcription error in the `terms` builders: for valid inputs /// the relation `LHS − RHS = 0` holds exactly, so every partial sum is divisible by 256. -fn limb_carries(relation: &str, terms: &[i128; 64]) -> [i64; 64] { +pub(crate) fn limb_carries(relation: &str, terms: &[i128; 64]) -> [i64; 64] { let mut c = [0i64; 64]; let mut carry: i128 = 0; for i in 0..64 { @@ -134,7 +134,12 @@ fn limb_carries(relation: &str, terms: &[i128; 64]) -> [i64; 64] { // ========================================================================= /// ECSM `x2` relation: `xG^2 − x2 − q0·p = 0`. -fn carries_x2(xg: &[i128; 64], x2: &[i128; 64], q0: &[i128; 64], pp: &[i128; 64]) -> [i64; 64] { +pub(crate) fn carries_x2( + xg: &[i128; 64], + x2: &[i128; 64], + q0: &[i128; 64], + pp: &[i128; 64], +) -> [i64; 64] { let mut terms = [0i128; 64]; for i in 0..64 { terms[i] = conv(xg, xg, i) - x2[i] - conv(q0, pp, i); @@ -143,7 +148,7 @@ fn carries_x2(xg: &[i128; 64], x2: &[i128; 64], q0: &[i128; 64], pp: &[i128; 64] } /// ECSM `yG` relation: `yG^2 + p^2 − xG·x2 − b − q1·p = 0`. -fn carries_yg( +pub(crate) fn carries_yg( yg: &[i128; 64], pp: &[i128; 64], x2: &[i128; 64], @@ -244,7 +249,7 @@ fn carries_yr( // ========================================================================= /// Little-endian 33 bytes of a non-negative value that fits in 264 bits. -fn to_le_33(relation: &str, v: &BigUint) -> [u8; 33] { +pub(crate) fn to_le_33(relation: &str, v: &BigUint) -> [u8; 33] { let mut bytes = v.to_bytes_le(); assert!( bytes.len() <= 33, @@ -258,7 +263,12 @@ fn to_le_33(relation: &str, v: &BigUint) -> [u8; 33] { /// `r + numerator / p`, where `numerator` must be divisible by `p`. Asserts divisibility /// and that the result is non-negative (guaranteed by the spec quotient ranges). -fn shifted_quotient(relation: &str, numerator: &BigInt, p_big: &BigInt, r_big: &BigInt) -> BigUint { +pub(crate) fn shifted_quotient( + relation: &str, + numerator: &BigInt, + p_big: &BigInt, + r_big: &BigInt, +) -> BigUint { let (q, rem) = numerator.div_rem(p_big); assert!( rem.is_zero(), @@ -280,6 +290,14 @@ fn shifted_quotient(relation: &str, numerator: &BigInt, p_big: &BigInt, r_big: & /// little-endian 32-byte values. This is the prover's entry point. pub fn compute_witness(k_le: &[u8; 32], xg_le: &[u8; 32]) -> Result { let (k, g) = prepare(k_le, xg_le)?; + Ok(witness_for_point(&k, &g, k_le)) +} + +/// [`compute_witness`] for a scalar already checked to be in `[1, N)` and a point already +/// on the curve with canonical coordinates, whichever root its `y` is (`k_le` is `k`'s +/// bytes). The ECDSA-recovery call builds its two chains with it. +pub(crate) fn witness_for_point(k: &BigUint, g: &AffinePoint, k_le: &[u8; 32]) -> EcsmWitness { + let (k, g) = (k.clone(), g.clone()); let p_big = BigInt::from(p()); let r_big = BigInt::from(BigUint::from_bytes_le(&R_BYTES)); // r = 3p @@ -342,7 +360,7 @@ pub fn compute_witness(k_le: &[u8; 32], xg_le: &[u8; 32]) -> Result Result (u64, u64) { + let t = a as u128 + b as u128 + carry as u128; + (t as u64, (t >> 64) as u64) +} + +#[inline(always)] +fn sbb(a: u64, b: u64, borrow: u64) -> (u64, u64) { + let t = (a as u128).wrapping_sub(b as u128 + borrow as u128); + (t as u64, (t >> 127) as u64) +} + +impl Fe { + pub(crate) const ZERO: Fe = Fe([0; 4]); + pub(crate) const ONE: Fe = Fe([1, 0, 0, 0]); + pub(crate) const SEVEN: Fe = Fe([7, 0, 0, 0]); + + /// The value with these little-endian limbs (any value below 2^256). + pub(crate) const fn from_limbs(limbs: [u64; 4]) -> Fe { + Fe(limbs) + } + + /// The canonical value's limbs, below p. + #[inline] + fn canonical(self) -> [u64; 4] { + // self >= p exactly when self + C carries out of 2^256; the wrapped sum is then + // self − p. + let (r0, c) = adc(self.0[0], C, 0); + let (r1, c) = adc(self.0[1], 0, c); + let (r2, c) = adc(self.0[2], 0, c); + let (r3, c) = adc(self.0[3], 0, c); + if c != 0 { + [r0, r1, r2, r3] + } else { + self.0 + } + } + + /// The element of 32 little-endian bytes, or `None` if they encode p or more + /// (the same rule as `FieldElement::from_bytes`). + pub(crate) fn from_le_bytes(bytes: &[u8; 32]) -> Option { + let mut limbs = [0u64; 4]; + for (limb, chunk) in limbs.iter_mut().zip(bytes.chunks_exact(8)) { + let mut word = [0u8; 8]; + word.copy_from_slice(chunk); + *limb = u64::from_le_bytes(word); + } + let fe = Fe(limbs); + (fe.canonical() == limbs).then_some(fe) + } + + /// The element of 32 big-endian bytes, or `None` if they encode p or more. + pub(crate) fn from_be_bytes(bytes: &[u8; 32]) -> Option { + let mut limbs = [0u64; 4]; + for (limb, chunk) in limbs.iter_mut().zip(bytes.rchunks_exact(8)) { + let mut word = [0u8; 8]; + word.copy_from_slice(chunk); + *limb = u64::from_be_bytes(word); + } + let fe = Fe(limbs); + (fe.canonical() == limbs).then_some(fe) + } + + /// The canonical value as 32 little-endian bytes. + pub(crate) fn to_le_bytes(self) -> [u8; 32] { + let mut out = [0u8; 32]; + for (chunk, limb) in out.chunks_exact_mut(8).zip(self.canonical()) { + chunk.copy_from_slice(&limb.to_le_bytes()); + } + out + } + + /// The canonical value as 32 big-endian bytes. + pub(crate) fn to_be_bytes(self) -> [u8; 32] { + let mut out = [0u8; 32]; + for (chunk, limb) in out.rchunks_exact_mut(8).zip(self.canonical()) { + chunk.copy_from_slice(&limb.to_be_bytes()); + } + out + } + + /// The same element as k256's type. + pub(crate) fn from_k256(x: &FieldElement) -> Fe { + // `to_bytes` is canonical, so every limb pattern it gives is below p. + let bytes: [u8; 32] = x.to_bytes().into(); + let mut limbs = [0u64; 4]; + for (limb, chunk) in limbs.iter_mut().zip(bytes.rchunks_exact(8)) { + let mut word = [0u8; 8]; + word.copy_from_slice(chunk); + *limb = u64::from_be_bytes(word); + } + Fe(limbs) + } + + /// The same element as k256's type (always `Some`: the encoding is canonical). + pub(crate) fn to_k256(self) -> Option { + Option::from(FieldElement::from_bytes(&self.to_be_bytes().into())) + } + + /// Whether the element is zero. + pub(crate) fn is_zero(self) -> bool { + self.canonical() == [0; 4] + } + + /// Whether the two values are the same element. + pub(crate) fn same(self, other: Fe) -> bool { + self.canonical() == other.canonical() + } + + /// Whether the canonical value is odd. + pub(crate) fn is_odd(self) -> bool { + self.canonical()[0] & 1 == 1 + } + + pub(crate) fn double(self) -> Fe { + self + self + } + + /// The square, with the cross products computed once and doubled. + pub(crate) fn square(self) -> Fe { + let a = self.0; + let mut r = [0u64; 8]; + for i in 0..4 { + let mut carry: u128 = 0; + for j in (i + 1)..4 { + let t = r[i + j] as u128 + (a[i] as u128) * (a[j] as u128) + carry; + r[i + j] = t as u64; + carry = t >> 64; + } + r[i + 4] = carry as u64; + } + let mut top = 0u64; + for limb in r.iter_mut().skip(1) { + let next = *limb >> 63; + *limb = (*limb << 1) | top; + top = next; + } + let mut carry: u128 = 0; + for i in 0..4 { + let d = (a[i] as u128) * (a[i] as u128); + let t = r[2 * i] as u128 + (d as u64) as u128 + carry; + r[2 * i] = t as u64; + let t2 = r[2 * i + 1] as u128 + (d >> 64) + (t >> 64); + r[2 * i + 1] = t2 as u64; + carry = t2 >> 64; + } + reduce(r) + } +} + +/// The 512-bit `r` modulo p, in the weak form: the high half folded in with C twice. +#[inline(always)] +fn reduce(r: [u64; 8]) -> Fe { + let mut out = [0u64; 4]; + let mut carry: u128 = 0; + for i in 0..4 { + let t = r[i] as u128 + (r[i + 4] as u128) * (C as u128) + carry; + out[i] = t as u64; + carry = t >> 64; + } + // carry < 2^34, carry · C < 2^67. + let t = out[0] as u128 + carry * (C as u128); + out[0] = t as u64; + let mut c = (t >> 64) as u64; + for limb in out.iter_mut().skip(1) { + if c == 0 { + return Fe(out); + } + let (v, next) = adc(*limb, c, 0); + *limb = v; + c = next; + } + if c != 0 { + // Past 2^256 once more: the value is now below 2^67, so adding C cannot carry. + out[0] += C; + } + Fe(out) +} + +impl Add for Fe { + type Output = Fe; + + fn add(self, other: Fe) -> Fe { + let (r0, c) = adc(self.0[0], other.0[0], 0); + let (r1, c) = adc(self.0[1], other.0[1], c); + let (r2, c) = adc(self.0[2], other.0[2], c); + let (r3, c) = adc(self.0[3], other.0[3], c); + if c == 0 { + return Fe([r0, r1, r2, r3]); + } + // The sum wrapped 2^256 ≡ C: add C back. + let (r0, c) = adc(r0, C, 0); + let (r1, c) = adc(r1, 0, c); + let (r2, c) = adc(r2, 0, c); + let (r3, c) = adc(r3, 0, c); + if c == 0 { + return Fe([r0, r1, r2, r3]); + } + // Wrapped again: the value is now below C, so adding C cannot carry. + Fe([r0 + C, r1, r2, r3]) + } +} + +impl Sub for Fe { + type Output = Fe; + + fn sub(self, other: Fe) -> Fe { + let (r0, b) = sbb(self.0[0], other.0[0], 0); + let (r1, b) = sbb(self.0[1], other.0[1], b); + let (r2, b) = sbb(self.0[2], other.0[2], b); + let (r3, b) = sbb(self.0[3], other.0[3], b); + if b == 0 { + return Fe([r0, r1, r2, r3]); + } + // The difference wrapped by +2^256 ≡ +C: take C off. + let (r0, b) = sbb(r0, C, 0); + let (r1, b) = sbb(r1, 0, b); + let (r2, b) = sbb(r2, 0, b); + let (r3, b) = sbb(r3, 0, b); + if b == 0 { + return Fe([r0, r1, r2, r3]); + } + // Wrapped again (the value was below C): now at least 2^256 − C, so taking C + // off once more cannot borrow. + Fe([r0 - C, r1, r2, r3]) + } +} + +impl Neg for Fe { + type Output = Fe; + + fn neg(self) -> Fe { + Fe::ZERO - self + } +} + +impl Mul for Fe { + type Output = Fe; + + fn mul(self, other: Fe) -> Fe { + let (a, b) = (self.0, other.0); + let mut r = [0u64; 8]; + for i in 0..4 { + let mut carry: u128 = 0; + for j in 0..4 { + let t = r[i + j] as u128 + (a[i] as u128) * (b[j] as u128) + carry; + r[i + j] = t as u64; + carry = t >> 64; + } + r[i + 4] = carry as u64; + } + reduce(r) + } +} + +#[cfg(test)] +impl Fe { + /// The raw limbs, for the tests. + pub(crate) fn limbs(self) -> [u64; 4] { + self.0 + } +} diff --git a/crypto/ethrex-crypto/src/keccak_cache.rs b/crypto/ethrex-crypto/src/keccak_cache.rs new file mode 100644 index 000000000..8f2674cde --- /dev/null +++ b/crypto/ethrex-crypto/src/keccak_cache.rs @@ -0,0 +1,173 @@ +//! A bounded cache of Keccak-256 digests for the short inputs the guest hashes again +//! and again. +//! +//! The ethrex guest hashes the same 20-byte addresses (account trie keys), 32-byte +//! storage slots (storage trie keys) and 64-byte mapping slots (the `KECCAK256` +//! opcode) many times in a block, and the empty input too. Each repeat costs a +//! `keccak256` call (a KECCAK row and 24 KECCAK_RND rows in the proof per +//! permutation). At the median block the cache saves 6,852 of the guest's 96,384 +//! permutations. +//! +//! One direct-mapped table per length. A lookup compares the whole input with the +//! stored one, so a hit returns the digest of exactly that input; a different input +//! in the same slot is a miss and replaces it. The tables have a fixed number of +//! entries and are allocated once, zeroed (an entry is used only once its `full` word +//! is set), so the memory is bounded whatever the block does. A collision of the +//! slot function only costs a miss. + +#[cfg(target_arch = "riscv64")] +use std::cell::UnsafeCell; + +/// Keccak-256 of the empty input. +pub(crate) const EMPTY_DIGEST: [u8; 32] = [ + 0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, + 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70, +]; + +/// log2 of the entries in each table. +const SLOT_BITS: u32 = 12; + +#[derive(Clone, Copy)] +#[repr(C)] +struct Entry { + /// The input as little-endian doublewords, the last one zero-padded. + key: [u64; W], + digest: [u8; 32], + /// 1 once `key` and `digest` hold a pair, 0 before. + full: u64, +} + +/// A direct-mapped table for inputs of exactly `N` bytes, held in `W` doublewords. +struct Table { + entries: Box<[Entry]>, +} + +impl Table { + fn new() -> Self { + let entries = Box::<[Entry]>::new_zeroed_slice(1 << SLOT_BITS); + // SAFETY: every field of `Entry` is an integer or an integer array, for which + // all-zero bytes are a valid value. + Self { + entries: unsafe { entries.assume_init() }, + } + } + + /// `input` (`input.len() == N`) as little-endian doublewords, the last one + /// zero-padded. + #[inline(always)] + fn key(input: &[u8]) -> [u64; W] { + let mut key = [0u64; W]; + for (i, word) in key.iter_mut().enumerate() { + let mut bytes = [0u8; 8]; + let end = N.min(8 * i + 8); + bytes[..end - 8 * i].copy_from_slice(&input[8 * i..end]); + *word = u64::from_le_bytes(bytes); + } + key + } + + /// The digest of `input` (`input.len() == N`), from the table or from `hash`. + /// + /// The hit path is inlined into the provider; the miss path is a call of its own, + /// so that a hit pays for no register saves the sponge needs. + #[inline(always)] + fn digest(&mut self, input: &[u8], hash: impl FnOnce(&[u8]) -> [u8; 32]) -> [u8; 32] { + let key = Self::key(input); + let mut fold = 0u64; + for word in key { + fold ^= word; + } + let slot = (fold.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> (64 - SLOT_BITS)) as usize; + let entry = &mut self.entries[slot]; + // Word by word into one register, rather than an array compare that goes + // through memory: the hit path then needs no stack frame. + let mut diff = entry.full ^ 1; + for (stored, word) in entry.key.iter().zip(key) { + diff |= stored ^ word; + } + if diff == 0 { + return entry.digest; + } + Self::fill(entry, input, hash) + } + + /// Hash `input` and make it the entry's pair. + #[inline(never)] + fn fill(entry: &mut Entry, input: &[u8], hash: impl FnOnce(&[u8]) -> [u8; 32]) -> [u8; 32] { + let digest = hash(input); + entry.key = Self::key(input); + entry.digest = digest; + entry.full = 1; + digest + } +} + +/// The cache: one table per cached length. +pub(crate) struct KeccakCache { + t20: Table<20, 3>, + t32: Table<32, 4>, + t64: Table<64, 8>, +} + +impl KeccakCache { + pub(crate) fn new() -> Self { + Self { + t20: Table::new(), + t32: Table::new(), + t64: Table::new(), + } + } + + /// Keccak-256 of `input`: `hash(input)`, which must be Keccak-256, or the digest it + /// returned earlier for the same input. + #[inline(always)] + pub(crate) fn keccak256( + &mut self, + input: &[u8], + hash: impl FnOnce(&[u8]) -> [u8; 32], + ) -> [u8; 32] { + match input.len() { + 0 => EMPTY_DIGEST, + 20 => self.t20.digest(input, hash), + 32 => self.t32.digest(input, hash), + 64 => self.t64.digest(input, hash), + _ => hash(input), + } + } +} + +/// The guest's one cache, created on first use. +#[cfg(target_arch = "riscv64")] +/// +/// The guest is a single hart with no threads (the target is `singlethread`) and +/// nothing here re-enters: `hash` is the sponge, which never calls back into the +/// provider. So at most one `&mut` to the cell's contents exists at a time. +pub(crate) struct GuestCache(UnsafeCell>); + +// SAFETY: see the type's documentation; the guest never shares it between threads. +#[cfg(target_arch = "riscv64")] +unsafe impl Sync for GuestCache {} + +#[cfg(target_arch = "riscv64")] +impl GuestCache { + pub(crate) const fn new() -> Self { + Self(UnsafeCell::new(None)) + } + + #[inline(always)] + pub(crate) fn keccak256(&self, input: &[u8], hash: impl FnOnce(&[u8]) -> [u8; 32]) -> [u8; 32] { + // SAFETY: single hart, no re-entry (see the type's documentation). + let cell = unsafe { &mut *self.0.get() }; + let cache = match cell { + Some(cache) => cache, + None => Self::init(cell), + }; + cache.keccak256(input, hash) + } + + #[cold] + #[inline(never)] + fn init(cell: &mut Option) -> &mut KeccakCache { + cell.insert(KeccakCache::new()) + } +} diff --git a/crypto/ethrex-crypto/src/lib.rs b/crypto/ethrex-crypto/src/lib.rs index d3724f92c..996f26143 100644 --- a/crypto/ethrex-crypto/src/lib.rs +++ b/crypto/ethrex-crypto/src/lib.rs @@ -6,12 +6,15 @@ //! accelerator changes don't require an ethrex PR. //! //! Accelerated today: -//! - `keccak256`: a sponge over the `keccak_permute` precompile (riscv64; on -//! host it falls back to software keccak for tests). -//! - `secp256k1_ecrecover`: the ECDSA recovery's 2-term linear combination is -//! evaluated through the ECSM `ecsm_mul` precompile (riscv64), reconstructing -//! the full point from x-only queries; on host / degenerate inputs it falls -//! back to the pure-Rust `ProjectivePoint::lincomb`. +//! - `keccak256`: one `keccak256` VM call per hash (riscv64; the sponge over the +//! `keccak_permute` precompile for the rare buffer the call refuses; on host +//! software keccak for tests), with a bounded cache of +//! repeated 20-, 32- and 64-byte inputs and the empty input (`keccak_cache`). +//! - `secp256k1_ecrecover`: one ECREC precompile call recovers the key +//! (riscv64). When it bails, the recovery runs as before: the 2-term linear +//! combination through the ECSM `ecsm_mul` precompile, reconstructing the full +//! point from x-only queries; on host / degenerate inputs the pure-Rust +//! `ProjectivePoint::lincomb`. //! //! Every other `Crypto` method inherits the trait default (vetted pure-Rust //! crates: `ark-bn254`, `p256`, `sha2`, `ripemd`, …), except two families that @@ -34,11 +37,12 @@ use k256::{AffinePoint, FieldBytes, ProjectivePoint, Scalar, U256}; // Used only by the x-only point reconstruction (riscv accelerated path + the // host unit tests); unused on a non-test host build. -#[cfg(any(target_arch = "riscv64", test))] use k256::elliptic_curve::sec1::FromEncodedPoint; -#[cfg(any(target_arch = "riscv64", test))] use k256::{EncodedPoint, FieldElement}; +mod fe; +use fe::Fe; + /// LambdaVM crypto provider — inject via `Arc::new(LambdaVmEcsmCrypto)`. #[derive(Debug)] pub struct LambdaVmEcsmCrypto; @@ -55,9 +59,10 @@ impl Crypto for LambdaVmEcsmCrypto { } fn keccak256(&self, input: &[u8]) -> [u8; 32] { - // riscv64 guest: sponge over the keccak_permute precompile. + // riscv64 guest: the keccak256 VM call (`keccak256_via_lambdavm`), behind a bounded + // cache of the short inputs the guest hashes repeatedly (`keccak_cache`). #[cfg(target_arch = "riscv64")] - return keccak256_via_lambdavm(input); + return KECCAK_CACHE.keccak256(input, keccak256_via_lambdavm); // host (tests / non-guest): software keccak — the precompile syscall // isn't available off-target. #[cfg(not(target_arch = "riscv64"))] @@ -146,35 +151,52 @@ where /// non-residue regardless of the prover-chosen hint — the hint can only save work, /// never steer the accept/reject outcome. Off-target it uses the software /// decompress directly. -fn decompress_r(r_bytes: &FieldBytes, y_is_odd: bool) -> Option { +/// +/// Returns R's affine coordinates, which is what the recovery uses. +fn decompress_r_xy(r_bytes: &FieldBytes, y_is_odd: bool) -> Option<(Fe, Fe)> { #[cfg(target_arch = "riscv64")] { - decompress_r_with_oracle(r_bytes, y_is_odd, |rhs_be| { + decompress_r_xy_with_oracle(r_bytes, y_is_odd, |rhs_be| { get_hint(lambda_vm_syscalls::syscalls::HINT_FIELD_SQRT, rhs_be) }) } #[cfg(not(target_arch = "riscv64"))] { use k256::elliptic_curve::point::DecompressPoint; - AffinePoint::decompress(r_bytes, u8::from(y_is_odd).into()).into() + let r: Option = + AffinePoint::decompress(r_bytes, u8::from(y_is_odd).into()).into(); + let (x, y) = affine_xy(&r?)?; + Some((Fe::from_k256(&x), Fe::from_k256(&y))) } } -/// Core of [`decompress_r`], generic over the hint source for host tests: try the +/// [`decompress_r_xy_with_oracle`] as a curve point (the host tests' view). +#[cfg(test)] +fn decompress_r_with_oracle(r_bytes: &FieldBytes, y_is_odd: bool, hint: O) -> Option +where + O: FnOnce(&[u8; 32]) -> [u8; 32], +{ + let (x, y) = decompress_r_xy_with_oracle(r_bytes, y_is_odd, hint)?; + point_from_fe(x, y) +} + +/// Core of [`decompress_r_xy`], generic over the hint source for host tests: try the /// hinted sqrt, then fall back to the authoritative software decompress on any -/// failure. See [`decompress_r`] for the rationale. +/// failure. See [`decompress_r_xy`] for the rationale. #[cfg(any(target_arch = "riscv64", test))] -fn decompress_r_with_oracle(r_bytes: &FieldBytes, y_is_odd: bool, hint: O) -> Option +fn decompress_r_xy_with_oracle(r_bytes: &FieldBytes, y_is_odd: bool, hint: O) -> Option<(Fe, Fe)> where O: FnOnce(&[u8; 32]) -> [u8; 32], { - if let Some(p) = decompress_r_hinted(r_bytes, y_is_odd, hint) { - return Some(p); + if let Some(xy) = decompress_r_hinted_xy(r_bytes, y_is_odd, hint) { + return Some(xy); } // Hinted root absent / malformed / wrong, OR a genuine non-residue: the software // decompress is authoritative — `Some` for a residue, `None` for a non-residue. use k256::elliptic_curve::point::DecompressPoint; - AffinePoint::decompress(r_bytes, u8::from(y_is_odd).into()).into() + let r: Option = AffinePoint::decompress(r_bytes, u8::from(y_is_odd).into()).into(); + let (x, y) = affine_xy(&r?)?; + Some((Fe::from_k256(&x), Fe::from_k256(&y))) } /// The hint-accelerated decompress attempt: returns the point only if the hinted @@ -182,41 +204,24 @@ where /// the software decompress. Never the last word — a `None` here is not a decision /// that R is invalid, only that the fast path did not produce a verified root. #[cfg(any(target_arch = "riscv64", test))] -fn decompress_r_hinted(r_bytes: &FieldBytes, y_is_odd: bool, hint: O) -> Option +fn decompress_r_hinted_xy(r_bytes: &FieldBytes, y_is_odd: bool, hint: O) -> Option<(Fe, Fe)> where O: FnOnce(&[u8; 32]) -> [u8; 32], { - let x: FieldElement = Option::from(FieldElement::from_bytes(r_bytes))?; - // secp256k1: y² = x³ + 7. - let mut seven_bytes = [0u8; 32]; - seven_bytes[31] = 7; - let seven: FieldElement = Option::from(FieldElement::from_bytes(&seven_bytes.into()))?; - let x3: FieldElement = x.square() * x; - let rhs: FieldElement = x3 + seven; - // Hinted sqrt (BE in/out), then verify y² == rhs canonically. - let rhs_be: [u8; 32] = rhs.to_bytes().into(); - let y_be = hint(&rhs_be); - let mut y: FieldElement = Option::from(FieldElement::from_bytes(&y_be.into()))?; - let y2: FieldElement = y.square(); - // Verify the untrusted root: y² must equal x³+7. Negate `y2`, not `rhs`: - // `Neg` is `negate(1)`, whose debug assert requires magnitude <= 1. `square()` - // always returns magnitude 1, whereas `rhs` is a sum carrying magnitude 2, so - // negating it would trip that assert and panic in debug builds. (The value would - // still come out right — `negate(m)` computes `2*(m+1)*P_limb - self`, which for a - // magnitude-2 operand stays non-negative — so this is a build-configuration - // hazard, not a wrong answer.) - // (`ct_eq` is unusable here for the same reason as in `field_inv`.) - if !bool::from((rhs + y2.negate(1)).normalizes_to_zero()) { + // r ≥ p is no field element (k256's `from_bytes` rule). + let x = Fe::from_be_bytes(&(*r_bytes).into())?; + // secp256k1: y² = x³ + 7. Hinted sqrt (canonical big-endian in and out). + let rhs = x.square() * x + Fe::SEVEN; + let y_be = hint(&rhs.to_be_bytes()); + let y = Fe::from_be_bytes(&y_be)?; + // Verify the untrusted root. + if !y.square().same(rhs) { return None; } - // Select the root whose canonical LSB matches the requested parity. - let y_odd = (y.to_bytes()[31] & 1) == 1; - if y_odd != y_is_odd { - y = -y; - } - // Build the affine point; `from_encoded_point` re-checks it's on-curve. - let ep = EncodedPoint::from_affine_coordinates(&x.to_bytes(), &y.to_bytes(), false); - Option::from(AffinePoint::from_encoded_point(&ep)) + // Select the root whose canonical low bit matches the requested parity. + let y = if y.is_odd() != y_is_odd { -y } else { y }; + // `y² == x³+7` was verified above, so `(x, y)` is R; no curve-point round trip. + Some((x, y)) } /// Recover the uncompressed public key bytes (X‖Y, 64 bytes) from a 64-byte @@ -230,13 +235,71 @@ where /// /// Mirrors the pure-Rust recovery in the `Crypto` trait default /// (`pk = r⁻¹·(s·R − z·G)`), but evaluates the 2-term linear combination -/// `lincomb(G, u1, R, u2)` through the ECSM accelerator via [`ecsm_lincomb2`], +/// `lincomb(G, u1, R, u2)` through the ECSM accelerator via [`ecsm_lincomb2_xy`], /// falling back to the software `ProjectivePoint::lincomb` whenever the /// accelerated path declines (degenerate scalars/points, or non-riscv builds). /// We compute the recovery directly rather than calling k256's /// `recover_from_prehash`, which internally runs a *second* lincomb to /// re-verify the key — doubling the ECSM ecalls for no gain here. fn ecsm_ecrecover(sig: &[u8; 64], recid: u8, msg: &[u8; 32]) -> Result<[u8; 64], CryptoError> { + // riscv64: one ECREC call recovers the key; when it bails (an invalid signature, or + // one of the rare inputs its chains cannot represent) the path below decides. + #[cfg(target_arch = "riscv64")] + if let Some(pk) = ecrec_call(sig, recid, msg) { + return Ok(pk); + } + ecrecover_with(sig, recid, msg, decompress_r_xy, ecsm_lincomb2_xy) +} + +/// The key from the ECREC precompile, or `None` when the call bailed (status 1). +/// +/// A status-0 answer is proven by the ECREC chip: `Q = u1·G + u2·R` with `u1`, `u2` +/// the scalars of `(r, s, z)` mod N and `R` the curve point with x = r and the +/// requested parity. A bail proves nothing and changes nothing: the caller recovers +/// the key in software exactly as it does without the precompile, so a prover that +/// bails on a valid signature only costs the guest cycles. The call reads the +/// signature and hash in place (big-endian, the precompile's encoding) and writes the +/// key big-endian, the order the address hash takes. +/// +/// Operands the VM would refuse (the signature and hash overlapping, or one of the +/// three crossing a `2^32` address-word boundary) take the software path instead, +/// so the call can never make a block unprovable. The status starts nonzero, so a +/// call that wrote nothing reads as a bail rather than as the key `(0, 0)`. +#[cfg(target_arch = "riscv64")] +fn ecrec_call(sig: &[u8; 64], recid: u8, msg: &[u8; 32]) -> Option<[u8; 64]> { + use lambda_vm_syscalls::syscalls; + #[repr(C, align(8))] + struct Out([u8; 72]); + let mut out = Out([0u8; 72]); + out.0[64] = 1; + if !syscalls::ecrecover_fits(&out.0, sig, msg) { + return None; + } + syscalls::ecrecover(&mut out.0, sig, msg, u64::from(recid & 1)); + if out.0[64..] != [0u8; 8] { + return None; + } + let mut pk = [0u8; 64]; + pk.copy_from_slice(&out.0[..64]); + Some(pk) +} + +/// [`ecsm_ecrecover`], generic over how R is decompressed and how the linear +/// combination is evaluated, so the host tests can run the accelerated shape with +/// software oracles. Both work on affine coordinates in the four-limb field type +/// ([`Fe`]), which avoids converting points to and from their SEC1 encoding at +/// every step. +fn ecrecover_with( + sig: &[u8; 64], + recid: u8, + msg: &[u8; 32], + decompress: D, + lincomb: L, +) -> Result<[u8; 64], CryptoError> +where + D: FnOnce(&FieldBytes, bool) -> Option<(Fe, Fe)>, + L: FnOnce(&Fe, &Fe, &Scalar, &Fe, &Fe, &Scalar) -> Option<(Fe, Fe)>, +{ let r_bytes = <&FieldBytes>::from(&sig[..32]); let s_bytes = <&FieldBytes>::from(&sig[32..]); @@ -251,15 +314,14 @@ fn ecsm_ecrecover(sig: &[u8; 64], recid: u8, msg: &[u8; 32]) -> Result<[u8; 64], } // Decompress R from r and the recovery-id parity bit. - // recid >= 2 (R.x = r + n) has ~2^-128 probability and never occurs for the - // precompile; we don't handle it (decompression simply fails), matching the - // trait default. + // recid >= 2 would mean R.x = r + n (probability ~2^-128; never for the + // precompile). It is not handled: only the low bit is read, so recid 2 and 3 + // decompress the point with x = r exactly as 0 and 1 do (as the ECREC call + // does, which is passed recid & 1). let y_is_odd = (recid & 1) != 0; - let r_point: Option = decompress_r(r_bytes, y_is_odd); - let Some(r_point) = r_point else { + let Some((rx, ry)) = decompress(r_bytes, y_is_odd) else { return Err(CryptoError::RecoveryFailed); }; - let r_proj = ProjectivePoint::from(r_point); let z = >::reduce_bytes(&FieldBytes::from(*msg)); let r_inv: Option = scalar_inv(&r); @@ -269,15 +331,28 @@ fn ecsm_ecrecover(sig: &[u8; 64], recid: u8, msg: &[u8; 32]) -> Result<[u8; 64], let u1 = -(r_inv * z); let u2 = r_inv * s; - // pk = u1·G + u2·R, accelerated via ECSM with a software fallback. - // The ECSM path takes affine inputs and returns the affine result directly: - // its inputs (G, R) and output are Z=1 points, so passing affines avoids the - // wasteful projective→affine inversions (`to_affine` of a Z=1 point still runs - // a full constant-time field inversion in k256). The rare software fallback - // still converts via `to_affine`. - let g = ProjectivePoint::GENERATOR; - let pk_affine = ecsm_lincomb2(&AffinePoint::GENERATOR, &u1, &r_point, &u2) - .unwrap_or_else(|| ProjectivePoint::lincomb(&g, &u1, &r_proj, &u2).to_affine()); + // pk = u1·G + u2·R, accelerated via ECSM with a software fallback. The + // accelerated path takes and returns affine coordinates (G's are fixed, R's + // come from the decompression); a result it returns has been checked to be on + // the curve, and an affine point is never the identity. + if let Some((xq, yq)) = lincomb(&GENERATOR_X, &GENERATOR_Y, &u1, &rx, &ry, &u2) { + let mut pk_bytes = [0u8; 64]; + pk_bytes[..32].copy_from_slice(&xq.to_be_bytes()); + pk_bytes[32..].copy_from_slice(&yq.to_be_bytes()); + return Ok(pk_bytes); + } + + // Rare software fallback (degenerate scalars or points, or off-target). + let Some(r_point) = point_from_fe(rx, ry) else { + return Err(CryptoError::RecoveryFailed); + }; + let pk_affine = ProjectivePoint::lincomb( + &ProjectivePoint::GENERATOR, + &u1, + &ProjectivePoint::from(r_point), + &u2, + ) + .to_affine(); if bool::from(pk_affine.is_identity()) { return Err(CryptoError::RecoveryFailed); } @@ -289,53 +364,78 @@ fn ecsm_ecrecover(sig: &[u8; 64], recid: u8, msg: &[u8; 32]) -> Result<[u8; 64], Ok(pk_bytes) } +/// The generator's affine x (SEC 2), as four little-endian limbs. +const GENERATOR_X: Fe = Fe::from_limbs([ + 0x59F2_815B_16F8_1798, + 0x029B_FCDB_2DCE_28D9, + 0x55A0_6295_CE87_0B07, + 0x79BE_667E_F9DC_BBAC, +]); + +/// The generator's affine y (SEC 2), as four little-endian limbs. +const GENERATOR_Y: Fe = Fe::from_limbs([ + 0x9C47_D08F_FB10_D4B8, + 0xFD17_B448_A685_5419, + 0x5DA4_FBFC_0E11_08A8, + 0x483A_DA77_26A3_C465, +]); + /// ECSM-accelerated 2-term linear combination `k1·P1 + k2·P2`. /// /// On riscv64 this reconstructs the full affine result from four x-only ECSM -/// queries (see [`lincomb2_with_oracle`]); on other targets, and whenever a +/// queries (see [`lincomb2_xy_with_oracle`]); on other targets, and whenever a /// guard trips (degenerate input or oracle inconsistency), it returns `None` /// so the caller uses the pure-Rust `ProjectivePoint::lincomb`. #[cfg(target_arch = "riscv64")] -fn ecsm_lincomb2( - a1: &AffinePoint, +fn ecsm_lincomb2_xy( + x1: &Fe, + y1: &Fe, k1: &Scalar, - a2: &AffinePoint, + x2: &Fe, + y2: &Fe, k2: &Scalar, -) -> Option { - lincomb2_with_oracle(a1, k1, a2, k2, ecsm_oracle) +) -> Option<(Fe, Fe)> { + lincomb2_xy_with_oracle(x1, y1, k1, x2, y2, k2, ecsm_oracle) } #[cfg(not(target_arch = "riscv64"))] -fn ecsm_lincomb2( - _a1: &AffinePoint, +fn ecsm_lincomb2_xy( + _x1: &Fe, + _y1: &Fe, _k1: &Scalar, - _a2: &AffinePoint, + _x2: &Fe, + _y2: &Fe, _k2: &Scalar, -) -> Option { +) -> Option<(Fe, Fe)> { None } /// x-only scalar-mul oracle backed by the ECSM precompile: computes `x(k·P)` -/// for the curve point P whose x-coordinate is passed in. `x` must be the -/// x-coordinate of a curve point and `k` in `(0, N)` (N = curve order) — -/// guaranteed by the guards in [`lincomb2_with_oracle`]. Values cross the ABI -/// as 32-byte little-endian; `x_le` and `k_le` are distinct stack arrays so -/// the executor's `|addr_x_le − addr_k_le| ≥ 32` assumption holds by -/// construction. +/// for the curve point P whose x-coordinate is passed in, as 32 little-endian +/// bytes. `x` must be the x-coordinate of a curve point and `k` in `(0, N)` +/// (N = curve order) — guaranteed by the guards in [`lincomb2_xy_with_oracle`]. +/// Values cross the ABI as 32-byte little-endian, the limb order of [`Fe`]; +/// `x_le` (the caller's) and `k_le` (this function's) are distinct 32-byte +/// arrays, so the executor's `|addr_x_le − addr_k_le| ≥ 32` assumption holds by +/// construction. A result of p or more is rejected (`None`), as k256's +/// `from_bytes` did. #[cfg(target_arch = "riscv64")] -fn ecsm_oracle(x: &FieldElement, k: &Scalar) -> Option { - let x_be = x.to_bytes(); - let k_be = k.to_bytes(); - let mut x_le = [0u8; 32]; - let mut k_le = [0u8; 32]; - for i in 0..32 { - x_le[i] = x_be[31 - i]; - k_le[i] = k_be[31 - i]; - } +fn ecsm_oracle(x_le: &[u8; 32], k: &Scalar) -> Option { + let k_le = scalar_le(k); let mut xr_le = [0u8; 32]; - lambda_vm_syscalls::syscalls::ecsm_mul(&mut xr_le, &x_le, &k_le); - xr_le.reverse(); - Option::from(FieldElement::from_bytes(&xr_le.into())) + lambda_vm_syscalls::syscalls::ecsm_mul(&mut xr_le, x_le, &k_le); + Fe::from_le_bytes(&xr_le) +} + +/// `k` as 32 little-endian bytes: its canonical value's limbs, low limb first. +#[cfg(any(target_arch = "riscv64", test))] +fn scalar_le(k: &Scalar) -> [u8; 32] { + const { assert!(core::mem::size_of::() == 8) }; + let mut out = [0u8; 32]; + for (dst, word) in out.chunks_exact_mut(8).zip(U256::from(k).to_words()) { + dst.copy_from_slice(&word.to_le_bytes()); + } + out } /// Base-field inverse `x⁻¹ mod p`. @@ -347,7 +447,7 @@ fn ecsm_oracle(x: &FieldElement, k: &Scalar) -> Option { /// in software directly. Returns `None` only for a genuinely non-invertible input /// (`x = 0`), which the callers' degeneracy guards already exclude. #[cfg(any(target_arch = "riscv64", test))] -fn field_inv(x: &FieldElement) -> Option { +fn field_inv(x: &Fe) -> Option { #[cfg(target_arch = "riscv64")] { field_inv_with_oracle(x, |x_be| { @@ -356,39 +456,84 @@ fn field_inv(x: &FieldElement) -> Option { } #[cfg(not(target_arch = "riscv64"))] { - Option::from(x.invert()) + software_field_inv(x) } } +/// `x⁻¹` by k256's software inversion; `None` for zero. +#[cfg(any(target_arch = "riscv64", test))] +fn software_field_inv(x: &Fe) -> Option { + let inv: Option = x.to_k256()?.invert().into(); + Some(Fe::from_k256(&inv?)) +} + /// Core of [`field_inv`], generic over the hint source so host tests can inject an /// honest or a lying oracle and assert the software fallback keeps the result /// correct either way. See [`scalar_inv`] for the verify-then-fallback rationale. #[cfg(any(target_arch = "riscv64", test))] -fn field_inv_with_oracle(x: &FieldElement, hint: O) -> Option +fn field_inv_with_oracle(x: &Fe, hint: O) -> Option where O: FnOnce(&[u8; 32]) -> [u8; 32], { - let x_be: [u8; 32] = x.to_bytes().into(); - let inv_be = hint(&x_be); + let inv_be = hint(&x.to_be_bytes()); // Fast path: a canonical hint that verifies (x·inv == 1 mod p) is used as-is. - // Verify by asking whether the difference normalizes to zero — a value-level test - // that skips the two full normalizations a `to_bytes()` compare pays. `ct_eq` is - // NOT a substitute: k256's FieldElement compares raw limbs *and* the magnitude and - // `normalized` tags, so a `mul` result (magnitude 1, unnormalized) never compares - // equal to the normalized `ONE` constant whatever its value. - // `Neg` is `negate(1)`, valid here because `mul` yields magnitude 1. - if let Some(inv) = Option::::from(FieldElement::from_bytes(&inv_be.into())) { - if bool::from((*x * inv - FieldElement::ONE).normalizes_to_zero()) { + if let Some(inv) = Fe::from_be_bytes(&inv_be) { + if (*x * inv).same(Fe::ONE) { return Some(inv); } } // Hint absent / malformed / wrong: recompute authoritatively. `None` only for a // genuine `x = 0`, excluded by the callers' guards. - Option::from(x.invert()) + software_field_inv(x) +} + +/// `k1·P1 + k2·P2` on curve points (the host tests' view of +/// [`lincomb2_xy_with_oracle`]), with an oracle on k256 field elements. +#[cfg(test)] +fn lincomb2_with_oracle( + a1: &AffinePoint, + k1: &Scalar, + a2: &AffinePoint, + k2: &Scalar, + oracle: O, +) -> Option +where + O: Fn(&FieldElement, &Scalar) -> Option, +{ + if bool::from(a1.is_identity()) || bool::from(a2.is_identity()) { + return None; + } + let (x1, y1) = affine_xy(a1)?; + let (x2, y2) = affine_xy(a2)?; + let (xq, yq) = lincomb2_xy_with_oracle( + &Fe::from_k256(&x1), + &Fe::from_k256(&y1), + k1, + &Fe::from_k256(&x2), + &Fe::from_k256(&y2), + k2, + le_oracle(oracle), + )?; + point_from_fe(xq, yq) +} + +/// An oracle on k256 field elements as one on the precompile's little-endian x. +#[cfg(test)] +fn le_oracle(oracle: O) -> impl Fn(&[u8; 32], &Scalar) -> Option +where + O: Fn(&FieldElement, &Scalar) -> Option, +{ + move |x_le, k| { + let x = Fe::from_le_bytes(x_le)?.to_k256()?; + Some(Fe::from_k256(&oracle(&x, k)?)) + } } /// Computes `k1·P1 + k2·P2` from four x-only oracle queries, or `None` if any -/// degenerate-configuration guard trips. +/// degenerate-configuration guard trips. `(x1, y1)` and `(x2, y2)` are the affine +/// coordinates of two non-identity curve points; the oracle takes a point's x as 32 +/// little-endian bytes, the precompile's encoding, converted once per point. The +/// result has been checked to be on the curve. /// /// The lambda-vm ECSM precompile returns only `x(k·P)`. For `A = k1·P1` with /// `P1 = (xp, yp)` fully known, query `xa = x(k1·P1)` and `xc = x((k1+1)·P1)`. @@ -402,37 +547,34 @@ where /// /// Generic over the oracle so unit tests can substitute a software stand-in. #[cfg(any(target_arch = "riscv64", test))] -fn lincomb2_with_oracle( - a1: &AffinePoint, +fn lincomb2_xy_with_oracle( + x1: &Fe, + y1: &Fe, k1: &Scalar, - a2: &AffinePoint, + x2: &Fe, + y2: &Fe, k2: &Scalar, oracle: O, -) -> Option +) -> Option<(Fe, Fe)> where - O: Fn(&FieldElement, &Scalar) -> Option, + O: Fn(&[u8; 32], &Scalar) -> Option, { - // Inputs are affine already (the ecrecover path lifts them from known Z=1 - // points), so no projective→affine inversion is needed here. - if bool::from(a1.is_identity()) || bool::from(a2.is_identity()) { - return None; - } if scalar_near_edge(k1) || scalar_near_edge(k2) { return None; } - - let (x1, y1) = affine_xy(a1)?; - let (x2, y2) = affine_xy(a2)?; - - let xa = oracle(&x1, k1)?; - let xc1 = oracle(&x1, &(*k1 + Scalar::ONE))?; - let xb = oracle(&x2, k2)?; - let xc2 = oracle(&x2, &(*k2 + Scalar::ONE))?; - - let dx1 = (xa - x1).normalize(); - let dx2 = (xb - x2).normalize(); - let dxq = (xb - xa).normalize(); - if bool::from(dx1.is_zero()) || bool::from(dx2.is_zero()) || bool::from(dxq.is_zero()) { + let (x1, y1, x2, y2) = (*x1, *y1, *x2, *y2); + let x1_le = x1.to_le_bytes(); + let x2_le = x2.to_le_bytes(); + + let xa = oracle(&x1_le, k1)?; + let xc1 = oracle(&x1_le, &(*k1 + Scalar::ONE))?; + let xb = oracle(&x2_le, k2)?; + let xc2 = oracle(&x2_le, &(*k2 + Scalar::ONE))?; + + let dx1 = xa - x1; + let dx2 = xb - x2; + let dxq = xb - xa; + if dx1.is_zero() || dx2.is_zero() || dxq.is_zero() { return None; } @@ -449,36 +591,40 @@ where // Q = A + B, with A ≠ ±B ensured by dxq ≠ 0. let lq = (yb - ya) * inv_dxq; - let xq = (lq.square() - xa - xb).normalize(); - let yq = (lq * (xa - xq) - ya).normalize(); + let xq = lq.square() - xa - xb; + let yq = lq * (xa - xq) - ya; - // `point_from_xy` checks the result is on the curve as a cheap backstop: + // The result is checked to be on the curve (`y² == x³+7`) as a cheap backstop: // it rejects gross off-curve garbage and falls back to software, but // correctness rests on the algebra above — an on-curve-but-wrong point // would still pass this check. - point_from_xy(&xq, &yq) + if !is_on_curve(&xq, &yq) { + return None; + } + Some((xq, yq)) } -/// Recovers `y(k·P)` from `xa = x(k·P)` and `xc = x((k+1)·P)`. +/// `y² == x³ + 7`. +#[cfg(any(target_arch = "riscv64", test))] +fn is_on_curve(x: &Fe, y: &Fe) -> bool { + (x.square() * *x + Fe::SEVEN).same(y.square()) +} + +/// Recovers `y(k·P)` from `xa = x(k·P)` and `xc = x((k+1)·P)`, for a curve point +/// `P = (xp, yp)`. /// Returns `None` if `xc` is inconsistent with the computed `lambda` -/// (oracle misbehavior); degeneracy guards are in [`lincomb2_with_oracle`]. +/// (oracle misbehavior); degeneracy guards are in [`lincomb2_xy_with_oracle`]. #[cfg(any(target_arch = "riscv64", test))] -fn solve_y( - xp: &FieldElement, - yp: &FieldElement, - xa: &FieldElement, - xc: &FieldElement, - dx: &FieldElement, - inv_den: &FieldElement, -) -> Option { - let t = *xc + xa + xp; - let xa3 = xa.square() * xa; - let xp3 = xp.square() * xp; - let lambda = (xa3 - xp3 - t * dx.square()) * inv_den; - if lambda.square().normalize() != t.normalize() { +fn solve_y(xp: &Fe, yp: &Fe, xa: &Fe, xc: &Fe, dx: &Fe, inv_den: &Fe) -> Option { + let t = *xc + *xa + *xp; + let xa3 = xa.square() * *xa; + // `xa³ − xp³ − t·dx²`, with `xp³ = yp² − 7` since P is on the curve: one squaring + // instead of a squaring and a product. + let lambda = (xa3 + Fe::SEVEN - yp.square() - t * dx.square()) * *inv_den; + if !lambda.square().same(t) { return None; } - Some((*yp + lambda * dx).normalize()) + Some(*yp + lambda * *dx) } /// `k ∈ {0, 1, n−1}`: fast early-exit before oracle calls. @@ -493,7 +639,6 @@ fn scalar_near_edge(k: &Scalar) -> bool { /// Affine `(x, y)` of a non-identity point as field elements, via its SEC1 /// uncompressed encoding (k256 keeps `AffinePoint`'s coordinate fields private). -#[cfg(any(target_arch = "riscv64", test))] fn affine_xy(p: &AffinePoint) -> Option<(FieldElement, FieldElement)> { let ep = p.to_encoded_point(false); let x = Option::::from(FieldElement::from_bytes(ep.x()?))?; @@ -503,14 +648,30 @@ fn affine_xy(p: &AffinePoint) -> Option<(FieldElement, FieldElement)> { /// Builds an affine curve point from coordinates, returning `None` if the point /// is not on the curve (`AffinePoint::from_encoded_point` validates this). -#[cfg(any(target_arch = "riscv64", test))] +#[cfg(test)] fn point_from_xy(x: &FieldElement, y: &FieldElement) -> Option { let ep = EncodedPoint::from_affine_coordinates(&x.to_bytes(), &y.to_bytes(), false); Option::::from(AffinePoint::from_encoded_point(&ep)) } +/// [`point_from_xy`] for coordinates in the four-limb field type. +fn point_from_fe(x: Fe, y: Fe) -> Option { + let ep = EncodedPoint::from_affine_coordinates( + &x.to_be_bytes().into(), + &y.to_be_bytes().into(), + false, + ); + Option::::from(AffinePoint::from_encoded_point(&ep)) +} + // ── Keccak-256 over the keccak_permute precompile (riscv64 guest) ─────────── +#[cfg(any(target_arch = "riscv64", test))] +mod keccak_cache; + +#[cfg(target_arch = "riscv64")] +static KECCAK_CACHE: keccak_cache::GuestCache = keccak_cache::GuestCache::new(); + /// Keccak-256 sponge with an injected permutation function. /// /// Keccak-f[1600], rate 1088 bits (136 bytes), capacity 512 bits. @@ -524,24 +685,81 @@ fn point_from_xy(x: &FieldElement, y: &FieldElement) -> Option { fn keccak256_with_permute(input: &[u8], mut permute: F) -> [u8; 32] { const RATE: usize = 136; - let mut state = [0u64; 25]; - let mut offset = 0; + let mut state; + if cfg!(target_endian = "little") && input.len() < RATE { + state = [0u64; 25]; + // SAFETY: the local state is initialized, writable, and disjoint from + // input. Its 200-byte allocation exceeds this <136-byte copy. On LE + // hosts these bytes are exactly Keccak's little-endian lane encoding. + unsafe { + core::ptr::copy_nonoverlapping( + input.as_ptr(), + state.as_mut_ptr().cast::(), + input.len(), + ); + } + state[input.len() / 8] |= 1u64 << (8 * (input.len() % 8)); + state[RATE / 8 - 1] |= 0x80u64 << 56; + permute(&mut state); + } else { + let rest = if let Some((first, rest)) = input.split_at_checked(RATE) { + // The first block absorbed into the zero state is the block itself: its + // lanes are written straight into the state, with the capacity lanes + // zero, instead of zeroing 200 bytes and then XORing the block in. + state = core::array::from_fn(|i| { + if i < RATE / 8 { + let mut buf = [0u8; 8]; + buf.copy_from_slice(&first[8 * i..8 * i + 8]); + u64::from_le_bytes(buf) + } else { + 0 + } + }); + permute(&mut state); + rest + } else { + state = [0u64; 25]; + input + }; + let mut blocks = rest.chunks_exact(RATE); + for block in &mut blocks { + absorb_block(&mut state, block); + permute(&mut state); + } - while input.len() - offset >= RATE { - absorb_block(&mut state, &input[offset..offset + RATE]); + // Final block with multi-rate padding, absorbed straight into the lanes: the + // remainder's whole lanes from the input, its last 0..=7 bytes and the 0x01 pad + // byte assembled into one lane, then the 0x80 pad bit on byte RATE - 1. The same + // XORs as padding a zeroed RATE-byte copy of the remainder, without the copy: + // this runs once per hash, and the guest hashes every trie node. + let mut lanes = blocks.remainder().chunks_exact(8); + let mut lane = 0; + for chunk in &mut lanes { + let mut buf = [0u8; 8]; + buf.copy_from_slice(chunk); + state[lane] ^= u64::from_le_bytes(buf); + lane += 1; + } + let tail = lanes.remainder(); + let mut last = 0x01u64 << (8 * tail.len()); + if !tail.is_empty() && input.len() >= 8 { + // The input's last eight bytes end with the tail: load them as one lane and + // shift out the bytes before the tail, instead of assembling the tail a byte + // at a time. (On a little-endian target this path only sees inputs of a + // full block or more.) + let mut word = [0u8; 8]; + word.copy_from_slice(&input[input.len() - 8..]); + last |= u64::from_le_bytes(word) >> (64 - 8 * tail.len()); + } else { + for (i, byte) in tail.iter().enumerate() { + last |= u64::from(*byte) << (8 * i); + } + } + state[lane] ^= last; + state[RATE / 8 - 1] ^= 0x80u64 << 56; permute(&mut state); - offset += RATE; } - // Final block with multi-rate padding. - let mut last = [0u8; RATE]; - let remaining = input.len() - offset; - last[..remaining].copy_from_slice(&input[offset..]); - last[remaining] ^= 0x01; - last[RATE - 1] ^= 0x80; - absorb_block(&mut state, &last); - permute(&mut state); - // Squeeze the first 32 bytes (four lanes) as little-endian. let mut output = [0u8; 32]; for (i, lane) in state.iter().take(4).enumerate() { @@ -550,10 +768,19 @@ fn keccak256_with_permute(input: &[u8], mut permute: F output } -/// Keccak-256 via LambdaVM's `keccak_permute` syscall (riscv64 guest only). +/// Keccak-256 via LambdaVM (riscv64 guest only): one `keccak256` call that +/// absorbs, pads, permutes and squeezes in the VM, or — when the buffer or the +/// output would cross a 2^32 address-word boundary, which the call refuses — the +/// sponge above over the `keccak_permute` syscall. #[cfg(target_arch = "riscv64")] fn keccak256_via_lambdavm(input: &[u8]) -> [u8; 32] { - keccak256_with_permute(input, |s| lambda_vm_syscalls::syscalls::keccak_permute(s)) + use lambda_vm_syscalls::syscalls; + let mut out = [0u8; 32]; + if syscalls::keccak256_fits(input, &out) { + syscalls::keccak256(input, &mut out); + return out; + } + keccak256_with_permute(input, |s| syscalls::keccak_permute(s)) } /// XOR one rate-sized block of bytes into the state lanes (little-endian). diff --git a/crypto/ethrex-crypto/src/tests/ecrecover_tests.rs b/crypto/ethrex-crypto/src/tests/ecrecover_tests.rs index af2ab1f1d..4a9cae617 100644 --- a/crypto/ethrex-crypto/src/tests/ecrecover_tests.rs +++ b/crypto/ethrex-crypto/src/tests/ecrecover_tests.rs @@ -124,3 +124,241 @@ fn ecrecover_rejects_zero_r() { "expected InvalidSignature for zero r" ); } + +/// Software stand-in for the ECSM precompile: `x(k·P)` for the point with +/// x-coordinate `x` (parity-invariant, like the real ecall). +fn soft_ecsm(x: &FieldElement, k: &Scalar) -> Option { + let xn = x.normalize(); + let y2 = (xn.square() * xn + FieldElement::from_u64(7)).normalize(); + let y = Option::::from(y2.sqrt())?; + let p = point_from_xy(&xn, &y.normalize())?; + Some(affine_xy(&(p * k).to_affine())?.0) +} + +/// Software stand-in for the field square-root hint. +fn soft_sqrt(rhs_be: &[u8; 32]) -> [u8; 32] { + let rhs = Option::::from(FieldElement::from_bytes(&(*rhs_be).into())) + .expect("canonical"); + match Option::::from(rhs.sqrt()) { + Some(y) => y.to_bytes().into(), + None => [0u8; 32], + } +} + +/// The accelerated recovery shape (R decompressed through the hint, the linear +/// combination through x-only queries, all on affine coordinates) with software +/// oracles, against the expected public key and against the software path +/// `ecsm_ecrecover` takes on the host. Honest and lying hints, both parities. +#[test] +fn accelerated_shape_recovers_the_signing_key() { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + state.wrapping_mul(0x2545_F491_4F6C_DD1D) + }; + let mut checked = 0; + while checked < 300 { + let d = Scalar::from(next()) * Scalar::from(next()) + Scalar::from(next()); + let kk = Scalar::from(next()) * Scalar::from(next()) + Scalar::from(next()); + let mut msg = [0u8; 32]; + for chunk in msg.chunks_mut(8) { + chunk.copy_from_slice(&next().to_le_bytes()); + } + if bool::from(d.is_zero()) || bool::from(kk.is_zero()) { + continue; + } + // Skip the ~2^-128 nonces whose R.x needs the high-x recovery id. + let r_point = (ProjectivePoint::GENERATOR * kk).to_affine(); + let rx = affine_xy(&r_point).unwrap().0; + if >::reduce_bytes(&rx.to_bytes()).to_bytes() != rx.to_bytes() { + continue; + } + let (sig, recid, _) = make_ecdsa_fixture(d, kk, msg); + let q = (ProjectivePoint::GENERATOR * d).to_affine(); + let mut want = [0u8; 64]; + want.copy_from_slice(&q.to_encoded_point(false).as_bytes()[1..65]); + + for (lie, parity_flip) in [(false, false), (true, false), (false, true)] { + let recid = recid ^ u8::from(parity_flip); + let got = ecrecover_with( + &sig, + recid, + &msg, + |r, odd| { + if lie { + decompress_r_xy_with_oracle(r, odd, |_| [0x11; 32]) + } else { + decompress_r_xy_with_oracle(r, odd, soft_sqrt) + } + }, + |x1, y1, k1, x2, y2, k2| { + lincomb2_xy_with_oracle(x1, y1, k1, x2, y2, k2, le_oracle(soft_ecsm)) + }, + ); + let got = got.ok(); + let software = ecsm_ecrecover(&sig, recid, &msg).ok(); + assert_eq!( + got, software, + "round {checked} lie {lie} flip {parity_flip}" + ); + if !parity_flip { + assert_eq!(got, Some(want), "round {checked} lie {lie}"); + } + } + checked += 1; + } +} + +/// ethrex's own recovery (the `Crypto` trait's default, pure Rust k256): the +/// reference the accelerated path must match. +#[derive(Debug)] +struct TraitDefault; +impl Crypto for TraitDefault {} + +/// The accelerated shape with software oracles, as the guest runs it. +fn accelerated(sig: &[u8; 64], recid: u8, msg: &[u8; 32]) -> Result<[u8; 64], CryptoError> { + ecrecover_with( + sig, + recid, + msg, + |r, odd| decompress_r_xy_with_oracle(r, odd, soft_sqrt), + |x1, y1, k1, x2, y2, k2| { + lincomb2_xy_with_oracle(x1, y1, k1, x2, y2, k2, le_oracle(soft_ecsm)) + }, + ) +} + +/// The address a recovered key gives, or `None` for an error. +fn address(pk: Option<[u8; 64]>) -> Option<[u8; 32]> { + pk.map(keccak_hash) +} + +/// Edge inputs, each against the software path the host build takes and, for +/// recovery ids 0 and 1, against ethrex's trait-default recovery: the high-s and +/// low-s forms of every signature (which recover the same key with the opposite +/// parity), recovery ids 2..=255, a result at the point at infinity, an `r` that +/// is no curve point's x, and `r` or `s` at or above the curve order. +#[test] +fn accelerated_shape_edge_cases() { + let mut state = 0x2545_F491_4F6C_DD1Du64; + let mut next = move || { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + state.wrapping_mul(0x2545_F491_4F6C_DD1D) + }; + let mut cases = 0; + while cases < 60 { + let d = Scalar::from(next()) * Scalar::from(next()) + Scalar::ONE; + let kk = Scalar::from(next()) * Scalar::from(next()) + Scalar::ONE; + let mut msg = [0u8; 32]; + for chunk in msg.chunks_mut(8) { + chunk.copy_from_slice(&next().to_le_bytes()); + } + let rx = affine_xy(&(ProjectivePoint::GENERATOR * kk).to_affine()) + .unwrap() + .0; + if >::reduce_bytes(&rx.to_bytes()).to_bytes() != rx.to_bytes() { + continue; + } + let (sig, recid, want_addr) = make_ecdsa_fixture(d, kk, msg); + + // The signature and its other-s form (s -> n - s, parity flipped). + let s = >::reduce_bytes(<&FieldBytes>::from(&sig[32..])); + let mut high = sig; + high[32..].copy_from_slice(&(-s).to_bytes()); + for (form, recid) in [(sig, recid), (high, recid ^ 1)] { + let got = accelerated(&form, recid, &msg).ok(); + assert_eq!(address(got), Some(want_addr), "case {cases}"); + assert_eq!(got, ecsm_ecrecover(&form, recid, &msg).ok(), "case {cases}"); + assert_eq!( + TraitDefault.secp256k1_ecrecover(&form, recid, &msg).ok(), + Some(want_addr), + "case {cases}" + ); + } + + // Every recovery id, including the invalid ones, as the software path. + for recid in [0u8, 1, 2, 3, 4, 27, 28, 255] { + let got = accelerated(&sig, recid, &msg); + let soft = ecsm_ecrecover(&sig, recid, &msg); + assert_eq!(got.is_ok(), soft.is_ok(), "case {cases} recid {recid}"); + assert_eq!(got.ok(), soft.ok(), "case {cases} recid {recid}"); + } + cases += 1; + } + + // A signature whose recovered point is the point at infinity: with + // R = (z / s)·G, u1·G + u2·R = r⁻¹·(s·R − z·G) = 0. + let mut msg = [0u8; 32]; + msg[31] = 0x2a; + let z = >::reduce_bytes(&FieldBytes::from(msg)); + let s = Scalar::from(0x1234_5678u64); + let kk = z * s.invert_vartime().unwrap(); + let (rx, ry) = affine_xy(&(ProjectivePoint::GENERATOR * kk).to_affine()).unwrap(); + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&rx.to_bytes()); + sig[32..].copy_from_slice(&s.to_bytes()); + let recid = ry.normalize().to_bytes()[31] & 1; + assert!(matches!( + accelerated(&sig, recid, &msg), + Err(CryptoError::RecoveryFailed) + )); + assert!(matches!( + ecsm_ecrecover(&sig, recid, &msg), + Err(CryptoError::RecoveryFailed) + )); + + // An r that is no curve point's x (x³ + 7 a non-residue): no R to decompress. + let mut x = 1u64; + let bad_x = loop { + let fx = FieldElement::from_u64(x); + if bool::from( + (fx.square() * fx + FieldElement::from_u64(7)) + .sqrt() + .is_none(), + ) { + break fx; + } + x += 1; + }; + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&bad_x.to_bytes()); + sig[63] = 1; + for recid in [0u8, 1] { + assert!(matches!( + accelerated(&sig, recid, &msg), + Err(CryptoError::RecoveryFailed) + )); + assert!(matches!( + ecsm_ecrecover(&sig, recid, &msg), + Err(CryptoError::RecoveryFailed) + )); + } + + // r or s at or above the curve order n: rejected before any recovery. + let n_bytes: [u8; 32] = [ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, + 0x41, 0x41, + ]; + for (r_half, s_half) in [ + (n_bytes, [1u8; 32]), + ([1u8; 32], n_bytes), + ([0xFF; 32], [1u8; 32]), + ] { + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&r_half); + sig[32..].copy_from_slice(&s_half); + assert!(matches!( + accelerated(&sig, 0, &msg), + Err(CryptoError::InvalidSignature) + )); + assert!(matches!( + ecsm_ecrecover(&sig, 0, &msg), + Err(CryptoError::InvalidSignature) + )); + } +} diff --git a/crypto/ethrex-crypto/src/tests/ecsm_tests.rs b/crypto/ethrex-crypto/src/tests/ecsm_tests.rs index 42e80224b..adb42b0f7 100644 --- a/crypto/ethrex-crypto/src/tests/ecsm_tests.rs +++ b/crypto/ethrex-crypto/src/tests/ecsm_tests.rs @@ -147,6 +147,9 @@ fn solve_y_rejects_inconsistent_oracle_xc() { let inv_den = Option::::from((yp.double() * dx).invert()) .expect("dx is nonzero for k=12345"); + let f = Fe::from_k256; + let (xp, yp, xa, dx, inv_den) = (f(&xp), f(&yp), f(&xa), f(&dx), f(&inv_den)); + let (xc_correct, xc_wrong) = (f(&xc_correct), f(&xc_wrong)); assert!( solve_y(&xp, &yp, &xa, &xc_correct, &dx, &inv_den).is_some(), "correct xc must pass the lambda² check" @@ -181,3 +184,45 @@ fn odd_y_base_point_reconstructs_correctly() { .expect("odd-y base point is non-degenerate and must reconstruct correctly"); assert_eq!(got, expected.to_affine()); } + +/// The precompile's little-endian encodings: a field element's little-endian bytes +/// are the reversal of its big-endian ones, and `scalar_le` is the reversal of the +/// scalar's big-endian bytes, on edge values and seeded random ones. +#[test] +fn little_endian_encodings() { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + state.wrapping_mul(0x2545_F491_4F6C_DD1D) + }; + let mut scalars = vec![ + Scalar::ZERO, + Scalar::ONE, + -Scalar::ONE, + Scalar::from(u64::MAX), + ]; + for _ in 0..200 { + let mut bytes = [0u8; 32]; + for chunk in bytes.chunks_exact_mut(8) { + chunk.copy_from_slice(&next().to_le_bytes()); + } + let mut reversed = bytes; + reversed.reverse(); + if let Some(fe) = Fe::from_le_bytes(&bytes) { + assert_eq!(fe.to_le_bytes(), bytes); + assert_eq!(fe.to_be_bytes(), reversed); + assert_eq!( + Fe::from_be_bytes(&reversed).map(|f| f.to_le_bytes()), + Some(bytes) + ); + } + scalars.push(>::reduce_bytes(&bytes.into())); + } + for k in scalars { + let mut want: [u8; 32] = k.to_bytes().into(); + want.reverse(); + assert_eq!(scalar_le(&k), want); + } +} diff --git a/crypto/ethrex-crypto/src/tests/fe_tests.rs b/crypto/ethrex-crypto/src/tests/fe_tests.rs new file mode 100644 index 000000000..79c0847f5 --- /dev/null +++ b/crypto/ethrex-crypto/src/tests/fe_tests.rs @@ -0,0 +1,262 @@ +//! The four-limb base-field type (`fe.rs`) against k256's `FieldElement`: every +//! operation on edge values and seeded random ones, inputs in the weak form included +//! (limb patterns from p up to 2^256 − 1, which stand for value − p). + +use crate::fe::Fe; +use crate::*; +use k256::elliptic_curve::bigint::ArrayEncoding; + +/// p, big-endian. +const P_BE: [u8; 32] = [ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x2F, +]; + +fn p() -> U256 { + U256::from_be_slice(&P_BE) +} + +/// The k256 element a limb pattern stands for, reduced independently of `fe.rs`. +fn reference(limbs: [u64; 4]) -> FieldElement { + let mut v = U256::from_words(limbs); + if v >= p() { + v = v.wrapping_sub(&p()); + } + let be: [u8; 32] = v.to_be_byte_array().into(); + Option::::from(FieldElement::from_bytes(&be.into())).expect("below p") +} + +fn canonical_be(x: FieldElement) -> [u8; 32] { + x.normalize().to_bytes().into() +} + +/// Edge limb patterns: zero, one, around p, around 2^256, around C = 2^256 − p, and +/// patterns that make carries and borrows run through every limb. +fn edges() -> Vec<[u64; 4]> { + let p = p().to_words(); + let mut out = vec![ + [0, 0, 0, 0], + [1, 0, 0, 0], + [2, 0, 0, 0], + [7, 0, 0, 0], + [0x1_0000_03D0, 0, 0, 0], + [0x1_0000_03D1, 0, 0, 0], + [0x1_0000_03D2, 0, 0, 0], + [u64::MAX, 0, 0, 0], + [0, 1, 0, 0], + [0, 0, 0, 1 << 63], + [u64::MAX; 4], + [u64::MAX - 1, u64::MAX, u64::MAX, u64::MAX], + [0, u64::MAX, u64::MAX, u64::MAX], + [u64::MAX, u64::MAX, u64::MAX, 0], + [0, 0, 0, u64::MAX], + ]; + for d in [0u64, 1, 2, 977, 0x1_0000_03D0] { + // p + d and p − 1 − d. + let (lo, c) = p[0].overflowing_add(d); + if !c { + out.push([lo, p[1], p[2], p[3]]); + } + out.push([p[0] - 1 - d, p[1], p[2], p[3]]); + } + out +} + +fn random_limbs(next: &mut impl FnMut() -> u64, round: usize) -> [u64; 4] { + let mut l = [next(), next(), next(), next()]; + match round % 5 { + // Near 2^256: the weak range [p, 2^256) and just below it. + 0 => { + l[1] = u64::MAX; + l[2] = u64::MAX; + l[3] = u64::MAX; + } + // Small values. + 1 => { + l[1] = 0; + l[2] = 0; + l[3] = 0; + } + _ => {} + } + l +} + +fn rng() -> impl FnMut() -> u64 { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + move || { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + state.wrapping_mul(0x2545_F491_4F6C_DD1D) + } +} + +fn check_pair(a: [u64; 4], b: [u64; 4]) { + let (fa, fb) = (Fe::from_limbs(a), Fe::from_limbs(b)); + let (ka, kb) = (reference(a), reference(b)); + let ctx = || format!("a = {a:x?}, b = {b:x?}"); + assert_eq!( + (fa + fb).to_be_bytes(), + canonical_be(ka + kb), + "add {}", + ctx() + ); + assert_eq!( + (fa - fb).to_be_bytes(), + canonical_be(ka - kb), + "sub {}", + ctx() + ); + assert_eq!( + (fa * fb).to_be_bytes(), + canonical_be(ka * kb), + "mul {}", + ctx() + ); + assert_eq!( + fa.same(fb), + canonical_be(ka) == canonical_be(kb), + "same {}", + ctx() + ); +} + +fn check_one(a: [u64; 4]) { + let fa = Fe::from_limbs(a); + let ka = reference(a); + let ctx = || format!("a = {a:x?}"); + let want = canonical_be(ka); + assert_eq!(fa.to_be_bytes(), want, "to_be_bytes {}", ctx()); + let mut want_le = want; + want_le.reverse(); + assert_eq!(fa.to_le_bytes(), want_le, "to_le_bytes {}", ctx()); + assert_eq!( + fa.square().to_be_bytes(), + canonical_be(ka.square()), + "square {}", + ctx() + ); + assert_eq!( + fa.double().to_be_bytes(), + canonical_be(ka.double()), + "double {}", + ctx() + ); + assert_eq!((-fa).to_be_bytes(), canonical_be(-ka), "neg {}", ctx()); + assert_eq!( + fa.is_zero(), + bool::from(ka.normalizes_to_zero()), + "is_zero {}", + ctx() + ); + assert_eq!(fa.is_odd(), want[31] & 1 == 1, "is_odd {}", ctx()); + assert_eq!( + Fe::from_k256(&ka).to_be_bytes(), + want, + "from_k256 {}", + ctx() + ); + assert_eq!( + fa.to_k256().map(canonical_be), + Some(want), + "to_k256 {}", + ctx() + ); + // Decoding accepts exactly the canonical encodings, as `FieldElement::from_bytes`. + let mut raw_be = [0u8; 32]; + for (chunk, limb) in raw_be.rchunks_exact_mut(8).zip(a) { + chunk.copy_from_slice(&limb.to_be_bytes()); + } + let mut raw_le = raw_be; + raw_le.reverse(); + let k256_accepts = bool::from(FieldElement::from_bytes(&raw_be.into()).is_some()); + assert_eq!( + Fe::from_be_bytes(&raw_be).is_some(), + k256_accepts, + "from_be_bytes {}", + ctx() + ); + assert_eq!( + Fe::from_le_bytes(&raw_le).is_some(), + k256_accepts, + "from_le_bytes {}", + ctx() + ); + if k256_accepts { + assert_eq!( + Fe::from_be_bytes(&raw_be).map(Fe::to_be_bytes), + Some(raw_be) + ); + assert_eq!(Fe::from_le_bytes(&raw_le).map(Fe::limbs), Some(a)); + } +} + +#[test] +fn edge_values_match_k256() { + let edges = edges(); + for &a in &edges { + check_one(a); + for &b in &edges { + check_pair(a, b); + } + } +} + +#[test] +fn random_values_match_k256() { + let mut next = rng(); + let edges = edges(); + for round in 0..20_000 { + let a = random_limbs(&mut next, round); + let b = random_limbs(&mut next, round / 5); + check_one(a); + check_pair(a, b); + check_pair(a, edges[round % edges.len()]); + } +} + +/// Long chains in the weak form stay correct: 2,000 rounds of products, squares, +/// sums and differences fed back into each other, against the same chain in k256. +#[test] +fn chained_operations_match_k256() { + let mut next = rng(); + let mut f = [Fe::from_limbs([u64::MAX; 4]), Fe::from_limbs([3, 0, 0, 0])]; + let mut k = [reference([u64::MAX; 4]), reference([3, 0, 0, 0])]; + for round in 0..2_000 { + let i = round % 2; + let x = random_limbs(&mut next, round); + let (fx, kx) = (Fe::from_limbs(x), reference(x)); + match next() % 5 { + 0 => { + f[i] = f[i] * f[1 - i]; + k[i] = (k[i] * k[1 - i]).normalize(); + } + 1 => { + f[i] = f[i].square() + fx; + k[i] = (k[i].square() + kx).normalize(); + } + 2 => { + f[i] = f[i] - fx; + k[i] = (k[i] - kx).normalize(); + } + 3 => { + f[i] = -(f[i] + f[1 - i]); + k[i] = (-(k[i] + k[1 - i]).normalize()).normalize(); + } + _ => { + f[i] = f[i].double() * fx; + k[i] = (k[i].double() * kx).normalize(); + } + } + assert_eq!(f[i].to_be_bytes(), canonical_be(k[i]), "round {round}"); + } +} + +/// The generator constants are k256's generator. +#[test] +fn generator_constants_are_k256s() { + let (x, y) = affine_xy(&AffinePoint::GENERATOR).unwrap(); + assert_eq!(GENERATOR_X.to_be_bytes(), canonical_be(x)); + assert_eq!(GENERATOR_Y.to_be_bytes(), canonical_be(y)); +} diff --git a/crypto/ethrex-crypto/src/tests/hint_tests.rs b/crypto/ethrex-crypto/src/tests/hint_tests.rs index ace59f208..bec6d7e36 100644 --- a/crypto/ethrex-crypto/src/tests/hint_tests.rs +++ b/crypto/ethrex-crypto/src/tests/hint_tests.rs @@ -215,9 +215,10 @@ fn field_inv_honest_hint_matches_software() { for k in [1u64, 2, 3, 7, 1000, 12345] { let x = fe_from_u64(k); let sw = Option::::from(x.invert()).expect("k != 0 is invertible"); - let got = field_inv_with_oracle(&x, honest_field_inv).expect("inverse exists"); + let got = + field_inv_with_oracle(&Fe::from_k256(&x), honest_field_inv).expect("inverse exists"); assert_eq!( - got.normalize().to_bytes(), + FieldBytes::from(got.to_be_bytes()), sw.normalize().to_bytes(), "honest hint must equal the software inverse (k={k})" ); @@ -233,9 +234,10 @@ fn field_inv_lying_hint_falls_back_to_software() { for k in [1u64, 2, 12345] { let x = fe_from_u64(k); let sw = Option::::from(x.invert()).unwrap(); - let got = field_inv_with_oracle(&x, |_| lie).expect("fallback recomputes"); + let got = + field_inv_with_oracle(&Fe::from_k256(&x), |_| lie).expect("fallback recomputes"); assert_eq!( - got.normalize().to_bytes(), + FieldBytes::from(got.to_be_bytes()), sw.normalize().to_bytes(), "lying hint must fall back to the software inverse (k={k})" ); @@ -259,9 +261,10 @@ fn field_inv_canonical_but_wrong_hint_falls_back_to_software() { ("-inv", -sw), ] { let lie_be: [u8; 32] = lie.normalize().to_bytes().into(); - let got = field_inv_with_oracle(&x, |_| lie_be).expect("fallback recomputes"); + let got = + field_inv_with_oracle(&Fe::from_k256(&x), |_| lie_be).expect("fallback recomputes"); assert_eq!( - got.normalize().to_bytes(), + FieldBytes::from(got.to_be_bytes()), sw.to_bytes(), "a canonical-but-wrong hint ({name}) must be rejected and recomputed (k={k})" ); diff --git a/crypto/ethrex-crypto/src/tests/keccak_cache_tests.rs b/crypto/ethrex-crypto/src/tests/keccak_cache_tests.rs new file mode 100644 index 000000000..f754e423b --- /dev/null +++ b/crypto/ethrex-crypto/src/tests/keccak_cache_tests.rs @@ -0,0 +1,125 @@ +//! Host-side tests for the guest's Keccak-256 cache (`keccak_cache`), with ethrex's +//! reference `keccak_hash` as the hash behind it. + +use crate::keccak_cache::{KeccakCache, EMPTY_DIGEST}; +use ethrex_crypto::keccak::keccak_hash; + +/// Look `input` up and check the digest against the reference; returns whether the +/// cache had to hash (a miss). +fn lookup(cache: &mut KeccakCache, input: &[u8]) -> bool { + let mut hashed = false; + let got = cache.keccak256(input, |x| { + hashed = true; + keccak_hash(x) + }); + assert_eq!( + got, + keccak_hash(input), + "{}-byte input {input:02x?}", + input.len() + ); + hashed +} + +fn pattern(len: usize, seed: u64) -> Vec { + (0..len as u64) + .map(|i| { + ((i + 1) + .wrapping_mul(seed | 1) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + >> 56) as u8 + }) + .collect() +} + +#[test] +fn empty_digest_is_keccak_of_nothing() { + assert_eq!(EMPTY_DIGEST, keccak_hash([])); +} + +#[test] +fn every_length_to_three_blocks_twice() { + let mut cache = KeccakCache::new(); + for round in 0..2 { + for len in 0..=410 { + let hashed = lookup(&mut cache, &pattern(len, len as u64)); + let cached = matches!(len, 0 | 20 | 32 | 64); + // The empty input never hashes; the cached lengths hash only the first + // time; every other length hashes every time. + let want = len != 0 && !(cached && round == 1); + assert_eq!(hashed, want, "length {len} round {round}"); + } + } +} + +/// Two inputs that differ in doublewords `i` and `j` by the same value fold to the +/// same slot. Alternating them must replace the entry each time and never return +/// the other one's digest, whichever pair of words differs; for 20 bytes the delta +/// stays within the last word's four bytes. +#[test] +fn same_slot_inputs_never_share_a_digest() { + for (len, words) in [(20usize, 3usize), (32, 4), (64, 8)] { + for i in 0..words { + for j in i + 1..words { + let mut cache = KeccakCache::new(); + let a = pattern(len, (len * 100 + i * 10 + j) as u64); + let mut b = a.clone(); + // only the bytes both words have, so that the two folds stay equal + let bytes = (len - 8 * j).min(8); + let delta = 0x0102_0304_a5c3_5a3cu64 & (u64::MAX >> (64 - 8 * bytes)); + for w in [i, j] { + for k in 0..bytes { + b[8 * w + k] ^= (delta >> (8 * k)) as u8; + } + } + for _ in 0..3 { + assert!( + lookup(&mut cache, &a), + "len {len} words {i} {j}: a must miss" + ); + assert!( + lookup(&mut cache, &b), + "len {len} words {i} {j}: b must miss" + ); + } + } + } + } +} + +/// A one-byte change anywhere in a cached input is a different input. +#[test] +fn every_single_byte_change_misses() { + for len in [20usize, 32, 64] { + let mut cache = KeccakCache::new(); + let a = pattern(len, 7); + assert!(lookup(&mut cache, &a)); + for at in 0..len { + for x in [0x01u8, 0x80] { + let mut b = a.clone(); + b[at] ^= x; + lookup(&mut cache, &b); + lookup(&mut cache, &a); + } + } + } +} + +#[test] +fn many_inputs_stay_correct_past_capacity() { + let mut cache = KeccakCache::new(); + for round in 0..2 { + for seed in 0..20_000u64 { + for len in [20usize, 32, 64] { + let mut input = pattern(len, seed); + // small big-endian integers, like storage slot numbers + if seed % 3 == 0 { + input.fill(0); + input[len - 8..].copy_from_slice(&seed.to_be_bytes()); + } + lookup(&mut cache, &input); + } + } + let _ = round; + } +} diff --git a/crypto/ethrex-crypto/src/tests/keccak_tests.rs b/crypto/ethrex-crypto/src/tests/keccak_tests.rs index 14d497520..ef2409da6 100644 --- a/crypto/ethrex-crypto/src/tests/keccak_tests.rs +++ b/crypto/ethrex-crypto/src/tests/keccak_tests.rs @@ -53,6 +53,20 @@ fn keccak_sponge_matches_trusted_permutation() { check_keccak(&[0xbb; 271]); } +#[test] +fn keccak_sponge_every_length_to_three_blocks() { + // The final block is absorbed lane by lane, so every remainder length (whole + // lanes, a partial lane of 1..=7 bytes, the pad byte landing in either) is + // worth a case. A pattern with no short period, so a byte landing in the wrong + // position of its lane changes the hash. + let input: Vec = (0..3 * 136 + 2) + .map(|i: u32| (i.wrapping_mul(0x9E37_79B9) >> 24) as u8) + .collect(); + for n in 0..=input.len() { + check_keccak(&input[..n]); + } +} + #[test] fn keccak_sponge_known_answer_vectors() { // Vectors from the Ethereum Yellow Paper / EIP-155. These use Keccak-256 @@ -76,3 +90,33 @@ fn keccak_sponge_known_answer_vectors() { "4d741b6f1eb29cb2a9b9911c82f56fa8d73b04959d3d9d222895df6c0b28aa15", ); } + +#[test] +fn keccak_alignment_lengths_and_permutation_counts() { + for len in (0usize..=410).chain([544, 545, 1024, 4095, 4096, 16384]) { + for alignment in 0..8 { + for pattern in 0..4 { + let data: Vec = (0..len + alignment) + .map(|i| match pattern { + 0 => 0, + 1 => 255, + 2 => ((i as u32).wrapping_mul(0x9e3779b9) >> 24) as u8, + _ => i as u8, + }) + .collect(); + let input = &data[alignment..]; + let mut count = 0; + let got = keccak256_with_permute(input, |state| { + count += 1; + keccak::f1600(state); + }); + assert_eq!( + got, + keccak_hash(input), + "len={len}, alignment={alignment}, pattern={pattern}" + ); + assert_eq!(count, len / 136 + 1); + } + } + } +} diff --git a/crypto/ethrex-crypto/src/tests/mod.rs b/crypto/ethrex-crypto/src/tests/mod.rs index 37fc9b3a0..961a599bd 100644 --- a/crypto/ethrex-crypto/src/tests/mod.rs +++ b/crypto/ethrex-crypto/src/tests/mod.rs @@ -3,6 +3,10 @@ pub mod ecrecover_tests; #[cfg(test)] pub mod ecsm_tests; #[cfg(test)] +pub mod fe_tests; +#[cfg(test)] pub mod hint_tests; #[cfg(test)] +pub mod keccak_cache_tests; +#[cfg(test)] pub mod keccak_tests; diff --git a/crypto/math-cuda/Cargo.toml b/crypto/math-cuda/Cargo.toml index 2304af398..db1be530b 100644 --- a/crypto/math-cuda/Cargo.toml +++ b/crypto/math-cuda/Cargo.toml @@ -21,7 +21,9 @@ license.workspace = true # * This replaces the fragile per-script `sed` pin in scripts/gpu_test.sh. # To move the floor (e.g. to use a newer driver-API symbol), bump this one # feature deliberately — see crypto/math-cuda/build.rs and README "GPU Tests". -cudarc = { version = "0.19", default-features = false, features = [ +# The version is pinned exactly: 0.19.10 is the first with +# `CudaContext::new_stream_with_priority` (the stream classes). +cudarc = { version = "=0.19.10", default-features = false, features = [ "driver", "nvrtc", "std", @@ -34,6 +36,12 @@ rayon = "1.7" libloading = { version = "0.8", optional = true } [features] +# `blake3-6round` is on by default so the cubin's round count matches the host +# tree's, which P-a Stage 6 moved to 6 by default for the same reason. These are +# separate crates' features and the lockstep between them is what makes the +# device and host hashes one hash; defaulting only one of them is precisely the +# mismatch the parity tests would report as a wrong root. +default = ["blake3-6round"] # Test-only fault injection in FriCommitState. Production builds leave this # off so the fault check is fully elided at compile time. test-faults = [] @@ -41,9 +49,22 @@ test-faults = [] # stark/prover layers). Zero-cost when disabled; when enabled but # libnvToolsExt is absent at runtime, every call is a cheap no-op. nvtx = ["dep:libloading"] +# Compile `kernels/blake3.cu` for the 6-round internal BLAKE3 variant instead of +# the 7-round standard one. Same polarity as the host tree's `blake3-6round` +# (on = 6 rounds), and it has to be set in lockstep with it: they are separate +# crates' features and a mismatch is a GPU tree committing under a different hash +# than the CPU one. `blake3::device_rounds` reads the cubin's round count back so +# that is assertable rather than discoverable — see +# `tests/blake3_compress_parity.rs`. +blake3-6round = [] [dev-dependencies] +# 7-round-only, so it can anchor the standard arm of the BLAKE3 kernels and +# nothing else — the same dev-only role it has in `prover/Cargo.toml`. +blake3 = { version = "1.8.5", default-features = false, features = ["std", "pure"] } crypto = { path = "../crypto" } +# The host pipeline the WHIR commit parity test compares against. +multilinear = { path = "../multilinear" } rand = { version = "0.8.5", features = ["std"] } rand_chacha = "0.3.1" rayon = "1.7" diff --git a/crypto/math-cuda/build.rs b/crypto/math-cuda/build.rs index bbb9943b9..5443fa16c 100644 --- a/crypto/math-cuda/build.rs +++ b/crypto/math-cuda/build.rs @@ -74,12 +74,21 @@ fn to_real_arch(arch: &str) -> String { /// Single source for the barycentric multi-kernel eval-point cap. The CUDA /// side sizes a per-thread accumulator array with it (`BARY_MAX_K`, passed via -/// `-D` below) and the Rust dispatch asserts against it (generated into -/// `bary_consts.rs`) — defining it twice invites stack corruption in the -/// kernel the day one side moves without the other. +/// `-D` at the barycentric.cu call site below) and the Rust dispatch asserts +/// against it (generated into `bary_consts.rs`) — defining it twice invites +/// stack corruption in the kernel the day one side moves without the other. const BARY_MAX_EVAL_POINTS: usize = 8; -fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool) { +/// The RPX permutation variant of the default RPX cubin, `rpx_v.cubin`: +/// `RPX_PERMUTE_VARIANT` in `kernels/rpx.cu`, a bit set of limb multiply (1), +/// limb square (2), unrolled `square_n` (4) and unrolled lane loops (8). The +/// source is also built at 0, the 64-bit multiply, as `rpx_v0.cubin`; +/// `rpx_paths::limb_permute` picks one of the two when the backend loads. +/// Mirrored into Rust (`rpx_paths::LIMB_PERMUTE_VARIANT`) and into the cubin +/// table (`device.rs`) by the generated files below. +const RPX_LIMB_PERMUTE_VARIANT: u32 = 5; + +fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool, defines: &[&str]) { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let src_path = manifest_dir.join("kernels").join(src); @@ -90,6 +99,7 @@ fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool) { println!("cargo:rerun-if-env-changed=CUDA_PATH"); println!("cargo:rerun-if-env-changed=CUDARC_NVCC_ARCH"); println!("cargo:rerun-if-env-changed=LAMBDA_VM_NVCC_LINEINFO"); + println!("cargo:rerun-if-env-changed=LAMBDA_VM_RPX_MAXRREGCOUNT"); // When nvcc is missing from PATH, emit an empty cubin stub so the crate // still compiles. include_bytes! in src/device.rs needs the file to exist @@ -125,12 +135,25 @@ fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool) { let mut cmd = Command::new(nvcc_path()); cmd.args(["--cubin", "-O3", "-std=c++17", "-arch", &arch]); - cmd.arg(format!("-DBARY_MAX_K={BARY_MAX_EVAL_POINTS}")); + cmd.args(defines); // SASS→source line mapping for Nsight Compute. Unlike -G this does not // change codegen, but keep it opt-in so production cubins stay byte-stable. if env::var("LAMBDA_VM_NVCC_LINEINFO").is_ok_and(|v| v != "0" && !v.is_empty()) { cmd.arg("-lineinfo"); } + // Opt-in per-thread register cap for the round-3 occupancy discriminator. + // Capping registers lets more blocks reside per SM, tracing the RPX + // permutation's throughput-vs-occupancy curve across separate builds (the + // production grind kernel cannot carry a per-kernel `__launch_bounds__`, and + // it is what this sweep must move). Empty/unset/"0" ⇒ no cap ⇒ production + // cubins stay byte-stable, exactly like `-lineinfo` above. Diagnostic only; + // never set in a production or bench build. + if let Ok(r) = env::var("LAMBDA_VM_RPX_MAXRREGCOUNT") + && !r.is_empty() + && r != "0" + { + cmd.arg(format!("-maxrregcount={r}")); + } let status = cmd .arg("-o") .arg(&out_path) @@ -156,11 +179,31 @@ fn main() { ), ) .expect("failed to write bary_consts.rs"); + // The limb cubin's variant, and the two RPX cubins the backend chooses from. + let v = RPX_LIMB_PERMUTE_VARIANT; + fs::write( + out_dir.join("rpx_variants.rs"), + format!( + "/// The permutation variant `rpx_v.cubin` is built at (build.rs).\n\ + pub const LIMB_PERMUTE_VARIANT: u32 = {v};\n" + ), + ) + .expect("failed to write rpx_variants.rs"); + let cubin = |name: &str, file: &str| { + format!("const {name}: &[u8] = include_bytes!(concat!(env!(\"OUT_DIR\"), \"/{file}\"));\n") + }; + fs::write( + out_dir.join("rpx_cubins.rs"), + cubin("RPX_V0_CUBIN", "rpx_v0.cubin") + + &cubin("RPX_LIMB_CUBIN", &format!("rpx_v{v}.cubin")), + ) + .expect("failed to write rpx_cubins.rs"); // Headers aren't compiled, so emit rerun-if-changed to rebuild on // header edits. println!("cargo:rerun-if-changed=kernels/goldilocks.cuh"); println!("cargo:rerun-if-changed=kernels/ext3.cuh"); + println!("cargo:rerun-if-changed=kernels/ext3_inv.cuh"); // Probe for nvcc once. Workspace consumers (clippy, fmt, CPU-only test // runners) build math-cuda incidentally without using its kernels. Stub @@ -178,13 +221,68 @@ fn main() { ); } - compile_kernel("arith.cu", "arith.cubin", have_nvcc); - compile_kernel("ntt.cu", "ntt.cubin", have_nvcc); - compile_kernel("keccak.cu", "keccak.cubin", have_nvcc); - compile_kernel("barycentric.cu", "barycentric.cubin", have_nvcc); - compile_kernel("deep.cu", "deep.cubin", have_nvcc); - compile_kernel("fri.cu", "fri.cubin", have_nvcc); - compile_kernel("inverse.cu", "inverse.cubin", have_nvcc); - compile_kernel("logup.cu", "logup.cubin", have_nvcc); - compile_kernel("constraint_interp.cu", "constraint_interp.cubin", have_nvcc); + compile_kernel("arith.cu", "arith.cubin", have_nvcc, &[]); + // The table fills from records (`src/fill.rs`), loaded on first use, never + // by the backend. Pinned on the host by `tests/host_kat/fill_host_kat.cpp`. + compile_kernel("fill.cu", "fill.cubin", have_nvcc, &[]); + compile_kernel("ntt.cu", "ntt.cubin", have_nvcc, &[]); + // The column-major LDE engine (`src/lde_cm.rs`). Pinned on the host by + // `tests/host_kat/ntt_cm_host_kat.cpp` (`make test-ntt-cm-host-kat`). + compile_kernel("ntt_cm.cu", "ntt_cm.cubin", have_nvcc, &[]); + compile_kernel("keccak.cu", "keccak.cubin", have_nvcc, &[]); + let bary_define = format!("-DBARY_MAX_K={BARY_MAX_EVAL_POINTS}"); + compile_kernel( + "barycentric.cu", + "barycentric.cubin", + have_nvcc, + &[&bary_define], + ); + compile_kernel("deep.cu", "deep.cubin", have_nvcc, &[]); + compile_kernel("fri.cu", "fri.cubin", have_nvcc, &[]); + compile_kernel("inverse.cu", "inverse.cubin", have_nvcc, &[]); + // RPX256 (XHash12) leaves, parents and grind — the algebraic hash's device + // kernels, pinned on the host by `tests/host_kat/rpx_host_kat.cpp` at every + // variant. Built twice: at 0 (no `-D`, the source's default: the 64-bit + // multiply) and at the limb variant, the default the backend loads. + compile_kernel("rpx.cu", "rpx_v0.cubin", have_nvcc, &[]); + let limb_define = format!("-DRPX_PERMUTE_VARIANT={RPX_LIMB_PERMUTE_VARIANT}"); + compile_kernel( + "rpx.cu", + &format!("rpx_v{RPX_LIMB_PERMUTE_VARIANT}.cubin"), + have_nvcc, + &[&limb_define], + ); + compile_kernel("logup.cu", "logup.cubin", have_nvcc, &[]); + compile_kernel( + "constraint_interp.cu", + "constraint_interp.cubin", + have_nvcc, + &[], + ); + // The BLAKE3 kernels' round count is a compile-time knob with the same + // polarity as the host tree's `blake3-6round` feature: 7 rounds (standard + // BLAKE3) unless the feature selects the 6-round variant. The `.cu` defaults + // to 7 on its own, so a stale `-D` can never silently pick 6. + // `CARGO_FEATURE_*`, not `cfg!(feature = ..)`: cargo passes a build script + // the active features as environment variables and does NOT cfg them into + // its compilation, so the `cfg!` form here would silently always be false. + let blake3_defines: &[&str] = if env::var_os("CARGO_FEATURE_BLAKE3_6ROUND").is_some() { + &["-DBLAKE3_ROUNDS=6"] + } else { + &[] + }; + compile_kernel("blake3.cu", "blake3.cubin", have_nvcc, blake3_defines); + // The WHIR path's own kernels. They carry no compile-time knob, so they + // take the empty define list rather than gaining a second signature. + // D-HASH stage 1's Poseidon1 width-16 MEASUREMENT kernels (`src/p1w16.rs`), + // pinned on the host by `tests/host_kat/p1w16_host_kat.cpp`. Loaded only by + // that module, never by the backend. + println!("cargo:rerun-if-changed=kernels/p1w16_constants.cuh"); + println!("cargo:rerun-if-changed=kernels/p1w8_constants.cuh"); + compile_kernel("p1w16.cu", "p1w16.cubin", have_nvcc, &[]); + compile_kernel("sumcheck.cu", "sumcheck.cubin", have_nvcc, &[]); + compile_kernel("whir_fold.cu", "whir_fold.cubin", have_nvcc, &[]); + // The first-round cosets evaluated from the polynomial (`src/whir_coset.rs`), + // a measurement module loaded on first use, never by the backend. + compile_kernel("whir_coset.cu", "whir_coset.cubin", have_nvcc, &[]); } diff --git a/crypto/math-cuda/kernels/arith.cu b/crypto/math-cuda/kernels/arith.cu index b1a6bb8ab..8b3d1aea2 100644 --- a/crypto/math-cuda/kernels/arith.cu +++ b/crypto/math-cuda/kernels/arith.cu @@ -95,3 +95,88 @@ extern "C" __global__ void ext3_sub_kernel(const uint64_t *a_int, c_int[tid*3 + 1] = r.b; c_int[tid*3 + 2] = r.c; } + +// Widen a packed main trace (crypto/stark/src/narrow.rs): column c holds `rows` +// little-endian words of widths[c] bytes from byte offsets[c] of `data`; `out` +// is the row-major rows x cols trace of the same words, zero-extended. One +// thread per output word, so the writes coalesce. +extern "C" __global__ void widen_narrow_row_major(const uint8_t *data, + const uint64_t *offsets, + const uint8_t *widths, + uint64_t *out, + uint64_t rows, + uint64_t cols) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= rows * cols) return; + uint64_t r = tid / cols; + uint64_t c = tid - r * cols; + uint32_t w = widths[c]; + const uint8_t *p = data + offsets[c] + r * w; + uint64_t v = 0; + for (uint32_t i = 0; i < w; i++) v |= (uint64_t)p[i] << (8 * i); + out[tid] = v; +} + +// Column maxima of a column-major rows x cols trace (`src[c*rows + r]`), the +// widths of its packed form (crypto/stark/src/narrow.rs): blockIdx.y is the +// column, the x blocks stride its rows, and each block folds its maximum into +// max_out[c]. blockDim.x must be 256. +extern "C" __global__ void column_max_col_major(const uint64_t *src, + uint64_t rows, + unsigned long long *max_out) { + __shared__ unsigned long long s[256]; + const uint64_t *col = src + (uint64_t)blockIdx.y * rows; + unsigned long long m = 0; + for (uint64_t r = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; r < rows; + r += (uint64_t)gridDim.x * blockDim.x) { + unsigned long long v = col[r]; + m = v > m ? v : m; + } + s[threadIdx.x] = m; + __syncthreads(); + for (unsigned int k = blockDim.x / 2; k > 0; k >>= 1) { + if (threadIdx.x < k && s[threadIdx.x + k] > s[threadIdx.x]) s[threadIdx.x] = s[threadIdx.x + k]; + __syncthreads(); + } + if (threadIdx.x == 0) atomicMax(&max_out[blockIdx.y], s[0]); +} + +// Pack a column-major rows x cols trace at per-column widths: column c's word +// r goes to out[offsets[c] + r*widths[c]], little-endian (the layout +// widen_narrow_row_major reads). One thread per word. +extern "C" __global__ void pack_col_major(const uint64_t *src, + uint64_t rows, + uint64_t cols, + const uint64_t *offsets, + const uint8_t *widths, + uint8_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= rows * cols) return; + uint64_t c = tid / rows; + uint64_t r = tid - c * rows; + uint32_t w = widths[c]; + uint64_t v = src[tid]; + uint8_t *p = out + offsets[c] + r * w; + for (uint32_t i = 0; i < w; i++) p[i] = (uint8_t)(v >> (8 * i)); +} + +// Widen a packed trace column-major: the words widen_narrow_row_major reads, +// written as out[c*rows + r] — the layout a table's run of columns has in the +// multilinear prover's resident store. One thread per output word, so the +// writes coalesce and a warp reads consecutive cells of one column. +extern "C" __global__ void widen_narrow_col_major(const uint8_t *data, + const uint64_t *offsets, + const uint8_t *widths, + uint64_t *out, + uint64_t rows, + uint64_t cols) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= rows * cols) return; + uint64_t c = tid / rows; + uint64_t r = tid - c * rows; + uint32_t w = widths[c]; + const uint8_t *p = data + offsets[c] + r * w; + uint64_t v = 0; + for (uint32_t i = 0; i < w; i++) v |= (uint64_t)p[i] << (8 * i); + out[tid] = v; +} diff --git a/crypto/math-cuda/kernels/blake3.cu b/crypto/math-cuda/kernels/blake3.cu new file mode 100644 index 000000000..680a42afe --- /dev/null +++ b/crypto/math-cuda/kernels/blake3.cu @@ -0,0 +1,789 @@ +// BLAKE3 compression on device, round-count parameterized, plus the Merkle +// parent compressors and the field-element byte serialization the leaf kernels +// share with the CPU commit path. +// +// THE PARITY REFERENCE is the host `blake3_compress_rounds(h, m, t, block_len, +// flags, rounds)` in `prover/src/lfm/blake3.rs:125` — one function whose ONLY +// parameter is the round count. `blake3_compress` below is a +// transcription of it and must agree bit-for-bit at both 6 and 7 rounds; at 7 +// rounds both are standard BLAKE3, so the `blake3` crate anchors the pair from +// outside this tree. (P-a Stage 1 moves the host reference down into +// `crypto/crypto`; nothing here changes when it does.) +// +// ROUND COUNT is a compile-time knob with the same polarity as the host's +// `BLAKE3_ROUNDS`: 7 (standard BLAKE3) by default, 6 when build.rs passes +// `-DBLAKE3_ROUNDS=6` under math-cuda's `blake3-6round` feature. The two knobs +// are separate crates' features and nothing forces them equal — see +// `blake3_rounds_probe`, which exports this cubin's round count so a caller can +// assert the match rather than discover it as a wrong commitment. +// +// THE CHAINING CONSTRUCTION is `Blake3Chain`, specified in PA-PLAN §1.7 and +// implemented on host at `crypto/crypto/src/hash/blake3/chain.rs`: standard +// BLAKE3 restricted to a single chunk that never ends. `t = 0` on every block, +// CHUNK_START on the first, CHUNK_END|ROOT and the true byte count as +// `block_len` on the last, digest = the low 8 output words little-endian. The +// device `Blake3Chain` below is a transcription of that host type, and every +// leaf kernel here streams its message through one. +// +// ⚠ The construction is a DRAFT pending ratification of forks F1-F3 +// (PA-PLAN §1.7.3): `t = 0` throughout rather than a block counter, the +// three-flag schedule rather than one constant, and no leaf/parent domain +// separation. It is implemented as the working default by standing decision. If +// a fork is ratified the other way, the change lands in `Blake3Chain::finalize` +// and `compress_pending` — the leaf kernels themselves do not move. + +#include +#include "goldilocks.cuh" + +// 7 = standard BLAKE3. Overridden to 6 by build.rs; see the header comment. +#ifndef BLAKE3_ROUNDS +#define BLAKE3_ROUNDS 7 +#endif + +// The BLAKE3 IV (= SHA-256's initial state). Mirror of `BLAKE3_IV` +// (`blake3.rs:46`). `IV[0..4]` also seeds `v[8..12]` of the working state. +__device__ __constant__ uint32_t BLAKE3_IV[8] = { + 0x6A09E667u, 0xBB67AE85u, 0x3C6EF372u, 0xA54FF53Au, + 0x510E527Fu, 0x9B05688Cu, 0x1F83D9ABu, 0x5BE0CD19u, +}; + +// The three BLAKE3 domain flags this construction uses. Mirrors of `CHUNK_START` +// / `CHUNK_END` / `ROOT` in `crypto/crypto/src/hash/blake3/chain.rs:53-58`. +// `Blake3Chain` sets CHUNK_START on the first block only and CHUNK_END|ROOT on +// the last only; interior blocks carry no flags at all. +#define BLAKE3_FLAG_CHUNK_START 1u +#define BLAKE3_FLAG_CHUNK_END 2u +#define BLAKE3_FLAG_ROOT 8u + +// CHUNK_START | CHUNK_END | ROOT: the flags of a BLAKE3 hash whose whole message +// is one block of one chunk. At 7 rounds a compression under these flags with +// `h = IV` and `t = 0` IS `blake3::hash(message)`, which is what makes the crate +// an anchor for the framing and not just for the round function. Same framing +// the live LFM socket uses (`blake3_socket.rs:258` `FLAGS_LFMC = 0x0B`). +#define BLAKE3_FLAGS_ONE_BLOCK \ + (BLAKE3_FLAG_CHUNK_START | BLAKE3_FLAG_CHUNK_END | BLAKE3_FLAG_ROOT) + +// A Merkle parent's message is two 32-byte child digests = exactly 64 bytes. +#define BLAKE3_PARENT_BLOCK_LEN 64u + +__device__ __forceinline__ uint32_t rotr32(uint32_t x, uint32_t n) { + // Every call site passes 16, 12, 8 or 7, so the 32-n shift is never a + // shift-by-32. Kept as an explicit expression rather than __funnelshift_r + // so the transcription against the host `rotate_right` is readable. + return (x >> n) | (x << (32 - n)); +} + +// Reverse the four bytes of a 32-bit word. nvcc lowers this to a single PRMT. +__device__ __forceinline__ uint32_t bswap32(uint32_t x) { + return (x >> 24) | ((x >> 8) & 0x0000FF00u) | ((x << 8) & 0x00FF0000u) | (x << 24); +} + +// The BLAKE3 quarter-round G (spec §2.1). Mirror of `blake3_g` +// (`blake3.rs:89`); uint32_t arithmetic wraps, matching `wrapping_add`. +__device__ __forceinline__ void blake3_g(uint32_t *v, int a, int b, int c, int d, + uint32_t mx, uint32_t my) { + v[a] = v[a] + v[b] + mx; + v[d] = rotr32(v[d] ^ v[a], 16); + v[c] = v[c] + v[d]; + v[b] = rotr32(v[b] ^ v[c], 12); + v[a] = v[a] + v[b] + my; + v[d] = rotr32(v[d] ^ v[a], 8); + v[c] = v[c] + v[d]; + v[b] = rotr32(v[b] ^ v[c], 7); +} + +// The message-schedule permutation `m'[i] = m[PERM[i]]`, written out. The +// indices are `BLAKE3_MSG_PERMUTATION` (`blake3.rs:52`): +// [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] +// Spelled as literals rather than read from a constant array so it stays a +// register shuffle after unrolling; the parity tests are what check the +// transcription. +__device__ __forceinline__ void blake3_permute(uint32_t *m) { + uint32_t p[16] = {m[2], m[6], m[3], m[10], m[7], m[0], m[4], m[13], + m[1], m[11], m[12], m[5], m[9], m[14], m[15], m[8]}; + #pragma unroll + for (int i = 0; i < 16; ++i) m[i] = p[i]; +} + +// The BLAKE3 compression function `f` at `ROUNDS` rounds, full 16-word output. +// +// State init: `v[0..8] = h`, `v[8..12] = IV[0..4]`, `v[12] = t as u32`, +// `v[13] = (t >> 32) as u32`, `v[14] = block_len`, `v[15] = flags`. Each round +// is 8 G-calls (4 columns then 4 diagonals); the schedule is permuted between +// rounds only (`r < ROUNDS - 1` — the trailing permute is never consumed). +// Feed-forward: `out[i] = v[i] ^ v[i+8]`, `out[i+8] = v[i+8] ^ h[i]`; the +// truncated chaining value is `out[0..8]`. +// +// Callers that need only the chaining value still get the full 16 words: the +// second half is 8 XORs the compiler drops when they are unused, and one +// function is one place to be wrong. +template +__device__ __forceinline__ void blake3_compress(const uint32_t *h, const uint32_t *m_in, + uint64_t t, uint32_t block_len, uint32_t flags, + uint32_t *out) { + uint32_t v[16] = { + h[0], h[1], h[2], h[3], + h[4], h[5], h[6], h[7], + BLAKE3_IV[0], BLAKE3_IV[1], BLAKE3_IV[2], BLAKE3_IV[3], + (uint32_t)t, (uint32_t)(t >> 32), block_len, flags, + }; + + uint32_t m[16]; + #pragma unroll + for (int i = 0; i < 16; ++i) m[i] = m_in[i]; + + #pragma unroll + for (int r = 0; r < ROUNDS; ++r) { + // Mix the columns. + blake3_g(v, 0, 4, 8, 12, m[0], m[1]); + blake3_g(v, 1, 5, 9, 13, m[2], m[3]); + blake3_g(v, 2, 6, 10, 14, m[4], m[5]); + blake3_g(v, 3, 7, 11, 15, m[6], m[7]); + // Mix the diagonals. + blake3_g(v, 0, 5, 10, 15, m[8], m[9]); + blake3_g(v, 1, 6, 11, 12, m[10], m[11]); + blake3_g(v, 2, 7, 8, 13, m[12], m[13]); + blake3_g(v, 3, 4, 9, 14, m[14], m[15]); + if (r < ROUNDS - 1) blake3_permute(m); + } + + #pragma unroll + for (int i = 0; i < 8; ++i) { + out[i] = v[i] ^ v[i + 8]; + out[i + 8] = v[i + 8] ^ h[i]; + } +} + +// --------------------------------------------------------------------------- +// Byte serialization — shared with the leaf kernels and with the CPU commit. +// +// The leaf byte encoding does NOT move under P-a: `leaves_bit_reversed_grouped` +// (`crypto/stark/src/commitment.rs:55`) serializes each field element in +// canonical BIG-endian form and concatenates. BLAKE3 reads a 64-byte block as +// 16 LITTLE-endian u32 words, so one 8-byte element is two words: the +// byte-reverse of its high half, then of its low half. That transposition is +// the whole of the serialization difference from keccak, which absorbs the same +// bytes as one byte-swapped u64 lane. +// --------------------------------------------------------------------------- + +// The two BLAKE3 message words covered by one Goldilocks element's canonical +// big-endian bytes. `raw` may be non-canonical; canonicalising here matches +// `canonical_u64().to_be_bytes()` on host. +__device__ __forceinline__ void blake3_words_of_felt(uint64_t raw, uint32_t &w0, uint32_t &w1) { + uint64_t canon = goldilocks::canonical(raw); + w0 = bswap32((uint32_t)(canon >> 32)); + w1 = bswap32((uint32_t)canon); +} + +// A 64-byte BLAKE3 message block under construction. +// +// The SINK is deliberately the caller's: a leaf kernel compresses each full +// block into a chaining value, and which chaining construction that is (bare +// cv-chain vs standard chunk tree, PA-PLAN §1.6) is still open. Everything this +// struct does — word packing, block boundaries, zero-padding the tail, the byte +// count the final `block_len` comes from — is the same under either. +// +// Usage: `push_word` returns true when the block just filled, at which point the +// caller consumes `m` and calls `reset()`; pushing into a full block is the one +// way to misuse it. Field elements go in two words at a time (via +// `blake3_words_of_felt`) and straddle a block boundary whenever the element +// count is not a multiple of 8 — ext3 elements, at three felts, straddle +// routinely — which is why this works at word granularity and not element +// granularity. +struct Blake3Block { + uint32_t m[16]; + uint32_t nwords; // words filled in the current block, 0..15 between pushes + + __device__ __forceinline__ void init() { + nwords = 0; + #pragma unroll + for (int i = 0; i < 16; ++i) m[i] = 0; + } + + __device__ __forceinline__ bool push_word(uint32_t w) { + m[nwords++] = w; + return nwords == 16; + } + + __device__ __forceinline__ void reset() { init(); } + + // Bytes occupied in the pending (partial) block — the `block_len` a final + // compression over it takes. Zero exactly when no partial block is pending. + __device__ __forceinline__ uint32_t pending_bytes() const { return nwords * 4u; } +}; + +// --------------------------------------------------------------------------- +// `Blake3Chain` — the byte hash every leaf kernel commits with. +// +// Transcription of the host `Blake3Chain` (`crypto/crypto/src/hash/blake3/ +// chain.rs:98`), and the two are checked against each other by the leaf parity +// tests at the build's round count. The construction is PA-PLAN §1.7: +// +// n = max(1, ceil(|M| / 64)) blocks; the empty message is ONE +// L = |M| - 64*(n-1) 0 when |M| = 0, else 1..=64 +// F_i = (CHUNK_START if i = 0) | (CHUNK_END|ROOT if i = n-1) +// cv_0 = IV +// cv_i+1 = compress(cv_i, m_i, 0, 64, F_i)[0..8] for i < n-1 +// digest = compress(cv_n-1, m_n-1, 0, L, F_n-1)[0..8] little-endian +// +// ★ THE ONE SUBTLETY, and the reason this is a state machine rather than a +// loop: a FULL block is *held*, not compressed. The last block's flags and +// `block_len` differ from every other block's, and whether a block is the last +// is not known until the message ends — so a block is only folded into the +// chaining value once a further word proves it was not the last. `push_word` +// therefore compresses on the *next* push, never on filling. This mirrors the +// host `update`, which tests `block_len == BLOCK_LEN` at the top of the loop +// body and so only compresses when there is more input (`chain.rs:186-195`). +// +// Compressing eagerly on fill is the bug this shape exists to prevent: it would +// hash a 64-byte message as two blocks (one flagged CHUNK_START, one empty +// final) instead of one, breaking P2 — the property that a 64-byte message is +// exactly a Merkle parent — and with it the `StarkHash` two-element invariant. +// +// Word granularity, not byte: every message these kernels hash is a whole +// number of 8-byte field elements, so `block_len` is always a multiple of 4 and +// a partial word can never occur. `Blake3Block` is reused for the pending block +// so that the framing (packing, boundaries, zero-padding, the byte count) has +// exactly one implementation shared with `blake3_blocks_of_felts_probe`. +// --------------------------------------------------------------------------- +struct Blake3Chain { + uint32_t cv[8]; + Blake3Block block; + // Whether any block has been compressed — i.e. whether the pending block + // still carries CHUNK_START. Host counterpart: `started` (`chain.rs:109`). + bool started; + + __device__ __forceinline__ void init() { + #pragma unroll + for (int i = 0; i < 8; ++i) cv[i] = BLAKE3_IV[i]; + block.init(); + started = false; + } + + // The pending block's flags. CHUNK_START while nothing has been compressed + // yet; CHUNK_END|ROOT when this is the message's last block. Mirror of the + // host `flags(is_final)` (`chain.rs:160`). + __device__ __forceinline__ uint32_t flags(bool is_final) const { + uint32_t start = started ? 0u : BLAKE3_FLAG_CHUNK_START; + uint32_t end = is_final ? (BLAKE3_FLAG_CHUNK_END | BLAKE3_FLAG_ROOT) : 0u; + return start | end; + } + + // Fold the pending block — known NOT to be the last — into the chaining + // value, and clear it so the next block starts zero-padded. + __device__ __forceinline__ void compress_pending() { + uint32_t out[16]; + blake3_compress(cv, block.m, 0, 64u, flags(false), out); + #pragma unroll + for (int i = 0; i < 8; ++i) cv[i] = out[i]; + block.reset(); + started = true; + } + + // Absorb one message word. The full-block test comes FIRST: reaching here + // with a full block is what proves that block was not the last. + __device__ __forceinline__ void push_word(uint32_t w) { + if (block.nwords == 16) compress_pending(); + block.push_word(w); + } + + // Absorb one Goldilocks field element as its two message words — the + // canonical big-endian element bytes read back as little-endian words. + __device__ __forceinline__ void push_felt(uint64_t raw) { + uint32_t w0, w1; + blake3_words_of_felt(raw, w0, w1); + push_word(w0); + push_word(w1); + } + + // The 32-byte digest: one final compression over the pending block with the + // true byte count as `block_len` and CHUNK_END|ROOT set. The empty message + // takes this path with an all-zero block and `block_len = 0`, which is ONE + // compression, not zero. + // + // `dst` is 32-byte aligned at every call site (node buffers come from + // cuMemAlloc, 256-byte aligned, and every leaf sits at a multiple of 32), so + // the u32 store is safe. A digest's 32 bytes ARE its 8 output words + // little-endian and the device is little-endian, so this is a plain copy + // with no byte swapping — contrast the leaf INPUT path, whose field bytes + // are big-endian. + __device__ __forceinline__ void finalize(uint8_t *dst) { + uint32_t out[16]; + blake3_compress(cv, block.m, 0, block.pending_bytes(), flags(true), out); + uint32_t *w = reinterpret_cast(dst); + #pragma unroll + for (int i = 0; i < 8; ++i) w[i] = out[i]; + } +}; + +// --------------------------------------------------------------------------- +// Leaf kernels. +// +// Twins of the seven keccak leaf kernels, one for one, with the sponge replaced +// by a `Blake3Chain` and the lane byte-swap replaced by the two-word field +// serialization. THE READ PATTERN IS IDENTICAL IN EVERY CASE — same bit +// reversal, same column/component order, same row-pair ordering — because the +// leaf byte layout does not move under P-a: `leaves_bit_reversed_grouped` +// (`crypto/stark/src/commitment.rs:55`) serializes each element in canonical +// big-endian and concatenates, and only the hash over those bytes changes. +// +// So the correctness argument for each kernel below is two independent halves: +// the byte stream (identical to the keccak twin's, and checked against the CPU +// leaf helpers by the parity tests) and the hash over it (`Blake3Chain`, checked +// against the host chain by the same tests). +// --------------------------------------------------------------------------- + +// Goldilocks BASE-FIELD leaf hashing, one leaf per bit-reversed row. +// Twin of `keccak256_leaves_base_batched` (`keccak.cu:152`). +extern "C" __global__ void blake3_leaves_base_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + + // Read columns at the bit-reversed row, write the leaf at `tid` — matching + // the CPU per-row `commit_bit_reversed(.., 1)`. + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < num_cols; ++c) { + h.push_felt(columns_base_ptr[c * col_stride + br]); + } + h.finalize(hashed_leaves_out + tid * 32); +} + +// Goldilocks BASE-FIELD row-pair leaf hashing: leaf `tid` hashes bit-reversed +// rows `2*tid` and `2*tid+1`, each written column-by-column, first row then +// second. `num_leaves = num_rows / 2`. +// Twin of `keccak256_leaves_base_row_pair_batched` (`keccak.cu:196`). +extern "C" __global__ void blake3_leaves_base_row_pair_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < num_cols; ++c) { + h.push_felt(columns_base_ptr[c * col_stride + br_0]); + } + for (uint64_t c = 0; c < num_cols; ++c) { + h.push_felt(columns_base_ptr[c * col_stride + br_1]); + } + h.finalize(hashed_leaves_out + tid * 32); +} + +// Goldilocks EXT3 leaf hashing, one leaf per bit-reversed row. Components live +// in three separate base-field slabs: column `c` component `k` is at +// `columns_base_ptr[(c*3 + k)*col_stride + br]`, and per-element bytes are +// `[comp0, comp1, comp2]` each 8 big-endian bytes (matching +// `FieldElement::::write_bytes_be`). +// Twin of `keccak256_leaves_ext3_batched` (`keccak.cu:237`). +extern "C" __global__ void blake3_leaves_ext3_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, // number of ext3 columns (NOT slabs) + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < num_cols; ++c) { + #pragma unroll + for (int k = 0; k < 3; ++k) { + h.push_felt(columns_base_ptr[(c * 3 + (uint64_t)k) * col_stride + br]); + } + } + h.finalize(hashed_leaves_out + tid * 32); +} + +// R2 composition-polynomial leaf hashing: each leaf hashes `2 * num_parts` ext3 +// values from bit-reversed rows `2*tid` and `2*tid+1`, in (row 0: parts) then +// (row 1: parts) order, three base components per value. +// Twin of `keccak_comp_poly_leaves_ext3` (`keccak.cu:277`). +extern "C" __global__ void blake3_comp_poly_leaves_ext3( + const uint64_t *parts_base_ptr, + uint64_t col_stride, + uint64_t num_parts, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t p = 0; p < num_parts; ++p) { + #pragma unroll + for (int k = 0; k < 3; ++k) { + h.push_felt(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_0]); + } + } + for (uint64_t p = 0; p < num_parts; ++p) { + #pragma unroll + for (int k = 0; k < 3; ++k) { + h.push_felt(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_1]); + } + } + h.finalize(leaves_out + tid * 32); +} + +// FRI layer leaf hashing: each leaf hashes two consecutive ext3 values from an +// interleaved eval vector `[a0,a1,a2,b0,b1,b2,...]` = 48 bytes. No bit reversal +// and no slab layout. +// +// Note 48 bytes is under one block, so a FRI leaf is a SINGLE compression with +// `flags = 0x0B` and `block_len = 48` — the chain's degenerate one-block case, +// same shape as a Merkle parent but at a different length. +// Twin of `keccak_fri_leaves_ext3` (`keccak.cu:326`). +extern "C" __global__ void blake3_fri_leaves_ext3( + const uint64_t *evals_interleaved, // 3 * num_evals u64s (ext3 interleaved) + uint64_t num_leaves, // = num_evals / 2 + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + const uint64_t *left = evals_interleaved + 2 * tid * 3; // 3 u64s + const uint64_t *right = left + 3; + + Blake3Chain h; + h.init(); + #pragma unroll + for (int i = 0; i < 3; ++i) h.push_felt(left[i]); + #pragma unroll + for (int i = 0; i < 3; ++i) h.push_felt(right[i]); + h.finalize(leaves_out + tid * 32); +} + +// FRI GROUP-leaf hashing (S3): leaf `tid` hashes the `group` consecutive ext3 +// values `evals[tid*group .. (tid+1)*group]` of an interleaved eval vector (the +// `3*group` contiguous u64s at `evals_interleaved + tid*group*3`), each as its +// canonical big-endian components. The host `Batched` leaf over the group; at +// `group = 2` exactly `blake3_fri_leaves_ext3`'s message. A group of 2^d values +// is 24*2^d bytes, several blocks from d = 2 on — the chain handles any length. +// Twin of `keccak_fri_group_leaves_ext3`. +extern "C" __global__ void blake3_fri_group_leaves_ext3( + const uint64_t *evals_interleaved, // 3 * num_leaves * group u64s + uint64_t num_leaves, + uint64_t group, // ext3 values per leaf (2^d) + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + const uint64_t *g = evals_interleaved + tid * group * 3; + + Blake3Chain h; + h.init(); + for (uint64_t i = 0; i < 3 * group; ++i) h.push_felt(g[i]); + h.finalize(leaves_out + tid * 32); +} + +// Row-major ROW-PAIR leaf hashing: the row-major analog of +// `blake3_leaves_base_row_pair_batched`. Leaf `tid` hashes row +// `reverse_index(2*tid)` then row `reverse_index(2*tid+1)`, each `m` lanes read +// contiguously from `data + br * m`. `m` is the row stride in u64s: base trace = +// column count, ext3 trace = 3 * column count (an ext3 element's components are +// consecutive, matching `write_bytes_be`). +// Twin of `keccak256_leaves_base_row_major_row_pair` (`keccak.cu:473`). +extern "C" __global__ void blake3_leaves_base_row_major_row_pair( + const uint64_t *data, + uint64_t m, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) +{ + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < m; ++c) h.push_felt(row_0[c]); + for (uint64_t c = 0; c < m; ++c) h.push_felt(row_1[c]); + h.finalize(hashed_leaves_out + tid * 32); +} + +// Column-range variant: each leaf hashes only columns `[col_start, col_end)` of +// the two bit-reversed rows, while `m` stays the full row stride. Byte layout +// equals the CPU `commit_rows_bit_reversed_subset` — used for preprocessed +// tables, whose precomputed and multiplicity column ranges commit to separate +// Merkle trees over the same row-major LDE. +// Twin of `keccak256_leaves_base_row_major_row_pair_range` (`keccak.cu:511`). +extern "C" __global__ void blake3_leaves_base_row_major_row_pair_range( + const uint64_t *data, + uint64_t m, + uint64_t col_start, + uint64_t col_end, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) +{ + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + Blake3Chain h; + h.init(); + for (uint64_t c = col_start; c < col_end; ++c) h.push_felt(row_0[c]); + for (uint64_t c = col_start; c < col_end; ++c) h.push_felt(row_1[c]); + h.finalize(hashed_leaves_out + tid * 32); +} + +// Row-major ONE-ROW leaf hashing (S2, rows_per_leaf = 1): leaf `tid` hashes the +// single row `reverse_index(tid)`, columns `[col_start, col_end)` of the +// row-major buffer (`m` the full row stride). Byte stream = the CPU +// `commit_rows_bit_reversed_subset_with(.., 1)`. Twin of +// `keccak256_leaves_base_row_major_row_range`. +extern "C" __global__ void blake3_leaves_base_row_major_row_range( + const uint64_t *data, + uint64_t m, + uint64_t col_start, + uint64_t col_end, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) +{ + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + const uint64_t *row = data + br * m; + + Blake3Chain h; + h.init(); + for (uint64_t c = col_start; c < col_end; ++c) h.push_felt(row[c]); + h.finalize(hashed_leaves_out + tid * 32); +} + +// --------------------------------------------------------------------------- +// Merkle parent / level compressors. +// +// A parent is ONE compression over the 64 bytes of its two child digests: +// `h = IV`, `t = 0`, `block_len = 64`, `flags = CHUNK_START|CHUNK_END|ROOT`, +// digest = `out[0..8]` little-endian. That is `hash_bytes(left ‖ right)`, which +// is what `hash_new_parent` is on host for every existing backend +// (`hash_new_parent_bytes`, `field_element_vector.rs:74`) — so a parent needs no +// chaining and is construction-independent: with a single-block message the +// chunk tree and a bare cv-chain agree bit-for-bit. +// +// The u32 casts are byte-order-free in both directions and that is not an +// accident: a digest's 32 bytes ARE its 8 output words little-endian, and BLAKE3 +// reads message bytes as little-endian words, so on a little-endian device +// (every NVIDIA GPU) reading a child digest as `uint32_t[8]` yields exactly the +// message words, and storing `out[0..8]` as u32 yields exactly the digest bytes. +// No byte swapping anywhere on this path — contrast the leaf path above, whose +// input is big-endian field bytes. +// +// Node buffer layout mirrors `keccak.cu`'s and the CPU +// `crypto/crypto/src/merkle_tree/merkle.rs`: children at +// `nodes[parent_begin + n_pairs .. parent_begin + 3*n_pairs]`, parents at +// `nodes[parent_begin .. parent_begin + n_pairs]`, 32 bytes per node. +// --------------------------------------------------------------------------- +__device__ __forceinline__ void blake3_hash_merkle_parent(uint8_t *nodes, uint64_t parent_begin, + uint64_t n_pairs, uint64_t tid) { + // `nodes` comes from cuMemAlloc (256-byte aligned) and every 32-byte node + // sits at a 32-byte-aligned offset, so the u32 casts are safe. + const uint32_t *left = reinterpret_cast( + nodes + (parent_begin + n_pairs + 2 * tid) * 32); + const uint32_t *right = reinterpret_cast( + nodes + (parent_begin + n_pairs + 2 * tid + 1) * 32); + + uint32_t m[16]; + #pragma unroll + for (int i = 0; i < 8; ++i) { + m[i] = left[i]; + m[i + 8] = right[i]; + } + + uint32_t out[16]; + blake3_compress(BLAKE3_IV, m, 0, BLAKE3_PARENT_BLOCK_LEN, + BLAKE3_FLAGS_ONE_BLOCK, out); + + uint32_t *dst = reinterpret_cast(nodes + (parent_begin + tid) * 32); + #pragma unroll + for (int i = 0; i < 8; ++i) dst[i] = out[i]; +} + +// One level of the inner Merkle tree: each thread hashes one child pair. +extern "C" __global__ void blake3_merkle_level(uint8_t *nodes, + uint64_t parent_begin, // in 32-byte nodes + uint64_t n_pairs) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n_pairs) return; + blake3_hash_merkle_parent(nodes, parent_begin, n_pairs, tid); +} + +// Build every remaining level (from `level_begin` up to the root) in ONE +// single-block launch: each level's pairs are grid-strided over the block, with +// a __syncthreads() barrier between levels. Replaces log2 launches of +// `blake3_merkle_level` for the small top levels, whose per-level work is +// dwarfed by launch overhead. Twin of `keccak_merkle_tail`. +extern "C" __global__ void blake3_merkle_tail(uint8_t *nodes, uint64_t level_begin) { + uint64_t lb = level_begin; + while (lb != 0) { + uint64_t nb = lb / 2; + uint64_t n_pairs = lb - nb; + for (uint64_t tid = threadIdx.x; tid < n_pairs; tid += blockDim.x) { + blake3_hash_merkle_parent(nodes, nb, n_pairs, tid); + } + __syncthreads(); + lb = nb; + } +} + +// --------------------------------------------------------------------------- +// Parity-harness entry points. +// +// The device compression function is not otherwise reachable from host code, so +// there would be nothing to check it against the host reference with. These are +// that oracle — the same role `build_fri_layer_tree_from_evals_ext3` plays for +// the keccak tree. Not on any production path. +// --------------------------------------------------------------------------- + +// `n` independent compressions, full 16-word outputs. One thread per vector. +template +__device__ __forceinline__ void compress_probe_body(const uint32_t *h, const uint32_t *m, + const uint64_t *t, const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + blake3_compress(h + tid * 8, m + tid * 16, t[tid], block_len[tid], flags[tid], + out + tid * 16); +} + +extern "C" __global__ void blake3_compress_probe_6r(const uint32_t *h, const uint32_t *m, + const uint64_t *t, const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + compress_probe_body<6>(h, m, t, block_len, flags, n, out); +} + +extern "C" __global__ void blake3_compress_probe_7r(const uint32_t *h, const uint32_t *m, + const uint64_t *t, const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + compress_probe_body<7>(h, m, t, block_len, flags, n, out); +} + +// The same probe at the round count this cubin's PRODUCTION kernels are built +// for. Not redundant with the two above: it is the only way to observe from host +// code which of them `blake3_merkle_level` actually uses. +extern "C" __global__ void blake3_compress_probe_default(const uint32_t *h, const uint32_t *m, + const uint64_t *t, + const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + compress_probe_body(h, m, t, block_len, flags, n, out); +} + +// `n_words` message words streamed through the device `Blake3Chain`, digest out. +// +// ★ This is the harness that lets the device be checked against the COMMITTED +// KAT TABLE (`CHAIN_KAT_6ROUND`, `chain.rs:304`) rather than only against the +// host implementation. That distinction is the whole of risk R13: a device port +// checked solely against the Rust it was transcribed from is checked against +// nothing. The KAT digests came from a Python oracle, so asserting the device +// against them closes the loop with an artifact this tree did not produce. +// +// Word-granular, because that is all the device ever hashes: every production +// message is a whole number of 8-byte field elements. The KAT lengths that are +// not multiples of 4 are therefore unreachable from device code by construction, +// and the host tests cover them instead. +// +// Single-threaded on purpose — same shape as a leaf kernel, one thread hashing +// one whole message sequentially. +extern "C" __global__ void blake3_chain_probe(const uint32_t *words, uint64_t n_words, + uint8_t *out32) { + if (threadIdx.x != 0 || blockIdx.x != 0) return; + Blake3Chain h; + h.init(); + for (uint64_t i = 0; i < n_words; ++i) h.push_word(words[i]); + h.finalize(out32); +} + +// This cubin's compiled-in round count, so a caller can assert it against the +// host's `BLAKE3_ROUNDS` instead of discovering a mismatch as a wrong root. +extern "C" __global__ void blake3_rounds_probe(uint32_t *out) { + if (threadIdx.x == 0 && blockIdx.x == 0) *out = (uint32_t)BLAKE3_ROUNDS; +} + +// The two message words of each of `n` field elements, in order — the +// serialization contract on its own (canonicalisation, big-endian element bytes, +// little-endian word packing), with no hashing over it. +extern "C" __global__ void blake3_serialize_felts_probe(const uint64_t *vals, uint64_t n, + uint32_t *out_words) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint32_t w0, w1; + blake3_words_of_felt(vals[tid], w0, w1); + out_words[tid * 2] = w0; + out_words[tid * 2 + 1] = w1; +} + +// `n` field elements streamed through `Blake3Block`, with the completed blocks +// written out instead of compressed. Single-threaded on purpose: that is the +// shape a leaf kernel has (one thread hashes one whole leaf, sequentially), so +// this exercises the block framing on the code path the chaining loop will use. +// Writes `ceil(2n/16)` blocks of 16 words; the tail block is zero-padded. +extern "C" __global__ void blake3_blocks_of_felts_probe(const uint64_t *vals, uint64_t n, + uint32_t *out_blocks) { + if (threadIdx.x != 0 || blockIdx.x != 0) return; + + Blake3Block b; + b.init(); + uint64_t nblocks = 0; + for (uint64_t i = 0; i < n; ++i) { + uint32_t w0, w1; + blake3_words_of_felt(vals[i], w0, w1); + if (b.push_word(w0)) { + #pragma unroll + for (int k = 0; k < 16; ++k) out_blocks[nblocks * 16 + k] = b.m[k]; + ++nblocks; + b.reset(); + } + if (b.push_word(w1)) { + #pragma unroll + for (int k = 0; k < 16; ++k) out_blocks[nblocks * 16 + k] = b.m[k]; + ++nblocks; + b.reset(); + } + } + // Flush the partial tail block, zero-padded (`init` zeroed it, and + // `pending_bytes` is what a real final compression would pass as block_len). + if (b.pending_bytes() != 0) { + #pragma unroll + for (int k = 0; k < 16; ++k) out_blocks[nblocks * 16 + k] = b.m[k]; + } +} diff --git a/crypto/math-cuda/kernels/constraint_interp.cu b/crypto/math-cuda/kernels/constraint_interp.cu index 535a09fb5..288711de1 100644 --- a/crypto/math-cuda/kernels/constraint_interp.cu +++ b/crypto/math-cuda/kernels/constraint_interp.cu @@ -376,6 +376,11 @@ extern "C" __global__ void constraint_interp_kernel( // full mul on its embedding), z_inv is the cyclic base transition-zerofier // inverse, and the boundary term reads the resident trace at column `b_col[b]` // (main or aux). +// +// `row_stride` evaluates H on a sub-coset of the LDE coset: output row `out` +// is LDE row `out * row_stride`, for `num_rows / row_stride` output rows. Every +// read (trace frame, z_inv, boundary z_b_inv) uses the LDE row, so stride 1 is +// the full-coset evaluation unchanged. extern "C" __global__ void constraint_composition_kernel( // output: one H(row) per LDE row Fe3 *__restrict__ d_h, @@ -397,6 +402,7 @@ extern "C" __global__ void constraint_composition_kernel( uint64_t aux_stride, uint64_t next_step, uint64_t num_rows, + uint64_t row_stride, // transition accumulation const Fe3 *__restrict__ d_beta_trans, // [num_roots] const uint64_t *__restrict__ d_z_inv, // [z_len], cyclic @@ -425,7 +431,9 @@ extern "C" __global__ void constraint_composition_kernel( u.alpha = d_alpha_powers; u.offset = *d_table_offset; - for (uint64_t row = task_offset; row < num_rows; row += num_threads) { + uint64_t num_out = num_rows / row_stride; + for (uint64_t out = task_offset; out < num_out; out += num_threads) { + uint64_t row = out * row_stride; eval_program_row(vb, ve, vstride, d_nodes, num_nodes, u, row, next_step, num_rows, d_main, main_stride, d_aux, aux_stride); @@ -463,30 +471,33 @@ extern "C" __global__ void constraint_composition_kernel( h = ext3::add(h, ext3::mul(zb, bp)); } - d_h[row] = h; + d_h[out] = h; } } // ============================================================================ -// Degree-2 quotient decomposition, pointwise on the LDE coset: +// Degree-2 quotient decomposition, pointwise on the coset `h` was evaluated on: // H0[i] = two_inv * (h[i] + h[i+n]) -// H1[i] = inv_2x[i] * (h[i] - h[i+n]) +// H1[i] = inv_2x[i * inv_stride] * (h[i] - h[i+n]) // Reads the interleaved ext3 composition evals `h` (2n rows); writes the two // halves in slab layout (3 base slabs per half, `slab_stride` u64 each; rows -// n.. stay zero as the LDE zero-pad). +// n.. stay zero as the LDE zero-pad). `inv_stride` is the sub-coset stride `h` +// was evaluated at: `inv_2x` holds 1/(2x) for the full LDE coset's first half, +// so the sub-coset's point i is its entry i * inv_stride (1 on the full coset). extern "C" __global__ void decompose_d2_ext3( const uint64_t *__restrict__ h, const uint64_t *__restrict__ inv_2x, uint64_t two_inv, uint64_t n, uint64_t slab_stride, + uint64_t inv_stride, uint64_t *__restrict__ out) { for (uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (uint64_t)gridDim.x * blockDim.x) { Fe3 x = ext3::make(h[i * 3], h[i * 3 + 1], h[i * 3 + 2]); Fe3 y = ext3::make(h[(i + n) * 3], h[(i + n) * 3 + 1], h[(i + n) * 3 + 2]); Fe3 h0 = ext3::mul_base(ext3::add(x, y), two_inv); - Fe3 h1 = ext3::mul_base(ext3::sub(x, y), inv_2x[i]); + Fe3 h1 = ext3::mul_base(ext3::sub(x, y), inv_2x[i * inv_stride]); out[0 * slab_stride + i] = h0.a; out[1 * slab_stride + i] = h0.b; out[2 * slab_stride + i] = h0.c; diff --git a/crypto/math-cuda/kernels/deep.cu b/crypto/math-cuda/kernels/deep.cu index d58c37a2e..9fbafe5cc 100644 --- a/crypto/math-cuda/kernels/deep.cu +++ b/crypto/math-cuda/kernels/deep.cu @@ -130,3 +130,228 @@ extern "C" __global__ void bit_reverse_ext3_interleaved( out[i * 3 + 2] = in[j * 3 + 2]; } } + +// --------------------------------------------------------------------------- +// `deep_composition_ext3_row` with its denominators inverted in +// registers, per row, instead of read from the `inv_h` / `inv_t` buffers the +// six-kernel scan fills. +// +// Row `i` sits at the domain point `coset_points[i]`; its M = 1 + num_eval_points +// denominators are `x - z_scalars[m]` (DenomSign::XMinusZ: m = 0 is the H-term +// pole z^K, m = 1 + k the trace pole z·w^k — the layout the scan path's +// `compute_and_invert_denoms_ext3_dev` is called with). They are inverted +// together with ONE base-field inversion (`ext3_inv.cuh`), and the sum below is +// `deep_composition_ext3_row`'s, term for term. The inverses are the scan's as +// field elements, so the codeword is too. +// +// M is a template parameter so the inverses stay in registers; the host picks +// the instantiation and keeps the buffered kernel for any other M. +// `zero_flag` (nullable) as in `invert_denoms_rowwise_ext3_k*`. +// --------------------------------------------------------------------------- +#include "ext3_inv.cuh" + +template +__device__ __forceinline__ void deep_fused_body( + const uint64_t *__restrict__ main_lde, const uint64_t *__restrict__ aux_lde, + const uint64_t *__restrict__ h_lde, uint64_t lde_stride, uint64_t num_main, uint64_t num_aux, + uint64_t num_parts, uint64_t row_stride, uint64_t domain_size, + const uint64_t *__restrict__ h_ood, const uint64_t *__restrict__ trace_ood, + const uint64_t *__restrict__ gammas_h, const uint64_t *__restrict__ gammas_tr, + const uint64_t *__restrict__ coset_points, const uint64_t *__restrict__ z_scalars, + uint64_t *__restrict__ deep_out, uint32_t *zero_flag) { + constexpr int K = M - 1; // num_eval_points + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= domain_size) return; + uint64_t row = i * row_stride; + + uint64_t x = coset_points[i]; + ext3::Fe3 inv[M]; +#pragma unroll + for (int m = 0; m < M; ++m) inv[m] = ext3_inv::x_minus_z(x, z_scalars + 3 * m); + uint64_t product = ext3_inv::batch_inv(inv); + if (zero_flag != nullptr && goldilocks::canonical(product) == 0) atomicOr(zero_flag, 1u); + + ext3::Fe3 result = ext3::zero(); + + // H-terms + for (uint64_t j = 0; j < num_parts; ++j) { + ext3::Fe3 h_val = { + h_lde[(j * 3 + 0) * lde_stride + row], + h_lde[(j * 3 + 1) * lde_stride + row], + h_lde[(j * 3 + 2) * lde_stride + row], + }; + ext3::Fe3 h_ood_j = {h_ood[j * 3], h_ood[j * 3 + 1], h_ood[j * 3 + 2]}; + ext3::Fe3 num = ext3::sub(h_val, h_ood_j); + ext3::Fe3 gamma = {gammas_h[j * 3], gammas_h[j * 3 + 1], gammas_h[j * 3 + 2]}; + ext3::Fe3 tmp = ext3::mul(gamma, num); + tmp = ext3::mul(tmp, inv[0]); + result = ext3::add(result, tmp); + } + + // Main trace terms: t_val (base) - t_ood (ext3) + for (uint64_t j = 0; j < num_main; ++j) { + uint64_t t_val = main_lde[j * lde_stride + row]; +#pragma unroll + for (int k = 0; k < K; ++k) { + uint64_t idx = (j * K + k) * 3; + ext3::Fe3 t_ood = {trace_ood[idx], trace_ood[idx + 1], trace_ood[idx + 2]}; + ext3::Fe3 num = { + goldilocks::sub(t_val, t_ood.a), + goldilocks::neg(t_ood.b), + goldilocks::neg(t_ood.c), + }; + ext3::Fe3 gamma = {gammas_tr[idx], gammas_tr[idx + 1], gammas_tr[idx + 2]}; + ext3::Fe3 tmp = ext3::mul(gamma, num); + tmp = ext3::mul(tmp, inv[1 + k]); + result = ext3::add(result, tmp); + } + } + + // Aux trace terms: t_val (ext3) - t_ood (ext3) + for (uint64_t j = 0; j < num_aux; ++j) { + ext3::Fe3 t_val = { + aux_lde[(j * 3 + 0) * lde_stride + row], + aux_lde[(j * 3 + 1) * lde_stride + row], + aux_lde[(j * 3 + 2) * lde_stride + row], + }; + uint64_t trace_j = num_main + j; +#pragma unroll + for (int k = 0; k < K; ++k) { + uint64_t idx = (trace_j * K + k) * 3; + ext3::Fe3 t_ood = {trace_ood[idx], trace_ood[idx + 1], trace_ood[idx + 2]}; + ext3::Fe3 num = ext3::sub(t_val, t_ood); + ext3::Fe3 gamma = {gammas_tr[idx], gammas_tr[idx + 1], gammas_tr[idx + 2]}; + ext3::Fe3 tmp = ext3::mul(gamma, num); + tmp = ext3::mul(tmp, inv[1 + k]); + result = ext3::add(result, tmp); + } + } + + uint64_t out_idx = i * 3; + deep_out[out_idx + 0] = result.a; + deep_out[out_idx + 1] = result.b; + deep_out[out_idx + 2] = result.c; +} + +#define DEEP_FUSED(M) \ + extern "C" __global__ void deep_composition_ext3_fused_m##M( \ + const uint64_t *__restrict__ main_lde, const uint64_t *__restrict__ aux_lde, \ + const uint64_t *__restrict__ h_lde, uint64_t lde_stride, uint64_t num_main, \ + uint64_t num_aux, uint64_t num_parts, uint64_t row_stride, uint64_t domain_size, \ + const uint64_t *__restrict__ h_ood, const uint64_t *__restrict__ trace_ood, \ + const uint64_t *__restrict__ gammas_h, const uint64_t *__restrict__ gammas_tr, \ + const uint64_t *__restrict__ coset_points, const uint64_t *__restrict__ z_scalars, \ + uint64_t *__restrict__ deep_out, uint32_t *zero_flag) { \ + deep_fused_body(main_lde, aux_lde, h_lde, lde_stride, num_main, num_aux, num_parts, \ + row_stride, domain_size, h_ood, trace_ood, gammas_h, gammas_tr, \ + coset_points, z_scalars, deep_out, zero_flag); \ + } + +// num_eval_points 0..=3; the tables prove with 2 (transition offsets {0, 1}). +DEEP_FUSED(1) +DEEP_FUSED(2) +DEEP_FUSED(3) +DEEP_FUSED(4) + +// --------------------------------------------------------------------------- +// `deep_composition_ext3_fused_m*` with the row-independent part taken out of +// the row (`LAMBDA_VM_DEEP_HOISTED`; on by default). For each pole k, +// Σ_j γ_jk·(t_j(x) − o_jk) = Σ_j γ_jk·t_j(x) − C_k, C_k = Σ_j γ_jk·o_jk, +// and the H-terms alike (C_h = Σ_j γ_j·h_ood_j): a row costs one ext3×base +// product per (main column, pole), one ext3×ext3 per (aux column, pole) and per +// part, and 1 + K multiplies by the inverses, against two ext3×ext3 per term in +// `deep_fused_body`. `consts` holds C_h, then C_0 … C_{K−1} (ext3 interleaved), +// computed once on the host (`deep::hoisted_constants`). The same field element +// as the fused kernel at `row_stride = 1`, written canonical: the fused kernel's +// words are canonical but for a ≈ 2^-32 chance a word, so the two agree as raw +// words but there (measured on the card: 100 % at four shapes). +// --------------------------------------------------------------------------- +template +__device__ __forceinline__ void deep_hoisted_body( + const uint64_t *__restrict__ main_lde, const uint64_t *__restrict__ aux_lde, + const uint64_t *__restrict__ h_lde, uint64_t lde_stride, uint64_t num_main, uint64_t num_aux, + uint64_t num_parts, uint64_t domain_size, const uint64_t *__restrict__ consts, + const uint64_t *__restrict__ gammas_h, const uint64_t *__restrict__ gammas_tr, + const uint64_t *__restrict__ coset_points, const uint64_t *__restrict__ z_scalars, + uint64_t *__restrict__ deep_out, uint32_t *zero_flag) { + constexpr int K = M - 1; + constexpr int KA = K > 0 ? K : 1; + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= domain_size) return; + + uint64_t x = coset_points[i]; + ext3::Fe3 inv[M]; +#pragma unroll + for (int m = 0; m < M; ++m) inv[m] = ext3_inv::x_minus_z(x, z_scalars + 3 * m); + uint64_t product = ext3_inv::batch_inv(inv); + if (zero_flag != nullptr && goldilocks::canonical(product) == 0) atomicOr(zero_flag, 1u); + + ext3::Fe3 acc_h = ext3::zero(); + for (uint64_t j = 0; j < num_parts; ++j) { + ext3::Fe3 h_val = { + h_lde[(j * 3 + 0) * lde_stride + i], + h_lde[(j * 3 + 1) * lde_stride + i], + h_lde[(j * 3 + 2) * lde_stride + i], + }; + ext3::Fe3 gamma = {gammas_h[j * 3], gammas_h[j * 3 + 1], gammas_h[j * 3 + 2]}; + acc_h = ext3::add(acc_h, ext3::mul(gamma, h_val)); + } + + ext3::Fe3 acc[KA]; +#pragma unroll + for (int k = 0; k < KA; ++k) acc[k] = ext3::zero(); + for (uint64_t j = 0; j < num_main; ++j) { + uint64_t t_val = main_lde[j * lde_stride + i]; +#pragma unroll + for (int k = 0; k < K; ++k) { + uint64_t idx = (j * K + k) * 3; + ext3::Fe3 gamma = {gammas_tr[idx], gammas_tr[idx + 1], gammas_tr[idx + 2]}; + acc[k] = ext3::add(acc[k], ext3::mul_base(gamma, t_val)); + } + } + for (uint64_t j = 0; j < num_aux; ++j) { + ext3::Fe3 t_val = { + aux_lde[(j * 3 + 0) * lde_stride + i], + aux_lde[(j * 3 + 1) * lde_stride + i], + aux_lde[(j * 3 + 2) * lde_stride + i], + }; + uint64_t trace_j = num_main + j; +#pragma unroll + for (int k = 0; k < K; ++k) { + uint64_t idx = (trace_j * K + k) * 3; + ext3::Fe3 gamma = {gammas_tr[idx], gammas_tr[idx + 1], gammas_tr[idx + 2]}; + acc[k] = ext3::add(acc[k], ext3::mul(gamma, t_val)); + } + } + + ext3::Fe3 c_h = {consts[0], consts[1], consts[2]}; + ext3::Fe3 result = ext3::mul(ext3::sub(acc_h, c_h), inv[0]); +#pragma unroll + for (int k = 0; k < K; ++k) { + ext3::Fe3 c_k = {consts[3 + 3 * k], consts[4 + 3 * k], consts[5 + 3 * k]}; + result = ext3::add(result, ext3::mul(ext3::sub(acc[k], c_k), inv[1 + k])); + } + result = ext3::canonical(result); + deep_out[i * 3 + 0] = result.a; + deep_out[i * 3 + 1] = result.b; + deep_out[i * 3 + 2] = result.c; +} + +#define DEEP_HOISTED(M) \ + extern "C" __global__ void deep_composition_ext3_hoisted_m##M( \ + const uint64_t *__restrict__ main_lde, const uint64_t *__restrict__ aux_lde, \ + const uint64_t *__restrict__ h_lde, uint64_t lde_stride, uint64_t num_main, \ + uint64_t num_aux, uint64_t num_parts, uint64_t domain_size, \ + const uint64_t *__restrict__ consts, const uint64_t *__restrict__ gammas_h, \ + const uint64_t *__restrict__ gammas_tr, const uint64_t *__restrict__ coset_points, \ + const uint64_t *__restrict__ z_scalars, uint64_t *__restrict__ deep_out, \ + uint32_t *zero_flag) { \ + deep_hoisted_body(main_lde, aux_lde, h_lde, lde_stride, num_main, num_aux, num_parts, \ + domain_size, consts, gammas_h, gammas_tr, coset_points, z_scalars, \ + deep_out, zero_flag); \ + } + +DEEP_HOISTED(1) +DEEP_HOISTED(2) +DEEP_HOISTED(3) +DEEP_HOISTED(4) diff --git a/crypto/math-cuda/kernels/ext3_inv.cuh b/crypto/math-cuda/kernels/ext3_inv.cuh new file mode 100644 index 000000000..4d99e6a8b --- /dev/null +++ b/crypto/math-cuda/kernels/ext3_inv.cuh @@ -0,0 +1,114 @@ +// Row-local inversion of a handful of ext3 elements with ONE base-field +// inversion. +// +// The DEEP and OOD denominators are `x - z` (or `z - x`): `x` a base-field +// domain point, `z` one ext3 opening point per term. The scan path +// (`inverse.cu`) inverts a whole domain-sized array with prefix and suffix +// products and one Fermat inversion for the batch: six kernels, each a full +// pass over the array. A row only ever needs its own few denominators, so here +// each thread inverts them in registers: +// +// e^{-1} = adj(e) / N(e), N(e) = e · adj(e) in Fp, +// +// for e = a + b·w + c·w^2 in Fp[w]/(w^3 - 2): +// +// adj(e) = (a^2 - 2bc) + (2c^2 - ab)·w + (b^2 - ac)·w^2 +// N(e) = a·adj_0 + 2·(b·adj_2 + c·adj_1) +// +// (the w and w^2 components of e·adj(e) cancel identically), and the row's norms +// are inverted together by Montgomery's trick in the base field. One Fermat +// inversion per row, the rest a few dozen multiplications. +// +// The result is THE inverse, so it equals the scan's as a field element. Its raw +// u64 may differ by p — the device representation is non-canonical everywhere — +// which nothing downstream observes: every consumer is field arithmetic, the +// Merkle leaves absorb raw limbs into a representation-independent permutation, +// and the host canonicalises before it serialises or absorbs. +// +// A zero element (x equal to an opening point, which Fiat-Shamir makes +// negligible) has norm zero, which zeroes the row's product and so every +// inverse in that row. The scan path has the same release behaviour for its +// whole batch; its debug-build guard is mirrored by the kernels' zero flag. + +#pragma once +#include "goldilocks.cuh" +#include "ext3.cuh" + +namespace ext3_inv { + +using ext3::Fe3; + +// x^(p-2), which is x^{-1} for x != 0 and 0 for x == 0 (Fermat). +// +// p - 2 = 2^64 - 2^32 - 1 = (2^31 - 1)·2^33 + (2^32 - 1), built from +// t_k = x^(2^k - 1) by t_{j+k} = t_j^(2^k)·t_k: 64 squarings, 9 products. +__device__ __forceinline__ uint64_t sq_n(uint64_t v, int n) { + for (int i = 0; i < n; ++i) v = goldilocks::mul(v, v); + return v; +} + +__device__ __forceinline__ uint64_t gl_inv(uint64_t x) { + using goldilocks::mul; + uint64_t t2 = mul(mul(x, x), x); // x^(2^2 - 1) + uint64_t t3 = mul(mul(t2, t2), x); // x^(2^3 - 1) + uint64_t t6 = mul(sq_n(t3, 3), t3); // x^(2^6 - 1) + uint64_t t12 = mul(sq_n(t6, 6), t6); // x^(2^12 - 1) + uint64_t t24 = mul(sq_n(t12, 12), t12); // x^(2^24 - 1) + uint64_t t30 = mul(sq_n(t24, 6), t6); // x^(2^30 - 1) + uint64_t t31 = mul(mul(t30, t30), x); // x^(2^31 - 1) + uint64_t t32 = mul(mul(t31, t31), x); // x^(2^32 - 1) + return mul(sq_n(t31, 33), t32); // x^((2^31 - 1)·2^33 + 2^32 - 1) +} + +// The adjugate of `e` and its norm N(e) = e·adj(e) (a base-field value). +__device__ __forceinline__ void adj_norm(const Fe3 &e, Fe3 &adj, uint64_t &norm) { + using goldilocks::add; + using goldilocks::mul; + using goldilocks::sub; + uint64_t bc = mul(e.b, e.c); + uint64_t cc = mul(e.c, e.c); + uint64_t a0 = sub(mul(e.a, e.a), add(bc, bc)); // a^2 - 2bc + uint64_t a1 = sub(add(cc, cc), mul(e.a, e.b)); // 2c^2 - ab + uint64_t a2 = sub(mul(e.b, e.b), mul(e.a, e.c)); // b^2 - ac + uint64_t t = add(mul(e.b, a2), mul(e.c, a1)); + norm = add(mul(e.a, a0), add(t, t)); + adj = ext3::make(a0, a1, a2); +} + +// Invert d[0..M) in place with one base-field inversion. M is a compile-time +// count so every array below stays in registers. Returns the product of the +// norms, which is zero (possibly as the raw value p) exactly when some d[j] was. +template +__device__ __forceinline__ uint64_t batch_inv(Fe3 (&d)[M]) { + Fe3 adj[M]; + uint64_t norm[M]; + uint64_t prefix[M]; +#pragma unroll + for (int j = 0; j < M; ++j) { + adj_norm(d[j], adj[j], norm[j]); + prefix[j] = j == 0 ? norm[0] : goldilocks::mul(prefix[j - 1], norm[j]); + } + // inv = 1 / (norm[0]·…·norm[j]) as j walks down. + uint64_t inv = gl_inv(prefix[M - 1]); +#pragma unroll + for (int j = M - 1; j > 0; --j) { + uint64_t norm_inv = goldilocks::mul(inv, prefix[j - 1]); + inv = goldilocks::mul(inv, norm[j]); + d[j] = ext3::mul_base(adj[j], norm_inv); + } + d[0] = ext3::mul_base(adj[0], inv); + return prefix[M - 1]; +} + +// `x - z` for base `x` and ext3 `z` (DenomSign::XMinusZ), exactly as the scan +// path's `compute_denoms_ext3` builds it. +__device__ __forceinline__ Fe3 x_minus_z(uint64_t x, const uint64_t *z) { + return ext3::make(goldilocks::sub(x, z[0]), goldilocks::neg(z[1]), goldilocks::neg(z[2])); +} + +// `z - x` (DenomSign::ZMinusX), likewise. +__device__ __forceinline__ Fe3 z_minus_x(uint64_t x, const uint64_t *z) { + return ext3::make(goldilocks::sub(z[0], x), z[1], z[2]); +} + +} // namespace ext3_inv diff --git a/crypto/math-cuda/kernels/fill.cu b/crypto/math-cuda/kernels/fill.cu new file mode 100644 index 000000000..47d1b3efe --- /dev/null +++ b/crypto/math-cuda/kernels/fill.cu @@ -0,0 +1,1056 @@ +// A table's columns filled on the card from the records its rows come from, +// instead of filled packed on the host and widened here (`widen_narrow_col_major` +// in arith.cu): MEMW_R from its register rows, CPU from the executor's logs and +// the program's decode table, KECCAK_RND from its keccak operations. +// +// Each kernel writes exactly the words the prover's host fill writes for the +// same rows (`prover/src/tables/{memw_register,cpu,keccak_rnd}.rs`), every one +// canonical as `FE::from(u64)` makes it, column-major (`out[col * height + row]`): +// the layout a table's run of columns has in the multilinear prover's resident +// store, which is the layout the widen writes. A kernel covers every row of the +// table, the padding rows included, so its output needs no zeroing. +// +// The records are the `#[repr(C)]` structs of `src/fill.rs`, field for field; +// the static_asserts below and the const asserts there pin both sides. + +#include +#include + +#include "goldilocks.cuh" +#include "p1w16_constants.cuh" + +// --------------------------------------------------------------------------- +// The records +// --------------------------------------------------------------------------- + +// The executor's log of one cycle (`executor/src/vm/logs.rs`). +struct LogC { + uint64_t current_pc; + uint64_t next_pc; + uint64_t src1_val; + uint64_t src2_val; + uint64_t dst_val; +}; +static_assert(sizeof(LogC) == 40, "LogC is 40 bytes"); +static_assert(offsetof(LogC, dst_val) == 32, "LogC field order"); + +// One MEMW_R row (`memw_register.rs`, `RegRow`): `address` is the register +// index, `old_ts_lo` the old timestamp's low word (its high word is the +// timestamp's), `is_read` 0 or 1, `pad` 0. +struct RegRowC { + uint64_t timestamp; + uint32_t val0; + uint32_t val1; + uint32_t old0; + uint32_t old1; + uint32_t old_ts_lo; + uint16_t address; + uint8_t is_read; + uint8_t pad; +}; +static_assert(sizeof(RegRowC) == 32, "RegRowC is 32 bytes"); +static_assert(offsetof(RegRowC, old_ts_lo) == 24, "RegRowC field order"); +static_assert(offsetof(RegRowC, address) == 28, "RegRowC field order"); +static_assert(offsetof(RegRowC, is_read) == 30, "RegRowC field order"); + +// One DECODE row (`types.rs`, `DecodeEntry`): the pc, the extended immediate, +// then `ShrunkDecode`'s fields as bytes (flags 0 or 1). +struct DecodeEntryC { + uint64_t pc; + uint64_t imm; + uint8_t read_register1; + uint8_t read_register2; + uint8_t write_register; + uint8_t word_instr; + uint8_t alu; + uint8_t add; + uint8_t sub; + uint8_t memory; + uint8_t branch; + uint8_t ecall; + uint8_t rs1; + uint8_t rs2; + uint8_t rd; + uint8_t half_instruction_length; + uint8_t alu_flags; + uint8_t mem_flags; +}; +static_assert(sizeof(DecodeEntryC) == 32, "DecodeEntryC is 32 bytes"); +static_assert(offsetof(DecodeEntryC, read_register1) == 16, "DecodeEntryC field order"); +static_assert(offsetof(DecodeEntryC, mem_flags) == 31, "DecodeEntryC field order"); + +// A run of pcs 4 apart (`decode.rs`, `DecodeTable::runs`): pc `first_pc + 4k` +// is DECODE row `row + k` for `k < len`. The runs are ascending and disjoint. +struct DecodeRunC { + uint64_t first_pc; + uint32_t row; + uint32_t len; +}; +static_assert(sizeof(DecodeRunC) == 16, "DecodeRunC is 16 bytes"); +static_assert(offsetof(DecodeRunC, len) == 12, "DecodeRunC field order"); + +// One keccak permutation call (`keccak_rnd.rs`, `KeccakRoundOperation`): its 24 +// rows are the rounds from `input`. `output` is carried, not read. +struct KeccakOpC { + uint64_t timestamp; + uint64_t input[25]; + uint64_t output[25]; +}; +static_assert(sizeof(KeccakOpC) == 408, "KeccakOpC is 408 bytes"); +static_assert(offsetof(KeccakOpC, output) == 208, "KeccakOpC field order"); + +// The same call as a table held as its records carries it +// (`multilinear::records::KeccakRndOp`): the fields its rows read, no output. +struct KeccakOpInC { + uint64_t timestamp; + uint64_t input[25]; +}; +static_assert(sizeof(KeccakOpInC) == 208, "KeccakOpInC is 208 bytes"); +static_assert(offsetof(KeccakOpInC, input) == 8, "KeccakOpInC field order"); + +// One MEMW_A row (`memw_aligned.rs`, `AlignedRow`): the access's base address, +// timestamp and shared old timestamp, its value and old value packed in a +// word each (eight bytes for a memory access, the two 32-bit halves for a +// register access), its width, and flags 0 or 1. +struct AlignedRowC { + uint64_t base_address; + uint64_t timestamp; + uint64_t old_timestamp; + uint64_t value; + uint64_t old; + uint8_t width; + uint8_t is_read; + uint8_t is_register; +}; +static_assert(sizeof(AlignedRowC) == 48, "AlignedRowC is 48 bytes"); +static_assert(offsetof(AlignedRowC, width) == 40, "AlignedRowC field order"); +static_assert(offsetof(AlignedRowC, is_register) == 42, "AlignedRowC field order"); + +// One LOAD op (`load.rs`, `LoadOperation`): the base address, the timestamp, +// the width, whether it sign-extends (0 or 1) and the result's eight bytes. +struct LoadOpC { + uint64_t base_address; + uint64_t timestamp; + uint8_t width; + uint8_t is_signed; + uint8_t res[8]; +}; +static_assert(sizeof(LoadOpC) == 32, "LoadOpC is 32 bytes"); +static_assert(offsetof(LoadOpC, is_signed) == 17, "LoadOpC field order"); +static_assert(offsetof(LoadOpC, res) == 18, "LoadOpC field order"); + +// One STORE op (`store.rs`, `StoreOperation`): the base address, the +// timestamp, the value, and the width flags (0 or 1; none for one byte). +struct StoreOpC { + uint64_t base_address; + uint64_t timestamp; + uint64_t value; + uint8_t write2; + uint8_t write4; + uint8_t write8; +}; +static_assert(sizeof(StoreOpC) == 32, "StoreOpC is 32 bytes"); +static_assert(offsetof(StoreOpC, write2) == 24, "StoreOpC field order"); + +// One SHIFT op (`shift.rs`, `ShiftOperation`): the full shift amount, the input +// as four 16-bit halves, the amount's low byte, and flags 0 or 1. +struct ShiftOpC { + uint64_t shift_amount; + uint16_t in_halves[4]; + uint8_t shift; + uint8_t direction; + uint8_t is_signed; + uint8_t word_instr; +}; +static_assert(sizeof(ShiftOpC) == 24, "ShiftOpC is 24 bytes"); +static_assert(offsetof(ShiftOpC, in_halves) == 8, "ShiftOpC field order"); +static_assert(offsetof(ShiftOpC, word_instr) == 19, "ShiftOpC field order"); + +// One MEMW op (`memw.rs`, `MemwOperation`): the base address, the timestamp, +// the eight old timestamps, the eight value and old elements (a memory byte +// or a 32-bit register half each), and the flags and width. +struct MemwOpC { + uint64_t base_address; + uint64_t timestamp; + uint64_t old_timestamp[8]; + uint32_t value[8]; + uint32_t old[8]; + uint8_t is_register; + uint8_t width; + uint8_t is_read; +}; +static_assert(sizeof(MemwOpC) == 152, "MemwOpC is 152 bytes"); +static_assert(offsetof(MemwOpC, value) == 80, "MemwOpC field order"); +static_assert(offsetof(MemwOpC, is_register) == 144, "MemwOpC field order"); + +// --------------------------------------------------------------------------- +// The setters (`types.rs`, `VmTable`) +// --------------------------------------------------------------------------- + +namespace fill { + +// One row's cells: column `col` at `out[col * height + row]`. +struct RowWriter { + uint64_t *out; + uint64_t height; + uint64_t row; + + // `set_u64` (and `set_bool` / `set_byte` / `set_half` / `set_word`): one + // canonical word. + __device__ __forceinline__ void word(uint32_t col, uint64_t v) const { + out[(uint64_t)col * height + row] = goldilocks::canonical(v); + } + // `set_dword_wl`: the low and high 32-bit words. + __device__ __forceinline__ void dword_wl(uint32_t col, uint64_t v) const { + word(col, v & 0xFFFFFFFFull); + word(col + 1, v >> 32); + } + // `set_dword_hl`: four 16-bit halves, low first. + __device__ __forceinline__ void dword_hl(uint32_t col, uint64_t v) const { +#pragma unroll + for (uint32_t i = 0; i < 4; ++i) word(col + i, (v >> (16 * i)) & 0xFFFFull); + } + // `set_dword_bl`: eight bytes, low first. + __device__ __forceinline__ void dword_bl(uint32_t col, uint64_t v) const { +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) word(col + i, (v >> (8 * i)) & 0xFFull); + } + // Columns `[first, end)` zero: a padding row's. + __device__ __forceinline__ void zeros(uint32_t first, uint32_t end) const { + for (uint32_t c = first; c < end; ++c) out[(uint64_t)c * height + row] = 0; + } +}; + +} // namespace fill + +// --------------------------------------------------------------------------- +// MEMW_R (`memw_register.rs`, `generate_memw_register_trace_from_rows_as`) +// --------------------------------------------------------------------------- + +namespace memw_r_cols { +constexpr uint32_t ADDRESS = 0; +constexpr uint32_t TIMESTAMP_0 = 1; +constexpr uint32_t VAL_0 = 3; +constexpr uint32_t VAL_1 = 4; +constexpr uint32_t OLD_0 = 5; +constexpr uint32_t OLD_1 = 6; +constexpr uint32_t OLD_TIMESTAMP_LO = 7; +constexpr uint32_t MU_READ = 8; +constexpr uint32_t MU_WRITE = 9; +constexpr uint32_t NUM_COLUMNS = 10; +} // namespace memw_r_cols + +// Rows `[0, n)` from `rows`, rows `[n, height)` zero. One thread a row. +extern "C" __global__ void fill_memw_r(const RegRowC *rows, uint64_t n, uint64_t height, + uint64_t *out) { + namespace c = memw_r_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) return; + const fill::RowWriter w{out, height, row}; + if (row >= n) { + w.zeros(0, c::NUM_COLUMNS); + return; + } + const RegRowC r = rows[row]; + w.word(c::ADDRESS, r.address); + w.dword_wl(c::TIMESTAMP_0, r.timestamp); + w.word(c::VAL_0, r.val0); + w.word(c::VAL_1, r.val1); + w.word(c::OLD_0, r.old0); + w.word(c::OLD_1, r.old1); + w.word(c::OLD_TIMESTAMP_LO, r.old_ts_lo); + w.word(c::MU_READ, r.is_read != 0); + w.word(c::MU_WRITE, r.is_read == 0); +} + +// --------------------------------------------------------------------------- +// MEMW_A (`memw_aligned.rs`, `generate_memw_aligned_trace_segments_as`) +// --------------------------------------------------------------------------- + +namespace memw_a_cols { +constexpr uint32_t IS_REGISTER = 0; +constexpr uint32_t BASE_ADDRESS = 1; // DWordWHH: 3 columns +constexpr uint32_t VALUE = 4; // 8 columns +constexpr uint32_t TIMESTAMP_0 = 12; +constexpr uint32_t WRITE2 = 14; +constexpr uint32_t WRITE4 = 15; +constexpr uint32_t WRITE8 = 16; +constexpr uint32_t OLD = 17; // 8 columns +constexpr uint32_t OLD_TIMESTAMP_0 = 25; +constexpr uint32_t MU_READ = 27; +constexpr uint32_t MU_WRITE = 28; +constexpr uint32_t NUM_COLUMNS = 29; +} // namespace memw_a_cols + +namespace fill { + +// `AlignedRow::unpack` into eight columns from `col`: a register access's +// two 32-bit halves (the rest zero), or a memory access's eight bytes. +__device__ __forceinline__ void aligned_elements(const RowWriter &w, uint32_t col, uint64_t packed, + bool is_register) { +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) { + uint64_t element = is_register ? (i < 2 ? (packed >> (32 * i)) & 0xFFFFFFFFull : 0) + : (packed >> (8 * i)) & 0xFFull; + w.word(col + i, element); + } +} + +} // namespace fill + +// Rows `[0, n)` from `rows`, rows `[n, height)` zero. One thread a row. +extern "C" __global__ void fill_memw_a(const AlignedRowC *rows, uint64_t n, uint64_t height, + uint64_t *out) { + namespace c = memw_a_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) return; + const fill::RowWriter w{out, height, row}; + if (row >= n) { + w.zeros(0, c::NUM_COLUMNS); + return; + } + const AlignedRowC r = rows[row]; + const bool is_register = r.is_register != 0; + w.word(c::IS_REGISTER, is_register); + // `set_dword_whh`: the low and middle halves, then the high word. + w.word(c::BASE_ADDRESS, r.base_address & 0xFFFFull); + w.word(c::BASE_ADDRESS + 1, (r.base_address >> 16) & 0xFFFFull); + w.word(c::BASE_ADDRESS + 2, r.base_address >> 32); + fill::aligned_elements(w, c::VALUE, r.value, is_register); + w.dword_wl(c::TIMESTAMP_0, r.timestamp); + w.word(c::WRITE2, r.width == 2); + w.word(c::WRITE4, r.width == 4); + w.word(c::WRITE8, r.width == 8); + fill::aligned_elements(w, c::OLD, r.old, is_register); + w.dword_wl(c::OLD_TIMESTAMP_0, r.old_timestamp); + w.word(c::MU_READ, r.is_read != 0); + w.word(c::MU_WRITE, r.is_read == 0); +} + +// --------------------------------------------------------------------------- +// LOAD (`load.rs`, `generate_load_trace_segments_as`) +// --------------------------------------------------------------------------- + +namespace load_cols { +constexpr uint32_t BASE_ADDRESS_0 = 0; +constexpr uint32_t TIMESTAMP_0 = 2; +constexpr uint32_t READ2 = 4; +constexpr uint32_t READ4 = 5; +constexpr uint32_t READ8 = 6; +constexpr uint32_t SIGNED = 7; +constexpr uint32_t RES = 8; // 8 columns +constexpr uint32_t SIGN_BIT = 16; +constexpr uint32_t MU = 17; +constexpr uint32_t NUM_COLUMNS = 18; +} // namespace load_cols + +// Rows `[0, n)` from `ops`, rows `[n, height)` zero. One thread a row. +extern "C" __global__ void fill_load(const LoadOpC *ops, uint64_t n, uint64_t height, + uint64_t *out) { + namespace c = load_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) return; + const fill::RowWriter w{out, height, row}; + if (row >= n) { + w.zeros(0, c::NUM_COLUMNS); + return; + } + const LoadOpC op = ops[row]; + w.dword_wl(c::BASE_ADDRESS_0, op.base_address); + w.dword_wl(c::TIMESTAMP_0, op.timestamp); + w.word(c::READ2, op.width == 2); + w.word(c::READ4, op.width == 4); + w.word(c::READ8, op.width == 8); + w.word(c::SIGNED, op.is_signed != 0); +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) w.word(c::RES + i, op.res[i]); + // `compute_sign_bit`: the top bit of the highest byte read. + const uint32_t top = op.width == 2 ? 1 : (op.width == 4 ? 3 : (op.width == 8 ? 7 : 0)); + w.word(c::SIGN_BIT, (op.res[top] >> 7) & 1); + w.word(c::MU, 1); +} + +// --------------------------------------------------------------------------- +// STORE (`store.rs`, `generate_store_trace_segments_as`) +// --------------------------------------------------------------------------- + +namespace store_cols { +constexpr uint32_t BASE_ADDRESS_0 = 0; +constexpr uint32_t TIMESTAMP_0 = 2; +constexpr uint32_t WRITE2 = 4; +constexpr uint32_t WRITE4 = 5; +constexpr uint32_t WRITE8 = 6; +constexpr uint32_t VALUE = 7; // 8 columns +constexpr uint32_t MU = 15; +constexpr uint32_t NUM_COLUMNS = 16; +} // namespace store_cols + +// Rows `[0, n)` from `ops`, rows `[n, height)` zero. One thread a row. +extern "C" __global__ void fill_store(const StoreOpC *ops, uint64_t n, uint64_t height, + uint64_t *out) { + namespace c = store_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) return; + const fill::RowWriter w{out, height, row}; + if (row >= n) { + w.zeros(0, c::NUM_COLUMNS); + return; + } + const StoreOpC op = ops[row]; + w.dword_wl(c::BASE_ADDRESS_0, op.base_address); + w.dword_wl(c::TIMESTAMP_0, op.timestamp); + w.word(c::WRITE2, op.write2 != 0); + w.word(c::WRITE4, op.write4 != 0); + w.word(c::WRITE8, op.write8 != 0); + w.dword_bl(c::VALUE, op.value); + w.word(c::MU, 1); +} + +// --------------------------------------------------------------------------- +// SHIFT (`shift.rs`, `generate_shift_trace_segments_as` and +// `ShiftOperation::compute_aux`) +// --------------------------------------------------------------------------- + +namespace shift_cols { +constexpr uint32_t IN_0 = 0; // 4 columns +constexpr uint32_t SHIFT_AMOUNT = 4; +constexpr uint32_t DIRECTION = 5; +constexpr uint32_t SIGNED = 6; +constexpr uint32_t WORD_INSTR = 7; +constexpr uint32_t OUT_0 = 8; // 2 columns +constexpr uint32_t IS_NEGATIVE = 10; +constexpr uint32_t BIT_SHIFT = 11; +constexpr uint32_t ZBS = 12; +constexpr uint32_t X_0 = 13; // 5 columns +constexpr uint32_t Y_0 = 18; // 4 columns +constexpr uint32_t LIMB_SHIFT_RAW_0 = 22; // 3 columns +constexpr uint32_t MU = 25; +constexpr uint32_t SHIFT_B1 = 26; +constexpr uint32_t SHIFT_H1 = 27; +constexpr uint32_t SHIFT_HIGH = 28; +constexpr uint32_t NUM_COLUMNS = 29; +} // namespace shift_cols + +// Rows `[0, n)` from `ops`, rows `[n, height)` padding (ZBS 1, every other +// column zero). One thread a row. +extern "C" __global__ void fill_shift(const ShiftOpC *ops, uint64_t n, uint64_t height, + uint64_t *out) { + namespace c = shift_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) return; + const fill::RowWriter w{out, height, row}; + if (row >= n) { + w.zeros(0, c::NUM_COLUMNS); + w.word(c::ZBS, 1); + return; + } + const ShiftOpC op = ops[row]; + const bool left = op.direction == 0; + const bool right = !left; + const bool is_negative = op.is_signed != 0 && ((op.in_halves[3] >> 15) & 1) == 1; + const uint16_t extension = is_negative ? 0xFFFF : 0; + const uint8_t bit_shift = + left ? (uint8_t)(op.shift & 15) : (uint8_t)((uint16_t)(256 - (uint16_t)op.shift) & 15); + const bool zbs = bit_shift == 0; + // `hwsl`: (half << z) & 0xFFFF; `hwslc`: half >> (16 - z), 0 at z = 0. + uint16_t x[5] = {0, 0, 0, 0, 0}; + uint16_t y[4] = {0, 0, 0, 0}; + if (zbs) { +#pragma unroll + for (int i = 0; i < 4; ++i) { + if (left) { + x[i] = op.in_halves[i]; + } else { + y[i] = op.in_halves[i]; + } + } + } else { +#pragma unroll + for (int i = 0; i < 4; ++i) { + x[i] = (uint16_t)((uint32_t)op.in_halves[i] << bit_shift); + y[i] = (uint16_t)(op.in_halves[i] >> (16 - bit_shift)); + } + x[4] = (uint16_t)((uint32_t)extension << bit_shift); + } + // `limb_shift`: one-hot of (shift >> 4) & mask. + const int limb = op.word_instr != 0 ? ((op.shift >> 4) & 1) : ((op.shift >> 4) & 3); + // `compute_shifted`, 16-bit wrapping sums. + uint16_t shifted[4]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + uint16_t v = 0; + if (left && limb <= i) { + const int k = i - limb; + v = (uint16_t)(v + (k == 0 ? x[0] : (uint16_t)(x[k] + y[k - 1]))); + } + if (right) { + if (limb <= 3 - i) { + const int k = i + limb; + v = (uint16_t)(v + (uint16_t)(y[k] + x[k + 1])); + } else { + v = (uint16_t)(v + extension); + } + } + shifted[i] = v; + } + w.dword_hl(c::IN_0, (uint64_t)op.in_halves[0] | (uint64_t)op.in_halves[1] << 16 | + (uint64_t)op.in_halves[2] << 32 | (uint64_t)op.in_halves[3] << 48); + w.word(c::SHIFT_AMOUNT, op.shift); + w.word(c::SHIFT_B1, (op.shift_amount >> 8) & 0xFFull); + w.word(c::SHIFT_H1, (op.shift_amount >> 16) & 0xFFFFull); + w.word(c::SHIFT_HIGH, op.shift_amount >> 32); + w.word(c::DIRECTION, op.direction != 0); + w.word(c::SIGNED, op.is_signed != 0); + w.word(c::WORD_INSTR, op.word_instr != 0); + w.word(c::OUT_0, (uint64_t)shifted[0] | (uint64_t)shifted[1] << 16); + w.word(c::OUT_0 + 1, (uint64_t)shifted[2] | (uint64_t)shifted[3] << 16); + w.word(c::IS_NEGATIVE, is_negative); + w.word(c::BIT_SHIFT, bit_shift); + w.word(c::ZBS, zbs); +#pragma unroll + for (int i = 0; i < 5; ++i) w.word(c::X_0 + i, x[i]); +#pragma unroll + for (int i = 0; i < 4; ++i) w.word(c::Y_0 + i, y[i]); +#pragma unroll + for (int i = 0; i < 3; ++i) w.word(c::LIMB_SHIFT_RAW_0 + i, limb == i); + w.word(c::MU, 1); +} + +// --------------------------------------------------------------------------- +// MEMW (`memw.rs`, `generate_memw_trace_segments_as`) +// --------------------------------------------------------------------------- + +namespace memw_cols { +constexpr uint32_t IS_REGISTER = 0; +constexpr uint32_t BASE_ADDRESS_0 = 1; +constexpr uint32_t VALUE = 3; // 8 columns +constexpr uint32_t TIMESTAMP_0 = 11; +constexpr uint32_t WRITE2 = 13; +constexpr uint32_t WRITE4 = 14; +constexpr uint32_t WRITE8 = 15; +constexpr uint32_t OLD = 16; // 8 columns +constexpr uint32_t CARRY = 24; // 7 columns +constexpr uint32_t OLD_TIMESTAMP = 31; // 8 × 2 columns +constexpr uint32_t MU_READ = 47; +constexpr uint32_t MU_WRITE = 48; +constexpr uint32_t NUM_COLUMNS = 49; +} // namespace memw_cols + +// Rows `[0, n)` from `ops`, rows `[n, height)` zero. One thread a row. +extern "C" __global__ void fill_memw(const MemwOpC *ops, uint64_t n, uint64_t height, + uint64_t *out) { + namespace c = memw_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) return; + const fill::RowWriter w{out, height, row}; + if (row >= n) { + w.zeros(0, c::NUM_COLUMNS); + return; + } + const MemwOpC &op = ops[row]; + w.word(c::IS_REGISTER, op.is_register != 0); + w.dword_wl(c::BASE_ADDRESS_0, op.base_address); +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) w.word(c::VALUE + i, op.value[i]); + w.dword_wl(c::TIMESTAMP_0, op.timestamp); + w.word(c::WRITE2, op.width == 2); + w.word(c::WRITE4, op.width == 4); + w.word(c::WRITE8, op.width == 8); +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) w.word(c::OLD + i, op.old[i]); + // carry[i]: the low word plus i + 1 reaches 2^32. + const uint64_t lo = op.base_address & 0xFFFFFFFFull; +#pragma unroll + for (uint32_t i = 0; i < 7; ++i) w.word(c::CARRY + i, lo + i + 1 >= (1ull << 32)); +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) w.dword_wl(c::OLD_TIMESTAMP + 2 * i, op.old_timestamp[i]); + w.word(c::MU_READ, op.is_read != 0); + w.word(c::MU_WRITE, op.is_read == 0); +} + +// --------------------------------------------------------------------------- +// CPU (`cpu.rs`, `CpuOperation::from_log` then `fill_cpu_rows`) +// --------------------------------------------------------------------------- + +namespace cpu_cols { +constexpr uint32_t TIMESTAMP = 0; +constexpr uint32_t PC_0 = 1; +constexpr uint32_t RS1 = 3; +constexpr uint32_t RS2 = 4; +constexpr uint32_t RD = 5; +constexpr uint32_t READ_REGISTER1 = 6; +constexpr uint32_t READ_REGISTER2 = 7; +constexpr uint32_t WRITE_REGISTER = 8; +constexpr uint32_t IMM_0 = 9; +constexpr uint32_t HALF_INSTRUCTION_LENGTH = 11; +constexpr uint32_t WORD_INSTR = 12; +constexpr uint32_t ALU = 13; +constexpr uint32_t ALU_FLAGS = 14; +constexpr uint32_t ADD = 15; +constexpr uint32_t SUB = 16; +constexpr uint32_t MEMORY = 17; +constexpr uint32_t MEM_FLAGS = 18; +constexpr uint32_t BRANCH = 19; +constexpr uint32_t ECALL = 20; +constexpr uint32_t NEXT_PC_0 = 21; +constexpr uint32_t RVD_0 = 23; +constexpr uint32_t PC_DOUBLE_READ = 25; +constexpr uint32_t RV1_0 = 26; +constexpr uint32_t RV2_0 = 28; +constexpr uint32_t ARG2_0 = 30; +constexpr uint32_t RES_0 = 32; +constexpr uint32_t BRANCH_COND = 36; +constexpr uint32_t NUM_COLUMNS = 37; +} // namespace cpu_cols + +// `alu_op::EQ` / `alu_op::LT` (`types.rs`). +constexpr uint32_t FILL_ALU_EQ = 3; +constexpr uint32_t FILL_ALU_LT = 4; +// The CPU padding rows' pc (`cpu.rs`, `CPU_PADDING_PC`). +constexpr uint64_t FILL_CPU_PADDING_PC = 1; + +namespace fill { + +// `DecodeTable::row`: the row of the run holding `pc`, or -1. The runs are +// ascending and disjoint, so the only run that can hold `pc` is the last one +// starting at or below it. +__device__ __forceinline__ int64_t decode_row(const DecodeRunC *runs, uint32_t nruns, + uint64_t pc) { + uint32_t lo = 0, hi = nruns; + while (lo < hi) { + uint32_t mid = lo + (hi - lo) / 2; + if (runs[mid].first_pc <= pc) { + lo = mid + 1; + } else { + hi = mid; + } + } + if (lo == 0) return -1; + const DecodeRunC run = runs[lo - 1]; + uint64_t offset = pc - run.first_pc; + if (offset % 4 != 0 || offset / 4 >= run.len) return -1; + return (int64_t)run.row + (int64_t)(offset / 4); +} + +// `CpuOperation::branch_taken`: `alu_flags = op + 32·signed + 64·invert`. +__device__ __forceinline__ bool branch_taken(uint8_t alu_flags, uint64_t rv1, uint64_t rv2) { + uint32_t op = alu_flags & 0x1F; + bool is_signed = (alu_flags >> 5) & 1; + bool invert = (alu_flags >> 6) & 1; + bool cmp = false; + if (op == FILL_ALU_EQ) { + cmp = rv1 == rv2; + } else if (op == FILL_ALU_LT) { + cmp = is_signed ? (int64_t)rv1 < (int64_t)rv2 : rv1 < rv2; + } + return cmp != invert; +} + +} // namespace fill + +// Rows `[0, n)` from `logs`, each read with its DECODE entry; rows +// `[n, height)` the CPU padding rows. Every row's TIMESTAMP is the table's own +// preprocessed `4·(row + 1)` (`cpu.rs`, `local_timestamp`). A log whose pc has +// no DECODE row counts into `missing` and leaves its row zero: the caller +// refuses the table. One thread a row. +extern "C" __global__ void fill_cpu_from_logs(const LogC *logs, uint64_t n, + const DecodeEntryC *entries, + const DecodeRunC *runs, uint32_t nruns, + uint64_t height, uint64_t *out, + unsigned long long *missing) { + namespace c = cpu_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) return; + const fill::RowWriter w{out, height, row}; + const uint64_t timestamp = 4 * (row + 1); + + if (row >= n) { + // pc = next_pc = 1, every flag 0. + w.zeros(0, c::NUM_COLUMNS); + w.word(c::TIMESTAMP, timestamp); + w.word(c::PC_0, FILL_CPU_PADDING_PC); + w.word(c::NEXT_PC_0, FILL_CPU_PADDING_PC); + return; + } + + const LogC log = logs[row]; + int64_t at = fill::decode_row(runs, nruns, log.current_pc); + if (at < 0) { + atomicAdd(missing, 1ull); + w.zeros(0, c::NUM_COLUMNS); + return; + } + const DecodeEntryC e = entries[at]; + uint64_t length = 2 * (uint64_t)e.half_instruction_length; + bool word_instr = e.word_instr != 0; + + // `from_log`. A word instruction's row is a CPU32 delegate: its operational + // columns are zero and its pc advances by the length. + uint64_t next_pc = e.pc + length; + uint64_t rv1 = 0, rv2 = 0, arg2 = 0, res = 0, rvd = 0; + bool branch_cond = false; + if (!word_instr) { + // x255 is the pc register (AUIPC / JAL read it through rs1). + rv1 = e.rs1 == 255 ? log.current_pc : (e.read_register1 ? log.src1_val : 0); + rv2 = e.read_register2 ? log.src2_val : 0; + bool jalr = (e.mem_flags & 1) != 0; + arg2 = e.memory ? e.imm : (e.branch ? rv2 : rv2 + e.imm); + branch_cond = e.branch && (jalr || fill::branch_taken(e.alu_flags, rv1, rv2)); + if (e.add) { + res = rv1 + arg2; + } else if (e.sub) { + res = rv1 - arg2; + } else if (e.alu) { + res = e.branch ? (uint64_t)branch_cond : log.dst_val; + } + // Under MEMORY, mem_flags bit 0 is the store bit. + if (e.memory) { + rvd = jalr ? 0 : log.dst_val; + } else if (e.branch) { + rvd = e.pc + length; + } else { + rvd = res; + } + // An ECALL keeps pc + length (the executor's 0 signals the halt). + if (!e.ecall && branch_cond) next_pc = log.next_pc; + } + + // `fill_cpu_rows`. + bool live = !word_instr; + w.word(c::TIMESTAMP, timestamp); + w.dword_wl(c::PC_0, e.pc); + w.word(c::RS1, live ? e.rs1 : 0); + w.word(c::RS2, live ? e.rs2 : 0); + w.word(c::RD, live ? e.rd : 0); + // x0 is never read or written; x255 is read. + w.word(c::READ_REGISTER1, live && e.read_register1 && e.rs1 != 0); + w.word(c::READ_REGISTER2, live && e.read_register2 && e.rs2 != 0); + w.word(c::WRITE_REGISTER, live && e.write_register && e.rd != 0); + w.dword_wl(c::IMM_0, live ? e.imm : 0); + w.word(c::HALF_INSTRUCTION_LENGTH, e.half_instruction_length); + w.word(c::WORD_INSTR, word_instr); + w.word(c::ALU, live && e.alu); + w.word(c::ALU_FLAGS, live ? e.alu_flags : 0); + w.word(c::ADD, live && e.add); + w.word(c::SUB, live && e.sub); + w.word(c::MEMORY, live && e.memory); + w.word(c::MEM_FLAGS, live ? e.mem_flags : 0); + w.word(c::BRANCH, live && e.branch); + w.word(c::ECALL, live && e.ecall); + w.dword_wl(c::NEXT_PC_0, next_pc); + w.dword_wl(c::RVD_0, rvd); + w.dword_wl(c::RV1_0, rv1); + w.dword_wl(c::RV2_0, rv2); + w.dword_wl(c::ARG2_0, arg2); + w.dword_hl(c::RES_0, res); + w.word(c::BRANCH_COND, branch_cond); + // The inline pc read's column. + w.word(c::PC_DOUBLE_READ, live && e.read_register1 && e.rs1 == 255); +} + +// --------------------------------------------------------------------------- +// KECCAK_RND (`keccak_rnd.rs`, `generate_keccak_rnd_rows_as` and `fill_op`) +// --------------------------------------------------------------------------- + +namespace keccak_cols { +constexpr uint32_t TIMESTAMP_0 = 0; +constexpr uint32_t ROUND = 2; +constexpr uint32_t START = 3; +constexpr uint32_t CXZ = START + 200; +constexpr uint32_t CXZ_LEFT = CXZ + 160; +constexpr uint32_t CXZ_RIGHT = CXZ_LEFT + 40; +constexpr uint32_t DXZ = CXZ_RIGHT + 20; +constexpr uint32_t THETA = DXZ + 40; +constexpr uint32_t ROT_LEFT = THETA + 200; +constexpr uint32_t ROT_RIGHT = ROT_LEFT + 200; +constexpr uint32_t CHI_ANDS = ROT_RIGHT + 200; +constexpr uint32_t CHI = CHI_ANDS + 200; +constexpr uint32_t RC = CHI + 200; +constexpr uint32_t IOTA = RC + 8; +constexpr uint32_t MU = IOTA + 8; +constexpr uint32_t NUM_COLUMNS = MU + 1; +static_assert(NUM_COLUMNS == 1480, "KECCAK_RND has 1,480 columns"); +} // namespace keccak_cols + +// The kernel's grid.y: three runs of columns, each a function of the round's +// start state alone. Threads of different sections recompute the state. +// 0: timestamp, round, start, Cxz, Cxz_left, Cxz_right, Dxz [0, 463) +// 1: theta, rot_left, rot_right [463, 1063) +// 2: chi_ands, chi, rc, iota, mu [1063, 1480) +constexpr uint32_t FILL_KECCAK_SECTIONS = 3; + +__device__ __constant__ uint64_t FILL_KECCAK_RC[24] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL, +}; + +// `KECCAK_RHO[x][y]`, indexed by lane `x + 5y`. +__device__ __constant__ uint32_t FILL_KECCAK_RHO[25] = { + 0, 1, 62, 28, 27, // y = 0: x = 0..4 + 36, 44, 6, 55, 20, // y = 1 + 3, 10, 43, 25, 39, // y = 2 + 41, 45, 15, 21, 8, // y = 3 + 18, 2, 61, 56, 14, // y = 4 +}; + +namespace fill { + +__device__ __forceinline__ uint64_t rotl(uint64_t v, uint32_t n) { + return n == 0 ? v : (v << n) | (v >> (64 - n)); +} + +// θ's column parities `C[x]` and `D[x] = C[x−1] ⊕ rotl(C[x+1], 1)`. +__device__ __forceinline__ void theta_d(const uint64_t s[25], uint64_t cpar[5], + uint64_t d[5]) { +#pragma unroll + for (int x = 0; x < 5; ++x) cpar[x] = s[x] ^ s[x + 5] ^ s[x + 10] ^ s[x + 15] ^ s[x + 20]; +#pragma unroll + for (int x = 0; x < 5; ++x) d[x] = cpar[(x + 4) % 5] ^ rotl(cpar[(x + 1) % 5], 1); +} + +// θ applied, then ρ and π: `b[y + 5·((2x + 3y) mod 5)] = rotl(θ[x + 5y], ρ[x][y])`. +__device__ __forceinline__ void rho_pi(uint64_t s[25], const uint64_t d[5], uint64_t b[25]) { +#pragma unroll + for (int y = 0; y < 5; ++y) { +#pragma unroll + for (int x = 0; x < 5; ++x) { + b[y + 5 * ((2 * x + 3 * y) % 5)] = rotl(s[x + 5 * y] ^ d[x], FILL_KECCAK_RHO[x + 5 * y]); + } + } +} + +// One round of Keccak-f[1600] on `s`. +__device__ __forceinline__ void keccak_round(uint64_t s[25], uint64_t rc) { + uint64_t cpar[5], d[5], b[25]; + theta_d(s, cpar, d); + rho_pi(s, d, b); +#pragma unroll + for (int y = 0; y < 5; ++y) { +#pragma unroll + for (int x = 0; x < 5; ++x) { + s[x + 5 * y] = b[x + 5 * y] ^ (~b[(x + 1) % 5 + 5 * y] & b[(x + 2) % 5 + 5 * y]); + } + } + s[0] ^= rc; +} + +// `hwsl` on each halfword of `v` at once: the halfwords shifted left by `n` +// (their low part, `rot_left`) and the bits they carry out (`rot_right`). +__device__ __forceinline__ void halfword_shift(uint64_t v, uint32_t n, uint64_t &left, + uint64_t &right) { + if (n == 0) { + left = v; + right = 0; + return; + } + const uint64_t keep_high = ((0xFFFFull << n) & 0xFFFFull) * 0x0001000100010001ull; + const uint64_t keep_low = ((1ull << n) - 1) * 0x0001000100010001ull; + left = (v << n) & keep_high; + right = (v >> (16 - n)) & keep_low; +} + +} // namespace fill + +// Rows `[skip, skip + rows)` of the table `ops` make (24 rows an op, op `k`'s +// round `r` at row `24k + r`), as a table of `rows` rows; rows past the last +// op are zero (μ = 0). An op the slice cuts writes only its rows inside. Grid: +// x a run of rows, one thread a row, so a column's stores coalesce; y the +// column section (`FILL_KECCAK_SECTIONS`). `Op` is either record: the rows +// read its timestamp and input alone. +template +__device__ __forceinline__ void fill_keccak_rnd_rows(const Op *ops, uint32_t nops, uint64_t skip, + uint64_t rows, uint64_t *out) { + namespace c = keccak_cols; + uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + const uint32_t section = blockIdx.y; + const fill::RowWriter w{out, rows, row}; + const uint32_t first_col = section == 0 ? 0 : (section == 1 ? c::THETA : c::CHI_ANDS); + const uint32_t end_col = section == 0 ? c::THETA : (section == 1 ? c::CHI_ANDS : c::NUM_COLUMNS); + + const uint64_t at = skip + row; + const uint64_t op = at / 24; + const uint32_t round = (uint32_t)(at % 24); + if (op >= nops) { + w.zeros(first_col, end_col); + return; + } + + // The state at the start of this row's round. + uint64_t s[25]; +#pragma unroll + for (int i = 0; i < 25; ++i) s[i] = ops[op].input[i]; + for (uint32_t r = 0; r < round; ++r) fill::keccak_round(s, FILL_KECCAK_RC[r]); + + uint64_t cpar[5], d[5]; + fill::theta_d(s, cpar, d); + + if (section == 0) { + w.dword_wl(c::TIMESTAMP_0, ops[op].timestamp); + w.word(c::ROUND, round); +#pragma unroll + for (int lane = 0; lane < 25; ++lane) w.dword_bl(c::START + 8 * lane, s[lane]); +#pragma unroll + for (int x = 0; x < 5; ++x) { + // The parity chain: stage k is lanes y = 0..k+1 of column x. + uint64_t chain = s[x] ^ s[x + 5]; + w.dword_bl(c::CXZ + (4 * x) * 8, chain); +#pragma unroll + for (int stage = 1; stage < 4; ++stage) { + chain ^= s[x + 5 * (stage + 1)]; + w.dword_bl(c::CXZ + (4 * x + stage) * 8, chain); + } + // HWSL(C[x], 1): the halfwords shifted by one and their carry bits. + uint64_t left, right; + fill::halfword_shift(cpar[x], 1, left, right); + w.dword_bl(c::CXZ_LEFT + 8 * x, left); +#pragma unroll + for (int hw = 0; hw < 4; ++hw) { + w.word(c::CXZ_RIGHT + 4 * x + hw, (right >> (16 * hw)) & 1); + } + w.dword_bl(c::DXZ + 8 * x, d[x]); + } + } else if (section == 1) { +#pragma unroll + for (int y = 0; y < 5; ++y) { +#pragma unroll + for (int x = 0; x < 5; ++x) { + const int lane = x + 5 * y; + const uint64_t theta = s[lane] ^ d[x]; + w.dword_bl(c::THETA + 8 * lane, theta); + uint64_t left, right; + fill::halfword_shift(theta, FILL_KECCAK_RHO[lane] % 16, left, right); + w.dword_bl(c::ROT_LEFT + 8 * lane, left); + w.dword_bl(c::ROT_RIGHT + 8 * lane, right); + } + } + } else { + uint64_t b[25]; + fill::rho_pi(s, d, b); + const uint64_t rc = FILL_KECCAK_RC[round]; +#pragma unroll + for (int y = 0; y < 5; ++y) { +#pragma unroll + for (int x = 0; x < 5; ++x) { + const int lane = x + 5 * y; + const uint64_t ands = ~b[(x + 1) % 5 + 5 * y] & b[(x + 2) % 5 + 5 * y]; + const uint64_t chi = b[lane] ^ ands; + w.dword_bl(c::CHI_ANDS + 8 * lane, ands); + w.dword_bl(c::CHI + 8 * lane, chi); + if (lane == 0) w.dword_bl(c::IOTA, chi ^ rc); + } + } + w.dword_bl(c::RC, rc); + w.word(c::MU, 1); + } +} + +extern "C" __global__ void fill_keccak_rnd(const KeccakOpC *ops, uint32_t nops, uint64_t skip, + uint64_t rows, uint64_t *out) { + fill_keccak_rnd_rows(ops, nops, skip, rows, out); +} + +// The same, over the records a table held as its keccak operations uploads. +extern "C" __global__ void fill_keccak_rnd_in(const KeccakOpInC *ops, uint32_t nops, + uint64_t skip, uint64_t rows, uint64_t *out) { + fill_keccak_rnd_rows(ops, nops, skip, rows, out); +} + +// --------------------------------------------------------------------------- +// The LFM width-16 Poseidon1 socket (`prover/src/lfm/p1w16_socket.rs`): its +// row-major trace widened from the columns the host uploads — the preprocessed +// prefix and the sixteen input lanes, `narrow_w = prep_w + 16` words a row — +// to the whole row (`width` words): the prefix and inputs copied, then the +// round witness, `x³` and `x⁷` of every S-boxed lane of every round, in +// `fill_row`'s column order from `rounds_off`. Rows at or past `real_rows` are +// padding: their witness words are zero, as the host's calloc'd trace leaves +// them. Row-major in and out (`out[row · width + col]`), the R1 main commit's +// input layout. +// +// BIT-IDENTICAL to the host fill, not only congruent: the trace words feed the +// LDE and the commitment as stored. So every operation is the host's own on +// raw `[0, 2^64)` words: `goldilocks::add`/`mul` port the host's `add`/`mul` +// one to one; `x³ = (a·a)·a` and `x⁷ = (x³·x³)·a` keep `fill_row`'s +// association; the MDS is `poseidon1_w16::mds` — the exact 128-bit lane sum +// `Σ s_j·M[i][j]` over raw words, then `Fp::from(lo) + Fp::from(hi·EPSILON)`, +// where `Fp::from` reduces a word ≥ p once (`canonical`) and `hi·EPSILON` +// (< 2^41) is already reduced; the round constants go through `Fp::from` too. +// --------------------------------------------------------------------------- +namespace lfm_socket { + +constexpr int WIDTH = 16; +constexpr int ROUNDS = 30; +constexpr int HALF_FULL = 4; + +__device__ __forceinline__ bool full_round(int r) { return r < HALF_FULL || r >= ROUNDS - HALF_FULL; } + +// `poseidon1_w16::mds`, word for word. +__device__ __forceinline__ void mds(const uint64_t s[WIDTH], uint64_t out[WIDTH]) { + uint32_t lo32[WIDTH], hi32[WIDTH]; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + lo32[j] = (uint32_t)s[j]; + hi32[j] = (uint32_t)(s[j] >> 32); + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) { + // Σ_j c·lo32_j and Σ_j c·hi32_j, each < 371·2^32 < 2^41: the lane's + // exact sum is acc_hi·2^32 + acc_lo < 2^73. + uint64_t acc_lo = 0, acc_hi = 0; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + const uint64_t c = p1w16::MDS_ROW[(j + WIDTH - i) % WIDTH]; + acc_lo += c * (uint64_t)lo32[j]; + acc_hi += c * (uint64_t)hi32[j]; + } + const uint64_t lo = (acc_hi << 32) + acc_lo; + const uint64_t carry = (lo < acc_lo) ? 1ull : 0ull; + const uint64_t hi = (acc_hi >> 32) + carry; + out[i] = goldilocks::add(goldilocks::canonical(lo), hi * goldilocks::EPSILON); + } +} + +} // namespace lfm_socket + +extern "C" __global__ void fill_lfm_p1w16_socket(const uint64_t *__restrict__ narrow, uint64_t rows, + uint64_t real_rows, uint64_t prep_w, uint64_t width, + uint64_t rounds_off, uint64_t *__restrict__ out) { + using namespace lfm_socket; + const uint64_t narrow_w = prep_w + WIDTH; + for (uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; row < rows; + row += (uint64_t)gridDim.x * blockDim.x) { + const uint64_t *src = narrow + row * narrow_w; + uint64_t *dst = out + row * width; + for (uint64_t c = 0; c < narrow_w; ++c) dst[c] = src[c]; + if (row >= real_rows) { + for (uint64_t c = narrow_w; c < width; ++c) dst[c] = 0; + continue; + } + uint64_t s[WIDTH]; +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = src[prep_w + i]; + uint64_t off = rounds_off; + for (int r = 0; r < ROUNDS; ++r) { + uint64_t f[WIDTH]; +#pragma unroll + for (int i = 0; i < WIDTH; ++i) f[i] = goldilocks::add(s[i], goldilocks::canonical(p1w16::RC[r][i])); + const int lanes = full_round(r) ? WIDTH : 1; + for (int lane = 0; lane < lanes; ++lane) { + const uint64_t a = f[lane]; + const uint64_t x3 = goldilocks::mul(goldilocks::mul(a, a), a); + const uint64_t x7 = goldilocks::mul(goldilocks::mul(x3, x3), a); + dst[off + lane] = x3; + dst[off + lanes + lane] = x7; + f[lane] = x7; + } + off += 2 * (uint64_t)lanes; + mds(f, s); + } + } +} diff --git a/crypto/math-cuda/kernels/inverse.cu b/crypto/math-cuda/kernels/inverse.cu index 65d54afc0..e584277ae 100644 --- a/crypto/math-cuda/kernels/inverse.cu +++ b/crypto/math-cuda/kernels/inverse.cu @@ -344,3 +344,62 @@ extern "C" __global__ void invert_total_ext3( out[1] = r.b; out[2] = r.c; } + +// --------------------------------------------------------------------------- +// 8. invert_denoms_rowwise_ext3_k +// +// The same output as `compute_denoms_ext3` followed by the six-kernel batch +// inverse — `inv_out[(k * n + i) * 3 ..]` = 1 / (sign-dependent `x[i]`, `z[k]` +// denominator), `DenomSign` as in (1) — in ONE pass: thread `i` builds row +// `i`'s K denominators and inverts them in registers with one base-field +// inversion (`ext3_inv.cuh`). No prefix/suffix scratch, no recursion over block +// totals, no host-side or one-thread Fermat step. +// +// `zero_flag` (nullable) is set when a row's denominator product is zero, so a +// debug caller can keep the scan path's invertibility guard. +// --------------------------------------------------------------------------- +#include "ext3_inv.cuh" + +template +__device__ __forceinline__ void invert_denoms_rowwise_body(const uint64_t *__restrict__ x_base, + const uint64_t *__restrict__ z_scalars, + uint64_t n, uint64_t denom_sign, + uint64_t *__restrict__ inv_out, + uint32_t *zero_flag) { + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + uint64_t x = x_base[i]; + ext3::Fe3 d[K]; +#pragma unroll + for (int k = 0; k < K; ++k) { + d[k] = denom_sign == 0 ? ext3_inv::z_minus_x(x, z_scalars + 3 * k) + : ext3_inv::x_minus_z(x, z_scalars + 3 * k); + } + uint64_t product = ext3_inv::batch_inv(d); + if (zero_flag != nullptr && goldilocks::canonical(product) == 0) atomicOr(zero_flag, 1u); +#pragma unroll + for (int k = 0; k < K; ++k) { + uint64_t *out = inv_out + ((uint64_t)k * n + i) * 3; + out[0] = d[k].a; + out[1] = d[k].b; + out[2] = d[k].c; + } +} + +#define INVERT_DENOMS_ROWWISE(K) \ + extern "C" __global__ void invert_denoms_rowwise_ext3_k##K( \ + const uint64_t *__restrict__ x_base, const uint64_t *__restrict__ z_scalars, uint64_t n, \ + uint64_t denom_sign, uint64_t *__restrict__ inv_out, uint32_t *zero_flag) { \ + invert_denoms_rowwise_body(x_base, z_scalars, n, denom_sign, inv_out, zero_flag); \ + } + +// K up to the barycentric multi kernels' cap (BARY_MAX_EVAL_POINTS = 8) covers +// every R3 shape; R4 asks for 1 + num_eval_points. +INVERT_DENOMS_ROWWISE(1) +INVERT_DENOMS_ROWWISE(2) +INVERT_DENOMS_ROWWISE(3) +INVERT_DENOMS_ROWWISE(4) +INVERT_DENOMS_ROWWISE(5) +INVERT_DENOMS_ROWWISE(6) +INVERT_DENOMS_ROWWISE(7) +INVERT_DENOMS_ROWWISE(8) diff --git a/crypto/math-cuda/kernels/keccak.cu b/crypto/math-cuda/kernels/keccak.cu index 2762d7469..923a978b6 100644 --- a/crypto/math-cuda/kernels/keccak.cu +++ b/crypto/math-cuda/kernels/keccak.cu @@ -207,6 +207,29 @@ extern "C" __global__ void grind_search(const uint64_t *inner_lanes, // `c` is the contiguous slab `[c*col_stride .. c*col_stride + num_rows]`. The // remaining `col_stride - num_rows` entries (if any) are ignored. // --------------------------------------------------------------------------- +__device__ __forceinline__ void hash_base_row(const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t row, + uint8_t *out32) { + uint64_t st[25]; + #pragma unroll + for (int i = 0; i < 25; ++i) st[i] = 0; + + uint32_t rate_pos = 0; + for (uint64_t c = 0; c < num_cols; ++c) { + uint64_t v = columns_base_ptr[c * col_stride + row]; + // Canonicalise to match `canonical_u64().to_be_bytes()` on host. + uint64_t canon = goldilocks::canonical(v); + // The on-disk leaf bytes are canon.to_be_bytes(). Keccak reads those + // as a LE lane, which equals bswap64(canon). + uint64_t lane = bswap64(canon); + absorb_lane(st, rate_pos, lane); + } + + finalize_keccak256(st, rate_pos, out32); +} + extern "C" __global__ void keccak256_leaves_base_batched( const uint64_t *columns_base_ptr, uint64_t col_stride, @@ -220,23 +243,55 @@ extern "C" __global__ void keccak256_leaves_base_batched( // Bit-reverse the row index so we read columns at `br` but write the hashed // leaf at `tid` — matching the CPU per-row `commit_bit_reversed(.., 1)`. uint64_t br = __brevll(tid) >> (64 - log_num_rows); + hash_base_row(columns_base_ptr, col_stride, num_cols, br, + hashed_leaves_out + tid * 32); +} + +// --------------------------------------------------------------------------- +// Goldilocks BASE-FIELD strided-coset leaf hashing. +// +// Leaf `j` hashes `codeword[j + t * num_leaves]` for `t` in `[0, block)`, in +// coset order — the fold block a WHIR query opens. Rows are read in natural +// order, not bit-reversed: the codeword's own index order is what the fold +// pairs, so there is no permutation between the two. +// --------------------------------------------------------------------------- +extern "C" __global__ void keccak256_leaves_base_coset( + const uint64_t *codeword, + uint64_t num_leaves, + uint64_t block, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + // A coset is a stride of `num_leaves`, which is exactly `block` slabs of + // that stride read at the same offset. + hash_base_row(codeword, num_leaves, block, tid, hashed_leaves_out + tid * 32); +} + +// Leaf hashing for an ext3 codeword's fold blocks: leaf `j` hashes +// `codeword[j + t·num_leaves]` for `t` in `[0, block)`, each element as its +// three components in canonical big-endian order — what +// `FieldElement::::write_bytes_be` streams, and what the base-field +// coset kernel does one component at a time. +extern "C" __global__ void keccak256_leaves_ext3_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; uint64_t st[25]; - #pragma unroll +#pragma unroll for (int i = 0; i < 25; ++i) st[i] = 0; uint32_t rate_pos = 0; - for (uint64_t c = 0; c < num_cols; ++c) { - uint64_t v = columns_base_ptr[c * col_stride + br]; - // Canonicalise to match `canonical_u64().to_be_bytes()` on host. - uint64_t canon = goldilocks::canonical(v); - // The on-disk leaf bytes are canon.to_be_bytes(). Keccak reads those - // as a LE lane, which equals bswap64(canon). - uint64_t lane = bswap64(canon); - absorb_lane(st, rate_pos, lane); + for (uint64_t t = 0; t < block; ++t) { + const uint64_t *at = codeword + (tid + t * num_leaves) * 3; +#pragma unroll + for (int k = 0; k < 3; ++k) { + absorb_lane(st, rate_pos, bswap64(goldilocks::canonical(at[k]))); + } } - - finalize_keccak256(st, rate_pos, hashed_leaves_out + tid * 32); + finalize_keccak256(st, rate_pos, out + tid * 32); } // --------------------------------------------------------------------------- @@ -409,6 +464,37 @@ extern "C" __global__ void keccak_fri_leaves_ext3( finalize_keccak256(st, rate_pos, leaves_out + tid * 32); } +// --------------------------------------------------------------------------- +// FRI GROUP-leaf hashing (S3, higher-arity committed FRI layers). +// +// Leaf `tid` hashes the `group` consecutive ext3 values +// `evals[tid*group .. (tid+1)*group]` of an interleaved eval vector — the +// `3*group` contiguous u64s at `evals_interleaved + tid*group*3` — each value +// as its three components in canonical big-endian order. That is the host +// `Batched` leaf over the group (`hash_data_from_slices(group, [])`), and at +// `group = 2` exactly `keccak_fri_leaves_ext3`'s byte stream. No bit reversal. +// --------------------------------------------------------------------------- +extern "C" __global__ void keccak_fri_group_leaves_ext3( + const uint64_t *evals_interleaved, // 3 * num_leaves * group u64s + uint64_t num_leaves, + uint64_t group, // ext3 values per leaf (2^d) + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + uint64_t st[25]; + #pragma unroll + for (int i = 0; i < 25; ++i) st[i] = 0; + uint32_t rate_pos = 0; + + const uint64_t *g = evals_interleaved + tid * group * 3; + for (uint64_t i = 0; i < 3 * group; ++i) { + absorb_lane(st, rate_pos, bswap64(goldilocks::canonical(g[i]))); + } + + finalize_keccak256(st, rate_pos, leaves_out + tid * 32); +} + // --------------------------------------------------------------------------- // Merkle inner-tree pair hash: one level of the inner Merkle tree. // @@ -597,3 +683,43 @@ extern "C" __global__ void keccak256_leaves_base_row_major_row_pair_range( } finalize_keccak256(st, rate_pos, hashed_leaves_out + tid * 32); } + +// --------------------------------------------------------------------------- +// Row-major ONE-ROW leaf hashing (S2, rows_per_leaf = 1). +// +// Leaf `tid` hashes the single row `reverse_index(tid)` (bit reversal over +// `log_num_rows` bits), columns `[col_start, col_end)` of the contiguous +// row-major buffer (`data + br * m`, `m` the full row stride), as canonical +// big-endian lanes. `num_leaves = num_rows`. Byte layout equals the CPU +// `commit_rows_bit_reversed_subset_with(data, m, col_start, col_end, 1)`; the +// whole row (`[0, m)`) is `commit_rows_bit_reversed_with(data, m, 1)`. +// +// NOT the row-pair kernels at another width: those read rows `brev(2·tid)` and +// `brev(2·tid + 1)` over `log_num_rows` bits, which is a different row set, +// so one row per leaf needs its own read pattern. +// --------------------------------------------------------------------------- +extern "C" __global__ void keccak256_leaves_base_row_major_row_range( + const uint64_t *data, + uint64_t m, + uint64_t col_start, + uint64_t col_end, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) +{ + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + const uint64_t *row = data + br * m; + + uint64_t st[25]; + #pragma unroll + for (int i = 0; i < 25; ++i) st[i] = 0; + + uint32_t rate_pos = 0; + for (uint64_t c = col_start; c < col_end; ++c) { + absorb_lane(st, rate_pos, bswap64(goldilocks::canonical(row[c]))); + } + finalize_keccak256(st, rate_pos, hashed_leaves_out + tid * 32); +} diff --git a/crypto/math-cuda/kernels/ntt.cu b/crypto/math-cuda/kernels/ntt.cu index 1e6c83f5c..dc1c59f5a 100644 --- a/crypto/math-cuda/kernels/ntt.cu +++ b/crypto/math-cuda/kernels/ntt.cu @@ -13,6 +13,24 @@ using goldilocks::add; using goldilocks::sub; using goldilocks::mul; +/// The lift's permutation and the NTT's, composed — which turns out to be +/// neither of them. +/// +/// The lift reverses a coefficient's index over `log_evals` bits and the NTT +/// wants its input reversed over `log_n = log_evals + log_blowup`. Reversing +/// twice around the zero padding leaves coefficient `m` at `m << log_blowup` +/// and a zero at every other index: the two passes over the whole codeword, +/// each of them a scattered read, are one pass that also writes the padding. +extern "C" __global__ void lift_spread(const uint64_t *__restrict__ coeffs, + uint64_t n, + uint32_t log_blowup, + uint64_t *__restrict__ out) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= n) return; + uint64_t mask = ((uint64_t)1 << log_blowup) - 1; + out[j] = (j & mask) == 0 ? coeffs[j >> log_blowup] : 0; +} + /// Reverse the low `log_n` bits of each index and swap x[i] ↔ x[rev(i)]. /// One thread per index; guarded by `tid < rev` to avoid double-swap. extern "C" __global__ void bit_reverse_permute(uint64_t *x, @@ -30,6 +48,212 @@ extern "C" __global__ void bit_reverse_permute(uint64_t *x, } } +/// One level of the Möbius transform that turns a multilinear's hypercube +/// evaluations into its monomial coefficients: every index whose `stride` bit +/// is set loses its partner below. `n/2` threads; the level's blocks are +/// independent, so nothing crosses a block boundary. +extern "C" __global__ void mobius_level(uint64_t *x, + uint64_t half, + uint64_t stride) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= half) return; + + // Spread `tid` over the indices that have the stride bit set: the low + // `log2(stride)` bits stay, the rest move up one. + uint64_t low = tid & (stride - 1); + uint64_t high = tid - low; + uint64_t i = (high << 1) | stride | low; + x[i] = sub(x[i], x[i ^ stride]); +} + +/// Columns a Mobius tile spans: one warp, so the load and the store coalesce. +#define MOBIUS_TILE_COLS 32 + +/// The first `k_levels` Mobius levels, fused through shared memory. +/// +/// Levels below eight move only the low eight bits of an index, so a block of +/// 256 consecutive elements already holds both sides of every pair it has to +/// subtract — no remap, and the load and the store are contiguous. +extern "C" __global__ void mobius_low_levels(uint64_t *x, uint32_t k_levels) { + __shared__ uint64_t tile[256]; + uint64_t base = (uint64_t)blockIdx.x * 256; + tile[threadIdx.x] = x[base + threadIdx.x]; + __syncthreads(); + + for (uint32_t l = 0; l < k_levels; ++l) { + uint32_t half = 1u << l; + // Only the element with the level's bit set is written; the other is + // what it subtracts. + if (threadIdx.x & half) { + tile[threadIdx.x] = sub(tile[threadIdx.x], tile[threadIdx.x ^ half]); + } + __syncthreads(); + } + + x[base + threadIdx.x] = tile[threadIdx.x]; +} + +/// `k_levels` Mobius levels from `base`, fused through shared memory. +/// +/// The elements that interact across levels `base .. base + k_levels - 1` are +/// exactly those differing only in the middle `k_levels` bits of their index: +/// write `i = q*2^(base+k) + r*2^base + c` and the levels move `r` while `q` +/// and `c` stay put. A block takes one `q`, a warp's worth of consecutive `c`, +/// and every `r` — so `threadIdx.x` walks what is contiguous in memory and +/// `threadIdx.y` walks the stride the subtractions work on. Gathering along +/// the stride instead loses more to uncoalesced traffic than the fused passes +/// save, which is what the NTT's tile is shaped this way for. +/// +/// Requires `2^base >= MOBIUS_TILE_COLS`; the levels below that are +/// [`mobius_low_levels`]. +extern "C" __global__ void mobius_tile(uint64_t *x, uint64_t base, uint32_t k_levels) { + // rows x (MOBIUS_TILE_COLS + 1): the odd stride keeps a column off one bank. + extern __shared__ uint64_t tile[]; + const uint32_t cols = MOBIUS_TILE_COLS; + const uint32_t pitch = cols + 1; + + uint64_t low = (uint64_t)1 << base; + uint64_t c = (uint64_t)blockIdx.x * cols + threadIdx.x; + // The block's `q`: grid.z carries the multiples of grid.y that grid.y's + // 65,535 cannot (see `ntt::split_grid_y`). + uint64_t q = (uint64_t)blockIdx.z * gridDim.y + blockIdx.y; + uint64_t i = q * (low << k_levels) + (uint64_t)threadIdx.y * low + c; + + tile[threadIdx.y * pitch + threadIdx.x] = x[i]; + __syncthreads(); + + for (uint32_t l = 0; l < k_levels; ++l) { + uint32_t half = 1u << l; + if (threadIdx.y & half) { + uint32_t r1 = threadIdx.y; + uint32_t r0 = r1 ^ half; + tile[r1 * pitch + threadIdx.x] = + sub(tile[r1 * pitch + threadIdx.x], tile[r0 * pitch + threadIdx.x]); + } + __syncthreads(); + } + + x[i] = tile[threadIdx.y * pitch + threadIdx.x]; +} + +// --------------------------------------------------------------------------- +// ★ The Möbius transform in three passes instead of five (the WHIR encoding's +// default, `LAMBDA_VM_WHIR_MOBIUS_WIDE`; `src/whir.rs` `mobius_wide`). Each element +// takes exactly the operations, in exactly the level order, that +// `mobius_low_levels` / `mobius_tile` give it — the element with the level's +// bit set loses its partner — so the words are theirs bit for bit. Two kernels: +// mobius_wide_low the first `k_levels <= 11` levels over contiguous tiles +// of 2^11 elements, 1,024 threads; with `parts` it GATHERS +// the stack from its parts on the way in (zero where no +// part covers), so the stack copy and its memset never run. +// mobius_wide_tile `k_levels <= 8` strided levels from `base` over a tile of +// 2^k rows × 32 columns, each of the 1,024 threads holding +// 2^k / 32 rows (dynamic shared memory up to 66 KiB); with +// `parts` it gathers the stack the same way (the fused +// encoding, `LAMBDA_VM_WHIR_MOBIUS_FUSED`, whose low +// levels run in the NTT's first pass, starts here). +// --------------------------------------------------------------------------- + +#define MOBIUS_WIDE_LOG 11 + +/// The stack's word at `e`: the part covering it, or zero. `parts` is the +/// table `(offset, len, src)` per part, sorted by offset, not overlapping; +/// `[p0, p1)` are the parts that meet this tile (`p0 < p1`). +__device__ __forceinline__ uint64_t mobius_gather(const uint64_t *parts, uint32_t p0, uint32_t p1, + uint64_t e) { + uint32_t lo = p0, hi = p1; // the last part with offset <= e, else p0 + while (hi - lo > 1) { + const uint32_t mid = (lo + hi) / 2; + if (parts[3 * mid] <= e) { + lo = mid; + } else { + hi = mid; + } + } + const uint64_t off = parts[3 * lo], len = parts[3 * lo + 1]; + if (e < off || e >= off + len) return 0; + const uint64_t *src = reinterpret_cast(parts[3 * lo + 2]); + return src[e - off]; +} + +extern "C" __global__ void mobius_wide_low(uint64_t *x, const uint64_t *parts, uint32_t n_parts, + uint32_t k_levels) { + __shared__ uint64_t tile[1 << MOBIUS_WIDE_LOG]; + __shared__ uint32_t span[2]; + const uint32_t n = 1u << MOBIUS_WIDE_LOG; + const uint64_t base = (uint64_t)blockIdx.x << MOBIUS_WIDE_LOG; + if (parts != nullptr) { + if (threadIdx.x == 0) { + // The first part that ends after the tile's start, then every part + // that starts before its end. + uint32_t lo = 0, hi = n_parts; + while (lo < hi) { + const uint32_t mid = (lo + hi) / 2; + if (parts[3 * mid] + parts[3 * mid + 1] <= base) { + lo = mid + 1; + } else { + hi = mid; + } + } + uint32_t end = lo; + while (end < n_parts && parts[3 * end] < base + n) ++end; + span[0] = lo; + span[1] = end; + } + __syncthreads(); + const uint32_t p0 = span[0], p1 = span[1]; + for (uint32_t j = threadIdx.x; j < n; j += blockDim.x) + tile[j] = (p0 < p1) ? mobius_gather(parts, p0, p1, base + j) : 0; + } else { + for (uint32_t j = threadIdx.x; j < n; j += blockDim.x) tile[j] = x[base + j]; + } + __syncthreads(); + for (uint32_t l = 0; l < k_levels; ++l) { + const uint32_t half = 1u << l; + for (uint32_t t = threadIdx.x; t < n / 2; t += blockDim.x) { + const uint32_t i1 = ((t >> l) << (l + 1)) | half | (t & (half - 1u)); + tile[i1] = sub(tile[i1], tile[i1 ^ half]); + } + __syncthreads(); + } + for (uint32_t j = threadIdx.x; j < n; j += blockDim.x) x[base + j] = tile[j]; +} + +extern "C" __global__ void mobius_wide_tile(uint64_t *x, uint64_t base, uint32_t k_levels, + const uint64_t *parts, uint32_t n_parts) { + extern __shared__ uint64_t tile[]; + const uint32_t cols = MOBIUS_TILE_COLS; + const uint32_t pitch = cols + 1; + const uint32_t rows = 1u << k_levels; + const uint64_t low = (uint64_t)1 << base; + const uint64_t c = (uint64_t)blockIdx.x * cols + threadIdx.x; + const uint64_t q = (uint64_t)blockIdx.z * gridDim.y + blockIdx.y; + const uint64_t i0 = q * (low << k_levels) + c; + if (parts != nullptr) { + // Each element's part by a search of the whole table (zero where none + // covers it, and for an empty table). + for (uint32_t r = threadIdx.y; r < rows; r += blockDim.y) + tile[r * pitch + threadIdx.x] = + n_parts ? mobius_gather(parts, 0, n_parts, i0 + r * low) : 0; + } else { + for (uint32_t r = threadIdx.y; r < rows; r += blockDim.y) + tile[r * pitch + threadIdx.x] = x[i0 + r * low]; + } + __syncthreads(); + for (uint32_t l = 0; l < k_levels; ++l) { + const uint32_t half = 1u << l; + for (uint32_t r = threadIdx.y; r < rows; r += blockDim.y) { + if (r & half) { + tile[r * pitch + threadIdx.x] = + sub(tile[r * pitch + threadIdx.x], tile[(r ^ half) * pitch + threadIdx.x]); + } + } + __syncthreads(); + } + for (uint32_t r = threadIdx.y; r < rows; r += blockDim.y) + x[i0 + r * low] = tile[r * pitch + threadIdx.x]; +} + /// Pointwise multiply: x[i] *= w[i]. Used for coset scaling (w = g^i weights). extern "C" __global__ void pointwise_mul(uint64_t *x, const uint64_t *w, @@ -227,6 +451,74 @@ extern "C" __global__ void ntt_dit_level(uint64_t *x, /// before the first kernel launch). /// /// Assumes `n` is a multiple of 256, i.e. `log_n >= 8`. +/// Columns a tile spans: one warp, so the load and the store are coalesced. +#define NTT_TILE_COLS 32 + +/// `k_levels` DIT levels from `base`, fused through shared memory. +/// +/// The elements that interact across levels `base .. base + k_levels - 1` are +/// exactly those differing only in the middle `k_levels` bits of their index: +/// write `i = q·2^(base+k) + r·2^base + c` and the levels move `r` while `q` +/// and `c` stay put. So a block takes one `q`, a warp's worth of *consecutive* +/// `c`, and every `r`. +/// +/// That last part is the whole point. The older fused path at `base > 0` +/// gathered its tile along the stride — consecutive threads a group apart — +/// and lost more to uncoalesced traffic than it saved in passes. Here +/// `threadIdx.x` walks `c`, which is contiguous in memory, and the stride is +/// `threadIdx.y`, which is the dimension the butterflies work on. +/// +/// Requires `2^base >= NTT_TILE_COLS`; the low levels are the fused +/// `ntt_dit_8_levels`, which is already coalesced at `base = 0`. +extern "C" __global__ void ntt_dit_tile(uint64_t *x, + const uint64_t *tw, + uint64_t n, + uint64_t log_n, + uint64_t base, + uint32_t k_levels) { + // rows × (NTT_TILE_COLS + 1): the odd stride keeps a column off one bank. + extern __shared__ uint64_t tile[]; + const uint32_t cols = NTT_TILE_COLS; + const uint32_t rows = 1u << k_levels; + const uint32_t pitch = cols + 1; + + uint64_t low = (uint64_t)1 << base; + uint64_t c = (uint64_t)blockIdx.x * cols + threadIdx.x; + // The block's `q`: grid.z carries the multiples of grid.y that grid.y's + // 65,535 cannot (see `ntt::split_grid_y`). + uint64_t q = (uint64_t)blockIdx.z * gridDim.y + blockIdx.y; + uint64_t i = q * (low << k_levels) + (uint64_t)threadIdx.y * low + c; + + tile[threadIdx.y * pitch + threadIdx.x] = x[i]; + __syncthreads(); + + // Half the rows hold a butterfly's lower element at each level, the same + // way the 256-element kernel uses half its threads. + for (uint32_t l = 0; l < k_levels; ++l) { + if (threadIdx.y < (rows >> 1)) { + uint32_t half = 1u << l; + uint32_t grp = threadIdx.y >> l; + uint32_t pos = threadIdx.y & (half - 1); + uint32_t r0 = (grp << (l + 1)) + pos; + uint32_t r1 = r0 + half; + + // The twiddle index is the element's low `base + l` bits, which is + // `pos` in the middle bits and `c` in the low ones — the same + // `k << (log_n - level - 1)` the per-level kernel uses. + uint64_t k_index = (uint64_t)pos * low + c; + uint64_t w = tw[k_index << (log_n - base - l - 1)]; + + uint64_t u = tile[r0 * pitch + threadIdx.x]; + uint64_t v = mul(w, tile[r1 * pitch + threadIdx.x]); + tile[r0 * pitch + threadIdx.x] = add(u, v); + tile[r1 * pitch + threadIdx.x] = sub(u, v); + } + __syncthreads(); + } + + x[i] = tile[threadIdx.y * pitch + threadIdx.x]; +} + extern "C" __global__ void ntt_dit_8_levels(uint64_t *x, const uint64_t *tw, uint64_t n, diff --git a/crypto/math-cuda/kernels/ntt_cm.cu b/crypto/math-cuda/kernels/ntt_cm.cu new file mode 100644 index 000000000..bd18dfa38 --- /dev/null +++ b/crypto/math-cuda/kernels/ntt_cm.cu @@ -0,0 +1,545 @@ +// Column-major coset LDE engine: the pass kernels. +// +// WHY THIS EXISTS. The row-major LDE in `ntt.cu` runs one butterfly level per +// launch over the whole matrix, so an LDE of a 2^22 x 245 table makes ~33 +// whole-matrix DRAM passes at the DRAM roof. This engine transforms a few +// columns at a time — a chunk sized to stay resident in L2 — and runs 4..8 +// levels per launch out of registers and shared memory, so DRAM sees the trace +// about once and the LDE about once (the WHIR encoding's wide plan takes up to +// 10, see below). The driver is `src/lde_cm.rs`. +// +// THE PASS. A launch runs one PASS: levels [s, s + k) of a length-2^L radix-2 +// transform, over every column of a chunk (gridDim.y = column). Write an index +// as i = hi·2^(s+k) + mid·2^s + lo. For each (hi, lo) the pass is a length-2^k +// transform over `mid` together with the TWIST +// +// x[mid] *= ω_{2^(s+k)}^(lo · rev_k(mid)) +// +// applied BEFORE the transform for DIT and AFTER it for DIF. That is the +// four-step factorisation of levels [s, s + k): what is left inside the pass +// uses only the powers of ω_{2^k}, which fit a shared table of 2^(k-1) +// entries (128 for every k <= 8). The levels themselves are the textbook +// radix-2 ones: +// +// DIT (bit-reversed in, natural out; levels ascending): level u pairs i and +// i + 2^u (bit u of i clear): t = x[i + 2^u]·ω_{2^(u+1)}^(i mod 2^u), +// (x[i], x[i + 2^u]) <- (x[i] + t, x[i] - t). +// DIF (natural in, bit-reversed out; levels descending): (a, b) <- +// (a + b, (a - b)·ω_{2^(u+1)}^(i mod 2^u)). +// +// With the inverse root table either direction computes the inverse transform +// WITHOUT the 1/2^L factor, which is how the prover's weights expect it (they +// carry the 1/n). +// +// THE TILE. A block owns T consecutive `lo` (or, when s = 0, T consecutive +// `hi`) and all 2^k `mid`: a K x T tile. A thread holds E = 16 elements in +// registers: 4 of the mid bits vary over its registers, the other k - 4 bits +// are its group `g`, and threadIdx = g·T + t. Each ROUND runs up to four +// levels in registers; a pass of k <= 8 levels is one or two rounds with one +// shared-memory exchange between them, a WIDE pass of k = 9..12 levels three +// rounds with two. Global loads and stores walk `lo` across the lanes of a +// warp, so they are coalesced for any s >= 4 (a wide pass's T = 4096 >> k +// lanes make row segments of 8 words at k = 9 and 4 at k = 10: whole 32-byte +// sectors); at s = 0 the tile is staged through shared memory instead. +// +// ROOTS. Every root is a power of root32, the field's primitive 2^32-th root +// (`TWO_ADIC_PRIMITVE_ROOT_OF_UNITY`), or of its inverse: +// ω_{2^M}^x = root32^(x << (32 - M)), a 32-bit exponent that wraps exactly +// as the root's order does. `roots` is the 4 x 256 windowed table +// roots[w·256 + j] = root32^(j << 8w), so any power is at most three +// multiplications, and roots[768 + j] = ω_256^j is the radix table of a pass +// of k <= 8 levels. A wide pass builds its ω_{2^k}^j from the windows. +// +// FUSIONS (contiguous passes only, flags): +// F_SPREAD DIT s = 0: the input is COMPACT — n = 2^(L - lb) values `y`, and +// position B·q of the zero-padded transform input is y[q], every +// other position zero (the coset spread of the LDE). The zeros are +// never read or stored. +// F_GATHER with F_SPREAD: the compact input is in NATURAL order, so y[q] is +// read at rev_{L-lb}(q) and multiplied by wtab[q]. +// F_STORE_W s = 0: multiply the value at column position p by wtab[p] on the +// way out (the iNTT's last pass applies the bit-reversed coset +// weights here). +// F_MOBIUS with F_SPREAD: before the rounds, the encoding's Möbius levels +// over the block's compact values — all of them, the low +// KLOG + log T - lb bits of a value's index — one level a phase +// (`phase_contig_mobius`). The WHIR encoding runs its other levels +// first; the levels commute. +// +// HOST REPLAY. Every phase is a plain function of (tid, bx, col) so that +// `tests/host_kat/ntt_cm_host_kat.cpp` can compile this file through the CUDA +// host shim and replay a launch phase by phase, every thread of a phase before +// the next — exactly the ordering `__syncthreads` gives the device. The +// `__global__` wrappers are device-only. + +#include "goldilocks.cuh" + +namespace ntt_cm { + +using goldilocks::add; +using goldilocks::mul; +using goldilocks::sub; + +// Elements one thread holds in registers, and log2 of it. +constexpr int ELOG = 4; +constexpr int E = 1 << ELOG; + +// The radix table of a KLOG-level pass: rad[j] = ω_{2^(LOG+1)}^j for +// j < 2^LOG, the powers every level of the pass takes (level u reads entry +// xi << (LOG - u)). Every k <= 8 shares today's table, ω_256^j for j < 128. +template +struct Radix { + static constexpr int LOG = KLOG <= 8 ? 7 : KLOG - 1; + static constexpr int SIZE = 1 << LOG; +}; + +constexpr uint32_t F_SPREAD = 1u; +constexpr uint32_t F_GATHER = 2u; +constexpr uint32_t F_STORE_W = 4u; +constexpr uint32_t F_MOBIUS = 8u; + +// root32^e from the windowed table (at most three multiplications). +__device__ __forceinline__ uint64_t pow_root32(const uint64_t *roots, uint32_t e) { + uint64_t r = roots[e & 255u]; + const uint32_t e1 = (e >> 8) & 255u; + const uint32_t e2 = (e >> 16) & 255u; + const uint32_t e3 = e >> 24; + if (e1) r = mul(r, roots[256 + e1]); + if (e2) r = mul(r, roots[512 + e2]); + if (e3) r = mul(r, roots[768 + e3]); + return r; +} + +// Reverse the low `bits` bits of x (bits >= 1). +__device__ __forceinline__ uint64_t rev_bits(uint64_t x, uint32_t bits) { + return __brevll(x) >> (64 - bits); +} + +// Reverse a 4-bit register index — a compile-time constant in every use. +__device__ __forceinline__ constexpr int rev4(int v) { + return ((v & 1) << 3) | ((v & 2) << 1) | ((v & 4) >> 1) | ((v & 8) >> 3); +} + +// Round J of a KLOG-level pass: its levels [U0, U1] and the lowest of the four +// mid bits the registers vary over, A. Full rounds cover four levels; when k +// is not a multiple of four the short round keeps four varying bits and some +// of them are passive (the round's butterflies pair only along its levels). +// DIT: rounds ascend; the short round is the last and ends at bit k - 1. +// DIF: rounds descend; the short round is the last and starts at bit 0. +template +struct Round { + static constexpr int kDitU0 = 4 * J; + static constexpr int kDitU1 = (4 * J + 3 < KLOG - 1) ? 4 * J + 3 : KLOG - 1; + static constexpr int kDitA = (4 * J < KLOG - 4) ? 4 * J : KLOG - 4; + static constexpr int kDifU1 = KLOG - 1 - 4 * J; + static constexpr int kDifU0 = (kDifU1 - 3 > 0) ? kDifU1 - 3 : 0; + static constexpr int U0 = DIT ? kDitU0 : kDifU0; + static constexpr int U1 = DIT ? kDitU1 : kDifU1; + static constexpr int A = DIT ? kDitA : kDifU0; +}; + +// The mid index of register v of group g in a round whose varying bits are +// [A, A + 4): g fills the other k - 4 bits in order. +template +__device__ __forceinline__ uint32_t mid_base(uint32_t g) { + return ((g >> A) << (A + ELOG)) | (g & ((1u << A) - 1u)); +} + +// One level u of the in-register transform: pairs registers v and v | 2^(u-A). +template +__device__ __forceinline__ void level(uint64_t (&x)[E], uint32_t base, const uint64_t *rad) { + constexpr int O = U - A; +#pragma unroll + for (int v = 0; v < E; ++v) { + if (v & (1 << O)) continue; + const int w = v | (1 << O); + uint64_t tw = 1; + if constexpr (U > 0) { + const uint32_t xi = (base | ((uint32_t)v << A)) & ((1u << U) - 1u); + tw = rad[xi << (RLOG - U)]; + } + if constexpr (DIT) { + const uint64_t t = (U > 0) ? mul(x[w], tw) : x[w]; + x[w] = sub(x[v], t); + x[v] = add(x[v], t); + } else { + const uint64_t a = x[v]; + const uint64_t b = x[w]; + x[v] = add(a, b); + x[w] = (U > 0) ? mul(sub(a, b), tw) : sub(a, b); + } + } +} + +// Levels U .. UEND of a round, ascending (DIT) or descending (DIF). +template +__device__ __forceinline__ void levels(uint64_t (&x)[E], uint32_t base, const uint64_t *rad) { + level(x, base, rad); + if constexpr (U != UEND) { + constexpr int NEXT = DIT ? U + 1 : U - 1; + levels(x, base, rad); + } +} + +template +__device__ __forceinline__ void run_round(uint64_t (&x)[E], uint32_t g, const uint64_t *rad) { + using R = Round; + constexpr int RLOG = Radix::LOG; + static_assert(KLOG >= ELOG && KLOG <= 12, "a pass runs 4..12 levels"); + static_assert(J >= 0 && J < (KLOG + 3) / 4, "round index out of range"); + static_assert(R::U0 <= R::U1 && R::A >= 0 && R::A <= R::U0 && R::U1 < R::A + ELOG, + "a round's levels lie inside its four varying bits"); + const uint32_t base = mid_base(g); + if constexpr (DIT) { + levels(x, base, rad); + } else { + levels(x, base, rad); + } +} + +// Tile row pitch: T + 1 keeps a column of the tile off a single bank. +__device__ __forceinline__ uint32_t pitch_of(uint32_t log_t) { return (1u << log_t) + 1u; } + +// The tile slot of row `mid`, lane t. A wide pass (k > 8) spans T <= 8 lanes, +// so a half-warp holds 16 / T groups, and in the first round's layout (A = 0) +// their rows lie 16 apart: at pitch T + 1 alone they would share banks. A skew +// of T words every 16 rows puts them on distinct banks; the staged copies (16 +// consecutive rows, one lane) keep distinct banks through the odd pitch. +// k <= 8 keeps today's layout. +template +__device__ __forceinline__ uint32_t tile_at(uint32_t mid, uint32_t log_t, uint32_t t) { + uint32_t i = mid * pitch_of(log_t) + t; + if constexpr (KLOG > 8) i += (mid >> ELOG) << log_t; + return i; +} + +// Words the tile spans (the dynamic shared memory, mirrored by the driver's +// `tile_bytes`). +template +__device__ __forceinline__ uint32_t tile_words(uint32_t log_t) { + uint32_t w = (1u << KLOG) * pitch_of(log_t); + if constexpr (KLOG > 8) w += (1u << (KLOG - ELOG)) << log_t; + return w; +} + +template +__device__ __forceinline__ void tile_write(const uint64_t (&x)[E], uint64_t *tile, uint32_t log_t, + uint32_t t, uint32_t g) { + const uint32_t base = mid_base(g); +#pragma unroll + for (int v = 0; v < E; ++v) tile[tile_at(base | ((uint32_t)v << A), log_t, t)] = x[v]; +} + +template +__device__ __forceinline__ void tile_read(uint64_t (&x)[E], const uint64_t *tile, uint32_t log_t, + uint32_t t, uint32_t g) { + const uint32_t base = mid_base(g); +#pragma unroll + for (int v = 0; v < E; ++v) x[v] = tile[tile_at(base | ((uint32_t)v << A), log_t, t)]; +} + +// The twist of a strided pass, in the A = 0 register layout that both the +// first DIT round and the last DIF round use: mid = (g << 4) | v, so +// rev_k(mid) = rev4(v)·2^(k-4) + rev_{k-4}(g) and +// ω_{2^(s+k)}^(lo·rev_k(mid)) = c · d^rev4(v), +// c = ω_{2^(s+k)}^(lo·rev_{k-4}(g)), d = ω_{2^(s+4)}^lo. +template +__device__ __forceinline__ void twist(uint64_t (&x)[E], const uint64_t *roots, uint32_t s, + uint32_t lo, uint32_t g) { + constexpr int GBITS = KLOG - ELOG; + uint32_t rg = 0; + if constexpr (GBITS > 0) rg = (uint32_t)rev_bits(g, GBITS); + const uint64_t c = pow_root32(roots, (lo * rg) << (32 - s - KLOG)); + const uint64_t d = pow_root32(roots, lo << (32 - s - ELOG)); + uint64_t f = c; +#pragma unroll + for (int j = 0; j < E; ++j) { + x[rev4(j)] = mul(x[rev4(j)], f); + if (j + 1 < E) f = mul(f, d); + } +} + +// Threads in a block: T lanes x K/16 groups. +template +__device__ __forceinline__ uint32_t threads_of(uint32_t log_t) { + return (1u << log_t) << (KLOG - ELOG); +} + +// Load the pass's radix table into shared memory: today's ω_256^j straight +// from the top window, a wide pass's ω_{2^k}^j = root32^(j << (32 - k)) from +// the windows (at most two multiplications an entry). +template +__device__ __forceinline__ void phase_rad(uint64_t *rad, const uint64_t *roots, uint32_t tid, + uint32_t nthreads) { + using RX = Radix; + for (uint32_t j = tid; j < (uint32_t)RX::SIZE; j += nthreads) { + if constexpr (RX::LOG == 7) { + rad[j] = roots[768 + j]; + } else { + rad[j] = pow_root32(roots, j << (31 - RX::LOG)); + } + } +} + +// --------------------------------------------------------------------------- +// Strided pass (s >= log T >= 0 with s > 0): a block is one hi and T +// consecutive lo. Element (t, mid) of the tile is column index +// hi·2^(s+k) + mid·2^s + lo0 + t. +// --------------------------------------------------------------------------- + +__device__ __forceinline__ uint64_t strided_tile_base(uint32_t bx, uint32_t s, uint32_t klog, + uint32_t log_t, uint32_t t) { + const uint32_t lo_blocks_log = s - log_t; + const uint64_t hi = (uint64_t)(bx >> lo_blocks_log); + const uint64_t lo0 = (uint64_t)(bx & ((1u << lo_blocks_log) - 1u)) << log_t; + return (hi << (s + klog)) + lo0 + t; +} + +// First (or only) phase of a strided pass: load in the first round's layout, +// DIT twist, first round; then either the exchange write (two or three +// rounds) or the DIF twist and the store (one round). +template +__device__ __forceinline__ void phase_strided_first( + const uint64_t *in, uint64_t in_stride, uint64_t *out, uint64_t out_stride, + const uint64_t *roots, const uint64_t *rad, uint64_t *tile, uint32_t L, uint32_t s, + uint32_t log_t, uint32_t tid, uint32_t bx, uint32_t col) { + constexpr int ROUNDS = (KLOG + 3) / 4; + using R0 = Round; + const uint32_t t = tid & ((1u << log_t) - 1u); + const uint32_t g = tid >> log_t; + const uint64_t tb = strided_tile_base(bx, s, KLOG, log_t, t); + const uint32_t lo = (uint32_t)(tb & ((1ull << s) - 1ull)); + const uint64_t *src = in + (uint64_t)col * in_stride; + + uint64_t x[E]; + { + const uint32_t base = mid_base(g); +#pragma unroll + for (int v = 0; v < E; ++v) x[v] = src[tb + ((uint64_t)(base | ((uint32_t)v << R0::A)) << s)]; + } + if constexpr (DIT) twist(x, roots, s, lo, g); + run_round(x, g, rad); + if constexpr (ROUNDS >= 2) { + tile_write(x, tile, log_t, t, g); + } else { + if constexpr (!DIT) twist(x, roots, s, lo, g); + uint64_t *dst = out + (uint64_t)col * out_stride; + const uint32_t base = mid_base(g); +#pragma unroll + for (int v = 0; v < E; ++v) dst[tb + ((uint64_t)(base | ((uint32_t)v << R0::A)) << s)] = x[v]; + } +} + +// Last phase of a strided pass of two or three rounds: exchange read, last +// round, DIF twist, store. (A three-round pass runs its middle round tile to +// tile between the two, `phase_tile_round`.) +template +__device__ __forceinline__ void phase_strided_last( + uint64_t *out, uint64_t out_stride, const uint64_t *roots, const uint64_t *rad, + const uint64_t *tile, uint32_t L, uint32_t s, uint32_t log_t, uint32_t tid, uint32_t bx, + uint32_t col) { + constexpr int LAST = (KLOG + 3) / 4 - 1; + using RL = Round; + const uint32_t t = tid & ((1u << log_t) - 1u); + const uint32_t g = tid >> log_t; + const uint64_t tb = strided_tile_base(bx, s, KLOG, log_t, t); + const uint32_t lo = (uint32_t)(tb & ((1ull << s) - 1ull)); + + uint64_t x[E]; + tile_read(x, tile, log_t, t, g); + run_round(x, g, rad); + if constexpr (!DIT) twist(x, roots, s, lo, g); + uint64_t *dst = out + (uint64_t)col * out_stride; + const uint32_t base = mid_base(g); +#pragma unroll + for (int v = 0; v < E; ++v) dst[tb + ((uint64_t)(base | ((uint32_t)v << RL::A)) << s)] = x[v]; +} + +// --------------------------------------------------------------------------- +// Contiguous pass (s = 0): a block is T consecutive sub-transforms of K +// consecutive elements, hi0 = bx·T. The tile is staged through shared memory +// so that global accesses walk consecutive addresses. +// --------------------------------------------------------------------------- + +// Stage the block's T·K elements into the tile. +template +__device__ __forceinline__ void phase_contig_stage(const uint64_t *in, uint64_t in_stride, + const uint64_t *wtab, uint64_t *tile, + uint32_t L, uint32_t lb, uint32_t log_t, + uint32_t flags, uint32_t tid, uint32_t bx, + uint32_t col) { + constexpr uint32_t K = 1u << KLOG; + const uint32_t nthreads = threads_of(log_t); + const uint32_t count = K << log_t; + const uint64_t *src = in + (uint64_t)col * in_stride; + const uint64_t block0 = (uint64_t)bx << (KLOG + log_t); + if (flags & F_SPREAD) { + // Compact input: the block covers compact positions + // [block0 >> lb, (block0 + count) >> lb); position B·q of the + // transform input is y[q], the rest is zero. + const uint32_t bmask = (1u << lb) - 1u; + const uint32_t log_n = L - lb; + for (uint32_t e = tid; e < count; e += nthreads) { + const uint32_t t = e >> KLOG; + const uint32_t mid = e & (K - 1u); + uint64_t v = 0; + if ((mid & bmask) == 0) { + const uint64_t q = (block0 + e) >> lb; + if (flags & F_GATHER) { + v = mul(src[rev_bits(q, log_n)], wtab[q]); + } else { + v = src[q]; + } + } + tile[tile_at(mid, log_t, t)] = v; + } + } else { + for (uint32_t e = tid; e < count; e += nthreads) { + const uint32_t t = e >> KLOG; + const uint32_t mid = e & (K - 1u); + tile[tile_at(mid, log_t, t)] = src[block0 + e]; + } + } +} + +// One Möbius level of the block's compact values (F_MOBIUS): the value at +// index j, staged at position B·j, loses its partner below when bit `level` +// of j is set — `mobius_wide_low`'s operation, x[j] -= x[j ^ 2^level]. Each +// pair belongs to one thread, so a level needs no barrier inside it. +template +__device__ __forceinline__ void phase_contig_mobius(uint64_t *tile, uint32_t log_t, uint32_t lb, + uint32_t level, uint32_t tid) { + constexpr uint32_t K = 1u << KLOG; + const uint32_t nthreads = threads_of(log_t); + const uint32_t pairs = 1u << (KLOG + log_t - lb - 1); + const uint32_t h = 1u << level; + for (uint32_t u = tid; u < pairs; u += nthreads) { + const uint32_t j1 = ((u >> level) << (level + 1)) | h | (u & (h - 1u)); + const uint32_t e1 = j1 << lb, e0 = (j1 ^ h) << lb; + const uint32_t i1 = tile_at(e1 & (K - 1u), log_t, e1 >> KLOG); + tile[i1] = sub(tile[i1], tile[tile_at(e0 & (K - 1u), log_t, e0 >> KLOG)]); + } +} + +// Round J tile to tile — every round of a contiguous pass, and the middle +// round of a three-round strided one: each thread reads and writes back +// exactly the tile slots of its own group, so no barrier is needed between +// its read and its write. +template +__device__ __forceinline__ void phase_tile_round(uint64_t *tile, const uint64_t *rad, + uint32_t log_t, uint32_t tid) { + using R = Round; + const uint32_t t = tid & ((1u << log_t) - 1u); + const uint32_t g = tid >> log_t; + uint64_t x[E]; + tile_read(x, tile, log_t, t, g); + run_round(x, g, rad); + tile_write(x, tile, log_t, t, g); +} + +// Store the block's T·K elements from the tile, optionally weighted. +template +__device__ __forceinline__ void phase_contig_store(uint64_t *out, uint64_t out_stride, + const uint64_t *wtab, const uint64_t *tile, + uint32_t log_t, uint32_t flags, uint32_t tid, + uint32_t bx, uint32_t col) { + constexpr uint32_t K = 1u << KLOG; + const uint32_t nthreads = threads_of(log_t); + const uint32_t count = K << log_t; + uint64_t *dst = out + (uint64_t)col * out_stride; + const uint64_t block0 = (uint64_t)bx << (KLOG + log_t); + for (uint32_t e = tid; e < count; e += nthreads) { + const uint32_t t = e >> KLOG; + const uint32_t mid = e & (K - 1u); + uint64_t v = tile[tile_at(mid, log_t, t)]; + if (flags & F_STORE_W) v = mul(v, wtab[block0 + e]); + dst[block0 + e] = v; + } +} + +} // namespace ntt_cm + +#if defined(__CUDACC__) + +// One pass. `s == 0` selects the contiguous (staged) form. The dynamic shared +// memory is the tile (`tile_words`); the radix table is static. +template +__device__ __forceinline__ void ntt_cm_pass(const uint64_t *in, uint64_t in_stride, uint64_t *out, + uint64_t out_stride, const uint64_t *roots, + const uint64_t *wtab, uint32_t L, uint32_t s, + uint32_t log_t, uint32_t lb, uint32_t flags) { + using namespace ntt_cm; + extern __shared__ uint64_t ntt_cm_tile[]; + __shared__ uint64_t rad[Radix::SIZE]; + constexpr int ROUNDS = (KLOG + 3) / 4; + const uint32_t tid = threadIdx.x; + const uint32_t bx = blockIdx.x; + const uint32_t col = blockIdx.y; + const uint32_t nthreads = threads_of(log_t); + + phase_rad(rad, roots, tid, nthreads); + if (s > 0) { + __syncthreads(); + phase_strided_first(in, in_stride, out, out_stride, roots, rad, ntt_cm_tile, L, s, + log_t, tid, bx, col); + if constexpr (ROUNDS == 3) { + __syncthreads(); + phase_tile_round(ntt_cm_tile, rad, log_t, tid); + } + if constexpr (ROUNDS >= 2) { + __syncthreads(); + phase_strided_last(out, out_stride, roots, rad, ntt_cm_tile, L, s, log_t, + tid, bx, col); + } + } else { + phase_contig_stage(in, in_stride, wtab, ntt_cm_tile, L, lb, log_t, flags, tid, bx, col); + __syncthreads(); + if (flags & F_MOBIUS) { + for (uint32_t l = 0; l < KLOG + log_t - lb; ++l) { + phase_contig_mobius(ntt_cm_tile, log_t, lb, l, tid); + __syncthreads(); + } + } + phase_tile_round(ntt_cm_tile, rad, log_t, tid); + if constexpr (ROUNDS >= 2) { + __syncthreads(); + phase_tile_round(ntt_cm_tile, rad, log_t, tid); + } + if constexpr (ROUNDS == 3) { + __syncthreads(); + phase_tile_round(ntt_cm_tile, rad, log_t, tid); + } + __syncthreads(); + phase_contig_store(out, out_stride, wtab, ntt_cm_tile, log_t, flags, tid, bx, col); + } +} + +// The driver never launches more than 256 threads (T lanes x K/16 groups is +// capped at 4096/16); two resident blocks per SM bound a thread at 128 +// registers, which holds the 16 elements and the round's temporaries. The +// wide passes are DIT only: they serve the WHIR encoding's forward transform +// (`lde_cm::plan_wide`), and at k = 10 the tile and the radix table take +// 46 KiB of the 48 a block has without an opt-in. +#define NTT_CM_KERNEL(NAME, KLOG, DIT) \ + extern "C" __global__ void __launch_bounds__(256, 2) \ + NAME(const uint64_t *in, uint64_t in_stride, uint64_t *out, uint64_t out_stride, \ + const uint64_t *roots, const uint64_t *wtab, uint32_t L, uint32_t s, \ + uint32_t log_t, uint32_t lb, uint32_t flags) { \ + ntt_cm_pass(in, in_stride, out, out_stride, roots, wtab, L, s, log_t, lb, \ + flags); \ + } + +NTT_CM_KERNEL(ntt_cm_dit_k4, 4, true) +NTT_CM_KERNEL(ntt_cm_dit_k5, 5, true) +NTT_CM_KERNEL(ntt_cm_dit_k6, 6, true) +NTT_CM_KERNEL(ntt_cm_dit_k7, 7, true) +NTT_CM_KERNEL(ntt_cm_dit_k8, 8, true) +NTT_CM_KERNEL(ntt_cm_dit_k9, 9, true) +NTT_CM_KERNEL(ntt_cm_dit_k10, 10, true) +NTT_CM_KERNEL(ntt_cm_dif_k4, 4, false) +NTT_CM_KERNEL(ntt_cm_dif_k5, 5, false) +NTT_CM_KERNEL(ntt_cm_dif_k6, 6, false) +NTT_CM_KERNEL(ntt_cm_dif_k7, 7, false) +NTT_CM_KERNEL(ntt_cm_dif_k8, 8, false) + +#endif // __CUDACC__ diff --git a/crypto/math-cuda/kernels/p1w16.cu b/crypto/math-cuda/kernels/p1w16.cu new file mode 100644 index 000000000..f351322dc --- /dev/null +++ b/crypto/math-cuda/kernels/p1w16.cu @@ -0,0 +1,1365 @@ +// Poseidon1 over Goldilocks at width 16 on device — the D-HASH stage-1 +// MEASUREMENT kernels: the permutation, the rate-12 leaf sponge over WHIR fold +// cosets, the 4-ary Merkle node, and the grind search. Nothing on a proving +// path loads this module (`src/p1w16.rs`). +// +// CLEAN-ROOM. Written from the Poseidon paper (eprint 2019/458) and this +// repository's host reference `crypto/crypto/src/hash/poseidon1_w16/mod.rs`, +// with the field arithmetic of `goldilocks.cuh` and the limb multiply of our +// own `rpx.cu`. No third-party GPU Poseidon code was read (pil2-stark's is +// AGPL-3.0). +// +// THE ORACLE is the host reference, lane for lane: +// permute 4 full rounds, 22 partial (S-box on lane 0), 4 full; each +// round adds RC[r], applies x^7, then state <- M·state with the +// circulant M[i][j] = MDS_ROW[(j − i) mod 16]; +// sponge leaf rate 12 overwrite duplex; capacity lane 12 = felts mod 12, +// lane 13 = DOMAIN_LEAF ("P1WL"), lanes 14, 15 = 0; the tail +// block zero-padded; an empty leaf never permutes; +// 4-ary node the four child digests fill all 16 lanes, permute, keep 0..4; +// grind head lane 0 of sponge_leaf([inner0..3, nonce]) — five felts, one +// permutation — the RPX grind's construction (`rpx.cu`) with +// this sponge. +// `tests/host_kat/p1w16_host_kat.cpp` checks all four on the host against the +// vectors the Python reference, Plonky3's generator and Plonky3's Rust +// permutation agree on; `tests/p1w16_device.rs` checks the device against the +// Rust host reference. +// +// DIGESTS are four canonical u64s (32 bytes, native order) — a measurement +// format, not the RPX kernels' big-endian commitment bytes. +// +// COST MODEL (one permutation): 150 S-boxes × 4 multiplications = 600 field +// multiplications; 30 MDS products of 256 small-constant 32×32→64 MACs on each +// 32-bit half (15,360 MACs) plus 16 reductions. Against RPX's 2,736 field +// multiplications + 2,016 MACs. +// +// VARIANTS (template `V`; every entry below is instantiated so one job +// measures them all): +// V = 0 every product through `goldilocks::mul`; +// V = 1 products and squares through the 32-bit-limb multiply (`rpx.cu`'s +// `RPX_V_LIMB_MUL | RPX_V_LIMB_SQR`, reproduced below); +// V = 2 V = 1's multiply with the partial rounds in the Fourier domain of +// the circulant MDS (`permute_fourier`); +// V = 3 the Grain Cauchy MDS alternative (`permute_cauchy`: dense full +// rounds, the paper's sparse partial rounds, V = 1's multiply); +// V = 4 V = 2 with leaner addition chains (`permute_fourier_lean`); the +// production kernels' `_lean` twins run it (`LAMBDA_VM_P1_LEAN`, on by default). +// Every variant computes the same field values and `permute` canonicalises. +// +// ZISK'S INSTANCE (stage P1 of the Poseidon1 base STARK, at the end of the +// file): the same permutation with ZisK's leaf hash (`zisk_leaf`: zero +// capacity, the previous digest carried into it) over cosets and the STARK's +// LDE rows, and ZisK's width-8 grinding permutation (`p1w8`, its own Fourier +// form). Oracle: `crypto::hash::poseidon1_stark` / `poseidon1_w8`, whose +// vectors come from ZisK's own code (`tests/host_kat/p1_zisk_kat_vectors.h`). + +#include +#include "goldilocks.cuh" +#include "p1w16_constants.cuh" + +namespace p1w16 { + +enum : int { + WIDTH = 16, + RATE = 12, + DIGEST = 4, + HALF_FULL = 4, + PARTIAL = 22, + ROUNDS = 30, + CAP_PAD_LANE = RATE + 0, + CAP_DOMAIN_LANE = RATE + 1, +}; + +// `u32::from_le_bytes(*b"P1WL")`. +__device__ constexpr uint64_t DOMAIN_LEAF = 0x4C573150ull; +// Five felts: the inner hash's four and the nonce. +__device__ constexpr uint64_t GRIND_FELTS = 5; + +// --------------------------------------------------------------------------- +// The 32-bit-limb multiply and square — `rpx.cu`'s `reduce_limbs`, `mul_limb`, +// `sqr_limb` (see the derivation there), reproduced so this module has no +// dependency on the RPX kernel file. +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint64_t reduce_limbs(uint32_t t0, uint32_t t1, uint32_t t2, + uint32_t t3) { +#if defined(__CUDA_ARCH__) + const uint32_t w = 0xFFFFFFFFu; + asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.cc.u32 %1, %2, %3, %1; addc.u32 %2, 0, 0;" + : "+r"(t0), "+r"(t1), "+r"(t2) + : "r"(w)); + asm("sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, 0; subc.u32 %2, %2, 0;" + : "+r"(t0), "+r"(t1), "+r"(t2) + : "r"(t3)); + const uint32_t hi_sub = (t2 == 1u) ? 0xFFFFFFFFu : 0u; + asm("sub.cc.u32 %0, %0, %2; subc.u32 %1, %1, %3;" : "+r"(t0), "+r"(t1) : "r"(t2), "r"(hi_sub)); + return ((uint64_t)t1 << 32) | t0; +#else + const uint64_t s = ((uint64_t)t1 << 32) | t0; + const uint64_t s1 = s + (uint64_t)t2 * 0xFFFFFFFFull; + int c = (s1 < s) ? 1 : 0; + const uint64_t s2 = s1 - t3; + c -= (s1 < t3) ? 1 : 0; + if (c == 1) return s2 + 0xFFFFFFFFull; + if (c == -1) return s2 - 0xFFFFFFFFull; + return s2; +#endif +} + +__device__ __forceinline__ uint64_t mul_limb(uint64_t a, uint64_t b) { + const uint32_t a0 = (uint32_t)a, a1 = (uint32_t)(a >> 32); + const uint32_t b0 = (uint32_t)b, b1 = (uint32_t)(b >> 32); +#if defined(__CUDA_ARCH__) + uint32_t t0, t1, t2, t3, c; + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t0), "=r"(t1) : "r"(a0), "r"(b0)); + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t2), "=r"(t3) : "r"(a1), "r"(b1)); + asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;" + : "+r"(t1), "+r"(t2), "=r"(c) + : "r"(a0), "r"(b1)); + asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, %2, %5;" + : "+r"(t1), "+r"(t2), "+r"(t3) + : "r"(a1), "r"(b0), "r"(c)); + return reduce_limbs(t0, t1, t2, t3); +#else + const uint64_t p00 = (uint64_t)a0 * b0, p01 = (uint64_t)a0 * b1; + const uint64_t p10 = (uint64_t)a1 * b0, p11 = (uint64_t)a1 * b1; + const uint64_t mid = (p00 >> 32) + (uint32_t)p01 + (uint32_t)p10; + const uint64_t top = p11 + (p01 >> 32) + (p10 >> 32) + (mid >> 32); + return reduce_limbs((uint32_t)p00, (uint32_t)mid, (uint32_t)top, (uint32_t)(top >> 32)); +#endif +} + +__device__ __forceinline__ uint64_t sqr_limb(uint64_t a) { + const uint32_t a0 = (uint32_t)a, a1 = (uint32_t)(a >> 32); +#if defined(__CUDA_ARCH__) + uint32_t t0, t1, t2, t3, x0, x1; + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t0), "=r"(t1) : "r"(a0), "r"(a0)); + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t2), "=r"(t3) : "r"(a1), "r"(a1)); + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(x0), "=r"(x1) : "r"(a0), "r"(a1)); + asm("add.cc.u32 %0, %0, %3; addc.cc.u32 %1, %1, %4; addc.u32 %2, %2, 0;" + : "+r"(t1), "+r"(t2), "+r"(t3) + : "r"(x0), "r"(x1)); + asm("add.cc.u32 %0, %0, %3; addc.cc.u32 %1, %1, %4; addc.u32 %2, %2, 0;" + : "+r"(t1), "+r"(t2), "+r"(t3) + : "r"(x0), "r"(x1)); + return reduce_limbs(t0, t1, t2, t3); +#else + const uint64_t p00 = (uint64_t)a0 * a0, x = (uint64_t)a0 * a1, p11 = (uint64_t)a1 * a1; + const uint64_t mid = (p00 >> 32) + 2 * (uint64_t)(uint32_t)x; + const uint64_t top = p11 + 2 * (x >> 32) + (mid >> 32); + return reduce_limbs((uint32_t)p00, (uint32_t)mid, (uint32_t)top, (uint32_t)(top >> 32)); +#endif +} + +template +__device__ __forceinline__ uint64_t fmul(uint64_t a, uint64_t b) { + if constexpr (V >= 1) { + return mul_limb(a, b); + } else { + return goldilocks::mul(a, b); + } +} + +template +__device__ __forceinline__ uint64_t fsqr(uint64_t a) { + if constexpr (V >= 1) { + return sqr_limb(a); + } else { + return goldilocks::mul(a, a); + } +} + +// x^7 as x², x³ = x²·x, x⁶ = (x³)², x⁷ = x⁶·x — the host's association. +template +__device__ __forceinline__ uint64_t sbox(uint64_t x) { + const uint64_t x2 = fsqr(x); + const uint64_t x3 = fmul(x2, x); + const uint64_t x6 = fsqr(x3); + return fmul(x6, x); +} + +// The circulant MDS, fully unrolled over compile-time entries: each lane is +// Σ_j c·lo_j and Σ_j c·hi_j over the 32-bit halves (each < 371·2^32 < 2^41), +// recombined to `hi·2^64 + lo` with `hi < 2^10` and folded by +// `2^64 ≡ EPSILON` — `rpx.cu`'s `mds` at width 16. Inputs may be raw +// `[0, 2^64)` storage; the output is congruent, not canonical. +__device__ __forceinline__ void mds(uint64_t s[WIDTH]) { + uint32_t lo32[WIDTH], hi32[WIDTH]; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + lo32[j] = (uint32_t)s[j]; + hi32[j] = (uint32_t)(s[j] >> 32); + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) { + uint64_t acc_lo = 0, acc_hi = 0; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + const uint64_t c = MDS_ROW[(j + WIDTH - i) % WIDTH]; + acc_lo += c * (uint64_t)lo32[j]; + acc_hi += c * (uint64_t)hi32[j]; + } + const uint64_t lo = (acc_hi << 32) + acc_lo; + const uint64_t carry = (lo < acc_lo) ? 1ull : 0ull; + const uint64_t hi = (acc_hi >> 32) + carry; + s[i] = goldilocks::add(lo, hi * goldilocks::EPSILON); + } +} + +// One full round: constants, the S-box on every lane, the circulant MDS. +template +__device__ __forceinline__ void full_round(uint64_t s[WIDTH], int r) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = sbox(goldilocks::add(s[i], RC[r][i])); + mds(s); +} + +// The 16-point DFT in registers, radix-2 decimation in time over a +// bit-reversed copy: X[k] = sum_n x[n]·omega^(±nk), omega = 2^12 (order 16). +// `INV` takes omega^-1 and does NOT scale by 1/16 (the caller folds it into +// the last partial round's eigenvalues). Every twiddle index is a +// compile-time constant after unrolling, so each product is by an immediate. +template +__device__ __forceinline__ void dft16(uint64_t x[WIDTH]) { + constexpr int BITREV[WIDTH] = {0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15}; + uint64_t y[WIDTH]; +#pragma unroll + for (int i = 0; i < WIDTH; ++i) y[i] = x[BITREV[i]]; +#pragma unroll + for (int h = 1; h < WIDTH; h <<= 1) { +#pragma unroll + for (int start = 0; start < WIDTH; start += 2 * h) { +#pragma unroll + for (int j = 0; j < h; ++j) { + const int e = j * (8 / h); // omega_{2h}^j = omega^(j·16/(2h)) + const int ei = INV ? ((WIDTH - e) & (WIDTH - 1)) : e; + const uint64_t a = y[start + j]; + const uint64_t t = (e == 0) ? y[start + j + h] : fmul(y[start + j + h], OMEGA_POW[ei]); + y[start + j] = goldilocks::add(a, t); + y[start + j + h] = goldilocks::sub(a, t); + } + } + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) x[i] = y[i]; +} + +// ★ The permutation with its 22 partial rounds in the Fourier domain of the +// circulant MDS (`scripts/poseidon1/p1_fourier.py`, checked equal to the +// textbook form there): the last initial full round's MDS becomes D·F, each +// partial round is s0 = (1/16)·Σŝ, δ = (s0 + c0)^7 − (s0 + c0), +// ŝ_j ← d_j·(ŝ_j + ĉ_j + δ), and one inverse DFT precedes the terminal full +// rounds. 21 products a partial round instead of a dense MDS. +template +__device__ __forceinline__ void permute_fourier(uint64_t s[WIDTH]) { +#pragma unroll 1 + for (int r = 0; r < HALF_FULL - 1; ++r) full_round(s, r); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = sbox(goldilocks::add(s[i], RC[HALF_FULL - 1][i])); + dft16(s); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = fmul(s[i], FD[i]); +#pragma unroll 1 + for (int k = 0; k < PARTIAL; ++k) { + uint64_t sum = s[0]; +#pragma unroll + for (int i = 1; i < WIDTH; ++i) sum = goldilocks::add(sum, s[i]); + const uint64_t a0 = goldilocks::add(fmul(sum, INV16), RC[HALF_FULL + k][0]); + const uint64_t delta = goldilocks::sub(sbox(a0), a0); + if (k + 1 < PARTIAL) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) + s[i] = fmul(goldilocks::add(goldilocks::add(s[i], FC[k][i]), delta), FD[i]); + } else { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) + s[i] = fmul(goldilocks::add(goldilocks::add(s[i], FC[k][i]), delta), FD_LAST[i]); + } + } + dft16(s); +#pragma unroll 1 + for (int r = HALF_FULL + PARTIAL; r < ROUNDS; ++r) full_round(s, r); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = goldilocks::canonical(s[i]); +} + +// --------------------------------------------------------------------------- +// ★ LEAN (V = 4): `permute_fourier`'s field values by +// fewer instructions. Every word stays in [0, 2^64) and congruent to V = 2's +// mod p, and the last step canonicalises, so the output is V = 2's bit for +// bit. Three cuts, each on an addition chain (the field's adds, compares and +// selects are ≈ 64 % of V = 2's instructions, its multiplies ≈ 24 %): +// add_c a + c with c ≤ p (a canonical constant, or a small bound): ONE +// carry fix. On overflow the wrapped sum is below c ≤ p, so adding +// EPSILON cannot overflow again (`goldilocks::add` checks twice). +// add3 a + c + d, the carries counted and folded once (≤ 2·EPSILON). +// sum16 the partial round's 16-lane sum, carries counted, folded once. +// div16 x/16 mod p by a shift: p ≡ 1 (mod 16), so x + k·p with +// k = −x mod 16 is a multiple of 16 below 2^68. +// The multiplies (`mul_limb`, `sqr_limb`) take any word in [0, 2^64), so a +// sum need not be canonical before one. +// --------------------------------------------------------------------------- + +__device__ __forceinline__ uint64_t add_c(uint64_t a, uint64_t c) { + const uint64_t s = a + c; + return s + ((s < a) ? goldilocks::EPSILON : 0ull); +} + +__device__ __forceinline__ uint64_t add3(uint64_t a, uint64_t c, uint64_t d) { + const uint64_t t = a + c; + const uint64_t u = t + d; + const uint64_t carries = (uint64_t)(t < a) + (uint64_t)(u < t); + return add_c(u, carries * goldilocks::EPSILON); +} + +__device__ __forceinline__ uint64_t sum16(const uint64_t s[WIDTH]) { + uint64_t lo = s[0], carries = 0; +#pragma unroll + for (int i = 1; i < WIDTH; ++i) { + lo += s[i]; + carries += (lo < s[i]) ? 1ull : 0ull; + } + return add_c(lo, carries * goldilocks::EPSILON); +} + +__device__ __forceinline__ uint64_t div16(uint64_t x) { + const uint64_t k = (0ull - x) & 15ull; + // x + k·p = x + k − k·2^32 + k·2^64, as a 128-bit (hi, lo). + uint64_t lo = x + k; + uint64_t hi = k + ((lo < x) ? 1ull : 0ull); + const uint64_t sub = k << 32; + hi -= (lo < sub) ? 1ull : 0ull; + lo -= sub; + return (lo >> 4) | (hi << 60); +} + +// `mds` with its recombination folded by `add_c` (`hi · EPSILON < 2^42 ≤ p`). +__device__ __forceinline__ void mds_lean(uint64_t s[WIDTH]) { + uint32_t lo32[WIDTH], hi32[WIDTH]; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + lo32[j] = (uint32_t)s[j]; + hi32[j] = (uint32_t)(s[j] >> 32); + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) { + uint64_t acc_lo = 0, acc_hi = 0; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + const uint64_t c = MDS_ROW[(j + WIDTH - i) % WIDTH]; + acc_lo += c * (uint64_t)lo32[j]; + acc_hi += c * (uint64_t)hi32[j]; + } + const uint64_t lo = (acc_hi << 32) + acc_lo; + const uint64_t carry = (lo < acc_lo) ? 1ull : 0ull; + const uint64_t hi = (acc_hi >> 32) + carry; + s[i] = add_c(lo, hi * goldilocks::EPSILON); + } +} + +__device__ __forceinline__ void full_round_lean(uint64_t s[WIDTH], int r) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = sbox<4>(add_c(s[i], RC[r][i])); + mds_lean(s); +} + +__device__ __forceinline__ void permute_fourier_lean(uint64_t s[WIDTH]) { +#pragma unroll 1 + for (int r = 0; r < HALF_FULL - 1; ++r) full_round_lean(s, r); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = sbox<4>(add_c(s[i], RC[HALF_FULL - 1][i])); + dft16<4, false>(s); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = fmul<4>(s[i], FD[i]); +#pragma unroll 1 + for (int k = 0; k < PARTIAL; ++k) { + const uint64_t a0 = add_c(div16(sum16(s)), RC[HALF_FULL + k][0]); + const uint64_t delta = goldilocks::sub(sbox<4>(a0), a0); + if (k + 1 < PARTIAL) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = fmul<4>(add3(s[i], FC[k][i], delta), FD[i]); + } else { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = fmul<4>(add3(s[i], FC[k][i], delta), FD_LAST[i]); + } + } + dft16<4, true>(s); +#pragma unroll 1 + for (int r = HALF_FULL + PARTIAL; r < ROUNDS; ++r) full_round_lean(s, r); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = goldilocks::canonical(s[i]); +} + +// A dot product of full-field constants with raw state words, accumulated +// as a 132-bit sum (lo, hi, top) and reduced once: 2^128 ≡ −2^32 (mod p), +// because 2^96 ≡ −1. `N` terms, the constants at compile-time indices. +template +__device__ __forceinline__ uint64_t dot_lazy(const uint64_t *c, const uint64_t *x) { + uint64_t lo = 0, hi = 0, top = 0; +#pragma unroll + for (int j = 0; j < N; ++j) { + const uint64_t pl = c[j] * x[j]; + const uint64_t ph = __umul64hi(c[j], x[j]); + lo += pl; + const uint64_t c1 = (lo < pl) ? 1ull : 0ull; + const uint64_t h2 = hi + ph; + uint64_t c2 = (h2 < hi) ? 1ull : 0ull; + hi = h2 + c1; + c2 += (hi < h2) ? 1ull : 0ull; + top += c2; + } + return goldilocks::sub(goldilocks::reduce128(lo, hi), top << 32); +} + +// The dense product by the Grain Cauchy matrix — the MDS alternative. 256 +// full products, where the circulant needs 512 small-constant MACs. +__device__ __forceinline__ void mds_cauchy(uint64_t s[WIDTH]) { + uint64_t out[WIDTH]; +#pragma unroll + for (int i = 0; i < WIDTH; ++i) out[i] = dot_lazy(CAUCHY_M[i], s); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = out[i]; +} + +template +__device__ __forceinline__ void full_round_cauchy(uint64_t s[WIDTH], const uint64_t *rc) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = sbox(goldilocks::add(s[i], rc[i])); + mds_cauchy(s); +} + +// ★ The Cauchy alternative, with the paper's Appendix B sparse partial rounds +// (`scripts/poseidon1/p1_sparse.py`, checked equal to the textbook form there +// for this matrix): dense full rounds; CAUCHY_B0 on lanes 1..16 once; per +// partial round x0 = (s0 + C0[k])^7, new0 = N00[k]·x0 + W[k]·s[1..], +// s_i += V[k][i-1]·x0; the folded carry rides in CAUCHY_RC_T0. The Fourier +// trick does not apply: a Cauchy matrix is not circulant. +template +__device__ __forceinline__ void permute_cauchy(uint64_t s[WIDTH]) { +#pragma unroll 1 + for (int r = 0; r < HALF_FULL; ++r) full_round_cauchy(s, RC[r]); + { + uint64_t t[WIDTH - 1]; +#pragma unroll + for (int i = 0; i < WIDTH - 1; ++i) t[i] = dot_lazy(CAUCHY_B0[i], s + 1); +#pragma unroll + for (int i = 0; i < WIDTH - 1; ++i) s[i + 1] = t[i]; + } +#pragma unroll 1 + for (int k = 0; k < PARTIAL; ++k) { + const uint64_t x0 = sbox(goldilocks::add(s[0], CAUCHY_C0[k])); + const uint64_t new0 = + goldilocks::add(fmul(CAUCHY_N00[k], x0), dot_lazy(CAUCHY_W[k], s + 1)); +#pragma unroll + for (int i = 1; i < WIDTH; ++i) s[i] = goldilocks::add(s[i], fmul(CAUCHY_V[k][i - 1], x0)); + s[0] = new0; + } + full_round_cauchy(s, CAUCHY_RC_T0); +#pragma unroll 1 + for (int r = HALF_FULL + PARTIAL + 1; r < ROUNDS; ++r) full_round_cauchy(s, RC[r]); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = goldilocks::canonical(s[i]); +} + +// ★ The permutation. V = 2 takes the Fourier-domain partial rounds above, +// V = 3 the Cauchy alternative, V = 4 the lean Fourier form; otherwise one rolled round loop with a +// warp-uniform branch on the round kind, so the MDS body is emitted once. +template +__device__ __forceinline__ void permute(uint64_t s[WIDTH]) { + if constexpr (V == 2) { + permute_fourier(s); + return; + } + if constexpr (V == 3) { + permute_cauchy(s); + return; + } + if constexpr (V == 4) { + permute_fourier_lean(s); + return; + } +#pragma unroll 1 + for (int r = 0; r < ROUNDS; ++r) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = goldilocks::add(s[i], RC[r][i]); + if (r < HALF_FULL || r >= HALF_FULL + PARTIAL) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = sbox(s[i]); + } else { + s[0] = sbox(s[0]); + } + mds(s); + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = goldilocks::canonical(s[i]); +} + +// The leaf sponge over `num_felts` felts read through `load(i)`. Blocks are +// written with compile-time lane indices (no dynamic state indexing). +template +__device__ __forceinline__ void sponge_leaf(uint64_t num_felts, Load load, uint64_t digest[DIGEST]) { + uint64_t s[WIDTH]; +#pragma unroll + for (int i = 0; i < RATE; ++i) s[i] = 0; + s[CAP_PAD_LANE] = num_felts % RATE; + s[CAP_DOMAIN_LANE] = DOMAIN_LEAF; + s[CAP_DOMAIN_LANE + 1] = 0; + s[CAP_DOMAIN_LANE + 2] = 0; + uint64_t t = 0; +#pragma unroll 1 + for (; t + RATE <= num_felts; t += RATE) { +#pragma unroll + for (int k = 0; k < RATE; ++k) s[k] = load(t + k); + permute(s); + } + if (t < num_felts) { +#pragma unroll + for (int k = 0; k < RATE; ++k) s[k] = (t + k < num_felts) ? load(t + k) : 0; + permute(s); + } +#pragma unroll + for (int i = 0; i < DIGEST; ++i) digest[i] = s[i]; +} + +template +__device__ __forceinline__ void compress4(const uint64_t *children, uint64_t out[DIGEST]) { + uint64_t s[WIDTH]; +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = children[i]; + permute(s); +#pragma unroll + for (int i = 0; i < DIGEST; ++i) out[i] = s[i]; +} + +// ZisK's leaf hash (`linear_hash_seq` at width 16): each block of up to 12 +// felts overwrites lanes 0..12, zero-filled past its end; the capacity lanes +// 12..16 are the previous output's lanes 0..4 for every block after the first. +// The first block's capacity is zero under TAG = false (ZisK's own leaf, host +// oracle `poseidon1_stark::zisk_linear_hash`: the measurement kernels) and the +// width tag [num_felts, DOMAIN_LEAF, 0, 0] under TAG = true (the STARK's leaf, +// host oracle `poseidon1_stark::linear_hash`: the production `p1s_*` kernels). +// An empty leaf never permutes. +template +__device__ __forceinline__ void zisk_leaf(uint64_t num_felts, Load load, uint64_t digest[DIGEST]) { + uint64_t s[WIDTH]; +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = 0; + if (TAG) { + s[CAP_PAD_LANE] = num_felts; + s[CAP_DOMAIN_LANE] = DOMAIN_LEAF; + } + uint64_t t = 0; +#pragma unroll 1 + for (; t + RATE <= num_felts; t += RATE) { + if (t > 0) { +#pragma unroll + for (int i = 0; i < DIGEST; ++i) s[RATE + i] = s[i]; + } +#pragma unroll + for (int k = 0; k < RATE; ++k) s[k] = load(t + k); + permute(s); + } + if (t < num_felts) { + if (t > 0) { +#pragma unroll + for (int i = 0; i < DIGEST; ++i) s[RATE + i] = s[i]; + } +#pragma unroll + for (int k = 0; k < RATE; ++k) s[k] = (t + k < num_felts) ? load(t + k) : 0; + permute(s); + } +#pragma unroll + for (int i = 0; i < DIGEST; ++i) digest[i] = s[i]; +} + +} // namespace p1w16 + +// --------------------------------------------------------------------------- +// Width 8: ZisK's grinding permutation (`crypto::hash::poseidon1_w8`). The same +// recipe at t = 8: x^7, R_F 8, R_P 22, Grain constants, the circulant +// `[7, 1, 3, 8, 8, 3, 4, 9]`; the 8-point DFT (omega = 2^24) diagonalises it. +// V = 1 textbook rounds (rolled loop, dense circulant every round); +// V = 2 the partial rounds in the Fourier domain (`p1_fourier.py` at T = 8). +// --------------------------------------------------------------------------- +#include "p1w8_constants.cuh" + +namespace p1w8 { + +enum : int { WIDTH = 8 }; +using p1w16::HALF_FULL; +using p1w16::PARTIAL; +using p1w16::ROUNDS; + +// The circulant over 32-bit halves, as `p1w16::mds`: each half-sum is below +// 43·2^32 < 2^38, `hi < 2^7`. +__device__ __forceinline__ void mds(uint64_t s[WIDTH]) { + uint32_t lo32[WIDTH], hi32[WIDTH]; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + lo32[j] = (uint32_t)s[j]; + hi32[j] = (uint32_t)(s[j] >> 32); + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) { + uint64_t acc_lo = 0, acc_hi = 0; +#pragma unroll + for (int j = 0; j < WIDTH; ++j) { + const uint64_t c = MDS_ROW[(j + WIDTH - i) % WIDTH]; + acc_lo += c * (uint64_t)lo32[j]; + acc_hi += c * (uint64_t)hi32[j]; + } + const uint64_t lo = (acc_hi << 32) + acc_lo; + const uint64_t carry = (lo < acc_lo) ? 1ull : 0ull; + const uint64_t hi = (acc_hi >> 32) + carry; + s[i] = goldilocks::add(lo, hi * goldilocks::EPSILON); + } +} + +template +__device__ __forceinline__ void full_round(uint64_t s[WIDTH], int r) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = p1w16::sbox(goldilocks::add(s[i], RC[r][i])); + mds(s); +} + +// The 8-point DFT, `p1w16::dft16`'s radix-2 form at width 8. +template +__device__ __forceinline__ void dft8(uint64_t x[WIDTH]) { + constexpr int BITREV[WIDTH] = {0, 4, 2, 6, 1, 5, 3, 7}; + uint64_t y[WIDTH]; +#pragma unroll + for (int i = 0; i < WIDTH; ++i) y[i] = x[BITREV[i]]; +#pragma unroll + for (int h = 1; h < WIDTH; h <<= 1) { +#pragma unroll + for (int start = 0; start < WIDTH; start += 2 * h) { +#pragma unroll + for (int j = 0; j < h; ++j) { + const int e = j * (4 / h); // omega_{2h}^j = omega^(j·8/(2h)) + const int ei = INV ? ((WIDTH - e) & (WIDTH - 1)) : e; + const uint64_t a = y[start + j]; + const uint64_t t = + (e == 0) ? y[start + j + h] : p1w16::fmul(y[start + j + h], OMEGA_POW[ei]); + y[start + j] = goldilocks::add(a, t); + y[start + j + h] = goldilocks::sub(a, t); + } + } + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) x[i] = y[i]; +} + +// ★ The permutation; the output is canonical. +template +__device__ __forceinline__ void permute(uint64_t s[WIDTH]) { + if constexpr (V == 2) { +#pragma unroll 1 + for (int r = 0; r < HALF_FULL - 1; ++r) full_round(s, r); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) + s[i] = p1w16::sbox(goldilocks::add(s[i], RC[HALF_FULL - 1][i])); + dft8(s); +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = p1w16::fmul(s[i], FD[i]); +#pragma unroll 1 + for (int k = 0; k < PARTIAL; ++k) { + uint64_t sum = s[0]; +#pragma unroll + for (int i = 1; i < WIDTH; ++i) sum = goldilocks::add(sum, s[i]); + const uint64_t a0 = goldilocks::add(p1w16::fmul(sum, INV8), RC[HALF_FULL + k][0]); + const uint64_t delta = goldilocks::sub(p1w16::sbox(a0), a0); + if (k + 1 < PARTIAL) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) + s[i] = p1w16::fmul(goldilocks::add(goldilocks::add(s[i], FC[k][i]), delta), FD[i]); + } else { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) + s[i] = p1w16::fmul(goldilocks::add(goldilocks::add(s[i], FC[k][i]), delta), + FD_LAST[i]); + } + } + dft8(s); +#pragma unroll 1 + for (int r = HALF_FULL + PARTIAL; r < ROUNDS; ++r) full_round(s, r); + } else { +#pragma unroll 1 + for (int r = 0; r < ROUNDS; ++r) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = goldilocks::add(s[i], RC[r][i]); + if (r < HALF_FULL || r >= HALF_FULL + PARTIAL) { +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = p1w16::sbox(s[i]); + } else { + s[0] = p1w16::sbox(s[0]); + } + mds(s); + } + } +#pragma unroll + for (int i = 0; i < WIDTH; ++i) s[i] = goldilocks::canonical(s[i]); +} + +} // namespace p1w8 + +// --------------------------------------------------------------------------- +// The `extern "C"` surface, one entry per variant. +// --------------------------------------------------------------------------- + +// Parity probe: `n` independent permutations, one thread each. +template +__device__ __forceinline__ void permute_probe(const uint64_t *states, uint64_t n, uint64_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint64_t s[p1w16::WIDTH]; +#pragma unroll + for (int i = 0; i < p1w16::WIDTH; ++i) s[i] = states[tid * p1w16::WIDTH + i]; + p1w16::permute(s); +#pragma unroll + for (int i = 0; i < p1w16::WIDTH; ++i) out[tid * p1w16::WIDTH + i] = s[i]; +} + +// Base-field coset leaves: leaf `tid` hashes `codeword[tid + t·num_leaves]`, +// `t` in `[0, block)` — `rpx_leaves_base_coset`'s geometry. +template +__device__ __forceinline__ void leaves_base_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint64_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::sponge_leaf( + block, [&](uint64_t i) { return codeword[tid + i * num_leaves]; }, d); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) out[tid * p1w16::DIGEST + i] = d[i]; +} + +// Ext3 coset leaves: each element's three components in order. +template +__device__ __forceinline__ void leaves_ext3_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint64_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::sponge_leaf( + 3 * block, + [&](uint64_t i) { return codeword[(tid + (i / 3) * num_leaves) * 3 + i % 3]; }, d); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) out[tid * p1w16::DIGEST + i] = d[i]; +} + +// One 4-ary level: parent `tid` = compress4(children[4·tid .. 4·tid + 4]). +template +__device__ __forceinline__ void merkle_level4(const uint64_t *__restrict__ children, + uint64_t *__restrict__ parents, uint64_t n_parents) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n_parents) return; + uint64_t d[p1w16::DIGEST]; + p1w16::compress4(children + tid * p1w16::WIDTH, d); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) parents[tid * p1w16::DIGEST + i] = d[i]; +} + +// The grind search: `rpx_grind_search`'s loop, first-hit `atomicMin` and poll, +// with this sponge. The head is lane 0, canonical after `permute`. +template +__device__ __forceinline__ void grind_search(const uint64_t *inner, uint64_t limit, uint64_t base, + uint64_t count, volatile unsigned long long *result) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + const uint64_t f[4] = {inner[0], inner[1], inner[2], inner[3]}; + for (uint64_t i = tid; i < count; i += stride) { + uint64_t nonce = base + i; + if (nonce < base) break; + if (nonce >= (uint64_t)*result) break; + uint64_t s[p1w16::WIDTH]; +#pragma unroll + for (int k = 0; k < 4; ++k) s[k] = f[k]; + s[4] = goldilocks::canonical(nonce); +#pragma unroll + for (int k = 5; k < p1w16::RATE; ++k) s[k] = 0; + s[p1w16::CAP_PAD_LANE] = p1w16::GRIND_FELTS % p1w16::RATE; + s[p1w16::CAP_DOMAIN_LANE] = p1w16::DOMAIN_LEAF; + s[p1w16::CAP_DOMAIN_LANE + 1] = 0; + s[p1w16::CAP_DOMAIN_LANE + 2] = 0; + p1w16::permute(s); + if (s[0] < limit) { + atomicMin((unsigned long long *)result, (unsigned long long)nonce); + } + } +} + +// Register caps for the occupancy sweep: `__launch_bounds__(128, minb)` asks for +// `minb` resident 128-thread blocks per SM, i.e. at most 65536 / (128·minb) +// registers per thread (128 at 4, 102 at 5, 85 at 6). The host shim has none. +#if defined(__CUDACC__) +#define P1W16_LB(minb) __launch_bounds__(128, minb) +#else +#define P1W16_LB(minb) +#endif + +#define P1W16_ENTRIES(NAME, V, BOUND) \ + extern "C" __global__ void BOUND p1w16_permute_probe_##NAME(const uint64_t *states, \ + uint64_t n, uint64_t *out) { \ + permute_probe(states, n, out); \ + } \ + extern "C" __global__ void BOUND p1w16_leaves_base_coset_##NAME( \ + const uint64_t *__restrict__ codeword, uint64_t num_leaves, uint64_t block, \ + uint64_t *__restrict__ out) { \ + leaves_base_coset(codeword, num_leaves, block, out); \ + } \ + extern "C" __global__ void BOUND p1w16_leaves_ext3_coset_##NAME( \ + const uint64_t *__restrict__ codeword, uint64_t num_leaves, uint64_t block, \ + uint64_t *__restrict__ out) { \ + leaves_ext3_coset(codeword, num_leaves, block, out); \ + } \ + extern "C" __global__ void BOUND p1w16_merkle_level4_##NAME( \ + const uint64_t *__restrict__ children, uint64_t *__restrict__ parents, \ + uint64_t n_parents) { \ + merkle_level4(children, parents, n_parents); \ + } \ + extern "C" __global__ void BOUND p1w16_grind_search_##NAME( \ + const uint64_t *inner, uint64_t limit, uint64_t base, uint64_t count, \ + volatile unsigned long long *result) { \ + grind_search(inner, limit, base, count, result); \ + } + +// The two multiply variants, the Fourier-domain partial rounds, the Cauchy +// alternative, and the lean Fourier form (`src/p1w16.rs` VARIANTS lists the +// same names in the same order). FAST job +// 281 measured the register caps (4/5/6 blocks of 128) SLOWER than uncapped +// (+6 %, +63 %, ×4.6), so no capped variant is instantiated; `P1W16_LB` stays +// for the next sweep. +P1W16_ENTRIES(v0, 0, ) +P1W16_ENTRIES(v1, 1, ) +P1W16_ENTRIES(v2, 2, ) +P1W16_ENTRIES(c1, 3, ) +P1W16_ENTRIES(l2, 4, ) + +// --------------------------------------------------------------------------- +// ZisK's instance (stage P1 of the Poseidon1 base STARK): its leaf hash over +// the coset geometry above and over the STARK's column-major LDE rows, and its +// width-8 grind. The Fourier-domain permutation (V = 2) only; the 4-ary node +// is `p1w16_merkle_level4_v2` unchanged. Digests are four canonical u64s. +// --------------------------------------------------------------------------- + +// Coset leaves under ZisK's leaf hash: leaf `tid` hashes +// `codeword[tid + t·num_leaves]` (base) or that element's three components +// (ext3), `t` in `[0, block)`. +extern "C" __global__ void p1w16_zleaves_base_coset_v2(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint64_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, false>( + block, [&](uint64_t i) { return codeword[tid + i * num_leaves]; }, d); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) out[tid * p1w16::DIGEST + i] = d[i]; +} + +extern "C" __global__ void p1w16_zleaves_ext3_coset_v2(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint64_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, false>( + 3 * block, + [&](uint64_t i) { return codeword[(tid + (i / 3) * num_leaves) * 3 + i % 3]; }, d); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) out[tid * p1w16::DIGEST + i] = d[i]; +} + +// One leaf per bit-reversed LDE row: column `c` of row `br` at +// `columns[c * col_stride + br]`, absorbed column by column — +// `rpx_leaves_base_batched`'s geometry. An ext3 matrix stored as three base +// slabs per column (`rpx_leaves_ext3_batched`) is this kernel over +// `num_cols = 3 · ext3 columns`: felt `3c + k` is slab `3c + k`. +extern "C" __global__ void p1w16_zleaves_rows_v2(const uint64_t *__restrict__ columns, + uint64_t col_stride, uint64_t num_cols, + uint64_t num_rows, uint64_t log_num_rows, + uint64_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + const uint64_t br = __brevll(tid) >> (64 - log_num_rows); + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, false>( + num_cols, [&](uint64_t c) { return columns[c * col_stride + br]; }, d); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) out[tid * p1w16::DIGEST + i] = d[i]; +} + +// Row-pair leaves: leaf `tid` hashes bit-reversed rows `2·tid` then `2·tid + 1`, +// each column by column — `rpx_leaves_base_row_pair_batched`'s geometry. +extern "C" __global__ void p1w16_zleaves_row_pair_v2(const uint64_t *__restrict__ columns, + uint64_t col_stride, uint64_t num_cols, + uint64_t num_rows, uint64_t log_num_rows, + uint64_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows / 2) return; + const uint64_t br0 = __brevll(2 * tid) >> (64 - log_num_rows); + const uint64_t br1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, false>( + 2 * num_cols, + [&](uint64_t i) { + return i < num_cols ? columns[i * col_stride + br0] + : columns[(i - num_cols) * col_stride + br1]; + }, + d); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) out[tid * p1w16::DIGEST + i] = d[i]; +} + +// Width-8 parity probe: `n` independent permutations. +template +__device__ __forceinline__ void p1w8_probe(const uint64_t *states, uint64_t n, uint64_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint64_t s[p1w8::WIDTH]; +#pragma unroll + for (int i = 0; i < p1w8::WIDTH; ++i) s[i] = states[tid * p1w8::WIDTH + i]; + p1w8::permute(s); +#pragma unroll + for (int i = 0; i < p1w8::WIDTH; ++i) out[tid * p1w8::WIDTH + i] = s[i]; +} + +// ZisK's grind: the smallest nonce in `[base, base + count)` whose width-8 +// permutation of `[c0, c1, c2, nonce, 0, 0, 0, 0]` has lane 0 below `limit` +// (`crypto::hash::poseidon1_stark::grinding_lane0`). The grid-stride loop, +// first-hit `atomicMin` and poll of `grind_search` above. +template +__device__ __forceinline__ void p1w8_grind(const uint64_t *challenge, uint64_t limit, uint64_t base, + uint64_t count, volatile unsigned long long *result) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + const uint64_t c0 = challenge[0], c1 = challenge[1], c2 = challenge[2]; + for (uint64_t i = tid; i < count; i += stride) { + uint64_t nonce = base + i; + if (nonce < base) break; + if (nonce >= (uint64_t)*result) break; + uint64_t s[p1w8::WIDTH] = {c0, c1, c2, goldilocks::canonical(nonce), 0, 0, 0, 0}; + p1w8::permute(s); + if (s[0] < limit) { + atomicMin((unsigned long long *)result, (unsigned long long)nonce); + } + } +} + +#define P1W8_ENTRIES(NAME, V) \ + extern "C" __global__ void p1w8_permute_probe_##NAME(const uint64_t *states, uint64_t n, \ + uint64_t *out) { \ + p1w8_probe(states, n, out); \ + } \ + extern "C" __global__ void p1w8_grind_search_##NAME(const uint64_t *challenge, uint64_t limit, \ + uint64_t base, uint64_t count, \ + volatile unsigned long long *result) { \ + p1w8_grind(challenge, limit, base, count, result); \ + } + +P1W8_ENTRIES(v1, 1) +P1W8_ENTRIES(v2, 2) + +// Bench fill: `out[i]` = a SplitMix64 of `i`, canonicalised — a codeword the +// microbench hashes without a host upload. +extern "C" __global__ void p1w16_fill(uint64_t *out, uint64_t n) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint64_t z = tid + 0x9E3779B97F4A7C15ull; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + out[tid] = goldilocks::canonical(z ^ (z >> 31)); +} + +// =========================================================================== +// PRODUCTION (`p1/*` P2): ZisK's instance on the STARK's device commit paths, +// launched by `src/p1_stark.rs` for `DeviceHash::Poseidon1`. +// +// NODE BYTES are the host's commitment bytes, as `rpx.cu`'s: four canonical +// felts, each eight BIG-endian bytes (`algebraic_commit::digest_to_commitment`), +// so a device node buffer is the host's byte for byte and a parent reads its +// children back with `commitment_to_digest`'s decoding. +// +// TREES are 4-ary in the host's arity-4 layout (`crypto::merkle_tree::utils:: +// level_offsets4`): the levels top-down, root at node 0, leaves last; a level +// holds ⌈below / 4⌉ nodes, so the level above the one at `off` with `n` nodes +// starts at `off − ⌈n / 4⌉`. A short group's missing children are the zero +// digest (ZisK's rule; `P1BatchBackend::padding_node`) and are never stored. +// A parent is `compress4`: the four child digests fill the sixteen lanes, +// one permutation, lanes 0..4. +// +// LEAVES are `zisk_leaf` over the felt sequence the host leaf hashes: the read +// patterns of `rpx.cu`'s leaf kernels (bit-reversed rows, column by column; a +// row pair is the first row then the second; an ext3 element as its three +// components), which is what `element_felts` / `felts_from_bytes` give on the +// host. +// =========================================================================== + +namespace p1s { + +// Byte-swap a u64 (`rpx::bswap64`). +__device__ __forceinline__ uint64_t bswap64(uint64_t x) { + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x >> 8) & 0x00FF00FF00FF00FFull); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x >> 16) & 0x0000FFFF0000FFFFull); + return (x << 32) | (x >> 32); +} + +__device__ __forceinline__ void store_be(const uint64_t d[p1w16::DIGEST], uint8_t *node) { + uint64_t *dst = reinterpret_cast(node); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) dst[i] = bswap64(d[i]); +} + +__device__ __forceinline__ void load_be(const uint8_t *node, uint64_t *d) { + const uint64_t *src = reinterpret_cast(node); +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) d[i] = bswap64(src[i]); +} + +// Parent `p` of the level at `child_off` (`n_children` nodes), written at +// `parent_off + p`. +__device__ __forceinline__ void parent4(uint8_t *nodes, uint64_t child_off, uint64_t n_children, + uint64_t parent_off, uint64_t p) { + uint64_t s[p1w16::WIDTH]; +#pragma unroll + for (int c = 0; c < 4; ++c) { + const uint64_t idx = 4 * p + (uint64_t)c; + if (idx < n_children) { + load_be(nodes + (child_off + idx) * 32, s + 4 * c); + } else { +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) s[4 * c + i] = 0; + } + } + p1w16::permute<2>(s); + store_be(s, nodes + (parent_off + p) * 32); +} + +// `parent4` under the lean permutation (the `_lean` twins below). +__device__ __forceinline__ void parent4_lean(uint8_t *nodes, uint64_t child_off, uint64_t n_children, + uint64_t parent_off, uint64_t p) { + uint64_t s[p1w16::WIDTH]; +#pragma unroll + for (int c = 0; c < 4; ++c) { + const uint64_t idx = 4 * p + (uint64_t)c; + if (idx < n_children) { + load_be(nodes + (child_off + idx) * 32, s + 4 * c); + } else { +#pragma unroll + for (int i = 0; i < p1w16::DIGEST; ++i) s[4 * c + i] = 0; + } + } + p1w16::permute<4>(s); + store_be(s, nodes + (parent_off + p) * 32); +} + +} // namespace p1s + +// One leaf per bit-reversed row of a column-major matrix: column `c` of row +// `br` at `cols[c * col_stride + br]` (`rpx_leaves_base_batched`'s geometry). +// An ext3 matrix stored as three base slabs per column +// (`rpx_leaves_ext3_batched`) is this kernel over `num_cols = 3 · columns`. +extern "C" __global__ void p1s_leaves_cols_row(const uint64_t *__restrict__ cols, uint64_t col_stride, + uint64_t num_cols, uint64_t num_rows, + uint64_t log_num_rows, uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + const uint64_t br = __brevll(tid) >> (64 - log_num_rows); + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + num_cols, [&](uint64_t c) { return cols[c * col_stride + br]; }, d); + p1s::store_be(d, out + tid * 32); +} + +// Row-pair leaves of a column-major matrix: leaf `tid` hashes bit-reversed rows +// `2·tid` then `2·tid + 1`, each column by column +// (`rpx_leaves_base_row_pair_batched`). Over `num_cols = 3 · parts` slabs it is +// the composition tree's ext3 row-pair leaf (`rpx_comp_poly_leaves_ext3`). +extern "C" __global__ void p1s_leaves_cols_pair(const uint64_t *__restrict__ cols, uint64_t col_stride, + uint64_t num_cols, uint64_t num_rows, + uint64_t log_num_rows, uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows / 2) return; + const uint64_t br0 = __brevll(2 * tid) >> (64 - log_num_rows); + const uint64_t br1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + 2 * num_cols, + [&](uint64_t i) { + return i < num_cols ? cols[i * col_stride + br0] : cols[(i - num_cols) * col_stride + br1]; + }, + d); + p1s::store_be(d, out + tid * 32); +} + +// Row-major row-pair leaves over columns `[col_start, col_end)` of rows of +// stride `m` (`rpx_leaves_base_row_major_row_pair[_range]`; the full row is +// the range `[0, m)`). +extern "C" __global__ void p1s_leaves_rm_pair(const uint64_t *__restrict__ data, uint64_t m, + uint64_t col_start, uint64_t col_end, + uint64_t num_rows, uint64_t log_num_rows, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows / 2) return; + const uint64_t *row0 = data + (__brevll(2 * tid) >> (64 - log_num_rows)) * m + col_start; + const uint64_t *row1 = data + (__brevll(2 * tid + 1) >> (64 - log_num_rows)) * m + col_start; + const uint64_t w = col_end - col_start; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + 2 * w, [&](uint64_t i) { return i < w ? row0[i] : row1[i - w]; }, d); + p1s::store_be(d, out + tid * 32); +} + +// Row-major one-row leaves over columns `[col_start, col_end)` +// (`rpx_leaves_base_row_major_row_range`). +extern "C" __global__ void p1s_leaves_rm_row(const uint64_t *__restrict__ data, uint64_t m, + uint64_t col_start, uint64_t col_end, + uint64_t num_rows, uint64_t log_num_rows, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + const uint64_t *row = data + (__brevll(tid) >> (64 - log_num_rows)) * m + col_start; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + col_end - col_start, [&](uint64_t i) { return row[i]; }, d); + p1s::store_be(d, out + tid * 32); +} + +// FRI leaves: leaf `tid` hashes the `3 · group` contiguous felts of the `group` +// consecutive ext3 values from `tid · group` of an interleaved eval vector +// (`rpx_fri_group_leaves_ext3`; the pair leaf is `group = 2`, six felts, as +// `P1PairBackend::hash_data`). +extern "C" __global__ void p1s_fri_group_leaves(const uint64_t *__restrict__ evals, uint64_t num_leaves, + uint64_t group, uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + const uint64_t *g = evals + tid * group * 3; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + 3 * group, [&](uint64_t i) { return g[i]; }, d); + p1s::store_be(d, out + tid * 32); +} + +// WHIR coset leaves (`P1BatchBackend::hash_data` over a fold block): leaf +// `tid` hashes `codeword[tid + t·num_leaves]` (base) or that element's three +// components (ext3), `t` in `[0, block)` — `rpx_leaves_*_coset`'s geometry +// under the tagged leaf. +extern "C" __global__ void p1s_leaves_base_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + block, [&](uint64_t i) { return codeword[tid + i * num_leaves]; }, d); + p1s::store_be(d, out + tid * 32); +} + +// The same leaves placed in the coset order: leaf `tid` (the block that folds +// onto `tid`) at position `bitrev(tid)` over `leaf_bits` bits, so the leaves +// under an aligned subtree are one coset of the domain +// (`multilinear::whir_commit::LeafOrder::Coset`). The reads are +// `p1s_leaves_base_coset`'s; only the digest's slot moves. +extern "C" __global__ void p1s_leaves_base_coset_rev(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint32_t leaf_bits, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + block, [&](uint64_t i) { return codeword[tid + i * num_leaves]; }, d); + const uint64_t at = leaf_bits == 0 ? tid : (__brevll(tid) >> (64 - leaf_bits)); + p1s::store_be(d, out + at * 32); +} + +extern "C" __global__ void p1s_leaves_ext3_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<2, true>( + 3 * block, + [&](uint64_t i) { return codeword[(tid + (i / 3) * num_leaves) * 3 + i % 3]; }, d); + p1s::store_be(d, out + tid * 32); +} + +// One 4-ary level in place: parent `tid` of the level at `child_off` +// (`n_children` nodes) into the level at `parent_off` (`n_parents` nodes). +extern "C" __global__ void p1s_merkle_level4(uint8_t *nodes, uint64_t child_off, uint64_t n_children, + uint64_t parent_off, uint64_t n_parents) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n_parents) return; + p1s::parent4(nodes, child_off, n_children, parent_off, tid); +} + +// Every level from the one at `child_off` (`n_children` nodes) up to the root, +// in one block, a barrier between levels. +extern "C" __global__ void p1s_merkle_tail4(uint8_t *nodes, uint64_t child_off, uint64_t n_children) { + while (n_children > 1) { + const uint64_t n_parents = (n_children + 3) / 4; + const uint64_t parent_off = child_off - n_parents; + for (uint64_t p = threadIdx.x; p < n_parents; p += blockDim.x) { + p1s::parent4(nodes, child_off, n_children, parent_off, p); + } + __syncthreads(); + child_off = parent_off; + n_children = n_parents; + } +} + +// --------------------------------------------------------------------------- +// The `_lean` twins: the kernels above with the lean +// permutation (`p1w16::permute_fourier_lean`), the same bytes in the same +// slots. `src/p1_stark.rs` loads them in their twins' places under +// `LAMBDA_VM_P1_LEAN` (on by default; `=0` loads the plain kernels). +// --------------------------------------------------------------------------- + +extern "C" __global__ void p1s_leaves_cols_row_lean(const uint64_t *__restrict__ cols, + uint64_t col_stride, uint64_t num_cols, + uint64_t num_rows, uint64_t log_num_rows, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + const uint64_t br = __brevll(tid) >> (64 - log_num_rows); + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<4, true>( + num_cols, [&](uint64_t c) { return cols[c * col_stride + br]; }, d); + p1s::store_be(d, out + tid * 32); +} + +extern "C" __global__ void p1s_leaves_cols_pair_lean(const uint64_t *__restrict__ cols, + uint64_t col_stride, uint64_t num_cols, + uint64_t num_rows, uint64_t log_num_rows, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows / 2) return; + const uint64_t br0 = __brevll(2 * tid) >> (64 - log_num_rows); + const uint64_t br1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<4, true>( + 2 * num_cols, + [&](uint64_t i) { + return i < num_cols ? cols[i * col_stride + br0] : cols[(i - num_cols) * col_stride + br1]; + }, + d); + p1s::store_be(d, out + tid * 32); +} + +extern "C" __global__ void p1s_leaves_base_coset_lean(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<4, true>( + block, [&](uint64_t i) { return codeword[tid + i * num_leaves]; }, d); + p1s::store_be(d, out + tid * 32); +} + +extern "C" __global__ void p1s_leaves_base_coset_rev_lean(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint32_t leaf_bits, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<4, true>( + block, [&](uint64_t i) { return codeword[tid + i * num_leaves]; }, d); + const uint64_t at = leaf_bits == 0 ? tid : (__brevll(tid) >> (64 - leaf_bits)); + p1s::store_be(d, out + at * 32); +} + +extern "C" __global__ void p1s_leaves_ext3_coset_lean(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + uint64_t d[p1w16::DIGEST]; + p1w16::zisk_leaf<4, true>( + 3 * block, + [&](uint64_t i) { return codeword[(tid + (i / 3) * num_leaves) * 3 + i % 3]; }, d); + p1s::store_be(d, out + tid * 32); +} + +extern "C" __global__ void p1s_merkle_level4_lean(uint8_t *nodes, uint64_t child_off, + uint64_t n_children, uint64_t parent_off, + uint64_t n_parents) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n_parents) return; + p1s::parent4_lean(nodes, child_off, n_children, parent_off, tid); +} + +extern "C" __global__ void p1s_merkle_tail4_lean(uint8_t *nodes, uint64_t child_off, + uint64_t n_children) { + while (n_children > 1) { + const uint64_t n_parents = (n_children + 3) / 4; + const uint64_t parent_off = child_off - n_parents; + for (uint64_t p = threadIdx.x; p < n_parents; p += blockDim.x) { + p1s::parent4_lean(nodes, child_off, n_children, parent_off, p); + } + __syncthreads(); + child_off = parent_off; + n_children = n_parents; + } +} + +// Authentication paths: query `tid`'s path is `depth4` levels from the leaves +// up, each the three other children of its group in child order, the zero +// digest where the group is short — `MerkleTree::get_proof_by_pos` at arity 4. +// `total_nodes` is the tree's node count (the leaves start at +// `total_nodes − leaves_len`). +extern "C" __global__ void p1s_gather_paths4(const uint8_t *__restrict__ nodes, + const uint32_t *__restrict__ positions, uint32_t nq, + uint64_t leaves_len, uint64_t total_nodes, + uint32_t depth4, uint8_t *__restrict__ out) { + uint64_t q = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (q >= nq) return; + uint64_t i = positions[q]; + uint64_t n = leaves_len; + uint64_t off = total_nodes - leaves_len; + uint64_t *dst = reinterpret_cast(out + q * (uint64_t)depth4 * 3 * 32); + for (uint32_t level = 0; level < depth4; ++level) { + const uint64_t first = i & ~3ull; + for (uint64_t c = first; c < first + 4; ++c) { + if (c == i) continue; + if (c < n) { + const uint64_t *src = reinterpret_cast(nodes + (off + c) * 32); +#pragma unroll + for (int k = 0; k < 4; ++k) dst[k] = src[k]; + } else { +#pragma unroll + for (int k = 0; k < 4; ++k) dst[k] = 0; + } + dst += 4; + } + const uint64_t parents = (n + 3) / 4; + off -= parents; + n = parents; + i >>= 2; + } +} + +// The STARK grind (`P1GrindDigest`, `crypto::grinding`'s two-level form): the +// smallest nonce in `[base, base + count)` whose width-8 permutation of +// `[inner0, inner1, inner2, inner3, nonce, 0, 0, 0]` has lane 0 below `limit`. +// `inner` is the inner hash's four big-endian felts (`inner_hash_felts`). The +// grid-stride loop, first-hit `atomicMin` and poll of `p1w8_grind`. +extern "C" __global__ void p1s_grind_w8(const uint64_t *inner, uint64_t limit, uint64_t base, + uint64_t count, volatile unsigned long long *result) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + const uint64_t i0 = inner[0], i1 = inner[1], i2 = inner[2], i3 = inner[3]; + for (uint64_t i = tid; i < count; i += stride) { + uint64_t nonce = base + i; + if (nonce < base) break; + if (nonce >= (uint64_t)*result) break; + uint64_t s[p1w8::WIDTH] = {i0, i1, i2, i3, goldilocks::canonical(nonce), 0, 0, 0}; + p1w8::permute<2>(s); + if (s[0] < limit) { + atomicMin((unsigned long long *)result, (unsigned long long)nonce); + } + } +} diff --git a/crypto/math-cuda/kernels/p1w16_constants.cuh b/crypto/math-cuda/kernels/p1w16_constants.cuh new file mode 100644 index 000000000..6da0f0635 --- /dev/null +++ b/crypto/math-cuda/kernels/p1w16_constants.cuh @@ -0,0 +1,164 @@ +// Poseidon1 over Goldilocks at width 16: round constants and the MDS row. +// GENERATED by `scripts/poseidon1/p1_params.py cuda`; do not edit by hand. +// The same tables as `crypto/crypto/src/hash/poseidon1_w16/constants.rs`. +#pragma once +#include +namespace p1w16 { +__device__ __constant__ uint64_t RC[30][16] = { + {0x15ebea3fc73397c3ull, 0xd73cd9fbfe8e275cull, 0x8c096bfce77f6c26ull, 0x4e128f68b53d8feaull, 0x29b779a36b2763f6ull, 0xfe2adc6fb65acd08ull, 0x8d2520e725ad0955ull, 0x1c2392b214624d2aull, 0x37482118206dcc6eull, 0x2f829bed19be019aull, 0x2fe298cb6f8159b0ull, 0x2bbad982deccdbbfull, 0xbad568b8cc60a81eull, 0xb86a814265baad10ull, 0xbec2005513b3acb3ull, 0x6bf89b59a07c2a94ull}, + {0xa25deeb835e230f5ull, 0x3c5bad8512b8b12aull, 0x7230f73c3cb7a4f2ull, 0xa70c87f095c74d0full, 0x6b7606b830bb2e80ull, 0x6cd467cfc4f24274ull, 0xfeed794df42a9b0aull, 0x8cf7cf6163b7dbd3ull, 0x9a6e9dda597175a0ull, 0xaa52295a684faf7bull, 0x017b811cc3589d8dull, 0x55bfb699b6181648ull, 0xc2ccaf71501c2421ull, 0x1707950327596402ull, 0xdd2fcdcd42a8229full, 0x8b9d7d5b27778a21ull}, + {0xac9a05525f9cf512ull, 0x2ba125c58627b5e8ull, 0xc74e91250a8147a5ull, 0xa3e64b640d5bb384ull, 0xf53047d18d1f9292ull, 0xbaaeddacae3a6374ull, 0xf2d0914a808b3db1ull, 0x18af1a3742bfa3b0ull, 0x9a621ef50c55bdb8ull, 0xc615f4d1cc5466f3ull, 0xb7fbac19a35cf793ull, 0xd2b1a15ba517e46dull, 0x4a290c4d7fd26f6full, 0x4f0cf1bb1770c4c4ull, 0x548345386cd377f5ull, 0x33978d2789fddd42ull}, + {0xab78c59deb77e211ull, 0xc485b2a933d2be7full, 0xbde3792c00c03c53ull, 0xab4cefe8f893d247ull, 0xc5c0e752eab7f85full, 0xdbf5a76f893bafeaull, 0xa91f6003e3d984deull, 0x099539077f311e87ull, 0x097ec52232f9559eull, 0x53641bdf8991e48cull, 0x2afe9711d5ed9d7cull, 0xa7b13d3661b5d117ull, 0x5a0e243fe7af6556ull, 0x1076fae8932d5f00ull, 0x9b53a83d434934e3ull, 0xed3fd595a3c0344aull}, + {0x28eff4b01103d100ull, 0x60400ca3e2685a45ull, 0x1c8636beb3389b84ull, 0xac1332b60e13eff0ull, 0x2adafcc364e20f87ull, 0x79ffc2b14054ea0bull, 0x3f98e4c0908f0a05ull, 0xcdb230bc4e8a06c4ull, 0x1bcaf7705b152a74ull, 0xd9bca249a82a7470ull, 0x91e24af19bf82551ull, 0xa62b43ba5cb78858ull, 0xb4898117472e797full, 0xb3228bca606cdaa0ull, 0x844461051bca39c9ull, 0xf3411581f6617d68ull}, + {0xf7fd50646782b533ull, 0x6ca664253c18fb48ull, 0x2d2fcdec0886a08full, 0x29da00dd799b575eull, 0x47d966cc3b6e1e93ull, 0xde884e9a17ced59eull, 0xdacf46dc1c31a045ull, 0x5d2e3c121eb387f2ull, 0x51f8b0658b124499ull, 0x1e7dbd1daa72167dull, 0x8275015a25c55b88ull, 0xe8521c24ac7a70b3ull, 0x6521d121c40b3f67ull, 0xac12de797de135b0ull, 0xafa28ead79f6ed6aull, 0x685174a7a8d26f0bull}, + {0xeff92a08d35d9874ull, 0x3058734b76dd123aull, 0xfa55dcfba429f79cull, 0x559294d4324c7728ull, 0x7a770f53012dc178ull, 0xedd8f7c408f3883bull, 0x39b533cf8d795fa5ull, 0x160ef9de243a8c0aull, 0x431d52da6215fe3full, 0x54c51a2a2ef6d528ull, 0x9b13892b46ff9d16ull, 0x263c46fcee210289ull, 0xb738c96d25aabdc4ull, 0x5c33a5203996d38full, 0x2626496e7c98d8ddull, 0xc669e0a52785903aull}, + {0xaecde726c8ae1f47ull, 0x039343ef3a81e999ull, 0x2615ceaf044a54f9ull, 0x7e41e834662b66e1ull, 0x4ca5fd4895335783ull, 0x64b334d02916f2b0ull, 0x87268837389a6981ull, 0x034b75bcb20a6274ull, 0x58e658296cc2cd6eull, 0xe2d0f759acc31df4ull, 0x81a652e435093e20ull, 0x0b72b6e0172eaf47ull, 0x4aec43cec577d66dull, 0xde78365b028a84e6ull, 0x444e19569adc0ee4ull, 0x942b2451fa40d1daull}, + {0xe24506623ea5bd6cull, 0x082854bf2ef7c743ull, 0x69dbbc566f59d62eull, 0x248c38d02a7b5cb2ull, 0x4f4e8f8c09d15edbull, 0xd96682f188d310cfull, 0x6f9a25d56818b54cull, 0xb6cefed606546cd9ull, 0x5bc07523da38a67bull, 0x7df5a3c35b8111cfull, 0xaaa2cc5d4db34bb0ull, 0x9e673ff22a4653f8ull, 0xbd8b278d60739c62ull, 0xe10d20f6925b8815ull, 0xf6c87b91dd4da2bfull, 0xfed623e2f71b6f1aull}, + {0xa0f02fa52a94d0d3ull, 0xbb5794711b39fa16ull, 0xd3b94fba9d005c7full, 0x15a26e89fad946c9ull, 0xf3cb87db8a67cf49ull, 0x400d2bf56aa2a577ull, 0x56045ba28fd1dbb1ull, 0x06851d67ed5acacdull, 0xf7019bc57980d178ull, 0x0b56dd650d860d28ull, 0xcd095759d842636dull, 0x17180482b913fb6dull, 0xf085ee1e1911b728ull, 0x44549837566a0bfdull, 0xadd0271c8a246c59ull, 0x4292bcfb6e12c8ceull}, + {0xb9335e2407759a82ull, 0x08eb37a7111d5618ull, 0x2b7c80b429d54278ull, 0x53ea5d3e1999f4ceull, 0xedee2a45b06fa613ull, 0xacaf4772a36459c2ull, 0xcc0b4aa757746b2full, 0x52536a16f47129c0ull, 0x9b0abd0b07ccc510ull, 0x3d3b86ee7806a3eeull, 0x533072e923c588e5ull, 0x14555d5ea03b4428ull, 0xd9e7702629648bd3ull, 0x273c5dc3b8b6ed83ull, 0x26315afb96c27395ull, 0xbabc1eb9506c3767ull}, + {0x0d0281a2fb57b4c7ull, 0x2c709a3b40cce2bdull, 0x9ab9a14cafb72112ull, 0x189aed3cd14d0f5eull, 0xc941490f292f3566ull, 0x29d339bc4c4c5ed1ull, 0x51886b29d3b0d26eull, 0x26ecf673ad01666full, 0x58f20c05df6136e4ull, 0xcaae02f891d9a693ull, 0x99ea2de13df560c7ull, 0xfb7e604032a6c89bull, 0x9484e6a84078ffabull, 0x33eb1f78ea2bf882ull, 0x305fba6c763df8a4ull, 0x855a940575a2f5ccull}, + {0x3bbe96637669d8a9ull, 0xe0bf92ec5aa783b5ull, 0xd2d8f7be3c581792ull, 0x25d1dbdc95699756ull, 0x6811562dd7d380e7ull, 0xd2d0c07c8cb96126ull, 0xed7de2464aab8093ull, 0x28e68f8ff765cdddull, 0x000f4302468decf6ull, 0xf408a8da2ec86476ull, 0xa910d923652cbc51ull, 0x61ff0fdd666285f9ull, 0x0d35de1fb1d92032ull, 0xc33f743297168e6bull, 0x5280166634968f2full, 0x8029701a169b8c58ull}, + {0x442b1659f53fc92cull, 0xb80d1df09583aae7ull, 0x3f68f6f41e84d283ull, 0x2dfd848230eeab8eull, 0x464724542108bcceull, 0x9cbd9af019eb40d8ull, 0x9cd056128537dca6ull, 0x0fdc70699871ec1full, 0x91b4ed0840036aa9ull, 0x0426a03ab3b941aeull, 0x3620ce1d1da9d9eaull, 0xa909c10f133b12dfull, 0xdff78cb9ce5e72afull, 0x355a0507ea3d39e9ull, 0x33ab02012ebb8e90ull, 0x66e681125ad48d09ull}, + {0x754e7875a4cba209ull, 0xb4f4092cd96fe585ull, 0xe0beb61c71cd590bull, 0x289d6191348815afull, 0xc7f9310eebd89190ull, 0xc404c95833496b5dull, 0x67cd987a2e477666ull, 0x83a966eaaba6f2d0ull, 0xa843095304ddb411ull, 0xab9bd2ffaf5ce005ull, 0x4b8b976e23ed8d5eull, 0xe853009d6815ce32ull, 0x30b462f67decb8bdull, 0x4fbe35dfc8561ddeull, 0xb182e2119eb521e0ull, 0x9a37c025983b008bull}, + {0x9a8036d5c0b0ddffull, 0x530e4069d0629e2full, 0xb5d7149987ef9062ull, 0x6a7120ccc56ca201ull, 0xd4fb060e7aed5a56ull, 0x9d01fca4ebb8856bull, 0xe4bafe3340849d38ull, 0x0632a3df994f0c09ull, 0xf084664f0b9c8659ull, 0xbed5c89cfd8f9a70ull, 0x2732cb3f2103948cull, 0x56b9259a2307cb16ull, 0xe9c504996e7d3273ull, 0x1c8a11e45261ef10ull, 0xdd694710e5643a96ull, 0xf84538050c75ba9cull}, + {0x6d21207c2dd2ffddull, 0x398156c24d3ee292ull, 0x5dfe3b4a26dfdb31ull, 0x5264c0dce0fc508dull, 0x68db82e30df8c0b6ull, 0x0bb69f951e7dae8full, 0xa4a00a693a6a9dc6ull, 0x7d2ed77fb67e1edbull, 0x3a29153665b5deb6ull, 0xde05f000f7734b79ull, 0x2d3298b50e9b8ee3ull, 0x0a2801c5da77fe8dull, 0xe4dae42bd80dc286ull, 0x2a92a223212d9bb1ull, 0x4a9e772dc0e7ae65ull, 0xf91ad425467514a6ull}, + {0xc947db43432f981cull, 0xfab35dc94da6ff86ull, 0x30e86b98128ca1d8ull, 0x73045f05de00acdbull, 0x877a90860890a505ull, 0x904b699fae24adf6ull, 0x2fd36112b8db92eaull, 0x37c8e7b709efcfeeull, 0x117825ee6efb194dull, 0xef084994ba70e740ull, 0x3bbbb3e35ddff9ffull, 0x94599a930c7d397eull, 0x3c962b6062d37a0bull, 0x8cedf279060e80cfull, 0xe3d6fee148661835ull, 0xac6dc576dcf69a1bull}, + {0x4395a68ba78162dcull, 0x1509b345505d0cfcull, 0x78ea2a2c0ff44da3ull, 0x3b8453ac216d9dccull, 0xeeea1ecb6e9739daull, 0x998a5a4578f042a1ull, 0x85b4d1176041379eull, 0xf1c548623858b53eull, 0x84f76ecd462db219ull, 0x7d478acf677e40c9ull, 0x78bd49e6248a3f8bull, 0xbaa176467eac8dc0ull, 0xa1d87308ce5f171bull, 0x6ecdc783e58e6dedull, 0x0a27470d8a54c073ull, 0x303dedb68be27affull}, + {0x1283fa6b9fab0360ull, 0x09b9238f8990d3c6ull, 0x783fe452c827ce70ull, 0x51053b0a0b311026ull, 0x913af1029780fd02ull, 0x2768db16e909ab50ull, 0xe2711e4680cd49d4ull, 0xae38b891ffd47d23ull, 0x747a4123904482efull, 0x7590a5536989e07full, 0x597bcf699cb8f029ull, 0xcf1022c720e9c768ull, 0x6be4d569436457d0ull, 0x1ea4854422964af6ull, 0x9ae378da59b64449ull, 0x01111bdcbee5f71aull}, + {0x3e5b8b050fbb6ffeull, 0x5a14f21fc4493541ull, 0x1ce4f54dfef557a1ull, 0x5cd4599991cc8d4full, 0xc564b336f4eb42a2ull, 0xdcb6c46e887ac22eull, 0x8a75e51054cd014full, 0xb0ff306616755d5eull, 0x4dfee54c8dd19e47ull, 0x7ca27c634f5d6f20ull, 0x377a3f686a605c43ull, 0x942f96fe69242828ull, 0x63363826c868c690ull, 0xf622c7bbff21a41eull, 0x7413fe06ae65a228ull, 0xf27e12c8f2b63a12ull}, + {0xcb9cdc2782a601b6ull, 0x4c355b11ae9d1501ull, 0x97b3caa2ca1983c8ull, 0x6f253eb18a69a3d6ull, 0x9daaad35dbe278d0ull, 0x9b154395fc9479deull, 0xb88a4c10b121859bull, 0x23f574023668b146ull, 0xe014dd76f0daeb2aull, 0x47ca3455c99abbfcull, 0xe15bcce7d6626f93ull, 0xe785e1d29b697a2aull, 0x72f6e6335f2779b2ull, 0x63354be802a8d806ull, 0xe8680400a8b7da56ull, 0xa57878f6cd7dc1f5ull}, + {0xa3242e1666842e56ull, 0x9d4763931e5c80b8ull, 0xf5ad9042f9c2a2eaull, 0xfde9819dcf522ccfull, 0x2c1b7126142f99b2ull, 0x512a49e97331bcdfull, 0xdc36f6f22179f9a4ull, 0x1287293218e2af1dull, 0x9547747ddf7cd847ull, 0x27e9d7e39b288667ull, 0xf53ed68c44662e1cull, 0xccd3913a837bf56dull, 0xa54b0c67cee88c45ull, 0xc9204bc4f678e860ull, 0x3b2b3332135618afull, 0x3bfe057360941901ull}, + {0x7ea1c40408898d03ull, 0x65735bc04a989af5ull, 0xc0d0f9741e308f84ull, 0x274505af11e8561cull, 0x00c01a31a2a8adf7ull, 0xae16a9b5d19bd9ebull, 0xeb47bf1cf2f8c536ull, 0x6281eff11a965684ull, 0x16fbec6cb544e37bull, 0x43661c50ecbaf61dull, 0x48ededeefe678215ull, 0x41454e2ac92e020full, 0x677ab8b3c9ba50e2ull, 0x7fe3981c7c556ff9ull, 0x35c8b8589248beddull, 0x8466ae507b53ba99ull}, + {0x3c1ce7a6557cd52cull, 0xb547ce487a0afa54ull, 0x05edb6bc9e4c7961ull, 0x50573b9cb7beeefeull, 0xb26c40ecb1535571ull, 0x4a9d2fb483061eb1ull, 0x9738a691d84dd72cull, 0xedfb4e4efb79d5f8ull, 0xbfdb2e3c4d600b54ull, 0xa44a4620bb87c023ull, 0xedfd03c8933b6509ull, 0x6409ecdd7113b38full, 0xbc7dd042ff7eb60bull, 0x1c41de22a36b7a75ull, 0x177fdf40e9646dedull, 0x8eeff87cd83ea10dull}, + {0xf4ba0b7ed08bafebull, 0xe5b3ede57adb7f0dull, 0xd8bf37d380f76ba9ull, 0xb5e07f4e8f36d2f7ull, 0xf2014366c2e22b88ull, 0x876f5726bb600720ull, 0x2fdd356780329918ull, 0xa60294429a5d0408ull, 0x0fd714f70b67a879ull, 0xd3efd294503511e7ull, 0x9ae10ed6c78b28c6ull, 0xae221f0e399fa644ull, 0x7d75772d7d8bf011ull, 0x0a6321da6233b471ull, 0x9bb7b0e6658281beull, 0x7b16cc63c27fe8f4ull}, + {0xbf53f47f87f56a6dull, 0xe635bf90172d71e6ull, 0xccf02f48250d3fb5ull, 0x3d36aa7d16141aafull, 0xae1eb04558b772b4ull, 0x1d58ab28e8083c4full, 0x1d9bda2e12d3e534ull, 0x423dd55f154482efull, 0x72b65b610654133bull, 0x487d3548f56a85f0ull, 0xd8c52e8889a97c5aull, 0x505ec42c4db39362ull, 0xc217b41e7d0ef6e7ull, 0xa6fd6c3cda5c8473ull, 0xff5dd9dfcd788211ull, 0x303ff3284e6f7308ull}, + {0xa1aa11255463d565ull, 0x78e87d336ccc6981ull, 0xcbe6f13680f9259eull, 0x43ed60b3f0605305ull, 0x285bb9012ad02b65ull, 0x5e3be4930ba77f91ull, 0x7d0b7f76259514deull, 0x196f1021318d4d14ull, 0x32aeee488c159236ull, 0x7042b7b5a3393989ull, 0xefe7a9290c7ff77cull, 0x429dc22cfde71457ull, 0x70a0a8b0cd2e4e92ull, 0x2a66e414a2e69a2dull, 0xeb6d94e92221051cull, 0x704157837371e9beull}, + {0x8a085f6f2d59f8bbull, 0xd195560e9297a989ull, 0x8ef6701b17a6930cull, 0xbdb996211667ba57ull, 0xe240f4c1c4b2ad06ull, 0xfc2f101d588e0907ull, 0x791d9d059d498e68ull, 0xd21b9dfad1421e14ull, 0x3f40701396692e53ull, 0x837e836ab5a382caull, 0x6b7358c4631537d0ull, 0x7e3b633af145a310ull, 0xbac53ee64fd8b153ull, 0xb5e07a6e5b2b5242ull, 0xe754f45f540247cbull, 0x23ada0ccd4a9e8e9ull}, + {0x5bc4109d42b4c5c8ull, 0x754fa15d43d39e06ull, 0x9a99178df6fb2088ull, 0xe8fca9a16c030c7aull, 0xbb9f10f3f43aef47ull, 0x11350f1b54ae9adcull, 0x665e172a588ee1dbull, 0x08302ce9f0ee5277ull, 0x494e7e049554756cull, 0xded40cca27f6c2aeull, 0xfa62692d0d41fc3cull, 0x7b2f0fdcde013e9bull, 0x0998315134e1d6a9ull, 0x5e59c08deb844be8ull, 0x3a37c9734e09cfb2ull, 0x31719e2d681d7414ull}, +}; +// First ROW of the circulant MDS, `M[i][j] = MDS_ROW[(j - i) mod 16]`. `constexpr`, so a +// fully unrolled product folds every entry into an immediate. Row sum 371 < 2^9. +__device__ constexpr uint32_t MDS_ROW[16] = {1, 1, 51, 1, 11, 17, 2, 1, 101, 63, 15, 2, 67, 22, 13, 3}; +// Fourier-domain partial rounds (`scripts/poseidon1/p1_fourier.py`): OMEGA_POW[k] = 2^(12k) mod p +// (omega = 2^12 has order 16); FD = the circulant's eigenvalues (DFT of its first column); FD_LAST = +// FD / 16, the inverse DFT's scale folded into the last partial round; FC[k] = DFT(RC[4 + k]). +__device__ constexpr uint64_t OMEGA_POW[16] = {0x0000000000000001ull, 0x0000000000001000ull, 0x0000000001000000ull, 0x0000001000000000ull, 0x0001000000000000ull, 0x1000000000000000ull, 0x000000ffffffff00ull, 0x000ffffffff00000ull, 0xffffffff00000000ull, 0xfffffffefffff001ull, 0xfffffffeff000001ull, 0xffffffef00000001ull, 0xfffeffff00000001ull, 0xefffffff00000001ull, 0xfffffeff00000101ull, 0xffefffff00100001ull}; +__device__ constexpr uint64_t FD[16] = {0x0000000000000173ull, 0x1417dc500720439cull, 0xffcce6ff01001919ull, 0xdfb80b22dc0fa499ull, 0xff9fffff00000064ull, 0x1fe827dff54fcb9cull, 0x003300ffe6ffff18ull, 0x4fe7f51023e3ea9cull, 0x0000000000000097ull, 0xec57dbaf0ee0039dull, 0xffcd18fefeffe719ull, 0x1fd80adbdbf044a0ull, 0x0060000000000063ull, 0xe088201ef4afeb9dull, 0x0032ff0019000118ull, 0xafa7f4ef241c2a9dull}; +__device__ constexpr uint64_t FD_LAST[16] = {0xcfffffff30000018ull, 0x41417dc4c072043aull, 0x7ffcce6f80100192ull, 0x7dfb80b1bdc0fa4aull, 0xcff9ffff30000007ull, 0x41fe827dbf54fcbaull, 0x8003300f7e6ffff2ull, 0x44fe7f50c23e3eaaull, 0x8fffffff7000000aull, 0x3ec57dbac0ee003aull, 0x7ffcd18f7feffe72ull, 0x01fd80adbdbf044aull, 0xd005ffff30000007ull, 0x3e088201bf4afebaull, 0x80032fef81900012ull, 0x3afa7f4ec241c2aaull}; +__device__ constexpr uint64_t INV16 = 0xefffffff10000001ull; +__device__ __constant__ uint64_t FC[22][16] = { + {0x10b5eb90eebf18e9ull, 0x0ef71ac620710815ull, 0x7032d8f562ecbc86ull, 0xf30b6a3d1cc124f1ull, 0x89801e21dbe5b04dull, 0x5db71621c5484776ull, 0xaf2600846cf738b8ull, 0x163ff94d3611f9d3ull, 0x1c14775638a7f94cull, 0x519f5c6d8857e638ull, 0xcc5d4942a54f0d93ull, 0xfb22ee6bf638a6fdull, 0xda3326e65d594f63ull, 0xbd16de767afb145aull, 0xa9a396558aeec6ecull, 0xe9552c377d5d2486ull}, + {0x1e72f9a22059bda5ull, 0xd4282eb42aa1229bull, 0xc341d8ad3521f929ull, 0xb86f0c803a269989ull, 0x9b7a2286acff014eull, 0x61d53f97b75060c0ull, 0x5887e4f59c90066bull, 0x5d408d51bf7b57d0ull, 0x439cc1754cac056bull, 0xfa0eaddba2bf8187ull, 0x46fce951bd9dc3ceull, 0x8a122238679570aaull, 0xde3b0547ae349ab4ull, 0x3b60415f9667551cull, 0x11247c7c3d1eabe5ull, 0x24f6e6666733c8ceull}, + {0x817d19bda613bc3dull, 0xcf67f1e1195de91aull, 0x847b4d16bb17350dull, 0x069146cc9223290cull, 0x90bdee6ef30a55a8ull, 0xa03ae1cc64c2c5c2ull, 0x534d5f2f0a69aeb8ull, 0x6edbfe70b6ea6472ull, 0x3299585afcfc0a01ull, 0x8009dcb769c53704ull, 0x0883bd5d904acc97ull, 0xd6f09aae92b37e51ull, 0x4e44f60ddb163bceull, 0xea2a1a60c5a207efull, 0x254e26e7e4a0ad81ull, 0x40aa0ec300f3d809ull}, + {0x5d322325d971efb6ull, 0x56df5aad39b2e895ull, 0x371c1a84053663ffull, 0xa12063ac4b750f99ull, 0xce13a68716001d82ull, 0xf5a30094eafa8d87ull, 0x7ebbdc9f5878ae29ull, 0x6a3b74d28ecec6d4ull, 0xc7bc63f0605a5c8bull, 0xdbabfd6390543e00ull, 0xde9aada7b04a9542ull, 0xd97dfb20b7aa3e88ull, 0x8a17d404f0a400cdull, 0x19af223cd4c44d74ull, 0x2c1554185d1d53aaull, 0x88852966c3a67845ull}, + {0x7eea94a97d6fd797ull, 0x30fb6ff6acec899eull, 0xcd5ae89a5656d83cull, 0x65f6a6e8059d9be0ull, 0xb6465bf97fc0c9cdull, 0x3b9677758677028dull, 0xd2b11c24594377a8ull, 0xc4875205e941982bull, 0x0c9624d48dbdda7aull, 0xb8e3df450ef91b0cull, 0xf73b8f8b3843ad94ull, 0x253f5ed4ef5276acull, 0xe9b5b50d819f00abull, 0x9e7b23bfaf5ec808ull, 0x2d677d66d287a532ull, 0x207647c0537b9d91ull}, + {0xe1bceeb1cfefbf2eull, 0x1bab37ff961d29faull, 0x79304bdc83bc929aull, 0x90b51d6aefc4397eull, 0xa31ff202cfc27116ull, 0xe317ac19a89c43cbull, 0x73fae628c00afe15ull, 0xdc741b4a715bfc3cull, 0x5ff7e7c8dda0a22bull, 0xb8645efcf93acf8aull, 0x267f159d34cfb4a6ull, 0xe6b90ee07aab607full, 0x0c383d1fa0e9d075ull, 0xb80cd1bd2565edb2ull, 0xbad70e1e89d92a16ull, 0x8c5e428c4f7a39a6ull}, + {0x1c5ef61b08da16faull, 0xfde60937ec30082eull, 0x1d72f39780f86b4aull, 0xa8aa120adc7ab7fdull, 0x72b702e88ce65c06ull, 0x20107bdd701b5e6full, 0x26580af60f40c922ull, 0x7e9d5da64a73d796ull, 0xfd9ba7a340f66030ull, 0xe0d63151ad6348f7ull, 0x7084ccb54df01867ull, 0x6328602e123bb920ull, 0xe39d35cecda372a6ull, 0xcb53ea270c7639efull, 0x7d5237c6f78f69e0ull, 0x9cb49856adf7795eull}, + {0x8f848089abb38278ull, 0x407803a96469e5d8ull, 0xb73e4dbba2a03dfdull, 0x8c40340a620a1ebaull, 0xd810ab21ee535ccfull, 0xda00045e057e2cf1ull, 0xda4ad7d749cb3fb6ull, 0xfff2d553dbb3efbbull, 0x6508e3c54c4558d0ull, 0x14856f216d9e2d68ull, 0x19dbb59fec508247ull, 0xaa1ab2d50c0759aeull, 0x424ce6152b384ad4ull, 0x01cc02220eb74193ull, 0x75549c8ceb86da74ull, 0x396c7764afb10537ull}, + {0x08b633231e789995ull, 0xc85368dfeaac415aull, 0x4544397b6927e463ull, 0x4df6c40c7308021full, 0xda76e794a0c9a5acull, 0xe51bc0f784f0608full, 0xd9f5a844cda21064ull, 0x32fac8625ec27b38ull, 0xd1437b65b05dfb1full, 0xc0763793e1207bf8ull, 0x807555394d03a84dull, 0x8ffa3de7bca323adull, 0x0fe3a0b0aaf1607full, 0xe3e484008e788d96ull, 0x7a6b5d64495ef507ull, 0x7ac4eb44113c1120ull}, + {0x1e3966cb99a219daull, 0xde39313ffc098fffull, 0x744a4fbe855b60a0ull, 0x413ca22e26627508ull, 0x0cf0ad410d71f538ull, 0xd3d296750f4702dcull, 0x2b8fc5c7c6e8e609ull, 0x6973d30d21002151ull, 0x660e3c648ff6dc0aull, 0xf0d638262de4dc17ull, 0x4761655f2c624d7eull, 0x39ad805158348f49ull, 0x5f4281565b9ea225ull, 0xaddbf0103d552737ull, 0xd749ce689ec97d3cull, 0x5e95650e93c13854ull}, + {0xfefe428fdb12450full, 0x9f5c6e42ae51e285ull, 0xf6d036a6f06cbee0ull, 0xc824183fe713e3fbull, 0x9a52f1a35e91384bull, 0xf2267b84ea58c8d5ull, 0x4f4a3ee39ed694bdull, 0x50df65f989c44751ull, 0xb8b579401139f916ull, 0xcb007e409417abafull, 0x953b365ae95da14bull, 0x143efd17dbb8b64full, 0x06f5a9cb02dd0b26ull, 0x73580c2e1a914c18ull, 0xb83a0b2586ef3a4eull, 0x6b3d89876b8aeb0aull}, + {0x740506ce1ed9ceaaull, 0x086a820e259d8546ull, 0xe187b78209e5ed63ull, 0x932c3591af53cb39ull, 0x19390083f1976ac3ull, 0xd439d94f6374d1a8ull, 0x3e59952abff52185ull, 0xfc0831d74e4f0a29ull, 0x5de0930fea4e0d05ull, 0x94ed9c5a08506972ull, 0x866a738a86b885ddull, 0x8ad7e1afe125cf25ull, 0x3bf404dcdc207e06ull, 0xdbbd7c203d1abfa3ull, 0x8ac689b93af7c97aull, 0xe882c73cfb5c98aeull}, + {0x9016e920e72212eeull, 0x2dc18e38317a665dull, 0x56e6e7ce6106b079ull, 0x27cdb34920729308ull, 0x8a7513212e8e64a0ull, 0x3f01c8e92407b032ull, 0xeccb22ca551b1858ull, 0x1f6a34b441eed294ull, 0x4ec8fb946d981d28ull, 0xa26dd552b3eb97e2ull, 0x3ce329c9552e5a00ull, 0x04939c104ea359c6ull, 0x6aad7b3562f4f280ull, 0x304bd84e9c2bee7dull, 0xe5ba0627aab94a90ull, 0x0c77d15dea4aace8ull}, + {0x11a8e6cd1cec7d54ull, 0x5df35e31612a8947ull, 0xa48b6b55c8aa8ce6ull, 0xa73fc05a6d1d0bdcull, 0x405b288f4f2f833cull, 0x3eff2eab226b12ffull, 0xb64e951bef26ecdeull, 0x35218194c7dfa788ull, 0x2c959248018db184ull, 0x8634761a4b4e6cc3ull, 0xe951fd9545d50784ull, 0x62e8f4a4be40bed7ull, 0xfca952c108918fcbull, 0xfc966951ed562facull, 0x169117241d73c81eull, 0x5f7607cbf22c4b86ull}, + {0x8da59354c469453eull, 0x1962cd62c798141aull, 0x52b372518ab1f655ull, 0xa03f3147c402c01dull, 0xda28ccfaabbeac26ull, 0xff30443c60e159dcull, 0x7d0d72a00618cac8ull, 0xb0b0d89b9492748bull, 0x2800d37acf0a910dull, 0x65a2656edf86af6dull, 0xfef628861ee43a46ull, 0x28b921c57d56a513ull, 0xd56f68f16b631530ull, 0xb649e8c8ce42f085ull, 0x10730096133414a2ull, 0x46c9326d5e6e9e7bull}, + {0x67e4a85c33c91e27ull, 0xd7baf9ec8d0eb40full, 0xbf7a07350b4962e4ull, 0x886db6cacf9c7582ull, 0x8acbcb01d473848full, 0xa0a2dd2e9de304c5ull, 0x1a6315790b0dbdd1ull, 0xb2536d4ff0601937ull, 0x3e77f15b60a93180ull, 0xe68e9c319ef7e378ull, 0xee71f7c965954290ull, 0xe8a8eaa235ee02a6ull, 0xdf4fa336c26d984aull, 0x1c67c242054a7fb1ull, 0x5f2ac011d83c62b3ull, 0x518f85ecb6155634ull}, + {0x45f0a1f966c8c65eull, 0xcddfe3ec98328286ull, 0xe279dae858cafb48ull, 0x22e70acda4bd7d35ull, 0xe6b8db08277213e6ull, 0x03b9d9e1af0ea9c6ull, 0x743080975169ccd3ull, 0xefca7b4bf1cadd9dull, 0xc9cc4505280a1741ull, 0x9ad00dbd9f28cccfull, 0xa91dfb42c46996f0ull, 0x0cfe365fa3a7f8a4ull, 0xdd5facbab53f6c53ull, 0x31917d5e34fb12ffull, 0x8f35bd081245b546ull, 0xc53a285cb9b92e2cull}, + {0x88b8610f4a0ee6c1ull, 0x4718c94096d5fba2ull, 0x4bace0c9e089d67cull, 0x367084cb2f87988full, 0x455fb326a2cdcc2aull, 0x77df8959299d06faull, 0x465b22db94e3d971ull, 0x8a24c376702d24deull, 0x23f2084307b17e90ull, 0x64e2d6195167888dull, 0xa039f8e60171e0c0ull, 0xa2380198ee4fbef0ull, 0xff4317aec59d4c04ull, 0xa2c6c4f91947314eull, 0x39fe9c4a6afc58caull, 0x32d0bdf9d5327b90ull}, + {0x04dec3c18b86a69cull, 0x9baa5f0010f2b68full, 0x81b80bea7753cc44ull, 0x467504309cf109afull, 0x592ee61826792c18ull, 0x8520edb50ae01f5bull, 0x7a5e8b40fb10dd55ull, 0x56bc976f937f9d3bull, 0x13629e73ac9d7934ull, 0x19931dd9705615d1ull, 0x430e50d3d5263b42ull, 0x2b35dfba8cfa19acull, 0xb5d8384245c76661ull, 0xc5c190edfcfd3c25ull, 0x5cefac1a44189dbdull, 0xa65e55e9f1a9c805ull}, + {0x4eee8e33c250493bull, 0x8f74eae73aa2b0a2ull, 0x9581c4dd42066c25ull, 0x5543aab84bbe568bull, 0x0c7f38c0c08f3e07ull, 0xca6b26ed2b0eddfdull, 0x29fd42b68e59e30bull, 0x68aaa60bc7bca8d5ull, 0x0261362fd5c5c0c5ull, 0x28387eeb27e1bd2eull, 0xc02b24e580f5771cull, 0x19298dc47628fa9full, 0x9993103750207552ull, 0x16ca315739899184ull, 0x35e149b2f4580049ull, 0xcd341c1b496474f0ull}, + {0xff42f8f79f787ca7ull, 0x9d8c84eff0473161ull, 0x083ed1787d5836a1ull, 0x804bce0d26a98a10ull, 0xb5444633ad7ad9dbull, 0xf40bae606baee704ull, 0x8db874f6db5e311cull, 0x25e5355cf0eb9686ull, 0x1bc7d5e3ee59a250ull, 0x342ed21aac0e1187ull, 0xade41796fd580959ull, 0x1d1e374d44554f83ull, 0xdb398741136eb717ull, 0x6b7ea78b55c85e6cull, 0xf05cb4c1721ce2feull, 0xed78e39a872f5657ull}, + {0x83cf408958f0d5f5ull, 0x177fda20f46be8d8ull, 0xe2715587ba366b40ull, 0xad5e8da1a1d9dc14ull, 0x55987102b3b23b6dull, 0x5038db89334b519full, 0xeea947b4ec8282baull, 0x17ffbf3ff503abdfull, 0xe2aacf833c417087ull, 0x1bc3e74c17b53fbfull, 0xfbae3bd3bc19f8a7ull, 0xe6ea457da0e19c18ull, 0x140ceb2128a14e03ull, 0x2ed05284a43cbc07ull, 0x87a0be710b420c90ull, 0xc88232680db7e144ull}, +}; +// The Grain Cauchy MDS (the instance's alternative, `CAUCHY_MDS` on the host) and its sparse +// partial rounds (`scripts/poseidon1/p1_sparse.py`, the paper's Appendix B): CAUCHY_B0 acts on +// lanes 1..16 once after the last initial full round; partial round k adds CAUCHY_C0[k] to lane 0, +// S-boxes it, then new0 = N00[k]·x0 + W[k]·x[1..], new_i = V[k][i-1]·x0 + x_i; the folded +// carry joins the first terminal round's constants (CAUCHY_RC_T0). +__device__ constexpr uint64_t CAUCHY_M[16][16] = { + {0x5988b85748e01d64ull, 0x6ce5b2a51c028cd5ull, 0x97151726a74103ebull, 0x48da805da22db860ull, 0xab527bddf567ce5dull, 0xa62c7a8802d557bfull, 0x682f001eff55d953ull, 0x739f833269079c91ull, 0x52f36cc427b3c2d1ull, 0x207f3edd31fec0c1ull, 0xe861a14803653f31ull, 0x7f75391531d1351aull, 0x9c1ff22b7b6519c0ull, 0x33b1b3b9f6243b4dull, 0x39a21454890d9e9full, 0x5153a55d83d32ad0ull}, + {0x2c7fa8fb1e89be4aull, 0x53aaf9e104e4e906ull, 0x5466ca73c94286dbull, 0x01533ab42def6283ull, 0xa1d91119905de8f2ull, 0xc4a70e83b3da8175ull, 0x9c4fa46f80a875b7ull, 0x2ab64e023d7a4574ull, 0x95870ac3b43221adull, 0x8ec4c7dfe0b1e529ull, 0x29af3c3d9caab861ull, 0x0f5e03a41985622full, 0x6e351f8b80d25233ull, 0x34c392b28480b011ull, 0xa2e21b3c21902702ull, 0x98c40b93bfa7d635ull}, + {0x3669f4cbc4539d7eull, 0xf8fee0879daec673ull, 0xb732c1f23b761aeeull, 0xceee88ee79daf441ull, 0x86c99724e2cef313ull, 0x1cf1f3ab213a0d3cull, 0xd5ea051783a3122aull, 0x22352d4ea73f6045ull, 0xf383c05e03cd7c62ull, 0x81dfe9c8a0051bbeull, 0x3f07ffcdd9266723ull, 0xc93603d00f5dc893ull, 0xe69d2645321274abull, 0xd4d06b40d3b2458dull, 0x3c3b9bf934fc9fd7ull, 0x6c3ed734df9cf691ull}, + {0x3c3edb0d8e367cbeull, 0x4d141f636fb94d24ull, 0x593dc28f61908b6dull, 0x51429b67ced5b2eaull, 0xb8ad92e65afe1948ull, 0x7de771ac1136a634ull, 0xf2bc3d4d9f8c12b4ull, 0xa00899fee1163f9aull, 0xefd9361aba69c5f9ull, 0x692af10ed750d360ull, 0x203a34814f5fdba1ull, 0x22aa4b0fc8e4f127ull, 0x8404c6ef28881b6dull, 0x332c23191cb610fdull, 0xe03b41f9d0a5d63full, 0xf6bc920a03f7752bull}, + {0x35accf0a901fcb8dull, 0x8dcff51324521386ull, 0x57e239c58d0c8e89ull, 0xfaed102b92f35770ull, 0x3e4bfd2caf5c21f7ull, 0x3cd53e002583952aull, 0xea2a4a22eb681b2cull, 0x670a2ee21d7014ebull, 0xdde0a9aa6df4b771ull, 0x95b1fbc56e1a6c39ull, 0x6346cd06cc929757ull, 0x6ae4bda639af92e0ull, 0x77314dfd51e7d92cull, 0x0391e3978ff24b17ull, 0xe27611f117d6d97full, 0x04883758be9f54e3ull}, + {0xe6d3dda85e6395c2ull, 0x0cd42c05f66b3c1cull, 0x698359374266b60bull, 0x9e9eaa8893e52d50ull, 0x034432c8f57c7559ull, 0x4674cb4bc8006eaeull, 0x92010504468f2681ull, 0x182baea6c583b9c6ull, 0xfd348e1d8e95d693ull, 0xc2f2fd9b9078ece9ull, 0x1aba1a45a6419bf9ull, 0xbcc2b6584c1af690ull, 0xad0b58758270f4caull, 0xd4e24b7e35851406ull, 0x9572342e0aec74bdull, 0xf681bb067f7ac76eull}, + {0x30f21444e981732eull, 0x0a4b4c356cf81323ull, 0xe0ddde574b7bf32bull, 0xc3f99cc853e76784ull, 0xe1aa2dd912c46532ull, 0x54531f9f67108f80ull, 0x8639b004a91e6a28ull, 0x8cd1ddb584f7a9acull, 0xd78dad0f1337bba4ull, 0xc481f8f3a4b87f91ull, 0x4c24f9db5e94ab02ull, 0x1c62f47a8c19eddbull, 0xbf16cec4fba38913ull, 0x4ed589b4fd6bbaf0ull, 0x86cff751dac92c05ull, 0x86249af3a8eff0f6ull}, + {0xe1033c0d774c285dull, 0x2f8586c1921d8fb4ull, 0x7441a10af45e5808ull, 0x8d8feb70aded8628ull, 0x7da45cc544d9ae35ull, 0x767f12ba8a5a0c14ull, 0x620efed7f8d3a459ull, 0xdd7ff646c99f70adull, 0x3678cc601846b44aull, 0x6755b4209c3401ccull, 0x8e0ae4aa7035b0ebull, 0xba9b1267dd354322ull, 0xfadd3d5f0683e9b6ull, 0xbc5c35ae68f2dfe2ull, 0x73ae2e3c9c090886ull, 0xcc0c1c1e8d9bef49ull}, + {0x8671921f7a0a1794ull, 0x9cd5f1eac6037499ull, 0x3e527bd0148ade83ull, 0xaa717d8659a564c3ull, 0x548583860ccd6913ull, 0x0df5a873cf6cef5full, 0x6b1a27e8227f6df8ull, 0xdb3e44aa830df1c7ull, 0x71353fc324da6e34ull, 0x519ed83e6503d741ull, 0x81d74e3914f76769ull, 0x6c8a3d26ef84b973ull, 0x48033d71b77fe8c1ull, 0x0b845bc4926fb555ull, 0x9e07dbc3ba174490ull, 0x0ec39acc7f68d8ceull}, + {0x7677a79600252463ull, 0xf3e2c73584829d44ull, 0x554b023d8621a4deull, 0xf8afcc7e5c120419ull, 0x385e05151f117f7cull, 0x6f349729b45b5be6ull, 0x3a98b59f73759ee1ull, 0xd98152e584082edbull, 0x795f0382526cb465ull, 0x3d7631494b438675ull, 0x1003b04cc5497839ull, 0x0f224853d28542feull, 0xf83d612bda7234b5ull, 0xdbfc370b88104851ull, 0xc1dd2e22be7c3a16ull, 0x17ca5948341a933bull}, + {0xa9d70e4e8c01ad89ull, 0x47bb5a0c5a8da9b0ull, 0xc7b5e1da10d895f7ull, 0x5f0cc2c2aaa7b03eull, 0xbc8097f44e63863full, 0x70c8e880581bd43eull, 0x758cfe2ffa079581ull, 0x839395f285d3ece8ull, 0xa2f27080ac482121ull, 0x3f62d1fcb8f9d784ull, 0x4b2c662ed6ae100cull, 0x14f3fbce248827f0ull, 0xbda2a9225eda8792ull, 0x1ed0e69117ef3c36ull, 0x3af8405a89b5c1a5ull, 0x3429a7904ea79c9dull}, + {0x9d8119220744a23bull, 0x730bf95139f665b1ull, 0xf71f347cf06eb11bull, 0x1e16d46ef71b8e86ull, 0xffb5517c741183a7ull, 0x9de7bdf6e7763c76ull, 0x641535f22ebbb8b5ull, 0xcc9a2ac6e3525ff2ull, 0x6ca74aadfd1870b2ull, 0x26c7b78d1a569b93ull, 0x6771174eb61e5363ull, 0xfece43af74186a6cull, 0xe2d04ce9d92aac6cull, 0x069bce14795ab3eeull, 0x454663cdef4ad361ull, 0xc9d1d1b13aa366baull}, + {0x2a14bb5529f46c5eull, 0x9284dc3ec5a3298eull, 0xc32a357d30aa5298ull, 0x6378633155977074ull, 0xd42fbe0d1987c1d4ull, 0x812a7402da911fc3ull, 0x0aa377c56139e86dull, 0xf738501437323e35ull, 0x716814c20125dd1aull, 0x4b8f884ea190466eull, 0x73af1afa89a8b0f8ull, 0x581257e0a6514156ull, 0x2a3e9650920f7c21ull, 0x3a4b286082ce7d23ull, 0xaa89fb9040aed6faull, 0xcbcd108576e6d1baull}, + {0xa0e6af946333f9ccull, 0x5b9fa025d5f91de4ull, 0xa37f3398a3ff534cull, 0x5d807c5491b334ccull, 0xdde0491f392f54b3ull, 0xd31f9255564584e5ull, 0xb682b5175a329122ull, 0x36ac21cacdf0a097ull, 0xbd69c3e422624ed0ull, 0xfd03026489b74c0bull, 0xb2b862a95c6f5fc9ull, 0x7a6ec2a715549e03ull, 0x6a3cf97409861804ull, 0xdd31032efb7843ccull, 0x080d82ec2393f0c3ull, 0x5ca5e139dcf07d41ull}, + {0xf57bae2bb3999ef3ull, 0x1b3241817e6f40c5ull, 0x12f1d5f21156e7b5ull, 0x44261d736bf454a8ull, 0xe1194d1a1cd17a51ull, 0x9394f656c1246dc8ull, 0x050fe6e2bb26bab4ull, 0xcb184bfb067f33a3ull, 0x87b5f94c9d2c7907ull, 0x456daf135d96f358ull, 0x116eba266dcc0cb3ull, 0xa2dbf3c480582332ull, 0x1302f5ef13049a4dull, 0xd2ac7ba5adc5269aull, 0x43b32aedb14c08ccull, 0xe00837b7287de52dull}, + {0xd9553f671aad6e14ull, 0x4bb5271bb83f98a0ull, 0x2c84ca7932ff0d9eull, 0x0ac13c2ec615ff05ull, 0xfd066e3f8f82e427ull, 0x6cddcb7116f97320ull, 0xce298f1a321e7087ull, 0x74c9ceb2944dc1b5ull, 0x6afbdadac658ec67ull, 0x8c32f317e710b3bdull, 0xc380701ec43dc56dull, 0x96b0a79e387441a4ull, 0xcac77034b3035048ull, 0x015fd2a7e0a193acull, 0x8f848885347161d7ull, 0x110276a5157b853aull}, +}; +__device__ constexpr uint64_t CAUCHY_B0[15][15] = { + {0x559d1b8e8b3e4024ull, 0x1397f67357f0186aull, 0x32d3361d249892f6ull, 0x2855f062fce3be81ull, 0x136cdda8b9d5c7b8ull, 0xbcbae04443e7ab0full, 0xce3d73ae3d0b6fbeull, 0x3ea1cf9803af8191ull, 0x50712b5d381c341full, 0xcf5d4c0e3647c571ull, 0x57532e272531b749ull, 0x20748e6c7f50d289ull, 0x2670acadcfcf7ad0ull, 0xe98366f8f36969a5ull, 0x7d294991b9ad19a8ull}, + {0xc4ee6ac78f74cb10ull, 0x84c443ffd432eaedull, 0x0295acb35bd579ecull, 0x3e76bd946ce86d98ull, 0xb91aed4e102a8759ull, 0x6796661dd9cb7698ull, 0x0a1f3ad035369182ull, 0xf535bfa4f214bbf5ull, 0x19315b9a3a96c6fcull, 0x80833bcf93d87a2dull, 0x459aa19620f859b4ull, 0x0754d188b4651faaull, 0xfb3cea4c26e3fcf6ull, 0x86b09c03d0ea4c45ull, 0xd8a5f8ffe24ec4ebull}, + {0x3cec8b2de54e5913ull, 0x8d26e7a2e6f0639dull, 0xae2556adf17782f9ull, 0x419345661c863f06ull, 0xb9e47cbe39e6c4abull, 0xe9772fb2a7b4feb3ull, 0x4dd69109821e7f11ull, 0xd8c68bb1d4f1fd2cull, 0x3753a7ea7778b6b3ull, 0x0b98dbd36cd73241ull, 0x73bb482cac611072ull, 0xb70ad83fd457464cull, 0xd2b1daa126e7afd4ull, 0xb7ee25700147fd35ull, 0xa794f817f66bd15eull}, + {0xcf129d394c00d855ull, 0xfb024afbefa84911ull, 0x7f2433bae92446dcull, 0xd954925b033ae4f5ull, 0xf367b5825aa39848ull, 0xcfcfeb4ec242581full, 0x2c05f6d8e434be2full, 0x79fd56917608dc75ull, 0xeebeb593b3f40213ull, 0x3b478ff7078d0de5ull, 0x9028a8c228ba6407ull, 0xa59ff096eaf95679ull, 0x7a39bbe867eae9a0ull, 0x1e738dc9b9a5fc68ull, 0xc7b21942ac090d93ull}, + {0x5f2b4d373d5848f3ull, 0x03d1878ba215cbc2ull, 0x69f591ac542e5b32ull, 0x0273d5ea865825b6ull, 0xc3cedde16211c8a3ull, 0x6a5ed60000793e5full, 0xe1635ea18a50b3bcull, 0xb15875a154a66dd9ull, 0x0014cacd53d61805ull, 0x5bff9804c699ebd4ull, 0x69f01779e9a88412ull, 0x3b6978c545d5669dull, 0xf27d206842b1beb8ull, 0x76876ad1f691f3d6ull, 0x2867edbb56802b93ull}, + {0x43b2b3896929fb8dull, 0x654908340051fabeull, 0x78d32edc1b7fd4b7ull, 0x20e3c29875f57c00ull, 0x75e09581bd9df08eull, 0xe422000ecfad6bafull, 0x5df6eb0254d16021ull, 0x0ace093566795ea6ull, 0x7e1aaeb88cdfa14aull, 0x8eec7a8e23fb09abull, 0x827484382cc03df8ull, 0x1146d7f4b1826096ull, 0x755ad381928047eeull, 0x37e4912b7d9fbbf0ull, 0x6b24abcdd510e693ull}, + {0xd812b53b2d5cc829ull, 0x7a8c0c932752d4fdull, 0x728c0fb4a24c4f87ull, 0x8eb73c9b649e087full, 0x8abcc7c507d5b9b5ull, 0x0fbecdd48b86354eull, 0x631615fc3c10f710ull, 0x52ade4de22488123ull, 0x5fadfbea34707fe5ull, 0xd10ccf129a7981aeull, 0xa6ec425e3fee20d9ull, 0x55cf736220e4c5d0ull, 0xc663df9e5fd71bb1ull, 0xb899f86948345ed2ull, 0x2c5ee71cf9f502a2ull}, + {0x6fdc911ef663054full, 0xd4dba9470bfda7afull, 0x29b981ba5e9f19e1ull, 0x86ffb723a114a384ull, 0x5e21d6d7c7ce8bafull, 0xfd6a85aaf540f9d3ull, 0xcfd58a581ab8cb6dull, 0x8c7a61791706bc51ull, 0x7fb71883c40dfe3bull, 0x8e8d2a480f97e941ull, 0x4a7bf87d697ed0c8ull, 0x6701e174b86be60dull, 0x1e8d4ba8ea63203aull, 0x4083d2deae16e0adull, 0x559bb4fa1723e820ull}, + {0x8061819eff966b42ull, 0xaf2f62975d768aedull, 0x709582e84457550full, 0x9712867f3458caa5ull, 0x9acb09c5e449c97cull, 0x0f7c8a5be97f0e80ull, 0xe6fe1995a8ad4f3aull, 0x31912fd0a507b93dull, 0x6820d729758a2ce6ull, 0xff6066277f8f8e02ull, 0xbd642958b7049f47ull, 0x81ea527ae2599714ull, 0x5831da7c495a34f3ull, 0x2f55cd35d917db7dull, 0x9f69baa44558e2b7ull}, + {0xb9301f8ef88358e9ull, 0xf2f5ed02a9d96ab3ull, 0x26ebc858ce77bb60ull, 0x4de9d4062b84614aull, 0xa33c130c4fdc2133ull, 0x96ce8b035cf1bab1ull, 0xfdc4d29224666515ull, 0x13deffe42c2fb55full, 0xbfe2e9fc6dfceeeeull, 0x750c56619f2e3992ull, 0xd43eed4428061768ull, 0xf271b2188ceca2f0ull, 0x68e0d07180bd28ceull, 0xe5af1c59f325a409ull, 0x05a5888cae904351ull}, + {0xe5c6f200fe3d26c2ull, 0xb8cd6af9aa4e9d45ull, 0x32ab26847f1cba47ull, 0x6e32e5626a358755ull, 0xc27efbc10a5289eaull, 0xb3dfba9dbf86dbdeull, 0x2bd2a14a8fa2101full, 0xab67eed3898e3276ull, 0xe23a50d0e4429bdcull, 0x408236671be3497aull, 0x61fb25c0524abc11ull, 0xfc6d88bc7ee7218dull, 0x83c2f0f5b81f329eull, 0x7bf75974fb7ebd2full, 0x64ba1013871e1886ull}, + {0xd14130fe8b153e3cull, 0x80bcc03a6b6754b7ull, 0xdafe5980e7322b44ull, 0x482387134dada3d7ull, 0x5489b2f5f0e4be53ull, 0x31a56acce544d03dull, 0x35fa085c63c90308ull, 0x2cdf1f25a659c81dull, 0x5142e0c3ef1c615bull, 0x475344586f6b2f84ull, 0x0b9e2cf2e96e072dull, 0x6470da091be84331ull, 0xa39ebaefdf449d9bull, 0x7633676073853c18ull, 0x246d164a86c60e41ull}, + {0xdd22613d6780719eull, 0x84a890f507740351ull, 0x18f951c60a994107ull, 0xeaad1e7da92379f2ull, 0xda66880d9299f3c6ull, 0x542188854fc1bc24ull, 0xdefe3f2df857a907ull, 0x669f40e9261c0413ull, 0x883b63d8b781db34ull, 0x6292763383750670ull, 0xf59cec3273e64cccull, 0x1dfcea105d26b219ull, 0xb804e059e39250b7ull, 0xfc8ff58d00ee82fdull, 0xa83dbd6485588da6ull}, + {0xbcbfd09e59fb34bfull, 0xd69ca89c22ff6a49ull, 0xa2985cb015a85596ull, 0x684efb2ed9f55181ull, 0xc848d747a33dff63ull, 0x832b1b8eb1bdf99cull, 0x118c47150e2735ebull, 0xb5bffe1863cfdddeull, 0x813340c337165979ull, 0xb4857c56cc781247ull, 0x0aa4eea357f0cdd5ull, 0xee5494bf91c44b72ull, 0x3598d1556200532bull, 0x960cef171f7d3eecull, 0x5778dc4fd0b27eeeull}, + {0x143b9f409a8163cdull, 0x024c50f810929465ull, 0xa816f917fb98cea7ull, 0x202964327d60d466ull, 0x06624dd884c5bc63ull, 0x0894f5bdef7002aaull, 0x4287900d769aa563ull, 0x190f08376f6fc031ull, 0xcaf795056ba3f750ull, 0xe664c38623cbbf74ull, 0x26e9d7793f325fb2ull, 0x92d64f871a7be92cull, 0x46f27e19a39cb575ull, 0x5335539edbe522dfull, 0xc43233f028b7bb44ull}, +}; +__device__ __constant__ uint64_t CAUCHY_C0[22] = {0x28eff4b01103d100ull, 0x217f03f61640248aull, 0x6495bb39a57eea2cull, 0x95cb5f5e448e4a11ull, 0x3b849dd45ab59647ull, 0xc8e5023ba7f7dbefull, 0xedde999aff8d0895ull, 0x09a4e28f85ca7bdbull, 0x3b39353eb6408246ull, 0x2af66528c0d36099ull, 0x55f047a193e7f8bdull, 0xc54e4289757518caull, 0x3283aeb50ca24583ull, 0x13fc13c17ceb85bdull, 0x13a36c7d33ed227cull, 0x88f8fa956dbaba55ull, 0x62a1b3743fe440a4ull, 0xefa5e0c905008e63ull, 0x2518fea84a7829f0ull, 0xcd072119089ace60ull, 0x716eea92d49fab22ull, 0x88b8cc50b731e711ull}; +__device__ __constant__ uint64_t CAUCHY_N00[22] = {0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull, 0x5988b85748e01d64ull}; +__device__ __constant__ uint64_t CAUCHY_W[22][15] = { + {0xda9bdc3c4e53a24bull, 0xdc8ab3c6a0aada65ull, 0x180e4a11e624b8b4ull, 0xc2c18fc4a3631950ull, 0xd6cbcc51dae2ca6bull, 0xd050c63b62c44d8cull, 0x53e58bf06e847624ull, 0xb1800ab08e5d3eb2ull, 0x67efdd62fdc33060ull, 0x9c53674d46d93492ull, 0xb8c5f0f2235b0c81ull, 0x87cc335bcc56fafbull, 0x259a7314ecfb12a7ull, 0xc8f764c610867069ull, 0xb3996f1e02f1433cull}, + {0xcc64116f3b0ad6e7ull, 0x894638143505f25bull, 0xb1e9ed35ed28ddc7ull, 0x2c22889c49e73a5aull, 0xaf124824ea389c59ull, 0x97c57561e1f7ce43ull, 0xbfbd60b5bff5eaf3ull, 0x151918194808cf18ull, 0xd0466e385181b7a1ull, 0x885d27cfdd92109full, 0x2a4f864cbd0bb1d4ull, 0x9ea0f412ea4a9e66ull, 0xbc719b492d1f014aull, 0x1ce7729c89bf56cfull, 0x5eec4f075731e98bull}, + {0xb3b294ae3bad1b2cull, 0x7f5d1ebffdf3f346ull, 0x13ed77a93cff565bull, 0x3198b6fe59701b80ull, 0xf8ae739952a5e119ull, 0x742a6f47d06efa64ull, 0x5a46103efe025629ull, 0x6d3c693b52aeddf2ull, 0xa8a1137557ac770aull, 0x0d1c80fcaa87e2e6ull, 0x1320a78a858ac140ull, 0x9ee6b02979e749e3ull, 0x1ad3240c8c589c7bull, 0x8123a386282f071aull, 0x099792bfd7970f8eull}, + {0x990e97a11b84a9b3ull, 0x489e5b247460cd5bull, 0x9b3cdb9f9233dd6dull, 0xc9fc89cd6307269dull, 0xf583862249eb0cfcull, 0x9f1b7bcae75bbdf4ull, 0xacd1da32b03cc334ull, 0xc28d6e31a0e071d8ull, 0x8b99e4b388dd918cull, 0xd183951158089445ull, 0x01a28d8e09eb2efaull, 0x1016643ea51883b0ull, 0x1d52b9231eb403e0ull, 0xc19f26c8fc7d8337ull, 0x97ee9670ceca9449ull}, + {0xcbe63bb9fd569002ull, 0x0c89218eeaaaa365ull, 0xc6df6144a5ff1829ull, 0xf14ee86eca82e85bull, 0xa3b3945d8dff1eb0ull, 0x252569d4f55a5f1cull, 0x18fe8bb55a336efeull, 0x5fa47b298582e2abull, 0x71c14a9d4952abb2ull, 0x238ce3b173707e8dull, 0x044fa3d8d4fd8ec4ull, 0xc7fcc4ed69040347ull, 0x85f172c1ef6ad540ull, 0x43bcc534ae03d074ull, 0xd87865e6bfa61987ull}, + {0x25ada4f4be566e39ull, 0x2fa30d52d8ebe269ull, 0xea010797ede26abaull, 0x3ced3e8ed9ee0598ull, 0x814802ec77a62350ull, 0xcc50fba1150996ebull, 0x8b4071219eff723eull, 0x535132b778ba43e9ull, 0x2d2b6ae4cc56cd82ull, 0xac86e9845166be60ull, 0x7780fc69d4e949d4ull, 0xc6038e6a82273424ull, 0x4c3c627feabbf720ull, 0x635fd24cc2d4226dull, 0xf75b293365d9d993ull}, + {0x5ba7da49c3cab6fbull, 0x803705bb82ed38d6ull, 0xb23e23be91da09dbull, 0x6fc25cabac0937e3ull, 0xfd9e38e6ab0f5186ull, 0xf7a7fdd2765cdce4ull, 0xfa886d67857eaa29ull, 0xfae86f609822ade1ull, 0xa176772559f39816ull, 0x68cf842ed5e46433ull, 0x74d5de78191cc535ull, 0x5e9bfa491789da92ull, 0xde53e681791a254full, 0xe76aa196ac12ba21ull, 0xace11d0f8d890153ull}, + {0xc045677ac28c2325ull, 0xe7f914aa01f42593ull, 0x0846ebccc2501ff7ull, 0x2e9804ae9538ef74ull, 0xc8d4d252c1bd7fd8ull, 0x8cde2fd074ce1891ull, 0x746011eccad80f27ull, 0x52be5e82e109e294ull, 0x3ea49f749047d443ull, 0x7e02894af640132bull, 0xdac33fd5405e7a2dull, 0x9c2bd51ec53eee24ull, 0xa8dc3c11c453e6c1ull, 0x887b43b53ca6e6aaull, 0xa33631fae25d6323ull}, + {0x6a3213edbb8f0fd1ull, 0x652546b3ff5cfa5cull, 0x8cfadd5eb0bf518cull, 0x6565c3d30035191dull, 0xb3da736231536935ull, 0xe4cd7b36aa19723full, 0x74b5a69cc0dc7a21ull, 0x50c0561024e84ef5ull, 0x810dd6e9dee3f858ull, 0x54c49cbfbcdb7cfaull, 0xca0ebce5c487945aull, 0xafa8f8c967459e35ull, 0xfd41dfd75c44a3e7ull, 0x2ab1668271bc19e6ull, 0x9d3b1f2239306d08ull}, + {0x4cef9c3c4c0a6980ull, 0x6f0e80d79b4c824cull, 0x98bf753c8c778a87ull, 0x6f5193544ccee4d2ull, 0x0d669d58c13437f9ull, 0xf4bd7e6a1cafc6abull, 0x71364c3789069110ull, 0x811c611d15a8d8afull, 0xf2888eca813ca1f0ull, 0x50cf3f1ca153ca49ull, 0x6c29c427529881d1ull, 0x07110f5bf44ec70eull, 0x8dca61ba9008b16bull, 0x4120a34081d6df9aull, 0x7272d469f2a27342ull}, + {0xa54e15ab27ece0f2ull, 0x8975467e73292cafull, 0x935519402854e996ull, 0xa1c2411a1d35c88cull, 0x03d3f713becf647cull, 0xcc08b0ef3ac5c9c6ull, 0xad957f25cebd0561ull, 0x0417b22b5190e512ull, 0xf7a9a4d6e886dd22ull, 0xafa8e38cdeac1cb0ull, 0xd2aeb503d96cc1b4ull, 0x740aae9e0cd731e7ull, 0xe72b7b2dad84973dull, 0x63eac4a16b927a21ull, 0x21de0c44b0723e6cull}, + {0xe5d48b956d8a3f6aull, 0x4d4450216e0d7998ull, 0x5dd90c0aafd6ac59ull, 0x9441bb73562d8c98ull, 0x40fbd2d79a2bd076ull, 0xf28c7cbdbac7a61cull, 0xc853ff35d06d4845ull, 0x9cda159c8be8c7cdull, 0x54945958d2d473ebull, 0x2beee11dd45c8dedull, 0x6c88606cfab5cc1bull, 0xf372845c54534623ull, 0xaf285bcd71ab60b6ull, 0x53f3a918b3bb9d1aull, 0xe2196d8b90aacef6ull}, + {0xf85f92d531fde335ull, 0x1159a41a28463500ull, 0xe8101bb3c582399full, 0xd485ceb8847000c4ull, 0xbf4eded27ba2ccf1ull, 0x6de5f3f461037514ull, 0x29460a580281bd5dull, 0x72498aca7dca6ec7ull, 0x8965adbe0f09d4bdull, 0xf9a001da0fd03612ull, 0xc639cbd8acf4ab13ull, 0xf76b3b8f461023c7ull, 0x4351c9a9b1579328ull, 0xbbfcffa6721185b4ull, 0xa7d3ec6a9efd142aull}, + {0x5b6aa9a225fc58a2ull, 0x0bad61e471ca4927ull, 0xfadd80064edbfff4ull, 0xa66f15081bb8dcfcull, 0x35f597162b56bb6full, 0x9de7e56d8233c666ull, 0xc33e1d4819a8e538ull, 0xe11a1e8a42de152full, 0xe340cf6fb53f99f4ull, 0x1f79822a19d8313dull, 0x7075a8d0bdd0f431ull, 0x493539312e48e3d0ull, 0x277d52b10042b9b9ull, 0x7d3e4573397dae4aull, 0x324cf7f88af1e4f6ull}, + {0x2d2c65d276e64330ull, 0xc383f771d05af65eull, 0xae489f5d93843694ull, 0x67e52cf1d9a183d7ull, 0x8b0937cb215dac9cull, 0xcf1878dc486e1324ull, 0xb2d5ecffff067689ull, 0x128f9f9a06e73ea8ull, 0xc9c73d37838a8dcaull, 0xc7fd34eefefcacd4ull, 0x55187e5f63f5d3d2ull, 0x29af3db7e0a964c4ull, 0x0603504e288f9f21ull, 0x8f4d0c002ddcc55aull, 0x631657c86eb2e78bull}, + {0x31becb31f32b0723ull, 0x625d913fc1e56fd7ull, 0xe8850e85bc04aae3ull, 0xb528507181c2d37eull, 0x4131be5701c09779ull, 0x9dc828c13b095feaull, 0x9bab730f66c644b4ull, 0xab13c0e460879a14ull, 0x954e03be4fc138f2ull, 0x8f218dd2bbbfe45full, 0xb39a9feda9e12366ull, 0x563b5128884cb95cull, 0xa48a3326e01629a7ull, 0x35423abc412de206ull, 0xde7ce264e3f9d507ull}, + {0x03653222b124bb4full, 0x2b34403f9348b57eull, 0xfdcf70d5d9254eeaull, 0xbbea2dff9e607f95ull, 0x653ea68d5bc260eeull, 0xe15c2334dbd8e9a2ull, 0x6ecea57c4838079full, 0x9e85a012b82f30feull, 0x0ad1a95a9a8478f6ull, 0xee1dd1f6326c52fbull, 0x6e98e6ddb3f7cd37ull, 0x41d4a5def648f696ull, 0xa9243efb5849b6adull, 0xf7f6119b407eee70ull, 0x3208ed0d4f25c31bull}, + {0x5c1e57d7ba9f5b6aull, 0x76ca6cb60cb034c9ull, 0xa6bdb1f5345ed89full, 0x30babfcbfdfb109aull, 0xbf2a519b65474d96ull, 0x7557597029414fe3ull, 0xeeb713f380aeeaf0ull, 0x02f6bd83f1f819baull, 0xb08aaeda7170305bull, 0x46e5c354770f6795ull, 0x293495166e17008aull, 0x901a67f2b2032b4bull, 0x06a58cf7fb251858ull, 0x4c8866e571f4886aull, 0x36d66559fd250007ull}, + {0xabcd4d44d91977b0ull, 0x99f20bcc147642b5ull, 0xefbba37729a1e70full, 0xbe08dcacb34bc3ceull, 0xe70255c85f6e02e9ull, 0xf447e3d59abcc907ull, 0x17b51429ac619cc3ull, 0x0bc3d0efa9977f16ull, 0x96eae50dd3afa83full, 0x69cbbf30d7aa577cull, 0x51a8b8f39293b782ull, 0xa3d871d1d322187dull, 0x5a5db39a3b1412b5ull, 0x4a27c7b6459972c7ull, 0xe02216746361168dull}, + {0x61752a156e2c80bcull, 0xcf0be0dd6d24592aull, 0x4f436a11e52b4c32ull, 0xde6626628d1ef21cull, 0x80507068e8bead2bull, 0x7607816f8984a47full, 0x0658f0ea37061407ull, 0x258b51b1eef1ee98ull, 0xe441e84ec47246e7ull, 0x58de4099c448911dull, 0x4d7822b6d7982401ull, 0x1bc4604a7987fb4eull, 0x361303a0148b5d39ull, 0x5e8a4e3abe283f72ull, 0xa6eef3785ebb385aull}, + {0xcd9a6fb9c550da8bull, 0xb4c9fc68e07feab7ull, 0x7184b49bb7ffaae6ull, 0x7ecd4c62efe61ea7ull, 0x0880955ddb1389d8ull, 0xf3798f93b1d7915full, 0xd92be292de0d4a6aull, 0x61f7afe1a2e8b24cull, 0x018dde109a326589ull, 0x833a7b5c1e46a4aaull, 0x928ffe25937a99bfull, 0xc9fe60de55b5a0b6ull, 0x4f86c8957a5bc055ull, 0xb5d1f9d0c3fcf715ull, 0x6987bfb754383bc2ull}, + {0xe7c631cca41a534full, 0x9c6e7d7bb8398e42ull, 0x4cc79b69d9f50959ull, 0x65ea4f0d08470b5aull, 0xed186c9fcacc5cdbull, 0x0b0d3411e70a0459ull, 0xe5898a042bd64324ull, 0x3f0547740aae663full, 0x8222f3cb44ccc6bbull, 0x554511199d81399dull, 0x39da2920295f821aull, 0x64b43caa71f815a9ull, 0x92bf3817d44ea334ull, 0x90608266aea17de6ull, 0xe1e1dab00de4b3bbull}, +}; +__device__ __constant__ uint64_t CAUCHY_V[22][15] = { + {0xb2c11154e1e17d06ull, 0x08a54769a9c34a1aull, 0xa875b2bf597af7cfull, 0x94d9866b0418d19full, 0x497d6712de056585ull, 0xf31f856a168705f0ull, 0x6729a995e8c5bcecull, 0x13500e762b721bc0ull, 0xf4aeb710eb508e7dull, 0xffdc836bad54d9a4ull, 0x0a224a83de7f6a10ull, 0x1375ac5fdb3e43e1ull, 0xc11483b53270bb94ull, 0xa918fc1dccab2e30ull, 0xf7adee46053ea9b2ull}, + {0xb2e3ea7121f5b603ull, 0x8c2ae1be22d373fcull, 0x2879d85b34cee02cull, 0x0750ec695890c2eaull, 0x648f66940d3372bfull, 0x31b9d2c68e1fe7ddull, 0xf006de7371b1441bull, 0x91af44c7ba15c648ull, 0xdc8becd51224555cull, 0x590037ec49909e12ull, 0xa3771da405a226fcull, 0xea6e80141ed0c720ull, 0x0081636aaa57b8bdull, 0x2214b051535d87e3ull, 0xfa64e83f8b6a0dd7ull}, + {0x05f1e6f2fccb5361ull, 0xa0ed3c0478578f01ull, 0xedfddcd46f8f387eull, 0x5e1413216db5bc6aull, 0xb31a6842f92003fdull, 0x3a8b0c91cf09f1bcull, 0x41541bd0f120f2a6ull, 0x9d7074abc178d405ull, 0xc2076602dc331be3ull, 0xb907a565199f712bull, 0x24ab08c680da03f5ull, 0xcaef381c0448e9fbull, 0x36a649fda032c484ull, 0x8978de40d5bc6472ull, 0x4814e5c83005a7abull}, + {0xfbf9988294057f31ull, 0x6393bb7c663eb532ull, 0xb10a2058fe9cb4f3ull, 0x38e99f09215a3c03ull, 0x7f851fde9f1194acull, 0x0eabd636ed5eab66ull, 0x3189a600db811ddbull, 0xa4734bd7273c14f6ull, 0xbd3af602c3b516f0ull, 0xacc9b52e21dcbfabull, 0x254adc65705d5151ull, 0xa50f302ed98908d3ull, 0x60cfa580acf370cdull, 0x14135efc0b4a2382ull, 0x8f347cdbd45eea56ull}, + {0x3ad5bec49ed20330ull, 0x0d1e7ff18170b1b9ull, 0x01746d622eb8905cull, 0xed10a354f2a4c1c3ull, 0xc25c3a082b2a7b10ull, 0xe842f8834ddf9d29ull, 0xd7ae1636fe53550aull, 0x94fcf7d66abe620cull, 0xbc8250815c0ca141ull, 0x456d7ed00bb5c946ull, 0x13ab7d11911715dcull, 0xf014cf954ae53866ull, 0x8851346a04b8abd0ull, 0xaf101798a152eb77ull, 0xbac421ce07bba7b3ull}, + {0x5abc689fd94bbd03ull, 0x646c8057e646fec7ull, 0x3ba484e2d829a61full, 0x70b1aefad2bf62d5ull, 0x8936124044ac9d96ull, 0x8698a9ef11144e5aull, 0x42a07cf7699e59b6ull, 0x81b0261c71a82767ull, 0x928f147408a5b73aull, 0x41524a39c06b6f34ull, 0xa1ca38599aca4c22ull, 0x0d9c2761640f174aull, 0x3d3f97c56e580848ull, 0x3be85e39a27687b4ull, 0x9e4c216c7216a327ull}, + {0x957451069396d420ull, 0xd4bb95b8ff3b8e05ull, 0x00f0e0cda8c381bdull, 0x05b6de735a285801ull, 0x06fe49c0df7fbb7eull, 0xc7e0bf5a0b20c1f8ull, 0xed90d92e5606df04ull, 0x27f433b78e282669ull, 0x91932a0bce60c87dull, 0x3ec44932b706bce7ull, 0x4e5accdd95706a47ull, 0xca89603b8bb5a1d5ull, 0xb77ccfcb2a1f8388ull, 0x19c429bc4343dcbdull, 0xd033ffe7192a900cull}, + {0xef23b28cb89f6a35ull, 0xe62112336e66da5eull, 0xff14457b59c9d82cull, 0xc5979e6852ff0fa0ull, 0x990c4f1e95a1af67ull, 0x5de31a41b6fbda9cull, 0x882796cb77a69039ull, 0xb9e21153d93b4763ull, 0x48fa221ee530ead0ull, 0x4265b64d2d1d6983ull, 0xbf9b9bdb61f654b8ull, 0x101f9b2e804818bdull, 0xd6243c2fea690ca2ull, 0x51db1154ad3c7d18ull, 0xa69c09018636e10cull}, + {0x3562df8adb9fcf3dull, 0xa28082787217d240ull, 0xbb8567fa9a6ff5d1ull, 0xcfc25733ca1c2cc3ull, 0x00c458123b68e1caull, 0x60c4dcb82e4f2d96ull, 0xd751f16685ebd4cbull, 0xe6e2fe94e750d2c5ull, 0xe171abdb26363af5ull, 0xa26d9909f0402474ull, 0x7eb14ec40cce375eull, 0x41b1d43d01381ad9ull, 0x7b6d03a437d5ed99ull, 0x05ea64f0696cb159ull, 0xf3f3c9917591b143ull}, + {0xf1f7ee38df7e2646ull, 0xe0281d35320946c6ull, 0x2f99c30eed95860dull, 0x7ac11021e9e2ed53ull, 0xbf9354903601c67aull, 0x6358c08276e039ccull, 0x26abd13faab591d6ull, 0x922aaa86bbcc53f2ull, 0xdc2333383a5ef85aull, 0x31e6db868edf754cull, 0xf9fcf629fb132e8eull, 0x28ce6b740372b6b6ull, 0xde98a42f43ba3fc6ull, 0x00fc04ea98d905aaull, 0xff911559d4958842ull}, + {0xc5f8952f039198cbull, 0x575f1f2ca296c418ull, 0x687d96641bc939b5ull, 0xb8985fa635f0b6caull, 0x8c3f8ab170a581adull, 0xfc66c76ef791130aull, 0x3ff4530f9aa5879aull, 0xd01dba3f5119ae66ull, 0xafcc1da91c70a3d8ull, 0x5a88d4f11036f159ull, 0xf908ae15fc86174eull, 0xd4ad31e52876e616ull, 0xe893d04c933dee3bull, 0x43e546ce340f0735ull, 0xfef3c4a739017a72ull}, + {0x056b88012bbc70d0ull, 0x7e5a211e08130898ull, 0x9aed73dba56abf2cull, 0x957aea53a9f1c874ull, 0x5aecf9feb0c02d4bull, 0xd60009118783a8e9ull, 0x4e099b36c663c9baull, 0xf77c3d6cbbd3a835ull, 0xcd443eeb013bd030ull, 0x567e0a15186bd965ull, 0xa946f2766b45927dull, 0x6f1548a2ee2020efull, 0xb39eef2afafc5c36ull, 0x1be12eaf897c6446ull, 0x568ff39a4b748d26ull}, + {0x0e0301ffe64bf167ull, 0x35e4135be327e57bull, 0x7bed8bc6486e6b66ull, 0x736c39e3233da49dull, 0x7081f839fbffcb19ull, 0x4513be609be750ffull, 0x9a9bf344aaca41fbull, 0x1730c9cd8cd169d5ull, 0xb1c39feac88311fdull, 0x6dc356752833dd11ull, 0x985562d033023c43ull, 0x7448c65d21e4f194ull, 0xb67b799db894bae7ull, 0x5dd25ab256aa31e1ull, 0x575dc89b31eae077ull}, + {0xf40e040f72cf6ba8ull, 0xb4821be4147f2c55ull, 0x35f2d8ca870b0077ull, 0xf99f93b3549ce90eull, 0xcb70c65308e6db5aull, 0xe70c0f455cfd9f4aull, 0x3ba507762cf6621bull, 0x2d2b4a10dbe21c28ull, 0x3d436451bb9e581eull, 0xdbb494ee0c5f4771ull, 0x30f6bc7b4ec09bf9ull, 0xeb49767a64c3512dull, 0x70a55243372f8b3dull, 0xccc6b5bc6035c2f3ull, 0xb512be7312291f61ull}, + {0x3eb8982dcfd9c694ull, 0x9eb4572ce5c4f342ull, 0xc32b2e9f9a96342aull, 0x1a88997901cea63dull, 0xa2839bffee8b1402ull, 0x77e0a6b7f5954e91ull, 0xee9e11682aa5fc87ull, 0x38cad310c5bb85dbull, 0xbc1455acf3426765ull, 0xd56b94ee060d9496ull, 0xb5f3ffaefa6aa6b4ull, 0x7b3acc191fffc6eeull, 0xc3bb2836dd10260cull, 0xe5ee696b118cc7f0ull, 0xdd9b8d355c997e75ull}, + {0x36771f4b30b78dffull, 0xcc778aefb0bb2822ull, 0x7d998ec4235ba6f4ull, 0xbfb7ece81aa7bb13ull, 0x615ee68a959658d6ull, 0x8c12377a6a6c93eeull, 0x8fc9ebad7c7a7212ull, 0x3bcfa54d96372e73ull, 0x105a5ccfd094e12eull, 0x75d7c34c8ccf4d94ull, 0x4c1375e0a3bfb398ull, 0x48ba3462eb46db8eull, 0xb236db0492c15525ull, 0x9ed26ed8c2f679feull, 0x07a1e420fe176193ull}, + {0x7242f3438cc951dbull, 0xba721ac130194ab1ull, 0x3390f76254916524ull, 0x6254146d6270cd0eull, 0xfca3e8c7985823b3ull, 0x5785f92043564486ull, 0xee990a328eb4a1c2ull, 0x30a80e6e0402b859ull, 0x6b0843a5020d9144ull, 0xc31ff6b4b3328c19ull, 0x57031d9b06eb9711ull, 0x3b4ada658eb3ce83ull, 0x9dd6c960aef32976ull, 0xdb69892f30d98c15ull, 0x736b8e9aaf6270abull}, + {0x3fe0eaed7002908eull, 0xdcb8126e33084066ull, 0x718d428879838b58ull, 0xd31f0593098a0421ull, 0x7bb098ed91f24f1full, 0x14333b0cbb721642ull, 0x200daba0bf897ac4ull, 0xbed37458ef658a95ull, 0xcb40770e6ef5865dull, 0x791b55a0391a0daeull, 0x94a2ca1f6ef545aaull, 0x7866d4a3256ec250ull, 0xd9284c0b172f69b8ull, 0xc34f63aaf58e29f8ull, 0xaee0e33ed6708677ull}, + {0x6b9b3c9f3bf900c2ull, 0x0a656ac64c38bc01ull, 0x04cc9c497b9aa976ull, 0x2ecc93352d332f79ull, 0x4f197ec8a7edde1bull, 0x60b0a38f6114b355ull, 0x188495a938d66ce7ull, 0xbb76ba93a58649d6ull, 0xe5e002da6b804c0cull, 0x3382be5298f96753ull, 0xf689d10b9bbd71b0ull, 0x5a8a2a7789ee2415ull, 0xb67d435e5e1a16b2ull, 0x213b969c6709f348ull, 0x32bbe39e2ea90068ull}, + {0xe67dfa42e0827ddcull, 0x540c4f5a21a65374ull, 0xdee53c6258c575e9ull, 0xea9440812745665eull, 0x00570ce98f7a5198ull, 0xe63e3f4e4522adb9ull, 0xbb87944a95d59adcull, 0xeb0a38def2af1fc5ull, 0x521e716480794842ull, 0xdc010e9f1e4a18a3ull, 0x2e6a9139deb66aefull, 0x000a26d02e66b788ull, 0x9e2e9f1646e62d9full, 0x8dee07680440d65aull, 0x6213c72e2e5afe0dull}, + {0x830b72fd6d111bfdull, 0xc05d59ce24b2251bull, 0x22608a5ae40c689bull, 0xc1c1f97f901e0215ull, 0xea0abfe5845a97bcull, 0xd62097b430e126e6ull, 0x3721daef7d7c334aull, 0x45f8fbf5bfbda5f6ull, 0xb8c5b8d51b3ca41aull, 0x3b4b4ff8cdc94106ull, 0xa29f96ed6261f0f0ull, 0x40cba3dc61ad24f2ull, 0x2b77f4bd247f2084ull, 0xae0c82cb17eb728eull, 0x7b6d6ec95c26c100ull}, + {0x2c7fa8fb1e89be4aull, 0x3669f4cbc4539d7eull, 0x3c3edb0d8e367cbeull, 0x35accf0a901fcb8dull, 0xe6d3dda85e6395c2ull, 0x30f21444e981732eull, 0xe1033c0d774c285dull, 0x8671921f7a0a1794ull, 0x7677a79600252463ull, 0xa9d70e4e8c01ad89ull, 0x9d8119220744a23bull, 0x2a14bb5529f46c5eull, 0xa0e6af946333f9ccull, 0xf57bae2bb3999ef3ull, 0xd9553f671aad6e14ull}, +}; +__device__ constexpr uint64_t CAUCHY_RC_T0[16] = {0x6322b77453fbb998ull, 0xc8101162981033a1ull, 0x29ee9a33d14f8d65ull, 0x762e55340fa17b85ull, 0x22250616ec25550bull, 0x2f8bded9dc622004ull, 0xddcbae7b50667fabull, 0x2cec3ba64669f2beull, 0xaa6a9f32829985c7ull, 0xe261a5156f1ce657ull, 0x290fa01b196d8503ull, 0x572f2a9c72712130ull, 0x813f6bc44ae5ef8cull, 0xf291d8b58a62ab5bull, 0x06631ffb71b13f84ull, 0xe542a86b0c464a0eull}; +} // namespace p1w16 diff --git a/crypto/math-cuda/kernels/p1w8_constants.cuh b/crypto/math-cuda/kernels/p1w8_constants.cuh new file mode 100644 index 000000000..d938de163 --- /dev/null +++ b/crypto/math-cuda/kernels/p1w8_constants.cuh @@ -0,0 +1,71 @@ +// Poseidon1 over Goldilocks at width 8 (ZisK's grinding permutation): round constants, the MDS +// row and the Fourier-domain partial rounds. GENERATED by `scripts/poseidon1/p1_params.py cuda8`; +// do not edit by hand. The same tables as `crypto/crypto/src/hash/poseidon1_w8/constants.rs`. +#pragma once +#include +namespace p1w8 { +__device__ __constant__ uint64_t RC[30][8] = { + {0xdd5743e7f2a5a5d9ull, 0xcb3a864e58ada44bull, 0xffa2449ed32f8cdcull, 0x42025f65d6bd13eeull, 0x7889175e25506323ull, 0x34b98bb03d24b737ull, 0xbdcc535ecc4faa2aull, 0x5b20ad869fc0d033ull}, + {0xf1dda5b9259dfcb4ull, 0x27515210be112d59ull, 0x4227d1718c766c3full, 0x26d333161a5bd794ull, 0x49b938957bf4b026ull, 0x4a56b5938b213669ull, 0x1120426b48c8353dull, 0x6b323c3f10a56cadull}, + {0xce57d6245ddca6b2ull, 0xb1fc8d402bba1eb1ull, 0xb5c5096ca959bd04ull, 0x6db55cd306d31f7full, 0xc49d293a81cb9641ull, 0x1ce55a4fe979719full, 0xa92e60a9d178a4d1ull, 0x002cc64973bcfd8cull}, + {0xcea721cce82fb11bull, 0xe5b55eb8098ece81ull, 0x4e30525c6f1ddd66ull, 0x43c6702827070987ull, 0xaca68430a7b5762aull, 0x3674238634df9c93ull, 0x88cee1c825e33433ull, 0xde99ae8d74b57176ull}, + {0x488897d85ff51f56ull, 0x1140737ccb162218ull, 0xa7eeb9215866ed35ull, 0x9bd2976fee49fcc9ull, 0xc0c8f0de580a3fccull, 0x4fb2dae6ee8fc793ull, 0x343a89f35f37395bull, 0x223b525a77ca72c8ull}, + {0x56ccb62574aaa918ull, 0xc4d507d8027af9edull, 0xa080673cf0b7e95cull, 0xf0184884eb70dcf8ull, 0x044f10b0cb3d5c69ull, 0xe9e3f7993938f186ull, 0x1b761c80e772f459ull, 0x606cec607a1b5facull}, + {0x14a0c2e1d45f03cdull, 0x4eace8855398574full, 0xf905ca7103eff3e6ull, 0xf8c8f8d20862c059ull, 0xb524fe8bdd678e5aull, 0xfbb7865901a1ec41ull, 0x014ef1197d341346ull, 0x9725e20825d07394ull}, + {0xfdb25aef2c5bae3bull, 0xbe5402dc598c971eull, 0x93a5711f04cdca3dull, 0xc45a9a5b2f8fb97bull, 0xfe8946a924933545ull, 0x2af997a27369091cull, 0xaa62c88e0b294011ull, 0x058eb9d810ce9f74ull}, + {0xb3cb23eced349ae4ull, 0xa3648177a77b4a84ull, 0x43153d905992d95dull, 0xf4e2a97cda44aa4bull, 0x5baa2702b908682full, 0x082923bdf4f750d1ull, 0x98ae09a325893803ull, 0xf8a6475077968838ull}, + {0xceb0735bf00b2c5full, 0x0a1a5d953888e072ull, 0x2fcb190489f94475ull, 0xb5be06270dec69fcull, 0x739cb934b09acf8bull, 0x537750b75ec7f25bull, 0xe9dd318bae1f3961ull, 0xf7462137299efe1aull}, + {0xb1f6b8eee9adb940ull, 0xbdebcc8a809dfe6bull, 0x40fc1f791b178113ull, 0x3ac1c3362d014864ull, 0x9a016184bdb8aebaull, 0x95f2394459fbc25eull, 0xe3f34a07a76a66c2ull, 0x8df25f9ad98b1b96ull}, + {0x85ffc27171439d9dull, 0xddcb9a2dcfd26910ull, 0x26b5ba4bf3afb94eull, 0xffff9cc7c7651e2full, 0x8c88364698280b55ull, 0xebc114167b910501ull, 0x2d77b4d89ecfb516ull, 0x332e0828eba151f2ull}, + {0x46fa6a6450dd4735ull, 0xd00db7dd92384a33ull, 0x5fd4fb751f3a5fc5ull, 0x496fb90c0bb65ea2ull, 0xf3baec0bb87cc5c7ull, 0x862a3c0a7d4c7713ull, 0xbf5f38336a3f47d8ull, 0x41ad9dbc1394a20cull}, + {0xcc535945b7dbf0f7ull, 0x82af2bc93685bcecull, 0x8e4c8d0c8cebfccdull, 0x17cb39417e84597eull, 0xd4a965a8c749b232ull, 0xa2cab040f33f3ee5ull, 0xa98811a1fed4e3a6ull, 0x1cc48b54f377e2a1ull}, + {0xe40cd4f6c5609a27ull, 0x11de79ebca97a4a4ull, 0x9177c73d8b7e929dull, 0x2a6fe8085797e792ull, 0x3de6e93329f8d5aeull, 0x3f7af9125da962ffull, 0xd710682cfc77d3acull, 0x48faf05f3b053cf4ull}, + {0x287db8630da89c8bull, 0x4d0de32053cb30e9ull, 0x8b37a4f20c5ada7bull, 0xe7cc6ebe78c84ecfull, 0x240bdc0a66a2610dull, 0x8299e7f02caa1650ull, 0x380a53fefb6e754eull, 0x684a1d8cf8eb6810ull}, + {0xe839452eb4b8a5e1ull, 0xb03fa62e90626af4ull, 0x11a688602fbc5efcull, 0x30dda75c355a2d62ull, 0x0f712adcb73810deull, 0xffdc1102187f1ae1ull, 0x40c34f398254b99cull, 0xede021b9dc289a4aull}, + {0x8b7b05225c4e7dadull, 0x3bc794346f9d9ff9ull, 0xfccb5a57f2ca86ffull, 0xbb1502015a7da9d4ull, 0xd7e0a35d4352a015ull, 0x27af7a44f8160931ull, 0xc37442f6782f4615ull, 0xbdf392a9bd095dcbull}, + {0xc17f55037cf00de9ull, 0xbcffedd34c71a874ull, 0x5eb45d2a8133d1f2ull, 0xbabe251e1612ebdfull, 0x3efeb9fbe438c536ull, 0x2d7cef97b4afe1cfull, 0xe5de1b4660016c0bull, 0xcdcc26c332f5657cull}, + {0xe01dd653daf15809ull, 0xb0a6bdd4b41094b5ull, 0x27eac858b0b03a05ull, 0x51d43b5e93adbdc0ull, 0x8b89a23b0fea5fc9ull, 0xdc8ac3b14f7f2fc1ull, 0xe793f82f1efec039ull, 0x9f6f2cf8969e7b80ull}, + {0x49d45382e0f21d4aull, 0x5f4ad1797cd72786ull, 0x4dc3dbebfd45f795ull, 0x03a3ef84dba6e1bcull, 0x204bc9b3d3fc4c01ull, 0x9ad706081e89b9baull, 0x638bfb4d840e9f89ull, 0x5ef2938cd095ae35ull}, + {0x42cca18ebeb265c8ull, 0xb7b2ec5c29aecbf8ull, 0x0d84f9535dc78f0full, 0x04e64ad942e77b8cull, 0xb4880dffffc9da0bull, 0x16db16d9c29adeb1ull, 0x09bbaf2a0590cd1eull, 0x76460e74961fcf8dull}, + {0xed12a2276dfa1553ull, 0x0b5acec5de0436fdull, 0x3c6cfea033a1f0a8ull, 0x2b5ecefe546cac15ull, 0x6e2d82884cd3bf6full, 0xc134878d1add7b83ull, 0x997963422eb7a280ull, 0x5e834537ac648cf6ull}, + {0x89e779214737c0b7ull, 0x1a8c05e8581ad95bull, 0x8d18b72796437cf7ull, 0xe7252c949e04b106ull, 0x53267c4fd174585aull, 0xa16ef5d9c81dad47ull, 0xda65191937270a46ull, 0xcb2a5b55f2df664cull}, + {0x854aee2dc1924137ull, 0xf37013c9d479ece6ull, 0x0e163bc0630c4696ull, 0x384ee64955048f76ull, 0xf65d814e28ee4ec5ull, 0xe57bc564fd82f1b1ull, 0x4b338937b6876614ull, 0x66ee0b04ed43cd8dull}, + {0x49884bf25f4ef15dull, 0xeb51fe28de1c6f54ull, 0x2cd64e84fce8dfccull, 0x29164a96a541a013ull, 0x173ce7558f4cacb8ull, 0xeb5b1ce5877c89e9ull, 0x5faff4b0f5217bf6ull, 0xac42d0b1c20f205eull}, + {0xfb1d6bf0ca43221bull, 0x97b0a1b01d6a2955ull, 0x08c60bd622952b30ull, 0x43f2be0f9e24147cull, 0xfa7268b7d3730f5dull, 0x43a6c419a23983bbull, 0xcd77c1f7b29b113cull, 0xcfa43c9db8eec29full}, + {0xcaaa95a6c7365decull, 0x0a91193f798f3be0ull, 0x1104497652735dc6ull, 0x35aecb93663b515eull, 0x8dbc9916065aa858ull, 0xada8f7a0266579edull, 0x524dee7bec1ea789ull, 0xa93aee9dd5af9521ull}, + {0x9d1f1b54750d707eull, 0x7c9feab87096d5dcull, 0xa2e1fb19f9d4261bull, 0xb714deb448de6346ull, 0x225d1f0d011c5403ull, 0x1549b7f1d28cedc0ull, 0xaef3e46f97d43942ull, 0x6dfc7ffe0b38bf08ull}, + {0x7de853fdc542b663ull, 0xa68ecc96610657b2ull, 0xe88bb5428af289b1ull, 0xd7cfa1504c5569f5ull, 0x78a9aad0d642d30aull, 0xd68315f2353dce52ull, 0x46e56300f86fcfd5ull, 0x323d95332b145fd6ull}, +}; +// First ROW of the circulant MDS, `M[i][j] = MDS_ROW[(j - i) mod 8]`. Row sum 43 < 2^6. +__device__ constexpr uint32_t MDS_ROW[8] = {7, 1, 3, 8, 8, 3, 4, 9}; +// Fourier-domain partial rounds (`scripts/poseidon1/p1_fourier.py`): OMEGA_POW[k] = 2^(24k) mod p +// (omega = 2^24 has order 8); FD = the circulant's eigenvalues; FD_LAST = FD / 8; FC[k] = DFT(RC[4 + k]). +__device__ constexpr uint64_t OMEGA_POW[8] = {0x0000000000000001ull, 0x0000000001000000ull, 0x0001000000000000ull, 0x000000ffffffff00ull, 0xffffffff00000000ull, 0xfffffffeff000001ull, 0xfffeffff00000001ull, 0xfffffeff00000101ull}; +__device__ constexpr uint64_t FD[8] = {0x000000000000002bull, 0x0001020000fffdffull, 0x000d000000000008ull, 0xffff00ff01ffff00ull, 0x0000000000000001ull, 0x0000fdffff0001ffull, 0xfff2ffff00000009ull, 0xfffefefefe000100ull}; +__device__ constexpr uint64_t FD_LAST[8] = {0x9fffffff60000006ull, 0x2000203fe01fffc0ull, 0x0001a00000000001ull, 0x1fffe01fe03fffe0ull, 0xdfffffff20000001ull, 0x20001fbfdfe00040ull, 0xfffe5fff00000002ull, 0x1fffdfdfdfc00020ull}; +__device__ constexpr uint64_t INV8 = 0xdfffffff20000001ull; +__device__ __constant__ uint64_t FC[22][8] = { + {0x047c03fc8f57deebull, 0x7fefb4a2472bce98ull, 0x0bdb99339bc8421bull, 0x410249d723ac9a4dull, 0xc679939d4fe32c76ull, 0x559f8baf6a4d16b6ull, 0x4e74f21064fa2f09ull, 0x086d11bc4a85fe90ull}, + {0x16507eeeb9530b49ull, 0x265e2d42c4fc5cdeull, 0x1880193f9d30f371ull, 0x79c8042e0c8189d9ull, 0x17d4163b76d2bb21ull, 0xfe1b302ff8651fe2ull, 0x25ca6cf032495c28ull, 0xabb53430dbd22c24ull}, + {0x9e6dc6b4b65810ccull, 0xd764fb7bf6e872bdull, 0x73182ce99c9ea97full, 0x663381a9a921167eull, 0xe9c7333daf7d21d8ull, 0x5b819aa544557b46ull, 0x2bc9dedbc4a66c78ull, 0xe4d4f987f77ed150ull}, + {0xed7ac9fb6e39e6f3ull, 0x2d799224d3a9b185ull, 0x4bc9f482faac2d36ull, 0x73d7a90838a15417ull, 0x870cec945391f4a4ull, 0x365289afeac57a99ull, 0x309cdb548743852dull, 0x25008c3a281163a4ull}, + {0x844f282a13a6e247ull, 0x95d8034ec81fd984ull, 0x50ec4e5372b9e917ull, 0x333265c9bbc56811ull, 0x5221fc1f370b469cull, 0xc4f85e98385ecf05ull, 0x1677b924db87fa4full, 0xd2812bf7146cba3bull}, + {0x668b4ccfa79ab49full, 0x789cdc3da9f0c61bull, 0x1a4a41c5e1a06dc1ull, 0x1917bae7a52f11ddull, 0x515fa17609e23eddull, 0x22a44fc505ffaffdull, 0x36ff823aef7a8e67ull, 0xb7f601b2a8a1eb5bull}, + {0x8d79ac984b0e748eull, 0xc43335391dcbd6eaull, 0x66d5850001e620eeull, 0x656f362533e9b8b2ull, 0x54555b5488c22b0cull, 0x4b3c60f48f3c9cb7ull, 0xe73bdce4c7e2df5dull, 0xeaf69153cee1fdc7ull}, + {0x636fbb159a54f584ull, 0xde3569274d555dc9ull, 0xc59e21f06d980bd2ull, 0xd0d8dbb8ced7e87eull, 0x69fb14a59d813926ull, 0x281058ed59fb2a8cull, 0xb716f1368040694aull, 0x0ebf92dbee45d84full}, + {0x3b3ed4ccc1a37689ull, 0x2389989a10fd628aull, 0x79391300e2bfaa0bull, 0xb3e3ccea224b21b0ull, 0x78943f686403f2a5ull, 0x3952ce0b9940f572ull, 0xbdc9328d1d0120b4ull, 0x3c3dc5ce94f88c10ull}, + {0x32dafe41a6a7bb88ull, 0x20d589b269231de3ull, 0x404dd808dbf01a03ull, 0x961c7fca1e081bb0ull, 0x7ec7bcfd6f254babull, 0xf7a179b4812c79f1ull, 0x920268770ad96b69ull, 0x30144b40b9f14792ull}, + {0x4f4038fd322e0244ull, 0x30116aabd9e2d06dull, 0x371f2462cccc95faull, 0x1f64098d46d41a82ull, 0xc5b7a22fbc71a9f4ull, 0x583d8ae99ecc25a8ull, 0x3bb7f91b01f97d1full, 0xf0e4afecae1c014cull}, + {0x2f89e4bd6e3d4b76ull, 0xc76a40da901072eaull, 0x5866aa3d2dbd1f7dull, 0xa16bc3d921645420ull, 0xf00d360089eb4f4bull, 0xadb999ae1c153be0ull, 0xba288cb9ab463c23ull, 0xfb37d2fdce8eeb11ull}, + {0x18edc7efd8661cd4ull, 0x150a275ca890f8c9ull, 0x51842fd0cbc4756aull, 0x7f2daf53087ed650ull, 0x7b3ac75c639d81d8ull, 0x59936817e02334a6ull, 0xf8fd0112a7fac6e4ull, 0x75552a8264cf504bull}, + {0x001ae8f789d59b9aull, 0x6fbd851333cab0efull, 0xbe755b5cbada1614ull, 0x3a8044d099afb8a4ull, 0x471ba2aa8b603a0cull, 0xa80e33abcf69a25eull, 0x87c2bb03ae748b4aull, 0x7c1d8981c70b6a72ull}, + {0xb817b0c08c87ecb6ull, 0x83f96411ebbf6dc9ull, 0x865d4ea6ee6a7b17ull, 0xf74c2dc9f8e91026ull, 0xd2095e22f834357full, 0xd09e1461c1e7ef8cull, 0xf179de74117caf2full, 0xbe1ec5dfbc4cb552ull}, + {0xf99b22f7e866afc2ull, 0xd104b3767ab96441ull, 0x008d914b01fd4862ull, 0x01b30ae42e1e4dd5ull, 0xfcb14f388caeb45bull, 0x6c0ed81d7b02e82full, 0xb7c3dec3345c32c6ull, 0x138a39eb084146bbull}, + {0x78284f057de07198ull, 0xe4894b9a36037b04ull, 0x5e8f3520df294b7eull, 0x35fb9f62b3b645feull, 0xbeb799dbeea58f39ull, 0xdfdcba7122ab60b1ull, 0x131156d8880a58ddull, 0xabc081cc27722373ull}, + {0x584fb491e72591c0ull, 0xe512d6f85335f43cull, 0xe98c1a53b13b7d02ull, 0xfa52e99e1cc07e1bull, 0xc4dafb875c83a63full, 0x4fab009096486b40ull, 0xd69bf3cf050c4a4aull, 0x0a018d0ff5635161ull}, + {0x8797f11e16da5372ull, 0xa89790bf1cd5a217ull, 0x40ebbade165706dcull, 0x828f79138eb46f75ull, 0xdab51c0923747c5full, 0x283eb853eebbb9f5ull, 0xc9c7cabc9a917c58ull, 0xa82ebc55ea538c0full}, + {0xb2d6496297333e3eull, 0x7ad4300dcc2040d2ull, 0x58b7b483d76af8d5ull, 0x61f4d257b3795cccull, 0xd641420434fa025bull, 0x142c87cde34a6c48ull, 0x926895dabf182ad5ull, 0xea0e69117429978full}, + {0x4d1afef51859783cull, 0x479eeddbf9706155ull, 0x8bd33a38e90b19eeull, 0x371716e949116728ull, 0x5cc969f5efcf010dull, 0xfc5144ead2c6a4c2ull, 0xb8ea1acfb8ceacb5ull, 0xc0ae69ca4d475c8dull}, + {0x9951acd7ad8fb382ull, 0x65cbb8addb34def3ull, 0x0cd0ee59fcca42b7ull, 0xf8e6cdce489297c3ull, 0x4145402513bc402bull, 0x7a1f201a1128007cull, 0x9bacf1c8fc5841f0ull, 0xf05bebdb0b199b64ull}, +}; +} // namespace p1w8 diff --git a/crypto/math-cuda/kernels/rpx.cu b/crypto/math-cuda/kernels/rpx.cu new file mode 100644 index 000000000..006cd49bb --- /dev/null +++ b/crypto/math-cuda/kernels/rpx.cu @@ -0,0 +1,1699 @@ +// RPX256 (Rescue-Prime eXtended / XHash12) over Goldilocks at width 12 on +// device — the permutation, the rate-8 overwrite-duplex leaf sponge and the +// Merkle parent (lane K phase 1, arithmetic), then the leaf/tree kernels that +// stream table rows through `rpx::Sponge` and `rpx::compress` and the +// permutation probe (phase 2, the `extern "C"` surface at the end of the +// file, kernel for kernel the twin of `blake3.cu:338-620`). +// +// THE ORACLE is the Rust host implementation, byte for byte: +// `prover/src/lfm/rpx.rs` `Rpx256::permute` (:280-316) — schedule FB E FB E FB E M, +// `cubic_ext::{mul, power7}` (:118-140); +// `prover/src/lfm/rpo.rs` ARK1/ARK2 (:119-321, RPX imports RPO's tables +// literally), `sbox` (:455-460), `inv_sbox_layer` +// (:481-509, the 72-multiplication chain), `mds` +// (:539-557, the u128 accumulation); +// `prover/src/lfm/algebraic_commit.rs` `leaf_capacity` (:142-147), +// `sponge_leaf` (:169-184), `parent` (:248-252); +// `prover/src/lfm/hash.rs` `permute_two_cells` (:95-108): `[a ‖ b ‖ iv]`, +// digest = lanes 0..4. +// +// PROVENANCE, layered exactly as the Rust module's own (rpx.rs "PROVENANCE"): +// the FB round IS RPO's round with RPO's constants, and those are pinned by +// nineteen EXTERNAL miden-crypto vectors, which `tests/host_kat/rpx_host_kat.cpp` +// replays through `fb_round(s, r)` composed seven times. The E round (the cubic +// extension) and the schedule have no published vector anywhere; they are +// pinned to the Rust oracle's output (`prover/tests/rpx_host_kat_vectors.rs`) +// and, independently, to naive polynomial arithmetic in the harness. +// +// REPRESENTATION. Inputs may be raw `[0, 2^64)` Goldilocks storage exactly as +// `goldilocks.cuh` allows everywhere else; every step here (`add`, `mul`, +// `dot3`, the MDS bound) accepts that. `permute` CANONICALISES its output, so +// digests are canonical `< p` and their big-endian bytes are what +// `digest_to_commitment` (algebraic_commit.rs:112-118) writes — the device +// Merkle tree can be compared to the host's byte for byte. +// +// ⚠ TWO CUBIC EXTENSIONS EXIST AND THIS FILE USES THE OTHER ONE. `ext3.cuh` is +// the VM's `w³ = 2`; RPX's is `φ³ = φ + 1` (rpx.rs:98-103). Only the GENERIC +// three-term dot product `ext3::dot3` is borrowed from that header — never +// `ext3::mul`. The reduction polynomial lives in `rpx::ext_mul` alone. +// +// COST MODEL (one permutation; counted by the harness's op counters, static +// for the MDS): +// FB round ×3 : 12·(4 + 72) = 912 Goldilocks multiplications (48 forward +// S-box, 864 inverse), 2 MDS, 24 constant adds; +// E round ×3 : 4 triples × 4 extension products = 16 `ext_mul` = 144 wide +// 64×64 products folded into 48 reductions (`dot3`), 12 +// constant adds, 32 operand pre-adds; +// M round ×1 : 1 MDS, 12 constant adds; +// MDS ×7 : 288 32×32→64 multiply-adds + 12 reductions each — the ported +// u128 property (see `mds`), ~6× under twelve field +// multiplications per lane. +// Total: 2736 field multiplications + 144 dot3 (432 wide products) + 300 adds +// + 2016 narrow MACs. The inverse S-box is 2592/2736 = 95% of the field +// multiplications; RPO spends 7 such layers, RPX 3 — that is the whole +// reason RPX exists (rpx.rs:22-28). +// +// PHASE-2 TUNING NOTES (not done here, do not guess at them): `inv_sbox` is a +// serial 72-deep chain per lane — one thread per permutation interleaves twelve +// of them; register pressure is what to measure (`-Xptxas -v`). `Sponge::absorb` +// indexes the state dynamically, which nvcc lowers to local memory unless the +// caller's loop is unrolled — the same trade `Blake3Chain::push_word` makes. +// ARK reads are warp-uniform constant-bank operands and cost nothing. + +#include +#include "goldilocks.cuh" +#include "ext3.cuh" + +// `permute` is a REAL device function, never inlined (see its CODE SHAPE +// note). The host shim has no `__noinline__`; on the host the attribute only +// matters to the code-size probe, which asks for it explicitly. +#if defined(__CUDACC__) +#define RPX_NOINLINE __noinline__ +#define RPX_LAUNCH_BOUNDS(n) __launch_bounds__(n) +#elif defined(RPX_HOST_NOINLINE) +#define RPX_NOINLINE __attribute__((noinline)) +#define RPX_LAUNCH_BOUNDS(n) +#else +#define RPX_NOINLINE +#define RPX_LAUNCH_BOUNDS(n) +#endif + +// ★ RPX_PERMUTE_VARIANT — which field multiply the permutation runs, as a +// compile-time bit set. 0 is `goldilocks::mul` for every product and square, +// and every kernel in a cubin uses `RPX_PERMUTE_VARIANT`. build.rs compiles this +// file twice: `rpx_v0.cubin` at 0 and `rpx_v5.cubin` at 5 (limb multiply, +// `square_n` unrolled), the default; `LAMBDA_VM_RPX_LIMB_PERMUTE=0` loads the +// former (math-cuda `rpx_paths`). The bits: +// RPX_V_LIMB_MUL products through `mul_limb` (32-bit limbs, carry chains); +// RPX_V_LIMB_SQR squarings through `sqr_limb` (three limb products); +// RPX_V_UNROLL_SQN `square_n` unrolled by four; +// RPX_V_UNROLL_LANES the FB round's twelve lane loops unrolled. +// ⚠ Every variant computes the same FIELD values; only the raw representation +// of intermediates may differ, and `permute` canonicalises its output, so +// digests, nodes and grind heads are bit-identical across variants. +#define RPX_V_LIMB_MUL 1 +#define RPX_V_LIMB_SQR 2 +#define RPX_V_UNROLL_SQN 4 +#define RPX_V_UNROLL_LANES 8 +#ifndef RPX_PERMUTE_VARIANT +#define RPX_PERMUTE_VARIANT 0 +#endif + +namespace rpx { + +enum : int { + STATE_FELTS = 12, + RATE_FELTS = 8, + CAPACITY_FELTS = 4, + DIGEST_FELTS = 4, + NUM_ROUNDS = 7, + EXT_DEGREE = 3, + EXT_ELEMENTS = 4, + // Absolute lanes of the two capacity cells the socket names: the padding + // flag (`rpo.rs:339` CAPACITY_PAD_LANE = 0 within the capacity) and the + // domain tag (`rpo.rs:343` CAPACITY_DOMAIN_LANE = 1). Capacity = lanes 8..12. + CAPACITY_PAD_LANE = RATE_FELTS + 0, + CAPACITY_DOMAIN_LANE = RATE_FELTS + 1, +}; + +// The Merkle-parent domain is ZERO on purpose (rpo.rs:350): a parent is a +// standard `Rpx256::merge`, checkable against miden without this codebase. +__device__ constexpr uint64_t DOMAIN_COMPRESS = 0; +// The leaf domain: `u32::from_le_bytes(b"LFML")` (rpo.rs:358) = 1280132684. +__device__ constexpr uint64_t DOMAIN_LEAF = 0x4C4D464CULL; + +// --------------------------------------------------------------------------- +// Constants. Transcribed MECHANICALLY (a script over rpo.rs, not by hand) from +// `rpo.rs` ARK1 (:119-218), ARK2 (:222-321) and MDS_CIRC_ROW (:114). RPX +// imports exactly these (rpx.rs:69); `rpx_uses_rpos_constant_tables` asserts +// the import on the host, and the miden vectors in the harness pin them here. +// Only the FB rounds (0, 2, 4) consume ARK2; E and M rounds add ARK1 alone. +// --------------------------------------------------------------------------- +__device__ __constant__ uint64_t ARK1[NUM_ROUNDS][STATE_FELTS] = { + {5789762306288267392ull, 6522564764413701783ull, 17809893479458208203ull, 107145243989736508ull, + 6388978042437517382ull, 15844067734406016715ull, 9975000513555218239ull, 3344984123768313364ull, + 9959189626657347191ull, 12960773468763563665ull, 9602914297752488475ull, 16657542370200465908ull}, + {12987190162843096997ull, 653957632802705281ull, 4441654670647621225ull, 4038207883745915761ull, + 5613464648874830118ull, 13222989726778338773ull, 3037761201230264149ull, 16683759727265180203ull, + 8337364536491240715ull, 3227397518293416448ull, 8110510111539674682ull, 2872078294163232137ull}, + {18072785500942327487ull, 6200974112677013481ull, 17682092219085884187ull, 10599526828986756440ull, + 975003873302957338ull, 8264241093196931281ull, 10065763900435475170ull, 2181131744534710197ull, + 6317303992309418647ull, 1401440938888741532ull, 8884468225181997494ull, 13066900325715521532ull}, + {5674685213610121970ull, 5759084860419474071ull, 13943282657648897737ull, 1352748651966375394ull, + 17110913224029905221ull, 1003883795902368422ull, 4141870621881018291ull, 8121410972417424656ull, + 14300518605864919529ull, 13712227150607670181ull, 17021852944633065291ull, 6252096473787587650ull}, + {4887609836208846458ull, 3027115137917284492ull, 9595098600469470675ull, 10528569829048484079ull, + 7864689113198939815ull, 17533723827845969040ull, 5781638039037710951ull, 17024078752430719006ull, + 109659393484013511ull, 7158933660534805869ull, 2955076958026921730ull, 7433723648458773977ull}, + {16308865189192447297ull, 11977192855656444890ull, 12532242556065780287ull, 14594890931430968898ull, + 7291784239689209784ull, 5514718540551361949ull, 10025733853830934803ull, 7293794580341021693ull, + 6728552937464861756ull, 6332385040983343262ull, 13277683694236792804ull, 2600778905124452676ull}, + {7123075680859040534ull, 1034205548717903090ull, 7717824418247931797ull, 3019070937878604058ull, + 11403792746066867460ull, 10280580802233112374ull, 337153209462421218ull, 13333398568519923717ull, + 3596153696935337464ull, 8104208463525993784ull, 14345062289456085693ull, 17036731477169661256ull}, +}; + +__device__ __constant__ uint64_t ARK2[NUM_ROUNDS][STATE_FELTS] = { + {6077062762357204287ull, 15277620170502011191ull, 5358738125714196705ull, 14233283787297595718ull, + 13792579614346651365ull, 11614812331536767105ull, 14871063686742261166ull, 10148237148793043499ull, + 4457428952329675767ull, 15590786458219172475ull, 10063319113072092615ull, 14200078843431360086ull}, + {6202948458916099932ull, 17690140365333231091ull, 3595001575307484651ull, 373995945117666487ull, + 1235734395091296013ull, 14172757457833931602ull, 707573103686350224ull, 15453217512188187135ull, + 219777875004506018ull, 17876696346199469008ull, 17731621626449383378ull, 2897136237748376248ull}, + {8023374565629191455ull, 15013690343205953430ull, 4485500052507912973ull, 12489737547229155153ull, + 9500452585969030576ull, 2054001340201038870ull, 12420704059284934186ull, 355990932618543755ull, + 9071225051243523860ull, 12766199826003448536ull, 9045979173463556963ull, 12934431667190679898ull}, + {18389244934624494276ull, 16731736864863925227ull, 4440209734760478192ull, 17208448209698888938ull, + 8739495587021565984ull, 17000774922218161967ull, 13533282547195532087ull, 525402848358706231ull, + 16987541523062161972ull, 5466806524462797102ull, 14512769585918244983ull, 10973956031244051118ull}, + {6982293561042362913ull, 14065426295947720331ull, 16451845770444974180ull, 7139138592091306727ull, + 9012006439959783127ull, 14619614108529063361ull, 1394813199588124371ull, 4635111139507788575ull, + 16217473952264203365ull, 10782018226466330683ull, 6844229992533662050ull, 7446486531695178711ull}, + {3736792340494631448ull, 577852220195055341ull, 6689998335515779805ull, 13886063479078013492ull, + 14358505101923202168ull, 7744142531772274164ull, 16135070735728404443ull, 12290902521256031137ull, + 12059913662657709804ull, 16456018495793751911ull, 4571485474751953524ull, 17200392109565783176ull}, + {17130398059294018733ull, 519782857322261988ull, 9625384390925085478ull, 1664893052631119222ull, + 7629576092524553570ull, 3485239601103661425ull, 9755891797164033838ull, 15218148195153269027ull, + 16460604813734957368ull, 9643968136937729763ull, 3611348709641382851ull, 18256379591337759196ull}, +}; + +// First ROW of the circulant MDS, `M[i][j] = ROW[(j − i) mod 12]` +// (rpo.rs:107-114), stored TWICE so that `MDS_CIRC_ROW2[j + 12 − i]` is the +// entry with no modulo: the output-lane loop in `mds` is rolled, so `i` is a +// runtime value there. 32-bit so each MDS term is one 32×32→64 MAC. The row +// sums to 160, which is the bound `mds` rests on. +__device__ __constant__ uint32_t MDS_CIRC_ROW2[2 * STATE_FELTS] = { + 7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8, 7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8}; + +// --------------------------------------------------------------------------- +// Field-op forwarders. Under nvcc they are the `goldilocks.cuh` / `ext3.cuh` +// primitives, nothing more. The host-KAT harness defines RPX_HOST_OP_COUNT +// before including this file so it can COUNT them per round kind and print the +// cost model above as a measurement rather than a claim. +// --------------------------------------------------------------------------- +#ifdef RPX_HOST_OP_COUNT +struct OpCount { + unsigned long long mul, dot3, add; +}; +static OpCount g_ops = {0, 0, 0}; +#define RPX_COUNT(field) (++g_ops.field) +#else +#define RPX_COUNT(field) ((void)0) +#endif + +// --------------------------------------------------------------------------- +// The 32-bit-limb multiply. The 128-bit product is built from four +// 32×32 partial products with the carries in the add chain, then folded to +// `[0, 2^64)` in three steps (the reduction sppark's `gl64_t` ships, Apache-2.0, +// which pil2-stark uses): `2^64 ≡ 2^32 − 1` and `2^96 ≡ −1 (mod p)`. +// +// The result is congruent to `a·b` and below 2^64, NOT canonical, and may differ +// in representation from `goldilocks::mul`'s — which is why only this file uses +// it: `permute` canonicalises its output, whereas the NTT/LDE parity tests +// compare `goldilocks::mul`'s raw words. +// +// Why the steps cannot wrap (inputs anywhere in `[0, 2^64)`): limbs t0..t3 of +// the product; step 1 forms `c·2^64 + s1 = (t1:t0) + t2·(2^32 − 1)` with +// `c ∈ {0, 1}`, and `c = 1` forces `s1 < 2^64 − 2^33 + 1`; step 2 subtracts t3 +// with the borrow going into `c`, leaving `c ∈ {−1, 0, 1}`; step 3 subtracts +// `c·p` as ONE 64-bit subtraction: `c = 1` subtracts p (≡ adding 2^32 − 1 mod +// 2^64, and s2 ≤ s1 < 2^64 − 2^33 + 1 leaves room), `c = −1` subtracts +// 2^32 − 1 (s2 = s1 − t3 + 2^64 > 2^64 − 2^32 there, so no borrow). +// --------------------------------------------------------------------------- +__device__ __forceinline__ uint64_t reduce_limbs(uint32_t t0, uint32_t t1, uint32_t t2, + uint32_t t3) { +#if defined(__CUDA_ARCH__) + const uint32_t w = 0xFFFFFFFFu; + // Step 1: (t1:t0) += t2·(2^32 − 1); the carry out replaces t2. + asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.cc.u32 %1, %2, %3, %1; addc.u32 %2, 0, 0;" + : "+r"(t0), "+r"(t1), "+r"(t2) + : "r"(w)); + // Step 2: − t3, the borrow running into t2. + asm("sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, 0; subc.u32 %2, %2, 0;" + : "+r"(t0), "+r"(t1), "+r"(t2) + : "r"(t3)); + // Step 3: − c·p with c = t2 ∈ {0, 1, 0xFFFFFFFF}: (c, 0xFFFFFFFF) for c = 1, + // (0xFFFFFFFF, 0) for c = −1, nothing for c = 0. + const uint32_t hi_sub = (t2 == 1u) ? 0xFFFFFFFFu : 0u; + asm("sub.cc.u32 %0, %0, %2; subc.u32 %1, %1, %3;" : "+r"(t0), "+r"(t1) : "r"(t2), "r"(hi_sub)); + return ((uint64_t)t1 << 32) | t0; +#else + // The same three steps in portable C, for the host KAT. + const uint64_t s = ((uint64_t)t1 << 32) | t0; + const uint64_t s1 = s + (uint64_t)t2 * 0xFFFFFFFFull; + int c = (s1 < s) ? 1 : 0; + const uint64_t s2 = s1 - t3; + c -= (s1 < t3) ? 1 : 0; + if (c == 1) return s2 + 0xFFFFFFFFull; + if (c == -1) return s2 - 0xFFFFFFFFull; + return s2; +#endif +} + +__device__ __forceinline__ uint64_t mul_limb(uint64_t a, uint64_t b) { + const uint32_t a0 = (uint32_t)a, a1 = (uint32_t)(a >> 32); + const uint32_t b0 = (uint32_t)b, b1 = (uint32_t)(b >> 32); +#if defined(__CUDA_ARCH__) + uint32_t t0, t1, t2, t3, c; + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t0), "=r"(t1) : "r"(a0), "r"(b0)); + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t2), "=r"(t3) : "r"(a1), "r"(b1)); + asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;" + : "+r"(t1), "+r"(t2), "=r"(c) + : "r"(a0), "r"(b1)); + asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, %2, %5;" + : "+r"(t1), "+r"(t2), "+r"(t3) + : "r"(a1), "r"(b0), "r"(c)); + return reduce_limbs(t0, t1, t2, t3); +#else + const uint64_t p00 = (uint64_t)a0 * b0, p01 = (uint64_t)a0 * b1; + const uint64_t p10 = (uint64_t)a1 * b0, p11 = (uint64_t)a1 * b1; + const uint64_t mid = (p00 >> 32) + (uint32_t)p01 + (uint32_t)p10; + const uint64_t top = p11 + (p01 >> 32) + (p10 >> 32) + (mid >> 32); + return reduce_limbs((uint32_t)p00, (uint32_t)mid, (uint32_t)top, (uint32_t)(top >> 32)); +#endif +} + +// `a²` with the cross product `a0·a1` formed once and added twice: three limb +// products where `mul_limb` needs four, paid for in adds. +__device__ __forceinline__ uint64_t sqr_limb(uint64_t a) { + const uint32_t a0 = (uint32_t)a, a1 = (uint32_t)(a >> 32); +#if defined(__CUDA_ARCH__) + uint32_t t0, t1, t2, t3, x0, x1; + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t0), "=r"(t1) : "r"(a0), "r"(a0)); + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(t2), "=r"(t3) : "r"(a1), "r"(a1)); + asm("mul.lo.u32 %0, %2, %3; mul.hi.u32 %1, %2, %3;" : "=r"(x0), "=r"(x1) : "r"(a0), "r"(a1)); + asm("add.cc.u32 %0, %0, %3; addc.cc.u32 %1, %1, %4; addc.u32 %2, %2, 0;" + : "+r"(t1), "+r"(t2), "+r"(t3) + : "r"(x0), "r"(x1)); + asm("add.cc.u32 %0, %0, %3; addc.cc.u32 %1, %1, %4; addc.u32 %2, %2, 0;" + : "+r"(t1), "+r"(t2), "+r"(t3) + : "r"(x0), "r"(x1)); + return reduce_limbs(t0, t1, t2, t3); +#else + const uint64_t p00 = (uint64_t)a0 * a0, x = (uint64_t)a0 * a1, p11 = (uint64_t)a1 * a1; + const uint64_t mid = (p00 >> 32) + 2 * (uint64_t)(uint32_t)x; + const uint64_t top = p11 + 2 * (x >> 32) + (mid >> 32); + return reduce_limbs((uint32_t)p00, (uint32_t)mid, (uint32_t)top, (uint32_t)(top >> 32)); +#endif +} + +// The multiply and the square as the variant `V` spells them. Both count as +// one multiplication for the host cost model. +template +__device__ __forceinline__ uint64_t fmul_v(uint64_t a, uint64_t b) { + RPX_COUNT(mul); + if constexpr ((V & RPX_V_LIMB_MUL) != 0) { + return mul_limb(a, b); + } else { + return goldilocks::mul(a, b); + } +} + +template +__device__ __forceinline__ uint64_t fsqr_v(uint64_t a) { + if constexpr ((V & RPX_V_LIMB_SQR) != 0) { + RPX_COUNT(mul); + return sqr_limb(a); + } else { + return fmul_v(a, a); + } +} + +__device__ __forceinline__ uint64_t fmul(uint64_t a, uint64_t b) { + return fmul_v(a, b); +} + +__device__ __forceinline__ uint64_t fadd(uint64_t a, uint64_t b) { + RPX_COUNT(add); + return goldilocks::add(a, b); +} + +// `a0·b0 + a1·b1 + a2·b2` with ONE reduction — the generic part of `ext3.cuh`, +// independent of that header's reduction polynomial. +__device__ __forceinline__ uint64_t fdot3(uint64_t a0, uint64_t b0, uint64_t a1, uint64_t b1, + uint64_t a2, uint64_t b2) { + RPX_COUNT(dot3); + return ext3::dot3(a0, b0, a1, b1, a2, b2); +} + +// --------------------------------------------------------------------------- +// The circulant MDS, `out_i = Σ_j MDS_CIRC_ROW[(j − i) mod 12] · s_j`, in +// `rpo.rs:539-557`'s orientation (the one the miden vectors pin). +// +// ★ THE PORTED PROPERTY (rpo.rs:527-536): one accumulation and ONE reduction +// per output lane, no per-term field multiplication. Every coefficient is ≤ 26 +// and every stored lane is < 2^64, so the twelve-term row sum is < 12·26·2^64 +// < 2^73 and needs no reduction before the end. The host accumulates it in a +// u128; the device has no u128, so the SAME integer is assembled from 32-bit +// halves. With `s_j = h_j·2^32 + l_j`, +// +// acc = 2^32 · Σ_j c_j·h_j + Σ_j c_j·l_j , +// +// and each half-sum is ≤ 160·(2^32 − 1) < 2^40 — the row sums to 160 — so both +// fit a u64 with 24 bits to spare and every term is a single 32×32→64 +// multiply-add (no 64-bit multiplier anywhere in the MDS). The halves are then +// recombined into the u128's `(lo, hi)` exactly as the host holds them and +// reduced the host's way: `acc = hi·2^64 + lo ≡ lo + hi·EPSILON (mod p)`, with +// `hi < 2^9` so `hi·EPSILON < 2^41` needs no reduction of its own +// (`the_mds_row_sum_cannot_overflow_a_u128` asserts the same bound on the host). +// --------------------------------------------------------------------------- +__device__ __forceinline__ void mds(uint64_t s[STATE_FELTS]) { + uint32_t lo32[STATE_FELTS], hi32[STATE_FELTS]; +#pragma unroll + for (int j = 0; j < STATE_FELTS; ++j) { + lo32[j] = (uint32_t)s[j]; + hi32[j] = (uint32_t)(s[j] >> 32); + } + uint64_t out[STATE_FELTS]; + // Rolled over output lanes: twelve iterations of twenty-four MACs, one + // twelfth of the unrolled body's code for the same instruction count. +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) { + uint64_t acc_lo = 0, acc_hi = 0; // Σ c·l_j and Σ c·h_j, each < 2^40 + const int rot = STATE_FELTS - i; // MDS_CIRC_ROW2[j + rot] = ROW[(j − i) mod 12] +#pragma unroll + for (int j = 0; j < STATE_FELTS; ++j) { + const uint32_t c = MDS_CIRC_ROW2[j + rot]; + acc_lo += (uint64_t)c * (uint64_t)lo32[j]; + acc_hi += (uint64_t)c * (uint64_t)hi32[j]; + } + // acc = acc_hi·2^32 + acc_lo, exactly. Split it at bit 64. + const uint64_t lo = (acc_hi << 32) + acc_lo; + const uint64_t carry = (lo < acc_lo) ? 1ull : 0ull; + const uint64_t hi = (acc_hi >> 32) + carry; // < 2^9 + out[i] = fadd(lo, hi * goldilocks::EPSILON); + } +#pragma unroll + for (int i = 0; i < STATE_FELTS; ++i) s[i] = out[i]; +} + +// --------------------------------------------------------------------------- +// S-boxes. +// --------------------------------------------------------------------------- + +// `x^7` in the association the AIR's degree-3 lowering uses (rpo.rs:455-460): +// `x², x³ = x²·x, x^7 = (x³)²·x`. Two squarings, two products. +template +__device__ __forceinline__ uint64_t sbox_v(uint64_t x) { + const uint64_t x2 = fsqr_v(x); + const uint64_t x3 = fmul_v(x2, x); + const uint64_t x6 = fsqr_v(x3); + return fmul_v(x6, x); +} + +template +__device__ __forceinline__ uint64_t square_n_v(uint64_t x) { + // Rolled: the chain is serial anyway, and unrolled it is what made one + // permutation ~49k lines of PTX. The unroll factor here is a tuning knob + // (RPX_V_UNROLL_SQN). + if constexpr ((V & RPX_V_UNROLL_SQN) != 0) { +#pragma unroll 4 + for (int i = 0; i < N; ++i) x = fsqr_v(x); + } else { +#pragma unroll 1 + for (int i = 0; i < N; ++i) x = fsqr_v(x); + } + return x; +} + +// `base^(2^M) · tail` — the inverse chain's one building block (rpo.rs:483-495). +template +__device__ __forceinline__ uint64_t exp_acc_v(uint64_t base, uint64_t tail) { + return fmul_v(square_n_v(base), tail); +} + +// `x^{1/7} = x^10540996611094048183` by miden-crypto's addition chain, as +// `rpo.rs:481-509` runs it lane-wise: 63 squarings + 9 products = 72 +// multiplications against ~93 for square-and-multiply. Per lane rather than +// whole-state: on a GPU the twelve lanes' independence is the compiler's to +// interleave, and a lane-wise body keeps only six values live. +template +__device__ __forceinline__ uint64_t inv_sbox_v(uint64_t x) { + const uint64_t t1 = fsqr_v(x); // x^2 + const uint64_t t2 = fsqr_v(t1); // x^4 + const uint64_t t3 = exp_acc_v(t2, t2); // x^36 + const uint64_t t4 = exp_acc_v(t3, t3); // x^(36·65) + const uint64_t t5 = exp_acc_v(t4, t4); // x^(36·65·4097) + const uint64_t t6 = exp_acc_v(t5, t3); // x^0x24924924 + const uint64_t t7 = exp_acc_v(t6, t6); // x^0x1249249224924924 + // ((t7² · t6)²)² · ((t1 · t2) · x) — rpo.rs:504-508. + const uint64_t a = square_n_v(fmul_v(fsqr_v(t7), t6)); + const uint64_t b = fmul_v(fmul_v(t1, t2), x); + return fmul_v(a, b); +} + +// The names every kernel calls, at this cubin's variant (the host KAT calls these). +__device__ __forceinline__ uint64_t sbox(uint64_t x) { return sbox_v(x); } + +template +__device__ __forceinline__ uint64_t square_n(uint64_t x) { + return square_n_v(x); +} + +template +__device__ __forceinline__ uint64_t exp_acc(uint64_t base, uint64_t tail) { + return exp_acc_v(base, tail); +} + +__device__ __forceinline__ uint64_t inv_sbox(uint64_t x) { + return inv_sbox_v(x); +} + +// --------------------------------------------------------------------------- +// The cubic extension `GF(p)[φ] / (φ³ − φ − 1)` — rpx.rs:98-140. NOT `ext3.cuh`'s. +// --------------------------------------------------------------------------- +struct CubicExt { + uint64_t c0, c1, c2; // c0 + c1·φ + c2·φ² +}; + +// The product reduced by `φ³ = φ + 1`, `φ⁴ = φ² + φ`. `rpx.rs:118-125`'s +// closed form, regrouped so each coefficient is ONE three-term dot product +// with a single reduction (the same fold `dot_product_3` gives the VM's own +// extension): +// c0 = a0·b0 + a1·b2 + a2·b1 +// c1 = a0·b1 + a1·(b0 + b2) + a2·(b1 + b2) [= a0b1 + a1b0 + a1b2 + a2b1 + a2b2] +// c2 = a0·b2 + a1·b1 + a2·(b0 + b2) [= a0b2 + a1b1 + a2b0 + a2b2] +// Nine wide products, three reductions, two operand pre-adds. +__device__ __forceinline__ CubicExt ext_mul(const CubicExt &a, const CubicExt &b) { + const uint64_t b02 = fadd(b.c0, b.c2); + const uint64_t b12 = fadd(b.c1, b.c2); + CubicExt r; + r.c0 = fdot3(a.c0, b.c0, a.c1, b.c2, a.c2, b.c1); + r.c1 = fdot3(a.c0, b.c1, a.c1, b02, a.c2, b12); + r.c2 = fdot3(a.c0, b.c2, a.c1, b.c1, a.c2, b02); + return r; +} + +// One function for squaring and product, as on the host (rpx.rs:128-130). +__device__ __forceinline__ CubicExt ext_square(const CubicExt &a) { return ext_mul(a, a); } + +// `a^7` by `a² → a³ → a⁶ → a⁷` (rpx.rs:135-140): two squarings, two products. +__device__ __forceinline__ CubicExt ext_power7(const CubicExt &a) { + const CubicExt a2 = ext_square(a); + const CubicExt a3 = ext_mul(a2, a); + const CubicExt a6 = ext_square(a3); + return ext_mul(a6, a); +} + +// --------------------------------------------------------------------------- +// Rounds. `r` is the round index into ARK1/ARK2 — a runtime value, so one copy +// of each round body serves every round; the constant-bank address is +// computed, which costs nothing next to the round's arithmetic. Every lane +// loop is rolled for the same reason (see `permute`'s CODE SHAPE note). +// --------------------------------------------------------------------------- + +// FB: `MDS → +ARK1 → x^7 → MDS → +ARK2 → x^{1/7}` — RPO's round exactly +// (rpo.rs:561-582, rpx.rs:283-295). RPX runs it at R = 0, 2, 4; RPO at 0..7. +template +__device__ __forceinline__ void fb_round_v(uint64_t s[STATE_FELTS], int r) { + mds(s); + if constexpr ((V & RPX_V_UNROLL_LANES) != 0) { +#pragma unroll + for (int i = 0; i < STATE_FELTS; ++i) s[i] = sbox_v(fadd(s[i], ARK1[r][i])); + mds(s); +#pragma unroll + for (int i = 0; i < STATE_FELTS; ++i) s[i] = inv_sbox_v(fadd(s[i], ARK2[r][i])); + } else { +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK1[r][i]); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = sbox_v(s[i]); + mds(s); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK2[r][i]); + // The twelve chains are independent; a GPU hides their latency with + // other warps, not by unrolling one thread's twelve chains into + // straight line. +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = inv_sbox_v(s[i]); + } +} + +__device__ __forceinline__ void fb_round(uint64_t s[STATE_FELTS], int r) { + fb_round_v(s, r); +} + +// E: `+ARK1 → x^7` in the cubic extension on four lane-triples, NO linear +// layer (rpx.rs:296-307; the design, not an omission — rpx.rs:275-279). +__device__ __forceinline__ void ext_round(uint64_t s[STATE_FELTS], int r) { +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK1[r][i]); +#pragma unroll 1 + for (int e = 0; e < EXT_ELEMENTS; ++e) { + const int base = e * EXT_DEGREE; + CubicExt x; + x.c0 = s[base]; + x.c1 = s[base + 1]; + x.c2 = s[base + 2]; + const CubicExt y = ext_power7(x); + s[base] = y.c0; + s[base + 1] = y.c1; + s[base + 2] = y.c2; + } +} + +// M: `MDS → +ARK1`, a linear finish with no S-box (rpx.rs:308-313). +__device__ __forceinline__ void final_round(uint64_t s[STATE_FELTS], int r) { + mds(s); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK1[r][i]); +} + +// The permutation: `FB E FB E FB E M` (rpx.rs:280-316), output CANONICAL. +// +// ★ CODE SHAPE. A real (`RPX_NOINLINE`) function with rolled loops, on +// purpose. The first cubin build of the fully inlined, fully unrolled form ran +// 41 minutes and emitted 56 MB of PTX: one permutation was ~49k straight-line +// lines (the inverse S-box chain unrolled over twelve lanes, three times) and +// every leaf kernel carried one copy per `permute` call site — seven in the +// comp-poly kernel. Rolled and called, the whole file is a few thousand lines +// and every kernel shares one body. The price is loop overhead of order 10% of +// the permutation's instructions and the state living in local memory across +// the call; the `-Xptxas -v` report and the unroll factors of `square_n` and +// the lane loops are the tuning knobs, in that order. +// +// `permute_v` is the permutation at variant V; every instantiation +// is its own called body. `permute` — what every kernel calls — is this +// cubin's variant. +template +RPX_NOINLINE __device__ void permute_v(uint64_t s[STATE_FELTS]) { +#pragma unroll 1 + for (int r = 0; r + 1 < NUM_ROUNDS; r += 2) { + fb_round_v(s, r); + ext_round(s, r + 1); + } + final_round(s, NUM_ROUNDS - 1); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = goldilocks::canonical(s[i]); +} + +__device__ __forceinline__ void permute(uint64_t s[STATE_FELTS]) { + permute_v(s); +} + +// --------------------------------------------------------------------------- +// The socket's two constructions over the permutation. +// --------------------------------------------------------------------------- + +// The rate-8 OVERWRITE duplex — `algebraic_commit::sponge_leaf` (:169-184) +// with `leaf_capacity` (:142-147), streamed. Capacity lane 8 carries the +// padding flag `len mod 8`, lane 9 the LEAF domain, lanes 10-11 zero. Each +// block OVERWRITES the eight rate lanes (spec §2.6): absorption is a store, no +// field arithmetic. The total length is needed BEFORE the first permutation +// (algebraic_commit.rs "A1"), hence `init(num_felts)`; callers absorb exactly +// that many felts. +struct Sponge { + uint64_t s[STATE_FELTS]; + int pos; + + __device__ __forceinline__ void init(uint64_t num_felts) { +#pragma unroll + for (int i = 0; i < RATE_FELTS; ++i) s[i] = 0; + s[CAPACITY_PAD_LANE] = num_felts % RATE_FELTS; + s[CAPACITY_DOMAIN_LANE] = DOMAIN_LEAF; + s[CAPACITY_DOMAIN_LANE + 1] = 0; + s[CAPACITY_DOMAIN_LANE + 2] = 0; + pos = 0; + } + + __device__ __forceinline__ void absorb(uint64_t felt) { + s[pos++] = felt; + if (pos == RATE_FELTS) { + permute(s); + pos = 0; + } + } + + // A pending partial block is zero-padded and permuted. An exact multiple of + // the rate spends no trailing permutation — including the EMPTY leaf, whose + // digest is therefore the untouched zero rate lanes, exactly what + // `sponge_leaf` returns for `felts.is_empty()` (:174-176). + __device__ __forceinline__ void finalize(uint64_t digest[DIGEST_FELTS]) { + if (pos != 0) { + for (int k = pos; k < RATE_FELTS; ++k) s[k] = 0; + permute(s); + pos = 0; + } +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) digest[i] = s[i]; + } +}; + +// `sponge_leaf` over a contiguous array — the one-call form for the KAT and +// for any phase-2 kernel that has its felts in hand. +__device__ __forceinline__ void sponge_leaf(const uint64_t *felts, uint64_t num_felts, + uint64_t digest[DIGEST_FELTS]) { + Sponge sp; + sp.init(num_felts); + for (uint64_t i = 0; i < num_felts; ++i) sp.absorb(felts[i]); + sp.finalize(digest); +} + +// The Merkle parent: ONE permutation of `[left ‖ right ‖ capacity]` with the +// compress domain, which is zero (algebraic_commit.rs:248-252 → +// hash.rs:95-108). Capacity = `domain_iv(0)` = all zeros. +__device__ __forceinline__ void compress(const uint64_t left[DIGEST_FELTS], + const uint64_t right[DIGEST_FELTS], + uint64_t out[DIGEST_FELTS]) { + uint64_t s[STATE_FELTS]; +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) { + s[i] = left[i]; + s[DIGEST_FELTS + i] = right[i]; + s[RATE_FELTS + i] = 0; + } + s[CAPACITY_DOMAIN_LANE] = DOMAIN_COMPRESS; + permute(s); +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) out[i] = s[i]; +} + +} // namespace rpx + +// =========================================================================== +// PHASE 2 — the device-facing surface: node bytes, leaf kernels, Merkle +// compressors and the permutation probe. Kernel for kernel the twin of +// `blake3.cu:338-620`, with the chain replaced by `rpx::Sponge` and the parent +// by `rpx::compress`. +// +// NODE BYTES. A node is four canonical felts, each stored as eight BIG-ENDIAN +// bytes — `digest_to_commitment` (algebraic_commit.rs:112-118) — so 32 bytes, +// the same slot width as a BLAKE3 or keccak node, and the device tree's bytes +// equal the host's. Digests leave `permute` canonical; a parent reads its +// children back with `commitment_to_digest`'s big-endian decoding. The device +// is little-endian, so both directions byte-swap (`bswap64`); the 32-byte node +// offsets inside a 256-byte-aligned `cuMemAlloc` buffer make the u64 accesses +// aligned, the same precondition the BLAKE3 u32 accesses rest on. +// +// A LEAF absorbs exactly the felt sequence the host leaf hashes: the same +// read pattern as the BLAKE3 kernel it twins (`leaves_bit_reversed_grouped`, +// commitment.rs:67 — bit-reversed rows, each column by column, an ext3 element +// as its three components), which is the sequence `felts_from_bytes` rebuilds +// from the leaf bytes, so `hash_bytes == hash_data` holds on device by +// construction. The felt count is known before the loop, as the overwrite +// duplex's padding flag needs it (A1). Raw `[0, 2^64)` storage is absorbed as +// is: the permutation is representation-independent, and the host +// canonicalises before serialising — same field value, same digest. +// =========================================================================== + +namespace rpx { + +// Byte-swap a u64: the device reads a host big-endian felt from a node and +// writes one back. Plain shifts so the host shim compiles it; nvcc lowers it +// to two PRMTs. +__device__ __forceinline__ uint64_t bswap64(uint64_t x) { + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x >> 8) & 0x00FF00FF00FF00FFull); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x >> 16) & 0x0000FFFF0000FFFFull); + return (x << 32) | (x >> 32); +} + +// Four felts → one 32-byte node, `digest_to_commitment`'s layout. +__device__ __forceinline__ void store_digest_be(const uint64_t digest[DIGEST_FELTS], uint8_t *node) { + uint64_t *dst = reinterpret_cast(node); +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) dst[i] = bswap64(digest[i]); +} + +// One 32-byte node → four felts, `commitment_to_digest`'s decoding. +__device__ __forceinline__ void load_digest_be(const uint8_t *node, uint64_t digest[DIGEST_FELTS]) { + const uint64_t *src = reinterpret_cast(node); +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) digest[i] = bswap64(src[i]); +} + +// A Merkle parent in place in the node buffer — `parent` (algebraic_commit.rs +// :248-252): decode both children, `compress`, encode. Node buffer layout as +// `blake3.cu` / `keccak.cu` / the CPU `merkle.rs`: children at +// `nodes[parent_begin + n_pairs .. parent_begin + 3*n_pairs]`, parents at +// `nodes[parent_begin .. parent_begin + n_pairs]`, 32 bytes per node. +__device__ __forceinline__ void hash_merkle_parent(uint8_t *nodes, uint64_t parent_begin, + uint64_t n_pairs, uint64_t tid) { + uint64_t left[DIGEST_FELTS], right[DIGEST_FELTS], out[DIGEST_FELTS]; + load_digest_be(nodes + (parent_begin + n_pairs + 2 * tid) * 32, left); + load_digest_be(nodes + (parent_begin + n_pairs + 2 * tid + 1) * 32, right); + compress(left, right, out); + store_digest_be(out, nodes + (parent_begin + tid) * 32); +} + +} // namespace rpx + +// --------------------------------------------------------------------------- +// Leaf kernels. Twins of `blake3_leaves_*` / `blake3_comp_poly_leaves_ext3` / +// `blake3_fri_leaves_ext3`, argument for argument; one thread hashes one leaf. +// --------------------------------------------------------------------------- + +// Goldilocks BASE-FIELD leaf hashing, one leaf per bit-reversed row: column +// `c` of row `br` at `columns_base_ptr[c * col_stride + br]`. +// Twin of `blake3_leaves_base_batched` (`blake3.cu:346`). +extern "C" __global__ void rpx_leaves_base_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(num_cols); + for (uint64_t c = 0; c < num_cols; ++c) sp.absorb(columns_base_ptr[c * col_stride + br]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// BASE-FIELD row-pair leaf hashing: leaf `tid` hashes bit-reversed rows +// `2*tid` and `2*tid+1`, each column by column, first row then second. +// `num_leaves = num_rows / 2`. Twin of `blake3_leaves_base_row_pair_batched`. +extern "C" __global__ void rpx_leaves_base_row_pair_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(2 * num_cols); + for (uint64_t c = 0; c < num_cols; ++c) sp.absorb(columns_base_ptr[c * col_stride + br_0]); + for (uint64_t c = 0; c < num_cols; ++c) sp.absorb(columns_base_ptr[c * col_stride + br_1]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// EXT3 leaf hashing, one leaf per bit-reversed row, components in three +// separate base slabs: column `c` component `k` at +// `columns_base_ptr[(c*3 + k) * col_stride + br]`; an element is absorbed as +// `[comp0, comp1, comp2]`, matching `write_bytes_be`. +// Twin of `blake3_leaves_ext3_batched`. +extern "C" __global__ void rpx_leaves_ext3_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, // number of ext3 columns (NOT slabs) + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(3 * num_cols); + for (uint64_t c = 0; c < num_cols; ++c) { +#pragma unroll + for (int k = 0; k < 3; ++k) { + sp.absorb(columns_base_ptr[(c * 3 + (uint64_t)k) * col_stride + br]); + } + } + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// Composition-polynomial leaf hashing: each leaf absorbs `2 * num_parts` ext3 +// values from bit-reversed rows `2*tid` and `2*tid+1`, (row 0: parts) then +// (row 1: parts), three base components per value. +// Twin of `blake3_comp_poly_leaves_ext3`. +extern "C" __global__ void rpx_comp_poly_leaves_ext3( + const uint64_t *parts_base_ptr, + uint64_t col_stride, + uint64_t num_parts, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(2 * 3 * num_parts); + for (uint64_t p = 0; p < num_parts; ++p) { +#pragma unroll + for (int k = 0; k < 3; ++k) { + sp.absorb(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_0]); + } + } + for (uint64_t p = 0; p < num_parts; ++p) { +#pragma unroll + for (int k = 0; k < 3; ++k) { + sp.absorb(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_1]); + } + } + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, leaves_out + tid * 32); +} + +// FRI layer leaf hashing: each leaf absorbs two consecutive ext3 values from an +// interleaved eval vector `[a0,a1,a2,b0,b1,b2,...]` — six felts, so a single +// block, no padding flag (`6 mod 8 = 6` in capacity lane 8). No bit reversal. +// The host is `AlgebraicPairBackend::hash_data` (algebraic_commit.rs:318-329). +// Twin of `blake3_fri_leaves_ext3`. +extern "C" __global__ void rpx_fri_leaves_ext3( + const uint64_t *evals_interleaved, // 3 * num_evals u64s + uint64_t num_leaves, // = num_evals / 2 + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + const uint64_t *pair = evals_interleaved + 2 * tid * 3; + + rpx::Sponge sp; + sp.init(6); +#pragma unroll + for (int i = 0; i < 6; ++i) sp.absorb(pair[i]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, leaves_out + tid * 32); +} + +// FRI GROUP-leaf hashing (S3): leaf `tid` absorbs the `group` consecutive ext3 +// values `evals[tid*group .. (tid+1)*group]` of an interleaved eval vector — the +// `3*group` contiguous felts at `evals_interleaved + tid*group*3` — in order, a +// sponge over `3*group` felts (the count keys the padding). The host +// `AlgebraicBatchBackend` leaf over the group; at `group = 2` exactly +// `rpx_fri_leaves_ext3`'s six felts. Twin of `keccak_fri_group_leaves_ext3`. +extern "C" __global__ void rpx_fri_group_leaves_ext3( + const uint64_t *evals_interleaved, // 3 * num_leaves * group u64s + uint64_t num_leaves, + uint64_t group, // ext3 values per leaf (2^d) + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + const uint64_t *g = evals_interleaved + tid * group * 3; + + rpx::Sponge sp; + sp.init(3 * group); + for (uint64_t i = 0; i < 3 * group; ++i) sp.absorb(g[i]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, leaves_out + tid * 32); +} + +// Row-major ROW-PAIR leaf hashing: leaf `tid` absorbs row `reverse_index(2*tid)` +// then row `reverse_index(2*tid+1)`, each `m` lanes read contiguously from +// `data + br * m`. `m` is the row stride in u64s: base trace = column count, +// ext3 trace = 3 * column count (an ext3 element's components are consecutive). +// Twin of `blake3_leaves_base_row_major_row_pair`; the fused LDE+commit +// pipeline's leaf kernel (`lde.rs` `coset_lde_row_major_inner`). +extern "C" __global__ void rpx_leaves_base_row_major_row_pair( + const uint64_t *data, + uint64_t m, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + rpx::Sponge sp; + sp.init(2 * m); + for (uint64_t c = 0; c < m; ++c) sp.absorb(row_0[c]); + for (uint64_t c = 0; c < m; ++c) sp.absorb(row_1[c]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// Column-range variant: each leaf absorbs only columns `[col_start, col_end)` +// of the two bit-reversed rows while `m` stays the full row stride — the CPU +// `commit_rows_bit_reversed_subset`, how preprocessed tables commit their +// precomputed and multiplicity column ranges to separate trees over one LDE. +// Twin of `blake3_leaves_base_row_major_row_pair_range`. +extern "C" __global__ void rpx_leaves_base_row_major_row_pair_range( + const uint64_t *data, + uint64_t m, + uint64_t col_start, + uint64_t col_end, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + rpx::Sponge sp; + sp.init(2 * (col_end - col_start)); + for (uint64_t c = col_start; c < col_end; ++c) sp.absorb(row_0[c]); + for (uint64_t c = col_start; c < col_end; ++c) sp.absorb(row_1[c]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// Row-major ONE-ROW leaf hashing (S2, rows_per_leaf = 1): leaf `tid` absorbs the +// single row `reverse_index(tid)`, columns `[col_start, col_end)` of the +// row-major buffer (`m` the full row stride) — a sponge over +// `col_end - col_start` felts (the count keys the padding). The CPU +// `commit_rows_bit_reversed_subset_with(.., 1)`. Twin of +// `keccak256_leaves_base_row_major_row_range`. +extern "C" __global__ void rpx_leaves_base_row_major_row_range( + const uint64_t *data, + uint64_t m, + uint64_t col_start, + uint64_t col_end, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + const uint64_t *row = data + br * m; + + rpx::Sponge sp; + sp.init(col_end - col_start); + for (uint64_t c = col_start; c < col_end; ++c) sp.absorb(row[c]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// --------------------------------------------------------------------------- +// COSET leaf hashing — the WHIR shape, and the two kernels the per-table branch +// has no twin for. +// +// Every other leaf kernel in this file hashes a ROW GROUP: a leaf is a row (or +// a row pair) read across the columns. WHIR's leaf is a fold COSET: leaf `j` +// holds the `2^log_folding` codeword positions that fold onto `j`, which are +// strided by `num_leaves`. Twins of `keccak256_leaves_base_coset` / +// `keccak256_leaves_ext3_coset` (`keccak.cu`), argument for argument, with the +// sponge swapped. +// +// The felt count is known before the loop, which the overwrite duplex needs for +// its padding flag: a base leaf is `block` felts, an ext3 leaf `3 * block`. Raw +// `[0, 2^64)` storage is absorbed as is — the permutation is +// representation-independent and the host canonicalises before serialising, so +// the same field value gives the same digest either way. +// --------------------------------------------------------------------------- + +// Goldilocks BASE-FIELD coset leaves: leaf `tid` hashes +// `codeword[tid + t * num_leaves]` for `t` in `[0, block)`. +extern "C" __global__ void rpx_leaves_base_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + rpx::Sponge sp; + sp.init(block); + for (uint64_t t = 0; t < block; ++t) sp.absorb(codeword[tid + t * num_leaves]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, out + tid * 32); +} + +// EXT3 coset leaves: the same stride, each element as its three components in +// order — what `element_felts` produces for a cubic-extension element, and what +// the base kernel above does one component at a time. +extern "C" __global__ void rpx_leaves_ext3_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + rpx::Sponge sp; + sp.init(block * 3); + for (uint64_t t = 0; t < block; ++t) { + const uint64_t *at = codeword + (tid + t * num_leaves) * 3; +#pragma unroll + for (int k = 0; k < 3; ++k) sp.absorb(at[k]); + } + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, out + tid * 32); +} + +// --------------------------------------------------------------------------- +// Merkle level / tail. Same launch split as BLAKE3's: one thread per pair per +// level while a level is wide, then ONE single-block launch that grid-strides +// every remaining level with a barrier between them. +// --------------------------------------------------------------------------- + +// One level of the inner tree: each thread compresses one child pair. +extern "C" __global__ void rpx_merkle_level(uint8_t *nodes, + uint64_t parent_begin, // in 32-byte nodes + uint64_t n_pairs) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n_pairs) return; + rpx::hash_merkle_parent(nodes, parent_begin, n_pairs, tid); +} + +// Every remaining level from `level_begin` up to the root, in one block. +// Twin of `blake3_merkle_tail`. +extern "C" __global__ void rpx_merkle_tail(uint8_t *nodes, uint64_t level_begin) { + uint64_t lb = level_begin; + while (lb != 0) { + uint64_t nb = lb / 2; + uint64_t n_pairs = lb - nb; + for (uint64_t tid = threadIdx.x; tid < n_pairs; tid += blockDim.x) { + rpx::hash_merkle_parent(nodes, nb, n_pairs, tid); + } + __syncthreads(); + lb = nb; + } +} + +// --------------------------------------------------------------------------- +// Parity-harness entry point: `n` independent permutations, one thread each. +// The bare device permutation is otherwise unreachable from host code; this is +// what lets the GPU be checked against the host `Rpx256` (and the host-KAT's +// oracle tables) before any tree is built. Not on any production path. +// --------------------------------------------------------------------------- +extern "C" __global__ void rpx_permute_probe(const uint64_t *states, uint64_t n, uint64_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint64_t s[rpx::STATE_FELTS]; +#pragma unroll + for (int i = 0; i < rpx::STATE_FELTS; ++i) s[i] = states[tid * rpx::STATE_FELTS + i]; + rpx::permute(s); +#pragma unroll + for (int i = 0; i < rpx::STATE_FELTS; ++i) out[tid * rpx::STATE_FELTS + i] = s[i]; +} + +// --------------------------------------------------------------------------- +// Proof-of-work grinding search, RPX arm. +// +// Twin of `keccak.cu`'s `grind_search`, same signature shape and the same +// first-hit reduction; only the outer hash differs. The host path it replaces +// is `stark::grinding::generate_nonce`, a per-table ~2^grinding_factor search +// that is the prover's dominant CPU cost once the transcript is algebraic. +// +// THE MAPPING this reproduces, derived from the host (✓ VERIFIED against the +// sources named) and stated here so it is not re-derived at each reading: +// +// host predicate stark/src/grinding.rs::is_valid_nonce_for_inner_hash: +// valid ⇔ u64::from_be_bytes(D::digest(inner ‖ nonce.to_be_bytes())[..8]) < limit, +// limit = 1 << (64 − grinding_factor); inner = D::digest(PREFIX ‖ seed ‖ factor), +// 41 bytes, computed ONCE per table on the host and never on device. +// D for RPX prover/src/lfm/algebraic_commit.rs AlgebraicDigest: +// D::digest(bytes) = digest_to_commitment(sponge_leaf(Rpx, felts_from_bytes(bytes))) +// — the LEAF construction, on purpose. +// bytes → felts felts_from_bytes: consecutive 8-byte groups, each +// FE::from(u64::from_be_bytes(group)) — BIG-endian, and `FE::from` is +// `from_u64`, which maps a raw value ≥ p to raw − p (ONE subtraction, +// goldilocks.rs:172-178 — exactly `goldilocks::canonical`). +// The 40-byte outer block is therefore EXACTLY five felts: +// f0..f3 = the inner hash's four big-endian u64s (canonical already — +// they are digest_to_commitment output, so each < p), +// f4 = the nonce. +// ⚠ Big-endian, unlike keccak's `inner_hash_lanes` (LITTLE-endian lanes). +// A separate host helper, `stark::grinding::inner_hash_felts` (BE), feeds +// this kernel; feeding it the keccak lanes is a silent wrong hash. +// sponge mode sponge_leaf over five felts: ONE permutation of +// [f0, f1, f2, f3, nonce, 0, 0, 0 | 5, 0x4C4D464C, 0, 0] +// — rate lanes 5..8 zero-padded, capacity lane 8 = padding flag +// `5 mod 8 = 5`, lane 9 = DOMAIN_LEAF ("LFML"), lanes 10, 11 = 0. +// ★ Built here through `rpx::Sponge` — `init(5)`, five `absorb`s, +// `finalize` — rather than by writing those twelve lanes out, so the +// capacity rule has ONE statement on device and a change to it cannot +// leave the grind behind. +// the head digest_to_commitment writes lane 0 CANONICAL as 8 big-endian bytes and +// the host reads those 8 bytes back big-endian, so `seed_head` IS the +// canonical value of state lane 0 after the permutation — no byte +// reinterpretation. `permute` canonicalises its output, so on device the +// predicate is just `digest[0] < limit`. +// +// THE NONCE LANE is `goldilocks::canonical(nonce)`, matching `FE::from(nonce)` +// exactly. It is a no-op for every nonce this search can reach (the first +// nonce ≥ p is 2^64 − 2^32 + 1, and the launcher's range walk bails long +// before), and the device representation is lazy anyway — but absorbing the +// canonical value is what makes "the device absorbs what `FE::from` produces" +// true by inspection rather than by an argument about reachability. +// +// Each thread strides over `[base, base+count)` and `atomicMin`s the smallest +// valid nonce it finds into `*result` (initialised to U64_MAX by the caller), +// so the launch returns the globally smallest valid nonce in the searched +// block — deterministic despite the parallel grid, and any valid nonce +// satisfies the verifier. +// --------------------------------------------------------------------------- + +// The outer block is `inner_hash ‖ nonce`: 40 bytes, five felts. Named because +// the capacity's padding flag is `5 mod 8` and the count is what `init` needs. +__device__ constexpr uint64_t GRIND_FELTS = 5; + +extern "C" __global__ void rpx_grind_search(const uint64_t *inner_felts, + uint64_t limit, + uint64_t base, + uint64_t count, + volatile unsigned long long *result) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + const uint64_t f0 = inner_felts[0], f1 = inner_felts[1], f2 = inner_felts[2], + f3 = inner_felts[3]; + for (uint64_t i = tid; i < count; i += stride) { + uint64_t nonce = base + i; + // Guard the u64 wrap on the final block (the launcher bails before it, + // so this is unreachable in practice): a wrapped nonce is < base, so + // stop rather than re-scan from 0. + if (nonce < base) break; + // A thread's nonces only increase, so once a smaller valid one is known + // this thread can never beat it — stop scanning. `result` is volatile + // so this load re-reads L2 (where the atomicMin writes land) instead of + // being hoisted into a register or served stale from L1; the early exit + // depends on that, though correctness does not. + if (nonce >= (uint64_t)*result) break; + rpx::Sponge sp; + sp.init(GRIND_FELTS); + sp.absorb(f0); + sp.absorb(f1); + sp.absorb(f2); + sp.absorb(f3); + sp.absorb(goldilocks::canonical(nonce)); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + if (digest[0] < limit) { + atomicMin((unsigned long long *)result, (unsigned long long)nonce); + } + } +} + +// --------------------------------------------------------------------------- +// ⛔ A DIAGNOSTIC TWIN, NEVER A PROVING PATH. +// +// `rpx_grind_search` above is the shipped kernel and this file does not change +// it. This twin is the same search with three counters, and it exists to answer +// ONE question that no timing can: when a launch is slow, does it EXECUTE MORE +// PERMUTATIONS, or the same ones more slowly? +// +// The question is not idle. The paired microbench reads a mean 20% above the +// model on the record posture while the MEDIAN seed sits on it, and the excess +// saturates in `count` rather than growing with it: converted to iterations per +// thread the excess fits `T·(1 − e^(−(N−8)/τ))` with τ ≈ 19 on three +// independent points. That is the shape of a thread that keeps scanning for a +// bounded TIME after the answer is known — a poll of `*result` served stale — +// and not the shape of a thread running to the loop bound. `executed` settles +// it: stale polls mean real extra permutations, bounded by time and therefore +// the SAME at scan 8 and scan 64; anything else means the work is unchanged and +// the cost is outside this loop. +// +// ⚠ WASTED WORK, NEVER A WRONG ANSWER. The search returns the globally smallest +// valid nonce whatever any thread does after the `atomicMin`, which is why the +// counters can be read at leisure while the answer stays pinned by +// `gpu_grind_returns_smallest_valid_nonce`. Nothing here is a soundness matter. +// +// ⛔ `#if defined(__CUDACC__)`, and that is deliberate rather than defensive: +// the host-KAT compiles this file through `cuda_host_shim.h`, which supplies +// `gridDim`, `blockDim` and `atomicMin` but NOT `__shfl_down_sync`. A KAT has +// no answer to check here anyway — this kernel produces no digest — and its +// nonce agreement with the shipped kernel is asserted on the device, seed by +// seed, by the bench that launches it. +// +// SIZING, before the first cubin, by the rule this file learnt the hard way +// (`permute` stays a called function; see its CODE SHAPE note): one more +// `permute` CALL SITE, not one more inlined copy, so this adds an entry of +// order 150-250 PTX lines against a file of 6,241 — not a duplicated +// permutation body. If the `.ptx` grows by thousands, something inlined that +// must not. +#if defined(__CUDACC__) + +// One warp's reduction of a sum and a max, so the counters cost four atomics a +// warp instead of one per thread. +// +// ⛔ A per-thread `atomicAdd` would be 131,072 serialised updates of one L2 +// line per launch — the same order as the 5-11 ms effect being measured. An +// instrument that manufactures its own signal answers a different question. +// Every thread in a warp reaches this (the loop's `break`s leave the loop, not +// the function), so the full mask is correct. +// ⛔ `unsigned long long`, NOT `uint64_t`, and that is not a style choice. The +// shuffle intrinsics are overloaded on `int`, `unsigned int`, `long long`, +// `unsigned long long`, `float` and `double`. On an LP64 host `uint64_t` is +// `unsigned long`, which is NONE of them — the call would be ambiguous or +// absent rather than wrong, so it fails at compile time on the box and not +// here, where no nvcc runs. The file already casts for exactly this reason +// where it calls `atomicMin`. +__device__ __forceinline__ void warp_reduce_counts(unsigned long long &sum, + unsigned long long &max_v, + unsigned long long &ends) { +#pragma unroll + for (int off = 16; off > 0; off >>= 1) { + sum += __shfl_down_sync(0xffffffffu, sum, off); + ends += __shfl_down_sync(0xffffffffu, ends, off); + const unsigned long long other = __shfl_down_sync(0xffffffffu, max_v, off); + if (other > max_v) max_v = other; + } +} + +// The three counters live in ONE device array so a search reads them back in a +// single copy: `counts[COUNT_EXECUTED]` and `counts[COUNT_RAN_TO_END]` are +// accumulated with `atomicAdd` across every launch of the search, +// `counts[COUNT_MAX_ITERS]` with `atomicMax`. The host mirrors these names. +__device__ constexpr int COUNT_EXECUTED = 0; +__device__ constexpr int COUNT_MAX_ITERS = 1; +__device__ constexpr int COUNT_RAN_TO_END = 2; + +// ⭐ THE POLL-RATE KNOB (`poll_period`, the RPX grind k-sweep's k). The shipped +// kernel polls `*result` every iteration; this twin polls it every +// `poll_period`-th iteration, STAGGERED across threads. It answers whether the +// stale-poll overrun is driven by CONTENTION on that one address: 131,072 +// resident threads each issue a system-scope load to it every permutation, and +// the finder's `atomicMin` queues behind that flood. If lowering the poll rate +// makes the overrun FALL, contention is the cause and a warp- or block-level +// poll is the fix; if it RISES ~linearly in `poll_period`, polling less just +// wastes more scanning and the poll family is dead. `poll_period == 1` +// reproduces the shipped every-iteration poll exactly, which is the sweep's +// admissibility control. ⚠ Wasted work, never a wrong answer — the search still +// returns the globally smallest valid nonce whatever the poll rate; see the +// soundness note above. +extern "C" __global__ void rpx_grind_search_counted(const uint64_t *inner_felts, + uint64_t limit, + uint64_t base, + uint64_t count, + volatile unsigned long long *result, + unsigned long long *counts, + uint64_t poll_period) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + const uint64_t f0 = inner_felts[0], f1 = inner_felts[1], f2 = inner_felts[2], + f3 = inner_felts[3]; + // Permutations THIS thread ran. Counted after both exit tests and before + // the sponge, so it counts work done and never work declined. + uint64_t iters = 0; + // Did this thread leave by the loop bound rather than by an early exit? + // Only meaningful for a thread that did some work: a thread whose `tid` is + // past `count` never enters the loop and must not be scored as having + // scanned to the end. + uint64_t to_end = 0; + // ⭐ STAGGERED POLL. `poll_period` is a power of two (caller-guaranteed), so + // the period is a bitmask; `(n + tid) & poll_mask == 0` fires once every + // `poll_period` iterations with a phase that differs per thread, so at any + // one iteration only 1/poll_period of threads issue the system-scope load. + // A synchronised `n & poll_mask` would fire on the SAME iteration for every + // thread, holding the peak request rate constant and changing only the duty + // cycle. poll_mask == 0 (poll_period == 1) polls every iteration, identical + // to the shipped kernel. + const uint64_t poll_mask = poll_period - 1; + // This thread's own loop-iteration counter, kept independent of `i / stride` + // (that identity holds only while `tid < stride`; a counter cannot be broken + // by a future change to the indexing). + uint64_t n = 0; + uint64_t i = tid; + for (; i < count; i += stride) { + uint64_t nonce = base + i; + if (nonce < base) break; + if (((n + tid) & poll_mask) == 0 && nonce >= (uint64_t)*result) break; + ++n; + ++iters; + rpx::Sponge sp; + sp.init(GRIND_FELTS); + sp.absorb(f0); + sp.absorb(f1); + sp.absorb(f2); + sp.absorb(f3); + sp.absorb(goldilocks::canonical(nonce)); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + if (digest[0] < limit) { + atomicMin((unsigned long long *)result, (unsigned long long)nonce); + } + } + if (i >= count && iters > 0) to_end = 1; + + unsigned long long sum = (unsigned long long)iters; + unsigned long long max_v = (unsigned long long)iters; + unsigned long long ends = (unsigned long long)to_end; + warp_reduce_counts(sum, max_v, ends); + if ((threadIdx.x & 31u) == 0u) { + atomicAdd(&counts[COUNT_EXECUTED], sum); + atomicAdd(&counts[COUNT_RAN_TO_END], ends); + atomicMax(&counts[COUNT_MAX_ITERS], max_v); + } +} + +#endif // __CUDACC__ + +// =========================================================================== +// ★ The half-warp Merkle kernels and the work-queue grind (the defaults, which +// math-cuda `rpx_paths` switches back to `rpx_merkle_level`/`rpx_merkle_tail` +// and `rpx_grind_search`), and the probes the parity tests and benches use. +// Every kernel here produces the same bytes (nodes, nonces, digests) as the +// kernel it stands in for. +// =========================================================================== + +// --------------------------------------------------------------------------- +// K5 probes. `rpx_permute_chain_probe_v` permutes each thread's state `k` +// times at variant V — chained, so a launch is compute rather than memory and +// a wrong step anywhere reaches the output. `rpx_limb_probe` returns the two +// limb primitives and `goldilocks::mul` side by side on raw pairs, so the PTX +// is checked against 128-bit arithmetic on the device itself. V is a template +// argument, so probe v0 is the 64-bit multiply in every cubin. +// --------------------------------------------------------------------------- +template +__device__ __forceinline__ void permute_chain_probe(const uint64_t *states, uint64_t n, + uint64_t k, uint64_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint64_t s[rpx::STATE_FELTS]; +#pragma unroll + for (int i = 0; i < rpx::STATE_FELTS; ++i) s[i] = states[tid * rpx::STATE_FELTS + i]; + for (uint64_t j = 0; j < k; ++j) rpx::permute_v(s); +#pragma unroll + for (int i = 0; i < rpx::STATE_FELTS; ++i) out[tid * rpx::STATE_FELTS + i] = s[i]; +} + +#define RPX_CHAIN_PROBE(V) \ + extern "C" __global__ void rpx_permute_chain_probe_v##V(const uint64_t *states, uint64_t n, \ + uint64_t k, uint64_t *out) { \ + permute_chain_probe(states, n, k, out); \ + } +RPX_CHAIN_PROBE(0) +RPX_CHAIN_PROBE(1) +RPX_CHAIN_PROBE(3) +RPX_CHAIN_PROBE(4) +RPX_CHAIN_PROBE(5) +RPX_CHAIN_PROBE(7) +RPX_CHAIN_PROBE(9) +#undef RPX_CHAIN_PROBE + +extern "C" __global__ void rpx_limb_probe(const uint64_t *a, const uint64_t *b, uint64_t n, + uint64_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + out[3 * tid] = rpx::mul_limb(a[tid], b[tid]); + out[3 * tid + 1] = rpx::sqr_limb(a[tid]); + out[3 * tid + 2] = goldilocks::mul(a[tid], b[tid]); +} + +// The poll of a word other threads update with atomics: a volatile load on the +// device (the stride grind's `LDG.E.64.STRONG.SYS`), an atomic load under the +// host SIMT shim, whose lanes are real threads. +__device__ __forceinline__ unsigned long long rpx_poll_u64(unsigned long long *p) { +#if defined(__CUDA_ARCH__) + return *(volatile unsigned long long *)p; +#else + return __atomic_load_n(p, __ATOMIC_SEQ_CST); +#endif +} + +#if defined(__CUDACC__) || defined(RPX_HOST_SIMT) + +// --------------------------------------------------------------------------- +// K4 — the grind with nonces claimed from a WORK QUEUE, in increasing order. +// +// The stride kernel gives thread `t` the fixed nonces `t, t + stride, …`, so a +// warp that runs ahead scans nonces far above the answer while a slow warp is +// still below it; the search cannot stop until the slowest owner of a nonce +// below the answer gets there. The counted twin measured the cost: 3.30 strides +// of over-scan past `nonce + stride` at the record posture, with a ZERO +// contention component (the O6 poll k-sweep), i.e. skew, not stale polls. +// +// Here a warp claims the next 32 unclaimed nonces (one per lane) with ONE +// `atomicAdd` on a queue head, so the nonces are processed in (nearly) +// increasing order whatever the warps' speeds, and the waste is bounded by the +// work in flight when the answer lands — at most one permutation per lane. +// +// ★ THE ANSWER IS UNCHANGED: the smallest valid nonce in `[base, base+count)`, +// or the sentinel. The head only grows and the result only shrinks, to valid +// nonces. A warp stops when its claimed chunk starts at or above the result +// (every later chunk starts higher still) or past `count`. A chunk holding a +// nonce below the final result is therefore always claimed — the head passed +// it before any warp could stop above it — and a claimed chunk that starts +// below the result is hashed in full. So the launch returns what +// `rpx_grind_search` returns, which the parity test pins seed by seed. +// +// `state[0]` = the result (U64_MAX sentinel), `state[1]` = the queue head (an +// offset into `[0, count)`); the launcher resets both before each launch. +// `blockDim.x` must be a multiple of 32 (the claim is per warp). +// --------------------------------------------------------------------------- +__device__ constexpr int GRIND_Q_RESULT = 0; +__device__ constexpr int GRIND_Q_HEAD = 1; +__device__ constexpr unsigned long long GRIND_Q_CHUNK = 32; + +__device__ __forceinline__ unsigned long long shfl_u64(unsigned long long v, int src) { + return __shfl_sync(0xffffffffu, v, src); +} + +// Warp butterfly for the counted twin: sum, max and the ran-to-end count, left +// on every lane. `__shfl_sync` only, so the host SIMT shim runs it too. +__device__ __forceinline__ void warp_sum_max(unsigned long long &sum, unsigned long long &max_v, + unsigned long long &ends) { + const int lane = (int)(threadIdx.x & 31u); +#pragma unroll + for (int off = 16; off > 0; off >>= 1) { + sum += shfl_u64(sum, lane ^ off); + ends += shfl_u64(ends, lane ^ off); + const unsigned long long other = shfl_u64(max_v, lane ^ off); + if (other > max_v) max_v = other; + } +} + +template +__device__ __forceinline__ void grind_queue(const uint64_t *inner_felts, uint64_t limit, + uint64_t base, uint64_t count, + unsigned long long *state, + unsigned long long *counts) { + const unsigned lane = threadIdx.x & 31u; + const uint64_t f0 = inner_felts[0], f1 = inner_felts[1], f2 = inner_felts[2], + f3 = inner_felts[3]; + unsigned long long iters = 0, to_end = 0; + for (;;) { + unsigned long long off = 0, best = 0; + if (lane == 0) { + off = atomicAdd(&state[GRIND_Q_HEAD], GRIND_Q_CHUNK); + best = rpx_poll_u64(&state[GRIND_Q_RESULT]); + } + off = shfl_u64(off, 0); + best = shfl_u64(best, 0); + if (off >= count) { + to_end = iters > 0 ? 1 : 0; + break; + } + const uint64_t first = base + off; + // `first < base`: the u64 wrap on the final block, unreachable in + // practice (the launcher bails first), as in the stride kernel. + if (first < base || first >= best) break; + const uint64_t i = off + lane; + if (i < count) { + const uint64_t nonce = base + i; + rpx::Sponge sp; + sp.init(GRIND_FELTS); + sp.absorb(f0); + sp.absorb(f1); + sp.absorb(f2); + sp.absorb(f3); + sp.absorb(goldilocks::canonical(nonce)); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + if (digest[0] < limit) { + atomicMin(&state[GRIND_Q_RESULT], (unsigned long long)nonce); + } + if constexpr (COUNTED) ++iters; + } + } + if constexpr (COUNTED) { + unsigned long long sum = iters, max_v = iters, ends = to_end; + warp_sum_max(sum, max_v, ends); + if (lane == 0) { + atomicAdd(&counts[0], sum); + atomicMax(&counts[1], max_v); + atomicAdd(&counts[2], ends); + } + } +} + +extern "C" __global__ void rpx_grind_search_queue(const uint64_t *inner_felts, uint64_t limit, + uint64_t base, uint64_t count, + unsigned long long *state) { + grind_queue(inner_felts, limit, base, count, state, nullptr); +} + +// ⛔ DIAGNOSTIC twin: the same search plus the counters `rpx_grind_search_counted` +// keeps, in the same slots (executed, max iterations, ran to end). +extern "C" __global__ void rpx_grind_search_queue_counted(const uint64_t *inner_felts, + uint64_t limit, uint64_t base, + uint64_t count, + unsigned long long *state, + unsigned long long *counts) { + grind_queue(inner_felts, limit, base, count, state, counts); +} + +// --------------------------------------------------------------------------- +// K3 — one permutation across the lanes of a HALF-WARP. +// +// Narrow Merkle levels and the tail are latency-bound: one thread runs one +// permutation as a single dependent chain (~119 µs at the block's clock), +// whatever the width. Here lane `e` of a 16-lane half holds state element `e`, +// so the twelve inverse S-box chains — 95% of the multiplications — run side by +// side, and the MDS and the cubic-extension rounds read the other elements by +// `__shfl_sync`. Lanes 12-15 mirror elements 0-3 (they compute the same values +// and never store), so every shuffle has its whole warp and nothing diverges. +// +// ★ SAME WORDS as `permute`: every element sees the same field operations in +// the same order — `mds` per output lane is the same twelve exact integer terms +// (summed in another order; each half-sum is < 2^40, so no reduction happens +// before the end), and the E round's triple is gathered and raised by the same +// `ext_power7`, each of its three lanes keeping its own coefficient. +// --------------------------------------------------------------------------- +namespace rpx { + +struct WarpLane { + int e; // this lane's state element, 0..11 (idle lanes mirror 0..3) + int group; // the half-warp's first lane: 0 or 16 + int tri; // the first lane of this element's cubic-extension triple + int k; // e mod 3: which coefficient of the triple this lane keeps + bool idle; // lanes 12..15 of a half +}; + +__device__ __forceinline__ WarpLane warp_lane() { + const int lane = (int)(threadIdx.x & 31u); + const int l16 = lane & 15; + WarpLane w; + w.idle = l16 >= STATE_FELTS; + w.e = w.idle ? l16 - STATE_FELTS : l16; + w.group = lane & 16; + w.tri = w.group + EXT_DEGREE * (w.e / EXT_DEGREE); + w.k = w.e % EXT_DEGREE; + return w; +} + +// `mds` for this lane's element. `out_i = Σ_j ROW[(j − i) mod 12]·s_j` with +// `j = (e + k) mod 12` is `Σ_k ROW[k]·s_{(e+k) mod 12}`: the coefficient is +// warp-uniform (a compile-time index) and the source lane varies instead. +__device__ __forceinline__ uint64_t mds_lane(uint64_t v, const WarpLane &w) { + uint64_t acc_lo = 0, acc_hi = 0; // Σ c·l_j and Σ c·h_j, each < 2^40 +#pragma unroll + for (int k = 0; k < STATE_FELTS; ++k) { + int j = w.e + k; + if (j >= STATE_FELTS) j -= STATE_FELTS; + const uint64_t sj = shfl_u64(v, w.group + j); + const uint64_t c = MDS_CIRC_ROW2[k]; + acc_lo += c * (uint64_t)(uint32_t)sj; + acc_hi += c * (uint64_t)(uint32_t)(sj >> 32); + } + const uint64_t lo = (acc_hi << 32) + acc_lo; + const uint64_t carry = (lo < acc_lo) ? 1ull : 0ull; + const uint64_t hi = (acc_hi >> 32) + carry; + return fadd(lo, hi * goldilocks::EPSILON); +} + +template +__device__ __forceinline__ uint64_t fb_round_lane(uint64_t v, int r, const WarpLane &w) { + v = mds_lane(v, w); + v = sbox_v(fadd(v, ARK1[r][w.e])); + v = mds_lane(v, w); + return inv_sbox_v(fadd(v, ARK2[r][w.e])); +} + +__device__ __forceinline__ uint64_t ext_round_lane(uint64_t v, int r, const WarpLane &w) { + v = fadd(v, ARK1[r][w.e]); + CubicExt x; + x.c0 = shfl_u64(v, w.tri); + x.c1 = shfl_u64(v, w.tri + 1); + x.c2 = shfl_u64(v, w.tri + 2); + const CubicExt y = ext_power7(x); + return w.k == 0 ? y.c0 : (w.k == 1 ? y.c1 : y.c2); +} + +// `permute_v` with the state spread over a half-warp: this lane's element +// in, this lane's element out, canonical. +template +__device__ __forceinline__ uint64_t permute_warp_v(uint64_t v, const WarpLane &w) { +#pragma unroll 1 + for (int r = 0; r + 1 < NUM_ROUNDS; r += 2) { + v = fb_round_lane(v, r, w); + v = ext_round_lane(v, r + 1, w); + } + v = mds_lane(v, w); + v = fadd(v, ARK1[NUM_ROUNDS - 1][w.e]); + return goldilocks::canonical(v); +} + +// This lane's element of `compress`'s state `[left ‖ right ‖ capacity]`. The +// two children are adjacent nodes, so elements 0..7 are the eight big-endian +// felts of the 64 bytes at `left_node`; the capacity is zero but for the +// compress domain (itself zero). +__device__ __forceinline__ uint64_t load_compress_lane(const uint8_t *nodes, uint64_t left_node, + int e) { + if (e == CAPACITY_DOMAIN_LANE) return DOMAIN_COMPRESS; + if (e >= 2 * DIGEST_FELTS) return 0; + const uint64_t *src = reinterpret_cast(nodes + left_node * 32); + return bswap64(src[e]); +} + +// One parent from the lanes holding its digest: `store_digest_be`, a felt per lane. +__device__ __forceinline__ void store_parent_lane(uint8_t *nodes, uint64_t parent, uint64_t v, + const WarpLane &w, bool live) { + if (live && !w.idle && w.e < DIGEST_FELTS) { + uint64_t *dst = reinterpret_cast(nodes + parent * 32); + dst[w.e] = bswap64(v); + } +} + +} // namespace rpx + +// One inner level, two parents per warp (one per half). The node layout is +// `rpx_merkle_level`'s: children at `parent_begin + n_pairs + 2p` and `+ 1`, +// parent `p` at `parent_begin + p`. +extern "C" __global__ void rpx_merkle_level_warp(uint8_t *nodes, uint64_t parent_begin, + uint64_t n_pairs) { + const rpx::WarpLane w = rpx::warp_lane(); + const uint64_t first = (((uint64_t)blockIdx.x * blockDim.x + threadIdx.x) >> 5) * 2; + if (first >= n_pairs) return; // warp-uniform + const uint64_t pair = first + (uint64_t)(w.group >> 4); + const bool live = pair < n_pairs; + // A dead half (odd n_pairs) recomputes its neighbour's pair and stores nothing. + const uint64_t p = live ? pair : first; + uint64_t v = rpx::load_compress_lane(nodes, parent_begin + n_pairs + 2 * p, w.e); + v = rpx::permute_warp_v(v, w); + rpx::store_parent_lane(nodes, parent_begin + p, v, w, live); +} + +// Every remaining level from `level_begin` up to the root, in ONE block, with a +// barrier between levels — `rpx_merkle_tail`'s job with a warp per two parents, +// so a level of up to `blockDim.x / 16` pairs is one permutation latency. +extern "C" __global__ void RPX_LAUNCH_BOUNDS(1024) + rpx_merkle_tail_warp(uint8_t *nodes, uint64_t level_begin) { + const rpx::WarpLane w = rpx::warp_lane(); + const uint64_t first_of_warp = (uint64_t)(threadIdx.x >> 5) * 2; + const uint64_t pairs_per_pass = (uint64_t)(blockDim.x >> 5) * 2; + const uint64_t half = (uint64_t)(w.group >> 4); + uint64_t lb = level_begin; + while (lb != 0) { + const uint64_t nb = lb / 2; + const uint64_t n_pairs = lb - nb; + // Warp-uniform bounds: both halves run every pass together. + for (uint64_t first = first_of_warp; first < n_pairs; first += pairs_per_pass) { + const uint64_t pair = first + half; + const bool live = pair < n_pairs; + const uint64_t p = live ? pair : first; + uint64_t v = rpx::load_compress_lane(nodes, nb + n_pairs + 2 * p, w.e); + v = rpx::permute_warp_v(v, w); + rpx::store_parent_lane(nodes, nb + p, v, w, live); + } + __syncthreads(); + lb = nb; + } +} + +// Parity probe for the half-warp permutation: state `i` is permuted by half +// `i mod 2` of warp `i / 2`; all twelve lanes are written back, raw. +extern "C" __global__ void rpx_permute_warp_probe(const uint64_t *states, uint64_t n, + uint64_t *out) { + const rpx::WarpLane w = rpx::warp_lane(); + const uint64_t first = (((uint64_t)blockIdx.x * blockDim.x + threadIdx.x) >> 5) * 2; + if (first >= n) return; + const uint64_t idx = first + (uint64_t)(w.group >> 4); + const bool live = idx < n; + const uint64_t p = live ? idx : first; + uint64_t v = states[p * rpx::STATE_FELTS + w.e]; + v = rpx::permute_warp_v(v, w); + if (live && !w.idle) out[p * rpx::STATE_FELTS + w.e] = v; +} + +#endif // __CUDACC__ || RPX_HOST_SIMT diff --git a/crypto/math-cuda/kernels/sumcheck.cu b/crypto/math-cuda/kernels/sumcheck.cu new file mode 100644 index 000000000..d536e39e3 --- /dev/null +++ b/crypto/math-cuda/kernels/sumcheck.cu @@ -0,0 +1,1426 @@ +// One sumcheck round over the Boolean hypercube, for a batch given as +// straight-line code over its factors (`multilinear::program::Program`, lowered +// by `crypto/multilinear/src/gpu.rs`). +// +// Design: +// * One thread per (cube index, interpolation node), grid-stride in both: +// `blockIdx.x` walks the cube and `blockIdx.y` the nodes. A late round has +// a cube smaller than the device is wide and nothing left to hide the slot +// file's latency behind, and the nodes are the only parallelism left to +// give it. The nodes stay the inner loop, so a factor's `lo` and `hi` are +// read once and stay in cache across the nodes a block owns. +// * Every value is ext3 — the sumcheck runs in one field, and the factors +// were lifted when they were built. +// * The lowering assigns each step a slot with liveness reuse, so the +// per-thread scratch is the program's max live set, not its step count +// (25k steps on the precompile tables). Slots are strided by thread: +// slot `s` component `k` is `d_slots[(s*3 + k) * num_threads + tid]`. +// * The round sums over the whole cube, so each block reduces its threads in +// shared memory and writes one partial per interpolation node. Field +// addition is associative and Goldilocks compares canonically, so the +// order the partials are summed in is not observable. +// +// Op tags and the node packing MUST stay in sync with +// `crypto/multilinear/src/gpu.rs`. + +#include "goldilocks.cuh" +#include "ext3.cuh" + +using ext3::Fe3; + +#define OP_FIXED 0u +#define OP_VAR 1u +#define OP_ADD 2u +#define OP_SUB 3u +#define OP_MUL 4u +#define OP_NEG 5u + +// Interpolation nodes a round can ask for. The host declines above this. +#define MAX_NODES 16 + +// A lowered step: `word0 = op | (a << 32)`, `word1 = b | (res << 32)`. +struct Node { + uint32_t op, a, b, res; +}; + +__device__ __forceinline__ Node load_node(const uint64_t *d_nodes, uint64_t i) { + uint64_t w0 = d_nodes[2 * i]; + uint64_t w1 = d_nodes[2 * i + 1]; + Node n; + n.op = (uint32_t)(w0 & 0xFFFFFFFFull); + n.a = (uint32_t)(w0 >> 32); + n.b = (uint32_t)(w1 & 0xFFFFFFFFull); + n.res = (uint32_t)(w1 >> 32); + return n; +} + +__device__ __forceinline__ Fe3 load_ext(const uint64_t *p) { + return ext3::make(p[0], p[1], p[2]); +} + +__device__ __forceinline__ Fe3 load_slot(const uint64_t *slots, uint64_t stride, uint32_t slot) { + const uint64_t *p = slots + (uint64_t)slot * 3 * stride; + return ext3::make(p[0], p[stride], p[2 * stride]); +} + +__device__ __forceinline__ void store_slot(uint64_t *slots, uint64_t stride, uint32_t slot, + const Fe3 &v) { + uint64_t *p = slots + (uint64_t)slot * 3 * stride; + p[0] = v.a; + p[stride] = v.b; + p[2 * stride] = v.c; +} + +// The program's value at cube index `j`, with every factor extended to the +// interpolation node `t`: `f(j) + t·(f(j + half) − f(j))`. +// +// `INT` (D-ARGUE S1-1, `LAMBDA_VM_ARGUE_INT_NODES`): the node is a base-field +// integer `(k, 0, 0)` — every round's nodes are `1..=D` — so `t·(hi − lo)` is +// the componentwise `k·(hi − lo)`: three products where the full multiply +// spends nine. The same value; the host only takes this variant when every +// node's upper limbs are zero. +template +__device__ __forceinline__ Fe3 eval_program_t(const uint64_t *__restrict__ d_nodes, + uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, + const uint64_t *const *__restrict__ d_factors, + uint64_t j, uint64_t half, const Fe3 &t, + uint64_t *slots, uint64_t stride, + uint32_t root_slot) { + for (uint64_t i = 0; i < num_nodes; i++) { + Node nd = load_node(d_nodes, i); + switch (nd.op) { + case OP_VAR: { + const uint64_t *column = d_factors[nd.a]; + Fe3 lo = load_ext(column + j * 3); + Fe3 hi = load_ext(column + (j + half) * 3); + Fe3 v = INT ? ext3::add(lo, ext3::mul_base(ext3::sub(hi, lo), t.a)) + : ext3::add(lo, ext3::mul(t, ext3::sub(hi, lo))); + store_slot(slots, stride, nd.res, v); + break; + } + case OP_FIXED: + store_slot(slots, stride, nd.res, load_ext(d_consts + (uint64_t)nd.a * 3)); + break; + case OP_ADD: + store_slot(slots, stride, nd.res, + ext3::add(load_slot(slots, stride, nd.a), load_slot(slots, stride, nd.b))); + break; + case OP_SUB: + store_slot(slots, stride, nd.res, + ext3::sub(load_slot(slots, stride, nd.a), load_slot(slots, stride, nd.b))); + break; + case OP_MUL: + store_slot(slots, stride, nd.res, + ext3::mul(load_slot(slots, stride, nd.a), load_slot(slots, stride, nd.b))); + break; + case OP_NEG: + store_slot(slots, stride, nd.res, ext3::neg(load_slot(slots, stride, nd.a))); + break; + default: + break; + } + } + return load_slot(slots, stride, root_slot); +} + +__device__ __forceinline__ Fe3 eval_program(const uint64_t *__restrict__ d_nodes, + uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, + const uint64_t *const *__restrict__ d_factors, + uint64_t j, uint64_t half, const Fe3 &t, + uint64_t *slots, uint64_t stride, uint32_t root_slot) { + return eval_program_t(d_nodes, num_nodes, d_consts, d_factors, j, half, t, slots, + stride, root_slot); +} + +// Capped at the block size the launcher uses and two blocks per SM: without a +// bound the compiler spends 67 registers a thread and occupancy stops at half. +template +__device__ __forceinline__ void sumcheck_round_body( + const uint64_t *const *__restrict__ d_factors, uint64_t half, + const uint64_t *__restrict__ d_nodes, uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, uint32_t root_slot, + const uint64_t *__restrict__ d_t, uint32_t num_t, uint64_t *__restrict__ d_slots, + uint64_t *__restrict__ d_partials); + +extern "C" __global__ __launch_bounds__(256, 2) void sumcheck_round_ext3( + // one device pointer per factor; factor `k` at cube index `j`, component + // `c`, is `d_factors[k][j*3 + c]` + const uint64_t *const *__restrict__ d_factors, + // cube indices this round: `lo` at `j`, `hi` at `j + half` + uint64_t half, + // the program + const uint64_t *__restrict__ d_nodes, uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, uint32_t root_slot, + // interpolation nodes, ext3 + const uint64_t *__restrict__ d_t, uint32_t num_t, + // per-thread slot file + uint64_t *__restrict__ d_slots, + // out: one partial per (node, block), ext3 + uint64_t *__restrict__ d_partials) { + sumcheck_round_body(d_factors, half, d_nodes, num_nodes, d_consts, root_slot, d_t, + num_t, d_slots, d_partials); +} + +// The same round with integer nodes (D-ARGUE S1-1): see `eval_program_t`. +extern "C" __global__ __launch_bounds__(256, 2) void sumcheck_round_ext3_int( + const uint64_t *const *__restrict__ d_factors, uint64_t half, + const uint64_t *__restrict__ d_nodes, uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, uint32_t root_slot, + const uint64_t *__restrict__ d_t, uint32_t num_t, uint64_t *__restrict__ d_slots, + uint64_t *__restrict__ d_partials) { + sumcheck_round_body(d_factors, half, d_nodes, num_nodes, d_consts, root_slot, d_t, + num_t, d_slots, d_partials); +} + +template +__device__ __forceinline__ void sumcheck_round_body( + const uint64_t *const *__restrict__ d_factors, uint64_t half, + const uint64_t *__restrict__ d_nodes, uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, uint32_t root_slot, + const uint64_t *__restrict__ d_t, uint32_t num_t, uint64_t *__restrict__ d_slots, + uint64_t *__restrict__ d_partials) { + // The slot file is per thread and a thread is a (cube index, node) pair, + // so both grid dimensions go into the address. + uint64_t tid = ((uint64_t)blockIdx.y * gridDim.x + blockIdx.x) * blockDim.x + threadIdx.x; + uint64_t num_threads = (uint64_t)gridDim.x * gridDim.y * blockDim.x; + uint64_t *slots = d_slots + tid; + + uint64_t index = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t index_stride = (uint64_t)gridDim.x * blockDim.x; + + // Each node belongs to exactly one `blockIdx.y`, so the partial it writes + // below has one writer whatever the grid's second dimension is. + Fe3 acc[MAX_NODES]; + for (uint32_t ti = blockIdx.y; ti < num_t; ti += gridDim.y) { + acc[ti] = ext3::make(0, 0, 0); + } + + for (uint64_t j = index; j < half; j += index_stride) { + for (uint32_t ti = blockIdx.y; ti < num_t; ti += gridDim.y) { + Fe3 t = load_ext(d_t + (uint64_t)ti * 3); + Fe3 v = eval_program_t(d_nodes, num_nodes, d_consts, d_factors, j, half, t, + slots, num_threads, root_slot); + acc[ti] = ext3::add(acc[ti], v); + } + } + + // One node at a time through the same shared buffer: the round's degree is + // a handful, and a buffer per node would bound the block size instead. + extern __shared__ uint64_t shared[]; + for (uint32_t ti = blockIdx.y; ti < num_t; ti += gridDim.y) { + shared[threadIdx.x * 3 + 0] = acc[ti].a; + shared[threadIdx.x * 3 + 1] = acc[ti].b; + shared[threadIdx.x * 3 + 2] = acc[ti].c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 x = load_ext(shared + threadIdx.x * 3); + Fe3 y = load_ext(shared + (threadIdx.x + width) * 3); + Fe3 sum = ext3::add(x, y); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t at = ((uint64_t)ti * gridDim.x + blockIdx.x) * 3; + d_partials[at + 0] = shared[0]; + d_partials[at + 1] = shared[1]; + d_partials[at + 2] = shared[2]; + } + __syncthreads(); + } +} + +// Sums a round's per-block partials, one block per interpolation node. +// +// The round leaves `num_t × blocks` extension values and the transcript wants +// `num_t`. Reducing them here instead of on the host is what keeps a round's +// answer at a few dozen bytes: the partials are megabytes for a wide launch, +// and every round of every sumcheck waits for that copy. +extern "C" __global__ void sum_partials_ext3(const uint64_t *__restrict__ d_partials, + uint64_t blocks, uint64_t *__restrict__ out) { + const uint64_t *base = d_partials + (uint64_t)blockIdx.x * blocks * 3; + Fe3 acc = ext3::make(0, 0, 0); + for (uint64_t i = threadIdx.x; i < blocks; i += blockDim.x) { + acc = ext3::add(acc, load_ext(base + i * 3)); + } + extern __shared__ uint64_t shared[]; + shared[threadIdx.x * 3 + 0] = acc.a; + shared[threadIdx.x * 3 + 1] = acc.b; + shared[threadIdx.x * 3 + 2] = acc.c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 x = load_ext(shared + threadIdx.x * 3); + Fe3 y = load_ext(shared + (threadIdx.x + width) * 3); + Fe3 sum = ext3::add(x, y); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t *at = out + (uint64_t)blockIdx.x * 3; + at[0] = shared[0]; + at[1] = shared[1]; + at[2] = shared[2]; + } +} + +// A round's sums without the second launch (`LAMBDA_VM_ARGUE_ROUND_SLOT`): every block of a `_slot` +// kernel calls this once its partials are written (`rows × gridDim.x` of them, row `r`'s at +// `(r·gridDim.x + b)·3`). The last block to finish — the ticket says which — sums each row's +// partials in `sum_partials_ext3`'s order exactly (256 lanes, lane `v` adding partials `v, v + 256, +// …` from zero, then the halving tree), so the sums are the limbs today's second launch writes; +// then it puts them in `d_slot[0 .. 3·rows)`, resets the ticket for the next launch, and writes +// `seq` to `d_slot[96]` last, behind a system fence: a host that reads `seq` there finds the sums +// before it. +// +// `d_slot` is pinned host memory mapped into the card (`crate::device::MappedSlot`). The shared +// buffer needs `max(blockDim.x, 256)·3 + 1` u64. +__device__ void slot_sums(const uint64_t *d_partials, uint32_t rows, unsigned int *d_ticket, + uint64_t *d_slot, uint64_t seq) { + extern __shared__ uint64_t shared[]; + const uint32_t lanes = 256; + uint64_t *last = shared + 3 * (blockDim.x > lanes ? blockDim.x : lanes); + __syncthreads(); + if (threadIdx.x == 0) { + // This block's partials, out before its ticket. + __threadfence(); + unsigned int ticket = atomicAdd(d_ticket, 1u); + *last = ticket == gridDim.x * gridDim.y - 1; + } + __syncthreads(); + if (!*last) return; + // Every other block's partials, seen after its ticket. + __threadfence(); + const uint64_t blocks = gridDim.x; + for (uint32_t row = 0; row < rows; row++) { + const volatile uint64_t *base = d_partials + (uint64_t)row * blocks * 3; + for (uint32_t v = threadIdx.x; v < lanes; v += blockDim.x) { + Fe3 acc = ext3::make(0, 0, 0); + for (uint64_t i = v; i < blocks; i += lanes) { + acc = ext3::add(acc, ext3::make(base[i * 3], base[i * 3 + 1], base[i * 3 + 2])); + } + shared[v * 3 + 0] = acc.a; + shared[v * 3 + 1] = acc.b; + shared[v * 3 + 2] = acc.c; + } + __syncthreads(); + for (uint32_t width = lanes / 2; width > 0; width >>= 1) { + for (uint32_t v = threadIdx.x; v < width; v += blockDim.x) { + Fe3 sum = ext3::add(load_ext(shared + v * 3), load_ext(shared + (v + width) * 3)); + shared[v * 3 + 0] = sum.a; + shared[v * 3 + 1] = sum.b; + shared[v * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + d_slot[row * 3 + 0] = shared[0]; + d_slot[row * 3 + 1] = shared[1]; + d_slot[row * 3 + 2] = shared[2]; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + *d_ticket = 0; + __threadfence_system(); + *(volatile uint64_t *)(d_slot + 96) = seq; + } +} + +// The program's value at every row, written out rather than summed: the LogUp +// input layer is one of these per interaction per side. +// +// Reuses the round's walk with `half = 0` and `t = 0`, which makes `OP_VAR` +// read `lo` and extend it by nothing — the plain value at the row. +extern "C" __global__ void program_map_ext3(const uint64_t *const *__restrict__ d_factors, + uint64_t num_rows, + const uint64_t *__restrict__ d_nodes, + uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, + uint32_t root_slot, uint64_t *__restrict__ d_slots, + uint64_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_threads = (uint64_t)gridDim.x * blockDim.x; + uint64_t *slots = d_slots + tid; + Fe3 zero = ext3::make(0, 0, 0); + + for (uint64_t row = tid; row < num_rows; row += num_threads) { + Fe3 v = eval_program(d_nodes, num_nodes, d_consts, d_factors, row, 0, zero, slots, + num_threads, root_slot); + uint64_t *at = out + row * 3; + at[0] = v.a; + at[1] = v.b; + at[2] = v.c; + } +} + +// Builds a table's factors out of its base columns: factor `slot` at row `j` is +// the column it reads, taken at its frame-step offset and lifted into the +// extension. +// +// `d_plan` is three u64 per committed factor — where its column starts inside +// `d_columns`, how far it is shifted (already reduced mod `rows`), and the slot +// it fills. The public factors are not here: they are in the extension already +// and are copied in as they are. +// +// The point of this kernel is that the columns are a third of what the factors +// are, so what crosses the bus is the trace and not its lift. +extern "C" __global__ void factors_from_columns_ext3(const uint64_t *__restrict__ d_columns, + const uint64_t *__restrict__ d_plan, + uint64_t num_plan, uint64_t rows, + uint64_t *__restrict__ out) { + uint64_t total = num_plan * rows; + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < total; + task += (uint64_t)gridDim.x * blockDim.x) { + uint64_t p = task / rows; + uint64_t j = task - p * rows; + uint64_t base = d_plan[p * 3 + 0]; + uint64_t shift = d_plan[p * 3 + 1]; + uint64_t slot = d_plan[p * 3 + 2]; + uint64_t k = j + shift; + if (k >= rows) k -= rows; + uint64_t *at = out + (slot * rows + j) * 3; + at[0] = d_columns[base + k]; + at[1] = 0; + at[2] = 0; + } +} + +// Fills a range with one ext3 value — the padding interactions of an input +// layer, whose numerators vanish and whose denominators are one. +extern "C" __global__ void fill_ext3(uint64_t *__restrict__ dst, uint64_t count, + const uint64_t *__restrict__ value) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= count) return; + uint64_t *at = dst + j * 3; + at[0] = value[0]; + at[1] = value[1]; + at[2] = value[2]; +} + +// Lifts a base-field table into the extension: `out[j] = {in[j], 0, 0}`. +extern "C" __global__ void mle_lift_base_ext3(const uint64_t *__restrict__ in, uint64_t count, + uint64_t *__restrict__ out) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= count) return; + uint64_t *at = out + j * 3; + at[0] = in[j]; + at[1] = 0; + at[2] = 0; +} + +// `dst[j] += scale · src[j]`, the shape a weight takes when a round adds the +// next claim to it. +extern "C" __global__ void add_scaled_ext3(uint64_t *__restrict__ dst, + const uint64_t *__restrict__ src, uint64_t count, + const uint64_t *__restrict__ scale) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= count) return; + Fe3 term = ext3::mul(load_ext(src + j * 3), load_ext(scale)); + Fe3 sum = ext3::add(load_ext(dst + j * 3), term); + uint64_t *at = dst + j * 3; + at[0] = sum.a; + at[1] = sum.b; + at[2] = sum.c; +} + +// One level of the eq table's doubling: `dst[j + half] = dst[j]·r` and +// `dst[j] = dst[j]·(1 − r)`, the halves disjoint so one thread owns both. The +// host seeds `dst[0]` and walks the variables back to front, which is what +// puts variable 0 in the high bit. +extern "C" __global__ void eq_expand_level_ext3(uint64_t *__restrict__ dst, uint64_t half, + const uint64_t *__restrict__ r) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= half) return; + Fe3 value = load_ext(dst + j * 3); + Fe3 challenge = load_ext(r); + Fe3 hi = ext3::mul(value, challenge); + Fe3 lo = ext3::sub(value, hi); + uint64_t *at = dst + j * 3; + at[0] = lo.a; + at[1] = lo.b; + at[2] = lo.c; + uint64_t *up = dst + (j + half) * 3; + up[0] = hi.a; + up[1] = hi.b; + up[2] = hi.c; +} + +// A stacked polynomial's weight is one `eq` table per column, each in its own +// subcube. Building them one at a time is a launch per level per column — +// tens of thousands of launches for a hundred milliseconds of work — so these +// two do every column at once. +// +// The shares arrive sorted by variable count, descending, so the ones still +// doubling at level `l` are exactly the first `active` of them. Three u64 per +// share: where its subcube starts, how many variables it has, and where its +// point starts in `d_points`. + +// Seeds every share's first cell with its scale. The table is a product over +// the variables, so one more factor at the start scales every cell. +extern "C" __global__ void eq_seed_shares_ext3(uint64_t *__restrict__ dst, + const uint64_t *__restrict__ d_shares, + const uint64_t *__restrict__ d_scales, + uint64_t count) { + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= count) return; + uint64_t *at = dst + d_shares[i * 3] * 3; + at[0] = d_scales[i * 3 + 0]; + at[1] = d_scales[i * 3 + 1]; + at[2] = d_scales[i * 3 + 2]; +} + +// One level of the doubling, for every share still doubling at it. The body is +// `eq_expand_level_ext3`'s, with the share's own coordinate and subcube. +extern "C" __global__ void eq_expand_level_shares_ext3(uint64_t *__restrict__ dst, + const uint64_t *__restrict__ d_shares, + const uint64_t *__restrict__ d_points, + uint64_t active, uint64_t level) { + uint64_t filled = (uint64_t)1 << level; + uint64_t total = active * filled; + for (uint64_t t = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; t < total; + t += (uint64_t)gridDim.x * blockDim.x) { + uint64_t s = t >> level; + uint64_t j = t & (filled - 1); + uint64_t offset = d_shares[s * 3 + 0]; + uint64_t vars = d_shares[s * 3 + 1]; + uint64_t point_at = d_shares[s * 3 + 2]; + + uint64_t *cube = dst + offset * 3; + // Variables go back to front, which is what leaves variable 0 in the + // high bit. + Fe3 challenge = load_ext(d_points + (point_at + vars - 1 - level) * 3); + Fe3 value = load_ext(cube + j * 3); + Fe3 hi = ext3::mul(value, challenge); + Fe3 lo = ext3::sub(value, hi); + uint64_t *at = cube + j * 3; + at[0] = lo.a; + at[1] = lo.b; + at[2] = lo.c; + uint64_t *up = cube + (j + filled) * 3; + up[0] = hi.a; + up[1] = hi.b; + up[2] = hi.c; + } +} + +// One level of the fraction tree: `p' = p_lo·q_hi + p_hi·q_lo`, `q' = q_lo·q_hi` +// over the halves of the layer below. Out of place — the halves are read by +// threads that write the level above. +extern "C" __global__ void fraction_fold_ext3(const uint64_t *__restrict__ p, + const uint64_t *__restrict__ q, uint64_t half, + uint64_t *__restrict__ p_out, + uint64_t *__restrict__ q_out) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= half) return; + Fe3 p_lo = load_ext(p + j * 3); + Fe3 p_hi = load_ext(p + (j + half) * 3); + Fe3 q_lo = load_ext(q + j * 3); + Fe3 q_hi = load_ext(q + (j + half) * 3); + Fe3 numerator = ext3::add(ext3::mul(p_lo, q_hi), ext3::mul(p_hi, q_lo)); + Fe3 denominator = ext3::mul(q_lo, q_hi); + uint64_t *at = p_out + j * 3; + at[0] = numerator.a; + at[1] = numerator.b; + at[2] = numerator.c; + uint64_t *down = q_out + j * 3; + down[0] = denominator.a; + down[1] = denominator.b; + down[2] = denominator.c; +} + +// The same, for a layer whose upper half runs out: the interactions are padded +// up to a power of two with the fraction 0/1, and a fraction nobody wrote is +// one nobody has to store. The lower half is always there — the padding is +// less than half the cube, because the count is rounded *up* to the power of +// two above it. +extern "C" __global__ void fraction_fold_padded_ext3(const uint64_t *__restrict__ p, + const uint64_t *__restrict__ q, uint64_t half, + uint64_t real, + uint64_t *__restrict__ p_out, + uint64_t *__restrict__ q_out) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= half) return; + Fe3 p_lo = load_ext(p + j * 3); + Fe3 q_lo = load_ext(q + j * 3); + uint64_t hi = j + half; + Fe3 p_hi = {0, 0, 0}; + Fe3 q_hi = {1, 0, 0}; + if (hi < real) { + p_hi = load_ext(p + hi * 3); + q_hi = load_ext(q + hi * 3); + } + Fe3 numerator = ext3::add(ext3::mul(p_lo, q_hi), ext3::mul(p_hi, q_lo)); + Fe3 denominator = ext3::mul(q_lo, q_hi); + uint64_t *at = p_out + j * 3; + at[0] = numerator.a; + at[1] = numerator.b; + at[2] = numerator.c; + uint64_t *down = q_out + j * 3; + down[0] = denominator.a; + down[1] = denominator.b; + down[2] = denominator.c; +} + +// The first fold of a base-field table, which lifts it: +// out[j] = in[j] + r·(in[j + half] − in[j]) +// with `in` base and `out` ext3. Later folds stay in the extension and go +// through `sumcheck_fold_ext3` with a single factor. +extern "C" __global__ void mle_fold_base_ext3(const uint64_t *__restrict__ in, uint64_t half, + const uint64_t *__restrict__ r, + uint64_t *__restrict__ out) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= half) return; + uint64_t lo = in[j]; + uint64_t delta = goldilocks::sub(in[j + half], lo); + // `delta·r` with `delta` in the base field, then `lo +` it: the mixed-field + // shortcuts, bit-identical to the full ext3 ops on the embedding. + Fe3 scaled = ext3::mul_base(ext3::make(r[0], r[1], r[2]), delta); + uint64_t *at = out + j * 3; + at[0] = goldilocks::add(lo, scaled.a); + at[1] = scaled.b; + at[2] = scaled.c; +} + +// The same for many base tables at once, laid out end to end: one thread per +// (table, index) pair, and the output is one ext3 half per table in the same +// order. Everything above this level is ext3, so the folds that follow are the +// ordinary ones over a list of factors. +extern "C" __global__ void mle_fold_base_ext3_many(const uint64_t *__restrict__ in, uint64_t half, + uint64_t num_tables, + const uint64_t *__restrict__ r, + uint64_t *__restrict__ out) { + uint64_t total = half * num_tables; + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < total; + task += (uint64_t)gridDim.x * blockDim.x) { + uint64_t table = task / half; + uint64_t j = task - table * half; + const uint64_t *src = in + table * half * 2; + uint64_t lo = src[j]; + uint64_t delta = goldilocks::sub(src[j + half], lo); + Fe3 scaled = ext3::mul_base(ext3::make(r[0], r[1], r[2]), delta); + uint64_t *at = out + task * 3; + at[0] = goldilocks::add(lo, scaled.a); + at[1] = scaled.b; + at[2] = scaled.c; + } +} + +// Many base columns' values at ONE point, each column read once: +// +// value_k = Σ_row column_k[row] · eq(p, row), +// eq(p, row) = hi[row >> lo_bits] · lo[row & (2^lo_bits − 1)] +// +// with `hi` and `lo` the point's `eq` table in two halves (its first and its +// last `lo_bits` coordinates, the first variable the high bit), a few KiB each +// instead of the table. The columns are base-field, laid end to end, `rows` +// each. Block `(b, k)` takes column `k` and every `gridDim.x`-th hi row from +// `b`: a thread sums its lo slots of one hi row against `lo` — base × ext3, +// three products a value — and scales that by the row's `hi` once. One +// partial per (column, block), in `sum_partials_ext3`'s layout with the +// columns as its rows. A commit-time out-of-domain answer's column values +// (`multilinear::stacked_eval::ood_claims`): a pass over the columns at the +// bandwidth of their base words, where a fold writes and reads back an ext3 +// half at every level. +extern "C" __global__ void eq_dot_base_ext3_many(const uint64_t *__restrict__ columns, + uint64_t rows, uint32_t lo_bits, + const uint64_t *__restrict__ hi, + const uint64_t *__restrict__ lo, + uint64_t *__restrict__ d_partials) { + const uint64_t *column = columns + (uint64_t)blockIdx.y * rows; + uint64_t lo_len = (uint64_t)1 << lo_bits; + uint64_t his = rows >> lo_bits; + Fe3 acc = ext3::zero(); + for (uint64_t h = blockIdx.x; h < his; h += gridDim.x) { + const uint64_t *row = column + (h << lo_bits); + Fe3 s = ext3::zero(); + for (uint64_t l = threadIdx.x; l < lo_len; l += blockDim.x) { + s = ext3::add(s, ext3::mul_base(load_ext(lo + l * 3), row[l])); + } + acc = ext3::add(acc, ext3::mul(s, load_ext(hi + h * 3))); + } + extern __shared__ uint64_t shared[]; + shared[threadIdx.x * 3 + 0] = acc.a; + shared[threadIdx.x * 3 + 1] = acc.b; + shared[threadIdx.x * 3 + 2] = acc.c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 sum = ext3::add(load_ext(shared + threadIdx.x * 3), + load_ext(shared + (threadIdx.x + width) * 3)); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t *at = d_partials + ((uint64_t)blockIdx.y * gridDim.x + blockIdx.x) * 3; + at[0] = shared[0]; + at[1] = shared[1]; + at[2] = shared[2]; + } +} + +// The claim reduce's batched column for one offset, out of the epoch's resident +// columns: `out[row] = Σ_m weight[m]·column[member[m]][row]`. The columns are +// base-field and laid end to end, `rows` each, from the start of the table's run; +// `members` is each term's column in that run and `weights` its ext3 weight. +// +// One thread per row, looping over the members: the threads of a warp read one +// column's consecutive rows, and every thread reads the same weight. The sum is +// the host's (`claim_reduce::batched_column`) term for term — base × ext3 is the +// componentwise product either way — in a different order, which a field does +// not see. +extern "C" __global__ void batched_column_ext3(const uint64_t *__restrict__ columns, + uint64_t rows, + const uint64_t *__restrict__ members, + const uint64_t *__restrict__ weights, + uint64_t num_members, + uint64_t *__restrict__ out) { + for (uint64_t row = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; row < rows; + row += (uint64_t)gridDim.x * blockDim.x) { + Fe3 acc = ext3::zero(); + for (uint64_t m = 0; m < num_members; ++m) { + uint64_t value = columns[members[m] * rows + row]; + acc = ext3::add(acc, ext3::mul_base(load_ext(weights + m * 3), value)); + } + uint64_t *at = out + row * 3; + at[0] = acc.a; + at[1] = acc.b; + at[2] = acc.c; + } +} + +// A session's end, read back in one copy: factor `k`'s first `cells` ext3 +// values — what the folds left of it, at its own base — gathered to +// `out[k·cells·3 ..]`. One thread per u64, so a warp reads one factor's +// consecutive words and writes consecutive words. +extern "C" __global__ void gather_factor_heads_ext3(uint64_t *const *__restrict__ d_factors, + uint64_t width, uint64_t cells, + uint64_t *__restrict__ out) { + uint64_t span = cells * 3; + uint64_t total = width * span; + for (uint64_t word = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; word < total; + word += (uint64_t)gridDim.x * blockDim.x) { + uint64_t k = word / span; + out[word] = d_factors[k][word - k * span]; + } +} + +// Binds the round's variable: `f(j) <- f(j) + r·(f(j + half) − f(j))` for every +// factor, halving the cube. One thread per (factor, index) pair. +extern "C" __global__ void sumcheck_fold_ext3(uint64_t *const *__restrict__ d_factors, + uint64_t half, uint64_t width, + const uint64_t *__restrict__ d_r) { + uint64_t total = width * half; + Fe3 r = load_ext(d_r); + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < total; + task += (uint64_t)gridDim.x * blockDim.x) { + uint64_t k = task / half; + uint64_t j = task - k * half; + uint64_t *column = d_factors[k]; + Fe3 lo = load_ext(column + j * 3); + Fe3 hi = load_ext(column + (j + half) * 3); + Fe3 v = ext3::add(lo, ext3::mul(r, ext3::sub(hi, lo))); + column[j * 3 + 0] = v.a; + column[j * 3 + 1] = v.b; + column[j * 3 + 2] = v.c; + } +} + +// ── D-ARGUE stage 1: the fused zerocheck (S1-2, S1-4, S1-5) ────────────────── +// +// A table's zerocheck batch is `eq(r,x)·C(x) + eq(ρ,x)·L(x)`, `C = Σ β_i·sel_i·root_i` +// the constraint part and `L` the bus's two rules as one affine column +// (`multilinear::fused`, the host reference these kernels are checked against). +// Rounds 0 and 1 come from one base-field pass over the 4-row groups +// `(a, b, x'')` — rows `a·2q + b·q + x''`, `q` a quarter of the cube — on the +// grid `{0..d}²` (`zc_grid01`, `zc_bus_u`); the factors are then folded by both +// challenges at once into a quarter-size ext3 buffer (`zc_fold2`), `L` is built +// there (`zc_bus_column`), and the later rounds walk `C` alone at the integer +// nodes `{0, 2..d}` under split `eq` weights (`zc_round_gruen`). +// +// The constraint program is the table's base DAG with one extra op, `ACC`: +// `acc += β[res]·(value[a]·factor[b])` (`b == NO_SELECTOR`: no factor), placed +// right after each root — so the roots are summed as they are made and never +// held to the end. Op tags and the packing MUST stay in sync with +// `crypto/multilinear/src/gpu_fused.rs`. + +#define OP_ACC 6u +#define NO_SELECTOR 0xFFFFFFFFu + +// A factor's value on the grid point `(a, b)` of group `x`, from its four rows +// — base limbs of the lifted ext3 factor (`stride` 3), whose upper limbs are +// zero before the first fold (the base-field precondition), or the base column +// itself (`stride` 1, D-BATCH M1-2: no lift). Bilinear, exact. +__device__ __forceinline__ uint64_t grid_value(const uint64_t *__restrict__ column, uint64_t x, + uint64_t q, uint64_t a, uint64_t b, + uint64_t stride) { + uint64_t f00 = column[x * stride]; + uint64_t f01 = column[(x + q) * stride]; + uint64_t f10 = column[(x + 2 * q) * stride]; + uint64_t f11 = column[(x + 3 * q) * stride]; + uint64_t da = goldilocks::sub(f10, f00); + uint64_t db = goldilocks::sub(f01, f00); + uint64_t dab = goldilocks::sub(goldilocks::sub(f11, f10), db); + uint64_t v = f00; + if (a) v = goldilocks::add(v, goldilocks::mul(da, a)); + if (b) v = goldilocks::add(v, goldilocks::mul(db, b)); + if (a && b) v = goldilocks::add(v, goldilocks::mul(dab, goldilocks::mul(a, b))); + return v; +} + +// One thread per (group, grid point): `gridDim.y` is the point count and each +// block owns one point. `T(a, b) = Σ_x eq_r[x]·C(a, b, x)` summed per block +// into `d_partials[(point·gridDim.x + block)·3]`. A corner (`a, b < 2`) is on +// the list only when the corners are checked or kept. Checked +// (`keep_corners == 0`): its `C` must be zero on every row, the first row that +// is not is kept in `d_violation` (atomicMin), and it adds nothing to `T`. +// Kept (`keep_corners == 1`, random columns in a test): it adds to `T` like +// any point. +extern "C" __global__ __launch_bounds__(256) void zc_grid01( + const uint64_t *const *__restrict__ d_factors, uint64_t q, + const uint64_t *__restrict__ d_nodes, uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, const uint64_t *__restrict__ d_betas, + const uint64_t *__restrict__ d_points, const uint64_t *__restrict__ d_eq_r, + uint64_t *__restrict__ d_slots, uint64_t *__restrict__ d_partials, + unsigned long long *__restrict__ d_violation, uint32_t keep_corners, uint64_t stride) { + uint64_t tid = ((uint64_t)blockIdx.y * gridDim.x + blockIdx.x) * blockDim.x + threadIdx.x; + uint64_t num_threads = (uint64_t)gridDim.x * gridDim.y * blockDim.x; + uint64_t *slots = d_slots + tid; + uint64_t point = d_points[blockIdx.y]; + uint64_t a = point & 0xFFFFFFFFull; + uint64_t b = point >> 32; + bool corner = a < 2 && b < 2; + + Fe3 t_acc = ext3::zero(); + for (uint64_t x = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; x < q; + x += (uint64_t)gridDim.x * blockDim.x) { + Fe3 c = ext3::zero(); + for (uint64_t i = 0; i < num_nodes; i++) { + Node nd = load_node(d_nodes, i); + uint64_t *out = slots + (uint64_t)nd.res * num_threads; + switch (nd.op) { + case OP_VAR: + *out = grid_value(d_factors[nd.a], x, q, a, b, stride); + break; + case OP_FIXED: + *out = d_consts[nd.a]; + break; + case OP_ADD: + *out = goldilocks::add(slots[(uint64_t)nd.a * num_threads], + slots[(uint64_t)nd.b * num_threads]); + break; + case OP_SUB: + *out = goldilocks::sub(slots[(uint64_t)nd.a * num_threads], + slots[(uint64_t)nd.b * num_threads]); + break; + case OP_MUL: + *out = goldilocks::mul(slots[(uint64_t)nd.a * num_threads], + slots[(uint64_t)nd.b * num_threads]); + break; + case OP_NEG: + *out = goldilocks::neg(slots[(uint64_t)nd.a * num_threads]); + break; + case OP_ACC: { + uint64_t v = slots[(uint64_t)nd.a * num_threads]; + if (nd.b != NO_SELECTOR) { + v = goldilocks::mul(v, grid_value(d_factors[nd.b], x, q, a, b, stride)); + } + c = ext3::add(c, ext3::mul_base(load_ext(d_betas + (uint64_t)nd.res * 3), v)); + break; + } + default: + break; + } + } + if (corner && !keep_corners) { + Fe3 k = ext3::canonical(c); + if (k.a | k.b | k.c) { + atomicMin(d_violation, (unsigned long long)(a * 2 * q + b * q + x)); + } + continue; + } + t_acc = ext3::add(t_acc, ext3::mul(load_ext(d_eq_r + x * 3), c)); + } + + extern __shared__ uint64_t shared[]; + shared[threadIdx.x * 3 + 0] = t_acc.a; + shared[threadIdx.x * 3 + 1] = t_acc.b; + shared[threadIdx.x * 3 + 2] = t_acc.c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 sum = ext3::add(load_ext(shared + threadIdx.x * 3), + load_ext(shared + (threadIdx.x + width) * 3)); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t at = ((uint64_t)blockIdx.y * gridDim.x + blockIdx.x) * 3; + d_partials[at + 0] = shared[0]; + d_partials[at + 1] = shared[1]; + d_partials[at + 2] = shared[2]; + } +} + +// `U(b, c) = Σ_x eq_rho[x]·L(b, c, x)` for `(b, c) ∈ {0,1}²`, `L = a₀ + Σ a_k·f_k` +// from each term's four base rows. One thread per group; the block's four sums +// go to `d_partials[(bc·gridDim.x + block)·3]`, `bc = 2b + c`. +extern "C" __global__ __launch_bounds__(256) void zc_bus_u( + const uint64_t *const *__restrict__ d_factors, uint64_t q, + const uint32_t *__restrict__ d_term_slots, const uint64_t *__restrict__ d_term_coeffs, + uint64_t num_terms, const uint64_t *__restrict__ d_constant, + const uint64_t *__restrict__ d_eq_rho, uint64_t *__restrict__ d_partials, uint64_t stride) { + Fe3 u[4] = {ext3::zero(), ext3::zero(), ext3::zero(), ext3::zero()}; + Fe3 a0 = load_ext(d_constant); + for (uint64_t x = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; x < q; + x += (uint64_t)gridDim.x * blockDim.x) { + Fe3 l[4] = {a0, a0, a0, a0}; + for (uint64_t k = 0; k < num_terms; k++) { + const uint64_t *column = d_factors[d_term_slots[k]]; + Fe3 coeff = load_ext(d_term_coeffs + k * 3); + for (uint32_t bc = 0; bc < 4; bc++) { + l[bc] = ext3::add(l[bc], ext3::mul_base(coeff, column[(x + bc * q) * stride])); + } + } + Fe3 w = load_ext(d_eq_rho + x * 3); + for (uint32_t bc = 0; bc < 4; bc++) { + u[bc] = ext3::add(u[bc], ext3::mul(w, l[bc])); + } + } + extern __shared__ uint64_t shared[]; + for (uint32_t bc = 0; bc < 4; bc++) { + shared[threadIdx.x * 3 + 0] = u[bc].a; + shared[threadIdx.x * 3 + 1] = u[bc].b; + shared[threadIdx.x * 3 + 2] = u[bc].c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 sum = ext3::add(load_ext(shared + threadIdx.x * 3), + load_ext(shared + (threadIdx.x + width) * 3)); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t at = ((uint64_t)bc * gridDim.x + blockIdx.x) * 3; + d_partials[at + 0] = shared[0]; + d_partials[at + 1] = shared[1]; + d_partials[at + 2] = shared[2]; + } + __syncthreads(); + } +} + +// Both folds at once, base to ext3: `out_k[x] = Σ_{a,b} w[2a+b]·f_k(a, b, x)` over +// the four rows of each group, into factor `k`'s quarter-size slab of `out`. +// One thread per (factor, group). +extern "C" __global__ void zc_fold2(const uint64_t *const *__restrict__ d_factors, uint64_t q, + uint64_t width, const uint64_t *__restrict__ d_w, + uint64_t *__restrict__ out, uint64_t stride) { + Fe3 w0 = load_ext(d_w), w1 = load_ext(d_w + 3), w2 = load_ext(d_w + 6), + w3 = load_ext(d_w + 9); + uint64_t total = width * q; + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < total; + task += (uint64_t)gridDim.x * blockDim.x) { + uint64_t k = task / q; + uint64_t x = task - k * q; + const uint64_t *column = d_factors[k]; + Fe3 v = ext3::mul_base(w0, column[x * stride]); + v = ext3::add(v, ext3::mul_base(w1, column[(x + q) * stride])); + v = ext3::add(v, ext3::mul_base(w2, column[(x + 2 * q) * stride])); + v = ext3::add(v, ext3::mul_base(w3, column[(x + 3 * q) * stride])); + uint64_t *at = out + task * 3; + at[0] = v.a; + at[1] = v.b; + at[2] = v.c; + } +} + +// `L[x] = a₀ + Σ a_k·F_k[x]` over the folded (ext3) factors. One thread per row. +extern "C" __global__ void zc_bus_column(const uint64_t *const *__restrict__ d_factors, + uint64_t rows, const uint32_t *__restrict__ d_term_slots, + const uint64_t *__restrict__ d_term_coeffs, + uint64_t num_terms, + const uint64_t *__restrict__ d_constant, + uint64_t *__restrict__ out) { + Fe3 a0 = load_ext(d_constant); + for (uint64_t x = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; x < rows; + x += (uint64_t)gridDim.x * blockDim.x) { + Fe3 acc = a0; + for (uint64_t k = 0; k < num_terms; k++) { + acc = ext3::add(acc, ext3::mul(load_ext(d_term_coeffs + k * 3), + load_ext(d_factors[d_term_slots[k]] + x * 3))); + } + uint64_t *at = out + x * 3; + at[0] = acc.a; + at[1] = acc.b; + at[2] = acc.c; + } +} + +// `eq(a_{>j+1}, ·)` from `eq(a_{>j}, ·)` in place: `t[x] += t[x + half]`. +extern "C" __global__ void zc_halve(uint64_t *__restrict__ table, uint64_t half) { + for (uint64_t x = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; x < half; + x += (uint64_t)gridDim.x * blockDim.x) { + Fe3 v = ext3::add(load_ext(table + x * 3), load_ext(table + (x + half) * 3)); + table[x * 3 + 0] = v.a; + table[x * 3 + 1] = v.b; + table[x * 3 + 2] = v.c; + } +} + +// A later round (S1-4 with S1-1): `gridDim.y` is `num_a + 2` and each block owns +// one of them. Block row `y < num_a` sums `eq_r[j]·C(j, node[y])`, every factor +// read at the integer node `k = node[y]` as `lo + k·(hi − lo)` (`lo` itself at +// `k = 0`); row `num_a + b` sums `eq_rho[j]·L[j + b·half]`. The per-block sums go +// to `d_partials[(y·gridDim.x + block)·3]`. +extern "C" __global__ __launch_bounds__(256, 2) void zc_round_gruen( + const uint64_t *const *__restrict__ d_factors, uint64_t half, + const uint64_t *__restrict__ d_nodes, uint64_t num_nodes, + const uint64_t *__restrict__ d_consts, const uint64_t *__restrict__ d_betas, + const uint32_t *__restrict__ d_node_ints, uint32_t num_a, + const uint64_t *__restrict__ d_eq_r, const uint64_t *__restrict__ d_eq_rho, + const uint64_t *__restrict__ d_l, uint64_t *__restrict__ d_slots, + uint64_t *__restrict__ d_partials) { + uint64_t tid = ((uint64_t)blockIdx.y * gridDim.x + blockIdx.x) * blockDim.x + threadIdx.x; + uint64_t num_threads = (uint64_t)gridDim.x * gridDim.y * blockDim.x; + uint64_t *slots = d_slots + tid; + uint64_t start = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t step = (uint64_t)gridDim.x * blockDim.x; + + Fe3 acc = ext3::zero(); + if (blockIdx.y < num_a) { + uint64_t k = d_node_ints[blockIdx.y]; + for (uint64_t j = start; j < half; j += step) { + Fe3 c = ext3::zero(); + for (uint64_t i = 0; i < num_nodes; i++) { + Node nd = load_node(d_nodes, i); + switch (nd.op) { + case OP_VAR: { + const uint64_t *column = d_factors[nd.a]; + Fe3 lo = load_ext(column + j * 3); + Fe3 v = lo; + if (k) { + Fe3 hi = load_ext(column + (j + half) * 3); + v = ext3::add(lo, ext3::mul_base(ext3::sub(hi, lo), k)); + } + store_slot(slots, num_threads, nd.res, v); + break; + } + case OP_FIXED: + store_slot(slots, num_threads, nd.res, load_ext(d_consts + (uint64_t)nd.a * 3)); + break; + case OP_ADD: + store_slot(slots, num_threads, nd.res, + ext3::add(load_slot(slots, num_threads, nd.a), + load_slot(slots, num_threads, nd.b))); + break; + case OP_SUB: + store_slot(slots, num_threads, nd.res, + ext3::sub(load_slot(slots, num_threads, nd.a), + load_slot(slots, num_threads, nd.b))); + break; + case OP_MUL: + store_slot(slots, num_threads, nd.res, + ext3::mul(load_slot(slots, num_threads, nd.a), + load_slot(slots, num_threads, nd.b))); + break; + case OP_NEG: + store_slot(slots, num_threads, nd.res, + ext3::neg(load_slot(slots, num_threads, nd.a))); + break; + case OP_ACC: { + Fe3 v = load_slot(slots, num_threads, nd.a); + if (nd.b != NO_SELECTOR) { + const uint64_t *column = d_factors[nd.b]; + Fe3 lo = load_ext(column + j * 3); + Fe3 s = lo; + if (k) { + Fe3 hi = load_ext(column + (j + half) * 3); + s = ext3::add(lo, ext3::mul_base(ext3::sub(hi, lo), k)); + } + v = ext3::mul(v, s); + } + c = ext3::add(c, ext3::mul(load_ext(d_betas + (uint64_t)nd.res * 3), v)); + break; + } + default: + break; + } + } + acc = ext3::add(acc, ext3::mul(load_ext(d_eq_r + j * 3), c)); + } + } else { + uint64_t b = blockIdx.y - num_a; + for (uint64_t j = start; j < half; j += step) { + acc = ext3::add(acc, ext3::mul(load_ext(d_eq_rho + j * 3), + load_ext(d_l + (j + b * half) * 3))); + } + } + + extern __shared__ uint64_t shared[]; + shared[threadIdx.x * 3 + 0] = acc.a; + shared[threadIdx.x * 3 + 1] = acc.b; + shared[threadIdx.x * 3 + 2] = acc.c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 sum = ext3::add(load_ext(shared + threadIdx.x * 3), + load_ext(shared + (threadIdx.x + width) * 3)); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t at = ((uint64_t)blockIdx.y * gridDim.x + blockIdx.x) * 3; + d_partials[at + 0] = shared[0]; + d_partials[at + 1] = shared[1]; + d_partials[at + 2] = shared[2]; + } +} + +// ── a GKR layer's rounds with Gruen's split (D-ARGUE S1-3, `crate::gkr::GruenLayer`) ───────────── +// +// The layer relation is `eq(u, x)·h(x)`, `h = p_lo·q_hi + p_hi·q_lo + λ·q_lo·q_hi`, over `m` +// variables. Round `j` needs only `H(t) = Σ_{x'} eq(u_{>j}, x')·h(s_{= ((uint64_t)1 << (rounds - 1 - j))) { + r -= (uint64_t)1 << (rounds - 1 - j); + j++; + } + first = j + 1; + bits = rounds - 1 - j; + index = r; + } + Fe3 acc = ext3::one(); + for (uint32_t i = 0; i < bits; i++) { + Fe3 u = load_ext(point + (uint64_t)(first + i) * 3); + bool set = (index >> (bits - 1 - i)) & 1; + acc = ext3::mul(acc, set ? u : ext3::sub(ext3::one(), u)); + } + uint64_t *at = out + t * 3; + at[0] = acc.a; + at[1] = acc.b; + at[2] = acc.c; + } +} + +// `h` at one node, from the four halves' values there. +__device__ __forceinline__ Fe3 gkr_h(const Fe3 &a, const Fe3 &b, const Fe3 &c, const Fe3 &d, + const Fe3 &lambda) { + Fe3 cd = ext3::mul(c, d); + return ext3::add(ext3::add(ext3::mul(a, d), ext3::mul(b, c)), ext3::mul(lambda, cd)); +} + +// An input layer kept without its padding (`crate::gkr::DeviceFractionTree::from_truncated_input`): +// the upper halves `p_hi` and `q_hi` (factors 1 and 3) hold only their first `stored` cells, and +// every cell from there on is the padding's fraction — `0` in `p`, `1` in `q`. The padding is the +// tail of the layer (the interaction slot is the high bits of a cell's index, and fewer than half +// the slots are padding), so it lies in the upper halves only, as one run to their end. A fold of +// two padding cells gives the padding back, so after a fold the stored run is the shorter of the +// one before and the new half. `REAL` is false for a layer stored whole, and then every cell is +// read and written as before. +// +// The constants are the limbs the padded layer is written with (`ext3::zero()`, `ext3::one()`, +// `gkr_input_from_columns`) and a fold of two of them writes the same limbs back, so a round over +// the stored cells reads raw-identical values to one over the whole layer. +template +__device__ __forceinline__ bool gkr_stored(int k, uint64_t i, uint64_t stored) { + return !REAL || !(k & 1) || i < stored; +} + +template +__device__ __forceinline__ Fe3 gkr_cell(const uint64_t *f, int k, uint64_t i, uint64_t stored) { + if (gkr_stored(k, i, stored)) return load_ext(f + i * 3); + return k < 2 ? ext3::zero() : ext3::one(); +} + +// One round over `quarter` cube indices. The four halves are `p`, `p + half`, `q`, `q + half` +// (`half` ext3 cells each). With `fold`, they still hold the previous round's cube, twice as long, +// and each is bound to `s` on the way in — `f(x) + s·(f(x + 2Q) − f(x))` at `x` and `x + Q` — and +// written back where the next round reads it: thread `x` owns cells `x, x+Q, x+2Q, x+3Q` and no +// one else touches them. Then `h` at `t = 1` (`hi`), `t = 2` (`2·hi − lo`) and, with `want_h0`, +// `t = 0` (`lo`), each weighted by `e_hi[x >> low]·e_lo[x mod 2^low]`. Block partials go to +// `d_partials[(row·gridDim.x + block)·3]`, rows `H(1), H(2), H(0)`. +// +// `gkr_round_gruen_real` is the same round over a layer kept without its padding (see `gkr_cell`): +// `stored` is the run of the upper halves' cells held before this round's fold. A cell past it is +// read as the padding and a folded cell past it is not written — it is the padding again. +template +__device__ __forceinline__ void gkr_round_gruen_body( + uint64_t *__restrict__ p, uint64_t *__restrict__ q, uint64_t half, uint64_t quarter, + uint32_t fold, uint64_t s0, uint64_t s1, uint64_t s2, uint64_t l0, uint64_t l1, uint64_t l2, + const uint64_t *__restrict__ e_hi, const uint64_t *__restrict__ e_lo, uint32_t low, + uint32_t want_h0, uint64_t stored, uint64_t *__restrict__ d_partials, unsigned int *d_ticket, + uint64_t *d_slot, uint64_t seq); + +extern "C" __global__ __launch_bounds__(256) void gkr_round_gruen( + uint64_t *__restrict__ p, uint64_t *__restrict__ q, uint64_t half, uint64_t quarter, + uint32_t fold, uint64_t s0, uint64_t s1, uint64_t s2, uint64_t l0, uint64_t l1, uint64_t l2, + const uint64_t *__restrict__ e_hi, const uint64_t *__restrict__ e_lo, uint32_t low, + uint32_t want_h0, uint64_t *__restrict__ d_partials) { + gkr_round_gruen_body(p, q, half, quarter, fold, s0, s1, s2, l0, l1, l2, e_hi, + e_lo, low, want_h0, 0, d_partials, nullptr, nullptr, 0); +} + +extern "C" __global__ __launch_bounds__(256) void gkr_round_gruen_real( + uint64_t *__restrict__ p, uint64_t *__restrict__ q, uint64_t half, uint64_t quarter, + uint32_t fold, uint64_t s0, uint64_t s1, uint64_t s2, uint64_t l0, uint64_t l1, uint64_t l2, + const uint64_t *__restrict__ e_hi, const uint64_t *__restrict__ e_lo, uint32_t low, + uint32_t want_h0, uint64_t stored, uint64_t *__restrict__ d_partials) { + gkr_round_gruen_body(p, q, half, quarter, fold, s0, s1, s2, l0, l1, l2, e_hi, + e_lo, low, want_h0, stored, d_partials, nullptr, nullptr, 0); +} + +// The two rounds again with their sums through a slot (`slot_sums`): no second launch. +extern "C" __global__ __launch_bounds__(256) void gkr_round_gruen_slot( + uint64_t *__restrict__ p, uint64_t *__restrict__ q, uint64_t half, uint64_t quarter, + uint32_t fold, uint64_t s0, uint64_t s1, uint64_t s2, uint64_t l0, uint64_t l1, uint64_t l2, + const uint64_t *__restrict__ e_hi, const uint64_t *__restrict__ e_lo, uint32_t low, + uint32_t want_h0, uint64_t *__restrict__ d_partials, unsigned int *d_ticket, uint64_t *d_slot, + uint64_t seq) { + gkr_round_gruen_body(p, q, half, quarter, fold, s0, s1, s2, l0, l1, l2, e_hi, + e_lo, low, want_h0, 0, d_partials, d_ticket, d_slot, seq); +} + +extern "C" __global__ __launch_bounds__(256) void gkr_round_gruen_real_slot( + uint64_t *__restrict__ p, uint64_t *__restrict__ q, uint64_t half, uint64_t quarter, + uint32_t fold, uint64_t s0, uint64_t s1, uint64_t s2, uint64_t l0, uint64_t l1, uint64_t l2, + const uint64_t *__restrict__ e_hi, const uint64_t *__restrict__ e_lo, uint32_t low, + uint32_t want_h0, uint64_t stored, uint64_t *__restrict__ d_partials, unsigned int *d_ticket, + uint64_t *d_slot, uint64_t seq) { + gkr_round_gruen_body(p, q, half, quarter, fold, s0, s1, s2, l0, l1, l2, e_hi, e_lo, + low, want_h0, stored, d_partials, d_ticket, d_slot, seq); +} + +template +__device__ __forceinline__ void gkr_round_gruen_body( + uint64_t *__restrict__ p, uint64_t *__restrict__ q, uint64_t half, uint64_t quarter, + uint32_t fold, uint64_t s0, uint64_t s1, uint64_t s2, uint64_t l0, uint64_t l1, uint64_t l2, + const uint64_t *__restrict__ e_hi, const uint64_t *__restrict__ e_lo, uint32_t low, + uint32_t want_h0, uint64_t stored, uint64_t *__restrict__ d_partials, unsigned int *d_ticket, + uint64_t *d_slot, uint64_t seq) { + Fe3 s = ext3::make(s0, s1, s2); + Fe3 lambda = ext3::make(l0, l1, l2); + uint64_t *factors[4] = {p, p + half * 3, q, q + half * 3}; + uint64_t mask = ((uint64_t)1 << low) - 1; + Fe3 acc1 = ext3::zero(), acc2 = ext3::zero(), acc0 = ext3::zero(); + for (uint64_t x = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; x < quarter; + x += (uint64_t)gridDim.x * blockDim.x) { + Fe3 lo[4], hi[4]; +#pragma unroll + for (int k = 0; k < 4; k++) { + uint64_t *f = factors[k]; + if (fold) { + Fe3 a = gkr_cell(f, k, x, stored); + Fe3 b = gkr_cell(f, k, x + quarter, stored); + Fe3 c = gkr_cell(f, k, x + 2 * quarter, stored); + Fe3 d = gkr_cell(f, k, x + 3 * quarter, stored); + lo[k] = ext3::add(a, ext3::mul(s, ext3::sub(c, a))); + hi[k] = ext3::add(b, ext3::mul(s, ext3::sub(d, b))); + if (gkr_stored(k, x, stored)) { + f[x * 3 + 0] = lo[k].a; + f[x * 3 + 1] = lo[k].b; + f[x * 3 + 2] = lo[k].c; + } + if (gkr_stored(k, x + quarter, stored)) { + f[(x + quarter) * 3 + 0] = hi[k].a; + f[(x + quarter) * 3 + 1] = hi[k].b; + f[(x + quarter) * 3 + 2] = hi[k].c; + } + } else { + lo[k] = gkr_cell(f, k, x, stored); + hi[k] = gkr_cell(f, k, x + quarter, stored); + } + } + Fe3 w = ext3::mul(load_ext(e_hi + (x >> low) * 3), load_ext(e_lo + (x & mask) * 3)); + acc1 = ext3::add(acc1, ext3::mul(w, gkr_h(hi[0], hi[1], hi[2], hi[3], lambda))); + Fe3 two[4]; +#pragma unroll + for (int k = 0; k < 4; k++) { + two[k] = ext3::sub(ext3::add(hi[k], hi[k]), lo[k]); + } + acc2 = ext3::add(acc2, ext3::mul(w, gkr_h(two[0], two[1], two[2], two[3], lambda))); + if (want_h0) { + acc0 = ext3::add(acc0, ext3::mul(w, gkr_h(lo[0], lo[1], lo[2], lo[3], lambda))); + } + } + + extern __shared__ uint64_t shared[]; + uint32_t rows = want_h0 ? 3 : 2; + for (uint32_t row = 0; row < rows; row++) { + Fe3 mine = row == 0 ? acc1 : row == 1 ? acc2 : acc0; + shared[threadIdx.x * 3 + 0] = mine.a; + shared[threadIdx.x * 3 + 1] = mine.b; + shared[threadIdx.x * 3 + 2] = mine.c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 sum = ext3::add(load_ext(shared + threadIdx.x * 3), + load_ext(shared + (threadIdx.x + width) * 3)); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t at = ((uint64_t)row * gridDim.x + blockIdx.x) * 3; + d_partials[at + 0] = shared[0]; + d_partials[at + 1] = shared[1]; + d_partials[at + 2] = shared[2]; + } + __syncthreads(); + } + if (SLOT) slot_sums(d_partials, rows, d_ticket, d_slot, seq); +} + +// The rounds' end: each half bound to the last challenge `s` (with `fold`) from its `2·cells` cube +// down to `cells`, written in place and to `out[(k·cells + y)·3]`, halves in the order `p_lo, p_hi, +// q_lo, q_hi` — what the host tail starts from. Thread `y` reads cells `y` and `y + cells` and +// writes cell `y` only. +// +// `gkr_gruen_finish_real` is the same over a layer kept without its padding: `stored` is the run of +// the upper halves' cells held (see `gkr_cell`). Every cell of `out` is written. +template +__device__ __forceinline__ void gkr_gruen_finish_body(uint64_t *__restrict__ p, + uint64_t *__restrict__ q, uint64_t half, + uint64_t cells, uint32_t fold, uint64_t s0, + uint64_t s1, uint64_t s2, uint64_t stored, + uint64_t *__restrict__ out); + +extern "C" __global__ void gkr_gruen_finish(uint64_t *__restrict__ p, uint64_t *__restrict__ q, + uint64_t half, uint64_t cells, uint32_t fold, + uint64_t s0, uint64_t s1, uint64_t s2, + uint64_t *__restrict__ out) { + gkr_gruen_finish_body(p, q, half, cells, fold, s0, s1, s2, 0, out); +} + +extern "C" __global__ void gkr_gruen_finish_real(uint64_t *__restrict__ p, + uint64_t *__restrict__ q, uint64_t half, + uint64_t cells, uint32_t fold, uint64_t s0, + uint64_t s1, uint64_t s2, uint64_t stored, + uint64_t *__restrict__ out) { + gkr_gruen_finish_body(p, q, half, cells, fold, s0, s1, s2, stored, out); +} + +template +__device__ __forceinline__ void gkr_gruen_finish_body(uint64_t *__restrict__ p, + uint64_t *__restrict__ q, uint64_t half, + uint64_t cells, uint32_t fold, uint64_t s0, + uint64_t s1, uint64_t s2, uint64_t stored, + uint64_t *__restrict__ out) { + Fe3 s = ext3::make(s0, s1, s2); + uint64_t total = 4 * cells; + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < total; + task += (uint64_t)gridDim.x * blockDim.x) { + uint64_t k = task / cells; + uint64_t y = task - k * cells; + uint64_t *f = (k < 2 ? p : q) + (k & 1) * half * 3; + Fe3 v = gkr_cell(f, (int)k, y, stored); + if (fold) { + Fe3 hi = gkr_cell(f, (int)k, y + cells, stored); + v = ext3::add(v, ext3::mul(s, ext3::sub(hi, v))); + if (gkr_stored((int)k, y, stored)) { + f[y * 3 + 0] = v.a; + f[y * 3 + 1] = v.b; + f[y * 3 + 2] = v.c; + } + } + uint64_t *at = out + task * 3; + at[0] = v.a; + at[1] = v.b; + at[2] = v.c; + } +} + +// The fraction tree's input layer straight from the table's base columns (D-BATCH M1-2, +// `crate::gkr::input_from_columns`): cell `(s, r)` of `p` and `q` for every interaction slot +// `s < slots` and row `r`, both sides in one launch, where the old path lifts every column into +// the extension and runs one program launch per side. Side `2s` (numerator) and `2s + 1` +// (denominator) are `constant + Σ coeff·column[(r + shift) mod rows]` over their terms — base × +// ext3 products, the same field values the lifted program computes. Slots from `interactions` on +// are the padding's `0/1`. +// +// `d_side_start` (2·interactions + 1 entries) indexes the terms; term `t` is two u64 in `d_terms` +// — where its column starts in `d_columns` and its shift, already reduced mod `rows` — and its +// coefficient at `d_coeffs[3t]`. A side's constant is at `d_constants[3·side]`. +extern "C" __global__ void gkr_input_from_columns( + const uint64_t *__restrict__ d_columns, uint64_t rows, + const uint32_t *__restrict__ d_side_start, const uint64_t *__restrict__ d_terms, + const uint64_t *__restrict__ d_coeffs, const uint64_t *__restrict__ d_constants, + uint64_t interactions, uint64_t slots, uint64_t *__restrict__ p, uint64_t *__restrict__ q) { + uint64_t total = slots * rows; + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < total; + task += (uint64_t)gridDim.x * blockDim.x) { + uint64_t s = task / rows; + uint64_t r = task - s * rows; + Fe3 side[2]; + if (s < interactions) { +#pragma unroll + for (int h = 0; h < 2; h++) { + uint64_t which = 2 * s + h; + Fe3 acc = load_ext(d_constants + which * 3); + for (uint32_t t = d_side_start[which]; t < d_side_start[which + 1]; t++) { + uint64_t k = r + d_terms[2 * t + 1]; + if (k >= rows) k -= rows; + uint64_t value = d_columns[d_terms[2 * t] + k]; + acc = ext3::add(acc, ext3::mul_base(load_ext(d_coeffs + (uint64_t)t * 3), value)); + } + side[h] = acc; + } + } else { + side[0] = ext3::zero(); + side[1] = ext3::one(); + } + uint64_t *at = p + task * 3; + at[0] = side[0].a; + at[1] = side[0].b; + at[2] = side[0].c; + at = q + task * 3; + at[0] = side[1].a; + at[1] = side[1].b; + at[2] = side[1].c; + } +} diff --git a/crypto/math-cuda/kernels/whir_coset.cu b/crypto/math-cuda/kernels/whir_coset.cu new file mode 100644 index 000000000..e973c7012 --- /dev/null +++ b/crypto/math-cuda/kernels/whir_coset.cu @@ -0,0 +1,434 @@ +// The values a first-round query's kept subtree holds, evaluated from the +// stacked multilinear instead of read out of its re-encoded codeword +// (`src/whir_coset.rs`, which states the identity and pins every kernel here +// against the host). +// +// The multilinear `f` has `2^n` hypercube values; its codeword is +// `P(ω^j)`, `P(x) = f̂(x^(2^(n-1)), …, x^2, x)` with hypercube bit `s` sent to +// `x^(2^(n-1-s))`. On the coset `ω^c·⟨ζ⟩`, `|⟨ζ⟩| = 2^L`, every bit `s < n-L` +// sees a constant `γ_s(c) = ω^(c·2^(n-1-s))` and folds away: +// +// f'(rest) = f(rest, 0) + γ·(f(rest, 1) − f(rest, 0)) +// +// Bit `s`'s constant depends on `c` modulo `2^(b+1+s)` only, so the low bits +// are folded once per residue class and shared by every coset in it: the +// tile kernel folds the low `A` bits of each `2^A`-value tile for every class +// any coset needs, the rest kernel folds the remaining bits per coset, and +// what is left (`2^L` values a coset) goes through its Möbius transform, a +// twist by `ω^(c·i)` and one radix-2 NTT over `ζ`. +// +// Values are non-canonical u64 throughout (`goldilocks.cuh`); the host +// compares canonical values. + +#include "goldilocks.cuh" + +using goldilocks::add; +using goldilocks::mul; +using goldilocks::sub; + +// Tiles one block folds before it writes: a run of this many consecutive tile +// indices per class, so the writes are whole sectors. +#define COSET_TILES_PER_BLOCK 8 +// Most tile levels (`A`): a tile of 256 values, the block's width. +#define COSET_MAX_TILE_LEVELS 8 + +__device__ __forceinline__ uint64_t coset_fold(uint64_t f0, uint64_t f1, uint64_t gamma) { + return add(f0, mul(gamma, sub(f1, f0))); +} + +__device__ __forceinline__ uint64_t coset_pow(uint64_t base, uint64_t exp) { + uint64_t acc = 1; + while (exp > 0) { + if (exp & 1ull) acc = mul(acc, base); + base = mul(base, base); + exp >>= 1; + } + return acc; +} + +// Folds the low `levels` bits of every `2^levels`-value tile of `f` for every +// class the plan lists. +// +// Level `t` lists `count[t]` classes at `offset[t]` in `parent`/`gamma`: +// class `k`'s parent is its residue one level down (row `parent[k]` of the +// level before; level 0's parent is the tile itself, row 0), and `gamma[k]` is +// its constant for bit `t`. Level `t` holds `count[t]` rows of +// `2^(levels-1-t)` values, so a level's buffer is `buf_len` at most. +// +// Output: `slab[k·num_tiles + T]`, `k` a class of the last level, `T` the tile. +// +// Shared memory (dynamic): gamma[total] · buf0[buf_len] · buf1[buf_len] · +// fin[count[levels-1]·TILES] · parent[total] (u32). +extern "C" __global__ void __launch_bounds__(256) +coset_fold_tiles(const uint64_t *__restrict__ f, uint64_t num_tiles, uint32_t levels, + const uint32_t *__restrict__ count, const uint32_t *__restrict__ offset, + const uint32_t *__restrict__ parent, const uint64_t *__restrict__ gamma, + uint32_t total, uint32_t buf_len, uint64_t *__restrict__ slab) { + extern __shared__ uint64_t coset_smem[]; + __shared__ uint32_t s_count[COSET_MAX_TILE_LEVELS]; + __shared__ uint32_t s_offset[COSET_MAX_TILE_LEVELS]; + + const uint32_t tid = threadIdx.x; + const uint32_t bdim = blockDim.x; + if (tid < levels) { + s_count[tid] = count[tid]; + s_offset[tid] = offset[tid]; + } + __syncthreads(); + const uint32_t last_count = s_count[levels - 1]; + + uint64_t *s_gamma = coset_smem; + uint64_t *buf0 = s_gamma + total; + uint64_t *buf1 = buf0 + buf_len; + uint64_t *fin = buf1 + buf_len; + uint32_t *s_parent = (uint32_t *)(fin + (uint64_t)last_count * COSET_TILES_PER_BLOCK); + for (uint32_t x = tid; x < total; x += bdim) { + s_gamma[x] = gamma[x]; + s_parent[x] = parent[x]; + } + + const uint32_t tile = 1u << levels; + // A pass a group of tiles, the grid striding over the groups: one pass a + // block at the full grid, more under a cap. + for (uint64_t first = (uint64_t)blockIdx.x * COSET_TILES_PER_BLOCK; first < num_tiles; + first += (uint64_t)gridDim.x * COSET_TILES_PER_BLOCK) { + for (uint32_t i = 0; i < COSET_TILES_PER_BLOCK; ++i) { + const uint64_t at = first + i; + if (at >= num_tiles) break; // the same for every thread of the block + const uint64_t *src = f + at * tile; + // The previous tile's last level read buf0's partner; the barrier at + // that level's end ordered it before this write. + for (uint32_t x = tid; x < tile; x += bdim) buf0[x] = src[x]; + __syncthreads(); + uint64_t *in = buf0; + uint64_t *out = buf1; + uint32_t width = tile; + for (uint32_t t = 0; t < levels; ++t) { + const uint32_t cnt = s_count[t]; + const uint32_t off = s_offset[t]; + const uint32_t log_half = levels - 1 - t; + const uint32_t half = 1u << log_half; + const uint32_t outputs = cnt << log_half; + const bool last = t + 1 == levels; + for (uint32_t o = tid; o < outputs; o += bdim) { + const uint32_t k = o >> log_half; + const uint32_t j = o & (half - 1); + const uint64_t *row = in + (uint64_t)s_parent[off + k] * width + 2 * j; + const uint64_t r = coset_fold(row[0], row[1], s_gamma[off + k]); + if (last) { + fin[(uint64_t)k * COSET_TILES_PER_BLOCK + i] = r; + } else { + out[o] = r; + } + } + __syncthreads(); + uint64_t *swap = in; + in = out; + out = swap; + width = half; + } + } + + for (uint32_t x = tid; x < last_count * COSET_TILES_PER_BLOCK; x += bdim) { + const uint32_t k = x / COSET_TILES_PER_BLOCK; + const uint64_t at = first + x % COSET_TILES_PER_BLOCK; + if (at < num_tiles) slab[(uint64_t)k * num_tiles + at] = fin[x]; + } + // The next pass's tiles write `fin` again. + __syncthreads(); + } +} + +// The stack read where it lies: its parts (columns), sorted by their offset in +// the stack and disjoint, are `part_len[p]` values at `part_off[p]` in the stack +// and at `part_at[p]` in the store's buffer `data`; every other value is zero. +// +// A tile's map entry is the part that holds all of it, `COSET_TILE_ZERO` when +// no part touches it, or `COSET_TILE_MIXED` (shorter columns, a column's end, +// zeros) when its values are looked up one by one. +#define COSET_TILE_ZERO 0xffffffffu +#define COSET_TILE_MIXED 0xfffffffeu + +// The number of parts whose offset is at most `e`: the last of them is the only +// one that can hold `e`. +__device__ __forceinline__ uint32_t coset_parts_upto(const uint64_t *__restrict__ part_off, + uint32_t nparts, uint64_t e) { + uint32_t lo = 0, hi = nparts; + while (lo < hi) { + const uint32_t mid = (lo + hi) >> 1; + if (part_off[mid] <= e) lo = mid + 1; + else hi = mid; + } + return lo; +} + +// One thread a tile of `2^tile_bits` values, the grid striding over them. +extern "C" __global__ void coset_tile_map(const uint64_t *__restrict__ part_off, + const uint64_t *__restrict__ part_len, uint32_t nparts, + uint32_t tile_bits, uint64_t num_tiles, + uint32_t *__restrict__ map) { + for (uint64_t tile = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; tile < num_tiles; + tile += (uint64_t)gridDim.x * blockDim.x) { + const uint64_t start = tile << tile_bits; + const uint64_t end = start + (1ull << tile_bits); + const uint32_t upto = coset_parts_upto(part_off, nparts, start); + uint32_t kind = COSET_TILE_ZERO; + if (upto > 0) { + const uint32_t p = upto - 1; + const uint64_t past = part_off[p] + part_len[p]; + if (past >= end) kind = p; + else if (past > start) kind = COSET_TILE_MIXED; + } + // The next part, past `start`, starting before the tile's end. + if (kind == COSET_TILE_ZERO && upto < nparts && part_off[upto] < end) + kind = COSET_TILE_MIXED; + map[tile] = kind; + } +} + +// `coset_fold_tiles` over the stack read where it lies (the map above): a tile +// one part holds is read from the store; a zero tile folds to zero for every +// class (`f0 + γ·(f1 − f0)` at `f0 = f1 = 0` is the word 0), so its outputs are +// written as zeros and its levels skipped; a mixed tile is read value by value. +// The same words as `coset_fold_tiles` over the assembled stack. +extern "C" __global__ void __launch_bounds__(256) +coset_fold_tiles_parts(const uint64_t *__restrict__ data, const uint32_t *__restrict__ map, + const uint64_t *__restrict__ part_off, const uint64_t *__restrict__ part_len, + const uint64_t *__restrict__ part_at, uint32_t nparts, uint64_t num_tiles, + uint32_t levels, const uint32_t *__restrict__ count, + const uint32_t *__restrict__ offset, const uint32_t *__restrict__ parent, + const uint64_t *__restrict__ gamma, uint32_t total, uint32_t buf_len, + uint64_t *__restrict__ slab) { + extern __shared__ uint64_t coset_smem[]; + __shared__ uint32_t s_count[COSET_MAX_TILE_LEVELS]; + __shared__ uint32_t s_offset[COSET_MAX_TILE_LEVELS]; + + const uint32_t tid = threadIdx.x; + const uint32_t bdim = blockDim.x; + if (tid < levels) { + s_count[tid] = count[tid]; + s_offset[tid] = offset[tid]; + } + __syncthreads(); + const uint32_t last_count = s_count[levels - 1]; + + uint64_t *s_gamma = coset_smem; + uint64_t *buf0 = s_gamma + total; + uint64_t *buf1 = buf0 + buf_len; + uint64_t *fin = buf1 + buf_len; + uint32_t *s_parent = (uint32_t *)(fin + (uint64_t)last_count * COSET_TILES_PER_BLOCK); + for (uint32_t x = tid; x < total; x += bdim) { + s_gamma[x] = gamma[x]; + s_parent[x] = parent[x]; + } + + const uint32_t tile = 1u << levels; + // A pass a group of tiles, the grid striding over the groups, as in + // `coset_fold_tiles`. + for (uint64_t first = (uint64_t)blockIdx.x * COSET_TILES_PER_BLOCK; first < num_tiles; + first += (uint64_t)gridDim.x * COSET_TILES_PER_BLOCK) { + for (uint32_t i = 0; i < COSET_TILES_PER_BLOCK; ++i) { + const uint64_t at = first + i; + if (at >= num_tiles) break; // the same for every thread of the block + const uint32_t kind = map[at]; // the same for every thread of the block + if (kind == COSET_TILE_ZERO) { + for (uint32_t k = tid; k < last_count; k += bdim) + fin[(uint64_t)k * COSET_TILES_PER_BLOCK + i] = 0; + continue; + } + const uint64_t base = at << levels; + if (kind == COSET_TILE_MIXED) { + for (uint32_t x = tid; x < tile; x += bdim) { + const uint64_t e = base + x; + const uint32_t upto = coset_parts_upto(part_off, nparts, e); + uint64_t v = 0; + if (upto > 0) { + const uint32_t p = upto - 1; + if (e < part_off[p] + part_len[p]) + v = data[part_at[p] + (e - part_off[p])]; + } + buf0[x] = v; + } + } else { + const uint64_t *src = data + part_at[kind] + (base - part_off[kind]); + for (uint32_t x = tid; x < tile; x += bdim) buf0[x] = src[x]; + } + // The previous folded tile's last level read buf0's partner; the + // barrier at that level's end ordered it before this write. + __syncthreads(); + uint64_t *in = buf0; + uint64_t *out = buf1; + uint32_t width = tile; + for (uint32_t t = 0; t < levels; ++t) { + const uint32_t cnt = s_count[t]; + const uint32_t off = s_offset[t]; + const uint32_t log_half = levels - 1 - t; + const uint32_t half = 1u << log_half; + const uint32_t outputs = cnt << log_half; + const bool last = t + 1 == levels; + for (uint32_t o = tid; o < outputs; o += bdim) { + const uint32_t k = o >> log_half; + const uint32_t j = o & (half - 1); + const uint64_t *row = in + (uint64_t)s_parent[off + k] * width + 2 * j; + const uint64_t r = coset_fold(row[0], row[1], s_gamma[off + k]); + if (last) { + fin[(uint64_t)k * COSET_TILES_PER_BLOCK + i] = r; + } else { + out[o] = r; + } + } + __syncthreads(); + uint64_t *swap = in; + in = out; + out = swap; + width = half; + } + } + // A zero tile's outputs were written without a barrier after them. + __syncthreads(); + + for (uint32_t x = tid; x < last_count * COSET_TILES_PER_BLOCK; x += bdim) { + const uint32_t k = x / COSET_TILES_PER_BLOCK; + const uint64_t at = first + x % COSET_TILES_PER_BLOCK; + if (at < num_tiles) slab[(uint64_t)k * num_tiles + at] = fin[x]; + } + // The next pass's tiles write `fin` again. + __syncthreads(); + } +} + +// Folds the bits the tile kernel left (`rest` ≤ 5 of them) per coset: one +// warp a value, lane `l` holding slab entry `h·2^rest + l` of the coset's +// class, the pairs a level folds `2^l` lanes apart. +// +// `g[q·2^log_free + h]`, `h < 2^log_free`. `rest_gamma[q·rest + l]` is coset +// `q`'s constant for its `l`-th remaining bit; `slab_of[q]` its class row. +extern "C" __global__ void __launch_bounds__(256) +coset_fold_rest(const uint64_t *__restrict__ slab, uint64_t num_tiles, uint32_t rest, + uint32_t cosets, const uint32_t *__restrict__ slab_of, + const uint64_t *__restrict__ rest_gamma, uint32_t log_free, + uint64_t *__restrict__ g) { + const uint32_t lane = threadIdx.x & 31; + // Whole warps stride over the values (blockDim a multiple of 32). + for (uint64_t warp = ((uint64_t)blockIdx.x * blockDim.x + threadIdx.x) >> 5; + warp < ((uint64_t)cosets << log_free); warp += ((uint64_t)gridDim.x * blockDim.x) >> 5) { + const uint64_t q = warp >> log_free; + const uint64_t h = warp & ((1ull << log_free) - 1); + const uint64_t *src = slab + (uint64_t)slab_of[q] * num_tiles + (h << rest); + uint64_t v = lane < (1u << rest) ? src[lane] : 0; + for (uint32_t l = 0; l < rest; ++l) { + const uint64_t partner = + (uint64_t)__shfl_down_sync(0xffffffffu, (unsigned long long)v, 1u << l); + if ((lane & ((2u << l) - 1)) == 0) v = coset_fold(v, partner, rest_gamma[q * rest + l]); + } + if (lane == 0) g[warp] = v; + } +} + +// One level of the Möbius transform over every coset's `2^log_size` values, +// for a level the chunk kernel does not reach: `hi −= lo`, `2^level` apart. +extern "C" __global__ void coset_mobius_level(uint64_t *__restrict__ g, uint64_t pairs, + uint32_t log_size, uint32_t level) { + const uint64_t idx = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= pairs) return; + const uint64_t q = idx >> (log_size - 1); + const uint64_t r = idx & ((1ull << (log_size - 1)) - 1); + const uint64_t lo = ((r >> level) << (level + 1)) | (r & ((1ull << level) - 1)); + uint64_t *base = g + (q << log_size); + base[lo + (1ull << level)] = sub(base[lo + (1ull << level)], base[lo]); +} + +// One `2^chunk_levels` chunk of a coset's values, in shared memory: the +// Möbius levels below `chunk_levels`, the twist, and the NTT's first +// `chunk_levels` stages. +// +// After the Möbius transform position `h` holds the coefficient of +// `x^bitrev_L(h)` — the input order a decimation-in-time NTT takes — so the +// twist multiplies it by `w^bitrev_L(h)`, `w = twist[q] = ω^c`, and the stages +// run in place. `twiddles[j] = ζ^j`, `j < 2^(log_size-1)`. One thread a pair. +extern "C" __global__ void coset_chunk(uint64_t *__restrict__ g, uint32_t log_size, + uint32_t chunk_levels, + const uint64_t *__restrict__ twist, + const uint64_t *__restrict__ twiddles) { + extern __shared__ uint64_t coset_chunk_smem[]; + uint64_t *x = coset_chunk_smem; + const uint32_t tid = threadIdx.x; + const uint32_t half = 1u << (chunk_levels - 1); + const uint64_t chunks = 1ull << (log_size - chunk_levels); + const uint64_t q = blockIdx.x / chunks; + const uint64_t at = (blockIdx.x % chunks) << chunk_levels; + uint64_t *base = g + (q << log_size) + at; + x[tid] = base[tid]; + x[tid + half] = base[tid + half]; + __syncthreads(); + + for (uint32_t l = 0; l < chunk_levels; ++l) { + const uint32_t lo = ((tid >> l) << (l + 1)) | (tid & ((1u << l) - 1)); + x[lo + (1u << l)] = sub(x[lo + (1u << l)], x[lo]); + __syncthreads(); + } + + const uint64_t w = twist[q]; + for (uint32_t e = tid; e < 2 * half; e += half) { + const uint64_t i = __brevll(at + e) >> (64 - log_size); + x[e] = mul(x[e], coset_pow(w, i)); + } + __syncthreads(); + + for (uint32_t s = 1; s <= chunk_levels; ++s) { + const uint32_t h = 1u << (s - 1); + const uint32_t i0 = ((tid >> (s - 1)) << s) | (tid & (h - 1)); + const uint64_t tw = twiddles[(uint64_t)(tid & (h - 1)) << (log_size - s)]; + const uint64_t u = x[i0]; + const uint64_t v = mul(x[i0 + h], tw); + x[i0] = add(u, v); + x[i0 + h] = sub(u, v); + __syncthreads(); + } + + base[tid] = x[tid]; + base[tid + half] = x[tid + half]; +} + +// One NTT stage `s` (pairs `2^(s-1)` apart) over every coset's values, for a +// stage past the chunk kernel's. +extern "C" __global__ void coset_ntt_stage(uint64_t *__restrict__ g, uint64_t pairs, + uint32_t log_size, uint32_t s, + const uint64_t *__restrict__ twiddles) { + const uint64_t idx = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= pairs) return; + const uint64_t q = idx >> (log_size - 1); + const uint64_t r = idx & ((1ull << (log_size - 1)) - 1); + const uint64_t h = 1ull << (s - 1); + const uint64_t j = r & (h - 1); + const uint64_t i0 = ((r >> (s - 1)) << s) | j; + uint64_t *base = g + (q << log_size); + const uint64_t u = base[i0]; + const uint64_t v = mul(base[i0 + h], twiddles[j << (log_size - s)]); + base[i0] = add(u, v); + base[i0 + h] = sub(u, v); +} + +// The coset's values, in domain order (`g[q][u]` = the codeword at +// `c + u·2^(D-L)`), written in the order its kept subtree's leaves take them: +// leaf `r` (position `r` under the node) is `j = c + bitrev_kept(r)·2^(D-L)`, +// and its `2^fold` values are `u = bitrev_kept(r) + t·2^kept`, `t` ascending. +// `stride == 0`: leaf-major, leaf `r` of coset `q` at `(q·2^kept + r)·2^fold`. +// Otherwise strided as the coset leaf kernels read a codeword: value `t` of +// gathered leaf `q·2^kept + r` at `(q·2^kept + r) + t·stride`. +extern "C" __global__ void coset_leaf_order(const uint64_t *__restrict__ g, uint64_t total, + uint32_t log_size, uint32_t kept, uint64_t stride, + uint64_t *__restrict__ out) { + const uint64_t idx = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= total) return; + const uint64_t q = idx >> log_size; + const uint64_t u = idx & ((1ull << log_size) - 1); + const uint64_t low = u & ((1ull << kept) - 1); + const uint64_t r = kept == 0 ? 0 : __brevll(low) >> (64 - kept); + const uint64_t t = u >> kept; + if (stride == 0) { + out[(q << log_size) | (r << (log_size - kept)) | t] = g[idx]; + } else { + out[((q << kept) | r) + t * stride] = g[idx]; + } +} diff --git a/crypto/math-cuda/kernels/whir_fold.cu b/crypto/math-cuda/kernels/whir_fold.cu new file mode 100644 index 000000000..aecaf71e8 --- /dev/null +++ b/crypto/math-cuda/kernels/whir_fold.cu @@ -0,0 +1,890 @@ +// Folding a WHIR codeword on device. +// +// One fold halves the codeword, pairing `j` with `j + half` — the two points +// `x` and `−x`, which is half a period apart on a multiplicative domain: +// +// out[j] = two_inv·(a + b) + (two_inv·g^{-j})·(a − b)·alpha +// +// where `a = in[j]`, `b = in[j + half]`. A transliteration of `fold_codeword` +// in `crypto/multilinear/src/whir.rs`, and `tests/whir_fold.rs` pins it there. +// +// `g^{-j}` is computed per thread by squaring rather than read from a table: +// the fold is bound by the codeword it streams, and a table would be another +// pass over memory the exponentiation does not need. +// +// Two entry points: the first fold of a chain reads the committed base-field +// codeword and lifts it, every later one is ext3 throughout. Ext3 values are +// interleaved (`[a0,b0,c0, a1,b1,c1, ...]`), the layout the rest of the crate +// uses. + +#include "goldilocks.cuh" +#include "ext3.cuh" + +using ext3::Fe3; + +__device__ __forceinline__ uint64_t pow_base(uint64_t base, uint64_t exp) { + uint64_t acc = 1; + uint64_t square = base; + while (exp > 0) { + if (exp & 1ull) { + acc = goldilocks::mul(acc, square); + } + square = goldilocks::mul(square, square); + exp >>= 1; + } + return acc; +} + +extern "C" __global__ void whir_fold_base_ext3(const uint64_t *__restrict__ in, uint64_t half, + uint64_t two_inv, uint64_t g_inv, + const uint64_t *__restrict__ alpha, + uint64_t *__restrict__ out) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= half) return; + + uint64_t a = in[j]; + uint64_t b = in[j + half]; + uint64_t even = goldilocks::mul(two_inv, goldilocks::add(a, b)); + uint64_t scale = goldilocks::mul(two_inv, pow_base(g_inv, j)); + uint64_t odd = goldilocks::mul(scale, goldilocks::sub(a, b)); + + // `odd·alpha` with `odd` in the base field, then `even +` it: the two + // mixed-field shortcuts the tower gives, both bit-identical to the full + // ext3 ops on the embedded operand. + Fe3 term = ext3::mul_base(ext3::make(alpha[0], alpha[1], alpha[2]), odd); + uint64_t *at = out + j * 3; + at[0] = goldilocks::add(even, term.a); + at[1] = term.b; + at[2] = term.c; +} + +extern "C" __global__ void whir_fold_ext3(const uint64_t *__restrict__ in, uint64_t half, + uint64_t two_inv, uint64_t g_inv, + const uint64_t *__restrict__ alpha, + uint64_t *__restrict__ out) { + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= half) return; + + const uint64_t *lo = in + j * 3; + const uint64_t *hi = in + (j + half) * 3; + Fe3 a = ext3::make(lo[0], lo[1], lo[2]); + Fe3 b = ext3::make(hi[0], hi[1], hi[2]); + Fe3 even = ext3::mul_base(ext3::add(a, b), two_inv); + uint64_t scale = goldilocks::mul(two_inv, pow_base(g_inv, j)); + Fe3 odd = ext3::mul_base(ext3::sub(a, b), scale); + Fe3 res = ext3::add(even, ext3::mul(odd, ext3::make(alpha[0], alpha[1], alpha[2]))); + + uint64_t *at = out + j * 3; + at[0] = res.a; + at[1] = res.b; + at[2] = res.c; +} + +// The fold blocks a round's queries open, gathered into one buffer: +// `out[(q*block + t)*limbs ..]` is `codeword[(index[q] + t*num_leaves)*limbs]`. +// One launch and one copy back, against one of each per value. +extern "C" __global__ void gather_cosets(const uint64_t *__restrict__ codeword, + const uint64_t *__restrict__ indices, uint64_t queries, + uint64_t num_leaves, uint64_t block, uint64_t limbs, + uint64_t *__restrict__ out) { + uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (task >= queries * block) return; + uint64_t q = task / block; + uint64_t t = task - q * block; + const uint64_t *from = codeword + (indices[q] + t * num_leaves) * limbs; + uint64_t *at = out + task * limbs; + for (uint64_t k = 0; k < limbs; ++k) { + at[k] = from[k]; + } +} + +// The same gather into the strided layout the coset leaf kernels read: leaf `i` +// of the `leaves` gathered (`indices[i]`) at slot `t` goes to +// `out[(i + t*leaves)*limbs ..]`, so `rpx_leaves_*_coset` / `keccak256_leaves_*_coset` +// over `out` with `num_leaves = leaves` hash exactly the committed leaves. What a +// retired tree's queried blocks are re-hashed from, on the card. +extern "C" __global__ void gather_cosets_strided(const uint64_t *__restrict__ codeword, + const uint64_t *__restrict__ indices, + uint64_t leaves, uint64_t num_leaves, + uint64_t block, uint64_t limbs, + uint64_t *__restrict__ out) { + uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (task >= leaves * block) return; + uint64_t i = task / block; + uint64_t t = task - i * block; + const uint64_t *from = codeword + (indices[i] + t * num_leaves) * limbs; + uint64_t *at = out + (i + t * leaves) * limbs; + for (uint64_t k = 0; k < limbs; ++k) { + at[k] = from[k]; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// EVERY LEVEL OF A ROUND'S FOLD IN ONE PASS. +// +// Folding `k` levels one launch at a time writes every intermediate codeword: +// the first level alone is `2^(n+1)` extension values — 1.5× the committed +// base codeword — and it is live beside the second level's (0.75×) while the +// committed codeword still is. But output `j` depends only on the strided +// coset `{j + t·n_out : t < 2^k}` (`n_out = N / 2^k`), so one thread folds its +// coset to the end in registers and the only thing written is the output. +// +// Inside a coset, level `L` pairs `(t, t + 2^(k−1−L))` — the TOP bit of that +// level's index — so the coset is a binary tree over its index bits from the +// top down. The thread walks the leaf pairs in bit-reversed order, which makes +// the two halves of every later pair adjacent in the walk, and merges them on a +// stack holding one pending value per level (a binary counter): `k` extension +// values of state rather than `2^(k−1)`. +// +// ★ RAW-IDENTICAL, not merely equal. A proof serializes raw limbs, which may be +// non-canonical, so every pair is folded with the exact operation sequence of +// `whir_fold_base_ext3` / `whir_fold_ext3` above — including `pow_base`'s +// square-and-multiply for `g^{-pos}`, replayed from a table of the same squares +// in the same ascending-bit order — on the same raw inputs. The output is the +// level-by-level result bit for bit (`tests/whir_fold.rs`, +// `tests/host_kat/whir_host_kat.cpp`). +// ───────────────────────────────────────────────────────────────────────────── + +#define WHIR_MAX_FOLD 6 +// Positions stay below 2^32: a codeword longer than that is refused on the host +// (`commit_from`), so 32 squares cover every exponent `pow_base` could see. +#define WHIR_POW_BITS 32 + +__device__ __forceinline__ uint32_t whir_reverse_low_bits(uint32_t x, uint32_t bits) { + uint32_t r = 0; + for (uint32_t i = 0; i < bits; ++i) { + r = (r << 1) | ((x >> i) & 1u); + } + return r; +} + +// `pow_base(base, exp)` from the squares `pow_base` would compute, in the order +// it would multiply them: bit for bit its result. +__device__ __forceinline__ uint64_t whir_pow_from_squares(const uint64_t *squares, uint64_t exp) { + uint64_t acc = 1; + uint32_t i = 0; + while (exp > 0) { + if (exp & 1ull) { + acc = goldilocks::mul(acc, squares[i]); + } + exp >>= 1; + ++i; + } + return acc; +} + +// `whir_fold_base_ext3`'s arithmetic for one pair, `x_pow = g^{-pos}`. +__device__ __forceinline__ Fe3 whir_fold_pair_base(uint64_t a, uint64_t b, uint64_t two_inv, + uint64_t x_pow, const Fe3 &alpha) { + uint64_t even = goldilocks::mul(two_inv, goldilocks::add(a, b)); + uint64_t scale = goldilocks::mul(two_inv, x_pow); + uint64_t odd = goldilocks::mul(scale, goldilocks::sub(a, b)); + Fe3 term = ext3::mul_base(alpha, odd); + return ext3::make(goldilocks::add(even, term.a), term.b, term.c); +} + +// `whir_fold_ext3`'s arithmetic for one pair. +__device__ __forceinline__ Fe3 whir_fold_pair_ext3(const Fe3 &a, const Fe3 &b, uint64_t two_inv, + uint64_t x_pow, const Fe3 &alpha) { + Fe3 even = ext3::mul_base(ext3::add(a, b), two_inv); + uint64_t scale = goldilocks::mul(two_inv, x_pow); + Fe3 odd = ext3::mul_base(ext3::sub(a, b), scale); + return ext3::add(even, ext3::mul(odd, alpha)); +} + +// The squares `pow_base(g_inv, ·)` walks for level `level`'s inverse +// generator: `squares[level·WHIR_POW_BITS + i] = g_inv^(2^i)`, each the square +// of the one before, exactly as `pow_base` computes them. +__device__ __forceinline__ void whir_fold_squares(uint64_t *squares, uint32_t level, + uint64_t g_inv) { + uint64_t square = g_inv; + for (uint32_t i = 0; i < WHIR_POW_BITS; ++i) { + squares[level * WHIR_POW_BITS + i] = square; + square = goldilocks::mul(square, square); + } +} + +// Output `j` of a `k`-level fold of `in` (`n_out · 2^k` values, base-field when +// `base`), level `L` at challenge `alphas[3L..3L+3]` and at the inverse +// generator whose squares are `squares[L·WHIR_POW_BITS ..]`. +__device__ Fe3 whir_fold_coset(const uint64_t *__restrict__ in, bool base, uint64_t j, + uint64_t n_out, uint32_t k, uint64_t two_inv, + const uint64_t *squares, const uint64_t *__restrict__ alphas) { + Fe3 pending[WHIR_MAX_FOLD]; + const uint32_t leaf_half = 1u << (k - 1); + Fe3 v = ext3::zero(); + for (uint32_t m = 0; m < leaf_half; ++m) { + // Level 0: the pair `(t0, t0 + 2^(k−1))` of the coset. + uint32_t t0 = whir_reverse_low_bits(m, k - 1); + uint64_t lo = j + (uint64_t)t0 * n_out; + uint64_t hi = j + (uint64_t)(t0 + leaf_half) * n_out; + uint64_t x_pow = whir_pow_from_squares(squares, lo); + Fe3 alpha0 = ext3::make(alphas[0], alphas[1], alphas[2]); + if (base) { + v = whir_fold_pair_base(in[lo], in[hi], two_inv, x_pow, alpha0); + } else { + v = whir_fold_pair_ext3(ext3::make(in[lo * 3], in[lo * 3 + 1], in[lo * 3 + 2]), + ext3::make(in[hi * 3], in[hi * 3 + 1], in[hi * 3 + 2]), + two_inv, x_pow, alpha0); + } + // Merge up while `v` is the right half of a pair whose left is pending: + // at level `L` the left half is level-(L−1) node `2·(m >> L)` in walk + // order, whose index in that level's array is its bit reversal. + uint32_t level = 1; + uint32_t counter = m; + while (level < k && (counter & 1u)) { + uint32_t t_left = whir_reverse_low_bits(2u * (m >> level), k - level); + uint64_t pos = j + (uint64_t)t_left * n_out; + uint64_t xl = whir_pow_from_squares(squares + level * WHIR_POW_BITS, pos); + Fe3 alpha = ext3::make(alphas[3 * level], alphas[3 * level + 1], alphas[3 * level + 2]); + v = whir_fold_pair_ext3(pending[level], v, two_inv, xl, alpha); + counter >>= 1; + ++level; + } + if (level < k) { + pending[level] = v; + } + } + return v; +} + +// The squares are the same for every thread, so a block builds them once in +// shared memory — one thread per level — before any thread folds. +__device__ __forceinline__ void whir_fold_k_kernel(const uint64_t *__restrict__ in, bool base, + uint64_t n_out, uint32_t k, uint64_t two_inv, + const uint64_t *__restrict__ g_invs, + const uint64_t *__restrict__ alphas, + uint64_t *__restrict__ out) { + __shared__ uint64_t squares[WHIR_MAX_FOLD * WHIR_POW_BITS]; + if (threadIdx.x < k) { + whir_fold_squares(squares, threadIdx.x, g_invs[threadIdx.x]); + } + __syncthreads(); + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= n_out) return; + Fe3 v = whir_fold_coset(in, base, j, n_out, k, two_inv, squares, alphas); + out[j * 3] = v.a; + out[j * 3 + 1] = v.b; + out[j * 3 + 2] = v.c; +} + +extern "C" __global__ void whir_fold_k_base_ext3(const uint64_t *__restrict__ in, uint64_t n_out, + uint32_t k, uint64_t two_inv, + const uint64_t *__restrict__ g_invs, + const uint64_t *__restrict__ alphas, + uint64_t *__restrict__ out) { + whir_fold_k_kernel(in, true, n_out, k, two_inv, g_invs, alphas, out); +} + +extern "C" __global__ void whir_fold_k_ext3(const uint64_t *__restrict__ in, uint64_t n_out, + uint32_t k, uint64_t two_inv, + const uint64_t *__restrict__ g_invs, + const uint64_t *__restrict__ alphas, + uint64_t *__restrict__ out) { + whir_fold_k_kernel(in, false, n_out, k, two_inv, g_invs, alphas, out); +} + +// ───────────────────────────────────────────────────────────────────────────── +// A WHIR CHAIN'S FIRST ROUNDS OVER ITS FACTORS IN SHARE FORM. +// +// The opening's factors are the weight `w` and the message `f`, each `2^n` +// extension values once materialised — 1.5× the committed base codeword +// together — and the first group of rounds reads them at full width before +// binding anything. They need not exist at that width: +// +// f(x) = the column holding `x`, at its row; +// w(x) = scale_c · eq(z_c, row) for the column `c` holding `x`, else 0, +// +// where a column is a SHARE: a stack offset, its data, its point and its scale. +// Rounds bind the index's top bits first (`Mle::fix_first_variable` pairs `j` +// with `j + half`), so after `s − 1` of them the factors at `(bit s, rest)` are +// `Σ_p eq(α_{> lo_bits] · +// lo[row & (2^lo_bits − 1)]`: the two halves of the point's table, each a few +// kilobytes, instead of the table. +// ───────────────────────────────────────────────────────────────────────────── + +#define WHIR_LEAN_NONE 0xFFFFu +#define WHIR_LEAN_SHARE_WORDS 9 +#define WHIR_LEAN_MAX_T 4 + +// The share holding each stack position, by binary search over the shares' +// offsets (sorted, disjoint): `NONE` for a position no column covers. +extern "C" __global__ void whir_lean_colmap(uint16_t *__restrict__ colmap, uint64_t len, + const uint64_t *__restrict__ starts, + const uint64_t *__restrict__ ends, uint32_t count) { + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t x = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; x < len; x += stride) { + uint32_t lo = 0, hi = count; + while (lo < hi) { + uint32_t mid = (lo + hi) / 2; + if (starts[mid] <= x) { + lo = mid + 1; + } else { + hi = mid; + } + } + uint16_t id = WHIR_LEAN_NONE; + if (lo > 0 && x < ends[lo - 1]) { + id = (uint16_t)(lo - 1); + } + colmap[x] = id; + } +} + +// `(w(x), f(x))` from the shares. +__device__ __forceinline__ void whir_lean_value(const uint16_t *__restrict__ colmap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, uint64_t x, + Fe3 &w, uint64_t &f) { + uint16_t id = colmap[x]; + if (id == WHIR_LEAN_NONE) { + w = ext3::zero(); + f = 0; + return; + } + const uint64_t *share = shares + (uint64_t)id * WHIR_LEAN_SHARE_WORDS; + uint64_t row = x - share[0]; + f = data[share[1] + row]; + uint64_t lo_bits = share[2]; + uint64_t hi_at = (share[3] + (row >> lo_bits)) * 3; + uint64_t lo_at = (share[4] + (row & ((1ull << lo_bits) - 1))) * 3; + Fe3 eq = ext3::mul(ext3::make(eqbuf[hi_at], eqbuf[hi_at + 1], eqbuf[hi_at + 2]), + ext3::make(eqbuf[lo_at], eqbuf[lo_at + 1], eqbuf[lo_at + 2])); + w = ext3::mul(ext3::make(share[5], share[6], share[7]), eq); +} + +// One rest-position's share of round `s` (1-based) over `n` variables: the +// factors at `(p ‖ b ‖ rest)` for the `2^(s−1)` bound prefixes `p` (weights +// `eqc[p]`) and `b ∈ {0, 1}`, evaluated at each node `t` as `lo + t·(hi − lo)` +// and multiplied — the product rule the opening's program is. +__device__ void whir_lean_round_at(const uint16_t *__restrict__ colmap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, uint32_t n, uint32_t s, + const uint64_t *__restrict__ eqc, + const uint64_t *__restrict__ d_t, uint32_t num_t, + uint64_t rest, Fe3 *acc) { + uint32_t below = n - s; // bits under the round's variable + uint64_t half = 1ull << below; + Fe3 w_lo = ext3::zero(), w_hi = ext3::zero(), f_lo = ext3::zero(), f_hi = ext3::zero(); + uint64_t prefixes = 1ull << (s - 1); + for (uint64_t p = 0; p < prefixes; ++p) { + Fe3 c = ext3::make(eqc[p * 3], eqc[p * 3 + 1], eqc[p * 3 + 2]); + uint64_t x_lo = (p << (below + 1)) | rest; + Fe3 w; + uint64_t f; + whir_lean_value(colmap, shares, eqbuf, data, x_lo, w, f); + w_lo = ext3::add(w_lo, ext3::mul(c, w)); + f_lo = ext3::add(f_lo, ext3::mul_base(c, f)); + whir_lean_value(colmap, shares, eqbuf, data, x_lo | half, w, f); + w_hi = ext3::add(w_hi, ext3::mul(c, w)); + f_hi = ext3::add(f_hi, ext3::mul_base(c, f)); + } + Fe3 w_d = ext3::sub(w_hi, w_lo), f_d = ext3::sub(f_hi, f_lo); + for (uint32_t ti = 0; ti < num_t; ++ti) { + Fe3 t = ext3::make(d_t[ti * 3], d_t[ti * 3 + 1], d_t[ti * 3 + 2]); + Fe3 w_t = ext3::add(w_lo, ext3::mul(t, w_d)); + Fe3 f_t = ext3::add(f_lo, ext3::mul(t, f_d)); + acc[ti] = ext3::add(acc[ti], ext3::mul(w_t, f_t)); + } +} + +// Round `s`'s evaluations at the `num_t` nodes, one partial per (node, block) in +// `sum_partials_ext3`'s layout. Grid-stride over the `2^(n−s)` rest positions. +extern "C" __global__ void whir_lean_round(const uint16_t *__restrict__ colmap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, uint32_t n, + uint32_t s, const uint64_t *__restrict__ eqc, + const uint64_t *__restrict__ d_t, uint32_t num_t, + uint64_t *__restrict__ d_partials) { + Fe3 acc[WHIR_LEAN_MAX_T]; + for (uint32_t ti = 0; ti < num_t; ++ti) acc[ti] = ext3::zero(); + uint64_t rests = 1ull << (n - s); + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t rest = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; rest < rests; + rest += stride) { + whir_lean_round_at(colmap, shares, eqbuf, data, n, s, eqc, d_t, num_t, rest, acc); + } + extern __shared__ uint64_t shared[]; + for (uint32_t ti = 0; ti < num_t; ++ti) { + shared[threadIdx.x * 3 + 0] = acc[ti].a; + shared[threadIdx.x * 3 + 1] = acc[ti].b; + shared[threadIdx.x * 3 + 2] = acc[ti].c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 x = ext3::make(shared[threadIdx.x * 3], shared[threadIdx.x * 3 + 1], + shared[threadIdx.x * 3 + 2]); + Fe3 y = ext3::make(shared[(threadIdx.x + width) * 3], + shared[(threadIdx.x + width) * 3 + 1], + shared[(threadIdx.x + width) * 3 + 2]); + Fe3 sum = ext3::add(x, y); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t at = ((uint64_t)ti * gridDim.x + blockIdx.x) * 3; + d_partials[at + 0] = shared[0]; + d_partials[at + 1] = shared[1]; + d_partials[at + 2] = shared[2]; + } + __syncthreads(); + } +} + +// The factors bound at the `bound` challenges so far, materialised: +// `out[y] = Σ_b eqfull[b] · factor(b ‖ y)` over the `2^bound` prefixes, for the +// weight and the (lifted) message, `2^(n − bound)` extension values each. +__device__ void whir_lean_materialize_at(const uint16_t *__restrict__ colmap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, uint32_t n, + uint32_t bound, const uint64_t *__restrict__ eqfull, + uint64_t y, Fe3 &w_out, Fe3 &f_out) { + uint32_t below = n - bound; + Fe3 w_acc = ext3::zero(), f_acc = ext3::zero(); + uint64_t prefixes = 1ull << bound; + for (uint64_t p = 0; p < prefixes; ++p) { + Fe3 c = ext3::make(eqfull[p * 3], eqfull[p * 3 + 1], eqfull[p * 3 + 2]); + Fe3 w; + uint64_t f; + whir_lean_value(colmap, shares, eqbuf, data, (p << below) | y, w, f); + w_acc = ext3::add(w_acc, ext3::mul(c, w)); + f_acc = ext3::add(f_acc, ext3::mul_base(c, f)); + } + w_out = w_acc; + f_out = f_acc; +} + +extern "C" __global__ void whir_lean_materialize(const uint16_t *__restrict__ colmap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, uint32_t n, + uint32_t bound, + const uint64_t *__restrict__ eqfull, + uint64_t *__restrict__ out_w, + uint64_t *__restrict__ out_f) { + uint64_t len = 1ull << (n - bound); + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t y = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; y < len; y += stride) { + Fe3 w, f; + whir_lean_materialize_at(colmap, shares, eqbuf, data, n, bound, eqfull, y, w, f); + out_w[y * 3] = w.a; + out_w[y * 3 + 1] = w.b; + out_w[y * 3 + 2] = w.c; + out_f[y * 3] = f.a; + out_f[y * 3 + 1] = f.b; + out_f[y * 3 + 2] = f.c; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// THE LEAN ROUNDS FROM ONE PASS OVER THE MESSAGE: THE TILE FORM. +// +// `whir_lean_round` walks every stack position in every round and rebuilds the +// weight there. The walk is not needed more than once. Cut the stack into TILES +// of `2^u` positions, `u` no more than the smallest column's variables: columns +// sit at multiples of their own size, so a tile lies inside one column or one +// gap, and inside a column +// +// w(T, j) = ω(T) · eq(λ, j), ω(T) = scale · eq(z_hi, T − first tile), +// +// with `λ` the last `u` coordinates of the column's point and `z_hi` the rest. +// Columns with the same `λ` are one CLASS, named by where its `eq(λ, ·)` table +// sits in `eqbuf` (the shares' `lo_at`, which the host gives once per class). +// +// A COLUMN OF TILES is the `2^L` tiles that differ only in the `L` top bits +// the lean rounds bind. Number the classes met in it `κ = 0, 1, …` (its +// SLOTS, at most `2^L`) and define, over the tiles, +// +// Ω_κ(T) = ω(T) if tile T is of slot κ, else 0 +// Φ_κ(T) = Σ_j eq(λ_κ, j) · f(T, j) +// +// Then the first `L` round polynomials of the opening are the first `L` round +// polynomials of the sumcheck of `Σ_κ Ω_κ·Φ_κ` over the tiles, at the same +// challenges: bound in its first `s` variables the weight at `(R, j)` is +// `Σ_κ Ω̃_κ(R)·eq(λ_κ, j)` — the slot numbering depends only on the bits of `R` +// below the lean rounds — and `Σ_j eq(λ_κ, j)·f̃(R, j) = Φ̃_κ(R)`. +// +// So `Φ` is one pass over the message, three products a position and class, +// `Ω` one product a tile, and every round after that reads `2^(n−u)` values a +// slot instead of `2^n` positions. Bound at the rounds' challenges, the weight +// is `Σ_κ Ω_κ^bound(R)·eq(λ_κ, j)` and the message is what +// `whir_lean_materialize` writes: `whir_tile_materialize` writes both from the +// tile form, with no position map. +// +// ★ EQUAL, not raw-identical, as the share form above is: the same field +// elements by another sequence of operations. The bound message alone is +// written by the share form's own sequence, so that table is raw-identical. +// +// The share rows are the share form's with `lo_bits = u` in every row. The +// tile map is `whir_lean_colmap` run over tiles (starts and ends in tiles). +// ───────────────────────────────────────────────────────────────────────────── + +#define WHIR_TILE_MAX_SLOTS 8 +#define WHIR_TILE_NO_SLOT 0xFFFFFFFFFFFFFFFFull + +__device__ __forceinline__ Fe3 whir_load3(const uint64_t *__restrict__ at) { + return ext3::make(at[0], at[1], at[2]); +} + +__device__ __forceinline__ void whir_store3(uint64_t *__restrict__ at, const Fe3 &v) { + at[0] = v.a; + at[1] = v.b; + at[2] = v.c; +} + +// One column of tiles: its slots — `slot_lo[col·slots + κ]`, the class of slot +// `κ` or `NO_SLOT` — and `Ω` for its `2^lean` tiles, every slot written. `nu` +// is the tiles' variable count, `n − u`. The host promises `slots` is at least +// the classes any column meets. +__device__ void whir_tile_plan_at(const uint16_t *__restrict__ tilemap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, uint32_t nu, uint32_t lean, + uint32_t u, uint32_t slots, uint64_t col, + uint64_t *__restrict__ slot_lo, uint64_t *__restrict__ omega) { + uint64_t cols = 1ull << (nu - lean); + uint64_t tiles = 1ull << nu; + uint64_t found[WHIR_TILE_MAX_SLOTS]; + uint32_t used = 0; + for (uint64_t q = 0; q < (1ull << lean); ++q) { + uint64_t tile = q * cols + col; + uint16_t id = tilemap[tile]; + uint32_t slot = slots; + Fe3 weight = ext3::zero(); + if (id != WHIR_LEAN_NONE) { + const uint64_t *share = shares + (uint64_t)id * WHIR_LEAN_SHARE_WORDS; + uint64_t lo_at = share[4]; + slot = 0; + while (slot < used && found[slot] != lo_at) ++slot; + if (slot == used && used < slots) { + found[used] = lo_at; + ++used; + } + uint64_t hi_at = (share[3] + (tile - (share[0] >> u))) * 3; + weight = ext3::mul(ext3::make(share[5], share[6], share[7]), whir_load3(eqbuf + hi_at)); + } + for (uint32_t s = 0; s < slots; ++s) { + whir_store3(omega + ((uint64_t)s * tiles + tile) * 3, + s == slot ? weight : ext3::zero()); + } + } + for (uint32_t s = 0; s < slots; ++s) { + slot_lo[col * slots + s] = s < used ? found[s] : WHIR_TILE_NO_SLOT; + } +} + +extern "C" __global__ void whir_tile_plan(const uint16_t *__restrict__ tilemap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, uint32_t nu, + uint32_t lean, uint32_t u, uint32_t slots, + uint64_t *__restrict__ slot_lo, + uint64_t *__restrict__ omega) { + uint64_t cols = 1ull << (nu - lean); + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t col = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; col < cols; + col += stride) { + whir_tile_plan_at(tilemap, shares, eqbuf, nu, lean, u, slots, col, slot_lo, omega); + } +} + +// `Φ_κ(tile)` for every slot of the tile's column, over the cells `first, +// first + step, …` of the tile: all of it at `(0, 1)`, one lane's part of it +// at `(lane, lanes)`. A tile no column covers, and a slot its column does not +// use, are zero. +__device__ void whir_tile_contract_at(const uint16_t *__restrict__ tilemap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, + const uint64_t *__restrict__ slot_lo, uint32_t nu, + uint32_t lean, uint32_t u, uint32_t slots, uint64_t tile, + uint64_t first, uint64_t step, Fe3 *acc) { + for (uint32_t s = 0; s < slots; ++s) acc[s] = ext3::zero(); + uint16_t id = tilemap[tile]; + if (id == WHIR_LEAN_NONE) return; + const uint64_t *share = shares + (uint64_t)id * WHIR_LEAN_SHARE_WORDS; + const uint64_t *from = data + share[1] + ((tile - (share[0] >> u)) << u); + const uint64_t *lo_of = slot_lo + (tile & ((1ull << (nu - lean)) - 1)) * slots; + // Slots fill from zero, so the used ones are a prefix. + const uint64_t *lo[WHIR_TILE_MAX_SLOTS]; + uint32_t used = 0; + while (used < slots && lo_of[used] != WHIR_TILE_NO_SLOT) { + lo[used] = eqbuf + lo_of[used] * 3; + ++used; + } + uint64_t cells = 1ull << u; + for (uint64_t j = first; j < cells; j += step) { + uint64_t f = from[j]; + for (uint32_t s = 0; s < used; ++s) { + acc[s] = ext3::add(acc[s], ext3::mul_base(whir_load3(lo[s] + j * 3), f)); + } + } +} + +// `Φ`, one thread a tile. +extern "C" __global__ void whir_tile_contract(const uint16_t *__restrict__ tilemap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, + const uint64_t *__restrict__ slot_lo, uint32_t nu, + uint32_t lean, uint32_t u, uint32_t slots, + uint64_t *__restrict__ phi) { + uint64_t tiles = 1ull << nu; + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t tile = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; tile < tiles; + tile += stride) { + Fe3 acc[WHIR_TILE_MAX_SLOTS]; + whir_tile_contract_at(tilemap, shares, eqbuf, data, slot_lo, nu, lean, u, slots, tile, 0, + 1, acc); + for (uint32_t s = 0; s < slots; ++s) { + whir_store3(phi + ((uint64_t)s * tiles + tile) * 3, acc[s]); + } + } +} + +// The block's threads' values summed into thread 0's `out`. Every thread of +// the block calls it, with its own `value`. +__device__ __forceinline__ void whir_tile_block_sum(const Fe3 &value, Fe3 &out) { + extern __shared__ uint64_t shared[]; + whir_store3(shared + threadIdx.x * 3, value); + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 sum = ext3::add(whir_load3(shared + threadIdx.x * 3), + whir_load3(shared + (threadIdx.x + width) * 3)); + whir_store3(shared + threadIdx.x * 3, sum); + } + __syncthreads(); + } + if (threadIdx.x == 0) out = whir_load3(shared); + __syncthreads(); +} + +// `Φ`, one block a tile: the block's threads take the tile's cells in turn, so +// a warp reads consecutive words, and their parts are summed in the block. +// The same values as `whir_tile_contract`. The block is a power of two. +extern "C" __global__ void whir_tile_contract_wide(const uint16_t *__restrict__ tilemap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, + const uint64_t *__restrict__ slot_lo, + uint32_t nu, uint32_t lean, uint32_t u, + uint32_t slots, uint64_t *__restrict__ phi) { + uint64_t tiles = 1ull << nu; + for (uint64_t tile = blockIdx.x; tile < tiles; tile += gridDim.x) { + Fe3 acc[WHIR_TILE_MAX_SLOTS]; + whir_tile_contract_at(tilemap, shares, eqbuf, data, slot_lo, nu, lean, u, slots, tile, + threadIdx.x, blockDim.x, acc); + for (uint32_t s = 0; s < slots; ++s) { + Fe3 sum = ext3::zero(); + whir_tile_block_sum(acc[s], sum); + if (threadIdx.x == 0) { + whir_store3(phi + ((uint64_t)s * tiles + tile) * 3, sum); + } + } + } +} + +// One pair's share of a round over the slot tables as they stand — `2·half` +// live values at the front of each, `stride` values apart: the product rule at +// each node, summed over the slots. +__device__ void whir_tile_round_at(const uint64_t *__restrict__ omega, + const uint64_t *__restrict__ phi, uint64_t stride, + uint32_t slots, uint64_t half, + const uint64_t *__restrict__ d_t, uint32_t num_t, uint64_t i, + Fe3 *acc) { + for (uint32_t s = 0; s < slots; ++s) { + const uint64_t *w = omega + (uint64_t)s * stride * 3; + const uint64_t *f = phi + (uint64_t)s * stride * 3; + Fe3 w_lo = whir_load3(w + i * 3), w_hi = whir_load3(w + (i + half) * 3); + Fe3 f_lo = whir_load3(f + i * 3), f_hi = whir_load3(f + (i + half) * 3); + Fe3 w_d = ext3::sub(w_hi, w_lo), f_d = ext3::sub(f_hi, f_lo); + for (uint32_t ti = 0; ti < num_t; ++ti) { + Fe3 t = whir_load3(d_t + ti * 3); + Fe3 w_t = ext3::add(w_lo, ext3::mul(t, w_d)); + Fe3 f_t = ext3::add(f_lo, ext3::mul(t, f_d)); + acc[ti] = ext3::add(acc[ti], ext3::mul(w_t, f_t)); + } + } +} + +// A round's evaluations at the `num_t` nodes, one partial per (node, block) in +// `sum_partials_ext3`'s layout, as `whir_lean_round` leaves them. +extern "C" __global__ void whir_tile_round(const uint64_t *__restrict__ omega, + const uint64_t *__restrict__ phi, uint64_t stride, + uint32_t slots, uint64_t half, + const uint64_t *__restrict__ d_t, uint32_t num_t, + uint64_t *__restrict__ d_partials) { + Fe3 acc[WHIR_LEAN_MAX_T]; + for (uint32_t ti = 0; ti < num_t; ++ti) acc[ti] = ext3::zero(); + uint64_t step = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; i < half; i += step) { + whir_tile_round_at(omega, phi, stride, slots, half, d_t, num_t, i, acc); + } + for (uint32_t ti = 0; ti < num_t; ++ti) { + Fe3 sum = ext3::zero(); + whir_tile_block_sum(acc[ti], sum); + if (threadIdx.x == 0) { + whir_store3(d_partials + ((uint64_t)ti * gridDim.x + blockIdx.x) * 3, sum); + } + } +} + +// `x[i] ← x[i] + r·(x[i + half] − x[i])`: one table bound at a round's +// challenge, in place, the order `Mle::fix_first_variable` binds in. +__device__ __forceinline__ void whir_tile_fold_at(uint64_t *__restrict__ x, uint64_t half, + uint64_t i, const Fe3 &r) { + Fe3 lo = whir_load3(x + i * 3), hi = whir_load3(x + (i + half) * 3); + whir_store3(x + i * 3, ext3::add(lo, ext3::mul(r, ext3::sub(hi, lo)))); +} + +// Every slot table of `Ω` and of `Φ` bound at `r`. +extern "C" __global__ void whir_tile_fold(uint64_t *__restrict__ omega, uint64_t *__restrict__ phi, + uint64_t stride, uint32_t slots, uint64_t half, + const uint64_t *__restrict__ r) { + Fe3 challenge = whir_load3(r); + uint64_t tasks = 2ull * slots * half; + uint64_t step = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < tasks; + task += step) { + uint64_t table = task / half; + uint64_t i = task - table * half; + uint64_t *x = table < slots ? omega + table * stride * 3 : phi + (table - slots) * stride * 3; + whir_tile_fold_at(x, half, i, challenge); + } +} + +// The factors at the `bound` challenges so far, from the tile form, at +// `y < 2^(n − bound)`: what `whir_lean_materialize_at` writes there, without a +// position map. The weight is `Σ_κ Ω_κ(t)·eq(λ_κ, j)` over the slots of the +// column of tile `t = y >> u`, `j` the cell in it, with `Ω` bound at the same +// challenges: EQUAL to the share form's weight. The message is the share +// form's own sequence, `Σ_p eqfull[p]·f(p ‖ y)` with a gap's cells zero, each +// cell found through the tile map: RAW-identical to it. +__device__ void whir_tile_materialize_at(const uint16_t *__restrict__ tilemap, + const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, + const uint64_t *__restrict__ data, + const uint64_t *__restrict__ slot_lo, + const uint64_t *__restrict__ omega, uint32_t n, + uint32_t lean, uint32_t u, uint32_t slots, + uint32_t bound, const uint64_t *__restrict__ eqfull, + uint64_t y, Fe3 &w_out, Fe3 &f_out) { + uint32_t nu = n - u; + uint64_t tiles = 1ull << nu; + uint64_t tile = y >> u, j = y & ((1ull << u) - 1); + uint64_t column = tile & ((1ull << (nu - lean)) - 1); // below the lean rounds' bits + const uint64_t *lo_of = slot_lo + column * slots; + Fe3 w = ext3::zero(); + for (uint32_t s = 0; s < slots && lo_of[s] != WHIR_TILE_NO_SLOT; ++s) { + w = ext3::add(w, ext3::mul(whir_load3(omega + ((uint64_t)s * tiles + tile) * 3), + whir_load3(eqbuf + (lo_of[s] + j) * 3))); + } + uint32_t below = n - bound; + Fe3 f_acc = ext3::zero(); + uint64_t prefixes = 1ull << bound; + for (uint64_t p = 0; p < prefixes; ++p) { + Fe3 c = whir_load3(eqfull + p * 3); + uint64_t x = (p << below) | y; + uint16_t id = tilemap[x >> u]; + uint64_t f = 0; + if (id != WHIR_LEAN_NONE) { + const uint64_t *share = shares + (uint64_t)id * WHIR_LEAN_SHARE_WORDS; + f = data[share[1] + (x - share[0])]; + } + f_acc = ext3::add(f_acc, ext3::mul_base(c, f)); + } + w_out = w; + f_out = f_acc; +} + +extern "C" __global__ void whir_tile_materialize( + const uint16_t *__restrict__ tilemap, const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, const uint64_t *__restrict__ data, + const uint64_t *__restrict__ slot_lo, const uint64_t *__restrict__ omega, uint32_t n, + uint32_t lean, uint32_t u, uint32_t slots, uint32_t bound, + const uint64_t *__restrict__ eqfull, uint64_t *__restrict__ out_w, + uint64_t *__restrict__ out_f) { + uint64_t len = 1ull << (n - bound); + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t y = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; y < len; y += stride) { + Fe3 w, f; + whir_tile_materialize_at(tilemap, shares, eqbuf, data, slot_lo, omega, n, lean, u, slots, + bound, eqfull, y, w, f); + whir_store3(out_w + y * 3, w); + whir_store3(out_f + y * 3, f); + } +} + +// The same values against tables already written — the share form's, when a +// checking opening writes them — as canonical limbs: one is added to +// `*differing` for every position where either factor differs. +extern "C" __global__ void whir_tile_materialize_check( + const uint16_t *__restrict__ tilemap, const uint64_t *__restrict__ shares, + const uint64_t *__restrict__ eqbuf, const uint64_t *__restrict__ data, + const uint64_t *__restrict__ slot_lo, const uint64_t *__restrict__ omega, uint32_t n, + uint32_t lean, uint32_t u, uint32_t slots, uint32_t bound, + const uint64_t *__restrict__ eqfull, const uint64_t *__restrict__ table_w, + const uint64_t *__restrict__ table_f, unsigned long long *__restrict__ differing) { + uint64_t len = 1ull << (n - bound); + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + for (uint64_t y = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; y < len; y += stride) { + Fe3 w, f; + whir_tile_materialize_at(tilemap, shares, eqbuf, data, slot_lo, omega, n, lean, u, slots, + bound, eqfull, y, w, f); + Fe3 mw = ext3::canonical(w), mf = ext3::canonical(f); + Fe3 tw = ext3::canonical(whir_load3(table_w + y * 3)); + Fe3 tf = ext3::canonical(whir_load3(table_f + y * 3)); + if (mw.a != tw.a || mw.b != tw.b || mw.c != tw.c || mf.a != tf.a || mf.b != tf.b || + mf.c != tf.c) { + atomicAdd(differing, 1ull); + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// A chain's successor from its bound message (D-SPEED lever 5a). +// +// The fold of a committed codeword is the encoding of the message its rounds +// bound, on the folded domain — so the successor can be encoded from the +// `2^(n−k)` message instead of folded from the `2^(n+blowup)` codeword. The +// encoding is the base field's (Möbius, the lift's spread, the NTT), and it is +// linear over the base field, so an ext3 message is encoded one limb at a +// time: each limb comes out of the interleaved message as a plane, is encoded +// as a base-field codeword, and goes back into its place in the interleaved +// successor. +// ───────────────────────────────────────────────────────────────────────────── + +// `plane[i] = values[3·i + limb]` for `i < n`. +extern "C" __global__ void ext3_limb_gather(const uint64_t *__restrict__ values, uint64_t n, + uint32_t limb, uint64_t *__restrict__ plane) { + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + plane[i] = values[i * 3 + limb]; +} + +// `values[3·i + limb] = plane[i]` for `i < n`. +extern "C" __global__ void ext3_limb_scatter(const uint64_t *__restrict__ plane, uint64_t n, + uint32_t limb, uint64_t *__restrict__ values) { + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + values[i * 3 + limb] = plane[i]; +} diff --git a/crypto/math-cuda/src/argue_fused.rs b/crypto/math-cuda/src/argue_fused.rs new file mode 100644 index 000000000..834d5c1d2 --- /dev/null +++ b/crypto/math-cuda/src/argue_fused.rs @@ -0,0 +1,650 @@ +//! D-ARGUE stage 1 on the card: a table's zerocheck rounds the fused way +//! (`thoughts/zf/gap2/fix2/D-ARGUE.md` §2.3–2.6, §4.2), over the factors the +//! GKR input layer already lifted. +//! +//! The kernels are in `kernels/sumcheck.cu` (`zc_*`); the host that drives them +//! — the program with its `ACC` steps, the bus column's coefficients, the +//! round messages out of the sums — is `multilinear::gpu_fused`, and the host +//! reference both are checked against is `multilinear::fused`. +//! +//! The lifted factors are only read: rounds 0 and 1 read their base limbs, and +//! both folds write a quarter-size copy the later rounds fold in place. So +//! today's rounds can still run over them afterwards, which is what the +//! cross-check does. + +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, LaunchConfig, PushKernelArg}; +use std::sync::Arc; + +use crate::Result; +use crate::device::{DeviceReservation, alloc_or_trim, alloc_zeros_or_trim, backend, htod_or_trim}; +use crate::sumcheck::{FactorView, eq_table_ext3, launch_shape}; + +const BLOCK_DIM: u32 = 256; +const MAX_GRID: u32 = 4096; + +/// Scratch ceiling for the per-thread slot files, as the round kernel's. +const SLOT_BUDGET_BYTES: u64 = 512 * 1024 * 1024; +const MAX_THREADS: u64 = 1 << 20; + +/// The constraint part, lowered with its `ACC` steps: `nodes` two u64 a step; +/// its constants once as base values (the grid pass) and once as ext3 (the +/// later rounds); `betas` three u64 a root; `num_slots` the live set. +#[derive(Clone, Copy, Debug)] +pub struct FusedProgram<'a> { + pub nodes: &'a [u64], + pub base_consts: &'a [u64], + pub ext_consts: &'a [u64], + pub betas: &'a [u64], + pub num_slots: usize, +} + +/// The bus column `L = a₀ + Σ a_k·f_k`: each term's factor slot, its ext3 +/// coefficient (three u64), and `a₀`. +#[derive(Clone, Copy, Debug)] +pub struct FusedBus<'a> { + pub slots: &'a [u32], + pub coeffs: &'a [u64], + pub constant: [u64; 3], +} + +/// One table's fused zerocheck on the card. +pub struct FusedZerocheck<'f> { + stream: Arc, + source: FactorView<'f>, + width: usize, + /// The cube the lifted factors span. + rows: usize, + /// Cube indices left in the folded buffer (0 before `fold2`). + len: usize, + nodes: CudaSlice, + num_nodes: u64, + base_consts: CudaSlice, + ext_consts: CudaSlice, + betas: CudaSlice, + num_slots: usize, + term_slots: CudaSlice, + term_coeffs: CudaSlice, + num_terms: u64, + constant: CudaSlice, + eq_r: CudaSlice, + eq_rho: CudaSlice, + /// `width` factor slabs and then `L`'s, a quarter of the cube each. + folded: CudaSlice, + folded_ptrs: CudaSlice, + /// The fold's pointer table — every folded factor, then `L` — built once: + /// the addresses never move, and a table built per fold would have to be + /// waited for before it could be dropped. + fold_ptrs: CudaSlice, + slots: CudaSlice, + partials: CudaSlice, + sums: CudaSlice, + points: CudaSlice, + node_ints: CudaSlice, + scratch: CudaSlice, + violation: CudaSlice, + _room: DeviceReservation, +} + +/// Threads a slot file of `bytes_per_thread` affords under the budget. +fn ceiling(bytes_per_thread: u64) -> u64 { + (SLOT_BUDGET_BYTES / bytes_per_thread.max(1)).clamp(32, MAX_THREADS) +} + +/// Threads a fused launch takes in all: past this the card is full (170 SMs of +/// a few blocks each) and a wider launch only grows the slot file, which is +/// VRAM beside the argue's and DRAM traffic. +const MAX_LAUNCH_THREADS: u64 = 1 << 17; + +/// Grid points and nodes a launch may own: one block row each. +pub const MAX_ROWS: usize = 32; + +/// Threads a launch of `work` indices over `rows` block rows takes, a slot +/// file of `words` u64 a thread, at most `cap_threads` in all: `(grid.x, block)`. +fn fit(work: u64, rows: usize, words: u64, cap_threads: u64) -> Option<(u32, u32)> { + if rows == 0 { + return None; + } + let per_row = (ceiling(words * 8) / rows as u64) + .min(cap_threads / rows as u64) + .min(MAX_LAUNCH_THREADS / rows as u64); + if per_row < 32 { + return None; + } + let (mut grid, block) = launch_shape(per_row, work); + grid = grid.min(MAX_GRID); + while grid > 1 && grid as u64 * block as u64 > per_row { + grid -= 1; + } + Some((grid, block)) +} + +/// The slot file's u64s: the grid pass's first launch (base slots) or the +/// first later round's (ext3), whichever is wider. +fn slot_words(rows: usize, num_slots: usize, grid_rows: usize, gruen_rows: usize) -> usize { + let s = num_slots.max(1) as u64; + let q = (rows / 4) as u64; + let words = |work: u64, rows: usize, per: u64| -> u64 { + fit(work, rows, per, u64::MAX).map_or(0, |(g, b)| g as u64 * b as u64 * rows as u64 * per) + }; + words(q, grid_rows, s).max(words((q / 2).max(1), gruen_rows, 3 * s)) as usize +} + +impl<'f> FusedZerocheck<'f> { + /// Bytes the session holds beside the lifted factors: the folded copy and + /// `L` at a quarter, the two weights at a quarter, the slot file its first + /// launches need, and the partials. + pub fn device_bytes( + width: usize, + rows: usize, + num_slots: usize, + grid_rows: usize, + gruen_rows: usize, + ) -> u64 { + let q = (rows / 4) as u64; + let folded = (width as u64 + 1) * q * 24; + let weights = 2 * q * 24; + let slots = slot_words(rows, num_slots, grid_rows, gruen_rows) as u64 * 8; + let partials = (MAX_ROWS as u64) * (MAX_GRID as u64) * 24; + folded + weights + slots + partials + (1 << 20) + } + + /// Sets the session up, or `Ok(None)` when the card's budget refuses it — + /// before anything is absorbed, so the caller runs today's rounds. + /// + /// `r_tail` and `rho_tail` are `r[2..]` and `ρ[2..]` (three u64 a + /// coordinate): the grid pass's weights. + pub fn new( + source: FactorView<'f>, + program: FusedProgram<'_>, + bus: FusedBus<'_>, + r_tail: &[u64], + rho_tail: &[u64], + grid_rows: usize, + gruen_rows: usize, + ) -> Result> { + let rows = source.rows; + let width = source.width; + assert!( + rows >= 8 && rows.is_power_of_two(), + "the grid needs four rows a group" + ); + assert_eq!( + 1usize << (r_tail.len() / 3), + rows / 4, + "r[2..] spans a quarter" + ); + assert_eq!(r_tail.len(), rho_tail.len(), "ρ[2..] spans a quarter"); + assert_eq!( + bus.slots.len() * 3, + bus.coeffs.len(), + "three u64 a coefficient" + ); + let be = backend()?; + assert!( + grid_rows <= MAX_ROWS && gruen_rows <= MAX_ROWS, + "rows per launch" + ); + let words = slot_words(rows, program.num_slots, grid_rows, gruen_rows); + if words == 0 { + return Ok(None); + } + let Some(room) = be.reserve(Self::device_bytes( + width, + rows, + program.num_slots, + grid_rows, + gruen_rows, + )) else { + return Ok(None); + }; + let stream = source.stream.clone(); + let q = rows / 4; + let nonempty = |v: &[u64]| -> Vec { if v.is_empty() { vec![0] } else { v.to_vec() } }; + let nodes = htod_or_trim(&stream, &nonempty(program.nodes))?; + let base_consts = htod_or_trim(&stream, &nonempty(program.base_consts))?; + let ext_consts = htod_or_trim(&stream, &nonempty(program.ext_consts))?; + let betas = htod_or_trim(&stream, &nonempty(program.betas))?; + let term_slots = htod_or_trim( + &stream, + if bus.slots.is_empty() { + &[0u32][..] + } else { + bus.slots + }, + )?; + let term_coeffs = htod_or_trim(&stream, &nonempty(bus.coeffs))?; + let constant = htod_or_trim(&stream, &bus.constant[..])?; + let eq_r = eq_table_ext3(&stream, r_tail, q)?; + let eq_rho = eq_table_ext3(&stream, rho_tail, q)?; + // SAFETY: every slab is written by `fold2` and `zc_bus_column` before any + // round reads it. + let folded = unsafe { alloc_or_trim::(&stream, (width + 1) * q * 3) }?; + let addresses: Vec = { + let (base, _record) = folded.device_ptr(&stream); + (0..=width).map(|k| base + (k * q * 24) as u64).collect() + }; + let folded_ptrs = htod_or_trim(&stream, &addresses[..width])?; + let fold_ptrs = htod_or_trim(&stream, &addresses)?; + // SAFETY: a thread writes a slot before it reads it. + let slots = unsafe { alloc_or_trim::(&stream, words) }?; + let partials = unsafe { alloc_or_trim::(&stream, MAX_ROWS * MAX_GRID as usize * 3) }?; + let sums = alloc_zeros_or_trim::(&stream, MAX_ROWS * 3)?; + let points = alloc_zeros_or_trim::(&stream, MAX_ROWS)?; + let node_ints = alloc_zeros_or_trim::(&stream, MAX_ROWS)?; + let scratch = alloc_zeros_or_trim::(&stream, 12)?; + let violation = alloc_zeros_or_trim::(&stream, 1)?; + Ok(Some(Self { + stream, + source, + width, + rows, + len: 0, + nodes, + num_nodes: (program.nodes.len() / 2) as u64, + base_consts, + ext_consts, + betas, + num_slots: program.num_slots.max(1), + term_slots, + term_coeffs, + num_terms: bus.slots.len() as u64, + constant, + eq_r, + eq_rho, + folded, + folded_ptrs, + fold_ptrs, + slots, + partials, + sums, + points, + node_ints, + scratch, + violation, + _room: room, + })) + } + + /// A launch over `work` indices with `rows` block rows and `words` slot + /// u64s a thread, inside the slot file this session holds. + fn shape(&self, work: u64, rows: usize, words: u64) -> Result<(u32, u32)> { + fit(work, rows, words, self.slots.len() as u64 / words.max(1)).ok_or( + cudarc::driver::DriverError(cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY), + ) + } + + /// Sums `rows × grid` partials into `rows` ext3 values and reads them back. + fn reduce(&mut self, rows: usize, grid: u32) -> Result> { + self.reduce_enqueue(rows, grid)?; + self.collect(rows) + } + + /// [`reduce`](Self::reduce)'s launch, not waited for. + fn reduce_enqueue(&mut self, rows: usize, grid: u32) -> Result<()> { + let be = backend()?; + crate::round_slot::note_second_launch(); + let reduce_block = 256u32; + unsafe { + self.stream + .launch_builder(&be.sum_partials_ext3) + .arg(&self.partials) + .arg(&(grid as u64)) + .arg(&mut self.sums) + .launch(LaunchConfig { + grid_dim: (rows as u32, 1, 1), + block_dim: (reduce_block, 1, 1), + shared_mem_bytes: reduce_block * 3 * 8, + })?; + } + Ok(()) + } + + /// The `rows` sums a queued launch left, read back once the stream is idle. + pub fn collect(&self, rows: usize) -> Result> { + let out = crate::readback::clone_dtoh(&self.stream, &self.sums.slice(0..rows * 3))?; + self.stream.synchronize()?; + Ok(out) + } + + /// Rounds 0 and 1's pass (S1-5): `T` at each grid point `(a, b)` of + /// `points` (three u64 each, in order), `U(b, c)` at `2b + c`, and the + /// first row whose corner is not zero, if a corner was on the list and one + /// was not. With `keep_corners` a corner is summed into `T` like any + /// point and checked for nothing — random columns, in a test. An empty + /// list — a table without roots, whose grid is all corners — launches no + /// grid pass: `T` is zero. + #[allow(clippy::type_complexity)] + pub fn grid( + &mut self, + points: &[(u32, u32)], + keep_corners: bool, + ) -> Result<(Vec, Vec, Option)> { + assert!(points.len() <= MAX_ROWS, "grid points per launch"); + let (t, violation) = if points.is_empty() { + (Vec::new(), None) + } else { + self.grid_pass(points, keep_corners)? + }; + let be = backend()?; + let q = (self.rows / 4) as u64; + let (grid, block) = ( + q.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32, + BLOCK_DIM, + ); + let factors = self.source.ptrs; + unsafe { + self.stream + .launch_builder(&be.zc_bus_u) + .arg(factors) + .arg(&q) + .arg(&self.term_slots) + .arg(&self.term_coeffs) + .arg(&self.num_terms) + .arg(&self.constant) + .arg(&self.eq_rho) + .arg(&mut self.partials) + .arg(&self.source.stride) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: block * 3 * 8, + })?; + } + let u = self.reduce(4, grid)?; + Ok((t, u, violation)) + } + + /// `zc_grid01` over a non-empty `points`: `T` at each, three u64 a point, + /// and the first row whose corner is not zero. + fn grid_pass( + &mut self, + points: &[(u32, u32)], + keep_corners: bool, + ) -> Result<(Vec, Option)> { + let be = backend()?; + let q = (self.rows / 4) as u64; + let packed: Vec = points + .iter() + .map(|&(a, b)| u64::from(a) | (u64::from(b) << 32)) + .collect(); + { + let mut head = self.points.slice_mut(0..packed.len()); + self.stream.memcpy_htod(&packed, &mut head)?; + } + self.stream.memcpy_htod(&[u64::MAX], &mut self.violation)?; + let (grid, block) = self.shape(q, points.len(), self.num_slots as u64)?; + let factors = self.source.ptrs; + unsafe { + self.stream + .launch_builder(&be.zc_grid01) + .arg(factors) + .arg(&q) + .arg(&self.nodes) + .arg(&self.num_nodes) + .arg(&self.base_consts) + .arg(&self.betas) + .arg(&self.points) + .arg(&self.eq_r) + .arg(&mut self.slots) + .arg(&mut self.partials) + .arg(&mut self.violation) + .arg(&u32::from(keep_corners)) + .arg(&self.source.stride) + .launch(LaunchConfig { + grid_dim: (grid, points.len() as u32, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: block * 3 * 8, + })?; + } + let t = self.reduce(points.len(), grid)?; + let violation = crate::readback::clone_dtoh(&self.stream, &self.violation)?; + self.stream.synchronize()?; + Ok((t, (violation[0] != u64::MAX).then_some(violation[0]))) + } + + /// Both folds at once (`w[2a + b] = eq₁(s₀, a)·eq₁(s₁, b)`, twelve u64), into + /// the quarter-size copy; then `L` there, and the weights halved to round + /// 2's `eq(·_{>2})`. + pub fn fold2(&mut self, w: &[u64; 12]) -> Result<()> { + assert_eq!(self.len, 0, "the double fold is the first"); + let be = backend()?; + let q = (self.rows / 4) as u64; + self.stream.memcpy_htod(&w[..], &mut self.scratch)?; + let width = self.width as u64; + let total = width * q; + let grid = total.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + let factors = self.source.ptrs; + unsafe { + self.stream + .launch_builder(&be.zc_fold2) + .arg(factors) + .arg(&q) + .arg(&width) + .arg(&self.scratch) + .arg(&mut self.folded) + .arg(&self.source.stride) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + self.len = q as usize; + self.bus_column()?; + self.halve_weights() + } + + /// `L` over the folded factors, into its slab. + fn bus_column(&mut self) -> Result<()> { + let be = backend()?; + let rows = self.len as u64; + let grid = rows.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + let q = self.rows / 4; + let mut l = self + .folded + .slice_mut(self.width * q * 3..(self.width + 1) * q * 3); + unsafe { + self.stream + .launch_builder(&be.zc_bus_column) + .arg(&self.folded_ptrs) + .arg(&rows) + .arg(&self.term_slots) + .arg(&self.term_coeffs) + .arg(&self.num_terms) + .arg(&self.constant) + .arg(&mut l) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + Ok(()) + } + + /// Both weights to the next round's `eq(·_{>j+1})`: their halves added. The + /// weights are always half the folded cube after this. + pub fn halve_weights(&mut self) -> Result<()> { + let be = backend()?; + let half = (self.len / 2) as u64; + let grid = half.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + for table in [&mut self.eq_r, &mut self.eq_rho] { + unsafe { + self.stream + .launch_builder(&be.zc_halve) + .arg(table) + .arg(&half) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + } + Ok(()) + } + + /// A later round (S1-4): `A` at each integer node of `nodes`, then `B(0)` + /// and `B(1)`, three u64 each. + pub fn round(&mut self, nodes: &[u32]) -> Result> { + let rows = self.round_enqueue(nodes)?; + self.collect(rows) + } + + /// [`round`](Self::round)'s launches, queued and not waited for; the + /// caller reads the sums with [`collect`](Self::collect) and the returned + /// row count. The batched argue queues every table's round before it + /// reads any back. + pub fn round_enqueue(&mut self, nodes: &[u32]) -> Result { + assert!(self.len >= 2, "a round needs a variable to bind"); + let rows = nodes.len() + 2; + assert!(rows <= MAX_ROWS, "nodes per launch"); + let be = backend()?; + let half = (self.len / 2) as u64; + { + let mut head = self.node_ints.slice_mut(0..nodes.len().max(1)); + let ints: Vec = if nodes.is_empty() { + vec![0] + } else { + nodes.to_vec() + }; + self.stream.memcpy_htod(&ints, &mut head)?; + } + let (grid, block) = self.shape(half, rows, 3 * self.num_slots as u64)?; + let num_a = nodes.len() as u32; + let q = self.rows / 4; + let l = self + .folded + .slice(self.width * q * 3..(self.width + 1) * q * 3); + unsafe { + self.stream + .launch_builder(&be.zc_round_gruen) + .arg(&self.folded_ptrs) + .arg(&half) + .arg(&self.nodes) + .arg(&self.num_nodes) + .arg(&self.ext_consts) + .arg(&self.betas) + .arg(&self.node_ints) + .arg(&num_a) + .arg(&self.eq_r) + .arg(&self.eq_rho) + .arg(&l) + .arg(&mut self.slots) + .arg(&mut self.partials) + .launch(LaunchConfig { + grid_dim: (grid, rows as u32, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: block * 3 * 8, + })?; + } + self.reduce_enqueue(rows, grid)?; + Ok(rows) + } + + /// Binds the round's variable to `s` (three u64) in every folded factor and + /// in `L`; the weights follow when `more` rounds are left on the card. + pub fn fold(&mut self, s: &[u64; 3], more: bool) -> Result<()> { + assert!(self.len >= 2, "a fold needs a variable to bind"); + let be = backend()?; + let half = (self.len / 2) as u64; + self.stream + .memcpy_htod(&s[..], &mut self.scratch.slice_mut(0..3))?; + // The factors and `L`, through the one pointer table of `width + 1`. + let width = self.width as u64 + 1; + let total = width * half; + let grid = total.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + unsafe { + self.stream + .launch_builder(&be.sumcheck_fold_ext3) + .arg(&mut self.fold_ptrs) + .arg(&half) + .arg(&width) + .arg(&self.scratch) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + self.len /= 2; + if more { + self.halve_weights()?; + } + Ok(()) + } + + /// Cube indices left in the folded copy. + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// What every factor has left — `width` of them, three u64 an element — + /// gathered on the card and read back in one copy. + pub fn values(&self) -> Result>> { + let be = backend()?; + let span = self.len * 3; + let total = self.width * span; + // SAFETY: the kernel writes every word of it, one thread per word. + let mut out = unsafe { alloc_or_trim::(&self.stream, total) }?; + let width = self.width as u64; + let cells = self.len as u64; + let grid = (total as u64) + .div_ceil(BLOCK_DIM as u64) + .clamp(1, MAX_GRID as u64) as u32; + unsafe { + self.stream + .launch_builder(&be.gather_factor_heads_ext3) + .arg(&self.folded_ptrs) + .arg(&width) + .arg(&cells) + .arg(&mut out) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + let flat = crate::readback::clone_dtoh(&self.stream, &out)?; + self.stream.synchronize()?; + Ok(flat.chunks_exact(span).map(<[u64]>::to_vec).collect()) + } + + /// `L`'s remaining values (a test's view). + pub fn bus_values(&self) -> Result> { + let q = self.rows / 4; + let at = self.width * q * 3; + let out = + crate::readback::clone_dtoh(&self.stream, &self.folded.slice(at..at + self.len * 3))?; + self.stream.synchronize()?; + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A table without roots has `d_C = 0`, so its grid `{0,1}²` is all + /// corners and, with the corners skipped, no point is left to launch — + /// FAST job 273's B arm, which divided by that empty row count. + #[test] + fn a_grid_of_corners_only_sizes_no_grid_launch() { + let words = slot_words(1 << 20, 1, 0, 3); + assert!(words > 0, "the later rounds still need their slot file"); + assert_eq!(fit(1 << 18, 0, 1, u64::MAX), None, "no rows, no launch"); + } + + /// S1-1's default, pinned: integer nodes unless the variable says `0`. + #[test] + fn integer_nodes_are_on_unless_the_knob_says_0() { + use crate::sumcheck::int_nodes_from; + assert!(int_nodes_from(None)); + assert!(int_nodes_from(Some("1"))); + assert!(!int_nodes_from(Some("0"))); + } +} diff --git a/crypto/math-cuda/src/argue_probe.rs b/crypto/math-cuda/src/argue_probe.rs new file mode 100644 index 000000000..a59473554 --- /dev/null +++ b/crypto/math-cuda/src/argue_probe.rs @@ -0,0 +1,130 @@ +//! ⛔ ROUND-3 ARGUE DISCRIMINATOR (diagnostic; on NO production decision path). +//! +//! The occupancy sweep put the RPX permutation at its arithmetic floor, so round +//! 3's only remaining stage is `argue` (the WHIR field-arithmetic argument: +//! sumcheck + gkr + columns, ~16.6s, per-table dominated by KECCAK_RND). STEP 1 +//! (the O5 device-fallback counter, read from the round-2 logs) established argue +//! does NOT host-fall-back at the tip — it is device-side and stable. This +//! instrument answers STEP 2: is argue's device work MEMORY-bound (a layout/reuse +//! lever exists) or COMPUTE-bound (near a floor, like the permutation)? +//! +//! ncu is unavailable on the box (ERR_NVGPUCTRPERM), so the memory roofline is +//! read by ARITHMETIC instead of a hardware counter: this records, per argue +//! surface, the DEVICE-path bytes RESERVED and the number of device operations, +//! at the same five `reserve` sites the fallback counter already guards +//! (sumcheck.rs, gkr.rs, columns.rs). The harness prints the totals; divided by +//! the known argue wall time they give an achieved HBM bandwidth: +//! near the card roofline (~1.7 TB/s on the 5090) ⇒ MEMORY-bound ⇒ the lever is +//! MLE layout/reuse (keep folds resident, fuse fold+eval, stop +//! re-materialising factors); +//! far below it ⇒ the resident data is reused from cache, argue is +//! COMPUTE-bound on the extension-field arithmetic ⇒ near a floor. +//! +//! Reserved bytes is the DEVICE working set per operation, so summed over the run +//! it is the HBM traffic to first order (a resident buffer re-read within an op +//! is L2/L1, not HBM). It is a robust, additive quantity — unlike a wall-timer at +//! these sites, which would double-count when one surface's reserved scope nests +//! another's (gkr's layer sumcheck inside a reserve). A finer per-surface TIME / +//! device-vs-host split is STEP 2b, added only if the byte roofline is borderline. + +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The three argue device surfaces, matching the crates the reserve sites live +/// in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Surface { + Sumcheck, + Gkr, + Columns, +} + +struct Counters { + /// Device-path bytes reserved (the working set promised to the card). + bytes: AtomicU64, + /// Device operations that got their reservation (i.e. ran on the device). + calls: AtomicU64, +} + +impl Counters { + const fn zero() -> Self { + Self { + bytes: AtomicU64::new(0), + calls: AtomicU64::new(0), + } + } +} + +static SUMCHECK: Counters = Counters::zero(); +static GKR: Counters = Counters::zero(); +static COLUMNS: Counters = Counters::zero(); + +fn counters(surface: Surface) -> &'static Counters { + match surface { + Surface::Sumcheck => &SUMCHECK, + Surface::Gkr => &GKR, + Surface::Columns => &COLUMNS, + } +} + +/// Record one device-path reservation on `surface`: the bytes it promised and +/// one operation. Called at each `reserve`-SUCCESS site, beside the +/// `note_device_fallback()` the else-branch already calls. +pub fn note_device(surface: Surface, bytes: u64) { + let c = counters(surface); + c.bytes.fetch_add(bytes, Ordering::Relaxed); + c.calls.fetch_add(1, Ordering::Relaxed); +} + +/// `(bytes, calls)` for one surface, whole-run scope. +pub fn surface_totals(surface: Surface) -> (u64, u64) { + let c = counters(surface); + ( + c.bytes.load(Ordering::Relaxed), + c.calls.load(Ordering::Relaxed), + ) +} + +// ── device-busy sizing (round-3 argue idle-fraction discriminator) ────────── +// +// The concurrency lever is soundness-dead (the whole argue is one sequential +// Fiat-Shamir chain — the transcript order IS the proof), so argue's ~2% util is +// mostly INHERENT per-round host-sync latency. This measures how much of the +// argue wall the round kernels are actually BUSY: the sizing timer records CUDA +// events around the sumcheck round kernels and reads the elapsed after the +// EXISTING per-round synchronize(), accumulating device-busy nanoseconds. idle = +// argue_wall − device_busy sizes the total gap; the recoverable-WITHOUT-a-rewrite +// fraction is only what transcript-independent prefetch can fill (the +// orchestration read sizes that), so a large idle here is NOT a large recoverable +// win by itself. + +static DEVICE_BUSY_NS: AtomicU64 = AtomicU64::new(0); + +/// Enabled only when `LAMBDA_VM_ARGUE_BUSY_PROBE` is set — read ONCE and cached, +/// so a production round() pays a single relaxed bool load and nothing else. The +/// event recording (and its one-time timing-event creation) happens only when on. +pub fn busy_probe_enabled() -> bool { + static EN: OnceLock = OnceLock::new(); + *EN.get_or_init(|| std::env::var_os("LAMBDA_VM_ARGUE_BUSY_PROBE").is_some()) +} + +/// Add device-busy nanoseconds measured across one round's kernels. +pub fn add_device_busy_ns(ns: u64) { + DEVICE_BUSY_NS.fetch_add(ns, Ordering::Relaxed); +} + +/// Device-busy nanoseconds accumulated over the run (round kernels only; fold and +/// setup are excluded, so idle = wall − this is a slight OVER-estimate). +pub fn device_busy_ns() -> u64 { + DEVICE_BUSY_NS.load(Ordering::Relaxed) +} + +/// Zero every surface — call before a run whose totals are to be read, exactly +/// as the fallback counter is reset. +pub fn reset() { + for c in [&SUMCHECK, &GKR, &COLUMNS] { + c.bytes.store(0, Ordering::Relaxed); + c.calls.store(0, Ordering::Relaxed); + } + DEVICE_BUSY_NS.store(0, Ordering::Relaxed); +} diff --git a/crypto/math-cuda/src/barycentric.rs b/crypto/math-cuda/src/barycentric.rs index d6df604ce..dfaa6852e 100644 --- a/crypto/math-cuda/src/barycentric.rs +++ b/crypto/math-cuda/src/barycentric.rs @@ -70,7 +70,7 @@ pub fn barycentric_base( .arg(&mut out_dev) .launch(cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; stream.synchronize()?; Ok(out) } @@ -127,7 +127,7 @@ pub fn barycentric_ext3( .arg(&mut out_dev) .launch(cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; stream.synchronize()?; Ok(out) } @@ -183,7 +183,7 @@ pub fn barycentric_base_on_device( .arg(&mut out_dev) .launch(cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; stream.synchronize()?; Ok(out) } @@ -197,6 +197,12 @@ pub fn barycentric_base_on_device( /// `compute_and_invert_denoms_ext3_dev`. `inv_offset_u64` is the start /// of this eval point's block (in u64s), so the kernel reads /// `inv_denoms_dev[inv_offset_u64 .. inv_offset_u64 + 3*n]`. +/// +/// The sums run on the row-chunked multi kernel at one point +/// ([`barycentric_base_chunked_with_dev_inv_denoms`], `num_cols × chunks` +/// blocks); under `LAMBDA_VM_DEEP_INV_LEGACY=1` ([`crate::deep_inv`]) on one +/// block per column ([`barycentric_base_one_block_with_dev_inv_denoms`]). The +/// same sums, added in a different order. pub fn barycentric_base_on_device_with_dev_inv_denoms( stream: &Arc, main_handle: &GpuLdeBase, @@ -208,6 +214,41 @@ pub fn barycentric_base_on_device_with_dev_inv_denoms( ) -> Result> { #[cfg(feature = "test-faults")] crate::faults::check_sticky(&crate::faults::FAULT_BARYCENTRIC_STICKY)?; + if crate::deep_inv::rowwise_enabled() { + crate::deep_inv::count_chunked_ood_sum(); + return barycentric_base_chunked_with_dev_inv_denoms( + stream, + main_handle, + row_stride, + coset_points_dev, + inv_denoms_dev, + inv_offset_u64, + n, + ); + } + barycentric_base_one_block_with_dev_inv_denoms( + stream, + main_handle, + row_stride, + coset_points_dev, + inv_denoms_dev, + inv_offset_u64, + n, + ) +} + +/// [`barycentric_base_on_device_with_dev_inv_denoms`]'s legacy path: one +/// block per column over all `n` rows. Public so the parity tests can run it +/// beside the chunked kernel whatever the process setting says. +pub fn barycentric_base_one_block_with_dev_inv_denoms( + stream: &Arc, + main_handle: &GpuLdeBase, + row_stride: usize, + coset_points_dev: &CudaSlice, + inv_denoms_dev: &CudaSlice, + inv_offset_u64: usize, + n: usize, +) -> Result> { main_handle.wait_ready_on(stream)?; assert!(coset_points_dev.len() >= n); let inv_end = inv_offset_u64 @@ -245,7 +286,7 @@ pub fn barycentric_base_on_device_with_dev_inv_denoms( .arg(&mut out_dev) .launch(cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(stream, &out_dev)?; stream.synchronize()?; Ok(out) } @@ -299,12 +340,14 @@ pub fn barycentric_ext3_on_device( .arg(&mut out_dev) .launch(cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; stream.synchronize()?; Ok(out) } -/// Ext3 counterpart of [`barycentric_base_on_device_with_dev_inv_denoms`]. +/// Ext3 counterpart of [`barycentric_base_on_device_with_dev_inv_denoms`], and +/// the one the R3 composition-parts OOD calls: 1–2 columns, which one block per +/// column would run as 1–2 blocks on the whole card. pub fn barycentric_ext3_on_device_with_dev_inv_denoms( stream: &Arc, aux_handle: &GpuLdeExt3, @@ -316,6 +359,39 @@ pub fn barycentric_ext3_on_device_with_dev_inv_denoms( ) -> Result> { #[cfg(feature = "test-faults")] crate::faults::check_sticky(&crate::faults::FAULT_BARYCENTRIC_STICKY)?; + if crate::deep_inv::rowwise_enabled() { + crate::deep_inv::count_chunked_ood_sum(); + return barycentric_ext3_chunked_with_dev_inv_denoms( + stream, + aux_handle, + row_stride, + coset_points_dev, + inv_denoms_dev, + inv_offset_u64, + n, + ); + } + barycentric_ext3_one_block_with_dev_inv_denoms( + stream, + aux_handle, + row_stride, + coset_points_dev, + inv_denoms_dev, + inv_offset_u64, + n, + ) +} + +/// Ext3 counterpart of [`barycentric_base_one_block_with_dev_inv_denoms`]. +pub fn barycentric_ext3_one_block_with_dev_inv_denoms( + stream: &Arc, + aux_handle: &GpuLdeExt3, + row_stride: usize, + coset_points_dev: &CudaSlice, + inv_denoms_dev: &CudaSlice, + inv_offset_u64: usize, + n: usize, +) -> Result> { aux_handle.wait_ready_on(stream)?; assert!(coset_points_dev.len() >= n); let inv_end = inv_offset_u64 @@ -353,11 +429,85 @@ pub fn barycentric_ext3_on_device_with_dev_inv_denoms( .arg(&mut out_dev) .launch(cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(stream, &out_dev)?; stream.synchronize()?; Ok(out) } +/// [`barycentric_base_on_device_with_dev_inv_denoms`]'s sums from the +/// row-chunked multi kernel at one point — `num_cols × chunks` blocks +/// instead of `num_cols`. The same sums, added in a different order. +pub fn barycentric_base_chunked_with_dev_inv_denoms( + stream: &Arc, + main_handle: &GpuLdeBase, + row_stride: usize, + coset_points_dev: &CudaSlice, + inv_denoms_dev: &CudaSlice, + inv_offset_u64: usize, + n: usize, +) -> Result> { + main_handle.wait_ready_on(stream)?; + assert!(coset_points_dev.len() >= n); + let inv_end = inv_offset_u64 + .checked_add(3 * n) + .expect("barycentric inv_denoms range overflow"); + assert!(inv_end <= inv_denoms_dev.len()); + let num_cols = main_handle.m; + if num_cols == 0 || n == 0 { + return Ok(vec![0; 3 * num_cols]); + } + let be = backend()?; + launch_multi( + stream, + &be.barycentric_base_strided_multi, + main_handle.buf.as_ref(), + main_handle.lde_size, + num_cols, + row_stride, + &coset_points_dev.slice(0..n), + &inv_denoms_dev.slice(inv_offset_u64..inv_end), + n, + 1, + bary_num_chunks_filled(num_cols, n), + ) +} + +/// Ext3 counterpart of [`barycentric_base_chunked_with_dev_inv_denoms`]. +pub fn barycentric_ext3_chunked_with_dev_inv_denoms( + stream: &Arc, + aux_handle: &GpuLdeExt3, + row_stride: usize, + coset_points_dev: &CudaSlice, + inv_denoms_dev: &CudaSlice, + inv_offset_u64: usize, + n: usize, +) -> Result> { + aux_handle.wait_ready_on(stream)?; + assert!(coset_points_dev.len() >= n); + let inv_end = inv_offset_u64 + .checked_add(3 * n) + .expect("barycentric inv_denoms range overflow"); + assert!(inv_end <= inv_denoms_dev.len()); + let num_cols = aux_handle.m; + if num_cols == 0 || n == 0 { + return Ok(vec![0; 3 * num_cols]); + } + let be = backend()?; + launch_multi( + stream, + &be.barycentric_ext3_strided_multi, + aux_handle.buf.as_ref(), + aux_handle.lde_size, + num_cols, + row_stride, + &coset_points_dev.slice(0..n), + &inv_denoms_dev.slice(inv_offset_u64..inv_end), + n, + 1, + bary_num_chunks_filled(num_cols, n), + ) +} + include!(concat!(env!("OUT_DIR"), "/bary_consts.rs")); /// Row-chunk count for the multi kernels: enough `cols * chunks` blocks to @@ -369,39 +519,50 @@ fn bary_num_chunks(num_cols: usize, n: usize) -> usize { by_occupancy.min(by_rows).min(64) } -/// Multi-eval-point counterpart of -/// [`barycentric_base_on_device_with_dev_inv_denoms`]: one pass over the LDE -/// column data computes the barycentric sums for ALL `k_points` evaluation -/// points (their inv_denom blocks live contiguously in `inv_denoms_dev`, the -/// layout `compute_and_invert_denoms_ext3_dev` produces). Returns -/// `3 * k_points * num_cols` u64: `k_points` concatenated per-column blocks, -/// each in the same layout as the single-point kernels. -pub fn barycentric_base_multi_on_device( +/// [`bary_num_chunks`] without its 64-chunk cap, which left a +/// launch of a few columns at a fraction of a wave (4 × 64 blocks on 170 +/// multiprocessors). The occupancy and row floors are unchanged, so a chunk +/// still covers at least 8192 rows; the combine pass folds at most 2048 +/// partials per column. +fn bary_num_chunks_filled(num_cols: usize, n: usize) -> usize { + let by_occupancy = (2048 / num_cols.max(1)).max(1); + let by_rows = (n / 8192).max(1); + by_occupancy.min(by_rows) +} + +/// The chunk count the multi kernels launch with: [`bary_num_chunks_filled`], +/// or [`bary_num_chunks`] under `LAMBDA_VM_DEEP_INV_LEGACY=1`. +fn multi_num_chunks(num_cols: usize, n: usize) -> usize { + if crate::deep_inv::rowwise_enabled() { + bary_num_chunks_filled(num_cols, n) + } else { + bary_num_chunks(num_cols, n) + } +} + +/// One multi-point launch over `num_cols` resident columns plus the combine +/// pass: the body of both `*_multi_on_device` functions, and of the chunked +/// single-point path, which is this at `k_points = 1` over the point's slice +/// of the inverse buffer. +#[allow(clippy::too_many_arguments)] +fn launch_multi( stream: &Arc, - main_handle: &GpuLdeBase, + kernel: &cudarc::driver::CudaFunction, + columns: &CudaSlice, + col_stride: usize, + num_cols: usize, row_stride: usize, - coset_points_dev: &CudaSlice, - inv_denoms_dev: &CudaSlice, + points_view: &cudarc::driver::CudaView<'_, u64>, + inv_view: &cudarc::driver::CudaView<'_, u64>, n: usize, k_points: usize, + num_chunks: usize, ) -> Result> { - main_handle.wait_ready_on(stream)?; - assert!((1..=BARY_MAX_EVAL_POINTS).contains(&k_points)); - assert!(coset_points_dev.len() >= n); - assert!(inv_denoms_dev.len() >= k_points * 3 * n); - let num_cols = main_handle.m; - if num_cols == 0 || n == 0 { - return Ok(vec![0; 3 * k_points * num_cols]); - } let be = backend()?; - let num_chunks = bary_num_chunks(num_cols, n); let total = k_points * num_cols; let mut partials = stream.alloc_zeros::(total * num_chunks * 3)?; let mut out_dev = stream.alloc_zeros::(3 * total)?; - let points_view = coset_points_dev.slice(0..n); - let inv_view = inv_denoms_dev.slice(0..k_points * 3 * n); - - let col_stride_u64 = main_handle.lde_size as u64; + let col_stride_u64 = col_stride as u64; let row_stride_u64 = row_stride as u64; let n_u64 = n as u64; let k_u64 = k_points as u64; @@ -414,12 +575,12 @@ pub fn barycentric_base_multi_on_device( }; unsafe { stream - .launch_builder(&be.barycentric_base_strided_multi) - .arg(main_handle.buf.as_ref()) + .launch_builder(kernel) + .arg(columns) .arg(&col_stride_u64) .arg(&row_stride_u64) - .arg(&points_view) - .arg(&inv_view) + .arg(points_view) + .arg(inv_view) .arg(&n_u64) .arg(&k_u64) .arg(&chunks_u64) @@ -440,11 +601,51 @@ pub fn barycentric_base_multi_on_device( .arg(&mut out_dev) .launch(combine_cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(stream, &out_dev)?; stream.synchronize()?; Ok(out) } +/// Multi-eval-point counterpart of +/// [`barycentric_base_on_device_with_dev_inv_denoms`]: one pass over the LDE +/// column data computes the barycentric sums for ALL `k_points` evaluation +/// points (their inv_denom blocks live contiguously in `inv_denoms_dev`, the +/// layout `compute_and_invert_denoms_ext3_dev` produces). Returns +/// `3 * k_points * num_cols` u64: `k_points` concatenated per-column blocks, +/// each in the same layout as the single-point kernels. +pub fn barycentric_base_multi_on_device( + stream: &Arc, + main_handle: &GpuLdeBase, + row_stride: usize, + coset_points_dev: &CudaSlice, + inv_denoms_dev: &CudaSlice, + n: usize, + k_points: usize, +) -> Result> { + main_handle.wait_ready_on(stream)?; + assert!((1..=BARY_MAX_EVAL_POINTS).contains(&k_points)); + assert!(coset_points_dev.len() >= n); + assert!(inv_denoms_dev.len() >= k_points * 3 * n); + let num_cols = main_handle.m; + if num_cols == 0 || n == 0 { + return Ok(vec![0; 3 * k_points * num_cols]); + } + let be = backend()?; + launch_multi( + stream, + &be.barycentric_base_strided_multi, + main_handle.buf.as_ref(), + main_handle.lde_size, + num_cols, + row_stride, + &coset_points_dev.slice(0..n), + &inv_denoms_dev.slice(0..k_points * 3 * n), + n, + k_points, + multi_num_chunks(num_cols, n), + ) +} + /// Ext3 counterpart of [`barycentric_base_multi_on_device`]. pub fn barycentric_ext3_multi_on_device( stream: &Arc, @@ -464,55 +665,19 @@ pub fn barycentric_ext3_multi_on_device( return Ok(vec![0; 3 * k_points * num_cols]); } let be = backend()?; - let num_chunks = bary_num_chunks(num_cols, n); - let total = k_points * num_cols; - let mut partials = stream.alloc_zeros::(total * num_chunks * 3)?; - let mut out_dev = stream.alloc_zeros::(3 * total)?; - let points_view = coset_points_dev.slice(0..n); - let inv_view = inv_denoms_dev.slice(0..k_points * 3 * n); - - let col_stride_u64 = aux_handle.lde_size as u64; - let row_stride_u64 = row_stride as u64; - let n_u64 = n as u64; - let k_u64 = k_points as u64; - let chunks_u64 = num_chunks as u64; - let total_u64 = total as u64; - let cfg = LaunchConfig { - grid_dim: (num_cols as u32, num_chunks as u32, 1), - block_dim: (BLOCK_DIM, 1, 1), - shared_mem_bytes: 0, - }; - unsafe { - stream - .launch_builder(&be.barycentric_ext3_strided_multi) - .arg(aux_handle.buf.as_ref()) - .arg(&col_stride_u64) - .arg(&row_stride_u64) - .arg(&points_view) - .arg(&inv_view) - .arg(&n_u64) - .arg(&k_u64) - .arg(&chunks_u64) - .arg(&mut partials) - .launch(cfg)?; - } - let combine_cfg = LaunchConfig { - grid_dim: (total.div_ceil(BLOCK_DIM as usize) as u32, 1, 1), - block_dim: (BLOCK_DIM, 1, 1), - shared_mem_bytes: 0, - }; - unsafe { - stream - .launch_builder(&be.barycentric_combine_partials) - .arg(&partials) - .arg(&chunks_u64) - .arg(&total_u64) - .arg(&mut out_dev) - .launch(combine_cfg)?; - } - let out = stream.clone_dtoh(&out_dev)?; - stream.synchronize()?; - Ok(out) + launch_multi( + stream, + &be.barycentric_ext3_strided_multi, + aux_handle.buf.as_ref(), + aux_handle.lde_size, + num_cols, + row_stride, + &coset_points_dev.slice(0..n), + &inv_denoms_dev.slice(0..k_points * 3 * n), + n, + k_points, + multi_num_chunks(num_cols, n), + ) } /// Gather full rows from a device-resident base-field LDE handle. `rows` are LDE @@ -551,7 +716,7 @@ pub fn gather_rows_base_on_device( .arg(&mut out) .launch(cfg)?; } - let host = stream.clone_dtoh(&out)?; + let host = crate::readback::clone_dtoh(stream, &out)?; stream.synchronize()?; Ok(host) } @@ -590,14 +755,38 @@ pub fn gather_rows_ext3_on_device( .arg(&mut out) .launch(cfg)?; } - let host = stream.clone_dtoh(&out)?; + let host = crate::readback::clone_dtoh(stream, &out)?; stream.synchronize()?; Ok(host) } #[cfg(test)] mod tests { - use super::bary_num_chunks; + use super::{bary_num_chunks, bary_num_chunks_filled}; + + /// The default count lifts the 64-chunk cap and nothing else: where the cap did + /// not bind the two agree, and at production shapes the filled count is the + /// occupancy or rows bound the cap hid. + #[test] + fn bary_num_chunks_filled_lifts_only_the_cap() { + for (cols, n) in [ + (100, 1 << 14), + (256, 1 << 17), + (0, 0), + (usize::MAX, 1 << 20), + (1, 0), + ] { + assert_eq!(bary_num_chunks_filled(cols, n), bary_num_chunks(cols, n)); + } + // The R3 parts OOD (1–2 columns at 2^22 rows): 64 → 512 blocks. + assert_eq!(bary_num_chunks(1, 1 << 22), 64); + assert_eq!(bary_num_chunks_filled(1, 1 << 22), 512); + assert_eq!(bary_num_chunks_filled(2, 1 << 22), 512); + // Occupancy-bound: 2048 / 5 columns. + assert_eq!(bary_num_chunks_filled(5, 1 << 22), 409); + // Rows-bound: 2^20 / 8192. + assert_eq!(bary_num_chunks_filled(4, 1 << 20), 128); + } /// Pins which of the three terms binds, per regime. Pure arithmetic — the /// kernels' parity across chunk counts is covered by diff --git a/crypto/math-cuda/src/blake3.rs b/crypto/math-cuda/src/blake3.rs new file mode 100644 index 000000000..da551a27a --- /dev/null +++ b/crypto/math-cuda/src/blake3.rs @@ -0,0 +1,995 @@ +//! GPU BLAKE3 for Merkle commits — the leaf kernels, the parent/level +//! compressors, and the parity-harness handles on the device compression +//! function, byte serialization and chain construction. +//! +//! Twin of [`crate::merkle`]'s keccak path, kernel for kernel, so the two read +//! against each other. Production dispatch reaches this module through +//! [`crate::DeviceHash::Blake3`]: the fused LDE+commit pipelines ([`crate::lde`]), +//! the comp-poly tree builders and the FRI layer commits ([`crate::fri`]) all +//! select between the keccak launchers and these at every leaf, level and +//! FRI-layer launch. +//! +//! # What a parent is +//! +//! `hash_new_parent(left, right)` is one BLAKE3 compression over the 64 bytes of +//! the two child digests: `h = IV`, `t = 0`, `block_len = 64`, `flags = +//! CHUNK_START|CHUNK_END|ROOT`, digest = the low 8 output words little-endian. +//! That is `hash_bytes(left ‖ right)`, which is what `hash_new_parent` already +//! is for every host backend (`hash_new_parent_bytes`, +//! `crypto/crypto/src/merkle_tree/backends/field_element_vector.rs:74`), and at +//! 7 rounds it is literally `blake3::hash(left ‖ right)`. +//! +//! A parent is therefore construction-independent: its message is a single +//! block, and over a single block every candidate chaining construction agrees +//! bit-for-bit. Only multi-block messages — leaves — depend on the construction, +//! which is why the parent compressor could land before it was settled and the +//! leaf kernels could not. +//! +//! # What a leaf is +//! +//! A leaf's bytes are unchanged from the keccak path — `leaves_bit_reversed_grouped` +//! (`crypto/stark/src/commitment.rs:55`) serializes each element in canonical +//! big-endian and concatenates, and only the hash over those bytes moves. The +//! hash is `Blake3Chain` (PA-PLAN §1.7): standard BLAKE3 restricted to a single +//! chunk that never ends, host implementation at +//! `crypto/crypto/src/hash/blake3/chain.rs`. +//! +//! ⚠ That construction is a DRAFT pending ratification of forks F1-F3 +//! (PA-PLAN §1.7.3), implemented here as the working default by standing +//! decision. +//! +//! # Coverage +//! +//! All seven leaf kernels, both tree compressors and the wrapper twins are +//! here; the device-side launchers (`launch_*`) are what the dispatch sites in +//! [`crate::lde`] and [`crate::fri`] call. + +use cudarc::driver::{CudaSlice, CudaStream, CudaViewMut, LaunchConfig, PushKernelArg}; +use std::sync::Arc; + +use crate::Result; +use crate::device::{Backend, backend}; +use crate::lde::pack_ext3_to_pinned_slabs; + +/// Threads per block for the BLAKE3 kernels. +/// +/// Wider than [`crate::merkle`]'s 128 because the register footprint is a third +/// of keccak's: 16 working-state words + 16 message words + the output, all u32, +/// against keccak's 25 u64 lanes plus a 25-lane scratch. The 128 there is a +/// Blackwell register-file limit, not a shape this path shares. +const BLAKE3_BLOCK_DIM: u32 = 256; + +pub(crate) fn blake3_launch_cfg(num_threads: u64) -> LaunchConfig { + debug_assert!( + num_threads <= u32::MAX as u64, + "blake3_launch_cfg: num_threads ({num_threads}) exceeds u32 grid range", + ); + let grid = (num_threads as u32).div_ceil(BLAKE3_BLOCK_DIM); + LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLAKE3_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + } +} + +/// BLAKE3 leaf hashing over a base-field column buffer. Twin of +/// [`crate::merkle::keccak_leaves_base`], argument for argument. +/// +/// `columns` must hold `num_cols * col_stride` u64s with column `c`'s data at +/// `[c*col_stride .. c*col_stride + num_rows]`. `rows_per_leaf` selects the leaf +/// layout: `1` = one leaf per bit-reversed row (`num_rows` leaves), `2` = one +/// leaf per bit-reversed row pair (`num_rows/2` leaves, the trace-commit +/// layout). Returns `(num_rows / rows_per_leaf) * 32` hash bytes. +pub fn leaves_base( + columns: &[u64], + col_stride: usize, + num_cols: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + assert!(num_rows.is_power_of_two()); + assert!(rows_per_leaf == 1 || rows_per_leaf == 2); + assert!( + num_rows >= 2, + "num_rows must be at least 2 for bit-reversed GPU leaf hashing" + ); + assert!( + col_stride >= num_rows, + "col_stride must be >= num_rows to keep per-column reads in-bounds" + ); + let total = num_cols + .checked_mul(col_stride) + .expect("num_cols * col_stride overflows usize"); + assert!(columns.len() >= total); + let be = backend()?; + let stream = be.next_stream(); + let cols_dev = stream.clone_htod(&columns[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / rows_per_leaf) * 32)?; + let launch = if rows_per_leaf == 2 { + launch_leaves_base_row_pair + } else { + launch_leaves_base + }; + launch( + stream.as_ref(), + &cols_dev, + col_stride as u64, + num_cols as u64, + num_rows as u64, + &mut out_dev.as_view_mut(), + )?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Ext3 variant of [`leaves_base`]: columns arrive as three base slabs per ext3 +/// column, so `columns.len() >= num_cols * 3 * col_stride`. Twin of +/// [`crate::merkle::keccak_leaves_ext3`]. +pub fn leaves_ext3( + columns: &[u64], + col_stride: usize, + num_cols: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + assert!(num_rows.is_power_of_two()); + assert!(rows_per_leaf == 1 || rows_per_leaf == 2); + assert!( + num_rows >= 2, + "num_rows must be at least 2 for bit-reversed GPU leaf hashing" + ); + assert!( + col_stride >= num_rows, + "col_stride must be >= num_rows to keep per-column reads in-bounds" + ); + let total = num_cols + .checked_mul(3) + .and_then(|v| v.checked_mul(col_stride)) + .expect("num_cols * 3 * col_stride overflows usize"); + assert!(columns.len() >= total); + let be = backend()?; + let stream = be.next_stream(); + let cols_dev = stream.clone_htod(&columns[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / rows_per_leaf) * 32)?; + // Row-pair ext3 leaves reuse the comp-poly kernel, exactly as the keccak + // path does: hashing all ext3 columns of rows `2i`, `2i+1` is the same + // traversal whether the columns are called "aux trace" or "parts". + let launch = if rows_per_leaf == 2 { + launch_ext3_row_pair + } else { + launch_leaves_ext3 + }; + launch( + stream.as_ref(), + &cols_dev, + col_stride as u64, + num_cols as u64, + num_rows as u64, + &mut out_dev.as_view_mut(), + )?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + +pub(crate) fn launch_leaves_base( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + // The kernel computes `__brevll(tid) >> (64 - log_num_rows)`, which is UB + // for `log_num_rows == 0` (single-row trees are degenerate anyway). + debug_assert!(num_rows >= 2, "blake3 leaf kernel: num_rows must be >= 2"); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = blake3_launch_cfg(num_rows); + unsafe { + stream + .launch_builder(&be.blake3_leaves_base_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_leaves_base_row_pair( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "blake3 row-pair leaf kernel: num_rows must be >= 2" + ); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + // One thread per leaf (= row pair). + let cfg = blake3_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.blake3_leaves_base_row_pair_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_leaves_ext3( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!(num_rows >= 2, "blake3 leaf kernel: num_rows must be >= 2"); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = blake3_launch_cfg(num_rows); + unsafe { + stream + .launch_builder(&be.blake3_leaves_ext3_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_ext3_row_pair( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "blake3 row-pair leaf kernel: num_rows must be >= 2" + ); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = blake3_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.blake3_comp_poly_leaves_ext3) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +/// Row-major row-pair leaf hashing: leaf `i` hashes the two consecutive +/// bit-reversed rows `reverse_index(2i)`, `reverse_index(2i+1)`, each `m` lanes +/// read contiguously from the row-major `data`. Matches the CPU +/// `commit_bit_reversed(.., 2)`; twin of the keccak launcher in +/// [`crate::lde`]. +/// +/// Returns `(num_rows / 2) * 32` hash bytes. Public because the blake3 path has +/// no production caller yet — the parity tests are what reach this kernel, and +/// the keccak twin's private launcher is called from the LDE pipeline instead. +pub fn leaves_base_row_major_row_pair(data: &[u64], m: usize, num_rows: usize) -> Result> { + leaves_row_major_row_pair_inner(data, m, 0, m, num_rows, false) +} + +/// Column-range variant of [`leaves_base_row_major_row_pair`]: each leaf hashes +/// only columns `[col_start, col_end)` of the row pair, while `m` stays the full +/// row stride. Matches the CPU `commit_rows_bit_reversed_subset`, which is how +/// preprocessed tables commit their precomputed and multiplicity column ranges +/// to separate Merkle trees over one row-major LDE. +pub fn leaves_base_row_major_row_pair_range( + data: &[u64], + m: usize, + col_start: usize, + col_end: usize, + num_rows: usize, +) -> Result> { + leaves_row_major_row_pair_inner(data, m, col_start, col_end, num_rows, true) +} + +fn leaves_row_major_row_pair_inner( + data: &[u64], + m: usize, + col_start: usize, + col_end: usize, + num_rows: usize, + ranged: bool, +) -> Result> { + assert!(num_rows.is_power_of_two()); + assert!(num_rows >= 2, "num_rows must be at least 2"); + assert!( + col_start < col_end && col_end <= m, + "column range in bounds" + ); + let total = num_rows + .checked_mul(m) + .expect("num_rows * m overflows usize"); + assert!(data.len() >= total); + + let be = backend()?; + let stream = be.next_stream(); + let data_dev = stream.clone_htod(&data[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / 2) * 32)?; + + let m_u64 = m as u64; + let num_rows_u64 = num_rows as u64; + let log_num_rows = num_rows.trailing_zeros() as u64; + if ranged { + launch_leaves_base_row_major_row_pair_range( + stream.as_ref(), + be, + &data_dev, + m_u64, + col_start as u64, + col_end as u64, + num_rows_u64, + log_num_rows, + &mut out_dev.as_view_mut(), + )?; + } else { + launch_leaves_base_row_major_row_pair( + stream.as_ref(), + be, + &data_dev, + m_u64, + num_rows_u64, + log_num_rows, + &mut out_dev.as_view_mut(), + )?; + } + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Row-major ROW-PAIR leaf hashing under BLAKE3: leaf `i` hashes the two +/// consecutive bit-reversed rows `reverse_index(2i)`, `reverse_index(2i+1)` +/// (each `m` lanes, read contiguously from the row-major `buf`), producing +/// `num_rows / 2` leaves. Device-buffer twin of the keccak launcher the fused +/// LDE pipeline dispatches against; matches the CPU `commit_bit_reversed(.., 2)` +/// and the verifier's `verify_opening_pair`. +pub(crate) fn launch_leaves_base_row_major_row_pair( + stream: &CudaStream, + be: &Backend, + buf: &CudaSlice, + m: u64, + num_rows: u64, + log_num_rows: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + // The kernel derives rows as `__brevll(2*tid + k) >> (64 - log_num_rows)`; + // a 64-bit shift is UB at `log_num_rows == 0`, so require `num_rows >= 2` + // (also the minimum for a single row pair). + debug_assert!( + num_rows >= 2, + "row-major row-pair blake3 requires num_rows >= 2" + ); + let cfg = blake3_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.blake3_leaves_base_row_major_row_pair) + .arg(buf) + .arg(&m) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// Column-range variant of [`launch_leaves_base_row_major_row_pair`]: leaves +/// hash only columns `[col_start, col_end)` of each bit-reversed row pair +/// (`m` stays the full row stride). Matches the CPU +/// `commit_rows_bit_reversed_subset`. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_leaves_base_row_major_row_pair_range( + stream: &CudaStream, + be: &Backend, + buf: &CudaSlice, + m: u64, + col_start: u64, + col_end: u64, + num_rows: u64, + log_num_rows: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "row-major row-pair blake3 requires num_rows >= 2" + ); + debug_assert!( + col_start < col_end && col_end <= m, + "column range in bounds" + ); + let cfg = blake3_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.blake3_leaves_base_row_major_row_pair_range) + .arg(buf) + .arg(&m) + .arg(&col_start) + .arg(&col_end) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// Composition-part leaf hashing under BLAKE3: leaf `i` hashes the ext3 +/// components of every part at the two bit-reversed rows `2i`, `2i+1`, read +/// from per-component slabs with stride `col_stride`. Device-buffer twin of +/// the keccak launch the comp-poly tree build dispatches against. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_comp_poly_leaves_ext3( + stream: &CudaStream, + be: &Backend, + buf: &CudaSlice, + col_stride: u64, + num_parts: u64, + num_rows: u64, + log_num_rows: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "comp-poly blake3 leaves require num_rows >= 2" + ); + let cfg = blake3_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.blake3_comp_poly_leaves_ext3) + .arg(buf) + .arg(&col_stride) + .arg(&num_parts) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// FRI-layer leaf hashing under BLAKE3: leaf `i` hashes the two consecutive +/// ext3 evals `2i`, `2i+1` of an interleaved eval vector (48 bytes — a single +/// compression). Device-buffer twin of the keccak launch the FRI layer commit +/// dispatches against. +pub(crate) fn launch_fri_leaves_ext3( + stream: &CudaStream, + be: &Backend, + evals: &CudaSlice, + num_leaves: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + let cfg = blake3_launch_cfg(num_leaves); + unsafe { + stream + .launch_builder(&be.blake3_fri_leaves_ext3) + .arg(evals) + .arg(&num_leaves) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// Walk the inner Merkle tree on device under BLAKE3. `nodes_dev` already has +/// the `leaves_len` hashed leaves written into the tail; this fills in the inner +/// nodes bottom-up. Twin of [`crate::merkle::build_inner_tree_levels`], and the +/// tail cutover has the same rationale: one single-block launch takes over once a +/// level is no wider than the block, where per-level launch overhead dominates +/// the work and the tail's grid-striding adds no serialization over the launches +/// it replaces. +pub(crate) fn build_inner_tree_levels( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + leaves_len: usize, +) -> Result<()> { + const TAIL_MAX_PAIRS: u64 = BLAKE3_BLOCK_DIM as u64; + let mut level_begin: u64 = (leaves_len - 1) as u64; + while level_begin != 0 { + let new_begin = level_begin / 2; + let n_pairs = level_begin - new_begin; + if n_pairs <= TAIL_MAX_PAIRS { + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (BLAKE3_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_merkle_tail) + .arg(&mut *nodes_dev) + .arg(&level_begin) + .launch(cfg)?; + } + return Ok(()); + } + let cfg = blake3_launch_cfg(n_pairs); + unsafe { + stream + .launch_builder(&be.blake3_merkle_level) + .arg(&mut *nodes_dev) + .arg(&new_begin) + .arg(&n_pairs) + .launch(cfg)?; + } + level_begin = new_begin; + } + Ok(()) +} + +/// Given `hashed_leaves` of length `leaves_len * 32`, build the full BLAKE3 +/// Merkle tree on device and return the `(2*leaves_len - 1) * 32`-byte node +/// buffer in the standard layout: `nodes[0..leaves_len - 1]` are inner nodes +/// (root at index 0) and `nodes[leaves_len - 1..]` are the leaves themselves. +/// +/// Matches the CPU `crypto/crypto/src/merkle_tree/merkle.rs` construction, so +/// the result plugs into `MerkleTree::from_precomputed_nodes` the same way +/// [`crate::merkle::build_merkle_tree_on_device`]'s does. +/// +/// `leaves_len` must be a power of two and >= 2. +pub fn build_merkle_tree_on_device(hashed_leaves: &[u8]) -> Result> { + assert!(hashed_leaves.len().is_multiple_of(32)); + let leaves_len = hashed_leaves.len() / 32; + assert!(leaves_len >= 2, "tree needs at least two leaves"); + assert!( + leaves_len.is_power_of_two(), + "leaves_len must be a power of two" + ); + + let total_nodes = 2 * leaves_len - 1; + let be = backend()?; + let stream = be.next_stream(); + + // SAFETY: every byte is written before it is read — leaves by the H2D + // below, inner nodes by the level walk that follows. + let mut nodes_dev = unsafe { stream.alloc::(total_nodes * 32) }?; + let leaves_offset_bytes = (leaves_len - 1) * 32; + { + let mut slice = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + hashed_leaves.len()); + stream.memcpy_htod(hashed_leaves, &mut slice)?; + } + + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, leaves_len)?; + + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Build the composition Merkle tree under BLAKE3 straight from a +/// device-resident slab buffer (`3*m` slabs of `lde_size` u64s, component `k` of +/// part `c` at `(c*3 + k) * lde_size` — the [`crate::lde::GpuLdeExt3`] layout). +/// No host staging and no H2D: the leaf kernel reads `buf` in place on `stream`. +/// +/// Twin of [`crate::merkle::build_comp_poly_tree_from_slabs_dev`]. +pub fn build_comp_poly_tree_from_slabs_dev( + stream: &Arc, + buf: &CudaSlice, + m: usize, + lde_size: usize, +) -> Result { + build_comp_poly_tree_from_slabs_dev_rpl(stream, buf, m, lde_size, 2) +} + +/// [`build_comp_poly_tree_from_slabs_dev`] with `rows_per_leaf` rows per leaf +/// (2 = row pair, 1 = S2 one row: `lde_size` leaves, the one-row ext3 kernel). +pub fn build_comp_poly_tree_from_slabs_dev_rpl( + stream: &Arc, + buf: &CudaSlice, + m: usize, + lde_size: usize, + rows_per_leaf: usize, +) -> Result { + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); + // Same sticky hook as the keccak twin: the comp-tree cliff test arms one + // counter and must reach it under whichever hash the build pins. + #[cfg(feature = "test-faults")] + crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; + assert!(m > 0); + assert!(lde_size.is_power_of_two() && lde_size >= 2); + assert_eq!(buf.len(), 3 * m * lde_size, "slab buffer shape"); + let num_leaves = lde_size / rows_per_leaf; + let tight_total_nodes = 2 * num_leaves - 1; + let be = backend()?; + + // SAFETY: every byte is written before it is read — leaves by the kernel + // below, inner nodes by the level walk after it. + let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; + let leaves_offset_bytes = (num_leaves - 1) * 32; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); + let launch = if rows_per_leaf == 2 { + launch_ext3_row_pair + } else { + launch_leaves_ext3 + }; + launch( + stream.as_ref(), + buf, + lde_size as u64, + m as u64, + lde_size as u64, + &mut leaves_view, + )?; + } + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + let mut root = [0u8; 32]; + crate::readback::memcpy_dtoh(stream, &nodes_dev.slice(0..32), &mut root)?; + stream.synchronize()?; + Ok(crate::lde::GpuMerkleTree { + nodes: Arc::new(nodes_dev), + leaves_len: num_leaves, + arity: 2, + root, + }) +} + +/// Build the composition Merkle tree under BLAKE3 from host-side interleaved +/// ext3 parts, keeping the nodes device-resident so openings can gather paths on +/// device. `parts_interleaved` is `num_parts` slices, each `[a0,a1,a2,b0,b1,b2,…]` +/// of length `3*lde_size`. Leaves hash row pairs, so `leaves_len = lde_size / 2`. +/// +/// Twin of [`crate::merkle::build_comp_poly_tree_from_evals_ext3_keep`], and it +/// stages through the same pinned de-interleave buffer for the same reason. +pub fn build_comp_poly_tree_from_evals_ext3_keep( + parts_interleaved: &[&[u64]], +) -> Result { + build_comp_poly_tree_from_evals_ext3_keep_rpl(parts_interleaved, 2) +} + +/// [`build_comp_poly_tree_from_evals_ext3_keep`] with `rows_per_leaf` rows per +/// leaf (2 = row pair, 1 = S2 one row). +pub fn build_comp_poly_tree_from_evals_ext3_keep_rpl( + parts_interleaved: &[&[u64]], + rows_per_leaf: usize, +) -> Result { + #[cfg(feature = "test-faults")] + crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; + assert!(!parts_interleaved.is_empty()); + let m = parts_interleaved.len(); + let ext3_elems = parts_interleaved[0].len() / 3; + assert_eq!( + parts_interleaved[0].len(), + 3 * ext3_elems, + "ext3 buffer length must be 3 * lde_size" + ); + for p in parts_interleaved.iter() { + assert_eq!(p.len(), 3 * ext3_elems); + } + let lde_size = ext3_elems; + assert!(lde_size.is_power_of_two() && lde_size >= 2); + + let be = backend()?; + let stream = be.next_stream(); + let staging_slot = be.pinned_staging(); + + // Stage: de-interleave each part into 3 base slabs in pinned memory. + let mb = 3 * m; + let mut staging = staging_slot.lock().unwrap(); + staging.ensure_capacity(mb * lde_size, &be.ctx)?; + let pinned = unsafe { staging.as_mut_slice(mb * lde_size) }; + + pack_ext3_to_pinned_slabs(parts_interleaved, pinned, lde_size); + + // H2D the de-interleaved parts, then release the staging lock: the tree + // build reads the device `buf`, not `pinned`. Synchronize first so the async + // H2D has consumed `pinned` before it can be freed or reused. + let mut buf = stream.alloc_zeros::(mb * lde_size)?; + stream.memcpy_htod(&pinned[..mb * lde_size], &mut buf)?; + stream.synchronize()?; + drop(staging); + + build_comp_poly_tree_from_slabs_dev_rpl(&stream, &buf, m, lde_size, rows_per_leaf) +} + +/// Build a FRI-layer Merkle tree on device under BLAKE3 from an interleaved ext3 +/// eval vector, returning the full host node buffer so tests can compare it byte +/// for byte against the CPU. Each leaf hashes two consecutive ext3 values; +/// `num_leaves = evals.len() / 6`. Returns `(2*num_leaves - 1) * 32` bytes in +/// standard layout. +/// +/// Twin of [`crate::merkle::build_fri_layer_tree_from_evals_ext3`], and like it a +/// parity harness rather than a production path: production folds and commits +/// through [`crate::fri::FriLayer::fold_and_commit_layer`]. +pub fn build_fri_layer_tree_from_evals_ext3(evals: &[u64]) -> Result> { + assert!( + evals.len().is_multiple_of(6), + "evals must hold whole pair-leaves" + ); + let num_evals = evals.len() / 3; + let num_leaves = num_evals / 2; + assert!(num_leaves.is_power_of_two() && num_leaves >= 2); + let tight_total_nodes = 2 * num_leaves - 1; + + let be = backend()?; + let stream = be.next_stream(); + + let evals_dev = stream.clone_htod(evals)?; + // SAFETY: leaves are written by the kernel below, inner nodes by the level + // walk after it, before either is read. + let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; + + let leaves_offset_bytes = (num_leaves - 1) * 32; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); + let num_leaves_u64 = num_leaves as u64; + let cfg = blake3_launch_cfg(num_leaves as u64); + unsafe { + stream + .launch_builder(&be.blake3_fri_leaves_ext3) + .arg(&evals_dev) + .arg(&num_leaves_u64) + .arg(&mut leaves_view) + .launch(cfg)?; + } + } + + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// One compression's inputs, in the argument order of the host reference +/// `blake3_compress_rounds(h, m, t, block_len, flags, rounds)`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CompressInput { + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, +} + +/// Which round count [`compress_probe`] should run. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProbeRounds { + /// 6 — the internal variant. + Six, + /// 7 — standard BLAKE3, where the `blake3` crate is an external anchor. + Seven, + /// Whatever the cubin's production kernels are compiled for. The only way to + /// observe from host code which of the two `blake3_merkle_level` uses. + CompiledIn, +} + +/// Parity harness: run the device compression function over `inputs` and return +/// each full 16-word output. +/// +/// Not a production path — the device compression is otherwise unreachable from +/// host code, so without this there would be nothing to check it against the +/// host reference with. +pub fn compress_probe(inputs: &[CompressInput], rounds: ProbeRounds) -> Result> { + if inputs.is_empty() { + return Ok(Vec::new()); + } + let n = inputs.len(); + let mut h = Vec::with_capacity(n * 8); + let mut m = Vec::with_capacity(n * 16); + let mut t = Vec::with_capacity(n); + let mut block_len = Vec::with_capacity(n); + let mut flags = Vec::with_capacity(n); + for i in inputs { + h.extend_from_slice(&i.h); + m.extend_from_slice(&i.m); + t.push(i.t); + block_len.push(i.block_len); + flags.push(i.flags); + } + + let be = backend()?; + let stream = be.next_stream(); + let h_dev = stream.clone_htod(&h)?; + let m_dev = stream.clone_htod(&m)?; + let t_dev = stream.clone_htod(&t)?; + let bl_dev = stream.clone_htod(&block_len)?; + let fl_dev = stream.clone_htod(&flags)?; + let mut out_dev = stream.alloc_zeros::(n * 16)?; + + let kernel = match rounds { + ProbeRounds::Six => &be.blake3_compress_probe_6r, + ProbeRounds::Seven => &be.blake3_compress_probe_7r, + ProbeRounds::CompiledIn => &be.blake3_compress_probe_default, + }; + let n_u64 = n as u64; + let cfg = blake3_launch_cfg(n_u64); + unsafe { + stream + .launch_builder(kernel) + .arg(&h_dev) + .arg(&m_dev) + .arg(&t_dev) + .arg(&bl_dev) + .arg(&fl_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let flat = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(flat + .chunks_exact(16) + .map(|c| { + let mut w = [0u32; 16]; + w.copy_from_slice(c); + w + }) + .collect()) +} + +/// The round count `kernels/blake3.cu` was compiled for. +/// +/// The host tree's round count and this one are separate crates' features, so +/// nothing forces them equal; a mismatch would be a GPU tree committing under a +/// different hash than the CPU one, with no symptom short of a failing verify. +/// Reading it back makes that assertable. +pub fn device_rounds() -> Result { + let be = backend()?; + let stream = be.next_stream(); + let mut out_dev = stream.alloc_zeros::(1)?; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_rounds_probe) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out[0]) +} + +/// Parity harness: the BLAKE3 message words each of `vals` serializes to — two +/// per element, the byte-reverse of its canonical value's high then low half. +/// +/// This is the serialization the leaf kernels share with the CPU commit path +/// (`leaves_bit_reversed_grouped`, `crypto/stark/src/commitment.rs:55`), isolated +/// from any hashing: canonicalisation, big-endian element bytes, little-endian +/// word packing. +pub fn serialize_felts(vals: &[u64]) -> Result> { + if vals.is_empty() { + return Ok(Vec::new()); + } + let be = backend()?; + let stream = be.next_stream(); + let vals_dev = stream.clone_htod(vals)?; + let mut out_dev = stream.alloc_zeros::(vals.len() * 2)?; + let n_u64 = vals.len() as u64; + let cfg = blake3_launch_cfg(n_u64); + unsafe { + stream + .launch_builder(&be.blake3_serialize_felts_probe) + .arg(&vals_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Parity harness: `words` streamed through the device `Blake3Chain`, returning +/// the 32-byte digest. +/// +/// ★ This is what lets the device be asserted against the COMMITTED KAT TABLE +/// (`crypto::hash::blake3::chain::CHAIN_KAT_6ROUND`) rather than only against +/// the host implementation — the difference risk R13 is about. The KAT digests +/// were produced by a Python oracle, so a device digest matching them is checked +/// against an artifact this tree did not compute. +/// +/// Word-granular because that is all the device ever hashes: production messages +/// are whole numbers of 8-byte field elements. KAT lengths that are not +/// multiples of 4 are unreachable from device code by construction and are +/// covered by the host tests instead. +pub fn chain_probe(words: &[u32]) -> Result<[u8; 32]> { + let be = backend()?; + let stream = be.next_stream(); + // The empty message is a legitimate input (one compression, `block_len = 0`), + // so an empty slice must still reach the kernel. `clone_htod` of an empty + // slice is not portable, so allocate a one-word buffer and pass `n = 0`. + let words_dev = if words.is_empty() { + stream.alloc_zeros::(1)? + } else { + stream.clone_htod(words)? + }; + let mut out_dev = stream.alloc_zeros::(32)?; + let n_words = words.len() as u64; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_chain_probe) + .arg(&words_dev) + .arg(&n_words) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + let mut digest = [0u8; 32]; + digest.copy_from_slice(&out); + Ok(digest) +} + +/// Parity harness: `vals` streamed through the device block builder, returning +/// the `ceil(2*len/16)` completed 64-byte blocks as 16 words each, tail block +/// zero-padded. +/// +/// Exercises the block framing on the code path a leaf kernel will use — one +/// thread streaming a whole leaf — with the compression sink replaced by a copy +/// out. Small inputs only; it is single-threaded by design. +pub fn blocks_of_felts(vals: &[u64]) -> Result> { + if vals.is_empty() { + return Ok(Vec::new()); + } + let n_words = vals.len() * 2; + let n_blocks = n_words.div_ceil(16); + let be = backend()?; + let stream = be.next_stream(); + let vals_dev = stream.clone_htod(vals)?; + let mut out_dev = stream.alloc_zeros::(n_blocks * 16)?; + let n_u64 = vals.len() as u64; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_blocks_of_felts_probe) + .arg(&vals_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} diff --git a/crypto/math-cuda/src/columns.rs b/crypto/math-cuda/src/columns.rs new file mode 100644 index 000000000..7da1e9bc0 --- /dev/null +++ b/crypto/math-cuda/src/columns.rs @@ -0,0 +1,389 @@ +//! The epoch's trace columns, on the card once. +//! +//! Four things read the same columns — the commitment, the sumcheck's factors, +//! the evaluation at the reduction point, and the opening's message — and each +//! used to upload its own copy. That is the same forty gigabytes crossing the +//! bus four times, and at the speed pageable host memory gives it is a fifth of +//! the prove. They are put here once and read where they lie. +//! +//! A table's columns are a contiguous run of equal height, which is the layout +//! the factor gather and the batched evaluation already want; the commitment and +//! the opening scatter theirs, which on the card is a copy at device bandwidth. + +use std::sync::Arc; + +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, DevicePtrMut}; + +use crate::Result; +use crate::device::{DeviceReservation, alloc_or_trim, backend}; + +/// `LAMBDA_VM_CARD_UPLOAD_TIMES=1`: each packed table's copy and widen, and +/// each records table's copy and fill, in a group's upload is timed by events +/// on its stream and summed ([`upload_kernel_times`]). Read once; unset, no +/// event is made. A measurement; the store is the same. +fn timed_uploads() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| std::env::var("LAMBDA_VM_CARD_UPLOAD_TIMES").is_ok_and(|v| v.trim() == "1")) +} + +/// The tables timed and their summed nanoseconds: packed (copy and widen), +/// records (copy and fill). +static WIDEN_TABLES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static WIDEN_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static FILL_TABLES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static FILL_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Under `LAMBDA_VM_CARD_UPLOAD_TIMES=1`, the tables whose upload was timed +/// since the process started and their card seconds: +/// `((packed tables, copy and widen s), (records tables, copy and fill s))`. +/// Zeros otherwise. +pub fn upload_kernel_times() -> ((u64, f64), (u64, f64)) { + use std::sync::atomic::Ordering::Relaxed; + let s = |ns: &std::sync::atomic::AtomicU64| ns.load(Relaxed) as f64 * 1e-9; + ( + (WIDEN_TABLES.load(Relaxed), s(&WIDEN_NS)), + (FILL_TABLES.load(Relaxed), s(&FILL_NS)), + ) +} + +/// One table's upload timed on its stream: the events either side of it. +struct TableClock { + records: bool, + start: cudarc::driver::CudaEvent, + done: cudarc::driver::CudaEvent, +} + +impl TableClock { + /// Events on `stream`, the first recorded now; `None` when uploads are not + /// timed or an event cannot be made. + fn start(stream: &Arc, records: bool) -> Option { + if !timed_uploads() { + return None; + } + let flags = Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT); + let ctx = stream.context(); + let clock = Self { + records, + start: ctx.new_event(flags).ok()?, + done: ctx.new_event(flags).ok()?, + }; + clock.start.record(stream).ok()?; + Some(clock) + } + + /// Adds the span to the totals, once the stream ran it. + fn add(self) { + use std::sync::atomic::Ordering::Relaxed; + let Ok(ms) = self.start.elapsed_ms(&self.done) else { + return; + }; + let ns = (f64::from(ms) * 1e6) as u64; + let (tables, total) = if self.records { + (&FILL_TABLES, &FILL_NS) + } else { + (&WIDEN_TABLES, &WIDEN_NS) + }; + tables.fetch_add(1, Relaxed); + total.fetch_add(ns, Relaxed); + } +} + +/// Where a set of columns is: still here, or already there. +pub enum Columns<'a> { + Host(&'a [&'a [u64]]), + /// A run of `width` columns of `rows` each, starting at column `first`. + Device { + store: &'a DeviceColumns, + first: usize, + width: usize, + }, +} + +impl Columns<'_> { + pub fn width(&self) -> usize { + match self { + Self::Host(columns) => columns.len(), + Self::Device { width, .. } => *width, + } + } + + pub fn rows(&self) -> usize { + match self { + Self::Host(columns) => columns.first().map_or(0, |c| c.len()), + Self::Device { store, first, .. } => store.spans[*first].1, + } + } + + pub fn is_empty(&self) -> bool { + self.width() == 0 + } +} + +/// A table's columns as they go up: as they are, packed (each column at the +/// bytes its words need, [`crate::narrow`]) and widened on the card, or as the +/// records its rows come from and filled on the card ([`crate::fill`]). +#[derive(Clone, Copy)] +pub enum TableUpload<'a> { + /// The columns, every one the table's height. + Wide(&'a [&'a [u64]]), + Narrow(crate::narrow::NarrowInput<'a>), + Records(crate::fill::RecordsInput<'a>), +} + +impl TableUpload<'_> { + /// Words the table takes on the card. + pub fn cells(&self) -> usize { + match self { + Self::Wide(columns) => columns.iter().map(|c| c.len()).sum(), + Self::Narrow(packed) => packed.rows() * packed.cols(), + Self::Records(records) => records.rows() * records.cols(), + } + } +} + +/// The columns themselves, laid end to end in one allocation. +pub struct DeviceColumns { + stream: Arc, + /// Shared so a reader that outlives a borrow of the store — a WHIR opening's + /// first rounds read the message where it lies — keeps it alive. + buffer: Arc>, + /// `(offset, len)` in elements, per column, in the order uploaded. + spans: Vec<(usize, usize)>, + _room: DeviceReservation, +} + +impl DeviceColumns { + /// `None` when the card will not promise the room, in which case every + /// caller uploads its own copy as before. + pub fn upload(columns: &[&[u64]]) -> Option { + if columns.is_empty() { + return None; + } + let total: usize = columns.iter().map(|c| c.len()).sum(); + let be = backend().ok()?; + let Some(room) = be.reserve(total as u64 * 8) else { + crate::device::note_device_fallback(); + return None; + }; + crate::argue_probe::note_device(crate::argue_probe::Surface::Columns, total as u64 * 8); + let stream = be.next_stream(); + // SAFETY: every element is written by the copies below. + let mut buffer = unsafe { alloc_or_trim::(&stream, total) }.ok()?; + let mut spans = Vec::with_capacity(columns.len()); + let mut at = 0usize; + for column in columns { + let mut slab = buffer.slice_mut(at..at + column.len()); + stream.memcpy_htod(*column, &mut slab).ok()?; + spans.push((at, column.len())); + at += column.len(); + } + stream.synchronize().ok()?; + Some(Self { + stream, + buffer: Arc::new(buffer), + spans, + _room: room, + }) + } + + /// [`Self::upload`] of whole tables, each as it is held: a packed one + /// crosses the bus at its packed bytes and is widened on the card, column + /// by column, into the run its columns take — the same words, at the same + /// offsets, as its widened columns uploaded; one held as records crosses it + /// at their bytes and is filled there, into the same run. `None` as for + /// [`Self::upload`], or when a packed table is too big for one launch. + pub fn upload_tables(tables: &[TableUpload<'_>]) -> Option { + let total: usize = tables.iter().map(TableUpload::cells).sum(); + if total == 0 + || tables.iter().any(|table| { + matches!(table, TableUpload::Narrow(_)) && u32::try_from(table.cells()).is_err() + }) + { + return None; + } + let be = backend().ok()?; + let Some(room) = be.reserve(total as u64 * 8) else { + crate::device::note_device_fallback(); + return None; + }; + // The packed bytes (or records) sit on the card only while their + // table widens (or fills), one table at a time in stream order: + // promised for the largest of them. + let staging = tables + .iter() + .map(|table| match table { + TableUpload::Wide(_) => 0, + TableUpload::Narrow(packed) => packed.bytes() as u64, + TableUpload::Records(records) => records.bytes() as u64, + }) + .max() + .unwrap_or(0); + let _staging = match staging { + 0 => None, + bytes => { + let Some(promise) = be.reserve(bytes) else { + crate::device::note_device_fallback(); + return None; + }; + Some(promise) + } + }; + crate::argue_probe::note_device(crate::argue_probe::Surface::Columns, total as u64 * 8); + let stream = be.next_stream(); + // SAFETY: every element is written by the copies and widens below. + let mut buffer = unsafe { alloc_or_trim::(&stream, total) }.ok()?; + let mut spans = Vec::new(); + let mut at = 0usize; + let mut checks = crate::fill::FillChecks::default(); + let mut clocks = Vec::new(); + for table in tables { + match table { + TableUpload::Wide(columns) => { + for column in columns.iter() { + let mut slab = buffer.slice_mut(at..at + column.len()); + stream.memcpy_htod(*column, &mut slab).ok()?; + spans.push((at, column.len())); + at += column.len(); + } + } + TableUpload::Narrow(packed) => { + let (rows, cols) = (packed.rows(), packed.cols()); + let mut slab = buffer.slice_mut(at..at + rows * cols); + let clock = TableClock::start(&stream, false); + crate::narrow::widen_col_major_into( + &stream, be, *packed, rows, cols, &mut slab, + ) + .ok()?; + clocks.extend(clock.filter(|c| c.done.record(&stream).is_ok())); + spans.extend((0..cols).map(|k| (at + k * rows, rows))); + at += rows * cols; + } + TableUpload::Records(records) => { + let (rows, cols) = (records.rows(), records.cols()); + let mut slab = buffer.slice_mut(at..at + rows * cols); + let clock = TableClock::start(&stream, true); + crate::fill::fill_col_major_into(&stream, *records, &mut slab, &mut checks) + .ok()?; + clocks.extend(clock.filter(|c| c.done.record(&stream).is_ok())); + spans.extend((0..cols).map(|k| (at + k * rows, rows))); + at += rows * cols; + } + } + } + stream.synchronize().ok()?; + clocks.into_iter().for_each(TableClock::add); + // A fill whose records it refused (a CPU log at a pc with no DECODE + // row): the card declines the group, and its readers take the host's + // columns. The host fill refuses the same log, so that table reads as + // no columns and the proof is one the verifier refuses. An honest run + // never gets here: the walk refused such a log before any chunk. + if !checks.passed(&stream).ok()? { + crate::device::note_device_fallback(); + return None; + } + Some(Self { + stream, + buffer: Arc::new(buffer), + spans, + _room: room, + }) + } + + /// The run of `width` columns from `first`, packed on the card (each + /// column at the bytes its words need, [`crate::narrow`]): its widths and + /// its packed bytes, the only bytes that come back. `None` when the columns + /// are not a run or are too many words for one launch. + pub fn pack_run(&self, first: usize, width: usize) -> Result, Vec)>> { + if !self.is_run(first, width) { + return Ok(None); + } + let rows = self.spans[first].1; + if u32::try_from(rows * width).is_err() { + return Ok(None); + } + let be = backend()?; + let stream = be.next_stream(); + crate::narrow::pack_col_major_on_stream(&stream, be, &self.view(first, width), rows, width) + .map(Some) + } + + pub fn num_columns(&self) -> usize { + self.spans.len() + } + + /// Every column back on the host, in upload order: what a test compares. + pub fn download(&self) -> Result>> { + let all = crate::readback::clone_dtoh(&self.stream, &*self.buffer)?; + self.stream.synchronize()?; + Ok(self + .spans + .iter() + .map(|&(at, len)| all[at..at + len].to_vec()) + .collect()) + } + + /// Whether `width` columns from `first` are a run of equal height — which + /// is what the kernels that read a table's columns in place need. + pub fn is_run(&self, first: usize, width: usize) -> bool { + if width == 0 || first + width > self.spans.len() { + return false; + } + let (start, rows) = self.spans[first]; + (0..width).all(|k| self.spans[first + k] == (start + k * rows, rows)) + } + + /// The one allocation every column lives in, shared. + pub fn buffer(&self) -> Arc> { + self.buffer.clone() + } + + /// Column `k`'s `(offset, len)` in elements, within [`buffer`](Self::buffer). + pub fn span(&self, k: usize) -> (usize, usize) { + self.spans[k] + } + + /// The device address of column `k` and its length in elements. + pub fn at(&self, k: usize) -> (u64, usize) { + let (offset, len) = self.spans[k]; + let (base, _guard) = self.buffer.device_ptr(&self.stream); + (base + (offset * 8) as u64, len) + } + + /// A view over `width` columns from `first`, which must be a run. + pub fn view(&self, first: usize, width: usize) -> cudarc::driver::CudaView<'_, u64> { + let (offset, rows) = self.spans[first]; + self.buffer.slice(offset..offset + width * rows) + } + + pub fn stream(&self) -> &Arc { + &self.stream + } + + /// Copies column `k` into `dst` at `offset`, on the card. + /// + /// Issued on the **destination's** stream, so whatever reads `dst` next is + /// ordered behind it. The store is written once and synchronized at upload + /// and read-only after, so another stream reading it races with nothing. + pub fn copy_into( + &self, + k: usize, + dst: &mut CudaSlice, + offset: usize, + stream: &Arc, + ) -> Result<()> { + let (src, len) = self.at(k); + let (base, _guard) = dst.device_ptr_mut(stream); + // SAFETY: both ranges are inside allocations this call holds a handle + // to, and the caller checked `offset + len` against `dst`. + unsafe { + cudarc::driver::sys::cuMemcpyDtoDAsync_v2( + base + (offset * 8) as u64, + src, + len * 8, + stream.cu_stream(), + ) + .result()?; + } + Ok(()) + } +} diff --git a/crypto/math-cuda/src/constraint_interp.rs b/crypto/math-cuda/src/constraint_interp.rs index 0c2a620ad..1ddcf7aa5 100644 --- a/crypto/math-cuda/src/constraint_interp.rs +++ b/crypto/math-cuda/src/constraint_interp.rs @@ -174,6 +174,10 @@ pub struct CompositionAccum<'a> { /// device buffer (kernel indexing `b * num_rows + row`) — no PCIe traffic /// per dispatch. pub b_z_inv: &'a [&'a GpuBaseVec], + /// Sub-coset stride: `H` is produced at LDE rows `0, s, 2s, …` only + /// (`num_rows / s` output rows). 1 is the whole LDE coset. Must divide + /// `num_rows`. + pub row_stride: usize, } /// A base-field column resident on device, uploaded once and reused across @@ -248,6 +252,13 @@ fn eval_composition_launch( "b_beta ext3 per boundary" ); + let row_stride = accum.row_stride; + assert!( + row_stride >= 1 && num_rows.is_multiple_of(row_stride), + "row_stride must divide the LDE row count" + ); + let num_out = num_rows / row_stride; + let be = backend()?; let stream = be.next_stream(); main.wait_ready_on(&stream)?; @@ -284,14 +295,14 @@ fn eval_composition_launch( } let max_grid = MAX_THREADS / BLOCK_DIM; - let grid = (num_rows as u32).div_ceil(BLOCK_DIM).clamp(1, max_grid); + let grid = (num_out as u32).div_ceil(BLOCK_DIM).clamp(1, max_grid); let num_threads = (grid as usize) * (BLOCK_DIM as usize); // Per-thread slot scratch, uninitialized (the walk writes before reading). let mut d_vals_base = unsafe { stream.alloc::((num_base_slots * num_threads).max(1)) }?; let mut d_vals_ext = unsafe { stream.alloc::((num_ext_slots * 3 * num_threads).max(1)) }?; - // Output: every row is written by the grid-stride loop. - let mut d_h = unsafe { stream.alloc::(num_rows * 3) }?; + // Output: every output row is written by the grid-stride loop. + let mut d_h = unsafe { stream.alloc::(num_out * 3) }?; let num_nodes_u64 = num_nodes as u64; let num_roots_u64 = num_roots as u64; @@ -299,6 +310,7 @@ fn eval_composition_launch( let aux_stride = aux.lde_size as u64; let next_step_u64 = next_step as u64; let num_rows_u64 = num_rows as u64; + let row_stride_u64 = row_stride as u64; let z_len_u64 = (accum.z_inv.len() as u64).max(1); let num_boundary_u64 = num_boundary as u64; @@ -326,6 +338,7 @@ fn eval_composition_launch( .arg(&aux_stride) .arg(&next_step_u64) .arg(&num_rows_u64) + .arg(&row_stride_u64) .arg(&d_beta_trans) .arg(&d_z_inv) .arg(&z_len_u64) @@ -344,8 +357,9 @@ fn eval_composition_launch( /// Evaluate the constraints AND fuse the composition accumulation on-device: /// `H(row) = z_inv[row]·Σ βᵢ·Cᵢ + Σ_b z_b_inv[row]·β_b·(trace_b − value_b)`, -/// returning `H` as raw ext3 limbs (`num_rows * 3` u64, `out[row*3 + k]`). No -/// per-constraint matrix is materialized. +/// returning `H` as raw ext3 limbs (`(num_rows / accum.row_stride) * 3` u64, +/// `out[out_row*3 + k]` for LDE row `out_row * row_stride`). No per-constraint +/// matrix is materialized. /// /// Uniform-zerofier case only (the VM has no end-exemptions); the caller gates /// on `is_uniform` and falls back to CPU otherwise. @@ -397,7 +411,8 @@ pub fn eval_composition_on_device( /// The composition evals `H` resident on device (interleaved ext3, /// `num_rows * 3` u64), with the stream that produced them: downstream device -/// consumers enqueue on the same stream for ordering. +/// consumers enqueue on the same stream for ordering. `num_rows` counts the +/// rows `H` was evaluated at — the LDE size divided by the sub-coset stride. pub struct GpuCompH { buf: CudaSlice, pub num_rows: usize, @@ -442,7 +457,7 @@ pub fn eval_composition_on_device_keep( )?; Ok(GpuCompH { buf, - num_rows, + num_rows: num_rows / accum.row_stride, stream, }) } @@ -462,6 +477,27 @@ pub fn download_comp_h(h: &GpuCompH) -> Result> { Ok(out) } +/// [`download_comp_h`] through the staging pairs (32 MiB pinned chunks, +/// double-buffered) instead of the shared pinned slab. The slab grows to the +/// largest drain it has served, and a growth is a `cuMemHostAlloc` of the new +/// size that holds the driver's lock — every other thread's CUDA call waits +/// behind it (≈ 0.23 s for a 192 MiB `H`, the card idle meanwhile). The pairs +/// never grow. Same words. +pub fn download_comp_h_staged(h: &GpuCompH) -> Result> { + use cudarc::driver::DevicePtr; + let mut out = Vec::new(); + let src = { + let (ptr, _record) = h.buf.device_ptr(&h.stream); + ptr + }; + // SAFETY: `src` is `h.buf`, `h.buf.len()` words written by kernels queued on + // `h.stream` before this call, and kept alive by `h` past the last chunk. + unsafe { + crate::device::dtoh_staged_into(&h.stream, src, h.buf.len(), &mut out, || Ok(()))?; + } + Ok(out) +} + /// Degree-2 quotient decomposition on device: splits a resident `H` (2n rows) /// into the two halves `H0/H1`, written in zero-padded slab layout (6 slabs of /// `lde_size = 2n` u64, first `n` filled) ready for the batched slab LDE. @@ -470,11 +506,29 @@ pub fn decompose_d2_into_slabs( h: &GpuCompH, inv_2x: &GpuBaseVec, two_inv: u64, +) -> Result<(CudaSlice, Arc, usize)> { + decompose_d2_into_slabs_strided(h, inv_2x, two_inv, 1, h.num_rows) +} + +/// [`decompose_d2_into_slabs`] for an `H` evaluated on a sub-coset of the LDE +/// coset (every `inv_stride`-th LDE row; 2n rows): `inv_2x` is the full coset's +/// `1/(2x)` table, read at `i * inv_stride`, and the slabs are `lde_size` u64 +/// each (first `n` filled, the rest the zero-pad of an n → `lde_size` LDE). +pub fn decompose_d2_into_slabs_strided( + h: &GpuCompH, + inv_2x: &GpuBaseVec, + two_inv: u64, + inv_stride: usize, + lde_size: usize, ) -> Result<(CudaSlice, Arc, usize)> { let n = h.num_rows / 2; assert_eq!(h.num_rows, n * 2, "H row count must be even"); - assert!(inv_2x.len() >= n, "inv_2x must cover the half domain"); - let lde_size = h.num_rows; + assert!(inv_stride >= 1, "inv_stride must be positive"); + assert!( + n == 0 || inv_2x.len() > (n - 1) * inv_stride, + "inv_2x must cover the sub-coset's half" + ); + assert!(lde_size >= h.num_rows, "slabs must hold the evaluated rows"); let be = backend()?; let stream = h.stream.clone(); let mut out = stream.alloc_zeros::(6 * lde_size)?; @@ -489,6 +543,7 @@ pub fn decompose_d2_into_slabs( }; let n_u64 = n as u64; let stride_u64 = lde_size as u64; + let inv_stride_u64 = inv_stride as u64; unsafe { stream .launch_builder(&be.decompose_d2_kernel) @@ -497,6 +552,7 @@ pub fn decompose_d2_into_slabs( .arg(&two_inv) .arg(&n_u64) .arg(&stride_u64) + .arg(&inv_stride_u64) .arg(&mut out) .launch(cfg)?; } diff --git a/crypto/math-cuda/src/deep.rs b/crypto/math-cuda/src/deep.rs index b0eefd61d..3330771ee 100644 --- a/crypto/math-cuda/src/deep.rs +++ b/crypto/math-cuda/src/deep.rs @@ -353,6 +353,15 @@ pub fn deep_composition_ext3_fully_resident_keep( row_stride, domain_size, )?; + into_fri_order(stream, &deep_out, domain_size) +} + +/// Bit-reverse-permute a natural-order DEEP codeword into FRI order on device. +fn into_fri_order( + stream: &Arc, + deep_out: &CudaSlice, + domain_size: usize, +) -> Result { let be = backend()?; // SAFETY: every element is written by the permutation kernel below. let mut reversed = unsafe { stream.alloc::(domain_size * 3) }?; @@ -367,7 +376,7 @@ pub fn deep_composition_ext3_fully_resident_keep( unsafe { stream .launch_builder(&be.bit_reverse_ext3_kernel) - .arg(&deep_out) + .arg(deep_out) .arg(&mut reversed) .arg(&n_u64) .arg(&log_n) @@ -380,6 +389,314 @@ pub fn deep_composition_ext3_fully_resident_keep( }) } +impl GpuDeepCodeword { + /// The codeword on host (FRI order, ext3 interleaved), for parity tests. + pub fn download(&self) -> Result> { + let out = crate::readback::clone_dtoh(&self.stream, &self.buf)?; + self.stream.synchronize()?; + Ok(out) + } +} + +/// `LAMBDA_VM_DEEP_HOISTED` as given: on when unset, `1` or `on`; off when +/// empty, `0` or `off`; anything else is on (the default), said once on stderr. +fn deep_hoisted_setting(value: Option<&str>) -> bool { + match value.map(str::trim) { + Some("" | "0" | "off") => false, + None | Some("1" | "on") => true, + Some(other) => { + eprintln!( + "math-cuda: LAMBDA_VM_DEEP_HOISTED={other} is not 0, 1, off or on: on (the default)" + ); + true + } + } +} + +/// Runtime override of [`deep_hoisted`]: 0 = the environment's choice, 1 = off, +/// 2 = on. +static DEEP_HOISTED_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Force `LAMBDA_VM_DEEP_HOISTED` on, off, or back to the environment (tests). +pub fn set_deep_hoisted(on: Option) { + let v = match on { + None => 0, + Some(false) => 1, + Some(true) => 2, + }; + DEEP_HOISTED_OVERRIDE.store(v, std::sync::atomic::Ordering::Relaxed); +} + +/// Whether the fused DEEP runs with its row-independent part hoisted +/// (`LAMBDA_VM_DEEP_HOISTED`; on by default, `=0` turns it off): the same codeword from +/// `deep_composition_ext3_hoisted_m*`, one ext3×base product per (main +/// column, pole) instead of two ext3×ext3, the poles' constant terms summed +/// once on the host ([`hoisted_constants`]). +pub fn deep_hoisted() -> bool { + match DEEP_HOISTED_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + static ENV: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENV.get_or_init(|| { + deep_hoisted_setting(std::env::var("LAMBDA_VM_DEEP_HOISTED").ok().as_deref()) + }) +} + +/// Fused DEEP launches through the hoisted kernel, and those that kept the +/// fused one while it was on (a `row_stride` other than 1), since the process +/// started. +static HOISTED_LAUNCHES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static HOISTED_FALLBACKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// One line for the log, in both arms: the setting and its launches. +pub fn deep_hoisted_report() -> String { + use std::sync::atomic::Ordering::Relaxed; + format!( + "{} · hoisted launches {} · fused kept {} (row stride ≠ 1)", + if deep_hoisted() { "on" } else { "off" }, + HOISTED_LAUNCHES.load(Relaxed), + HOISTED_FALLBACKS.load(Relaxed), + ) +} + +/// The hoisted DEEP's row-independent constants (raw ext3 words, interleaved): +/// `C_h = Σ_j γ_h[j]·h_ood[j]`, then for each pole `k` +/// `C_k = Σ_j γ_tr[j, k]·o[j, k]` over every trace column `j` (main, then aux), +/// in the fused kernel's layouts (`h_ood` / `gammas_h` a part each, +/// `trace_ood` / `gammas_tr` indexed `(j · K + k) · 3`). +pub fn hoisted_constants( + h_ood: &[u64], + trace_ood: &[u64], + gammas_h: &[u64], + gammas_tr: &[u64], + num_parts: usize, + num_cols: usize, + num_eval_points: usize, +) -> Vec { + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + type Fp3 = FieldElement; + let at = |v: &[u64], i: usize| -> Fp3 { + Fp3::new([ + FieldElement::::from_raw(v[3 * i]), + FieldElement::::from_raw(v[3 * i + 1]), + FieldElement::::from_raw(v[3 * i + 2]), + ]) + }; + let mut sums = + vec![(0..num_parts).fold(Fp3::zero(), |a, j| a + at(gammas_h, j) * at(h_ood, j))]; + for k in 0..num_eval_points { + sums.push((0..num_cols).fold(Fp3::zero(), |a, j| { + let i = j * num_eval_points + k; + a + at(gammas_tr, i) * at(trace_ood, i) + })); + } + sums.iter() + .flat_map(|e| { + [ + *e.value()[0].value(), + *e.value()[1].value(), + *e.value()[2].value(), + ] + }) + .collect() +} + +/// Largest `num_eval_points` the fused DEEP kernels take +/// (`deep_composition_ext3_fused_m1` … `_m4`, `M = 1 + num_eval_points`). +pub const FUSED_MAX_EVAL_POINTS: usize = 3; + +/// [`deep_composition_ext3_fully_resident_keep`] without the inverted +/// denominators buffer: each row inverts its own +/// `1 + num_eval_points` denominators `x - z` in registers with one base-field +/// inversion. `coset_points_dev` holds the domain points (`domain_size` u64, the +/// coset the buffered path's `compute_and_invert_denoms_ext3_dev` call reads) and +/// `z_scalars` the poles `[z^K, z·w^0, z·w^1, …]` (3 u64 each), the list that +/// call takes. The codeword equals the buffered kernel's as field elements. +/// +/// `Err(CUDA_ERROR_NOT_SUPPORTED)` for more than [`FUSED_MAX_EVAL_POINTS`] +/// points: the caller keeps the buffered path. +#[allow(clippy::too_many_arguments)] +pub fn deep_composition_ext3_fused_keep( + stream: &Arc, + main_lde: &GpuLdeBase, + aux_lde: Option<&GpuLdeExt3>, + h_parts_dev: &GpuLdeExt3, + coset_points_dev: &CudaSlice, + z_scalars: &[u64], + h_ood: &[u64], + trace_ood: &[u64], + gammas_h: &[u64], + gammas_tr: &[u64], + num_parts: usize, + num_main: usize, + num_aux: usize, + num_eval_points: usize, + row_stride: usize, + domain_size: usize, +) -> Result { + #[cfg(feature = "test-faults")] + crate::faults::check_sticky(&crate::faults::FAULT_DEEP_STICKY)?; + if num_eval_points > FUSED_MAX_EVAL_POINTS { + return Err(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_NOT_SUPPORTED, + )); + } + assert!( + domain_size.is_power_of_two() && domain_size >= 2, + "bit-reverse needs a power-of-two codeword" + ); + main_lde.wait_ready_on(stream)?; + if let Some(aux) = aux_lde { + aux.wait_ready_on(stream)?; + } + h_parts_dev.wait_ready_on(stream)?; + assert_eq!(main_lde.m, num_main); + assert_eq!(h_parts_dev.m, num_parts); + assert_eq!(h_parts_dev.lde_size, main_lde.lde_size); + if let Some(a) = aux_lde { + assert_eq!(a.m, num_aux); + assert_eq!(a.lde_size, main_lde.lde_size); + } else { + assert_eq!(num_aux, 0); + } + assert_eq!(h_ood.len(), num_parts * 3); + let num_total_cols = num_main + num_aux; + assert_eq!(trace_ood.len(), num_total_cols * num_eval_points * 3); + assert_eq!(gammas_h.len(), num_parts * 3); + assert_eq!(gammas_tr.len(), num_total_cols * num_eval_points * 3); + assert_eq!( + z_scalars.len(), + (1 + num_eval_points) * 3, + "one pole per term" + ); + assert!(coset_points_dev.len() >= domain_size, "one point per row"); + let max_row = (domain_size - 1) + .checked_mul(row_stride) + .expect("deep composition: (domain_size - 1) * row_stride overflow"); + assert!( + max_row < main_lde.lde_size, + "deep composition: kernel row {max_row} out of LDE stride {}", + main_lde.lde_size + ); + + let be = backend()?; + // With no trace terms (num_eval_points = 0) their tables are empty, and the + // driver refuses a zero-byte allocation: upload one unread zero instead. + let nonempty = |xs: &[u64]| stream.clone_htod(if xs.is_empty() { &[0u64][..] } else { xs }); + let (h_ood_dev, trace_ood_dev, gammas_h_dev, gammas_tr_dev, z_dev) = ( + nonempty(h_ood)?, + nonempty(trace_ood)?, + nonempty(gammas_h)?, + nonempty(gammas_tr)?, + stream.clone_htod(z_scalars)?, + ); + // SAFETY: every output slot is written by the kernel. + let mut deep_out = unsafe { stream.alloc::(domain_size * 3) }?; + let dummy_aux; + let aux_slice = if let Some(a) = aux_lde { + a.buf.as_ref() + } else { + dummy_aux = stream.alloc_zeros::(1)?; + &dummy_aux + }; + + let lde_stride = main_lde.lde_size as u64; + let num_main_u = num_main as u64; + let num_aux_u = num_aux as u64; + let num_parts_u = num_parts as u64; + let row_stride_u = row_stride as u64; + let domain_size_u = domain_size as u64; + // The buffered path's invertibility guard, on the same builds. + #[cfg(any(debug_assertions, feature = "test-faults"))] + let zero_flag = stream.alloc_zeros::(1)?; + #[cfg(any(debug_assertions, feature = "test-faults"))] + let flag_arg = &zero_flag; + #[cfg(not(any(debug_assertions, feature = "test-faults")))] + let flag_arg = &0u64; // nullptr: release keeps the pass free of host syncs + + let cfg = LaunchConfig { + grid_dim: ((domain_size as u32).div_ceil(128), 1, 1), + block_dim: (128, 1, 1), + shared_mem_bytes: 0, + }; + // With its row-independent part hoisted (`LAMBDA_VM_DEEP_HOISTED`): the + // same codeword, written canonical. The hoisted kernel reads every row at + // stride 1, the R4 callers' stride; any other keeps the fused kernel. + let hoisted = deep_hoisted() && row_stride == 1; + if deep_hoisted() && !hoisted { + HOISTED_FALLBACKS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + if hoisted { + let consts = stream.clone_htod(&hoisted_constants( + h_ood, + trace_ood, + gammas_h, + gammas_tr, + num_parts, + num_total_cols, + num_eval_points, + ))?; + unsafe { + stream + .launch_builder(&be.deep_composition_ext3_hoisted[num_eval_points]) + .arg(main_lde.buf.as_ref()) + .arg(aux_slice) + .arg(h_parts_dev.buf.as_ref()) + .arg(&lde_stride) + .arg(&num_main_u) + .arg(&num_aux_u) + .arg(&num_parts_u) + .arg(&domain_size_u) + .arg(&consts) + .arg(&gammas_h_dev) + .arg(&gammas_tr_dev) + .arg(coset_points_dev) + .arg(&z_dev) + .arg(&mut deep_out) + .arg(flag_arg) + .launch(cfg)?; + } + HOISTED_LAUNCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + drop(h_ood_dev); + drop(trace_ood_dev); + } else { + unsafe { + stream + .launch_builder(&be.deep_composition_ext3_fused[num_eval_points]) + .arg(main_lde.buf.as_ref()) + .arg(aux_slice) + .arg(h_parts_dev.buf.as_ref()) + .arg(&lde_stride) + .arg(&num_main_u) + .arg(&num_aux_u) + .arg(&num_parts_u) + .arg(&row_stride_u) + .arg(&domain_size_u) + .arg(&h_ood_dev) + .arg(&trace_ood_dev) + .arg(&gammas_h_dev) + .arg(&gammas_tr_dev) + .arg(coset_points_dev) + .arg(&z_dev) + .arg(&mut deep_out) + .arg(flag_arg) + .launch(cfg)?; + } + } + #[cfg(any(debug_assertions, feature = "test-faults"))] + { + let mut host = [0u32; 1]; + crate::readback::memcpy_dtoh(stream, &zero_flag, &mut host)?; + stream.synchronize()?; + assert_eq!(host[0], 0, "fused DEEP: a zero denominator has no inverse"); + } + into_fri_order(stream, &deep_out, domain_size) +} + #[allow(clippy::too_many_arguments)] fn deep_composition_ext3_impl( stream: &Arc, @@ -526,3 +843,130 @@ fn deep_composition_ext3_impl( pending.wait_into_u64(&mut out)?; Ok(out) } + +#[cfg(test)] +mod hoisted_tests { + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use rand::{Rng, SeedableRng}; + use rand_chacha::ChaCha8Rng; + + use super::{deep_hoisted_setting, hoisted_constants}; + + type Fp = FieldElement; + type Fp3 = FieldElement; + + fn fp3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) + } + + fn flat(v: &[Fp3]) -> Vec { + v.iter() + .flat_map(|e| { + [ + *e.value()[0].value(), + *e.value()[1].value(), + *e.value()[2].value(), + ] + }) + .collect() + } + + fn at(v: &[u64], i: usize) -> Fp3 { + Fp3::new([ + Fp::from_raw(v[3 * i]), + Fp::from_raw(v[3 * i + 1]), + Fp::from_raw(v[3 * i + 2]), + ]) + } + + #[test] + fn the_knob_reads_as_documented() { + for v in [Some(""), Some("0"), Some("off"), Some(" 0 ")] { + assert!(!deep_hoisted_setting(v), "{v:?}"); + } + for v in [None, Some("1"), Some("on"), Some(" 1 "), Some("2")] { + assert!(deep_hoisted_setting(v), "{v:?}"); + } + } + + /// The hoisted kernel's algebra on the host, in the kernel's layouts: for + /// random rows, `Σ γ(v − o)/(x − z)` term by term (the fused kernel) equals + /// `(Σ γ v − C)/(x − z)` pole by pole with [`hoisted_constants`]'s `C`, at + /// every pole count and with and without aux columns. + #[test] + fn the_hoisted_sum_is_the_fused_sum() { + let mut rng = ChaCha8Rng::seed_from_u64(7); + for (num_main, num_aux, parts, k) in [ + (5, 3, 2, 3), + (4, 0, 1, 0), + (3, 2, 2, 1), + (23, 3, 2, 2), + (1, 1, 3, 3), + ] { + let cols = num_main + num_aux; + let h_ood: Vec = (0..parts).map(|_| fp3(&mut rng)).collect(); + let gh: Vec = (0..parts).map(|_| fp3(&mut rng)).collect(); + let tood: Vec = (0..cols * k).map(|_| fp3(&mut rng)).collect(); + let gt: Vec = (0..cols * k).map(|_| fp3(&mut rng)).collect(); + let consts = hoisted_constants( + &flat(&h_ood), + &flat(&tood), + &flat(&gh), + &flat(>), + parts, + cols, + k, + ); + assert_eq!(consts.len(), 3 * (1 + k)); + for _ in 0..8 { + let inv: Vec = (0..1 + k).map(|_| fp3(&mut rng)).collect(); + let h: Vec = (0..parts).map(|_| fp3(&mut rng)).collect(); + let main: Vec = (0..num_main) + .map(|_| Fp::from_raw(rng.r#gen::())) + .collect(); + let aux: Vec = (0..num_aux).map(|_| fp3(&mut rng)).collect(); + let mut fused = Fp3::zero(); + for j in 0..parts { + fused += gh[j] * (h[j] - h_ood[j]) * inv[0]; + } + for (j, t) in main.iter().enumerate() { + for kk in 0..k { + let t3 = t.to_extension::(); + fused += gt[j * k + kk] * (t3 - tood[j * k + kk]) * inv[1 + kk]; + } + } + for (j, t) in aux.iter().enumerate() { + for kk in 0..k { + let i = (num_main + j) * k + kk; + fused += gt[i] * (*t - tood[i]) * inv[1 + kk]; + } + } + let mut acc_h = Fp3::zero(); + for j in 0..parts { + acc_h += gh[j] * h[j]; + } + let mut hoisted = (acc_h - at(&consts, 0)) * inv[0]; + for kk in 0..k { + let mut acc = Fp3::zero(); + for (j, t) in main.iter().enumerate() { + acc += gt[j * k + kk] * t.to_extension::(); + } + for (j, t) in aux.iter().enumerate() { + acc += gt[(num_main + j) * k + kk] * *t; + } + hoisted += (acc - at(&consts, 1 + kk)) * inv[1 + kk]; + } + assert_eq!( + fused, hoisted, + "main {num_main} aux {num_aux} parts {parts} k {k}" + ); + } + } + } +} diff --git a/crypto/math-cuda/src/deep_inv.rs b/crypto/math-cuda/src/deep_inv.rs new file mode 100644 index 000000000..a41f8e4d2 --- /dev/null +++ b/crypto/math-cuda/src/deep_inv.rs @@ -0,0 +1,62 @@ +//! Where the DEEP and out-of-domain (OOD) denominators are inverted. +//! +//! By default every row inverts its own few denominators in registers: the R3 +//! inverse buffers come from one row-wise kernel +//! ([`crate::inverse::invert_denoms_rowwise_dev`]), the fully resident R4 DEEP +//! kernel inverts its row's `1 + K` denominators itself +//! ([`crate::deep::deep_composition_ext3_fused_keep`]), and the single-point +//! OOD sums run row-chunked like the multi-point ones ([`crate::barycentric`]). +//! +//! `LAMBDA_VM_DEEP_INV_LEGACY=1` sends all three back to the legacy path — the +//! six-kernel global scan, the buffered DEEP kernel, one block per OOD column — +//! an A/B and rollback switch, not a tuning knob: both compute the same field +//! elements, so the same roots and proofs. Read once per process. + +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The legacy switch. Only `1` selects the legacy path; the banner +/// [`rowwise_enabled`] prints says which one a process took. +pub const LEGACY_ENV: &str = "LAMBDA_VM_DEEP_INV_LEGACY"; + +/// Whether this process inverts the DEEP and OOD denominators row-wise. +pub fn rowwise_enabled() -> bool { + static ENV: OnceLock = OnceLock::new(); + *ENV.get_or_init(|| { + let legacy = std::env::var(LEGACY_ENV).is_ok_and(|v| v == "1"); + // One line per process, so every box log states which path it ran. + eprintln!( + "[gpu] DEEP/OOD denominators: {}", + if legacy { + "the legacy scan, buffered DEEP, one block per OOD column \ + (LAMBDA_VM_DEEP_INV_LEGACY=1)" + } else { + "inverted row-wise, in the DEEP kernel itself, OOD sums row-chunked" + } + ); + !legacy + }) +} + +static ROWWISE_INVERSIONS: AtomicU64 = AtomicU64::new(0); +static CHUNKED_OOD_SUMS: AtomicU64 = AtomicU64::new(0); + +/// Denominator inversions the row-wise kernel has done, process-wide: the +/// counter a test reads to know the row-wise path, and not the scan, did the +/// work. +pub fn rowwise_inversions() -> u64 { + ROWWISE_INVERSIONS.load(Ordering::Relaxed) +} + +/// Single-point OOD sums the row-chunked kernel has done, process-wide. +pub fn chunked_ood_sums() -> u64 { + CHUNKED_OOD_SUMS.load(Ordering::Relaxed) +} + +pub(crate) fn count_rowwise_inversion() { + ROWWISE_INVERSIONS.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn count_chunked_ood_sum() { + CHUNKED_OOD_SUMS.fetch_add(1, Ordering::Relaxed); +} diff --git a/crypto/math-cuda/src/device.rs b/crypto/math-cuda/src/device.rs index 3a2f1db2a..72c4a428f 100644 --- a/crypto/math-cuda/src/device.rs +++ b/crypto/math-cuda/src/device.rs @@ -4,7 +4,7 @@ //! on a single CUDA context; a pool of streams lets rayon-parallel callers //! overlap H2D / compute / D2H. -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use cudarc::driver::{CudaContext, CudaFunction, CudaSlice, CudaStream}; @@ -15,6 +15,16 @@ use math::field::traits::IsFFTField; use crate::Result; use crate::ntt::{twiddles_forward, twiddles_inverse}; +/// Host bytes the pinned staging buffers hold ([`PinnedStaging`]), process +/// wide: page-locked, so they count in the process's resident set but in no +/// allocator's statistics. +static PINNED_HOST_BYTES: AtomicUsize = AtomicUsize::new(0); + +/// [`PINNED_HOST_BYTES`]: what the pinned staging buffers hold now. +pub fn pinned_host_bytes() -> usize { + PINNED_HOST_BYTES.load(Ordering::Relaxed) +} + /// Reusable pinned host staging buffer. Shared across all streams via a /// `Mutex` (see `Backend::pinned_staging`); the LDE call holds the lock /// across the D2H + memcpy-to-user-Vecs window. @@ -59,6 +69,10 @@ impl PinnedStaging { unsafe { let _ = cudarc::driver::sys::cuMemFreeHost(self.ptr as *mut _); } + PINNED_HOST_BYTES.fetch_sub( + self.capacity_elems * std::mem::size_of::(), + Ordering::Relaxed, + ); self.ptr = std::ptr::null_mut(); self.capacity_elems = 0; } @@ -67,6 +81,7 @@ impl PinnedStaging { let ptr = unsafe { cudarc::driver::result::malloc_host(bytes, 0 /* flags: non-WC */)? } as *mut u64; + PINNED_HOST_BYTES.fetch_add(bytes, Ordering::Relaxed); self.ptr = ptr; self.capacity_elems = new_cap; Ok(()) @@ -116,6 +131,10 @@ impl Drop for PinnedStaging { unsafe { let _ = cudarc::driver::sys::cuMemFreeHost(self.ptr as *mut _); } + PINNED_HOST_BYTES.fetch_sub( + self.capacity_elems * std::mem::size_of::(), + Ordering::Relaxed, + ); } } } @@ -127,7 +146,12 @@ impl Drop for PinnedStaging { // falls back to CPU. const ARITH_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/arith.cubin")); const NTT_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ntt.cubin")); +const NTT_CM_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ntt_cm.cubin")); const KECCAK_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/keccak.cubin")); +// `RPX_V0_CUBIN` and `RPX_LIMB_CUBIN`: `rpx.cu` at permutation variant 0 and at +// `rpx_paths::LIMB_PERMUTE_VARIANT`, one of which the backend loads (see +// `rpx_paths::limb_permute`). Generated by build.rs. +include!(concat!(env!("OUT_DIR"), "/rpx_cubins.rs")); const BARY_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/barycentric.cubin")); const DEEP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/deep.cubin")); const FRI_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/fri.cubin")); @@ -135,6 +159,9 @@ const INVERSE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/inverse.c const LOGUP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/logup.cubin")); const CONSTRAINT_INTERP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/constraint_interp.cubin")); +const BLAKE3_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/blake3.cubin")); +const SUMCHECK_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/sumcheck.cubin")); +const WHIR_FOLD_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/whir_fold.cubin")); /// Number of CUDA streams in the pool. Larger pools let many rayon-parallel /// callers overlap on the GPU without serializing on stream ownership. The @@ -144,7 +171,9 @@ const STREAM_POOL_SIZE: usize = 32; pub struct Backend { pub ctx: Arc, - streams: Vec>, + /// The stream pool, and the background pool beside it unless + /// `LAMBDA_VM_STREAM_CLASSES=off` ([`crate::stream_class`]). + streams: crate::stream_class::StreamPools>, /// Per-rayon-worker pinned staging buffers. Indexed by /// `rayon::current_thread_index()` (0 for non-rayon callers). Each slot /// grows lazily on first use, idle slots stay at zero allocation. @@ -155,15 +184,24 @@ pub struct Backend { /// alongside the LDE staging so the GPU→host D2H runs at PCIe line-rate. pinned_hashes: Vec>, util_stream: Arc, + /// Fixed-size double-buffered pinned pairs, one per concurrent + /// staged transfer. See [`htod_staged`]. + staging_pairs: StagingPool, /// Free-list of pre-created events for [`Backend::take_event`]. event_pool: Mutex>, - next: AtomicUsize, - /// VRAM budget (bytes) for table-session admission control. See + /// VRAM budget (bytes) for admission control. See /// [`detect_vram_budget_bytes`]. vram_budget_bytes: u64, + /// Device bytes promised to structures that are still alive. See + /// [`Backend::reserve`]. + reserved: AtomicU64, // arith.cubin pub vector_add_u64: CudaFunction, + pub widen_narrow_row_major: CudaFunction, + pub widen_narrow_col_major: CudaFunction, + pub column_max_col_major: CudaFunction, + pub pack_col_major: CudaFunction, pub gl_add: CudaFunction, pub gl_sub: CudaFunction, pub gl_mul: CudaFunction, @@ -174,8 +212,15 @@ pub struct Backend { // ntt.cubin pub bit_reverse_permute: CudaFunction, + pub lift_spread: CudaFunction, + pub mobius_level: CudaFunction, + pub mobius_low_levels: CudaFunction, + pub mobius_tile: CudaFunction, + pub mobius_wide_low: CudaFunction, + pub mobius_wide_tile: CudaFunction, pub ntt_dit_level: CudaFunction, pub ntt_dit_8_levels: CudaFunction, + pub ntt_dit_tile: CudaFunction, pub pointwise_mul: CudaFunction, pub scalar_mul: CudaFunction, pub bit_reverse_permute_batched: CudaFunction, @@ -190,18 +235,110 @@ pub struct Backend { pub pointwise_mul_row_major: CudaFunction, pub matrix_transpose_strided: CudaFunction, + // ntt_cm.cubin — the column-major LDE engine's passes, indexed by + // `k - 4` for a pass of k = 4..=8 levels (`crate::lde_cm`), and the WHIR + // encoding's wide DIT passes by `k - 9` for k = 9, 10 (`lde_cm::plan_wide`). + pub ntt_cm_dit: [CudaFunction; 5], + pub ntt_cm_dif: [CudaFunction; 5], + pub ntt_cm_dit_wide: [CudaFunction; 2], + // keccak.cubin pub keccak256_leaves_base_row_major_row_pair: CudaFunction, pub keccak256_leaves_base_row_major_row_pair_range: CudaFunction, + /// S2 one-row leaves (`rows_per_leaf = 1`): row `reverse_index(i)`, a column range. + pub keccak256_leaves_base_row_major_row_range: CudaFunction, pub keccak256_leaves_base_batched: CudaFunction, + pub keccak256_leaves_base_coset: CudaFunction, + pub keccak256_leaves_ext3_coset: CudaFunction, pub keccak256_leaves_base_row_pair_batched: CudaFunction, pub keccak256_leaves_ext3_batched: CudaFunction, pub grind_search: CudaFunction, pub keccak_comp_poly_leaves_ext3: CudaFunction, pub keccak_fri_leaves_ext3: CudaFunction, + /// S3 group-leaf FRI layers: `group` consecutive ext3 values per leaf. + pub keccak_fri_group_leaves_ext3: CudaFunction, pub keccak_merkle_level: CudaFunction, pub keccak_merkle_tail: CudaFunction, pub merkle_gather_paths: CudaFunction, + // blake3.cubin — the leaf kernels, the Merkle level/tail compressors, and + // the parity-harness probes that are the only host-visible handle on the + // device compression function, byte serialization and chain construction + // (see `kernels/blake3.cu`). Twin for twin with the keccak set above, and in + // the same order. `merkle_gather_paths` has no twin: path gathering copies + // nodes and never hashes, so it is hash-agnostic and both trees share it. + // + // Keccak stays the prover's default, so no production dispatch reaches these + // yet — they exist so the GPU can follow the CPU's hash switch (PA-PLAN §6.1). + pub blake3_leaves_base_row_major_row_pair: CudaFunction, + pub blake3_leaves_base_row_major_row_pair_range: CudaFunction, + /// S2 one-row leaves (`rows_per_leaf = 1`): row `reverse_index(i)`, a column range. + pub blake3_leaves_base_row_major_row_range: CudaFunction, + pub blake3_leaves_base_batched: CudaFunction, + pub blake3_leaves_base_row_pair_batched: CudaFunction, + pub blake3_leaves_ext3_batched: CudaFunction, + pub blake3_comp_poly_leaves_ext3: CudaFunction, + pub blake3_fri_leaves_ext3: CudaFunction, + /// S3 group-leaf FRI layers: `group` consecutive ext3 values per leaf. + pub blake3_fri_group_leaves_ext3: CudaFunction, + pub blake3_merkle_level: CudaFunction, + pub blake3_merkle_tail: CudaFunction, + pub blake3_compress_probe_6r: CudaFunction, + pub blake3_compress_probe_7r: CudaFunction, + pub blake3_compress_probe_default: CudaFunction, + pub blake3_rounds_probe: CudaFunction, + pub blake3_serialize_felts_probe: CudaFunction, + pub blake3_blocks_of_felts_probe: CudaFunction, + pub blake3_chain_probe: CudaFunction, + + // The RPX module — the RPX256 (XHash12) leaf kernels, Merkle level/tail + // compressors and the permutation probe (see `kernels/rpx.cu`). Twin for + // twin with the blake3 set above and in the same order; the probe is the + // only host-visible handle on the bare device permutation, which the parity + // tests check against the host `Rpx256`. + pub rpx_leaves_base_row_major_row_pair: CudaFunction, + pub rpx_leaves_base_row_major_row_pair_range: CudaFunction, + /// S2 one-row leaves (`rows_per_leaf = 1`): row `reverse_index(i)`, a column range. + pub rpx_leaves_base_row_major_row_range: CudaFunction, + pub rpx_leaves_base_batched: CudaFunction, + pub rpx_leaves_base_row_pair_batched: CudaFunction, + pub rpx_leaves_ext3_batched: CudaFunction, + pub rpx_comp_poly_leaves_ext3: CudaFunction, + pub rpx_fri_leaves_ext3: CudaFunction, + /// S3 group-leaf FRI layers: `group` consecutive ext3 values per leaf. + pub rpx_fri_group_leaves_ext3: CudaFunction, + pub rpx_merkle_level: CudaFunction, + pub rpx_merkle_tail: CudaFunction, + pub rpx_permute_probe: CudaFunction, + pub rpx_grind_search: CudaFunction, + /// ⛔ DIAGNOSTIC ONLY — the grind search with its executed-permutation + /// counters. Nothing on a proving path launches it; its one caller is + /// [`crate::grinding::search_counted`], which reads whether a slow launch + /// does MORE work or the same work more slowly. + pub rpx_grind_search_counted: CudaFunction, + /// The grind with nonces claimed from a work queue (the default RPX grind, + /// `rpx_paths::grind_queue`), and its counted twin (diagnostic, the same + /// counters as `rpx_grind_search_counted`). + pub rpx_grind_search_queue: CudaFunction, + pub rpx_grind_search_queue_counted: CudaFunction, + /// A Merkle level and the tail with one permutation per half-warp (the + /// default Merkle tops, `rpx_paths::warp_merkle`), and the half-warp + /// permutation's parity probe. + pub rpx_merkle_level_warp: CudaFunction, + pub rpx_merkle_tail_warp: CudaFunction, + pub rpx_permute_warp_probe: CudaFunction, + /// Permutation-variant probes: the limb primitives, and the permutation + /// chained at each variant the source instantiates, as `(variant, kernel)`. + pub rpx_limb_probe: CudaFunction, + pub rpx_permute_chain_probes: Vec<(u32, CudaFunction)>, + + // The RPX module — the algebraic hash's twins of the keccak entries above. + // Only the ones the WHIR path reaches are bound: the coset leaves, the two + // tree compressors and the grind. The row-group leaf kernels the per-table + // branch uses are in the cubin but are not loaded here, because nothing on + // this path launches them and an unused handle is a claim that something + // does. + pub rpx_leaves_base_coset: CudaFunction, + pub rpx_leaves_ext3_coset: CudaFunction, // barycentric.cubin pub barycentric_base_batched: CudaFunction, @@ -217,6 +354,14 @@ pub struct Backend { // deep.cubin pub deep_composition_ext3_row: CudaFunction, pub bit_reverse_ext3_kernel: CudaFunction, + /// `deep_composition_ext3_fused_m{M}` at index `M - 1`, the + /// DEEP row with its own `M = 1 + num_eval_points` denominators inverted in + /// registers. + pub deep_composition_ext3_fused: [CudaFunction; 4], + /// [`Self::deep_composition_ext3_fused`] with the row-independent part + /// hoisted (`LAMBDA_VM_DEEP_HOISTED`, `deep_composition_ext3_hoisted_m1` … + /// `_m4`). + pub deep_composition_ext3_hoisted: [CudaFunction; 4], // fri.cubin pub fri_fold_ext3: CudaFunction, @@ -231,6 +376,8 @@ pub struct Backend { pub apply_block_offsets_rev_ext3: CudaFunction, pub batch_inverse_combine_ext3: CudaFunction, pub invert_total_ext3: CudaFunction, + /// `invert_denoms_rowwise_ext3_k{K}` at index `K - 1`. + pub invert_denoms_rowwise_ext3: [CudaFunction; 8], pub logup_fingerprint_ext3: CudaFunction, pub logup_term_ext3: CudaFunction, pub logup_row_sum_ext3: CudaFunction, @@ -239,6 +386,76 @@ pub struct Backend { pub logup_finalize_accum_ext3: CudaFunction, pub logup_assemble_aux_ext3: CudaFunction, + // sumcheck.cubin + pub sumcheck_round_ext3: CudaFunction, + /// D-ARGUE S1-1: the round with integer interpolation nodes. + pub sumcheck_round_ext3_int: CudaFunction, + pub sum_partials_ext3: CudaFunction, + // D-ARGUE stage 1's fused zerocheck (`crate::argue_fused`). + pub zc_grid01: CudaFunction, + pub zc_bus_u: CudaFunction, + pub zc_fold2: CudaFunction, + pub zc_bus_column: CudaFunction, + pub zc_halve: CudaFunction, + pub zc_round_gruen: CudaFunction, + // D-ARGUE S1-3's GKR layer rounds (`crate::gkr::GruenLayer`). + pub gkr_eq_levels_ext3: CudaFunction, + pub gkr_round_gruen: CudaFunction, + pub gkr_gruen_finish: CudaFunction, + // The same over an input layer kept without its padding + // (`crate::gkr::DeviceFractionTree::from_truncated_input`). + pub gkr_round_gruen_real: CudaFunction, + pub gkr_gruen_finish_real: CudaFunction, + // The GKR round kernels whose last block reduces the sums into a slot + // (`crate::round_slot`). + pub gkr_round_gruen_slot: CudaFunction, + pub gkr_round_gruen_real_slot: CudaFunction, + // D-BATCH M1-2's input layer from the base columns (`crate::gkr::input_from_columns`). + pub gkr_input_from_columns: CudaFunction, + pub sumcheck_fold_ext3: CudaFunction, + pub mle_fold_base_ext3: CudaFunction, + pub eq_expand_level_ext3: CudaFunction, + pub eq_seed_shares_ext3: CudaFunction, + pub eq_expand_level_shares_ext3: CudaFunction, + pub program_map_ext3: CudaFunction, + pub factors_from_columns_ext3: CudaFunction, + pub mle_lift_base_ext3: CudaFunction, + pub add_scaled_ext3: CudaFunction, + pub fill_ext3: CudaFunction, + pub fraction_fold_ext3: CudaFunction, + pub fraction_fold_padded_ext3: CudaFunction, + pub mle_fold_base_ext3_many: CudaFunction, + pub eq_dot_base_ext3_many: CudaFunction, + pub batched_column_ext3: CudaFunction, + pub gather_factor_heads_ext3: CudaFunction, + + // whir_fold.cubin + pub whir_fold_base_ext3: CudaFunction, + pub whir_fold_ext3: CudaFunction, + pub gather_cosets: CudaFunction, + pub gather_cosets_strided: CudaFunction, + pub whir_fold_k_base_ext3: CudaFunction, + pub whir_fold_k_ext3: CudaFunction, + pub whir_lean_colmap: CudaFunction, + pub whir_lean_round: CudaFunction, + pub whir_lean_materialize: CudaFunction, + /// The lean rounds in tile form (`whir_open::TileSpec`): each column of + /// tiles' slots and weights, the one pass over the message (a thread a + /// tile, or a block a tile), a round over the slot tables, their fold, + /// the bound tables written from them, and that write compared with the + /// share form's. + pub whir_tile_plan: CudaFunction, + pub whir_tile_contract: CudaFunction, + pub whir_tile_contract_wide: CudaFunction, + pub whir_tile_round: CudaFunction, + pub whir_tile_fold: CudaFunction, + pub whir_tile_materialize: CudaFunction, + pub whir_tile_materialize_check: CudaFunction, + /// An interleaved ext3 array's limb as a plane, and back (a successor + /// encoded from its bound message, `whir::encode_successor`). + pub ext3_limb_gather: CudaFunction, + pub ext3_limb_scatter: CudaFunction, + // constraint_interp.cubin pub constraint_interp_kernel: CudaFunction, pub constraint_composition_kernel: CudaFunction, @@ -250,17 +467,186 @@ pub struct Backend { inv_twiddles: Mutex>>>>, } -/// Raise the device default memory pool's release threshold so freed -/// stream-ordered allocations are kept for reuse instead of returned to the OS -/// at each sync. Best-effort: any failure (e.g. a device/driver without -/// stream-ordered allocator support) leaves the default behaviour untouched. +/// The environment knob for the device default memory pool's release +/// threshold, in MiB: the bytes of freed stream-ordered memory the pool keeps +/// before handing memory back to the OS at the next sync. Unset means +/// [`DEFAULT_MEMPOOL_RELEASE_THRESHOLD_BYTES`]. The VRAM sampler runs set it +/// to `0`, so `total - free` reads the live working set and not the retained +/// pool. +pub const MEMPOOL_RELEASE_ENV: &str = "LAMBDA_VM_MEMPOOL_RELEASE_MB"; + +/// Retain every freed block (`u64::MAX`): a same-shape allocation skips the +/// driver and reuses the block, which is what the per-table pipeline's +/// repeated LDE/FRI buffers want. +/// +/// Measured, not guessed (RTX 5090, 2026-09-07, `one_lde_buffer::vram_arm` +/// at 2^21 × 316 @ blowup 2, five commits, in-process 1 kHz peak): retain-all +/// 1176.9 / 1057.9 / 1055.3 / 1056.2 / 1055.6 ms against release-0 +/// 1178.5 / 1057.2 / 1077.5 / 1081.2 / 1079.3 ms — retention ≈2% faster once +/// the first commit has populated the pool — and a peak of 15.67 GiB under +/// both: a same-shape allocation reuses the retained block, so retention adds +/// nothing to the peak. The unequal-shape case is covered at block scale by +/// the multi-table q=41 wrap rung under this default (VRAM peak 28,976 MiB, no +/// device decline): the stream-ordered allocator serves a new request from the +/// physical chunks it retains, and the release threshold governs only what a +/// sync hands back to the OS. The explicit release for a moment reuse cannot +/// serve is [`Backend::trim_mempool_to`]; the sampler runs set the knob to `0` +/// so `total - free` reads the live set rather than the pool. +pub const DEFAULT_MEMPOOL_RELEASE_THRESHOLD_BYTES: u64 = u64::MAX; + +/// The effective release threshold in bytes: the knob when set and parseable, +/// the default otherwise. Read once per process; the prover's diagnostics +/// print it so every box log states the posture its run had. +pub fn mempool_release_threshold_bytes() -> u64 { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + std::env::var(MEMPOOL_RELEASE_ENV) + .ok() + .and_then(|s| s.parse::().ok()) + .map(|mb| mb.saturating_mul(1024 * 1024)) + .unwrap_or(DEFAULT_MEMPOOL_RELEASE_THRESHOLD_BYTES) + }) +} + +/// The device default memory pool, or `None` on a device/driver without +/// stream-ordered allocator support. +/// +/// # Safety +/// +/// `ctx` must be a live context; its device is queried directly. +unsafe fn default_mempool(ctx: &CudaContext) -> Option { + use cudarc::driver::sys; + let mut pool: sys::CUmemoryPool = std::ptr::null_mut(); + // SAFETY: the out-pointer is a valid stack slot; the device is the + // context's own. + unsafe { + sys::cuDeviceGetDefaultMemPool(&mut pool as *mut _, ctx.cu_device()) + .result() + .ok() + .map(|()| pool) + } +} + +/// Set the device default memory pool's release threshold +/// ([`mempool_release_threshold_bytes`]) so freed stream-ordered allocations +/// are kept for reuse instead of returned to the OS at each sync. Best-effort: +/// any failure leaves the driver default (release everything) untouched, and +/// the one-line report says so. fn retain_default_mempool(ctx: &CudaContext) { use cudarc::driver::sys; - // SAFETY: raw CUDA driver calls. `ctx.cu_device()` is a valid device for - // the just-created context; the out-pointers are valid stack slots; the + let threshold = mempool_release_threshold_bytes(); + // SAFETY: raw CUDA driver calls on the just-created context's device; the // threshold is read as a u64 by the driver. Errors are swallowed. + let set = unsafe { + default_mempool(ctx).is_some_and(|pool| { + sys::cuMemPoolSetAttribute( + pool, + sys::CUmemPool_attribute_enum::CU_MEMPOOL_ATTR_RELEASE_THRESHOLD, + &threshold as *const u64 as *mut core::ffi::c_void, + ) + .result() + .is_ok() + }) + }; + // One line per process, so the box log states the posture the run had. + eprintln!( + "[gpu] mempool release threshold: {}{}", + match threshold { + u64::MAX => "retain all freed blocks".to_string(), + t => format!("{} MiB", t >> 20), + }, + if set { + "" + } else { + " (driver refused; the release-on-sync default stays)" + } + ); +} + +/// Device bytes held for as long as this lives. See [`Backend::reserve`]. +/// +/// ★ The count is atomic and the reservation is GROWABLE, because a chain +/// promises its room once and then discovers more of it: the tree a commitment +/// caches is not known when the codeword reserves, and it is shared through an +/// `Arc` by the time it is. Growing this rather than taking a second +/// reservation is what keeps ONE number answering "what does this chain hold" — +/// two accountings for one working set is a shape this codebase has paid for +/// before. +#[derive(Debug)] +pub struct DeviceReservation { + bytes: AtomicU64, +} + +impl DeviceReservation { + /// Bytes this reservation currently accounts for. + pub fn bytes(&self) -> u64 { + self.bytes.load(Ordering::Relaxed) + } + + /// Promise `extra` more against the same budget, under this reservation. + /// + /// Returns false and changes nothing if the budget will not take it — the + /// caller then does without whatever it wanted the bytes for, rather than + /// holding memory the accounting cannot see. + pub fn grow(&self, extra: u64) -> bool { + let Ok(be) = backend() else { return false }; + let mut held = be.reserved.load(Ordering::Relaxed); + loop { + if held.saturating_add(extra) > be.vram_budget_bytes { + return false; + } + match be.reserved.compare_exchange_weak( + held, + held + extra, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + self.bytes.fetch_add(extra, Ordering::Relaxed); + note_reserved(held + extra); + return true; + } + Err(seen) => held = seen, + } + } + } + + /// Give `given` of them back, when what they were promised for is dropped + /// before the reservation is. + pub fn shrink(&self, given: u64) { + let given = given.min(self.bytes.load(Ordering::Relaxed)); + if given == 0 { + return; + } + self.bytes.fetch_sub(given, Ordering::Relaxed); + if let Ok(be) = backend() { + be.reserved.fetch_sub(given, Ordering::Relaxed); + } + } +} + +impl Drop for DeviceReservation { + fn drop(&mut self) { + if let Ok(be) = backend() { + be.reserved + .fetch_sub(self.bytes.load(Ordering::Relaxed), Ordering::Relaxed); + } + } +} + +/// Hands the device default memory pool's retained blocks back to the OS. +/// +/// The pool keeps freed stream-ordered allocations forever by design (see +/// [`retain_default_mempool`]), which is what makes repeated allocations cheap +/// — and also what makes a *new* shape of allocation fail while the pool sits +/// on memory it is not using. Best-effort: a failure leaves things as they are. +fn trim_default_mempool() { + use cudarc::driver::sys; + let Ok(be) = backend() else { return }; + // SAFETY: raw driver calls. The device is the backend's, the out-pointer a + // stack slot, and the target size is read as a u64. Errors are swallowed. unsafe { - let dev = ctx.cu_device(); + let dev = be.ctx.cu_device(); let mut pool: sys::CUmemoryPool = std::ptr::null_mut(); if sys::cuDeviceGetDefaultMemPool(&mut pool as *mut _, dev) .result() @@ -268,21 +654,312 @@ fn retain_default_mempool(ctx: &CudaContext) { { return; } - // Default: retain freed stream-ordered blocks indefinitely (u64::MAX) - // for reuse. `LAMBDA_VM_MEMPOOL_RELEASE_MB` overrides the cap (bytes the - // pool keeps before returning memory to the OS) when retained-pool - // growth needs bounding. - let threshold: u64 = std::env::var("LAMBDA_VM_MEMPOOL_RELEASE_MB") - .ok() - .and_then(|s| s.parse::().ok()) - .map(|mb| mb.saturating_mul(1024 * 1024)) - .unwrap_or(u64::MAX); - let _ = sys::cuMemPoolSetAttribute( + let _ = sys::cuMemPoolTrimTo(pool, 0).result(); + } +} + +/// Lands every stream's pending frees and hands the pool's retained blocks +/// back. +/// +/// The frees are stream-ordered, so a buffer dropped on another stream is not +/// free until that stream reaches the drop — and the pool cannot return what +/// it has not been given yet. Draining the whole context first is what makes +/// the trim worth doing. +/// +/// Public because a test that asks the driver how much memory this process has +/// taken needs the pool empty first, or it is measuring the pool's retention +/// instead of the caller's — see `a_group_holds_only_its_codewords_before_any_open`. +pub fn drain_and_trim() -> Result<()> { + TRIMS.fetch_add(1, Ordering::Relaxed); + let be = backend()?; + be.ctx.synchronize()?; + trim_default_mempool(); + Ok(()) +} + +/// Calls to [`drain_and_trim`] this process has made. Each one waits for every +/// stream of the context, so work running on two threads at once stalls +/// together on it. Monotone: a reader brackets the span it cares about. +static TRIMS: AtomicU64 = AtomicU64::new(0); + +/// [`TRIMS`]: the [`drain_and_trim`] calls so far. +pub fn trims() -> u64 { + TRIMS.load(Ordering::Relaxed) +} + +/// ★ Bytes the default memory pool has handed out and not taken back: `(now, +/// the most at any instant since the last [`reset_pool_high_water`])`. +/// +/// Every allocation this crate makes is stream-ordered and comes from that +/// pool — cudarc allocates with `cuMemAllocAsync` whenever the device has +/// pools, which every card this runs on does — so the second number is the +/// PEAK of what the process held. That is what a working-set claim is tested +/// against: a reservation says what was promised, and `cuMemGetInfo` only what +/// is free at the instant it is asked. Drain first ([`drain_and_trim`]): a free +/// counts when its stream reaches it. +pub fn pool_used_bytes() -> Result<(u64, u64)> { + use cudarc::driver::sys; + let be = backend()?; + let read = |attribute| -> Result { + let mut value = 0u64; + // SAFETY: raw driver calls on the backend's own device; the value is a + // u64 stack slot, the type both attributes are documented to write. + unsafe { + let pool = default_mempool(&be.ctx).ok_or(cudarc::driver::DriverError( + sys::CUresult::CUDA_ERROR_NOT_SUPPORTED, + ))?; + sys::cuMemPoolGetAttribute( + pool, + attribute, + &mut value as *mut u64 as *mut core::ffi::c_void, + ) + .result()?; + } + Ok(value) + }; + Ok(( + read(sys::CUmemPool_attribute_enum::CU_MEMPOOL_ATTR_USED_MEM_CURRENT)?, + read(sys::CUmemPool_attribute_enum::CU_MEMPOOL_ATTR_USED_MEM_HIGH)?, + )) +} + +/// Restarts [`pool_used_bytes`]'s high-water mark. The driver only resets it +/// to zero, and the next allocation raises it to what is then in use — so a +/// reader takes the larger of it and the `now` it read before the reset. +pub fn reset_pool_high_water() -> Result<()> { + use cudarc::driver::sys; + let be = backend()?; + let zero = 0u64; + // SAFETY: as in `pool_used_bytes`; zero is the one value the driver takes + // for this attribute. + unsafe { + let pool = default_mempool(&be.ctx).ok_or(cudarc::driver::DriverError( + sys::CUresult::CUDA_ERROR_NOT_SUPPORTED, + ))?; + sys::cuMemPoolSetAttribute( pool, - sys::CUmemPool_attribute_enum::CU_MEMPOOL_ATTR_RELEASE_THRESHOLD, - &threshold as *const u64 as *mut core::ffi::c_void, + sys::CUmemPool_attribute_enum::CU_MEMPOOL_ATTR_USED_MEM_HIGH, + &zero as *const u64 as *mut core::ffi::c_void, ) - .result(); + .result()?; + } + Ok(()) +} + +/// Promises `bytes` against the budget for a caller whose structure outlives +/// the type that spends them. +pub fn reserve(bytes: u64) -> Option { + backend().ok()?.reserve(bytes) +} + +/// [`reserve`] that never asks the retention for its spare bytes: a caller +/// beside another proof's work, whose cache it must not take, waits instead. +pub fn reserve_without_eviction(bytes: u64) -> Option { + backend().ok()?.reserve_with(bytes, false) +} + +/// Argue-surface device fallbacks: the reservation refusals in math-cuda's +/// `sumcheck`, `gkr` and `columns`, and the GKR tree's whole-tree promise in +/// `multilinear::gpu`, counted where each one's `reserve` returns `None` and its +/// work moves to the host. +/// +/// ⛔ WHY THIS EXISTS. Until this counter the only fallback number the campaign +/// read was `multilinear::gpu::host_fallbacks()`, which has ONE caller — the +/// COMMIT path (`multilinear/src/whir_chain.rs`) — so every `host fallbacks 0` +/// certified that no COMMITMENT fell back and said NOTHING about the per-table +/// ARGUMENT. wt16 was net-negative for exactly that blind spot: the leaf-layer +/// retention grew `be.reserved`, argue's `reserve` then refused and moved to +/// the host UNCOUNTED, and the slot-level reading looked like a clean win. +/// Read beside `host_fallbacks()`, this makes "the device did the work" +/// distinguishable from "it quietly did not" on the argue surface. +/// +/// SCOPE, stated precisely, by function (line numbers go stale). SIX sites +/// count here, each where its reservation is refused and its work moves to the +/// host: +/// - the FIVE in `crypto/math-cuda/src`: `sumcheck.rs` (×3), `gkr.rs` (×1), +/// `columns.rs` (×1); +/// - the GKR tree's whole-tree promise in `multilinear::gpu`'s +/// `input_layer_tree_impl`, on the consume path only: the host then builds +/// the table's factors and fraction tree. `multilinear::gpu::gkr_tree_refusals` +/// counts it apart as well. +/// +/// The other argue-side `reserve` calls in `multilinear::gpu` are NOT +/// fallbacks, and must never be counted here: +/// - `input_layer_tree_impl`'s carry (`reserve(eager)`) is SPECULATIVE: its +/// `None` picks the lazy tree, still on the device; +/// - a prefetch's (`input_layer_tree_deferred`) refusal only means no prefetch; +/// - a kept input layer's tree (`truncated_tree`, `LAMBDA_VM_ARGUE_INPUT_REAL`) +/// is also speculative: its `None` builds today's tree instead; +/// - `reserve_room`'s two callers keep the work on the device. `take_turn` opens +/// unpromised and counts its own refusals (`room_turn_refusals`); the group +/// commit's retention room makes the commit transient, so its leaf pass is +/// redone. +/// +/// This is NOT the whole device surface: the commit path has its own counters +/// (`multilinear::gpu::host_fallbacks`, `commit_errors`). Putting a wrong site +/// into the very counter meant to end false numbers is the one thing to avoid. +static DEVICE_FALLBACKS: AtomicU64 = AtomicU64::new(0); + +/// Argue-surface device fallbacks this process has taken — see +/// [`DEVICE_FALLBACKS`] for the enumerated sites and the scope it does not +/// cover. Read alongside `multilinear::gpu::host_fallbacks()` (the commit-side +/// count) for both surfaces. +pub fn device_fallbacks() -> u64 { + DEVICE_FALLBACKS.load(Ordering::Relaxed) +} + +/// Zero the process-wide counter. For a test that wants to assert a delta, and +/// for a harness that reads one prove's worth from a reused process. +pub fn reset_device_fallbacks() { + DEVICE_FALLBACKS.store(0, Ordering::Relaxed); +} + +/// Record one argue-side reservation refusal — bumped at each of the six +/// sites [`DEVICE_FALLBACKS`] enumerates, and nowhere else. +pub fn note_device_fallback() { + DEVICE_FALLBACKS.fetch_add(1, Ordering::Relaxed); +} + +/// ★ The high-water mark of `be.reserved` — the PEAK simultaneous device +/// reservation this process reached, updated wherever the total rises +/// ([`Backend::reserve`] and [`DeviceReservation::grow`]). +/// +/// This is the reservation quantity argue's `reserve` is checked against, which +/// the raw device trace cannot report: the raw peak includes the never-purge +/// pool's retained blocks and sits above the reservation budget, while THIS is +/// exactly what the budget gates. A control run reads argue's peak reservation +/// demand here; and while the evictable retention holds only spare bytes, this +/// stays below the budget by construction. +static RESERVED_HIGH_WATER: AtomicU64 = AtomicU64::new(0); + +/// ★ The same peak over a WINDOW a caller opens and closes — one phase of one +/// proof — beside the whole-run one, which it never touches. +/// +/// The whole-run number says how close the process came to the budget; it +/// cannot say WHEN. A change that moves one phase's reservations — giving a +/// group's room back across the argument, say — is tested by that phase's peak, +/// epoch by epoch, and a run's single maximum hides every epoch but one. +static WINDOW_HIGH_WATER: AtomicU64 = AtomicU64::new(0); + +/// Note that `be.reserved` just rose to `now`, keeping both peaks. +fn note_reserved(now: u64) { + RESERVED_HIGH_WATER.fetch_max(now, Ordering::Relaxed); + WINDOW_HIGH_WATER.fetch_max(now, Ordering::Relaxed); +} + +/// The peak simultaneous device reservation this process reached — see +/// [`RESERVED_HIGH_WATER`]. +pub fn reserved_high_water() -> u64 { + RESERVED_HIGH_WATER.load(Ordering::Relaxed) +} + +/// Zero the reservation high-water. For a test asserting a delta. +pub fn reset_reserved_high_water() { + RESERVED_HIGH_WATER.store(0, Ordering::Relaxed); +} + +/// Opens a window ([`WINDOW_HIGH_WATER`]): its peak starts at what is promised +/// NOW, not at zero, because the total only reports when it RISES — a phase +/// that reserved nothing peaked at whatever it inherited, and a window started +/// at zero would read that as nothing held at all. +/// +/// Process-wide, like the whole-run peak: a window is one phase's only while +/// nothing else in the process reserves during it. The WHIR base proves one +/// epoch at a time; a caller that cannot say the same must mark what it reads. +pub fn reset_window_high_water() { + let now = backend().map(|be| be.reserved_bytes()).unwrap_or(0); + WINDOW_HIGH_WATER.store(now, Ordering::Relaxed); +} + +/// The peak promised since the last [`reset_window_high_water`]. +pub fn window_high_water() -> u64 { + WINDOW_HIGH_WATER.load(Ordering::Relaxed) +} + +/// ★ THE RETENTION EVICTOR — the callback the WHIR leaf-layer retention installs +/// so the allocator can reclaim spare retained bytes UNDER PRESSURE, on a real +/// caller's behalf, without `device.rs` depending on `whir.rs`. +/// +/// This is what makes the retention a PRIORITY scheme, not a byte race +/// ([[gpu-two-vramgates-overlap]]): it holds only genuinely-spare bytes, and the +/// instant a real caller (argue) cannot get its reservation, the layers are +/// given back. A plain `fn` pointer — Send + Sync, no allocation — installed once +/// via `OnceLock`; it takes a byte TARGET and returns how many it actually freed. +/// +/// ⛔ CONTRACT the evictor MUST honour (see `whir::evict_retained_layers`): it +/// frees by dropping retained layers and calling `DeviceReservation::shrink`, and +/// it MUST NOT call `reserve` or `grow` — those would re-enter the allocator that +/// called it and deadlock. +static EVICTOR: OnceLock u64> = OnceLock::new(); + +/// Install the retention evictor. Idempotent — only the first install sticks. +pub fn set_retention_evictor(evictor: fn(u64) -> u64) { + let _ = EVICTOR.set(evictor); +} + +/// Ask the retention to give back at least `target` bytes of budget; returns how +/// many it freed (0 if none is installed or nothing was reclaimable). +fn try_evict_retained(target: u64) -> u64 { + match EVICTOR.get() { + Some(evict) => evict(target), + None => 0, + } +} + +/// Allocates on `stream`, and if the device says no, gives the pool's retained +/// blocks back and asks once more. +/// +/// "Out of memory" from the stream-ordered allocator usually means the pool is +/// holding blocks of the wrong shape, not that the device is full — a second +/// prover on the same card is enough. It is worth one retry: a prove whose +/// tables are already on the device has nowhere to fall back to, so a failure +/// here is a failed proof. +/// +/// # Safety +/// The caller must write every element before reading it, as with +/// `CudaStream::alloc`. +pub unsafe fn alloc_or_trim( + stream: &Arc, + len: usize, +) -> Result> { + // SAFETY: the caller's, forwarded. + match unsafe { stream.alloc::(len) } { + Ok(slice) => Ok(slice), + Err(_) => { + drain_and_trim()?; + // SAFETY: the caller's, forwarded. + unsafe { stream.alloc::(len) } + } + } +} + +/// Uploads on `stream`, with the same one retry as [`alloc_or_trim`]: a copy +/// to the device allocates too, and the small ones are no less fatal for being +/// small. +pub fn htod_or_trim( + stream: &Arc, + src: &[T], +) -> Result> { + match stream.clone_htod(src) { + Ok(slice) => Ok(slice), + Err(_) => { + drain_and_trim()?; + stream.clone_htod(src) + } + } +} + +/// The same, zeroed. +pub fn alloc_zeros_or_trim( + stream: &Arc, + len: usize, +) -> Result> { + match stream.alloc_zeros::(len) { + Ok(slice) => Ok(slice), + Err(_) => { + drain_and_trim()?; + stream.alloc_zeros::(len) + } } } @@ -344,7 +1021,16 @@ impl Backend { let arith = ctx.load_module(Ptx::from_binary(ARITH_CUBIN.to_vec()))?; let ntt = ctx.load_module(Ptx::from_binary(NTT_CUBIN.to_vec()))?; + let ntt_cm = ctx.load_module(Ptx::from_binary(NTT_CM_CUBIN.to_vec()))?; let keccak = ctx.load_module(Ptx::from_binary(KECCAK_CUBIN.to_vec()))?; + // The whole RPX module at the permutation `rpx_paths::limb_permute` + // picks: every RPX kernel of the process runs that variant. + let rpx_cubin = if crate::rpx_paths::limb_permute() { + RPX_LIMB_CUBIN + } else { + RPX_V0_CUBIN + }; + let rpx = ctx.load_module(Ptx::from_binary(rpx_cubin.to_vec()))?; let bary = ctx.load_module(Ptx::from_binary(BARY_CUBIN.to_vec()))?; let deep = ctx.load_module(Ptx::from_binary(DEEP_CUBIN.to_vec()))?; let fri = ctx.load_module(Ptx::from_binary(FRI_CUBIN.to_vec()))?; @@ -352,10 +1038,42 @@ impl Backend { let logup = ctx.load_module(Ptx::from_binary(LOGUP_CUBIN.to_vec()))?; let constraint_interp = ctx.load_module(Ptx::from_binary(CONSTRAINT_INTERP_CUBIN.to_vec()))?; + let blake3 = ctx.load_module(Ptx::from_binary(BLAKE3_CUBIN.to_vec()))?; + let sumcheck = ctx.load_module(Ptx::from_binary(SUMCHECK_CUBIN.to_vec()))?; + let whir_fold = ctx.load_module(Ptx::from_binary(WHIR_FOLD_CUBIN.to_vec()))?; + // The stream classes (`LAMBDA_VM_STREAM_CLASSES`): off, every stream as + // it always was; otherwise a background pool beside the pool, under + // `priority` the pool at the device's greatest priority and the + // background at its least. + let classes = crate::stream_class::setting(); + let (foreground_priority, background_priority) = match classes { + crate::stream_class::StreamClasses::Priority => { + ctx.bind_to_thread()?; + let (least, greatest) = cudarc::driver::result::stream::get_priority_range()?; + (Some(greatest), Some(least)) + } + _ => (None, None), + }; + let make_stream = |priority: Option| match priority { + Some(p) => ctx.new_stream_with_priority(p), + None => ctx.new_stream(), + }; let mut streams = Vec::with_capacity(STREAM_POOL_SIZE); for _ in 0..STREAM_POOL_SIZE { - streams.push(ctx.new_stream()?); + streams.push(make_stream(foreground_priority)?); + } + let mut background = Vec::new(); + if classes.split() { + for _ in 0..crate::stream_class::BACKGROUND_STREAMS { + background.push(make_stream(background_priority)?); + } + eprintln!( + "[gpu] stream classes: {classes:?} · {STREAM_POOL_SIZE} streams at priority {} · {} background streams at priority {}", + foreground_priority.unwrap_or(0), + background.len(), + background_priority.unwrap_or(0), + ); } // One slot per rayon worker. `current_thread_index()` returns // `0..current_num_threads()`, and non-rayon callers (None) map to slot 0, @@ -391,7 +1109,7 @@ impl Backend { }; // Separate "utility" stream for twiddle uploads and other bookkeeping; // not part of the pool that callers rotate through. - let util_stream = ctx.new_stream()?; + let util_stream = make_stream(foreground_priority)?; // Cache is indexed by log_n. Valid range is [0, TWO_ADICITY] since // Goldilocks has roots of unity for orders 2^0..=2^TWO_ADICITY only. @@ -402,6 +1120,10 @@ impl Backend { Ok(Self { vector_add_u64: arith.load_function("vector_add_u64")?, + widen_narrow_row_major: arith.load_function("widen_narrow_row_major")?, + widen_narrow_col_major: arith.load_function("widen_narrow_col_major")?, + column_max_col_major: arith.load_function("column_max_col_major")?, + pack_col_major: arith.load_function("pack_col_major")?, gl_add: arith.load_function("gl_add_kernel")?, gl_sub: arith.load_function("gl_sub_kernel")?, gl_mul: arith.load_function("gl_mul_kernel")?, @@ -410,8 +1132,15 @@ impl Backend { ext3_add: arith.load_function("ext3_add_kernel")?, ext3_sub: arith.load_function("ext3_sub_kernel")?, bit_reverse_permute: ntt.load_function("bit_reverse_permute")?, + lift_spread: ntt.load_function("lift_spread")?, + mobius_level: ntt.load_function("mobius_level")?, + mobius_low_levels: ntt.load_function("mobius_low_levels")?, + mobius_tile: ntt.load_function("mobius_tile")?, + mobius_wide_low: ntt.load_function("mobius_wide_low")?, + mobius_wide_tile: ntt.load_function("mobius_wide_tile")?, ntt_dit_level: ntt.load_function("ntt_dit_level")?, ntt_dit_8_levels: ntt.load_function("ntt_dit_8_levels")?, + ntt_dit_tile: ntt.load_function("ntt_dit_tile")?, pointwise_mul: ntt.load_function("pointwise_mul")?, scalar_mul: ntt.load_function("scalar_mul")?, bit_reverse_permute_batched: ntt.load_function("bit_reverse_permute_batched")?, @@ -424,20 +1153,102 @@ impl Backend { ntt_dit_8_levels_row_major: ntt.load_function("ntt_dit_8_levels_row_major")?, pointwise_mul_row_major: ntt.load_function("pointwise_mul_row_major")?, matrix_transpose_strided: ntt.load_function("matrix_transpose_strided")?, + ntt_cm_dit: [ + ntt_cm.load_function("ntt_cm_dit_k4")?, + ntt_cm.load_function("ntt_cm_dit_k5")?, + ntt_cm.load_function("ntt_cm_dit_k6")?, + ntt_cm.load_function("ntt_cm_dit_k7")?, + ntt_cm.load_function("ntt_cm_dit_k8")?, + ], + ntt_cm_dif: [ + ntt_cm.load_function("ntt_cm_dif_k4")?, + ntt_cm.load_function("ntt_cm_dif_k5")?, + ntt_cm.load_function("ntt_cm_dif_k6")?, + ntt_cm.load_function("ntt_cm_dif_k7")?, + ntt_cm.load_function("ntt_cm_dif_k8")?, + ], + ntt_cm_dit_wide: [ + ntt_cm.load_function("ntt_cm_dit_k9")?, + ntt_cm.load_function("ntt_cm_dit_k10")?, + ], keccak256_leaves_base_row_major_row_pair: keccak .load_function("keccak256_leaves_base_row_major_row_pair")?, keccak256_leaves_base_row_major_row_pair_range: keccak .load_function("keccak256_leaves_base_row_major_row_pair_range")?, + keccak256_leaves_base_row_major_row_range: keccak + .load_function("keccak256_leaves_base_row_major_row_range")?, keccak256_leaves_base_batched: keccak.load_function("keccak256_leaves_base_batched")?, + keccak256_leaves_base_coset: keccak.load_function("keccak256_leaves_base_coset")?, + keccak256_leaves_ext3_coset: keccak.load_function("keccak256_leaves_ext3_coset")?, keccak256_leaves_base_row_pair_batched: keccak .load_function("keccak256_leaves_base_row_pair_batched")?, keccak256_leaves_ext3_batched: keccak.load_function("keccak256_leaves_ext3_batched")?, grind_search: keccak.load_function("grind_search")?, keccak_comp_poly_leaves_ext3: keccak.load_function("keccak_comp_poly_leaves_ext3")?, keccak_fri_leaves_ext3: keccak.load_function("keccak_fri_leaves_ext3")?, + keccak_fri_group_leaves_ext3: keccak.load_function("keccak_fri_group_leaves_ext3")?, keccak_merkle_level: keccak.load_function("keccak_merkle_level")?, keccak_merkle_tail: keccak.load_function("keccak_merkle_tail")?, merkle_gather_paths: keccak.load_function("merkle_gather_paths")?, + blake3_leaves_base_row_major_row_pair: blake3 + .load_function("blake3_leaves_base_row_major_row_pair")?, + blake3_leaves_base_row_major_row_pair_range: blake3 + .load_function("blake3_leaves_base_row_major_row_pair_range")?, + blake3_leaves_base_row_major_row_range: blake3 + .load_function("blake3_leaves_base_row_major_row_range")?, + blake3_leaves_base_batched: blake3.load_function("blake3_leaves_base_batched")?, + blake3_leaves_base_row_pair_batched: blake3 + .load_function("blake3_leaves_base_row_pair_batched")?, + blake3_leaves_ext3_batched: blake3.load_function("blake3_leaves_ext3_batched")?, + blake3_comp_poly_leaves_ext3: blake3.load_function("blake3_comp_poly_leaves_ext3")?, + blake3_fri_leaves_ext3: blake3.load_function("blake3_fri_leaves_ext3")?, + blake3_fri_group_leaves_ext3: blake3.load_function("blake3_fri_group_leaves_ext3")?, + blake3_merkle_level: blake3.load_function("blake3_merkle_level")?, + blake3_merkle_tail: blake3.load_function("blake3_merkle_tail")?, + blake3_compress_probe_6r: blake3.load_function("blake3_compress_probe_6r")?, + blake3_compress_probe_7r: blake3.load_function("blake3_compress_probe_7r")?, + blake3_compress_probe_default: blake3.load_function("blake3_compress_probe_default")?, + blake3_rounds_probe: blake3.load_function("blake3_rounds_probe")?, + blake3_serialize_felts_probe: blake3.load_function("blake3_serialize_felts_probe")?, + blake3_blocks_of_felts_probe: blake3.load_function("blake3_blocks_of_felts_probe")?, + blake3_chain_probe: blake3.load_function("blake3_chain_probe")?, + + rpx_leaves_base_row_major_row_pair: rpx + .load_function("rpx_leaves_base_row_major_row_pair")?, + rpx_leaves_base_row_major_row_pair_range: rpx + .load_function("rpx_leaves_base_row_major_row_pair_range")?, + rpx_leaves_base_row_major_row_range: rpx + .load_function("rpx_leaves_base_row_major_row_range")?, + rpx_leaves_base_batched: rpx.load_function("rpx_leaves_base_batched")?, + rpx_leaves_base_row_pair_batched: rpx + .load_function("rpx_leaves_base_row_pair_batched")?, + rpx_leaves_ext3_batched: rpx.load_function("rpx_leaves_ext3_batched")?, + rpx_comp_poly_leaves_ext3: rpx.load_function("rpx_comp_poly_leaves_ext3")?, + rpx_fri_leaves_ext3: rpx.load_function("rpx_fri_leaves_ext3")?, + rpx_fri_group_leaves_ext3: rpx.load_function("rpx_fri_group_leaves_ext3")?, + rpx_merkle_level: rpx.load_function("rpx_merkle_level")?, + rpx_merkle_tail: rpx.load_function("rpx_merkle_tail")?, + rpx_permute_probe: rpx.load_function("rpx_permute_probe")?, + // The WHIR path's leaves are a fold COSET, not a row group, so these + // two are additional kernels rather than alternatives to the seven + // above. `rpx_merkle_level` / `rpx_merkle_tail` are shared. + rpx_leaves_base_coset: rpx.load_function("rpx_leaves_base_coset")?, + rpx_leaves_ext3_coset: rpx.load_function("rpx_leaves_ext3_coset")?, + rpx_grind_search: rpx.load_function("rpx_grind_search")?, + rpx_grind_search_counted: rpx.load_function("rpx_grind_search_counted")?, + rpx_grind_search_queue: rpx.load_function("rpx_grind_search_queue")?, + rpx_grind_search_queue_counted: rpx.load_function("rpx_grind_search_queue_counted")?, + rpx_merkle_level_warp: rpx.load_function("rpx_merkle_level_warp")?, + rpx_merkle_tail_warp: rpx.load_function("rpx_merkle_tail_warp")?, + rpx_permute_warp_probe: rpx.load_function("rpx_permute_warp_probe")?, + rpx_limb_probe: rpx.load_function("rpx_limb_probe")?, + rpx_permute_chain_probes: [0u32, 1, 3, 4, 5, 7, 9] + .into_iter() + .map(|v| { + rpx.load_function(&format!("rpx_permute_chain_probe_v{v}")) + .map(|f| (v, f)) + }) + .collect::, _>>()?, barycentric_base_batched: bary.load_function("barycentric_base_batched")?, barycentric_ext3_batched: bary.load_function("barycentric_ext3_batched")?, barycentric_base_batched_strided: bary @@ -451,6 +1262,18 @@ impl Backend { gather_rows_ext3: bary.load_function("gather_rows_ext3")?, deep_composition_ext3_row: deep.load_function("deep_composition_ext3_row")?, bit_reverse_ext3_kernel: deep.load_function("bit_reverse_ext3_interleaved")?, + deep_composition_ext3_fused: [ + deep.load_function("deep_composition_ext3_fused_m1")?, + deep.load_function("deep_composition_ext3_fused_m2")?, + deep.load_function("deep_composition_ext3_fused_m3")?, + deep.load_function("deep_composition_ext3_fused_m4")?, + ], + deep_composition_ext3_hoisted: [ + deep.load_function("deep_composition_ext3_hoisted_m1")?, + deep.load_function("deep_composition_ext3_hoisted_m2")?, + deep.load_function("deep_composition_ext3_hoisted_m3")?, + deep.load_function("deep_composition_ext3_hoisted_m4")?, + ], fri_fold_ext3: fri.load_function("fri_fold_ext3")?, gather_ext3_at: fri.load_function("gather_ext3_at")?, fri_update_twiddles: fri.load_function("fri_update_twiddles")?, @@ -463,6 +1286,16 @@ impl Backend { apply_block_offsets_rev_ext3: inverse.load_function("apply_block_offsets_rev_ext3")?, batch_inverse_combine_ext3: inverse.load_function("batch_inverse_combine_ext3")?, invert_total_ext3: inverse.load_function("invert_total_ext3")?, + invert_denoms_rowwise_ext3: [ + inverse.load_function("invert_denoms_rowwise_ext3_k1")?, + inverse.load_function("invert_denoms_rowwise_ext3_k2")?, + inverse.load_function("invert_denoms_rowwise_ext3_k3")?, + inverse.load_function("invert_denoms_rowwise_ext3_k4")?, + inverse.load_function("invert_denoms_rowwise_ext3_k5")?, + inverse.load_function("invert_denoms_rowwise_ext3_k6")?, + inverse.load_function("invert_denoms_rowwise_ext3_k7")?, + inverse.load_function("invert_denoms_rowwise_ext3_k8")?, + ], logup_fingerprint_ext3: logup.load_function("logup_fingerprint_ext3")?, logup_term_ext3: logup.load_function("logup_term_ext3")?, logup_row_sum_ext3: logup.load_function("logup_row_sum_ext3")?, @@ -470,6 +1303,57 @@ impl Backend { logup_apply_offsets_add_ext3: logup.load_function("logup_apply_offsets_add_ext3")?, logup_finalize_accum_ext3: logup.load_function("logup_finalize_accum_ext3")?, logup_assemble_aux_ext3: logup.load_function("logup_assemble_aux_ext3")?, + whir_fold_base_ext3: whir_fold.load_function("whir_fold_base_ext3")?, + whir_fold_ext3: whir_fold.load_function("whir_fold_ext3")?, + gather_cosets: whir_fold.load_function("gather_cosets")?, + gather_cosets_strided: whir_fold.load_function("gather_cosets_strided")?, + whir_fold_k_base_ext3: whir_fold.load_function("whir_fold_k_base_ext3")?, + whir_fold_k_ext3: whir_fold.load_function("whir_fold_k_ext3")?, + whir_lean_colmap: whir_fold.load_function("whir_lean_colmap")?, + whir_lean_round: whir_fold.load_function("whir_lean_round")?, + whir_lean_materialize: whir_fold.load_function("whir_lean_materialize")?, + whir_tile_plan: whir_fold.load_function("whir_tile_plan")?, + whir_tile_contract: whir_fold.load_function("whir_tile_contract")?, + whir_tile_contract_wide: whir_fold.load_function("whir_tile_contract_wide")?, + whir_tile_round: whir_fold.load_function("whir_tile_round")?, + whir_tile_fold: whir_fold.load_function("whir_tile_fold")?, + whir_tile_materialize: whir_fold.load_function("whir_tile_materialize")?, + whir_tile_materialize_check: whir_fold.load_function("whir_tile_materialize_check")?, + ext3_limb_gather: whir_fold.load_function("ext3_limb_gather")?, + ext3_limb_scatter: whir_fold.load_function("ext3_limb_scatter")?, + sumcheck_round_ext3: sumcheck.load_function("sumcheck_round_ext3")?, + sumcheck_round_ext3_int: sumcheck.load_function("sumcheck_round_ext3_int")?, + sum_partials_ext3: sumcheck.load_function("sum_partials_ext3")?, + zc_grid01: sumcheck.load_function("zc_grid01")?, + zc_bus_u: sumcheck.load_function("zc_bus_u")?, + zc_fold2: sumcheck.load_function("zc_fold2")?, + zc_bus_column: sumcheck.load_function("zc_bus_column")?, + zc_halve: sumcheck.load_function("zc_halve")?, + zc_round_gruen: sumcheck.load_function("zc_round_gruen")?, + gkr_eq_levels_ext3: sumcheck.load_function("gkr_eq_levels_ext3")?, + gkr_round_gruen: sumcheck.load_function("gkr_round_gruen")?, + gkr_gruen_finish: sumcheck.load_function("gkr_gruen_finish")?, + gkr_round_gruen_real: sumcheck.load_function("gkr_round_gruen_real")?, + gkr_gruen_finish_real: sumcheck.load_function("gkr_gruen_finish_real")?, + gkr_round_gruen_slot: sumcheck.load_function("gkr_round_gruen_slot")?, + gkr_round_gruen_real_slot: sumcheck.load_function("gkr_round_gruen_real_slot")?, + gkr_input_from_columns: sumcheck.load_function("gkr_input_from_columns")?, + sumcheck_fold_ext3: sumcheck.load_function("sumcheck_fold_ext3")?, + mle_fold_base_ext3: sumcheck.load_function("mle_fold_base_ext3")?, + eq_expand_level_ext3: sumcheck.load_function("eq_expand_level_ext3")?, + eq_seed_shares_ext3: sumcheck.load_function("eq_seed_shares_ext3")?, + eq_expand_level_shares_ext3: sumcheck.load_function("eq_expand_level_shares_ext3")?, + program_map_ext3: sumcheck.load_function("program_map_ext3")?, + factors_from_columns_ext3: sumcheck.load_function("factors_from_columns_ext3")?, + mle_lift_base_ext3: sumcheck.load_function("mle_lift_base_ext3")?, + add_scaled_ext3: sumcheck.load_function("add_scaled_ext3")?, + fill_ext3: sumcheck.load_function("fill_ext3")?, + fraction_fold_ext3: sumcheck.load_function("fraction_fold_ext3")?, + fraction_fold_padded_ext3: sumcheck.load_function("fraction_fold_padded_ext3")?, + mle_fold_base_ext3_many: sumcheck.load_function("mle_fold_base_ext3_many")?, + eq_dot_base_ext3_many: sumcheck.load_function("eq_dot_base_ext3_many")?, + batched_column_ext3: sumcheck.load_function("batched_column_ext3")?, + gather_factor_heads_ext3: sumcheck.load_function("gather_factor_heads_ext3")?, constraint_interp_kernel: constraint_interp .load_function("constraint_interp_kernel")?, constraint_composition_kernel: constraint_interp @@ -479,27 +1363,151 @@ impl Backend { fwd_twiddles: Mutex::new(vec![None; max_log]), inv_twiddles: Mutex::new(vec![None; max_log]), ctx, - streams, + streams: crate::stream_class::StreamPools::new(streams, background), pinned_staging, pinned_hashes, + staging_pairs: StagingPool::default(), event_pool, util_stream, - next: AtomicUsize::new(0), vram_budget_bytes, + reserved: AtomicU64::new(0), }) } - /// VRAM budget in bytes for table-session admission control. `u64::MAX` + /// VRAM budget in bytes for admission control. `u64::MAX` /// when budgeting is disabled (query failed). See the field docs. pub fn vram_budget_bytes(&self) -> u64 { self.vram_budget_bytes } + /// Live `(free, total)` device memory in bytes, for diagnostics — the + /// admission gates never read it (they must answer the same at R1 and at + /// R4). `None` when the query fails. + pub fn device_mem_info(&self) -> Option<(u64, u64)> { + self.ctx + .mem_get_info() + .ok() + .map(|(free, total)| (free as u64, total as u64)) + } + + /// Hand the default memory pool's unused reserved memory back to the OS, + /// keeping at most `keep_bytes` (`cuMemPoolTrimTo`). Under the retained + /// posture ([`mempool_release_threshold_bytes`]) a sync never releases; + /// this is the explicit release for the moments reuse cannot serve — a + /// differently-shaped table after a large one, or a sampler that must read + /// the live working set. Best effort: `false` when the pool cannot be + /// queried or the trim fails. + pub fn trim_mempool_to(&self, keep_bytes: u64) -> bool { + use cudarc::driver::sys; + // SAFETY: raw driver calls on this backend's live context; the trim + // takes a plain byte count. + unsafe { + default_mempool(&self.ctx).is_some_and(|pool| { + sys::cuMemPoolTrimTo(pool, keep_bytes as usize) + .result() + .is_ok() + }) + } + } + + /// Bytes the device reports free, right now. + /// + /// The driver's own accounting rather than this module's: [`reserve`] + /// counts what callers PROMISED, which is silent about anything allocated + /// without a reservation. A test that wants to know whether a structure is + /// holding device memory it never declared has to ask the device, and this + /// is how — see `a_group_holds_only_its_codewords_before_any_open`. + /// + /// ⚠ The stream-ordered pool retains freed blocks, so this falls as memory + /// is used and does not always rise as it is released. It answers "how + /// much has this process taken from the device", not "how much is live", + /// which is the question a retention test is asking. + pub fn free_vram_bytes(&self) -> Result { + use cudarc::driver::sys; + self.ctx.bind_to_thread()?; + // SAFETY: a raw driver query writing into two stack slots, with the + // context bound to this thread on the line above. + unsafe { + let mut free: usize = 0; + let mut total: usize = 0; + sys::cuMemGetInfo_v2(&mut free as *mut usize, &mut total as *mut usize).result()?; + Ok(free as u64) + } + } + + /// Promises `bytes` of the device to something about to be built there, or + /// refuses. + /// + /// Asking the driver how much is free does not answer this: two callers + /// can both be told yes and both be right at the moment they ask. What a + /// structure needs is that the room stays its own until it is done — a + /// codeword that is admitted and then cannot fold has nowhere to go, since + /// there is no copy on the host by then. + /// + /// Refusing is cheap wherever it is asked, because everything that asks + /// has a host path. The budget binds only when several proofs share a + /// card: one of them is enough to fill it. + pub fn reserve(&self, bytes: u64) -> Option { + self.reserve_with(bytes, true) + } + + /// [`Self::reserve`], asking the retention for spare bytes on a miss only + /// when `evict`. + fn reserve_with(&self, bytes: u64, evict: bool) -> Option { + let mut evicted = !evict; + let mut held = self.reserved.load(Ordering::Relaxed); + loop { + if held.saturating_add(bytes) > self.vram_budget_bytes { + // ★ Before giving up, ask the retention for spare bytes ONCE. The + // retention holds only genuinely-spare leaf layers and gives them + // back here, so a real caller (argue) is never displaced by a + // cache — the fix for wt16, where the retention won the shared + // budget and argue fell to the host. Bounded to one pass by + // `evicted`, so a persistent miss returns None (and the call + // site's `note_device_fallback` counts it) rather than spinning. + // A miss the layers cannot cover may keep them + // (`whir::keep_futile`): nothing is freed, and this returns None + // as it would after evicting them. + if !evicted { + evicted = true; + let deficit = held.saturating_add(bytes) - self.vram_budget_bytes; + if try_evict_retained(deficit) > 0 { + held = self.reserved.load(Ordering::Relaxed); + continue; + } + } + return None; + } + match self.reserved.compare_exchange_weak( + held, + held + bytes, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + note_reserved(held + bytes); + return Some(DeviceReservation { + bytes: AtomicU64::new(bytes), + }); + } + Err(seen) => held = seen, + } + } + } + + /// Bytes promised across every live reservation — what `reserve` checks + /// the budget against. Exposed so a test can assert the number rather than + /// assert that nothing crashed. + pub fn reserved_bytes(&self) -> u64 { + self.reserved.load(Ordering::Relaxed) + } + /// Round-robin over the stream pool. Concurrent callers get different - /// streams so their kernel launches overlap on the GPU. + /// streams so their kernel launches overlap on the GPU. A thread marked + /// background draws from the background pool when one exists + /// ([`crate::stream_class`]). pub fn next_stream(&self) -> Arc { - let idx = self.next.fetch_add(1, Ordering::Relaxed) % self.streams.len(); - self.streams[idx].clone() + self.streams.next() } /// Per-rayon-worker pinned staging buffer. Returns the slot for the @@ -586,11 +1594,30 @@ impl Backend { /// /// Returns `Err` when CUDA initialisation fails (no driver, no GPU, PTX load /// failure). Initialisation is retried on the next call until one succeeds — -/// only a successful `Backend` is cached. The race window where two threads -/// init concurrently is harmless: at most one extra `Backend::init()` runs -/// and the loser is dropped. +/// only a successful `Backend` is cached. One initialisation runs at a time: +/// a caller that finds another initialising waits for it rather than +/// building a second context. pub fn backend() -> Result<&'static Backend> { + backend_once() +} + +/// Sets the backend up ahead of its first use ([`backend`]), without counting +/// as a device entry: a prove that will reach the card starts it as the +/// prove starts, on a thread of its own, so its first upload does not wait for +/// the context, the modules and the staging pools. A failure is left for the +/// first use to meet. +pub fn set_up_backend() { + let _ = backend_once(); +} + +/// [`backend`] without the entry count. +fn backend_once() -> Result<&'static Backend> { static BACKEND: OnceLock = OnceLock::new(); + static INIT: Mutex<()> = Mutex::new(()); + if let Some(b) = BACKEND.get() { + return Ok(b); + } + let _init = INIT.lock().unwrap_or_else(|e| e.into_inner()); if let Some(b) = BACKEND.get() { return Ok(b); } @@ -722,7 +1749,9 @@ pub fn htod_via( // synced below before this memcpy overwrites the slab, so the slab is // never read (by an in-flight DMA) and written at the same time. unsafe { + let copy_t0 = std::time::Instant::now(); std::ptr::copy_nonoverlapping(src.add(byte_off), staging.ptr as *mut u8, this_bytes); + STAGING_STATS.note_in(false, this_bytes, copy_t0.elapsed()); let r = cudarc::driver::sys::cuMemcpyHtoDAsync_v2( dst_base + byte_off as u64, staging.ptr as *const core::ffi::c_void, @@ -816,6 +1845,758 @@ impl Drop for DrainOnErr<'_> { } } +// ── Per-transfer, double-buffered pinned staging ───────────────────────────── + +/// The opt-out from per-transfer staging. `1` sends the row-major commit's trace +/// upload and its retained-LDE download through the shared per-worker slab of +/// [`htod_via`] / [`async_dtoh_via`]; anything else (the default) gives each of +/// those transfers a staging pair of its own ([`htod_staged`], +/// [`dtoh_staged_into`]). +/// +/// ⛔ WHY PAIRS. The per-table scheduler's driver threads are not rayon workers, +/// so [`Backend::worker_slot`] sends every one of them to slot 0. Through the +/// shared slab every driver stages through ONE slab, one single-buffered chunk at +/// a time, and a driver copying a retained LDE out holds that slab's mutex for +/// the whole host copy while the others queue with the card idle. The slab also +/// grows to the largest retained LDE's next power of two and never shrinks, which +/// is pinned host memory for the rest of the process. +pub const STAGING_SHARED_SLAB_ENV: &str = "LAMBDA_VM_STAGING_SHARED_SLAB"; + +thread_local! { + /// The calling thread's override of [`staging_pairs_enabled`], so one test + /// process can run both paths. `None` reads the environment. + static STAGING_PAIRS_OVERRIDE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Whether the calling thread stages the row-major commit's transfers through +/// pairs: its override if set, else unless [`STAGING_SHARED_SLAB_ENV`] is `1`. +/// The environment is read once per process and named once on stderr, so every +/// log states which staging it ran. +pub fn staging_pairs_enabled() -> bool { + if let Some(on) = STAGING_PAIRS_OVERRIDE.with(|o| o.get()) { + return on; + } + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| { + let shared = std::env::var(STAGING_SHARED_SLAB_ENV).is_ok_and(|v| v == "1"); + eprintln!( + "[gpu] transfer staging: {}", + if shared { + "the shared per-worker slab (LAMBDA_VM_STAGING_SHARED_SLAB=1)" + } else { + "per-transfer pinned pairs" + } + ); + !shared + }) +} + +/// Force [`staging_pairs_enabled`] on the calling thread; `None` restores the +/// environment's answer. +pub fn set_staging_pairs_override(on: Option) { + STAGING_PAIRS_OVERRIDE.with(|o| o.set(on)); +} + +thread_local! { + /// The calling thread's [`grid_cap`]. + static GRID_CAP: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// The most blocks a long kernel launched from the calling thread may take, +/// its work walked by a grid-stride loop: `None` (the default) launches the +/// full grid. Set by [`with_grid_cap`] around work that runs beside +/// latency-bound kernels, so that it holds at most that many blocks resident +/// and the others find SMs free. The values are the same at any grid. +pub fn grid_cap() -> Option { + GRID_CAP.with(|c| c.get()) +} + +/// `full` blocks, cut to the calling thread's [`grid_cap`] (at least one). +pub fn capped_grid(full: u32) -> u32 { + match grid_cap() { + Some(cap) => full.min(cap.max(1)), + None => full, + } +} + +/// Runs `f` with the calling thread's [`grid_cap`] set to `cap`, the previous +/// cap restored after (also on a panic), so calls nest. +pub fn with_grid_cap(cap: Option, f: impl FnOnce() -> R) -> R { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + GRID_CAP.with(|c| c.set(self.0)); + } + } + let _restore = Restore(GRID_CAP.with(|c| c.replace(cap))); + f() +} + +/// Bytes in one staging buffer. A pair holds two. +pub const STAGED_CHUNK_BYTES: usize = 32 << 20; + +/// The most pairs a process ever allocates, which bounds the pairs' pinned +/// footprint at `MAX_STAGING_PAIRS × 2 × STAGED_CHUNK_BYTES` = 512 MiB and the +/// number of pinned allocations at sixteen. Pairs are made on demand, lent to +/// one transfer at a time and never freed or grown — the bound whose absence +/// made pre-sized per-worker slabs a measured regression (see the note on +/// pinned-staging size hints in `stark::prover::multi_prove`). +pub const MAX_STAGING_PAIRS: usize = 8; + +/// Two fixed-size pinned buffers, each carrying the event of the last DMA that +/// read or wrote it. +struct StagingPair { + bufs: [PinnedStaging; 2], +} + +impl StagingPair { + fn new(ctx: &Arc) -> Result { + let mut bufs = [PinnedStaging::empty(), PinnedStaging::empty()]; + for buf in &mut bufs { + buf.ensure_capacity(STAGED_CHUNK_BYTES / 8, ctx)?; + buf.event = Some(ctx.new_event(None)?); + } + Ok(Self { bufs }) + } +} + +/// The pairs not on loan, and how many exist. +#[derive(Default)] +struct StagingPool { + free: Mutex>, + returned: std::sync::Condvar, + created: AtomicUsize, +} + +impl StagingPool { + /// A pair for one transfer: a free one, else a new one while fewer than + /// [`MAX_STAGING_PAIRS`] exist, else the next one handed back. Never nested + /// (a transfer holds one pair and asks for nothing else while it does), so + /// the wait cannot deadlock. + fn lend(&self, ctx: &Arc) -> Result> { + let mut free = self.free.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if let Some(pair) = free.pop() { + return Ok(PairLoan { + pool: self, + pair: Some(pair), + }); + } + if self.created.load(Ordering::Relaxed) < MAX_STAGING_PAIRS { + // Counted under the lock, so the cap cannot be overshot. + self.created.fetch_add(1, Ordering::Relaxed); + drop(free); + return match StagingPair::new(ctx) { + Ok(pair) => Ok(PairLoan { + pool: self, + pair: Some(pair), + }), + Err(e) => { + self.created.fetch_sub(1, Ordering::Relaxed); + Err(e) + } + }; + } + STAGING_STATS.pair_waits.fetch_add(1, Ordering::Relaxed); + free = self.returned.wait(free).unwrap_or_else(|e| e.into_inner()); + } + } + + /// Makes pairs until `want` exist (at most [`MAX_STAGING_PAIRS`]) and puts + /// them in the free list; returns how many this call made. One pair at a + /// time, the lock not held across a pinned allocation, so a transfer that + /// lends meanwhile is never kept waiting behind this. + fn prewarm(&self, ctx: &Arc, want: usize) -> Result { + let want = want.min(MAX_STAGING_PAIRS); + let mut made = 0; + loop { + { + let _free = self.free.lock().unwrap_or_else(|e| e.into_inner()); + if self.created.load(Ordering::Relaxed) >= want { + return Ok(made); + } + // Counted under the lock, as `lend` counts, so the cap holds. + self.created.fetch_add(1, Ordering::Relaxed); + } + let pair = match StagingPair::new(ctx) { + Ok(pair) => pair, + Err(e) => { + self.created.fetch_sub(1, Ordering::Relaxed); + return Err(e); + } + }; + self.free + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(pair); + self.returned.notify_one(); + made += 1; + } + } +} + +/// Makes the staging pairs a later transfer would make on first use, until +/// `want` exist (at most [`MAX_STAGING_PAIRS`]); returns how many this call +/// made. Each pair is two pinned allocations, which take the driver's lock for +/// several milliseconds: a caller that runs this before the transfers that +/// need the pairs takes those allocations out of their windows. The pairs are +/// the ones [`htod_staged`] would make, so nothing a transfer moves changes. +pub fn prewarm_staging_pairs(want: usize) -> Result { + let be = backend()?; + be.ctx.bind_to_thread()?; + be.staging_pairs.prewarm(&be.ctx, want) +} + +/// How many staging pairs exist (made on first use or by +/// [`prewarm_staging_pairs`]). +pub fn staging_pairs_created() -> usize { + backend().map_or(0, |be| be.staging_pairs.created.load(Ordering::Relaxed)) +} + +/// A pair on loan to one transfer, handed back when dropped. Every exit path of +/// a transfer leaves no DMA in flight on a buffer without that buffer's event +/// recorded after it, so the next borrower's event waits are enough. +struct PairLoan<'a> { + pool: &'a StagingPool, + pair: Option, +} + +impl PairLoan<'_> { + fn buf(&mut self, k: usize) -> &mut PinnedStaging { + &mut self.pair.as_mut().expect("present until drop").bufs[k & 1] + } +} + +impl Drop for PairLoan<'_> { + fn drop(&mut self) { + if let Some(pair) = self.pair.take() { + self.pool + .free + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(pair); + self.pool.returned.notify_one(); + } + } +} + +/// What the staging surface moved and what its host copies cost, by path — +/// counted on BOTH arms so an A/B reads the same quantities on each. `in` is the +/// commit's trace upload (host memcpy into pinned memory); `out` is the retained +/// LDE's download, timed from the DMA's enqueue to the host holding every value. +struct StagingStats { + /// `[shared slab, staged]`. + in_bytes: [AtomicU64; 2], + in_nanos: [AtomicU64; 2], + out_bytes: [AtomicU64; 2], + out_nanos: [AtomicU64; 2], + pair_waits: AtomicU64, +} + +static STAGING_STATS: StagingStats = StagingStats { + in_bytes: [AtomicU64::new(0), AtomicU64::new(0)], + in_nanos: [AtomicU64::new(0), AtomicU64::new(0)], + out_bytes: [AtomicU64::new(0), AtomicU64::new(0)], + out_nanos: [AtomicU64::new(0), AtomicU64::new(0)], + pair_waits: AtomicU64::new(0), +}; + +impl StagingStats { + fn note_in(&self, staged: bool, bytes: usize, took: std::time::Duration) { + let i = usize::from(staged); + self.in_bytes[i].fetch_add(bytes as u64, Ordering::Relaxed); + self.in_nanos[i].fetch_add(took.as_nanos() as u64, Ordering::Relaxed); + } + + fn note_out(&self, staged: bool, bytes: usize, took: std::time::Duration) { + let i = usize::from(staged); + self.out_bytes[i].fetch_add(bytes as u64, Ordering::Relaxed); + self.out_nanos[i].fetch_add(took.as_nanos() as u64, Ordering::Relaxed); + } +} + +/// Record one retained-LDE download taken through the shared slab: `bytes` +/// landed in a host `Vec` `took` after the DMA was queued. +pub fn note_shared_slab_download(bytes: usize, took: std::time::Duration) { + STAGING_STATS.note_out(false, bytes, took); +} + +/// The staging counters as numbers, for a test that asserts a path was taken. +#[derive(Clone, Copy, Debug)] +pub struct StagingTotals { + pub shared_in_bytes: u64, + pub staged_in_bytes: u64, + pub shared_out_bytes: u64, + pub staged_out_bytes: u64, + /// Staging pairs allocated so far. + pub pairs: usize, + /// Times a transfer waited because every pair was on loan. + pub pair_waits: u64, +} + +/// The staging counters since the process started. +pub fn staging_totals() -> StagingTotals { + let s = &STAGING_STATS; + StagingTotals { + shared_in_bytes: s.in_bytes[0].load(Ordering::Relaxed), + staged_in_bytes: s.in_bytes[1].load(Ordering::Relaxed), + shared_out_bytes: s.out_bytes[0].load(Ordering::Relaxed), + staged_out_bytes: s.out_bytes[1].load(Ordering::Relaxed), + pairs: backend().map_or(0, |be| be.staging_pairs.created.load(Ordering::Relaxed)), + pair_waits: s.pair_waits.load(Ordering::Relaxed), + } +} + +/// How a preprocessed commit brings its precomputed tree's node buffer to the +/// host on a tree-cache miss. Unset or `staged` (the default): through a +/// staging pair ([`dtoh_staged_uncounted`]) straight into the host tree's node +/// vector, with the multiplicity tree queued behind the last chunk. `legacy` +/// or `pageable` (the opt-out): one pageable copy into a zeroed byte vector, +/// converted to nodes at the end of the commit — the path before the knob +/// existed. The nodes are the same bytes either way; only the transfer +/// differs. #1013's median 25475471 recursion took 2.9 s (ULTRA) and 6.3 s +/// (RYZEN) less staged (I-COPIES §6, §8). +pub const TREE_DOWNLOAD_ENV: &str = "LAMBDA_VM_TREE_DOWNLOAD"; + +thread_local! { + /// The calling thread's override of [`tree_download_staged`], so one test + /// process can run both paths. `None` reads the environment. + static TREE_DOWNLOAD_OVERRIDE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Whether the calling thread downloads precomputed trees staged: its override +/// if set, else [`TREE_DOWNLOAD_ENV`], read once per process and named once on +/// stderr, so every log states which download it ran. +pub fn tree_download_staged() -> bool { + if let Some(on) = TREE_DOWNLOAD_OVERRIDE.with(|o| o.get()) { + return on; + } + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| { + let (staged, why) = tree_download_setting(std::env::var(TREE_DOWNLOAD_ENV).ok().as_deref()); + eprintln!( + "[gpu] precomputed-tree download: {} ({why})", + if staged { "staged" } else { "pageable" } + ); + staged + }) +} + +/// [`TREE_DOWNLOAD_ENV`]'s value read: whether it selects the staged download, +/// and how the value was taken. An unknown value keeps the default (staged) +/// and says so, rather than failing a prove over a typo. +fn tree_download_setting(value: Option<&str>) -> (bool, String) { + match value.map(str::trim) { + None => (true, format!("{TREE_DOWNLOAD_ENV} unset, the default")), + Some("staged") => (true, format!("{TREE_DOWNLOAD_ENV}=staged")), + Some(v @ ("legacy" | "pageable")) => (false, format!("{TREE_DOWNLOAD_ENV}={v}")), + Some(v) => ( + true, + format!("{TREE_DOWNLOAD_ENV}={v:?} is not staged, legacy or pageable: the default"), + ), + } +} + +/// Force [`tree_download_staged`] on the calling thread; `None` restores the +/// environment's answer. +pub fn set_tree_download_override(on: Option) { + TREE_DOWNLOAD_OVERRIDE.with(|o| o.set(on)); +} + +/// Precomputed-tree downloads by path (`[pageable, staged]`): trees, bytes, and +/// the time from the tree being built to the host holding every node — the +/// pageable path's zero-fill, copy and conversion to nodes included. +static TREE_DOWNLOADS: [[AtomicU64; 3]; 2] = [ + [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)], + [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)], +]; + +/// Record one precomputed-tree download of `bytes` that took `took`. +pub fn note_tree_download(staged: bool, bytes: usize, took: std::time::Duration) { + let c = &TREE_DOWNLOADS[usize::from(staged)]; + c[0].fetch_add(1, Ordering::Relaxed); + c[1].fetch_add(bytes as u64, Ordering::Relaxed); + c[2].fetch_add(took.as_nanos() as u64, Ordering::Relaxed); +} + +/// The precomputed-tree download counters, `[pageable, staged]` each. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TreeDownloadTotals { + pub trees: [u64; 2], + pub bytes: [u64; 2], + pub nanos: [u64; 2], +} + +impl TreeDownloadTotals { + /// What moved between `earlier` and `self`. + pub fn since(&self, earlier: &Self) -> Self { + let d = |a: [u64; 2], b: [u64; 2]| [a[0] - b[0], a[1] - b[1]]; + Self { + trees: d(self.trees, earlier.trees), + bytes: d(self.bytes, earlier.bytes), + nanos: d(self.nanos, earlier.nanos), + } + } + + /// One line: `pageable trees in () | staged …`. + pub fn line(&self) -> String { + let one = |i: usize| { + let gb = self.bytes[i] as f64 / 1e9; + let s = self.nanos[i] as f64 / 1e9; + format!( + "{} {} trees {gb:.2} GB in {s:.2} s ({:.2} GB/s)", + ["pageable", "staged"][i], + self.trees[i], + if s > 0.0 { gb / s } else { 0.0 } + ) + }; + format!("{} | {}", one(0), one(1)) + } +} + +/// The precomputed-tree download counters since the process started. +pub fn tree_download_totals() -> TreeDownloadTotals { + let load = |k: usize| { + [ + TREE_DOWNLOADS[0][k].load(Ordering::Relaxed), + TREE_DOWNLOADS[1][k].load(Ordering::Relaxed), + ] + }; + TreeDownloadTotals { + trees: load(0), + bytes: load(1), + nanos: load(2), + } +} + +#[cfg(test)] +mod tree_download_tests { + use super::{TreeDownloadTotals, tree_download_setting}; + + /// The staged download is the default: unset, `staged` and anything + /// unknown select it; only `legacy` and `pageable` opt out, and the reason + /// names the value. + #[test] + fn staged_is_the_default_and_only_legacy_or_pageable_opt_out() { + let (on, why) = tree_download_setting(None); + assert!(on, "unset must select the staged download"); + assert!(why.contains("the default"), "{why}"); + assert!(tree_download_setting(Some("staged")).0); + assert!(tree_download_setting(Some(" staged ")).0); + assert!(!tree_download_setting(Some("legacy")).0); + assert!(!tree_download_setting(Some(" legacy ")).0); + assert!(!tree_download_setting(Some("pageable")).0); + let (on, why) = tree_download_setting(Some("legcay")); + assert!(on, "an unknown value keeps the default"); + assert!(why.contains("\"legcay\""), "{why}"); + } + + #[test] + fn totals_subtract_and_print_per_path() { + let earlier = TreeDownloadTotals { + trees: [1, 2], + bytes: [1_000_000_000, 0], + nanos: [500_000_000, 0], + }; + let now = TreeDownloadTotals { + trees: [3, 2], + bytes: [5_000_000_000, 0], + nanos: [2_500_000_000, 0], + }; + let d = now.since(&earlier); + assert_eq!(d.trees, [2, 0]); + assert_eq!( + d.line(), + "pageable 2 trees 4.00 GB in 2.00 s (2.00 GB/s) | staged 0 trees 0.00 GB in 0.00 s (0.00 GB/s)" + ); + } +} + +/// The shared slabs' pinned footprint, LDE staging and Merkle hashes together: +/// `(bytes, slots allocated, slots busy)`. A slab grows to a power of two and +/// never shrinks, so this is its high-water so far. A slot another thread holds +/// right now is counted as busy, not waited for. +fn shared_slab_footprint(be: &Backend) -> (u64, usize, usize) { + slab_footprint(be.pinned_staging.iter().chain(&be.pinned_hashes)) +} + +/// [`shared_slab_footprint`] over any set of slots. +fn slab_footprint<'a>( + slots: impl Iterator>, +) -> (u64, usize, usize) { + let (mut bytes, mut allocated, mut busy) = (0u64, 0usize, 0usize); + for slot in slots { + let elems = match slot.try_lock() { + Ok(s) => s.capacity_elems, + Err(std::sync::TryLockError::Poisoned(p)) => p.into_inner().capacity_elems, + Err(std::sync::TryLockError::WouldBlock) => { + busy += 1; + continue; + } + }; + if elems > 0 { + allocated += 1; + bytes += (elems * std::mem::size_of::()) as u64; + } + } + (bytes, allocated, busy) +} + +/// One line for the log: the staging surface's traffic and host-copy seconds per +/// path since the process started, the staging pairs' pool size and waits, and the +/// shared slabs' pinned footprint (what the retained-LDE downloads grow while +/// the shared-slab path is taken). +pub fn staging_report() -> String { + staging_line( + backend().map_or(0, |be| be.staging_pairs.created.load(Ordering::Relaxed)), + backend().map_or((0, 0, 0), shared_slab_footprint), + ) +} + +/// [`staging_report`]'s line from the counters, the pairs allocated and the +/// shared slabs' `(bytes, slots allocated, slots busy)`. +fn staging_line(pairs: usize, (slab_bytes, slabs, busy): (u64, usize, usize)) -> String { + let s = &STAGING_STATS; + let gb = |a: &AtomicU64| a.load(Ordering::Relaxed) as f64 / 1e9; + let secs = |a: &AtomicU64| a.load(Ordering::Relaxed) as f64 / 1e9; + let rate = |b: &AtomicU64, t: &AtomicU64| { + let t = secs(t); + if t > 0.0 { gb(b) / t } else { 0.0 } + }; + format!( + "staging: upload shared-slab {:.2} GB in {:.2} s memcpy ({:.1} GB/s) · staged {:.2} GB in \ + {:.2} s ({:.1} GB/s) | retained-LDE download shared-slab {:.2} GB in {:.2} s ({:.1} GB/s) · \ + staged {:.2} GB in {:.2} s ({:.1} GB/s) | staging pairs {} of {MAX_STAGING_PAIRS} · waits {} \ + | shared slabs pinned {:.2} GiB in {} slot(s), {} busy", + gb(&s.in_bytes[0]), + secs(&s.in_nanos[0]), + rate(&s.in_bytes[0], &s.in_nanos[0]), + gb(&s.in_bytes[1]), + secs(&s.in_nanos[1]), + rate(&s.in_bytes[1], &s.in_nanos[1]), + gb(&s.out_bytes[0]), + secs(&s.out_nanos[0]), + rate(&s.out_bytes[0], &s.out_nanos[0]), + gb(&s.out_bytes[1]), + secs(&s.out_nanos[1]), + rate(&s.out_bytes[1], &s.out_nanos[1]), + pairs, + s.pair_waits.load(Ordering::Relaxed), + slab_bytes as f64 / (1u64 << 30) as f64, + slabs, + busy, + ) +} + +/// Host→device copy through a staging pair of its own, double-buffered: the +/// host fills one buffer while the previous chunk's DMA drains the other. +/// +/// Returns once the LAST chunk is queued, without waiting for it: every host +/// read of `src_host` is done by then, so it is reusable on return, and work +/// the caller queues on `stream` next is ordered behind the copy. The buffers +/// stay safe because each carries the event of the DMA that last read it, and +/// whoever refills one — this call two chunks on, or the pair's next borrower on +/// any stream — waits on that event first. +/// +/// Under [`STAGING_SHARED_SLAB_ENV`] the upload takes [`htod_via`] instead. +pub fn htod_staged( + stream: &Arc, + src_host: &[T], + dst: &mut cudarc::driver::CudaViewMut<'_, T>, +) -> Result<()> { + use cudarc::driver::DevicePtrMut; + assert!( + dst.len() >= src_host.len(), + "htod_staged: destination shorter than source" + ); + let elem_size = std::mem::size_of::(); + assert!( + elem_size <= STAGED_CHUNK_BYTES, + "htod_staged: an element larger than a staging buffer" + ); + let n_elems = src_host.len(); + if std::mem::size_of_val(src_host) == 0 { + return Ok(()); + } + // Whole elements per chunk, so a `T` never straddles two buffers. + let chunk_elems = STAGED_CHUNK_BYTES / elem_size.max(1); + + let be = backend()?; + let mut loan = be.staging_pairs.lend(&be.ctx)?; + be.ctx.bind_to_thread()?; + // SAFETY: `device_ptr_mut` yields the destination's base address and orders + // the writes on `stream`; `dst.len() >= src_host.len()` (asserted), so every + // chunk's range lies inside `dst`. + let (dst_base, _record) = dst.device_ptr_mut(stream); + // Declared after `loan`, so it drops FIRST: a DMA queued but not yet covered + // by its buffer's event is drained before the pair can be lent again. + let mut drain = DrainOnErr { + stream, + armed: false, + }; + let src = src_host.as_ptr() as *const u8; + let (mut elem_off, mut k) = (0usize, 0usize); + while elem_off < n_elems { + let this_elems = (n_elems - elem_off).min(chunk_elems); + let this_bytes = this_elems * elem_size; + let byte_off = elem_off * elem_size; + let buf = loan.buf(k); + // The DMA that last read this buffer — two chunks back, or the previous + // borrower's — lands before the host overwrites it. + buf.sync_event()?; + // SAFETY: the buffer holds `STAGED_CHUNK_BYTES >= this_bytes`, and its + // last reader has landed (above), so nothing reads it while it is filled. + unsafe { + let copy_t0 = std::time::Instant::now(); + std::ptr::copy_nonoverlapping(src.add(byte_off), buf.ptr as *mut u8, this_bytes); + STAGING_STATS.note_in(true, this_bytes, copy_t0.elapsed()); + let r = cudarc::driver::sys::cuMemcpyHtoDAsync_v2( + dst_base + byte_off as u64, + buf.ptr as *const core::ffi::c_void, + this_bytes, + stream.cu_stream(), + ) + .result(); + // Armed even on failure: the driver may have queued the copy before + // reporting the error. + drain.armed = true; + r?; + } + buf.record_event(stream)?; + // The buffer's event now covers the DMA, for whoever refills it next. + drain.armed = false; + elem_off += this_elems; + k += 1; + } + Ok(()) +} + +/// Device→host copy of `n` `u64`s from device address `src` into `dst`, through +/// a staging pair of its own: two chunks in flight while the host copies each +/// landed one straight into `dst`'s spare capacity, with no zero-fill pass. `dst` +/// must be empty; on `Ok` it holds exactly the `n` values, and every DMA of this +/// call has landed. +/// +/// `after_last_enqueue` runs exactly once, as soon as the LAST chunk's DMA is +/// queued on `stream` — the first moment the caller may queue work that +/// overwrites the source, such as the retained LDE's in-place transpose — and +/// its value is returned. It does not run if an earlier step fails. +/// +/// # Safety +/// `src` must address at least `n` `u64`s of one live device allocation whose +/// writers are all ordered before this call on `stream`. Nothing may write them +/// before `after_last_enqueue` runs; work queued on `stream` from then on is +/// ordered behind the reads, and the allocation must outlive them (a +/// stream-ordered free on `stream` does). +pub unsafe fn dtoh_staged_into( + stream: &Arc, + src: cudarc::driver::sys::CUdeviceptr, + n: usize, + dst: &mut Vec, + after_last_enqueue: impl FnOnce() -> Result, +) -> Result { + if n == 0 { + assert!(dst.is_empty(), "dtoh_staged_into: dst must start empty"); + return after_last_enqueue(); + } + let started = std::time::Instant::now(); + // SAFETY: the caller's contract is `dtoh_staged_uncounted`'s, for `u64`s. + let result = unsafe { dtoh_staged_uncounted(stream, src, n, dst, after_last_enqueue) }?; + STAGING_STATS.note_out(true, n * 8, started.elapsed()); + Ok(result) +} + +/// [`dtoh_staged_into`] for any plain element type, and counted by its caller +/// instead of in the retained-LDE download's counters: the same pair, chunks and +/// ordering, with `n` counting `T`s. `T`'s size must divide +/// [`STAGED_CHUNK_BYTES`], so no element straddles two chunks. +/// +/// # Safety +/// As [`dtoh_staged_into`], with `src` addressing at least `n` `T`s, and every +/// bit pattern the device wrote there a valid `T` (plain data). +pub unsafe fn dtoh_staged_uncounted( + stream: &Arc, + src: cudarc::driver::sys::CUdeviceptr, + n: usize, + dst: &mut Vec, + after_last_enqueue: impl FnOnce() -> Result, +) -> Result { + assert!(dst.is_empty(), "dtoh_staged_into: dst must start empty"); + let size = std::mem::size_of::(); + assert!( + size > 0 && STAGED_CHUNK_BYTES.is_multiple_of(size), + "dtoh_staged_into: the element size must divide the chunk" + ); + if n == 0 { + return after_last_enqueue(); + } + dst.reserve_exact(n); + let chunk = STAGED_CHUNK_BYTES / size; + let n_chunks = n.div_ceil(chunk); + let chunk_len = |k: usize| (n - k * chunk).min(chunk); + + let be = backend()?; + let mut loan = be.staging_pairs.lend(&be.ctx)?; + be.ctx.bind_to_thread()?; + // ⛔ Declared after `loan`, so it drops FIRST, and armed from the first + // enqueue to the last landing: a DMA WRITES the pinned buffer, and a pair + // handed back with one in flight would let it land in the next borrower's + // data. Every early return drains the stream before the pair is lent again. + let mut drain = DrainOnErr { + stream, + armed: false, + }; + let mut after = Some(after_last_enqueue); + let mut result = None; + let mut next = 0usize; + for k in 0..n_chunks { + // Keep two chunks in flight: queue up to chunk `k + 1`. + while next < n_chunks && next < k + 2 { + let buf = loan.buf(next); + // Whatever last touched this buffer — this call's chunk `next - 2`, + // already copied out, or the previous borrower's DMA — is done. + buf.sync_event()?; + // SAFETY: the buffer holds `STAGED_CHUNK_BYTES >= chunk_len(next) * + // size`, nothing reads or writes it (above), and the source range + // lies inside the caller's `n` values. + unsafe { + let r = cudarc::driver::sys::cuMemcpyDtoHAsync_v2( + buf.ptr as *mut core::ffi::c_void, + src + (next * chunk * size) as u64, + chunk_len(next) * size, + stream.cu_stream(), + ) + .result(); + drain.armed = true; + r?; + } + buf.record_event(stream)?; + next += 1; + if next == n_chunks { + let queue_behind = after.take().expect("the last chunk is queued once"); + result = Some(queue_behind()?); + } + } + let buf = loan.buf(k); + buf.sync_event()?; + // SAFETY: the event orders this chunk's DMA before the read; `dst` has + // capacity for `n` values (reserved above) and chunk `k` covers + // `[k * chunk, k * chunk + chunk_len(k))`. + unsafe { + std::ptr::copy_nonoverlapping( + buf.ptr as *const T, + dst.as_mut_ptr().add(k * chunk), + chunk_len(k), + ); + } + } + // Every chunk landed and was copied: nothing of this call is in flight. + drain.armed = false; + // SAFETY: the loop wrote all `n` values. + unsafe { dst.set_len(n) }; + Ok(result.expect("the closure ran at the last enqueue")) +} + impl PendingD2H<'_> { /// Number of bytes the copy deposits. pub fn len_bytes(&self) -> usize { @@ -897,3 +2678,172 @@ impl Backend { Ok(PooledEvent { event: Some(ev) }) } } + +#[cfg(test)] +mod device_fallback_counter_tests { + use super::{ + device_fallbacks, note_device_fallback, note_reserved, reserved_high_water, + reset_device_fallbacks, reset_reserved_high_water, + }; + use std::sync::Mutex; + + /// The counter is process-wide, so a test asserting an absolute value + /// serialises against anything else in this binary that might move it. + static COUNTER: Mutex<()> = Mutex::new(()); + + /// The plumbing, card-free: a bump reads as one, bumps accumulate, and a + /// reset reads as zero. This is the CONTROL for the box test — if + /// `note`/`device_fallbacks`/`reset` did not agree here, no site test could + /// be trusted. + #[test] + fn note_bumps_read_and_reset_zeroes() { + let _g = COUNTER.lock().unwrap_or_else(|e| e.into_inner()); + reset_device_fallbacks(); + assert_eq!(device_fallbacks(), 0, "reset must zero the counter"); + note_device_fallback(); + assert_eq!(device_fallbacks(), 1, "one note reads one"); + note_device_fallback(); + assert_eq!(device_fallbacks(), 2, "notes accumulate"); + reset_device_fallbacks(); + assert_eq!(device_fallbacks(), 0, "reset must zero it again"); + } + + /// ⛔ THE FOUR UNDRIVEN SITES' FALSIFIER. The box test drives ONE site + /// (`DeviceColumns::upload`); this arm is what lets the other four fail + /// without four card fixtures. `note_device_fallback` must be CALLED at + /// exactly the five math-cuda argue-surface sites the counter's doc + /// enumerates — three in `sumcheck.rs`, one in `gkr.rs`, one in + /// `columns.rs`. Removing the call at ANY site changes the tuple and reddens + /// this test by name, which localises the loss to the file it happened in. + /// The sixth site, `multilinear::gpu`'s GKR tree, has its own census there. + /// + /// The pattern carries the `crate::device::` prefix so it counts CALLS and + /// never the definition (`pub fn note_device_fallback`). + #[test] + fn note_device_fallback_is_called_at_exactly_the_five_argue_sites() { + const PATTERN: &str = "crate::device::note_device_fallback()"; + let sumcheck = include_str!("sumcheck.rs").matches(PATTERN).count(); + let gkr = include_str!("gkr.rs").matches(PATTERN).count(); + let columns = include_str!("columns.rs").matches(PATTERN).count(); + assert_eq!( + (sumcheck, gkr, columns), + (3, 1, 1), + "the argue-surface device-fallback counter must be bumped at exactly \ + the five sites the counter's doc names: sumcheck ×3, gkr ×1, \ + columns ×1 (found sumcheck {sumcheck}, gkr {gkr}, columns {columns})" + ); + } + + /// The reservation high-water is a MONOTONE peak: it takes the max, so a + /// smaller rise never lowers it and a larger one raises it. Card-free — it + /// exercises `note_reserved` directly (the same call `reserve`/`grow` make on + /// every successful rise of `be.reserved`), so a peak that failed to track + /// would show here before any card run relied on the number. + #[test] + fn reserved_high_water_keeps_the_max() { + let _g = COUNTER.lock().unwrap_or_else(|e| e.into_inner()); + reset_reserved_high_water(); + assert_eq!(reserved_high_water(), 0, "reset must zero the high-water"); + note_reserved(100); + assert_eq!(reserved_high_water(), 100, "the first rise sets the peak"); + note_reserved(50); + assert_eq!( + reserved_high_water(), + 100, + "a smaller rise must not lower it" + ); + note_reserved(200); + assert_eq!(reserved_high_water(), 200, "a larger rise raises it"); + reset_reserved_high_water(); + assert_eq!(reserved_high_water(), 0, "reset must zero it again"); + } +} + +#[cfg(test)] +mod staging_report_tests { + use super::{PinnedStaging, slab_footprint, staging_line}; + use std::sync::Mutex; + + /// A slot claiming `elems` of capacity with no allocation behind it: the + /// null pointer keeps `Drop` from freeing anything. + fn slot(elems: usize) -> Mutex { + let mut s = PinnedStaging::empty(); + s.capacity_elems = elems; + Mutex::new(s) + } + + /// Card-free: the footprint sums the allocated slots' bytes and skips empty + /// ones, counts a slot someone holds as busy instead of waiting for it, and + /// still reads a poisoned one. + #[test] + fn the_slab_footprint_counts_allocated_busy_and_poisoned_slots() { + let slots = [ + slot(0), + slot(1 << 20), + slot(1 << 29), + slot(1 << 20), + slot(1 << 22), + ]; + assert_eq!(slab_footprint(slots[..1].iter()), (0, 0, 0)); + assert_eq!( + slab_footprint(slots[..3].iter()), + (((1 << 20) + (1 << 29)) * 8, 2, 0) + ); + let _held = slots[3].lock().unwrap(); + std::thread::scope(|s| { + let _ = s + .spawn(|| { + let _g = slots[4].lock().unwrap(); + panic!("poisoning the slot on purpose"); + }) + .join(); + }); + assert!(slots[4].is_poisoned()); + assert_eq!( + slab_footprint(slots.iter()), + (((1 << 20) + (1 << 29) + (1 << 22)) * 8, 3, 1) + ); + } + + /// The line's shape is a contract with the IDLE-B box script's parsers: four + /// ` GB in` figures (upload shared, staged; download shared, staged), then + /// the slab segment last, in GiB. + #[test] + fn the_staging_line_ends_with_the_shared_slab_footprint() { + let line = staging_line(6, (4 << 30, 2, 1)); + assert_eq!(line.matches(" GB in ").count(), 4, "{line}"); + assert!(line.contains("| staging pairs 6 of 8 · waits "), "{line}"); + assert!( + line.ends_with(" | shared slabs pinned 4.00 GiB in 2 slot(s), 1 busy"), + "{line}" + ); + } +} + +#[cfg(test)] +mod grid_cap_tests { + use super::{capped_grid, grid_cap, with_grid_cap}; + + /// No cap: the full grid. A cap cuts it, never below one block, nests, + /// and is restored after its closure, a panic included. + #[test] + fn a_grid_cap_cuts_the_grid_nests_and_is_restored() { + assert_eq!((grid_cap(), capped_grid(65_536)), (None, 65_536)); + with_grid_cap(Some(85), || { + assert_eq!(capped_grid(65_536), 85); + assert_eq!(capped_grid(12), 12, "a grid under the cap keeps its size"); + with_grid_cap(Some(0), || { + assert_eq!(capped_grid(65_536), 1, "at least one block") + }); + with_grid_cap(None, || assert_eq!(capped_grid(65_536), 65_536)); + assert_eq!(grid_cap(), Some(85), "the outer cap restored"); + }); + assert_eq!(grid_cap(), None); + let caught = std::panic::catch_unwind(|| with_grid_cap(Some(3), || panic!("inside"))); + assert!(caught.is_err()); + assert_eq!(grid_cap(), None, "restored after a panic"); + std::thread::spawn(|| assert_eq!(grid_cap(), None, "a new thread starts uncapped")) + .join() + .expect("thread"); + } +} diff --git a/crypto/math-cuda/src/fill.rs b/crypto/math-cuda/src/fill.rs new file mode 100644 index 000000000..7df651ccf --- /dev/null +++ b/crypto/math-cuda/src/fill.rs @@ -0,0 +1,1107 @@ +//! A table's columns filled on the card from the records its rows come from +//! (`kernels/fill.cu`), instead of filled packed on the host and widened on the +//! card ([`crate::narrow`]): MEMW_R from its register rows, CPU from the +//! executor's logs and the program's decode table, KECCAK_RND from its keccak +//! operations. +//! +//! Each kernel writes the words the prover's host fill writes for the same +//! rows, padding rows included, column-major (`out[col · height + row]`): a +//! table's run of columns in the resident store ([`crate::columns`]), the +//! layout [`crate::narrow::widen_col_major_to_host`] writes. The records cross +//! the bus as the `#[repr(C)]` structs below, which are the kernels' structs +//! field for field (their sizes and offsets are asserted on both sides). +//! +//! Each wrapper uploads its records from host slices the way the widen path +//! uploads packed bytes (a pageable copy on the destination's stream), launches +//! its kernel into the destination and returns the upload's and the kernel's +//! event-timed milliseconds ([`FillTimes`]). [`widen_col_major_timed`] is +//! today's path timed the same way, for the comparison. +//! +//! The module is loaded on first use from its own cubin, never by +//! [`crate::device::Backend::init`]. + +use std::mem::{offset_of, size_of}; +use std::sync::{Arc, OnceLock}; + +use cudarc::driver::{ + CudaEvent, CudaFunction, CudaModule, CudaSlice, CudaStream, CudaViewMut, DeviceRepr, + LaunchConfig, PushKernelArg, +}; +use cudarc::nvrtc::Ptx; + +use crate::Result; +use crate::device::backend; +use crate::narrow::NarrowInput; + +const FILL_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/fill.cubin")); + +/// MEMW_R's columns (`memw_register::cols::NUM_COLUMNS`). +pub const MEMW_R_COLUMNS: usize = 10; +/// MEMW_A's columns (`memw_aligned::cols::NUM_COLUMNS`). +pub const MEMW_A_COLUMNS: usize = 29; +/// LOAD's columns (`load::cols::NUM_COLUMNS`). +pub const LOAD_COLUMNS: usize = 18; +/// STORE's columns (`store::cols::NUM_COLUMNS`). +pub const STORE_COLUMNS: usize = 16; +/// SHIFT's columns (`shift::cols::NUM_COLUMNS`). +pub const SHIFT_COLUMNS: usize = 29; +/// MEMW's columns (`memw::cols::NUM_COLUMNS`). +pub const MEMW_COLUMNS: usize = 49; +/// CPU's columns (`cpu::cols::NUM_COLUMNS`). +pub const CPU_COLUMNS: usize = 37; +/// KECCAK_RND's columns (`keccak_rnd::cols::NUM_COLUMNS`). +pub const KECCAK_RND_COLUMNS: usize = 1480; +/// Rows a keccak operation fills. +pub const KECCAK_RND_ROWS_PER_OP: usize = 24; + +/// The column sections the KECCAK_RND kernel's grid splits a row into +/// (`FILL_KECCAK_SECTIONS`). +const KECCAK_SECTIONS: u32 = 3; +/// Threads a block for the row kernels. +const ROW_BLOCK: u32 = 256; +const CPU_BLOCK: u32 = 256; +const KECCAK_BLOCK: u32 = 128; + +/// The executor's log of one cycle (`executor::vm::logs::Log`), field for +/// field. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LogC { + pub current_pc: u64, + pub next_pc: u64, + pub src1_val: u64, + pub src2_val: u64, + pub dst_val: u64, +} + +/// One MEMW_R row (the prover's `RegRow`): `address` the register index, +/// `old_ts_lo` the old timestamp's low word, `is_read` 0 or 1, `pad` 0. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RegRowC { + pub timestamp: u64, + pub val0: u32, + pub val1: u32, + pub old0: u32, + pub old1: u32, + pub old_ts_lo: u32, + pub address: u16, + pub is_read: u8, + pub pad: u8, +} + +/// One DECODE row (the prover's `DecodeEntry`): the pc, the extended +/// immediate, then `ShrunkDecode`'s fields as bytes (flags 0 or 1). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DecodeEntryC { + pub pc: u64, + pub imm: u64, + pub read_register1: u8, + pub read_register2: u8, + pub write_register: u8, + pub word_instr: u8, + pub alu: u8, + pub add: u8, + pub sub: u8, + pub memory: u8, + pub branch: u8, + pub ecall: u8, + pub rs1: u8, + pub rs2: u8, + pub rd: u8, + pub half_instruction_length: u8, + pub alu_flags: u8, + pub mem_flags: u8, +} + +/// A run of pcs 4 apart (the prover's `DecodeTable` runs): pc +/// `first_pc + 4k` is DECODE row `row + k` for `k < len`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DecodeRunC { + pub first_pc: u64, + pub row: u32, + pub len: u32, +} + +/// One keccak permutation call (the prover's `KeccakRoundOperation`): its 24 +/// rows are the rounds from `input`. `output` is carried, not read. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KeccakOpC { + pub timestamp: u64, + pub input: [u64; 25], + pub output: [u64; 25], +} + +/// The same call as a table held as its keccak operations carries it +/// (`multilinear::records::KeccakRndOp`): the fields its rows read. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KeccakOpInC { + pub timestamp: u64, + pub input: [u64; 25], +} + +/// One MEMW_A row (the prover's `AlignedRow`): the base address, timestamp and +/// shared old timestamp, the value and old value packed a word each, the +/// width, and flags 0 or 1. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AlignedRowC { + pub base_address: u64, + pub timestamp: u64, + pub old_timestamp: u64, + pub value: u64, + pub old: u64, + pub width: u8, + pub is_read: u8, + pub is_register: u8, +} + +/// One LOAD op (the prover's `LoadOperation`): the base address, the +/// timestamp, the width, whether it sign-extends (0 or 1) and the result's +/// bytes. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LoadOpC { + pub base_address: u64, + pub timestamp: u64, + pub width: u8, + pub is_signed: u8, + pub res: [u8; 8], +} + +/// One STORE op (the prover's `StoreOperation`): the base address, the +/// timestamp, the value, and the width flags (0 or 1; none for one byte). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StoreOpC { + pub base_address: u64, + pub timestamp: u64, + pub value: u64, + pub write2: u8, + pub write4: u8, + pub write8: u8, +} + +/// One SHIFT op (the prover's `ShiftOperation`): the full shift amount, the +/// input as four 16-bit halves, the amount's low byte, and flags 0 or 1. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ShiftOpC { + pub shift_amount: u64, + pub in_halves: [u16; 4], + pub shift: u8, + pub direction: u8, + pub is_signed: u8, + pub word_instr: u8, +} + +/// One MEMW op (the prover's `MemwOperation`): the base address, the +/// timestamp, the eight old timestamps, the eight value and old elements, and +/// the flags and width. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MemwOpC { + pub base_address: u64, + pub timestamp: u64, + pub old_timestamp: [u64; 8], + pub value: [u32; 8], + pub old: [u32; 8], + pub is_register: u8, + pub width: u8, + pub is_read: u8, +} + +const _: () = { + assert!(size_of::() == 40); + assert!(offset_of!(LogC, dst_val) == 32); + assert!(size_of::() == 32); + assert!(offset_of!(RegRowC, old_ts_lo) == 24); + assert!(offset_of!(RegRowC, address) == 28); + assert!(offset_of!(RegRowC, is_read) == 30); + assert!(size_of::() == 32); + assert!(offset_of!(DecodeEntryC, read_register1) == 16); + assert!(offset_of!(DecodeEntryC, mem_flags) == 31); + assert!(size_of::() == 16); + assert!(offset_of!(DecodeRunC, len) == 12); + assert!(size_of::() == 408); + assert!(offset_of!(KeccakOpC, output) == 208); + assert!(size_of::() == 208); + assert!(offset_of!(KeccakOpInC, input) == 8); + assert!(size_of::() == 48); + assert!(offset_of!(AlignedRowC, width) == 40); + assert!(offset_of!(AlignedRowC, is_register) == 42); + assert!(size_of::() == 32); + assert!(offset_of!(LoadOpC, is_signed) == 17); + assert!(offset_of!(LoadOpC, res) == 18); + assert!(size_of::() == 32); + assert!(offset_of!(StoreOpC, write2) == 24); + assert!(size_of::() == 24); + assert!(offset_of!(ShiftOpC, in_halves) == 8); + assert!(offset_of!(ShiftOpC, word_instr) == 19); + assert!(size_of::() == 152); + assert!(offset_of!(MemwOpC, value) == 80); + assert!(offset_of!(MemwOpC, is_register) == 144); +}; + +// SAFETY: each is `#[repr(C)]` plain data whose padding no kernel reads +// (`RegRowC::pad` is a named field; the row families' structs end in +// padding), so its bytes on the card are the struct the kernel declares. +unsafe impl DeviceRepr for LogC {} +unsafe impl DeviceRepr for RegRowC {} +unsafe impl DeviceRepr for DecodeEntryC {} +unsafe impl DeviceRepr for DecodeRunC {} +unsafe impl DeviceRepr for KeccakOpC {} +unsafe impl DeviceRepr for KeccakOpInC {} +unsafe impl DeviceRepr for AlignedRowC {} +unsafe impl DeviceRepr for LoadOpC {} +unsafe impl DeviceRepr for StoreOpC {} +unsafe impl DeviceRepr for ShiftOpC {} +unsafe impl DeviceRepr for MemwOpC {} + +impl DecodeRunC { + /// The DECODE row of `pc` in `runs` (ascending and disjoint), as the CPU + /// kernel finds it: the last run starting at or below `pc`, if it holds it. + pub fn find(runs: &[DecodeRunC], pc: u64) -> Option { + let at = runs.partition_point(|run| run.first_pc <= pc); + let run = runs.get(at.checked_sub(1)?)?; + let offset = pc - run.first_pc; + (offset.is_multiple_of(4) && offset / 4 < u64::from(run.len)) + .then(|| run.row + (offset / 4) as u32) + } +} + +/// The event-timed phases of one fill or widen, on its stream. +#[derive(Clone, Copy, Debug)] +pub struct FillTimes { + /// The records' (or packed bytes') copy to the card, its allocation + /// included. + pub upload_ms: f64, + /// The kernel. + pub kernel_ms: f64, + /// The bytes copied. + pub upload_bytes: usize, +} + +struct Module { + _module: Arc, + memw_r: CudaFunction, + memw_a: CudaFunction, + load: CudaFunction, + store: CudaFunction, + shift: CudaFunction, + memw: CudaFunction, + cpu: CudaFunction, + keccak_rnd: CudaFunction, + keccak_rnd_in: CudaFunction, + lfm_p1w16_socket: CudaFunction, +} + +fn module() -> Result<&'static Module> { + static MODULE: OnceLock = OnceLock::new(); + if let Some(m) = MODULE.get() { + return Ok(m); + } + let be = backend()?; + let module = be.ctx.load_module(Ptx::from_binary(FILL_CUBIN.to_vec()))?; + let m = Module { + memw_r: module.load_function("fill_memw_r")?, + memw_a: module.load_function("fill_memw_a")?, + load: module.load_function("fill_load")?, + store: module.load_function("fill_store")?, + shift: module.load_function("fill_shift")?, + memw: module.load_function("fill_memw")?, + cpu: module.load_function("fill_cpu_from_logs")?, + keccak_rnd: module.load_function("fill_keccak_rnd")?, + keccak_rnd_in: module.load_function("fill_keccak_rnd_in")?, + lfm_p1w16_socket: module.load_function("fill_lfm_p1w16_socket")?, + _module: module, + }; + Ok(MODULE.get_or_init(|| m)) +} + +/// Three timing events on a stream: queued, uploaded, done. +struct Clock { + start: CudaEvent, + uploaded: CudaEvent, + done: CudaEvent, +} + +impl Clock { + /// Events that time, the first recorded now. + fn start(stream: &Arc) -> Result { + let flags = Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT); + let ctx = stream.context(); + let clock = Self { + start: ctx.new_event(flags)?, + uploaded: ctx.new_event(flags)?, + done: ctx.new_event(flags)?, + }; + clock.start.record(stream)?; + Ok(clock) + } + + /// Waits for the work queued so far and reads the two phases. + fn finish(&self, upload_bytes: usize) -> Result { + Ok(FillTimes { + upload_ms: f64::from(self.start.elapsed_ms(&self.uploaded)?), + kernel_ms: f64::from(self.uploaded.elapsed_ms(&self.done)?), + upload_bytes, + }) + } +} + +/// `records` on the card: a pageable copy on `stream`, as the widen path +/// copies packed bytes. An empty list uploads one default record so the +/// kernel gets a pointer; it reads none. +fn upload( + stream: &Arc, + records: &[T], +) -> Result> { + if records.is_empty() { + stream.clone_htod(std::slice::from_ref(&T::default())) + } else { + stream.clone_htod(records) + } +} + +fn row_grid(rows: usize, block: u32) -> Result { + u32::try_from(rows.div_ceil(block as usize)).map_err(|_| crate::invalid_value()) +} + +/// MEMW_R's `height` rows into `dst` (`height × MEMW_R_COLUMNS` words, +/// column-major): `rows`, then zero rows. All on `stream`; returns once the +/// kernel is done. +pub fn fill_memw_r_into( + stream: &Arc, + rows: &[RegRowC], + height: usize, + dst: &mut CudaViewMut, +) -> Result { + ensure!(rows.len() <= height, "more rows than the table"); + ensure_eq!(dst.len(), height * MEMW_R_COLUMNS, "the table's words"); + if height == 0 { + return Ok(FillTimes { + upload_ms: 0.0, + kernel_ms: 0.0, + upload_bytes: 0, + }); + } + let m = module()?; + let clock = Clock::start(stream)?; + let records = upload(stream, rows)?; + clock.uploaded.record(stream)?; + let (n, height_u64) = (rows.len() as u64, height as u64); + unsafe { + stream + .launch_builder(&m.memw_r) + .arg(&records) + .arg(&n) + .arg(&height_u64) + .arg(dst) + .launch(LaunchConfig { + grid_dim: (row_grid(height, ROW_BLOCK)?, 1, 1), + block_dim: (ROW_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + clock.done.record(stream)?; + clock.finish(size_of_val(rows)) +} + +/// A program's decode table on the card, uploaded once: the entries in pc +/// order and the runs of pcs 4 apart. +pub struct CardDecode { + entries: CudaSlice, + runs: CudaSlice, + nruns: u32, + /// What [`Self::resident`] promised the backend for it. + _room: Option, +} + +impl CardDecode { + /// `entries` and `runs` on `stream`. Refused unless the runs are ascending + /// and disjoint and every row they name is an entry. + pub fn upload( + stream: &Arc, + entries: &[DecodeEntryC], + runs: &[DecodeRunC], + ) -> Result { + ensure!(!entries.is_empty() && !runs.is_empty(), "a program"); + let nruns = u32::try_from(runs.len()).map_err(|_| crate::invalid_value())?; + let mut end_pc = 0u64; + for (k, run) in runs.iter().enumerate() { + ensure!(run.len > 0, "a run holds a pc"); + ensure!( + k == 0 || run.first_pc > end_pc, + "runs ascending and disjoint" + ); + ensure!( + run.row as usize + run.len as usize <= entries.len(), + "a run's rows are entries" + ); + end_pc = run + .first_pc + .checked_add(4 * (u64::from(run.len) - 1)) + .ok_or_else(crate::invalid_value)?; + } + Ok(Self { + entries: stream.clone_htod(entries)?, + runs: stream.clone_htod(runs)?, + nruns, + _room: None, + }) + } + + /// [`Self::upload`] for every stream's use, under a promise of its bytes + /// to the backend held as long as it is: uploaded on a stream of the pool + /// and waited for, so a fill on any stream reads it. Refused when the + /// backend declines the room. + pub fn resident(entries: &[DecodeEntryC], runs: &[DecodeRunC]) -> Result { + let be = backend()?; + let bytes = (size_of_val(entries) + size_of_val(runs)) as u64; + let Some(room) = be.reserve(bytes) else { + crate::device::note_device_fallback(); + return Err(crate::invalid_value()); + }; + let stream = be.next_stream(); + let mut decode = Self::upload(&stream, entries, runs)?; + stream.synchronize()?; + decode._room = Some(room); + Ok(decode) + } +} + +/// CPU's `height` rows into `dst` (`height × CPU_COLUMNS` words, +/// column-major): one a log, read with its entry of `decode`, then the padding +/// rows, every row's timestamp the table's own `4·(row + 1)`. Refused when a log's pc has +/// no DECODE row (the host fill's `MissingInstruction`). All on `stream`; +/// returns once the kernel is done. +pub fn fill_cpu_from_logs_into( + stream: &Arc, + logs: &[LogC], + decode: &CardDecode, + height: usize, + dst: &mut CudaViewMut, +) -> Result { + ensure!(logs.len() <= height, "more logs than the table"); + ensure_eq!(dst.len(), height * CPU_COLUMNS, "the table's words"); + if height == 0 { + return Ok(FillTimes { + upload_ms: 0.0, + kernel_ms: 0.0, + upload_bytes: 0, + }); + } + let m = module()?; + let mut missing = stream.alloc_zeros::(1)?; + let clock = Clock::start(stream)?; + let records = upload(stream, logs)?; + clock.uploaded.record(stream)?; + let (n, height_u64) = (logs.len() as u64, height as u64); + unsafe { + stream + .launch_builder(&m.cpu) + .arg(&records) + .arg(&n) + .arg(&decode.entries) + .arg(&decode.runs) + .arg(&decode.nruns) + .arg(&height_u64) + .arg(dst) + .arg(&mut missing) + .launch(LaunchConfig { + grid_dim: (row_grid(height, CPU_BLOCK)?, 1, 1), + block_dim: (CPU_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + clock.done.record(stream)?; + let times = clock.finish(size_of_val(logs))?; + let missing = crate::readback::clone_dtoh(stream, &missing)?; + stream.synchronize()?; + ensure_eq!(missing[0], 0, "every log's pc has a DECODE row"); + Ok(times) +} + +/// KECCAK_RND's rows `[skip, skip + rows)` of the table `ops` make (24 rows an +/// op) into `dst` (`rows × KECCAK_RND_COLUMNS` words, column-major), rows past +/// the last op zero. All on `stream`; returns once the kernel is done. +pub fn fill_keccak_rnd_into( + stream: &Arc, + ops: &[KeccakOpC], + skip: usize, + rows: usize, + dst: &mut CudaViewMut, +) -> Result { + ensure_eq!(dst.len(), rows * KECCAK_RND_COLUMNS, "the table's words"); + let nops = u32::try_from(ops.len()).map_err(|_| crate::invalid_value())?; + if rows == 0 { + return Ok(FillTimes { + upload_ms: 0.0, + kernel_ms: 0.0, + upload_bytes: 0, + }); + } + let m = module()?; + let clock = Clock::start(stream)?; + let records = upload(stream, ops)?; + clock.uploaded.record(stream)?; + launch_keccak_rnd(stream, &m.keccak_rnd, &records, nops, skip, rows, dst)?; + clock.done.record(stream)?; + clock.finish(size_of_val(ops)) +} + +/// `fill_keccak_rnd`'s launch (or `fill_keccak_rnd_in`'s, as `function` says) +/// over records already on the card. +fn launch_keccak_rnd( + stream: &Arc, + function: &CudaFunction, + records: &CudaSlice, + nops: u32, + skip: usize, + rows: usize, + dst: &mut CudaViewMut, +) -> Result<()> { + let (skip_u64, rows_u64) = (skip as u64, rows as u64); + unsafe { + stream + .launch_builder(function) + .arg(records) + .arg(&nops) + .arg(&skip_u64) + .arg(&rows_u64) + .arg(dst) + .launch(LaunchConfig { + grid_dim: (row_grid(rows, KECCAK_BLOCK)?, KECCAK_SECTIONS, 1), + block_dim: (KECCAK_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + Ok(()) +} + +/// A table's records as they go up for the card to fill its columns from +/// them ([`crate::columns::TableUpload::Records`]). +#[derive(Clone, Copy)] +pub enum RecordsInput<'a> { + /// KECCAK_RND's rows `[skip, skip + rows)` of the table `ops` make, 24 + /// rows an op; rows past the last op zero. + KeccakRnd { + ops: &'a [KeccakOpInC], + skip: usize, + rows: usize, + }, + /// CPU's `rows` rows: one a log of `logs`' pieces in order, read with its + /// row of `decode` at the table's own timestamps; then the padding rows. + Cpu { + logs: &'a [&'a [LogC]], + decode: &'a CardDecode, + rows: usize, + }, + /// MEMW_R's `rows` rows: `records`' pieces in order, then zero rows. + MemwR { + records: &'a [&'a [RegRowC]], + rows: usize, + }, + /// MEMW_A's `rows` rows: `records`' pieces in order, then zero rows. + MemwA { + records: &'a [&'a [AlignedRowC]], + rows: usize, + }, + /// LOAD's `rows` rows: `records`' pieces in order, then zero rows. + Load { + records: &'a [&'a [LoadOpC]], + rows: usize, + }, + /// STORE's `rows` rows: `records`' pieces in order, then zero rows. + Store { + records: &'a [&'a [StoreOpC]], + rows: usize, + }, + /// SHIFT's `rows` rows: `records`' pieces in order, then padding rows. + Shift { + records: &'a [&'a [ShiftOpC]], + rows: usize, + }, + /// MEMW's `rows` rows: `records`' pieces in order, then zero rows. + Memw { + records: &'a [&'a [MemwOpC]], + rows: usize, + }, +} + +impl std::fmt::Debug for RecordsInput<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let family = match self { + Self::KeccakRnd { .. } => "KECCAK_RND", + Self::Cpu { .. } => "CPU", + Self::MemwR { .. } => "MEMW_R", + Self::MemwA { .. } => "MEMW_A", + Self::Load { .. } => "LOAD", + Self::Store { .. } => "STORE", + Self::Shift { .. } => "SHIFT", + Self::Memw { .. } => "MEMW", + }; + write!( + f, + "{family} records: {} rows, {} bytes", + self.rows(), + self.bytes() + ) + } +} + +impl RecordsInput<'_> { + /// Rows of the table. + pub fn rows(&self) -> usize { + match self { + Self::KeccakRnd { rows, .. } + | Self::Cpu { rows, .. } + | Self::MemwR { rows, .. } + | Self::MemwA { rows, .. } + | Self::Load { rows, .. } + | Self::Store { rows, .. } + | Self::Shift { rows, .. } + | Self::Memw { rows, .. } => *rows, + } + } + + /// Columns of the table. + pub fn cols(&self) -> usize { + match self { + Self::KeccakRnd { .. } => KECCAK_RND_COLUMNS, + Self::Cpu { .. } => CPU_COLUMNS, + Self::MemwR { .. } => MEMW_R_COLUMNS, + Self::MemwA { .. } => MEMW_A_COLUMNS, + Self::Load { .. } => LOAD_COLUMNS, + Self::Store { .. } => STORE_COLUMNS, + Self::Shift { .. } => SHIFT_COLUMNS, + Self::Memw { .. } => MEMW_COLUMNS, + } + } + + /// The records' bytes (a CPU table's decode table aside: it is on the + /// card already). + pub fn bytes(&self) -> usize { + match self { + Self::KeccakRnd { ops, .. } => size_of_val(*ops), + Self::Cpu { logs, .. } => logs.iter().map(|piece| size_of_val(*piece)).sum(), + Self::MemwR { records, .. } => records.iter().map(|piece| size_of_val(*piece)).sum(), + Self::MemwA { records, .. } => records.iter().map(|piece| size_of_val(*piece)).sum(), + Self::Load { records, .. } => records.iter().map(|piece| size_of_val(*piece)).sum(), + Self::Store { records, .. } => records.iter().map(|piece| size_of_val(*piece)).sum(), + Self::Shift { records, .. } => records.iter().map(|piece| size_of_val(*piece)).sum(), + Self::Memw { records, .. } => records.iter().map(|piece| size_of_val(*piece)).sum(), + } + } + + /// The records' count: logs, rows or ops. + fn len(&self) -> usize { + match self { + Self::KeccakRnd { ops, .. } => ops.len(), + Self::Cpu { logs, .. } => logs.iter().map(|piece| piece.len()).sum(), + Self::MemwR { records, .. } => records.iter().map(|piece| piece.len()).sum(), + Self::MemwA { records, .. } => records.iter().map(|piece| piece.len()).sum(), + Self::Load { records, .. } => records.iter().map(|piece| piece.len()).sum(), + Self::Store { records, .. } => records.iter().map(|piece| piece.len()).sum(), + Self::Shift { records, .. } => records.iter().map(|piece| piece.len()).sum(), + Self::Memw { records, .. } => records.iter().map(|piece| piece.len()).sum(), + } + } +} + +/// What the fills of one upload check once the stream's work is done +/// ([`Self::passed`]): a CPU log whose pc has no DECODE row. +#[derive(Default)] +pub(crate) struct FillChecks { + missing: Option>, +} + +impl FillChecks { + /// The counter of logs with no DECODE row, zeroed on `stream` at first use. + fn missing(&mut self, stream: &Arc) -> Result<&mut CudaSlice> { + if self.missing.is_none() { + self.missing = Some(stream.alloc_zeros::(1)?); + } + self.missing.as_mut().ok_or_else(crate::invalid_value) + } + + /// Whether every fill's check passed, read after `stream` ran them: every + /// CPU log's pc found its DECODE row (the host fill's + /// `MissingInstruction` otherwise). + pub(crate) fn passed(&self, stream: &Arc) -> Result { + let Some(missing) = &self.missing else { + return Ok(true); + }; + let missing = crate::readback::clone_dtoh(stream, missing)?; + stream.synchronize()?; + Ok(missing.first() == Some(&0)) + } +} + +/// `pieces` on the card, one after another in one buffer: each a pageable +/// copy on `stream`, as [`upload`] copies one slice. No record uploads one +/// default record so the kernel gets a pointer; it reads none. +fn upload_pieces( + stream: &Arc, + pieces: &[&[T]], +) -> Result> { + let n: usize = pieces.iter().map(|piece| piece.len()).sum(); + if n == 0 { + return upload(stream, &[]); + } + if let [piece] = pieces { + return upload(stream, piece); + } + // SAFETY: every element is written by the copies below, which cover + // `0..n` piece by piece. + let mut records = unsafe { crate::device::alloc_or_trim::(stream, n) }?; + let mut at = 0usize; + for piece in pieces.iter().filter(|piece| !piece.is_empty()) { + let mut view = records.slice_mut(at..at + piece.len()); + stream.memcpy_htod(*piece, &mut view)?; + at += piece.len(); + } + Ok(records) +} + +/// Test-only fault: with the `test-faults` feature, a table filled from its +/// records is filled this many rows late (plus +/// `LAMBDA_VM_TEST_FAULT_CARD_FILL_SKIP`, read at the first fill): other words +/// than the host fill's, which the card tests and a proof's verifier must +/// catch. +#[cfg(feature = "test-faults")] +pub static FAULT_RECORDS_SKIP: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + +#[cfg(feature = "test-faults")] +fn fault_records_skip() -> usize { + static FROM_ENV: OnceLock = OnceLock::new(); + let from_env = *FROM_ENV.get_or_init(|| { + std::env::var("LAMBDA_VM_TEST_FAULT_CARD_FILL_SKIP") + .ok() + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0) + }); + from_env + FAULT_RECORDS_SKIP.load(std::sync::atomic::Ordering::Relaxed) +} + +/// Test-only fault: with the `test-faults` feature, a CPU table filled from +/// its records has two of its columns swapped, as a kernel that wrote each +/// into the other's place would: other words than the host fill's, which the +/// card tests and a proof's verifier must catch. The pair is +/// `LAMBDA_VM_TEST_FAULT_CARD_FILL_SWAP=,` (read at the first fill), or +/// else what this holds (`1 + a · CPU_COLUMNS + b`; 0 for none). +#[cfg(feature = "test-faults")] +pub static FAULT_CPU_SWAP: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +#[cfg(feature = "test-faults")] +fn fault_cpu_swap() -> Option<(usize, usize)> { + static FROM_ENV: OnceLock> = OnceLock::new(); + let from_env = *FROM_ENV.get_or_init(|| { + let value = std::env::var("LAMBDA_VM_TEST_FAULT_CARD_FILL_SWAP").ok()?; + let (a, b) = value.trim().split_once(',')?; + Some((a.trim().parse().ok()?, b.trim().parse().ok()?)) + }); + let held = FAULT_CPU_SWAP.load(std::sync::atomic::Ordering::Relaxed); + from_env + .or_else(|| { + held.checked_sub(1) + .map(|k| (k / CPU_COLUMNS, k % CPU_COLUMNS)) + }) + .filter(|&(a, b)| a != b && a < CPU_COLUMNS && b < CPU_COLUMNS) +} + +/// Columns `a` and `b` of the `rows`-row table in `dst` swapped, on `stream`. +#[cfg(feature = "test-faults")] +fn swap_columns( + stream: &Arc, + dst: &mut CudaViewMut, + rows: usize, + (a, b): (usize, usize), +) -> Result<()> { + let column_a = stream.clone_dtod(&dst.slice(a * rows..(a + 1) * rows))?; + let column_b = stream.clone_dtod(&dst.slice(b * rows..(b + 1) * rows))?; + stream.memcpy_dtod(&column_b, &mut dst.slice_mut(a * rows..(a + 1) * rows))?; + stream.memcpy_dtod(&column_a, &mut dst.slice_mut(b * rows..(b + 1) * rows))?; + Ok(()) +} + +/// `records`' table into `dst` (`rows × cols` words, column-major), all on +/// `stream`: the records copied up as the widen path copies packed bytes +/// (piece by piece into one buffer) and the family's kernel launched, the copy +/// freed in stream order, so a reader on the same stream needs no further +/// ordering. What the kernel checks is counted into `checks`, read once the +/// stream's work is done ([`FillChecks::passed`]). +pub(crate) fn fill_col_major_into( + stream: &Arc, + records: RecordsInput<'_>, + dst: &mut CudaViewMut, + checks: &mut FillChecks, +) -> Result<()> { + ensure_eq!( + dst.len(), + records.rows() * records.cols(), + "the table's words" + ); + ensure!(records.len() <= records.rows() || matches!(records, RecordsInput::KeccakRnd { .. })); + if records.rows() == 0 { + return Ok(()); + } + let m = module()?; + match records { + RecordsInput::KeccakRnd { ops, skip, rows } => { + #[cfg(feature = "test-faults")] + let skip = skip + fault_records_skip(); + let nops = u32::try_from(ops.len()).map_err(|_| crate::invalid_value())?; + let ops_dev = upload(stream, ops)?; + launch_keccak_rnd(stream, &m.keccak_rnd_in, &ops_dev, nops, skip, rows, dst) + } + RecordsInput::Cpu { logs, decode, rows } => { + let n = records.len() as u64; + let logs_dev = upload_pieces(stream, logs)?; + let height = rows as u64; + let missing = checks.missing(stream)?; + unsafe { + stream + .launch_builder(&m.cpu) + .arg(&logs_dev) + .arg(&n) + .arg(&decode.entries) + .arg(&decode.runs) + .arg(&decode.nruns) + .arg(&height) + .arg(&mut *dst) + .arg(missing) + .launch(LaunchConfig { + grid_dim: (row_grid(rows, CPU_BLOCK)?, 1, 1), + block_dim: (CPU_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + #[cfg(feature = "test-faults")] + if let Some(pair) = fault_cpu_swap() { + swap_columns(stream, dst, rows, pair)?; + } + Ok(()) + } + RecordsInput::MemwR { + records: pieces, + rows, + } => { + let records_dev = upload_pieces(stream, pieces)?; + launch_rows(stream, &m.memw_r, &records_dev, records.len(), rows, dst) + } + RecordsInput::MemwA { + records: pieces, + rows, + } => { + let records_dev = upload_pieces(stream, pieces)?; + launch_rows(stream, &m.memw_a, &records_dev, records.len(), rows, dst) + } + RecordsInput::Load { + records: pieces, + rows, + } => { + let records_dev = upload_pieces(stream, pieces)?; + launch_rows(stream, &m.load, &records_dev, records.len(), rows, dst) + } + RecordsInput::Store { + records: pieces, + rows, + } => { + let records_dev = upload_pieces(stream, pieces)?; + launch_rows(stream, &m.store, &records_dev, records.len(), rows, dst) + } + RecordsInput::Shift { + records: pieces, + rows, + } => { + let records_dev = upload_pieces(stream, pieces)?; + launch_rows(stream, &m.shift, &records_dev, records.len(), rows, dst) + } + RecordsInput::Memw { + records: pieces, + rows, + } => { + let records_dev = upload_pieces(stream, pieces)?; + launch_rows(stream, &m.memw, &records_dev, records.len(), rows, dst) + } + } +} + +/// A one-record-a-row family's fill over the `n` records on the card into +/// `dst`'s `rows` rows (zero past the records), on `stream`. +fn launch_rows( + stream: &Arc, + function: &CudaFunction, + records: &CudaSlice, + n: usize, + rows: usize, + dst: &mut CudaViewMut, +) -> Result<()> { + let (n, height) = (n as u64, rows as u64); + unsafe { + stream + .launch_builder(function) + .arg(records) + .arg(&n) + .arg(&height) + .arg(&mut *dst) + .launch(LaunchConfig { + grid_dim: (row_grid(rows, ROW_BLOCK)?, 1, 1), + block_dim: (ROW_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + Ok(()) +} + +/// Today's path, timed as the fills are: `packed`'s bytes, offsets and widths +/// uploaded, then widened column-major into `dst` (`rows × cols` words) by the +/// kernel [`crate::columns::DeviceColumns::upload_tables`] launches. All on +/// `stream`; returns once the kernel is done. +pub fn widen_col_major_timed( + stream: &Arc, + packed: NarrowInput, + dst: &mut CudaViewMut, +) -> Result { + let (rows, cols) = (packed.rows(), packed.cols()); + let cells = rows * cols; + ensure_eq!(dst.len(), cells, "the table's words"); + let cells_u32 = u32::try_from(cells).map_err(|_| crate::invalid_value())?; + if cells == 0 { + return Ok(FillTimes { + upload_ms: 0.0, + kernel_ms: 0.0, + upload_bytes: 0, + }); + } + let be = backend()?; + let clock = Clock::start(stream)?; + let (data, offsets, widths) = packed.parts(); + let data_dev = stream.clone_htod(data)?; + let offsets_dev = stream.clone_htod(offsets)?; + let widths_dev = stream.clone_htod(widths)?; + clock.uploaded.record(stream)?; + let (rows_u64, cols_u64) = (rows as u64, cols as u64); + unsafe { + stream + .launch_builder(&be.widen_narrow_col_major) + .arg(&data_dev) + .arg(&offsets_dev) + .arg(&widths_dev) + .arg(dst) + .arg(&rows_u64) + .arg(&cols_u64) + .launch(LaunchConfig::for_num_elems(cells_u32))?; + } + clock.done.record(stream)?; + clock.finish(size_of_val(data) + size_of_val(offsets) + size_of_val(widths)) +} + +/// The LFM width-16 Poseidon1 socket's row-major trace (`rows × width` words) +/// on the card, from `narrow`: each row's preprocessed prefix (`prep_w` words) +/// and its sixteen input lanes, `rows × (prep_w + 16)` words row-major. The +/// kernel (`fill_lfm_p1w16_socket`) copies those and writes the round witness +/// from `rounds_off` for the first `real_rows` rows, zeros past them — word for +/// word the host's `p1w16_socket::fill_row` trace. The narrow words cross the +/// bus through the staging pairs, as the R1 commit's own upload would; the +/// stream is synchronized before return, so the buffer is ready for any stream. +pub fn lfm_p1w16_socket_rows( + narrow: &[u64], + rows: usize, + real_rows: usize, + prep_w: usize, + width: usize, + rounds_off: usize, +) -> Result> { + const LANES: usize = 16; + ensure!(rows > 0 && real_rows <= rows, "real rows within the table"); + ensure!( + rounds_off >= prep_w + LANES && rounds_off <= width, + "the witness after the inputs" + ); + ensure_eq!(narrow.len(), rows * (prep_w + LANES), "the narrow words"); + let m = module()?; + let stream = backend()?.next_stream(); + let mut narrow_dev = unsafe { stream.alloc::(narrow.len()) }?; + crate::device::htod_staged(&stream, narrow, &mut narrow_dev.slice_mut(..))?; + let mut out = unsafe { stream.alloc::(rows * width) }?; + let (rows_u64, real_u64, prep_u64, width_u64, off_u64) = ( + rows as u64, + real_rows as u64, + prep_w as u64, + width as u64, + rounds_off as u64, + ); + unsafe { + stream + .launch_builder(&m.lfm_p1w16_socket) + .arg(&narrow_dev) + .arg(&rows_u64) + .arg(&real_u64) + .arg(&prep_u64) + .arg(&width_u64) + .arg(&off_u64) + .arg(&mut out) + .launch(LaunchConfig { + grid_dim: (row_grid(rows, ROW_BLOCK)?, 1, 1), + block_dim: (ROW_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + stream.synchronize()?; + drop(narrow_dev); + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn runs() -> Vec { + vec![ + DecodeRunC { + first_pc: 0x1000, + row: 0, + len: 3, + }, + DecodeRunC { + first_pc: 0x2000, + row: 3, + len: 1, + }, + DecodeRunC { + first_pc: 0x2008, + row: 4, + len: 2, + }, + ] + } + + /// Every pc of the runs finds its row, and the pcs around them none. + #[test] + fn a_pc_finds_the_row_of_its_run() { + let runs = runs(); + let want = [ + (0x1000, Some(0)), + (0x1004, Some(1)), + (0x1008, Some(2)), + (0x2000, Some(3)), + (0x2008, Some(4)), + (0x200c, Some(5)), + ]; + for (pc, row) in want { + assert_eq!(DecodeRunC::find(&runs, pc), row, "pc {pc:#x}"); + } + for pc in [0, 0xffc, 0x1002, 0x100c, 0x1ffc, 0x2004, 0x2010, u64::MAX] { + assert_eq!(DecodeRunC::find(&runs, pc), None, "pc {pc:#x}"); + } + assert_eq!(DecodeRunC::find(&[], 0x1000), None); + } +} diff --git a/crypto/math-cuda/src/fri.rs b/crypto/math-cuda/src/fri.rs index 533ff6e32..1da06d331 100644 --- a/crypto/math-cuda/src/fri.rs +++ b/crypto/math-cuda/src/fri.rs @@ -12,9 +12,9 @@ use cudarc::driver::{CudaSlice, CudaStream, LaunchConfig, PushKernelArg}; use std::sync::Arc; +use crate::DeviceHash; use crate::Result; use crate::device::backend; -use crate::merkle::build_inner_tree_levels; /// Test-only fault injection. When the `test-faults` feature is on, setting /// this to a finite value forces the next `fold_and_commit_layer` call to @@ -47,6 +47,8 @@ fn check_fault_injection() -> Result<()> { /// of the evals exists. Freed when the last holder drops. pub struct FriCommitState { pub stream: Arc, + /// Which hash family commits each layer's tree. + hash: DeviceHash, /// Current fold input. Each fold allocates a fresh output buffer that is /// both returned to the caller (kept resident for the query phase) and /// becomes the next fold's input. @@ -61,7 +63,12 @@ impl FriCommitState { /// H2D the starting evals (ext3 interleaved, 3 * n0 u64) and the /// initial inv_twiddles (base field, n0/2 u64). `n0` must be a power of /// two and >= 2. - pub fn new(evals_host: &[u64], inv_tw_host: &[u64], n0: usize) -> Result { + pub fn new( + evals_host: &[u64], + inv_tw_host: &[u64], + n0: usize, + hash: DeviceHash, + ) -> Result { assert!(n0 >= 2 && n0.is_power_of_two()); assert_eq!(evals_host.len(), 3 * n0); assert_eq!(inv_tw_host.len(), n0 / 2); @@ -76,6 +83,7 @@ impl FriCommitState { Ok(Self { stream, + hash, current: Arc::new(evals), inv_tw, current_n: n0, @@ -84,7 +92,11 @@ impl FriCommitState { /// Like [`Self::new`], but adopts a device-resident codeword (already in /// FRI bit-reversed order) and its producing stream — no evals H2D. - pub fn new_dev(codeword: crate::deep::GpuDeepCodeword, inv_tw_host: &[u64]) -> Result { + pub fn new_dev( + codeword: crate::deep::GpuDeepCodeword, + inv_tw_host: &[u64], + hash: DeviceHash, + ) -> Result { let crate::deep::GpuDeepCodeword { buf, n, stream } = codeword; assert!(n >= 2 && n.is_power_of_two()); assert_eq!(buf.len(), 3 * n); @@ -94,13 +106,15 @@ impl FriCommitState { Ok(Self { stream, + hash, current: Arc::new(buf), inv_tw, current_n: n, }) } - /// Fold the current layer using `zeta`, run the row-pair Keccak leaves and + /// Fold the current layer using `zeta`, run the row-pair leaf kernels of + /// the configured hash family and /// pair-hash Merkle tree kernels on the result, and return the layer's /// evals — device-resident Arc, plus a host copy only when `want_host` — /// with its resident Merkle tree (root D2H'd, 32 bytes). @@ -130,7 +144,7 @@ impl FriCommitState { // Row-pair leaves: each leaf hashes two consecutive ext3 evals. let num_leaves = n_out / 2; - let tight_total_nodes = 2 * num_leaves - 1; + let tight_total_nodes = crate::tree_nodes(self.hash, num_leaves); // H2D zeta. let zeta_dev = self.stream.clone_htod(&zeta_raw)?; @@ -162,27 +176,78 @@ impl FriCommitState { // and build_inner_tree_levels writes every inner node [0, num_leaves-1), so all // tight_total_nodes * 32 bytes are initialised before the D2H below reads them. let mut nodes_dev = unsafe { self.stream.alloc::(tight_total_nodes * 32) }?; - let leaves_offset_bytes = (num_leaves - 1) * 32; + let leaves_offset_bytes = crate::leaves_offset(self.hash, num_leaves) * 32; { let mut leaves_view = nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); let num_leaves_u64 = num_leaves as u64; - let grid = (num_leaves as u32).div_ceil(128); - let kcfg = LaunchConfig { - grid_dim: (grid, 1, 1), - block_dim: (128, 1, 1), - shared_mem_bytes: 0, - }; - unsafe { - self.stream - .launch_builder(&be.keccak_fri_leaves_ext3) - .arg(&out) - .arg(&num_leaves_u64) - .arg(&mut leaves_view) - .launch(kcfg)?; + match self.hash { + DeviceHash::Keccak256 => { + let grid = (num_leaves as u32).div_ceil(128); + let kcfg = LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (128, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + self.stream + .launch_builder(&be.keccak_fri_leaves_ext3) + .arg(&out) + .arg(&num_leaves_u64) + .arg(&mut leaves_view) + .launch(kcfg)?; + } + } + DeviceHash::Blake3 => crate::blake3::launch_fri_leaves_ext3( + self.stream.as_ref(), + be, + &out, + num_leaves_u64, + &mut leaves_view, + )?, + DeviceHash::Rpx256 => crate::rpx::launch_fri_leaves_ext3( + self.stream.as_ref(), + be, + &out, + num_leaves_u64, + &mut leaves_view, + )?, + // The pair leaf is the two-value group leaf (six felts). + DeviceHash::Poseidon1 => crate::p1_stark::launch_fri_group_leaves( + self.stream.as_ref(), + &out, + num_leaves_u64, + 2, + &mut leaves_view, + )?, } } - build_inner_tree_levels(self.stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + match self.hash { + DeviceHash::Keccak256 => crate::merkle::build_inner_tree_levels( + self.stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + DeviceHash::Keccak256, + )?, + DeviceHash::Blake3 => crate::blake3::build_inner_tree_levels( + self.stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + )?, + DeviceHash::Rpx256 => crate::rpx::build_inner_tree_levels( + self.stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + )?, + DeviceHash::Poseidon1 => crate::p1_stark::build_inner_tree_levels( + self.stream.as_ref(), + &mut nodes_dev, + num_leaves, + )?, + } // Update inv_twiddles for the next layer: `new[j] = old[2j]^2` for // j in 0..n_out/2. (If n_out == 1, skip; no next fold.) Writes into @@ -230,8 +295,7 @@ impl FriCommitState { // This pageable copy drains the stream (including any evals DMA above), // so the pending wait after it is instant — one block covers both. let mut root = [0u8; 32]; - self.stream - .memcpy_dtoh(&nodes_dev.slice(0..32), &mut root)?; + crate::readback::memcpy_dtoh(&self.stream, &nodes_dev.slice(0..32), &mut root)?; let layer_evals = match pending { Some(p) => { let mut v = vec![0u64; n_evals]; @@ -248,10 +312,254 @@ impl FriCommitState { let tree = crate::lde::GpuMerkleTree { nodes: std::sync::Arc::new(nodes_dev), leaves_len: num_leaves, + arity: self.hash.arity(), root, }; Ok((layer_evals, out, tree)) } + + /// One binary fold of the current codeword with `zeta_raw` into a fresh + /// buffer (the same `fri_fold_ext3` launch as [`Self::fold_and_commit_layer`]), + /// then the twiddle update for the halved domain. The output becomes the + /// current codeword; the input is released once no caller holds it. + fn fold_once(&mut self, be: &crate::device::Backend, zeta_raw: [u64; 3]) -> Result<()> { + let n_out = self.current_n / 2; + assert!(n_out >= 1, "fold_once: nothing left to fold"); + let zeta_dev = self.stream.clone_htod(&zeta_raw)?; + let cfg = LaunchConfig { + grid_dim: ((n_out as u32).div_ceil(128), 1, 1), + block_dim: (128, 1, 1), + shared_mem_bytes: 0, + }; + let n_out_u64 = n_out as u64; + // SAFETY: the fold kernel writes all 3 * n_out slots before any read. + let mut out = unsafe { self.stream.alloc::(3 * n_out) }?; + unsafe { + self.stream + .launch_builder(&be.fri_fold_ext3) + .arg(self.current.as_ref()) + .arg(&n_out_u64) + .arg(&self.inv_tw) + .arg(&zeta_dev) + .arg(&mut out) + .launch(cfg)?; + } + // `new[j] = old[2j]^2` into a fresh buffer (see `fold_and_commit_layer` + // for why not in place). + let tw_next = n_out / 2; + if tw_next > 0 { + // SAFETY: the update kernel writes all tw_next slots. + let mut tw_out = unsafe { self.stream.alloc::(tw_next) }?; + let cfg = LaunchConfig { + grid_dim: ((tw_next as u32).div_ceil(128), 1, 1), + block_dim: (128, 1, 1), + shared_mem_bytes: 0, + }; + let tw_next_u64 = tw_next as u64; + unsafe { + self.stream + .launch_builder(&be.fri_update_twiddles) + .arg(&self.inv_tw) + .arg(&mut tw_out) + .arg(&tw_next_u64) + .launch(cfg)?; + } + self.inv_tw = tw_out; + } + self.current = Arc::new(out); + self.current_n = n_out; + Ok(()) + } + + /// The S3 (higher-arity committed FRI) step: fold the current codeword + /// `zeta_powers.len()` times — fold `ℓ` with `zeta_powers[ℓ]`, which the + /// caller sets to `ζ^{2^ℓ}` — then commit the result as a layer whose leaf + /// `g` hashes the `2^group_log` consecutive ext3 values + /// `[g·2^group_log, (g+1)·2^group_log)` (the configured hash's `Batched` + /// leaf over the group), with the pair-hash inner tree on top. + /// + /// The fold count and the group size are separate on purpose: committed + /// layer `j` is reached by the PREVIOUS layer's `d_{j−1}` folds and grouped + /// by its own `d_j`. Only the last fold's output is kept; the + /// intermediate codewords are released as the chain advances. + /// + /// Returns what [`Self::fold_and_commit_layer`] returns: the layer's evals + /// (host copy only when `want_host`), the resident evals, and the resident + /// tree with its root D2H'd. + #[allow(clippy::type_complexity)] + pub fn fold_and_commit_group( + &mut self, + zeta_powers: &[[u64; 3]], + group_log: u32, + want_host: bool, + ) -> Result<( + Option>, + Arc>, + crate::lde::GpuMerkleTree, + )> { + #[cfg(feature = "test-faults")] + check_fault_injection()?; + let be = backend()?; + for &z in zeta_powers { + self.fold_once(be, z)?; + } + let n = self.current_n; + assert!( + group_log >= 1 && (n >> group_log) >= 2 && (n >> group_log) << group_log == n, + "fold_and_commit_group: a layer of {n} values cannot hold >= 2 groups of 2^{group_log}" + ); + let num_leaves = n >> group_log; + let nodes_dev = commit_group_leaves( + &self.stream, + be, + self.hash, + self.current.as_ref(), + num_leaves, + 1u64 << group_log, + )?; + + let n_evals = 3 * n; + let pending = if want_host { + Some(crate::device::async_dtoh_via( + &self.stream, + be.pinned_staging(), + &be.ctx, + self.current.as_ref(), + n_evals, + )?) + } else { + None + }; + // The pageable root copy drains the stream, the evals DMA included. + let mut root = [0u8; 32]; + crate::readback::memcpy_dtoh(&self.stream, &nodes_dev.slice(0..32), &mut root)?; + let layer_evals = match pending { + Some(p) => { + let mut v = vec![0u64; n_evals]; + p.wait_into_u64(&mut v)?; + Some(v) + } + None => None, + }; + let tree = crate::lde::GpuMerkleTree { + nodes: Arc::new(nodes_dev), + leaves_len: num_leaves, + arity: self.hash.arity(), + root, + }; + Ok((layer_evals, Arc::clone(&self.current), tree)) + } + + /// Fold the current codeword `zeta_powers.len()` times (fold `ℓ` with + /// `zeta_powers[ℓ]`) and copy the result to the host, with no commitment: + /// the S3 fold into the terminal codeword after the last committed layer. + pub fn fold_to_host(&mut self, zeta_powers: &[[u64; 3]]) -> Result> { + #[cfg(feature = "test-faults")] + check_fault_injection()?; + let be = backend()?; + for &z in zeta_powers { + self.fold_once(be, z)?; + } + let out = crate::readback::clone_dtoh(&self.stream, self.current.as_ref())?; + self.stream.synchronize()?; + Ok(out) + } +} + +/// Hash `num_leaves` group leaves of `group` consecutive ext3 values each from +/// the interleaved `evals` (`3 · num_leaves · group` u64) and build the inner +/// tree on top: the full `(2·num_leaves − 1) · 32`-byte node buffer, root at 0. +fn commit_group_leaves( + stream: &Arc, + be: &crate::device::Backend, + hash: DeviceHash, + evals: &CudaSlice, + num_leaves: usize, + group: u64, +) -> Result> { + assert!(num_leaves >= 2 && num_leaves.is_power_of_two()); + assert!(evals.len() as u64 >= 3 * num_leaves as u64 * group); + let tight_total_nodes = crate::tree_nodes(hash, num_leaves); + // SAFETY: the leaf kernel writes the leaves (the buffer's tail) and the + // inner-level walk every node before them, before any read. + let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; + let leaves_offset_bytes = crate::leaves_offset(hash, num_leaves) * 32; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); + let num_leaves_u64 = num_leaves as u64; + let (kernel, cfg) = match hash { + DeviceHash::Poseidon1 => ( + &crate::p1_stark::kernels()?.fri_group_leaves, + crate::p1_stark::launch_cfg(num_leaves_u64), + ), + DeviceHash::Keccak256 => ( + &be.keccak_fri_group_leaves_ext3, + crate::merkle::keccak_launch_cfg(num_leaves_u64), + ), + DeviceHash::Blake3 => ( + &be.blake3_fri_group_leaves_ext3, + crate::blake3::blake3_launch_cfg(num_leaves_u64), + ), + DeviceHash::Rpx256 => ( + &be.rpx_fri_group_leaves_ext3, + crate::rpx::rpx_launch_cfg(num_leaves_u64), + ), + }; + unsafe { + stream + .launch_builder(kernel) + .arg(evals) + .arg(&num_leaves_u64) + .arg(&group) + .arg(&mut leaves_view) + .launch(cfg)?; + } + } + match hash { + DeviceHash::Keccak256 => crate::merkle::build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + DeviceHash::Keccak256, + )?, + DeviceHash::Blake3 => { + crate::blake3::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)? + } + DeviceHash::Rpx256 => { + crate::rpx::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)? + } + DeviceHash::Poseidon1 => { + crate::p1_stark::build_inner_tree_levels(stream.as_ref(), &mut nodes_dev, num_leaves)? + } + } + Ok(nodes_dev) +} + +/// Parity harness (not a production path): commit an interleaved ext3 eval +/// vector as an S3 group-leaf FRI layer — leaf `g` = the `2^group_log` +/// consecutive values from `g·2^group_log` — under `hash`, and return the full +/// host node buffer (`(2·num_leaves − 1) · 32` bytes, standard layout) so tests +/// can compare it node for node with the host tree. Production commits through +/// [`FriCommitState::fold_and_commit_group`], over the same kernels. +pub fn build_fri_group_tree_from_evals_ext3( + evals: &[u64], + group_log: u32, + hash: DeviceHash, +) -> Result> { + assert!(evals.len().is_multiple_of(3)); + let n = evals.len() / 3; + let num_leaves = n >> group_log; + assert!(num_leaves << group_log == n, "whole groups only"); + let be = backend()?; + let stream = be.next_stream(); + let evals_dev = stream.clone_htod(evals)?; + let nodes_dev = + commit_group_leaves(&stream, be, hash, &evals_dev, num_leaves, 1u64 << group_log)?; + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; + stream.synchronize()?; + Ok(out) } /// Gather interleaved ext3 elements at `positions` from a resident evals @@ -293,7 +601,7 @@ pub fn gather_ext3_at( .arg(&mut out_dev) .launch(cfg)?; } - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(stream, &out_dev)?; stream.synchronize()?; Ok(out) } diff --git a/crypto/math-cuda/src/gkr.rs b/crypto/math-cuda/src/gkr.rs new file mode 100644 index 000000000..7ff11b2f4 --- /dev/null +++ b/crypto/math-cuda/src/gkr.rs @@ -0,0 +1,1092 @@ +//! A LogUp fraction tree resident on device, and the sumchecks over its +//! layers. +//! +//! The tree is the biggest structure a table's argument holds — its input +//! layer spans `interactions × rows` fractions — and GKR walks every level of +//! it. Building it here and leaving it here means the layers are never copied: +//! a layer's halves are the sumcheck's factors in place, and the round that +//! binds them is the same kernel the batched statements use. +//! +//! Each layer is visited exactly once, top down, so the rounds fold it where +//! it lies and nothing reads it again. + +use std::sync::Arc; + +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, LaunchConfig, PushKernelArg}; + +use crate::Result; +use crate::device::backend; +use crate::sumcheck::SumcheckSession; + +/// One level: numerators and denominators over the same cube. +struct DeviceLayer { + p: Arc>, + q: Arc>, + num_vars: usize, + /// The cells `p` and `q` hold: the whole cube, `2^num_vars`, except for an + /// input layer kept without its padding + /// ([`DeviceFractionTree::from_truncated_input`]), whose cells from here + /// on are the padding's `0/1`. + real: usize, +} + +impl DeviceLayer { + fn whole(&self) -> bool { + self.real == 1usize << self.num_vars + } +} + +fn invalid() -> cudarc::driver::DriverError { + cudarc::driver::DriverError(cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE) +} + +/// The whole tree, `layers[0]` the output fraction and layer `layers.len()` +/// the input — the order `multilinear::gkr::FractionTree` uses. +pub struct DeviceFractionTree { + stream: Arc, + /// Every level above the input layer. + layers: Vec, + /// The input layer, when this tree carries it. A tree built from the + /// factors does not: it is the biggest thing here — as many fractions as + /// interactions times rows — and it is read exactly twice, to fold the + /// level above it and by its own sumcheck at the very end. Between those + /// two the caller rebuilds it rather than carry it. + input: Option, + input_num_vars: usize, + /// The room the tree promised itself, when it is the tree that promised + /// it. A tree that hands its input layer back does not: the promise has to + /// outlive it, so the caller holds it. + _room: Option, +} + +/// A layer's two halves as whoever wrote them leaves them: `p` and `q` over +/// the same cells, before anything decides whether they are a whole cube. +pub type Halves = (CudaSlice, CudaSlice); + +/// One layer's two halves on device — what a caller hands back when the tree +/// it built asks for its input layer again. +pub struct InputLayer { + stream: Arc, + p: Arc>, + q: Arc>, + num_vars: usize, +} + +impl InputLayer { + pub fn new( + stream: Arc, + p: CudaSlice, + q: CudaSlice, + num_vars: usize, + ) -> Self { + assert_eq!(p.len(), q.len(), "a layer's halves span one cube"); + assert_eq!(p.len(), (1usize << num_vars) * 3, "the cube is the layer"); + Self { + stream, + p: Arc::new(p), + q: Arc::new(q), + num_vars, + } + } + + /// The layer's own sumcheck, the one that ends GKR. + pub fn sumcheck( + &self, + point: &[u64], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + sumcheck_over_layer( + &self.stream, + &self.p, + &self.q, + self.num_vars, + point, + nodes, + consts, + num_slots, + root_slot, + ) + } +} + +impl DeviceFractionTree { + /// Uploads the input layer and folds every level above it. + /// + /// `p` and `q` are interleaved ext3 over the same cube. + pub fn build(p: &[u64], q: &[u64]) -> Result { + assert_eq!(p.len(), q.len(), "a layer's halves span one cube"); + assert!(p.len().is_multiple_of(3), "three u64 per ext3 element"); + let elements = p.len() / 3; + assert!(elements.is_power_of_two(), "the cube is a power of two"); + + let be = backend()?; + let stream = be.next_stream(); + let input_p = crate::device::htod_or_trim(&stream, p)?; + let input_q = crate::device::htod_or_trim(&stream, q)?; + Self::from_device(stream, input_p, input_q) + } + + /// The same for an input layer the device already holds — what the LogUp + /// fingerprints write straight into. + pub fn from_device( + stream: Arc, + p: CudaSlice, + q: CudaSlice, + ) -> Result { + assert_eq!(p.len(), q.len(), "a layer's halves span one cube"); + assert!(p.len().is_multiple_of(3), "three u64 per ext3 element"); + let elements = p.len() / 3; + assert!(elements.is_power_of_two(), "the cube is a power of two"); + + let be = backend()?; + // The levels above the input layer halve, so the whole tree is twice + // it — and the input layer is `p` and `q` together. + let Some(room) = be.reserve(p.len() as u64 * 8 * 4) else { + crate::device::note_device_fallback(); + return Err(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY, + )); + }; + crate::argue_probe::note_device(crate::argue_probe::Surface::Gkr, p.len() as u64 * 8 * 4); + let input = DeviceLayer { + p: Arc::new(p), + q: Arc::new(q), + num_vars: elements.trailing_zeros() as usize, + real: elements, + }; + let layers = fold_upwards(&stream, be, &input.p, &input.q, elements, elements)?; + Ok(Self { + stream, + layers, + input_num_vars: input.num_vars, + input: Some(input), + _room: Some(room), + }) + } + + /// The same for an input layer that stops where its real fractions do. + /// + /// The interactions are padded up to a power of two with `0/1`, and that + /// padding is half the cube for the widest precompiles. `p` and `q` hold + /// `real` fractions of a cube of `1 << num_vars`; the first fold reads the + /// rest as the `0/1` it is, and **the input layer is let go as soon as the + /// level above it exists**. Its sumcheck comes last, by which time every + /// level above is spent, so the caller rebuilds it then — which is why + /// this tree asks for a fraction of what carrying it costs. + /// + /// **The caller holds the room**, because the promise has to cover the + /// input layer it hands back after this tree is gone. + /// [`padded_peak_bytes`] is what to promise. + pub fn from_padded_input( + stream: Arc, + p: CudaSlice, + q: CudaSlice, + real: usize, + num_vars: usize, + ) -> Result { + assert_eq!(p.len(), q.len(), "a layer's halves span one cube"); + assert_eq!(p.len(), real * 3, "three u64 per ext3 element"); + let full = 1usize << num_vars; + assert!(real <= full, "the real fractions fit the cube"); + assert!( + num_vars == 0 || real > full / 2, + "a count rounded up to a power of two leaves less than half padding" + ); + + let be = backend()?; + let layers = { + let p = Arc::new(p); + let q = Arc::new(q); + fold_upwards(&stream, be, &p, &q, real, full)? + // `p` and `q` go here, which is the point of this constructor. + }; + Ok(Self { + stream, + layers, + input: None, + input_num_vars: num_vars, + _room: None, + }) + } + + /// The same truncated input layer, **kept**: the first fold reads its + /// padding as the `0/1` it is, as [`from_padded_input`](Self::from_padded_input)'s + /// does, and the layer's own Gruen rounds read it that way too + /// ([`layer_gruen`](Self::layer_gruen)), so it is never written. That is + /// the cells of the padding left out of the write, the first fold and the + /// input layer's first rounds, which are as large as the layer. + /// + /// Only the Gruen rounds read a layer kept this way; today's session over + /// it ([`layer_sumcheck`](Self::layer_sumcheck)) is refused, and its caller + /// writes the whole layer again. + /// + /// **The caller holds the room** — [`truncated_peak_bytes`] — because a + /// caller that has to write the whole layer for today's session does so + /// after this tree is gone. A shape no count rounded up to a power of two + /// gives is refused before anything runs. + pub fn from_truncated_input( + stream: Arc, + p: CudaSlice, + q: CudaSlice, + real: usize, + num_vars: usize, + ) -> Result { + let full = 1usize.checked_shl(num_vars as u32).ok_or_else(invalid)?; + let rounded_up = real <= full && real > full / 2; + if p.len() != real * 3 || q.len() != real * 3 || !rounded_up { + return Err(invalid()); + } + let be = backend()?; + crate::argue_probe::note_device( + crate::argue_probe::Surface::Gkr, + truncated_peak_bytes(real, full), + ); + let input = DeviceLayer { + p: Arc::new(p), + q: Arc::new(q), + num_vars, + real, + }; + let layers = fold_upwards(&stream, be, &input.p, &input.q, real, full)?; + Ok(Self { + stream, + layers, + input: Some(input), + input_num_vars: num_vars, + _room: None, + }) + } + + /// Every level plus the input layer, whether or not this tree holds it. + pub fn num_layers(&self) -> usize { + self.layers.len() + 1 + } + + pub fn layer_num_vars(&self, layer: usize) -> usize { + self.layers + .get(layer) + .map_or(self.input_num_vars, |level| level.num_vars) + } + + /// Whether [`layer_sumcheck`](Self::layer_sumcheck) can still prove the + /// input layer, or the caller has to hand it back first. + pub fn holds_input(&self) -> bool { + self.input.is_some() + } + + /// Whether the input layer it holds was kept without its padding + /// ([`from_truncated_input`](Self::from_truncated_input)). + pub fn input_truncated(&self) -> bool { + self.input.as_ref().is_some_and(|input| !input.whole()) + } + + /// The output fraction `(p, q)`, the one the bus balance is read off. + pub fn output(&self) -> Result<([u64; 3], [u64; 3])> { + let top = self + .layers + .first() + .or(self.input.as_ref()) + .expect("a tree has a top"); + let p = crate::readback::clone_dtoh(&self.stream, &top.p.slice(0..3))?; + let q = crate::readback::clone_dtoh(&self.stream, &top.q.slice(0..3))?; + self.stream.synchronize()?; + Ok(([p[0], p[1], p[2]], [q[0], q[1], q[2]])) + } + + /// A level's halves back on the host, interleaved ext3. + /// + /// Only the levels near the output are worth asking for: a round over a + /// cube of a few hundred is a launch and a wait around a kernel with + /// almost nothing to sum, and the whole level is a few kilobytes. The + /// input layer is never one of them — it is the biggest thing here, and a + /// tree that dropped it has nothing to hand over. + pub fn layer_to_host(&self, layer: usize) -> Result<(Vec, Vec)> { + let level = self.layers.get(layer).ok_or(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE, + ))?; + let p = crate::readback::clone_dtoh(&self.stream, &level.p.slice(0..level.p.len()))?; + let q = crate::readback::clone_dtoh(&self.stream, &level.q.slice(0..level.q.len()))?; + self.stream.synchronize()?; + Ok((p, q)) + } + + /// A sumcheck over layer `layer`, with `eq(point, ·)` as factor 0 and the + /// layer's four halves — `p_lo`, `p_hi`, `q_lo`, `q_hi` — as factors 1..5. + /// + /// The rounds fold those halves where they lie. GKR visits each layer once, + /// so nothing reads them afterwards, and what the fold leaves behind is + /// exactly the four values the layer reduces to. + #[allow(clippy::too_many_arguments)] + pub fn layer_sumcheck( + &self, + layer: usize, + point: &[u64], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + let level = match self.layers.get(layer) { + Some(level) => level, + // The input layer, which this tree only has if it kept it. + None => self.input.as_ref().ok_or(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE, + ))?, + }; + // This session reads every cell of the halves: a layer kept without + // its padding is the Gruen rounds' alone. + if !level.whole() { + return Err(invalid()); + } + sumcheck_over_layer( + &self.stream, + &level.p, + &level.q, + level.num_vars, + point, + nodes, + consts, + num_slots, + root_slot, + ) + } +} + +/// Every level above an input layer of `full` fractions, of which the first +/// `real` are stored — the rest are the padding's `0/1` and the first fold +/// reads them as that. Returned with the output first, the way the tree +/// indexes them. +fn fold_upwards( + stream: &Arc, + be: &crate::device::Backend, + p: &Arc>, + q: &Arc>, + real: usize, + full: usize, +) -> Result> { + let mut layers: Vec = Vec::new(); + let mut num_vars = full.trailing_zeros() as usize; + while num_vars > 0 { + let half = (1usize << num_vars) / 2; + // SAFETY: the kernel writes every element of the level it produces. + let mut p_out = unsafe { crate::device::alloc_or_trim::(stream, half * 3) }?; + let mut q_out = unsafe { crate::device::alloc_or_trim::(stream, half * 3) }?; + let (below_p, below_q) = match layers.last() { + Some(level) => (level.p.clone(), level.q.clone()), + None => (p.clone(), q.clone()), + }; + let half_arg = half as u64; + let padded = layers.is_empty() && real != full; + unsafe { + if padded { + let real_arg = real as u64; + stream + .launch_builder(&be.fraction_fold_padded_ext3) + .arg(below_p.as_ref()) + .arg(below_q.as_ref()) + .arg(&half_arg) + .arg(&real_arg) + .arg(&mut p_out) + .arg(&mut q_out) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } else { + stream + .launch_builder(&be.fraction_fold_ext3) + .arg(below_p.as_ref()) + .arg(below_q.as_ref()) + .arg(&half_arg) + .arg(&mut p_out) + .arg(&mut q_out) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + } + num_vars -= 1; + layers.push(DeviceLayer { + p: Arc::new(p_out), + q: Arc::new(q_out), + num_vars, + real: half, + }); + } + layers.reverse(); + Ok(layers) +} + +/// A sumcheck over one layer's halves, with `eq(point, ·)` as factor 0 and +/// `p_lo`, `p_hi`, `q_lo`, `q_hi` as factors 1..5. +/// +/// The rounds fold those halves where they lie. GKR visits each layer once, so +/// nothing reads them afterwards, and what the fold leaves behind is exactly +/// the four values the layer reduces to. +#[allow(clippy::too_many_arguments)] +pub fn sumcheck_over_layer( + stream: &Arc, + p: &Arc>, + q: &Arc>, + num_vars: usize, + point: &[u64], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, +) -> Result { + assert!(num_vars > 0, "the output layer has nothing to bind"); + let half_vars = num_vars - 1; + assert_eq!(point.len(), half_vars * 3, "the point spans the halves"); + + let half = 1usize << half_vars; + let eq = Arc::new(crate::sumcheck::eq_table_ext3(stream, point, half)?); + let addresses = { + let (eq_at, _eq_guard) = eq.device_ptr(stream); + let (p_at, _p_guard) = p.device_ptr(stream); + let (q_at, _q_guard) = q.device_ptr(stream); + let stride = (half * 3 * 8) as u64; + vec![eq_at, p_at, p_at + stride, q_at, q_at + stride] + }; + + SumcheckSession::from_device( + stream.clone(), + &addresses, + half, + vec![eq, p.clone(), q.clone()], + nodes, + consts, + num_slots, + root_slot, + ) +} + +/// What a tree built by [`DeviceFractionTree::from_padded_input`] holds at +/// once: the truncated input layer plus the level above it while the first +/// fold runs, against every level once the input is gone — and, later, the +/// full input layer handed back for its own sumcheck, by which time the levels +/// are spent. +pub fn padded_peak_bytes(real: usize, full: usize) -> u64 { + ((2 * real + full).max(2 * full)) as u64 * 24 +} + +/// What a tree built by [`DeviceFractionTree::from_truncated_input`] holds at +/// once: the truncated input layer, kept, and every level above it (`full − 1` +/// cells of each side) — against `4·full` cells for a tree that carries its +/// padding. Also covers the whole layer written again for today's session +/// once the tree is gone. +pub fn truncated_peak_bytes(real: usize, full: usize) -> u64 { + (2 * real + 2 * full) as u64 * 24 +} + +/// The run of an upper half's cells a layer kept without its padding still +/// holds after a round over `quarter` indices: the fold leaves a half of +/// `2·quarter` cells, and a cell past the run held before it folds two padding +/// cells into the padding again. +pub fn stored_after_round(stored: usize, quarter: usize) -> usize { + stored.min(2 * quarter) +} + +/// Blocks a Gruen round launches at most; past it the grid-stride loop takes +/// over. A few blocks an SM on a 170-SM card, and the partials stay small. +const GRUEN_MAX_GRID: u32 = 2048; + +/// One GKR layer's device rounds with Gruen's split (D-ARGUE S1-3): the round +/// sums `H(1)` and `H(2)` of `h = p_lo·q_hi + p_hi·q_lo + λ·q_lo·q_hi` under +/// `eq(u_{>j}, ·)`, and the host puts `eq(u_{≤j})` back in (see +/// `multilinear::gkr_gruen`). The fold by a round's challenge rides the next +/// round's pass, and the weight is two small tables a layer instead of an `eq` +/// table the size of the layer folded every round. +/// +/// Folds the layer's halves in place, as [`sumcheck_over_layer`]'s session +/// does: GKR visits each layer once. +pub struct GruenLayer { + stream: Arc, + /// The buffers the halves live in, kept alive; the kernels see addresses. + p: Arc>, + q: Arc>, + p_at: u64, + q_at: u64, + /// Cells in each half: `2^m`. + half: usize, + /// The run of cells the upper halves `p_hi`, `q_hi` hold before the next + /// fold — every cell of the cube they hold now, for a layer stored whole. + /// Past it is the padding of a layer kept without it + /// ([`DeviceFractionTree::from_truncated_input`]), read as `0/1`. + stored: usize, + /// Whether the layer was opened without its padding. + truncated: bool, + /// The point's limbs, `m` ext3 values. + point: Vec, + /// `L`: the variables left to the host, `2^L` cells. + low: usize, + /// `J = m − L`: the rounds this layer runs here. + rounds: usize, + /// Every `eq` level ([`gkr_eq_levels_ext3`'s layout](crate) — `e_lo`, then + /// one `e_hi` level per round). + eq: CudaSlice, + partials: CudaSlice, + sums: CudaSlice, + /// Rounds run so far. + done: usize, + /// The rounds' sums through a slot (`LAMBDA_VM_ARGUE_ROUND_SLOT`): + /// reduced by the round kernel's last block, read from pinned host memory. + slot: Option, +} + +impl GruenLayer { + /// Over one layer's halves: `p` and `q` of `2^num_vars` cells each — or + /// the first `real` of them, more than half, for an input layer kept + /// without its padding — the point `m = num_vars − 1` ext3 values, and at + /// most `low` variables left to the host (fewer when the layer is smaller). + #[allow(clippy::too_many_arguments)] + fn new( + stream: &Arc, + p: &Arc>, + q: &Arc>, + num_vars: usize, + real: usize, + point: &[u64], + low: usize, + ) -> Result { + assert!(num_vars > 0, "the output layer has nothing to bind"); + let m = num_vars - 1; + assert_eq!(point.len(), m * 3, "the point spans the halves"); + let half = 1usize << m; + if p.len() != real * 3 || q.len() != real * 3 || real <= half || real > 2 * half { + return Err(invalid()); + } + let low = low.min(m); + let rounds = m - low; + + let be = backend()?; + let cells = (1usize << low) + (1usize << rounds) - 1; + // SAFETY: the kernel writes every cell. + let mut eq = unsafe { crate::device::alloc_or_trim::(stream, cells * 3) }?; + let point_dev = crate::device::htod_or_trim( + stream, + if point.is_empty() { &[0u64][..] } else { point }, + )?; + let low_arg = low as u32; + let rounds_arg = rounds as u32; + unsafe { + stream + .launch_builder(&be.gkr_eq_levels_ext3) + .arg(&point_dev) + .arg(&low_arg) + .arg(&rounds_arg) + .arg(&mut eq) + .launch(LaunchConfig { + grid_dim: ((cells as u64).div_ceil(256).min(4096) as u32, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + })?; + } + let partials = unsafe { + crate::device::alloc_or_trim::(stream, 3 * GRUEN_MAX_GRID as usize * 3) + }?; + let sums = crate::device::alloc_zeros_or_trim::(stream, 3 * 3)?; + let p_at = { + let (at, _guard) = p.device_ptr(stream); + at + }; + let q_at = { + let (at, _guard) = q.device_ptr(stream); + at + }; + Ok(Self { + stream: stream.clone(), + p: p.clone(), + q: q.clone(), + p_at, + q_at, + half, + stored: real - half, + truncated: real < 2 * half, + point: point.to_vec(), + low, + rounds, + eq, + partials, + sums, + done: 0, + slot: None, + }) + } + + /// The rounds' sums through a slot from here on (`crate::round_slot`) + /// when `mode` asks for one and one can be had; `false` keeps today's + /// second launch and read-back. + pub fn use_slot(&mut self, mode: crate::round_slot::RoundSlot) -> bool { + self.slot = crate::round_slot::SlotRounds::open(&self.stream, mode); + self.slot.is_some() + } + + /// The rounds this layer runs on the card, `J`. + pub fn rounds(&self) -> usize { + self.rounds + } + + /// Whether the layer was opened without its padding. + pub fn truncated(&self) -> bool { + self.truncated + } + + /// The variables left to the host, `L`: the halves come back over `2^L`. + pub fn low(&self) -> usize { + self.low + } + + /// The next round's sums, `H(1)` and `H(2)` — and `H(0)` third with + /// `want_h0` — three u64 each. `fold_by` is the previous round's challenge, + /// and every round but the first has one. + pub fn round( + &mut self, + fold_by: Option<&[u64; 3]>, + lambda: &[u64; 3], + want_h0: bool, + ) -> Result> { + let rows = self.round_enqueue(fold_by, lambda, want_h0)?; + self.round_collect(rows) + } + + /// [`round`](Self::round)'s launches, queued without waiting for them — + /// so the batched argue's ladder can queue every tree's round before it + /// reads any back. Returns the rows [`round_collect`](Self::round_collect) + /// reads. + pub fn round_enqueue( + &mut self, + fold_by: Option<&[u64; 3]>, + lambda: &[u64; 3], + want_h0: bool, + ) -> Result { + assert!(self.done < self.rounds, "every card round has run"); + assert_eq!( + fold_by.is_some(), + self.done > 0, + "a challenge to fold by after the first round" + ); + let be = backend()?; + let m = self.half.trailing_zeros() as usize; + let quarter = 1u64 << (m - self.done - 1); + let (grid, block) = gruen_shape(quarter); + let rows = if want_h0 { 3u32 } else { 2 }; + let s = fold_by.copied().unwrap_or([0; 3]); + let fold = u32::from(fold_by.is_some()); + let at = + (1usize << self.low) + (1usize << self.rounds) - (1usize << (self.rounds - self.done)); + let e_lo = self.eq.slice(0..(1usize << self.low) * 3); + let e_hi = self + .eq + .slice(at * 3..(at + (1usize << (self.rounds - 1 - self.done))) * 3); + let half = self.half as u64; + let low = self.low as u32; + let want = u32::from(want_h0); + let stored = self.stored as u64; + // The cube the upper halves hold coming into this round; once their + // stored run covers it, the padding is spent and today's kernel reads + // every cell. + let cube = if fold_by.is_some() { 4 } else { 2 } * quarter; + let kernel = match (stored < cube, self.slot.is_some()) { + (true, false) => &be.gkr_round_gruen_real, + (false, false) => &be.gkr_round_gruen, + (true, true) => &be.gkr_round_gruen_real_slot, + (false, true) => &be.gkr_round_gruen_slot, + }; + let config = LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: if self.slot.is_some() { + crate::round_slot::shared_bytes(block) + } else { + block * 3 * 8 + }, + }; + // Today's second launch, unless the slot's last block does its work. + let second = self.slot.as_ref().is_none_or(|slot| slot.checking()); + let mut slot_args = self.slot.as_mut().map(|slot| slot.launch()); + unsafe { + let mut launch = self.stream.launch_builder(kernel); + launch + .arg(&self.p_at) + .arg(&self.q_at) + .arg(&half) + .arg(&quarter) + .arg(&fold) + .arg(&s[0]) + .arg(&s[1]) + .arg(&s[2]) + .arg(&lambda[0]) + .arg(&lambda[1]) + .arg(&lambda[2]) + .arg(&e_hi) + .arg(&e_lo) + .arg(&low) + .arg(&want); + if stored < cube { + launch.arg(&stored); + } + launch.arg(&mut self.partials); + if let Some((ticket, dev, seq)) = slot_args.as_mut() { + launch.arg(&mut **ticket).arg(&*dev).arg(&*seq); + } + launch.launch(config)?; + if second { + crate::round_slot::note_second_launch(); + let reduce_block = 256u32; + self.stream + .launch_builder(&be.sum_partials_ext3) + .arg(&self.partials) + .arg(&(grid as u64)) + .arg(&mut self.sums) + .launch(LaunchConfig { + grid_dim: (rows, 1, 1), + block_dim: (reduce_block, 1, 1), + shared_mem_bytes: reduce_block * 3 * 8, + })?; + } + } + self.stored = stored_after_round(self.stored, quarter as usize); + self.done += 1; + Ok(rows) + } + + /// The sums a [`round_enqueue`](Self::round_enqueue) queued, `rows` of + /// them, three u64 each. + pub fn round_collect(&self, rows: u32) -> Result> { + let today = || -> Result> { + let sums = + crate::readback::clone_dtoh(&self.stream, &self.sums.slice(0..rows as usize * 3))?; + self.stream.synchronize()?; + Ok(sums) + }; + match self.slot.as_ref() { + Some(slot) => slot.collect(rows as usize, today), + None => today(), + } + } + + /// After the last card round: the four halves bound to its challenge + /// (`fold_by`, `None` for a layer that ran no card round), over `2^L` + /// cells, laid end to end — `p_lo, p_hi, q_lo, q_hi`, three u64 a cell. + /// Also left in place, where a [`shadow`](Self::shadow) reads them. + pub fn finish(&mut self, fold_by: Option<&[u64; 3]>) -> Result> { + assert_eq!(self.done, self.rounds, "the card's rounds are done"); + assert_eq!( + fold_by.is_some(), + self.rounds > 0, + "a challenge to fold by after a card round" + ); + let be = backend()?; + let cells = 1u64 << self.low; + // SAFETY: the kernel writes every cell. + let mut out = + unsafe { crate::device::alloc_or_trim::(&self.stream, 4 * cells as usize * 3) }?; + let s = fold_by.copied().unwrap_or([0; 3]); + let fold = u32::from(fold_by.is_some()); + let half = self.half as u64; + let stored = self.stored as u64; + // The cube the halves hold: twice the cells with a fold to run. + let cube = if fold_by.is_some() { 2 * cells } else { cells }; + let kernel = if stored < cube { + &be.gkr_gruen_finish_real + } else { + &be.gkr_gruen_finish + }; + unsafe { + let mut launch = self.stream.launch_builder(kernel); + launch + .arg(&self.p_at) + .arg(&self.q_at) + .arg(&half) + .arg(&cells) + .arg(&fold) + .arg(&s[0]) + .arg(&s[1]) + .arg(&s[2]); + if stored < cube { + launch.arg(&stored); + } + launch.arg(&mut out).launch(LaunchConfig { + grid_dim: ((4 * cells).div_ceil(256).min(4096) as u32, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + })?; + } + let values = crate::readback::clone_dtoh(&self.stream, &out)?; + self.stream.synchronize()?; + Ok(values) + } + + /// Today's session over the same halves, for checking these rounds against + /// it (`LAMBDA_VM_ARGUE_GKR_GRUEN_XCHECK`): factor 0 its own `eq(u, ·)` + /// table, factors 1..5 the halves where they lie. Run its round after this + /// layer's round `j` has folded the halves, with + /// [`SumcheckSession::fold_first`] keeping its `eq` in step. + /// + /// Refused for a layer opened without its padding: today's session reads + /// every cell of the halves. + pub fn shadow( + &self, + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + if self.truncated { + return Err(invalid()); + } + let eq = Arc::new(crate::sumcheck::eq_table_ext3( + &self.stream, + &self.point, + self.half, + )?); + let eq_at = { + let (at, _guard) = eq.device_ptr(&self.stream); + at + }; + let stride = (self.half * 3 * 8) as u64; + let addresses = vec![ + eq_at, + self.p_at, + self.p_at + stride, + self.q_at, + self.q_at + stride, + ]; + SumcheckSession::from_device( + self.stream.clone(), + &addresses, + self.half, + vec![eq, self.p.clone(), self.q.clone()], + nodes, + consts, + num_slots, + root_slot, + ) + } + + /// Bytes [`shadow`](Self::shadow) allocates: the `eq` table and today's + /// round scratch. + pub fn shadow_bytes(&self, num_slots: usize) -> u64 { + self.half as u64 * 24 + SumcheckSession::scratch_bytes(self.half, num_slots) + } +} + +/// A Gruen round's launch: 256-thread blocks for a wide cube, narrower ones — +/// down to a warp — for a narrow one, and at most [`GRUEN_MAX_GRID`] blocks. +fn gruen_shape(quarter: u64) -> (u32, u32) { + let block = quarter.next_power_of_two().clamp(32, 256) as u32; + let grid = quarter + .div_ceil(block as u64) + .clamp(1, GRUEN_MAX_GRID as u64) as u32; + (grid, block) +} + +impl DeviceFractionTree { + /// Layer `layer`'s Gruen rounds ([`GruenLayer`]), at most `low` variables + /// left to the host. + pub fn layer_gruen(&self, layer: usize, point: &[u64], low: usize) -> Result { + let level = match self.layers.get(layer) { + Some(level) => level, + None => self.input.as_ref().ok_or(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE, + ))?, + }; + GruenLayer::new( + &self.stream, + &level.p, + &level.q, + level.num_vars, + level.real, + point, + low, + ) + } +} + +impl InputLayer { + /// The input layer's Gruen rounds, the ones that end GKR. + pub fn gruen(&self, point: &[u64], low: usize) -> Result { + GruenLayer::new( + &self.stream, + &self.p, + &self.q, + self.num_vars, + 1usize << self.num_vars, + point, + low, + ) + } +} + +/// An input layer's plan on the card (D-BATCH M1-2): each side's range of +/// terms, the terms — where a term's column starts in the table's run of base +/// columns and its shift — their ext3 coefficients, and each side's constant. +/// Uploaded once a table and kept for the layer's rewrite. +pub struct InputPlan { + side_start: CudaSlice, + terms: CudaSlice, + coeffs: CudaSlice, + constants: CudaSlice, + interactions: usize, +} + +impl InputPlan { + /// `side_start` has `2·interactions + 1` entries (side `2i` the + /// numerator of interaction `i`, `2i + 1` its denominator); `terms` two + /// u64 a term, `coeffs` three, `constants` three a side. + pub fn upload( + stream: &Arc, + side_start: &[u32], + terms: &[u64], + coeffs: &[u64], + constants: &[u64], + ) -> Result { + assert!( + side_start.len() % 2 == 1, + "two sides an interaction, and the end" + ); + let interactions = side_start.len() / 2; + let count = *side_start.last().expect("the end") as usize; + assert_eq!(terms.len(), 2 * count, "two u64 a term"); + assert_eq!(coeffs.len(), 3 * count, "an ext3 coefficient a term"); + assert_eq!(constants.len(), 6 * interactions, "an ext3 constant a side"); + // An allocation to point at when a table's sides have no terms. + let or_one = |v: &[u64]| if v.is_empty() { vec![0u64] } else { v.to_vec() }; + Ok(Self { + side_start: crate::device::htod_or_trim(stream, side_start)?, + terms: crate::device::htod_or_trim(stream, &or_one(terms))?, + coeffs: crate::device::htod_or_trim(stream, &or_one(coeffs))?, + constants: crate::device::htod_or_trim(stream, &or_one(constants))?, + interactions, + }) + } + + pub fn interactions(&self) -> usize { + self.interactions + } +} + +/// The input layer written from the table's base columns in one launch +/// (`gkr_input_from_columns`): `columns` is the table's run, `rows` a column, +/// and the slots are the interactions, padded to a power of two with `0/1` +/// when `padding` — the layout the lifted path writes. +pub fn input_from_columns( + stream: &Arc, + columns: &cudarc::driver::CudaView<'_, u64>, + rows: usize, + plan: &InputPlan, + padding: bool, +) -> Result { + let be = backend()?; + let interactions = plan.interactions; + let slots = if padding { + interactions.next_power_of_two() + } else { + interactions + }; + let cells = slots * rows; + // SAFETY: the kernel writes every cell of both. + let mut p = unsafe { crate::device::alloc_or_trim::(stream, cells * 3) }?; + let mut q = unsafe { crate::device::alloc_or_trim::(stream, cells * 3) }?; + let rows_arg = rows as u64; + let interactions_arg = interactions as u64; + let slots_arg = slots as u64; + unsafe { + stream + .launch_builder(&be.gkr_input_from_columns) + .arg(columns) + .arg(&rows_arg) + .arg(&plan.side_start) + .arg(&plan.terms) + .arg(&plan.coeffs) + .arg(&plan.constants) + .arg(&interactions_arg) + .arg(&slots_arg) + .arg(&mut p) + .arg(&mut q) + .launch(LaunchConfig { + grid_dim: ((cells as u64).div_ceil(256).clamp(1, 8192) as u32, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + })?; + } + Ok((p, q)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Where the padding of an upper half starts, and that it runs to the end. + fn padding_start(pad: &[bool]) -> usize { + let start = pad.iter().position(|&p| p).unwrap_or(pad.len()); + assert!( + pad[start..].iter().all(|&p| p), + "the padding is one run to the end" + ); + start + } + + /// ★ The run a kept layer's upper halves hold is exact: worked out cell + /// by cell (a folded cell is padding exactly when both cells it folds + /// are), the padding is one run to the end of the live half after every + /// fold, starting where [`stored_after_round`] says — so a kernel that + /// reads the run and the constants past it reads every cell the whole + /// layer would. And a round reads its cube through the kept kernel + /// exactly while padding is left in it. + #[test] + fn the_stored_run_is_where_the_padding_starts_after_every_fold() { + for m in 1..=12usize { + let half = 1usize << m; + for stored in [1, 2, half / 3 + 1, half / 2, half / 2 + 1, half - 1, half] { + if stored == 0 || stored > half { + continue; + } + let mut pad: Vec = (0..half).map(|i| i >= stored).collect(); + let mut run = stored; + for j in 0..m { + let what = format!("m {m}, stored {stored}, round {j}"); + let quarter = 1usize << (m - j - 1); + // The cube the halves hold coming into round `j`: round + // `j > 0` folds the previous round's on the way in. + let cube = if j > 0 { 4 * quarter } else { 2 * quarter }; + assert_eq!(pad.len(), cube, "{what}: the live half"); + let start = padding_start(&pad); + assert_eq!(run.min(cube), start, "{what}: the run held coming in"); + assert_eq!( + run < cube, + start < cube, + "{what}: the kept kernel exactly while padding is left" + ); + if j > 0 { + pad = (0..2 * quarter) + .map(|x| pad[x] && pad[x + 2 * quarter]) + .collect(); + } + run = stored_after_round(run, quarter); + assert_eq!(run, padding_start(&pad), "{what}: the run held after it"); + } + } + } + } + + /// The room a kept tree promises: the kept layer and every level above + /// it — what carrying it costs when there is no padding to leave out. + #[test] + fn a_kept_tree_promises_its_layer_and_the_levels_above() { + let full = 1usize << 20; + assert_eq!(truncated_peak_bytes(full, full), (4 * full) as u64 * 24); + // A CPU table: 20 of 32 slots real, 2^21 rows. + let (rows, slots, interactions) = (1usize << 21, 32usize, 20usize); + let (real, full) = (interactions * rows, slots * rows); + assert_eq!( + truncated_peak_bytes(real, full), + (2 * real + 2 * full) as u64 * 24 + ); + assert!(truncated_peak_bytes(real, full) < (4 * full) as u64 * 24); + // It covers the kept layer plus every level above (full − 1 cells a + // side), and the whole layer written again once the tree is gone. + let levels = 2 * (full - 1); + assert!(truncated_peak_bytes(real, full) >= (2 * real + levels) as u64 * 24); + assert!(truncated_peak_bytes(real, full) >= (2 * full) as u64 * 24); + } +} diff --git a/crypto/math-cuda/src/grinding.rs b/crypto/math-cuda/src/grinding.rs index fe7803eb9..761ee7091 100644 --- a/crypto/math-cuda/src/grinding.rs +++ b/crypto/math-cuda/src/grinding.rs @@ -1,51 +1,595 @@ -//! GPU proof-of-work grinding: a parallel Keccak nonce search that mirrors the -//! host `stark::grinding::generate_nonce`, offloading the ~2^grinding_factor -//! hashes it does per table per epoch from the CPU (where they dominate the -//! prove) to the otherwise-idle GPU. +//! GPU proof-of-work grinding: a parallel nonce search that mirrors the host +//! `crypto::grinding::generate_nonce`, offloading the ~2^grinding_factor hashes +//! it does per grind from the CPU — where they dominate the prove — to the +//! otherwise-idle GPU. +//! +//! Two arms, one per outer hash: [`generate_nonce_gpu`] for keccak-256 and +//! [`generate_nonce_rpx_gpu`] for RPX256. They differ in the kernel and in **how +//! the 32-byte inner hash is read into four `u64`s** — LITTLE-endian lanes for +//! keccak, BIG-endian felts for RPX. Everything else (the min-factor gate, the +//! block sizing, the sentinel loop, the first-hit reduction) is one policy, +//! written once in [`search`]. +//! +//! # ⚠ Why the two entry points are named rather than flagged +//! +//! The endianness is the whole difference and it is not cosmetic. An algebraic +//! digest reads consecutive eight-byte groups big-endian, so those four `u64`s +//! ARE the felts the host sponge absorbs; keccak reads its lanes little-endian. +//! Crossing them compiles, runs, and searches for a nonce under a message the +//! host never hashes: every returned nonce fails the host check, the prover +//! falls back to the CPU on every grind, and nothing says so louder than one +//! warning line. Two functions with two doc comments is the cheapest way to +//! make that mistake hard to type. use cudarc::driver::{LaunchConfig, PushKernelArg}; use crate::device::backend; -const BLOCK_DIM: u32 = 256; -const GRID_DIM: u32 = 1024; +/// Threads per block for the keccak arm. +/// +/// ⛔ NOT a knob, and neither is its RPX twin. The two block dims are tuned per +/// kernel against register pressure, which is a property of the kernel body; +/// the GRID is what a launch chooses, and that is [`GRID_ENV`]. +pub const BLOCK_DIM: u32 = 256; + +/// Threads per block for the RPX arm. +/// +/// Half keccak's, for the reason every RPX kernel in this crate launches +/// narrow: a thread carries a twelve-lane `u64` state plus the inverse S-box's +/// live temporaries across a non-inlined `permute` call, so occupancy is bought +/// with registers rather than threads. +pub const RPX_BLOCK_DIM: u32 = 128; /// Below this grinding factor the CPU search finds a valid nonce in well under /// a microsecond, so a device launch + shared-stream `synchronize` (which also /// stalls whatever a rayon peer queued on that stream) is pure loss. Bounce /// those to the CPU. The production factor is 20; only tests use tiny factors. -const GRIND_MIN_FACTOR: u8 = 12; - -/// Smallest nonce whose grind head is `< limit`, or `None` when the CUDA path -/// is unavailable/errors (the caller then runs the CPU search). -/// -/// `inner_lanes` are the four little-endian-read u64 lanes of the 32-byte -/// inner hash — build them with `stark::grinding::inner_hash_lanes`, which is -/// what the prover and the tests here both call. `grinding_factor` (1..=64) -/// fixes `limit = 1 << (64 - grinding_factor)` and sizes the search: the -/// expected first valid nonce is ~`2^grinding_factor`, so each launch scans a -/// contiguous block several times that, from 0 upward, and the first block that -/// hits yields the globally smallest valid nonce (the kernel `atomicMin`s it). +pub const GRIND_MIN_FACTOR: u8 = 12; + +/// Smallest per-launch block, so a tiny grinding factor still fills the grid. +const MIN_BLOCK: u64 = 1 << 18; + +/// Largest per-launch block, so a huge grinding factor does not ask for an +/// absurd single launch. A miss just advances `base` and relaunches. +const MAX_BLOCK: u64 = 1 << 28; + +/// ★ `LAMBDA_VM_GRIND_SCAN_FACTOR` — how many expected hit distances one launch +/// covers. Default [`SCAN_FACTOR_DEFAULT`]. +/// +/// # ⛔ This is a CEILING on the block, not a multiplier on the work +/// +/// Both kernels carry an early exit — `if (nonce >= *result) break;` against a +/// `volatile` result the `atomicMin` writes through L2 — and the stride walk +/// `for (i = tid; i < count; i += gridDim*blockDim)` gives every nonce in +/// `[base, base+count)` exactly one owner. So once the first valid nonce `h` is +/// recorded, every thread stops within one stride round: the permutations +/// actually executed are `h + stride`. Launches before the hitting one cover +/// exactly the part of `[0, h)` below it, so the total over the whole search is +/// `h + stride` **for any scan factor**. +/// +/// What the factor buys is only the probability that one launch suffices, +/// `P = 1 − e^−k`: 99.97% at the default 8, 63% at 1. Lowering it removes no +/// permutations — they were never executed — and adds `1/(1 − e^−k)` expected +/// launches, each a sentinel H2D, a launch, an 8-byte D2H and a stream +/// synchronize. +/// +/// ⇒ The knob exists so that reading is measurable on the card rather than +/// argued from the source. Nothing here touches the grinding factor itself +/// (the security parameter), the kernels, or the host re-validation of every +/// nonce the device returns. +pub const SCAN_FACTOR_ENV: &str = "LAMBDA_VM_GRIND_SCAN_FACTOR"; + +/// The default, and the posture every recorded measurement was taken under. +pub const SCAN_FACTOR_DEFAULT: u32 = 8; + +/// What [`SCAN_FACTOR_ENV`] accepts, inclusive, and what its error names. +/// +/// 0 is refused rather than clamped: `0 * expected` is 0, which the clamp would +/// turn into a fixed [`MIN_BLOCK`] block — a different search, not a smaller +/// one, and silently so. +pub const SCAN_FACTOR_RANGE: std::ops::RangeInclusive = 1..=64; + +/// ★ `LAMBDA_VM_GRIND_GRID` — blocks per launch. Default [`GRID_DEFAULT`]. +/// +/// # Why the GRID and not the block dims +/// +/// `stride = grid × block_dim` is the number of nonces the walk advances per +/// iteration, and it is the term the early exit above leaves behind: the search +/// executes `h + stride` permutations, so the overshoot past the first hit is +/// `stride/h` — 12.5% at the default against `h = 2^20`. +/// +/// That gives the knob two opposite edges and the sweep has to run BOTH ways: +/// - while the card is **not** filled, a wider grid raises throughput faster +/// than it raises the overshoot, and the wall falls; +/// - once the card **is** filled, extra blocks only queue, and the wider stride +/// is pure added work — narrower is then strictly better. +/// +/// Which edge the default sits on is a residency question, and residency is +/// decided by registers per thread, which is why [`BLOCK_DIM`] and +/// [`RPX_BLOCK_DIM`] are NOT knobs: they are tuned per kernel body (lane K's +/// territory), and moving them changes what an occupancy reading means. +/// [`device_fill`] reads the answer off the driver instead of estimating it. +pub const GRID_ENV: &str = "LAMBDA_VM_GRIND_GRID"; + +/// The default, and the posture every recorded measurement was taken under. +pub const GRID_DEFAULT: u32 = 1024; + +/// What [`GRID_ENV`] accepts, inclusive. +/// +/// The top is where the knob stops meaning anything rather than where CUDA +/// stops accepting it (the driver allows 2^31−1 blocks in x): at 65,536 blocks +/// the RPX stride is 8.4 M against an expected hit at 2^20, so the search would +/// be overshoot and nothing else. 0 is refused because a launch of no blocks +/// scans nothing and the retry loop would spin forever. +pub const GRID_RANGE: std::ops::RangeInclusive = 1..=65_536; + +/// ★ The two launch knobs, read together so one line can print both. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Knobs { + /// How many expected hit distances one launch covers. + pub scan: u32, + /// Blocks per launch. + pub grid: u32, +} + +impl Knobs { + /// The record posture: what an unset environment reads. + pub const DEFAULT: Self = Self { + scan: SCAN_FACTOR_DEFAULT, + grid: GRID_DEFAULT, + }; + + /// Nonces the walk advances per iteration — `grid × block_dim`, and the + /// term the early exit leaves behind as overshoot. + pub const fn stride(self, block_dim: u32) -> u64 { + self.grid as u64 * block_dim as u64 + } + + /// Nonces one launch covers: the expected hit distance `2^grinding_factor` + /// times the scan factor, clamped to `[MIN_BLOCK, MAX_BLOCK]`. + /// + /// `2^grinding_factor` can overflow u64 (factor 64), so saturate. + pub fn block(self, grinding_factor: u8) -> u64 { + let expected = 1u64.checked_shl(grinding_factor as u32).unwrap_or(u64::MAX); + expected + .saturating_mul(self.scan as u64) + .clamp(MIN_BLOCK, MAX_BLOCK) + } +} + +/// ★ The knobs for this process, read once and cached. +/// +/// Prints ONE line on every setting including the defaults, carrying both +/// knobs AND the stride each arm gets — the stride is the mechanism, so it is +/// printed as a read rather than left for the reader to multiply. A log with no +/// `★ GRIND KNOBS:` line is a run that never reached the device search. +/// +/// Aborts on a value outside its range for the reason `LAMBDA_VM_WHIR_HASH` +/// aborts on an unknown hash: a measurement taken under a silently ignored knob +/// is worse than no measurement. +pub fn knobs_in_effect() -> Knobs { + static KNOBS: std::sync::OnceLock = std::sync::OnceLock::new(); + *KNOBS.get_or_init(|| { + let knobs = Knobs { + scan: read_knob(SCAN_FACTOR_ENV, SCAN_FACTOR_DEFAULT, &SCAN_FACTOR_RANGE), + grid: read_knob(GRID_ENV, GRID_DEFAULT, &GRID_RANGE), + }; + println!( + "★ GRIND KNOBS: scan {} · grid {} · stride rpx {} / keccak {}", + knobs.scan, + knobs.grid, + knobs.stride(RPX_BLOCK_DIM), + knobs.stride(BLOCK_DIM), + ); + knobs + }) +} + +/// ★ The nonces one launch would cover at `grinding_factor` under the knobs in +/// effect — what [`search`] passes the kernel as `count`. +pub fn per_launch_block(grinding_factor: u8) -> u64 { + knobs_in_effect().block(grinding_factor) +} + +/// One knob, parsed from the environment or defaulted, refused outside its +/// range with the offending value and the range both named. +fn read_knob(env: &str, default: u32, range: &std::ops::RangeInclusive) -> u32 { + match std::env::var(env) { + Err(_) => default, + Ok(raw) => parse_knob(raw.trim(), range).unwrap_or_else(|| { + // eprintln then abort rather than a panic: a configuration error at + // startup, where the operator needs the accepted range and not a + // backtrace through the prover. + eprintln!( + "{env}={raw:?} is not a value this path accepts. Accepted: an integer in {}..={}.", + range.start(), + range.end() + ); + std::process::abort() + }), + } +} + +/// The accepted spellings: a plain decimal integer inside `range`. +fn parse_knob(raw: &str, range: &std::ops::RangeInclusive) -> Option { + let value: u32 = raw.parse().ok()?; + range.contains(&value).then_some(value) +} + +/// ★ What the driver says about filling this card with the RPX grind kernel. +/// +/// Every field is READ, not estimated: the residency question the grid knob +/// turns on is decided by registers per thread, and guessing that is how a +/// sweep gets sized against a card nobody measured. +#[derive(Clone, Copy, Debug)] +pub struct DeviceFill { + /// Multiprocessors on the device. + pub sm_count: u32, + /// The device's own ceiling on resident threads per multiprocessor. + pub max_threads_per_sm: u32, + /// Registers the RPX grind kernel uses per thread. + pub rpx_regs_per_thread: i32, + /// Blocks of [`RPX_BLOCK_DIM`] the driver will keep resident per + /// multiprocessor — the occupancy the register count actually buys. + pub rpx_blocks_per_sm: u32, + /// The block dim those blocks carry. + pub rpx_block_dim: u32, +} + +impl DeviceFill { + /// ⭐ Blocks that can be resident at once. A grid ABOVE this queues: the + /// extra blocks buy no parallelism and their stride is pure overshoot. + pub const fn resident_blocks(&self) -> u64 { + self.sm_count as u64 * self.rpx_blocks_per_sm as u64 + } + + /// Threads that can be resident at once, by the same reading. + pub const fn resident_threads(&self) -> u64 { + self.resident_blocks() * self.rpx_block_dim as u64 + } + + /// What fraction of the resident ceiling a grid of `grid` blocks asks for. + /// Above 1.0 the surplus queues. + pub fn fill(&self, grid: u32) -> f64 { + grid as f64 / self.resident_blocks() as f64 + } +} + +/// Reads [`DeviceFill`] off the driver, or `None` where there is no device. +pub fn device_fill() -> Option { + use cudarc::driver::sys::CUdevice_attribute; + let be = backend().ok()?; + let sm_count = be + .ctx + .attribute(CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) + .ok()?; + let max_threads_per_sm = be + .ctx + .attribute(CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR) + .ok()?; + let rpx_regs_per_thread = be.rpx_grind_search.num_regs().ok()?; + let rpx_blocks_per_sm = be + .rpx_grind_search + .occupancy_max_active_blocks_per_multiprocessor(RPX_BLOCK_DIM, 0, None) + .ok()?; + Some(DeviceFill { + sm_count: sm_count.max(0) as u32, + max_threads_per_sm: max_threads_per_sm.max(0) as u32, + rpx_regs_per_thread, + rpx_blocks_per_sm, + rpx_block_dim: RPX_BLOCK_DIM, + }) +} + +/// Which outer hash the search runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Arm { + Keccak256, + Rpx256, + Poseidon1, +} + +/// Smallest nonce whose keccak grind head is `< limit`, or `None` when the CUDA +/// path is unavailable/errors (the caller then runs the CPU search). +/// +/// `inner_lanes` are the four **little-endian**-read `u64` lanes of the 32-byte +/// inner hash — build them with `crypto::grinding::inner_hash_lanes`. pub fn generate_nonce_gpu(inner_lanes: &[u64; 4], grinding_factor: u8) -> Option { + search( + Arm::Keccak256, + inner_lanes, + grinding_factor, + knobs_in_effect(), + ) +} + +/// Smallest nonce whose Poseidon1 grind head is `< limit` (`p1/*` +/// exploration branch: `prover::lfm::p1_commit::P1GrindDigest`), or `None` when +/// the CUDA path is unavailable or errors. +/// +/// The host predicate hashes `inner_hash ‖ nonce.to_be_bytes()`, 40 bytes = +/// five big-endian felts, zero-padded to the eight lanes of ONE width-8 +/// permutation, and compares lane 0 with `limit`; `p1s_grind_w8` permutes +/// `[f0, f1, f2, f3, canonical(nonce), 0, 0, 0]` and compares the same lane. +/// `inner_felts` are the four big-endian `u64`s of the 32-byte inner hash +/// (`crypto::grinding::inner_hash_felts`), canonical as a digest's output. +pub fn generate_nonce_p1_gpu(inner_felts: &[u64; 4], grinding_factor: u8) -> Option { + search( + Arm::Poseidon1, + inner_felts, + grinding_factor, + knobs_in_effect(), + ) +} + +/// [`generate_nonce_gpu`] at knobs given here rather than read from the +/// environment. +/// +/// ⛔ Not a second policy — the SAME [`search`], with the one thing the +/// environment would have decided passed in. It exists because the knobs are +/// cached in a `OnceLock`, so a process cannot compare two settings through the +/// environment: a sweep would need one process per arm, and two processes are +/// two device contexts, two cubin loads and two clock domains. The arms that +/// matter (the nonce is unmoved by the block size; what a grind costs at each +/// setting) are exact only when they share a process. +pub fn generate_nonce_gpu_at( + inner_lanes: &[u64; 4], + grinding_factor: u8, + knobs: Knobs, +) -> Option { + search(Arm::Keccak256, inner_lanes, grinding_factor, knobs) +} + +/// Smallest nonce whose RPX grind head is `< limit`, or `None` when the CUDA +/// path is unavailable/errors (the caller then runs the CPU search). +/// +/// ⚠ `inner_felts` are the four **big-endian**-read `u64`s of the 32-byte inner +/// hash — build them with `crypto::grinding::inner_hash_felts`, never with +/// `inner_hash_lanes`. See the module header for what crossing them does. +/// +/// # The preimage, stated where the kernel is called +/// +/// The host predicate hashes `inner_hash ‖ nonce.to_be_bytes()` — **40 bytes, +/// which is five felts, which is one rate-8 block and therefore exactly one +/// permutation**. Its capacity is `leaf_capacity(5)`: lane 0 the padding flag +/// `5 mod 8 = 5`, lane 1 the LEAF domain. `rpx_grind_search` builds the same +/// block — `init(5)`, absorb `f0..f3` then `canonical(nonce)` — and compares +/// `digest[0] < limit`, which is the same number the host compares because +/// `u64::from_be_bytes(digest[..8])` IS felt 0's canonical value. +/// +/// The four `inner_felts` need no reduction on either side: they are an +/// algebraic digest's own output, which `digest_to_commitment` writes as four +/// canonical big-endian `u64`s. +pub fn generate_nonce_rpx_gpu(inner_felts: &[u64; 4], grinding_factor: u8) -> Option { + // The work-queue kernel on a card-filling grid (the default, + // `rpx_paths::grind_queue`). Same nonce — see `rpx_grind_search_queue`. + if crate::rpx_paths::grind_queue() { + return search_queue( + inner_felts, + grinding_factor, + knobs_in_effect(), + queue_grid()?, + false, + ) + .map(|c| c.nonce); + } + search(Arm::Rpx256, inner_felts, grinding_factor, knobs_in_effect()) +} + +/// The RPX grind through the work-queue kernel, at the scan factor of `knobs` +/// and a grid of `grid` blocks — the production arm passes [`queue_grid`]. +/// Returns the same nonce as [`generate_nonce_rpx_gpu_at`]: the smallest valid +/// nonce, a function of the inner hash and the factor alone. Exposed so a test +/// can run both arms in one process. +pub fn generate_nonce_rpx_gpu_queue_at( + inner_felts: &[u64; 4], + grinding_factor: u8, + knobs: Knobs, + grid: u32, +) -> Option { + search_queue(inner_felts, grinding_factor, knobs, grid, false).map(|c| c.nonce) +} + +/// ⛔ DIAGNOSTIC: [`generate_nonce_rpx_gpu_queue_at`] with the counted twin, so +/// a bench reads the permutations the queue EXECUTED — what the queue saves — in +/// the same slots as [`search_counted`]. +pub fn search_queue_counted( + inner_felts: &[u64; 4], + grinding_factor: u8, + knobs: Knobs, + grid: u32, +) -> Option { + search_queue(inner_felts, grinding_factor, knobs, grid, true) +} + +/// Blocks of [`RPX_BLOCK_DIM`] the queue kernel keeps resident on this card — +/// multiprocessors × the driver's occupancy for the kernel — so one launch +/// fills the card exactly (the stride kernel's grid of 1024 is 0.86 of it on a +/// 5090). Printed once. +/// +/// ⚠ The occupancy query needs the context current on the CALLING thread. +/// cudarc binds it before `num_regs` but not before this query, and +/// `backend()` binds only the thread that created it. Unbound, the query +/// failed, and while the failure was cached every RPX grind of the process +/// then fell to the host search. So this binds first, and caches only a grid +/// it read: a failed read sends that one grind to the host and the next grind +/// reads again. `tests/grind_queue_grid_thread.rs` pins it. +pub fn queue_grid() -> Option { + use cudarc::driver::sys::CUdevice_attribute; + static GRID: std::sync::OnceLock = std::sync::OnceLock::new(); + if let Some(&grid) = GRID.get() { + return Some(grid); + } + // `backend()` warns on its own failure. + let be = backend().ok()?; + let read = || -> crate::Result<(u32, u32)> { + be.ctx.bind_to_thread()?; + let sms = be + .ctx + .attribute(CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)? + .max(1) as u32; + let per_sm = be + .rpx_grind_search_queue + .occupancy_max_active_blocks_per_multiprocessor(RPX_BLOCK_DIM, 0, None)? + .max(1); + Ok((sms, per_sm)) + }; + match read() { + Ok((sms, per_sm)) => Some(*GRID.get_or_init(|| { + let regs = be.rpx_grind_search_queue.num_regs().unwrap_or(0); + eprintln!( + "[gpu] RPX grind queue: grid {} = {sms} SMs x {per_sm} blocks of {RPX_BLOCK_DIM} \ + ({regs} regs/thread), 32 nonces per warp claim", + sms * per_sm + ); + sms * per_sm + })), + Err(e) => { + // eprintln for the reason the crypto dispatch gives for its own + // fallback line: it is the only sign the queue grind left the device. + eprintln!( + "[gpu] RPX grind queue: the grid could not be read ({e}); this grind falls \ + back to the CPU search" + ); + None + } + } +} + +/// The queue arm of [`search`]: the same contiguous range walk from 0, the +/// same per-launch block, and a two-word device state — the running minimum +/// and the queue head — reset before every launch. +fn search_queue( + inner: &[u64; 4], + grinding_factor: u8, + knobs: Knobs, + grid: u32, + counted: bool, +) -> Option { + if !(GRIND_MIN_FACTOR..=64).contains(&grinding_factor) || grid == 0 { + return None; + } + let limit: u64 = 1u64 << (64 - grinding_factor); + let be = backend().ok()?; + let stream = be.next_stream(); + let inner_dev = stream.clone_htod(inner.as_slice()).ok()?; + let count = knobs.block(grinding_factor); + let cfg = LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (RPX_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + // `[result, head]`: the sentinel and an empty queue. A named binding so it + // outlives every async H2D below. + let reset = [u64::MAX, 0u64]; + let mut state_dev = stream.clone_htod(&reset).ok()?; + // The counted twin's three counters, zeroed once and accumulated into by + // every launch; the production arm has none. + let zeros = [0u64; 3]; + let mut counts_dev = if counted { + Some(stream.clone_htod(&zeros).ok()?) + } else { + None + }; + + let mut base: u64 = 0; + let mut launches: u64 = 0; + loop { + stream.memcpy_htod(&reset, &mut state_dev).ok()?; + launches += 1; + // SAFETY: the kernel reads four inner felts, and only `atomicAdd`s, + // `atomicMin`s and reads the two state words (and, counted, the three + // counters) — every buffer is allocated at those sizes above. + unsafe { + match counts_dev.as_mut() { + Some(counts) => stream + .launch_builder(&be.rpx_grind_search_queue_counted) + .arg(&inner_dev) + .arg(&limit) + .arg(&base) + .arg(&count) + .arg(&mut state_dev) + .arg(counts) + .launch(cfg) + .ok()?, + None => stream + .launch_builder(&be.rpx_grind_search_queue) + .arg(&inner_dev) + .arg(&limit) + .arg(&base) + .arg(&count) + .arg(&mut state_dev) + .launch(cfg) + .ok()?, + }; + } + let host = crate::readback::clone_dtoh(&stream, &state_dev).ok()?; + stream.synchronize().ok()?; + if host[0] != u64::MAX { + let counts = match counts_dev.as_ref() { + Some(counts) => { + let c = crate::readback::clone_dtoh(&stream, counts).ok()?; + stream.synchronize().ok()?; + c + } + None => vec![0; 3], + }; + return Some(GrindCounts { + nonce: host[0], + executed: counts[0], + max_iters: counts[1], + ran_to_end: counts[2], + launches, + poll_period: 1, + }); + } + base = base.checked_add(count)?; + } +} + +/// [`generate_nonce_rpx_gpu`] at knobs given here rather than read from the +/// environment. See [`generate_nonce_gpu_at`] for why this exists. +pub fn generate_nonce_rpx_gpu_at( + inner_felts: &[u64; 4], + grinding_factor: u8, + knobs: Knobs, +) -> Option { + search(Arm::Rpx256, inner_felts, grinding_factor, knobs) +} + +/// The range walk both arms share. +/// +/// `grinding_factor` (1..=64) fixes `limit = 1 << (64 - grinding_factor)` and +/// sizes the search: the expected first valid nonce is ~`2^grinding_factor`, so +/// each launch scans a contiguous block several times that, from 0 upward, and +/// the first block that hits yields the globally smallest valid nonce (the +/// kernels `atomicMin` it). +/// +/// ★ **The returned nonce is therefore a function of the inner hash and the +/// grinding factor alone** — the blocks are contiguous from 0 and the first one +/// to hit returns its minimum, so the block SIZE cannot move it. That is what +/// `tests/grinding.rs::gpu_grind_returns_smallest_valid_nonce` pins, and it is +/// why sweeping [`SCAN_FACTOR_ENV`] moves no proof byte. +fn search(arm: Arm, inner: &[u64; 4], grinding_factor: u8, knobs: Knobs) -> Option { if !(GRIND_MIN_FACTOR..=64).contains(&grinding_factor) { return None; } let limit: u64 = 1u64 << (64 - grinding_factor); let be = backend().ok()?; + let (kernel, block_dim) = match arm { + Arm::Keccak256 => (&be.grind_search, BLOCK_DIM), + Arm::Rpx256 => (&be.rpx_grind_search, RPX_BLOCK_DIM), + Arm::Poseidon1 => crate::p1_stark::grind_kernel().ok()?, + }; let stream = be.next_stream(); - let inner_dev = stream.clone_htod(inner_lanes.as_slice()).ok()?; + let inner_dev = stream.clone_htod(inner.as_slice()).ok()?; - // Per-launch block size: ~8× the expected hit distance, clamped so tiny - // factors still launch a full grid and huge factors don't ask for an - // absurd single block. `2^grinding_factor` can overflow u64 (factor 64), so - // saturate. - let expected = 1u64.checked_shl(grinding_factor as u32).unwrap_or(u64::MAX); - let count = expected.saturating_mul(8).clamp(1 << 18, 1 << 28); + let count = knobs.block(grinding_factor); let cfg = LaunchConfig { - grid_dim: (GRID_DIM, 1, 1), - block_dim: (BLOCK_DIM, 1, 1), + grid_dim: (knobs.grid, 1, 1), + block_dim: (block_dim, 1, 1), shared_mem_bytes: 0, }; @@ -60,7 +604,7 @@ pub fn generate_nonce_gpu(inner_lanes: &[u64; 4], grinding_factor: u8) -> Option stream.memcpy_htod(&sentinel, &mut result_dev).ok()?; unsafe { stream - .launch_builder(&be.grind_search) + .launch_builder(kernel) .arg(&inner_dev) .arg(&limit) .arg(&base) @@ -69,7 +613,7 @@ pub fn generate_nonce_gpu(inner_lanes: &[u64; 4], grinding_factor: u8) -> Option .launch(cfg) .ok()?; } - let host = stream.clone_dtoh(&result_dev).ok()?; + let host = crate::readback::clone_dtoh(&stream, &result_dev).ok()?; stream.synchronize().ok()?; if host[0] != u64::MAX { return Some(host[0]); @@ -79,3 +623,317 @@ pub fn generate_nonce_gpu(inner_lanes: &[u64; 4], grinding_factor: u8) -> Option base = base.checked_add(count)?; } } + +/// ⛔ DIAGNOSTIC: what one grind EXECUTED, beside what it returned. +/// +/// Every field is counted ON THE DEVICE by the threads that did the work, so +/// `executed - (h + stride)` is a READ rather than a model. See +/// [`search_counted`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct GrindCounts { + /// The nonce the search returned — must equal the shipped kernel's. + pub nonce: u64, + /// Permutations every thread of every launch actually ran. + pub executed: u64, + /// The most iterations any single thread ran, over every launch. + pub max_iters: u64, + /// Threads that left by the loop bound rather than by the early exit. + pub ran_to_end: u64, + /// Launches this search made: the hit's block, plus every miss before it. + pub launches: u64, + /// The poll period (`k`) this search launched the twin with, echoed back + /// from the launch so a report quotes the knob READ on the path rather than + /// the one the caller believes it passed. `1` is the shipped + /// every-iteration poll; the sweep uses powers of two. + pub poll_period: u64, +} + +/// ⛔ THE DIAGNOSTIC TWIN OF [`search`], RPX ONLY, ON NO PROVING PATH. +/// +/// Identical walk, identical exits, identical `atomicMin` — plus three device +/// counters. It exists to separate two explanations of the same slow launch +/// that no stopwatch can tell apart: MORE PERMUTATIONS (a thread that did not +/// observe the early exit and kept hashing) against THE SAME PERMUTATIONS MORE +/// SLOWLY (a cost outside this loop entirely). +/// +/// ⚠ ITS OWN ADMISSIBILITY IS THE CALLER'S JOB, and it is not optional: a +/// counted kernel with different register pressure has different occupancy and +/// therefore measures a different kernel. The bench asserts this twin +/// reproduces the shipped kernel's milliseconds per seed within the measured +/// noise floor, and reports the instrument as having changed the phenomenon if +/// it does not. +/// +/// The counters accumulate ACROSS the launches of one search — `atomicAdd` for +/// the two sums, `atomicMax` for the deepest thread — so a miss-and-relaunch +/// seed reports the whole search and not only its last block. `launches` says +/// how many blocks that was. +/// +/// `poll_period` sets how often each thread polls `*result` for the early +/// exit: every iteration at `1` (the shipped kernel's rate), every `k`-th at +/// `k`, staggered across threads so the request rate falls by `k` in both +/// average and peak. It exists to test whether the stale-poll overrun is +/// contention on that one address — lower the rate and read whether the overrun +/// falls (contention) or rises (a poll that is simply too coarse). It must be a +/// power of two; it changes only how much a thread over-scans, never the nonce +/// the search returns. +/// +/// Returns `None` for the same reasons [`search`] does, plus a factor outside +/// the supported range or a `poll_period` that is zero or not a power of two. +pub fn search_counted( + inner: &[u64; 4], + grinding_factor: u8, + knobs: Knobs, + poll_period: u64, +) -> Option { + if !(GRIND_MIN_FACTOR..=64).contains(&grinding_factor) { + return None; + } + // The kernel derives its stagger mask as `poll_period - 1` and polls when + // `(n + tid) & mask == 0`, which is a clean period of `poll_period` only + // when `poll_period` is a power of two. `poll_period == 1` (mask 0) polls + // every iteration — the shipped kernel's rate, the sweep's control; a zero + // would wrap the mask to all-ones and poll essentially never. Reject both. + if poll_period == 0 || !poll_period.is_power_of_two() { + return None; + } + let limit: u64 = 1u64 << (64 - grinding_factor); + + let be = backend().ok()?; + let stream = be.next_stream(); + let inner_dev = stream.clone_htod(inner.as_slice()).ok()?; + + let count = knobs.block(grinding_factor); + let cfg = LaunchConfig { + grid_dim: (knobs.grid, 1, 1), + block_dim: (RPX_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + + let sentinel = [u64::MAX]; + let mut result_dev = stream.clone_htod(&sentinel).ok()?; + // One slot each, zeroed once and accumulated into by every launch. + let zeros = [0u64; 3]; + let mut counts_dev = stream.clone_htod(&zeros).ok()?; + + let mut base: u64 = 0; + let mut launches: u64 = 0; + loop { + stream.memcpy_htod(&sentinel, &mut result_dev).ok()?; + launches += 1; + // SAFETY: the same contract as `search`'s launch, with three more + // device words the kernel only ever adds into or maxes against. + unsafe { + stream + .launch_builder(&be.rpx_grind_search_counted) + .arg(&inner_dev) + .arg(&limit) + .arg(&base) + .arg(&count) + .arg(&mut result_dev) + .arg(&mut counts_dev) + .arg(&poll_period) + .launch(cfg) + .ok()?; + } + let host = crate::readback::clone_dtoh(&stream, &result_dev).ok()?; + stream.synchronize().ok()?; + if host[0] != u64::MAX { + let counts = crate::readback::clone_dtoh(&stream, &counts_dev).ok()?; + stream.synchronize().ok()?; + return Some(GrindCounts { + nonce: host[0], + executed: counts[0], + max_iters: counts[1], + ran_to_end: counts[2], + launches, + poll_period, + }); + } + base = base.checked_add(count)?; + } +} + +/// Round-3 occupancy discriminator: the SHIPPED grind kernel's registers/thread +/// and blocks/SM as the current build compiled it. Reads the ACHIEVED cap (from +/// the loaded function) so a sweep over `LAMBDA_VM_RPX_MAXRREGCOUNT` builds can +/// plot the grind's throughput against its real occupancy, not the requested +/// cap. Diagnostic; touches no production path. +pub fn grind_occupancy() -> Option<(u32, u32)> { + let be = backend().ok()?; + let regs = be.rpx_grind_search.num_regs().ok()?.max(0) as u32; + let blocks_per_sm = be + .rpx_grind_search + .occupancy_max_active_blocks_per_multiprocessor(RPX_BLOCK_DIM, 0, None) + .ok()?; + Some((regs, blocks_per_sm)) +} + +#[cfg(test)] +mod tests { + //! The knobs' parser and their launch arithmetic, card-free: nothing here + //! touches `backend()`, so these run wherever the crate compiles. + //! + //! `knobs_in_effect()` itself is deliberately NOT exercised — it caches in + //! a `OnceLock`, so a test that set a variable would fix the value for + //! every other test in the binary and the second assertion would pass on + //! the first one's cache. That is also why the sweep goes through + //! `generate_nonce_rpx_gpu_at` rather than through the environment. + + use super::{ + BLOCK_DIM, GRID_DEFAULT, GRID_RANGE, Knobs, MAX_BLOCK, MIN_BLOCK, RPX_BLOCK_DIM, + SCAN_FACTOR_DEFAULT, SCAN_FACTOR_RANGE, parse_knob, + }; + + /// A knob at a named scan factor, the grid left at the record posture. + fn at_scan(scan: u32) -> Knobs { + Knobs { + scan, + grid: GRID_DEFAULT, + } + } + + /// Every arm of the scan sweep parses, and so does every arm of the grid + /// sweep — both directions, since the grid's optimum can sit on either side + /// of the default. + #[test] + fn the_parser_accepts_every_arm_of_both_sweeps() { + for k in [8u32, 4, 2, 1] { + assert_eq!( + parse_knob(&k.to_string(), &SCAN_FACTOR_RANGE), + Some(k), + "scan factor {k} is a sweep arm and must parse" + ); + } + for g in [256u32, 512, 1024, 2048, 4096] { + assert_eq!( + parse_knob(&g.to_string(), &GRID_RANGE), + Some(g), + "grid {g} is a sweep arm and must parse" + ); + } + assert_eq!(parse_knob("64", &SCAN_FACTOR_RANGE), Some(64), "scan top"); + assert_eq!(parse_knob("65536", &GRID_RANGE), Some(65_536), "grid top"); + } + + /// The refusing half, executed on both ranges. Each of these would + /// otherwise reach the launch and become a DIFFERENT search that no log + /// could distinguish from the one the operator asked for. + #[test] + fn the_parser_refuses_what_would_silently_change_the_search() { + for raw in ["0", "abc", "", "-1", "8.0", "0x8", " ", "4294967296"] { + assert_eq!( + parse_knob(raw, &SCAN_FACTOR_RANGE), + None, + "scan {raw:?} must be refused, not defaulted" + ); + assert_eq!( + parse_knob(raw, &GRID_RANGE), + None, + "grid {raw:?} must be refused, not defaulted" + ); + } + // Each range refuses just past its own top, and the two tops differ — + // so a range accidentally shared between the knobs reddens here. + assert_eq!(parse_knob("65", &SCAN_FACTOR_RANGE), None, "scan past top"); + assert_eq!(parse_knob("65537", &GRID_RANGE), None, "grid past top"); + assert_eq!( + parse_knob("1024", &SCAN_FACTOR_RANGE), + None, + "a grid value is not a scan factor" + ); + } + + /// The block size at the production grinding factor, written out as + /// arithmetic rather than as the expression under test. + /// + /// ⇒ A drifted default or a dropped multiply reddens here by value. + #[test] + fn the_block_is_the_hit_distance_times_the_scan_factor() { + assert_eq!(at_scan(8).block(20), 8 * 1_048_576, "2^20 * 8 = 2^23"); + assert_eq!(at_scan(4).block(20), 4 * 1_048_576, "2^20 * 4 = 2^22"); + assert_eq!(at_scan(2).block(20), 2 * 1_048_576, "2^20 * 2 = 2^21"); + assert_eq!(at_scan(1).block(20), 1_048_576, "2^20 * 1 = 2^20"); + } + + /// ★ The stride — the term the kernels' early exit leaves behind as + /// overshoot, and the whole mechanism of the grid knob. Written as the + /// products themselves, so a changed block dim reddens rather than being + /// absorbed by the formula under test. + #[test] + fn the_stride_is_the_grid_times_the_block_dim() { + assert_eq!(RPX_BLOCK_DIM, 128, "the RPX arm's block dim"); + assert_eq!(BLOCK_DIM, 256, "the keccak arm's block dim"); + assert_eq!( + Knobs::DEFAULT.stride(RPX_BLOCK_DIM), + 131_072, + "1024 blocks * 128 threads" + ); + assert_eq!( + Knobs::DEFAULT.stride(BLOCK_DIM), + 262_144, + "1024 blocks * 256 threads" + ); + // Both directions of the sweep, as products. + for (grid, rpx) in [ + (256u32, 32_768u64), + (512, 65_536), + (1024, 131_072), + (2048, 262_144), + (4096, 524_288), + ] { + let knobs = Knobs { scan: 8, grid }; + assert_eq!( + knobs.stride(RPX_BLOCK_DIM), + rpx, + "grid {grid} on the RPX arm" + ); + } + } + + /// The defaults are the record posture: every measurement in the campaign + /// was taken at a 2^23 block over a 1024-block grid, and only an ABBA may + /// move either. + #[test] + fn the_defaults_are_the_record_posture() { + assert_eq!(SCAN_FACTOR_DEFAULT, 8, "the record posture's scan factor"); + assert_eq!(GRID_DEFAULT, 1024, "the record posture's grid"); + assert_eq!(Knobs::DEFAULT.scan, 8, "DEFAULT carries the scan factor"); + assert_eq!(Knobs::DEFAULT.grid, 1024, "DEFAULT carries the grid"); + assert_eq!( + Knobs::DEFAULT.block(20), + 1 << 23, + "the record posture's per-launch block at grinding factor 20" + ); + } + + /// Both ends of the clamp still bind, including the u64 overflow the + /// saturating multiply exists for. The grid does not enter the clamp at + /// all — it sizes the stride, not the block — and that separation is the + /// thing this asserts. + #[test] + fn the_clamp_binds_at_both_ends_and_the_grid_does_not_touch_it() { + assert_eq!( + at_scan(1).block(12), + MIN_BLOCK, + "the min-factor gate's own factor, at the narrowest scan, floors" + ); + assert_eq!(at_scan(8).block(26), MAX_BLOCK, "2^26 * 8 = 2^29 ceils"); + assert_eq!( + at_scan(8).block(64), + MAX_BLOCK, + "2^64 saturates before the multiply, then ceils" + ); + assert!( + (MIN_BLOCK..=MAX_BLOCK).contains(&Knobs::DEFAULT.block(20)), + "the record posture sits strictly inside the clamp, so neither end \ + is silently setting it" + ); + for grid in [256u32, 1024, 4096] { + assert_eq!( + Knobs { scan: 8, grid }.block(20), + 1 << 23, + "the grid must not move the per-launch block" + ); + } + } +} diff --git a/crypto/math-cuda/src/inverse.rs b/crypto/math-cuda/src/inverse.rs index 1087e2ae4..4678a97c0 100644 --- a/crypto/math-cuda/src/inverse.rs +++ b/crypto/math-cuda/src/inverse.rs @@ -138,7 +138,7 @@ fn launch_invert_total( #[cfg(any(debug_assertions, feature = "test-faults"))] { let mut host = [0u64; 3]; - stream.memcpy_dtoh(&out.slice(0..3), &mut host)?; + crate::readback::memcpy_dtoh(stream, &out.slice(0..3), &mut host)?; stream.synchronize()?; assert_ne!( host, [0u64; 3], @@ -148,6 +148,56 @@ fn launch_invert_total( Ok(()) } +/// Whether a one-dimensional launch over `total` elements is expressible. +/// +/// The launches in this crate build their grid as `(total as u32).div_ceil( +/// BLOCK_SIZE)` blocks of `BLOCK_SIZE` threads, and every kernel behind them +/// (`logup.cu`, `inverse.cu`) computes its element index as the 64-bit +/// `blockIdx.x * (uint64_t)blockDim.x + threadIdx.x`. The only thing that +/// silently breaks is therefore the cast: past `u32::MAX` elements `total as +/// u32` wraps, too few blocks launch, and a tail of the output is never +/// written. Up to `u32::MAX` the grid is at most 16,777,216 blocks, far inside +/// `gridDim.x`'s 2^31 − 1, and the index arithmetic is exact. +/// +/// Until 2026-09 this bound was `u32::MAX / BLOCK_SIZE` — 256× too strict, as +/// if the cast applied to the block count rather than the element count. Every +/// LogUp aux build with more than 16,777,215 interaction·rows (at 2^22 rows, +/// any table with five or more interactions) and every R3/R4 batch inverse +/// past that size was refused before any launch and fell to the host, and the +/// refusal wore a `CUDA_ERROR_INVALID_VALUE` that looked like the driver's. +pub(crate) const fn launch_total_fits(total: usize) -> bool { + total <= u32::MAX as usize +} + +#[cfg(test)] +mod launch_bound_tests { + use super::*; + + /// LOCAL_TO_GLOBAL at 2^22 rows with 6 interactions — the first shape the + /// loud aux-build admission surfaced — is expressible; so is the CPU table + /// at 2^22 with its full 24, and LFM_BLAKE3 at 2^21 with its 1,261. + #[test] + fn the_shapes_that_were_refused_fit() { + assert!(launch_total_fits(6 << 22)); + assert!(launch_total_fits(24 << 22)); + assert!(launch_total_fits(1261 << 21)); + assert!(launch_total_fits(u32::MAX as usize)); + } + + /// The truncation hazard is real one past `u32::MAX`, and the old bound + /// sat 256× below it. + #[test] + fn the_bound_is_the_cast_not_the_block_count() { + assert!(!launch_total_fits(u32::MAX as usize + 1)); + assert!(launch_total_fits( + u32::MAX as usize / BLOCK_SIZE as usize + 1 + )); + let blocks = (u32::MAX).div_ceil(BLOCK_SIZE); + assert_eq!(blocks, 1 << 24); + assert!((blocks as u64) < (1u64 << 31)); + } +} + /// Device-input batch inverse. Allocates and returns a fresh `CudaSlice` /// of length `3 * n` holding the inverses. Requires `n >= 1`. /// @@ -160,12 +210,11 @@ pub fn batch_inverse_ext3_dev( stream: &Arc, ) -> Result> { assert!(n >= 1, "batch_inverse_ext3_dev requires n >= 1"); - // Runtime guard (not debug_assert): a u32 grid_dim is truncated past - // u32::MAX / BLOCK_SIZE, which would silently launch too few blocks - // and leave a tail uninverted. Reachable on LDE size 2^23+ × multi- - // eval-point R4. Returning Err lets the dispatcher's Err(_) => None - // route the caller to the CPU `inplace_batch_inverse` fallback. - if n > u32::MAX as usize / BLOCK_SIZE as usize { + // Runtime guard (not debug_assert), see [`launch_total_fits`]: past the + // bound the u32 element count the launch is built from would truncate + // and leave a tail uninverted. Returning Err lets the dispatcher route + // the caller to the CPU `inplace_batch_inverse` fallback, or abort. + if !launch_total_fits(n) { return Err(cudarc::driver::DriverError( cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE, )); @@ -235,9 +284,15 @@ pub enum DenomSign { } /// Compute `denoms[k*n + i] = sign-dependent (z, x) combination` on -/// device, then batch-invert. Returns a fresh `CudaSlice` of length +/// device and invert them. Returns a fresh `CudaSlice` of length /// `3 * k_scalars * n` holding the inverted denominators. Entire pipeline /// stays on device (no PCIe traffic beyond the small `z_scalars` upload). +/// +/// Up to [`ROWWISE_MAX_K`] denominators per row this is one row-wise kernel +/// ([`invert_denoms_rowwise_dev`]); above that, or under +/// `LAMBDA_VM_DEEP_INV_LEGACY=1` ([`crate::deep_inv`]), the six-kernel global +/// scan ([`compute_and_invert_denoms_ext3_scan_dev`]). The same layout and the +/// same field elements either way. pub fn compute_and_invert_denoms_ext3_dev( x_lde_dev: &CudaSlice, z_scalars_host: &[u64], @@ -249,20 +304,43 @@ pub fn compute_and_invert_denoms_ext3_dev( // Fault-injection hook lives here (not in the shared `batch_inverse_ext3_dev`) // so `schedule_inverse_fault(N)` targets exactly the Nth R3/R4 denominator // inversion the fallback test exercises — not the LogUp aux inverses that - // also route through `batch_inverse_ext3_dev` earlier in the prove. + // also route through `batch_inverse_ext3_dev` earlier in the prove. The + // fully resident R4 DEEP inverts inside its own kernel and never comes here, + // so on that path the Nth inversion is R3's. #[cfg(feature = "test-faults")] check_inverse_fault_injection()?; assert_eq!(z_scalars_host.len(), k_scalars * 3); assert!(n >= 1 && k_scalars >= 1); + if crate::deep_inv::rowwise_enabled() && k_scalars <= ROWWISE_MAX_K { + crate::deep_inv::count_rowwise_inversion(); + return invert_denoms_rowwise_dev(x_lde_dev, z_scalars_host, n, k_scalars, sign, stream); + } + compute_and_invert_denoms_ext3_scan_dev(x_lde_dev, z_scalars_host, n, k_scalars, sign, stream) +} + +/// [`compute_and_invert_denoms_ext3_dev`]'s legacy path: the denominators are +/// computed into a domain-sized buffer and inverted by the six-kernel global +/// scan ([`batch_inverse_ext3_dev`]). Public so the parity tests can run it +/// beside [`invert_denoms_rowwise_dev`] whatever the process setting says. +pub fn compute_and_invert_denoms_ext3_scan_dev( + x_lde_dev: &CudaSlice, + z_scalars_host: &[u64], + n: usize, + k_scalars: usize, + sign: DenomSign, + stream: &Arc, +) -> Result> { + assert_eq!(z_scalars_host.len(), k_scalars * 3); + assert!(n >= 1 && k_scalars >= 1); + let be = backend()?; let total = k_scalars .checked_mul(n) - .expect("compute_and_invert_denoms_ext3_dev: k_scalars * n overflow"); - // See `batch_inverse_ext3_dev` for the rationale: runtime Err, not - // debug_assert, so release builds also route past the silent-truncation - // hazard via the caller's CPU fallback. - if total > u32::MAX as usize / BLOCK_SIZE as usize { + .expect("compute_and_invert_denoms_ext3_scan_dev: k_scalars * n overflow"); + // See [`launch_total_fits`]: runtime Err, not debug_assert, so release + // builds also route past the truncation hazard. + if !launch_total_fits(total) { return Err(cudarc::driver::DriverError( cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE, )); @@ -299,6 +377,83 @@ pub fn compute_and_invert_denoms_ext3_dev( batch_inverse_ext3_dev(&denoms, total, stream) } +/// Most denominators per row the row-wise kernels take +/// (`invert_denoms_rowwise_ext3_k1` … `_k8`), the barycentric multi kernels' +/// point cap. Above it [`compute_and_invert_denoms_ext3_dev`] keeps the scan. +pub const ROWWISE_MAX_K: usize = 8; + +/// [`compute_and_invert_denoms_ext3_dev`]'s output — the same layout, the same +/// field elements — from ONE kernel: thread `i` builds row `i`'s `k_scalars` +/// denominators and inverts them in registers with a single base-field +/// inversion (`kernels/ext3_inv.cuh`), instead of the six-kernel global scan +/// with its prefix and suffix scratch. Stream-ordered, no host round-trip. +/// Public so the parity tests can run it beside the scan whatever the process +/// setting says. +pub fn invert_denoms_rowwise_dev( + x_lde_dev: &CudaSlice, + z_scalars_host: &[u64], + n: usize, + k_scalars: usize, + sign: DenomSign, + stream: &Arc, +) -> Result> { + assert_eq!(z_scalars_host.len(), k_scalars * 3); + assert!(n >= 1 && (1..=ROWWISE_MAX_K).contains(&k_scalars)); + assert!(x_lde_dev.len() >= n, "one coset point per row"); + let total = k_scalars + .checked_mul(n) + .expect("invert_denoms_rowwise_dev: k_scalars * n overflow"); + if !launch_total_fits(total) { + return Err(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE, + )); + } + let be = backend()?; + let z_dev = stream.clone_htod(z_scalars_host)?; + // SAFETY: the kernel writes all `k_scalars` inverses of every row. + let mut out = unsafe { stream.alloc::(3 * total) }?; + let n_u64 = n as u64; + let denom_sign_u64: u64 = match sign { + DenomSign::ZMinusX => 0, + DenomSign::XMinusZ => 1, + }; + let cfg = LaunchConfig { + grid_dim: ((n as u32).div_ceil(BLOCK_SIZE), 1, 1), + block_dim: (BLOCK_SIZE, 1, 1), + shared_mem_bytes: 0, + }; + // The scan path's invertibility guard, kept on the same builds: a zero + // denominator zeroes its row's inverses, and a valid inverse never is. + #[cfg(any(debug_assertions, feature = "test-faults"))] + let zero_flag = stream.alloc_zeros::(1)?; + #[cfg(any(debug_assertions, feature = "test-faults"))] + let flag_arg = &zero_flag; + #[cfg(not(any(debug_assertions, feature = "test-faults")))] + let flag_arg = &0u64; // nullptr: release keeps the pass free of host syncs + unsafe { + stream + .launch_builder(&be.invert_denoms_rowwise_ext3[k_scalars - 1]) + .arg(x_lde_dev) + .arg(&z_dev) + .arg(&n_u64) + .arg(&denom_sign_u64) + .arg(&mut out) + .arg(flag_arg) + .launch(cfg)?; + } + #[cfg(any(debug_assertions, feature = "test-faults"))] + { + let mut host = [0u32; 1]; + crate::readback::memcpy_dtoh(stream, &zero_flag, &mut host)?; + stream.synchronize()?; + assert_eq!( + host[0], 0, + "row-wise inverse: a zero denominator has no inverse" + ); + } + Ok(out) +} + // ============================================================================= // Multi-block recursive scan driver // ============================================================================= diff --git a/crypto/math-cuda/src/lde.rs b/crypto/math-cuda/src/lde.rs index 9bbd9958d..21394a2aa 100644 --- a/crypto/math-cuda/src/lde.rs +++ b/crypto/math-cuda/src/lde.rs @@ -10,10 +10,15 @@ //! On-device steps, picks a stream from the shared pool so rayon-parallel //! callers overlap on the GPU. Twiddles are cached in the backend. -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; -use cudarc::driver::{CudaSlice, CudaStream, LaunchConfig, PushKernelArg}; +use cudarc::driver::sys; +use cudarc::driver::{ + CudaSlice, CudaStream, CudaView, CudaViewMut, DevicePtr, DevicePtrMut, LaunchConfig, + PushKernelArg, +}; +use crate::DeviceHash; use crate::Result; use crate::device::{Backend, backend}; use crate::merkle::{keccak_launch_cfg, launch_keccak_base, launch_keccak_base_row_pair}; @@ -33,7 +38,7 @@ fn assert_u32_domain(n: usize, what: &str) { /// Output shape requested from the fused LDE + Keccak entry points. #[derive(Copy, Clone, PartialEq, Eq)] -enum KeccakCommit { +enum TreeCommit { /// Only the keccak-256 leaves; no inner-tree build. Caller receives /// `num_leaves * 32` bytes. LeavesOnly, @@ -42,18 +47,18 @@ enum KeccakCommit { FullTree, } -impl KeccakCommit { - fn total_nodes_bytes(self, num_leaves: usize) -> usize { +impl TreeCommit { + fn total_nodes_bytes(self, hash: DeviceHash, num_leaves: usize) -> usize { match self { - KeccakCommit::LeavesOnly => num_leaves * 32, - KeccakCommit::FullTree => (2 * num_leaves - 1) * 32, + TreeCommit::LeavesOnly => num_leaves * 32, + TreeCommit::FullTree => crate::tree_nodes(hash, num_leaves) * 32, } } - fn leaves_offset_bytes(self, num_leaves: usize) -> usize { + fn leaves_offset_bytes(self, hash: DeviceHash, num_leaves: usize) -> usize { match self { - KeccakCommit::LeavesOnly => 0, - KeccakCommit::FullTree => (num_leaves - 1) * 32, + TreeCommit::LeavesOnly => 0, + TreeCommit::FullTree => crate::leaves_offset(hash, num_leaves) * 32, } } } @@ -391,24 +396,30 @@ fn launch_keccak_base_row_major_row_pair_range( Ok(()) } -/// Transpose row-major `lde_size × cols` → column-major with stride `lde_size`, -/// returning the new device buffer. Used to convert the row-major LDE output to -/// the column-major layout expected by downstream GPU kernels (DEEP, barycentric). -/// No synchronize — callers on the same stream are ordered; other streams must -/// synchronize themselves. -fn launch_row_to_col_major( - stream: &Arc, +/// One `matrix_transpose_strided` launch: `dst[c * out_stride + r] = +/// src[r * cols + c]` for `r < rows`, `c < cols`. `src` holds `rows * cols` +/// contiguous row-major elements; `dst` must reach `(cols - 1) * out_stride + +/// rows`. No synchronize — callers on the same stream are ordered; other +/// streams must synchronize themselves. +fn launch_transpose_tiles( + stream: &CudaStream, be: &Backend, - src: &CudaSlice, - lde_size: usize, + src: &CudaView<'_, u64>, + dst: &mut CudaViewMut<'_, u64>, + rows: usize, cols: usize, - lde_u64: u64, -) -> Result> { - let mut dst = stream.alloc_zeros::(lde_size * cols)?; + out_stride: u64, +) -> Result<()> { + debug_assert!(rows >= 1 && cols >= 1, "empty transpose"); + debug_assert!(src.len() >= rows * cols, "transpose source too short"); + debug_assert!( + dst.len() >= (cols - 1) * out_stride as usize + rows, + "transpose destination too short" + ); let cfg = LaunchConfig { grid_dim: ( (cols as u32).div_ceil(32), - (lde_size as u32).div_ceil(32).min(65535), + (rows as u32).div_ceil(32).min(65535), 1, ), block_dim: (32, 32, 1), @@ -418,15 +429,510 @@ fn launch_row_to_col_major( stream .launch_builder(&be.matrix_transpose_strided) .arg(src) - .arg(&mut dst) - .arg(&(lde_size as u32)) + .arg(dst) + .arg(&(rows as u32)) .arg(&(cols as u32)) - .arg(&lde_u64) + .arg(&out_stride) .launch(cfg)?; } + Ok(()) +} + +/// Transpose the first `rows` row-major rows of `src` (`cols` wide) into a +/// NEW column-major buffer with column stride `out_stride` (the trace-domain +/// snapshot the LogUp fingerprint kernel reads). The LDE itself is never +/// transposed this way — see [`transpose_lde_in_place`], which keeps one +/// LDE-sized buffer live instead of two. +fn launch_row_to_col_major( + stream: &Arc, + be: &Backend, + src: &CudaSlice, + rows: usize, + cols: usize, + out_stride: u64, +) -> Result> { + let mut dst = stream.alloc_zeros::(out_stride as usize * cols)?; + launch_transpose_tiles( + stream, + be, + &src.slice(0..rows * cols), + &mut dst.slice_mut(..), + rows, + cols, + out_stride, + )?; Ok(dst) } +// ── In-place row-major → column-major transpose of the LDE ────────────────── +// +// The fused commit computes the LDE row-major (one H2D, row-major NTT and leaf +// kernels) but every downstream kernel reads it column-major. Transposing into +// a fresh buffer held TWO LDE-sized allocations live at once — the peak of the +// whole per-table commit, and the term that pushed 2^21-row tables past 32 GiB. +// Here the transpose happens inside the one allocation, in two passes over the +// same `matrix_transpose_strided` kernel plus device-to-device copies, so the +// bytes that land are exactly the ones the out-of-place kernel used to write: +// +// 1. Block pass. The `rows × cols` matrix is `blocks` row blocks of +// `rows_per_block` rows. Each block is transposed on its own into `cols` +// runs of `rows_per_block` consecutive rows of one column. Blocks ping-pong +// through one spare block of scratch: block 0 lands in the scratch, block +// `b` lands where block `b - 1` was, and the scratch finally lands in the +// last slot — so slot `s` holds block `(s + 1) mod blocks`. +// 2. Run pass. Column-major wants the runs ordered `(column, block)`; the +// block pass left them ordered `(slot, column)`. That is a permutation of +// whole runs, followed cycle by cycle in place with the scratch runs as the +// parking space, and issued as batched device copies. +// +// Scratch is one block plus one run — capped at +// `INPLACE_TRANSPOSE_SCRATCH_BYTES` — instead of a second LDE. + +/// Cap on the in-place transpose's device scratch: one transposed row block +/// (`cols` runs) plus one parking run for the cycle walk. +const INPLACE_TRANSPOSE_SCRATCH_BYTES: usize = 256 << 20; + +/// Row-block geometry of the in-place transpose: `rows == blocks * +/// rows_per_block`, both powers of two, `blocks >= 2`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct TransposeGeometry { + rows_per_block: usize, + blocks: usize, +} + +fn transpose_geometry(rows: usize, cols: usize) -> TransposeGeometry { + debug_assert!(rows >= 2 && rows.is_power_of_two(), "rows: {rows}"); + debug_assert!(cols >= 1, "cols: {cols}"); + // Longest power-of-two run whose scratch (`cols + 1` runs) fits the cap. + let run_cap = (INPLACE_TRANSPOSE_SCRATCH_BYTES / ((cols + 1) * 8)).max(1); + let run_cap = 1usize << run_cap.ilog2(); + // At least eight blocks whenever the matrix has the rows for it, so the + // small shapes the parity suites drive take the same multi-block + // permutation as production instead of a trivial two-block one. + let rows_per_block = run_cap.min((rows / 8).max(1)); + TransposeGeometry { + rows_per_block, + blocks: rows / rows_per_block, + } +} + +/// A run — `rows_per_block` consecutive rows of one column — by run index, +/// either inside the LDE buffer or inside the scratch. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Run { + Buf(usize), + Scratch(usize), +} + +/// Sink for the run copies the permutation issues. The pairs inside one +/// `batch` call are independent — destinations are distinct and no destination +/// aliases a source of the same batch — so a sink may execute them in any +/// order or concurrently. Successive batches are ordered. +trait RunCopier { + fn batch(&mut self, copies: &[(Run, Run)]) -> Result<()>; +} + +/// After the block pass, run position `p = slot * cols + c` holds column `c` +/// of block `(slot + 1) mod blocks`; column-major needs that run at +/// `c * blocks + block`. This is the inverse map: the position whose run must +/// end up at `q`. +fn run_source_of(q: usize, blocks: usize, cols: usize) -> usize { + let (c, block) = (q / blocks, q % blocks); + ((block + blocks - 1) % blocks) * cols + c +} + +/// Move every run to its column-major position, in place, cycle by cycle. +/// A cycle `[p0, p1, ..]` (run `p_{i+1}` moves to `p_i`) is served in +/// segments of at most `cols` moves: one batch parks the segment's sources in +/// scratch runs `0..m`, the next lands them; run `p0` waits in scratch run +/// `cols` until the cycle closes. About `2 * blocks + 3 * cycles` batches. +fn permute_runs_to_col_major( + blocks: usize, + cols: usize, + copier: &mut impl RunCopier, +) -> Result<()> { + let total = blocks * cols; + let parked = Run::Scratch(cols); + let mut visited = vec![false; total]; + let mut cycle: Vec = Vec::new(); + let mut copies: Vec<(Run, Run)> = Vec::with_capacity(cols + 1); + for p0 in 0..total { + if visited[p0] { + continue; + } + cycle.clear(); + let mut p = p0; + loop { + visited[p] = true; + cycle.push(p); + p = run_source_of(p, blocks, cols); + if p == p0 { + break; + } + } + let k = cycle.len(); + if k == 1 { + continue; + } + copier.batch(&[(parked, Run::Buf(p0))])?; + let mut i = 0; + while i < k - 1 { + let m = cols.min(k - 1 - i); + copies.clear(); + copies.extend((0..m).map(|j| (Run::Scratch(j), Run::Buf(cycle[i + 1 + j])))); + copier.batch(&copies)?; + copies.clear(); + copies.extend((0..m).map(|j| (Run::Buf(cycle[i + j]), Run::Scratch(j)))); + if i + m == k - 1 { + copies.push((Run::Buf(cycle[k - 1]), parked)); + } + copier.batch(&copies)?; + i += m; + } + } + Ok(()) +} + +/// `LAMBDA_VM_LDE_TRANSPOSE_UNBATCHED=1` issues the run pass as one +/// device-to-device copy per run instead of `cuMemcpyBatchAsync` batches. +/// Same bytes either way; a measurement and diagnostic knob only. +fn transpose_copies_batched() -> bool { + static BATCHED: OnceLock = OnceLock::new(); + *BATCHED.get_or_init(|| std::env::var_os("LAMBDA_VM_LDE_TRANSPOSE_UNBATCHED").is_none()) +} + +/// Issues run copies on the device: a batch is one `cuMemcpyBatchAsync` +/// (source access in stream order, both sides device memory), or one +/// `cuMemcpyDtoDAsync` per run when unbatched or for a single copy. The +/// caller keeps the context bound to this thread and both pointers valid +/// while the copier lives. +struct DeviceRunCopier<'a> { + stream: &'a CudaStream, + device: sys::CUdevice, + buf: sys::CUdeviceptr, + scratch: sys::CUdeviceptr, + run_bytes: usize, + batched: bool, + dsts: Vec, + srcs: Vec, + sizes: Vec, +} + +impl DeviceRunCopier<'_> { + fn addr(&self, run: Run) -> sys::CUdeviceptr { + match run { + Run::Buf(p) => self.buf + (p * self.run_bytes) as sys::CUdeviceptr, + Run::Scratch(j) => self.scratch + (j * self.run_bytes) as sys::CUdeviceptr, + } + } +} + +impl RunCopier for DeviceRunCopier<'_> { + fn batch(&mut self, copies: &[(Run, Run)]) -> Result<()> { + if copies.is_empty() { + return Ok(()); + } + if !self.batched || copies.len() == 1 { + for &(dst, src) in copies { + // SAFETY: both addresses lie inside allocations the caller + // keeps alive (`buf`, `scratch`), each run is `run_bytes` + // long, and the copy is queued on the caller's stream. + unsafe { + sys::cuMemcpyDtoDAsync_v2( + self.addr(dst), + self.addr(src), + self.run_bytes, + self.stream.cu_stream(), + ) + .result()?; + } + } + return Ok(()); + } + self.dsts.clear(); + self.srcs.clear(); + self.sizes.clear(); + for &(dst, src) in copies { + self.dsts.push(self.addr(dst)); + self.srcs.push(self.addr(src)); + self.sizes.push(self.run_bytes); + } + let location = sys::CUmemLocation { + type_: sys::CUmemLocationType_enum::CU_MEM_LOCATION_TYPE_DEVICE, + id: self.device, + }; + let mut attrs = sys::CUmemcpyAttributes { + srcAccessOrder: sys::CUmemcpySrcAccessOrder_enum::CU_MEMCPY_SRC_ACCESS_ORDER_STREAM, + srcLocHint: location, + dstLocHint: location, + flags: 0, + }; + let mut attrs_idxs = [0usize]; + let mut fail_idx = 0usize; + // SAFETY: the three arrays are `copies.len()` long and outlive the + // call; one attribute set covers the whole batch (`attrsIdxs[0] == + // 0`); every address is inside `buf` or `scratch`, which the caller + // keeps alive; the batch is queued on the caller's stream, and the + // pairs are independent (the `RunCopier` contract). + unsafe { + sys::cuMemcpyBatchAsync( + self.dsts.as_mut_ptr(), + self.srcs.as_mut_ptr(), + self.sizes.as_mut_ptr(), + copies.len(), + &mut attrs, + attrs_idxs.as_mut_ptr(), + 1, + &mut fail_idx, + self.stream.cu_stream(), + ) + .result()?; + } + Ok(()) + } +} + +/// Transpose the row-major `rows × cols` LDE in `buf` to column-major +/// (column `c` at `c * rows`) IN PLACE and hand the same allocation back. +/// Byte for byte the result of the out-of-place kernel — the two passes only +/// move runs of values — with one block plus one run of scratch instead of a +/// second LDE. Everything is queued on `stream`; nothing synchronizes. +fn transpose_lde_in_place( + stream: &Arc, + be: &Backend, + mut buf: CudaSlice, + rows: usize, + cols: usize, +) -> Result> { + assert_eq!(buf.len(), rows * cols, "in-place transpose shape"); + if rows < 2 || cols < 2 { + // A single row or a single column reads the same in both layouts. + return Ok(buf); + } + let TransposeGeometry { + rows_per_block, + blocks, + } = transpose_geometry(rows, cols); + let block_elems = rows_per_block * cols; + // `alloc`, not `alloc_zeros`: every scratch element is written before it is + // read — the block pass fills the spare block, and each parking batch + // fills the runs the next batch drains. + let mut scratch = unsafe { stream.alloc::((cols + 1) * rows_per_block) }?; + + // Block pass: block 0 → scratch, block b → slot b - 1, scratch → last slot. + { + let src = buf.slice(0..block_elems); + let mut spare = scratch.slice_mut(0..block_elems); + launch_transpose_tiles( + stream, + be, + &src, + &mut spare, + rows_per_block, + cols, + rows_per_block as u64, + )?; + } + for block in 1..blocks { + let (mut lo, hi) = buf.split_at_mut(block * block_elems); + let src = hi.slice(0..block_elems); + let mut dst = lo.slice_mut((block - 1) * block_elems..block * block_elems); + launch_transpose_tiles( + stream, + be, + &src, + &mut dst, + rows_per_block, + cols, + rows_per_block as u64, + )?; + } + stream.memcpy_dtod( + &scratch.slice(0..block_elems), + &mut buf.slice_mut((blocks - 1) * block_elems..blocks * block_elems), + )?; + + // Run pass: raw driver copies, so the context must be current here. + be.ctx.bind_to_thread()?; + { + let (buf_ptr, _buf_record) = buf.device_ptr_mut(stream); + let (scratch_ptr, _scratch_record) = scratch.device_ptr_mut(stream); + let mut copier = DeviceRunCopier { + stream, + device: be.ctx.cu_device(), + buf: buf_ptr, + scratch: scratch_ptr, + run_bytes: rows_per_block * 8, + batched: transpose_copies_batched(), + dsts: Vec::with_capacity(cols + 1), + srcs: Vec::with_capacity(cols + 1), + sizes: Vec::with_capacity(cols + 1), + }; + permute_runs_to_col_major(blocks, cols, &mut copier)?; + } + // `scratch` drops here: freed stream-ordered behind the copies that read it. + Ok(buf) +} + +#[cfg(test)] +mod inplace_transpose_tests { + use super::*; + + #[test] + fn geometry_is_power_of_two_blocks_under_the_scratch_cap() { + for log_rows in 1..=27u32 { + let rows = 1usize << log_rows; + for cols in [1usize, 2, 3, 9, 37, 316, 436, 612, 2048, 65535] { + let g = transpose_geometry(rows, cols); + assert!(g.rows_per_block.is_power_of_two()); + assert!(g.blocks.is_power_of_two()); + assert_eq!(g.rows_per_block * g.blocks, rows, "{rows}x{cols}"); + assert!(g.blocks >= 2, "{rows}x{cols}: {g:?}"); + assert!( + g.rows_per_block == 1 + || (cols + 1) * g.rows_per_block * 8 <= INPLACE_TRANSPOSE_SCRATCH_BYTES, + "{rows}x{cols}: {g:?} busts the scratch cap" + ); + if rows >= 8 && (cols + 1) * (rows / 8) * 8 <= INPLACE_TRANSPOSE_SCRATCH_BYTES { + assert_eq!(g.blocks, 8, "{rows}x{cols}: {g:?}"); + } + } + } + // The production shapes the brief sizes: LDE 2^22 × 316 and 2^22 × 436. + assert_eq!( + transpose_geometry(1 << 22, 316), + TransposeGeometry { + rows_per_block: 1 << 16, + blocks: 64 + } + ); + assert_eq!( + transpose_geometry(1 << 22, 436), + TransposeGeometry { + rows_per_block: 1 << 16, + blocks: 64 + } + ); + } + + /// Host model of the buffer after the block pass: one label `(block, col)` + /// per run. Checks the independence contract on every batch. + struct ModelCopier { + buf: Vec>, + scratch: Vec>, + batches: usize, + copies: usize, + } + + impl ModelCopier { + fn get(&self, run: Run) -> Option<(usize, usize)> { + match run { + Run::Buf(p) => self.buf[p], + Run::Scratch(j) => self.scratch[j], + } + } + fn set(&mut self, run: Run, v: Option<(usize, usize)>) { + match run { + Run::Buf(p) => self.buf[p] = v, + Run::Scratch(j) => self.scratch[j] = v, + } + } + } + + impl RunCopier for ModelCopier { + fn batch(&mut self, copies: &[(Run, Run)]) -> Result<()> { + for (i, (dst, src)) in copies.iter().enumerate() { + assert!( + copies[..i].iter().all(|(d, _)| d != dst), + "duplicate destination {dst:?} in a batch" + ); + assert!( + copies.iter().all(|(_, s)| s != dst), + "destination {dst:?} aliases a source of the same batch" + ); + assert!(self.get(*src).is_some(), "copy from unwritten run {src:?}"); + } + // Independent pairs: read everything, then write everything. + let values: Vec<_> = copies.iter().map(|(_, src)| self.get(*src)).collect(); + for ((dst, _), v) in copies.iter().zip(values) { + self.set(*dst, v); + } + self.batches += 1; + self.copies += copies.len(); + Ok(()) + } + } + + #[test] + fn run_pass_lands_every_run_at_its_column_major_position() { + for (blocks, cols) in [ + (2usize, 2usize), + (2, 3), + (4, 1), + (4, 3), + (8, 2), + (8, 7), + (8, 9), + (8, 316), + (8, 436), + (64, 9), + (64, 316), + (64, 436), + (128, 316), + (128, 612), + (16, 2048), + ] { + let mut model = ModelCopier { + buf: (0..blocks * cols) + .map(|p| Some(((p / cols + 1) % blocks, p % cols))) + .collect(), + scratch: vec![None; cols + 1], + batches: 0, + copies: 0, + }; + permute_runs_to_col_major(blocks, cols, &mut model).unwrap(); + for q in 0..blocks * cols { + assert_eq!( + model.buf[q], + Some((q % blocks, q / blocks)), + "{blocks}x{cols}: run {q}" + ); + } + // Cycle census of the same permutation, independently walked. + let total = blocks * cols; + let mut seen = vec![false; total]; + let (mut cycles, mut fixed) = (0usize, 0usize); + for p0 in 0..total { + if seen[p0] { + continue; + } + let (mut p, mut len) = (p0, 0usize); + loop { + seen[p] = true; + len += 1; + p = run_source_of(p, blocks, cols); + if p == p0 { + break; + } + } + if len == 1 { + fixed += 1; + } else { + cycles += 1; + } + } + // Every moving run is parked once and landed once — no more. + assert_eq!(model.copies, 2 * (total - fixed), "{blocks}x{cols}"); + // One parking batch per cycle, then two batches per `cols` moves. + assert!( + model.batches <= 2 * blocks + 3 * cycles, + "{blocks}x{cols}: {} batches for {cycles} cycles", + model.batches + ); + } + } +} + /// Row-major LDE input: either a host slice (uploaded) or an already-resident /// device buffer (copied device-to-device, no PCIe upload). enum InnerInput<'a> { @@ -467,7 +973,11 @@ fn expand_row_major_on_stream( match input { InnerInput::Host(h) if h.len() >= PINNED_H2D_MIN_U64 => { let mut dst = buf.slice_mut(0..n * total_cols); - crate::device::htod_via(stream, be.pinned_staging(), &be.ctx, h, &mut dst)?; + if crate::device::staging_pairs_enabled() { + crate::device::htod_staged(stream, h, &mut dst)?; + } else { + crate::device::htod_via(stream, be.pinned_staging(), &be.ctx, h, &mut dst)?; + } } InnerInput::Host(h) => stream.memcpy_htod(h, &mut buf.slice_mut(0..n * total_cols))?, InnerInput::Dev(d) => stream.memcpy_dtod(d, &mut buf.slice_mut(0..n * total_cols))?, @@ -520,7 +1030,216 @@ fn expand_row_major_on_stream( Ok((buf, trace_col_major)) } -/// Shared row-major LDE + Keccak + Merkle pipeline for the base and ext3 paths. +/// The column-major form of [`expand_row_major_on_stream`] (see [`crate::lde_cm`]): +/// the same LDE, produced COLUMN-major (column `c` at `c · lde_size`) — the +/// layout every downstream kernel reads — so there is no row-major LDE to +/// transpose afterwards, and no zero fill. +/// +/// The trace still arrives row-major. A host trace is staged at the TAIL of +/// the LDE buffer. The column-major trace-domain source the engine reads is +/// then either the snapshot, when the caller retains one (the engine only +/// reads its source), or the buffer's HEAD (column `c` at `c · n`), which the +/// engine walks in descending chunks so no chunk overwrites a column it has +/// not read yet (`ChunkOrder::Descending`). Head and tail are disjoint because +/// the blowup is at least 2 (`lde_cm::supports`). +#[allow(clippy::too_many_arguments)] +fn expand_col_major_on_stream( + stream: &Arc, + be: &Backend, + input: InnerInput, + n: usize, + total_cols: usize, + blowup_factor: usize, + weights: &[u64], + retain_trace_col_major: bool, +) -> Result<(CudaSlice, Option>)> { + use crate::lde_cm::{ChunkOrder, Cols, Input, lde_columns}; + debug_assert!(crate::lde_cm::supports(n, blowup_factor)); + let lde_size = n * blowup_factor; + let total = lde_size * total_cols; + let trace = n * total_cols; + let tail = total - trace; + // SAFETY: every element is written before it is read — the staging and the + // head source by the upload and the transpose, the LDE columns by each + // column's forward transform, which stores all `lde_size` rows. + let mut buf = unsafe { stream.alloc::(total) }?; + let buf_ptr = buf.device_ptr(stream).0; + const PINNED_H2D_MIN_U64: usize = 1 << 20; + if let InnerInput::Host(h) = input { + let mut staged = buf.slice_mut(tail..total); + if h.len() >= PINNED_H2D_MIN_U64 && crate::device::staging_pairs_enabled() { + crate::device::htod_staged(stream, h, &mut staged)?; + } else if h.len() >= PINNED_H2D_MIN_U64 { + crate::device::htod_via(stream, be.pinned_staging(), &be.ctx, h, &mut staged)?; + } else { + stream.memcpy_htod(h, &mut staged)?; + } + } + + let (snapshot, src, order) = { + let (mut head, staged) = buf.split_at_mut(tail); + let row_major = match input { + InnerInput::Host(_) => staged.slice(0..trace), + InnerInput::Dev(d) => d.slice(0..trace), + }; + if retain_trace_col_major { + // SAFETY: the transpose writes all `n × total_cols` elements. + let mut snap = unsafe { stream.alloc::(trace) }?; + launch_transpose_tiles( + stream, + be, + &row_major, + &mut snap.slice_mut(..), + n, + total_cols, + n as u64, + )?; + let ptr = snap.device_ptr(stream).0; + let src = Cols { + ptr, + stride: n as u64, + }; + (Some(snap), src, ChunkOrder::Ascending) + } else { + launch_transpose_tiles( + stream, + be, + &row_major, + &mut head.slice_mut(0..trace), + n, + total_cols, + n as u64, + )?; + let src = Cols { + ptr: buf_ptr, + stride: n as u64, + }; + (None, src, ChunkOrder::Descending) + } + }; + lde_columns( + stream, + be, + src, + buf_ptr, + total_cols, + n, + blowup_factor, + weights, + Input::Evals, + order, + )?; + Ok((buf, snapshot)) +} + +/// Leaves of the column range `[col_start, col_end)` of a column-major LDE +/// (`lde_size` rows per column) into the leaf slab of `nodes_dev`. +#[allow(clippy::too_many_arguments)] +fn col_major_leaves_into_tree( + hash: DeviceHash, + stream: &Arc, + be: &Backend, + buf: &CudaSlice, + lde_size: usize, + col_start: usize, + col_end: usize, + rows_per_leaf: usize, + nodes_dev: &mut CudaSlice, + leaves_offset: usize, +) -> Result<()> { + let cols = buf.device_ptr(stream).0 + (col_start * lde_size * 8) as u64; + let leaves = nodes_dev.device_ptr(stream).0 + leaves_offset as u64; + crate::lde_cm::launch_col_major_leaves( + hash, + stream, + be, + cols, + lde_size as u64, + (col_end - col_start) as u64, + lde_size as u64, + rows_per_leaf, + leaves, + ) +} + +/// The column-major form of [`coset_lde_row_major_inner`]: the same tree and the +/// same column-major handle, with the LDE built column-major +/// ([`expand_col_major_on_stream`]) and its leaves hashed from that layout by +/// the column-major twins of the row-major leaf kernels. Returns +/// `(tree, column-major LDE, optional snapshot, ready)`; the host LDE copy is +/// never produced here (the callers route `retain_host_lde` to the legacy +/// path, whose host copy is row-major). +#[allow(clippy::type_complexity)] +#[allow(clippy::too_many_arguments)] +fn coset_lde_row_major_inner_col_major( + input: InnerInput, + hash: DeviceHash, + n: usize, + total_cols: usize, + blowup_factor: usize, + weights: &[u64], + retain_trace_col_major: bool, + rows_per_leaf: usize, +) -> Result<( + GpuMerkleTree, + CudaSlice, + Option>, + Arc, +)> { + let lde_size = n * blowup_factor; + let num_leaves = lde_size / rows_per_leaf; + let nodes_bytes = TreeCommit::FullTree.total_nodes_bytes(hash, num_leaves); + let leaves_offset = TreeCommit::FullTree.leaves_offset_bytes(hash, num_leaves); + + let be = backend()?; + let stream = be.next_stream(); + let (buf, trace_col_major) = expand_col_major_on_stream( + &stream, + be, + input, + n, + total_cols, + blowup_factor, + weights, + retain_trace_col_major, + )?; + + // SAFETY: the leaf kernel fills the leaf slab, the inner levels the rest. + let mut nodes_dev = unsafe { stream.alloc::(nodes_bytes) }?; + col_major_leaves_into_tree( + hash, + &stream, + be, + &buf, + lde_size, + 0, + total_cols, + rows_per_leaf, + &mut nodes_dev, + leaves_offset, + )?; + build_inner_tree_levels_for(hash, stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + let mut root = [0u8; 32]; + crate::readback::memcpy_dtoh(&stream, &nodes_dev.slice(0..32), &mut root)?; + let ready = be.take_event()?; + ready.event().record(&stream)?; + let tree = GpuMerkleTree { + nodes: Arc::new(nodes_dev), + leaves_len: num_leaves, + arity: hash.arity(), + root, + }; + Ok((tree, buf, trace_col_major, Arc::new(ready))) +} + +/// Whether a row-major commit takes the column-major form: the engine is on, the shape +/// fits the engine, and the caller wants no host copy of the row-major LDE. +fn row_major_commit_takes_engine(n: usize, blowup_factor: usize, retain_host_lde: bool) -> bool { + !retain_host_lde && crate::lde_cm::engine_enabled() && crate::lde_cm::supports(n, blowup_factor) +} + +/// Shared row-major LDE + leaf-hash + Merkle pipeline for the base and ext3 +/// paths, committing with the kernel family `hash` selects. /// /// `total_cols` is the number of base-field columns in the row-major layout: /// `m` for base, `m * 3` for ext3. Because `Fp3 = [u64; 3]`, the three ext3 @@ -533,10 +1252,229 @@ fn expand_row_major_on_stream( /// `retain_trace_col_major`). The buffer is transposed to column-major (as /// required by the downstream GPU kernels DEEP/barycentric); callers wrap it in /// the appropriate LDE handle. +/// Walk the inner Merkle levels with the kernel family `hash` selects. +fn build_inner_tree_levels_for( + hash: DeviceHash, + stream: &CudaStream, + be: &crate::device::Backend, + nodes_dev: &mut CudaSlice, + leaves_len: usize, +) -> Result<()> { + match hash { + DeviceHash::Keccak256 => crate::merkle::build_inner_tree_levels( + stream, + be, + nodes_dev, + leaves_len, + DeviceHash::Keccak256, + ), + DeviceHash::Blake3 => { + crate::blake3::build_inner_tree_levels(stream, be, nodes_dev, leaves_len) + } + DeviceHash::Rpx256 => { + crate::rpx::build_inner_tree_levels(stream, be, nodes_dev, leaves_len) + } + DeviceHash::Poseidon1 => { + crate::p1_stark::build_inner_tree_levels(stream, nodes_dev, leaves_len) + } + } +} + +/// Hash the leaves of a row-major commit over `buf` (`num_rows` rows of stride +/// `m`), columns `[col_start, col_end)`, `rows_per_leaf` rows per leaf, into +/// `leaves_out` (`num_rows / rows_per_leaf` leaves), with the kernel family +/// `hash` selects: +/// +/// - `rows_per_leaf = 2` (today): leaf `i` = rows `reverse_index(2i)`, +/// `reverse_index(2i + 1)` — the row-pair kernels, the full-row one when the +/// range is the whole row (so the default launches exactly what it did). +/// - `rows_per_leaf = 1` (S2): leaf `i` = the row `reverse_index(i)` — the +/// one-row kernels (`*_leaves_base_row_major_row_range`). The CPU twin is +/// `commit_rows_bit_reversed_subset_with(.., rows_per_leaf)`. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_row_major_leaves( + hash: DeviceHash, + stream: &CudaStream, + be: &Backend, + buf: &CudaSlice, + m: u64, + col_start: u64, + col_end: u64, + num_rows: u64, + rows_per_leaf: usize, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); + // Every kernel derives rows as `__brevll(..) >> (64 - log_num_rows)`, UB at + // `log_num_rows == 0`. + assert!(num_rows >= 2 && num_rows.is_power_of_two()); + assert!( + col_start < col_end && col_end <= m, + "column range in bounds" + ); + let log_num_rows = num_rows.trailing_zeros() as u64; + if hash == DeviceHash::Poseidon1 { + return crate::p1_stark::launch_leaves_row_major( + stream, + buf, + m, + col_start, + col_end, + num_rows, + rows_per_leaf, + leaves_out, + ); + } + let full = col_start == 0 && col_end == m; + if rows_per_leaf == 2 { + return match (hash, full) { + (DeviceHash::Keccak256, true) => launch_keccak_base_row_major_row_pair( + stream, + be, + buf, + m, + num_rows, + log_num_rows, + leaves_out, + ), + (DeviceHash::Keccak256, false) => launch_keccak_base_row_major_row_pair_range( + stream, + be, + buf, + m, + col_start, + col_end, + num_rows, + log_num_rows, + leaves_out, + ), + (DeviceHash::Blake3, true) => crate::blake3::launch_leaves_base_row_major_row_pair( + stream, + be, + buf, + m, + num_rows, + log_num_rows, + leaves_out, + ), + (DeviceHash::Blake3, false) => { + crate::blake3::launch_leaves_base_row_major_row_pair_range( + stream, + be, + buf, + m, + col_start, + col_end, + num_rows, + log_num_rows, + leaves_out, + ) + } + (DeviceHash::Rpx256, true) => crate::rpx::launch_leaves_base_row_major_row_pair( + stream, + be, + buf, + m, + num_rows, + log_num_rows, + leaves_out, + ), + (DeviceHash::Rpx256, false) => crate::rpx::launch_leaves_base_row_major_row_pair_range( + stream, + be, + buf, + m, + col_start, + col_end, + num_rows, + log_num_rows, + leaves_out, + ), + // Launched above (`p1_stark::launch_leaves_row_major`). + (DeviceHash::Poseidon1, _) => Err(crate::invalid_value()), + }; + } + // One row per leaf: one thread per row. + let (kernel, cfg) = match hash { + DeviceHash::Keccak256 => ( + &be.keccak256_leaves_base_row_major_row_range, + keccak_launch_cfg(num_rows), + ), + DeviceHash::Blake3 => ( + &be.blake3_leaves_base_row_major_row_range, + crate::blake3::blake3_launch_cfg(num_rows), + ), + DeviceHash::Rpx256 => ( + &be.rpx_leaves_base_row_major_row_range, + crate::rpx::rpx_launch_cfg(num_rows), + ), + // Launched above (`p1_stark::launch_leaves_row_major`). + DeviceHash::Poseidon1 => return Err(crate::invalid_value()), + }; + unsafe { + stream + .launch_builder(kernel) + .arg(buf) + .arg(&m) + .arg(&col_start) + .arg(&col_end) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// Row-major leaf hashing of a HOST row-major matrix under `hash` with +/// `rows_per_leaf` rows per leaf, columns `[col_start, col_end)`: the leaf +/// hashes alone (`num_rows / rows_per_leaf` × 32 bytes). A parity harness for +/// [`launch_row_major_leaves`] against the CPU leaf spec; nothing on a proving +/// path calls it. +pub fn row_major_leaves( + hash: DeviceHash, + data: &[u64], + m: usize, + col_start: usize, + col_end: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + assert!(num_rows.is_power_of_two() && num_rows >= 2); + assert!(rows_per_leaf == 1 || rows_per_leaf == 2); + let total = num_rows + .checked_mul(m) + .expect("num_rows * m overflows usize"); + assert!(data.len() >= total); + let be = backend()?; + let stream = be.next_stream(); + let data_dev = stream.clone_htod(&data[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / rows_per_leaf) * 32)?; + launch_row_major_leaves( + hash, + stream.as_ref(), + be, + &data_dev, + m as u64, + col_start as u64, + col_end as u64, + num_rows as u64, + rows_per_leaf, + &mut out_dev.as_view_mut(), + )?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + #[allow(clippy::type_complexity)] #[allow(clippy::too_many_arguments)] fn coset_lde_row_major_inner( input: InnerInput, + hash: DeviceHash, n: usize, total_cols: usize, blowup_factor: usize, @@ -544,6 +1482,7 @@ fn coset_lde_row_major_inner( what: &str, retain_trace_col_major: bool, retain_host_lde: bool, + rows_per_leaf: usize, ) -> Result<( GpuMerkleTree, CudaSlice, @@ -562,13 +1501,30 @@ fn coset_lde_row_major_inner( let lde_size = n * blowup_factor; assert_u32_domain(lde_size, what); - // Row-pair trace commit: one Merkle leaf per bit-reversed row pair (rows 2i, - // 2i+1), matching the CPU `commit_bit_reversed(.., ROWS_PER_LEAF=2)` and the - // verifier's `verify_opening_pair`. `lde_size` is a power of two >= 2, so it - // is always even. - let num_leaves = lde_size / 2; - let nodes_bytes = KeccakCommit::FullTree.total_nodes_bytes(num_leaves); - let log_lde = lde_size.trailing_zeros() as u64; + // Trace commit with `rows_per_leaf` bit-reversed rows per Merkle leaf: row + // pairs (rows 2i, 2i+1) today, matching the CPU `commit_bit_reversed(.., + // ROWS_PER_LEAF=2)` and the verifier's `verify_opening_pair`; one row (S2) + // under `rows_per_leaf = 1`. `lde_size` is a power of two >= 2, so it is + // always a multiple of either. + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); + if row_major_commit_takes_engine(n, blowup_factor, retain_host_lde) { + let (tree, col_major_dev, trace_col_major, ready) = coset_lde_row_major_inner_col_major( + input, + hash, + n, + total_cols, + blowup_factor, + weights, + retain_trace_col_major, + rows_per_leaf, + )?; + return Ok((tree, col_major_dev, Vec::new(), trace_col_major, ready)); + } + let num_leaves = lde_size / rows_per_leaf; + let nodes_bytes = TreeCommit::FullTree.total_nodes_bytes(hash, num_leaves); let lde_u64 = lde_size as u64; let cols_u64 = total_cols as u64; @@ -586,24 +1542,28 @@ fn coset_lde_row_major_inner( retain_trace_col_major, )?; - // Keccak + Merkle on-device. Each row-pair leaf reads two bit-reversed rows - // of `total_cols` consecutive u64s (`lde_u64` is the bit-reverse modulus; the - // kernel emits `lde_size / 2` leaves). + // Leaf hashing + Merkle on-device, with the kernel family `hash` selects. + // Each leaf reads `rows_per_leaf` bit-reversed rows of `total_cols` + // consecutive u64s (`lde_u64` is the bit-reverse modulus; the kernel emits + // `lde_size / rows_per_leaf` leaves). let mut nodes_dev = unsafe { stream.alloc::(nodes_bytes) }?; - let leaves_offset = KeccakCommit::FullTree.leaves_offset_bytes(num_leaves); + let leaves_offset = TreeCommit::FullTree.leaves_offset_bytes(hash, num_leaves); { let mut leaves_view = nodes_dev.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); - launch_keccak_base_row_major_row_pair( + launch_row_major_leaves( + hash, stream.as_ref(), be, &buf, cols_u64, + 0, + cols_u64, lde_u64, - log_lde, + rows_per_leaf, &mut leaves_view, )?; } - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels_for(hash, stream.as_ref(), be, &mut nodes_dev, num_leaves)?; // Copy the 32-byte root BEFORE queueing the big drain/transpose: this // pageable copy host-blocks until everything queued so far lands, so @@ -612,44 +1572,69 @@ fn coset_lde_row_major_inner( // The Merkle tree stays resident on device; query openings gather paths // from it (see merkle::gather_merkle_paths_dev). let mut root = [0u8; 32]; - stream.memcpy_dtoh(&nodes_dev.slice(0..32), &mut root)?; + crate::readback::memcpy_dtoh(&stream, &nodes_dev.slice(0..32), &mut root)?; + // Transpose row-major buf into column-major for the handle, in place. + // Downstream kernels (DEEP, barycentric) expect buf[c * lde_size + r]. + // No host synchronize after it: the handle carries a `ready` event instead, + // and consumers on other streams wait on it device-side + // (`wait_ready_on`). On the device-only path this makes the whole + // commit's tail (transpose) run behind the host's next work. + let transpose = |buf: CudaSlice| -> Result<(CudaSlice, crate::device::PooledEvent)> { + let col_major_dev = transpose_lde_in_place(&stream, be, buf, lde_size, total_cols)?; + let ready = be.take_event()?; + ready.event().record(&stream)?; + Ok((col_major_dev, ready)) + }; // D2H the row-major LDE (skipped when `retain_host_lde` is false — the // full-residency path keeps the LDE device-only; that skip is the big - // transfer/alloc win, and we return an empty host Vec). - let lde_pending = if retain_host_lde { - Some(crate::device::async_dtoh_via( + // transfer/alloc win, and we return an empty host Vec). The transpose is + // queued behind the D2H either way, so the host copy sees the row-major + // bytes. + let (col_major_dev, ready, lde_out) = if retain_host_lde + && crate::device::staging_pairs_enabled() + { + // A staging pair of this call's own: the host copies one chunk out + // while the next lands, and the transpose is queued the moment the last + // chunk's read is. + let src = { + let (ptr, _record) = buf.device_ptr(&stream); + ptr + }; + let mut out = Vec::new(); + // SAFETY: `src` is `buf`, whose writers (the LDE and leaf kernels) are + // queued on `stream` above; its next writer is the transpose, which the + // closure queues on `stream` behind the last read, and `buf` lives on as + // the transposed handle. + let (col_major_dev, ready) = unsafe { + crate::device::dtoh_staged_into(&stream, src, lde_size * total_cols, &mut out, || { + transpose(buf) + }) + }?; + (col_major_dev, ready, out) + } else if retain_host_lde { + let started = std::time::Instant::now(); + let pending = crate::device::async_dtoh_via( &stream, be.pinned_staging(), &be.ctx, &buf, lde_size * total_cols, - )?) + )?; + let (col_major_dev, ready) = transpose(buf)?; + let mut out = vec![0u64; lde_size * total_cols]; + pending.wait_into_u64(&mut out)?; + crate::device::note_shared_slab_download(out.len() * 8, started.elapsed()); + (col_major_dev, ready, out) } else { - None - }; - - // Transpose row-major buf into column-major for the handle. Downstream - // kernels (DEEP, barycentric) expect buf[c * lde_size + r] (column-major). - let col_major_dev = launch_row_to_col_major(&stream, be, &buf, lde_size, total_cols, lde_u64)?; - // No host synchronize here: the handle carries a `ready` event instead, - // and consumers on other streams wait on it device-side - // (`wait_ready_on`). On the device-only path this makes the whole - // commit's tail (transpose) run behind the host's next work. - let ready = be.take_event()?; - ready.event().record(&stream)?; - let lde_out = match lde_pending { - Some(p) => { - let mut out = vec![0u64; lde_size * total_cols]; - p.wait_into_u64(&mut out)?; - out - } - None => Vec::new(), + let (col_major_dev, ready) = transpose(buf)?; + (col_major_dev, ready, Vec::new()) }; let tree = GpuMerkleTree { nodes: Arc::new(nodes_dev), leaves_len: num_leaves, + arity: hash.arity(), root, }; Ok(( @@ -661,7 +1646,47 @@ fn coset_lde_row_major_inner( )) } -/// Row-major LDE + Keccak + Merkle, all on-device, keeping the Merkle tree +/// The row-major coset LDE **alone** — no leaf hashing, no Merkle tree. +/// +/// ★ This is exactly what a Round-1 RESIDENCY RELEASE would re-run. At the +/// Round-1 barrier a table's tree is already built and its root already +/// absorbed, so dropping the LDE buffer and rebuilding it later costs this and +/// nothing else. The entry exists so that cost can be PRICED before anyone +/// builds the release: lane P5's arithmetic puts the break-even at 24 ms per +/// rebuild for an all-tables policy and 423 ms for the heaviest table only, and +/// those are decided by a measurement, not by a model. +/// +/// ⛔ IT SYNCHRONISES, and that is the whole point. The launches below are +/// asynchronous, so a caller that timed this without the sync would time a +/// queue submission — microseconds — and report that the rebuild is free. A +/// probe that cannot fail is worse than no probe. +/// +/// Nothing on the proving path calls this today. +pub fn coset_lde_row_major_no_tree( + row_major: &[u64], + n: usize, + m: usize, + blowup_factor: usize, + weights: &[u64], +) -> Result> { + let be = backend()?; + let stream = be.next_stream(); + let (buf, _) = expand_row_major_on_stream( + &stream, + be, + InnerInput::Host(row_major), + n, + m, + blowup_factor, + weights, + false, + )?; + stream.synchronize()?; + Ok(buf) +} + +/// Row-major LDE + leaf hashing + Merkle, all on-device, keeping the Merkle +/// tree /// resident on device (in the handle's `tree`). The host tree is not built, so /// the whole tree copy to host is eliminated; query openings gather paths from /// the device tree. @@ -671,14 +1696,44 @@ fn coset_lde_row_major_inner( /// critical path), the expansion D2D-copies from it instead of a fresh H2D. /// Returns the `GpuLdeBase` handle (column-major buf, plus the device tree) /// and the row-major LDE Vec. +#[allow(clippy::too_many_arguments)] pub fn coset_lde_row_major_with_merkle_tree_keep( row_major: &[u64], predev: Option<&CudaSlice>, + hash: DeviceHash, + n: usize, + m: usize, + blowup_factor: usize, + weights: &[u64], + retain_host_lde: bool, +) -> Result<(GpuLdeBase, Vec)> { + coset_lde_row_major_with_merkle_tree_keep_rpl( + row_major, + predev, + hash, + n, + m, + blowup_factor, + weights, + retain_host_lde, + 2, + ) +} + +/// [`coset_lde_row_major_with_merkle_tree_keep`] with `rows_per_leaf` rows per +/// Merkle leaf: 2 is today's row pair, 1 the S2 one-row tree (twice the +/// leaves, `(2·lde − 1)·32` node bytes instead of `(lde − 1)·32`). +#[allow(clippy::too_many_arguments)] +pub fn coset_lde_row_major_with_merkle_tree_keep_rpl( + row_major: &[u64], + predev: Option<&CudaSlice>, + hash: DeviceHash, n: usize, m: usize, blowup_factor: usize, weights: &[u64], retain_host_lde: bool, + rows_per_leaf: usize, ) -> Result<(GpuLdeBase, Vec)> { let input = match predev { Some(d) if d.len() == row_major.len() => InnerInput::Dev(d), @@ -686,6 +1741,7 @@ pub fn coset_lde_row_major_with_merkle_tree_keep( }; let (tree, col_major_dev, lde_out, trace_col_major, ready) = coset_lde_row_major_inner( input, + hash, n, m, blowup_factor, @@ -693,6 +1749,7 @@ pub fn coset_lde_row_major_with_merkle_tree_keep( "coset_lde_row_major lde_size", true, retain_host_lde, + rows_per_leaf, )?; let handle = GpuLdeBase { buf: Arc::new(col_major_dev), @@ -706,17 +1763,103 @@ pub fn coset_lde_row_major_with_merkle_tree_keep( Ok((handle, lde_out)) } +/// A precomputed tree's nodes on their way to the host: still bytes (the +/// pageable download, converted to nodes at the end of the commit, where the +/// conversion has always run) or already nodes (the staged download). +enum PrecomputedHost { + Bytes { + bytes: Vec, + took: std::time::Duration, + }, + Nodes(Vec<[u8; 32]>), +} + +impl PrecomputedHost { + fn into_nodes(self) -> Vec<[u8; 32]> { + match self { + PrecomputedHost::Nodes(nodes) => nodes, + PrecomputedHost::Bytes { bytes, took } => { + let started = std::time::Instant::now(); + let nodes: Vec<[u8; 32]> = bytes + .chunks_exact(32) + .map(|c| { + let mut node = [0u8; 32]; + node.copy_from_slice(c); + node + }) + .collect(); + crate::device::note_tree_download(false, bytes.len(), took + started.elapsed()); + nodes + } + } + } +} + +/// Bring a built precomputed tree (`nodes_dev`, a full node buffer) to the +/// host, then run `then` — the multiplicity tree's build — on the same stream. +/// [`crate::device::tree_download_staged`] picks the transfer: +/// +/// - pageable (the `legacy` opt-out): one pageable copy into a zeroed byte +/// vector, the buffer freed, then `then`; +/// - staged (the default): through a staging pair straight into the node vector, the buffer +/// freed and `then` queued as soon as the last chunk's DMA is, so the +/// multiplicity tree hashes while the host copies the last chunks. Both +/// frees are stream-ordered behind the reads, so one tree is the device peak +/// either way (as `stark::device_set::CommitDeviceSet::tree_bytes` counts). +/// +/// The timer starts once the tree is built (an event waited on first), so the +/// counters time the transfer, not the hashing before it. +fn download_precomputed_then( + stream: &Arc, + be: &Backend, + nodes_dev: CudaSlice, + then: impl FnOnce() -> Result, +) -> Result<(PrecomputedHost, X)> { + let n_bytes = nodes_dev.len(); + let built = be.take_event()?; + built.event().record(stream)?; + built.event().synchronize()?; + let started = std::time::Instant::now(); + if !crate::device::tree_download_staged() { + let mut bytes = vec![0u8; n_bytes]; + crate::readback::memcpy_dtoh(stream, &nodes_dev, &mut bytes)?; + drop(nodes_dev); + let took = started.elapsed(); + let x = then()?; + return Ok((PrecomputedHost::Bytes { bytes, took }, x)); + } + assert_eq!(n_bytes % 32, 0, "a node buffer holds whole nodes"); + let src = { + let (ptr, _record) = nodes_dev.device_ptr(stream); + ptr + }; + let mut nodes: Vec<[u8; 32]> = Vec::new(); + // SAFETY: `src` is `nodes_dev`, `n_bytes / 32` nodes, whose writers (the + // tree build) are queued on `stream` before this call; nothing writes it + // again, and the closure frees it stream-ordered on `stream`, behind the + // last chunk's read. Every 32-byte pattern is a valid node. + let x = unsafe { + crate::device::dtoh_staged_uncounted(stream, src, n_bytes / 32, &mut nodes, move || { + drop(nodes_dev); + then() + }) + }?; + crate::device::note_tree_download(true, n_bytes, started.elapsed()); + Ok((PrecomputedHost::Nodes(nodes), x)) +} + /// Row-major LDE + TWO subset Merkle trees for preprocessed tables: the /// precomputed columns `[0, split_col)` and the multiplicity columns /// `[split_col, m)` commit to separate trees over the same row-major LDE, /// mirroring the CPU `commit_rows_bit_reversed_subset` pair. /// /// The precomputed tree's complete node buffer is downloaded to host -/// (`(2*num_leaves - 1) * 32` bytes, inner nodes first, root at offset 0, -/// leaves at the tail — the exact `MerkleTree::from_precomputed_nodes` -/// layout) because it feeds the process-wide host tree cache; it is only -/// built when `build_precomputed` is true (the caller skips it on a cache -/// hit). The multiplicity tree stays resident in `handle.tree` — openings +/// (`2*num_leaves - 1` nodes, inner nodes first, root at index 0, leaves at +/// the tail — the exact `MerkleTree::from_precomputed_nodes` layout) because +/// it feeds the process-wide host tree cache; it is only built when +/// `build_precomputed` is true (the caller skips it on a cache hit). How it +/// travels is [`crate::device::TREE_DOWNLOAD_ENV`]'s choice; the nodes are the +/// same bytes either way. The multiplicity tree stays resident in `handle.tree` — openings /// gather its paths on device. /// /// Returns `(precomputed_nodes, handle, row_major_lde)`. The handle also @@ -726,6 +1869,39 @@ pub fn coset_lde_row_major_with_merkle_tree_keep( pub fn coset_lde_row_major_split_trees( row_major: &[u64], predev: Option<&CudaSlice>, + hash: DeviceHash, + n: usize, + m: usize, + blowup_factor: usize, + weights: &[u64], + split_col: usize, + build_precomputed: bool, + retain_host_lde: bool, +) -> Result<(Option>, GpuLdeBase, Vec)> { + coset_lde_row_major_split_trees_rpl( + row_major, + predev, + hash, + n, + m, + blowup_factor, + weights, + split_col, + build_precomputed, + retain_host_lde, + 2, + ) +} + +/// [`coset_lde_row_major_split_trees`] with `rows_per_leaf` rows per Merkle +/// leaf in BOTH subset trees (a table has one leaf layout): 2 = row pair, +/// 1 = S2 one row. +#[allow(clippy::type_complexity)] +#[allow(clippy::too_many_arguments)] +pub fn coset_lde_row_major_split_trees_rpl( + row_major: &[u64], + predev: Option<&CudaSlice>, + hash: DeviceHash, n: usize, m: usize, blowup_factor: usize, @@ -733,7 +1909,8 @@ pub fn coset_lde_row_major_split_trees( split_col: usize, build_precomputed: bool, retain_host_lde: bool, -) -> Result<(Option>, GpuLdeBase, Vec)> { + rows_per_leaf: usize, +) -> Result<(Option>, GpuLdeBase, Vec)> { assert!(split_col > 0 && split_col < m, "split inside the row"); assert!(n.is_power_of_two(), "n must be a power of two"); assert_eq!(weights.len(), n, "weights length must match n"); @@ -744,10 +1921,13 @@ pub fn coset_lde_row_major_split_trees( assert_eq!(row_major.len(), n * m, "row-major input shape"); let lde_size = n * blowup_factor; assert_u32_domain(lde_size, "coset_lde_row_major_split lde_size"); - let num_leaves = lde_size / 2; - let nodes_bytes = KeccakCommit::FullTree.total_nodes_bytes(num_leaves); - let leaves_offset = KeccakCommit::FullTree.leaves_offset_bytes(num_leaves); - let log_lde = lde_size.trailing_zeros() as u64; + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); + let num_leaves = lde_size / rows_per_leaf; + let nodes_bytes = TreeCommit::FullTree.total_nodes_bytes(hash, num_leaves); + let leaves_offset = TreeCommit::FullTree.leaves_offset_bytes(hash, num_leaves); let lde_u64 = lde_size as u64; let cols_u64 = m as u64; @@ -758,80 +1938,201 @@ pub fn coset_lde_row_major_split_trees( Some(d) if d.len() == row_major.len() => InnerInput::Dev(d), _ => InnerInput::Host(row_major), }; + if row_major_commit_takes_engine(n, blowup_factor, retain_host_lde) { + return coset_lde_row_major_split_trees_col_major( + &stream, + be, + input, + hash, + n, + m, + blowup_factor, + weights, + split_col, + build_precomputed, + rows_per_leaf, + ); + } let (buf, trace_col_major) = expand_row_major_on_stream(&stream, be, input, n, m, blowup_factor, weights, true)?; - // One subset tree per column range, built sequentially on the stream. - let build_subset_tree_dev = |col_start: u64, col_end: u64| -> Result> { + // One subset tree per column range, built sequentially on the stream. + let build_subset_tree_dev = |col_start: u64, col_end: u64| -> Result> { + let mut nodes_dev = unsafe { stream.alloc::(nodes_bytes) }?; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); + launch_row_major_leaves( + hash, + stream.as_ref(), + be, + &buf, + cols_u64, + col_start, + col_end, + lde_u64, + rows_per_leaf, + &mut leaves_view, + )?; + } + build_inner_tree_levels_for(hash, stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + Ok(nodes_dev) + }; + + // Precomputed subset tree: full nodes to host (feeds the process-wide + // host tree cache keyed by root; built once per prove on cache miss). + // Multiplicity subset tree: resident (per-epoch; the ~2x-leaves node + // download and host rebuild it used to pay are dropped — R4 openings + // gather paths on device). Built behind the precomputed tree's download. + let build_mult = || build_subset_tree_dev(split_col as u64, cols_u64); + let (precomputed_host, mult_nodes_dev) = if build_precomputed { + let nodes_dev = build_subset_tree_dev(0, split_col as u64)?; + let (host, mult) = download_precomputed_then(&stream, be, nodes_dev, build_mult)?; + (Some(host), mult) + } else { + (None, build_mult()?) + }; + let mult_tree = { + let nodes_dev = mult_nodes_dev; + let mut root = [0u8; 32]; + crate::readback::memcpy_dtoh(&stream, &nodes_dev.slice(0..32), &mut root)?; + GpuMerkleTree { + nodes: Arc::new(nodes_dev), + leaves_len: num_leaves, + arity: hash.arity(), + root, + } + }; + + // Column-major handle for downstream GPU rounds (DEEP, barycentric, + // constraint composition): transposed in place, behind the D2H below. + let transpose = |buf: CudaSlice| -> Result<(CudaSlice, crate::device::PooledEvent)> { + let col_major_dev = transpose_lde_in_place(&stream, be, buf, lde_size, m)?; + let ready = be.take_event()?; + ready.event().record(&stream)?; + Ok((col_major_dev, ready)) + }; + // D2H the row-major LDE only when the caller keeps a host copy; under + // device-only every downstream consumer reads the handle. + let (col_major_dev, ready, lde_out) = if retain_host_lde + && crate::device::staging_pairs_enabled() + { + // A staging pair of this call's own, as in `coset_lde_row_major_inner`. + let src = { + let (ptr, _record) = buf.device_ptr(&stream); + ptr + }; + let mut out = Vec::new(); + // SAFETY: `src` is `buf`, whose writers are queued on `stream` above; + // its next writer is the transpose, which the closure queues on + // `stream` behind the last read, and `buf` lives on as the handle. + let (col_major_dev, ready) = unsafe { + crate::device::dtoh_staged_into(&stream, src, lde_size * m, &mut out, || transpose(buf)) + }?; + (col_major_dev, ready, out) + } else if retain_host_lde { + let started = std::time::Instant::now(); + let pending = crate::device::async_dtoh_via( + &stream, + be.pinned_staging(), + &be.ctx, + &buf, + lde_size * m, + )?; + let (col_major_dev, ready) = transpose(buf)?; + let mut out = vec![0u64; lde_size * m]; + pending.wait_into_u64(&mut out)?; + crate::device::note_shared_slab_download(out.len() * 8, started.elapsed()); + (col_major_dev, ready, out) + } else { + let (col_major_dev, ready) = transpose(buf)?; + (col_major_dev, ready, Vec::new()) + }; + + let handle = GpuLdeBase { + buf: Arc::new(col_major_dev), + m, + lde_size, + tree: Some(mult_tree), + ready: Some(Arc::new(ready)), + trace_dev: trace_col_major.map(Arc::new), + trace_rows: n, + }; + let precomputed_nodes = precomputed_host.map(PrecomputedHost::into_nodes); + Ok((precomputed_nodes, handle, lde_out)) +} + +/// The column-major form of [`coset_lde_row_major_split_trees_rpl`]: the LDE built +/// column-major ([`expand_col_major_on_stream`], snapshot retained), each +/// subset tree hashed from its contiguous run of columns. Same trees, same +/// handle, no transpose; the host LDE copy is never produced here. +#[allow(clippy::type_complexity)] +#[allow(clippy::too_many_arguments)] +fn coset_lde_row_major_split_trees_col_major( + stream: &Arc, + be: &Backend, + input: InnerInput, + hash: DeviceHash, + n: usize, + m: usize, + blowup_factor: usize, + weights: &[u64], + split_col: usize, + build_precomputed: bool, + rows_per_leaf: usize, +) -> Result<(Option>, GpuLdeBase, Vec)> { + let lde_size = n * blowup_factor; + let num_leaves = lde_size / rows_per_leaf; + let nodes_bytes = TreeCommit::FullTree.total_nodes_bytes(hash, num_leaves); + let leaves_offset = TreeCommit::FullTree.leaves_offset_bytes(hash, num_leaves); + let (buf, trace_col_major) = + expand_col_major_on_stream(stream, be, input, n, m, blowup_factor, weights, true)?; + + let build_subset_tree_dev = |col_start: usize, col_end: usize| -> Result> { + // SAFETY: the leaf kernel fills the leaf slab, the inner levels the rest. let mut nodes_dev = unsafe { stream.alloc::(nodes_bytes) }?; - { - let mut leaves_view = - nodes_dev.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); - launch_keccak_base_row_major_row_pair_range( - stream.as_ref(), - be, - &buf, - cols_u64, - col_start, - col_end, - lde_u64, - log_lde, - &mut leaves_view, - )?; - } - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + col_major_leaves_into_tree( + hash, + stream, + be, + &buf, + lde_size, + col_start, + col_end, + rows_per_leaf, + &mut nodes_dev, + leaves_offset, + )?; + build_inner_tree_levels_for(hash, stream.as_ref(), be, &mut nodes_dev, num_leaves)?; Ok(nodes_dev) }; - // Precomputed subset tree: full nodes to host (feeds the process-wide - // host tree cache keyed by root; built once per prove on cache miss). - let precomputed_nodes = if build_precomputed { - let nodes_dev = build_subset_tree_dev(0, split_col as u64)?; - let mut nodes_host = vec![0u8; nodes_bytes]; - stream.memcpy_dtoh(&nodes_dev, &mut nodes_host)?; - Some(nodes_host) + // As the legacy path: the precomputed tree's nodes go to host (the + // process-wide tree cache), the multiplicity tree stays resident. + let build_mult = || build_subset_tree_dev(split_col, m); + let (precomputed_host, mult_nodes_dev) = if build_precomputed { + let nodes_dev = build_subset_tree_dev(0, split_col)?; + let (host, mult) = download_precomputed_then(stream, be, nodes_dev, build_mult)?; + (Some(host), mult) } else { - None + (None, build_mult()?) }; - // Multiplicity subset tree: resident (per-epoch; the ~2x-leaves node - // download and host rebuild it used to pay are dropped — R4 openings - // gather paths on device). let mult_tree = { - let nodes_dev = build_subset_tree_dev(split_col as u64, cols_u64)?; + let nodes_dev = mult_nodes_dev; let mut root = [0u8; 32]; - stream.memcpy_dtoh(&nodes_dev.slice(0..32), &mut root)?; + crate::readback::memcpy_dtoh(stream, &nodes_dev.slice(0..32), &mut root)?; GpuMerkleTree { nodes: Arc::new(nodes_dev), leaves_len: num_leaves, + arity: hash.arity(), root, } }; - - // D2H the row-major LDE only when the caller keeps a host copy; under - // device-only every downstream consumer reads the handle. - let lde_pending = retain_host_lde - .then(|| { - crate::device::async_dtoh_via(&stream, be.pinned_staging(), &be.ctx, &buf, lde_size * m) - }) - .transpose()?; - - // Column-major handle for downstream GPU rounds (DEEP, barycentric, - // constraint composition). - let col_major_dev = launch_row_to_col_major(&stream, be, &buf, lde_size, m, lde_u64)?; let ready = be.take_event()?; - ready.event().record(&stream)?; - - let lde_out = match lde_pending { - Some(pending) => { - let mut out = vec![0u64; lde_size * m]; - pending.wait_into_u64(&mut out)?; - out - } - None => Vec::new(), - }; + ready.event().record(stream)?; let handle = GpuLdeBase { - buf: Arc::new(col_major_dev), + buf: Arc::new(buf), m, lde_size, tree: Some(mult_tree), @@ -839,10 +2140,11 @@ pub fn coset_lde_row_major_split_trees( trace_dev: trace_col_major.map(Arc::new), trace_rows: n, }; - Ok((precomputed_nodes, handle, lde_out)) + let precomputed_nodes = precomputed_host.map(PrecomputedHost::into_nodes); + Ok((precomputed_nodes, handle, Vec::new())) } -/// Row-major ext3 LDE + Keccak + Merkle, all on-device. +/// Row-major ext3 LDE + leaf hashing + Merkle, all on-device. /// /// `Fp3` is `[u64; 3]` in memory, so row-major ext3 with `m` ext3 columns is /// identical to row-major base-field with `m3 = m * 3`. The same row-major NTT @@ -854,14 +2156,41 @@ pub fn coset_lde_row_major_split_trees( /// Returns (merkle_nodes, GpuLdeExt3 handle, row-major ext3 LDE Vec). pub fn coset_lde_ext3_row_major_with_merkle_tree_keep( row_major: &[u64], + hash: DeviceHash, + n: usize, + m: usize, + blowup_factor: usize, + weights: &[u64], + retain_host_lde: bool, +) -> Result<(GpuLdeExt3, Vec)> { + coset_lde_ext3_row_major_with_merkle_tree_keep_rpl( + row_major, + hash, + n, + m, + blowup_factor, + weights, + retain_host_lde, + 2, + ) +} + +/// [`coset_lde_ext3_row_major_with_merkle_tree_keep`] with `rows_per_leaf` rows per Merkle leaf (2 = row pair, 1 = S2 +/// one row). +#[allow(clippy::too_many_arguments)] +pub fn coset_lde_ext3_row_major_with_merkle_tree_keep_rpl( + row_major: &[u64], + hash: DeviceHash, n: usize, m: usize, blowup_factor: usize, weights: &[u64], retain_host_lde: bool, + rows_per_leaf: usize, ) -> Result<(GpuLdeExt3, Vec)> { let (tree, col_major_dev, lde_out, _, ready) = coset_lde_row_major_inner( InnerInput::Host(row_major), + hash, n, m * 3, blowup_factor, @@ -869,6 +2198,7 @@ pub fn coset_lde_ext3_row_major_with_merkle_tree_keep( "coset_lde_ext3_row_major lde_size", false, retain_host_lde, + rows_per_leaf, )?; let handle = GpuLdeExt3 { buf: Arc::new(col_major_dev), @@ -886,14 +2216,41 @@ pub fn coset_lde_ext3_row_major_with_merkle_tree_keep( /// scratch. Used by the resident LogUp aux path. pub fn coset_lde_ext3_row_major_with_merkle_tree_keep_dev( input_dev: &CudaSlice, + hash: DeviceHash, + n: usize, + m: usize, + blowup_factor: usize, + weights: &[u64], + retain_host_lde: bool, +) -> Result<(GpuLdeExt3, Vec)> { + coset_lde_ext3_row_major_with_merkle_tree_keep_dev_rpl( + input_dev, + hash, + n, + m, + blowup_factor, + weights, + retain_host_lde, + 2, + ) +} + +/// [`coset_lde_ext3_row_major_with_merkle_tree_keep_dev`] with `rows_per_leaf` rows per Merkle leaf (2 = row pair, 1 = S2 +/// one row). +#[allow(clippy::too_many_arguments)] +pub fn coset_lde_ext3_row_major_with_merkle_tree_keep_dev_rpl( + input_dev: &CudaSlice, + hash: DeviceHash, n: usize, m: usize, blowup_factor: usize, weights: &[u64], retain_host_lde: bool, + rows_per_leaf: usize, ) -> Result<(GpuLdeExt3, Vec)> { let (tree, col_major_dev, lde_out, _, ready) = coset_lde_row_major_inner( InnerInput::Dev(input_dev), + hash, n, m * 3, blowup_factor, @@ -901,6 +2258,7 @@ pub fn coset_lde_ext3_row_major_with_merkle_tree_keep_dev( "coset_lde_ext3_row_major_dev lde_size", false, retain_host_lde, + rows_per_leaf, )?; let handle = GpuLdeExt3 { buf: Arc::new(col_major_dev), @@ -977,16 +2335,31 @@ impl GpuLdeExt3 { /// paths on device instead of copying the whole tree to host. Node layout /// matches the CPU tree (`crypto/crypto/src/merkle_tree`): `nodes[0..leaves_len-1]` /// are inner nodes (root at 0), `nodes[leaves_len-1..]` are the leaves, each 32 -/// bytes. Freed when the `nodes` Arc drops. +/// bytes. At [`Self::arity`] 4 it is the CPU tree's arity-4 layout instead +/// ([`crate::p1_stark`]). Freed when the `nodes` Arc drops. #[derive(Clone)] pub struct GpuMerkleTree { pub nodes: Arc>, pub leaves_len: usize, + /// Children per node: 2, or 4 under [`DeviceHash::Poseidon1`]. + pub arity: usize, /// The Merkle root (node 0), copied to host at build time so the commitment /// is available without copying the whole tree. pub root: [u8; 32], } +impl GpuMerkleTree { + /// Stored nodes: the binary heap's `2·leaves − 1`, or the arity-4 layout's + /// count. + pub fn total_nodes(&self) -> usize { + if self.arity == crate::p1_stark::ARITY { + crate::p1_stark::tree_nodes(self.leaves_len) + } else { + 2 * self.leaves_len - 1 + } + } +} + pub fn coset_lde_base(evals: &[u64], blowup_factor: usize, weights: &[u64]) -> Result> { let n = evals.len(); // Empty input must short-circuit before the power-of-two assert @@ -1058,7 +2431,7 @@ pub fn coset_lde_base(evals: &[u64], blowup_factor: usize, weights: &[u64]) -> R } run_ntt_body(stream.as_ref(), &mut buf, fwd_tw.as_ref(), lde_u64, log_lde)?; - let out = stream.clone_dtoh(&buf)?; + let out = crate::readback::clone_dtoh(&stream, &buf)?; stream.synchronize()?; Ok(out) } @@ -1095,8 +2468,6 @@ pub fn coset_lde_batch_base( } let lde_size = n * blowup_factor; assert_u32_domain(lde_size, "coset_lde_batch_base lde_size"); - let log_n = n.trailing_zeros() as u64; - let log_lde = lde_size.trailing_zeros() as u64; let be = backend()?; let stream = be.next_stream(); @@ -1118,9 +2489,16 @@ pub fn coset_lde_batch_base( pinned[c * n..c * n + n].copy_from_slice(col); } - // Column layout: `buf[c * lde_size + r]`. Zeroed so the [n, lde_size) - // tail of each column is already the zero-pad the CPU path does. - let mut buf = stream.alloc_zeros::(m * lde_size)?; + // Column layout: `buf[c * lde_size + r]`. Zeroed for the legacy + // transform, so the [n, lde_size) tail of each column is already the + // zero-pad the CPU path does; the column-major engine reads only the first n. + let engine = slabs_take_engine(n, blowup_factor); + let mut buf = if engine { + // SAFETY: the upload writes each slab's first n, the LDE the rest. + unsafe { stream.alloc::(m * lde_size) }? + } else { + stream.alloc_zeros::(m * lde_size)? + }; // Any `?` between the first upload below and `sync_event` would release // the slot with async H2D reads of the slab still in flight; this guard // (declared after `staging`, so it drops first) drains the stream on @@ -1140,68 +2518,16 @@ pub fn coset_lde_batch_base( // point. It is waited just before the D2H drain re-acquires the slot. staging.record_event(&stream)?; - let inv_tw = be.inv_twiddles_for(log_n)?; - let fwd_tw = be.fwd_twiddles_for(log_lde)?; - let weights_dev = stream.clone_htod(weights)?; - - let n_u64 = n as u64; - let lde_u64 = lde_size as u64; - let col_stride_u64 = lde_size as u64; - let m_u32 = m as u32; - - // === 1. Bit-reverse first N of every column === - launch_bit_reverse_batched( - stream.as_ref(), - be, - &mut buf, - n_u64, - log_n, - col_stride_u64, - m_u32, - )?; - - // === 2. iNTT body over all columns === - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - inv_tw.as_ref(), - n_u64, - log_n, - col_stride_u64, - m_u32, - )?; - - // === 3. Pointwise multiply by coset weights (includes 1/N) === - launch_pointwise_mul_batched( - stream.as_ref(), - be, - &mut buf, - &weights_dev, - n_u64, - col_stride_u64, - m_u32, - )?; - - // === 4. Bit-reverse full LDE of every column === - launch_bit_reverse_batched( - stream.as_ref(), + lde_in_slabs( + &stream, be, &mut buf, - lde_u64, - log_lde, - col_stride_u64, - m_u32, - )?; - - // === 5. Forward NTT on full LDE of every column === - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - fwd_tw.as_ref(), - lde_u64, - log_lde, - col_stride_u64, - m_u32, + m, + n, + blowup_factor, + weights, + SlabInput::Evals, + engine, )?; // Release the staging slot before the drain: the uploads have landed once @@ -1280,8 +2606,6 @@ pub fn coset_lde_batch_base_into( assert_eq!(o.len(), lde_size, "each output must be lde_size"); } assert_u32_domain(lde_size, "coset_lde_batch_base_into lde_size"); - let log_n = n.trailing_zeros() as u64; - let log_lde = lde_size.trailing_zeros() as u64; let be = backend()?; let stream = be.next_stream(); @@ -1295,7 +2619,15 @@ pub fn coset_lde_batch_base_into( pinned[c * n..c * n + n].copy_from_slice(col); } - let mut buf = stream.alloc_zeros::(m * lde_size)?; + // Zeroed for the legacy transform, which reads the [n, lde_size) tail of + // each slab as the zero padding; the column-major engine reads only the first n. + let engine = slabs_take_engine(n, blowup_factor); + let mut buf = if engine { + // SAFETY: the upload writes each slab's first n, the LDE the rest. + unsafe { stream.alloc::(m * lde_size) }? + } else { + stream.alloc_zeros::(m * lde_size)? + }; for c in 0..m { let mut dst = buf.slice_mut(c * lde_size..c * lde_size + n); stream.memcpy_htod(&pinned[c * n..c * n + n], &mut dst)?; @@ -1304,60 +2636,16 @@ pub fn coset_lde_batch_base_into( // slot stays locked until this event fires (waited before the drain). staging.record_event(&stream)?; - let inv_tw = be.inv_twiddles_for(log_n)?; - let fwd_tw = be.fwd_twiddles_for(log_lde)?; - let weights_dev = stream.clone_htod(weights)?; - - let n_u64 = n as u64; - let lde_u64 = lde_size as u64; - let col_stride_u64 = lde_size as u64; - let m_u32 = m as u32; - - // iNTT bit-reverse + body, pointwise mul, forward bit-reverse + body. - launch_bit_reverse_batched( - stream.as_ref(), - be, - &mut buf, - n_u64, - log_n, - col_stride_u64, - m_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - inv_tw.as_ref(), - n_u64, - log_n, - col_stride_u64, - m_u32, - )?; - launch_pointwise_mul_batched( - stream.as_ref(), - be, - &mut buf, - &weights_dev, - n_u64, - col_stride_u64, - m_u32, - )?; - launch_bit_reverse_batched( - stream.as_ref(), + lde_in_slabs( + &stream, be, &mut buf, - lde_u64, - log_lde, - col_stride_u64, - m_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - fwd_tw.as_ref(), - lde_u64, - log_lde, - col_stride_u64, - m_u32, + m, + n, + blowup_factor, + weights, + SlabInput::Evals, + engine, )?; // Release the staging slot before the drain: the uploads have landed once @@ -1385,7 +2673,8 @@ pub fn coset_lde_batch_base_into( Ok(()) } -/// Fused LDE + row-pair Keccak-256 leaf hashing. Caller receives +/// Fused LDE + row-pair leaf hashing under the family `hash` selects. +/// Caller receives /// `(lde_size / 2) * 32` bytes of leaf hashes in `hashed_leaves_out` (one /// 32-byte digest per bit-reversed row pair, in natural leaf order, matching /// `commit_bit_reversed(.., 2)` on the CPU side). Thin wrapper over @@ -1393,6 +2682,7 @@ pub fn coset_lde_batch_base_into( /// inner-tree build, no device handle. pub fn coset_lde_batch_base_into_with_leaf_hash( columns: &[&[u64]], + hash: DeviceHash, blowup_factor: usize, weights: &[u64], outputs: &mut [&mut [u64]], @@ -1400,11 +2690,12 @@ pub fn coset_lde_batch_base_into_with_leaf_hash( ) -> Result<()> { coset_lde_batch_base_into_with_merkle_tree_inner( columns, + hash, blowup_factor, weights, outputs, hashed_leaves_out, - KeccakCommit::LeavesOnly, + TreeCommit::LeavesOnly, false, 2, ) @@ -1414,11 +2705,12 @@ pub fn coset_lde_batch_base_into_with_leaf_hash( #[allow(clippy::too_many_arguments)] fn coset_lde_batch_base_into_with_merkle_tree_inner( columns: &[&[u64]], + hash: DeviceHash, blowup_factor: usize, weights: &[u64], outputs: &mut [&mut [u64]], nodes_out: &mut [u8], - commit: KeccakCommit, + commit: TreeCommit, keep_device_buf: bool, // 1 = one leaf per bit-reversed row; 2 = one leaf per row pair (2i, 2i+1), // matching the CPU `commit_bit_reversed(.., 2)` used for the trace commit. @@ -1452,10 +2744,8 @@ fn coset_lde_batch_base_into_with_merkle_tree_inner( ); assert_eq!(lde_size % rows_per_leaf, 0); let num_leaves = lde_size / rows_per_leaf; - let nodes_dev_bytes = commit.total_nodes_bytes(num_leaves); + let nodes_dev_bytes = commit.total_nodes_bytes(hash, num_leaves); assert_eq!(nodes_out.len(), nodes_dev_bytes); - let log_n = n.trailing_zeros() as u64; - let log_lde = lde_size.trailing_zeros() as u64; let be = backend()?; let stream = be.next_stream(); @@ -1471,7 +2761,15 @@ fn coset_lde_batch_base_into_with_merkle_tree_inner( pinned[c * n..c * n + n].copy_from_slice(col); } - let mut buf = stream.alloc_zeros::(m * lde_size)?; + // Zeroed for the legacy transform, which reads the [n, lde_size) tail of + // each slab as the zero padding; the column-major engine reads only the first n. + let engine = slabs_take_engine(n, blowup_factor); + let mut buf = if engine { + // SAFETY: the upload writes each slab's first n, the LDE the rest. + unsafe { stream.alloc::(m * lde_size) }? + } else { + stream.alloc_zeros::(m * lde_size)? + }; for c in 0..m { let mut dst = buf.slice_mut(c * lde_size..c * lde_size + n); stream.memcpy_htod(&pinned[c * n..c * n + n], &mut dst)?; @@ -1480,63 +2778,21 @@ fn coset_lde_batch_base_into_with_merkle_tree_inner( // slot stays locked until this event fires (waited before the drain). staging.record_event(&stream)?; - let inv_tw = be.inv_twiddles_for(log_n)?; - let fwd_tw = be.fwd_twiddles_for(log_lde)?; - let weights_dev = stream.clone_htod(weights)?; - - let n_u64 = n as u64; - let lde_u64 = lde_size as u64; - let col_stride_u64 = lde_size as u64; - let m_u32 = m as u32; - - // iNTT - launch_bit_reverse_batched( - stream.as_ref(), - be, - &mut buf, - n_u64, - log_n, - col_stride_u64, - m_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - inv_tw.as_ref(), - n_u64, - log_n, - col_stride_u64, - m_u32, - )?; - launch_pointwise_mul_batched( - stream.as_ref(), - be, - &mut buf, - &weights_dev, - n_u64, - col_stride_u64, - m_u32, - )?; - // forward NTT at LDE size - launch_bit_reverse_batched( - stream.as_ref(), + lde_in_slabs( + &stream, be, &mut buf, - lde_u64, - log_lde, - col_stride_u64, - m_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - fwd_tw.as_ref(), - lde_u64, - log_lde, - col_stride_u64, - m_u32, + m, + n, + blowup_factor, + weights, + SlabInput::Evals, + engine, )?; + let lde_u64 = lde_size as u64; + let col_stride_u64 = lde_size as u64; + // Allocate the device output buffer. In `LeavesOnly` mode this is just // `num_leaves * 32` bytes (the leaves themselves); in `FullTree` mode it's // `(2*num_leaves - 1) * 32` bytes (leaves in the tail + inner nodes filled @@ -1544,33 +2800,73 @@ fn coset_lde_batch_base_into_with_merkle_tree_inner( // written before any reader sees it: the keccak kernel fills the // leaves slab, the inner-tree pass (when present) fills the head. let mut nodes_dev = unsafe { stream.alloc::(nodes_dev_bytes) }?; - let leaves_offset_bytes = commit.leaves_offset_bytes(num_leaves); + let leaves_offset_bytes = commit.leaves_offset_bytes(hash, num_leaves); { let mut leaves_view = nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); - if rows_per_leaf == 2 { - launch_keccak_base_row_pair( + match (hash, rows_per_leaf == 2) { + (DeviceHash::Keccak256, true) => launch_keccak_base_row_pair( + stream.as_ref(), + &buf, + col_stride_u64, + m as u64, + lde_u64, + &mut leaves_view, + )?, + (DeviceHash::Keccak256, false) => launch_keccak_base( + stream.as_ref(), + &buf, + col_stride_u64, + m as u64, + lde_u64, + &mut leaves_view, + )?, + (DeviceHash::Blake3, true) => crate::blake3::launch_leaves_base_row_pair( stream.as_ref(), &buf, col_stride_u64, m as u64, lde_u64, &mut leaves_view, - )?; - } else { - launch_keccak_base( + )?, + (DeviceHash::Blake3, false) => crate::blake3::launch_leaves_base( stream.as_ref(), &buf, col_stride_u64, m as u64, lde_u64, &mut leaves_view, - )?; + )?, + (DeviceHash::Rpx256, true) => crate::rpx::launch_leaves_base_row_pair( + stream.as_ref(), + &buf, + col_stride_u64, + m as u64, + lde_u64, + &mut leaves_view, + )?, + (DeviceHash::Rpx256, false) => crate::rpx::launch_leaves_base( + stream.as_ref(), + &buf, + col_stride_u64, + m as u64, + lde_u64, + &mut leaves_view, + )?, + (DeviceHash::Poseidon1, _) => crate::p1_stark::launch_leaves_cols( + stream.as_ref(), + &buf, + col_stride_u64, + m as u64, + lde_u64, + rows_per_leaf, + &mut leaves_view, + )?, } } - if commit == KeccakCommit::FullTree { - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + if commit == TreeCommit::FullTree { + build_inner_tree_levels_for(hash, stream.as_ref(), be, &mut nodes_dev, num_leaves)?; } // Release the staging slot before the drain: the uploads have landed once @@ -1635,6 +2931,9 @@ pub fn evaluate_poly_coset_batch_ext3_into( ) -> Result<()> { evaluate_poly_coset_batch_ext3_into_inner( coefs, + // No tree on this face (`merkle_nodes_out: None`): the hash dispatch + // key is never read. + DeviceHash::Keccak256, n, blowup_factor, weights, @@ -1657,6 +2956,9 @@ pub fn evaluate_poly_coset_batch_ext3_into_keep( ) -> Result { let opt = evaluate_poly_coset_batch_ext3_into_inner( coefs, + // No tree on this face (`merkle_nodes_out: None` below): the hash + // dispatch key is never read. + DeviceHash::Keccak256, n, blowup_factor, weights, @@ -1667,8 +2969,10 @@ pub fn evaluate_poly_coset_batch_ext3_into_keep( Ok(opt.expect("keep_device_buf=true must return Some")) } +#[allow(clippy::too_many_arguments)] fn evaluate_poly_coset_batch_ext3_into_inner( coefs: &[&[u64]], + hash: DeviceHash, n: usize, blowup_factor: usize, weights: &[u64], @@ -1714,7 +3018,15 @@ fn evaluate_poly_coset_batch_ext3_into_inner( pack_ext3_to_pinned_slabs(coefs, pinned, n); - let mut buf = stream.alloc_zeros::(mb * lde_size)?; + // Zeroed for the legacy transform, which reads the [n, lde_size) tail of + // each slab as the zero padding; the column-major engine reads only the first n. + let engine = slabs_take_engine(n, blowup_factor); + let mut buf = if engine { + // SAFETY: the upload writes each slab's first n, the LDE the rest. + unsafe { stream.alloc::(mb * lde_size) }? + } else { + stream.alloc_zeros::(mb * lde_size)? + }; for s in 0..mb { let mut dst = buf.slice_mut(s * lde_size..s * lde_size + n); stream.memcpy_htod(&pinned[s * n..s * n + n], &mut dst)?; @@ -1723,72 +3035,83 @@ fn evaluate_poly_coset_batch_ext3_into_inner( // slot stays locked until this event fires (waited before the drain). staging.record_event(&stream)?; - let fwd_tw = be.fwd_twiddles_for(log_lde)?; - let weights_dev = stream.clone_htod(weights)?; - - let n_u64 = n as u64; - let lde_u64 = lde_size as u64; - let col_stride_u64 = lde_size as u64; - let mb_u32 = mb as u32; - - // Apply coset scaling: x[k] *= weights[k] for k in 0..n (no iFFT first). - launch_pointwise_mul_batched( - stream.as_ref(), + lde_in_slabs( + &stream, be, &mut buf, - &weights_dev, - n_u64, - col_stride_u64, - mb_u32, + mb, + n, + blowup_factor, + weights, + SlabInput::Coeffs, + engine, )?; - // Bit-reverse full lde_size slab, then forward DIT NTT. - launch_bit_reverse_batched( - stream.as_ref(), - be, - &mut buf, - lde_u64, - log_lde, - col_stride_u64, - mb_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - fwd_tw.as_ref(), - lde_u64, - log_lde, - col_stride_u64, - mb_u32, - )?; + let lde_u64 = lde_size as u64; + let col_stride_u64 = lde_size as u64; // Optional R2-style row-pair Merkle tree build on the LDE buffer, queued // ahead of the drains below. let nodes = if let Some(nodes_out) = merkle_nodes_out { let num_leaves = lde_size / 2; - let tight_total_nodes = 2 * num_leaves - 1; + let tight_total_nodes = crate::tree_nodes(hash, num_leaves); assert_eq!(nodes_out.len(), tight_total_nodes * 32); let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; - let leaves_offset_bytes = (num_leaves - 1) * 32; + let leaves_offset_bytes = crate::leaves_offset(hash, num_leaves) * 32; { let mut leaves_view = nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); let log_num_rows = log_lde; let num_parts_u64 = m as u64; - let cfg = keccak_launch_cfg(num_leaves as u64); - unsafe { - stream - .launch_builder(&be.keccak_comp_poly_leaves_ext3) - .arg(&buf) - .arg(&col_stride_u64) - .arg(&num_parts_u64) - .arg(&lde_u64) - .arg(&log_num_rows) - .arg(&mut leaves_view) - .launch(cfg)?; + match hash { + DeviceHash::Keccak256 => { + let cfg = keccak_launch_cfg(num_leaves as u64); + unsafe { + stream + .launch_builder(&be.keccak_comp_poly_leaves_ext3) + .arg(&buf) + .arg(&col_stride_u64) + .arg(&num_parts_u64) + .arg(&lde_u64) + .arg(&log_num_rows) + .arg(&mut leaves_view) + .launch(cfg)?; + } + } + DeviceHash::Blake3 => crate::blake3::launch_comp_poly_leaves_ext3( + stream.as_ref(), + be, + &buf, + col_stride_u64, + num_parts_u64, + lde_u64, + log_num_rows, + &mut leaves_view, + )?, + DeviceHash::Rpx256 => crate::rpx::launch_comp_poly_leaves_ext3( + stream.as_ref(), + be, + &buf, + col_stride_u64, + num_parts_u64, + lde_u64, + log_num_rows, + &mut leaves_view, + )?, + // The composition leaf is the column-major row-pair leaf over + // the parts' 3·m slabs (`p1s_leaves_cols_pair`). + DeviceHash::Poseidon1 => crate::p1_stark::launch_leaves_cols( + stream.as_ref(), + &buf, + col_stride_u64, + 3 * num_parts_u64, + lde_u64, + 2, + &mut leaves_view, + )?, } } - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels_for(hash, stream.as_ref(), be, &mut nodes_dev, num_leaves)?; Some((nodes_dev, nodes_out)) } else { None @@ -1841,6 +3164,7 @@ fn evaluate_poly_coset_batch_ext3_into_inner( /// `(lde_size - 1) * 32`. Requires `lde_size >= 2`. pub fn evaluate_poly_coset_batch_ext3_into_with_merkle_tree( coefs: &[&[u64]], + hash: DeviceHash, n: usize, blowup_factor: usize, weights: &[u64], @@ -1849,6 +3173,7 @@ pub fn evaluate_poly_coset_batch_ext3_into_with_merkle_tree( ) -> Result<()> { evaluate_poly_coset_batch_ext3_into_inner( coefs, + hash, n, blowup_factor, weights, @@ -1909,8 +3234,6 @@ pub fn coset_lde_batch_ext3_into( assert_eq!(o.len(), 3 * lde_size, "each output must be 3*lde_size u64s"); } assert_u32_domain(lde_size, "coset_lde_batch_ext3_into lde_size"); - let log_n = n.trailing_zeros() as u64; - let log_lde = lde_size.trailing_zeros() as u64; // 3 base slabs per ext3 column; slab index `c*3 + k` holds component `k`. let mb = 3 * m; @@ -1925,8 +3248,16 @@ pub fn coset_lde_batch_ext3_into( pack_ext3_to_pinned_slabs(columns, pinned, n); - // Allocate + zero-pad device buffer holding 3M slabs of `lde_size`. - let mut buf = stream.alloc_zeros::(mb * lde_size)?; + // Device buffer holding 3M slabs of `lde_size`, zeroed for the legacy + // transform, which reads the [n, lde_size) tail of each slab as the zero + // padding; the column-major engine reads only the first n. + let engine = slabs_take_engine(n, blowup_factor); + let mut buf = if engine { + // SAFETY: the upload writes each slab's first n, the LDE the rest. + unsafe { stream.alloc::(mb * lde_size) }? + } else { + stream.alloc_zeros::(mb * lde_size)? + }; // H2D: slab by slab into the first N slots of each `lde_size`-slab. for s in 0..mb { let mut dst = buf.slice_mut(s * lde_size..s * lde_size + n); @@ -1936,61 +3267,16 @@ pub fn coset_lde_batch_ext3_into( // slot stays locked until this event fires (waited before the drain). staging.record_event(&stream)?; - let inv_tw = be.inv_twiddles_for(log_n)?; - let fwd_tw = be.fwd_twiddles_for(log_lde)?; - let weights_dev = stream.clone_htod(weights)?; - - let n_u64 = n as u64; - let lde_u64 = lde_size as u64; - let col_stride_u64 = lde_size as u64; - let mb_u32 = mb as u32; - - // === Butterflies: identical to the base-field batched path, but with - // grid.y = 3M instead of M. === - launch_bit_reverse_batched( - stream.as_ref(), - be, - &mut buf, - n_u64, - log_n, - col_stride_u64, - mb_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - inv_tw.as_ref(), - n_u64, - log_n, - col_stride_u64, - mb_u32, - )?; - launch_pointwise_mul_batched( - stream.as_ref(), - be, - &mut buf, - &weights_dev, - n_u64, - col_stride_u64, - mb_u32, - )?; - launch_bit_reverse_batched( - stream.as_ref(), + lde_in_slabs( + &stream, be, &mut buf, - lde_u64, - log_lde, - col_stride_u64, - mb_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - fwd_tw.as_ref(), - lde_u64, - log_lde, - col_stride_u64, - mb_u32, + mb, + n, + blowup_factor, + weights, + SlabInput::Evals, + engine, )?; // Release the staging slot before the drain: the uploads have landed once @@ -2049,63 +3335,19 @@ pub fn coset_lde_batch_ext3_slabs_keep( } } assert_u32_domain(lde_size, "coset_lde_batch_ext3_slabs_keep lde_size"); - let log_n = n.trailing_zeros() as u64; - let log_lde = lde_size.trailing_zeros() as u64; let be = backend()?; - let inv_tw = be.inv_twiddles_for(log_n)?; - let fwd_tw = be.fwd_twiddles_for(log_lde)?; - let weights_dev = stream.clone_htod(weights)?; - - let n_u64 = n as u64; - let lde_u64 = lde_size as u64; - let col_stride_u64 = lde_size as u64; - let mb_u32 = mb as u32; - - launch_bit_reverse_batched( - stream.as_ref(), - be, - &mut buf, - n_u64, - log_n, - col_stride_u64, - mb_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - inv_tw.as_ref(), - n_u64, - log_n, - col_stride_u64, - mb_u32, - )?; - launch_pointwise_mul_batched( - stream.as_ref(), - be, - &mut buf, - &weights_dev, - n_u64, - col_stride_u64, - mb_u32, - )?; - launch_bit_reverse_batched( - stream.as_ref(), + let engine = slabs_take_engine(n, blowup_factor); + lde_in_slabs( + stream, be, &mut buf, - lde_u64, - log_lde, - col_stride_u64, - mb_u32, - )?; - run_batched_ntt_body( - stream.as_ref(), - &mut buf, - fwd_tw.as_ref(), - lde_u64, - log_lde, - col_stride_u64, - mb_u32, + mb, + n, + blowup_factor, + weights, + SlabInput::Evals, + engine, )?; let ready = match outputs { @@ -2143,6 +3385,124 @@ pub fn coset_lde_batch_ext3_slabs_keep( }) } +/// What the first `n` elements of each slab hold before [`lde_in_slabs`]. +#[derive(Clone, Copy, PartialEq, Eq)] +enum SlabInput { + /// Trace-domain evaluations; `weights[i] = g^i / n`. + Evals, + /// Coefficients; `weights[k] = g^k` (no iNTT, so no `1/n`). + Coeffs, +} + +/// Whether the batched column-major entry points take the column-major engine for +/// an `n`-row, blowup-`blowup` LDE. Decided once per call, before the buffer +/// is allocated: under the engine nothing reads the zero padding, so the call skips +/// the zero fill. +fn slabs_take_engine(n: usize, blowup: usize) -> bool { + crate::lde_cm::engine_enabled() && crate::lde_cm::supports(n, blowup) +} + +/// The transform every batched column-major entry point runs: `slabs` slabs of +/// `n · blowup` in `buf`, the first `n` of each holding values or +/// coefficients (`input`); afterwards every slab holds its coset LDE. `engine` +/// picks the column-major engine ([`crate::lde_cm`], which reads nothing +/// beyond each slab's first `n`) or the legacy per-level batched kernels, +/// which need the slabs' tails zeroed. +#[allow(clippy::too_many_arguments)] +fn lde_in_slabs( + stream: &Arc, + be: &Backend, + buf: &mut CudaSlice, + slabs: usize, + n: usize, + blowup: usize, + weights: &[u64], + input: SlabInput, + engine: bool, +) -> Result<()> { + let lde_size = n * blowup; + if engine { + let ptr = buf.device_ptr(stream).0; + // Each slab's source is its own first `n`: a chunk copies or + // transforms it into the engine's scratch before it writes the slab, + // and no chunk writes another chunk's slabs, so any order is safe. + return crate::lde_cm::lde_columns( + stream, + be, + crate::lde_cm::Cols { + ptr, + stride: lde_size as u64, + }, + ptr, + slabs, + n, + blowup, + weights, + match input { + SlabInput::Evals => crate::lde_cm::Input::Evals, + SlabInput::Coeffs => crate::lde_cm::Input::Coeffs, + }, + crate::lde_cm::ChunkOrder::Ascending, + ); + } + let log_n = n.trailing_zeros() as u64; + let log_lde = lde_size.trailing_zeros() as u64; + let n_u64 = n as u64; + let lde_u64 = lde_size as u64; + let col_stride_u64 = lde_size as u64; + let slabs_u32 = slabs as u32; + let fwd_tw = be.fwd_twiddles_for(log_lde)?; + let weights_dev = stream.clone_htod(weights)?; + if input == SlabInput::Evals { + let inv_tw = be.inv_twiddles_for(log_n)?; + launch_bit_reverse_batched( + stream.as_ref(), + be, + buf, + n_u64, + log_n, + col_stride_u64, + slabs_u32, + )?; + run_batched_ntt_body( + stream.as_ref(), + buf, + inv_tw.as_ref(), + n_u64, + log_n, + col_stride_u64, + slabs_u32, + )?; + } + launch_pointwise_mul_batched( + stream.as_ref(), + be, + buf, + &weights_dev, + n_u64, + col_stride_u64, + slabs_u32, + )?; + launch_bit_reverse_batched( + stream.as_ref(), + be, + buf, + lde_u64, + log_lde, + col_stride_u64, + slabs_u32, + )?; + run_batched_ntt_body( + stream.as_ref(), + buf, + fwd_tw.as_ref(), + lde_u64, + log_lde, + col_stride_u64, + slabs_u32, + ) +} + /// Run the DIT butterfly body of a bit-reversed-input NTT over `m` batched /// columns in one device buffer. Same fusion strategy as `run_ntt_body`: /// first 8 levels shmem-fused (coalesced), subsequent levels one kernel each. @@ -2219,3 +3579,223 @@ fn run_batched_ntt_body( } Ok(()) } + +/// LDE microbenchmark: the row-major commit's stages under the legacy +/// engine and under the column-major one, on one `n × m` matrix already +/// resident on the device, with a host synchronize after each stage — the +/// same stages and shapes as the W3/K0 same-input comparison with ZisK +/// (`thoughts/zf/gap/W3-KERNELS.md`). Measurement only; nothing on a proving +/// path calls it. +#[doc(hidden)] +pub mod lde_bench { + use super::*; + use std::time::Instant; + + /// Stage times of one commit, and its root's first eight bytes (the two + /// engines must agree on it). + #[derive(Debug, Clone, Copy)] + pub struct StageTimes { + /// Everything before the leaves: staging, snapshot, the LDE — the + /// production expansion of the engine under test. + pub expand_ms: f64, + /// Engine only, timed after the root is read by re-running the expansion + /// in two timed halves: `stage_ms` puts the column-major trace where + /// the engine reads it (0 when the snapshot is retained: it is the + /// source), `lde_ms` is the engine alone, column-major source to + /// column-major LDE — the like-for-like with ZisK's `LDE(dst, src)`. + pub stage_ms: f64, + pub lde_ms: f64, + pub leaves_ms: f64, + pub inner_ms: f64, + /// Legacy only: the in-place transpose to column-major; 0 for the engine. + pub transpose_ms: f64, + pub root0: u64, + } + + /// Upload a deterministic `n × m` row-major matrix once. + pub fn upload(n: usize, m: usize) -> Result> { + let be = backend()?; + let stream = be.next_stream(); + let host: Vec = (0..n * m) + .map(|i| (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) % 0xFFFF_FFFF_0000_0001) + .collect(); + let d = stream.clone_htod(&host)?; + stream.synchronize()?; + Ok(d) + } + + fn weights(n: usize) -> Vec { + (0..n as u64).map(|i| i.wrapping_mul(7) + 1).collect() + } + + fn ms(t: Instant) -> f64 { + t.elapsed().as_secs_f64() * 1e3 + } + + /// One commit of the device-resident `input` (`n × m`, row-major) at + /// `blowup`, `rows_per_leaf` rows per leaf, column-major engine or legacy. + /// `snapshot` retains the trace-domain snapshot as the main commits do. + #[allow(clippy::too_many_arguments)] + pub fn run( + input: &CudaSlice, + hash: DeviceHash, + n: usize, + m: usize, + blowup: usize, + rows_per_leaf: usize, + snapshot: bool, + engine: bool, + ) -> Result { + let be = backend()?; + let stream = be.next_stream(); + let w = weights(n); + let lde_size = n * blowup; + let num_leaves = lde_size / rows_per_leaf; + let nodes_bytes = TreeCommit::FullTree.total_nodes_bytes(hash, num_leaves); + let leaves_offset = TreeCommit::FullTree.leaves_offset_bytes(hash, num_leaves); + + stream.synchronize()?; + let t0 = Instant::now(); + let (mut buf, snap) = if engine { + expand_col_major_on_stream( + &stream, + be, + InnerInput::Dev(input), + n, + m, + blowup, + &w, + snapshot, + )? + } else { + expand_row_major_on_stream( + &stream, + be, + InnerInput::Dev(input), + n, + m, + blowup, + &w, + snapshot, + )? + }; + stream.synchronize()?; + let expand_ms = ms(t0); + + let mut nodes_dev = unsafe { stream.alloc::(nodes_bytes) }?; + stream.synchronize()?; + let t1 = Instant::now(); + if engine { + col_major_leaves_into_tree( + hash, + &stream, + be, + &buf, + lde_size, + 0, + m, + rows_per_leaf, + &mut nodes_dev, + leaves_offset, + )?; + } else { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); + launch_row_major_leaves( + hash, + stream.as_ref(), + be, + &buf, + m as u64, + 0, + m as u64, + lde_size as u64, + rows_per_leaf, + &mut leaves_view, + )?; + } + stream.synchronize()?; + let leaves_ms = ms(t1); + + let t2 = Instant::now(); + build_inner_tree_levels_for(hash, stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + stream.synchronize()?; + let inner_ms = ms(t2); + let mut root = [0u8; 32]; + crate::readback::memcpy_dtoh(&stream, &nodes_dev.slice(0..32), &mut root)?; + drop(nodes_dev); + + let (mut stage_ms, mut lde_ms, mut transpose_ms) = (0.0, 0.0, 0.0); + if engine { + // The root is read; the LDE is recomputed in place, in two timed + // halves, from the same kind of source the expansion used. + let buf_ptr = buf.device_ptr(&stream).0; + let (src, order) = match &snap { + Some(s) => ( + crate::lde_cm::Cols { + ptr: s.device_ptr(&stream).0, + stride: n as u64, + }, + crate::lde_cm::ChunkOrder::Ascending, + ), + None => { + stream.synchronize()?; + let t = Instant::now(); + launch_transpose_tiles( + &stream, + be, + &input.slice(0..n * m), + &mut buf.slice_mut(0..n * m), + n, + m, + n as u64, + )?; + stream.synchronize()?; + stage_ms = ms(t); + ( + crate::lde_cm::Cols { + ptr: buf_ptr, + stride: n as u64, + }, + crate::lde_cm::ChunkOrder::Descending, + ) + } + }; + stream.synchronize()?; + let t = Instant::now(); + crate::lde_cm::lde_columns( + &stream, + be, + src, + buf_ptr, + m, + n, + blowup, + &w, + crate::lde_cm::Input::Evals, + order, + )?; + stream.synchronize()?; + lde_ms = ms(t); + drop(buf); + } else { + stream.synchronize()?; + let t3 = Instant::now(); + let cm = transpose_lde_in_place(&stream, be, buf, lde_size, m)?; + stream.synchronize()?; + drop(cm); + transpose_ms = ms(t3); + } + drop(snap); + stream.synchronize()?; + Ok(StageTimes { + expand_ms, + stage_ms, + lde_ms, + leaves_ms, + inner_ms, + transpose_ms, + root0: u64::from_le_bytes(root[..8].try_into().unwrap()), + }) + } +} diff --git a/crypto/math-cuda/src/lde_cm.rs b/crypto/math-cuda/src/lde_cm.rs new file mode 100644 index 000000000..44ee1314d --- /dev/null +++ b/crypto/math-cuda/src/lde_cm.rs @@ -0,0 +1,936 @@ +//! Column-major coset LDE engine — the device LDE of the commits. +//! +//! The legacy LDEs in [`crate::lde`] run one butterfly level per launch over +//! the whole matrix: an LDE of `2^22 × 245` at blowup 2 makes ~33 whole-matrix +//! DRAM passes. This engine instead transforms the columns a CHUNK at a time, +//! the chunk sized so its working set stays in L2 between launches, and runs +//! 4..8 levels per launch out of registers and shared memory +//! (`kernels/ntt_cm.cu`). DRAM then sees the trace about once and the LDE about +//! once, and the LDE comes out column-major — the layout every downstream kernel +//! reads — so the row-major commits also lose their in-place transpose. On the +//! W3/K0 shapes it runs within 1.2–1.4× of ZisK's production LDE and cuts the +//! whole row-major commit to 0.58–0.67× (`thoughts/zf/gap/fix/NTT.md`). +//! +//! Per chunk of `C` columns the driver runs: +//! +//! 1. the iNTT of each column, DIF (natural in, bit-reversed out) with the +//! inverse roots and no `1/n`: the first pass reads the caller's source and +//! writes the chunk's scratch `W` (`C × n`), so the source is never written; +//! the last pass multiplies position `p` by `weights[rev(p)]` — the caller's +//! coset weights, which carry the `1/n`; +//! 2. the forward NTT at `N = n·blowup`, DIT (bit-reversed in, natural out): the +//! first pass reads `W` with the coset spread fused in (position `B·q` is +//! `W[q]`, every other position zero — never stored), the rest run in place +//! in the destination column. +//! +//! The field values are those of the legacy pipeline — the LDE of a column is +//! one polynomial evaluated on one coset, and both pipelines use the caller's +//! weights and the same roots of unity — so roots and every downstream value +//! are unchanged; only the non-canonical representation of a value may differ, +//! which nothing downstream observes (the hashes and the serialisation +//! canonicalise). +//! +//! The WHIR commit's encoding (`whir::commit_from`) is one column of the same +//! forward transform with no coset: the coefficients, spread at the blowup's +//! spacing, go through the forward passes ([`spread_ntt_column`]) with the +//! spread fused into the first. It reads an 8 KiB windowed root table, where +//! the legacy encoding reads `Backend::fwd_twiddles_for`: `2^(log_n − 1)` +//! values per codeword size, built on the host and cached on the card for the +//! life of the process, outside the VRAM ledger (512 MiB for a `2^27` +//! codeword, 2 GiB for `2^29`). The encoding takes passes of up to 10 levels +//! ([`plan_wide`]): three passes at the production `2^28` instead of four +//! ([`NTT_WIDE_ENV`]`=0` keeps [`plan`]'s). Every pass is bound by memory +//! bandwidth, so the passes are the cost; the trace LDEs keep [`plan`]. +//! +//! It is the default wherever it applies ([`engine_enabled`]). The legacy path +//! stays for what the engine does not take — a caller that wants the row-major +//! LDE back on the host, and shapes outside [`supports`] (for the encoding, +//! [`spread_supports`]) — and, whole, under [`LEGACY_ENV`]. + +use std::cell::Cell; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; + +use cudarc::driver::sys; +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, LaunchConfig, PushKernelArg}; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsFFTField; + +use crate::Result; +use crate::device::Backend; + +/// `LAMBDA_VM_LDE_LEGACY=1` sends every LDE back to the legacy per-level +/// pipeline, the commits' trace LDEs and the WHIR commit's encoding alike — an +/// A/B and rollback switch, not a tuning knob: both produce the same values, +/// roots and proofs. Read once per process. +pub const LEGACY_ENV: &str = "LAMBDA_VM_LDE_LEGACY"; + +thread_local! { + /// Per-thread override of [`LEGACY_ENV`], for parity tests that run both + /// pipelines in one process. See [`with_engine`]. + static ENGINE_OVERRIDE: Cell> = const { Cell::new(None) }; +} + +/// Whether the calling thread's LDEs and WHIR encodings take this engine (where +/// it applies). +pub fn engine_enabled() -> bool { + if let Some(on) = ENGINE_OVERRIDE.with(Cell::get) { + return on; + } + static ENV: OnceLock = OnceLock::new(); + *ENV.get_or_init(|| { + let legacy = std::env::var(LEGACY_ENV).is_ok_and(|v| v == "1"); + // One line per process, so every box log states which LDE it ran. + eprintln!( + "[gpu] LDE: {}", + if legacy { + "legacy per-level (LAMBDA_VM_LDE_LEGACY=1)" + } else { + "column-major engine" + } + ); + !legacy + }) +} + +/// Run `f` with the engine forced on or off for the calling thread — the +/// entry points decide on the calling thread before any work is queued, so a +/// test can compare both pipelines on the same inputs in one process. +#[doc(hidden)] +pub fn with_engine(on: bool, f: impl FnOnce() -> R) -> R { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + ENGINE_OVERRIDE.with(|c| c.set(self.0)); + } + } + let _restore = Restore(ENGINE_OVERRIDE.with(|c| c.replace(Some(on)))); + f() +} + +/// Columns this engine has extended, process-wide: the counter a test reads to +/// know the engine, and not the legacy path, did the work. +static COLUMNS_EXTENDED: AtomicU64 = AtomicU64::new(0); + +/// See [`COLUMNS_EXTENDED`]. +pub fn columns_extended() -> u64 { + COLUMNS_EXTENDED.load(Ordering::Relaxed) +} + +/// Codewords [`spread_ntt_column`] has encoded, process-wide: the counter a +/// test reads to know the engine, and not the legacy path, did a WHIR encoding. +static CODEWORDS_ENCODED: AtomicU64 = AtomicU64::new(0); + +/// See [`CODEWORDS_ENCODED`]. +pub fn codewords_encoded() -> u64 { + CODEWORDS_ENCODED.load(Ordering::Relaxed) +} + +/// The WHIR encoding's transform ([`spread_ntt_column`]) runs in +/// [`plan_wide`]'s passes of up to 10 levels by default; +/// `LAMBDA_VM_WHIR_NTT_WIDE=0` (or `off`) keeps [`plan`]'s passes of up to 8. +/// The codeword's values are the same either way; only a value's +/// non-canonical representation may differ, as between this engine and the +/// legacy one. Read once per process. +pub const NTT_WIDE_ENV: &str = "LAMBDA_VM_WHIR_NTT_WIDE"; + +thread_local! { + /// Per-thread override of [`NTT_WIDE_ENV`], for parity tests that run + /// both plans in one process. See [`with_ntt_wide`]. + static NTT_WIDE_OVERRIDE: Cell> = const { Cell::new(None) }; +} + +/// `LAMBDA_VM_WHIR_NTT_WIDE`'s value read: empty, `0` or `off` keep the +/// 8-level plan; anything else (unset included) is the wide plan. +pub fn ntt_wide_setting(value: Option<&str>) -> bool { + !matches!(value.map(str::trim), Some("" | "0" | "off")) +} + +/// Whether the calling thread's WHIR encodings take [`plan_wide`]. +pub fn ntt_wide_enabled() -> bool { + if let Some(on) = NTT_WIDE_OVERRIDE.with(Cell::get) { + return on; + } + static ENV: OnceLock = OnceLock::new(); + *ENV.get_or_init(|| { + let on = ntt_wide_setting(std::env::var(NTT_WIDE_ENV).ok().as_deref()); + if on { + eprintln!( + "[gpu] WHIR NTT: wide, passes of up to 10 levels (the default; {NTT_WIDE_ENV}=0 keeps passes of up to 8)" + ); + } + on + }) +} + +/// Run `f` with the wide plan forced on or off for the calling thread. +#[doc(hidden)] +pub fn with_ntt_wide(on: bool, f: impl FnOnce() -> R) -> R { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + NTT_WIDE_OVERRIDE.with(|c| c.set(self.0)); + } + } + let _restore = Restore(NTT_WIDE_OVERRIDE.with(|c| c.replace(Some(on)))); + f() +} + +/// Codewords [`spread_ntt_column`] encoded with a pass of more than 8 levels, +/// process-wide: what a test reads to know the wide kernels ran. +static WIDE_CODEWORDS_ENCODED: AtomicU64 = AtomicU64::new(0); + +/// See [`WIDE_CODEWORDS_ENCODED`]. +pub fn wide_codewords_encoded() -> u64 { + WIDE_CODEWORDS_ENCODED.load(Ordering::Relaxed) +} + +/// Smallest trace a column can have here: a pass holds 16 elements per +/// thread, so a transform needs at least 2^4 of them. +pub const MIN_LOG_N: u32 = 4; + +/// Largest blowup the fused spread covers: the first forward pass must hold a +/// whole coset of `B` spread positions (`B <= 2^k`, `k >= 4`). +pub const MAX_LOG_BLOWUP: u32 = 4; + +/// Whether an `n`-row, blowup-`b` LDE fits the engine's shapes. Callers take +/// the legacy path otherwise. +pub fn supports(n: usize, blowup: usize) -> bool { + n.is_power_of_two() + && blowup.is_power_of_two() + && blowup >= 2 + && n.trailing_zeros() >= MIN_LOG_N + && blowup.trailing_zeros() <= MAX_LOG_BLOWUP + && (n * blowup).trailing_zeros() <= GoldilocksField::TWO_ADICITY as u32 +} + +/// Whether a length-`2^log_n` transform of an input spread at spacing +/// `2^log_blowup` fits [`spread_ntt_column`]: the first pass must hold a whole +/// spacing (`2^log_blowup <= 2^k`, `k >= 4`). Spacing 1 (`log_blowup = 0`) is a +/// plain transform of a bit-reversed input. +pub fn spread_supports(log_n: u32, log_blowup: u32) -> bool { + log_n >= MIN_LOG_N + && log_n <= GoldilocksField::TWO_ADICITY as u32 + && log_blowup <= MAX_LOG_BLOWUP + && log_blowup <= log_n +} + +/// Pass sizes for a length-`2^l` transform: `ceil(l / 8)` passes of 4..=8 +/// levels, within one of each other. Mirrored by the host KAT's `plan`. +pub(crate) fn plan(l: u32) -> Vec { + assert!(l >= MIN_LOG_N, "transform too short for the engine: 2^{l}"); + let p = l.div_ceil(8); + let mut ks = vec![l / p; p as usize]; + for k in ks.iter_mut().take((l % p) as usize) { + *k += 1; + } + debug_assert!(ks.iter().all(|&k| (4..=8).contains(&k))); + ks +} + +/// The largest pass [`plan_wide`] takes: the kernels instantiate wide passes +/// of 9 and 10 levels (`ntt_cm_dit_k9`, `_k10`). +pub(crate) const WIDE_MAX_K: u32 = 10; + +/// Pass sizes for the WHIR encoding's length-`2^l` transform under +/// [`NTT_WIDE_ENV`]: `ceil(l / 10)` passes of 4..=10 levels, within one of +/// each other — `[10, 9, 9]` at `2^28`. Where that is as many passes as +/// [`plan`] it is [`plan`]. Mirrored by the host KAT's `plan_wide`. +pub(crate) fn plan_wide(l: u32) -> Vec { + assert!(l >= MIN_LOG_N, "transform too short for the engine: 2^{l}"); + let p = l.div_ceil(WIDE_MAX_K); + let mut ks = vec![l / p; p as usize]; + for k in ks.iter_mut().take((l % p) as usize) { + *k += 1; + } + debug_assert!(ks.iter().all(|&k| (4..=WIDE_MAX_K).contains(&k))); + ks +} + +/// log2 of the lanes a block spans: 256 threads at 16 elements each hold +/// `4096 >> k` sub-transforms' worth, capped by what the level window allows +/// (`2^s` lanes of `lo` for a strided pass, `2^(l-k)` sub-transforms for the +/// contiguous one). Mirrored by the host KAT's `pass_log_t`. +fn pass_log_t(l: u32, s: u32, k: u32) -> u32 { + let cap = 12 - k; + let lim = if s > 0 { s } else { l - k }; + cap.min(lim) +} + +/// A pass's dynamic shared memory: its `2^k × (T + 1)` tile, and on a wide +/// pass (k > 8) a skew of `T` words every 16 rows. Mirrors the kernels' +/// `tile_words`. The radix table is static: `max(2^(k−1), 128)` words. +fn tile_bytes(k: u32, log_t: u32) -> u32 { + let mut words = (1u32 << k) * ((1u32 << log_t) + 1); + if k > 8 { + words += (1u32 << (k - 4)) << log_t; + } + words * 8 +} + +/// Kernel flags, as in `kernels/ntt_cm.cu`. +const F_SPREAD: u32 = 1; +const F_GATHER: u32 = 2; +const F_STORE_W: u32 = 4; +const F_MOBIUS: u32 = 8; + +/// The windowed root tables the kernels take: `t[w·256 + j] = root^(j << 8w)` +/// for the field's primitive `2^32`-th root (`fwd`) and its inverse (`inv`). +pub(crate) struct RootTables { + fwd: CudaSlice, + inv: CudaSlice, +} + +fn windowed(root: FieldElement) -> Vec { + let mut out = Vec::with_capacity(1024); + let mut base = root; + for _ in 0..4 { + let mut acc = FieldElement::::one(); + for _ in 0..256 { + out.push(*acc.value()); + acc = &acc * &base; + } + // base^(2^8) for the next window. + for _ in 0..8 { + base = base.square(); + } + } + out +} + +pub(crate) fn root_tables(be: &Backend) -> Result<&'static RootTables> { + static TABLES: OnceLock = OnceLock::new(); + if let Some(t) = TABLES.get() { + return Ok(t); + } + let root = GoldilocksField::get_primitive_root_of_unity(32).expect("2^32-th root"); + let root_inv = root.inv().expect("root is non-zero"); + let stream = be.next_stream(); + let fwd = stream.clone_htod(&windowed(root))?; + let inv = stream.clone_htod(&windowed(root_inv))?; + stream.synchronize()?; + // A racing initialiser's tables are dropped; either set is the same bytes. + let _ = TABLES.set(RootTables { fwd, inv }); + Ok(TABLES.get().expect("root tables just set")) +} + +/// Columns per chunk: as many whole columns (trace scratch plus LDE) as fit +/// in 60% of L2, at least one, at most `m` — ZisK's chunk rule, which leaves +/// room for the tables and the stream's other traffic. The scratch is one +/// chunk of trace columns, so it too stays within 60% of L2. +fn chunk_cols(be: &Backend, n: usize, lde: usize, m: usize) -> usize { + static L2: OnceLock = OnceLock::new(); + let l2 = *L2.get_or_init(|| { + be.ctx + .attribute(sys::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE) + .ok() + .filter(|&b| b > 0) + .map(|b| b as usize) + .unwrap_or(32 << 20) + }); + let per_col = (n + lde) * 8; + // gridDim.y is the column: at most 65535 per launch. + (l2 * 3 / 5 / per_col).clamp(1, m.clamp(1, 65535)) +} + +/// The order a chunk loop walks the columns in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ChunkOrder { + /// Any source that no chunk's output overlaps. + Ascending, + /// A compact source at the head of the destination buffer (column `c` at + /// `c·n`): chunk `[c0, c0 + C)` writes `[c0·N, (c0 + C)·N)`, which covers + /// source columns `[c0·B, (c0 + C)·B)` — all at or above `c0`, so walking + /// down, every one of them has been read (the chunk's own ones by its + /// first pass, before its output is written). + Descending, +} + +/// What the source columns hold. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Input { + /// Evaluations on the trace domain, natural order: iNTT, weights, NTT. + Evals, + /// Coefficients, natural order: weights, NTT (no iNTT). + Coeffs, +} + +/// Where column `c` of an operand lives: `ptr + c·stride` elements. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Cols { + pub ptr: sys::CUdeviceptr, + pub stride: u64, +} + +impl Cols { + fn at(self, c: usize) -> sys::CUdeviceptr { + self.ptr + (c as u64) * self.stride * 8 + } +} + +/// One pass launch over `cols` columns. +#[allow(clippy::too_many_arguments)] +fn launch_pass( + stream: &CudaStream, + be: &Backend, + dit: bool, + k: u32, + l: u32, + s: u32, + lb: u32, + flags: u32, + input: Cols, + output: Cols, + roots: sys::CUdeviceptr, + wtab: sys::CUdeviceptr, + cols: usize, +) -> Result<()> { + let log_t = pass_log_t(l, s, k); + let threads = (1u32 << log_t) << (k - 4); + let cfg = LaunchConfig { + grid_dim: (1u32 << (l - k - log_t), cols as u32, 1), + block_dim: (threads, 1, 1), + shared_mem_bytes: tile_bytes(k, log_t), + }; + // The wide passes are DIT only (the encoding's forward transform). + debug_assert!(dit || k <= 8, "no DIF pass of {k} levels"); + let f = match (dit, k) { + (true, 9..) => &be.ntt_cm_dit_wide[(k - 9) as usize], + (true, _) => &be.ntt_cm_dit[(k - 4) as usize], + (false, _) => &be.ntt_cm_dif[(k - 4) as usize], + }; + unsafe { + stream + .launch_builder(f) + .arg(&input.ptr) + .arg(&input.stride) + .arg(&output.ptr) + .arg(&output.stride) + .arg(&roots) + .arg(&wtab) + .arg(&l) + .arg(&s) + .arg(&log_t) + .arg(&lb) + .arg(&flags) + .launch(cfg)?; + } + Ok(()) +} + +/// The coset LDE of `m` columns on `stream`: column `c` of `src` (`n` values, +/// natural order, `input` says evaluations or coefficients) becomes column `c` +/// of `dst` (`n·blowup` evaluations on the coset, natural order, stride +/// `n·blowup`). `weights` are the caller's coset weights in natural order: +/// `g^j / n` for [`Input::Evals`], `g^j` for [`Input::Coeffs`]. +/// +/// `src` is only read, and is left intact unless it lies inside `dst` — which +/// [`ChunkOrder::Descending`] then makes safe for a compact head source. +/// Nothing synchronises; the scratch is freed stream-ordered. +#[allow(clippy::too_many_arguments)] +pub(crate) fn lde_columns( + stream: &Arc, + be: &Backend, + src: Cols, + dst: sys::CUdeviceptr, + m: usize, + n: usize, + blowup: usize, + weights: &[u64], + input: Input, + order: ChunkOrder, +) -> Result<()> { + assert!( + supports(n, blowup), + "column-engine LDE shape n={n} blowup={blowup}" + ); + assert_eq!(weights.len(), n, "one weight per trace row"); + if m == 0 { + return Ok(()); + } + let lde = n * blowup; + let log_n = n.trailing_zeros(); + let lb = blowup.trailing_zeros(); + let log_lde = log_n + lb; + let tables = root_tables(be)?; + let dst = Cols { + ptr: dst, + stride: lde as u64, + }; + + // The weights at bit-reversed positions: the iNTT leaves coefficient + // `rev(p)` at `p`, and the coefficient form gathers at the same index. + let mut wbr = stream.clone_htod(weights)?; + unsafe { + stream + .launch_builder(&be.bit_reverse_permute) + .arg(&mut wbr) + .arg(&(n as u64)) + .arg(&(log_n as u64)) + .launch(LaunchConfig::for_num_elems(n as u32))?; + } + + let chunk = chunk_cols(be, n, lde, m); + // The chunk's trace-domain scratch: the iNTT's output (evaluations), or a + // copy of the coefficients (so a source inside `dst` is read before the + // chunk's output overwrites it). SAFETY: every element a later pass reads + // is written first — by the iNTT's first pass or by the copy. + let scratch = unsafe { stream.alloc::(chunk * n) }?; + let (fwd_roots, _fr) = tables.fwd.device_ptr(stream); + let (inv_roots, _ir) = tables.inv.device_ptr(stream); + let (wtab, _wr) = wbr.device_ptr(stream); + let (scratch_ptr, _sr) = scratch.device_ptr(stream); + let w = Cols { + ptr: scratch_ptr, + stride: n as u64, + }; + if input == Input::Coeffs { + // Raw driver copies below: the context must be current on this thread. + be.ctx.bind_to_thread()?; + } + + let inv_plan = plan(log_n); + let fwd_plan = plan(log_lde); + let chunks: Vec<(usize, usize)> = (0..m) + .step_by(chunk) + .map(|c0| (c0, chunk.min(m - c0))) + .collect(); + let walk: Box> = match order { + ChunkOrder::Ascending => Box::new(chunks.iter()), + ChunkOrder::Descending => Box::new(chunks.iter().rev()), + }; + for &(c0, cols) in walk { + let chunk_src = Cols { + ptr: src.at(c0), + stride: src.stride, + }; + let chunk_dst = Cols { + ptr: dst.at(c0), + stride: dst.stride, + }; + match input { + Input::Evals => { + // DIF passes descend to s = 0. + let mut s = log_n; + for (i, &k) in inv_plan.iter().enumerate() { + s -= k; + let last = i + 1 == inv_plan.len(); + launch_pass( + stream, + be, + false, + k, + log_n, + s, + 0, + if last { F_STORE_W } else { 0 }, + if i == 0 { chunk_src } else { w }, + w, + inv_roots, + wtab, + cols, + )?; + } + } + Input::Coeffs => { + for i in 0..cols { + // SAFETY: both ranges are `n` elements inside live + // allocations (the caller's source column, the scratch), + // queued on the stream every pass of this chunk runs on. + unsafe { + sys::cuMemcpyDtoDAsync_v2( + w.at(i), + chunk_src.at(i), + n * 8, + stream.cu_stream(), + ) + .result()?; + } + } + } + } + let spread = match input { + Input::Evals => F_SPREAD, + Input::Coeffs => F_SPREAD | F_GATHER, + }; + // DIT passes ascend from s = 0. + let mut s = 0; + for (i, &k) in fwd_plan.iter().enumerate() { + launch_pass( + stream, + be, + true, + k, + log_lde, + s, + lb, + if i == 0 { spread } else { 0 }, + if i == 0 { w } else { chunk_dst }, + chunk_dst, + fwd_roots, + wtab, + cols, + )?; + s += k; + } + } + COLUMNS_EXTENDED.fetch_add(m as u64, Ordering::Relaxed); + Ok(()) +} + +/// The plan [`spread_ntt_column`] takes on the calling thread. +fn spread_plan(log_n: u32) -> Vec { + if ntt_wide_enabled() { + plan_wide(log_n) + } else { + plan(log_n) + } +} + +/// How many low Möbius levels [`spread_ntt_column`]'s first pass holds for a +/// length-`2^log_n` transform at spacing `2^log_blowup`, in the plan in force +/// on the calling thread: a block stages `2^(k + log T)` positions, so +/// `2^(k + log T − log_blowup)` consecutive coefficients — `12 − log_blowup` +/// levels once the transform reaches 2^12. +pub(crate) fn fused_mobius_levels(log_n: u32, log_blowup: u32) -> u32 { + let k = spread_plan(log_n)[0]; + k + pass_log_t(log_n, 0, k) - log_blowup +} + +/// The forward transform of ONE spread column — the WHIR commit's encoding +/// (`whir::commit_from`; the legacy path is `ntt.cu`'s `lift_spread` then +/// `ntt::run_ntt_body`), fused: position `2^log_blowup · q` of the +/// length-`2^log_n` DIT input (bit-reversed in, natural out) is `src[q]`, +/// every other position zero, and `dst` receives the natural-order transform. +/// The spread is the first pass's load, so the padding is never stored or +/// read, and the transform is `ceil(log_n / 8)` passes instead of the spread +/// pass, the fused 8-level pass and one 5-level tile per five levels above it +/// — `ceil(log_n / 10)` under [`NTT_WIDE_ENV`] ([`plan_wide`]). +/// `src` (`2^(log_n - log_blowup)` values) and `dst` (`2^log_n`) must not +/// overlap. Nothing synchronises. +/// +/// With `mobius_levels` (non-zero, and then exactly +/// [`fused_mobius_levels`]) the first pass also runs the encoding's Möbius +/// levels `[0, mobius_levels)` over the coefficients it stages: the caller has +/// run the others (the levels commute). +pub(crate) fn spread_ntt_column( + stream: &CudaStream, + be: &Backend, + src: sys::CUdeviceptr, + dst: sys::CUdeviceptr, + log_n: u32, + log_blowup: u32, + mobius_levels: u32, +) -> Result<()> { + assert!( + spread_supports(log_n, log_blowup), + "a spread transform of 2^{log_n} at spacing 2^{log_blowup} is outside the engine" + ); + assert!( + mobius_levels == 0 || mobius_levels == fused_mobius_levels(log_n, log_blowup), + "the first pass of 2^{log_n} holds {} Möbius levels, not {mobius_levels}", + fused_mobius_levels(log_n, log_blowup) + ); + let tables = root_tables(be)?; + let (fwd_roots, _fr) = tables.fwd.device_ptr(stream); + let one = |ptr| Cols { ptr, stride: 0 }; + let ks = spread_plan(log_n); + let first_flags = if mobius_levels > 0 { + F_SPREAD | F_MOBIUS + } else { + F_SPREAD + }; + let mut s = 0; + for (i, &k) in ks.iter().enumerate() { + let first = i == 0; + launch_pass( + stream, + be, + true, + k, + log_n, + s, + log_blowup, + if first { first_flags } else { 0 }, + one(if first { src } else { dst }), + one(dst), + fwd_roots, + 0, + 1, + )?; + s += k; + } + CODEWORDS_ENCODED.fetch_add(1, Ordering::Relaxed); + if ks.iter().any(|&k| k > 8) { + WIDE_CODEWORDS_ENCODED.fetch_add(1, Ordering::Relaxed); + } + Ok(()) +} + +/// Hash the leaves of a column-major matrix: `num_cols` columns of `num_rows` +/// rows at `cols` (column stride `col_stride` elements), `rows_per_leaf` +/// bit-reversed rows per leaf, into `leaves` (`num_rows / rows_per_leaf` +/// digests of 32 bytes). The column-major twins of the row-major leaf +/// kernels: each absorbs the same elements in the same order (row by row, +/// column by column), which the parity suite pins. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_col_major_leaves( + hash: crate::DeviceHash, + stream: &CudaStream, + be: &Backend, + cols: sys::CUdeviceptr, + col_stride: u64, + num_cols: u64, + num_rows: u64, + rows_per_leaf: usize, + leaves: sys::CUdeviceptr, +) -> Result<()> { + use crate::DeviceHash; + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); + // Every kernel derives rows as `__brevll(..) >> (64 - log_num_rows)`, UB + // at `log_num_rows == 0`. + assert!(num_rows >= 2 && num_rows.is_power_of_two()); + if hash == DeviceHash::Poseidon1 { + return crate::p1_stark::launch_leaves_cols_ptr( + stream, + cols, + col_stride, + num_cols, + num_rows, + rows_per_leaf, + leaves, + ); + } + let log_num_rows = num_rows.trailing_zeros() as u64; + let threads = num_rows / rows_per_leaf as u64; + let (kernel, cfg) = match (hash, rows_per_leaf) { + (DeviceHash::Keccak256, 2) => ( + &be.keccak256_leaves_base_row_pair_batched, + crate::merkle::keccak_launch_cfg(threads), + ), + (DeviceHash::Keccak256, _) => ( + &be.keccak256_leaves_base_batched, + crate::merkle::keccak_launch_cfg(threads), + ), + (DeviceHash::Blake3, 2) => ( + &be.blake3_leaves_base_row_pair_batched, + crate::blake3::blake3_launch_cfg(threads), + ), + (DeviceHash::Blake3, _) => ( + &be.blake3_leaves_base_batched, + crate::blake3::blake3_launch_cfg(threads), + ), + (DeviceHash::Rpx256, 2) => ( + &be.rpx_leaves_base_row_pair_batched, + crate::rpx::rpx_launch_cfg(threads), + ), + (DeviceHash::Rpx256, _) => ( + &be.rpx_leaves_base_batched, + crate::rpx::rpx_launch_cfg(threads), + ), + (DeviceHash::Poseidon1, _) => unreachable!("Poseidon1 leaves launched above"), + }; + unsafe { + stream + .launch_builder(kernel) + .arg(&cols) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(&leaves) + .launch(cfg)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plans_cover_every_level_in_passes_of_four_to_eight() { + for l in MIN_LOG_N..=32 { + let ks = plan(l); + assert_eq!(ks.iter().sum::(), l, "l = {l}"); + assert_eq!(ks.len() as u32, l.div_ceil(8), "l = {l}: pass count"); + assert!(ks.iter().all(|&k| (4..=8).contains(&k)), "l = {l}: {ks:?}"); + let (lo, hi) = (ks.iter().min().unwrap(), ks.iter().max().unwrap()); + assert!(hi - lo <= 1, "l = {l}: uneven {ks:?}"); + } + } + + #[test] + fn wide_plans_cover_every_level_in_passes_of_four_to_ten() { + for l in MIN_LOG_N..=32 { + let ks = plan_wide(l); + assert_eq!(ks.iter().sum::(), l, "l = {l}"); + assert_eq!(ks.len() as u32, l.div_ceil(10), "l = {l}: pass count"); + assert!( + ks.iter().all(|&k| (4..=WIDE_MAX_K).contains(&k)), + "l = {l}: {ks:?}" + ); + let (lo, hi) = (ks.iter().min().unwrap(), ks.iter().max().unwrap()); + assert!(hi - lo <= 1, "l = {l}: uneven {ks:?}"); + if ks.len() == plan(l).len() { + assert_eq!( + ks, + plan(l), + "l = {l}: as many passes as today's, today's plan" + ); + } + } + assert_eq!(plan_wide(28), [10, 9, 9], "the production encoding"); + assert_eq!(plan(28), [7, 7, 7, 7], "today's production encoding"); + // The sizes where the plans differ up to 2^30 (the host KAT pins 9, 10 + // and 17..20 against today's plan). + let differ: Vec = (MIN_LOG_N..=30) + .filter(|&l| plan_wide(l) != plan(l)) + .collect(); + assert_eq!(differ, [9, 10, 17, 18, 19, 20, 25, 26, 27, 28, 29, 30]); + } + + /// A first pass at the lane cap stages 2^12 positions in either plan, so it + /// holds the low `12 − log_blowup` Möbius levels; below 2^12, all of them. + #[test] + fn the_first_pass_holds_the_low_mobius_levels() { + for wide in [false, true] { + with_ntt_wide(wide, || { + for lb in 0..=MAX_LOG_BLOWUP { + for log_n in 12.max(lb + MIN_LOG_N)..=30 { + assert_eq!(fused_mobius_levels(log_n, lb), 12 - lb, "2^{log_n} lb {lb}"); + } + } + assert_eq!( + fused_mobius_levels(11, 1), + 10, + "every level of a 2^10 stack" + ); + assert_eq!(fused_mobius_levels(28, 1), 11, "the production encoding"); + }); + } + } + + /// The wide plan unless the knob says `0`, `off` or nothing. + #[test] + fn the_wide_ntt_runs_unless_zero() { + assert!(ntt_wide_setting(None), "the default is the wide plan"); + assert!(ntt_wide_setting(Some("1"))); + assert!(ntt_wide_setting(Some("on"))); + assert!(!ntt_wide_setting(Some("0"))); + assert!(!ntt_wide_setting(Some(" 0"))); + assert!(!ntt_wide_setting(Some("off"))); + assert!(!ntt_wide_setting(Some(""))); + assert!(with_ntt_wide(true, ntt_wide_enabled)); + assert!(!with_ntt_wide(false, ntt_wide_enabled)); + } + + /// Every (s, k) window the driver launches: DIT ascending from s = 0, DIF + /// descending to s = 0, and the encoding's wide DIT passes. + fn windows(l: u32) -> Vec<(u32, u32)> { + let ks = plan(l); + let mut out = Vec::new(); + let mut s = 0; + for &k in &ks { + out.push((s, k)); + s += k; + } + let mut s = l; + for &k in &ks { + s -= k; + out.push((s, k)); + } + let mut s = 0; + for &k in &plan_wide(l) { + out.push((s, k)); + s += k; + } + out + } + + #[test] + fn a_block_never_exceeds_256_threads_or_its_window() { + for l in MIN_LOG_N..=32 { + for (s, k) in windows(l) { + let log_t = pass_log_t(l, s, k); + let threads = (1u32 << log_t) << (k - 4); + assert!(threads <= 256, "l={l} s={s} k={k}: {threads} threads"); + if s > 0 { + assert!(log_t <= s, "l={l} s={s} k={k}: lanes beyond 2^s"); + } else { + assert!( + log_t <= l - k, + "l={l} k={k}: more sub-transforms than exist" + ); + } + assert!(s + k <= l, "l={l} s={s} k={k}: window past the transform"); + // The tile, and the static radix table: 2^(k-1) words, 128 at least. + let smem = tile_bytes(k, log_t) + (1u32 << (k - 1).max(7)) * 8; + assert!( + smem <= 48 * 1024, + "l={l} s={s} k={k}: {smem} B of shared memory" + ); + if k <= 8 { + assert_eq!( + tile_bytes(k, log_t), + (1u32 << k) * ((1u32 << log_t) + 1) * 8, + "l={l} s={s} k={k}: today's tile" + ); + } + } + } + } + + #[test] + fn windowed_tables_are_powers_of_the_two_adic_root() { + let root = GoldilocksField::get_primitive_root_of_unity(32).unwrap(); + let t = windowed(root); + assert_eq!(t.len(), 1024); + for w in 0..4u32 { + for j in [0u64, 1, 2, 17, 128, 255] { + let want = root.pow(j << (8 * w)); + assert_eq!( + FieldElement::::from_raw(t[(w * 256) as usize + j as usize]), + want, + "window {w} entry {j}" + ); + } + } + // The top window is the radix table the kernels read: ω_256^j. + let w256 = GoldilocksField::get_primitive_root_of_unity(8).unwrap(); + for j in 0..128u64 { + assert_eq!( + FieldElement::::from_raw(t[768 + j as usize]), + w256.pow(j), + "radix entry {j}" + ); + } + } + + #[test] + fn spread_shapes_outside_the_engine_are_declined() { + assert!(spread_supports(4, 0), "spacing 1 is a plain transform"); + assert!(spread_supports(24, 2), "the WHIR production shape"); + assert!(spread_supports(8, 4)); + assert!(!spread_supports(3, 0), "transform below 2^4"); + assert!( + !spread_supports(20, 5), + "spacing beyond one first-pass tile" + ); + assert!(!spread_supports(33, 2), "beyond the two-adicity"); + } + + #[test] + fn shapes_outside_the_engine_are_declined() { + assert!(supports(1 << 4, 2)); + assert!(supports(1 << 22, 16)); + assert!(!supports(1 << 3, 2), "trace below 2^4"); + assert!(!supports(1 << 10, 1), "blowup 1 has no spread"); + assert!(!supports(1 << 10, 32), "blowup beyond the fused spread"); + assert!(!supports(3 << 10, 2), "not a power of two"); + } +} diff --git a/crypto/math-cuda/src/lib.rs b/crypto/math-cuda/src/lib.rs index 838bf9044..e749f27c9 100644 --- a/crypto/math-cuda/src/lib.rs +++ b/crypto/math-cuda/src/lib.rs @@ -5,20 +5,62 @@ //! Everything else (`ntt`, element-wise arith) is either internal to those //! pipelines or used by the parity test suite. +/// A launcher's precondition, refused as an error rather than a panic: the +/// caller handed a shape the kernel does not take, which is +/// `CUDA_ERROR_INVALID_VALUE` (the callers map a device error to a refusal or +/// a host fallback). The message is the check's documentation; it is not +/// evaluated. +macro_rules! ensure { + ($cond:expr $(, $($msg:tt)+)?) => { + if !$cond { + return Err($crate::invalid_value()); + } + }; +} + +/// [`ensure!`] for an equality. +macro_rules! ensure_eq { + ($left:expr, $right:expr $(, $($msg:tt)+)?) => { + if $left != $right { + return Err($crate::invalid_value()); + } + }; +} + +pub mod argue_fused; +pub mod argue_probe; pub mod barycentric; +pub mod blake3; +pub mod columns; pub mod constraint_interp; pub mod deep; +pub mod deep_inv; pub mod device; #[cfg(feature = "test-faults")] pub mod faults; +pub mod fill; pub mod fri; +pub mod gkr; pub mod grinding; pub mod inverse; pub mod lde; +pub mod lde_cm; pub mod logup; pub mod merkle; +pub mod narrow; pub mod ntt; pub mod nvtx; +pub mod p1_stark; +pub mod p1w16; +pub mod readback; +pub mod round_slot; +pub mod rpx; +pub mod rpx_paths; +pub mod stream_class; +pub mod sumcheck; +pub mod whir; +pub mod whir_coset; +pub mod whir_open; // Re-exported for downstream crates so they can refer to CUDA primitive // types without depending on cudarc directly. @@ -30,6 +72,87 @@ use crate::device::{Backend, backend}; pub type Result = std::result::Result; +/// The error a refused launcher precondition returns ([`ensure!`]). +pub(crate) fn invalid_value() -> cudarc::driver::DriverError { + cudarc::driver::DriverError(cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE) +} + +/// Which hash family a device tree build launches. +/// +/// The fused LDE+commit pipelines ([`lde`]), the composition-poly tree +/// builders ([`merkle`] / [`blake3`]) and the FRI layer commits ([`fri`]) +/// each exist kernel-for-kernel in both BYTE families; this enum is the +/// dispatch key callers pass down. It deliberately carries no round counts or +/// parameters: within one build each family is a single concrete hash +/// (keccak-256, or `Blake3Chain` at the compiled round count), exactly as on +/// the host. +/// +/// ★ Of the three ALGEBRAIC keys, [`DeviceHash::Rpx256`] is ported +/// ([`rpx`]); RPO256 and Poseidon name hashes whose device kernels are not. +/// Every dispatch site in this crate carries an arm for the unported keys that +/// aborts with `unimplemented!` naming the hash — never an arm that launches a +/// byte-hash kernel in its place. The keys exist ahead of their kernels so the +/// host side (`stark::config::DeviceTreeBackend`) can name every commitment +/// hash under `cuda`: a tree labelled RPO is then built by RPO kernels or not +/// built at all. Porting a family means replacing those arms with launches, +/// and the set of arms is the checklist. +/// +/// ★ **Why the key travels with the request at all.** On the host a Merkle +/// backend both NAMES a hash and computes it, so a tree cannot wear a name its +/// own code did not produce. On the device the backend only names it — the +/// kernels hash — so without a key travelling alongside, a tree labelled RPX +/// could be built by keccak's kernels and nothing would notice. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DeviceHash { + /// Keccak-256 leaves and parents. + Keccak256, + /// `Blake3Chain` leaves and parents at the compiled round count. + Blake3, + /// RPX256 (XHash12) leaves and parents — [`rpx`]'s kernels, the + /// algebraic family's first device port. + Rpx256, + /// ZisK's Poseidon1 instance (width 16, rate 12, arity-4 trees) — the + /// `p1/*` exploration branch. Kernels in [`p1w16`]; a dispatch site not yet + /// wired aborts loudly on this key. + Poseidon1, +} + +impl DeviceHash { + /// The name a tree built under this key may be called by. + pub const fn name(self) -> &'static str { + match self { + Self::Keccak256 => "keccak256", + Self::Blake3 => "blake3-chain", + Self::Rpx256 => "rpx256", + Self::Poseidon1 => "poseidon1-w16", + } + } + + /// Children per Merkle node: 4 under ZisK's Poseidon1 (its trees are + /// 4-ary, [`p1_stark`]), 2 under every other key. + pub const fn arity(self) -> usize { + match self { + Self::Poseidon1 => p1_stark::ARITY, + Self::Keccak256 | Self::Blake3 | Self::Rpx256 => 2, + } + } +} + +/// Stored nodes of a device tree over `leaves` leaves under `hash`: the binary +/// heap's `2·leaves − 1`, or the arity-4 layout's count ([`p1_stark::tree_nodes`]). +pub fn tree_nodes(hash: DeviceHash, leaves: usize) -> usize { + match hash.arity() { + 2 => 2 * leaves - 1, + _ => p1_stark::tree_nodes(leaves), + } +} + +/// Where the leaves start in that node buffer, in nodes: both layouts store +/// the leaves last. +pub fn leaves_offset(hash: DeviceHash, leaves: usize) -> usize { + tree_nodes(hash, leaves) - leaves +} + /// Toolchain sanity: plain wrapping u64 vector add. Not a field op. pub fn vector_add_u64(a: &[u64], b: &[u64]) -> Result> { launch_binary_u64(a, b, |be| &be.vector_add_u64) @@ -70,7 +193,7 @@ pub fn gl_neg_u64(a: &[u64]) -> Result> { .launch(cfg)?; } - let out = stream.clone_dtoh(&c_dev)?; + let out = crate::readback::clone_dtoh(&stream, &c_dev)?; stream.synchronize()?; Ok(out) } @@ -100,7 +223,7 @@ pub fn ext3_mul_u64(a: &[u64], b: &[u64]) -> Result> { .arg(&n_u64) .launch(cfg)?; } - let out = stream.clone_dtoh(&c_dev)?; + let out = crate::readback::clone_dtoh(&stream, &c_dev)?; stream.synchronize()?; Ok(out) } @@ -129,7 +252,7 @@ pub fn ext3_sub_u64(a: &[u64], b: &[u64]) -> Result> { .arg(&n_u64) .launch(cfg)?; } - let out = stream.clone_dtoh(&c_dev)?; + let out = crate::readback::clone_dtoh(&stream, &c_dev)?; stream.synchronize()?; Ok(out) } @@ -158,7 +281,7 @@ pub fn ext3_add_u64(a: &[u64], b: &[u64]) -> Result> { .arg(&n_u64) .launch(cfg)?; } - let out = stream.clone_dtoh(&c_dev)?; + let out = crate::readback::clone_dtoh(&stream, &c_dev)?; stream.synchronize()?; Ok(out) } @@ -191,7 +314,7 @@ where .launch(cfg)?; } - let out = stream.clone_dtoh(&c_dev)?; + let out = crate::readback::clone_dtoh(&stream, &c_dev)?; stream.synchronize()?; Ok(out) } diff --git a/crypto/math-cuda/src/logup.rs b/crypto/math-cuda/src/logup.rs index ac449e989..bc507a000 100644 --- a/crypto/math-cuda/src/logup.rs +++ b/crypto/math-cuda/src/logup.rs @@ -44,12 +44,11 @@ pub struct LogupDescriptor<'a> { } fn cfg(total: usize) -> Result { - // See `batch_inverse_ext3_dev` for the rationale: a u32 grid_dim is - // truncated past u32::MAX / BLOCK_SIZE, which would silently launch too - // few blocks and leave a tail of the (uninitialized) output unwritten. - // Runtime Err, not debug_assert, so release builds also route to the - // caller's CPU fallback. - if total > u32::MAX as usize / BLOCK_SIZE as usize { + // See `inverse::launch_total_fits` for the bound and its reasoning: past + // it the u32 element count would truncate and leave a tail of the + // (uninitialized) output unwritten. Runtime Err, not debug_assert, so + // release builds also route to the caller's fallback or abort. + if !crate::inverse::launch_total_fits(total) { return Err(cudarc::driver::DriverError( cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE, )); @@ -453,7 +452,8 @@ pub fn logup_aux_resident( let t_accum_done = std::time::Instant::now(); // L = table_contribution = S[n-1] (sum of all term columns, all rows). - let l_host: Vec = stream.clone_dtoh(&row_sum.slice((num_rows - 1) * 3..num_rows * 3))?; + let l_host: Vec = + crate::readback::clone_dtoh(stream, &row_sum.slice((num_rows - 1) * 3..num_rows * 3))?; stream.synchronize()?; if timing { let t_end = std::time::Instant::now(); @@ -483,3 +483,26 @@ pub fn logup_aux_resident( table_contribution: [l_host[0], l_host[1], l_host[2]], }) } + +#[cfg(test)] +mod launch_cfg_tests { + use super::*; + + /// 6 interactions × 2^22 rows (LOCAL_TO_GLOBAL in a 2^22 epoch) launches + /// 98,304 blocks; it used to be refused as if it were 256× larger. + #[test] + fn the_first_refused_aux_build_shape_launches() { + let c = cfg(6 << 22).expect("expressible"); + assert_eq!(c.grid_dim, (98_304, 1, 1)); + assert_eq!(c.block_dim, (BLOCK_SIZE, 1, 1)); + } + + /// The largest expressible launch is exactly u32::MAX elements; one more + /// is refused with the same error the driver would have used. + #[test] + fn the_bound_is_u32_max_elements() { + let c = cfg(u32::MAX as usize).expect("expressible"); + assert_eq!(c.grid_dim.0, 1 << 24); + assert!(cfg(u32::MAX as usize + 1).is_err()); + } +} diff --git a/crypto/math-cuda/src/merkle.rs b/crypto/math-cuda/src/merkle.rs index 02532f6de..f8daf9853 100644 --- a/crypto/math-cuda/src/merkle.rs +++ b/crypto/math-cuda/src/merkle.rs @@ -72,7 +72,7 @@ pub fn keccak_leaves_base( num_rows as u64, &mut out_dev.as_view_mut(), )?; - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; stream.synchronize()?; Ok(out) } @@ -123,7 +123,7 @@ pub fn keccak_leaves_ext3( num_rows as u64, &mut out_dev.as_view_mut(), )?; - let out = stream.clone_dtoh(&out_dev)?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; stream.synchronize()?; Ok(out) } @@ -152,12 +152,42 @@ pub(crate) fn keccak_launch_cfg(num_rows: u64) -> LaunchConfig { /// `log2(leaves_len)` times invoking `keccak_merkle_level` to fill in the /// inner nodes from the bottom up. Mirrors the CPU `build(nodes, leaves_len)` /// scan in `crypto/crypto/src/merkle_tree/merkle.rs`. +/// Build every inner level of a Merkle tree, under the hash `hash` names. +/// +/// The two kernel families have identical signatures and identical node layout, +/// so the dispatch is a choice of handle and nothing else — which is the whole +/// reason an algebraic hash costs two kernels here rather than a second tree +/// builder. pub(crate) fn build_inner_tree_levels( stream: &CudaStream, be: &Backend, nodes_dev: &mut CudaSlice, leaves_len: usize, + hash: crate::DeviceHash, ) -> Result<()> { + let (level_fn, tail_fn) = match hash { + crate::DeviceHash::Keccak256 => (&be.keccak_merkle_level, &be.keccak_merkle_tail), + // The half-warp Merkle tops (the default, `rpx_paths::warp_merkle`) + // live in `crate::rpx`'s walk; switched off, RPX takes the walk below. + crate::DeviceHash::Rpx256 if crate::rpx_paths::warp_merkle() => { + return crate::rpx::build_inner_tree_levels(stream, be, nodes_dev, leaves_len); + } + crate::DeviceHash::Rpx256 => (&be.rpx_merkle_level, &be.rpx_merkle_tail), + // ZisK's Poseidon1 trees are 4-ary in their own layout + // (`crate::p1_stark`): the caller allocated `crate::tree_nodes` nodes + // for it and wrote the leaves at `crate::leaves_offset`. + crate::DeviceHash::Poseidon1 => { + return crate::p1_stark::build_inner_tree_levels(stream, nodes_dev, leaves_len); + } + // ⚠ REFUSE rather than fall through. BLAKE3 does have inner-tree + // kernels, but they are walked by `crate::blake3` with its own tail + // geometry; reaching them from here would build a second, differently + // shaped tree under the same name. RPO and Poseidon have none at all. + other => unimplemented!( + "no device Merkle kernels reachable here for {} ({other:?})", + other.name() + ), + }; // Once a level fits this many pairs, one single-block launch // (`keccak_merkle_tail`) builds all remaining levels with barriers // between them: the top levels of a big tree are each smaller than the @@ -186,7 +216,7 @@ pub(crate) fn build_inner_tree_levels( }; unsafe { stream - .launch_builder(&be.keccak_merkle_tail) + .launch_builder(tail_fn) .arg(&mut *nodes_dev) .arg(&level_begin) .launch(cfg)?; @@ -196,7 +226,7 @@ pub(crate) fn build_inner_tree_levels( let cfg = keccak_launch_cfg(n_pairs); unsafe { stream - .launch_builder(&be.keccak_merkle_level) + .launch_builder(level_fn) .arg(&mut *nodes_dev) .arg(&new_begin) .arg(&n_pairs) @@ -341,9 +371,15 @@ pub fn build_merkle_tree_on_device(hashed_leaves: &[u8]) -> Result> { stream.memcpy_htod(hashed_leaves, &mut slice)?; } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, leaves_len)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + leaves_len, + crate::DeviceHash::Keccak256, + )?; - let out = stream.clone_dtoh(&nodes_dev)?; + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; stream.synchronize()?; Ok(out) } @@ -415,6 +451,47 @@ pub fn gather_merkle_paths_dev( Ok(host) } +/// Read the Merkle cap at height `cap_height` off a device-resident tree: the +/// `2^c` nodes `c` levels below the root, left to right, as `2^c * 32` bytes. +/// +/// No kernel: the device heap has the host layout (root at node 0, the level +/// with `2^c` nodes at `[2^c - 1, 2^{c+1} - 1)`), so the cap is one D2H of the +/// heap slice `[(2^c - 1) * 32, (2^{c+1} - 1) * 32)`. The +/// same nodes `MerkleTree::cap` returns on the host tree, byte for byte. +/// `cap_height = 0` is the root. Runs on the caller's `stream`, after the work +/// already queued on it, and waits for the copy. +/// +/// Panics on a shape no caller may pass (the same contract as +/// [`gather_merkle_paths_dev`]): `leaves_len` not a power of two, a cap taller +/// than the tree, or a node buffer too short for the heap it claims to hold. +pub fn read_cap_dev( + nodes_dev: &CudaSlice, + leaves_len: usize, + cap_height: usize, + stream: &Arc, +) -> Result> { + assert!( + leaves_len.is_power_of_two(), + "read_cap_dev: leaves_len must be a power of two" + ); + let depth = leaves_len.trailing_zeros() as usize; + assert!( + cap_height <= depth, + "read_cap_dev: cap height {cap_height} exceeds the tree depth {depth}" + ); + let start = ((1usize << cap_height) - 1) * 32; + let end = ((2usize << cap_height) - 1) * 32; + assert!( + end <= nodes_dev.len(), + "read_cap_dev: node buffer of {} bytes is shorter than the cap slice end {end}", + nodes_dev.len() + ); + let mut host = vec![0u8; end - start]; + crate::readback::memcpy_dtoh(stream, &nodes_dev.slice(start..end), &mut host)?; + stream.synchronize()?; + Ok(host) +} + /// Build the composition Merkle tree on device. `parts_interleaved` is /// `num_parts` slices, each an ext3 LDE column interleaved as /// `[a0,a1,a2, b0,b1,b2, ...]` of length `3*lde_size`. Leaves hash row pairs, so @@ -422,7 +499,12 @@ pub fn gather_merkle_paths_dev( /// and the stream it was built on. Used by the device keep wrapper below. fn build_comp_poly_tree_nodes_dev( parts_interleaved: &[&[u64]], + rows_per_leaf: usize, ) -> Result<(CudaSlice, usize, Arc)> { + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); assert!(!parts_interleaved.is_empty()); let m = parts_interleaved.len(); let ext3_elems = parts_interleaved[0].len() / 3; @@ -436,7 +518,7 @@ fn build_comp_poly_tree_nodes_dev( } let lde_size = ext3_elems; assert!(lde_size.is_power_of_two() && lde_size >= 2); - let num_leaves = lde_size / 2; + let num_leaves = lde_size / rows_per_leaf; let tight_total_nodes = 2 * num_leaves - 1; let be = backend()?; @@ -470,9 +552,16 @@ fn build_comp_poly_tree_nodes_dev( let num_rows_u64 = lde_size as u64; let log_num_rows = lde_size.trailing_zeros() as u64; let cfg = keccak_launch_cfg(num_leaves as u64); + // Row pairs: rows `2i`, `2i+1` of every part; one row (S2): the row + // `reverse_index(i)` alone — the one-row ext3 kernel, same arguments. + let kernel = if rows_per_leaf == 2 { + &be.keccak_comp_poly_leaves_ext3 + } else { + &be.keccak256_leaves_ext3_batched + }; unsafe { stream - .launch_builder(&be.keccak_comp_poly_leaves_ext3) + .launch_builder(kernel) .arg(&buf) .arg(&col_stride_u64) .arg(&num_parts_u64) @@ -483,7 +572,13 @@ fn build_comp_poly_tree_nodes_dev( } } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; Ok((nodes_dev, num_leaves, stream)) } @@ -497,12 +592,28 @@ pub fn build_comp_poly_tree_from_slabs_dev( m: usize, lde_size: usize, ) -> Result { + build_comp_poly_tree_from_slabs_dev_rpl(stream, buf, m, lde_size, 2) +} + +/// [`build_comp_poly_tree_from_slabs_dev`] with `rows_per_leaf` rows per leaf +/// (2 = row pair, 1 = S2 one row: `lde_size` leaves). +pub fn build_comp_poly_tree_from_slabs_dev_rpl( + stream: &Arc, + buf: &CudaSlice, + m: usize, + lde_size: usize, + rows_per_leaf: usize, +) -> Result { + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); #[cfg(feature = "test-faults")] crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; assert!(m > 0); assert!(lde_size.is_power_of_two() && lde_size >= 2); assert_eq!(buf.len(), 3 * m * lde_size, "slab buffer shape"); - let num_leaves = lde_size / 2; + let num_leaves = lde_size / rows_per_leaf; let tight_total_nodes = 2 * num_leaves - 1; let be = backend()?; @@ -516,9 +627,16 @@ pub fn build_comp_poly_tree_from_slabs_dev( let num_rows_u64 = lde_size as u64; let log_num_rows = lde_size.trailing_zeros() as u64; let cfg = keccak_launch_cfg(num_leaves as u64); + // Row pairs: rows `2i`, `2i+1` of every part; one row (S2): the row + // `reverse_index(i)` alone — the one-row ext3 kernel, same arguments. + let kernel = if rows_per_leaf == 2 { + &be.keccak_comp_poly_leaves_ext3 + } else { + &be.keccak256_leaves_ext3_batched + }; unsafe { stream - .launch_builder(&be.keccak_comp_poly_leaves_ext3) + .launch_builder(kernel) .arg(buf) .arg(&col_stride_u64) .arg(&num_parts_u64) @@ -528,13 +646,20 @@ pub fn build_comp_poly_tree_from_slabs_dev( .launch(cfg)?; } } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; let mut root = [0u8; 32]; - stream.memcpy_dtoh(&nodes_dev.slice(0..32), &mut root)?; + crate::readback::memcpy_dtoh(stream, &nodes_dev.slice(0..32), &mut root)?; stream.synchronize()?; Ok(crate::lde::GpuMerkleTree { nodes: Arc::new(nodes_dev), leaves_len: num_leaves, + arity: 2, root, }) } @@ -545,16 +670,27 @@ pub fn build_comp_poly_tree_from_slabs_dev( /// tree to host. `leaves_len = lde_size / 2` (row pair leaves). pub fn build_comp_poly_tree_from_evals_ext3_keep( parts_interleaved: &[&[u64]], +) -> Result { + build_comp_poly_tree_from_evals_ext3_keep_rpl(parts_interleaved, 2) +} + +/// [`build_comp_poly_tree_from_evals_ext3_keep`] with `rows_per_leaf` rows per +/// leaf (2 = row pair, 1 = S2 one row: `lde_size` leaves). +pub fn build_comp_poly_tree_from_evals_ext3_keep_rpl( + parts_interleaved: &[&[u64]], + rows_per_leaf: usize, ) -> Result { #[cfg(feature = "test-faults")] crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; - let (nodes_dev, num_leaves, stream) = build_comp_poly_tree_nodes_dev(parts_interleaved)?; + let (nodes_dev, num_leaves, stream) = + build_comp_poly_tree_nodes_dev(parts_interleaved, rows_per_leaf)?; let mut root = [0u8; 32]; - stream.memcpy_dtoh(&nodes_dev.slice(0..32), &mut root)?; + crate::readback::memcpy_dtoh(&stream, &nodes_dev.slice(0..32), &mut root)?; stream.synchronize()?; Ok(crate::lde::GpuMerkleTree { nodes: Arc::new(nodes_dev), leaves_len: num_leaves, + arity: 2, root, }) } @@ -598,9 +734,15 @@ pub fn build_fri_layer_tree_from_evals_ext3(evals: &[u64]) -> Result> { } } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; - let out = stream.clone_dtoh(&nodes_dev)?; + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; stream.synchronize()?; Ok(out) } diff --git a/crypto/math-cuda/src/narrow.rs b/crypto/math-cuda/src/narrow.rs new file mode 100644 index 000000000..f9f55a22d --- /dev/null +++ b/crypto/math-cuda/src/narrow.rs @@ -0,0 +1,295 @@ +//! Widening a packed main trace on the device (the stark crate's `NarrowMain` +//! layout): column `c` holds `rows` little-endian words of `widths[c]` bytes +//! (1, 2, 4 or 8) from byte `offsets[c]` of the packed data. The words come +//! back bit for bit, row-major, as the LDE reads its input, or column-major, +//! as a table's run of columns lies in the multilinear prover's resident store +//! ([`widen_col_major_into`]). + +use std::sync::Arc; + +use cudarc::driver::{CudaStream, CudaView, CudaViewMut, LaunchConfig, PushKernelArg}; + +use crate::Result; +use crate::device::{Backend, backend}; + +/// A packed trace on the host (see the module docs). +#[derive(Clone, Copy, Debug)] +pub struct NarrowInput<'a> { + data: &'a [u8], + offsets: &'a [u64], + widths: &'a [u8], + rows: usize, +} + +impl<'a> NarrowInput<'a> { + /// `rows × cols` words packed as the module docs say. + pub fn new( + data: &'a [u8], + offsets: &'a [u64], + widths: &'a [u8], + rows: usize, + cols: usize, + ) -> Self { + assert_eq!(offsets.len(), cols, "packed trace: one offset per column"); + assert_eq!(widths.len(), cols, "packed trace: one width per column"); + Self { + data, + offsets, + widths, + rows, + } + } + + /// Rows of the trace. + pub fn rows(&self) -> usize { + self.rows + } + + /// Columns of the trace. + pub fn cols(&self) -> usize { + self.widths.len() + } + + /// The packed bytes. + pub fn bytes(&self) -> usize { + self.data.len() + } + + /// The packed bytes, the columns' offsets and their widths. + pub(crate) fn parts(&self) -> (&'a [u8], &'a [u64], &'a [u8]) { + (self.data, self.offsets, self.widths) + } +} + +/// Upload `t`'s packed columns and widen them into `dst` (`rows × cols` +/// row-major words), all on `stream`: a consumer on the same stream needs no +/// further ordering, and the packed copy is freed in stream order. +pub(crate) fn widen_into( + stream: &Arc, + be: &Backend, + t: NarrowInput, + rows: usize, + cols: usize, + dst: &mut CudaViewMut, +) -> Result<()> { + assert_eq!(t.rows, rows, "packed trace: rows"); + assert_eq!(t.widths.len(), cols, "packed trace: columns"); + let cells = rows * cols; + assert_eq!(dst.len(), cells, "packed trace: destination size"); + if cells == 0 { + return Ok(()); + } + let data_dev = stream.clone_htod(t.data)?; + let offsets_dev = stream.clone_htod(t.offsets)?; + let widths_dev = stream.clone_htod(t.widths)?; + let cfg = + LaunchConfig::for_num_elems(u32::try_from(cells).expect("packed trace: cells fit u32")); + let (rows_u64, cols_u64) = (rows as u64, cols as u64); + unsafe { + stream + .launch_builder(&be.widen_narrow_row_major) + .arg(&data_dev) + .arg(&offsets_dev) + .arg(&widths_dev) + .arg(dst) + .arg(&rows_u64) + .arg(&cols_u64) + .launch(cfg)?; + } + Ok(()) +} + +/// [`widen_into`] writing `dst` column-major (`rows × cols` words, column `c` +/// at `c · rows`): the same words, laid out as a table's run of columns in the +/// resident store. All on `stream`, so the packed copy is freed in stream order +/// and a reader on the same stream needs no further ordering. +pub(crate) fn widen_col_major_into( + stream: &Arc, + be: &Backend, + t: NarrowInput, + rows: usize, + cols: usize, + dst: &mut CudaViewMut, +) -> Result<()> { + assert_eq!(t.rows, rows, "packed trace: rows"); + assert_eq!(t.widths.len(), cols, "packed trace: columns"); + let cells = rows * cols; + assert_eq!(dst.len(), cells, "packed trace: destination size"); + if cells == 0 { + return Ok(()); + } + let data_dev = stream.clone_htod(t.data)?; + let offsets_dev = stream.clone_htod(t.offsets)?; + let widths_dev = stream.clone_htod(t.widths)?; + let cfg = + LaunchConfig::for_num_elems(u32::try_from(cells).expect("packed trace: cells fit u32")); + let (rows_u64, cols_u64) = (rows as u64, cols as u64); + unsafe { + stream + .launch_builder(&be.widen_narrow_col_major) + .arg(&data_dev) + .arg(&offsets_dev) + .arg(&widths_dev) + .arg(dst) + .arg(&rows_u64) + .arg(&cols_u64) + .launch(cfg)?; + } + Ok(()) +} + +/// [`widen_col_major_into`] on a stream of its own, downloaded: the device's +/// column-major widen, for the round-trip tests. +pub fn widen_col_major_to_host( + data: &[u8], + offsets: &[u64], + widths: &[u8], + rows: usize, + cols: usize, +) -> Result> { + let be = backend()?; + let stream = be.next_stream(); + // SAFETY: `widen_col_major_into` writes all `rows × cols` words. + let mut out = unsafe { stream.alloc::(rows * cols) }?; + widen_col_major_into( + &stream, + be, + NarrowInput::new(data, offsets, widths, rows, cols), + rows, + cols, + &mut out.slice_mut(..), + )?; + let words = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(words) +} + +/// [`widen_into`] on a stream of its own, downloaded: the device widen's words, +/// for the round-trip tests. +pub fn widen_to_host( + data: &[u8], + offsets: &[u64], + widths: &[u8], + rows: usize, + cols: usize, +) -> Result> { + let be = backend()?; + let stream = be.next_stream(); + // SAFETY: `widen_into` writes all `rows × cols` words. + let mut out = unsafe { stream.alloc::(rows * cols) }?; + widen_into( + &stream, + be, + NarrowInput::new(data, offsets, widths, rows, cols), + rows, + cols, + &mut out.slice_mut(..), + )?; + let words = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(words) +} + +/// The bytes a word needs: 1, 2, 4 or 8 (the stark crate's `NarrowMain` rule). +pub fn width_of(max: u64) -> u8 { + match max { + 0..=0xff => 1, + 0x100..=0xffff => 2, + 0x1_0000..=0xffff_ffff => 4, + _ => 8, + } +} + +/// Pack the trace snapshot a commit left on the device (`handle.trace_dev`, +/// column-major) the way the stark crate's `NarrowMain` packs a host trace: +/// `(widths, data)`, column `c` from byte `Σ rows · widths[c' < c]` of `data`. +/// `None` when the handle kept no snapshot. Runs on a stream of its own after +/// the commit's `ready` event; only the packed bytes cross the bus. +pub fn pack_trace_snapshot(handle: &crate::lde::GpuLdeBase) -> Result, Vec)>> { + let Some(src) = handle.trace_dev.as_deref() else { + return Ok(None); + }; + let (rows, cols) = (handle.trace_rows, handle.m); + if rows == 0 || cols == 0 || src.len() < rows * cols { + return Ok(None); + } + let be = backend()?; + let stream = be.next_stream(); + handle.wait_ready_on(&stream)?; + pack_col_major_on_stream(&stream, be, &src.slice(..rows * cols), rows, cols).map(Some) +} + +/// Pack the column-major `rows × cols` words `src` holds, on `stream`: the +/// widths and the packed columns, downloaded. `src` is read only after the +/// work already queued on `stream`. +pub(crate) fn pack_col_major_on_stream( + stream: &Arc, + be: &Backend, + src: &CudaView<'_, u64>, + rows: usize, + cols: usize, +) -> Result<(Vec, Vec)> { + let mut max_dev = stream.alloc_zeros::(cols)?; + let rows_u64 = rows as u64; + let cfg = LaunchConfig { + grid_dim: ( + u32::try_from(rows.div_ceil(256).min(64)).expect("at most 64"), + u32::try_from(cols).expect("packed trace: columns fit u32"), + 1, + ), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.column_max_col_major) + .arg(src) + .arg(&rows_u64) + .arg(&mut max_dev) + .launch(cfg)?; + } + let maxes = crate::readback::clone_dtoh(stream, &max_dev)?; + stream.synchronize()?; + let widths: Vec = maxes.into_iter().map(width_of).collect(); + let mut offsets = Vec::with_capacity(cols); + let mut total = 0u64; + for &w in &widths { + offsets.push(total); + total += rows_u64 * u64::from(w); + } + let offsets_dev = stream.clone_htod(&offsets)?; + let widths_dev = stream.clone_htod(&widths)?; + // SAFETY: the kernel writes every byte: each column's `rows × width`. + let mut out = unsafe { stream.alloc::(total as usize) }?; + let cells = rows * cols; + let cfg = + LaunchConfig::for_num_elems(u32::try_from(cells).expect("packed trace: cells fit u32")); + let cols_u64 = cols as u64; + unsafe { + stream + .launch_builder(&be.pack_col_major) + .arg(src) + .arg(&rows_u64) + .arg(&cols_u64) + .arg(&offsets_dev) + .arg(&widths_dev) + .arg(&mut out) + .launch(cfg)?; + } + let data = crate::readback::clone_dtoh(stream, &out)?; + stream.synchronize()?; + Ok((widths, data)) +} + +/// [`pack_col_major_on_stream`] of a host column-major trace: the device pack, +/// for the tests that compare it with the host pack. +pub fn pack_col_major_from_host( + col_major: &[u64], + rows: usize, + cols: usize, +) -> Result<(Vec, Vec)> { + let be = backend()?; + let stream = be.next_stream(); + let src = stream.clone_htod(col_major)?; + pack_col_major_on_stream(&stream, be, &src.slice(..), rows, cols) +} diff --git a/crypto/math-cuda/src/ntt.rs b/crypto/math-cuda/src/ntt.rs index f5accc3b1..c370d2633 100644 --- a/crypto/math-cuda/src/ntt.rs +++ b/crypto/math-cuda/src/ntt.rs @@ -114,13 +114,73 @@ fn ntt_inplace(input: &[u64], forward: bool) -> Result> { } } - let out = stream.clone_dtoh(&x_dev)?; + let out = crate::readback::clone_dtoh(&stream, &x_dev)?; stream.synchronize()?; Ok(out) } +/// Largest grid.y a tile launch uses before it moves the excess into grid.z. +/// +/// CUDA caps grid.y (and grid.z) at 65,535 on every compute capability, and a +/// tile grid is a power of two, so 2^15 is the largest y that fits. A tile's +/// y count is `n >> (level + k)`: the NTT's first 5-level tile needs +/// `n >> 13`, which reaches 65,536 at a 2^29 codeword — the WHIR base commit at +/// stack 27, blowup 4 — and the launch failed there, every such commit falling +/// back to the host (`thoughts/zf/gap/fix/STRUCT.md` §9.2). Every launch at or +/// below the cap is the one it always was. +const GRID_Y_CAP: u64 = 1 << 15; + +thread_local! { + /// Per-thread override of [`GRID_Y_CAP`]. See [`with_grid_y_cap`]. + static GRID_Y_CAP_OVERRIDE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// `blocks` tile blocks (a power of two) as `(grid.y, grid.z)`: all of them in +/// y up to the cap, the rest as multiples of it in z. The tile kernels read +/// their block's high index as `blockIdx.z * gridDim.y + blockIdx.y`. +pub(crate) fn split_grid_y(blocks: u64) -> (u32, u32) { + let cap = GRID_Y_CAP_OVERRIDE + .with(std::cell::Cell::get) + .unwrap_or(GRID_Y_CAP); + assert!( + blocks.is_power_of_two(), + "a tile grid is a power of two, got {blocks}" + ); + let y = blocks.min(cap); + let z = blocks / y; + assert!( + z <= 65_535, + "a tile grid of {blocks} blocks exceeds grid.y x grid.z" + ); + (y as u32, z as u32) +} + +/// Run `f` with the tile grids split at `cap` blocks in y instead of 2^15 on +/// the calling thread (the launches happen there), so a test can drive the +/// grid.z path at a size that fits a test. `cap` must be a power of two. +#[doc(hidden)] +pub fn with_grid_y_cap(cap: u64, f: impl FnOnce() -> R) -> R { + assert!(cap.is_power_of_two(), "the y cap is a power of two"); + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + GRID_Y_CAP_OVERRIDE.with(|c| c.set(self.0)); + } + } + let _restore = Restore(GRID_Y_CAP_OVERRIDE.with(|c| c.replace(Some(cap)))); + f() +} + /// Run the butterfly body of a bit-reversed-input DIT NTT. Split out so the /// LDE orchestrator can reuse it on the same device buffer. +/// Columns a fused tile spans — one warp, which is what keeps it coalesced. +/// Mirrors `NTT_TILE_COLS` in `kernels/ntt.cu`. +const TILE_COLS: u32 = 32; + +/// Levels a fused tile takes at once. `32 × 32` threads is a full block and +/// `32 × 33 × 8` bytes of shared memory, which is room to spare. +const TILE_LEVELS: u64 = 5; + pub(crate) fn run_ntt_body( stream: &cudarc::driver::CudaStream, x_dev: &mut cudarc::driver::CudaSlice, @@ -168,11 +228,41 @@ pub(crate) fn run_ntt_body( } } - // Levels 8..log_n: per-level kernels. Loads are fully coalesced in the - // per-level path; switching to fused-with-row-remap at base_step>0 tanks - // DRAM throughput enough to wipe out the launch savings. + // Levels 8..log_n, five at a time through a 2D tile: `threadIdx.x` walks + // the contiguous low bits and `threadIdx.y` the stride the butterflies + // move, so both the load and the store stay coalesced. The older + // fused-with-row-remap path gathered along the stride instead and lost + // more bandwidth than it saved in passes — that is why the per-level + // fallback is still here for what does not tile. let half_cfg = LaunchConfig::for_num_elems((n / 2) as u32); - for level in fused..log_n { + let mut level = fused; + while level < log_n { + let k = core::cmp::min(TILE_LEVELS, log_n - level); + // A tile needs a warp of contiguous indices below it, and is only + // worth its idle half when it fuses more than a level. + if k >= 2 && (1u64 << level) >= TILE_COLS as u64 { + let rows = 1u32 << k; + let (grid_y, grid_z) = split_grid_y(n >> (level + k)); + let cfg = LaunchConfig { + grid_dim: (((1u64 << level) / TILE_COLS as u64) as u32, grid_y, grid_z), + block_dim: (TILE_COLS, rows, 1), + shared_mem_bytes: rows * (TILE_COLS + 1) * 8, + }; + let k_u32 = k as u32; + unsafe { + stream + .launch_builder(&be.ntt_dit_tile) + .arg(&mut *x_dev) + .arg(tw_dev) + .arg(&n) + .arg(&log_n) + .arg(&level) + .arg(&k_u32) + .launch(cfg)?; + } + level += k; + continue; + } unsafe { stream .launch_builder(&be.ntt_dit_level) @@ -183,6 +273,7 @@ pub(crate) fn run_ntt_body( .arg(&level) .launch(half_cfg)?; } + level += 1; } Ok(()) } @@ -210,7 +301,51 @@ pub fn pointwise_mul(x: &[u64], w: &[u64]) -> Result> { .launch(LaunchConfig::for_num_elems(n as u32))?; } - let out = stream.clone_dtoh(&x_dev)?; + let out = crate::readback::clone_dtoh(&stream, &x_dev)?; stream.synchronize()?; Ok(out) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tile_grids_split_into_z_only_past_the_cap() { + // Up to 2^30 blocks fit (2^15 in y times 2^15 in z); the largest tile + // grid a Goldilocks transform can ask for is 2^32 >> 13 = 2^19. + for log in 0..=30u32 { + let blocks = 1u64 << log; + let (y, z) = split_grid_y(blocks); + assert_eq!( + u64::from(y) * u64::from(z), + blocks, + "2^{log}: every block launched" + ); + assert!( + y <= 65_535 && z <= 65_535, + "2^{log}: ({y}, {z}) past the device limits" + ); + if blocks <= GRID_Y_CAP { + assert_eq!( + (y, z), + (blocks as u32, 1), + "2^{log}: the launch it always was" + ); + } + } + // The WHIR base codewords at stacks 27 and 28 (blowup 4): the first + // 5-level tile's n >> 13. + assert_eq!(split_grid_y((1 << 29) >> 13), (1 << 15, 2)); + assert_eq!(split_grid_y((1 << 30) >> 13), (1 << 15, 4)); + // A test cap moves the split down, and restores on return. + assert_eq!(with_grid_y_cap(2, || split_grid_y(64)), (2, 32)); + assert_eq!(split_grid_y(64), (64, 1)); + } + + #[test] + #[should_panic(expected = "exceeds grid.y x grid.z")] + fn a_tile_grid_past_both_dimensions_is_refused_not_truncated() { + split_grid_y(1 << 31); + } +} diff --git a/crypto/math-cuda/src/p1_stark.rs b/crypto/math-cuda/src/p1_stark.rs new file mode 100644 index 000000000..657497d94 --- /dev/null +++ b/crypto/math-cuda/src/p1_stark.rs @@ -0,0 +1,641 @@ +//! ZisK's Poseidon1 on the STARK's device commit paths: the +//! [`crate::DeviceHash::Poseidon1`] arm of every leaf, tree, path and grind +//! dispatch (`p1/*` exploration branch; nothing reaches it unless a +//! configuration names `CommitmentHash::Poseidon1`). +//! +//! The kernels are the production section of `kernels/p1w16.cu` (`p1s_*`), +//! loaded on first use from that cubin, never by +//! [`crate::device::Backend::init`], so a process that commits under another +//! hash never loads them. +//! +//! # Nodes and trees +//! +//! A node is the host's commitment bytes (four canonical felts, each eight +//! big-endian bytes), as under [`crate::rpx`]. A tree is 4-ary in the host's +//! arity-4 layout (`crypto::merkle_tree::utils::level_offsets4`): levels +//! top-down, root at node 0, leaves last, each level `⌈below / 4⌉` nodes, a +//! short group padded with the zero digest when its parent is hashed. So a +//! device node buffer is what `MerkleTree::::from_precomputed_nodes` +//! takes, and a path is three siblings per level ([`gather_paths_dev`]). +//! +//! # Leaves +//! +//! The STARK's leaf hash (`poseidon1_stark::linear_hash`: ZisK's chain with the +//! width tag `[len, LEAF_DOMAIN, 0, 0]` as the first capacity) over the felt +//! sequence the host leaf hashes, in the read patterns of [`crate::rpx`]'s +//! kernels. + +use std::sync::{Arc, OnceLock}; + +use cudarc::driver::{ + CudaFunction, CudaModule, CudaSlice, CudaStream, CudaViewMut, LaunchConfig, PushKernelArg, sys, +}; +use cudarc::nvrtc::Ptx; + +use crate::Result; +use crate::device::backend; + +const P1W16_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/p1w16.cubin")); + +/// Children per node. +pub const ARITY: usize = 4; + +/// Threads per block for the leaf and level kernels (a 16-lane state: the +/// RPX kernels' 128). +const BLOCK_DIM: u32 = 128; + +/// Threads of the one-block tail ([`TAIL_MAX_PARENTS`] parents a pass); at +/// ~140 registers a thread, 1,024 would not fit one multiprocessor. +const TAIL_BLOCK_DIM: u32 = 256; + +/// The widest level the one-block tail takes over: below it a per-level +/// launch costs more than the level's work. +pub const TAIL_MAX_PARENTS: u64 = TAIL_BLOCK_DIM as u64; + +/// The production kernels. +pub struct Kernels { + _module: Arc, + pub leaves_cols_row: CudaFunction, + pub leaves_cols_pair: CudaFunction, + pub leaves_rm_pair: CudaFunction, + pub leaves_rm_row: CudaFunction, + pub fri_group_leaves: CudaFunction, + /// WHIR's coset leaves over a base / ext3 codeword (`crate::whir`). + pub leaves_base_coset: CudaFunction, + /// [`Self::leaves_base_coset`] with leaf `j` placed at `bitrev(j)`. + pub leaves_base_coset_rev: CudaFunction, + pub leaves_ext3_coset: CudaFunction, + pub merkle_level4: CudaFunction, + pub merkle_tail4: CudaFunction, + pub gather_paths4: CudaFunction, + pub grind_w8: CudaFunction, +} + +/// Whether the Poseidon1 kernels run the lean permutation +/// (`LAMBDA_VM_P1_LEAN`; on by default, `=0` turns it off): the `_lean` twins +/// of the WHIR coset leaves, the column leaves and the 4-ary levels load in +/// their twins' places, with the same bytes in the same slots. Read once, when +/// the kernels first load. +pub fn lean() -> bool { + static LEAN: OnceLock = OnceLock::new(); + *LEAN.get_or_init(|| lean_setting(std::env::var("LAMBDA_VM_P1_LEAN").ok().as_deref())) +} + +/// `LAMBDA_VM_P1_LEAN`'s value read: empty, `0` or `off` load the plain +/// kernels; anything else (unset included) the lean twins. +pub fn lean_setting(value: Option<&str>) -> bool { + !matches!(value.map(str::trim), Some("" | "0" | "off")) +} + +/// Load the kernels once, under [`lean`]. +pub fn kernels() -> Result<&'static Kernels> { + static KERNELS: OnceLock = OnceLock::new(); + if let Some(k) = KERNELS.get() { + return Ok(k); + } + let lean = lean(); + let k = load_kernels(lean)?; + if lean { + eprintln!( + "[gpu] Poseidon1 kernels: the lean twins (the default; LAMBDA_VM_P1_LEAN=0 loads the plain ones)" + ); + } + Ok(KERNELS.get_or_init(|| k)) +} + +/// The kernel set, loaded afresh: the lean twins where they exist when `lean`, +/// today's kernels otherwise. [`kernels`] caches one; the parity tests hold +/// both. +pub fn load_kernels(lean: bool) -> Result { + let be = backend()?; + let module = be.ctx.load_module(Ptx::from_binary(P1W16_CUBIN.to_vec()))?; + let twin = |name: &str| { + if lean { + format!("{name}_lean") + } else { + name.to_string() + } + }; + Ok(Kernels { + leaves_cols_row: module.load_function(&twin("p1s_leaves_cols_row"))?, + leaves_cols_pair: module.load_function(&twin("p1s_leaves_cols_pair"))?, + leaves_rm_pair: module.load_function("p1s_leaves_rm_pair")?, + leaves_rm_row: module.load_function("p1s_leaves_rm_row")?, + fri_group_leaves: module.load_function("p1s_fri_group_leaves")?, + leaves_base_coset: module.load_function(&twin("p1s_leaves_base_coset"))?, + leaves_base_coset_rev: module.load_function(&twin("p1s_leaves_base_coset_rev"))?, + leaves_ext3_coset: module.load_function(&twin("p1s_leaves_ext3_coset"))?, + merkle_level4: module.load_function(&twin("p1s_merkle_level4"))?, + merkle_tail4: module.load_function(&twin("p1s_merkle_tail4"))?, + gather_paths4: module.load_function("p1s_gather_paths4")?, + grind_w8: module.load_function("p1s_grind_w8")?, + _module: module, + }) +} + +#[cfg(test)] +mod lean_setting_tests { + use super::lean_setting; + + /// The lean twins unless the knob says `0`, `off` or nothing. + #[test] + fn the_lean_kernels_load_unless_zero() { + assert!(lean_setting(None), "the default is the lean twins"); + assert!(lean_setting(Some("1"))); + assert!(lean_setting(Some("on"))); + assert!(!lean_setting(Some("0"))); + assert!(!lean_setting(Some(" 0 "))); + assert!(!lean_setting(Some("off"))); + assert!(!lean_setting(Some(""))); + } +} + +pub(crate) fn launch_cfg(threads: u64) -> LaunchConfig { + debug_assert!(threads <= u32::MAX as u64, "p1 launch: {threads} threads"); + LaunchConfig { + grid_dim: ((threads as u32).div_ceil(BLOCK_DIM).max(1), 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + } +} + +// =========================================================================== +// The arity-4 layout +// =========================================================================== + +/// Level sizes over `leaves ≥ 1` leaves, leaves first, root (1) last +/// (`crypto::merkle_tree::utils::level_sizes4`). +pub fn level_sizes(leaves: usize) -> Vec { + let mut sizes = vec![leaves]; + while let Some(&n) = sizes.last().filter(|&&n| n > 1) { + sizes.push(n.div_ceil(ARITY)); + } + sizes +} + +/// Stored nodes of a tree over `leaves` leaves. +pub fn tree_nodes(leaves: usize) -> usize { + level_sizes(leaves).iter().sum() +} + +/// Levels above the leaves: the number of three-sibling groups on a path. +pub fn depth(leaves: usize) -> usize { + level_sizes(leaves).len() - 1 +} + +/// How many nodes the top `levels` levels hold (the root's level first): the +/// node-buffer prefix those levels occupy. +pub fn top_levels_nodes(leaves: usize, levels: usize) -> usize { + level_sizes(leaves).iter().rev().take(levels).sum() +} + +/// Where each level of [`level_sizes`] starts in the node buffer, which holds +/// the levels top-down (`crypto::merkle_tree::utils::level_offsets4`). +pub fn level_offsets(sizes: &[usize]) -> Vec { + let mut offsets = vec![0; sizes.len()]; + for j in (0..sizes.len().saturating_sub(1)).rev() { + offsets[j] = offsets[j + 1] + sizes[j + 1]; + } + offsets +} + +// =========================================================================== +// Leaves +// =========================================================================== + +/// Leaves of a column-major matrix (`num_cols` columns of `num_rows` rows at +/// `cols`, column stride `col_stride`): one bit-reversed row per leaf +/// (`rows_per_leaf = 1`) or a row pair (`2`), into `leaves`. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_leaves_cols_ptr( + stream: &CudaStream, + cols: sys::CUdeviceptr, + col_stride: u64, + num_cols: u64, + num_rows: u64, + rows_per_leaf: usize, + leaves: sys::CUdeviceptr, +) -> Result<()> { + ensure!(rows_per_leaf == 1 || rows_per_leaf == 2); + ensure!(num_rows >= 2 && num_rows.is_power_of_two()); + let k = kernels()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let (kernel, threads) = if rows_per_leaf == 2 { + (&k.leaves_cols_pair, num_rows / 2) + } else { + (&k.leaves_cols_row, num_rows) + }; + unsafe { + stream + .launch_builder(kernel) + .arg(&cols) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(&leaves) + .launch(launch_cfg(threads))?; + } + Ok(()) +} + +/// [`launch_leaves_cols_ptr`] over a buffer and a leaf view. +pub(crate) fn launch_leaves_cols( + stream: &CudaStream, + cols: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + rows_per_leaf: usize, + leaves: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + ensure!(rows_per_leaf == 1 || rows_per_leaf == 2); + ensure!(num_rows >= 2 && num_rows.is_power_of_two()); + let k = kernels()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let (kernel, threads) = if rows_per_leaf == 2 { + (&k.leaves_cols_pair, num_rows / 2) + } else { + (&k.leaves_cols_row, num_rows) + }; + unsafe { + stream + .launch_builder(kernel) + .arg(cols) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves) + .launch(launch_cfg(threads))?; + } + Ok(()) +} + +/// Leaves of a row-major matrix (`num_rows` rows of stride `m`), columns +/// `[col_start, col_end)`, `rows_per_leaf` bit-reversed rows a leaf. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_leaves_row_major( + stream: &CudaStream, + buf: &CudaSlice, + m: u64, + col_start: u64, + col_end: u64, + num_rows: u64, + rows_per_leaf: usize, + leaves: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + ensure!(rows_per_leaf == 1 || rows_per_leaf == 2); + ensure!(num_rows >= 2 && num_rows.is_power_of_two()); + ensure!( + col_start < col_end && col_end <= m, + "column range in bounds" + ); + let k = kernels()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let (kernel, threads) = if rows_per_leaf == 2 { + (&k.leaves_rm_pair, num_rows / 2) + } else { + (&k.leaves_rm_row, num_rows) + }; + unsafe { + stream + .launch_builder(kernel) + .arg(buf) + .arg(&m) + .arg(&col_start) + .arg(&col_end) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves) + .launch(launch_cfg(threads))?; + } + Ok(()) +} + +/// FRI leaves of `group` consecutive ext3 values each from an interleaved eval +/// vector (`group = 2`: the pair leaf). +pub(crate) fn launch_fri_group_leaves( + stream: &CudaStream, + evals: &CudaSlice, + num_leaves: u64, + group: u64, + leaves: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + let k = kernels()?; + unsafe { + stream + .launch_builder(&k.fri_group_leaves) + .arg(evals) + .arg(&num_leaves) + .arg(&group) + .arg(leaves) + .launch(launch_cfg(num_leaves))?; + } + Ok(()) +} + +// =========================================================================== +// Trees +// =========================================================================== + +/// Fill every inner level of a tree whose `leaves_len` leaf digests sit at +/// the tail of `nodes_dev` (a [`tree_nodes`]`(leaves_len)`-node buffer): +/// one launch per level while a level has more than [`TAIL_MAX_PARENTS`] +/// parents, then one block for the rest. +pub fn build_inner_tree_levels( + stream: &CudaStream, + nodes_dev: &mut CudaSlice, + leaves_len: usize, +) -> Result<()> { + let k = kernels()?; + let total = tree_nodes(leaves_len) as u64; + ensure!( + nodes_dev.len() as u64 >= total * 32, + "p1 tree: a node buffer of {} bytes holds no {leaves_len}-leaf tree", + nodes_dev.len() + ); + let mut child_off = total - leaves_len as u64; + let mut n_children = leaves_len as u64; + while n_children > 1 { + let n_parents = n_children.div_ceil(ARITY as u64); + if n_parents <= TAIL_MAX_PARENTS { + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (TAIL_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&k.merkle_tail4) + .arg(&mut *nodes_dev) + .arg(&child_off) + .arg(&n_children) + .launch(cfg)?; + } + return Ok(()); + } + let parent_off = child_off - n_parents; + unsafe { + stream + .launch_builder(&k.merkle_level4) + .arg(&mut *nodes_dev) + .arg(&child_off) + .arg(&n_children) + .arg(&parent_off) + .arg(&n_parents) + .launch(launch_cfg(n_parents))?; + } + child_off = parent_off; + n_children = n_parents; + } + Ok(()) +} + +/// Parity harness: the full tree over `hashed_leaves` (32 bytes each), as the +/// host node buffer. +pub fn build_merkle_tree_on_device(hashed_leaves: &[u8]) -> Result> { + ensure!(hashed_leaves.len().is_multiple_of(32)); + let leaves_len = hashed_leaves.len() / 32; + ensure!(leaves_len >= 1); + let total = tree_nodes(leaves_len); + let be = backend()?; + let stream = be.next_stream(); + // SAFETY: the leaves are written by the copy below, every inner node by + // the level walk, before anything reads them. + let mut nodes_dev = unsafe { stream.alloc::(total * 32) }?; + { + let off = (total - leaves_len) * 32; + let mut slice = nodes_dev.slice_mut(off..off + hashed_leaves.len()); + stream.memcpy_htod(hashed_leaves, &mut slice)?; + } + build_inner_tree_levels(stream.as_ref(), &mut nodes_dev, leaves_len)?; + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// The composition tree straight from a resident slab buffer (`3·m` slabs of +/// `lde_size`, component `k` of part `c` at `(c·3 + k)·lde_size`), +/// `rows_per_leaf` rows a leaf. Twin of +/// [`crate::rpx::build_comp_poly_tree_from_slabs_dev_rpl`]. +pub fn build_comp_poly_tree_from_slabs_dev_rpl( + stream: &Arc, + buf: &CudaSlice, + m: usize, + lde_size: usize, + rows_per_leaf: usize, +) -> Result { + ensure!(rows_per_leaf == 1 || rows_per_leaf == 2); + #[cfg(feature = "test-faults")] + crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; + ensure!(m > 0); + ensure!(lde_size.is_power_of_two() && lde_size >= 2); + ensure_eq!(buf.len(), 3 * m * lde_size, "slab buffer shape"); + let num_leaves = lde_size / rows_per_leaf; + let total = tree_nodes(num_leaves); + // SAFETY: the leaf kernel writes the leaves, the level walk the rest. + let mut nodes_dev = unsafe { stream.alloc::(total * 32) }?; + { + let off = (total - num_leaves) * 32; + let mut leaves = nodes_dev.slice_mut(off..off + num_leaves * 32); + launch_leaves_cols( + stream.as_ref(), + buf, + lde_size as u64, + 3 * m as u64, + lde_size as u64, + rows_per_leaf, + &mut leaves, + )?; + } + build_inner_tree_levels(stream.as_ref(), &mut nodes_dev, num_leaves)?; + let mut root = [0u8; 32]; + crate::readback::memcpy_dtoh(stream, &nodes_dev.slice(0..32), &mut root)?; + stream.synchronize()?; + Ok(crate::lde::GpuMerkleTree { + nodes: Arc::new(nodes_dev), + leaves_len: num_leaves, + arity: ARITY, + root, + }) +} + +/// The composition tree from host interleaved ext3 parts, kept resident. Twin +/// of [`crate::rpx::build_comp_poly_tree_from_evals_ext3_keep_rpl`]. +pub fn build_comp_poly_tree_from_evals_ext3_keep_rpl( + parts_interleaved: &[&[u64]], + rows_per_leaf: usize, +) -> Result { + #[cfg(feature = "test-faults")] + crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; + ensure!(!parts_interleaved.is_empty()); + let m = parts_interleaved.len(); + let lde_size = parts_interleaved[0].len() / 3; + for p in parts_interleaved { + ensure_eq!( + p.len(), + 3 * lde_size, + "ext3 buffer length must be 3 * lde_size" + ); + } + ensure!(lde_size.is_power_of_two() && lde_size >= 2); + let be = backend()?; + let stream = be.next_stream(); + let staging_slot = be.pinned_staging(); + let mb = 3 * m; + let mut staging = staging_slot.lock().unwrap(); + staging.ensure_capacity(mb * lde_size, &be.ctx)?; + let pinned = unsafe { staging.as_mut_slice(mb * lde_size) }; + crate::lde::pack_ext3_to_pinned_slabs(parts_interleaved, pinned, lde_size); + let mut buf = stream.alloc_zeros::(mb * lde_size)?; + stream.memcpy_htod(&pinned[..mb * lde_size], &mut buf)?; + stream.synchronize()?; + drop(staging); + build_comp_poly_tree_from_slabs_dev_rpl(&stream, &buf, m, lde_size, rows_per_leaf) +} + +/// Parity harness: a FRI layer tree over an interleaved ext3 eval vector, +/// `group` values a leaf (2 = the pair layer), as the host node buffer. +pub fn build_fri_tree_from_evals_ext3(evals: &[u64], group: usize) -> Result> { + ensure!(evals.len().is_multiple_of(3 * group)); + let num_leaves = evals.len() / (3 * group); + ensure!(num_leaves >= 1); + let total = tree_nodes(num_leaves); + let be = backend()?; + let stream = be.next_stream(); + let evals_dev = stream.clone_htod(evals)?; + // SAFETY: as in `build_merkle_tree_on_device`. + let mut nodes_dev = unsafe { stream.alloc::(total * 32) }?; + { + let off = (total - num_leaves) * 32; + let mut leaves = nodes_dev.slice_mut(off..off + num_leaves * 32); + launch_fri_group_leaves( + stream.as_ref(), + &evals_dev, + num_leaves as u64, + group as u64, + &mut leaves, + )?; + } + build_inner_tree_levels(stream.as_ref(), &mut nodes_dev, num_leaves)?; + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Parity harness: column-major leaves over a host matrix (`num_cols` columns +/// of `num_rows`, column-major), `rows_per_leaf` rows a leaf. +pub fn leaves_cols( + columns: &[u64], + num_cols: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + ensure_eq!(columns.len(), num_cols * num_rows); + let be = backend()?; + let stream = be.next_stream(); + let cols_dev = stream.clone_htod(columns)?; + let mut out = stream.alloc_zeros::(num_rows / rows_per_leaf * 32)?; + launch_leaves_cols( + stream.as_ref(), + &cols_dev, + num_rows as u64, + num_cols as u64, + num_rows as u64, + rows_per_leaf, + &mut out.as_view_mut(), + )?; + let host = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(host) +} + +// =========================================================================== +// Paths and grinding +// =========================================================================== + +/// Authentication paths for `positions` against a resident tree of +/// `leaves_len` leaves: `positions.len() · 3 · depth · 32` bytes, query `q`'s +/// path at `q · 3 · depth · 32`, the siblings of each level from the leaves +/// up in child order — the host `get_proof_by_pos` at arity 4. +pub fn gather_paths_dev( + nodes_dev: &CudaSlice, + leaves_len: usize, + positions: &[u32], + stream: &Arc, +) -> Result> { + let nq = positions.len(); + if nq == 0 { + return Ok(Vec::new()); + } + ensure!( + positions.iter().all(|&p| (p as usize) < leaves_len), + "p1 gather_paths_dev: leaf position >= leaves_len" + ); + let total = tree_nodes(leaves_len); + ensure!( + nodes_dev.len() >= total * 32, + "p1 gather: node buffer too short" + ); + let depth4 = depth(leaves_len); + let path_bytes = 3 * depth4 * 32; + if path_bytes == 0 { + return Ok(Vec::new()); + } + let k = kernels()?; + let be = backend()?; + let pos_dev = stream.clone_htod(positions)?; + // SAFETY: the kernel writes every (query, level, sibling) slot. + let mut out = unsafe { stream.alloc::(nq * path_bytes) }?; + let nq_u32 = nq as u32; + let leaves_u64 = leaves_len as u64; + let total_u64 = total as u64; + let depth_u32 = depth4 as u32; + unsafe { + stream + .launch_builder(&k.gather_paths4) + .arg(nodes_dev) + .arg(&pos_dev) + .arg(&nq_u32) + .arg(&leaves_u64) + .arg(&total_u64) + .arg(&depth_u32) + .arg(&mut out) + .launch(launch_cfg(nq as u64))?; + } + let pending = + crate::device::async_dtoh_via(stream, be.pinned_hashes(), &be.ctx, &out, out.len())?; + let mut host = vec![0u8; out.len()]; + pending.wait_into_bytes(&mut host)?; + Ok(host) +} + +/// The grind kernel and its block size, for [`crate::grinding`]'s range walk. +pub(crate) fn grind_kernel() -> Result<(&'static CudaFunction, u32)> { + Ok((&kernels()?.grind_w8, BLOCK_DIM)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The layout helpers against the host's (`level_sizes4` / + /// `level_offsets4`) on a few shapes, written out. + #[test] + fn the_arity4_layout_counts_real_nodes_only() { + assert_eq!(level_sizes(1), vec![1]); + assert_eq!(level_sizes(2), vec![2, 1]); + assert_eq!(level_sizes(8), vec![8, 2, 1]); + assert_eq!(level_sizes(32), vec![32, 8, 2, 1]); + assert_eq!(level_sizes(64), vec![64, 16, 4, 1]); + assert_eq!(tree_nodes(64), 85); + assert_eq!(tree_nodes(32), 43); + assert_eq!(depth(1 << 21), 11); + assert_eq!(depth(1 << 22), 11); + assert_eq!(depth(2), 1); + // The top two levels of a 64-leaf tree: the root and its four children. + assert_eq!(top_levels_nodes(64, 2), 5); + assert_eq!(top_levels_nodes(64, 9), 85); + } +} diff --git a/crypto/math-cuda/src/p1w16.rs b/crypto/math-cuda/src/p1w16.rs new file mode 100644 index 000000000..1de0569c3 --- /dev/null +++ b/crypto/math-cuda/src/p1w16.rs @@ -0,0 +1,768 @@ +//! ⛔ D-HASH stage 1 (diagnostic; on NO proving path): the Poseidon1 +//! width-16 measurement kernels (`kernels/p1w16.cu`), their parity entry +//! points, and the microbenchmark against the RPX kernels. +//! +//! The module is loaded on first use from its own cubin, never by +//! [`crate::device::Backend::init`], so a proving process never loads it. +//! Digests are four canonical `u64`s per node (a measurement format). + +use std::sync::OnceLock; +use std::time::Instant; + +use cudarc::driver::{CudaFunction, CudaModule, LaunchConfig, PushKernelArg}; +use cudarc::nvrtc::Ptx; + +use crate::Result; +use crate::device::backend; + +const P1W16_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/p1w16.cubin")); + +/// Threads per block, as RPX's kernels. +const BLOCK_DIM: u32 = 128; + +/// Felts per permutation state, rate, digest. +pub const WIDTH: usize = 16; +pub const RATE: usize = 12; +pub const DIGEST: usize = 4; + +/// The kernel variants the cubin instantiates, in `p1w16.cu`'s order: the two +/// multiply variants and the Fourier-domain partial rounds of the circulant +/// instance, then `c1`, the Grain Cauchy MDS alternative, then `l2`, the +/// Fourier form with leaner addition chains (the same permutation as `v2`). +pub const VARIANTS: [&str; 5] = ["v0", "v1", "v2", "c1", "l2"]; + +/// Is variant `v` the Cauchy alternative (a different permutation, checked +/// against `permute_cauchy` on the host)? +pub fn is_cauchy(v: usize) -> bool { + VARIANTS[v].starts_with('c') +} + +/// One variant's kernels. +pub struct Kernels { + pub permute_probe: CudaFunction, + pub leaves_base_coset: CudaFunction, + pub leaves_ext3_coset: CudaFunction, + pub merkle_level4: CudaFunction, + pub grind_search: CudaFunction, +} + +/// ZisK's instance (stage P1 of the Poseidon1 base STARK): its leaf hash over +/// the coset and LDE-row geometries (Fourier-domain permutation), and the +/// width-8 permutation and grind at [`W8_VARIANTS`]. +pub struct Zisk { + pub leaves_base_coset: CudaFunction, + pub leaves_ext3_coset: CudaFunction, + pub leaves_rows: CudaFunction, + pub leaves_row_pair: CudaFunction, + pub w8_permute_probe: Vec, + pub w8_grind_search: Vec, +} + +/// The width-8 variants: textbook rounds, then the Fourier-domain partial rounds. +pub const W8_VARIANTS: [&str; 2] = ["v1", "v2"]; + +/// The loaded module: every variant, ZisK's instance and the bench fill. +pub struct Module { + _module: std::sync::Arc, + pub variants: Vec, + pub zisk: Zisk, + pub fill: CudaFunction, +} + +/// Load the module once. +pub fn module() -> Result<&'static Module> { + static MODULE: OnceLock = OnceLock::new(); + if let Some(m) = MODULE.get() { + return Ok(m); + } + let be = backend()?; + let module = be.ctx.load_module(Ptx::from_binary(P1W16_CUBIN.to_vec()))?; + let kernels = |v: &str| -> Result { + Ok(Kernels { + permute_probe: module.load_function(&format!("p1w16_permute_probe_{v}"))?, + leaves_base_coset: module.load_function(&format!("p1w16_leaves_base_coset_{v}"))?, + leaves_ext3_coset: module.load_function(&format!("p1w16_leaves_ext3_coset_{v}"))?, + merkle_level4: module.load_function(&format!("p1w16_merkle_level4_{v}"))?, + grind_search: module.load_function(&format!("p1w16_grind_search_{v}"))?, + }) + }; + let zisk = Zisk { + leaves_base_coset: module.load_function("p1w16_zleaves_base_coset_v2")?, + leaves_ext3_coset: module.load_function("p1w16_zleaves_ext3_coset_v2")?, + leaves_rows: module.load_function("p1w16_zleaves_rows_v2")?, + leaves_row_pair: module.load_function("p1w16_zleaves_row_pair_v2")?, + w8_permute_probe: W8_VARIANTS + .iter() + .map(|v| module.load_function(&format!("p1w8_permute_probe_{v}"))) + .collect::>()?, + w8_grind_search: W8_VARIANTS + .iter() + .map(|v| module.load_function(&format!("p1w8_grind_search_{v}"))) + .collect::>()?, + }; + let m = Module { + variants: VARIANTS.iter().map(|v| kernels(v)).collect::>()?, + zisk, + fill: module.load_function("p1w16_fill")?, + _module: module, + }; + Ok(MODULE.get_or_init(|| m)) +} + +fn cfg(threads: u64) -> LaunchConfig { + LaunchConfig { + grid_dim: ((threads as u32).div_ceil(BLOCK_DIM), 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + } +} + +/// `n` permutations of `states` (`n·16` felts) on device. +pub fn permute_many(variant: usize, states: &[u64]) -> Result> { + assert_eq!(states.len() % WIDTH, 0); + let n = (states.len() / WIDTH) as u64; + let k = &module()?.variants[variant]; + let stream = backend()?.next_stream(); + let input = stream.clone_htod(states)?; + let mut out = stream.alloc_zeros::(states.len())?; + unsafe { + stream + .launch_builder(&k.permute_probe) + .arg(&input) + .arg(&n) + .arg(&mut out) + .launch(cfg(n))?; + } + let host = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(host) +} + +/// Coset leaves over a base (`ext3 = false`) or ext3 codeword: `num_leaves` +/// digests of `block` elements each, element `t` of leaf `j` at `j + t·num_leaves`. +pub fn leaves_coset(variant: usize, codeword: &[u64], block: u64, ext3: bool) -> Result> { + let per = if ext3 { 3 } else { 1 }; + let num_leaves = codeword.len() as u64 / (block * per); + let k = &module()?.variants[variant]; + let stream = backend()?.next_stream(); + let input = stream.clone_htod(codeword)?; + let mut out = stream.alloc_zeros::(num_leaves as usize * DIGEST)?; + let f = if ext3 { + &k.leaves_ext3_coset + } else { + &k.leaves_base_coset + }; + unsafe { + stream + .launch_builder(f) + .arg(&input) + .arg(&num_leaves) + .arg(&block) + .arg(&mut out) + .launch(cfg(num_leaves))?; + } + let host = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(host) +} + +/// One 4-ary level: `children.len() / 16` parents. +pub fn merkle_level4(variant: usize, children: &[u64]) -> Result> { + assert_eq!(children.len() % WIDTH, 0); + let n = (children.len() / WIDTH) as u64; + let k = &module()?.variants[variant]; + let stream = backend()?.next_stream(); + let input = stream.clone_htod(children)?; + let mut out = stream.alloc_zeros::(n as usize * DIGEST)?; + unsafe { + stream + .launch_builder(&k.merkle_level4) + .arg(&input) + .arg(&mut out) + .arg(&n) + .launch(cfg(n))?; + } + let host = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(host) +} + +/// The smallest nonce in `[0, count)` whose grind head is `< limit`, if any. +pub fn grind( + variant: usize, + inner: &[u64; 4], + limit: u64, + count: u64, + grid: u32, +) -> Result> { + let k = &module()?.variants[variant]; + let stream = backend()?.next_stream(); + let inner_dev = stream.clone_htod(inner.as_slice())?; + let mut result = stream.clone_htod(&[u64::MAX])?; + let base = 0u64; + unsafe { + stream + .launch_builder(&k.grind_search) + .arg(&inner_dev) + .arg(&limit) + .arg(&base) + .arg(&count) + .arg(&mut result) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + let host = crate::readback::clone_dtoh(&stream, &result)?; + stream.synchronize()?; + Ok((host[0] != u64::MAX).then_some(host[0])) +} + +/// Median of `reps` timings of `f` (each ends in a stream synchronize), in ns. +fn median_ns(reps: u32, mut f: impl FnMut() -> Result<()>) -> Result { + f()?; // warm-up (module load, clocks) + let mut t = Vec::with_capacity(reps as usize); + for _ in 0..reps { + let start = Instant::now(); + f()?; + t.push(start.elapsed().as_nanos() as f64); + } + t.sort_by(|a, b| a.partial_cmp(b).expect("finite")); + Ok(t[t.len() / 2]) +} + +/// ★ The stage-1 microbenchmark: one `2^log_len`-element base codeword hashed +/// as 64-element WHIR fold cosets, then its tree, then a grind scan, under RPX +/// (the production kernels and walk) and under both Poseidon1 variants. Every +/// line is `BENCH ns/perm ( perms, ms)`. +pub fn microbench(log_len: u32, reps: u32, grind_count: u64) -> Result> { + let be = backend()?; + let m = module()?; + let stream = be.next_stream(); + let len = 1u64 << log_len; + let block = 64u64; + let num_leaves = len / block; + let mut lines = Vec::new(); + let mut line = |what: &str, hash: &str, ns: f64, perms: u64| { + let s = format!( + "BENCH {what:<5} {hash:<9} {:>7.3} ns/perm ({perms} perms, {:.2} ms)", + ns / perms as f64, + ns / 1e6 + ); + println!("{s}"); + lines.push(s); + }; + + let mut codeword = stream.alloc_zeros::(len as usize)?; + unsafe { + stream + .launch_builder(&m.fill) + .arg(&mut codeword) + .arg(&len) + .launch(cfg(len))?; + } + stream.synchronize()?; + + // RPX: the production coset leaves into a tree buffer, then the production walk. + let total_nodes = 2 * num_leaves as usize - 1; + let mut nodes = stream.alloc_zeros::(total_nodes * 32)?; + let leaves_off = (num_leaves as usize - 1) * 32; + let ns = median_ns(reps, || { + let mut leaves = nodes.slice_mut(leaves_off..leaves_off + num_leaves as usize * 32); + unsafe { + stream + .launch_builder(&be.rpx_leaves_base_coset) + .arg(&codeword) + .arg(&num_leaves) + .arg(&block) + .arg(&mut leaves) + .launch(crate::merkle::keccak_launch_cfg(num_leaves))?; + } + stream.synchronize() + })?; + line("leaf", "rpx", ns, num_leaves * block.div_ceil(8)); + let ns = median_ns(reps, || { + crate::rpx::build_inner_tree_levels(&stream, be, &mut nodes, num_leaves as usize)?; + stream.synchronize() + })?; + line("node", "rpx", ns, num_leaves - 1); + drop(nodes); + + // Poseidon1 W16, both variants: the coset leaves, then 4-ary levels down to + // fewer than four nodes. + for (v, tag) in VARIANTS.iter().enumerate() { + let k = &m.variants[v]; + let name = format!("p1w16-{tag}"); + let mut leaves = stream.alloc_zeros::(num_leaves as usize * DIGEST)?; + let ns = median_ns(reps, || { + unsafe { + stream + .launch_builder(&k.leaves_base_coset) + .arg(&codeword) + .arg(&num_leaves) + .arg(&block) + .arg(&mut leaves) + .launch(cfg(num_leaves))?; + } + stream.synchronize() + })?; + line("leaf", &name, ns, num_leaves * block.div_ceil(RATE as u64)); + + let mut sizes = Vec::new(); + let mut n = num_leaves; + while n >= 4 { + n /= 4; + sizes.push(n); + } + let mut levels: Vec<_> = sizes + .iter() + .map(|&s| stream.alloc_zeros::(s as usize * DIGEST)) + .collect::>()?; + let ns = median_ns(reps, || { + for (i, &n_parents) in sizes.iter().enumerate() { + let (done, rest) = levels.split_at_mut(i); + let parents = &mut rest[0]; + let children = if i == 0 { &leaves } else { &done[i - 1] }; + unsafe { + stream + .launch_builder(&k.merkle_level4) + .arg(children) + .arg(parents) + .arg(&n_parents) + .launch(cfg(n_parents))?; + } + } + stream.synchronize() + })?; + line("node4", &name, ns, sizes.iter().sum()); + drop(levels); + drop(leaves); + } + + // Grind: a full scan (limit 0 never hits) of `grind_count` nonces, the + // simple grid-stride kernel of each hash at two grids. + let inner = [1u64, 2, 3, 4]; + let inner_dev = stream.clone_htod(inner.as_slice())?; + let limit = 0u64; + let base = 0u64; + for grid in [ + crate::grinding::GRID_DEFAULT, + 4 * crate::grinding::GRID_DEFAULT, + ] { + let launch = LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + let mut result = stream.clone_htod(&[u64::MAX])?; + let ns = median_ns(reps, || { + unsafe { + stream + .launch_builder(&be.rpx_grind_search) + .arg(&inner_dev) + .arg(&limit) + .arg(&base) + .arg(&grind_count) + .arg(&mut result) + .launch(launch)?; + } + stream.synchronize() + })?; + line("grind", &format!("rpx-g{grid}"), ns, grind_count); + for (v, tag) in VARIANTS.iter().enumerate() { + let k = &m.variants[v]; + let ns = median_ns(reps, || { + unsafe { + stream + .launch_builder(&k.grind_search) + .arg(&inner_dev) + .arg(&limit) + .arg(&base) + .arg(&grind_count) + .arg(&mut result) + .launch(launch)?; + } + stream.synchronize() + })?; + line("grind", &format!("p1-{tag}-g{grid}"), ns, grind_count); + } + } + Ok(lines) +} + +/// ZisK's leaf hash over coset leaves of a base (`ext3 = false`) or ext3 +/// codeword, the geometry of [`leaves_coset`]. +pub fn zisk_leaves_coset(codeword: &[u64], block: u64, ext3: bool) -> Result> { + let per = if ext3 { 3 } else { 1 }; + let num_leaves = codeword.len() as u64 / (block * per); + let z = &module()?.zisk; + let stream = backend()?.next_stream(); + let input = stream.clone_htod(codeword)?; + let mut out = stream.alloc_zeros::(num_leaves as usize * DIGEST)?; + let f = if ext3 { + &z.leaves_ext3_coset + } else { + &z.leaves_base_coset + }; + unsafe { + stream + .launch_builder(f) + .arg(&input) + .arg(&num_leaves) + .arg(&block) + .arg(&mut out) + .launch(cfg(num_leaves))?; + } + let host = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(host) +} + +/// ZisK's leaf hash over a column-major LDE matrix (column `c` of row `r` at +/// `columns[c·col_stride + r]`, rows read bit-reversed): one leaf per row, or +/// per row pair when `pair`. `num_rows` is a power of two, at least 2. +pub fn zisk_leaves_rows( + columns: &[u64], + col_stride: u64, + num_cols: u64, + num_rows: u64, + pair: bool, +) -> Result> { + assert!(num_rows.is_power_of_two() && num_rows >= 2); + let z = &module()?.zisk; + let stream = backend()?.next_stream(); + let input = stream.clone_htod(columns)?; + let num_leaves = if pair { num_rows / 2 } else { num_rows }; + let mut out = stream.alloc_zeros::(num_leaves as usize * DIGEST)?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let f = if pair { + &z.leaves_row_pair + } else { + &z.leaves_rows + }; + unsafe { + stream + .launch_builder(f) + .arg(&input) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(&mut out) + .launch(cfg(num_leaves))?; + } + let host = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(host) +} + +/// `n` width-8 permutations of `states` (`n·8` felts) at [`W8_VARIANTS`]`[variant]`. +pub fn w8_permute_many(variant: usize, states: &[u64]) -> Result> { + assert_eq!(states.len() % 8, 0); + let n = (states.len() / 8) as u64; + let f = &module()?.zisk.w8_permute_probe[variant]; + let stream = backend()?.next_stream(); + let input = stream.clone_htod(states)?; + let mut out = stream.alloc_zeros::(states.len())?; + unsafe { + stream + .launch_builder(f) + .arg(&input) + .arg(&n) + .arg(&mut out) + .launch(cfg(n))?; + } + let host = crate::readback::clone_dtoh(&stream, &out)?; + stream.synchronize()?; + Ok(host) +} + +/// ZisK's grind: the smallest nonce in `[0, count)` whose width-8 permutation +/// of `[c0, c1, c2, nonce, 0, 0, 0, 0]` has lane 0 below `limit`, if any. +pub fn w8_grind( + variant: usize, + challenge: &[u64; 3], + limit: u64, + count: u64, + grid: u32, +) -> Result> { + let f = &module()?.zisk.w8_grind_search[variant]; + let stream = backend()?.next_stream(); + let challenge_dev = stream.clone_htod(challenge.as_slice())?; + let mut result = stream.clone_htod(&[u64::MAX])?; + let base = 0u64; + unsafe { + stream + .launch_builder(f) + .arg(&challenge_dev) + .arg(&limit) + .arg(&base) + .arg(&count) + .arg(&mut result) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + let host = crate::readback::clone_dtoh(&stream, &result)?; + stream.synchronize()?; + Ok((host[0] != u64::MAX).then_some(host[0])) +} + +/// The LDE-row widths the P1 bench hashes: row pairs (the trace-commit layout) +/// and single rows (an ext3 table's slabs). +const ROW_PAIR_WIDTHS: [u64; 3] = [16, 48, 96]; +const ROW_WIDTHS: [u64; 2] = [24, 96]; + +/// ★ Stage P1's microbenchmark: ZisK's instance against the production RPX +/// kernels on one `2^log_len`-felt buffer. +/// +/// - `leaf`: 64-felt cosets (the I-HASH geometry): RPX, the I-HASH leaf (v2, +/// the continuity control for FAST 282) and ZisK's leaf. +/// - `node` / `node4`, and `tree` (the whole tree's time per leaf): RPX's +/// binary production walk against 4-ary levels. +/// - `rows*`: the STARK's column-major LDE rows at [`ROW_PAIR_WIDTHS`] and +/// [`ROW_WIDTHS`]: RPX's production leaf kernels against ZisK's leaf, per +/// permutation and per leaf. +/// - `grind`: a full scan of `grind_count` nonces: RPX, the I-HASH W16 grind +/// (v2) and ZisK's W8 grind at both variants, at two grids. +/// +/// Every line is `BENCH ns/perm [ ns/leaf] ( perms, ms)`. +pub fn microbench_zisk(log_len: u32, reps: u32, grind_count: u64) -> Result> { + let be = backend()?; + let m = module()?; + let stream = be.next_stream(); + let len = 1u64 << log_len; + let mut lines = Vec::new(); + let mut line = |what: &str, hash: &str, ns: f64, perms: u64, leaves: Option| { + let per_leaf = leaves + .map(|l| format!(" {:>8.3} ns/leaf", ns / l as f64)) + .unwrap_or_default(); + let s = format!( + "BENCH {what:<14} {hash:<10} {:>7.3} ns/perm{per_leaf} ({perms} perms, {:.2} ms)", + ns / perms as f64, + ns / 1e6 + ); + println!("{s}"); + lines.push(s); + }; + + let mut buf = stream.alloc_zeros::(len as usize)?; + unsafe { + stream + .launch_builder(&m.fill) + .arg(&mut buf) + .arg(&len) + .launch(cfg(len))?; + } + stream.synchronize()?; + + // Cosets: RPX's production leaves into a tree buffer, then its walk. + let block = 64u64; + let num_leaves = len / block; + let total_nodes = 2 * num_leaves as usize - 1; + let mut nodes = stream.alloc_zeros::(total_nodes * 32)?; + let leaves_off = (num_leaves as usize - 1) * 32; + let ns = median_ns(reps, || { + let mut leaves = nodes.slice_mut(leaves_off..leaves_off + num_leaves as usize * 32); + unsafe { + stream + .launch_builder(&be.rpx_leaves_base_coset) + .arg(&buf) + .arg(&num_leaves) + .arg(&block) + .arg(&mut leaves) + .launch(crate::merkle::keccak_launch_cfg(num_leaves))?; + } + stream.synchronize() + })?; + line( + "leaf", + "rpx", + ns, + num_leaves * block.div_ceil(8), + Some(num_leaves), + ); + let ns = median_ns(reps, || { + crate::rpx::build_inner_tree_levels(&stream, be, &mut nodes, num_leaves as usize)?; + stream.synchronize() + })?; + line("node", "rpx", ns, num_leaves - 1, None); + line("tree", "rpx", ns, num_leaves - 1, Some(num_leaves)); + drop(nodes); + + let p1_perms = num_leaves * block.div_ceil(RATE as u64); + let mut leaves = stream.alloc_zeros::(num_leaves as usize * DIGEST)?; + for (tag, f) in [ + ("p1-v2", &m.variants[2].leaves_base_coset), + ("p1-zisk", &m.zisk.leaves_base_coset), + ] { + let ns = median_ns(reps, || { + unsafe { + stream + .launch_builder(f) + .arg(&buf) + .arg(&num_leaves) + .arg(&block) + .arg(&mut leaves) + .launch(cfg(num_leaves))?; + } + stream.synchronize() + })?; + line("leaf", tag, ns, p1_perms, Some(num_leaves)); + } + let mut sizes = Vec::new(); + let mut n = num_leaves; + while n >= 4 { + n /= 4; + sizes.push(n); + } + let mut levels: Vec<_> = sizes + .iter() + .map(|&s| stream.alloc_zeros::(s as usize * DIGEST)) + .collect::>()?; + let level4 = &m.variants[2].merkle_level4; + let ns = median_ns(reps, || { + for (i, &n_parents) in sizes.iter().enumerate() { + let (done, rest) = levels.split_at_mut(i); + let parents = &mut rest[0]; + let children = if i == 0 { &leaves } else { &done[i - 1] }; + unsafe { + stream + .launch_builder(level4) + .arg(children) + .arg(parents) + .arg(&n_parents) + .launch(cfg(n_parents))?; + } + } + stream.synchronize() + })?; + let node_perms: u64 = sizes.iter().sum(); + line("node4", "p1-v2", ns, node_perms, None); + line("tree", "p1-v2", ns, node_perms, Some(num_leaves)); + drop(levels); + drop(leaves); + + // LDE rows: the buffer read as `num_cols` columns of `2^k` rows. + for (pair, widths) in [(true, &ROW_PAIR_WIDTHS[..]), (false, &ROW_WIDTHS[..])] { + for &num_cols in widths { + let num_rows = 1u64 << (63 - (len / num_cols).leading_zeros()); + let num_leaves = if pair { num_rows / 2 } else { num_rows }; + let felts = if pair { 2 * num_cols } else { num_cols }; + let what = format!("rows{}-w{num_cols}", if pair { "-pair" } else { "" }); + let mut out = stream.alloc_zeros::(num_leaves as usize * 32)?; + let ns = median_ns(reps, || { + let mut view = out.slice_mut(..); + if pair { + crate::rpx::launch_leaves_base_row_pair( + &stream, &buf, num_rows, num_cols, num_rows, &mut view, + )?; + } else { + crate::rpx::launch_leaves_base( + &stream, &buf, num_rows, num_cols, num_rows, &mut view, + )?; + } + stream.synchronize() + })?; + line( + &what, + "rpx", + ns, + num_leaves * felts.div_ceil(8), + Some(num_leaves), + ); + drop(out); + let mut out = stream.alloc_zeros::(num_leaves as usize * DIGEST)?; + let f = if pair { + &m.zisk.leaves_row_pair + } else { + &m.zisk.leaves_rows + }; + let log_num_rows = num_rows.trailing_zeros() as u64; + let ns = median_ns(reps, || { + unsafe { + stream + .launch_builder(f) + .arg(&buf) + .arg(&num_rows) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(&mut out) + .launch(cfg(num_leaves))?; + } + stream.synchronize() + })?; + line( + &what, + "p1-zisk", + ns, + num_leaves * felts.div_ceil(RATE as u64), + Some(num_leaves), + ); + } + } + + // Grind: full scans (limit 0 never hits), the simple grid-stride kernels. + let inner = [1u64, 2, 3, 4]; + let inner_dev = stream.clone_htod(inner.as_slice())?; + let limit = 0u64; + let base = 0u64; + for grid in [ + crate::grinding::GRID_DEFAULT, + 4 * crate::grinding::GRID_DEFAULT, + ] { + let launch = LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + let mut result = stream.clone_htod(&[u64::MAX])?; + let mut arms: Vec<(String, &CudaFunction)> = vec![ + (format!("rpx-g{grid}"), &be.rpx_grind_search), + (format!("p1w16-v2-g{grid}"), &m.variants[2].grind_search), + ]; + for (v, tag) in W8_VARIANTS.iter().enumerate() { + arms.push((format!("p1w8-{tag}-g{grid}"), &m.zisk.w8_grind_search[v])); + } + for (tag, f) in arms { + let ns = median_ns(reps, || { + unsafe { + stream + .launch_builder(f) + .arg(&inner_dev) + .arg(&limit) + .arg(&base) + .arg(&grind_count) + .arg(&mut result) + .launch(launch)?; + } + stream.synchronize() + })?; + line("grind", &tag, ns, grind_count, None); + } + } + Ok(lines) +} + +/// A device buffer of `len` distinct canonical felts (the bench fill), for +/// benches outside this crate that need a large random device matrix without +/// a host upload. +pub fn random_device_matrix(len: usize) -> Result> { + let m = module()?; + let stream = backend()?.next_stream(); + let mut out = stream.alloc_zeros::(len)?; + let n = len as u64; + unsafe { + stream + .launch_builder(&m.fill) + .arg(&mut out) + .arg(&n) + .launch(cfg(n))?; + } + stream.synchronize()?; + Ok(out) +} diff --git a/crypto/math-cuda/src/readback.rs b/crypto/math-cuda/src/readback.rs new file mode 100644 index 000000000..821a8e53b --- /dev/null +++ b/crypto/math-cuda/src/readback.rs @@ -0,0 +1,300 @@ +//! Device-to-host read-backs that can stay off the driver's pageable path. +//! +//! A copy from the card into pageable host memory returns only once it has +//! completed, and the driver stages it through a buffer of its own. Two +//! threads reading back that way at once wait on each other inside the +//! driver: under phase B's overlap, the argument's round trips sat behind the +//! openings' read-backs (in a trace of a median block, 91 % of the argument +//! thread's lock waits overlapped another thread's pageable read-back). +//! +//! While [`pinned`] is on, a thread in the background class +//! ([`crate::stream_class`]) reads back through a pinned buffer: an +//! asynchronous copy, then a blocking-sync event the thread sleeps on, then a +//! host copy into the caller's memory. The bytes are the same either way. +//! Every other thread, and every thread while it is off (the default), reads +//! back the stream's own way. The buffers come from a pool: a thread takes +//! one at its first pinned read-back and gives it back when it ends, so the +//! short-lived threads phase B spawns do not pin memory of their own each. + +use std::cell::RefCell; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use cudarc::driver::{CudaStream, DevicePtr, DeviceRepr, DriverError}; + +use crate::stream_class::{self, StreamClass}; + +static PINNED: AtomicBool = AtomicBool::new(false); +static PINNED_COPIES: AtomicU64 = AtomicU64::new(0); +static PAGEABLE_COPIES: AtomicU64 = AtomicU64::new(0); +static PINNED_BYTES: AtomicU64 = AtomicU64::new(0); + +/// Buffers threads gave back, for the next thread to take. +static POOL: Mutex> = Mutex::new(Vec::new()); + +/// Buffers the pool keeps; past it a buffer given back is freed. +const POOL_KEEPS: usize = 16; + +thread_local! { + static STAGING: RefCell = const { RefCell::new(Lent(None)) }; +} + +/// While it lives, background-class threads read back through pinned +/// staging. Taken by the thread that runs phase B. +pub struct PinnedReadbacks { + previous: bool, +} + +/// Turns pinned read-backs on until the guard drops. +pub fn pinned() -> PinnedReadbacks { + PinnedReadbacks { + previous: PINNED.swap(true, Ordering::AcqRel), + } +} + +impl Drop for PinnedReadbacks { + fn drop(&mut self) { + PINNED.store(self.previous, Ordering::Release); + } +} + +/// Background-class read-backs so far, `(through pinned staging, the +/// stream's own way)`, process wide. +pub fn background_copies() -> (u64, u64) { + ( + PINNED_COPIES.load(Ordering::Relaxed), + PAGEABLE_COPIES.load(Ordering::Relaxed), + ) +} + +/// Host bytes the pinned read-back buffers hold now, lent or pooled +/// (page-locked: resident, and in no allocator's statistics). +pub fn pinned_bytes() -> u64 { + PINNED_BYTES.load(Ordering::Relaxed) +} + +/// Whether this thread's read-backs go through pinned staging; counts a +/// background-class read-back either way. +fn through_staging() -> bool { + if stream_class::current() != StreamClass::Background { + return false; + } + let pinned = PINNED.load(Ordering::Acquire); + let counter = if pinned { + &PINNED_COPIES + } else { + &PAGEABLE_COPIES + }; + counter.fetch_add(1, Ordering::Relaxed); + pinned +} + +/// A pinned buffer and the blocking-sync event its copies record. +struct Staging { + ptr: *mut u8, + bytes: usize, + event: cudarc::driver::sys::CUevent, +} + +// SAFETY: the buffer and the event are used by one thread at a time: the +// thread that holds the `Staging`, or the pool's mutex. +unsafe impl Send for Staging {} + +impl Staging { + /// A buffer of at least `bytes`, from the pool or new, in the context bound + /// to this thread. + fn take(bytes: usize) -> Result { + use cudarc::driver::sys; + let pooled = POOL.lock().map(|mut pool| pool.pop()).unwrap_or(None); + let mut staging = match pooled { + Some(staging) => staging, + None => { + let mut event = std::ptr::null_mut(); + let flags = sys::CUevent_flags::CU_EVENT_BLOCKING_SYNC as u32 + | sys::CUevent_flags::CU_EVENT_DISABLE_TIMING as u32; + // SAFETY: `event` is a valid out-pointer. + unsafe { sys::cuEventCreate(&mut event, flags) }.result()?; + Staging { + ptr: std::ptr::null_mut(), + bytes: 0, + event, + } + } + }; + staging.grow(bytes)?; + Ok(staging) + } + + /// Room for `bytes`: the buffer replaced by a larger one if it is short. + fn grow(&mut self, bytes: usize) -> Result<(), DriverError> { + if self.bytes >= bytes { + return Ok(()); + } + self.free(); + let size = bytes.next_power_of_two().max(1 << 16); + // SAFETY: a fresh host allocation the driver pins; freed by `free`. + self.ptr = unsafe { cudarc::driver::result::malloc_host(size, 0) }? as *mut u8; + self.bytes = size; + PINNED_BYTES.fetch_add(size as u64, Ordering::Relaxed); + Ok(()) + } + + /// Frees the buffer; every copy into it was waited for before its + /// read-back returned. + fn free(&mut self) { + if !self.ptr.is_null() { + // SAFETY: the buffer is idle and ours. + unsafe { + let _ = cudarc::driver::sys::cuMemFreeHost(self.ptr as *mut _); + } + PINNED_BYTES.fetch_sub(self.bytes as u64, Ordering::Relaxed); + self.ptr = std::ptr::null_mut(); + self.bytes = 0; + } + } +} + +impl Drop for Staging { + fn drop(&mut self) { + self.free(); + // SAFETY: the event is idle and ours. + unsafe { + let _ = cudarc::driver::sys::cuEventDestroy_v2(self.event); + } + } +} + +/// The buffer a thread holds; given back to the pool when the thread ends. +struct Lent(Option); + +impl Drop for Lent { + fn drop(&mut self) { + if let Some(staging) = self.0.take() + && let Ok(mut pool) = POOL.lock() + && pool.len() < POOL_KEEPS + { + pool.push(staging); + } + } +} + +/// Copies `src` into `dst` (at least `src.len()` elements) through this +/// thread's pinned buffer and waits for it on a blocking-sync event. +fn through_pinned>( + stream: &Arc, + src: &Src, + dst: &mut [T], +) -> Result<(), DriverError> { + let bytes = std::mem::size_of_val(&dst[..src.len()]); + if bytes == 0 { + return Ok(()); + } + stream.context().bind_to_thread()?; + STAGING.with(|cell| { + let mut lent = cell.borrow_mut(); + let staging = match lent.0.take() { + Some(mut staging) => { + staging.grow(bytes)?; + lent.0.insert(staging) + } + None => lent.0.insert(Staging::take(bytes)?), + }; + { + let (src_ptr, _record) = src.device_ptr(stream); + // SAFETY: `staging` is pinned and holds `bytes`; the copy is + // ordered after the stream's earlier work on `src`, and waited for + // below before the buffer is read or reused. + unsafe { + cudarc::driver::sys::cuMemcpyDtoHAsync_v2( + staging.ptr as *mut _, + src_ptr, + bytes, + stream.cu_stream(), + ) + .result()?; + let recorded = + cudarc::driver::sys::cuEventRecord(staging.event, stream.cu_stream()).result(); + if let Err(e) = recorded { + let _ = stream.synchronize(); + return Err(e); + } + } + } + // SAFETY: the event was recorded after the copy on its stream. + unsafe { cudarc::driver::sys::cuEventSynchronize(staging.event) }.result()?; + // SAFETY: the copy has completed; `dst` holds at least `bytes` and + // does not overlap the pinned buffer. + unsafe { + std::ptr::copy_nonoverlapping(staging.ptr, dst.as_mut_ptr() as *mut u8, bytes); + } + Ok(()) + }) +} + +/// `stream.clone_dtoh(src)`, or through a pinned buffer on a background-class +/// thread while [`pinned`] is on. +pub fn clone_dtoh>( + stream: &Arc, + src: &Src, +) -> Result, DriverError> { + if !through_staging() { + return stream.clone_dtoh(src); + } + let mut out = Vec::with_capacity(src.len()); + // SAFETY: every element is written by the copy before `out` is read; a + // failed copy drops `out` without reading it (`T: DeviceRepr` is plain + // data with no drop glue). + #[allow(clippy::uninit_vec)] + unsafe { + out.set_len(src.len()) + }; + through_pinned(stream, src, &mut out)?; + Ok(out) +} + +/// `stream.memcpy_dtoh(src, dst)` into host memory, or through a pinned +/// buffer on a background-class thread while [`pinned`] is on. +pub fn memcpy_dtoh>( + stream: &Arc, + src: &Src, + dst: &mut [T], +) -> Result<(), DriverError> { + assert!(dst.len() >= src.len()); + if !through_staging() { + return stream.memcpy_dtoh(src, dst); + } + through_pinned(stream, src, dst) +} + +#[cfg(test)] +mod tests { + use super::{background_copies, pinned, through_staging}; + use crate::stream_class::{StreamClass, enter}; + + /// Only a background-class thread takes the pinned path, and only while + /// the guard lives; each background read-back is counted on the path it + /// took, a foreground one on neither. + #[test] + fn only_background_threads_read_back_pinned_while_it_is_on() { + let background = || { + std::thread::scope(|s| { + s.spawn(|| { + let _bg = enter(StreamClass::Background); + through_staging() + }) + .join() + .expect("joined") + }) + }; + let before = background_copies(); + assert!(!through_staging() && !background()); + { + let _on = pinned(); + assert!(!through_staging(), "a foreground thread"); + assert!(background(), "a background thread"); + } + assert!(!background()); + let after = background_copies(); + assert_eq!((after.0 - before.0, after.1 - before.1), (1, 2)); + } +} diff --git a/crypto/math-cuda/src/round_slot.rs b/crypto/math-cuda/src/round_slot.rs new file mode 100644 index 000000000..02d8a9f30 --- /dev/null +++ b/crypto/math-cuda/src/round_slot.rs @@ -0,0 +1,517 @@ +//! A round's sums without the second launch and the pageable read-back +//! (`LAMBDA_VM_ARGUE_ROUND_SLOT`): the round kernel's last block reduces them +//! (`slot_sums` in `kernels/sumcheck.cu`) into a slot of pinned host memory the +//! card writes through a mapping, and the host spins on the slot's sequence +//! word instead of copying and waiting. +//! +//! The sums are the limbs today's `sum_partials_ext3` writes, and the caller +//! reduces them by `ext3_from_raw` as it does today, so no value the host sees +//! moves. `Check` runs today's second launch and read-back as well, compares +//! the two limb for limb, counts a difference and uses today's. +//! +//! A slot is 128 u64: the sums at `[0, 96)` (32 rows of three), then the +//! sequence word at [`SEQ`]; the words after it are unused. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use cudarc::driver::{CudaSlice, CudaStream}; + +use crate::Result; + +/// Words a slot spans. +pub const SLOT_WORDS: usize = 128; +/// Where the last block writes the launch's sequence number, after the sums. +pub const SEQ: usize = 96; +/// Rows a slot holds. +pub const MAX_ROWS: usize = 32; + +/// How a session's round sums come back. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RoundSlot { + /// Today's second launch and pageable read-back. + Off, + /// The last block's reduction, through a slot. + On, + /// Both, compared every round; today's are used. + Check, +} + +/// Shared memory a `_slot` kernel's block asks for: its own reduction's, or +/// the last block's 256 lanes when that is more, and the last block's flag. +pub fn shared_bytes(block: u32) -> u32 { + block.max(256) * 24 + 16 +} + +static SLOT_ROUNDS: AtomicU64 = AtomicU64::new(0); +static FALL_BACKS: AtomicU64 = AtomicU64::new(0); +static DECLINES: AtomicU64 = AtomicU64::new(0); +static SECOND_LAUNCHES: AtomicU64 = AtomicU64::new(0); +static CHECKED: AtomicU64 = AtomicU64::new(0); +static MISMATCHES: AtomicU64 = AtomicU64::new(0); + +/// What the slots did this process. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Counts { + /// Rounds whose sums were read from a slot. + pub slot_rounds: u64, + /// Of those, the ones whose spin ran out and that waited on the stream. + pub fall_backs: u64, + /// Sessions that asked for a slot and got none (today's path instead). + pub declines: u64, + /// Today's second launches (`sum_partials_ext3`) by the GKR layers' and + /// the fused zerocheck's rounds. + pub second_launches: u64, + /// Rounds `Check` compared. + pub checked: u64, + /// Rounds whose sums `Check` found differing. + pub mismatches: u64, +} + +pub fn counts() -> Counts { + Counts { + slot_rounds: SLOT_ROUNDS.load(Ordering::Relaxed), + fall_backs: FALL_BACKS.load(Ordering::Relaxed), + declines: DECLINES.load(Ordering::Relaxed), + second_launches: SECOND_LAUNCHES.load(Ordering::Relaxed), + checked: CHECKED.load(Ordering::Relaxed), + mismatches: MISMATCHES.load(Ordering::Relaxed), + } +} + +pub fn reset_counts() { + for c in [ + &SLOT_ROUNDS, + &FALL_BACKS, + &DECLINES, + &SECOND_LAUNCHES, + &CHECKED, + &MISMATCHES, + ] { + c.store(0, Ordering::Relaxed); + } +} + +/// Called where a round launches today's second launch. +pub(crate) fn note_second_launch() { + SECOND_LAUNCHES.fetch_add(1, Ordering::Relaxed); +} + +/// The spin before yielding, and the patience before waiting on the stream. +const SPIN: Duration = Duration::from_micros(50); +const PATIENCE: Duration = Duration::from_millis(5); + +/// Waits until `words[SEQ]` is `seq`: a spin for `spin`, then yields for +/// `patience`, then `fall_back` (the stream's synchronize) and one more look. +/// Returns the first `3·rows` words, read after the sequence word with an +/// acquire fence, and whether the wait fell back. A word that is still not +/// `seq` after the fall-back is an error: the launch never wrote its slot. +/// +/// # Safety +/// `words` points to [`SLOT_WORDS`] u64 that stay valid for the call; another +/// agent (the card, or a test's thread) may write them meanwhile. +pub(crate) unsafe fn wait_on( + words: *const u64, + seq: u64, + rows: usize, + spin: Duration, + patience: Duration, + fall_back: impl FnOnce() -> Result<()>, +) -> Result<(Vec, bool)> { + if rows > MAX_ROWS { + return Err(invalid()); + } + let arrived = || unsafe { std::ptr::read_volatile(words.add(SEQ)) } == seq; + let started = Instant::now(); + let mut fell_back = false; + let mut seen = arrived(); + while !seen && started.elapsed() < spin { + for _ in 0..64 { + std::hint::spin_loop(); + } + seen = arrived(); + } + while !seen && started.elapsed() < spin + patience { + std::thread::yield_now(); + seen = arrived(); + } + if !seen { + fall_back()?; + fell_back = true; + if !arrived() { + return Err(invalid()); + } + } + std::sync::atomic::fence(Ordering::Acquire); + let sums = (0..3 * rows) + .map(|i| unsafe { std::ptr::read_volatile(words.add(i)) }) + .collect(); + Ok((sums, fell_back)) +} + +fn invalid() -> cudarc::driver::DriverError { + cudarc::driver::DriverError(cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE) +} + +/// Sequence numbers, one a launch, never repeated in a process: a slot taken +/// again cannot mistake an earlier owner's word for its own launch's. +static NEXT_SEQ: AtomicU64 = AtomicU64::new(1); + +fn next_seq() -> u64 { + NEXT_SEQ.fetch_add(1, Ordering::Relaxed) +} + +/// A slot's host and device addresses. +#[derive(Clone, Copy)] +struct Place { + host: usize, + dev: u64, +} + +/// Slots no session holds, from 64 KiB pages of pinned, card-mapped memory +/// that are never given back (at most [`MAX_PAGES`]). +static FREE: Mutex> = Mutex::new(Vec::new()); +static PAGES: AtomicU64 = AtomicU64::new(0); +const PAGE_BYTES: usize = 64 * 1024; +const MAX_PAGES: u64 = 16; + +fn new_page() -> Option> { + use cudarc::driver::sys; + if PAGES.fetch_add(1, Ordering::Relaxed) >= MAX_PAGES { + PAGES.fetch_sub(1, Ordering::Relaxed); + return None; + } + let be = crate::device::backend().ok()?; + be.ctx.bind_to_thread().ok()?; + // SAFETY: a fresh allocation of `PAGE_BYTES`, mapped into the card's + // address space; never freed, so the addresses below stay valid. + unsafe { + let host = cudarc::driver::result::malloc_host( + PAGE_BYTES, + sys::CU_MEMHOSTALLOC_DEVICEMAP | sys::CU_MEMHOSTALLOC_PORTABLE, + ) + .ok()? as *mut u64; + let mut dev: sys::CUdeviceptr = 0; + if sys::cuMemHostGetDevicePointer_v2(&mut dev, host as *mut _, 0) + .result() + .is_err() + { + let _ = sys::cuMemFreeHost(host as *mut _); + PAGES.fetch_sub(1, Ordering::Relaxed); + return None; + } + std::ptr::write_bytes(host, 0, PAGE_BYTES / 8); + let stride = SLOT_WORDS * 8; + Some( + (0..PAGE_BYTES / stride) + .map(|i| Place { + host: host as usize + i * stride, + dev: dev + (i * stride) as u64, + }) + .collect(), + ) + } +} + +/// One session's slot, given back to the pool when the session drops — after +/// its stream is idle, so no launch of this owner can still write it. +pub struct MappedSlot { + place: Place, + stream: Arc, +} + +// SAFETY: the slot's memory is only touched through this handle (and by the +// card, ordered by the stream it was launched on), and it is not shared. +unsafe impl Send for MappedSlot {} +unsafe impl Sync for MappedSlot {} + +impl MappedSlot { + /// A slot for launches on `stream`, or `None` when the pool is spent or + /// the card will not map host memory. + pub fn take(stream: &Arc) -> Option { + let place = { + let mut free = FREE.lock().ok()?; + match free.pop() { + Some(place) => place, + None => { + let mut page = new_page()?; + let place = page.pop()?; + free.extend(page); + place + } + } + }; + // SAFETY: the slot is this handle's; no launch of an earlier owner is + // in flight (that owner's stream was synchronized before it gave the + // slot back). + unsafe { std::ptr::write_bytes(place.host as *mut u64, 0, SLOT_WORDS) }; + Some(Self { + place, + stream: stream.clone(), + }) + } + + /// The slot's address on the card. + pub fn dev(&self) -> u64 { + self.place.dev + } + + fn words(&self) -> *mut u64 { + self.place.host as *mut u64 + } + + /// Waits for the launch numbered `seq` and returns its `rows` sums. + pub fn wait(&self, seq: u64, rows: usize) -> Result> { + let stream = self.stream.clone(); + // SAFETY: the slot's words are valid while `self` is. + let (sums, fell_back) = unsafe { + wait_on(self.words(), seq, rows, SPIN, PATIENCE, || { + stream.synchronize() + }) + }?; + SLOT_ROUNDS.fetch_add(1, Ordering::Relaxed); + if fell_back { + FALL_BACKS.fetch_add(1, Ordering::Relaxed); + } + Ok(sums) + } + + /// The slot's host words, for a test that writes a flag beside the sums + /// (the capability probe). Not for production callers. + #[doc(hidden)] + pub fn host_words_for_tests(&self) -> *mut u64 { + self.words() + } +} + +impl Drop for MappedSlot { + fn drop(&mut self) { + // Every launch that writes this slot is on this stream. + let _ = self.stream.synchronize(); + if let Ok(mut free) = FREE.lock() { + free.push(self.place); + } + } +} + +/// A session's rounds through a slot: the slot, the ticket its launches' +/// blocks take, and whether today's second launch runs beside them. +pub struct SlotRounds { + slot: MappedSlot, + ticket: CudaSlice, + check: bool, + seq: u64, +} + +impl SlotRounds { + /// For `mode` on `stream`; `None` for `Off`, and — counted — when no slot + /// or ticket can be had, so the session runs today's rounds. + pub fn open(stream: &Arc, mode: RoundSlot) -> Option { + if mode == RoundSlot::Off { + return None; + } + let made = MappedSlot::take(stream).and_then(|slot| { + let ticket = crate::device::alloc_zeros_or_trim::(stream, 1).ok()?; + Some(Self { + slot, + ticket, + check: mode == RoundSlot::Check, + seq: 0, + }) + }); + if made.is_none() { + DECLINES.fetch_add(1, Ordering::Relaxed); + } + made + } + + /// Whether today's second launch runs beside the slot's. + pub fn checking(&self) -> bool { + self.check + } + + /// The next launch's arguments: the ticket, the slot's card address and + /// its sequence number. + pub fn launch(&mut self) -> (&mut CudaSlice, u64, u64) { + self.seq = next_seq(); + (&mut self.ticket, self.slot.dev(), self.seq) + } + + /// The last launch's `rows` sums. Under `Check`, `today` reads today's + /// second launch's; the two are compared limb for limb and today's are + /// returned. + pub fn collect( + &self, + rows: usize, + today: impl FnOnce() -> Result>, + ) -> Result> { + let sums = self.slot.wait(self.seq, rows)?; + if !self.check { + return Ok(sums); + } + let want = today()?; + CHECKED.fetch_add(1, Ordering::Relaxed); + if want != sums || fault_armed() { + MISMATCHES.fetch_add(1, Ordering::Relaxed); + } + Ok(want) + } +} + +thread_local! { + /// ⛔ A FAULT: while armed on a thread, every round `Check` compares counts + /// as differing — so a test can show the count reaches the harness line. + static FAULT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[doc(hidden)] +pub fn force_check_fault(on: bool) { + FAULT.with(|cell| cell.set(on)); +} + +fn fault_armed() -> bool { + FAULT.with(std::cell::Cell::get) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn slot() -> Box<[u64; SLOT_WORDS]> { + Box::new([0u64; SLOT_WORDS]) + } + + /// A word that arrives late is waited for, and the sums written before it + /// are the ones read. + #[test] + fn a_late_sequence_word_is_waited_for() { + let mut words = slot(); + let read = words.as_mut_ptr(); + let at = read as usize; + let writer = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(1)); + let w = at as *mut u64; + unsafe { + for i in 0..6 { + std::ptr::write_volatile(w.add(i), 100 + i as u64); + } + std::sync::atomic::fence(Ordering::Release); + std::ptr::write_volatile(w.add(SEQ), 7); + } + }); + let (sums, fell_back) = unsafe { + wait_on( + read as *const u64, + 7, + 2, + Duration::from_micros(50), + Duration::from_secs(5), + || Ok(()), + ) + } + .unwrap(); + writer.join().unwrap(); + assert_eq!(sums, vec![100, 101, 102, 103, 104, 105]); + assert!(!fell_back, "the word came inside the patience"); + } + + /// ⛔ Another launch's sequence word is not this one's: the wait falls + /// back to the stream and, the word still wrong, fails. + #[test] + fn a_wrong_sequence_word_is_not_taken() { + let mut words = slot(); + words[SEQ] = 6; + let mut fell = false; + let got = unsafe { + wait_on( + words.as_ptr(), + 7, + 1, + Duration::from_micros(10), + Duration::from_micros(10), + || { + fell = true; + Ok(()) + }, + ) + }; + assert!(fell, "the wait fell back to the stream"); + assert!(got.is_err(), "a word that never arrives is an error"); + } + + /// The fall-back is what makes a slow launch's word arrive: the stream's + /// synchronize returns after the card wrote it. + #[test] + fn the_fall_back_waits_for_the_launch() { + let mut words = slot(); + let at = words.as_mut_ptr(); + let (sums, fell_back) = unsafe { + wait_on( + at as *const u64, + 9, + 1, + Duration::from_micros(1), + Duration::from_micros(1), + || { + std::ptr::write_volatile(at.add(0), 1); + std::ptr::write_volatile(at.add(1), 2); + std::ptr::write_volatile(at.add(2), 3); + std::ptr::write_volatile(at.add(SEQ), 9); + Ok(()) + }, + ) + } + .unwrap(); + assert!(fell_back); + assert_eq!(sums, vec![1, 2, 3]); + } + + /// ⛔ The fall-back's own error is the wait's. + #[test] + fn a_failed_stream_wait_is_the_waits_error() { + let words = slot(); + let got = unsafe { + wait_on(words.as_ptr(), 1, 1, Duration::ZERO, Duration::ZERO, || { + Err(invalid()) + }) + }; + assert!(got.is_err()); + } + + /// ⛔ More rows than a slot holds is refused before anything is read. + #[test] + fn more_rows_than_a_slot_holds_is_refused() { + let words = slot(); + let got = unsafe { + wait_on( + words.as_ptr(), + 0, + MAX_ROWS + 1, + Duration::ZERO, + Duration::ZERO, + || Ok(()), + ) + }; + assert!(got.is_err()); + } + + /// The layout the kernel writes: the sums of 32 rows before the sequence + /// word, inside the slot. + #[test] + fn the_slot_layout_holds_what_the_kernel_writes() { + assert_eq!(MAX_ROWS * 3, SEQ, "the sums end where seq is"); + const { assert!(SEQ < SLOT_WORDS) }; + assert_eq!(shared_bytes(32), 256 * 24 + 16); + assert_eq!(shared_bytes(256), 256 * 24 + 16); + assert_eq!(shared_bytes(512), 512 * 24 + 16); + } + + /// Sequence numbers are never repeated. + #[test] + fn sequence_numbers_grow() { + let a = next_seq(); + let b = next_seq(); + assert!(b > a); + } +} diff --git a/crypto/math-cuda/src/rpx.rs b/crypto/math-cuda/src/rpx.rs new file mode 100644 index 000000000..b32d0903b --- /dev/null +++ b/crypto/math-cuda/src/rpx.rs @@ -0,0 +1,1353 @@ +//! GPU RPX256 (XHash12) for Merkle commits — the leaf kernels, the parent/level +//! compressors, and the permutation probe that is the host's only handle on the +//! bare device permutation. +//! +//! Twin of [`crate::blake3`], launcher for launcher, so the two read against +//! each other; production dispatch reaches this module through +//! [`crate::DeviceHash::Rpx256`] from the fused LDE+commit pipelines +//! ([`crate::lde`]), the comp-poly tree builders (`stark::gpu_lde`) and the FRI +//! layer commits ([`crate::fri`]). +//! +//! # What a parent is +//! +//! `hash_new_parent(left, right)` is ONE permutation of `[left ‖ right ‖ 0⁴]` +//! with the digest read from lanes 0..4 — `algebraic_commit::parent`, which is +//! `Rpx256::merge` in miden's terms. The children are decoded from their node +//! bytes as `commitment_to_digest` does (four big-endian u64s) and the parent +//! is encoded back as `digest_to_commitment` does, so device nodes are the +//! host's bytes. +//! +//! # What a leaf is +//! +//! The rate-8 OVERWRITE duplex `algebraic_commit::sponge_leaf` over the felt +//! sequence the host leaf hashes — the same read pattern as the BLAKE3 kernel +//! each leaf kernel twins (`leaves_bit_reversed_grouped`), which is exactly the +//! sequence `felts_from_bytes` rebuilds from the leaf bytes, so the +//! `hash_bytes == hash_data` contract holds on device by construction. +//! +//! # Coverage +//! +//! All seven leaf kernels, both tree compressors and the wrapper twins are +//! here; the `launch_*` functions are what the dispatch sites call. The +//! permutation itself is pinned without a GPU by +//! `tests/host_kat/rpx_host_kat.cpp` (`make test-rpx-host-kat`); the device +//! build is pinned against the host by `prover/tests/rpx_device_parity.rs`. + +use cudarc::driver::{CudaSlice, CudaStream, CudaViewMut, LaunchConfig, PushKernelArg}; +use std::sync::Arc; + +use crate::Result; +use crate::device::{Backend, backend}; +use crate::lde::pack_ext3_to_pinned_slabs; + +/// Felts in one permutation state. +pub const STATE_FELTS: usize = 12; + +/// Threads per block for the RPX kernels. +/// +/// [`crate::merkle`]'s 128 rather than BLAKE3's 256: a thread carries a +/// twelve-lane u64 state plus the inverse S-box's live temporaries, a register +/// footprint closer to keccak's 25 u64 lanes than to BLAKE3's 32 u32 words, and +/// 128 is the Blackwell register-file limit that keccak already runs at. To be +/// re-measured with `-Xptxas -v` (phase-2 gate); this is the safe default. +const RPX_BLOCK_DIM: u32 = 128; + +pub(crate) fn rpx_launch_cfg(num_threads: u64) -> LaunchConfig { + debug_assert!( + num_threads <= u32::MAX as u64, + "rpx_launch_cfg: num_threads ({num_threads}) exceeds u32 grid range", + ); + let grid = (num_threads as u32).div_ceil(RPX_BLOCK_DIM); + LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (RPX_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + } +} + +/// RPX leaf hashing over a base-field column buffer. Twin of +/// [`crate::blake3::leaves_base`], argument for argument. +/// +/// `columns` must hold `num_cols * col_stride` u64s with column `c`'s data at +/// `[c*col_stride .. c*col_stride + num_rows]`. `rows_per_leaf` selects the leaf +/// layout: `1` = one leaf per bit-reversed row (`num_rows` leaves), `2` = one +/// leaf per bit-reversed row pair (`num_rows/2` leaves, the trace-commit +/// layout). Returns `(num_rows / rows_per_leaf) * 32` hash bytes. +pub fn leaves_base( + columns: &[u64], + col_stride: usize, + num_cols: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + ensure!(num_rows.is_power_of_two()); + ensure!(rows_per_leaf == 1 || rows_per_leaf == 2); + ensure!( + num_rows >= 2, + "num_rows must be at least 2 for bit-reversed GPU leaf hashing" + ); + ensure!( + col_stride >= num_rows, + "col_stride must be >= num_rows to keep per-column reads in-bounds" + ); + let total = num_cols + .checked_mul(col_stride) + .expect("num_cols * col_stride overflows usize"); + ensure!(columns.len() >= total); + let be = backend()?; + let stream = be.next_stream(); + let cols_dev = stream.clone_htod(&columns[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / rows_per_leaf) * 32)?; + let launch = if rows_per_leaf == 2 { + launch_leaves_base_row_pair + } else { + launch_leaves_base + }; + launch( + stream.as_ref(), + &cols_dev, + col_stride as u64, + num_cols as u64, + num_rows as u64, + &mut out_dev.as_view_mut(), + )?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Ext3 variant of [`leaves_base`]: columns arrive as three base slabs per ext3 +/// column, so `columns.len() >= num_cols * 3 * col_stride`. Twin of +/// [`crate::blake3::leaves_ext3`]. +pub fn leaves_ext3( + columns: &[u64], + col_stride: usize, + num_cols: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + ensure!(num_rows.is_power_of_two()); + ensure!(rows_per_leaf == 1 || rows_per_leaf == 2); + ensure!( + num_rows >= 2, + "num_rows must be at least 2 for bit-reversed GPU leaf hashing" + ); + ensure!( + col_stride >= num_rows, + "col_stride must be >= num_rows to keep per-column reads in-bounds" + ); + let total = num_cols + .checked_mul(3) + .and_then(|v| v.checked_mul(col_stride)) + .expect("num_cols * 3 * col_stride overflows usize"); + ensure!(columns.len() >= total); + let be = backend()?; + let stream = be.next_stream(); + let cols_dev = stream.clone_htod(&columns[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / rows_per_leaf) * 32)?; + // Row-pair ext3 leaves reuse the comp-poly kernel, as the keccak and BLAKE3 + // paths do: hashing all ext3 columns of rows `2i`, `2i+1` is the same + // traversal whether the columns are called "aux trace" or "parts". + let launch = if rows_per_leaf == 2 { + launch_ext3_row_pair + } else { + launch_leaves_ext3 + }; + launch( + stream.as_ref(), + &cols_dev, + col_stride as u64, + num_cols as u64, + num_rows as u64, + &mut out_dev.as_view_mut(), + )?; + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + +pub(crate) fn launch_leaves_base( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + // The kernel computes `__brevll(tid) >> (64 - log_num_rows)`, which is UB + // for `log_num_rows == 0` (single-row trees are degenerate anyway). + debug_assert!(num_rows >= 2, "rpx leaf kernel: num_rows must be >= 2"); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = rpx_launch_cfg(num_rows); + unsafe { + stream + .launch_builder(&be.rpx_leaves_base_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_leaves_base_row_pair( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "rpx row-pair leaf kernel: num_rows must be >= 2" + ); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + // One thread per leaf (= row pair). + let cfg = rpx_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.rpx_leaves_base_row_pair_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_leaves_ext3( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!(num_rows >= 2, "rpx leaf kernel: num_rows must be >= 2"); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = rpx_launch_cfg(num_rows); + unsafe { + stream + .launch_builder(&be.rpx_leaves_ext3_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_ext3_row_pair( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "rpx row-pair leaf kernel: num_rows must be >= 2" + ); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = rpx_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.rpx_comp_poly_leaves_ext3) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +/// Row-major row-pair leaf hashing: leaf `i` hashes the two consecutive +/// bit-reversed rows `reverse_index(2i)`, `reverse_index(2i+1)`, each `m` lanes +/// read contiguously from the row-major `data`. Matches the CPU +/// `commit_bit_reversed(.., 2)`; twin of [`crate::blake3::leaves_base_row_major_row_pair`]. +/// +/// Returns `(num_rows / 2) * 32` hash bytes. +pub fn leaves_base_row_major_row_pair(data: &[u64], m: usize, num_rows: usize) -> Result> { + leaves_row_major_row_pair_inner(data, m, 0, m, num_rows, false) +} + +/// Column-range variant of [`leaves_base_row_major_row_pair`]: each leaf hashes +/// only columns `[col_start, col_end)` of the row pair, while `m` stays the full +/// row stride. Matches the CPU `commit_rows_bit_reversed_subset`. +pub fn leaves_base_row_major_row_pair_range( + data: &[u64], + m: usize, + col_start: usize, + col_end: usize, + num_rows: usize, +) -> Result> { + leaves_row_major_row_pair_inner(data, m, col_start, col_end, num_rows, true) +} + +fn leaves_row_major_row_pair_inner( + data: &[u64], + m: usize, + col_start: usize, + col_end: usize, + num_rows: usize, + ranged: bool, +) -> Result> { + ensure!(num_rows.is_power_of_two()); + ensure!(num_rows >= 2, "num_rows must be at least 2"); + ensure!( + col_start < col_end && col_end <= m, + "column range in bounds" + ); + let total = num_rows + .checked_mul(m) + .expect("num_rows * m overflows usize"); + ensure!(data.len() >= total); + + let be = backend()?; + let stream = be.next_stream(); + let data_dev = stream.clone_htod(&data[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / 2) * 32)?; + + let m_u64 = m as u64; + let num_rows_u64 = num_rows as u64; + let log_num_rows = num_rows.trailing_zeros() as u64; + if ranged { + launch_leaves_base_row_major_row_pair_range( + stream.as_ref(), + be, + &data_dev, + m_u64, + col_start as u64, + col_end as u64, + num_rows_u64, + log_num_rows, + &mut out_dev.as_view_mut(), + )?; + } else { + launch_leaves_base_row_major_row_pair( + stream.as_ref(), + be, + &data_dev, + m_u64, + num_rows_u64, + log_num_rows, + &mut out_dev.as_view_mut(), + )?; + } + let out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Row-major ROW-PAIR leaf hashing under RPX: leaf `i` hashes the two +/// consecutive bit-reversed rows `reverse_index(2i)`, `reverse_index(2i+1)` +/// (each `m` lanes, read contiguously from the row-major `buf`), producing +/// `num_rows / 2` leaves. Device-buffer twin of the BLAKE3 launcher the fused +/// LDE pipeline dispatches against; matches the CPU `commit_bit_reversed(.., 2)`. +pub(crate) fn launch_leaves_base_row_major_row_pair( + stream: &CudaStream, + be: &Backend, + buf: &CudaSlice, + m: u64, + num_rows: u64, + log_num_rows: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + // The kernel derives rows as `__brevll(2*tid + k) >> (64 - log_num_rows)`; + // a 64-bit shift is UB at `log_num_rows == 0`, so require `num_rows >= 2`. + debug_assert!( + num_rows >= 2, + "row-major row-pair rpx requires num_rows >= 2" + ); + let cfg = rpx_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.rpx_leaves_base_row_major_row_pair) + .arg(buf) + .arg(&m) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// Column-range variant of [`launch_leaves_base_row_major_row_pair`]: leaves +/// hash only columns `[col_start, col_end)` of each bit-reversed row pair +/// (`m` stays the full row stride). Matches the CPU +/// `commit_rows_bit_reversed_subset`. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_leaves_base_row_major_row_pair_range( + stream: &CudaStream, + be: &Backend, + buf: &CudaSlice, + m: u64, + col_start: u64, + col_end: u64, + num_rows: u64, + log_num_rows: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "row-major row-pair rpx requires num_rows >= 2" + ); + debug_assert!( + col_start < col_end && col_end <= m, + "column range in bounds" + ); + let cfg = rpx_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.rpx_leaves_base_row_major_row_pair_range) + .arg(buf) + .arg(&m) + .arg(&col_start) + .arg(&col_end) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// Composition-part leaf hashing under RPX: leaf `i` hashes the ext3 +/// components of every part at the two bit-reversed rows `2i`, `2i+1`, read +/// from per-component slabs with stride `col_stride`. Device-buffer twin of +/// the BLAKE3 launch the comp-poly tree build dispatches against. +#[allow(clippy::too_many_arguments)] +pub(crate) fn launch_comp_poly_leaves_ext3( + stream: &CudaStream, + be: &Backend, + buf: &CudaSlice, + col_stride: u64, + num_parts: u64, + num_rows: u64, + log_num_rows: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!(num_rows >= 2, "comp-poly rpx leaves require num_rows >= 2"); + let cfg = rpx_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.rpx_comp_poly_leaves_ext3) + .arg(buf) + .arg(&col_stride) + .arg(&num_parts) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// FRI-layer leaf hashing under RPX: leaf `i` hashes the two consecutive ext3 +/// evals `2i`, `2i+1` of an interleaved eval vector (six felts — one block). +/// Device-buffer twin of the BLAKE3 launch the FRI layer commit dispatches +/// against; the host is `AlgebraicPairBackend::hash_data`. +pub(crate) fn launch_fri_leaves_ext3( + stream: &CudaStream, + be: &Backend, + evals: &CudaSlice, + num_leaves: u64, + leaves_out: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + let cfg = rpx_launch_cfg(num_leaves); + unsafe { + stream + .launch_builder(&be.rpx_fri_leaves_ext3) + .arg(evals) + .arg(&num_leaves) + .arg(leaves_out) + .launch(cfg)?; + } + Ok(()) +} + +/// Walk the inner Merkle tree on device under RPX. `nodes_dev` already has the +/// `leaves_len` hashed leaves written into the tail; this fills in the inner +/// nodes bottom-up. Twin of [`crate::blake3::build_inner_tree_levels`], with +/// the same tail cutover: one single-block launch takes over once a level is no +/// wider than the block, where per-level launch overhead dominates the work. +/// +/// The walk is [`TreeWalk::from_switch`]'s: the half-warp levels and tail by +/// default, a thread per parent under `LAMBDA_VM_RPX_WARP_MERKLE=0` — the same +/// nodes, byte for byte. +pub(crate) fn build_inner_tree_levels( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + leaves_len: usize, +) -> Result<()> { + build_inner_tree_levels_with(stream, be, nodes_dev, leaves_len, TreeWalk::from_switch()) +} + +/// Which kernels walk an RPX tree's inner levels. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TreeWalk { + /// `rpx_merkle_level` per level (a thread per parent), then + /// `rpx_merkle_tail` (one 128-thread block) from 128 pairs up. + ThreadPerParent, + /// `rpx_merkle_level` while a level is wider than `level_max_pairs`; + /// `rpx_merkle_level_warp` (a half-warp per parent) down to + /// `tail_max_pairs`; then ONE `rpx_merkle_tail_warp` block for the rest. + Warp { + level_max_pairs: u64, + tail_max_pairs: u64, + }, +} + +impl TreeWalk { + /// The default walk. A level of up to 16,384 pairs is one latency of the + /// half-warp permutation (or close to it) where a thread per parent pays + /// one single-thread chain (~119 µs at the block's clock, NCU-5090 §2); + /// wider levels already run a thread per parent at the leaf rate. The tail + /// takes the last 64 pairs, one pass of a 1,024-thread block per level. + pub const WARP: Self = Self::Warp { + level_max_pairs: 16_384, + tail_max_pairs: TAIL_WARP_MAX_PAIRS, + }; + + /// The walk every production tree takes: [`Self::WARP`], or + /// [`Self::ThreadPerParent`] when `rpx_paths::warp_merkle` is off. + pub fn from_switch() -> Self { + if crate::rpx_paths::warp_merkle() { + Self::WARP + } else { + Self::ThreadPerParent + } + } +} + +/// Pairs one `rpx_merkle_tail_warp` block of 1,024 threads covers in one pass +/// (32 warps, two parents each); the kernel's `__launch_bounds__`. +pub const TAIL_WARP_MAX_PAIRS: u64 = 64; + +/// Threads per block for `rpx_merkle_level_warp`: four warps, eight parents. +const LEVEL_WARP_BLOCK_DIM: u32 = 128; + +/// [`build_inner_tree_levels`] under an explicit walk, so tests and benches +/// can run both in one process. +pub fn build_inner_tree_levels_with( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + leaves_len: usize, + walk: TreeWalk, +) -> Result<()> { + let (level_max_pairs, tail_max_pairs) = match walk { + TreeWalk::ThreadPerParent => { + return build_inner_tree_levels_per_parent(stream, be, nodes_dev, leaves_len); + } + TreeWalk::Warp { + level_max_pairs, + tail_max_pairs, + } => ( + level_max_pairs, + tail_max_pairs.clamp(1, TAIL_WARP_MAX_PAIRS), + ), + }; + let mut level_begin: u64 = (leaves_len - 1) as u64; + while level_begin != 0 { + let new_begin = level_begin / 2; + let n_pairs = level_begin - new_begin; + if n_pairs <= tail_max_pairs { + launch_merkle_tail_warp(stream, be, nodes_dev, level_begin, n_pairs)?; + return Ok(()); + } + if n_pairs <= level_max_pairs { + launch_merkle_level_warp(stream, be, nodes_dev, new_begin, n_pairs)?; + } else { + launch_merkle_level(stream, be, nodes_dev, new_begin, n_pairs)?; + } + level_begin = new_begin; + } + Ok(()) +} + +fn launch_merkle_level( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + parent_begin: u64, + n_pairs: u64, +) -> Result<()> { + unsafe { + stream + .launch_builder(&be.rpx_merkle_level) + .arg(&mut *nodes_dev) + .arg(&parent_begin) + .arg(&n_pairs) + .launch(rpx_launch_cfg(n_pairs))?; + } + Ok(()) +} + +fn launch_merkle_level_warp( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + parent_begin: u64, + n_pairs: u64, +) -> Result<()> { + // Two parents per warp. + let pairs_per_block = (LEVEL_WARP_BLOCK_DIM / 16) as u64; + let cfg = LaunchConfig { + grid_dim: (n_pairs.div_ceil(pairs_per_block) as u32, 1, 1), + block_dim: (LEVEL_WARP_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.rpx_merkle_level_warp) + .arg(&mut *nodes_dev) + .arg(&parent_begin) + .arg(&n_pairs) + .launch(cfg)?; + } + Ok(()) +} + +/// Every level from `level_begin` (whose width is `n_pairs`) to the root in one +/// block, sized to the widest of those levels: a warp per two of its pairs, up +/// to 1,024 threads. +fn launch_merkle_tail_warp( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + level_begin: u64, + n_pairs: u64, +) -> Result<()> { + let warps = n_pairs.div_ceil(2).clamp(1, TAIL_WARP_MAX_PAIRS / 2) as u32; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (32 * warps, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.rpx_merkle_tail_warp) + .arg(&mut *nodes_dev) + .arg(&level_begin) + .launch(cfg)?; + } + Ok(()) +} + +fn build_inner_tree_levels_per_parent( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + leaves_len: usize, +) -> Result<()> { + const TAIL_MAX_PAIRS: u64 = RPX_BLOCK_DIM as u64; + let mut level_begin: u64 = (leaves_len - 1) as u64; + while level_begin != 0 { + let new_begin = level_begin / 2; + let n_pairs = level_begin - new_begin; + if n_pairs <= TAIL_MAX_PAIRS { + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (RPX_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.rpx_merkle_tail) + .arg(&mut *nodes_dev) + .arg(&level_begin) + .launch(cfg)?; + } + return Ok(()); + } + let cfg = rpx_launch_cfg(n_pairs); + unsafe { + stream + .launch_builder(&be.rpx_merkle_level) + .arg(&mut *nodes_dev) + .arg(&new_begin) + .arg(&n_pairs) + .launch(cfg)?; + } + level_begin = new_begin; + } + Ok(()) +} + +/// Given `hashed_leaves` of length `leaves_len * 32`, build the full RPX +/// Merkle tree on device and return the `(2*leaves_len - 1) * 32`-byte node +/// buffer in the standard layout: `nodes[0..leaves_len - 1]` are inner nodes +/// (root at index 0) and `nodes[leaves_len - 1..]` are the leaves themselves. +/// +/// Matches the CPU `crypto/crypto/src/merkle_tree/merkle.rs` construction, so +/// the result plugs into `MerkleTree::from_precomputed_nodes` the same way +/// [`crate::blake3::build_merkle_tree_on_device`]'s does. +/// +/// `leaves_len` must be a power of two and >= 2. +pub fn build_merkle_tree_on_device(hashed_leaves: &[u8]) -> Result> { + build_merkle_tree_on_device_with(hashed_leaves, TreeWalk::from_switch()) +} + +/// [`build_merkle_tree_on_device`] under an explicit [`TreeWalk`] (the parity +/// tests build every tree both ways in one process). +pub fn build_merkle_tree_on_device_with(hashed_leaves: &[u8], walk: TreeWalk) -> Result> { + ensure!(hashed_leaves.len().is_multiple_of(32)); + let leaves_len = hashed_leaves.len() / 32; + ensure!(leaves_len >= 2, "tree needs at least two leaves"); + ensure!( + leaves_len.is_power_of_two(), + "leaves_len must be a power of two" + ); + + let total_nodes = 2 * leaves_len - 1; + let be = backend()?; + let stream = be.next_stream(); + + // SAFETY: every byte is written before it is read — leaves by the H2D + // below, inner nodes by the level walk that follows. + let mut nodes_dev = unsafe { stream.alloc::(total_nodes * 32) }?; + let leaves_offset_bytes = (leaves_len - 1) * 32; + { + let mut slice = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + hashed_leaves.len()); + stream.memcpy_htod(hashed_leaves, &mut slice)?; + } + + build_inner_tree_levels_with(stream.as_ref(), be, &mut nodes_dev, leaves_len, walk)?; + + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Build the composition Merkle tree under RPX straight from a device-resident +/// slab buffer (`3*m` slabs of `lde_size` u64s, component `k` of part `c` at +/// `(c*3 + k) * lde_size` — the [`crate::lde::GpuLdeExt3`] layout). No host +/// staging and no H2D: the leaf kernel reads `buf` in place on `stream`. +/// +/// Twin of [`crate::blake3::build_comp_poly_tree_from_slabs_dev`]. +pub fn build_comp_poly_tree_from_slabs_dev( + stream: &Arc, + buf: &CudaSlice, + m: usize, + lde_size: usize, +) -> Result { + build_comp_poly_tree_from_slabs_dev_rpl(stream, buf, m, lde_size, 2) +} + +/// [`build_comp_poly_tree_from_slabs_dev`] with `rows_per_leaf` rows per leaf +/// (2 = row pair, 1 = S2 one row: `lde_size` leaves, the one-row ext3 kernel). +pub fn build_comp_poly_tree_from_slabs_dev_rpl( + stream: &Arc, + buf: &CudaSlice, + m: usize, + lde_size: usize, + rows_per_leaf: usize, +) -> Result { + ensure!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "rows_per_leaf must be 1 or 2" + ); + // Same sticky hook as the keccak and BLAKE3 twins: the comp-tree cliff test + // arms one counter and must reach it under whichever hash the build pins. + #[cfg(feature = "test-faults")] + crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; + ensure!(m > 0); + ensure!(lde_size.is_power_of_two() && lde_size >= 2); + ensure_eq!(buf.len(), 3 * m * lde_size, "slab buffer shape"); + let num_leaves = lde_size / rows_per_leaf; + let tight_total_nodes = 2 * num_leaves - 1; + let be = backend()?; + + // SAFETY: every byte is written before it is read — leaves by the kernel + // below, inner nodes by the level walk after it. + let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; + let leaves_offset_bytes = (num_leaves - 1) * 32; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); + let launch = if rows_per_leaf == 2 { + launch_ext3_row_pair + } else { + launch_leaves_ext3 + }; + launch( + stream.as_ref(), + buf, + lde_size as u64, + m as u64, + lde_size as u64, + &mut leaves_view, + )?; + } + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + let mut root = [0u8; 32]; + crate::readback::memcpy_dtoh(stream, &nodes_dev.slice(0..32), &mut root)?; + stream.synchronize()?; + Ok(crate::lde::GpuMerkleTree { + nodes: Arc::new(nodes_dev), + leaves_len: num_leaves, + arity: 2, + root, + }) +} + +/// Build the composition Merkle tree under RPX from host-side interleaved ext3 +/// parts, keeping the nodes device-resident so openings can gather paths on +/// device. `parts_interleaved` is `num_parts` slices, each `[a0,a1,a2,b0,b1,b2,…]` +/// of length `3*lde_size`. Leaves hash row pairs, so `leaves_len = lde_size / 2`. +/// +/// Twin of [`crate::blake3::build_comp_poly_tree_from_evals_ext3_keep`], and it +/// stages through the same pinned de-interleave buffer for the same reason. +pub fn build_comp_poly_tree_from_evals_ext3_keep( + parts_interleaved: &[&[u64]], +) -> Result { + build_comp_poly_tree_from_evals_ext3_keep_rpl(parts_interleaved, 2) +} + +/// [`build_comp_poly_tree_from_evals_ext3_keep`] with `rows_per_leaf` rows per +/// leaf (2 = row pair, 1 = S2 one row). +pub fn build_comp_poly_tree_from_evals_ext3_keep_rpl( + parts_interleaved: &[&[u64]], + rows_per_leaf: usize, +) -> Result { + #[cfg(feature = "test-faults")] + crate::faults::check_sticky(&crate::faults::FAULT_COMP_TREE_STICKY)?; + ensure!(!parts_interleaved.is_empty()); + let m = parts_interleaved.len(); + let ext3_elems = parts_interleaved[0].len() / 3; + ensure_eq!( + parts_interleaved[0].len(), + 3 * ext3_elems, + "ext3 buffer length must be 3 * lde_size" + ); + for p in parts_interleaved.iter() { + ensure_eq!(p.len(), 3 * ext3_elems); + } + let lde_size = ext3_elems; + ensure!(lde_size.is_power_of_two() && lde_size >= 2); + + let be = backend()?; + let stream = be.next_stream(); + let staging_slot = be.pinned_staging(); + + // Stage: de-interleave each part into 3 base slabs in pinned memory. + let mb = 3 * m; + let mut staging = staging_slot.lock().unwrap(); + staging.ensure_capacity(mb * lde_size, &be.ctx)?; + let pinned = unsafe { staging.as_mut_slice(mb * lde_size) }; + + pack_ext3_to_pinned_slabs(parts_interleaved, pinned, lde_size); + + // H2D the de-interleaved parts, then release the staging lock: the tree + // build reads the device `buf`, not `pinned`. Synchronize first so the async + // H2D has consumed `pinned` before it can be freed or reused. + let mut buf = stream.alloc_zeros::(mb * lde_size)?; + stream.memcpy_htod(&pinned[..mb * lde_size], &mut buf)?; + stream.synchronize()?; + drop(staging); + + build_comp_poly_tree_from_slabs_dev_rpl(&stream, &buf, m, lde_size, rows_per_leaf) +} + +/// Build a FRI-layer Merkle tree on device under RPX from an interleaved ext3 +/// eval vector, returning the full host node buffer so tests can compare it byte +/// for byte against the CPU `AlgebraicPairBackend` tree. Each leaf hashes two +/// consecutive ext3 values; `num_leaves = evals.len() / 6`. Returns +/// `(2*num_leaves - 1) * 32` bytes in standard layout. +/// +/// Twin of [`crate::blake3::build_fri_layer_tree_from_evals_ext3`], and like it +/// a parity harness rather than a production path: production folds and commits +/// through [`crate::fri::FriCommitState::fold_and_commit_layer`], which +/// dispatches to the same two kernels. +pub fn build_fri_layer_tree_from_evals_ext3(evals: &[u64]) -> Result> { + ensure!( + evals.len().is_multiple_of(6), + "evals must hold whole pair-leaves" + ); + let num_evals = evals.len() / 3; + let num_leaves = num_evals / 2; + ensure!(num_leaves.is_power_of_two() && num_leaves >= 2); + let tight_total_nodes = 2 * num_leaves - 1; + + let be = backend()?; + let stream = be.next_stream(); + + let evals_dev = stream.clone_htod(evals)?; + // SAFETY: leaves are written by the kernel below, inner nodes by the level + // walk after it, before either is read. + let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; + + let leaves_offset_bytes = (num_leaves - 1) * 32; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); + launch_fri_leaves_ext3( + stream.as_ref(), + be, + &evals_dev, + num_leaves as u64, + &mut leaves_view, + )?; + } + + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + + let out = crate::readback::clone_dtoh(&stream, &nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Parity harness: run the device permutation over `states` and return each +/// output state, canonical. +/// +/// Not a production path — the bare device permutation is otherwise +/// unreachable from host code, so without this there would be nothing to check +/// it against the host `Rpx256` (or the host-KAT's oracle tables) with before a +/// whole tree is built. Inputs may be raw `[0, 2^64)` storage. +pub fn permute_probe(states: &[[u64; STATE_FELTS]]) -> Result> { + if states.is_empty() { + return Ok(Vec::new()); + } + let n = states.len(); + let flat: Vec = states.iter().flatten().copied().collect(); + let be = backend()?; + let stream = be.next_stream(); + let states_dev = stream.clone_htod(&flat)?; + let mut out_dev = stream.alloc_zeros::(n * STATE_FELTS)?; + let n_u64 = n as u64; + let cfg = rpx_launch_cfg(n_u64); + unsafe { + stream + .launch_builder(&be.rpx_permute_probe) + .arg(&states_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let flat_out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(flat_out + .chunks_exact(STATE_FELTS) + .map(|c| { + let mut s = [0u64; STATE_FELTS]; + s.copy_from_slice(c); + s + }) + .collect()) +} + +/// One point of the round-3 occupancy discriminator: the pure permutation's +/// throughput and the register cap the current build actually achieved. +/// +/// NOT a production path. It exists to answer whether the RPX permutation is +/// latency-bound (raising occupancy — fewer registers per thread, more blocks +/// per SM — speeds it) or compute/issue-bound (it does not), which decides round +/// 3's landing lever. The register cap is a whole-cubin `-maxrregcount` set at +/// build time (`LAMBDA_VM_RPX_MAXRREGCOUNT`); one build gives one point, so the +/// caller runs this once per cap and reads the curve across builds. +#[derive(Clone, Copy, Debug)] +pub struct ProbeSweepPoint { + /// States permuted per launch. + pub n: u64, + /// Timed launches (excludes a warm-up launch). + pub iters: u64, + /// Wall time for the `iters` launches (host-to-device copy done ONCE before + /// the clock, so the cross-build delta is kernel time, not transfer). + pub secs: f64, + /// The probe kernel's registers/thread as compiled — the ACHIEVED cap, read + /// back from the loaded function, not the requested `-maxrregcount`. + pub regs: u32, + /// Blocks/SM the driver reports resident at [`RPX_BLOCK_DIM`] for that cap. + pub blocks_per_sm: u32, +} + +/// Time `iters` launches of the permute probe over `n` states and report the +/// achieved register cap. See [`ProbeSweepPoint`]. Deterministic dummy states; +/// the outputs are not checked here (parity is [`permute_probe`]'s job, and a +/// register cap cannot change a result — it only moves spills and occupancy). +pub fn permute_probe_sweep(n: usize, iters: usize) -> Result { + let be = backend()?; + let stream = be.next_stream(); + let flat: Vec = (0..(n as u64) * (STATE_FELTS as u64)).collect(); + let states_dev = stream.clone_htod(&flat)?; + let mut out_dev = stream.alloc_zeros::(n * STATE_FELTS)?; + let n_u64 = n as u64; + let cfg = rpx_launch_cfg(n_u64); + // Warm-up (cubin resident, caches primed), excluded by name. + unsafe { + stream + .launch_builder(&be.rpx_permute_probe) + .arg(&states_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + stream.synchronize()?; + let start = std::time::Instant::now(); + for _ in 0..iters { + unsafe { + stream + .launch_builder(&be.rpx_permute_probe) + .arg(&states_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + } + stream.synchronize()?; + let secs = start.elapsed().as_secs_f64(); + let regs = be.rpx_permute_probe.num_regs().unwrap_or(0).max(0) as u32; + let blocks_per_sm = be + .rpx_permute_probe + .occupancy_max_active_blocks_per_multiprocessor(RPX_BLOCK_DIM, 0, None) + .unwrap_or(0); + Ok(ProbeSweepPoint { + n: n_u64, + iters: iters as u64, + secs, + regs, + blocks_per_sm, + }) +} + +// =========================================================================== +// Parity probes and benches for the half-warp Merkle kernels and the +// permutation variants. None of these is a proving path; each exists so a test +// or a bench can run both sides of a `rpx_paths` switch in one process. +// =========================================================================== + +/// Parity harness: the half-warp permutation (`rpx_permute_warp_probe`, +/// this cubin's variant) over `states`, each output canonical — to be compared +/// with [`permute_probe`] and the host `Rpx256`, raw. +pub fn permute_warp_probe(states: &[[u64; STATE_FELTS]]) -> Result> { + if states.is_empty() { + return Ok(Vec::new()); + } + let n = states.len(); + let flat: Vec = states.iter().flatten().copied().collect(); + let be = backend()?; + let stream = be.next_stream(); + let states_dev = stream.clone_htod(&flat)?; + let mut out_dev = stream.alloc_zeros::(n * STATE_FELTS)?; + let n_u64 = n as u64; + // Two states per warp, four warps per block. + let cfg = LaunchConfig { + grid_dim: (n.div_ceil(8) as u32, 1, 1), + block_dim: (128, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.rpx_permute_warp_probe) + .arg(&states_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let flat_out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(flat_out + .chunks_exact(STATE_FELTS) + .map(|c| core::array::from_fn(|i| c[i])) + .collect()) +} + +/// The permutation variants this cubin carries a chained probe for. +pub fn chain_probe_variants() -> Result> { + Ok(backend()? + .rpx_permute_chain_probes + .iter() + .map(|(v, _)| *v) + .collect()) +} + +fn chain_probe_kernel(be: &Backend, variant: u32) -> &cudarc::driver::CudaFunction { + &be.rpx_permute_chain_probes + .iter() + .find(|(v, _)| *v == variant) + .unwrap_or_else(|| panic!("no chained permutation probe for variant {variant}")) + .1 +} + +/// Parity harness: `k` chained permutations of each state at +/// `variant` (`rpx_permute_chain_probe_v`), raw. +pub fn permute_chain_probe( + variant: u32, + states: &[[u64; STATE_FELTS]], + k: u64, +) -> Result> { + if states.is_empty() { + return Ok(Vec::new()); + } + let n = states.len(); + let flat: Vec = states.iter().flatten().copied().collect(); + let be = backend()?; + let stream = be.next_stream(); + let states_dev = stream.clone_htod(&flat)?; + let mut out_dev = stream.alloc_zeros::(n * STATE_FELTS)?; + let n_u64 = n as u64; + unsafe { + stream + .launch_builder(chain_probe_kernel(be, variant)) + .arg(&states_dev) + .arg(&n_u64) + .arg(&k) + .arg(&mut out_dev) + .launch(rpx_launch_cfg(n_u64))?; + } + let flat_out = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(flat_out + .chunks_exact(STATE_FELTS) + .map(|c| core::array::from_fn(|i| c[i])) + .collect()) +} + +/// One timed point of the chained probe at one variant. +#[derive(Clone, Copy, Debug)] +pub struct ChainBench { + pub variant: u32, + /// States (threads) per launch. + pub n: u64, + /// Permutations per thread per launch. + pub k: u64, + /// Timed launches (a warm-up launch is excluded). + pub iters: u64, + pub secs: f64, + /// The probe kernel's registers/thread and blocks/SM, as compiled. + pub regs: u32, + pub blocks_per_sm: u32, +} + +impl ChainBench { + /// Nanoseconds per permutation over the whole card. + pub fn ns_per_perm(&self) -> f64 { + self.secs * 1e9 / (self.n * self.k * self.iters) as f64 + } +} + +/// Threads of [`RPX_BLOCK_DIM`] the chained probe at `variant` keeps +/// resident on this card (multiprocessors × the driver's occupancy for that +/// kernel), so a bench can launch whole waves: variants differ in registers, +/// and a fixed launch size would give each a different wave tail. +pub fn chain_probe_resident_threads(variant: u32) -> Result { + use cudarc::driver::sys::CUdevice_attribute; + let be = backend()?; + // The occupancy query needs the context current on this thread, and cudarc + // does not bind it for that call (see `grinding::queue_grid`). + be.ctx.bind_to_thread()?; + let sms = be + .ctx + .attribute(CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)? + .max(1) as u64; + let per_sm = chain_probe_kernel(be, variant) + .occupancy_max_active_blocks_per_multiprocessor(RPX_BLOCK_DIM, 0, None)? + .max(1) as u64; + Ok(sms * per_sm * RPX_BLOCK_DIM as u64) +} + +/// Bench: `iters` launches of `n` threads × `k` chained permutations +/// at `variant`, after one excluded warm-up; the states are copied in once. +pub fn permute_chain_bench(variant: u32, n: usize, k: u64, iters: usize) -> Result { + let be = backend()?; + let kernel = chain_probe_kernel(be, variant); + let stream = be.next_stream(); + let flat: Vec = (0..(n as u64) * (STATE_FELTS as u64)).collect(); + let states_dev = stream.clone_htod(&flat)?; + let mut out_dev = stream.alloc_zeros::(n * STATE_FELTS)?; + let n_u64 = n as u64; + let cfg = rpx_launch_cfg(n_u64); + let launch = |out: &mut CudaSlice| -> Result<()> { + unsafe { + stream + .launch_builder(kernel) + .arg(&states_dev) + .arg(&n_u64) + .arg(&k) + .arg(out) + .launch(cfg)?; + } + Ok(()) + }; + launch(&mut out_dev)?; + stream.synchronize()?; + let start = std::time::Instant::now(); + for _ in 0..iters { + launch(&mut out_dev)?; + } + stream.synchronize()?; + let secs = start.elapsed().as_secs_f64(); + Ok(ChainBench { + variant, + n: n_u64, + k, + iters: iters as u64, + secs, + regs: kernel.num_regs().unwrap_or(0).max(0) as u32, + blocks_per_sm: kernel + .occupancy_max_active_blocks_per_multiprocessor(RPX_BLOCK_DIM, 0, None) + .unwrap_or(0), + }) +} + +/// Parity harness: `[mul_limb(a, b), sqr_limb(a), goldilocks::mul(a, b)]` +/// per pair, computed by the device PTX. +pub fn limb_probe(a: &[u64], b: &[u64]) -> Result> { + ensure_eq!( + a.len(), + b.len(), + "limb_probe: operand vectors differ in length" + ); + if a.is_empty() { + return Ok(Vec::new()); + } + let n = a.len(); + let be = backend()?; + let stream = be.next_stream(); + let a_dev = stream.clone_htod(a)?; + let b_dev = stream.clone_htod(b)?; + let mut out_dev = stream.alloc_zeros::(3 * n)?; + let n_u64 = n as u64; + unsafe { + stream + .launch_builder(&be.rpx_limb_probe) + .arg(&a_dev) + .arg(&b_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(rpx_launch_cfg(n_u64))?; + } + let flat = crate::readback::clone_dtoh(&stream, &out_dev)?; + stream.synchronize()?; + Ok(flat.chunks_exact(3).map(|c| [c[0], c[1], c[2]]).collect()) +} + +/// Which kernel [`level_bench`] times. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LevelKernel { + /// `rpx_merkle_level`: a thread per parent. + ThreadPerParent, + /// `rpx_merkle_level_warp`: a half-warp per parent. + Warp, + /// `rpx_merkle_tail`: one 128-thread block, every level from `n_pairs` up. + BlockTail, + /// `rpx_merkle_tail_warp`: one block of a warp per two pairs (≤ 1,024 + /// threads), every level from `n_pairs` up. + WarpTail, +} + +/// Random node bytes for a tree of `leaves_len` leaves, on the device. Any +/// bytes are fair for timing and for kernel-versus-kernel parity: the kernels +/// decode big-endian words without reducing them. +fn random_tree_dev( + stream: &Arc, + leaves_len: usize, + seed: u64, +) -> Result> { + // Only the leaf region carries data (a word at a time); every inner node + // is written by a walk before it is read. + let mut host = vec![0u8; (2 * leaves_len - 1) * 32]; + let mut s = seed; + for word in host[(leaves_len - 1) * 32..].chunks_exact_mut(8) { + s = s.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + word.copy_from_slice(&(z ^ (z >> 31)).to_le_bytes()); + } + stream.clone_htod(&host) +} + +/// Bench: seconds per launch of one level kernel at `n_pairs` (a +/// power of two), `iters` launches back to back on one stream after an excluded +/// warm-up — launch gaps included, as in a tree walk. A tail kernel runs every +/// level from `n_pairs` up to the root. +pub fn level_bench(kind: LevelKernel, n_pairs: u64, iters: usize) -> Result { + ensure!( + n_pairs.is_power_of_two(), + "level_bench: n_pairs must be a power of two" + ); + let be = backend()?; + let stream = be.next_stream(); + // The bottom level of a tree of 2·n_pairs leaves. + let leaves_len = (2 * n_pairs) as usize; + let mut nodes = random_tree_dev(&stream, leaves_len, n_pairs)?; + let level_begin = (leaves_len - 1) as u64; + let parent_begin = level_begin / 2; + let once = |nodes: &mut CudaSlice| -> Result<()> { + match kind { + LevelKernel::ThreadPerParent => { + launch_merkle_level(&stream, be, nodes, parent_begin, n_pairs) + } + LevelKernel::Warp => { + launch_merkle_level_warp(&stream, be, nodes, parent_begin, n_pairs) + } + LevelKernel::BlockTail => { + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (RPX_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.rpx_merkle_tail) + .arg(&mut *nodes) + .arg(&level_begin) + .launch(cfg)?; + } + Ok(()) + } + LevelKernel::WarpTail => { + launch_merkle_tail_warp(&stream, be, nodes, level_begin, n_pairs) + } + } + }; + once(&mut nodes)?; + stream.synchronize()?; + let start = std::time::Instant::now(); + for _ in 0..iters { + once(&mut nodes)?; + } + stream.synchronize()?; + Ok(start.elapsed().as_secs_f64() / iters as f64) +} + +/// Bench: seconds per inner-tree walk of `leaves_len` random leaves +/// under `walk`, `iters` walks back to back after an excluded warm-up (the +/// leaves stay in place, so every walk is the same work). +pub fn tree_bench(leaves_len: usize, walk: TreeWalk, iters: usize) -> Result { + ensure!(leaves_len.is_power_of_two() && leaves_len >= 2); + let be = backend()?; + let stream = be.next_stream(); + let mut nodes = random_tree_dev(&stream, leaves_len, leaves_len as u64)?; + build_inner_tree_levels_with(stream.as_ref(), be, &mut nodes, leaves_len, walk)?; + stream.synchronize()?; + let start = std::time::Instant::now(); + for _ in 0..iters { + build_inner_tree_levels_with(stream.as_ref(), be, &mut nodes, leaves_len, walk)?; + } + stream.synchronize()?; + Ok(start.elapsed().as_secs_f64() / iters as f64) +} diff --git a/crypto/math-cuda/src/rpx_paths.rs b/crypto/math-cuda/src/rpx_paths.rs new file mode 100644 index 000000000..f5adc569b --- /dev/null +++ b/crypto/math-cuda/src/rpx_paths.rs @@ -0,0 +1,167 @@ +//! Which RPX device path each of three switches takes: the Merkle tree tops, +//! the proof-of-work grind, and the permutation's field multiply. +//! +//! Each switch has its default stated ONCE, as a `*_DEFAULT` constant below, +//! and one variable that overrides it (`1` on, `0` off, unset = the default), +//! read once per process. A pipeline that needs one of them off by default +//! flips that constant, one line, the way `lfm::chunking`'s +//! `HASH_SPLIT_DEFAULT` differs between the two pipelines. +//! +//! Every setting produces the same bytes: the nodes, roots, grind nonces and +//! digests are the other path's, which `prover/tests/rpx_device_paths.rs` pins +//! path against path and against the host oracle. These switch speed, never +//! the proof: a block proved either way has the same program ids. +//! +//! Each prints one line the first time it is read, so every log says which +//! paths it ran — on, then off: +//! +//! - `LAMBDA_VM_RPX_WARP_MERKLE`: +//! `[gpu] RPX Merkle: a half-warp per parent up to 16384 pairs, then one +//! 1024-thread tail block (…)` / +//! `[gpu] RPX Merkle: a thread per parent, then one 128-thread tail block (…)` +//! - `LAMBDA_VM_RPX_GRIND_QUEUE`: +//! `[gpu] RPX grind: the work-queue kernel on a card-filling grid (…)` / +//! `[gpu] RPX grind: the stride kernel on LAMBDA_VM_GRIND_GRID blocks (…)` +//! - `LAMBDA_VM_RPX_LIMB_PERMUTE`: +//! `[gpu] RPX permutation: variant 5, 32-bit limb multiplies, squarings +//! unrolled by four (rpx_v5.cubin) (…)` / +//! `[gpu] RPX permutation: variant 0, the 64-bit multiply (rpx_v0.cubin) (…)` +//! +//! `(…)` reads `the pipeline default` when the variable is unset, else the +//! variable and its value. The grind queue adds its grid once it has read it +//! (`[gpu] RPX grind queue: grid …`). Any value other than `0` or `1` aborts +//! and names what is accepted: a typo read as "unset" would turn one arm of an +//! A/B into the other. + +use std::env::VarError; +use std::sync::OnceLock; + +// `LIMB_PERMUTE_VARIANT`: the permutation variant of the default RPX cubin. +// Generated by build.rs from the variant it compiles, so the two cannot drift. +include!(concat!(env!("OUT_DIR"), "/rpx_variants.rs")); + +/// The Merkle tops: levels of up to 16,384 pairs through +/// `rpx_merkle_level_warp` (a half-warp per parent) and the last 64 pairs +/// through one `rpx_merkle_tail_warp` block, in every RPX tree walk (the STARK +/// trace, aux, composition and FRI trees, the WHIR trees and their query-time +/// rebuilds). Off: `rpx_merkle_level` at every level and `rpx_merkle_tail` +/// from 128 pairs up. +pub const WARP_MERKLE_DEFAULT: bool = true; + +/// The RPX grind: `rpx_grind_search_queue`, warps claiming 32 nonces at a time +/// from a queue head, on a grid of multiprocessors × the kernel's occupancy. +/// Off: `rpx_grind_search`'s stride walk on `LAMBDA_VM_GRIND_GRID` blocks. The +/// same smallest nonce either way. +pub const GRIND_QUEUE_DEFAULT: bool = true; + +/// The permutation's field multiply: every RPX kernel from +/// `rpx_v.cubin` (32-bit limb multiplies with carry +/// chains, `square_n` unrolled by four). Off: `rpx_v0.cubin`, the 64-bit +/// multiply. Both cubins are always built, so flipping this is one line. +pub const LIMB_PERMUTE_DEFAULT: bool = true; + +pub const WARP_MERKLE_ENV: &str = "LAMBDA_VM_RPX_WARP_MERKLE"; +pub const GRIND_QUEUE_ENV: &str = "LAMBDA_VM_RPX_GRIND_QUEUE"; +pub const LIMB_PERMUTE_ENV: &str = "LAMBDA_VM_RPX_LIMB_PERMUTE"; + +/// Whether RPX Merkle walks take the half-warp levels and the one-block tail. +pub fn warp_merkle() -> bool { + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| { + read( + WARP_MERKLE_ENV, + WARP_MERKLE_DEFAULT, + "RPX Merkle: a half-warp per parent up to 16384 pairs, then one 1024-thread tail block", + "RPX Merkle: a thread per parent, then one 128-thread tail block", + ) + }) +} + +/// Whether the RPX grind takes the work-queue kernel. +pub fn grind_queue() -> bool { + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| { + read( + GRIND_QUEUE_ENV, + GRIND_QUEUE_DEFAULT, + "RPX grind: the work-queue kernel on a card-filling grid", + "RPX grind: the stride kernel on LAMBDA_VM_GRIND_GRID blocks", + ) + }) +} + +/// Whether the RPX module loads at [`LIMB_PERMUTE_VARIANT`] rather than at +/// variant 0. Read once, when the backend loads its modules. +pub fn limb_permute() -> bool { + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| { + read( + LIMB_PERMUTE_ENV, + LIMB_PERMUTE_DEFAULT, + &format!( + "RPX permutation: variant {LIMB_PERMUTE_VARIANT}, 32-bit limb multiplies, \ + squarings unrolled by four (rpx_v{LIMB_PERMUTE_VARIANT}.cubin)" + ), + "RPX permutation: variant 0, the 64-bit multiply (rpx_v0.cubin)", + ) + }) +} + +/// One switch resolved and printed: the variable if it is set, else the +/// default. A value the switch does not accept aborts. The WHIR retention's +/// switch (`whir::keep_futile`) reads through here too. +pub(crate) fn read(env: &str, default: bool, on: &str, off: &str) -> bool { + let (value, source) = match std::env::var(env) { + Err(VarError::NotPresent) => (default, "the pipeline default".to_string()), + Ok(raw) => match parse(raw.trim()) { + Some(value) => (value, format!("{env}={}", u8::from(value))), + None => refuse(env, &format!("{raw:?}"), default), + }, + Err(VarError::NotUnicode(raw)) => refuse(env, &format!("{raw:?}"), default), + }; + eprintln!("[gpu] {} ({source})", if value { on } else { off }); + value +} + +/// eprintln then abort, as the grind knobs do: a configuration error at +/// startup needs the accepted values, not a backtrace. +fn refuse(env: &str, raw: &str, default: bool) -> ! { + eprintln!( + "{env}={raw} is not a value this switch accepts. Accepted: 1 (on) or 0 (off); unset \ + is the pipeline default ({}).", + u8::from(default) + ); + std::process::abort() +} + +/// `1` is on and `0` is off; anything else is refused. +fn parse(raw: &str) -> Option { + match raw { + "1" => Some(true), + "0" => Some(false), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::{LIMB_PERMUTE_VARIANT, parse}; + + #[test] + fn a_switch_accepts_only_zero_and_one() { + assert_eq!(parse("1"), Some(true)); + assert_eq!(parse("0"), Some(false)); + for raw in ["", "on", "off", "true", "2", "01", "yes", "-1", "+1"] { + assert_eq!(parse(raw), None, "{raw:?} must be refused, not read"); + } + } + + /// The variant the limb cubin is built at is the one the bench measured + /// fastest (0.8128 of variant 0 per permutation on the 5090): limb + /// multiply (1) plus `square_n` unrolled by four (4). Another variant is + /// another measurement, not a rename. + #[test] + fn the_limb_permutation_is_variant_five() { + assert_eq!(LIMB_PERMUTE_VARIANT, 5); + } +} diff --git a/crypto/math-cuda/src/stream_class.rs b/crypto/math-cuda/src/stream_class.rs new file mode 100644 index 000000000..6297fa498 --- /dev/null +++ b/crypto/math-cuda/src/stream_class.rs @@ -0,0 +1,252 @@ +//! Two classes of device work, so a proof run beside another can yield the card. +//! +//! Every caller is in the foreground unless its thread says otherwise +//! ([`enter`]). With `LAMBDA_VM_STREAM_CLASSES` unset (or `priority`), the +//! foreground pool is created at the device's greatest stream priority and a +//! background pool of [`BACKGROUND_STREAMS`] at its least, so the card's block +//! scheduler starts a foreground kernel's blocks before a background kernel's +//! pending ones; with `equal`, the background pool exists but every stream has +//! the default priority (the control for `priority`); with `off` the class is +//! read by nobody: one pool of streams serves every thread, as it did before +//! the classes. A process with no background thread runs every kernel on the +//! foreground pool whichever it is. +//! +//! The class is a property of the THREAD that draws the stream. A thread a +//! background proof spawns must be marked again (`stark`'s per-table drivers +//! copy their spawner's class); a rayon worker of the global pool is never +//! marked, so a stream drawn inside a parallel iterator is a foreground one. +//! [`draws`] counts every draw by class, which is how a leak is seen. + +use std::cell::Cell; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +/// Streams in the background pool when one exists. +pub const BACKGROUND_STREAMS: usize = 8; + +/// The knob that creates the background pool. +pub const STREAM_CLASSES_ENV: &str = "LAMBDA_VM_STREAM_CLASSES"; + +/// Which class of work a thread's stream draws are for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StreamClass { + /// Everything that does not say otherwise. + Foreground, + /// Work run beside the foreground's, which should take the card only where + /// the foreground leaves it. + Background, +} + +/// What the backend builds its streams as (`LAMBDA_VM_STREAM_CLASSES`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StreamClasses { + /// One pool for every thread (`off`): the streams before the classes. + Off, + /// The foreground pool at the greatest priority, a background pool at the + /// least (unset, `priority`). + Priority, + /// A background pool, every stream at the default priority (`equal`). + Equal, +} + +impl StreamClasses { + /// The knob's reading: unset or `priority` is [`Self::Priority`], `equal` + /// the two pools at one priority, `off` [`Self::Off`]; anything else is an + /// error naming the knob. + pub fn parse(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("priority") => Ok(Self::Priority), + Some("off") => Ok(Self::Off), + Some("equal") => Ok(Self::Equal), + Some(other) => Err(format!( + "{STREAM_CLASSES_ENV}={other}: off, priority or equal" + )), + } + } + + /// Whether a background pool exists. + pub fn split(self) -> bool { + self != Self::Off + } +} + +/// The process's setting, read once. A value the knob does not take is read as +/// [`StreamClasses::Off`] and said once: the backend has no caller to refuse +/// to, and a measurement that needs the pools checks [`setting`] itself. +pub fn setting() -> StreamClasses { + static SETTING: OnceLock = OnceLock::new(); + *SETTING.get_or_init(|| { + StreamClasses::parse(std::env::var(STREAM_CLASSES_ENV).ok().as_deref()).unwrap_or_else( + |e| { + eprintln!("[gpu] stream classes: {e} — read as off"); + StreamClasses::Off + }, + ) + }) +} + +thread_local! { + static CLASS: Cell = const { Cell::new(StreamClass::Foreground) }; +} + +/// This thread's class. +pub fn current() -> StreamClass { + CLASS.with(Cell::get) +} + +/// Mark this thread's stream draws as `class` until the guard drops, which +/// puts the previous class back (also as a panic unwinds). +pub fn enter(class: StreamClass) -> ClassGuard { + let previous = CLASS.with(|c| c.replace(class)); + ClassGuard { + previous, + _this_thread: std::marker::PhantomData, + } +} + +/// Puts the thread's previous class back when it drops. Not `Send`: the class +/// belongs to the thread that entered it. +pub struct ClassGuard { + previous: StreamClass, + _this_thread: std::marker::PhantomData<*const ()>, +} + +impl Drop for ClassGuard { + fn drop(&mut self) { + CLASS.with(|c| c.set(self.previous)); + } +} + +/// Stream draws by class since the process started: (foreground pool, +/// background pool). A background thread's draw is counted in the pool it +/// came from, so with no background pool every draw is a foreground one. +static FOREGROUND_DRAWS: AtomicU64 = AtomicU64::new(0); +static BACKGROUND_DRAWS: AtomicU64 = AtomicU64::new(0); + +/// (foreground-pool draws, background-pool draws) since the process started. +/// Monotone: a reader brackets the span it cares about and subtracts. +pub fn draws() -> (u64, u64) { + ( + FOREGROUND_DRAWS.load(Ordering::Relaxed), + BACKGROUND_DRAWS.load(Ordering::Relaxed), + ) +} + +/// The two pools a backend draws its streams from, round-robin within each. +pub(crate) struct StreamPools { + foreground: Vec, + background: Vec, + next_foreground: AtomicUsize, + next_background: AtomicUsize, +} + +impl StreamPools { + /// `background` empty is one pool for every thread. + pub(crate) fn new(foreground: Vec, background: Vec) -> Self { + assert!(!foreground.is_empty(), "a stream pool needs a stream"); + Self { + foreground, + background, + next_foreground: AtomicUsize::new(0), + next_background: AtomicUsize::new(0), + } + } + + /// The next stream for this thread's class: the background pool's for a + /// background thread when there is one, the foreground's otherwise. + pub(crate) fn next(&self) -> T { + if current() == StreamClass::Background && !self.background.is_empty() { + BACKGROUND_DRAWS.fetch_add(1, Ordering::Relaxed); + let idx = self.next_background.fetch_add(1, Ordering::Relaxed) % self.background.len(); + return self.background[idx].clone(); + } + FOREGROUND_DRAWS.fetch_add(1, Ordering::Relaxed); + let idx = self.next_foreground.fetch_add(1, Ordering::Relaxed) % self.foreground.len(); + self.foreground[idx].clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two pools of labelled stand-ins: foreground 0..4, background 100..102. + fn pools(split: bool) -> StreamPools { + StreamPools::new( + (0..4).collect(), + if split { (100..102).collect() } else { vec![] }, + ) + } + + /// A thread marked background draws only from the background pool, an + /// unmarked one never does, and each pool is taken round-robin. + #[test] + fn a_background_thread_draws_only_background_streams() { + let p = pools(true); + let foreground: Vec = (0..8).map(|_| p.next()).collect(); + assert_eq!(foreground, vec![0, 1, 2, 3, 0, 1, 2, 3]); + let background: Vec = std::thread::scope(|s| { + s.spawn(|| { + let _bg = enter(StreamClass::Background); + (0..5).map(|_| p.next()).collect() + }) + .join() + .expect("the background thread") + }); + assert_eq!(background, vec![100, 101, 100, 101, 100]); + // Another thread, never marked, is in the foreground. + let other = std::thread::scope(|s| s.spawn(|| p.next()).join().expect("a plain thread")); + assert!(other < 100, "an unmarked thread drew {other}"); + } + + /// With no background pool (the knob off) a background thread draws the + /// foreground's streams: today's behaviour whatever a thread is marked. + #[test] + fn without_a_background_pool_every_thread_draws_the_one_pool() { + let p = pools(false); + let _bg = enter(StreamClass::Background); + let drawn: Vec = (0..6).map(|_| p.next()).collect(); + assert_eq!(drawn, vec![0, 1, 2, 3, 0, 1]); + } + + /// The guard puts the previous class back, nested and on unwind. + #[test] + fn the_guard_restores_the_previous_class() { + assert_eq!(current(), StreamClass::Foreground); + { + let _outer = enter(StreamClass::Background); + { + let _inner = enter(StreamClass::Foreground); + assert_eq!(current(), StreamClass::Foreground); + } + assert_eq!(current(), StreamClass::Background); + } + assert_eq!(current(), StreamClass::Foreground); + let unwound = std::panic::catch_unwind(|| { + let _bg = enter(StreamClass::Background); + panic!("unwinding through the guard"); + }); + assert!(unwound.is_err()); + assert_eq!(current(), StreamClass::Foreground, "an unwind restores it"); + } + + /// The knob's words, unset read as `priority`, and a word it does not take + /// refused by name. + #[test] + fn the_knob_reads_off_priority_and_equal() { + assert_eq!(StreamClasses::parse(None), Ok(StreamClasses::Priority)); + assert_eq!(StreamClasses::parse(Some("off")), Ok(StreamClasses::Off)); + assert_eq!( + StreamClasses::parse(Some(" priority ")), + Ok(StreamClasses::Priority) + ); + assert_eq!( + StreamClasses::parse(Some("equal")), + Ok(StreamClasses::Equal) + ); + let err = StreamClasses::parse(Some("high")).expect_err("not a setting"); + assert!(err.contains(STREAM_CLASSES_ENV), "{err}"); + assert!(!StreamClasses::Off.split()); + assert!(StreamClasses::Priority.split() && StreamClasses::Equal.split()); + } +} diff --git a/crypto/math-cuda/src/sumcheck.rs b/crypto/math-cuda/src/sumcheck.rs new file mode 100644 index 000000000..fd928edfe --- /dev/null +++ b/crypto/math-cuda/src/sumcheck.rs @@ -0,0 +1,1994 @@ +//! A sumcheck's rounds on device, with the factors resident between them. +//! +//! The protocol is interactive by construction: every round's evaluations are +//! absorbed and a challenge is drawn before the next one, so the host is in the +//! loop once per round. What stays on device is the expensive part — the pass +//! over the cube and the fold that halves it — and what crosses the bus is a +//! handful of field elements per round. +//! +//! The program is `multilinear::program::Program` lowered to the flat node blob +//! `kernels/sumcheck.cu` reads; the lowering lives in +//! `crypto/multilinear/src/gpu.rs`, which is also where the op tags are +//! defined. + +use std::sync::Arc; + +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, LaunchConfig, PushKernelArg}; + +use crate::Result; +use crate::device::{alloc_or_trim, alloc_zeros_or_trim, backend}; + +/// Interpolation nodes per round the kernel has accumulator room for. Mirrors +/// `MAX_NODES` in `kernels/sumcheck.cu`. +pub const MAX_NODES: usize = 16; + +const BLOCK_DIM: u32 = 256; + +/// Smallest block a launch will take: one warp. Below this a block is not a +/// unit of scheduling any more. +const MIN_BLOCK: u32 = 32; + +/// Blocks a launch aims for before it starts making them wider. +/// +/// A round's threads are one per cube index, so a small cube — which is every +/// late round, and the late rounds are most of them — used to arrive as a +/// single block of 256 on a device with a hundred and seventy +/// multiprocessors. Spreading the same threads over narrow blocks puts them on +/// different multiprocessors, which is where the latency of the slot file gets +/// hidden. +const SPREAD_BLOCKS: u32 = 512; + +/// Scratch ceiling for the per-thread slot file, which is what caps the grid: +/// a wider program buys fewer threads. 512 MiB leaves the factors and the +/// resident codewords room on a 32 GiB device. +const SLOT_BUDGET_BYTES: u64 = 512 * 1024 * 1024; + +/// Most threads a round launches, whatever the slot file allows. +const MAX_THREADS: u64 = 1 << 20; + +/// Blocks an elementwise kernel here may take. Enough to fill the device; past +/// it the grid-stride loop takes over. +const MAX_GRID: u32 = 4096; + +/// One sumcheck's device state: the factors, the program, and the scratch the +/// rounds reuse. +/// +/// The factors reach the kernels as a table of device pointers, so they need +/// not be one allocation: a batch uploads them together, while a GKR layer's +/// are halves of buffers the fraction tree already holds. +pub struct SumcheckSession { + stream: Arc, + /// One device address per factor, and the same list on the host so a + /// bound factor can be read back without owning its buffer. + factor_ptrs: CudaSlice, + addresses: Vec, + /// The buffers the factors point into, kept alive for the session's + /// lifetime. A session built by uploading owns one; a session over a + /// fraction tree's layer shares the tree's. + held: Vec>>, + /// True when `held[0]` is this session's own upload, laid out as `width` + /// slabs of `stride` — the only shape [`download`](Self::download) knows. + uploaded: bool, + stride: usize, + width: usize, + /// Cube indices left. Halves with every fold. + len: usize, + nodes: CudaSlice, + num_nodes: usize, + consts: CudaSlice, + root_slot: u32, + slots: CudaSlice, + /// Threads `slots` was sized for, which is the first round's. Later + /// rounds need fewer for the cube, and spend the difference on nodes. + slot_threads: u64, + partials: CudaSlice, + /// Where the partials are summed, so a round's answer crosses the bus at + /// three u64 per interpolation node instead of three per block. + sums: CudaSlice, + /// The threads this session's program can afford at once. Every round's + /// launch is shaped out of it and the indices it has left. + thread_ceiling: u64, + /// The interpolation nodes as the device last saw them, and the buffer + /// they live in. They are the same every round of a sumcheck, so the send + /// happens once and the comparison is what decides that. + t_dev: CudaSlice, + t_host: Vec, + /// Three u64 of scratch for the round's challenge, written in place rather + /// than allocated per fold. + r_dev: CudaSlice, +} + +thread_local! { + // ROUND-3 ARGUE DEVICE-BUSY SIZING (diagnostic; `LAMBDA_VM_ARGUE_BUSY_PROBE`). + // A reusable TIMING-enabled CUDA event pair — created ONCE per thread, never + // per round (a mid-prove cuEventCreate convoys the driver lock), and reused to + // bracket the round kernels. Off by default: production round() only ever + // reads the cached flag and skips all of this. + static BUSY_EVENTS: std::cell::RefCell< + Option<(cudarc::driver::CudaEvent, cudarc::driver::CudaEvent)>, + > = const { std::cell::RefCell::new(None) }; +} + +impl SumcheckSession { + /// Uploads `factors` (each `2^num_vars` ext3 values, interleaved as three + /// u64 per element) and the lowered program. + /// + /// `nodes` is two u64 per step, `consts` three u64 per constant, and + /// `num_slots` the program's max live set — see the lowering. + pub fn new( + factors: &[&[u64]], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + assert!(!factors.is_empty(), "a sumcheck needs a factor"); + assert!(nodes.len().is_multiple_of(2), "two u64 per step"); + assert!( + consts.len().is_multiple_of(3), + "three u64 per ext3 constant" + ); + let stride = factors[0].len() / 3; + assert!(stride.is_power_of_two(), "the cube is a power of two"); + assert!( + factors.iter().all(|f| f.len() == stride * 3), + "every factor spans the same cube" + ); + assert!(num_slots > 0, "a program writes at least one slot"); + + let be = backend()?; + let stream = be.next_stream(); + + let width = factors.len(); + let mut buffer = unsafe { alloc_or_trim::(&stream, width * stride * 3) }?; + for (k, factor) in factors.iter().enumerate() { + let at = k * stride * 3; + let mut slab = buffer.slice_mut(at..at + factor.len()); + stream.memcpy_htod(*factor, &mut slab)?; + } + // The base address plus the slab offsets, which stay inside the + // allocation by construction. The guard orders the read on `stream` + // and is dropped here — every later use of the pointers is on the same + // stream, which is what orders them. + let addresses: Vec = { + let (base, _record) = buffer.device_ptr(&stream); + (0..width) + .map(|k| base + (k * stride * 3 * 8) as u64) + .collect() + }; + let factor_ptrs = crate::device::htod_or_trim(&stream, &addresses)?; + + let ceiling = thread_ceiling(num_slots); + let (grid, block) = launch_shape(ceiling, (stride / 2) as u64); + let num_threads = grid as u64 * block as u64; + let widest = widest_grid(ceiling, (stride / 2) as u64); + + let nodes_dev = crate::device::htod_or_trim(&stream, nodes)?; + // A program with no constants still needs an allocation to point at. + let consts_dev = crate::device::htod_or_trim( + &stream, + if consts.is_empty() { + &[0u64][..] + } else { + consts + }, + )?; + let slots = unsafe { alloc_or_trim::(&stream, num_slots * 3 * num_threads as usize) }?; + // Every partial the round reads is one the round wrote. + let partials = unsafe { alloc_or_trim::(&stream, MAX_NODES * widest as usize * 3) }?; + let t_dev = crate::device::alloc_zeros_or_trim::(&stream, MAX_NODES * 3)?; + let r_dev = crate::device::alloc_zeros_or_trim::(&stream, 3)?; + let sums = crate::device::alloc_zeros_or_trim::(&stream, MAX_NODES * 3)?; + + Ok(Self { + stream, + factor_ptrs, + addresses, + held: vec![Arc::new(buffer)], + uploaded: true, + stride, + width, + len: stride, + nodes: nodes_dev, + num_nodes: nodes.len() / 2, + consts: consts_dev, + root_slot, + slots, + slot_threads: num_threads, + partials, + sums, + thread_ceiling: ceiling, + t_dev, + t_host: Vec::new(), + r_dev, + }) + } + + /// A session over factors that already live on device. + /// + /// `addresses` is one device pointer per factor, `len` the cube they span, + /// and `held` the allocations they point into — kept alive here, because + /// the kernels only see addresses. The rounds fold those buffers in place. + #[allow(clippy::too_many_arguments)] + pub fn from_device( + stream: Arc, + addresses: &[u64], + len: usize, + held: Vec>>, + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + Self::from_device_spread( + stream, addresses, len, held, nodes, consts, num_slots, root_slot, 1, + ) + } + + /// [`from_device`](Self::from_device) with the slot file sized for + /// `spread` threads per cube index, up to the slot budget. + /// + /// A round gives each index one thread and spreads the interpolation nodes + /// over the grid's second dimension only as far as the slot file has room + /// (`round`'s `nodes_wide`). Sized for one thread an index, a session's + /// first round walks every node in turn on one thread; sized for `spread` + /// — the round's degree — the nodes run side by side from the first round. + #[allow(clippy::too_many_arguments)] + pub fn from_device_spread( + stream: Arc, + addresses: &[u64], + len: usize, + held: Vec>>, + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + spread: usize, + ) -> Result { + assert!(!addresses.is_empty(), "a sumcheck needs a factor"); + assert!(len.is_power_of_two(), "the cube is a power of two"); + assert!(nodes.len().is_multiple_of(2), "two u64 per step"); + assert!( + consts.len().is_multiple_of(3), + "three u64 per ext3 constant" + ); + assert!(num_slots > 0, "a program writes at least one slot"); + + let be = backend()?; + let width = addresses.len(); + let ceiling = thread_ceiling(num_slots); + let (grid, block) = launch_shape( + ceiling, + (len / 2) as u64 * spread.clamp(1, MAX_NODES) as u64, + ); + let num_threads = grid as u64 * block as u64; + let widest = widest_grid(ceiling, (len / 2) as u64); + let factor_ptrs = crate::device::htod_or_trim(&stream, addresses)?; + let nodes_dev = crate::device::htod_or_trim(&stream, nodes)?; + let consts_dev = crate::device::htod_or_trim( + &stream, + if consts.is_empty() { + &[0u64][..] + } else { + consts + }, + )?; + let slots = unsafe { alloc_or_trim::(&stream, num_slots * 3 * num_threads as usize) }?; + let partials = unsafe { alloc_or_trim::(&stream, MAX_NODES * widest as usize * 3) }?; + let t_dev = crate::device::alloc_zeros_or_trim::(&stream, MAX_NODES * 3)?; + let r_dev = crate::device::alloc_zeros_or_trim::(&stream, 3)?; + let sums = crate::device::alloc_zeros_or_trim::(&stream, MAX_NODES * 3)?; + let _ = be; + + Ok(Self { + stream, + factor_ptrs, + addresses: addresses.to_vec(), + held, + uploaded: false, + stride: len, + width, + len, + nodes: nodes_dev, + num_nodes: nodes.len() / 2, + consts: consts_dev, + root_slot, + slots, + slot_threads: num_threads, + partials, + sums, + thread_ceiling: ceiling, + t_dev, + t_host: Vec::new(), + r_dev, + }) + } + + /// Bytes this session holds on device, for admission control. + pub fn device_bytes(factors: usize, cube: usize, num_slots: usize) -> u64 { + let per_thread = num_slots as u64 * 3 * 8; + let (grid, block) = launch_shape(thread_ceiling(num_slots), (cube / 2) as u64); + let threads = grid as u64 * block as u64; + factors as u64 * cube as u64 * 24 + threads * per_thread + } + + /// Bytes a session over factors that are already on the card allocates + /// for its rounds ([`from_device`](Self::from_device)): the slot file, the + /// per-block partials of its widest round, and the node, challenge and sum + /// scratch. + pub fn scratch_bytes(cube: usize, num_slots: usize) -> u64 { + let ceiling = thread_ceiling(num_slots); + let (grid, block) = launch_shape(ceiling, (cube / 2) as u64); + let threads = grid as u64 * block as u64; + let widest = widest_grid(ceiling, (cube / 2) as u64) as u64; + let nodes = MAX_NODES as u64; + threads * num_slots as u64 * 24 + (nodes * widest + 2 * nodes + 1) * 24 + } + + /// A second session over this one's factors, walking another program: + /// the same buffers, and a slot file of its own sized for one thread an + /// index. It never folds — [`follow`](Self::follow) takes the cube size + /// of the session it shadows after that one folds the buffers they share. + /// For checking one program's rounds against another's over the same + /// factors, in the same process. + pub fn shadow( + &self, + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + Self::from_device( + self.stream.clone(), + &self.addresses, + self.len, + self.held.clone(), + nodes, + consts, + num_slots, + root_slot, + ) + } + + /// Takes `leader`'s cube size: it folded the factors this session shares + /// with it ([`shadow`](Self::shadow)). + pub fn follow(&mut self, leader: &Self) { + assert_eq!( + self.addresses, leader.addresses, + "a shadow follows the session whose factors it reads" + ); + assert!(leader.len <= self.len, "a cube only shrinks"); + self.len = leader.len; + } + + /// Cube indices left to bind. + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// One round: the program's sum over the cube at each interpolation node. + /// + /// `t` holds the nodes as ext3 (three u64 each). Returns one ext3 sum per + /// node, as `t.len()/3` triples. + pub fn round(&mut self, t: &[u64]) -> Result> { + let num_t = self.round_enqueue(t)?; + self.round_collect(num_t) + } + + /// [`round`](Self::round)'s launches, queued on the session's stream and + /// not waited for: the caller collects the answer with + /// [`round_collect`](Self::round_collect). Several sessions' rounds can so + /// be queued before one wait — the batched argue's ladder, whose active + /// trees each run a session and share one challenge a round. Returns the + /// node count `round_collect` reads. + pub fn round_enqueue(&mut self, t: &[u64]) -> Result { + assert!(t.len().is_multiple_of(3), "three u64 per ext3 node"); + let num_t = t.len() / 3; + assert!(num_t > 0 && num_t <= MAX_NODES, "nodes per round"); + assert!(self.len >= 2, "a round needs a variable to bind"); + + let be = backend()?; + let half = (self.len / 2) as u64; + // The launch follows the cube down, and narrows its blocks as it goes: + // the same threads across more multiprocessors is what a late round + // needs, and a block past the indices left writes a partial with + // nothing in it. + let (grid, block) = launch_shape(self.thread_ceiling, half); + // Once the cube stops filling the slot file, the nodes do: a late round + // is the same walk of the program at each of them, and they are + // independent. The width comes out of the scratch the session already + // has — the first round fills it with indices and every round after + // frees half of it — so this costs no memory at all. The partials keep + // their shape either way: a node has one owning block in the second + // dimension. + let per_node = grid as u64 * block as u64; + let nodes_wide = (self.slot_threads / per_node.max(1)).clamp(1, num_t as u64) as u32; + if self.t_host != t { + let mut head = self.t_dev.slice_mut(0..t.len()); + self.stream.memcpy_htod(t, &mut head)?; + self.t_host.clear(); + self.t_host.extend_from_slice(t); + } + let cfg = LaunchConfig { + grid_dim: (grid, nodes_wide, 1), + block_dim: (block, 1, 1), + // One ext3 accumulator per thread, reduced one node at a time. + shared_mem_bytes: block * 3 * 8, + }; + let num_nodes = self.num_nodes as u64; + let num_t_u32 = num_t as u32; + // Round-3 argue device-busy sizing: bracket the round kernels with a + // reusable event pair, read the elapsed after the EXISTING sync below. + let busy_probe = crate::argue_probe::busy_probe_enabled(); + if busy_probe { + BUSY_EVENTS.with(|cell| -> Result<()> { + let mut ev = cell.borrow_mut(); + if ev.is_none() { + let f = cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT; + *ev = Some((be.ctx.new_event(Some(f))?, be.ctx.new_event(Some(f))?)); + } + ev.as_ref().unwrap().0.record(&self.stream)?; + Ok(()) + })?; + } + // S1-1: integer nodes take the componentwise multiply — only when + // every node is `(k, 0, 0)`, where the two are the same value. + let kernel = if int_nodes() && t.chunks_exact(3).all(|n| n[1] == 0 && n[2] == 0) { + &be.sumcheck_round_ext3_int + } else { + &be.sumcheck_round_ext3 + }; + unsafe { + self.stream + .launch_builder(kernel) + .arg(&self.factor_ptrs) + .arg(&half) + .arg(&self.nodes) + .arg(&num_nodes) + .arg(&self.consts) + .arg(&self.root_slot) + .arg(&self.t_dev) + .arg(&num_t_u32) + .arg(&mut self.slots) + .arg(&mut self.partials) + .launch(cfg)?; + } + + // The partials are summed here, on the device, and what crosses the bus + // is the round's answer: three u64 per interpolation node. A wide + // launch leaves megabytes of them, and every round of every sumcheck + // waits for that copy before the transcript can move — which for a + // whole proof is more time than the rounds themselves. + let reduce_block = 256u32; + unsafe { + self.stream + .launch_builder(&be.sum_partials_ext3) + .arg(&self.partials) + .arg(&(grid as u64)) + .arg(&mut self.sums) + .launch(LaunchConfig { + grid_dim: (num_t as u32, 1, 1), + block_dim: (reduce_block, 1, 1), + shared_mem_bytes: reduce_block * 3 * 8, + })?; + } + if busy_probe { + BUSY_EVENTS.with(|cell| -> Result<()> { + cell.borrow().as_ref().unwrap().1.record(&self.stream)?; + Ok(()) + })?; + } + Ok(num_t) + } + + /// The answer of the round [`round_enqueue`](Self::round_enqueue) queued: + /// three u64 per interpolation node, read back once the stream is idle. + pub fn round_collect(&self, num_t: usize) -> Result> { + let sums = crate::readback::clone_dtoh(&self.stream, &self.sums.slice(0..num_t * 3))?; + self.stream.synchronize()?; + if crate::argue_probe::busy_probe_enabled() { + BUSY_EVENTS.with(|cell| -> Result<()> { + let ev = cell.borrow(); + let (start, end) = ev.as_ref().unwrap(); + crate::argue_probe::add_device_busy_ns( + (start.elapsed_ms(end)? as f64 * 1.0e6) as u64, + ); + Ok(()) + })?; + } + Ok(sums) + } + + /// Binds the round's variable to `r` (ext3, three u64) in every factor. + pub fn fold(&mut self, r: &[u64]) -> Result<()> { + assert_eq!(r.len(), 3, "an ext3 challenge"); + assert!(self.len >= 2, "a fold needs a variable to bind"); + let be = backend()?; + let half = (self.len / 2) as u64; + self.stream.memcpy_htod(r, &mut self.r_dev)?; + let total = self.width as u64 * half; + let grid = total.div_ceil(BLOCK_DIM as u64).min(4096) as u32; + let cfg = LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + let width = self.width as u64; + unsafe { + self.stream + .launch_builder(&be.sumcheck_fold_ext3) + .arg(&mut self.factor_ptrs) + .arg(&half) + .arg(&width) + .arg(&self.r_dev) + .launch(cfg)?; + } + self.len /= 2; + Ok(()) + } + + /// Binds the round's variable to `r` in the first `count` factors only, + /// and takes the cube down with them: for a shadow whose other factors a + /// leader folds where they lie ([`crate::gkr::GruenLayer::shadow`]). + pub fn fold_first(&mut self, count: usize, r: &[u64]) -> Result<()> { + assert_eq!(r.len(), 3, "an ext3 challenge"); + assert!(self.len >= 2, "a fold needs a variable to bind"); + assert!(count <= self.width, "the factors to fold are the session's"); + let be = backend()?; + let half = (self.len / 2) as u64; + self.stream.memcpy_htod(r, &mut self.r_dev)?; + let total = count as u64 * half; + let grid = total.div_ceil(BLOCK_DIM as u64).min(4096) as u32; + let width = count as u64; + unsafe { + self.stream + .launch_builder(&be.sumcheck_fold_ext3) + .arg(&mut self.factor_ptrs) + .arg(&half) + .arg(&width) + .arg(&self.r_dev) + .launch(LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + self.len /= 2; + Ok(()) + } + + /// Whether [`download`](Self::download) can read the factors back: only a + /// session that uploaded them knows their layout. + pub fn can_download(&self) -> bool { + self.uploaded + } + + /// What every factor has left, read where it lies rather than through the + /// buffer it sits in: a session over a fraction tree's layer does not own + /// one, and the fold writes each factor back over its own base — so what + /// is left of a factor is the first `len` elements at its address, + /// whatever wrote them. + pub fn values(&self) -> Result>> { + self.stream.synchronize()?; + let mut out = Vec::with_capacity(self.addresses.len()); + for address in &self.addresses { + let mut values = vec![0u64; self.len * 3]; + // SAFETY: the address is a factor's base and the cube it spans is + // `len` elements wide, the stream being idle (synchronized above). + unsafe { + cudarc::driver::sys::cuMemcpyDtoH_v2( + values.as_mut_ptr() as *mut core::ffi::c_void, + *address, + self.len * 24, + ) + .result()?; + } + out.push(values); + } + Ok(out) + } + + /// [`values`](Self::values) — the same bytes — gathered into one buffer on + /// the card and read back in one copy, where `values` makes a synchronous + /// copy per factor: a table of fourteen hundred factors is fourteen hundred + /// round trips at every session's end, for a few kilobytes each. + /// + /// Pageable, not pinned: what this saves is the count of copies, and a + /// session's end is at most a megabyte or so. + pub fn values_gathered(&self) -> Result>> { + let be = backend()?; + let span = self.len * 3; + let total = self.width * span; + // SAFETY: the kernel writes every word of it, one thread per word. + let mut out = unsafe { alloc_or_trim::(&self.stream, total) }?; + let width = self.width as u64; + let cells = self.len as u64; + let grid = (total as u64) + .div_ceil(BLOCK_DIM as u64) + .clamp(1, MAX_GRID as u64) as u32; + unsafe { + self.stream + .launch_builder(&be.gather_factor_heads_ext3) + .arg(&self.factor_ptrs) + .arg(&width) + .arg(&cells) + .arg(&mut out) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + let flat = crate::readback::clone_dtoh(&self.stream, &out)?; + self.stream.synchronize()?; + Ok(flat.chunks_exact(span).map(<[u64]>::to_vec).collect()) + } + + /// Factor `k`'s remaining values, read where it lies: one factor, where + /// [`values`](Self::values) reads every one — so a check of a table the + /// card built reads that table and not the trace beside it. + pub fn factor(&self, k: usize) -> Result> { + self.stream.synchronize()?; + let mut values = vec![0u64; self.len * 3]; + // SAFETY: as in `values`: the address is factor `k`'s base, it spans + // `len` elements, and the stream is idle. + unsafe { + cudarc::driver::sys::cuMemcpyDtoH_v2( + values.as_mut_ptr() as *mut core::ffi::c_void, + self.addresses[k], + self.len * 24, + ) + .result()?; + } + Ok(values) + } + + /// Writes `value` over cell `index` of factor `k`: the way in for a fault + /// that corrupts a table the card built, so the checks on it can be shown + /// to see one. + pub fn set_cell(&self, k: usize, index: usize, value: &[u64; 3]) -> Result<()> { + assert!(index < self.len, "the cell is inside the cube"); + self.stream.synchronize()?; + // SAFETY: the cell is inside factor `k`'s `len` elements, and the + // stream is idle, so nothing reads or writes it meanwhile. + unsafe { + cudarc::driver::sys::cuMemcpyHtoD_v2( + self.addresses[k] + (index * 24) as u64, + value.as_ptr() as *const core::ffi::c_void, + 24, + ) + .result()?; + } + Ok(()) + } + + /// What each factor has been bound to, once every variable is gone. + pub fn bound_values(&self) -> Result> { + assert_eq!(self.len, 1, "a factor is bound once every variable is"); + Ok(self + .values()? + .into_iter() + .map(|value| [value[0], value[1], value[2]]) + .collect()) + } + + /// Every factor's remaining values, interleaved as three u64 per element — + /// what the host needs to carry on where the device stopped. + pub fn download(&self) -> Result>> { + assert!( + self.uploaded, + "only a session that uploaded its factors knows their layout" + ); + let factors = &self.held[0]; + let mut out = Vec::with_capacity(self.width); + for k in 0..self.width { + let at = k * self.stride * 3; + out.push(crate::readback::clone_dtoh( + &self.stream, + &factors.slice(at..at + self.len * 3), + )?); + } + self.stream.synchronize()?; + Ok(out) + } +} + +/// Whether a round with integer nodes takes the componentwise multiply +/// (D-ARGUE S1-1): on by default, `LAMBDA_VM_ARGUE_INT_NODES=0` the opt-out. +/// The same value either way: `(k, 0, 0)·d = (k·d₀, k·d₁, k·d₂)`. Read once, +/// with a banner. Turned on with the fused zerocheck, by their A/B together +/// (FAST jobs 274 and 270). +pub fn int_nodes() -> bool { + match INT_NODES_FORCED.load(std::sync::atomic::Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + let on = int_nodes_from(std::env::var("LAMBDA_VM_ARGUE_INT_NODES").ok().as_deref()); + eprintln!( + "[gpu] sumcheck rounds: {}", + if on { + "integer nodes (the default; LAMBDA_VM_ARGUE_INT_NODES=0 is the full multiply)" + } else { + "the full ext3 multiply (LAMBDA_VM_ARGUE_INT_NODES=0)" + } + ); + on + }) +} + +/// The knob's reading of its variable: only `0` turns it off. +pub(crate) fn int_nodes_from(value: Option<&str>) -> bool { + value != Some("0") +} + +static INT_NODES_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// A test's own setting in place of the environment's (`None` = the environment). +#[doc(hidden)] +pub fn force_int_nodes(on: Option) { + INT_NODES_FORCED.store( + match on { + None => 0, + Some(false) => 1, + Some(true) => 2, + }, + std::sync::atomic::Ordering::Relaxed, + ); +} + +/// The most threads a program of `num_slots` live values may run at once: the +/// slot file is per thread, so it is the thread count that gives way to a wider +/// program. +pub fn thread_ceiling(num_slots: usize) -> u64 { + let per_thread = num_slots as u64 * 3 * 8; + (SLOT_BUDGET_BYTES / per_thread.max(1)) + .min(MAX_THREADS) + .max(MIN_BLOCK as u64) +} + +/// The `(grid, block)` a launch of `work` indices takes, given the thread +/// ceiling: one thread per index, spread over [`SPREAD_BLOCKS`] blocks before +/// any block is made wider than a warp. +pub(crate) fn launch_shape(ceiling: u64, work: u64) -> (u32, u32) { + let threads = work.clamp(1, ceiling); + let wide = (threads / SPREAD_BLOCKS as u64).max(1); + let block = (1u64 << (63 - wide.leading_zeros() as u64)) + .clamp(MIN_BLOCK as u64, BLOCK_DIM as u64) as u32; + let grid = threads.div_ceil(block as u64).max(1) as u32; + (grid, block) +} + +/// The widest grid any round of a session will take, which is what the +/// per-block partials have to have room for. The cube halves every round and +/// the blocks narrow as it does, so the widest is not always the first. +fn widest_grid(ceiling: u64, first_half: u64) -> u32 { + let mut widest = 1; + let mut work = first_half.max(1); + loop { + widest = widest.max(launch_shape(ceiling, work).0); + if work <= 1 { + return widest; + } + work /= 2; + } +} +/// A multilinear's value at `point`, bound one variable at a time on device. +/// +/// `table` is `2^point_len` ext3 values interleaved, `point` three u64 per +/// coordinate. Mirrors `Mle::evaluate_at`: the same folds in the same order, +/// with the table halving under them. +pub fn evaluate_mle_ext3(table: &[u64], point: &[u64]) -> Result<[u64; 3]> { + assert!(point.len().is_multiple_of(3), "three u64 per coordinate"); + let vars = point.len() / 3; + assert_eq!( + table.len(), + (1usize << vars) * 3, + "the table spans the point" + ); + assert!(vars > 0, "a point with no coordinates is the table itself"); + + let be = backend()?; + let stream = be.next_stream(); + let mut values = crate::device::htod_or_trim(&stream, table)?; + let stride = 1u64 << vars; + fold_to_one(&stream, be, &mut values, stride, point)?; + let out = crate::readback::clone_dtoh(&stream, &values.slice(0..3))?; + stream.synchronize()?; + Ok([out[0], out[1], out[2]]) +} + +/// The same for a base-field table at a point in the extension: the first fold +/// lifts, the rest stay up. +pub fn evaluate_mle_base(table: &[u64], point: &[u64]) -> Result<[u64; 3]> { + assert!(point.len().is_multiple_of(3), "three u64 per coordinate"); + let vars = point.len() / 3; + assert_eq!(table.len(), 1usize << vars, "the table spans the point"); + assert!(vars > 0, "a point with no coordinates is the table itself"); + + let be = backend()?; + let stream = be.next_stream(); + let base = crate::device::htod_or_trim(&stream, table)?; + let half = table.len() / 2; + let r = crate::device::htod_or_trim(&stream, &point[..3])?; + // SAFETY: the kernel writes every element of the half it produces. + let mut values = unsafe { stream.alloc::(half * 3) }?; + let half_arg = half as u64; + unsafe { + stream + .launch_builder(&be.mle_fold_base_ext3) + .arg(&base) + .arg(&half_arg) + .arg(&r) + .arg(&mut values) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + drop(base); + fold_to_one(&stream, be, &mut values, half as u64, &point[3..])?; + let out = crate::readback::clone_dtoh(&stream, &values.slice(0..3))?; + stream.synchronize()?; + Ok([out[0], out[1], out[2]]) +} + +/// Every base-field table's value at `point`, in one pass per level. +/// +/// A table is evaluated by folding it down to one element, which is a launch +/// per level and an upload of the table. Doing that per column turns a table's +/// argument into hundreds of round trips over the same trace; here the columns +/// go up together and every level is one launch for all of them. +/// +/// `columns` are base-field slices of `2^point.len()/3` values each. Returns +/// one ext3 value per column, in order. +pub fn evaluate_many_base( + columns: crate::columns::Columns<'_>, + point: &[u64], +) -> Result> { + assert!(point.len().is_multiple_of(3), "three u64 per coordinate"); + let vars = point.len() / 3; + let rows = 1usize << vars; + assert!(vars > 0, "a point with no coordinates is the table itself"); + if columns.is_empty() { + return Ok(Vec::new()); + } + assert_eq!(columns.rows(), rows, "every column spans the point"); + let num_columns = columns.width(); + + let be = backend()?; + let stream = be.next_stream(); + // How many go up at a time: the base copy plus the ext3 half it folds to + // is 20 bytes a row, and this keeps that transient bounded however wide + // the table is. + let per_column = rows as u64 * 20; + let chunk = (CHUNK_BUDGET_BYTES / per_column.max(1)).clamp(1, num_columns as u64) as usize; + + let mut out = Vec::with_capacity(num_columns); + for start in (0..num_columns).step_by(chunk) { + let group = start..(start + chunk).min(num_columns); + let group_len = group.len(); + let half = rows / 2; + // Promised before it is used, like everything else that takes a slab + // of the card: the caller's fallback is to evaluate the columns one at + // a time, which needs almost nothing. + let Some(_room) = crate::device::reserve(group_len as u64 * per_column) else { + crate::device::note_device_fallback(); + return Err(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY, + )); + }; + crate::argue_probe::note_device( + crate::argue_probe::Surface::Sumcheck, + group_len as u64 * per_column, + ); + // Read where they lie when they are already there; a copy otherwise. + let uploaded; + let base = match &columns { + crate::columns::Columns::Device { store, first, .. } => { + store.view(first + group.start, group_len) + } + crate::columns::Columns::Host(host) => { + let mut up = unsafe { alloc_or_trim::(&stream, group_len * rows) }?; + for (k, column) in host[group.clone()].iter().enumerate() { + let at = k * rows; + let mut slab = up.slice_mut(at..at + rows); + stream.memcpy_htod(*column, &mut slab)?; + } + uploaded = up; + uploaded.slice(0..group_len * rows) + } + }; + let r = crate::device::htod_or_trim(&stream, &point[..3])?; + // SAFETY: the kernel writes every element of the halves it produces. + let mut values = unsafe { alloc_or_trim::(&stream, group_len * half * 3) }?; + let half_arg = half as u64; + let tables = group_len as u64; + let total = half_arg * tables; + let grid = total.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + unsafe { + stream + .launch_builder(&be.mle_fold_base_ext3_many) + .arg(&base) + .arg(&half_arg) + .arg(&tables) + .arg(&r) + .arg(&mut values) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + // From here every table is ext3 and they all fold together: one launch + // per level, over the list of addresses. A table of one variable has + // no level left — the launch above bound it — and uploading its empty + // rest of the point would ask the driver for a zero-byte allocation. + if vars > 1 { + let addresses: Vec = { + let (at, _guard) = values.device_ptr(&stream); + (0..group_len) + .map(|k| at + (k * half * 3 * 8) as u64) + .collect() + }; + let mut factors = crate::device::htod_or_trim(&stream, &addresses)?; + let mut span = half; + let mut r_dev = crate::device::htod_or_trim(&stream, &point[3..6])?; + for coordinate in point[3..].chunks_exact(3) { + let fold_half = (span / 2) as u64; + if fold_half == 0 { + break; + } + stream.memcpy_htod(coordinate, &mut r_dev)?; + let width = group_len as u64; + let total = width * fold_half; + let grid = total.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + unsafe { + stream + .launch_builder(&be.sumcheck_fold_ext3) + .arg(&mut factors) + .arg(&fold_half) + .arg(&width) + .arg(&r_dev) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + span /= 2; + } + } + + // Three u64 per column, where each one's fold left it — not the + // buffer, which is a level's worth of values nobody needs. + // + // Gathered there and brought back in one copy. A trace has thousands of + // columns, and reading each one's head on its own is a transfer and a + // stream synchronize apiece for twenty-four bytes. + let heads: Vec = (0..group_len).map(|k| (k * half) as u32).collect(); + let packed = crate::fri::gather_ext3_at(&values, &heads, &stream)?; + for head in packed.chunks_exact(3) { + out.push([head[0], head[1], head[2]]); + } + } + Ok(out) +} + +/// Many resident base columns' values at ONE point, each column read once +/// (`eq_dot_base_ext3_many`): `hi` and `lo` are the point's `eq` table in two +/// halves, `eq(p, row) = hi[row >> lo_bits]·lo[row & (2^lo_bits − 1)]`, +/// three u64 per value. The store's columns `first..first + width` must be one +/// run. Returns one ext3 value per column, in order. +/// +/// Where [`evaluate_many_base`] folds a chunk of columns level by level, +/// writing and reading back an ext3 half of each at every level, this reads +/// each base word once and writes a partial per (column, block): the +/// commit-time out-of-domain answers' column values, read while the group's +/// columns are on the card. +pub fn eq_dot_many_base( + store: &crate::columns::DeviceColumns, + first: usize, + width: usize, + hi: &[u64], + lo: &[u64], + lo_bits: u32, +) -> Result> { + if width == 0 { + return Ok(Vec::new()); + } + assert!(store.is_run(first, width), "the columns are one run"); + let rows = store.span(first).1; + assert_eq!(lo.len(), 3 << lo_bits, "the low half spans its bits"); + assert_eq!( + (hi.len() / 3) << lo_bits, + rows, + "the halves span the column" + ); + let be = backend()?; + let stream = be.next_stream(); + let his = (rows >> lo_bits) as u64; + let blocks = his.clamp(1, EQ_DOT_BLOCKS as u64) as u32; + let hi_dev = crate::device::htod_or_trim(&stream, hi)?; + let lo_dev = crate::device::htod_or_trim(&stream, lo)?; + let rows_arg = rows as u64; + let mut out = Vec::with_capacity(width); + for start in (0..width).step_by(EQ_DOT_MAX_COLUMNS) { + let columns = (width - start).min(EQ_DOT_MAX_COLUMNS); + // SAFETY: every partial is written by its block, every sum by the + // reduction, before either is read. + let mut partials = unsafe { alloc_or_trim::(&stream, columns * blocks as usize * 3) }?; + let mut sums = unsafe { alloc_or_trim::(&stream, columns * 3) }?; + let view = store.view(first + start, columns); + unsafe { + stream + .launch_builder(&be.eq_dot_base_ext3_many) + .arg(&view) + .arg(&rows_arg) + .arg(&lo_bits) + .arg(&hi_dev) + .arg(&lo_dev) + .arg(&mut partials) + .launch(LaunchConfig { + grid_dim: (blocks, columns as u32, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: BLOCK_DIM * 3 * 8, + })?; + stream + .launch_builder(&be.sum_partials_ext3) + .arg(&partials) + .arg(&(blocks as u64)) + .arg(&mut sums) + .launch(LaunchConfig { + grid_dim: (columns as u32, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: BLOCK_DIM * 3 * 8, + })?; + } + let host = crate::readback::clone_dtoh(&stream, &sums)?; + stream.synchronize()?; + out.extend(host.chunks_exact(3).map(|v| [v[0], v[1], v[2]])); + } + Ok(out) +} + +/// Blocks [`eq_dot_many_base`] gives one column: enough to fill the card +/// beside a run of a few columns, few enough that the partials stay small. +const EQ_DOT_BLOCKS: u32 = 64; + +/// Columns one [`eq_dot_many_base`] launch takes: the grid's `y` limit. +const EQ_DOT_MAX_COLUMNS: usize = 65_535; + +/// How much a batched evaluation keeps on the device at once. +/// +/// Small: what batching saves is the launches, and those dominate for a short +/// column — a tall one spends its time on the upload either way. Keeping the +/// slab small leaves the card to everything that has no host path. +const CHUNK_BUDGET_BYTES: u64 = 64 << 20; + +/// Binds every coordinate of `point` in a resident ext3 table of `len` +/// elements, leaving the value in its first slot. +fn fold_to_one( + stream: &Arc, + be: &crate::device::Backend, + values: &mut CudaSlice, + len: u64, + point: &[u64], +) -> Result<()> { + // The fold kernel takes a table of factors; here there is one, and it + // folds in place, so its address holds for every level. + let address = { + let (base, _record) = values.device_ptr(stream); + [base] + }; + let mut table = crate::device::htod_or_trim(stream, &address)?; + + let point_dev = crate::device::htod_or_trim(stream, point)?; + let width = 1u64; + let mut half = len / 2; + for at in (0..point.len()).step_by(3) { + let r = point_dev.slice(at..at + 3); + let cfg = LaunchConfig::for_num_elems((half * width) as u32); + unsafe { + stream + .launch_builder(&be.sumcheck_fold_ext3) + .arg(&mut table) + .arg(&half) + .arg(&width) + .arg(&r) + .launch(cfg)?; + } + half /= 2; + } + Ok(()) +} + +/// A weight a sumcheck over resident factors adds on top of them. +/// +/// A weight is `2^vars` cells; an `eq` weight is made of `vars` coordinates. +/// Given as its point it is built where it will be folded, and what crosses the +/// bus is the point. +#[derive(Clone, Copy, Debug)] +pub enum Extra<'a> { + /// A table the host built: `len` ext3 values, three u64 each. + Table(&'a [u64]), + /// `eq(point, ·)`, the point three u64 per coordinate. + Eq(&'a [u64]), +} + +/// A table's factors, uploaded once and read by everything that walks them: +/// the LogUp input layer, and then the batched sumcheck. +pub struct DeviceFactors { + stream: Arc, + buffer: Arc>, + addresses: Vec, + /// The room this table's argument promised itself: the factors, the + /// columns they are built from, and the rounds' scratch. + _room: crate::device::DeviceReservation, + /// The same list on device, where every program that walks these factors + /// reads it — one send, not one per interaction. + factor_ptrs: CudaSlice, + len: usize, +} + +impl DeviceFactors { + /// Uploads `factors`, each `len` ext3 values interleaved. + /// + /// The prover builds them with [`from_columns`](Self::from_columns) + /// instead; this is what that is checked against. + pub fn upload(factors: &[&[u64]]) -> Result { + assert!(!factors.is_empty(), "a table has factors"); + let span = factors[0].len(); + assert!(span.is_multiple_of(3), "three u64 per ext3 element"); + assert!( + factors.iter().all(|f| f.len() == span), + "every factor spans the same cube" + ); + let len = span / 3; + + let be = backend()?; + let Some(room) = be.reserve(factors.len() as u64 * span as u64 * 8) else { + crate::device::note_device_fallback(); + return Err(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY, + )); + }; + crate::argue_probe::note_device( + crate::argue_probe::Surface::Sumcheck, + factors.len() as u64 * span as u64 * 8, + ); + let stream = be.next_stream(); + let mut buffer = unsafe { alloc_or_trim::(&stream, factors.len() * span) }?; + for (k, factor) in factors.iter().enumerate() { + let at = k * span; + let mut slab = buffer.slice_mut(at..at + span); + stream.memcpy_htod(*factor, &mut slab)?; + } + let addresses: Vec = { + let (base, _record) = buffer.device_ptr(&stream); + (0..factors.len()) + .map(|k| base + (k * span * 8) as u64) + .collect() + }; + let factor_ptrs = crate::device::htod_or_trim(&stream, &addresses)?; + Ok(Self { + stream, + buffer: Arc::new(buffer), + addresses, + _room: room, + factor_ptrs, + len, + }) + } + + /// Builds the factors from the table's base columns rather than taking + /// them built. + /// + /// A factor is a column read at a frame-step offset and lifted into the + /// extension, so the columns are a third of what the factors are: building + /// them here sends the trace instead of its lift, and spares the host the + /// copy. + /// + /// `columns` is one base-field slice of `rows` values each; `plan` is three + /// u64 per committed factor — the column it reads, its shift (already + /// reduced mod `rows`), and the slot it fills; `public` is the extension + /// tables that are not views of a column, each with the slot it goes to. + pub fn from_columns( + columns: crate::columns::Columns<'_>, + plan: &[u64], + public: &[(usize, &[u64])], + rows: usize, + width: usize, + ) -> Result { + assert!(rows.is_power_of_two(), "the cube is a power of two"); + assert!(width > 0, "a table has factors"); + assert!( + plan.len().is_multiple_of(3), + "three u64 per committed factor" + ); + assert_eq!( + plan.len() / 3 + public.len(), + width, + "every slot is filled once" + ); + assert!( + columns.is_empty() || columns.rows() == rows, + "every column spans the cube" + ); + + let be = backend()?; + // What stays: the factors. The base columns they are gathered from are + // a third of that and are freed as soon as the gather has read them. + let Some(room) = be.reserve(width as u64 * rows as u64 * 24) else { + crate::device::note_device_fallback(); + return Err(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY, + )); + }; + crate::argue_probe::note_device( + crate::argue_probe::Surface::Sumcheck, + width as u64 * rows as u64 * 24, + ); + let stream = be.next_stream(); + + let span = rows * 3; + // SAFETY: every cell is written below — the public slots by their + // copies, the rest by the kernel, which covers every (factor, row). + let mut buffer = unsafe { alloc_or_trim::(&stream, width * span) }?; + for (slot, table) in public { + assert_eq!(table.len(), span, "a public factor spans the cube"); + let at = slot * span; + let mut slab = buffer.slice_mut(at..at + span); + stream.memcpy_htod(*table, &mut slab)?; + } + + if !plan.is_empty() { + // Read where they lie when they are already there; a copy otherwise. + let uploaded; + let base = match &columns { + crate::columns::Columns::Device { + store, + first, + width, + } => store.view(*first, *width), + crate::columns::Columns::Host(host) => { + // SAFETY: every cell is written by the copies below. + let mut up = unsafe { alloc_or_trim::(&stream, host.len() * rows) }?; + for (k, column) in host.iter().enumerate() { + let at = k * rows; + let mut slab = up.slice_mut(at..at + rows); + stream.memcpy_htod(*column, &mut slab)?; + } + uploaded = up; + uploaded.slice(0..host.len() * rows) + } + }; + let plan_dev = crate::device::htod_or_trim(&stream, plan)?; + let num_plan = (plan.len() / 3) as u64; + let rows_arg = rows as u64; + let total = num_plan * rows_arg; + let grid = total.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + let cfg = LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.factors_from_columns_ext3) + .arg(&base) + .arg(&plan_dev) + .arg(&num_plan) + .arg(&rows_arg) + .arg(&mut buffer) + .launch(cfg)?; + } + // A copy made here is spent and goes at the end of this block, + // which is stream-ordered behind the kernel that just read it; a + // view of the epoch's columns frees nothing, because they stay. + } + + let addresses: Vec = { + let (base, _record) = buffer.device_ptr(&stream); + (0..width).map(|k| base + (k * span * 8) as u64).collect() + }; + let factor_ptrs = crate::device::htod_or_trim(&stream, &addresses)?; + Ok(Self { + stream, + buffer: Arc::new(buffer), + addresses, + _room: room, + factor_ptrs, + len: rows, + }) + } + + /// The factors as the fused zerocheck reads them: lifted, three u64 a + /// cell, of which the first is the base value before any fold. + pub fn view(&self) -> FactorView<'_> { + FactorView { + stream: &self.stream, + ptrs: &self.factor_ptrs, + stride: 3, + rows: self.len, + width: self.addresses.len(), + } + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn width(&self) -> usize { + self.addresses.len() + } + + pub fn stream(&self) -> &Arc { + &self.stream + } + + /// Writes `program`'s value at every row into a fresh buffer. + /// + /// This is how a bus interaction's two sides become an input-layer slab: + /// the affine expression is the program, and the rows are the cube. + pub fn map_program( + &self, + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + out: &mut CudaSlice, + offset: usize, + ) -> Result<()> { + assert!(nodes.len().is_multiple_of(2), "two u64 per step"); + assert!( + consts.len().is_multiple_of(3), + "three u64 per ext3 constant" + ); + assert!( + out.len() >= (offset + self.len) * 3, + "the output holds the rows" + ); + + let be = backend()?; + let (grid, block) = launch_shape(thread_ceiling(num_slots), self.len as u64); + let num_threads = grid as u64 * block as u64; + let nodes_dev = crate::device::htod_or_trim(&self.stream, nodes)?; + let consts_dev = crate::device::htod_or_trim( + &self.stream, + if consts.is_empty() { + &[0u64][..] + } else { + consts + }, + )?; + let mut slots = unsafe { + self.stream + .alloc::(num_slots * 3 * num_threads as usize) + }?; + let mut target = out.slice_mut(offset * 3..(offset + self.len) * 3); + let rows = self.len as u64; + let num_nodes = (nodes.len() / 2) as u64; + let cfg = LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + self.stream + .launch_builder(&be.program_map_ext3) + .arg(&self.factor_ptrs) + .arg(&rows) + .arg(&nodes_dev) + .arg(&num_nodes) + .arg(&consts_dev) + .arg(&root_slot) + .arg(&mut slots) + .arg(&mut target) + .launch(cfg)?; + } + Ok(()) + } + + /// A sumcheck over these factors followed by `extra`, which is uploaded + /// here — the weight tables a batch adds on top of the trace's. + /// + /// The rounds fold the factors in place, so the handle is spent for + /// everything else once this runs. + #[allow(clippy::too_many_arguments)] + pub fn session( + &self, + extra: &[&[u64]], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + let extra: Vec> = extra.iter().map(|table| Extra::Table(table)).collect(); + self.session_with(&extra, nodes, consts, num_slots, root_slot) + } + + /// The same, with each weight a table to upload or the point of an `eq` + /// table to build here ([`Extra`]). + #[allow(clippy::too_many_arguments)] + pub fn session_with( + &self, + extra: &[Extra<'_>], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + self.session_spread(extra, nodes, consts, num_slots, root_slot, 1) + } + + /// [`session_with`](Self::session_with) with the slot file sized for + /// `spread` threads per cube index + /// ([`SumcheckSession::from_device_spread`]). + #[allow(clippy::too_many_arguments)] + pub fn session_spread( + &self, + extra: &[Extra<'_>], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + spread: usize, + ) -> Result { + let span = self.len * 3; + let mut addresses = self.addresses.clone(); + let mut held = vec![self.buffer.clone()]; + if !extra.is_empty() { + for weight in extra { + match weight { + Extra::Table(table) => { + assert_eq!(table.len(), span, "every factor spans the same cube") + } + Extra::Eq(point) => assert!( + point.len().is_multiple_of(3) && 1usize << (point.len() / 3) == self.len, + "an eq weight's point spans the cube" + ), + } + } + // SAFETY: every slab is written below — a table by its copy, an + // `eq` table by its seed and levels, which cover all of it. + let mut buffer = unsafe { self.stream.alloc::(extra.len() * span) }?; + for (k, weight) in extra.iter().enumerate() { + let at = k * span; + match weight { + Extra::Table(table) => { + let mut slab = buffer.slice_mut(at..at + span); + self.stream.memcpy_htod(*table, &mut slab)?; + } + Extra::Eq(point) => { + eq_expand_into(&self.stream, &mut buffer, k * self.len, point, &[1, 0, 0])? + } + } + } + { + let (base, _record) = buffer.device_ptr(&self.stream); + addresses.extend((0..extra.len()).map(|k| base + (k * span * 8) as u64)); + } + held.push(Arc::new(buffer)); + } + SumcheckSession::from_device_spread( + self.stream.clone(), + &addresses, + self.len, + held, + nodes, + consts, + num_slots, + root_slot, + spread, + ) + } +} + +/// Fills `count` ext3 cells of `dst` from `offset` with one value. +pub fn fill_ext3( + stream: &Arc, + dst: &mut CudaSlice, + offset: usize, + count: usize, + value: &[u64], +) -> Result<()> { + assert_eq!(value.len(), 3, "an ext3 value"); + if count == 0 { + return Ok(()); + } + let be = backend()?; + let value_dev = crate::device::htod_or_trim(stream, value)?; + let mut target = dst.slice_mut(offset * 3..(offset + count) * 3); + let count_arg = count as u64; + unsafe { + stream + .launch_builder(&be.fill_ext3) + .arg(&mut target) + .arg(&count_arg) + .arg(&value_dev) + .launch(LaunchConfig::for_num_elems(count as u32))?; + } + Ok(()) +} + +/// `eq(point, ·)` as a table of `len` ext3 cells, doubled a variable at a time. +/// +/// Variables go in back to front, which is what leaves variable 0 in the high +/// bit — the indexing every table here folds on. +pub fn eq_table_ext3( + stream: &Arc, + point: &[u64], + len: usize, +) -> Result> { + assert_eq!( + 1usize << (point.len() / 3), + len, + "the point spans the table" + ); + let mut table = alloc_zeros_or_trim::(stream, len * 3)?; + eq_expand_into(stream, &mut table, 0, point, &[1, 0, 0])?; + Ok(table) +} + +/// The same, scaled by `seed` and written into `dst` from `offset`. +/// +/// The scale rides the seed: the table is a product over the variables, so one +/// more factor at the start scales every cell. That is what lets a stacked +/// polynomial's weight be written column by column into one buffer. +pub fn eq_expand_into( + stream: &Arc, + dst: &mut CudaSlice, + offset: usize, + point: &[u64], + seed: &[u64], +) -> Result<()> { + assert!(point.len().is_multiple_of(3), "three u64 per coordinate"); + assert_eq!(seed.len(), 3, "an ext3 seed"); + let len = 1usize << (point.len() / 3); + assert!(dst.len() >= (offset + len) * 3, "the range holds the table"); + let be = backend()?; + { + let mut head = dst.slice_mut(offset * 3..offset * 3 + 3); + stream.memcpy_htod(seed, &mut head)?; + } + // The whole point goes up once and each level reads its coordinate where + // it lies: a stacked polynomial's weight is one of these per column, so a + // send per level is tens of thousands of them for three u64 each. + let point_dev = crate::device::htod_or_trim(stream, point)?; + let vars = point.len() / 3; + for level in 0..vars { + let at = (vars - 1 - level) * 3; + let r = point_dev.slice(at..at + 3); + let filled = 1u64 << level; + let mut range = dst.slice_mut(offset * 3..(offset + len) * 3); + unsafe { + stream + .launch_builder(&be.eq_expand_level_ext3) + .arg(&mut range) + .arg(&filled) + .arg(&r) + .launch(LaunchConfig::for_num_elems(filled as u32))?; + } + } + Ok(()) +} + +/// Every share of a stacked polynomial's weight, built in one pass per level +/// instead of one pass per level per share. +/// +/// A share is `(offset, point, scale)`: its subcube's start, the point its +/// `eq` table is over, and what that table is scaled by. They are disjoint, so +/// the only reason to do them one at a time was that the kernel took one — and +/// on a real trace that is tens of thousands of launches for a tenth of a +/// second of work. +/// +/// `dst` must be zero where no share writes: the gaps between subcubes are +/// part of the weight. +pub fn eq_expand_shares_ext3( + stream: &Arc, + dst: &mut CudaSlice, + shares: &[(usize, Vec, [u64; 3])], +) -> Result<()> { + if shares.is_empty() { + return Ok(()); + } + let be = backend()?; + + // Sorted by variable count, descending: then the shares still doubling at + // level `l` are a prefix, and a thread finds its share by shifting. + let mut order: Vec = (0..shares.len()).collect(); + order.sort_by_key(|i| std::cmp::Reverse(shares[*i].1.len())); + + let mut table = Vec::with_capacity(order.len() * 3); + let mut points = Vec::new(); + let mut scales = Vec::with_capacity(order.len() * 3); + for &i in &order { + let (offset, point, scale) = &shares[i]; + assert!(point.len().is_multiple_of(3), "three u64 per coordinate"); + table.push(*offset as u64); + table.push((point.len() / 3) as u64); + table.push((points.len() / 3) as u64); + points.extend_from_slice(point); + scales.extend_from_slice(scale); + } + let max_vars = shares[order[0]].1.len() / 3; + + let table_dev = crate::device::htod_or_trim(stream, &table)?; + let scales_dev = crate::device::htod_or_trim(stream, &scales)?; + let points_dev = crate::device::htod_or_trim( + stream, + if points.is_empty() { + &[0u64][..] + } else { + &points + }, + )?; + + let count = order.len() as u64; + unsafe { + stream + .launch_builder(&be.eq_seed_shares_ext3) + .arg(&mut *dst) + .arg(&table_dev) + .arg(&scales_dev) + .arg(&count) + .launch(LaunchConfig::for_num_elems(order.len() as u32))?; + } + + for level in 0..max_vars { + // The prefix still doubling: `vars > level`, and the list is sorted. + let active = order + .iter() + .take_while(|i| shares[**i].1.len() / 3 > level) + .count() as u64; + if active == 0 { + break; + } + let total = active << level; + let grid = total.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + let level_arg = level as u64; + unsafe { + stream + .launch_builder(&be.eq_expand_level_shares_ext3) + .arg(&mut *dst) + .arg(&table_dev) + .arg(&points_dev) + .arg(&active) + .arg(&level_arg) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + } + Ok(()) +} + +/// The claim reduce's session with its tables built here, out of the epoch's +/// resident columns: per offset, in the reduce's order, the shift table and the +/// batched column. +/// +/// A shift table is `eq(alpha)` read `offset` rows back, cyclically — the shift +/// kernel at a corner `y` is the equality kernel at `y − offset`, which +/// `multilinear::eq`'s `a_shift_table_is_the_eq_table_rotated` pins — so +/// `eq(alpha)` is built once and each offset's table is two copies of it. A +/// batched column is `Σ weight·column` over the offset's members +/// (`batched_column_ext3`). +/// +/// `alpha` is three u64 per coordinate. `groups` is one `(offset, members, +/// weights)` per offset: each member a column of the run `first..first + width` +/// of `store`, each weight three u64. What crosses the bus is the point, the +/// members and their weights: a few kilobytes where the tables are gigabytes. +#[allow(clippy::too_many_arguments)] +pub fn reduce_session( + store: &crate::columns::DeviceColumns, + first: usize, + width: usize, + alpha: &[u64], + groups: &[(usize, Vec, Vec)], + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, +) -> Result { + assert!( + alpha.len().is_multiple_of(3) && !alpha.is_empty(), + "a point of at least one coordinate, three u64 each" + ); + let len = 1usize << (alpha.len() / 3); + assert!( + store.is_run(first, width) && store.span(first).1 == len, + "the columns are a run spanning the cube" + ); + assert!(!groups.is_empty(), "a reduction reads at an offset"); + for (_, members, weights) in groups { + assert!(!members.is_empty(), "an offset has a column reading at it"); + assert_eq!( + weights.len(), + members.len() * 3, + "one ext3 weight per member" + ); + assert!( + members.iter().all(|&m| (m as usize) < width), + "a member is a column of the run" + ); + } + + let be = backend()?; + let stream = be.next_stream(); + let span = len * 3; + // SAFETY: every slab is written below — a shift slab by `eq(alpha)` or by + // its two copies, a batched slab by the kernel, which covers every row. + let mut buffer = unsafe { alloc_or_trim::(&stream, 2 * groups.len() * span) }?; + // `eq(alpha)` is the first shift slab when the first offset is no shift, + // and a table of its own, freed once copied, otherwise. + let in_place = groups[0].0.is_multiple_of(len); + let own = if in_place { + eq_expand_into(&stream, &mut buffer, 0, alpha, &[1, 0, 0])?; + None + } else { + Some(eq_table_ext3(&stream, alpha, len)?) + }; + let base = { + let (base, _record) = buffer.device_ptr(&stream); + base + }; + let eq_at = match &own { + Some(table) => { + let (at, _record) = table.device_ptr(&stream); + at + } + None => base, + }; + let copy = |dst: u64, src: u64, bytes: usize| -> Result<()> { + if bytes == 0 { + return Ok(()); + } + // SAFETY: both ranges lie inside allocations this call holds, they do + // not overlap — a shift slab is never the table it is copied from — + // and the copy is ordered on `stream` behind whatever wrote the source. + unsafe { + cudarc::driver::sys::cuMemcpyDtoDAsync_v2(dst, src, bytes, stream.cu_stream()) + .result()?; + } + Ok(()) + }; + let columns = store.view(first, width); + let rows = len as u64; + let grid = rows.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32; + for (i, (offset, members, weights)) in groups.iter().enumerate() { + if !(i == 0 && in_place) { + // `shift[y] = eq[(y − k) mod len]`: the table's tail lands at its + // head, and the rest after it. + let k = offset % len; + let shift_at = base + (2 * i * span * 8) as u64; + copy(shift_at + (k * 24) as u64, eq_at, (len - k) * 24)?; + copy(shift_at, eq_at + ((len - k) * 24) as u64, k * 24)?; + } + let members_dev = crate::device::htod_or_trim(&stream, members)?; + let weights_dev = crate::device::htod_or_trim(&stream, weights)?; + let count = members.len() as u64; + let at = (2 * i + 1) * span; + let mut out = buffer.slice_mut(at..at + span); + unsafe { + stream + .launch_builder(&be.batched_column_ext3) + .arg(&columns) + .arg(&rows) + .arg(&members_dev) + .arg(&weights_dev) + .arg(&count) + .arg(&mut out) + .launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + })?; + } + } + // Freed on `stream`, so behind the copies that read it. + drop(own); + let addresses: Vec = (0..2 * groups.len()) + .map(|k| base + (k * span * 8) as u64) + .collect(); + SumcheckSession::from_device( + stream, + &addresses, + len, + vec![Arc::new(buffer)], + nodes, + consts, + num_slots, + root_slot, + ) +} + +/// A resident table's value at `point`, binding it in place. +/// +/// The table is spent: what is left in its first cell is the value. +pub fn evaluate_resident_ext3( + stream: &Arc, + values: &mut CudaSlice, + len: usize, + point: &[u64], +) -> Result<[u64; 3]> { + assert!(point.len().is_multiple_of(3), "three u64 per coordinate"); + assert_eq!( + 1usize << (point.len() / 3), + len, + "the point spans the table" + ); + let be = backend()?; + fold_to_one(stream, be, values, len as u64, point)?; + let out = crate::readback::clone_dtoh(stream, &values.slice(0..3))?; + stream.synchronize()?; + Ok([out[0], out[1], out[2]]) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A launch never runs more threads than the cube has indices, nor more + /// than the slot file can hold, and always fills at least one warp. + #[test] + fn a_launch_takes_a_thread_per_index_and_no_more() { + for slots in [1usize, 6, 64, 522, 1036, 8192] { + let ceiling = thread_ceiling(slots); + for log_work in 0..24u32 { + let work = 1u64 << log_work; + let (grid, block) = launch_shape(ceiling, work); + assert!((MIN_BLOCK..=BLOCK_DIM).contains(&block), "block {block}"); + let threads = grid as u64 * block as u64; + assert!(threads >= work.min(ceiling), "{threads} threads for {work}"); + assert!( + threads < work.min(ceiling) + block as u64, + "{threads} threads for {work}: a whole block of nothing", + ); + } + } + } + + /// A small cube spreads over blocks instead of arriving as one of them: + /// that is the whole point, and it is what a late round looks like. + #[test] + fn a_small_cube_spreads_over_the_device() { + let ceiling = thread_ceiling(1036); + for work in [128u64, 1024, 8192] { + let (grid, _) = launch_shape(ceiling, work); + assert!(grid >= 4, "{work} indices arrived as {grid} block(s)"); + } + } + + /// The partials buffer is sized for the widest round, which is not always + /// the first — the blocks narrow as the cube shrinks. + #[test] + fn the_partials_hold_every_round() { + for slots in [6usize, 1036] { + let ceiling = thread_ceiling(slots); + for log_half in 0..22u32 { + let first = 1u64 << log_half; + let widest = widest_grid(ceiling, first); + let mut work = first; + loop { + assert!(launch_shape(ceiling, work).0 <= widest); + if work <= 1 { + break; + } + work /= 2; + } + } + } + } +} + +/// A table's factors as the fused zerocheck's base-field rounds read them +/// (`crate::argue_fused`): one device address a factor, `stride` u64 between +/// rows — three for lifted factors, whose first limb is the base value, one +/// for base columns read where they lie (D-BATCH M1-2). +#[derive(Clone, Copy)] +pub struct FactorView<'f> { + pub stream: &'f Arc, + pub ptrs: &'f CudaSlice, + pub stride: u64, + pub rows: usize, + pub width: usize, +} + +/// Where a factor's base values are, for [`ColumnFactors`]: column `k` of the +/// table's resident run, or a public table's base values (uploaded). +pub enum ColumnSlot<'a> { + Column(usize), + Public(&'a [u64]), +} + +/// A table's factors as base columns on the card, with no lift (D-BATCH M1-2): +/// a committed factor is its column where the epoch's columns reside, a public +/// one a base copy uploaded here. Holds no column: the caller keeps the run it +/// points into alive for as long as this lives. +pub struct ColumnFactors { + stream: Arc, + ptrs: CudaSlice, + /// The public factors' base copies, end to end. + _public: Option>, + rows: usize, + width: usize, +} + +impl ColumnFactors { + /// `run` is the table's resident columns, `rows` each, end to end; `slots` + /// one per factor, in slot order. + pub fn new( + run: &cudarc::driver::CudaView<'_, u64>, + rows: usize, + slots: &[ColumnSlot<'_>], + ) -> Result { + assert!(rows.is_power_of_two(), "the cube is a power of two"); + let be = backend()?; + let stream = be.next_stream(); + let publics: Vec<&[u64]> = slots + .iter() + .filter_map(|slot| match slot { + ColumnSlot::Public(table) => Some(*table), + ColumnSlot::Column(_) => None, + }) + .collect(); + assert!( + publics.iter().all(|t| t.len() == rows), + "a public table spans the cube" + ); + let public = if publics.is_empty() { + None + } else { + Some(crate::device::htod_or_trim(&stream, &publics.concat())?) + }; + let run_at = { + let (at, _guard) = run.device_ptr(&stream); + at + }; + let public_at = public.as_ref().map(|buffer: &CudaSlice| { + let (at, _guard) = buffer.device_ptr(&stream); + at + }); + let mut next_public = 0u64; + let addresses: Vec = slots + .iter() + .map(|slot| match slot { + ColumnSlot::Column(k) => run_at + (k * rows * 8) as u64, + ColumnSlot::Public(_) => { + let at = public_at.expect("a public slot has its copy") + + next_public * rows as u64 * 8; + next_public += 1; + at + } + }) + .collect(); + let ptrs = crate::device::htod_or_trim(&stream, &addresses)?; + Ok(Self { + stream, + ptrs, + _public: public, + rows, + width: slots.len(), + }) + } + + pub fn view(&self) -> FactorView<'_> { + FactorView { + stream: &self.stream, + ptrs: &self.ptrs, + stride: 1, + rows: self.rows, + width: self.width, + } + } +} diff --git a/crypto/math-cuda/src/whir.rs b/crypto/math-cuda/src/whir.rs new file mode 100644 index 000000000..3e870b2f2 --- /dev/null +++ b/crypto/math-cuda/src/whir.rs @@ -0,0 +1,3179 @@ +//! Committing one stacked polynomial of the multilinear path on device. +//! +//! Mirrors the host pipeline in `multilinear`: the Möbius transform that turns +//! hypercube evaluations into monomial coefficients, the lift's bit-reverse, +//! one NTT onto the blown-up domain, then the strided-coset leaf hash and the +//! Merkle tree. Parity against that pipeline is checked by `tests/whir_commit.rs`. + +use std::mem::ManuallyDrop; +use std::sync::{Arc, Mutex, Once, Weak}; + +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, LaunchConfig, PushKernelArg}; + +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use crate::Result; +use crate::device::{DeviceReservation, alloc_or_trim, backend}; + +/// Leaf-hash passes over a codeword — one per tree actually built. +/// +/// The quantity H4 is about: a commitment that is opened used to cost TWO of +/// these, one for the root and one for the paths. It counts launches, not +/// leaves, because "how many times was this codeword's leaf layer hashed" is +/// the question, and a test can assert an integer. +static LEAF_HASH_CALLS: AtomicU64 = AtomicU64::new(0); + +pub fn leaf_hash_calls() -> u64 { + LEAF_HASH_CALLS.load(Ordering::Relaxed) +} + +pub fn reset_leaf_hash_calls() { + LEAF_HASH_CALLS.store(0, Ordering::Relaxed); +} + +/// Leaf-hash passes over ONE codeword. +/// +/// ⚠ The global [`LEAF_HASH_CALLS`] is a diagnostic: it is process-wide, so a +/// test asserting on it is asserting about every other test sharing the binary +/// too. That is not a hypothetical — the first gate run of this file had a +/// counting test read 5 instead of 1 purely because its neighbours were +/// committing at the same time. A per-codeword count is what an assertion can +/// actually be about, and it localises a failure to the codeword that caused +/// it rather than to whoever ran alongside. +type BuildCount = Arc; + +/// Trees ASSEMBLED, process-wide — the twin of [`LEAF_HASH_CALLS`] and no +/// longer the same number as it. +/// +/// ★ Before the leaf layer was retained these two counted the same event, which +/// is exactly why the retention needs both: a tree is still assembled for every +/// opening (the inner levels are rebuilt), but its LEAF PASS is skipped when a +/// matching layer is in hand. `leaf_hash_calls` well below `tree_builds` is the +/// retention working; the two equal is the retention not taken. +static TREE_BUILDS: AtomicU64 = AtomicU64::new(0); + +pub fn tree_builds() -> u64 { + TREE_BUILDS.load(Ordering::Relaxed) +} + +/// Leaf layers this process asked to retain, and what happened. +/// +/// ⛔ REPORTED, ALWAYS. A run that retains nothing must say so on its own line +/// rather than reading as a lever that quietly did not fire: the refusal path +/// below is what makes the scheme safe at full size, so how often it fires is +/// the first thing any reading of the lever has to know. +static RETAIN_ADMITTED: AtomicU64 = AtomicU64::new(0); +static RETAIN_REFUSED: AtomicU64 = AtomicU64::new(0); +static RETAIN_BYTES_ASKED: AtomicU64 = AtomicU64::new(0); +static RETAIN_BYTES_ADMITTED: AtomicU64 = AtomicU64::new(0); +/// Bytes the budget still had when it first refused — zero if it never did. +static RETAIN_FIRST_REFUSAL_HEADROOM: AtomicU64 = AtomicU64::new(0); +/// Leaf passes SKIPPED because a matching layer was in hand. The saving, counted +/// where it happens rather than inferred from two other counters. +static LEAF_PASSES_SAVED: AtomicU64 = AtomicU64::new(0); +/// ★ The SIMULTANEOUS retained device bytes held RIGHT NOW — `fetch_add` in +/// [`KeptNodes::promise`], `fetch_sub` in its `Drop`. Unlike +/// [`RETAIN_BYTES_ADMITTED`], which only ever rises (cumulative over the run), +/// this rises and falls with the live layers, so it is the TRUE footprint: the +/// quantity that contends with argue for the shared budget, and the one an +/// eviction gives back. wt16 reported the cumulative 39,057 MiB and had no name +/// for the ~2 GiB that actually bound. +static RETAIN_BYTES_LIVE: AtomicU64 = AtomicU64::new(0); +/// ★ The PEAK of [`RETAIN_BYTES_LIVE`] over the run — the number the print wants. +/// The instantaneous live count is ~0 by the time the base readback prints (the +/// codewords, and their layers, have dropped), so the reporting quantity is this +/// high-water: the largest simultaneous retained footprint the run ever held. +static RETAIN_BYTES_LIVE_PEAK: AtomicU64 = AtomicU64::new(0); + +/// What the leaf-layer retention did this process: admitted, refused, the bytes +/// on each side, the headroom at the first refusal, and the leaf passes skipped. +pub fn retention_report() -> (u64, u64, u64, u64, u64, u64) { + ( + RETAIN_ADMITTED.load(Ordering::Relaxed), + RETAIN_REFUSED.load(Ordering::Relaxed), + RETAIN_BYTES_ASKED.load(Ordering::Relaxed), + RETAIN_BYTES_ADMITTED.load(Ordering::Relaxed), + RETAIN_FIRST_REFUSAL_HEADROOM.load(Ordering::Relaxed), + LEAF_PASSES_SAVED.load(Ordering::Relaxed), + ) +} + +/// ★ The retained leaf-layer device bytes held RIGHT NOW — the live footprint, +/// as opposed to `retention_report`'s cumulative `admitted` bytes. This is the +/// quantity that contends with argue for the budget; a reading well below the +/// cumulative total is the layers being freed as their codewords drop (or an +/// eviction giving them back). +pub fn retained_bytes_live() -> u64 { + RETAIN_BYTES_LIVE.load(Ordering::Relaxed) +} + +/// ★ The PEAK simultaneous retained device footprint over the run — the largest +/// [`retained_bytes_live`] ever reached. This is the number that bound argue in +/// wt16 (~2 GiB); the instantaneous count is ~0 once the codewords drop, so this +/// is what a report prints. Process-wide and monotone (a high-water), so a fresh +/// process — which is how the launcher runs each prove — starts it at 0. +pub fn retained_bytes_peak() -> u64 { + RETAIN_BYTES_LIVE_PEAK.load(Ordering::Relaxed) +} + +/// What a codeword keeps of its tree past the call that built it: the whole +/// tree (the default, [`whole_trees`]) or its leaf layer alone — and in either +/// case NOTHING outside a promise. +/// +/// ⛔ THE INVARIANT: every byte kept here is promised — grown into the room of +/// the codeword that captured it, given back when it drops — and the evictor +/// hands it to any request it can cover. H4 kept whole node arrays OUTSIDE any +/// promise, so a full card pushed commits to the host at ~1.5 GiB a chain, and +/// it lost ~15 s. The leaf layer came first — half the bytes, most of the +/// saving: a round-0 leaf absorbs a whole `2^k` coset, an inner node two +/// digests — and the whole tree came back once both were promised and +/// evictable (job 248, −1.00 s): what separates it from H4 is where the bytes +/// sit, not how many there are. +/// +/// ⛔ THE KEY IS PART OF THE OBJECT. A leaf is the `2^log_folding` coset that +/// folds onto one position, so a layer is valid ONLY for the width it was built +/// at, and only for the hash that built it. `paths()` takes `log_folding` as a +/// PARAMETER, and `whir_tree_cache.rs`'s blocking test opens the same codeword +/// at two widths on purpose: served across widths, this would hand back paths +/// that are internally consistent and WRONG, which is the outcome that file +/// exists to forbid. An exact match or a rebuild; there is no near miss. +struct RetainedLeaves { + /// The leaf layer, or the whole node array when `whole`, with the promise + /// it holds. Shared so an opening served a whole tree can read it after the + /// slot's lock is let go; the promise goes with the LAST handle, never with + /// the slot's alone ([`KeptNodes`]). + nodes: Arc, + log_folding: usize, + hash: crate::DeviceHash, + num_leaves: usize, + /// The whole tree rather than the leaf layer alone. + whole: bool, +} + +impl RetainedLeaves { + /// The bytes this holds, all of them promised. + fn bytes(&self) -> u64 { + self.nodes.bytes + } + + /// Whether an opening holds this right now: the slot's handle is not the + /// only one. + /// + /// ⛔ READ IT UNDER THE SLOT'S LOCK. An opening takes its handle under that + /// lock ([`DeviceCodeword::retained_tree`]) and lets it go without one, so + /// under the lock the count can fall but never rise: a `false` read there + /// stays right for as long as the lock is held, and a `true` can only go + /// stale in the safe direction — a tree kept a moment longer than it had + /// to be. + fn read_by_an_opening(&self) -> bool { + Arc::strong_count(&self.nodes) > 1 + } +} + +/// A kept device buffer that answers for its own promise: grown into its room +/// when it is admitted, given back by its own `Drop`. +/// +/// ⛔ THE PROMISE LEAVES WITH THE BUFFER, NOT WITH THE SLOT. An opening served +/// a whole tree reads it through its own handle after the slot's lock is let +/// go, so emptying the slot — an eviction, or the codeword dropping — does not +/// free the buffer while that opening reads. The bytes are given back when the +/// LAST handle drops, which is also when the buffer is freed; given back by +/// whoever emptied the slot, the ledger would count less than the card holds +/// for as long as the reading lasted. +/// +/// ⛔ AND THE ROOM LIVES AS LONG AS THIS DOES. The handle is strong, so a room +/// cannot drop — giving back every byte it counts, these included — while a +/// buffer it counts is still held (the evictor's walk can hold a slot past its +/// codeword for as long as it looks at it). A kept buffer outlives its +/// codeword only there, so the rest of the room stays counted for that long, +/// on the safe side. +/// +/// The room is the codeword's, which a fold SHARES with the codeword it came +/// from. So it is shrunk HERE, not by whoever dropped the layer: a fold drops +/// at the end of its round while the source lives on to the end of the proof, +/// and shrunk only by the evictor, a fold's layer stayed promised until the +/// source dropped — every opened chain leaving its folds' layers in the budget +/// for the rest of the group, and a commitment held across epochs gaining a set +/// per epoch. +struct KeptNodes { + /// Freed in `Drop`, before the promise goes back. + buffer: ManuallyDrop>, + bytes: u64, + room: Arc, +} + +impl KeptNodes { + /// Promise `bytes` for `buffer` in `room`, or hand the buffer back when the + /// budget will not take them — + /// [`grow`](DeviceReservation::grow) changes nothing when it refuses. + fn promise( + buffer: CudaSlice, + bytes: u64, + room: &Arc, + ) -> core::result::Result> { + if !room.grow(bytes) { + return Err(buffer); + } + // The live footprint rises here and falls in `Drop`, so the two balance + // over the buffer's lifetime; the peak is kept for the print, since the + // instantaneous count is ~0 by the time it is read. + let now_live = RETAIN_BYTES_LIVE.fetch_add(bytes, Ordering::Relaxed) + bytes; + RETAIN_BYTES_LIVE_PEAK.fetch_max(now_live, Ordering::Relaxed); + Ok(Self { + buffer: ManuallyDrop::new(buffer), + bytes, + room: Arc::clone(room), + }) + } + + fn buffer(&self) -> &CudaSlice { + &self.buffer + } + + fn buffer_mut(&mut self) -> &mut CudaSlice { + &mut self.buffer + } +} + +impl Drop for KeptNodes { + /// The other half of both accountings: what [`KeptNodes::promise`] added to + /// [`RETAIN_BYTES_LIVE`] and to the room goes back here, once the free is + /// enqueued. cudarc's `CudaSlice::drop` frees on the slice's own stream, + /// after any work queued there on it. + fn drop(&mut self) { + // SAFETY: `buffer` is dropped exactly once, here, and nothing reads it + // afterwards: `self` is being dropped. + unsafe { ManuallyDrop::drop(&mut self.buffer) }; + RETAIN_BYTES_LIVE.fetch_sub(self.bytes, Ordering::Relaxed); + self.room.shrink(self.bytes); + } +} + +/// What [`DeviceCodeword::hold_kept_tree`] hands a test: a served opening's +/// handle on a kept tree. The tree, and its promise, stay while this lives. +#[doc(hidden)] +pub struct KeptTreeHold { + _tree: Arc, +} + +/// Retention evictions this process has done, and the bytes they gave back — the +/// evictable scheme working under pressure. `evicted > 0` with `device fallbacks +/// 0` is argue getting its budget FROM the retention instead of from the host. +static RETAIN_EVICTED: AtomicU64 = AtomicU64::new(0); +static RETAIN_BYTES_EVICTED: AtomicU64 = AtomicU64::new(0); + +/// Kept trees an eviction passed over because an opening was reading them — +/// see [`evict_retained_layers`]. +static RETAIN_READ_PASSES: AtomicU64 = AtomicU64::new(0); + +/// Kept trees the evictor passed over this process because an opening was +/// reading them at that moment. +pub fn retention_read_passes() -> u64 { + RETAIN_READ_PASSES.load(Ordering::Relaxed) +} + +/// (evictions, bytes evicted) this process — see [`RETAIN_EVICTED`]. +pub fn retention_evictions() -> (u64, u64) { + ( + RETAIN_EVICTED.load(Ordering::Relaxed), + RETAIN_BYTES_EVICTED.load(Ordering::Relaxed), + ) +} + +/// Budget misses the retained layers could not have covered, and the bytes the +/// layers could give back at each — see [`keep_futile`]. +static RETAIN_FUTILE_MISSES: AtomicU64 = AtomicU64::new(0); +static RETAIN_FUTILE_BYTES: AtomicU64 = AtomicU64::new(0); + +/// (futile misses, the layers' reclaimable bytes summed over them, whether they +/// were kept) this process — see [`keep_futile`]. Evicted, the bytes are what +/// went for nothing; kept, a layer held through two misses counts twice. +pub fn retention_futile() -> (u64, u64, bool) { + ( + RETAIN_FUTILE_MISSES.load(Ordering::Relaxed), + RETAIN_FUTILE_BYTES.load(Ordering::Relaxed), + keep_futile(), + ) +} + +/// The switch [`keep_futile`] reads. +const KEEP_FUTILE_ENV: &str = "LFM_WHIR_KEEP_FUTILE"; +/// Keep. Job 246 (A B B A on the block): the whole run −1.25 s and the openings' +/// tree rebuilds −1.39 s, with no layer evicted. The argue's peak in epochs 3–5 +/// rose by the layers it kept, to 25,034 of the 25,688 MiB budget. +const KEEP_FUTILE_DEFAULT: bool = true; + +/// ★ Whether a budget miss the retained layers CANNOT cover leaves them held. +/// +/// `Backend::reserve` asks the evictor once for its deficit before it gives up. +/// When every reclaimable layer together is less than that deficit, dropping +/// them cannot make the request fit: the reserve fails anyway, and each opening +/// that needed a dropped layer hashes its leaves again. The GKR input-layer +/// tree's eager probe is such a request: it asks for the whole carry, and its +/// `None` only picks the lazy tree. +/// +/// On the block every eviction was of this kind. Job 243 evicted fifteen layers +/// (2,432 MiB), and its whole-run ledger peak, 24,342 MiB, is below the 24,855 +/// MiB a request that fit after evicting would have left. Its epochs 3–5 +/// rebuilt 1.42 s more trees than epochs of the same shape, of the openings' +/// 2.41 s — by the permutation count, those layers hashed again. +/// +/// A miss the layers CAN cover evicts as before, so every `reserve` gets the +/// answer it got without this switch; only the layers a failing request would +/// have taken with it stay. +/// +/// `LFM_WHIR_KEEP_FUTILE` unset or `1` keeps them (the default); `0` evicts +/// them as before the switch; any other value aborts. Read once and printed +/// once; a test forces it with [`force_keep_futile`]. +fn keep_futile() -> bool { + match KEEP_FUTILE_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + crate::rpx_paths::read( + KEEP_FUTILE_ENV, + KEEP_FUTILE_DEFAULT, + "WHIR retention: a budget miss the leaf layers cannot cover keeps them", + "WHIR retention: a budget miss evicts leaf layers even when they cannot cover it", + ) + }) + } + } +} + +static KEEP_FUTILE_FORCED: core::sync::atomic::AtomicU8 = core::sync::atomic::AtomicU8::new(0); + +/// Overrides `LFM_WHIR_KEEP_FUTILE` for the whole process — for a test that +/// takes a futile miss both ways in one binary. `None` restores the +/// environment's setting. Not for production callers. +#[doc(hidden)] +pub fn force_keep_futile(on: Option) { + KEEP_FUTILE_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Openings served a kept whole tree, which built nothing — see [`whole_trees`]. +static WHOLE_TREE_SERVES: AtomicU64 = AtomicU64::new(0); + +/// (whether whole trees are kept, the openings they served) this process. +pub fn retention_whole() -> (bool, u64) { + (whole_trees(), WHOLE_TREE_SERVES.load(Ordering::Relaxed)) +} + +/// The switch [`whole_trees`] reads. +const WHOLE_TREES_ENV: &str = "LFM_WHIR_WHOLE_TREES"; +/// Whole trees. Job 248 (A B B A on the block): the whole run −1.00 s, the base +/// −0.90 s, the openings' tree rebuilds −0.82 s; 952 openings served a kept +/// tree, and epoch 4's argue evicted two kept trees. The argue's peaks in +/// epochs 3–5 were 25,633 / 25,306 / 25,681 of the 25,688 MiB budget — epoch 5 +/// within 7 MiB of it, and safe because everything kept is evictable. +const WHOLE_TREES_DEFAULT: bool = true; + +/// ★ Whether a tree is kept WHOLE past the call that built it, not just its +/// leaf layer. +/// +/// With the leaf layer kept, every opening still builds the inner levels, and +/// on the block (job 246) that was the openings' 1.02 s of tree rebuilds: one +/// permutation a node, level by level, the top levels in one block. The whole +/// tree costs one more layer's bytes (`num_leaves − 1` nodes), and it is +/// promised the same way — grown into the codeword's room, and given back to +/// the evictor under pressure, whole — so a request it can cover still gets its +/// bytes, and fallbacks cannot rise. The worst case is the old rebuild. +/// +/// `LFM_WHIR_WHOLE_TREES` unset or `1` keeps whole trees (the default); `0` +/// keeps leaf layers, as before the switch; any other value aborts. Read once +/// and printed once; a test forces it with [`force_whole_trees`]. +fn whole_trees() -> bool { + match WHOLE_TREES_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + crate::rpx_paths::read( + WHOLE_TREES_ENV, + WHOLE_TREES_DEFAULT, + "WHIR retention: whole trees, the inner levels kept with the leaves", + "WHIR retention: leaf layers, the inner levels rebuilt at each opening", + ) + }) + } + } +} + +static WHOLE_TREES_FORCED: core::sync::atomic::AtomicU8 = core::sync::atomic::AtomicU8::new(0); + +/// Overrides `LFM_WHIR_WHOLE_TREES` for the whole process — for a test that +/// opens the same codeword both ways in one binary. `None` restores the +/// environment's setting. Not for production callers. +#[doc(hidden)] +pub fn force_whole_trees(on: Option) { + WHOLE_TREES_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Whether evicting `reclaimable` bytes of layers is too little for a miss of +/// `target`: the reserve fails whatever the evictor does. Zero reclaimable is +/// not futile — there is nothing to keep. +fn is_futile(target: u64, reclaimable: u64) -> bool { + reclaimable > 0 && reclaimable < target +} + +/// Whether the evictor walks its layers for a miss of `target`: always, as +/// before the switch, unless the miss is futile and the switch keeps them. +fn evicts(target: u64, reclaimable: u64, keep_futile: bool) -> bool { + !(keep_futile && is_futile(target, reclaimable)) +} + +/// Seconds since the Unix epoch, the clock the prover's `t=` stamps read. +fn unix_secs() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()) +} + +/// Whether the leaf-layer retention is enabled this process. `LFM_WHIR_RETENTION=0` +/// turns it OFF (capture becomes a no-op), which is how the ABBA's control arm +/// runs off the SAME binary as the retention arm — the two differ only by this +/// flag. Read once and cached; default ON. +fn retention_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + std::env::var("LFM_WHIR_RETENTION") + .map(|v| v != "0") + .unwrap_or(true) + }) +} + +/// A handle the allocator can walk to reclaim a codeword's retained layer under +/// pressure. `Weak`, so the registry never keeps a codeword alive: `leaves` +/// points at the codeword's `Arc>>` — the MUTEX +/// outlives any single layer, so this entry SURVIVES an eviction and a later +/// re-capture refills the same `Option`. The kept buffer knows its own room and +/// gives the budget back when its last handle drops ([`KeptNodes`]). A dead +/// `leaves` upgrade means the codeword is gone and the eviction walk prunes the +/// entry. +struct RetentionHandle { + leaves: Weak>>, +} + +/// Every retaining codeword's handle, registered once at its first capture and +/// pruned when the codeword drops. Walked only on the rare eviction path. +static RETENTION_REGISTRY: Mutex> = Mutex::new(Vec::new()); + +/// Install [`evict_retained_layers`] as the allocator's evictor, once, and say +/// which way it treats a miss it cannot cover ([`keep_futile`]) and what it +/// keeps ([`whole_trees`]). +fn ensure_evictor_installed() { + static INSTALLED: Once = Once::new(); + INSTALLED.call_once(|| { + keep_futile(); + whole_trees(); + crate::device::set_retention_evictor(evict_retained_layers) + }); +} + +/// Register a codeword's layer slot for eviction. Called OUTSIDE the layer lock +/// (it takes only the registry lock), so the one lock nesting in the system is +/// the evictor's registry→layer and there is no cycle. +fn register_retained(leaves: &Arc>>) { + let handle = RetentionHandle { + leaves: Arc::downgrade(leaves), + }; + if let Ok(mut reg) = RETENTION_REGISTRY.lock() { + reg.push(handle); + } +} + +/// ⛔ THE EVICTOR — installed into `device::reserve`, called on its budget-miss +/// path on the RESERVING thread. Frees at least `target` bytes of BUDGET by +/// dropping retained leaf layers and whole trees, FIFO (oldest first), pruning +/// dead entries as it goes; returns the bytes freed. +/// +/// ★ When what it can give back cannot cover `target` the miss is FUTILE: the +/// reserve fails whatever is dropped. It is counted and said either way, and +/// under [`keep_futile`] nothing is dropped and this returns 0. +/// +/// ★ A kept tree an opening is reading is PASSED OVER, and not counted as +/// reclaimable: its promise stays until the opening lets go ([`KeptNodes`]), so +/// dropping the slot's handle would give nothing back and lose the tree. A +/// request only that tree could cover fails, counted, rather than being handed +/// bytes the card still holds. +/// +/// SAFETY, the load-bearing facts: +/// - NO REENTRANCY. Dropping the last handle of a [`KeptNodes`] is two counter +/// subtracts, one of them `room.shrink` (a lock-free `be.reserved` subtract), +/// and a stream-ordered free. It NEVER calls `reserve`/`grow`, so it cannot +/// re-enter the allocator that called it. +/// - NO UAF, AND NO BYTE UNCOUNTED. It takes the layer out UNDER the codeword's +/// own `leaves` mutex, so a concurrent [`serve_retained_leaves`] either ran +/// first (its `memcpy_dtod` is already enqueued on the codeword's stream) or +/// sees `None` and rebuilds. An opening served a whole tree takes its own +/// handle under the same mutex, so what the walk takes has no other handle +/// (checked there) and can get none once the slot is empty: the buffer is +/// freed and its promise given back as the walk drops it. The evicted buffer +/// was allocated on the codeword's stream, and cudarc 0.19.4 +/// `CudaSlice::drop` (core.rs:776-795) first waits on the slice's own read/write +/// events, then frees on the slice's OWN stream — `free_async(ptr, +/// self.stream.cu_stream)` when `has_async_alloc`, else `synchronize()` + +/// `free_sync`. So the free is ordered after any serve copy on that stream (or +/// a full device sync). The only unsafe shape — an async free on a DIFFERENT +/// stream than the copy — cannot arise here: both are the codeword's stream. +/// - LOCK ORDER. Registry mutex, then ONE layer mutex at a time (released before +/// the next); `be.reserved` is lock-free. Capture registers OUTSIDE the layer +/// lock, so nothing ever holds layer→registry — acyclic. +/// - `shrink` frees the BUDGET, not the bytes: never-purge keeps the raw device +/// allocation pooled for reuse, and the budget is exactly what argue's +/// `reserve` is gated on ([[gpu-two-vramgates-overlap]]). +fn evict_retained_layers(target: u64) -> u64 { + let mut freed = 0u64; + let mut reg = match RETENTION_REGISTRY.lock() { + Ok(reg) => reg, + Err(poisoned) => poisoned.into_inner(), + }; + // What an eviction can give back now: everything kept that no opening is + // reading. Same lock order as the walk below. + let (mut reclaimable, mut being_read) = (0u64, 0u64); + reg.retain(|handle| { + let Some(leaves) = handle.leaves.upgrade() else { + return false; + }; + let slot = match leaves.lock() { + Ok(slot) => slot, + Err(poisoned) => poisoned.into_inner(), + }; + match slot.as_ref() { + Some(layer) if layer.read_by_an_opening() => being_read += 1, + Some(layer) => reclaimable += layer.bytes(), + None => {} + } + true + }); + let keep = keep_futile(); + let futile = is_futile(target, reclaimable); + if futile { + let misses = RETAIN_FUTILE_MISSES.fetch_add(1, Ordering::Relaxed) + 1; + RETAIN_FUTILE_BYTES.fetch_add(reclaimable, Ordering::Relaxed); + eprintln!( + "[whir] retention: a reserve missed the budget by {} MiB; the retention can give back {} MiB — {}; \ + futile misses {misses} t={:.3}{}", + target >> 20, + reclaimable >> 20, + if keep { + "kept" + } else { + "evicted anyway, and the reserve still misses" + }, + unix_secs(), + BeingRead(being_read), + ); + } + if !evicts(target, reclaimable, keep) { + return 0; + } + let (mut layers, mut trees, mut passed) = (0u64, 0u64, 0u64); + reg.retain(|handle| { + let Some(leaves) = handle.leaves.upgrade() else { + return false; // the codeword is gone; prune this dead entry + }; + if freed >= target { + return true; // enough freed; keep the rest for next time + } + // Take the layer out under its own mutex — a concurrent serve holds this + // same lock across its copy enqueue, so the free cannot race it — unless + // an opening is reading it, which keeps it until the opening lets go. + let taken = { + let mut slot = match leaves.lock() { + Ok(slot) => slot, + Err(poisoned) => poisoned.into_inner(), + }; + if slot + .as_ref() + .is_some_and(RetainedLeaves::read_by_an_opening) + { + passed += 1; + None + } else { + slot.take() + } + }; + if let Some(layer) = taken { + let bytes = layer.bytes(); + if layer.whole { + trees += 1; + } else { + layers += 1; + } + // The walk holds the only handle: `KeptNodes`'s drop frees the + // buffer (stream-ordered) and gives the BUDGET back to its room + // (for argue), here. + drop(layer); + freed += bytes; + RETAIN_EVICTED.fetch_add(1, Ordering::Relaxed); + RETAIN_BYTES_EVICTED.fetch_add(bytes, Ordering::Relaxed); + } + true // keep the entry: the mutex lives with the codeword and may refill + }); + RETAIN_READ_PASSES.fetch_add(passed, Ordering::Relaxed); + if !futile && (layers + trees > 0 || passed > 0) { + eprintln!( + "[whir] retention: a reserve missed the budget by {} MiB; evicted {trees} whole tree(s) and {layers} \ + leaf layer(s), {} MiB, to cover it t={:.3}{}", + target >> 20, + freed >> 20, + unix_secs(), + BeingRead(passed), + ); + } + freed +} + +/// The eviction lines' note on kept trees an opening was reading, and so kept: +/// nothing when there were none. +struct BeingRead(u64); + +impl core::fmt::Display for BeingRead { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0 { + 0 => Ok(()), + n => write!( + f, + " · {n} kept tree(s) passed over: an opening was reading them" + ), + } + } +} + +/// May a layer built under `kept` be served for a tree asked for under `want`? +/// +/// ⛔ A FREE FUNCTION ON PURPOSE. Everything else on this path needs a device, +/// so this predicate would otherwise be checkable only on the box — and it is +/// the one piece of the retention whose failure is not slowness but WRONG +/// PATHS, internally consistent and verifying against nothing. Lifted out, it +/// takes unit cases on any machine. +/// +/// All three parts must agree. `num_leaves` is not redundant with +/// `log_folding`: the same width over a different codeword length is a +/// different tree, and a clone of a `DeviceCodeword` shares this layer. +fn leaf_key_matches( + kept: (usize, crate::DeviceHash, usize), + want: (usize, crate::DeviceHash, usize), +) -> bool { + kept.0 == want.0 && kept.1 == want.1 && kept.2 == want.2 +} + +#[cfg(test)] +mod leaf_key_tests { + use super::leaf_key_matches; + use crate::DeviceHash; + + /// The exact match is the only match, and each part is shown to matter on + /// its own — a predicate that only ever saw agreement would pass while + /// ignoring two of its three arguments. + #[test] + fn a_retained_leaf_layer_is_served_only_under_its_own_key() { + let kept = (4usize, DeviceHash::Rpx256, 1024usize); + assert!(leaf_key_matches(kept, kept), "an exact match must serve"); + assert!( + !leaf_key_matches(kept, (2, DeviceHash::Rpx256, 1024)), + "a different FOLD WIDTH describes a different tree: serving it would hand back paths of the wrong depth that verify against themselves" + ); + assert!( + !leaf_key_matches(kept, (4, DeviceHash::Keccak256, 1024)), + "a different HASH describes a different tree" + ); + assert!( + !leaf_key_matches(kept, (4, DeviceHash::Rpx256, 512)), + "the same width over a different codeword length is a different tree" + ); + assert!( + !leaf_key_matches(kept, (2, DeviceHash::Keccak256, 512)), + "and all three wrong is still not a match" + ); + } +} + +#[cfg(test)] +mod futile_miss_tests { + use super::{evicts, is_futile}; + + const MIB: u64 = 1 << 20; + + /// A miss is futile exactly when the layers hold something and less than the + /// deficit. Both edges are pinned: at equality the eviction covers the miss, + /// and with nothing held there is nothing to keep. + #[test] + fn a_miss_is_futile_only_when_the_layers_hold_less_than_it() { + assert!(is_futile(900 * MIB, 832 * MIB), "832 MiB cannot cover 900"); + assert!(!is_futile(832 * MIB, 832 * MIB), "equality covers the miss"); + assert!(!is_futile(100 * MIB, 832 * MIB), "a small miss is covered"); + assert!(!is_futile(900 * MIB, 0), "nothing held is nothing to keep"); + } + + /// The switch changes one cell of the table and no other: a futile miss + /// under `keep`. A covered miss evicts either way, which is what keeps every + /// `reserve`'s answer the same; and without the switch every miss walks the + /// layers, as before it existed. + #[test] + fn only_a_futile_miss_under_the_switch_keeps_the_layers() { + for (target, reclaimable) in [ + (900 * MIB, 832 * MIB), + (100 * MIB, 832 * MIB), + (900 * MIB, 0), + ] { + assert!( + evicts(target, reclaimable, false), + "switch off: every miss walks the layers ({target}, {reclaimable})" + ); + } + assert!( + !evicts(900 * MIB, 832 * MIB, true), + "switch on: a futile miss keeps them" + ); + assert!( + evicts(100 * MIB, 832 * MIB, true), + "switch on: a covered miss still evicts, or a reserve that fit before would fail" + ); + assert!( + evicts(900 * MIB, 0, true), + "switch on: nothing held, so the walk runs (and only prunes)" + ); + } + + /// ★ The default keeps the layers through a futile miss (job 246). Going + /// back is `LFM_WHIR_KEEP_FUTILE=0` at run time; a change here moves the + /// default for every run, and must fail this first. + #[test] + fn the_default_keeps_the_layers_through_a_futile_miss() { + assert!( + !evicts(900 * MIB, 832 * MIB, super::KEEP_FUTILE_DEFAULT), + "the default must keep the layers through a miss they cannot cover" + ); + assert!( + evicts(100 * MIB, 832 * MIB, super::KEEP_FUTILE_DEFAULT), + "and still evict for a miss they can" + ); + } +} + +#[cfg(test)] +mod whole_tree_tests { + use super::tree_bytes; + + /// ★ The default keeps whole trees (job 248). Going back is + /// `LFM_WHIR_WHOLE_TREES=0` at run time; a change here moves every run's + /// default, and must fail this first. + #[test] + fn the_default_keeps_whole_trees() { + let default = super::WHOLE_TREES_DEFAULT; + assert!(default, "the default must keep whole trees"); + } + + /// What a whole tree adds to the promise over its leaf layer is its inner + /// levels: one node fewer than the leaves. So a kept tree costs one more + /// layer, less 32 bytes — the figure the argue's margin was sized with. + #[test] + fn a_kept_tree_is_its_leaf_layer_and_one_layer_more_less_a_node() { + for log_leaves in [1usize, 12, 23] { + let layer = 32u64 << log_leaves; + assert_eq!(tree_bytes(log_leaves), 2 * layer - 32); + } + } +} + +#[cfg(test)] +mod fold_transient_tests { + use super::fold_transient_bytes; + + /// ★ The round-0 fold's transient, in numbers: level by level the first + /// two levels' outputs are live together — 2.25× the committed base + /// codeword `C` — while fused it is only the output, `3C/64` at six levels + /// (extension values, a 64th as many). At the fat stack's 2^28 variables + /// (a 2^30 codeword, `C` = 8 GiB) that is 18 GiB against 384 MiB. + #[test] + fn a_fused_fold_holds_only_its_output() { + for n in [25usize, 27, 28] { + let log_elements = n + 2; + let codeword = 8u64 << log_elements; + let stepped = fold_transient_bytes(log_elements, 6, false); + let fused = fold_transient_bytes(log_elements, 6, true); + println!( + "2^{n}: level by level {:.2} C, fused {:.4} C", + stepped as f64 / codeword as f64, + fused as f64 / codeword as f64 + ); + assert_eq!(stepped * 4, codeword * 9); + assert_eq!(fused * 64, codeword * 3); + assert!(fused * 48 == stepped); + } + assert_eq!( + fold_transient_bytes(20, 1, false), + fold_transient_bytes(20, 1, true) + ); + } + + /// ★ A commit holds its coefficients or its tree, never both — and at the + /// production first fold of six the coefficients are the larger, a quarter + /// of the codeword `C`. Under a uniform fold of four the tree is (half of + /// `C`, less a node), and two of those in flight are, to 64 bytes, the one + /// codeword a group's room always promised: that room was a commit turn. + #[test] + fn a_commit_holds_its_coefficients_or_its_tree_never_both() { + use super::commit_transient_bytes; + for n in [16usize, 25, 27, 28] { + let codeword = 8u64 << (n + 2); + assert_eq!(commit_transient_bytes(n, 2, 6) * 4, codeword); + let uniform = commit_transient_bytes(n, 2, 4); + assert_eq!(uniform, codeword / 2 - 32); + assert_eq!(codeword - 2 * uniform, 64); + } + } +} + +use crate::merkle::{build_inner_tree_levels, keccak_launch_cfg}; + +/// The coset-leaf kernel `hash` runs over a base or ext3 codeword, with its +/// launch geometry for `num_leaves` leaves. A key with no WHIR kernels is a +/// refused launch, never another hash's kernel under its name. +fn coset_leaf_kernel( + be: &crate::device::Backend, + hash: crate::DeviceHash, + base: bool, + num_leaves: u64, +) -> Result<(&cudarc::driver::CudaFunction, LaunchConfig)> { + let kernel = match (hash, base) { + (crate::DeviceHash::Keccak256, true) => &be.keccak256_leaves_base_coset, + (crate::DeviceHash::Keccak256, false) => &be.keccak256_leaves_ext3_coset, + (crate::DeviceHash::Rpx256, true) => &be.rpx_leaves_base_coset, + (crate::DeviceHash::Rpx256, false) => &be.rpx_leaves_ext3_coset, + (crate::DeviceHash::Poseidon1, base) => { + let k = crate::p1_stark::kernels()?; + let kernel = if base { + &k.leaves_base_coset + } else { + &k.leaves_ext3_coset + }; + return Ok((kernel, crate::p1_stark::launch_cfg(num_leaves))); + } + (crate::DeviceHash::Blake3, _) => return Err(crate::invalid_value()), + }; + Ok((kernel, keccak_launch_cfg(num_leaves))) +} + +/// The authentication paths of `positions` against a resident tree under +/// `hash`: `depth` siblings a path in the binary heap, three a level under +/// Poseidon1's arity-4 layout (`crate::p1_stark::gather_paths_dev`). +fn gather_paths( + hash: crate::DeviceHash, + nodes: &CudaSlice, + num_leaves: usize, + positions: &[u32], + stream: &Arc, +) -> Result> { + if hash.arity() == crate::p1_stark::ARITY { + crate::p1_stark::gather_paths_dev(nodes, num_leaves, positions, stream) + } else { + crate::merkle::gather_merkle_paths_dev(nodes, num_leaves, positions, stream) + } +} + +/// Where a tree puts the leaf of fold block `j`: at position `j`, or at +/// `bitrev(j)` over the tree's depth (`multilinear::whir_commit::LeafOrder`). +/// Only a round-0 tree is ever reversed, and only under Poseidon1. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum LeafPlacement { + #[default] + Natural, + Reversed, +} + +impl LeafPlacement { + /// The leaf at tree position `pos` of a tree of `2^depth` leaves (the map + /// is its own inverse). + pub fn leaf_at(self, pos: u64, depth: u32) -> u64 { + match self { + Self::Natural => pos, + Self::Reversed if depth == 0 => pos, + Self::Reversed => pos.reverse_bits() >> (64 - depth), + } + } +} + +/// A codeword the device holds, base-field or ext3. +/// +/// The commit leaves one here and the chain folds it here: it is the biggest +/// array the proof moves, and the host only ever needs the few values a query +/// opens. +#[derive(Clone)] +pub struct DeviceCodeword { + buffer: Arc>, + stream: Arc, + elements: usize, + base: bool, + /// Where this codeword's tree puts its leaves; set by the commit that + /// builds the tree, and the same for every rebuild. + placement: LeafPlacement, + /// Trees this codeword has built: the commit's, and one for each opening + /// that no kept whole tree served. + builds: BuildCount, + /// Leaf-hash passes this codeword actually paid for. Diverges from + /// [`builds`](Self::tree_builds) exactly when a retained layer was served. + leaf_passes: BuildCount, + /// ★ What is kept of the tree past the call that built it — the whole + /// tree or its leaf layer — with the key it is valid under. `Arc` for the + /// same reason `room` is one: `DeviceCodeword` is `Clone` and the folds + /// share the original's accounting, so a clone must share what is kept + /// rather than silently rebuild beside it. + leaves: Arc>>, + /// The room the chain promised itself: this codeword and the folds that + /// halve it, shared with those folds because they live inside it. + /// + /// ⛔ Whatever is kept of a tree is IN this number, and nothing of a tree + /// outside it: a capture grows the room by exactly what it keeps (the whole + /// tree, or its leaf layer), and the bytes leave it when that drops or is + /// evicted. A tree an opening builds and does not keep lives only for that + /// call — see [`with_tree`](Self::with_tree). + room: Arc, + /// Whether this codeword has been registered with the eviction registry. + /// Set once, on the first capture, so a rebuild-after-eviction re-capture + /// does not push a duplicate handle. `Arc` so all clones share the one flag, + /// as they share `leaves` and `room`. + registered: Arc, +} + +impl DeviceCodeword { + pub fn elements(&self) -> usize { + self.elements + } + + pub fn is_base(&self) -> bool { + self.base + } + + /// Where this codeword's tree puts its leaves. + pub fn placement(&self) -> LeafPlacement { + self.placement + } + + /// Every limb, read back — one per value for a base codeword, three for an + /// extension one. For the parity tests; a chain only ever gathers cosets. + pub fn to_host(&self) -> Result> { + let values = crate::readback::clone_dtoh(&self.stream, self.buffer.as_ref())?; + self.stream.synchronize()?; + Ok(values) + } + + /// Waits for the work queued on this codeword's stream — an encode's + /// transform — to finish, so a caller's stopwatch reads it where it ran. + pub fn synchronize(&self) -> Result<()> { + self.stream.synchronize()?; + Ok(()) + } + + /// The first value, which is what the last fold leaves behind. + pub fn first(&self) -> Result<[u64; 3]> { + let limbs = if self.base { 1 } else { 3 }; + let head = crate::readback::clone_dtoh(&self.stream, &self.buffer.slice(0..limbs))?; + self.stream.synchronize()?; + Ok(if self.base { + [head[0], 0, 0] + } else { + [head[0], head[1], head[2]] + }) + } + + /// The Merkle tree over this codeword's fold blocks, built here. + /// + /// A leaf is the `2^log_folding` coset that folds onto one position, and + /// the layout is the host's: `2*num_leaves - 1` nodes of 32 bytes, root + /// first — or, under [`crate::DeviceHash::Poseidon1`], the host's arity-4 + /// layout ([`crate::tree_nodes`] nodes, the leaves last). + fn build_tree( + &self, + log_folding: usize, + hash: crate::DeviceHash, + ) -> Result<(CudaSlice, usize)> { + let num_leaves = self.elements >> log_folding; + assert!(num_leaves >= 2, "tree needs at least two leaves"); + let be = backend()?; + let total_nodes = crate::tree_nodes(hash, num_leaves); + // SAFETY: every byte is written before it is read — the leaves by the + // kernel below, the inner nodes by the level loop after it. + let mut nodes = + unsafe { crate::device::alloc_or_trim::(&self.stream, total_nodes * 32) }?; + let leaves_offset = crate::leaves_offset(hash, num_leaves) * 32; + // ★ THE ONE BRANCH THIS CHANGE ADDS. A matching layer means the leaf + // pass is a device-to-device copy instead of a hash of every element; + // the inner levels below are built either way, so `build_tree` still + // returns the same tree it always did and `build_inner_tree_levels` is + // untouched. + let served = + self.serve_retained_leaves(&mut nodes, leaves_offset, num_leaves, log_folding, hash)?; + if !served { + { + let mut leaves = nodes.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); + let num_leaves_u64 = num_leaves as u64; + let block = 1u64 << log_folding; + // ★ The hash is chosen HERE, not by the host backend that will + // label the result. `hash` is the key the caller's `WhirHash` + // supplied, so a tree labelled RPX was hashed by RPX's kernels or + // was not built here at all. + match self.placement { + LeafPlacement::Natural => { + let (kernel, cfg) = coset_leaf_kernel(be, hash, self.base, num_leaves_u64)?; + unsafe { + self.stream + .launch_builder(kernel) + .arg(self.buffer.as_ref()) + .arg(&num_leaves_u64) + .arg(&block) + .arg(&mut leaves) + .launch(cfg)?; + } + } + // Reversed leaves exist for Poseidon1's base trees alone + // (the round-0 trees of the production hash). + LeafPlacement::Reversed => { + if hash != crate::DeviceHash::Poseidon1 || !self.base { + return Err(crate::invalid_value()); + } + let k = crate::p1_stark::kernels()?; + let leaf_bits = num_leaves.trailing_zeros(); + unsafe { + self.stream + .launch_builder(&k.leaves_base_coset_rev) + .arg(self.buffer.as_ref()) + .arg(&num_leaves_u64) + .arg(&block) + .arg(&leaf_bits) + .arg(&mut leaves) + .launch(crate::p1_stark::launch_cfg(num_leaves_u64))?; + } + } + } + } + // The borrow of `nodes` ends at the brace above, which is what lets + // the capture below read the region it just wrote. + // The pass was PAID here, so it is counted here — and the layer is + // offered for retention while it is in hand. Under `whole_trees` the + // whole tree is offered instead, once it is built (`with_tree`). + LEAF_HASH_CALLS.fetch_add(1, Ordering::Relaxed); + self.leaf_passes.fetch_add(1, Ordering::Relaxed); + if !whole_trees() { + self.capture_leaves(&nodes, leaves_offset, num_leaves, log_folding, hash); + } + } else { + LEAF_PASSES_SAVED.fetch_add(1, Ordering::Relaxed); + } + build_inner_tree_levels(self.stream.as_ref(), be, &mut nodes, num_leaves, hash)?; + TREE_BUILDS.fetch_add(1, Ordering::Relaxed); + self.builds.fetch_add(1, Ordering::Relaxed); + Ok((nodes, num_leaves)) + } + + /// Copy a retained layer into the node buffer's leaf region, if one matches. + /// + /// ⛔ THE MATCH IS EXACT ON BOTH KEY PARTS AND ON THE SHAPE. A layer built + /// at another fold width describes a different tree, and one built under + /// another hash describes a different tree again; either served here would + /// produce authentication paths that verify against themselves and against + /// nothing else. + fn serve_retained_leaves( + &self, + nodes: &mut CudaSlice, + leaves_offset: usize, + num_leaves: usize, + log_folding: usize, + hash: crate::DeviceHash, + ) -> Result { + let held = match self.leaves.lock() { + Ok(held) => held, + // A poisoned lock is not a reason to serve a layer nobody can + // vouch for: rebuild instead. + Err(_) => return Ok(false), + }; + let Some(kept) = held.as_ref() else { + return Ok(false); + }; + if !leaf_key_matches( + (kept.log_folding, kept.hash, kept.num_leaves), + (log_folding, hash, num_leaves), + ) { + return Ok(false); + } + let mut region = nodes.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); + if kept.whole { + // A whole tree's leaves sit at the same offset in its own buffer. + let leaves = kept + .nodes + .buffer() + .slice(leaves_offset..leaves_offset + num_leaves * 32); + self.stream.memcpy_dtod(&leaves, &mut region)?; + } else { + self.stream.memcpy_dtod(kept.nodes.buffer(), &mut region)?; + } + Ok(true) + } + + /// The whole tree kept under this key, if one is: a handle an opening reads + /// after the slot's lock is let go, with its leaf count. Taken under that + /// lock, which is what lets the evictor tell a tree being read from one + /// that is not ([`RetainedLeaves::read_by_an_opening`]); the tree's promise + /// stays until the last handle goes ([`KeptNodes`]). + fn retained_tree( + &self, + log_folding: usize, + hash: crate::DeviceHash, + ) -> Option<(Arc, usize)> { + let num_leaves = self.elements >> log_folding; + let held = self.leaves.lock().ok()?; + let kept = held.as_ref()?; + (kept.whole + && leaf_key_matches( + (kept.log_folding, kept.hash, kept.num_leaves), + (log_folding, hash, num_leaves), + )) + .then(|| (kept.nodes.clone(), kept.num_leaves)) + } + + /// Offer the tree just built and used for retention WHOLE, and take the + /// answer: `None` when it is kept, the buffer back when the budget will not + /// take it — the caller then keeps the leaf layer, as without the switch. + /// + /// The whole tree replaces a leaf layer held under the same key: the tree + /// is promised first, then the layer's bytes go back when it drops, so the + /// ledger never counts less than it holds. + fn capture_tree( + &self, + nodes: CudaSlice, + num_leaves: usize, + log_folding: usize, + hash: crate::DeviceHash, + ) -> Option> { + if !retention_enabled() { + return Some(nodes); + } + ensure_evictor_installed(); + let Ok(mut held) = self.leaves.lock() else { + return Some(nodes); + }; + if held.as_ref().is_some_and(|kept| kept.whole) { + return Some(nodes); + } + // `tree_bytes` for a binary tree; the arity-4 layout's own count + // under Poseidon1. + let bytes = (crate::tree_nodes(hash, num_leaves) * 32) as u64; + RETAIN_BYTES_ASKED.fetch_add(bytes, Ordering::Relaxed); + let kept = match KeptNodes::promise(nodes, bytes, &self.room) { + Ok(kept) => kept, + Err(nodes) => { + Self::note_refusal(); + return Some(nodes); + } + }; + RETAIN_ADMITTED.fetch_add(1, Ordering::Relaxed); + RETAIN_BYTES_ADMITTED.fetch_add(bytes, Ordering::Relaxed); + let replaced = held.replace(RetainedLeaves { + nodes: Arc::new(kept), + log_folding, + hash, + num_leaves, + whole: true, + }); + // As in `capture_leaves`: the layer lock goes before the registry is + // touched. A leaf layer the tree supersedes gives its bytes back as it + // drops, after the tree was promised. + drop(held); + drop(replaced); + if !self.registered.swap(true, Ordering::Relaxed) { + register_retained(&self.leaves); + } + None + } + + /// Offer the layer just hashed for retention, and take the answer. + /// + /// ⛔ ALLOCATE, THEN PROMISE, AND GIVE THE PROMISE BACK IF THE ALLOCATION + /// FAILED — in that order. Promising first and then failing to allocate + /// would leave the budget permanently short by bytes nothing holds, and + /// every later commitment would be refused because of it. `grow` returns + /// false and changes nothing when the budget will not take it, and + /// `shrink` gives back what an allocation could not use, so neither + /// direction can leak. + /// + /// ⚠ EVERY FAILURE PATH IS "NO RETENTION", NEVER AN ERROR. This is a cache: + /// the tree it would have saved is built anyway, the commit cannot fail + /// because of it, and `commit_stacked`'s device attempt cannot start + /// returning `None` — which is what would send a commitment to the host and + /// cost the 1.5 GiB per fallen-back chain that killed H4. + fn capture_leaves( + &self, + nodes: &CudaSlice, + leaves_offset: usize, + num_leaves: usize, + log_folding: usize, + hash: crate::DeviceHash, + ) { + // ⛔ LFM_WHIR_RETENTION=0 disables the retention entirely (the ABBA's + // control arm, off the SAME binary): capture is a no-op, nothing is + // retained, and the print reads admitted 0. Default ON. + if !retention_enabled() { + return; + } + // Install the evictor before any layer exists, so a later `reserve` that + // needs the budget can reclaim one. Idempotent (a `Once`). + ensure_evictor_installed(); + let Ok(mut held) = self.leaves.lock() else { + return; + }; + if held.is_some() { + return; + } + let bytes = (num_leaves as u64) * 32; + RETAIN_BYTES_ASKED.fetch_add(bytes, Ordering::Relaxed); + // SAFETY: every byte is written by the copy below before anything reads + // it, and the slice is dropped on every path that does not copy. + let Ok(copy) = (unsafe { alloc_or_trim::(&self.stream, num_leaves * 32) }) else { + Self::note_refusal(); + return; + }; + let Ok(mut kept) = KeptNodes::promise(copy, bytes, &self.room) else { + Self::note_refusal(); + return; + }; + let region = nodes.slice(leaves_offset..leaves_offset + num_leaves * 32); + if self.stream.memcpy_dtod(®ion, kept.buffer_mut()).is_err() { + // Dropping it gives the promise back. + drop(kept); + Self::note_refusal(); + return; + } + RETAIN_ADMITTED.fetch_add(1, Ordering::Relaxed); + RETAIN_BYTES_ADMITTED.fetch_add(bytes, Ordering::Relaxed); + *held = Some(RetainedLeaves { + nodes: Arc::new(kept), + log_folding, + hash, + num_leaves, + whole: false, + }); + // Release the layer lock BEFORE touching the registry, so the only lock + // nesting anywhere is the evictor's registry→layer — acyclic. Register + // ONCE: the handle is a Weak to this mutex, survives an eviction, and a + // re-capture refills the same slot, so a second push would only duplicate. + drop(held); + if !self.registered.swap(true, Ordering::Relaxed) { + register_retained(&self.leaves); + } + } + + /// One refusal, with the headroom the budget had the FIRST time it happened + /// — the number that says whether the scheme was short by a little or by a + /// lot, and which a later refusal would overwrite with a smaller one. + fn note_refusal() { + RETAIN_REFUSED.fetch_add(1, Ordering::Relaxed); + // `max(1)` so "no headroom at all" is still distinguishable from "never + // refused", which is what a zero in this slot means. + let headroom = backend() + .map(|be| { + be.vram_budget_bytes() + .saturating_sub(be.reserved_bytes()) + .max(1) + }) + .unwrap_or(1); + let _ = RETAIN_FIRST_REFUSAL_HEADROOM.compare_exchange( + 0, + headroom, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + + /// Run `f` against this codeword's tree: a kept whole tree when one + /// matches, else one built here — kept afterwards, whole or as its leaf + /// layer, inside the codeword's promise. + /// + /// # ⛔ The invariant: nothing of a tree is held outside a promise + /// + /// A commitment that is opened needs its tree twice — for the root and for + /// the paths — and the window between is forced by the protocol, not by + /// this file. `StackedCommitment::commit` builds EVERY chain's commitment + /// before it returns, because all the roots go into the transcript before + /// any query index is drawn; the openings come afterwards, one chain at a + /// time. So whatever is kept of the last chain's tree lives from its commit + /// to its opening, and all N exist before the first opening. + /// + /// H4 kept whole trees and lost, ~+15 s in both hashes: its node arrays sat + /// OUTSIDE any promise, so ten chains at half a gigabyte put the card at + /// 96%, device allocations failed, commits silently fell back to the host, + /// and the host grew ~1.5 GiB per fallen-back chain. The objection was + /// never the bytes but where they sat. What is kept now is promised and + /// evictable: + /// + /// - **The capture asks first.** + /// [`DeviceReservation::grow`](crate::device::DeviceReservation::grow) + /// declines without changing anything when the budget will not take the + /// bytes, and a refusal costs exactly the pass the retention would have + /// saved. The cliff is unreachable rather than unmeasured. + /// - **The evictor gives it back** to any request it can cover + /// ([`evict_retained_layers`]), so a request gets the budget it would get + /// with nothing kept, and fallbacks cannot rise. A miss nothing kept can + /// cover keeps it ([`keep_futile`]): that request fails either way. + /// - **The leaf layer, then the whole tree.** The layer came first: half the + /// bytes and most of the saving, since a round-0 leaf absorbs a whole + /// `2^k` coset against two digests for an inner node. The whole tree + /// ([`whole_trees`], the default since job 248: −1.00 s) keeps the inner + /// levels too, and an opening served one builds nothing. Evicted, its + /// worst case is the leaf pass again. + /// + /// The counters keep both kinds of work visible — + /// [`tree_builds`](Self::tree_builds), [`leaf_hash_calls`] and + /// [`retention_whole`] — and the group-scale test in + /// `tests/whir_tree_cache.rs` fails if anything of a tree is ever held + /// outside its promise. + fn with_tree( + &self, + log_folding: usize, + hash: crate::DeviceHash, + f: impl FnOnce(&CudaSlice, usize) -> Result, + ) -> Result { + if let Some((tree, num_leaves)) = self.retained_tree(log_folding, hash) { + WHOLE_TREE_SERVES.fetch_add(1, Ordering::Relaxed); + return f(tree.buffer(), num_leaves); + } + let (nodes, num_leaves) = self.build_tree(log_folding, hash)?; + let out = f(&nodes, num_leaves)?; + if whole_trees() + && let Some(nodes) = self.capture_tree(nodes, num_leaves, log_folding, hash) + { + // The budget would not take the whole tree: keep its leaf layer, as + // without the switch (a no-op when a layer is already held). + self.capture_leaves( + &nodes, + crate::leaves_offset(hash, num_leaves) * 32, + num_leaves, + log_folding, + hash, + ); + } + Ok(out) + } + + /// Bytes this codeword's chain has promised the device budget. For tests + /// and diagnostics. + pub fn reserved_bytes(&self) -> u64 { + self.room.bytes() + } + + /// ★ How many trees THIS codeword has built. + /// + /// One after a commit, and one more for each round that opens it — none + /// for an opening served a kept whole tree ([`whole_trees`]). Unlike the + /// process-wide counter this number is unaffected by whatever else shares + /// the test binary, so an assertion on it is about this codeword. + pub fn tree_builds(&self) -> u64 { + self.builds.load(Ordering::Relaxed) + } + + /// ★ Leaf-hash passes over THIS codeword — the number the retention moves. + /// + /// Equal to [`tree_builds`](Self::tree_builds) when nothing is retained, and + /// 1 however many times the codeword is opened when a leaf layer or a whole + /// tree is kept. The two together are what make the retention assertable + /// in BOTH directions: a cache that stopped working reads them equal, and + /// an opening served a kept whole tree leaves both where they were. + pub fn leaf_passes(&self) -> u64 { + self.leaf_passes.load(Ordering::Relaxed) + } + + /// Bytes this codeword is holding retained — its leaf layer, or its whole + /// tree under [`whole_trees`] — or zero. + pub fn retained_leaf_bytes(&self) -> u64 { + self.leaves + .lock() + .ok() + .and_then(|h| h.as_ref().map(RetainedLeaves::bytes)) + .unwrap_or(0) + } + + /// An opening's handle on the whole tree kept under this key, without the + /// opening: what a served opening holds while it reads the tree, for a test + /// that must keep one across an eviction. Not for production callers. + #[doc(hidden)] + pub fn hold_kept_tree( + &self, + log_folding: usize, + hash: crate::DeviceHash, + ) -> Option { + self.retained_tree(log_folding, hash) + .map(|(tree, _)| KeptTreeHold { _tree: tree }) + } + + /// The root of that tree, which is the commitment. + /// + /// ★ The tree is kept past this call, inside the codeword's promise — whole + /// by default, or its leaf layer — because the other thing anyone wants + /// from it is a path per query, and building it again then would cost a + /// second leaf-hash pass over the whole codeword ([`with_tree`](Self::with_tree)). + pub fn commit(&self, log_folding: usize, hash: crate::DeviceHash) -> Result<[u8; 32]> { + self.with_tree(log_folding, hash, |nodes, _| { + let head = crate::readback::clone_dtoh(&self.stream, &nodes.slice(0..32))?; + self.stream.synchronize()?; + let mut root = [0u8; 32]; + root.copy_from_slice(&head); + Ok(root) + }) + } + + /// The whole tree in the host node layout — what a caller that walks it + /// here needs, and what the parity test compares against. + pub fn nodes_to_host(&self, log_folding: usize, hash: crate::DeviceHash) -> Result> { + self.with_tree(log_folding, hash, |nodes, _| { + let out = crate::readback::clone_dtoh(&self.stream, nodes)?; + self.stream.synchronize()?; + Ok(out) + }) + } + + /// The first `nodes` nodes of the tree in the host layout (root first): + /// its top levels, which is what a prover that lets the codeword go keeps + /// to answer the paths later. `nodes` must be `2^(t+1) − 1` for some level + /// `t` of the tree — under Poseidon1, the arity-4 layout's top `t` levels + /// ([`crate::p1_stark::top_levels_nodes`]); anything else is refused. + pub fn top_nodes_to_host( + &self, + log_folding: usize, + hash: crate::DeviceHash, + nodes: usize, + ) -> Result> { + self.with_tree(log_folding, hash, |tree, num_leaves| { + if hash.arity() == crate::p1_stark::ARITY { + let levels = crate::p1_stark::level_sizes(num_leaves).len(); + ensure!( + (1..=levels).any(|t| crate::p1_stark::top_levels_nodes(num_leaves, t) == nodes), + "not whole top levels of an arity-4 tree" + ); + } else { + assert!( + (nodes + 1).is_power_of_two() && nodes < 2 * num_leaves, + "{nodes} nodes are not whole top levels of a tree of {num_leaves} leaves" + ); + } + let out = crate::readback::clone_dtoh(&self.stream, &tree.slice(0..nodes * 32))?; + self.stream.synchronize()?; + Ok(out) + }) + } + + /// The authentication paths of `positions`, against the same tree. + /// + /// ★ Read from the kept whole tree when one matches; otherwise the tree is + /// built here, its leaves served from a kept layer when one matches. The + /// tree is never brought home — a pageable copy of half a gigabyte is the + /// slowest thing in the commit, and what the host needs of a tree is a + /// kilobyte per query. + pub fn paths( + &self, + log_folding: usize, + positions: &[u32], + hash: crate::DeviceHash, + ) -> Result> { + self.with_tree(log_folding, hash, |nodes, num_leaves| { + gather_paths(hash, nodes, num_leaves, positions, &self.stream) + }) + } + + /// [`paths`](Self::paths) and the tree's Merkle cap at `cap_height`, from + /// ONE rebuild: the cap is the heap slice `[2^c − 1, 2^{c+1} − 1)` of the + /// same node buffer the paths are gathered from (root at node 0, the host + /// `MerkleTree` layout), so the two cannot come from different trees. + /// + /// Returns `(paths, cap)`: the paths exactly as [`paths`](Self::paths) + /// returns them (full depth — the caller cuts them to the cap), and + /// `2^cap_height` 32-byte nodes, left to right. `cap_height = 0` gives the + /// root. No kernel: the cap is a device-to-host copy of `2^c` nodes. + pub fn paths_and_cap( + &self, + log_folding: usize, + positions: &[u32], + cap_height: usize, + hash: crate::DeviceHash, + ) -> Result<(Vec, Vec)> { + self.with_tree(log_folding, hash, |nodes, num_leaves| { + let (start, end) = if hash.arity() == crate::p1_stark::ARITY { + // The real nodes of the 4-ary level `cap_height` below the root. + let sizes = crate::p1_stark::level_sizes(num_leaves); + ensure!(cap_height < sizes.len(), "a cap taller than the tree"); + let level = sizes.len() - 1 - cap_height; + let at = crate::p1_stark::level_offsets(&sizes)[level]; + (at * 32, (at + sizes[level]) * 32) + } else { + assert!( + num_leaves.is_power_of_two() + && cap_height <= num_leaves.trailing_zeros() as usize, + "a cap of height {cap_height} does not fit a tree of {num_leaves} leaves" + ); + ( + ((1usize << cap_height) - 1) * 32, + ((2usize << cap_height) - 1) * 32, + ) + }; + let paths = gather_paths(hash, nodes, num_leaves, positions, &self.stream)?; + let cap = crate::readback::clone_dtoh(&self.stream, &nodes.slice(start..end))?; + self.stream.synchronize()?; + Ok((paths, cap)) + }) + } + + /// The fold blocks `indices` open — `block` values at stride `num_leaves` + /// from each — gathered where they lie, one launch and one copy back. + /// + /// Query `q`'s block is `out[q*block*limbs ..]`, with one limb per value + /// for a base codeword and three for an extension one. + pub fn cosets(&self, indices: &[u64], num_leaves: usize, block: usize) -> Result> { + assert!(!indices.is_empty(), "a round opens at least one block"); + let limbs = if self.base { 1usize } else { 3 }; + let be = backend()?; + let index_dev = self.stream.clone_htod(indices)?; + let total = indices.len() * block; + // SAFETY: the kernel writes every value it is sized for. + let mut out = unsafe { alloc_or_trim::(&self.stream, total * limbs) }?; + let queries = indices.len() as u64; + let num_leaves_u64 = num_leaves as u64; + let block_u64 = block as u64; + let limbs_u64 = limbs as u64; + unsafe { + self.stream + .launch_builder(&be.gather_cosets) + .arg(self.buffer.as_ref()) + .arg(&index_dev) + .arg(&queries) + .arg(&num_leaves_u64) + .arg(&block_u64) + .arg(&limbs_u64) + .arg(&mut out) + .launch(LaunchConfig::for_num_elems(total as u32))?; + } + let values = crate::readback::clone_dtoh(&self.stream, &out)?; + self.stream.synchronize()?; + Ok(values) + } + + /// The subtrees of `blocks` — each block the `2^dropped` consecutive leaves + /// `[b·2^dropped, (b+1)·2^dropped)` of this codeword's tree at + /// `log_folding` — hashed and built here, from the codeword where it lies: + /// what a retired tree's queried blocks are re-hashed from, rather than + /// bringing their cosets home. + /// + /// Returns the nodes of ONE tree over the gathered leaves, in the host + /// layout (`2M − 1` nodes of 32 bytes, root first, `M` = the leaves rounded + /// up to a power of two, the last leaf repeated as padding). The blocks are + /// `2^dropped`-aligned in it, so the nodes `dropped` levels above the leaves + /// are the blocks' subtree roots, in `blocks`' order; the levels above them + /// mix blocks and mean nothing. Leaves and levels are hashed by the same + /// kernels the commit used, so each subtree is the committed one, byte for + /// byte — which the caller checks against the node it kept. + pub fn block_subtrees( + &self, + log_folding: usize, + blocks: &[u64], + dropped: usize, + hash: crate::DeviceHash, + ) -> Result> { + self.block_subtrees_placed(log_folding, blocks, dropped, hash, LeafPlacement::Natural) + } + + /// [`Self::block_subtrees`] of a tree whose leaves sit in `placement`: a + /// block is still `2^dropped` consecutive POSITIONS, and the leaf at each + /// is the fold block the placement puts there. + pub fn block_subtrees_placed( + &self, + log_folding: usize, + blocks: &[u64], + dropped: usize, + hash: crate::DeviceHash, + placement: LeafPlacement, + ) -> Result> { + assert!(!blocks.is_empty(), "a round opens at least one block"); + let num_leaves = self.elements >> log_folding; + // `dropped` counts the tree's own levels: a block is `2^dropped` + // leaves, or `4^dropped` under Poseidon1 (whose nodes, `dropped` + // 4-ary levels up, are then the blocks' roots in the arity-4 layout). + let span = if hash.arity() == crate::p1_stark::ARITY { + 1u64 << (2 * dropped) + } else { + 1u64 << dropped + }; + let real = blocks.len() * span as usize; + let leaves = real.next_power_of_two().max(2); + let depth = num_leaves.trailing_zeros(); + let mut indices: Vec = blocks + .iter() + .flat_map(|&b| (b * span)..((b + 1) * span)) + .map(|pos| placement.leaf_at(pos, depth)) + .collect(); + assert!( + ((blocks[blocks.len() - 1] + 1) * span) as usize <= num_leaves, + "block past the tree's {num_leaves} leaves" + ); + let last = *indices.last().expect("at least one leaf"); + indices.resize(leaves, last); + let limbs = if self.base { 1u64 } else { 3 }; + let block = 1u64 << log_folding; + let be = backend()?; + let index_dev = self.stream.clone_htod(&indices)?; + let values = leaves * block as usize * limbs as usize; + // SAFETY: the gather writes every value it is sized for. + let mut gathered = unsafe { alloc_or_trim::(&self.stream, values) }?; + let leaves_u64 = leaves as u64; + let num_leaves_u64 = num_leaves as u64; + unsafe { + self.stream + .launch_builder(&be.gather_cosets_strided) + .arg(self.buffer.as_ref()) + .arg(&index_dev) + .arg(&leaves_u64) + .arg(&num_leaves_u64) + .arg(&block) + .arg(&limbs) + .arg(&mut gathered) + .launch(LaunchConfig::for_num_elems((leaves as u64 * block) as u32))?; + } + let total_nodes = crate::tree_nodes(hash, leaves); + // SAFETY: every byte is written before it is read — the leaves by the + // kernel below, the inner nodes by the level loop after it. + let mut nodes = unsafe { alloc_or_trim::(&self.stream, total_nodes * 32) }?; + let leaves_offset = crate::leaves_offset(hash, leaves) * 32; + { + let mut out = nodes.slice_mut(leaves_offset..leaves_offset + leaves * 32); + let (kernel, cfg) = coset_leaf_kernel(be, hash, self.base, leaves_u64)?; + unsafe { + self.stream + .launch_builder(kernel) + .arg(&gathered) + .arg(&leaves_u64) + .arg(&block) + .arg(&mut out) + .launch(cfg)?; + } + } + build_inner_tree_levels(self.stream.as_ref(), be, &mut nodes, leaves, hash)?; + let out = crate::readback::clone_dtoh(&self.stream, &nodes)?; + self.stream.synchronize()?; + Ok(out) + } +} + +/// The codeword and the Merkle nodes of one stacked polynomial. +/// +/// `evals` holds the multilinear's `2^m` hypercube values in canonical +/// Goldilocks form. The codeword is `2^(m + log_blowup)` values in domain +/// order — the same array the host prover folds — and the nodes are the tree in +/// the host layout (`2*num_leaves - 1` nodes of 32 bytes, root first), so they +/// plug straight into a `MerkleTree`. +/// +/// `log_folding` is the first fold's width: a leaf is the `2^log_folding` coset +/// that folds onto one position. +/// The same, over a stacked polynomial that is never assembled on the host. +/// +/// A stacked polynomial is its columns written at their offsets and zeros +/// everywhere else, so the parts go straight into the device buffer: what the +/// host would have built is a copy of them, and building it costs a pass over +/// every byte the commit is about to upload anyway. +/// +/// `parts` is `(column, offset in elements)`; `log_evals` is the stacked +/// polynomial's variable count. +pub fn commit_codeword_parts( + parts: &[(&[u64], usize)], + log_evals: usize, + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::DeviceHash, +) -> Result<(DeviceCodeword, [u8; 32])> { + commit_codeword_parts_placed( + parts, + log_evals, + log_blowup, + log_folding, + transient, + hash, + LeafPlacement::Natural, + ) +} + +/// [`commit_codeword_parts`] with the tree's leaves in `placement`. +pub fn commit_codeword_parts_placed( + parts: &[(&[u64], usize)], + log_evals: usize, + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::DeviceHash, + placement: LeafPlacement, +) -> Result<(DeviceCodeword, [u8; 32])> { + commit_from( + Source::Parts { parts, log_evals }, + log_blowup, + log_folding, + transient, + hash, + placement, + ) +} + +/// The same for parts the card already holds: `(column index, offset)` into the +/// epoch's columns. The scatter is then a copy at device bandwidth rather than +/// the trace crossing the bus again. +pub fn commit_codeword_resident( + store: &crate::columns::DeviceColumns, + parts: &[(usize, usize)], + log_evals: usize, + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::DeviceHash, +) -> Result<(DeviceCodeword, [u8; 32])> { + commit_codeword_resident_placed( + store, + parts, + log_evals, + log_blowup, + log_folding, + transient, + hash, + LeafPlacement::Natural, + ) +} + +/// [`commit_codeword_resident`] with the tree's leaves in `placement`. +#[allow(clippy::too_many_arguments)] +pub fn commit_codeword_resident_placed( + store: &crate::columns::DeviceColumns, + parts: &[(usize, usize)], + log_evals: usize, + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::DeviceHash, + placement: LeafPlacement, +) -> Result<(DeviceCodeword, [u8; 32])> { + commit_from( + Source::Resident { + store, + parts, + log_evals, + }, + log_blowup, + log_folding, + transient, + hash, + placement, + ) +} + +/// Where a commit's coefficients come from: one slab the host holds, or the +/// columns a stacked polynomial is made of. +enum Source<'a> { + Whole(&'a [u64]), + Parts { + parts: &'a [(&'a [u64], usize)], + log_evals: usize, + }, + Resident { + store: &'a crate::columns::DeviceColumns, + parts: &'a [(usize, usize)], + log_evals: usize, + }, + /// Slices of one device buffer: `(element offset in it, elements, offset + /// in the stack)` — a stack the card still holds after the borrow of its + /// store ended (`crate::whir_coset::StackSlices`). + Slices { + buffer: &'a CudaSlice, + parts: &'a [(usize, usize, usize)], + log_evals: usize, + }, +} + +impl Source<'_> { + fn log_evals(&self) -> u64 { + match self { + Self::Whole(evals) => evals.len().trailing_zeros() as u64, + Self::Parts { log_evals, .. } + | Self::Resident { log_evals, .. } + | Self::Slices { log_evals, .. } => *log_evals as u64, + } + } + + /// Fills `coeffs`, which is `2^log_evals` elements long. + fn write(&self, stream: &Arc, coeffs: &mut CudaSlice) -> Result<()> { + match self { + Self::Whole(evals) => stream.memcpy_htod(*evals, coeffs), + Self::Parts { parts, .. } => { + // Everything the parts do not cover is the stacking's padding, + // and that is zero by definition. + stream.memset_zeros(coeffs)?; + for (column, offset) in *parts { + let mut at = coeffs.slice_mut(*offset..*offset + column.len()); + stream.memcpy_htod(*column, &mut at)?; + } + Ok(()) + } + Self::Resident { store, parts, .. } => { + stream.memset_zeros(coeffs)?; + for (column, offset) in *parts { + store.copy_into(*column, coeffs, *offset, stream)?; + } + Ok(()) + } + Self::Slices { buffer, parts, .. } => copy_slices(stream, buffer, parts, coeffs), + } + } +} + +/// Writes a stack of `coeffs.len()` elements from `parts` of `buffer` — +/// `(element offset in buffer, elements, offset in the stack)` — and zeros +/// between, on `stream`. +pub(crate) fn copy_slices( + stream: &Arc, + buffer: &CudaSlice, + parts: &[(usize, usize, usize)], + coeffs: &mut CudaSlice, +) -> Result<()> { + ensure!( + parts + .iter() + .all(|&(at, len, offset)| at + len <= buffer.len() && offset + len <= coeffs.len()) + ); + stream.memset_zeros(coeffs)?; + let (src, _src_guard) = buffer.device_ptr(stream); + let (dst, _dst_guard) = cudarc::driver::DevicePtrMut::device_ptr_mut(coeffs, stream); + for &(at, len, offset) in parts { + // SAFETY: both ranges are inside allocations held here, checked above. + unsafe { + cudarc::driver::sys::cuMemcpyDtoDAsync_v2( + dst + (offset * 8) as u64, + src + (at * 8) as u64, + len * 8, + stream.cu_stream(), + ) + .result()?; + } + } + Ok(()) +} + +/// The codeword of a stack held as slices of one device buffer +/// ([`copy_slices`]), with no tree: [`encode_codeword_resident`]'s values. +pub fn encode_codeword_slices( + buffer: &CudaSlice, + parts: &[(usize, usize, usize)], + log_evals: usize, + log_blowup: usize, + transient: bool, +) -> Result { + encode_from( + Source::Slices { + buffer, + parts, + log_evals, + }, + log_blowup, + transient, + ) +} + +pub fn commit_codeword( + evals: &[u64], + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::DeviceHash, +) -> Result<(DeviceCodeword, [u8; 32])> { + commit_from( + Source::Whole(evals), + log_blowup, + log_folding, + transient, + hash, + LeafPlacement::Natural, + ) +} + +fn commit_from( + source: Source<'_>, + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::DeviceHash, + placement: LeafPlacement, +) -> Result<(DeviceCodeword, [u8; 32])> { + let log_n = source.log_evals() + log_blowup as u64; + assert!( + log_folding as u64 <= log_n, + "a leaf cannot exceed the domain" + ); + assert!( + (1usize << log_n) >> log_folding >= 2, + "tree needs at least two leaves" + ); + let mut codeword = encode_from(source, log_blowup, transient)?; + codeword.placement = placement; + let root = codeword.commit(log_folding, hash)?; + Ok((codeword, root)) +} + +/// The codeword of a stacked polynomial whose columns the card already holds, +/// with NO tree: what a prover that committed it earlier, kept the root and +/// let the codeword go, rebuilds before its opening. The transform is +/// [`commit_codeword_resident`]'s, so the values are the committed ones; the +/// caller answers for the tree (the block prover keeps its top levels on the +/// host and re-hashes the bottom ones a query needs). +pub fn encode_codeword_resident( + store: &crate::columns::DeviceColumns, + parts: &[(usize, usize)], + log_evals: usize, + log_blowup: usize, + transient: bool, +) -> Result { + encode_from( + Source::Resident { + store, + parts, + log_evals, + }, + log_blowup, + transient, + ) +} + +/// [`encode_codeword_resident`] over columns the host holds. +pub fn encode_codeword_parts( + parts: &[(&[u64], usize)], + log_evals: usize, + log_blowup: usize, + transient: bool, +) -> Result { + encode_from(Source::Parts { parts, log_evals }, log_blowup, transient) +} + +/// The Reed–Solomon codeword of `source`, on the card, reserved like a +/// commit's: the codeword, plus its working set when `transient`. +fn encode_from(source: Source<'_>, log_blowup: usize, transient: bool) -> Result { + let log_evals = source.log_evals(); + let log_n = log_evals + log_blowup as u64; + let n = 1usize << log_n; + assert!( + n <= u32::MAX as usize, + "codeword length {n} exceeds u32 range — kernel grid would silently truncate", + ); + + let be = backend()?; + // The codeword itself stays until this commitment's opening. On top of it + // there is a working set — the tree this commit builds, and later the + // folds and the trees the chain's rounds build — which is another + // codeword's worth and is **transient**: one commit or one opening uses it + // at a time. A caller committing a group of polynomials promises that once + // for all of them and passes `transient: false`; one committing alone + // promises it here. + let promise = if transient { 2 } else { 1 }; + let Some(room) = be.reserve(n as u64 * 8 * promise) else { + return Err(cudarc::driver::DriverError( + cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY, + )); + }; + let stream = be.next_stream(); + + // The coefficients get a buffer of their own: the Möbius transform runs + // over them, and the spread below reads them while it writes the codeword. + // SAFETY: every element is written by the copy below, or by the wide + // Möbius pass that gathers the stack in its place. + let mut coeffs = unsafe { alloc_or_trim::(&stream, 1usize << log_evals) }?; + let fused = mobius_fused_levels(log_evals, log_blowup); + let x = if mobius_wide_gathered(&stream, be, &source, &mut coeffs, log_evals, fused)? { + spread_encode( + &stream, + be, + coeffs, + log_evals, + log_blowup, + fused.unwrap_or(0), + )? + } else { + source.write(&stream, &mut coeffs)?; + encode_values(&stream, be, coeffs, log_evals, log_blowup)? + }; + + let codeword = DeviceCodeword { + buffer: Arc::new(x), + stream, + elements: n, + base: true, + builds: BuildCount::default(), + leaf_passes: BuildCount::default(), + leaves: Arc::new(Mutex::new(None)), + room: Arc::new(room), + registered: Arc::new(AtomicBool::new(false)), + placement: LeafPlacement::Natural, + }; + Ok(codeword) +} + +/// The codeword of the multilinear whose `2^log_evals` hypercube values +/// `values` holds, base field: the Möbius transform in place, the lift's +/// spread, the NTT onto the `2^log_blowup`-times larger domain, in domain +/// order — `encode(lift_coefficients(·))` on the card. `values` is spent. +fn encode_values( + stream: &Arc, + be: &crate::device::Backend, + mut coeffs: CudaSlice, + log_evals: u64, + log_blowup: usize, +) -> Result> { + let len = 1usize << log_evals; + let fused = mobius_fused_levels(log_evals, log_blowup); + if let Some(from) = fused { + let (x, _guard) = cudarc::driver::DevicePtrMut::device_ptr_mut(&mut coeffs, stream); + mobius_wide_tiles(stream.as_ref(), be, x, len, log_evals, from, None)?; + } else { + mobius(stream.as_ref(), be, &mut coeffs, len, log_evals)?; + } + spread_encode( + stream, + be, + coeffs, + log_evals, + log_blowup, + fused.unwrap_or(0), + ) +} + +/// [`encode_values`] after its Möbius transform — all of it, or with +/// `mobius_levels` its levels from there up ([`mobius_fused_levels`]): the +/// lift's spread and the NTT onto the `2^log_blowup`-times larger domain, the +/// low Möbius levels then fused into the NTT's first pass. `coeffs` is spent. +fn spread_encode( + stream: &Arc, + be: &crate::device::Backend, + coeffs: CudaSlice, + log_evals: u64, + log_blowup: usize, + mobius_levels: u32, +) -> Result> { + let log_n = log_evals + log_blowup as u64; + let n = 1usize << log_n; + + // The lift's bit-reverse and the NTT's cancel around the zero padding — + // see `lift_spread`. What was two scattered passes over the codeword plus + // the memset that zeroed it is one pass that writes all of it. + // SAFETY: the spread writes every element, padding included (on the column + // engine its first pass does, storing all of its tile). + let mut x = unsafe { alloc_or_trim::(stream, n) }?; + let n_u64 = n as u64; + let log_blowup_u32 = log_blowup as u32; + if crate::lde_cm::engine_enabled() + && crate::lde_cm::spread_supports(log_n as u32, log_blowup_u32) + { + // The same spread and transform in the column engine's ceil(log_n / 8) + // passes, the spread fused into the first one, and no twiddle table + // (see `lde_cm`). `LAMBDA_VM_LDE_LEGACY=1` takes the branch below. + let src = coeffs.device_ptr(stream).0; + let dst = x.device_ptr(stream).0; + crate::lde_cm::spread_ntt_column( + stream, + be, + src, + dst, + log_n as u32, + log_blowup_u32, + mobius_levels, + )?; + if mobius_levels > 0 { + MOBIUS_FUSED_ENCODES.fetch_add(1, Ordering::Relaxed); + } + // Spent: the first pass has read them, and the free is stream-ordered. + drop(coeffs); + } else { + assert_eq!( + mobius_levels, 0, + "the legacy encoding cannot take fused Möbius levels" + ); + unsafe { + stream + .launch_builder(&be.lift_spread) + .arg(&coeffs) + .arg(&n_u64) + .arg(&log_blowup_u32) + .arg(&mut x) + .launch(LaunchConfig::for_num_elems(n as u32))?; + } + // Spent: the spread has read them, and the free is stream-ordered. + drop(coeffs); + let twiddles = be.fwd_twiddles_for(log_n)?; + crate::ntt::run_ntt_body(stream.as_ref(), &mut x, twiddles.as_ref(), n_u64, log_n)?; + } + Ok(x) +} + +/// ★ The successor [`fold_resident_fused`] would make of `codeword` over +/// `levels` levels, encoded from the message those levels' rounds bound +/// instead (D-SPEED lever 5a). +/// +/// Folding the committed codeword with `α` encodes the message with its first +/// variables fixed to `α`, on the folded domain (`multilinear`'s +/// `every_successor_is_the_bound_message_encoded_at_the_production_shape`), so +/// the `2^(n+blowup)`-element pass the fold reads becomes an encode of the +/// `2^(n−levels)` message: per limb, a gather out of the interleaved message, +/// [`encode_values`], and a scatter into the interleaved successor. The +/// encoding is base-field linear, so the limbs are independent. +/// +/// `message` is interleaved ext3 and its first `len` values are the bound +/// message, read on `stream` — the stream that bound it. The successor is +/// ordered on that stream too, and lives inside `codeword`'s room, as a fold +/// does. +/// +/// ★ EQUAL to the fold, not raw-identical: the same field values by another +/// sequence of operations, so raw limbs agree wherever both are canonical — +/// the parity the lean rounds have (`whir_lean_round`). Trees hash canonical +/// values (or absorb field elements), so roots and the transcript agree. +pub fn encode_successor( + codeword: &DeviceCodeword, + levels: usize, + stream: &Arc, + message: &CudaSlice, + len: usize, +) -> Result { + encode_successor_sized( + codeword.elements, + &codeword.room, + levels, + stream, + message, + len, + ) +} + +/// [`encode_successor`] of a codeword of `elements` values that need not +/// exist: the successor's size is `elements >> levels`, and it lives inside +/// `room` as a fold would. What a round-0 codeword evaluated coset by coset +/// (`crate::whir_coset`) is succeeded by. +pub fn encode_successor_sized( + elements: usize, + room: &Arc, + levels: usize, + stream: &Arc, + message: &CudaSlice, + len: usize, +) -> Result { + assert!(len.is_power_of_two(), "the bound message spans a cube"); + assert!(message.len() >= len * 3, "three u64 per ext3 value"); + assert!( + elements >> levels >= len && elements >> levels << levels == elements, + "{levels} folds of {elements} elements do not cover a message of {len}" + ); + let n_out = elements >> levels; + let log_evals = len.trailing_zeros() as u64; + let log_blowup = (n_out / len).trailing_zeros() as usize; + assert_eq!( + len << log_blowup, + n_out, + "the successor is the message's blowup" + ); + + let be = backend()?; + // SAFETY: the three scatters below write every limb of every element. + let mut out = unsafe { alloc_or_trim::(stream, n_out * 3) }?; + let len_arg = len as u64; + let n_out_arg = n_out as u64; + for limb in 0..3u32 { + // SAFETY: the gather writes every element. + let mut plane = unsafe { alloc_or_trim::(stream, len) }?; + unsafe { + stream + .launch_builder(&be.ext3_limb_gather) + .arg(message) + .arg(&len_arg) + .arg(&limb) + .arg(&mut plane) + .launch(LaunchConfig::for_num_elems(len as u32))?; + } + let encoded = encode_values(stream, be, plane, log_evals, log_blowup)?; + unsafe { + stream + .launch_builder(&be.ext3_limb_scatter) + .arg(&encoded) + .arg(&n_out_arg) + .arg(&limb) + .arg(&mut out) + .launch(LaunchConfig::for_num_elems(n_out as u32))?; + } + // Spent once scattered; the free is stream-ordered. + drop(encoded); + } + + Ok(DeviceCodeword { + buffer: Arc::new(out), + stream: stream.clone(), + elements: n_out, + base: false, + // Its own codeword, as a fold's is. + builds: BuildCount::default(), + leaf_passes: BuildCount::default(), + leaves: Arc::new(Mutex::new(None)), + room: room.clone(), + registered: Arc::new(AtomicBool::new(false)), + placement: LeafPlacement::Natural, + }) +} + +/// `Σ_p scales[p] · parts[p]`, the parts extension codewords of one length: +/// a grouped chain's round-0 successor from its polynomials' folds, each +/// weighted by its slot's `eq(ρ, p)` (three limbs a scale). Written on +/// `parts[0]`'s stream and inside its room, as a fold of it would be; every +/// part's queued work is waited for first, and the sums before this returns, +/// since each part is freed on its own stream. +/// +/// The same field values as the host's combination of the same folds; raw +/// limbs agree wherever both are canonical. +pub fn combine_codewords(parts: &[&DeviceCodeword], scales: &[[u64; 3]]) -> Result { + ensure!( + !parts.is_empty() && parts.len() == scales.len(), + "one scale a part" + ); + let first = parts[0]; + let n = first.elements; + ensure!( + parts.iter().all(|part| !part.base && part.elements == n), + "extension codewords of one length" + ); + let be = backend()?; + let stream = first.stream.clone(); + for part in parts { + part.stream.synchronize()?; + } + let mut out = crate::device::alloc_zeros_or_trim::(&stream, n * 3)?; + let count = n as u64; + for (part, scale) in parts.iter().zip(scales) { + let scale_dev = crate::device::htod_or_trim(&stream, scale)?; + unsafe { + stream + .launch_builder(&be.add_scaled_ext3) + .arg(&mut out) + .arg(part.buffer.as_ref()) + .arg(&count) + .arg(&scale_dev) + .launch(LaunchConfig::for_num_elems(n as u32))?; + } + } + stream.synchronize()?; + Ok(DeviceCodeword { + buffer: Arc::new(out), + stream, + elements: n, + base: false, + // Its own codeword, as a fold's is. + builds: BuildCount::default(), + leaf_passes: BuildCount::default(), + leaves: Arc::new(Mutex::new(None)), + room: first.room.clone(), + registered: Arc::new(AtomicBool::new(false)), + // A successor: its tree's leaves are in their natural order, whatever + // the round-0 trees' are. + placement: LeafPlacement::Natural, + }) +} + +/// The most [`encode_successor`] holds beside its inputs for a successor of +/// `2^log_out` elements from a message of `2^log_evals`: the interleaved +/// successor, one limb's plane and one limb's encoding. +pub const fn encode_successor_transient_bytes(log_out: usize, log_evals: usize) -> u64 { + (24u64 << log_out) + (8u64 << log_out) + (8u64 << log_evals) +} + +/// Whether a chain's first successor is encoded from its bound message +/// ([`encode_successor`]) when `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR` does not say +/// otherwise. Either way proves the same values. +/// +/// On: the fold reads the whole `2^(n+blowup)` base codeword, the encode a +/// `2^(n−k)` message (I-COPIES §12). `=0` is the way back. +pub const ENCODE_SUCCESSOR_DEFAULT: bool = true; + +/// `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR`'s value read: `0` folds, `1` encodes, +/// unset or anything else is [`ENCODE_SUCCESSOR_DEFAULT`]. +pub fn encode_successor_setting(value: Option<&str>) -> bool { + match value.map(str::trim) { + Some("0") => false, + Some("1") => true, + _ => ENCODE_SUCCESSOR_DEFAULT, + } +} + +#[cfg(test)] +mod encode_successor_tests { + use super::encode_successor_setting; + + /// The first successor is encoded unless the knob says `0`. + #[test] + fn the_first_successor_is_encoded_by_default_and_zero_folds() { + assert!(encode_successor_setting(None), "the default encodes"); + assert!(encode_successor_setting(Some("1"))); + assert!(!encode_successor_setting(Some("0")), "the way back"); + assert!(!encode_successor_setting(Some(" 0 "))); + assert!(encode_successor_setting(Some(""))); + assert!(encode_successor_setting(Some("off")), "only 0 opts out"); + } +} + +/// Levels a tile fuses at once: 32 rows of 32 columns is a full block, and the +/// shared tile it needs is a few kilobytes. +const MOBIUS_TILE_LEVELS: u32 = 5; +/// Columns a tile spans — one warp, matching the kernel. +const MOBIUS_TILE_COLS: u32 = 32; +/// Levels the contiguous kernel takes: a block of 256 holds both sides of +/// every pair the first eight levels make. +const MOBIUS_LOW_LEVELS: u32 = 8; + +/// The Mobius transform over `coeffs`, a window of levels at a time. +/// +/// Each level is one pass over the whole array, so run one per launch and the +/// transform is bound by how many times it reads the array rather than by the +/// subtractions. The windows below fuse the levels that share a tile, which is +/// the shape the NTT's levels are already fused in. +fn mobius( + stream: &CudaStream, + be: &crate::device::Backend, + coeffs: &mut CudaSlice, + len: usize, + log_evals: u64, +) -> Result<()> { + if mobius_wide_enabled() && log_evals >= MOBIUS_WIDE_LOW as u64 { + let (x, _guard) = cudarc::driver::DevicePtrMut::device_ptr_mut(coeffs, stream); + return mobius_wide(stream, be, x, len, log_evals, None); + } + let mut level: u64 = 0; + + // The contiguous window, when there is a whole block of elements to hold. + if log_evals >= MOBIUS_LOW_LEVELS as u64 && len >= 256 { + let k = MOBIUS_LOW_LEVELS; + unsafe { + stream + .launch_builder(&be.mobius_low_levels) + .arg(&mut *coeffs) + .arg(&k) + .launch(LaunchConfig { + grid_dim: ((len / 256) as u32, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + })?; + } + level = k as u64; + } + + while level < log_evals { + let low = 1u64 << level; + let k = MOBIUS_TILE_LEVELS.min((log_evals - level) as u32); + // A tile needs a warp of consecutive low indices to stay coalesced; a + // level below that is left to the one-level kernel. + if low < MOBIUS_TILE_COLS as u64 { + let half = (len / 2) as u64; + let stride = low; + unsafe { + stream + .launch_builder(&be.mobius_level) + .arg(&mut *coeffs) + .arg(&half) + .arg(&stride) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + level += 1; + continue; + } + let rows = 1u32 << k; + let pitch = MOBIUS_TILE_COLS + 1; + // Past 2^15 blocks in y the excess goes to grid.z (`ntt::split_grid_y`): + // the first tile needs `len >> 13`, 65,536 at 2^29 evaluations. + let (grid_y, grid_z) = crate::ntt::split_grid_y(len as u64 / (low << k)); + unsafe { + stream + .launch_builder(&be.mobius_tile) + .arg(&mut *coeffs) + .arg(&level) + .arg(&k) + .launch(LaunchConfig { + grid_dim: ((low / MOBIUS_TILE_COLS as u64) as u32, grid_y, grid_z), + block_dim: (MOBIUS_TILE_COLS, rows, 1), + shared_mem_bytes: rows * pitch * 8, + })?; + } + level += k as u64; + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// ★ The WHIR encoding's wide Möbius transform: three passes over the +// stack instead of five — the first 11 levels over contiguous tiles of 2^11, +// then strided tiles of up to 8 levels — and, for a stack the card already +// holds, the stack GATHERED from its parts by the first pass instead of copied +// into a zeroed buffer first. The kernels (`mobius_wide_low`, +// `mobius_wide_tile` in `kernels/ntt.cu`) give every element the operations +// the five-pass transform gives it, in the same level order, so the +// coefficients — and so the codeword and its tree — are the same words. +// --------------------------------------------------------------------------- + +/// The wide transform runs by default; `LAMBDA_VM_WHIR_MOBIUS_WIDE=0` (or +/// `off`) keeps the five-pass one. Read once per process. +pub const MOBIUS_WIDE_ENV: &str = "LAMBDA_VM_WHIR_MOBIUS_WIDE"; + +/// Levels the contiguous wide pass takes: 2^11 elements are 16 KiB of shared +/// memory, half of them updated a level, one per thread of 1,024. +const MOBIUS_WIDE_LOW: u32 = 11; +/// Levels a strided wide tile takes: 2^8 rows × 33 words of 8 bytes. +const MOBIUS_WIDE_TILE: u32 = 8; +/// The strided tile's shared memory at [`MOBIUS_WIDE_TILE`] levels. +const MOBIUS_WIDE_TILE_BYTES: u32 = (1 << MOBIUS_WIDE_TILE) * (MOBIUS_TILE_COLS + 1) * 8; + +thread_local! { + /// Per-thread override of [`MOBIUS_WIDE_ENV`], for parity tests that run + /// both transforms in one process. See [`with_mobius_wide`]. + static MOBIUS_WIDE_OVERRIDE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// `LAMBDA_VM_WHIR_MOBIUS_WIDE`'s value read: empty, `0` or `off` keep the +/// five-pass transform; anything else (unset included) is the wide one. +pub fn mobius_wide_setting(value: Option<&str>) -> bool { + !matches!(value.map(str::trim), Some("" | "0" | "off")) +} + +/// Whether the calling thread's WHIR encodings take the wide Möbius transform. +pub fn mobius_wide_enabled() -> bool { + if let Some(on) = MOBIUS_WIDE_OVERRIDE.with(std::cell::Cell::get) { + return on; + } + static ENV: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENV.get_or_init(|| { + let on = mobius_wide_setting(std::env::var(MOBIUS_WIDE_ENV).ok().as_deref()); + if on { + eprintln!( + "[gpu] WHIR Möbius: wide, three passes, the stack gathered (the default; {MOBIUS_WIDE_ENV}=0 keeps five)" + ); + } + on + }) +} + +/// Run `f` with the wide transform forced on or off for the calling thread. +#[doc(hidden)] +pub fn with_mobius_wide(on: bool, f: impl FnOnce() -> R) -> R { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + MOBIUS_WIDE_OVERRIDE.with(|c| c.set(self.0)); + } + } + let _restore = Restore(MOBIUS_WIDE_OVERRIDE.with(|c| c.replace(Some(on)))); + f() +} + +/// Encodings whose stack the wide pass gathered, process-wide: what a test +/// reads to know the gather, and not the copy, ran. +static MOBIUS_WIDE_GATHERS: AtomicU64 = AtomicU64::new(0); + +/// See [`MOBIUS_WIDE_GATHERS`]. +pub fn mobius_wide_gathers() -> u64 { + MOBIUS_WIDE_GATHERS.load(Ordering::Relaxed) +} + +/// The fused encoding (`LAMBDA_VM_WHIR_MOBIUS_FUSED`) folds the low Möbius levels into +/// the NTT's first pass: a first-pass block stages `2^(12 − log_blowup)` +/// consecutive coefficients ([`crate::lde_cm::fused_mobius_levels`]), exactly +/// the levels the wide transform's contiguous pass runs, so that pass goes. +/// The strided levels run first (the first one gathering the stack, as the +/// wide transform's contiguous pass did); the levels commute, so the field +/// values are today's — the order of the subtractions changes, so a word's +/// representation may not be. Where it does not apply (the legacy encoding, +/// a stack the first pass holds whole) the encoding is the wide transform's +/// or the five-pass one. On by default; `LAMBDA_VM_WHIR_MOBIUS_FUSED=0` (or +/// `off`) runs every Möbius level in the transform. Read once per process. +pub const MOBIUS_FUSED_ENV: &str = "LAMBDA_VM_WHIR_MOBIUS_FUSED"; + +thread_local! { + /// Per-thread override of [`MOBIUS_FUSED_ENV`]. See [`with_mobius_fused`]. + static MOBIUS_FUSED_OVERRIDE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// `LAMBDA_VM_WHIR_MOBIUS_FUSED`'s value read: empty, `0` or `off` do not +/// fuse; anything else (unset included) does. +pub fn mobius_fused_setting(value: Option<&str>) -> bool { + !matches!(value.map(str::trim), Some("" | "0" | "off")) +} + +/// Whether the calling thread's WHIR encodings fold the low Möbius levels into +/// the NTT. +pub fn mobius_fused_enabled() -> bool { + if let Some(on) = MOBIUS_FUSED_OVERRIDE.with(std::cell::Cell::get) { + return on; + } + static ENV: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENV.get_or_init(|| { + let on = mobius_fused_setting(std::env::var(MOBIUS_FUSED_ENV).ok().as_deref()); + if on { + eprintln!( + "[gpu] WHIR Möbius: fused, the low levels in the NTT's first pass (the default; {MOBIUS_FUSED_ENV}=0 runs them in the transform)" + ); + } + on + }) +} + +/// Run `f` with the fused encoding forced on or off for the calling thread. +#[doc(hidden)] +pub fn with_mobius_fused(on: bool, f: impl FnOnce() -> R) -> R { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + MOBIUS_FUSED_OVERRIDE.with(|c| c.set(self.0)); + } + } + let _restore = Restore(MOBIUS_FUSED_OVERRIDE.with(|c| c.replace(Some(on)))); + f() +} + +/// Encodings whose NTT ran the low Möbius levels, process-wide: what a test +/// reads to know the fused encoding ran. +static MOBIUS_FUSED_ENCODES: AtomicU64 = AtomicU64::new(0); + +/// See [`MOBIUS_FUSED_ENCODES`]. +pub fn mobius_fused_encodes() -> u64 { + MOBIUS_FUSED_ENCODES.load(Ordering::Relaxed) +} + +/// The Möbius level the fused encoding of a `2^log_evals` stack at spacing +/// `2^log_blowup` starts its strided passes at — the levels below it run in +/// the NTT's first pass — or `None` where it does not apply: the knob off, the +/// legacy encoding, or no strided level left. +fn mobius_fused_levels(log_evals: u64, log_blowup: usize) -> Option { + if !mobius_fused_enabled() || !crate::lde_cm::engine_enabled() { + return None; + } + let (log_n, lb) = ((log_evals + log_blowup as u64) as u32, log_blowup as u32); + if !crate::lde_cm::spread_supports(log_n, lb) { + return None; + } + let from = crate::lde_cm::fused_mobius_levels(log_n, lb); + ((from as u64) < log_evals && from >= MOBIUS_TILE_COLS.trailing_zeros()).then_some(from) +} + +/// The wide transform over the `len = 2^log_evals` words at `x` on `stream`; +/// with `parts` (a device table of `(offset, len, src)` triples and its +/// length), the first pass reads the stack from them instead of from `x`. +fn mobius_wide( + stream: &CudaStream, + be: &crate::device::Backend, + x: cudarc::driver::sys::CUdeviceptr, + len: usize, + log_evals: u64, + parts: Option<(cudarc::driver::sys::CUdeviceptr, u32)>, +) -> Result<()> { + debug_assert!(log_evals >= MOBIUS_WIDE_LOW as u64 && len == 1usize << log_evals); + let (table, n_parts) = parts.unwrap_or((0, 0)); + let first = MOBIUS_WIDE_LOW.min(log_evals as u32); + unsafe { + stream + .launch_builder(&be.mobius_wide_low) + .arg(&x) + .arg(&table) + .arg(&n_parts) + .arg(&first) + .launch(LaunchConfig { + grid_dim: ((len >> MOBIUS_WIDE_LOW) as u32, 1, 1), + block_dim: (1024, 1, 1), + shared_mem_bytes: 0, + })?; + } + mobius_wide_tiles(stream, be, x, len, log_evals, first, None) +} + +/// The wide transform's strided levels `[from, log_evals)`, up to +/// [`MOBIUS_WIDE_TILE`] a pass; with `parts` the first pass gathers the stack +/// from them instead of reading `x` (the fused encoding's start). +fn mobius_wide_tiles( + stream: &CudaStream, + be: &crate::device::Backend, + x: cudarc::driver::sys::CUdeviceptr, + len: usize, + log_evals: u64, + from: u32, + parts: Option<(cudarc::driver::sys::CUdeviceptr, u32)>, +) -> Result<()> { + debug_assert!(len == 1usize << log_evals && (1u32 << from) >= MOBIUS_TILE_COLS); + static TILE_ATTR: Once = Once::new(); + let mut attr = Ok(()); + TILE_ATTR.call_once(|| { + attr = be.ctx.bind_to_thread().and_then(|()| { + be.mobius_wide_tile.set_attribute( + cudarc::driver::sys::CUfunction_attribute::CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, + MOBIUS_WIDE_TILE_BYTES as i32, + ) + }); + }); + attr?; + let mut gather = parts; + let mut level = from as u64; + while level < log_evals { + let k = MOBIUS_WIDE_TILE.min((log_evals - level) as u32); + let low = 1u64 << level; + let (grid_y, grid_z) = crate::ntt::split_grid_y(len as u64 / (low << k)); + let (table, n_parts) = gather.take().unwrap_or((0, 0)); + unsafe { + stream + .launch_builder(&be.mobius_wide_tile) + .arg(&x) + .arg(&level) + .arg(&k) + .arg(&table) + .arg(&n_parts) + .launch(LaunchConfig { + grid_dim: ((low / MOBIUS_TILE_COLS as u64) as u32, grid_y, grid_z), + block_dim: (MOBIUS_TILE_COLS, 32, 1), + shared_mem_bytes: (1u32 << k) * (MOBIUS_TILE_COLS + 1) * 8, + })?; + } + level += k as u64; + } + Ok(()) +} + +/// The wide transform with the stack gathered from a source the card holds +/// (a resident store or a buffer's slices), straight into `coeffs`: no copy +/// and no memset. With `fused` (the fused encoding's first strided level) it +/// runs only the strided levels from there, the first gathering, and leaves +/// the low ones to the NTT. `false` (nothing queued) when it does not apply — +/// the knobs off, a stack under 2^11, a host source, or parts that overlap or +/// run past the stack — and the caller copies and transforms as before. +fn mobius_wide_gathered( + stream: &Arc, + be: &crate::device::Backend, + source: &Source<'_>, + coeffs: &mut CudaSlice, + log_evals: u64, + fused: Option, +) -> Result { + if (fused.is_none() && !mobius_wide_enabled()) || log_evals < MOBIUS_WIDE_LOW as u64 { + return Ok(false); + } + let len = 1usize << log_evals; + // (offset in the stack, words, source address); the guards hold the + // borrowed buffers' stream bookkeeping until the first pass is queued. + let mut parts: Vec<(u64, u64, u64)> = Vec::new(); + let mut guards = Vec::new(); + match source { + Source::Resident { + store, parts: p, .. + } => { + for &(column, offset) in p.iter() { + let (src, words) = store.at(column); + parts.push((offset as u64, words as u64, src)); + } + } + Source::Slices { + buffer, parts: p, .. + } => { + let (base, guard) = buffer.device_ptr(stream); + guards.push(guard); + for &(at, words, offset) in p.iter() { + if at + words > buffer.len() { + return Ok(false); + } + parts.push((offset as u64, words as u64, base + (at * 8) as u64)); + } + } + Source::Whole(_) | Source::Parts { .. } => return Ok(false), + } + parts.retain(|&(_, words, _)| words > 0); + parts.sort_by_key(|&(offset, _, _)| offset); + let mut end = 0u64; + for &(offset, words, _) in &parts { + if offset < end || offset + words > len as u64 { + return Ok(false); + } + end = offset + words; + } + let Ok(n_parts) = u32::try_from(parts.len()) else { + return Ok(false); + }; + let flat: Vec = parts.iter().flat_map(|&(o, w, s)| [o, w, s]).collect(); + // One word when there is no part, so the kernel gets an address; it reads + // none (every element is then zero, as the copy's memset leaves it). + let table = stream.clone_htod(if flat.is_empty() { + &[0u64][..] + } else { + &flat[..] + })?; + { + let (tab, _tab_guard) = table.device_ptr(stream); + let (x, _x_guard) = cudarc::driver::DevicePtrMut::device_ptr_mut(coeffs, stream); + match fused { + Some(from) => mobius_wide_tiles( + stream.as_ref(), + be, + x, + len, + log_evals, + from, + Some((tab, n_parts)), + )?, + None => mobius_wide(stream.as_ref(), be, x, len, log_evals, Some((tab, n_parts)))?, + } + } + drop(guards); + // Freed stream-ordered, after the first pass has read it. + drop(table); + MOBIUS_WIDE_GATHERS.fetch_add(1, Ordering::Relaxed); + Ok(true) +} + +#[cfg(test)] +mod mobius_wide_tests { + use super::mobius_wide_setting; + + /// The wide transform unless the knob says `0`, `off` or nothing. + #[test] + fn the_wide_mobius_runs_unless_zero() { + assert!( + mobius_wide_setting(None), + "the default is the wide transform" + ); + assert!(mobius_wide_setting(Some("1"))); + assert!(mobius_wide_setting(Some("on"))); + assert!(!mobius_wide_setting(Some("0"))); + assert!(!mobius_wide_setting(Some(" 0"))); + assert!(!mobius_wide_setting(Some("off"))); + assert!(!mobius_wide_setting(Some(""))); + } + + /// Fusing unless the knob says `0`, `off` or nothing. + #[test] + fn the_fused_mobius_runs_unless_zero() { + use super::mobius_fused_setting; + assert!(mobius_fused_setting(None), "the default fuses"); + assert!(mobius_fused_setting(Some("1"))); + assert!(!mobius_fused_setting(Some("0"))); + assert!(!mobius_fused_setting(Some("off"))); + assert!(!mobius_fused_setting(Some(""))); + } +} + +/// The same, with the codeword brought back — what a caller that folds on the +/// host needs. +pub fn commit_codeword_to_host( + evals: &[u64], + log_blowup: usize, + log_folding: usize, + hash: crate::DeviceHash, +) -> Result<(Vec, Vec)> { + let (codeword, _root) = commit_codeword(evals, log_blowup, log_folding, true, hash)?; + let values = crate::readback::clone_dtoh(&codeword.stream, codeword.buffer.as_ref())?; + codeword.stream.synchronize()?; + let nodes = codeword.nodes_to_host(log_folding, hash)?; + Ok((values, nodes)) +} + +/// Folds a codeword `levels` times in one residency, lifting the base field on +/// the first fold. +/// +/// `g_invs` is each level's inverse domain generator (the domain squares as the +/// codeword halves) and `alphas` the folding challenges, three u64 per level. +/// Returns the folded codeword as interleaved ext3. +pub fn fold_codeword_base( + codeword: &[u64], + two_inv: u64, + g_invs: &[u64], + alphas: &[u64], +) -> Result> { + let levels = g_invs.len(); + assert!(levels > 0, "a fold needs a level"); + assert_eq!(alphas.len(), levels * 3, "three u64 per challenge"); + assert!( + codeword.len().is_power_of_two(), + "a codeword is a power of two" + ); + assert!( + codeword.len() >> levels >= 1, + "{levels} folds do not fit a codeword of {}", + codeword.len() + ); + + let be = backend()?; + let stream = be.next_stream(); + let mut half = codeword.len() / 2; + let input = stream.clone_htod(codeword)?; + let alpha = stream.clone_htod(alphas)?; + + // SAFETY: the kernel writes every element of the half it produces. + let mut current = unsafe { alloc_or_trim::(&stream, half * 3) }?; + let half_arg = half as u64; + let g_inv = g_invs[0]; + unsafe { + stream + .launch_builder(&be.whir_fold_base_ext3) + .arg(&input) + .arg(&half_arg) + .arg(&two_inv) + .arg(&g_inv) + .arg(&alpha.slice(0..3)) + .arg(&mut current) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + drop(input); + + for (level, g_inv) in g_invs.iter().enumerate().skip(1) { + half /= 2; + // SAFETY: as above. + let mut next = unsafe { alloc_or_trim::(&stream, half * 3) }?; + let half_arg = half as u64; + unsafe { + stream + .launch_builder(&be.whir_fold_ext3) + .arg(¤t) + .arg(&half_arg) + .arg(&two_inv) + .arg(g_inv) + .arg(&alpha.slice(level * 3..level * 3 + 3)) + .arg(&mut next) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + current = next; + } + + let out = crate::readback::clone_dtoh(&stream, ¤t)?; + stream.synchronize()?; + Ok(out) +} + +/// Folds a resident codeword `levels` times, leaving the result resident. +/// +/// The chain folds the same array level after level and only opens a handful +/// of its values, so it never has to come back. +pub fn fold_resident( + codeword: &DeviceCodeword, + two_inv: u64, + g_invs: &[u64], + alphas: &[u64], +) -> Result { + let levels = g_invs.len(); + assert!(levels > 0, "a fold needs a level"); + assert_eq!(alphas.len(), levels * 3, "three u64 per challenge"); + assert!( + codeword.elements >> levels >= 1, + "{levels} folds do not fit" + ); + + let be = backend()?; + let stream = codeword.stream.clone(); + let alpha = stream.clone_htod(alphas)?; + let mut half = codeword.elements / 2; + + // SAFETY: the kernel writes every element of the half it produces. + let mut current = unsafe { alloc_or_trim::(&stream, half * 3) }?; + let half_arg = half as u64; + let kernel = if codeword.base { + &be.whir_fold_base_ext3 + } else { + &be.whir_fold_ext3 + }; + unsafe { + stream + .launch_builder(kernel) + .arg(codeword.buffer.as_ref()) + .arg(&half_arg) + .arg(&two_inv) + .arg(&g_invs[0]) + .arg(&alpha.slice(0..3)) + .arg(&mut current) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + + for (level, g_inv) in g_invs.iter().enumerate().skip(1) { + half /= 2; + // SAFETY: as above. + let mut next = unsafe { alloc_or_trim::(&stream, half * 3) }?; + let half_arg = half as u64; + unsafe { + stream + .launch_builder(&be.whir_fold_ext3) + .arg(¤t) + .arg(&half_arg) + .arg(&two_inv) + .arg(g_inv) + .arg(&alpha.slice(level * 3..level * 3 + 3)) + .arg(&mut next) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + current = next; + } + + Ok(DeviceCodeword { + buffer: Arc::new(current), + stream, + elements: half, + base: false, + // A fold is its OWN codeword: its own cache slot and its own count. It + // is committed and opened in its own right, and sharing the parent's + // slot would make one of them evict the other every round. + builds: BuildCount::default(), + leaf_passes: BuildCount::default(), + leaves: Arc::new(Mutex::new(None)), + // The fold lives inside the room the codeword it came from promised: + // it is half of it, and that one is still alive. + room: codeword.room.clone(), + // A fold is its own codeword for retention too: its own slot, its own + // registry entry once it captures. + registered: Arc::new(AtomicBool::new(false)), + placement: LeafPlacement::Natural, + }) +} + +/// The widest fold [`fold_resident_fused`] takes in one launch +/// (`WHIR_MAX_FOLD` in `whir_fold.cu`): the chain's `MAX_FOLD`. +pub const FUSED_MAX_FOLD: usize = 6; + +/// The most a `levels`-level fold of a `2^log_elements` codeword holds beside +/// its input: level by level, the first level's output and the second's live +/// together (`3·8·(2^(N−1) + 2^(N−2))` bytes; one level holds just its own); +/// fused, only the output (`3·8·2^(N−levels)`). +pub const fn fold_transient_bytes(log_elements: usize, levels: usize, fused: bool) -> u64 { + if fused || levels == 1 { + 24u64 << (log_elements - levels) + } else { + (24u64 << (log_elements - 1)) + (24u64 << (log_elements - 2)) + } +} + +/// Bytes a Merkle tree over `2^log_leaves` leaves holds while it is built: +/// `2·2^log_leaves − 1` nodes of 32 bytes, the host's layout. +pub const fn tree_bytes(log_leaves: usize) -> u64 { + 32 * ((2u64 << log_leaves) - 1) +} + +/// ★ The most one commit of a `2^log_evals` polynomial holds on the card beside +/// the codeword it leaves there. +/// +/// Two buffers, and they take turns rather than stack: the coefficients +/// (`2^log_evals` base values), which the spread reads into the codeword and +/// which are freed before the transform runs; and the tree over the codeword's +/// `2^log_folding`-value blocks, built after it. So the peak is the larger of +/// the two, not their sum. The transform runs in place, and its twiddles are +/// cached for the process outside every promise. What the commit then keeps of +/// the tree — the whole tree by default, or its leaf layer — is not a transient +/// and is not in this number: the codeword's own room grows by exactly that +/// when it is kept. +pub const fn commit_transient_bytes( + log_evals: usize, + log_blowup: usize, + log_folding: usize, +) -> u64 { + let coefficients = 8u64 << log_evals; + let tree = tree_bytes(log_evals + log_blowup - log_folding); + if coefficients > tree { + coefficients + } else { + tree + } +} + +/// [`fold_resident`]'s result in ONE launch, with nothing but the output +/// allocated. +/// +/// Level by level, the first fold of a committed codeword writes `2^(n+1)` +/// extension values — 1.5× the codeword — and holds them beside the second +/// level's while the codeword is still resident: a 2.25× transient that grows +/// with the stack. Here each output folds its own strided coset in registers +/// (`whir_fold_k_*` in `whir_fold.cu`), so the transient is the output alone, +/// `3·2^(n+2−levels)` limbs. +/// +/// ★ Raw-identical to [`fold_resident`], not merely equal: every pair runs +/// that kernel's operation sequence on the same raw inputs, `pow_base`'s +/// squares and multiplication order included (`tests/whir_fold.rs`). +pub fn fold_resident_fused( + codeword: &DeviceCodeword, + two_inv: u64, + g_invs: &[u64], + alphas: &[u64], +) -> Result { + let levels = g_invs.len(); + assert!( + (1..=FUSED_MAX_FOLD).contains(&levels), + "a fused fold takes 1..={FUSED_MAX_FOLD} levels, not {levels}" + ); + assert_eq!(alphas.len(), levels * 3, "three u64 per challenge"); + assert!( + codeword.elements >> levels >= 1, + "{levels} folds do not fit" + ); + + let be = backend()?; + let stream = codeword.stream.clone(); + let g_dev = crate::device::htod_or_trim(&stream, g_invs)?; + let alpha = crate::device::htod_or_trim(&stream, alphas)?; + let n_out = codeword.elements >> levels; + // SAFETY: the kernel writes every output element. + let mut out = unsafe { alloc_or_trim::(&stream, n_out * 3) }?; + let n_out_arg = n_out as u64; + let levels_arg = levels as u32; + let kernel = if codeword.base { + &be.whir_fold_k_base_ext3 + } else { + &be.whir_fold_k_ext3 + }; + // 256 threads: a thread carries a pending value per level, and a 1024-wide + // block would cap its registers below what that needs. + let block = 256u32; + let grid = (n_out as u64).div_ceil(block as u64) as u32; + unsafe { + stream + .launch_builder(kernel) + .arg(codeword.buffer.as_ref()) + .arg(&n_out_arg) + .arg(&levels_arg) + .arg(&two_inv) + .arg(&g_dev) + .arg(&alpha) + .arg(&mut out) + .launch(LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: 0, + })?; + } + + Ok(DeviceCodeword { + buffer: Arc::new(out), + stream, + elements: n_out, + base: false, + // A fold is its own codeword, as `fold_resident`'s is. + builds: BuildCount::default(), + leaf_passes: BuildCount::default(), + leaves: Arc::new(Mutex::new(None)), + room: codeword.room.clone(), + registered: Arc::new(AtomicBool::new(false)), + placement: LeafPlacement::Natural, + }) +} + +/// The same for a codeword already in the extension. +pub fn fold_codeword_ext3( + codeword: &[u64], + two_inv: u64, + g_invs: &[u64], + alphas: &[u64], +) -> Result> { + let levels = g_invs.len(); + assert!(levels > 0, "a fold needs a level"); + assert_eq!(alphas.len(), levels * 3, "three u64 per challenge"); + assert!( + codeword.len().is_multiple_of(3), + "three u64 per ext3 element" + ); + let elements = codeword.len() / 3; + assert!(elements.is_power_of_two(), "a codeword is a power of two"); + assert!(elements >> levels >= 1, "{levels} folds do not fit"); + + let be = backend()?; + let stream = be.next_stream(); + let mut half = elements / 2; + let alpha = stream.clone_htod(alphas)?; + let mut current = stream.clone_htod(codeword)?; + + for (level, g_inv) in g_invs.iter().enumerate() { + // SAFETY: the kernel writes every element of the half it produces. + let mut next = unsafe { alloc_or_trim::(&stream, half * 3) }?; + let half_arg = half as u64; + unsafe { + stream + .launch_builder(&be.whir_fold_ext3) + .arg(¤t) + .arg(&half_arg) + .arg(&two_inv) + .arg(g_inv) + .arg(&alpha.slice(level * 3..level * 3 + 3)) + .arg(&mut next) + .launch(LaunchConfig::for_num_elems(half as u32))?; + } + current = next; + half /= 2; + } + + let out = crate::readback::clone_dtoh(&stream, ¤t)?; + stream.synchronize()?; + Ok(out) +} + +/// Merkle-commits an ext3 codeword's fold blocks on device, returning the tree +/// in the host node layout. +/// +/// The codeword itself stays where the caller has it: a folded codeword is the +/// next round's input on the host side, so only the tree comes back. +pub fn commit_codeword_ext3( + codeword: &[u64], + log_folding: usize, + hash: crate::DeviceHash, +) -> Result> { + assert!( + codeword.len().is_multiple_of(3), + "three u64 per ext3 element" + ); + let elements = codeword.len() / 3; + assert!(elements.is_power_of_two(), "a codeword is a power of two"); + assert!( + log_folding <= elements.trailing_zeros() as usize, + "a leaf cannot exceed the codeword" + ); + let num_leaves = elements >> log_folding; + assert!(num_leaves >= 2, "tree needs at least two leaves"); + + let be = backend()?; + let stream = be.next_stream(); + let values = stream.clone_htod(codeword)?; + + let total_nodes = crate::tree_nodes(hash, num_leaves); + // SAFETY: every byte is written before it is read — the leaves by the + // kernel below, the inner nodes by the level loop after it. + let mut nodes = unsafe { alloc_or_trim::(&stream, total_nodes * 32) }?; + { + let leaves_offset = crate::leaves_offset(hash, num_leaves) * 32; + let mut leaves = nodes.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); + let num_leaves_u64 = num_leaves as u64; + let block = 1u64 << log_folding; + let (kernel, cfg) = coset_leaf_kernel(be, hash, false, num_leaves_u64)?; + unsafe { + stream + .launch_builder(kernel) + .arg(&values) + .arg(&num_leaves_u64) + .arg(&block) + .arg(&mut leaves) + .launch(cfg)?; + } + } + build_inner_tree_levels(stream.as_ref(), be, &mut nodes, num_leaves, hash)?; + + let out = crate::readback::clone_dtoh(&stream, &nodes)?; + stream.synchronize()?; + Ok(out) +} diff --git a/crypto/math-cuda/src/whir_coset.rs b/crypto/math-cuda/src/whir_coset.rs new file mode 100644 index 000000000..ac9338b07 --- /dev/null +++ b/crypto/math-cuda/src/whir_coset.rs @@ -0,0 +1,1895 @@ +//! ⛔ MEASUREMENT (on no proving path): the values a first-round query's kept +//! subtree holds, evaluated from the stacked multilinear instead of read out of +//! its re-encoded codeword. +//! +//! Today a retired first-round codeword is rebuilt whole +//! ([`crate::whir::encode_codeword_resident`]: `2^(n+b)` values) only for its +//! openings to read the `2^kept` leaves under each query's kept tree node. A +//! leaf `j` is the `2^fold` values `word[j + t·2^(D−fold)]`, `D = n + b`. If +//! the tree put leaf `j` at position `bitrev(j)`, the leaves under one kept +//! node would be `j ≡ c (mod 2^(D−L))`, `L = fold + kept`, and their values the +//! codeword on ONE coset `ω^c·⟨ζ⟩`, `ζ = ω^(2^(D−L))`, `2^L` points. +//! +//! The codeword is `P(ω^j)` with `P(x) = f̂(x^(2^(n−1)), …, x^2, x)`: hypercube +//! bit `s` goes to `x^(2^(n−1−s))` (`multilinear::whir::lift_coefficients`). +//! On the coset `x^(2^m)` is the constant `ω^(c·2^m)` for every `m ≥ L`, so +//! the bits `s < n − L` fold away against constants, +//! `f'(rest) = f(rest, 0) + γ·(f(rest, 1) − f(rest, 0))`, +//! `γ_s(c) = ω^(c·2^(n−1−s))`, and what is left is a multilinear of `L` +//! variables: its Möbius transform, a twist by `ω^(c·i)` and one NTT of `2^L` +//! over `ζ` give the coset's values. Bit `s`'s constant depends on `c` modulo +//! `2^(b+1+s)` only, so the low bits fold once per residue class for every +//! coset in it (the tile kernel), the rest per coset. +//! +//! No Möbius transform of the whole polynomial and no `2^(n+b)` transform. +//! [`CosetPlan::evaluate_on_host`] is the same algorithm on the host, and the +//! tests pin both against `encode(lift_coefficients(f))`. +//! +//! The module is loaded on first use from its own cubin, never by +//! [`crate::device::Backend::init`]. + +use std::sync::{Arc, OnceLock}; + +use cudarc::driver::{ + CudaFunction, CudaModule, CudaSlice, CudaStream, LaunchConfig, PushKernelArg, +}; +use cudarc::nvrtc::Ptx; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::{IsFFTField, IsField}; + +use crate::Result; +use crate::device::{alloc_or_trim, backend, htod_or_trim}; + +const WHIR_COSET_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/whir_coset.cubin")); + +/// Tile bits the tile kernel folds at most: a 256-value tile, its block. +const MAX_TILE_LEVELS: usize = 8; +/// Bits the rest kernel folds at most: one warp's lanes. +const MAX_REST_LEVELS: usize = 5; +/// Tiles a block of the tile kernel folds (`COSET_TILES_PER_BLOCK`). +const TILES_PER_BLOCK: usize = 8; +/// NTT stages and Möbius levels the chunk kernel takes in shared memory. +const MAX_CHUNK_LEVELS: usize = 10; +/// Dynamic shared memory the tile kernel may ask for: under the 48 KiB a +/// block takes without an attribute, with room for its static arrays. +const SHARED_LIMIT: usize = 47 * 1024; +const BLOCK: u32 = 256; + +struct Module { + _module: Arc, + fold_tiles: CudaFunction, + fold_tiles_parts: CudaFunction, + tile_map: CudaFunction, + fold_rest: CudaFunction, + mobius_level: CudaFunction, + chunk: CudaFunction, + ntt_stage: CudaFunction, + leaf_order: CudaFunction, +} + +fn module() -> Result<&'static Module> { + static MODULE: OnceLock = OnceLock::new(); + if let Some(m) = MODULE.get() { + return Ok(m); + } + let be = backend()?; + let module = be + .ctx + .load_module(Ptx::from_binary(WHIR_COSET_CUBIN.to_vec()))?; + let m = Module { + fold_tiles: module.load_function("coset_fold_tiles")?, + fold_tiles_parts: module.load_function("coset_fold_tiles_parts")?, + tile_map: module.load_function("coset_tile_map")?, + fold_rest: module.load_function("coset_fold_rest")?, + mobius_level: module.load_function("coset_mobius_level")?, + chunk: module.load_function("coset_chunk")?, + ntt_stage: module.load_function("coset_ntt_stage")?, + leaf_order: module.load_function("coset_leaf_order")?, + _module: module, + }; + Ok(MODULE.get_or_init(|| m)) +} + +fn fmul(a: u64, b: u64) -> u64 { + GoldilocksField::mul(&a, &b) +} + +fn fadd(a: u64, b: u64) -> u64 { + GoldilocksField::add(&a, &b) +} + +fn fsub(a: u64, b: u64) -> u64 { + GoldilocksField::sub(&a, &b) +} + +fn fpow(mut base: u64, mut exp: u64) -> u64 { + let mut acc = 1u64; + while exp > 0 { + if exp & 1 == 1 { + acc = fmul(acc, base); + } + base = fmul(base, base); + exp >>= 1; + } + acc +} + +fn fold(f0: u64, f1: u64, gamma: u64) -> u64 { + fadd(f0, fmul(gamma, fsub(f1, f0))) +} + +fn reverse(value: u64, bits: usize) -> u64 { + if bits == 0 { + 0 + } else { + value.reverse_bits() >> (64 - bits) + } +} + +/// Everything the kernels read besides the polynomial, for one set of queried +/// leaves: the cosets they fall in and the folding constants. +#[derive(Clone, Debug)] +pub struct CosetPlan { + num_vars: usize, + log_blowup: usize, + log_coset: usize, + kept: usize, + /// The cosets, as `c = j mod 2^(D−L)` over the queried leaves, ascending + /// and distinct. Output block `q` is coset `classes[q]`. + classes: Vec, + tile_levels: usize, + rest_levels: usize, + level_count: Vec, + level_offset: Vec, + parents: Vec, + gammas: Vec, + buf_len: usize, + slab_of: Vec, + rest_gammas: Vec, + twists: Vec, + twiddles: Vec, +} + +impl CosetPlan { + /// The plan for a codeword of a `num_vars`-variable multilinear at + /// `log_blowup`, leaves of `2^log_fold` values, `kept` binary levels under + /// a kept node, and the queried `leaves` (each `< 2^(D−log_fold)`). + /// + /// `None` for a shape the kernels do not take: no bit to fold + /// (`num_vars ≤ L`), more than 13 bits to fold (8 tile + 5 rest), a + /// domain past the field's two-adicity, a leaf out of range, or no leaf. + pub fn new( + num_vars: usize, + log_blowup: usize, + log_fold: usize, + kept: usize, + leaves: &[u64], + ) -> Option { + let log_coset = log_fold + kept; + let log_domain = num_vars + log_blowup; + if leaves.is_empty() + || log_coset < 1 + || num_vars <= log_coset + || log_domain > GoldilocksField::TWO_ADICITY as usize + || leaves.iter().any(|&j| j >> (log_domain - log_fold) != 0) + { + return None; + } + let folded = num_vars - log_coset; + let tile_levels = folded.min(MAX_TILE_LEVELS); + let rest_levels = folded - tile_levels; + if rest_levels > MAX_REST_LEVELS { + return None; + } + let omega = *GoldilocksField::get_primitive_root_of_unity(log_domain as u64) + .ok()? + .value(); + // ω^(2^m), m = 0..D. + let mut omega_pow2 = Vec::with_capacity(log_domain + 1); + let mut w = omega; + for _ in 0..=log_domain { + omega_pow2.push(w); + w = fmul(w, w); + } + // Bit `s`'s constant for the class of `c`: ω^(c·2^(n−1−s)). + let gamma_of = |c: u64, s: usize| fpow(omega_pow2[num_vars - 1 - s], c); + + let class_mod = 1u64 << (log_domain - log_coset); + let mut classes: Vec = leaves.iter().map(|&j| j & (class_mod - 1)).collect(); + classes.sort_unstable(); + classes.dedup(); + + // Level t lists the residues c mod 2^(b+1+t), ascending. + let mut keys: Vec> = Vec::with_capacity(tile_levels); + let mut level_count = Vec::with_capacity(tile_levels); + let mut level_offset = Vec::with_capacity(tile_levels); + let mut parents = Vec::new(); + let mut gammas = Vec::new(); + for t in 0..tile_levels { + let modulus = 1u64 << (log_blowup + 1 + t); + let mut level: Vec = classes.iter().map(|&c| c & (modulus - 1)).collect(); + level.sort_unstable(); + level.dedup(); + level_offset.push(parents.len() as u32); + level_count.push(level.len() as u32); + for &key in &level { + let parent = if t == 0 { + 0 + } else { + keys[t - 1] + .binary_search(&(key & ((modulus >> 1) - 1))) + .ok()? + }; + parents.push(parent as u32); + gammas.push(gamma_of(key, t)); + } + keys.push(level); + } + let buf_len = (0..tile_levels) + .map(|t| (level_count[t] as usize) << (tile_levels - 1 - t)) + .chain(std::iter::once(1usize << tile_levels)) + .max() + .unwrap_or(1); + + let last = keys.last()?; + let last_mod = 1u64 << (log_blowup + tile_levels); + let mut slab_of = Vec::with_capacity(classes.len()); + let mut rest_gammas = Vec::with_capacity(classes.len() * rest_levels); + for &c in &classes { + slab_of.push(last.binary_search(&(c & (last_mod - 1))).ok()? as u32); + for l in 0..rest_levels { + rest_gammas.push(gamma_of(c, tile_levels + l)); + } + } + let twists = classes.iter().map(|&c| fpow(omega, c)).collect(); + let zeta = omega_pow2[log_domain - log_coset]; + let mut twiddles = Vec::with_capacity(1 << (log_coset - 1)); + let mut z = 1u64; + for _ in 0..1usize << (log_coset - 1) { + twiddles.push(z); + z = fmul(z, zeta); + } + + let plan = Self { + num_vars, + log_blowup, + log_coset, + kept, + classes, + tile_levels, + rest_levels, + level_count, + level_offset, + parents, + gammas, + buf_len, + slab_of, + rest_gammas, + twists, + twiddles, + }; + (plan.tile_shared_bytes() <= SHARED_LIMIT).then_some(plan) + } + + /// The cosets, ascending; output block `q` is `classes()[q]`. + pub fn classes(&self) -> &[u64] { + &self.classes + } + + /// Values a coset holds: `2^L`. + pub fn coset_len(&self) -> usize { + 1 << self.log_coset + } + + /// The output block of the coset `leaf` falls in. + pub fn block_of(&self, leaf: u64) -> Option { + let class_mod = 1u64 << self.classes_bits(); + self.classes.binary_search(&(leaf & (class_mod - 1))).ok() + } + + /// Where `leaf` sits among the blocks: its coset's output block `q` and + /// its position `r` under that coset's kept node (`leaf = + /// classes()[q] + bitrev_kept(r)·2^(D−L)`). + pub fn slot_of(&self, leaf: u64) -> Option<(usize, usize)> { + let q = self.block_of(leaf)?; + let r = reverse(leaf >> self.classes_bits(), self.kept) as usize; + Some((q, r)) + } + + /// The tree position, among the kept nodes, of output block `q`'s node: + /// the coset's class with its `D − L` bits reversed. + pub fn block_position(&self, q: usize) -> usize { + reverse(self.classes[q], self.classes_bits()) as usize + } + + /// Binary tree levels under a kept node: a block is `2^kept` leaves. + pub fn kept(&self) -> usize { + self.kept + } + + /// `D − L`: there are `2^(D−L)` cosets of `2^L` points in the domain. + fn classes_bits(&self) -> usize { + self.num_vars + self.log_blowup - self.log_coset + } + + /// The leaves of output block `q` in its order: entry `r` is the leaf at + /// position `r` under the kept node, `c + bitrev_kept(r)·2^(D−L)`. + pub fn block_leaves(&self, q: usize) -> Vec { + let stride = 1u64 << self.classes_bits(); + (0..1u64 << self.kept) + .map(|r| self.classes[q] + reverse(r, self.kept) * stride) + .collect() + } + + /// Tiles of `2^tile_levels` values the polynomial splits into. + fn tiles(&self) -> usize { + 1 << (self.num_vars - self.tile_levels) + } + + /// Classes the last tile level lists: the slab's rows. + fn slab_rows(&self) -> usize { + *self.level_count.last().unwrap_or(&0) as usize + } + + /// Möbius levels and NTT stages the chunk kernel takes. + fn chunk_levels(&self) -> usize { + self.log_coset.min(MAX_CHUNK_LEVELS) + } + + /// Bytes the tile kernel's dynamic shared memory takes: the constants, + /// two level buffers, the block's last-level values, the parents. + fn tile_shared_bytes(&self) -> usize { + let total = self.gammas.len(); + 8 * (total + 2 * self.buf_len + self.slab_rows() * TILES_PER_BLOCK) + 4 * total + } + + /// Bytes the device evaluation allocates for a polynomial: the assembled + /// stack, the classes' slabs, the cosets twice (transform and leaf order). + pub fn device_bytes(&self) -> usize { + 8 * ((1 << self.num_vars) + + self.slab_rows() * self.tiles() + + 2 * self.classes.len() * self.coset_len()) + } + + /// The cosets' values from the multilinear's `2^n` hypercube values, on the + /// host, by the kernels' algorithm: block `q` is `2^L` values in leaf order + /// (leaf `r`'s `2^fold` values at `r·2^fold`). + pub fn evaluate_on_host(&self, values: &[u64]) -> Vec { + assert_eq!( + values.len(), + 1 << self.num_vars, + "one value a hypercube point" + ); + let tile = 1usize << self.tile_levels; + let tiles = self.tiles(); + let mut slab = vec![0u64; self.slab_rows() * tiles]; + for at in 0..tiles { + let mut rows = vec![values[at * tile..(at + 1) * tile].to_vec()]; + for t in 0..self.tile_levels { + let off = self.level_offset[t] as usize; + rows = (0..self.level_count[t] as usize) + .map(|k| { + let row = &rows[self.parents[off + k] as usize]; + row.chunks_exact(2) + .map(|p| fold(p[0], p[1], self.gammas[off + k])) + .collect() + }) + .collect(); + } + for (k, row) in rows.iter().enumerate() { + slab[k * tiles + at] = row[0]; + } + } + + let n = self.coset_len(); + let fold_bits = self.log_coset - self.kept; + let mut out = Vec::with_capacity(self.classes.len() * n); + for q in 0..self.classes.len() { + let row = &slab[self.slab_of[q] as usize * tiles..][..tiles]; + let gammas = &self.rest_gammas[q * self.rest_levels..][..self.rest_levels]; + let mut g: Vec = row + .chunks_exact(1 << self.rest_levels) + .map(|chunk| { + let mut v = chunk.to_vec(); + for &gamma in gammas { + v = v.chunks_exact(2).map(|p| fold(p[0], p[1], gamma)).collect(); + } + v[0] + }) + .collect(); + for level in 0..self.log_coset { + for i in 0..n { + if i >> level & 1 == 1 { + g[i] = fsub(g[i], g[i ^ (1 << level)]); + } + } + } + for (h, v) in g.iter_mut().enumerate() { + *v = fmul(*v, fpow(self.twists[q], reverse(h as u64, self.log_coset))); + } + for s in 1..=self.log_coset { + let half = 1usize << (s - 1); + for block in (0..n).step_by(2 * half) { + for j in 0..half { + let w = self.twiddles[j << (self.log_coset - s)]; + let u = g[block + j]; + let v = fmul(g[block + j + half], w); + g[block + j] = fadd(u, v); + g[block + j + half] = fsub(u, v); + } + } + } + let mut block = vec![0u64; n]; + for (u, &v) in g.iter().enumerate() { + let r = reverse(u as u64 & ((1 << self.kept) - 1), self.kept) as usize; + block[(r << fold_bits) | (u >> self.kept)] = v; + } + out.extend_from_slice(&block); + } + out + } +} + +/// The cosets' values on the card: block `q` (`2^L` values, leaf order) for +/// coset `plan.classes()[q]`, queued on `stream`. +pub struct CosetValues { + buffer: CudaSlice, + stream: Arc, +} + +impl CosetValues { + /// Waits for the evaluation queued on this buffer's stream. + pub fn synchronize(&self) -> Result<()> { + self.stream.synchronize() + } + + /// Every block, read back. + pub fn to_host(&self) -> Result> { + let values = crate::readback::clone_dtoh(&self.stream, &self.buffer)?; + self.stream.synchronize()?; + Ok(values) + } +} + +/// Wall-clock laps of an evaluation's phases, read only when asked for. +struct Laps<'a> { + split: Option<&'a mut Vec<(&'static str, f64)>>, + clock: std::time::Instant, +} + +impl Laps<'_> { + /// Synchronizes and records the phase just queued, when laps are kept. + fn lap(&mut self, name: &'static str, stream: &Arc) -> Result<()> { + if let Some(split) = self.split.as_deref_mut() { + stream.synchronize()?; + split.push((name, self.clock.elapsed().as_secs_f64() * 1e3)); + self.clock = std::time::Instant::now(); + } + Ok(()) + } +} + +/// The plan's cosets over a stacked polynomial whose columns the card holds: +/// `(column, offset)` into `store`, zeros between, as +/// [`crate::whir::encode_codeword_resident`] assembles it. +/// +/// `split`, when given, synchronizes after each phase and records its wall +/// milliseconds (assemble, tiles, rest, transform); the timed path passes +/// `None` and queues everything. +pub fn evaluate_resident( + store: &crate::columns::DeviceColumns, + parts: &[(usize, usize)], + plan: &CosetPlan, + split: Option<&mut Vec<(&'static str, f64)>>, +) -> Result { + let be = backend()?; + let m = module()?; + let stream = be.next_stream(); + let n = 1usize << plan.num_vars; + let mut laps = Laps { + split, + clock: std::time::Instant::now(), + }; + for &(column, offset) in parts { + ensure!(column < store.num_columns() && offset + store.span(column).1 <= n); + } + // SAFETY: the memset writes every element. + let mut f = unsafe { alloc_or_trim::(&stream, n) }?; + stream.memset_zeros(&mut f)?; + for &(column, offset) in parts { + store.copy_into(column, &mut f, offset, &stream)?; + } + laps.lap("assemble", &stream)?; + let len = plan.classes.len() * plan.coset_len(); + let out = evaluate_from(&stream, m, Tiles::Assembled(f), plan, &mut laps, 0, len)?; + Ok(CosetValues { + buffer: out, + stream, + }) +} + +/// [`evaluate_resident`] over a [`StackSlices`]: the stack read where it lies +/// when `in_place` and its parts allow ([`StackSlices::parts_in_place`]; the +/// kinds of its tiles returned), assembled into a fresh buffer otherwise +/// (`None`). The same values either way, word for word. +pub fn evaluate_slices( + stack: &StackSlices, + plan: &CosetPlan, + in_place: bool, + split: Option<&mut Vec<(&'static str, f64)>>, +) -> Result<(CosetValues, Option)> { + ensure!(plan.num_vars == stack.num_vars && plan.log_blowup == stack.log_blowup); + let be = backend()?; + let m = module()?; + let stream = be.next_stream(); + let mut laps = Laps { + split, + clock: std::time::Instant::now(), + }; + let parts = if in_place { + stack.parts_in_place() + } else { + None + }; + let (tiles, kinds) = match &parts { + Some(parts) => ( + Tiles::InPlace { + data: stack.buffer.as_ref(), + parts, + }, + Some(parts.tile_kinds(plan.num_vars, plan.tile_levels)), + ), + None => (Tiles::Assembled(stack.assemble(&stream)?), None), + }; + laps.lap("assemble", &stream)?; + let len = plan.classes.len() * plan.coset_len(); + let out = evaluate_from(&stream, m, tiles, plan, &mut laps, 0, len)?; + Ok(( + CosetValues { + buffer: out, + stream, + }, + kinds, + )) +} + +/// What the tile kernel reads: the assembled stack (spent by it), or the stack +/// where it lies, by its parts. +enum Tiles<'a> { + Assembled(CudaSlice), + InPlace { + data: &'a CudaSlice, + parts: &'a StackParts, + }, +} + +/// A stack's tiles by kind ([`StackParts::tile_kinds`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TileKinds { + /// Tiles one part holds all of: read straight from the store. + pub full: u64, + /// Tiles no part touches: folded to zero without being read. + pub zero: u64, + /// Tiles of shorter columns, a column's end or zeros: read value by value. + pub mixed: u64, +} + +/// A stack's parts as the in-place kernels read them: sorted by their offset +/// in the stack, disjoint, the empty ones dropped; part `p` is `len[p]` values +/// at `off[p]` in the stack and at `at[p]` in the store's buffer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StackParts { + off: Vec, + len: Vec, + at: Vec, +} + +impl StackParts { + /// `parts` — `(element offset in the buffer, elements, offset in the + /// stack)` — of a `2^num_vars`-value stack in a buffer of `buffer_len` + /// elements. `None` for parts the kernels cannot place: overlapping, past + /// the stack or past the buffer. + pub fn new( + parts: &[(usize, usize, usize)], + num_vars: usize, + buffer_len: usize, + ) -> Option { + let n = 1u64 << num_vars; + let mut sorted: Vec<(u64, u64, u64)> = parts + .iter() + .filter(|&&(_, len, _)| len > 0) + .map(|&(at, len, off)| (off as u64, len as u64, at as u64)) + .collect(); + sorted.sort_unstable(); + let mut end = 0u64; + for &(off, len, at) in &sorted { + if off < end || off.checked_add(len)? > n || at.checked_add(len)? > buffer_len as u64 { + return None; + } + end = off + len; + } + // The map keeps two words for its kinds. + (sorted.len() < u32::MAX as usize - 1).then(|| Self { + off: sorted.iter().map(|p| p.0).collect(), + len: sorted.iter().map(|p| p.1).collect(), + at: sorted.iter().map(|p| p.2).collect(), + }) + } + + /// How many of the stack's `2^tile_bits`-value tiles each kind takes, from + /// the parts alone (no tile is visited): the tiles whole inside a part, the + /// tiles no part touches, and the rest. + pub fn tile_kinds(&self, num_vars: usize, tile_bits: usize) -> TileKinds { + let tile = 1u64 << tile_bits; + let tiles = 1u64 << (num_vars - tile_bits); + let (mut full, mut touched, mut reached) = (0u64, 0u64, 0u64); + for (&off, &len) in self.off.iter().zip(&self.len) { + let end = off + len; + full += (end / tile).saturating_sub(off.div_ceil(tile)); + // Sorted and disjoint: the touched spans, merged as they come. + let (from, to) = ((off / tile).max(reached), end.div_ceil(tile)); + if to > from { + touched += to - from; + reached = to; + } + } + TileKinds { + full, + zero: tiles - touched, + mixed: touched - full, + } + } +} + +/// The cosets from the stack `tiles` reads (an assembled one is spent), into a +/// buffer of `out_len` values laid out by `stride` (`coset_leaf_order`), on +/// `stream`. +fn evaluate_from( + stream: &Arc, + m: &Module, + tiles_from: Tiles<'_>, + plan: &CosetPlan, + laps: &mut Laps<'_>, + stride: u64, + out_len: usize, +) -> Result> { + let tiles = plan.tiles(); + let last = plan.slab_rows(); + let count = htod_or_trim(stream, &plan.level_count)?; + let offset = htod_or_trim(stream, &plan.level_offset)?; + let parents = htod_or_trim(stream, &plan.parents)?; + let gammas = htod_or_trim(stream, &plan.gammas)?; + // SAFETY: the tile kernel writes every (class, tile) entry. + let mut slab = unsafe { alloc_or_trim::(stream, last * tiles) }?; + let tiles_u64 = tiles as u64; + let levels_u32 = plan.tile_levels as u32; + let total_u32 = plan.gammas.len() as u32; + let buf_u32 = plan.buf_len as u32; + // The long kernels' grids are cut to the calling thread's cap, if any + // (`device::grid_cap`); each strides over its work, the same values. + let capped = crate::device::capped_grid; + let tile_grid = LaunchConfig { + grid_dim: (capped(tiles.div_ceil(TILES_PER_BLOCK) as u32), 1, 1), + block_dim: (BLOCK, 1, 1), + shared_mem_bytes: plan.tile_shared_bytes() as u32, + }; + match tiles_from { + Tiles::Assembled(f) => { + unsafe { + stream + .launch_builder(&m.fold_tiles) + .arg(&f) + .arg(&tiles_u64) + .arg(&levels_u32) + .arg(&count) + .arg(&offset) + .arg(&parents) + .arg(&gammas) + .arg(&total_u32) + .arg(&buf_u32) + .arg(&mut slab) + .launch(tile_grid)?; + } + // Spent; the free is stream-ordered. + drop(f); + } + Tiles::InPlace { data, parts } => { + // A stack with no part (all zeros) still uploads one word a + // table: a zero-length upload is not an allocation. + let upload = |v: &[u64]| { + if v.is_empty() { + htod_or_trim(stream, &[0u64]) + } else { + htod_or_trim(stream, v) + } + }; + let (off, len, at) = (upload(&parts.off)?, upload(&parts.len)?, upload(&parts.at)?); + let nparts = parts.off.len() as u32; + let tile_bits = plan.tile_levels as u32; + // SAFETY: the map kernel writes every tile's entry. + let mut map = unsafe { alloc_or_trim::(stream, tiles) }?; + unsafe { + stream + .launch_builder(&m.tile_map) + .arg(&off) + .arg(&len) + .arg(&nparts) + .arg(&tile_bits) + .arg(&tiles_u64) + .arg(&mut map) + .launch(LaunchConfig { + grid_dim: (capped(tiles.div_ceil(BLOCK as usize) as u32), 1, 1), + block_dim: (BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + stream + .launch_builder(&m.fold_tiles_parts) + .arg(data) + .arg(&map) + .arg(&off) + .arg(&len) + .arg(&at) + .arg(&nparts) + .arg(&tiles_u64) + .arg(&levels_u32) + .arg(&count) + .arg(&offset) + .arg(&parents) + .arg(&gammas) + .arg(&total_u32) + .arg(&buf_u32) + .arg(&mut slab) + .launch(tile_grid)?; + } + } + } + laps.lap("tiles", stream)?; + + let cosets = plan.classes.len(); + let len = plan.coset_len(); + let log_len = plan.log_coset as u32; + let slab_of = htod_or_trim(stream, &plan.slab_of)?; + // A zero-length upload is not an allocation the driver takes; the kernel + // reads no constant when there is no bit left. + let rest_gammas = if plan.rest_gammas.is_empty() { + htod_or_trim(stream, &[0u64])? + } else { + htod_or_trim(stream, &plan.rest_gammas)? + }; + // SAFETY: the rest kernel writes every coset value. + let mut g = unsafe { alloc_or_trim::(stream, cosets * len) }?; + let rest_u32 = plan.rest_levels as u32; + let cosets_u32 = cosets as u32; + let warps = (cosets * len) as u64; + unsafe { + stream + .launch_builder(&m.fold_rest) + .arg(&slab) + .arg(&tiles_u64) + .arg(&rest_u32) + .arg(&cosets_u32) + .arg(&slab_of) + .arg(&rest_gammas) + .arg(&log_len) + .arg(&mut g) + .launch(LaunchConfig { + grid_dim: (capped((warps * 32).div_ceil(BLOCK as u64) as u32), 1, 1), + block_dim: (BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + drop(slab); + laps.lap("rest", stream)?; + + let twists = htod_or_trim(stream, &plan.twists)?; + let twiddles = htod_or_trim(stream, &plan.twiddles)?; + let chunk_levels = plan.chunk_levels(); + let pairs = (cosets * len / 2) as u64; + let pair_grid = LaunchConfig { + grid_dim: (pairs.div_ceil(BLOCK as u64) as u32, 1, 1), + block_dim: (BLOCK, 1, 1), + shared_mem_bytes: 0, + }; + // The Möbius levels commute, so the ones past the chunk go first and the + // chunk kernel ends the transform where the NTT starts. + for level in chunk_levels..plan.log_coset { + let level_u32 = level as u32; + unsafe { + stream + .launch_builder(&m.mobius_level) + .arg(&mut g) + .arg(&pairs) + .arg(&log_len) + .arg(&level_u32) + .launch(pair_grid)?; + } + } + let chunk_u32 = chunk_levels as u32; + unsafe { + stream + .launch_builder(&m.chunk) + .arg(&mut g) + .arg(&log_len) + .arg(&chunk_u32) + .arg(&twists) + .arg(&twiddles) + .launch(LaunchConfig { + grid_dim: ((cosets << (plan.log_coset - chunk_levels)) as u32, 1, 1), + block_dim: (1 << (chunk_levels - 1), 1, 1), + shared_mem_bytes: (8usize << chunk_levels) as u32, + })?; + } + for stage in chunk_levels + 1..=plan.log_coset { + let stage_u32 = stage as u32; + unsafe { + stream + .launch_builder(&m.ntt_stage) + .arg(&mut g) + .arg(&pairs) + .arg(&log_len) + .arg(&stage_u32) + .arg(&twiddles) + .launch(pair_grid)?; + } + } + // SAFETY: the permutation writes every value of a coset; past them (a + // strided layout padded to a power of two) the memset below does. + let mut out = unsafe { alloc_or_trim::(stream, out_len) }?; + if out_len > cosets * len { + stream.memset_zeros(&mut out)?; + } + let total = (cosets * len) as u64; + let kept_u32 = plan.kept as u32; + unsafe { + stream + .launch_builder(&m.leaf_order) + .arg(&g) + .arg(&total) + .arg(&log_len) + .arg(&kept_u32) + .arg(&stride) + .arg(&mut out) + .launch(LaunchConfig { + grid_dim: (total.div_ceil(BLOCK as u64) as u32, 1, 1), + block_dim: (BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + drop(g); + laps.lap("transform", stream)?; + Ok(out) +} + +/// A stacked polynomial the card holds as slices of one buffer — `(element +/// offset in the buffer, elements, offset in the stack)`, zeros between — +/// with its own handle on the buffer, so a revived commitment can evaluate its +/// cosets after the borrow of the store it was revived from has ended. +#[derive(Clone)] +pub struct StackSlices { + buffer: Arc>, + parts: Vec<(usize, usize, usize)>, + num_vars: usize, + log_blowup: usize, + /// What a successor encoded from this stack's bound message lives inside, + /// as a fold lives inside its codeword's room. + room: Arc, +} + +impl StackSlices { + /// The stack `parts` — `(column, offset)` into `store` — make, at + /// `2^num_vars` values and a code of `2^log_blowup` times that. + pub fn from_resident( + store: &crate::columns::DeviceColumns, + parts: &[(usize, usize)], + num_vars: usize, + log_blowup: usize, + ) -> Result { + let n = 1usize << num_vars; + let mut held = Vec::with_capacity(parts.len()); + for &(column, offset) in parts { + ensure!(column < store.num_columns()); + let (at, len) = store.span(column); + ensure!(offset + len <= n); + held.push((at, len, offset)); + } + let room = crate::device::reserve(0).ok_or_else(crate::invalid_value)?; + Ok(Self { + buffer: store.buffer(), + parts: held, + num_vars, + log_blowup, + room: Arc::new(room), + }) + } + + /// Values the codeword would have: `2^(n + b)`. + pub fn elements(&self) -> usize { + 1 << (self.num_vars + self.log_blowup) + } + + pub fn num_vars(&self) -> usize { + self.num_vars + } + + pub fn log_blowup(&self) -> usize { + self.log_blowup + } + + pub fn room(&self) -> &Arc { + &self.room + } + + /// The whole codeword, as the re-encode makes it: what a fold, or a plan + /// the kernels do not take, falls back to. + pub fn encode(&self, transient: bool) -> Result { + crate::whir::encode_codeword_slices( + &self.buffer, + &self.parts, + self.num_vars, + self.log_blowup, + transient, + ) + } + + /// The parts as the in-place kernels read them, or `None` when they cannot + /// place them ([`StackParts::new`]): the caller assembles the stack. + pub fn parts_in_place(&self) -> Option { + StackParts::new(&self.parts, self.num_vars, self.buffer.len()) + } + + fn assemble(&self, stream: &Arc) -> Result> { + // SAFETY: `copy_slices` writes every element (zeros, then the parts). + let mut f = unsafe { alloc_or_trim::(stream, 1 << self.num_vars) }?; + crate::whir::copy_slices(stream, &self.buffer, &self.parts, &mut f)?; + Ok(f) + } +} + +/// What a retired round-0 tree in the coset order needs for its openings, +/// evaluated from the stack: one tree over the plan's blocks (block `q` = the +/// gathered leaves `[q·2^kept, (q+1)·2^kept)`, padded to `leaves`), in the +/// hash's node layout as `DeviceCodeword::block_subtrees` returns it, and the +/// queried blocks' values. +pub struct CosetOpenings { + pub nodes: Vec, + pub leaves: usize, + /// `2^fold` values a query, in the order the queries were given. + pub values: Vec, + /// The stack's tiles by kind when it was read where it lies; `None` when + /// it was assembled (not asked for, or parts the kernels cannot place). + pub in_place: Option, +} + +/// The kept blocks' subtrees and the queried blocks' values of a round-0 +/// tree in the coset order, from the stack: the cosets evaluated straight into +/// the strided layout the coset leaf kernel reads, hashed and built by the +/// commit's own kernels (so each block's subtree is the committed one, which +/// the caller checks against its kept node), and the queried blocks gathered +/// from the same buffer. `queried` are leaf indices, each in one of the plan's +/// cosets. Poseidon1 trees only. `in_place`: the stack read where it lies +/// ([`evaluate_slices`]); the same words either way. +pub fn coset_openings( + stack: &StackSlices, + plan: &CosetPlan, + queried: &[u64], + hash: crate::DeviceHash, + in_place: bool, +) -> Result { + ensure!(hash == crate::DeviceHash::Poseidon1); + ensure!(plan.num_vars == stack.num_vars && plan.log_blowup == stack.log_blowup); + let be = backend()?; + let m = module()?; + let stream = be.next_stream(); + let block = 1usize << (plan.log_coset - plan.kept); + let leaves = (plan.classes.len() << plan.kept).next_power_of_two().max(2); + let mut at = Vec::with_capacity(queried.len()); + for &leaf in queried { + let (q, r) = plan.slot_of(leaf).ok_or_else(crate::invalid_value)?; + at.push(((q << plan.kept) | r) as u64); + } + + let parts = if in_place { + stack.parts_in_place() + } else { + None + }; + let (tiles, kinds) = match &parts { + Some(parts) => ( + Tiles::InPlace { + data: stack.buffer.as_ref(), + parts, + }, + Some(parts.tile_kinds(plan.num_vars, plan.tile_levels)), + ), + None => (Tiles::Assembled(stack.assemble(&stream)?), None), + }; + let mut laps = Laps { + split: None, + clock: std::time::Instant::now(), + }; + let out = evaluate_from( + &stream, + m, + tiles, + plan, + &mut laps, + leaves as u64, + leaves * block, + )?; + + let total_nodes = crate::tree_nodes(hash, leaves); + // SAFETY: every byte is written before it is read — the leaves by the + // kernel below, the inner nodes by the level loop after it. + let mut nodes = unsafe { alloc_or_trim::(&stream, total_nodes * 32) }?; + let leaves_offset = crate::leaves_offset(hash, leaves) * 32; + let leaves_u64 = leaves as u64; + let block_u64 = block as u64; + { + let mut region = nodes.slice_mut(leaves_offset..leaves_offset + leaves * 32); + let k = crate::p1_stark::kernels()?; + unsafe { + stream + .launch_builder(&k.leaves_base_coset) + .arg(&out) + .arg(&leaves_u64) + .arg(&block_u64) + .arg(&mut region) + .launch(crate::p1_stark::launch_cfg(leaves_u64))?; + } + } + crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes, leaves, hash)?; + let nodes_host = crate::readback::clone_dtoh(&stream, &nodes)?; + + let index_dev = htod_or_trim(&stream, &at)?; + // SAFETY: the gather writes every value it is sized for. + let mut values = unsafe { alloc_or_trim::(&stream, queried.len().max(1) * block) }?; + if !queried.is_empty() { + let queries = queried.len() as u64; + let limbs = 1u64; + unsafe { + stream + .launch_builder(&be.gather_cosets) + .arg(&out) + .arg(&index_dev) + .arg(&queries) + .arg(&leaves_u64) + .arg(&block_u64) + .arg(&limbs) + .arg(&mut values) + .launch(LaunchConfig::for_num_elems((queried.len() * block) as u32))?; + } + } + let mut values_host = crate::readback::clone_dtoh(&stream, &values)?; + stream.synchronize()?; + values_host.truncate(queried.len() * block); + Ok(CosetOpenings { + nodes: nodes_host, + leaves, + values: values_host, + in_place: kinds, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::element::FieldElement; + use multilinear::mle::Mle; + use multilinear::whir::{Domain, encode, lift_coefficients}; + + type FE = FieldElement; + const P: u64 = 0xFFFF_FFFF_0000_0001; + + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + } + + fn canonical(v: u64) -> u64 { + if v >= P { v - P } else { v } + } + + /// The codeword the commit encodes: `encode(lift_coefficients(f))` over + /// the `2^(n+b)` subgroup. + fn codeword(values: &[u64], num_vars: usize, log_blowup: usize) -> Vec { + let mle = Mle::new(values.iter().map(|&v| FE::from(v)).collect()).expect("mle"); + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + encode::(&lift_coefficients(&mle), &domain) + .expect("encode") + .iter() + .map(|v| canonical(*v.value())) + .collect() + } + + /// Every block equals the codeword at its leaves, leaf by leaf and value + /// by value: block `q`'s entry `r·2^fold + t` is `word[leaf_r + t·2^(D−fold)]`. + fn the_host_plan_is_the_codeword( + num_vars: usize, + log_blowup: usize, + log_fold: usize, + kept: usize, + queries: usize, + seed: u64, + ) { + let mut rng = Rng(seed); + let values: Vec = (0..1usize << num_vars).map(|_| rng.next() % P).collect(); + let num_leaves = 1u64 << (num_vars + log_blowup - log_fold); + let leaves: Vec = (0..queries).map(|_| rng.next() % num_leaves).collect(); + let plan = CosetPlan::new(num_vars, log_blowup, log_fold, kept, &leaves).expect("plan"); + let word = codeword(&values, num_vars, log_blowup); + let got = plan.evaluate_on_host(&values); + let n = plan.coset_len(); + assert_eq!(got.len(), plan.classes().len() * n); + let emulated = emulate::kernels(&plan, &values); + assert_eq!( + emulated.iter().map(|&v| canonical(v)).collect::>(), + got.iter().map(|&v| canonical(v)).collect::>(), + "the kernels' emulation and the host algorithm" + ); + // The long kernels cut to 1, 2 or 7 blocks give the full grid's words. + for cap in [1, 2, 7] { + assert_eq!( + emulate::kernels_capped(&plan, &values, Some(cap)), + emulated, + "{cap} blocks a kernel" + ); + } + for (q, block) in got.chunks_exact(n).enumerate() { + for (r, leaf) in plan.block_leaves(q).into_iter().enumerate() { + for t in 0..1usize << log_fold { + let want = word[leaf as usize + t * num_leaves as usize]; + assert_eq!( + canonical(block[(r << log_fold) + t]), + want, + "n {num_vars} b {log_blowup} fold {log_fold} kept {kept}: block {q} leaf {leaf} value {t}" + ); + } + } + } + for &leaf in &leaves { + let q = plan.block_of(leaf).expect("every query has a block"); + assert!( + plan.block_leaves(q).contains(&leaf), + "a query's block holds its leaf" + ); + } + } + + /// `kernels/whir_coset.cu` transcribed: each kernel's blocks, threads and + /// warps run in turn (a phase between two barriers at a time), over the + /// shared-memory layout and the launch parameters [`evaluate_resident`] + /// passes. What the laptop can say about the device code without a card: + /// an index, a layout or a launch-size slip shows here as a wrong value + /// or an out-of-bounds panic. + // A transcription keeps the kernels' thread loops, indices and all. + #[allow(clippy::needless_range_loop)] + mod emulate { + use super::super::*; + + pub fn kernels(plan: &CosetPlan, values: &[u64]) -> Vec { + kernels_capped(plan, values, None) + } + + /// [`kernels`] with the long kernels' grids cut to `cap` blocks + /// (`device::capped_grid`), each striding over its work. + pub fn kernels_capped(plan: &CosetPlan, values: &[u64], cap: Option) -> Vec { + let len = plan.classes.len() * plan.coset_len(); + kernels_into(plan, values, 0, len, cap) + } + + /// The blocks a capped launch takes: `full`, or `cap` (at least one). + fn grid_of(full: usize, cap: Option) -> usize { + cap.map_or(full, |c| full.min(c.max(1))) + } + + /// [`kernels`] into the strided layout `coset_openings` hashes: + /// `leaves` gathered leaves (padded to a power of two), value `t` of + /// leaf `i` at `i + t·leaves`. + pub fn kernels_strided(plan: &CosetPlan, values: &[u64]) -> (Vec, usize) { + let leaves = (plan.classes.len() << plan.kept).next_power_of_two().max(2); + let block = 1usize << (plan.log_coset - plan.kept); + ( + kernels_into(plan, values, leaves as u64, leaves * block, None), + leaves, + ) + } + + fn kernels_into( + plan: &CosetPlan, + values: &[u64], + stride: u64, + out_len: usize, + cap: Option, + ) -> Vec { + let tiles = plan.tiles(); + let mut slab = vec![0u64; plan.slab_rows() * tiles]; + fold_tiles(plan, values, &mut slab, cap); + finish(plan, &slab, stride, out_len, cap) + } + + /// [`kernels`] over the stack read where it lies: `data` the store's + /// buffer, `parts` its parts. Returns the cosets and the tile map. + pub fn kernels_in_place( + plan: &CosetPlan, + parts: &StackParts, + data: &[u64], + cap: Option, + ) -> (Vec, Vec) { + let tiles = plan.tiles(); + let map = tile_map(parts, plan.tile_levels, tiles, cap); + let mut slab = vec![0u64; plan.slab_rows() * tiles]; + fold_tiles_parts(plan, data, &map, parts, &mut slab, cap); + let len = plan.classes.len() * plan.coset_len(); + (finish(plan, &slab, 0, len, cap), map) + } + + pub const ZERO: u32 = 0xffff_ffff; + pub const MIXED: u32 = 0xffff_fffe; + + fn parts_upto(off: &[u64], e: u64) -> usize { + let (mut lo, mut hi) = (0usize, off.len()); + while lo < hi { + let mid = (lo + hi) >> 1; + if off[mid] <= e { + lo = mid + 1; + } else { + hi = mid; + } + } + lo + } + + /// `coset_tile_map`, one thread a tile, the grid striding over them. + /// An entry no thread writes stays `UNWRITTEN`. + pub fn tile_map( + parts: &StackParts, + tile_bits: usize, + num_tiles: usize, + cap: Option, + ) -> Vec { + const UNWRITTEN: u32 = 0xdead_beef; + let threads = grid_of(num_tiles.div_ceil(BLOCK as usize), cap) * BLOCK as usize; + let mut map = vec![UNWRITTEN; num_tiles]; + for thread in 0..threads { + for tile in (thread..num_tiles).step_by(threads) { + let tile = tile as u64; + let start = tile << tile_bits; + let end = start + (1 << tile_bits); + let upto = parts_upto(&parts.off, start); + let mut kind = ZERO; + if upto > 0 { + let p = upto - 1; + let past = parts.off[p] + parts.len[p]; + if past >= end { + kind = p as u32; + } else if past > start { + kind = MIXED; + } + } + if kind == ZERO && upto < parts.off.len() && parts.off[upto] < end { + kind = MIXED; + } + map[tile as usize] = kind; + } + } + map + } + + /// `coset_fold_tiles_parts`, a block's tiles in turn as in + /// [`fold_tiles`]. + fn fold_tiles_parts( + plan: &CosetPlan, + data: &[u64], + map: &[u32], + parts: &StackParts, + slab: &mut [u64], + cap: Option, + ) { + const TILES: usize = TILES_PER_BLOCK; + let levels = plan.tile_levels; + let num_tiles = plan.tiles(); + let buf_len = plan.buf_len; + let last_count = plan.level_count[levels - 1] as usize; + let grid = grid_of(num_tiles.div_ceil(TILES), cap); + for block in 0..grid { + // A block's shared memory, kept across its passes. + let mut bufs = [vec![0u64; buf_len], vec![0u64; buf_len]]; + let mut fin = vec![0u64; last_count * TILES]; + let tile = 1usize << levels; + for first in (block * TILES..num_tiles).step_by(grid * TILES) { + for i in 0..TILES { + let at = first + i; + if at >= num_tiles { + break; + } + let kind = map[at]; + if kind == ZERO { + for k in 0..last_count { + fin[k * TILES + i] = 0; + } + continue; + } + let base = (at as u64) << levels; + if kind == MIXED { + for x in 0..tile { + let e = base + x as u64; + let upto = parts_upto(&parts.off, e); + let mut v = 0; + if upto > 0 { + let p = upto - 1; + if e < parts.off[p] + parts.len[p] { + v = data[(parts.at[p] + (e - parts.off[p])) as usize]; + } + } + bufs[0][x] = v; + } + } else { + let p = kind as usize; + let src = (parts.at[p] + (base - parts.off[p])) as usize; + bufs[0][..tile].copy_from_slice(&data[src..src + tile]); + } + let (mut inp, mut out) = (0usize, 1usize); + let mut width = tile; + for t in 0..levels { + let cnt = plan.level_count[t] as usize; + let off = plan.level_offset[t] as usize; + let log_half = levels - 1 - t; + let half = 1usize << log_half; + let outputs = cnt << log_half; + let last = t + 1 == levels; + let mut wrote = vec![0u64; buf_len]; + for o in 0..outputs { + let k = o >> log_half; + let j = o & (half - 1); + let row = plan.parents[off + k] as usize * width + 2 * j; + let r = + fold(bufs[inp][row], bufs[inp][row + 1], plan.gammas[off + k]); + if last { + fin[k * TILES + i] = r; + } else { + wrote[o] = r; + } + } + if !last { + bufs[out][..outputs].copy_from_slice(&wrote[..outputs]); + } + std::mem::swap(&mut inp, &mut out); + width = half; + } + } + for x in 0..last_count * TILES { + let k = x / TILES; + let at = first + x % TILES; + if at < num_tiles { + slab[k * num_tiles + at] = fin[x]; + } + } + } + } + } + + fn finish( + plan: &CosetPlan, + slab: &[u64], + stride: u64, + out_len: usize, + cap: Option, + ) -> Vec { + let cosets = plan.classes.len(); + let len = plan.coset_len(); + let mut g = vec![0u64; cosets * len]; + fold_rest(plan, slab, &mut g, cap); + let cl = plan.chunk_levels(); + let pairs = cosets * len / 2; + for level in cl..plan.log_coset { + mobius_level(&mut g, pairs, plan.log_coset, level); + } + chunk(plan, &mut g, cl); + for stage in cl + 1..=plan.log_coset { + ntt_stage(&mut g, pairs, plan.log_coset, stage, &plan.twiddles); + } + let mut out = vec![0u64; out_len]; + leaf_order(&g, plan.log_coset, plan.kept, stride, &mut out); + out + } + + fn fold_tiles(plan: &CosetPlan, f: &[u64], slab: &mut [u64], cap: Option) { + const TILES: usize = TILES_PER_BLOCK; + let levels = plan.tile_levels; + let num_tiles = plan.tiles(); + let total = plan.gammas.len(); + let buf_len = plan.buf_len; + let last_count = plan.level_count[levels - 1] as usize; + // The kernel's dynamic layout, in u64 words, and the bytes asked for. + let fin_at = total + 2 * buf_len; + let parent_at_bytes = 8 * (fin_at + last_count * TILES); + assert_eq!(parent_at_bytes + 4 * total, plan.tile_shared_bytes()); + let grid = grid_of(num_tiles.div_ceil(TILES), cap); + for block in 0..grid { + // A block's shared memory, kept across its passes. + let mut bufs = [vec![0u64; buf_len], vec![0u64; buf_len]]; + let mut fin = vec![0u64; last_count * TILES]; + let tile = 1usize << levels; + for first in (block * TILES..num_tiles).step_by(grid * TILES) { + for i in 0..TILES { + let at = first + i; + if at >= num_tiles { + break; + } + bufs[0][..tile].copy_from_slice(&f[at * tile..(at + 1) * tile]); + let (mut inp, mut out) = (0usize, 1usize); + let mut width = tile; + for t in 0..levels { + let cnt = plan.level_count[t] as usize; + let off = plan.level_offset[t] as usize; + let log_half = levels - 1 - t; + let half = 1usize << log_half; + let outputs = cnt << log_half; + let last = t + 1 == levels; + let mut wrote = vec![0u64; buf_len]; + for o in 0..outputs { + let k = o >> log_half; + let j = o & (half - 1); + let row = plan.parents[off + k] as usize * width + 2 * j; + let r = + fold(bufs[inp][row], bufs[inp][row + 1], plan.gammas[off + k]); + if last { + fin[k * TILES + i] = r; + } else { + wrote[o] = r; + } + } + if !last { + bufs[out][..outputs].copy_from_slice(&wrote[..outputs]); + } + std::mem::swap(&mut inp, &mut out); + width = half; + } + } + for x in 0..last_count * TILES { + let k = x / TILES; + let at = first + x % TILES; + if at < num_tiles { + slab[k * num_tiles + at] = fin[x]; + } + } + } + } + } + + fn fold_rest(plan: &CosetPlan, slab: &[u64], g: &mut [u64], cap: Option) { + let rest = plan.rest_levels; + let num_tiles = plan.tiles(); + let log_free = plan.log_coset; + let warps = plan.classes.len() << log_free; + let threads = grid_of((warps * 32).div_ceil(BLOCK as usize), cap) * BLOCK as usize; + let launched = threads / 32; + for first in 0..launched { + for warp in (first..warps).step_by(launched) { + let q = warp >> log_free; + let h = warp & ((1 << log_free) - 1); + let src = plan.slab_of[q] as usize * num_tiles + (h << rest); + let mut v: Vec = (0..32) + .map(|lane| { + if lane < 1 << rest { + slab[src + lane] + } else { + 0 + } + }) + .collect(); + for l in 0..rest { + let snap = v.clone(); + for lane in 0..32 { + let partner = if lane + (1 << l) < 32 { + snap[lane + (1 << l)] + } else { + snap[lane] + }; + if lane & ((2 << l) - 1) == 0 { + v[lane] = fold(snap[lane], partner, plan.rest_gammas[q * rest + l]); + } + } + } + g[warp] = v[0]; + } + } + } + + fn mobius_level(g: &mut [u64], pairs: usize, log_size: usize, level: usize) { + for idx in 0..pairs { + let q = idx >> (log_size - 1); + let r = idx & ((1 << (log_size - 1)) - 1); + let lo = ((r >> level) << (level + 1)) | (r & ((1 << level) - 1)); + let base = q << log_size; + g[base + lo + (1 << level)] = fsub(g[base + lo + (1 << level)], g[base + lo]); + } + } + + fn chunk(plan: &CosetPlan, g: &mut [u64], cl: usize) { + let log_size = plan.log_coset; + let half = 1usize << (cl - 1); + let chunks = 1usize << (log_size - cl); + for block in 0..plan.classes.len() << (log_size - cl) { + let q = block / chunks; + let at = (block % chunks) << cl; + let base = (q << log_size) + at; + let mut x = g[base..base + 2 * half].to_vec(); + for l in 0..cl { + for tid in 0..half { + let lo = ((tid >> l) << (l + 1)) | (tid & ((1 << l) - 1)); + x[lo + (1 << l)] = fsub(x[lo + (1 << l)], x[lo]); + } + } + let w = plan.twists[q]; + for tid in 0..half { + for e in [tid, tid + half] { + let i = reverse((at + e) as u64, log_size); + x[e] = fmul(x[e], fpow(w, i)); + } + } + for s in 1..=cl { + let h = 1usize << (s - 1); + for tid in 0..half { + let i0 = ((tid >> (s - 1)) << s) | (tid & (h - 1)); + let tw = plan.twiddles[(tid & (h - 1)) << (log_size - s)]; + let u = x[i0]; + let v = fmul(x[i0 + h], tw); + x[i0] = fadd(u, v); + x[i0 + h] = fsub(u, v); + } + } + g[base..base + 2 * half].copy_from_slice(&x); + } + } + + fn ntt_stage(g: &mut [u64], pairs: usize, log_size: usize, s: usize, twiddles: &[u64]) { + for idx in 0..pairs { + let q = idx >> (log_size - 1); + let r = idx & ((1 << (log_size - 1)) - 1); + let h = 1usize << (s - 1); + let j = r & (h - 1); + let i0 = ((r >> (s - 1)) << s) | j; + let base = q << log_size; + let u = g[base + i0]; + let v = fmul(g[base + i0 + h], twiddles[j << (log_size - s)]); + g[base + i0] = fadd(u, v); + g[base + i0 + h] = fsub(u, v); + } + } + + fn leaf_order(g: &[u64], log_size: usize, kept: usize, stride: u64, out: &mut [u64]) { + for (idx, &v) in g.iter().enumerate() { + let q = idx >> log_size; + let u = idx & ((1 << log_size) - 1); + let low = u & ((1 << kept) - 1); + let r = if kept == 0 { + 0 + } else { + reverse(low as u64, kept) as usize + }; + let t = u >> kept; + if stride == 0 { + out[(q << log_size) | (r << (log_size - kept)) | t] = v; + } else { + out[((q << kept) | r) + t * stride as usize] = v; + } + } + } + } + + /// The strided layout the openings hash: value `t` of gathered leaf + /// `q·2^kept + r` is the codeword at the leaf `slot_of` names, the + /// padding is zero, and each block's tree position is its class reversed. + #[test] + fn the_strided_layout_holds_each_slot_s_leaf() { + let (num_vars, log_blowup, log_fold, kept) = (14, 2, 2, 3); + let mut rng = Rng(21); + let values: Vec = (0..1usize << num_vars).map(|_| rng.next() % P).collect(); + let num_leaves = 1u64 << (num_vars + log_blowup - log_fold); + let leaves: Vec = (0..13).map(|_| rng.next() % num_leaves).collect(); + let plan = CosetPlan::new(num_vars, log_blowup, log_fold, kept, &leaves).expect("plan"); + let word = codeword(&values, num_vars, log_blowup); + let (strided, gathered) = emulate::kernels_strided(&plan, &values); + let block = 1usize << log_fold; + let real = plan.classes().len() << kept; + assert_eq!(gathered, real.next_power_of_two()); + let depth = (num_vars + log_blowup - log_fold) as u32; + for q in 0..plan.classes().len() { + for (r, leaf) in plan.block_leaves(q).into_iter().enumerate() { + assert_eq!(plan.slot_of(leaf), Some((q, r)), "leaf {leaf}"); + for t in 0..block { + assert_eq!( + canonical(strided[((q << kept) | r) + t * gathered]), + word[leaf as usize + t * num_leaves as usize] + ); + } + // Block q's node sits at the position its class reversed + // names: the leaf's tree position has those top bits. + let position = leaf.reverse_bits() >> (64 - depth); + assert_eq!(position as usize, (plan.block_position(q) << kept) | r); + } + } + for i in real..gathered { + for t in 0..block { + assert_eq!(strided[i + t * gathered], 0, "padding"); + } + } + } + + /// Tile levels only, the rest kernel copying. + #[test] + fn cosets_equal_the_codeword_with_tile_levels_only() { + the_host_plan_is_the_codeword(10, 2, 3, 4, 9, 1); + the_host_plan_is_the_codeword(12, 2, 2, 3, 20, 2); + } + + /// Tile levels and rest levels both, at blowups 2, 4 and 8. + #[test] + fn cosets_equal_the_codeword_with_rest_levels() { + the_host_plan_is_the_codeword(14, 2, 2, 2, 17, 3); + the_host_plan_is_the_codeword(15, 1, 3, 1, 11, 4); + the_host_plan_is_the_codeword(16, 3, 2, 1, 33, 5); + } + + /// The production geometry's coset (first fold 6, 256 leaves a kept node: + /// 2^14 points) over the smallest stack that still folds bits. + #[test] + fn cosets_equal_the_codeword_at_the_production_coset() { + the_host_plan_is_the_codeword(16, 2, 6, 8, 12, 6); + } + + /// The production plan's shape: 13 bits fold (8 tile, 5 rest), the + /// classes and the device footprint, without a polynomial. + #[test] + fn the_production_plan_folds_eight_tile_and_five_rest_bits() { + let mut rng = Rng(7); + let leaves: Vec = (0..114).map(|_| rng.next() % (1 << 23)).collect(); + let plan = CosetPlan::new(27, 2, 6, 8, &leaves).expect("plan"); + assert_eq!((plan.tile_levels, plan.rest_levels), (8, 5)); + assert_eq!(plan.coset_len(), 1 << 14); + assert!(plan.classes().len() <= 114); + // Level t lists at most the 2^(3+t) residues, and at most the queries. + for (t, &count) in plan.level_count.iter().enumerate() { + assert!( + count as usize <= (8usize << t).min(114), + "level {t}: {count}" + ); + } + assert!(plan.tile_shared_bytes() <= SHARED_LIMIT); + assert!(plan.buf_len <= 1024); + for &leaf in &leaves { + let q = plan.block_of(leaf).expect("block"); + assert!(plan.block_leaves(q).contains(&leaf)); + } + } + + /// A stack's parts as a store holds them: columns of the given heights + /// placed as `StackedLayout::build` does (each at a multiple of its + /// height, the cursor only growing), `short` values cut off every third + /// column's end (a column ending mid-tile), then laid in the buffer in + /// another order with junk between (a read past a part shows). Returns + /// `(buffer offset, elements, stack offset)` parts and the buffer. + fn store_layout( + num_vars: usize, + heights: &[usize], + short: usize, + seed: u64, + ) -> (Vec<(usize, usize, usize)>, Vec) { + let mut rng = Rng(seed); + let mut placed = Vec::new(); + let mut cursor = 0usize; + for (i, &h) in heights.iter().enumerate() { + let len = 1usize << h; + let off = cursor.next_multiple_of(len); + if off + len > 1 << num_vars { + break; + } + cursor = off + len; + let len = if i % 3 == 2 { + len - short.min(len - 1) + } else { + len + }; + placed.push((len, off)); + } + let mut order: Vec = (0..placed.len()).collect(); + for i in (1..order.len()).rev() { + order.swap(i, rng.next() as usize % (i + 1)); + } + let mut data = Vec::new(); + let mut parts = Vec::new(); + for &i in &order { + let (len, off) = placed[i]; + for _ in 0..rng.next() % 5 { + data.push(rng.next() % P); + } + parts.push((data.len(), len, off)); + data.extend((0..len).map(|_| rng.next() % P)); + } + data.push(rng.next() % P); + (parts, data) + } + + /// The stack `parts` make out of `data`, zeros between. + fn assembled(num_vars: usize, parts: &[(usize, usize, usize)], data: &[u64]) -> Vec { + let mut f = vec![0u64; 1 << num_vars]; + for &(at, len, off) in parts { + f[off..off + len].copy_from_slice(&data[at..at + len]); + } + f + } + + /// Each tile's kind by looking at every value of it. + fn kinds_by_hand(num_vars: usize, tile_bits: usize, parts: &StackParts) -> Vec { + let owner = |e: u64| { + (0..parts.off.len()).find(|&p| parts.off[p] <= e && e < parts.off[p] + parts.len[p]) + }; + (0..1u64 << (num_vars - tile_bits)) + .map(|tile| { + let owners: Vec> = (tile << tile_bits..(tile + 1) << tile_bits) + .map(owner) + .collect(); + if owners.iter().all(Option::is_none) { + emulate::ZERO + } else if owners.iter().all(|o| *o == owners[0]) { + owners[0].unwrap() as u32 + } else { + emulate::MIXED + } + }) + .collect() + } + + /// The stack read where it lies gives the assembled stack's words, raw, + /// and the tile map gives each tile the kind its values have, the counts + /// `tile_kinds` says without visiting a tile. Both paths give the same + /// words, and the same map, with their grids cut to 1, 2 or 7 blocks. + fn in_place_is_the_assembled_stack( + plan: &CosetPlan, + parts: &[(usize, usize, usize)], + data: &[u64], + what: &str, + ) -> TileKinds { + let n = plan.num_vars; + let placed = StackParts::new(parts, n, data.len()).expect("the parts place"); + let (got, map) = emulate::kernels_in_place(plan, &placed, data, None); + let whole = assembled(n, parts, data); + let reference = emulate::kernels(plan, &whole); + assert_eq!( + got, reference, + "{what}: in place and assembled, word for word" + ); + for cap in [1, 2, 7] { + let (capped, capped_map) = emulate::kernels_in_place(plan, &placed, data, Some(cap)); + assert_eq!(capped, got, "{what}: in place, {cap} blocks a kernel"); + assert_eq!(capped_map, map, "{what}: the map at {cap} blocks"); + assert_eq!( + emulate::kernels_capped(plan, &whole, Some(cap)), + reference, + "{what}: assembled, {cap} blocks a kernel" + ); + } + assert_eq!( + map, + kinds_by_hand(n, plan.tile_levels, &placed), + "{what}: the map" + ); + let kinds = placed.tile_kinds(n, plan.tile_levels); + let count = |want: &dyn Fn(u32) -> bool| map.iter().filter(|&&k| want(k)).count() as u64; + assert_eq!( + kinds, + TileKinds { + full: count(&|k| k < emulate::MIXED), + zero: count(&|k| k == emulate::ZERO), + mixed: count(&|k| k == emulate::MIXED), + }, + "{what}: the kinds counted from the parts" + ); + kinds + } + + /// The production tile (8 bits), with and without rest bits, at blowups 4 + /// and 2, and a 5-bit tile, over the layouts the stacks have: columns down + /// to a value, columns ending mid-tile, alignment gaps, a zero tail; one + /// column; no column. + #[test] + fn the_stack_read_in_place_is_the_assembled_stack() { + let mut rng = Rng(31); + for (num_vars, log_blowup, log_fold, kept) in [(12, 2, 2, 2), (16, 1, 2, 3), (11, 2, 3, 3)] + { + let num_leaves = 1u64 << (num_vars + log_blowup - log_fold); + let leaves: Vec = (0..19).map(|_| rng.next() % num_leaves).collect(); + let plan = CosetPlan::new(num_vars, log_blowup, log_fold, kept, &leaves).expect("plan"); + let tb = plan.tile_levels; + assert!(tb >= 3, "{num_vars}: tile bits {tb}"); + // Production-like: tall columns first, then shorter ones down to a + // value, the cursor stopping ≈ 15 % short of the stack's end. + let mut heights = vec![num_vars - 2, num_vars - 3]; + heights.extend((0..40).map(|i| (num_vars - 4).saturating_sub(i % (num_vars - 3)))); + let (mut parts, data) = store_layout(num_vars, &heights, 7, rng.next()); + let cap = (1usize << num_vars) * 85 / 100; + parts.retain(|&(_, _, off)| off < cap); + let kinds = in_place_is_the_assembled_stack(&plan, &parts, &data, "production-like"); + assert!( + kinds.full > 0 && kinds.zero > 0 && kinds.mixed > 0, + "{kinds:?}" + ); + // Shortened columns at unaligned ends, sub-tile columns only. + let small: Vec = (0..300).map(|i| [0, 1, 2, 5, 3][i % 5]).collect(); + let (parts, data) = store_layout(num_vars, &small, 1, rng.next()); + in_place_is_the_assembled_stack(&plan, &parts, &data, "sub-tile columns"); + // One column, the whole stack. + let (parts, data) = store_layout(num_vars, &[num_vars], 0, rng.next()); + let kinds = in_place_is_the_assembled_stack(&plan, &parts, &data, "one column"); + assert_eq!((kinds.zero, kinds.mixed), (0, 0)); + // Columns meeting inside a tile and at a tile's edge, and one a + // value short of its last tile's end, a tile of junk after it. + let t = 1usize << tb; + let parts = [ + (0, t + 3, 0), + (t + 3, t - 3, t + 3), + (2 * t, t, 2 * t), + (3 * t, 2 * t - 1, 4 * t), + ]; + let data: Vec = (0..6 * t).map(|_| rng.next() % P).collect(); + in_place_is_the_assembled_stack(&plan, &parts, &data, "columns meeting"); + // No column: every tile zero, every coset value zero. + let kinds = in_place_is_the_assembled_stack(&plan, &[], &[], "no column"); + assert_eq!((kinds.full, kinds.mixed), (0, 0)); + } + } + + /// The parts the in-place kernels cannot place are refused (the caller + /// assembles the stack); empty ones are dropped and the rest sorted. + #[test] + fn parts_the_in_place_kernels_cannot_place_are_refused() { + let n = 10; + assert!( + StackParts::new(&[(0, 8, 0), (8, 8, 4)], n, 64).is_none(), + "overlapping" + ); + assert!( + StackParts::new(&[(0, 8, 1020)], n, 64).is_none(), + "past the stack" + ); + assert!( + StackParts::new(&[(60, 8, 0)], n, 64).is_none(), + "past the buffer" + ); + assert!( + StackParts::new(&[(0, 8, 0), (8, 8, 8)], n, 16).is_some(), + "adjacent" + ); + let placed = + StackParts::new(&[(40, 8, 512), (0, 0, 4), (8, 4, 16)], n, 64).expect("placed"); + assert_eq!( + (placed.off, placed.len, placed.at), + (vec![16, 512], vec![4, 8], vec![8, 40]) + ); + } + + #[test] + fn shapes_the_kernels_do_not_take_are_refused() { + assert!( + CosetPlan::new(14, 2, 6, 8, &[0]).is_none(), + "no bit to fold" + ); + assert!( + CosetPlan::new(28, 2, 6, 8, &[0]).is_none(), + "14 bits to fold" + ); + assert!(CosetPlan::new(20, 2, 6, 8, &[]).is_none(), "no query"); + assert!( + CosetPlan::new(20, 2, 6, 8, &[1 << 16]).is_none(), + "leaf out of range" + ); + assert!(CosetPlan::new(20, 2, 6, 8, &[(1 << 16) - 1]).is_some()); + } +} diff --git a/crypto/math-cuda/src/whir_open.rs b/crypto/math-cuda/src/whir_open.rs new file mode 100644 index 000000000..d9780b119 --- /dev/null +++ b/crypto/math-cuda/src/whir_open.rs @@ -0,0 +1,2079 @@ +//! The opening's two factors, resident across the groups of rounds. +//! +//! A chained WHIR does not run one sumcheck: it runs a group of rounds, folds +//! its codeword, commits the successor, answers an out-of-domain point and +//! folds the next group's weight into the one it carries. The factors — the +//! message and the weight, each the width of a stacked polynomial — are the +//! same two tables throughout, so they stay here and the host drives them. +//! +//! The message arrives in the base field, which is where it is committed, and +//! is lifted on the way in: the sumcheck runs in one field. + +use std::sync::Arc; + +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, LaunchConfig, PushKernelArg}; + +use crate::Result; +use crate::device::backend; +use crate::sumcheck::SumcheckSession; + +/// The weight and the message, on device, with the cube they currently span. +pub struct OpeningSession { + stream: Arc, + weight: Arc>, + message: Arc>, + len: usize, +} + +impl OpeningSession { + /// `weight` is interleaved ext3; `message` is the base-field polynomial + /// being opened, lifted here. + pub fn new(weight: &[u64], message: &[u64]) -> Result { + assert!(weight.len().is_multiple_of(3), "three u64 per ext3 element"); + let len = weight.len() / 3; + let be = backend()?; + let stream = be.next_stream(); + let weight_dev = crate::device::htod_or_trim(&stream, weight)?; + Self::with_weight(stream, weight_dev, len, message) + } + + /// The same with the weight built here: each share is a column's subcube + /// offset, the point it is claimed at, and the scale it carries, and its + /// cells are `scale·eq(point, ·)`. The gaps stay zero. + /// + /// A stacked polynomial's weight is as wide as the polynomial — writing it + /// on the host and sending it costs more than the rounds that read it. + pub fn from_shares( + shares: &[(usize, Vec, [u64; 3])], + len: usize, + message: &[u64], + ) -> Result { + let be = backend()?; + let stream = be.next_stream(); + // Zeroed because the gaps between the shares' subcubes are part of the + // weight and nothing writes them. + let mut weight = crate::device::alloc_zeros_or_trim::(&stream, len * 3)?; + crate::sumcheck::eq_expand_shares_ext3(&stream, &mut weight, shares)?; + Self::with_weight(stream, weight, len, message) + } + + /// The same again with the message given as the columns it is stacked from, + /// so it is never assembled on the host — `(column, offset in elements)`. + pub fn from_shares_and_parts( + shares: &[(usize, Vec, [u64; 3])], + len: usize, + parts: &[(&[u64], usize)], + ) -> Result { + Self::from_shares_and_source(shares, len, |stream, base| { + for (column, offset) in parts { + let mut at = base.slice_mut(*offset..*offset + column.len()); + stream.memcpy_htod(*column, &mut at)?; + } + Ok(()) + }) + } + + /// The same for parts the card already holds: `(column index, offset)` into + /// the epoch's columns, so the message is assembled there instead of + /// crossing the bus a second time. + pub fn from_shares_and_resident( + shares: &[(usize, Vec, [u64; 3])], + len: usize, + store: &crate::columns::DeviceColumns, + parts: &[(usize, usize)], + ) -> Result { + Self::from_shares_and_source(shares, len, |stream, base| { + for (column, offset) in parts { + store.copy_into(*column, base, *offset, stream)?; + } + Ok(()) + }) + } + + fn from_shares_and_source( + shares: &[(usize, Vec, [u64; 3])], + len: usize, + write: impl FnOnce(&Arc, &mut CudaSlice) -> Result<()>, + ) -> Result { + let be = backend()?; + let stream = be.next_stream(); + let mut weight = crate::device::alloc_zeros_or_trim::(&stream, len * 3)?; + crate::sumcheck::eq_expand_shares_ext3(&stream, &mut weight, shares)?; + // Zeroed because what the parts do not cover is the stacking's padding. + let mut base = crate::device::alloc_zeros_or_trim::(&stream, len)?; + write(&stream, &mut base)?; + Self::lift_into(stream, weight, len, base) + } + + fn with_weight( + stream: Arc, + weight_dev: CudaSlice, + len: usize, + message: &[u64], + ) -> Result { + assert_eq!(message.len(), len, "the message spans the weight's cube"); + let base = crate::device::htod_or_trim(&stream, message)?; + Self::lift_into(stream, weight_dev, len, base) + } + + fn lift_into( + stream: Arc, + weight_dev: CudaSlice, + len: usize, + base: CudaSlice, + ) -> Result { + assert!(len.is_power_of_two(), "the cube is a power of two"); + + let be = backend()?; + // SAFETY: the kernel writes every element it is sized for. + let mut lifted = unsafe { stream.alloc::(len * 3) }?; + let count = len as u64; + unsafe { + stream + .launch_builder(&be.mle_lift_base_ext3) + .arg(&base) + .arg(&count) + .arg(&mut lifted) + .launch(LaunchConfig::for_num_elems(len as u32))?; + } + + Ok(Self { + stream, + weight: Arc::new(weight_dev), + message: Arc::new(lifted), + len, + }) + } + + /// A session over factors already on the card, both extension-valued and + /// `len` wide — what [`LeanRound0::materialize`] hands over once its + /// rounds have bound the first variables. + pub fn from_tables( + stream: Arc, + weight: CudaSlice, + message: CudaSlice, + len: usize, + ) -> Self { + assert!(len.is_power_of_two(), "the cube is a power of two"); + assert!( + weight.len() >= len * 3 && message.len() >= len * 3, + "the tables span the cube" + ); + Self { + stream, + weight: Arc::new(weight), + message: Arc::new(message), + len, + } + } + + /// What a session over `2^num_vars` built from shares holds — the weight + /// and the lifted message, three limbs each — and the base-field staging it + /// builds the message through, which is freed once the lift has run. + pub const fn planned_bytes(num_vars: usize) -> (u64, u64) { + (48u64 << num_vars, 8u64 << num_vars) + } + + /// Device bytes the two tables occupy — their allocations, which the rounds + /// fold in place and never shrink. + pub fn device_bytes(&self) -> u64 { + ((self.weight.len() + self.message.len()) * 8) as u64 + } + + /// The weight's and the message's live cube, `len` ext3 values each, read + /// back — for the parity tests against the lean opening. + pub fn tables_to_host(&self) -> Result<(Vec, Vec)> { + let weight = + crate::readback::clone_dtoh(&self.stream, &self.weight.slice(0..self.len * 3))?; + let message = + crate::readback::clone_dtoh(&self.stream, &self.message.slice(0..self.len * 3))?; + self.stream.synchronize()?; + Ok((weight, message)) + } + + /// Cube indices the factors still span. + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn num_vars(&self) -> usize { + self.len.trailing_zeros() as usize + } + + /// A sumcheck over `[weight, message]` as they stand. + /// + /// The rounds fold them in place, so the caller tells this session how far + /// they got with [`bound`](Self::bound). + pub fn sumcheck( + &self, + nodes: &[u64], + consts: &[u64], + num_slots: usize, + root_slot: u32, + ) -> Result { + let addresses = { + let (weight, _weight_guard) = self.weight.device_ptr(&self.stream); + let (message, _message_guard) = self.message.device_ptr(&self.stream); + [weight, message] + }; + SumcheckSession::from_device( + self.stream.clone(), + &addresses, + self.len, + vec![self.weight.clone(), self.message.clone()], + nodes, + consts, + num_slots, + root_slot, + ) + } + + /// Records that `rounds` variables have been bound by a sumcheck over + /// these factors. + pub fn bound(&mut self, rounds: usize) { + self.len >>= rounds; + } + + /// The message's value at `point`, which the out-of-domain answer needs. + /// + /// Folds a copy: the message is the next group's factor and the rounds + /// have not bound this point. + pub fn evaluate_message(&self, point: &[u64]) -> Result<[u64; 3]> { + assert_eq!(point.len(), self.num_vars() * 3, "the point spans the cube"); + let mut copy = self.stream.alloc_zeros::(self.len * 3)?; + { + let source = self.message.slice(0..self.len * 3); + self.stream.memcpy_dtod(&source, &mut copy)?; + } + crate::sumcheck::evaluate_resident_ext3(&self.stream, &mut copy, self.len, point) + } + + /// The successor a `levels`-level fold of `codeword` would make, encoded + /// from the message as these factors have bound it + /// ([`crate::whir::encode_successor`]). Reads a copy: the message stays the + /// next group's factor. + pub fn encode_successor( + &self, + codeword: &crate::whir::DeviceCodeword, + levels: usize, + ) -> Result { + crate::whir::encode_successor(codeword, levels, &self.stream, &self.message, self.len) + } + + /// ★ A grouped chain's index rounds' factors: `parts`' live tables laid + /// slot by slot, `2^index_vars` slots of `parts[0].len()`, the slots past + /// the parts zero — slot `p` at cube indices `[p·len, (p + 1)·len)`, so the + /// index is the cube's HIGH variables and a sumcheck over the result binds + /// it first, slot index `y_0` (its most significant bit) before `y_1`. + /// The layout a grouped chain's host index rounds read, so the same rounds + /// and the same tables at the end. + /// + /// Written into `parts[0]`'s own allocation when it has the room and no + /// one else holds it (a session's tables stay allocated at the width they + /// were written at, and the rounds that bound them since left `len` of + /// it live), and into a fresh pair otherwise. Every part is drained of + /// its queued work first, and the copies before any part is let go: each + /// part's tables are freed on its own stream. + pub fn gather_slots(mut parts: Vec, index_vars: usize) -> Result { + ensure!( + !parts.is_empty() && parts.len() <= 1usize << index_vars, + "at least one part, and no more than the slots" + ); + let len = parts[0].len; + ensure!( + parts.iter().all(|part| part.len == len), + "every slot spans the same cube" + ); + let wide = len << index_vars; + for part in &parts { + part.stream.synchronize()?; + } + let first = parts.remove(0); + let stream = first.stream.clone(); + // In place only when nothing else holds the tables: a sumcheck over + // them keeps a clone of each for as long as it lives. + let in_place = [&first.weight, &first.message] + .iter() + .all(|table| table.len() >= wide * 3 && Arc::strong_count(table) == 1); + let (mut weight, mut message) = match (in_place, first.weight, first.message) { + (true, weight, message) => match (Arc::try_unwrap(weight), Arc::try_unwrap(message)) { + (Ok(weight), Ok(message)) => (weight, message), + _ => return Err(crate::invalid_value()), + }, + (false, first_weight, first_message) => { + let mut weight = crate::device::alloc_zeros_or_trim::(&stream, wide * 3)?; + let mut message = crate::device::alloc_zeros_or_trim::(&stream, wide * 3)?; + stream.memcpy_dtod( + &first_weight.slice(0..len * 3), + &mut weight.slice_mut(0..len * 3), + )?; + stream.memcpy_dtod( + &first_message.slice(0..len * 3), + &mut message.slice_mut(0..len * 3), + )?; + // Part 0's tables are freed on its stream: not before the copy. + stream.synchronize()?; + (weight, message) + } + }; + for (slot, part) in parts.iter().enumerate() { + let at = (slot + 1) * len * 3; + stream.memcpy_dtod( + &part.weight.slice(0..len * 3), + &mut weight.slice_mut(at..at + len * 3), + )?; + stream.memcpy_dtod( + &part.message.slice(0..len * 3), + &mut message.slice_mut(at..at + len * 3), + )?; + } + // The slots no polynomial fills: zero in both factors, so they add + // nothing to any round and fold away under any challenge. In place, + // what the earlier rounds left there is overwritten. + let filled = (parts.len() + 1) * len * 3; + if filled < wide * 3 { + stream.memset_zeros(&mut weight.slice_mut(filled..wide * 3))?; + stream.memset_zeros(&mut message.slice_mut(filled..wide * 3))?; + } + // The other parts are freed on their own streams: not before the + // copies that read them have run. + stream.synchronize()?; + drop(parts); + Ok(Self { + stream, + weight: Arc::new(weight), + message: Arc::new(message), + len: wide, + }) + } + + /// [`Self::encode_successor`] of a codeword of `elements` values that is + /// not held, inside `room` (`crate::whir::encode_successor_sized`). + pub fn encode_successor_sized( + &self, + elements: usize, + room: &std::sync::Arc, + levels: usize, + ) -> Result { + crate::whir::encode_successor_sized( + elements, + room, + levels, + &self.stream, + &self.message, + self.len, + ) + } + + /// `weight += scale · eq(point, ·)`, the weight the next group carries. + pub fn add_scaled_eq(&self, point: &[u64], scale: &[u64]) -> Result<()> { + assert_eq!(point.len(), self.num_vars() * 3, "the point spans the cube"); + assert_eq!(scale.len(), 3, "an ext3 scale"); + let be = backend()?; + let eq = crate::sumcheck::eq_table_ext3(&self.stream, point, self.len)?; + let scale_dev = crate::device::htod_or_trim(&self.stream, scale)?; + let count = self.len as u64; + // The kernel writes through a shared reference, the way the tree's + // layers are folded: what orders these is the stream, and the weight + // has one owner. + unsafe { + self.stream + .launch_builder(&be.add_scaled_ext3) + .arg(self.weight.as_ref()) + .arg(&eq) + .arg(&count) + .arg(&scale_dev) + .launch(LaunchConfig::for_num_elems(self.len as u32))?; + } + Ok(()) + } +} + +/// Nodes a lean round evaluates at (`WHIR_LEAN_MAX_T` in `whir_fold.cu`): the +/// opening's product is degree two, so two. +const LEAN_MAX_T: usize = 4; +/// The partial-sum grid a lean round is capped at, as the sumcheck's rounds are. +const LEAN_MAX_GRID: u32 = 4096; +const LEAN_BLOCK: u32 = 256; +/// Share ids are `u16` on the card, and `0xFFFF` marks a position no column +/// covers. +const LEAN_MAX_SHARES: usize = 0xFFFF; + +/// Rounds a stacked opening runs over its shares before it materialises its +/// factors at `2^(n − rounds)`, when `LAMBDA_VM_WHIR_LEAN_ROUNDS` does not say +/// otherwise. Every setting proves the same values. +/// +/// 3, not the 6 WBATCH chose at the 2^25 stack as a memory lever: at today's +/// 2^27 stack each lean round is a pass over the whole stack, and three of them +/// plus the materialisation cost less than six. ULTRA 129 (D-SPEED §12): +/// median phase-B openings −2.74 s (t −141), whole −2.54 s, the top proof's +/// bytes unchanged at 6, 3 and 2, no refusals; the open peak +672 MiB (20.5 of +/// 25.1 GiB usable). `LAMBDA_VM_WHIR_LEAN_ROUNDS=6` is the way back. +pub const LEAN_ROUNDS_DEFAULT: usize = 3; + +/// `LAMBDA_VM_WHIR_LEAN_ROUNDS`'s value read: a count, clamped to the widest +/// fold ([`crate::whir::FUSED_MAX_FOLD`]); unset or not a number is +/// [`LEAN_ROUNDS_DEFAULT`]. +pub fn lean_rounds_setting(value: Option<&str>) -> usize { + value + .and_then(|v| v.trim().parse::().ok()) + .map(|v| v.min(crate::whir::FUSED_MAX_FOLD)) + .unwrap_or(LEAN_ROUNDS_DEFAULT) +} +/// The round scratch a lean opening keeps: the nodes, one partial per node +/// and block, and the sums. +const LEAN_SCRATCH_BYTES: u64 = + ((LEAN_MAX_T * 3 + LEAN_MAX_T * LEAN_MAX_GRID as usize * 3 + LEAN_MAX_T * 3) * 8) as u64; + +/// Slot tables a tile-form opening may hold (`WHIR_TILE_MAX_SLOTS` in +/// `whir_fold.cu`). +pub const TILE_MAX_SLOTS: usize = 8; + +/// ★ The lean rounds from ONE pass over the message: the tile form +/// (`whir_tile_*` in `whir_fold.cu`, where the identity is written out). +/// +/// The share form walks every stack position in every round. Here the stack is +/// cut into tiles of `2^tile_bits` positions — every share at least that long, +/// its point's `eq` table split there — so the weight inside a tile is one +/// number times a small table its class shares. One pass over the message +/// reduces each tile to a value per slot (`Φ`), one multiply a tile gives the +/// weight's (`Ω`), and every round then reads those tables, `2^tile_bits` +/// times smaller than the stack. The bound tables are written from `Ω` and +/// the classes' tables, the message through a `u16` a tile: the share form's +/// map of a `u16` a position is never built. The same field elements as the +/// share form's rounds and tables; the bound message raw-identical. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TileSpec { + /// A tile is `2^tile_bits` stack positions: every share's `lo_bits`. + pub tile_bits: usize, + /// The rounds the opening runs before it writes its tables. A column of + /// tiles is the `2^lean_rounds` tiles those rounds bind together. + pub lean_rounds: usize, + /// Slot tables held: at least the classes (distinct `lo_at`) any column + /// of tiles meets. + pub slots: usize, + /// The pass over the message as one block a tile (a warp reads consecutive + /// words) rather than one thread a tile. The same values. + pub wide: bool, + /// Every round also run in share form and compared + /// ([`tile_check_counts`]), and the bound tables written by the share + /// form and compared with the tile form's write + /// ([`tile_table_check_counts`]): a difference is counted, and the share + /// form's values are the ones sent and kept, so the proof is today's + /// either way. The share form's map is built for it, so a checking + /// opening holds the map and the tile form at once. + pub check: bool, +} + +impl TileSpec { + /// Device bytes the tile form holds through its rounds: a `u16` per tile, + /// each column's slots, `Ω` and `Φ` (three limbs a tile and slot each), + /// and a challenge. + pub const fn device_bytes(&self, num_vars: usize) -> u64 { + self.write_bytes(num_vars) + ((24 * self.slots as u64) << (num_vars - self.tile_bits)) + } + + /// What it keeps beside the bound tables while it writes them: all of + /// the above but `Φ`, which is freed first. + pub const fn write_bytes(&self, num_vars: usize) -> u64 { + let tiles = 1u64 << (num_vars - self.tile_bits); + let columns = tiles >> self.lean_rounds; + let slots = self.slots as u64; + 2 * tiles + 8 * slots * columns + 24 * slots * tiles + 24 + } + + /// Whether a stack of `2^num_vars` takes this form: the rounds bind tile + /// variables only, the slots are ones the kernels have room for, and the + /// opening never holds more than the share form does when it writes its + /// tables — the position map (a `u16` a position, which the tile form + /// does not allocate outside [`check`](Self::check)) beside the bound + /// tables (`48 · 2^(num_vars − lean_rounds)` bytes). Through the rounds + /// ALL the form holds is no larger than those tables, which are allocated + /// once `Φ` is freed; through the write what it keeps is no larger than + /// the map. + pub const fn fits(&self, num_vars: usize) -> bool { + self.tile_bits <= num_vars + && self.lean_rounds >= 1 + && self.lean_rounds <= num_vars - self.tile_bits + && self.slots >= 1 + && self.slots <= TILE_MAX_SLOTS + && self.device_bytes(num_vars) <= 48u64 << (num_vars - self.lean_rounds) + && self.write_bytes(num_vars) <= 2u64 << num_vars + } +} + +/// What `LAMBDA_VM_WHIR_LEAN_TILES` asks of a stacked opening's lean rounds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LeanTiles { + /// The share form: `whir_lean_round` over every position, every round + /// (`0`). + Off, + /// The tile form ([`TileSpec`]), one thread a tile (`1`). + On, + /// The tile form with the pass over the message one block a tile + /// (`wide`, and [`LEAN_TILES_DEFAULT`]). + Wide, + /// The tile form, every round also run in share form and compared; the + /// share form's values are the ones sent. + Check, +} + +/// A stacked opening's lean rounds when `LAMBDA_VM_WHIR_LEAN_TILES` is unset: +/// the tile form, its pass over the message one block a tile. The same values +/// as the share form. Against the share form on one card (a 5090 held at +/// 400 W, a 9950X), two runs of each: the whole median block −5.6 s and the +/// whole large block −13.0 s, the openings' sumcheck a quarter of its +/// seconds, an opening in tile form 0.23 GiB lighter on the card. At 2^27 a +/// block a tile was the faster pass where a column of tiles meets one class +/// (most chains), a thread a tile where it meets two or more. `=0` is the way +/// back. +pub const LEAN_TILES_DEFAULT: LeanTiles = LeanTiles::Wide; + +/// `LAMBDA_VM_WHIR_LEAN_TILES`'s value read: unset, [`LEAN_TILES_DEFAULT`]; +/// `1` the tile form a thread a tile, `wide` a block a tile, `check` the tile +/// form compared with the share form on every round; `0` or anything else is +/// the share form. +pub fn lean_tiles_setting(value: Option<&str>) -> LeanTiles { + match value.map(str::trim) { + None => LEAN_TILES_DEFAULT, + Some("1") => LeanTiles::On, + Some("wide") => LeanTiles::Wide, + Some("check") => LeanTiles::Check, + _ => LeanTiles::Off, + } +} + +static TILE_CHECKED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static TILE_MISMATCHED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Rounds a checking tile-form opening ([`TileSpec::check`]) compared with +/// the share form since the process started, and how many differed. A round +/// that differed still sent the share form's values. +pub fn tile_check_counts() -> (u64, u64) { + use std::sync::atomic::Ordering::Relaxed; + (TILE_CHECKED.load(Relaxed), TILE_MISMATCHED.load(Relaxed)) +} + +static LEAN_PEAKS: [std::sync::atomic::AtomicU64; 3] = [ + std::sync::atomic::AtomicU64::new(0), + std::sync::atomic::AtomicU64::new(0), + std::sync::atomic::AtomicU64::new(0), +]; + +/// ★ The most one lean opening held on the card at once since the process +/// started, by form — `[share, tile, checking]` — counted from its own +/// allocations: once it is built (its rounds allocate nothing that lasts) +/// and while its bound tables are written. What a block gate reads to show +/// the tile form holds no more than the share form. +pub fn lean_peak_bytes() -> [u64; 3] { + use std::sync::atomic::Ordering::Relaxed; + [ + LEAN_PEAKS[0].load(Relaxed), + LEAN_PEAKS[1].load(Relaxed), + LEAN_PEAKS[2].load(Relaxed), + ] +} + +static TILE_TABLES_CHECKED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static TILE_TABLES_DIFFERING: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Openings whose bound tables a checking opening ([`TileSpec::check`]) +/// compared with the tile form's write since the process started, and the +/// positions where either factor differed. The tables kept are the share +/// form's either way. +pub fn tile_table_check_counts() -> (u64, u64) { + use std::sync::atomic::Ordering::Relaxed; + ( + TILE_TABLES_CHECKED.load(Relaxed), + TILE_TABLES_DIFFERING.load(Relaxed), + ) +} + +static TILE_CHECK_FAULT: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Moves every tile-form round value a checking opening compares by one, and +/// one entry of the bound weight it compares the tile form's write with, +/// before the comparison and nowhere else — for the test that shows the +/// comparisons count a difference. What the opening sends and keeps is +/// unchanged. Not for production callers. +#[doc(hidden)] +pub fn force_tile_check_fault(on: bool) { + TILE_CHECK_FAULT.store(on, std::sync::atomic::Ordering::Relaxed); +} + +/// Whether `shares` are a stack the tile form reads under `spec`: the spec +/// fits, every share is split at the tile, is at least a tile long and sits at +/// a multiple of its own size, and the slots cover the classes a column of +/// tiles can meet. +fn tiles_read(shares: &[LeanShare], num_vars: usize, spec: &TileSpec) -> bool { + if !spec.fits(num_vars) { + return false; + } + let mut classes = std::collections::BTreeSet::new(); + for share in shares { + if share.lo_bits != spec.tile_bits + || share.num_vars < spec.tile_bits + || share.stack_offset & ((1usize << share.num_vars) - 1) != 0 + { + return false; + } + classes.insert(share.lo_at); + } + spec.slots >= classes.len().min(1usize << spec.lean_rounds) +} + +/// The tile form's device state. +struct Tiles { + spec: TileSpec, + tilemap: CudaSlice, + slot_lo: CudaSlice, + omega: CudaSlice, + /// `None` once the rounds are spent: the write does not read it. + phi: Option>, + r_dev: CudaSlice, + /// Challenges the slot tables are bound at. The rest of the opening's + /// are folded in before the next round reads them. + folded: usize, +} + +impl Tiles { + /// The tile map, each column's slots and `Ω`, and the one pass over the + /// message for `Φ`, queued on `stream`. + #[allow(clippy::too_many_arguments)] + fn build( + stream: &Arc, + spec: TileSpec, + num_vars: usize, + shares: &CudaSlice, + eq: &CudaSlice, + data: &CudaSlice, + starts: &[u64], + ends: &[u64], + ) -> Result { + let be = backend()?; + let nu = num_vars - spec.tile_bits; + let tiles = 1usize << nu; + let columns = tiles >> spec.lean_rounds; + let tile_starts: Vec = starts.iter().map(|s| s >> spec.tile_bits).collect(); + let tile_ends: Vec = ends.iter().map(|e| e >> spec.tile_bits).collect(); + let starts_dev = crate::device::htod_or_trim(stream, &tile_starts)?; + let ends_dev = crate::device::htod_or_trim(stream, &tile_ends)?; + // SAFETY: each kernel below writes every element of what it is handed. + let mut tilemap = unsafe { crate::device::alloc_or_trim::(stream, tiles) }?; + let mut slot_lo = + unsafe { crate::device::alloc_or_trim::(stream, columns * spec.slots) }?; + let mut omega = + unsafe { crate::device::alloc_or_trim::(stream, spec.slots * tiles * 3) }?; + let mut phi = + unsafe { crate::device::alloc_or_trim::(stream, spec.slots * tiles * 3) }?; + let r_dev = crate::device::alloc_zeros_or_trim::(stream, 3)?; + let grid_for = |work: usize| { + (work as u64) + .div_ceil(LEAN_BLOCK as u64) + .min(LEAN_MAX_GRID as u64) as u32 + }; + let count = starts.len() as u32; + let (nu_arg, lean_arg, u_arg, slots_arg) = ( + nu as u32, + spec.lean_rounds as u32, + spec.tile_bits as u32, + spec.slots as u32, + ); + unsafe { + stream + .launch_builder(&be.whir_lean_colmap) + .arg(&mut tilemap) + .arg(&(tiles as u64)) + .arg(&starts_dev) + .arg(&ends_dev) + .arg(&count) + .launch(LaunchConfig { + grid_dim: (grid_for(tiles).max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + stream + .launch_builder(&be.whir_tile_plan) + .arg(&tilemap) + .arg(shares) + .arg(eq) + .arg(&nu_arg) + .arg(&lean_arg) + .arg(&u_arg) + .arg(&slots_arg) + .arg(&mut slot_lo) + .arg(&mut omega) + .launch(LaunchConfig { + grid_dim: (grid_for(columns).max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + if spec.wide { + // One block a tile at a time, its threads the tile's cells in + // turn: never more threads than cells. + let block = LEAN_BLOCK.min(1u32 << spec.tile_bits.min(31)); + stream + .launch_builder(&be.whir_tile_contract_wide) + .arg(&tilemap) + .arg(shares) + .arg(eq) + .arg(data) + .arg(&slot_lo) + .arg(&nu_arg) + .arg(&lean_arg) + .arg(&u_arg) + .arg(&slots_arg) + .arg(&mut phi) + .launch(LaunchConfig { + grid_dim: ((tiles as u64).min(LEAN_MAX_GRID as u64) as u32, 1, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: block * 3 * 8, + })?; + } else { + stream + .launch_builder(&be.whir_tile_contract) + .arg(&tilemap) + .arg(shares) + .arg(eq) + .arg(data) + .arg(&slot_lo) + .arg(&nu_arg) + .arg(&lean_arg) + .arg(&u_arg) + .arg(&slots_arg) + .arg(&mut phi) + .launch(LaunchConfig { + grid_dim: (grid_for(tiles).max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + } + Ok(Self { + spec, + tilemap, + slot_lo, + omega, + phi: Some(phi), + r_dev, + folded: 0, + }) + } + + fn device_bytes(&self) -> u64 { + let phi = self.phi.as_ref().map_or(0, CudaSlice::len); + (self.tilemap.len() * 2 + + (self.slot_lo.len() + self.omega.len() + phi + self.r_dev.len()) * 8) as u64 + } +} + +/// One column of a stacked polynomial as the lean rounds read it: where it sits +/// in the stack, how many variables it spans, its weight's scale, and where its +/// point's `eq` table sits in the shared buffer, in two halves. +/// +/// `eq(z, row) = hi[row >> lo_bits] · lo[row & (2^lo_bits − 1)]`: the high half +/// is the table of the point's first `num_vars − lo_bits` coordinates, the low +/// half of the rest, each indexed with its most significant bit on its first +/// coordinate, as every `eq` table here is. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LeanShare { + /// The first stack index the column occupies. + pub stack_offset: usize, + /// Its variables: it spans `2^num_vars` stack indices. + pub num_vars: usize, + /// Variables in the low half of its point's `eq` table. + pub lo_bits: usize, + /// Where the high and the low half start in the `eq` buffer, in ext3 + /// elements. + pub hi_at: usize, + pub lo_at: usize, + /// The weight's scale, as ext3 limbs. + pub scale: [u64; 3], +} + +/// Where a lean opening reads the message. +pub enum LeanMessage<'a> { + /// In the epoch's resident columns: `(store column, stack offset)` per + /// part. Nothing is copied. + Resident { + store: &'a crate::columns::DeviceColumns, + parts: &'a [(usize, usize)], + }, + /// Host columns at their stack offsets, staged once into a base-field + /// buffer the width of the stack — what the materialised path uploads too, + /// before lifting it. + Parts(&'a [(&'a [u64], usize)]), +} + +/// A WHIR chain's first rounds over its factors in SHARE form +/// (`whir_lean_*` in `whir_fold.cu`). +/// +/// The materialised opening ([`OpeningSession`]) holds the weight and the +/// lifted message at the stack's full width — `2 · 2^n` extension values, 1.5× +/// the committed base codeword, plus the base staging while it is built — for +/// rounds that bind the top variables first. Here the weight stays a list of +/// shares and small `eq` tables, the message is read where the columns lie, a +/// `u16` per stack index says which column holds it, and each round's +/// evaluations are computed from those with the challenges bound so far. The +/// factors exist only once [`materialize`](Self::materialize) writes them at +/// `2^(n − bound)`. +/// +/// ★ The evaluations and the bound tables EQUAL the materialised path's +/// (`tests/whir_lean.rs`): the same field values by a different sequence of +/// operations, so raw limbs agree whenever both are canonical — the parity the +/// device path already has with the host. +pub struct LeanRound0 { + stream: Arc, + /// The stack's variables. + num_vars: usize, + /// The share holding each position: `None` for an opening in tile form + /// that does not check itself, which never reads it. + colmap: Option>, + shares: CudaSlice, + eq: CudaSlice, + data: Arc>, + /// Whether `data` is a staging buffer this opening allocated. + staged: bool, + /// The challenges bound so far, three limbs each. + bound: Vec, + t_dev: CudaSlice, + t_host: Vec, + partials: CudaSlice, + sums: CudaSlice, + /// The rounds' tables in tile form ([`TileSpec`]), when the opening was + /// built with one; `None` is the share form. + tiles: Option, +} + +impl LeanRound0 { + /// `Ok(None)` when the shares do not describe a stack this can read — out + /// of order, overlapping, past the end, more than a `u16` can name, or a + /// part missing for one of them — and the caller materialises instead. + pub fn new( + shares: &[LeanShare], + eq: &[u64], + num_vars: usize, + message: LeanMessage<'_>, + ) -> Result> { + Self::with_tiles(shares, eq, num_vars, message, None) + } + + /// [`new`](Self::new), with the rounds in tile form when `tiles` is given + /// ([`TileSpec`]). `Ok(None)` too when the shares are not a stack the + /// tile form reads under that spec — a share shorter than a tile, split + /// elsewhere or off its own alignment, more classes than slots, a form + /// larger than the tables written after it — and the caller opens in + /// share form instead. + pub fn with_tiles( + shares: &[LeanShare], + eq: &[u64], + num_vars: usize, + message: LeanMessage<'_>, + tiles: Option, + ) -> Result> { + let len = 1usize << num_vars; + if shares.is_empty() || shares.len() >= LEAN_MAX_SHARES || eq.is_empty() { + return Ok(None); + } + if let Some(spec) = &tiles + && !tiles_read(shares, num_vars, spec) + { + return Ok(None); + } + let mut end = 0usize; + for share in shares { + let span = 1usize << share.num_vars; + if share.stack_offset < end + || share.stack_offset + span > len + || share.lo_bits > share.num_vars + || (share.hi_at + (span >> share.lo_bits)) * 3 > eq.len() + || (share.lo_at + (1usize << share.lo_bits)) * 3 > eq.len() + { + return Ok(None); + } + end = share.stack_offset + span; + } + + let be = backend()?; + let stream = be.next_stream(); + // Each share's data offset: where its column starts in `data`. + let (data, staged, data_offsets): (Arc>, bool, Vec) = match message { + LeanMessage::Resident { store, parts } => { + let mut offsets = Vec::with_capacity(shares.len()); + for share in shares { + let Some(&(column, _)) = parts + .iter() + .find(|(_, offset)| *offset == share.stack_offset) + else { + return Ok(None); + }; + let (at, rows) = store.span(column); + if rows != 1usize << share.num_vars { + return Ok(None); + } + offsets.push(at); + } + (store.buffer(), false, offsets) + } + LeanMessage::Parts(parts) => { + // Zeroed: what the parts do not cover is the stacking's padding. + let mut base = crate::device::alloc_zeros_or_trim::(&stream, len)?; + for (column, offset) in parts { + if offset + column.len() > len { + return Ok(None); + } + let mut at = base.slice_mut(*offset..*offset + column.len()); + stream.memcpy_htod(*column, &mut at)?; + } + let offsets = shares.iter().map(|s| s.stack_offset).collect(); + (Arc::new(base), true, offsets) + } + }; + + let mut rows = Vec::with_capacity(shares.len() * 9); + let mut starts = Vec::with_capacity(shares.len()); + let mut ends = Vec::with_capacity(shares.len()); + for (share, data_at) in shares.iter().zip(&data_offsets) { + rows.extend_from_slice(&[ + share.stack_offset as u64, + *data_at as u64, + share.lo_bits as u64, + share.hi_at as u64, + share.lo_at as u64, + share.scale[0], + share.scale[1], + share.scale[2], + 0, + ]); + starts.push(share.stack_offset as u64); + ends.push((share.stack_offset + (1usize << share.num_vars)) as u64); + } + let shares_dev = crate::device::htod_or_trim(&stream, &rows)?; + let eq_dev = crate::device::htod_or_trim(&stream, eq)?; + // The share form's map, a `u16` a position: what its rounds and its + // write read, and a checking tile form's. The tile form reads a `u16` + // a tile instead. + let colmap = if tiles.is_none_or(|spec| spec.check) { + let starts_dev = crate::device::htod_or_trim(&stream, &starts)?; + let ends_dev = crate::device::htod_or_trim(&stream, &ends)?; + // SAFETY: the kernel writes every position. + let mut colmap = unsafe { crate::device::alloc_or_trim::(&stream, len) }?; + let count = shares.len() as u32; + let len_arg = len as u64; + let grid = (len as u64) + .div_ceil(LEAN_BLOCK as u64) + .min(LEAN_MAX_GRID as u64) as u32; + unsafe { + stream + .launch_builder(&be.whir_lean_colmap) + .arg(&mut colmap) + .arg(&len_arg) + .arg(&starts_dev) + .arg(&ends_dev) + .arg(&count) + .launch(LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + Some(colmap) + } else { + None + }; + let t_dev = crate::device::alloc_zeros_or_trim::(&stream, LEAN_MAX_T * 3)?; + let partials = crate::device::alloc_zeros_or_trim::( + &stream, + LEAN_MAX_T * LEAN_MAX_GRID as usize * 3, + )?; + let sums = crate::device::alloc_zeros_or_trim::(&stream, LEAN_MAX_T * 3)?; + let tiles = match tiles { + Some(spec) => Some(Tiles::build( + &stream, + spec, + num_vars, + &shares_dev, + &eq_dev, + data.as_ref(), + &starts, + &ends, + )?), + None => None, + }; + let lean = Self { + stream, + num_vars, + colmap, + shares: shares_dev, + eq: eq_dev, + data, + staged, + bound: Vec::new(), + t_dev, + t_host: Vec::new(), + partials, + sums, + tiles, + }; + lean.note_peak(lean.device_bytes()); + Ok(Some(lean)) + } + + /// Adds what this opening holds right now to [`lean_peak_bytes`], under + /// its form. + fn note_peak(&self, bytes: u64) { + let form = match &self.tiles { + None => 0, + Some(tiles) if !tiles.spec.check => 1, + Some(_) => 2, + }; + LEAN_PEAKS[form].fetch_max(bytes, std::sync::atomic::Ordering::Relaxed); + } + + /// Whether the rounds run in tile form. + pub fn is_tiled(&self) -> bool { + self.tiles.is_some() + } + + /// Waits for everything queued on this opening's stream — for a test or a + /// measurement that times its steps one by one. + pub fn synchronize(&self) -> Result<()> { + self.stream.synchronize() + } + + /// Variables still free. + pub fn num_vars(&self) -> usize { + self.num_vars - self.bound.len() / 3 + } + + /// Variables bound so far. + pub fn bound(&self) -> usize { + self.bound.len() / 3 + } + + /// What a lean opening of `2^num_vars` holds with `shares` columns and + /// `eq_values` extension values of `eq` halves: a `u16` per position, nine + /// u64 per share, the halves, the round scratch — and the message staged + /// in the base field when it is not resident. + pub const fn planned_bytes( + num_vars: usize, + shares: usize, + eq_values: usize, + staged: bool, + ) -> u64 { + let staging = if staged { 8u64 << num_vars } else { 0 }; + (2u64 << num_vars) + + 72 * shares as u64 + + 24 * eq_values as u64 + + LEAN_SCRATCH_BYTES + + staging + } + + /// Device bytes this opening holds: the share map when it has one, the + /// shares, the `eq` halves, the round scratch, the staged message when it + /// staged one, and the tile form's tables. + pub fn device_bytes(&self) -> u64 { + let staged = if self.staged { self.data.len() * 8 } else { 0 }; + (self.colmap.as_ref().map_or(0, CudaSlice::len) * 2 + + (self.shares.len() + self.eq.len() + self.partials.len() + self.sums.len()) * 8 + + self.t_dev.len() * 8 + + staged) as u64 + + self.tiles.as_ref().map_or(0, Tiles::device_bytes) + } + + /// `eq(bound challenges, ·)` over the `2^bound` prefixes, on the card — + /// `[1]` before any is bound. + fn prefix_weights(&self) -> Result> { + if self.bound.is_empty() { + return crate::device::htod_or_trim(&self.stream, &[1u64, 0, 0]); + } + crate::sumcheck::eq_table_ext3(&self.stream, &self.bound, 1usize << self.bound()) + } + + /// The next round's evaluations at the nodes `t` (ext3, three u64 each), + /// as a [`SumcheckSession::round`] over the materialised factors would + /// return them. + pub fn round(&mut self, t: &[u64]) -> Result> { + assert!(t.len().is_multiple_of(3), "three u64 per ext3 node"); + let num_t = t.len() / 3; + assert!(num_t > 0 && num_t <= LEAN_MAX_T, "nodes per round"); + assert!(self.num_vars() >= 1, "a round needs a variable to bind"); + match self.tiles.as_ref().map(|tiles| tiles.spec.check) { + None => self.share_round(t), + Some(false) => self.tile_round(t), + Some(true) => { + use std::sync::atomic::Ordering::Relaxed; + let mut tiled = self.tile_round(t)?; + let shared = self.share_round(t)?; + TILE_CHECKED.fetch_add(1, Relaxed); + if TILE_CHECK_FAULT.load(Relaxed) { + tiled[0] = tiled[0].wrapping_add(1) % 0xFFFF_FFFF_0000_0001; + } + let canonical = |limbs: &[u64]| -> Vec { + const P: u64 = 0xFFFF_FFFF_0000_0001; + limbs + .iter() + .map(|&v| if v >= P { v - P } else { v }) + .collect() + }; + if canonical(&tiled) != canonical(&shared) { + // Every one counted, the first few printed. + if TILE_MISMATCHED.fetch_add(1, Relaxed) < 4 { + eprintln!( + "[gpu] WHIR lean tiles: round {} of a 2^{} stack differs from the \ + share form: {tiled:?} against {shared:?}", + self.bound() + 1, + self.num_vars + ); + } + } + Ok(shared) + } + } + } + + /// The nodes on the card, sent when they change: they are the same every + /// round of an opening. + fn send_nodes(&mut self, t: &[u64]) -> Result<()> { + if self.t_host != t { + let mut head = self.t_dev.slice_mut(0..t.len()); + self.stream.memcpy_htod(t, &mut head)?; + self.t_host.clear(); + self.t_host.extend_from_slice(t); + } + Ok(()) + } + + /// Binds the slot tables at the challenges they have not been bound at + /// yet, each one fold over what the one before left. + fn fold_tiles(&mut self) -> Result<()> { + let Some(tiles) = self.tiles.as_mut() else { + return Ok(()); + }; + let be = backend()?; + let stride = 1u64 << (self.num_vars - tiles.spec.tile_bits); + let slots = tiles.spec.slots as u32; + let Tiles { + omega, + phi, + r_dev, + folded, + .. + } = tiles; + let phi = phi + .as_mut() + .expect("Φ is held until the tables are written"); + while *folded < self.bound.len() / 3 { + let at = *folded * 3; + self.stream.memcpy_htod(&self.bound[at..at + 3], r_dev)?; + let half = (stride >> *folded) / 2; + let tasks = 2 * slots as u64 * half; + let grid = tasks.div_ceil(LEAN_BLOCK as u64).min(LEAN_MAX_GRID as u64) as u32; + unsafe { + self.stream + .launch_builder(&be.whir_tile_fold) + .arg(&mut *omega) + .arg(&mut *phi) + .arg(&stride) + .arg(&slots) + .arg(&half) + .arg(&*r_dev) + .launch(LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: 0, + })?; + } + *folded += 1; + } + Ok(()) + } + + /// [`round`](Self::round) over the slot tables: no stack position is read. + fn tile_round(&mut self, t: &[u64]) -> Result> { + let num_t = t.len() / 3; + let be = backend()?; + self.fold_tiles()?; + self.send_nodes(t)?; + let tiles = self.tiles.as_ref().expect("a tile round runs in tile form"); + // Past its lean rounds a pair would join two columns of tiles, whose + // slots name different classes. + assert!( + tiles.folded < tiles.spec.lean_rounds, + "a tile round binds one of the lean rounds' variables" + ); + let stride = 1u64 << (self.num_vars - tiles.spec.tile_bits); + let half = (stride >> tiles.folded) / 2; + let grid = half.div_ceil(LEAN_BLOCK as u64).min(LEAN_MAX_GRID as u64) as u32; + let slots = tiles.spec.slots as u32; + let num_t_arg = num_t as u32; + unsafe { + self.stream + .launch_builder(&be.whir_tile_round) + .arg(&tiles.omega) + .arg( + tiles + .phi + .as_ref() + .expect("Φ is held until the tables are written"), + ) + .arg(&stride) + .arg(&slots) + .arg(&half) + .arg(&self.t_dev) + .arg(&num_t_arg) + .arg(&mut self.partials) + .launch(LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: LEAN_BLOCK * 3 * 8, + })?; + self.stream + .launch_builder(&be.sum_partials_ext3) + .arg(&self.partials) + .arg(&(grid.max(1) as u64)) + .arg(&mut self.sums) + .launch(LaunchConfig { + grid_dim: (num_t as u32, 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: LEAN_BLOCK * 3 * 8, + })?; + } + let sums = crate::readback::clone_dtoh(&self.stream, &self.sums.slice(0..num_t * 3))?; + self.stream.synchronize()?; + Ok(sums) + } + + /// [`round`](Self::round) in share form: every stack position, read where + /// the columns lie. + fn share_round(&mut self, t: &[u64]) -> Result> { + let num_t = t.len() / 3; + let be = backend()?; + let round = self.bound() + 1; + let eqc = self.prefix_weights()?; + self.send_nodes(t)?; + let rests = 1u64 << (self.num_vars - round); + let grid = rests.div_ceil(LEAN_BLOCK as u64).min(LEAN_MAX_GRID as u64) as u32; + let n_arg = self.num_vars as u32; + let s_arg = round as u32; + let num_t_arg = num_t as u32; + let colmap = self + .colmap + .as_ref() + .expect("the share form runs where its map is built"); + unsafe { + self.stream + .launch_builder(&be.whir_lean_round) + .arg(colmap) + .arg(&self.shares) + .arg(&self.eq) + .arg(self.data.as_ref()) + .arg(&n_arg) + .arg(&s_arg) + .arg(&eqc) + .arg(&self.t_dev) + .arg(&num_t_arg) + .arg(&mut self.partials) + .launch(LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: LEAN_BLOCK * 3 * 8, + })?; + self.stream + .launch_builder(&be.sum_partials_ext3) + .arg(&self.partials) + .arg(&(grid.max(1) as u64)) + .arg(&mut self.sums) + .launch(LaunchConfig { + grid_dim: (num_t as u32, 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: LEAN_BLOCK * 3 * 8, + })?; + } + let sums = crate::readback::clone_dtoh(&self.stream, &self.sums.slice(0..num_t * 3))?; + self.stream.synchronize()?; + Ok(sums) + } + + /// Binds the round's variable to `r` (ext3, three u64). Nothing moves on + /// the card: the next round reads the shares with one more prefix bound. + pub fn bind(&mut self, r: &[u64]) { + assert_eq!(r.len(), 3, "an ext3 challenge"); + assert!(self.num_vars() >= 1, "a bind needs a variable"); + self.bound.extend_from_slice(r); + } + + /// The factors at the challenges bound so far, written out at + /// `2^(n − bound)`: the session the rest of the chain runs on. + /// + /// In tile form they are written from the slot tables + /// (`whir_tile_materialize`): `Ω` bound at every challenge so far, and + /// `Φ` freed before the tables are allocated ([`TileSpec::fits`]). A + /// checking opening writes the share form's tables, keeps them, and + /// compares the tile form's write with them + /// ([`tile_table_check_counts`]). + pub fn materialize(mut self) -> Result { + let be = backend()?; + self.fold_tiles()?; + if let Some(tiles) = self.tiles.as_mut() { + tiles.phi = None; + } + let bound = self.bound(); + let len = 1usize << (self.num_vars - bound); + let eqfull = self.prefix_weights()?; + // SAFETY: the kernel writes every element of both. + let mut weight = unsafe { crate::device::alloc_or_trim::(&self.stream, len * 3) }?; + let mut message = unsafe { crate::device::alloc_or_trim::(&self.stream, len * 3) }?; + self.note_peak( + self.device_bytes() + ((weight.len() + message.len() + eqfull.len()) * 8) as u64, + ); + let n_arg = self.num_vars as u32; + let bound_arg = bound as u32; + let grid = (len as u64) + .div_ceil(LEAN_BLOCK as u64) + .min(LEAN_MAX_GRID as u64) as u32; + let config = LaunchConfig { + grid_dim: (grid.max(1), 1, 1), + block_dim: (LEAN_BLOCK, 1, 1), + shared_mem_bytes: 0, + }; + match &self.tiles { + Some(tiles) if !tiles.spec.check => { + let (lean_arg, u_arg, slots_arg) = ( + tiles.spec.lean_rounds as u32, + tiles.spec.tile_bits as u32, + tiles.spec.slots as u32, + ); + unsafe { + self.stream + .launch_builder(&be.whir_tile_materialize) + .arg(&tiles.tilemap) + .arg(&self.shares) + .arg(&self.eq) + .arg(self.data.as_ref()) + .arg(&tiles.slot_lo) + .arg(&tiles.omega) + .arg(&n_arg) + .arg(&lean_arg) + .arg(&u_arg) + .arg(&slots_arg) + .arg(&bound_arg) + .arg(&eqfull) + .arg(&mut weight) + .arg(&mut message) + .launch(config)?; + } + } + tiles => { + let colmap = self + .colmap + .as_ref() + .expect("the share form writes where its map is built"); + unsafe { + self.stream + .launch_builder(&be.whir_lean_materialize) + .arg(colmap) + .arg(&self.shares) + .arg(&self.eq) + .arg(self.data.as_ref()) + .arg(&n_arg) + .arg(&bound_arg) + .arg(&eqfull) + .arg(&mut weight) + .arg(&mut message) + .launch(config)?; + } + if let Some(tiles) = tiles { + self.check_write(tiles, &eqfull, &mut weight, &message, config)?; + } + } + } + Ok(OpeningSession::from_tables( + self.stream.clone(), + weight, + message, + len, + )) + } + + /// A checking opening's comparison of the tile form's write with the + /// share form's tables just written: the positions where either factor + /// differs, counted on the card and added to + /// [`tile_table_check_counts`]. The tables are left as written. + fn check_write( + &self, + tiles: &Tiles, + eqfull: &CudaSlice, + weight: &mut CudaSlice, + message: &CudaSlice, + config: LaunchConfig, + ) -> Result<()> { + use std::sync::atomic::Ordering::Relaxed; + let be = backend()?; + // The fault moves the weight's first entry for the comparison only. + let kept = if TILE_CHECK_FAULT.load(Relaxed) { + let kept = crate::readback::clone_dtoh(&self.stream, &weight.slice(0..3))?; + self.stream.synchronize()?; + let mut moved = kept.clone(); + moved[0] = moved[0].wrapping_add(1) % 0xFFFF_FFFF_0000_0001; + self.stream + .memcpy_htod(&moved, &mut weight.slice_mut(0..3))?; + self.stream.synchronize()?; + Some(kept) + } else { + None + }; + let mut differing = crate::device::alloc_zeros_or_trim::(&self.stream, 1)?; + let (n_arg, lean_arg, u_arg, slots_arg, bound_arg) = ( + self.num_vars as u32, + tiles.spec.lean_rounds as u32, + tiles.spec.tile_bits as u32, + tiles.spec.slots as u32, + self.bound() as u32, + ); + unsafe { + self.stream + .launch_builder(&be.whir_tile_materialize_check) + .arg(&tiles.tilemap) + .arg(&self.shares) + .arg(&self.eq) + .arg(self.data.as_ref()) + .arg(&tiles.slot_lo) + .arg(&tiles.omega) + .arg(&n_arg) + .arg(&lean_arg) + .arg(&u_arg) + .arg(&slots_arg) + .arg(&bound_arg) + .arg(eqfull) + .arg(&*weight) + .arg(message) + .arg(&mut differing) + .launch(config)?; + } + let count = crate::readback::clone_dtoh(&self.stream, &differing)?; + self.stream.synchronize()?; + if let Some(kept) = kept { + self.stream + .memcpy_htod(&kept, &mut weight.slice_mut(0..3))?; + self.stream.synchronize()?; + } + TILE_TABLES_CHECKED.fetch_add(1, Relaxed); + if count[0] > 0 && TILE_TABLES_DIFFERING.fetch_add(count[0], Relaxed) == 0 { + eprintln!( + "[gpu] WHIR lean tiles: the tables written at {} bound of a 2^{} stack differ from \ + the share form's at {} positions", + self.bound(), + self.num_vars, + count[0] + ); + } + Ok(()) + } +} + +/// What one chain of a stacked opening allocates on the card depends on: the +/// stack, the chain's first two folds, which kernels run it, and what the lean +/// rounds read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OpeningShape { + /// The stacked polynomial's variables. + pub num_vars: usize, + pub log_blowup: usize, + /// The chain's first and second fold widths; `next_fold` is 0 for a chain + /// of one round. + pub first_fold: usize, + pub next_fold: usize, + /// Rounds run over the shares before the factors are written; 0 writes + /// them at full width up front. + pub lean_rounds: usize, + /// Whether a fold runs every level in one launch. + pub fused: bool, + /// Whether the first successor may be encoded from the bound message + /// instead of folded ([`crate::whir::encode_successor`]). + pub encoded: bool, + /// Whether the message is sent rather than read where the columns lie. + pub staged: bool, + /// Columns in the stacked polynomial, and the extension values of their + /// points' `eq` halves. + pub shares: usize, + pub eq_values: usize, + /// Slots the opening's program lowers to. + pub slots: usize, +} + +/// ★ The most one chain of a stacked opening holds on the card beside the +/// codeword it opens — what a group's openings take their turn for, one chain +/// at a time. +/// +/// Three phases, each of which frees what it no longer needs before the next +/// allocates, so the turn is the largest of them rather than their sum: +/// +/// - **Share form** (the lean rounds): the share map, the shares, the `eq` +/// halves, the round scratch and the staged message when there is one — +/// [`LeanRound0::planned_bytes`] — until the factors are written at the +/// width the rounds leave, beside it. Without the lean rounds the factors +/// are written at full width from a base copy of the message, whether that +/// copy is staged or gathered from the resident columns. +/// - **Rounds** over the written factors: the factors and a session's slot +/// file and partials ([`SumcheckSession::scratch_bytes`]). +/// - **The first fold and what follows it**: the factors, then the fold's +/// transient ([`fold_transient_bytes`]) — or, when the successor may be +/// encoded from the bound message instead, the larger of that and the +/// encode's ([`crate::whir::encode_successor_transient_bytes`]), since a +/// declined encode folds; after it, beside the fold's output, +/// one at a time: the successor's tree, the tree over the committed codeword +/// its queries' paths are read from, and the out-of-domain answer's copy of +/// the message with the next weight's `eq` table. +/// +/// Every later round works on a codeword at most `2^first_fold` times +/// smaller, so it is inside these. A leaf layer kept for retention is promised +/// by the codeword it belongs to, not by the turn. +pub fn opening_transient_bytes(shape: &OpeningShape) -> u64 { + use crate::whir::{ + FUSED_MAX_FOLD, encode_successor_transient_bytes, fold_transient_bytes, tree_bytes, + }; + + let n = shape.num_vars; + let lean = shape.lean_rounds > 0 && shape.shares > 0 && shape.shares < LEAN_MAX_SHARES; + let bound = if lean { + shape.lean_rounds.min(shape.first_fold).min(n) + } else { + 0 + }; + let factors = 48u64 << (n - bound); + let written = if lean { + LeanRound0::planned_bytes(n, shape.shares, shape.eq_values, shape.staged) + factors + } else { + factors + (8u64 << n) + }; + let rounds = factors + SumcheckSession::scratch_bytes(1usize << (n - bound), shape.slots); + let log_codeword = n + shape.log_blowup; + let folded = log_codeword - shape.first_fold; + let transient = fold_transient_bytes( + log_codeword, + shape.first_fold, + shape.fused && shape.first_fold <= FUSED_MAX_FOLD, + ); + let transient = if shape.encoded { + transient.max(encode_successor_transient_bytes( + folded, + n - shape.first_fold.min(n), + )) + } else { + transient + }; + let successor = if shape.next_fold > 0 { + tree_bytes(folded - shape.next_fold) + } else { + 0 + }; + let out_of_domain = 2 * (24u64 << (n - shape.first_fold.min(n))); + let beside = tree_bytes(folded).max(successor).max(out_of_domain); + let fold = factors + transient.max((24u64 << folded) + beside); + written.max(rounds).max(fold) +} + +/// ★ [`opening_transient_bytes`] for a GROUPED chain over `polys` stacked +/// polynomials of `shape` (`GroupFold::Indexed`, `index_vars` = ⌈log2 polys⌉): +/// its round 0 holds every polynomial's opening at once, where a +/// per-polynomial chain holds one. One chain at a time either way, so this is +/// a grouped run's turn; `polys` of 1 is a per-polynomial chain's. +/// +/// Four phases, each freeing what it no longer needs before the next: +/// +/// - **The openings written**: every polynomial's lean opening built up +/// front, then each written at the width the lean rounds leave, one after +/// another, the last beside the ones written before it. Without the lean +/// rounds each is written at full width as it opens, through one base copy +/// at a time. +/// - **The lockstep rounds** over the written factors: every polynomial's +/// factors and a session's slot file and partials beside each. +/// - **The index rounds**: the factors laid slot by slot +/// ([`OpeningSession::gather_slots`]) — in the first polynomial's own +/// allocation when the data rounds left it room for `2^index_vars` slots, +/// in a fresh pair otherwise — and one session over them. +/// - **The successor and what follows it**: the combined factors (that +/// allocation), then the encode's transient — or, when it is not encoded, +/// every polynomial's fold beside the sum they are combined into +/// ([`crate::whir::combine_codewords`]); after it, as for a +/// per-polynomial chain, the successor's tree, the round-0 trees' rebuilt +/// paths one tree at a time, and the out-of-domain answer's copy. +pub fn grouped_opening_transient_bytes( + shape: &OpeningShape, + polys: usize, + index_vars: usize, +) -> u64 { + use crate::whir::{ + FUSED_MAX_FOLD, encode_successor_transient_bytes, fold_transient_bytes, tree_bytes, + }; + + if polys < 2 { + return opening_transient_bytes(shape); + } + let p = polys as u64; + let n = shape.num_vars; + let first_fold = shape.first_fold.min(n); + let lean = shape.lean_rounds > 0 && shape.shares > 0 && shape.shares < LEAN_MAX_SHARES; + let bound = if lean { + shape.lean_rounds.min(first_fold) + } else { + 0 + }; + let factors = 48u64 << (n - bound); + let written = if lean { + let opening = LeanRound0::planned_bytes(n, shape.shares, shape.eq_values, shape.staged); + factors + opening + (p - 1) * factors.max(opening) + } else { + p * factors + (8u64 << n) + }; + let rounds = p * (factors + SumcheckSession::scratch_bytes(1usize << (n - bound), shape.slots)); + let slots_at = n - first_fold; + let gathered = 48u64 << (slots_at + index_vars); + let in_place = bound + index_vars <= first_fold; + let kept = if in_place { factors } else { gathered }; + let index = p * factors + + if in_place { 0 } else { gathered } + + SumcheckSession::scratch_bytes(1usize << (slots_at + index_vars), shape.slots); + let log_codeword = n + shape.log_blowup; + let folded = log_codeword - first_fold; + let one_fold = fold_transient_bytes( + log_codeword, + first_fold, + shape.fused && first_fold <= FUSED_MAX_FOLD, + ); + let output = 24u64 << folded; + let combined = ((p - 1) * output + one_fold).max((p + 1) * output); + let transient = if shape.encoded { + combined.max(encode_successor_transient_bytes(folded, slots_at)) + } else { + combined + }; + let successor = if shape.next_fold > 0 { + tree_bytes(folded - shape.next_fold) + } else { + 0 + }; + let out_of_domain = 2 * (24u64 << slots_at); + let beside = tree_bytes(folded).max(successor).max(out_of_domain); + let fold = kept + transient.max(output + beside); + written.max(rounds).max(index).max(fold) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// ★ The working set the lean opening exists to shrink, in numbers, at the + /// production stack (25) and at two fatter ones (27, 28). The materialised + /// session is `1.5 C` held plus `0.25 C` staging (`C` the committed base + /// codeword, `2^(n+2)·8` bytes); the lean one over resident columns is + /// `C/16` for the share map plus kilobytes — 400 columns and 30 points' + /// `eq` halves at `2^11` values each is generous. + #[test] + fn the_lean_opening_holds_a_sixteenth_of_a_codeword() { + for n in [25usize, 27, 28] { + let codeword = 8u64 << (n + 2); + let (held, staging) = OpeningSession::planned_bytes(n); + let lean = LeanRound0::planned_bytes(n, 400, 30 * 2 * 2048, false); + let lean_staged = LeanRound0::planned_bytes(n, 400, 30 * 2 * 2048, true); + println!( + "2^{n}: materialised {:.2} C + {:.2} C staging; lean {:.4} C resident, {:.4} C staged", + held as f64 / codeword as f64, + staging as f64 / codeword as f64, + lean as f64 / codeword as f64, + lean_staged as f64 / codeword as f64, + ); + assert_eq!(held * 2, codeword * 3); + assert!(lean * 15 < codeword, "2^{n}: {lean} B"); + assert!( + lean * 20 < held + staging, + "2^{n}: {lean} B against {} B", + held + staging + ); + } + } + + /// Three lean rounds unless the knob says otherwise; 7 or more is the + /// widest fold; unreadable is the default. + #[test] + fn three_lean_rounds_by_default_and_the_knob_clamps_to_the_widest_fold() { + use crate::whir::FUSED_MAX_FOLD; + assert_eq!(LEAN_ROUNDS_DEFAULT, 3); + assert_eq!(lean_rounds_setting(None), 3); + assert_eq!(lean_rounds_setting(Some("6")), 6, "the way back"); + assert_eq!(lean_rounds_setting(Some(" 2 ")), 2); + assert_eq!(lean_rounds_setting(Some("0")), 0, "materialise up front"); + assert_eq!(FUSED_MAX_FOLD, 6); + assert_eq!(lean_rounds_setting(Some("7")), FUSED_MAX_FOLD); + assert_eq!(lean_rounds_setting(Some("99")), FUSED_MAX_FOLD); + assert_eq!(lean_rounds_setting(Some("three")), LEAN_ROUNDS_DEFAULT); + assert_eq!(lean_rounds_setting(Some("-1")), LEAN_ROUNDS_DEFAULT); + } + + /// A stack of `LEAN_MAX_SHARES` shares or more cannot be named by the + /// card's `u16` share ids: the lean rounds decline before the device is + /// touched, and the opening materialises at full width as before. + #[test] + fn a_stack_of_too_many_shares_declines_the_lean_rounds() { + let shares: Vec = (0..LEAN_MAX_SHARES) + .map(|i| LeanShare { + stack_offset: i, + num_vars: 0, + lo_bits: 0, + hi_at: 0, + lo_at: 0, + scale: [1, 0, 0], + }) + .collect(); + let eq = vec![1u64, 0, 0]; + let lean = LeanRound0::new(&shares, &eq, 16, LeanMessage::Parts(&[])) + .expect("declining touches no device"); + assert!(lean.is_none(), "{} shares must materialise", shares.len()); + } + + /// `LAMBDA_VM_WHIR_LEAN_TILES`: unset, the tile form a block a tile; the + /// share form at `0` or any word but `1`, `wide` and `check`. + #[test] + fn the_lean_rounds_run_in_tile_form_unless_the_knob_says_otherwise() { + assert_eq!(LEAN_TILES_DEFAULT, LeanTiles::Wide); + assert_eq!(lean_tiles_setting(None), LEAN_TILES_DEFAULT); + assert_eq!(lean_tiles_setting(Some("0")), LeanTiles::Off); + assert_eq!(lean_tiles_setting(Some("")), LeanTiles::Off); + assert_eq!(lean_tiles_setting(Some("yes")), LeanTiles::Off); + assert_eq!(lean_tiles_setting(Some("1")), LeanTiles::On); + assert_eq!(lean_tiles_setting(Some(" 1 ")), LeanTiles::On); + assert_eq!(lean_tiles_setting(Some("wide")), LeanTiles::Wide); + assert_eq!(lean_tiles_setting(Some("check")), LeanTiles::Check); + } + + fn tile_share(stack_offset: usize, num_vars: usize, lo_bits: usize, lo_at: usize) -> LeanShare { + LeanShare { + stack_offset, + num_vars, + lo_bits, + hi_at: 0, + lo_at, + scale: [1, 0, 0], + } + } + + fn spec(tile_bits: usize, lean_rounds: usize, slots: usize) -> TileSpec { + TileSpec { + tile_bits, + lean_rounds, + slots, + wide: false, + check: false, + } + } + + /// ★ The tile form adds nothing to an opening's working set. It never + /// allocates the share form's position map (outside `check`); wherever it + /// fits, ALL it holds through the rounds is no larger than the tables + /// written once they are spent — `materialize` frees `Φ` before it + /// allocates them — and what it keeps beside those tables while it writes + /// them is no larger than the map. So at no step does an opening hold + /// more than the share form at its write, and the turn a group's openings + /// take ([`opening_transient_bytes`]) is what it was. At the production + /// stack and three rounds a slot is 6.4 MiB against the tables' 768 MiB, + /// and 3.3 MiB against the map's 256 MiB. + #[test] + fn the_tile_form_is_no_larger_than_the_tables_written_after_it() { + for n in [16usize, 22, 25, 27, 28] { + for lean_rounds in 1..=6usize { + for tile_bits in 0..=10usize { + for slots in 1..=TILE_MAX_SLOTS { + let form = spec(tile_bits, lean_rounds, slots); + if form.fits(n) { + let tables = 48u64 << (n - lean_rounds); + let map = 2u64 << n; + assert!(form.device_bytes(n) <= tables); + assert!(form.write_bytes(n) <= map); + assert_eq!( + form.device_bytes(n) - form.write_bytes(n), + (24 * slots as u64) << (n - tile_bits), + "the write frees Φ alone" + ); + } + } + } + } + } + assert_eq!(spec(10, 3, 1).write_bytes(27), (3 << 20) + (3 << 17) + 24); + assert!( + spec(3, 1, 1).device_bytes(27) <= 48 << 26 && !spec(3, 1, 1).fits(27), + "at one round, tiles of eight positions: the write would keep 480 MiB beside the \ + tables, more than the 256 MiB map" + ); + assert!(spec(4, 1, 1).fits(27)); + assert_eq!(spec(10, 3, 1).device_bytes(27), (6 << 20) + (3 << 17) + 24); + assert_eq!(spec(10, 3, 8).device_bytes(27), (49 << 20) + (1 << 18) + 24); + assert!(spec(10, 3, 8).fits(27) && spec(8, 3, 8).fits(27) && spec(7, 3, 8).fits(27)); + assert!( + !spec(6, 3, 8).fits(27), + "eight slots of 2^21 tiles are 788 MiB" + ); + assert!(spec(4, 3, 1).fits(27) && !spec(3, 3, 1).fits(27)); + // Not a form at all: no round, a round past the tiles, no slot, too many. + assert!(!spec(10, 0, 1).fits(27) && !spec(10, 18, 1).fits(27)); + assert!(!spec(10, 3, 0).fits(27) && !spec(10, 3, TILE_MAX_SLOTS + 1).fits(27)); + assert!(!spec(28, 1, 1).fits(27)); + } + + /// What the tile form reads: every share split at the tile, at least a + /// tile long, at a multiple of its own size, and no more classes in a + /// column of tiles than slots. + #[test] + fn the_tile_form_reads_aligned_shares_split_at_the_tile() { + let table = [ + tile_share(0, 12, 6, 0), + tile_share(1 << 12, 12, 6, 0), + tile_share(1 << 13, 8, 6, 64), + ]; + assert!(tiles_read(&table, 16, &spec(6, 3, 2))); + assert!( + !tiles_read(&table, 16, &spec(6, 3, 1)), + "two classes, one slot" + ); + // Two rounds bind four tiles: a column of tiles meets two classes at most. + let many: Vec = (0..5).map(|c| tile_share(c << 12, 12, 6, c * 64)).collect(); + assert!(!tiles_read(&many, 16, &spec(6, 3, 4))); + assert!(tiles_read(&many, 16, &spec(6, 2, 4))); + assert!( + !tiles_read(&[tile_share(0, 12, 5, 0)], 16, &spec(6, 3, 1)), + "split elsewhere" + ); + assert!( + !tiles_read(&[tile_share(0, 5, 6, 0)], 16, &spec(6, 3, 1)), + "shorter than a tile" + ); + assert!( + !tiles_read(&[tile_share(1 << 11, 12, 6, 0)], 16, &spec(6, 3, 1)), + "off its alignment" + ); + assert!( + !tiles_read(&table, 16, &spec(6, 11, 2)), + "the spec does not fit" + ); + } + + /// A stack the tile form does not read declines before the device is + /// touched, and the caller opens in share form. + #[test] + fn a_stack_the_tile_form_cannot_read_declines_before_the_device() { + let shares = [tile_share(0, 12, 5, 0)]; + let eq = vec![1u64, 0, 0]; + let lean = LeanRound0::with_tiles( + &shares, + &eq, + 16, + LeanMessage::Parts(&[]), + Some(spec(6, 3, 1)), + ) + .expect("declining touches no device"); + assert!(lean.is_none()); + } + + /// The production chain's shape at `2^n`: first6 then four, 400 columns, + /// the default lean rounds. + fn production(n: usize) -> OpeningShape { + OpeningShape { + num_vars: n, + log_blowup: 2, + first_fold: 6, + next_fold: 4, + lean_rounds: LEAN_ROUNDS_DEFAULT, + fused: true, + encoded: false, + staged: false, + shares: 400, + eq_values: 30 * 2 * 2048, + slots: 2, + } + } + + /// ★ The turn a group's openings take, in numbers. + /// + /// With both kernels the largest phase is the first fold's: the factors + /// bound three variables down at the default (`3C/16`), the fold's output + /// (`3C/64`) and the tree its queries' paths come from (`C/8`) — `23/64` of + /// the committed base codeword `C`, over a third, at every stack. Six lean + /// rounds (the way back) bind the factors to `3C/128` and the turn to about + /// a fifth. Without the kernels it is the factors at full width (`1.5 C`) + /// beside a level-by-level fold (`2.25 C`): three and three quarters, + /// EXACTLY — the working set that ran nearly three codewords past the + /// one-codeword room every opening had promised. + #[test] + fn an_opening_turn_is_a_third_of_a_codeword_with_the_kernels_a_fifth_at_six() { + for n in [16usize, 22, 25, 27, 28] { + let codeword = 8u64 << (n + 2); + let lean = opening_transient_bytes(&production(n)); + let six = opening_transient_bytes(&OpeningShape { + lean_rounds: 6, + ..production(n) + }); + let today = opening_transient_bytes(&OpeningShape { + lean_rounds: 0, + fused: false, + ..production(n) + }); + let staged = opening_transient_bytes(&OpeningShape { + staged: true, + ..production(n) + }); + println!( + "2^{n}: opening turn {:.4} C resident, {:.4} C staged, {:.4} C at six rounds; \ + {:.4} C without the kernels", + lean as f64 / codeword as f64, + staged as f64 / codeword as f64, + six as f64 / codeword as f64, + today as f64 / codeword as f64, + ); + if n >= 22 { + assert!( + lean * 3 > codeword && lean * 64 <= codeword * 23, + "2^{n}: {lean} B is not 23/64 of {codeword} B" + ); + assert!( + six * 5 < codeword && six * 6 > codeword, + "2^{n}: {six} B at six rounds is not about a fifth of {codeword} B" + ); + } + assert_eq!(today * 4, codeword * 15, "2^{n}: {today} B"); + assert!(staged >= lean && staged - lean <= 8u64 << n); + } + // Each kernel alone: the lean rounds without the fused fold still meet + // the level-by-level transient, and the fused fold without the lean + // rounds still meets the factors at full width. + let n = 25; + let codeword = 8u64 << (n + 2); + let lean_only = opening_transient_bytes(&OpeningShape { + fused: false, + ..production(n) + }); + let fused_only = opening_transient_bytes(&OpeningShape { + lean_rounds: 0, + ..production(n) + }); + assert!(lean_only * 4 > codeword * 9, "{lean_only} B"); + assert!(fused_only * 2 > codeword * 3, "{fused_only} B"); + } + + /// ★ A grouped chain's turn, in numbers: its round 0 holds every + /// polynomial's factors at once — `P'·3C/16` at the default three lean + /// rounds, beside one polynomial's share-form opening as the last is + /// written — where a per-polynomial chain's turn is its first fold's, + /// `23/64 C`. A run of one is a per-polynomial chain's turn exactly; a run + /// of `P'` never holds less than its `P'` factors; and at six lean rounds, + /// where the factors are written at `2^(n−6)`, the slots no longer fit the + /// first polynomial's allocation and the gather takes a pair of its own. + #[test] + fn a_grouped_turn_holds_every_polynomials_factors_at_once() { + for n in [16usize, 22, 25, 27] { + let codeword = 8u64 << (n + 2); + let shape = production(n); + let one = opening_transient_bytes(&shape); + assert_eq!(grouped_opening_transient_bytes(&shape, 1, 0), one, "2^{n}"); + let factors = 48u64 << (n - LEAN_ROUNDS_DEFAULT); + let opening = LeanRound0::planned_bytes(n, shape.shares, shape.eq_values, false); + for (polys, index_vars) in [(2usize, 1usize), (3, 2), (4, 2)] { + let grouped = grouped_opening_transient_bytes(&shape, polys, index_vars); + let p = polys as u64; + println!( + "2^{n}: a run of {polys} {:.4} C against {:.4} C a polynomial", + grouped as f64 / codeword as f64, + one as f64 / codeword as f64, + ); + assert!(grouped >= one, "2^{n} run of {polys}"); + // Every polynomial's factors and the last one's share-form + // opening while it is written, `(3P' + 1)·C/16`; or the + // lockstep rounds, each factor pair beside its session's slot + // file — at a small stack the slot file is not small. + let scratch = SumcheckSession::scratch_bytes( + 1usize << (n - LEAN_ROUNDS_DEFAULT), + shape.slots, + ); + let written = p * factors + opening; + let rounds = p * (factors + scratch); + assert!(grouped >= written.max(rounds), "2^{n} run of {polys}"); + if n == 27 { + assert_eq!(grouped, written, "2^{n} run of {polys}: the writes bind"); + } + } + let six = OpeningShape { + lean_rounds: 6, + ..production(n) + }; + let gathered = 48u64 << (n - 6 + 2); + assert!( + grouped_opening_transient_bytes(&six, 4, 2) >= 4 * (48u64 << (n - 6)) + gathered, + "2^{n} at six rounds: the gather's own pair" + ); + } + } + + /// The shares and the stack decide which path the turn is sized for: past + /// what a `u16` can name, the lean opening declines and the turn is the + /// materialised one's, as the opening itself will be. + #[test] + fn a_stack_the_lean_rounds_cannot_read_is_sized_as_materialised() { + let n = 25; + let too_many = opening_transient_bytes(&OpeningShape { + shares: LEAN_MAX_SHARES, + ..production(n) + }); + let materialised = opening_transient_bytes(&OpeningShape { + lean_rounds: 0, + shares: LEAN_MAX_SHARES, + ..production(n) + }); + assert_eq!(too_many, materialised); + assert!(too_many > opening_transient_bytes(&production(n))); + } + + /// A successor encoded from the bound message holds more beside the + /// factors than the fused fold — the interleaved successor, one limb's + /// plane and its encoding (`17/8 · 2^n` bytes at a first fold of six) + /// against the output alone (`3/2 · 2^n`) — but never more than the fold's + /// output and the tree the first round's paths are read from + /// (`32·2^f + 8·2^(n−k) ≤ 40·2^f` against `88·2^f` less a node, `f` the + /// folded codeword's log size), so the turn is the one the fold takes, at + /// every first fold. + #[test] + fn an_encoded_successor_fits_the_turn_the_fold_takes() { + use crate::whir::{encode_successor_transient_bytes, fold_transient_bytes, tree_bytes}; + for n in [16usize, 22, 25, 27, 28] { + let folded = n + 2 - 6; + let encode = encode_successor_transient_bytes(folded, n - 6); + assert_eq!(encode * 8, 17u64 << n); + assert_eq!(fold_transient_bytes(n + 2, 6, true) * 2, 3u64 << n); + for first_fold in 1..=6 { + for lean_rounds in [0, 3, 6] { + let shape = OpeningShape { + first_fold, + lean_rounds, + ..production(n) + }; + let folded = n + 2 - first_fold; + assert!( + encode_successor_transient_bytes(folded, n - first_fold) + < (24u64 << folded) + tree_bytes(folded) + ); + assert_eq!( + opening_transient_bytes(&OpeningShape { + encoded: true, + ..shape + }), + opening_transient_bytes(&shape), + "2^{n}, first fold {first_fold}, {lean_rounds} lean rounds" + ); + } + } + } + } +} diff --git a/crypto/math-cuda/tests/blake3_chain_kat.rs b/crypto/math-cuda/tests/blake3_chain_kat.rs new file mode 100644 index 000000000..4b3687d6e --- /dev/null +++ b/crypto/math-cuda/tests/blake3_chain_kat.rs @@ -0,0 +1,243 @@ +//! ★ Known-answer tests for the DEVICE `Blake3Chain`, against references that +//! are not this tree's Rust. +//! +//! # Why this file exists (risk R13) +//! +//! Every other parity test here asserts device == host. That is necessary and +//! not sufficient: the device kernels were transcribed from the host reference, +//! so a shared misreading of the construction passes all of them. R13 is exactly +//! that gap — "track G would then be checking a device port against the same +//! code path it was derived from". +//! +//! Two references close it, and neither is Rust in this repository: +//! +//! - **At 7 rounds, the official `blake3` crate.** `Blake3Chain` over any +//! message of at most one chunk (1024 bytes) IS `blake3::hash`, because +//! standard BLAKE3's first chunk is this chain and a one-chunk message has +//! that chunk's output as its root (PA-PLAN §1.7.2, P1). So for the whole +//! range the prover actually hashes in, the device is checked against a +//! published, externally maintained implementation with nothing in between. +//! - **At 6 rounds, `CHAIN_KAT_6ROUND`.** Those digests came from #903's Python +//! oracle, not from this code (`chain.rs:280-294`). Asserting the device +//! against them is a check against an artifact this tree did not compute. +//! +//! # Why the coverage is at multiples of four bytes +//! +//! The device chain is word-granular, because that is all it ever hashes: every +//! production message is a whole number of 8-byte field elements. The KAT +//! lengths that are not multiples of 4 (1, 31, 63, 127) are unreachable from +//! device code by construction and are covered by the host tests in +//! `crypto/crypto/src/hash/blake3/chain.rs` instead. What remains — 0, 64, 128, +//! 192, 256, 1024, 1088 — still covers every structural case PA-PLAN §1.7.4 +//! names except the partial-tail ones: the empty message is one block (0), a +//! 64-byte message is the parent form (64), an exact multiple of 64 emits no +//! spurious final block (128), interior blocks carry no flags (192, 256, 1024), +//! and 1088 is where this construction leaves standard BLAKE3. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::{expected_device_rounds, merkle_parent}; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::hash::blake3::chain::{ + CHAIN_KAT_6ROUND, CHAIN_KAT_LENS, blake3_chain_rounds, kat_message_byte, +}; +use math_cuda::blake3::chain_probe; + +/// The KAT message of a given length: byte `i` is `37i + 11 (mod 256)`. +fn message(len: usize) -> Vec { + (0..len).map(kat_message_byte).collect() +} + +/// A byte message as the little-endian u32 words the device chain absorbs. +/// Panics on a length that is not a whole number of words — see the module docs +/// for why that case cannot arise on device. +fn words(msg: &[u8]) -> Vec { + assert!(msg.len().is_multiple_of(4), "device chain is word-granular"); + msg.chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect() +} + +fn assert_lockstep() { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep" + ); +} + +/// The KAT lengths the device can hash, paired with their index into +/// `CHAIN_KAT_6ROUND`. +fn device_reachable_lengths() -> Vec<(usize, usize)> { + CHAIN_KAT_LENS + .iter() + .enumerate() + .filter(|&(_, &len)| len.is_multiple_of(4)) + .map(|(i, &len)| (i, len)) + .collect() +} + +/// ★ THE EXTERNAL ANCHOR. At 7 rounds the device chain must be the `blake3` +/// crate's hash, for every reachable length up to one full chunk. +/// +/// This is the strongest statement available about the device port: no oracle, +/// no table, no transcription — a published implementation computes the same +/// bytes. It pins the block splitting, the zero padding, the final block's +/// `block_len`, the CHUNK_START/CHUNK_END/ROOT schedule, `t = 0` throughout, and +/// the little-endian digest read-back, all at once. +/// +/// Only meaningful when the cubin is built for 7 rounds; under `blake3-6round` +/// nothing external recomputes this, which is PA-PLAN §1.6's premise and why the +/// 6-round arm needs the committed table instead. +#[test] +fn device_chain_is_the_blake3_crate_at_seven_rounds() { + assert_lockstep(); + if expected_device_rounds() != 7 { + return; + } + for (_, len) in device_reachable_lengths() { + if len > 1024 { + continue; + } + let msg = message(len); + let device = chain_probe(&words(&msg)).unwrap(); + assert_eq!( + device, + *blake3::hash(&msg).as_bytes(), + "device chain must equal the blake3 crate at length {len}" + ); + } +} + +/// ★ P3, on device. Past one chunk the construction deliberately leaves standard +/// BLAKE3 — the standard would start chunk 1 with `t = 1` and a reset chaining +/// value, this keeps chaining. Without this the test above would pass +/// identically if the kernels had implemented the whole chunk tree, so "the +/// device implements the single-chunk chain" would be unfalsifiable. +#[test] +fn device_chain_leaves_the_blake3_crate_past_one_chunk() { + assert_lockstep(); + if expected_device_rounds() != 7 { + return; + } + // 1024 is the last length where they agree; 1088 the first reachable one + // past it. Asserting both locates the divergence rather than just observing + // one. + let agreeing = message(1024); + assert_eq!( + chain_probe(&words(&agreeing)).unwrap(), + *blake3::hash(&agreeing).as_bytes(), + "1024 bytes is still one chunk and must agree" + ); + let diverging = message(1088); + assert_ne!( + chain_probe(&words(&diverging)).unwrap(), + *blake3::hash(&diverging).as_bytes(), + "past one chunk the device must leave the standard" + ); +} + +/// ★ THE 6-ROUND ANCHOR. The device must reproduce the committed KAT table, +/// whose digests came from a Python oracle rather than from this code. +/// +/// This is the assertion R13 asks for: at the round count the campaign actually +/// ships, the device port is pinned by numbers no Rust in this tree produced. +#[test] +fn device_chain_matches_the_committed_table_at_six_rounds() { + assert_lockstep(); + if expected_device_rounds() != 6 { + return; + } + for (i, len) in device_reachable_lengths() { + let device = chain_probe(&words(&message(len))).unwrap(); + assert_eq!( + device, CHAIN_KAT_6ROUND[i], + "device chain must match the committed 6-round KAT at length {len}" + ); + } +} + +/// The device chain against the host chain at whatever round count this build +/// uses. Weaker than the two anchors above — both sides are ours — but it is the +/// one that runs in every configuration, and it is the property the commitment +/// path actually needs: a GPU tree and a CPU tree over the same leaves must be +/// the same tree. +#[test] +fn device_chain_matches_the_host_chain() { + assert_lockstep(); + let rounds = expected_device_rounds(); + for (_, len) in device_reachable_lengths() { + let msg = message(len); + assert_eq!( + chain_probe(&words(&msg)).unwrap(), + blake3_chain_rounds(&msg, rounds), + "device/host chain mismatch at length {len}, {rounds} rounds" + ); + } + // Lengths off the KAT list, stepping through several block boundaries, so + // the agreement is not an artifact of the seven lengths chosen above. + for len in (0..=520usize).step_by(4) { + let msg = message(len); + assert_eq!( + chain_probe(&words(&msg)).unwrap(), + blake3_chain_rounds(&msg, rounds), + "device/host chain mismatch at length {len}" + ); + } +} + +/// ★ P2, on device: a 64-byte message through the chain is exactly the Merkle +/// parent compression. +/// +/// This is the invariant that lets the leaf and parent layers be one hash — and +/// the reason `blake3_hash_merkle_parent` can be a single compression with no +/// chaining at all. If the chain's flag schedule or `block_len` moved, the two +/// would part here while every leaf test still passed. +#[test] +fn a_sixty_four_byte_chain_is_the_parent_compression() { + assert_lockstep(); + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(7)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(31).wrapping_add(3)); + let mut msg = [0u8; 64]; + msg[..32].copy_from_slice(&left); + msg[32..].copy_from_slice(&right); + + assert_eq!( + chain_probe(&words(&msg)).unwrap(), + merkle_parent(&left, &right, expected_device_rounds()), + "a 64-byte device chain must be the parent form" + ); +} + +/// NEGATIVE CONTROL: distinct lengths must give distinct digests, or the tests +/// above would pass with a probe that ignored its input length. In particular a +/// chain that ignored `block_len` would collide 0 with nothing visible here, but +/// one that dropped the final partial block would collide 64 with 128. +#[test] +fn device_digests_are_distinct_across_lengths() { + assert_lockstep(); + let mut seen: Vec<[u8; 32]> = Vec::new(); + for len in (0..=256usize).step_by(4) { + let d = chain_probe(&words(&message(len))).unwrap(); + assert!( + !seen.contains(&d), + "length {len} collides with a shorter message" + ); + seen.push(d); + } +} + +/// The cubin's compiled-in round count must be the one the Rust side thinks it +/// is. Reading it back is the only way to observe from host code which arm +/// `blake3_merkle_level` and the leaf kernels were built for; a mismatch here is +/// a GPU tree committing under a different hash with no other symptom. +#[test] +fn the_cubin_round_count_is_what_the_feature_selected() { + assert_eq!( + math_cuda::blake3::device_rounds().unwrap() as usize, + expected_device_rounds(), + "cubin round count disagrees with math-cuda's blake3-6round feature" + ); +} diff --git a/crypto/math-cuda/tests/blake3_comp_poly_tree.rs b/crypto/math-cuda/tests/blake3_comp_poly_tree.rs new file mode 100644 index 000000000..71231968b --- /dev/null +++ b/crypto/math-cuda/tests/blake3_comp_poly_tree.rs @@ -0,0 +1,159 @@ +//! Parity: the device BLAKE3 composition-polynomial tree must equal the CPU +//! tree node for node, through BOTH wrappers that build it. +//! +//! `build_comp_poly_tree_from_evals_ext3_keep` takes host-side interleaved parts +//! and stages them through the pinned de-interleave buffer; +//! `build_comp_poly_tree_from_slabs_dev` takes an already-resident slab buffer +//! and never touches the host. They share the leaf kernel and the level walk but +//! not the staging, so a de-interleave bug shows in the first and not the second +//! — which is why both are exercised here rather than only the one the leaf test +//! happens to call. +//! +//! CPU reference is the production leaf function plus the production tree +//! builder, so nothing in the comparison is written for the test. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::expected_device_rounds; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::merkle_tree::backends::types::BatchBlake3Backend; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use stark::commitment::leaves_bit_reversed_grouped; + +type Fp = FieldElement; +type Fp3 = FieldElement; +type Ext3 = Degree3GoldilocksExtensionField; + +fn rand_ext3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) +} + +fn assert_lockstep() { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep" + ); +} + +/// The CPU node buffer for these parts: production row-pair leaves, production +/// tree walk. +fn cpu_nodes(parts: &[Vec]) -> Vec<[u8; 32]> { + let leaves = leaves_bit_reversed_grouped::>(parts, 2); + let tree = MerkleTree::>::build_from_hashed_leaves(leaves).unwrap(); + tree.nodes().to_vec() +} + +fn interleave(parts: &[Vec], lde_size: usize) -> Vec> { + parts + .iter() + .map(|p| { + let mut v = vec![0u64; 3 * lde_size]; + for (i, e) in p.iter().enumerate() { + v[i * 3] = *e.value()[0].value(); + v[i * 3 + 1] = *e.value()[1].value(); + v[i * 3 + 2] = *e.value()[2].value(); + } + v + }) + .collect() +} + +/// The de-interleaved slab layout the device wrapper consumes directly: +/// component `k` of part `c` at `(c*3 + k) * lde_size`. +fn slabs(parts: &[Vec], lde_size: usize) -> Vec { + let mut buf = vec![0u64; 3 * parts.len() * lde_size]; + for (c, p) in parts.iter().enumerate() { + for (r, e) in p.iter().enumerate() { + buf[(c * 3) * lde_size + r] = *e.value()[0].value(); + buf[(c * 3 + 1) * lde_size + r] = *e.value()[1].value(); + buf[(c * 3 + 2) * lde_size + r] = *e.value()[2].value(); + } + } + buf +} + +fn assert_nodes_eq(gpu: &[u8], cpu: &[[u8; 32]], what: &str) { + assert_eq!(gpu.len(), cpu.len() * 32, "{what}: node count"); + for (i, expected) in cpu.iter().enumerate() { + assert_eq!( + &gpu[i * 32..(i + 1) * 32], + &expected[..], + "{what}: node {i} mismatch" + ); + } +} + +fn run_parity(log_lde: u32, num_parts: usize, seed: u64) { + assert_lockstep(); + let lde_size = 1usize << log_lde; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let parts: Vec> = (0..num_parts) + .map(|_| (0..lde_size).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + let expected = cpu_nodes(&parts); + let what = format!("log_lde={log_lde} parts={num_parts}"); + + let be = math_cuda::device::backend().unwrap(); + + // Route 1: host-side interleaved parts through the pinned staging path. + let interleaved = interleave(&parts, lde_size); + let slices: Vec<&[u64]> = interleaved.iter().map(|v| v.as_slice()).collect(); + let keep = math_cuda::blake3::build_comp_poly_tree_from_evals_ext3_keep(&slices).unwrap(); + { + let stream = be.next_stream(); + let nodes: Vec = stream.clone_dtoh(&*keep.nodes).unwrap(); + assert_nodes_eq(&nodes, &expected, &format!("keep {what}")); + assert_eq!(&keep.root[..], &expected[0][..], "keep {what}: root"); + assert_eq!(keep.leaves_len, lde_size / 2, "keep {what}: leaf count"); + } + + // Route 2: an already-resident slab buffer, no host staging. + { + let stream = be.next_stream(); + let buf = stream.clone_htod(&slabs(&parts, lde_size)).unwrap(); + stream.synchronize().unwrap(); + let dev = math_cuda::blake3::build_comp_poly_tree_from_slabs_dev( + &stream, &buf, num_parts, lde_size, + ) + .unwrap(); + let nodes: Vec = stream.clone_dtoh(&*dev.nodes).unwrap(); + assert_nodes_eq(&nodes, &expected, &format!("slabs {what}")); + assert_eq!(&dev.root[..], &expected[0][..], "slabs {what}: root"); + } +} + +/// Small trees: the tail kernel builds every level in one launch. +#[test] +fn blake3_comp_poly_tree_small() { + for log_lde in [2u32, 4, 6, 8] { + for num_parts in [1usize, 2, 5] { + run_parity(log_lde, num_parts, 300 + log_lde as u64 + num_parts as u64); + } + } +} + +/// Deep enough to cross from the per-level kernel into the tail. +#[test] +fn blake3_comp_poly_tree_medium() { + for log_lde in [10u32, 12, 14] { + run_parity(log_lde, 17, 700 + log_lde as u64); + } +} + +#[test] +fn blake3_comp_poly_tree_large() { + run_parity(18, 3, 4242); +} diff --git a/crypto/math-cuda/tests/blake3_compress_parity.rs b/crypto/math-cuda/tests/blake3_compress_parity.rs new file mode 100644 index 000000000..9032ba525 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_compress_parity.rs @@ -0,0 +1,217 @@ +//! Parity: the device BLAKE3 compression function must equal the host reference +//! bit-for-bit, at both round counts. +//! +//! The reference is one function whose only parameter is the round count +//! (`crypto::hash::blake3::blake3_compress_rounds`, re-exported through +//! `blake3_reference` — the same function the host commitment backends and the +//! in-circuit chip use, not a copy of it). So the 7-round arm, where the `blake3` crate +//! is an external known-answer test, certifies the whole device code path — the +//! G function, the message schedule, the counter split, the feed-forward — and +//! the 6-round arm differs from it by a loop bound alone. That is why the +//! anchor below is worth more than a table of 6-round vectors would be. +//! +//! Every test here needs a GPU, like the rest of this crate's parity suite. + +mod blake3_reference; + +use blake3_reference::{ + BLAKE3_IV, BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS, FLAGS_ONE_BLOCK, blake3_compress_rounds, + expected_device_rounds, +}; +use math_cuda::blake3::{CompressInput, ProbeRounds, compress_probe, device_rounds}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +/// Inputs that between them move every field of the compression's framing. +/// +/// `block_len` and `flags` are state words, not lengths the kernel loops over, so +/// a port that dropped either would still pass on a single value of it — hence +/// the spread, including the 18..=64 range the host's canonical vectors cover and +/// the 36 the LFM socket uses. `t` carries values whose halves differ, since the +/// counter split is a real way to be wrong and a symmetric `t` cannot see it. +fn vectors(seed: u64) -> Vec { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let block_lens = [0u32, 1, 4, 18, 36, 63, 64]; + let flags = [0u32, 1, 2, 8, FLAGS_ONE_BLOCK, 0x0C, 0xFFFF_FFFF]; + let counters = [ + 0u64, + 1, + 0xFFFF_FFFF, + 0x1_0000_0000, + 0xB4E1_357D_4A84_EB03, + u64::MAX, + ]; + + let mut out = Vec::new(); + // Walk the framing exhaustively over random h/m: 7 × 7 × 6 = 294, plus the + // eight `h = IV` cases below for 302 — not a multiple of the kernel's block + // width, so the `tid >= n` guard is exercised too. + for &block_len in block_lens.iter() { + for &fl in flags.iter() { + for &t in counters.iter() { + out.push(CompressInput { + h: core::array::from_fn(|_| rng.r#gen::()), + m: core::array::from_fn(|_| rng.r#gen::()), + t, + block_len, + flags: fl, + }); + } + } + } + // `h = IV` is the case every real call site uses, and a random h would never + // hit it: the feed-forward `out[i+8] = v[i+8] ^ h[i]` reads h a second time. + for _ in 0..8 { + out.push(CompressInput { + h: BLAKE3_IV, + m: core::array::from_fn(|_| rng.r#gen::()), + t: 0, + block_len: 64, + flags: FLAGS_ONE_BLOCK, + }); + } + out +} + +fn host_outputs(inputs: &[CompressInput], rounds: usize) -> Vec<[u32; 16]> { + inputs + .iter() + .map(|i| blake3_compress_rounds(&i.h, &i.m, i.t, i.block_len, i.flags, rounds)) + .collect() +} + +fn assert_parity(rounds: usize, probe: ProbeRounds, seed: u64) { + let inputs = vectors(seed); + let device = compress_probe(&inputs, probe).unwrap(); + let host = host_outputs(&inputs, rounds); + assert_eq!(device.len(), host.len()); + for (i, (d, h)) in device.iter().zip(host.iter()).enumerate() { + assert_eq!( + d, h, + "vector {i} mismatch at {rounds} rounds: input {:?}", + inputs[i] + ); + } +} + +#[test] +fn device_compression_matches_host_at_six_rounds() { + assert_parity(BLAKE3_SIX_ROUNDS, ProbeRounds::Six, 6001); +} + +#[test] +fn device_compression_matches_host_at_seven_rounds() { + assert_parity(BLAKE3_STANDARD_ROUNDS, ProbeRounds::Seven, 7001); +} + +/// ★ **The external anchor.** At 7 rounds a message of at most 64 bytes is one +/// chunk and one block, so the whole tree hasher collapses to a single `f`: +/// `h = IV`, the block zero-padded and read as 16 little-endian words, `t = 0`, +/// `block_len` the true length, `flags = CHUNK_START|CHUNK_END|ROOT`. The digest +/// is `out[0..8]` little-endian. +/// +/// Mirrors `prover/src/lfm/blake3.rs`'s `seven_rounds_is_the_blake3_crate`, over +/// the same 65 lengths and for the same reason: the length keys both `block_len` +/// and the padding, and a port that ignored `block_len` would pass at one length. +#[test] +fn seven_rounds_on_device_is_the_blake3_crate() { + let inputs: Vec = (0..=64usize) + .map(|len| { + let mut block = [0u8; 64]; + for (i, b) in block.iter_mut().take(len).enumerate() { + *b = (i as u8).wrapping_mul(37).wrapping_add(11); + } + CompressInput { + h: BLAKE3_IV, + m: core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }), + t: 0, + block_len: len as u32, + flags: FLAGS_ONE_BLOCK, + } + }) + .collect(); + + let device = compress_probe(&inputs, ProbeRounds::Seven).unwrap(); + for (len, out) in device.iter().enumerate() { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect(); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_eq!( + ours, + *blake3::hash(&msg).as_bytes(), + "device 7-round compression must equal the blake3 crate at length {len}" + ); + } +} + +/// NEGATIVE CONTROL for the anchor above: at 6 rounds the device must NOT match +/// the crate. Without it, the anchor would pass just as well if the round count +/// were being ignored on device — the one bug that makes the whole +/// external-anchor argument vacuous, since the 6-round arm's only defence is +/// "the same code path with the loop bound changed". +#[test] +fn six_rounds_on_device_is_not_the_blake3_crate() { + let msg: [u8; 36] = core::array::from_fn(|i| i as u8); + let mut block = [0u8; 64]; + block[..36].copy_from_slice(&msg); + let input = CompressInput { + h: BLAKE3_IV, + m: core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }), + t: 0, + block_len: 36, + flags: FLAGS_ONE_BLOCK, + }; + let out = compress_probe(&[input], ProbeRounds::Six).unwrap()[0]; + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_ne!(ours, *blake3::hash(&msg).as_bytes()); +} + +/// The two round counts must actually differ on device. Guards the shape of the +/// port itself: `blake3_compress` is one template instantiated twice, and a +/// template that collapsed (or a `#pragma unroll` that outran the bound) would +/// leave both parity tests above passing against a single arm. +#[test] +fn the_two_device_round_counts_differ() { + let inputs = vectors(4242); + let six = compress_probe(&inputs, ProbeRounds::Six).unwrap(); + let seven = compress_probe(&inputs, ProbeRounds::Seven).unwrap(); + for (i, (a, b)) in six.iter().zip(seven.iter()).enumerate() { + assert_ne!(a, b, "6r and 7r agree on vector {i}"); + } +} + +/// The round count the production kernels are compiled for must be the one the +/// `blake3-6round` feature selects. +/// +/// This is the tripwire for a cross-crate feature mismatch: math-cuda's feature +/// and the host tree's are separate, nothing forces them equal, and the symptom +/// of a mismatch is a GPU tree that commits under a different hash than the CPU +/// one — no panic, no log line, just a proof that fails to verify. Asserting the +/// cubin's own round count is what turns that into a test failure. +#[test] +fn the_compiled_in_round_count_is_the_feature() { + assert_eq!( + device_rounds().unwrap() as usize, + expected_device_rounds(), + "kernels/blake3.cu was compiled for a different round count than the \ + math-cuda `blake3-6round` feature selects — check build.rs's -D plumbing" + ); + + // And the default probe must be the corresponding explicit arm, which is what + // ties `blake3_merkle_level`'s hash to the number reported above. + let inputs = vectors(909); + let default = compress_probe(&inputs, ProbeRounds::CompiledIn).unwrap(); + let host = host_outputs(&inputs, expected_device_rounds()); + assert_eq!(default, host); +} diff --git a/crypto/math-cuda/tests/blake3_fri_layer_tree.rs b/crypto/math-cuda/tests/blake3_fri_layer_tree.rs new file mode 100644 index 000000000..9bdaa33ba --- /dev/null +++ b/crypto/math-cuda/tests/blake3_fri_layer_tree.rs @@ -0,0 +1,96 @@ +//! Parity: the device BLAKE3 FRI-layer tree must equal the CPU tree node for +//! node — leaves and inner nodes alike. +//! +//! Mirror of `fri_layer_tree.rs` with the backend swapped. The CPU side is the +//! production `MerkleTree::build` over `PairBlake3Backend`, so what is compared +//! is the kernel pair against the real commitment path, not against a tree +//! builder written for the test. +//! +//! `blake3_leaves.rs` already pins the leaf layer alone; this adds the inner +//! nodes, which is where the level/tail launch split lives. Deep trees cross the +//! threshold where `blake3_merkle_level` hands over to `blake3_merkle_tail`, so +//! both kernels run. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::expected_device_rounds; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::merkle_tree::backends::types::PairBlake3Backend; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math_cuda::blake3::build_fri_layer_tree_from_evals_ext3; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; +type Fp3 = FieldElement; +type Ext3 = Degree3GoldilocksExtensionField; + +fn rand_ext3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) +} + +fn run_parity(log_num_leaves: u32, seed: u64) { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep" + ); + + let num_leaves = 1usize << log_num_leaves; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let evals: Vec = (0..num_leaves * 2).map(|_| rand_ext3(&mut rng)).collect(); + + let mut evals_u64 = Vec::with_capacity(evals.len() * 3); + for e in &evals { + evals_u64.push(*e.value()[0].value()); + evals_u64.push(*e.value()[1].value()); + evals_u64.push(*e.value()[2].value()); + } + + let leaves: Vec<[Fp3; 2]> = evals.chunks_exact(2).map(|c| [c[0], c[1]]).collect(); + let cpu_tree = MerkleTree::>::build(&leaves).unwrap(); + let cpu_nodes = cpu_tree.nodes(); + + let gpu_bytes = build_fri_layer_tree_from_evals_ext3(&evals_u64).unwrap(); + + assert_eq!(cpu_nodes.len() * 32, gpu_bytes.len(), "node count"); + for (i, expected) in cpu_nodes.iter().enumerate() { + assert_eq!( + &gpu_bytes[i * 32..(i + 1) * 32], + &expected[..], + "node {i} mismatch at log_num_leaves={log_num_leaves}" + ); + } +} + +/// Small trees: every level fits the block width, so the tail kernel builds the +/// whole tree in one launch. +#[test] +fn blake3_fri_layer_tree_small() { + for log in 1u32..=6 { + run_parity(log, 100 + log as u64); + } +} + +/// Deep enough that the per-level kernel runs first and hands over to the tail +/// partway up — the launch path a real commit takes. +#[test] +fn blake3_fri_layer_tree_medium() { + for log in [10u32, 12, 14] { + run_parity(log, 500 + log as u64); + } +} + +#[test] +fn blake3_fri_layer_tree_large() { + run_parity(18, 9999); +} diff --git a/crypto/math-cuda/tests/blake3_fused_parity.rs b/crypto/math-cuda/tests/blake3_fused_parity.rs new file mode 100644 index 000000000..12a655190 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_fused_parity.rs @@ -0,0 +1,261 @@ +//! The fused GPU pipelines under BLAKE3 must produce the same Merkle root as +//! the CPU BLAKE3 path — the fused twin of `merkle_root_parity`, which pins +//! the same pipelines under keccak. +//! +//! `merkle_root_parity` covers the leaf/tree kernels through the generic +//! entry points; this file drives the FUSED entries (`coset_lde_row_major_ +//! with_merkle_tree_keep` and the ext3 variant) with `DeviceHash::Blake3`, +//! i.e. exactly the dispatch the production commit path takes on a cuda +//! build, and compares against the CPU row-major LDE + `Blake3StarkHash` +//! commit byte for byte. A tamper arm proves the comparison is not vacuous. + +use math::fft::two_half_fft::TwoHalfTwiddles; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::polynomial::Polynomial; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use stark::config::Blake3StarkHash; +use stark::prover::{GenericProver, IsStarkProver}; + +/// The BLAKE3 prover, named: these tests compare against the CUDA BLAKE3 +/// kernels, so the CPU side must say BLAKE3 explicitly rather than follow +/// the default alias — the comparison stays meaningful even if the default +/// ever moves. +type Prover = GenericProver; + +type Fp3 = FieldElement; +type Fp = FieldElement; + +fn coset_weights(n: usize, g: u64) -> Vec { + let inv_n = Fp::from(n as u64).inv().unwrap(); + let g_fp = Fp::from_raw(g); + let mut w = Vec::with_capacity(n); + let mut cur = inv_n; + for _ in 0..n { + w.push(cur); + cur = &cur * &g_fp; + } + w +} + +fn coset_weights_u64(n: usize, g: u64) -> Vec { + coset_weights(n, g).iter().map(|w| *w.value()).collect() +} + +fn cpu_row_major_blake3_root( + columns: &[Vec], + blowup: usize, + weights: &[Fp], + inv_tw: &TwoHalfTwiddles, + fwd_tw: &TwoHalfTwiddles, +) -> [u8; 32] { + let n = columns[0].len(); + let num_cols = columns.len(); + + let mut buf: Vec = vec![Fp::from(0u64); n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, &v) in col.iter().enumerate() { + buf[r * num_cols + c] = Fp::from_raw(v); + } + } + + Polynomial::::coset_lde_full_expand_row_major::( + &mut buf, num_cols, blowup, weights, inv_tw, fwd_tw, + ) + .expect("CPU row-major LDE"); + + let (_, root) = + Prover::::commit_rows_bit_reversed(&buf, num_cols) + .expect("CPU BLAKE3 commit"); + + root +} + +/// Device fused row-major LDE + BLAKE3 leaves + Merkle, root only. +fn gpu_fused_blake3_root(columns: &[Vec], blowup: usize, weights_u64: &[u64]) -> [u8; 32] { + let n = columns[0].len(); + let num_cols = columns.len(); + + // Row-major input: data[row * num_cols + col]. + let mut row_major = vec![0u64; n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, &v) in col.iter().enumerate() { + row_major[r * num_cols + c] = v; + } + } + + let (handle, _lde) = math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep( + &row_major, + None, + math_cuda::DeviceHash::Blake3, + n, + num_cols, + blowup, + weights_u64, + true, + ) + .expect("fused BLAKE3 GPU pipeline"); + handle.tree.as_ref().expect("resident merkle tree").root +} + +#[test] +fn blake3_fused_base_root_matches_cpu() { + const COSET_OFFSET: u64 = 7; + + for log_n in [4usize, 6, 8, 10] { + for blowup in [2usize, 4] { + for num_cols in [1usize, 3, 8] { + let n = 1usize << log_n; + let log_lde = (n * blowup).trailing_zeros() as usize; + let mut rng = + ChaCha8Rng::seed_from_u64((log_n * 1000 + blowup * 100 + num_cols) as u64); + + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rng.r#gen::()).collect()) + .collect(); + + let weights_u64 = coset_weights_u64(n, COSET_OFFSET); + let weights_fp = coset_weights(n, COSET_OFFSET); + let inv_tw = + TwoHalfTwiddles::::new(log_n, true).expect("inv twiddles"); + let fwd_tw = + TwoHalfTwiddles::::new(log_lde, false).expect("fwd twiddles"); + + let gpu_root = gpu_fused_blake3_root(&columns, blowup, &weights_u64); + let cpu_root = + cpu_row_major_blake3_root(&columns, blowup, &weights_fp, &inv_tw, &fwd_tw); + + assert_eq!( + gpu_root, cpu_root, + "BLAKE3 fused root mismatch: log_n={log_n} blowup={blowup} num_cols={num_cols}" + ); + } + } + } +} + +fn rand_ext3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + FieldElement::::from_raw(rng.r#gen::()), + FieldElement::::from_raw(rng.r#gen::()), + FieldElement::::from_raw(rng.r#gen::()), + ]) +} + +fn cpu_ext3_row_major_blake3_root( + columns: &[Vec], + blowup: usize, + weights: &[Fp], + inv_tw: &TwoHalfTwiddles, + fwd_tw: &TwoHalfTwiddles, +) -> [u8; 32] { + let n = columns[0].len(); + let num_cols = columns.len(); + + let mut buf: Vec = vec![Fp3::from(0u64); n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, v) in col.iter().enumerate() { + buf[r * num_cols + c] = *v; + } + } + + Polynomial::::coset_lde_full_expand_row_major::( + &mut buf, num_cols, blowup, weights, inv_tw, fwd_tw, + ) + .expect("CPU ext3 row-major LDE"); + + let (_, root) = + Prover::::commit_rows_bit_reversed( + &buf, num_cols, + ) + .expect("CPU ext3 BLAKE3 commit"); + + root +} + +#[test] +fn blake3_fused_ext3_root_matches_cpu() { + const COSET_OFFSET: u64 = 7; + + for log_n in [4usize, 6, 8] { + for blowup in [2usize, 4] { + for num_cols in [1usize, 3, 5] { + let n = 1usize << log_n; + let log_lde = (n * blowup).trailing_zeros() as usize; + let mut rng = ChaCha8Rng::seed_from_u64( + (log_n * 1000 + blowup * 100 + num_cols) as u64 + 4242, + ); + + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + // Row-major ext3 = row-major base with 3 * num_cols lanes. + let mut row_major = vec![0u64; n * num_cols * 3]; + for (c, col) in columns.iter().enumerate() { + for (r, v) in col.iter().enumerate() { + for k in 0..3 { + row_major[(r * num_cols + c) * 3 + k] = *v.value()[k].value(); + } + } + } + + let weights_u64 = coset_weights_u64(n, COSET_OFFSET); + let weights_fp = coset_weights(n, COSET_OFFSET); + let inv_tw = + TwoHalfTwiddles::::new(log_n, true).expect("inv twiddles"); + let fwd_tw = + TwoHalfTwiddles::::new(log_lde, false).expect("fwd twiddles"); + + let (handle, _lde) = + math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep( + &row_major, + math_cuda::DeviceHash::Blake3, + n, + num_cols, + blowup, + &weights_u64, + true, + ) + .expect("fused ext3 BLAKE3 GPU pipeline"); + let gpu_root = handle.tree.as_ref().expect("resident merkle tree").root; + + let cpu_root = + cpu_ext3_row_major_blake3_root(&columns, blowup, &weights_fp, &inv_tw, &fwd_tw); + + assert_eq!( + gpu_root, cpu_root, + "BLAKE3 fused ext3 root mismatch: log_n={log_n} blowup={blowup} num_cols={num_cols}" + ); + } + } + } +} + +/// Negative control: one corrupted input element must move the device root. +/// Proves the equality assertions above compare live data, not fixed points. +#[test] +fn blake3_fused_tamper_diverges() { + const COSET_OFFSET: u64 = 7; + let n = 1usize << 6; + let num_cols = 3usize; + let mut rng = ChaCha8Rng::seed_from_u64(777); + + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rng.r#gen::()).collect()) + .collect(); + let weights_u64 = coset_weights_u64(n, COSET_OFFSET); + + let honest = gpu_fused_blake3_root(&columns, 2, &weights_u64); + + let mut tampered = columns.clone(); + tampered[1][n / 2] ^= 1; + let forged = gpu_fused_blake3_root(&tampered, 2, &weights_u64); + + assert_ne!( + honest, forged, + "a corrupted input element must move the root" + ); +} diff --git a/crypto/math-cuda/tests/blake3_leaves.rs b/crypto/math-cuda/tests/blake3_leaves.rs new file mode 100644 index 000000000..193d149ea --- /dev/null +++ b/crypto/math-cuda/tests/blake3_leaves.rs @@ -0,0 +1,386 @@ +//! Parity: the device BLAKE3 leaf kernels must reproduce the CPU prover's leaf +//! hashes byte for byte. +//! +//! Structural mirror of `keccak_leaves.rs`, and deliberately so: the leaf BYTE +//! layout does not move under P-a. `leaves_bit_reversed_grouped` +//! (`crypto/stark/src/commitment.rs:55`) serializes each element in canonical +//! big-endian, concatenates, and hashes the buffer once — the same bytes for +//! both hashes. What changes is only the hash over them, so the CPU reference +//! here is the *production* leaf function instantiated at the BLAKE3 backend +//! rather than a second implementation written for the test. +//! +//! That makes each assertion below a check of two things at once: that the +//! kernel's read pattern (bit reversal, column order, component order, row-pair +//! ordering) matches the CPU's, and that the device `Blake3Chain` matches the +//! host one over multi-block messages. +//! +//! Needs a GPU. See `RESUME-TRACKG.md` for the run command. + +mod blake3_reference; + +use blake3_reference::expected_device_rounds; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::merkle_tree::backends::types::{BatchBlake3Backend, PairBlake3Backend}; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsField; +use math::traits::{AsBytes, ByteConversion}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use stark::commitment::leaves_bit_reversed_grouped; +use stark::config::Commitment; + +type Fp = FieldElement; +type Fp3 = FieldElement; + +/// The CPU leaf hashes for `columns`, through the production leaf function at +/// the BLAKE3 batched backend. +fn cpu_leaves(columns: &[Vec>], rows_per_leaf: usize) -> Vec +where + E: IsField, + FieldElement: AsBytes + Sync + Send + ByteConversion, +{ + leaves_bit_reversed_grouped::>(columns, rows_per_leaf) +} + +/// ★ LOCKSTEP GUARD. `crypto`'s `blake3-6round` and `math-cuda`'s are separate +/// features and nothing forces them equal. Out of lockstep, every assertion in +/// this file compares a 6-round device tree against a 7-round host one (or the +/// reverse) and fails with a wall of unequal bytes that says nothing about the +/// cause. Failing here first names it. +/// +/// This is the same guard `blake3_reference`'s parent test carries, repeated +/// because a leaf-kernel failure has the same ambiguity and a developer running +/// only this file would not see the other one. +fn assert_round_lockstep() { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep: the GPU \ + kernels would commit under a different hash than the CPU backend. Set \ + both features or neither." + ); +} + +fn rand_base(rng: &mut ChaCha8Rng) -> Fp { + Fp::from_raw(rng.r#gen::()) +} + +fn rand_ext3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) +} + +/// Base columns into the contiguous `[col * stride + row]` slab the kernels read +/// — the layout `coset_lde_batch_base_into` writes to pinned staging. +fn base_slabs(columns: &[Vec], n: usize) -> Vec { + let mut flat = vec![0u64; columns.len() * n]; + for (c, col) in columns.iter().enumerate() { + for (r, e) in col.iter().enumerate() { + flat[c * n + r] = *e.value(); + } + } + flat +} + +/// Ext3 columns into three base slabs per column: `[col*3 + k]`, each a +/// contiguous slab of `n` u64s. +fn ext3_slabs(columns: &[Vec], n: usize) -> Vec { + let mut flat = vec![0u64; columns.len() * 3 * n]; + for (c, col) in columns.iter().enumerate() { + for (r, e) in col.iter().enumerate() { + flat[(c * 3) * n + r] = *e.value()[0].value(); + flat[(c * 3 + 1) * n + r] = *e.value()[1].value(); + flat[(c * 3 + 2) * n + r] = *e.value()[2].value(); + } + } + flat +} + +fn assert_leaves_eq(gpu: &[u8], cpu: &[Commitment], what: &str) { + assert_eq!(gpu.len(), cpu.len() * 32, "{what}: leaf count"); + for (i, expected) in cpu.iter().enumerate() { + assert_eq!( + &gpu[i * 32..(i + 1) * 32], + &expected[..], + "{what}: leaf {i} mismatch" + ); + } +} + +/// Column counts are chosen to straddle the 64-byte block boundary in both +/// directions: 8 base elements fill a block exactly, so 1/5/17/41 columns give +/// leaves that end mid-block, on a boundary, and several blocks in. That is +/// where a chaining bug lives — a kernel that compressed eagerly on fill, or +/// mis-set `CHUNK_START` on a later block, agrees with the host at one column +/// count and not at the next. +#[test] +fn blake3_leaves_base_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10, 12] { + for num_cols in [1usize, 5, 8, 17, 41] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(100 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_base(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 1); + let flat = base_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_base(&flat, n, num_cols, n, 1).unwrap(); + assert_leaves_eq(&gpu, &cpu, &format!("base log_n={log_n} cols={num_cols}")); + } + } +} + +#[test] +fn blake3_leaves_base_row_pair_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10, 12] { + for num_cols in [1usize, 5, 8, 17, 41] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(500 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_base(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 2); + assert_eq!(cpu.len(), n / 2); + let flat = base_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_base(&flat, n, num_cols, n, 2).unwrap(); + assert_leaves_eq( + &gpu, + &cpu, + &format!("base row-pair log_n={log_n} cols={num_cols}"), + ); + } + } +} + +/// Ext3 elements are three felts = six words, so they straddle block boundaries +/// on most column counts rather than only on a few — the case the word-granular +/// (rather than element-granular) block builder exists for. +#[test] +fn blake3_leaves_ext3_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + for num_cols in [1usize, 3, 11, 20] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(200 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 1); + let flat = ext3_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_ext3(&flat, n, num_cols, n, 1).unwrap(); + assert_leaves_eq(&gpu, &cpu, &format!("ext3 log_n={log_n} cols={num_cols}")); + } + } +} + +#[test] +fn blake3_leaves_ext3_row_pair_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + for num_cols in [1usize, 3, 11, 20] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(600 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 2); + assert_eq!(cpu.len(), n / 2); + let flat = ext3_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_ext3(&flat, n, num_cols, n, 2).unwrap(); + assert_leaves_eq( + &gpu, + &cpu, + &format!("ext3 row-pair log_n={log_n} cols={num_cols}"), + ); + } + } +} + +/// FRI leaves are 48 bytes — under one block — so this is the chain's degenerate +/// single-compression case: `flags = 0x0B`, `block_len = 48`. Same shape as a +/// Merkle parent at a different length, which is why it is worth pinning +/// separately from the multi-block leaves above. +#[test] +fn blake3_fri_leaves_matches_cpu() { + assert_round_lockstep(); + for log_lde in [2u32, 4, 6, 8, 10, 12] { + let lde_size = 1usize << log_lde; + let mut rng = ChaCha8Rng::seed_from_u64(400 + log_lde as u64); + let evals: Vec = (0..lde_size).map(|_| rand_ext3(&mut rng)).collect(); + + let cpu: Vec<[u8; 32]> = evals + .chunks_exact(2) + .map(|c| PairBlake3Backend::::hash_data(&[c[0], c[1]])) + .collect(); + + let mut evals_interleaved = vec![0u64; 3 * lde_size]; + for (i, e) in evals.iter().enumerate() { + evals_interleaved[i * 3] = *e.value()[0].value(); + evals_interleaved[i * 3 + 1] = *e.value()[1].value(); + evals_interleaved[i * 3 + 2] = *e.value()[2].value(); + } + let nodes = + math_cuda::blake3::build_fri_layer_tree_from_evals_ext3(&evals_interleaved).unwrap(); + let num_leaves = lde_size / 2; + let leaves_offset = (num_leaves - 1) * 32; + assert_leaves_eq( + &nodes[leaves_offset..leaves_offset + num_leaves * 32], + &cpu, + &format!("fri log_lde={log_lde}"), + ); + } +} + +/// The comp-poly kernel through the production keep path, checked at the leaf +/// layer: the resident node buffer's leaf half must be the CPU's row-pair leaves. +#[test] +fn blake3_comp_poly_leaves_matches_cpu() { + assert_round_lockstep(); + for log_lde in [2u32, 4, 6, 8, 10, 12] { + for num_parts in [1usize, 2, 5, 17] { + let lde_size = 1usize << log_lde; + let mut rng = ChaCha8Rng::seed_from_u64(300 + log_lde as u64 + num_parts as u64); + let parts: Vec> = (0..num_parts) + .map(|_| (0..lde_size).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + let cpu = cpu_leaves(&parts, 2); + + let parts_interleaved: Vec> = parts + .iter() + .map(|p| { + let mut v = vec![0u64; 3 * lde_size]; + for (i, e) in p.iter().enumerate() { + v[i * 3] = *e.value()[0].value(); + v[i * 3 + 1] = *e.value()[1].value(); + v[i * 3 + 2] = *e.value()[2].value(); + } + v + }) + .collect(); + let parts_slices: Vec<&[u64]> = + parts_interleaved.iter().map(|v| v.as_slice()).collect(); + + let tree = math_cuda::blake3::build_comp_poly_tree_from_evals_ext3_keep(&parts_slices) + .unwrap(); + let be = math_cuda::device::backend().unwrap(); + let stream = be.next_stream(); + let nodes: Vec = stream.clone_dtoh(&*tree.nodes).unwrap(); + let num_leaves = lde_size / 2; + let leaves_offset = (num_leaves - 1) * 32; + assert_leaves_eq( + &nodes[leaves_offset..leaves_offset + num_leaves * 32], + &cpu, + &format!("comp-poly log_lde={log_lde} parts={num_parts}"), + ); + } + } +} + +/// Row-major row-pair leaves. The CPU reference is the same +/// `leaves_bit_reversed_grouped(.., 2)` — over the column-major view of the same +/// buffer, which is exactly the equivalence the row-major kernel exists to +/// exploit (`commit_rows_bit_reversed` reads rows contiguously instead of +/// transposing). +#[test] +fn blake3_leaves_row_major_row_pair_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + for m in [1usize, 5, 8, 17] { + let n = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(800 + log_n as u64 + m as u64); + // Row-major: row r occupies `data[r*m .. r*m + m]`. + let data: Vec = (0..n * m).map(|_| rand_base(&mut rng)).collect(); + + let columns: Vec> = (0..m) + .map(|c| (0..n).map(|r| data[r * m + c]).collect()) + .collect(); + let cpu = cpu_leaves(&columns, 2); + + let raw: Vec = data.iter().map(|e| *e.value()).collect(); + let gpu = math_cuda::blake3::leaves_base_row_major_row_pair(&raw, m, n).unwrap(); + assert_leaves_eq(&gpu, &cpu, &format!("row-major log_n={log_n} m={m}")); + } + } +} + +/// The column-range variant, which is how preprocessed tables commit their +/// precomputed and multiplicity ranges to separate trees over one LDE. The +/// reference is the same function over just those columns — so this pins that +/// `m` stays the full row stride while only `[col_start, col_end)` is hashed. +#[test] +fn blake3_leaves_row_major_row_pair_range_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + let n = 1usize << log_n; + let m = 13usize; + let mut rng = ChaCha8Rng::seed_from_u64(900 + log_n as u64); + let data: Vec = (0..n * m).map(|_| rand_base(&mut rng)).collect(); + + // A split that is not on a block boundary either side of it. + for (col_start, col_end) in [(0usize, 5usize), (5, 13), (0, 13), (3, 4)] { + let columns: Vec> = (col_start..col_end) + .map(|c| (0..n).map(|r| data[r * m + c]).collect()) + .collect(); + let cpu = cpu_leaves(&columns, 2); + + let raw: Vec = data.iter().map(|e| *e.value()).collect(); + let gpu = math_cuda::blake3::leaves_base_row_major_row_pair_range( + &raw, m, col_start, col_end, n, + ) + .unwrap(); + assert_leaves_eq( + &gpu, + &cpu, + &format!("row-major range log_n={log_n} cols=[{col_start},{col_end})"), + ); + } + } +} + +/// ★ NEGATIVE CONTROL for the whole file. +/// +/// Every test above asserts device == host. All of them would pass just as well +/// if both sides were a constant, or if the kernel ignored its input entirely +/// and the CPU reference happened to be compared against itself. This asserts +/// the leaves actually depend on the data: two column sets differing in one +/// element must give different leaves, and distinct rows must give distinct +/// leaves. +#[test] +fn leaves_depend_on_the_data() { + assert_round_lockstep(); + let n = 64usize; + let num_cols = 5usize; + let mut rng = ChaCha8Rng::seed_from_u64(4242); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_base(&mut rng)).collect()) + .collect(); + + let flat = base_slabs(&columns, n); + let a = math_cuda::blake3::leaves_base(&flat, n, num_cols, n, 1).unwrap(); + + // Perturb one element and re-hash. + let mut perturbed = columns.clone(); + perturbed[2][7] += Fp::from(1u64); + let flat2 = base_slabs(&perturbed, n); + let b = math_cuda::blake3::leaves_base(&flat2, n, num_cols, n, 1).unwrap(); + assert_ne!(a, b, "a one-element change must move some leaf"); + + // And the leaves are not all the same digest. + let first = &a[0..32]; + assert!( + a.chunks_exact(32).any(|c| c != first), + "all leaves identical — the kernel is not reading its row index" + ); +} diff --git a/crypto/math-cuda/tests/blake3_merkle_gather.rs b/crypto/math-cuda/tests/blake3_merkle_gather.rs new file mode 100644 index 000000000..5afbed0e3 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_merkle_gather.rs @@ -0,0 +1,102 @@ +//! Parity: authentication paths gathered from a BLAKE3 tree must be the paths +//! the CPU `MerkleTree::get_proof_by_pos` returns. +//! +//! `merkle_gather_paths` is HASH-AGNOSTIC — it copies sibling nodes and never +//! hashes — so PA-PLAN §6.1 correctly says it needs no BLAKE3 twin, and none is +//! written. What is not free is the claim that it walks a BLAKE3 tree correctly: +//! that depends on `blake3::build_merkle_tree_on_device` laying nodes out in the +//! same order the keccak builder does, which is a property of the new code. This +//! file is that check, and it is why the gather is reused rather than twinned +//! *and tested*, rather than reused on the strength of the argument alone. +//! +//! Mirror of `merkle_gather.rs` with the tree builder swapped. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::{expected_device_rounds, merkle_parent}; +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +/// The host parent hash as a Merkle backend, so the CPU reference is the +/// production tree walk. Leaves arrive already hashed, so `hash_data` is +/// unreachable; it is wired to the same parent function rather than to +/// `unimplemented!()` so the backend stays a total function. +#[derive(Clone, Default)] +struct Blake3ParentBackend; + +impl IsMerkleTreeBackend for Blake3ParentBackend { + type Node = [u8; 32]; + type Data = [u8; 32]; + + fn hash_data(leaf: &Self::Data) -> Self::Node { + merkle_parent(leaf, leaf, expected_device_rounds()) + } + + fn hash_new_parent(a: &Self::Node, b: &Self::Node) -> Self::Node { + merkle_parent(a, b, expected_device_rounds()) + } +} + +fn run_gather_parity(log_n: u32, seed: u64) { + let leaves_len = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let leaves: Vec<[u8; 32]> = (0..leaves_len) + .map(|_| core::array::from_fn(|_| rng.r#gen::())) + .collect(); + let flat: Vec = leaves.iter().flatten().copied().collect(); + + // Build the BLAKE3 tree on device, then upload its nodes back as the + // resident buffer the gather reads. + let gpu_nodes_bytes = math_cuda::blake3::build_merkle_tree_on_device(&flat).unwrap(); + let cpu_tree = MerkleTree::::build_from_hashed_leaves(leaves).unwrap(); + + // A spread of positions: first, last, and random interior ones. + let mut positions: Vec = vec![0, (leaves_len - 1) as u32]; + let mut r = ChaCha8Rng::seed_from_u64(seed ^ 0xabcd); + for _ in 0..16usize.min(leaves_len) { + positions.push(r.gen_range(0..leaves_len) as u32); + } + + let be = math_cuda::device::backend().unwrap(); + let stream = be.next_stream(); + let nodes_dev = stream.clone_htod(&gpu_nodes_bytes).unwrap(); + stream.synchronize().unwrap(); + + let depth = log_n as usize; + let paths = + math_cuda::merkle::gather_merkle_paths_dev(&nodes_dev, leaves_len, &positions, &stream) + .unwrap(); + assert_eq!(paths.len(), positions.len() * depth * 32); + + for (q, &pos) in positions.iter().enumerate() { + let cpu_proof = cpu_tree.get_proof_by_pos(pos as usize).unwrap(); + assert_eq!( + cpu_proof.merkle_path.len(), + depth, + "depth mismatch at log_n={log_n} pos={pos}" + ); + for (level, cpu_node) in cpu_proof.merkle_path.iter().enumerate() { + assert_eq!( + &paths[(q * depth + level) * 32..(q * depth + level + 1) * 32], + &cpu_node[..], + "path node mismatch: log_n={log_n} pos={pos} level={level}" + ); + } + } +} + +#[test] +fn blake3_merkle_gather_small() { + for log_n in 1u32..=6 { + run_gather_parity(log_n, 200 + log_n as u64); + } +} + +#[test] +fn blake3_merkle_gather_large() { + run_gather_parity(18, 7777); +} diff --git a/crypto/math-cuda/tests/blake3_merkle_tree.rs b/crypto/math-cuda/tests/blake3_merkle_tree.rs new file mode 100644 index 000000000..562d2be15 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_merkle_tree.rs @@ -0,0 +1,118 @@ +//! Parity: the device BLAKE3 Merkle tree must equal the CPU tree node for node. +//! +//! The CPU side is the *production* tree walk — `MerkleTree::build_from_hashed_leaves` +//! over a backend whose only new code is `hash_new_parent` — so what this compares +//! is the parent compression and the node layout, not a second tree builder. +//! Both the per-level kernel and the single-block tail kernel are exercised: the +//! tail takes over once a level is no wider than the block, so a tree deep enough +//! to cross that threshold runs both, and the small trees run the tail alone. +//! +//! Mirrors `merkle_root_parity.rs` / `merkle_tree.rs` in structure. Needs a GPU. + +mod blake3_reference; + +use blake3_reference::{expected_device_rounds, merkle_parent}; +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math_cuda::blake3::build_merkle_tree_on_device; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +/// The host parent hash under test, wrapped as a Merkle backend so the CPU +/// reference is the production tree walk rather than a hand-rolled one. +/// +/// `hash_data` is unreachable here — leaves are supplied already hashed — and is +/// wired to the same parent function rather than to `unimplemented!()` so the +/// backend stays a total function if a later test does call it. +#[derive(Clone, Default)] +struct Blake3ParentBackend; + +impl IsMerkleTreeBackend for Blake3ParentBackend { + type Node = [u8; 32]; + type Data = [u8; 32]; + + fn hash_data(leaf: &Self::Data) -> Self::Node { + merkle_parent(leaf, leaf, expected_device_rounds()) + } + + fn hash_new_parent(a: &Self::Node, b: &Self::Node) -> Self::Node { + merkle_parent(a, b, expected_device_rounds()) + } +} + +fn random_leaves(count: usize, seed: u64) -> Vec<[u8; 32]> { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + (0..count) + .map(|_| core::array::from_fn(|_| rng.r#gen::())) + .collect() +} + +fn run_parity(log_num_leaves: u32, seed: u64) { + let num_leaves = 1usize << log_num_leaves; + let leaves = random_leaves(num_leaves, seed); + + let cpu = MerkleTree::::build_from_hashed_leaves(leaves.clone()).unwrap(); + let cpu_nodes = cpu.nodes(); + + let flat: Vec = leaves.iter().flatten().copied().collect(); + let gpu = build_merkle_tree_on_device(&flat).unwrap(); + + assert_eq!(cpu_nodes.len() * 32, gpu.len(), "node count"); + for (i, expected) in cpu_nodes.iter().enumerate() { + assert_eq!( + &gpu[i * 32..(i + 1) * 32], + &expected[..], + "node {i} mismatch at log_num_leaves = {log_num_leaves}" + ); + } +} + +/// Small trees: every level fits the block width, so the tail kernel builds the +/// whole tree in one launch. +#[test] +fn blake3_merkle_tree_small() { + for log in 1u32..=8 { + run_parity(log, 300 + log as u64); + } +} + +/// Deep enough that the per-level kernel runs first and hands over to the tail +/// partway up — the launch path a real commit takes. +#[test] +fn blake3_merkle_tree_medium() { + for log in [10u32, 12, 14] { + run_parity(log, 700 + log as u64); + } +} + +#[test] +fn blake3_merkle_tree_large() { + run_parity(18, 4242); +} + +/// The parent is `hash_bytes(left ‖ right)` — a plain library call at 7 rounds, +/// which is the property that makes the framing (`h = IV`, `t = 0`, +/// `block_len = 64`, `flags = CHUNK_START|CHUNK_END|ROOT`) externally anchored +/// rather than merely self-consistent. +/// +/// Only meaningful when the kernels are built for 7 rounds; under +/// `blake3-6round` there is nothing in the world that recomputes the parent, which +/// is exactly PA-PLAN §1.6's premise. +#[test] +fn the_parent_is_the_blake3_crate_at_seven_rounds() { + if expected_device_rounds() != 7 { + return; + } + let leaves = random_leaves(2, 31337); + let flat: Vec = leaves.iter().flatten().copied().collect(); + let gpu = build_merkle_tree_on_device(&flat).unwrap(); + + let mut msg = Vec::with_capacity(64); + msg.extend_from_slice(&leaves[0]); + msg.extend_from_slice(&leaves[1]); + assert_eq!( + &gpu[0..32], + blake3::hash(&msg).as_bytes(), + "a two-leaf root must be blake3::hash(left ‖ right)" + ); +} diff --git a/crypto/math-cuda/tests/blake3_reference/mod.rs b/crypto/math-cuda/tests/blake3_reference/mod.rs new file mode 100644 index 000000000..574b383bb --- /dev/null +++ b/crypto/math-cuda/tests/blake3_reference/mod.rs @@ -0,0 +1,189 @@ +//! The host BLAKE3 compression reference the device kernels are checked against. +//! +//! **Not a copy any more.** The compression function, the IV, the permutation +//! and the round-count constants are re-exported from `crypto::hash::blake3`, +//! which P-a Stage 1 made their single home — so the device kernels, the host +//! commitment backends and the in-circuit chip are now all checked against one +//! function rather than three transcriptions of it. `crypto` is a dev-dependency +//! of this crate, which is what makes the re-export legal: `prover` (the old +//! home) depends on this crate, so it could never have been imported here. +//! +//! What stays local is [`merkle_parent`] — the *framing* a device Merkle parent +//! uses, which is a property of the kernel, not of the primitive. It is checked +//! two ways: against the `blake3` crate at 7 rounds, and against the production +//! host backend at the build's round count, so the reference cannot drift from +//! either the standard or the thing the CPU prover actually commits with. +//! +//! Lives in a subdirectory of `tests/`, so cargo treats it as a shared module +//! the parity tests `mod blake3_reference;` rather than as a test binary of its +//! own. `#![allow(dead_code)]` because not every includer uses every item. + +#![allow(dead_code)] + +pub use crypto::hash::blake3::chain::FLAGS_ONE_BLOCK; +pub use crypto::hash::blake3::{ + BLAKE3_IV, BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS, blake3_compress_rounds, +}; + +/// A Merkle parent: one compression over the 64 bytes of two child digests, with +/// the digest read back out little-endian. The host `hash_new_parent` for a +/// BLAKE3 backend, and the reference for `blake3_merkle_level`. +pub fn merkle_parent(left: &[u8; 32], right: &[u8; 32], rounds: usize) -> [u8; 32] { + let mut m = [0u32; 16]; + for i in 0..8 { + m[i] = u32::from_le_bytes(left[4 * i..4 * i + 4].try_into().unwrap()); + m[i + 8] = u32::from_le_bytes(right[4 * i..4 * i + 4].try_into().unwrap()); + } + let out = blake3_compress_rounds(&BLAKE3_IV, &m, 0, 64, FLAGS_ONE_BLOCK, rounds); + let mut digest = [0u8; 32]; + for i in 0..8 { + digest[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + digest +} + +/// The round count the kernels are compiled for, as the Rust side of the +/// feature. Mirrors `math-cuda`'s `blake3-6round`, which build.rs turns into +/// `-DBLAKE3_ROUNDS=6`. +pub const fn expected_device_rounds() -> usize { + if cfg!(feature = "blake3-6round") { + BLAKE3_SIX_ROUNDS + } else { + BLAKE3_STANDARD_ROUNDS + } +} + +/// The compression function is shared now, but [`merkle_parent`]'s framing is +/// still written here, so it gets its own checks that it did not drift — +/// otherwise a device-vs-host parity failure would be ambiguous between "the +/// kernel is wrong" and "the reference is wrong". +/// +/// Host-only: no GPU, so these run wherever the suite compiles, including the +/// laptops where the kernels are stubbed out. +#[cfg(test)] +mod tests { + use super::*; + + /// At 7 rounds the reference must be the `blake3` crate, over every length a + /// single block can hold — `block_len` and the zero-padding both key off the + /// length, and one length would not see a port that ignored either. + #[test] + fn the_reference_copy_is_the_blake3_crate_at_seven_rounds() { + for len in 0..=64usize { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect(); + let mut block = [0u8; 64]; + block[..len].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + let out = blake3_compress_rounds( + &BLAKE3_IV, + &words, + 0, + len as u32, + FLAGS_ONE_BLOCK, + BLAKE3_STANDARD_ROUNDS, + ); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_eq!( + ours, + *blake3::hash(&msg).as_bytes(), + "reference copy diverged from the blake3 crate at length {len}" + ); + } + } + + /// And the parent framing on top of it: `hash_new_parent(a, b)` is + /// `blake3::hash(a ‖ b)` at 7 rounds. Pins `block_len = 64`, `t = 0`, + /// `h = IV`, the flag set, and the little-endian digest read-back in one shot. + #[test] + fn the_reference_parent_is_the_blake3_crate_at_seven_rounds() { + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(7)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(31).wrapping_add(3)); + let mut msg = Vec::with_capacity(64); + msg.extend_from_slice(&left); + msg.extend_from_slice(&right); + assert_eq!( + merkle_parent(&left, &right, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&msg).as_bytes() + ); + } + + /// ★ The reference parent is what the **host commitment backend** computes. + /// + /// The two checks above anchor the framing against the standard; this one + /// anchors it against the thing the CPU prover actually commits with, so a + /// GPU tree and a CPU tree over the same leaves are the same tree. Without + /// it, the device could be faithful to `blake3::hash(a ‖ b)` and still + /// disagree with the backend the proof is verified against. + /// + /// It runs at [`expected_device_rounds`], and so it doubles as the LOCKSTEP + /// alarm for the two crates' `blake3-6round` features: they are separate + /// features and nothing forces them equal, and a mismatch means a GPU tree + /// committing under a different hash than the CPU one. If this fails with + /// the round counts differing, set both features or neither — `make lint` + /// has a combined pass that compiles them together for the same reason. + #[test] + fn the_reference_parent_is_the_host_commitment_backend() { + use crypto::hash::blake3::BLAKE3_ROUNDS; + use crypto::merkle_tree::backends::types::BatchBlake3Backend; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + use math::field::goldilocks::GoldilocksField; + + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep: the GPU \ + kernels would commit under a different hash than the CPU backend" + ); + + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(11).wrapping_add(5)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(23)); + assert_eq!( + merkle_parent(&left, &right, expected_device_rounds()), + as IsMerkleTreeBackend>::hash_new_parent( + &left, &right + ), + ); + } + + /// NEGATIVE CONTROL: at 6 rounds neither must match, or the two checks above + /// would pass with `rounds` ignored. + #[test] + fn six_rounds_is_not_the_blake3_crate() { + let msg: [u8; 36] = core::array::from_fn(|i| i as u8); + let mut block = [0u8; 64]; + block[..36].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + let out = blake3_compress_rounds( + &BLAKE3_IV, + &words, + 0, + 36, + FLAGS_ONE_BLOCK, + BLAKE3_SIX_ROUNDS, + ); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_ne!(ours, *blake3::hash(&msg).as_bytes()); + + let left = [1u8; 32]; + let right = [2u8; 32]; + let mut pmsg = Vec::with_capacity(64); + pmsg.extend_from_slice(&left); + pmsg.extend_from_slice(&right); + assert_ne!( + merkle_parent(&left, &right, BLAKE3_SIX_ROUNDS), + *blake3::hash(&pmsg).as_bytes() + ); + } +} diff --git a/crypto/math-cuda/tests/blake3_serialize.rs b/crypto/math-cuda/tests/blake3_serialize.rs new file mode 100644 index 000000000..3ecd8ccc7 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_serialize.rs @@ -0,0 +1,125 @@ +//! Parity: the device field-element serialization and 64-byte block framing must +//! reproduce the bytes the CPU commit path hashes. +//! +//! The leaf byte encoding does not move under P-a: `leaves_bit_reversed_grouped` +//! (`crypto/stark/src/commitment.rs:55`) writes each element in canonical +//! big-endian form and concatenates, and `hash_bytes` hashes that buffer. BLAKE3 +//! reads a 64-byte block as 16 little-endian u32 words, so the device has to +//! transpose: one element becomes the byte-reverse of its canonical high half, +//! then of its low half. This pins that transposition, the canonicalisation in +//! front of it, and the block boundaries and zero-padding around it — everything +//! a leaf kernel needs that does not depend on the still-open chaining +//! construction (PA-PLAN §1.6). + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::traits::AsBytes; +use math_cuda::blake3::{blocks_of_felts, serialize_felts}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; + +const PRIME: u64 = 0xFFFF_FFFF_0000_0001; + +/// Raw values that include the ones canonicalisation is the only thing standing +/// between: `p` and above are representable in the prover's non-canonical u64 +/// form and serialize as their reduced value, so a kernel that skipped the +/// reduction would differ from the CPU on exactly these. +fn raws(seed: u64, n: usize) -> Vec { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let mut v = vec![ + 0u64, + 1, + PRIME - 1, + PRIME, + PRIME + 1, + PRIME + 12345, + u64::MAX, + ]; + v.truncate(n.min(7)); + while v.len() < n { + // Half in range, half deliberately non-canonical. + let x = rng.r#gen::(); + v.push(if v.len().is_multiple_of(2) { + x % PRIME + } else { + x + }); + } + v +} + +/// The bytes the CPU hashes for these elements, via the same `AsBytes` route +/// `leaves_bit_reversed_grouped` serializes through. +fn cpu_bytes(raws: &[u64]) -> Vec { + let mut out = Vec::with_capacity(raws.len() * 8); + for &r in raws { + out.extend_from_slice(&Fp::from_raw(r).as_bytes()); + } + out +} + +/// Those bytes as BLAKE3 message words, zero-padded to whole 64-byte blocks. +fn cpu_block_words(bytes: &[u8]) -> Vec { + let n_blocks = bytes.len().div_ceil(64); + let mut padded = bytes.to_vec(); + padded.resize(n_blocks * 64, 0); + padded + .chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect() +} + +#[test] +fn device_serialization_is_the_cpu_leaf_bytes() { + for n in [1usize, 2, 7, 8, 9, 64, 1000] { + let vals = raws(11 + n as u64, n); + let device = serialize_felts(&vals).unwrap(); + let expected = cpu_block_words(&cpu_bytes(&vals)); + // `serialize_felts` emits exactly two words per element with no padding, + // so compare against the unpadded prefix of the block view. + assert_eq!(device.len(), 2 * n); + assert_eq!( + device[..], + expected[..2 * n], + "serialization mismatch at n = {n}" + ); + } +} + +/// A field element is 8 bytes = 2 words, and a block is 16 words, so elements +/// straddle a block boundary only when the count is not a multiple of 8 — but +/// ext3 elements are 6 words and straddle routinely, which is why the builder +/// works at word granularity. Both cases are covered by the counts below. +#[test] +fn device_block_framing_matches_the_cpu_byte_stream() { + for n in [1usize, 3, 8, 9, 16, 17, 63, 64, 255] { + let vals = raws(500 + n as u64, n); + let device = blocks_of_felts(&vals).unwrap(); + let expected = cpu_block_words(&cpu_bytes(&vals)); + assert_eq!( + device.len(), + expected.len(), + "block count mismatch at n = {n}" + ); + assert_eq!(device, expected, "block words mismatch at n = {n}"); + } +} + +/// The tail block must be zero-padded, not left holding stale words. The check +/// above would catch that only if the padding happened to differ from whatever +/// was there; asserting the padded region directly is what makes it a test of the +/// padding rather than of the allocator. +#[test] +fn the_tail_block_is_zero_padded() { + // 9 elements = 18 words = one full block plus 2 words, leaving 14 to pad. + let vals = raws(77, 9); + let device = blocks_of_felts(&vals).unwrap(); + assert_eq!(device.len(), 32, "expected exactly two blocks"); + assert!( + device[18..].iter().all(|&w| w == 0), + "tail block not zero-padded: {:?}", + &device[18..] + ); +} diff --git a/crypto/math-cuda/tests/comp_poly_tree.rs b/crypto/math-cuda/tests/comp_poly_tree.rs index 51b826dd1..3646de764 100644 --- a/crypto/math-cuda/tests/comp_poly_tree.rs +++ b/crypto/math-cuda/tests/comp_poly_tree.rs @@ -167,6 +167,7 @@ fn run_parity(log_n: u32, blowup: usize, num_parts: usize, seed: u64) { math_cuda::lde::evaluate_poly_coset_batch_ext3_into_with_merkle_tree( &coefs_slices, + math_cuda::DeviceHash::Keccak256, n, blowup, &weights, diff --git a/crypto/math-cuda/tests/deep_hoisted.rs b/crypto/math-cuda/tests/deep_hoisted.rs new file mode 100644 index 000000000..e8501971f --- /dev/null +++ b/crypto/math-cuda/tests/deep_hoisted.rs @@ -0,0 +1,176 @@ +//! `LAMBDA_VM_DEEP_HOISTED`: the fused DEEP with its row-independent part +//! hoisted (`deep_composition_ext3_hoisted_m*`) against the fused kernel it +//! replaces, through `deep_composition_ext3_fused_keep` with the setting off +//! then on: the hoisted codeword is the fused one's canonical words, word for +//! word, at every instantiation (0..=3 eval points), with and without aux +//! columns; it is canonical throughout; and a row stride other than 1 keeps +//! the fused kernel. +//! +//! Needs a GPU, like every test in this directory. A binary of its own, so the +//! setting's override touches no other test. + +use std::sync::Arc; + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use math::field::traits::IsPrimeField; +use math_cuda::deep::{deep_composition_ext3_fused_keep, deep_hoisted_report, set_deep_hoisted}; +use math_cuda::device::backend; +use math_cuda::lde::{GpuLdeBase, GpuLdeExt3}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; +type Fp3 = FieldElement; + +fn rand_fp3(rng: &mut ChaCha8Rng) -> [u64; 3] { + let e = Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]); + [ + *e.value()[0].value(), + *e.value()[1].value(), + *e.value()[2].value(), + ] +} + +fn canon(xs: &[u64]) -> Vec { + xs.iter().map(Gl::canonical).collect() +} + +/// The fused DEEP's codeword (FRI order) on random inputs, `hoisted` set as +/// given; `row_stride` rows apart over an LDE `row_stride` times the domain. +#[allow(clippy::too_many_arguments)] +fn codeword( + domain: usize, + num_main: usize, + num_aux: usize, + parts: usize, + k: usize, + row_stride: usize, + seed: u64, + hoisted: bool, +) -> Vec { + let lde = domain * row_stride; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let be = backend().expect("cuda backend"); + let stream = be.next_stream(); + let main_flat: Vec = (0..num_main.max(1) * lde) + .map(|_| rng.r#gen::()) + .collect(); + let ext = |cols: usize, rng: &mut ChaCha8Rng| -> Vec { + (0..cols.max(1) * 3 * lde) + .map(|_| rng.r#gen::() % 0xFFFF_FFFF_0000_0001) + .collect() + }; + let aux_flat = ext(num_aux, &mut rng); + let parts_flat = ext(parts, &mut rng); + let main = GpuLdeBase { + ready: None, + buf: Arc::new(stream.clone_htod(&main_flat).expect("htod")), + m: num_main, + lde_size: lde, + tree: None, + trace_dev: None, + trace_rows: 0, + }; + let aux = (num_aux > 0).then(|| GpuLdeExt3 { + ready: None, + buf: Arc::new(stream.clone_htod(&aux_flat).expect("htod")), + m: num_aux, + lde_size: lde, + tree: None, + }); + let parts_dev = GpuLdeExt3 { + ready: None, + buf: Arc::new(stream.clone_htod(&parts_flat).expect("htod")), + m: parts, + lde_size: lde, + tree: None, + }; + let coset: Vec = (0..lde).map(|_| rng.r#gen::() | 1).collect(); + let coset_dev = stream.clone_htod(&coset).expect("htod"); + let cols = num_main + num_aux; + let z: Vec = (0..1 + k).flat_map(|_| rand_fp3(&mut rng)).collect(); + let h_ood: Vec = (0..parts).flat_map(|_| rand_fp3(&mut rng)).collect(); + let trace_ood: Vec = (0..cols * k).flat_map(|_| rand_fp3(&mut rng)).collect(); + let gammas_h: Vec = (0..parts).flat_map(|_| rand_fp3(&mut rng)).collect(); + let gammas_tr: Vec = (0..cols * k).flat_map(|_| rand_fp3(&mut rng)).collect(); + stream.synchronize().expect("sync"); + set_deep_hoisted(Some(hoisted)); + let out = deep_composition_ext3_fused_keep( + &stream, + &main, + aux.as_ref(), + &parts_dev, + &coset_dev, + &z, + &h_ood, + &trace_ood, + &gammas_h, + &gammas_tr, + parts, + num_main, + num_aux, + k, + row_stride, + domain, + ) + .expect("the fused DEEP runs") + .download() + .expect("download"); + set_deep_hoisted(None); + out +} + +#[test] +fn the_hoisted_codeword_is_the_fused_one_word_for_word() { + for (domain, num_main, num_aux, parts, k, seed) in [ + (1usize << 10, 5usize, 3usize, 2usize, 3usize, 1u64), + (1 << 10, 4, 0, 1, 0, 2), + (1 << 11, 3, 2, 2, 1, 3), + (1 << 9, 6, 4, 2, 2, 4), + (1 << 16, 23, 3, 2, 3, 5), + (1 << 12, 40, 6, 2, 3, 6), + ] { + let fused = codeword(domain, num_main, num_aux, parts, k, 1, seed, false); + let hoisted = codeword(domain, num_main, num_aux, parts, k, 1, seed, true); + let what = format!("domain {domain} main {num_main} aux {num_aux} parts {parts} k {k}"); + assert_eq!(hoisted.len(), fused.len(), "{what}: length"); + assert!( + hoisted.iter().all(|v| *v < 0xFFFF_FFFF_0000_0001), + "{what}: a non-canonical word" + ); + let want = canon(&fused); + if let Some(i) = hoisted.iter().zip(&want).position(|(a, b)| a != b) { + panic!( + "{what}: word {i} differs: {:#x} != {:#x} (fused raw {:#x})", + hoisted[i], want[i], fused[i] + ); + } + } + let line = deep_hoisted_report(); + assert!( + line.contains("hoisted launches 6"), + "six hoisted launches: {line}" + ); +} + +/// A row stride other than 1 keeps the fused kernel, counted, with the same +/// codeword. +#[test] +fn a_row_stride_other_than_one_keeps_the_fused_kernel() { + let before = deep_hoisted_report(); + let fused = codeword(1 << 9, 4, 2, 2, 2, 4, 9, false); + let on = codeword(1 << 9, 4, 2, 2, 2, 4, 9, true); + assert_eq!( + on, fused, + "the fused kernel both times, so the same raw words" + ); + let after = deep_hoisted_report(); + assert_ne!(before, after, "the fallback is counted: {before} → {after}"); + assert!(after.contains("fused kept"), "{after}"); +} diff --git a/crypto/math-cuda/tests/deep_inv_parity.rs b/crypto/math-cuda/tests/deep_inv_parity.rs new file mode 100644 index 000000000..60927ae27 --- /dev/null +++ b/crypto/math-cuda/tests/deep_inv_parity.rs @@ -0,0 +1,549 @@ +//! The row-wise DEEP/OOD denominator paths against the legacy paths they +//! replace by default (`math_cuda::deep_inv`). +//! +//! Each computes the same field elements as its legacy path, so each test runs +//! both on the same inputs and compares canonically — the device representation +//! is non-canonical, and a different operation order may land a value on `v` or +//! `v + p`, which nothing downstream can observe: +//! +//! - `invert_denoms_rowwise_dev` against the six-kernel scan +//! (`compute_and_invert_denoms_ext3_scan_dev`) and the definition, +//! `d · d⁻¹ = 1`, on a sample of rows; +//! - `deep_composition_ext3_fused_keep` against the buffered +//! `deep_composition_ext3_fully_resident_keep` fed the scan's inverses, and +//! against a host port of the DEEP sum; +//! - `barycentric_*_chunked_with_dev_inv_denoms` against the one-block-per- +//! column `barycentric_*_one_block_with_dev_inv_denoms`, at a nonzero offset +//! into a multi-point buffer. +//! +//! Every test names both of its paths, so none reads the process setting +//! (`LAMBDA_VM_DEEP_INV_LEGACY`); `tests/deep_inv_setting.rs` does. +//! +//! Needs a GPU, like every test in this directory. + +use std::sync::Arc; + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use math::field::traits::IsPrimeField; +use math_cuda::device::backend; +use math_cuda::inverse::{ + DenomSign, compute_and_invert_denoms_ext3_scan_dev, invert_denoms_rowwise_dev, +}; +use math_cuda::lde::{GpuLdeBase, GpuLdeExt3}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; +type Fp3 = FieldElement; + +fn rand_fp3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) +} + +fn raw3(e: &Fp3) -> [u64; 3] { + [ + *e.value()[0].value(), + *e.value()[1].value(), + *e.value()[2].value(), + ] +} + +fn flat3(xs: &[Fp3]) -> Vec { + xs.iter().flat_map(raw3).collect() +} + +fn canon(xs: &[u64]) -> Vec { + xs.iter().map(Gl::canonical).collect() +} + +fn from_raw3(x: &[u64]) -> Fp3 { + Fp3::new([Fp::from_raw(x[0]), Fp::from_raw(x[1]), Fp::from_raw(x[2])]) +} + +// --------------------------------------------------------------------------- +// Row-wise denominator inversion +// --------------------------------------------------------------------------- + +/// Row-wise == scan == host, canonically, for `k` points over `n` domain points +/// whose limbs include non-canonical values. +fn rowwise_matches_scan(n: usize, k: usize, sign: DenomSign, seed: u64) { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let x: Vec = (0..n).map(|_| rng.r#gen::()).collect(); + let z: Vec = (0..k).map(|_| rand_fp3(&mut rng)).collect(); + let z_raw = flat3(&z); + + let be = backend().unwrap(); + let stream = be.next_stream(); + let x_dev = stream.clone_htod(&x).unwrap(); + let scan = + compute_and_invert_denoms_ext3_scan_dev(&x_dev, &z_raw, n, k, sign, &stream).unwrap(); + let rowwise = invert_denoms_rowwise_dev(&x_dev, &z_raw, n, k, sign, &stream).unwrap(); + let scan: Vec = stream.clone_dtoh(&scan).unwrap(); + let rowwise: Vec = stream.clone_dtoh(&rowwise).unwrap(); + stream.synchronize().unwrap(); + assert_eq!(rowwise.len(), 3 * n * k); + assert_eq!( + canon(&rowwise), + canon(&scan), + "row-wise != scan at n={n} k={k} seed={seed}" + ); + + // And the definition, on a sample of rows: d · d^{-1} = 1. + for i in (0..n).step_by((n / 97).max(1)) { + for (kk, zk) in z.iter().enumerate() { + let xi = Fp::from_raw(x[i]).to_extension::(); + let d = match sign { + DenomSign::ZMinusX => zk - &xi, + DenomSign::XMinusZ => &xi - zk, + }; + let at = (kk * n + i) * 3; + assert_eq!( + d * from_raw3(&rowwise[at..at + 3]), + Fp3::one(), + "not an inverse at row {i}, point {kk}" + ); + } + } +} + +#[test] +fn rowwise_inversion_matches_the_scan() { + // Every instantiation, both signs; the R3 (trace, k points), R3 parts + // (k = 1) and R4 (k = 1 + points) shapes; a size that is not a multiple of + // the block. + for k in 1..=math_cuda::inverse::ROWWISE_MAX_K { + rowwise_matches_scan(1000 + k, k, DenomSign::ZMinusX, 10 + k as u64); + rowwise_matches_scan(777, k, DenomSign::XMinusZ, 20 + k as u64); + } + rowwise_matches_scan(1 << 16, 2, DenomSign::ZMinusX, 31); + rowwise_matches_scan(1 << 16, 1, DenomSign::ZMinusX, 32); + rowwise_matches_scan(1 << 18, 3, DenomSign::XMinusZ, 33); +} + +/// A zero denominator (a domain point equal to an opening point) zeroes the +/// row's inverses in release — the scan's release behaviour for its batch — +/// and leaves every other row exact. +#[cfg(not(any(debug_assertions, feature = "test-faults")))] +#[test] +fn rowwise_zero_denominator_zeroes_only_its_row() { + let n = 300; + let mut rng = ChaCha8Rng::seed_from_u64(5); + let mut x: Vec = (0..n).map(|_| rng.r#gen::()).collect(); + // Point 1 lies in the base field, and row 17 sits on it. + let z = [ + rand_fp3(&mut rng), + Fp3::new([Fp::from(42u64), Fp::zero(), Fp::zero()]), + ]; + x[17] = 42; + let be = backend().unwrap(); + let stream = be.next_stream(); + let x_dev = stream.clone_htod(&x).unwrap(); + let out = + invert_denoms_rowwise_dev(&x_dev, &flat3(&z), n, 2, DenomSign::XMinusZ, &stream).unwrap(); + let out: Vec = stream.clone_dtoh(&out).unwrap(); + stream.synchronize().unwrap(); + for k in 0..2 { + let at = (k * n + 17) * 3; + assert_eq!(canon(&out[at..at + 3]), vec![0, 0, 0], "row 17 point {k}"); + } + for i in [0usize, 16, 18, n - 1] { + let at = i * 3; + let d = Fp::from_raw(x[i]).to_extension::() - z[0]; + assert_eq!(d * from_raw3(&out[at..at + 3]), Fp3::one(), "row {i}"); + } +} + +/// The same input trips the debug guard, as the scan's zero-total guard does. +#[cfg(any(debug_assertions, feature = "test-faults"))] +#[test] +#[should_panic(expected = "a zero denominator has no inverse")] +fn rowwise_zero_denominator_trips_the_guard() { + let n = 300; + let mut rng = ChaCha8Rng::seed_from_u64(5); + let mut x: Vec = (0..n).map(|_| rng.r#gen::()).collect(); + let z = [ + rand_fp3(&mut rng), + Fp3::new([Fp::from(42u64), Fp::zero(), Fp::zero()]), + ]; + x[17] = 42; + let be = backend().unwrap(); + let stream = be.next_stream(); + let x_dev = stream.clone_htod(&x).unwrap(); + let _ = invert_denoms_rowwise_dev(&x_dev, &flat3(&z), n, 2, DenomSign::XMinusZ, &stream); +} + +// --------------------------------------------------------------------------- +// Fused DEEP +// --------------------------------------------------------------------------- + +struct DeepCase { + main: Vec>, + aux: Vec>, + parts: Vec>, + coset: Vec, + z: Vec, + h_ood: Vec, + trace_ood: Vec>, + gammas_h: Vec, + gammas_tr: Vec>, +} + +fn deep_case( + lde: usize, + num_main: usize, + num_aux: usize, + parts: usize, + k: usize, + seed: u64, +) -> DeepCase { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let total = num_main + num_aux; + DeepCase { + main: (0..num_main) + .map(|_| (0..lde).map(|_| rng.r#gen::()).collect()) + .collect(), + aux: (0..num_aux) + .map(|_| (0..lde).map(|_| rand_fp3(&mut rng)).collect()) + .collect(), + parts: (0..parts) + .map(|_| (0..lde).map(|_| rand_fp3(&mut rng)).collect()) + .collect(), + coset: (0..lde).map(|_| rng.r#gen::()).collect(), + z: (0..1 + k).map(|_| rand_fp3(&mut rng)).collect(), + h_ood: (0..parts).map(|_| rand_fp3(&mut rng)).collect(), + trace_ood: (0..total) + .map(|_| (0..k).map(|_| rand_fp3(&mut rng)).collect()) + .collect(), + gammas_h: (0..parts).map(|_| rand_fp3(&mut rng)).collect(), + gammas_tr: (0..total) + .map(|_| (0..k).map(|_| rand_fp3(&mut rng)).collect()) + .collect(), + } +} + +fn ext3_slabs(cols: &[Vec], lde: usize) -> Vec { + let mut out = vec![0u64; cols.len().max(1) * 3 * lde]; + for (c, col) in cols.iter().enumerate() { + for (r, v) in col.iter().enumerate() { + let [a, b, cc] = raw3(v); + out[(c * 3) * lde + r] = a; + out[(c * 3 + 1) * lde + r] = b; + out[(c * 3 + 2) * lde + r] = cc; + } + } + out +} + +/// Host DEEP at row `i` (natural order), the formula both kernels implement. +fn host_deep_row(case: &DeepCase, i: usize) -> Fp3 { + let x = Fp::from_raw(case.coset[i]).to_extension::(); + let inv: Vec = case.z.iter().map(|z| (&x - z).inv().unwrap()).collect(); + let mut acc = Fp3::zero(); + for (j, part) in case.parts.iter().enumerate() { + acc += case.gammas_h[j] * (part[i] - case.h_ood[j]) * inv[0]; + } + let k = case.z.len() - 1; + for (j, col) in case.main.iter().enumerate() { + let t = Fp::from_raw(col[i]).to_extension::(); + for kk in 0..k { + acc += case.gammas_tr[j][kk] * (&t - &case.trace_ood[j][kk]) * inv[1 + kk]; + } + } + for (j, col) in case.aux.iter().enumerate() { + let jj = case.main.len() + j; + for kk in 0..k { + acc += case.gammas_tr[jj][kk] * (col[i] - case.trace_ood[jj][kk]) * inv[1 + kk]; + } + } + acc +} + +fn fused_deep_matches_buffered( + lde: usize, + num_main: usize, + num_aux: usize, + parts: usize, + k: usize, + seed: u64, +) { + let case = deep_case(lde, num_main, num_aux, parts, k, seed); + let be = backend().unwrap(); + let stream = be.next_stream(); + + let mut main_flat = vec![0u64; num_main.max(1) * lde]; + for (c, col) in case.main.iter().enumerate() { + main_flat[c * lde..(c + 1) * lde].copy_from_slice(col); + } + let main = GpuLdeBase { + ready: None, + buf: Arc::new(stream.clone_htod(&main_flat).unwrap()), + m: num_main, + lde_size: lde, + tree: None, + trace_dev: None, + trace_rows: 0, + }; + let aux = (num_aux > 0).then(|| GpuLdeExt3 { + ready: None, + buf: Arc::new(stream.clone_htod(&ext3_slabs(&case.aux, lde)).unwrap()), + m: num_aux, + lde_size: lde, + tree: None, + }); + let parts_dev = GpuLdeExt3 { + ready: None, + buf: Arc::new(stream.clone_htod(&ext3_slabs(&case.parts, lde)).unwrap()), + m: parts, + lde_size: lde, + tree: None, + }; + let coset_dev = stream.clone_htod(&case.coset).unwrap(); + let z_raw = flat3(&case.z); + let h_ood = flat3(&case.h_ood); + let trace_ood: Vec = case.trace_ood.iter().flat_map(|c| flat3(c)).collect(); + let gammas_h = flat3(&case.gammas_h); + let gammas_tr: Vec = case.gammas_tr.iter().flat_map(|c| flat3(c)).collect(); + stream.synchronize().unwrap(); + + let fused = math_cuda::deep::deep_composition_ext3_fused_keep( + &stream, + &main, + aux.as_ref(), + &parts_dev, + &coset_dev, + &z_raw, + &h_ood, + &trace_ood, + &gammas_h, + &gammas_tr, + parts, + num_main, + num_aux, + k, + 1, + lde, + ) + .unwrap() + .download() + .unwrap(); + + // The buffered path: the scan's inverses, then the kernel that reads them. + // (It uploads the empty trace tables as they are, which the driver refuses + // at k = 0 — no table proves with zero points; the host sum below covers it.) + if k > 0 { + let inv = compute_and_invert_denoms_ext3_scan_dev( + &coset_dev, + &z_raw, + lde, + 1 + k, + DenomSign::XMinusZ, + &stream, + ) + .unwrap(); + let buffered = math_cuda::deep::deep_composition_ext3_fully_resident_keep( + &stream, + &main, + aux.as_ref(), + &parts_dev, + &inv, + &h_ood, + &trace_ood, + &gammas_h, + &gammas_tr, + parts, + num_main, + num_aux, + k, + 1, + lde, + ) + .unwrap() + .download() + .unwrap(); + assert_eq!( + canon(&fused), + canon(&buffered), + "fused != buffered DEEP at lde={lde} main={num_main} aux={num_aux} parts={parts} k={k}" + ); + } + + // The host sum on a sample of rows (the codeword is in FRI order: + // position p holds natural row bitrev(p)). + let log = lde.trailing_zeros(); + for p in (0..lde).step_by((lde / 61).max(1)) { + let i = p.reverse_bits() >> (usize::BITS - log); + assert_eq!( + from_raw3(&fused[p * 3..p * 3 + 3]), + host_deep_row(&case, i), + "fused DEEP != host at row {i}" + ); + } +} + +#[test] +fn fused_deep_matches_the_buffered_kernel() { + // Every instantiation (0..=3 eval points), with and without aux, one and + // two composition parts. + fused_deep_matches_buffered(1 << 10, 5, 3, 2, 2, 1); + fused_deep_matches_buffered(1 << 10, 4, 0, 1, 0, 2); + fused_deep_matches_buffered(1 << 11, 3, 2, 2, 1, 3); + fused_deep_matches_buffered(1 << 9, 6, 4, 2, 3, 4); + fused_deep_matches_buffered(1 << 16, 12, 5, 2, 2, 5); +} + +/// More points than the fused kernels take is a clean decline, not a launch. +#[test] +fn fused_deep_declines_past_its_cap() { + let lde = 1 << 8; + let k = math_cuda::deep::FUSED_MAX_EVAL_POINTS + 1; + let be = backend().unwrap(); + let stream = be.next_stream(); + let main = GpuLdeBase { + ready: None, + buf: Arc::new(stream.alloc_zeros::(lde).unwrap()), + m: 1, + lde_size: lde, + tree: None, + trace_dev: None, + trace_rows: 0, + }; + let parts_dev = GpuLdeExt3 { + ready: None, + buf: Arc::new(stream.alloc_zeros::(3 * lde).unwrap()), + m: 1, + lde_size: lde, + tree: None, + }; + let coset = stream.alloc_zeros::(lde).unwrap(); + let r = math_cuda::deep::deep_composition_ext3_fused_keep( + &stream, + &main, + None, + &parts_dev, + &coset, + &vec![0; 3 * (1 + k)], + &[0; 3], + &vec![0; 3 * k], + &[0; 3], + &vec![0; 3 * k], + 1, + 1, + 0, + k, + 1, + lde, + ); + assert!(r.is_err(), "{k} points must decline"); +} + +// --------------------------------------------------------------------------- +// Chunked single-point OOD sums +// --------------------------------------------------------------------------- + +fn chunked_bary_matches(log_n: u32, blowup: usize, cols: usize, seed: u64) { + use math_cuda::barycentric::{ + barycentric_base_chunked_with_dev_inv_denoms, + barycentric_base_one_block_with_dev_inv_denoms, + barycentric_ext3_chunked_with_dev_inv_denoms, + barycentric_ext3_one_block_with_dev_inv_denoms, + }; + let n = 1usize << log_n; + let lde = n * blowup; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let base: Vec = (0..cols * lde).map(|_| rng.r#gen::()).collect(); + let ext: Vec = (0..cols * 3 * lde).map(|_| rng.r#gen::()).collect(); + let points: Vec = (0..n).map(|_| rng.r#gen::()).collect(); + // Three points' inverses back to back; the sums read the middle one. + let inv: Vec = (0..3 * 3 * n).map(|_| rng.r#gen::()).collect(); + let offset = 3 * n; + + let be = backend().unwrap(); + let stream = be.next_stream(); + let main = GpuLdeBase { + ready: None, + buf: Arc::new(stream.clone_htod(&base).unwrap()), + m: cols, + lde_size: lde, + tree: None, + trace_dev: None, + trace_rows: 0, + }; + let aux = GpuLdeExt3 { + ready: None, + buf: Arc::new(stream.clone_htod(&ext).unwrap()), + m: cols, + lde_size: lde, + tree: None, + }; + let points_dev = stream.clone_htod(&points).unwrap(); + let inv_dev = stream.clone_htod(&inv).unwrap(); + stream.synchronize().unwrap(); + + let one_block = barycentric_base_one_block_with_dev_inv_denoms( + &stream, + &main, + blowup, + &points_dev, + &inv_dev, + offset, + n, + ) + .unwrap(); + let chunked = barycentric_base_chunked_with_dev_inv_denoms( + &stream, + &main, + blowup, + &points_dev, + &inv_dev, + offset, + n, + ) + .unwrap(); + assert_eq!( + canon(&chunked), + canon(&one_block), + "base sums, 2^{log_n} x {cols}" + ); + + let one_block = barycentric_ext3_one_block_with_dev_inv_denoms( + &stream, + &aux, + blowup, + &points_dev, + &inv_dev, + offset, + n, + ) + .unwrap(); + let chunked = barycentric_ext3_chunked_with_dev_inv_denoms( + &stream, + &aux, + blowup, + &points_dev, + &inv_dev, + offset, + n, + ) + .unwrap(); + assert_eq!( + canon(&chunked), + canon(&one_block), + "ext3 sums, 2^{log_n} x {cols}" + ); +} + +#[test] +fn chunked_single_point_sums_match_one_block_per_column() { + // The parts OOD's 1–2 columns at a size that chunks 64-fold and past the + // old cap, a rows-bound size, and a single-chunk one. + chunked_bary_matches(20, 2, 1, 1); + chunked_bary_matches(20, 4, 2, 2); + chunked_bary_matches(16, 2, 3, 3); + chunked_bary_matches(12, 2, 2, 4); +} diff --git a/crypto/math-cuda/tests/deep_inv_setting.rs b/crypto/math-cuda/tests/deep_inv_setting.rs new file mode 100644 index 000000000..cdd1e7ab3 --- /dev/null +++ b/crypto/math-cuda/tests/deep_inv_setting.rs @@ -0,0 +1,96 @@ +//! The DEEP/OOD denominators follow the process setting: inverted row-wise and +//! summed row-chunked by default, the legacy scan and one block per column +//! under `LAMBDA_VM_DEEP_INV_LEGACY=1` (`math_cuda::deep_inv::LEGACY_ENV`). +//! +//! ```text +//! cargo test --release -p math-cuda --test deep_inv_setting -- --nocapture +//! LAMBDA_VM_DEEP_INV_LEGACY=1 cargo test --release -p math-cuda --test deep_inv_setting -- --nocapture +//! ``` +//! +//! The parity tests name both paths per call (`tests/deep_inv_parity.rs`), so +//! none of them reads the setting; this one does. It is its own binary with one +//! test, so the process-wide counters move for its two calls alone, and the +//! once-per-process `[gpu] DEEP/OOD denominators: …` banner is printed by its +//! first call (visible under `--nocapture`). + +use std::sync::Arc; + +use math_cuda::barycentric::barycentric_ext3_on_device_with_dev_inv_denoms; +use math_cuda::deep_inv::{LEGACY_ENV, chunked_ood_sums, rowwise_inversions}; +use math_cuda::device::backend; +use math_cuda::inverse::{DenomSign, compute_and_invert_denoms_ext3_dev}; +use math_cuda::lde::GpuLdeExt3; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +#[test] +fn the_denominators_follow_the_process_setting() { + let setting = std::env::var(LEGACY_ENV).unwrap_or_else(|_| "".into()); + let legacy = setting == "1"; + let mut rng = ChaCha8Rng::seed_from_u64(7); + let mut elem = || rng.r#gen::() % P; + let be = backend().expect("cuda backend"); + let stream = be.next_stream(); + + // One inversion: two OOD points over a 2^12-point domain. + let n = 1usize << 12; + let points: Vec = (0..n).map(|_| elem()).collect(); + let z: Vec = (0..6).map(|_| elem()).collect(); + let points_dev = stream.clone_htod(&points).expect("upload the points"); + let before = rowwise_inversions(); + let inv = + compute_and_invert_denoms_ext3_dev(&points_dev, &z, n, 2, DenomSign::ZMinusX, &stream) + .expect("the inversion"); + let rowwise = rowwise_inversions() - before; + + // One single-point OOD sum: the second point's inverses over one ext3 + // column at blowup 2. + let blowup = 2; + let column: Vec = (0..3 * n * blowup).map(|_| elem()).collect(); + let aux = GpuLdeExt3 { + ready: None, + buf: Arc::new(stream.clone_htod(&column).expect("upload the column")), + m: 1, + lde_size: n * blowup, + tree: None, + }; + let before = chunked_ood_sums(); + barycentric_ext3_on_device_with_dev_inv_denoms( + &stream, + &aux, + blowup, + &points_dev, + &inv, + 3 * n, + n, + ) + .expect("the OOD sum"); + let chunked = chunked_ood_sums() - before; + stream.synchronize().expect("sync"); + + println!( + "deep_inv_setting: {LEGACY_ENV}={setting}: the inversion took the {}, the OOD sum took {}", + if rowwise == 1 { + "row-wise kernel" + } else { + "scan" + }, + if chunked == 1 { + "the chunked kernel" + } else { + "one block per column" + }, + ); + assert_eq!( + rowwise, + u64::from(!legacy), + "{LEGACY_ENV}={setting}: the inversion did not follow the process setting" + ); + assert_eq!( + chunked, + u64::from(!legacy), + "{LEGACY_ENV}={setting}: the OOD sum did not follow the process setting" + ); +} diff --git a/crypto/math-cuda/tests/fraction_tree_padded.rs b/crypto/math-cuda/tests/fraction_tree_padded.rs new file mode 100644 index 000000000..fe2da8a81 --- /dev/null +++ b/crypto/math-cuda/tests/fraction_tree_padded.rs @@ -0,0 +1,90 @@ +//! A fraction tree that does not carry its padding against one that does. +//! +//! Runs on the merge-queue GPU box via `make test-math-cuda` — the tree needs a +//! real device, like the other tests here. +//! +//! The interactions are padded up to a power of two with the fraction `0/1`, +//! and the widest precompiles are barely over the power below, so that padding +//! is half the layer. The tree built from the truncated layer has to be the +//! same tree: what is compared is the output fraction, which is a product and +//! sum chain over every cell of every level — a level that differed anywhere +//! would have to collide in the field to come out equal here. + +use cudarc::driver::CudaStream; +use math_cuda::device::backend; +use math_cuda::gkr::DeviceFractionTree; +use std::sync::Arc; + +/// Fractions with no structure a kernel could accidentally satisfy. +fn layer(cells: usize, seed: u64) -> Vec { + (0..cells as u64) + .flat_map(|i| { + let mix = + |k: u64| (i.wrapping_mul(6364136223846793005 + k).wrapping_add(seed) >> 9) + 1; + [mix(1), mix(7), mix(13)] + }) + .collect() +} + +/// `real` fractions of a cube of `full`, with the rest written out as the +/// `0/1` the padding is. +fn padded(real: usize, full: usize, seed: u64) -> (Vec, Vec) { + let mut p = layer(real, seed); + let mut q = layer(real, seed ^ 0x5eed); + for _ in real..full { + p.extend_from_slice(&[0, 0, 0]); + q.extend_from_slice(&[1, 0, 0]); + } + (p, q) +} + +fn upload(stream: &Arc, values: &[u64]) -> cudarc::driver::CudaSlice { + math_cuda::device::htod_or_trim(stream, values).expect("upload") +} + +#[test] +fn a_tree_that_drops_its_padding_is_the_same_tree() { + let Ok(be) = backend() else { + eprintln!("no device; skipping"); + return; + }; + // 5 interactions over 8 rows: rounded up to 8 slots, so three eighths of + // the cube is padding — the shape of a precompile, in miniature. + for (interactions, rows) in [(5usize, 8usize), (3, 4), (7, 32), (1, 16)] { + let slots = interactions.next_power_of_two(); + let full = slots * rows; + let real = interactions * rows; + let (p, q) = padded(real, full, 0x1234 + interactions as u64); + + let stream = be.next_stream(); + let carried = DeviceFractionTree::from_device( + stream.clone(), + upload(&stream, &p), + upload(&stream, &q), + ) + .expect("the tree that carries its padding"); + + let stream = be.next_stream(); + let dropped = DeviceFractionTree::from_padded_input( + stream.clone(), + upload(&stream, &p[..real * 3]), + upload(&stream, &q[..real * 3]), + real, + full.trailing_zeros() as usize, + ) + .expect("the tree that drops it"); + + assert_eq!( + carried.num_layers(), + dropped.num_layers(), + "{interactions}x{rows}: same tree, same levels" + ); + assert_eq!( + carried.output().expect("output"), + dropped.output().expect("output"), + "{interactions}x{rows}: the output fraction differs" + ); + assert!(carried.holds_input(), "the first one keeps its input layer"); + assert!(!dropped.holds_input(), "the second one gives it back"); + } +} diff --git a/crypto/math-cuda/tests/fri_group_tree.rs b/crypto/math-cuda/tests/fri_group_tree.rs new file mode 100644 index 000000000..5bdd51d42 --- /dev/null +++ b/crypto/math-cuda/tests/fri_group_tree.rs @@ -0,0 +1,336 @@ +//! S3 group-leaf FRI layers on the device. +//! +//! - The group-leaf trees (`build_fri_group_tree_from_evals_ext3`, the kernels +//! `FriCommitState::fold_and_commit_group` commits with) equal the host tree +//! node for node: leaf `g` = the configuration's `Batched` leaf over the +//! `2^d` consecutive values from `g·2^d`, parents the pair hash. Keccak and +//! Blake3 against the host backends, d = 1..=6 at several sizes. +//! - Against the checked-in S3 vectors (`crypto/stark/tests/vectors/zf_fri`): +//! (c) the first-leaf digest and the layer root of the KAT codeword for +//! d = 1..=6 under Keccak, Blake3 AND RPX (the host RPX backend lives in the +//! prover crate; the vector is its output); (b) the KAT codeword folded d +//! times on the device with ζ, ζ², … equals the vector's `folded`. +//! - At d = 1 the group kernel IS the legacy pair-leaf kernel (the two-element +//! invariant, on the device), under all three hashes. +//! +//! Needs a GPU. + +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsStreamingLeafBackend; +use math::fft::bit_reversing::in_place_bit_reverse_permute; +use math::fft::roots_of_unity::get_powers_of_primitive_root_coset; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math_cuda::DeviceHash; +use math_cuda::fri::{FriCommitState, build_fri_group_tree_from_evals_ext3}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use stark::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Fp = FieldElement; +type Fp3 = FieldElement; + +fn vectors_dir() -> std::path::PathBuf { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../stark/tests/vectors/zf_fri") +} + +fn read_vector(name: &str) -> String { + std::fs::read_to_string(vectors_dir().join(name)).unwrap_or_else(|e| panic!("{name}: {e}")) +} + +/// Every decimal integer in `s`, in order (the vectors' ext limbs). +fn u64s(s: &str) -> Vec { + let mut out = Vec::new(); + let mut cur: Option = None; + for ch in s.chars() { + match ch.to_digit(10) { + Some(d) => cur = Some(cur.unwrap_or(0) * 10 + u64::from(d)), + None => { + if let Some(v) = cur.take() { + out.push(v); + } + } + } + } + if let Some(v) = cur { + out.push(v); + } + out +} + +/// The value of `"key": [...]` on `line`, up to the bracket that closes it. +fn json_array<'a>(line: &'a str, key: &str) -> &'a str { + let start = line + .find(&format!("\"{key}\": [")) + .unwrap_or_else(|| panic!("no {key}")) + + key.len() + + 4; + let mut depth = 0i32; + for (i, ch) in line[start..].char_indices() { + match ch { + '[' => depth += 1, + ']' => { + depth -= 1; + if depth == 0 { + return &line[start..start + i + 1]; + } + } + _ => {} + } + } + panic!("unterminated {key}") +} + +fn hex32(s: &str) -> [u8; 32] { + let mut out = [0u8; 32]; + for (i, b) in out.iter_mut().enumerate() { + *b = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).expect("hex"); + } + out +} + +/// The vector (b) KAT codeword as interleaved limbs, and per d its ζ and the +/// folded codeword. +#[allow(clippy::type_complexity)] +fn fold_vector() -> (Vec, Vec<(u32, [u64; 3], Vec)>) { + let text = read_vector("b_group_folds.json"); + let codeword_line = text + .lines() + .find(|l| l.trim_start().starts_with("\"codeword\"")) + .expect("codeword line"); + let codeword = u64s(json_array(codeword_line, "codeword")); + assert_eq!(codeword.len(), 3 * 128); + let mut folds = Vec::new(); + for line in text.lines().filter(|l| l.contains("\"folded\"")) { + let d = u64s(&line[..line.find("\"zeta\"").expect("zeta")])[0] as u32; + let z = u64s(json_array(line, "zeta")); + let folded = u64s(json_array(line, "folded")); + assert_eq!(folded.len(), 3 * (128 >> d)); + folds.push((d, [z[0], z[1], z[2]], folded)); + } + assert_eq!(folds.len(), 6); + (codeword, folds) +} + +/// The vector (c) digests for `hash`: per d, (first leaf, layer root). +fn leaf_vector(hash: &str) -> Vec<(u32, [u8; 32], [u8; 32])> { + let text = read_vector(&format!("c_leaf_digests_{hash}.json")); + let mut out = Vec::new(); + for line in text.lines().filter(|l| l.contains("\"first_leaf\"")) { + let field = |key: &str| { + let at = line.find(&format!("\"{key}\": \"")).expect(key) + key.len() + 5; + hex32(&line[at..at + 64]) + }; + let d = u64s(&line[..line.find("\"first_leaf\"").expect("first_leaf")])[0] as u32; + out.push((d, field("first_leaf"), field("layer_root"))); + } + assert_eq!(out.len(), 6, "{hash}: d = 1..=6"); + out +} + +fn limbs(v: &[Fp3]) -> Vec { + v.iter() + .flat_map(|e| { + let c = e.value(); + [c[0].canonical(), c[1].canonical(), c[2].canonical()] + }) + .collect() +} + +fn from_limbs(v: &[u64]) -> Vec { + v.chunks_exact(3) + .map(|c| Fp3::new([Fp::from(c[0]), Fp::from(c[1]), Fp::from(c[2])])) + .collect() +} + +/// The host group tree: `H::Batched` leaves over consecutive groups, the pair +/// hash above (as `stark::fri::group_tree`). +fn host_group_nodes(evals: &[Fp3], group: usize) -> Vec<[u8; 32]> { + let leaves: Vec<[u8; 32]> = evals + .chunks_exact(group) + .map(|g| as IsStreamingLeafBackend>::hash_data_from_slices(g, &[])) + .collect(); + MerkleTree::>::build_from_hashed_leaves(leaves) + .expect("tree") + .nodes() + .to_vec() +} + +fn assert_nodes_eq(device: &[u8], host: &[[u8; 32]], what: &str) { + assert_eq!(device.len(), host.len() * 32, "{what}: node count"); + for (i, h) in host.iter().enumerate() { + assert_eq!(&device[i * 32..(i + 1) * 32], &h[..], "{what}: node {i}"); + } +} + +fn random_evals(n: usize, seed: u64) -> Vec { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + (0..n) + .map(|_| { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) + }) + .collect() +} + +fn host_parity(hash: DeviceHash, name: &str) { + for d in 1..=6u32 { + for extra in [1u32, 4, 9] { + let n = 1usize << (d + extra); + let evals = random_evals(n, 1000 + u64::from(d * 16 + extra)); + let raw: Vec = evals + .iter() + .flat_map(|e| { + let c = e.value(); + [*c[0].value(), *c[1].value(), *c[2].value()] + }) + .collect(); + let device = build_fri_group_tree_from_evals_ext3(&raw, d, hash).expect("device"); + assert_nodes_eq( + &device, + &host_group_nodes::(&evals, 1 << d), + &format!("{name} d={d} n=2^{}", d + extra), + ); + } + } +} + +#[test] +fn group_tree_matches_host_keccak() { + host_parity::(DeviceHash::Keccak256, "keccak"); +} + +#[test] +fn group_tree_matches_host_blake3() { + host_parity::(DeviceHash::Blake3, "blake3"); +} + +#[test] +fn group_tree_matches_the_leaf_vectors() { + let (codeword, _) = fold_vector(); + for (hash, name) in [ + (DeviceHash::Keccak256, "keccak"), + (DeviceHash::Blake3, "blake3"), + (DeviceHash::Rpx256, "rpx"), + ] { + for (d, first_leaf, root) in leaf_vector(name) { + let nodes = build_fri_group_tree_from_evals_ext3(&codeword, d, hash).expect("device"); + let num_leaves = 128usize >> d; + let leaf0 = (num_leaves - 1) * 32; + assert_eq!( + &nodes[leaf0..leaf0 + 32], + &first_leaf, + "{name} d={d}: first leaf" + ); + assert_eq!(&nodes[..32], &root, "{name} d={d}: layer root"); + } + } +} + +#[test] +fn group_of_two_is_the_pair_leaf_kernel() { + let evals = random_evals(1 << 12, 77); + let raw = limbs(&evals); + for (hash, pair) in [ + ( + DeviceHash::Keccak256, + math_cuda::merkle::build_fri_layer_tree_from_evals_ext3(&raw).expect("keccak"), + ), + ( + DeviceHash::Blake3, + math_cuda::blake3::build_fri_layer_tree_from_evals_ext3(&raw).expect("blake3"), + ), + ( + DeviceHash::Rpx256, + math_cuda::rpx::build_fri_layer_tree_from_evals_ext3(&raw).expect("rpx"), + ), + ] { + let group = build_fri_group_tree_from_evals_ext3(&raw, 1, hash).expect("group"); + assert_eq!(group, pair, "{hash:?}: d = 1 group tree != pair tree"); + } +} + +/// `compute_coset_twiddles_inv`: the inverses of the coset points at the even +/// bit-reversed positions (`o·ω^i`, `i < n/2`, bit-reversed, inverted). +fn fold_twiddles(offset: u64, n: usize) -> Vec { + let mut pts = get_powers_of_primitive_root_coset::( + n.trailing_zeros() as u64, + n / 2, + &Fp::from(offset), + ) + .expect("roots"); + in_place_bit_reverse_permute(&mut pts); + pts.iter() + .map(|p| p.inv().expect("nonzero").canonical()) + .collect() +} + +fn zeta_powers(z: [u64; 3], d: u32) -> Vec<[u64; 3]> { + let mut zeta = from_limbs(&z)[0]; + let mut out = Vec::new(); + for level in 0..d { + out.push(limbs(&[zeta]).try_into().expect("3 limbs")); + if level + 1 < d { + zeta = zeta.square(); + } + } + out +} + +/// (b): the device folds (`fold_to_host`, the S3 terminal step) reproduce the +/// vector's `folded` for d = 1..=6; and `fold_and_commit_group` with no fold +/// commits the KAT codeword itself to the (c) root, with d − 1 folds then a +/// group of 2 to the root of the folded codeword's pair tree. +#[test] +fn device_folds_match_the_fold_vector() { + let (codeword, folds) = fold_vector(); + let tw = fold_twiddles(3, 128); + for (d, z, folded) in &folds { + let mut st = + FriCommitState::new(&codeword, &tw, 128, DeviceHash::Keccak256).expect("state"); + let out = st.fold_to_host(&zeta_powers(*z, *d)).expect("fold"); + assert_eq!(limbs(&from_limbs(&out)), *folded, "d={d}: folded codeword"); + } + for (hash, name) in [ + (DeviceHash::Keccak256, "keccak"), + (DeviceHash::Blake3, "blake3"), + (DeviceHash::Rpx256, "rpx"), + ] { + for (d, _, root) in leaf_vector(name) { + let mut st = FriCommitState::new(&codeword, &tw, 128, hash).expect("state"); + let (host, _, tree) = st.fold_and_commit_group(&[], d, true).expect("commit"); + assert_eq!(tree.root, root, "{name} d={d}: zero-fold group commit root"); + assert_eq!(tree.leaves_len, 128 >> d); + assert_eq!( + host.expect("drained"), + codeword, + "{name} d={d}: layer evals" + ); + } + // Folds then a commit: d folds, then groups of 2 over the result. + for (d, z, folded) in folds.iter().filter(|(d, _, _)| *d <= 5) { + let mut st = FriCommitState::new(&codeword, &tw, 128, hash).expect("state"); + let (host, _, tree) = st + .fold_and_commit_group(&zeta_powers(*z, *d), 1, true) + .expect("commit"); + let host = host.expect("drained"); + assert_eq!( + limbs(&from_limbs(&host)), + *folded, + "{name} d={d}: layer evals" + ); + let expect = build_fri_group_tree_from_evals_ext3(folded, 1, hash).expect("tree"); + assert_eq!( + &tree.root[..], + &expect[..32], + "{name} d={d}: folded layer root" + ); + } + } +} diff --git a/crypto/math-cuda/tests/fri_layer_tree.rs b/crypto/math-cuda/tests/fri_layer_tree.rs index ad524b59a..78772815a 100644 --- a/crypto/math-cuda/tests/fri_layer_tree.rs +++ b/crypto/math-cuda/tests/fri_layer_tree.rs @@ -7,7 +7,7 @@ use math::field::goldilocks::GoldilocksField; use math_cuda::merkle::build_fri_layer_tree_from_evals_ext3; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha8Rng; -use stark::config::FriLayerMerkleTree; +use stark::config::KeccakFriLayerMerkleTree as FriLayerMerkleTree; type Fp = FieldElement; type Fp3 = FieldElement; diff --git a/crypto/math-cuda/tests/gkr_gruen_truncated.rs b/crypto/math-cuda/tests/gkr_gruen_truncated.rs new file mode 100644 index 000000000..95f242e77 --- /dev/null +++ b/crypto/math-cuda/tests/gkr_gruen_truncated.rs @@ -0,0 +1,652 @@ +//! A GKR input layer kept on the card without its padding against the same +//! layer stored whole (`DeviceFractionTree::from_truncated_input`, +//! `LAMBDA_VM_ARGUE_INPUT_REAL`). +//! +//! ```text +//! cargo test --release -p math-cuda --test gkr_gruen_truncated -- --test-threads=1 +//! ``` +//! +//! Needs a GPU, like the other tests here. The interactions are padded up to a +//! power of two with `0/1`; a kept layer stores only the real fractions and its +//! rounds read the padding as those constants. Everything the card hands back +//! — the tree's output and levels, every round's sums, the halves after the +//! last round — is compared limb for limb with today's: the kept layer is +//! meant to be raw-identical, not just equal. +//! +//! The ignored test is the card micro-benchmark: the input layer's write, the +//! tree's folds and the input layer's own rounds at a CPU table's shape and a +//! Keccak-round table's, whole against kept. + +use std::sync::Arc; + +use cudarc::driver::{CudaSlice, CudaStream}; +use math_cuda::device::backend; +use math_cuda::gkr::{DeviceFractionTree, GruenLayer, InputPlan}; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|e| e.into_inner()) +} + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + /// A raw limb, every eighth one a non-canonical representative in + /// `[p, 2^64)` — what the card may hold. + fn limb(&mut self) -> u64 { + let v = self.next() % P; + if self.next() & 7 == 0 && v < 0xFFFF_FFFF { + v + P + } else { + v + } + } + + fn ext(&mut self) -> [u64; 3] { + [self.limb(), self.limb(), self.limb()] + } +} + +/// An input layer of `interactions` slots rounded up to a power of two, `rows` +/// a slot, the slot in the high bits: random fractions, the padding written +/// as today writes it (`p` 0, `q` 1). +fn padded_layer(interactions: usize, rows: usize, seed: u64) -> (Vec, Vec) { + let full = interactions.next_power_of_two() * rows; + let real = interactions * rows; + let mut rng = Rng(seed | 1); + let mut p = Vec::with_capacity(full * 3); + let mut q = Vec::with_capacity(full * 3); + for _ in 0..real { + p.extend_from_slice(&rng.ext()); + q.extend_from_slice(&rng.ext()); + } + for _ in real..full { + p.extend_from_slice(&[0, 0, 0]); + q.extend_from_slice(&[1, 0, 0]); + } + (p, q) +} + +fn upload(stream: &Arc, values: &[u64]) -> CudaSlice { + math_cuda::device::htod_or_trim(stream, values).expect("upload") +} + +/// Both trees over one layer: today's, carrying the padding, and the kept one. +fn both_trees( + interactions: usize, + rows: usize, + seed: u64, +) -> (DeviceFractionTree, DeviceFractionTree, usize, usize) { + let be = backend().expect("a device"); + let full = interactions.next_power_of_two() * rows; + let real = interactions * rows; + let (p, q) = padded_layer(interactions, rows, seed); + let stream = be.next_stream(); + let whole = + DeviceFractionTree::from_device(stream.clone(), upload(&stream, &p), upload(&stream, &q)) + .expect("today's tree"); + let stream = be.next_stream(); + let kept = DeviceFractionTree::from_truncated_input( + stream.clone(), + upload(&stream, &p[..real * 3]), + upload(&stream, &q[..real * 3]), + real, + full.trailing_zeros() as usize, + ) + .expect("the kept tree"); + (whole, kept, real, full) +} + +/// Every round of a layer's Gruen session and its finish, raw: the sums of +/// each round, then the halves. +fn run_layer( + layer: &mut GruenLayer, + lambda: &[u64; 3], + challenges: &[[u64; 3]], + h0: impl Fn(usize) -> bool, +) -> (Vec>, Vec) { + let mut sums = Vec::new(); + for j in 0..layer.rounds() { + let fold = (j > 0).then(|| &challenges[j - 1]); + sums.push(layer.round(fold, lambda, h0(j)).expect("a round")); + } + let last = (layer.rounds() > 0).then(|| &challenges[layer.rounds() - 1]); + (sums, layer.finish(last).expect("the finish")) +} + +/// ★ A kept tree is today's tree: the same output and levels, and its input +/// layer's Gruen rounds hand back today's sums every round and today's halves +/// at the end, limb for limb — at shapes whose padding starts one row into +/// the upper halves, at a precompile's three eighths, at a CPU table's twelve +/// slots of 32 and a Keccak-round table's 1,017 of 2,048, with no padding, +/// for every host tail and with `H(0)` summed on the card or not. +#[test] +fn a_kept_layer_runs_todays_rounds_limb_for_limb() { + let _serial = serial(); + if backend().is_err() { + eprintln!("no device; skipping"); + return; + } + let shapes: [(usize, usize); 9] = [ + (5, 8), + (3, 4), + (7, 32), + (17, 1 << 6), + (20, 1 << 10), + (1031, 1 << 3), + (9, 1 << 11), + (8, 1 << 9), + (1025, 1), + ]; + let mut checked = 0; + for (n, &(interactions, rows)) in shapes.iter().enumerate() { + for (low, h0) in [(6usize, 0u8), (0, 1), (9, 2), (2, 0)] { + let what = format!("{interactions} × {rows} rows, low {low}, h0 {h0}"); + let seed = 0xA5A5 + (n * 16 + low) as u64; + let (whole, kept, real, full) = both_trees(interactions, rows, seed); + assert_eq!( + whole.num_layers(), + kept.num_layers(), + "{what}: the same levels" + ); + assert_eq!( + whole.output().unwrap(), + kept.output().unwrap(), + "{what}: the output fraction" + ); + assert!( + kept.holds_input(), + "{what}: the kept tree holds its input layer" + ); + assert_eq!( + kept.input_truncated(), + real < full, + "{what}: kept without its padding" + ); + let input = whole.num_layers() - 1; + if input >= 1 { + assert_eq!( + whole.layer_to_host(input - 1).unwrap(), + kept.layer_to_host(input - 1).unwrap(), + "{what}: the level above the input layer, raw" + ); + } + let m = whole.layer_num_vars(input) - 1; + let mut rng = Rng(seed ^ 0x77); + let point: Vec = (0..m).flat_map(|_| rng.ext()).collect(); + let lambda = rng.ext(); + let challenges: Vec<[u64; 3]> = (0..m).map(|_| rng.ext()).collect(); + let h0_of = |j: usize| h0 == 1 || (h0 == 2 && j % 2 == 1); + let mut a = whole.layer_gruen(input, &point, low).unwrap(); + let mut b = kept.layer_gruen(input, &point, low).unwrap(); + assert!(!a.truncated(), "{what}: today's layer is whole"); + assert_eq!( + b.truncated(), + real < full, + "{what}: the kept layer is opened kept" + ); + let (sums_a, halves_a) = run_layer(&mut a, &lambda, &challenges, h0_of); + let (sums_b, halves_b) = run_layer(&mut b, &lambda, &challenges, h0_of); + for (j, (x, y)) in sums_a.iter().zip(&sums_b).enumerate() { + assert_eq!(x, y, "{what}: round {j}'s sums, raw"); + } + assert_eq!( + halves_a, halves_b, + "{what}: the halves handed to the host, raw" + ); + checked += 1; + } + } + eprintln!("gkr gruen truncated: {checked} layers, every round and finish raw-identical"); +} + +/// The same at a CPU table's shape (20 interactions of 32 slots, 2^21 rows) +/// and a Keccak-round table's (1,031 of 2,048, 2^16 rows), one tree at a time: +/// today's values first, then the kept tree's against them. +#[test] +fn a_kept_layer_runs_todays_rounds_at_production_shapes() { + let _serial = serial(); + let Ok(be) = backend() else { + eprintln!("no device; skipping"); + return; + }; + for (name, interactions, rows) in [("CPU", 20usize, 1usize << 21), ("KECCAK", 1031, 1 << 16)] { + let full = interactions.next_power_of_two() * rows; + let real = interactions * rows; + let num_vars = full.trailing_zeros() as usize; + let (p, q) = padded_layer(interactions, rows, 0xC0FFEE); + let mut rng = Rng(0xBEEF); + let point: Vec = (0..num_vars - 1).flat_map(|_| rng.ext()).collect(); + let lambda = rng.ext(); + let challenges: Vec<[u64; 3]> = (0..num_vars).map(|_| rng.ext()).collect(); + let low = 6; + let run = |tree: &DeviceFractionTree| { + let input = tree.num_layers() - 1; + let mut layer = tree.layer_gruen(input, &point, low).unwrap(); + let truncated = layer.truncated(); + let (sums, halves) = run_layer(&mut layer, &lambda, &challenges, |_| false); + (tree.output().unwrap(), sums, halves, truncated) + }; + let want = { + let stream = be.next_stream(); + let tree = DeviceFractionTree::from_device( + stream.clone(), + upload(&stream, &p), + upload(&stream, &q), + ) + .expect("today's tree"); + run(&tree) + }; + let got = { + let stream = be.next_stream(); + let tree = DeviceFractionTree::from_truncated_input( + stream.clone(), + upload(&stream, &p[..real * 3]), + upload(&stream, &q[..real * 3]), + real, + num_vars, + ) + .expect("the kept tree"); + run(&tree) + }; + assert!(!want.3 && got.3, "{name}: today's whole, the kept one kept"); + assert_eq!(want.0, got.0, "{name}: the output fraction"); + assert_eq!(want.1.len(), got.1.len(), "{name}: the rounds"); + for (j, (x, y)) in want.1.iter().zip(&got.1).enumerate() { + assert_eq!(x, y, "{name}: round {j}'s sums, raw"); + } + assert_eq!(want.2, got.2, "{name}: the halves handed to the host, raw"); + eprintln!( + "gkr gruen truncated: {name} ({interactions} of {} slots × 2^{} rows): {} rounds and the finish raw-identical", + interactions.next_power_of_two(), + rows.trailing_zeros(), + got.1.len() + ); + } +} + +/// ⛔ Only the Gruen rounds read a kept layer: today's session over it and the +/// cross-check's shadow refuse it, before anything runs. The same tree's +/// levels above it are whole, and their Gruen sessions open as before. +#[test] +fn todays_session_and_the_shadow_refuse_a_kept_layer() { + let _serial = serial(); + if backend().is_err() { + eprintln!("no device; skipping"); + return; + } + let invalid = cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE; + let (_whole, kept, _, _) = both_trees(5, 1 << 8, 0x51); + let input = kept.num_layers() - 1; + let m = kept.layer_num_vars(input) - 1; + let point = vec![7u64; m * 3]; + let refused = kept.layer_sumcheck(input, &point, &[], &[], 1, 0); + assert_eq!( + refused.err().map(|e| e.0), + Some(invalid), + "today's session refuses it" + ); + let layer = kept.layer_gruen(input, &point, 4).unwrap(); + assert_eq!( + layer.shadow(&[], &[], 1, 0).err().map(|e| e.0), + Some(invalid), + "the shadow refuses it" + ); + let above = kept + .layer_gruen(input - 1, &point[3..], 4) + .expect("a level above opens as before"); + assert!(!above.truncated(), "the levels above are whole"); +} + +/// ⛔ A shape no count rounded up to a power of two gives is refused, never +/// folded: half the cube or less, more than the cube, halves that disagree. +#[test] +fn a_kept_tree_of_a_shape_no_count_gives_is_refused() { + let _serial = serial(); + let Ok(be) = backend() else { + eprintln!("no device; skipping"); + return; + }; + let invalid = cudarc::driver::sys::CUresult::CUDA_ERROR_INVALID_VALUE; + let num_vars = 6; + let (p, q) = padded_layer(5, 8, 3); + let stream = be.next_stream(); + let attempt = |p: &[u64], q: &[u64], real: usize| { + DeviceFractionTree::from_truncated_input( + stream.clone(), + upload(&stream, p), + upload(&stream, q), + real, + num_vars, + ) + .err() + .map(|e| e.0) + }; + assert_eq!( + attempt(&p[..32 * 3], &q[..32 * 3], 32), + Some(invalid), + "half the cube" + ); + assert_eq!( + attempt(&p[..60], &q[..60], 20), + Some(invalid), + "less than half" + ); + let mut longer = p.clone(); + longer.extend_from_slice(&[0, 0, 0]); + assert_eq!( + attempt(&longer, &longer, 65), + Some(invalid), + "more than the cube" + ); + assert_eq!( + attempt(&p[..40 * 3], &q[..41 * 3], 40), + Some(invalid), + "halves that disagree" + ); + assert!( + attempt(&p[..40 * 3], &q[..40 * 3], 40).is_none(), + "the real shape builds" + ); +} + +/// The kept tree holds less on the card: the pool's high water over the +/// input layer's upload, the build and the input layer's rounds, against +/// today's, by at least nine tenths of the padding it does not store (48 +/// bytes a cell). +#[test] +fn a_kept_tree_holds_less_than_todays() { + let _serial = serial(); + let Ok(be) = backend() else { + eprintln!("no device; skipping"); + return; + }; + let (interactions, rows) = (20usize, 1usize << 16); + let full = interactions.next_power_of_two() * rows; + let real = interactions * rows; + let (p, q) = padded_layer(interactions, rows, 9); + let peak = |kept: bool| -> u64 { + math_cuda::device::drain_and_trim().unwrap(); + math_cuda::device::reset_pool_high_water().unwrap(); + let stream = be.next_stream(); + let (dp, dq) = if kept { + ( + upload(&stream, &p[..real * 3]), + upload(&stream, &q[..real * 3]), + ) + } else { + (upload(&stream, &p), upload(&stream, &q)) + }; + let tree = if kept { + DeviceFractionTree::from_truncated_input( + stream.clone(), + dp, + dq, + real, + full.trailing_zeros() as usize, + ) + } else { + DeviceFractionTree::from_device(stream.clone(), dp, dq) + } + .unwrap(); + let input = tree.num_layers() - 1; + let m = tree.layer_num_vars(input) - 1; + let mut layer = tree.layer_gruen(input, &vec![5u64; m * 3], 6).unwrap(); + run_layer(&mut layer, &[3, 0, 0], &vec![[2, 0, 0]; m], |_| false); + stream.synchronize().unwrap(); + drop(layer); + drop(tree); + math_cuda::device::drain_and_trim().unwrap(); + math_cuda::device::pool_used_bytes().unwrap().1 + }; + let (whole, kept) = (peak(false), peak(true)); + let padding = ((full - real) * 48) as u64; + eprintln!( + "gkr gruen truncated: pool high water today {} MiB, kept {} MiB, padding {} MiB", + whole >> 20, + kept >> 20, + padding >> 20 + ); + assert!( + kept + padding * 9 / 10 <= whole, + "the kept tree holds at least nine tenths of the padding less than today's" + ); +} + +// ── the card micro-benchmark ──────────────────────────────────────────────── + +/// What one tree hands back: its output fraction, its input layer's sums +/// round by round, and the halves after the last round. +type LayerValues = (([u64; 3], [u64; 3]), Vec>, Vec); + +/// Median, min and max of a sample, as `median/min/max` in milliseconds. +fn median_min_max(mut v: Vec) -> String { + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + format!("{:.3}/{:.3}/{:.3}", v[v.len() / 2], v[0], v[v.len() - 1]) +} + +fn median(mut v: Vec) -> f64 { + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + v[v.len() / 2] +} + +/// A table of `width` base columns over `rows`, and a plan of `interactions` +/// whose numerators are a multiplicity column times an integer and whose +/// denominators are `z − Σ α^k·column` over two to four columns — the shape +/// of a bus interaction. +fn columns_and_plan( + stream: &Arc, + width: usize, + rows: usize, + interactions: usize, + seed: u64, +) -> (CudaSlice, InputPlan) { + let mut rng = Rng(seed | 1); + let columns: Vec = (0..width * rows).map(|_| rng.next() % P).collect(); + let mut side_start = vec![0u32]; + let (mut terms, mut coeffs, mut constants) = (Vec::new(), Vec::new(), Vec::new()); + for _ in 0..interactions { + // numerator: ± multiplicity + let column = (rng.next() as usize) % width; + terms.extend_from_slice(&[(column * rows) as u64, 0]); + coeffs.extend_from_slice(&[rng.next() % 3 + 1, 0, 0]); + constants.extend_from_slice(&[0, 0, 0]); + side_start.push((terms.len() / 2) as u32); + // denominator: z − fingerprint + for _ in 0..2 + rng.next() % 3 { + let column = (rng.next() as usize) % width; + terms.extend_from_slice(&[(column * rows) as u64, rng.next() % 2]); + coeffs.extend_from_slice(&rng.ext()); + } + constants.extend_from_slice(&rng.ext()); + side_start.push((terms.len() / 2) as u32); + } + let plan = InputPlan::upload(stream, &side_start, &terms, &coeffs, &constants).unwrap(); + (upload(stream, &columns), plan) +} + +/// ⏱ The kill test's measurement (run with `--ignored --nocapture`): at a CPU +/// table's shape and a Keccak-round table's, the input layer's write (padded +/// against real slots), the tree's folds (today's against the kept tree's, +/// whose first fold reads the padding as `0/1`), and the input layer's own +/// Gruen rounds to the host tail plus the finish — 3 warm-ups and 10 timed +/// each, every step waited for, medians. The first timed repetition's values +/// are compared raw. `ARGUE BENCH`, `ARGUE VALUES` and `ARGUE MEM` lines. +#[test] +#[ignore] +fn the_kept_input_layer_at_production_shapes() { + let _serial = serial(); + let be = backend().expect("a device"); + const WARM: usize = 3; + const TIMED: usize = 10; + let time = |f: &mut dyn FnMut()| { + let from = std::time::Instant::now(); + f(); + from.elapsed().as_secs_f64() * 1e3 + }; + for (name, interactions, rows, width) in [ + ("CPU", 20usize, 1usize << 21, 38usize), + ("KECCAK", 1031, 1 << 16, 96), + ] { + let full = interactions.next_power_of_two() * rows; + let real = interactions * rows; + let num_vars = full.trailing_zeros() as usize; + + // The write, from columns resident on the card. + { + let stream = be.next_stream(); + let (columns, plan) = columns_and_plan(&stream, width, rows, interactions, 0x11); + let view = columns.slice(..); + for padding in [true, false] { + let mut times = Vec::new(); + for rep in 0..WARM + TIMED { + stream.synchronize().unwrap(); + let ms = time(&mut || { + let halves = math_cuda::gkr::input_from_columns( + &stream, &view, rows, &plan, padding, + ) + .unwrap(); + stream.synchronize().unwrap(); + drop(halves); + }); + if rep >= WARM { + times.push(ms); + } + } + println!( + "ARGUE BENCH shape={name} form={} step=write reps={TIMED} ms={}", + if padding { "whole" } else { "kept" }, + median_min_max(times) + ); + } + } + + // The folds and the input layer's rounds. + let (p, q) = padded_layer(interactions, rows, 0xF00D); + let mut rng = Rng(0xD00D); + let point: Vec = (0..num_vars - 1).flat_map(|_| rng.ext()).collect(); + let lambda = rng.ext(); + let challenges: Vec<[u64; 3]> = (0..num_vars).map(|_| rng.ext()).collect(); + let low = 6; + let mut values: Vec = Vec::new(); + let mut medians = Vec::new(); + let mut peaks = Vec::new(); + for kept in [false, true] { + let form = if kept { "kept" } else { "whole" }; + let stream = be.next_stream(); + let cells = if kept { real } else { full }; + let pristine_p = upload(&stream, &p[..cells * 3]); + let pristine_q = upload(&stream, &q[..cells * 3]); + stream.synchronize().unwrap(); + let (mut folds, mut rounds, mut first3) = (Vec::new(), Vec::new(), Vec::new()); + let mut peak = 0u64; + for rep in 0..WARM + TIMED { + math_cuda::device::reset_pool_high_water().unwrap(); + let mut input = Some(( + stream.clone_dtod(&pristine_p).unwrap(), + stream.clone_dtod(&pristine_q).unwrap(), + )); + stream.synchronize().unwrap(); + let mut tree = None; + let fold_ms = time(&mut || { + let (dp, dq) = input.take().unwrap(); + let built = if kept { + DeviceFractionTree::from_truncated_input( + stream.clone(), + dp, + dq, + real, + num_vars, + ) + } else { + DeviceFractionTree::from_device(stream.clone(), dp, dq) + } + .unwrap(); + stream.synchronize().unwrap(); + tree = Some(built); + }); + let tree = tree.unwrap(); + let input = tree.num_layers() - 1; + let mut layer = tree.layer_gruen(input, &point, low).unwrap(); + let mut sums = Vec::new(); + let mut three = 0.0; + let rounds_ms = time(&mut || { + for j in 0..layer.rounds() { + let from = std::time::Instant::now(); + let fold = (j > 0).then(|| &challenges[j - 1]); + sums.push(layer.round(fold, &lambda, false).unwrap()); + if j < 3 { + three += from.elapsed().as_secs_f64() * 1e3; + } + } + }); + let last = (layer.rounds() > 0).then(|| &challenges[layer.rounds() - 1]); + let finish_from = std::time::Instant::now(); + let halves = layer.finish(last).unwrap(); + let rounds_ms = rounds_ms + finish_from.elapsed().as_secs_f64() * 1e3; + // The window's high water less the pristine copy held beside it. + let held = (cells * 48) as u64; + peak = peak.max( + math_cuda::device::pool_used_bytes() + .unwrap() + .1 + .saturating_sub(held), + ); + if rep == WARM { + values.push((tree.output().unwrap(), sums, halves)); + } + if rep >= WARM { + folds.push(fold_ms); + rounds.push(rounds_ms); + first3.push(three); + } + drop(layer); + drop(tree); + } + println!( + "ARGUE BENCH shape={name} form={form} step=folds reps={TIMED} ms={}", + median_min_max(folds.clone()) + ); + println!( + "ARGUE BENCH shape={name} form={form} step=rounds reps={TIMED} ms={} first3_ms={}", + median_min_max(rounds.clone()), + median_min_max(first3) + ); + medians.push((median(folds), median(rounds))); + peaks.push(peak); + } + let same_output = values[0].0 == values[1].0; + let same_rounds = values[0].1 == values[1].1; + let same_halves = values[0].2 == values[1].2; + println!( + "ARGUE VALUES shape={name} output_equal={same_output} rounds_equal={same_rounds} halves_equal={same_halves} rounds={}", + values[1].1.len() + ); + println!( + "ARGUE MEM shape={name} promise_whole_mib={} promise_kept_mib={} pool_peak_whole_mib={} pool_peak_kept_mib={}", + ((4 * full) as u64 * 24) >> 20, + math_cuda::gkr::truncated_peak_bytes(real, full) >> 20, + peaks[0] >> 20, + peaks[1] >> 20 + ); + println!( + "ARGUE RATIO shape={name} rounds_kept_over_whole={:.3} folds_kept_over_whole={:.3}", + medians[1].1 / medians[0].1, + medians[1].0 / medians[0].0 + ); + assert!( + same_output && same_rounds && same_halves, + "{name}: the kept layer's values" + ); + } +} diff --git a/crypto/math-cuda/tests/grid_limits.rs b/crypto/math-cuda/tests/grid_limits.rs new file mode 100644 index 000000000..b6e6c77f8 --- /dev/null +++ b/crypto/math-cuda/tests/grid_limits.rs @@ -0,0 +1,254 @@ +//! CUDA caps grid.y at 65,535. The NTT's and the Möbius transform's 2-D tile +//! launches put a tile's high index in grid.y, and at the WHIR base's largest +//! stacks that index needs more: the NTT's first 5-level tile launches +//! `n >> 13` blocks in y, 65,536 at a 2^29 codeword (stack 27, blowup 4), and +//! the launch failed — every such commit fell back to the host +//! (`thoughts/zf/gap/fix/STRUCT.md` §9.2). The launches now move the excess +//! into grid.z (`ntt::split_grid_y`); the butterflies are the same, so the +//! output is too. +//! +//! The cheap tests force the split at every tile (`with_grid_y_cap(1, ..)`) on +//! small shapes and compare against the unsplit launch — the index math the +//! big shapes rely on. The `#[ignore]`d ones commit at the real sizes, where +//! the unsplit launch cannot run: stack 27 against the host pipeline those +//! commits fell back to, and stacks 27 and 28 between the two encodings: the +//! legacy one, whose tiles split, and the column engine's (the default), which +//! uses grid.x alone. Run them alone: several GiB of device and host memory +//! each. +//! +//! ```text +//! cargo test --release -p math-cuda --test grid_limits +//! cargo test --release -p math-cuda --test grid_limits -- --ignored --test-threads=1 --nocapture +//! ``` + +use math_cuda::DeviceHash; +use math_cuda::lde_cm::with_engine; +use math_cuda::ntt::with_grid_y_cap; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// Deterministic canonical field elements (splitmix64), fast enough for 2^29. +fn felts(seed: u64, len: usize) -> Vec { + let mut x = seed; + (0..len) + .map(|_| { + x = x.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = x; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + (z ^ (z >> 31)) % P + }) + .collect() +} + +fn canon(v: u64) -> u64 { + if v >= P { v - P } else { v } +} + +/// A position-sensitive digest of a codeword's canonical values, so two large +/// codewords compare without both being held on the host. +fn digest(values: &[u64]) -> u64 { + values.iter().fold(0x243F_6A88_85A3_08D3u64, |acc, &v| { + acc.rotate_left(7).wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ canon(v) + }) +} + +#[test] +fn ntt_tiles_split_into_grid_z_match_the_single_y_launch() { + for log_n in [13u32, 14, 16, 18] { + let x = felts(0x6E77 + u64::from(log_n), 1 << log_n); + for forward in [true, false] { + let run = || { + if forward { + math_cuda::ntt::forward(&x) + } else { + math_cuda::ntt::inverse(&x) + } + }; + let whole = run().expect("unsplit launch"); + let split = with_grid_y_cap(1, run).expect("every tile split into z"); + assert_eq!( + whole, split, + "2^{log_n} forward={forward}: the grid.z launch changed the transform" + ); + } + } +} + +#[test] +fn whir_commit_tiles_split_into_grid_z_match_the_single_y_launch() { + // The Möbius tiles start at 2^13 evaluations and the NTT tiles at a 2^13 + // codeword; spacing 1 and 4 both. + for (log_evals, log_blowup) in [(13usize, 1usize), (14, 2), (16, 0), (17, 2)] { + let evals = felts(0x3141 + log_evals as u64, 1 << log_evals); + for engine in [false, true] { + let commit = || { + with_engine(engine, || { + math_cuda::whir::commit_codeword_to_host( + &evals, + log_blowup, + 2, + DeviceHash::Rpx256, + ) + }) + }; + let whole = commit().expect("unsplit commit"); + let split = with_grid_y_cap(1, commit).expect("split commit"); + assert_eq!( + whole.0.iter().map(|&v| canon(v)).collect::>(), + split.0.iter().map(|&v| canon(v)).collect::>(), + "2^{log_evals} at 2^{log_blowup} (engine {engine}): codeword" + ); + assert_eq!( + whole.1, split.1, + "2^{log_evals} (engine {engine}): Merkle nodes" + ); + } + } +} + +/// One commit's root and codeword digest, under the legacy encoding or the +/// column engine's. +fn commit_digest(evals: &[u64], log_blowup: usize, engine: bool) -> ([u8; 32], u64) { + let t = std::time::Instant::now(); + let (values, nodes) = with_engine(engine, || { + math_cuda::whir::commit_codeword_to_host(evals, log_blowup, 4, DeviceHash::Rpx256) + }) + .unwrap_or_else(|e| panic!("commit (engine {engine}) failed: {e:?}")); + let root: [u8; 32] = nodes[..32].try_into().unwrap(); + let d = digest(&values); + println!( + "grid_limits: 2^{} evaluations at 2^{log_blowup}, engine {engine}: committed in {:.2} s, root0 {:016x}", + evals.len().trailing_zeros(), + t.elapsed().as_secs_f64(), + u64::from_le_bytes(root[..8].try_into().unwrap()) + ); + (root, d) +} + +/// Stack 27 against the host pipeline its commits fell back to (`multilinear`'s +/// encode and tree, as `tests/whir_commit.rs` uses at small sizes), at the +/// production shape: blowup 4, first6 leaves, RPX, under both encodings — the +/// legacy one, whose first tile is past what the unsplit launch could run at +/// this 2^29 codeword, and the column engine's, the default. Equal codewords +/// and roots are the pipeline's bytes at that size: what the fallback +/// produced, now from the device. `commit_codeword_to_host` has no host +/// fallback of its own: it runs the kernels or errors. +#[test] +#[ignore = "a 4 GiB codeword on each side and a host encode: run alone on the box"] +fn the_stack_27_commit_matches_the_host_pipeline() { + use math::field::element::FieldElement; + use math::field::goldilocks::GoldilocksField as F; + use multilinear::mle::Mle; + use multilinear::whir::{self, Domain}; + use multilinear::whir_commit::CodewordCommitment; + use multilinear::whir_hash::RpxWhir; + + const LOG_EVALS: usize = 27; + const LOG_BLOWUP: usize = 2; + const LOG_FOLDING: usize = 6; + let evals = felts(0x2727_0006, 1 << LOG_EVALS); + + let t = std::time::Instant::now(); + let f = Mle::new( + evals + .iter() + .map(|&v| FieldElement::::from_raw(v)) + .collect(), + ) + .expect("power of two"); + let domain = Domain::::new(LOG_EVALS + LOG_BLOWUP).expect("domain"); + let host_codeword = + whir::encode::(&whir::lift_coefficients(&f), &domain).expect("host encode"); + drop(f); + let host = CodewordCommitment::::from_codeword(host_codeword, LOG_FOLDING) + .expect("host commit"); + let host_values = host.codeword().host().expect("a host codeword"); + println!( + "grid_limits: stack 27 host encode and tree in {:.2} s", + t.elapsed().as_secs_f64() + ); + + for engine in [false, true] { + let t = std::time::Instant::now(); + let (device, nodes) = with_engine(engine, || { + math_cuda::whir::commit_codeword_to_host( + &evals, + LOG_BLOWUP, + LOG_FOLDING, + DeviceHash::Rpx256, + ) + }) + .unwrap_or_else(|e| panic!("the stack-27 device commit (engine {engine}) failed: {e:?}")); + println!( + "grid_limits: stack 27 device commit (engine {engine}) in {:.2} s", + t.elapsed().as_secs_f64() + ); + assert_eq!(device.len(), host_values.len()); + if let Some(i) = device + .iter() + .zip(host_values) + .position(|(&d, h)| canon(d) != canon(*h.value())) + { + panic!("stack 27 (engine {engine}): codeword position {i} differs from the host"); + } + let root: [u8; 32] = nodes[..32].try_into().unwrap(); + assert_eq!( + root, + host.root(), + "stack 27 (engine {engine}): the device root is not the host's" + ); + println!( + "grid_limits: stack 27 (engine {engine}) codeword and root equal to the host's, \ + root0 {:016x}", + u64::from_le_bytes(root[..8].try_into().unwrap()) + ); + } +} + +/// The WHIR base commits at stacks 27 and 28 (blowup 4): codewords of 2^29 and +/// 2^30, whose first NTT tile needs 65,536 and 131,072 blocks in y. +#[test] +#[ignore = "4–8 GiB codewords: run alone on the box"] +fn whir_commit_at_stacks_27_and_28() { + for log_evals in [27usize, 28] { + let evals = felts(0x2728 + log_evals as u64, 1 << log_evals); + let legacy = commit_digest(&evals, 2, false); + let engine = commit_digest(&evals, 2, true); + assert_eq!( + legacy, engine, + "2^{log_evals}: legacy and engine encodings disagree" + ); + } +} + +/// 2^29 evaluations at spacing 1: the Möbius tile's first launch needs 65,536 +/// blocks in y too, and so does the NTT's on the 2^29 codeword. +#[test] +#[ignore = "a 4 GiB codeword and 4 GiB of coefficients: run alone on the box"] +fn whir_commit_with_the_mobius_past_the_limit() { + let evals = felts(0x2929, 1 << 29); + let legacy = commit_digest(&evals, 0, false); + let engine = commit_digest(&evals, 0, true); + assert_eq!( + legacy, engine, + "2^29 at spacing 1: legacy and engine encodings disagree" + ); +} + +/// The control: with the split pushed back past the limit (65,536 blocks in +/// y, the old geometry), the same 2^29 transform must be refused by the device +/// — the failure the fix removes, reproduced in the same binary, so the passing +/// tests above are not passing for some other reason. +#[test] +#[ignore = "a 4 GiB transform: run alone on the box"] +fn the_unsplit_launch_is_refused_at_65536_blocks() { + let x = felts(0x6553_6000, 1 << 29); + // Not `expect_err`: its message would print the 2^29 values on success. + match with_grid_y_cap(1 << 16, || math_cuda::ntt::forward(&x)) { + Ok(v) => panic!("a 65,536-block grid.y launched ({} values back)", v.len()), + Err(err) => println!("grid_limits: unsplit 2^29 forward NTT refused as expected: {err:?}"), + } + let split = math_cuda::ntt::forward(&x).expect("the split launch runs"); + assert_eq!(split.len(), x.len()); +} diff --git a/crypto/math-cuda/tests/grind_queue_grid_thread.rs b/crypto/math-cuda/tests/grind_queue_grid_thread.rs new file mode 100644 index 000000000..62ac80db7 --- /dev/null +++ b/crypto/math-cuda/tests/grind_queue_grid_thread.rs @@ -0,0 +1,59 @@ +//! ★ The grind queue's grid must read the same on every thread. +//! +//! `grinding::queue_grid` sizes the queue grind by the queue kernel's occupancy, a +//! driver query that needs the CUDA context current on the CALLING thread. +//! `backend()` makes the context current only on the thread that created it, +//! and cudarc binds it before `num_regs` but not before the occupancy query. So +//! a first queue grind on any other thread got an error there, the `None` was +//! cached for the whole process, and every RPX grind after it fell to the host +//! search. Nothing failed; the device counter just stopped moving. +//! `rpx_grind_device` showed it on the runs where its keccak test happened to +//! be the one that created the backend. +//! +//! ONE test, in its own binary, on purpose: the grid is cached per process, so +//! a read by any earlier test on a thread that had the context would hide it. +//! +//! Runs on the merge-queue GPU box via `make test-math-cuda`, like the other +//! tests here. + +use std::thread; + +use cudarc::driver::sys::CUdevice_attribute; +use math_cuda::device::backend; +use math_cuda::grinding::{RPX_BLOCK_DIM, queue_grid}; + +#[test] +fn the_queue_grid_reads_on_a_thread_that_did_not_create_the_backend() { + // The backend, and with it the current context, on a thread of its own. + thread::spawn(|| { + backend().expect("a CUDA device"); + }) + .join() + .unwrap(); + + // What the driver answers on a fresh thread, printed rather than asserted: + // this test can see the regression only while this is an error. + let unbound = thread::spawn(|| { + backend() + .expect("a CUDA device") + .rpx_grind_search_queue + .occupancy_max_active_blocks_per_multiprocessor(RPX_BLOCK_DIM, 0, None) + }) + .join() + .unwrap(); + println!("occupancy query on a thread with no current context: {unbound:?}"); + + let grid = thread::spawn(queue_grid) + .join() + .unwrap() + .expect("the queue grid must read on a thread that did not create the backend"); + let sms = backend() + .expect("a CUDA device") + .ctx + .attribute(CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) + .expect("the multiprocessor count") as u32; + assert!( + grid >= sms, + "a grid of {grid} blocks must hold at least one per multiprocessor ({sms})" + ); +} diff --git a/crypto/math-cuda/tests/grinding.rs b/crypto/math-cuda/tests/grinding.rs index 84bc5e624..f755ec9d1 100644 --- a/crypto/math-cuda/tests/grinding.rs +++ b/crypto/math-cuda/tests/grinding.rs @@ -19,6 +19,11 @@ use stark::grinding::{inner_hash_lanes, is_valid_nonce}; +/// The kernel's outer hash is keccak, so the lanes and the validity pin are +/// keccak's grinding digest — the same type the dispatch in +/// `generate_nonce_maybe_gpu` keys the device search on. +type Keccak = stark::config::GrindingDigest; + /// At a moderate factor the kernel returns a valid nonce, and it is the /// smallest one (the exhaustive CPU scan below it is cheap at factor 14). /// @@ -33,14 +38,15 @@ use stark::grinding::{inner_hash_lanes, is_valid_nonce}; fn gpu_grind_returns_smallest_valid_nonce() { let seed = [14u8; 32]; let factor = 14u8; - let nonce = math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes(&seed, factor), factor) - .expect("GPU grind (needs a GPU)"); + let nonce = + math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes::(&seed, factor), factor) + .expect("GPU grind (needs a GPU)"); assert!( - is_valid_nonce(&seed, nonce, factor), + is_valid_nonce::(&seed, nonce, factor), "GPU nonce {nonce} fails is_valid_nonce (factor {factor})" ); assert!( - (0..nonce).all(|n| !is_valid_nonce(&seed, n, factor)), + (0..nonce).all(|n| !is_valid_nonce::(&seed, n, factor)), "GPU nonce {nonce} is not the smallest valid nonce (factor {factor})" ); } @@ -51,10 +57,11 @@ fn gpu_grind_returns_smallest_valid_nonce() { fn gpu_grind_valid_at_production_factor() { let seed = [20u8; 32]; let factor = 20u8; - let nonce = math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes(&seed, factor), factor) - .expect("GPU grind (needs a GPU)"); + let nonce = + math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes::(&seed, factor), factor) + .expect("GPU grind (needs a GPU)"); assert!( - is_valid_nonce(&seed, nonce, factor), + is_valid_nonce::(&seed, nonce, factor), "GPU nonce {nonce} fails is_valid_nonce (factor {factor})" ); } @@ -65,7 +72,94 @@ fn gpu_grind_valid_at_production_factor() { fn gpu_grind_declines_below_min_factor() { let seed = [1u8; 32]; assert!( - math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes(&seed, 1), 1).is_none(), + math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes::(&seed, 1), 1).is_none(), "GPU grind should decline factor 1" ); } + +/// ★ THE BLOCK SIZE AND THE GRID CANNOT MOVE THE ANSWER, EXECUTED. +/// +/// `search` scans contiguous blocks from zero and returns the first hitting +/// block's minimum, so the nonce is a function of the inner hash and the +/// grinding factor alone. That is what makes a sweep of either knob move no +/// proof byte — and it is a property that can fail: a stride or bounds defect +/// would return a different valid nonce at a different stride, and only an +/// equality across settings can see it. +/// +/// Through `generate_nonce_gpu_at` rather than the environment because the +/// knobs cache in a `OnceLock`: one process cannot read two settings through +/// `LAMBDA_VM_GRIND_*`, and two processes would be two device contexts. +#[test] +fn the_nonce_is_the_same_at_every_scan_factor_and_grid() { + let seed = [20u8; 32]; + let factor = 20u8; + let lanes = inner_hash_lanes::(&seed, factor); + + let record = math_cuda::grinding::Knobs::DEFAULT; + let expected = math_cuda::grinding::generate_nonce_gpu_at(&lanes, factor, record) + .expect("GPU grind at the record posture (needs a GPU)"); + assert!( + is_valid_nonce::(&seed, expected, factor), + "the record posture's nonce {expected} fails is_valid_nonce" + ); + + // Both knobs, both directions, including the pair the ruling names. + for knobs in [ + math_cuda::grinding::Knobs { + scan: 1, + grid: 1024, + }, + math_cuda::grinding::Knobs { + scan: 2, + grid: 1024, + }, + math_cuda::grinding::Knobs { + scan: 8, + grid: 4096, + }, + math_cuda::grinding::Knobs { scan: 8, grid: 256 }, + math_cuda::grinding::Knobs { + scan: 1, + grid: 4096, + }, + ] { + let nonce = math_cuda::grinding::generate_nonce_gpu_at(&lanes, factor, knobs) + .expect("GPU grind (needs a GPU)"); + assert!( + is_valid_nonce::(&seed, nonce, factor), + "nonce {nonce} from {knobs:?} fails is_valid_nonce" + ); + assert_eq!( + nonce, expected, + "{knobs:?} returned {nonce}, the record posture returned {expected} — \ + the launch geometry moved the answer, so the search is not scanning \ + contiguously from zero" + ); + } +} + +/// And it is still the SMALLEST at every setting, not merely the same one. +/// +/// Factor 14 so the exhaustive host scan below the answer stays cheap. Equality +/// across settings (the test above) would be satisfied by a search that +/// consistently skipped the same range; minimality is what rules that out. +#[test] +fn the_nonce_is_still_the_smallest_at_a_narrow_grid() { + let seed = [14u8; 32]; + let factor = 14u8; + let lanes = inner_hash_lanes::(&seed, factor); + for knobs in [ + math_cuda::grinding::Knobs { scan: 1, grid: 256 }, + math_cuda::grinding::Knobs { + scan: 8, + grid: 4096, + }, + ] { + let nonce = math_cuda::grinding::generate_nonce_gpu_at(&lanes, factor, knobs) + .expect("GPU grind (needs a GPU)"); + assert!( + (0..nonce).all(|n| !is_valid_nonce::(&seed, n, factor)), + "nonce {nonce} from {knobs:?} is not the smallest valid nonce" + ); + } +} diff --git a/crypto/math-cuda/tests/host_kat/blake3_host_kat.cpp b/crypto/math-cuda/tests/host_kat/blake3_host_kat.cpp new file mode 100644 index 000000000..2def5c3c3 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/blake3_host_kat.cpp @@ -0,0 +1,829 @@ +// Known-answer tests for `kernels/blake3.cu`, run on the host. +// +// WHY THIS EXISTS. The GPU parity suite (`tests/blake3_compress_parity.rs` and +// friends) is the authority on these kernels, but it runs only where a GPU does, +// and per-PR CI has none — GPU CI is merge_group-only. Without this the kernels +// have no per-PR gate at all: an edit to `blake3.cu` that broke the hash would +// reach the merge queue before anything noticed. This compiles the real kernel +// source through `cuda_host_shim.h` and runs external known-answer vectors +// through it, in seconds, with no GPU and no cargo. +// +// WHAT IT COVERS: the compression function at both round counts, the field +// element serialization, the 64-byte block framing, the Merkle parent, the +// `Blake3Chain` construction over multi-block messages, and every leaf kernel's +// byte stream (replayed thread by thread through the shim). +// +// WHAT IT DOES NOT COVER, and what the GPU tests are still required for: +// whether nvcc accepts the file, and every property of execution rather than +// arithmetic — grid indexing, `__syncthreads` ordering up the Merkle levels, +// device memory alignment, and register pressure. Passing here is necessary, +// never sufficient. +// +// HOW THE ANCHORING LAYERS. Nothing here is checked against itself: +// 1. The compression function is anchored by the OFFICIAL BLAKE3 vectors at 7 +// rounds (Table 1) and by the canonical vectors at 6 (Table 2). +// 2. `HostChain` below — a byte-level transcription of the construction — is +// anchored by the OFFICIAL multi-block vectors at 7 rounds (Table 3) and +// the committed 6-round chain KAT (Table 4). It is built ON the device +// compression, so layer 1 carries into it. +// 3. The device `Blake3Chain` is checked against `HostChain`, at word +// granularity, which is all the kernels ever need. +// 4. Each leaf kernel is replayed on host and checked against `HostChain` over +// the byte stream `leaves_bit_reversed_grouped` specifies — so the read +// pattern and the hash are anchored separately rather than together. +// +// ★ TABLES 1 AND 2 ARE COMPLEMENTARY, NOT REDUNDANT, and the difference is not +// obvious enough to leave unwritten. The official-vector path hashes whole +// messages, so it only ever exercises `t = 0`. A build with `v[12]` and `v[13]` +// transposed — the counter split inverted — reproduces the official vectors at +// ALL 65 single-block lengths, and is caught only by Table 2, whose ten vectors +// all carry `t >= 2^32`. That was measured, not assumed. Do not retire Table 2 +// as "covered by the standard", and do not describe Table 1 as subsuming it. +// +// ⚠ CONVERSELY, one thing here pins LESS than it appears to. The compression +// loop's `if (r < ROUNDS - 1)` permutation guard is UNOBSERVABLE: always +// permuting produces identical output at both round counts, because the schedule +// permuted after the final round is never read. It is an optimization, not a +// convention any known-answer test can validate — upstream expresses the same +// schedule as an indexed table with no guard at all. Do not cite these vectors +// as evidence the guard is correct; they cannot be. +// +// Build and run with `make test-blake3-host-kat`. + +#include +#include +#include +#include + +#include "cuda_host_shim.h" + +// The kernel under test. Included, not linked: the shim turns its device +// functions into host functions, and there is no other way to call them. +#include "blake3.cu" + +#include "blake3_kat_vectors.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char *what) { + if (!ok) { + printf("FAIL: %s\n", what); + ++failures; + } +} + +// The official vectors' input: the first `len` bytes of the repeating 251-byte +// sequence 0, 1, ..., 250. +void official_input(uint32_t len, uint8_t *out) { + for (uint32_t i = 0; i < len; ++i) out[i] = (uint8_t)(i % 251); +} + +// The 32-byte digest of a message of at most 64 bytes: ONE compression with +// `h = IV`, `t = 0`, the block zero-padded and read as little-endian words, +// `block_len` the true length, and the one-block flag set. The digest is the low +// eight output words, little-endian. +std::string hash_one_block(const uint8_t *msg, uint32_t len, int rounds) { + uint8_t block[64] = {0}; + memcpy(block, msg, len); + uint32_t m[16]; + for (int i = 0; i < 16; ++i) { + m[i] = (uint32_t)block[4 * i] | ((uint32_t)block[4 * i + 1] << 8) | + ((uint32_t)block[4 * i + 2] << 16) | ((uint32_t)block[4 * i + 3] << 24); + } + uint32_t out[16]; + if (rounds == 6) { + blake3_compress<6>(BLAKE3_IV, m, 0, len, BLAKE3_FLAGS_ONE_BLOCK, out); + } else { + blake3_compress<7>(BLAKE3_IV, m, 0, len, BLAKE3_FLAGS_ONE_BLOCK, out); + } + char hex[65]; + for (int i = 0; i < 8; ++i) { + for (int b = 0; b < 4; ++b) { + snprintf(hex + (i * 4 + b) * 2, 3, "%02x", (unsigned)((out[i] >> (8 * b)) & 0xff)); + } + } + return std::string(hex, 64); +} + +// ★ The external anchor. At 7 rounds the kernel must BE standard BLAKE3. +// +// Run over every length a single block can hold rather than one: the length keys +// both `block_len` and the zero-padding, so a port that ignored either would +// still pass at a single length. +void official_vectors_at_seven_rounds() { + check(NUM_OFFICIAL_VECTORS == 11, "official vector table lost entries"); + for (int i = 0; i < NUM_OFFICIAL_VECTORS; ++i) { + const OfficialVector &v = OFFICIAL_VECTORS[i]; + uint8_t msg[64]; + official_input(v.input_len, msg); + std::string got = hash_one_block(msg, v.input_len, 7); + if (got != v.hash_hex) { + printf("FAIL official vector len=%u\n got %s\n want %s\n", v.input_len, got.c_str(), + v.hash_hex); + ++failures; + } + } + printf("official BLAKE3 vectors at 7 rounds: %d checked\n", NUM_OFFICIAL_VECTORS); +} + +// NEGATIVE CONTROL for the anchor above: at 6 rounds nothing must match. +// +// Without this the anchor would pass just as well if the round count were being +// ignored — the one bug that makes the whole external-anchor argument vacuous, +// since the 6-round arm's only defence is "the same code path with the loop +// bound changed". The zero-length case is skipped: an empty message is the one +// input where the rounds have nothing to diffuse and a collision would not be +// evidence of anything. +void six_rounds_is_not_standard_blake3() { + int discriminated = 0; + for (int i = 0; i < NUM_OFFICIAL_VECTORS; ++i) { + const OfficialVector &v = OFFICIAL_VECTORS[i]; + if (v.input_len == 0) continue; + uint8_t msg[64]; + official_input(v.input_len, msg); + check(hash_one_block(msg, v.input_len, 6) != v.hash_hex, + "6 rounds reproduced an official 7-round vector"); + ++discriminated; + } + printf("6-round negative control: %d lengths discriminated\n", discriminated); +} + +// ★ The 6-round known-answer test, and the reason it is worth more than a +// self-comparison: `out6` came from #903's Python oracle, not from any code in +// this tree. All 16 output words are checked, not just the chaining value. +void canonical_vectors_at_both_round_counts() { + check(NUM_CANONICAL_VECTORS == 10, "canonical vector table lost entries"); + for (int i = 0; i < NUM_CANONICAL_VECTORS; ++i) { + const CanonicalVector &v = CANONICAL_VECTORS[i]; + uint32_t out6[16], out7[16]; + blake3_compress<6>(v.h, v.m, v.t, v.block_len, v.flags, out6); + blake3_compress<7>(v.h, v.m, v.t, v.block_len, v.flags, out7); + for (int w = 0; w < 16; ++w) { + if (out6[w] != v.out6[w]) { + printf("FAIL canonical %d word %d at 6 rounds: got %08x want %08x\n", i, w, out6[w], + v.out6[w]); + ++failures; + } + if (out7[w] != v.out7[w]) { + printf("FAIL canonical %d word %d at 7 rounds: got %08x want %08x\n", i, w, out7[w], + v.out7[w]); + ++failures; + } + } + } + printf("canonical vectors at 6 AND 7 rounds: %d checked, all 16 words each\n", + NUM_CANONICAL_VECTORS); +} + +// The serialization: one field element becomes the two message words its +// canonical big-endian bytes are read as, little-endian. The non-canonical raws +// are the cases where the reduction is the only thing that matters. +void serialization_is_the_canonical_big_endian_bytes() { + const uint64_t P = 0xFFFFFFFF00000001ull; + const uint64_t raws[] = {0, 1, P - 1, P, P + 1, P + 12345, ~0ull, 0x0123456789ABCDEFull}; + for (uint64_t raw : raws) { + uint32_t w0, w1; + blake3_words_of_felt(raw, w0, w1); + uint64_t canon = raw >= P ? raw - P : raw; + uint8_t be[8]; + for (int i = 0; i < 8; ++i) be[i] = (uint8_t)(canon >> (56 - 8 * i)); + uint32_t e0 = (uint32_t)be[0] | ((uint32_t)be[1] << 8) | ((uint32_t)be[2] << 16) | + ((uint32_t)be[3] << 24); + uint32_t e1 = (uint32_t)be[4] | ((uint32_t)be[5] << 8) | ((uint32_t)be[6] << 16) | + ((uint32_t)be[7] << 24); + check(w0 == e0 && w1 == e1, "blake3_words_of_felt"); + } + printf("serialization: %zu elements checked, non-canonical raws included\n", + sizeof(raws) / sizeof(raws[0])); +} + +// The block framing: nine elements are eighteen words, so one block completes and +// a two-word tail stays pending with fourteen words of zero padding behind it. +void block_framing_completes_and_pads() { + Blake3Block b; + b.init(); + int completed = 0; + uint32_t blocks[2][16] = {{0}}; + for (int i = 0; i < 9; ++i) { + uint32_t w0, w1; + blake3_words_of_felt((uint64_t)(i + 1) * 0x1111111111111111ull, w0, w1); + if (b.push_word(w0)) { + memcpy(blocks[completed++], b.m, 64); + b.reset(); + } + if (b.push_word(w1)) { + memcpy(blocks[completed++], b.m, 64); + b.reset(); + } + } + check(completed == 1, "exactly one block should have completed"); + check(b.pending_bytes() == 8, "the pending tail should be 8 bytes"); + memcpy(blocks[1], b.m, 64); + bool padded = true; + for (int k = 2; k < 16; ++k) padded = padded && blocks[1][k] == 0; + check(padded, "the tail block must be zero-padded"); + printf("block framing: 1 completed block + an 8-byte zero-padded tail\n"); +} + +// The Merkle parent: one compression over the 64 bytes of two child digests, so +// it must equal the one-block hash of their concatenation — which at 7 rounds is +// a plain `blake3::hash` call, and is what makes the parent framing externally +// anchored rather than merely self-consistent. +void parent_is_the_one_block_hash_of_its_children() { + uint8_t children[64]; + official_input(64, children); + uint8_t nodes[3 * 32]; + memcpy(nodes + 32, children, 32); // node 1 = left child + memcpy(nodes + 64, children + 32, 32); // node 2 = right child + blake3_hash_merkle_parent(nodes, 0, 1, 0); + char hex[65]; + for (int i = 0; i < 32; ++i) snprintf(hex + i * 2, 3, "%02x", (unsigned)nodes[i]); + check(std::string(hex, 64) == hash_one_block(children, 64, BLAKE3_ROUNDS), + "parent must equal the one-block hash of left || right"); + printf("Merkle parent at BLAKE3_ROUNDS=%d: %s\n", BLAKE3_ROUNDS, hex); +} + +// =========================================================================== +// The chain construction. +// =========================================================================== + +// The device compression with the round count as a run-time argument, so the +// reference below can be evaluated at either arm from one code path. +void compress_dyn(const uint32_t *h, const uint32_t *m, uint64_t t, uint32_t block_len, + uint32_t flags, int rounds, uint32_t *out) { + if (rounds == 6) { + blake3_compress<6>(h, m, t, block_len, flags, out); + } else { + blake3_compress<7>(h, m, t, block_len, flags, out); + } +} + +// `Blake3Chain` at BYTE granularity — PA-PLAN §1.7.1 written out directly. +// +// Why this exists when `blake3.cu` already has a `Blake3Chain`: the device one +// is word-granular, because every message the kernels hash is a whole number of +// 8-byte field elements. The official vectors are not — 65, 127, 1023 — and +// those lengths are where a final-block `block_len` bug lives. So the vectors +// anchor THIS, and the device chain is then checked against it at the word +// lengths it can actually reach. +// +// It is a transcription of the same spec as the host Rust `Blake3Chain`, not of +// the device struct, and it holds a full block rather than compressing it for +// the same reason: whether a block is the last is unknown until the message ends. +struct HostChain { + uint32_t cv[8]; + uint8_t block[64]; + uint32_t block_len; + bool started; + int rounds; + + void init(int r) { + memcpy(cv, BLAKE3_IV, sizeof(cv)); + memset(block, 0, sizeof(block)); + block_len = 0; + started = false; + rounds = r; + } + + void block_words(uint32_t *m) const { + for (int i = 0; i < 16; ++i) { + m[i] = (uint32_t)block[4 * i] | ((uint32_t)block[4 * i + 1] << 8) | + ((uint32_t)block[4 * i + 2] << 16) | ((uint32_t)block[4 * i + 3] << 24); + } + } + + uint32_t flags(bool is_final) const { + return (started ? 0u : BLAKE3_FLAG_CHUNK_START) | + (is_final ? (BLAKE3_FLAG_CHUNK_END | BLAKE3_FLAG_ROOT) : 0u); + } + + void compress_pending() { + uint32_t m[16], out[16]; + block_words(m); + compress_dyn(cv, m, 0, 64, flags(false), rounds, out); + memcpy(cv, out, sizeof(cv)); + memset(block, 0, sizeof(block)); + block_len = 0; + started = true; + } + + void update(const uint8_t *in, size_t n) { + while (n != 0) { + // Only now is the pending block known not to be the last. + if (block_len == 64) compress_pending(); + size_t take = 64 - block_len; + if (take > n) take = n; + memcpy(block + block_len, in, take); + block_len += (uint32_t)take; + in += take; + n -= take; + } + } + + void finalize(uint8_t *out32) const { + uint32_t m[16], out[16]; + block_words(m); + compress_dyn(cv, m, 0, block_len, flags(true), rounds, out); + for (int i = 0; i < 8; ++i) { + out32[4 * i] = (uint8_t)(out[i] & 0xff); + out32[4 * i + 1] = (uint8_t)((out[i] >> 8) & 0xff); + out32[4 * i + 2] = (uint8_t)((out[i] >> 16) & 0xff); + out32[4 * i + 3] = (uint8_t)((out[i] >> 24) & 0xff); + } + } +}; + +void host_chain(const uint8_t *msg, size_t len, int rounds, uint8_t *out32) { + HostChain c; + c.init(rounds); + c.update(msg, len); + c.finalize(out32); +} + +std::string to_hex(const uint8_t *b, size_t n) { + std::string s(n * 2, '\0'); + for (size_t i = 0; i < n; ++i) snprintf(&s[i * 2], 3, "%02x", (unsigned)b[i]); + return s; +} + +// The KAT message for Table 4: byte `i` is `37i + 11 (mod 256)`. +void kat_message(size_t len, std::vector &out) { + out.resize(len); + for (size_t i = 0; i < len; ++i) out[i] = (uint8_t)((i * 37 + 11) & 0xff); +} + +// ★ The chain's external anchor: over multi-block messages of at most one +// chunk, at 7 rounds, the construction IS standard BLAKE3, so the official +// vectors are direct known-answer tests for the framing — the flag schedule +// across blocks, the chaining value, and the final block's `block_len`. +// +// The `agrees == false` rows are the P3 negative control and are not decoration: +// without them, every matching row would pass identically if the full chunk tree +// had been implemented instead of the single unbounded chunk. +void chain_against_official_multiblock_vectors() { + check(NUM_CHAIN_VECTORS == 8, "chain vector table lost entries"); + int matched = 0, diverged = 0; + for (int i = 0; i < NUM_CHAIN_VECTORS; ++i) { + const ChainVector &v = CHAIN_VECTORS[i]; + std::vector msg(v.input_len); + for (uint32_t k = 0; k < v.input_len; ++k) msg[k] = (uint8_t)(k % 251); + uint8_t digest[32]; + host_chain(msg.data(), msg.size(), 7, digest); + std::string got = to_hex(digest, 32); + if (v.agrees) { + if (got != v.hash_hex) { + printf("FAIL chain vector len=%u\n got %s\n want %s\n", v.input_len, got.c_str(), + v.hash_hex); + ++failures; + } else { + ++matched; + } + } else { + check(got != v.hash_hex, + "past one chunk the chain must LEAVE standard BLAKE3 (P3 control)"); + ++diverged; + } + } + printf("chain vs official multi-block vectors at 7 rounds: %d must-match, %d P3 controls\n", + matched, diverged); +} + +// ★ The 6-round chain anchor: the committed table, whose digests came from the +// Python oracle rather than from any code in this tree. +void chain_against_the_committed_six_round_table() { + check(NUM_CHAIN_KAT_6ROUND == 12, "6-round chain KAT table lost entries"); + for (int i = 0; i < NUM_CHAIN_KAT_6ROUND; ++i) { + const ChainKat6Round &v = CHAIN_KAT_6ROUND[i]; + std::vector msg; + kat_message(v.input_len, msg); + uint8_t digest[32]; + host_chain(msg.data(), msg.size(), 6, digest); + std::string got = to_hex(digest, 32); + if (got != v.hash_hex) { + printf("FAIL 6-round chain KAT len=%u\n got %s\n want %s\n", v.input_len, got.c_str(), + v.hash_hex); + ++failures; + } + } + printf("chain vs committed 6-round KAT: %d lengths checked\n", NUM_CHAIN_KAT_6ROUND); +} + +// The DEVICE chain against the anchored reference, at every word-multiple length +// through several block boundaries. This is what carries the anchors above onto +// the struct the kernels actually use. +// +// The step of 4 is the device chain's granularity, and the range crosses the +// first, second and eighth block boundaries — the places a mis-set CHUNK_START, +// an eagerly compressed final block, or a wrong `block_len` would show. +void device_chain_matches_the_reference() { + int checked = 0; + for (size_t len = 0; len <= 600; len += 4) { + std::vector msg; + kat_message(len, msg); + + Blake3Chain dev; + dev.init(); + for (size_t i = 0; i < len; i += 4) { + uint32_t w = (uint32_t)msg[i] | ((uint32_t)msg[i + 1] << 8) | + ((uint32_t)msg[i + 2] << 16) | ((uint32_t)msg[i + 3] << 24); + dev.push_word(w); + } + uint8_t got[32]; + dev.finalize(got); + + uint8_t want[32]; + host_chain(msg.data(), msg.size(), BLAKE3_ROUNDS, want); + if (memcmp(got, want, 32) != 0) { + printf("FAIL device chain at len=%zu\n got %s\n want %s\n", len, + to_hex(got, 32).c_str(), to_hex(want, 32).c_str()); + ++failures; + break; + } + ++checked; + } + printf("device chain vs reference at BLAKE3_ROUNDS=%d: %d lengths (0..600 step 4)\n", + BLAKE3_ROUNDS, checked); +} + +// ★ P2 on the device struct: a 64-byte message through the chain must be the +// parent compression. This is the invariant that lets the leaf and parent layers +// be one hash, and it is why `blake3_hash_merkle_parent` needs no chaining. +void device_chain_at_64_bytes_is_the_parent() { + uint8_t children[64]; + official_input(64, children); + uint8_t nodes[3 * 32]; + memcpy(nodes + 32, children, 32); + memcpy(nodes + 64, children + 32, 32); + blake3_hash_merkle_parent(nodes, 0, 1, 0); + + Blake3Chain dev; + dev.init(); + for (int i = 0; i < 16; ++i) { + uint32_t w = (uint32_t)children[4 * i] | ((uint32_t)children[4 * i + 1] << 8) | + ((uint32_t)children[4 * i + 2] << 16) | ((uint32_t)children[4 * i + 3] << 24); + dev.push_word(w); + } + uint8_t got[32]; + dev.finalize(got); + check(memcmp(got, nodes, 32) == 0, "a 64-byte device chain must be the parent compression"); + printf("P2: a 64-byte chain is the Merkle parent compression\n"); +} + +// =========================================================================== +// The leaf kernels, replayed thread by thread. +// =========================================================================== + +const uint64_t GOLDILOCKS_P = 0xFFFFFFFF00000001ull; + +uint64_t canon(uint64_t raw) { return raw >= GOLDILOCKS_P ? raw - GOLDILOCKS_P : raw; } + +// `reverse_index(i, n)` — the CPU commit's row permutation, and what the kernels +// compute as `__brevll(tid) >> (64 - log_num_rows)`. +uint64_t reverse_index(uint64_t i, uint32_t log_n) { return __brevll(i) >> (64 - log_n); } + +// Append a field element's canonical BIG-endian bytes — the serialization +// `leaves_bit_reversed_grouped` writes and every leaf kernel must reproduce. +void push_be(std::vector &buf, uint64_t raw) { + uint64_t c = canon(raw); + for (int i = 0; i < 8; ++i) buf.push_back((uint8_t)(c >> (56 - 8 * i))); +} + +// A deterministic value stream, including deliberately non-canonical raws so the +// reduction is exercised rather than assumed. +uint64_t sample(uint64_t seed, uint64_t i) { + uint64_t x = seed * 0x9E3779B97F4A7C15ull + i * 0xBF58476D1CE4E5B9ull; + x ^= x >> 31; + x *= 0x94D049BB133111EBull; + x ^= x >> 29; + // Every fifth value is left above the modulus. + return (i % 5 == 0) ? x : x % GOLDILOCKS_P; +} + +void check_leaves(const std::vector &got, const std::vector> &want, + const char *what) { + if (got.size() != want.size() * 32) { + printf("FAIL %s: leaf count %zu vs %zu\n", what, got.size() / 32, want.size()); + ++failures; + return; + } + for (size_t i = 0; i < want.size(); ++i) { + uint8_t expect[32]; + host_chain(want[i].data(), want[i].size(), BLAKE3_ROUNDS, expect); + if (memcmp(got.data() + i * 32, expect, 32) != 0) { + printf("FAIL %s: leaf %zu\n got %s\n want %s\n", what, i, + to_hex(got.data() + i * 32, 32).c_str(), to_hex(expect, 32).c_str()); + ++failures; + return; + } + } +} + +// The two column-major base kernels: one leaf per bit-reversed row, and one per +// bit-reversed row pair. The expected byte stream is built from the CPU leaf +// spec — rows in bit-reversed order, each written column by column in canonical +// big-endian — so what is compared is the kernel's READ PATTERN against that +// spec, with the hash anchored separately above. +void base_leaf_kernels_read_the_specified_bytes() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 5ull, 8ull, 17ull}) { + uint64_t n = 1ull << log_n; + std::vector cols(num_cols * n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t r = 0; r < n; ++r) cols[c * n + r] = sample(log_n * 31 + num_cols, c * n + r); + } + + // rows_per_leaf = 1 + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) push_be(want[leaf], cols[c * n + br]); + } + check_leaves(out, want, "blake3_leaves_base_batched"); + } + + // rows_per_leaf = 2 + { + uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_pair_batched(cols.data(), n, num_cols, n, log_n, + out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < num_cols; ++c) push_be(want[leaf], cols[c * n + br]); + } + } + check_leaves(out, want, "blake3_leaves_base_row_pair_batched"); + } + } + } + printf("base leaf kernels: read pattern matches the CPU leaf spec\n"); +} + +// The ext3 kernels, over the de-interleaved three-slab layout. An ext3 element +// is three consecutive components, each 8 big-endian bytes — six words, so +// elements straddle block boundaries routinely, which is the case the +// word-granular block builder exists for. +void ext3_leaf_kernels_read_the_specified_bytes() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 3ull, 11ull}) { + uint64_t n = 1ull << log_n; + std::vector cols(num_cols * 3 * n); + for (uint64_t s = 0; s < num_cols * 3; ++s) { + for (uint64_t r = 0; r < n; ++r) cols[s * n + r] = sample(log_n * 17 + num_cols, s * n + r); + } + + // One leaf per bit-reversed row. + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_ext3_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) push_be(want[leaf], cols[(c * 3 + k) * n + br]); + } + } + check_leaves(out, want, "blake3_leaves_ext3_batched"); + } + + // Row pairs — the comp-poly kernel, which the aux trace also uses. + { + uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_comp_poly_leaves_ext3(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int j = 0; j < 2; ++j) { + uint64_t br = reverse_index(2 * leaf + j, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) + push_be(want[leaf], cols[(c * 3 + k) * n + br]); + } + } + } + check_leaves(out, want, "blake3_comp_poly_leaves_ext3"); + } + } + } + printf("ext3 + comp-poly leaf kernels: read pattern matches the CPU leaf spec\n"); +} + +// FRI leaves: two consecutive ext3 values from an interleaved vector, 48 bytes, +// no bit reversal. Under one block, so this is the chain's single-compression +// case at a length that is neither 64 nor a block multiple. +void fri_leaf_kernel_reads_the_specified_bytes() { + for (uint64_t num_leaves : {1ull, 2ull, 8ull, 33ull}) { + std::vector evals(num_leaves * 2 * 3); + for (size_t i = 0; i < evals.size(); ++i) evals[i] = sample(0xF41, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_fri_leaves_ext3(evals.data(), num_leaves, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int i = 0; i < 6; ++i) push_be(want[leaf], evals[leaf * 6 + i]); + } + check_leaves(out, want, "blake3_fri_leaves_ext3"); + } + printf("FRI leaf kernel: read pattern matches the CPU leaf spec\n"); +} + +// The row-major row-pair kernels, plain and column-ranged. `m` is the row +// stride; the ranged variant hashes only `[col_start, col_end)` while the stride +// stays full, which is how preprocessed tables commit two column ranges to +// separate trees over one LDE. +void row_major_leaf_kernels_read_the_specified_bytes() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t m : {1ull, 5ull, 13ull}) { + uint64_t n = 1ull << log_n; + uint64_t num_leaves = n / 2; + std::vector data(n * m); + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(log_n * 7 + m, i); + + { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair(data.data(), m, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < m; ++c) push_be(want[leaf], data[br * m + c]); + } + } + check_leaves(out, want, "blake3_leaves_base_row_major_row_pair"); + } + + // Every non-empty column range, so the boundary handling is checked + // rather than sampled. + for (uint64_t cs = 0; cs < m; ++cs) { + for (uint64_t ce = cs + 1; ce <= m; ++ce) { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair_range(data.data(), m, cs, ce, n, + log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = cs; c < ce; ++c) push_be(want[leaf], data[br * m + c]); + } + } + check_leaves(out, want, "blake3_leaves_base_row_major_row_pair_range"); + } + } + } + } + printf("row-major leaf kernels: read pattern matches the CPU leaf spec, all column ranges\n"); +} + +// The row-major ONE-ROW kernel (S2, rows_per_leaf = 1): leaf `i` is the single +// row `reverse_index(i)` over `log_n` bits, every non-empty column range. Also +// the control that it is NOT the row-pair kernel's first row: at n >= 4 the +// one-row leaf 1 is row brev(1) = n/2, the row-pair leaf 0's second row, never +// row brev(2) (the pair kernel's leaf 1 first row). +void row_major_one_row_kernel_reads_the_specified_bytes() { + for (uint32_t log_n : {1u, 2u, 4u, 6u}) { + for (uint64_t m : {1ull, 5ull, 13ull}) { + uint64_t n = 1ull << log_n; + std::vector data(n * m); + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(log_n * 11 + m, i); + for (uint64_t cs = 0; cs < m; ++cs) { + for (uint64_t ce = cs + 1; ce <= m; ++ce) { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_row_major_row_range(data.data(), m, cs, ce, n, log_n, + out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = cs; c < ce; ++c) push_be(want[leaf], data[br * m + c]); + } + check_leaves(out, want, "blake3_leaves_base_row_major_row_range"); + } + } + } + } + // The one-row tree has TWICE the leaves of the row-pair tree over the same + // rows, and its leaves are not the pair tree's: a one-row kernel that read + // row pairs would match neither the spec above nor differ here. + { + const uint32_t log_n = 4; + const uint64_t n = 1ull << log_n, m = 3; + std::vector data(n * m); + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(0x0E, i); + std::vector one(n * 32, 0), pair((n / 2) * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_row_major_row_range(data.data(), m, 0, m, n, log_n, one.data()); + } + CUDA_HOST_FOR_EACH_THREAD(t, n / 2) { + blake3_leaves_base_row_major_row_pair(data.data(), m, n, log_n, pair.data()); + } + check(memcmp(one.data(), pair.data(), 32) != 0, + "a one-row leaf must not equal the row-pair leaf over the same first row"); + } + printf("row-major one-row kernel: read pattern matches the CPU one-row leaf spec, all column ranges\n"); +} + +// The full-range ranged kernel must be the unranged one — the same bytes by two +// code paths. A cheap check that the range arithmetic has no off-by-one at the +// boundary it is most likely to have one at. +void the_full_range_variant_equals_the_plain_one() { + const uint32_t log_n = 5; + const uint64_t n = 1ull << log_n, m = 7, num_leaves = n / 2; + std::vector data(n * m); + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(0xBEEF, i); + + std::vector plain(num_leaves * 32, 0), ranged(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair(data.data(), m, n, log_n, plain.data()); + } + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair_range(data.data(), m, 0, m, n, log_n, ranged.data()); + } + check(plain == ranged, "the full-range kernel must equal the unranged one"); + printf("row-major range [0, m) equals the plain kernel\n"); +} + +// NEGATIVE CONTROL for the leaf checks: the leaves must depend on the data and +// on the row index. Every check above compares kernel output to an expectation +// built from the same buffer, and all of them would pass if the kernel emitted a +// constant and the expectation happened to be that constant. +void leaves_depend_on_data_and_row() { + const uint32_t log_n = 4; + const uint64_t n = 1ull << log_n, num_cols = 3; + std::vector cols(num_cols * n); + for (size_t i = 0; i < cols.size(); ++i) cols[i] = sample(0xD00D, i); + + std::vector a(n * 32, 0), b(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_batched(cols.data(), n, num_cols, n, log_n, a.data()); + } + cols[n + 3] ^= 1ull; // one element of one column + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_batched(cols.data(), n, num_cols, n, log_n, b.data()); + } + check(a != b, "a one-element change must move some leaf"); + + bool all_same = true; + for (uint64_t i = 1; i < n; ++i) { + if (memcmp(a.data(), a.data() + i * 32, 32) != 0) { + all_same = false; + break; + } + } + check(!all_same, "all leaves identical — the kernel is not reading its row index"); + printf("negative control: leaves depend on the data and on the row index\n"); +} + +} // namespace + +int main() { + printf("BLAKE3 device-kernel known-answer tests, host-compiled from " + "crypto/math-cuda/kernels/blake3.cu\n\n"); + official_vectors_at_seven_rounds(); + six_rounds_is_not_standard_blake3(); + canonical_vectors_at_both_round_counts(); + serialization_is_the_canonical_big_endian_bytes(); + block_framing_completes_and_pads(); + parent_is_the_one_block_hash_of_its_children(); + printf("\n-- chain construction --\n"); + chain_against_official_multiblock_vectors(); + chain_against_the_committed_six_round_table(); + device_chain_matches_the_reference(); + device_chain_at_64_bytes_is_the_parent(); + printf("\n-- leaf kernels --\n"); + base_leaf_kernels_read_the_specified_bytes(); + ext3_leaf_kernels_read_the_specified_bytes(); + fri_leaf_kernel_reads_the_specified_bytes(); + row_major_leaf_kernels_read_the_specified_bytes(); + the_full_range_variant_equals_the_plain_one(); + row_major_one_row_kernel_reads_the_specified_bytes(); + leaves_depend_on_data_and_row(); + if (failures != 0) { + printf("\n*** %d FAILURE(S) ***\n", failures); + return 1; + } + printf("\nALL HOST KAT CHECKS PASS\n"); + printf("NOTE: arithmetic only. nvcc acceptance and GPU execution are covered " + "by tests/blake3_*.rs, which need a GPU.\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/blake3_kat_vectors.h b/crypto/math-cuda/tests/host_kat/blake3_kat_vectors.h new file mode 100644 index 000000000..9901f28ae --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/blake3_kat_vectors.h @@ -0,0 +1,360 @@ +// Known-answer vectors for the BLAKE3 device kernels, embedded rather than +// parsed at run time. +// +// Embedded on purpose: a test that reads its vectors from a file passes +// silently when the read finds nothing, which is a failure mode that has +// already happened once on this harness. A table cannot have a zero-vector +// run, and `main` asserts the counts below as well. +// +// This file is DATA. It is transcribed, never computed, and the two tables +// come from outside this crate — see each one's provenance note. +#pragma once +#include + +// --------------------------------------------------------------------------- +// Table 1 — the OFFICIAL BLAKE3 test vectors, standard 7-round hash. +// +// Transcribed from `tooling/blake3-second-source/blake3-oracle/official_test_vectors.json` +// (tracked in this repo, sourced from the BLAKE3 reference implementation). +// Only the cases with `input_len <= 64` appear: a message that fits one block +// of one chunk is a SINGLE compression, which is what the device function +// computes. Longer cases need the chunk tree and are not this kernel's job. +// +// The input for length N is the first N bytes of the repeating 251-byte +// sequence 0, 1, 2, ..., 250 — the generator the vector file specifies. +// --------------------------------------------------------------------------- +struct OfficialVector { + uint32_t input_len; + const char *hash_hex; // the first 32 bytes of the extended output +}; + +inline constexpr int NUM_OFFICIAL_VECTORS = 11; +inline constexpr OfficialVector OFFICIAL_VECTORS[NUM_OFFICIAL_VECTORS] = { + { 0, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"}, + { 1, "2d3adedff11b61f14c886e35afa036736dcd87a74d27b5c1510225d0f592e213"}, + { 2, "7b7015bb92cf0b318037702a6cdd81dee41224f734684c2c122cd6359cb1ee63"}, + { 3, "e1be4d7a8ab5560aa4199eea339849ba8e293d55ca0a81006726d184519e647f"}, + { 4, "f30f5ab28fe047904037f77b6da4fea1e27241c5d132638d8bedce9d40494f32"}, + { 5, "b40b44dfd97e7a84a996a91af8b85188c66c126940ba7aad2e7ae6b385402aa2"}, + { 6, "06c4e8ffb6872fad96f9aaca5eee1553eb62aed0ad7198cef42e87f6a616c844"}, + { 7, "3f8770f387faad08faa9d8414e9f449ac68e6ff0417f673f602a646a891419fe"}, + { 8, "2351207d04fc16ade43ccab08600939c7c1fa70a5c0aaca76063d04c3228eaeb"}, + {63, "e9bc37a594daad83be9470df7f7b3798297c3d834ce80ba85d6e207627b7db7b"}, + {64, "4eed7141ea4a5cd4b788606bd23f46e212af9cacebacdc7d1f4c6dc7f2511b98"}, +}; + +// --------------------------------------------------------------------------- +// Table 2 — the ten canonical vectors, at BOTH round counts. +// +// Inputs and `out6` are transcribed from `CANONICAL_VECTORS` +// (now `crypto/crypto/src/hash/blake3/vectors.rs`), whose 6-round outputs came +// from #903's Python oracle rather than from any Rust code. `out7` is +// `CANONICAL_OUT_7ROUND` from the same file, itself pinned by the official crate. +// +// This is the point of the table: it gives the SIX-round arm a known-answer +// test whose expected values no implementation in this tree produced. +// +// ★ It ALSO does a job Table 1 structurally CANNOT, and the reason is not +// obvious enough to leave unwritten: the official-vector path hashes whole +// messages, so it only ever exercises `t = 0`. A compression with the counter +// split inverted (`v[12]` and `v[13]` transposed) reproduces the official +// vectors at every single-block length, and is caught only here — all ten of +// these vectors carry `t >= 2^32`. Measured, not assumed. This table is not +// redundant with Table 1 and must not be retired as "covered by the standard". +// +// ★ PROVENANCE, strengthened 2026-08-15 — the 6-round column is no longer +// pinned by Python alone. `tooling/blake3-second-source/reference-impl/` holds UPSTREAM +// BLAKE3 1.8.5 with a 2 KB reviewable diff (`PARAMETERISATION.diff`) whose only +// functional edit replaces seven unrolled `round_fn` calls with a loop bounded +// by `BLAKE3_ROUNDS_PARAM`. Built at both round counts and run over these ten +// inputs, it reproduces `out6` AND `out7` 10/10, all 16 words. It is C rather +// than Python, upstream's own code rather than a transcription, and it encodes +// the message schedule as an INDEXED TABLE (`MSG_SCHEDULE[r]`) rather than as an +// in-place permutation between rounds — a structurally different expression of +// the same convention, so its agreement cross-validates the schedule instead of +// restating it. Rebuild with `tooling/blake3-second-source/reference-impl/build.sh`: a ~1 +// second C compile, no cargo, no GPU. +// --------------------------------------------------------------------------- +struct CanonicalVector { + uint32_t h[8]; + uint32_t m[16]; + uint64_t t; + uint32_t block_len; + uint32_t flags; + uint32_t out6[16]; + uint32_t out7[16]; +}; + +inline constexpr int NUM_CANONICAL_VECTORS = 10; +inline constexpr CanonicalVector CANONICAL_VECTORS[NUM_CANONICAL_VECTORS] = { + { + {0xD82C07CDu, 0x6BAA9455u, 0x82E2E662u, 0x7A024204u, + 0xE87A1613u, 0x81332876u, 0x48268673u, 0xC17C6279u}, + {0xE6F4590Bu, 0x4F65D4D9u, 0xBAD640FBu, 0xAF19922Au, + 0x19C78DF4u, 0x6F25E2A2u, 0xE9BB17BCu, 0x7A1D5006u, + 0x42AF9FC3u, 0x03983CA8u, 0xDE1B372Au, 0xDED733E8u, + 0x9148624Fu, 0xF7B0B7D2u, 0x72AE2244u, 0xEECE328Bu}, + 0xB4E1357D4A84EB03ull, 42u, 52u, + {0xCED9D1FFu, 0xC248EEABu, 0xBD109B7Fu, 0x911B48F6u, + 0x923D62C0u, 0xD804903Fu, 0x5974223Eu, 0xAA4F0C80u, + 0xAD61007Fu, 0xB50B8DDBu, 0xE7372BE1u, 0x33D3D6C3u, + 0x42AA284Bu, 0xC5A25F28u, 0x79AC8370u, 0xB75F3915u}, + {0xEE79E5DCu, 0xEA647B8Cu, 0x964C097Eu, 0xE2F3383Au, + 0xFE2E6D00u, 0x78EE613Au, 0xC33C8572u, 0xCD444391u, + 0x0C890604u, 0xC3209591u, 0x45633FF8u, 0xCB171C6Au, + 0x760247AEu, 0xF6D0FC1Eu, 0xCD550F20u, 0xCD54BF83u}, + }, + { + {0xC386BBC4u, 0x414C343Cu, 0x7311D8A3u, 0xA6CECC1Bu, + 0xC9E9C616u, 0x18072E8Cu, 0xD5F4B3B2u, 0x7204E52Du}, + {0xF1FD42A2u, 0xE6C3F339u, 0x07D4BEDCu, 0x8A9A021Eu, + 0x3BAB6C39u, 0x05805975u, 0xA46D6753u, 0xDC2574BDu, + 0xAB99254Au, 0x4DA98F1Du, 0xE1EA24C4u, 0x815A47C5u, + 0x08D6AF57u, 0xCC22AF58u, 0x2C4A3698u, 0x5FEC898Fu}, + 0xC74803E31BA16215ull, 50u, 94u, + {0xF2A972E9u, 0x81FDB8ECu, 0x40C50EBCu, 0x4BA1CAF9u, + 0x9EE9E930u, 0x6B1A16B2u, 0xE9156F47u, 0xA89FB436u, + 0xA2F616B3u, 0x12874C12u, 0x30768035u, 0xE01A17D9u, + 0xBEE5C17Cu, 0xD61C0BE0u, 0x3041FF46u, 0xDFB91125u}, + {0xD68593D0u, 0xDBC8157Au, 0xF6E1687Cu, 0x52A60555u, + 0xB56D418Au, 0x0CCBB863u, 0xADBFB51Eu, 0x8BF7D125u, + 0x75C23432u, 0xF484D7A6u, 0x06E85F4Au, 0x2771FE96u, + 0x00F6E24Du, 0x48368A3Eu, 0x04EE7E88u, 0x501D8539u}, + }, + { + {0x0E7A269Fu, 0x15BA2BDDu, 0xD5E34124u, 0x4EE207F8u, + 0x9B1F282Eu, 0x9B575BD1u, 0xF30B94FAu, 0x0706A045u}, + {0x6148A86Fu, 0x8697BBD0u, 0x8F7D9B78u, 0x3C729578u, + 0x061B9030u, 0x533C9135u, 0x829E07B0u, 0xE4C11AB2u, + 0xCBF87544u, 0xC34C769Fu, 0x5A91C89Bu, 0xF63F23D0u, + 0xC1066932u, 0x87C56473u, 0x7D718D73u, 0xECC1CB63u}, + 0x7604E4B4E73695C3ull, 58u, 124u, + {0x5AA6B114u, 0xC9D6740Cu, 0x8738CAF4u, 0xAC5F4B72u, + 0x9FC6B9DEu, 0x3F2EFB8Fu, 0x8CB7A912u, 0xF497A285u, + 0x3D062266u, 0x7F22380Cu, 0xAFD468FAu, 0x122CBA80u, + 0x446B156Du, 0xB239D8C2u, 0xC3EAB2CFu, 0x775F2F92u}, + {0xBC92D7C4u, 0x56542092u, 0x3490E2CBu, 0x2E3328CDu, + 0x13E3746Fu, 0xA5B88E66u, 0x2B5FE530u, 0x92C7AD52u, + 0xFF502AE5u, 0x1F088FBFu, 0x9163752Fu, 0x8A0C8B4Du, + 0xB557B0E8u, 0xE76F23CBu, 0xD054C959u, 0x74813CFDu}, + }, + { + {0x8B529B4Au, 0x9A9A80FDu, 0xD6645FA9u, 0x3BFD1D33u, + 0x79F248B0u, 0x268ECC45u, 0xA2863A7Fu, 0x85EF3430u}, + {0xBDC2AE99u, 0x10645D51u, 0x97524D6Au, 0xDD933160u, + 0xE0F9E038u, 0xEBCD1F5Eu, 0xEF829C88u, 0xE0FD67DDu, + 0x18F2C41Cu, 0x22CEDAFBu, 0x378C74DCu, 0x4D100D8Fu, + 0x95C76AB4u, 0x95918694u, 0xE779C470u, 0xEDCF6109u}, + 0x92D3043AFCF249F3ull, 36u, 31u, + {0xEED92FABu, 0x138D9358u, 0x915BFE3Cu, 0x13718B01u, + 0xB506E277u, 0xBE4007CDu, 0x35847E06u, 0xCE1C6896u, + 0x52FA01B5u, 0x4AA26AF8u, 0xB1078A61u, 0x2C517AEDu, + 0xA08867A0u, 0xEA6ECFEAu, 0x6D33D3B0u, 0xDC293166u}, + {0xCF4FB929u, 0x1DBADE2Au, 0x70E63AAFu, 0x2E0FFB48u, + 0x60123045u, 0x798AEAE8u, 0x5A911D30u, 0x15977C61u, + 0x6F7C8334u, 0x5EB0BCE2u, 0xAB240F17u, 0x66B7A3CDu, + 0xA9064E0Bu, 0x6AC4747Bu, 0x1206F62Bu, 0x9F3E91ECu}, + }, + { + {0x3C6DA5D7u, 0x656412A9u, 0x27AC435Au, 0x11072231u, + 0xEAFF1A09u, 0xC3E1B258u, 0x8963DC6Eu, 0x1B2ED40Eu}, + {0xED6F0B09u, 0xCE80C4B0u, 0xCCEA2645u, 0x3184FF27u, + 0x4F5253A0u, 0xE14B0190u, 0x9B191BF4u, 0xABF4A07Cu, + 0x81862FC9u, 0x2D83A823u, 0x793D0E45u, 0x4CDCE7A6u, + 0xE8ABB93Fu, 0xE1DF8AF9u, 0x8224B122u, 0x69F85E31u}, + 0x49C7B59B995253FDull, 57u, 41u, + {0xCA00BDA3u, 0x84239A3Au, 0xE7C88E6Du, 0x33A8A3D6u, + 0x09DCD1CEu, 0xA1B10212u, 0xF48E1156u, 0x8F039915u, + 0x8A055EAAu, 0xFF5B11D5u, 0xB725085Bu, 0x2E1AB267u, + 0x6AE7323Du, 0xB2FF6FA8u, 0x7102C8A1u, 0x7561EB37u}, + {0xFF525F0Fu, 0xD892E3D2u, 0xFB566B40u, 0x3BDF4ED0u, + 0x78B961CDu, 0x9CB86B48u, 0x6AB54F3Du, 0x3EF5F695u, + 0xBD896ED8u, 0x6265AC08u, 0xF6695D78u, 0x9F3795EAu, + 0x943E0342u, 0xD1437B3Bu, 0x4F6BAF78u, 0x85DFD2C9u}, + }, + { + {0x9F767C45u, 0xBDE5C099u, 0xF17FD374u, 0xA6233255u, + 0xE6A16A3Bu, 0x1CFB10F6u, 0x3F1F65A8u, 0x8B33E968u}, + {0x92EDCF45u, 0x377B9AA2u, 0x478C281Du, 0xC4069545u, + 0xCC11D357u, 0x9E115E4Bu, 0x206F5C66u, 0xDF1461AAu, + 0xFB7FF337u, 0xDF561D80u, 0x4A0FE75Du, 0xF6236BF2u, + 0x346C6E2Bu, 0xB0CDE917u, 0xE4CC4132u, 0x4C7D6DF0u}, + 0x6A3753915C76F18Aull, 18u, 67u, + {0x14A9F66Fu, 0x101BDFE8u, 0x9B0A50DDu, 0xEE4BB45Bu, + 0x7A914502u, 0x77B3486Bu, 0x59BFC114u, 0xA1AD2AFDu, + 0xC194DDE6u, 0x894EC54Du, 0xAD36C805u, 0x9018F3F5u, + 0x165AF5D8u, 0x3E85B598u, 0x78E76653u, 0xBB7A485Du}, + {0xD22912BBu, 0x627F992Cu, 0xE883AF5Du, 0x50E58A48u, + 0xF3D071C6u, 0xB20D47A4u, 0x29011151u, 0xFE50E232u, + 0x594B76A3u, 0x8706296Bu, 0x2C1D1E31u, 0x6A478D0Du, + 0x64004E61u, 0xA072DA1Eu, 0xAB3FCA42u, 0x09BB269Eu}, + }, + { + {0xD26B9496u, 0x42F9A039u, 0x001D9A88u, 0x5F877031u, + 0xC527E279u, 0x45CF8AA4u, 0xCD4A5557u, 0xAE9AF169u}, + {0xAF895F5Bu, 0xD822E2F9u, 0x17D7AB26u, 0xCCDF540Bu, + 0xCE06294Du, 0x4A8B0188u, 0xF38D2E64u, 0x5C41D5C5u, + 0xE8D5B9E3u, 0x5C832A51u, 0x9A0C1B76u, 0x4DE8344Eu, + 0x96D2F9E0u, 0x8677A5F2u, 0xA9A967C1u, 0x323BBEAFu}, + 0x390567C27BD6AA42ull, 26u, 3u, + {0x32A6FF70u, 0xC30560BCu, 0xD1C777C8u, 0xF1871821u, + 0x7207AB54u, 0x9F5B83C7u, 0xB6561C5Du, 0x991E738Fu, + 0xB38B62B9u, 0x0EF6D156u, 0x994BECB1u, 0x09A85D0Eu, + 0x32221741u, 0xADA3CC5Fu, 0x5B654ED6u, 0x2A7A62B2u}, + {0xA101CEABu, 0x9232E0ECu, 0x2FE4B24Eu, 0x35F7F4FEu, + 0x61A5AB42u, 0xBE417503u, 0xEB740D5Eu, 0x8BB2FE96u, + 0xC6863DA9u, 0x1F31FF5Du, 0x5763EA12u, 0xDC862699u, + 0x1A60ADE2u, 0x9E3E6745u, 0xE3C8F87Eu, 0xD3EFB0EAu}, + }, + { + {0x269E0D37u, 0xA6A3A450u, 0x892F902Bu, 0x81E74EF5u, + 0x099950D8u, 0x6F03675Au, 0x11E20B8Fu, 0x6CAD4A26u}, + {0xF29D0DA9u, 0x658CDA14u, 0xF9EBDACCu, 0xDBC496CBu, + 0x4A23D596u, 0x2E44158Bu, 0xA38FD547u, 0x5F557203u, + 0x34B9B5DFu, 0x506BF2EFu, 0x7403E430u, 0x4CBD87ADu, + 0xCB5C7427u, 0x3E7D1BFBu, 0x930D6EAFu, 0x86734721u}, + 0x12BD4ACEFAECBD38ull, 53u, 42u, + {0xA632AD45u, 0x12CE41F4u, 0xD21B2CBDu, 0x76795C62u, + 0x6BEC36C1u, 0xDAFAFCDEu, 0x53CA87B7u, 0x92E8465Bu, + 0x7B424F5Du, 0xE1E6AD7Fu, 0x753BA387u, 0xCCC50824u, + 0x69AEDF6Du, 0xBBBBF253u, 0x78D04883u, 0xF3F33689u}, + {0x318604BEu, 0x22A35843u, 0x6CA63195u, 0xA2E7E2F8u, + 0x48769A04u, 0xC462F1E3u, 0x5CF053C7u, 0xFD1EE629u, + 0x69366332u, 0x0ACC819Bu, 0xBBD2456Au, 0xF1DA9DB6u, + 0x4A7B7D68u, 0x6DD1A843u, 0x61555466u, 0xBDA36F28u}, + }, + { + {0x3A096533u, 0xF658F7A7u, 0x205738D1u, 0xB46EE1DAu, + 0x15CEB3A1u, 0x359B1548u, 0xA4517D6Cu, 0x7589CA4Au}, + {0x74007CB4u, 0xD49D0AC1u, 0x16EDC5D4u, 0x685CA8AFu, + 0x4223AA56u, 0x10269470u, 0x60908405u, 0xA92D04A3u, + 0x56A3E957u, 0xB0F91306u, 0xE6C08269u, 0xF2306D4Au, + 0x31A06A7Cu, 0x9436D6F6u, 0xE18692E2u, 0xE0C99F3Eu}, + 0x329911DA9FBD8735ull, 19u, 91u, + {0x913B2AE1u, 0xC7F73082u, 0x45E1C023u, 0x6F1F3F82u, + 0x20AEE6F5u, 0xDAF21D94u, 0xF2C1E4AFu, 0xD4F7D4ACu, + 0x44A45F87u, 0xF4C40CE5u, 0x613E9B94u, 0x08CE53DEu, + 0x4FF07AA4u, 0x456BF2E2u, 0x2066EA7Fu, 0x3C5A654Bu}, + {0x87584719u, 0x15C73090u, 0x851C1A4Au, 0x99D21014u, + 0x821A82A8u, 0xC7307CD5u, 0x6797EFE2u, 0xCF38CEDFu, + 0x777C177Du, 0x202BE3EAu, 0x19421985u, 0x3176132Du, + 0x7BB8BC22u, 0x65C9804Bu, 0x22C68EA3u, 0x92504162u}, + }, + { + {0x5F915EF0u, 0x237751AAu, 0x01A5BA50u, 0x80B65386u, + 0x14B044D7u, 0x61076DC3u, 0xB99DE255u, 0x283B73A6u}, + {0x3CEE5E2Cu, 0x1C670EA9u, 0x972651DAu, 0x4A8AA593u, + 0xAC9ABB0Cu, 0x35BB5C11u, 0x47FBB3B4u, 0xCF3C17E5u, + 0xE2EB17C8u, 0xE11E99FBu, 0x7DE0D208u, 0x0602FE0Cu, + 0x98CAE043u, 0x9425B3E2u, 0x33FB4B4Fu, 0x15607DF9u}, + 0xEAEB999B8A2E547Eull, 64u, 21u, + {0xF5EE9114u, 0x856CABB8u, 0x29BE2CF1u, 0x603BE91Cu, + 0x94A7DD0Eu, 0x28FC3E27u, 0xB64E2CC8u, 0x2D2C67FFu, + 0x69FAC1BAu, 0x0C949090u, 0xD68DE435u, 0xCE91A527u, + 0xE80C1815u, 0x6D44EFE6u, 0x87C7B175u, 0xD18A8B94u}, + {0xDC60D189u, 0xE6311F18u, 0x9DC3E078u, 0x304BB43Eu, + 0x5C616E7Du, 0xE168D00Fu, 0x2E197872u, 0x175B9188u, + 0x5A99C462u, 0xEF311A88u, 0xC61836FDu, 0x9FFD4DE3u, + 0x36AE4940u, 0x4D813D81u, 0x9B058DA9u, 0x9017D38Cu}, + }, +}; + +// --------------------------------------------------------------------------- +// Table 3 — MULTI-BLOCK official BLAKE3 vectors, for the `Blake3Chain` +// construction rather than the bare compression function. +// +// Table 1 above stops at 64 bytes because a single compression is all it can +// check. The chain spans many blocks, and its flag schedule, its `block_len` +// handling and its chaining value are only exercised past the first block — so +// it needs vectors Table 1 cannot supply. +// +// ★ These are still the OFFICIAL vectors, not an oracle's. `Blake3Chain` over a +// message of at most one chunk (1024 bytes) IS `blake3::hash` — standard +// BLAKE3's first chunk is exactly this chain, and a one-chunk message has that +// chunk's output as its root (PA-PLAN §1.7.2, P1). So for every length here up +// to 1024 the published hash is a direct known-answer test for the device +// chain, with no oracle and no transcription of anything computed in this repo. +// +// Transcribed from `tooling/blake3-second-source/blake3-oracle/official_test_vectors.json` +// (tracked), first 32 bytes of each case's `hash`. Input for length N is the +// first N bytes of the repeating 251-byte sequence 0, 1, ..., 250 — the same +// generator Table 1 uses. +// +// `agrees` marks whether the chain must MATCH the published hash. The 1025 and +// 2048 rows are the P3 negative control: past one chunk standard BLAKE3 starts +// chunk 1 with a reset chaining value and builds a tree, and this construction +// deliberately does not. Without them "we implement the single-chunk chain" +// would be unfalsifiable — the matching rows alone would pass identically if the +// whole chunk tree had been implemented instead. +// +// ★ THE BOUNDARY IS LOCATED, not sampled. All 35 official cases were swept +// (2026-08-15): agreement holds for every length up to and including 1024, and +// fails for every one of the 18 lengths >= 1025. Max agreeing 1024, min +// differing 1025 — the divergence sits exactly on the one-chunk edge, which is +// what P3 predicts. The 1024/1025 pair below is that boundary; the other rows +// cover the multi-block cases in between. +// +// The input generator `i % 251` was itself verified empirically rather than +// taken from the file's prose: it reproduces the `hash` field for 35/35 cases. +// Note it differs from the `(37i + 11) mod 256` generator Table 4 uses — the two +// tables come from different sources and do NOT share a message. +// --------------------------------------------------------------------------- +struct ChainVector { + uint32_t input_len; + const char *hash_hex; + bool agrees; // false = must DIFFER (past one chunk) +}; + +inline constexpr int NUM_CHAIN_VECTORS = 8; +inline constexpr ChainVector CHAIN_VECTORS[NUM_CHAIN_VECTORS] = { + { 65, "de1e5fa0be70df6d2be8fffd0e99ceaa8eb6e8c93a63f2d8d1c30ecb6b263dee", true}, + { 127, "d81293fda863f008c09e92fc382a81f5a0b4a1251cba1634016a0f86a6bd640d", true}, + { 128, "f17e570564b26578c33bb7f44643f539624b05df1a76c81f30acd548c44b45ef", true}, + { 129, "683aaae9f3c5ba37eaaf072aed0f9e30bac0865137bae68b1fde4ca2aebdcb12", true}, + { 1023, "10108970eeda3eb932baac1428c7a2163b0e924c9a9e25b35bba72b28f70bd11", true}, + { 1024, "42214739f095a406f3fc83deb889744ac00df831c10daa55189b5d121c855af7", true}, + { 1025, "d00278ae47eb27b34faecf67b4fe263f82d5412916c1ffd97c8cb7fb814b8444", false}, + { 2048, "e776b6028c7cd22a4d0ba182a8bf62205d2ef576467e838ed6f2529b85fba24a", false}, +}; + +// --------------------------------------------------------------------------- +// Table 4 — the committed 6-ROUND chain KAT. +// +// A byte-for-byte transcription of `CHAIN_KAT_6ROUND` +// (`crypto/crypto/src/hash/blake3/chain.rs:304`), which is the same table +// PA-PLAN §1.7.5 records. Message of length N is byte `i = 37i + 11 (mod 256)`. +// +// Provenance, and why it is worth more than a self-comparison: those digests +// were produced by #903's Python oracle +// (`tooling/blake3-second-source/blake3-oracle/blake3_ref.py`, tracked), a full +// standard-BLAKE3 implementation with the round count as a parameter, whose +// 7-round arm reproduces the official package bit-for-bit. So at the round count +// the campaign actually ships, this pins the device chain against numbers no +// Rust and no CUDA in this tree computed. +// +// Duplicating the table here rather than sharing one copy is deliberate: this +// harness compiles as standalone C++ with no cargo and no Rust in the build, so +// there is nothing to share it with. A drift between the two copies is caught by +// the Rust-side `device_chain_matches_the_committed_table_at_six_rounds`, which +// reads the Rust constant directly. +// --------------------------------------------------------------------------- +struct ChainKat6Round { + uint32_t input_len; + const char *hash_hex; +}; + +inline constexpr int NUM_CHAIN_KAT_6ROUND = 12; +inline constexpr ChainKat6Round CHAIN_KAT_6ROUND[NUM_CHAIN_KAT_6ROUND] = { + { 0, "3c3bbb1f335a31ea86464b651c0206fc81d33262ae00ea1a65f3d1d04afaefc9"}, + { 1, "2a50e45b8921f9efa008d9f39f7165600cf48a7f0e859c2122e3ccb6b9677ee5"}, + { 31, "c38bf62f506040b2600273778d281b8943621e2b8a9f59e2379f8fd7e5c85125"}, + { 63, "c373f51a5eb8b27ea05bb1f6f4e62e924ff4d8a279f0d05afa5cd519391d6389"}, + { 64, "5900a1e398bb2bf6d3ba7f1a29197b79c86b71ad2c2631f4ac736c82db043cb5"}, + { 65, "53953fcadc39b8623901af7b534f2f6933e312f50299331334e6c0a7c9dbc2be"}, + { 127, "9e0dd8168d199a04590c2cba439b270776e42715d518f68655e56692483e505e"}, + { 128, "5caffc8784e817bbba991b2108c26a3dfdf804245ef63ae1040a3c34f1b362ff"}, + { 192, "399d6b9adeb2f88450775f773e9dec08836c135713c2c5dd09f4ceceb0ed3888"}, + { 256, "fbcab3699a4959fa37190e98ca5142ddbc88330f2e7d12335db9c6c8881a0b87"}, + { 1024, "f395e7e2150363b6d200487515425b0204eea424072183b701176eccbe0ffe1b"}, + { 1088, "b4738ede77a6ec166ee97667118d4793cbf2b08b45aac7c6d52943b5d298c688"}, +}; diff --git a/crypto/math-cuda/tests/host_kat/cuda_host_shim.h b/crypto/math-cuda/tests/host_kat/cuda_host_shim.h new file mode 100644 index 000000000..8993489d2 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/cuda_host_shim.h @@ -0,0 +1,106 @@ +// Enough of the CUDA language to compile a `.cu` kernel file as ordinary host +// C++, so its arithmetic can be checked without a GPU. +// +// This exists because the GPU parity suite (`crypto/math-cuda/tests/blake3_*.rs`) +// runs only where a GPU does, and per-PR CI has none. Including a kernel through +// this shim turns its device functions into plain functions a host program can +// call, which is all a known-answer test needs. +// +// ⚠ What it CANNOT check, and what therefore still belongs to the GPU tests: +// anything about execution rather than arithmetic — thread/block indexing, +// `__syncthreads` ordering, memory alignment on device, register pressure, and +// whether nvcc accepts the file at all. A kernel that passes here can still be +// wrong on a GPU. Treat this as a lower bound on correctness, never a substitute. +#pragma once + +#include + +// The execution-space and inlining qualifiers carry no meaning on host. +#define __device__ +#define __constant__ +#define __forceinline__ inline +#define __global__ + +// Single-threaded host execution: one thread, block 0, and a barrier that has +// nothing to wait for. Kernel thread coordinates are ordinary mutable globals, +// so a caller can drive them (see `CUDA_HOST_FOR_EACH_THREAD`) and replay a +// whole launch's worth of thread slices one at a time. +#define __syncthreads() ((void)0) +struct CudaHostDim3 { + unsigned x = 0, y = 0, z = 0; +}; +static CudaHostDim3 blockIdx; +static CudaHostDim3 threadIdx; +static CudaHostDim3 cuda_host_block_dim; +static CudaHostDim3 cuda_host_grid_dim; +#define blockDim cuda_host_block_dim +#define gridDim cuda_host_grid_dim + +// The one atomic the grid-stride search kernels use. Single-threaded on host, +// so the read-modify-write needs no protection; it returns the OLD value, as +// CUDA's does, and takes a non-volatile pointer because the kernels cast the +// volatility away at the call (the `volatile` there is for the *reads* that +// drive the early exit, which the shim's single thread makes moot). +static inline unsigned long long atomicMin(unsigned long long *address, + unsigned long long val) { + unsigned long long old = *address; + if (val < old) *address = val; + return old; +} + +// And the counter a comparison kernel keeps (`whir_tile_materialize_check`): +// the same single-threaded read-modify-write, returning the old value. +static inline unsigned long long atomicAdd(unsigned long long *address, unsigned long long val) { + unsigned long long old = *address; + *address = old + val; + return old; +} + +// `goldilocks.cuh`'s field multiply needs this intrinsic. `blake3.cu` only uses +// `goldilocks::canonical`, but the header compiles as a whole, so supply it. +static inline uint64_t __umul64hi(uint64_t a, uint64_t b) { + return (uint64_t)(((unsigned __int128)a * (unsigned __int128)b) >> 64); +} + +// Bit-reverse a 64-bit word. Every leaf kernel derives its row index as +// `__brevll(tid) >> (64 - log_num_rows)`, so replaying one on host needs it. +// Written out rather than deferring to a compiler builtin so the shim stays +// toolchain-neutral. +static inline uint64_t __brevll(uint64_t x) { + x = ((x & 0x5555555555555555ull) << 1) | ((x >> 1) & 0x5555555555555555ull); + x = ((x & 0x3333333333333333ull) << 2) | ((x >> 2) & 0x3333333333333333ull); + x = ((x & 0x0F0F0F0F0F0F0F0Full) << 4) | ((x >> 4) & 0x0F0F0F0F0F0F0F0Full); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x >> 8) & 0x00FF00FF00FF00FFull); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x >> 16) & 0x0000FFFF0000FFFFull); + return (x << 32) | (x >> 32); +} + +// Replay a `__global__` kernel once per thread index, sequentially, by driving +// the shim's thread coordinates. A kernel computing +// `tid = blockIdx.x * blockDim.x + threadIdx.x` sees `tid = i` on iteration `i`, +// so a whole launch can be reproduced on host: +// +// CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) some_leaf_kernel(args...); +// +// ⚠ Only valid for kernels whose threads are independent — which the leaf +// kernels are (one thread, one leaf, disjoint output) and the Merkle *tail* is +// not. It says nothing about `__syncthreads` ordering, races or occupancy. +#define CUDA_HOST_FOR_EACH_THREAD(i, n) \ + for (unsigned i = 0; \ + i < (unsigned)(n) && \ + (blockIdx.x = 0, blockDim.x = 0, threadIdx.x = i, true); \ + ++i) + +// Replay a GRID-STRIDE kernel as ONE thread that covers the whole range: +// `tid = 0`, `stride = gridDim.x * blockDim.x = 1`, so a kernel written as +// `for (i = tid; i < count; i += stride)` scans `[0, count)` in order. +// +// `CUDA_HOST_FOR_EACH_THREAD` cannot do this — it leaves `blockDim.x = 0`, +// which is a zero stride and an unterminated loop. +// +// ⚠ Says nothing about the parallel reduction a real launch performs. What it +// checks is the per-candidate arithmetic and the loop's bounds; that +// `atomicMin` over many threads yields the same answer is a property of the +// reduction, pinned on a GPU. +#define CUDA_HOST_SINGLE_THREAD() \ + (gridDim.x = 1, blockDim.x = 1, blockIdx.x = 0, threadIdx.x = 0, (void)0) diff --git a/crypto/math-cuda/tests/host_kat/cuda_host_simt_shim.h b/crypto/math-cuda/tests/host_kat/cuda_host_simt_shim.h new file mode 100644 index 000000000..e4be76f7a --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/cuda_host_simt_shim.h @@ -0,0 +1,156 @@ +// Enough of the CUDA language to run a kernel that COOPERATES ACROSS LANES as +// host C++: every thread of a block is a real `std::thread`, `__shfl_sync` is an +// exchange through a per-warp slot array between two barriers, `__syncthreads` +// is a block-wide barrier, and the atomics are host atomics. +// +// `cuda_host_shim.h` replays a launch one thread at a time, which is exact for +// kernels whose threads never talk to each other and meaningless for the ones +// that do (a shuffle from lane 5 needs lane 5 to be running). This shim exists +// for the second kind: the half-warp permutation and the work-queue grind. +// +// ⚠ What it CANNOT check, as for the single-thread shim: nvcc acceptance, +// register pressure, timing, and anything the hardware does that a converged +// warp of host threads does not (it assumes every lane of a warp reaches every +// `__shfl_sync`, which is what the kernels guarantee and what the hardware +// requires). Blocks run one after another, not concurrently. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#define __device__ +#define __constant__ +#define __forceinline__ inline +#define __global__ + +struct CudaHostDim3 { + unsigned x = 0, y = 0, z = 0; +}; +static thread_local CudaHostDim3 threadIdx; +static thread_local CudaHostDim3 blockIdx; +static CudaHostDim3 cuda_host_block_dim; +static CudaHostDim3 cuda_host_grid_dim; +#define blockDim cuda_host_block_dim +#define gridDim cuda_host_grid_dim + +// A reusable barrier for a fixed party count (C++17 has none). +class SimtBarrier { + public: + explicit SimtBarrier(unsigned parties) : parties_(parties) {} + void arrive_and_wait() { + std::unique_lock lock(m_); + const unsigned gen = generation_; + if (++arrived_ == parties_) { + arrived_ = 0; + ++generation_; + cv_.notify_all(); + } else { + cv_.wait(lock, [&] { return generation_ != gen; }); + } + } + + private: + std::mutex m_; + std::condition_variable cv_; + unsigned parties_; + unsigned arrived_ = 0; + unsigned generation_ = 0; +}; + +struct SimtWarp { + explicit SimtWarp(unsigned lanes) : bar(lanes) {} + SimtBarrier bar; + unsigned long long slots[32] = {}; +}; + +// The running block's warps and block barrier (blocks run one at a time). +static std::vector> *simt_warps = nullptr; +static SimtBarrier *simt_block_barrier = nullptr; + +inline unsigned long long __shfl_sync(unsigned /*mask*/, unsigned long long v, int src) { + SimtWarp &w = *(*simt_warps)[threadIdx.x / 32]; + w.slots[threadIdx.x & 31u] = v; + w.bar.arrive_and_wait(); + const unsigned long long r = w.slots[(unsigned)src & 31u]; + w.bar.arrive_and_wait(); + return r; +} + +#define __syncthreads() (simt_block_barrier->arrive_and_wait()) + +inline unsigned long long atomicAdd(unsigned long long *p, unsigned long long v) { + return __atomic_fetch_add(p, v, __ATOMIC_SEQ_CST); +} + +inline unsigned int atomicAdd(unsigned int *p, unsigned int v) { + return __atomic_fetch_add(p, v, __ATOMIC_SEQ_CST); +} + +// A launch's last block learns it is last from a ticket the others took after a fence; on the +// host every thread's stores are fenced the same way. +inline void __threadfence() { __atomic_thread_fence(__ATOMIC_SEQ_CST); } +inline void __threadfence_system() { __atomic_thread_fence(__ATOMIC_SEQ_CST); } + +inline unsigned long long atomicMin(unsigned long long *p, unsigned long long v) { + unsigned long long old = __atomic_load_n(p, __ATOMIC_SEQ_CST); + while (v < old && + !__atomic_compare_exchange_n(p, &old, v, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + } + return old; +} + +inline unsigned long long atomicMax(unsigned long long *p, unsigned long long v) { + unsigned long long old = __atomic_load_n(p, __ATOMIC_SEQ_CST); + while (v > old && + !__atomic_compare_exchange_n(p, &old, v, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + } + return old; +} + +// `goldilocks.cuh`'s multiply and the leaf kernels' row index, as in the +// single-thread shim. +static inline uint64_t __umul64hi(uint64_t a, uint64_t b) { + return (uint64_t)(((unsigned __int128)a * (unsigned __int128)b) >> 64); +} + +static inline uint64_t __brevll(uint64_t x) { + x = ((x & 0x5555555555555555ull) << 1) | ((x >> 1) & 0x5555555555555555ull); + x = ((x & 0x3333333333333333ull) << 2) | ((x >> 2) & 0x3333333333333333ull); + x = ((x & 0x0F0F0F0F0F0F0F0Full) << 4) | ((x >> 4) & 0x0F0F0F0F0F0F0F0Full); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x >> 8) & 0x00FF00FF00FF00FFull); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x >> 16) & 0x0000FFFF0000FFFFull); + return (x << 32) | (x >> 32); +} + +// Launch `kernel` over `grid` blocks of `block` threads: each block's threads +// run concurrently as real threads; blocks run in order. `block` must be a +// multiple of 32, as the warp kernels require. +inline void simt_launch(unsigned grid, unsigned block, const std::function &kernel) { + cuda_host_grid_dim.x = grid; + cuda_host_grid_dim.y = 1; + cuda_host_block_dim.x = block; + for (unsigned b = 0; b < grid; ++b) { + std::vector> warps; + for (unsigned w = 0; w < block / 32; ++w) warps.emplace_back(new SimtWarp(32)); + SimtBarrier block_barrier(block); + simt_warps = &warps; + simt_block_barrier = &block_barrier; + std::vector threads; + threads.reserve(block); + for (unsigned t = 0; t < block; ++t) { + threads.emplace_back([&kernel, b, t] { + blockIdx.x = b; + threadIdx.x = t; + kernel(); + }); + } + for (auto &th : threads) th.join(); + simt_warps = nullptr; + simt_block_barrier = nullptr; + } +} diff --git a/crypto/math-cuda/tests/host_kat/eq_dot_host_kat.cpp b/crypto/math-cuda/tests/host_kat/eq_dot_host_kat.cpp new file mode 100644 index 000000000..2da6c4cf4 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/eq_dot_host_kat.cpp @@ -0,0 +1,177 @@ +// Known-answer tests for the commit-time out-of-domain answers' column values +// (`kernels/sumcheck.cu`: `eq_dot_base_ext3_many`, then `sum_partials_ext3` +// over its partials, as `math_cuda::sumcheck::eq_dot_many_base` launches +// them), run on the host through `cuda_host_simt_shim.h` — every thread of a +// block a real thread, so the block reductions run as written. No GPU, no +// nvcc. +// +// WHAT IT COVERS, for every shape (the columns' variables, how many low ones +// the `lo` half takes, how many columns, the blocks a column gets — fewer +// than its hi rows, as many, more — and the block's threads, more than a hi +// row's lo slots and fewer): +// each column's value against a written-out reference: the sum over its +// rows of the value times `eq(p, row)` as a product over the row's bits, +// the first variable the high bit (canonical values). The columns' words +// include non-canonical representatives, as a card may hold them. +// +// WHAT IT DOES NOT COVER, and the card tests still own: whether nvcc accepts +// the file, the hardware's memory and timing, the driver's launch +// (`multilinear/tests/whir_commit_ood_device.rs` on a box). +// +// Build and run with `make test-eq-dot-host-kat`. + +#include +#include +#include + +#include "cuda_host_simt_shim.h" + +// The kernels' block-shared buffer: blocks run one at a time here, so one +// buffer serves every block. `extern "C"` because the kernels that name it +// are. The launch bounds mean nothing on the host. +#define __shared__ +#define __launch_bounds__(...) +extern "C" { +uint64_t shared[3 * 1024]; +} + +#include "sumcheck.cu" + +namespace { + +int failures = 0; + +void check(bool ok, const std::string &what) { + if (!ok) { + printf("FAIL: %s\n", what.c_str()); + ++failures; + } +} + +const uint64_t P = 0xFFFFFFFF00000001ull; + +uint64_t state = 0x13198A2E03707344ull; +uint64_t next_u64() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; +} + +// A raw limb: canonical most of the time, and every eighth one a +// non-canonical representative in [p, 2^64). +uint64_t raw_limb() { + uint64_t v = next_u64() % P; + if ((next_u64() & 7) == 0 && v < 0xFFFFFFFFull) v += P; + return v; +} + +Fe3 random_ext() { return ext3::make(raw_limb(), raw_limb(), raw_limb()); } + +bool same_value(const Fe3 &a, const Fe3 &b) { + Fe3 x = ext3::canonical(a), y = ext3::canonical(b); + return x.a == y.a && x.b == y.b && x.c == y.c; +} + +// `eq(r, x)` for every `x` of `r.size()` bits, the first variable the high +// bit (`multilinear::eq::eq_evals`). +std::vector eq_table(const std::vector &r) { + size_t n = r.size(); + std::vector t(1, ext3::make(1, 0, 0)); + for (size_t j = 0; j < n; ++j) { + std::vector next(t.size() * 2); + Fe3 one_minus = ext3::sub(ext3::make(1, 0, 0), r[j]); + for (size_t i = 0; i < t.size(); ++i) { + next[2 * i] = ext3::mul(t[i], one_minus); + next[2 * i + 1] = ext3::mul(t[i], r[j]); + } + t.swap(next); + } + std::vector raw; + for (const Fe3 &v : t) { + raw.push_back(v.a); + raw.push_back(v.b); + raw.push_back(v.c); + } + return raw; +} + +struct Shape { + unsigned vars; + unsigned lo_bits; + unsigned columns; + unsigned blocks; + unsigned threads; +}; + +void run(const Shape &s) { + uint64_t rows = 1ull << s.vars; + std::vector columns(rows * s.columns); + for (auto &v : columns) v = raw_limb(); + std::vector point(s.vars); + for (auto &c : point) c = random_ext(); + std::vector high(point.begin(), point.end() - s.lo_bits); + std::vector low(point.end() - s.lo_bits, point.end()); + std::vector hi = eq_table(high), lo = eq_table(low); + + std::vector partials(3 * (uint64_t)s.columns * s.blocks, 0); + for (unsigned y = 0; y < s.columns; ++y) { + simt_launch(s.blocks, s.threads, [&] { + blockIdx.y = y; + eq_dot_base_ext3_many(columns.data(), rows, s.lo_bits, hi.data(), lo.data(), + partials.data()); + }); + } + std::vector sums(3 * s.columns, 0); + simt_launch(s.columns, 256, [&] { + sum_partials_ext3(partials.data(), s.blocks, sums.data()); + }); + + for (unsigned k = 0; k < s.columns; ++k) { + Fe3 want = ext3::zero(); + for (uint64_t row = 0; row < rows; ++row) { + Fe3 eq = ext3::make(1, 0, 0); + for (unsigned j = 0; j < s.vars; ++j) { + bool bit = (row >> (s.vars - 1 - j)) & 1; + eq = ext3::mul(eq, bit ? point[j] : ext3::sub(ext3::make(1, 0, 0), point[j])); + } + want = ext3::add(want, ext3::mul_base(eq, columns[k * rows + row])); + } + Fe3 got = ext3::make(sums[k * 3], sums[k * 3 + 1], sums[k * 3 + 2]); + check(same_value(got, want), + "vars " + std::to_string(s.vars) + " lo " + std::to_string(s.lo_bits) + + " columns " + std::to_string(s.columns) + " blocks " + + std::to_string(s.blocks) + " threads " + std::to_string(s.threads) + + ": column " + std::to_string(k)); + } +} + +} // namespace + +int main() { + std::vector shapes = { + // One hi row, lo slots fewer than the threads. + {1, 1, 1, 1, 32}, + {3, 3, 2, 1, 32}, + {5, 5, 3, 1, 64}, + // One hi row of 2^10 lo slots, four a thread. + {10, 10, 3, 1, 256}, + // Several hi rows: blocks fewer than them (each strides), as many, and + // more (the extra blocks write zero partials). + {12, 10, 4, 2, 256}, + {12, 10, 2, 4, 256}, + {12, 10, 2, 8, 256}, + {13, 10, 2, 3, 128}, + // A short low half, as a narrow point takes it. + {8, 4, 5, 7, 32}, + {11, 3, 2, 64, 64}, + }; + for (const Shape &s : shapes) run(s); + if (failures) { + printf("eq_dot host KAT: %d FAILED\n", failures); + return 1; + } + printf("eq_dot host KAT: %zu shapes, every column's value the written-out sum\n", + shapes.size()); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/fill_host_kat.cpp b/crypto/math-cuda/tests/host_kat/fill_host_kat.cpp new file mode 100644 index 000000000..1d9d0fa61 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/fill_host_kat.cpp @@ -0,0 +1,283 @@ +// Known-answer tests for `kernels/fill.cu`, the table fills from records, run +// on the host. +// +// WHY THIS EXISTS. A fill is right when every column of every row gets the +// word the prover's host fill writes there: which record field, which setter, +// which column, which row. That is arithmetic and indexing, and this checks it +// without a GPU: the kernel source compiles through `cuda_host_shim.h`, and a +// launch is replayed one thread at a time (the kernels' threads are +// independent: one thread a row, disjoint cells). The expected words come from +// the prover's host fill (`fill_kat_vectors.h`, written and pinned by the +// prover's `tables::card_fill_inputs`); nothing here computes them. +// +// WHAT IT COVERS: +// 1. MEMW_R: thirteen register rows, then zero rows, every word. +// 1a. MEMW_A: thirteen aligned rows (memory and register accesses, reads and +// writes, every width), then zero rows, every word. +// 1b. LOAD: nineteen ops (every width, signed or not, sign bits set and +// clear), then zero rows, every word. +// 1c. STORE: eleven ops at every width, then zero rows, every word. +// 1d. SHIFT: 123 ops, every direction, signedness and width at sixteen edge +// amounts (bit shift 0, limb shifts, 32 and 64 bits, high amount bits), +// then padding rows (ZBS 1), every word. +// 1e. MEMW: thirteen ops (memory and register, every width, reads and +// writes, low words about to carry, an old timestamp of 2^64 - 1), then +// zero rows, every word. +// 2. CPU: 29 logs over 24 decode entries in three runs (every op class the +// fill branches on, each entry in turn), then padding rows, every word; +// and a log whose pc no run holds, counted as missing with its row left +// zero. +// 3. KECCAK_RND: a table cut into its first op, holding every round of the +// second and zero rows after it, and one cut at both ends: every row's +// fifteen column groups, by digest; the same through the entry that reads +// the records a table held as its keccak operations uploads. +// Every cell of each table starts as a sentinel, so a cell no thread writes +// is a mismatch, and threads past the table must leave a guard alone. +// +// WHAT IT DOES NOT COVER: whether nvcc accepts the file, and everything about +// execution: the grid, coalescing, the sections of a KECCAK_RND row running as +// separate blocks. That is the card test's (`prover/src/tables/card_fill_tests.rs`). +// Passing here is necessary, never sufficient. +// +// Build and run with `make test-fill-host-kat`. + +#include +#include +#include + +#include "cuda_host_shim.h" + +#include "fill.cu" + +#include "fill_kat_vectors.h" + +namespace { + +int g_failures = 0; +int g_checks = 0; + +#define CHECK(cond, ...) \ + do { \ + ++g_checks; \ + if (!(cond)) { \ + ++g_failures; \ + std::printf("FAIL %s:%d: ", __FILE__, __LINE__); \ + std::printf(__VA_ARGS__); \ + std::printf("\n"); \ + } \ + } while (0) + +constexpr uint64_t UNWRITTEN = 0x5A5A5A5A5A5A5A5Aull; +constexpr uint64_t GUARD = 0xDEADBEEFCAFEF00Dull; +constexpr size_t GUARD_WORDS = 64; +// Threads launched past a table's rows. +constexpr unsigned EXTRA_THREADS = 7; + +// A table of `words` cells, each UNWRITTEN, then a guard. +std::vector table(size_t words) { + std::vector t(words + GUARD_WORDS, GUARD); + for (size_t i = 0; i < words; ++i) t[i] = UNWRITTEN; + return t; +} + +bool guard_intact(const std::vector &t, size_t words) { + for (size_t i = words; i < t.size(); ++i) { + if (t[i] != GUARD) return false; + } + return true; +} + +// One launch of a row kernel, replayed: thread `i` of column section `y` is row +// `i`, for every row and EXTRA_THREADS more. +template +void each_row(uint64_t rows, unsigned y, Launch launch) { + for (unsigned i = 0; i < rows + EXTRA_THREADS; ++i) { + blockIdx.x = 0; + blockIdx.y = y; + blockDim.x = 0; + threadIdx.x = i; + launch(); + } +} + +// Every word of a column-major table against the expected; the first few +// mismatches named. +void check_words(const char *name, const std::vector &got, const uint64_t *want, + uint64_t rows, uint64_t cols) { + int named = 0; + uint64_t bad = 0; + for (uint64_t c = 0; c < cols; ++c) { + for (uint64_t r = 0; r < rows; ++r) { + uint64_t i = c * rows + r; + if (got[i] != want[i]) { + ++bad; + if (named++ < 8) { + std::printf(" %s column %llu row %llu: got %llu, want %llu\n", name, + (unsigned long long)c, (unsigned long long)r, + (unsigned long long)got[i], (unsigned long long)want[i]); + } + } + } + } + CHECK(bad == 0, "%s: %llu of %llu words differ", name, (unsigned long long)bad, + (unsigned long long)(rows * cols)); + CHECK(guard_intact(got, rows * cols), "%s: a thread past the table wrote", name); +} + +// FNV-1a over the words' little-endian bytes (the vectors' digest). +uint64_t fnv1a(const uint64_t *words, uint64_t n, uint64_t stride) { + uint64_t h = 0xCBF29CE484222325ull; + for (uint64_t k = 0; k < n; ++k) { + uint64_t w = words[k * stride]; + for (int b = 0; b < 8; ++b) { + h = (h ^ ((w >> (8 * b)) & 0xFF)) * 0x00000100000001B3ull; + } + } + return h; +} + +void test_memw_r() { + const uint64_t height = KAT_MEMW_R_HEIGHT; + const uint64_t cols = memw_r_cols::NUM_COLUMNS; + auto out = table(height * cols); + each_row(height, 0, [&] { fill_memw_r(KAT_MEMW_R_ROWS, KAT_MEMW_R_N, height, out.data()); }); + check_words("MEMW_R", out, KAT_MEMW_R_WORDS, height, cols); +} + +void test_memw_a() { + const uint64_t height = KAT_MEMW_A_HEIGHT; + const uint64_t cols = memw_a_cols::NUM_COLUMNS; + auto out = table(height * cols); + each_row(height, 0, [&] { fill_memw_a(KAT_MEMW_A_ROWS, KAT_MEMW_A_N, height, out.data()); }); + check_words("MEMW_A", out, KAT_MEMW_A_WORDS, height, cols); +} + +void test_load() { + const uint64_t height = KAT_LOAD_HEIGHT; + const uint64_t cols = load_cols::NUM_COLUMNS; + auto out = table(height * cols); + each_row(height, 0, [&] { fill_load(KAT_LOAD_OPS, KAT_LOAD_N, height, out.data()); }); + check_words("LOAD", out, KAT_LOAD_WORDS, height, cols); +} + +void test_store() { + const uint64_t height = KAT_STORE_HEIGHT; + const uint64_t cols = store_cols::NUM_COLUMNS; + auto out = table(height * cols); + each_row(height, 0, [&] { fill_store(KAT_STORE_OPS, KAT_STORE_N, height, out.data()); }); + check_words("STORE", out, KAT_STORE_WORDS, height, cols); +} + +void test_shift() { + const uint64_t height = KAT_SHIFT_HEIGHT; + const uint64_t cols = shift_cols::NUM_COLUMNS; + auto out = table(height * cols); + each_row(height, 0, [&] { fill_shift(KAT_SHIFT_OPS, KAT_SHIFT_N, height, out.data()); }); + check_words("SHIFT", out, KAT_SHIFT_WORDS, height, cols); +} + +void test_memw() { + const uint64_t height = KAT_MEMW_HEIGHT; + const uint64_t cols = memw_cols::NUM_COLUMNS; + auto out = table(height * cols); + each_row(height, 0, [&] { fill_memw(KAT_MEMW_OPS, KAT_MEMW_N, height, out.data()); }); + check_words("MEMW", out, KAT_MEMW_WORDS, height, cols); +} + +void test_cpu() { + const uint64_t height = KAT_CPU_HEIGHT; + const uint64_t cols = cpu_cols::NUM_COLUMNS; + auto out = table(height * cols); + unsigned long long missing = 0; + each_row(height, 0, [&] { + fill_cpu_from_logs(KAT_CPU_LOGS, KAT_CPU_N, KAT_CPU_ENTRIES, + KAT_CPU_RUNS, KAT_CPU_RUNS_N, height, out.data(), &missing); + }); + CHECK(missing == 0, "CPU: %llu logs missing their decode row", missing); + check_words("CPU", out, KAT_CPU_WORDS, height, cols); + + // A log whose pc no run holds: counted, its row zero, every other row as before. + std::vector logs(KAT_CPU_LOGS, KAT_CPU_LOGS + KAT_CPU_N); + const uint64_t absent_row = 5; + logs[absent_row].current_pc = KAT_CPU_ABSENT_PC; + std::vector want(KAT_CPU_WORDS, KAT_CPU_WORDS + height * cols); + for (uint64_t c = 0; c < cols; ++c) want[c * height + absent_row] = 0; + auto refused = table(height * cols); + missing = 0; + each_row(height, 0, [&] { + fill_cpu_from_logs(logs.data(), KAT_CPU_N, KAT_CPU_ENTRIES, + KAT_CPU_RUNS, KAT_CPU_RUNS_N, height, refused.data(), &missing); + }); + CHECK(missing == 1, "CPU: an absent pc counted %llu times, not once", missing); + check_words("CPU with an absent pc", refused, want.data(), height, cols); +} + +// Every KECCAK_RND case through `fill(case, out)`, one of the kernel's +// entries, against the digests. +template +void check_keccak_rnd(const char *name, Fill fill) { + const uint64_t cols = keccak_cols::NUM_COLUMNS; + CHECK(KAT_KECCAK_GROUP_END[KAT_KECCAK_GROUPS - 1] == cols, "the groups end at the last column"); + size_t digest = 0; + for (int k = 0; k < KAT_KECCAK_CASES_N; ++k) { + const KatKeccakCase cs = KAT_KECCAK_CASES[k]; + auto out = table(cs.rows * cols); + for (unsigned section = 0; section < FILL_KECCAK_SECTIONS; ++section) { + each_row(cs.rows, section, [&] { fill(cs, out.data()); }); + } + CHECK(guard_intact(out, cs.rows * cols), "%s case %d: a thread past the table wrote", name, + k); + int bad = 0; + for (uint64_t r = 0; r < cs.rows; ++r) { + uint32_t start = 0; + for (int g = 0; g < KAT_KECCAK_GROUPS; ++g) { + const uint32_t end = KAT_KECCAK_GROUP_END[g]; + const uint64_t got = fnv1a(out.data() + start * cs.rows + r, end - start, cs.rows); + if (got != KAT_KECCAK_DIGESTS[digest]) { + if (bad++ < 8) { + std::printf(" %s case %d row %llu columns [%u, %u) differ\n", name, k, + (unsigned long long)r, start, end); + } + } + ++digest; + start = end; + } + } + CHECK(bad == 0, "%s case %d: %d row groups differ", name, k, bad); + } + CHECK(digest == sizeof(KAT_KECCAK_DIGESTS) / sizeof(KAT_KECCAK_DIGESTS[0]), + "every digest read"); +} + +void test_keccak_rnd() { + check_keccak_rnd("KECCAK_RND", [](const KatKeccakCase &cs, uint64_t *out) { + fill_keccak_rnd(KAT_KECCAK_OPS + cs.first_op, cs.nops, cs.skip, cs.rows, out); + }); + // The records a table held as its keccak operations uploads: the same + // calls without their outputs, the same words. + std::vector ops; + for (const KeccakOpC &op : KAT_KECCAK_OPS) { + KeccakOpInC in{}; + in.timestamp = op.timestamp; + for (int i = 0; i < 25; ++i) in.input[i] = op.input[i]; + ops.push_back(in); + } + check_keccak_rnd("KECCAK_RND from records", [&](const KatKeccakCase &cs, uint64_t *out) { + fill_keccak_rnd_in(ops.data() + cs.first_op, cs.nops, cs.skip, cs.rows, out); + }); +} + +} // namespace + +int main() { + test_memw_r(); + test_memw_a(); + test_load(); + test_store(); + test_shift(); + test_memw(); + test_cpu(); + test_keccak_rnd(); + std::printf("fill host KAT: %d checks, %d failures\n", g_checks, g_failures); + return g_failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/crypto/math-cuda/tests/host_kat/fill_kat_vectors.h b/crypto/math-cuda/tests/host_kat/fill_kat_vectors.h new file mode 100644 index 000000000..e6084c106 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/fill_kat_vectors.h @@ -0,0 +1,1790 @@ +// Known-answer vectors for the table fills (`kernels/fill.cu`), from the prover's host +// fill. GENERATED by `tables::card_fill_inputs` (prover); do not edit. Regenerate with +// CARD_FILL_KAT_WRITE=1 cargo test -p lambda-vm-prover --lib \ +// tables::card_fill_inputs::tests::the_host_kat_vectors_are_the_host_fills +// Include after `fill.cu` (it declares the record structs). The CPU table's decode is +// seeded random instruction words (no guest ELF): these are host-generated vectors, +// not a guest's rows. +#pragma once +#include + +// MEMW_R: 13 rows, then zero rows to 16. +inline constexpr uint64_t KAT_MEMW_R_N = 13; +inline constexpr uint64_t KAT_MEMW_R_HEIGHT = 16; +inline const RegRowC KAT_MEMW_R_ROWS[KAT_MEMW_R_N] = { + {634543647ull, 3275349215u, 129944420u, 215479676u, 2406529433u, 634519699u, 3, 1, 0}, + {12619125296ull, 2868119902u, 2385902209u, 2212915922u, 2313682349u, 4029151156u, 93, 1, 0}, + {1209625953ull, 3950022460u, 3223135919u, 3087992781u, 39700391u, 1209619787u, 227, 1, 0}, + {8339747195ull, 578205858u, 3892543858u, 3604565851u, 566881254u, 4044731010u, 99, 1, 0}, + {4991810409ull, 3600250396u, 1792327763u, 2619281666u, 3007122281u, 696803863u, 194, 1, 0}, + {3977431703ull, 3271503573u, 3797432940u, 3303276053u, 1418444600u, 3977421190u, 204, 1, 0}, + {12892639480ull, 2399297619u, 2856663987u, 1941055731u, 3994242571u, 7695437u, 173, 1, 0}, + {9805258060ull, 1767392942u, 2598999003u, 2272726162u, 1445679030u, 1215311095u, 208, 1, 0}, + {6188540510ull, 1022701490u, 250911734u, 2707595402u, 2727821415u, 1893511672u, 139, 0, 0}, + {11187809324ull, 590819500u, 113990439u, 617484667u, 2156506046u, 2597833583u, 42, 0, 0}, + {15122480225ull, 2791337564u, 3240115697u, 1202437322u, 931346773u, 2237568483u, 31, 1, 0}, + {4289463752ull, 1811008214u, 2186692200u, 1812759666u, 728207178u, 4289452249u, 54, 1, 0}, + {6373477900ull, 2117900341u, 3552637126u, 339731000u, 1990560270u, 2078475868u, 191, 0, 0}, +}; +// Column-major: column c's rows from c * KAT_MEMW_R_HEIGHT. +inline const uint64_t KAT_MEMW_R_WORDS[] = { + 3ull, 93ull, 227ull, 99ull, 194ull, 204ull, + 173ull, 208ull, 139ull, 42ull, 31ull, 54ull, + 191ull, 0ull, 0ull, 0ull, 634543647ull, 4029190704ull, + 1209625953ull, 4044779899ull, 696843113ull, 3977431703ull, 7737592ull, 1215323468ull, + 1893573214ull, 2597874732ull, 2237578337ull, 4289463752ull, 2078510604ull, 0ull, + 0ull, 0ull, 0ull, 2ull, 0ull, 1ull, + 1ull, 0ull, 3ull, 2ull, 1ull, 2ull, + 3ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 3275349215ull, 2868119902ull, 3950022460ull, 578205858ull, 3600250396ull, 3271503573ull, + 2399297619ull, 1767392942ull, 1022701490ull, 590819500ull, 2791337564ull, 1811008214ull, + 2117900341ull, 0ull, 0ull, 0ull, 129944420ull, 2385902209ull, + 3223135919ull, 3892543858ull, 1792327763ull, 3797432940ull, 2856663987ull, 2598999003ull, + 250911734ull, 113990439ull, 3240115697ull, 2186692200ull, 3552637126ull, 0ull, + 0ull, 0ull, 215479676ull, 2212915922ull, 3087992781ull, 3604565851ull, + 2619281666ull, 3303276053ull, 1941055731ull, 2272726162ull, 2707595402ull, 617484667ull, + 1202437322ull, 1812759666ull, 339731000ull, 0ull, 0ull, 0ull, + 2406529433ull, 2313682349ull, 39700391ull, 566881254ull, 3007122281ull, 1418444600ull, + 3994242571ull, 1445679030ull, 2727821415ull, 2156506046ull, 931346773ull, 728207178ull, + 1990560270ull, 0ull, 0ull, 0ull, 634519699ull, 4029151156ull, + 1209619787ull, 4044731010ull, 696803863ull, 3977421190ull, 7695437ull, 1215311095ull, + 1893511672ull, 2597833583ull, 2237568483ull, 4289452249ull, 2078475868ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, +}; + +// MEMW_A: 13 rows, then zero rows to 16. +inline constexpr uint64_t KAT_MEMW_A_N = 13; +inline constexpr uint64_t KAT_MEMW_A_HEIGHT = 16; +inline const AlignedRowC KAT_MEMW_A_ROWS[KAT_MEMW_A_N] = { + {9631758199009817824ull, 1093151982096ull, 529127942528ull, 7645878369810480747ull, 1777635049527331647ull, 1, 1, 0}, + {17425809405337730072ull, 80339804308ull, 66635081317ull, 2385105407910523327ull, 3680567861905348533ull, 2, 1, 1}, + {6539617429112846888ull, 555577561756ull, 358747572326ull, 4637095072736623228ull, 4349360174217104096ull, 4, 1, 0}, + {9923978884303165368ull, 933217040615ull, 430152386855ull, 8335065718154963217ull, 10178598578214111582ull, 8, 1, 0}, + {7853589285439487296ull, 611139207300ull, 201776833736ull, 554173175233465038ull, 9838768561454728814ull, 1, 1, 1}, + {15080800646611361728ull, 1067280363572ull, 1023074798322ull, 8194342221160436911ull, 15407584087209807920ull, 2, 0, 0}, + {4289661623292839048ull, 634949879259ull, 494719009298ull, 8055350011983263693ull, 13054936850680642099ull, 4, 1, 0}, + {17474960791978755032ull, 628492072900ull, 349024566847ull, 15450051060413575496ull, 14558466771771382098ull, 8, 0, 1}, + {5856138096254178632ull, 609030522168ull, 530677148884ull, 3121178269785347015ull, 17146266709291884162ull, 1, 0, 0}, + {3562905347674050096ull, 917999004109ull, 103052934894ull, 11877099407915219071ull, 10543805777883472329ull, 2, 0, 0}, + {1704514759688843608ull, 728386309148ull, 38604692288ull, 1790391755749850258ull, 8020347863009492482ull, 4, 0, 1}, + {3201066851601757824ull, 743525300365ull, 671006065093ull, 18138251620724940571ull, 17208064548798569269ull, 8, 1, 0}, + {17992568124395956840ull, 323618198745ull, 312598527800ull, 7092789789913090722ull, 12661757980123898317ull, 1, 1, 0}, +}; +// Column-major: column c's rows from c * KAT_MEMW_A_HEIGHT. +inline const uint64_t KAT_MEMW_A_WORDS[] = { + 0ull, 1ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 46304ull, 64536ull, + 31272ull, 6072ull, 15680ull, 24512ull, 50312ull, 50136ull, + 54600ull, 36400ull, 61784ull, 13952ull, 10856ull, 0ull, + 0ull, 0ull, 65135ull, 2256ull, 17721ull, 15710ull, + 21103ull, 24000ull, 52990ull, 1477ull, 18140ull, 24691ull, + 40361ull, 28672ull, 25507ull, 0ull, 0ull, 0ull, + 2242568460ull, 4057262420ull, 1522623335ull, 2310606391ull, 1828556248ull, 3511272521ull, + 998764676ull, 4068706369ull, 1363488402ull, 829553545ull, 396863268ull, 745306455ull, + 4189221217ull, 0ull, 0ull, 0ull, 107ull, 1314828735ull, + 124ull, 17ull, 366430926ull, 175ull, 205ull, 3784913224ull, + 199ull, 127ull, 3795975314ull, 27ull, 162ull, 0ull, + 0ull, 0ull, 126ull, 555325627ull, 58ull, 169ull, + 129028497ull, 212ull, 119ull, 3597245332ull, 83ull, 248ull, + 416858064ull, 159ull, 138ull, 0ull, 0ull, 0ull, + 101ull, 0ull, 46ull, 41ull, 0ull, 64ull, + 179ull, 0ull, 57ull, 201ull, 0ull, 246ull, + 73ull, 0ull, 0ull, 0ull, 36ull, 0ull, + 120ull, 14ull, 0ull, 172ull, 37ull, 0ull, + 131ull, 226ull, 0ull, 155ull, 22ull, 0ull, + 0ull, 0ull, 222ull, 0ull, 102ull, 28ull, + 0ull, 213ull, 40ull, 0ull, 25ull, 251ull, + 0ull, 191ull, 188ull, 0ull, 0ull, 0ull, + 157ull, 0ull, 69ull, 26ull, 0ull, 38ull, + 90ull, 0ull, 167ull, 238ull, 0ull, 3ull, + 166ull, 0ull, 0ull, 0ull, 27ull, 0ull, + 90ull, 172ull, 0ull, 184ull, 202ull, 0ull, + 80ull, 211ull, 0ull, 184ull, 110ull, 0ull, + 0ull, 0ull, 106ull, 0ull, 64ull, 115ull, + 0ull, 113ull, 111ull, 0ull, 43ull, 164ull, + 0ull, 251ull, 98ull, 0ull, 0ull, 0ull, + 2230288912ull, 3030392980ull, 1526780572ull, 1209137383ull, 1253851268ull, 2128474164ull, + 3589686747ull, 1426847684ull, 3440133432ull, 3170970061ull, 2536836124ull, 495958157ull, + 1495651545ull, 0ull, 0ull, 0ull, 254ull, 18ull, + 129ull, 217ull, 142ull, 248ull, 147ull, 146ull, + 141ull, 213ull, 169ull, 173ull, 75ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 63ull, 1284921269ull, 224ull, 94ull, + 3972679278ull, 48ull, 51ull, 365091154ull, 130ull, 201ull, + 1170112002ull, 53ull, 205ull, 0ull, 0ull, 0ull, + 75ull, 856948984ull, 10ull, 137ull, 2290766816ull, 16ull, + 250ull, 3389657189ull, 222ull, 161ull, 1867382755ull, 155ull, + 161ull, 0ull, 0ull, 0ull, 92ull, 0ull, + 113ull, 191ull, 0ull, 26ull, 249ull, 0ull, + 114ull, 116ull, 0ull, 33ull, 64ull, 0ull, + 0ull, 0ull, 195ull, 0ull, 216ull, 117ull, + 0ull, 45ull, 205ull, 0ull, 47ull, 211ull, + 0ull, 239ull, 77ull, 0ull, 0ull, 0ull, + 168ull, 0ull, 8ull, 122ull, 0ull, 24ull, + 229ull, 0ull, 186ull, 125ull, 0ull, 136ull, + 204ull, 0ull, 0ull, 0ull, 109ull, 0ull, + 8ull, 165ull, 0ull, 188ull, 115ull, 0ull, + 198ull, 31ull, 0ull, 83ull, 153ull, 0ull, + 0ull, 0ull, 171ull, 0ull, 92ull, 65ull, + 0ull, 210ull, 44ull, 0ull, 243ull, 83ull, + 0ull, 207ull, 183ull, 0ull, 0ull, 0ull, + 24ull, 0ull, 60ull, 141ull, 0ull, 213ull, + 181ull, 0ull, 237ull, 146ull, 0ull, 238ull, + 175ull, 0ull, 0ull, 0ull, 846965120ull, 2210571877ull, + 2265286758ull, 655657255ull, 4208338120ull, 872581874ull, 797770258ull, 1132215871ull, + 2396171476ull, 4268687086ull, 4244953920ull, 991166917ull, 3360882488ull, 0ull, + 0ull, 0ull, 123ull, 15ull, 83ull, 100ull, + 46ull, 238ull, 115ull, 81ull, 123ull, 23ull, + 8ull, 156ull, 72ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 1ull, + 1ull, 1ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 0ull, +}; + +// LOAD: 19 ops, then zero rows to 32. +inline constexpr uint64_t KAT_LOAD_N = 19; +inline constexpr uint64_t KAT_LOAD_HEIGHT = 32; +inline const LoadOpC KAT_LOAD_OPS[KAT_LOAD_N] = { + {1568532246794916528ull, 1072818390687ull, 1, 0, {86, 65, 86, 89, 132, 109, 155, 161}}, + {3452037299261216263ull, 904600845957ull, 2, 0, {237, 254, 234, 254, 224, 242, 151, 100}}, + {605977712323159034ull, 1019548766389ull, 4, 0, {172, 247, 85, 122, 148, 25, 235, 116}}, + {16076329484220639362ull, 41948024667ull, 8, 0, {69, 53, 170, 244, 137, 19, 116, 192}}, + {7193777689344888541ull, 459120508716ull, 1, 1, {40, 45, 2, 39, 145, 9, 131, 180}}, + {14734695211004367604ull, 543607160502ull, 2, 1, {239, 221, 143, 96, 92, 152, 174, 89}}, + {17179637227398587538ull, 686760908534ull, 4, 1, {3, 33, 175, 204, 231, 238, 235, 151}}, + {1846668067482655586ull, 734116685131ull, 8, 1, {158, 108, 193, 156, 201, 251, 127, 18}}, + {8380049116650266713ull, 957620625488ull, 1, 0, {157, 235, 16, 32, 59, 138, 244, 117}}, + {1121891651050010518ull, 5675762741ull, 2, 0, {199, 205, 113, 86, 168, 151, 100, 208}}, + {7990977878521476009ull, 545431982019ull, 4, 0, {24, 13, 2, 216, 78, 182, 62, 96}}, + {14023592196834508768ull, 853364558121ull, 8, 0, {151, 212, 222, 31, 130, 241, 178, 247}}, + {2453685364608796325ull, 1020607526951ull, 1, 1, {192, 230, 96, 243, 13, 80, 212, 237}}, + {11758734794247959978ull, 402097422611ull, 2, 1, {88, 79, 183, 63, 145, 129, 107, 167}}, + {11076364353336972332ull, 458940754924ull, 4, 1, {206, 21, 29, 18, 3, 57, 25, 222}}, + {12708616932199599668ull, 452358648828ull, 8, 1, {32, 52, 216, 169, 152, 233, 96, 175}}, + {3633929476716935911ull, 595539573110ull, 1, 0, {77, 57, 11, 225, 33, 20, 144, 86}}, + {14595222962553927814ull, 995371482878ull, 2, 0, {228, 43, 33, 156, 140, 213, 222, 210}}, + {18071084022723084802ull, 700443711479ull, 4, 0, {8, 18, 117, 139, 162, 247, 35, 235}}, +}; +// Column-major: column c's rows from c * KAT_LOAD_HEIGHT. +inline const uint64_t KAT_LOAD_WORDS[] = { + 2633290416ull, 1306334727ull, 3436405754ull, 1729898626ull, 3062386397ull, 93360884ull, + 1697342610ull, 3294387042ull, 2923683929ull, 961542038ull, 3779523497ull, 2269298656ull, + 3289712293ull, 3006068138ull, 3144881196ull, 1178963508ull, 243841767ull, 2762389638ull, + 2292548098ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 365202372ull, 803740066ull, 141090180ull, 3743062141ull, + 1674931889ull, 3430688570ull, 3999945993ull, 429960914ull, 1951132229ull, 261210755ull, + 1860544522ull, 3265121997ull, 571293142ull, 2737793790ull, 2578917041ull, 2958955460ull, + 846090139ull, 3398215156ull, 4207502124ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 3371533983ull, 2657713797ull, + 1641517237ull, 3293319003ull, 3853975340ull, 2441281206ull, 3861108470ull, 3972244811ull, + 4137885776ull, 1380795445ull, 4266102723ull, 2961033513ull, 2700277799ull, 2665464083ull, + 3674221548ull, 1387082748ull, 2834086262ull, 3234037502ull, 364042231ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 249ull, 210ull, 237ull, 9ull, 106ull, 126ull, + 159ull, 170ull, 222ull, 1ull, 126ull, 198ull, + 237ull, 93ull, 106ull, 105ull, 138ull, 231ull, + 163ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 86ull, 237ull, + 172ull, 69ull, 40ull, 239ull, 3ull, 158ull, + 157ull, 199ull, 24ull, 151ull, 192ull, 88ull, + 206ull, 32ull, 77ull, 228ull, 8ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 65ull, 254ull, 247ull, 53ull, 45ull, 221ull, + 33ull, 108ull, 235ull, 205ull, 13ull, 212ull, + 230ull, 79ull, 21ull, 52ull, 57ull, 43ull, + 18ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 86ull, 234ull, 85ull, 170ull, + 2ull, 143ull, 175ull, 193ull, 16ull, 113ull, + 2ull, 222ull, 96ull, 183ull, 29ull, 216ull, + 11ull, 33ull, 117ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 89ull, 254ull, + 122ull, 244ull, 39ull, 96ull, 204ull, 156ull, + 32ull, 86ull, 216ull, 31ull, 243ull, 63ull, + 18ull, 169ull, 225ull, 156ull, 139ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 132ull, 224ull, 148ull, 137ull, 145ull, 92ull, + 231ull, 201ull, 59ull, 168ull, 78ull, 130ull, + 13ull, 145ull, 3ull, 152ull, 33ull, 140ull, + 162ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 109ull, 242ull, 25ull, 19ull, + 9ull, 152ull, 238ull, 251ull, 138ull, 151ull, + 182ull, 241ull, 80ull, 129ull, 57ull, 233ull, + 20ull, 213ull, 247ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 155ull, 151ull, + 235ull, 116ull, 131ull, 174ull, 235ull, 127ull, + 244ull, 100ull, 62ull, 178ull, 212ull, 107ull, + 25ull, 96ull, 144ull, 222ull, 35ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 161ull, 100ull, 116ull, 192ull, 180ull, 89ull, + 151ull, 18ull, 117ull, 208ull, 96ull, 247ull, + 237ull, 167ull, 222ull, 175ull, 86ull, 210ull, + 235ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 1ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, +}; + +// STORE: 11 ops, then zero rows to 16. +inline constexpr uint64_t KAT_STORE_N = 11; +inline constexpr uint64_t KAT_STORE_HEIGHT = 16; +inline const StoreOpC KAT_STORE_OPS[KAT_STORE_N] = { + {3882708967309460720ull, 997896696899ull, 9596568836759627169ull, 0, 0, 0}, + {7633517603624463862ull, 409139576017ull, 6462542903376522072ull, 1, 0, 0}, + {727390276247621193ull, 755076943378ull, 2483203373287819223ull, 0, 1, 0}, + {299960893569381612ull, 1035485332371ull, 13185304593035917809ull, 0, 0, 1}, + {5700762956143270975ull, 924322354550ull, 14221597735584558835ull, 0, 0, 0}, + {2058814834748344091ull, 620476738741ull, 13880369393543343830ull, 1, 0, 0}, + {2098276135678504849ull, 1087793995775ull, 3987873980192719094ull, 0, 1, 0}, + {197015077595671378ull, 538334602650ull, 15211489648062662013ull, 0, 0, 1}, + {2316767623969456756ull, 645156576607ull, 9632383314857573026ull, 0, 0, 0}, + {11848447504349320453ull, 288916926627ull, 3420602715032296732ull, 1, 0, 0}, + {6103412655939982226ull, 151335820147ull, 8827411741291200602ull, 0, 1, 0}, +}; +// Column-major: column c's rows from c * KAT_STORE_HEIGHT. +inline const uint64_t KAT_STORE_WORDS[] = { + 4206118128ull, 4273614326ull, 886050377ull, 3954586860ull, 1704991807ull, 595248923ull, + 1003220881ull, 1400662866ull, 366166644ull, 3164263685ull, 720180114ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 904013627ull, 1777316816ull, + 169358746ull, 69840087ull, 1327312308ull, 479355183ull, 488542983ull, 45871147ull, + 539414497ull, 2758681658ull, 1421061497ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1464284227ull, 1117682897ull, 3457666578ull, 398214035ull, + 904385910ull, 2001448117ull, 1167269887ull, 1463690650ull, 911482207ull, 1154117795ull, + 1011964787ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 232ull, 95ull, 175ull, 241ull, 215ull, 144ull, + 253ull, 125ull, 150ull, 67ull, 35ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 161ull, 88ull, + 215ull, 241ull, 243ull, 214ull, 246ull, 125ull, + 162ull, 28ull, 90ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 37ull, 171ull, 71ull, 121ull, + 110ull, 238ull, 0ull, 133ull, 58ull, 157ull, + 64ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 164ull, 122ull, 223ull, 59ull, 4ull, 90ull, + 105ull, 78ull, 124ull, 60ull, 198ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 33ull, 213ull, + 185ull, 43ull, 144ull, 13ull, 158ull, 177ull, + 30ull, 211ull, 133ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 131ull, 135ull, 79ull, 171ull, + 202ull, 108ull, 65ull, 61ull, 151ull, 11ull, + 53ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 220ull, 144ull, 28ull, 156ull, 67ull, 250ull, + 198ull, 17ull, 25ull, 108ull, 68ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 45ull, 175ull, + 118ull, 251ull, 93ull, 160ull, 87ull, 26ull, + 173ull, 120ull, 129ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 133ull, 89ull, 34ull, 182ull, + 197ull, 192ull, 55ull, 211ull, 133ull, 47ull, + 122ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, +}; + +// SHIFT: 123 ops (every direction, signedness and width at each edge amount), +// then padding rows (ZBS 1) to 128. +inline constexpr uint64_t KAT_SHIFT_N = 123; +inline constexpr uint64_t KAT_SHIFT_HEIGHT = 128; +inline const ShiftOpC KAT_SHIFT_OPS[KAT_SHIFT_N] = { + {0ull, {7275, 55603, 47851, 2175}, 0, 0, 0, 0}, + {0ull, {51654, 51932, 56659, 57485}, 0, 1, 0, 0}, + {0ull, {36168, 8916, 12977, 9448}, 0, 0, 1, 0}, + {0ull, {12382, 47640, 37139, 35580}, 0, 1, 1, 0}, + {0ull, {19813, 9380, 21071, 22065}, 0, 0, 0, 1}, + {0ull, {31839, 5976, 1230, 29830}, 0, 1, 0, 1}, + {0ull, {11240, 12981, 11209, 27594}, 0, 0, 1, 1}, + {0ull, {33047, 57175, 50978, 28983}, 0, 1, 1, 1}, + {1ull, {24203, 50738, 63992, 57676}, 1, 0, 0, 0}, + {1ull, {56288, 32160, 12601, 38919}, 1, 1, 0, 0}, + {1ull, {10602, 59410, 23655, 20435}, 1, 0, 1, 0}, + {1ull, {50787, 38269, 20903, 24390}, 1, 1, 1, 0}, + {1ull, {24139, 9649, 4837, 23142}, 1, 0, 0, 1}, + {1ull, {28022, 25570, 54059, 17658}, 1, 1, 0, 1}, + {1ull, {15707, 34469, 2673, 31964}, 1, 0, 1, 1}, + {1ull, {49239, 58492, 62038, 39113}, 1, 1, 1, 1}, + {4ull, {28587, 62391, 11782, 46392}, 4, 0, 0, 0}, + {4ull, {30604, 48672, 2247, 17}, 4, 1, 0, 0}, + {4ull, {25087, 34581, 47741, 27908}, 4, 0, 1, 0}, + {4ull, {53947, 56963, 52778, 19541}, 4, 1, 1, 0}, + {4ull, {3633, 41625, 48039, 65371}, 4, 0, 0, 1}, + {4ull, {22491, 36178, 32077, 47923}, 4, 1, 0, 1}, + {4ull, {18244, 40765, 60074, 34688}, 4, 0, 1, 1}, + {4ull, {17782, 59962, 52028, 42040}, 4, 1, 1, 1}, + {15ull, {12989, 6808, 6996, 42322}, 15, 0, 0, 0}, + {15ull, {44393, 3492, 31686, 7069}, 15, 1, 0, 0}, + {15ull, {38933, 40205, 8635, 7751}, 15, 0, 1, 0}, + {15ull, {55169, 43170, 48965, 36395}, 15, 1, 1, 0}, + {15ull, {28675, 11654, 17385, 16590}, 15, 0, 0, 1}, + {15ull, {8786, 54213, 52673, 33885}, 15, 1, 0, 1}, + {15ull, {7131, 41717, 24545, 6554}, 15, 0, 1, 1}, + {15ull, {48967, 12220, 39209, 1506}, 15, 1, 1, 1}, + {16ull, {52313, 18720, 1220, 31145}, 16, 0, 0, 0}, + {16ull, {58347, 29330, 14554, 14223}, 16, 1, 0, 0}, + {16ull, {48366, 39415, 9248, 20072}, 16, 0, 1, 0}, + {16ull, {41598, 8, 16876, 51527}, 16, 1, 1, 0}, + {16ull, {26439, 63019, 59534, 2331}, 16, 0, 0, 1}, + {16ull, {38770, 23568, 34312, 20357}, 16, 1, 0, 1}, + {16ull, {61456, 42350, 21746, 33506}, 16, 0, 1, 1}, + {16ull, {46525, 7755, 41143, 14853}, 16, 1, 1, 1}, + {17ull, {53941, 2373, 51763, 535}, 17, 0, 0, 0}, + {17ull, {14305, 23679, 59354, 13555}, 17, 1, 0, 0}, + {17ull, {45269, 16837, 55333, 11404}, 17, 0, 1, 0}, + {17ull, {47592, 50544, 44181, 57855}, 17, 1, 1, 0}, + {17ull, {60233, 45411, 28687, 56897}, 17, 0, 0, 1}, + {17ull, {930, 55480, 58127, 48768}, 17, 1, 0, 1}, + {17ull, {30786, 18278, 19763, 6206}, 17, 0, 1, 1}, + {17ull, {62988, 7509, 27694, 2449}, 17, 1, 1, 1}, + {31ull, {13071, 30344, 19068, 4666}, 31, 0, 0, 0}, + {31ull, {577, 61748, 19362, 29452}, 31, 1, 0, 0}, + {31ull, {59841, 37916, 56750, 42372}, 31, 0, 1, 0}, + {31ull, {54841, 39107, 58833, 35596}, 31, 1, 1, 0}, + {31ull, {59037, 28075, 28865, 12982}, 31, 0, 0, 1}, + {31ull, {65116, 23631, 21457, 32927}, 31, 1, 0, 1}, + {31ull, {29509, 64502, 16460, 35108}, 31, 0, 1, 1}, + {31ull, {60746, 15536, 56325, 60163}, 31, 1, 1, 1}, + {32ull, {29682, 55200, 39663, 33608}, 32, 0, 0, 0}, + {32ull, {18642, 3247, 53970, 12520}, 32, 1, 0, 0}, + {32ull, {53299, 8534, 59031, 10129}, 32, 0, 1, 0}, + {32ull, {15607, 36139, 16933, 11562}, 32, 1, 1, 0}, + {32ull, {26502, 12734, 4255, 62065}, 32, 0, 0, 1}, + {32ull, {1157, 16104, 44947, 55906}, 32, 1, 0, 1}, + {32ull, {65025, 55352, 5838, 24612}, 32, 0, 1, 1}, + {32ull, {64509, 39362, 24164, 45601}, 32, 1, 1, 1}, + {33ull, {62844, 43152, 39232, 24103}, 33, 0, 0, 0}, + {33ull, {38274, 35392, 18101, 8803}, 33, 1, 0, 0}, + {33ull, {1125, 2794, 37856, 15663}, 33, 0, 1, 0}, + {33ull, {15026, 22585, 64798, 37597}, 33, 1, 1, 0}, + {33ull, {55002, 43858, 41548, 29385}, 33, 0, 0, 1}, + {33ull, {42607, 39330, 43355, 4872}, 33, 1, 0, 1}, + {33ull, {20018, 6036, 64563, 7}, 33, 0, 1, 1}, + {33ull, {21590, 12727, 62461, 5072}, 33, 1, 1, 1}, + {47ull, {52987, 6066, 22054, 53623}, 47, 0, 0, 0}, + {47ull, {62642, 31830, 18102, 55759}, 47, 1, 0, 0}, + {47ull, {28687, 56432, 47653, 61391}, 47, 0, 1, 0}, + {47ull, {3295, 48116, 59740, 4700}, 47, 1, 1, 0}, + {47ull, {5309, 29902, 14200, 27937}, 47, 0, 0, 1}, + {47ull, {3745, 29222, 62967, 40614}, 47, 1, 0, 1}, + {47ull, {39770, 49896, 32077, 36356}, 47, 0, 1, 1}, + {47ull, {8144, 558, 5085, 9698}, 47, 1, 1, 1}, + {48ull, {37806, 20727, 47586, 32703}, 48, 0, 0, 0}, + {48ull, {60901, 53773, 27558, 52351}, 48, 1, 0, 0}, + {48ull, {65010, 3754, 49863, 14760}, 48, 0, 1, 0}, + {48ull, {63257, 37069, 11231, 39357}, 48, 1, 1, 0}, + {48ull, {27705, 10978, 29690, 13345}, 48, 0, 0, 1}, + {48ull, {49252, 12506, 31671, 55474}, 48, 1, 0, 1}, + {48ull, {6092, 25817, 12976, 24516}, 48, 0, 1, 1}, + {48ull, {49083, 37665, 56202, 17968}, 48, 1, 1, 1}, + {63ull, {23059, 23986, 1990, 14862}, 63, 0, 0, 0}, + {63ull, {41151, 30699, 15244, 63785}, 63, 1, 0, 0}, + {63ull, {34750, 19993, 2072, 8266}, 63, 0, 1, 0}, + {63ull, {42905, 19412, 29376, 63071}, 63, 1, 1, 0}, + {63ull, {13795, 57590, 21218, 57472}, 63, 0, 0, 1}, + {63ull, {46136, 65529, 35151, 64556}, 63, 1, 0, 1}, + {63ull, {63413, 16451, 52572, 45008}, 63, 0, 1, 1}, + {63ull, {13723, 21680, 35429, 9574}, 63, 1, 1, 1}, + {71ull, {47313, 30668, 1839, 41941}, 71, 0, 0, 0}, + {71ull, {34626, 2841, 33520, 23001}, 71, 1, 0, 0}, + {71ull, {11657, 55509, 41598, 62332}, 71, 0, 1, 0}, + {71ull, {41180, 23780, 13295, 38774}, 71, 1, 1, 0}, + {71ull, {49091, 57445, 20126, 16608}, 71, 0, 0, 1}, + {71ull, {15185, 23468, 48479, 25186}, 71, 1, 0, 1}, + {71ull, {11135, 46680, 36430, 62458}, 71, 0, 1, 1}, + {71ull, {16247, 56069, 63080, 14242}, 71, 1, 1, 1}, + {255ull, {43387, 1458, 4703, 36147}, 255, 0, 0, 0}, + {255ull, {16891, 45931, 53013, 28771}, 255, 1, 0, 0}, + {255ull, {39352, 55239, 16517, 61776}, 255, 0, 1, 0}, + {255ull, {24744, 18729, 17418, 60571}, 255, 1, 1, 0}, + {255ull, {1188, 62205, 4318, 62727}, 255, 0, 0, 1}, + {255ull, {47729, 4404, 34390, 53311}, 255, 1, 0, 1}, + {255ull, {20585, 6561, 37976, 18978}, 255, 0, 1, 1}, + {255ull, {18605, 60180, 9267, 46078}, 255, 1, 1, 1}, + {4294967555ull, {11318, 32215, 49721, 6884}, 3, 0, 0, 0}, + {4294967555ull, {18853, 49087, 55433, 55674}, 3, 1, 0, 0}, + {4294967555ull, {55661, 57731, 16433, 16182}, 3, 0, 1, 0}, + {4294967555ull, {29325, 49301, 26084, 8839}, 3, 1, 1, 0}, + {4294967555ull, {56316, 33263, 43310, 20166}, 3, 0, 0, 1}, + {4294967555ull, {45950, 4725, 37154, 22913}, 3, 1, 0, 1}, + {4294967555ull, {49945, 16930, 60809, 58671}, 3, 0, 1, 1}, + {4294967555ull, {54095, 52189, 3842, 18877}, 3, 1, 1, 1}, + {18446744073709551614ull, {2493, 15028, 23771, 33011}, 254, 0, 0, 0}, + {18446744073709551614ull, {36007, 57209, 17412, 57672}, 254, 1, 0, 0}, + {18446744073709551614ull, {11552, 36436, 31555, 60762}, 254, 0, 1, 0}, +}; +// Column-major: column c's rows from c * KAT_SHIFT_HEIGHT. +inline const uint64_t KAT_SHIFT_WORDS[] = { + 7275ull, 51654ull, 36168ull, 12382ull, 19813ull, 31839ull, + 11240ull, 33047ull, 24203ull, 56288ull, 10602ull, 50787ull, + 24139ull, 28022ull, 15707ull, 49239ull, 28587ull, 30604ull, + 25087ull, 53947ull, 3633ull, 22491ull, 18244ull, 17782ull, + 12989ull, 44393ull, 38933ull, 55169ull, 28675ull, 8786ull, + 7131ull, 48967ull, 52313ull, 58347ull, 48366ull, 41598ull, + 26439ull, 38770ull, 61456ull, 46525ull, 53941ull, 14305ull, + 45269ull, 47592ull, 60233ull, 930ull, 30786ull, 62988ull, + 13071ull, 577ull, 59841ull, 54841ull, 59037ull, 65116ull, + 29509ull, 60746ull, 29682ull, 18642ull, 53299ull, 15607ull, + 26502ull, 1157ull, 65025ull, 64509ull, 62844ull, 38274ull, + 1125ull, 15026ull, 55002ull, 42607ull, 20018ull, 21590ull, + 52987ull, 62642ull, 28687ull, 3295ull, 5309ull, 3745ull, + 39770ull, 8144ull, 37806ull, 60901ull, 65010ull, 63257ull, + 27705ull, 49252ull, 6092ull, 49083ull, 23059ull, 41151ull, + 34750ull, 42905ull, 13795ull, 46136ull, 63413ull, 13723ull, + 47313ull, 34626ull, 11657ull, 41180ull, 49091ull, 15185ull, + 11135ull, 16247ull, 43387ull, 16891ull, 39352ull, 24744ull, + 1188ull, 47729ull, 20585ull, 18605ull, 11318ull, 18853ull, + 55661ull, 29325ull, 56316ull, 45950ull, 49945ull, 54095ull, + 2493ull, 36007ull, 11552ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 55603ull, 51932ull, 8916ull, 47640ull, + 9380ull, 5976ull, 12981ull, 57175ull, 50738ull, 32160ull, + 59410ull, 38269ull, 9649ull, 25570ull, 34469ull, 58492ull, + 62391ull, 48672ull, 34581ull, 56963ull, 41625ull, 36178ull, + 40765ull, 59962ull, 6808ull, 3492ull, 40205ull, 43170ull, + 11654ull, 54213ull, 41717ull, 12220ull, 18720ull, 29330ull, + 39415ull, 8ull, 63019ull, 23568ull, 42350ull, 7755ull, + 2373ull, 23679ull, 16837ull, 50544ull, 45411ull, 55480ull, + 18278ull, 7509ull, 30344ull, 61748ull, 37916ull, 39107ull, + 28075ull, 23631ull, 64502ull, 15536ull, 55200ull, 3247ull, + 8534ull, 36139ull, 12734ull, 16104ull, 55352ull, 39362ull, + 43152ull, 35392ull, 2794ull, 22585ull, 43858ull, 39330ull, + 6036ull, 12727ull, 6066ull, 31830ull, 56432ull, 48116ull, + 29902ull, 29222ull, 49896ull, 558ull, 20727ull, 53773ull, + 3754ull, 37069ull, 10978ull, 12506ull, 25817ull, 37665ull, + 23986ull, 30699ull, 19993ull, 19412ull, 57590ull, 65529ull, + 16451ull, 21680ull, 30668ull, 2841ull, 55509ull, 23780ull, + 57445ull, 23468ull, 46680ull, 56069ull, 1458ull, 45931ull, + 55239ull, 18729ull, 62205ull, 4404ull, 6561ull, 60180ull, + 32215ull, 49087ull, 57731ull, 49301ull, 33263ull, 4725ull, + 16930ull, 52189ull, 15028ull, 57209ull, 36436ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 47851ull, 56659ull, + 12977ull, 37139ull, 21071ull, 1230ull, 11209ull, 50978ull, + 63992ull, 12601ull, 23655ull, 20903ull, 4837ull, 54059ull, + 2673ull, 62038ull, 11782ull, 2247ull, 47741ull, 52778ull, + 48039ull, 32077ull, 60074ull, 52028ull, 6996ull, 31686ull, + 8635ull, 48965ull, 17385ull, 52673ull, 24545ull, 39209ull, + 1220ull, 14554ull, 9248ull, 16876ull, 59534ull, 34312ull, + 21746ull, 41143ull, 51763ull, 59354ull, 55333ull, 44181ull, + 28687ull, 58127ull, 19763ull, 27694ull, 19068ull, 19362ull, + 56750ull, 58833ull, 28865ull, 21457ull, 16460ull, 56325ull, + 39663ull, 53970ull, 59031ull, 16933ull, 4255ull, 44947ull, + 5838ull, 24164ull, 39232ull, 18101ull, 37856ull, 64798ull, + 41548ull, 43355ull, 64563ull, 62461ull, 22054ull, 18102ull, + 47653ull, 59740ull, 14200ull, 62967ull, 32077ull, 5085ull, + 47586ull, 27558ull, 49863ull, 11231ull, 29690ull, 31671ull, + 12976ull, 56202ull, 1990ull, 15244ull, 2072ull, 29376ull, + 21218ull, 35151ull, 52572ull, 35429ull, 1839ull, 33520ull, + 41598ull, 13295ull, 20126ull, 48479ull, 36430ull, 63080ull, + 4703ull, 53013ull, 16517ull, 17418ull, 4318ull, 34390ull, + 37976ull, 9267ull, 49721ull, 55433ull, 16433ull, 26084ull, + 43310ull, 37154ull, 60809ull, 3842ull, 23771ull, 17412ull, + 31555ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 2175ull, 57485ull, 9448ull, 35580ull, 22065ull, 29830ull, + 27594ull, 28983ull, 57676ull, 38919ull, 20435ull, 24390ull, + 23142ull, 17658ull, 31964ull, 39113ull, 46392ull, 17ull, + 27908ull, 19541ull, 65371ull, 47923ull, 34688ull, 42040ull, + 42322ull, 7069ull, 7751ull, 36395ull, 16590ull, 33885ull, + 6554ull, 1506ull, 31145ull, 14223ull, 20072ull, 51527ull, + 2331ull, 20357ull, 33506ull, 14853ull, 535ull, 13555ull, + 11404ull, 57855ull, 56897ull, 48768ull, 6206ull, 2449ull, + 4666ull, 29452ull, 42372ull, 35596ull, 12982ull, 32927ull, + 35108ull, 60163ull, 33608ull, 12520ull, 10129ull, 11562ull, + 62065ull, 55906ull, 24612ull, 45601ull, 24103ull, 8803ull, + 15663ull, 37597ull, 29385ull, 4872ull, 7ull, 5072ull, + 53623ull, 55759ull, 61391ull, 4700ull, 27937ull, 40614ull, + 36356ull, 9698ull, 32703ull, 52351ull, 14760ull, 39357ull, + 13345ull, 55474ull, 24516ull, 17968ull, 14862ull, 63785ull, + 8266ull, 63071ull, 57472ull, 64556ull, 45008ull, 9574ull, + 41941ull, 23001ull, 62332ull, 38774ull, 16608ull, 25186ull, + 62458ull, 14242ull, 36147ull, 28771ull, 61776ull, 60571ull, + 62727ull, 53311ull, 18978ull, 46078ull, 6884ull, 55674ull, + 16182ull, 8839ull, 20166ull, 22913ull, 58671ull, 18877ull, + 33011ull, 57672ull, 60762ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 4ull, 4ull, 4ull, 4ull, 4ull, 4ull, + 4ull, 4ull, 15ull, 15ull, 15ull, 15ull, + 15ull, 15ull, 15ull, 15ull, 16ull, 16ull, + 16ull, 16ull, 16ull, 16ull, 16ull, 16ull, + 17ull, 17ull, 17ull, 17ull, 17ull, 17ull, + 17ull, 17ull, 31ull, 31ull, 31ull, 31ull, + 31ull, 31ull, 31ull, 31ull, 32ull, 32ull, + 32ull, 32ull, 32ull, 32ull, 32ull, 32ull, + 33ull, 33ull, 33ull, 33ull, 33ull, 33ull, + 33ull, 33ull, 47ull, 47ull, 47ull, 47ull, + 47ull, 47ull, 47ull, 47ull, 48ull, 48ull, + 48ull, 48ull, 48ull, 48ull, 48ull, 48ull, + 63ull, 63ull, 63ull, 63ull, 63ull, 63ull, + 63ull, 63ull, 71ull, 71ull, 71ull, 71ull, + 71ull, 71ull, 71ull, 71ull, 255ull, 255ull, + 255ull, 255ull, 255ull, 255ull, 255ull, 255ull, + 3ull, 3ull, 3ull, 3ull, 3ull, 3ull, + 3ull, 3ull, 254ull, 254ull, 254ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 3644005483ull, 3403467206ull, + 584355144ull, 3122147422ull, 614747493ull, 391674975ull, 850734056ull, 3747053847ull, + 2355412246ull, 3201330672ull, 3492041428ull, 3401507633ull, 1264762006ull, 2985375419ull, + 222984886ull, 1916690475ull, 997653168ull, 2078410616ull, 1901469680ull, 2917678379ull, + 697361168ull, 3637847421ull, 4090786880ull, 3466830935ull, 425623552ull, 4153154377ull, + 3423240192ull, 2123059525ull, 939622400ull, 2609096586ull, 2381152256ull, 844259193ull, + 3428384768ull, 953840274ull, 3169714176ull, 1105985544ull, 1732706304ull, 2248694800ull, + 4027580416ull, 2696355403ull, 2775187456ull, 4092407359ull, 1638531072ull, 3595231928ull, + 3599892480ull, 1904733276ull, 4035182592ull, 3054964394ull, 2147483648ull, 3860371269ull, + 2147483648ull, 370789283ull, 2147483648ull, 20883362ull, 2147483648ull, 3590830090ull, + 0ull, 820564690ull, 0ull, 757744165ull, 834561926ull, 1055392901ull, + 3627613697ull, 2579692541ull, 0ull, 288465754ull, 0ull, 3379494543ull, + 1453698484ull, 3436270391ull, 791190628ull, 2564532779ull, 0ull, 111518ull, + 0ull, 9401ull, 173965312ull, 3958301772ull, 1303183360ull, 666502236ull, + 0ull, 52351ull, 0ull, 4294941117ull, 1815674880ull, 2075603162ull, + 399245312ull, 3683291937ull, 0ull, 1ull, 0ull, 4294967295ull, + 2147483648ull, 4166587039ull, 2147483648ull, 1254954186ull, 3864815744ull, 3759551246ull, + 1788265600ull, 3736717633ull, 853533056ull, 3199686774ull, 739622784ull, 3518368382ull, + 0ull, 0ull, 0ull, 4294967295ull, 0ull, 2692680876ull, + 2147483648ull, 1744586855ull, 4005126576ull, 938993972ull, 203344744ull, 2551361105ull, + 259973088ull, 1112454767ull, 286660808ull, 1501280873ull, 0ull, 3ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 142588651ull, 3767393619ull, 619197105ull, 2331808019ull, 1446072911ull, 1954940110ull, + 1808411593ull, 1899480866ull, 3264869361ull, 1275304092ull, 2678503631ull, 799221971ull, + 3033277898ull, 578644373ull, 4189590755ull, 3429169451ull, 1401086063ull, 69772ull, + 3494619096ull, 80043234ull, 4122720890ull, 196294612ull, 2014227113ull, 4198730931ull, + 229248332ull, 14138ull, 2430455430ull, 4294909015ull, 569677507ull, 67771ull, + 804311418ull, 3013ull, 79972640ull, 14223ull, 606116343ull, 4294953287ull, + 3901683243ull, 20357ull, 1425188206ull, 14853ull, 2489717387ull, 6777ull, + 2957673355ull, 4294963455ull, 3760153287ull, 24384ull, 2590412492ull, 1224ull, + 994318727ull, 0ull, 1242461408ull, 4294967295ull, 3067474766ull, 1ull, + 2113616290ull, 4294967295ull, 3617616882ull, 0ull, 559337523ull, 0ull, + 4067496095ull, 3663900563ull, 1612977870ull, 2988531300ull, 1361177336ull, 0ull, + 366217418ull, 4294967295ull, 3851633817ull, 159667373ull, 1046630ull, 166230526ull, + 1736278016ull, 0ull, 940015616ull, 0ull, 2612804199ull, 81229ull, + 1051124084ull, 19396ull, 2477654016ull, 0ull, 4260495360ull, 4294967295ull, + 1945774818ull, 55474ull, 850420953ull, 17968ull, 2147483648ull, 0ull, + 0ull, 4294967295ull, 1887116017ull, 1ull, 539098074ull, 0ull, + 3934492603ull, 11776773ull, 3192995692ull, 4281265255ull, 1881624432ull, 12895610ull, + 4249298779ull, 7292396ull, 2147483648ull, 0ull, 0ull, 4294967295ull, + 2038334034ull, 1ull, 215001140ull, 4294967295ull, 3609596363ull, 456088337ull, + 4189192591ull, 72412348ull, 1983203700ull, 187707940ull, 696216650ull, 154640864ull, + 1073741824ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 15ull, 1ull, 15ull, 1ull, 15ull, + 1ull, 15ull, 4ull, 12ull, 4ull, 12ull, + 4ull, 12ull, 4ull, 12ull, 15ull, 1ull, + 15ull, 1ull, 15ull, 1ull, 15ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 15ull, 1ull, 15ull, + 1ull, 15ull, 1ull, 15ull, 15ull, 1ull, + 15ull, 1ull, 15ull, 1ull, 15ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 15ull, 1ull, 15ull, + 1ull, 15ull, 1ull, 15ull, 15ull, 1ull, + 15ull, 1ull, 15ull, 1ull, 15ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 15ull, 1ull, 15ull, 1ull, + 15ull, 1ull, 15ull, 1ull, 7ull, 9ull, + 7ull, 9ull, 7ull, 9ull, 7ull, 9ull, + 15ull, 1ull, 15ull, 1ull, 15ull, 1ull, + 15ull, 1ull, 3ull, 13ull, 3ull, 13ull, + 3ull, 13ull, 3ull, 13ull, 14ull, 2ull, + 14ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 7275ull, 0ull, 36168ull, 0ull, + 19813ull, 0ull, 11240ull, 0ull, 48406ull, 0ull, + 21204ull, 32768ull, 48278ull, 0ull, 31414ull, 32768ull, + 64176ull, 49152ull, 8176ull, 45056ull, 58128ull, 45056ull, + 29760ull, 24576ull, 32768ull, 23250ull, 32768ull, 44802ull, + 32768ull, 17572ull, 32768ull, 32398ull, 52313ull, 0ull, + 48366ull, 0ull, 26439ull, 0ull, 61456ull, 0ull, + 42346ull, 32768ull, 25002ull, 0ull, 54930ull, 0ull, + 61572ull, 0ull, 32768ull, 1154ull, 32768ull, 44146ull, + 32768ull, 64696ull, 32768ull, 55956ull, 29682ull, 0ull, + 53299ull, 0ull, 26502ull, 0ull, 65025ull, 0ull, + 60152ull, 0ull, 2250ull, 0ull, 44468ull, 32768ull, + 40036ull, 0ull, 32768ull, 59748ull, 32768ull, 6590ull, + 32768ull, 7490ull, 0ull, 16288ull, 37806ull, 0ull, + 65010ull, 0ull, 27705ull, 0ull, 6092ull, 0ull, + 32768ull, 16766ull, 0ull, 20274ull, 32768ull, 26736ull, + 32768ull, 27446ull, 26752ull, 33792ull, 50304ull, 47104ull, + 57728ull, 41472ull, 49024ull, 60928ull, 32768ull, 33782ull, + 0ull, 49488ull, 0ull, 29922ull, 32768ull, 37210ull, + 25008ull, 40960ull, 52072ull, 40960ull, 57312ull, 49152ull, + 6344ull, 57344ull, 16384ull, 12956ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 55603ull, 0ull, + 8916ull, 0ull, 9380ull, 0ull, 12981ull, 0ull, + 35940ull, 0ull, 53284ull, 32768ull, 19298ull, 0ull, + 3402ull, 0ull, 15216ull, 0ull, 29008ull, 12288ull, + 10640ull, 8192ull, 62416ull, 40960ull, 0ull, 6984ull, + 32768ull, 20804ull, 0ull, 42890ull, 32768ull, 24440ull, + 18720ull, 0ull, 39415ull, 0ull, 63019ull, 0ull, + 42350ull, 0ull, 4746ull, 32768ull, 33674ull, 0ull, + 25286ull, 0ull, 36556ull, 32768ull, 0ull, 57960ull, + 0ull, 12678ull, 32768ull, 47262ull, 0ull, 31072ull, + 55200ull, 0ull, 8534ull, 0ull, 12734ull, 0ull, + 55352ull, 0ull, 20768ull, 0ull, 5588ull, 32768ull, + 22180ull, 0ull, 12072ull, 32768ull, 0ull, 63660ull, + 0ull, 30696ull, 0ull, 58444ull, 0ull, 1116ull, + 20727ull, 0ull, 3754ull, 0ull, 10978ull, 0ull, + 25817ull, 0ull, 0ull, 61398ull, 32768ull, 38824ull, + 0ull, 65522ull, 32768ull, 43360ull, 58880ull, 12800ull, + 27264ull, 51200ull, 12928ull, 22528ull, 11264ull, 2560ull, + 0ull, 26326ull, 32768ull, 37458ull, 32768ull, 8808ull, + 32768ull, 54824ull, 61112ull, 57344ull, 3096ull, 40960ull, + 3960ull, 40960ull, 4368ull, 40960ull, 0ull, 32228ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 47851ull, 0ull, 12977ull, 0ull, 21071ull, 0ull, + 11209ull, 0ull, 62448ull, 32768ull, 47310ull, 32768ull, + 9674ull, 32768ull, 5346ull, 0ull, 57440ull, 28672ull, + 42960ull, 40960ull, 47728ull, 53248ull, 43680ull, 49152ull, + 0ull, 63372ull, 32768ull, 32394ull, 32768ull, 39810ull, + 32768ull, 12882ull, 1220ull, 0ull, 9248ull, 0ull, + 59534ull, 0ull, 21746ull, 0ull, 37990ull, 0ull, + 45130ull, 32768ull, 57374ull, 32768ull, 39526ull, 0ull, + 0ull, 38724ull, 0ull, 52130ull, 32768ull, 42914ull, + 0ull, 47114ull, 39663ull, 0ull, 59031ull, 0ull, + 4255ull, 0ull, 5838ull, 0ull, 12928ull, 32768ull, + 10176ull, 0ull, 17560ull, 32768ull, 63590ull, 32768ull, + 0ull, 36204ull, 32768ull, 53944ull, 0ull, 60398ull, + 32768ull, 10170ull, 47586ull, 0ull, 49863ull, 0ull, + 29690ull, 0ull, 12976ull, 0ull, 0ull, 30488ull, + 0ull, 58752ull, 0ull, 4766ull, 0ull, 5322ull, + 38784ull, 57344ull, 16128ull, 56832ull, 20224ull, 48640ull, + 9984ull, 53248ull, 32768ull, 40490ull, 32768ull, 34836ull, + 0ull, 3244ull, 0ull, 18534ull, 4552ull, 8192ull, + 392ull, 32768ull, 18800ull, 16384ull, 27720ull, 16384ull, + 49152ull, 4112ull, 49152ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 2175ull, 0ull, 9448ull, 0ull, + 22065ull, 0ull, 27594ull, 0ull, 49816ull, 32768ull, + 40870ull, 0ull, 46284ull, 0ull, 63928ull, 32768ull, + 21376ull, 4096ull, 53312ull, 20480ull, 62896ull, 12288ull, + 30720ull, 32768ull, 0ull, 14138ull, 32768ull, 7254ull, + 0ull, 2234ull, 0ull, 3012ull, 31145ull, 0ull, + 20072ull, 0ull, 2331ull, 0ull, 33506ull, 0ull, + 1070ull, 32768ull, 22808ull, 32768ull, 48258ull, 0ull, + 12412ull, 32768ull, 0ull, 58904ull, 0ull, 5656ull, + 0ull, 318ull, 0ull, 54790ull, 33608ull, 0ull, + 10129ull, 0ull, 62065ull, 0ull, 24612ull, 0ull, + 48206ull, 32768ull, 31326ull, 32768ull, 58770ull, 0ull, + 14ull, 0ull, 32768ull, 45982ull, 32768ull, 9400ull, + 32768ull, 15692ull, 0ull, 19396ull, 32703ull, 0ull, + 14760ull, 0ull, 13345ull, 0ull, 24516ull, 0ull, + 0ull, 62034ull, 0ull, 60606ull, 0ull, 63576ull, + 0ull, 19148ull, 60032ull, 45568ull, 48640ull, 60416ull, + 28672ull, 50176ull, 64768ull, 17408ull, 32768ull, 57542ull, + 0ull, 55606ull, 32768ull, 41086ull, 0ull, 26620ull, + 55072ull, 16384ull, 63920ull, 57344ull, 30256ull, 8192ull, + 10616ull, 40960ull, 49152ull, 34080ull, 32768ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 32768ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 65520ull, 61440ull, 0ull, 0ull, + 0ull, 65534ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 32768ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 32768ull, 65534ull, 0ull, 0ull, 32768ull, 65534ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 32768ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 32768ull, 0ull, 0ull, 0ull, 32768ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 65534ull, + 0ull, 0ull, 32768ull, 0ull, 0ull, 0ull, + 65408ull, 65024ull, 0ull, 0ull, 65408ull, 0ull, + 0ull, 0ull, 32768ull, 65534ull, 0ull, 0ull, + 0ull, 65534ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 65528ull, 0ull, 0ull, 0ull, + 49152ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 51654ull, 0ull, 12382ull, 0ull, 31839ull, + 0ull, 33047ull, 0ull, 28144ull, 0ull, 25393ull, + 0ull, 14011ull, 0ull, 24619ull, 6ull, 1912ull, + 6ull, 3371ull, 0ull, 1405ull, 4ull, 1111ull, + 6494ull, 1ull, 19466ull, 1ull, 14337ull, 0ull, + 3565ull, 1ull, 0ull, 58347ull, 0ull, 41598ull, + 0ull, 38770ull, 0ull, 46525ull, 1ull, 7152ull, + 1ull, 23796ull, 1ull, 465ull, 0ull, 31494ull, + 6535ull, 0ull, 29920ull, 1ull, 29518ull, 1ull, + 14754ull, 1ull, 0ull, 18642ull, 0ull, 15607ull, + 0ull, 1157ull, 0ull, 64509ull, 1ull, 19137ull, + 0ull, 7513ull, 1ull, 21303ull, 0ull, 10795ull, + 26493ull, 1ull, 14343ull, 0ull, 2654ull, 0ull, + 19885ull, 0ull, 0ull, 60901ull, 0ull, 63257ull, + 0ull, 49252ull, 0ull, 49083ull, 11529ull, 1ull, + 17375ull, 1ull, 6897ull, 1ull, 31706ull, 0ull, + 92ull, 270ull, 22ull, 321ull, 95ull, 118ull, + 21ull, 126ull, 21693ull, 0ull, 19676ull, 0ull, + 594ull, 1ull, 10292ull, 0ull, 1ull, 2356ull, + 6ull, 3665ull, 6ull, 5743ull, 6ull, 6761ull, + 623ull, 2ull, 2888ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 51932ull, 0ull, 47640ull, + 0ull, 5976ull, 0ull, 57175ull, 1ull, 16080ull, + 1ull, 19134ull, 0ull, 12785ull, 1ull, 29246ull, + 15ull, 3042ull, 8ull, 3560ull, 10ull, 2261ull, + 9ull, 3747ull, 3404ull, 0ull, 20102ull, 1ull, + 5827ull, 1ull, 20858ull, 0ull, 0ull, 29330ull, + 0ull, 8ull, 0ull, 23568ull, 0ull, 7755ull, + 0ull, 11839ull, 0ull, 25272ull, 1ull, 27740ull, + 0ull, 3754ull, 15172ull, 1ull, 18958ull, 1ull, + 14037ull, 0ull, 32251ull, 0ull, 0ull, 3247ull, + 0ull, 36139ull, 0ull, 16104ull, 0ull, 39362ull, + 1ull, 17696ull, 0ull, 11292ull, 1ull, 19665ull, + 0ull, 6363ull, 3033ull, 0ull, 28216ull, 1ull, + 14951ull, 0ull, 24948ull, 0ull, 0ull, 53773ull, + 0ull, 37069ull, 0ull, 12506ull, 0ull, 37665ull, + 11993ull, 0ull, 9996ull, 0ull, 28795ull, 1ull, + 8225ull, 0ull, 59ull, 22ull, 108ull, 185ull, + 112ull, 183ull, 91ull, 438ull, 729ull, 1ull, + 27619ull, 0ull, 31102ull, 0ull, 3280ull, 1ull, + 3ull, 6135ull, 7ull, 6162ull, 4ull, 590ull, + 2ull, 6523ull, 3757ull, 3ull, 9109ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 56659ull, + 0ull, 37139ull, 0ull, 1230ull, 0ull, 50978ull, + 1ull, 6300ull, 0ull, 10451ull, 0ull, 27029ull, + 0ull, 31019ull, 2ull, 140ull, 11ull, 3298ull, + 11ull, 2004ull, 14ull, 3251ull, 3498ull, 0ull, + 4317ull, 1ull, 8692ull, 1ull, 12272ull, 1ull, + 0ull, 14554ull, 0ull, 16876ull, 0ull, 34312ull, + 0ull, 41143ull, 1ull, 29677ull, 1ull, 22090ull, + 0ull, 29063ull, 0ull, 13847ull, 9534ull, 0ull, + 28375ull, 1ull, 14432ull, 0ull, 8230ull, 1ull, + 0ull, 53970ull, 0ull, 16933ull, 0ull, 44947ull, + 0ull, 24164ull, 1ull, 9050ull, 1ull, 32399ull, + 1ull, 21677ull, 1ull, 31230ull, 11027ull, 0ull, + 23826ull, 1ull, 7100ull, 1ull, 16038ull, 0ull, + 0ull, 27558ull, 0ull, 11231ull, 0ull, 31671ull, + 0ull, 56202ull, 995ull, 0ull, 1036ull, 0ull, + 10609ull, 1ull, 26286ull, 1ull, 3ull, 261ull, + 81ull, 103ull, 39ull, 378ull, 71ull, 492ull, + 2351ull, 1ull, 8258ull, 0ull, 2159ull, 1ull, + 18988ull, 0ull, 6ull, 6929ull, 2ull, 3260ull, + 5ull, 4644ull, 7ull, 480ull, 5942ull, 1ull, + 7888ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 57485ull, 0ull, 35580ull, 0ull, 29830ull, + 0ull, 28983ull, 1ull, 19459ull, 0ull, 12195ull, + 0ull, 8829ull, 0ull, 19556ull, 11ull, 1ull, + 6ull, 1221ull, 15ull, 2995ull, 8ull, 2627ull, + 21161ull, 0ull, 3875ull, 1ull, 8295ull, 1ull, + 3277ull, 0ull, 0ull, 14223ull, 0ull, 51527ull, + 0ull, 20357ull, 0ull, 14853ull, 0ull, 6777ull, + 0ull, 28927ull, 1ull, 24384ull, 0ull, 1224ull, + 2333ull, 0ull, 21186ull, 1ull, 6491ull, 1ull, + 17554ull, 1ull, 0ull, 12520ull, 0ull, 11562ull, + 0ull, 55906ull, 0ull, 45601ull, 0ull, 4401ull, + 0ull, 18798ull, 0ull, 2436ull, 0ull, 2536ull, + 26811ull, 1ull, 30695ull, 0ull, 13968ull, 1ull, + 18178ull, 0ull, 0ull, 52351ull, 0ull, 39357ull, + 0ull, 55474ull, 0ull, 17968ull, 7431ull, 1ull, + 4133ull, 1ull, 28736ull, 1ull, 22504ull, 0ull, + 81ull, 179ull, 121ull, 302ull, 32ull, 196ull, + 121ull, 111ull, 18073ull, 0ull, 30888ull, 1ull, + 31363ull, 1ull, 9489ull, 1ull, 0ull, 6959ull, + 1ull, 1104ull, 2ull, 2864ull, 7ull, 2359ull, + 8252ull, 3ull, 15190ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 255ull, 255ull, + 255ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 65535ull, 65535ull, 65535ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 4294967295ull, 4294967295ull, 4294967295ull, 0ull, + 0ull, 0ull, 0ull, 0ull, +}; + +// MEMW: 13 ops, then zero rows to 16. +inline constexpr uint64_t KAT_MEMW_N = 13; +inline constexpr uint64_t KAT_MEMW_HEIGHT = 16; +inline const MemwOpC KAT_MEMW_OPS[KAT_MEMW_N] = { + {2196302393769983992ull, 9081178967921045189ull, + {2954885813557003648ull, 1661472893429511991ull, 3734671298466394841ull, 2804435598620097887ull, 2878566473777557094ull, 1494187763453411750ull, 3180384316123522860ull, 4227787398478374558ull}, + {93u, 94u, 129u, 210u, 173u, 49u, 25u, 21u}, + {227u, 60u, 175u, 205u, 167u, 245u, 147u, 248u}, 0, 1, 1}, + {9168491704955699193ull, 824055680800087121ull, + {725295466165624924ull, 538840363712271318ull, 4006218208199736057ull, 3422980854186058479ull, 490669743657964488ull, 46440874509084244ull, 1258153541990992240ull, 1911481139145858439ull}, + {1792327763u, 2619281666u, 3007122281u, 2126367403u, 3476572207u, 1116743952u, 4258151372u, 3271503573u}, + {3797432940u, 3303276053u, 1418444600u, 2490694231u, 2148267216u, 801154218u, 2157854893u, 2399297619u}, 1, 2, 0}, + {6483759454121623546ull, 4165077609662190713ull, + {496554143037838210ull, 2756998485883754593ull, 4293964851084894913ull, 2301337772901485589ull, 3619548821125451828ull, 2695129525285949867ull, 2307884313329951990ull, 2035393032657683236ull}, + {182u, 103u, 14u, 101u, 139u, 178u, 246u, 138u}, + {103u, 184u, 172u, 188u, 42u, 172u, 39u, 123u}, 0, 4, 1}, + {17137636439865950203ull, 2277468098299651171ull, + {1998753417335254990ull, 2322304771306178975ull, 3611124266713159943ull, 10638503693523863ull, 2981588694873935740ull, 525872920542244658ull, 2118199539006492245ull, 2813510239113273210ull}, + {232u, 238u, 54u, 214u, 104u, 114u, 74u, 197u}, + {20u, 175u, 191u, 53u, 198u, 56u, 14u, 56u}, 0, 8, 0}, + {8301575109131370492ull, 3696266597942444955ull, + {1525922660825425784ull, 3743982974522745862ull, 3461738336423173089ull, 1872885701690551189ull, 2209645889423875001ull, 192047156818613367ull, 768800933207181595ull, 3224215871763977292ull}, + {610387765u, 1254924969u, 2205510012u, 3116234940u, 2276692144u, 3962942534u, 596110680u, 3324776147u}, + {722663786u, 2105655426u, 866039207u, 1708790782u, 1556902463u, 2963559917u, 464946901u, 3991779705u}, 1, 1, 1}, + {1759505860444291069ull, 4557377184466370123ull, + {2456716564870803979ull, 4472618789715286119ull, 1591099849829788352ull, 18446744073709551615ull, 3958588045215174355ull, 287686489480725851ull, 135577209866610411ull, 3442578233704241357ull}, + {90u, 188u, 112u, 194u, 73u, 42u, 166u, 51u}, + {121u, 43u, 132u, 111u, 185u, 14u, 158u, 215u}, 0, 2, 0}, + {3778345007316991998ull, 4489848373588757409ull, + {4336349674250550900ull, 340395615713414508ull, 3212225615816820825ull, 3193530707837933053ull, 2227970912412159345ull, 783779615247488044ull, 1340974627655140014ull, 3251741792972581193ull}, + {179u, 209u, 142u, 119u, 238u, 6u, 211u, 94u}, + {202u, 81u, 177u, 229u, 233u, 180u, 175u, 51u}, 0, 4, 1}, + {7760563316082081791ull, 5989954969848584048ull, + {3729920077674538576ull, 1700987671907318318ull, 2833602738064837823ull, 3716611885553609621ull, 2582093758564906122ull, 3310865308276243275ull, 1675306767855572364ull, 3204225296198178234ull}, + {3589701959u, 2115314294u, 3661998622u, 2201061761u, 1417397106u, 3538148509u, 3351409000u, 2001232663u}, + {1432466479u, 3227825056u, 981526728u, 2627187194u, 3055721352u, 3279833680u, 2249625064u, 1218313875u}, 1, 8, 0}, + {16915589108858879992ull, 3497619950256507815ull, + {819739137613000190ull, 3236397084258259938ull, 3162804072217323706ull, 3405030582503668538ull, 1706904116101527121ull, 1465615467312552862ull, 1387700871113711709ull, 35104741372201040ull}, + {20u, 183u, 120u, 161u, 195u, 129u, 173u, 251u}, + {238u, 168u, 28u, 169u, 100u, 128u, 149u, 243u}, 0, 1, 1}, + {3278754153748234233ull, 6947241879897815600ull, + {1228157821991555394ull, 3717571730141111693ull, 881137600169767929ull, 1256931607090186940ull, 3568013674145294917ull, 2731111278837879252ull, 1235531932991409149ull, 2714687935692706761ull}, + {128u, 180u, 118u, 39u, 98u, 74u, 154u, 99u}, + {12u, 183u, 170u, 28u, 48u, 98u, 229u, 6u}, 0, 2, 0}, + {173765365954772986ull, 6670201659442247711ull, + {1561011233634449549ull, 4093678442872381943ull, 3910600373195959553ull, 4591475827895699557ull, 1898634050408108461ull, 2265969318370764013ull, 168166693317244455ull, 3392444601375953114ull}, + {1777558993u, 2075714399u, 3441943106u, 3384625345u, 1601054126u, 4204046316u, 3083579879u, 2904534582u}, + {2630144845u, 2154451461u, 1874451909u, 3866949867u, 2513529116u, 1186144462u, 3039232492u, 1126333962u}, 1, 4, 1}, + {17761019808662421499ull, 365784363734156217ull, + {1688629848014891874ull, 2756610353844752467ull, 494047915170602873ull, 2560605749489036704ull, 2502829609701754797ull, 1365347485999347989ull, 4166446637069243383ull, 4183997100564958197ull}, + {52u, 90u, 30u, 36u, 53u, 33u, 142u, 82u}, + {24u, 177u, 46u, 65u, 131u, 34u, 28u, 133u}, 0, 8, 0}, + {8182478075489943548ull, 2129355515390709960ull, + {2172301215861080189ull, 460340283214038847ull, 1280852517770519772ull, 2432403070865057414ull, 477515050803297584ull, 2548358794965430431ull, 3793725434275240494ull, 2615043946631216554ull}, + {226u, 70u, 135u, 183u, 136u, 156u, 63u, 103u}, + {233u, 215u, 59u, 177u, 127u, 179u, 9u, 217u}, 0, 1, 1}, +}; +// Column-major: column c's rows from c * KAT_MEMW_HEIGHT. +inline const uint64_t KAT_MEMW_WORDS[] = { + 0ull, 1ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 4294967288ull, 4294967289ull, + 4294967290ull, 4294967291ull, 4294967292ull, 4294967293ull, 4294967294ull, 4294967295ull, + 4294967288ull, 4294967289ull, 4294967290ull, 4294967291ull, 4294967292ull, 0ull, + 0ull, 0ull, 511366499ull, 2134705824ull, 1509617886ull, 3990166922ull, + 1932861075ull, 409666881ull, 879714499ull, 1806896951ull, 3938467499ull, 763394439ull, + 40457901ull, 4135309673ull, 1905131636ull, 0ull, 0ull, 0ull, + 93ull, 1792327763ull, 182ull, 232ull, 610387765ull, 90ull, + 179ull, 3589701959ull, 20ull, 128ull, 1777558993ull, 52ull, + 226ull, 0ull, 0ull, 0ull, 94ull, 2619281666ull, + 103ull, 238ull, 1254924969ull, 188ull, 209ull, 2115314294ull, + 183ull, 180ull, 2075714399ull, 90ull, 70ull, 0ull, + 0ull, 0ull, 129ull, 3007122281ull, 14ull, 54ull, + 2205510012ull, 112ull, 142ull, 3661998622ull, 120ull, 118ull, + 3441943106ull, 30ull, 135ull, 0ull, 0ull, 0ull, + 210ull, 2126367403ull, 101ull, 214ull, 3116234940ull, 194ull, + 119ull, 2201061761ull, 161ull, 39ull, 3384625345ull, 36ull, + 183ull, 0ull, 0ull, 0ull, 173ull, 3476572207ull, + 139ull, 104ull, 2276692144ull, 73ull, 238ull, 1417397106ull, + 195ull, 98ull, 1601054126ull, 53ull, 136ull, 0ull, + 0ull, 0ull, 49ull, 1116743952ull, 178ull, 114ull, + 3962942534ull, 42ull, 6ull, 3538148509ull, 129ull, 74ull, + 4204046316ull, 33ull, 156ull, 0ull, 0ull, 0ull, + 25ull, 4258151372ull, 246ull, 74ull, 596110680ull, 166ull, + 211ull, 3351409000ull, 173ull, 154ull, 3083579879ull, 142ull, + 63ull, 0ull, 0ull, 0ull, 21ull, 3271503573ull, + 138ull, 197ull, 3324776147ull, 51ull, 94ull, 2001232663ull, + 251ull, 99ull, 2904534582ull, 82ull, 103ull, 0ull, + 0ull, 0ull, 1018080965ull, 2436586577ull, 3118011513ull, 980378723ull, + 494945179ull, 999718475ull, 3139406753ull, 375986032ull, 1180421031ull, 2215033392ull, + 2873538591ull, 3636741049ull, 929157320ull, 0ull, 0ull, 0ull, + 2114376744ull, 191865414ull, 969757700ull, 530264363ull, 860604131ull, 1061097063ull, + 1045374286ull, 1394645071ull, 814353104ull, 1617530798ull, 1553027345ull, 85165808ull, + 495779215ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 227ull, 3797432940ull, + 103ull, 20ull, 722663786ull, 121ull, 202ull, 1432466479ull, + 238ull, 12ull, 2630144845ull, 24ull, 233ull, 0ull, + 0ull, 0ull, 60ull, 3303276053ull, 184ull, 175ull, + 2105655426ull, 43ull, 81ull, 3227825056ull, 168ull, 183ull, + 2154451461ull, 177ull, 215ull, 0ull, 0ull, 0ull, + 175ull, 1418444600ull, 172ull, 191ull, 866039207ull, 132ull, + 177ull, 981526728ull, 28ull, 170ull, 1874451909ull, 46ull, + 59ull, 0ull, 0ull, 0ull, 205ull, 2490694231ull, + 188ull, 53ull, 1708790782ull, 111ull, 229ull, 2627187194ull, + 169ull, 28ull, 3866949867ull, 65ull, 177ull, 0ull, + 0ull, 0ull, 167ull, 2148267216ull, 42ull, 198ull, + 1556902463ull, 185ull, 233ull, 3055721352ull, 100ull, 48ull, + 2513529116ull, 131ull, 127ull, 0ull, 0ull, 0ull, + 245ull, 801154218ull, 172ull, 56ull, 2963559917ull, 14ull, + 180ull, 3279833680ull, 128ull, 98ull, 1186144462ull, 34ull, + 179ull, 0ull, 0ull, 0ull, 147ull, 2157854893ull, + 39ull, 14ull, 464946901ull, 158ull, 175ull, 2249625064ull, + 149ull, 229ull, 3039232492ull, 28ull, 9ull, 0ull, + 0ull, 0ull, 248ull, 2399297619ull, 123ull, 56ull, + 3991779705ull, 215ull, 51ull, 1218313875ull, 243ull, 6ull, + 1126333962ull, 133ull, 217ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 0ull, 1ull, 1ull, 1ull, + 1ull, 0ull, 0ull, 0ull, 3840411008ull, 973135964ull, + 3146044290ull, 2849877966ull, 1228400504ull, 317796875ull, 3281370740ull, 1986225744ull, + 166227454ull, 871141698ull, 1194129549ull, 1745660770ull, 1284276349ull, 0ull, + 0ull, 0ull, 687987965ull, 168871010ull, 115613020ull, 465371044ull, + 355281555ull, 571998899ull, 1009635085ull, 868439692ull, 190860391ull, 285952776ull, + 363451250ull, 393164774ull, 505778290ull, 0ull, 0ull, 0ull, + 818837303ull, 3048625110ull, 90445921ull, 2850441631ull, 3381257222ull, 2696386663ull, + 1303563628ull, 2136652334ull, 1062392802ull, 3256378765ull, 3289719287ull, 4245983315ull, + 1892058943ull, 0ull, 0ull, 0ull, 386841803ull, 125458548ull, + 641913732ull, 540703714ull, 871713965ull, 1041362711ull, 79254530ull, 396042054ull, + 753532416ull, 865564618ull, 953133786ull, 641823362ull, 107181324ull, 0ull, + 0ull, 0ull, 1106227929ull, 141720313ull, 451781313ull, 1429936391ull, + 131783649ull, 659040960ull, 2972452953ull, 1559088319ull, 3802933434ull, 2556163065ull, + 4141542657ull, 365174649ull, 593519836ull, 0ull, 0ull, 0ull, + 869545922ull, 932770364ull, 999766600ull, 840780387ull, 805998765ull, 370456802ull, + 747904557ull, 659749549ull, 736397707ull, 205155834ull, 910507601ull, 115029494ull, + 298221716ull, 0ull, 0ull, 0ull, 2201353567ull, 2126062319ull, + 3992669205ull, 2845318039ull, 1781296021ull, 4294967295ull, 1006327293ull, 2255154069ull, + 796457786ull, 462572220ull, 1417939045ull, 3424513440ull, 2642093702ull, 0ull, + 0ull, 0ull, 652958545ull, 796974835ull, 535821954ull, 2476969ull, + 436065183ull, 4294967295ull, 743551810ull, 865341137ull, 792795462ull, 292652195ull, + 1069036272ull, 596187484ull, 566337972ull, 0ull, 0ull, 0ull, + 3822857830ull, 1019466696ull, 3260928052ull, 4031254396ull, 2927827897ull, 2267582163ull, + 2486881649ull, 3355686026ull, 31602257ull, 1361750597ull, 84496813ull, 4221192109ull, + 2718465328ull, 0ull, 0ull, 0ull, 670218484ull, 114242952ull, + 842741881ull, 694205214ull, 514473274ull, 921680602ull, 518739901ull, 601190551ull, + 397419584ull, 830742920ull, 442060188ull, 582735428ull, 111180136ull, 0ull, + 0ull, 0ull, 3026207142ull, 3022792276ull, 441848235ull, 2448092978ull, + 575411319ull, 2217575771ull, 1292162092ull, 3338156875ull, 2591792030ull, 4046574036ull, + 3849086189ull, 4193696021ull, 2116563103ull, 0ull, 0ull, 0ull, + 347892698ull, 10812858ull, 627508742ull, 122439330ull, 44714463ull, 66982230ull, + 182487912ull, 770870900ull, 341240192ull, 635886396ull, 527587094ull, 317894733ull, + 593336018ull, 0ull, 0ull, 0ull, 1673799468ull, 4282763632ull, + 649749750ull, 232836693ull, 516786459ull, 163374827ull, 2922529454ull, 332846476ull, + 87793757ull, 916114429ull, 1916401191ull, 1109529591ull, 3042384430ull, 0ull, + 0ull, 0ull, 740490927ull, 292936698ull, 537346190ull, 493181762ull, + 179000416ull, 31566529ull, 312219985ull, 390062753ull, 323099287ull, 287669695ull, + 39154359ull, 970076452ull, 883295534ull, 0ull, 0ull, 0ull, + 942483102ull, 4121288071ull, 3789407012ull, 3863306106ull, 3014333516ull, 4218345677ull, + 2808883529ull, 540448186ull, 2272004176ull, 1660982217ull, 422875354ull, 4168109045ull, + 1191001514ull, 0ull, 0ull, 0ull, 984358461ull, 445051383ull, + 473901869ull, 655071399ull, 750696256ull, 801537705ull, 757105134ull, 746041838ull, + 8173459ull, 632062539ull, 789865060ull, 974162737ull, 608862365ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 1ull, 0ull, + 1ull, 0ull, 1ull, 0ull, 1ull, 0ull, + 1ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 1ull, 0ull, 1ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, +}; + +// CPU: 24 decode entries in 3 runs; 29 logs (their ops from cycle +// 1073741804), each entry in turn, then padding rows to 32; every row's +// timestamp the table's own 4·(row + 1). KAT_CPU_ABSENT_PC is no entry's. +inline constexpr uint64_t KAT_CPU_N = 29; +inline constexpr uint64_t KAT_CPU_HEIGHT = 32; +inline constexpr uint64_t KAT_CPU_ABSENT_PC = 65538ull; +inline constexpr uint32_t KAT_CPU_ENTRIES_N = 24; +inline constexpr uint32_t KAT_CPU_RUNS_N = 3; +inline const DecodeEntryC KAT_CPU_ENTRIES[KAT_CPU_ENTRIES_N] = { + {65536ull, 18446744073709551467ull, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 15, 0, 9, 2, 0, 6}, + {65540ull, 18446744073709550848ull, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 21, 0, 22, 2, 36, 0}, + {65544ull, 1294721024ull, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 255, 0, 7, 2, 0, 0}, + {65548ull, 18446744073709550718ull, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 19, 0, 23, 2, 0, 0}, + {65552ull, 868ull, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 4, 20, 0, 2, 0, 5}, + {65556ull, 0ull, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 31, 30, 20, 2, 4, 0}, + {65560ull, 590659584ull, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 26, 2, 0, 0}, + {65564ull, 0ull, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 28, 13, 11, 2, 102, 0}, + {4194304ull, 18446744073709550795ull, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 12, 0, 23, 2, 0, 1}, + {4194308ull, 18446744073709186234ull, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 255, 0, 11, 2, 0, 1}, + {4194312ull, 18446744073709548546ull, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 2, 25, 0, 2, 3, 0}, + {4194316ull, 448ull, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 3, 9, 0, 2, 67, 0}, + {4194320ull, 18446744073709547868ull, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 26, 9, 0, 2, 36, 0}, + {4194324ull, 18446744073709550390ull, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 27, 17, 0, 2, 100, 0}, + {4194328ull, 18446744073709548670ull, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 24, 31, 0, 2, 4, 0}, + {4194332ull, 18446744073709547992ull, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 27, 29, 0, 2, 68, 0}, + {2147483648ull, 0ull, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 0, 2, 0, 0}, + {2147483652ull, 18446744073709462302ull, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 255, 0, 27, 2, 0, 1}, + {2147483656ull, 314436ull, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 255, 0, 7, 2, 0, 1}, + {2147483660ull, 16101376ull, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0}, + {2147483664ull, 1226ull, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 31, 0, 31, 2, 0, 16}, + {2147483668ull, 1518ull, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 18, 25, 0, 2, 36, 0}, + {2147483672ull, 18446744073709550619ull, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 26, 7, 0, 2, 0, 1}, + {2147483676ull, 0ull, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 19, 5, 5, 2, 0, 0}, +}; +inline const DecodeRunC KAT_CPU_RUNS[KAT_CPU_RUNS_N] = { + {65536ull, 0u, 8u}, + {4194304ull, 8u, 8u}, + {2147483648ull, 16u, 8u}, +}; +inline const LogC KAT_CPU_LOGS[KAT_CPU_N] = { + {65536ull, 65540ull, 4424988285630774666ull, 4424988285630774666ull, 18412148599395821942ull}, + {65540ull, 65544ull, 5960244934289483965ull, 1279977502728950153ull, 10788761249356058668ull}, + {65544ull, 65548ull, 10848412296834775076ull, 15689971538380043454ull, 6463848764750327784ull}, + {65548ull, 65552ull, 14948615613208466271ull, 14948615613208466271ull, 14412314415353964661ull}, + {65552ull, 65556ull, 6035435346081234459ull, 3423647205869163346ull, 651493761594111282ull}, + {65556ull, 65560ull, 9736314039822082370ull, 494373199572039951ull, 11512092647157196513ull}, + {65560ull, 65564ull, 12826766831794759114ull, 12826766831794759114ull, 17875598521845564709ull}, + {65564ull, 65568ull, 16684756930793824349ull, 17542606376304805231ull, 17070076722726251049ull}, + {4194304ull, 15787819435294302448ull, 9412844368937603194ull, 1208511819238993803ull, 14275292988177590773ull}, + {4194308ull, 4556673577045029952ull, 17496984900428930331ull, 17496984900428930331ull, 2569174710114771346ull}, + {4194312ull, 17887679673943296100ull, 5880119809985717223ull, 3987246593240169931ull, 11902221261797707878ull}, + {4194316ull, 6770703458175501848ull, 11805353509771578509ull, 13444140911507565574ull, 9230963099927369204ull}, + {4194320ull, 2965231511091852016ull, 3580147778431846789ull, 3580147778431846789ull, 15419036417094155511ull}, + {4194324ull, 119947997428345652ull, 9395083095236583466ull, 4631706875218869986ull, 6506806413312930716ull}, + {4194328ull, 4959415229374886436ull, 3253652646988073065ull, 1863968155119670720ull, 3179652840312867773ull}, + {4194332ull, 11311981632689615864ull, 16105200575807210881ull, 16105200575807210881ull, 9572621909934232754ull}, + {2147483648ull, 2147483652ull, 2062178634883773396ull, 17064953472395344149ull, 6241452547151611159ull}, + {2147483652ull, 5050024515479318284ull, 11140343034284772033ull, 7295133808540743224ull, 14854941012277360512ull}, + {2147483656ull, 15928706157484481808ull, 950335977927407995ull, 950335977927407995ull, 4304388707255785660ull}, + {2147483660ull, 2147483664ull, 4212601604506736926ull, 16144172961723867635ull, 947567545262650006ull}, + {2147483664ull, 2147483668ull, 10276760257794315736ull, 4166867176358383013ull, 17088046638902171367ull}, + {2147483668ull, 5106177096531089388ull, 5877275692953215290ull, 5877275692953215290ull, 2093707535188836546ull}, + {2147483672ull, 2147483676ull, 3355817143395868575ull, 4561139659940551790ull, 4630800053681630750ull}, + {2147483676ull, 2147483680ull, 14276149692267561406ull, 16489870132977831324ull, 15064721321420835040ull}, + {65536ull, 65540ull, 12245343449633703455ull, 12245343449633703455ull, 17128525636800697761ull}, + {65540ull, 65544ull, 18244907948753815990ull, 9443869848232899421ull, 15942204915329471555ull}, + {65544ull, 65548ull, 6199823000492333955ull, 4791469188991947545ull, 3051808446652994050ull}, + {65548ull, 65552ull, 12086316318820355627ull, 12086316318820355627ull, 18285688532243783015ull}, + {65552ull, 65556ull, 2263206762550924118ull, 12079518076941974946ull, 2936901075920567508ull}, +}; +// Column-major: column c's rows from c * KAT_CPU_HEIGHT. +inline const uint64_t KAT_CPU_WORDS[] = { + 4ull, 8ull, 12ull, 16ull, 20ull, 24ull, + 28ull, 32ull, 36ull, 40ull, 44ull, 48ull, + 52ull, 56ull, 60ull, 64ull, 68ull, 72ull, + 76ull, 80ull, 84ull, 88ull, 92ull, 96ull, + 100ull, 104ull, 108ull, 112ull, 116ull, 120ull, + 124ull, 128ull, 65536ull, 65540ull, 65544ull, 65548ull, + 65552ull, 65556ull, 65560ull, 65564ull, 4194304ull, 4194308ull, + 4194312ull, 4194316ull, 4194320ull, 4194324ull, 4194328ull, 4194332ull, + 2147483648ull, 2147483652ull, 2147483656ull, 2147483660ull, 2147483664ull, 2147483668ull, + 2147483672ull, 2147483676ull, 65536ull, 65540ull, 65544ull, 65548ull, + 65552ull, 1ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 15ull, 21ull, 255ull, 0ull, 4ull, 31ull, + 0ull, 0ull, 12ull, 255ull, 2ull, 3ull, + 26ull, 27ull, 24ull, 27ull, 17ull, 255ull, + 255ull, 0ull, 31ull, 18ull, 26ull, 19ull, + 15ull, 21ull, 255ull, 0ull, 4ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 20ull, 30ull, 0ull, 0ull, 0ull, 0ull, + 25ull, 9ull, 9ull, 17ull, 31ull, 29ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 25ull, + 7ull, 5ull, 0ull, 0ull, 0ull, 0ull, + 20ull, 0ull, 0ull, 0ull, 9ull, 22ull, + 7ull, 0ull, 0ull, 20ull, 26ull, 0ull, + 23ull, 11ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 27ull, 7ull, 6ull, + 31ull, 0ull, 0ull, 5ull, 9ull, 22ull, + 7ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 0ull, 1ull, 1ull, + 0ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 0ull, 1ull, 1ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 0ull, 0ull, 1ull, 1ull, 0ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 1ull, 1ull, + 1ull, 0ull, 0ull, 1ull, 1ull, 1ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 4294967147ull, 4294966528ull, 1294721024ull, 0ull, 868ull, 0ull, + 590659584ull, 0ull, 4294966475ull, 4294601914ull, 4294964226ull, 448ull, + 4294963548ull, 4294966070ull, 4294964350ull, 4294963672ull, 0ull, 4294877982ull, + 314436ull, 16101376ull, 1226ull, 1518ull, 4294966299ull, 0ull, + 4294967147ull, 4294966528ull, 1294721024ull, 0ull, 868ull, 0ull, + 0ull, 0ull, 4294967295ull, 4294967295ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 4294967295ull, 4294967295ull, + 4294967295ull, 0ull, 4294967295ull, 4294967295ull, 4294967295ull, 4294967295ull, + 0ull, 4294967295ull, 0ull, 0ull, 0ull, 0ull, + 4294967295ull, 0ull, 4294967295ull, 4294967295ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 2ull, 2ull, + 2ull, 2ull, 2ull, 2ull, 2ull, 2ull, + 2ull, 2ull, 2ull, 2ull, 2ull, 2ull, + 2ull, 2ull, 2ull, 2ull, 2ull, 2ull, + 2ull, 2ull, 2ull, 2ull, 2ull, 2ull, + 2ull, 2ull, 2ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 36ull, + 0ull, 0ull, 0ull, 4ull, 0ull, 0ull, + 0ull, 0ull, 3ull, 67ull, 36ull, 100ull, + 4ull, 68ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 36ull, 0ull, 0ull, 0ull, 36ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 1ull, 0ull, 1ull, 0ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 1ull, 0ull, 1ull, 0ull, + 1ull, 0ull, 1ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 1ull, 0ull, 1ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 6ull, 0ull, 0ull, 0ull, 5ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 1ull, 0ull, 16ull, 0ull, 1ull, 0ull, + 6ull, 0ull, 0ull, 0ull, 5ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 1ull, + 1ull, 1ull, 1ull, 1ull, 1ull, 1ull, + 0ull, 1ull, 1ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 65540ull, 65544ull, 65548ull, 65552ull, 65556ull, 65560ull, + 65564ull, 65568ull, 45719792ull, 3710743616ull, 4194316ull, 672089624ull, + 4194324ull, 4194328ull, 4194332ull, 2318742520ull, 2147483652ull, 1747496716ull, + 946904336ull, 2147483664ull, 2147483668ull, 2147483672ull, 2147483676ull, 2147483680ull, + 65540ull, 65544ull, 65548ull, 65552ull, 65556ull, 1ull, + 1ull, 1ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 3675888161ull, 1060933241ull, + 0ull, 1576427244ull, 0ull, 0ull, 0ull, 2633775964ull, + 0ull, 1175800458ull, 3708690907ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1924245878ull, 3932033068ull, + 1294786568ull, 0ull, 0ull, 3610306273ull, 590659584ull, 0ull, + 4194308ull, 4194312ull, 4194316ull, 4194320ull, 4194324ull, 4194328ull, + 4194332ull, 4194336ull, 0ull, 2147483656ull, 2147483660ull, 16101376ull, + 3104986855ull, 2147483672ull, 0ull, 3950081058ull, 3685569953ull, 2998080579ull, + 1294786568ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 4286912409ull, 2511954225ull, 0ull, 0ull, 0ull, 2680367940ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 3978620897ull, 0ull, 0ull, 3779545340ull, + 3988045648ull, 3711833831ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1ull, 1ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 1ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 3024715146ull, 1655007421ull, + 65544ull, 0ull, 1543650843ull, 3040764226ull, 0ull, 0ull, + 143642746ull, 4194308ull, 519021543ull, 1037601933ull, 2698348933ull, 1054515242ull, + 104581225ull, 1030210945ull, 3819287508ull, 2147483652ull, 2147483656ull, 0ull, + 2142148056ull, 2239996218ull, 2068242335ull, 3219718590ull, 605210143ull, 489425334ull, + 65544ull, 0ull, 1058804566ull, 0ull, 0ull, 0ull, + 1030272870ull, 1387727664ull, 0ull, 0ull, 1405234296ull, 2266912264ull, + 0ull, 0ull, 2191598613ull, 0ull, 1369072080ull, 2748648056ull, + 833568111ull, 2187463244ull, 757550040ull, 3749784216ull, 480138378ull, 0ull, + 0ull, 0ull, 2392744705ull, 1368409882ull, 781337065ull, 3323925121ull, + 2851091197ull, 4247973661ull, 0ull, 0ull, 526943887ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 2518094674ull, 1123481871ull, 0ull, 0ull, 0ull, 0ull, + 3052895691ull, 3434376198ull, 2698348933ull, 4180131554ull, 1317819840ull, 1030210945ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 2239996218ull, + 3518225518ull, 3564604828ull, 0ull, 0ull, 0ull, 0ull, + 3792256418ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 797129982ull, 115105230ull, 0ull, 0ull, + 0ull, 0ull, 928353190ull, 3130207981ull, 833568111ull, 1078403292ull, + 433988905ull, 3749784216ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 1368409882ull, 1061973082ull, 3839347076ull, 0ull, 0ull, + 0ull, 0ull, 2812481968ull, 0ull, 0ull, 0ull, + 4294967147ull, 4294966528ull, 1294721024ull, 0ull, 868ull, 1123481871ull, + 590659584ull, 0ull, 0ull, 0ull, 3052895691ull, 3434376198ull, + 2698348933ull, 4180131554ull, 1317819840ull, 1030210945ull, 0ull, 0ull, + 0ull, 16101376ull, 1226ull, 2239996218ull, 4294966299ull, 3564604828ull, + 4294967147ull, 4294966528ull, 1294721024ull, 0ull, 868ull, 0ull, + 0ull, 0ull, 4294967295ull, 4294967295ull, 0ull, 0ull, + 0ull, 115105230ull, 0ull, 0ull, 0ull, 0ull, + 928353190ull, 3130207981ull, 833568111ull, 1078403292ull, 433988905ull, 3749784216ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 1368409882ull, + 4294967295ull, 3839347076ull, 4294967295ull, 4294967295ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 31989ull, 4140ull, + 57352ull, 0ull, 16767ull, 59105ull, 49152ull, 0ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 0ull, + 0ull, 1ull, 0ull, 0ull, 0ull, 45056ull, + 39586ull, 0ull, 56250ull, 29730ull, 50570ull, 5187ull, + 57352ull, 0ull, 5818ull, 0ull, 0ull, 0ull, + 46153ull, 59998ull, 19756ull, 0ull, 23554ull, 55088ull, + 9012ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 245ull, 32686ull, 0ull, 31558ull, 60273ull, + 9234ull, 45747ull, 19756ull, 0ull, 16156ull, 0ull, + 0ull, 0ull, 46950ull, 24881ull, 0ull, 0ull, + 11384ull, 11076ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 25345ull, 0ull, + 16873ull, 18684ull, 13053ull, 5863ull, 0ull, 0ull, + 34447ull, 0ull, 0ull, 0ull, 15720ull, 38329ull, + 0ull, 0ull, 21442ull, 40899ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 36510ull, 0ull, 11922ull, 57671ull, 43504ull, 56638ull, + 0ull, 0ull, 8040ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 1ull, 1ull, 0ull, 1ull, + 0ull, 0ull, 0ull, 1ull, 0ull, 1ull, + 1ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, + 0ull, 0ull, +}; + +// KECCAK_RND: 4 ops; each case a table of rows [skip, skip + rows) of ops +// [first_op, first_op + nops); KAT_KECCAK_DIGESTS: per case, per row, per column group +// (columns [KAT_KECCAK_GROUP_END[g - 1], KAT_KECCAK_GROUP_END[g])), FNV-1a of the words' +// little-endian bytes. +inline constexpr int KAT_KECCAK_GROUPS = 15; +inline const uint32_t KAT_KECCAK_GROUP_END[KAT_KECCAK_GROUPS] = {2, 3, 203, 363, 403, 423, 463, 663, 863, 1063, 1263, 1463, 1471, 1479, 1480}; +struct KatKeccakCase { + uint32_t first_op; + uint32_t nops; + uint64_t skip; + uint64_t rows; +}; +inline constexpr int KAT_KECCAK_CASES_N = 2; +inline const KatKeccakCase KAT_KECCAK_CASES[KAT_KECCAK_CASES_N] = { + {0, 2, 21, 32}, + {2, 2, 10, 20}, +}; +inline const KeccakOpC KAT_KECCAK_OPS[4] = { + {4294966580ull, + {12674425323606611166ull, 7224583924505288613ull, 9253841140082780077ull, 15646447405310943616ull, 13902937212018515148ull, 17950215022809280635ull, 6624612683388723710ull, 18135366143176431890ull, 8247482728277031880ull, 5442814875136548487ull, 15928138076144925489ull, 6399128500011934564ull, 8502553403947463345ull, 13505255346702155355ull, 5612681234594373481ull, 16110180623266301015ull, 12748661194488041223ull, 12682745422687540869ull, 10864167830753512092ull, 4794152377681422488ull, 2005441664290232424ull, 3676767222173704279ull, 11671720315174187911ull, 16658153039762329236ull, 4958105345246185154ull}, + {2683597047092003659ull, 12826186436926684669ull, 8019650074904710360ull, 2150863586502558975ull, 5579760690090483989ull, 8080514585465249909ull, 3268304943918751921ull, 6139797853715278938ull, 12712040775490795809ull, 5852191730910666006ull, 511457036382311934ull, 4205938097282005975ull, 8228300477832805506ull, 854249159378616277ull, 13406968705669952484ull, 764676947809387456ull, 9900241518608029605ull, 9233136529966560106ull, 11915359974701909768ull, 8583121543250500747ull, 13303132703523276655ull, 6967661612317830134ull, 3239197285203246135ull, 12212423627561678584ull, 6071944868653966700ull}}, + {4294968804ull, + {308902593903867720ull, 8956868455053569351ull, 11843135764724078676ull, 6911140041862556430ull, 9798960420268402643ull, 17498726121255981328ull, 9335786755567227584ull, 5915882731460741489ull, 14021267657156268279ull, 6866466221687330660ull, 16382050180825158610ull, 6918422444313652148ull, 1941829575868020014ull, 1145815108730150579ull, 3628677934705480543ull, 18132823242354233122ull, 16063926841770727771ull, 13693863664527975929ull, 12630896139916174297ull, 6431682534514855861ull, 5336953394787147136ull, 12340414390232068082ull, 13248361877032509976ull, 1474145841288233373ull, 2403457084688226646ull}, + {3400586733915177118ull, 4913557236797452348ull, 10523809581827630904ull, 2429299634500162274ull, 16116507637409399992ull, 653365456954681629ull, 940584035424922035ull, 11869466847570456515ull, 18343863689146775167ull, 3102996752834236562ull, 8226328626116032580ull, 10567959678077839309ull, 1362811608861759160ull, 11311385942922720199ull, 17334780965435123032ull, 14242084687249706088ull, 12382915423299710054ull, 10392233342743712004ull, 4026245380792332669ull, 17935648474313015197ull, 13947457496502485539ull, 3425184099387171215ull, 10374259363361165240ull, 4363344802997950812ull, 3108278771138854259ull}}, + {4294972368ull, + {12457457393399288183ull, 5087635810258151956ull, 10686658854481118124ull, 15526602598769291195ull, 3107114212025771946ull, 7420396598984617778ull, 1115692637770830993ull, 4711989809983422589ull, 188738188489819656ull, 6711766703421988061ull, 11940507507528414061ull, 13742168237093757788ull, 11478560964550855200ull, 17127131227365652502ull, 3540858862079130635ull, 2294435845716551383ull, 6685030174765472872ull, 4381958460800187671ull, 2006184903740412421ull, 2297952048952982841ull, 7869131555081335689ull, 1059099975666924475ull, 13660295694471017210ull, 15682200453237518308ull, 4312256092096166579ull}, + {11582309675411281375ull, 15078539088660122020ull, 1997151618835736396ull, 3189999572166387317ull, 10979401860448291720ull, 17899270687354501482ull, 7664517702011109772ull, 5155921762307525714ull, 13101109632001527422ull, 5630321492955556222ull, 11909682267889586183ull, 7765555292371428703ull, 12966066369240326387ull, 3135640438798739494ull, 9840361264133014843ull, 219708709484435156ull, 13397904961911397658ull, 3688148651851013095ull, 79392555156086061ull, 14950292751958754095ull, 16745498471767433146ull, 13825969458461018888ull, 12846426637736237216ull, 565850077199459217ull, 3648512169578199281ull}}, + {4294975148ull, + {9648502122111253536ull, 7617448266142704140ull, 5636502310290544919ull, 17735399777426238736ull, 4049281545136372076ull, 476715735203365136ull, 16416173087533220323ull, 13326809340514359012ull, 11400886365557851217ull, 936620259879375513ull, 17867362534873306849ull, 5966862180131126339ull, 10498282743610586841ull, 239193180700419305ull, 18217650366353955201ull, 7862377735435122183ull, 17478885802313394277ull, 10136497119052738695ull, 10149889235772388497ull, 4369425081772528306ull, 4445446991621161831ull, 8368870315339710323ull, 5016911852782737759ull, 7534262086685514747ull, 13852267238018217382ull}, + {5507083841617193461ull, 16168264075704244758ull, 3903393148940489018ull, 6642242857904470681ull, 7033570715438523634ull, 13680993549573879617ull, 16379393348251695391ull, 12725206723243303036ull, 1397547520519414837ull, 11088680704560552619ull, 6758389056649124205ull, 9138334011870786968ull, 16154014116313716008ull, 1663896489873524486ull, 8937130985924373977ull, 12595898129365221545ull, 6199936745607912350ull, 10430614718694846221ull, 101774096265735881ull, 12194925575267115162ull, 9093187224011580386ull, 2384600699575754614ull, 1052398156785583837ull, 5748830314198527992ull, 12551869045076620004ull}}, +}; +inline const uint64_t KAT_KECCAK_DIGESTS[] = { + 8444736256554381952ull, 18270687615976106288ull, 8507147051953507757ull, 7809025194088770294ull, + 9582487095640738580ull, 5149934775982280293ull, 304259556555973788ull, 16599296527761185172ull, + 3037550400542729379ull, 304378558097735946ull, 8968275084465380186ull, 9297306764767040371ull, + 14871655760800376997ull, 109626154617237274ull, 9929646806074584996ull, 8444736256554381952ull, + 6520889763169322899ull, 9775510067500148979ull, 13877996751680257988ull, 1059513467440100776ull, + 16229633438060689189ull, 8710898417510695870ull, 12958700894073134504ull, 16443472434022862093ull, + 18041979704286812384ull, 14371486333644219096ull, 11373584951148850485ull, 6706413662153930404ull, + 14019056312594933358ull, 9929646806074584996ull, 8444736256554381952ull, 4288574356201733490ull, + 7776603201904697332ull, 18048385138274973545ull, 4688234263078239846ull, 1147371612499146341ull, + 15998859047722338551ull, 1931063831546386278ull, 13523265759558890383ull, 1558850420436763389ull, + 11156895287836856209ull, 13566140348326483747ull, 2235630609106157229ull, 3264832315523878764ull, + 9929646806074584996ull, 3431702464375352019ull, 12161962213042174405ull, 13193442654796683565ull, + 8334590212385724984ull, 15357449038890419092ull, 15837399225138575972ull, 4881747830283588061ull, + 15457419410096056341ull, 5032351535080145706ull, 15991665899349589484ull, 9934698833611660127ull, + 15965227733791834841ull, 12328143194273367844ull, 13372448871341388368ull, 9929646806074584996ull, + 3431702464375352019ull, 9929646806074584996ull, 13735461740483716568ull, 14982622454577743023ull, + 7435217960374110879ull, 14401823068531858021ull, 9016909727059087266ull, 7803883680080415839ull, + 4026290534972648267ull, 10795174486436525272ull, 10847882792969313447ull, 9337160245915445172ull, + 16192782323258333735ull, 103093238518686591ull, 9929646806074584996ull, 3431702464375352019ull, + 16626593026977353223ull, 8856113364939820470ull, 607923798490175660ull, 14746615506037896899ull, + 12356753349559203365ull, 14779538159487245776ull, 11137603547599430019ull, 2903209585752766649ull, + 3600250278459823796ull, 12738161514871901742ull, 2894986398833427299ull, 15202719380792455855ull, + 3463822072348202393ull, 9929646806074584996ull, 3431702464375352019ull, 14394277620009763814ull, + 15667136192267456873ull, 10608062864970450438ull, 1194732673652370332ull, 9746400522416399077ull, + 5438722676143559184ull, 9773747647927506076ull, 5119879930265316625ull, 17231856798224323375ull, + 6248421631441809955ull, 5312580966437055837ull, 12260904537780205221ull, 15260212911613784766ull, + 9929646806074584996ull, 3431702464375352019ull, 3232700585171816769ull, 9650634813105075421ull, + 18150793954620035729ull, 1255294920566153716ull, 6946892816177486437ull, 17256324596697066764ull, + 9064872560280733492ull, 16897685356011075017ull, 14939888438587711918ull, 8703612334675782558ull, + 14456055057927096761ull, 14934276586463937070ull, 10736831261721078847ull, 9929646806074584996ull, + 3431702464375352019ull, 1000385178204227360ull, 17994768618115889650ull, 6978850856305961202ull, + 14701600945896099275ull, 2835411922388146724ull, 4036570694837616797ull, 16829595688903779466ull, + 5989618240221916984ull, 8073800715309075972ull, 13086311043755292090ull, 6223882010014204550ull, + 6706413662153930404ull, 11249491403605533491ull, 9929646806074584996ull, 3431702464375352019ull, + 7697331399106995587ull, 718478703045465095ull, 13475808604046439783ull, 12827775488285280865ull, + 9152632141343974340ull, 7876875865506209346ull, 8659743236343876000ull, 17827631628586370004ull, + 10445525673023536156ull, 4509329771401153637ull, 14983464567303961261ull, 7820920359258290212ull, + 12930717912034219730ull, 9929646806074584996ull, 3431702464375352019ull, 5465015992139406178ull, + 17586931231182937260ull, 17970315207746565971ull, 7444601518716311927ull, 2994699725400872964ull, + 10759573379209432767ull, 6910503027886841270ull, 13099139457217738603ull, 11611935005913403424ull, + 16927279644915856457ull, 3519296852540855682ull, 6428354271802945324ull, 18376713052359267011ull, + 9929646806074584996ull, 3431702464375352019ull, 11573741395073338061ull, 3419237664924877195ull, + 10000557748717886640ull, 5158839027686148089ull, 13178594325029082532ull, 13416266880269612758ull, + 3223190212137770872ull, 17343702988820206866ull, 6336386618808657943ull, 18375978358793372530ull, + 11633121456120324023ull, 11077234374588830383ull, 8180204338630545054ull, 9929646806074584996ull, + 3431702464375352019ull, 9341425988105748652ull, 7177049025331364797ull, 4959928107689718256ull, + 5159176997360528853ull, 1331538611343586308ull, 14414985995998047500ull, 16309659978970882388ull, + 11319426391562296090ull, 3147542845147905272ull, 7808454228194984297ull, 588613566610964363ull, + 8971866838329954477ull, 10167695865699004236ull, 9929646806074584996ull, 3431702464375352019ull, + 16038372209008516879ull, 5718770807426144771ull, 13734018057415793961ull, 15380829863647024904ull, + 2076900057563131365ull, 11855693280904757422ull, 1036270753464392648ull, 12645904500633926133ull, + 4505134575788375569ull, 13747260766921882106ull, 9035473588797807070ull, 3014872817714212652ull, + 3091176465180369473ull, 9929646806074584996ull, 3431702464375352019ull, 13806056802040927470ull, + 13583494234369102039ull, 9837302318424076726ull, 2825634088546339232ull, 5275383672714511844ull, + 9731281448305029907ull, 6061193435299616993ull, 13874754679159550496ull, 7555963712425646206ull, + 2102283275668783319ull, 5733087702344704201ull, 18285935111577747887ull, 17001339639394697318ull, + 9929646806074584996ull, 3431702464375352019ull, 2644479767202980425ull, 1080902025688370627ull, + 11857441523096527685ull, 1032554748434012009ull, 13184222185283909573ull, 3451838179545068431ull, + 211858218902640425ull, 6589990505971079665ull, 8758863053197249378ull, 15476780931617393705ull, + 2363216312166459770ull, 2109262044873822894ull, 13210384081609662070ull, 9929646806074584996ull, + 3431702464375352019ull, 412164360235391016ull, 14387234344997518769ull, 7019837939240681569ull, + 9343219793494410412ull, 1713241759714182212ull, 4576764081045560377ull, 17032680829048578093ull, + 11034106804659870134ull, 17254780615482689560ull, 2715779596200417245ull, 4709281321957878195ull, + 989339620251222318ull, 16387139209673312472ull, 9929646806074584996ull, 3431702464375352019ull, + 7109110581138159243ull, 15876768600806209784ull, 2055807417991868731ull, 2967240684959102974ull, + 16861027235162456005ull, 15060086465613936099ull, 15108105020256312702ull, 4798571550112838045ull, + 12147577628520154821ull, 2672253757378624433ull, 11012398248118034729ull, 3793698063996891052ull, + 14192380692230185360ull, 9929646806074584996ull, 3431702464375352019ull, 4876795174170569834ull, + 6415223491846048288ull, 15440203674283382254ull, 15770939448601912590ull, 10303879746669770948ull, + 9129508526108090506ull, 7566812634646466639ull, 9342513139847534316ull, 2583212593423889265ull, + 17394302303999227727ull, 4862824118059625938ull, 488573764319528998ull, 2182726028422319597ull, + 9929646806074584996ull, 3431702464375352019ull, 10985520577104501717ull, 1722620875866364753ull, + 1330953375236028975ull, 13016771554077924109ull, 1957243932866248837ull, 17505535194597138553ull, + 3454710910137292429ull, 2588728158773447268ull, 3896815640332977269ull, 1242432642703862772ull, + 5759726139356603685ull, 11360709492458056231ull, 11322820717063119955ull, 9929646806074584996ull, + 3431702464375352019ull, 8753205170136912308ull, 3266260137831309351ull, 14317651032660366710ull, + 2595696204080302369ull, 5005019107248970821ull, 18056244996701624227ull, 16671148369694216353ull, + 17192890128476992110ull, 3372374369492008463ull, 14874634228248935661ull, 17186942698938417100ull, + 9585607679502620965ull, 5330611514576973503ull, 9929646806074584996ull, 3431702464375352019ull, + 15450151391039680535ull, 6706695279434477772ull, 3898973846032338714ull, 16426222375228309526ull, + 3298583537716228165ull, 12780842357225884671ull, 17868433793332746134ull, 850620973344744465ull, + 13307433097827310109ull, 1625492676009976678ull, 693113809262650154ull, 10370646549992178351ull, + 11915819701152280554ull, 9929646806074584996ull, 3431702464375352019ull, 13217835984072091126ull, + 16241106754843188000ull, 16159215073436017114ull, 2473238957724558926ull, 12686955773096649317ull, + 15453275012172611147ull, 4555685926564653262ull, 17461223914346450926ull, 14905649388478306090ull, + 8456041017007585703ull, 8300615010208981475ull, 9250724125369577775ull, 6851628177429024696ull, + 9929646806074584996ull, 3431702464375352019ull, 2056258949234144081ull, 17960188792256126185ull, + 6248027919678976759ull, 5884554688825838845ull, 6587708148855725669ull, 18072969703117911665ull, + 10627671335969334205ull, 16829596849950769704ull, 625195922224587841ull, 1089939125976698227ull, + 13051273475920467674ull, 7820920359258290212ull, 15866132849546632615ull, 9929646806074584996ull, + 3431702464375352019ull, 18270687615976106288ull, 5508666052666974171ull, 10218458133261192228ull, + 10866495873235391672ull, 1179815459809367173ull, 7663668949466309574ull, 8216161350835101624ull, + 5361981758203061759ull, 8650916081886692640ull, 15316599184646277809ull, 15322639613652786606ull, + 14871655760800376997ull, 7215206224531212151ull, 9929646806074584996ull, 3431702464375352019ull, + 6520889763169322899ull, 3841957037555937838ull, 10665174528894959951ull, 10368136919060946099ull, + 7721277914738202692ull, 2754624916616902265ull, 15837409298092517746ull, 18078560969645441914ull, + 10724340601660215611ull, 14835948283450207307ull, 6759621847658594698ull, 6706413662153930404ull, + 10460725108710611029ull, 9929646806074584996ull, 3431702464375352019ull, 4288574356201733490ull, + 8763328487769941515ull, 700732781097283655ull, 16959966825370378712ull, 13017126584466384005ull, + 16617980689532025078ull, 7682040416437574040ull, 11791357319445204128ull, 11427135332308542094ull, + 4893461070494745476ull, 10442094867997493034ull, 2235630609106157229ull, 10911120786643641041ull, + 9929646806074584996ull, 9808874869469701221ull, 12161962213042174405ull, 12197969927824375845ull, + 10823637164633491237ull, 17320409490924411941ull, 9345366988845399973ull, 17320409490924411941ull, + 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, + 12197969927824375845ull, 13380826962402805797ull, 13380826962402805797ull, 12161962213042174405ull, + 9808874869469701221ull, 12161962213042174405ull, 12197969927824375845ull, 10823637164633491237ull, + 17320409490924411941ull, 9345366988845399973ull, 17320409490924411941ull, 12197969927824375845ull, + 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, + 13380826962402805797ull, 13380826962402805797ull, 12161962213042174405ull, 9808874869469701221ull, + 12161962213042174405ull, 12197969927824375845ull, 10823637164633491237ull, 17320409490924411941ull, + 9345366988845399973ull, 17320409490924411941ull, 12197969927824375845ull, 12197969927824375845ull, + 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, 13380826962402805797ull, + 13380826962402805797ull, 12161962213042174405ull, 9808874869469701221ull, 12161962213042174405ull, + 12197969927824375845ull, 10823637164633491237ull, 17320409490924411941ull, 9345366988845399973ull, + 17320409490924411941ull, 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, + 12197969927824375845ull, 12197969927824375845ull, 13380826962402805797ull, 13380826962402805797ull, + 12161962213042174405ull, 9808874869469701221ull, 12161962213042174405ull, 12197969927824375845ull, + 10823637164633491237ull, 17320409490924411941ull, 9345366988845399973ull, 17320409490924411941ull, + 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, 12197969927824375845ull, + 12197969927824375845ull, 13380826962402805797ull, 13380826962402805797ull, 12161962213042174405ull, + 10245844859216114261ull, 16038372209008516879ull, 4254646028709477699ull, 12300074492682826684ull, + 15908467192642411976ull, 7950156206358012996ull, 15043164804985601647ull, 12783314913722373385ull, + 16792642165047377928ull, 2238281901097204143ull, 8711620613655762925ull, 17417142276522077386ull, + 3014872817714212652ull, 12635679875810846655ull, 9929646806074584996ull, 10245844859216114261ull, + 13806056802040927470ull, 4003541699522263875ull, 16596867321842826449ull, 16664792748624813960ull, + 14132989249505005636ull, 15410644256742794287ull, 16777549432581831049ull, 11014337354272729423ull, + 12575510482750747585ull, 6937872996993320494ull, 7873369128419528160ull, 18285935111577747887ull, + 1559960375268139812ull, 9929646806074584996ull, 10245844859216114261ull, 2644479767202980425ull, + 6459089341819708778ull, 966653120609482452ull, 12416900401777903066ull, 9831653568391621828ull, + 8973702348826892436ull, 5808588642145717979ull, 2773856019070395392ull, 17661806233274843864ull, + 9496146201909608876ull, 7147922039555270068ull, 2109262044873822894ull, 5643732296656521371ull, + 9929646806074584996ull, 10245844859216114261ull, 412164360235391016ull, 16893053361661322175ull, + 17385172586960186106ull, 6465398564150572945ull, 598852157427535461ull, 16074826988892074891ull, + 13970266224040560593ull, 11784735133004810318ull, 953226739955908145ull, 8969976679625886228ull, + 12029653552559363627ull, 989339620251222318ull, 4301440202830369560ull, 9929646806074584996ull, + 10245844859216114261ull, 7109110581138159243ull, 8026734438605766752ull, 4253932397883257457ull, + 17311852144259772975ull, 5138707742610939557ull, 17047145035282307114ull, 18252905126938170543ull, + 1710985378236454104ull, 7644606896570782349ull, 10731322431763316666ull, 14602262179685641903ull, + 3793698063996891052ull, 5491017416662145418ull, 9929646806074584996ull, 10245844859216114261ull, + 4876795174170569834ull, 10918450909810791462ull, 48966951802344970ull, 13365923898320945666ull, + 3434363327021900389ull, 17586450556037117889ull, 8571847369069111298ull, 11164984465534005783ull, + 14626279846032423183ull, 3623580842157849513ull, 1388843580232209521ull, 488573764319528998ull, + 17448373154710829308ull, 9929646806074584996ull, 10245844859216114261ull, 10985520577104501717ull, + 13361911375096022130ull, 3869393120385085659ull, 17423722436464472587ull, 2291627512304891012ull, + 16609454694013617821ull, 10390844718655603850ull, 6673988922889799768ull, 14454933237410568547ull, + 13555544843919144400ull, 2836989055438452971ull, 11360709492458056231ull, 4349808880239301807ull, + 9929646806074584996ull, 10245844859216114261ull, 8753205170136912308ull, 16743760734573427305ull, + 8250350423723034597ull, 7283222790831149629ull, 8209942966163734597ull, 12079245050547410545ull, + 3005884804184556477ull, 16381422990939436297ull, 8220394426965175085ull, 8353520243613371096ull, + 12601865447941974588ull, 9585607679502620965ull, 4910536422200807894ull, 9929646806074584996ull, + 10245844859216114261ull, 15450151391039680535ull, 5792676497014479932ull, 6387556661574179130ull, + 17287773342443167575ull, 12841592598198050404ull, 9811631404770394255ull, 17904296496655120086ull, + 5334660256499167901ull, 4039662765688679551ull, 17480536656621705838ull, 9077829344096662092ull, + 10370646549992178351ull, 119092456390158968ull, 9929646806074584996ull, 10245844859216114261ull, + 13217835984072091126ull, 9466095989884506182ull, 8791450861578970234ull, 9817995344206357603ull, + 9072342392419849349ull, 12191654521315667328ull, 3079068229028071011ull, 152737164443752339ull, + 12700053295334055845ull, 2337558397364801863ull, 1930178646267556081ull, 9250724125369577775ull, + 4299520301997249611ull, 9929646806074584996ull, 10245844859216114261ull, 2056258949234144081ull, + 9022985483432757627ull, 3158147517679243999ull, 6000598513667501048ull, 12164788277490775524ull, + 17133287433839438439ull, 673956482863793273ull, 8623484103056217706ull, 6389809404065816401ull, + 3552530134142743990ull, 6533023279035251917ull, 7820920359258290212ull, 3029967279686905816ull, + 9929646806074584996ull, 10245844859216114261ull, 18270687615976106288ull, 10717843613671707276ull, + 8527280534105252101ull, 16616641737246663671ull, 7400675049835196485ull, 5608064026514283998ull, + 4607330859784478583ull, 3170994706702534159ull, 18256622923546096607ull, 11251980000933183315ull, + 35980232621132739ull, 14871655760800376997ull, 1224982096426648260ull, 9929646806074584996ull, + 10245844859216114261ull, 6520889763169322899ull, 18187976246647208771ull, 6544886890507334915ull, + 2115815003454214600ull, 18276728862442793029ull, 18158979233548083054ull, 7547212739353040328ull, + 9738531155519268778ull, 9973249718744891033ull, 3380497772055080907ull, 3318387629497154296ull, + 6706413662153930404ull, 3011639468714315709ull, 9929646806074584996ull, 10245844859216114261ull, + 4288574356201733490ull, 14296329261825483449ull, 8318391134491374838ull, 2506972485852023996ull, + 12577350250469416068ull, 3720578441542291297ull, 17341391028088200381ull, 4205809385229478131ull, + 9455589803458283425ull, 3347090296395901195ull, 12840983972433303321ull, 2235630609106157229ull, + 4979009263625600118ull, 9929646806074584996ull, 10906065300532129270ull, 12161962213042174405ull, + 4459598103981495085ull, 9207947867349445348ull, 8019421683980492692ull, 7027475233620407300ull, + 7077685313696407581ull, 5534538284821761493ull, 8836197566063881670ull, 10016581987595975395ull, + 7427358572612232098ull, 8223521773421637255ull, 12328143194273367844ull, 4533810997039206025ull, + 9929646806074584996ull, 10906065300532129270ull, 9929646806074584996ull, 17891168194564123014ull, + 4160904766410350119ull, 8474724122642769410ull, 8515635158629812900ull, 12109342475119015584ull, + 8713919464803087427ull, 17108399248271322102ull, 10170291737768683638ull, 7502061669052043513ull, + 15647068379399201337ull, 16192782323258333735ull, 18185152579243956680ull, 9929646806074584996ull, + 10906065300532129270ull, 16626593026977353223ull, 5116688293803162683ull, 12998163066859041943ull, + 12371932470324220152ull, 10814168203011406917ull, 9604240991739613990ull, 5292461005704548088ull, + 2116880173682349193ull, 7287681621341525323ull, 15358372066348372456ull, 6833296685369735530ull, + 15202719380792455855ull, 17586389509072535890ull, 9929646806074584996ull, 10906065300532129270ull, + 14394277620009763814ull, 6572638171534480096ull, 5239686895372129413ull, 17445091950031255087ull, + 2498011192851537988ull, 7384066550948509035ull, 762549597265980846ull, 1854557634220774704ull, + 18172331339239140220ull, 14052569221893701052ull, 12597429044243204272ull, 12260904537780205221ull, + 10672413254672234391ull, 9929646806074584996ull, 10906065300532129270ull, 3232700585171816769ull, + 5172379653384660016ull, 2796027124317499372ull, 18063170839876232174ull, 11349200814308868772ull, + 15660606307652611578ull, 5239293564670470255ull, 13929780168506220530ull, 11370508876060845074ull, + 2923190992572625715ull, 12747294915904714451ull, 14934276586463937070ull, 15084332468920295324ull, + 9929646806074584996ull, 10906065300532129270ull, 1000385178204227360ull, 9942873459550220568ull, + 2532536929896784885ull, 3786164554705898655ull, 16283625318780699172ull, 14415033135274681379ull, + 7350162293479816414ull, 11524209855056886731ull, 991887806978362128ull, 3108217720964037117ull, + 12538881514310806246ull, 6706413662153930404ull, 637690377021991326ull, 9929646806074584996ull, +}; diff --git a/crypto/math-cuda/tests/host_kat/gkr_host_kat.cpp b/crypto/math-cuda/tests/host_kat/gkr_host_kat.cpp new file mode 100644 index 000000000..55f27f91b --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/gkr_host_kat.cpp @@ -0,0 +1,581 @@ +// Known-answer tests for the GKR layer rounds over an input layer kept on the +// card without its padding (`kernels/sumcheck.cu`: `gkr_round_gruen_real`, +// `gkr_gruen_finish_real`; `LAMBDA_VM_ARGUE_INPUT_REAL`), run on the host +// through `cuda_host_simt_shim.h` — every thread of a block a real thread, so +// the kernels' block reductions run as written. No GPU, no nvcc. +// +// WHAT IT COVERS, for every shape (the layer's `m`, where its padding starts, +// the cube left to the host, which rounds sum `H(0)`): +// 1. TODAY'S KERNELS UNCHANGED: `gkr_round_gruen` and `gkr_gruen_finish` as +// shipped against a copy of the source they had before the kept layer +// (below), over the whole layer: every round's block partials and the +// folded halves, RAW-identical. +// 2. THE KEPT LAYER: the upper halves stored only to where the padding +// starts, poison past that, rounds launched the way the driver launches +// them (`math_cuda::gkr::GruenLayer`: the kept kernel while the stored run +// is shorter than the cube, today's after) and, on the smaller shapes, +// the kept kernel every round — against today's kernels over the whole +// layer: every round's block partials, the halves' stored cells after +// every round, the finish's output, RAW-identical; the poison never read +// (a read would move a sum) and never written. +// 3. A WRITTEN-OUT REFERENCE: the sums `H(1)`, `H(2)`, `H(0)` and the +// finish's halves computed here from the whole layer — its own fold, its +// own `eq(u_{>j}, x)` as a product over the bits of `x` — EQUAL to the +// kernels' (canonical values). +// 4. Today's path keeps the padding as the raw limbs `0/1`: after every fold +// the whole layer's cells past the stored run are those limbs, which is +// what makes the kept kernel's constants raw-identical. +// +// WHAT IT DOES NOT COVER, and the card tests still own: whether nvcc accepts +// the file, the hardware's warps and memory, launch shapes from the driver +// (`gkr_gruen_truncated.rs`, `argue_gkr_truncated.rs` on a box). +// +// Build and run with `make test-gkr-host-kat`. + +#include +#include +#include +#include +#include + +#include "cuda_host_simt_shim.h" + +// The kernels' block-shared buffer: blocks run one at a time here, so one +// buffer serves every block. `extern "C"` because the kernels that name it +// are. The launch bounds mean nothing on the host. +#define __shared__ +#define __launch_bounds__(...) +extern "C" { +uint64_t shared[3 * 1024]; +} + +#include "sumcheck.cu" + +// ── today's two kernels, as they were before the kept layer ───────────────── +// Copied from `sumcheck.cu` at the commit this change starts from, renamed. +// Item 1 runs them beside the shipped ones. +void gkr_round_gruen_before( + uint64_t *__restrict__ p, uint64_t *__restrict__ q, uint64_t half, uint64_t quarter, + uint32_t fold, uint64_t s0, uint64_t s1, uint64_t s2, uint64_t l0, uint64_t l1, uint64_t l2, + const uint64_t *__restrict__ e_hi, const uint64_t *__restrict__ e_lo, uint32_t low, + uint32_t want_h0, uint64_t *__restrict__ d_partials) { + Fe3 s = ext3::make(s0, s1, s2); + Fe3 lambda = ext3::make(l0, l1, l2); + uint64_t *factors[4] = {p, p + half * 3, q, q + half * 3}; + uint64_t mask = ((uint64_t)1 << low) - 1; + Fe3 acc1 = ext3::zero(), acc2 = ext3::zero(), acc0 = ext3::zero(); + for (uint64_t x = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; x < quarter; + x += (uint64_t)gridDim.x * blockDim.x) { + Fe3 lo[4], hi[4]; +#pragma unroll + for (int k = 0; k < 4; k++) { + uint64_t *f = factors[k]; + if (fold) { + Fe3 a = load_ext(f + x * 3); + Fe3 b = load_ext(f + (x + quarter) * 3); + Fe3 c = load_ext(f + (x + 2 * quarter) * 3); + Fe3 d = load_ext(f + (x + 3 * quarter) * 3); + lo[k] = ext3::add(a, ext3::mul(s, ext3::sub(c, a))); + hi[k] = ext3::add(b, ext3::mul(s, ext3::sub(d, b))); + f[x * 3 + 0] = lo[k].a; + f[x * 3 + 1] = lo[k].b; + f[x * 3 + 2] = lo[k].c; + f[(x + quarter) * 3 + 0] = hi[k].a; + f[(x + quarter) * 3 + 1] = hi[k].b; + f[(x + quarter) * 3 + 2] = hi[k].c; + } else { + lo[k] = load_ext(f + x * 3); + hi[k] = load_ext(f + (x + quarter) * 3); + } + } + Fe3 w = ext3::mul(load_ext(e_hi + (x >> low) * 3), load_ext(e_lo + (x & mask) * 3)); + acc1 = ext3::add(acc1, ext3::mul(w, gkr_h(hi[0], hi[1], hi[2], hi[3], lambda))); + Fe3 two[4]; +#pragma unroll + for (int k = 0; k < 4; k++) { + two[k] = ext3::sub(ext3::add(hi[k], hi[k]), lo[k]); + } + acc2 = ext3::add(acc2, ext3::mul(w, gkr_h(two[0], two[1], two[2], two[3], lambda))); + if (want_h0) { + acc0 = ext3::add(acc0, ext3::mul(w, gkr_h(lo[0], lo[1], lo[2], lo[3], lambda))); + } + } + + extern __shared__ uint64_t shared[]; + uint32_t rows = want_h0 ? 3 : 2; + for (uint32_t row = 0; row < rows; row++) { + Fe3 mine = row == 0 ? acc1 : row == 1 ? acc2 : acc0; + shared[threadIdx.x * 3 + 0] = mine.a; + shared[threadIdx.x * 3 + 1] = mine.b; + shared[threadIdx.x * 3 + 2] = mine.c; + __syncthreads(); + for (uint32_t width = blockDim.x / 2; width > 0; width >>= 1) { + if (threadIdx.x < width) { + Fe3 sum = ext3::add(load_ext(shared + threadIdx.x * 3), + load_ext(shared + (threadIdx.x + width) * 3)); + shared[threadIdx.x * 3 + 0] = sum.a; + shared[threadIdx.x * 3 + 1] = sum.b; + shared[threadIdx.x * 3 + 2] = sum.c; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + uint64_t at = ((uint64_t)row * gridDim.x + blockIdx.x) * 3; + d_partials[at + 0] = shared[0]; + d_partials[at + 1] = shared[1]; + d_partials[at + 2] = shared[2]; + } + __syncthreads(); + } +} + +// The rounds' end: each half bound to the last challenge `s` (with `fold`) from its `2·cells` cube +// down to `cells`, written in place and to `out[(k·cells + y)·3]`, halves in the order `p_lo, p_hi, +// q_lo, q_hi` — what the host tail starts from. Thread `y` reads cells `y` and `y + cells` and +// writes cell `y` only. +void gkr_gruen_finish_before(uint64_t *__restrict__ p, uint64_t *__restrict__ q, + uint64_t half, uint64_t cells, uint32_t fold, + uint64_t s0, uint64_t s1, uint64_t s2, + uint64_t *__restrict__ out) { + Fe3 s = ext3::make(s0, s1, s2); + uint64_t total = 4 * cells; + for (uint64_t task = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; task < total; + task += (uint64_t)gridDim.x * blockDim.x) { + uint64_t k = task / cells; + uint64_t y = task - k * cells; + uint64_t *f = (k < 2 ? p : q) + (k & 1) * half * 3; + Fe3 v = load_ext(f + y * 3); + if (fold) { + Fe3 hi = load_ext(f + (y + cells) * 3); + v = ext3::add(v, ext3::mul(s, ext3::sub(hi, v))); + f[y * 3 + 0] = v.a; + f[y * 3 + 1] = v.b; + f[y * 3 + 2] = v.c; + } + uint64_t *at = out + task * 3; + at[0] = v.a; + at[1] = v.b; + at[2] = v.c; + } +} + +namespace { + +int failures = 0; + +void check(bool ok, const std::string &what) { + if (!ok) { + printf("FAIL: %s\n", what.c_str()); + ++failures; + } +} + +const uint64_t P = 0xFFFFFFFF00000001ull; + +uint64_t state = 0x243F6A8885A308D3ull; +uint64_t next_u64() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; +} + +// A raw limb: canonical most of the time, and every eighth one a +// non-canonical representative in [p, 2^64) — what a card may hold. +uint64_t raw_limb() { + uint64_t v = next_u64() % P; + if ((next_u64() & 7) == 0 && v < 0xFFFFFFFFull) v += P; + return v; +} + +Fe3 random_ext() { return ext3::make(raw_limb(), raw_limb(), raw_limb()); } + +bool same_value(const Fe3 &a, const Fe3 &b) { + Fe3 x = ext3::canonical(a), y = ext3::canonical(b); + return x.a == y.a && x.b == y.b && x.c == y.c; +} + +Fe3 at(const std::vector &v, uint64_t i) { + return ext3::make(v[i * 3], v[i * 3 + 1], v[i * 3 + 2]); +} + +void put(std::vector &v, uint64_t i, const Fe3 &x) { + v[i * 3] = x.a; + v[i * 3 + 1] = x.b; + v[i * 3 + 2] = x.c; +} + +// `math_cuda::gkr::gruen_shape`. +void gruen_shape(uint64_t quarter, unsigned &grid, unsigned &block) { + uint64_t b = 1; + while (b < quarter) b <<= 1; + block = (unsigned)std::min(std::max(b, 32), 256); + grid = (unsigned)std::min(std::max((quarter + block - 1) / block, 1), 2048); +} + +// A poison cell: no value a kernel computes from the layer, and never `0/1`. +const uint64_t POISON = 0xDEAD5EED0BADF00Dull; + +struct Shape { + unsigned m; // the halves' variables: each half `2^m` cells + uint64_t stored; // the upper halves' real cells, (0, 2^m] + unsigned low; // at most this many variables left to the host + int h0; // which rounds sum H(0): 0 none, 1 every one, 2 the odd ones + bool every_round; // also run the kept kernel on every round + bool before; // also run the copy of today's kernels +}; + +// One whole layer and its kept copy, and every round over both. +struct Run { + Shape s; + uint64_t half, rounds, low; + std::vector point, e; // u (m ext3), the eq levels as the kernel lays them out + Fe3 lambda; + std::vector challenges; // one a round + // Buffers: p and q, each the whole layer's 2·half cells. + std::vector wp, wq; // whole, today's kernels + std::vector bp, bq; // whole, the copy of today's kernels + std::vector kp, kq; // kept, the driver's choice of kernel + std::vector ap, aq; // kept, the kept kernel every round + std::vector ref[4]; // the reference: p_lo, p_hi, q_lo, q_hi, whole +}; + +std::string label(const Shape &s) { + char b[160]; + snprintf(b, sizeof b, "m %u stored %llu low %u h0 %d", s.m, (unsigned long long)s.stored, s.low, + s.h0); + return b; +} + +bool want_h0(const Shape &s, uint64_t j) { return s.h0 == 1 || (s.h0 == 2 && (j & 1)); } + +// `eq(u_{j+1..m−1}, x)` written out: a product over the bits of `x`, variable +// `j + 1` in the high bit. +Fe3 eq_weight(const Run &r, uint64_t j, uint64_t x) { + uint64_t bits = r.s.m - j - 1; + Fe3 acc = ext3::one(); + for (uint64_t i = 0; i < bits; ++i) { + Fe3 u = at(r.point, j + 1 + i); + bool set = (x >> (bits - 1 - i)) & 1; + acc = ext3::mul(acc, set ? u : ext3::sub(ext3::one(), u)); + } + return acc; +} + +Fe3 h_of(const Fe3 v[4], const Fe3 &lambda) { + return ext3::add(ext3::add(ext3::mul(v[0], v[3]), ext3::mul(v[1], v[2])), + ext3::mul(lambda, ext3::mul(v[2], v[3]))); +} + +enum Kernel { TODAY, BEFORE, KEPT }; + +// One round's launch over one pair of buffers; returns its block partials. +std::vector launch_round(Run &r, Kernel kernel, uint64_t *p, uint64_t *q, uint64_t j, + uint64_t stored) { + uint64_t quarter = r.half >> (j + 1); + unsigned grid, block; + gruen_shape(quarter, grid, block); + uint32_t fold = j > 0; + Fe3 s = j > 0 ? r.challenges[j - 1] : ext3::zero(); + uint64_t at_level = (1ull << r.low) + (1ull << r.rounds) - (1ull << (r.rounds - j)); + const uint64_t *e_lo = r.e.data(); + const uint64_t *e_hi = r.e.data() + at_level * 3; + uint32_t want = want_h0(r.s, j); + std::vector partials(3 * grid * 3, 0); + uint64_t half = r.half; + uint32_t low = (uint32_t)r.low; + Fe3 l = r.lambda; + simt_launch(grid, block, [&] { + if (kernel == TODAY) + gkr_round_gruen(p, q, half, quarter, fold, s.a, s.b, s.c, l.a, l.b, l.c, e_hi, e_lo, low, + want, partials.data()); + else if (kernel == BEFORE) + gkr_round_gruen_before(p, q, half, quarter, fold, s.a, s.b, s.c, l.a, l.b, l.c, e_hi, + e_lo, low, want, partials.data()); + else + gkr_round_gruen_real(p, q, half, quarter, fold, s.a, s.b, s.c, l.a, l.b, l.c, e_hi, + e_lo, low, want, stored, partials.data()); + }); + partials.resize((want ? 3 : 2) * grid * 3); + return partials; +} + +std::vector launch_finish(Run &r, Kernel kernel, uint64_t *p, uint64_t *q, + uint64_t stored) { + uint64_t cells = 1ull << r.low; + uint32_t fold = r.rounds > 0; + Fe3 s = fold ? r.challenges[r.rounds - 1] : ext3::zero(); + std::vector out(4 * cells * 3, POISON); + uint64_t half = r.half; + simt_launch(1, 32, [&] { + if (kernel == TODAY) + gkr_gruen_finish(p, q, half, cells, fold, s.a, s.b, s.c, out.data()); + else if (kernel == BEFORE) + gkr_gruen_finish_before(p, q, half, cells, fold, s.a, s.b, s.c, out.data()); + else + gkr_gruen_finish_real(p, q, half, cells, fold, s.a, s.b, s.c, stored, out.data()); + }); + return out; +} + +// The driver's choice (`GruenLayer::round_enqueue`, `finish`): the kept kernel +// while the stored run is shorter than the cube the halves hold. +bool driver_keeps(uint64_t stored, uint64_t cube) { return stored < cube; } + +// `math_cuda::gkr::stored_after_round`. +uint64_t stored_after_round(uint64_t stored, uint64_t quarter) { + return std::min(stored, 2 * quarter); +} + +// The reference's sums at round `j`, after folding its whole layer by the +// last challenge. +void reference_round(Run &r, uint64_t j, Fe3 sums[3]) { + uint64_t quarter = r.half >> (j + 1); + if (j > 0) { + Fe3 s = r.challenges[j - 1]; + for (auto &g : r.ref) { + for (uint64_t x = 0; x < 2 * quarter; ++x) + g[x] = ext3::add(g[x], ext3::mul(s, ext3::sub(g[x + 2 * quarter], g[x]))); + g.resize(2 * quarter); + } + } + for (int t = 0; t < 3; ++t) sums[t] = ext3::zero(); + for (uint64_t x = 0; x < quarter; ++x) { + Fe3 lo[4], hi[4], two[4]; + for (int k = 0; k < 4; ++k) { + lo[k] = r.ref[k][x]; + hi[k] = r.ref[k][x + quarter]; + two[k] = ext3::sub(ext3::add(hi[k], hi[k]), lo[k]); + } + Fe3 w = eq_weight(r, j, x); + sums[0] = ext3::add(sums[0], ext3::mul(w, h_of(hi, r.lambda))); + sums[1] = ext3::add(sums[1], ext3::mul(w, h_of(two, r.lambda))); + sums[2] = ext3::add(sums[2], ext3::mul(w, h_of(lo, r.lambda))); + } +} + +// The cells of factor `k` (0 p_lo, 1 p_hi, 2 q_lo, 3 q_hi) at `[0, n)`. +const uint64_t *factor(const std::vector &p, const std::vector &q, int k, + uint64_t half) { + return (k < 2 ? p.data() : q.data()) + (k & 1) * half * 3; +} + +bool same_raw(const uint64_t *a, const uint64_t *b, uint64_t cells) { + return memcmp(a, b, cells * 3 * sizeof(uint64_t)) == 0; +} + +// The kept buffers' cells past the real ones still hold the poison. +bool poison_intact(const std::vector &v, uint64_t half, uint64_t stored) { + for (uint64_t i = (half + stored) * 3; i < v.size(); ++i) + if (v[i] != POISON) return false; + return true; +} + +uint64_t layers_run = 0, rounds_run = 0, kept_rounds = 0, today_rounds_on_kept = 0; + +void run_shape(const Shape &shape) { + Run r; + r.s = shape; + r.half = 1ull << shape.m; + r.low = std::min(shape.low, shape.m); + r.rounds = shape.m - r.low; + const std::string what = label(shape); + const uint64_t half = r.half, full = 2 * half; + + // The layer: random fractions, the padding written out as today writes it. + r.wp.resize(full * 3); + r.wq.resize(full * 3); + for (uint64_t i = 0; i < full; ++i) { + bool padding = i >= half + shape.stored; + put(r.wp, i, padding ? ext3::zero() : random_ext()); + put(r.wq, i, padding ? ext3::one() : random_ext()); + } + r.bp = r.wp; + r.bq = r.wq; + // The kept copies: the real cells, poison where the padding was. + r.kp.assign(full * 3, POISON); + r.kq.assign(full * 3, POISON); + std::copy(r.wp.begin(), r.wp.begin() + (half + shape.stored) * 3, r.kp.begin()); + std::copy(r.wq.begin(), r.wq.begin() + (half + shape.stored) * 3, r.kq.begin()); + r.ap = r.kp; + r.aq = r.kq; + for (int k = 0; k < 4; ++k) { + const uint64_t *f = factor(r.wp, r.wq, k, half); + r.ref[k].resize(half); + for (uint64_t x = 0; x < half; ++x) r.ref[k][x] = ext3::make(f[x * 3], f[x * 3 + 1], f[x * 3 + 2]); + } + r.point.resize(shape.m * 3); + for (unsigned i = 0; i < shape.m; ++i) put(r.point, i, random_ext()); + r.lambda = random_ext(); + for (uint64_t j = 0; j < r.rounds; ++j) r.challenges.push_back(random_ext()); + + // The eq levels, by the kernel the driver launches. + uint64_t cells = (1ull << r.low) + (1ull << r.rounds) - 1; + r.e.assign(cells * 3, 0); + { + uint32_t low = (uint32_t)r.low, rounds = (uint32_t)r.rounds; + const uint64_t *pt = r.point.empty() ? nullptr : r.point.data(); + std::vector zero(3, 0); + if (!pt) pt = zero.data(); + simt_launch(1, 32, [&] { gkr_eq_levels_ext3(pt, low, rounds, r.e.data()); }); + } + + uint64_t stored_k = shape.stored, stored_a = shape.stored; + for (uint64_t j = 0; j < r.rounds; ++j) { + const std::string rw = what + " round " + std::to_string(j); + uint64_t quarter = half >> (j + 1); + uint64_t cube = (j > 0 ? 4 : 2) * quarter; + std::vector today = launch_round(r, TODAY, r.wp.data(), r.wq.data(), j, 0); + if (shape.before) { + std::vector before = launch_round(r, BEFORE, r.bp.data(), r.bq.data(), j, 0); + check(before == today, rw + ": today's round = the copy of it from before, raw"); + check(r.bp == r.wp && r.bq == r.wq, rw + ": today's folded halves = the copy's, raw"); + } + bool keeps = driver_keeps(stored_k, cube); + kept_rounds += keeps; + today_rounds_on_kept += !keeps; + std::vector kept = + launch_round(r, keeps ? KEPT : TODAY, r.kp.data(), r.kq.data(), j, stored_k); + check(kept == today, rw + ": the kept layer's partials = the whole layer's, raw"); + if (shape.every_round) { + std::vector always = launch_round(r, KEPT, r.ap.data(), r.aq.data(), j, stored_a); + check(always == today, rw + ": the kept kernel's partials = today's, raw (every round)"); + } + // The reference. + Fe3 want[3]; + reference_round(r, j, want); + unsigned grid, block; + gruen_shape(quarter, grid, block); + int rows = want_h0(shape, j) ? 3 : 2; + for (int row = 0; row < rows; ++row) { + Fe3 got = ext3::zero(); + for (unsigned b = 0; b < grid; ++b) got = ext3::add(got, at(today, row * grid + b)); + check(same_value(got, want[row]), + rw + ": H(" + std::to_string(row == 0 ? 1 : row == 1 ? 2 : 0) + ") = the reference's"); + } + // The halves after the round: the live half is `2·quarter`. + uint64_t next_k = stored_after_round(stored_k, quarter); + uint64_t next_a = stored_after_round(stored_a, quarter); + for (int k = 0; k < 4; ++k) { + bool upper = k & 1; + uint64_t live = 2 * quarter; + uint64_t n = upper ? std::min(next_k, live) : live; + check(same_raw(factor(r.kp, r.kq, k, half), factor(r.wp, r.wq, k, half), n), + rw + ": factor " + std::to_string(k) + "'s stored cells = the whole layer's, raw"); + if (shape.every_round) { + uint64_t na = upper ? std::min(next_a, live) : live; + check(same_raw(factor(r.ap, r.aq, k, half), factor(r.wp, r.wq, k, half), na), + rw + ": factor " + std::to_string(k) + "'s stored cells = the whole layer's, raw (every round)"); + } + if (upper) { + // Item 4: the whole layer's padding is still the raw `0/1`. + const uint64_t *f = factor(r.wp, r.wq, k, half); + bool limbs = true; + for (uint64_t x = next_k; x < live; ++x) + limbs &= f[x * 3] == (k == 3 ? 1u : 0u) && f[x * 3 + 1] == 0 && f[x * 3 + 2] == 0; + check(limbs, rw + ": factor " + std::to_string(k) + "'s padding is the raw 0/1 after the fold"); + } + } + check(poison_intact(r.kp, half, shape.stored) && poison_intact(r.kq, half, shape.stored), + rw + ": nothing written past the real cells"); + if (shape.every_round) + check(poison_intact(r.ap, half, shape.stored) && poison_intact(r.aq, half, shape.stored), + rw + ": nothing written past the real cells (every round)"); + stored_k = next_k; + stored_a = next_a; + ++rounds_run; + } + + // The finish. + { + const std::string rw = what + " finish"; + uint64_t cells_l = 1ull << r.low; + uint64_t cube = r.rounds > 0 ? 2 * cells_l : cells_l; + std::vector today = launch_finish(r, TODAY, r.wp.data(), r.wq.data(), 0); + if (shape.before) { + std::vector before = launch_finish(r, BEFORE, r.bp.data(), r.bq.data(), 0); + check(before == today && r.bp == r.wp && r.bq == r.wq, + rw + ": today's finish = the copy of it from before, raw"); + } + bool keeps = driver_keeps(stored_k, cube); + std::vector kept = + launch_finish(r, keeps ? KEPT : TODAY, r.kp.data(), r.kq.data(), stored_k); + check(kept == today, rw + ": the kept layer's halves = the whole layer's, raw"); + if (shape.every_round) { + std::vector always = launch_finish(r, KEPT, r.ap.data(), r.aq.data(), stored_a); + check(always == today, rw + ": the kept kernel's halves = today's, raw (every round)"); + } + // The reference's halves. + bool same = true; + for (int k = 0; k < 4; ++k) { + for (uint64_t y = 0; y < cells_l; ++y) { + Fe3 v = r.ref[k][y]; + if (r.rounds > 0) { + Fe3 s = r.challenges[r.rounds - 1]; + v = ext3::add(v, ext3::mul(s, ext3::sub(r.ref[k][y + cells_l], v))); + } + same &= same_value(at(today, k * cells_l + y), v); + } + } + check(same, rw + ": the halves = the reference's"); + uint64_t n = std::min(stored_k, cells_l); + for (int k = 0; k < 4; ++k) { + uint64_t m = (k & 1) ? n : cells_l; + check(same_raw(factor(r.kp, r.kq, k, half), factor(r.wp, r.wq, k, half), m), + rw + ": factor " + std::to_string(k) + "'s stored cells = the whole layer's, raw"); + } + check(poison_intact(r.kp, half, shape.stored) && poison_intact(r.kq, half, shape.stored), + rw + ": nothing written past the real cells"); + } + ++layers_run; +} + +} // namespace + +int main() { + std::vector shapes; + // Every split of card rounds and host tail, and every H(0) mode, on small + // layers, with the padding's start at both ends and between. + for (unsigned m = 1; m <= 6; ++m) { + uint64_t half = 1ull << m; + for (uint64_t stored : {uint64_t(1), half / 2 + 1, half - 1, half}) { + if (stored == 0 || stored > half) continue; + for (unsigned low : {0u, 1u, 2u, m}) { + for (int h0 : {0, 1, 2}) shapes.push_back({m, stored, low, h0, true, true}); + } + } + } + // m 7..14: the padding one cell in, one cell before the end, none, a CPU + // table's (20 of 32 slots) and a Keccak-round table's (1,031 of 2,048) — + // each a run of whole rows. From m 12 on, where a layer is thousands of + // host threads a round, the two production shapes alone. + const unsigned lows[] = {0, 6, 2, 5, 9}; + unsigned n = 0; + for (unsigned m = 7; m <= 14; ++m) { + uint64_t half = 1ull << m, full = 2 * half; + uint64_t cpu = (20 - 16) * (full / 32), keccak = (1031 - 1024) * (full / 2048); + std::vector starts = {cpu}; + if (full >= 2048) starts.push_back(keccak); + if (m <= 11) { + starts.push_back(1); + starts.push_back(half - 1); + starts.push_back(half); + } + for (uint64_t stored : starts) { + shapes.push_back({m, stored, lows[n % 5], (int)(n % 3), m <= 10, m <= 11}); + ++n; + } + } + for (const Shape &s : shapes) run_shape(s); + printf("gkr host KAT: %llu layers (m 1..14), %llu card rounds (%llu by the kept kernel, %llu by today's " + "once the padding was spent): today's kernels = their copy from before; kept = whole, raw; sums " + "and halves = the written-out reference\n", + (unsigned long long)layers_run, (unsigned long long)rounds_run, + (unsigned long long)kept_rounds, (unsigned long long)today_rounds_on_kept); + if (failures) { + printf("gkr host KAT: %d FAILED\n", failures); + return 1; + } + printf("gkr host KAT: all passed\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/ntt_cm_host_kat.cpp b/crypto/math-cuda/tests/host_kat/ntt_cm_host_kat.cpp new file mode 100644 index 000000000..77fa0c6f7 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/ntt_cm_host_kat.cpp @@ -0,0 +1,846 @@ +// Known-answer tests for `kernels/ntt_cm.cu`, the column-major LDE engine's +// passes, run on the host. +// +// WHY THIS EXISTS. The column-major LDE engine's correctness is index math and +// roots: which element a register holds, which root a butterfly takes, what +// the twist multiplies by. All of that is arithmetic, and this checks it +// without a GPU: the kernel source compiles through `cuda_host_shim.h`, and a +// launch is replayed phase by phase — every thread of a phase before any +// thread of the next, which is the ordering the kernel's `__syncthreads` give +// it on the device. +// +// WHAT IT COVERS, each against an independent reference (schoolbook +// `__int128` field arithmetic and naive O(n^2) transforms — no code shared +// with the kernel): +// 1. the windowed root table against repeated multiplication, and the wide +// passes' radix tables (k = 9..12) against ω_{2^k}^j; +// 2. every pass shape (k = 4..12, DIT and DIF, contiguous and strided, every +// start level s) against the textbook radix-2 levels [s, s + k); +// 3. whole transforms composed of passes against the naive DFT; +// 4. the LDE the driver runs — DIF iNTT out of place, bit-reversed weights +// on the last pass, DIT with the fused coset spread — against direct +// interpolation and evaluation on the coset, blowups 2..16, several +// columns with a column stride, non-canonical inputs; +// 5. the coefficient-input form (F_SPREAD | F_GATHER) against direct +// evaluation; +// 6. the single-column spread transform of WHIR's encoding (spacing 1..16, +// no weights) against the naive DFT; +// 7. the same transform in the wide plan (`lde_cm::plan_wide`, passes of up +// to 10 levels in three register rounds) against the naive DFT on small +// sizes, and against today's plan — the oracle — at every size up to 2^20 +// where the two plans differ, edge words planted; +// 8. the tile layout: every slot of every (k, T) in bounds and distinct, and +// a wide pass's first-round and staged accesses on 16 distinct banks a +// half-warp (a speed property, checked because the layout exists for it); +// 9. the Möbius levels fused into the first pass (F_MOBIUS): the encoding's +// other Möbius levels on the host, then the fused transform, against the +// whole Möbius transform on the host then today's spread transform, in +// both plans, spacings 1..16, edge words planted. +// +// WHAT IT DOES NOT COVER: whether nvcc accepts the file, and everything about +// execution — grid sizing, shared-memory size, races the replay order cannot +// show, register pressure. That is the GPU parity suite's +// (`tests/ntt_cm_parity.rs`). Passing here is necessary, never sufficient. +// +// Build and run with `make test-ntt-cm-host-kat`. + +#include +#include +#include +#include +#include +#include + +#include "cuda_host_shim.h" + +#include "ntt_cm.cu" + +namespace { + +using u64 = uint64_t; +using u128 = unsigned __int128; + +constexpr u64 P = 0xFFFFFFFF00000001ull; +constexpr u64 ROOT32 = 1753635133440165772ull; // TWO_ADIC_PRIMITVE_ROOT_OF_UNITY + +int g_failures = 0; +int g_checks = 0; + +#define CHECK(cond, ...) \ + do { \ + ++g_checks; \ + if (!(cond)) { \ + ++g_failures; \ + std::printf("FAIL %s:%d: ", __FILE__, __LINE__); \ + std::printf(__VA_ARGS__); \ + std::printf("\n"); \ + } \ + } while (0) + +// Schoolbook field arithmetic — the definition, independent of goldilocks.cuh. +u64 canon(u64 a) { return a >= P ? a - P : a; } +u64 fadd(u64 a, u64 b) { return (u64)(((u128)canon(a) + canon(b)) % P); } +u64 fsub(u64 a, u64 b) { return (u64)(((u128)canon(a) + P - canon(b)) % P); } +u64 fmul(u64 a, u64 b) { return (u64)(((u128)canon(a) * canon(b)) % P); } +u64 fpow(u64 b, u64 e) { + u64 r = 1; + b = canon(b); + while (e) { + if (e & 1) r = fmul(r, b); + b = fmul(b, b); + e >>= 1; + } + return r; +} +u64 finv(u64 a) { return fpow(a, P - 2); } + +// ω_{2^m} = ROOT32^(2^(32-m)), by squaring — the math crate's own derivation. +u64 omega(unsigned m, bool inverse) { + u64 r = ROOT32; + for (unsigned i = 0; i < 32 - m; ++i) r = fmul(r, r); + return inverse ? finv(r) : r; +} + +std::vector root_table(bool inverse) { + const u64 root = inverse ? finv(ROOT32) : ROOT32; + std::vector t(1024); + for (int w = 0; w < 4; ++w) { + const u64 base = fpow(root, (u64)1 << (8 * w)); + u64 acc = 1; + for (int j = 0; j < 256; ++j) { + t[w * 256 + j] = acc; + acc = fmul(acc, base); + } + } + return t; +} + +u64 rev(u64 x, unsigned bits) { + u64 r = 0; + for (unsigned i = 0; i < bits; ++i) r |= ((x >> i) & 1) << (bits - 1 - i); + return r; +} + +// Deterministic, deliberately non-canonical inputs: a slice lands in [P, 2^64). +u64 g_rng = 0x9E3779B97F4A7C15ull; +u64 next_u64() { + g_rng ^= g_rng << 13; + g_rng ^= g_rng >> 7; + g_rng ^= g_rng << 17; + return g_rng; +} +u64 random_felt() { + const u64 r = next_u64(); + return (r & 7) == 0 ? P + (r % (0xFFFFFFFFull)) : r; +} + +// Textbook levels [s, s + k) of a length-2^L transform, in place. +void textbook_levels(std::vector &x, unsigned L, unsigned s, unsigned k, bool dit, + bool inverse) { + const u64 n = (u64)1 << L; + auto one_level = [&](unsigned u) { + const u64 half = (u64)1 << u; + const u64 w1 = omega(u + 1, inverse); + std::vector pw(half); + u64 acc = 1; + for (u64 j = 0; j < half; ++j) { + pw[j] = acc; + acc = fmul(acc, w1); + } + for (u64 i = 0; i < n; ++i) { + if (i & half) continue; + const u64 tw = pw[i & (half - 1)]; + const u64 a = x[i], b = x[i + half]; + if (dit) { + const u64 t = fmul(b, tw); + x[i] = fadd(a, t); + x[i + half] = fsub(a, t); + } else { + x[i] = fadd(a, b); + x[i + half] = fmul(fsub(a, b), tw); + } + } + }; + if (dit) { + for (unsigned u = s; u < s + k; ++u) one_level(u); + } else { + for (unsigned u = s + k; u-- > s;) one_level(u); + } +} + +// Naive DFT: out[j] = Σ_i in[i] ω^(ij). +std::vector naive_dft(const std::vector &in, unsigned L, bool inverse) { + const u64 n = (u64)1 << L; + const u64 w = omega(L, inverse); + std::vector out(n); + for (u64 j = 0; j < n; ++j) { + const u64 wj = fpow(w, j); + u64 acc = 0, pw = 1; + for (u64 i = 0; i < n; ++i) { + acc = fadd(acc, fmul(in[i], pw)); + pw = fmul(pw, wj); + } + out[j] = acc; + } + return out; +} + +// --------------------------------------------------------------------------- +// Launch replay: one pass over `cols` columns, exactly the geometry the Rust +// driver uses (see `lde_cm.rs`): T = min(4096 >> k, 2^s) for strided passes, +// T = min(4096 >> k, 2^(L - k)) for contiguous ones. +// --------------------------------------------------------------------------- + +unsigned pass_log_t(unsigned L, unsigned s, unsigned k) { + const unsigned cap = 12 - k; // log2(4096 >> k) + const unsigned lim = s > 0 ? s : L - k; + return cap < lim ? cap : lim; +} + +// Words past the tile that no phase may write (checked after every block). +constexpr size_t TILE_GUARD = 64; +constexpr u64 GUARD_WORD = 0xA5A5A5A5DEADF00Dull; + +template +void replay_pass_k(const u64 *in, u64 in_stride, u64 *out, u64 out_stride, const u64 *roots, + const u64 *wtab, unsigned L, unsigned s, unsigned lb, unsigned flags, + unsigned cols) { + using namespace ntt_cm; + constexpr int ROUNDS = (KLOG + 3) / 4; + const unsigned log_t = pass_log_t(L, s, KLOG); + const unsigned nthreads = threads_of(log_t); + const unsigned blocks = 1u << (L - KLOG - log_t); + const size_t words = tile_words(log_t); + std::vector tile(words + TILE_GUARD, GUARD_WORD); + std::vector rad(Radix::SIZE); + for (unsigned col = 0; col < cols; ++col) { + for (unsigned bx = 0; bx < blocks; ++bx) { + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_rad(rad.data(), roots, tid, nthreads); + if (s > 0) { + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_strided_first(in, in_stride, out, out_stride, roots, rad.data(), + tile.data(), L, s, log_t, tid, bx, col); + if constexpr (ROUNDS == 3) { + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_tile_round(tile.data(), rad.data(), log_t, tid); + } + if constexpr (ROUNDS >= 2) { + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_strided_last(out, out_stride, roots, rad.data(), + tile.data(), L, s, log_t, tid, bx, col); + } + } else { + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_contig_stage(in, in_stride, wtab, tile.data(), L, lb, log_t, flags, + tid, bx, col); + if (flags & F_MOBIUS) { + for (unsigned l = 0; l < KLOG + log_t - lb; ++l) + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_contig_mobius(tile.data(), log_t, lb, l, tid); + } + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_tile_round(tile.data(), rad.data(), log_t, tid); + if constexpr (ROUNDS >= 2) { + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_tile_round(tile.data(), rad.data(), log_t, tid); + } + if constexpr (ROUNDS == 3) { + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_tile_round(tile.data(), rad.data(), log_t, tid); + } + for (unsigned tid = 0; tid < nthreads; ++tid) + phase_contig_store(out, out_stride, wtab, tile.data(), log_t, flags, tid, + bx, col); + } + bool guard_ok = true; + for (size_t i = words; i < words + TILE_GUARD; ++i) guard_ok &= tile[i] == GUARD_WORD; + if (!guard_ok) { + CHECK(false, "k=%d L=%u s=%u: a phase wrote past the tile's %zu words", KLOG, L, s, + words); + std::fill(tile.begin() + words, tile.end(), GUARD_WORD); + } + } + } +} + +void replay_pass(unsigned k, bool dit, const u64 *in, u64 in_stride, u64 *out, u64 out_stride, + const u64 *roots, const u64 *wtab, unsigned L, unsigned s, unsigned lb, + unsigned flags, unsigned cols) { +#define DISPATCH(KK) \ + case KK: \ + if (dit) \ + replay_pass_k(in, in_stride, out, out_stride, roots, wtab, L, s, lb, flags, \ + cols); \ + else \ + replay_pass_k(in, in_stride, out, out_stride, roots, wtab, L, s, lb, flags, \ + cols); \ + return; + switch (k) { + DISPATCH(4) + DISPATCH(5) + DISPATCH(6) + DISPATCH(7) + DISPATCH(8) + DISPATCH(9) + DISPATCH(10) + DISPATCH(11) + DISPATCH(12) + default: + std::printf("bad k %u\n", k); + std::exit(2); + } +#undef DISPATCH +} + +// The driver's pass plan (mirrors `lde_cm::plan`): ceil(L/8) passes, sizes as +// equal as possible. +std::vector plan(unsigned L) { + const unsigned p = (L + 7) / 8; + std::vector ks(p, L / p); + for (unsigned i = 0; i < L % p; ++i) ks[i] += 1; + return ks; +} + +// The encoding's wide plan (mirrors `lde_cm::plan_wide`): ceil(L/10) passes, +// sizes as equal as possible — [10, 9, 9] at the production 2^28. +std::vector plan_wide(unsigned L) { + const unsigned p = (L + 9) / 10; + std::vector ks(p, L / p); + for (unsigned i = 0; i < L % p; ++i) ks[i] += 1; + return ks; +} + +bool eq_canon(const std::vector &a, const std::vector &b, size_t *where) { + if (a.size() != b.size()) { + *where = (size_t)-1; + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (canon(a[i]) != canon(b[i])) { + *where = i; + return false; + } + } + return true; +} + +// --------------------------------------------------------------------------- +// 1. Windowed roots. +// --------------------------------------------------------------------------- +void test_root_table() { + for (bool inverse : {false, true}) { + const std::vector t = root_table(inverse); + const u64 root = inverse ? finv(ROOT32) : ROOT32; + for (int i = 0; i < 2000; ++i) { + const uint32_t e = (uint32_t)next_u64(); + CHECK(canon(ntt_cm::pow_root32(t.data(), e)) == fpow(root, e), + "pow_root32 inverse=%d e=%u", (int)inverse, e); + } + CHECK(canon(ntt_cm::pow_root32(t.data(), 0)) == 1, "pow_root32(0) != 1"); + // The top window is the radix table ω_256^j. + const u64 w256 = omega(8, inverse); + for (int j = 0; j < 128; ++j) + CHECK(t[768 + j] == fpow(w256, j), "radix table entry %d", j); + // A wide pass's radix table, built from the windows: ω_{2^k}^j. + auto wide = [&](auto kk) { + constexpr int K = decltype(kk)::value; + using RX = ntt_cm::Radix; + static_assert(RX::LOG == K - 1, "a wide pass takes ω_{2^k}"); + std::vector rad(RX::SIZE, 0); + for (uint32_t tid = 0; tid < 256; ++tid) + ntt_cm::phase_rad(rad.data(), t.data(), tid, 256); + const u64 wk = omega(K, inverse); + for (int j = 0; j < RX::SIZE; ++j) + CHECK(canon(rad[j]) == fpow(wk, j), "k=%d inverse=%d radix entry %d", K, + (int)inverse, j); + }; + wide(std::integral_constant{}); + wide(std::integral_constant{}); + wide(std::integral_constant{}); + wide(std::integral_constant{}); + static_assert(ntt_cm::Radix<8>::SIZE == 128 && ntt_cm::Radix<4>::SIZE == 128, + "every k <= 8 keeps today's table"); + } + // ω_{2^32} really has order 2^32 (and not 2^31). + CHECK(fpow(ROOT32, (u64)1 << 31) == P - 1, "ROOT32^(2^31) != -1"); +} + +// --------------------------------------------------------------------------- +// 2. Every pass shape against the textbook levels. +// --------------------------------------------------------------------------- +void test_single_passes() { + const std::vector fwd = root_table(false), inv = root_table(true); + for (unsigned L = 4; L <= 14; ++L) { + for (unsigned k = 4; k <= 12 && k <= L; ++k) { + for (unsigned s = 0; s + k <= L; ++s) { + // Strided passes need a warp's worth of lo only for speed; the + // math holds for any s >= log T, which the plan guarantees. + for (bool dit : {true, false}) { + for (bool inverse : {false, true}) { + const unsigned cols = 2; + const u64 n = (u64)1 << L; + const u64 stride = n + 3; // a column stride that is not the length + std::vector buf(stride * cols); + for (auto &v : buf) v = random_felt(); + std::vector> want(cols); + for (unsigned c = 0; c < cols; ++c) { + want[c].assign(buf.begin() + c * stride, buf.begin() + c * stride + n); + textbook_levels(want[c], L, s, k, dit, inverse); + } + // Out of place for the strided form (as the iNTT's + // first pass runs), in place for the contiguous one. + std::vector out = s > 0 ? std::vector(stride * cols, 0) : buf; + replay_pass(k, dit, buf.data(), stride, out.data(), stride, + inverse ? inv.data() : fwd.data(), nullptr, L, s, 0, 0, cols); + for (unsigned c = 0; c < cols; ++c) { + std::vector got(out.begin() + c * stride, + out.begin() + c * stride + n); + size_t where = 0; + CHECK(eq_canon(got, want[c], &where), + "pass L=%u s=%u k=%u dit=%d inv=%d col=%u first diff at %zu", L, s, + k, (int)dit, (int)inverse, c, where); + } + } + } + } + } + } +} + +// --------------------------------------------------------------------------- +// 3. Whole transforms composed of passes against the naive DFT. +// --------------------------------------------------------------------------- +void test_whole_transforms() { + const std::vector fwd = root_table(false), inv = root_table(true); + for (unsigned L = 4; L <= 12; ++L) { + const u64 n = (u64)1 << L; + // Today's plan and the encoding's wide one, forward and inverse. + for (int variant = 0; variant < 4; ++variant) { + const bool wide = variant & 2, inverse = variant & 1; + std::vector x(n); + for (auto &v : x) v = random_felt(); + const std::vector want = naive_dft(x, L, inverse); + const std::vector ks = wide ? plan_wide(L) : plan(L); + + // DIT: bit-reversed input, passes ascending from s = 0. + { + std::vector buf(n); + for (u64 i = 0; i < n; ++i) buf[i] = x[rev(i, L)]; + unsigned s = 0; + for (unsigned k : ks) { + replay_pass(k, true, buf.data(), n, buf.data(), n, + inverse ? inv.data() : fwd.data(), nullptr, L, s, 0, 0, 1); + s += k; + } + size_t where = 0; + CHECK(eq_canon(buf, want, &where), "DIT L=%u wide=%d inv=%d first diff at %zu", L, + (int)wide, (int)inverse, where); + } + // DIF: natural input, passes descending to s = 0, bit-reversed out. + { + std::vector buf = x; + unsigned s = L; + for (unsigned k : ks) { + s -= k; + replay_pass(k, false, buf.data(), n, buf.data(), n, + inverse ? inv.data() : fwd.data(), nullptr, L, s, 0, 0, 1); + } + std::vector nat(n); + for (u64 i = 0; i < n; ++i) nat[rev(i, L)] = buf[i]; + size_t where = 0; + CHECK(eq_canon(nat, want, &where), "DIF L=%u wide=%d inv=%d first diff at %zu", L, + (int)wide, (int)inverse, where); + } + } + } +} + +// --------------------------------------------------------------------------- +// 4. The LDE as the driver runs it, and 5. the coefficient-input form. +// --------------------------------------------------------------------------- + +// weights[j] = g^j / n (the prover's `coset_weights`). +std::vector coset_weights(u64 n, u64 g, bool with_inv_n) { + std::vector w(n); + u64 acc = with_inv_n ? finv(n % P) : 1; + for (u64 j = 0; j < n; ++j) { + w[j] = acc; + acc = fmul(acc, g); + } + return w; +} + +// Direct: interpolate (naive inverse DFT / n), evaluate at g·ω_N^i. +std::vector naive_lde(const std::vector &evals, unsigned lb, u64 g) { + const u64 n = evals.size(); + unsigned log_n = 0; + while (((u64)1 << log_n) < n) ++log_n; + const unsigned log_N = log_n + lb; + const u64 N = (u64)1 << log_N; + std::vector c = naive_dft(evals, log_n, true); + const u64 inv_n = finv(n % P); + for (auto &v : c) v = fmul(v, inv_n); + std::vector out(N); + const u64 wN = omega(log_N, false); + for (u64 i = 0; i < N; ++i) { + const u64 x = fmul(g, fpow(wN, i)); + u64 acc = 0, pw = 1; + for (u64 j = 0; j < n; ++j) { + acc = fadd(acc, fmul(c[j], pw)); + pw = fmul(pw, x); + } + out[i] = acc; + } + return out; +} + +// Run the driver's sequence on host: DIF iNTT from `src` (column stride +// src_stride) into W (stride n), last pass weighted by wbr; DIT from W with the +// coset spread into dst (stride N). +void run_lde(const std::vector &src, u64 src_stride, std::vector &dst, unsigned cols, + unsigned log_n, unsigned lb, const std::vector &weights) { + const std::vector fwd = root_table(false), inv = root_table(true); + const u64 n = (u64)1 << log_n; + const unsigned log_N = log_n + lb; + const u64 N = (u64)1 << log_N; + std::vector wbr(n); + for (u64 p = 0; p < n; ++p) wbr[p] = weights[rev(p, log_n)]; + std::vector W(n * cols, 0xDEADBEEFull); + + const std::vector ki = plan(log_n); + unsigned s = log_n; + for (size_t i = 0; i < ki.size(); ++i) { + s -= ki[i]; + const bool first = i == 0, last = i + 1 == ki.size(); + replay_pass(ki[i], false, first ? src.data() : W.data(), first ? src_stride : n, W.data(), + n, inv.data(), wbr.data(), log_n, s, 0, last ? ntt_cm::F_STORE_W : 0, cols); + } + const std::vector kf = plan(log_N); + s = 0; + for (size_t i = 0; i < kf.size(); ++i) { + const bool first = i == 0; + replay_pass(kf[i], true, first ? W.data() : dst.data(), first ? n : N, dst.data(), N, + fwd.data(), nullptr, log_N, s, lb, first ? ntt_cm::F_SPREAD : 0, cols); + s += kf[i]; + } +} + +void test_lde() { + const u64 g = 7; + struct Shape { + unsigned log_n, lb, cols; + }; + const Shape shapes[] = { + {4, 1, 1}, {4, 2, 2}, {5, 1, 3}, {6, 3, 2}, {7, 4, 1}, {8, 1, 2}, + {9, 2, 2}, {10, 1, 1}, {10, 3, 1}, {11, 2, 1}, {9, 4, 1}, + }; + for (const Shape &sh : shapes) { + const u64 n = (u64)1 << sh.log_n; + const u64 N = n << sh.lb; + const u64 src_stride = n + 5; + std::vector src(src_stride * sh.cols); + for (auto &v : src) v = random_felt(); + const std::vector weights = coset_weights(n, g, true); + std::vector dst(N * sh.cols, 0xBADC0FFEEull); + const std::vector src_before = src; + run_lde(src, src_stride, dst, sh.cols, sh.log_n, sh.lb, weights); + for (unsigned c = 0; c < sh.cols; ++c) { + std::vector evals(src.begin() + c * src_stride, src.begin() + c * src_stride + n); + const std::vector want = naive_lde(evals, sh.lb, g); + std::vector got(dst.begin() + c * N, dst.begin() + (c + 1) * N); + size_t where = 0; + CHECK(eq_canon(got, want, &where), "LDE log_n=%u lb=%u col=%u first diff at %zu", + sh.log_n, sh.lb, c, where); + } + // The source survives, raw u64 for raw u64: the iNTT's first pass + // reads it and writes the scratch, which is what lets the main commit + // keep its trace-domain snapshot. + CHECK(src == src_before, "LDE log_n=%u lb=%u clobbered its source", sh.log_n, sh.lb); + } +} + +void test_coeff_form() { + const u64 offset = 7; + for (unsigned log_n = 4; log_n <= 9; ++log_n) { + for (unsigned lb = 1; lb <= 3; ++lb) { + const u64 n = (u64)1 << log_n; + const unsigned log_N = log_n + lb; + const u64 N = (u64)1 << log_N; + const unsigned cols = 2; + std::vector coef(n * cols); + for (auto &v : coef) v = random_felt(); + const std::vector weights = coset_weights(n, offset, false); + std::vector wbr(n); + for (u64 p = 0; p < n; ++p) wbr[p] = weights[rev(p, log_n)]; + std::vector dst(N * cols, 0); + const std::vector fwd = root_table(false); + const std::vector kf = plan(log_N); + unsigned s = 0; + for (size_t i = 0; i < kf.size(); ++i) { + const bool first = i == 0; + replay_pass(kf[i], true, first ? coef.data() : dst.data(), first ? n : N, + dst.data(), N, fwd.data(), wbr.data(), log_N, s, lb, + first ? (ntt_cm::F_SPREAD | ntt_cm::F_GATHER) : 0, cols); + s += kf[i]; + } + const u64 wN = omega(log_N, false); + for (unsigned c = 0; c < cols; ++c) { + std::vector want(N); + for (u64 i = 0; i < N; ++i) { + const u64 x = fmul(offset, fpow(wN, i)); + u64 acc = 0, pw = 1; + for (u64 j = 0; j < n; ++j) { + acc = fadd(acc, fmul(coef[c * n + j], pw)); + pw = fmul(pw, x); + } + want[i] = acc; + } + std::vector got(dst.begin() + c * N, dst.begin() + (c + 1) * N); + size_t where = 0; + CHECK(eq_canon(got, want, &where), "coeff form log_n=%u lb=%u col=%u diff at %zu", + log_n, lb, c, where); + } + } + } +} + +// 6. The single-column spread transform the WHIR commit's encoding runs +// (`lde_cm::spread_ntt_column`): position B·q of the DIT input is src[q], the +// rest zero, no weights — the legacy `lift_spread` then the DIT body. A DIT with +// bit-reversed input position p computes Σ_p z[p]·ω^(i·rev_N(p)), and +// rev_N(B·q) = rev_n(q), so the answer is the DFT of y[j] = src[rev_n(j)] (j < n). +void test_spread_transform() { + const std::vector fwd = root_table(false); + for (unsigned log_n = 4; log_n <= 11; ++log_n) { + for (unsigned lb = 0; lb <= 4 && lb <= log_n; ++lb) { + const u64 N = (u64)1 << log_n; + const unsigned log_c = log_n - lb; + const u64 n = (u64)1 << log_c; + std::vector src(n); + for (auto &v : src) v = random_felt(); + std::vector dst(N, 0xFEEDFACEull); + const std::vector ks = plan(log_n); + unsigned s = 0; + for (size_t i = 0; i < ks.size(); ++i) { + const bool first = i == 0; + replay_pass(ks[i], true, first ? src.data() : dst.data(), 0, dst.data(), 0, fwd.data(), + nullptr, log_n, s, lb, first ? ntt_cm::F_SPREAD : 0, 1); + s += ks[i]; + } + std::vector y(N, 0); + for (u64 j = 0; j < n; ++j) y[j] = src[log_c ? rev(j, log_c) : 0]; + const std::vector want = naive_dft(y, log_n, false); + size_t where = 0; + CHECK(eq_canon(dst, want, &where), "spread transform log_n=%u lb=%u first diff at %zu", log_n, + lb, where); + } + } +} + +// The spread transform on one column in a given plan, as `spread_ntt_column` +// launches it, the first pass's extra flags (F_MOBIUS) or-ed in. +std::vector spread_transform(const std::vector &src, unsigned log_n, unsigned lb, + const std::vector &ks, unsigned first_flags = 0) { + const std::vector fwd = root_table(false); + std::vector dst((u64)1 << log_n, 0xFEEDFACEull); + unsigned s = 0; + for (size_t i = 0; i < ks.size(); ++i) { + const bool first = i == 0; + replay_pass(ks[i], true, first ? src.data() : dst.data(), 0, dst.data(), 0, fwd.data(), + nullptr, log_n, s, lb, first ? (ntt_cm::F_SPREAD | first_flags) : 0, 1); + s += ks[i]; + } + return dst; +} + +// Möbius levels [from, to) on the host, the definition: every index with the +// level's bit set loses its partner below. +void host_mobius(std::vector &x, unsigned from, unsigned to) { + for (unsigned l = from; l < to; ++l) { + const u64 h = (u64)1 << l; + for (u64 i = 0; i < x.size(); ++i) + if (i & h) x[i] = fsub(x[i], x[i ^ h]); + } +} + +// Raw words with an edge value at every seventh slot (the card test's mix). +std::vector edge_words(u64 n) { + static const u64 EDGES[8] = {0, 1, P - 1, P, P + 1, ~0ull, 1ull << 63, 0xFFFFFFFFull}; + std::vector v(n); + for (u64 i = 0; i < n; ++i) v[i] = i % 7 == 0 ? EDGES[(i / 7) % 8] : next_u64(); + return v; +} + +// 7. The wide plan: against the naive DFT where that is cheap, and against +// today's plan — the 4..8-level passes the encoding runs today, themselves +// pinned by 2. and 6. — at every size up to 2^20 where the plans differ. +void test_wide_spread_transform() { + for (unsigned log_n = 4; log_n <= 12; ++log_n) { + for (unsigned lb = 0; lb <= 4 && lb <= log_n; ++lb) { + const unsigned log_c = log_n - lb; + const u64 n = (u64)1 << log_c, N = (u64)1 << log_n; + const std::vector src = edge_words(n); + const std::vector got = spread_transform(src, log_n, lb, plan_wide(log_n)); + std::vector y(N, 0); + for (u64 j = 0; j < n; ++j) y[j] = src[log_c ? rev(j, log_c) : 0]; + size_t where = 0; + CHECK(eq_canon(got, naive_dft(y, log_n, false), &where), + "wide spread transform log_n=%u lb=%u first diff at %zu", log_n, lb, where); + } + } + unsigned differing = 0; + for (unsigned log_n = 4; log_n <= 20; ++log_n) { + const std::vector today = plan(log_n), wide = plan_wide(log_n); + if (today == wide) continue; + ++differing; + for (unsigned lb : {0u, 1u, 2u, 4u}) { + const u64 n = (u64)1 << (log_n - lb); + const std::vector src = edge_words(n); + const std::vector want = spread_transform(src, log_n, lb, today); + const std::vector got = spread_transform(src, log_n, lb, wide); + size_t where = 0; + CHECK(eq_canon(got, want, &where), + "wide vs today's plan log_n=%u lb=%u (%zu vs %zu passes) first diff at %zu", + log_n, lb, wide.size(), today.size(), where); + } + } + // 9, 10 and 17..20 (the production 2^28's windows, (0, 10), (10, 9) and + // (19, 9), are pinned per pass by 2. and on the card). + CHECK(differing == 6, "the plans differ at %u sizes up to 2^20, expected 6", differing); + CHECK(plan_wide(28) == std::vector({10, 9, 9}), "plan_wide(28) is not [10, 9, 9]"); + CHECK(plan(28) == std::vector({7, 7, 7, 7}), "plan(28) is not [7, 7, 7, 7]"); +} + +// 8. The tile layout. +template +void tile_layout_k() { + using namespace ntt_cm; + for (unsigned log_t = 0; log_t + KLOG <= 12; ++log_t) { + const uint32_t words = tile_words(log_t); + std::vector seen(words, 0); + bool ok = true; + for (uint32_t mid = 0; mid < (1u << KLOG); ++mid) + for (uint32_t t = 0; t < (1u << log_t); ++t) { + const uint32_t i = tile_at(mid, log_t, t); + if (i >= words || seen[i]) ok = false; + else seen[i] = 1; + } + CHECK(ok, "k=%d log_t=%u: a tile slot out of bounds or shared", KLOG, log_t); + } + if constexpr (KLOG > 8 && KLOG < 12) { + // At the driver's lane count T = 4096 >> k, a half-warp is 16 / T + // groups of T lanes. 64-bit words on 32 four-byte banks: a half-warp's + // 16 words are conflict-free iff they are distinct mod 16. (At k = 12 + // the single lane makes the pitch even; no kernel instantiates it.) + const uint32_t log_t = 12 - KLOG, T = 1u << log_t; + auto distinct16 = [](const uint32_t (&a)[16]) { + bool hit[16] = {}; + for (uint32_t x : a) { + if (hit[x % 16]) return false; + hit[x % 16] = true; + } + return true; + }; + bool first = true, staged = true; + for (uint32_t h = 0; h < (threads_of(log_t) >> 4); ++h) + for (int v = 0; v < E; ++v) { + uint32_t a[16]; + for (uint32_t l = 0; l < 16; ++l) { + const uint32_t tid = h * 16 + l, t = tid & (T - 1), g = tid >> log_t; + a[l] = tile_at(mid_base<0>(g) | (uint32_t)v, log_t, t); + } + first &= distinct16(a); + } + for (uint32_t e0 = 0; e0 < ((1u << KLOG) << log_t); e0 += 16) { + uint32_t a[16]; + for (uint32_t l = 0; l < 16; ++l) { + const uint32_t e = e0 + l; + a[l] = tile_at(e & ((1u << KLOG) - 1u), log_t, e >> KLOG); + } + staged &= distinct16(a); + } + CHECK(first, "k=%d: the first round's half-warp shares a bank", KLOG); + CHECK(staged, "k=%d: the staged copy's half-warp shares a bank", KLOG); + } +} + +// 9. F_MOBIUS: the first pass's block holds the low F = k + log T - lb bits of +// a compact index, so the host runs levels [F, log_c) and the pass levels +// [0, F); the oracle runs every level, then today's transform. +void test_fused_mobius() { + for (unsigned log_n = 4; log_n <= 20; ++log_n) { + for (unsigned lb : {0u, 1u, 2u, 4u}) { + if (lb > log_n || (log_n > 13 && log_n < 17 && lb != 1)) continue; + const unsigned log_c = log_n - lb; + const std::vector src = edge_words((u64)1 << log_c); + std::vector full = src; + host_mobius(full, 0, log_c); + const std::vector want = spread_transform(full, log_n, lb, plan(log_n)); + for (bool wide : {false, true}) { + const std::vector ks = wide ? plan_wide(log_n) : plan(log_n); + const unsigned f = ks[0] + pass_log_t(log_n, 0, ks[0]) - lb; + std::vector part = src; + host_mobius(part, f, log_c); + const std::vector got = spread_transform(part, log_n, lb, ks, ntt_cm::F_MOBIUS); + size_t where = 0; + CHECK(eq_canon(got, want, &where), + "fused Möbius log_n=%u lb=%u wide=%d (%u levels fused) first diff at %zu", log_n, + lb, (int)wide, f, where); + if (log_n >= 12) CHECK(f == 12 - lb, "log_n=%u lb=%u: %u levels fused, not 12 - lb", log_n, lb, f); + } + } + } + // The fused levels are load-bearing: without the flag the same input is not the codeword. + const unsigned log_n = 13, lb = 1; + const std::vector src = edge_words((u64)1 << (log_n - lb)); + std::vector full = src, part = src; + host_mobius(full, 0, log_n - lb); + host_mobius(part, 12 - lb, log_n - lb); + size_t where = 0; + CHECK(!eq_canon(spread_transform(part, log_n, lb, plan(log_n)), + spread_transform(full, log_n, lb, plan(log_n)), &where), + "the unfused partial transform equals the codeword: the check cannot fail"); +} + +void test_tile_layout() { + tile_layout_k<4>(); + tile_layout_k<7>(); + tile_layout_k<8>(); + tile_layout_k<9>(); + tile_layout_k<10>(); + tile_layout_k<11>(); + tile_layout_k<12>(); + // Today's layout for every k <= 8: K x (T + 1), unchanged. + CHECK(ntt_cm::tile_words<7>(5) == 128u * 33u, "k=7 tile is not today's"); + CHECK(ntt_cm::tile_at<8>(17, 4, 3) == 17u * 17u + 3u, "k=8 slot is not today's"); +} + +} // namespace + +int main() { + test_root_table(); + std::printf("roots: %d checks, %d failures\n", g_checks, g_failures); + test_single_passes(); + std::printf("+ single passes: %d checks, %d failures\n", g_checks, g_failures); + test_whole_transforms(); + std::printf("+ whole transforms: %d checks, %d failures\n", g_checks, g_failures); + test_lde(); + std::printf("+ LDE: %d checks, %d failures\n", g_checks, g_failures); + test_coeff_form(); + std::printf("+ coefficient form: %d checks, %d failures\n", g_checks, g_failures); + test_spread_transform(); + std::printf("+ spread transform: %d checks, %d failures\n", g_checks, g_failures); + test_wide_spread_transform(); + std::printf("+ wide spread transform: %d checks, %d failures\n", g_checks, g_failures); + test_tile_layout(); + std::printf("+ tile layout: %d checks, %d failures\n", g_checks, g_failures); + test_fused_mobius(); + std::printf("+ fused Möbius: %d checks, %d failures\n", g_checks, g_failures); + if (g_failures) { + std::printf("NTT_CM HOST KAT: FAILED (%d of %d)\n", g_failures, g_checks); + return 1; + } + std::printf("NTT_CM HOST KAT: OK (%d checks)\n", g_checks); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/p1_stark_kat_vectors.h b/crypto/math-cuda/tests/host_kat/p1_stark_kat_vectors.h new file mode 100644 index 000000000..6c006628a --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/p1_stark_kat_vectors.h @@ -0,0 +1,46 @@ +// ZisK's arity-4 Merkle vectors for the PRODUCTION kernels of `kernels/p1w16.cu` +// (`p1s_merkle_level4`, `p1s_merkle_tail4`, `p1s_gather_paths4`): a transcription of +// `crypto/crypto/src/hash/poseidon1_stark/zisk_kat.rs` (MERKLE4_ROOTS, PATH_ROOT, PATHS), +// which `scripts/poseidon1/zisk_crosscheck` generates from ZisK's own Rust +// (`proofman-fields`, pil2-proofman v1.3.0-alpha, d485fac2). Do not edit by hand. +#pragma once +#include +static const uint64_t ZK_SEED_TREE = 0x7eeeull; +static const uint64_t ZK_SEED_ROWS = 0x2075ull; +static const uint64_t ZK_PATH_ROWS = 21, ZK_PATH_WIDTH = 7; +static const uint64_t ZK_TREE_N[9] = {1, 2, 3, 4, 5, 16, 17, 21, 64}; +static const uint64_t ZK_TREE_ROOT[9][4] = { + {0x8664c94dbefbe325ull, 0x30089173afe6c9d0ull, 0x04f90fe597272adfull, 0x0b7edc901f59ab82ull}, + {0x2eb364ea6cdadf19ull, 0xccb753f846d8a40aull, 0xf4f07ab8162431e4ull, 0x3c5ee032d67fdf59ull}, + {0x86e1acde79585807ull, 0x9abd8ea8b7cbe033ull, 0x11362fa741f967a5ull, 0x84b6f84e20992320ull}, + {0xe4439b945904eb77ull, 0x0cd8326d455ffdb1ull, 0x54b2f128a7109cf2ull, 0xcf984c5f8ad6848full}, + {0x20ac5f24df34a743ull, 0x3087e607ce719605ull, 0x5db55fcbca8a11e2ull, 0x983403c51793fa1aull}, + {0x5cb2c8680856be1bull, 0xcacd01bd16ca97bdull, 0xa929172d0b2e569full, 0x0f0511a671a5dd95ull}, + {0x199efcaf762e1698ull, 0xa6a300a7f88b0629ull, 0xa495edf09b591039ull, 0x9fcfb670ed6704b9ull}, + {0x28690a55266d917aull, 0xeb6bec957c51abc5ull, 0x571cbe2b6f5c05f2ull, 0xf101f6580769df1full}, + {0x37efafed6fd1bf30ull, 0xc1369a094e1dc939ull, 0x5cea7a4244a6e116ull, 0xdf140e80cc1b026eull}, +}; +static const uint64_t ZK_PATH_ROOT[4] = {0x23f761a0cb6ff00cull, 0x052127d1e609285bull, 0x78609f05b5b25a69ull, 0x364c6d50e3240899ull}; +static const uint64_t ZK_PATH_INDEX[4] = {0, 3, 13, 20}; +static const uint64_t ZK_PATH_SIBLINGS[4][3][12] = { + { + {0x6ebf87658577973bull, 0xb2af426aa2df3d12ull, 0x646457bf9de8f505ull, 0xbd423e46c0b7c70eull, 0xdc3e47b347cdde6dull, 0x8c557853b1ae4858ull, 0x25a1799325aa3076ull, 0xec5f1caa799417d2ull, 0x690cf267be58acffull, 0xed5b61bdf0c7addcull, 0xda3abba88dc03526ull, 0x06ef8ab52eeaae95ull}, + {0xdaf7006956ca017aull, 0x097dd24a97ba5646ull, 0xad5994585e0cd6e2ull, 0x1ba415ba1aa8761dull, 0xd2b7f30720d8ee9full, 0x89a016d3ff144dacull, 0x9517c5102664d681ull, 0x173fdf02efed4246ull, 0x95761bd58616b352ull, 0x2cf97fe34ea40dc2ull, 0x5106d057dcfce9cdull, 0xf767e484fb091b81ull}, + {0x0035d153eb31324dull, 0x37c538e270be601aull, 0x21f73928c5344f95ull, 0x0b0e4f442e8de209ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + }, + { + {0x281e019d55d54de1ull, 0xf0ab5f7cc3764d6full, 0x2cb6b556780ba645ull, 0xaa6d879899c5cf80ull, 0x6ebf87658577973bull, 0xb2af426aa2df3d12ull, 0x646457bf9de8f505ull, 0xbd423e46c0b7c70eull, 0xdc3e47b347cdde6dull, 0x8c557853b1ae4858ull, 0x25a1799325aa3076ull, 0xec5f1caa799417d2ull}, + {0xdaf7006956ca017aull, 0x097dd24a97ba5646ull, 0xad5994585e0cd6e2ull, 0x1ba415ba1aa8761dull, 0xd2b7f30720d8ee9full, 0x89a016d3ff144dacull, 0x9517c5102664d681ull, 0x173fdf02efed4246ull, 0x95761bd58616b352ull, 0x2cf97fe34ea40dc2ull, 0x5106d057dcfce9cdull, 0xf767e484fb091b81ull}, + {0x0035d153eb31324dull, 0x37c538e270be601aull, 0x21f73928c5344f95ull, 0x0b0e4f442e8de209ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + }, + { + {0x40dd5da5d4363562ull, 0x6dff0c226f49789full, 0x07147d0a82d4ef3aull, 0xa3c79042cf6dddcfull, 0x1b6988f938cd9ddaull, 0xa195a6dc7990340eull, 0x1bb1c4cd027f331eull, 0xb3814bb7304c2ae4ull, 0x268ebd7c101b16ddull, 0xc8701b039ebf6a61ull, 0xcc3c5195235401e8ull, 0x86d620a4dc0a8a4aull}, + {0xc4a0fe4b7c48ac77ull, 0x6380dc66613847b3ull, 0xb0aa8463bff724e2ull, 0x4f2d28e9d19d26a3ull, 0xdaf7006956ca017aull, 0x097dd24a97ba5646ull, 0xad5994585e0cd6e2ull, 0x1ba415ba1aa8761dull, 0xd2b7f30720d8ee9full, 0x89a016d3ff144dacull, 0x9517c5102664d681ull, 0x173fdf02efed4246ull}, + {0x0035d153eb31324dull, 0x37c538e270be601aull, 0x21f73928c5344f95ull, 0x0b0e4f442e8de209ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + }, + { + {0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0x5e833b6c497a8a05ull, 0xee643677f38238bcull, 0xb9a53292f76ab34dull, 0x08409edcc0a6b39eull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0x889e95bbeee05d35ull, 0xd6b76c8d22d89bd6ull, 0x795cc749c96b7575ull, 0xd9684e68cd1b09e3ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + }, +}; diff --git a/crypto/math-cuda/tests/host_kat/p1_zisk_kat_vectors.h b/crypto/math-cuda/tests/host_kat/p1_zisk_kat_vectors.h new file mode 100644 index 000000000..cba432962 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/p1_zisk_kat_vectors.h @@ -0,0 +1,41 @@ +// Known answers for the ZisK-instance device kernels (`kernels/p1w16.cu`), computed by +// ZisK's own Rust implementation (`proofman-fields`, pil2-proofman v1.3.0-alpha, d485fac2, +// MIT OR Apache-2.0). GENERATED by `scripts/poseidon1/zisk_crosscheck` (`-- c`); do not edit. +// Leaf inputs are the LCG stream `zk_felts(ZK_SEED_LEAF, n)`. +#pragma once +#include +static const uint64_t ZK_PERM8_IN[4][8] = { + {0x0000000000000000ull, 0x0000000000000001ull, 0x0000000000000002ull, 0x0000000000000003ull, 0x0000000000000004ull, 0x0000000000000005ull, 0x0000000000000006ull, 0x0000000000000007ull}, + {0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull}, + {0x2ceaee21bf46bc00ull, 0xaa80754d1a1a8d4full, 0xb3c4904a6d278932ull, 0xbc69cf4276846d19ull, 0x377b2fd56a5b15b4ull, 0x64d815deeaf29df3ull, 0xf66e100db2d7d206ull, 0x1069e6a57e06665dull}, +}; +static const uint64_t ZK_PERM8_OUT[4][8] = { + {0x21bd740749dd30d7ull, 0x82d56e9f77070d02ull, 0x5ec0547e3dbc8dccull, 0xeaa429642119fae9ull, 0x8c9b30f79af9b031ull, 0xf224d564607084bdull, 0x7dd2af74636e87d6ull, 0xa7daeb4dc21c22d7ull}, + {0x967b858fdabc48f5ull, 0xad350bc2b4a0e728ull, 0x2df1898fef94687aull, 0x03615aabe4616577ull, 0x964840faf7231998ull, 0x2c23d347f3c7cddbull, 0xc3c52c0cb7d2776cull, 0xa1021e181fe6ea58ull}, + {0x00f79505ba6ae588ull, 0xa94c3a530e38c062ull, 0xc33cd2c7bad60fa5ull, 0x9eab666a9418b97cull, 0xba59588e735134f7ull, 0xe20003ce19565758ull, 0xdd6e988c7e479219ull, 0xa7f53d8b009dd2e7ull}, + {0x6f4b8bcff6c46360ull, 0x55fa4e327ba9a074ull, 0x64f61d05b00bb015ull, 0x503249cfdf8949a2ull, 0xee0c105d8e205184ull, 0xa296e0f105460d27ull, 0x2a1dd0e592a7148bull, 0x3796cef45525bcecull}, +}; +static const uint64_t ZK_SEED_LEAF = 0x1eafull; +static const uint64_t ZK_LEAF_N[15] = {1, 3, 4, 5, 8, 11, 12, 13, 23, 24, 25, 36, 64, 100, 245}; +static const uint64_t ZK_LEAF_DIGEST[15][4] = { + {0x7d03e575421c5b23ull, 0xe7205b8b16d02caeull, 0xe7403af7b190c9b8ull, 0x33dce167235a4161ull}, + {0x6640aacb19b99fefull, 0x8cfa8c6eded38947ull, 0x983df120f9673f0eull, 0x1310238c565b68c3ull}, + {0x5bbe522545759068ull, 0x8be0c21c95272122ull, 0x4c1d411056f08119ull, 0xfd4884cefaae238eull}, + {0xf9e35072a2cbb968ull, 0x00eecaa4551e4bc6ull, 0x2967536c85b1e981ull, 0xcc0e4f974a68a925ull}, + {0x37699b63c7c41ff8ull, 0x9b77ac0ed9cfa6baull, 0xe8e79112528274faull, 0x0c97bd2a1719f7a9ull}, + {0x2fb335f8ba81a625ull, 0xc60103e3cd048002ull, 0x6ad161bff910fe7full, 0xda3990535144a065ull}, + {0x56d3b861d55fd314ull, 0xa5a31a15748ecf02ull, 0x7b8a70ffc050b14aull, 0x1ea870b034c04761ull}, + {0x970e5d8788b6ff13ull, 0x662a81f429cf5db9ull, 0x4b0b0ab73c68d61dull, 0x9f39ab57a883f85aull}, + {0xb24ab0a82ce7788full, 0x01e278923e6f556aull, 0x1d5fcb3b6c26b2dfull, 0xb854b08d821532a7ull}, + {0x94f5a3dd9e653ad2ull, 0x3170c14426de671bull, 0x2b2be28f1e5de7e6ull, 0x121dc669632d4af2ull}, + {0x92f4915f35848d9full, 0x23d820916e1a5313ull, 0xde776805d1a21bf7ull, 0x9e2ecd73e3fcced1ull}, + {0x042694abd8c60e18ull, 0x64e6ea0ecb0e3dacull, 0x089c1f4d45e9b9b7ull, 0xa13674c9fe4c3eedull}, + {0x8794e906c2322944ull, 0x0e3e180c1411cf2aull, 0xd4f728e29cf2838full, 0x5972df3e1ef71b06ull}, + {0x61ca36b7bf6d6dd3ull, 0x54822101148cfe34ull, 0x019fda1d3ba125c2ull, 0xff29312691d434ebull}, + {0x7c9f951fe2d785d0ull, 0x1a234f39f5f1997dull, 0xfa9e991c9153c45cull, 0x17e9375cff057d09ull}, +}; +static const uint64_t ZK_GRIND_CHALLENGE[3] = {0x9e2dc8d5767b6c68ull, 0x600b7718fe692797ull, 0xc430fd99b96a5fdaull}; +static const uint64_t ZK_GRIND_LANE0[4] = {0xc82edce9946878a9ull, 0xa97fc9da5e04914eull, 0x4d160560aa29bea0ull, 0x03007779902fd6ebull}; +static const uint64_t ZK_GRIND_BITS = 10; +static const uint64_t ZK_GRIND_SMALLEST_NONCE = 2408; diff --git a/crypto/math-cuda/tests/host_kat/p1w16_host_kat.cpp b/crypto/math-cuda/tests/host_kat/p1w16_host_kat.cpp new file mode 100644 index 000000000..2b612e719 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/p1w16_host_kat.cpp @@ -0,0 +1,701 @@ +// Known-answer tests for `kernels/p1w16.cu` (the D-HASH stage-1 Poseidon1 +// width-16 measurement kernels), run on the HOST through the shim: no GPU, no +// nvcc. Every check runs at every variant (the two multiplies and the +// Fourier-domain partial rounds against the circulant's vectors; the Cauchy +// alternative against its own). +// +// The vectors (`p1w16_kat_vectors.h`) come from the Python reference +// (`scripts/poseidon1/p1_params.py cuda`); the permutation vectors are the ones +// the Python reference, Plonky3's generator's permutation, Plonky3's Rust +// `Poseidon1` and the Rust host reference agree on. +// +// ⚠ Arithmetic and index math only — execution (threads, occupancy, the grid +// reduction) stays with `tests/p1w16_device.rs` on a GPU. +// +// make test-p1w16-host-kat + +#include +#include + +#include "cuda_host_shim.h" +#include "p1w16.cu" +#include "p1w16_kat_vectors.h" +#include "p1_zisk_kat_vectors.h" +#include "p1_stark_kat_vectors.h" + +static int g_fail = 0; + +static void check(bool ok, const char *what, int v) { + if (!ok) { + std::printf("FAIL [v%d] %s\n", v, what); + g_fail++; + } +} + +// One instance's expected answers: the circulant's (P1_*) or the Cauchy +// alternative's (P1C_*), over the same inputs. +struct Vectors { + const uint64_t (*perm_out)[16]; + const uint64_t (*leaf_digest)[4]; + const uint64_t *node4; + const uint64_t *grind_head; +}; +static const Vectors CIRCULANT = {P1_PERM_OUT, P1_LEAF_DIGEST, P1_NODE4, P1_GRIND_HEAD}; +static const Vectors CAUCHY = {P1C_PERM_OUT, P1C_LEAF_DIGEST, P1C_NODE4, P1C_GRIND_HEAD}; + +template +static void run(const Vectors &want) { + // Permutation: the four vectors through the probe kernel. + uint64_t out[4 * 16]; + uint64_t in[4 * 16]; + std::memcpy(in, P1_PERM_IN, sizeof(in)); + CUDA_HOST_FOR_EACH_THREAD(t, 4) permute_probe(in, 4, out); + check(std::memcmp(out, want.perm_out, sizeof(out)) == 0, "permutation vectors", V); + + // Mutation control: one flipped input bit must move the output. + in[15] ^= 1; + uint64_t out2[4 * 16]; + CUDA_HOST_FOR_EACH_THREAD(t, 4) permute_probe(in, 4, out2); + check(std::memcmp(out2, want.perm_out, 16 * 8) != 0, "a flipped input moves the output", V); + + // Leaves: a one-leaf coset of n felts is the leaf of those n felts. + for (int k = 0; k < (int)(sizeof(P1_LEAF_N) / sizeof(P1_LEAF_N[0])); ++k) { + const uint64_t n = P1_LEAF_N[k]; + uint64_t felts[64 + 1]; + for (uint64_t i = 0; i < n; ++i) felts[i] = i * 0x0123456789abcdefull + n; + uint64_t d[4] = {~0ull, ~0ull, ~0ull, ~0ull}; + CUDA_HOST_FOR_EACH_THREAD(t, 1) leaves_base_coset(felts, 1, n, d); + char what[64]; + std::snprintf(what, sizeof(what), "leaf of %llu felts", (unsigned long long)n); + check(std::memcmp(d, want.leaf_digest[k], 32) == 0, what, V); + } + + // Coset geometry: 4 leaves of block 16 over a strided codeword equal the + // one-leaf form over each gathered coset. + { + const uint64_t num_leaves = 4, block = 16; + uint64_t cw[64]; + for (int i = 0; i < 64; ++i) cw[i] = 0x9e3779b97f4a7c15ull * (i + 1); + uint64_t got[16]; + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) leaves_base_coset(cw, num_leaves, block, got); + bool ok = true; + for (uint64_t j = 0; j < num_leaves; ++j) { + uint64_t coset[16], want[4]; + for (uint64_t t = 0; t < block; ++t) coset[t] = cw[j + t * num_leaves]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) leaves_base_coset(coset, 1, block, want); + ok &= std::memcmp(got + 4 * j, want, 32) == 0; + } + check(ok, "base coset stride", V); + + // Ext3: element `t` of leaf `j` is the triple at `(j + t·num_leaves)·3`; + // its leaf is the base leaf over the 3·block components in order. + uint64_t cw3[4 * 4 * 3]; + for (int i = 0; i < 48; ++i) cw3[i] = 0x1234567ull * (i + 7); + uint64_t got3[16]; + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) leaves_ext3_coset(cw3, num_leaves, 4, got3); + ok = true; + for (uint64_t j = 0; j < num_leaves; ++j) { + uint64_t flat[12], want[4]; + for (uint64_t t = 0; t < 4; ++t) + for (int c = 0; c < 3; ++c) flat[3 * t + c] = cw3[(j + t * num_leaves) * 3 + c]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) leaves_base_coset(flat, 1, 12, want); + ok &= std::memcmp(got3 + 4 * j, want, 32) == 0; + } + check(ok, "ext3 coset stride", V); + } + + // The 4-ary node over the four digests perm_out[0][4c..4c+4]. + { + uint64_t parent[4]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) merkle_level4(want.perm_out[0], parent, 1); + check(std::memcmp(parent, want.node4, 32) == 0, "4-ary node", V); + } + + // Grind: with limit = head(k) + 1 for the k-th vector, the search over + // [0, 8) returns the smallest nonce whose head is below it. + for (int k = 0; k < 8; ++k) { + const uint64_t limit = want.grind_head[k] + 1; + uint64_t first = ~0ull; + for (int n = 0; n < 8; ++n) + if (want.grind_head[n] < limit) { + first = n; + break; + } + unsigned long long result = ~0ull; + CUDA_HOST_SINGLE_THREAD(); + grind_search(P1_GRIND_INNER, limit, 0, 8, &result); + char what[64]; + std::snprintf(what, sizeof(what), "grind, limit head[%d]+1", k); + check(result == first, what, V); + } +} + +// The LCG of `scripts/poseidon1/p1_params.py kat_inputs`, reduced mod p — the +// zisk_crosscheck harness's input stream. +static void zk_felts(uint64_t seed, uint64_t n, uint64_t *out) { + const uint64_t p = 0xFFFFFFFF00000001ull; + uint64_t x = seed; + for (uint64_t i = 0; i < n; ++i) { + x = x * 6364136223846793005ull + 1442695040888963407ull; + out[i] = x % p; + } +} + +// ZisK's instance against ZisK's own vectors (`p1_zisk_kat_vectors.h`): the +// width-8 permutation at both variants, the leaf hash, the row and row-pair +// geometries, and the width-8 grind. +template +static void run_w8() { + uint64_t out[4 * 8]; + uint64_t in[4 * 8]; + std::memcpy(in, ZK_PERM8_IN, sizeof(in)); + CUDA_HOST_FOR_EACH_THREAD(t, 4) p1w8_probe(in, 4, out); + check(std::memcmp(out, ZK_PERM8_OUT, sizeof(out)) == 0, "W8 permutation vectors (ZisK)", V); + in[7] ^= 1; + CUDA_HOST_FOR_EACH_THREAD(t, 4) p1w8_probe(in, 4, out); + check(std::memcmp(out, ZK_PERM8_OUT, 8 * 8) != 0, "W8: a flipped input moves the output", V); + + // Grind lanes, then the smallest nonce at ZK_GRIND_BITS. + for (uint64_t n = 0; n < 4; ++n) { + uint64_t s[8] = {ZK_GRIND_CHALLENGE[0], ZK_GRIND_CHALLENGE[1], ZK_GRIND_CHALLENGE[2], n, 0, 0, 0, 0}; + uint64_t o[8]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) p1w8_probe(s, 1, o); + check(o[0] == ZK_GRIND_LANE0[n], "W8 grind lane 0 (ZisK)", V); + } + unsigned long long result = ~0ull; + CUDA_HOST_SINGLE_THREAD(); + p1w8_grind(ZK_GRIND_CHALLENGE, 1ull << (64 - ZK_GRIND_BITS), 0, 4096, &result); + check(result == ZK_GRIND_SMALLEST_NONCE, "W8 grind smallest nonce (ZisK)", V); + result = ~0ull; + CUDA_HOST_SINGLE_THREAD(); + p1w8_grind(ZK_GRIND_CHALLENGE, 0, 0, 64, &result); + check(result == ~0ull, "W8 grind: limit 0 finds nothing", V); +} + +static void run_zisk_leaves() { + const int V = 2; + uint64_t felts[256]; + for (int k = 0; k < (int)(sizeof(ZK_LEAF_N) / sizeof(ZK_LEAF_N[0])); ++k) { + const uint64_t n = ZK_LEAF_N[k]; + zk_felts(ZK_SEED_LEAF, n, felts); + uint64_t d[4] = {~0ull, ~0ull, ~0ull, ~0ull}; + CUDA_HOST_FOR_EACH_THREAD(t, 1) p1w16_zleaves_base_coset_v2(felts, 1, n, d); + char what[64]; + std::snprintf(what, sizeof(what), "ZisK leaf of %llu felts", (unsigned long long)n); + check(std::memcmp(d, ZK_LEAF_DIGEST[k], 32) == 0, what, V); + // Mutation: the measurement leaf (padding flag and domain lane) is a + // different hash, so it must miss every ZisK vector. + CUDA_HOST_FOR_EACH_THREAD(t, 1) leaves_base_coset(felts, 1, n, d); + std::snprintf(what, sizeof(what), "measurement leaf != ZisK leaf at %llu", (unsigned long long)n); + check(std::memcmp(d, ZK_LEAF_DIGEST[k], 32) != 0, what, V); + } + + // Ext3 coset: element `t` of leaf `j` is the triple at `(j + t·num_leaves)·3`. + { + const uint64_t num_leaves = 4, block = 5; + uint64_t cw3[4 * 5 * 3]; + zk_felts(0x3e3, 60, cw3); + uint64_t got[16]; + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1w16_zleaves_ext3_coset_v2(cw3, num_leaves, block, got); + bool ok = true; + for (uint64_t j = 0; j < num_leaves; ++j) { + uint64_t flat[15], want[4]; + for (uint64_t t = 0; t < block; ++t) + for (int c = 0; c < 3; ++c) flat[3 * t + c] = cw3[(j + t * num_leaves) * 3 + c]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) p1w16_zleaves_base_coset_v2(flat, 1, 15, want); + ok &= std::memcmp(got + 4 * j, want, 32) == 0; + } + check(ok, "ZisK ext3 coset stride", V); + } + + // Rows and row pairs over a column-major matrix with a padded stride: each + // leaf is the leaf of its bit-reversed row(s), gathered column by column. + { + const uint64_t num_cols = 7, num_rows = 8, log_rows = 3, stride = 11; + uint64_t m[7 * 11]; + zk_felts(0x7015, 7 * 11, m); + uint64_t got[8 * 4]; + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1w16_zleaves_rows_v2(m, stride, num_cols, num_rows, log_rows, got); + bool ok = true; + for (uint64_t r = 0; r < num_rows; ++r) { + const uint64_t br = __brevll(r) >> (64 - log_rows); + uint64_t row[7], want[4]; + for (uint64_t c = 0; c < num_cols; ++c) row[c] = m[c * stride + br]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) p1w16_zleaves_base_coset_v2(row, 1, num_cols, want); + ok &= std::memcmp(got + 4 * r, want, 32) == 0; + } + check(ok, "ZisK row leaves", V); + + CUDA_HOST_FOR_EACH_THREAD(t, num_rows / 2) + p1w16_zleaves_row_pair_v2(m, stride, num_cols, num_rows, log_rows, got); + ok = true; + for (uint64_t j = 0; j < num_rows / 2; ++j) { + uint64_t pair[14], want[4]; + for (int h = 0; h < 2; ++h) { + const uint64_t br = __brevll(2 * j + h) >> (64 - log_rows); + for (uint64_t c = 0; c < num_cols; ++c) pair[h * num_cols + c] = m[c * stride + br]; + } + CUDA_HOST_FOR_EACH_THREAD(t, 1) p1w16_zleaves_base_coset_v2(pair, 1, 2 * num_cols, want); + ok &= std::memcmp(got + 4 * j, want, 32) == 0; + } + check(ok, "ZisK row-pair leaves", V); + } +} + +// --------------------------------------------------------------------------- +// The PRODUCTION kernels (`p1s_*`, `src/p1_stark.rs`). Nodes are big-endian +// bytes, so every check decodes them the host's way. The tree and path kernels +// are checked against ZisK's own trees (`p1_stark_kat_vectors.h`); the leaf +// kernels against the ZisK leaf above (`p1w16_zleaves_*`, itself pinned to +// ZisK's vectors) over the same felts. +// --------------------------------------------------------------------------- + +static uint64_t bswap(uint64_t x) { return p1s::bswap64(x); } + +// A node's four felts, decoded as `commitment_to_digest` decodes them. +static void node_felts(const uint8_t *node, uint64_t out[4]) { + uint64_t w[4]; + std::memcpy(w, node, 32); + for (int i = 0; i < 4; ++i) out[i] = bswap(w[i]); +} + +static void put_node(uint8_t *node, const uint64_t d[4]) { + uint64_t w[4]; + for (int i = 0; i < 4; ++i) w[i] = bswap(d[i]); + std::memcpy(node, w, 32); +} + +static uint64_t tree_nodes4(uint64_t n) { + uint64_t total = n; + while (n > 1) { + n = (n + 3) / 4; + total += n; + } + return total; +} + +// The host driver's walk (`p1_stark::build_inner_tree_levels`) with the tail +// threshold `tail_max`: per-level launches while a level has more parents, +// then the one-block tail. +static void build_tree4(uint8_t *nodes, uint64_t leaves, uint64_t tail_max) { + uint64_t child_off = tree_nodes4(leaves) - leaves, n_children = leaves; + while (n_children > 1) { + const uint64_t n_parents = (n_children + 3) / 4; + if (n_parents <= tail_max) { + CUDA_HOST_SINGLE_THREAD(); + p1s_merkle_tail4(nodes, child_off, n_children); + return; + } + const uint64_t parent_off = child_off - n_parents; + CUDA_HOST_FOR_EACH_THREAD(t, n_parents) + p1s_merkle_level4(nodes, child_off, n_children, parent_off, n_parents); + child_off = parent_off; + n_children = n_parents; + } +} + +static void run_stark_kernels() { + const int V = 2; + // Trees: ZisK's roots at 1 … 64 leaves (padded levels included), through + // the per-level kernel alone, the tail alone, and the two together. + for (int k = 0; k < (int)(sizeof(ZK_TREE_N) / sizeof(ZK_TREE_N[0])); ++k) { + const uint64_t n = ZK_TREE_N[k]; + uint64_t felts[4 * 64]; + zk_felts(ZK_SEED_TREE, 4 * n, felts); + const uint64_t tails[4] = {0, 1, 4, 256}; + for (uint64_t tail_max : tails) { + const uint64_t total = tree_nodes4(n); + uint8_t nodes[128 * 32]; + std::memset(nodes, 0xab, sizeof(nodes)); + for (uint64_t i = 0; i < n; ++i) put_node(nodes + (total - n + i) * 32, felts + 4 * i); + build_tree4(nodes, n, tail_max); + uint64_t root[4]; + node_felts(nodes, root); + char what[96]; + std::snprintf(what, sizeof(what), "p1s tree root of %llu leaves (tail at %llu)", + (unsigned long long)n, (unsigned long long)tail_max); + check(std::memcmp(root, ZK_TREE_ROOT[k], 32) == 0, what, V); + } + } + + // Paths: ZisK's 21-row tree (leaf = ZisK's linear hash of a 7-felt row); + // the gather returns its siblings per level in child order. + { + const uint64_t n = ZK_PATH_ROWS, w = ZK_PATH_WIDTH; + uint64_t rows[21 * 7]; + zk_felts(ZK_SEED_ROWS, n * w, rows); + const uint64_t total = tree_nodes4(n); + uint8_t nodes[32 * 32]; + std::memset(nodes, 0xab, sizeof(nodes)); + for (uint64_t r = 0; r < n; ++r) { + uint64_t d[4]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) p1w16_zleaves_base_coset_v2(rows + r * w, 1, w, d); + put_node(nodes + (total - n + r) * 32, d); + } + build_tree4(nodes, n, 256); + uint64_t root[4]; + node_felts(nodes, root); + check(std::memcmp(root, ZK_PATH_ROOT, 32) == 0, "p1s tree root over ZisK's path rows", V); + const int nq = (int)(sizeof(ZK_PATH_INDEX) / sizeof(ZK_PATH_INDEX[0])); + uint32_t pos[8]; + for (int q = 0; q < nq; ++q) pos[q] = (uint32_t)ZK_PATH_INDEX[q]; + uint8_t paths[8 * 3 * 3 * 32]; + CUDA_HOST_FOR_EACH_THREAD(t, nq) p1s_gather_paths4(nodes, pos, nq, n, total, 3, paths); + bool ok = true; + for (int q = 0; q < nq; ++q) + for (int level = 0; level < 3; ++level) + for (int sib = 0; sib < 3; ++sib) { + uint64_t got[4]; + node_felts(paths + ((q * 3 + level) * 3 + sib) * 32, got); + ok &= std::memcmp(got, ZK_PATH_SIBLINGS[q][level] + 4 * sib, 32) == 0; + } + check(ok, "p1s gathered paths are ZisK's", V); + // Mutation: a node count one too high moves every level's offset, so + // the gather must stop matching. + CUDA_HOST_FOR_EACH_THREAD(t, nq) p1s_gather_paths4(nodes, pos, nq, n, total + 1, 3, paths); + uint64_t got[4]; + node_felts(paths, got); + check(std::memcmp(got, ZK_PATH_SIBLINGS[0][0], 32) != 0, + "p1s gather with a wrong shape misses", V); + } + + // Leaves: each geometry equals the STARK leaf (ZisK's chain with the width + // tag [n, DOMAIN_LEAF, 0, 0] in the first capacity, host oracle + // `poseidon1_stark::linear_hash`) of the felts it reads, and differs from + // ZisK's untagged leaf of the same felts (the measurement kernels). + { + const uint64_t num_cols = 7, num_rows = 8, log_rows = 3, stride = 11; + uint64_t m[7 * 11]; + zk_felts(0x7015, 7 * 11, m); + uint8_t got[8 * 32]; + uint64_t want[8 * 4], untagged[8 * 4]; + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1s_leaves_cols_row(m, stride, num_cols, num_rows, log_rows, got); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1w16_zleaves_rows_v2(m, stride, num_cols, num_rows, log_rows, untagged); + for (uint64_t r = 0; r < num_rows; ++r) { + const uint64_t br = __brevll(r) >> (64 - log_rows); + p1w16::zisk_leaf<2, true>( + num_cols, [&](uint64_t c) { return m[c * stride + br]; }, want + 4 * r); + } + bool ok = true, apart = true; + for (uint64_t r = 0; r < num_rows; ++r) { + uint64_t d[4]; + node_felts(got + 32 * r, d); + ok &= std::memcmp(d, want + 4 * r, 32) == 0; + apart &= std::memcmp(d, untagged + 4 * r, 32) != 0; + } + check(ok, "p1s column-major row leaves", V); + check(apart, "p1s row leaves carry the width tag (!= ZisK's untagged leaf)", V); + + CUDA_HOST_FOR_EACH_THREAD(t, num_rows / 2) + p1s_leaves_cols_pair(m, stride, num_cols, num_rows, log_rows, got); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows / 2) + p1w16_zleaves_row_pair_v2(m, stride, num_cols, num_rows, log_rows, untagged); + for (uint64_t j = 0; j < num_rows / 2; ++j) { + const uint64_t br0 = __brevll(2 * j) >> (64 - log_rows); + const uint64_t br1 = __brevll(2 * j + 1) >> (64 - log_rows); + p1w16::zisk_leaf<2, true>( + 2 * num_cols, + [&](uint64_t i) { + return i < num_cols ? m[i * stride + br0] : m[(i - num_cols) * stride + br1]; + }, + want + 4 * j); + } + ok = true; + apart = true; + for (uint64_t j = 0; j < num_rows / 2; ++j) { + uint64_t d[4]; + node_felts(got + 32 * j, d); + ok &= std::memcmp(d, want + 4 * j, 32) == 0; + apart &= std::memcmp(d, untagged + 4 * j, 32) != 0; + } + check(ok, "p1s column-major row-pair leaves", V); + check(apart, "p1s row-pair leaves carry the width tag (!= ZisK's untagged leaf)", V); + + // The same matrix row-major (stride `num_cols`): the row-pair and + // one-row kernels over the whole row equal the column-major ones, + // and over a column range equal the column-major kernels on that range. + uint64_t rm[8 * 7]; + for (uint64_t r = 0; r < num_rows; ++r) + for (uint64_t c = 0; c < num_cols; ++c) rm[r * num_cols + c] = m[c * stride + r]; + uint8_t got2[8 * 32]; + CUDA_HOST_FOR_EACH_THREAD(t, num_rows / 2) + p1s_leaves_rm_pair(rm, num_cols, 0, num_cols, num_rows, log_rows, got2); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows / 2) + p1s_leaves_cols_pair(m, stride, num_cols, num_rows, log_rows, got); + check(std::memcmp(got, got2, 4 * 32) == 0, "p1s row-major row pairs = column-major", V); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1s_leaves_rm_row(rm, num_cols, 0, num_cols, num_rows, log_rows, got2); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1s_leaves_cols_row(m, stride, num_cols, num_rows, log_rows, got); + check(std::memcmp(got, got2, 8 * 32) == 0, "p1s row-major rows = column-major", V); + const uint64_t cs = 2, ce = 5; + CUDA_HOST_FOR_EACH_THREAD(t, num_rows / 2) + p1s_leaves_rm_pair(rm, num_cols, cs, ce, num_rows, log_rows, got2); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows / 2) + p1s_leaves_cols_pair(m + cs * stride, stride, ce - cs, num_rows, log_rows, got); + check(std::memcmp(got, got2, 4 * 32) == 0, "p1s row-major row-pair range", V); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1s_leaves_rm_row(rm, num_cols, cs, ce, num_rows, log_rows, got2); + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1s_leaves_cols_row(m + cs * stride, stride, ce - cs, num_rows, log_rows, got); + check(std::memcmp(got, got2, 8 * 32) == 0, "p1s row-major one-row range", V); + // Mutation: the range must matter. + CUDA_HOST_FOR_EACH_THREAD(t, num_rows) + p1s_leaves_rm_row(rm, num_cols, cs, ce + 1, num_rows, log_rows, got2); + check(std::memcmp(got, got2, 8 * 32) != 0, "p1s row-major range end moves the leaf", V); + } + + // The width-tagged leaf against the Rust oracle's pinned digests + // (`poseidon1_stark::tests::TAGGED_LEAF`: `linear_hash(felts(SEED_LEAF, n))`). + { + static const uint64_t TAGGED_N[2] = {12, 25}; + static const uint64_t TAGGED[2][4] = { + {4458286375898503949ull, 6509762154028324312ull, 350313438890380232ull, + 5326189538293941466ull}, + {14038525980236200321ull, 10177624454212016161ull, 9232529760025479834ull, + 85431238162842744ull}, + }; + uint64_t felts[32]; + for (int k = 0; k < 2; ++k) { + zk_felts(ZK_SEED_LEAF, TAGGED_N[k], felts); + uint64_t d[4]; + p1w16::zisk_leaf<2, true>(TAGGED_N[k], [&](uint64_t i) { return felts[i]; }, d); + char what[64]; + std::snprintf(what, sizeof(what), "tagged leaf of %llu felts (Rust pin)", + (unsigned long long)TAGGED_N[k]); + check(std::memcmp(d, TAGGED[k], 32) == 0, what, V); + } + } + + // FRI group leaves: leaf `j` is the STARK (width-tagged) leaf of its + // `3·group` contiguous felts. + { + uint64_t ev[4 * 8 * 3]; + zk_felts(0xf41, 4 * 8 * 3, ev); + const uint64_t groups[3] = {2, 4, 8}; + for (uint64_t group : groups) { + const uint64_t num_leaves = 4 * 8 / group; + uint8_t got[16 * 32]; + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1s_fri_group_leaves(ev, num_leaves, group, got); + bool ok = true; + for (uint64_t j = 0; j < num_leaves; ++j) { + uint64_t want[4], d[4]; + const uint64_t *leaf = ev + j * group * 3; + p1w16::zisk_leaf<2, true>(3 * group, [&](uint64_t i) { return leaf[i]; }, want); + node_felts(got + 32 * j, d); + ok &= std::memcmp(d, want, 32) == 0; + } + char what[64]; + std::snprintf(what, sizeof(what), "p1s FRI group leaves (group %llu)", + (unsigned long long)group); + check(ok, what, V); + } + } + + // The STARK grind: the smallest nonce whose width-8 permutation of + // [inner0..3, nonce, 0, 0, 0] has lane 0 below the limit — checked against + // a scan through the permutation probe. + { + const uint64_t inner[4] = {0x0123456789abcdefull, 0x1111222233334444ull, + 0x5555666677778888ull, 0x00000000deadbeefull}; + const uint64_t bits = 8, limit = 1ull << (64 - bits); + uint64_t want = ~0ull; + for (uint64_t nonce = 0; nonce < 4096 && want == ~0ull; ++nonce) { + uint64_t s[8] = {inner[0], inner[1], inner[2], inner[3], nonce, 0, 0, 0}, o[8]; + CUDA_HOST_FOR_EACH_THREAD(t, 1) p1w8_probe<2>(s, 1, o); + if (o[0] < limit) want = nonce; + } + check(want != ~0ull, "p1s grind reference finds a nonce in 4096", V); + unsigned long long result = ~0ull; + CUDA_HOST_SINGLE_THREAD(); + p1s_grind_w8(inner, limit, 0, 4096, &result); + check(result == want, "p1s grind smallest nonce", V); + result = ~0ull; + CUDA_HOST_SINGLE_THREAD(); + p1s_grind_w8(inner, 0, 0, 64, &result); + check(result == ~0ull, "p1s grind: limit 0 finds nothing", V); + } +} + + +// --------------------------------------------------------------------------- +// LEAN (V = 4): the same permutation as V = 2 by leaner +// addition chains. Its preconditions (every constant `add_c` adds is +// canonical), then V = 4 against V = 2 over edge states and a raw-word sweep, +// and the production `_lean` twins against today's kernels, byte for byte. +// --------------------------------------------------------------------------- + +static const uint64_t P_MOD = 0xFFFFFFFF00000001ull; + +static uint64_t splitmix(uint64_t &z) { + z += 0x9E3779B97F4A7C15ull; + uint64_t x = z; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ull; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBull; + return x ^ (x >> 31); +} + +static void run_lean() { + const int V = 4; + // add_c's precondition: the round constants and the Fourier constants it + // adds are canonical. + bool canon = true; + for (int r = 0; r < p1w16::ROUNDS; ++r) + for (int i = 0; i < 16; ++i) canon &= p1w16::RC[r][i] < P_MOD; + for (int k = 0; k < p1w16::PARTIAL; ++k) + for (int i = 0; i < 16; ++i) canon &= p1w16::FC[k][i] < P_MOD; + check(canon, "lean: RC and FC canonical (add_c's precondition)", V); + + // The helpers against the field, on edge words: 0, 1, p−1, p, p+1, 2^64−2^32 + // (= p−1), 2^64−2^32±1, 2^64−2, 2^64−1 (all ones), 2^63, 2^32, 2^32−1. + const uint64_t edge[] = {0ull, 1ull, P_MOD - 1, P_MOD, P_MOD + 1, 0xFFFFFFFF00000000ull, + 0xFFFFFFFEFFFFFFFFull, 0xFFFFFFFF00000002ull, ~0ull - 1, ~0ull, + 1ull << 63, 1ull << 32, 0xFFFFFFFFull, 0x8000000080000000ull}; + const int ne = (int)(sizeof(edge) / sizeof(edge[0])); + auto modp = [](unsigned __int128 x) { return (uint64_t)(x % P_MOD); }; + bool ok = true; + for (int a = 0; a < ne; ++a) { + // div16: 16·y ≡ x. + ok &= modp((unsigned __int128)p1w16::div16(edge[a]) * 16) == modp(edge[a]); + for (int b = 0; b < ne; ++b) { + const uint64_t c = edge[b] % P_MOD; // add_c's c is canonical + ok &= modp(p1w16::add_c(edge[a], c)) == modp((unsigned __int128)edge[a] + c); + for (int d = 0; d < ne; ++d) + ok &= modp(p1w16::add3(edge[a], c, edge[d])) == + modp((unsigned __int128)edge[a] + c + edge[d]); + } + } + { + uint64_t s[16]; + for (int pat = 0; pat < ne; ++pat) { + unsigned __int128 want = 0; + for (int i = 0; i < 16; ++i) { + s[i] = edge[(pat + i) % ne]; + want += s[i]; + } + ok &= modp(p1w16::sum16(s)) == modp(want); + } + for (int i = 0; i < 16; ++i) s[i] = ~0ull; // all-ones lanes: 15 carries + ok &= modp(p1w16::sum16(s)) == modp((unsigned __int128)(~0ull) * 16); + } + { + uint64_t z = 0x1ea4; + for (int n = 0; n < 200000; ++n) { + const uint64_t a = splitmix(z), c = splitmix(z) % P_MOD, d = splitmix(z); + ok &= modp(p1w16::add_c(a, c)) == modp((unsigned __int128)a + c); + ok &= modp(p1w16::add3(a, c, d)) == modp((unsigned __int128)a + c + d); + ok &= modp((unsigned __int128)p1w16::div16(a) * 16) == modp(a); + } + } + check(ok, "lean: add_c, add3, sum16, div16 against the field (edge words + 200k raw)", V); + + // The permutation: V = 4 = V = 2 on edge states (every lane one edge word; + // mixed edges; all-ones) and on raw-word states. + { + uint64_t in[64 * 16], o2[64 * 16], o4[64 * 16]; + int n = 0; + for (int e = 0; e < ne; ++e, ++n) + for (int i = 0; i < 16; ++i) in[n * 16 + i] = edge[e]; + for (int e = 0; e < ne; ++e, ++n) + for (int i = 0; i < 16; ++i) in[n * 16 + i] = edge[(e + i) % ne]; + uint64_t z = 0x51de; + while (n < 64) { + for (int i = 0; i < 16; ++i) in[n * 16 + i] = splitmix(z); + ++n; + } + CUDA_HOST_FOR_EACH_THREAD(t, 64) permute_probe<2>(in, 64, o2); + CUDA_HOST_FOR_EACH_THREAD(t, 64) permute_probe<4>(in, 64, o4); + check(std::memcmp(o2, o4, sizeof(o2)) == 0, "lean permutation = v2 (edge + raw states)", V); + uint64_t z2 = 0xfeed; + bool all = true; + for (int rep = 0; rep < 64; ++rep) { + for (int k = 0; k < 64 * 16; ++k) in[k] = splitmix(z2); + CUDA_HOST_FOR_EACH_THREAD(t, 64) permute_probe<2>(in, 64, o2); + CUDA_HOST_FOR_EACH_THREAD(t, 64) permute_probe<4>(in, 64, o4); + all &= std::memcmp(o2, o4, sizeof(o2)) == 0; + } + check(all, "lean permutation = v2 (4096 raw states)", V); + } + + // The production twins, byte for byte: coset leaves (both orders, ext3), + // column-row and row-pair leaves, and whole 4-ary trees (levels + tail). + { + const uint64_t block = 64, num_leaves = 16, leaf_bits = 4, n = block * num_leaves; + static uint64_t cw[64 * 16 * 3]; + uint64_t z = 0xc0de; + for (uint64_t i = 0; i < 3 * n; ++i) cw[i] = (i % 7 == 0) ? edge[i % ne] : splitmix(z); + static uint8_t a[16 * 32], b[16 * 32]; + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1s_leaves_base_coset_rev(cw, num_leaves, block, leaf_bits, a); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1s_leaves_base_coset_rev_lean(cw, num_leaves, block, leaf_bits, b); + check(std::memcmp(a, b, sizeof(a)) == 0, "lean twin: coset leaves, coset order", V); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1s_leaves_base_coset(cw, num_leaves, block, a); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1s_leaves_base_coset_lean(cw, num_leaves, block, b); + check(std::memcmp(a, b, sizeof(a)) == 0, "lean twin: coset leaves, natural order", V); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1s_leaves_ext3_coset(cw, num_leaves, block, a); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) p1s_leaves_ext3_coset_lean(cw, num_leaves, block, b); + check(std::memcmp(a, b, sizeof(a)) == 0, "lean twin: ext3 coset leaves", V); + // 13 columns of 16 rows (log 4): 2 chained blocks plus a tail. + CUDA_HOST_FOR_EACH_THREAD(t, 16) p1s_leaves_cols_row(cw, 16, 13, 16, 4, a); + CUDA_HOST_FOR_EACH_THREAD(t, 16) p1s_leaves_cols_row_lean(cw, 16, 13, 16, 4, b); + check(std::memcmp(a, b, sizeof(a)) == 0, "lean twin: column-row leaves", V); + CUDA_HOST_FOR_EACH_THREAD(t, 8) p1s_leaves_cols_pair(cw, 16, 13, 16, 4, a); + CUDA_HOST_FOR_EACH_THREAD(t, 8) p1s_leaves_cols_pair_lean(cw, 16, 13, 16, 4, b); + check(std::memcmp(a, b, 8 * 32) == 0, "lean twin: row-pair leaves", V); + // Trees over 64 leaves (3 levels; a padded group at 23 leaves). + const uint64_t sizes[2] = {64, 23}; + for (uint64_t leaves : sizes) { + const uint64_t total = tree_nodes4(leaves); + static uint8_t ta[128 * 32], tb[128 * 32]; + std::memset(ta, 0, sizeof(ta)); + for (uint64_t j = 0; j < leaves; ++j) { + uint64_t d[4] = {splitmix(z) % P_MOD, splitmix(z) % P_MOD, splitmix(z) % P_MOD, + splitmix(z) % P_MOD}; + put_node(ta + (total - leaves + j) * 32, d); + } + std::memcpy(tb, ta, sizeof(ta)); + // Today's: two levels by launch, then the tail. + uint64_t child_off = total - leaves, nc = leaves; + for (int lvl = 0; lvl < 3 && nc > 1; ++lvl) { + const uint64_t np = (nc + 3) / 4, po = child_off - np; + if (lvl < 1) { + CUDA_HOST_FOR_EACH_THREAD(t, np) p1s_merkle_level4(ta, child_off, nc, po, np); + CUDA_HOST_FOR_EACH_THREAD(t, np) p1s_merkle_level4_lean(tb, child_off, nc, po, np); + child_off = po; + nc = np; + } else { + CUDA_HOST_SINGLE_THREAD(); + p1s_merkle_tail4(ta, child_off, nc); + CUDA_HOST_SINGLE_THREAD(); + p1s_merkle_tail4_lean(tb, child_off, nc); + break; + } + } + char what[64]; + std::snprintf(what, sizeof(what), "lean twin: 4-ary tree over %llu leaves", (unsigned long long)leaves); + check(std::memcmp(ta, tb, total * 32) == 0, what, V); + } + } +} + +int main() { + run_stark_kernels(); + run_w8<1>(); + run_w8<2>(); + run_zisk_leaves(); + run<0>(CIRCULANT); + run<1>(CIRCULANT); + run<2>(CIRCULANT); + run<3>(CAUCHY); + run<4>(CIRCULANT); + run_lean(); + if (g_fail == 0) { + std::printf("p1w16 host KAT: all checks pass (circulant v0, v1, v2; Cauchy c1; ZisK leaf, " + "rows, row pairs; W8 v1, v2 and its grind; the p1s production kernels; lean l2 and its twins)\n"); + return 0; + } + std::printf("p1w16 host KAT: %d FAILED\n", g_fail); + return 1; +} diff --git a/crypto/math-cuda/tests/host_kat/p1w16_kat_vectors.h b/crypto/math-cuda/tests/host_kat/p1w16_kat_vectors.h new file mode 100644 index 000000000..b0486f2e8 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/p1w16_kat_vectors.h @@ -0,0 +1,52 @@ +// Known answers for `kernels/p1w16.cu`, from the Python reference +// (`scripts/poseidon1/p1_params.py cuda`); the permutation vectors are the ones four +// implementations agree on (`crypto::hash::poseidon1_w16::kat`). +#pragma once +#include +static const uint64_t P1_PERM_IN[4][16] = { + {0x0000000000000000ull, 0x0000000000000001ull, 0x0000000000000002ull, 0x0000000000000003ull, 0x0000000000000004ull, 0x0000000000000005ull, 0x0000000000000006ull, 0x0000000000000007ull, 0x0000000000000008ull, 0x0000000000000009ull, 0x000000000000000aull, 0x000000000000000bull, 0x000000000000000cull, 0x000000000000000dull, 0x000000000000000eull, 0x000000000000000full}, + {0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull, 0xffffffff00000000ull}, + {0x2ceaee21bf46bc00ull, 0xaa80754d1a1a8d4full, 0xb3c4904a6d278932ull, 0xbc69cf4276846d19ull, 0x377b2fd56a5b15b4ull, 0x64d815deeaf29df3ull, 0xf66e100db2d7d206ull, 0x1069e6a57e06665dull, 0x7be902917b70a2a8ull, 0x68901baf16ad70d7ull, 0xf2caa7c38002001aull, 0x5a55db213cf06be1ull, 0x4c88c014892416dcull, 0xa0c1ac9a9822a9fbull, 0x9e5c77e75ae9e76eull, 0x37e83672575ac1a5ull}, +}; +static const uint64_t P1_PERM_OUT[4][16] = { + {0x81c2ff551d3dd1a3ull, 0xa6f3ddabab7998e2ull, 0x4372186243233825ull, 0xd2bd8442c6cc6df7ull, 0x051a796f67578f23ull, 0x3b597e26481062caull, 0x19c3c48645baaabbull, 0x7e142fc8bf48c2ceull, 0x599ca659bfbf033full, 0x84e132ca4afd703dull, 0xb758d5776f5185c3ull, 0xaf58bfc9cb74204eull, 0x7015309157ec7e9cull, 0xe57e7f42acfff2e0ull, 0x57043250e11a11bbull, 0x656c21727540ab90ull}, + {0x078a54a5999c1f89ull, 0xba0c619c6e9a0ff0ull, 0xe2f9a11694354835ull, 0xd2c8e968b04cc9a6ull, 0x81d0e75f2327654bull, 0x20912557baff31b0ull, 0x69bc2fc6d13e33cdull, 0x032c5a376d7cfc13ull, 0xecef36c7bccf4d56ull, 0x80b4817b062829dcull, 0xf93659d1793a7952ull, 0x1f0dc30f44cb3138ull, 0x8b564149bfa10efaull, 0xc7b30100325f4879ull, 0xf694a1841608a1e8ull, 0xc595e9d1f1914be9ull}, + {0x491ef464b2792bd3ull, 0x28450225e7342b0eull, 0x1299abb383c26dbeull, 0x96056d60d5b031caull, 0xb6efaa51f392fb67ull, 0x8e428e525552bc22ull, 0xcda509fb6d800175ull, 0x4b641572c6984696ull, 0x01f93fc0af917f75ull, 0xf28ed5d932aeee76ull, 0x9d2467d3ac8a6c3aull, 0x6c6dc438a4757fd1ull, 0xc927d1ffd408297aull, 0x4ed8f3f9228f45eeull, 0x469eb77a91504639ull, 0x1e5a53837d729653ull}, + {0xb077e251291e3c50ull, 0x802ac7ee5069af14ull, 0x3b40e1590405fd9eull, 0xcfa2bc2aec5966c6ull, 0x554dd87f58bdb066ull, 0xc11dcdf422d22bb0ull, 0x6b6bcf24fa68d8bfull, 0x9cafadebd618a6c3ull, 0xf2fd3eacc96f8243ull, 0x6fd97ccffe5b67f1ull, 0xfbc70f8784230214ull, 0x6d96b2b2a82fd02eull, 0x4c73dc266ce6f482ull, 0x83d58cae06543d83ull, 0x326b8c8355c4bf52ull, 0x63ad58b9fcdcb60dull}, +}; +// Leaf of n felts, felt i = (i * 0x0123456789abcdef + n) mod p. +static const uint64_t P1_LEAF_N[7] = {0, 1, 11, 12, 13, 48, 64}; +static const uint64_t P1_LEAF_DIGEST[7][4] = { + {0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0xe415de404a2953afull, 0x9572871df89532fbull, 0x422e0e2b1e010d7cull, 0x8d52c08bd5433b76ull}, + {0x08859e42f65a9057ull, 0x540336d531267ee1ull, 0x824830ddc5b71850ull, 0x16c2f0a23db5e110ull}, + {0xaecd04f747eff7f8ull, 0xdd201f145b1f681full, 0x1b451c69be18046dull, 0xc914cef09f53f0dfull}, + {0xad8b01cd8c471f14ull, 0x8b56e5f65391eb7bull, 0xc4d5a161ef1186a5ull, 0x5b1bce9e120cb1a1ull}, + {0x80a3635a372a2313ull, 0xeab7a5d2c2bc5a03ull, 0xd24c114e223f3ec9ull, 0xd0942016f536e335ull}, + {0x964b4d22a7976fc3ull, 0x6ea18f3764f7e566ull, 0xb86810f36a0e7ce4ull, 0xdc3c8124425aa320ull}, +}; +// The 4-ary node over the four digests P1_PERM_OUT[0][4c..4c+4]. +static const uint64_t P1_NODE4[4] = {0x1ddf6497ce8bd322ull, 0xe04a80a11f0adc0bull, 0xa6a017e08ba8470aull, 0x082b6f6c76fd7bdeull}; +// Grind head: lane 0 of sponge_leaf([1, 2, 3, 4, nonce]) for nonce 0..8. +static const uint64_t P1_GRIND_INNER[4] = {1, 2, 3, 4}; +static const uint64_t P1_GRIND_HEAD[8] = {0xf531bbf388c97eb1ull, 0x5cc396c83fb3167eull, 0xe8002eb0fcef46c1ull, 0xd3fcdabf15080d86ull, 0x834b28099e44c2c0ull, 0xe309f1c5ab496ab2ull, 0xe726b46acd5ec1f8ull, 0x3ffa4087ddb82244ull}; +// The same vectors under the Grain Cauchy MDS (`CAUCHY_*` on the host). +static const uint64_t P1C_PERM_OUT[4][16] = { + {0x6a84bf02be1f328dull, 0xec14d274b936a21aull, 0xc0539d7bd4eb66deull, 0xb317ecf41fa8d55bull, 0x80b0d36f66671f8aull, 0x74a1592b9a16e832ull, 0x65e53afadfadc8c3ull, 0xa0007e5ee96ee4b2ull, 0x6dd5661a877003a8ull, 0xc36a09c2dc25cd6eull, 0xcbda3d58f7cf85f4ull, 0x34cb1d63c35596cfull, 0x4fcd09b24769e281ull, 0x6c514f906998c65dull, 0xc447035d8d71952bull, 0x591863454267826full}, + {0x6739c9388c4e2ef0ull, 0xf30b2e3b622598adull, 0xe439dd54cf0a515full, 0x7e44fb022e6afd5eull, 0x6de7468bd2bc085dull, 0xd436b57bc7acc226ull, 0xdd008c3a59bda176ull, 0x58b4a5d48066e997ull, 0x156c5b75c7cac11eull, 0x1c76a3e41d07970eull, 0x839518ae672fd49eull, 0x82f2ac09ce6b0b1bull, 0xf8217338ffb9bdb0ull, 0x3ed3402e1f33629dull, 0x8b62125da646935full, 0x3b27d003425f5481ull}, + {0x7a6e22f23f763ef2ull, 0x30e26031b6b94bb4ull, 0x269b80f1cbb79100ull, 0x2c724492563f5f30ull, 0xb7d98baf70824ef1ull, 0x99b7c1939dc54660ull, 0x6a12ad6e9d7969b3ull, 0x460f0a2fd8afc57bull, 0x68c84cb2bdce79fdull, 0x8330e46629523758ull, 0xfcdbe2677bd08018ull, 0xff0ba6fdd3cda119ull, 0xee98d495f18bf150ull, 0xeb2db2f0b53ca5c2ull, 0xf6b4a4d35797d71dull, 0xd90df4ca56b64d46ull}, + {0x558ecb3ef284e7c9ull, 0xf5e658c0d4210d0eull, 0xd156f6ecf4a2dc39ull, 0x87398c68e9b1f36cull, 0x2ac68e0cd15c0e64ull, 0x63de5d8e79208b7aull, 0xb0c975579f5615e2ull, 0x6762becbdc2ac612ull, 0x48b6834353d9ba1dull, 0xb82339f848872615ull, 0x9bba130306252a7full, 0xc7e1d0c57fb3b2deull, 0x784ad3867f9ba9b1ull, 0x35c0d82ec7544ec2ull, 0x014c39294b4f3beeull, 0x361200fea1f613fcull}, +}; +static const uint64_t P1C_LEAF_DIGEST[7][4] = { + {0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0x0f5a8171129f0e7cull, 0x369de019fc631f9aull, 0x11bcc39be3795febull, 0x938bda6a985e2cc0ull}, + {0xaace96db56764328ull, 0x77b54185481ffa22ull, 0x6bb417ebc6dab7fbull, 0x7f0d29c8a4287602ull}, + {0xe29c1b20e2c3b82cull, 0x0912444adbf7e934ull, 0xdd36f632aab73791ull, 0xd4f5efd18bbc032dull}, + {0xd54b8304dd89f2acull, 0xff1de01d10fc30a2ull, 0x22dae57424c724d4ull, 0x7aa829fb45c98951ull}, + {0x20f9f35b98a59030ull, 0x2923be0334af9725ull, 0x2b737bdff4fa2692ull, 0x0208ffa011fdb210ull}, + {0x70a38779ef6e8fcaull, 0x6b8f35f6286ba1ffull, 0xdf914b7e3a041f39ull, 0xbb45d9892022e65dull}, +}; +// The 4-ary node over the four digests P1C_PERM_OUT[0][4c..4c+4]. +static const uint64_t P1C_NODE4[4] = {0x9aecf43b4bf5d329ull, 0x02c7d6eb8e9cc56aull, 0xecd69de566b23205ull, 0xf109da81efb2a483ull}; +static const uint64_t P1C_GRIND_HEAD[8] = {0xc77181c9446adee7ull, 0x6a27860d2b6b4ce0ull, 0x339336e869b457d7ull, 0x4831192e98131bc8ull, 0x57c664111dfafb33ull, 0x1724183da9b71e4full, 0xada78e17c78069deull, 0xaa346d67efce0e64ull}; diff --git a/crypto/math-cuda/tests/host_kat/round_slot_host_kat.cpp b/crypto/math-cuda/tests/host_kat/round_slot_host_kat.cpp new file mode 100644 index 000000000..92b16368a --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/round_slot_host_kat.cpp @@ -0,0 +1,206 @@ +// Known-answer tests for the round kernels whose sums leave through a slot (`kernels/sumcheck.cu`: +// `slot_sums` and the `_slot` kernels; `LAMBDA_VM_ARGUE_ROUND_SLOT`), run on the host through +// `cuda_host_simt_shim.h` — every thread of a block a real thread, blocks one after another — so the +// last block's reduction, its barriers and the ticket run as written. No GPU, no nvcc. +// +// WHAT IT COVERS, for each slot kernel (`gkr_round_gruen_slot`, `gkr_round_gruen_real_slot`) +// against today's kernel followed by `sum_partials_ext3`, over grids of 1, 2, 3, 5, 37 and 2,048 +// blocks and 2 and 3 rows: +// 1. the slot's sums RAW-identical to today's second launch's (the proof reduces them by +// `ext3_from_raw`, but the kernel promises the same limbs); +// 2. every block's partials, and everything the kernel writes in place, raw-identical; +// 3. the ticket back at 0 after the launch, and a second launch over the same ticket finding its +// last block again (a new `seq` in the slot); +// 4. `seq` written AFTER the system fence and the sums BEFORE it (the shim's fence takes a snapshot +// of the slot). +// +// WHAT IT DOES NOT COVER, and the card tests still own: concurrent blocks racing for the ticket (the +// shim runs blocks in order), the card's memory model, mapped host memory over PCIe, nvcc. +// +// Build and run with `make test-round-slot-host-kat`. + +#include +#include +#include +#include + +#include "cuda_host_simt_shim.h" + +#define __shared__ +#define __launch_bounds__(...) +extern "C" { +uint64_t shared[3 * 1024 + 8]; +} + +// The system fence, observed: the slot as it stands when the last block fences. +static uint64_t *kat_slot = nullptr; +static std::vector kat_snapshot; +static int kat_fences = 0; +static void kat_fence_system() { + __atomic_thread_fence(__ATOMIC_SEQ_CST); + if (kat_slot) kat_snapshot.assign(kat_slot, kat_slot + 128); + ++kat_fences; +} +#define __threadfence_system() kat_fence_system() + +#include "sumcheck.cu" + +namespace { + +int failures = 0; +int checks = 0; + +void check(bool ok, const std::string &what) { + ++checks; + if (!ok) { + printf("FAIL: %s\n", what.c_str()); + ++failures; + } +} + +const uint64_t P = 0xFFFFFFFF00000001ull; +uint64_t state = 0x7A3C5E1F9B2D4680ull; +uint64_t next_u64() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; +} + +// A raw limb: every fourth one non-canonical, in [p, 2^64) — so additions carry, and an order of +// summation that differs shows in the limbs. +uint64_t raw_limb() { + uint64_t v = next_u64() % P; + if ((next_u64() & 3) == 0 && v < 0xFFFFFFFFull) v += P; + return v; +} + +std::vector random_words(size_t n) { + std::vector v(n); + for (auto &x : v) x = raw_limb(); + return v; +} + +const uint64_t SENTINEL = 0x5E5E5E5E5E5E5E5Eull; + +struct Slot { + std::vector words = std::vector(128, 0); + unsigned int ticket = 0; + Slot() { words[96] = SENTINEL; } +}; + +// Today's second launch: `sum_partials_ext3` over `rows × blocks` partials. +std::vector todays_sums(const std::vector &partials, unsigned rows, + unsigned blocks) { + std::vector out(rows * 3, 0); + uint64_t b = blocks; + simt_launch(rows, 256, [&] { sum_partials_ext3(partials.data(), b, out.data()); }); + return out; +} + +// The checks every slot launch answers: the sums, the ticket, the fence's order. `run` launches +// the slot kernel once with the given `seq`. +template +void check_slot(const std::string &what, Slot &slot, const std::vector &want, + unsigned rows, Run run) { + kat_slot = slot.words.data(); + kat_fences = 0; + kat_snapshot.clear(); + run(uint64_t(41)); + bool sums = memcmp(slot.words.data(), want.data(), rows * 3 * 8) == 0; + check(sums, what + ": the slot's sums = today's second launch's, raw"); + check(slot.words[96] == 41, what + ": seq in the slot"); + check(slot.ticket == 0, what + ": the ticket back at 0"); + check(kat_fences == 1, what + ": one system fence, by the last block"); + bool ordered = kat_snapshot.size() == 128 && kat_snapshot[96] == SENTINEL && + memcmp(kat_snapshot.data(), want.data(), rows * 3 * 8) == 0; + check(ordered, what + ": the sums out before the fence, seq after it"); + // A second launch over the same ticket finds its last block again. + kat_fences = 0; + run(uint64_t(42)); + check(slot.words[96] == 42 && kat_fences == 1 && slot.ticket == 0, + what + ": a second launch over the same ticket writes its seq"); + kat_slot = nullptr; +} + +unsigned launches = 0; + +// ── the GKR layer's round ───────────────────────────────────────────────────────────────────── + +void gkr_case(unsigned m, unsigned grid, unsigned block, bool fold, bool want_h0, bool real) { + uint64_t half = 1ull << m, quarter = fold ? half / 4 : half / 2; + uint64_t stored = real ? half / 2 + 1 : half; // a kept layer's run, or a whole one + std::vector p = random_words(2 * half * 3), q = random_words(2 * half * 3); + for (uint64_t i = stored; i < half; ++i) { + // what a kept layer's padding reads as: the whole layer holds it as raw 0/1 + p[(half + i) * 3] = p[(half + i) * 3 + 1] = p[(half + i) * 3 + 2] = 0; + q[(half + i) * 3] = 1; + q[(half + i) * 3 + 1] = q[(half + i) * 3 + 2] = 0; + } + std::vector e_hi = random_words(half * 3), e_lo = random_words(4 * 3); + uint32_t low = 2; + std::vector s = random_words(3), l = random_words(3); + unsigned rows = want_h0 ? 3 : 2; + std::vector pa = p, qa = q, pb = p, qb = q; + std::vector partials_a(3 * grid * 3, 0), partials_b(3 * grid * 3, 0); + std::string what = "gkr m " + std::to_string(m) + " grid " + std::to_string(grid) + " block " + + std::to_string(block) + (fold ? " fold" : "") + (want_h0 ? " h0" : "") + + (real ? " kept" : ""); + simt_launch(grid, block, [&] { + if (real) + gkr_round_gruen_real(pa.data(), qa.data(), half, quarter, fold, s[0], s[1], s[2], l[0], + l[1], l[2], e_hi.data(), e_lo.data(), low, want_h0, stored, + partials_a.data()); + else + gkr_round_gruen(pa.data(), qa.data(), half, quarter, fold, s[0], s[1], s[2], l[0], l[1], + l[2], e_hi.data(), e_lo.data(), low, want_h0, partials_a.data()); + }); + std::vector want = todays_sums(partials_a, rows, grid); + Slot slot; + bool first = true; + check_slot(what, slot, want, rows, [&](uint64_t seq) { + // The second launch starts from the same layer, so its sums are the first's. + std::vector pc = p, qc = q; + uint64_t *pp = first ? pb.data() : pc.data(), *qq = first ? qb.data() : qc.data(); + simt_launch(grid, block, [&] { + if (real) + gkr_round_gruen_real_slot(pp, qq, half, quarter, fold, s[0], s[1], s[2], l[0], l[1], + l[2], e_hi.data(), e_lo.data(), low, want_h0, stored, + partials_b.data(), &slot.ticket, slot.words.data(), seq); + else + gkr_round_gruen_slot(pp, qq, half, quarter, fold, s[0], s[1], s[2], l[0], l[1], + l[2], e_hi.data(), e_lo.data(), low, want_h0, + partials_b.data(), &slot.ticket, slot.words.data(), seq); + }); + if (first) { + check(partials_a == partials_b, what + ": every block's partials, raw"); + check(pa == pb && qa == qb, what + ": the halves folded in place, raw"); + } + first = false; + ++launches; + }); +} + +} // namespace + +int main() { + // The GKR round: today's and the kept layer's, fold and not, H(0) and not, over 1 / 2 / 37 + // blocks (a grid wider than the work leaves zero partials, which the reduction still sums) and + // 2,048 blocks every one of which has work (every lane of the reduction, and lanes that wrap). + for (unsigned grid : {1u, 2u, 37u}) { + gkr_case(10, grid, 64, false, false, false); + gkr_case(10, grid, 64, true, true, true); + } + gkr_case(9, 3, 256, true, false, false); + gkr_case(12, 5, 128, false, true, true); + gkr_case(17, 2048, 32, true, true, true); + printf("round slot host KAT: %u slot launches, %d checks: the slot's sums = today's second launch's (raw); " + "partials and in-place writes raw; ticket reset; seq after the fence\n", + launches, checks); + if (failures) { + printf("round slot host KAT: %d FAILED\n", failures); + return 1; + } + printf("round slot host KAT: all passed\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/rpx_canon_witness.py b/crypto/math-cuda/tests/host_kat/rpx_canon_witness.py new file mode 100644 index 000000000..7c8dfd17d --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/rpx_canon_witness.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Derives the "canonicalisation witness" row of `rpx_kat_vectors.h`. + +WHY. `rpx::permute` (kernels/rpx.cu) ends in a loop that canonicalises the +state, which is what makes device digests byte-comparable to the host's. A +known-answer check cannot see that loop unless some output lane is a raw twin +(`value + p`, in `[p, 2^64)`) before it — a 2^-32 event per lane on random +inputs. This script builds an input for which it is certain. + +HOW. The permutation's last operation is `out_i = add(m_i, ARK1[6][i])`, where +`m_i` is the M-round MDS output. With `m_i` canonical and +`m_i + ARK1[6][i] < 2^64`, the device `add` returns `m_i + ARK1[6][i]` as is; if +that sum lies in `[p, 2^64)` it is the raw twin of `sum − p`. So choose the +canonical MDS output `u` with `u_0 = p − ARK1[6][0] + 1` (raw `out_0 = p + 1`, +field value 1), fill the other eleven lanes at random, invert the MDS to get the +M-round input, and invert rounds 5..0 — `x^{1/7}` in `GF(p³)` for the E rounds, +`x^7` / `MDS⁻¹` / `x^{1/7}` / `MDS⁻¹` for the FB rounds — to get the +permutation input. `m_0` cannot itself be a twin (`u_0 + p > 2^64`), so the raw +lane is deterministic whatever representation the earlier rounds happen to +carry. + +TRUST. This is a THIRD transcription of the permutation, so it trusts nothing +about itself: before printing, it reproduces every row of the header's Table 2 +forward and inverts each one back to its input. Run from anywhere: + + python3 crypto/math-cuda/tests/host_kat/rpx_canon_witness.py + +The printed input goes into `prover/tests/rpx_host_kat_vectors.rs` +(`permutation_inputs`, the row named "canonicalisation witness"); its output +row comes from that generator, never from here. +""" +import pathlib +import random +import re + +REPO = pathlib.Path(__file__).resolve().parents[4] +P = (1 << 64) - (1 << 32) + 1 +INV_ALPHA = 10540996611094048183 # rpo.rs:96 +assert (7 * INV_ALPHA) % (P - 1) == 1 +ROW = [7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8] # rpo.rs:114 + +RPO_RS = (REPO / "prover/src/lfm/rpo.rs").read_text() + + +def constant_table(name): + m = re.search( + r"pub const %s: \[\[u64; HASH_STATE_FELTS\]; NUM_ROUNDS\] = \[(.*?)\n\];" % name, + RPO_RS, + re.S, + ) + rows = re.findall(r"\[\s*((?:\d+,\s*)+)\]", m.group(1)) + vals = [[int(x) for x in re.findall(r"\d+", r)] for r in rows] + assert len(vals) == 7 and all(len(r) == 12 for r in vals), name + return vals + + +ARK1, ARK2 = constant_table("ARK1"), constant_table("ARK2") + + +# --- the field, the MDS and its inverse, the cubic extension ------------------------- + +def mds(s): + return [sum(ROW[(j - i) % 12] * s[j] for j in range(12)) % P for i in range(12)] + + +def matrix_inverse_mod_p(m): + n = len(m) + a = [row[:] + [1 if i == j else 0 for j in range(n)] for i, row in enumerate(m)] + for col in range(n): + piv = next(r for r in range(col, n) if a[r][col] % P) + a[col], a[piv] = a[piv], a[col] + inv = pow(a[col][col], P - 2, P) + a[col] = [(v * inv) % P for v in a[col]] + for r in range(n): + if r != col and a[r][col]: + f = a[r][col] + a[r] = [(vr - f * vc) % P for vr, vc in zip(a[r], a[col])] + return [row[n:] for row in a] + + +MDS_INV = matrix_inverse_mod_p([[ROW[(j - i) % 12] for j in range(12)] for i in range(12)]) + + +def mds_inv(s): + return [sum(MDS_INV[i][j] * s[j] for j in range(12)) % P for i in range(12)] + + +def ext_mul(a, b): # rpx.rs:118-125, φ³ = φ + 1 + return [ + (a[0] * b[0] + a[1] * b[2] + a[2] * b[1]) % P, + (a[0] * b[1] + a[1] * b[0] + a[1] * b[2] + a[2] * b[1] + a[2] * b[2]) % P, + (a[0] * b[2] + a[1] * b[1] + a[2] * b[0] + a[2] * b[2]) % P, + ] + + +def ext_pow(a, e): + r, b = [1, 0, 0], a[:] + while e: + if e & 1: + r = ext_mul(r, b) + b = ext_mul(b, b) + e >>= 1 + return r + + +EXT_INV7 = pow(7, -1, P**3 - 1) # x ↦ x^7 permutes GF(p³) (rpx.rs tests), so this exists + + +# --- the permutation, forward (rpx.rs:280-316) and inverse ---------------------------- + +def add_constants(s, table, r, sign=1): + return [(v + sign * table[r][i]) % P for i, v in enumerate(s)] + + +def fb_round(s, r): + s = add_constants(mds(s), ARK1, r) + s = mds([pow(v, 7, P) for v in s]) + return [pow(v, INV_ALPHA, P) for v in add_constants(s, ARK2, r)] + + +def ext_round(s, r): + s = add_constants(s, ARK1, r) + return sum((ext_pow(s[3 * e:3 * e + 3], 7) for e in range(4)), []) + + +def final_round(s): + return add_constants(mds(s), ARK1, 6) + + +def permute(s): + for r in range(6): + s = fb_round(s, r) if r % 2 == 0 else ext_round(s, r) + return final_round(s) + + +def fb_round_inv(s, r): + s = add_constants([pow(v, 7, P) for v in s], ARK2, r, -1) + s = [pow(v, INV_ALPHA, P) for v in mds_inv(s)] + return mds_inv(add_constants(s, ARK1, r, -1)) + + +def ext_round_inv(s, r): + s = sum((ext_pow(s[3 * e:3 * e + 3], EXT_INV7) for e in range(4)), []) + return add_constants(s, ARK1, r, -1) + + +def rounds_0_to_5_inv(t): + for r in (5, 4, 3, 2, 1, 0): + t = ext_round_inv(t, r) if r % 2 == 1 else fb_round_inv(t, r) + return t + + +# --- self-check against the header's oracle table before trusting any of the above ---- + +HEADER = (REPO / "crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h").read_text() +body = HEADER.split("RPX_PERMUTATION_VECTORS[NUM_RPX_PERMUTATION_VECTORS] = {")[1].split("};")[0] +rows = re.findall(r'\{"([^"]*)",\s*\{([^}]*)\},\s*\{([^}]*)\}\}', body) +assert len(rows) >= 8, "oracle table has %d rows" % len(rows) +for name, inp, outp in rows: + x = [int(v) for v in re.findall(r"\d+", inp)] + y = [int(v) for v in re.findall(r"\d+", outp)] + assert permute(x) == y, "forward transcription disagrees with the oracle on %r" % name + t = rounds_0_to_5_inv(mds_inv(add_constants(y, ARK1, 6, -1))) + assert t == x, "inverse permutation does not round-trip on %r" % name +print("self-check: %d/%d oracle rows reproduced forward and inverted back" % (len(rows), len(rows))) + +# --- the witness ------------------------------------------------------------------------- + +c0 = ARK1[6][0] +rng = random.Random(0x4B57) # "KW"; one generator, eleven draws +u = [P - c0 + 1] + [rng.randrange(P) for _ in range(11)] +assert P - c0 <= u[0] < P - c0 + (1 << 32) - 1 +x = rounds_0_to_5_inv(mds_inv(u)) +y = permute(x) +assert y[0] == 1 and y == add_constants(u, ARK1, 6) +print("witness input :", ", ".join(str(v) for v in x)) +print("witness output:", ", ".join(str(v) for v in y), " (lane 0 raw on device: %d = p + 1)" % (u[0] + c0)) + +# The generator's hard-coded row must be exactly this input, or the header's +# witness and this derivation have drifted apart. +GENERATOR = (REPO / "prover/tests/rpx_host_kat_vectors.rs").read_text() +block = GENERATOR.split('"canonicalisation witness"', 1)[1].split("]", 1)[0] +assert [int(v) for v in re.findall(r"\d+", block)] == x, "the generator's witness row is not this derivation's" +print("generator row check: prover/tests/rpx_host_kat_vectors.rs carries this exact input") diff --git a/crypto/math-cuda/tests/host_kat/rpx_host_kat.cpp b/crypto/math-cuda/tests/host_kat/rpx_host_kat.cpp new file mode 100644 index 000000000..26580e64c --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/rpx_host_kat.cpp @@ -0,0 +1,1364 @@ +// Known-answer tests for `kernels/rpx.cu`, run on the host. +// +// WHY THIS EXISTS. The GPU parity suite runs only where a GPU does, and per-PR +// CI has none — GPU CI is merge_group-only. This compiles the real kernel +// source through `cuda_host_shim.h` and pins its arithmetic in seconds, with no +// GPU and no cargo, exactly as `blake3_host_kat.cpp` does for BLAKE3. +// +// WHAT IT COVERS: the field primitives the kernel is built from, the MDS, both +// S-boxes, the cubic extension, the seven-round schedule, the rate-8 overwrite +// duplex leaf, the Merkle parent, and the raw-vs-canonical representation. +// +// WHAT IT DOES NOT COVER, and what the GPU tests are still required for: +// whether nvcc accepts the file, and every property of execution rather than +// arithmetic — grid indexing, register pressure, local-memory spills from the +// sponge's dynamic indexing. Passing here is necessary, never sufficient. +// +// HOW THE ANCHORING LAYERS. Nothing here is checked only against itself: +// 1. The field primitives (`goldilocks::mul/add`, `ext3::dot3`) against +// schoolbook `__int128` arithmetic — the definition, no shared code. +// 2. The MDS against its per-term definition; the S-boxes against generic +// exponentiation (including `x^{1/7}` as `x^INV_ALPHA`); the cubic +// extension against naive polynomial multiplication reduced by +// `φ³ = φ + 1` — the same independent algorithms `rpx.rs`'s own tests use. +// 3. ★ EXTERNAL: RPX's FB round IS RPO's round with RPO's constants. Seven +// `fb_round(s, r)` compose to RPO256, and that composition is replayed over +// miden-crypto's nineteen `hash_elements` vectors, which nothing in this +// tree produced. That pins ARK1/ARK2, the MDS row and orientation, both +// S-box chains and the sponge lane convention from outside. +// 4. ★ THE ORACLE: the Rust host `Rpx256` (`prover/src/lfm/rpx.rs`), through +// the tables `prover/tests/rpx_host_kat_vectors.rs` prints — the bare +// permutation, the leaf sponge at seven lengths, the parent. miden +// publishes no RPX vector, so the E round and the schedule rest on this +// layer alone, as the Rust module's own provenance note says they must. +// 5. Negative controls: RPX ≠ RPO on the same state; every input lane +// reaches the output; raw (`≥ p`) and canonical inputs agree; outputs are +// canonical. +// 6. The cost model, COUNTED rather than asserted from a comment. +// 7. Every leaf kernel, both Merkle compressors and the permutation probe +// replayed thread by thread through the shim against the CPU leaf spec +// and the host parent — the read patterns and the node encoding, with the +// hash over them anchored by the layers above. +// 8. The proof-of-work grind kernel against the HOST predicate +// `stark::grinding::is_valid_nonce` over `AlgebraicDigest` — +// the nonce, its minimality, that `base` participates, and an endianness +// control (the little-endian reading of the same inner hash finds nothing +// where the big-endian one finds the nonce). +// +// Build and run with `make test-rpx-host-kat`. + +#include +#include +#include +#include +#include + +#include "cuda_host_shim.h" + +// The kernel under test. Included, not linked: the shim turns its device +// functions into host functions, and there is no other way to call them. +// RPX_HOST_OP_COUNT turns on its field-op counters (layer 6). +#define RPX_HOST_OP_COUNT +#include "rpx.cu" + +#include "rpx_kat_vectors.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char *what) { + if (!ok) { + printf("FAIL: %s\n", what); + ++failures; + } +} + +typedef unsigned __int128 u128; +const uint64_t P = 0xFFFFFFFF00000001ull; +// `7^{-1} mod (p − 1)` — rpo.rs:96. Re-derived below rather than trusted. +const uint64_t INV_ALPHA = 10540996611094048183ull; + +uint64_t canon(uint64_t x) { return x >= P ? x - P : x; } + +// =========================================================================== +// Reference arithmetic: schoolbook over `__int128`. It shares no code with the +// kernel — it is the definition the kernel's shortcuts are checked against. +// =========================================================================== + +uint64_t ref_mul(uint64_t a, uint64_t b) { + return (uint64_t)(((u128)canon(a) * (u128)canon(b)) % P); +} + +uint64_t ref_add(uint64_t a, uint64_t b) { + return (uint64_t)(((u128)canon(a) + (u128)canon(b)) % P); +} + +uint64_t ref_pow(uint64_t x, uint64_t e) { + uint64_t r = 1, b = canon(x); + while (e != 0) { + if (e & 1) r = ref_mul(r, b); + b = ref_mul(b, b); + e >>= 1; + } + return r; +} + +struct RefExt { + uint64_t c[3]; +}; + +// Naive polynomial multiplication reduced by `φ³ = φ + 1`, `φ⁴ = φ² + φ` — the +// obvious slow way, as `rpx.rs:341-352` writes it, so it shares no structure +// with the kernel's regrouped closed form. +RefExt ref_ext_mul(const RefExt &a, const RefExt &b) { + uint64_t c[5] = {0, 0, 0, 0, 0}; + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) c[i + j] = ref_add(c[i + j], ref_mul(a.c[i], b.c[j])); + } + RefExt r; + r.c[0] = ref_add(c[0], c[3]); + r.c[1] = ref_add(ref_add(c[1], c[3]), c[4]); + r.c[2] = ref_add(c[2], c[4]); + return r; +} + +RefExt ref_ext_pow(RefExt a, unsigned e) { + RefExt r = {{1, 0, 0}}; + while (e != 0) { + if (e & 1) r = ref_ext_mul(r, a); + a = ref_ext_mul(a, a); + e >>= 1; + } + return r; +} + +// The MDS as defined: `out_i = Σ_j ROW[(j − i) mod 12] · s_j`, one reduced +// field multiplication per term (rpo.rs:522-524). +void ref_mds(const uint64_t in[12], uint64_t out[12]) { + static const uint64_t ROW[12] = {7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8}; + for (int i = 0; i < 12; ++i) { + uint64_t acc = 0; + for (int j = 0; j < 12; ++j) acc = ref_add(acc, ref_mul(ROW[(j + 12 - i) % 12], in[j])); + out[i] = acc; + } +} + +// A deterministic value stream. Every fifth value is a RAW representation in +// `[p, 2^64)` — the field's non-canonical storage, which the kernel must read +// as `value − p` — so the reduction paths are exercised rather than assumed. +uint64_t splitmix(uint64_t &seed) { + seed += 0x9E3779B97F4A7C15ull; + uint64_t z = seed; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +uint64_t sample(uint64_t &seed, uint64_t i) { + uint64_t x = splitmix(seed); + // Raw values above p exist only for canonical values below 2^32 − 1. + return (i % 5 == 0) ? (x % 0xFFFFFFFFull) + P : x % P; +} + +// Values at every edge of the representation: zero, one, the modulus and its +// neighbours (raw zero, raw one), EPSILON and 2^32, the top of the u64 range. +const uint64_t EDGES[] = {0ull, 1ull, 2ull, P - 1, P, + P + 1, 0xFFFFFFFFull, 0x100000000ull, 1ull << 63, ~0ull, + ~0ull - 1, 0x0123456789ABCDEFull}; +const int NUM_EDGES = (int)(sizeof(EDGES) / sizeof(EDGES[0])); + +// =========================================================================== +// Layer 1 — the field primitives the kernel is built from. +// =========================================================================== + +void field_primitives_match_schoolbook_arithmetic() { + int checked = 0; + for (int i = 0; i < NUM_EDGES; ++i) { + for (int j = 0; j < NUM_EDGES; ++j) { + const uint64_t a = EDGES[i], b = EDGES[j]; + check(canon(goldilocks::mul(a, b)) == ref_mul(a, b), "goldilocks::mul at an edge"); + check(canon(goldilocks::add(a, b)) == ref_add(a, b), "goldilocks::add at an edge"); + // Three equal products: the 128-bit sum overflows for the large edges. + const uint64_t want = ref_add(ref_add(ref_mul(a, b), ref_mul(a, b)), ref_mul(a, b)); + check(canon(ext3::dot3(a, b, a, b, a, b)) == want, "ext3::dot3 at an edge (3 equal terms)"); + ++checked; + } + } + // The two-overflow case explicitly: six maximal operands. + { + const uint64_t m = ~0ull; + const uint64_t want = ref_add(ref_add(ref_mul(m, m), ref_mul(m, m)), ref_mul(m, m)); + check(canon(ext3::dot3(m, m, m, m, m, m)) == want, "ext3::dot3 with two 2^128 overflows"); + const uint64_t want1 = ref_add(ref_mul(m, m), ref_mul(m, m)); + check(canon(ext3::dot3(m, m, m, m, 0, 0)) == want1, "ext3::dot3 with one 2^128 overflow"); + } + uint64_t seed = 0xF1E1D; + for (int k = 0; k < 500; ++k) { + uint64_t v[6]; + for (int t = 0; t < 6; ++t) v[t] = sample(seed, (uint64_t)k * 6 + t); + const uint64_t want = + ref_add(ref_add(ref_mul(v[0], v[1]), ref_mul(v[2], v[3])), ref_mul(v[4], v[5])); + check(canon(ext3::dot3(v[0], v[1], v[2], v[3], v[4], v[5])) == want, "ext3::dot3 on random"); + check(canon(goldilocks::mul(v[0], v[1])) == ref_mul(v[0], v[1]), "goldilocks::mul on random"); + ++checked; + } + printf("field primitives vs schoolbook __int128: %d edge pairs + random, mul/add/dot3\n", checked); +} + +// =========================================================================== +// Layer 2 — the building blocks against independent algorithms. +// =========================================================================== + +void mds_matches_its_per_term_definition() { + std::vector> states; + states.push_back(std::vector(12, 0)); + states.push_back(std::vector(12, P - 1)); + states.push_back(std::vector(12, ~0ull)); // the raw maximum: the u128 bound's worst case + for (int k = 0; k < 12; ++k) { // one-hot lanes pin the orientation + std::vector s(12, 0); + s[k] = 1; + states.push_back(s); + } + uint64_t seed = 0x3D5; + for (int k = 0; k < 64; ++k) { + std::vector s(12); + for (int i = 0; i < 12; ++i) s[i] = sample(seed, (uint64_t)k * 12 + i); + states.push_back(s); + } + for (size_t n = 0; n < states.size(); ++n) { + uint64_t got[12], want[12]; + memcpy(got, states[n].data(), sizeof(got)); + rpx::mds(got); + ref_mds(states[n].data(), want); + for (int i = 0; i < 12; ++i) { + if (canon(got[i]) != want[i]) { + printf("FAIL mds state %zu lane %d: got %llu want %llu\n", n, i, + (unsigned long long)canon(got[i]), (unsigned long long)want[i]); + ++failures; + break; + } + } + } + printf("MDS (u128-property port) vs per-term definition: %zu states incl. raw-max and one-hot\n", + states.size()); +} + +void sboxes_are_the_seventh_power_and_its_inverse() { + // `7 · INV_ALPHA ≡ 1 (mod p − 1)`, re-derived as rpo.rs:797-806 does. + const u128 p_minus_one = (u128)P - 1; + check(((u128)7 * (u128)INV_ALPHA) % p_minus_one == 1, "INV_ALPHA must invert 7 in the exponent group"); + + std::vector xs(EDGES, EDGES + NUM_EDGES); + uint64_t seed = 0x5B0; + for (int k = 0; k < 48; ++k) xs.push_back(sample(seed, (uint64_t)k)); + for (size_t n = 0; n < xs.size(); ++n) { + const uint64_t x = xs[n]; + check(canon(rpx::sbox(x)) == ref_pow(x, 7), "sbox(x) must be x^7"); + check(canon(rpx::inv_sbox(x)) == ref_pow(x, INV_ALPHA), "inv_sbox(x) must be x^INV_ALPHA"); + check(canon(rpx::sbox(rpx::inv_sbox(x))) == canon(x), "sbox(inv_sbox(x)) must be x"); + check(canon(rpx::inv_sbox(rpx::sbox(x))) == canon(x), "inv_sbox(sbox(x)) must be x"); + } + check(rpx::inv_sbox(0) == 0, "inv_sbox(0) must be 0 (the padding row's fixed point)"); + check(canon(rpx::inv_sbox(P)) == 0, "inv_sbox(raw zero) must be 0"); + check(canon(rpx::inv_sbox(1)) == 1, "inv_sbox(1) must be 1"); + printf("S-boxes vs generic exponentiation: %zu values, x^7, x^{1/7}, both compositions\n", + xs.size()); +} + +void cubic_extension_matches_naive_polynomial_arithmetic() { + // The reduction rule itself, pinned on the basis: φ·φ² = φ³ = 1 + φ, and + // φ²·φ² = φ⁴ = φ + φ². + { + rpx::CubicExt phi = {0, 1, 0}, phi2 = {0, 0, 1}, one = {1, 0, 0}; + rpx::CubicExt r = rpx::ext_mul(phi, phi2); + check(canon(r.c0) == 1 && canon(r.c1) == 1 && canon(r.c2) == 0, "φ·φ² must be 1 + φ"); + r = rpx::ext_mul(phi2, phi2); + check(canon(r.c0) == 0 && canon(r.c1) == 1 && canon(r.c2) == 1, "φ²·φ² must be φ + φ²"); + r = rpx::ext_mul(phi2, one); + check(canon(r.c0) == 0 && canon(r.c1) == 0 && canon(r.c2) == 1, "1 must be the identity"); + } + std::vector as, bs; + as.push_back(RefExt{{P - 1, P - 1, P - 1}}); + bs.push_back(RefExt{{P - 1, P - 1, P - 1}}); + as.push_back(RefExt{{~0ull, ~0ull, ~0ull}}); // raw maxima + bs.push_back(RefExt{{~0ull, ~0ull, ~0ull}}); + as.push_back(RefExt{{0, 0, 0}}); + bs.push_back(RefExt{{P - 1, 0, 1}}); + uint64_t seed = 0xE3; + for (int k = 0; k < 64; ++k) { + RefExt a, b; + for (int t = 0; t < 3; ++t) { + a.c[t] = sample(seed, (uint64_t)k * 6 + t); + b.c[t] = sample(seed, (uint64_t)k * 6 + 3 + t); + } + as.push_back(a); + bs.push_back(b); + } + for (size_t n = 0; n < as.size(); ++n) { + const rpx::CubicExt a = {as[n].c[0], as[n].c[1], as[n].c[2]}; + const rpx::CubicExt b = {bs[n].c[0], bs[n].c[1], bs[n].c[2]}; + const rpx::CubicExt m = rpx::ext_mul(a, b); + const RefExt mw = ref_ext_mul(as[n], bs[n]); + check(canon(m.c0) == mw.c[0] && canon(m.c1) == mw.c[1] && canon(m.c2) == mw.c[2], + "ext_mul must equal the naive polynomial product"); + const rpx::CubicExt s = rpx::ext_square(a); + const RefExt sw = ref_ext_mul(as[n], as[n]); + check(canon(s.c0) == sw.c[0] && canon(s.c1) == sw.c[1] && canon(s.c2) == sw.c[2], + "ext_square must equal the naive square"); + const rpx::CubicExt p7 = rpx::ext_power7(a); + const RefExt pw = ref_ext_pow(as[n], 7); + check(canon(p7.c0) == pw.c[0] && canon(p7.c1) == pw.c[1] && canon(p7.c2) == pw.c[2], + "ext_power7 must equal generic exponentiation to 7"); + } + printf("cubic extension (φ³ = φ + 1) vs naive polynomial arithmetic: %zu pairs, mul/square/power7\n", + as.size()); +} + +// =========================================================================== +// Layer 3 — ★ the EXTERNAL anchor: seven FB rounds are RPO256. +// =========================================================================== + +// RPO256's permutation composed from the kernel's FB round — rpo.rs:567-583. +void rpo_permute(uint64_t s[12]) { + rpx::fb_round(s, 0); + rpx::fb_round(s, 1); + rpx::fb_round(s, 2); + rpx::fb_round(s, 3); + rpx::fb_round(s, 4); + rpx::fb_round(s, 5); + rpx::fb_round(s, 6); + for (int i = 0; i < 12; ++i) s[i] = goldilocks::canonical(s[i]); +} + +// miden's `hash_elements` in this lane convention — a transcription of the +// test-only `rpo.rs:747-767`: capacity lane 8 takes `len % 8`, the rate is +// OVERWRITTEN, the tail zero-padded, the digest is lanes 0-3. +void miden_hash_elements(const uint64_t *elements, size_t n, uint64_t out[4]) { + uint64_t state[12] = {0}; + state[8] = (uint64_t)(n % 8); + size_t i = 0; + for (size_t k = 0; k < n; ++k) { + state[i++] = elements[k]; + if (i == 8) { + rpo_permute(state); + i = 0; + } + } + if (i > 0) { + for (; i < 8; ++i) state[i] = 0; + rpo_permute(state); + } + for (int d = 0; d < 4; ++d) out[d] = state[d]; +} + +void seven_fb_rounds_reproduce_the_miden_rpo_vectors() { + check(NUM_MIDEN_HASH_ELEMENTS == 19, "miden vector table lost entries"); + int matched = 0; + for (int n = 0; n < NUM_MIDEN_HASH_ELEMENTS; ++n) { + uint64_t elements[19]; + for (int k = 0; k <= n; ++k) elements[k] = (uint64_t)k; + uint64_t got[4]; + miden_hash_elements(elements, (size_t)n + 1, got); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && got[d] == MIDEN_HASH_ELEMENTS[n][d]; + if (!ok) { + printf("FAIL miden hash_elements(0..=%d)\n got %llu %llu %llu %llu\n want %llu %llu %llu %llu\n", + n, (unsigned long long)got[0], (unsigned long long)got[1], + (unsigned long long)got[2], (unsigned long long)got[3], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][0], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][1], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][2], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][3]); + ++failures; + } else { + ++matched; + } + } + // The compress layout, pinned the way rpo.rs:789-795 pins it: one + // permutation of `[0..8 ‖ 0⁴]` is the eight-element vector, so + // `[left ‖ right ‖ zero capacity]` with left = 0..4, right = 4..8 IS + // `Rpo256::merge` — the layout `rpx::compress` builds. + { + uint64_t s[12] = {0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0}; + rpo_permute(s); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && s[d] == MIDEN_HASH_ELEMENTS[7][d]; + check(ok, "permute([0..8 ‖ 0⁴]) must be miden's eight-element vector (compress layout)"); + } + printf("★ EXTERNAL: seven fb_round(s, r) = RPO256 vs miden-crypto hash_elements: %d/19 matched\n", + matched); +} + +// =========================================================================== +// Layer 4 — ★ the Rust oracle. +// +// ⚠ Every comparison here is RAW: `s[i] == v.output[i]`, never +// `canon(s[i]) == …`. The tables are canonical by construction (the generator +// canonicalises), and `permute` ends in a canonicalisation loop that makes +// digests byte-comparable to the host's; a check that canonicalised the kernel +// side would pass with that loop deleted, and so would a raw check on outputs +// that merely happen to be canonical — all but a 2^-32 slice per lane. The +// "canonicalisation witness" row and `the_canonicalisation_loop_is_pinned…` +// below are what make the loop observable. +// =========================================================================== + +void rpx_permutation_matches_the_rust_oracle() { + check(NUM_RPX_PERMUTATION_VECTORS >= 8, + "Rust-oracle permutation table must hold >= 8 vectors (run the generator, see rpx_kat_vectors.h)"); + bool saw_zero = false, saw_p_minus_one = false; + int matched = 0; + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + const RpxPermutationVector &v = RPX_PERMUTATION_VECTORS[n]; + bool all_zero = true, all_pm1 = true; + uint64_t s[12]; + for (int i = 0; i < 12; ++i) { + s[i] = v.input[i]; + all_zero = all_zero && v.input[i] == 0; + all_pm1 = all_pm1 && v.input[i] == P - 1; + } + saw_zero = saw_zero || all_zero; + saw_p_minus_one = saw_p_minus_one || all_pm1; + rpx::permute(s); + bool ok = true; + for (int i = 0; i < 12; ++i) ok = ok && s[i] == v.output[i]; + if (!ok) { + printf("FAIL rpx permutation vector %d (%s)\n", n, v.name); + for (int i = 0; i < 12; ++i) { + if (s[i] != v.output[i]) { + printf(" lane %2d got %llu (raw) want %llu\n", i, (unsigned long long)s[i], + (unsigned long long)v.output[i]); + } + } + ++failures; + } else { + ++matched; + } + } + check(saw_zero, "the permutation table must include the all-zero state"); + check(saw_p_minus_one, "the permutation table must include the all-(p-1) state"); + printf("★ ORACLE: rpx::permute vs Rust Rpx256::permute: %d/%d vectors matched\n", matched, + NUM_RPX_PERMUTATION_VECTORS); +} + +// The array-form transcription of `algebraic_commit::sponge_leaf` (:169-184), +// over the kernel's permutation — so the STREAMING struct's block bookkeeping +// is checked against the direct transcription at every length, independently +// of which lengths the oracle table carries. +void ref_sponge_leaf(const uint64_t *felts, size_t n, uint64_t digest[4]) { + uint64_t state[12] = {0}; + state[8] = (uint64_t)(n % 8); + state[9] = 0x4C4D464Cull; // u32::from_le_bytes(b"LFML") + if (n == 0) { + for (int d = 0; d < 4; ++d) digest[d] = state[d]; + return; + } + for (size_t start = 0; start < n; start += 8) { + for (size_t lane = 0; lane < 8; ++lane) { + state[lane] = (start + lane < n) ? felts[start + lane] : 0; + } + rpx::permute(state); + } + for (int d = 0; d < 4; ++d) digest[d] = state[d]; +} + +void leaf_sponge_matches_the_rust_oracle() { + // The streaming struct against the array transcription, lengths 0..40. + { + uint64_t seed = 0x1EAF; + std::vector felts(40); + for (size_t i = 0; i < felts.size(); ++i) felts[i] = sample(seed, i); + for (size_t n = 0; n <= felts.size(); ++n) { + uint64_t got[4], want[4]; + rpx::sponge_leaf(felts.data(), n, got); + ref_sponge_leaf(felts.data(), n, want); + check(memcmp(got, want, sizeof(got)) == 0, "rpx::Sponge must equal the sponge_leaf transcription"); + } + uint64_t empty[4] = {1, 1, 1, 1}; + rpx::sponge_leaf(felts.data(), 0, empty); + check(empty[0] == 0 && empty[1] == 0 && empty[2] == 0 && empty[3] == 0, + "the empty leaf's digest is the zero rate lanes, with NO permutation"); + uint64_t one[4]; + rpx::sponge_leaf(felts.data(), 1, one); + check(one[0] != 0 || one[1] != 0 || one[2] != 0 || one[3] != 0, "a one-felt leaf must permute"); + printf("leaf: rpx::Sponge vs sponge_leaf transcription at 41 lengths (0..40)\n"); + } + // The oracle table: exactly the gate's seven lengths. + std::set lengths; + for (int n = 0; n < NUM_RPX_LEAF_VECTORS; ++n) lengths.insert(RPX_LEAF_VECTORS[n].len); + const uint32_t required[7] = {0, 1, 7, 8, 9, 16, 17}; + bool all_present = NUM_RPX_LEAF_VECTORS > 0; + for (int k = 0; k < 7; ++k) all_present = all_present && lengths.count(required[k]) == 1; + check(all_present, + "Rust-oracle leaf table must hold lengths 0, 1, 7, 8, 9, 16, 17 (run the generator, see rpx_kat_vectors.h)"); + int matched = 0; + for (int n = 0; n < NUM_RPX_LEAF_VECTORS; ++n) { + const RpxLeafVector &v = RPX_LEAF_VECTORS[n]; + check(v.len <= (uint32_t)RPX_LEAF_KAT_MAX_FELTS, "leaf vector wider than the table row"); + uint64_t got[4]; + rpx::sponge_leaf(v.felts, v.len, got); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && got[d] == v.digest[d]; + if (!ok) { + printf("FAIL rpx leaf vector len=%u\n got %llu %llu %llu %llu\n want %llu %llu %llu %llu\n", + v.len, (unsigned long long)got[0], (unsigned long long)got[1], + (unsigned long long)got[2], (unsigned long long)got[3], + (unsigned long long)v.digest[0], (unsigned long long)v.digest[1], + (unsigned long long)v.digest[2], (unsigned long long)v.digest[3]); + ++failures; + } else { + ++matched; + } + } + printf("★ ORACLE: rpx::sponge_leaf vs Rust sponge_leaf(Rpx): %d/%d lengths matched\n", matched, + NUM_RPX_LEAF_VECTORS); +} + +void parent_matches_the_rust_oracle() { + check(NUM_RPX_PARENT_VECTORS >= 1, + "Rust-oracle parent table must hold >= 1 vector (run the generator, see rpx_kat_vectors.h)"); + int matched = 0; + for (int n = 0; n < NUM_RPX_PARENT_VECTORS; ++n) { + const RpxParentVector &v = RPX_PARENT_VECTORS[n]; + uint64_t got[4]; + rpx::compress(v.left, v.right, got); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && got[d] == v.digest[d]; + if (!ok) { + printf("FAIL rpx parent vector %d (%s)\n got %llu %llu %llu %llu\n want %llu %llu %llu %llu\n", + n, v.name, (unsigned long long)got[0], (unsigned long long)got[1], + (unsigned long long)got[2], (unsigned long long)got[3], + (unsigned long long)v.digest[0], (unsigned long long)v.digest[1], + (unsigned long long)v.digest[2], (unsigned long long)v.digest[3]); + ++failures; + } else { + ++matched; + } + // Structure: a parent is ONE permutation of `[l ‖ r ‖ 0⁴]`, and the + // order of the children matters. + uint64_t s[12] = {v.left[0], v.left[1], v.left[2], v.left[3], v.right[0], v.right[1], + v.right[2], v.right[3], 0, 0, 0, 0}; + rpx::permute(s); + check(memcmp(s, got, sizeof(got)) == 0, "compress must be permute([l ‖ r ‖ 0⁴]) truncated"); + uint64_t swapped[4]; + rpx::compress(v.right, v.left, swapped); + bool same_children = memcmp(v.left, v.right, sizeof(swapped)) == 0; + check(same_children || memcmp(swapped, got, sizeof(got)) != 0, "compress(r, l) must differ from compress(l, r)"); + } + printf("★ ORACLE: rpx::compress vs Rust HasherKind::Rpx.compress: %d/%d parents matched\n", matched, + NUM_RPX_PARENT_VECTORS); +} + +// ★ The pin on the canonicalisation loop. The witness row's M-round MDS output +// lane 0 is `p − ARK1[6][0] + 1`, so the device's final `add` returns the raw +// twin `p + 1` for a field value of 1 — deterministically, since neither that +// sum nor the MDS reduction can wrap there. Replaying the rounds without the +// loop must therefore show a lane ≥ p (or the witness has gone stale and no +// longer witnesses anything), and `permute` must then return the oracle's +// canonical digits RAW — which a kernel without the loop cannot. +void the_canonicalisation_loop_is_pinned_by_the_witness() { + const RpxPermutationVector *w = nullptr; + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + if (strcmp(RPX_PERMUTATION_VECTORS[n].name, "canonicalisation witness") == 0) { + w = &RPX_PERMUTATION_VECTORS[n]; + } + } + check(w != nullptr, + "the permutation table must carry the 'canonicalisation witness' row (run the generator, see rpx_kat_vectors.h)"); + if (w == nullptr) return; + + uint64_t s[12]; + memcpy(s, w->input, sizeof(s)); + rpx::fb_round(s, 0); + rpx::ext_round(s, 1); + rpx::fb_round(s, 2); + rpx::ext_round(s, 3); + rpx::fb_round(s, 4); + rpx::ext_round(s, 5); + rpx::final_round(s, 6); + int twins = 0; + for (int i = 0; i < 12; ++i) twins += (s[i] >= P) ? 1 : 0; + check(twins > 0, "the witness must leave a raw lane >= p before the canonicalisation loop"); + check(s[0] == P + 1, "the witness's lane 0 must be the raw twin p + 1 before the loop"); + for (int i = 0; i < 12; ++i) { + check(canon(s[i]) == w->output[i], "the witness's field values must be the oracle's"); + } + + uint64_t full[12]; + memcpy(full, w->input, sizeof(full)); + rpx::permute(full); + const bool loop_present = memcmp(full, w->output, sizeof(full)) == 0; + check(loop_present, + "permute must return the witness's digits RAW — the canonicalisation loop is missing"); + if (loop_present) { + printf("★ canonicalisation pin: witness leaves %d raw lane(s) >= p before the loop; permute() returns them canonical\n", + twins); + } +} + +// =========================================================================== +// Layer 5 — negative controls and the representation. +// =========================================================================== + +void rpx_is_not_rpo() { + // rpx.rs:577-580: the two share constants, an MDS and three of seven + // rounds, so a schedule bug could collapse one into the other. + uint64_t a[12], b[12]; + for (int i = 0; i < 12; ++i) a[i] = b[i] = (uint64_t)i; + rpx::permute(a); + rpo_permute(b); + check(memcmp(a, b, sizeof(a)) != 0, "RPX must not be RPO on the same state"); + uint64_t z[12] = {0}; + rpx::permute(z); + bool nonzero = false; + for (int i = 0; i < 12; ++i) nonzero = nonzero || z[i] != 0; + check(nonzero, "with its constants present, permute(0) must not be 0"); + printf("negative control: RPX(0..12) != RPO(0..12); permute(0) != 0\n"); +} + +void raw_and_canonical_inputs_agree_and_outputs_are_canonical() { + uint64_t seed = 0xCA0; + for (int k = 0; k < 32; ++k) { + uint64_t raw[12], can[12]; + for (int i = 0; i < 12; ++i) { + // Canonical values below 2^32 − 1 have a raw twin `c + p`; alternate + // lanes between the twin and a plain canonical value. + const uint64_t c = splitmix(seed) % 0xFFFFFFFFull; + const bool twin = ((k + i) % 3) != 0; + can[i] = twin ? c : splitmix(seed) % P; + raw[i] = twin ? c + P : can[i]; + } + uint64_t r1[12], c1[12]; + memcpy(r1, raw, sizeof(r1)); + memcpy(c1, can, sizeof(c1)); + rpx::permute(r1); + rpx::permute(c1); + check(memcmp(r1, c1, sizeof(r1)) == 0, "permute(raw) must equal permute(canonical)"); + for (int i = 0; i < 12; ++i) check(c1[i] < P, "permute output must be canonical"); + + uint64_t d_raw[4], d_can[4]; + rpx::sponge_leaf(raw, 12, d_raw); + rpx::sponge_leaf(can, 12, d_can); + check(memcmp(d_raw, d_can, sizeof(d_raw)) == 0, "sponge_leaf(raw) must equal sponge_leaf(canonical)"); + + uint64_t p_raw[4], p_can[4]; + rpx::compress(raw, raw + 4, p_raw); + rpx::compress(can, can + 4, p_can); + check(memcmp(p_raw, p_can, sizeof(p_raw)) == 0, "compress(raw) must equal compress(canonical)"); + } + printf("representation: raw [p, 2^64) inputs agree with canonical; outputs canonical (32 states)\n"); +} + +void every_input_lane_reaches_the_output() { + uint64_t seed = 0x1A4E; + uint64_t base[12]; + for (int i = 0; i < 12; ++i) base[i] = splitmix(seed) % P; + uint64_t out0[12]; + memcpy(out0, base, sizeof(out0)); + rpx::permute(out0); + for (int k = 0; k < 12; ++k) { + uint64_t s[12]; + memcpy(s, base, sizeof(s)); + s[k] = (s[k] + 1) % P; + rpx::permute(s); + check(memcmp(s, out0, sizeof(s)) != 0, "changing one input lane must move the output"); + } + printf("negative control: each of the 12 input lanes moves the output\n"); +} + +// =========================================================================== +// Layer 6 — the cost model, counted. +// =========================================================================== + +struct Counted { + unsigned long long mul, dot3, add; +}; + +template +Counted count_ops(F f) { + rpx::g_ops = rpx::OpCount{0, 0, 0}; + f(); + return Counted{rpx::g_ops.mul, rpx::g_ops.dot3, rpx::g_ops.add}; +} + +void the_cost_model_is_what_the_header_claims() { + uint64_t s[12]; + for (int i = 0; i < 12; ++i) s[i] = (uint64_t)i + 1; + const Counted fb = count_ops([&] { rpx::fb_round(s, 0); }); + const Counted ext = count_ops([&] { rpx::ext_round(s, 1); }); + const Counted fin = count_ops([&] { rpx::final_round(s, 6); }); + const Counted all = count_ops([&] { rpx::permute(s); }); + const Counted rpo = count_ops([&] { rpo_permute(s); }); + const Counted inv = count_ops([&] { (void)rpx::inv_sbox(s[0]); }); + const Counted fwd = count_ops([&] { (void)rpx::sbox(s[0]); }); + const Counted emul = count_ops([&] { + rpx::CubicExt a = {s[0], s[1], s[2]}; + (void)rpx::ext_mul(a, a); + }); + + printf("op counts (Goldilocks mul | 3-term dot3 | add); MDS = 288 narrow 32x32 MACs each, uncounted:\n"); + printf(" x^7 (sbox) %4llu | %3llu | %3llu\n", fwd.mul, fwd.dot3, fwd.add); + printf(" x^{1/7} (inv_sbox) %4llu | %3llu | %3llu (63 squarings + 9 products)\n", inv.mul, + inv.dot3, inv.add); + printf(" ext_mul %4llu | %3llu | %3llu (9 wide products in 3 reductions)\n", emul.mul, + emul.dot3, emul.add); + printf(" FB round %4llu | %3llu | %3llu + 2 MDS\n", fb.mul, fb.dot3, fb.add); + printf(" E round %4llu | %3llu | %3llu (4 triples x power7)\n", ext.mul, ext.dot3, + ext.add); + printf(" M round %4llu | %3llu | %3llu + 1 MDS\n", fin.mul, fin.dot3, fin.add); + printf(" RPX permutation %4llu | %3llu | %3llu + 7 MDS (2016 MACs)\n", all.mul, all.dot3, + all.add); + printf(" RPO permutation %4llu | %3llu | %3llu + 14 MDS (4032 MACs), for comparison\n", + rpo.mul, rpo.dot3, rpo.add); + printf(" inverse S-box share of RPX field multiplications: %llu / %llu\n", 3ull * 12ull * inv.mul, + all.mul); + + check(fwd.mul == 4 && inv.mul == 72, "S-box costs must be 4 and 72 multiplications"); + check(emul.mul == 0 && emul.dot3 == 3 && emul.add == 2, "ext_mul must be 3 dot3 + 2 adds"); + check(fb.mul == 912 && fb.dot3 == 0 && fb.add == 48, "FB round must be 912 mul / 48 add"); + check(ext.mul == 0 && ext.dot3 == 48 && ext.add == 44, "E round must be 48 dot3 / 44 add"); + check(fin.mul == 0 && fin.dot3 == 0 && fin.add == 24, "M round must be 24 add"); + check(all.mul == 2736 && all.dot3 == 144 && all.add == 300, "RPX permutation must be 2736 mul / 144 dot3 / 300 add"); + check(rpo.mul == 6384 && rpo.dot3 == 0 && rpo.add == 336, "RPO permutation must be 6384 mul / 336 add"); +} + +// =========================================================================== +// Layer 7 — the leaf kernels, the Merkle compressors and the probe, replayed +// thread by thread through the shim. +// +// What a leaf hashes is the CPU `leaves_bit_reversed_grouped` sequence — +// bit-reversed rows, each column by column, an ext3 element as its three +// components — and the hash over it is the `sponge_leaf` transcription pinned +// in layer 4. So each kernel is checked for its READ PATTERN and its node +// ENCODING (`digest_to_commitment`: four canonical felts, big-endian), with the +// permutation anchored separately above. Raw `[p, 2^64)` values are fed in, +// since that is what an LDE buffer holds. +// =========================================================================== + +uint64_t reverse_index(uint64_t i, uint32_t log_n) { return __brevll(i) >> (64 - log_n); } + +// The host leaf over `felts`: `sponge_leaf`, then `digest_to_commitment`. +void expected_leaf(const std::vector &felts, uint8_t out[32]) { + uint64_t d[4]; + ref_sponge_leaf(felts.data(), felts.size(), d); + for (int i = 0; i < 4; ++i) { + const uint64_t c = canon(d[i]); + for (int b = 0; b < 8; ++b) out[i * 8 + b] = (uint8_t)(c >> (56 - 8 * b)); + } +} + +std::string hex32(const uint8_t *b) { + std::string s(64, '\0'); + for (int i = 0; i < 32; ++i) snprintf(&s[i * 2], 3, "%02x", (unsigned)b[i]); + return s; +} + +void check_leaves(const std::vector &got, const std::vector> &want, + const char *what) { + if (got.size() != want.size() * 32) { + printf("FAIL %s: leaf count %zu vs %zu\n", what, got.size() / 32, want.size()); + ++failures; + return; + } + for (size_t i = 0; i < want.size(); ++i) { + uint8_t expect[32]; + expected_leaf(want[i], expect); + if (memcmp(got.data() + i * 32, expect, 32) != 0) { + printf("FAIL %s: leaf %zu\n got %s\n want %s\n", what, i, hex32(got.data() + i * 32).c_str(), + hex32(expect).c_str()); + ++failures; + return; + } + } +} + +// The two column-major base kernels: one leaf per bit-reversed row, and one per +// bit-reversed row pair. +void base_leaf_kernels_read_the_specified_felts() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 5ull, 8ull, 17ull}) { + const uint64_t n = 1ull << log_n; + std::vector cols(num_cols * n); + uint64_t seed = log_n * 31 + num_cols; + for (size_t i = 0; i < cols.size(); ++i) cols[i] = sample(seed, i); + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + rpx_leaves_base_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + const uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) want[leaf].push_back(cols[c * n + br]); + } + check_leaves(out, want, "rpx_leaves_base_batched"); + } + { + const uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_row_pair_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + const uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < num_cols; ++c) want[leaf].push_back(cols[c * n + br]); + } + } + check_leaves(out, want, "rpx_leaves_base_row_pair_batched"); + } + } + } + printf("base leaf kernels: read pattern + node encoding match the CPU leaf spec\n"); +} + +// The ext3 kernels over the de-interleaved three-slab layout. +void ext3_leaf_kernels_read_the_specified_felts() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 3ull, 11ull}) { + const uint64_t n = 1ull << log_n; + std::vector cols(num_cols * 3 * n); + uint64_t seed = log_n * 17 + num_cols; + for (size_t i = 0; i < cols.size(); ++i) cols[i] = sample(seed, i); + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + rpx_leaves_ext3_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + const uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) want[leaf].push_back(cols[(c * 3 + k) * n + br]); + } + } + check_leaves(out, want, "rpx_leaves_ext3_batched"); + } + { + const uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_comp_poly_leaves_ext3(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int j = 0; j < 2; ++j) { + const uint64_t br = reverse_index(2 * leaf + j, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) want[leaf].push_back(cols[(c * 3 + k) * n + br]); + } + } + } + check_leaves(out, want, "rpx_comp_poly_leaves_ext3"); + } + } + } + printf("ext3 + comp-poly leaf kernels: read pattern + node encoding match the CPU leaf spec\n"); +} + +// ★ COSET leaves — the WHIR shape, and the only two kernels in this file that +// the per-table branch has no twin for, so this is the ONLY place their read +// pattern is pinned without a GPU. +// +// Leaf `j` holds the fold coset of `j`: the `block` codeword positions strided +// by `num_leaves`. Every other leaf kernel here reads a ROW; these read a +// STRIDE, which is exactly the kind of index arithmetic that compiles, runs and +// silently hashes the wrong elements. The expectation below is built from the +// definition — `codeword[j + t * num_leaves]` — not from a second call to the +// kernel. +void coset_leaf_kernels_read_the_specified_felts() { + // `block` is `2^log_folding`; 16 is the shipped posture, the others bracket + // it. `num_leaves` deliberately includes a value that is not a multiple of + // any block, to catch a bound computed from the wrong quantity. + for (uint64_t block : {2ull, 4ull, 16ull}) { + for (uint64_t num_leaves : {1ull, 2ull, 8ull, 13ull}) { + // --- base field ------------------------------------------------- + { + std::vector codeword(num_leaves * block); + uint64_t seed = 0xC05E7; + for (size_t i = 0; i < codeword.size(); ++i) codeword[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_coset(codeword.data(), num_leaves, block, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t j = 0; j < num_leaves; ++j) { + for (uint64_t t = 0; t < block; ++t) { + want[j].push_back(codeword[j + t * num_leaves]); + } + } + check_leaves(out, want, "rpx_leaves_base_coset"); + } + // --- cubic extension -------------------------------------------- + { + std::vector codeword(num_leaves * block * 3); + uint64_t seed = 0xC05E8; + for (size_t i = 0; i < codeword.size(); ++i) codeword[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_ext3_coset(codeword.data(), num_leaves, block, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t j = 0; j < num_leaves; ++j) { + for (uint64_t t = 0; t < block; ++t) { + const uint64_t *at = codeword.data() + (j + t * num_leaves) * 3; + for (int k = 0; k < 3; ++k) want[j].push_back(at[k]); + } + } + check_leaves(out, want, "rpx_leaves_ext3_coset"); + } + } + } + + // ✓ A coset is NOT a contiguous run. At block > 1 the strided read and the + // contiguous one differ, so a kernel that had dropped the stride would have + // passed everything above only if `num_leaves == 1`. This is the control + // that says the stride is really being exercised. + { + const uint64_t num_leaves = 4, block = 4; + std::vector codeword(num_leaves * block); + uint64_t seed = 0xC05E9; + for (size_t i = 0; i < codeword.size(); ++i) codeword[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_coset(codeword.data(), num_leaves, block, out.data()); + } + std::vector contiguous(codeword.begin(), codeword.begin() + block); + uint8_t as_contiguous[32]; + expected_leaf(contiguous, as_contiguous); + if (memcmp(out.data(), as_contiguous, 32) == 0) { + printf("FAIL rpx_leaves_base_coset: leaf 0 equals the CONTIGUOUS run, so the " + "stride is not being read\n"); + ++failures; + } + } + printf("coset leaf kernels: strided read pattern + node encoding match the WHIR leaf spec\n"); +} + +// FRI leaves: two consecutive ext3 values from an interleaved vector, six felts, +// no bit reversal — the Pair backend's `hash_data`. +void fri_leaf_kernel_reads_the_specified_felts() { + for (uint64_t num_leaves : {1ull, 2ull, 8ull, 33ull}) { + std::vector evals(num_leaves * 6); + uint64_t seed = 0xF41; + for (size_t i = 0; i < evals.size(); ++i) evals[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { rpx_fri_leaves_ext3(evals.data(), num_leaves, out.data()); } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int i = 0; i < 6; ++i) want[leaf].push_back(evals[leaf * 6 + i]); + } + check_leaves(out, want, "rpx_fri_leaves_ext3"); + } + printf("FRI leaf kernel: read pattern + node encoding match the Pair backend's leaf\n"); +} + +// The row-major row-pair kernels, plain and column-ranged, every non-empty +// range. +void row_major_leaf_kernels_read_the_specified_felts() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t m : {1ull, 5ull, 13ull}) { + const uint64_t n = 1ull << log_n; + const uint64_t num_leaves = n / 2; + std::vector data(n * m); + uint64_t seed = log_n * 7 + m; + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(seed, i); + { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_row_major_row_pair(data.data(), m, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + const uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < m; ++c) want[leaf].push_back(data[br * m + c]); + } + } + check_leaves(out, want, "rpx_leaves_base_row_major_row_pair"); + } + for (uint64_t cs = 0; cs < m; ++cs) { + for (uint64_t ce = cs + 1; ce <= m; ++ce) { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_row_major_row_pair_range(data.data(), m, cs, ce, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + const uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = cs; c < ce; ++c) want[leaf].push_back(data[br * m + c]); + } + } + check_leaves(out, want, "rpx_leaves_base_row_major_row_pair_range"); + } + } + } + } + printf("row-major leaf kernels: read pattern + node encoding match the CPU leaf spec, all column ranges\n"); +} + +// The row-major ONE-ROW kernel (S2, rows_per_leaf = 1): leaf `i` absorbs the +// single row `reverse_index(i)` over `log_n` bits, every non-empty column range +// (the felt count keys the padding, so each range length is its own sponge). +void row_major_one_row_kernel_reads_the_specified_felts() { + for (uint32_t log_n : {1u, 2u, 4u, 6u}) { + for (uint64_t m : {1ull, 5ull, 13ull}) { + const uint64_t n = 1ull << log_n; + std::vector data(n * m); + uint64_t seed = log_n * 11 + m; + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(seed, i); + for (uint64_t cs = 0; cs < m; ++cs) { + for (uint64_t ce = cs + 1; ce <= m; ++ce) { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + rpx_leaves_base_row_major_row_range(data.data(), m, cs, ce, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + const uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = cs; c < ce; ++c) want[leaf].push_back(data[br * m + c]); + } + check_leaves(out, want, "rpx_leaves_base_row_major_row_range"); + } + } + } + } + printf("row-major one-row kernel: read pattern + node encoding match the CPU one-row leaf spec, all column ranges\n"); +} + +// The host parent over two nodes: decode big-endian, compress, encode. +void expected_parent(const uint8_t *left, const uint8_t *right, uint8_t out[32]) { + uint64_t l[4], r[4], d[4]; + for (int i = 0; i < 4; ++i) { + l[i] = r[i] = 0; + for (int b = 0; b < 8; ++b) { + l[i] = (l[i] << 8) | left[i * 8 + b]; + r[i] = (r[i] << 8) | right[i * 8 + b]; + } + } + rpx::compress(l, r, d); + for (int i = 0; i < 4; ++i) { + for (int b = 0; b < 8; ++b) out[i * 8 + b] = (uint8_t)(canon(d[i]) >> (56 - 8 * b)); + } +} + +// The Merkle level kernel replayed thread by thread up a 16-leaf tree, and the +// tail kernel replayed as a one-thread block (the shim's barrier is a no-op, +// so a single thread walking every pair in order is the tail's sequential +// meaning), both against the host parent over the same node buffer. +void merkle_compressors_match_the_host_parent() { + const uint64_t num_leaves = 16; + const uint64_t total = 2 * num_leaves - 1; + // Nodes must be VALID digests (canonical big-endian felts) for the decode to + // be meaningful, so the leaves are hashes of random felts, not random bytes. + std::vector leaves(num_leaves * 32); + uint64_t seed = 0x3E11; + for (uint64_t i = 0; i < num_leaves; ++i) { + std::vector f = {sample(seed, i), sample(seed, i + 1000)}; + expected_leaf(f, leaves.data() + i * 32); + } + + std::vector want(total * 32, 0); + memcpy(want.data() + (num_leaves - 1) * 32, leaves.data(), leaves.size()); + for (uint64_t parent = num_leaves - 1; parent-- > 0;) { + expected_parent(want.data() + (2 * parent + 1) * 32, want.data() + (2 * parent + 2) * 32, + want.data() + parent * 32); + } + + // Level by level. + std::vector by_level(total * 32, 0); + memcpy(by_level.data() + (num_leaves - 1) * 32, leaves.data(), leaves.size()); + uint64_t level_begin = num_leaves - 1; + while (level_begin != 0) { + const uint64_t new_begin = level_begin / 2; + const uint64_t n_pairs = level_begin - new_begin; + CUDA_HOST_FOR_EACH_THREAD(t, n_pairs) { rpx_merkle_level(by_level.data(), new_begin, n_pairs); } + level_begin = new_begin; + } + check(by_level == want, "rpx_merkle_level must reproduce the host tree"); + + // The tail, in one go. + std::vector by_tail(total * 32, 0); + memcpy(by_tail.data() + (num_leaves - 1) * 32, leaves.data(), leaves.size()); + blockIdx.x = 0; + threadIdx.x = 0; + blockDim.x = 1; + rpx_merkle_tail(by_tail.data(), num_leaves - 1); + check(by_tail == want, "rpx_merkle_tail must reproduce the host tree"); + printf("Merkle compressors: level and tail kernels reproduce the host parent over a 16-leaf tree\n"); +} + +// The permutation probe replayed over the oracle table: pins its indexing. +void permute_probe_matches_the_oracle_table() { + std::vector in(NUM_RPX_PERMUTATION_VECTORS * 12), out(NUM_RPX_PERMUTATION_VECTORS * 12, 0); + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + for (int i = 0; i < 12; ++i) in[n * 12 + i] = RPX_PERMUTATION_VECTORS[n].input[i]; + } + CUDA_HOST_FOR_EACH_THREAD(t, NUM_RPX_PERMUTATION_VECTORS) { + rpx_permute_probe(in.data(), (uint64_t)NUM_RPX_PERMUTATION_VECTORS, out.data()); + } + bool ok = true; + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + for (int i = 0; i < 12; ++i) ok = ok && out[n * 12 + i] == RPX_PERMUTATION_VECTORS[n].output[i]; + } + check(ok, "rpx_permute_probe must reproduce the oracle table, raw"); + printf("permute probe: %d oracle states reproduced through the kernel entry point\n", + NUM_RPX_PERMUTATION_VECTORS); +} + +// --------------------------------------------------------------------------- +// Layer 8 — the proof-of-work grind kernel. +// +// `rpx_grind_search` is the one kernel whose correctness is a statement about +// a HOST predicate rather than about the permutation: it has to search for the +// nonces `stark::grinding::is_valid_nonce` accepts over +// `AlgebraicDigest`, and the two reach the sponge by different +// routes — the host through a byte buffer and `felts_from_bytes`, the kernel by +// building the five felts directly. Table 5's rows are that agreement, pinned +// on the Rust side (the generator asserts the two routes match over the whole +// scanned range) and reproduced here through the kernel entry point. +// +// Driven single-threaded, so what this covers is the per-candidate arithmetic +// and the loop bounds. That the parallel `atomicMin` reduction returns the same +// answer is a property of a real launch, and belongs to the GPU test. +// --------------------------------------------------------------------------- +uint64_t run_grind(const uint64_t inner[4], uint8_t factor, uint64_t base, uint64_t count) { + const uint64_t limit = (uint64_t)1 << (64 - factor); + // ⛔⛔ `unsigned long long`, NOT `uint64_t`, AND THAT DIFFERENCE WAS THE BUG. + // The kernel takes `volatile unsigned long long *` because that is the type + // CUDA's `atomicMin` overload wants, so this object's address is handed out + // under it. `uint64_t` is `unsigned long long` on Darwin/arm64 and + // `unsigned long` on LP64 glibc — a DIFFERENT type of the same width — so on + // Linux the cast type-punned, and with `#include "rpx.cu"` putting the whole + // kernel in this translation unit, GCC 13.3 at -O2 was free under TBAA to + // assume a write through `unsigned long long *` could not touch an + // `unsigned long`, and to keep `result` in a register across the inlined call. + // + // It did. `run_grind` returned `UINT64_MAX` for every input on the box, so + // every check below that expects the SENTINEL passed vacuously while every + // check that expects a FOUND nonce failed — six rows, at every sha back to + // the gated base `c00342c1f`, while the same source passed on a clang/arm64 + // laptop where the two types coincide. Measured on the box 2026-09-20: + // `-O2` → 6 failures; `-O2 -fno-strict-aliasing` → all pass; `-O0` → all pass. + // + // ⚠ NONE OF THAT WAS EVER A STATEMENT ABOUT THE DEVICE GRIND. This file is a + // HOST replay of the kernel source through `cuda_host_shim.h`; the defect + // was in the harness holding the result, not in the kernel it was testing. + // ⛔ Do not "tidy" this back to `uint64_t`: matching the pointer type the + // kernel is given is what makes the access well-defined, and the box gate + // carries a mutation that restores `uint64_t` and requires those six rows + // back — so the tidy-up would be caught, loudly, by a red nobody wants again. + unsigned long long result = UINT64_MAX; + CUDA_HOST_SINGLE_THREAD(); + rpx_grind_search(inner, limit, base, count, (volatile unsigned long long *)&result); + return result; +} + +void grind_kernel_finds_the_nonce_the_host_predicate_accepts() { + for (int n = 0; n < NUM_RPX_GRIND_VECTORS; ++n) { + const RpxGrindVector &v = RPX_GRIND_VECTORS[n]; + char what[160]; + + // The nonce is in range: the kernel returns exactly it, and it is the + // SMALLEST — a stride or bounds defect would still return a *valid* + // nonce, just not the first one, which plain validity cannot see. + snprintf(what, sizeof what, + "rpx_grind_search must return the host's nonce %llu (seed 0x%02x, factor %u)", + (unsigned long long)v.nonce, v.seed_byte, v.factor); + check(run_grind(v.inner_felts, v.factor, 0, v.nonce + 1) == v.nonce, what); + + // One short of it: nothing in `[0, nonce)` passes, so the kernel must + // leave the sentinel alone. This is what says the nonce above is the + // first — and it exercises the not-found path the launcher's range walk + // depends on. + snprintf(what, sizeof what, + "rpx_grind_search must find nothing below %llu (seed 0x%02x, factor %u)", + (unsigned long long)v.nonce, v.seed_byte, v.factor); + check(run_grind(v.inner_felts, v.factor, 0, v.nonce) == UINT64_MAX, what); + + // Offset base: the same nonce is found when the block starts inside the + // range, which pins that `base` participates rather than being ignored. + if (v.nonce > 0) { + snprintf(what, sizeof what, + "rpx_grind_search must honour base (seed 0x%02x, factor %u)", v.seed_byte, + v.factor); + check(run_grind(v.inner_felts, v.factor, v.nonce, 1) == v.nonce, what); + } + + // ★ THE ENDIANNESS CONTROL. `felts_from_bytes` reads big-endian and + // keccak's `inner_hash_lanes` reads little-endian; feeding the kernel + // the wrong one compiles, runs, and searches a message the host never + // hashes. The same kernel on the byte-swapped inner hash must give the + // oracle's `le_nonce` — which is the sentinel for every row here, i.e. + // it finds NOTHING where the correct reading finds the nonce. + uint64_t swapped[4]; + for (int i = 0; i < 4; ++i) { + uint64_t x = v.inner_felts[i], y = 0; + for (int b = 0; b < 8; ++b) y |= ((x >> (8 * b)) & 0xffull) << (8 * (7 - b)); + swapped[i] = y; + } + snprintf(what, sizeof what, + "the LE reading must not answer the BE one (seed 0x%02x, factor %u)", v.seed_byte, + v.factor); + check(run_grind(swapped, v.factor, 0, v.nonce + 1) == v.le_nonce, what); + } + printf("grind kernel: %d oracle rows — nonce, minimality, base, and the endianness control\n", + NUM_RPX_GRIND_VECTORS); +} + +// --------------------------------------------------------------------------- +// Layer 9 — the limb multiply and every permutation variant. +// +// `mul_limb` / `sqr_limb` return a word congruent to the product and below +// 2^64, not `goldilocks::mul`'s word, so they are checked against the +// schoolbook value after canonicalising. The permutation variants must then +// reproduce the oracle table RAW, which is the property the kernels rely on: +// intermediates may differ in representation, the canonical output may not. +// On the host the limb primitives run their portable-C twin of the device PTX; +// the PTX itself is pinned on the GPU (`rpx_limb_probe`). +// --------------------------------------------------------------------------- +void limb_primitives_match_schoolbook_arithmetic() { + int checked = 0; + for (int i = 0; i < NUM_EDGES; ++i) { + for (int j = 0; j < NUM_EDGES; ++j) { + const uint64_t a = EDGES[i], b = EDGES[j]; + check(canon(rpx::mul_limb(a, b)) == ref_mul(a, b), "mul_limb at an edge"); + ++checked; + } + check(canon(rpx::sqr_limb(EDGES[i])) == ref_mul(EDGES[i], EDGES[i]), "sqr_limb at an edge"); + } + // Operands that drive the fold through each carry case: a carry out of + // step 1, a borrow from t3 in step 2, and both. + const uint64_t forced[][2] = { + {~0ull, ~0ull}, + {P - 1, P - 1}, + {0xFFFFFFFF00000000ull, ~0ull}, + {0x00000000FFFFFFFFull, ~0ull}, + {1ull << 63, 1ull << 63}, + {(1ull << 63) + 1, ~0ull - 1}, + {0xFFFFFFFF00000001ull, 0xFFFFFFFF00000001ull}, + }; + for (const auto &f : forced) { + check(canon(rpx::mul_limb(f[0], f[1])) == ref_mul(f[0], f[1]), "mul_limb at a forced carry"); + check(canon(rpx::sqr_limb(f[0])) == ref_mul(f[0], f[0]), "sqr_limb at a forced carry"); + ++checked; + } + uint64_t seed = 0x11B; + for (int k = 0; k < 200000; ++k) { + const uint64_t a = splitmix(seed), b = splitmix(seed); + if (canon(rpx::mul_limb(a, b)) != ref_mul(a, b) || canon(rpx::sqr_limb(a)) != ref_mul(a, a)) { + printf("FAIL limb primitive at a = %llu, b = %llu\n", (unsigned long long)a, + (unsigned long long)b); + ++failures; + break; + } + ++checked; + } + printf("limb mul/sqr vs schoolbook __int128: %d cases (edges, forced carries, 200k raw words)\n", + checked); +} + +template +bool variant_reproduces_the_oracle_table() { + bool ok = true; + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + uint64_t s[12]; + for (int i = 0; i < 12; ++i) s[i] = RPX_PERMUTATION_VECTORS[n].input[i]; + rpx::permute_v(s); + for (int i = 0; i < 12; ++i) ok = ok && s[i] == RPX_PERMUTATION_VECTORS[n].output[i]; + } + return ok; +} + +// The chained probe replayed thread by thread: `k` permutations per state at +// variant V must equal `k` applications of the shipped permutation, raw. +template +bool chain_probe_matches_the_shipped_permutation(Kernel kernel, uint64_t k) { + const unsigned n = 24; + std::vector in(n * 12), out(n * 12, 0); + uint64_t seed = 0xC4A1; + for (unsigned t = 0; t < n * 12; ++t) in[t] = sample(seed, t); + CUDA_HOST_FOR_EACH_THREAD(t, n) { kernel(in.data(), (uint64_t)n, k, out.data()); } + for (unsigned t = 0; t < n; ++t) { + uint64_t s[12]; + for (int i = 0; i < 12; ++i) s[i] = in[t * 12 + i]; + for (uint64_t j = 0; j < k; ++j) rpx::permute_v<0>(s); + for (int i = 0; i < 12; ++i) { + if (out[t * 12 + i] != s[i]) return false; + } + } + return true; +} + +void every_permutation_variant_reproduces_the_oracle() { + check(variant_reproduces_the_oracle_table<0>(), "variant 0 must reproduce the oracle table raw"); + check(variant_reproduces_the_oracle_table<1>(), "variant 1 (limb mul) must reproduce the oracle table raw"); + check(variant_reproduces_the_oracle_table<3>(), "variant 3 (limb mul+sqr) must reproduce the oracle table raw"); + check(variant_reproduces_the_oracle_table<4>(), "variant 4 (unrolled sqn) must reproduce the oracle table raw"); + check(variant_reproduces_the_oracle_table<5>(), "variant 5 must reproduce the oracle table raw"); + check(variant_reproduces_the_oracle_table<7>(), "variant 7 must reproduce the oracle table raw"); + check(variant_reproduces_the_oracle_table<9>(), "variant 9 (limb mul, lanes unrolled) must reproduce the oracle table raw"); + check(chain_probe_matches_the_shipped_permutation(rpx_permute_chain_probe_v0, 3), "chain probe v0"); + check(chain_probe_matches_the_shipped_permutation(rpx_permute_chain_probe_v1, 3), "chain probe v1"); + check(chain_probe_matches_the_shipped_permutation(rpx_permute_chain_probe_v3, 3), "chain probe v3"); + check(chain_probe_matches_the_shipped_permutation(rpx_permute_chain_probe_v4, 3), "chain probe v4"); + check(chain_probe_matches_the_shipped_permutation(rpx_permute_chain_probe_v5, 3), "chain probe v5"); + check(chain_probe_matches_the_shipped_permutation(rpx_permute_chain_probe_v7, 3), "chain probe v7"); + check(chain_probe_matches_the_shipped_permutation(rpx_permute_chain_probe_v9, 3), "chain probe v9"); + // A control that can fail: a chain of 2 is not a chain of 3. + const unsigned n = 4; + std::vector in(n * 12, 5), out(n * 12, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { rpx_permute_chain_probe_v1(in.data(), (uint64_t)n, 2, out.data()); } + uint64_t s[12]; + for (int i = 0; i < 12; ++i) s[i] = 5; + for (int j = 0; j < 3; ++j) rpx::permute_v<0>(s); + check(memcmp(out.data(), s, sizeof(s)) != 0, "the chain-probe comparison must be able to fail"); + printf("permutation variants 0/1/3/4/5/7/9: oracle table raw + chain probes (k=3) + a failing control\n"); +} + +} // namespace + +int main() { + printf("RPX device-kernel known-answer tests, host-compiled from crypto/math-cuda/kernels/rpx.cu\n\n"); + printf("-- layer 1/2: primitives and building blocks vs independent algorithms --\n"); + field_primitives_match_schoolbook_arithmetic(); + mds_matches_its_per_term_definition(); + sboxes_are_the_seventh_power_and_its_inverse(); + cubic_extension_matches_naive_polynomial_arithmetic(); + printf("\n-- layer 3: the external anchor --\n"); + seven_fb_rounds_reproduce_the_miden_rpo_vectors(); + printf("\n-- layer 4: the Rust oracle --\n"); + rpx_permutation_matches_the_rust_oracle(); + leaf_sponge_matches_the_rust_oracle(); + parent_matches_the_rust_oracle(); + the_canonicalisation_loop_is_pinned_by_the_witness(); + printf("\n-- layer 5: negative controls and representation --\n"); + rpx_is_not_rpo(); + raw_and_canonical_inputs_agree_and_outputs_are_canonical(); + every_input_lane_reaches_the_output(); + printf("\n-- layer 6: cost model --\n"); + the_cost_model_is_what_the_header_claims(); + printf("\n-- layer 7: leaf kernels, Merkle compressors and the probe, replayed thread by thread --\n"); + base_leaf_kernels_read_the_specified_felts(); + ext3_leaf_kernels_read_the_specified_felts(); + fri_leaf_kernel_reads_the_specified_felts(); + coset_leaf_kernels_read_the_specified_felts(); + row_major_leaf_kernels_read_the_specified_felts(); + row_major_one_row_kernel_reads_the_specified_felts(); + merkle_compressors_match_the_host_parent(); + permute_probe_matches_the_oracle_table(); + printf("\n-- layer 8: the proof-of-work grind kernel against the host predicate --\n"); + grind_kernel_finds_the_nonce_the_host_predicate_accepts(); + printf("\n-- layer 9: limb arithmetic and the permutation variants --\n"); + limb_primitives_match_schoolbook_arithmetic(); + every_permutation_variant_reproduces_the_oracle(); + if (failures != 0) { + printf("\n*** %d FAILURE(S) ***\n", failures); + return 1; + } + printf("\nALL HOST KAT CHECKS PASS\n"); + printf("NOTE: arithmetic only. nvcc acceptance and GPU execution are phase 2's GPU tests.\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h b/crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h new file mode 100644 index 000000000..c89900c7a --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h @@ -0,0 +1,195 @@ +// Known-answer vectors for the RPX device kernel (`kernels/rpx.cu`), embedded +// rather than parsed at run time — a table cannot have a zero-vector run, and +// `rpx_host_kat.cpp` asserts every count below as well. +// +// This file is DATA. Table 1 is transcribed; Tables 2-4 are printed by the Rust +// oracle and pasted. Nothing here is computed by the harness. +#pragma once +#include + +// --------------------------------------------------------------------------- +// Table 1 — miden-crypto's RPO256 `hash_elements` known-answer vectors. +// EXTERNAL: nothing in this repository produced these seventy-six numbers. +// +// Transcribed mechanically (a script over the source, not by hand) from +// `prover/src/lfm/rpo.rs:624-739` `MIDEN_HASH_ELEMENTS`, itself transcribed +// from miden-crypto `src/hash/algebraic_sponge/rescue/rpo/tests.rs`. Entry `n` +// is the digest of the field elements `[0, 1, …, n]` under miden's convention: +// capacity lane 8 = `len mod 8`, rate OVERWRITTEN, zero-padded tail, digest = +// lanes 0..4 (rpo.rs:741-767). +// +// What they pin in `rpx.cu`: RPX's FB round IS RPO's round, so seven +// `fb_round<0..7>` compose to RPO256 and must reproduce this table. That pins +// ARK1/ARK2 (all seven rows), the MDS row AND its orientation, both S-box +// exponents including the 72-step inverse chain, the u128-property MDS, and the +// sponge lane convention — externally. Entries 1-7 and 9-19 exercise padding, +// 8 and 16 the exact-block path, everything above 8 the capacity carry. +// --------------------------------------------------------------------------- +inline constexpr int NUM_MIDEN_HASH_ELEMENTS = 19; +inline constexpr uint64_t MIDEN_HASH_ELEMENTS[NUM_MIDEN_HASH_ELEMENTS][4] = { + {8563248028282119176ull, 14757918088501470722ull, 14042820149444308297ull, 7607140247535155355ull}, + {8762449007102993687ull, 4386081033660325954ull, 5000814629424193749ull, 8171580292230495897ull}, + {16710087681096729759ull, 10808706421914121430ull, 14661356949236585983ull, 5683478730832134441ull}, + {5309818427047650994ull, 17172251659920546244ull, 8288476618870804357ull, 18080473279382182941ull}, + {3647545403045515695ull, 3358383208908083302ull, 8797161010298072910ull, 2412100201132087248ull}, + {8409780526028662686ull, 214479528340808320ull, 13626616722984122219ull, 13991752159726061594ull}, + {4800410126693035096ull, 8293686005479024958ull, 16849389505608627981ull, 12129312715917897796ull}, + {5421234586123900205ull, 9738602082989433872ull, 7017816005734536787ull, 8635896173743411073ull}, + {11707446879505873182ull, 7588005580730590001ull, 4664404372972250366ull, 17613162115550587316ull}, + {6991094187713033844ull, 10140064581418506488ull, 1235093741254112241ull, 16755357411831959519ull}, + {18007834547781860956ull, 5262789089508245576ull, 4752286606024269423ull, 15626544383301396533ull}, + {5419895278045886802ull, 10747737918518643252ull, 14861255521757514163ull, 3291029997369465426ull}, + {16916426112258580265ull, 8714377345140065340ull, 14207246102129706649ull, 6226142825442954311ull}, + {7320977330193495928ull, 15630435616748408136ull, 10194509925259146809ull, 15938750299626487367ull}, + {9872217233988117092ull, 5336302253150565952ull, 9650742686075483437ull, 8725445618118634861ull}, + {12539853708112793207ull, 10831674032088582545ull, 11090804155187202889ull, 105068293543772992ull}, + {7287113073032114129ull, 6373434548664566745ull, 8097061424355177769ull, 14780666619112596652ull}, + {17147873541222871127ull, 17350918081193545524ull, 5785390176806607444ull, 12480094913955467088ull}, + {17273934282489765074ull, 8007352780590012415ull, 16690624932024962846ull, 8137543572359747206ull}, +}; + +// --------------------------------------------------------------------------- +// Tables 2-4 — THE RUST ORACLE. miden publishes no RPX known-answer table +// (rpx.rs "PROVENANCE"), so the host `Rpx256` is the oracle the kernel is +// pinned to. Printed by `prover/tests/rpx_host_kat_vectors.rs`: +// +// cargo test -p lambda-vm-prover --test rpx_host_kat_vectors -- --ignored --nocapture +// +// and pasted verbatim between the `>>> BEGIN` / `<<< END` markers. Inputs are +// derived there from fixed seeds and printed next to the outputs, so this file +// is self-contained. All values are canonical (`< p`). +// +// Table 2 — the bare permutation: all-zero, all-(p−1), `0..12`, alternating, +// two one-hot lanes, four seeded random states, and the row named +// "canonicalisation witness" — an input whose output lane 0 is the +// raw twin `p + 1` before the kernel's final canonicalisation loop +// (derived by `rpx_canon_witness.py`; the harness replays it). +// Table 3 — the leaf sponge (`algebraic_commit::sponge_leaf`) at 0, 1, 7, 8, +// 9, 16 and 17 felts; `felts[]` is zero beyond `len`. +// Table 4 — the parent `compress(l, r)`. +// Table 5 — the proof-of-work grind: the inner hash's four BIG-endian felts +// and the smallest valid nonce, at three grinding factors, taken +// through the production predicate `stark::grinding::is_valid_nonce` +// over `GrindingDigest`; `le_nonce` is what the same +// kernel answers on the LITTLE-endian reading of the same inner +// hash, and must differ. +// --------------------------------------------------------------------------- +struct RpxPermutationVector { + const char *name; + uint64_t input[12]; + uint64_t output[12]; +}; + +inline constexpr int RPX_LEAF_KAT_MAX_FELTS = 17; +struct RpxLeafVector { + uint32_t len; + uint64_t felts[RPX_LEAF_KAT_MAX_FELTS]; + uint64_t digest[4]; +}; + +struct RpxParentVector { + const char *name; + uint64_t left[4]; + uint64_t right[4]; + uint64_t digest[4]; +}; + +struct RpxGrindVector { + uint8_t seed_byte; // the 32-byte grinding seed is this byte, repeated + uint8_t factor; // limit = 1 << (64 - factor) + uint64_t inner_felts[4]; // BIG-endian reading of the 32-byte inner hash + uint64_t nonce; // the smallest nonce the host predicate accepts + uint64_t le_nonce; // the same kernel's answer on the LE reading +}; + +// >>> BEGIN RUST-ORACLE TABLES — generated by +// cargo test -p lambda-vm-prover --test rpx_host_kat_vectors -- --ignored --nocapture +// (prover/tests/rpx_host_kat_vectors.rs). Paste verbatim; do not edit by hand. + +inline constexpr int NUM_RPX_PERMUTATION_VECTORS = 11; +inline constexpr RpxPermutationVector RPX_PERMUTATION_VECTORS[NUM_RPX_PERMUTATION_VECTORS] = { + {"all-zero", + {0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {8760086638283468260ull, 18228666152919569253ull, 4041825754230271128ull, 16906183286731764961ull, 4664375192219530269ull, 271590372761485506ull, 5612474514543166805ull, 8933101171974180471ull, 1556877437237031065ull, 7026397410864970258ull, 15101742939622740655ull, 4524429088483979565ull}}, + {"all-(p-1)", + {18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull}, + {7040074528728887770ull, 10474261017970959672ull, 6160748039461781206ull, 9121740959127811013ull, 7259505444118573102ull, 6771278935515018093ull, 18386914479072470354ull, 17160039764143535473ull, 1815780993504974800ull, 17309055307915657636ull, 5977169316478634398ull, 4250629519753691035ull}}, + {"lanes 0..12", + {0ull, 1ull, 2ull, 3ull, 4ull, 5ull, 6ull, 7ull, 8ull, 9ull, 10ull, 11ull}, + {3614697924784493998ull, 4917065433670799835ull, 12893407190838344317ull, 16769932886818781879ull, 17010299523770013195ull, 9826755761378503206ull, 1872785960340665977ull, 7783788981462778586ull, 45778307605882514ull, 7437259891664617628ull, 17010253034795346176ull, 6863075881906649113ull}}, + {"alternating 0 / p-1", + {0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull}, + {12839024277220712229ull, 1805658617972785851ull, 11708832562581917975ull, 2207339757364837492ull, 457975798096500050ull, 15656130651128894835ull, 3485815494872446363ull, 10687968103458402677ull, 10384294655078062232ull, 1487178939946482695ull, 12310600107129561463ull, 18388841767871832735ull}}, + {"one-hot lane 0", + {1ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {8423002511501289529ull, 6761734748202534392ull, 17987336675889252592ull, 14012777376234247391ull, 15293807115397414812ull, 15290017247514670316ull, 10548590320248089637ull, 9459855167724924903ull, 10549768014422457033ull, 13045952392708592140ull, 3310663857881768756ull, 7584810783597460418ull}}, + {"one-hot lane 11", + {0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 1ull}, + {18436166275486246010ull, 14000894557392395452ull, 10767551609857089912ull, 12516698445112165012ull, 13131066481882004069ull, 9858979976142754244ull, 11402636824743634507ull, 10600727647028701714ull, 11200928220555719329ull, 7317761145158236061ull, 16857331551667002769ull, 16879508045812612150ull}}, + {"random #1", + {303661977215735624ull, 5244312915552057691ull, 9817756985327366386ull, 15550273871372065883ull, 5764353057648779642ull, 16198122637140758912ull, 7462824619408935181ull, 3819703627846067891ull, 10378249170554155646ull, 11473525795005675318ull, 8246620909628934680ull, 4793144044164964625ull}, + {15068850129045079395ull, 15287067578585128518ull, 13369562146120321575ull, 10561395445440413441ull, 9652992371859647144ull, 4276856065313043669ull, 5527444075954724606ull, 7786060382866009904ull, 16451772069079981395ull, 198876956612152837ull, 15815343923951857286ull, 16122126005548441717ull}}, + {"random #2", + {5204068831683694011ull, 601380814908431653ull, 258667317409904638ull, 8486618912357792900ull, 16418043790810515027ull, 10319906524521615844ull, 8286207029444254408ull, 17770698039797916230ull, 12310900488678790115ull, 11195649432216834664ull, 13332813278057623446ull, 16898620073423657296ull}, + {9523479656024648568ull, 5510889535488554715ull, 8599619832581755346ull, 3318619196771576895ull, 12581966946741818379ull, 12200018864226225973ull, 4385075405488142149ull, 8051813774684357414ull, 3019406547981393239ull, 7453667634993074437ull, 9864259903669275905ull, 6156796699962990553ull}}, + {"random #3", + {13533914130435405040ull, 15234815373149021432ull, 10183913914233800905ull, 9526239132464493568ull, 5375977297676405297ull, 5765388458641153407ull, 4908125521970473579ull, 4421030864271922041ull, 15641279279696351384ull, 16893076439662162884ull, 7253714011824234117ull, 14616467593891397000ull}, + {15514260962038810700ull, 190255547175148079ull, 15766300047716671382ull, 10145444481310349528ull, 6135237967701788176ull, 11361125511081474273ull, 9927005018743801106ull, 17211086950078547559ull, 10833199580085782023ull, 13634008743082439065ull, 6687522208929839355ull, 3545879585555314384ull}}, + {"random #4", + {389113379214421922ull, 1947929307647562990ull, 667333451960644926ull, 3487966933876559811ull, 4195385248066926332ull, 2153180418459341747ull, 2727969323864685845ull, 29633526854483411ull, 990649808851061115ull, 1355410330370587755ull, 11605520071788416946ull, 4884409355120715354ull}, + {7025469669435110295ull, 17270957437800346011ull, 13702589935335807876ull, 3666927270871270796ull, 16666721215101099684ull, 531487850530305024ull, 15550553335698242665ull, 8959489596577675281ull, 11020601500923732075ull, 16110845767020565054ull, 4778394010005480449ull, 7715575140819562371ull}}, + {"canonicalisation witness", + {15055324559807314153ull, 10242425218814686878ull, 9326602342065331773ull, 15451135068213333861ull, 17942679252967467289ull, 9284164080268346300ull, 5090350781253234438ull, 9328738269791029498ull, 18385380985273671691ull, 3238854716908013220ull, 5495049682105235955ull, 15773368383738726538ull}, + {1ull, 9023883145409261355ull, 5839950281880325605ull, 5697668523532261268ull, 13033383890974728246ull, 14801658261553133914ull, 3025695522291518949ull, 12907720598453111556ull, 14827640614007773288ull, 14642633917625231592ull, 3090884930034198616ull, 2894057710100710233ull}}, +}; + +inline constexpr int NUM_RPX_LEAF_VECTORS = 7; +inline constexpr RpxLeafVector RPX_LEAF_VECTORS[NUM_RPX_LEAF_VECTORS] = { + {0u, + {0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {0ull, 0ull, 0ull, 0ull}}, + {1u, + {14681136968691612469ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {16400186102935428425ull, 12817983163740802970ull, 13449009006350391325ull, 2209445548780258712ull}}, + {7u, + {2664695409302073823ull, 17298518342786888931ull, 17367242851809685948ull, 13566833943477212382ull, 6789339537410032387ull, 5202847705797706501ull, 6869254230765949416ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {2289345357069865559ull, 8509266780934512918ull, 13810958145049281723ull, 5769431894700133303ull}}, + {8u, + {3521541860211663897ull, 5585621328801039182ull, 3314063895810834828ull, 6286715337571703139ull, 9272399501810688383ull, 17378448552699642502ull, 9663403628134293866ull, 8225575178453385283ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {14052993739410942603ull, 8384701950754250190ull, 11473922331550289114ull, 16644313465254305812ull}}, + {9u, + {15923052634311126246ull, 10423360080185943333ull, 4604695570423031111ull, 15959212651715575539ull, 4341333374822801132ull, 3169961389438585383ull, 7059846953207312362ull, 6231597079039193598ull, 14413065529971692326ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {15453186885173297365ull, 11395279108043639065ull, 15954005188014354330ull, 2854892578083306874ull}}, + {16u, + {9660685076555889599ull, 4027567791223379602ull, 11432600011703367870ull, 6441517771629429252ull, 8272264386868866348ull, 16565648022353132158ull, 16844837242675693755ull, 12942506659476152817ull, 11839051358503478840ull, 1846358602548732379ull, 118703897581348635ull, 14480592082795401517ull, 12015885875590073011ull, 7433808365622677077ull, 13247077855319202624ull, 17837888200692576115ull, 0ull}, + {18135965004560326100ull, 1948492279228612931ull, 17772968542724134453ull, 12116464713281646840ull}}, + {17u, + {14169068543591784110ull, 12906798066534908639ull, 1898134805181953282ull, 3700382130787856361ull, 10455317549184205797ull, 1564511190292879407ull, 5954886065046464361ull, 10320234224067579215ull, 17095047743397986079ull, 8434180870595516882ull, 17706992797230203878ull, 813257427175065251ull, 13312284969041468023ull, 15899260221184366980ull, 5770785055252949875ull, 11176385994046687487ull, 8142444693260481147ull}, + {430819886588247494ull, 10400188655761849356ull, 3003730485848167815ull, 13484379440855863704ull}}, +}; + +inline constexpr int NUM_RPX_PARENT_VECTORS = 2; +inline constexpr RpxParentVector RPX_PARENT_VECTORS[NUM_RPX_PARENT_VECTORS] = { + {"digits 0..8", + {0ull, 1ull, 2ull, 3ull}, + {4ull, 5ull, 6ull, 7ull}, + {10386438340626196987ull, 10820383641790274229ull, 5711121060683785078ull, 11046870009967209474ull}}, + {"random", + {10430052842846219471ull, 4016318112082366688ull, 17186674839268073878ull, 16606021345024473049ull}, + {1405896845186672283ull, 13799610513837549656ull, 17571522367612218822ull, 18082329703565322844ull}, + {18019606657308693634ull, 10494109104368286361ull, 7943124261980338770ull, 17971490172695632899ull}}, +}; + +inline constexpr int NUM_RPX_GRIND_VECTORS = 3; +inline constexpr RpxGrindVector RPX_GRIND_VECTORS[NUM_RPX_GRIND_VECTORS] = { + {90u, 12u, + {17047917526726690733ull, 2027278666509702433ull, 4678289907902145381ull, 4242003890993108442ull}, + 1342ull, 18446744073709551615ull}, + {17u, 13u, + {3807340077325453675ull, 129745844021573959ull, 15014385560057355003ull, 944573484564438641ull}, + 300ull, 18446744073709551615ull}, + {32u, 14u, + {5597071933014793605ull, 8702110216523445336ull, 2882478612521280078ull, 9429844132731097150ull}, + 705ull, 18446744073709551615ull}, +}; +// <<< END RUST-ORACLE TABLES diff --git a/crypto/math-cuda/tests/host_kat/rpx_simt_host_kat.cpp b/crypto/math-cuda/tests/host_kat/rpx_simt_host_kat.cpp new file mode 100644 index 000000000..1d97bb993 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/rpx_simt_host_kat.cpp @@ -0,0 +1,253 @@ +// Host checks for the RPX kernels whose lanes COOPERATE — the half-warp +// permutation and Merkle kernels, and the work-queue grind — run through +// `cuda_host_simt_shim.h`, where every lane is a real thread. +// +// Every check here is against the SHIPPED kernel or the shipped device +// function, byte for byte (raw words for the permutation): these kernels exist +// to produce exactly what `permute`, `rpx_merkle_level` / `rpx_merkle_tail` and +// `rpx_grind_search` produce, faster. The shipped side is itself pinned to the +// Rust oracle by `rpx_host_kat.cpp`. +// +// Build and run with `make test-rpx-host-kat` (the second binary). What the +// GPU tests still own: nvcc acceptance, the real warp scheduler, timing. + +#include +#include +#include + +#include "cuda_host_simt_shim.h" + +#define RPX_HOST_SIMT +#include "rpx.cu" + +namespace { + +int failures = 0; + +void check(bool ok, const char *what) { + if (!ok) { + printf("FAIL: %s\n", what); + ++failures; + } +} + +const uint64_t P = 0xFFFFFFFF00000001ull; + +uint64_t splitmix(uint64_t &seed) { + seed += 0x9E3779B97F4A7C15ull; + uint64_t z = seed; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +// Every fifth word a raw representation in [p, 2^64), as an LDE buffer holds. +uint64_t sample(uint64_t &seed, uint64_t i) { + const uint64_t x = splitmix(seed); + return (i % 5 == 0) ? (x % 0xFFFFFFFFull) + P : x % P; +} + +// --------------------------------------------------------------------------- +// The half-warp permutation against `permute`, raw. +// --------------------------------------------------------------------------- +void warp_permutation_matches_the_shipped_permutation() { + // An odd count, so the last warp has a dead half. + const uint64_t n = 21; + std::vector in(n * 12), out(n * 12, ~0ull); + uint64_t seed = 0x5117; + for (uint64_t t = 0; t < n * 12; ++t) in[t] = sample(seed, t); + for (int i = 0; i < 12; ++i) in[i] = 0; // the all-zero state + for (int i = 0; i < 12; ++i) in[12 + i] = P - 1; // the all-(p-1) state + for (int i = 0; i < 12; ++i) in[24 + i] = ~0ull; // raw maxima + // 64 threads = 2 warps = 4 states per block. + simt_launch((unsigned)((n + 3) / 4), 64, + [&] { rpx_permute_warp_probe(in.data(), n, out.data()); }); + int matched = 0; + for (uint64_t t = 0; t < n; ++t) { + uint64_t s[12]; + memcpy(s, &in[t * 12], sizeof(s)); + rpx::permute(s); + if (memcmp(s, &out[t * 12], sizeof(s)) == 0) { + ++matched; + } else { + printf("FAIL warp permutation state %llu\n", (unsigned long long)t); + ++failures; + } + } + printf("half-warp permutation vs rpx::permute, raw: %d/%llu states (odd count, edges, raw words)\n", + matched, (unsigned long long)n); +} + +// Random node bytes: both kernels decode big-endian words without reducing +// them, and the permutation reads raw words as field elements, so any bytes are +// a fair input for a kernel-versus-kernel comparison. +std::vector tree_with_random_leaves(uint64_t num_leaves, uint64_t seed) { + std::vector nodes((2 * num_leaves - 1) * 32, 0); + for (uint64_t i = (num_leaves - 1) * 32; i < nodes.size(); ++i) { + nodes[i] = (uint8_t)splitmix(seed); + } + return nodes; +} + +// The shipped walk: one `rpx_merkle_level` launch per level, threads replayed. +void shipped_tree(std::vector &nodes, uint64_t num_leaves) { + uint64_t level_begin = num_leaves - 1; + while (level_begin != 0) { + const uint64_t new_begin = level_begin / 2; + const uint64_t n_pairs = level_begin - new_begin; + const unsigned grid = (unsigned)((n_pairs + 63) / 64); + simt_launch(grid, 64, [&] { rpx_merkle_level(nodes.data(), new_begin, n_pairs); }); + level_begin = new_begin; + } +} + +void warp_merkle_kernels_match_the_shipped_tree() { + int trees = 0; + for (uint64_t log_n = 1; log_n <= 7; ++log_n) { + const uint64_t num_leaves = 1ull << log_n; + std::vector want = tree_with_random_leaves(num_leaves, 0x7EE + log_n); + std::vector by_level = want, by_tail = want, mixed = want; + shipped_tree(want, num_leaves); + + // Every level through `rpx_merkle_level_warp` (2 warps per block). + uint64_t level_begin = num_leaves - 1; + while (level_begin != 0) { + const uint64_t new_begin = level_begin / 2; + const uint64_t n_pairs = level_begin - new_begin; + simt_launch((unsigned)((n_pairs + 3) / 4), 64, + [&] { rpx_merkle_level_warp(by_level.data(), new_begin, n_pairs); }); + level_begin = new_begin; + } + check(by_level == want, "rpx_merkle_level_warp must reproduce the shipped tree"); + + // The whole tree in one `rpx_merkle_tail_warp` block of 2 warps: 4 + // pairs per pass, so the wide levels take several passes. + simt_launch(1, 64, [&] { rpx_merkle_tail_warp(by_tail.data(), num_leaves - 1); }); + check(by_tail == want, "rpx_merkle_tail_warp must reproduce the shipped tree"); + + // The walker's split: shipped levels while wide, warp levels, then the + // warp tail — here cut at 8 and 2 pairs. + level_begin = num_leaves - 1; + while (level_begin != 0) { + const uint64_t new_begin = level_begin / 2; + const uint64_t n_pairs = level_begin - new_begin; + if (n_pairs <= 2) { + simt_launch(1, 32, [&] { rpx_merkle_tail_warp(mixed.data(), level_begin); }); + break; + } + if (n_pairs <= 8) { + simt_launch((unsigned)((n_pairs + 3) / 4), 64, + [&] { rpx_merkle_level_warp(mixed.data(), new_begin, n_pairs); }); + } else { + simt_launch((unsigned)((n_pairs + 63) / 64), 64, + [&] { rpx_merkle_level(mixed.data(), new_begin, n_pairs); }); + } + level_begin = new_begin; + } + check(mixed == want, "shipped + warp levels + warp tail must reproduce the shipped tree"); + ++trees; + } + // A control that can fail: one flipped leaf byte moves the root. + std::vector a = tree_with_random_leaves(8, 0xF00), b = a; + b[(8 - 1) * 32 + 5] ^= 1; + simt_launch(1, 64, [&] { rpx_merkle_tail_warp(a.data(), 7); }); + simt_launch(1, 64, [&] { rpx_merkle_tail_warp(b.data(), 7); }); + check(memcmp(a.data(), b.data(), 32) != 0, "a flipped leaf byte must move the warp tail's root"); + printf("warp level + warp tail + mixed walk vs the per-parent level walk: %d trees (2..128 leaves) + a failing control\n", + trees); +} + +// --------------------------------------------------------------------------- +// The work-queue grind against the stride grind. +// --------------------------------------------------------------------------- + +// The shipped kernel as ONE thread: stride 1, so it scans `[base, base+count)` +// in order and returns the smallest valid nonce there, or the sentinel. +uint64_t shipped_grind(const uint64_t inner[4], uint64_t limit, uint64_t base, uint64_t count) { + unsigned long long result = ~0ull; + simt_launch(1, 1, [&] { + rpx_grind_search(inner, limit, base, count, (volatile unsigned long long *)&result); + }); + return result; +} + +struct QueueRun { + uint64_t nonce, executed, max_iters, ran_to_end; +}; + +QueueRun queue_grind(const uint64_t inner[4], uint64_t limit, uint64_t base, uint64_t count, + unsigned grid, unsigned block) { + unsigned long long state[2] = {~0ull, 0}; + unsigned long long counts[3] = {0, 0, 0}; + simt_launch(grid, block, [&] { + rpx_grind_search_queue_counted(inner, limit, base, count, state, counts); + }); + unsigned long long plain[2] = {~0ull, 0}; + simt_launch(grid, block, [&] { rpx_grind_search_queue(inner, limit, base, count, plain); }); + check(plain[0] == state[0], "the plain and counted queue kernels must agree"); + return QueueRun{state[0], counts[0], counts[1], counts[2]}; +} + +void queue_grind_returns_the_shipped_nonce() { + int runs = 0, found = 0; + double past_answer = 0; + uint64_t seed = 0x9A1; + for (int k = 0; k < 24; ++k) { + uint64_t inner[4]; + for (int i = 0; i < 4; ++i) inner[i] = splitmix(seed) % P; + for (unsigned factor : {5u, 7u, 9u}) { + const uint64_t limit = 1ull << (64 - factor); + for (uint64_t base : {0ull, 37ull}) { + // A range that ends before most answers, and one that holds them. + for (uint64_t count : {100ull, 4096ull}) { + const uint64_t want = shipped_grind(inner, limit, base, count); + const QueueRun got = queue_grind(inner, limit, base, count, 2, 64); + char what[160]; + snprintf(what, sizeof what, + "queue grind must return the shipped nonce (k %d, factor %u, base %llu, count %llu)", + k, factor, (unsigned long long)base, (unsigned long long)count); + check(got.nonce == want, what); + if (want != ~0ull) { + ++found; + // What is GUARANTEED: every chunk up to the one holding + // the answer is hashed in full, and nothing is hashed + // twice. How far past the answer the other warps got + // depends on scheduling (here, the host's), so it is + // reported, not asserted. + const uint64_t through = ((want - base) / 32 + 1) * 32; + check(got.executed >= through && got.executed <= count, + "queue grind must hash every chunk through the answer's, once"); + past_answer += (double)(got.executed - through); + } else { + check(got.executed == count, "a miss must hash the whole range exactly once"); + } + ++runs; + } + } + } + } + // A control that can fail: a different inner hash finds a different nonce. + uint64_t a[4] = {1, 2, 3, 4}, b[4] = {1, 2, 3, 5}; + const uint64_t limit = 1ull << (64 - 7); + check(queue_grind(a, limit, 0, 4096, 2, 64).nonce != queue_grind(b, limit, 0, 4096, 2, 64).nonce, + "different inner hashes must (here) give different nonces"); + printf("queue grind vs the stride grind: %d runs (%d with a nonce), misses hash the range once, a failing control\n", + runs, found); + printf(" (informational: %.1f nonces hashed past the answer's chunk on average, 4 host warps)\n", + found ? past_answer / found : 0.0); +} + +} // namespace + +int main() { + printf("RPX cooperative-lane kernels (half-warp Merkle, queue grind), host SIMT replay of crypto/math-cuda/kernels/rpx.cu\n\n"); + warp_permutation_matches_the_shipped_permutation(); + warp_merkle_kernels_match_the_shipped_tree(); + queue_grind_returns_the_shipped_nonce(); + if (failures != 0) { + printf("\n*** %d FAILURE(S) ***\n", failures); + return 1; + } + printf("\nALL SIMT HOST KAT CHECKS PASS\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/whir_host_kat.cpp b/crypto/math-cuda/tests/host_kat/whir_host_kat.cpp new file mode 100644 index 000000000..2ab268f2a --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/whir_host_kat.cpp @@ -0,0 +1,808 @@ +// Known-answer tests for the WHIR round kernels in `kernels/whir_fold.cu`, run +// on the host through `cuda_host_shim.h` — no GPU, no nvcc, a second. +// +// WHAT IT COVERS: +// 1. The FUSED fold (`whir_fold_coset`, every level of a round in one pass) +// against the level-by-level kernels it replaces (`whir_fold_base_ext3`, +// `whir_fold_ext3`), replayed thread by thread: RAW-identical, limb for +// limb, over base and extension inputs, k = 1..6, with non-canonical raw +// inputs mixed in — the proof serializes raw limbs, so equal values are not +// enough for this one. +// 2. The LEAN first rounds (`whir_lean_round_at`, `whir_lean_materialize_at`, +// `whir_lean_colmap`) against the materialised opening they replace: a +// weight written out from its shares as `scale·eq(z, row)` coordinate by +// coordinate (no half tables), the message lifted, the round evaluated at +// the nodes 1 and 2 over the pairs `(j, j + half)` and bound in place as +// `lo + α(hi − lo)` — the order `Mle::fix_first_variable` and the device +// session bind in. Six rounds, then the bound tables. EQUAL values +// (canonical), which is what the lean path promises; the transcript +// absorbs canonical bytes. +// 3. The TILE form of the lean rounds (`whir_tile_plan_at`, +// `whir_tile_contract_at`, `whir_tile_round_at`, `whir_tile_fold_at`) — +// one pass over the message, then every round on tables `2^u` times +// smaller — against BOTH the lean rounds of 2 and the materialised +// opening, round by round, on random stacks: one to eight classes meeting +// in a column of tiles, gaps, a column taller than the bound blocks, equal +// points apart, columns read from a store that is not laid out as the +// stack, one to six rounds. EQUAL values. Then the bound tables written +// from the tile form (`whir_tile_materialize_at`) at every count of bound +// challenges: the weight EQUAL to the share form's write, the message +// RAW-identical to it; and the comparison a checking opening runs +// (`whir_tile_materialize_check`) counts one moved entry of either +// table. The planner of these stacks is this file's own: the host +// planner in Rust is not what it is checked against. +// +// WHAT IT DOES NOT COVER, and the GPU tests are still required for: whether +// nvcc accepts the file, the shared-memory squares and block reductions, +// grid-stride bounds under a real launch, register pressure. The field +// primitives themselves are pinned by `rpx_host_kat.cpp`'s schoolbook layer. +// +// Build and run with `make test-whir-host-kat`. + +#include +#include +#include +#include + +#include "cuda_host_shim.h" + +// The kernels' shared memory, for a host that has no such thing. The KAT calls +// the device functions directly, never the block-reducing kernels, so these +// are only here for the file to link. +// `extern "C"` because the kernels that name it are. +#define __shared__ +extern "C" { +uint64_t shared[3 * 1024]; +} + +#include "whir_fold.cu" + +namespace { + +int failures = 0; + +void check(bool ok, const char *what) { + if (!ok) { + printf("FAIL: %s\n", what); + ++failures; + } +} + +const uint64_t P = 0xFFFFFFFF00000001ull; + +uint64_t state = 0x9E3779B97F4A7C15ull; +uint64_t next_u64() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; +} + +// A raw limb: canonical most of the time, and every eighth one a non-canonical +// representative in [p, 2^64) — the representation a device may hold. +uint64_t raw_limb() { + uint64_t v = next_u64() % P; + if ((next_u64() & 7) == 0 && v < 0xFFFFFFFFull) { + v += P; + } + return v; +} + +Fe3 random_ext() { return ext3::make(raw_limb(), raw_limb(), raw_limb()); } + +bool same_value(const Fe3 &a, const Fe3 &b) { + Fe3 x = ext3::canonical(a), y = ext3::canonical(b); + return x.a == y.a && x.b == y.b && x.c == y.c; +} + +// ─────────────────────────────── the fused fold ────────────────────────────── + +// Level by level, as the device folds today: one launch per level, replayed. +std::vector fold_level_by_level(const std::vector &input, bool base, + uint32_t k, uint64_t two_inv, + const std::vector &g_invs, + const std::vector &alphas) { + std::vector current = input; + bool current_base = base; + uint64_t elements = base ? input.size() : input.size() / 3; + for (uint32_t level = 0; level < k; ++level) { + uint64_t half = elements / 2; + std::vector out(half * 3); + if (current_base) { + CUDA_HOST_FOR_EACH_THREAD(j, half) + whir_fold_base_ext3(current.data(), half, two_inv, g_invs[level], + alphas.data() + level * 3, out.data()); + } else { + CUDA_HOST_FOR_EACH_THREAD(j, half) + whir_fold_ext3(current.data(), half, two_inv, g_invs[level], + alphas.data() + level * 3, out.data()); + } + current = out; + current_base = false; + elements = half; + } + return current; +} + +std::vector fold_fused(const std::vector &input, bool base, uint32_t k, + uint64_t two_inv, const std::vector &g_invs, + const std::vector &alphas) { + uint64_t elements = base ? input.size() : input.size() / 3; + uint64_t n_out = elements >> k; + std::vector squares(WHIR_MAX_FOLD * WHIR_POW_BITS); + for (uint32_t level = 0; level < k; ++level) { + whir_fold_squares(squares.data(), level, g_invs[level]); + } + std::vector out(n_out * 3); + for (uint64_t j = 0; j < n_out; ++j) { + Fe3 v = whir_fold_coset(input.data(), base, j, n_out, k, two_inv, squares.data(), + alphas.data()); + out[j * 3] = v.a; + out[j * 3 + 1] = v.b; + out[j * 3 + 2] = v.c; + } + return out; +} + +void fused_fold_is_raw_identical() { + int cases = 0; + for (int base = 1; base >= 0; --base) { + for (uint32_t k = 1; k <= WHIR_MAX_FOLD; ++k) { + for (uint32_t log_n : {k, k + 1u, k + 4u, 13u}) { + if (log_n < k) continue; + uint64_t elements = 1ull << log_n; + std::vector input(base ? elements : elements * 3); + for (auto &limb : input) limb = raw_limb(); + std::vector g_invs(k), alphas(k * 3); + for (auto &g : g_invs) g = raw_limb(); + for (auto &a : alphas) a = raw_limb(); + uint64_t two_inv = 0x7FFFFFFF80000001ull; // 2^{-1} mod p + std::vector want = + fold_level_by_level(input, base, k, two_inv, g_invs, alphas); + std::vector got = fold_fused(input, base, k, two_inv, g_invs, alphas); + char what[128]; + snprintf(what, sizeof what, "fused fold, %s, k=%u, n=2^%u: raw limbs", + base ? "base" : "ext3", k, log_n); + check(want == got, what); + ++cases; + } + } + } + // A control: the comparison sees a single limb. + std::vector input(1u << 8); + for (auto &limb : input) limb = raw_limb(); + std::vector g_invs = {raw_limb(), raw_limb()}, alphas(6); + for (auto &a : alphas) a = raw_limb(); + std::vector a = fold_fused(input, true, 2, 5, g_invs, alphas); + input[17] = goldilocks::add(input[17], 1); + std::vector b = fold_fused(input, true, 2, 5, g_invs, alphas); + check(a != b, "control: one input limb moves the fused output"); + printf("fused fold: %d cases raw-identical to the level-by-level kernels\n", cases); +} + +// ───────────────────────────── the lean first rounds ───────────────────────── + +struct Share { + uint64_t offset; + uint32_t num_vars; + std::vector point; + Fe3 scale; +}; + +// `eq(z, i)` with `i`'s most significant bit on `z[0]` — the layout every +// `eq` table in the prover has (`eq::eq_evals_into`). +Fe3 eq_at(const std::vector &z, uint64_t i) { + Fe3 acc = ext3::one(); + uint32_t m = z.size(); + for (uint32_t c = 0; c < m; ++c) { + bool bit = (i >> (m - 1 - c)) & 1; + Fe3 factor = bit ? z[c] : ext3::sub(ext3::one(), z[c]); + acc = ext3::mul(acc, factor); + } + return acc; +} + +std::vector eq_table(const std::vector &z) { + std::vector out; + for (uint64_t i = 0; i < (1ull << z.size()); ++i) { + Fe3 v = eq_at(z, i); + out.push_back(v.a); + out.push_back(v.b); + out.push_back(v.c); + } + return out; +} + +void lean_rounds_equal_the_materialised_opening() { + const uint32_t n = 11; + const uint64_t len = 1ull << n; + // A stack of mixed heights, aligned to their sizes, with gaps: a table of + // three columns sharing a point, a tall column crossing the first rounds' + // blocks, short ones inside one block, and padding at the end. + std::vector shares; + std::vector table_point; + for (int i = 0; i < 7; ++i) table_point.push_back(random_ext()); + for (int c = 0; c < 3; ++c) { + shares.push_back({(uint64_t)c << 7, 7, table_point, random_ext()}); + } + std::vector tall; + for (int i = 0; i < 10; ++i) tall.push_back(random_ext()); + shares.push_back({1ull << 10, 10, tall, random_ext()}); + for (int c = 0; c < 6; ++c) { + std::vector p; + uint32_t vars = 2 + c % 3; + for (uint32_t i = 0; i < vars; ++i) p.push_back(random_ext()); + uint64_t at = (1ull << 9) + ((uint64_t)c << 5); + shares.push_back({at, vars, p, random_ext()}); + } + // In stack order: the column map is a binary search over the offsets, and + // the lean opening takes its shares sorted (`LeanRound0::new` refuses + // otherwise). + std::sort(shares.begin(), shares.end(), + [](const Share &a, const Share &b) { return a.offset < b.offset; }); + // The message: base values where a column is, zero elsewhere. + std::vector data(len, 0); + for (const Share &s : shares) { + for (uint64_t r = 0; r < (1ull << s.num_vars); ++r) data[s.offset + r] = raw_limb(); + } + + // ── the materialised opening ── + std::vector w(len, ext3::zero()), f(len); + for (uint64_t x = 0; x < len; ++x) f[x] = ext3::make(data[x], 0, 0); + for (const Share &s : shares) { + for (uint64_t r = 0; r < (1ull << s.num_vars); ++r) { + w[s.offset + r] = ext3::mul(s.scale, eq_at(s.point, r)); + } + } + + // ── the lean opening's inputs ── + std::vector eqbuf, rows, starts, ends; + for (const Share &s : shares) { + uint32_t lo_bits = s.num_vars / 2; + std::vector hi(s.point.begin(), s.point.end() - lo_bits); + std::vector lo(s.point.end() - lo_bits, s.point.end()); + uint64_t hi_at = eqbuf.size() / 3; + for (uint64_t v : eq_table(hi)) eqbuf.push_back(v); + uint64_t lo_at = eqbuf.size() / 3; + for (uint64_t v : eq_table(lo)) eqbuf.push_back(v); + uint64_t row[WHIR_LEAN_SHARE_WORDS] = {s.offset, s.offset, lo_bits, hi_at, lo_at, + s.scale.a, s.scale.b, s.scale.c, 0}; + rows.insert(rows.end(), row, row + WHIR_LEAN_SHARE_WORDS); + starts.push_back(s.offset); + ends.push_back(s.offset + (1ull << s.num_vars)); + } + std::vector colmap(len); + CUDA_HOST_SINGLE_THREAD(); + whir_lean_colmap(colmap.data(), len, starts.data(), ends.data(), shares.size()); + uint64_t covered = 0; + for (uint64_t x = 0; x < len; ++x) covered += colmap[x] != WHIR_LEAN_NONE; + uint64_t cells = 0; + for (const Share &s : shares) cells += 1ull << s.num_vars; + check(covered == cells, "the column map covers exactly the columns' cells"); + + // The share-form value at every position is the materialised one. + bool values_ok = true; + for (uint64_t x = 0; x < len; ++x) { + Fe3 wv; + uint64_t fv; + whir_lean_value(colmap.data(), rows.data(), eqbuf.data(), data.data(), x, wv, fv); + values_ok &= same_value(wv, w[x]) && goldilocks::canonical(fv) == + goldilocks::canonical(f[x].a); + } + check(values_ok, "the share form reads back the materialised weight and message"); + + const uint64_t t[6] = {1, 0, 0, 2, 0, 0}; + std::vector bound; // three limbs per challenge + uint64_t width = len; + for (uint32_t s = 1; s <= 6; ++s) { + // The materialised round: pairs `(j, j + half)`, nodes 1 and 2. + uint64_t half = width / 2; + Fe3 want[2] = {ext3::zero(), ext3::zero()}; + for (uint64_t j = 0; j < half; ++j) { + for (int ti = 0; ti < 2; ++ti) { + Fe3 node = ext3::make(t[ti * 3], 0, 0); + Fe3 wt = ext3::add(w[j], ext3::mul(node, ext3::sub(w[j + half], w[j]))); + Fe3 ft = ext3::add(f[j], ext3::mul(node, ext3::sub(f[j + half], f[j]))); + want[ti] = ext3::add(want[ti], ext3::mul(wt, ft)); + } + } + // The lean round over the shares. + std::vector prefix; + for (size_t i = 0; i < bound.size(); i += 3) { + prefix.push_back(ext3::make(bound[i], bound[i + 1], bound[i + 2])); + } + std::vector eqc = eq_table(prefix); + Fe3 got[WHIR_LEAN_MAX_T] = {ext3::zero(), ext3::zero(), ext3::zero(), ext3::zero()}; + for (uint64_t rest = 0; rest < (1ull << (n - s)); ++rest) { + whir_lean_round_at(colmap.data(), rows.data(), eqbuf.data(), data.data(), n, s, + eqc.data(), t, 2, rest, got); + } + char what[96]; + snprintf(what, sizeof what, "round %u: the evaluation at node 1", s); + check(same_value(got[0], want[0]), what); + snprintf(what, sizeof what, "round %u: the evaluation at node 2", s); + check(same_value(got[1], want[1]), what); + + // Bind: in place for the materialised tables, a challenge for the lean. + Fe3 alpha = random_ext(); + for (uint64_t j = 0; j < half; ++j) { + w[j] = ext3::add(w[j], ext3::mul(alpha, ext3::sub(w[j + half], w[j]))); + f[j] = ext3::add(f[j], ext3::mul(alpha, ext3::sub(f[j + half], f[j]))); + } + width = half; + bound.push_back(alpha.a); + bound.push_back(alpha.b); + bound.push_back(alpha.c); + } + + // The tables the lean opening writes once its rounds are spent. + std::vector prefix; + for (size_t i = 0; i < bound.size(); i += 3) { + prefix.push_back(ext3::make(bound[i], bound[i + 1], bound[i + 2])); + } + std::vector eqfull = eq_table(prefix); + bool tables_ok = true; + for (uint64_t y = 0; y < width; ++y) { + Fe3 wy, fy; + whir_lean_materialize_at(colmap.data(), rows.data(), eqbuf.data(), data.data(), n, 6, + eqfull.data(), y, wy, fy); + tables_ok &= same_value(wy, w[y]) && same_value(fy, f[y]); + } + check(tables_ok, "the materialised tables after six rounds"); + + // A control: a column whose scale moves moves the first round. + rows[5] = goldilocks::add(rows[5], 1); + std::vector eqc = eq_table({}); + Fe3 moved[WHIR_LEAN_MAX_T] = {ext3::zero(), ext3::zero(), ext3::zero(), ext3::zero()}; + Fe3 base_line[WHIR_LEAN_MAX_T] = {ext3::zero(), ext3::zero(), ext3::zero(), ext3::zero()}; + for (uint64_t rest = 0; rest < (1ull << (n - 1)); ++rest) { + whir_lean_round_at(colmap.data(), rows.data(), eqbuf.data(), data.data(), n, 1, + eqc.data(), t, 2, rest, moved); + } + rows[5] = goldilocks::sub(rows[5], 1); + for (uint64_t rest = 0; rest < (1ull << (n - 1)); ++rest) { + whir_lean_round_at(colmap.data(), rows.data(), eqbuf.data(), data.data(), n, 1, + eqc.data(), t, 2, rest, base_line); + } + check(!same_value(moved[1], base_line[1]), "control: a share's scale moves the round"); + printf("lean first rounds: 6 rounds and the bound tables equal the materialised opening\n"); +} + +// ───────────────────────────────── the tile form ───────────────────────────── + +// A stacked polynomial as the tile form reads it: shares of at least `2^u` +// positions each, their points' tables split at `u` in every row, one `lo` +// table per class (points whose last `u` coordinates are equal, limb for +// limb), and the columns' data in a store that is NOT laid out as the stack. +struct TileStack { + uint32_t n, u, lean, slots, classes; + std::vector shares; + std::vector store; // the columns, back to front + std::vector eqbuf, rows, starts, ends; // the share form's inputs, `lo_bits = u` + std::vector tile_starts, tile_ends; // the same bounds in tiles +}; + +bool same_limbs(const std::vector &a, const std::vector &b) { + if (a.size() != b.size()) return false; + for (size_t i = 0; i < a.size(); ++i) { + if (a[i].a != b[i].a || a[i].b != b[i].b || a[i].c != b[i].c) return false; + } + return true; +} + +// `suffixes` is how many distinct last-`u`-coordinate tails the points draw +// from: the classes. `tall` puts one column of `2^(n−1)` first. +TileStack tile_stack(uint32_t n, uint32_t u, uint32_t lean, uint32_t suffixes, bool tall) { + TileStack t; + t.n = n; + t.u = u; + t.lean = lean; + std::vector> tails(suffixes); + for (auto &tail : tails) { + for (uint32_t i = 0; i < u; ++i) tail.push_back(random_ext()); + } + // A few heads per height, so columns of one height share whole points too. + std::vector>> heads(n + 1); + auto point_for = [&](uint32_t m) { + auto &pool = heads[m]; + size_t pick = next_u64() % 3; + while (pool.size() <= pick) { + std::vector head; + for (uint32_t i = 0; i + u < m; ++i) head.push_back(random_ext()); + pool.push_back(head); + } + std::vector point = pool[pick]; + const auto &tail = tails[next_u64() % suffixes]; + point.insert(point.end(), tail.begin(), tail.end()); + return point; + }; + uint64_t len = 1ull << n, cursor = 0, stop = len; + if (tall) { + uint64_t at = (next_u64() & 1) ? len / 2 : 0; + t.shares.push_back({at, n - 1, point_for(n - 1), random_ext()}); + if (at == 0) { + cursor = len / 2; + } else { + stop = len / 2; + } + } + while (true) { + // Mostly the sizes around a bound block, where columns of several + // points meet in one column of tiles. + uint32_t top = n - lean + 1 < n - 1 ? n - lean + 1 : n - 1; + if (top < u) top = u; + uint32_t m = u + next_u64() % (top - u + 1); + if (next_u64() % 4 == 0) m = u + next_u64() % 2; + if (m > n - 1) m = n - 1; + uint64_t size = 1ull << m; + uint64_t at = (cursor + size - 1) / size * size; + if (next_u64() % 5 == 0) at += size; // a gap + if (at + size > stop) break; + t.shares.push_back({at, m, point_for(m), random_ext()}); + cursor = at + size; + } + std::sort(t.shares.begin(), t.shares.end(), + [](const Share &a, const Share &b) { return a.offset < b.offset; }); + + // The store: the last column first. + std::vector data_at(t.shares.size()); + for (size_t c = t.shares.size(); c-- > 0;) { + data_at[c] = t.store.size(); + for (uint64_t r = 0; r < (1ull << t.shares[c].num_vars); ++r) t.store.push_back(raw_limb()); + } + std::vector, uint64_t>> lo_tables; // (tail, where) + for (size_t c = 0; c < t.shares.size(); ++c) { + const Share &s = t.shares[c]; + std::vector hi(s.point.begin(), s.point.end() - u); + std::vector lo(s.point.end() - u, s.point.end()); + uint64_t hi_at = t.eqbuf.size() / 3; + for (uint64_t v : eq_table(hi)) t.eqbuf.push_back(v); + uint64_t lo_at = 0; + bool known = false; + for (const auto &seen : lo_tables) { + if (same_limbs(seen.first, lo)) { + lo_at = seen.second; + known = true; + } + } + if (!known) { + lo_at = t.eqbuf.size() / 3; + for (uint64_t v : eq_table(lo)) t.eqbuf.push_back(v); + lo_tables.push_back({lo, lo_at}); + } + uint64_t row[WHIR_LEAN_SHARE_WORDS] = {s.offset, data_at[c], u, hi_at, lo_at, + s.scale.a, s.scale.b, s.scale.c, 0}; + t.rows.insert(t.rows.end(), row, row + WHIR_LEAN_SHARE_WORDS); + t.starts.push_back(s.offset); + t.ends.push_back(s.offset + (1ull << s.num_vars)); + t.tile_starts.push_back(s.offset >> u); + t.tile_ends.push_back((s.offset + (1ull << s.num_vars)) >> u); + } + t.classes = lo_tables.size(); + t.slots = t.classes < (1u << lean) ? t.classes : (1u << lean); + return t; +} + +// The tile form's tables for a stack: the tile map, each column's slots, `Ω` +// and `Φ` (`slots` tables of `2^(n−u)` each). +struct TileTables { + std::vector tilemap; + std::vector slot_lo, omega, phi; +}; + +TileTables tile_tables(const TileStack &t) { + const uint32_t nu = t.n - t.u; + const uint64_t tiles = 1ull << nu, cols = 1ull << (nu - t.lean); + TileTables out; + out.tilemap.resize(tiles); + out.slot_lo.resize(cols * t.slots); + out.omega.resize(t.slots * tiles * 3); + out.phi.resize(t.slots * tiles * 3); + CUDA_HOST_SINGLE_THREAD(); + whir_lean_colmap(out.tilemap.data(), tiles, t.tile_starts.data(), t.tile_ends.data(), + t.shares.size()); + for (uint64_t col = 0; col < cols; ++col) { + whir_tile_plan_at(out.tilemap.data(), t.rows.data(), t.eqbuf.data(), nu, t.lean, t.u, + t.slots, col, out.slot_lo.data(), out.omega.data()); + } + for (uint64_t tile = 0; tile < tiles; ++tile) { + Fe3 acc[WHIR_TILE_MAX_SLOTS]; + whir_tile_contract_at(out.tilemap.data(), t.rows.data(), t.eqbuf.data(), t.store.data(), + out.slot_lo.data(), nu, t.lean, t.u, t.slots, tile, 0, 1, acc); + for (uint32_t s = 0; s < t.slots; ++s) { + whir_store3(out.phi.data() + (s * tiles + tile) * 3, acc[s]); + } + } + return out; +} + +// One stack through the three: the materialised opening, the lean rounds and +// the tile form. Returns the most slots a column of tiles used. +uint32_t tile_form_on(const TileStack &t, const char *name) { + const uint32_t n = t.n, u = t.u, lean = t.lean, slots = t.slots, nu = n - u; + const uint64_t len = 1ull << n, tiles = 1ull << nu, cols = 1ull << (nu - lean); + char what[160]; + + // ── the materialised opening ── + std::vector w(len, ext3::zero()), f(len, ext3::zero()); + for (size_t c = 0; c < t.shares.size(); ++c) { + const Share &s = t.shares[c]; + for (uint64_t r = 0; r < (1ull << s.num_vars); ++r) { + w[s.offset + r] = ext3::mul(s.scale, eq_at(s.point, r)); + f[s.offset + r] = ext3::make(t.store[t.rows[c * WHIR_LEAN_SHARE_WORDS + 1] + r], 0, 0); + } + } + Fe3 claim = ext3::zero(); + for (uint64_t x = 0; x < len; ++x) claim = ext3::add(claim, ext3::mul(w[x], f[x])); + + // ── the lean rounds' position map against the tile form's tile map ── + std::vector colmap(len); + CUDA_HOST_SINGLE_THREAD(); + whir_lean_colmap(colmap.data(), len, t.starts.data(), t.ends.data(), t.shares.size()); + TileTables tt = tile_tables(t); + bool map_ok = true; + for (uint64_t x = 0; x < len; ++x) map_ok &= tt.tilemap[x >> u] == colmap[x]; + snprintf(what, sizeof what, "%s: a tile lies in one column or one gap", name); + check(map_ok, what); + + // ── the plan and the one pass ── + uint32_t most = 0; + bool prefix_ok = true; + for (uint64_t col = 0; col < cols; ++col) { + uint32_t used = 0; + while (used < slots && tt.slot_lo[col * slots + used] != WHIR_TILE_NO_SLOT) ++used; + for (uint32_t s = used; s < slots; ++s) { + prefix_ok &= tt.slot_lo[col * slots + s] == WHIR_TILE_NO_SLOT; + } + most = used > most ? used : most; + } + snprintf(what, sizeof what, "%s: a column's slots are a prefix", name); + check(prefix_ok, what); + bool lanes_ok = true; + const uint64_t lanes = (1ull << u) < 4 ? (1ull << u) : 4; + for (uint64_t tile = 0; tile < tiles; ++tile) { + Fe3 part[WHIR_TILE_MAX_SLOTS], sum[WHIR_TILE_MAX_SLOTS]; + for (uint32_t s = 0; s < slots; ++s) sum[s] = ext3::zero(); + for (uint64_t lane = 0; lane < lanes; ++lane) { + whir_tile_contract_at(tt.tilemap.data(), t.rows.data(), t.eqbuf.data(), t.store.data(), + tt.slot_lo.data(), nu, lean, u, slots, tile, lane, lanes, part); + for (uint32_t s = 0; s < slots; ++s) sum[s] = ext3::add(sum[s], part[s]); + } + for (uint32_t s = 0; s < slots; ++s) { + lanes_ok &= same_value(sum[s], whir_load3(tt.phi.data() + (s * tiles + tile) * 3)); + } + } + snprintf(what, sizeof what, "%s: the pass in lanes sums to the pass in one", name); + check(lanes_ok, what); + Fe3 tile_claim = ext3::zero(); + for (uint64_t i = 0; i < slots * tiles; ++i) { + tile_claim = ext3::add(tile_claim, ext3::mul(whir_load3(tt.omega.data() + i * 3), + whir_load3(tt.phi.data() + i * 3))); + } + snprintf(what, sizeof what, "%s: the tiles' sum is the opening's claim", name); + check(same_value(tile_claim, claim), what); + + // ── the bound tables written from the tile form, at the challenges bound + // so far: the weight EQUAL to the share form's write and to the + // materialised opening, the message RAW-identical to the share form's + // write. `last` also runs the comparison kernel a checking opening + // uses: nothing on the share form's tables, then one position counted + // for one moved entry of either table. ── + std::vector bound; + uint64_t width = len; + auto written = [&](bool last) { + const uint32_t b = bound.size() / 3; + std::vector prefix; + for (size_t i = 0; i < bound.size(); i += 3) prefix.push_back(whir_load3(&bound[i])); + std::vector eqfull = eq_table(prefix); + std::vector share_w(width * 3), share_f(width * 3); + bool weight_ok = true, message_raw = true, opening_ok = true; + for (uint64_t y = 0; y < width; ++y) { + Fe3 tw, tf, sw, sf; + whir_tile_materialize_at(tt.tilemap.data(), t.rows.data(), t.eqbuf.data(), + t.store.data(), tt.slot_lo.data(), tt.omega.data(), n, lean, + u, slots, b, eqfull.data(), y, tw, tf); + whir_lean_materialize_at(colmap.data(), t.rows.data(), t.eqbuf.data(), t.store.data(), + n, b, eqfull.data(), y, sw, sf); + weight_ok &= same_value(tw, sw); + message_raw &= tf.a == sf.a && tf.b == sf.b && tf.c == sf.c; + opening_ok &= same_value(tw, w[y]) && same_value(tf, f[y]); + whir_store3(share_w.data() + y * 3, sw); + whir_store3(share_f.data() + y * 3, sf); + } + snprintf(what, sizeof what, "%s: written at %u bound: the weight = the share form's", name, + b); + check(weight_ok, what); + snprintf(what, sizeof what, + "%s: written at %u bound: the message raw-identical to the share form's", name, b); + check(message_raw, what); + snprintf(what, sizeof what, "%s: written at %u bound: both = the materialised opening", + name, b); + check(opening_ok, what); + if (!last) return; + auto differing = [&]() { + unsigned long long count = 0; + CUDA_HOST_SINGLE_THREAD(); + whir_tile_materialize_check(tt.tilemap.data(), t.rows.data(), t.eqbuf.data(), + t.store.data(), tt.slot_lo.data(), tt.omega.data(), n, + lean, u, slots, b, eqfull.data(), share_w.data(), + share_f.data(), &count); + return count; + }; + snprintf(what, sizeof what, "%s: the comparison finds the share form's tables equal", name); + check(differing() == 0, what); + const uint64_t at = (width / 2) * 3; + share_w[at + 1] = goldilocks::add(share_w[at + 1], 1); + snprintf(what, sizeof what, "%s: the comparison counts one moved weight entry", name); + check(differing() == 1, what); + share_w[at + 1] = goldilocks::sub(share_w[at + 1], 1); + share_f[at] = goldilocks::add(share_f[at], 1); + snprintf(what, sizeof what, "%s: the comparison counts one moved message entry", name); + check(differing() == 1, what); + }; + + // ── the rounds ── + const uint64_t nodes[6] = {1, 0, 0, 2, 0, 0}; + uint64_t tile_half = tiles / 2; + for (uint32_t s = 1; s <= lean; ++s) { + written(false); + uint64_t half = width / 2; + Fe3 want[2] = {ext3::zero(), ext3::zero()}; + for (uint64_t j = 0; j < half; ++j) { + for (int ti = 0; ti < 2; ++ti) { + Fe3 node = ext3::make(nodes[ti * 3], 0, 0); + Fe3 wt = ext3::add(w[j], ext3::mul(node, ext3::sub(w[j + half], w[j]))); + Fe3 ft = ext3::add(f[j], ext3::mul(node, ext3::sub(f[j + half], f[j]))); + want[ti] = ext3::add(want[ti], ext3::mul(wt, ft)); + } + } + std::vector prefix; + for (size_t i = 0; i < bound.size(); i += 3) prefix.push_back(whir_load3(&bound[i])); + std::vector eqc = eq_table(prefix); + Fe3 share_form[WHIR_LEAN_MAX_T] = {ext3::zero(), ext3::zero(), ext3::zero(), ext3::zero()}; + for (uint64_t rest = 0; rest < (1ull << (n - s)); ++rest) { + whir_lean_round_at(colmap.data(), t.rows.data(), t.eqbuf.data(), t.store.data(), n, s, + eqc.data(), nodes, 2, rest, share_form); + } + Fe3 got[WHIR_LEAN_MAX_T] = {ext3::zero(), ext3::zero(), ext3::zero(), ext3::zero()}; + for (uint64_t i = 0; i < tile_half; ++i) { + whir_tile_round_at(tt.omega.data(), tt.phi.data(), tiles, slots, tile_half, nodes, 2, i, + got); + } + for (int ti = 0; ti < 2; ++ti) { + snprintf(what, sizeof what, "%s: round %u node %d: tiles = the materialised opening", + name, s, ti + 1); + check(same_value(got[ti], want[ti]), what); + snprintf(what, sizeof what, "%s: round %u node %d: tiles = the lean round", name, s, + ti + 1); + check(same_value(got[ti], share_form[ti]), what); + } + Fe3 alpha = random_ext(); + for (uint64_t j = 0; j < half; ++j) { + w[j] = ext3::add(w[j], ext3::mul(alpha, ext3::sub(w[j + half], w[j]))); + f[j] = ext3::add(f[j], ext3::mul(alpha, ext3::sub(f[j + half], f[j]))); + } + for (uint32_t k = 0; k < slots; ++k) { + for (uint64_t i = 0; i < tile_half; ++i) { + whir_tile_fold_at(tt.omega.data() + k * tiles * 3, tile_half, i, alpha); + whir_tile_fold_at(tt.phi.data() + k * tiles * 3, tile_half, i, alpha); + } + } + width = half; + tile_half /= 2; + bound.push_back(alpha.a); + bound.push_back(alpha.b); + bound.push_back(alpha.c); + } + written(true); + + // ── the bound weight is `Σ_κ Ω_κ^bound(R)·eq(λ_κ, j)`, and `Φ^bound` the + // bound message's tiles at their slots' points ── + bool weight_ok = true, message_ok = true; + for (uint64_t y = 0; y < width; ++y) { + uint64_t tile = y >> u, j = y & ((1ull << u) - 1); + Fe3 sum = ext3::zero(); + for (uint32_t k = 0; k < slots; ++k) { + uint64_t lo_at = tt.slot_lo[(tile & (cols - 1)) * slots + k]; + if (lo_at == WHIR_TILE_NO_SLOT) continue; + sum = ext3::add(sum, ext3::mul(whir_load3(tt.omega.data() + (k * tiles + tile) * 3), + whir_load3(t.eqbuf.data() + (lo_at + j) * 3))); + } + weight_ok &= same_value(sum, w[y]); + } + for (uint64_t tile = 0; tile < (width >> u); ++tile) { + for (uint32_t k = 0; k < slots; ++k) { + uint64_t lo_at = tt.slot_lo[(tile & (cols - 1)) * slots + k]; + if (lo_at == WHIR_TILE_NO_SLOT) continue; + Fe3 sum = ext3::zero(); + for (uint64_t j = 0; j < (1ull << u); ++j) { + sum = ext3::add(sum, ext3::mul(whir_load3(t.eqbuf.data() + (lo_at + j) * 3), + f[(tile << u) + j])); + } + message_ok &= same_value(sum, whir_load3(tt.phi.data() + (k * tiles + tile) * 3)); + } + } + snprintf(what, sizeof what, "%s: the bound weight from the bound slot tables", name); + check(weight_ok, what); + snprintf(what, sizeof what, "%s: the bound slot tables of the message", name); + check(message_ok, what); + return most; +} + +// The first round's node-2 value from the tile form alone, for the controls. +Fe3 tile_first_round(const TileStack &t) { + const uint64_t tiles = 1ull << (t.n - t.u); + TileTables tt = tile_tables(t); + const uint64_t nodes[6] = {1, 0, 0, 2, 0, 0}; + Fe3 got[WHIR_LEAN_MAX_T] = {ext3::zero(), ext3::zero(), ext3::zero(), ext3::zero()}; + for (uint64_t i = 0; i < tiles / 2; ++i) { + whir_tile_round_at(tt.omega.data(), tt.phi.data(), tiles, t.slots, tiles / 2, nodes, 2, i, + got); + } + return got[1]; +} + +void tile_form_equals_the_lean_rounds_and_the_materialised_opening() { + int stacks = 0, by_slots[WHIR_TILE_MAX_SLOTS + 1] = {0}, gaps = 0, talls = 0; + for (uint32_t trial = 0; trial < 240; ++trial) { + uint32_t n = 9 + trial % 4; // 9..12 + uint32_t u = 2 + (trial / 4) % 4; // 2..5 + uint32_t lean = 1 + (trial / 16) % 6; // 1..6 + if (lean > n - u) lean = n - u; + uint32_t suffixes = 1 + trial % 9; // classes offered: 1..9 + bool tall = trial % 5 == 0; + TileStack t = tile_stack(n, u, lean, suffixes, tall); + if (t.shares.empty() || t.slots > WHIR_TILE_MAX_SLOTS) continue; + uint64_t cells = 0; + for (const Share &s : t.shares) cells += 1ull << s.num_vars; + gaps += cells < (1ull << n); + talls += tall; + char name[96]; + snprintf(name, sizeof name, "tile form #%u (n=%u u=%u rounds=%u classes=%u)", trial, n, u, + lean, t.classes); + uint32_t most = tile_form_on(t, name); + ++by_slots[most]; + ++stacks; + } + check(by_slots[1] > 0 && by_slots[2] > 0 && by_slots[4] > 0 && by_slots[8] > 0, + "the stacks cover one, two, four and eight classes meeting in a column of tiles"); + check(gaps > 0 && talls > 0, "the stacks cover gaps and a column taller than the bound blocks"); + + // Controls: the comparison sees one cell of the message, one entry of a + // class's table, and a share's scale. + TileStack t = tile_stack(10, 3, 3, 2, false); + int before = failures; + tile_form_on(t, "control: untouched"); + check(failures == before, "control: the untouched stack passes"); + Fe3 base_line = tile_first_round(t); + TileStack cell = t; + cell.store[cell.rows[1]] = goldilocks::add(cell.store[cell.rows[1]], 1); + check(!same_value(tile_first_round(cell), base_line), + "control: one cell of the message moves the round"); + TileStack table = t; + table.eqbuf[(table.rows[4] + 1) * 3] = goldilocks::add(table.eqbuf[(table.rows[4] + 1) * 3], 1); + check(!same_value(tile_first_round(table), base_line), + "control: one entry of a class's table moves the round"); + TileStack scale = t; + scale.rows[5] = goldilocks::add(scale.rows[5], 1); + check(!same_value(tile_first_round(scale), base_line), "control: a share's scale moves the round"); + printf("tile form: %d stacks (most classes meeting in a column of tiles 1/2/3/4/5/6/7/8: " + "%d/%d/%d/%d/%d/%d/%d/%d; %d with gaps, %d with a tall column) equal the lean rounds " + "and the materialised opening, their written tables the share form's\n", + stacks, by_slots[1], by_slots[2], by_slots[3], by_slots[4], by_slots[5], by_slots[6], + by_slots[7], by_slots[8], gaps, talls); +} + +} // namespace + +int main() { + fused_fold_is_raw_identical(); + lean_rounds_equal_the_materialised_opening(); + tile_form_equals_the_lean_rounds_and_the_materialised_opening(); + if (failures) { + printf("whir host KAT: %d FAILURE(S)\n", failures); + return 1; + } + printf("whir host KAT: all passed\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/keccak_leaves.rs b/crypto/math-cuda/tests/keccak_leaves.rs index 087ccde14..be098c633 100644 --- a/crypto/math-cuda/tests/keccak_leaves.rs +++ b/crypto/math-cuda/tests/keccak_leaves.rs @@ -11,7 +11,7 @@ use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; use math::field::goldilocks::GoldilocksField; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha8Rng; -use stark::config::FriLayerMerkleTreeBackend; +use stark::config::KeccakFriLayerMerkleTreeBackend as FriLayerMerkleTreeBackend; use stark::prover::{keccak_leaves_bit_reversed, keccak_leaves_row_pair_bit_reversed}; type Fp = FieldElement; diff --git a/crypto/math-cuda/tests/merkle_cap.rs b/crypto/math-cuda/tests/merkle_cap.rs new file mode 100644 index 000000000..283bcb54c --- /dev/null +++ b/crypto/math-cuda/tests/merkle_cap.rs @@ -0,0 +1,143 @@ +//! Parity: `read_cap_dev` must return, for every cap height, exactly the nodes +//! the host `MerkleTree::cap` returns — the `2^c` nodes `c` levels below the +//! root, left to right, byte for byte. This is the gate for reading a +//! device-resident tree's Merkle cap in the STARK R4 cap post-pass +//! instead of copying the whole tree. + +use crypto::merkle_tree::backends::field_element_vector::FieldElementVectorBackend; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::goldilocks::GoldilocksField; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use sha3::Keccak256; + +type CpuTree = MerkleTree>; + +fn random_leaves(leaves_len: usize, seed: u64) -> Vec<[u8; 32]> { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + (0..leaves_len) + .map(|_| { + let mut arr = [0u8; 32]; + rng.fill(&mut arr[..]); + arr + }) + .collect() +} + +fn flat(leaves: &[[u8; 32]]) -> Vec { + leaves.iter().flat_map(|l| l.iter().copied()).collect() +} + +/// Every height `c <= min(depth, 6)` of a keccak tree with `2^log_n` leaves: +/// the device read equals the host cap, and the heap slice of the device's own +/// node buffer. +fn keccak_cap_parity(log_n: u32, seed: u64) { + let leaves_len = 1usize << log_n; + let leaves = random_leaves(leaves_len, seed); + let gpu_nodes = math_cuda::merkle::build_merkle_tree_on_device(&flat(&leaves)).unwrap(); + let cpu_tree = CpuTree::build_from_hashed_leaves(leaves).unwrap(); + + let be = math_cuda::device::backend().unwrap(); + let stream = be.next_stream(); + let nodes_dev = stream.clone_htod(&gpu_nodes).unwrap(); + stream.synchronize().unwrap(); + + let depth = log_n as usize; + for c in 0..=depth.min(6) { + let got = math_cuda::merkle::read_cap_dev(&nodes_dev, leaves_len, c, &stream).unwrap(); + let want: Vec = cpu_tree + .cap(c) + .unwrap() + .iter() + .flat_map(|n| n.iter().copied()) + .collect(); + assert_eq!(got.len(), (1 << c) * 32, "log_n={log_n} c={c}"); + assert_eq!(got, want, "keccak cap mismatch: log_n={log_n} c={c}"); + assert_eq!( + got, + gpu_nodes[((1 << c) - 1) * 32..((2 << c) - 1) * 32], + "log_n={log_n} c={c}: not the heap slice" + ); + } + let root = math_cuda::merkle::read_cap_dev(&nodes_dev, leaves_len, 0, &stream).unwrap(); + assert_eq!(root, cpu_tree.root.to_vec(), "c = 0 is the root"); +} + +#[test] +fn keccak_cap_matches_the_host_cap_small() { + for log_n in 1u32..=8 { + keccak_cap_parity(log_n, 300 + log_n as u64); + } +} + +#[test] +fn keccak_cap_matches_the_host_cap_large() { + for log_n in [12u32, 18, 22] { + keccak_cap_parity(log_n, 9000 + log_n as u64); + } +} + +/// RPX trees: the read is hash-agnostic (a D2H of the heap slice), so it is +/// pinned against the device builder's own full node buffer, whose layout the +/// existing RPX tree parity tests pin against the host. +#[test] +fn rpx_cap_is_the_heap_slice() { + for log_n in [1u32, 2, 5, 10, 16] { + let leaves_len = 1usize << log_n; + // RPX digests are four canonical Goldilocks limbs; reduce the random + // bytes below the modulus so the device hashes valid field elements. + let leaves: Vec<[u8; 32]> = random_leaves(leaves_len, 77 + log_n as u64) + .into_iter() + .map(|mut l| { + for limb in l.chunks_exact_mut(8) { + limb[7] &= 0x7f; + } + l + }) + .collect(); + let gpu_nodes = math_cuda::rpx::build_merkle_tree_on_device(&flat(&leaves)).unwrap(); + let be = math_cuda::device::backend().unwrap(); + let stream = be.next_stream(); + let nodes_dev = stream.clone_htod(&gpu_nodes).unwrap(); + stream.synchronize().unwrap(); + let depth = log_n as usize; + for c in 0..=depth.min(6) { + let got = math_cuda::merkle::read_cap_dev(&nodes_dev, leaves_len, c, &stream).unwrap(); + assert_eq!( + got, + gpu_nodes[((1 << c) - 1) * 32..((2 << c) - 1) * 32], + "rpx: log_n={log_n} c={c}" + ); + } + } +} + +/// The resident tree a real R2 commit keeps (`GpuMerkleTree`): the cap read +/// off it equals its full node buffer's heap slice, and `c = 0` its root. +#[test] +fn a_kept_composition_tree_serves_its_cap() { + let lde_size = 1usize << 12; + let mut rng = ChaCha8Rng::seed_from_u64(4242); + let parts: Vec> = (0..2) + .map(|_| { + (0..3 * lde_size) + .map(|_| rng.gen_range(0..0xFFFF_FFFF_0000_0001u64)) + .collect() + }) + .collect(); + let refs: Vec<&[u64]> = parts.iter().map(|p| p.as_slice()).collect(); + let tree = math_cuda::merkle::build_comp_poly_tree_from_evals_ext3_keep(&refs).unwrap(); + let be = math_cuda::device::backend().unwrap(); + let stream = be.next_stream(); + let all = stream.clone_dtoh(tree.nodes.as_ref()).unwrap(); + stream.synchronize().unwrap(); + let depth = tree.leaves_len.trailing_zeros() as usize; + assert_eq!(tree.leaves_len, lde_size / 2); + for c in 0..=depth.min(6) { + let got = + math_cuda::merkle::read_cap_dev(&tree.nodes, tree.leaves_len, c, &stream).unwrap(); + assert_eq!(got, all[((1 << c) - 1) * 32..((2 << c) - 1) * 32], "c={c}"); + } + let root = math_cuda::merkle::read_cap_dev(&tree.nodes, tree.leaves_len, 0, &stream).unwrap(); + assert_eq!(root, tree.root.to_vec()); +} diff --git a/crypto/math-cuda/tests/merkle_root_parity.rs b/crypto/math-cuda/tests/merkle_root_parity.rs index 410828268..53fae8a1e 100644 --- a/crypto/math-cuda/tests/merkle_root_parity.rs +++ b/crypto/math-cuda/tests/merkle_root_parity.rs @@ -15,7 +15,13 @@ use math::field::goldilocks::GoldilocksField; use math::polynomial::Polynomial; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha8Rng; -use stark::prover::{IsStarkProver, Prover}; +use stark::config::KeccakStarkHash; +use stark::prover::{GenericProver, IsStarkProver}; + +/// The keccak prover, named: these tests compare against the CUDA keccak +/// kernels, so the CPU side must say keccak rather than follow the default +/// alias (BLAKE3 since the P-a flip). +type Prover = GenericProver; type Fp3 = FieldElement; @@ -302,6 +308,7 @@ fn new_row_major_pipeline_base_root_matches_cpu() { let (handle, _lde) = math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep( &row_major, None, + math_cuda::DeviceHash::Keccak256, n, num_cols, blowup, @@ -366,6 +373,7 @@ fn new_row_major_pipeline_ext3_root_matches_cpu() { let (handle, _lde) = math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep( &row_major, + math_cuda::DeviceHash::Keccak256, n, num_cols, blowup, diff --git a/crypto/math-cuda/tests/ntt_cm_parity.rs b/crypto/math-cuda/tests/ntt_cm_parity.rs new file mode 100644 index 000000000..d0fc15339 --- /dev/null +++ b/crypto/math-cuda/tests/ntt_cm_parity.rs @@ -0,0 +1,558 @@ +//! The column-major LDE engine (`math_cuda::lde_cm`) against the legacy +//! pipelines, entry point by entry point, on the same inputs in one process +//! (`with_engine` forces the pipeline per call). +//! +//! What must hold for the engine to be a pure performance change: +//! - the Merkle roots — and for the split trees the downloaded precomputed +//! node bytes — are identical; +//! - the device LDE handles hold the same field values (compared canonically: +//! the engine may leave a different non-canonical representative, which +//! nothing downstream observes — the hashes and serialisation canonicalise); +//! - the trace-domain snapshot is identical, raw u64 for raw u64; +//! - the host outputs of the batched entry points are identical (canonically); +//! - the WHIR commit's codeword (canonically) and Merkle nodes (bytes) are +//! identical under either encoding. +//! +//! The engine's arithmetic is also pinned on the host by +//! `tests/host_kat/ntt_cm_host_kat.cpp`; `engine_matches_the_cpu_lde` anchors the +//! device engine to the CPU `Polynomial::coset_lde_full_expand` independently +//! of the legacy GPU path. + +use math::fft::bowers_fft::LayerTwiddles; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsPrimeField; +use math::polynomial::Polynomial; +use math_cuda::DeviceHash; +use math_cuda::lde_cm::{codewords_encoded, columns_extended, spread_supports, with_engine}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; + +const COSET_OFFSET: u64 = 7; + +/// `weights[i] = g^i / n` (evaluations) or `g^i` (coefficients). +fn weights_u64(n: usize, g: u64, with_inv_n: bool) -> Vec { + let mut cur = if with_inv_n { + Fp::from(n as u64).inv().unwrap() + } else { + Fp::one() + }; + let g_fp = Fp::from_raw(g); + let mut w = Vec::with_capacity(n); + for _ in 0..n { + w.push(*cur.value()); + cur = &cur * &g_fp; + } + w +} + +fn random(rng: &mut ChaCha8Rng, len: usize) -> Vec { + (0..len).map(|_| rng.r#gen::()).collect() +} + +fn canon(xs: &[u64]) -> Vec { + xs.iter().map(GoldilocksField::canonical).collect() +} + +fn assert_same_values(engine: &[u64], legacy: &[u64], what: &str) { + assert_eq!(engine.len(), legacy.len(), "{what}: length"); + let (a, b) = (canon(engine), canon(legacy)); + if let Some(i) = (0..a.len()).find(|&i| a[i] != b[i]) { + panic!( + "{what}: first difference at {i}: engine {:#x} legacy {:#x}", + a[i], b[i] + ); + } +} + +/// D2H a handle's buffer once its `ready` event has fired. +fn download( + buf: &math_cuda::CudaSlice, + ready: Option<&math_cuda::device::PooledEvent>, +) -> Vec { + let be = math_cuda::device::backend().expect("cuda backend"); + let stream = be.next_stream(); + if let Some(ev) = ready { + stream.wait(ev.event()).expect("wait on the ready event"); + } + let out = stream.clone_dtoh(buf).expect("D2H"); + stream.synchronize().expect("D2H sync"); + out +} + +const HASHES: [DeviceHash; 3] = [ + DeviceHash::Rpx256, + DeviceHash::Keccak256, + DeviceHash::Blake3, +]; + +/// (log_n, blowup, cols). The small shapes run in milliseconds; the last two +/// put several columns in several L2 chunks (a 2^20-row column at blowup 4 +/// is ~40 MB, so every column is its own chunk), which is what exercises the +/// chunk walk — and, for the aux commit, its descending order over a source +/// that lives inside the destination. +const SHAPES: &[(usize, usize, usize)] = &[ + (4, 2, 1), + (4, 16, 3), + (5, 4, 7), + (6, 2, 37), + (8, 8, 5), + (9, 2, 316), + (10, 4, 13), + (12, 2, 64), + (13, 4, 9), + (16, 2, 11), + (20, 4, 3), + (21, 2, 2), +]; + +/// The main (base) commit: host and pre-uploaded input, both leaf layouts. +#[test] +fn base_commit_engine_matches_legacy() { + let be = math_cuda::device::backend().expect("cuda backend"); + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + let n = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(0xB45E_0000 + i as u64); + let row_major = random(&mut rng, n * cols); + let weights = weights_u64(n, COSET_OFFSET, true); + let predev = { + let stream = be.next_stream(); + let d = stream.clone_htod(&row_major).expect("predev upload"); + stream.synchronize().expect("predev sync"); + d + }; + for (j, rpl) in [2usize, 1].into_iter().enumerate() { + let hash = HASHES[(i + j) % HASHES.len()]; + for dev_input in [false, true] { + let what = format!( + "base log_n={log_n} blowup={blowup} cols={cols} rpl={rpl} {hash:?} predev={dev_input}" + ); + let run = |on: bool| { + with_engine(on, || { + math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep_rpl( + &row_major, + dev_input.then_some(&predev), + hash, + n, + cols, + blowup, + &weights, + false, + rpl, + ) + }) + .expect("base commit") + }; + let before = columns_extended(); + let (engine, engine_host) = run(true); + assert!( + columns_extended() - before >= cols as u64, + "{what}: the entry point did not route through the engine" + ); + let (legacy, legacy_host) = run(false); + assert!( + engine_host.is_empty() && legacy_host.is_empty(), + "{what}: device-only" + ); + assert_eq!( + engine.tree.as_ref().unwrap().root, + legacy.tree.as_ref().unwrap().root, + "{what}: root" + ); + assert_eq!( + engine.tree.as_ref().unwrap().leaves_len, + legacy.tree.as_ref().unwrap().leaves_len + ); + assert_eq!( + (engine.m, engine.lde_size, engine.trace_rows), + (legacy.m, legacy.lde_size, legacy.trace_rows) + ); + assert_same_values( + &download(&engine.buf, engine.ready.as_deref()), + &download(&legacy.buf, legacy.ready.as_deref()), + &what, + ); + assert_eq!( + download(engine.trace_dev.as_ref().unwrap(), engine.ready.as_deref()), + download(legacy.trace_dev.as_ref().unwrap(), legacy.ready.as_deref()), + "{what}: trace snapshot" + ); + } + } + } +} + +/// The aux (ext3) commits: no snapshot, so the engine reads a column-major +/// copy of the trace staged at the head of the LDE buffer itself. +#[test] +fn ext3_commit_engine_matches_legacy() { + let be = math_cuda::device::backend().expect("cuda backend"); + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + // `cols` ext3 columns: keep the widest shapes affordable. + let cols = cols.min(40); + let n = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(0xE473_0000 + i as u64); + let row_major = random(&mut rng, n * cols * 3); + let weights = weights_u64(n, COSET_OFFSET, true); + let dev_input = { + let stream = be.next_stream(); + let d = stream.clone_htod(&row_major).expect("upload"); + stream.synchronize().expect("sync"); + d + }; + for (j, rpl) in [2usize, 1].into_iter().enumerate() { + let hash = HASHES[(i + j + 1) % HASHES.len()]; + let what = format!("ext3 log_n={log_n} blowup={blowup} cols={cols} rpl={rpl} {hash:?}"); + let host = |on: bool| { + with_engine(on, || { + math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep_rpl( + &row_major, hash, n, cols, blowup, &weights, false, rpl, + ) + }) + .expect("ext3 commit") + .0 + }; + let dev = |on: bool| { + with_engine(on, || { + math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep_dev_rpl( + &dev_input, hash, n, cols, blowup, &weights, false, rpl, + ) + }) + .expect("ext3 dev commit") + .0 + }; + let legacy = host(false); + let legacy_vals = download(&legacy.buf, legacy.ready.as_deref()); + for (engine, form) in [(host(true), "host"), (dev(true), "dev")] { + assert_eq!( + engine.tree.as_ref().unwrap().root, + legacy.tree.as_ref().unwrap().root, + "{what} {form}: root" + ); + assert_eq!((engine.m, engine.lde_size), (legacy.m, legacy.lde_size)); + assert_same_values( + &download(&engine.buf, engine.ready.as_deref()), + &legacy_vals, + &format!("{what} {form}"), + ); + } + } + } +} + +/// The preprocessed commit: two subset trees over one LDE. +#[test] +fn split_trees_engine_matches_legacy() { + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + if cols < 2 { + continue; // the split needs a column on each side + } + let n = 1usize << log_n; + let split = (cols / 3).max(1); + let mut rng = ChaCha8Rng::seed_from_u64(0x5B11_0000 + i as u64); + let row_major = random(&mut rng, n * cols); + let weights = weights_u64(n, COSET_OFFSET, true); + for (j, rpl) in [2usize, 1].into_iter().enumerate() { + let hash = HASHES[(i + j + 2) % HASHES.len()]; + let what = + format!("split log_n={log_n} blowup={blowup} cols={cols} rpl={rpl} {hash:?}"); + let run = |on: bool| { + with_engine(on, || { + math_cuda::lde::coset_lde_row_major_split_trees_rpl( + &row_major, None, hash, n, cols, blowup, &weights, split, true, false, rpl, + ) + }) + .expect("split commit") + }; + let (engine_pre, engine, _) = run(true); + let (legacy_pre, legacy, _) = run(false); + assert_eq!(engine_pre, legacy_pre, "{what}: precomputed tree nodes"); + assert_eq!( + engine.tree.as_ref().unwrap().root, + legacy.tree.as_ref().unwrap().root, + "{what}: multiplicity root" + ); + assert_same_values( + &download(&engine.buf, engine.ready.as_deref()), + &download(&legacy.buf, legacy.ready.as_deref()), + &what, + ); + assert_eq!( + download(engine.trace_dev.as_ref().unwrap(), engine.ready.as_deref()), + download(legacy.trace_dev.as_ref().unwrap(), legacy.ready.as_deref()), + "{what}: trace snapshot" + ); + } + } +} + +/// A caller that wants the row-major host LDE keeps the legacy path even with +/// the engine on (the engine's LDE is column-major), and gets the same bytes. +#[test] +fn host_lde_callers_keep_the_legacy_path() { + let (log_n, blowup, cols) = (10, 2, 5); + let n = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(0x4057); + let row_major = random(&mut rng, n * cols); + let weights = weights_u64(n, COSET_OFFSET, true); + let run = |on: bool| { + with_engine(on, || { + math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep( + &row_major, + None, + DeviceHash::Rpx256, + n, + cols, + blowup, + &weights, + true, + ) + }) + .expect("base commit with host copy") + }; + let (on, on_host) = run(true); + let (off, off_host) = run(false); + assert_eq!( + on_host.len(), + n * blowup * cols, + "the host copy is produced" + ); + assert_eq!(on_host, off_host, "identical row-major host LDE"); + assert_eq!(on.tree.unwrap().root, off.tree.unwrap().root); +} + +/// Every batched (column-major, in-slab) entry point. +#[test] +fn batched_entry_points_engine_match_legacy() { + let be = math_cuda::device::backend().expect("cuda backend"); + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + let cols = cols.min(24); + let n = 1usize << log_n; + let lde = n * blowup; + let mut rng = ChaCha8Rng::seed_from_u64(0xBA7C_0000 + i as u64); + let what = format!("batched log_n={log_n} blowup={blowup} cols={cols}"); + let w_evals = weights_u64(n, COSET_OFFSET, true); + let w_coeffs = weights_u64(n, COSET_OFFSET, false); + + // Base columns. + let base: Vec> = (0..cols).map(|_| random(&mut rng, n)).collect(); + let slices: Vec<&[u64]> = base.iter().map(Vec::as_slice).collect(); + let batch = |on: bool| { + with_engine(on, || { + math_cuda::lde::coset_lde_batch_base(&slices, blowup, &w_evals) + }) + .expect("batch base") + }; + let (a, b) = (batch(true), batch(false)); + for c in 0..cols { + assert_same_values(&a[c], &b[c], &format!("{what} batch_base col {c}")); + } + let into = |on: bool| { + let mut outs = vec![vec![0u64; lde]; cols]; + { + let mut views: Vec<&mut [u64]> = outs.iter_mut().map(Vec::as_mut_slice).collect(); + with_engine(on, || { + math_cuda::lde::coset_lde_batch_base_into(&slices, blowup, &w_evals, &mut views) + }) + .expect("batch base into"); + } + outs + }; + assert_eq!( + into(true).iter().map(|c| canon(c)).collect::>(), + into(false).iter().map(|c| canon(c)).collect::>(), + "{what}: batch_base_into" + ); + let hash = HASHES[i % HASHES.len()]; + let leaf_hash = |on: bool| { + let mut outs = vec![vec![0u64; lde]; cols]; + let mut leaves = vec![0u8; (lde / 2) * 32]; + { + let mut views: Vec<&mut [u64]> = outs.iter_mut().map(Vec::as_mut_slice).collect(); + with_engine(on, || { + math_cuda::lde::coset_lde_batch_base_into_with_leaf_hash( + &slices, + hash, + blowup, + &w_evals, + &mut views, + &mut leaves, + ) + }) + .expect("batch base leaf hash"); + } + (outs.iter().map(|c| canon(c)).collect::>(), leaves) + }; + assert_eq!( + leaf_hash(true), + leaf_hash(false), + "{what}: leaf hash {hash:?}" + ); + + // Ext3 columns (interleaved), their slab-resident form, and the + // coefficient form. + let ext: Vec> = (0..cols).map(|_| random(&mut rng, 3 * n)).collect(); + let ext_slices: Vec<&[u64]> = ext.iter().map(Vec::as_slice).collect(); + let ext3_into = |on: bool, coeffs: bool| { + let mut outs = vec![vec![0u64; 3 * lde]; cols]; + { + let mut views: Vec<&mut [u64]> = outs.iter_mut().map(Vec::as_mut_slice).collect(); + with_engine(on, || { + if coeffs { + math_cuda::lde::evaluate_poly_coset_batch_ext3_into( + &ext_slices, + n, + blowup, + &w_coeffs, + &mut views, + ) + } else { + math_cuda::lde::coset_lde_batch_ext3_into( + &ext_slices, + n, + blowup, + &w_evals, + &mut views, + ) + } + }) + .expect("ext3 into"); + } + outs.iter().map(|c| canon(c)).collect::>() + }; + assert_eq!( + ext3_into(true, false), + ext3_into(false, false), + "{what}: ext3_into" + ); + assert_eq!( + ext3_into(true, true), + ext3_into(false, true), + "{what}: evaluate_poly ext3 (coefficients)" + ); + let comp_tree = |on: bool| { + let mut outs = vec![vec![0u64; 3 * lde]; cols]; + let mut nodes = vec![0u8; (lde - 1) * 32]; + { + let mut views: Vec<&mut [u64]> = outs.iter_mut().map(Vec::as_mut_slice).collect(); + with_engine(on, || { + math_cuda::lde::evaluate_poly_coset_batch_ext3_into_with_merkle_tree( + &ext_slices, + hash, + n, + blowup, + &w_coeffs, + &mut views, + &mut nodes, + ) + }) + .expect("evaluate with tree"); + } + (outs.iter().map(|c| canon(c)).collect::>(), nodes) + }; + assert_eq!( + comp_tree(true), + comp_tree(false), + "{what}: evaluate_poly tree {hash:?}" + ); + + let slabs_keep = |on: bool| { + let stream = be.next_stream(); + let mut host = vec![0u64; 3 * cols * lde]; + for s in 0..3 * cols { + for r in 0..n { + host[s * lde + r] = ext[s / 3][r * 3 + s % 3]; + } + } + let buf = stream.clone_htod(&host).expect("slab upload"); + let handle = with_engine(on, || { + math_cuda::lde::coset_lde_batch_ext3_slabs_keep( + &stream, buf, cols, n, blowup, &w_evals, None, + ) + }) + .expect("slabs keep"); + download(&handle.buf, handle.ready.as_deref()) + }; + assert_same_values( + &slabs_keep(true), + &slabs_keep(false), + &format!("{what}: slabs_keep"), + ); + } +} + +/// The engine against the CPU LDE, independently of the legacy GPU path. +#[test] +fn engine_matches_the_cpu_lde() { + for &(log_n, blowup) in &[(4usize, 2usize), (7, 8), (11, 4), (14, 2), (18, 4)] { + let n = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(0xC9A0 + log_n as u64); + let evals = random(&mut rng, n); + let weights = weights_u64(n, COSET_OFFSET, true); + let gpu = with_engine(true, || { + math_cuda::lde::coset_lde_batch_base(&[evals.as_slice()], blowup, &weights) + }) + .expect("engine lde") + .remove(0); + + let log_lde = (n * blowup).trailing_zeros() as u64; + let inv_tw = LayerTwiddles::::new_inverse(log_n as u64).unwrap(); + let fwd_tw = LayerTwiddles::::new(log_lde).unwrap(); + let w_fe: Vec = weights.iter().map(|&w| Fp::from_raw(w)).collect(); + let mut cpu: Vec = evals.iter().map(|&x| Fp::from_raw(x)).collect(); + Polynomial::coset_lde_full_expand::( + &mut cpu, blowup, &w_fe, &inv_tw, &fwd_tw, + ) + .expect("cpu lde"); + let cpu: Vec = cpu.iter().map(|e| *e.value()).collect(); + assert_same_values(&gpu, &cpu, &format!("cpu log_n={log_n} blowup={blowup}")); + } +} + +/// The WHIR commit's encoding (spread + NTT of one codeword) through the column +/// engine against the legacy `lift_spread` + `run_ntt_body`: the codeword +/// (canonically) and the Merkle nodes (bytes) are identical. `(1, 2)` is below +/// the engine's smallest transform and must stay on the legacy path. +#[test] +fn whir_codeword_engine_matches_legacy() { + let shapes: &[(usize, usize, usize)] = &[ + // (log_evals, log_blowup, log_folding) + (1, 2, 1), + (2, 2, 1), + (4, 0, 2), + (6, 2, 4), + (8, 4, 4), + (10, 1, 4), + (12, 2, 4), + (16, 3, 4), + (18, 2, 4), + (21, 2, 4), + ]; + for (i, &(log_evals, log_blowup, log_folding)) in shapes.iter().enumerate() { + let mut rng = ChaCha8Rng::seed_from_u64(0x3171_0000 + i as u64); + // The commit takes canonical hypercube values. + let evals = canon(&random(&mut rng, 1usize << log_evals)); + let hash = [DeviceHash::Rpx256, DeviceHash::Keccak256][i % 2]; + let what = format!("whir log_evals={log_evals} log_blowup={log_blowup} {hash:?}"); + let before = codewords_encoded(); + let (engine_vals, engine_nodes) = with_engine(true, || { + math_cuda::whir::commit_codeword_to_host(&evals, log_blowup, log_folding, hash) + }) + .expect("engine commit"); + let routed = codewords_encoded() > before; + let supported = spread_supports((log_evals + log_blowup) as u32, log_blowup as u32); + // Other tests may encode concurrently, so only the supported shapes' + // "it routed" is certain; an unsupported one is checked by its values. + if supported { + assert!(routed, "{what}: the encoding did not take the engine"); + } + let (vals, nodes) = with_engine(false, || { + math_cuda::whir::commit_codeword_to_host(&evals, log_blowup, log_folding, hash) + }) + .expect("legacy commit"); + assert_same_values(&engine_vals, &vals, &what); + assert_eq!(engine_nodes, nodes, "{what}: Merkle nodes"); + } +} diff --git a/crypto/math-cuda/tests/one_lde_buffer.rs b/crypto/math-cuda/tests/one_lde_buffer.rs new file mode 100644 index 000000000..7bf8945fc --- /dev/null +++ b/crypto/math-cuda/tests/one_lde_buffer.rs @@ -0,0 +1,217 @@ +//! One LDE buffer: the fused row-major commits transpose their LDE to +//! column-major IN PLACE, so no second LDE-sized device allocation exists. +//! +//! The in-place transpose only moves whole runs of values, so its output must +//! equal, u64 for u64, the row-major host copy the very same call drains +//! before transposing. That is the pin here, for every entry point that +//! transposes: base, ext3 and the preprocessed split-tree pair. Roots against +//! the CPU commit are pinned by `merkle_root_parity` / `blake3_fused_parity`. +//! +//! `vram_arm` is the `#[ignore]`d measurement arm for the 10 Hz sampler at +//! the production shape (2^21 × 316 by default; env-overridable). Run it +//! with `LAMBDA_VM_MEMPOOL_RELEASE_MB=0` and the sampler attached: +//! +//! ```text +//! LAMBDA_VM_MEMPOOL_RELEASE_MB=0 cargo test -p math-cuda --release \ +//! --test one_lde_buffer vram_arm -- --ignored --nocapture +//! ``` + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math_cuda::DeviceHash; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; + +const COSET_OFFSET: u64 = 7; + +/// `weights[i] = g^i / n` — the `LdeTwiddles::coset_weights` format. +fn coset_weights_u64(n: usize, g: u64) -> Vec { + let inv_n = Fp::from(n as u64).inv().unwrap(); + let g_fp = Fp::from_raw(g); + let mut w = Vec::with_capacity(n); + let mut cur = inv_n; + for _ in 0..n { + w.push(*cur.value()); + cur = &cur * &g_fp; + } + w +} + +fn random_row_major(rng: &mut ChaCha8Rng, n: usize, cols: usize) -> Vec { + (0..n * cols).map(|_| rng.r#gen::()).collect() +} + +/// D2H the handle's column-major buffer once its `ready` event has fired. +fn download( + buf: &math_cuda::CudaSlice, + ready: Option<&math_cuda::device::PooledEvent>, +) -> Vec { + let be = math_cuda::device::backend().expect("cuda backend"); + let stream = be.next_stream(); + if let Some(ev) = ready { + stream + .wait(ev.event()) + .expect("wait on the handle's ready event"); + } + stream.clone_dtoh(buf).expect("D2H of the column-major LDE") +} + +/// `dev[c * rows + r] == host[r * cols + c]` for every element, raw u64s. +fn assert_column_major_of(dev: &[u64], host: &[u64], rows: usize, cols: usize, what: &str) { + assert_eq!(dev.len(), rows * cols, "{what}: device length"); + assert_eq!(host.len(), rows * cols, "{what}: host length"); + for c in 0..cols { + let col = &dev[c * rows..(c + 1) * rows]; + for (r, &v) in col.iter().enumerate() { + let want = host[r * cols + c]; + assert_eq!( + v, want, + "{what}: row {r} col {c}: device {v:#x} != host {want:#x}" + ); + } + } +} + +fn hash_for(i: usize) -> DeviceHash { + if i.is_multiple_of(2) { + DeviceHash::Keccak256 + } else { + DeviceHash::Blake3 + } +} + +/// Shapes small enough to run in seconds yet wide enough to exercise the +/// multi-block run permutation (8 blocks from 2^3 rows up) with odd column +/// counts, a single column (identity), and both production blowups. +const SHAPES: &[(usize, usize, usize)] = &[ + // (log_n, blowup, cols) + (1, 2, 2), + (1, 4, 3), + (2, 2, 7), + (3, 2, 1), + (3, 4, 5), + (5, 2, 37), + (6, 4, 3), + (8, 2, 316), + (9, 4, 37), + (10, 2, 13), + (12, 2, 316), + (12, 4, 9), +]; + +#[test] +fn base_handle_is_the_in_place_transpose_of_the_host_lde() { + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + let n = 1usize << log_n; + let rows = n * blowup; + let mut rng = ChaCha8Rng::seed_from_u64(0x1DE_0000 + i as u64); + let row_major = random_row_major(&mut rng, n, cols); + let weights = coset_weights_u64(n, COSET_OFFSET); + let (handle, host_lde) = math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep( + &row_major, + None, + hash_for(i), + n, + cols, + blowup, + &weights, + true, + ) + .expect("fused base commit"); + assert_eq!(handle.m, cols); + assert_eq!(handle.lde_size, rows); + let dev = download(handle.buf.as_ref(), handle.ready.as_deref()); + assert_column_major_of( + &dev, + &host_lde, + rows, + cols, + &format!("base log_n={log_n} blowup={blowup} cols={cols}"), + ); + // The trace snapshot is the plain transpose of the input rows. + let snap = handle + .trace_dev + .as_ref() + .expect("base keep path retains the trace"); + let snap = download(snap.as_ref(), handle.ready.as_deref()); + assert_column_major_of( + &snap, + &row_major, + n, + cols, + &format!("snapshot log_n={log_n} cols={cols}"), + ); + } +} + +#[test] +fn ext3_handle_is_the_in_place_transpose_of_the_host_lde() { + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + let n = 1usize << log_n; + let rows = n * blowup; + // `cols` ext3 columns = `3 * cols` base-field columns in row-major. + let base_cols = cols * 3; + let mut rng = ChaCha8Rng::seed_from_u64(0x31DE_0000 + i as u64); + let row_major = random_row_major(&mut rng, n, base_cols); + let weights = coset_weights_u64(n, COSET_OFFSET); + let (handle, host_lde) = math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep( + &row_major, + hash_for(i + 1), + n, + cols, + blowup, + &weights, + true, + ) + .expect("fused ext3 commit"); + assert_eq!(handle.m, cols); + assert_eq!(handle.lde_size, rows); + let dev = download(handle.buf.as_ref(), handle.ready.as_deref()); + assert_column_major_of( + &dev, + &host_lde, + rows, + base_cols, + &format!("ext3 log_n={log_n} blowup={blowup} cols={cols}"), + ); + } +} + +#[test] +fn split_tree_handle_is_the_in_place_transpose_of_the_host_lde() { + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + if cols < 2 { + continue; // the split needs a column on each side + } + let n = 1usize << log_n; + let rows = n * blowup; + let split_col = (cols / 2).max(1); + let mut rng = ChaCha8Rng::seed_from_u64(0x51DE_0000 + i as u64); + let row_major = random_row_major(&mut rng, n, cols); + let weights = coset_weights_u64(n, COSET_OFFSET); + let (pre_nodes, handle, host_lde) = math_cuda::lde::coset_lde_row_major_split_trees( + &row_major, + None, + hash_for(i), + n, + cols, + blowup, + &weights, + split_col, + true, + true, + ) + .expect("fused split-tree commit"); + assert!(pre_nodes.is_some(), "precomputed tree requested"); + let dev = download(handle.buf.as_ref(), handle.ready.as_deref()); + assert_column_major_of( + &dev, + &host_lde, + rows, + cols, + &format!("split log_n={log_n} blowup={blowup} cols={cols}"), + ); + } +} diff --git a/crypto/math-cuda/tests/p1_device_parity.rs b/crypto/math-cuda/tests/p1_device_parity.rs new file mode 100644 index 000000000..c73ddcbaa --- /dev/null +++ b/crypto/math-cuda/tests/p1_device_parity.rs @@ -0,0 +1,467 @@ +//! ZisK's Poseidon1 on the device must produce the host's bytes: every +//! `DeviceHash::Poseidon1` leaf geometry the kernels have, the 4-ary trees, +//! the path gather and the grind, against the host backends +//! (`crypto::merkle_tree::backends::p1`, `crypto::fiat_shamir::p1_transcript`), +//! whose primitives are pinned to ZisK's own vectors +//! (`crypto::hash::poseidon1_stark::zisk_kat`). The kernels' arithmetic and +//! index math are also pinned without a GPU (`make test-p1w16-host-kat`); what +//! needs the GPU is here. Needs a GPU: +//! +//! cargo test -p math-cuda --release --test p1_device_parity -- --test-threads=1 + +use crypto::fiat_shamir::p1_transcript::P1GrindDigest; +use crypto::merkle_tree::backends::p1::{P1BatchBackend, P1PairBackend}; +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::fft::bit_reversing::reverse_index; +use math::fft::two_half_fft::TwoHalfTwiddles; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsField; +use math::polynomial::Polynomial; +use math::traits::AsBytes; + +type Ext3 = Degree3GoldilocksExtensionField; +type Fp3 = FieldElement; +type Fp = FieldElement; + +const COSET_OFFSET: u64 = 7; + +/// splitmix64. +struct SplitMix(u64); + +impl SplitMix { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +fn coset_weights(n: usize, g: u64) -> Vec { + let inv_n = Fp::from(n as u64).inv().unwrap(); + let g_fp = Fp::from_raw(g); + let mut w = Vec::with_capacity(n); + let mut cur = inv_n; + for _ in 0..n { + w.push(cur); + cur = &cur * &g_fp; + } + w +} + +fn rand_ext3(rng: &mut SplitMix) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.next()), + Fp::from_raw(rng.next()), + Fp::from_raw(rng.next()), + ]) +} + +fn nodes_of(bytes: &[u8]) -> Vec<[u8; 32]> { + bytes + .chunks_exact(32) + .map(|c| c.try_into().expect("32-byte node")) + .collect() +} + +/// The host commit the STARK's row trees make: leaf `i` hashes the +/// bit-reversed rows `R·i .. R·i + R − 1` of the row-major `data`. +fn host_row_tree( + data: &[FieldElement], + num_cols: usize, + rows_per_leaf: usize, +) -> MerkleTree> +where + E: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + Vec>: Sync + Send, +{ + let num_rows = data.len() / num_cols; + let leaves: Vec>> = (0..num_rows / rows_per_leaf) + .map(|leaf| { + (0..rows_per_leaf) + .flat_map(|k| { + let r = reverse_index(rows_per_leaf * leaf + k, num_rows as u64); + data[r * num_cols..(r + 1) * num_cols].to_vec() + }) + .collect() + }) + .collect(); + MerkleTree::>::build(&leaves).expect("non-empty") +} + +/// The whole node buffer of a device tree, root first. +fn device_nodes(tree: &math_cuda::lde::GpuMerkleTree) -> Vec<[u8; 32]> { + let be = math_cuda::device::backend().expect("backend"); + let stream = be.next_stream(); + let n = tree.total_nodes(); + let mut bytes = vec![0u8; n * 32]; + stream + .memcpy_dtoh(&tree.nodes.slice(0..n * 32), &mut bytes) + .expect("node download"); + stream.synchronize().expect("sync"); + nodes_of(&bytes) +} + +// =========================================================================== +// The fused LDE + commit pipelines (the STARK's trace trees): both leaf +// layouts, even and odd 4-ary depths. +// =========================================================================== + +#[test] +fn p1_fused_trace_roots_match_the_host() { + for log_n in [4usize, 5, 7, 10] { + for blowup in [2usize, 4] { + for num_cols in [1usize, 5, 12, 13] { + for rows_per_leaf in [1usize, 2] { + // `retain_host_lde = true` takes the row-major leaf kernels, + // `false` the column-major engine's. + for retain in [true, false] { + let n = 1usize << log_n; + let mut rng = + SplitMix((log_n * 10_000 + blowup * 1000 + num_cols * 10) as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rng.next()).collect()) + .collect(); + let mut row_major = vec![0u64; n * num_cols]; + let mut buf: Vec = vec![Fp::from(0u64); n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, &v) in col.iter().enumerate() { + row_major[r * num_cols + c] = v; + buf[r * num_cols + c] = Fp::from_raw(v); + } + } + let weights_fp = coset_weights(n, COSET_OFFSET); + let weights: Vec = weights_fp.iter().map(|w| *w.value()).collect(); + let (handle, _) = + math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep_rpl( + &row_major, + None, + math_cuda::DeviceHash::Poseidon1, + n, + num_cols, + blowup, + &weights, + retain, + rows_per_leaf, + ) + .expect("fused P1 pipeline"); + let tree = handle.tree.as_ref().expect("resident tree"); + assert_eq!(tree.arity, 4); + + let log_lde = (n * blowup).trailing_zeros() as usize; + let inv_tw = TwoHalfTwiddles::::new(log_n, true).unwrap(); + let fwd_tw = + TwoHalfTwiddles::::new(log_lde, false).unwrap(); + Polynomial::::coset_lde_full_expand_row_major::( + &mut buf, + num_cols, + blowup, + &weights_fp, + &inv_tw, + &fwd_tw, + ) + .expect("CPU row-major LDE"); + assert_eq!( + tree.root, + host_row_tree(&buf, num_cols, rows_per_leaf).root, + "log_n={log_n} blowup={blowup} cols={num_cols} rpl={rows_per_leaf} \ + retain={retain}" + ); + } + } + } + } + } +} + +/// The ext3 (aux) trace commit, row pairs. +#[test] +fn p1_fused_ext3_roots_match_the_host() { + for log_n in [4usize, 7] { + for num_cols in [1usize, 4] { + for rows_per_leaf in [1usize, 2] { + let n = 1usize << log_n; + let blowup = 4usize; + let mut rng = SplitMix((log_n * 100 + num_cols) as u64 + 77); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + let mut row_major = vec![0u64; n * num_cols * 3]; + let mut buf: Vec = vec![Fp3::from(0u64); n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, v) in col.iter().enumerate() { + buf[r * num_cols + c] = *v; + for k in 0..3 { + row_major[(r * num_cols + c) * 3 + k] = *v.value()[k].value(); + } + } + } + let weights_fp = coset_weights(n, COSET_OFFSET); + let weights: Vec = weights_fp.iter().map(|w| *w.value()).collect(); + let (handle, _) = + math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep_rpl( + &row_major, + math_cuda::DeviceHash::Poseidon1, + n, + num_cols, + blowup, + &weights, + true, + rows_per_leaf, + ) + .expect("fused ext3 P1 pipeline"); + let log_lde = (n * blowup).trailing_zeros() as usize; + let inv_tw = TwoHalfTwiddles::::new(log_n, true).unwrap(); + let fwd_tw = TwoHalfTwiddles::::new(log_lde, false).unwrap(); + Polynomial::::coset_lde_full_expand_row_major::( + &mut buf, + num_cols, + blowup, + &weights_fp, + &inv_tw, + &fwd_tw, + ) + .unwrap(); + assert_eq!( + handle.tree.as_ref().expect("resident tree").root, + host_row_tree(&buf, num_cols, rows_per_leaf).root, + "ext3 log_n={log_n} cols={num_cols} rpl={rows_per_leaf}" + ); + } + } + } +} + +/// Column ranges, leaf for leaf against the batched backend's `hash_data` +/// over the same felts (the tagged leaf at every width). +#[test] +fn p1_row_major_range_leaves_match_the_host() { + let log_n = 6u32; + let n = 1usize << log_n; + let m = 7usize; + let mut rng = SplitMix(0xBEEF); + let data: Vec = (0..n * m).map(|_| rng.next()).collect(); + for rows_per_leaf in [1usize, 2] { + for (cs, ce) in [(0usize, m), (0, 3), (3, m), (2, 5)] { + let gpu = math_cuda::lde::row_major_leaves( + math_cuda::DeviceHash::Poseidon1, + &data, + m, + cs, + ce, + n, + rows_per_leaf, + ) + .expect("device ranged leaves"); + assert_eq!(gpu.len(), n / rows_per_leaf * 32); + for leaf in 0..n / rows_per_leaf { + let mut felts: Vec = Vec::new(); + for k in 0..rows_per_leaf { + let br = reverse_index(rows_per_leaf * leaf + k, n as u64); + for c in cs..ce { + felts.push(Fp::from_raw(data[br * m + c])); + } + } + let want = + as IsMerkleTreeBackend>::hash_data(&felts); + assert_eq!( + &gpu[leaf * 32..(leaf + 1) * 32], + &want[..], + "leaf {leaf}, columns [{cs}, {ce}), rows_per_leaf {rows_per_leaf}" + ); + } + } + } +} + +// =========================================================================== +// The composition tree, the FRI trees and the bare 4-ary walk, node for node. +// =========================================================================== + +#[test] +fn p1_comp_poly_trees_match_the_host() { + for (log_lde, m) in [(4usize, 1usize), (5, 2), (10, 3), (13, 4)] { + for rows_per_leaf in [1usize, 2] { + let lde_size = 1usize << log_lde; + let mut rng = SplitMix((log_lde * 10 + m) as u64 + 99); + let parts: Vec> = (0..m) + .map(|_| (0..lde_size).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + let parts_u64: Vec> = parts + .iter() + .map(|p| { + p.iter() + .flat_map(|e| e.value().iter().map(|c| *c.value())) + .collect() + }) + .collect(); + let raw: Vec<&[u64]> = parts_u64.iter().map(|p| p.as_slice()).collect(); + let dev = math_cuda::p1_stark::build_comp_poly_tree_from_evals_ext3_keep_rpl( + &raw, + rows_per_leaf, + ) + .expect("device comp-poly tree"); + assert_eq!(dev.leaves_len, lde_size / rows_per_leaf); + let mut buf: Vec = vec![Fp3::from(0u64); lde_size * m]; + for (c, p) in parts.iter().enumerate() { + for (r, v) in p.iter().enumerate() { + buf[r * m + c] = *v; + } + } + let host = host_row_tree(&buf, m, rows_per_leaf); + assert_eq!( + dev.root, host.root, + "log_lde={log_lde} m={m} rpl={rows_per_leaf}" + ); + + // The whole node buffer, then paths. + let all = device_nodes(&dev); + assert_eq!( + all.as_slice(), + host.nodes(), + "node buffer log_lde={log_lde}" + ); + let be = math_cuda::device::backend().expect("backend"); + let stream = be.next_stream(); + let positions: Vec = [0usize, 1, 3, dev.leaves_len / 2, dev.leaves_len - 1] + .iter() + .map(|&p| p as u32) + .collect(); + let bytes = math_cuda::p1_stark::gather_paths_dev( + &dev.nodes, + dev.leaves_len, + &positions, + &stream, + ) + .expect("device paths"); + let per = 3 * math_cuda::p1_stark::depth(dev.leaves_len) * 32; + for (q, &p) in positions.iter().enumerate() { + let want_path = host + .get_proof_by_pos(p as usize) + .expect("host path") + .merkle_path; + assert_eq!( + nodes_of(&bytes[q * per..(q + 1) * per]), + want_path, + "path {p} log_lde={log_lde} rpl={rows_per_leaf}" + ); + } + } + } +} + +fn fri_tree_parity(num_leaves: usize, group: usize, seed: u64) { + let mut rng = SplitMix(seed); + let evals: Vec = (0..num_leaves * group) + .map(|_| rand_ext3(&mut rng)) + .collect(); + let evals_u64: Vec = evals + .iter() + .flat_map(|e| e.value().iter().map(|c| *c.value())) + .collect(); + let host_nodes: Vec<[u8; 32]> = if group == 2 { + let leaves: Vec<[Fp3; 2]> = evals.chunks_exact(2).map(|c| [c[0], c[1]]).collect(); + MerkleTree::>::build(&leaves) + .unwrap() + .nodes() + .to_vec() + } else { + let leaves: Vec> = evals.chunks_exact(group).map(<[Fp3]>::to_vec).collect(); + MerkleTree::>::build(&leaves) + .unwrap() + .nodes() + .to_vec() + }; + let gpu = math_cuda::p1_stark::build_fri_tree_from_evals_ext3(&evals_u64, group).unwrap(); + assert_eq!( + nodes_of(&gpu), + host_nodes, + "FRI tree: {num_leaves} leaves of {group}" + ); +} + +/// Pair and group trees: shallow (the tail alone) and deep enough that the +/// per-level kernel runs first, at even and odd 4-ary depths. Groups of 16 +/// ext3 values are WHIR's fold-block leaf (48 felts). +#[test] +fn p1_fri_trees_match_the_host() { + for log in [1u32, 2, 3, 5, 6, 11, 14, 15] { + for group in [2usize, 4, 16] { + fri_tree_parity(1 << log, group, 300 + log as u64 * 7 + group as u64); + } + } +} + +/// The bare 4-ary walk over random leaf digests, node for node, from two +/// leaves (the tail alone) to 2^13 (per-level launches, then the tail). +#[test] +fn p1_bare_trees_match_the_host() { + for leaves in [2usize, 4, 8, 32, 64, 1 << 10, 1 << 13] { + let mut rng = SplitMix(leaves as u64); + let hashed: Vec<[u8; 32]> = (0..leaves) + .map(|_| { + let d = [ + Fp::from(rng.next()), + Fp::from(rng.next()), + Fp::from(rng.next()), + Fp::from(rng.next()), + ]; + crypto::hash::rpx::digest_to_commitment(&d) + }) + .collect(); + let host = + MerkleTree::>::build_from_hashed_leaves(hashed.clone()) + .unwrap(); + let flat: Vec = hashed.iter().flatten().copied().collect(); + let gpu = math_cuda::p1_stark::build_merkle_tree_on_device(&flat).unwrap(); + assert_eq!(nodes_of(&gpu), host.nodes(), "{leaves} leaves"); + } +} + +// =========================================================================== +// The grind. +// =========================================================================== + +/// A real launch returns the smallest nonce the host predicate accepts, at +/// factors from the device floor (`GRIND_MIN_FACTOR`) up, and WHIR's 18; below +/// the floor the device declines (`None`) and the dispatcher's host search +/// takes the grind. +#[test] +fn p1_gpu_grind_returns_the_smallest_valid_nonce() { + let floor = math_cuda::grinding::GRIND_MIN_FACTOR; + for (seed, factor) in [ + ([14u8; 32], floor), + ([3u8; 32], 14u8), + ([77u8; 32], 16), + ([5u8; 32], 18), + ] { + let nonce = math_cuda::grinding::generate_nonce_p1_gpu( + &crypto::grinding::inner_hash_felts::(&seed, factor), + factor, + ) + .unwrap_or_else(|| panic!("GPU P1 grind at factor {factor} (needs a GPU)")); + assert!( + crypto::grinding::is_valid_nonce::(&seed, nonce, factor), + "GPU nonce {nonce} fails is_valid_nonce (factor {factor})" + ); + assert_eq!( + crypto::grinding::generate_nonce_smallest::(&seed, factor), + Some(nonce), + "GPU nonce {nonce} is not the smallest (factor {factor})" + ); + } + let below = floor - 2; + assert!( + math_cuda::grinding::generate_nonce_p1_gpu( + &crypto::grinding::inner_hash_felts::(&[3u8; 32], below), + below, + ) + .is_none(), + "the device grind must decline factor {below}, below its floor {floor}" + ); +} diff --git a/crypto/math-cuda/tests/p1_lean.rs b/crypto/math-cuda/tests/p1_lean.rs new file mode 100644 index 000000000..cdac73cda --- /dev/null +++ b/crypto/math-cuda/tests/p1_lean.rs @@ -0,0 +1,274 @@ +//! The Poseidon1 kernels' `_lean` twins +//! (`LAMBDA_VM_P1_LEAN`, on by default) write the plain kernels' bytes, on the card. +//! +//! The permutation itself is checked against the Rust host reference with the +//! other variants (`p1w16_device`, variant `l2`), and on the host against +//! `v2` and the field over edge words (`make test-p1w16-host-kat`). Here the +//! production twins run beside today's kernels, both sets loaded in one +//! process (`p1_stark::load_kernels`), over codewords of raw words with the +//! edge values planted, and over whole 4-ary trees. Needs a GPU (the box): +//! +//! cargo test -p math-cuda --release --test p1_lean + +use std::sync::Arc; + +use cudarc::driver::{CudaSlice, CudaStream, LaunchConfig, PushKernelArg}; +use math_cuda::device::backend; +use math_cuda::p1_stark::{Kernels, TAIL_MAX_PARENTS, load_kernels, tree_nodes}; + +const P: u64 = 0xFFFF_FFFF_0000_0001; +const BLOCK_DIM: u32 = 128; + +/// The words a lean addition chain must survive: 0, 1, p − 1, p, p + 1, the +/// neighbours of 2^64 − 2^32, all ones, 2^63, 2^32 and its neighbour. +const EDGES: [u64; 13] = [ + 0, + 1, + P - 1, + P, + P + 1, + 0xFFFF_FFFE_FFFF_FFFF, + 0xFFFF_FFFF_0000_0002, + u64::MAX - 1, + u64::MAX, + 1 << 63, + 1 << 32, + 0xFFFF_FFFF, + 0x8000_0000_8000_0000, +]; + +/// Raw words (SplitMix64, any value in [0, 2^64)) with an edge word at every +/// fifth slot, and one stretch of all ones. +fn words(seed: u64, n: usize) -> Vec { + let mut z = seed; + let mut v: Vec = (0..n) + .map(|i| { + z = z.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut x = z; + x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + if i % 5 == 0 { + EDGES[(i / 5) % EDGES.len()] + } else { + x ^ (x >> 31) + } + }) + .collect(); + for w in v.iter_mut().take(256) { + *w = u64::MAX; + } + v +} + +fn cfg(threads: u64) -> LaunchConfig { + LaunchConfig { + grid_dim: ((threads as u32).div_ceil(BLOCK_DIM).max(1), 1, 1), + block_dim: (BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + } +} + +fn read(stream: &Arc, s: &CudaSlice) -> Vec { + stream.synchronize().unwrap(); + stream.clone_dtoh(s).unwrap() +} + +/// The leaf kernels of one set over `codeword` (`num_leaves` leaves of +/// `block` values; ext3 reads three limbs a value), the bytes back. +fn coset_leaves( + stream: &Arc, + k: &Kernels, + codeword: &CudaSlice, + num_leaves: u64, + block: u64, +) -> [Vec; 3] { + let leaf_bits = num_leaves.trailing_zeros(); + let mut out = stream.alloc_zeros::(num_leaves as usize * 32).unwrap(); + let mut got = Vec::new(); + // SAFETY: each kernel reads at most `3 · num_leaves · block` values of the + // codeword (allocated so) and writes `num_leaves` digests. + unsafe { + stream + .launch_builder(&k.leaves_base_coset_rev) + .arg(codeword) + .arg(&num_leaves) + .arg(&block) + .arg(&leaf_bits) + .arg(&mut out) + .launch(cfg(num_leaves)) + .unwrap(); + } + got.push(read(stream, &out)); + for f in [&k.leaves_base_coset, &k.leaves_ext3_coset] { + stream.memset_zeros(&mut out).unwrap(); + unsafe { + stream + .launch_builder(f) + .arg(codeword) + .arg(&num_leaves) + .arg(&block) + .arg(&mut out) + .launch(cfg(num_leaves)) + .unwrap(); + } + got.push(read(stream, &out)); + } + [got.remove(0), got.remove(0), got.remove(0)] +} + +/// One set's column-row and row-pair leaves over `num_cols` columns of +/// `2^log_rows` rows. +fn column_leaves( + stream: &Arc, + k: &Kernels, + cols: &CudaSlice, + num_cols: u64, + log_rows: u64, +) -> [Vec; 2] { + let rows = 1u64 << log_rows; + let mut out = stream.alloc_zeros::(rows as usize * 32).unwrap(); + // SAFETY: both read `num_cols · rows` values at column stride `rows` and + // write at most `rows` digests. + unsafe { + stream + .launch_builder(&k.leaves_cols_row) + .arg(cols) + .arg(&rows) + .arg(&num_cols) + .arg(&rows) + .arg(&log_rows) + .arg(&mut out) + .launch(cfg(rows)) + .unwrap(); + } + let row = read(stream, &out); + stream.memset_zeros(&mut out).unwrap(); + unsafe { + stream + .launch_builder(&k.leaves_cols_pair) + .arg(cols) + .arg(&rows) + .arg(&num_cols) + .arg(&rows) + .arg(&log_rows) + .arg(&mut out) + .launch(cfg(rows / 2)) + .unwrap(); + } + [row, read(stream, &out)] +} + +/// A whole 4-ary tree over `leaves` digests with one set's level and tail +/// kernels (`p1_stark::build_inner_tree_levels`'s walk), the node buffer back. +fn tree(stream: &Arc, k: &Kernels, leaf_bytes: &[u8]) -> Vec { + let leaves = leaf_bytes.len() / 32; + let total = tree_nodes(leaves); + let mut host = vec![0u8; total * 32]; + host[(total - leaves) * 32..].copy_from_slice(leaf_bytes); + let mut nodes = stream.clone_htod(&host).unwrap(); + let mut child_off = (total - leaves) as u64; + let mut n_children = leaves as u64; + while n_children > 1 { + let n_parents = n_children.div_ceil(4); + // SAFETY: the node buffer holds the whole tree (`tree_nodes`). + unsafe { + if n_parents <= TAIL_MAX_PARENTS { + let tail = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (TAIL_MAX_PARENTS as u32, 1, 1), + shared_mem_bytes: 0, + }; + stream + .launch_builder(&k.merkle_tail4) + .arg(&mut nodes) + .arg(&child_off) + .arg(&n_children) + .launch(tail) + .unwrap(); + break; + } + let parent_off = child_off - n_parents; + stream + .launch_builder(&k.merkle_level4) + .arg(&mut nodes) + .arg(&child_off) + .arg(&n_children) + .arg(&parent_off) + .arg(&n_parents) + .launch(cfg(n_parents)) + .unwrap(); + child_off = parent_off; + n_children = n_parents; + } + } + read(stream, &nodes) +} + +/// ★ Every twin writes today's bytes: coset leaves (coset and natural order, +/// ext3), column-row and row-pair leaves (12, 13 and 25 columns: whole +/// blocks, a partial tail, two tails), and whole trees over 2^14 leaves and +/// over a ragged 5,000. One planted change moves exactly one leaf under the +/// twin too, so the comparison can fail. +#[test] +fn the_lean_twins_write_todays_bytes() { + let be = backend().expect("a GPU"); + let stream = be.ctx.new_stream().expect("stream"); + let today = load_kernels(false).expect("today's kernels"); + let lean = load_kernels(true).expect("the lean twins"); + let (num_leaves, block) = (1u64 << 14, 64u64); + let mut host = words(0x1ea4, (3 * num_leaves * block) as usize); + let codeword = stream.clone_htod(&host).unwrap(); + let a = coset_leaves(&stream, &today, &codeword, num_leaves, block); + let b = coset_leaves(&stream, &lean, &codeword, num_leaves, block); + for (name, (x, y)) in ["coset order", "natural order", "ext3"] + .iter() + .zip(a.iter().zip(&b)) + { + assert_eq!(x, y, "the lean coset leaves ({name}) differ from today's"); + } + let log_rows = 12; + for num_cols in [12u64, 13, 25] { + let cols = stream + .clone_htod(&words(num_cols, (num_cols as usize) << log_rows)) + .unwrap(); + let x = column_leaves(&stream, &today, &cols, num_cols, log_rows); + let y = column_leaves(&stream, &lean, &cols, num_cols, log_rows); + assert_eq!( + x[0], y[0], + "the lean column-row leaves differ ({num_cols} columns)" + ); + assert_eq!( + x[1], y[1], + "the lean row-pair leaves differ ({num_cols} columns)" + ); + } + assert_eq!( + tree(&stream, &today, &a[0]), + tree(&stream, &lean, &a[0]), + "the lean tree over 2^14 leaves differs" + ); + let ragged = &a[1][..5000 * 32]; + assert_eq!( + tree(&stream, &today, ragged), + tree(&stream, &lean, ragged), + "the lean tree over 5,000 leaves differs" + ); + + // The comparison can fail: one value of leaf 7's coset moved. + host[7 + 9 * num_leaves as usize] ^= 1 << 40; + let moved = stream.clone_htod(&host).unwrap(); + let c = coset_leaves(&stream, &lean, &moved, num_leaves, block); + let differ = (0..num_leaves as usize) + .filter(|&j| c[1][32 * j..32 * j + 32] != a[1][32 * j..32 * j + 32]) + .count(); + assert_eq!(differ, 1, "one moved value moves one natural-order leaf"); +} + +/// The measurement kernels' ns a permutation (leaf, 4-ary node, grind) at +/// every variant over a 2^29 codeword, `l2` beside `v2`: the mechanism row of +/// the lean kernels' measurement (`BENCH` lines). +#[test] +#[ignore = "box tier: a measurement"] +fn the_variants_ns_a_permutation() { + math_cuda::p1w16::microbench(29, 7, 1 << 28).expect("microbench"); +} diff --git a/crypto/math-cuda/tests/p1w16_device.rs b/crypto/math-cuda/tests/p1w16_device.rs new file mode 100644 index 000000000..805bc2ac6 --- /dev/null +++ b/crypto/math-cuda/tests/p1w16_device.rs @@ -0,0 +1,280 @@ +//! D-HASH stage 1: the Poseidon1 width-16 measurement kernels against the Rust +//! host reference (`crypto::hash::poseidon1_w16`), at both multiply variants, +//! and the microbenchmark against the RPX kernels. +//! +//! Needs a GPU (the box); the host half of the arithmetic is pinned per PR by +//! `make test-p1w16-host-kat`. +//! +//! cargo test -p math-cuda --release --test p1w16_device +//! P1W16_LOG_LEN=29 cargo test -p math-cuda --release --test p1w16_device \ +//! -- --ignored --nocapture p1w16_microbench + +use crypto::hash::poseidon1_w16 as host; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math_cuda::p1w16; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn canon(v: &[Fp]) -> Vec { + v.iter().map(|f| f.canonical()).collect() +} + +fn random_felts(rng: &mut ChaCha8Rng, n: usize) -> Vec { + // Raw u64s, some at or above p: the device must accept raw storage. + (0..n) + .map(|i| { + if i % 17 == 0 { + P + (i as u64 % 7) + } else { + rng.r#gen::() % P + } + }) + .collect() +} + +fn fp(v: &[u64]) -> Vec { + v.iter().map(|x| Fp::from(*x)).collect() +} + +/// The host reference for variant `v`: the circulant instance, or the Cauchy +/// alternative for `c1`. +fn host_permute(v: usize) -> fn([Fp; 16]) -> [Fp; 16] { + if p1w16::is_cauchy(v) { + host::permute_cauchy + } else { + host::permute + } +} + +fn host_leaf(v: usize) -> fn(&[Fp]) -> host::Digest { + if p1w16::is_cauchy(v) { + host::sponge_leaf_cauchy + } else { + host::sponge_leaf + } +} + +fn host_node(v: usize) -> fn(&[host::Digest; 4]) -> host::Digest { + if p1w16::is_cauchy(v) { + host::compress4_cauchy + } else { + host::compress4 + } +} + +#[test] +fn the_device_permutation_matches_the_host_reference() { + let mut rng = ChaCha8Rng::seed_from_u64(1); + let n = 4099; + let states = random_felts(&mut rng, n * 16); + for v in 0..p1w16::VARIANTS.len() { + let got = p1w16::permute_many(v, &states).expect("device"); + for (i, chunk) in states.chunks(16).enumerate() { + let s: [Fp; 16] = core::array::from_fn(|k| Fp::from(chunk[k])); + assert_eq!( + got[i * 16..(i + 1) * 16], + canon(&host_permute(v)(s))[..], + "variant {v} state {i}" + ); + } + } +} + +#[test] +fn the_device_coset_leaves_match_the_host_sponge() { + let mut rng = ChaCha8Rng::seed_from_u64(2); + for (block, ext3) in [ + (64u64, false), + (16, false), + (5, false), + (16, true), + (4, true), + ] { + let per = if ext3 { 3 } else { 1 }; + let num_leaves = 1027u64; + let cw = random_felts(&mut rng, (num_leaves * block * per) as usize); + for v in 0..p1w16::VARIANTS.len() { + let got = p1w16::leaves_coset(v, &cw, block, ext3).expect("device"); + for j in 0..num_leaves { + let mut felts = Vec::new(); + for t in 0..block { + let at = (j + t * num_leaves) as usize; + if ext3 { + felts.extend_from_slice(&cw[3 * at..3 * at + 3]); + } else { + felts.push(cw[at]); + } + } + let want = canon(&host_leaf(v)(&fp(&felts))); + assert_eq!( + got[4 * j as usize..4 * j as usize + 4], + want[..], + "variant {v} block {block} ext3 {ext3} leaf {j}" + ); + } + } + } +} + +#[test] +fn the_device_4ary_level_matches_the_host_node() { + let mut rng = ChaCha8Rng::seed_from_u64(3); + let n = 2049; + let children = random_felts(&mut rng, n * 16); + for v in 0..p1w16::VARIANTS.len() { + let got = p1w16::merkle_level4(v, &children).expect("device"); + for i in 0..n { + let c = &children[i * 16..(i + 1) * 16]; + let kids: [host::Digest; 4] = + core::array::from_fn(|q| core::array::from_fn(|l| Fp::from(c[4 * q + l]))); + assert_eq!( + got[4 * i..4 * i + 4], + canon(&host_node(v)(&kids))[..], + "variant {v} node {i}" + ); + } + } +} + +/// The grind returns the smallest nonce the host scan finds, at a factor where +/// the scan is cheap; a limit of zero finds nothing. +#[test] +fn the_device_grind_returns_the_smallest_valid_nonce() { + let inner = [11u64, 22, 33, 44]; + let limit = 1u64 << (64 - 12); + for v in 0..p1w16::VARIANTS.len() { + let leaf = host_leaf(v); + let head = |nonce: u64| { + let felts = fp(&[inner[0], inner[1], inner[2], inner[3], nonce]); + leaf(&felts)[0].canonical() + }; + let want = (0..1u64 << 20) + .find(|&n| head(n) < limit) + .expect("a hit below 2^20"); + for grid in [1u32, 1024] { + let got = p1w16::grind(v, &inner, limit, 1 << 20, grid).expect("device"); + assert_eq!(got, Some(want), "variant {v} grid {grid}"); + } + assert_eq!( + p1w16::grind(v, &inner, 0, 1 << 16, 64).expect("device"), + None + ); + } +} + +// --------------------------------------------------------------------------- +// ZisK's instance (stage P1): the kernels against `crypto::hash::poseidon1_stark` +// and `crypto::hash::poseidon1_w8`, whose vectors come from ZisK's own code. +// --------------------------------------------------------------------------- + +use crypto::hash::{poseidon1_stark as zisk, poseidon1_w8 as w8}; + +#[test] +fn the_device_zisk_coset_leaves_match_the_host() { + let mut rng = ChaCha8Rng::seed_from_u64(4); + for (block, ext3) in [ + (64u64, false), + (13, false), + (12, false), + (16, true), + (5, true), + ] { + let per = if ext3 { 3 } else { 1 }; + let num_leaves = 1027u64; + let cw = random_felts(&mut rng, (num_leaves * block * per) as usize); + let got = p1w16::zisk_leaves_coset(&cw, block, ext3).expect("device"); + for j in 0..num_leaves { + let mut felts = Vec::new(); + for t in 0..block { + let at = (j + t * num_leaves) as usize; + if ext3 { + felts.extend_from_slice(&cw[3 * at..3 * at + 3]); + } else { + felts.push(cw[at]); + } + } + assert_eq!( + got[4 * j as usize..4 * j as usize + 4], + canon(&zisk::zisk_linear_hash(&fp(&felts)))[..], + "block {block} ext3 {ext3} leaf {j}" + ); + } + } +} + +#[test] +fn the_device_zisk_row_leaves_match_the_host() { + let mut rng = ChaCha8Rng::seed_from_u64(5); + let num_rows = 1u64 << 10; + for (num_cols, pair) in [(7u64, false), (24, false), (16, true), (25, true)] { + let col_stride = num_rows + 3; + let m = random_felts(&mut rng, (num_cols * col_stride) as usize); + let got = + p1w16::zisk_leaves_rows(&m, col_stride, num_cols, num_rows, pair).expect("device"); + let log = num_rows.trailing_zeros(); + let row = |r: u64| -> Vec { + let br = r.reverse_bits() >> (64 - log); + (0..num_cols) + .map(|c| m[(c * col_stride + br) as usize]) + .collect() + }; + let num_leaves = if pair { num_rows / 2 } else { num_rows }; + for j in 0..num_leaves { + let felts = if pair { + [row(2 * j), row(2 * j + 1)].concat() + } else { + row(j) + }; + assert_eq!( + got[4 * j as usize..4 * j as usize + 4], + canon(&zisk::zisk_linear_hash(&fp(&felts)))[..], + "cols {num_cols} pair {pair} leaf {j}" + ); + } + } +} + +#[test] +fn the_device_width_8_permutation_matches_the_host() { + let mut rng = ChaCha8Rng::seed_from_u64(6); + let n = 4099; + let states = random_felts(&mut rng, n * 8); + for v in 0..p1w16::W8_VARIANTS.len() { + let got = p1w16::w8_permute_many(v, &states).expect("device"); + for (i, chunk) in states.chunks(8).enumerate() { + let s: [Fp; 8] = core::array::from_fn(|k| Fp::from(chunk[k])); + assert_eq!( + got[i * 8..(i + 1) * 8], + canon(&w8::permute(s))[..], + "W8 variant {v} state {i}" + ); + } + } +} + +/// ZisK's grind returns the smallest nonce the host scan finds; a limit of zero +/// finds nothing. +#[test] +fn the_device_zisk_grind_returns_the_smallest_valid_nonce() { + let challenge = [11u64, 22, 33]; + let bits = 12; + let ch: [Fp; 3] = core::array::from_fn(|i| Fp::from(challenge[i])); + let want = (0..1u64 << 20) + .find(|&n| zisk::grinding_ok(&ch, &Fp::from(n), bits)) + .expect("a hit below 2^20"); + for v in 0..p1w16::W8_VARIANTS.len() { + for grid in [1u32, 1024] { + let got = + p1w16::w8_grind(v, &challenge, 1u64 << (64 - bits), 1 << 20, grid).expect("device"); + assert_eq!(got, Some(want), "W8 variant {v} grid {grid}"); + } + assert_eq!( + p1w16::w8_grind(v, &challenge, 0, 1 << 16, 64).expect("device"), + None + ); + } +} diff --git a/crypto/math-cuda/tests/round_slot.rs b/crypto/math-cuda/tests/round_slot.rs new file mode 100644 index 000000000..9687e6fe9 --- /dev/null +++ b/crypto/math-cuda/tests/round_slot.rs @@ -0,0 +1,462 @@ +//! A GKR layer's rounds with their sums through a slot against today's second +//! launch and read-back (`crate::round_slot`, `LAMBDA_VM_ARGUE_ROUND_SLOT`). +//! +//! ```text +//! cargo test --release -p math-cuda --test round_slot -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. Every round's sums and the halves after the last round are +//! compared limb for limb: the slot's last-block reduction promises today's +//! limbs, not only today's values. The ignored test is the capability probe +//! that decides whether the next round is ever queued behind a card-side wait: +//! the stream-memory attributes, and three round trips timed. + +use std::sync::Arc; + +use cudarc::driver::{CudaSlice, CudaStream, DevicePtr, LaunchConfig, PushKernelArg}; +use math_cuda::device::backend; +use math_cuda::gkr::{DeviceFractionTree, GruenLayer}; +use math_cuda::round_slot::{self, MappedSlot, RoundSlot}; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +struct Restore { + _serial: std::sync::MutexGuard<'static, ()>, +} + +impl Drop for Restore { + fn drop(&mut self) { + round_slot::force_check_fault(false); + } +} + +fn serial() -> Restore { + Restore { + _serial: SERIAL.lock().unwrap_or_else(|e| e.into_inner()), + } +} + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn limb(&mut self) -> u64 { + let v = self.next() % P; + if self.next() & 7 == 0 && v < 0xFFFF_FFFF { + v + P + } else { + v + } + } + + fn ext(&mut self) -> [u64; 3] { + [self.limb(), self.limb(), self.limb()] + } +} + +fn upload(stream: &Arc, values: &[u64]) -> CudaSlice { + math_cuda::device::htod_or_trim(stream, values).expect("upload") +} + +/// A tree over random fractions, the padding `0/1` past `real` (a kept tree +/// when `kept`, today's otherwise). +fn tree(interactions: usize, rows: usize, kept: bool, seed: u64) -> DeviceFractionTree { + let be = backend().expect("a device"); + let full = interactions.next_power_of_two() * rows; + let real = interactions * rows; + let mut rng = Rng(seed | 1); + let mut p = Vec::with_capacity(full * 3); + let mut q = Vec::with_capacity(full * 3); + for i in 0..full { + if i < real { + p.extend_from_slice(&rng.ext()); + q.extend_from_slice(&rng.ext()); + } else { + p.extend_from_slice(&[0, 0, 0]); + q.extend_from_slice(&[1, 0, 0]); + } + } + let stream = be.next_stream(); + if kept { + DeviceFractionTree::from_truncated_input( + stream.clone(), + upload(&stream, &p[..real * 3]), + upload(&stream, &q[..real * 3]), + real, + full.trailing_zeros() as usize, + ) + } else { + DeviceFractionTree::from_device(stream.clone(), upload(&stream, &p), upload(&stream, &q)) + } + .expect("the tree") +} + +/// Every round of a layer and its finish, raw. +fn run( + layer: &mut GruenLayer, + lambda: &[u64; 3], + challenges: &[[u64; 3]], +) -> (Vec>, Vec) { + let mut sums = Vec::new(); + for j in 0..layer.rounds() { + let fold = (j > 0).then(|| &challenges[j - 1]); + sums.push(layer.round(fold, lambda, j % 3 == 1).expect("a round")); + } + let last = (layer.rounds() > 0).then(|| &challenges[layer.rounds() - 1]); + (sums, layer.finish(last).expect("the finish")) +} + +/// One layer's rounds today and through a slot under `mode`, over twin trees. +fn compare(interactions: usize, rows: usize, kept: bool, low: usize, mode: RoundSlot, seed: u64) { + let what = format!("{interactions} x {rows} rows, kept {kept}, low {low}, {mode:?}"); + let today_tree = tree(interactions, rows, kept, seed); + let slot_tree = tree(interactions, rows, kept, seed); + let input = today_tree.num_layers() - 1; + let m = today_tree.layer_num_vars(input) - 1; + let mut rng = Rng(seed ^ 0x51); + let point: Vec = (0..m).flat_map(|_| rng.ext()).collect(); + let lambda = rng.ext(); + let challenges: Vec<[u64; 3]> = (0..m).map(|_| rng.ext()).collect(); + let mut a = today_tree.layer_gruen(input, &point, low).unwrap(); + let mut b = slot_tree.layer_gruen(input, &point, low).unwrap(); + assert!(b.use_slot(mode), "{what}: a slot was had"); + let before = round_slot::counts(); + let (sums_a, halves_a) = run(&mut a, &lambda, &challenges); + let mid = round_slot::counts(); + let (sums_b, halves_b) = run(&mut b, &lambda, &challenges); + let after = round_slot::counts(); + for (j, (x, y)) in sums_a.iter().zip(&sums_b).enumerate() { + assert_eq!(x, y, "{what}: round {j}'s sums, raw"); + } + assert_eq!(halves_a, halves_b, "{what}: the halves, raw"); + let rounds = sums_b.len() as u64; + assert_eq!( + mid.second_launches - before.second_launches, + rounds, + "{what}: today's launches" + ); + assert_eq!( + after.slot_rounds - mid.slot_rounds, + rounds, + "{what}: every round through the slot" + ); + let second = after.second_launches - mid.second_launches; + match mode { + RoundSlot::On => assert_eq!(second, 0, "{what}: no second launch"), + _ => { + assert_eq!(second, rounds, "{what}: check runs today's beside it"); + assert_eq!( + after.checked - mid.checked, + rounds, + "{what}: every round compared" + ); + assert_eq!(after.mismatches, mid.mismatches, "{what}: none differed"); + } + } +} + +/// ★ A layer's rounds through a slot hand back today's sums every round and +/// today's halves at the end, limb for limb, with no second launch: whole and +/// kept layers, every host tail, rounds summing `H(0)` and not. +#[test] +fn the_slot_hands_back_todays_sums_limb_for_limb() { + let _restore = serial(); + if backend().is_err() { + eprintln!("no device; skipping"); + return; + } + let mut n = 0; + for (interactions, rows) in [ + (5usize, 8usize), + (20, 1 << 10), + (1031, 1 << 3), + (8, 1 << 9), + (3, 1 << 12), + ] { + for (kept, low) in [(false, 6usize), (true, 0), (true, 6), (false, 0)] { + compare(interactions, rows, kept, low, RoundSlot::On, 0x5107 + n); + n += 1; + } + } + eprintln!("round slot: {n} layers, every round and finish raw-identical, no second launch"); +} + +/// `Check` runs today's second launch beside the slot every round, compares +/// the two and finds none differing. +#[test] +fn the_check_compares_every_round() { + let _restore = serial(); + if backend().is_err() { + eprintln!("no device; skipping"); + return; + } + compare(20, 1 << 12, true, 6, RoundSlot::Check, 0xC4EC); + compare(1031, 1 << 4, false, 0, RoundSlot::Check, 0xC4ED); +} + +/// ⛔ The check can fail: with the fault armed every compared round counts as +/// differing, and the values handed back are still today's. +#[test] +fn a_differing_round_is_counted_and_todays_is_used() { + let _restore = serial(); + if backend().is_err() { + eprintln!("no device; skipping"); + return; + } + let today_tree = tree(20, 1 << 10, false, 7); + let slot_tree = tree(20, 1 << 10, false, 7); + let input = today_tree.num_layers() - 1; + let m = today_tree.layer_num_vars(input) - 1; + let mut rng = Rng(8); + let point: Vec = (0..m).flat_map(|_| rng.ext()).collect(); + let lambda = rng.ext(); + let challenges: Vec<[u64; 3]> = (0..m).map(|_| rng.ext()).collect(); + let mut a = today_tree.layer_gruen(input, &point, 6).unwrap(); + let mut b = slot_tree.layer_gruen(input, &point, 6).unwrap(); + assert!(b.use_slot(RoundSlot::Check)); + let want = run(&mut a, &lambda, &challenges); + round_slot::force_check_fault(true); + let before = round_slot::counts(); + let got = run(&mut b, &lambda, &challenges); + let after = round_slot::counts(); + round_slot::force_check_fault(false); + assert_eq!(want, got, "today's values are used"); + assert_eq!( + after.mismatches - before.mismatches, + got.0.len() as u64, + "every compared round counted" + ); +} + +/// The pool hands out at most its pages' slots, declines past them (the +/// session keeps today's rounds, counted), and takes slots back on drop. +#[test] +fn the_pool_declines_past_its_pages_and_takes_slots_back() { + let _restore = serial(); + let Ok(be) = backend() else { + eprintln!("no device; skipping"); + return; + }; + let stream = be.next_stream(); + let mut held = Vec::new(); + while let Some(slot) = MappedSlot::take(&stream) { + held.push(slot); + assert!(held.len() <= 16 * 64, "no more slots than 16 pages of 64"); + } + assert_eq!(held.len(), 16 * 64, "every page's slots were handed out"); + let before = round_slot::counts(); + let declined = round_slot::SlotRounds::open(&stream, RoundSlot::On); + assert!(declined.is_none(), "a spent pool declines"); + assert_eq!( + round_slot::counts().declines - before.declines, + 1, + "the decline counted" + ); + drop(held); + assert!( + MappedSlot::take(&stream).is_some(), + "a slot given back is taken again" + ); +} + +// ── the capability probe ──────────────────────────────────────────────────── + +/// The word the queued round waits on: the slot's 104th u64, past the +/// sequence word, which the kernel does not write. +const FLAG: usize = round_slot::SEQ + 7; + +fn median(mut v: Vec) -> f64 { + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + v[v.len() / 2] +} + +/// ⏱ The probe (run with `--ignored --nocapture`): whether the card takes stream +/// memory operations, and one GKR round's round trip timed three ways over a +/// small layer (a quarter of 256, so the trip is all fixed cost): +/// today's (the round, the second launch, a pageable read-back and a stream +/// wait); a slot (the slot round and a host spin); and a slot with the round +/// already queued behind `cuStreamWaitValue32` on a word the host writes — +/// timed from that write to the spin's end. `PROBE ATTR`, `PROBE TRIP` and +/// `PROBE DECISION` lines; the queued round is worth building only if the +/// attribute is set and the queued trip is at most 0.6 of the spin's. +#[test] +#[ignore] +fn the_round_trip_three_ways() { + use cudarc::driver::sys; + let _restore = serial(); + let be = backend().expect("a device"); + const WARM: usize = 200; + const TIMED: usize = 10_000; + let attr = |a: sys::CUdevice_attribute| -> i32 { + let mut v = 0i32; + unsafe { sys::cuDeviceGetAttribute(&mut v, a, be.ctx.cu_device()) }; + v + }; + let mem_ops = attr(sys::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_CAN_USE_STREAM_MEM_OPS_V1); + let wait_nor = attr(sys::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_CAN_USE_STREAM_WAIT_VALUE_NOR); + println!("PROBE ATTR stream_mem_ops={mem_ops} wait_value_nor={wait_nor}"); + + let stream = be.next_stream(); + let half = 512u64; + let quarter = 256u64; + let mut rng = Rng(0x9E0); + let p: Vec = (0..2 * half * 3).map(|_| rng.limb()).collect(); + let q: Vec = (0..2 * half * 3).map(|_| rng.limb()).collect(); + let p_dev = upload(&stream, &p); + let q_dev = upload(&stream, &q); + let e_hi = upload(&stream, &vec![1u64; 64 * 3]); + let e_lo = upload(&stream, &[1u64; 4 * 3]); + let mut partials = upload(&stream, &vec![0u64; 3 * 2048 * 3]); + let mut sums = upload(&stream, &[0u64; 9]); + let mut ticket = math_cuda::device::alloc_zeros_or_trim::(&stream, 1).unwrap(); + let slot = MappedSlot::take(&stream).expect("a slot"); + let (p_at, q_at) = { + let (a, _g) = p_dev.device_ptr(&stream); + let (b, _h) = q_dev.device_ptr(&stream); + (a, b) + }; + let (block, grid) = (256u32, 1u32); + let (zero, fold, low, want) = (0u64, 0u32, 2u32, 0u32); + let lambda = [3u64, 0, 0]; + let launch_round = + |slotted: bool, seq: u64, partials: &mut CudaSlice, ticket: &mut CudaSlice| { + let kernel = if slotted { + &be.gkr_round_gruen_slot + } else { + &be.gkr_round_gruen + }; + let dev = slot.dev(); + unsafe { + let mut l = stream.launch_builder(kernel); + l.arg(&p_at) + .arg(&q_at) + .arg(&half) + .arg(&quarter) + .arg(&fold) + .arg(&zero) + .arg(&zero) + .arg(&zero) + .arg(&lambda[0]) + .arg(&lambda[1]) + .arg(&lambda[2]) + .arg(&e_hi) + .arg(&e_lo) + .arg(&low) + .arg(&want) + .arg(partials); + if slotted { + l.arg(ticket).arg(&dev).arg(&seq); + } + l.launch(LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (block, 1, 1), + shared_mem_bytes: if slotted { + round_slot::shared_bytes(block) + } else { + block * 24 + }, + }) + .unwrap(); + } + }; + // Today's trip. + let mut today = Vec::new(); + for i in 0..WARM + TIMED { + let from = std::time::Instant::now(); + launch_round(false, 0, &mut partials, &mut ticket); + unsafe { + stream + .launch_builder(&be.sum_partials_ext3) + .arg(&partials) + .arg(&(grid as u64)) + .arg(&mut sums) + .launch(LaunchConfig { + grid_dim: (2, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 256 * 24, + }) + .unwrap(); + } + let _ = stream.clone_dtoh(&sums.slice(0..6)).unwrap(); + stream.synchronize().unwrap(); + if i >= WARM { + today.push(from.elapsed().as_secs_f64() * 1e6); + } + } + // The slot's trip; the sequence numbers here are the probe's own. + let mut spin = Vec::new(); + for i in 0..WARM + TIMED { + let seq = 1_000_000 + i as u64; + let from = std::time::Instant::now(); + launch_round(true, seq, &mut partials, &mut ticket); + let _ = slot.wait(seq, 2).unwrap(); + if i >= WARM { + spin.push(from.elapsed().as_secs_f64() * 1e6); + } + } + // The queued trip: the round waits on a word the host writes, then runs. + let mut queued = Vec::new(); + if mem_ops != 0 { + let flag = slot.dev() + FLAG as u64 * 8; + let flag_host = |v: u32| unsafe { + // The host's view of the same word (the slot's 104th u64). + let words = slot_host(&slot); + std::ptr::write_volatile(words.add(FLAG) as *mut u32, v); + }; + flag_host(0); + for i in 0..WARM + TIMED { + let seq = 2_000_000 + i as u64; + let value = (i + 1) as u32; + unsafe { + sys::cuStreamWaitValue32_v2( + stream.cu_stream(), + flag, + value, + sys::CUstreamWaitValue_flags::CU_STREAM_WAIT_VALUE_GEQ as u32, + ) + .result() + .unwrap(); + } + launch_round(true, seq, &mut partials, &mut ticket); + std::thread::sleep(std::time::Duration::from_micros(50)); + let from = std::time::Instant::now(); + flag_host(value); + let _ = slot.wait(seq, 2).unwrap(); + if i >= WARM { + queued.push(from.elapsed().as_secs_f64() * 1e6); + } + } + } + let (t, s) = (median(today), median(spin)); + let qd = (!queued.is_empty()).then(|| median(queued)); + println!( + "PROBE TRIP reps={TIMED} today_us={t:.2} slot_us={s:.2} queued_us={} slot_over_today={:.3} queued_over_slot={}", + qd.map_or("-".to_string(), |v| format!("{v:.2}")), + s / t, + qd.map_or("-".to_string(), |v| format!("{:.3}", v / s)) + ); + let build = wait_nor != 0 && mem_ops != 0 && qd.is_some_and(|v| v <= 0.6 * s); + println!( + "PROBE DECISION build the queued round: {} (attribute {} and queued <= 0.6 x slot)", + if build { "yes" } else { "no" }, + if wait_nor != 0 && mem_ops != 0 { + "set" + } else { + "unset" + } + ); +} + +/// The slot's host words, for the probe's flag (the slot hands out only its +/// card address and its own reads). +fn slot_host(slot: &MappedSlot) -> *mut u64 { + slot.host_words_for_tests() +} diff --git a/crypto/math-cuda/tests/staged_transfers.rs b/crypto/math-cuda/tests/staged_transfers.rs new file mode 100644 index 000000000..2dd8a9c67 --- /dev/null +++ b/crypto/math-cuda/tests/staged_transfers.rs @@ -0,0 +1,382 @@ +//! The per-transfer, double-buffered pinned staging (`htod_staged`, +//! `dtoh_staged_into`) must move every byte exactly, across every chunk +//! boundary, from any number of threads at once — and the fused commits, the +//! row-major ones and the column-major engine's, must produce the SAME roots, +//! the same host LDE and the same column-major handle through it as through the +//! shared slab (`LAMBDA_VM_STAGING_SHARED_SLAB=1`). +//! +//! Device tests: they need the card, like every other file in this directory. + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math_cuda::DeviceHash; +use math_cuda::device::{ + STAGED_CHUNK_BYTES, STAGING_SHARED_SLAB_ENV, backend, dtoh_staged_into, htod_staged, + set_staging_pairs_override, staging_pairs_enabled, staging_totals, +}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; + +/// u64s per staging buffer. +const CHUNK: usize = STAGED_CHUNK_BYTES / 8; + +fn random(n: usize, seed: u64) -> Vec { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + (0..n).map(|_| rng.r#gen::()).collect() +} + +/// The first differing offset, so a failure names the chunk it is in. +fn assert_same(got: &[u64], want: &[u64], what: &str) { + assert_eq!(got.len(), want.len(), "{what}: length"); + if let Some(i) = got.iter().zip(want).position(|(a, b)| a != b) { + panic!( + "{what}: first difference at u64 {i} (chunk {}, offset {} in it): {:#x} != {:#x}", + i / CHUNK, + i % CHUNK, + got[i], + want[i] + ); + } +} + +/// Upload through `htod_staged`, read back plainly. +fn upload_roundtrip(src: &[u64]) -> Vec { + let be = backend().expect("cuda backend"); + let stream = be.next_stream(); + let mut dst = stream.alloc_zeros::(src.len()).expect("device alloc"); + htod_staged(&stream, src, &mut dst.slice_mut(0..src.len())).expect("htod_staged"); + let back = stream.clone_dtoh(&dst).expect("dtoh"); + stream.synchronize().expect("sync"); + back +} + +/// Upload plainly, read back through `dtoh_staged_into`; returns the values +/// and how many times the closure ran. +fn download_roundtrip(src: &[u64]) -> (Vec, usize) { + use cudarc::driver::DevicePtr; + let be = backend().expect("cuda backend"); + let stream = be.next_stream(); + let dev = stream.clone_htod(src).expect("htod"); + let ptr = { + let (p, _record) = dev.device_ptr(&stream); + p + }; + let mut out = Vec::new(); + let mut calls = 0usize; + // SAFETY: `ptr` is `dev`, written by the upload queued on `stream` above, + // written by nothing after, and alive until this function returns. + let tag = unsafe { + dtoh_staged_into(&stream, ptr, src.len(), &mut out, || { + calls += 1; + Ok(0xC0FFEEu32) + }) + } + .expect("dtoh_staged_into"); + assert_eq!(tag, 0xC0FFEE, "the closure's value is returned"); + drop(dev); + (out, calls) +} + +#[test] +fn staged_upload_is_exact_across_chunk_boundaries() { + for (n, seed) in [ + (1usize, 1u64), + (CHUNK - 1, 2), + (CHUNK, 3), + (CHUNK + 1, 4), + // Three full buffers and a partial tail: both buffers reused. + (3 * CHUNK + 12_345, 5), + ] { + let src = random(n, seed); + assert_same(&upload_roundtrip(&src), &src, &format!("upload n={n}")); + } +} + +#[test] +fn staged_download_is_exact_and_runs_the_closure_once() { + for (n, seed) in [ + (1usize, 11u64), + (CHUNK - 1, 12), + (CHUNK, 13), + (CHUNK + 1, 14), + (2 * CHUNK, 15), + (3 * CHUNK + 777, 16), + ] { + let src = random(n, seed); + let (got, calls) = download_roundtrip(&src); + assert_eq!(calls, 1, "n={n}: the closure must run exactly once"); + assert_same(&got, &src, &format!("download n={n}")); + } + // Zero values: the closure still runs once and nothing lands. + let (got, calls) = download_roundtrip(&[]); + assert!(got.is_empty()); + assert_eq!(calls, 1, "n=0: the closure must still run once"); +} + +/// ⛔ THE HAZARD THE EVENTS EXIST FOR. `htod_staged` returns without waiting +/// for its last DMA, so the SAME thread's next transfer can borrow the same +/// pair while that DMA still reads a buffer. A download into that buffer must +/// wait for it; if it did not, the upload would arrive with the download's +/// bytes in it. Interleaved back to back on different streams, many times. +#[test] +fn a_download_never_overwrites_a_buffer_an_upload_is_still_reading() { + let be = backend().expect("cuda backend"); + for round in 0..8u64 { + let up = random(2 * CHUNK + 99, 100 + round); + let down = random(2 * CHUNK + 55, 200 + round); + let s_up = be.next_stream(); + let mut up_dev = s_up.alloc_zeros::(up.len()).expect("alloc"); + htod_staged(&s_up, &up, &mut up_dev.slice_mut(0..up.len())).expect("htod_staged"); + // No sync: the upload's DMAs may still be reading the pair. + let (got, _) = download_roundtrip(&down); + assert_same(&got, &down, &format!("round {round}: download")); + let back = s_up.clone_dtoh(&up_dev).expect("dtoh"); + s_up.synchronize().expect("sync"); + assert_same(&back, &up, &format!("round {round}: upload")); + } +} + +/// More threads than pairs: every transfer exact, and the pool never grows +/// past its cap (the extra threads wait for a pair instead). +#[test] +fn concurrent_transfers_beyond_the_pool_cap_are_exact() { + const THREADS: usize = 12; + std::thread::scope(|s| { + for t in 0..THREADS { + s.spawn(move || { + for round in 0..3u64 { + let n = CHUNK + 1000 * (t + 1) + round as usize; + let seed = 1_000 + (t as u64) * 10 + round; + let src = random(n, seed); + assert_same( + &upload_roundtrip(&src), + &src, + &format!("thread {t} round {round}: upload"), + ); + let (got, _) = download_roundtrip(&src); + assert_same(&got, &src, &format!("thread {t} round {round}: download")); + } + }); + } + }); + let totals = staging_totals(); + assert!( + totals.pairs <= math_cuda::device::MAX_STAGING_PAIRS, + "the pool grew past its cap: {totals:?}" + ); +} + +// ── The fused commits, one arm against the other ───────────────────────────── + +const COSET_OFFSET: u64 = 7; + +/// `weights[i] = g^i / n` — the `LdeTwiddles::coset_weights` format. +fn coset_weights_u64(n: usize, g: u64) -> Vec { + let inv_n = Fp::from(n as u64).inv().unwrap(); + let g_fp = Fp::from_raw(g); + let mut w = Vec::with_capacity(n); + let mut cur = inv_n; + for _ in 0..n { + w.push(*cur.value()); + cur = &cur * &g_fp; + } + w +} + +/// D2H a handle's column-major buffer once its `ready` event has fired. +fn download_handle( + buf: &math_cuda::CudaSlice, + ready: Option<&math_cuda::device::PooledEvent>, +) -> Vec { + let be = backend().expect("cuda backend"); + let stream = be.next_stream(); + if let Some(ev) = ready { + stream.wait(ev.event()).expect("wait on the ready event"); + } + let v = stream.clone_dtoh(buf).expect("D2H of the handle"); + stream.synchronize().expect("sync"); + v +} + +/// Runs `f` through the shared slab, then through pairs, on this thread. +fn both_paths(f: impl Fn() -> T) -> (T, T) { + set_staging_pairs_override(Some(false)); + let shared = f(); + set_staging_pairs_override(Some(true)); + let pairs = f(); + set_staging_pairs_override(None); + (shared, pairs) +} + +/// Shapes whose host input clears the 8 MB staged-upload floor and whose LDE +/// spans one buffer, exactly two, or two plus a tail. +const SHAPES: &[(usize, usize, usize)] = &[ + // (log_n, blowup, base columns) + (18, 2, 5), // 10.5 MB in, 21 MB LDE: one chunk + (17, 4, 9), // 9.4 MB in, 37.7 MB LDE: a chunk and a tail + (16, 4, 32), // 16.8 MB in, 67 MB LDE: exactly two chunks +]; + +#[test] +fn the_base_commit_is_identical_through_either_staging() { + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + let n = 1usize << log_n; + let row_major = random(n * cols, 0xB5E0 + i as u64); + let weights = coset_weights_u64(n, COSET_OFFSET); + let before = staging_totals(); + let (off, on) = both_paths(|| { + let (handle, host) = math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep_rpl( + &row_major, + None, + DeviceHash::Rpx256, + n, + cols, + blowup, + &weights, + true, + 2, + ) + .expect("fused base commit"); + let dev = download_handle(handle.buf.as_ref(), handle.ready.as_deref()); + let root = handle.tree.as_ref().expect("keep path keeps the tree").root; + (root, host, dev) + }); + let what = format!("base log_n={log_n} blowup={blowup} cols={cols}"); + assert_eq!(off.0, on.0, "{what}: root"); + assert_same(&on.1, &off.1, &format!("{what}: host LDE")); + assert_same(&on.2, &off.2, &format!("{what}: column-major handle")); + // ⛔ The check that can fail if the override stopped routing: the staged + // path must have moved this shape's bytes. + let after = staging_totals(); + assert!( + after.staged_in_bytes >= before.staged_in_bytes + (n * cols * 8) as u64, + "{what}: the upload did not go through the staged path: {before:?} -> {after:?}" + ); + assert!( + after.staged_out_bytes >= before.staged_out_bytes + (n * blowup * cols * 8) as u64, + "{what}: the download did not go through the staged path: {before:?} -> {after:?}" + ); + } +} + +#[test] +fn the_ext3_commit_is_identical_through_either_staging() { + // `m` ext3 columns = `3 m` base columns row-major. + for (i, &(log_n, blowup, m)) in [(16usize, 4usize, 6usize), (17, 2, 3)].iter().enumerate() { + let n = 1usize << log_n; + let row_major = random(n * m * 3, 0xE530 + i as u64); + let weights = coset_weights_u64(n, COSET_OFFSET); + let (off, on) = both_paths(|| { + let (handle, host) = + math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep_rpl( + &row_major, + DeviceHash::Rpx256, + n, + m, + blowup, + &weights, + true, + 2, + ) + .expect("fused ext3 commit"); + let dev = download_handle(handle.buf.as_ref(), handle.ready.as_deref()); + let root = handle.tree.as_ref().expect("keep path keeps the tree").root; + (root, host, dev) + }); + let what = format!("ext3 log_n={log_n} blowup={blowup} m={m}"); + assert_eq!(off.0, on.0, "{what}: root"); + assert_same(&on.1, &off.1, &format!("{what}: host LDE")); + assert_same(&on.2, &off.2, &format!("{what}: column-major handle")); + } +} + +#[test] +fn the_split_tree_commit_is_identical_through_either_staging() { + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + let n = 1usize << log_n; + let split_col = (cols / 2).max(1); + let row_major = random(n * cols, 0x5B17 + i as u64); + let weights = coset_weights_u64(n, COSET_OFFSET); + let (off, on) = both_paths(|| { + let (pre, handle, host) = math_cuda::lde::coset_lde_row_major_split_trees_rpl( + &row_major, + None, + DeviceHash::Rpx256, + n, + cols, + blowup, + &weights, + split_col, + true, + true, + 2, + ) + .expect("fused split-tree commit"); + let dev = download_handle(handle.buf.as_ref(), handle.ready.as_deref()); + let mult_root = handle.tree.as_ref().expect("the mult tree").root; + (pre, mult_root, host, dev) + }); + let what = format!("split log_n={log_n} blowup={blowup} cols={cols}"); + assert_eq!(off.0, on.0, "{what}: precomputed tree nodes"); + assert_eq!(off.1, on.1, "{what}: multiplicity root"); + assert_same(&on.2, &off.2, &format!("{what}: host LDE")); + assert_same(&on.3, &off.3, &format!("{what}: column-major handle")); + } +} + +/// The column-major engine's device-only commit (the default for the shapes it +/// takes) stages its upload the same way: the same root and column-major handle +/// through either staging, no host LDE, and the upload's bytes moved staged. +#[test] +fn the_engine_commit_is_identical_through_either_staging() { + for (i, &(log_n, blowup, cols)) in SHAPES.iter().enumerate() { + let n = 1usize << log_n; + assert!( + math_cuda::lde_cm::supports(n, blowup), + "log_n={log_n} blowup={blowup}: the engine must take this shape" + ); + let row_major = random(n * cols, 0xC011 + i as u64); + let weights = coset_weights_u64(n, COSET_OFFSET); + let before = staging_totals(); + let (off, on) = both_paths(|| { + math_cuda::lde_cm::with_engine(true, || { + let (handle, host) = math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep_rpl( + &row_major, + None, + DeviceHash::Rpx256, + n, + cols, + blowup, + &weights, + false, + 2, + ) + .expect("device-only base commit"); + assert!(host.is_empty(), "a device-only commit returns no host LDE"); + let dev = download_handle(handle.buf.as_ref(), handle.ready.as_deref()); + let root = handle.tree.as_ref().expect("keep path keeps the tree").root; + (root, dev) + }) + }); + let what = format!("engine log_n={log_n} blowup={blowup} cols={cols}"); + assert_eq!(off.0, on.0, "{what}: root"); + assert_same(&on.1, &off.1, &format!("{what}: column-major handle")); + let after = staging_totals(); + assert!( + after.staged_in_bytes >= before.staged_in_bytes + (n * cols * 8) as u64, + "{what}: the upload did not go through the staged path: {before:?} -> {after:?}" + ); + } +} + +/// The process's setting as the commits read it: pairs unless +/// `LAMBDA_VM_STAGING_SHARED_SLAB=1`. Card-free; with `--nocapture` the one +/// `[gpu] transfer staging: ...` line it names is visible, which is how the gate +/// shows the default and the opt-out READ. +#[test] +fn the_staging_setting_reads_the_environment() { + let shared = std::env::var(STAGING_SHARED_SLAB_ENV).is_ok_and(|v| v == "1"); + assert_eq!(staging_pairs_enabled(), !shared); +} diff --git a/crypto/math-cuda/tests/sumcheck.rs b/crypto/math-cuda/tests/sumcheck.rs new file mode 100644 index 000000000..4fc9fd811 --- /dev/null +++ b/crypto/math-cuda/tests/sumcheck.rs @@ -0,0 +1,416 @@ +//! The sumcheck round and fold kernels against the host they replace. +//! +//! Runs on the merge-queue GPU box via `make test-math-cuda` — `SumcheckSession` +//! needs a real device, like the other tests here. +//! +//! The reference is `multilinear`'s own program evaluation and multilinear +//! fold, so the two cannot drift: what is compared is a sum over the whole cube +//! at each interpolation node, and the tables the fold leaves behind. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use multilinear::gpu::{ext3_from_raw, ext3_raw, lower}; +use multilinear::mle::Mle; +use multilinear::program::{Builder, Program}; +use multilinear::sumcheck::round_evaluations_for_program; + +type FE = FieldElement; + +/// A factor with no structure a kernel could accidentally satisfy. +fn factor(num_vars: usize, seed: u64) -> Mle { + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| { + let mix = |k: u64| i.wrapping_mul(6364136223846793005 + k).wrapping_add(seed) >> 9; + FE::new([ + FieldElement::::from(mix(1)), + FieldElement::::from(mix(7)), + FieldElement::::from(mix(13)), + ]) + }) + .collect(); + Mle::new(evals).expect("power of two") +} + +/// Every op, a constant, and a chain long enough to recycle slots — the shape +/// a compiled batch has, in miniature. +fn program(width: usize) -> Program { + let mut b = Builder::::new(); + let mut acc = b.var(0); + for slot in 1..width { + let v = b.var(slot); + let doubled = b.add(v, v); + let scaled = b.mul(doubled, acc); + let shifted = b.sub(scaled, v); + acc = b.neg(shifted); + } + let seven = b.fixed(FE::from(7u64)); + let root = b.mul(acc, seven); + b.finish(root).expect("a root") +} + +fn parity(num_vars: usize, width: usize, degree: usize) { + let factors: Vec> = (0..width).map(|k| factor(num_vars, k as u64 + 1)).collect(); + let program = program(width); + let lowered = lower(&program).expect("ext3 lowers"); + let raw: Vec<&[u64]> = factors + .iter() + .map(|f| unsafe { + core::slice::from_raw_parts(f.evals().as_ptr() as *const u64, f.len() * 3) + }) + .collect(); + + let mut session = math_cuda::sumcheck::SumcheckSession::new( + &raw, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .expect("a session (needs a GPU)"); + + let mut t = Vec::new(); + for node in 1..=degree { + t.extend_from_slice(&ext3_raw(&FE::from(node as u64)).expect("ext3")); + } + + // Two rounds, so the fold's output is what the second one reads. + let mut folded = factors; + for round in 0..2 { + let sums = session.round(&t).expect("a round"); + let device: Vec = sums.chunks_exact(3).map(ext3_from_raw::).collect(); + let host = + round_evaluations_for_program(&folded, &program, degree).expect("the host round"); + assert_eq!( + device, host, + "round {round} sums differ at 2^{num_vars}, width {width}" + ); + + let r = FE::new([ + FieldElement::::from(11 + round as u64), + FieldElement::::from(5), + FieldElement::::from(2), + ]); + session.fold(&ext3_raw(&r).expect("ext3")).expect("a fold"); + for f in &mut folded { + f.fix_first_variable_in_place(&r).expect("a variable"); + } + + let downloaded = session.download().expect("the folded factors"); + for (k, (device, host)) in downloaded.iter().zip(&folded).enumerate() { + let device: Vec = device.chunks_exact(3).map(ext3_from_raw::).collect(); + assert_eq!( + device, + host.evals(), + "factor {k} differs after round {round}" + ); + } + } +} + +#[test] +fn device_rounds_match_the_host_sumcheck() { + // Wider than one block, narrower than one, and a cube the grid strides over. + parity(10, 6, 3); + parity(14, 4, 5); + parity(9, 12, 2); +} + +/// The factors a device builds from the columns must be the ones the host +/// builds from the same columns and uploads. +/// +/// The two disagree silently otherwise: a factor read at the wrong offset is a +/// perfectly well-formed proof of a different statement. +fn factor_parity(num_vars: usize, columns: usize, offsets: &[usize], publics: usize) { + use math_cuda::sumcheck::DeviceFactors; + + let rows = 1usize << num_vars; + let base: Vec> = (0..columns) + .map(|k| { + let evals: Vec> = (0..rows as u64) + .map(|i| FieldElement::::from(i.wrapping_mul(2654435761 + k as u64) >> 3)) + .collect(); + Mle::new(evals).expect("power of two") + }) + .collect(); + let public: Vec> = (0..publics) + .map(|k| factor(num_vars, 900 + k as u64)) + .collect(); + + // One factor per (column, offset), then the public tables — the order + // `kinds` gives them in. + let mut host: Vec> = Vec::new(); + let mut plan: Vec = Vec::new(); + for (k, column) in base.iter().enumerate() { + for offset in offsets { + let shift = offset % rows; + let (wrapped, rest) = column.evals().split_at(shift); + host.push( + Mle::new( + rest.iter() + .chain(wrapped) + .map(|v| (*v).to_extension::()) + .collect(), + ) + .expect("power of two"), + ); + plan.push((k * rows) as u64); + plan.push(shift as u64); + plan.push((host.len() - 1) as u64); + } + } + let mut public_slots = Vec::new(); + for table in &public { + public_slots.push(host.len()); + host.push(table.clone()); + } + + let raw_columns: Vec> = base + .iter() + .map(|column| column.evals().iter().map(|v| *v.value()).collect()) + .collect(); + let raw_public: Vec> = public + .iter() + .map(|table| { + table + .evals() + .iter() + .flat_map(|v| ext3_raw(v).expect("ext3")) + .collect() + }) + .collect(); + + let columns_ref: Vec<&[u64]> = raw_columns.iter().map(Vec::as_slice).collect(); + let public_ref: Vec<(usize, &[u64])> = public_slots + .iter() + .zip(&raw_public) + .map(|(slot, table)| (*slot, table.as_slice())) + .collect(); + + let built = DeviceFactors::from_columns( + math_cuda::columns::Columns::Host(&columns_ref), + &plan, + &public_ref, + rows, + host.len(), + ) + .expect("factors on device (needs a GPU)"); + + // Read them back the only way a `DeviceFactors` can be read: a sumcheck + // that binds nothing yet, whose session owns the same buffer. + let program = program(host.len()); + let lowered = lower(&program).expect("lowers"); + let mut session = built + .session( + &[], + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .expect("a session"); + let mut t = Vec::new(); + for node in 1..=2u64 { + t.extend_from_slice(&ext3_raw(&FE::from(node)).expect("ext3")); + } + let device = session.round(&t).expect("a round"); + let expected = round_evaluations_for_program(&host, &program, 2).expect("the host round"); + let device: Vec = device.chunks_exact(3).map(ext3_from_raw::).collect(); + assert_eq!( + device, expected, + "factors built on device differ at 2^{num_vars}, {columns} columns" + ); +} + +#[test] +fn factors_built_on_device_match_the_host() { + factor_parity(10, 4, &[0, 1], 2); + factor_parity(12, 3, &[0, 1, 7], 0); + factor_parity(9, 2, &[0], 3); +} + +/// The weight built share by share and the weight built all at once must be +/// the same buffer. +/// +/// They are the same `eq` doubling either way; what differs is that one walks +/// a level of every share per launch and the other a level of one share. A +/// share reading the wrong coordinate — the index runs backwards — would be a +/// perfectly well-formed proof of a different claim. +fn weight_parity(len: usize, shares: &[(usize, usize)]) { + use math_cuda::device::backend; + + let built: Vec<(usize, Vec, [u64; 3])> = shares + .iter() + .enumerate() + .map(|(k, (offset, vars))| { + let point: Vec = (0..*vars) + .flat_map(|i| { + ext3_raw(&FE::new([ + FieldElement::::from((7 * k + i + 1) as u64), + FieldElement::::from((3 * i + 2) as u64), + FieldElement::::from((k + 5) as u64), + ])) + .expect("ext3") + }) + .collect(); + let scale = ext3_raw(&FE::from((k + 1) as u64)).expect("ext3"); + (*offset, point, [scale[0], scale[1], scale[2]]) + }) + .collect(); + + let be = backend().expect("a device"); + let stream = be.next_stream(); + + let mut one_at_a_time = stream.alloc_zeros::(len * 3).expect("alloc"); + for (offset, point, scale) in &built { + math_cuda::sumcheck::eq_expand_into(&stream, &mut one_at_a_time, *offset, point, scale) + .expect("a share"); + } + let mut all_at_once = stream.alloc_zeros::(len * 3).expect("alloc"); + math_cuda::sumcheck::eq_expand_shares_ext3(&stream, &mut all_at_once, &built) + .expect("the shares"); + + let expected = stream.clone_dtoh(&one_at_a_time).expect("read back"); + let got = stream.clone_dtoh(&all_at_once).expect("read back"); + stream.synchronize().expect("sync"); + assert_eq!(expected, got, "the batched weight differs at len {len}"); +} + +#[test] +fn a_weight_built_all_at_once_matches_one_share_at_a_time() { + // Shares of different heights, in and out of descending order, with gaps. + weight_parity(1 << 10, &[(0, 9), (512, 8), (768, 6), (832, 4)]); + weight_parity(1 << 12, &[(0, 6), (64, 11), (2112, 5), (2144, 10)]); + weight_parity(1 << 8, &[(0, 8)]); +} + +/// Columns evaluated together against the same columns evaluated one at a +/// time, which is what the batched fold replaces. +/// +/// The short shapes are the ones `LAMBDA_VM_ARGUE_DEVICE_COLUMNS` sends here: +/// under 2^16 rows, from a handful of columns to past the widest precompile's +/// 1,480, and two chunks of `CHUNK_BUDGET_BYTES` at the tallest (102 columns +/// of 2^15 rows fill one). +#[test] +fn columns_folded_together_match_one_at_a_time() { + let Ok(_) = math_cuda::device::backend() else { + eprintln!("no device; skipping"); + return; + }; + let shapes = [ + (16usize, 5usize), + (17, 12), + (16, 1), + (2, 7), + (5, 2000), + (10, 64), + (15, 200), + ]; + for (vars, width) in shapes { + let rows = 1usize << vars; + let columns: Vec> = (0..width) + .map(|k| { + (0..rows as u64) + .map(|i| i.wrapping_mul(6364136223846793005 + k as u64) >> 11) + .collect() + }) + .collect(); + let point: Vec = (0..vars) + .flat_map(|i| { + ext3_raw(&FE::new([ + FieldElement::::from((i + 3) as u64), + FieldElement::::from((2 * i + 1) as u64), + FieldElement::::from((i * i + 7) as u64), + ])) + .expect("ext3") + }) + .collect(); + + let borrowed: Vec<&[u64]> = columns.iter().map(|c| c.as_slice()).collect(); + let together = math_cuda::sumcheck::evaluate_many_base( + math_cuda::columns::Columns::Host(&borrowed), + &point, + ) + .expect("the batch"); + assert_eq!(together.len(), width); + for (k, column) in borrowed.iter().enumerate() { + let alone = math_cuda::sumcheck::evaluate_mle_base(column, &point).expect("one"); + assert_eq!( + together[k], alone, + "column {k} of {width} at {vars} vars differs" + ); + } + } +} + +/// A session's end read in one gathered copy is the per-factor read, bit for +/// bit (the Gruen rounds' cross-check reads its shadow so): after rounds and +/// folds, for a session that uploaded its factors and for one over resident +/// factors with weights added, from one factor to the widest precompile's +/// 1,482, at cubes the sessions stop at and ones they do not. +#[test] +fn a_session_end_read_in_one_copy_is_the_per_factor_read() { + let Ok(_) = math_cuda::device::backend() else { + eprintln!("no device; skipping"); + return; + }; + let mut t = Vec::new(); + for node in 1..=2u64 { + t.extend_from_slice(&ext3_raw(&FE::from(node)).expect("ext3")); + } + let r = ext3_raw(&FE::from(7u64)).expect("ext3"); + for (vars, width, folds) in [ + (6usize, 1usize, 1usize), + (8, 5, 3), + (10, 5, 1), + (7, 1482, 2), + (11, 40, 0), + ] { + let factors: Vec> = (0..width).map(|k| factor(vars, k as u64 + 3)).collect(); + let raw: Vec<&[u64]> = factors + .iter() + .map(|f| unsafe { + core::slice::from_raw_parts(f.evals().as_ptr() as *const u64, f.len() * 3) + }) + .collect(); + let program = program(width.min(8)); + let lowered = lower(&program).expect("ext3 lowers"); + + let mut uploaded = math_cuda::sumcheck::SumcheckSession::new( + &raw, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .expect("a session (needs a GPU)"); + let resident = math_cuda::sumcheck::DeviceFactors::upload(&raw).expect("the factors"); + let point: Vec = (0..vars as u64) + .flat_map(|i| ext3_raw(&FE::from(i + 2)).expect("ext3")) + .collect(); + let mut over_resident = resident + .session_with( + &[ + math_cuda::sumcheck::Extra::Eq(&point), + math_cuda::sumcheck::Extra::Table(raw[0]), + ], + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .expect("a session over resident factors"); + for session in [&mut uploaded, &mut over_resident] { + for _ in 0..folds { + session.round(&t).expect("a round"); + session.fold(&r).expect("a fold"); + } + let one_at_a_time = session.values().expect("the per-factor read"); + let gathered = session.values_gathered().expect("the gathered read"); + assert_eq!( + gathered, one_at_a_time, + "2^{vars} × {width} after {folds} folds: the gathered read differs" + ); + } + } +} diff --git a/crypto/math-cuda/tests/tree_download.rs b/crypto/math-cuda/tests/tree_download.rs new file mode 100644 index 000000000..2d3664cc5 --- /dev/null +++ b/crypto/math-cuda/tests/tree_download.rs @@ -0,0 +1,254 @@ +//! The precomputed tree's staged download (`LAMBDA_VM_TREE_DOWNLOAD=staged`) +//! must hand the host the same nodes as the pageable download, byte for byte, +//! across every chunk boundary and a ragged tail (a full node buffer is +//! `2^k − 32` bytes), in both forms of the preprocessed commit (the legacy +//! row-major one and the column-major engine's), and leave the multiplicity +//! tree, the host LDE and the column-major handle as they were. +//! +//! Device tests: they need the card, like every other file in this directory. + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math_cuda::DeviceHash; +use math_cuda::device::{ + STAGED_CHUNK_BYTES, backend, dtoh_staged_uncounted, set_tree_download_override, + tree_download_totals, +}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; + +/// Nodes per staging buffer. +const NODE_CHUNK: usize = STAGED_CHUNK_BYTES / 32; + +const COSET_OFFSET: u64 = 7; + +fn random(n: usize, seed: u64) -> Vec { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + (0..n).map(|_| rng.r#gen::()).collect() +} + +/// `weights[i] = g^i / n` — the `LdeTwiddles::coset_weights` format. +fn coset_weights_u64(n: usize, g: u64) -> Vec { + let inv_n = Fp::from(n as u64).inv().unwrap(); + let g_fp = Fp::from_raw(g); + let mut w = Vec::with_capacity(n); + let mut cur = inv_n; + for _ in 0..n { + w.push(*cur.value()); + cur = &cur * &g_fp; + } + w +} + +/// The first differing node, so a failure names the chunk it is in. +fn assert_same_nodes(got: &[[u8; 32]], want: &[[u8; 32]], what: &str) { + assert_eq!(got.len(), want.len(), "{what}: node count"); + if let Some(i) = got.iter().zip(want).position(|(a, b)| a != b) { + panic!( + "{what}: first difference at node {i} (chunk {}, node {} in it)", + i / NODE_CHUNK, + i % NODE_CHUNK + ); + } +} + +fn assert_same_u64(got: &[u64], want: &[u64], what: &str) { + assert_eq!(got.len(), want.len(), "{what}: length"); + if let Some(i) = got.iter().zip(want).position(|(a, b)| a != b) { + panic!("{what}: first difference at u64 {i}"); + } +} + +/// D2H a handle's column-major buffer once its `ready` event has fired. +fn download_handle( + buf: &math_cuda::CudaSlice, + ready: Option<&math_cuda::device::PooledEvent>, +) -> Vec { + let be = backend().expect("cuda backend"); + let stream = be.next_stream(); + if let Some(ev) = ready { + stream.wait(ev.event()).expect("wait on the ready event"); + } + let v = stream.clone_dtoh(buf).expect("D2H of the handle"); + stream.synchronize().expect("sync"); + v +} + +/// 32-byte elements through the generic staged download: every node exact, +/// the closure run once, at sizes around one chunk and past three. +#[test] +fn staged_node_download_is_exact_across_chunk_boundaries() { + use cudarc::driver::DevicePtr; + for (n, seed) in [ + (1usize, 1u64), + (NODE_CHUNK - 1, 2), + (NODE_CHUNK, 3), + (NODE_CHUNK + 1, 4), + // A full node buffer's shape: three buffers and a tail one node short. + (4 * NODE_CHUNK - 1, 5), + ] { + let bytes: Vec = random(n * 4, seed) + .iter() + .flat_map(|w| w.to_le_bytes()) + .collect(); + let want: Vec<[u8; 32]> = bytes + .chunks_exact(32) + .map(|c| c.try_into().expect("32 bytes")) + .collect(); + let be = backend().expect("cuda backend"); + let stream = be.next_stream(); + let dev = stream.clone_htod(&bytes).expect("htod"); + let ptr = { + let (p, _record) = dev.device_ptr(&stream); + p + }; + let mut got: Vec<[u8; 32]> = Vec::new(); + let mut calls = 0usize; + // SAFETY: `ptr` is `dev`, `n` nodes written by the upload queued on + // `stream` above, written by nothing after, alive until the loop's end. + unsafe { + dtoh_staged_uncounted(&stream, ptr, n, &mut got, || { + calls += 1; + Ok(()) + }) + } + .expect("staged node download"); + drop(dev); + assert_eq!(calls, 1, "n={n}: the closure must run exactly once"); + assert_same_nodes(&got, &want, &format!("nodes n={n}")); + } +} + +/// What one preprocessed commit hands back, for comparing the two downloads. +struct Commit { + pre: Option>, + mult_root: [u8; 32], + host_lde: Vec, + handle: Vec, +} + +#[allow(clippy::too_many_arguments)] +fn commit( + staged: bool, + engine: bool, + row_major: &[u64], + n: usize, + cols: usize, + blowup: usize, + split: usize, + build_precomputed: bool, + retain_host_lde: bool, + rows_per_leaf: usize, +) -> Commit { + let weights = coset_weights_u64(n, COSET_OFFSET); + set_tree_download_override(Some(staged)); + let (pre, handle, host_lde) = math_cuda::lde_cm::with_engine(engine, || { + math_cuda::lde::coset_lde_row_major_split_trees_rpl( + row_major, + None, + DeviceHash::Rpx256, + n, + cols, + blowup, + &weights, + split, + build_precomputed, + retain_host_lde, + rows_per_leaf, + ) + }) + .expect("split-tree commit"); + set_tree_download_override(None); + Commit { + pre, + mult_root: handle.tree.as_ref().expect("the multiplicity tree").root, + host_lde, + handle: download_handle(handle.buf.as_ref(), handle.ready.as_deref()), + } +} + +/// Both downloads over one commit, in both forms, with and without the +/// precomputed tree: the same nodes, multiplicity root, host LDE and handle; +/// each arm's counter moved by exactly its tree; no counter moved on a cache hit. +#[test] +fn the_staged_tree_download_is_the_pageable_download() { + // (log_n, blowup, cols, split, rows per leaf, engine, retain host LDE) + let cases: &[(u32, usize, usize, usize, usize, bool, bool)] = &[ + // Legacy row-major form (a kept host LDE takes it): 2^21 row-pair + // leaves, a 128 MiB − 32 B tree = three chunks and a ragged tail. + (20, 4, 3, 1, 2, false, true), + // The column-major engine: 2^22 one-row leaves, 256 MiB − 32 B. + (20, 4, 3, 2, 1, true, false), + // One chunk, both forms. + (12, 2, 4, 2, 2, false, true), + (12, 2, 4, 2, 1, true, false), + ]; + for (i, &(log_n, blowup, cols, split, rpl, engine, retain)) in cases.iter().enumerate() { + let n = 1usize << log_n; + if engine { + assert!( + math_cuda::lde_cm::supports(n, blowup), + "log_n={log_n} blowup={blowup}: the engine must take this shape" + ); + } + let row_major = random(n * cols, 0x7D0 + i as u64); + let tree_bytes = (2 * (n * blowup / rpl) - 1) * 32; + for build in [true, false] { + let what = format!( + "log_n={log_n} blowup={blowup} cols={cols} split={split} rpl={rpl} \ + engine={engine} retain={retain} build_precomputed={build}" + ); + let t0 = tree_download_totals(); + let pageable = commit( + false, engine, &row_major, n, cols, blowup, split, build, retain, rpl, + ); + let t1 = tree_download_totals(); + let staged = commit( + true, engine, &row_major, n, cols, blowup, split, build, retain, rpl, + ); + let t2 = tree_download_totals(); + + assert_eq!( + pageable.pre.is_some(), + build, + "{what}: pageable tree iff requested" + ); + assert_eq!( + staged.pre.is_some(), + build, + "{what}: staged tree iff requested" + ); + if let (Some(p), Some(s)) = (&pageable.pre, &staged.pre) { + assert_eq!(p.len() * 32, tree_bytes, "{what}: full node buffer"); + assert_same_nodes(s, p, &format!("{what}: precomputed nodes")); + } + assert_eq!( + pageable.mult_root, staged.mult_root, + "{what}: multiplicity root" + ); + assert_same_u64( + &staged.host_lde, + &pageable.host_lde, + &format!("{what}: host LDE"), + ); + assert_same_u64(&staged.handle, &pageable.handle, &format!("{what}: handle")); + + let (dp, ds) = (t1.since(&t0), t2.since(&t1)); + let one = u64::from(build); + assert_eq!(dp.trees, [one, 0], "{what}: the pageable arm's counter"); + assert_eq!( + dp.bytes, + [one * tree_bytes as u64, 0], + "{what}: pageable bytes" + ); + assert_eq!(ds.trees, [0, one], "{what}: the staged arm's counter"); + assert_eq!( + ds.bytes, + [0, one * tree_bytes as u64], + "{what}: staged bytes" + ); + } + } +} diff --git a/crypto/math-cuda/tests/whir_cap.rs b/crypto/math-cuda/tests/whir_cap.rs new file mode 100644 index 000000000..09c22cc4a --- /dev/null +++ b/crypto/math-cuda/tests/whir_cap.rs @@ -0,0 +1,298 @@ +//! W1 on the device: `DeviceCodeword::paths_and_cap` returns the paths and the +//! Merkle cap of ONE rebuilt tree, and both equal the host tree's. +//! +//! Needs a GPU (`make test-math-cuda`). The reference is `multilinear`'s host +//! commitment over the same codeword (`CodewordCommitment::new` on a base +//! codeword hashes on the host): its full paths (`open_many`) and its +//! owner-encoded capped paths (`open_many_capped`), whose first path ends with +//! the host tree's cap. +//! +//! Three regimes for what the commit kept, because the cap must come from the +//! same tree as the paths whatever supplied it: SERVED (same blocking as the +//! commit: the kept whole tree is read, or the tree is built on the kept leaf +//! layer), REHASHED (another blocking, so the kept object's key does not match +//! and the leaves are hashed), and EVICTED (what was kept is reclaimed by the +//! allocator's evictor before the opening). In each, the cap costs no extra +//! tree build: `tree_builds` rises by one per call, or by none when a kept +//! whole tree serves it. SERVED and EVICTED run in both retention modes +//! (`LFM_WHIR_WHOLE_TREES`, forced). + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use math_cuda::DeviceHash; +use multilinear::mle::Mle; +use multilinear::whir::{self, Domain}; +use multilinear::whir_commit::CodewordCommitment; +use multilinear::whir_hash::{DeviceHashKey, KeccakWhir, RpxWhir, WhirHash}; +use std::sync::Mutex; + +type FE = FieldElement; + +/// Every test here commits, and eviction moves the process-wide reservation +/// total, so they take turns (the `whir_tree_cache.rs` pattern). +static DEVICE_GLOBALS: Mutex<()> = Mutex::new(()); + +fn exclusive() -> std::sync::MutexGuard<'static, ()> { + DEVICE_GLOBALS.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// `LFM_WHIR_WHOLE_TREES` forced for one test, and the environment's setting +/// restored when this drops, on a panic too. +struct WholeTrees; + +impl WholeTrees { + fn force(on: bool) -> Self { + math_cuda::whir::force_whole_trees(Some(on)); + Self + } +} + +impl Drop for WholeTrees { + fn drop(&mut self) { + math_cuda::whir::force_whole_trees(None); + } +} + +/// Mirror of `DeviceHashKey::into_math_cuda` (cuda-gated on `multilinear`). +fn key() -> DeviceHash { + match H::DEVICE { + DeviceHashKey::Keccak256 => DeviceHash::Keccak256, + DeviceHashKey::Rpx256 => DeviceHash::Rpx256, + DeviceHashKey::Poseidon1 => DeviceHash::Poseidon1, + } +} + +fn poly(num_vars: usize, seed: u64) -> Mle { + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(6364136223846793005).wrapping_add(seed) >> 11)) + .collect(); + Mle::new(evals).expect("power of two") +} + +fn nodes(bytes: &[u8]) -> Vec<[u8; 32]> { + bytes + .chunks_exact(32) + .map(|n| n.try_into().expect("32 bytes")) + .collect() +} + +/// The device result against the host tree at blocking `k`, every cap height +/// up to `min(depth, 6)`. `before_each(c)` runs before the call at height `c` +/// (the EVICTED regime evicts there, so every height meets a rebuilt tree). +/// `builds` is the trees each call must build: one, or none when a kept whole +/// tree serves it. +#[allow(clippy::too_many_arguments)] +fn assert_matches_host( + name: &str, + device: &math_cuda::whir::DeviceCodeword, + host: &CodewordCommitment, + k: usize, + positions: &[usize], + mut before_each: impl FnMut(usize), + builds: u64, + expect_leaf_pass: impl Fn(u64) -> bool, +) { + let depth = host.depth(); + let full = host.open_many(positions).expect("host paths"); + let pos32: Vec = positions.iter().map(|p| *p as u32).collect(); + for c in 0..=depth.min(6) { + before_each(c); + let before = device.tree_builds(); + let passes = device.leaf_passes(); + let (paths, cap) = device + .paths_and_cap(k, &pos32, c, key::()) + .unwrap_or_else(|e| panic!("{name} k={k} c={c}: paths_and_cap: {e:?}")); + assert_eq!( + device.tree_builds(), + before + builds, + "{name} k={k} c={c}: paths and cap must come from ONE tree — built by this \ + call, or the kept one — with no extra build for the cap" + ); + assert!( + expect_leaf_pass(device.leaf_passes() - passes), + "{name} k={k} c={c}: unexpected leaf-pass count {} -> {}", + passes, + device.leaf_passes() + ); + let paths = nodes(&paths); + let cap = nodes(&cap); + assert_eq!(cap.len(), 1 << c, "{name} k={k} c={c}: cap length"); + if c == 0 { + assert_eq!( + cap[0], + host.root(), + "{name} k={k}: the height-0 cap is the root" + ); + } + // Full paths: byte-identical to the host tree's, query by query. + for (q, opening) in full.iter().enumerate() { + assert_eq!( + &paths[q * depth..(q + 1) * depth], + opening.proof.merkle_path.as_slice(), + "{name} k={k} c={c}: path {q}" + ); + } + // The owner encoding the host produces ends with the host tree's cap. + let capped = host + .open_many_capped(positions, c, true) + .expect("host capped paths"); + if c > 0 { + assert_eq!( + &capped[0].proof.merkle_path[depth - c..], + cap.as_slice(), + "{name} k={k} c={c}: the device cap is the host tree's cap" + ); + } + } +} + +fn setup( + num_vars: usize, + k_commit: usize, +) -> (math_cuda::whir::DeviceCodeword, Vec, Domain) { + let f = poly(num_vars, 3); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + let (device, _root) = math_cuda::whir::commit_codeword(&raw, 2, k_commit, false, key::()) + .expect("device commit (needs a GPU)"); + let domain = Domain::::new(num_vars + 2).expect("domain"); + let host_codeword = + whir::encode::(&whir::lift_coefficients(&f), &domain).expect("encode"); + (device, host_codeword, domain) +} + +/// SERVED and REHASHED, k = 1..5, both hashes, both retention modes. +#[test] +fn paths_and_cap_are_the_host_trees_served_or_rehashed() { + let _exclusive = exclusive(); + fn run(name: &str, whole: bool) { + let num_vars = 12; + for k_commit in 1..=5usize { + let (device, host_codeword, _) = setup::(num_vars, k_commit); + let leaves = (host_codeword.len()) >> k_commit; + let positions = [0usize, 1, leaves / 3, leaves - 1]; + let host = + CodewordCommitment::<_, H>::new(&host_codeword, k_commit).expect("host commit"); + // Same blocking as the commit: the kept whole tree is read (no + // build), or the tree is built on the kept layer (no leaf pass). + assert_matches_host( + name, + &device, + &host, + k_commit, + &positions, + |_| {}, + u64::from(!whole), + |d| d == 0, + ); + // Another blocking: what was kept does not match, the leaves are + // hashed, and a tree is built every call. + let k_other = if k_commit == 5 { 3 } else { k_commit + 1 }; + let other = + CodewordCommitment::<_, H>::new(&host_codeword, k_other).expect("host commit"); + let leaves = host_codeword.len() >> k_other; + let positions = [0usize, leaves / 2, leaves - 1]; + assert_matches_host( + name, + &device, + &other, + k_other, + &positions, + |_| {}, + 1, + |d| d == 1, + ); + } + } + for whole in [false, true] { + let _mode = WholeTrees::force(whole); + run::( + if whole { + "keccak, whole trees" + } else { + "keccak, leaf layers" + }, + whole, + ); + run::( + if whole { + "rpx, whole trees" + } else { + "rpx, leaf layers" + }, + whole, + ); + } +} + +/// EVICTED: what the commit kept — the whole tree, or its leaf layer — is +/// reclaimed by the allocator's evictor, and the next opening rebuilds the +/// whole tree — its cap still the host's. Both retention modes. +/// +/// A rebuild RE-CAPTURES what was kept (by design: the evictor keeps the +/// codeword's registry entry because "the mutex lives with the codeword and may +/// refill", `math-cuda/src/whir.rs`), so a second opening after one eviction is +/// SERVED, not rebuilt. The eviction therefore runs before EVERY cap height: +/// each height meets a rebuilt tree and pays exactly one leaf pass, and what +/// was kept is back after each rebuild (the next eviction's precondition says +/// so). +#[test] +fn paths_and_cap_after_the_retained_layer_is_evicted() { + let _exclusive = exclusive(); + for whole in [false, true] { + let _mode = WholeTrees::force(whole); + evicted_regime(whole); + } +} + +fn evicted_regime(whole: bool) { + let be = math_cuda::device::backend().expect("eviction test needs a GPU"); + let k = 4; + let (device, host_codeword, _) = setup::(14, k); + let layer_bytes = device.retained_leaf_bytes(); + assert!(layer_bytes > 0, "precondition: the commit retained a layer"); + + let mut evictions = 0usize; + let evict = |c: usize| { + assert_eq!( + device.retained_leaf_bytes(), + layer_bytes, + "c={c}: the layer is retained (by the commit, or re-captured by the last rebuild)" + ); + let gap = layer_bytes / 2; + let hog_bytes = be + .vram_budget_bytes() + .saturating_sub(be.reserved_bytes()) + .saturating_sub(gap); + let hog = math_cuda::device::reserve(hog_bytes).expect("the hog reservation cannot fail"); + let got = math_cuda::device::reserve(layer_bytes) + .expect("the reserve must succeed by evicting the retained layer"); + assert_eq!( + device.retained_leaf_bytes(), + 0, + "c={c}: the layer was evicted" + ); + drop(got); + drop(hog); + evictions += 1; + }; + + let host = CodewordCommitment::<_, RpxWhir>::new(&host_codeword, k).expect("host commit"); + let leaves = host_codeword.len() >> k; + let positions = [0usize, 5, leaves / 2, leaves - 1]; + let name = if whole { + "rpx evicted, whole trees" + } else { + "rpx evicted, leaf layers" + }; + assert_matches_host(name, &device, &host, k, &positions, evict, 1, |d| d == 1); + assert_eq!( + evictions, + host.depth().min(6) + 1, + "one eviction per cap height" + ); + assert_eq!( + device.retained_leaf_bytes(), + layer_bytes, + "the last rebuild re-captured the layer" + ); +} diff --git a/crypto/math-cuda/tests/whir_commit.rs b/crypto/math-cuda/tests/whir_commit.rs new file mode 100644 index 000000000..aed15fd19 --- /dev/null +++ b/crypto/math-cuda/tests/whir_commit.rs @@ -0,0 +1,202 @@ +//! The device commit of a stacked polynomial against the host pipeline it +//! mirrors: same codeword, same root, and openings that verify against the +//! device tree. +//! +//! Runs on the merge-queue GPU box via `make test-math-cuda` — `commit_codeword` +//! needs a real device, like the other tests here. +//! +//! The reference is `multilinear`'s own pipeline rather than a copy of it: a +//! second implementation of the Möbius transform in this file could drift from +//! the one the prover runs with every test still green. +//! +//! ⚠ **Both hashes, and the device path asserted TAKEN.** The commit falls back +//! to the host silently when the device declines (`gpu::commit_tree_ext3` +//! returns `None` below a size threshold, on a missing card, or under +//! `LAMBDA_VM_NO_GPU_WHIR_COMMIT`). A parity test that let that happen would be +//! comparing the host against itself and passing — the exact shape recon B +//! found in `whir_fold.rs`, where two host paths were checked against each +//! other. So `commit_codeword_to_host` is called directly, which has no host +//! fallback at all: it either runs the kernels or returns an error this test +//! turns into a failure. + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::mle::Mle; +use multilinear::whir::{self, Domain}; +use multilinear::whir_commit::{CodewordCommitment, verify_opening}; +use multilinear::whir_hash::{DeviceHashKey, KeccakWhir, P1Whir, RpxWhir, WhirHash}; + +type FE = FieldElement; + +/// The dispatch key `math-cuda` takes, from the configuration that names it. +/// +/// ⚠ A MIRROR of `DeviceHashKey::into_math_cuda`, not a call to it. That method +/// is `#[cfg(feature = "cuda")]` on `multilinear`, and `multilinear` is a plain +/// dev-dependency here — enabling its cuda feature from this crate would be a +/// dependency cycle through the crate under test. The production bridge is +/// asserted total and bijective at compile time inside `multilinear` +/// (`whir_hash.rs`); what this mirror can still get wrong is being written +/// backwards, which `the_device_dispatch_really_selects_the_kernel_family` +/// would catch — it requires the two keys to produce DIFFERENT trees, and a +/// swapped mirror produces the same two trees in the other order. +fn device_key() -> math_cuda::DeviceHash { + match H::DEVICE { + DeviceHashKey::Keccak256 => math_cuda::DeviceHash::Keccak256, + DeviceHashKey::Rpx256 => math_cuda::DeviceHash::Rpx256, + DeviceHashKey::Poseidon1 => math_cuda::DeviceHash::Poseidon1, + } +} + +/// A polynomial with no structure a kernel could accidentally satisfy. +fn poly(num_vars: usize, seed: u64) -> Mle { + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(6364136223846793005).wrapping_add(seed) >> 11)) + .collect(); + Mle::new(evals).expect("power of two") +} + +fn parity(num_vars: usize, log_blowup: usize, log_folding: usize) { + let f = poly(num_vars, 1 + num_vars as u64); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + // No host fallback on this entry point: it hashes on the device or errors. + let (device_codeword, nodes) = + math_cuda::whir::commit_codeword_to_host(&raw, log_blowup, log_folding, device_key::()) + .unwrap_or_else(|e| panic!("device commit under {} (needs a GPU): {e:?}", H::NAME)); + + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + let host_codeword = + whir::encode::(&whir::lift_coefficients(&f), &domain).expect("encode"); + let host = CodewordCommitment::<_, H>::new(&host_codeword, log_folding).expect("host commit"); + + assert_eq!(device_codeword.len(), host_codeword.len()); + for (i, (device, host)) in device_codeword.iter().zip(&host_codeword).enumerate() { + assert_eq!( + device, + host.value(), + "codeword position {i} differs at 2^{num_vars}, blowup 2^{log_blowup}" + ); + } + + let nodes: Vec<[u8; 32]> = nodes + .chunks_exact(32) + .map(|node| node.try_into().expect("32 bytes")) + .collect(); + let codeword: Vec = device_codeword.into_iter().map(FE::from_raw).collect(); + let device = CodewordCommitment::<_, H>::from_precomputed(codeword, nodes, log_folding) + .expect("device commitment"); + assert_eq!(device.root(), host.root(), "roots differ under {}", H::NAME); + assert_eq!(device.num_leaves(), host.num_leaves()); + + // The root alone would pass on a tree whose inner nodes are garbage below + // it, so an opening is checked against it too. + for index in [0, 1, device.num_leaves() / 3, device.num_leaves() - 1] { + let opening = device.open(index).expect("open"); + assert!( + verify_opening::<_, H>(&device.root(), device.depth(), index, &opening), + "device opening at {index} does not verify under {}", + H::NAME + ); + assert_eq!( + opening.values, + host.open(index).expect("open").values, + "opened block {index} differs" + ); + } +} + +/// The shapes: both sides of the fused-8-level NTT threshold, fold widths 1 to +/// 6 (6 = the `first6` schedule's first round), and the Möbius windows — below the contiguous +/// kernel, exactly one window, one window plus a tiled level, and several full +/// tiles with a partial one on top. +fn every_shape() { + parity::(14, 2, 4); + parity::(16, 2, 4); + parity::(11, 1, 1); + parity::(12, 3, 5); + + parity::(5, 2, 3); + parity::(8, 2, 4); + parity::(9, 1, 2); + parity::(13, 2, 5); + parity::(17, 1, 4); + + // ★ k = 6, the widest fold the stack runs (W2 `first6`: tree 0's leaves + // are 64 base felts, 512 bytes, one leaf-kernel thread each). A production + // height's shape, a smaller one, and four leaves. + parity::(14, 2, 6); + parity::(12, 2, 6); + parity::(7, 1, 6); +} + +#[test] +fn device_commit_matches_the_host_pipeline() { + every_shape::(); +} + +/// ★ The same, under the algebraic hash — the kernels this branch adds. +#[test] +fn device_commit_matches_the_host_pipeline_under_rpx() { + every_shape::(); +} + +/// ★ Blowup 2 (`LAMBDA_VM_ZF_WHIR_BLOWUP=2`): the device commit at the rate's +/// production shapes is the host's, under both block hashes — the first fold +/// of `first6` at a production-like height and two others, and a later round's +/// fold of four. Every tree is a level shallower than at blowup 4, so its depth +/// parity flips (the arity-4 layout's odd tops move). +#[test] +fn device_commit_matches_the_host_pipeline_at_blowup_two() { + for (num_vars, log_folding) in [(16usize, 6usize), (13, 6), (14, 4), (15, 4)] { + parity::(num_vars, 1, log_folding); + parity::(num_vars, 1, log_folding); + } +} + +/// ★★ The two hashes really do build DIFFERENT trees on the device. +/// +/// Without this, both tests above would pass on a dispatch that ignored its key +/// and always launched keccak's kernels: the RPX host reference would be +/// compared against a keccak device tree and fail — unless the host side had +/// been mis-wired the same way, which is exactly the failure a single-hash +/// parity test cannot see. Comparing the two device roots directly closes it. +#[test] +fn the_device_dispatch_really_selects_the_kernel_family() { + let f = poly(12, 7); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + + let (_, keccak) = + math_cuda::whir::commit_codeword_to_host(&raw, 2, 4, device_key::()) + .expect("device commit (needs a GPU)"); + let (_, rpx) = math_cuda::whir::commit_codeword_to_host(&raw, 2, 4, device_key::()) + .expect("device commit (needs a GPU)"); + + assert_eq!(keccak.len(), rpx.len(), "the node layout must not change"); + assert_ne!( + keccak, rpx, + "the two kernel families produced identical trees, so the key is not being read" + ); +} + +/// ★ The same under ZisK's Poseidon1: the tagged coset leaves and the 4-ary +/// tree in the host's arity-4 layout, at every shape (odd and even depths). +#[test] +fn device_commit_matches_the_host_pipeline_under_p1() { + every_shape::(); +} + +/// ★★ Poseidon1 is a family of its own on the device: its tree is the arity-4 +/// layout (fewer nodes than the binary heap) and its root is neither binary +/// hash's. +#[test] +fn the_p1_dispatch_builds_its_own_arity_4_tree() { + let f = poly(12, 7); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + let (_, rpx) = math_cuda::whir::commit_codeword_to_host(&raw, 2, 4, device_key::()) + .expect("device commit (needs a GPU)"); + let (_, p1) = math_cuda::whir::commit_codeword_to_host(&raw, 2, 4, device_key::()) + .expect("device commit (needs a GPU)"); + let leaves = 1usize << (12 + 2 - 4); + assert_eq!(rpx.len(), 32 * (2 * leaves - 1)); + assert_eq!(p1.len(), 32 * math_cuda::p1_stark::tree_nodes(leaves)); + assert_ne!(rpx[..32], p1[..32], "the roots"); +} diff --git a/crypto/math-cuda/tests/whir_coset.rs b/crypto/math-cuda/tests/whir_coset.rs new file mode 100644 index 000000000..66ccc658b --- /dev/null +++ b/crypto/math-cuda/tests/whir_coset.rs @@ -0,0 +1,743 @@ +//! The first-round cosets evaluated from the polynomial +//! ([`math_cuda::whir_coset`]) against the re-encoded codeword they replace, +//! on the device. +//! +//! Runs on a GPU box. The parity test compares, value by value, the coset +//! blocks with the same leaves gathered out of +//! [`math_cuda::whir::encode_codeword_resident`]'s codeword (the re-encode a +//! retired stack is revived with) and with the host algorithm +//! ([`CosetPlan::evaluate_on_host`], pinned against `encode(lift_coefficients)` +//! in the module's own tests). The measurement arm times both at the +//! production shape and prints one `COSET EVAL …` line per reading. +//! +//! The stack read where it lies (`evaluate_slices` / `coset_openings` with +//! `in_place`) is compared with the assembled stack word for word, raw: it +//! changes what the tile kernel reads, not its arithmetic. + +use std::time::Instant; + +use math_cuda::columns::DeviceColumns; +use math_cuda::whir_coset::{ + CosetPlan, StackParts, StackSlices, coset_openings, evaluate_resident, evaluate_slices, +}; + +const P: u64 = 0xFFFF_FFFF_0000_0001; +/// Production geometry: blowup 4, first fold 6, 256 leaves under a kept node. +const LOG_BLOWUP: usize = 2; +const LOG_FOLD: usize = 6; +const KEPT: usize = 8; + +fn canonical(v: u64) -> u64 { + if v >= P { v - P } else { v } +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } +} + +/// A stack of `2^num_vars` values as the block prover lays one out: fifteen +/// columns of `2^(num_vars-4)` end to end and a zero tail of one more, each a +/// separate resident column. Returns the store, its parts and the columns. +fn stack(num_vars: usize, seed: u64) -> (DeviceColumns, Vec<(usize, usize)>, Vec>) { + let mut rng = Rng(seed); + let column = 1usize << (num_vars - 4); + let columns: Vec> = (0..15) + .map(|_| (0..column).map(|_| rng.next() % P).collect()) + .collect(); + let refs: Vec<&[u64]> = columns.iter().map(Vec::as_slice).collect(); + let store = DeviceColumns::upload(&refs).expect("device columns (needs a GPU)"); + let parts: Vec<(usize, usize)> = (0..15).map(|k| (k, k * column)).collect(); + (store, parts, columns) +} + +/// A stack laid out as `StackedLayout::build` lays the block prover's: two +/// tall columns (3/8 of the stack), then about 340 more, three in four short +/// (`2^4`…`2^8` values, under a 256-value tile) and the rest `2^9`… +/// `2^(num_vars-6)`, as many as fit in 94 % of the stack; taller first, every +/// seventh pair out of order, each at a multiple of its height with the +/// cursor only growing (so alignment gaps); then the room left up to 94 % +/// filled, taller columns first (a zero tail of 6 % or more, as the block's +/// stacks have). Every fifth column is 5 values short of its height (an end +/// mid-tile). The store holds a junk column before each column and a tile of +/// junk after the last, none of it in the stack, so a read past a part's end +/// shows as a wrong value. Returns the store and the parts. +fn production_stack(num_vars: usize, seed: u64) -> (DeviceColumns, Vec<(usize, usize)>) { + let mut rng = Rng(seed); + let placed = production_layout(num_vars, &mut rng); + let mut columns: Vec> = Vec::new(); + let mut parts = Vec::new(); + for (offset, len) in placed { + columns.push((0..1 + rng.next() % 37).map(|_| rng.next() % P).collect()); + parts.push((columns.len(), offset)); + columns.push((0..len).map(|_| rng.next() % P).collect()); + } + columns.push((0..256).map(|_| rng.next() % P).collect()); + let refs: Vec<&[u64]> = columns.iter().map(Vec::as_slice).collect(); + let store = DeviceColumns::upload(&refs).expect("device columns (needs a GPU)"); + (store, parts) +} + +/// [`production_stack`]'s columns: `(offset in the stack, length)`, in the +/// order placed. +fn production_layout(num_vars: usize, rng: &mut Rng) -> Vec<(usize, usize)> { + let cap = (1usize << num_vars) * 94 / 100; + let mut room = cap - (3usize << (num_vars - 3)); + let mut heights = Vec::new(); + for _ in 0..338 { + let h = if rng.next().is_multiple_of(4) { + 9 + (rng.next() % (num_vars as u64 - 14)) as usize + } else { + 4 + (rng.next() % 5) as usize + }; + if (1usize << h) <= room { + room -= 1 << h; + heights.push(h); + } + } + heights.sort_unstable_by(|a, b| b.cmp(a)); + for i in (0..heights.len().saturating_sub(1)).step_by(7) { + heights.swap(i, i + 1); + } + let heights: Vec = [num_vars - 2, num_vars - 3] + .into_iter() + .chain(heights) + .collect(); + let mut placed = Vec::new(); + let mut cursor = 0usize; + for (i, &h) in heights.iter().enumerate() { + let size = 1usize << h; + let offset = cursor.next_multiple_of(size); + if offset + size > cap { + continue; + } + cursor = offset + size; + placed.push((offset, if i % 5 == 4 { size - 5 } else { size })); + } + for h in (4..=num_vars - 4).rev() { + let size = 1usize << h; + while cursor.next_multiple_of(size) + size <= cap { + let offset = cursor.next_multiple_of(size); + cursor = offset + size; + placed.push((offset, size)); + } + } + placed +} + +/// [`stack`]'s columns with a zero tail of 15 %: thirteen whole columns and +/// six tenths of the fourteenth, a tile of junk after it in the store (not in +/// the stack). +fn tail_stack(num_vars: usize, seed: u64) -> (DeviceColumns, Vec<(usize, usize)>) { + let mut rng = Rng(seed); + let column = 1usize << (num_vars - 4); + let columns: Vec> = (0..15) + .map(|k| { + let len = match k { + 13 => column * 6 / 10, + 14 => 256, + _ => column, + }; + (0..len).map(|_| rng.next() % P).collect() + }) + .collect(); + let refs: Vec<&[u64]> = columns.iter().map(Vec::as_slice).collect(); + let store = DeviceColumns::upload(&refs).expect("device columns (needs a GPU)"); + let parts: Vec<(usize, usize)> = (0..14).map(|k| (k, k * column)).collect(); + (store, parts) +} + +/// The stacked polynomial on the host, its zero tail included. +fn whole(columns: &[Vec], num_vars: usize) -> Vec { + let mut whole = columns.concat(); + whole.resize(1 << num_vars, 0); + whole +} + +fn queries(num_vars: usize, count: usize, seed: u64) -> Vec { + queries_at(num_vars, LOG_BLOWUP, count, seed) +} + +/// [`queries`] over a codeword of blowup `2^log_blowup`. +fn queries_at(num_vars: usize, log_blowup: usize, count: usize, seed: u64) -> Vec { + let mut rng = Rng(seed ^ 0x9e37_79b9_7f4a_7c15); + let num_leaves = 1u64 << (num_vars + log_blowup - LOG_FOLD); + (0..count).map(|_| rng.next() % num_leaves).collect() +} + +/// The re-encoded codeword's values at every block's leaves, in the blocks' +/// order: block `q`'s leaf `r` value `t` at `(q·2^kept + r)·2^fold + t`, for a +/// codeword of blowup `2^log_blowup`. +fn reencoded_at_blocks_at( + store: &DeviceColumns, + parts: &[(usize, usize)], + num_vars: usize, + log_blowup: usize, + plan: &CosetPlan, +) -> Vec { + let codeword = + math_cuda::whir::encode_codeword_resident(store, parts, num_vars, log_blowup, true) + .expect("re-encode"); + codeword.synchronize().expect("re-encode done"); + let leaves: Vec = (0..plan.classes().len()) + .flat_map(|q| plan.block_leaves(q)) + .collect(); + let num_leaves = 1usize << (num_vars + log_blowup - LOG_FOLD); + codeword + .cosets(&leaves, num_leaves, 1 << LOG_FOLD) + .expect("gather the leaves") +} + +fn mismatches(a: &[u64], b: &[u64]) -> usize { + assert_eq!(a.len(), b.len(), "the same number of values"); + a.iter() + .zip(b) + .filter(|(x, y)| canonical(**x) != canonical(**y)) + .count() +} + +/// Every coset block equals the re-encoded codeword at its leaves and the +/// host algorithm, at three stack heights: 2 bits folded (tiles only), 6 +/// (tiles only), 10 (8 tile + 2 rest); the split path once. +#[test] +fn coset_blocks_equal_the_reencoded_codeword() { + coset_blocks_at(LOG_BLOWUP, 114); +} + +/// Blowup 2 (`LAMBDA_VM_ZF_WHIR_BLOWUP=2`) and its 229 queries: the same +/// parity over a codeword half as long, whose kept cosets are the same width. +#[test] +fn coset_blocks_at_blowup_two_equal_the_reencoded_codeword() { + coset_blocks_at(1, 229); +} + +/// [`coset_blocks_equal_the_reencoded_codeword`] at blowup `2^log_blowup` with +/// `count` queries. +fn coset_blocks_at(log_blowup: usize, count: usize) { + for (num_vars, seed) in [(16usize, 11u64), (20, 12), (24, 13)] { + let (store, parts, columns) = stack(num_vars, seed); + let leaves = queries_at(num_vars, log_blowup, count, seed); + let plan = CosetPlan::new(num_vars, log_blowup, LOG_FOLD, KEPT, &leaves).expect("plan"); + let mut split = Vec::new(); + let device = evaluate_resident(&store, &parts, &plan, Some(&mut split)) + .expect("coset evaluation") + .to_host() + .expect("read back"); + assert_eq!(split.len(), 4, "four phases timed: {split:?}"); + let queued = evaluate_resident(&store, &parts, &plan, None) + .expect("coset evaluation") + .to_host() + .expect("read back"); + let gathered = reencoded_at_blocks_at(&store, &parts, num_vars, log_blowup, &plan); + let host = plan.evaluate_on_host(&whole(&columns, num_vars)); + let (vs_codeword, vs_host, vs_queued) = ( + mismatches(&device, &gathered), + mismatches(&device, &host), + mismatches(&device, &queued), + ); + // The blowup-4 line names no rate; blowup 2's names its rate at the + // end. + let rate = if log_blowup == LOG_BLOWUP { + String::new() + } else { + format!(" blowup_log={log_blowup}") + }; + println!( + "COSET EVAL PARITY n={num_vars} cosets={} values={} vs_codeword={vs_codeword} vs_host={vs_host} vs_queued={vs_queued}{rate}", + plan.classes().len(), + device.len() + ); + assert_eq!( + (vs_codeword, vs_host, vs_queued), + (0, 0, 0), + "n {num_vars} blowup 2^{log_blowup}: the cosets differ" + ); + for &leaf in &leaves { + let q = plan.block_of(leaf).expect("a block"); + assert!(plan.block_leaves(q).contains(&leaf)); + } + } +} + +/// The tiles a stack of `2^num_vars` values folds in: `2^tile_bits` values +/// each, as many bits as fold below the kept coset, eight at most. +fn tiles(num_vars: usize) -> u64 { + let tile_bits = (num_vars - LOG_FOLD - KEPT).min(8); + 1 << (num_vars - tile_bits) +} + +/// The production-like layouts the card tests and the measurement use (no +/// card needed): about 340 columns or more, 85-94 % of the stack filled, +/// alignment gaps, and tiles of every kind at the plan's tile width. +#[test] +fn the_production_like_stacks_have_every_tile_kind() { + for (num_vars, seed) in [ + (16usize, 21u64), + (20, 22), + (24, 23), + (20, 31), + (24, 32), + (27, 100), + ] { + let placed = production_layout(num_vars, &mut Rng(seed)); + let n = 1usize << num_vars; + let filled: usize = placed.iter().map(|&(_, len)| len).sum(); + let mut sorted = placed.clone(); + sorted.sort_unstable(); + let (mut gaps, mut end) = (0usize, 0usize); + for &(offset, len) in &sorted { + assert!(offset >= end, "n {num_vars}: columns overlap"); + gaps += offset - end; + end = offset + len; + } + let parts: Vec<(usize, usize, usize)> = placed + .iter() + .map(|&(offset, len)| (offset, len, offset)) + .collect(); + let tile_bits = (num_vars - LOG_FOLD - KEPT).min(8); + let kinds = StackParts::new(&parts, num_vars, n) + .expect("the parts place") + .tile_kinds(num_vars, tile_bits); + println!( + "COSET EVAL LAYOUT n={num_vars} seed={seed} parts={} fill={:.3} gaps={:.4} tiles_full={} tiles_zero={} tiles_mixed={}", + placed.len(), + filled as f64 / n as f64, + gaps as f64 / n as f64, + kinds.full, + kinds.zero, + kinds.mixed + ); + assert!( + placed.len() >= 150, + "n {num_vars}: {} columns", + placed.len() + ); + assert!( + (n * 85 / 100..=n * 94 / 100).contains(&filled), + "n {num_vars}: {filled} filled" + ); + assert!(gaps > 0, "n {num_vars}: no alignment gap"); + assert!(kinds.zero > 0 && kinds.mixed > 0, "n {num_vars}: {kinds:?}"); + } +} + +/// ★ The stack read where it lies is the assembled stack: the cosets equal +/// word for word, raw (the same arithmetic over the same words), and equal +/// the re-encoded codeword at every block's leaves; the tiles' kinds add up. +/// At blowups 4 and 2, stacks of `2^16` (4-value tiles), `2^20` (64) and +/// `2^24` (256, two rest bits), three layouts each: [`stack`]'s, a +/// production-like one ([`production_stack`]: short columns, ends mid-tile, +/// alignment gaps, junk between the columns in the store) and a 15 % zero +/// tail ([`tail_stack`]). +#[test] +fn coset_blocks_read_in_place_equal_the_assembled_stack() { + for (log_blowup, count) in [(LOG_BLOWUP, 114usize), (1, 229)] { + for (num_vars, seed) in [(16usize, 21u64), (20, 22), (24, 23)] { + let (bench, bench_parts, _) = stack(num_vars, seed); + let (production, production_parts) = production_stack(num_vars, seed); + let (tail, tail_parts) = tail_stack(num_vars, seed); + for (shape, store, parts) in [ + ("bench", &bench, &bench_parts), + ("production", &production, &production_parts), + ("tail", &tail, &tail_parts), + ] { + let label = format!("n {num_vars} blowup 2^{log_blowup} {shape}"); + let leaves = queries_at(num_vars, log_blowup, count, seed); + let plan = + CosetPlan::new(num_vars, log_blowup, LOG_FOLD, KEPT, &leaves).expect("plan"); + let slices = StackSlices::from_resident(store, parts, num_vars, log_blowup) + .expect("stack slices"); + let (assembled, none) = + evaluate_slices(&slices, &plan, false, None).expect("assembled"); + assert_eq!(none, None, "{label}: assembled, no tile kinds"); + let assembled = assembled.to_host().expect("read back"); + let mut split = Vec::new(); + let (in_place, kinds) = + evaluate_slices(&slices, &plan, true, Some(&mut split)).expect("in place"); + assert_eq!(split.len(), 4, "{label}: four phases timed: {split:?}"); + let in_place = in_place.to_host().expect("read back"); + let kinds = kinds.unwrap_or_else(|| panic!("{label}: the parts must place")); + let resident = evaluate_resident(store, parts, &plan, None) + .expect("resident") + .to_host() + .expect("read back"); + let gathered = reencoded_at_blocks_at(store, parts, num_vars, log_blowup, &plan); + let raw = in_place + .iter() + .zip(&assembled) + .filter(|(a, b)| a != b) + .count(); + let vs_codeword = mismatches(&in_place, &gathered); + println!( + "COSET EVAL INPLACE n={num_vars} blowup_log={log_blowup} shape={shape} tiles_full={} tiles_zero={} tiles_mixed={} raw_vs_assembled={raw} vs_codeword={vs_codeword}", + kinds.full, kinds.zero, kinds.mixed + ); + assert_eq!(in_place.len(), assembled.len(), "{label}: lengths"); + assert_eq!( + (raw, vs_codeword), + (0, 0), + "{label}: the stack read in place differs" + ); + assert_eq!(assembled, resident, "{label}: the two assembled paths"); + assert_eq!( + kinds.full + kinds.zero + kinds.mixed, + tiles(num_vars), + "{label}: every tile has one kind" + ); + if shape != "bench" { + assert!(kinds.zero > 0, "{label}: a zero tail: {kinds:?}"); + } + if shape == "production" { + assert!(kinds.mixed > 0, "{label}: mixed tiles: {kinds:?}"); + } + } + } + } +} + +/// ★ A round-0 tree's openings with the stack read where it lies are the +/// assembled stack's: the same subtrees' nodes, byte for byte, and the same +/// queried values; only the in-place run reports tile kinds. +#[test] +fn coset_openings_read_in_place_are_the_assembled_stacks() { + for (num_vars, seed) in [(20usize, 31u64), (24, 32)] { + let (store, parts) = production_stack(num_vars, seed); + let leaves = queries(num_vars, 114, seed); + let plan = CosetPlan::new(num_vars, LOG_BLOWUP, LOG_FOLD, KEPT, &leaves).expect("plan"); + let slices = + StackSlices::from_resident(&store, &parts, num_vars, LOG_BLOWUP).expect("slices"); + let hash = math_cuda::DeviceHash::Poseidon1; + let assembled = coset_openings(&slices, &plan, &leaves, hash, false).expect("assembled"); + let in_place = coset_openings(&slices, &plan, &leaves, hash, true).expect("in place"); + println!( + "COSET EVAL OPENINGS n={num_vars} nodes_equal={} values_equal={} in_place={:?}", + in_place.nodes == assembled.nodes, + in_place.values == assembled.values, + in_place.in_place + ); + assert_eq!(assembled.in_place, None, "n {num_vars}: assembled"); + assert!(in_place.in_place.is_some(), "n {num_vars}: read in place"); + assert_eq!(in_place.leaves, assembled.leaves, "n {num_vars}: leaves"); + assert!( + in_place.nodes == assembled.nodes, + "n {num_vars}: the subtrees' nodes differ" + ); + assert_eq!( + in_place.values, assembled.values, + "n {num_vars}: the queried values" + ); + } +} + +/// ★ The long kernels' grids cut to 1, 3 or 85 blocks (`device::with_grid_cap`, +/// as a background thread runs them beside latency-bound kernels): the same +/// words, raw, as the full grid, the stack assembled and read in place, and +/// the same openings' subtrees and values. At blowups 4 and 2, `2^20` and +/// `2^24`, the production-like stack. +#[test] +fn coset_kernels_under_a_grid_cap_give_the_full_grids_words() { + use math_cuda::device::{grid_cap, with_grid_cap}; + for (log_blowup, count) in [(LOG_BLOWUP, 114usize), (1, 229)] { + for (num_vars, seed) in [(20usize, 51u64), (24, 52)] { + let (store, parts) = production_stack(num_vars, seed); + let leaves = queries_at(num_vars, log_blowup, count, seed); + let plan = CosetPlan::new(num_vars, log_blowup, LOG_FOLD, KEPT, &leaves).expect("plan"); + let slices = + StackSlices::from_resident(&store, &parts, num_vars, log_blowup).expect("slices"); + let hash = math_cuda::DeviceHash::Poseidon1; + let run = || { + let assembled = evaluate_resident(&store, &parts, &plan, None) + .expect("assembled") + .to_host() + .expect("read back"); + let (in_place, kinds) = + evaluate_slices(&slices, &plan, true, None).expect("in place"); + assert!(kinds.is_some(), "read in place"); + let opened = coset_openings(&slices, &plan, &leaves, hash, true).expect("openings"); + ( + assembled, + in_place.to_host().expect("read back"), + opened.nodes, + opened.values, + ) + }; + assert_eq!(grid_cap(), None, "no cap outside with_grid_cap"); + let full = run(); + for cap in [1u32, 3, 85] { + let label = format!("n {num_vars} blowup 2^{log_blowup}, {cap} blocks"); + let capped = with_grid_cap(Some(cap), || { + assert_eq!(grid_cap(), Some(cap)); + run() + }); + assert_eq!(grid_cap(), None, "{label}: the cap restored"); + let raw = |a: &[u64], b: &[u64]| a.iter().zip(b).filter(|(x, y)| x != y).count(); + let (assembled, in_place) = (raw(&capped.0, &full.0), raw(&capped.1, &full.1)); + println!( + "COSET EVAL CAPPED n={num_vars} blowup_log={log_blowup} cap={cap} assembled_vs_full={assembled} in_place_vs_full={in_place} nodes_equal={} values_equal={}", + capped.2 == full.2, + capped.3 == full.3 + ); + assert_eq!( + (capped.0.len(), capped.1.len(), assembled, in_place), + (full.0.len(), full.1.len(), 0, 0), + "{label}: the cosets differ from the full grid's" + ); + assert!(capped.2 == full.2, "{label}: the subtrees' nodes differ"); + assert_eq!(capped.3, full.3, "{label}: the queried values differ"); + } + } + } +} + +/// ★ Parts the in-place kernels cannot place (two columns overlapping in the +/// stack: no stacked layout makes them, a stand-in for whatever might) are +/// assembled after all, the in-place request reporting no tile kinds — what +/// the caller counts as assembled after all — with the assembled values. +#[test] +fn parts_the_kernels_cannot_place_are_assembled_after_all() { + let num_vars = 20; + let (store, _, _) = stack(num_vars, 41); + let column = 1usize << (num_vars - 4); + let parts = [(0usize, 0usize), (1, column - 7), (2, 3 * column)]; + let leaves = queries(num_vars, 114, 41); + let plan = CosetPlan::new(num_vars, LOG_BLOWUP, LOG_FOLD, KEPT, &leaves).expect("plan"); + let slices = StackSlices::from_resident(&store, &parts, num_vars, LOG_BLOWUP).expect("slices"); + assert!( + slices.parts_in_place().is_none(), + "overlapping parts refused" + ); + let (asked, kinds) = evaluate_slices(&slices, &plan, true, None).expect("asked in place"); + let (off, _) = evaluate_slices(&slices, &plan, false, None).expect("assembled"); + let opened = coset_openings( + &slices, + &plan, + &leaves, + math_cuda::DeviceHash::Poseidon1, + true, + ) + .expect("openings"); + assert_eq!(kinds, None, "no tile kinds: assembled"); + assert_eq!(opened.in_place, None, "the openings assembled too"); + assert_eq!( + asked.to_host().expect("read back"), + off.to_host().expect("read back"), + "the assembled values" + ); +} + +fn env_usize(name: &str, default: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +fn summary(arm: &str, ms: &mut [f64]) { + ms.sort_by(|a, b| a.partial_cmp(b).expect("finite")); + let at = |p: f64| ms[((ms.len() - 1) as f64 * p).round() as usize]; + println!( + "COSET EVAL SUMMARY arm={arm} reps={} median_ms={:.3} min_ms={:.3} p10_ms={:.3} p90_ms={:.3} max_ms={:.3}", + ms.len(), + at(0.5), + ms[0], + at(0.1), + at(0.9), + ms[ms.len() - 1] + ); +} + +/// Peak pool bytes `run` holds beyond what was held before it, in MiB. +fn peak_mib(run: impl FnOnce()) -> f64 { + math_cuda::device::drain_and_trim().expect("drain"); + let (before, _) = math_cuda::device::pool_used_bytes().expect("pool"); + math_cuda::device::reset_pool_high_water().expect("reset"); + run(); + math_cuda::device::drain_and_trim().expect("drain"); + let (_, high) = math_cuda::device::pool_used_bytes().expect("pool"); + (high.max(before) - before) as f64 / (1 << 20) as f64 +} + +/// The measurement: one production-shape polynomial (`2^27` values, blowup 4, +/// first fold 6, 256 leaves under a kept node, 114 queries) — (A) today's +/// re-encode as a retired stack is revived (`encode_codeword_resident` on +/// resident columns, then a synchronize) against (B) the plan built from the +/// queries and the cosets evaluated from the assembled stack, and (B1) from +/// the stack read where it lies (each then a synchronize), interleaved +/// A B B1, `COSET_EVAL_WARM` warm-ups then `COSET_EVAL_REPS` timed each; the +/// same for a group of three polynomials one after another (A3, B3, B13); B's +/// and B1's phases; the arms' peak pool bytes; and the checks: B's and B1's +/// values equal A's codeword at every block's leaves, B1's equal B's raw. +/// +/// `COSET_EVAL_LOG` (27), `COSET_EVAL_QUERIES` (114), `COSET_EVAL_WARM` (3), +/// `COSET_EVAL_REPS` (20), `COSET_EVAL_BLOWUP` (the blowup's log, 2), +/// `COSET_EVAL_STACK` (`bench`: [`stack`]'s fifteen columns; `production`: +/// [`production_stack`]). +#[test] +#[ignore = "measurement arm: a minute on a card; run on the box"] +fn coset_eval_bench() { + let num_vars = env_usize("COSET_EVAL_LOG", 27); + let count = env_usize("COSET_EVAL_QUERIES", 114); + let warm = env_usize("COSET_EVAL_WARM", 3); + let reps = env_usize("COSET_EVAL_REPS", 20); + let log_blowup = env_usize("COSET_EVAL_BLOWUP", LOG_BLOWUP); + let shape = std::env::var("COSET_EVAL_STACK").unwrap_or_else(|_| "bench".into()); + let stacks: Vec<(DeviceColumns, Vec<(usize, usize)>)> = (0..3) + .map(|k| match shape.as_str() { + "bench" => { + let (store, parts, _) = stack(num_vars, 100 + k); + (store, parts) + } + "production" => production_stack(num_vars, 100 + k), + other => panic!("COSET_EVAL_STACK={other}: bench or production"), + }) + .collect(); + let sets: Vec> = (0..3) + .map(|k| queries_at(num_vars, log_blowup, count, 200 + k)) + .collect(); + let plan0 = CosetPlan::new(num_vars, log_blowup, LOG_FOLD, KEPT, &sets[0]).expect("plan"); + let slices: Vec = stacks + .iter() + .map(|(store, parts)| { + StackSlices::from_resident(store, parts, num_vars, log_blowup).expect("slices") + }) + .collect(); + println!( + "COSET EVAL SHAPE n={num_vars} blowup_log={log_blowup} fold={LOG_FOLD} kept={KEPT} queries={count} cosets={} coset_len={} device_mib_b={:.1} codeword_mib_a={:.1} stack={shape} parts={}", + plan0.classes().len(), + plan0.coset_len(), + plan0.device_bytes() as f64 / (1 << 20) as f64, + ((8usize << num_vars) + (8usize << (num_vars + log_blowup))) as f64 / (1 << 20) as f64, + stacks[0].1.len(), + ); + + let encode_one = |k: usize| { + let (store, parts) = &stacks[k]; + let codeword = + math_cuda::whir::encode_codeword_resident(store, parts, num_vars, log_blowup, false) + .expect("re-encode"); + codeword.synchronize().expect("re-encode done"); + }; + let cosets_one = |k: usize| { + let (store, parts) = &stacks[k]; + let plan = CosetPlan::new(num_vars, log_blowup, LOG_FOLD, KEPT, &sets[k]).expect("plan"); + let values = evaluate_resident(store, parts, &plan, None).expect("cosets"); + values.synchronize().expect("cosets done"); + }; + let in_place_one = |k: usize| { + let plan = CosetPlan::new(num_vars, log_blowup, LOG_FOLD, KEPT, &sets[k]).expect("plan"); + let (values, kinds) = evaluate_slices(&slices[k], &plan, true, None).expect("cosets"); + assert!(kinds.is_some(), "B1 read the stack in place"); + values.synchronize().expect("cosets done"); + }; + let time = |f: &dyn Fn()| { + let start = Instant::now(); + f(); + start.elapsed().as_secs_f64() * 1e3 + }; + + let mut arms: Vec> = vec![Vec::new(); 6]; + for rep in 0..warm + reps { + let ta = time(&|| encode_one(0)); + let tb = time(&|| cosets_one(0)); + let tb1 = time(&|| in_place_one(0)); + let ta3 = time(&|| (0..3).for_each(encode_one)); + let tb3 = time(&|| (0..3).for_each(cosets_one)); + let tb13 = time(&|| (0..3).for_each(in_place_one)); + let label = if rep < warm { "warm" } else { "timed" }; + println!( + "COSET EVAL REP {label} rep={rep} a_ms={ta:.3} b_ms={tb:.3} a3_ms={ta3:.3} b3_ms={tb3:.3} b1_ms={tb1:.3} b13_ms={tb13:.3}" + ); + if rep >= warm { + for (arm, t) in arms.iter_mut().zip([ta, tb, ta3, tb3, tb1, tb13]) { + arm.push(t); + } + } + } + for (name, arm) in ["A", "B", "A3", "B3", "B1", "B13"] + .iter() + .zip(arms.iter_mut()) + { + summary(name, arm); + } + + // B's and B1's phases, each synchronized (so they sum to a little more + // than the arm). B1's assemble is its parts' placement only; its tiles + // include the tile map. + let median = |v: &mut Vec| { + v.sort_by(|x, y| x.partial_cmp(y).expect("finite")); + v[v.len() / 2] + }; + for in_place in [false, true] { + let mut phases: Vec> = vec![Vec::new(); 5]; + for _ in 0..5 { + let start = Instant::now(); + let plan = + CosetPlan::new(num_vars, log_blowup, LOG_FOLD, KEPT, &sets[0]).expect("plan"); + phases[0].push(start.elapsed().as_secs_f64() * 1e3); + let mut split = Vec::new(); + if in_place { + evaluate_slices(&slices[0], &plan, true, Some(&mut split)).expect("cosets"); + } else { + let (store, parts) = &stacks[0]; + evaluate_resident(store, parts, &plan, Some(&mut split)).expect("cosets"); + } + for (i, (_, ms)) in split.iter().enumerate() { + phases[i + 1].push(*ms); + } + } + // B's line keeps its words; B1's names its arm at the end. + let tag = if in_place { " arm=B1" } else { "" }; + println!( + "COSET EVAL SPLIT plan_ms={:.3} assemble_ms={:.3} tiles_ms={:.3} rest_ms={:.3} transform_ms={:.3}{tag}", + median(&mut phases[0]), + median(&mut phases[1]), + median(&mut phases[2]), + median(&mut phases[3]), + median(&mut phases[4]), + ); + } + + let peak_a = peak_mib(|| encode_one(0)); + let peak_b = peak_mib(|| cosets_one(0)); + let peak_b1 = peak_mib(|| in_place_one(0)); + println!( + "COSET EVAL MEM peak_mib_a={peak_a:.1} peak_mib_b={peak_b:.1} peak_mib_b1={peak_b1:.1}" + ); + + let (store, parts) = &stacks[0]; + let device = evaluate_resident(store, parts, &plan0, None) + .expect("cosets") + .to_host() + .expect("read back"); + let (in_place, kinds) = evaluate_slices(&slices[0], &plan0, true, None).expect("cosets"); + let in_place = in_place.to_host().expect("read back"); + let kinds = kinds.expect("B1 read the stack in place"); + let gathered = reencoded_at_blocks_at(store, parts, num_vars, log_blowup, &plan0); + let differ = mismatches(&device, &gathered); + println!( + "COSET EVAL CHECK positions={} mismatches={differ} cosets={} queries={count}", + device.len(), + plan0.classes().len() + ); + let differ_b1 = mismatches(&in_place, &gathered); + let raw = in_place.iter().zip(&device).filter(|(a, b)| a != b).count(); + println!( + "COSET EVAL CHECK B1 positions={} mismatches={differ_b1} raw_vs_b={raw} tiles_full={} tiles_zero={} tiles_mixed={}", + in_place.len(), + kinds.full, + kinds.zero, + kinds.mixed + ); + assert_eq!(differ, 0, "the cosets differ from the re-encoded codeword"); + assert_eq!( + (differ_b1, raw, in_place.len()), + (0, 0, device.len()), + "the stack read in place differs" + ); +} diff --git a/crypto/math-cuda/tests/whir_encode_setting.rs b/crypto/math-cuda/tests/whir_encode_setting.rs new file mode 100644 index 000000000..987abb712 --- /dev/null +++ b/crypto/math-cuda/tests/whir_encode_setting.rs @@ -0,0 +1,55 @@ +//! The WHIR commit's encoding follows the process's LDE setting: the column +//! engine by default, the legacy `lift_spread` + tiled NTT under +//! `LAMBDA_VM_LDE_LEGACY=1` (`math_cuda::lde_cm::LEGACY_ENV`), the switch that +//! sends every device LDE back. +//! +//! ```text +//! cargo test --release -p math-cuda --test whir_encode_setting -- --nocapture +//! LAMBDA_VM_LDE_LEGACY=1 cargo test --release -p math-cuda --test whir_encode_setting -- --nocapture +//! ``` +//! +//! The parity tests force the pipeline per call (`with_engine`), so none of +//! them reads the setting; this one does. It is its own binary with one test, +//! so the process-wide engine counter moves for this commit alone, and the +//! once-per-process `[gpu] LDE: …` banner is printed by this commit's decision +//! (visible under `--nocapture`). + +use math_cuda::DeviceHash; +use math_cuda::lde_cm::{LEGACY_ENV, codewords_encoded, spread_supports}; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +#[test] +fn the_whir_encoding_follows_the_process_setting() { + // A 2^14 codeword at blowup 4: a shape the engine takes. + const LOG_EVALS: usize = 12; + const LOG_BLOWUP: usize = 2; + assert!(spread_supports( + (LOG_EVALS + LOG_BLOWUP) as u32, + LOG_BLOWUP as u32 + )); + let setting = std::env::var(LEGACY_ENV).unwrap_or_else(|_| "".into()); + let legacy = setting == "1"; + let evals: Vec = (0..1u64 << LOG_EVALS) + .map(|i| i.wrapping_mul(0x9E37_79B9_7F4A_7C15) % P) + .collect(); + + let before = codewords_encoded(); + math_cuda::whir::commit_codeword_to_host(&evals, LOG_BLOWUP, 4, DeviceHash::Rpx256) + .expect("the device commit"); + let through_engine = codewords_encoded() - before; + println!( + "whir_encode_setting: {LEGACY_ENV}={setting}: the encoding took the {} \ + ({through_engine} codeword through the engine)", + if through_engine == 0 { + "legacy path" + } else { + "column engine" + }, + ); + assert_eq!( + through_engine, + u64::from(!legacy), + "{LEGACY_ENV}={setting}: the WHIR encoding did not follow the process setting" + ); +} diff --git a/crypto/math-cuda/tests/whir_encode_successor.rs b/crypto/math-cuda/tests/whir_encode_successor.rs new file mode 100644 index 000000000..1a416f915 --- /dev/null +++ b/crypto/math-cuda/tests/whir_encode_successor.rs @@ -0,0 +1,215 @@ +//! A chain's first successor encoded from its bound message +//! ([`math_cuda::whir::encode_successor`], D-SPEED lever 5a) against the fold +//! it replaces, on the device. +//! +//! Runs on a GPU box. The message is bound by the opening session's own rounds +//! over the product program lowered by `multilinear::gpu::lower`, at the +//! challenges the fold then takes, so the pair compared is the production one: +//! [`OpeningSession::encode_successor`] and [`fold_resident_fused`] over the +//! committed codeword. As VALUES (canonical limbs): the encode computes the +//! fold's field elements by another sequence of operations, so raw limbs agree +//! wherever both are canonical; the ones that do not are counted and printed. +//! The host fold (`multilinear::whir::fold_codeword_k_on_host`) is the third +//! witness. +//! +//! [`fold_resident_fused`]: math_cuda::whir::fold_resident_fused + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use math_cuda::whir_open::OpeningSession; +use multilinear::whir::Domain; + +type FE = FieldElement; +type FE3 = FieldElement; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn canonical(limbs: &[u64]) -> Vec { + limbs + .iter() + .map(|&v| if v >= P { v - P } else { v }) + .collect() +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % P + } +} + +/// The opening's rule, `w·f`, lowered as the device session runs it. +fn product() -> multilinear::gpu::Lowered { + let mut builder = multilinear::program::Builder::::new(); + let weight = builder.var(0); + let message = builder.var(1); + let root = builder.mul(weight, message); + let program = builder.finish(root).expect("program"); + multilinear::gpu::lower(&program).expect("lowers") +} + +/// Two-inverse and the inverse generator of each of `levels` levels from a +/// domain with generator `g`: what `multilinear::gpu` passes a fold. +fn fold_scalars(g: FE, levels: usize) -> (u64, Vec) { + let two_inv = *FE::from(2u64).inv().expect("2 is invertible").value(); + let mut g_inv = g.inv().expect("a generator is invertible"); + let mut g_invs = Vec::with_capacity(levels); + for _ in 0..levels { + g_invs.push(*g_inv.value()); + g_inv = g_inv.square(); + } + (two_inv, g_invs) +} + +fn ext(raw: &[u64]) -> Vec { + raw.chunks_exact(3) + .map(|l| FE3::new([FE::from(l[0]), FE::from(l[1]), FE::from(l[2])])) + .collect() +} + +/// Commits `2^num_vars` base values at blowup 4 and a first fold of `levels`, +/// binds the message with the session's rounds, and compares the encoded +/// successor with the fused fold and the host fold at the same challenges. +fn the_encoded_successor_is_the_fold(num_vars: usize, levels: usize, seed: u64) { + the_encoded_successor_is_the_fold_at(num_vars, levels, seed, 2); +} + +/// [`the_encoded_successor_is_the_fold`] at blowup `2^log_blowup`. +fn the_encoded_successor_is_the_fold_at( + num_vars: usize, + levels: usize, + seed: u64, + log_blowup: usize, +) { + let mut rng = Rng(seed); + let len = 1usize << num_vars; + let evals: Vec = (0..len).map(|_| rng.next()).collect(); + let weight: Vec = (0..len * 3).map(|_| rng.next()).collect(); + let (committed, _root) = math_cuda::whir::commit_codeword( + &evals, + log_blowup, + levels, + true, + math_cuda::DeviceHash::Keccak256, + ) + .unwrap_or_else(|e| panic!("device commit (needs a GPU): {e:?}")); + + let mut session = OpeningSession::new(&weight, &evals).expect("session"); + let lowered = product(); + let mut rounds = session + .sumcheck( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .expect("session rounds"); + let mut alphas = Vec::with_capacity(levels * 3); + for _ in 0..levels { + rounds.round(&[1, 0, 0, 2, 0, 0]).expect("round"); + let alpha = [rng.next(), rng.next(), rng.next()]; + rounds.fold(&alpha).expect("fold"); + alphas.extend_from_slice(&alpha); + } + drop(rounds); + session.bound(levels); + let (_, message_before) = session.tables_to_host().expect("read back"); + + let encoded = session + .encode_successor(&committed, levels) + .expect("encode the successor"); + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + let (two_inv, g_invs) = fold_scalars(*domain.generator(), levels); + let folded = math_cuda::whir::fold_resident_fused(&committed, two_inv, &g_invs, &alphas) + .expect("fused fold"); + assert_eq!(encoded.elements(), folded.elements()); + assert_eq!(encoded.elements(), len << log_blowup >> levels); + assert!(!encoded.is_base(), "the successor is extension-valued"); + + let encoded_raw = encoded.to_host().expect("read back"); + let folded_raw = folded.to_host().expect("read back"); + let raw_differ = encoded_raw + .iter() + .zip(&folded_raw) + .filter(|(a, b)| a != b) + .count(); + println!( + "2^{num_vars}, fold {levels}: successor of {} values; raw limbs differing {raw_differ} of {}", + encoded.elements(), + encoded_raw.len() + ); + assert_eq!( + canonical(&encoded_raw), + canonical(&folded_raw), + "2^{num_vars}, fold {levels}: the encoded successor is not the fold" + ); + + // The host fold of the host's own codeword, at the same challenges. + let host_codeword = multilinear::whir::encode::( + &multilinear::whir::lift_coefficients( + &multilinear::mle::Mle::new(evals.iter().map(|v| FE::from(*v)).collect()) + .expect("power of two"), + ), + &domain, + ) + .expect("host encode"); + let (host, _) = multilinear::whir::fold_codeword_k_on_host::( + &host_codeword, + &domain, + &ext(&alphas), + ) + .expect("host fold"); + assert!( + host == ext(&encoded_raw), + "2^{num_vars}: the encoded successor is not the host fold" + ); + + // The message is the next group's factor: the encode read a copy. + let (_, message_after) = session.tables_to_host().expect("read back"); + assert_eq!( + message_after, message_before, + "the encode moved the message" + ); + + // A near miss must differ: the fold at another first challenge (one + // limb moved, which a fold of any number of levels has). + let mut moved = alphas.clone(); + moved[0] = if moved[0] == 0 { 1 } else { moved[0] - 1 }; + let other = math_cuda::whir::fold_resident_fused(&committed, two_inv, &g_invs, &moved) + .expect("fused fold"); + assert_ne!( + canonical(&other.to_host().expect("read back")), + canonical(&encoded_raw), + "the comparison must see the challenge" + ); +} + +/// The production first fold, at two stacks: the session's rounds and the +/// column engine's transform at more than one shape. +#[test] +fn the_encoded_successor_is_the_fold_at_the_production_first_fold() { + the_encoded_successor_is_the_fold(20, 6, 0x5A5A); + the_encoded_successor_is_the_fold(12, 6, 0x0A0A); +} + +/// Blowup 2 (`LAMBDA_VM_ZF_WHIR_BLOWUP=2`): the successor is the bound message +/// encoded at the rate the codeword has, which the encode reads off the sizes. +#[test] +fn the_encoded_successor_is_the_fold_at_blowup_two() { + the_encoded_successor_is_the_fold_at(20, 6, 0x6B6B, 1); + the_encoded_successor_is_the_fold_at(12, 6, 0x0B0B, 1); + the_encoded_successor_is_the_fold_at(16, 4, 0x1B1B, 1); +} + +/// Other first folds: the encode is not tied to six. +#[test] +fn the_encoded_successor_is_the_fold_at_other_first_folds() { + for levels in [1, 4] { + the_encoded_successor_is_the_fold(16, levels, 0x1234 + levels as u64); + } +} diff --git a/crypto/math-cuda/tests/whir_fold.rs b/crypto/math-cuda/tests/whir_fold.rs new file mode 100644 index 000000000..2eba872dc --- /dev/null +++ b/crypto/math-cuda/tests/whir_fold.rs @@ -0,0 +1,267 @@ +//! The codeword fold and the ext3 fold-block commit against the host. +//! +//! Runs on the merge-queue GPU box via `make test-math-cuda` — both need a real +//! device, like the other tests here. +//! +//! The references are the host arms themselves — +//! `whir::fold_codeword_k_on_host` and +//! `CodewordCommitment::from_codeword_on_host` — so the kernels cannot drift +//! from the protocol: what is compared is every folded value and the root over +//! the fold blocks. Naming them beats unsetting the kill switches, which are +//! read once per process and would leave the reference on the device. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use multilinear::mle::Mle; +use multilinear::whir::{self, Domain}; +use multilinear::whir_commit::CodewordCommitment; +use multilinear::whir_hash::KeccakWhir; + +type FE3 = FieldElement; +type FE = FieldElement; + +/// A codeword with no structure a kernel could accidentally satisfy: the +/// encoding of a pseudo-random multilinear, so it is a real codeword. +fn codeword(num_vars: usize, log_blowup: usize) -> (Vec, Domain) { + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(6364136223846793005).wrapping_add(11) >> 9)) + .collect(); + let f = Mle::new(evals).expect("power of two"); + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + let cw = whir::encode::(&whir::lift_coefficients(&f), &domain).expect("encode"); + (cw, domain) +} + +fn challenge(seed: u64) -> FE3 { + FE3::new([ + FE::from(seed * 31 + 7), + FE::from(seed * 17 + 5), + FE::from(seed + 3), + ]) +} + +/// `levels` folds against the host, then the same again on the ext3 codeword +/// the first group produced — the base entry point and the ext one. +#[test] +fn device_folds_match_the_host_fold() { + let (cw, domain) = codeword(14, 2); + let alphas: Vec = (1..=4).map(challenge).collect(); + + let (device, device_domain) = + whir::fold_codeword_k::(&cw, &domain, &alphas).expect("device fold"); + let (host, host_domain) = + whir::fold_codeword_k_on_host::(&cw, &domain, &alphas).expect("host fold"); + assert_eq!(device.len(), host.len()); + assert_eq!(device, host, "the base fold differs"); + assert_eq!(device_domain.log_size(), host_domain.log_size()); + + let alphas: Vec = (5..=7).map(challenge).collect(); + let (device_again, _) = + whir::fold_codeword_k::(&device, &device_domain, &alphas) + .expect("device fold"); + let (host_again, _) = + whir::fold_codeword_k_on_host::(&host, &host_domain, &alphas) + .expect("host fold"); + assert_eq!(device_again, host_again, "the extension fold differs"); +} + +/// The tree over an ext3 codeword's fold blocks must be the one the host +/// builds: same root, and openings that verify against it. +#[test] +fn the_device_ext3_commit_matches_the_host() { + let (cw, domain) = codeword(14, 2); + let alphas: Vec = (1..=4).map(challenge).collect(); + let (folded, _) = + whir::fold_codeword_k::(&cw, &domain, &alphas).expect("device fold"); + + let device = CodewordCommitment::<_, KeccakWhir>::from_codeword(folded.clone(), 4) + .expect("device commit"); + let host = + CodewordCommitment::<_, KeccakWhir>::from_codeword_on_host(folded, 4).expect("host commit"); + assert_eq!(device.root(), host.root(), "roots differ"); + for index in [0, 1, device.num_leaves() / 3, device.num_leaves() - 1] { + let opening = device.open(index).expect("open"); + assert!( + multilinear::whir_commit::verify_opening::<_, KeccakWhir>( + &device.root(), + device.depth(), + index, + &opening + ), + "device opening at {index} does not verify" + ); + } +} + +/// ★ Six levels in one residency (W2 `first6`: the first round folds 6 +/// variables of the base codeword), against the host arm. +/// +/// ⚠ Called on the device ENTRY POINT, not through `whir::fold_codeword_k`: +/// that wrapper returns `None` below its size threshold or under +/// `LAMBDA_VM_NO_GPU_WHIR_FOLD` and falls back to the host silently, so a +/// comparison through it can be the host against itself. This one either runs +/// the kernels or fails. +#[test] +fn the_device_folds_six_levels_as_the_host_does() { + for (num_vars, log_blowup) in [(14, 2), (12, 2), (7, 1)] { + let (cw, domain) = codeword(num_vars, log_blowup); + let alphas: Vec = (1..=6).map(challenge).collect(); + let (host, host_domain) = + whir::fold_codeword_k_on_host::(&cw, &domain, &alphas) + .expect("host fold"); + + // The arguments `multilinear::gpu::fold_codeword_k` builds. + let two_inv = *FE::from(2u64).inv().expect("2 is invertible").value(); + let mut g_inv = domain.generator().inv().expect("a generator is invertible"); + let mut g_invs = Vec::with_capacity(alphas.len()); + for _ in 0..alphas.len() { + g_invs.push(*g_inv.value()); + g_inv = g_inv.square(); + } + let raw_alphas: Vec = alphas + .iter() + .flat_map(|a| a.value().iter().map(|c| *c.value())) + .collect(); + let raw: Vec = cw.iter().map(|v| *v.value()).collect(); + let device = math_cuda::whir::fold_codeword_base(&raw, two_inv, &g_invs, &raw_alphas) + .unwrap_or_else(|e| panic!("device fold at 2^{num_vars} (needs a GPU): {e:?}")); + let device: Vec = device + .chunks_exact(3) + .map(|c| FE3::new([FE::from_raw(c[0]), FE::from_raw(c[1]), FE::from_raw(c[2])])) + .collect(); + + assert_eq!(host.len(), cw.len() >> 6); + assert_eq!(device.len(), host.len()); + assert_eq!( + device, host, + "the six-level base fold differs at 2^{num_vars}" + ); + assert_eq!(host_domain.log_size(), num_vars + log_blowup - 6); + } +} + +/// Two-inverse and the inverse generator of every level from `levels` down, +/// starting at a domain whose generator is `g`: what `multilinear::gpu` +/// passes a fold. +fn fold_scalars(g: FE, levels: usize) -> (u64, Vec) { + let two_inv = *FE::from(2u64).inv().expect("2 is invertible").value(); + let mut g_inv = g.inv().expect("a generator is invertible"); + let mut g_invs = Vec::with_capacity(levels); + for _ in 0..levels { + g_invs.push(*g_inv.value()); + g_inv = g_inv.square(); + } + (two_inv, g_invs) +} + +fn raw_challenges(from: u64, count: usize) -> Vec { + (from..from + count as u64) + .map(challenge) + .flat_map(|a| a.value().iter().map(|c| *c.value()).collect::>()) + .collect() +} + +/// ★ The one-launch fold IS the level-by-level fold, raw limb for limb: the +/// committed base codeword folded one to six levels, and an extension codeword +/// (the first fold's output) folded one to four more. Raw rather than equal, +/// because a proof serializes raw limbs and the fused path is the default. +/// +/// Called on the device entry points, so it either runs both kernels or fails. +#[test] +fn the_fused_fold_is_raw_identical_to_the_level_by_level_fold() { + for (num_vars, log_blowup) in [(14usize, 2usize), (9, 2)] { + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| i.wrapping_mul(6364136223846793005).wrapping_add(11) >> 9) + .collect(); + let (committed, _root) = math_cuda::whir::commit_codeword( + &evals, + log_blowup, + 1, + false, + math_cuda::DeviceHash::Keccak256, + ) + .unwrap_or_else(|e| panic!("device commit (needs a GPU): {e:?}")); + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + for levels in 1..=6usize { + let (two_inv, g_invs) = fold_scalars(*domain.generator(), levels); + let alphas = raw_challenges(1, levels); + let stepped = math_cuda::whir::fold_resident(&committed, two_inv, &g_invs, &alphas) + .expect("fold"); + let fused = math_cuda::whir::fold_resident_fused(&committed, two_inv, &g_invs, &alphas) + .expect("fused fold"); + assert_eq!(fused.elements(), stepped.elements()); + assert!(!fused.is_base()); + assert_eq!( + fused.to_host().expect("read back"), + stepped.to_host().expect("read back"), + "base fold, {levels} levels at 2^{num_vars}: raw limbs differ" + ); + } + + // The extension entry point: fold two levels, then k more both ways. + let (two_inv, g_invs) = fold_scalars(*domain.generator(), 2); + let ext = + math_cuda::whir::fold_resident(&committed, two_inv, &g_invs, &raw_challenges(9, 2)) + .expect("fold"); + let folded_domain = domain.squared().expect("square").squared().expect("square"); + for levels in 1..=4usize { + let (two_inv, g_invs) = fold_scalars(*folded_domain.generator(), levels); + let alphas = raw_challenges(20, levels); + let stepped = + math_cuda::whir::fold_resident(&ext, two_inv, &g_invs, &alphas).expect("fold"); + let fused = math_cuda::whir::fold_resident_fused(&ext, two_inv, &g_invs, &alphas) + .expect("fused fold"); + assert_eq!( + fused.to_host().expect("read back"), + stepped.to_host().expect("read back"), + "extension fold, {levels} levels at 2^{num_vars}: raw limbs differ" + ); + } + } +} + +/// ★ A fold keeps its tree's leaf layer against the room of the codeword it +/// came from, and gives those bytes back when the fold drops — not when the +/// codeword does. A chain drops each fold at the end of its round while the +/// committed codeword lives to the end of the group (of the run, for one held +/// across epochs), so a room that only shrank with the codeword kept every +/// opened chain's fold layers promised for all of that. +#[test] +fn a_folds_kept_layer_goes_back_to_the_room_when_the_fold_drops() { + let (num_vars, log_blowup, first) = (14usize, 2usize, 6usize); + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| i.wrapping_mul(6364136223846793005).wrapping_add(11) >> 9) + .collect(); + let hash = math_cuda::DeviceHash::Keccak256; + let (committed, _root) = + math_cuda::whir::commit_codeword(&evals, log_blowup, first, false, hash) + .unwrap_or_else(|e| panic!("device commit (needs a GPU): {e:?}")); + let held = committed.reserved_bytes(); + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + let (two_inv, g_invs) = fold_scalars(*domain.generator(), first); + let folded = math_cuda::whir::fold_resident_fused( + &committed, + two_inv, + &g_invs, + &raw_challenges(1, first), + ) + .expect("fused fold"); + folded.commit(4, hash).expect("the fold's tree"); + let kept = folded.retained_leaf_bytes(); + assert!( + kept > 0, + "precondition: the fold's commit kept its leaf layer" + ); + assert_eq!( + committed.reserved_bytes(), + held + kept, + "the fold's layer is promised under the room it shares with its source" + ); + drop(folded); + assert_eq!( + committed.reserved_bytes(), + held, + "the fold is gone and its layer's bytes are still promised" + ); +} diff --git a/crypto/math-cuda/tests/whir_group.rs b/crypto/math-cuda/tests/whir_group.rs new file mode 100644 index 000000000..1c8772107 --- /dev/null +++ b/crypto/math-cuda/tests/whir_group.rs @@ -0,0 +1,258 @@ +//! A grouped WHIR chain's two card steps against host references +//! (`GroupFold::Indexed`, D-ONECHAIN §2.5): the run's bound tables laid slot by +//! slot for the index rounds ([`OpeningSession::gather_slots`]), and its +//! polynomials' folds combined into the round-0 successor +//! ([`math_cuda::whir::combine_codewords`]). +//! +//! Runs on a GPU box (`cargo test -p math-cuda --release --test whir_group`). +//! As VALUES (canonical limbs), as every card step here is compared. The whole +//! grouped chain against the host's bytes is +//! `crypto/multilinear/tests/whir_group_device.rs`. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use math_cuda::whir_open::OpeningSession; +use multilinear::whir::Domain; + +type FE = FieldElement; +type FE3 = FieldElement; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn canonical(limbs: &[u64]) -> Vec { + limbs + .iter() + .map(|&v| if v >= P { v - P } else { v }) + .collect() +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % P + } +} + +fn ext(raw: &[u64]) -> Vec { + raw.chunks_exact(3) + .map(|l| FE3::new([FE::from(l[0]), FE::from(l[1]), FE::from(l[2])])) + .collect() +} + +/// `parts` sessions over `2^num_vars`, each then taken down to its first +/// `2^(num_vars − bound)` values as a sumcheck's rounds would leave it, laid +/// into `2^index_vars` slots: the gathered tables are each part's live values +/// in slot order, then zeros — in the first part's own allocation when it has +/// the room (`bound ≥ index_vars`), in a fresh one otherwise. +fn the_slots_are_the_parts_in_order( + parts: usize, + index_vars: usize, + num_vars: usize, + bound: usize, + seed: u64, +) { + let mut rng = Rng(seed); + let len = 1usize << num_vars; + let live = len >> bound; + let mut sessions = Vec::with_capacity(parts); + let mut want_weight = Vec::new(); + let mut want_message = Vec::new(); + for _ in 0..parts { + let weight: Vec = (0..len * 3).map(|_| rng.next()).collect(); + let message: Vec = (0..len).map(|_| rng.next()).collect(); + let mut session = OpeningSession::new(&weight, &message) + .unwrap_or_else(|e| panic!("session (needs a GPU): {e:?}")); + session.bound(bound); + want_weight.extend_from_slice(&weight[..live * 3]); + // The message is lifted on the card: `[v, 0, 0]`. + want_message.extend(message[..live].iter().flat_map(|&v| [v, 0, 0])); + sessions.push(session); + } + want_weight.resize((live * 3) << index_vars, 0); + want_message.resize((live * 3) << index_vars, 0); + let gathered = OpeningSession::gather_slots(sessions, index_vars).expect("gather"); + assert_eq!(gathered.len(), live << index_vars, "the slots' cube"); + let (weight, message) = gathered.tables_to_host().expect("read back"); + let tag = format!("{parts} parts in 2^{index_vars} slots, 2^{num_vars} bound {bound}"); + assert_eq!( + canonical(&weight), + canonical(&want_weight), + "{tag}: the weight" + ); + assert_eq!( + canonical(&message), + canonical(&want_message), + "{tag}: the message" + ); +} + +#[test] +fn gathered_slots_are_the_parts_in_slot_order_with_the_rest_zero() { + for (parts, index_vars) in [(2, 1), (3, 2), (4, 2)] { + // In place: the first part's allocation holds every slot. + the_slots_are_the_parts_in_order(parts, index_vars, 12, index_vars, 7 + parts as u64); + the_slots_are_the_parts_in_order(parts, index_vars, 12, 3, 11 + parts as u64); + // A fresh pair: it does not. + the_slots_are_the_parts_in_order(parts, index_vars, 12, index_vars - 1, 13 + parts as u64); + } +} + +#[test] +fn gathering_more_parts_than_slots_or_unequal_cubes_is_refused() { + let mut rng = Rng(5); + let session = |rng: &mut Rng, num_vars: usize| { + let len = 1usize << num_vars; + let weight: Vec = (0..len * 3).map(|_| rng.next()).collect(); + let message: Vec = (0..len).map(|_| rng.next()).collect(); + OpeningSession::new(&weight, &message) + .unwrap_or_else(|e| panic!("session (needs a GPU): {e:?}")) + }; + let three: Vec = (0..3).map(|_| session(&mut rng, 10)).collect(); + assert!( + OpeningSession::gather_slots(three, 1).is_err(), + "three parts in two slots" + ); + let unequal = vec![session(&mut rng, 10), session(&mut rng, 9)]; + assert!( + OpeningSession::gather_slots(unequal, 1).is_err(), + "cubes of two sizes" + ); +} + +/// Two-inverse and each level's inverse generator from a domain with +/// generator `g`: what `multilinear::gpu` passes a fold. +fn fold_scalars(g: FE, levels: usize) -> (u64, Vec) { + let two_inv = *FE::from(2u64).inv().expect("2 is invertible").value(); + let mut g_inv = g.inv().expect("a generator is invertible"); + let mut g_invs = Vec::with_capacity(levels); + for _ in 0..levels { + g_invs.push(*g_inv.value()); + g_inv = g_inv.square(); + } + (two_inv, g_invs) +} + +/// `parts` base codewords committed and folded `levels` times at one set of +/// challenges, then combined with one scale each: the sum is the host's +/// `Σ_p scale_p·fold_p` of the same folds, value for value. A near miss — two +/// scales exchanged — differs. +fn the_combined_successor_is_the_scaled_sum(parts: usize, num_vars: usize, seed: u64) { + the_combined_successor_is_the_scaled_sum_at(parts, num_vars, seed, 2); +} + +/// [`the_combined_successor_is_the_scaled_sum`] at blowup `2^log_blowup`. +fn the_combined_successor_is_the_scaled_sum_at( + parts: usize, + num_vars: usize, + seed: u64, + log_blowup: usize, +) { + let levels = 6; + let mut rng = Rng(seed); + let len = 1usize << num_vars; + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + let (two_inv, g_invs) = fold_scalars(*domain.generator(), levels); + let alphas: Vec = (0..levels * 3).map(|_| rng.next()).collect(); + let folds: Vec = (0..parts) + .map(|_| { + let evals: Vec = (0..len).map(|_| rng.next()).collect(); + let (committed, _root) = math_cuda::whir::commit_codeword( + &evals, + log_blowup, + levels, + true, + math_cuda::DeviceHash::Keccak256, + ) + .unwrap_or_else(|e| panic!("device commit (needs a GPU): {e:?}")); + math_cuda::whir::fold_resident_fused(&committed, two_inv, &g_invs, &alphas) + .expect("fused fold") + }) + .collect(); + let scales: Vec<[u64; 3]> = (0..parts) + .map(|_| [rng.next(), rng.next(), rng.next()]) + .collect(); + let refs: Vec<&math_cuda::whir::DeviceCodeword> = folds.iter().collect(); + let combined = math_cuda::whir::combine_codewords(&refs, &scales).expect("combine"); + assert!(!combined.is_base(), "the successor is extension-valued"); + assert_eq!(combined.elements(), folds[0].elements()); + + let host_folds: Vec> = folds + .iter() + .map(|f| ext(&f.to_host().expect("read back"))) + .collect(); + let scaled_sum = |scales: &[[u64; 3]]| -> Vec { + let mut sum = vec![FE3::zero(); host_folds[0].len()]; + for (fold, scale) in host_folds.iter().zip(scales) { + let scale = ext(scale)[0]; + for (acc, v) in sum.iter_mut().zip(fold) { + *acc += &scale * v; + } + } + sum + }; + let got = ext(&combined.to_host().expect("read back")); + assert!( + got == scaled_sum(&scales), + "{parts} parts at 2^{num_vars}: the combined successor is not the scaled sum" + ); + let mut exchanged = scales.clone(); + exchanged.swap(0, 1); + assert!( + got != scaled_sum(&exchanged), + "{parts} parts: the comparison must see which scale weighs which fold" + ); +} + +#[test] +fn a_combined_successor_is_the_eq_weighted_sum_of_the_folds() { + for parts in 2..=4 { + the_combined_successor_is_the_scaled_sum(parts, 14, 31 + parts as u64); + } +} + +/// Blowup 2 (`LAMBDA_VM_ZF_WHIR_BLOWUP=2`): the same sum over codewords half +/// as long. +#[test] +fn a_combined_successor_at_blowup_two_is_the_eq_weighted_sum_of_the_folds() { + for parts in 2..=4 { + the_combined_successor_is_the_scaled_sum_at(parts, 15, 41 + parts as u64, 1); + } +} + +#[test] +fn combining_codewords_of_two_lengths_or_a_base_one_is_refused() { + let mut rng = Rng(9); + let commit = |rng: &mut Rng, num_vars: usize| { + let evals: Vec = (0..1usize << num_vars).map(|_| rng.next()).collect(); + math_cuda::whir::commit_codeword(&evals, 2, 6, true, math_cuda::DeviceHash::Keccak256) + .unwrap_or_else(|e| panic!("device commit (needs a GPU): {e:?}")) + .0 + }; + let base = commit(&mut rng, 14); + let other = commit(&mut rng, 14); + let scale = [[1, 0, 0], [2, 0, 0]]; + assert!( + math_cuda::whir::combine_codewords(&[&base, &other], &scale).is_err(), + "base codewords" + ); + let domain = |n: usize| Domain::::new(n + 2).expect("domain"); + let fold = |c: &math_cuda::whir::DeviceCodeword, n: usize| { + let (two_inv, g_invs) = fold_scalars(*domain(n).generator(), 6); + math_cuda::whir::fold_resident_fused(c, two_inv, &g_invs, &[3u64; 18]).expect("fold") + }; + let short = commit(&mut rng, 13); + let (a, b) = (fold(&base, 14), fold(&short, 13)); + assert!( + math_cuda::whir::combine_codewords(&[&a, &b], &scale).is_err(), + "two lengths" + ); + assert!( + math_cuda::whir::combine_codewords(&[&a], &scale).is_err(), + "a scale without a part" + ); +} diff --git a/crypto/math-cuda/tests/whir_lean.rs b/crypto/math-cuda/tests/whir_lean.rs new file mode 100644 index 000000000..e6fe5a3ac --- /dev/null +++ b/crypto/math-cuda/tests/whir_lean.rs @@ -0,0 +1,411 @@ +//! The lean first rounds of a WHIR opening against the materialised session +//! they replace, on the device. +//! +//! Runs on a GPU box via `make test-math-cuda`. The reference is today's path +//! itself — [`OpeningSession`] built from the same shares and parts, rounds +//! through its [`SumcheckSession`] with the product program lowered by +//! `multilinear::gpu::lower` — so what is compared is every round's two +//! evaluations, the bound tables once the rounds are spent, and the message's +//! value at an out-of-domain-like point, as VALUES (canonical limbs): the lean +//! path computes the same field elements by a different sequence of +//! operations. `tests/host_kat/whir_host_kat.cpp` pins the same arithmetic on +//! the host. +//! +//! The two working-set tests (the lean opening's, and the fused fold's) are +//! here too, and every test in this binary takes [`DEVICE`]: they read the +//! driver's free memory before and after, and a neighbour allocating in the +//! same process at the same time would be counted as theirs. The parity tests +//! elsewhere run in parallel as usual; this binary does not. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use math_cuda::whir_open::{LeanMessage, LeanRound0, LeanShare, OpeningSession}; + +type FE = FieldElement; +type FE3 = FieldElement; + +/// Held by every test here, so a driver measurement never overlaps another +/// test's allocations. +static DEVICE: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn device() -> std::sync::MutexGuard<'static, ()> { + DEVICE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn canonical(limbs: &[u64]) -> Vec { + limbs + .iter() + .map(|&v| if v >= P { v - P } else { v }) + .collect() +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % P + } + + fn ext(&mut self) -> FE3 { + FE3::new([ + FE::from(self.next()), + FE::from(self.next()), + FE::from(self.next()), + ]) + } +} + +fn limbs(value: &FE3) -> [u64; 3] { + let [a, b, c] = value.value(); + [*a.value(), *b.value(), *c.value()] +} + +/// A stacked polynomial of `2^n`: a table of columns sharing a point, a tall +/// column, short ones scattered in the middle, and gaps — its parts, its weight +/// shares in the materialised session's form, and the same shares in lean form +/// with their points' `eq` halves. +struct Stack { + n: usize, + columns: Vec<(Vec, usize)>, + raw_shares: Vec<(usize, Vec, [u64; 3])>, + lean: Vec, + eq: Vec, +} + +fn stack(n: usize, seed: u64) -> Stack { + let mut rng = Rng(seed); + let mut placed: Vec<(usize, usize, Vec)> = Vec::new(); // (offset, vars, point) + let table_vars = n - 4; + let table_point: Vec = (0..table_vars).map(|_| rng.ext()).collect(); + for c in 0..3 { + placed.push((c << table_vars, table_vars, table_point.clone())); + } + let tall_vars = n - 1; + placed.push(( + 1 << tall_vars, + tall_vars, + (0..tall_vars).map(|_| rng.ext()).collect(), + )); + for c in 0..12usize { + let vars = 2 + c % 4; + let at = (1 << (n - 2)) + (c << 6); + placed.push((at, vars, (0..vars).map(|_| rng.ext()).collect())); + } + placed.sort_by_key(|p| p.0); + + let mut columns = Vec::new(); + let mut raw_shares = Vec::new(); + let mut lean = Vec::new(); + let mut eq: Vec = Vec::new(); + for (offset, vars, point) in &placed { + let column: Vec = (0..(1usize << vars)).map(|_| rng.next()).collect(); + columns.push((column, *offset)); + let scale = rng.ext(); + let raw_point: Vec = point.iter().flat_map(limbs).collect(); + raw_shares.push((*offset, raw_point, limbs(&scale))); + let lo_bits = vars / 2; + let (hi, lo) = point.split_at(vars - lo_bits); + let hi_at = eq.len() / 3; + for value in multilinear::eq::eq_evals(hi) { + eq.extend_from_slice(&limbs(&value)); + } + let lo_at = eq.len() / 3; + for value in multilinear::eq::eq_evals(lo) { + eq.extend_from_slice(&limbs(&value)); + } + lean.push(LeanShare { + stack_offset: *offset, + num_vars: *vars, + lo_bits, + hi_at, + lo_at, + scale: limbs(&scale), + }); + } + Stack { + n, + columns, + raw_shares, + lean, + eq, + } +} + +/// The opening's rule, `w·f`, lowered as the device session runs it. +fn product() -> multilinear::gpu::Lowered { + let mut builder = multilinear::program::Builder::::new(); + let weight = builder.var(0); + let message = builder.var(1); + let root = builder.mul(weight, message); + let program = builder.finish(root).expect("program"); + multilinear::gpu::lower(&program).expect("lowers") +} + +/// Nodes 1 and 2, as `run_rounds` passes them. +fn nodes() -> Vec { + vec![1, 0, 0, 2, 0, 0] +} + +/// `rounds` rounds both ways at the same challenges, then the bound tables and +/// the message at a point. +fn lean_matches_materialised(s: &Stack, resident: bool, rounds_wanted: usize) { + let parts: Vec<(&[u64], usize)> = s + .columns + .iter() + .map(|(column, offset)| (column.as_slice(), *offset)) + .collect(); + let store = math_cuda::columns::DeviceColumns::upload( + &s.columns + .iter() + .map(|(c, _)| c.as_slice()) + .collect::>(), + ); + let store_parts: Vec<(usize, usize)> = s + .columns + .iter() + .enumerate() + .map(|(i, (_, offset))| (i, *offset)) + .collect(); + let len = 1usize << s.n; + + let mut session = if resident { + OpeningSession::from_shares_and_resident( + &s.raw_shares, + len, + store.as_ref().expect("the card took the columns"), + &store_parts, + ) + } else { + OpeningSession::from_shares_and_parts(&s.raw_shares, len, &parts) + } + .unwrap_or_else(|e| panic!("materialised session (needs a GPU): {e:?}")); + let message = if resident { + LeanMessage::Resident { + store: store.as_ref().expect("the card took the columns"), + parts: &store_parts, + } + } else { + LeanMessage::Parts(&parts) + }; + let mut lean = LeanRound0::new(&s.lean, &s.eq, s.n, message) + .expect("device") + .expect("the lean opening takes this stack"); + + let lowered = product(); + let mut rounds = session + .sumcheck( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .expect("session rounds"); + let mut rng = Rng(0xABCD); + for round in 1..=rounds_wanted { + let want = rounds.round(&nodes()).expect("materialised round"); + let got = lean.round(&nodes()).expect("lean round"); + assert_eq!( + canonical(&got), + canonical(&want), + "round {round} of {rounds_wanted} ({}): the evaluations differ", + if resident { "resident" } else { "parts" } + ); + let alpha = limbs(&rng.ext()); + rounds.fold(&alpha).expect("fold"); + lean.bind(&alpha); + } + drop(rounds); + session.bound(rounds_wanted); + let (want_w, want_f) = session.tables_to_host().expect("read back"); + + let bound = lean.materialize().expect("materialise"); + let (got_w, got_f) = bound.tables_to_host().expect("read back"); + assert_eq!( + canonical(&got_w), + canonical(&want_w), + "the bound weight differs" + ); + assert_eq!( + canonical(&got_f), + canonical(&want_f), + "the bound message differs" + ); + + let point: Vec = (0..s.n - rounds_wanted) + .flat_map(|_| limbs(&rng.ext())) + .collect(); + assert_eq!( + canonical(&bound.evaluate_message(&point).expect("evaluate")), + canonical(&session.evaluate_message(&point).expect("evaluate")), + "the message at a point differs" + ); +} + +/// The default (three rounds) and `LAMBDA_VM_WHIR_LEAN_ROUNDS=6`, each against +/// the session materialised up front (`=0`). +const ROUNDS: [usize; 2] = [math_cuda::whir_open::LEAN_ROUNDS_DEFAULT, 6]; + +#[test] +fn the_lean_rounds_match_the_materialised_session_from_host_parts() { + let _device = device(); + for rounds in ROUNDS { + lean_matches_materialised(&stack(14, 0x51), false, rounds); + } +} + +#[test] +fn the_lean_rounds_match_the_materialised_session_from_resident_columns() { + let _device = device(); + for rounds in ROUNDS { + lean_matches_materialised(&stack(15, 0x77), true, rounds); + } +} + +/// ★ The accounting the lean path exists for. At 2^22 the materialised +/// session holds the weight and the lifted message, `2·2^22` extension values +/// (192 MiB), and built them through a base staging buffer (32 MiB more); the +/// lean opening over resident columns holds a `u16` per position (8 MiB), the +/// shares, the `eq` halves and the round scratch. Measured through the driver +/// with the pool drained, and by the structures' own accounting. +#[test] +fn the_lean_opening_holds_a_fraction_of_the_materialised_one() { + let _device = device(); + let s = stack(22, 0x99); + let store = math_cuda::columns::DeviceColumns::upload( + &s.columns + .iter() + .map(|(c, _)| c.as_slice()) + .collect::>(), + ) + .expect("the card took the columns (needs a GPU)"); + let store_parts: Vec<(usize, usize)> = s + .columns + .iter() + .enumerate() + .map(|(i, (_, offset))| (i, *offset)) + .collect(); + let len = 1usize << s.n; + let be = math_cuda::device::backend().expect("a device"); + + math_cuda::device::drain_and_trim().expect("drain"); + let before = be.free_vram_bytes().expect("free"); + let session = + OpeningSession::from_shares_and_resident(&s.raw_shares, len, &store, &store_parts) + .expect("session"); + be.ctx.synchronize().expect("sync"); + let materialised = before.saturating_sub(be.free_vram_bytes().expect("free")); + let materialised_accounted = session.device_bytes(); + drop(session); + + math_cuda::device::drain_and_trim().expect("drain"); + let before = be.free_vram_bytes().expect("free"); + let lean = LeanRound0::new( + &s.lean, + &s.eq, + s.n, + LeanMessage::Resident { + store: &store, + parts: &store_parts, + }, + ) + .expect("device") + .expect("the lean opening takes this stack"); + be.ctx.synchronize().expect("sync"); + let lean_taken = before.saturating_sub(be.free_vram_bytes().expect("free")); + let lean_accounted = lean.device_bytes(); + + let codeword = (1u64 << (s.n + 2)) * 8; + println!( + "opening working set at 2^{}: materialised {materialised} B measured / \ + {materialised_accounted} B accounted ({:.2}× the base codeword); lean \ + {lean_taken} B measured / {lean_accounted} B accounted ({:.3}×)", + s.n, + materialised_accounted as f64 / codeword as f64, + lean_accounted as f64 / codeword as f64, + ); + assert_eq!(materialised_accounted, (len as u64) * 48); + assert!(lean_accounted * 8 < materialised_accounted); + assert!( + lean_taken * 4 < materialised, + "lean took {lean_taken} B from the card against {materialised} B" + ); +} + +/// Two-inverse and the inverse generator of each of `levels` levels from a +/// domain with generator `g`: what `multilinear::gpu` passes a fold. +fn fold_scalars(g: FE, levels: usize) -> (u64, Vec) { + let two_inv = *FE::from(2u64).inv().expect("2 is invertible").value(); + let mut g_inv = g.inv().expect("a generator is invertible"); + let mut g_invs = Vec::with_capacity(levels); + for _ in 0..levels { + g_invs.push(*g_inv.value()); + g_inv = g_inv.square(); + } + (two_inv, g_invs) +} + +fn raw_challenges(from: u64, count: usize) -> Vec { + (from..from + count as u64) + .flat_map(|seed| [seed * 31 + 7, seed * 17 + 5, seed + 3]) + .collect() +} + +/// The fused fold's working set is its output: folding a committed codeword +/// six levels takes `2^(n+2−6)` extension values from the card, where the +/// level-by-level fold's first two levels alone take `2^(n+1) + 2^n` of them +/// while the codeword is resident. Measured through the driver, pool drained. +#[test] +fn the_fused_fold_takes_only_its_output_from_the_card() { + use multilinear::whir::Domain; + let _device = device(); + let (num_vars, log_blowup) = (20usize, 2usize); + let evals: Vec = (0..(1u64 << num_vars)).map(|i| i * 7 + 3).collect(); + let (committed, _root) = math_cuda::whir::commit_codeword( + &evals, + log_blowup, + 1, + false, + math_cuda::DeviceHash::Keccak256, + ) + .unwrap_or_else(|e| panic!("device commit (needs a GPU): {e:?}")); + let domain = Domain::::new(num_vars + log_blowup).expect("domain"); + let (two_inv, g_invs) = fold_scalars(*domain.generator(), 6); + let alphas = raw_challenges(1, 6); + let be = math_cuda::device::backend().expect("a device"); + + let taken = |fused: bool| -> u64 { + math_cuda::device::drain_and_trim().expect("drain"); + let before = be.free_vram_bytes().expect("free"); + let folded = if fused { + math_cuda::whir::fold_resident_fused(&committed, two_inv, &g_invs, &alphas) + } else { + math_cuda::whir::fold_resident(&committed, two_inv, &g_invs, &alphas) + } + .expect("fold"); + be.ctx.synchronize().expect("sync"); + let after = be.free_vram_bytes().expect("free"); + drop(folded); + before.saturating_sub(after) + }; + let stepped = taken(false); + let fused = taken(true); + let codeword = (1u64 << (num_vars + log_blowup)) * 8; + println!( + "fold working set at 2^{}: level by level {stepped} B ({:.2}× the codeword), fused {fused} B ({:.3}×)", + num_vars + log_blowup, + stepped as f64 / codeword as f64, + fused as f64 / codeword as f64, + ); + assert!( + fused * 8 < stepped, + "the fused fold took {fused} B against the level-by-level {stepped} B" + ); +} diff --git a/crypto/math-cuda/tests/whir_lean_tiles.rs b/crypto/math-cuda/tests/whir_lean_tiles.rs new file mode 100644 index 000000000..ccd2c3ebd --- /dev/null +++ b/crypto/math-cuda/tests/whir_lean_tiles.rs @@ -0,0 +1,936 @@ +//! The lean rounds in TILE form — one pass over the message, then every round +//! on tables a tile's size smaller — against the share form they replace and +//! the materialised session both stand in for, on the device. +//! +//! Runs on a GPU box via `make test-math-cuda`. What is compared is every +//! round's two evaluations three ways (the session over the written-out +//! factors, `whir_lean_round` over the shares, the tile form), then the bound +//! tables and the message at a point, as VALUES (canonical limbs): the tile +//! form computes the same field elements by another sequence of operations. +//! The plan of each stack is `multilinear::lean_tiles::plan`, the production +//! planner; `tests/host_kat/whir_host_kat.cpp` checks the same kernels on the +//! host against a planner of its own. +//! +//! One test measures the card's high water, so every test here takes +//! [`DEVICE`] as `tests/whir_lean.rs` does. +//! +//! `the_tile_form_at_the_production_stack` is the measurement (ignored: a +//! `2^27` stack, about a minute): the share form against the tile form at the +//! shapes a block's chains have, values compared, milliseconds printed. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use math_cuda::whir_open::{LeanMessage, LeanRound0, LeanShare, OpeningSession, TileSpec}; + +type FE = FieldElement; +type FE3 = FieldElement; + +static DEVICE: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn device() -> std::sync::MutexGuard<'static, ()> { + DEVICE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn canonical(limbs: &[u64]) -> Vec { + limbs + .iter() + .map(|&v| if v >= P { v - P } else { v }) + .collect() +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % P + } + + fn ext(&mut self) -> FE3 { + FE3::new([ + FE::from(self.next()), + FE::from(self.next()), + FE::from(self.next()), + ]) + } + + fn point(&mut self, vars: usize) -> Vec { + (0..vars).flat_map(|_| limbs(&self.ext())).collect() + } +} + +fn limbs(value: &FE3) -> [u64; 3] { + let [a, b, c] = value.value(); + [*a.value(), *b.value(), *c.value()] +} + +/// A stacked polynomial of `2^n`: its columns with their stack offsets, and +/// its weight's shares as `(offset, point, scale)`. +struct Stack { + n: usize, + columns: Vec<(Vec, usize)>, + raw_shares: Vec<(usize, Vec, [u64; 3])>, +} + +/// Columns `(offset, variables, point)` with random data and scales. +fn stack_of(n: usize, placed: Vec<(usize, usize, Vec)>, seed: u64) -> Stack { + let mut rng = Rng(seed | 1); + let mut columns = Vec::with_capacity(placed.len()); + let mut raw_shares = Vec::with_capacity(placed.len()); + for (offset, vars, point) in placed { + assert_eq!(point.len(), vars * 3); + columns.push(((0..1usize << vars).map(|_| rng.next()).collect(), offset)); + raw_shares.push((offset, point, limbs(&rng.ext()))); + } + Stack { + n, + columns, + raw_shares, + } +} + +/// Columns of at least `2^min_vars` rows at multiples of their own size, with +/// gaps, their points ending in one of `tails` tails of `min_vars` +/// coordinates and starting with one of a few heads a height; `tall` puts a +/// column of `2^(n−1)` first. +fn mixed_stack(n: usize, min_vars: usize, tails: usize, tall: bool, seed: u64) -> Stack { + let mut rng = Rng(seed | 1); + let tails: Vec> = (0..tails).map(|_| rng.point(min_vars)).collect(); + let mut heads: Vec>> = vec![Vec::new(); n + 1]; + let mut point = |m: usize, rng: &mut Rng| -> Vec { + let pick = (rng.next() % 3) as usize; + while heads[m].len() <= pick { + let head = rng.point(m - min_vars); + heads[m].push(head); + } + let mut point = heads[m][pick].clone(); + point.extend_from_slice(&tails[(rng.next() % tails.len() as u64) as usize]); + point + }; + let len = 1usize << n; + let (mut cursor, mut stop) = (0usize, len); + let mut placed = Vec::new(); + if tall { + let at = if rng.next() & 1 == 1 { len / 2 } else { 0 }; + placed.push((at, n - 1, point(n - 1, &mut rng))); + if at == 0 { + cursor = len / 2; + } else { + stop = len / 2; + } + } + loop { + let top = (n - 2).max(min_vars); + let m = min_vars + (rng.next() as usize) % (top - min_vars + 1); + let size = 1usize << m; + let mut at = cursor.div_ceil(size) * size; + if rng.next().is_multiple_of(5) { + at += size; + } + if at + size > stop { + break; + } + placed.push((at, m, point(m, &mut rng))); + cursor = at + size; + } + placed.sort_by_key(|p| p.0); + stack_of(n, placed, seed ^ 0x5EED) +} + +/// The shares as the share form reads them: each point's table split at half +/// its variables, a pair of halves a share. +fn share_form(s: &Stack) -> (Vec, Vec) { + let mut shares = Vec::with_capacity(s.raw_shares.len()); + let mut eq: Vec = Vec::new(); + for (offset, point, scale) in &s.raw_shares { + let vars = point.len() / 3; + let lo_bits = vars / 2; + let coordinates: Vec = point + .chunks_exact(3) + .map(multilinear::gpu::ext3_from_raw::) + .collect(); + let (hi, lo) = coordinates.split_at(vars - lo_bits); + let hi_at = eq.len() / 3; + for value in multilinear::eq::eq_evals(hi) { + eq.extend_from_slice(&limbs(&value)); + } + let lo_at = eq.len() / 3; + for value in multilinear::eq::eq_evals(lo) { + eq.extend_from_slice(&limbs(&value)); + } + shares.push(LeanShare { + stack_offset: *offset, + num_vars: vars, + lo_bits, + hi_at, + lo_at, + scale: *scale, + }); + } + (shares, eq) +} + +/// The shares as the tile form reads them, from the production planner. +fn tile_form( + s: &Stack, + rounds: usize, + wide: bool, + check: bool, +) -> (Vec, Vec, TileSpec) { + let plan = multilinear::lean_tiles::plan(&s.raw_shares, s.n, rounds) + .expect("the stack takes the tile form"); + let shares = plan + .shares + .iter() + .map(|share| LeanShare { + stack_offset: share.stack_offset, + num_vars: share.num_vars, + lo_bits: plan.tile_bits, + hi_at: share.hi_at, + lo_at: share.lo_at, + scale: share.scale, + }) + .collect(); + let spec = TileSpec { + tile_bits: plan.tile_bits, + lean_rounds: rounds, + slots: plan.slots, + wide, + check, + }; + (shares, plan.eq, spec) +} + +/// The opening's rule, `w·f`, lowered as the device session runs it. +fn product() -> multilinear::gpu::Lowered { + let mut builder = multilinear::program::Builder::::new(); + let weight = builder.var(0); + let message = builder.var(1); + let root = builder.mul(weight, message); + let program = builder.finish(root).expect("program"); + multilinear::gpu::lower(&program).expect("lowers") +} + +/// Nodes 1 and 2, as `run_rounds` passes them. +fn nodes() -> Vec { + vec![1, 0, 0, 2, 0, 0] +} + +/// `rounds` rounds three ways at the same challenges — the materialised +/// session, the share form, the tile form — then the bound tables and the +/// message at a point. Returns the tile form's slots. +fn tiles_match(s: &Stack, resident: bool, rounds: usize, wide: bool, check: bool) -> usize { + let what = format!( + "2^{} {} {rounds} rounds{}{}", + s.n, + if resident { "resident" } else { "parts" }, + if wide { " wide" } else { "" }, + if check { " check" } else { "" } + ); + let parts: Vec<(&[u64], usize)> = s + .columns + .iter() + .map(|(column, offset)| (column.as_slice(), *offset)) + .collect(); + let store = math_cuda::columns::DeviceColumns::upload( + &s.columns + .iter() + .map(|(c, _)| c.as_slice()) + .collect::>(), + ) + .expect("the card took the columns (needs a GPU)"); + let store_parts: Vec<(usize, usize)> = s + .columns + .iter() + .enumerate() + .map(|(i, (_, offset))| (i, *offset)) + .collect(); + let len = 1usize << s.n; + let message = || { + if resident { + LeanMessage::Resident { + store: &store, + parts: &store_parts, + } + } else { + LeanMessage::Parts(&parts) + } + }; + + let mut session = if resident { + OpeningSession::from_shares_and_resident(&s.raw_shares, len, &store, &store_parts) + } else { + OpeningSession::from_shares_and_parts(&s.raw_shares, len, &parts) + } + .expect("materialised session"); + let (old_shares, old_eq) = share_form(s); + let mut old = LeanRound0::new(&old_shares, &old_eq, s.n, message()) + .expect("device") + .expect("the share form takes this stack"); + let (new_shares, new_eq, spec) = tile_form(s, rounds, wide, check); + let mut new = LeanRound0::with_tiles(&new_shares, &new_eq, s.n, message(), Some(spec)) + .expect("device") + .expect("the tile form takes this stack"); + assert!( + new.is_tiled() && !old.is_tiled(), + "{what}: each opening is the form it was asked for" + ); + let (checked_before, mismatched_before) = math_cuda::whir_open::tile_check_counts(); + + let lowered = product(); + let mut reference = session + .sumcheck( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .expect("session rounds"); + let mut rng = Rng(0xABCD); + for round in 1..=rounds { + let want = reference.round(&nodes()).expect("materialised round"); + let share = old.round(&nodes()).expect("share-form round"); + let tile = new.round(&nodes()).expect("tile-form round"); + assert_eq!( + canonical(&share), + canonical(&want), + "{what}: round {round}: the share form" + ); + assert_eq!( + canonical(&tile), + canonical(&want), + "{what}: round {round}: the tile form" + ); + let alpha = limbs(&rng.ext()); + reference.fold(&alpha).expect("fold"); + old.bind(&alpha); + new.bind(&alpha); + } + drop(reference); + session.bound(rounds); + let (checked, mismatched) = math_cuda::whir_open::tile_check_counts(); + if check { + assert_eq!( + (checked - checked_before, mismatched - mismatched_before), + (rounds as u64, 0), + "{what}: every round was compared with the share form, and none differed" + ); + } + + let (want_w, want_f) = session.tables_to_host().expect("read back"); + let tables_before = math_cuda::whir_open::tile_table_check_counts(); + let old_bound = old.materialize().expect("materialise"); + let new_bound = new.materialize().expect("materialise"); + let (tables, positions) = math_cuda::whir_open::tile_table_check_counts(); + assert_eq!( + (tables - tables_before.0, positions - tables_before.1), + (u64::from(check), 0), + "{what}: a checking opening compares the tile form's write, and nothing differs" + ); + for (name, bound) in [("share", &old_bound), ("tile", &new_bound)] { + let (got_w, got_f) = bound.tables_to_host().expect("read back"); + assert_eq!( + canonical(&got_w), + canonical(&want_w), + "{what}: the {name} form's bound weight" + ); + assert_eq!( + canonical(&got_f), + canonical(&want_f), + "{what}: the {name} form's bound message" + ); + } + // The tile form writes the message by the share form's own sequence. + assert_eq!( + new_bound.tables_to_host().expect("read back").1, + old_bound.tables_to_host().expect("read back").1, + "{what}: the bound message raw-identical to the share form's" + ); + let point: Vec = rng.point(s.n - rounds); + assert_eq!( + canonical(&new_bound.evaluate_message(&point).expect("evaluate")), + canonical(&session.evaluate_message(&point).expect("evaluate")), + "{what}: the message at a point" + ); + spec.slots +} + +/// ★ The tile form's rounds and bound tables are the share form's and the +/// materialised session's: stacks of one, two and many classes, with gaps and +/// a column taller than the blocks the rounds bind, three rounds and six, the +/// message read from host parts and from resident columns, the pass over the +/// message a thread a tile and a block a tile. +#[test] +fn the_tile_rounds_match_the_share_form_and_the_materialised_session() { + let _device = device(); + let mut seen = std::collections::BTreeSet::new(); + for (case, (n, min_vars, tails, tall)) in [ + (16usize, 8usize, 1usize, false), + (16, 8, 2, true), + (17, 9, 8, false), + (17, 7, 3, true), + (18, 10, 2, false), + (18, 11, 5, true), + ] + .into_iter() + .enumerate() + { + let s = mixed_stack(n, min_vars, tails, tall, 0x71 + case as u64); + for rounds in [math_cuda::whir_open::LEAN_ROUNDS_DEFAULT, 6] { + if multilinear::lean_tiles::plan(&s.raw_shares, s.n, rounds).is_none() { + continue; + } + for (resident, wide) in [(false, false), (true, false), (true, true), (false, true)] { + seen.insert(tiles_match(&s, resident, rounds, wide, false)); + } + } + } + println!("slots covered: {seen:?}"); + assert!(seen.contains(&1) && seen.contains(&2) && seen.iter().any(|slots| *slots >= 4)); +} + +/// The checking form compares every round with the share form on the card, +/// counts it, and finds none different. +#[test] +fn the_checking_form_compares_every_round_and_counts_it() { + let _device = device(); + let s = mixed_stack(17, 8, 3, true, 0x99); + tiles_match(&s, true, 3, false, true); + tiles_match(&s, false, 3, false, true); +} + +/// Clears the check's fault on the way out, whatever happens. +struct FaultOff; + +impl Drop for FaultOff { + fn drop(&mut self) { + math_cuda::whir_open::force_tile_check_fault(false); + } +} + +/// ★ The comparisons are load-bearing: with every tile-form round value moved +/// by one before it is compared, every round counts as different, and with +/// one entry of the bound weight moved for the comparison of the write, one +/// position does — and the opening still sends the share form's values and +/// keeps its tables, today's. +#[test] +fn the_checking_form_counts_a_round_that_differs_and_sends_todays_values() { + let _device = device(); + let s = mixed_stack(17, 8, 3, true, 0x99); + let parts: Vec<(&[u64], usize)> = s + .columns + .iter() + .map(|(column, offset)| (column.as_slice(), *offset)) + .collect(); + let (old_shares, old_eq) = share_form(&s); + let mut old = LeanRound0::new(&old_shares, &old_eq, s.n, LeanMessage::Parts(&parts)) + .expect("device") + .expect("the share form takes this stack"); + let (new_shares, new_eq, spec) = tile_form(&s, 3, false, true); + let mut new = LeanRound0::with_tiles( + &new_shares, + &new_eq, + s.n, + LeanMessage::Parts(&parts), + Some(spec), + ) + .expect("device") + .expect("the tile form takes this stack"); + let (checked_before, mismatched_before) = math_cuda::whir_open::tile_check_counts(); + let fault = FaultOff; + math_cuda::whir_open::force_tile_check_fault(true); + let mut rng = Rng(0xABCD); + for round in 1..=3 { + let want = old.round(&nodes()).expect("share-form round"); + let sent = new.round(&nodes()).expect("checking round"); + assert_eq!( + canonical(&sent), + canonical(&want), + "round {round}: a checking opening sends the share form's values" + ); + let alpha = limbs(&rng.ext()); + old.bind(&alpha); + new.bind(&alpha); + } + let tables_before = math_cuda::whir_open::tile_table_check_counts(); + let old_bound = old.materialize().expect("materialise"); + let new_bound = new.materialize().expect("materialise"); + drop(fault); + let (checked, mismatched) = math_cuda::whir_open::tile_check_counts(); + assert_eq!( + (checked - checked_before, mismatched - mismatched_before), + (3, 3), + "every round compared, every one counted as different" + ); + let (tables, positions) = math_cuda::whir_open::tile_table_check_counts(); + assert_eq!( + (tables - tables_before.0, positions - tables_before.1), + (1, 1), + "the write compared, the one moved position counted" + ); + let (got_w, got_f) = new_bound.tables_to_host().expect("read back"); + let (want_w, want_f) = old_bound.tables_to_host().expect("read back"); + assert_eq!( + (canonical(&got_w), got_f), + (canonical(&want_w), want_f), + "a checking opening keeps the share form's tables as written, the moved entry put back" + ); +} + +/// An opening whose rounds stop before its lean rounds are spent — a group +/// shorter than they are — writes its tables from the slot tables bound that +/// far: the share form's values at every count, the message raw-identical. +#[test] +fn a_tile_opening_written_early_writes_the_share_forms_tables() { + let _device = device(); + let s = mixed_stack(17, 8, 3, true, 0x5A); + let parts: Vec<(&[u64], usize)> = s + .columns + .iter() + .map(|(column, offset)| (column.as_slice(), *offset)) + .collect(); + let (old_shares, old_eq) = share_form(&s); + let (new_shares, new_eq, spec) = tile_form(&s, 3, false, false); + for spent in 0..3usize { + let mut old = LeanRound0::new(&old_shares, &old_eq, s.n, LeanMessage::Parts(&parts)) + .expect("device") + .expect("the share form takes this stack"); + let mut new = LeanRound0::with_tiles( + &new_shares, + &new_eq, + s.n, + LeanMessage::Parts(&parts), + Some(spec), + ) + .expect("device") + .expect("the tile form takes this stack"); + let mut rng = Rng(0xE4); + for _ in 0..spent { + assert_eq!( + canonical(&new.round(&nodes()).expect("tile round")), + canonical(&old.round(&nodes()).expect("share round")) + ); + let alpha = limbs(&rng.ext()); + old.bind(&alpha); + new.bind(&alpha); + } + let (want_w, want_f) = old + .materialize() + .expect("materialise") + .tables_to_host() + .expect("read back"); + let (got_w, got_f) = new + .materialize() + .expect("materialise") + .tables_to_host() + .expect("read back"); + assert_eq!( + canonical(&got_w), + canonical(&want_w), + "{spent} of 3: the weight" + ); + assert_eq!(got_f, want_f, "{spent} of 3: the message, raw-identical"); + } +} + +/// The opening the tile form declines is the share form's: a stack with a +/// column shorter than the plan can cut is not planned, and `with_tiles` +/// refuses shares that are not split at the tile. +#[test] +fn a_stack_the_tile_form_does_not_take_opens_in_share_form() { + let _device = device(); + let mut rng = Rng(0x33); + // A column of four rows beside columns of 2^12: tiles of four positions + // would hold more than the tables written after three rounds. + let placed = vec![ + (0usize, 12usize, rng.point(12)), + (1 << 12, 12, rng.point(12)), + (1 << 13, 2, rng.point(2)), + ]; + let s = stack_of(16, placed, 0x44); + assert!(multilinear::lean_tiles::plan(&s.raw_shares, s.n, 3).is_none()); + let parts: Vec<(&[u64], usize)> = s + .columns + .iter() + .map(|(column, offset)| (column.as_slice(), *offset)) + .collect(); + let (shares, eq) = share_form(&s); + let spec = TileSpec { + tile_bits: 6, + lean_rounds: 3, + slots: 2, + wide: false, + check: false, + }; + let tiled = LeanRound0::with_tiles(&shares, &eq, s.n, LeanMessage::Parts(&parts), Some(spec)) + .expect("declining touches no device"); + assert!( + tiled.is_none(), + "shares split at half their variables are not the tile form's" + ); + let lean = LeanRound0::new(&shares, &eq, s.n, LeanMessage::Parts(&parts)) + .expect("device") + .expect("the share form takes this stack"); + assert!(!lean.is_tiled()); +} + +/// One opening's three rounds and its bound tables, with the pool's high +/// water over them. `tiles` is `None` for the share form. +fn high_water( + s: &Stack, + store: &math_cuda::columns::DeviceColumns, + tiles: Option, +) -> (u64, u64) { + let store_parts: Vec<(usize, usize)> = s + .columns + .iter() + .enumerate() + .map(|(i, (_, offset))| (i, *offset)) + .collect(); + let message = LeanMessage::Resident { + store, + parts: &store_parts, + }; + let be = math_cuda::device::backend().expect("a device"); + math_cuda::device::drain_and_trim().expect("drain"); + math_cuda::device::reset_pool_high_water().expect("reset"); + let (held_before, _) = math_cuda::device::pool_used_bytes().expect("pool"); + let mut lean = match tiles { + None => { + let (shares, eq) = share_form(s); + LeanRound0::new(&shares, &eq, s.n, message) + } + Some(wide) => { + let (shares, eq, spec) = tile_form(s, 3, wide, false); + LeanRound0::with_tiles(&shares, &eq, s.n, message, Some(spec)) + } + } + .expect("device") + .expect("the opening takes this stack"); + let mut rng = Rng(0xABCD); + for _ in 0..3 { + lean.round(&nodes()).expect("round"); + lean.bind(&limbs(&rng.ext())); + } + let accounted = lean.device_bytes(); + let bound = lean.materialize().expect("materialise"); + be.ctx.synchronize().expect("sync"); + let (_, peak) = math_cuda::device::pool_used_bytes().expect("pool"); + drop(bound); + (peak.saturating_sub(held_before), accounted) +} + +/// ★ The tile form adds nothing to an opening's working set, and takes the +/// share form's position map out of it. It never allocates that map (a `u16` +/// a position); through the rounds all it holds is smaller than the bound +/// tables, which `materialize` allocates once `Φ` is freed, and what it keeps +/// beside them while it writes them is smaller than the map +/// ([`TileSpec::fits`]). So the pool's high water over an opening's rounds and +/// its `materialize` is the share form's less about the map. +#[test] +fn the_tile_form_adds_nothing_to_an_openings_high_water() { + let _device = device(); + // 16 columns of 2^20 at one point: 2^24, the shape of a block's chain at + // an eighth of its size. The map is 32 MiB, the bound tables 96 MiB. + let mut rng = Rng(0x22); + let point = rng.point(20); + let s = stack_of( + 24, + (0..16).map(|c| (c << 20, 20, point.clone())).collect(), + 0x23, + ); + let store = math_cuda::columns::DeviceColumns::upload( + &s.columns + .iter() + .map(|(c, _)| c.as_slice()) + .collect::>(), + ) + .expect("the card took the columns (needs a GPU)"); + let (share_peak, share_accounted) = high_water(&s, &store, None); + let (tile_peak, tile_accounted) = high_water(&s, &store, Some(false)); + let (wide_peak, _) = high_water(&s, &store, Some(true)); + let tables = 48u64 << (24 - 3); + let map = 2u64 << 24; + println!( + "opening high water at 2^24, three rounds: share form {share_peak} B (accounted {share_accounted} B \ + before its tables), tile form {tile_peak} B (accounted {tile_accounted} B), wide {wide_peak} B; \ + the bound tables {tables} B, the map {map} B" + ); + assert!( + tile_accounted + map / 2 < share_accounted, + "the tile form holds no position map through its rounds" + ); + // Half the map of margin: the pool rounds what it hands out, and the + // tile form keeps a few small tables the share form does not. + assert!( + tile_peak + map / 2 <= share_peak && wide_peak + map / 2 <= share_peak, + "the tile form's high water is not the share form's less the map: {tile_peak} / {wide_peak} B \ + against {share_peak} B" + ); +} + +// ─────────────────────────────── the measurement ───────────────────────────── + +/// Which opening a measured chain runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Form { + Share, + Tile, + Wide, +} + +impl Form { + fn name(self) -> &'static str { + match self { + Form::Share => "share", + Form::Tile => "tile", + Form::Wide => "wide", + } + } +} + +/// One opening's steps, in milliseconds: the construction (the position map; +/// for the tile form also its plan and its pass over the message), each of +/// the three rounds, and the bound tables' write. +#[derive(Clone, Copy, Default)] +struct Steps { + setup: f64, + rounds: [f64; 3], + write: f64, +} + +impl Steps { + /// What the rounds cost a chain: everything but the write. + fn rounds_ms(&self) -> f64 { + self.setup + self.rounds.iter().sum::() + } +} + +/// What an opening leaves to compare: its rounds' evaluations, and the bound +/// weight and message, all as canonical limbs. +type Values = (Vec>, Vec, Vec); + +/// The three challenges every measured opening binds. +fn bench_challenges() -> Vec<[u64; 3]> { + let mut rng = Rng(0xBE7C); + (0..3).map(|_| limbs(&rng.ext())).collect() +} + +/// One opening from prepared shares: its steps, each waited for, its rounds' +/// values, and the session its tables are in. +fn timed_opening( + shares: &[LeanShare], + eq: &[u64], + spec: Option, + n: usize, + store: &math_cuda::columns::DeviceColumns, + store_parts: &[(usize, usize)], +) -> (Steps, Vec>, OpeningSession) { + let be = math_cuda::device::backend().expect("a device"); + let ms = |from: std::time::Instant| from.elapsed().as_secs_f64() * 1e3; + let mut steps = Steps::default(); + let from = std::time::Instant::now(); + let mut lean = LeanRound0::with_tiles( + shares, + eq, + n, + LeanMessage::Resident { + store, + parts: store_parts, + }, + spec, + ) + .expect("device") + .expect("the opening takes this stack"); + lean.synchronize().expect("sync"); + steps.setup = ms(from); + let mut values = Vec::with_capacity(3); + for (round, alpha) in bench_challenges().iter().enumerate() { + let from = std::time::Instant::now(); + values.push(lean.round(&nodes()).expect("round")); + steps.rounds[round] = ms(from); + lean.bind(alpha); + } + let from = std::time::Instant::now(); + let bound = lean.materialize().expect("materialise"); + be.ctx.synchronize().expect("sync"); + steps.write = ms(from); + (steps, values, bound) +} + +fn median_min_max(mut values: Vec) -> String { + values.sort_by(|a, b| a.partial_cmp(b).expect("a time")); + format!( + "{:.3}/{:.3}/{:.3}", + values[values.len() / 2], + values[0], + values[values.len() - 1] + ) +} + +/// The shapes a block's chains have, at the production stack `2^27`: +/// `S1` 64 columns of `2^21` rows at one point (a chain of full tables); +/// `S2` 2,048 columns of `2^16` at one point (a keccak-round chain); +/// `S3` sixteen columns of `2^20` at one point and 56 of `2^21` at another, so +/// every column of tiles meets two classes (a chain holding a comparison +/// table); `S4` eight points over eight blocks of eight columns of `2^21`, so +/// every column of tiles meets eight. +fn production_shape(name: &str) -> Stack { + let mut rng = Rng(0x5A9E ^ name.len() as u64 ^ (name.as_bytes()[1] as u64) << 8); + let placed: Vec<(usize, usize, Vec)> = match name { + "S1" => { + let point = rng.point(21); + (0..64).map(|c| (c << 21, 21, point.clone())).collect() + } + "S2" => { + let point = rng.point(16); + (0..2048).map(|c| (c << 16, 16, point.clone())).collect() + } + "S3" => { + let (short, tall) = (rng.point(20), rng.point(21)); + (0..16) + .map(|c| (c << 20, 20, short.clone())) + .chain((8..64).map(|c| (c << 21, 21, tall.clone()))) + .collect() + } + "S4" => { + let points: Vec> = (0..8).map(|_| rng.point(21)).collect(); + (0..64) + .map(|c| (c << 21, 21, points[c / 8].clone())) + .collect() + } + other => panic!("no shape {other}"), + }; + stack_of(27, placed, 0x27) +} + +/// ★ THE MEASUREMENT: one chain's lean rounds at the production stack, the +/// share form against the tile form (a thread a tile, and a block a tile), at +/// the four shapes. Three warm-ups and twenty timed openings a form, every +/// step waited for. Values first: the three rounds' evaluations and both +/// bound tables equal across the forms, as canonical limbs. Then, per shape +/// and form, `TILE BENCH` lines (median/min/max milliseconds: the +/// construction, each round, the bound tables' write), the pool's high water +/// over an opening (`TILE MEM`), and the host's planning time (`TILE PLAN`); +/// last, the most one opening held by form (`TILE PEAKS`). Whoever runs it +/// reads its verdict from these lines; nothing here asserts a time. +#[test] +#[ignore = "the measurement: a 2^27 stack on the card, about a minute"] +fn the_tile_form_at_the_production_stack() { + let _device = device(); + const WARM: usize = 3; + const TIMED: usize = 20; + for name in ["S1", "S2", "S3", "S4"] { + let s = production_shape(name); + let store = math_cuda::columns::DeviceColumns::upload( + &s.columns + .iter() + .map(|(c, _)| c.as_slice()) + .collect::>(), + ) + .expect("the card took the columns (needs a GPU)"); + let store_parts: Vec<(usize, usize)> = s + .columns + .iter() + .enumerate() + .map(|(i, (_, offset))| (i, *offset)) + .collect(); + + // The host's part, timed apart: it is in neither form's card time. + let plan_ms = |f: &dyn Fn()| { + let times: Vec = (0..5) + .map(|_| { + let from = std::time::Instant::now(); + f(); + from.elapsed().as_secs_f64() * 1e3 + }) + .collect(); + median_min_max(times) + }; + println!( + "TILE PLAN shape={name} share_plan_ms={} tile_plan_ms={}", + plan_ms(&|| drop(share_form(&s))), + plan_ms(&|| drop(tile_form(&s, 3, false, false))) + ); + let (share_shares, share_eq) = share_form(&s); + let (tile_shares, tile_eq, spec) = tile_form(&s, 3, false, false); + let wide = TileSpec { wide: true, ..spec }; + + let mut reference: Option = None; + let (mut rounds_equal, mut tables_equal) = (true, true); + for form in [Form::Share, Form::Tile, Form::Wide] { + let (shares, eq, spec) = match form { + Form::Share => (&share_shares, &share_eq, None), + Form::Tile => (&tile_shares, &tile_eq, Some(spec)), + Form::Wide => (&tile_shares, &tile_eq, Some(wide)), + }; + let mut all: Vec = Vec::with_capacity(TIMED); + for rep in 0..WARM + TIMED { + let (steps, values, bound) = + timed_opening(shares, eq, spec, s.n, &store, &store_parts); + if rep >= WARM { + all.push(steps); + } + if rep == 0 { + let values: Vec> = values.iter().map(|v| canonical(v)).collect(); + let (w, f) = bound.tables_to_host().expect("read back"); + let (w, f) = (canonical(&w), canonical(&f)); + match &reference { + None => reference = Some((values, w, f)), + Some((want, want_w, want_f)) => { + rounds_equal &= &values == want; + tables_equal &= &w == want_w && &f == want_f; + } + } + } + drop(bound); + } + let column = + |pick: &dyn Fn(&Steps) -> f64| median_min_max(all.iter().map(pick).collect()); + println!( + "TILE BENCH shape={name} form={} reps={TIMED} setup_ms={} round1_ms={} round2_ms={} \ + round3_ms={} write_ms={} rounds_ms={} chain_ms={}", + form.name(), + column(&|t| t.setup), + column(&|t| t.rounds[0]), + column(&|t| t.rounds[1]), + column(&|t| t.rounds[2]), + column(&|t| t.write), + column(&|t| t.rounds_ms()), + column(&|t| t.rounds_ms() + t.write), + ); + } + println!( + "TILE VALUES shape={name} rounds_equal={rounds_equal} tables_equal={tables_equal} \ + tile_bits={} slots={}", + spec.tile_bits, spec.slots + ); + let (share_peak, _) = high_water(&s, &store, None); + let (tile_peak, _) = high_water(&s, &store, Some(false)); + let (wide_peak, _) = high_water(&s, &store, Some(true)); + println!( + "TILE MEM shape={name} share_peak={share_peak} tile_peak={tile_peak} wide_peak={wide_peak}" + ); + assert!( + rounds_equal && tables_equal, + "{name}: the forms' values differ" + ); + } + // The most one opening held, by form, counted from its allocations: the + // number a block's gate reads. + let mib = |bytes: u64| bytes as f64 / (1u64 << 20) as f64; + let [share, tile, check] = math_cuda::whir_open::lean_peak_bytes(); + println!( + "TILE PEAKS share_mib={:.1} tile_mib={:.1} check_mib={:.1}", + mib(share), + mib(tile), + mib(check) + ); +} diff --git a/crypto/math-cuda/tests/whir_mobius_fused.rs b/crypto/math-cuda/tests/whir_mobius_fused.rs new file mode 100644 index 000000000..656b43224 --- /dev/null +++ b/crypto/math-cuda/tests/whir_mobius_fused.rs @@ -0,0 +1,357 @@ +//! The WHIR encoding with its low Möbius levels folded into the NTT's first +//! pass (`LAMBDA_VM_WHIR_MOBIUS_FUSED=1`: the strided levels first, the first +//! of them gathering the stack, then the low `12 − log_blowup` levels on the +//! coefficients the NTT's first pass stages) encodes today's codeword: the same +//! field values — word for word once canonicalised, since the subtractions run +//! in another order — and the same tree root. +//! +//! The fused and today's encodings run in one process over stacks of raw words +//! with edge values planted, from every source a commit takes, under both NTT +//! plans. Needs a GPU (the box): +//! +//! cargo test -p math-cuda --release --test whir_mobius_fused -- --test-threads=1 +//! cargo test -p math-cuda --release --test whir_mobius_fused -- --ignored --nocapture --test-threads=1 + +use std::time::Instant; + +use math_cuda::columns::DeviceColumns; +use math_cuda::lde_cm::{wide_codewords_encoded, with_ntt_wide}; +use math_cuda::whir::{ + commit_codeword, encode_codeword_parts, encode_codeword_resident, encode_codeword_slices, + mobius_fused_encodes, mobius_wide_gathers, with_mobius_fused, with_mobius_wide, +}; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// Raw words with an edge value at every seventh slot. +fn words(seed: u64, n: usize) -> Vec { + const EDGES: [u64; 8] = [0, 1, P - 1, P, P + 1, u64::MAX, 1 << 63, 0xFFFF_FFFF]; + let mut z = seed; + (0..n) + .map(|i| { + z = z.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut x = z; + x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + if i % 7 == 0 { + EDGES[(i / 7) % EDGES.len()] + } else { + x ^ (x >> 31) + } + }) + .collect() +} + +/// The field values of raw words, in place: every word in `[0, p)`. +fn canonical(mut v: Vec) -> Vec { + for x in &mut v { + if *x >= P { + *x -= P; + } + } + v +} + +/// Whether the fused encoding applies to a `2^l` stack at spacing `2^lb`: the +/// NTT's first pass holds `min(12, l + lb) − lb` low levels, and at least one +/// strided level must be left (`whir::mobius_fused_levels`). +fn fuses(l: usize, lb: usize) -> bool { + (l + lb).min(12) - lb < l +} + +/// A stack of `2^l` words as parts `(column, offset)`: a half, an eighth, a +/// ragged run at an odd offset, a 5-word stub near the end, gaps between them. +fn layout(l: usize) -> Vec<(Vec, usize)> { + let n = 1usize << l; + let ragged = 1000.min(n / 4 - 32); + let parts = vec![ + (words(21, n / 2), 0), + (words(22, n / 8), n / 2), + (words(23, ragged), n / 2 + n / 8 + 17), + (words(24, 5), n - 9), + ]; + let mut end = 0; + for (k, (c, o)) in parts.iter().enumerate() { + assert!( + *o >= end && o + c.len() <= n, + "2^{l}: the test's part {k} ({} words at {o}) overlaps or overruns the stack", + c.len() + ); + end = o + c.len(); + } + parts +} + +/// An encode's codeword read back, a named failure if either step fails. +fn read(what: &str, cw: math_cuda::Result) -> Vec { + cw.unwrap_or_else(|e| panic!("{what}: the encode failed: {e:?}")) + .to_host() + .unwrap_or_else(|e| panic!("{what}: the read-back failed: {e:?}")) +} + +/// Run `f` under one arm: the fused encoding on or off, the wide NTT on or +/// off, the wide Möbius transform off. Returns its result and how many +/// encodings fused and gathered. +fn arm(fused: bool, ntt: bool, f: impl FnOnce() -> R) -> (R, u64, u64) { + with_mobius_wide(false, || { + with_mobius_fused(fused, || { + with_ntt_wide(ntt, || { + let (f0, g0) = (mobius_fused_encodes(), mobius_wide_gathers()); + let r = f(); + (r, mobius_fused_encodes() - f0, mobius_wide_gathers() - g0) + }) + }) + }) +} + +/// ★ Every source a commit takes, at stacks of 2^11 (the first pass holds them +/// whole: nothing fuses) to 2^22, under both NTT plans; spacings 4 and 16 on a +/// host source; and a committed stack's root at 2^20 and 2^28: the fused +/// encoding's codeword is today's in field values, it fused exactly where it +/// applies, and the device sources gathered. Every comparison prints its +/// `WHIR MOBIUS FUSED … PARITY` line before it asserts, so the box read-out +/// counts the compares that ran. +#[test] +fn the_fused_mobius_encodes_todays_codeword() { + let be = math_cuda::device::backend().expect("a GPU"); + for l in [11usize, 12, 13, 19, 20, 22] { + let parts = layout(l); + let host: Vec<(&[u64], usize)> = parts.iter().map(|(c, o)| (c.as_slice(), *o)).collect(); + let (today, ..) = arm(false, false, || { + canonical(read( + "today's host encode", + encode_codeword_parts(&host, l, 1, false), + )) + }); + let cols: Vec<&[u64]> = parts.iter().map(|(c, _)| c.as_slice()).collect(); + let store = DeviceColumns::upload(&cols).expect("room for the store"); + let at: Vec<(usize, usize)> = parts + .iter() + .enumerate() + .map(|(k, (_, o))| (k, *o)) + .collect(); + let stream = be.next_stream(); + let mut flat = Vec::new(); + let mut slices = Vec::new(); + for (c, o) in parts.iter().rev() { + slices.push((flat.len(), c.len(), *o)); + flat.extend_from_slice(c); + } + let buffer = stream.clone_htod(&flat).expect("the slices' buffer"); + stream.synchronize().expect("the slices' upload"); + let applies = fuses(l, 1); + for ntt in [false, true] { + let plan = if ntt { "wide" } else { "today" }; + for source in ["host", "resident", "slices"] { + let (cw, fused, gathered) = arm(true, ntt, || { + let what = format!("the fused {source} encode"); + canonical(match source { + "host" => read(&what, encode_codeword_parts(&host, l, 1, false)), + "resident" => { + read(&what, encode_codeword_resident(&store, &at, l, 1, false)) + } + _ => read(&what, encode_codeword_slices(&buffer, &slices, l, 1, false)), + }) + }); + let equal = cw == today; + let want_gather = u64::from(applies && source != "host"); + println!( + "WHIR MOBIUS FUSED SMALL PARITY: 2^{l} · {source} · ntt {plan} · equal={equal} applies={applies} fused={fused} gathered={gathered} want_gathered={want_gather}" + ); + assert_eq!( + fused, + u64::from(applies), + "2^{l} {source}: fused where it does not apply, or not where it does" + ); + assert_eq!( + gathered, want_gather, + "2^{l} {source}: the gather count is not the source's" + ); + assert!( + equal, + "2^{l} {source} ntt {plan}: the fused codeword differs" + ); + } + } + } + + // Other spacings: the first pass then holds 12 − lb levels. + for (l, lb) in [(16usize, 2usize), (15, 4)] { + let evals = words(30 + lb as u64, 1 << l); + let (today, ..) = arm(false, false, || { + canonical(read( + "today's encode", + encode_codeword_parts(&[(&evals, 0)], l, lb, false), + )) + }); + let (cw, fused, _) = arm(true, true, || { + canonical(read( + "the fused encode", + encode_codeword_parts(&[(&evals, 0)], l, lb, false), + )) + }); + let equal = cw == today; + println!( + "WHIR MOBIUS FUSED SPACING PARITY: 2^{l} at spacing 2^{lb} · equal={equal} fused={fused}" + ); + assert_eq!(fused, 1, "2^{l} at 2^{lb}: not fused"); + assert!(equal, "2^{l} at 2^{lb}: the fused codeword differs"); + } + + // A whole stack, committed: the same field values give the same root. + let commit = |what: &str, fused: bool, evals: &[u64]| { + arm(fused, true, || { + commit_codeword(evals, 1, 6, true, math_cuda::DeviceHash::Poseidon1) + .unwrap_or_else(|e| panic!("{what}: the commit failed: {e:?}")) + }) + }; + let mut kept = None; + for l in [19usize, 27] { + let evals = words(80 + l as u64, 1 << l); + let ((a, ra), ..) = commit("today's whole commit", false, &evals); + let ((b, rb), fused, _) = commit("the fused whole commit", true, &evals); + let (wa, wb) = ( + canonical(a.to_host().expect("read back")), + canonical(b.to_host().expect("read back")), + ); + println!( + "WHIR MOBIUS FUSED ROOT PARITY: 2^{} · equal={} root_equal={} fused={fused}", + l + 1, + wa == wb, + ra == rb + ); + assert_eq!(fused, 1, "2^{l}: the commit did not fuse"); + assert_eq!(wa, wb, "2^{l}: a whole stack's codeword differs"); + assert_eq!(ra, rb, "2^{l}: a whole stack's root differs"); + if l == 19 { + kept = Some((evals, wa, ra)); + } + } + + // The comparison can fail: one stack word moved moves the codeword and + // the root. + let (evals, wa, ra) = kept.expect("the 2^19 commit"); + let mut moved = evals.clone(); + moved[54321] ^= 1; + let ((c, rc), ..) = commit("the fused moved commit", true, &moved); + let differs = canonical(c.to_host().expect("read back")) != wa; + println!( + "WHIR MOBIUS FUSED MOVED WORD: differs={differs} root_differs={}", + rc != ra + ); + assert!(differs, "a moved word must move the codeword"); + assert_ne!(rc, ra, "a moved word must move the root"); +} + +/// The production shape's encode, timed: a resident stack of 2^27 (64 columns +/// of 2^21) to its 2^28 codeword under five arms — today (T), the wide Möbius +/// (M), the fused encoding (MD), the wide Möbius and NTT (B), the fused +/// encoding and the wide NTT (BD) — interleaved (the order rotates a rep), +/// median of 9 after 2 warm-ups. Each arm's codeword is first checked against +/// today's in field values. Prints one `WHIR MOBIUS FUSED BENCH` line. +#[test] +#[ignore = "box tier: a measurement"] +fn the_fused_mobius_encode_at_the_production_shape() { + let l = 27; + let label = "64 columns of 2^21"; + // (name, wide Möbius, fused, wide NTT) + const ARMS: [(&str, bool, bool, bool); 5] = [ + ("today", false, false, false), + ("mobius", true, false, false), + ("fused", false, true, false), + ("both", true, false, true), + ("fused_ntt", false, true, true), + ]; + let cols: Vec> = (0..64).map(|k| words(k + 1, 1 << 21)).collect(); + let refs: Vec<&[u64]> = cols.iter().map(|c| c.as_slice()).collect(); + let store = DeviceColumns::upload(&refs).expect("room for the store"); + let at: Vec<(usize, usize)> = (0..64).map(|k| (k, k << 21)).collect(); + let encode = |(_, mobius, fused, ntt): (&str, bool, bool, bool)| { + with_mobius_wide(mobius, || { + with_mobius_fused(fused, || { + with_ntt_wide(ntt, || encode_codeword_resident(&store, &at, l, 1, false)) + }) + }) + }; + + // The production shape's codewords in field values, before any timing. + let today = canonical(read("today's encode", encode(ARMS[0]))); + for &a in &ARMS[1..] { + let equal = canonical(read(a.0, encode(a))) == today; + println!( + "WHIR MOBIUS FUSED PARITY: 2^{l} · {label} · {} · equal={equal}", + a.0 + ); + assert!( + equal, + "{label}: the {} arm's codeword differs at the production shape", + a.0 + ); + } + drop(today); + + let run = |a: (&str, bool, bool, bool)| { + let (g0, f0, w0) = ( + mobius_wide_gathers(), + mobius_fused_encodes(), + wide_codewords_encoded(), + ); + let t = Instant::now(); + let cw = encode(a).expect("a timed encode"); + cw.synchronize().unwrap(); + let ms = t.elapsed().as_secs_f64() * 1e3; + drop(cw); + // Each arm took exactly its own transforms: the times are the levers'. + assert_eq!( + mobius_wide_gathers() - g0, + u64::from(a.1 || a.2), + "{}: the gather count is not the arm's", + a.0 + ); + assert_eq!( + mobius_fused_encodes() - f0, + u64::from(a.2), + "{}: the fused count is not the arm's", + a.0 + ); + assert_eq!( + wide_codewords_encoded() - w0, + u64::from(a.3), + "{}: the wide-NTT count is not the arm's", + a.0 + ); + ms + }; + let mut times: [Vec; 5] = Default::default(); + for rep in 0..11 { + for i in 0..ARMS.len() { + let a = (rep + i) % ARMS.len(); + let ms = run(ARMS[a]); + if rep >= 2 { + times[a].push(ms); + } + } + } + let med: Vec = times + .iter_mut() + .map(|t| { + t.sort_by(f64::total_cmp); + t[t.len() / 2] + }) + .collect(); + println!( + "WHIR MOBIUS FUSED SPREAD: {}", + ARMS.iter() + .zip(×) + .map(|((arm, ..), t)| format!("{arm} {:.3}–{:.3} ms", t[0], t[t.len() - 1])) + .collect::>() + .join(" · ") + ); + let (today, mobius, fused, both, fused_ntt) = (med[0], med[1], med[2], med[3], med[4]); + println!( + "WHIR MOBIUS FUSED BENCH: resident 2^{l} → 2^{} · {label} · today {today:.3} ms · mobius {mobius:.3} ms · fused {fused:.3} ms · both {both:.3} ms · fused_ntt {fused_ntt:.3} ms · R_DM {:.3} · R_D {:.3}", + l + 1, + fused / mobius, + fused_ntt / both + ); +} diff --git a/crypto/math-cuda/tests/whir_mobius_wide.rs b/crypto/math-cuda/tests/whir_mobius_wide.rs new file mode 100644 index 000000000..4612d0175 --- /dev/null +++ b/crypto/math-cuda/tests/whir_mobius_wide.rs @@ -0,0 +1,310 @@ +//! The WHIR encoding's wide Möbius transform +//! (`LAMBDA_VM_WHIR_MOBIUS_WIDE`: three passes instead of five, and the +//! stack gathered from its parts instead of copied into a zeroed buffer) +//! encodes the five-pass transform's codeword, word for word, from every +//! source a commit takes. The encoding's other two knobs (the fused low +//! levels, the wide NTT), on by default, are held off ([`alone`]), so the two +//! arms differ in this transform only. +//! +//! Both transforms run in one process (`whir::with_mobius_wide`) over stacks of +//! raw words — edge values planted, parts of mixed sizes at unaligned offsets +//! with gaps of padding — and the codewords are compared whole. Needs a GPU +//! (the box): +//! +//! cargo test -p math-cuda --release --test whir_mobius_wide +//! cargo test -p math-cuda --release --test whir_mobius_wide -- --ignored --nocapture + +use std::time::Instant; + +use math_cuda::columns::DeviceColumns; +use math_cuda::lde_cm::with_ntt_wide; +use math_cuda::whir::{ + commit_codeword, encode_codeword_parts, encode_codeword_resident, encode_codeword_slices, + mobius_wide_gathers, with_mobius_fused, with_mobius_wide, +}; + +/// Runs `f` with the fused low levels and the wide NTT off on this thread, so +/// each arm's encoding differs from the other's in the Möbius transform only. +fn alone(f: impl FnOnce() -> R) -> R { + with_mobius_fused(false, || with_ntt_wide(false, f)) +} + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// Raw words with an edge value at every seventh slot. +fn words(seed: u64, n: usize) -> Vec { + const EDGES: [u64; 8] = [0, 1, P - 1, P, P + 1, u64::MAX, 1 << 63, 0xFFFF_FFFF]; + let mut z = seed; + (0..n) + .map(|i| { + z = z.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut x = z; + x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + if i % 7 == 0 { + EDGES[(i / 7) % EDGES.len()] + } else { + x ^ (x >> 31) + } + }) + .collect() +} + +/// A stack of `2^l` words as parts `(column, offset)`: a half, an eighth, +/// a ragged run (1,000 words, or less on a small stack) at an odd offset, a +/// 5-word stub near the end, and gaps of padding between them. Checked to fit: +/// a layout that overruns is the test's own fault, and it must say so rather +/// than panic inside a copy. +fn layout(l: usize) -> Vec<(Vec, usize)> { + let n = 1usize << l; + let ragged = 1000.min(n / 4 - 32); + let parts = vec![ + (words(1, n / 2), 0), + (words(2, n / 8), n / 2), + (words(3, ragged), n / 2 + n / 8 + 17), + (words(4, 5), n - 9), + ]; + let mut end = 0; + for (k, (c, o)) in parts.iter().enumerate() { + assert!( + *o >= end && o + c.len() <= n, + "2^{l}: the test's part {k} ({} words at {o}) overlaps or overruns the stack", + c.len() + ); + end = o + c.len(); + } + parts +} + +/// An encode's codeword read back, a named failure if either step fails. +fn read(what: &str, cw: math_cuda::Result) -> Vec { + cw.unwrap_or_else(|e| panic!("{what}: the encode failed: {e:?}")) + .to_host() + .unwrap_or_else(|e| panic!("{what}: the read-back failed: {e:?}")) +} + +/// ★ Every source a commit takes, at stacks of 2^11 (one contiguous pass) to +/// 2^22 (both strided passes, one short): the wide transform's codeword is +/// today's, word for word, and the device sources took the gather. Every +/// comparison prints its `WHIR MOBIUS WIDE … PARITY` line before it asserts, +/// so the box read-out counts the compares that ran. +#[test] +fn the_wide_mobius_encodes_todays_codeword() { + alone(|| { + for l in [11usize, 12, 13, 19, 20, 22] { + let parts = layout(l); + // A host source: the wide transform, no gather. + let host: Vec<(&[u64], usize)> = + parts.iter().map(|(c, o)| (c.as_slice(), *o)).collect(); + let today = with_mobius_wide(false, || { + read( + "today's host encode", + encode_codeword_parts(&host, l, 1, false), + ) + }); + let wide = with_mobius_wide(true, || { + read( + "the wide host encode", + encode_codeword_parts(&host, l, 1, false), + ) + }); + println!( + "WHIR MOBIUS WIDE SMALL PARITY: 2^{l} · host · equal={}", + today == wide + ); + assert_eq!( + today, wide, + "2^{l}: host parts differ under the wide transform" + ); + + // A resident store: gathered. + let cols: Vec<&[u64]> = parts.iter().map(|(c, _)| c.as_slice()).collect(); + let store = DeviceColumns::upload(&cols).expect("room for the store"); + let at: Vec<(usize, usize)> = parts + .iter() + .enumerate() + .map(|(k, (_, o))| (k, *o)) + .collect(); + let before = mobius_wide_gathers(); + let wide = with_mobius_wide(true, || { + read( + "the wide resident encode", + encode_codeword_resident(&store, &at, l, 1, false), + ) + }); + let gathered = mobius_wide_gathers() == before + 1; + let today_r = with_mobius_wide(false, || { + read( + "today's resident encode", + encode_codeword_resident(&store, &at, l, 1, false), + ) + }); + println!( + "WHIR MOBIUS WIDE SMALL PARITY: 2^{l} · resident · equal={} gathered={gathered} today_equal={}", + wide == today, + today_r == today + ); + assert!(gathered, "2^{l}: the resident source was not gathered"); + assert_eq!( + today_r, today, + "2^{l}: the resident and host sources differ today" + ); + assert_eq!(wide, today, "2^{l}: the gathered resident source differs"); + + // A buffer's slices, out of order in the list: gathered. + let be = math_cuda::device::backend().expect("a GPU"); + let stream = be.next_stream(); + let mut flat = Vec::new(); + let mut slices = Vec::new(); + for (c, o) in parts.iter().rev() { + slices.push((flat.len(), c.len(), *o)); + flat.extend_from_slice(c); + } + let buffer = stream.clone_htod(&flat).expect("the slices' buffer"); + stream.synchronize().expect("the slices' upload"); + let before = mobius_wide_gathers(); + let wide = with_mobius_wide(true, || { + read( + "the wide sliced encode", + encode_codeword_slices(&buffer, &slices, l, 1, false), + ) + }); + let gathered = mobius_wide_gathers() == before + 1; + println!( + "WHIR MOBIUS WIDE SMALL PARITY: 2^{l} · slices · equal={} gathered={gathered}", + wide == today + ); + assert!(gathered, "2^{l}: the sliced source was not gathered"); + assert_eq!(wide, today, "2^{l}: the gathered slices differ"); + } + + // A whole stack, committed: the same codeword gives the same root. + let l = 16; + let evals = words(9, 1 << l); + let commit = |what: &str, wide: bool, evals: &[u64]| { + with_mobius_wide(wide, || { + commit_codeword(evals, 1, 6, true, math_cuda::DeviceHash::Poseidon1) + .unwrap_or_else(|e| panic!("{what}: the commit failed: {e:?}")) + }) + }; + let (a, ra) = commit("today's whole commit", false, &evals); + let (b, rb) = commit("the wide whole commit", true, &evals); + let (wa, wb) = ( + a.to_host().expect("read back"), + b.to_host().expect("read back"), + ); + println!( + "WHIR MOBIUS WIDE WHOLE PARITY: 2^{l} · equal={} root_equal={}", + wa == wb, + ra == rb + ); + assert_eq!(wa, wb, "a whole stack's codeword differs"); + assert_eq!(ra, rb, "a whole stack's root differs"); + + // The comparison can fail: one stack word moved moves the codeword. + let mut moved = evals.clone(); + moved[12345] ^= 1; + let (c, _) = commit("the wide moved commit", true, &moved); + let differs = c.to_host().expect("read back") != wa; + println!("WHIR MOBIUS WIDE MOVED WORD: differs={differs}"); + assert!(differs, "a moved word must move the codeword"); + }) +} + +/// The production shape's encode, timed: a resident stack of 2^27 (64 columns +/// of 2^21, as a group's tables stack) to its 2^28 codeword, today's transform +/// against the wide one, interleaved, median of 9 after 2 warm-ups. Prints a +/// `WHIR MOBIUS WIDE BENCH` line a shape. +#[test] +#[ignore = "box tier: a measurement"] +fn the_wide_mobius_encode_at_the_production_shape() { + alone(|| { + let l = 27; + let shapes: [(&str, Vec); 2] = [ + ("64 columns of 2^21", vec![1 << 21; 64]), + ("mixed: 48 of 2^21, 24 of 2^20, 40 of 2^16 and gaps", { + let mut v = vec![1 << 21; 48]; + v.extend(vec![1 << 20; 24]); + v.extend(vec![1 << 16; 40]); + v + }), + ]; + for (label, lens) in shapes { + let cols: Vec> = lens + .iter() + .enumerate() + .map(|(k, &n)| words(k as u64 + 1, n)) + .collect(); + let refs: Vec<&[u64]> = cols.iter().map(|c| c.as_slice()).collect(); + let store = DeviceColumns::upload(&refs).expect("room for the store"); + // Columns at their running offsets, a 4,096-word gap after each 2^16 one. + let mut at = Vec::new(); + let mut off = 0usize; + for (k, &n) in lens.iter().enumerate() { + at.push((k, off)); + off += n + if n == 1 << 16 { 4096 } else { 0 }; + } + assert!(off <= 1 << l, "{label}: the layout overruns the stack"); + // The production shape's codewords, word for word, before any timing. + let today = with_mobius_wide(false, || { + encode_codeword_resident(&store, &at, l, 1, false) + .unwrap() + .to_host() + .unwrap() + }); + let wide = with_mobius_wide(true, || { + encode_codeword_resident(&store, &at, l, 1, false) + .unwrap() + .to_host() + .unwrap() + }); + let equal = today == wide; + println!("WHIR MOBIUS WIDE PARITY: 2^{l} · {label} · equal={equal}"); + assert!( + equal, + "{label}: the wide transform's codeword differs at the production shape" + ); + drop((today, wide)); + let run = |wide: bool| { + with_mobius_wide(wide, || { + let t = Instant::now(); + let before = mobius_wide_gathers(); + let cw = + encode_codeword_resident(&store, &at, l, 1, false).expect("a timed encode"); + cw.synchronize().unwrap(); + let ms = t.elapsed().as_secs_f64() * 1e3; + drop(cw); + // The wide arm took the gather and today's did not: the times are the lever's. + assert_eq!( + mobius_wide_gathers() - before, + u64::from(wide), + "{label}: the gather count is not the arm's" + ); + ms + }) + }; + let (mut a, mut b) = (Vec::new(), Vec::new()); + for rep in 0..11 { + let (x, y) = if rep % 2 == 0 { + (run(false), run(true)) + } else { + let y = run(true); + (run(false), y) + }; + if rep >= 2 { + a.push(x); + b.push(y); + } + } + a.sort_by(f64::total_cmp); + b.sort_by(f64::total_cmp); + let (ma, mb) = (a[a.len() / 2], b[b.len() / 2]); + println!( + "WHIR MOBIUS WIDE BENCH: resident 2^{l} → 2^{} · {label} · today {ma:.3} ms · wide {mb:.3} ms · R {:.3} · saved {:.3} ms", + l + 1, + mb / ma, + ma - mb + ); + } + }) +} diff --git a/crypto/math-cuda/tests/whir_ntt_wide.rs b/crypto/math-cuda/tests/whir_ntt_wide.rs new file mode 100644 index 000000000..26316725f --- /dev/null +++ b/crypto/math-cuda/tests/whir_ntt_wide.rs @@ -0,0 +1,357 @@ +//! The WHIR encoding's wide NTT (`LAMBDA_VM_WHIR_NTT_WIDE`: passes of up to +//! 10 levels, three instead of four at the production 2^28) encodes the +//! 8-level plan's codeword: the same field values — word for word once canonicalised, since a +//! different pass split multiplies by different twist factors and may leave a +//! value in its other representation — and the same tree root. +//! +//! Both plans run in one process (`lde_cm::with_ntt_wide`) over stacks of raw +//! words with edge values planted, at every codeword size where the plans +//! differ from 2^9 to 2^29 and at two where they agree; the legacy per-level +//! encoding (`lde_cm::with_engine(false, ..)`), which shares no pass code with +//! either plan, is a second reference. The encoding's other two knobs (the +//! wide Möbius transform, the fused low levels), on by default, are held off +//! ([`alone`]) except where a comparison sets them. Needs a GPU (the box): +//! +//! cargo test -p math-cuda --release --test whir_ntt_wide -- --test-threads=1 +//! cargo test -p math-cuda --release --test whir_ntt_wide -- --ignored --nocapture --test-threads=1 + +use std::time::Instant; + +use math_cuda::columns::DeviceColumns; +use math_cuda::lde_cm::{wide_codewords_encoded, with_engine, with_ntt_wide}; +use math_cuda::whir::{ + commit_codeword, encode_codeword_parts, encode_codeword_resident, mobius_wide_gathers, + with_mobius_fused, with_mobius_wide, +}; + +/// Runs `f` with the wide Möbius transform and the fused low levels off on +/// this thread, so the plans' arms differ in the NTT only; a comparison that +/// sets the Möbius transform itself overrides it inside. +fn alone(f: impl FnOnce() -> R) -> R { + with_mobius_fused(false, || with_mobius_wide(false, f)) +} + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// Raw words with an edge value at every seventh slot. +fn words(seed: u64, n: usize) -> Vec { + const EDGES: [u64; 8] = [0, 1, P - 1, P, P + 1, u64::MAX, 1 << 63, 0xFFFF_FFFF]; + let mut z = seed; + (0..n) + .map(|i| { + z = z.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut x = z; + x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + if i % 7 == 0 { + EDGES[(i / 7) % EDGES.len()] + } else { + x ^ (x >> 31) + } + }) + .collect() +} + +/// The field values of raw words, in place: every word in `[0, p)`. +fn canonical(mut v: Vec) -> Vec { + for x in &mut v { + if *x >= P { + *x -= P; + } + } + v +} + +/// Whether the wide plan of a `2^log_n` transform has a pass of more than 8 +/// levels (`lde_cm::plan_wide`: `ceil(l / 10)` passes, as even as possible). +/// Where it does not, it is today's plan. +fn wide_kernels_run(log_n: usize) -> bool { + let p = log_n.div_ceil(10); + log_n.div_ceil(p) > 8 +} + +/// A stack of `2^l` words as parts `(column, offset)`: a half, an eighth, a +/// ragged run at an odd offset, a 5-word stub near the end, gaps between them. +fn layout(l: usize) -> Vec<(Vec, usize)> { + let n = 1usize << l; + let ragged = 1000.min(n / 4 - 32); + let parts = vec![ + (words(11, n / 2), 0), + (words(12, n / 8), n / 2), + (words(13, ragged), n / 2 + n / 8 + 17), + (words(14, 5), n - 9), + ]; + let mut end = 0; + for (k, (c, o)) in parts.iter().enumerate() { + assert!( + *o >= end && o + c.len() <= n, + "2^{l}: the test's part {k} ({} words at {o}) overlaps or overruns the stack", + c.len() + ); + end = o + c.len(); + } + parts +} + +/// An encode's codeword read back, a named failure if either step fails. +fn read(what: &str, cw: math_cuda::Result) -> Vec { + cw.unwrap_or_else(|e| panic!("{what}: the encode failed: {e:?}")) + .to_host() + .unwrap_or_else(|e| panic!("{what}: the read-back failed: {e:?}")) +} + +/// Encode `evals` (one whole stack of `2^l`) at spacing `2^lb` under one plan: +/// the raw codeword, and how many encodings ran a wide pass. +fn encode(what: &str, wide: bool, evals: &[u64], l: usize, lb: usize) -> (Vec, u64) { + with_ntt_wide(wide, || { + let before = wide_codewords_encoded(); + let raw = read(what, encode_codeword_parts(&[(evals, 0)], l, lb, false)); + (raw, wide_codewords_encoded() - before) + }) +} + +/// ★ Every codeword size where the plans differ from 2^9 to 2^29 (one or two +/// wide passes, a short strided one, the production [10, 9, 9] and [10, 10, +/// 9]), spacings 2, 4 and 16, and two sizes where they agree: the wide plan's +/// codeword is today's in field values, its wide kernels ran exactly where the +/// plans differ, and a committed stack's root is today's. Every comparison +/// prints its `WHIR NTT WIDE … PARITY` line before it asserts, so the box +/// read-out counts the compares that ran. +#[test] +fn the_wide_ntt_encodes_todays_codeword() { + alone(|| { + // (log_evals, log_blowup): the codeword is 2^(log_evals + log_blowup). + const SHAPES: [(usize, usize); 15] = [ + (8, 1), + (9, 1), + (15, 1), + (16, 1), + (17, 1), + (18, 1), + (19, 1), + (16, 2), + (15, 4), + (23, 1), + (24, 1), + (25, 1), + (26, 1), + (27, 1), + (27, 2), + ]; + for (l, lb) in SHAPES { + let log_n = l + lb; + let evals = words(l as u64 * 8 + lb as u64, 1 << l); + let (today, today_k) = encode("today's encode", false, &evals, l, lb); + let (wide, wide_k) = encode("the wide encode", true, &evals, l, lb); + let legacy = with_engine(false, || { + read( + "the legacy encode", + encode_codeword_parts(&[(&evals, 0)], l, lb, false), + ) + }); + let expect = u64::from(wide_kernels_run(log_n)); + let raw_equal = today == wide; + let (today, wide) = (canonical(today), canonical(wide)); + let equal = today == wide; + let legacy_equal = canonical(legacy) == wide; + println!( + "WHIR NTT WIDE SMALL PARITY: 2^{log_n} (2^{l} at spacing 2^{lb}) · equal={equal} legacy_equal={legacy_equal} raw_equal={raw_equal} wide_kernels={wide_k} expected={expect} today_kernels={today_k}" + ); + assert_eq!(today_k, 0, "2^{log_n}: today's plan ran a wide pass"); + assert_eq!( + wide_k, expect, + "2^{log_n}: the wide kernels did not run where the plans differ" + ); + assert!( + equal, + "2^{log_n}: the wide plan's codeword differs in field values" + ); + assert!( + legacy_equal, + "2^{log_n}: the wide plan's codeword differs from the legacy encoding" + ); + } + + // A resident stack under both wide transforms (the Möbius gathered, the + // NTT wide) against today's: the two levers together. + for l in [19usize, 24] { + let parts = layout(l); + let cols: Vec<&[u64]> = parts.iter().map(|(c, _)| c.as_slice()).collect(); + let store = DeviceColumns::upload(&cols).expect("room for the store"); + let at: Vec<(usize, usize)> = parts + .iter() + .enumerate() + .map(|(k, (_, o))| (k, *o)) + .collect(); + let today = with_mobius_wide(false, || { + with_ntt_wide(false, || { + read( + "today's resident encode", + encode_codeword_resident(&store, &at, l, 1, false), + ) + }) + }); + let (g0, w0) = (mobius_wide_gathers(), wide_codewords_encoded()); + let both = with_mobius_wide(true, || { + with_ntt_wide(true, || { + read( + "the wide-wide resident encode", + encode_codeword_resident(&store, &at, l, 1, false), + ) + }) + }); + let (gathered, wide_k) = (mobius_wide_gathers() - g0, wide_codewords_encoded() - w0); + let equal = canonical(today) == canonical(both); + println!( + "WHIR NTT WIDE BOTH PARITY: 2^{} · resident · equal={equal} gathered={gathered} wide_kernels={wide_k}", + l + 1 + ); + assert_eq!(gathered, 1, "2^{l}: the resident source was not gathered"); + assert_eq!(wide_k, 1, "2^{l}: the wide kernels did not run"); + assert!(equal, "2^{l}: both wide transforms together differ"); + } + + // A whole stack, committed: the same field values give the same root. + let commit = |what: &str, wide: bool, evals: &[u64]| { + with_ntt_wide(wide, || { + commit_codeword(evals, 1, 6, true, math_cuda::DeviceHash::Poseidon1) + .unwrap_or_else(|e| panic!("{what}: the commit failed: {e:?}")) + }) + }; + let mut kept = None; + for l in [19usize, 27] { + let evals = words(90 + l as u64, 1 << l); + let (a, ra) = commit("today's whole commit", false, &evals); + let w0 = wide_codewords_encoded(); + let (b, rb) = commit("the wide whole commit", true, &evals); + let wide_k = wide_codewords_encoded() - w0; + let (wa, wb) = ( + canonical(a.to_host().expect("read back")), + canonical(b.to_host().expect("read back")), + ); + println!( + "WHIR NTT WIDE ROOT PARITY: 2^{} · equal={} root_equal={} wide_kernels={wide_k}", + l + 1, + wa == wb, + ra == rb + ); + assert_eq!(wide_k, 1, "2^{l}: the wide commit ran no wide pass"); + assert_eq!(wa, wb, "2^{l}: a whole stack's codeword differs"); + assert_eq!(ra, rb, "2^{l}: a whole stack's root differs"); + if l == 19 { + kept = Some((evals, wa, ra)); + } + } + + // The comparison can fail: one stack word moved moves the codeword and + // the root. + let (evals, wa, ra) = kept.expect("the 2^19 commit"); + let mut moved = evals.clone(); + moved[12345] ^= 1; + let (c, rc) = commit("the wide moved commit", true, &moved); + let differs = canonical(c.to_host().expect("read back")) != wa; + println!( + "WHIR NTT WIDE MOVED WORD: differs={differs} root_differs={}", + rc != ra + ); + assert!(differs, "a moved word must move the codeword"); + assert_ne!(rc, ra, "a moved word must move the root"); + }) +} + +/// The production shape's encode, timed: a resident stack of 2^27 (64 columns +/// of 2^21, as a group's tables stack) to its 2^28 codeword under four arms — +/// today, the wide Möbius alone, the wide NTT alone, both — interleaved (the +/// order rotates a rep), median of 9 after 2 warm-ups. Each arm's codeword is +/// first checked against today's in field values. Prints one `WHIR NTT WIDE +/// BENCH` line. +#[test] +#[ignore = "box tier: a measurement"] +fn the_wide_ntt_encode_at_the_production_shape() { + alone(|| { + let l = 27; + let label = "64 columns of 2^21"; + const ARMS: [(&str, bool, bool); 4] = [ + ("today", false, false), + ("mobius", true, false), + ("ntt", false, true), + ("both", true, true), + ]; + let cols: Vec> = (0..64).map(|k| words(k + 1, 1 << 21)).collect(); + let refs: Vec<&[u64]> = cols.iter().map(|c| c.as_slice()).collect(); + let store = DeviceColumns::upload(&refs).expect("room for the store"); + let at: Vec<(usize, usize)> = (0..64).map(|k| (k, k << 21)).collect(); + let encode = |mobius: bool, ntt: bool| { + with_mobius_wide(mobius, || { + with_ntt_wide(ntt, || encode_codeword_resident(&store, &at, l, 1, false)) + }) + }; + + // The production shape's codewords in field values, before any timing. + let today = canonical(read("today's encode", encode(false, false))); + for &(arm, mobius, ntt) in &ARMS[1..] { + let equal = canonical(read(arm, encode(mobius, ntt))) == today; + println!("WHIR NTT WIDE PARITY: 2^{l} · {label} · {arm} · equal={equal}"); + assert!( + equal, + "{label}: the {arm} arm's codeword differs at the production shape" + ); + } + drop(today); + + let run = |mobius: bool, ntt: bool| { + let (g0, w0) = (mobius_wide_gathers(), wide_codewords_encoded()); + let t = Instant::now(); + let cw = encode(mobius, ntt).expect("a timed encode"); + cw.synchronize().unwrap(); + let ms = t.elapsed().as_secs_f64() * 1e3; + drop(cw); + // Each arm took exactly its own transforms: the times are the levers'. + assert_eq!( + mobius_wide_gathers() - g0, + u64::from(mobius), + "the gather count is not the arm's" + ); + assert_eq!( + wide_codewords_encoded() - w0, + u64::from(ntt), + "the wide-NTT count is not the arm's" + ); + ms + }; + let mut times: [Vec; 4] = Default::default(); + for rep in 0..11 { + for i in 0..ARMS.len() { + let a = (rep + i) % ARMS.len(); + let ms = run(ARMS[a].1, ARMS[a].2); + if rep >= 2 { + times[a].push(ms); + } + } + } + let med: Vec = times + .iter_mut() + .map(|t| { + t.sort_by(f64::total_cmp); + t[t.len() / 2] + }) + .collect(); + let (today, mobius, ntt, both) = (med[0], med[1], med[2], med[3]); + println!( + "WHIR NTT WIDE SPREAD: {}", + ARMS.iter() + .zip(×) + .map(|((arm, ..), t)| format!("{arm} {:.3}–{:.3} ms", t[0], t[t.len() - 1])) + .collect::>() + .join(" · ") + ); + println!( + "WHIR NTT WIDE BENCH: resident 2^{l} → 2^{} · {label} · today {today:.3} ms · mobius {mobius:.3} ms · ntt {ntt:.3} ms · both {both:.3} ms · R_N {:.3} · R_BM {:.3} · saved {:.3} ms alone · {:.3} ms on the wide Möbius", + l + 1, + ntt / today, + both / mobius, + today - ntt, + mobius - both + ); + }) +} diff --git a/crypto/math-cuda/tests/whir_tree_cache.rs b/crypto/math-cuda/tests/whir_tree_cache.rs new file mode 100644 index 000000000..af29f5b44 --- /dev/null +++ b/crypto/math-cuda/tests/whir_tree_cache.rs @@ -0,0 +1,1278 @@ +//! ★★ H4's result of record, and what replaced it — a commitment keeps its tree +//! INSIDE ITS PROMISE: the whole tree by default, its leaf layer under +//! `LFM_WHIR_WHOLE_TREES=0`, and nothing of a tree outside the room it grew. +//! +//! ⛔ H4's finding stands and is not edited away below: keeping the whole node +//! array LOST, measured on the card at about +15 s, because its trees sat +//! OUTSIDE any promise — one per commitment IN THE GROUP, ten of them put the +//! device at 96%, and commits fell back to the host. What changed is WHERE the +//! bytes sit. The leaf layer came first (half the bytes, most of the saving: a +//! leaf absorbs a whole `2^k` coset while an inner node absorbs two digests), +//! promised through the codeword's room and evictable; the whole tree came +//! back on the same terms (job 248, −1.00 s). These tests pin BOTH sides of the +//! invariant: what is kept is held AND promised, and nothing of a tree is held +//! outside the promise. +//! +//! Needs a GPU: +//! +//! ```text +//! cargo test -p math-cuda --release --test whir_tree_cache -- --nocapture +//! ``` +//! +//! # What this is about +//! +//! `commit()` builds the tree and takes the root; `paths()` needs the same +//! tree to read a kilobyte per query out of it. Every commitment on this path +//! is opened, so without retention every one pays for two leaf-hash passes over +//! its codeword — ~25 s of RPX device hashing on a real block, about half of it +//! that second pass. H4 cached the first tree to remove the second pass. It was +//! measured on the card and it LOST, ~+15 s in both hashes. +//! +//! # Why H4's tree could not work, and what the tests pin instead +//! +//! Not because the cache missed — it returned exactly the hashing it promised. +//! Because the retention is one tree per commitment IN THE GROUP, not one tree. +//! `StackedCommitment::commit` builds every chain's commitment before it +//! returns, since all the roots enter the transcript before any query index is +//! drawn, and the openings follow one chain at a time. So the last chain's tree +//! lives from its commit to the end of the proof, and all N exist before the +//! first opening. Held outside the ledger, ten chains at half a gigabyte put +//! the card at 96%, after which device allocations fail, commits fall back to +//! the host, and the host grows ~1.5 GiB per fallen-back chain. +//! +//! Held INSIDE the ledger — grown into each codeword's room, and given back to +//! any request that needs the bytes — the same window costs nothing but the +//! re-hash of whatever a request took. That is the trade these tests pin. +//! +//! # ⚠ Why the counts are PER CODEWORD +//! +//! An earlier run of this file failed two tests for a reason that was not the +//! code under test: `leaf_hash_calls()` is process-wide, the tests share one +//! binary, and cargo runs them in parallel — so one test read 5 where it +//! expected 1 purely because its neighbours were committing at the same time. +//! An assertion on a global counter is an assertion about every test in the +//! binary. The counting assertions use `DeviceCodeword::tree_builds()`; the +//! process-wide counter keeps one test of its own, and that one takes the lock. +//! +//! # ⚠ Why the memory guard asks the DRIVER +//! +//! `Backend::reserved_bytes()` counts what callers promised, so it is silent +//! about device memory allocated without a reservation — and it reads baseline +//! while the card fills, which is how the H4 arm's retention stayed invisible +//! to a unit test that passed. So the guard samples `free_vram_bytes()` across +//! four live, unopened commitments and asserts two things: what they took is +//! their codewords and what each keeps of its tree, and that equals what they +//! PROMISED, give or take the pool. It fails if anything of a tree is ever held +//! outside the promise, wherever the holding is written. + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use math_cuda::DeviceHash; +use math_cuda::whir::{leaf_hash_calls, retention_report, tree_builds}; +use multilinear::mle::Mle; +use multilinear::whir::{self, Domain}; +use multilinear::whir_commit::{CodewordCommitment, verify_opening}; +use multilinear::whir_hash::{DeviceHashKey, KeccakWhir, RpxWhir, WhirHash}; +use std::sync::Mutex; + +type FE = FieldElement; + +/// ★ Taken by EVERY test in this file, because every one of them commits, and a +/// commit moves two process-wide quantities: the leaf-hash counter and the +/// device reservation total. +/// +/// The per-codeword counts added earlier make the *counting* assertions immune +/// to the scheduler, but one proposition is irreducibly global — "dropping a +/// codeword gives its promised bytes BACK" is a statement about +/// `Backend::reserved_bytes()`, and there is no per-codeword handle left to ask +/// once the codeword is gone. So these tests take turns. +/// +/// That costs nothing real: they share one card and serialise on it regardless. +/// What the lock buys over `--test-threads=1` is that the property holds however +/// the suite is invoked, rather than only when someone remembers the flag. +/// +/// Poisoning is ignored so one failure does not cascade into unrelated tests — +/// a lesson this branch learned once already, in `hash_metrics_tests`. +static DEVICE_GLOBALS: Mutex<()> = Mutex::new(()); + +fn exclusive() -> std::sync::MutexGuard<'static, ()> { + DEVICE_GLOBALS.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Mirror of `DeviceHashKey::into_math_cuda`, which is cuda-gated on +/// `multilinear` and so unreachable from this crate's dev-dependency. The +/// production bridge is asserted bijective at compile time inside +/// `multilinear`; a swapped mirror here would fail +/// [`the_two_hashes_build_different_trees`] below. +fn key() -> DeviceHash { + match H::DEVICE { + DeviceHashKey::Keccak256 => DeviceHash::Keccak256, + DeviceHashKey::Rpx256 => DeviceHash::Rpx256, + DeviceHashKey::Poseidon1 => DeviceHash::Poseidon1, + } +} + +/// A polynomial with no structure a kernel could accidentally satisfy. +fn poly(num_vars: usize) -> Mle { + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(6364136223846793005).wrapping_add(11) >> 11)) + .collect(); + Mle::new(evals).expect("power of two") +} + +/// Commit on the device at `log_blowup = 2`, above `COMMIT_THRESHOLD`, so the +/// device path is the one taken. +fn commit_on_device( + num_vars: usize, + log_folding: usize, + hash: DeviceHash, +) -> (math_cuda::whir::DeviceCodeword, [u8; 32]) { + let f = poly(num_vars); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit (needs a GPU)") +} + +/// `LFM_WHIR_WHOLE_TREES` forced for one test — whole trees or leaf layers — +/// and the environment's setting restored when this drops, on a panic too. +/// Every test that asserts a count or a byte figure that depends on what is +/// kept runs both ways under it, rather than at whatever the default is. +struct WholeTrees; + +impl WholeTrees { + fn force(on: bool) -> Self { + math_cuda::whir::force_whole_trees(Some(on)); + Self + } +} + +impl Drop for WholeTrees { + fn drop(&mut self) { + math_cuda::whir::force_whole_trees(None); + } +} + +/// What a codeword over `2^num_vars` values at `log_blowup = 2` keeps of its +/// tree at `log_folding`: the whole node array, or its leaf layer. +fn kept_bytes(num_vars: usize, log_folding: usize, whole: bool) -> u64 { + let leaves = ((1u64 << num_vars) << 2) >> log_folding; + if whole { + (2 * leaves - 1) * 32 + } else { + leaves * 32 + } +} + +/// ★★ (1) THE COUNT. One leaf-hash pass per codeword while what it keeps is +/// held, and a tree built at an opening only when no whole tree is kept. +/// +/// This is the cost H4 tried to remove, asserted as integers in both retention +/// modes. With leaf layers (`LFM_WHIR_WHOLE_TREES=0`) each opening builds its +/// own tree on the kept layer: `tree_builds` climbs, `leaf_passes` stays at 1. +/// With whole trees (the default) each opening is served the kept tree: both +/// stay at 1. A 2 in `leaf_passes` is the retention not working in either mode; +/// a build at an opening under whole trees is an opening that stopped reading +/// the tree it kept; and the kept bytes say which object the counts are about. +#[test] +fn a_commitment_hashes_its_leaves_once_per_tree_it_builds() { + let _exclusive = exclusive(); + for whole in [false, true] { + let _mode = WholeTrees::force(whole); + // Trees an opening builds: one with leaf layers, none with whole trees. + let per_open = u64::from(!whole); + for (name, hash) in [("keccak", key::()), ("rpx", key::())] { + let (codeword, _root) = commit_on_device(14, 4, hash); + assert_eq!( + codeword.tree_builds(), + 1, + "{name}, whole trees {whole}: the commit itself must hash the leaves exactly once" + ); + assert_eq!( + codeword.leaf_passes(), + 1, + "{name}, whole trees {whole}: the commit hashes the leaves once" + ); + + let _ = codeword.paths(4, &[0, 1, 7], hash).expect("paths"); + assert_eq!( + codeword.tree_builds(), + 1 + per_open, + "{name}, whole trees {whole}: an opening builds a tree only when no \ + whole tree is kept" + ); + // ★ THE OTHER DIRECTION, and it is the whole point of the retention: + // the leaves were not re-hashed, whether the opening built on the + // kept layer or read the kept tree. A 2 here is the retention not + // working. + assert_eq!( + codeword.leaf_passes(), + 1, + "{name}, whole trees {whole}: the opening must reuse what was kept — \ + a 2 here means nothing was kept, or it was not matched" + ); + + // …and again, because a cache that served once and then evicted + // would read 2 on the line above too. + let _ = codeword.paths(4, &[2, 3], hash).expect("paths"); + assert_eq!( + codeword.tree_builds(), + 1 + 2 * per_open, + "{name}, whole trees {whole}: and the second opening likewise" + ); + assert_eq!( + codeword.leaf_passes(), + 1, + "{name}, whole trees {whole}: and still one leaf pass — what served \ + once and was then evicted would read 2 here" + ); + assert_eq!( + codeword.retained_leaf_bytes(), + kept_bytes(14, 4, whole), + "{name}, whole trees {whole}: the codeword keeps the wrong object, so \ + the counts above are agreeing about the wrong thing" + ); + } + } +} + +/// ★ (2) THE PATHS ARE RIGHT. Two independent codewords over the same +/// evaluations give the same root and the same paths. +/// +/// The count alone is satisfied by a build that hands back stale or wrong +/// nodes: the paths would be internally consistent and wrong. +#[test] +fn the_paths_are_the_ones_a_fresh_tree_gives() { + let _exclusive = exclusive(); + for (name, hash) in [("keccak", key::()), ("rpx", key::())] { + let num_vars = 12; + let log_folding = 4; + let f = poly(num_vars); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + + let (codeword, root) = math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit (needs a GPU)"); + let leaves = (raw.len() << 2) >> log_folding; + let positions: Vec = [0usize, 1, leaves / 3, leaves - 1] + .iter() + .map(|p| *p as u32) + .collect(); + let first = codeword + .paths(log_folding, &positions, hash) + .expect("paths"); + + // A second, independent codeword over the same evaluations: same + // inputs, a tree built from scratch, nothing shared with the one above. + let (fresh_codeword, fresh_root) = + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit"); + let fresh = fresh_codeword + .paths(log_folding, &positions, hash) + .expect("paths"); + + assert_eq!( + root, fresh_root, + "{name}: the two commits disagree on the root" + ); + assert_eq!( + first, fresh, + "{name}: two builds over the same codeword disagree on the paths" + ); + } +} + +/// ★ The same, through the production types, so the openings are checked by +/// the verifier rather than only compared to each other. +#[test] +fn the_openings_verify_against_the_device_commitment() { + let _exclusive = exclusive(); + fn check(name: &str) { + let num_vars = 12; + let log_folding = 4; + let f = poly(num_vars); + let domain = Domain::::new(num_vars + 2).expect("domain"); + let host_codeword = + whir::encode::(&whir::lift_coefficients(&f), &domain).expect("encode"); + let host = + CodewordCommitment::<_, H>::new(&host_codeword, log_folding).expect("host commit"); + + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + let (_device, root) = + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, key::()) + .expect("device commit (needs a GPU)"); + assert_eq!(root, host.root(), "{name}: device and host roots differ"); + + for index in [0, 1, host.num_leaves() / 3, host.num_leaves() - 1] { + let opening = host.open(index).expect("open"); + assert!( + verify_opening::<_, H>(&root, host.depth(), index, &opening), + "{name}: opening {index} does not verify against the device root" + ); + } + } + check::("keccak"); + check::("rpx"); +} + +/// ★★ (3) THE RESERVATION SEES THE CODEWORD AND WHAT IT KEEPS, AND GETS IT +/// BACK. +/// +/// Device memory held outside the accounting is an under-count that nothing +/// reports until a second prover shares the card. What a codeword promises is +/// its own bytes, the folds that halve it, and what it keeps of its tree — the +/// whole tree or its leaf layer, grown into the promise when kept. A tree an +/// opening builds without keeping it is a transient of that call and never +/// enters the promise. Both retention modes, forced. +#[test] +fn the_codeword_is_inside_the_reservation_and_gives_it_back() { + let _exclusive = exclusive(); + let be = math_cuda::device::backend().expect("a device"); + let hash = key::(); + let num_vars = 14; + let log_folding = 4; + let codeword_bytes = ((1u64 << num_vars) << 2) * 8; + + for whole in [false, true] { + let _mode = WholeTrees::force(whole); + // ⚠ Read under the lock, and it is a BASELINE rather than an assumed + // zero: a sibling's reservation was what failed this test the first + // time it ran on a card (786,400 B of someone else's). What is asserted + // below is the DELTA this codeword is responsible for. + let before = be.reserved_bytes(); + let (codeword, _root) = commit_on_device(num_vars, log_folding, hash); + + // (a) This codeword's OWN promise covers its codeword — no global + // involved, so this half would hold even without the lock. + let held = codeword.reserved_bytes(); + assert!( + held >= codeword_bytes, + "whole trees {whole}: the reservation holds {held} B, which does not cover the \ + {codeword_bytes} B codeword" + ); + + // (b) …and the global grew by exactly that, as a delta. + let grown = be.reserved_bytes() - before; + assert_eq!( + grown, held, + "whole trees {whole}: this codeword's promise and the global's growth must be \ + the same bytes" + ); + + // (c) ★ What the commit kept of its tree is INSIDE the promise — H4's + // trees were the bytes that were not — and it is the object the mode + // names: the whole tree, or the leaf layer. + let kept = codeword.retained_leaf_bytes(); + assert_eq!( + kept, + kept_bytes(num_vars, log_folding, whole), + "whole trees {whole}: the commit kept the wrong object" + ); + assert!( + held >= codeword_bytes + kept, + "whole trees {whole}: the promise ({held} B) does not cover the codeword \ + ({codeword_bytes} B) and what it keeps ({kept} B): a kept tree is outside it" + ); + + // (d) …and an opening adds nothing to it: it reads the kept tree, or + // builds a transient one on the kept layer and frees it on return. + let after_open = { + let _ = codeword.paths(log_folding, &[0, 1], hash).expect("paths"); + codeword.reserved_bytes() + }; + assert_eq!( + after_open, held, + "whole trees {whole}: an opening changed the reservation, {after_open} B against \ + {held} B" + ); + + // (e) Dropping it gives every byte back, kept tree included. The + // irreducibly global proposition, and the reason this test holds the + // lock. + drop(codeword); + assert_eq!( + be.reserved_bytes(), + before, + "whole trees {whole}: dropping the codeword must return the accounting to its \ + baseline" + ); + } +} + +/// ★★★ (4) THE GUARD, AT GROUP SCALE. Four commitments, none opened, hold four +/// codewords and what each keeps of its tree — exactly what they promised — and +/// nothing outside the promise. +/// +/// This is the test H4 needed and did not have. The unit test that shipped +/// dropped ONE bare codeword and asserted the accounting returned to baseline; +/// it passed on the leaking prover, because an O(chains) peak is not a state +/// one codeword can be in and because the accounting it read is blind to bytes +/// nobody promised. Four LIVE, UNOPENED commitments is the state the group +/// actually reaches — `StackedCommitment::commit` builds all of them before the +/// first opening — and the driver's own free-memory count is the instrument +/// that cannot be fooled by where the retention is written. +/// +/// # The invariant, and its margins +/// +/// A codeword here is `2^20 << 2` u64 = 32 MiB. At `log_folding = 2` its tree +/// is `2^20` leaves: a leaf layer is 32 MiB — one codeword — and a whole tree +/// `(2·2^20 − 1)·32` B ≈ 64 MiB — two — which is the whole reason for that +/// blocking. Both retention modes, forced, each asserting three things: +/// +/// - **What is kept is the object the mode names, and it is promised.** Each +/// codeword keeps its leaf layer or its whole tree, and the ledger grew by +/// EXACTLY four codewords and those four objects (a commit alone promises its +/// codeword, and a capture grows that promise by what it keeps). +/// - **Nothing is held outside the promise.** The driver's count less the +/// ledger's — the pool's share, after a symmetric drain — stays under one +/// tree and one codeword (96 MiB). H4's shape, a tree kept outside the +/// promise, would put four trees there (256 MiB); a leaf layer kept outside +/// it, four codewords (128). The allowance is the whole-node-array mutation +/// run's pool share after the symmetric drain, 64 MiB — a fragmentation floor +/// of one largest transient, which a best-effort trim cannot release — plus a +/// codeword. +/// - **What is kept is really held.** The driver took more than the next +/// smaller retention would: four codewords alone (128 MiB) under leaf layers, +/// four codewords and their layers (256 MiB) under whole trees. Under leaf +/// layers it also took less than four codewords, their layers and one +/// codeword (288 MiB) — the bound this guard carried before whole trees, +/// kept as it was. +/// +/// ⚠ The slack is not decoration. `free_vram_bytes` reports what the PROCESS +/// has taken from the driver, which includes whatever one-time workspace and +/// twiddle caches the first commit of this size sets up, and those are counted +/// identically in both cases. The warm-up commit below pays those costs before +/// the sample, and the margins absorb what it misses. +/// +/// Keccak because this is a memory proposition and the two hash families build +/// identically shaped trees; the cheaper kernel keeps the test short. +#[test] +fn a_group_holds_only_its_codewords_before_any_open() { + let _exclusive = exclusive(); + let be = math_cuda::device::backend().expect("a device"); + let hash = key::(); + let num_vars = 20; + // ⚠ Not 4. At `log_folding = 2` the tree is TWO codewords rather than half + // of one, which is what puts a codeword or more between every case below. + let log_folding = 2; + let codeword_bytes = ((1u64 << num_vars) << 2) * 8; + let leaf_bytes = kept_bytes(num_vars, log_folding, false); + let tree_bytes = kept_bytes(num_vars, log_folding, true); + let mib = |b: u64| b / (1 << 20); + + for whole in [false, true] { + let _mode = WholeTrees::force(whole); + let kept = if whole { tree_bytes } else { leaf_bytes }; + + // One commit of this exact shape before the sample, so the one-time + // costs of the first — twiddles, workspaces, whatever the pool grows to + // hold them — are paid outside the window and not attributed to + // retention. + drop(commit_on_device(num_vars, log_folding, hash)); + + // ⚠ And without this the measurement is the POOL's, not the caller's: + // the stream-ordered allocator keeps freed blocks, and the warm-up's own + // codeword would silently serve one of the four commits below. Drain, + // then sample. + math_cuda::device::drain_and_trim().expect("drain"); + let free_before = be.free_vram_bytes().expect("cuMemGetInfo"); + // ★ THE SECOND INSTRUMENT, and it is the one that can tell the two + // stories apart. `free_vram_bytes` is the DRIVER's count and includes + // whatever the pool is sitting on; `reserved_bytes` is what the CODE + // promised and is blind to the pool by construction. Their difference is + // the pool's, and it is what "nothing outside the promise" is asserted on. + let reserved_before = be.reserved_bytes(); + + let held: Vec<_> = (0..4) + .map(|_| commit_on_device(num_vars, log_folding, hash)) + .collect(); + + // ⛔ SYMMETRIC SAMPLING. `free_before` is taken AFTER a drain, so this + // one must be too: a delta between two samples is about the code only if + // both are taken at the same pool state. The run of 2026-09-20 read + // EXACTLY nine codewords on a bound of nine before this drain existed — + // the pool's transients, not the code's holding — and every peak-demand + // model missed on both sides, the signature of driver suballocation. + math_cuda::device::drain_and_trim().expect("drain"); + let free_after = be.free_vram_bytes().expect("cuMemGetInfo"); + let taken = free_before.saturating_sub(free_after); + let promised = be.reserved_bytes().saturating_sub(reserved_before); + let pool_share = taken.saturating_sub(promised); + + // ★ THE TWO ACCOUNTINGS, SIDE BY SIDE, PRINTED EVERY TIME under + // `--nocapture` (which is how the box gate runs this suite), so the + // honest path's numbers are READ rather than inferred from a pass. + let ledger = { + let per: Vec = held + .iter() + .map(|(c, _)| { + format!( + "{}/{}", + mib(c.reserved_bytes()), + mib(c.retained_leaf_bytes()) + ) + }) + .collect(); + format!( + "whole trees {whole}: driver {} MiB · promised {} MiB · pool share {} MiB · \ + per codeword reserved/retained MiB: [{}]", + mib(taken), + mib(promised), + mib(pool_share), + per.join(", ") + ) + }; + println!(" group guard: {ledger}"); + + // (i) The object the mode names, and its promise. + for (c, _) in &held { + assert_eq!( + c.retained_leaf_bytes(), + kept, + "whole trees {whole}: a codeword keeps {} B where the mode keeps {} B\n {}", + c.retained_leaf_bytes(), + kept, + ledger, + ); + } + assert_eq!( + promised, + 4 * (codeword_bytes + kept), + "whole trees {whole}: four commits must promise exactly four codewords and \ + what each keeps\n {ledger}" + ); + + // (ii) ⛔ Nothing held outside the promise. + assert!( + pool_share < tree_bytes + codeword_bytes, + "whole trees {whole}: the device holds {} MiB beyond what the four commits \ + promised. A tree kept outside the promise, per commitment, reads {} MiB \ + here; a leaf layer, {} MiB.\n {}", + mib(pool_share), + mib(4 * tree_bytes), + mib(4 * leaf_bytes), + ledger, + ); + + // (iii) What is kept is really held. + let floor = if whole { + 4 * (codeword_bytes + leaf_bytes) + } else { + 4 * codeword_bytes + }; + assert!( + taken > floor, + "whole trees {whole}: four unopened commitments took only {} MiB, at or under \ + the {} MiB of the next smaller retention. What the mode keeps ({} MiB for \ + four) is NOT being held — the capture never ran or the budget refused it, \ + and every opening will pay for it again.\n {}", + mib(taken), + mib(floor), + mib(4 * kept), + ledger, + ); + if !whole { + let bound = 4 * (codeword_bytes + leaf_bytes) + codeword_bytes; + assert!( + taken < bound, + "leaf layers: four unopened commitments took {} MiB from the device, over \ + the {} MiB bound. Something bigger than a leaf layer is held per \ + commitment.\n {}", + mib(taken), + mib(bound), + ledger, + ); + } + + // The commitments are alive up to here, which is the whole point: a + // `drop` any earlier and the assertions would be about a group that had + // already been released. + drop(held); + } +} + +/// ★ (5) THE BLOCKING IS THE ONE THAT WAS ASKED FOR. +/// +/// The dangerous outcome a cache makes reachable — serving a tree that answers a +/// different question, whose paths are internally consistent and wrong — stays +/// unreachable: whatever is kept is keyed by its blocking, a call at another +/// blocking builds its own tree and hashes its own leaves, and the kept object +/// still serves the blocking it was built for. Both retention modes, forced. +#[test] +fn a_tree_is_built_for_the_blocking_that_is_asked_for() { + let _exclusive = exclusive(); + let hash = key::(); + for whole in [false, true] { + let _mode = WholeTrees::force(whole); + let (codeword, _root) = commit_on_device(14, 4, hash); + assert_eq!(codeword.tree_builds(), 1); + + // Same codeword, different blocking: its own tree, its own pass. + let at_two = codeword.paths(2, &[0, 1], hash).expect("paths at k=2"); + assert_eq!( + codeword.tree_builds(), + 2, + "whole trees {whole}: the opening must build a tree for the blocking it was given" + ); + // ★ THE KEY, ASSERTED WHERE IT CAN FAIL. What was kept was built at + // k=4; this opening is at k=2 and describes a DIFFERENT tree, so it must + // not be served and the leaves must be hashed again. A 1 here is a cache + // ignoring its key, which is the one way retention could hand back paths + // that are internally consistent and wrong. + assert_eq!( + codeword.leaf_passes(), + 2, + "whole trees {whole}: a k=2 opening must NOT be served what was kept at k=4" + ); + + // And the rebuild answered the question that was asked: at k=2 the tree + // has four times the leaves, so each path is two levels deeper. + let at_four = codeword.paths(4, &[0, 1], hash).expect("paths at k=4"); + // …and what was kept at k=4 IS still there and IS served — the whole + // tree, with no build, or the layer, under a third tree — so the key + // rejects a mismatch without throwing away a match. Without these lines + // the test above would also pass on a cache that had simply stopped + // working. + assert_eq!( + codeword.tree_builds(), + if whole { 2 } else { 3 }, + "whole trees {whole}: the k=4 opening builds a tree only when no whole tree is kept" + ); + assert_eq!( + codeword.leaf_passes(), + 2, + "whole trees {whole}: the k=4 opening matches what was kept and must not re-hash" + ); + assert_eq!( + at_two.len(), + at_four.len() + 2 * 2 * 32, + "whole trees {whole}: a k=2 tree's paths must be two levels deeper than a k=4 tree's" + ); + } +} + +/// ✓ The cache is per hash too — the same codeword under two keys must build +/// two different trees, or the dispatch key is being ignored one level up. +#[test] +fn the_two_hashes_build_different_trees() { + let _exclusive = exclusive(); + let f = poly(12); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + let (_k, keccak_root) = + math_cuda::whir::commit_codeword(&raw, 2, 4, false, key::()) + .expect("device commit"); + let (_r, rpx_root) = math_cuda::whir::commit_codeword(&raw, 2, 4, false, key::()) + .expect("device commit"); + assert_ne!(keccak_root, rpx_root, "the two kernel families agreed"); +} + +/// ✓ The PROCESS-WIDE counter tracks the same passes — it is what the bench +/// prints, so it needs a test of its own. +/// +/// Takes the lock across the whole window, because every other test in this +/// binary commits too and the counter cannot tell whose work it is counting. +/// That is exactly why the assertions above do not use it. +#[test] +fn the_process_wide_counter_tracks_the_same_passes() { + let _exclusive = exclusive(); + let hash = key::(); + + // ⛔ DELTAS, NOT ABSOLUTES. These counters are process-wide and not + // resettable as a set, and the retention makes them diverge on purpose, so + // the assertions below are about what THIS codeword moved. + let at = || { + ( + tree_builds(), + leaf_hash_calls(), + retention_report().5, + math_cuda::whir::retention_whole().1, + ) + }; + + for whole in [false, true] { + let _mode = WholeTrees::force(whole); + let (b0, p0, s0, w0) = at(); + let (codeword, _root) = commit_on_device(12, 4, hash); + let (b1, p1, s1, w1) = at(); + assert_eq!( + b1 - b0, + 1, + "whole trees {whole}: one commit, one tree assembled" + ); + assert_eq!( + p1 - p0, + 1, + "whole trees {whole}: and it paid for its own leaf pass" + ); + assert_eq!( + s1 - s0, + 0, + "whole trees {whole}: with nothing yet in hand to reuse" + ); + assert_eq!(w1 - w0, 0, "whole trees {whole}: and nothing served"); + + let _ = codeword.paths(4, &[0, 1], hash).expect("paths"); + let (b2, p2, s2, w2) = at(); + // ★ THE LINES THAT CHANGED WITH H4's REPLACEMENTS. A tree and a leaf + // pass were once the same event. With the leaf layer kept, the opening + // assembles a tree and does not re-hash the leaves; with the whole tree + // kept, it assembles nothing and is served. A leaf pass here, in either + // mode, is the retention failing to serve. + assert_eq!( + b2 - b1, + u64::from(!whole), + "whole trees {whole}: the opening assembles a tree only when no whole tree is kept" + ); + assert_eq!( + p2 - p1, + 0, + "whole trees {whole}: and does NOT re-hash the leaves" + ); + assert_eq!( + (s2 - s1, w2 - w1), + if whole { (0, 1) } else { (1, 0) }, + "whole trees {whole}: the saving is counted where it happens (layer, served)" + ); + + // ⭐ THE ACCOUNTING IDENTITIES, which fail in BOTH directions: every + // tree assembled either paid for its leaf pass or reused a kept layer, + // and every call that wanted a tree either assembled one or was served + // a kept one. + assert_eq!( + b2 - b0, + (p2 - p0) + (s2 - s0), + "whole trees {whole}: every tree either paid for its leaf pass or reused one; \ + trees {}, passes {}, savings {}", + b2 - b0, + p2 - p0, + s2 - s0 + ); + assert_eq!( + (b2 - b0) + (w2 - w0), + 2, + "whole trees {whole}: the commit and the opening each assembled a tree or were served one" + ); + } +} + +/// ⛔ THE ARGUE-SURFACE DEVICE-FALLBACK COUNTER FIRES AT A REAL SITE. +/// +/// wt16 read as a win at the slot level because argue's per-table device work +/// fell back to the host UNCOUNTED — `multilinear::gpu::host_fallbacks()` +/// counts the COMMIT path only. `math_cuda::device::device_fallbacks()` is the +/// counter that closes that blind spot; this test proves it actually moves when +/// an argue-surface `reserve` is refused, using the cheapest of the five sites, +/// `DeviceColumns::upload`. +/// +/// It forces the refusal WITHOUT a budget setter and WITHOUT allocating any +/// device memory: `Backend::reserve` is a pure atomic bump on the reservation +/// total (no `cuMemAlloc`), so reserving the whole remaining budget makes every +/// later `reserve` return `None` at no memory cost. The reservation is dropped +/// at the end, giving the budget back. +/// +/// The gate mutation is deleting the `note_device_fallback()` call at +/// `columns.rs`'s `reserve`→`None` site: the count then reads 0 and the final +/// assertion reddens by name. The four undriven sites are covered card-free by +/// `note_device_fallback_is_called_at_exactly_the_five_argue_sites` in +/// `device.rs`. +#[test] +fn an_argue_reservation_refusal_bumps_the_device_fallback_counter() { + let _exclusive = exclusive(); + let be = math_cuda::device::backend().expect("device fallback test needs a GPU"); + + // Take the whole remaining budget as one reservation — an atomic bump, no + // device memory — so any further `reserve` must be refused. Held to the end + // of the test, then dropped. + let remaining = be.vram_budget_bytes().saturating_sub(be.reserved_bytes()); + let _hog = math_cuda::device::reserve(remaining) + .expect("reserving the remaining budget is an accounting move and cannot fail"); + assert_eq!( + be.reserved_bytes(), + be.vram_budget_bytes(), + "the budget is now fully promised, so the next reserve must be refused" + ); + + math_cuda::device::reset_device_fallbacks(); + assert_eq!( + math_cuda::device::device_fallbacks(), + 0, + "the counter starts this measurement at zero" + ); + + // The cheapest argue site: one column of one element wants 8 bytes the + // budget cannot promise, so `upload` returns `None` at its `reserve` and + // the site records the fallback. + let refused = math_cuda::columns::DeviceColumns::upload(&[&[0u64]]); + assert!( + refused.is_none(), + "with the budget fully promised, the device upload must decline" + ); + assert_eq!( + math_cuda::device::device_fallbacks(), + 1, + "an argue-surface reserve was refused, so the device-fallback counter \ + must read exactly one — a 0 here is the counter not wired to the site" + ); + + // `_hog` is dropped here at scope end, returning the reserved budget. +} + +/// ⛔ THE LIVE FOOTPRINT AND RESERVED HIGH-WATER TRACK THE RETENTION. +/// +/// The two instruments the evictable retention is built on: `retained_bytes_live` +/// (the SIMULTANEOUS footprint, not the cumulative `held`) and `reserved_high_water` +/// (the peak `be.reserved`, the quantity argue's `reserve` is checked against). +/// This proves both move with a real retention and, crucially, that the layer's +/// bytes are GIVEN BACK when its codeword drops — the balance the eviction relies +/// on. A broken `KeptNodes` drop leaves the live count high and reddens the +/// final assertion; a missing `note_reserved` leaves the high-water below the +/// live reserved total and reddens the middle one. +#[test] +fn the_live_footprint_and_reserved_high_water_track_the_retention() { + let _exclusive = exclusive(); + let be = math_cuda::device::backend().expect("footprint test needs a GPU"); + let live_before = math_cuda::whir::retained_bytes_live(); + { + let (codeword, _root) = commit_on_device(14, 4, key::()); + let _ = codeword + .paths(4, &[0, 1, 7], key::()) + .expect("paths"); + // A leaf layer is captured and held on the codeword. + let live = math_cuda::whir::retained_bytes_live(); + assert!( + live > live_before, + "a held leaf layer must raise the live footprint (live {live}, before {live_before})" + ); + assert!( + math_cuda::whir::retained_bytes_peak() >= live, + "the peak footprint must be at least the current live count" + ); + assert!( + math_cuda::device::reserved_high_water() >= be.reserved_bytes(), + "the reservation high-water must be at least the current reserved total \ + — note_reserved must fire on every rise of be.reserved" + ); + } + // The codeword — and, synchronously in its Drop, the retained layer — is gone. + assert_eq!( + math_cuda::whir::retained_bytes_live(), + live_before, + "dropping the codeword must give the layer's bytes back: KeptNodes's drop \ + balances the admit, or the live footprint would only ever rise" + ); +} + +/// ⛔ A BUDGET MISS EVICTS A RETAINED LAYER AND THE RESERVE THEN SUCCEEDS. +/// +/// The whole point of the evictable retention: when a real caller (argue, here a +/// bare reserve) cannot get its bytes, the retention gives a layer back rather +/// than the caller falling to the host. Committing captures ONE base layer (no +/// folds), so exactly one layer is in the registry. Then the budget is filled to +/// leave LESS free than the layer's bytes, so a reserve of the layer's size must +/// miss — and it SUCCEEDS only because the evictor reclaims the layer. The +/// mutation that disables the evictor (skip the consult in reserve, or never +/// install it) makes this reserve return None and the test panic on the expect. +#[test] +fn a_budget_miss_evicts_a_retained_layer_and_the_reserve_succeeds() { + let _exclusive = exclusive(); + let be = math_cuda::device::backend().expect("eviction test needs a GPU"); + + // Commit captures the base leaf layer; hold the codeword so the layer stays. + let (codeword, _root) = commit_on_device(14, 4, key::()); + let layer_bytes = codeword.retained_leaf_bytes(); + assert!( + layer_bytes > 0, + "precondition: the commit must have retained a layer" + ); + let (evictions_before, _) = math_cuda::whir::retention_evictions(); + + // Fill the budget to leave a gap SMALLER than the layer, so a reserve of the + // layer's size cannot fit without eviction. reserve is a pure atomic bump, + // so the hog costs no device memory. + let gap = layer_bytes / 2; + let hog_bytes = be + .vram_budget_bytes() + .saturating_sub(be.reserved_bytes()) + .saturating_sub(gap); + let _hog = math_cuda::device::reserve(hog_bytes).expect("the hog reservation cannot fail"); + assert!( + be.vram_budget_bytes().saturating_sub(be.reserved_bytes()) < layer_bytes, + "the free budget must now be below the layer size, so the next reserve misses" + ); + + // This would return None without the evictor; with it, the layer is freed + // and the reserve succeeds. + let got = math_cuda::device::reserve(layer_bytes).expect( + "the reserve must SUCCEED by evicting the retained layer — a None here \ + is the evictor not consulted, or eviction freeing nothing", + ); + + let (evictions_after, bytes_evicted) = math_cuda::whir::retention_evictions(); + assert!( + evictions_after > evictions_before, + "an eviction must have been recorded ({evictions_before} -> {evictions_after})" + ); + assert!( + bytes_evicted >= layer_bytes, + "the eviction must have freed at least the layer's bytes" + ); + assert_eq!( + codeword.retained_leaf_bytes(), + 0, + "the evicted codeword's layer slot must read None (the sole registered layer)" + ); + + drop(got); + drop(_hog); +} + +/// ⛔ A MISS THE LAYERS CANNOT COVER KEEPS THEM UNDER `LFM_WHIR_KEEP_FUTILE`, AND +/// MOVES NO PATH EITHER WAY. +/// +/// The budget is filled to leave half a layer free, and the request asks for +/// two layers more than that. Evicting the one layer held cannot cover the +/// deficit, so the reserve fails whatever the evictor does. With the switch off +/// the layer goes anyway and the opening hashes its leaves again; on, it stays +/// and the opening is served from it. Both openings give the paths of a fresh +/// codeword over the same evaluations: the switch decides where a leaf layer +/// comes from, never what it is. A switch the evictor ignores, or a walk that +/// no longer asks whether the miss is futile, reddens the kept arm's eviction +/// count. +#[test] +fn a_miss_the_layers_cannot_cover_keeps_them_and_moves_no_path() { + let _exclusive = exclusive(); + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + math_cuda::whir::force_keep_futile(None); + } + } + let _restore = Restore; + let be = math_cuda::device::backend().expect("futile-miss test needs a GPU"); + let hash = key::(); + let (num_vars, log_folding) = (14, 4); + let raw: Vec = poly(num_vars).evals().iter().map(|v| *v.value()).collect(); + let leaves = (raw.len() << 2) >> log_folding; + let positions: Vec = [0usize, 1, leaves / 3, leaves - 1] + .iter() + .map(|p| *p as u32) + .collect(); + let (reference, reference_root) = + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit (needs a GPU)"); + let want = reference + .paths(log_folding, &positions, hash) + .expect("paths"); + // Its layer must not be the one a miss below could reclaim. + drop(reference); + + for keep in [false, true] { + math_cuda::whir::force_keep_futile(Some(keep)); + let (codeword, root) = math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit"); + assert_eq!(root, reference_root, "keep {keep}: the root moved"); + let layer = codeword.retained_leaf_bytes(); + assert!( + layer > 0, + "keep {keep}: precondition: the commit must have retained a layer" + ); + let (evictions_before, _) = math_cuda::whir::retention_evictions(); + let (futile_before, _, _) = math_cuda::whir::retention_futile(); + + // Half a layer free; the request's deficit is then two layers, against + // the one layer the evictor could give back. + let gap = layer / 2; + let hog_bytes = be + .vram_budget_bytes() + .saturating_sub(be.reserved_bytes()) + .saturating_sub(gap); + let hog = math_cuda::device::reserve(hog_bytes).expect("the hog reservation cannot fail"); + let miss = math_cuda::device::reserve(gap + 2 * layer); + assert!( + miss.is_none(), + "keep {keep}: a miss no eviction can cover must fail either way" + ); + drop(hog); + + let (evictions_after, _) = math_cuda::whir::retention_evictions(); + let (futile_after, _, kept) = math_cuda::whir::retention_futile(); + assert_eq!(kept, keep, "the switch must read back as forced"); + assert_eq!( + futile_after, + futile_before + 1, + "keep {keep}: the miss must be counted as futile exactly once" + ); + if keep { + assert_eq!( + evictions_after, evictions_before, + "kept: a futile miss must evict nothing" + ); + assert_eq!( + codeword.retained_leaf_bytes(), + layer, + "kept: the layer must still be held" + ); + } else { + assert!( + evictions_after > evictions_before, + "off: the layer goes, as it did before the switch" + ); + assert_eq!( + codeword.retained_leaf_bytes(), + 0, + "off: the layer must be gone" + ); + } + + let passes = codeword.leaf_passes(); + let got = codeword + .paths(log_folding, &positions, hash) + .expect("paths"); + assert_eq!( + codeword.leaf_passes(), + passes + u64::from(!keep), + "keep {keep}: the opening is served when the layer was kept and hashes it again when not" + ); + assert_eq!(got, want, "keep {keep}: the paths moved"); + } +} + +/// ⛔ A WHOLE TREE KEPT UNDER `LFM_WHIR_WHOLE_TREES` SERVES ITS OPENINGS AND +/// MOVES NO BYTE; EVICTED, THE NEXT OPENING BUILDS THE SAME TREE AND KEEPS IT +/// WHOLE AGAIN. +/// +/// The reference is a codeword opened with the switch off: its root, its paths, +/// and its paths with their Merkle cap. With the switch on the commit keeps the +/// whole node array, and two openings — the paths, then the paths and the cap — +/// build nothing (`tree_builds` stays 1) and give the reference's bytes. Then a +/// reserve the kept tree can cover evicts it and succeeds, as a layer's would, +/// and the next opening hashes the leaves again, gives the same paths and keeps +/// the tree whole again. An opening that stopped reading the kept tree reddens +/// the build count; a capture that kept only the leaves reddens the byte count. +#[test] +fn a_kept_whole_tree_serves_its_openings_and_moves_no_byte() { + let _exclusive = exclusive(); + // Leaf layers for the reference; switched to whole trees below. Dropped at + // the end, it restores the environment's setting. + let _mode = WholeTrees::force(false); + let be = math_cuda::device::backend().expect("whole-tree test needs a GPU"); + let hash = key::(); + let (num_vars, log_folding, cap_height) = (14, 4, 3); + let raw: Vec = poly(num_vars).evals().iter().map(|v| *v.value()).collect(); + let leaves = (raw.len() << 2) >> log_folding; + let positions: Vec = [0usize, 1, leaves / 3, leaves - 1] + .iter() + .map(|p| *p as u32) + .collect(); + + let (reference, reference_root) = + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit (needs a GPU)"); + let want = reference + .paths(log_folding, &positions, hash) + .expect("paths"); + let want_capped = reference + .paths_and_cap(log_folding, &positions, cap_height, hash) + .expect("paths and cap"); + assert_eq!( + reference.retained_leaf_bytes(), + leaves as u64 * 32, + "switch off: the leaf layer alone is kept" + ); + drop(reference); + + math_cuda::whir::force_whole_trees(Some(true)); + let (codeword, root) = + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash).expect("device commit"); + assert_eq!(root, reference_root, "the root moved"); + let whole = codeword.retained_leaf_bytes(); + assert_eq!( + whole, + (2 * leaves as u64 - 1) * 32, + "switch on: the commit must keep the whole tree" + ); + let (_, served_before) = math_cuda::whir::retention_whole(); + let got = codeword + .paths(log_folding, &positions, hash) + .expect("paths"); + let got_capped = codeword + .paths_and_cap(log_folding, &positions, cap_height, hash) + .expect("paths and cap"); + let (on, served_after) = math_cuda::whir::retention_whole(); + assert!(on, "the switch must read back as forced"); + assert_eq!( + served_after, + served_before + 2, + "both openings must be served the kept tree" + ); + assert_eq!( + codeword.tree_builds(), + 1, + "openings served a kept tree build none" + ); + assert_eq!(codeword.leaf_passes(), 1, "and hash no leaf"); + assert_eq!(got, want, "the paths moved"); + assert_eq!(got_capped, want_capped, "the paths or the cap moved"); + + // A reserve the kept tree can cover evicts it, whole, and succeeds. + let (evictions_before, _) = math_cuda::whir::retention_evictions(); + let gap = whole / 2; + let hog_bytes = be + .vram_budget_bytes() + .saturating_sub(be.reserved_bytes()) + .saturating_sub(gap); + let hog = math_cuda::device::reserve(hog_bytes).expect("the hog reservation cannot fail"); + let room = math_cuda::device::reserve(whole) + .expect("a miss the kept tree can cover must evict it and succeed"); + let (evictions_after, _) = math_cuda::whir::retention_evictions(); + assert!( + evictions_after > evictions_before, + "the kept tree must have been evicted" + ); + assert_eq!(codeword.retained_leaf_bytes(), 0, "the tree must be gone"); + drop(room); + drop(hog); + + let again = codeword + .paths(log_folding, &positions, hash) + .expect("paths"); + assert_eq!( + codeword.tree_builds(), + 2, + "evicted, the opening builds the tree again" + ); + assert_eq!(codeword.leaf_passes(), 2, "and hashes its leaves again"); + assert_eq!(again, want, "the rebuilt tree gives the same paths"); + assert_eq!( + codeword.retained_leaf_bytes(), + whole, + "and it is kept whole again" + ); +} + +/// ⛔ A KEPT TREE AN OPENING IS READING STAYS PROMISED UNTIL THE OPENING LETS +/// GO — through an eviction and through its codeword's drop. +/// +/// A served opening reads a kept whole tree through its own handle after the +/// slot's lock is let go. When the slot's drop gave the promise back, emptying +/// the slot while that opening read — an eviction, or the codeword dropping — +/// left the ledger counting less than the card held until the read ended. Now +/// the promise goes with the buffer's LAST handle (`KeptNodes`), and: +/// 1. the evictor passes over a tree an opening holds: a miss only that tree +/// could cover fails, the tree stays kept, and the ledger still counts it; +/// 2. once the opening lets go, the same miss evicts the tree and succeeds; +/// 3. a codeword dropped while an opening holds its kept tree gives back +/// nothing until the opening lets go — the tree's promise, and the room it +/// was grown in, go with the last handle — and then every byte. +/// +/// An evictor that ignores readers reddens (1); a promise given back by the +/// slot's drop reddens (3). +#[test] +fn a_kept_tree_an_opening_reads_stays_promised_until_the_opening_lets_go() { + let _exclusive = exclusive(); + let _mode = WholeTrees::force(true); + let be = math_cuda::device::backend().expect("reader test needs a GPU"); + let hash = key::(); + let (num_vars, log_folding) = (14, 4); + + // Baselines, not assumed zeros: what this test is responsible for is the + // delta. + let before = be.reserved_bytes(); + let live_before = math_cuda::whir::retained_bytes_live(); + let (codeword, _root) = commit_on_device(num_vars, log_folding, hash); + let tree = codeword.retained_leaf_bytes(); + assert_eq!( + tree, + kept_bytes(num_vars, log_folding, true), + "precondition: the commit keeps its whole tree" + ); + let with_codeword = be.reserved_bytes(); + + // (1) An opening holds the tree; a miss only it could cover. + let reader = codeword + .hold_kept_tree(log_folding, hash) + .expect("a kept tree to read"); + let passes_before = math_cuda::whir::retention_read_passes(); + let (evictions_before, _) = math_cuda::whir::retention_evictions(); + let gap = tree / 2; + let hog_bytes = be + .vram_budget_bytes() + .saturating_sub(be.reserved_bytes()) + .saturating_sub(gap); + let hog = math_cuda::device::reserve(hog_bytes).expect("the hog reservation cannot fail"); + let miss = math_cuda::device::reserve(tree); + assert!( + miss.is_none(), + "the only kept tree is being read: nothing can cover this miss" + ); + assert_eq!( + codeword.retained_leaf_bytes(), + tree, + "the tree an opening reads must stay kept" + ); + assert_eq!( + math_cuda::whir::retention_evictions().0, + evictions_before, + "nothing may be evicted" + ); + assert!( + math_cuda::whir::retention_read_passes() > passes_before, + "the evictor must count the tree it passed over" + ); + assert_eq!( + be.reserved_bytes(), + with_codeword + hog_bytes, + "the ledger must still count the tree an opening reads" + ); + + // (2) The opening lets go: the same miss evicts the tree and succeeds. + drop(reader); + let got = math_cuda::device::reserve(tree) + .expect("the opening let go: the miss must evict the tree and succeed"); + assert_eq!( + codeword.retained_leaf_bytes(), + 0, + "the tree must have been evicted" + ); + // The tree's bytes went back and the miss took as many again. + assert_eq!( + be.reserved_bytes(), + with_codeword + hog_bytes, + "the eviction must give back exactly the tree's bytes" + ); + drop(got); + drop(hog); + + // (3) The codeword drops while an opening holds its tree. + let _ = codeword.paths(log_folding, &[0, 1], hash).expect("paths"); + assert_eq!( + codeword.retained_leaf_bytes(), + tree, + "the rebuild keeps the tree whole again" + ); + let reader = codeword + .hold_kept_tree(log_folding, hash) + .expect("a kept tree to read"); + let held = be.reserved_bytes(); + let live_held = math_cuda::whir::retained_bytes_live(); + drop(codeword); + assert_eq!( + be.reserved_bytes(), + held, + "the codeword dropped while an opening reads its tree: nothing may be given back yet" + ); + assert_eq!( + math_cuda::whir::retained_bytes_live(), + live_held, + "and the tree must still count as held" + ); + drop(reader); + assert_eq!( + be.reserved_bytes(), + before, + "the opening let go: every byte must go back" + ); + assert_eq!( + math_cuda::whir::retained_bytes_live(), + live_before, + "and the live footprint with it" + ); +} diff --git a/crypto/math/src/fft/bit_reversing.rs b/crypto/math/src/fft/bit_reversing.rs index 8e830888b..d8e7386e5 100644 --- a/crypto/math/src/fft/bit_reversing.rs +++ b/crypto/math/src/fft/bit_reversing.rs @@ -2,7 +2,17 @@ use rayon::prelude::*; /// In-place bit-reverse permutation algorithm. Requires input length to be a power of two. -pub fn in_place_bit_reverse_permute(input: &mut [E]) { +/// +/// A power-of-two length is the whole of what the row-major variant needs to go +/// parallel, so it takes that path — one element per row. A length that is not +/// one keeps the serial loop rather than tripping the assert there, so a caller +/// outside the documented contract behaves exactly as it did. +pub fn in_place_bit_reverse_permute(input: &mut [E]) { + #[cfg(feature = "alloc")] + if input.len().is_power_of_two() { + in_place_bit_reverse_permute_row_major(input, 1); + return; + } for i in 0..input.len() { let bit_reversed_index = reverse_index(i, input.len() as u64); if bit_reversed_index > i { @@ -40,10 +50,7 @@ pub fn reverse_index(i: usize, size: u64) -> usize { /// `debug_assert!`): a non-power-of-two `n` would break the disjointness the /// parallel path relies on, turning a bad caller's input into a data race. #[cfg(feature = "alloc")] -pub(crate) fn in_place_bit_reverse_permute_row_major( - buf: &mut [E], - num_cols: usize, -) { +pub fn in_place_bit_reverse_permute_row_major(buf: &mut [E], num_cols: usize) { if num_cols == 0 || buf.is_empty() { return; } diff --git a/crypto/math/src/fft/two_half_fft.rs b/crypto/math/src/fft/two_half_fft.rs index 589435865..2724d296e 100644 --- a/crypto/math/src/fft/two_half_fft.rs +++ b/crypto/math/src/fft/two_half_fft.rs @@ -79,6 +79,11 @@ impl TwoHalfTwiddles { bitrev_tw, }) } + + /// The bytes its two twiddle tables hold. + pub fn heap_bytes(&self) -> usize { + (self.tw.capacity() + self.bitrev_tw.capacity()) * core::mem::size_of::>() + } } /// DIT butterfly over two equal-length row-slices, one twiddle for all pairs: diff --git a/crypto/math/src/field/element.rs b/crypto/math/src/field/element.rs index 861c8276a..a3e708430 100644 --- a/crypto/math/src/field/element.rs +++ b/crypto/math/src/field/element.rs @@ -41,9 +41,8 @@ use super::traits::{IsPrimeField, IsSubFieldOf, LegendreSymbol}; /// A field element with operations algorithms defined in `F` /// /// `#[repr(transparent)]` makes `FieldElement` byte-identical to -/// `F::BaseType`, which [`SpillSafe`](crate::spill_safe::SpillSafe) -/// requires. Changing the `repr` or adding fields breaks this and -/// is UB in any function that requires `T: SpillSafe`. +/// `F::BaseType`. Changing the `repr` or adding fields breaks any code that +/// reinterprets a `FieldElement` as its base type. #[allow(clippy::derived_hash_with_manual_eq)] #[repr(transparent)] #[derive(Debug, Clone, Hash, Copy)] diff --git a/crypto/math/src/field/extensions_goldilocks.rs b/crypto/math/src/field/extensions_goldilocks.rs index b4814a2c7..42eb51a59 100644 --- a/crypto/math/src/field/extensions_goldilocks.rs +++ b/crypto/math/src/field/extensions_goldilocks.rs @@ -579,6 +579,12 @@ impl HasDefaultTranscript for Degree3GoldilocksExtensionField { GoldilocksField::sample_field_element_from(&mut next_u64) })) } + + /// The coordinates are Goldilocks and the rejection loop runs per + /// coordinate, so the predicate is the base field's. + fn candidate_in_range(candidate: u64) -> bool { + GoldilocksField::candidate_in_range(candidate) + } } // ===================================================== diff --git a/crypto/math/src/field/goldilocks.rs b/crypto/math/src/field/goldilocks.rs index 39fd707b7..e5fa034bd 100644 --- a/crypto/math/src/field/goldilocks.rs +++ b/crypto/math/src/field/goldilocks.rs @@ -548,9 +548,13 @@ impl HasDefaultTranscript for GoldilocksField { fn sample_field_element_from(mut next_u64: impl FnMut() -> u64) -> FieldElement { loop { let candidate = next_u64(); - if candidate < GOLDILOCKS_PRIME { + if Self::candidate_in_range(candidate) { return FieldElement::from(candidate); } } } + + fn candidate_in_range(candidate: u64) -> bool { + candidate < GOLDILOCKS_PRIME + } } diff --git a/crypto/math/src/field/traits.rs b/crypto/math/src/field/traits.rs index a0e0a7fbc..1adc7992b 100644 --- a/crypto/math/src/field/traits.rs +++ b/crypto/math/src/field/traits.rs @@ -305,4 +305,19 @@ pub trait HasDefaultTranscript: IsField { /// straight from the Fiat-Shamir sponge, so no separate CSPRNG keystream is /// generated (see `DefaultTranscript`). fn sample_field_element_from(next_u64: impl FnMut() -> u64) -> FieldElement; + + /// Would [`Self::sample_field_element_from`] accept `candidate` as one + /// coordinate, or reject it and pull another? + /// + /// This exposes the acceptance predicate so a caller can pre-filter the + /// candidate stream and thereby control *how many* candidates a draw + /// consumes — see `DefaultTranscript`'s constant-consumption mode. It has to + /// agree with what `sample_field_element_from` actually does: a pre-filtering + /// caller that disagreed would hand the sampler values it rejects, and the + /// consumption schedule the caller believes it is enforcing would not be the + /// real one. + /// + /// For an extension field this is the predicate for a single *base* + /// coordinate, because that is the granularity the rejection loop runs at. + fn candidate_in_range(candidate: u64) -> bool; } diff --git a/crypto/math/src/lib.rs b/crypto/math/src/lib.rs index 11d5cda31..a13b75c4f 100644 --- a/crypto/math/src/lib.rs +++ b/crypto/math/src/lib.rs @@ -5,7 +5,6 @@ extern crate alloc; pub mod errors; pub mod field; -pub mod spill_safe; pub mod traits; pub mod unsigned_integer; diff --git a/crypto/math/src/spill_safe.rs b/crypto/math/src/spill_safe.rs deleted file mode 100644 index 7bcbdf103..000000000 --- a/crypto/math/src/spill_safe.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Marker trait for types whose in-memory bytes can be reinterpreted as the -//! same type without UB: no padding, every bit pattern valid, no indirection. -//! -//! Stricter than `Copy`, which permits types with restricted bit patterns -//! (e.g. `bool`, `NonZeroU32`). -//! -//! `unsafe impl` puts the layout invariants on the implementer. The -//! compiler does not check. - -use crate::field::{element::FieldElement, traits::IsField}; - -/// # Safety -/// Implementer asserts `Self`'s memory representation contains no padding, -/// every bit pattern is a valid value of `Self`, and `Self` carries no -/// indirection (heap pointers, references, etc.). Adding this `unsafe impl` -/// for a type that violates these invariants is UB at any byte cast. -pub unsafe trait SpillSafe: Copy + 'static {} - -unsafe impl SpillSafe for u8 {} -unsafe impl SpillSafe for u16 {} -unsafe impl SpillSafe for u32 {} -unsafe impl SpillSafe for u64 {} -unsafe impl SpillSafe for u128 {} -unsafe impl SpillSafe for i8 {} -unsafe impl SpillSafe for i16 {} -unsafe impl SpillSafe for i32 {} -unsafe impl SpillSafe for i64 {} -unsafe impl SpillSafe for i128 {} - -unsafe impl SpillSafe for [T; N] {} - -// `FieldElement` is `#[repr(transparent)]` over `F::BaseType`, so its -// layout matches the base type's exactly. SpillSafe propagates through. -unsafe impl SpillSafe for FieldElement where F::BaseType: SpillSafe {} diff --git a/crypto/multilinear/Cargo.toml b/crypto/multilinear/Cargo.toml new file mode 100644 index 000000000..b90518ff6 --- /dev/null +++ b/crypto/multilinear/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "multilinear" +description = "Multilinear extensions, sumcheck and zerocheck over the Boolean hypercube" +version = "0.1.0" +edition = "2024" +license.workspace = true + +[dependencies] +# Same feature set as `stark`: the proofs this crate produces have to serialize +# in the same format the univariate ones do. +math = { path = "../math", features = [ + "std", + "lambdaworks-serde-binary", + "rkyv", +] } +crypto = { path = "../crypto", features = ["std", "serde", "rkyv"] } +thiserror = "1.0.38" +serde = { version = "1.0", features = ["derive"] } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. +rkyv = { version = "0.8.10", default-features = false, features = [ + "alloc", + "bytecheck", + "aligned", + "pointer_width_64", +] } +rayon = { version = "1.8.0", optional = true } +# The device path for committing and folding — only linked when `cuda` is on. +math-cuda = { path = "../math-cuda", optional = true } + +[features] +parallel = ["dep:rayon", "math/parallel", "crypto/parallel"] +# The device path: the grind through `crypto`, the kernels through `math-cuda`. +cuda = ["crypto/cuda", "dep:math-cuda"] diff --git a/crypto/multilinear/src/batch.rs b/crypto/multilinear/src/batch.rs new file mode 100644 index 000000000..eb0480419 --- /dev/null +++ b/crypto/multilinear/src/batch.rs @@ -0,0 +1,1075 @@ +//! Several sumcheck statements over one cube, proved in a single pass. +//! +//! A zerocheck is `Σ_x eq(r,x)·C(x) = 0`. A LogUp-GKR input-layer claim is +//! `Σ_x eq(z,x)·P(x) = p(z)`. Both have the shape `Σ_x weight(x)·poly(x) = +//! claimed` over the same trace, so batching them with powers of a challenge +//! costs one pass instead of one per argument. +//! +//! The factors are **one list, shared by every statement**: a column read by a +//! constraint and by a bus fingerprint is folded once, not twice. Each +//! statement is a rule that indexes that list, so a statement's weight table is +//! just another factor it multiplies in. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{ + Error, challenge_powers, + mle::Mle, + poly::SumcheckPolynomial, + program::{self, Program}, + sumcheck::{self, SumcheckProof}, +}; + +/// A weight table the batch multiplies its statements by, as the caller has it: +/// built, or as the point of an `eq` table. +/// +/// The second is what lets a device build the table where it is folded — +/// `2^n` cells from `n` coordinates — instead of the host building it and +/// sending it across (`LAMBDA_VM_ARGUE_DEVICE_TABLES`). Whoever runs the rounds +/// on the host builds it here, the same table either way. +#[derive(Clone, Debug)] +pub enum Weight { + Table(Mle), + /// `eq(point, ·)`. + Eq(Vec>), +} + +impl Weight { + /// The table, built here if it came as a point. + pub fn materialize(self) -> Result, Error> { + match self { + Self::Table(table) => Ok(table), + Self::Eq(point) => crate::eq::eq_mle(&point), + } + } + + fn view(&self) -> crate::gpu::WeightView<'_, F> { + match self { + Self::Table(table) => crate::gpu::WeightView::Table(table), + Self::Eq(point) => crate::gpu::WeightView::Eq(point), + } + } +} + +/// One statement's rule: what it makes of the batch's factors. +/// +/// Reads the **whole** factor list, so a sub-argument that already combines a +/// prefix of it can be used unchanged. +pub struct Rule<'a, F: IsField> { + body: Body<'a, F>, + degree: usize, +} + +/// A rule is either a closure or the program it is. A statement that can say +/// which program it is gets the sumcheck's device path; the closure form is for +/// the ones that cannot. +enum Body<'a, F: IsField> { + Closure(RuleFn<'a, F>), + Compiled(Program), +} + +/// A statement's value from the batch's factor values. +type RuleFn<'a, F> = Box]) -> FieldElement + Sync + 'a>; + +impl<'a, F: IsField> Rule<'a, F> { + /// `degree` must upper-bound the rule's total degree in the factors, its + /// weight table included. + pub fn new( + degree: usize, + eval: impl Fn(&[FieldElement]) -> FieldElement + Sync + 'a, + ) -> Self { + Self { + body: Body::Closure(Box::new(eval)), + degree, + } + } + + /// The same rule as straight-line code over the factor values. + pub fn compiled(degree: usize, program: Program) -> Self { + Self { + body: Body::Compiled(program), + degree, + } + } + + pub fn degree(&self) -> usize { + self.degree + } + + /// The program this rule is, when it has one. + pub fn program(&self) -> Option<&Program> { + match &self.body { + Body::Compiled(program) => Some(program), + Body::Closure(_) => None, + } + } + + pub fn apply(&self, values: &[FieldElement]) -> FieldElement { + self.apply_in(values, &mut Vec::new()) + } + + /// The same, reusing the caller's scratch for a compiled rule's steps. + pub fn apply_in( + &self, + values: &[FieldElement], + scratch: &mut Vec>, + ) -> FieldElement { + match &self.body { + Body::Closure(eval) => eval(values), + Body::Compiled(program) => program.eval(values, scratch), + } + } +} + +/// The batch as one polynomial: `Σ_i lambda^i · rule_i`. +pub struct Batched<'a, F: IsField> { + polys: Vec>, + rules: Vec>, + lambdas: Vec>, + num_vars: usize, + degree: usize, + /// The whole batch as one program, when every rule is compiled. The round + /// loop runs this instead of the rules, and it is what a device gets. + program: Option>, + /// Whether the rounds may still go to a device. False once one has handed + /// its factors back: what it stopped for is that the cube got too small to + /// be worth sending anywhere. + dispatch: bool, +} + +impl<'a, F: IsField + 'static> Batched<'a, F> { + /// `lambdas` weights the statements; there must be one per rule. + pub fn new( + polys: Vec>, + rules: Vec>, + lambdas: Vec>, + ) -> Result { + if rules.is_empty() { + return Err(Error::EmptyPolynomial); + } + if lambdas.len() != rules.len() { + return Err(Error::VariableCountMismatch { + expected: rules.len(), + got: lambdas.len(), + }); + } + let num_vars = polys.first().map(|p| p.num_vars()).unwrap_or(0); + for p in &polys { + if p.num_vars() != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got: p.num_vars(), + }); + } + } + let degree = rules.iter().map(Rule::degree).max().unwrap_or(0); + let programs: Option>> = rules.iter().map(Rule::program).collect(); + let program = match programs { + Some(programs) => Some(program::combine(&programs, &lambdas)?), + None => None, + }; + Ok(Self { + polys, + rules, + lambdas, + num_vars, + degree, + program, + dispatch: true, + }) + } + + /// Puts `polys` in front of the ones already here. + /// + /// A batch whose first factors a device holds is built over the rest + /// alone; this is what brings them back when the device turns the rounds + /// down and the host has to run them. + pub fn prepend(&mut self, mut polys: Vec>) -> Result<(), Error> { + if polys.is_empty() { + return Ok(()); + } + let num_vars = polys[0].num_vars(); + for p in polys.iter().chain(&self.polys) { + if p.num_vars() != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got: p.num_vars(), + }); + } + } + polys.append(&mut self.polys); + self.polys = polys; + self.num_vars = num_vars; + Ok(()) + } + + /// The batch's factors, replaced whole by the ones a device folded. + /// + /// A batch whose leading factors a device holds is built over the rest + /// alone; when the device stops part-way it hands back **every** factor, + /// resident or not, as its rounds left them — so this replaces the list + /// rather than adding to it. The rounds that follow stay here: the cube + /// the device stopped at is the one it was no longer worth sending. + pub fn adopt(&mut self, polys: Vec>) -> Result<(), Error> { + let num_vars = polys.first().map(Mle::num_vars).unwrap_or(0); + for p in &polys { + if p.num_vars() != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got: p.num_vars(), + }); + } + } + self.polys = polys; + self.num_vars = num_vars; + self.dispatch = false; + Ok(()) + } + + /// The batch as one program, when it has one. + pub fn program(&self) -> Option<&Program> { + self.program.as_ref() + } +} + +impl SumcheckPolynomial for Batched<'_, F> { + fn num_vars(&self) -> usize { + self.num_vars + } + + fn degree(&self) -> usize { + self.degree + } + + fn polys(&self) -> &[Mle] { + &self.polys + } + + fn combine(&self, values: &[FieldElement]) -> FieldElement { + self.combine_in(values, &mut Vec::new()) + } + + fn combine_in( + &self, + values: &[FieldElement], + scratch: &mut Vec>, + ) -> FieldElement { + if let Some(program) = &self.program { + return program.eval(values, scratch); + } + self.rules + .iter() + .zip(&self.lambdas) + .fold(FieldElement::zero(), |acc, (rule, lambda)| { + acc + lambda * rule.apply_in(values, scratch) + }) + } + + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), Error> { + for p in &mut self.polys { + p.fix_first_variable_in_place(r)?; + } + self.num_vars -= 1; + Ok(()) + } + + fn program(&self) -> Option<&Program> { + self.program.as_ref().filter(|_| self.dispatch) + } + + fn accept_folded(&mut self, polys: Vec>) -> Result<(), Error> { + if polys.len() != self.polys.len() { + return Err(Error::VariableCountMismatch { + expected: self.polys.len(), + got: polys.len(), + }); + } + self.num_vars = polys.first().map(Mle::num_vars).unwrap_or(0); + self.polys = polys; + Ok(()) + } +} + +/// The degree the batched sumcheck runs at: the worst statement's. +pub fn degree_of(rules: &[Rule<'_, F>]) -> usize { + rules.iter().map(Rule::degree).max().unwrap_or(0) +} + +/// Proves every statement in one sumcheck, returning the proof and its point. +/// +/// `claims[i]` is what `Σ_x rule_i(x)` must come to. They are absorbed before +/// the batching challenge, so the prover cannot pick a statement after seeing +/// it. +pub fn prove( + polys: Vec>, + rules: Vec>, + claims: &[FieldElement], + transcript: &mut T, +) -> Result<(SumcheckProof, Vec>), Error> +where + F: IsField + 'static, + T: IsTranscript, +{ + if claims.len() != rules.len() { + return Err(Error::VariableCountMismatch { + expected: rules.len(), + got: claims.len(), + }); + } + for claim in claims { + transcript.append_field_element(claim); + } + let lambdas = challenge_powers(&transcript.sample_field_element(), rules.len()); + sumcheck::prove(Batched::new(polys, rules, lambdas)?, transcript) +} + +/// A batched sumcheck's proof, the point its rounds drew, and what every +/// factor slot was bound to there — empty when the factors were the host's own +/// and folded away as they went. +type ResidentProof = (SumcheckProof, Vec>, Vec>); + +/// The same, over factors a device already holds — the batch's first ones, in +/// the order the rules read them. +/// +/// Returns the proof, the point the rounds drew, and **what every slot was +/// bound to there**, in slot order. That last one is free on the device path — +/// the rounds fold the factors where they lie, so the values are already up +/// there — and empty when the host ran the rounds, which drops each factor as +/// it folds. +/// +/// `extra` is what the device does not have (the weight tables), and `absent` +/// makes what it does. That second one is a builder and not a value because on +/// the device path it is never called: building a table's factors here is most +/// of what the argument used to spend on the host, and the whole point of them +/// being up there is not to. +/// +/// It *is* called when the device turns the rounds down. Nothing has been said +/// about the factors to the transcript by then — only the claims and the +/// batching challenge, which are the same either way — so the host path +/// continues from where the device left off and produces the same proof. +/// +/// A weight given as a point ([`Weight::Eq`]) stays one until then: the device +/// builds it with the rounds, or the host does when they are turned down. +pub fn prove_resident( + extra: Vec>, + device: Option>, + absent: B, + rules: Vec>, + claims: &[FieldElement], + transcript: &mut T, +) -> Result, Error> +where + F: IsField + 'static, + FieldElement: Send + Sync, + T: IsTranscript, + B: Fn() -> Result>, Error>, +{ + prove_resident_with::(extra, device, absent, rules, claims, None, transcript) +} + +/// [`prove_resident`], with the zerocheck's rounds the stage-1 way when +/// `fused` describes the batch and `LAMBDA_VM_ARGUE_FUSED` is on +/// ([`crate::gpu_fused`]): the same messages, the same proof. A decline runs +/// today's device rounds, then the host's, as before. +#[allow(clippy::too_many_arguments)] +pub fn prove_resident_with( + extra: Vec>, + device: Option>, + absent: B, + rules: Vec>, + claims: &[FieldElement], + fused: Option>, + transcript: &mut T, +) -> Result, Error> +where + Base: IsField + math::field::traits::IsSubFieldOf + 'static, + F: IsField + 'static, + FieldElement: Send + Sync, + T: IsTranscript, + B: Fn() -> Result>, Error>, +{ + prove_resident_lazy( + extra, + device, + || None, + absent, + rules, + claims, + fused, + transcript, + ) +} + +/// [`prove_resident_with`] for factors that may not be lifted yet (D-BATCH +/// M1-2, `LAMBDA_VM_ARGUE_NO_LIFT`): with no `device` but the table's base +/// columns in `fused`, the fused rounds read the columns where they lie, and +/// `lift` makes the lifted factors only when today's rounds need them — the +/// fused rounds declined, or the cross-check replays today's beside them. The +/// same messages and proof either way. +#[allow(clippy::too_many_arguments)] +pub fn prove_resident_lazy( + extra: Vec>, + device: Option>, + lift: L, + absent: B, + rules: Vec>, + claims: &[FieldElement], + fused: Option>, + transcript: &mut T, +) -> Result, Error> +where + Base: IsField + math::field::traits::IsSubFieldOf + 'static, + F: IsField + 'static, + FieldElement: Send + Sync, + T: IsTranscript, + L: Fn() -> Option>, + B: Fn() -> Result>, Error>, +{ + let _ = (&fused, &lift); + if claims.len() != rules.len() { + return Err(Error::VariableCountMismatch { + expected: rules.len(), + got: claims.len(), + }); + } + // The base columns stand in for factors no one lifted, when the fused + // rounds will read them. + #[cfg(feature = "cuda")] + let columns = fused + .as_ref() + .and_then(|input| input.columns) + .filter(|_| device.is_none() && crate::gpu_fused::argue_fused()); + #[cfg(not(feature = "cuda"))] + let columns: Option<&crate::gpu::ColumnFactors> = None; + if device.is_none() && columns.is_none() { + let mut polys = absent()?; + for weight in extra { + polys.push(weight.materialize()?); + } + let (proof, point) = prove(polys, rules, claims, transcript)?; + return Ok((proof, point, Vec::new())); + } + #[allow(unused_mut)] + let mut device = device; + + for claim in claims { + transcript.append_field_element(claim); + } + let lambdas = challenge_powers(&transcript.sample_field_element(), rules.len()); + // Built weights go in the batch now, as they always did. When any came as a + // point, the batch starts empty and the weights wait: for the device to + // build them with its rounds, or for the host to, if it turns them down. + let pending = extra.iter().any(|weight| matches!(weight, Weight::Eq(_))); + let (built, waiting) = if pending { + (Vec::new(), extra) + } else { + let tables = extra + .into_iter() + .map(Weight::materialize) + .collect::, _>>()?; + (tables, Vec::new()) + }; + #[cfg(feature = "cuda")] + let weights = lambdas.clone(); + let mut batched = Batched::new(built, rules, lambdas)?; + let degree = batched.degree().max(1); + if let Some(program) = batched.program() { + let views: Vec> = if pending { + waiting.iter().map(Weight::view).collect() + } else { + batched + .polys() + .iter() + .map(crate::gpu::WeightView::Table) + .collect() + }; + #[cfg(feature = "cuda")] + let source = device + .as_deref() + .map(crate::gpu_fused::FusedSource::Lifted) + .or(columns.map(crate::gpu_fused::FusedSource::Columns)); + #[cfg(feature = "cuda")] + if let (Some(input), Some(source)) = ( + fused.as_ref().filter(|_| crate::gpu_fused::argue_fused()), + source, + ) { + let (width, rows) = source.shape(); + let claim = weights + .iter() + .zip(claims) + .fold(FieldElement::::zero(), |acc, (l, c)| acc + l * c); + let check = crate::gpu_fused::argue_fused_xcheck(); + let start = crate::whir_split::tick(); + let attempt = crate::gpu_fused::prove_fused( + source, + input, + &weights, + &claim, + degree, + check, + |evaluations| { + for e in evaluations { + transcript.append_field_element(e); + } + transcript.sample_field_element() + }, + ); + crate::whir_split::add_tick(&crate::whir_split::ZC_FUSED_TIME, start); + match attempt { + None => { + crate::gpu_fused::note_decline(); + crate::whir_split::bump(&crate::whir_split::ZC_FUSED_DECLINED); + if check || crate::gpu_fused::argue_fused_log() { + eprintln!( + "ARGUE FUSED: declined · factors {} n={} (today's rounds run)", + width, + rows.trailing_zeros(), + ); + } + } + Some(Err(error)) => return Err(error), + Some(Ok((mut rounds, mut point, factors, times))) => { + crate::whir_split::bump(&crate::whir_split::ZC_FUSED); + if check { + // Today's rounds replay over lifted factors, made now + // when the fused rounds read the columns. + if device.is_none() { + device = lift(); + } + let lifted = device.as_deref().ok_or(Error::DeviceFailed { + stage: "fused cross-check lift", + })?; + cross_check( + lifted, &views, program, degree, &rounds, &point, &factors, times, + )?; + } else if crate::gpu_fused::argue_fused_log() { + eprintln!( + "ARGUE FUSED: factors {} roots {} terms {} slots {} n={} · fused {:.3} ms (grid \ + {:.3} fold2 {:.3} rounds {:.3})", + width, + times.roots, + times.terms, + times.slots, + rows.trailing_zeros(), + times.total * 1e3, + times.grid * 1e3, + times.fold2 * 1e3, + times.rounds * 1e3, + ); + } + let host_tail = crate::whir_split::tick(); + batched.adopt(factors)?; + let left = batched.num_vars(); + let (tail, tail_point) = + sumcheck::prove_rounds(&mut batched, left, transcript)?; + rounds.extend(tail); + point.extend(tail_point); + let bound: Option>> = batched + .polys() + .iter() + .map(|factor| factor.as_constant().cloned()) + .collect(); + let bound = bound.ok_or(Error::NoVariablesLeft)?; + crate::whir_split::add_tick(&crate::whir_split::ZC_TAIL, host_tail); + return Ok((SumcheckProof { rounds }, point, bound)); + } + } + } + // Today's device rounds need the lifted factors: made now if no one + // lifted them (the fused rounds declined). + if device.is_none() { + device = lift(); + if device.is_some() { + crate::gpu::note_late_lift(); + } + } + let attempt = device.as_deref().and_then(|device| { + crate::gpu::prove_sumcheck_resident(device, &views, program, degree, |evaluations| { + for e in evaluations { + transcript.append_field_element(e); + } + transcript.sample_field_element() + }) + }); + if let Some(outcome) = attempt { + let (mut rounds, mut point, factors) = outcome?; + let host_tail = crate::whir_split::tick(); + // The device stopped where the cube stopped being worth sending; + // the rest of the rounds run over the factors it folded. + batched.adopt(factors)?; + let left = batched.num_vars(); + let (tail, tail_point) = sumcheck::prove_rounds(&mut batched, left, transcript)?; + rounds.extend(tail); + point.extend(tail_point); + // Binding every variable is evaluating at the point, so the factor + // values the caller needs are the factors themselves by now. + let bound: Option>> = batched + .polys() + .iter() + .map(|factor| factor.as_constant().cloned()) + .collect(); + let bound = bound.ok_or(Error::NoVariablesLeft)?; + crate::whir_split::add_tick(&crate::whir_split::ZC_TAIL, host_tail); + return Ok((SumcheckProof { rounds }, point, bound)); + } + } + // Declined before the first round: the host runs them, and for that the + // factors have to be here after all — and any weight that waited as a point. + if pending { + let weights = waiting + .into_iter() + .map(Weight::materialize) + .collect::, _>>()?; + batched.prepend(weights)?; + } + batched.prepend(absent()?)?; + let (proof, point) = sumcheck::prove(batched, transcript)?; + Ok((proof, point, Vec::new())) +} + +/// `LAMBDA_VM_ARGUE_FUSED_XCHECK`: today's device rounds over the same factors +/// with the fused rounds' challenges, every message and the factors at the +/// crossover compared, both timed. The fused rounds only read the lifted +/// factors, so today's can still fold them. A difference refuses the prove. +#[cfg(feature = "cuda")] +#[allow(clippy::too_many_arguments)] +fn cross_check( + device: &crate::gpu::DeviceFactors, + views: &[crate::gpu::WeightView<'_, F>], + program: &Program, + degree: usize, + rounds: &[crate::sumcheck::RoundProof], + point: &[FieldElement], + factors: &[Mle], + times: crate::gpu_fused::FusedTimes, +) -> Result<(), Error> +where + F: IsField + 'static, +{ + let inert = crate::gpu_fused::faults().xcheck_inert; + let mut at = 0usize; + let mut parted: Option = None; + let started = std::time::Instant::now(); + let replay = + crate::gpu::prove_sumcheck_resident(device, views, program, degree, |evaluations| { + let k = at.min(point.len() - 1); + if parted.is_none() && rounds.get(at).is_none_or(|r| r.evaluations != evaluations) { + parted = Some(at); + } + at += 1; + point[k].clone() + }); + let today = started.elapsed().as_secs_f64(); + let verdict = match replay { + None => "unchecked (today's rounds declined)".to_string(), + Some(Err(error)) => format!("unchecked (today's rounds failed: {error})"), + Some(Ok((today_rounds, _, today_factors))) => { + if parted.is_none() && today_rounds.len() != rounds.len() { + parted = Some(today_rounds.len().min(rounds.len())); + } + let same_factors = today_factors.len() == factors.len() + && today_factors + .iter() + .zip(factors) + .all(|(a, b)| a.evals() == b.evals()); + match (parted, same_factors) { + (None, true) => "ok".to_string(), + (Some(round), _) => format!("PARTED at round {round}"), + (None, false) => "PARTED at the crossover's factors".to_string(), + } + } + }; + eprintln!( + "ARGUE FUSED: factors {} roots {} terms {} slots {} n={} · fused {:.3} ms (grid {:.3} fold2 {:.3} rounds \ + {:.3}) · today {:.3} ms · R {:.3} · xcheck {verdict}", + device.width(), + times.roots, + times.terms, + times.slots, + device.len().trailing_zeros(), + times.total * 1e3, + times.grid * 1e3, + times.fold2 * 1e3, + times.rounds * 1e3, + today * 1e3, + times.total / today.max(1e-9), + ); + if verdict.starts_with("PARTED") && !inert { + return Err(Error::DeviceFailed { + stage: "fused cross-check", + }); + } + if verdict == "ok" { + crate::gpu_fused::note_xcheck(); + } + Ok(()) +} + +/// Checks the batched sumcheck against the factor values it reduces to. +/// +/// `values_at` is handed the sumcheck point and returns every factor's value +/// there. A weight table is not committed, so the verifier computes it in +/// closed form; the trace factors come from the proof, and binding *those* to +/// the committed columns is the caller's next step. +/// +/// Returns the point. +pub fn verify( + proof: &SumcheckProof, + rules: &[Rule<'_, F>], + claims: &[FieldElement], + values_at: V, + num_vars: usize, + transcript: &mut T, +) -> Result>, Error> +where + F: IsField + 'static, + T: IsTranscript, + V: FnOnce(&[FieldElement]) -> Result>, Error>, +{ + if rules.is_empty() { + return Err(Error::EmptyPolynomial); + } + if claims.len() != rules.len() { + return Err(Error::VariableCountMismatch { + expected: rules.len(), + got: claims.len(), + }); + } + for claim in claims { + transcript.append_field_element(claim); + } + let lambdas = challenge_powers(&transcript.sample_field_element(), rules.len()); + + let claimed = lambdas + .iter() + .zip(claims) + .fold(FieldElement::::zero(), |acc, (l, c)| acc + l * c); + let claim = sumcheck::verify(proof, claimed, num_vars, degree_of(rules), transcript)?; + + let values = values_at(&claim.point)?; + let rebuilt = rules + .iter() + .zip(&lambdas) + .fold(FieldElement::::zero(), |acc, (rule, lambda)| { + acc + lambda * rule.apply(&values) + }); + if rebuilt != claim.expected_evaluation { + return Err(Error::BatchMismatch); + } + + Ok(claim.point) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + eq::{eq_eval, eq_mle}, + gkr::{self, FractionLayer, FractionTree}, + }; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"batch-test") + } + + /// Factor layout shared by every statement below. + const MULT: usize = 0; + const VALUE: usize = 1; + const SQUARE: usize = 2; + const EQ_R: usize = 3; + const EQ_Z: usize = 4; + + /// A bus and a constraint over the same two columns: `mult` is the signed + /// multiplicity, `value` the fingerprint, `square` its square. Each + /// fingerprint is sent once and received once, so the bus balances. + fn columns(num_vars: usize) -> [Mle; 3] { + let size = 1usize << num_vars; + let half = size / 2; + let value: Vec = (0..size).map(|i| FE::from((i % half) as u64 + 1)).collect(); + let mult: Vec = (0..size) + .map(|i| if i < half { FE::one() } else { -FE::one() }) + .collect(); + let square: Vec = value.iter().map(|x| x * x).collect(); + [ + Mle::new(mult).unwrap(), + Mle::new(value).unwrap(), + Mle::new(square).unwrap(), + ] + } + + /// The three statements, all indexing the one factor list: + /// + /// - the zerocheck, `Σ_x eq(r,x)·(value² − square) = 0`; + /// - the bus's numerator claim, `Σ_x eq(z,x)·mult = p(z)`; + /// - its denominator claim, `Σ_x eq(z,x)·(alpha − value) = q(z)`. + /// + /// The denominator is never a column: it is `alpha − value`, read off the + /// committed `value`. + fn statements(alpha: FE) -> Vec> { + vec![ + Rule::new(3, |v: &[FE]| v[EQ_R] * (v[VALUE] * v[VALUE] - v[SQUARE])), + Rule::new(2, |v: &[FE]| v[EQ_Z] * v[MULT]), + Rule::new(2, move |v: &[FE]| v[EQ_Z] * (alpha - v[VALUE])), + ] + } + + /// Runs GKR over the bus, then settles its input-layer claim in the same + /// sumcheck as the constraint's zerocheck. + /// + /// `batch_mult` overrides the multiplicity column the *batch* reads, while + /// the tree keeps the original — a prover arguing about a different table + /// than the one it ran GKR over. + fn fuse( + cols: [Mle; 3], + num_vars: usize, + batch_mult: Option>, + ) -> Result { + let alpha = FE::from(97); + let [mult, value, square] = cols; + let batch_mult = batch_mult.unwrap_or_else(|| mult.clone()); + let denominator = Mle::new(value.evals().iter().map(|x| alpha - x).collect())?; + let tree = FractionTree::build(FractionLayer::new(mult.clone(), denominator)?)?; + let output = tree.output(); + + let mut prover = transcript(); + let gkr_out = gkr::prove(&tree, &mut prover)?; + // The zerocheck challenge, drawn from the same transcript. + let r: Vec = (0..num_vars) + .map(|_| prover.sample_field_element()) + .collect(); + let z = gkr_out.claim.point.clone(); + + let factors = || -> Result>, Error> { + Ok(vec![ + batch_mult.clone(), + value.clone(), + square.clone(), + eq_mle(&r)?, + eq_mle(&z)?, + ]) + }; + let claims = [FE::zero(), gkr_out.claim.p, gkr_out.claim.q]; + + let (proof, point) = prove(factors()?, statements(alpha), &claims, &mut prover)?; + let trace_values: Vec = factors()?[..=SQUARE] + .iter() + .map(|f| f.evaluate(&point)) + .collect::>()?; + + let mut verifier = transcript(); + let gkr_claim = gkr::verify(&gkr_out.proof, output, &mut verifier)?; + let vr: Vec = (0..num_vars) + .map(|_| verifier.sample_field_element()) + .collect(); + + let checked = verify( + &proof, + &statements(alpha), + &claims, + // The weights are not committed: the verifier computes them. + |at: &[FE]| { + let mut values = trace_values.clone(); + values.push(eq_eval(&vr, at)?); + values.push(eq_eval(&gkr_claim.point, at)?); + Ok(values) + }, + num_vars, + &mut verifier, + )?; + assert_eq!(checked, point); + Ok(proof.rounds.len()) + } + + /// The design decision this module exists for: the constraint's zerocheck + /// and the bus's input-layer claim share one sumcheck, so every column is + /// folded once. + #[test] + fn a_bus_claim_settles_in_the_constraint_s_sumcheck() { + for num_vars in 1..=4usize { + let rounds = fuse(columns(num_vars), num_vars, None) + .unwrap_or_else(|e| panic!("num_vars={num_vars}: {e:?}")); + // One pass over the cube for all three statements, not one each. + assert_eq!(rounds, num_vars); + } + } + + #[test] + fn a_constraint_broken_in_one_row_is_rejected() { + let mut cols = columns(3); + let mut square = cols[2].evals().to_vec(); + square[5] += FE::one(); + cols[2] = Mle::new(square).unwrap(); + + assert!(fuse(cols, 3, None).is_err()); + } + + /// The GKR's input-layer claim is only worth anything if it lands on the + /// same table the rest of the argument reads. + #[test] + fn a_bus_table_the_tree_was_not_built_on_is_rejected() { + let cols = columns(3); + let mut mult = cols[0].evals().to_vec(); + mult[2] += FE::one(); + + assert!(fuse(cols, 3, Some(Mle::new(mult).unwrap())).is_err()); + } + + // --------------------------------------------------------------- + // The batching itself. + // --------------------------------------------------------------- + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + /// Two statements over one factor list: `Σ eq(r,x)·a(x) = ã(r)` and + /// `Σ eq(r,x)·b(x) = b̃(r)`, sharing the weight table. + fn two_evaluation_claims(r: &[FE]) -> (Vec>, Vec>, [FE; 2]) { + let a = mle(&[3, 5, 8, 13]); + let b = mle(&[21, 34, 55, 89]); + let claims = [a.evaluate(r).unwrap(), b.evaluate(r).unwrap()]; + let polys = vec![a, b, eq_mle(r).unwrap()]; + let rules = vec![ + Rule::new(2, |v: &[FE]| v[2] * v[0]), + Rule::new(2, |v: &[FE]| v[2] * v[1]), + ]; + (polys, rules, claims) + } + + fn run_two(r: &[FE], claims: [FE; 2]) -> Result<(), Error> { + let (polys, rules, _) = two_evaluation_claims(r); + let values = polys.clone(); + let (proof, _) = prove(polys, rules, &claims, &mut transcript())?; + + let (_, rules, _) = two_evaluation_claims(r); + verify( + &proof, + &rules, + &claims, + |at: &[FE]| values.iter().map(|p| p.evaluate(at)).collect(), + r.len(), + &mut transcript(), + ) + .map(|_| ()) + } + + /// A weight given as its point proves exactly what its table proves. With + /// no device the host builds the point into the same table before the + /// rounds, so the proof, the point, the transcript after them — everything + /// — is the table's. + #[test] + fn a_weight_given_as_its_point_proves_what_its_table_proves() { + let r = [FE::from(11), FE::from(13)]; + let (polys, _, claims) = two_evaluation_claims(&r); + let run = |weight: Weight| { + let (_, rules, _) = two_evaluation_claims(&r); + let mut t = transcript(); + let (proof, point, bound) = prove_resident( + vec![weight], + None, + || Ok(polys[..2].to_vec()), + rules, + &claims, + &mut t, + ) + .unwrap(); + (proof, point, bound, t.state()) + }; + let from_table = run(Weight::Table(eq_mle(&r).unwrap())); + let from_point = run(Weight::Eq(r.to_vec())); + assert_eq!(from_table, from_point); + + let (_, rules, _) = two_evaluation_claims(&r); + verify( + &from_point.0, + &rules, + &claims, + |at: &[FE]| polys.iter().map(|p| p.evaluate(at)).collect(), + r.len(), + &mut transcript(), + ) + .expect("the proof from a point verifies"); + } + + #[test] + fn statements_sharing_a_weight_table_hold_it_once() { + let r = [FE::from(11), FE::from(13)]; + let (polys, _, claims) = two_evaluation_claims(&r); + // Three factors for two statements: the weight is shared. + assert_eq!(polys.len(), 3); + run_two(&r, claims).unwrap(); + } + + #[test] + fn a_false_claim_in_one_statement_is_rejected() { + let r = [FE::from(11), FE::from(13)]; + let (_, _, claims) = two_evaluation_claims(&r); + let lying = [claims[0], claims[1] + FE::one()]; + assert!(run_two(&r, lying).is_err()); + } + + #[test] + fn the_batch_degree_is_the_worst_statement_s() { + let rules: Vec> = vec![ + Rule::new(2, |v: &[FE]| v[0]), + Rule::new(5, |v: &[FE]| v[0]), + Rule::new(3, |v: &[FE]| v[0]), + ]; + assert_eq!(degree_of(&rules), 5); + } + + #[test] + fn a_claim_per_statement_is_required() { + let r = [FE::from(11), FE::from(13)]; + let (polys, rules, _) = two_evaluation_claims(&r); + assert_eq!( + prove(polys, rules, &[FE::zero()], &mut transcript()).unwrap_err(), + Error::VariableCountMismatch { + expected: 2, + got: 1 + } + ); + } + + #[test] + fn factors_of_differing_heights_are_rejected() { + let rules: Vec> = vec![Rule::new(1, |v: &[FE]| v[0])]; + let result = Batched::new( + vec![mle(&[1, 2]), mle(&[1, 2, 3, 4])], + rules, + vec![FE::one()], + ); + assert!(matches!( + result.err(), + Some(Error::VariableCountMismatch { .. }) + )); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let r = [FE::from(11), FE::from(13)]; + let (polys, rules, claims) = two_evaluation_claims(&r); + let values = polys.clone(); + let (proof, _) = prove(polys, rules, &claims, &mut transcript()).unwrap(); + + let (_, rules, _) = two_evaluation_claims(&r); + let mut other = DefaultTranscript::::new(b"a-different-statement"); + assert!( + verify( + &proof, + &rules, + &claims, + |at: &[FE]| values.iter().map(|p| p.evaluate(at)).collect(), + r.len(), + &mut other, + ) + .is_err() + ); + } +} diff --git a/crypto/multilinear/src/claim_reduce.rs b/crypto/multilinear/src/claim_reduce.rs new file mode 100644 index 000000000..5d6ab69d0 --- /dev/null +++ b/crypto/multilinear/src/claim_reduce.rs @@ -0,0 +1,837 @@ +//! Binding a shifted read to the column it shifts. +//! +//! A constraint that reads the next step becomes a sumcheck factor that is the +//! cyclic shift of a committed column. The zerocheck does not care where a +//! factor came from, so a prover free to pick both the column and its "shift" +//! could satisfy a constraint with unrelated tables. The shift kernel closes +//! that: `f_shift_k(alpha) = Σ_y shift_k(alpha, y)·f(y)`, which is a claim about +//! the column itself. +//! +//! Every factor's claim is batched into one degree-2 sumcheck, so the whole +//! trace costs a single pass and leaves **one evaluation claim per committed +//! column, all at the same point** — one commitment opening each. +//! +//! The guarantee is conditional, and the caller supplies the other half: *if* +//! the column values at the reduced point are the committed columns' true +//! values, then the factor values were the true shifted values at `alpha`. +//! Pinning those is the commitment scheme's job. +//! +//! The columns are base-field, like the trace; the challenges and the claims +//! are not, so the batching reads them through mixed products. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{ + element::FieldElement, + traits::{IsField, IsSubFieldOf}, +}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{ + Error, challenge_powers, + eq::{shift_eval, shift_mle}, + mle::Mle, + narrow::HostColumns, + poly::{Composed, SumcheckPolynomial}, + program::{Builder, Program}, + sumcheck::{self, SumcheckProof}, +}; + +/// Which committed column a sumcheck factor reads, and how many steps ahead. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FactorSource { + pub column: usize, + /// Frame-step offset; on the cube it is a cyclic shift by that many rows. + pub offset: usize, +} + +impl FactorSource { + /// The column itself. + pub const fn direct(column: usize) -> Self { + Self { column, offset: 0 } + } + + pub const fn shifted(column: usize, offset: usize) -> Self { + Self { column, offset } + } +} + +/// The batched reduction. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct ReduceProof { + pub sumcheck: SumcheckProof, + /// Every committed column's value at the reduced point. + pub column_values: Vec>, +} + +/// One evaluation claim per committed column, all at the same point. +#[derive(Clone, Debug)] +pub struct ReducedClaim { + pub point: Vec>, + pub column_values: Vec>, +} + +/// Distinct offsets in ascending order: the grouping both sides must agree on. +fn offsets(sources: &[FactorSource]) -> Vec { + let mut all: Vec = sources.iter().map(|s| s.offset).collect(); + all.sort_unstable(); + all.dedup(); + all +} + +/// `Σ_o K_o(y)·B_o(y)`, the factors laid out in kernel/column pairs. +fn pair_products(values: &[FieldElement]) -> FieldElement { + values + .chunks(2) + .fold(FieldElement::zero(), |acc, pair| acc + &pair[0] * &pair[1]) +} + +/// The same sum as straight-line code, over `pairs` kernel/column pairs. +fn pair_products_program(pairs: usize) -> Result, Error> { + let mut b = Builder::::new(); + let terms: Vec = (0..pairs) + .map(|pair| { + let kernel = b.var(2 * pair); + let column = b.var(2 * pair + 1); + b.mul(kernel, column) + }) + .collect(); + let root = b.sum(&terms); + b.finish(root) +} + +fn check_shape( + sources: &[FactorSource], + factor_values: &[FieldElement], + num_columns: usize, +) -> Result<(), Error> { + if sources.is_empty() { + return Err(Error::EmptyPolynomial); + } + if sources.len() != factor_values.len() { + return Err(Error::VariableCountMismatch { + expected: sources.len(), + got: factor_values.len(), + }); + } + if let Some(bad) = sources.iter().find(|s| s.column >= num_columns) { + return Err(Error::UnknownPolynomial { + index: bad.column, + len: num_columns, + }); + } + Ok(()) +} + +/// Accumulator cells a worker takes at a time. Small enough that a short table +/// still spreads, large enough that the columns streaming through one slab pay +/// for the handoff. +const ACCUMULATOR_CHUNK: usize = 1 << 12; + +/// `Σ_{i reads at `offset`} gamma^i · column_i`, the one polynomial that offset's +/// kernel multiplies. +pub(crate) fn batched_column( + columns: &[Mle], + sources: &[FactorSource], + weights: &[FieldElement], + offset: usize, + num_vars: usize, +) -> Result, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + // Which columns read at this offset, so the walk below carries its whole + // list and the accumulator is written once rather than once per column. + let members: Vec<(usize, usize)> = sources + .iter() + .enumerate() + .filter(|(_, source)| source.offset == offset) + .map(|(i, source)| (i, source.column)) + .collect(); + let mut acc = vec![FieldElement::::zero(); 1usize << num_vars]; + // A slab of the accumulator, with every column streamed through it: a + // table brings dozens of columns, and this way its cells are touched once. + let fill = |(index, slab): (usize, &mut [FieldElement])| { + let at = index * ACCUMULATOR_CHUNK; + for (weight, column) in &members { + let values = &columns[*column].evals()[at..at + slab.len()]; + for (slot, value) in slab.iter_mut().zip(values) { + // The base element on the left: the only direction the tower + // gives. + *slot += value * &weights[*weight]; + } + } + }; + #[cfg(feature = "parallel")] + acc.par_chunks_mut(ACCUMULATOR_CHUNK) + .enumerate() + .for_each(fill); + #[cfg(not(feature = "parallel"))] + acc.chunks_mut(ACCUMULATOR_CHUNK).enumerate().for_each(fill); + Mle::new(acc) +} + +/// Every column at `point`, one at a time: what runs when the card does not +/// take the table whole. +/// +/// Under `LAMBDA_VM_ARGUE_DEVICE_COLUMNS` the walk is spread over the pool when +/// each column is a host loop — the values are the same either way, every +/// column's fold being its own. Columns tall enough that `evaluate_in` sends +/// each to the device stay in order: they only get here when the card turned +/// the table down, and a pool of them would each ask it for a slab it has not +/// promised. +fn evaluate_each( + columns: &[Mle], + point: &[FieldElement], +) -> Result>, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let on_host = !crate::gpu::evaluates_on_device(columns.first().map_or(0, Mle::len)); + if on_host { + crate::gpu::note_host_evaluations(columns.len()); + } + let one = |column: &Mle| column.evaluate_in(point); + #[cfg(feature = "parallel")] + if on_host && crate::gpu::argue_device_columns() { + return columns.par_iter().map(one).collect(); + } + columns.iter().map(one).collect() +} + +/// `LAMBDA_VM_ARGUE_XCHECK`: every value the card handed back, recomputed here +/// and compared. A mismatch fails the proof and names the column, where a +/// verifier would only say that the proof does not verify. +fn check_against_the_host( + columns: &[Mle], + point: &[FieldElement], + device: &[FieldElement], +) -> Result<(), Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let one = |column: &Mle| column.evaluate_in_on_host(point); + #[cfg(feature = "parallel")] + let host: Vec> = columns.par_iter().map(one).collect::>()?; + #[cfg(not(feature = "parallel"))] + let host: Vec> = columns.iter().map(one).collect::>()?; + let differs = if host.len() != device.len() { + Some(host.len().min(device.len())) + } else { + (0..host.len()).find(|&k| host[k] != device[k]) + }; + if let Some(k) = differs { + eprintln!( + "[argue] XCHECK: the card's value of column {k} of {} (2^{} rows) is not the \ + host's: card {:?}, host {:?}", + columns.len(), + point.len(), + device.get(k), + host.get(k), + ); + return Err(Error::DeviceFailed { + stage: "column evaluation", + }); + } + crate::gpu::note_xchecked(columns.len()); + Ok(()) +} + +/// Reduces every factor's claimed value at `alpha` to one claim per column. +/// +/// Returns the proof and the reduced point. `factor_values[i]` must be the +/// value of the factor `sources[i]` describes — the column shifted by its +/// offset, evaluated at `alpha`. The columns are read on the host only by a +/// path the card does not take ([`HostColumns`]). +pub fn prove( + columns: &C, + sources: &[FactorSource], + factor_values: &[FieldElement], + alpha: &[FieldElement], + resident: Option<(&crate::gpu::ResidentColumns, usize)>, + transcript: &mut T, +) -> Result<(ReduceProof, Vec>), Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, + T: IsTranscript, + C: HostColumns + ?Sized, +{ + let t = crate::whir_split::tick(); + check_shape(sources, factor_values, columns.width())?; + match columns.rows() { + _ if columns.width() == 0 => {} + Some(rows) if rows == 1usize << alpha.len() => {} + Some(rows) => { + return Err(Error::VariableCountMismatch { + expected: alpha.len(), + got: rows.trailing_zeros() as usize, + }); + } + // Columns of different heights are host columns: name the first that + // is not the point's. + None => { + for column in columns.host() { + if column.num_vars() != alpha.len() { + return Err(Error::VariableCountMismatch { + expected: alpha.len(), + got: column.num_vars(), + }); + } + } + } + } + + // The claims are what is being reduced, so the batching challenge must come + // after them. + for value in factor_values { + transcript.append_field_element(value); + } + let weights = challenge_powers(&transcript.sample_field_element(), sources.len()); + + // Under `LAMBDA_VM_ARGUE_DEVICE_TABLES` the tables are built where they are + // folded: the card takes the point, the columns each offset reads and their + // weights, and hands back the tables folded to the crossover. + let on_card = if crate::gpu::argue_device_tables() { + let offsets = offsets(sources); + crate::gpu::prove_reduce_resident( + columns, + sources, + &weights, + &offsets, + alpha, + resident, + &pair_products_program::(offsets.len())?, + |evaluations| { + for e in evaluations { + transcript.append_field_element(e); + } + transcript.sample_field_element() + }, + ) + } else { + None + }; + let (sumcheck, point) = match on_card { + Some(outcome) => { + let (mut rounds, mut point, folded) = outcome?; + // The rounds past the crossover run here, over the tables the card + // folded — as they do after an upload, and through the same loop: + // a cube at the crossover is one no device takes. + let pairs = folded.len() / 2; + let mut rest = Composed::new(folded, pair_products::, 2)? + .with_program(pair_products_program::(pairs)?); + let left = rest.num_vars(); + let (tail, tail_point) = sumcheck::prove_rounds(&mut rest, left, transcript)?; + rounds.extend(tail); + point.extend(tail_point); + (SumcheckProof { rounds }, point) + } + None => { + let mut polys = Vec::with_capacity(2 * offsets(sources).len()); + for offset in offsets(sources) { + polys.push(shift_mle(alpha, offset)?); + polys.push(batched_column::( + columns.host(), + sources, + &weights, + offset, + alpha.len(), + )?); + } + + let pairs = polys.len() / 2; + sumcheck::prove( + Composed::new(polys, pair_products::, 2)? + .with_program(pair_products_program::(pairs)?), + transcript, + )? + } + }; + + crate::whir_split::add_tick(&crate::whir_split::REST_REDUCE, t); + + // All at the same point, so they fold together: one upload and one launch + // per level for the table instead of per column. + let t = crate::whir_split::tick(); + let column_values = match crate::gpu::evaluate_many_base(columns, &point, resident) { + Some(values) => { + if crate::gpu::argue_xcheck() { + check_against_the_host(columns.host(), &point, &values)?; + } + values + } + None => evaluate_each(columns.host(), &point)?, + }; + for value in &column_values { + transcript.append_field_element(value); + } + crate::whir_split::add_tick(&crate::whir_split::REST_COLUMNS, t); + + Ok(( + ReduceProof { + sumcheck, + column_values, + }, + point, + )) +} + +/// Checks the reduction and returns the claims the commitment scheme must +/// settle. +pub fn verify( + proof: &ReduceProof, + sources: &[FactorSource], + factor_values: &[FieldElement], + alpha: &[FieldElement], + num_columns: usize, + transcript: &mut T, +) -> Result, Error> +where + E: IsField + 'static, + T: IsTranscript, +{ + check_shape(sources, factor_values, num_columns)?; + if proof.column_values.len() != num_columns { + return Err(Error::QueryCountMismatch { + expected: num_columns, + got: proof.column_values.len(), + }); + } + + for value in factor_values { + transcript.append_field_element(value); + } + let weights = challenge_powers(&transcript.sample_field_element(), sources.len()); + + let claimed = weights + .iter() + .zip(factor_values) + .fold(FieldElement::::zero(), |acc, (w, v)| acc + w * v); + let claim = sumcheck::verify(&proof.sumcheck, claimed, alpha.len(), 2, transcript)?; + + // The kernels are closed forms, so the residual is entirely about the + // columns — and the columns are what the commitments answer for. + let mut rebuilt = FieldElement::::zero(); + for offset in offsets(sources) { + let batched = + sources + .iter() + .enumerate() + .fold(FieldElement::::zero(), |acc, (i, source)| { + if source.offset == offset { + acc + &weights[i] * &proof.column_values[source.column] + } else { + acc + } + }); + rebuilt += shift_eval(alpha, &claim.point, offset)? * batched; + } + if rebuilt != claim.expected_evaluation { + return Err(Error::ShiftedReadMismatch); + } + + for value in &proof.column_values { + transcript.append_field_element(value); + } + + Ok(ReducedClaim { + point: claim.point, + column_values: proof.column_values.clone(), + }) +} + +/// The factor `source` describes, materialized: the column shifted cyclically +/// by its offset. +/// A factor's value at `point`, without building the view when there is none +/// to build. +/// +/// An unshifted source *is* its column, and a column is the biggest thing the +/// argument holds: copying one to read a single value out of it is the whole +/// trace copied again, once per factor. +pub fn evaluate_source( + columns: &[Mle], + source: &FactorSource, + point: &[FieldElement], +) -> Result, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let column = columns.get(source.column).ok_or(Error::UnknownPolynomial { + index: source.column, + len: columns.len(), + })?; + if source.offset.is_multiple_of(column.len()) { + return column.evaluate_in(point); + } + materialize(columns, source)?.evaluate_in(point) +} + +pub fn materialize( + columns: &[Mle], + source: &FactorSource, +) -> Result, Error> { + let column = columns.get(source.column).ok_or(Error::UnknownPolynomial { + index: source.column, + len: columns.len(), + })?; + let size = column.len(); + let shift = source.offset % size; + if shift == 0 { + return Ok(column.clone()); + } + Mle::new( + (0..size) + .map(|i| column.evals()[(i + shift) % size].clone()) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"claim-reduce-test") + } + + fn column(num_vars: usize, seed: u64) -> Mle { + let vals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(seed).wrapping_add(seed * 7 + 1))) + .collect(); + Mle::new(vals).unwrap() + } + + fn point(num_vars: usize) -> Vec { + (0..num_vars).map(|i| FE::from(101 + i as u64)).collect() + } + + /// What an honest prover claims: each factor is its column shifted by its + /// offset, evaluated at `alpha`. + fn honest_values(columns: &[Mle], sources: &[FactorSource], alpha: &[FE]) -> Vec { + sources + .iter() + .map(|s| materialize(columns, s).unwrap().evaluate(alpha).unwrap()) + .collect() + } + + /// Proves honestly, then verifies whatever `factor_values` the caller wants + /// to present. + fn run( + columns: &[Mle], + sources: &[FactorSource], + prover_values: &[FE], + verifier_values: &[FE], + alpha: &[FE], + ) -> Result, Error> { + let (proof, _) = prove( + columns, + sources, + prover_values, + alpha, + None, + &mut transcript(), + )?; + verify( + &proof, + sources, + verifier_values, + alpha, + columns.len(), + &mut transcript(), + ) + } + + #[test] + fn honest_claims_reduce_to_the_columns() { + for num_vars in 1..=4usize { + let columns = vec![column(num_vars, 3), column(num_vars, 5)]; + let sources = [ + FactorSource::direct(0), + FactorSource::shifted(0, 1), + FactorSource::direct(1), + ]; + let alpha = point(num_vars); + let values = honest_values(&columns, &sources, &alpha); + + let claim = run(&columns, &sources, &values, &values, &alpha) + .unwrap_or_else(|e| panic!("num_vars={num_vars}: {e:?}")); + + // The claims handed on must be the columns' real values there. + for (c, value) in columns.iter().zip(&claim.column_values) { + assert_eq!(&c.evaluate(&claim.point).unwrap(), value); + } + } + } + + /// The reason this module exists: a "next step" factor that is not the shift + /// of the column it names cannot be passed off as one. + #[test] + fn an_unrelated_table_cannot_pass_as_a_shift() { + let num_vars = 3; + let columns = vec![column(num_vars, 3)]; + let sources = [FactorSource::direct(0), FactorSource::shifted(0, 1)]; + let alpha = point(num_vars); + + let mut lying = honest_values(&columns, &sources, &alpha); + lying[1] = column(num_vars, 11).evaluate(&alpha).unwrap(); + assert_ne!(lying[1], honest_values(&columns, &sources, &alpha)[1]); + + assert!(run(&columns, &sources, &lying, &lying, &alpha).is_err()); + } + + /// Same lie, but from a prover that proves the relation it wants: the + /// sumcheck is then internally consistent and the rebuild is what rejects. + #[test] + fn a_forged_column_value_is_rejected() { + let num_vars = 3; + let columns = vec![column(num_vars, 3), column(num_vars, 5)]; + let sources = [FactorSource::direct(0), FactorSource::shifted(1, 1)]; + let alpha = point(num_vars); + let values = honest_values(&columns, &sources, &alpha); + + let (mut proof, _) = + prove(&columns, &sources, &values, &alpha, None, &mut transcript()).unwrap(); + proof.column_values[1] += FE::one(); + + let err = verify( + &proof, + &sources, + &values, + &alpha, + columns.len(), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::ShiftedReadMismatch); + } + + #[test] + fn swapping_two_column_values_is_rejected() { + let num_vars = 3; + let columns = vec![column(num_vars, 3), column(num_vars, 5)]; + let sources = [FactorSource::direct(0), FactorSource::direct(1)]; + let alpha = point(num_vars); + let values = honest_values(&columns, &sources, &alpha); + + let (mut proof, _) = + prove(&columns, &sources, &values, &alpha, None, &mut transcript()).unwrap(); + proof.column_values.swap(0, 1); + + assert_eq!( + verify( + &proof, + &sources, + &values, + &alpha, + columns.len(), + &mut transcript() + ) + .unwrap_err(), + Error::ShiftedReadMismatch + ); + } + + #[test] + fn a_forged_factor_value_is_rejected() { + let num_vars = 3; + let columns = vec![column(num_vars, 7)]; + let sources = [FactorSource::direct(0), FactorSource::shifted(0, 1)]; + let alpha = point(num_vars); + let honest = honest_values(&columns, &sources, &alpha); + let mut forged = honest.clone(); + forged[0] += FE::one(); + + assert!(run(&columns, &sources, &honest, &forged, &alpha).is_err()); + } + + #[test] + fn offsets_beyond_one_reduce_too() { + // The example AIRs in this repo read two steps ahead, so the kernel is + // not specialized to the rotation. + let num_vars = 3; + let columns = vec![column(num_vars, 3)]; + let sources = [ + FactorSource::direct(0), + FactorSource::shifted(0, 1), + FactorSource::shifted(0, 2), + ]; + let alpha = point(num_vars); + let values = honest_values(&columns, &sources, &alpha); + + run(&columns, &sources, &values, &values, &alpha).unwrap(); + } + + #[test] + fn one_sumcheck_covers_every_factor() { + // The cost of the reduction is one pass over the cube, not one per + // factor. + let num_vars = 4; + let columns = vec![column(num_vars, 3), column(num_vars, 5)]; + let sources: Vec = (0..2) + .flat_map(|c| { + [ + FactorSource::direct(c), + FactorSource::shifted(c, 1), + FactorSource::shifted(c, 2), + ] + }) + .collect(); + let alpha = point(num_vars); + let values = honest_values(&columns, &sources, &alpha); + + let (proof, _) = + prove(&columns, &sources, &values, &alpha, None, &mut transcript()).unwrap(); + assert_eq!(proof.sumcheck.rounds.len(), num_vars); + assert_eq!(proof.column_values.len(), 2); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let num_vars = 3; + let columns = vec![column(num_vars, 3)]; + let sources = [FactorSource::direct(0), FactorSource::shifted(0, 1)]; + let alpha = point(num_vars); + let values = honest_values(&columns, &sources, &alpha); + + let (proof, _) = + prove(&columns, &sources, &values, &alpha, None, &mut transcript()).unwrap(); + let mut other = DefaultTranscript::::new(b"a-different-statement"); + assert!(verify(&proof, &sources, &values, &alpha, columns.len(), &mut other).is_err()); + } + + #[test] + fn a_source_naming_a_column_that_is_not_there_is_rejected() { + let columns = vec![column(3, 3)]; + let sources = [FactorSource::direct(1)]; + let alpha = point(3); + assert_eq!( + prove( + &columns, + &sources, + &[FE::zero()], + &alpha, + None, + &mut transcript() + ) + .unwrap_err(), + Error::UnknownPolynomial { index: 1, len: 1 } + ); + } + + #[test] + fn a_claim_per_factor_is_required() { + let columns = vec![column(3, 3)]; + let sources = [FactorSource::direct(0), FactorSource::shifted(0, 1)]; + let alpha = point(3); + assert_eq!( + prove( + &columns, + &sources, + &[FE::zero()], + &alpha, + None, + &mut transcript() + ) + .unwrap_err(), + Error::VariableCountMismatch { + expected: 2, + got: 1 + } + ); + } + + /// `LAMBDA_VM_ARGUE_DEVICE_COLUMNS` spreads the host walk over the pool, and + /// the proof must not notice: the same rounds, the same column values, the + /// same transcript after them. + /// + /// The knob is process-wide, so this is the only test here that sets it. + #[test] + fn the_device_columns_knob_moves_no_value_of_the_host_walk() { + let num_vars = 6; + let columns: Vec> = (0..9).map(|k| column(num_vars, 3 + 2 * k)).collect(); + let sources: Vec = (0..9) + .flat_map(|c| [FactorSource::direct(c), FactorSource::shifted(c, 1 + c % 3)]) + .collect(); + let alpha = point(num_vars); + let values = honest_values(&columns, &sources, &alpha); + + let run = |on: bool| { + crate::gpu::force_argue_device_columns(Some(on)); + let walked = crate::gpu::host_evaluate_calls(); + let mut t = transcript(); + let (proof, reduced) = prove(&columns, &sources, &values, &alpha, None, &mut t) + .unwrap_or_else(|e| panic!("knob {on}: {e:?}")); + ( + proof, + reduced, + t, + crate::gpu::host_evaluate_calls() - walked, + ) + }; + let (off, off_point, mut off_t, off_walked) = run(false); + let (on, on_point, mut on_t, on_walked) = run(true); + crate::gpu::force_argue_device_columns(None); + + assert_eq!(off.sumcheck, on.sumcheck, "the rounds moved"); + assert_eq!(off.column_values, on.column_values, "a column value moved"); + assert_eq!(off_point, on_point, "the reduced point moved"); + assert_eq!(off_t.state(), on_t.state(), "the transcripts parted"); + assert_eq!( + off_t.sample_field_element(), + on_t.sample_field_element(), + "the next challenge moved" + ); + // No card takes a base-field reduce, so both walked every column here. + assert!( + off_walked >= 9 && on_walked >= 9, + "the host walk went uncounted: {off_walked} and {on_walked} of 9" + ); + verify( + &on, + &sources, + &values, + &alpha, + columns.len(), + &mut transcript(), + ) + .expect("the proof with the knob on verifies"); + } + + #[test] + fn materializing_offset_zero_is_the_column_itself() { + let columns = vec![column(3, 3)]; + assert_eq!( + materialize(&columns, &FactorSource::direct(0)).unwrap(), + columns[0] + ); + } + + #[test] + fn materializing_a_shift_moves_every_row_up() { + let columns = vec![column(3, 3)]; + let shifted = materialize(&columns, &FactorSource::shifted(0, 1)).unwrap(); + let size = columns[0].len(); + for i in 0..size { + assert_eq!(shifted.evals()[i], columns[0].evals()[(i + 1) % size]); + } + } +} diff --git a/crypto/multilinear/src/commit_ood.rs b/crypto/multilinear/src/commit_ood.rs new file mode 100644 index 000000000..ce12dd4a9 --- /dev/null +++ b/crypto/multilinear/src/commit_ood.rs @@ -0,0 +1,479 @@ +//! WHIR's commitment-time out-of-domain sample, on a stack. +//! +//! WHIR's commitment does not end at the Merkle root. The verifier then draws +//! an out-of-domain point `z0` and the prover answers the committed +//! polynomial's value there, `y = F(ood_point(z0))`. A committed word may be +//! close to more than one codeword; the answer names one of them, since two +//! distinct codewords agree at a random point outside the domain only with +//! probability `(2^n − 1)/|E|`. Every challenge drawn after the answers then +//! argues about one codeword per committed word. +//! +//! The opening carries the answer as one more claim, which is the successor +//! rounds' out-of-domain step ([`crate::whir_chain`]) moved to round 0: with a +//! batching challenge `γ` drawn after the answer, the chain proves +//! +//! ```text +//! claim + γ·y under the weight w + γ·eq(ood_point(z0), ·) +//! ``` +//! +//! What lives here: +//! - the answers of a stack from its columns' own values: a stacked polynomial +//! is zero outside its columns, so `F_i(p) = Σ_c eq(corner_c, p_high)·c(p_low)` +//! over the columns `c` it holds, `p_low` the last `h_c` coordinates of `p` +//! ([`answers`]); +//! - the overlay one chain polynomial carries ([`Overlay`]); +//! - the overlay's share of the first sumcheck rounds while the opening's +//! weight is still in share form on a card ([`PendingOverlay`]). That share +//! needs `F(q ‖ p[K..])` for the `2^K` Boolean prefixes `q`, which the same +//! column values give ([`partial`]), so the card never writes +//! `γ·eq(p, ·)` at the stack's full width. + +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{Error, eq::eq_evals, mle::Mle, stacking::StackedLayout, whir_chain::MAX_FOLD}; + +/// `eq(a, x)` for one variable: `(1 − a)(1 − x) + a·x`. +fn eq1(a: &FieldElement, x: &FieldElement) -> FieldElement { + let one = FieldElement::::one(); + (&one - a) * (&one - x) + a * x +} + +/// `eq(point, bits)` for Boolean `bits`: the product of `point_j` where the bit +/// is set and `1 − point_j` where it is not. +fn eq_bits(point: &[FieldElement], bits: &[bool]) -> FieldElement { + point + .iter() + .zip(bits) + .fold(FieldElement::::one(), |acc, (p, &bit)| { + if bit { + acc * p + } else { + acc * (FieldElement::::one() - p) + } + }) +} + +/// Refuses `columns` unless it holds one value per column of `layout`, and +/// `point` unless it spans the stack. +fn check_shape( + layout: &StackedLayout, + point: &[FieldElement], + columns: &[FieldElement], +) -> Result<(), Error> { + if point.len() != layout.n_stack() { + return Err(Error::VariableCountMismatch { + expected: layout.n_stack(), + got: point.len(), + }); + } + if columns.len() != layout.placements().len() { + return Err(Error::QueryCountMismatch { + expected: layout.placements().len(), + got: columns.len(), + }); + } + Ok(()) +} + +/// The point column `c` of `layout` is read at for the stack's value at +/// `point`: its last `h_c` coordinates. +pub fn column_point<'a, E: IsField>( + layout: &StackedLayout, + column: usize, + point: &'a [FieldElement], +) -> &'a [FieldElement] { + let vars = layout.placements()[column].num_vars; + &point[point.len() - vars..] +} + +/// ★ Every stacked polynomial's value at `point`, from `columns`: column `c`'s +/// value at [`column_point`], one per column of `layout` in placement order. +/// +/// `F_i(point) = Σ_{c ∈ i} eq(corner_c, point[..n − h_c])·columns[c]`: the +/// closed form `stacked_eval`'s weight is evaluated in, read the other way. +pub fn answers( + layout: &StackedLayout, + point: &[FieldElement], + columns: &[FieldElement], +) -> Result>, Error> { + check_shape(layout, point, columns)?; + let mut out = vec![FieldElement::::zero(); layout.num_polys()]; + for (place, value) in layout.placements().iter().zip(columns) { + let corner = place.prefix_bits(); + out[place.poly] += eq_bits(&point[..corner.len()], &corner) * value; + } + Ok(out) +} + +/// The variables a [`partial`] table of polynomial `poly` may bind: at most +/// [`MAX_FOLD`], the widest a lean opening's first rounds run, and no more +/// than its shortest column prefix — a column taller than `2^(n − K)` spans +/// several of the table's prefixes, and its value at one of them is not a +/// value the commit evaluated. +pub fn partial_vars(layout: &StackedLayout, poly: usize) -> usize { + layout + .placements() + .iter() + .filter(|place| place.poly == poly) + .map(|place| place.n_stack - place.num_vars) + .fold(MAX_FOLD.min(layout.n_stack()), usize::min) +} + +/// `F_poly(q ‖ point[k..])` for every Boolean prefix `q ∈ {0,1}^k`, the first +/// variable the most significant bit of the index (the [`Mle`] convention): +/// what [`PendingOverlay`] reads its rounds from. Folding it at `point[..k]` +/// gives the answer. +/// +/// `k` at most [`partial_vars`]: each column then lies under one prefix, the +/// top `k` bits of its corner, and contributes +/// `eq(point[k..n − h_c], corner[k..])·columns[c]` there. +pub fn partial( + layout: &StackedLayout, + poly: usize, + point: &[FieldElement], + columns: &[FieldElement], + k: usize, +) -> Result>, Error> { + check_shape(layout, point, columns)?; + if k > partial_vars(layout, poly) { + return Err(Error::VariableCountMismatch { + expected: partial_vars(layout, poly), + got: k, + }); + } + let mut table = vec![FieldElement::::zero(); 1usize << k]; + for (place, value) in layout.placements().iter().zip(columns) { + if place.poly != poly { + continue; + } + let corner = place.prefix_bits(); + let prefix = corner[..k] + .iter() + .fold(0usize, |acc, &bit| (acc << 1) | usize::from(bit)); + table[prefix] += eq_bits(&point[k..corner.len()], &corner[k..]) * value; + } + Ok(table) +} + +/// One chain polynomial's commit-time claim, as its opening carries it: +/// `scale·eq(point, ·)` on its weight, and with it `scale·F(point)` on its +/// claim. +#[derive(Clone, Debug)] +pub struct Overlay<'a, E: IsField> { + /// `ood_point(z0, n)`, the whole stack's variables. + pub point: &'a [FieldElement], + /// The polynomial's batching weight, `γ^(i+1)`. + pub scale: FieldElement, + /// [`partial`] at `K = log2(len)`: what a card's first `K` rounds need. + pub partial: Vec>, +} + +impl Overlay<'_, E> +where + FieldElement: Send + Sync, +{ + /// The variables [`partial`](Self::partial) spans. + pub fn partial_vars(&self) -> usize { + self.partial.len().trailing_zeros() as usize + } + + /// `scale·eq(point, ·)` as a table, for a weight built on the host. + pub fn table(&self) -> Result, Error> { + let mut table = vec![FieldElement::::zero(); 1usize << self.point.len()]; + crate::eq::eq_evals_into(self.point, &self.scale, &mut table)?; + Mle::new(table) + } + + /// `weight + scale·eq(point, ·)`. + pub fn add_to(&self, weight: &Mle) -> Result, Error> { + let overlay = self.table()?; + if overlay.len() != weight.len() { + return Err(Error::VariableCountMismatch { + expected: weight.num_vars(), + got: self.point.len(), + }); + } + Mle::new( + weight + .evals() + .iter() + .zip(overlay.evals()) + .map(|(w, o)| w + o) + .collect(), + ) + } +} + +/// An [`Overlay`] not yet written into its opening's weight: while the weight +/// is in share form on a card, each round's share of `scale·eq(point, ·)·F` is +/// computed here, from the [`partial`] table, and added to the card's round +/// evaluations; once the table's variables are bound — or the caller needs the +/// weight whole — what is left, `scale·eq(point[..b], r)·eq(point[b..], ·)`, is +/// added to the weight where it lies ([`residual`](Self::residual)). +/// +/// Round `j` binds variable `j`. With the first `j` variables bound at `r` and +/// `A_x = F(r, x, point[j+1..])`, the overlay's round polynomial is +/// +/// ```text +/// scale·eq(point[..j], r)·eq(point_j, X)·((1 − X)·A_0 + X·A_1) +/// ``` +/// +/// since `Σ_rest eq(point[j+1..], rest)·F(r, X, rest) = F(r, X, point[j+1..])`. +#[derive(Clone, Debug)] +pub struct PendingOverlay { + point: Vec>, + scale: FieldElement, + /// `F(r, ·, point[K..])` over the variables `bound..K`, the bound ones at + /// their challenges. + table: Vec>, + /// `eq(point[..bound], r)`. + at: FieldElement, + bound: usize, +} + +impl PendingOverlay +where + FieldElement: Send + Sync, +{ + pub fn new(overlay: &Overlay<'_, E>) -> Self { + Self { + point: overlay.point.to_vec(), + scale: overlay.scale.clone(), + table: overlay.partial.clone(), + at: FieldElement::::one(), + bound: 0, + } + } + + /// Variables this can still take a round over. + pub fn rounds_left(&self) -> usize { + self.table.len().trailing_zeros() as usize + } + + /// The overlay's round polynomial at `nodes`, for the next unbound + /// variable. Refused when the table has none left. + pub fn round(&self, nodes: &[FieldElement]) -> Result>, Error> { + let left = self.rounds_left(); + if left == 0 { + return Err(Error::NoVariablesLeft); + } + let half = self.table.len() / 2; + let tail = eq_evals(&self.point[self.bound + 1..self.bound + left]); + let (lo, hi) = self.table.split_at(half); + let mut a0 = FieldElement::::zero(); + let mut a1 = FieldElement::::zero(); + for ((e, l), h) in tail.iter().zip(lo).zip(hi) { + a0 += e * l; + a1 += e * h; + } + let lead = &self.scale * &self.at; + let p = &self.point[self.bound]; + Ok(nodes + .iter() + .map(|x| &lead * eq1(p, x) * (&a0 + x * (&a1 - &a0))) + .collect()) + } + + /// Binds the next variable at `r`. + pub fn bind(&mut self, r: &FieldElement) -> Result<(), Error> { + if self.rounds_left() == 0 { + return Err(Error::NoVariablesLeft); + } + self.at = &self.at * eq1(&self.point[self.bound], r); + let half = self.table.len() / 2; + let (lo, hi) = self.table.split_at(half); + self.table = lo.iter().zip(hi).map(|(l, h)| l + r * (h - l)).collect(); + self.bound += 1; + Ok(()) + } + + /// What is left to add to the weight bound at the challenges so far: + /// `scale·at·eq(point[bound..], ·)`, as `(point[bound..], scale·at)`. + pub fn residual(&self) -> (&[FieldElement], FieldElement) { + (&self.point[self.bound..], &self.scale * &self.at) + } + + /// Variables bound so far. + pub fn bound(&self) -> usize { + self.bound + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + poly::{Composed, SumcheckPolynomial}, + sumcheck, + }; + use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, goldilocks::GoldilocksField, + }; + + type F = GoldilocksField; + type FE = FieldElement; + type EE = FieldElement; + + fn ext(seed: u64) -> EE { + EE::new([ + FE::from(seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) >> 3), + FE::from(seed.wrapping_mul(0xc2b2_ae3d_27d4_eb4f) >> 5), + FE::from(seed.wrapping_mul(0x1656_67b1_9e37_79f9) >> 7), + ]) + } + + /// Mixed heights with gaps the stacking leaves zero: three polynomials of + /// nine variables. + fn stack() -> (StackedLayout, Vec>) { + let heights = [3usize, 5, 4, 2, 6, 5, 9, 7, 7]; + let layout = StackedLayout::build(&heights, 9).unwrap(); + let columns = heights + .iter() + .enumerate() + .map(|(i, &m)| { + Mle::new( + (0..(1u64 << m)) + .map(|r| { + FE::from( + r.wrapping_add(31 * i as u64 + 5) + .wrapping_mul(6364136223846793005) + >> 9, + ) + }) + .collect(), + ) + .unwrap() + }) + .collect(); + (layout, columns) + } + + fn product(v: &[EE]) -> EE { + &v[0] * &v[1] + } + + fn lifted(f: &Mle) -> Mle { + Mle::new(f.evals().iter().map(|v| v.to_extension::()).collect()).unwrap() + } + + fn column_values(layout: &StackedLayout, columns: &[Mle], point: &[EE]) -> Vec { + columns + .iter() + .enumerate() + .map(|(c, column)| column.evaluate_in(column_point(layout, c, point)).unwrap()) + .collect() + } + + #[test] + fn the_answers_are_the_stacks_values_at_the_point() { + let (layout, columns) = stack(); + assert_eq!(layout.num_polys(), 3, "the stack this test means"); + let point = crate::whir_chain::ood_point(&ext(0x5eed), layout.n_stack()); + let values = column_values(&layout, &columns, &point); + let got = answers(&layout, &point, &values).unwrap(); + let stacked = layout.stack(&crate::stacking::borrow(&columns)).unwrap(); + for (poly, (answer, f)) in got.iter().zip(&stacked).enumerate() { + assert_eq!(*answer, f.evaluate_in(&point).unwrap(), "polynomial {poly}"); + } + assert!( + answers(&layout, &point, &values[1..]).is_err(), + "one value per column" + ); + } + + #[test] + fn the_partial_table_is_the_stack_at_each_prefix_and_folds_to_the_answer() { + let (layout, columns) = stack(); + let point = crate::whir_chain::ood_point(&ext(0xab), layout.n_stack()); + let values = column_values(&layout, &columns, &point); + let stacked = layout.stack(&crate::stacking::borrow(&columns)).unwrap(); + let ys = answers(&layout, &point, &values).unwrap(); + for poly in 0..layout.num_polys() { + let k = partial_vars(&layout, poly); + let table = partial(&layout, poly, &point, &values, k).unwrap(); + for (q, value) in table.iter().enumerate() { + let mut at: Vec = (0..k) + .map(|j| { + if (q >> (k - 1 - j)) & 1 == 1 { + EE::one() + } else { + EE::zero() + } + }) + .collect(); + at.extend_from_slice(&point[k..]); + assert_eq!( + *value, + stacked[poly].evaluate_in(&at).unwrap(), + "poly {poly} prefix {q}" + ); + } + assert_eq!( + Mle::new(table).unwrap().evaluate(&point[..k]).unwrap(), + ys[poly] + ); + assert!( + partial(&layout, poly, &point, &values, k + 1).is_err(), + "no wider than its shortest column prefix" + ); + } + // The polynomial holding the 2^9 column fills the stack: no prefix. + assert_eq!(partial_vars(&layout, 1), 0); + } + + /// ★ The pending overlay's rounds are the overlay's share of the host's + /// rounds over `(w + scale·eq(p, ·))·F`, and what is left after them is + /// the overlay's bound weight: so a card adding them to its rounds over `w` + /// proves what the host proves over the whole weight. + #[test] + fn the_pending_rounds_are_the_overlays_share_of_the_whole_weight() { + let (layout, columns) = stack(); + let stacked = layout.stack(&crate::stacking::borrow(&columns)).unwrap(); + let point = crate::whir_chain::ood_point(&ext(0x77), layout.n_stack()); + let values = column_values(&layout, &columns, &point); + let poly = 0; + let k = partial_vars(&layout, poly); + assert!(k >= 3, "the test wants a few rounds"); + let overlay = Overlay { + point: &point, + scale: ext(0x5ca1e), + partial: partial(&layout, poly, &point, &values, k).unwrap(), + }; + let message = lifted(&stacked[poly]); + let weight = Mle::new((0..message.len() as u64).map(ext).collect()).unwrap(); + + let mut whole = Composed::new( + vec![overlay.add_to(&weight).unwrap(), message.clone()], + product, + 2, + ) + .unwrap(); + let mut shares = Composed::new(vec![weight, message], product, 2).unwrap(); + let mut pending = PendingOverlay::new(&overlay); + let nodes = [EE::one(), EE::from(2u64)]; + for round in 0..k { + let want = sumcheck::round_evaluations(&whole, 2, false); + let base = sumcheck::round_evaluations(&shares, 2, false); + let extra = pending.round(&nodes).unwrap(); + let got: Vec = base.iter().zip(&extra).map(|(b, e)| b + e).collect(); + assert_eq!(got, want, "round {round}"); + let r = ext(1000 + round as u64); + whole.fix_first_variable(&r).unwrap(); + shares.fix_first_variable(&r).unwrap(); + pending.bind(&r).unwrap(); + } + assert!(pending.round(&nodes).is_err(), "the table is spent"); + // The residual completes the share-form weight to the whole one. + let (rest, scale) = pending.residual(); + let mut factors = shares.into_polys(); + let bound_shares = factors.remove(0); + let completed = Overlay { + point: rest, + scale, + partial: Vec::new(), + } + .add_to(&bound_shares) + .unwrap(); + assert_eq!(completed.evals(), whole.into_polys().remove(0).evals()); + } +} diff --git a/crypto/multilinear/src/constraint_argument.rs b/crypto/multilinear/src/constraint_argument.rs new file mode 100644 index 000000000..fcfac13a6 --- /dev/null +++ b/crypto/multilinear/src/constraint_argument.rs @@ -0,0 +1,2332 @@ +//! The argument for one table: statements over a **committed** trace, all in a +//! single sumcheck, settled against one opening per column. +//! +//! A statement is `Σ_x rule(x) = claimed`. A constraint is one — with `eq(r, ·)` +//! as its weight, the sum being zero is the zerocheck. A bus's LogUp-GKR +//! input-layer claim is another. [`prove_statements`] takes them together, so +//! the factors are folded once rather than once per argument. +//! +//! Only the trace's *columns* are committed, and only what cannot be recomputed +//! travels in the proof: +//! +//! - a read of the next step is a **view** of a column — materialized for the +//! sumcheck to fold, never committed, and its claimed value bound back to the +//! column through [`claim_reduce`]; +//! - a **public** factor, like a selector or a weight table, is committed by +//! nobody and claimed by nobody: the verifier evaluates it in closed form. +//! +//! And the columns that are committed are **stacked**: every one of them sits +//! in a subcube of a shared polynomial, so the whole trace is a handful of +//! commitments and a handful of openings — not one per column — however many +//! statements, shifted reads and public tables are in play. +//! +//! The columns live in the **base field**, like the trace they are, so the +//! stacked polynomial and its codeword do too. The sumcheck's factors have to +//! share a field, so the views are lifted for it; the codeword is not, and that +//! is the allocation that matters. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{ + Error, + batch::{self, Rule, Weight}, + claim_reduce::{self, FactorSource, ReduceProof}, + eq::{eq_eval, eq_mle}, + mle::Mle, + narrow::{HostColumns, NarrowColumns}, + records::Records, + stacked_eval::{self, StackedCommitment, StackedProof}, + stacking::StackedLayout, + sumcheck::SumcheckProof, + whir_chain::ChainConfig, + whir_commit::Commitment, + whir_hash::{KeccakWhir, WhirHash}, +}; + +/// The stack width that fits every column in a single polynomial. +/// +/// `num_vars + log_blowup + log2(columns)` must stay inside the base field's +/// two-adicity, or the evaluation domain does not exist — [`Domain::new`] says +/// so rather than producing something wrong. +/// +/// [`Domain::new`]: crate::whir::Domain::new +pub fn one_stack(num_vars: usize, num_columns: usize) -> usize { + num_vars + num_columns.next_power_of_two().trailing_zeros() as usize +} + +/// Where a sumcheck factor's table comes from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FactorKind { + /// A committed column, read at a frame-step offset. + Committed(FactorSource), + /// A table both sides can compute, so it is neither committed nor claimed. + Public, +} + +impl FactorKind { + pub const fn direct(column: usize) -> Self { + Self::Committed(FactorSource::direct(column)) + } + + pub const fn shifted(column: usize, offset: usize) -> Self { + Self::Committed(FactorSource::shifted(column, offset)) + } + + /// The column this factor reads, or `None` if it is public. + pub fn source(&self) -> Option { + match self { + Self::Committed(s) => Some(*s), + Self::Public => None, + } + } +} + +/// The committed factors' sources, in factor order — what [`claim_reduce`] +/// binds. Public factors drop out. +fn sources_of(kinds: &[FactorKind]) -> Vec { + kinds.iter().filter_map(FactorKind::source).collect() +} + +/// Weaves the committed factors' values back together with the public ones, in +/// factor order — the order the rules index. +/// +/// `public` covers the trace's public factors first; anything left over is a +/// weight table a statement appended, so it lands at the end. +/// +/// `pub` for the batched argue (`stark::multilinear_table::batched`), whose +/// verifier weaves each table's values the same way: one spelling of the order +/// the rules index. +pub fn weave( + kinds: &[FactorKind], + committed: &[FieldElement], + public: &[FieldElement], +) -> Result>, Error> { + let want_committed = kinds.iter().filter(|k| k.source().is_some()).count(); + if committed.len() != want_committed { + return Err(Error::QueryCountMismatch { + expected: want_committed, + got: committed.len(), + }); + } + let want_public = kinds.len() - want_committed; + if public.len() < want_public { + return Err(Error::QueryCountMismatch { + expected: want_public, + got: public.len(), + }); + } + + let (public, weights) = public.split_at(want_public); + let mut committed = committed.iter(); + let mut public = public.iter(); + let mut values: Vec> = kinds + .iter() + .map(|kind| match kind { + FactorKind::Committed(_) => committed.next(), + FactorKind::Public => public.next(), + }) + .map(|v| v.expect("counts were checked").clone()) + .collect(); + values.extend(weights.iter().cloned()); + Ok(values) +} + +/// A committed trace, ready to be argued about. +/// +/// Holds the columns that were committed and, for every trace-level factor, +/// where its table comes from. Weight tables are not in here: they depend on +/// challenges drawn after the commitments, so the statements bring their own. +pub struct CommittedTrace< + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField, + H: WhirHash = KeccakWhir, +> where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + data: TraceData, + stacked: StackedCommitment, +} + +/// A table's factors: its columns, the public tables, and what each factor +/// reads. +/// +/// Says nothing about where the columns are committed, which is the point: +/// several tables can share one commitment, and then no single table owns it. +#[derive(Debug)] +pub struct TraceData { + columns: Held, + /// The shape, read without touching the columns: a narrow table's are not + /// field elements until a host reader widens them. + num_columns: usize, + num_vars: usize, + /// The public factors' tables, in the order they appear in `kinds`. Held + /// because they are few — selectors and the like — while the shifted views + /// are rebuilt on demand rather than kept for the whole proof. + public: Vec>, + kinds: Vec, + /// The factors on a device, put there by whoever needed them first. The + /// input layer reads them and the sumcheck folds them, and they are the + /// biggest thing a table's argument holds — uploading them twice would + /// cost more than either use. + device: std::sync::Mutex>>, + /// The epoch's columns on the card and the index this table's start at. + /// Four things read the same columns; put there once, they are read where + /// they lie instead of uploaded again. + resident: Option<(std::sync::Arc, usize)>, +} + +/// How a table holds its committed columns between its commit and its +/// argument. +#[derive(Debug)] +enum Held { + /// A field element a cell. + Wide(Vec>), + /// Each column at the bytes its words need. A host reader widens them + /// once, and that copy is kept until [`TraceData::drop_widened`]; the card + /// widens its own (`gpu::upload_tables`) and never asks. + Narrow { + packed: NarrowColumns, + widened: std::sync::OnceLock>>, + }, + /// The records the rows come from ([`crate::records`]): the card fills the + /// columns from them (`gpu::upload_tables`), and a host reader gets the + /// host fill's, widened once and kept until [`TraceData::drop_widened`]. + /// Never packed, spilled or dropped: the records are what is held. + Records { + records: Records, + widened: std::sync::OnceLock>>, + }, + /// The packed columns are in a spill store's slot + /// ([`TraceData::take_narrow_for_spill`]); only + /// [`TraceData::restore_narrow`] brings them back. The block prover + /// refuses a group holding a table still spilled before any reader runs + /// (`stark::multilinear_block`, `SpillFailed`); a reader that met the + /// table here anyway would get no columns, a proof the verifier refuses. + Spilled, +} + +impl TraceData { + /// Points this table's columns at the epoch's device copy of them. + pub fn set_resident( + &mut self, + store: std::sync::Arc, + first: usize, + ) { + self.resident = Some((store, first)); + } + + /// Forgets the device copy [`set_resident`](Self::set_resident) pointed at, + /// so the store can go while the table (and its host columns) stay: the + /// block prover gives a group's columns back to the card between its commit + /// and its argument. + pub fn clear_resident(&mut self) { + self.resident = None; + } + + /// The table's factors as its resident base columns, with no lift + /// ([`crate::gpu::column_factors`]), or `None`. + pub fn column_factors(&self) -> Option { + crate::gpu::column_factors::( + self.num_columns, + 1usize << self.num_vars, + &self.kinds, + &self.public, + self.resident_shared(), + ) + } + + /// The same, shared: for a device tree that writes its input layer again + /// from the columns after this borrow is gone. + pub fn resident_shared(&self) -> Option<(std::sync::Arc, usize)> { + self.resident + .as_ref() + .map(|(store, first)| (store.clone(), *first)) + } + + /// The epoch's columns on the card, and where this table's start. + pub fn resident(&self) -> Option<(&crate::gpu::ResidentColumns, usize)> { + self.resident + .as_ref() + .map(|(store, first)| (&**store, *first)) + } + + /// Checks the shapes agree and that `kinds` asks for exactly the public + /// tables given. + pub fn new( + columns: Vec>, + kinds: Vec, + public: Vec>, + ) -> Result { + let num_vars = columns.first().map(Mle::num_vars).unwrap_or(0); + for got in columns + .iter() + .map(Mle::num_vars) + .chain(public.iter().map(Mle::num_vars)) + { + if got != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got, + }); + } + } + if columns.is_empty() { + return Err(Error::EmptyPolynomial); + } + let wanted = kinds.iter().filter(|k| k.source().is_none()).count(); + if public.len() != wanted { + return Err(Error::QueryCountMismatch { + expected: wanted, + got: public.len(), + }); + } + Ok(Self { + num_columns: columns.len(), + num_vars, + columns: Held::Wide(columns), + public, + device: std::sync::Mutex::new(None), + resident: None, + kinds, + }) + } + + /// [`Self::new`] over columns held narrow from the start: the packed + /// columns are the committed ones, and no field element is made for them + /// on the host unless a host reader asks. Refused unless `F` is Goldilocks + /// (whose elements are their raw words), the rows are a power of two, and + /// the public tables are the rows' height and the ones `kinds` asks for. + pub fn new_narrow( + packed: NarrowColumns, + kinds: Vec, + public: Vec>, + ) -> Result { + if std::any::TypeId::of::() + != std::any::TypeId::of::() + { + return Err(Error::EmptyPolynomial); + } + if packed.cols() == 0 { + return Err(Error::EmptyPolynomial); + } + if !packed.rows().is_power_of_two() { + return Err(Error::NotPowerOfTwo(packed.rows())); + } + let num_vars = packed.rows().trailing_zeros() as usize; + for got in public.iter().map(Mle::num_vars) { + if got != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got, + }); + } + } + let wanted = kinds.iter().filter(|k| k.source().is_none()).count(); + if public.len() != wanted { + return Err(Error::QueryCountMismatch { + expected: wanted, + got: public.len(), + }); + } + Ok(Self { + num_columns: packed.cols(), + num_vars, + columns: Held::Narrow { + packed, + widened: std::sync::OnceLock::new(), + }, + public, + device: std::sync::Mutex::new(None), + resident: None, + kinds, + }) + } + + /// [`Self::new_narrow`] over columns held as the records their rows come + /// from ([`crate::records`]): the card fills them, and no field element is + /// made for them on the host unless a host reader asks. Refused as + /// [`Self::new_narrow`] refuses. + pub fn new_records( + records: Records, + kinds: Vec, + public: Vec>, + ) -> Result { + if std::any::TypeId::of::() + != std::any::TypeId::of::() + { + return Err(Error::EmptyPolynomial); + } + if records.cols() == 0 { + return Err(Error::EmptyPolynomial); + } + if !records.rows().is_power_of_two() { + return Err(Error::NotPowerOfTwo(records.rows())); + } + let num_vars = records.rows().trailing_zeros() as usize; + for got in public.iter().map(Mle::num_vars) { + if got != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got, + }); + } + } + let wanted = kinds.iter().filter(|k| k.source().is_none()).count(); + if public.len() != wanted { + return Err(Error::QueryCountMismatch { + expected: wanted, + got: public.len(), + }); + } + Ok(Self { + num_columns: records.cols(), + num_vars, + columns: Held::Records { + records, + widened: std::sync::OnceLock::new(), + }, + public, + device: std::sync::Mutex::new(None), + resident: None, + kinds, + }) + } + + /// The committed columns on the host: widened here the first time a + /// narrow table is asked ([`crate::narrow::host_widens`] counts it), and + /// that copy kept until [`Self::drop_widened`]. + pub fn columns(&self) -> &[Mle] { + match &self.columns { + Held::Wide(columns) => columns, + Held::Narrow { packed, widened } => widened.get_or_init(|| { + crate::narrow::note_host_widen(packed.rows() * packed.cols()); + // Only Goldilocks columns are ever held narrow + // (`install_narrow`), so this widens; an empty table would make + // a proof the verifier refuses, not a wrong one it accepts. + let wide = packed.widen::(); + debug_assert!(wide.is_some(), "narrow columns that do not widen"); + wide.unwrap_or_default() + }), + Held::Records { records, widened } => widened.get_or_init(|| { + crate::narrow::note_host_widen(records.rows() * records.cols()); + crate::records::note_host_fill(); + // The host fill of the records the card would fill from; a + // fill of another shape is refused here, which makes a proof + // the verifier refuses, not a wrong one it accepts. + let wide = records + .host_fill() + .filter(|p| p.rows() == records.rows() && p.cols() == records.cols()) + .and_then(|packed| packed.widen::()); + debug_assert!(wide.is_some(), "a host fill that is not its records' table"); + wide.unwrap_or_default() + }), + Held::Spilled => { + debug_assert!(false, "a spilled table read before its columns came back"); + &[] + } + } + } + + /// How many committed columns. + pub fn num_columns(&self) -> usize { + self.num_columns + } + + /// The columns packed narrow, when they are held that way. + pub fn narrow(&self) -> Option<&NarrowColumns> { + match &self.columns { + Held::Wide(_) | Held::Records { .. } | Held::Spilled => None, + Held::Narrow { packed, .. } => Some(packed), + } + } + + /// The records the columns are filled from, when they are held that way. + pub fn records(&self) -> Option<&Records> { + match &self.columns { + Held::Records { records, .. } => Some(records), + Held::Wide(_) | Held::Narrow { .. } | Held::Spilled => None, + } + } + + /// Whether the packed columns are out in a spill store's slot. + pub fn is_spilled(&self) -> bool { + matches!(self.columns, Held::Spilled) + } + + /// Hands a narrow table's packed columns to a spill store, leaving the + /// table spilled until [`Self::restore_narrow`]. `None`, and nothing + /// changed, unless the table holds them narrow. + pub fn take_narrow_for_spill(&mut self) -> Option { + if !matches!(self.columns, Held::Narrow { .. }) { + return None; + } + match std::mem::replace(&mut self.columns, Held::Spilled) { + Held::Narrow { packed, .. } => Some(packed), + _ => None, + } + } + + /// Brings a spilled table's packed columns back from its store. Refused + /// (`false`, nothing changed) unless the table is spilled and `packed` has + /// its shape (as [`Self::install_narrow`] checks). + pub fn restore_narrow(&mut self, packed: NarrowColumns) -> bool { + self.is_spilled() && self.install_narrow(packed) + } + + /// The records out of a table held as its records, for a block's spill to + /// park or drop: the table is spilled until [`Self::restore_records`] puts + /// them back or [`Self::restore_narrow`] installs packed columns made from + /// them. `None`, nothing changed, for a table held otherwise. + pub fn take_records_for_spill(&mut self) -> Option { + if !matches!(self.columns, Held::Records { .. }) { + return None; + } + match std::mem::replace(&mut self.columns, Held::Spilled) { + Held::Records { records, .. } => Some(records), + _ => None, + } + } + + /// Puts a spilled table's records back. Refused (`false`, nothing + /// changed) unless the table is spilled and `records` have its shape. + pub fn restore_records(&mut self, records: Records) -> bool { + if !self.is_spilled() + || records.cols() != self.num_columns + || records.rows() != 1usize << self.num_vars + { + return false; + } + self.columns = Held::Records { + records, + widened: std::sync::OnceLock::new(), + }; + true + } + + /// Holds the columns as `packed` from here on, letting the field elements + /// go. Refused (`false`, nothing changed) unless `packed` has the table's + /// shape and `F` is Goldilocks, whose elements are their raw words — the + /// caller made `packed` from these very columns, which is what makes the + /// widened words the committed ones. A table held as its records is + /// refused too: the records are what it holds. + pub fn install_narrow(&mut self, packed: NarrowColumns) -> bool { + if std::any::TypeId::of::() + != std::any::TypeId::of::() + || matches!(self.columns, Held::Records { .. }) + || packed.cols() != self.num_columns + || packed.rows() != 1usize << self.num_vars + { + return false; + } + self.columns = Held::Narrow { + packed, + widened: std::sync::OnceLock::new(), + }; + true + } + + /// Packs the columns narrow here on the host ([`NarrowColumns::pack`]): + /// what the card does after a commit, for a test or a host without one. + /// `false` when they are held narrow already or cannot be. + pub fn pack_on_host(&mut self) -> bool { + let Held::Wide(columns) = &self.columns else { + return false; + }; + match NarrowColumns::pack_columns(columns) { + Some(packed) => self.install_narrow(packed), + None => false, + } + } + + /// Lets a narrow table's host widening go, keeping the packed columns (or + /// the records). + pub fn drop_widened(&mut self) { + if let Held::Narrow { widened, .. } | Held::Records { widened, .. } = &mut self.columns { + widened.take(); + } + } + + /// A test's hook: the packed columns, to break them. + #[doc(hidden)] + pub fn narrow_mut(&mut self) -> Option<&mut NarrowColumns> { + match &mut self.columns { + Held::Wide(_) | Held::Records { .. } | Held::Spilled => None, + Held::Narrow { packed, .. } => Some(packed), + } + } + + pub fn kinds(&self) -> &[FactorKind] { + &self.kinds + } + + pub fn num_vars(&self) -> usize { + self.num_vars + } + + /// The factors the sumcheck runs over: each committed column shifted by + /// its offset, and the public tables as they are. + /// + /// Materialized on each call rather than stored. The sumcheck has to own + /// and fold them anyway, so a second copy kept for the whole proof would be + /// one more resident copy of the trace and nothing else. + /// The factors on a device, if they are there. + /// + /// Whoever needs them first calls [`reside`](Self::reside); the handle is + /// shared from then on. Folding them — which the sumcheck does — spends + /// them, and nothing reads them after that. + pub fn device_factors(&self) -> Option> { + self.device.lock().ok()?.clone() + } + + /// Puts the factors on a device and keeps the handle, or leaves it empty + /// when the device declines. + /// + /// They are built there, out of the columns: a factor is a column read at + /// a frame-step offset and lifted into the extension, and both of those are + /// cheaper on the side that is going to fold them. This path never goes + /// through [`factors`](Self::factors), so on a device the host copy is + /// never made. + /// Lets go of the factors on the device. + /// + /// The sumcheck folds them where they lie, which spends them, and nothing + /// reads them afterwards — but the table outlives its own argument, so + /// without this every table's factors stay on the device until the whole + /// proof is done. On a real trace that is gigabytes held for nothing. + pub fn release_device(&self) { + if let Ok(mut slot) = self.device.lock() { + *slot = None; + } + } + + pub fn reside_from_columns(&self) -> Option> { + let mut slot = self.device.lock().ok()?; + if slot.is_none() { + *slot = crate::gpu::upload_factors_from_columns( + self, + &self.kinds, + &self.public, + self.resident + .as_ref() + .map(|(store, first)| (&**store, *first)), + ) + .map(std::sync::Arc::new); + } + slot.clone() + } + + pub fn factors(&self) -> Result>, Error> + where + F: IsSubFieldOf, + FieldElement: Send + Sync, + { + let columns = self.columns(); + // Which public table each public factor takes, resolved up front so the + // factors can be built out of order. + let mut public_at = Vec::with_capacity(self.kinds.len()); + let mut seen = 0usize; + for kind in &self.kinds { + public_at.push(seen); + if kind.source().is_none() { + seen += 1; + } + } + + // One lift of the whole trace into the extension, which is the biggest + // allocation the argument makes after the codeword. + let build = |(kind, at): (&FactorKind, &usize)| -> Result, Error> { + match kind { + // The sumcheck's factors share a field, so a base view is + // lifted for it. The codeword is what stays base. + // + // The shift rides the lift rather than preceding it: a view + // materialized first is a second copy of the column, and the + // rotation is two runs of consecutive cells, not a modulus per + // cell. + FactorKind::Committed(s) => { + let column = columns.get(s.column).ok_or(Error::UnknownPolynomial { + index: s.column, + len: columns.len(), + })?; + let shift = s.offset % column.len(); + let (wrapped, rest) = column.evals().split_at(shift); + Mle::new( + rest.iter() + .chain(wrapped) + .map(|v| v.clone().to_extension::()) + .collect(), + ) + } + FactorKind::Public => self.public.get(*at).cloned().ok_or(Error::EmptyPolynomial), + } + }; + #[cfg(feature = "parallel")] + return self + .kinds + .par_iter() + .zip(public_at.par_iter()) + .map(build) + .collect(); + #[cfg(not(feature = "parallel"))] + return self.kinds.iter().zip(public_at.iter()).map(build).collect(); + } +} + +impl HostColumns for TraceData +where + Self: Sync, +{ + fn width(&self) -> usize { + self.num_columns + } + + fn rows(&self) -> Option { + Some(1usize << self.num_vars) + } + + fn host(&self) -> &[Mle] { + self.columns() + } +} + +impl< + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, +> CommittedTrace +where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + /// Commits every column, each read unshifted as one factor. + pub fn commit(columns: Vec>, config: &ChainConfig) -> Result { + let kinds = (0..columns.len()).map(FactorKind::direct).collect(); + let n_stack = one_stack( + columns.first().map(|c| c.num_vars()).unwrap_or(0), + columns.len(), + ); + Self::commit_views(columns, kinds, Vec::new(), n_stack, config) + } + + /// Stacks and commits every column, and assembles the factors `kinds` + /// describes. + /// + /// `public` supplies the public factors' tables, in the order they appear + /// in `kinds`. Everything must agree on height. A shifted factor is + /// materialized here for the sumcheck to fold, but never committed. + /// + /// `n_stack` is the width of a stacked polynomial; [`one_stack`] is the + /// value that fits the whole trace in one. + pub fn commit_views( + columns: Vec>, + kinds: Vec, + public: Vec>, + n_stack: usize, + config: &ChainConfig, + ) -> Result { + let num_vars = columns.first().map(|c| c.num_vars()).unwrap_or(0); + for got in columns + .iter() + .map(Mle::num_vars) + .chain(public.iter().map(Mle::num_vars)) + { + if got != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got, + }); + } + } + if columns.is_empty() { + return Err(Error::EmptyPolynomial); + } + let layout = StackedLayout::build(&vec![num_vars; columns.len()], n_stack)?; + Self::commit_stacked(columns, kinds, public, layout, config) + } + + /// The same, against a layout the caller already holds. + /// + /// A verifier derives the layout from the column heights alone, so a prover + /// that built one to describe its statement passes that very one here + /// instead of building a second that could drift from it. + pub fn commit_stacked( + columns: Vec>, + kinds: Vec, + public: Vec>, + layout: StackedLayout, + config: &ChainConfig, + ) -> Result { + let stacked = StackedCommitment::::commit( + layout, + &crate::stacking::borrow(&columns), + None, + config, + )?; + Ok(Self { + data: TraceData::new(columns, kinds, public)?, + stacked, + }) + } + + /// The factors, without the commitment. + pub fn data(&self) -> &TraceData { + &self.data + } + + /// One root per stacked polynomial, not per column. + pub fn roots(&self) -> Vec { + self.stacked.roots() + } + + pub fn domain(&self) -> &crate::whir::Domain { + self.stacked.domain() + } + + pub fn layout(&self) -> &StackedLayout { + self.stacked.layout() + } + + pub fn num_vars(&self) -> usize { + self.data.num_vars + } + + pub fn kinds(&self) -> &[FactorKind] { + &self.data.kinds + } + + /// The factors the sumcheck runs over: each committed column shifted by its + /// offset, and the public tables as they are. + /// + /// Materialized on each call rather than stored. The sumcheck has to own + /// and fold them anyway, so a second copy kept for the whole proof would be + /// one more resident copy of the trace and nothing else. + pub fn factors(&self) -> Result>, Error> { + self.data.factors() + } + + pub fn columns(&self) -> &[Mle] { + self.data.columns() + } +} + +/// The public part of the statement: what was committed, over what domain, and +/// where each trace-level factor comes from. +#[derive(Clone, Copy, Debug)] +pub struct TraceClaim<'a, F: IsFFTField + IsPrimeField> { + /// One per stacked polynomial. + pub roots: &'a [Commitment], + pub kinds: &'a [FactorKind], + /// Where each column sits in the stack. Public, derived from the heights. + pub layout: &'a StackedLayout, + pub domain: &'a crate::whir::Domain, + pub num_vars: usize, +} + +/// A proof that the committed columns satisfy the statements. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct ConstraintProof { + pub core: ConstraintCore, + /// The columns' values at the reduced point, against the stack. + pub columns: StackedProof, +} + +/// A statement bundle's argument **up to** the columns' claims. +/// +/// Split out because the opening need not belong to one bundle: when several +/// tables share a commitment, each produces a core of its own and the stack is +/// opened once for all of them. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct ConstraintCore { + /// The one sumcheck every statement shares. + pub sumcheck: SumcheckProof, + /// Each **committed** factor's value at the sumcheck point. The public + /// factors are absent: the verifier computes those. + pub factor_values: Vec>, + /// Binds every committed factor value to the column it reads. + pub reduce: ReduceProof, +} + +/// Proves every statement in one sumcheck and settles it against the +/// commitments. +/// +/// The factor list is the trace's factors followed by `weights`, and that is +/// what `rules` index. `claims[i]` is what `Σ_x rules[i](x)` must come to. +/// +/// The caller must have absorbed the commitment roots and drawn whatever +/// challenges its statements need, identically on both sides. +pub fn prove_statements( + trace: &CommittedTrace, + weights: Vec>, + rules: Vec>, + claims: &[FieldElement], + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + let weights = weights.into_iter().map(Weight::Table).collect(); + let (core, reduced_point) = + prove_core::(&trace.data, weights, rules, claims, transcript)?; + + // Every column's value at one shared point, so the whole trace is settled + // against the stack in one go. + let columns = stacked_eval::prove::( + &trace.stacked, + &crate::stacking::borrow(trace.columns()), + trace.data().resident(), + &stacked_eval::Claimed::Shared(&reduced_point), + &core.reduce.column_values, + config, + transcript, + )?; + + Ok(ConstraintProof { core, columns }) +} + +/// The argument up to the columns' claims, leaving the opening to the caller. +/// +/// Returns the point the columns are claimed at, which is what a caller +/// settling several tables against one commitment collects before opening it +/// once. +pub fn prove_core( + trace: &TraceData, + weights: Vec>, + rules: Vec>, + claims: &[FieldElement], + transcript: &mut T, +) -> Result<(ConstraintCore, Vec>), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, +{ + prove_core_with(trace, weights, rules, claims, None, transcript) +} + +/// [`prove_core`], with the zerocheck batch described for the stage-1 rounds +/// (`batch::prove_resident_with`). +pub fn prove_core_with( + trace: &TraceData, + weights: Vec>, + rules: Vec>, + claims: &[FieldElement], + fused: Option>, + transcript: &mut T, +) -> Result<(ConstraintCore, Vec>), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, +{ + // The trace's factors are not built here when a device holds them: that + // build is the whole trace in the extension, and nothing on that path + // reads it. The closure is what makes them if the device turns the rounds + // down. + let resident = trace.device_factors(); + let t = crate::whir_split::tick(); + // With no lift (`LAMBDA_VM_ARGUE_NO_LIFT`) the factors are lifted here + // only if today's rounds need them. + let (sumcheck, point, bound) = batch::prove_resident_lazy( + weights, + resident, + || trace.reside_from_columns(), + || trace.factors(), + rules, + claims, + fused, + transcript, + )?; + crate::whir_split::add_tick(&crate::whir_split::REST_BATCH, t); + + // The sumcheck leaves a claim about the factors at its point. Settle it in + // two steps: reduce every committed factor's value there to a claim about + // the column it reads, then open each column once. The public factors need + // neither step. + let t = crate::whir_split::tick(); + let factor_values = if bound.len() >= trace.kinds.len() { + // The rounds folded every factor to exactly this, so reading it back is + // the whole of it. Slot order is `kinds` order, and the weight tables + // the batch added sit past the end. + trace + .kinds + .iter() + .zip(&bound) + .filter(|(kind, _)| kind.source().is_some()) + .map(|(_, value)| value.clone()) + .collect() + } else { + // The host ran the rounds and let each factor go as it folded, so the + // values have to be built again: a shifted view is rebuilt, read and + // dropped; an unshifted one is read where it lies. Either way this + // costs one column rather than a second copy of the whole factor list. + trace + .kinds + .iter() + .filter_map(FactorKind::source) + .map(|source| claim_reduce::evaluate_source(trace.columns(), &source, &point)) + .collect::, _>>()? + }; + crate::whir_split::add_tick(&crate::whir_split::REST_VALUES, t); + + let (reduce, reduced_point) = claim_reduce::prove::( + trace, + &sources_of(&trace.kinds), + &factor_values, + &point, + trace.resident(), + transcript, + )?; + + Ok(( + ConstraintCore { + sumcheck, + factor_values, + reduce, + }, + reduced_point, + )) +} + +/// Verifies the statements against the commitments. +/// +/// `rules` and `claims` must be the ones the prover used; they are the +/// statement, not part of the proof. `public_values` is handed the sumcheck +/// point and returns the public factors' values there: the trace's first, in +/// `kinds` order, then one per weight table. +/// +/// Returns each **column's** value at the reduced point. A caller that knows +/// what some column has to be — a preprocessed table, say — checks it there: +/// the commitment says the prover is consistent with what it committed, not +/// that what it committed is right. +#[must_use = "the column values are the only place a known column can be checked"] +pub fn verify_statements( + proof: &ConstraintProof, + claim_shape: TraceClaim<'_, F>, + rules: &[Rule<'_, E>], + claims: &[FieldElement], + public_values: P, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + P: FnOnce(&[FieldElement]) -> Result>, Error>, + H: WhirHash, +{ + let reduced = verify_core( + &proof.core, + claim_shape.kinds, + claim_shape.layout.placements().len(), + claim_shape.num_vars, + rules, + claims, + public_values, + transcript, + )?; + + stacked_eval::verify::( + &proof.columns, + claim_shape.layout, + claim_shape.roots, + &stacked_eval::Claimed::Shared(&reduced.point), + &reduced.column_values, + claim_shape.domain, + config, + transcript, + )?; + + Ok(reduced) +} + +/// The verifying half of [`prove_core`]: everything up to the columns' claims, +/// leaving the opening to the caller. +/// +/// `num_columns` is the table's own column count, which is what +/// [`claim_reduce`] reduces to — not the width of whatever stack those columns +/// end up sharing. +#[allow(clippy::too_many_arguments)] +#[must_use = "the column values are the only place a known column can be checked"] +pub fn verify_core( + core: &ConstraintCore, + kinds: &[FactorKind], + num_columns: usize, + num_vars: usize, + rules: &[Rule<'_, E>], + claims: &[FieldElement], + public_values: P, + transcript: &mut T, +) -> Result, Error> +where + E: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + P: FnOnce(&[FieldElement]) -> Result>, Error>, +{ + // The rules rebuild the statements from the factor values: the committed + // ones out of the proof, the public ones recomputed here. + let point = batch::verify( + &core.sumcheck, + rules, + claims, + |at: &[FieldElement]| weave(kinds, &core.factor_values, &public_values(at)?), + num_vars, + transcript, + )?; + + claim_reduce::verify( + &core.reduce, + &sources_of(kinds), + &core.factor_values, + &point, + num_columns, + transcript, + ) +} + +/// The single-constraint case: `Σ_x eq(r,x)·C(x) = 0`. +/// +/// Absorbs the roots, draws `r`, and adds `eq(r, ·)` as one more public factor +/// — so `combine` sees exactly the trace's factors, and the weight costs one +/// degree. +pub fn prove( + trace: &CommittedTrace, + combine: C, + degree: usize, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + C: Fn(&[FieldElement]) -> FieldElement + Sync, + H: WhirHash, +{ + for root in trace.roots() { + transcript.append_bytes(&root); + } + let r: Vec> = (0..trace.num_vars()) + .map(|_| transcript.sample_field_element()) + .collect(); + + let weight = trace.kinds().len(); + let rule = Rule::new(degree + 1, move |v: &[FieldElement]| { + &v[weight] * combine(&v[..weight]) + }); + prove_statements( + trace, + vec![eq_mle(&r)?], + vec![rule], + &[FieldElement::zero()], + config, + transcript, + ) +} + +/// Verifies the single-constraint case. See [`prove`]. +pub fn verify( + proof: &ConstraintProof, + claim_shape: TraceClaim<'_, F>, + combine: C, + public_values: P, + degree: usize, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + C: Fn(&[FieldElement]) -> FieldElement + Sync, + P: FnOnce(&[FieldElement]) -> Result>, Error>, + H: WhirHash, +{ + for root in claim_shape.roots { + transcript.append_bytes(root); + } + let r: Vec> = (0..claim_shape.num_vars) + .map(|_| transcript.sample_field_element()) + .collect(); + + let weight = claim_shape.kinds.len(); + let rule = Rule::new(degree + 1, move |v: &[FieldElement]| { + &v[weight] * combine(&v[..weight]) + }); + verify_statements::( + proof, + claim_shape, + &[rule], + &[FieldElement::zero()], + |at: &[FieldElement]| { + let mut values = public_values(at)?; + values.push(eq_eval(&r, at)?); + Ok(values) + }, + config, + transcript, + )?; + Ok(()) +} + +/// No public factors, so nothing for the verifier to recompute. +pub fn no_public_factors( + _point: &[FieldElement], +) -> Result>, Error> { + Ok(Vec::new()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + gkr::{self, FractionLayer, FractionTree}, + selector::Selector, + whir_chain::GrindBits, + whir_hash::KeccakWhir, + }; + + type FE = FieldElement; + + /// ★ These tests are the KECCAK instantiation, stated once. + /// + /// `prove` and `verify` are generic over [`WhirHash`] on the production + /// path, where the caller supplies it. Shadowing them here with pinned + /// wrappers keeps every test body reading exactly as it did on PR #988 — + /// which is what makes "the existing tests are unchanged and + /// byte-identical" a checkable statement rather than a hopeful one — while + /// still naming the hash in one visible place. + #[allow(clippy::too_many_arguments)] + fn prove( + trace: &CommittedTrace, + combine: C, + degree: usize, + config: &ChainConfig, + transcript: &mut T, + ) -> Result, Error> + where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + C: Fn(&[FieldElement]) -> FieldElement + Sync, + { + super::prove::(trace, combine, degree, config, transcript) + } + + #[allow(clippy::too_many_arguments)] + fn verify( + proof: &ConstraintProof, + claim_shape: TraceClaim<'_, F>, + combine: C, + public_values: P, + degree: usize, + config: &ChainConfig, + transcript: &mut T, + ) -> Result<(), Error> + where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + C: Fn(&[FieldElement]) -> FieldElement + Sync, + P: FnOnce(&[FieldElement]) -> Result>, Error>, + { + super::verify::( + proof, + claim_shape, + combine, + public_values, + degree, + config, + transcript, + ) + } + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"constraint-argument-test") + } + + fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: crate::whir_chain::ChainFormat::DEFAULT, + } + } + + /// The constraint: `a·b − c = 0`. Degree 2, three columns — the shape a + /// real AIR relation takes. + fn constraint(v: &[FE]) -> FE { + v[0] * v[1] - v[2] + } + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|x| FE::from(*x)).collect()).unwrap() + } + + /// Columns that satisfy `a·b = c` on every row. + fn satisfying(num_vars: usize) -> Vec> { + let size = 1usize << num_vars; + let a: Vec = (0..size as u64).map(|i| i * 3 + 1).collect(); + let b: Vec = (0..size as u64).map(|i| i * 5 + 2).collect(); + let c: Vec = a.iter().zip(&b).map(|(x, y)| x * y).collect(); + vec![mle(&a), mle(&b), mle(&c)] + } + + fn run(columns: Vec>) -> Result<(), Error> { + let num_vars = columns[0].num_vars(); + let trace = CommittedTrace::::commit(columns, &config()).unwrap(); + let proof = prove(&trace, constraint, 2, &config(), &mut transcript())?; + + verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + constraint, + no_public_factors, + 2, + &config(), + &mut transcript(), + ) + } + + /// ★ A trace held narrow proves the same bytes as held wide: a host path + /// widens the packed words back, bit for bit, the first time it reads them. + #[test] + fn a_trace_held_narrow_proves_the_same_bytes() { + let columns = satisfying(4); + let wide = CommittedTrace::::commit(columns.clone(), &config()).unwrap(); + let mut narrow = CommittedTrace::::commit(columns.clone(), &config()).unwrap(); + assert!(narrow.data.pack_on_host()); + assert!(!narrow.data.pack_on_host(), "packed once"); + assert!(narrow.data.narrow().is_some()); + assert_eq!((narrow.data.num_columns(), narrow.num_vars()), (3, 4)); + let widens = crate::narrow::host_widens().0; + let bytes = |trace: &CommittedTrace| { + let proof = prove(trace, constraint, 2, &config(), &mut transcript()).unwrap(); + rkyv::to_bytes::(&proof) + .unwrap() + .to_vec() + }; + assert_eq!(bytes(&narrow), bytes(&wide)); + assert!( + crate::narrow::host_widens().0 > widens, + "the host widened it" + ); + assert_eq!(narrow.columns(), columns.as_slice()); + narrow.data.drop_widened(); + assert!(narrow.data.narrow().is_some(), "still packed"); + assert_eq!(narrow.columns(), columns.as_slice()); + } + + /// A table held as records whose host fill is `packed`. + #[derive(Debug)] + struct HostFilled(NarrowColumns); + + impl crate::records::TableRecords for HostFilled { + fn rows(&self) -> usize { + self.0.rows() + } + + fn cols(&self) -> usize { + self.0.cols() + } + + fn fill(&self) -> crate::records::RecordsFill<'_> { + crate::records::RecordsFill::KeccakRnd { ops: &[], skip: 0 } + } + + fn host_fill(&self) -> Option { + Some(self.0.clone()) + } + } + + /// ★ A trace held as the records its rows come from proves the same bytes + /// as held wide: a host reader gets the host fill's words, filled once + /// until the widening is dropped. The records are what it holds: it is + /// never packed, spilled or handed packed columns. + #[test] + fn a_trace_held_as_records_proves_the_same_bytes() { + let columns = satisfying(4); + let wide = CommittedTrace::::commit(columns.clone(), &config()).unwrap(); + let mut held = CommittedTrace::::commit(columns.clone(), &config()).unwrap(); + let records = Records(std::sync::Arc::new(HostFilled( + NarrowColumns::pack_columns(&columns).unwrap(), + ))); + held.data = TraceData::new_records(records.clone(), held.data.kinds().to_vec(), Vec::new()) + .unwrap(); + assert_eq!(held.data.records(), Some(&records)); + assert!(held.data.narrow().is_none()); + assert_eq!((held.data.num_columns(), held.num_vars()), (3, 4)); + assert!(!held.data.pack_on_host()); + assert!(held.data.take_narrow_for_spill().is_none()); + assert!( + !held + .data + .install_narrow(NarrowColumns::pack_columns(&columns).unwrap()), + "the records are what it holds" + ); + let fills = crate::records::fills().1; + let bytes = |trace: &CommittedTrace| { + let proof = prove(trace, constraint, 2, &config(), &mut transcript()).unwrap(); + rkyv::to_bytes::(&proof) + .unwrap() + .to_vec() + }; + assert_eq!(bytes(&held), bytes(&wide)); + assert_eq!( + crate::records::fills().1, + fills + 1, + "filled on the host once" + ); + assert_eq!(held.columns(), columns.as_slice()); + held.data.drop_widened(); + assert_eq!(held.data.records(), Some(&records), "still the records"); + assert_eq!(held.columns(), columns.as_slice()); + } + + /// A table held as its records gives them up to a spill and takes back + /// only records of its shape, and only while spilled; packed columns made + /// from them install in their place. + #[test] + fn records_leave_for_the_spill_and_come_back() { + let columns = satisfying(4); + let packed = || NarrowColumns::pack_columns(&columns).unwrap(); + let held = Records(std::sync::Arc::new(HostFilled(packed()))); + let mut table = CommittedTrace::::commit(columns.clone(), &config()).unwrap(); + table.data = + TraceData::new_records(held.clone(), table.data.kinds().to_vec(), Vec::new()).unwrap(); + assert!(!table.data.restore_records(held.clone()), "not spilled"); + let out = table.data.take_records_for_spill().expect("records out"); + assert_eq!(out, held); + assert!(table.data.is_spilled()); + assert!(table.data.take_records_for_spill().is_none(), "already out"); + let other: Vec> = (0..3).map(|_| vec![1; 8]).collect(); + let raw: Vec<&[u64]> = other.iter().map(Vec::as_slice).collect(); + let taller = Records(std::sync::Arc::new(HostFilled( + NarrowColumns::pack(&raw).unwrap(), + ))); + assert!(!table.data.restore_records(taller), "another shape"); + assert!(table.data.restore_records(out), "its own records"); + assert_eq!(table.data.records(), Some(&held)); + assert_eq!(table.columns(), columns.as_slice()); + table.data.take_records_for_spill().expect("records out"); + assert!( + table.data.restore_narrow(packed()), + "packed columns made from them" + ); + assert!(table.data.narrow().is_some()); + assert_eq!(table.columns(), columns.as_slice()); + } + + /// Records are held only for a table of a power of two rows, with columns, + /// over Goldilocks. + #[test] + fn records_of_no_table_are_refused() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + + let kinds = || (0..3).map(FactorKind::direct).collect::>(); + let held = |packed: NarrowColumns| Records(std::sync::Arc::new(HostFilled(packed))); + let words: Vec> = (0..3).map(|_| vec![1, 2, 3]).collect(); + let raw: Vec<&[u64]> = words.iter().map(Vec::as_slice).collect(); + let odd = held(NarrowColumns::pack(&raw).unwrap()); + assert!(TraceData::::new_records(odd, kinds(), Vec::new()).is_err()); + let none = held(NarrowColumns::pack(&[]).unwrap()); + assert!(TraceData::::new_records(none, Vec::new(), Vec::new()).is_err()); + let packed = || NarrowColumns::pack_columns(&satisfying(3)).unwrap(); + assert!(TraceData::::new_records(held(packed()), kinds(), Vec::new()).is_err()); + assert!(TraceData::::new_records(held(packed()), kinds(), Vec::new()).is_ok()); + } + + /// Packed columns are installed only with the table's shape, and only for + /// Goldilocks columns, whose elements are their raw words. + #[test] + fn packed_columns_of_another_shape_or_field_are_refused() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + + let columns = satisfying(3); + let kinds = || (0..3).map(FactorKind::direct).collect::>(); + let mut trace = TraceData::::new(columns.clone(), kinds(), Vec::new()).unwrap(); + let fewer = NarrowColumns::pack_columns(&columns[..2]).unwrap(); + assert!(!trace.install_narrow(fewer)); + let shorter = NarrowColumns::pack_columns(&satisfying(2)).unwrap(); + assert!(!trace.install_narrow(shorter)); + assert!(trace.narrow().is_none()); + assert!(trace.install_narrow(NarrowColumns::pack_columns(&columns).unwrap())); + assert_eq!(trace.columns(), columns.as_slice()); + + let other: Vec> = (0..3) + .map(|c| Mle::new((0..8u64).map(|r| FieldElement::from(r + c)).collect()).unwrap()) + .collect(); + let mut trace = TraceData::::new(other, kinds(), Vec::new()).unwrap(); + let words: Vec> = (0..3u64).map(|c| (0..8).map(|r| r + c).collect()).collect(); + let raw: Vec<&[u64]> = words.iter().map(Vec::as_slice).collect(); + assert!(!trace.install_narrow(NarrowColumns::pack(&raw).unwrap())); + assert!(!trace.pack_on_host()); + } + + /// A narrow table's packed columns go to a spill store and come back: in + /// between the table offers no columns, and only its own shape returns. + #[test] + fn a_spilled_table_holds_nothing_until_its_columns_come_back() { + let columns = satisfying(3); + let kinds = || (0..3).map(FactorKind::direct).collect::>(); + let mut trace = TraceData::::new(columns.clone(), kinds(), Vec::new()).unwrap(); + assert!( + trace.take_narrow_for_spill().is_none(), + "a wide table has none" + ); + assert!(!trace.is_spilled()); + assert!(trace.pack_on_host()); + let held = trace.narrow().cloned().unwrap(); + let packed = trace.take_narrow_for_spill().expect("narrow, so it spills"); + assert_eq!(packed, held); + assert!(trace.is_spilled()); + assert!(trace.narrow().is_none()); + assert!(trace.take_narrow_for_spill().is_none(), "spilled once"); + assert!(!trace.pack_on_host()); + let shorter = NarrowColumns::pack_columns(&satisfying(2)).unwrap(); + assert!(!trace.restore_narrow(shorter)); + assert!(trace.is_spilled(), "a refused restore leaves it spilled"); + assert!(trace.restore_narrow(packed)); + assert!(!trace.is_spilled()); + assert_eq!(trace.narrow(), Some(&held)); + assert_eq!(trace.columns(), columns.as_slice()); + assert!(!trace.restore_narrow(held), "only a spilled table restores"); + } + + #[test] + fn a_satisfying_trace_verifies() { + for num_vars in 2..=4usize { + run(satisfying(num_vars)).unwrap_or_else(|e| panic!("num_vars={num_vars}: {e:?}")); + } + } + + /// The whole point: a trace that breaks the constraint in one row must be + /// rejected, end to end, against its own commitments. + #[test] + fn a_trace_that_breaks_the_constraint_is_rejected() { + let mut columns = satisfying(3); + let mut c = columns[2].evals().to_vec(); + c[5] += FE::one(); + columns[2] = Mle::new(c).unwrap(); + + assert!(run(columns).is_err()); + } + + #[test] + fn a_forged_factor_value_is_rejected() { + let columns = satisfying(3); + let num_vars = 3; + let trace = CommittedTrace::::commit(columns, &config()).unwrap(); + let mut proof = prove(&trace, constraint, 2, &config(), &mut transcript()).unwrap(); + + // Claim a different value for one factor, leaving everything else. + proof.core.factor_values[0] += FE::one(); + + let err = verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + constraint, + no_public_factors, + 2, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::BatchMismatch); + } + + #[test] + fn verifying_a_different_constraint_is_rejected() { + let columns = satisfying(3); + let trace = CommittedTrace::::commit(columns, &config()).unwrap(); + let proof = prove(&trace, constraint, 2, &config(), &mut transcript()).unwrap(); + + // `a·b + c` instead of `a·b − c`: same degree, same columns. + let err = verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars: 3, + }, + |v: &[FE]| v[0] * v[1] + v[2], + no_public_factors, + 2, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::BatchMismatch); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let columns = satisfying(3); + let trace = CommittedTrace::::commit(columns, &config()).unwrap(); + let proof = prove(&trace, constraint, 2, &config(), &mut transcript()).unwrap(); + + let mut other = DefaultTranscript::::new(b"a-different-statement"); + assert!( + verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars: 3, + }, + constraint, + no_public_factors, + 2, + &config(), + &mut other, + ) + .is_err() + ); + } + + #[test] + fn a_proof_missing_a_factor_is_rejected() { + let columns = satisfying(3); + let trace = CommittedTrace::::commit(columns, &config()).unwrap(); + let mut proof = prove(&trace, constraint, 2, &config(), &mut transcript()).unwrap(); + proof.core.factor_values.pop(); + + let err = verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars: 3, + }, + constraint, + no_public_factors, + 2, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert!(matches!(err, Error::QueryCountMismatch { expected: 3, .. })); + } + + #[test] + fn columns_of_differing_heights_are_rejected() { + let err = CommittedTrace::::commit(vec![mle(&[1, 2]), mle(&[1, 2, 3, 4])], &config()) + .err() + .unwrap(); + assert!(matches!(err, Error::VariableCountMismatch { .. })); + } + + /// The field tower in use: a base-field evaluation domain with + /// extension-valued columns, which is the shape a real trace has. + #[test] + fn the_argument_runs_over_a_field_tower() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + type ExtE = FieldElement; + + let num_vars = 3; + let size = 1usize << num_vars; + // The trace is base-field; only the challenges and claims are not. + let a: Vec = (0..size as u64).map(|i| FE::from(i * 3 + 1)).collect(); + let b: Vec = (0..size as u64).map(|i| FE::from(i * 5 + 2)).collect(); + let c: Vec = a.iter().zip(&b).map(|(x, y)| x * y).collect(); + let columns = vec![ + Mle::new(a).unwrap(), + Mle::new(b).unwrap(), + Mle::new(c).unwrap(), + ]; + + let cfg = ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: crate::whir_chain::ChainFormat::DEFAULT, + }; + // Domain in Goldilocks, values in its degree-3 extension. + let trace = CommittedTrace::::commit(columns, &cfg).unwrap(); + let roots = trace.roots(); + let constraint = |v: &[ExtE]| v[0] * v[1] - v[2]; + + let mut prover_t = DefaultTranscript::::new(b"tower"); + let proof = prove::(&trace, constraint, 2, &cfg, &mut prover_t).unwrap(); + + let mut verifier_t = DefaultTranscript::::new(b"tower"); + verify::( + &proof, + TraceClaim { + roots: &roots, + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + constraint, + no_public_factors, + 2, + &cfg, + &mut verifier_t, + ) + .unwrap(); + } + + #[test] + fn a_broken_row_over_the_tower_is_rejected() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + type ExtE = FieldElement; + + let num_vars = 3; + let size = 1usize << num_vars; + let a: Vec = (0..size as u64).map(|i| FE::from(i * 3 + 1)).collect(); + let b: Vec = (0..size as u64).map(|i| FE::from(i * 5 + 2)).collect(); + let mut c: Vec = a.iter().zip(&b).map(|(x, y)| x * y).collect(); + c[4] += FE::one(); + let columns = vec![ + Mle::new(a).unwrap(), + Mle::new(b).unwrap(), + Mle::new(c).unwrap(), + ]; + + let cfg = ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: crate::whir_chain::ChainFormat::DEFAULT, + }; + let trace = CommittedTrace::::commit(columns, &cfg).unwrap(); + let roots = trace.roots(); + let constraint = |v: &[ExtE]| v[0] * v[1] - v[2]; + + let mut prover_t = DefaultTranscript::::new(b"tower"); + let proof = prove::(&trace, constraint, 2, &cfg, &mut prover_t).unwrap(); + + let mut verifier_t = DefaultTranscript::::new(b"tower"); + assert!( + verify::( + &proof, + TraceClaim { + roots: &roots, + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + constraint, + no_public_factors, + 2, + &cfg, + &mut verifier_t, + ) + .is_err() + ); + } + + // --------------------------------------------------------------- + // Shifted reads: a constraint that reads the next step, with no + // commitment of its own for the shifted view. + // --------------------------------------------------------------- + + /// `next(a) − a − b = 0`, with the next-step read as factor 2. + fn transition(v: &[FE]) -> FE { + v[2] - v[0] - v[1] + } + + /// Two columns where `b` is `a`'s cyclic forward difference, so the + /// transition holds on every row including the wrap. + fn cyclic_columns(num_vars: usize) -> Vec> { + let size = 1usize << num_vars; + let a: Vec = (0..size as u64).map(|i| FE::from(i * 7 + 3)).collect(); + let b: Vec = (0..size).map(|i| a[(i + 1) % size] - a[i]).collect(); + vec![Mle::new(a).unwrap(), Mle::new(b).unwrap()] + } + + fn transition_kinds() -> Vec { + vec![ + FactorKind::direct(0), + FactorKind::direct(1), + FactorKind::shifted(0, 1), + ] + } + + fn run_transition(columns: Vec>) -> Result<(), Error> { + let num_vars = columns[0].num_vars(); + let n_stack = one_stack(num_vars, 2); + let trace = CommittedTrace::::commit_views( + columns, + transition_kinds(), + Vec::new(), + n_stack, + &config(), + ) + .unwrap(); + let proof = prove(&trace, transition, 1, &config(), &mut transcript())?; + + // Three factors, two columns, and one commitment holding both: the + // shifted view rides on the column it shifts, and the columns ride in + // one stack. + assert_eq!(trace.kinds().len(), 3); + assert_eq!(trace.roots().len(), 1); + assert_eq!(proof.columns.polys.len(), 1); + + verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + transition, + no_public_factors, + 1, + &config(), + &mut transcript(), + ) + } + + #[test] + fn a_constraint_reading_the_next_step_verifies() { + for num_vars in 2..=4usize { + run_transition(cyclic_columns(num_vars)) + .unwrap_or_else(|e| panic!("num_vars={num_vars}: {e:?}")); + } + } + + #[test] + fn a_broken_transition_is_rejected() { + let mut columns = cyclic_columns(3); + let mut b = columns[1].evals().to_vec(); + b[5] += FE::one(); + columns[1] = Mle::new(b).unwrap(); + + assert!(run_transition(columns).is_err()); + } + + /// The shifted factor's value is bound to the column through the reduction, + /// so tampering with what the columns are worth at the reduced point is + /// caught before any commitment is opened. + #[test] + fn a_forged_reduced_column_value_is_rejected() { + let columns = cyclic_columns(3); + let trace = CommittedTrace::::commit_views( + columns, + transition_kinds(), + Vec::new(), + one_stack(3, 2), + &config(), + ) + .unwrap(); + let mut proof = prove(&trace, transition, 1, &config(), &mut transcript()).unwrap(); + proof.core.reduce.column_values[0] += FE::one(); + + let err = verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars: 3, + }, + transition, + no_public_factors, + 1, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::ShiftedReadMismatch); + } + + /// A verifier told the factor reads the column unshifted, when the prover + /// shifted it. The sources are the statement, so the two must agree. + #[test] + fn disagreeing_about_a_factor_s_offset_is_rejected() { + let columns = cyclic_columns(3); + let trace = CommittedTrace::::commit_views( + columns, + transition_kinds(), + Vec::new(), + one_stack(3, 2), + &config(), + ) + .unwrap(); + let proof = prove(&trace, transition, 1, &config(), &mut transcript()).unwrap(); + + let mut kinds = transition_kinds(); + kinds[2] = FactorKind::direct(0); + + assert!( + verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: &kinds, + layout: trace.layout(), + domain: trace.domain(), + num_vars: 3, + }, + transition, + no_public_factors, + 1, + &config(), + &mut transcript(), + ) + .is_err() + ); + } + + // --------------------------------------------------------------- + // Public factors: a selector the verifier recomputes instead of + // taking anyone's word for. + // --------------------------------------------------------------- + + /// `s(x)·(a·b − c) = 0`, with the selector as factor 3. + fn selected(v: &[FE]) -> FE { + v[3] * (v[0] * v[1] - v[2]) + } + + fn selector() -> Selector { + Selector::except_last(1) + } + + /// Columns satisfying `a·b = c` everywhere **but the last row**, which is + /// exactly what the selector exempts. + fn satisfying_except_the_last(num_vars: usize) -> Vec> { + let mut columns = satisfying(num_vars); + let mut c = columns[2].evals().to_vec(); + let last = c.len() - 1; + c[last] += FE::one(); + columns[2] = Mle::new(c).unwrap(); + columns + } + + fn selected_kinds() -> Vec { + vec![ + FactorKind::direct(0), + FactorKind::direct(1), + FactorKind::direct(2), + FactorKind::Public, + ] + } + + fn run_selected(columns: Vec>) -> Result<(), Error> { + let num_vars = columns[0].num_vars(); + let trace = CommittedTrace::::commit_views( + columns, + selected_kinds(), + vec![selector().table(num_vars)?], + one_stack(num_vars, 3), + &config(), + )?; + let proof = prove(&trace, selected, 3, &config(), &mut transcript())?; + + // Four factors, one commitment for the three committed columns, and + // only those three factors' values in the proof. + assert_eq!(trace.kinds().len(), 4); + assert_eq!(trace.roots().len(), 1); + assert_eq!(proof.columns.polys.len(), 1); + assert_eq!(proof.core.factor_values.len(), 3); + + verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + selected, + |point: &[FE]| Ok(vec![selector().evaluate(point)?]), + 3, + &config(), + &mut transcript(), + ) + } + + #[test] + fn a_public_factor_needs_no_commitment() { + for num_vars in 2..=4usize { + run_selected(satisfying_except_the_last(num_vars)) + .unwrap_or_else(|e| panic!("num_vars={num_vars}: {e:?}")); + } + } + + #[test] + fn the_selector_does_not_hide_a_violation_it_does_not_exempt() { + let mut columns = satisfying_except_the_last(3); + let mut c = columns[2].evals().to_vec(); + c[4] += FE::one(); + columns[2] = Mle::new(c).unwrap(); + + assert!(run_selected(columns).is_err()); + } + + /// The verifier computes the public factor itself, so it decides what the + /// table is: a prover proving against a different one is rejected. + #[test] + fn a_prover_using_another_public_table_is_rejected() { + let columns = satisfying_except_the_last(3); + // The prover folds a selector that exempts nothing, so its constraint + // is violated on the last row while the verifier's masks it. + let trace = CommittedTrace::::commit_views( + columns, + selected_kinds(), + vec![Selector::ALL.table(3).unwrap()], + one_stack(3, 3), + &config(), + ) + .unwrap(); + let proof = prove(&trace, selected, 3, &config(), &mut transcript()).unwrap(); + + assert!( + verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars: 3, + }, + selected, + |point: &[FE]| Ok(vec![selector().evaluate(point)?]), + 3, + &config(), + &mut transcript(), + ) + .is_err() + ); + } + + #[test] + fn a_public_value_the_prover_did_not_fold_is_rejected() { + let columns = satisfying_except_the_last(3); + let trace = CommittedTrace::::commit_views( + columns, + selected_kinds(), + vec![selector().table(3).unwrap()], + one_stack(3, 3), + &config(), + ) + .unwrap(); + let proof = prove(&trace, selected, 3, &config(), &mut transcript()).unwrap(); + + let err = verify( + &proof, + TraceClaim { + roots: &trace.roots(), + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars: 3, + }, + selected, + |point: &[FE]| Ok(vec![selector().evaluate(point)? + FE::one()]), + 3, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::BatchMismatch); + } + + #[test] + fn a_public_table_of_the_wrong_height_is_rejected() { + let err = CommittedTrace::::commit_views( + satisfying(3), + selected_kinds(), + vec![selector().table(2).unwrap()], + one_stack(3, 3), + &config(), + ) + .err() + .unwrap(); + assert!(matches!(err, Error::VariableCountMismatch { .. })); + } + + #[test] + fn a_public_table_per_public_factor_is_required() { + let err = CommittedTrace::::commit_views( + satisfying(3), + selected_kinds(), + Vec::new(), + one_stack(3, 3), + &config(), + ) + .err() + .unwrap(); + assert_eq!( + err, + Error::QueryCountMismatch { + expected: 1, + got: 0 + } + ); + } + + // --------------------------------------------------------------- + // The whole composition: a transition constraint and a bus, over one + // committed trace, in one sumcheck. + // --------------------------------------------------------------- + + /// Factor layout of the fused argument. + const V: usize = 0; + const MULT: usize = 1; + const ACC: usize = 2; + const ACC_NEXT: usize = 3; + const SEL: usize = 4; + const EQ_R: usize = 5; + const EQ_Z: usize = 6; + + fn fused_kinds() -> Vec { + vec![ + FactorKind::direct(0), + FactorKind::direct(1), + FactorKind::direct(2), + FactorKind::shifted(2, 1), + FactorKind::Public, + ] + } + + /// The three statements, all indexing the one factor list: + /// + /// - `Σ_x eq(r,x)·s(x)·(next(acc) − acc − v) = 0`, the transition; + /// - `Σ_x eq(z,x)·mult = p(z)`, the bus's numerator claim; + /// - `Σ_x eq(z,x)·(alpha − v) = q(z)`, its denominator claim. + /// + /// The denominator is never a column: it is read off the committed `v`. + fn fused_rules(alpha: FE) -> Vec> { + vec![ + Rule::new(3, |f: &[FE]| { + f[EQ_R] * f[SEL] * (f[ACC_NEXT] - f[ACC] - f[V]) + }), + Rule::new(2, |f: &[FE]| f[EQ_Z] * f[MULT]), + Rule::new(2, move |f: &[FE]| f[EQ_Z] * (alpha - f[V])), + ] + } + + /// A trace with both shapes at once: a bus whose sends and receives cancel, + /// and a running sum whose recurrence holds off the wrap. + fn fused_columns(num_vars: usize) -> Vec> { + let size = 1usize << num_vars; + let half = size / 2; + let v: Vec = (0..size).map(|i| FE::from((i % half) as u64 + 1)).collect(); + let mult: Vec = (0..size) + .map(|i| if i < half { FE::one() } else { -FE::one() }) + .collect(); + let mut acc = vec![FE::zero()]; + for i in 1..size { + acc.push(acc[i - 1] + v[i - 1]); + } + vec![ + Mle::new(v).unwrap(), + Mle::new(mult).unwrap(), + Mle::new(acc).unwrap(), + ] + } + + /// Proves and verifies the fused argument, returning the sumcheck's round + /// count so a caller can check it really was one pass. + fn argue_table(columns: Vec>, num_vars: usize) -> Result { + let selector = Selector::except_last(1); + let trace = CommittedTrace::::commit_views( + columns, + fused_kinds(), + vec![selector.table(num_vars)?], + one_stack(num_vars, 3), + &config(), + )?; + let roots = trace.roots(); + + // ---- prover ---- + let mut prover = transcript(); + for root in &roots { + prover.append_bytes(root); + } + // The fingerprint challenge comes after the commitments. + let alpha: FE = prover.sample_field_element(); + + let v = trace.columns()[0].clone(); + let mult = trace.columns()[1].clone(); + let denominator = Mle::new(v.evals().iter().map(|x| alpha - x).collect())?; + let tree = FractionTree::build(FractionLayer::new(mult, denominator)?)?; + let (out_p, out_q) = tree.output(); + let gkr_out = gkr::prove(&tree, &mut prover)?; + + let r: Vec = (0..num_vars) + .map(|_| prover.sample_field_element()) + .collect(); + let z = gkr_out.claim.point.clone(); + let claims = [FE::zero(), gkr_out.claim.p, gkr_out.claim.q]; + + let proof = prove_statements( + &trace, + vec![eq_mle(&r)?, eq_mle(&z)?], + fused_rules(alpha), + &claims, + &config(), + &mut prover, + )?; + + // ---- verifier ---- + let mut verifier = transcript(); + for root in &roots { + verifier.append_bytes(root); + } + let alpha: FE = verifier.sample_field_element(); + // The bus balances exactly when the output numerator is zero; the + // denominator is the only part of the output that has to be sent. + let gkr_claim = gkr::verify(&gkr_out.proof, (FE::zero(), out_q), &mut verifier)?; + let vr: Vec = (0..num_vars) + .map(|_| verifier.sample_field_element()) + .collect(); + + verify_statements::( + &proof, + TraceClaim { + roots: &roots, + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + &fused_rules(alpha), + &claims, + // Trace publics first — the selector — then one per weight table. + |at: &[FE]| { + Ok(vec![ + selector.evaluate(at)?, + eq_eval(&vr, at)?, + eq_eval(&gkr_claim.point, at)?, + ]) + }, + &config(), + &mut verifier, + )?; + + assert_eq!(out_p, FE::zero(), "the bus was expected to balance"); + // Five factors over three columns, all in one commitment and settled by + // one opening. + assert_eq!(trace.kinds().len(), 5); + assert_eq!(roots.len(), 1); + assert_eq!(proof.columns.polys.len(), 1); + assert_eq!(proof.core.factor_values.len(), 4); + Ok(proof.core.sumcheck.rounds.len()) + } + + /// The composition this crate is being built for: a constraint that reads + /// the next step and a bus that has to balance, over three committed + /// columns, settled in **one** sumcheck with **one** opening per column. + #[test] + fn a_constraint_and_a_bus_argue_in_one_sumcheck() { + for num_vars in 2..=4usize { + let rounds = argue_table(fused_columns(num_vars), num_vars) + .unwrap_or_else(|e| panic!("num_vars={num_vars}: {e:?}")); + assert_eq!(rounds, num_vars, "one pass over the cube, not one each"); + } + } + + #[test] + fn a_broken_transition_breaks_the_fused_argument() { + let mut columns = fused_columns(3); + let mut acc = columns[2].evals().to_vec(); + acc[4] += FE::one(); + columns[2] = Mle::new(acc).unwrap(); + + assert!(argue_table(columns, 3).is_err()); + } + + #[test] + fn an_unbalanced_bus_breaks_the_fused_argument() { + // The verifier requires a zero output numerator, so a bus that does not + // cancel cannot get past the GKR. + let mut columns = fused_columns(3); + let mut mult = columns[1].evals().to_vec(); + mult[2] = FE::from(3); + columns[1] = Mle::new(mult).unwrap(); + + assert!(argue_table(columns, 3).is_err()); + } + + /// The transition and the bus both read `v` and it is folded once, and the + /// three columns are one commitment and one opening. + #[test] + fn columns_shared_by_two_statements_are_opened_once() { + argue_table(fused_columns(3), 3).unwrap(); + } +} diff --git a/crypto/multilinear/src/eq.rs b/crypto/multilinear/src/eq.rs new file mode 100644 index 000000000..6bfd6610e --- /dev/null +++ b/crypto/multilinear/src/eq.rs @@ -0,0 +1,611 @@ +//! The equality kernel `eq(r, x) = ∏_i (r_i·x_i + (1 - r_i)(1 - x_i))` and the +//! shift kernel, `shift_k(x, y) = 1` iff `index(y) = index(x) + k mod 2^n`, of +//! which the cyclic rotation `rot` is the `k = 1` case. + +use math::field::{element::FieldElement, traits::IsField}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{Error, mle::Mle}; + +/// Builds the table of `eq(r, x)` for every `x` in `{0,1}^n`, in `O(2^n)`. +/// +/// Doubles the table one variable at a time. Each step prepends its variable as +/// the new most significant bit, so the variables are consumed **back to +/// front**: that leaves variable 0 in the high bit, which is the indexing +/// convention [`Mle`](crate::mle::Mle) folds on. +pub fn eq_evals(r: &[FieldElement]) -> Vec> +where + FieldElement: Send + Sync, +{ + let mut table = vec![FieldElement::::zero(); 1usize << r.len()]; + // The length is `2^r.len()` by construction. + eq_evals_into(r, &FieldElement::::one(), &mut table).expect("the table is sized for r"); + table +} + +/// The same table, scaled by `by`, written into `dst`. +/// +/// The scale rides the seed rather than a pass over the result: the table is a +/// product over the variables, so one more factor at the start scales every +/// entry. And the doubling happens in place — a caller that wants the table +/// inside a bigger one (a stacked polynomial's weight, say) hands over that +/// range and pays no copy. +pub fn eq_evals_into( + r: &[FieldElement], + by: &FieldElement, + dst: &mut [FieldElement], +) -> Result<(), Error> +where + FieldElement: Send + Sync, +{ + if dst.len() != 1usize << r.len() { + return Err(Error::VariableCountMismatch { + expected: 1usize << r.len(), + got: dst.len(), + }); + } + dst[0] = by.clone(); + for (level, r_i) in r.iter().rev().enumerate() { + let one_minus = FieldElement::::one() - r_i; + // Each half of the doubled table is an independent scaling of the + // current one, and the halves are disjoint — so this is one pass over + // the level, and the last levels are the whole cube, which is where the + // pool is worth it. + let half = 1usize << level; + let (lo, hi) = dst[..half * 2].split_at_mut(half); + let scale = |(l, h): (&mut FieldElement, &mut FieldElement)| { + *h = &*l * r_i; + *l = &*l * &one_minus; + }; + #[cfg(feature = "parallel")] + if half >= crate::SERIAL_BELOW { + lo.par_iter_mut().zip(hi.par_iter_mut()).for_each(scale); + } else { + lo.iter_mut().zip(hi.iter_mut()).for_each(scale); + } + #[cfg(not(feature = "parallel"))] + lo.iter_mut().zip(hi.iter_mut()).for_each(scale); + } + Ok(()) +} + +/// The multilinear extension of `eq(r, ·)`. +pub fn eq_mle(r: &[FieldElement]) -> Result, Error> { + Mle::new(eq_evals(r)) +} + +/// Evaluates `eq(r, x)` directly, without materializing the table. +pub fn eq_eval( + r: &[FieldElement], + x: &[FieldElement], +) -> Result, Error> { + if r.len() != x.len() { + return Err(Error::VariableCountMismatch { + expected: r.len(), + got: x.len(), + }); + } + let one = FieldElement::::one(); + Ok(r.iter().zip(x).fold(one.clone(), |acc, (r_i, x_i)| { + acc * (r_i * x_i + (&one - r_i) * (&one - x_i)) + })) +} + +/// `rot` and `eq` together; the recursion needs both. +/// +/// `next_f(z) = Σ_y rot(z, y)·f(y)`, which is how a rotated table gets bound to +/// the committed column. +pub fn eq_and_rot_eval( + x: &[FieldElement], + y: &[FieldElement], +) -> Result<(FieldElement, FieldElement), Error> { + if x.len() != y.len() { + return Err(Error::VariableCountMismatch { + expected: x.len(), + got: y.len(), + }); + } + let one = FieldElement::::one(); + let mut eq = one.clone(); + let mut rot = one.clone(); + + // Recursion: rot(x, y) = y₀(1−x₀)·eq(rest) + (1−y₀)x₀·rot(rest), unrolled + // with the least significant variable outermost. `rot` on the tail carries + // "the low bits wrapped", which is what makes the next bit up increment. + // + // Our variable 0 is the most significant bit, so the fold runs front to + // back — the opposite of the usual least-significant-first presentation. + for (x_i, y_i) in x.iter().zip(y) { + rot = y_i * (&one - x_i) * &eq + (&one - y_i) * x_i * &rot; + eq *= x_i * y_i + (&one - x_i) * (&one - y_i); + } + Ok((eq, rot)) +} + +/// Just the rotation kernel. See [`eq_and_rot_eval`]. +pub fn rot_eval( + x: &[FieldElement], + y: &[FieldElement], +) -> Result, Error> { + Ok(eq_and_rot_eval(x, y)?.1) +} + +/// `shift_k(x, y) = 1` iff `index(y) = index(x) + k mod 2^n`, extended +/// multilinearly. `k = 0` is [`eq_eval`], `k = 1` is [`rot_eval`]. +/// +/// Adds the constant `k` bit by bit from the least significant end, which is +/// variable `n − 1`. `state[c]` is the weight of the bits handled so far having +/// produced carry `c`; the shift wraps, so both carries are accepted at the end. +pub fn shift_eval( + x: &[FieldElement], + y: &[FieldElement], + k: usize, +) -> Result, Error> { + if x.len() != y.len() { + return Err(Error::VariableCountMismatch { + expected: x.len(), + got: y.len(), + }); + } + let one = FieldElement::::one(); + let zero = FieldElement::::zero(); + let mut state = [one.clone(), zero.clone()]; + + for (t, (x_j, y_j)) in x.iter().zip(y).rev().enumerate() { + let eq_j = x_j * y_j + (&one - x_j) * (&one - y_j); + // y_j = 1 − x_j: x_j = 1 carries out, x_j = 0 does not. + let carry = x_j * (&one - y_j); + let no_carry = (&one - x_j) * y_j; + + let mut next = [zero.clone(), zero.clone()]; + for (c, weight) in state.iter().enumerate() { + match ((k >> t) & 1) + c { + 0 => next[0] += weight * &eq_j, + 1 => { + next[0] += weight * &no_carry; + next[1] += weight * &carry; + } + _ => next[1] += weight * &eq_j, + } + } + state = next; + } + + Ok(&state[0] + &state[1]) +} + +/// The table of `shift_k(x, ·)` over the cube, in `O(2^n)`. +/// +/// Same carry recursion as [`shift_eval`], but each step doubles the tables +/// instead of multiplying `y_j`'s weight out — prepending `y_j` as the new most +/// significant bit, which is the indexing [`Mle`] folds on. +/// +/// The two tables ride buffers the levels swap between rather than being +/// allocated per level: the last level alone is the whole cube, so allocating +/// per level is allocating the answer twice over. +pub fn shift_evals(x: &[FieldElement], k: usize) -> Vec> +where + FieldElement: Send + Sync, +{ + let size = 1usize << x.len(); + // Only the low `n` bits of `k` are ever read, so a shift by a multiple of + // the cube is no shift — and no shift is `eq`, which carries one table + // instead of two and is one pass per level instead of three. + if k.is_multiple_of(size) { + return eq_evals(x); + } + + let one = FieldElement::::one(); + let zero = FieldElement::::zero(); + // "No carry out" and "carry out". Only the seed has to be set: every cell + // a level reads is one that level or an earlier one wrote. + let mut cur = [vec![zero.clone(); size], vec![zero.clone(); size]]; + let mut next = [vec![zero.clone(); size], vec![zero; size]]; + cur[0][0] = one.clone(); + + let mut len = 1usize; + for (t, x_j) in x.iter().rev().enumerate() { + let one_minus = &one - x_j; + { + let [s0, s1] = &cur; + let (s0, s1) = (&s0[..len], &s1[..len]); + let [n0, n1] = &mut next; + let (n0_lo, n0_rest) = n0.split_at_mut(len); + let (n1_lo, n1_rest) = n1.split_at_mut(len); + let n0_hi = &mut n0_rest[..len]; + let n1_hi = &mut n1_rest[..len]; + // Every level is made of the same three blocks and one of zeros: + // b = (1 − x)·s0, m = x·s0 + (1 − x)·s1, h = x·s1 + // and all the shift's bit decides is where each one goes. + let (b, m, h, blank) = if (k >> t) & 1 == 0 { + (n0_lo, n0_hi, n1_lo, n1_hi) + } else { + (n0_hi, n1_lo, n1_hi, n0_lo) + }; + let scale_lo = |(o, v): (&mut FieldElement, &FieldElement)| *o = &one_minus * v; + let scale_hi = |(o, v): (&mut FieldElement, &FieldElement)| *o = x_j * v; + let mix = + |((o, a), c): ((&mut FieldElement, &FieldElement), &FieldElement)| { + *o = x_j * a + &one_minus * c + }; + let clear = |o: &mut FieldElement| *o = FieldElement::::zero(); + #[cfg(feature = "parallel")] + { + b.par_iter_mut().zip(s0.par_iter()).for_each(scale_lo); + h.par_iter_mut().zip(s1.par_iter()).for_each(scale_hi); + m.par_iter_mut() + .zip(s0.par_iter()) + .zip(s1.par_iter()) + .for_each(mix); + blank.par_iter_mut().for_each(clear); + } + #[cfg(not(feature = "parallel"))] + { + b.iter_mut().zip(s0.iter()).for_each(scale_lo); + h.iter_mut().zip(s1.iter()).for_each(scale_hi); + m.iter_mut().zip(s0.iter()).zip(s1.iter()).for_each(mix); + blank.iter_mut().for_each(clear); + } + } + len *= 2; + core::mem::swap(&mut cur, &mut next); + } + + let [no_carry, carried] = cur; + let add = |(a, b): (FieldElement, FieldElement)| a + b; + #[cfg(feature = "parallel")] + return no_carry.into_par_iter().zip(carried).map(add).collect(); + #[cfg(not(feature = "parallel"))] + return no_carry.into_iter().zip(carried).map(add).collect(); +} + +/// The multilinear extension of `shift_k(x, ·)`. +pub fn shift_mle(x: &[FieldElement], k: usize) -> Result, Error> +where + FieldElement: Send + Sync, +{ + Mle::new(shift_evals(x, k)) +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn point(vals: &[u64]) -> Vec { + vals.iter().map(|v| FE::from(*v)).collect() + } + + /// The hypercube corner for `index`, variable 0 most significant. + fn corner(index: usize, num_vars: usize) -> Vec { + (0..num_vars) + .map(|i| FE::from(((index >> (num_vars - 1 - i)) & 1) as u64)) + .collect() + } + + #[test] + fn is_the_indicator_on_the_hypercube() { + let n = 3; + for r_idx in 0..(1u64 << n) { + let r = point(&[(r_idx >> 2) & 1, (r_idx >> 1) & 1, r_idx & 1]); + let table = eq_evals(&r); + assert_eq!(table.len(), 1 << n); + for (x_idx, value) in table.iter().enumerate() { + let expected = if x_idx as u64 == r_idx { + FE::one() + } else { + FE::zero() + }; + assert_eq!(value, &expected, "eq(r={r_idx}, x={x_idx})"); + } + } + } + + #[test] + fn table_and_direct_evaluation_agree_on_corners() { + let r = point(&[5, 9, 2]); + for (x_idx, entry) in eq_evals(&r).into_iter().enumerate() { + let x = point(&[ + ((x_idx >> 2) & 1) as u64, + ((x_idx >> 1) & 1) as u64, + (x_idx & 1) as u64, + ]); + assert_eq!(entry, eq_eval(&r, &x).unwrap()); + } + } + + #[test] + fn table_is_the_multilinear_extension() { + // Evaluating the eq table as an MLE off the cube must match the + // product formula. + let r = point(&[3, 11]); + let mle = eq_mle(&r).unwrap(); + let x = point(&[7, 13]); + assert_eq!(mle.evaluate(&x).unwrap(), eq_eval(&r, &x).unwrap()); + } + + #[test] + fn sums_to_one_over_the_cube() { + // Σ_x eq(r, x) = 1 for any r, since eq interpolates a single corner. + let r = point(&[4, 6, 8]); + let total = eq_evals(&r).into_iter().fold(FE::zero(), |acc, v| acc + v); + assert_eq!(total, FE::one()); + } + + #[test] + fn is_symmetric_in_its_arguments() { + let a = point(&[2, 3]); + let b = point(&[5, 7]); + assert_eq!(eq_eval(&a, &b).unwrap(), eq_eval(&b, &a).unwrap()); + } + + #[test] + fn zero_variables_gives_the_empty_product() { + assert_eq!(eq_evals::(&[]), vec![FE::one()]); + assert_eq!(eq_eval::(&[], &[]).unwrap(), FE::one()); + } + + #[test] + fn rejects_mismatched_arity() { + assert_eq!( + eq_eval(&point(&[1, 2]), &point(&[1])).unwrap_err(), + Error::VariableCountMismatch { + expected: 2, + got: 1 + } + ); + } + + #[test] + fn rot_is_the_successor_indicator_on_the_hypercube() { + for num_vars in 1..=4usize { + let size = 1usize << num_vars; + for xi in 0..size { + for yi in 0..size { + let x = corner(xi, num_vars); + let y = corner(yi, num_vars); + let expected = if yi == (xi + 1) % size { + FE::one() + } else { + FE::zero() + }; + assert_eq!( + rot_eval(&x, &y).unwrap(), + expected, + "n={num_vars}, x={xi}, y={yi}" + ); + } + } + } + } + + #[test] + fn rot_wraps_the_last_index_to_the_first() { + let n = 3; + let last = corner(7, n); + let first = corner(0, n); + assert_eq!(rot_eval(&last, &first).unwrap(), FE::one()); + } + + #[test] + fn rot_reproduces_a_shifted_table() { + // The identity the commitment scheme will lean on: + // next_f(z) = Σ_y rot(z, y)·f(y), for z on the cube. + let n = 3; + let size = 1usize << n; + let f: Vec = (0..size as u64).map(|i| FE::from(i * 7 + 5)).collect(); + + for zi in 0..size { + let z = corner(zi, n); + let summed = (0..size).fold(FE::zero(), |acc, yi| { + acc + rot_eval(&z, &corner(yi, n)).unwrap() * f[yi] + }); + assert_eq!(summed, f[(zi + 1) % size], "z={zi}"); + } + } + + #[test] + fn rot_matches_its_table_off_the_cube() { + // The closed form must be the multilinear extension of the table, so + // the verifier can evaluate it at a random point. + let n = 3; + let size = 1usize << n; + let z = point(&[5, 9, 2]); + + // Brute-force the MLE of rot(·, y) for a fixed y by extending the table. + for yi in 0..size { + let table: Vec = (0..size) + .map(|xi| rot_eval(&corner(xi, n), &corner(yi, n)).unwrap()) + .collect(); + let mle = Mle::new(table).unwrap(); + assert_eq!( + mle.evaluate(&z).unwrap(), + rot_eval(&z, &corner(yi, n)).unwrap(), + "y={yi}" + ); + } + } + + #[test] + fn eq_and_rot_agree_with_the_standalone_helpers() { + let x = point(&[3, 11, 4]); + let y = point(&[7, 13, 2]); + let (eq, rot) = eq_and_rot_eval(&x, &y).unwrap(); + assert_eq!(eq, eq_eval(&x, &y).unwrap()); + assert_eq!(rot, rot_eval(&x, &y).unwrap()); + } + + #[test] + fn rot_rejects_mismatched_arity() { + assert!(rot_eval(&point(&[1, 2]), &point(&[1])).is_err()); + } + + #[test] + fn shift_is_the_offset_indicator_on_the_hypercube() { + for num_vars in 1..=4usize { + let size = 1usize << num_vars; + // Past `size` too, so wrapping is exercised as its own case. + for k in 0..(size + 3) { + for xi in 0..size { + for yi in 0..size { + let expected = if yi == (xi + k) % size { + FE::one() + } else { + FE::zero() + }; + assert_eq!( + shift_eval(&corner(xi, num_vars), &corner(yi, num_vars), k).unwrap(), + expected, + "n={num_vars}, k={k}, x={xi}, y={yi}" + ); + } + } + } + } + } + + #[test] + fn shift_zero_is_eq_and_shift_one_is_rot() { + // Off the cube, where agreeing on corners would not be enough. + let x = point(&[3, 11, 4]); + let y = point(&[7, 13, 2]); + assert_eq!(shift_eval(&x, &y, 0).unwrap(), eq_eval(&x, &y).unwrap()); + assert_eq!(shift_eval(&x, &y, 1).unwrap(), rot_eval(&x, &y).unwrap()); + } + + #[test] + fn shift_table_matches_the_closed_form() { + let x = point(&[5, 9, 2]); + for k in 0..10usize { + for (yi, entry) in shift_evals(&x, k).into_iter().enumerate() { + assert_eq!(entry, shift_eval(&x, &corner(yi, 3), k).unwrap(), "k={k}"); + } + } + } + + #[test] + fn shift_table_is_the_multilinear_extension() { + // The verifier evaluates the kernel at a random point while the prover + // folds the table, so the two must be the same polynomial. + let x = point(&[3, 11, 4]); + let z = point(&[7, 13, 2]); + for k in 0..8usize { + assert_eq!( + shift_mle(&x, k).unwrap().evaluate(&z).unwrap(), + shift_eval(&x, &z, k).unwrap(), + "k={k}" + ); + } + } + + #[test] + fn shift_reproduces_a_shifted_column() { + // The identity the reduction leans on: + // f_shift_k(z) = Σ_y shift_k(z, y)·f(y). + let n = 3; + let size = 1usize << n; + let f: Vec = (0..size as u64).map(|i| FE::from(i * 7 + 5)).collect(); + let z = point(&[5, 9, 2]); + + for k in 0..6usize { + let shifted = Mle::new((0..size).map(|i| f[(i + k) % size]).collect()).unwrap(); + let summed = shift_evals(&z, k) + .into_iter() + .zip(&f) + .fold(FE::zero(), |acc, (w, v)| acc + w * v); + assert_eq!(summed, shifted.evaluate(&z).unwrap(), "k={k}"); + } + } + + #[test] + fn shift_sums_to_one_over_the_cube() { + // The kernel picks out one corner, so its extension sums to one at any + // point — a cheap check that the carry recursion loses no weight. + let x = point(&[4, 6, 8]); + for k in 0..10usize { + let total = shift_evals(&x, k) + .into_iter() + .fold(FE::zero(), |acc, v| acc + v); + assert_eq!(total, FE::one(), "k={k}"); + } + } + + #[test] + fn shift_rejects_mismatched_arity() { + assert!(shift_eval(&point(&[1, 2]), &point(&[1]), 1).is_err()); + } + + /// `shift_evals(x, k)[y] == eq_evals(x)[(y − k) mod 2^n]` for every `y`. + fn shift_is_rotated_eq(x: &[FieldElement], offsets: &[usize]) + where + FieldElement: Send + Sync, + { + let eq = eq_evals(x); + let size = eq.len(); + for &k in offsets { + let shift = shift_evals(x, k); + assert_eq!(shift.len(), size); + for (y, value) in shift.iter().enumerate() { + assert_eq!( + value, + &eq[(y + size - k % size) % size], + "n={}, k={k}, y={y}", + x.len() + ); + } + } + } + + /// ★ The identity the claim reduce's tables on the card stand on + /// (`LAMBDA_VM_ARGUE_DEVICE_TABLES`): a shift table is the `eq` table read + /// `k` rows back, cyclically, so the card builds `eq(x, ·)` once and each + /// offset's table is a rotated copy of it. + /// + /// For a corner `y`, `shift_k(x, y)` is the indicator of `x = y − k` + /// extended multilinearly in `x`, which is `eq(x, y − k)`. The carry + /// recursion has to give that value for value, off the cube, at every + /// offset — every one below the cube's size for the small cubes, and the + /// ones past it, which wrap — in the base field and in the extension the + /// prover runs in. + #[test] + fn a_shift_table_is_the_eq_table_rotated() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + + for n in 1..=12usize { + let size = 1usize << n; + let offsets: Vec = if n <= 6 { + (0..size + 3).collect() + } else { + vec![ + 0, + 1, + 2, + 3, + 7, + size / 2, + size / 2 + 1, + size - 1, + size, + size + 1, + 2 * size + 5, + ] + }; + let base: Vec = (0..n as u64).map(|i| FE::from(3 * i + 5)).collect(); + shift_is_rotated_eq(&base, &offsets); + let ext: Vec> = (0..n as u64) + .map(|i| { + FieldElement::::new([ + FE::from(7 * i + 2), + FE::from(i + 11), + FE::from(5 * i * i + 3), + ]) + }) + .collect(); + shift_is_rotated_eq(&ext, &offsets); + } + } +} diff --git a/crypto/multilinear/src/front_loaded.rs b/crypto/multilinear/src/front_loaded.rs new file mode 100644 index 000000000..081920af1 --- /dev/null +++ b/crypto/multilinear/src/front_loaded.rs @@ -0,0 +1,579 @@ +//! One sumcheck over several polynomials of DIFFERENT sizes, front-loaded +//! (SWIRL Protocol 2.7.3; D-BATCH §2.2, phase C). +//! +//! Polynomial `t` has `n_t` variables and weight `w_t`. It is padded virtually +//! to `n_max` by the product of the variables it lacks: +//! +//! ```text +//! f̃_t(X) = f_t(X_{(point: &[FieldElement], vars: usize) -> FieldElement { + point + .iter() + .skip(vars) + .fold(FieldElement::one(), |acc, r| acc * r) +} + +/// One polynomial's side of a front-loaded batch: its round messages on +/// demand, bound by the batch's shared challenges. +/// +/// The host's is any [`SumcheckPolynomial`]; a device-resident table brings +/// its own (the fused zerocheck's stepper), and the batch cannot tell them +/// apart — which is what makes a device proof the host reference's bytes. +pub trait RoundStepper { + /// Variables left to bind. + fn num_vars(&self) -> usize; + /// The degree of its round polynomials. + fn degree(&self) -> usize; + /// This round's polynomial at `1..=degree` (`degree` at least its own). + fn round(&mut self, degree: usize) -> Result>, Error>; + /// Binds this round's variable to `r`. + fn bind(&mut self, r: &FieldElement) -> Result<(), Error>; + /// Its value once every variable is bound. + fn value(&self) -> Result, Error>; + /// Queues this round's work on a device without waiting for it; the + /// batch queues every table's before reading any back, so a round costs + /// one wait rather than one a table. + fn prefetch(&mut self) -> Result<(), Error> { + Ok(()) + } +} + +/// A host polynomial's rounds: [`round_evaluations`] and a fold, as the +/// plain sumcheck runs them. +macro_rules! host_stepper { + ($($ty:ty),*) => {$( + impl RoundStepper for $ty + where + F: IsField + 'static, + FieldElement: Send + Sync, + { + fn num_vars(&self) -> usize { + SumcheckPolynomial::num_vars(self) + } + + fn degree(&self) -> usize { + SumcheckPolynomial::degree(self) + } + + fn round(&mut self, degree: usize) -> Result>, Error> { + Ok(round_evaluations(self, degree, false)) + } + + fn bind(&mut self, r: &FieldElement) -> Result<(), Error> { + self.fix_first_variable(r) + } + + fn value(&self) -> Result, Error> { + if SumcheckPolynomial::num_vars(self) != 0 { + return Err(Error::VariableCountMismatch { + expected: 0, + got: SumcheckPolynomial::num_vars(self), + }); + } + Ok(self.eval_at_index(0)) + } + } + )*}; +} + +host_stepper!( + crate::batch::Batched<'_, F>, + crate::virtual_poly::VirtualPolynomial +); + +/// One table's rule in the batched constraint sumcheck (D-BATCH phase C): its +/// factors and weights on the host, or its fused rounds on a device down to +/// today's crossover and the host's over what the card folded from there. +/// Either way the same messages, so a device proof is the host reference's. +pub enum TableRounds<'a, E: IsField> { + Host(crate::batch::Batched<'a, E>), + #[cfg(feature = "cuda")] + Fused { + stepper: Option>>, + /// The rules and weights, adopting the factors at the crossover. + batched: crate::batch::Batched<'a, E>, + /// Variables left to bind. + vars: usize, + /// The program is a long one ([`crate::gpu_fused::TailTimes`]'s `[1]`). + long: bool, + }, +} + +impl<'a, E> TableRounds<'a, E> +where + E: IsField + 'static, + FieldElement: Send + Sync, +{ + /// A table's rounds, on its device factors — or its base columns on the + /// card, `columns`, when nothing was lifted — when the fused rounds take + /// them, else on the host over `factors()` and the two weights built from + /// their points. + /// + /// `claim` is the table's own `Σ λ^j·claim_j` — what the fused rounds + /// carry to reconstruct `A(1)`. The rules are `[zerocheck, numerator, + /// denominator]` and `lambdas` their `[1, λ, λ²]`, today's batch. + #[allow(clippy::too_many_arguments)] + pub fn new( + rules: Vec>, + lambdas: Vec>, + claim: &FieldElement, + weights: [&[FieldElement]; 2], + num_vars: usize, + device: Option<&'a crate::gpu::DeviceFactors>, + columns: Option<&'a crate::gpu::ColumnFactors>, + fused: Option>, + factors: P, + ) -> Result + where + B: IsField + math::field::traits::IsSubFieldOf + 'static, + P: FnOnce() -> Result>, Error>, + { + // The fused rounds read the lifted factors when a device holds them, + // else the table's base columns where they lie (no lift, D-BATCH M1-2). + #[cfg(feature = "cuda")] + let source = fused.as_ref().and_then(|_| { + device + .map(crate::gpu_fused::FusedSource::Lifted) + .or(columns.map(crate::gpu_fused::FusedSource::Columns)) + }); + #[cfg(feature = "cuda")] + if let (Some(source), Some(input)) = (source, fused.as_ref()) + && crate::gpu_fused::argue_fused() + { + let degree = rules + .iter() + .map(crate::batch::Rule::degree) + .max() + .unwrap_or(0) + .max(1); + let check = crate::gpu_fused::argue_fused_xcheck(); + match crate::gpu_fused::FusedStepper::new(source, input, &lambdas, claim, degree, check) + { + Some(Ok(stepper)) => { + crate::whir_split::bump(&crate::whir_split::ZC_FUSED); + let batched = crate::batch::Batched::new(Vec::new(), rules, lambdas)?; + let long = stepper.long(); + return Ok(Self::Fused { + stepper: Some(Box::new(stepper)), + batched, + vars: num_vars, + long, + }); + } + Some(Err(error)) => return Err(error), + None => { + crate::gpu_fused::note_decline(); + crate::whir_split::bump(&crate::whir_split::ZC_FUSED_DECLINED); + } + } + } + let _ = (device, columns, &fused, claim, num_vars); + let mut polys = factors()?; + polys.push(crate::eq::eq_mle(weights[0])?); + polys.push(crate::eq::eq_mle(weights[1])?); + Ok(Self::Host(crate::batch::Batched::new( + polys, rules, lambdas, + )?)) + } + + /// Every factor slot, once all are bound: the trace's, then the weights. + pub fn polys(&self) -> &[crate::mle::Mle] { + match self { + Self::Host(batched) => batched.polys(), + #[cfg(feature = "cuda")] + Self::Fused { batched, .. } => batched.polys(), + } + } + + /// Whether the card runs its rounds. + pub fn on_device(&self) -> bool { + match self { + Self::Host(_) => false, + #[cfg(feature = "cuda")] + Self::Fused { .. } => true, + } + } +} + +impl RoundStepper for TableRounds<'_, E> +where + E: IsField + 'static, + FieldElement: Send + Sync, +{ + fn num_vars(&self) -> usize { + match self { + Self::Host(batched) => SumcheckPolynomial::num_vars(batched), + #[cfg(feature = "cuda")] + Self::Fused { vars, .. } => *vars, + } + } + + fn degree(&self) -> usize { + match self { + Self::Host(batched) => SumcheckPolynomial::degree(batched), + #[cfg(feature = "cuda")] + Self::Fused { batched, .. } => SumcheckPolynomial::degree(batched), + } + } + + fn round(&mut self, degree: usize) -> Result>, Error> { + match self { + Self::Host(batched) => Ok(round_evaluations(batched, degree, false)), + #[cfg(feature = "cuda")] + Self::Fused { + stepper, + batched, + long, + .. + } => match stepper { + None => { + let tick = std::time::Instant::now(); + let out = round_evaluations(batched, degree, false); + let (long, spent) = (usize::from(*long), tick.elapsed().as_secs_f64()); + crate::gpu_fused::note_tail(|tail| { + tail.host_rounds[long] += 1; + tail.host[long] += spent; + }); + Ok(out) + } + Some(stepper) => { + let g = stepper.message()?; + if g.len() > degree { + return Err(Error::RoundDegreeMismatch { + round: 0, + expected: degree, + got: g.len(), + }); + } + // Past its own degree the message is the same polynomial, + // read at more nodes: `g(0)` from the claim, then + // interpolation. + let mut all = Vec::with_capacity(g.len() + 1); + all.push(stepper.claim() - &g[0]); + all.extend(g.iter().cloned()); + let mut out = g; + for node in out.len() + 1..=degree { + out.push(crate::sumcheck::interpolate( + &all, + &FieldElement::::from(node as u64), + )); + } + Ok(out) + } + }, + } + } + + fn bind(&mut self, r: &FieldElement) -> Result<(), Error> { + match self { + Self::Host(batched) => batched.fix_first_variable(r), + #[cfg(feature = "cuda")] + Self::Fused { + stepper, + batched, + vars, + long, + } => { + *vars -= 1; + match stepper.as_mut() { + None => { + let tick = std::time::Instant::now(); + let out = batched.fix_first_variable(r); + let (long, spent) = (usize::from(*long), tick.elapsed().as_secs_f64()); + crate::gpu_fused::note_tail(|tail| tail.host[long] += spent); + out + } + Some(on_card) => { + on_card.bind(r)?; + if on_card.rounds_left() == 0 { + let (factors, _) = stepper + .take() + .ok_or(Error::DeviceFailed { stage: "fused" })? + .into_factors()?; + batched.adopt(factors)?; + } + Ok(()) + } + } + } + } + } + + fn prefetch(&mut self) -> Result<(), Error> { + match self { + Self::Host(_) => Ok(()), + #[cfg(feature = "cuda")] + Self::Fused { stepper, .. } => match stepper { + Some(stepper) => stepper.prefetch(), + None => Ok(()), + }, + } + } + + fn value(&self) -> Result, Error> { + if RoundStepper::num_vars(self) != 0 { + return Err(Error::VariableCountMismatch { + expected: 0, + got: RoundStepper::num_vars(self), + }); + } + let batched = match self { + Self::Host(batched) => batched, + #[cfg(feature = "cuda")] + Self::Fused { batched, .. } => batched, + }; + Ok(batched.eval_at_index(0)) + } +} + +/// ⛔ A FAULT for the negative test: the named polynomial is padded the `2^Δ` +/// way — constant in the variables it lacks, its round polynomial `v·2^{rest}` — +/// which is a different batch than the one the verifier checks. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[doc(hidden)] +pub struct PaddingFault { + pub scaled: Option, +} + +/// Proves `Σ_t w_t·Σ_x f̃_t(x)` over `n_max = max_t n_t` rounds of `degree`, +/// on the host, leaving every polynomial bound at its own prefix of the point. +/// +/// `degree` must bound every polynomial's (and be at least one, for the +/// padding's `X`). The caller has absorbed the claims and drawn the weights. +pub fn prove( + polys: &mut [P], + weights: &[FieldElement], + degree: usize, + transcript: &mut T, +) -> Result<(SumcheckProof, Vec>), Error> +where + F: IsField + 'static, + T: IsTranscript, + P: RoundStepper, +{ + prove_with(polys, weights, degree, transcript, PaddingFault::default()) +} + +/// [`prove`], with a fault armed only by the negative test. +#[doc(hidden)] +pub fn prove_with( + polys: &mut [P], + weights: &[FieldElement], + degree: usize, + transcript: &mut T, + fault: PaddingFault, +) -> Result<(SumcheckProof, Vec>), Error> +where + F: IsField + 'static, + T: IsTranscript, + P: RoundStepper, +{ + if polys.len() != weights.len() { + return Err(Error::VariableCountMismatch { + expected: polys.len(), + got: weights.len(), + }); + } + let degree = degree.max(1); + if let Some(p) = polys.iter().find(|p| p.degree() > degree) { + return Err(Error::RoundDegreeMismatch { + round: 0, + expected: degree, + got: p.degree(), + }); + } + let rounds = polys.iter().map(|p| p.num_vars()).max().unwrap_or(0); + let steps: Vec> = (1..=degree) + .map(|t| FieldElement::::from(t as u64)) + .collect(); + // Per polynomial, once bound: its value, times the challenges drawn since. + let mut done: Vec>> = vec![None; polys.len()]; + let mut proofs = Vec::with_capacity(rounds); + let mut point = Vec::with_capacity(rounds); + + for j in 0..rounds { + for poly in polys.iter_mut() { + if poly.num_vars() > 0 { + poly.prefetch()?; + } + } + let mut sent = vec![FieldElement::::zero(); degree]; + for (t, (poly, weight)) in polys.iter_mut().zip(weights).enumerate() { + if poly.num_vars() > 0 { + let own = poly.round(degree)?; + if own.len() != degree { + return Err(Error::RoundDegreeMismatch { + round: j, + expected: degree, + got: own.len(), + }); + } + for (s, v) in sent.iter_mut().zip(&own) { + *s += weight * v; + } + continue; + } + let value = match &done[t] { + Some(value) => value.clone(), + None => { + let value = poly.value()?; + done[t] = Some(value.clone()); + value + } + }; + if fault.scaled == Some(t) { + let rest = FieldElement::::from(1u64 << (rounds - 1 - j)); + for s in sent.iter_mut() { + *s += weight * &value * &rest; + } + continue; + } + for (s, x) in sent.iter_mut().zip(&steps) { + *s += weight * &value * x; + } + } + for e in &sent { + transcript.append_field_element(e); + } + let r: FieldElement = transcript.sample_field_element(); + for (t, poly) in polys.iter_mut().enumerate() { + if poly.num_vars() > 0 { + poly.bind(&r)?; + } else if let Some(value) = done[t].as_mut() { + *value *= &r; + } + } + proofs.push(RoundProof { evaluations: sent }); + point.push(r); + } + Ok((SumcheckProof { rounds: proofs }, point)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + mle::Mle, + sumcheck, + virtual_poly::{Term, VirtualPolynomial}, + }; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"front-loaded-test") + } + + fn pseudo_mle(n: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << n)) + .map(|i| FE::from((i.wrapping_mul(6364136223846793005).wrapping_add(seed)) >> 11)) + .collect(), + ) + .unwrap() + } + + /// `a·b·c` over `n` variables — degree three. + fn cubic(n: usize, seed: u64) -> VirtualPolynomial { + VirtualPolynomial::new( + vec![ + pseudo_mle(n, seed), + pseudo_mle(n, seed + 1), + pseudo_mle(n, seed + 2), + ], + vec![Term::new(FE::one(), vec![0, 1, 2])], + ) + .unwrap() + } + + fn run(heights: &[usize], fault: PaddingFault) -> Result<(), String> { + let polys: Vec> = heights + .iter() + .enumerate() + .map(|(t, &n)| cubic(n, 7 * t as u64)) + .collect(); + let weights: Vec = (0..heights.len()).map(|t| FE::from(3 + t as u64)).collect(); + let claim = polys + .iter() + .zip(&weights) + .fold(FE::zero(), |acc, (p, w)| acc + w * p.sum_over_hypercube()); + let n_max = heights.iter().copied().max().unwrap_or(0); + + let mut bound = polys.clone(); + let (proof, point) = prove_with(&mut bound, &weights, 3, &mut transcript(), fault) + .map_err(|e| format!("{e:?}"))?; + let checked = sumcheck::verify(&proof, claim, n_max, 3, &mut transcript()) + .map_err(|e| format!("{e:?}"))?; + assert_eq!(checked.point, point); + let want = polys + .iter() + .zip(&weights) + .zip(heights) + .fold(FE::zero(), |acc, ((p, w), &n)| { + acc + w * p.evaluate(&point[..n]).unwrap() * padding_product(&point, n) + }); + // What the prover's own bound polynomials say agrees with the direct + // evaluation at the prefix. + for ((b, p), &n) in bound.iter().zip(&polys).zip(heights) { + assert_eq!(b.eval_at_index(0), p.evaluate(&point[..n]).unwrap()); + } + if want == checked.expected_evaluation { + Ok(()) + } else { + Err("residual".into()) + } + } + + /// Polynomials of different heights, one with no variables, batched into + /// one sumcheck: the residual is each at its prefix, padded by the rest. + #[test] + fn polynomials_of_different_heights_batch_front_loaded() { + run(&[4, 1, 0, 4, 2], PaddingFault::default()).unwrap(); + run(&[3], PaddingFault::default()).unwrap(); + run(&[0, 0], PaddingFault::default()).unwrap(); + } + + /// A prover padding one short polynomial the `2^Δ` way is refused. + #[test] + fn a_polynomial_padded_by_scaling_is_refused() { + assert_eq!( + run(&[4, 1, 4], PaddingFault { scaled: Some(1) }), + Err("residual".into()) + ); + } + + #[test] + fn the_padding_product_is_the_suffix() { + let point = [FE::from(2), FE::from(3), FE::from(5)]; + assert_eq!(padding_product(&point, 0), FE::from(30)); + assert_eq!(padding_product(&point, 2), FE::from(5)); + assert_eq!(padding_product(&point, 3), FE::one()); + } +} diff --git a/crypto/multilinear/src/fused.rs b/crypto/multilinear/src/fused.rs new file mode 100644 index 000000000..616f31e8f --- /dev/null +++ b/crypto/multilinear/src/fused.rs @@ -0,0 +1,2197 @@ +//! Stage 1 of the argue, on the host: a table's zerocheck batch — its +//! constraint rule beside the bus's two — with the round polynomials computed +//! the stage-1 way, and every message the one today's rounds send +//! (`thoughts/zf/gap2/fix2/D-ARGUE.md` §2). +//! +//! The batch is `eq(r,x)·C(x) + eq(ρ,x)·(γ·N(x) + γ²·D(x))`: `C` the table's +//! roots batched by `β`, `N` and `D` the bus's two sides at the GKR claim's row +//! point `ρ`. Three things change how its rounds are computed, none what they +//! are: +//! +//! - **the bus as one column** (S1-2). Both bus rules are affine in the factors +//! with constant coefficients, so `γ·N + γ²·D` is one table +//! `L = a₀ + Σ a_k·f_k`, built once ([`BusColumn`]), and the rounds read one +//! value where they read every interaction's side; +//! - **the weights pulled out** (S1-4, Gruen). Round `j`'s polynomial is +//! `E^r_j·eq₁(r_j,X)·A_j(X) + E^ρ_j·eq₁(ρ_j,X)·B_j(X)`, `A_j` the constraint's +//! sum under `eq(r_{>j}, ·)` and `B_j` the bus column's under `eq(ρ_{>j}, ·)`. +//! `A_j` is walked at `X ∈ {0, 2..d}` and `A_j(1)` is taken from the claim the +//! round carries in; `B_j` is linear; +//! - **rounds 0 and 1 in the base field** (S1-5). Before the first challenge every +//! factor is a base column, so one pass over the 4-row groups `(a, b, x'')` +//! evaluates `C` on the grid `{0..d}²` — each factor's grid by additions from +//! its group's four rows — and both messages come out of it. On a trace that +//! satisfies its AIR the four boolean corners are zero, and are not computed. +//! +//! A round polynomial is a function of the batch, the challenges drawn before it +//! and the round, so an exact algorithm for it sends what today sends (§2.7). +//! Where this one would not — a trace that breaks its AIR, under the corner skip +//! — both proofs are rejected; [`Options::check_corners`] computes the corners +//! anyway and refuses the first that is not zero. +//! +//! Host only. It is the reference the device kernels are checked against, and +//! what each variant does is counted by phase as it runs ([`Counters`]): priced +//! from each walk's step mix, which is what a kernel walking the same program +//! executes, whatever shortcut the host takes to the same values. [`model`] +//! states the same counts in closed form for any height, and a test holds the +//! two together. + +use std::collections::BTreeMap; + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{ + element::FieldElement, + traits::{IsField, IsSubFieldOf}, +}; + +use crate::{ + Error, challenge_powers, + eq::eq_evals, + logup::{self, Interaction}, + mle::Mle, + program::{self, Builder, Op, Program}, + sumcheck::{RoundProof, SumcheckProof, interpolate}, +}; + +/// A program's steps by kind: what one walk of it executes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Mix { + pub var: u64, + pub fixed: u64, + pub add: u64, + pub sub: u64, + pub mul: u64, + pub neg: u64, +} + +impl Mix { + pub fn of(steps: &[Op]) -> Self { + let mut mix = Self::default(); + for step in steps { + match step { + Op::Var(_) => mix.var += 1, + Op::Fixed(_) => mix.fixed += 1, + Op::Add(..) => mix.add += 1, + Op::Sub(..) => mix.sub += 1, + Op::Mul(..) => mix.mul += 1, + Op::Neg(_) => mix.neg += 1, + } + } + mix + } + + pub fn steps(&self) -> u64 { + self.var + self.fixed + self.add + self.sub + self.mul + self.neg + } + + /// Additions, subtractions and negations: one add each. + pub fn adds(&self) -> u64 { + self.add + self.sub + self.neg + } +} + +/// Work, by kind. +/// +/// A read interpolated to a node `t ≥ 1` — `lo + t·(hi − lo)` — is counted +/// apart from the multiply it is made of, so one count can be priced as +/// today's kernel does it (an extension multiply by `t`) or as S1-1 does (a +/// base one): [`Ops::units`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Ops { + pub base_mul: u64, + pub base_add: u64, + pub ext_mul: u64, + pub ext_add: u64, + /// A base value times an extension one. + pub mixed_mul: u64, + /// `lo + t·(hi − lo)` at a node `t ≥ 1`. + pub node_reads: u64, + /// Program steps an interpreter walks: its per-step overhead. + pub steps: u64, + /// Factor and table bytes read. + pub bytes: u64, +} + +impl Ops { + pub fn add(&mut self, other: &Ops) { + self.base_mul += other.base_mul; + self.base_add += other.base_add; + self.ext_mul += other.ext_mul; + self.ext_add += other.ext_add; + self.mixed_mul += other.mixed_mul; + self.node_reads += other.node_reads; + self.steps += other.steps; + self.bytes += other.bytes; + } + + /// In D-ARGUE §2.8's units, where one Goldilocks product or one reduction + /// is a unit: a base multiply 2, an extension multiply 12 (nine products, + /// three reductions), a base-by-extension one 6, a base add ½, an + /// extension add 1½. A read at a node is two extension adds and a multiply + /// by the node: an extension one today (15), a base one under S1-1 + /// (`int_nodes`, 9). An interpreter adds 4 a step. + pub fn units(&self, int_nodes: bool, interpreted: bool) -> f64 { + let read = if int_nodes { 9.0 } else { 15.0 }; + let step = if interpreted { 4.0 } else { 0.0 }; + 2.0 * self.base_mul as f64 + + 0.5 * self.base_add as f64 + + 12.0 * self.ext_mul as f64 + + 1.5 * self.ext_add as f64 + + 6.0 * self.mixed_mul as f64 + + read * self.node_reads as f64 + + step * self.steps as f64 + } +} + +/// What a variant did, by phase. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Counters { + /// `L`, built once (S1-2). + pub bus_column: Ops, + /// The `eq` weight tables. + pub weights: Ops, + /// Rounds 0 and 1's pass over the base columns (S1-5). + pub grid: Ops, + /// Every other round's walks. + pub rounds: Ops, + /// The folds between rounds, the grid's double fold among them. + pub fold: Ops, + /// The host's part of a round: its message out of the round's sums. + pub assembly: Ops, + /// The values at the point of the factors no round folded. + pub bound: Ops, +} + +impl Counters { + pub fn phases(&self) -> [(&'static str, &Ops); 7] { + [ + ("bus column", &self.bus_column), + ("weights", &self.weights), + ("grid", &self.grid), + ("rounds", &self.rounds), + ("fold", &self.fold), + ("assembly", &self.assembly), + ("bound", &self.bound), + ] + } + + pub fn total(&self) -> Ops { + let mut total = Ops::default(); + for (_, ops) in self.phases() { + total.add(ops); + } + total + } + + pub fn units(&self, int_nodes: bool, interpreted: bool) -> f64 { + self.total().units(int_nodes, interpreted) + } +} + +/// A table's constraint part: the roots' DAG over the factor values in the +/// base field, each root's step, and the factor slot of each root's selector. +/// +/// A table's own constraint set reads no challenge, and a constant it reads is +/// a lifted base one unless the AIR says otherwise — so the DAG runs in the +/// base field until the first fold, which is what rounds 0 and 1 run on. +#[derive(Clone, Debug)] +pub struct Constraints { + steps: Vec>, + roots: Vec, + selectors: Vec>, + degree: usize, +} + +impl Constraints { + /// The DAG as the extension holds it (`IrShape::steps_as_ops` and its + /// roots and selectors), brought down to the base field. + /// + /// `degree` is the constraint's — `C`'s, a root's DAG degree plus its + /// selector, at most — and the batch's round degree is one more. + /// + /// Refuses a constant with an extension part ([`Error::NotBaseField`]): the + /// precondition the base rounds rest on. + pub fn from_extension( + steps: &[Op], + roots: &[u32], + selectors: &[Option], + degree: usize, + ) -> Result + where + F: IsSubFieldOf, + E: IsField, + { + if roots.len() != selectors.len() { + return Err(Error::VariableCountMismatch { + expected: roots.len(), + got: selectors.len(), + }); + } + let mut base = Vec::with_capacity(steps.len()); + for (at, step) in steps.iter().enumerate() { + let operand = |a: u32| -> Result { + if a as usize >= at { + return Err(Error::UnknownPolynomial { + index: a as usize, + len: at, + }); + } + Ok(a) + }; + base.push(match *step { + Op::Fixed(ref c) => { + let limbs = c.clone().to_subfield_vec::(); + let (first, rest) = limbs.split_first().ok_or(Error::EmptyPolynomial)?; + if rest.iter().any(|limb| *limb != FieldElement::::zero()) { + return Err(Error::NotBaseField { step: at }); + } + Op::Fixed(first.clone()) + } + Op::Var(i) => Op::Var(i), + Op::Add(a, b) => Op::Add(operand(a)?, operand(b)?), + Op::Sub(a, b) => Op::Sub(operand(a)?, operand(b)?), + Op::Mul(a, b) => Op::Mul(operand(a)?, operand(b)?), + Op::Neg(a) => Op::Neg(operand(a)?), + }); + } + if let Some(&root) = roots.iter().find(|&&root| root as usize >= steps.len()) { + return Err(Error::UnknownPolynomial { + index: root as usize, + len: steps.len(), + }); + } + Ok(Self { + steps: base, + roots: roots.to_vec(), + selectors: selectors.to_vec(), + degree, + }) + } + + pub fn steps(&self) -> &[Op] { + &self.steps + } + + pub fn roots(&self) -> &[u32] { + &self.roots + } + + pub fn selectors(&self) -> &[Option] { + &self.selectors + } + + /// `C`'s degree in the factors, a selector counted. + pub fn degree(&self) -> usize { + self.degree + } + + /// The factor slots the constraint part reads, its DAG's and its + /// selectors', ascending. + pub fn reads(&self) -> Vec { + let mut reads: Vec = self + .steps + .iter() + .filter_map(|step| match step { + Op::Var(i) => Some(*i as usize), + _ => None, + }) + .chain(self.selectors.iter().flatten().copied()) + .collect(); + reads.sort_unstable(); + reads.dedup(); + reads + } + + /// Roots with a selector: each costs one more multiply in the combine. + pub fn selected(&self) -> usize { + self.selectors.iter().flatten().count() + } + + /// The DAG with every factor read through `read`, in whichever field the + /// values are in; each step's value lands in `out`. + fn run(&self, read: &impl Fn(usize) -> FieldElement, out: &mut Vec>) + where + F: IsSubFieldOf, + V: IsField, + { + out.clear(); + for step in &self.steps { + let value: FieldElement = match *step { + Op::Fixed(ref c) => c.clone().to_extension::(), + Op::Var(i) => read(i as usize), + Op::Add(a, b) => &out[a as usize] + &out[b as usize], + Op::Sub(a, b) => &out[a as usize] - &out[b as usize], + Op::Mul(a, b) => &out[a as usize] * &out[b as usize], + Op::Neg(a) => -&out[a as usize], + }; + out.push(value); + } + } + + /// `C = Σ β_i·sel_i·root_i` at a base-field point: each root and its + /// selector in the base field, `β_i` the only extension factor. + fn combine_base( + &self, + betas: &[FieldElement], + read: &impl Fn(usize) -> FieldElement, + out: &[FieldElement], + ) -> FieldElement + where + F: IsSubFieldOf, + E: IsField, + { + let mut sum = FieldElement::::zero(); + for ((&root, selector), beta) in self.roots.iter().zip(&self.selectors).zip(betas) { + let term: FieldElement = match selector { + Some(slot) => &read(*slot) * &out[root as usize], + None => out[root as usize].clone(), + }; + let scaled: FieldElement = &term * beta; + sum += scaled; + } + sum + } + + /// The same once the factors are folded into the extension. + fn combine_ext( + &self, + betas: &[FieldElement], + read: &impl Fn(usize) -> FieldElement, + out: &[FieldElement], + ) -> FieldElement + where + E: IsField, + { + let mut sum = FieldElement::::zero(); + for ((&root, selector), beta) in self.roots.iter().zip(&self.selectors).zip(betas) { + let term: FieldElement = match selector { + Some(slot) => &read(*slot) * &out[root as usize], + None => out[root as usize].clone(), + }; + let scaled: FieldElement = &term * beta; + sum += scaled; + } + sum + } + + /// Today's zerocheck rule — `weight · Σ β_i·sel_i·root_i` over the + /// extension, the weight at factor slot `weight` — written the way + /// `IrShape::program` writes it. + pub fn zerocheck_program( + &self, + betas: &[FieldElement], + weight: usize, + ) -> Result, Error> + where + F: IsSubFieldOf, + E: IsField + 'static, + { + if betas.len() != self.roots.len() { + return Err(Error::VariableCountMismatch { + expected: self.roots.len(), + got: betas.len(), + }); + } + let mut builder = Builder::::new(); + for step in &self.steps { + match *step { + Op::Fixed(ref c) => builder.fixed(c.clone().to_extension::()), + Op::Var(i) => builder.var(i as usize), + Op::Add(a, b) => builder.add(a, b), + Op::Sub(a, b) => builder.sub(a, b), + Op::Mul(a, b) => builder.mul(a, b), + Op::Neg(a) => builder.neg(a), + }; + } + let terms: Vec<(u32, FieldElement)> = self + .roots + .iter() + .zip(betas) + .zip(&self.selectors) + .map(|((&root, beta), selector)| { + let term = match selector { + Some(slot) => { + let s = builder.var(*slot); + builder.mul(root, s) + } + None => root, + }; + (term, beta.clone()) + }) + .collect(); + let sum = builder.weighted_sum(&terms); + let w = builder.var(weight); + let root = builder.mul(w, sum); + builder.finish(root) + } +} + +/// The bus's two rules as one column's coefficients (S1-2): +/// `λ_N·N + λ_D·D = a₀ + Σ_k a_k·f_k`. +/// +/// `N = Σ_i w_i·num_i(f)` and `D = Σ_i w_i·den_i(f) + padding`, with `w` the +/// interaction weights `eq(interaction point, i)` and `padding` the weight of +/// the `0/1` slots — as [`logup::claim_statements`] writes them — and every +/// side affine in the factors with constant coefficients. So the combination +/// is affine too, and its coefficients are sums over the interactions, made +/// once per table after the batching challenge is drawn. +/// +/// It is not a proof value: the verifier checks the batch against its own two +/// rules, which this equals. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BusColumn { + constant: FieldElement, + /// `(factor slot, coefficient)`, the slots ascending and distinct. + terms: Vec<(usize, FieldElement)>, +} + +impl BusColumn +where + FieldElement: Send + Sync, +{ + /// `numerator·N + denominator·D` at `interaction_point`: the batch's + /// weights on the bus's two rules. + pub fn new( + interactions: &[Interaction], + interaction_point: &[FieldElement], + numerator: &FieldElement, + denominator: &FieldElement, + ) -> Result { + if interactions.is_empty() { + return Err(Error::EmptyPolynomial); + } + let weights = eq_evals(interaction_point); + if weights.len() < interactions.len() { + return Err(Error::VariableCountMismatch { + expected: interactions.len().next_power_of_two().trailing_zeros() as usize, + got: interaction_point.len(), + }); + } + let padding = weights[interactions.len()..] + .iter() + .fold(FieldElement::::zero(), |acc, w| acc + w); + let mut constant = denominator * &padding; + let mut coefficients: BTreeMap> = BTreeMap::new(); + for (interaction, w) in interactions.iter().zip(&weights) { + for (side, scale) in [ + (&interaction.numerator, w * numerator), + (&interaction.denominator, w * denominator), + ] { + constant += &scale * side.constant_term(); + for (slot, coefficient) in side.terms() { + let term = &scale * coefficient; + *coefficients.entry(*slot).or_insert_with(FieldElement::zero) += term; + } + } + } + Ok(Self { + constant, + terms: coefficients.into_iter().collect(), + }) + } + + /// A column given by its coefficients — a test's forgery, for one. + pub fn from_parts(constant: FieldElement, terms: Vec<(usize, FieldElement)>) -> Self { + Self { constant, terms } + } + + pub fn constant(&self) -> &FieldElement { + &self.constant + } + + pub fn terms(&self) -> &[(usize, FieldElement)] { + &self.terms + } + + /// `L` at one point, from every factor's value there. + pub fn evaluate(&self, values: &[FieldElement]) -> FieldElement { + self.terms + .iter() + .fold(self.constant.clone(), |acc, (slot, a)| { + acc + a * &values[*slot] + }) + } + + /// `L`'s table over the cube, from the base factors: a base-by-extension + /// multiply-add per term a row. + pub fn table(&self, factors: &[Mle], ops: &mut Ops) -> Result>, Error> + where + F: IsSubFieldOf + 'static, + { + let size = factors + .first() + .map(Mle::len) + .ok_or(Error::EmptyPolynomial)?; + for (slot, _) in &self.terms { + let factor = factors.get(*slot).ok_or(Error::UnknownPolynomial { + index: *slot, + len: factors.len(), + })?; + if factor.len() != size { + return Err(Error::VariableCountMismatch { + expected: size.trailing_zeros() as usize, + got: factor.num_vars(), + }); + } + } + let mut table = vec![self.constant.clone(); size]; + for (slot, a) in &self.terms { + for (cell, value) in table.iter_mut().zip(factors[*slot].evals()) { + let term: FieldElement = value * a; + *cell += term; + } + } + column_ops(ops, size as u64, self.terms.len() as u64); + Ok(table) + } +} + +/// How a zerocheck batch's rounds are computed. Each variant adds a technique +/// to the one before it; every one sends the same messages. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Variant { + /// Today's rounds: the whole batch program walked at every node `1..=D` + /// of every index, the weights two factors among the rest, every factor + /// folded every round. + Today, + /// S1-2: the bus's two rules replaced by the column `L`; otherwise today's + /// rounds, the factors nothing reads left out of them. + Bus, + /// S1-2 and S1-4: no weight in the walk; `C` at `{0, 2..d}`, `A_j(1)` from + /// the claim, `B_j` from `L`'s two halves. + Gruen, + /// S1-2, S1-4 and S1-5: rounds 0 and 1 from one base-field pass over the + /// 4-row groups; S1-4 from round 2. + Grid, +} + +/// A variant and its switches. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Options { + pub variant: Variant, + /// [`Variant::Today`] and [`Variant::Bus`]: walk the batch program on + /// demand ([`Program::on_demand`]), as the head does for a big batch. The + /// values are the same; the reads, and so the counts, are not. + pub on_demand: bool, + /// [`Variant::Grid`]: the grid's four boolean corners are taken as zero — + /// what they are on a trace that satisfies its AIR. + pub skip_corners: bool, + /// [`Variant::Grid`]: compute the corners anyway and refuse the first that + /// is not zero ([`Error::ConstraintViolated`]) — XCHECK's self-check. + pub check_corners: bool, + /// ⛔ A FAULT, for a negative control: `L`'s term at this index off by one. + pub bus_fault: Option, +} + +impl Options { + pub fn new(variant: Variant) -> Self { + Self { + variant, + on_demand: false, + skip_corners: false, + check_corners: false, + bus_fault: None, + } + } +} + +/// What the batch is about, beside the factors: the constraint part, its +/// batching powers, the bus, and the two points. +pub struct Statement<'a, F: IsField, E: IsField> { + pub constraints: &'a Constraints, + /// `β^i`, one per root. + pub betas: &'a [FieldElement], + pub interactions: &'a [Interaction], + /// The GKR input-layer claim's point: the interaction bits, then the rows' + /// — whose tail is `ρ`. + pub claim_point: &'a [FieldElement], + /// The zerocheck's point. + pub r: &'a [FieldElement], +} + +/// The rounds, the point they drew, and every factor's value there in slot +/// order. +pub type Proved = (SumcheckProof, Vec>, Vec>); + +/// The batch's sumcheck the way `options` says, from the claims on. +/// +/// Absorbs `claims` — the zerocheck's, then the bus's `p̃` and `q̃` — draws the +/// batching challenge and runs every round, as [`crate::batch::prove`] does +/// over today's `Batched`; the proof, the point and the transcript after it +/// are that one's. `factors` are the table's, in the base field and slot +/// order: the committed views and the public selectors. +/// +/// The claims must be the batch's true sums. Today's rounds never read them; +/// S1-4 takes `A_j(1)` from the claim each round carries in, which is the true +/// one exactly when the first is. +pub fn prove( + statement: &Statement<'_, F, E>, + factors: &[Mle], + claims: &[FieldElement; 3], + options: Options, + transcript: &mut T, + counters: &mut Counters, +) -> Result, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, + FieldElement: Send + Sync, + T: IsTranscript, +{ + let num_vars = factors + .first() + .map(Mle::num_vars) + .ok_or(Error::EmptyPolynomial)?; + for factor in factors { + if factor.num_vars() != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got: factor.num_vars(), + }); + } + } + if statement.r.len() != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got: statement.r.len(), + }); + } + let expected = logup::input_layer_vars(statement.interactions.len(), num_vars); + if statement.claim_point.len() != expected { + return Err(Error::VariableCountMismatch { + expected, + got: statement.claim_point.len(), + }); + } + if statement.betas.len() != statement.constraints.roots.len() { + return Err(Error::VariableCountMismatch { + expected: statement.constraints.roots.len(), + got: statement.betas.len(), + }); + } + if let Some(&slot) = statement + .constraints + .reads() + .iter() + .find(|&&slot| slot >= factors.len()) + { + return Err(Error::UnknownPolynomial { + index: slot, + len: factors.len(), + }); + } + + for claim in claims { + transcript.append_field_element(claim); + } + let lambdas = challenge_powers(&transcript.sample_field_element(), 3); + let claim = lambdas + .iter() + .zip(claims) + .fold(FieldElement::::zero(), |acc, (l, c)| acc + l * c); + let degree = (statement.constraints.degree + 1).max(2); + let (interaction_point, _) = statement + .claim_point + .split_at(statement.claim_point.len() - num_vars); + let bus = || -> Result, Error> { + let mut bus = BusColumn::new( + statement.interactions, + interaction_point, + &lambdas[1], + &lambdas[2], + )?; + if let Some(term) = options.bus_fault { + let len = bus.terms.len(); + let (_, a) = bus + .terms + .get_mut(term) + .ok_or(Error::UnknownPolynomial { index: term, len })?; + *a += FieldElement::::one(); + } + Ok(bus) + }; + + match options.variant { + Variant::Today => { + let program = batch_program(statement, num_vars, factors.len(), &lambdas)?; + let rho = &statement.claim_point[interaction_point.len()..]; + let extra = vec![ + eq_table(statement.r, &mut counters.weights), + eq_table(rho, &mut counters.weights), + ]; + walk( + &program, + options.on_demand, + factors, + extra, + true, + degree, + transcript, + counters, + ) + } + Variant::Bus => { + let bus = bus()?; + let program = column_program(statement, factors.len())?; + let rho = &statement.claim_point[interaction_point.len()..]; + let extra = vec![ + eq_table(statement.r, &mut counters.weights), + eq_table(rho, &mut counters.weights), + bus.table(factors, &mut counters.bus_column)?, + ]; + walk( + &program, + options.on_demand, + factors, + extra, + false, + degree, + transcript, + counters, + ) + } + Variant::Gruen | Variant::Grid => { + let bus = bus()?; + gruen( + statement, factors, &bus, claim, degree, options, transcript, counters, + ) + } + } +} + +/// Today's batch over `factors` trace factors, the weights at the two slots +/// after them: the zerocheck rule and the bus's two, combined with `lambdas` +/// as `batch::prove_resident` combines them. +pub fn batch_program( + statement: &Statement<'_, F, E>, + num_vars: usize, + factors: usize, + lambdas: &[FieldElement], +) -> Result, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let (weight_r, weight_z) = (factors, factors + 1); + let zerocheck = statement + .constraints + .zerocheck_program(statement.betas, weight_r)?; + let rules = logup::claim_statements( + statement.interactions, + statement.claim_point, + num_vars, + weight_z, + )?; + program::combine( + &[ + &zerocheck, + rules.numerator.program().ok_or(Error::EmptyPolynomial)?, + rules.denominator.program().ok_or(Error::EmptyPolynomial)?, + ], + lambdas, + ) +} + +/// S1-2's batch over `factors` trace factors: the weights at the two slots +/// after them, `L` at the next. +pub fn column_program( + statement: &Statement<'_, F, E>, + factors: usize, +) -> Result, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let zerocheck = statement + .constraints + .zerocheck_program(statement.betas, factors)?; + bus_batch(&zerocheck, factors + 1, factors + 2) +} + +/// S1-2's batch: today's zerocheck rule plus the row weight times `L`, at +/// factor slots `weight` and `column`. +pub fn bus_batch( + zerocheck: &Program, + weight: usize, + column: usize, +) -> Result, Error> { + let mut builder = Builder::::new(); + let w = builder.var(weight); + let l = builder.var(column); + let root = builder.mul(w, l); + let bus = builder.finish(root)?; + program::combine( + &[zerocheck, &bus], + &[FieldElement::one(), FieldElement::one()], + ) +} + +/// `eq(point, ·)`'s table, counted as [`eq_evals`] builds it. +fn eq_table(point: &[FieldElement], ops: &mut Ops) -> Vec> +where + FieldElement: Send + Sync, +{ + eq_ops(ops, point.len() as u64); + eq_evals(point) +} + +/// `eq(a_{>j+1}, ·)` from `eq(a_{>j}, ·)`: the two halves added, since +/// `eq₁(a_{j+1}, 0) + eq₁(a_{j+1}, 1) = 1`. +fn halve(table: &mut Vec>, ops: &mut Ops) { + let half = table.len() / 2; + let (lo, hi) = table.split_at_mut(half); + for (l, h) in lo.iter_mut().zip(hi.iter()) { + *l = &*l + h; + } + table.truncate(half); + ops.ext_add += half as u64; +} + +/// `eq₁(a, t) = (1 − a)(1 − t) + a·t`. +pub(crate) fn eq1(a: &FieldElement, t: &FieldElement) -> FieldElement { + let one = FieldElement::::one(); + (&one - a) * (&one - t) + a * t +} + +/// A table folded on its first variable: `lo + s·(hi − lo)`. +fn fold(table: &mut Vec>, s: &FieldElement, ops: &mut Ops) { + let half = table.len() / 2; + let (lo, hi) = table.split_at_mut(half); + for (l, h) in lo.iter_mut().zip(hi.iter()) { + *l = &*l + s * &(h - &*l); + } + table.truncate(half); + fold_ops(ops, 1, half as u64); +} + +/// A factor's value at `point` from its base table: the first fold lifts, the +/// rest stay up — `Mle::evaluate_in`'s host path. +fn bound_at( + factor: &Mle, + point: &[FieldElement], + ops: &mut Ops, +) -> Result, Error> +where + F: IsSubFieldOf + 'static, + E: IsField + 'static, +{ + bound_ops(ops, 1, factor.len() as u64); + factor.evaluate_in_on_host(point) +} + +// ── what each piece of work costs ──────────────────────────────────────────── +// +// Every count below is linear in its last argument, so the executed paths call +// them once per round with that round's size, and [`model`] once per phase with +// the sizes summed over its rounds. What the two can disagree on is the sums, +// which is what `the_model_is_what_the_reference_counts` holds. + +/// `eq(point, ·)` over `len` variables: two multiplies a cell, a subtraction a +/// variable. +fn eq_ops(ops: &mut Ops, len: u64) { + ops.ext_mul += 2 * ((1u64 << len) - 1); + ops.ext_add += len; +} + +/// `L` over `cells` rows of `terms` terms: a base-by-extension multiply-add a +/// term, a base read. +fn column_ops(ops: &mut Ops, cells: u64, terms: u64) { + ops.mixed_mul += cells * terms; + ops.ext_add += cells * terms; + ops.bytes += 8 * cells * terms; +} + +/// `tables` extension tables folded over `indices` indices. +fn fold_ops(ops: &mut Ops, tables: u64, indices: u64) { + ops.ext_mul += tables * indices; + ops.ext_add += 2 * tables * indices; + ops.bytes += 48 * tables * indices; +} + +/// `count` base factors of `cells` rows evaluated at a point, as +/// [`bound_at`] does. +fn bound_ops(ops: &mut Ops, count: u64, cells: u64) { + let half = cells / 2; + if half == 0 { + return; + } + ops.base_add += count * half; + ops.mixed_mul += count * half; + ops.ext_add += count * half; + ops.bytes += count * 16 * half; + let rest = half - 1; + ops.ext_mul += count * rest; + ops.ext_add += count * 2 * rest; + ops.bytes += count * 48 * rest; +} + +/// `walks` walks of a batch program at nodes `t ≥ 1` — every read +/// interpolated, the sum accumulated — as today's kernel walks them. +fn walk_ops(ops: &mut Ops, mix: &Mix, walks: u64) { + ops.node_reads += walks * mix.var; + ops.bytes += walks * 48 * mix.var; + ops.ext_add += walks * (mix.adds() + 1); + ops.ext_mul += walks * mix.mul; + ops.steps += walks * mix.steps(); +} + +/// S1-4 over `indices` indices: `C` walked at the `d` nodes `0, 2..=d` (node +/// 0's reads plain, the rest interpolated), combined and weighted; `B`'s two +/// sums. +fn gruen_ops(ops: &mut Ops, mix: &Mix, roots: u64, selected: u64, d: u64, indices: u64) { + let walks = indices * d; + let interpolated = indices * (d - 1); + ops.node_reads += interpolated * mix.var; + ops.bytes += indices * 24 * mix.var + interpolated * 48 * mix.var; + ops.ext_add += walks * (mix.adds() + roots + 1) + 2 * indices; + ops.ext_mul += walks * (mix.mul + roots + selected + 1) + 2 * indices; + ops.steps += walks * (mix.steps() + 2 * roots + selected); + // The two weights, and `L`'s two halves. + ops.bytes += indices * (24 + 24 + 48); +} + +/// `rounds` messages out of their parts ([`message`]) — the host's side of a +/// round. Approximate, and far below anything a round walks: `A(D)` priced as +/// `(d+1)²` multiplies, each node's `g(t)` as eight. +fn message_ops(ops: &mut Ops, d: u64, rounds: u64) { + ops.ext_mul += rounds * ((d + 1) * (d + 1) + 8 * (d + 1)); + ops.ext_add += rounds * (6 * (d + 1) + 1); +} + +/// `A(1)` from the claim, `rounds` times. +fn derive_ops(ops: &mut Ops, rounds: u64) { + ops.ext_mul += 6 * rounds; + ops.ext_add += 5 * rounds; +} + +/// Round 1's `A₁(Y)` for each of the `d + 1` values of `Y`: a Lagrange form +/// in `s₀` over the grid's first variable. +fn lagrange_ops(ops: &mut Ops, d: u64) { + ops.ext_mul += (d + 1) * (d + 1) * (d + 1); +} + +/// The double fold over `groups` groups: each of `reads` base factors' +/// four rows into one, and `L`'s. +fn double_fold_ops(ops: &mut Ops, groups: u64, reads: u64) { + ops.ext_mul += 4; + ops.mixed_mul += 4 * groups * reads; + ops.ext_add += 3 * groups * (reads + 1); + ops.bytes += 32 * groups * reads + 96 * groups; + ops.ext_mul += 4 * groups; +} + +/// The grid pass over `groups` groups: each read factor's grid by additions, +/// the walks at the points computed, the sums at the points kept, and `U`. +#[allow(clippy::too_many_arguments)] +fn grid_ops( + ops: &mut Ops, + groups: u64, + reads: u64, + mix: &Mix, + roots: u64, + selected: u64, + d: u64, + options: Options, +) { + let side = d + 1; + let computed_corners = !options.skip_corners || options.check_corners; + let computed = side * side - if computed_corners { 0 } else { 4 }; + let kept = side * side - if options.skip_corners { 4 } else { 0 }; + ops.base_add += groups * reads * d * (d + 3); + ops.bytes += groups * reads * 32; + let walks = groups * computed; + ops.base_add += walks * mix.adds(); + ops.base_mul += walks * (mix.mul + selected); + ops.mixed_mul += walks * roots; + ops.ext_add += walks * roots; + ops.steps += walks * (mix.steps() + 2 * roots + selected); + ops.ext_mul += groups * kept; + ops.ext_add += groups * kept; + // `U`, and the two weights. + ops.ext_mul += 4 * groups; + ops.ext_add += 4 * groups; + ops.bytes += groups * (96 + 48); +} + +/// Today's round loop over `program`: at every index, every node `1..=degree`, +/// every factor interpolated and the program walked. A table is folded when +/// `fold_all` or when the program reads it; a trace factor none folded is +/// bound at the end. +/// +/// `extra` are the tables past the trace's factors (the weights, `L`), in +/// their slots. +#[allow(clippy::too_many_arguments)] +fn walk( + program: &Program, + on_demand: bool, + factors: &[Mle], + extra: Vec>>, + fold_all: bool, + degree: usize, + transcript: &mut T, + counters: &mut Counters, +) -> Result, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, + T: IsTranscript, +{ + let walked = if on_demand { + program.on_demand() + } else { + program.clone() + }; + let mix = Mix::of(walked.steps()); + let width = factors.len() + extra.len(); + let mut read = vec![false; width]; + for step in walked.steps() { + if let Op::Var(i) = step { + let slot = *i as usize; + if slot >= width { + return Err(Error::UnknownPolynomial { + index: slot, + len: width, + }); + } + read[slot] = true; + } + } + let mut extra = extra.into_iter(); + let mut tables: Vec>>> = (0..width) + .map(|slot| match factors.get(slot) { + Some(factor) if fold_all || read[slot] => Some( + factor + .evals() + .iter() + .map(|v| v.clone().to_extension::()) + .collect(), + ), + Some(_) => None, + None => extra.next(), + }) + .collect(); + let num_vars = factors.first().map(Mle::num_vars).unwrap_or(0); + let nodes: Vec> = (1..=degree as u64).map(FieldElement::from).collect(); + let mut values = vec![FieldElement::::zero(); width]; + let mut scratch = Vec::new(); + let mut rounds = Vec::with_capacity(num_vars); + let mut point = Vec::with_capacity(num_vars); + for _ in 0..num_vars { + let len = tables.iter().flatten().map(Vec::len).next().unwrap_or(1); + let half = len / 2; + let mut sums = vec![FieldElement::::zero(); degree]; + for i in 0..half { + for (sum, t) in sums.iter_mut().zip(&nodes) { + for (value, table) in values.iter_mut().zip(&tables) { + if let Some(table) = table { + let lo = &table[i]; + *value = lo + t * &(&table[i + half] - lo); + } + } + *sum += walked.eval(&values, &mut scratch); + } + } + walk_ops(&mut counters.rounds, &mix, (half * degree) as u64); + for e in &sums { + transcript.append_field_element(e); + } + let s = transcript.sample_field_element(); + for table in tables.iter_mut().flatten() { + fold(table, &s, &mut counters.fold); + } + rounds.push(RoundProof { evaluations: sums }); + point.push(s); + } + let bound = factors + .iter() + .zip(&tables) + .map(|(factor, table)| match table { + Some(table) => Ok(table[0].clone()), + None => bound_at(factor, &point, &mut counters.bound), + }) + .collect::, _>>()?; + Ok((SumcheckProof { rounds }, point, bound)) +} + +/// S1-4's rounds, from round 0 or — under [`Variant::Grid`], two variables up — +/// from round 2, after S1-5's pass has sent rounds 0 and 1. +#[allow(clippy::too_many_arguments)] +fn gruen( + statement: &Statement<'_, F, E>, + factors: &[Mle], + bus: &BusColumn, + claim: FieldElement, + degree: usize, + options: Options, + transcript: &mut T, + counters: &mut Counters, +) -> Result, Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, + FieldElement: Send + Sync, + T: IsTranscript, +{ + let constraints = statement.constraints; + let num_vars = factors.first().map(Mle::num_vars).unwrap_or(0); + let r = statement.r; + let rho = &statement.claim_point[statement.claim_point.len() - num_vars..]; + // `C`'s degree as the rounds see it: a constant constraint is a line. + let d = constraints.degree.max(1); + debug_assert_eq!( + degree, + d + 1, + "the batch's degree is the constraint's plus one" + ); + let reads = constraints.reads(); + let mix = Mix::of(constraints.steps()); + let (roots, selected) = ( + constraints.roots.len() as u64, + constraints.selected() as u64, + ); + let one = FieldElement::::one(); + let big = FieldElement::::from(degree as u64); + + let mut l = bus.table(factors, &mut counters.bus_column)?; + let mut claim = claim; + let (mut e_r, mut e_rho) = (one.clone(), one.clone()); + let mut rounds = Vec::with_capacity(num_vars); + let mut point = Vec::with_capacity(num_vars); + // A round's message sent and absorbed, its challenge drawn, and the claim + // the next round carries — what the verifier computes from the message. + let mut send = |g: Vec>, + claim: &mut FieldElement, + rounds: &mut Vec>, + point: &mut Vec>| { + for e in &g { + transcript.append_field_element(e); + } + let s = transcript.sample_field_element(); + let mut all = Vec::with_capacity(g.len() + 1); + all.push(&*claim - &g[0]); + all.extend(g.iter().cloned()); + *claim = interpolate(&all, &s); + rounds.push(RoundProof { evaluations: g }); + point.push(s.clone()); + s + }; + + // The tables of the factors the constraint part reads, by slot, as the + // rounds have folded them; the rest are bound at the end. + let mut tables: Vec>>> = vec![None; factors.len()]; + let (mut eq_r, mut eq_rho, start) = if options.variant == Variant::Grid && num_vars >= 2 { + let eq_r = eq_table(&r[2..], &mut counters.weights); + let eq_rho = eq_table(&rho[2..], &mut counters.weights); + let (t, u) = grid_pass( + statement, factors, &reads, &l, &eq_r, &eq_rho, d, options, counters, + )?; + let node = |k: usize| FieldElement::::from(k as u64); + // Round 0: `A₀(X) = Σ_c eq₁(r₁, c)·T(X, c)`, `B₀(b) = Σ_c eq₁(ρ₁, c)·U(b, c)`. + let a0: Vec> = (0..=d) + .map(|a| (&one - &r[1]) * &t[a][0] + &r[1] * &t[a][1]) + .collect(); + let b0: [FieldElement; 2] = + core::array::from_fn(|b| (&one - &rho[1]) * &u[b][0] + &rho[1] * &u[b][1]); + let g0 = message(&r[0], &rho[0], &e_r, &e_rho, &a0, &b0, &big); + message_ops(&mut counters.assembly, d as u64, 1); + let s0 = send(g0, &mut claim, &mut rounds, &mut point); + e_r = &e_r * eq1(&r[0], &s0); + e_rho = &e_rho * eq1(&rho[0], &s0); + // Round 1: `A₁(Y) = Σ_a ℓ_a(s₀)·T(a, Y)` — Lagrange in the first + // variable, exact since `C` has degree at most `d` in it — and + // `B₁(c) = (1 − s₀)·U(0, c) + s₀·U(1, c)`. + let a1: Vec> = (0..=d) + .map(|b| { + let column: Vec> = (0..=d).map(|a| t[a][b].clone()).collect(); + interpolate(&column, &s0) + }) + .collect(); + lagrange_ops(&mut counters.assembly, d as u64); + let b1: [FieldElement; 2] = + core::array::from_fn(|c| (&one - &s0) * &u[0][c] + &s0 * &u[1][c]); + let g1 = message(&r[1], &rho[1], &e_r, &e_rho, &a1, &b1, &big); + message_ops(&mut counters.assembly, d as u64, 1); + let s1 = send(g1, &mut claim, &mut rounds, &mut point); + e_r = &e_r * eq1(&r[1], &s1); + e_rho = &e_rho * eq1(&rho[1], &s1); + // Each group's four rows to one, base to extension: the two folds by + // `s₀` and `s₁` as one, `Σ_{a,b} eq₁(s₀, a)·eq₁(s₁, b)·f(a, b, x'')`. + let w: [[FieldElement; 2]; 2] = core::array::from_fn(|a| { + core::array::from_fn(|b| eq1(&s0, &node(a)) * eq1(&s1, &node(b))) + }); + let quarter = l.len() / 4; + let at = |a: usize, b: usize, x: usize| a * 2 * quarter + b * quarter + x; + for &slot in &reads { + let f = factors[slot].evals(); + tables[slot] = Some( + (0..quarter) + .map(|x| { + let mut v = FieldElement::::zero(); + for (a, row) in w.iter().enumerate() { + for (b, wab) in row.iter().enumerate() { + let term: FieldElement = &f[at(a, b, x)] * wab; + v += term; + } + } + v + }) + .collect(), + ); + } + l = (0..quarter) + .map(|x| { + let mut v = FieldElement::::zero(); + for (a, row) in w.iter().enumerate() { + for (b, wab) in row.iter().enumerate() { + v += &l[at(a, b, x)] * wab; + } + } + v + }) + .collect(); + double_fold_ops(&mut counters.fold, quarter as u64, reads.len() as u64); + let (mut eq_r, mut eq_rho) = (eq_r, eq_rho); + if num_vars > 2 { + halve(&mut eq_r, &mut counters.weights); + halve(&mut eq_rho, &mut counters.weights); + } + (eq_r, eq_rho, 2) + } else { + for &slot in &reads { + tables[slot] = Some( + factors[slot] + .evals() + .iter() + .map(|v| v.clone().to_extension::()) + .collect(), + ); + } + let from = 1.min(num_vars); + let eq_r = eq_table(&r[from..], &mut counters.weights); + let eq_rho = eq_table(&rho[from..], &mut counters.weights); + (eq_r, eq_rho, 0) + }; + + let mut values = vec![FieldElement::::zero(); factors.len()]; + let mut scratch: Vec> = Vec::new(); + // `A` is walked at `0` and `2..=d`; `A(1)` comes from the claim. + let walked: Vec = core::iter::once(0).chain(2..=d).collect(); + for j in start..num_vars { + let half = l.len() / 2; + // Every read factor at index `i`, extended to the node `k` — by a base + // multiply, the node being an integer (S1-1). + let at_node = |values: &mut Vec>, + tables: &[Option>>], + i: usize, + k: usize| { + let node = FieldElement::::from(k as u64); + for &slot in &reads { + let table = tables[slot].as_ref().expect("a read factor is folded"); + let lo = &table[i]; + values[slot] = if k == 0 { + lo.clone() + } else { + let delta = &table[i + half] - lo; + let step: FieldElement = &node * δ + lo + step + }; + } + }; + let mut a = vec![FieldElement::::zero(); d + 1]; + let mut b = [FieldElement::::zero(), FieldElement::::zero()]; + for i in 0..half { + for &k in &walked { + at_node(&mut values, &tables, i, k); + let read = |slot: usize| values[slot].clone(); + constraints.run(&read, &mut scratch); + let c = constraints.combine_ext(statement.betas, &read, &scratch); + a[k] += &eq_r[i] * &c; + } + b[0] += &eq_rho[i] * &l[i]; + b[1] += &eq_rho[i] * &l[i + half]; + } + gruen_ops( + &mut counters.rounds, + &mix, + roots, + selected, + d as u64, + half as u64, + ); + + // `A(1)` from the claim the round carries in, `g(0) + g(1)`: + // `E^r·((1 − r_j)·A(0) + r_j·A(1)) + E^ρ·((1 − ρ_j)·B(0) + ρ_j·B(1))`. + let (r_j, rho_j) = (&r[j], &rho[j]); + let divisor = &e_r * r_j; + let bus_part = &e_rho * ((&one - rho_j) * &b[0] + rho_j * &b[1]); + derive_ops(&mut counters.assembly, 1); + a[1] = match divisor.inv() { + Ok(inverse) => (&claim - &e_r * (&one - r_j) * &a[0] - bus_part) * inverse, + // `E^r_j·r_j` vanished, so the claim says nothing about `A(1)` and + // it is walked. A random point makes this `(j + 1)/|F|`. + Err(_) => { + let mut direct = FieldElement::::zero(); + for (i, weight) in eq_r.iter().enumerate().take(half) { + at_node(&mut values, &tables, i, 1); + let read = |slot: usize| values[slot].clone(); + constraints.run(&read, &mut scratch); + let c = constraints.combine_ext(statement.betas, &read, &scratch); + direct += weight * &c; + } + walk_ops(&mut counters.rounds, &mix, half as u64); + direct + } + }; + let g = message(r_j, rho_j, &e_r, &e_rho, &a, &b, &big); + message_ops(&mut counters.assembly, d as u64, 1); + let s = send(g, &mut claim, &mut rounds, &mut point); + e_r = &e_r * eq1(r_j, &s); + e_rho = &e_rho * eq1(rho_j, &s); + for table in tables.iter_mut().flatten() { + fold(table, &s, &mut counters.fold); + } + fold(&mut l, &s, &mut counters.fold); + if j + 1 < num_vars { + halve(&mut eq_r, &mut counters.weights); + halve(&mut eq_rho, &mut counters.weights); + } + } + + let bound = factors + .iter() + .zip(&tables) + .map(|(factor, table)| match table { + Some(table) => Ok(table[0].clone()), + None => bound_at(factor, &point, &mut counters.bound), + }) + .collect::, _>>()?; + Ok((SumcheckProof { rounds }, point, bound)) +} + +/// A round's message from its parts: `A` at `0..=d` (the constraint's, degree +/// `d`), `B` at `0` and `1` (the bus column's, linear), and the weights' +/// products so far. `g(t) = E^r·eq₁(r_j, t)·A(t) + E^ρ·eq₁(ρ_j, t)·B(t)` for +/// `t = 1..=D`, `D = d + 1`, with `A(D)` from the Lagrange form over `0..=d`. +pub(crate) fn message( + r_j: &FieldElement, + rho_j: &FieldElement, + e_r: &FieldElement, + e_rho: &FieldElement, + a: &[FieldElement], + b: &[FieldElement; 2], + big: &FieldElement, +) -> Vec> { + let d = a.len() - 1; + let a_big = interpolate(a, big); + let slope = &b[1] - &b[0]; + (1..=d + 1) + .map(|t| { + let node = FieldElement::::from(t as u64); + let at = if t <= d { &a[t] } else { &a_big }; + let bt = &b[0] + &node * &slope; + e_r * eq1(r_j, &node) * at + e_rho * eq1(rho_j, &node) * bt + }) + .collect() +} + +/// S1-5's pass: `T(a, b) = Σ_{x''} eq(r_{>1}, x'')·C(a, b, x'')` over the grid +/// `{0..d}²` and `U(b, c) = Σ_{x''} eq(ρ_{>1}, x'')·L(b, c, x'')` over +/// `{0,1}²`, group `x''`'s rows being `a·N/2 + b·N/4 + x''`. +/// +/// Each read factor's grid comes from its group's four rows by additions — the +/// first variable extended from the two rows of each `b`, then the second +/// from each `a`'s two — and `C` is walked in the base field at every point +/// that is not a corner, or at every point when the corners are not skipped +/// or are checked. +#[allow(clippy::too_many_arguments, clippy::type_complexity)] +fn grid_pass( + statement: &Statement<'_, F, E>, + factors: &[Mle], + reads: &[usize], + l: &[FieldElement], + eq_r: &[FieldElement], + eq_rho: &[FieldElement], + d: usize, + options: Options, + counters: &mut Counters, +) -> Result<(Vec>>, [[FieldElement; 2]; 2]), Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let constraints = statement.constraints; + let quarter = l.len() / 4; + let side = d + 1; + let mut t = vec![vec![FieldElement::::zero(); side]; side]; + let mut u: [[FieldElement; 2]; 2] = Default::default(); + let mut grid: Vec>> = vec![Vec::new(); factors.len()]; + for &slot in reads { + grid[slot] = vec![FieldElement::::zero(); side * side]; + } + let mut scratch: Vec> = Vec::new(); + let row = |a: usize, b: usize, x: usize| a * 2 * quarter + b * quarter + x; + let computed_corners = !options.skip_corners || options.check_corners; + for x in 0..quarter { + for &slot in reads { + let f = factors[slot].evals(); + let g = &mut grid[slot]; + for b in 0..2 { + g[b] = f[row(0, b, x)].clone(); + g[side + b] = f[row(1, b, x)].clone(); + let delta = &g[side + b] - &g[b]; + for a in 2..side { + g[a * side + b] = &g[(a - 1) * side + b] + δ + } + } + for a in 0..side { + let delta = &g[a * side + 1] - &g[a * side]; + for b in 2..side { + g[a * side + b] = &g[a * side + b - 1] + δ + } + } + } + for a in 0..side { + for b in 0..side { + let corner = a < 2 && b < 2; + if corner && !computed_corners { + continue; + } + let read = |slot: usize| grid[slot][a * side + b].clone(); + constraints.run::(&read, &mut scratch); + let c: FieldElement = constraints.combine_base(statement.betas, &read, &scratch); + if corner { + if options.check_corners && c != FieldElement::::zero() { + return Err(Error::ConstraintViolated { row: row(a, b, x) }); + } + if options.skip_corners { + continue; + } + } + t[a][b] += &eq_r[x] * &c; + } + } + for (b, lane) in u.iter_mut().enumerate() { + for (c, cell) in lane.iter_mut().enumerate() { + *cell += &eq_rho[x] * &l[row(b, c, x)]; + } + } + } + grid_ops( + &mut counters.grid, + quarter as u64, + reads.len() as u64, + &Mix::of(constraints.steps()), + constraints.roots.len() as u64, + constraints.selected() as u64, + d as u64, + options, + ); + Ok((t, u)) +} + +/// What the op model needs to know about a table's batch. +#[derive(Clone, Copy, Debug)] +pub struct Shape { + pub num_vars: usize, + /// `C`'s degree, a selector counted. + pub constraint_degree: usize, + /// The table's factors: its committed views and public selectors. + pub factors: usize, + /// The factors the constraint part reads ([`Constraints::reads`]). + pub constraint_reads: usize, + /// The trace factors S1-2's batch program reads. + pub bus_batch_reads: usize, + /// `L`'s terms: the distinct factors the bus reads. + pub bus_terms: usize, + pub roots: usize, + /// Roots with a selector. + pub selected: usize, + /// The constraint DAG's steps. + pub dag: Mix, + /// Today's batch program, as the rounds walk it. + pub batch: Mix, + /// S1-2's batch program ([`bus_batch`]), as the rounds walk it. + pub bus_batch: Mix, +} + +impl Shape { + /// `statement`'s shape over `factors` trace factors, at the height of its + /// zerocheck point: the batch programs as [`prove`] builds them with + /// `lambdas`, walked on demand when `on_demand`. + /// + /// A program's step mix depends on the challenges only where one is zero + /// or one — a coefficient `Builder::weighted_sum` drops, a product + /// `Program::simplify` folds — which a random challenge makes negligible. + pub fn of( + statement: &Statement<'_, F, E>, + factors: usize, + lambdas: &[FieldElement], + on_demand: bool, + ) -> Result + where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, + FieldElement: Send + Sync, + { + if lambdas.len() != 3 { + return Err(Error::VariableCountMismatch { + expected: 3, + got: lambdas.len(), + }); + } + let num_vars = statement.r.len(); + let walked = |program: Program| { + if on_demand { + program.on_demand() + } else { + program + } + }; + let batch = walked(batch_program(statement, num_vars, factors, lambdas)?); + let column = walked(column_program(statement, factors)?); + let mut read: Vec = column + .steps() + .iter() + .filter_map(|step| match step { + Op::Var(i) if (*i as usize) < factors => Some(*i as usize), + _ => None, + }) + .collect(); + read.sort_unstable(); + read.dedup(); + let split = statement.claim_point.len().saturating_sub(num_vars); + let bus = BusColumn::new( + statement.interactions, + &statement.claim_point[..split], + &lambdas[1], + &lambdas[2], + )?; + let constraints = statement.constraints; + Ok(Self { + num_vars, + constraint_degree: constraints.degree, + factors, + constraint_reads: constraints.reads().len(), + bus_batch_reads: read.len(), + bus_terms: bus.terms.len(), + roots: constraints.roots.len(), + selected: constraints.selected(), + dag: Mix::of(&constraints.steps), + batch: Mix::of(batch.steps()), + bus_batch: Mix::of(column.steps()), + }) + } +} + +/// Each variant's counts at `shape`'s height, in closed form: what [`prove`] +/// counts as it runs, for a table the host cannot afford to run. +/// +/// A round `j` works on `half_j = 2^{n−1−j}` indices, so rounds `from..n` on +/// `2^{n−from} − 1` of them. Assumes no `E^r_j·r_j` vanishes, which a random +/// point makes negligible. +pub fn model(shape: &Shape, options: Options) -> Counters { + let n = shape.num_vars as u64; + let cells = 1u64 << n; + // Indices over rounds `from..n`. + let over = |from: u64| (cells >> from.min(n)).saturating_sub(1); + let d = shape.constraint_degree.max(1) as u64; + let degree = d + 1; + let (roots, selected) = (shape.roots as u64, shape.selected as u64); + let factors = shape.factors as u64; + let reads = shape.constraint_reads as u64; + let mut c = Counters::default(); + match options.variant { + Variant::Today => { + eq_ops(&mut c.weights, n); + eq_ops(&mut c.weights, n); + walk_ops(&mut c.rounds, &shape.batch, degree * over(0)); + fold_ops(&mut c.fold, factors + 2, over(0)); + } + Variant::Bus => { + column_ops(&mut c.bus_column, cells, shape.bus_terms as u64); + eq_ops(&mut c.weights, n); + eq_ops(&mut c.weights, n); + walk_ops(&mut c.rounds, &shape.bus_batch, degree * over(0)); + fold_ops(&mut c.fold, shape.bus_batch_reads as u64 + 3, over(0)); + bound_ops(&mut c.bound, factors - shape.bus_batch_reads as u64, cells); + } + Variant::Gruen | Variant::Grid => { + column_ops(&mut c.bus_column, cells, shape.bus_terms as u64); + let start = if options.variant == Variant::Grid && n >= 2 { + let groups = cells / 4; + eq_ops(&mut c.weights, n - 2); + eq_ops(&mut c.weights, n - 2); + grid_ops( + &mut c.grid, + groups, + reads, + &shape.dag, + roots, + selected, + d, + options, + ); + message_ops(&mut c.assembly, d, 2); + lagrange_ops(&mut c.assembly, d); + double_fold_ops(&mut c.fold, groups, reads); + 2 + } else { + eq_ops(&mut c.weights, n.saturating_sub(1)); + eq_ops(&mut c.weights, n.saturating_sub(1)); + 0 + }; + // The halvings: each weight is built for the first round that + // walks it, and halved into every round's after that. + c.weights.ext_add += 2 * if start == 2 { over(2) } else { over(1) }; + gruen_ops(&mut c.rounds, &shape.dag, roots, selected, d, over(start)); + let rounds = n.saturating_sub(start); + derive_ops(&mut c.assembly, rounds); + message_ops(&mut c.assembly, d, rounds); + fold_ops(&mut c.fold, reads + 1, over(start)); + bound_ops(&mut c.bound, factors - reads, cells); + } + } + c +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + batch::{self, Rule}, + eq::eq_mle, + logup::Affine, + selector::Selector, + }; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3, + goldilocks::GoldilocksField as Gl, + }; + + type FB = FieldElement; + type FX = FieldElement; + + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn base(&mut self) -> FB { + FB::from(self.next()) + } + + fn ext(&mut self) -> FX { + FX::new([self.base(), self.base(), self.base()]) + } + + fn exts(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.ext()).collect() + } + } + + // The table's factors: four columns, a column's next-row view, the + // transition's selector, and a column only the bus reads. + const A: usize = 0; + const B: usize = 1; + const C: usize = 2; + const D: usize = 3; + const A_NEXT: usize = 4; + const SELECTOR: usize = 5; + const BUS_ONLY: usize = 6; + const WIDTH: usize = 7; + + struct Table { + constraints: Constraints, + interactions: Vec>, + betas: Vec, + } + + /// A table whose constraint part has degree `degree`. At 1: `c = a + b`. + /// From 2: `a_next = a + 1` under the selector (off on the last row), and + /// `c = a·b + a`, beside a root that is zero whatever the trace (a + /// negation and a constant in it). At 3: `d = a·b·c` too. Three bus + /// interactions, so the fourth slot is padding. + fn table(degree: usize, rng: &mut Rng) -> Table { + let mut b = Builder::::new(); + let a = b.var(A); + let bv = b.var(B); + let c = b.var(C); + let d = b.var(D); + let next = b.var(A_NEXT); + let one = b.fixed(FX::one()); + let seven = b.fixed(FX::from(7u64)); + let mut roots = Vec::new(); + let mut selectors = Vec::new(); + if degree == 1 { + let sum = b.add(a, bv); + roots.push(b.sub(c, sum)); + selectors.push(None); + } else { + let step = b.sub(next, a); + roots.push(b.sub(step, one)); + selectors.push(Some(SELECTOR)); + let ab = b.mul(a, bv); + let root = b.sub(c, ab); + roots.push(b.sub(root, a)); + selectors.push(None); + let negated = b.neg(ab); + let zero = b.add(negated, ab); + roots.push(b.mul(zero, seven)); + selectors.push(None); + if degree == 3 { + let abc = b.mul(ab, c); + roots.push(b.sub(d, abc)); + selectors.push(None); + } + } + let program = b.finish_verbatim(roots[0]).unwrap(); + let constraints = + Constraints::::from_extension(program.steps(), &roots, &selectors, degree).unwrap(); + let (alpha, z) = (rng.ext(), rng.ext()); + let interactions = vec![ + Interaction::new( + Affine::factor(BUS_ONLY), + Affine::new(vec![(A, alpha), (B, alpha * alpha)], z), + ), + Interaction::new( + Affine::new(vec![(SELECTOR, -FX::one())], FX::zero()), + Affine::new(vec![(C, alpha)], z + FX::from(3u64)), + ), + Interaction::new( + Affine::constant(FX::from(2u64)), + Affine::new(vec![(A, alpha * alpha * alpha), (D, FX::from(5u64))], z), + ), + ]; + let beta = rng.ext(); + let mut betas = vec![FX::one()]; + for _ in 1..roots.len() { + let next = betas[betas.len() - 1] * beta; + betas.push(next); + } + Table { + constraints, + interactions, + betas, + } + } + + /// The table's factors over `2^n` rows: a trace that satisfies it, or + /// random columns. The next-row view and the selector are what they are + /// either way. + fn factors(n: usize, degree: usize, satisfying: bool, rng: &mut Rng) -> Vec> { + let size = 1usize << n; + let random = |rng: &mut Rng| -> Vec { (0..size).map(|_| rng.base()).collect() }; + let a: Vec = if satisfying { + (0..size as u64).map(|i| FB::from(i + 7)).collect() + } else { + random(rng) + }; + let b = random(rng); + let c: Vec = match (satisfying, degree) { + (true, 1) => a.iter().zip(&b).map(|(x, y)| x + y).collect(), + (true, _) => a.iter().zip(&b).map(|(x, y)| x * y + x).collect(), + (false, _) => random(rng), + }; + let d: Vec = if satisfying && degree == 3 { + (0..size).map(|i| a[i] * b[i] * c[i]).collect() + } else { + random(rng) + }; + let next: Vec = (0..size).map(|i| a[(i + 1) % size]).collect(); + let bus_only = random(rng); + let selector = Selector::except_last(1).table::(n).unwrap(); + vec![ + Mle::new(a).unwrap(), + Mle::new(b).unwrap(), + Mle::new(c).unwrap(), + Mle::new(d).unwrap(), + Mle::new(next).unwrap(), + selector, + Mle::new(bus_only).unwrap(), + ] + } + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"fused-argue") + } + + fn lift(factors: &[Mle]) -> Vec> { + factors + .iter() + .map(|f| { + Mle::new(f.evals().iter().map(|v| v.to_extension::()).collect()).unwrap() + }) + .collect() + } + + /// Today's three rules and their factor list, as `multilinear_table::prove` + /// builds them: the trace's factors lifted, then `eq(r, ·)` and `eq(ρ, ·)`. + fn todays( + table: &Table, + factors: &[Mle], + r: &[FX], + claim_point: &[FX], + ) -> (Vec>, Vec>) { + let n = factors[0].num_vars(); + let zerocheck = Rule::compiled( + table.constraints.degree() + 1, + table + .constraints + .zerocheck_program(&table.betas, WIDTH) + .unwrap(), + ); + let interactions: &'static [Interaction] = + Box::leak(table.interactions.clone().into_boxed_slice()); + let bus = logup::claim_statements(interactions, claim_point, n, WIDTH + 1).unwrap(); + let mut polys = lift(factors); + polys.push(eq_mle(r).unwrap()); + polys.push(eq_mle(&bus.row_point).unwrap()); + (polys, vec![zerocheck, bus.numerator, bus.denominator]) + } + + /// Each rule's sum over the cube: the claims an honest batch carries. + fn true_claims(polys: &[Mle], rules: &[Rule<'_, Ext3>]) -> [FX; 3] { + let size = polys[0].len(); + core::array::from_fn(|k| { + (0..size).fold(FX::zero(), |acc, i| { + let values: Vec = polys.iter().map(|p| p.evals()[i]).collect(); + acc + rules[k].apply(&values) + }) + }) + } + + struct Case { + table: Table, + factors: Vec>, + r: Vec, + claim_point: Vec, + } + + fn case(n: usize, degree: usize, satisfying: bool, seed: u64) -> Case { + let mut rng = Rng(0x9E37_79B9_7F4A_7C15 ^ seed.wrapping_mul(0x2545_F491_4F6C_DD1D) | 1); + let table = table(degree, &mut rng); + let factors = factors(n, degree, satisfying, &mut rng); + let r = rng.exts(n); + let claim_point = rng.exts(logup::input_layer_vars(table.interactions.len(), n)); + Case { + table, + factors, + r, + claim_point, + } + } + + impl Case { + fn statement(&self) -> Statement<'_, Gl, Ext3> { + Statement { + constraints: &self.table.constraints, + betas: &self.table.betas, + interactions: &self.table.interactions, + claim_point: &self.claim_point, + r: &self.r, + } + } + + /// The claims: the true sums, or with the zerocheck's forced to zero — + /// what a prover claims whatever its trace. + fn claims(&self, zerocheck_zero: bool) -> [FX; 3] { + let (polys, rules) = todays(&self.table, &self.factors, &self.r, &self.claim_point); + let mut claims = true_claims(&polys, &rules); + if zerocheck_zero { + claims[0] = FX::zero(); + } + claims + } + + /// Today's proof through `batch::prove`, and the transcript after it. + fn today( + &self, + claims: &[FX; 3], + ) -> (SumcheckProof, Vec, DefaultTranscript) { + let (polys, rules) = todays(&self.table, &self.factors, &self.r, &self.claim_point); + let mut t = transcript(); + let (proof, point) = batch::prove(polys, rules, claims, &mut t).unwrap(); + (proof, point, t) + } + + fn fused( + &self, + claims: &[FX; 3], + options: Options, + ) -> ( + Result, Error>, + DefaultTranscript, + Counters, + ) { + let mut t = transcript(); + let mut counters = Counters::default(); + let proved = prove( + &self.statement(), + &self.factors, + claims, + options, + &mut t, + &mut counters, + ); + (proved, t, counters) + } + + /// The first round at which `options` sends another message than + /// today's, or `None`; with every other part of the result checked + /// when there is none. + fn parts_at(&self, claims: &[FX; 3], options: Options) -> Option { + let (proof, point, mut t_today) = self.today(claims); + let (proved, mut t, _) = self.fused(claims, options); + let (fused, fused_point, bound) = proved.unwrap(); + let parted = proof + .rounds + .iter() + .zip(&fused.rounds) + .position(|(a, b)| a.evaluations != b.evaluations); + if parted.is_some() { + return parted; + } + assert_eq!(fused.rounds.len(), proof.rounds.len(), "{options:?}"); + assert_eq!(fused_point, point, "{options:?}: the point"); + assert_eq!(t.state(), t_today.state(), "{options:?}: the transcript"); + assert_eq!( + t.sample_field_element(), + t_today.sample_field_element(), + "{options:?}: the next challenge" + ); + for (slot, (factor, value)) in self.factors.iter().zip(&bound).enumerate() { + assert_eq!( + *value, + factor.evaluate_in(&point).unwrap(), + "{options:?}: factor {slot} at the point" + ); + } + None + } + } + + fn every(satisfying: bool) -> Vec { + let mut all = Vec::new(); + for variant in [Variant::Today, Variant::Bus] { + for on_demand in [false, true] { + all.push(Options { + on_demand, + ..Options::new(variant) + }); + } + } + all.push(Options::new(Variant::Gruen)); + all.push(Options::new(Variant::Grid)); + if satisfying { + all.push(Options { + skip_corners: true, + ..Options::new(Variant::Grid) + }); + all.push(Options { + skip_corners: true, + check_corners: true, + ..Options::new(Variant::Grid) + }); + } + all + } + + /// ★ Every variant sends today's messages — the same rounds, point, + /// transcript and next challenge as `batch::prove` over today's batch, and + /// every factor's value at the point — at every height from none to 2^7, at + /// each constraint degree, on random columns (the corners computed) and on + /// a trace that satisfies the table (the corners skipped, and checked). + #[test] + fn every_variant_sends_todays_messages() { + for degree in 1..=3 { + for n in 0..=7 { + for satisfying in [false, true] { + let case = case(n, degree, satisfying, (degree * 100 + n) as u64); + let claims = case.claims(false); + if satisfying { + assert_eq!(claims[0], FX::zero(), "a satisfying trace's zerocheck"); + } + for options in every(satisfying) { + assert_eq!( + case.parts_at(&claims, options), + None, + "degree {degree}, n {n}, satisfying {satisfying}: {options:?}" + ); + } + } + } + } + } + + /// ⛔ The corner skip is only what today sends on a trace that satisfies + /// its AIR: on random columns the skipped corners are not zero and the + /// first message moves, and the corner check refuses at the first row. + #[test] + fn the_corner_skip_parts_from_today_on_random_columns() { + for degree in 1..=3 { + let case = case(5, degree, false, 7 + degree as u64); + let claims = case.claims(false); + let skip = Options { + skip_corners: true, + ..Options::new(Variant::Grid) + }; + assert_eq!(case.parts_at(&claims, skip), Some(0), "degree {degree}"); + let checked = Options { + check_corners: true, + ..skip + }; + assert_eq!( + case.fused(&claims, checked).0.unwrap_err(), + Error::ConstraintViolated { row: 0 }, + "degree {degree}" + ); + } + } + + /// ⛔ NEGATIVE CONTROL: one cell of a satisfying trace flipped. The prover + /// still claims zero for the zerocheck; today's rounds send the broken + /// trace's sums and the stage-1 ones do not, from round 0 — both proofs + /// rejected, with different bytes (D-ARGUE §2.7 (a)). The corner check + /// names the row. + #[test] + fn a_flipped_cell_parts_the_messages_and_the_check_names_its_row() { + for degree in 1..=3 { + let mut case = case(5, degree, true, 40 + degree as u64); + let row = 5; + let mut c = case.factors[C].evals().to_vec(); + c[row] += FB::one(); + case.factors[C] = Mle::new(c).unwrap(); + let claims = case.claims(true); + for options in [ + Options::new(Variant::Gruen), + Options { + skip_corners: true, + ..Options::new(Variant::Grid) + }, + ] { + assert_eq!( + case.parts_at(&claims, options), + Some(0), + "degree {degree}: {options:?}" + ); + } + let checked = Options { + skip_corners: true, + check_corners: true, + ..Options::new(Variant::Grid) + }; + assert_eq!( + case.fused(&claims, checked).0.unwrap_err(), + Error::ConstraintViolated { row }, + "degree {degree}" + ); + } + } + + /// ⛔ NEGATIVE CONTROL: one coefficient of `L` wrong. Every variant that + /// reads `L` parts from today at round 0. + #[test] + fn a_wrong_bus_coefficient_parts_the_messages() { + let case = case(6, 3, true, 77); + let claims = case.claims(false); + for fault in 0..3 { + for options in [ + Options::new(Variant::Bus), + Options::new(Variant::Gruen), + Options { + skip_corners: true, + ..Options::new(Variant::Grid) + }, + ] { + let faulty = Options { + bus_fault: Some(fault), + ..options + }; + assert_eq!(case.parts_at(&claims, faulty), Some(0), "{faulty:?}"); + } + } + } + + /// ★ The model is what the reference counts, phase by phase, for every + /// variant at every height it runs — which is what lets the model price a + /// table at a height the host cannot run. + #[test] + fn the_model_is_what_the_reference_counts() { + for degree in 1..=3 { + for n in [0, 1, 2, 3, 4, 6] { + for satisfying in [false, true] { + let case = case(n, degree, satisfying, 900 + (degree * 10 + n) as u64); + let claims = case.claims(false); + let mut t = transcript(); + for claim in &claims { + t.append_field_element(claim); + } + let lambdas = challenge_powers(&t.sample_field_element(), 3); + for options in every(satisfying) { + let shape = + Shape::of(&case.statement(), WIDTH, &lambdas, options.on_demand) + .unwrap(); + let (proved, _, counters) = case.fused(&claims, options); + proved.unwrap(); + assert_eq!( + model(&shape, options), + counters, + "degree {degree}, n {n}: {options:?}" + ); + } + } + } + } + } + + /// `L` is the bus's two rules, weighted: at a random point it is + /// `λ_N·N + λ_D·D`, the rules read with a row weight of one. + #[test] + fn the_bus_column_is_the_two_rules() { + let mut rng = Rng(0x5eed); + let table = table(3, &mut rng); + let n = 4; + let claim_point = rng.exts(logup::input_layer_vars(table.interactions.len(), n)); + let (numerator, denominator) = (rng.ext(), rng.ext()); + let split = claim_point.len() - n; + let column = BusColumn::new( + &table.interactions, + &claim_point[..split], + &numerator, + &denominator, + ) + .unwrap(); + let rules = logup::claim_statements(&table.interactions, &claim_point, n, WIDTH).unwrap(); + for _ in 0..4 { + let mut values = rng.exts(WIDTH); + values.push(FX::one()); + let expected = numerator * rules.numerator.apply(&values) + + denominator * rules.denominator.apply(&values); + assert_eq!(column.evaluate(&values), expected); + } + // Every factor the bus reads is one term. + let slots: Vec = column.terms().iter().map(|(slot, _)| *slot).collect(); + assert_eq!(slots, [A, B, C, D, SELECTOR, BUS_ONLY]); + } + + /// A constraint constant with an extension part cannot run in the base + /// field, and says which step it is. + #[test] + fn a_constant_with_an_extension_part_is_refused() { + let mut b = Builder::::new(); + let a = b.var(0); + let odd = b.fixed(FX::new([FB::one(), FB::one(), FB::zero()])); + let root = b.mul(a, odd); + let program = b.finish_verbatim(root).unwrap(); + assert_eq!( + Constraints::::from_extension(program.steps(), &[root], &[None], 1).unwrap_err(), + Error::NotBaseField { step: 1 } + ); + } +} diff --git a/crypto/multilinear/src/gkr.rs b/crypto/multilinear/src/gkr.rs new file mode 100644 index 000000000..80a31546d --- /dev/null +++ b/crypto/multilinear/src/gkr.rs @@ -0,0 +1,914 @@ +//! LogUp as a tree of fractions: `p₁/q₁ + p₂/q₂ = (p₁q₂ + p₂q₁)/(q₁q₂)`, added +//! pairwise up a binary tree with GKR proving each layer against the one below. +//! Only the input layer is ever committed. +//! +//! Proves the sum is whatever the output claims. Checking that the output +//! numerator is zero — the bus balance — is the caller's. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{element::FieldElement, traits::IsField}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{ + Error, + eq::{eq_eval, eq_mle}, + mle::Mle, + poly::SumcheckPolynomial, + program::{Builder, Program}, + sumcheck::{self, SumcheckProof}, + whir_split, +}; + +/// One level of the tree: numerators and denominators over the same cube. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FractionLayer { + pub p: Mle, + pub q: Mle, +} + +impl FractionLayer { + pub fn new(p: Mle, q: Mle) -> Result { + if p.num_vars() != q.num_vars() { + return Err(Error::VariableCountMismatch { + expected: p.num_vars(), + got: q.num_vars(), + }); + } + Ok(Self { p, q }) + } + + pub fn num_vars(&self) -> usize { + self.p.num_vars() + } + + /// Adds the two halves pointwise, giving the layer one level up. + pub fn fold(&self) -> Result + where + FieldElement: Send + Sync, + { + if self.num_vars() == 0 { + return Err(Error::NoVariablesLeft); + } + let half = self.p.len() / 2; + let (p_lo, p_hi) = self.p.evals().split_at(half); + let (q_lo, q_hi) = self.q.evals().split_at(half); + + // Every index folds on its own, and building the tree is a pass over + // the input layer at every level, so this is worth the pool. + let both = |i: usize| { + ( + &p_lo[i] * &q_hi[i] + &p_hi[i] * &q_lo[i], + &q_lo[i] * &q_hi[i], + ) + }; + #[cfg(feature = "parallel")] + let (next_p, next_q): (Vec<_>, Vec<_>) = if half >= crate::SERIAL_BELOW { + (0..half).into_par_iter().map(both).unzip() + } else { + (0..half).map(both).unzip() + }; + #[cfg(not(feature = "parallel"))] + let (next_p, next_q): (Vec<_>, Vec<_>) = (0..half).map(both).unzip(); + + Self::new(Mle::new(next_p)?, Mle::new(next_q)?) + } +} + +/// The whole tree, from the input layer down to the single output fraction. +/// +/// `layers[0]` is the output (zero variables); the last entry is the input. +#[derive(Debug)] +pub struct FractionTree { + /// Empty when the tree lives on a device, which holds every layer. + layers: Vec>, + device: Option, + num_layers: usize, + output: (FieldElement, FieldElement), +} + +impl FractionTree { + /// Builds every layer by repeated folding. + /// + /// On a device the layers stay there: the tree is the biggest thing a + /// table's argument holds, and GKR reads every level of it. + pub fn build(input: FractionLayer) -> Result + where + FieldElement: Send + Sync, + { + if let Some(device) = crate::gpu::build_tree(&input.p, &input.q) { + return Self::from_device(device); + } + + let mut layers = vec![input]; + while layers.last().expect("non-empty").num_vars() > 0 { + let next = layers.last().expect("non-empty").fold()?; + layers.push(next); + } + layers.reverse(); + let top = &layers[0]; + let output = (top.p.evals()[0].clone(), top.q.evals()[0].clone()); + let num_layers = layers.len(); + Ok(Self { + layers, + device: None, + num_layers, + output, + }) + } + + /// A tree a device already holds, layers and all. + pub fn from_device(device: crate::gpu::DeviceTree) -> Result { + let num_layers = device.num_layers(); + let output = device.output()?; + Ok(Self { + layers: Vec::new(), + device: Some(device), + num_layers, + output, + }) + } + + /// The output fraction `(p, q)`. The bus balances when `p` is zero. + pub fn output(&self) -> (FieldElement, FieldElement) { + self.output.clone() + } + + pub fn num_layers(&self) -> usize { + self.num_layers + } + + /// The layer, for a tree that kept them here. + pub fn layer(&self, i: usize) -> &FractionLayer { + &self.layers[i] + } + + /// The device holding every layer, when one does. + pub fn device(&self) -> Option<&crate::gpu::DeviceTree> { + self.device.as_ref() + } + + pub fn input_layer(&self) -> &FractionLayer { + self.layers.last().expect("non-empty") + } + + /// The levels near the output, back here — `prefix[i]` is layer `i`. + /// + /// Empty for a tree that already lives here. One download brings the whole + /// prefix: the levels above the deepest of them are its folds, and the + /// deepest is a few kilobytes. + fn host_prefix(&self) -> Vec> + where + FieldElement: Send + Sync, + { + let Some(device) = self.device.as_ref() else { + return Vec::new(); + }; + // Never the input layer: a tree that dropped it writes it again for its + // own sumcheck, and it is the one level too big to walk here. + let deepest = HOST_LAYER_VARS.min(self.num_layers.saturating_sub(2)); + let Some((p, q)) = device.layer_to_host::(deepest) else { + return Vec::new(); + }; + let Ok(layer) = FractionLayer::new(p, q) else { + return Vec::new(); + }; + if layer.num_vars() != deepest { + return Vec::new(); + } + let mut prefix = vec![layer]; + while prefix.last().expect("non-empty").num_vars() > 0 { + let Ok(next) = prefix.last().expect("non-empty").fold() else { + return Vec::new(); + }; + prefix.push(next); + } + prefix.reverse(); + prefix + } +} + +/// The deepest level that comes here whole: the one whose halves are already +/// a cube of [`crate::HOST_CUBE_DIRECT`], so its sumcheck never goes to a device. +const HOST_LAYER_VARS: usize = crate::HOST_CUBE_DIRECT.trailing_zeros() as usize + 1; + +/// The layer relation: `Σ_x eq(r,x)·[p_lo·q_hi + p_hi·q_lo + λ·q_lo·q_hi]`, +/// which equals `p_out(r) + λ·q_out(r)` when the layer really is the fold. +pub(crate) struct LayerRelation { + /// `[eq, p_lo, p_hi, q_lo, q_hi]`. + polys: Vec>, + lambda: FieldElement, + /// The same rule as straight-line code, for the device path. `None` for a + /// relation that is here on purpose — a level the tree handed over, or the + /// tail of one — so the dispatch does not send it back. + program: Option>, +} + +impl LayerRelation { + const EQ: usize = 0; + const P_LO: usize = 1; + const P_HI: usize = 2; + const Q_LO: usize = 3; + const Q_HI: usize = 4; + + fn new( + next: &FractionLayer, + r: &[FieldElement], + lambda: FieldElement, + dispatchable: bool, + ) -> Result { + let half = next.p.len() / 2; + let split = |m: &Mle| -> Result<(Mle, Mle), Error> { + Ok(( + Mle::new(m.evals()[..half].to_vec())?, + Mle::new(m.evals()[half..].to_vec())?, + )) + }; + let (p_lo, p_hi) = split(&next.p)?; + let (q_lo, q_hi) = split(&next.q)?; + Ok(Self { + polys: vec![eq_mle(r)?, p_lo, p_hi, q_lo, q_hi], + program: dispatchable + .then(|| Self::program_for(&lambda)) + .transpose()?, + lambda, + }) + } + + /// The same relation over factors a device already folded: the weight and + /// the four halves as its last round left them. + pub(crate) fn from_factors(polys: Vec>, lambda: FieldElement) -> Result { + if polys.len() != 5 { + return Err(Error::VariableCountMismatch { + expected: 5, + got: polys.len(), + }); + } + Ok(Self { + polys, + program: None, + lambda, + }) + } + + /// The five factors as the rounds left them, for a test to compare. + #[cfg(test)] + pub(crate) fn polys_for_tests(&self) -> &[Mle] { + &self.polys + } + + /// `eq·(p_lo·q_hi + p_hi·q_lo + lambda·q_lo·q_hi)`, the same expression + /// [`combine`](SumcheckPolynomial::combine) evaluates. + fn program_for(lambda: &FieldElement) -> Result, Error> { + let mut b = Builder::::new(); + let eq = b.var(Self::EQ); + let p_lo = b.var(Self::P_LO); + let p_hi = b.var(Self::P_HI); + let q_lo = b.var(Self::Q_LO); + let q_hi = b.var(Self::Q_HI); + let first = b.mul(p_lo, q_hi); + let second = b.mul(p_hi, q_lo); + let numerator = b.add(first, second); + let denominator = b.mul(q_lo, q_hi); + let weighted = b.fixed(lambda.clone()); + let scaled = b.mul(weighted, denominator); + let sum = b.add(numerator, scaled); + let root = b.mul(eq, sum); + b.finish(root) + } +} + +/// `eq` times a product of two layer values. +const LAYER_DEGREE: usize = 3; + +impl SumcheckPolynomial for LayerRelation { + fn num_vars(&self) -> usize { + self.polys[Self::EQ].num_vars() + } + + fn degree(&self) -> usize { + LAYER_DEGREE + } + + fn polys(&self) -> &[Mle] { + &self.polys + } + + fn combine(&self, v: &[FieldElement]) -> FieldElement { + let numerator = &v[Self::P_LO] * &v[Self::Q_HI] + &v[Self::P_HI] * &v[Self::Q_LO]; + let denominator = &v[Self::Q_LO] * &v[Self::Q_HI]; + &v[Self::EQ] * (numerator + &self.lambda * denominator) + } + + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), Error> { + for p in &mut self.polys { + p.fix_first_variable_in_place(r)?; + } + Ok(()) + } + + fn program(&self) -> Option<&Program> { + self.program.as_ref() + } + + /// The layer relation is four multiplications written out, not a program + /// the host walks — so its rounds are worth taking back much earlier. + fn host_cube(&self) -> usize { + crate::HOST_CUBE_DIRECT + } + + fn accept_folded(&mut self, polys: Vec>) -> Result<(), Error> { + if polys.len() != self.polys.len() { + return Err(Error::VariableCountMismatch { + expected: self.polys.len(), + got: polys.len(), + }); + } + self.polys = polys; + Ok(()) + } +} + +/// One layer's transcript: the sumcheck plus the four values it reduces to. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct LayerProof { + pub sumcheck: SumcheckProof, + pub p_lo: FieldElement, + pub p_hi: FieldElement, + pub q_lo: FieldElement, + pub q_hi: FieldElement, +} + +/// A proof for the whole tree, output layer first. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct GkrProof { + pub layers: Vec>, +} + +/// What the verifier is left holding about the **input** layer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GkrClaim { + pub point: Vec>, + pub p: FieldElement, + pub q: FieldElement, +} + +/// What proving leaves the caller holding. +/// +/// The claim is the same one [`verify`] arrives at: the prover needs it to +/// discharge the input layer, which is where the trace is. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GkrOutput { + pub proof: GkrProof, + pub claim: GkrClaim, +} + +/// `(1 − c)·lo + c·hi` — the multilinear interpolation that turns the two +/// restricted claims back into one claim on the fuller layer. +fn combine_halves( + lo: &FieldElement, + hi: &FieldElement, + c: &FieldElement, +) -> FieldElement { + lo + c * &(hi - lo) +} + +/// Values at `0, 1, 2, 3` interpolated at a point: [`sumcheck::interpolate`] for +/// a cubic, with the four Lagrange denominators inverted once rather than on +/// every call. +pub(crate) struct Cubic { + /// `1 / Π_{j≠i} (i − j)` for `i = 0..4`: `−1/6, 1/2, −1/2, 1/6`. + inverse: [FieldElement; 4], +} + +impl Cubic { + pub(crate) fn new() -> Option { + let six = FieldElement::::from(6u64); + let two = FieldElement::::from(2u64); + let mut inverse = [-six.clone(), two.clone(), -two, six]; + FieldElement::inplace_batch_inverse(&mut inverse).ok()?; + Some(Self { inverse }) + } + + pub(crate) fn at(&self, values: [&FieldElement; 4], x: &FieldElement) -> FieldElement { + let d: [FieldElement; 4] = + std::array::from_fn(|j| x - FieldElement::::from(j as u64)); + let low = &d[0] * &d[1]; + let high = &d[2] * &d[3]; + let others = [&d[1] * &high, &d[0] * &high, &low * &d[3], &low * &d[2]]; + values + .iter() + .zip(others.iter().zip(&self.inverse)) + .fold(FieldElement::zero(), |acc, (v, (o, inv))| { + acc + *v * o * inv + }) + } +} + +/// Proves the tree, from the output fraction down to the input layer. +pub fn prove(tree: &FractionTree, transcript: &mut T) -> Result, Error> +where + F: IsField + 'static, + T: IsTranscript, +{ + let mut layers = Vec::with_capacity(tree.num_layers().saturating_sub(1)); + // The output layer has no variables, so the first claim sits at the empty + // point and needs no challenge. + let mut point: Vec> = Vec::new(); + let (mut p_claim, mut q_claim) = tree.output(); + // The levels near the output, fetched once. Their rounds are over cubes a + // core walks in microseconds, and a device pays a launch for each. + let t = whir_split::tick(); + let prefix = tree.host_prefix(); + whir_split::add_tick(&whir_split::REST_PREFIX, t); + + for i in 0..tree.num_layers() - 1 { + // A layer the device runs, whose host work the instrument splits + // (`whir_split`, under `LAMBDA_VM_BASE_SPLIT` only). + let on_card = prefix.get(i + 1).is_none() && tree.device().is_some(); + let clock = || if on_card { whir_split::tick() } else { None }; + + let t = clock(); + let lambda: FieldElement = transcript.sample_field_element(); + whir_split::add_tick(&whir_split::GKR_LAMBDA, t); + + // What the device ran of this layer, and the relation it left behind. + // A tree the device holds proves its layer where it lies — the halves + // are the sumcheck's factors in place — until the cube reaches the + // crossover, and hands the factors over from there. + let (mut rounds, mut z, mut relation) = match prefix.get(i + 1) { + Some(here) => ( + Vec::new(), + Vec::new(), + LayerRelation::new(here, &point, lambda, false)?, + ), + None => match tree.device() { + // D-ARGUE S1-3: the same rounds, from the card's `H(1), H(2)`. + Some(device) if crate::gkr_gruen::argue_gkr_gruen() => { + let t = clock(); + // Today's program, only for the cross-check to walk beside. + let today = crate::gkr_gruen::argue_gkr_gruen_xcheck() + .then(|| LayerRelation::program_for(&lambda)) + .transpose()?; + whir_split::add_tick(&whir_split::GKR_PROGRAM, t); + let attempt = device.prove_layer_gruen( + i + 1, + &point, + &lambda, + &p_claim + &lambda * &q_claim, + crate::gkr_gruen::gkr_gruen_tail(), + today.as_ref(), + |sent| { + for value in sent { + transcript.append_field_element(value); + } + transcript.sample_field_element() + }, + ); + let Some(outcome) = attempt else { + return Err(Error::DeviceFailed { + stage: "layer sumcheck", + }); + }; + let (rounds, z, factors) = outcome?; + (rounds, z, LayerRelation::from_factors(factors, lambda)?) + } + Some(device) => { + let t = clock(); + let program = LayerRelation::program_for(&lambda)?; + whir_split::add_tick(&whir_split::GKR_PROGRAM, t); + let attempt = device.prove_layer( + i + 1, + &point, + &program, + LAYER_DEGREE, + crate::HOST_CUBE_DIRECT, + |sent| { + for value in sent { + transcript.append_field_element(value); + } + transcript.sample_field_element() + }, + ); + let Some(outcome) = attempt else { + return Err(Error::DeviceFailed { + stage: "layer sumcheck", + }); + }; + let (rounds, z, factors) = outcome?; + (rounds, z, LayerRelation::from_factors(factors, lambda)?) + } + None => ( + Vec::new(), + Vec::new(), + LayerRelation::new(tree.layer(i + 1), &point, lambda, true)?, + ), + }, + }; + + // The tail, however much of it is left: all of it for a level that came + // here whole, none for one the device ran out. + let left = relation.num_vars(); + let t = clock(); + let (tail, tail_z) = if t.is_some() { + // The same calls on the same transcript, with its share timed. + let mut timed = + whir_split::Timed::new(&mut *transcript, &whir_split::GKR_TAIL_TRANSCRIPT); + sumcheck::prove_rounds(&mut relation, left, &mut timed)? + } else { + sumcheck::prove_rounds(&mut relation, left, transcript)? + }; + whir_split::add_tick(&whir_split::GKR_TAIL, t); + if on_card { + whir_split::bump_by(&whir_split::GKR_TAIL_ROUNDS, left as u64); + } + rounds.extend(tail); + z.extend(tail_z); + let sumcheck = SumcheckProof { rounds }; + + // The four restricted values the verifier needs to close the round are + // what the sumcheck's own factors have become: binding every variable + // to `z` is the evaluation at `z`. Evaluating the halves again would be + // a second pass over the layer, and the layers are the biggest thing + // the fraction tree holds. + let t = clock(); + let bound = |slot: usize| -> Result, Error> { + relation.polys()[slot] + .as_constant() + .cloned() + .ok_or(Error::NoVariablesLeft) + }; + let p_lo = bound(LayerRelation::::P_LO)?; + let p_hi = bound(LayerRelation::::P_HI)?; + let q_lo = bound(LayerRelation::::Q_LO)?; + let q_hi = bound(LayerRelation::::Q_HI)?; + + for v in [&p_lo, &p_hi, &q_lo, &q_hi] { + transcript.append_field_element(v); + } + let c = transcript.sample_field_element(); + + // Next layer's claim lives at (c, z). + p_claim = combine_halves(&p_lo, &p_hi, &c); + q_claim = combine_halves(&q_lo, &q_hi, &c); + point = std::iter::once(c).chain(z).collect(); + + layers.push(LayerProof { + sumcheck, + p_lo, + p_hi, + q_lo, + q_hi, + }); + whir_split::add_tick(&whir_split::GKR_CLOSE, t); + } + + Ok(GkrOutput { + proof: GkrProof { layers }, + claim: GkrClaim { + point, + p: p_claim, + q: q_claim, + }, + }) +} + +/// Verifies the tree against a claimed output fraction. +pub fn verify( + proof: &GkrProof, + output: (FieldElement, FieldElement), + transcript: &mut T, +) -> Result, Error> +where + F: IsField + 'static, + T: IsTranscript, +{ + let (mut p_claim, mut q_claim) = output; + let mut point: Vec> = Vec::new(); + + for (i, layer) in proof.layers.iter().enumerate() { + let lambda = transcript.sample_field_element(); + let claimed_sum = &p_claim + &lambda * &q_claim; + + let claim = sumcheck::verify(&layer.sumcheck, claimed_sum, point.len(), 3, transcript)?; + + // The sumcheck's residual must be the layer relation at that point. + let eq_at = eq_eval(&point, &claim.point)?; + let numerator = &layer.p_lo * &layer.q_hi + &layer.p_hi * &layer.q_lo; + let denominator = &layer.q_lo * &layer.q_hi; + let expected = eq_at * (numerator + &lambda * denominator); + if expected != claim.expected_evaluation { + return Err(Error::LayerRelationMismatch { layer: i }); + } + + for v in [&layer.p_lo, &layer.p_hi, &layer.q_lo, &layer.q_hi] { + transcript.append_field_element(v); + } + let c = transcript.sample_field_element(); + + p_claim = combine_halves(&layer.p_lo, &layer.p_hi, &c); + q_claim = combine_halves(&layer.q_lo, &layer.q_hi, &c); + point = std::iter::once(c).chain(claim.point).collect(); + } + + Ok(GkrClaim { + point, + p: p_claim, + q: q_claim, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"gkr-test") + } + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + fn layer(p: &[u64], q: &[u64]) -> FractionLayer { + FractionLayer::new(mle(p), mle(q)).unwrap() + } + + /// Σ pᵢ/qᵢ computed directly, for comparison against the tree. + fn direct_sum(p: &[u64], q: &[u64]) -> FE { + p.iter().zip(q).fold(FE::zero(), |acc, (pi, qi)| { + acc + FE::from(*pi) * FE::from(*qi).inv().unwrap() + }) + } + + /// A LogUp-shaped input layer: `mult / (alpha - fingerprint)`, with the + /// sends and receives arranged to cancel. + fn balanced_logup_layer(num_vars: usize) -> FractionLayer { + let size = 1usize << num_vars; + let alpha = FE::from(0x9E37_79B9u64); + let mut p = Vec::with_capacity(size); + let mut q = Vec::with_capacity(size); + for i in 0..size { + // Each fingerprint appears once as a send (+1) and once as a + // receive (−1), so the whole bus balances. + let fingerprint = FE::from((i as u64 / 2) * 7 + 5); + let sign = if i % 2 == 0 { FE::one() } else { -FE::one() }; + p.push(sign); + q.push(alpha - fingerprint); + } + FractionLayer::new(Mle::new(p).unwrap(), Mle::new(q).unwrap()).unwrap() + } + + #[test] + fn folding_adds_the_two_halves() { + let l = layer(&[1, 2, 3, 4], &[5, 6, 7, 8]); + let folded = l.fold().unwrap(); + assert_eq!(folded.num_vars(), 1); + + // Variable 0 is the most significant bit, so index i pairs with i + 2. + let p = [1u64, 2, 3, 4]; + let q = [5u64, 6, 7, 8]; + for (i, (a, b)) in [(0usize, 2usize), (1, 3)].iter().enumerate() { + let expected = FE::from(p[*a]) * FE::from(q[*a]).inv().unwrap() + + FE::from(p[*b]) * FE::from(q[*b]).inv().unwrap(); + let got = folded.p.evals()[i] * folded.q.evals()[i].inv().unwrap(); + assert_eq!(expected, got, "pair ({a}, {b})"); + } + } + + #[test] + fn the_tree_output_is_the_sum_of_every_input_fraction() { + let p = [3u64, 1, 4, 1, 5, 9, 2, 6]; + let q = [2u64, 7, 1, 8, 2, 8, 1, 8]; + let tree = FractionTree::build(layer(&p, &q)).unwrap(); + + let (out_p, out_q) = tree.output(); + assert_eq!(out_p * out_q.inv().unwrap(), direct_sum(&p, &q)); + } + + #[test] + fn layer_count_is_one_per_variable_plus_the_output() { + let tree = FractionTree::build(balanced_logup_layer(4)).unwrap(); + assert_eq!(tree.num_layers(), 5); + assert_eq!(tree.layer(0).num_vars(), 0); + assert_eq!(tree.input_layer().num_vars(), 4); + } + + #[test] + fn a_balanced_bus_has_a_zero_numerator() { + let tree = FractionTree::build(balanced_logup_layer(4)).unwrap(); + let (p, q) = tree.output(); + assert_eq!(p, FE::zero()); + assert_ne!(q, FE::zero(), "denominators must not vanish"); + } + + #[test] + fn an_unbalanced_bus_does_not() { + let mut input = balanced_logup_layer(4); + // Drop one receive: the bus no longer cancels. + let mut p = input.p.evals().to_vec(); + p[3] = FE::zero(); + input = FractionLayer::new(Mle::new(p).unwrap(), input.q).unwrap(); + + let tree = FractionTree::build(input).unwrap(); + assert_ne!(tree.output().0, FE::zero()); + } + + #[test] + fn prove_and_verify_round_trip() { + let tree = FractionTree::build(balanced_logup_layer(5)).unwrap(); + let output = tree.output(); + + let proof = prove(&tree, &mut transcript()).unwrap().proof; + assert_eq!(proof.layers.len(), tree.num_layers() - 1); + + let claim = verify(&proof, output, &mut transcript()).unwrap(); + + // The residual claim must be the input layer at the final point. + let input = tree.input_layer(); + assert_eq!(claim.point.len(), input.num_vars()); + assert_eq!(input.p.evaluate(&claim.point).unwrap(), claim.p); + assert_eq!(input.q.evaluate(&claim.point).unwrap(), claim.q); + } + + #[test] + fn round_trips_on_an_unbalanced_bus_too() { + // GKR proves the sum is whatever it is; deciding that it is zero is the + // caller's check on the output numerator, not part of this protocol. + let mut p = balanced_logup_layer(4).p.evals().to_vec(); + p[3] = FE::from(9); + let input = FractionLayer::new(Mle::new(p).unwrap(), balanced_logup_layer(4).q).unwrap(); + let tree = FractionTree::build(input).unwrap(); + + let proof = prove(&tree, &mut transcript()).unwrap().proof; + let claim = verify(&proof, tree.output(), &mut transcript()).unwrap(); + assert_eq!( + tree.input_layer().p.evaluate(&claim.point).unwrap(), + claim.p + ); + } + + #[test] + fn the_prover_arrives_at_the_claim_the_verifier_does() { + // The prover has to discharge the input-layer claim against the trace, + // so it needs the same claim the verifier ends up holding. + let tree = FractionTree::build(balanced_logup_layer(3)).unwrap(); + let out = prove(&tree, &mut transcript()).unwrap(); + let claim = verify(&out.proof, tree.output(), &mut transcript()).unwrap(); + assert_eq!(out.claim, claim); + + // And it is the input layer's value at that point. + let input = tree.input_layer(); + assert_eq!(claim.p, input.p.evaluate(&claim.point).unwrap()); + assert_eq!(claim.q, input.q.evaluate(&claim.point).unwrap()); + } + + #[test] + fn a_wrong_output_claim_is_rejected() { + let tree = FractionTree::build(balanced_logup_layer(4)).unwrap(); + let (p, q) = tree.output(); + let proof = prove(&tree, &mut transcript()).unwrap().proof; + + assert!(verify(&proof, (p + FE::one(), q), &mut transcript()).is_err()); + } + + #[test] + fn a_tampered_half_value_is_rejected() { + let tree = FractionTree::build(balanced_logup_layer(4)).unwrap(); + let output = tree.output(); + let mut proof = prove(&tree, &mut transcript()).unwrap().proof; + + proof.layers[1].q_lo += FE::one(); + let err = verify(&proof, output, &mut transcript()).unwrap_err(); + assert!(matches!(err, Error::LayerRelationMismatch { layer: 1 })); + } + + #[test] + fn a_tampered_sumcheck_round_is_rejected() { + let tree = FractionTree::build(balanced_logup_layer(4)).unwrap(); + let output = tree.output(); + let mut proof = prove(&tree, &mut transcript()).unwrap().proof; + + proof.layers[2].sumcheck.rounds[0].evaluations[0] += FE::one(); + assert!(verify(&proof, output, &mut transcript()).is_err()); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let tree = FractionTree::build(balanced_logup_layer(4)).unwrap(); + let output = tree.output(); + let proof = prove(&tree, &mut transcript()).unwrap().proof; + + verify(&proof, output, &mut transcript()).unwrap(); + let mut other = DefaultTranscript::::new(b"another-statement"); + assert!(verify(&proof, output, &mut other).is_err()); + } + + #[test] + fn a_layer_carries_on_from_its_own_folded_factors() { + // What the crossover relies on: a relation rebuilt from the factors a + // few rounds left behind is the same relation. On a device those + // factors are read back off the layer; here the check is that picking + // them up mid-sumcheck changes nothing the verifier sees. + let tree = FractionTree::build(balanced_logup_layer(6)).unwrap(); + let lambda = FE::from(7); + let layer = tree.layer(4); + + let whole = { + let mut relation = LayerRelation::new(layer, &[FE::from(3)], lambda, true).unwrap(); + let vars = relation.num_vars(); + sumcheck::prove_rounds(&mut relation, vars, &mut transcript()).unwrap() + }; + + let mut relation = LayerRelation::new(layer, &[FE::from(3)], lambda, true).unwrap(); + let mut t = transcript(); + let (mut rounds, mut z) = sumcheck::prove_rounds(&mut relation, 1, &mut t).unwrap(); + let mut carried = LayerRelation::from_factors(relation.polys().to_vec(), lambda).unwrap(); + let left = carried.num_vars(); + let (tail, tail_z) = sumcheck::prove_rounds(&mut carried, left, &mut t).unwrap(); + rounds.extend(tail); + z.extend(tail_z); + + assert_eq!(whole, (rounds, z)); + } + + /// splitmix64: the cases' values, reproducible from a seed. + fn next(seed: &mut u64) -> u64 { + *seed = seed.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *seed; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn ext3(seed: &mut u64) -> FieldElement { + FieldElement::::new([ + FE::from(next(seed)), + FE::from(next(seed)), + FE::from(next(seed)), + ]) + } + + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + /// The cubic the claim is carried with is the verifier's interpolation. + #[test] + fn the_cubic_is_the_verifiers_interpolation() { + let cubic = Cubic::::new().unwrap(); + let mut seed = 3; + for _ in 0..32 { + let values: [FieldElement; 4] = std::array::from_fn(|_| ext3(&mut seed)); + for x in [ + ext3(&mut seed), + FieldElement::from(2u64), + FieldElement::zero(), + ] { + assert_eq!( + cubic.at([&values[0], &values[1], &values[2], &values[3]], &x), + sumcheck::interpolate(&values, &x) + ); + } + } + } + + #[test] + fn a_single_fraction_needs_no_layers() { + let tree = FractionTree::build(layer(&[7], &[3])).unwrap(); + assert_eq!(tree.num_layers(), 1); + + let proof = prove(&tree, &mut transcript()).unwrap().proof; + assert!(proof.layers.is_empty()); + + let claim = verify(&proof, tree.output(), &mut transcript()).unwrap(); + assert!(claim.point.is_empty()); + assert_eq!(claim.p, FE::from(7)); + assert_eq!(claim.q, FE::from(3)); + } +} diff --git a/crypto/multilinear/src/gkr_gruen.rs b/crypto/multilinear/src/gkr_gruen.rs new file mode 100644 index 000000000..576ac19cf --- /dev/null +++ b/crypto/multilinear/src/gkr_gruen.rs @@ -0,0 +1,781 @@ +//! A GKR layer's device rounds with Gruen's split (D-ARGUE S1-3, +//! `LAMBDA_VM_ARGUE_GKR_GRUEN`): the same round messages as today's, from half +//! the work. +//! +//! # What a round sends, and what the card sums +//! +//! A layer's relation is `eq(u, x)·h(x)` with `h = p_lo·q_hi + p_hi·q_lo + +//! λ·q_lo·q_hi` over `m` variables. Round `j`, after challenges `s_{j}, x')·h(s_{ bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + forced(&GRUEN_FORCED, || { + *ON.get_or_init(|| { + let on = not_off(std::env::var("LAMBDA_VM_ARGUE_GKR_GRUEN").ok().as_deref()); + eprintln!( + "★ ARGUE GKR GRUEN: {}", + if on { + format!( + "on (the default; LAMBDA_VM_ARGUE_GKR_GRUEN=0 is the old layer rounds; the host tail \ + from a cube of {})", + gkr_gruen_tail() + ) + } else { + "off (LAMBDA_VM_ARGUE_GKR_GRUEN=0)".to_string() + } + ); + on + }) + }) +} + +/// Whether every Gruen round is checked against today's +/// (`LAMBDA_VM_ARGUE_GKR_GRUEN_XCHECK`): today's program walks the same folded +/// halves beside it, with its own `eq` table folded on the same challenges, +/// and a difference fails the prove. The bound factors are compared too. +pub fn argue_gkr_gruen_xcheck() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + forced(&XCHECK_FORCED, || { + *ON.get_or_init(|| { + on_from( + std::env::var("LAMBDA_VM_ARGUE_GKR_GRUEN_XCHECK") + .ok() + .as_deref(), + ) + }) + }) +} + +/// The cube a Gruen layer hands the host (`LAMBDA_VM_ARGUE_GKR_GRUEN_TAIL`, a +/// power of two, at least 2; default [`GRUEN_TAIL_DEFAULT`]). +/// +/// Today's layers stop at `HOST_CUBE_DIRECT`, 512: a round there cost a +/// program walk, a fold launch and a round trip. A Gruen round is one launch +/// and one read-back, which undercuts a host round over more than a few dozen +/// pairs. +pub fn gkr_gruen_tail() -> usize { + match TAIL_FORCED.load(Ordering::Relaxed) { + 0 => { + static TAIL: std::sync::OnceLock = std::sync::OnceLock::new(); + *TAIL.get_or_init(|| { + tail_from( + std::env::var("LAMBDA_VM_ARGUE_GKR_GRUEN_TAIL") + .ok() + .as_deref(), + ) + }) + } + cube => cube, + } +} + +/// Where a Gruen layer's rounds stop unless the knob says otherwise. +pub const GRUEN_TAIL_DEFAULT: usize = 64; + +fn tail_from(value: Option<&str>) -> usize { + match value.map(str::parse::) { + None => GRUEN_TAIL_DEFAULT, + Some(Ok(cube)) if cube >= 2 && cube.is_power_of_two() => cube, + Some(_) => { + eprintln!( + "[gkr] LAMBDA_VM_ARGUE_GKR_GRUEN_TAIL must be a power of two ≥ 2; using {GRUEN_TAIL_DEFAULT}" + ); + GRUEN_TAIL_DEFAULT + } + } +} + +fn on_from(value: Option<&str>) -> bool { + value.is_some_and(|v| !v.is_empty() && v != "0") +} + +/// A default-on knob's reading of its variable: only `0` turns it off. +fn not_off(value: Option<&str>) -> bool { + value != Some("0") +} + +static GRUEN_FORCED: AtomicU8 = AtomicU8::new(0); +static XCHECK_FORCED: AtomicU8 = AtomicU8::new(0); +static TAIL_FORCED: AtomicUsize = AtomicUsize::new(0); + +fn forced(cell: &AtomicU8, env: impl FnOnce() -> bool) -> bool { + match cell.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => env(), + } +} + +fn store_forced(cell: &AtomicU8, on: Option) { + cell.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Overrides `LAMBDA_VM_ARGUE_GKR_GRUEN` for the whole process — for a test +/// that proves both ways in one binary. `None` restores the environment's. +#[doc(hidden)] +pub fn force_argue_gkr_gruen(on: Option) { + store_forced(&GRUEN_FORCED, on); +} + +/// The same for `LAMBDA_VM_ARGUE_GKR_GRUEN_XCHECK`. +#[doc(hidden)] +pub fn force_argue_gkr_gruen_xcheck(on: Option) { + store_forced(&XCHECK_FORCED, on); +} + +/// The same for `LAMBDA_VM_ARGUE_GKR_GRUEN_TAIL`; `None` restores it. +#[doc(hidden)] +pub fn force_gkr_gruen_tail(cube: Option) { + let cube = cube.unwrap_or(0); + assert!( + cube == 0 || (cube >= 2 && cube.is_power_of_two()), + "a tail cube is a power of two ≥ 2" + ); + TAIL_FORCED.store(cube, Ordering::Relaxed); +} + +/// On this thread, every round sums `H(0)` on the card as well, where it would +/// come from the claim — so a test can show that path sends the same messages. +/// Not for production callers. +#[doc(hidden)] +pub fn force_gkr_gruen_direct_h0(on: bool) { + DIRECT_H0.with(|cell| cell.set(on)); +} + +pub(crate) fn direct_h0_forced() -> bool { + DIRECT_H0.with(std::cell::Cell::get) +} + +thread_local! { + static DIRECT_H0: std::cell::Cell = const { std::cell::Cell::new(false) }; + + /// ⛔ A FAULT: while armed on a thread, each Gruen layer it proves adds one + /// to its first round's `s(1)` — what a round that summed one term wrong + /// would send — so the checks after it can be shown to see one. Per + /// thread: a test's fault must not reach a proof running beside it. + static GRUEN_FAULT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[doc(hidden)] +pub fn force_gkr_gruen_fault(on: bool) { + GRUEN_FAULT.with(|cell| cell.set(on)); +} + +pub(crate) fn gkr_gruen_fault() -> bool { + GRUEN_FAULT.with(std::cell::Cell::get) +} + +static GRUEN_LAYERS: AtomicU64 = AtomicU64::new(0); +static GRUEN_ROUNDS: AtomicU64 = AtomicU64::new(0); +static GRUEN_XCHECKED: AtomicU64 = AtomicU64::new(0); +static GRUEN_XCHECK_SKIPPED: AtomicU64 = AtomicU64::new(0); +static GRUEN_DIRECT_H0: AtomicU64 = AtomicU64::new(0); + +/// Device GKR layers that ran Gruen's rounds. +pub fn gruen_layers() -> u64 { + GRUEN_LAYERS.load(Ordering::Relaxed) +} + +/// Their card rounds. +pub fn gruen_rounds() -> u64 { + GRUEN_ROUNDS.load(Ordering::Relaxed) +} + +/// Of [`gruen_layers`], the ones the cross-check compared with today's rounds, +/// round by round and factor by factor, and found equal. +pub fn gruen_xchecked() -> u64 { + GRUEN_XCHECKED.load(Ordering::Relaxed) +} + +/// Layers the cross-check could not shadow: the card had no room for today's +/// `eq` table beside them. +pub fn gruen_xcheck_skipped() -> u64 { + GRUEN_XCHECK_SKIPPED.load(Ordering::Relaxed) +} + +/// Rounds that summed `H(0)` on the card (`1 − u_j = 0`, or forced). +pub fn gruen_direct_h0() -> u64 { + GRUEN_DIRECT_H0.load(Ordering::Relaxed) +} + +pub(crate) fn note_layer(rounds: u64, xchecked: bool) { + GRUEN_LAYERS.fetch_add(1, Ordering::Relaxed); + GRUEN_ROUNDS.fetch_add(rounds, Ordering::Relaxed); + if xchecked { + GRUEN_XCHECKED.fetch_add(1, Ordering::Relaxed); + } +} + +pub(crate) fn note_xcheck_skipped() { + GRUEN_XCHECK_SKIPPED.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn note_direct_h0() { + GRUEN_DIRECT_H0.fetch_add(1, Ordering::Relaxed); +} + +/// The host's half of a Gruen layer: the claim and `E` carried from round to +/// round, and each round's message made from the card's sums. +pub(crate) struct Chain { + claim: FieldElement, + /// `E = Π_{i, + cubic: Cubic, +} + +/// What a round sends, and the `s(0)` the claim implies — the fourth value the +/// next claim is interpolated from. +pub(crate) struct Message { + pub sent: Vec>, + pub s0: FieldElement, +} + +impl Chain { + /// From the claim the layer's sumcheck starts at, `p + λ·q` of the level + /// above. `None` if the cubic's denominators have no inverse (a field of + /// characteristic 2 or 3). + pub(crate) fn new(claim: FieldElement) -> Option { + Some(Self { + claim, + kappa: FieldElement::one(), + cubic: Cubic::new()?, + }) + } + + /// `E`, the scalar the rounds so far leave on the weight. + pub(crate) fn kappa(&self) -> &FieldElement { + &self.kappa + } + + /// Round `j`'s message from the card's `H(1)`, `H(2)` and, when there is + /// no `1/(1 − u_j)`, `H(0)`. `None` when neither is given. + pub(crate) fn message( + &self, + rho: &FieldElement, + inverse_left: Option<&FieldElement>, + h1: &FieldElement, + h2: &FieldElement, + h0: Option<&FieldElement>, + ) -> Option> { + let one = FieldElement::::one(); + let big1 = &self.kappa * h1; + let big2 = &self.kappa * h2; + let s1 = rho * &big1; + let s0 = &self.claim - &s1; + let big0 = match h0 { + Some(h0) => &self.kappa * h0, + None => &s0 * inverse_left?, + }; + let big3 = &big0 + (&big2 - &big1) * FieldElement::::from(3u64); + let step = rho + rho - &one; + let l2 = rho + &step; + let l3 = &l2 + &step; + Some(Message { + sent: vec![s1, &l2 * &big2, &l3 * &big3], + s0, + }) + } + + /// Past round `j`, whose challenge was `z`. + pub(crate) fn advance( + &mut self, + rho: &FieldElement, + message: &Message, + z: &FieldElement, + ) { + let one = FieldElement::::one(); + let [s1, s2, s3] = [&message.sent[0], &message.sent[1], &message.sent[2]]; + self.claim = self.cubic.at([&message.s0, s1, s2, s3], z); + let step = rho + rho - &one; + self.kappa = &self.kappa * ((&one - rho) + z * &step); + } +} + +/// `1/(1 − u_j)` for each round, `None` where there is none — those rounds sum +/// `H(0)` on the card. +pub(crate) fn inverse_lefts(rhos: &[FieldElement]) -> Vec>> { + let one = FieldElement::::one(); + let lefts: Vec> = rhos.iter().map(|r| &one - r).collect(); + let zero = FieldElement::::zero(); + let mut live: Vec> = lefts + .iter() + .map(|l| if *l == zero { one.clone() } else { l.clone() }) + .collect(); + if FieldElement::inplace_batch_inverse(&mut live).is_err() { + return vec![None; rhos.len()]; + } + lefts + .iter() + .zip(live) + .map(|(l, inv)| (*l != zero && !direct_h0_forced()).then_some(inv)) + .collect() +} + +/// The device algorithm on the host, value for value — the fused fold, the +/// split weight, the claim's `H(0)` — for checking its arithmetic against the +/// generic rounds without a card. Returns the rounds, their challenges and the +/// five factors (`eq` first) over `2^low` cells, as `gpu::DeviceTree::prove_layer` +/// hands them back. +/// +/// The upper halves `halves[1]` (`p_hi`) and `halves[3]` (`q_hi`) may stop +/// short of `2^m` cells, as an input layer kept without its padding does +/// (`crate::gkr_input_real`): every cell past the end is the padding's `0` +/// in `p` and `1` in `q`, a fold keeps only the cells below the run held +/// before it, and the run is never longer than the cube still live. +#[cfg(test)] +pub(crate) fn host_rounds( + halves: [Vec>; 4], + point: &[FieldElement], + lambda: &FieldElement, + claim: FieldElement, + low: usize, + transcript: &mut T, +) -> crate::gpu::LayerRounds +where + F: IsField + 'static, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + let m = point.len(); + let low = low.min(m); + let rounds = m - low; + // `e_lo = eq(u_{J..})`, and `e_hi` level j = `eq(u_{j+1..J})`, as the kernel lays them out. + let e_lo = crate::eq::eq_evals(&point[rounds..]); + let e_hi: Vec>> = (0..rounds) + .map(|j| crate::eq::eq_evals(&point[j + 1..rounds])) + .collect(); + let lefts = inverse_lefts(&point[..rounds]); + let mut f = halves; + let padding = [ + FieldElement::::zero(), + FieldElement::zero(), + FieldElement::one(), + FieldElement::one(), + ]; + let cell = |g: &[FieldElement], k: usize, i: usize| -> FieldElement { + g.get(i).cloned().unwrap_or_else(|| padding[k].clone()) + }; + let mut chain = Chain::new(claim).expect("a field with 2 and 3 invertible"); + let mut proofs = Vec::new(); + let mut challenges: Vec> = Vec::new(); + let h = |v: [&FieldElement; 4]| v[0] * v[3] + v[1] * v[2] + lambda * &(v[2] * v[3]); + for j in 0..rounds { + let quarter = 1usize << (m - j - 1); + if let Some(s) = challenges.last() { + for (k, g) in f.iter_mut().enumerate() { + let folded: Vec> = (0..g.len().min(2 * quarter)) + .map(|x| { + let lo = cell(g, k, x); + let hi = cell(g, k, x + 2 * quarter); + &lo + s * &(&hi - &lo) + }) + .collect(); + *g = folded; + } + } + let want_h0 = lefts[j].is_none(); + let (mut h1, mut h2, mut h0) = ( + FieldElement::zero(), + FieldElement::zero(), + FieldElement::zero(), + ); + for x in 0..quarter { + let w = &e_hi[j][x >> low] * &e_lo[x & ((1 << low) - 1)]; + let lo: [FieldElement; 4] = std::array::from_fn(|k| cell(&f[k], k, x)); + let hi: [FieldElement; 4] = std::array::from_fn(|k| cell(&f[k], k, x + quarter)); + let two: [FieldElement; 4] = std::array::from_fn(|k| &hi[k] + &hi[k] - &lo[k]); + h1 += &w * h([&hi[0], &hi[1], &hi[2], &hi[3]]); + h2 += &w * h([&two[0], &two[1], &two[2], &two[3]]); + if want_h0 { + h0 += &w * h([&lo[0], &lo[1], &lo[2], &lo[3]]); + } + } + let message = chain + .message( + &point[j], + lefts[j].as_ref(), + &h1, + &h2, + want_h0.then_some(&h0), + ) + .expect("H(0) or its inverse"); + for value in &message.sent { + transcript.append_field_element(value); + } + let z = transcript.sample_field_element(); + chain.advance(&point[j], &message, &z); + proofs.push(crate::sumcheck::RoundProof { + evaluations: message.sent, + }); + challenges.push(z); + } + let cells = 1usize << low; + let eq: Vec> = e_lo.iter().map(|v| chain.kappa() * v).collect(); + let mut factors = vec![crate::mle::Mle::new(eq).expect("a cube")]; + for (k, g) in f.iter().enumerate() { + let bound: Vec> = (0..cells) + .map(|y| match challenges.last() { + Some(s) => { + let lo = cell(g, k, y); + &lo + s * &(&cell(g, k, y + cells) - &lo) + } + None => cell(g, k, y), + }) + .collect(); + factors.push(crate::mle::Mle::new(bound).expect("a cube")); + } + (proofs, challenges, factors) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::gkr::LayerRelation; + use crypto::fiat_shamir::{default_transcript::DefaultTranscript, is_transcript::IsTranscript}; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + type FE = FieldElement; + + fn value(seed: &mut u64) -> FE { + let mut next = || { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed + }; + FE::new([next().into(), next().into(), next().into()]) + } + + /// A layer of `2^(m+1)` fractions, its point, `λ` and the claim the level + /// above makes of it — what `gkr::prove` hands a device layer. + fn layer(m: usize, seed: u64) -> ([Vec; 4], Vec, FE, FE) { + let mut seed = seed | 1; + let halves: [Vec; 4] = + std::array::from_fn(|_| (0..1 << m).map(|_| value(&mut seed)).collect()); + let point: Vec = (0..m).map(|_| value(&mut seed)).collect(); + let lambda = value(&mut seed); + let eq = crate::eq::eq_evals(&point); + let claim = (0..1usize << m).fold(FE::zero(), |acc, x| { + let [a, b, c, d] = [&halves[0][x], &halves[1][x], &halves[2][x], &halves[3][x]]; + acc + &eq[x] * (a * d + b * c + &lambda * &(c * d)) + }); + (halves, point, lambda, claim) + } + + /// Today's rounds over the same layer, as a device session runs them: the + /// generic relation, stopped at `2^low`. + fn today( + halves: &[Vec; 4], + point: &[FE], + lambda: &FE, + low: usize, + transcript: &mut DefaultTranscript, + ) -> crate::gpu::LayerRounds { + let mut polys = vec![crate::eq::eq_mle(point).unwrap()]; + polys.extend( + halves + .iter() + .map(|h| crate::mle::Mle::new(h.clone()).unwrap()), + ); + let mut relation = LayerRelation::from_factors(polys, *lambda).unwrap(); + let rounds = point.len() - low.min(point.len()); + let (proofs, challenges) = + crate::sumcheck::prove_rounds(&mut relation, rounds, transcript).unwrap(); + (proofs, challenges, relation.polys_for_tests().to_vec()) + } + + /// The rounds' values and the factors' tables, which compare. + fn values(layer: &crate::gpu::LayerRounds) -> (Vec>, Vec>) { + ( + layer.0.iter().map(|r| r.evaluations.clone()).collect(), + layer.2.iter().map(|f| f.evals().to_vec()).collect(), + ) + } + + fn assert_same(m: usize, low: usize, seed: u64) { + let (halves, point, lambda, claim) = layer(m, seed); + let mut a = DefaultTranscript::::new(b"gruen"); + let mut b = DefaultTranscript::::new(b"gruen"); + let want = today(&halves, &point, &lambda, low, &mut a); + let got = host_rounds(halves, &point, &lambda, claim, low, &mut b); + let (got_rounds, got_factors) = values(&got); + let (want_rounds, want_factors) = values(&want); + assert_eq!( + got_rounds, want_rounds, + "m {m}, low {low}: the rounds differ" + ); + assert_eq!(got.1, want.1, "m {m}, low {low}: the challenges differ"); + assert_eq!( + got_factors, want_factors, + "m {m}, low {low}: the factors handed to the tail differ" + ); + assert_eq!( + a.state(), + b.state(), + "m {m}, low {low}: the transcripts parted" + ); + } + + /// ★ The device algorithm, run on the host, sends today's rounds and hands + /// the tail today's factors, at every split of card rounds and host tail. + #[test] + fn gruen_rounds_are_todays() { + for m in 1..=9 { + for low in [0, 1, 2, 5, 6, 9] { + assert_same(m, low, 0x9E37_79B9 + (m * 31 + low) as u64); + } + } + } + + /// The same with `H(0)` summed on the card every round. + #[test] + fn gruen_rounds_with_a_direct_h0_are_todays() { + force_gkr_gruen_direct_h0(true); + let outcome = std::panic::catch_unwind(|| { + for m in [3, 7] { + assert_same(m, 2, 0xABCD + m as u64); + } + }); + force_gkr_gruen_direct_h0(false); + outcome.unwrap(); + } + + /// A point coordinate of one — `1 − u_j = 0`, no inverse — sums `H(0)` + /// directly for that round, and still sends today's rounds. + #[test] + fn a_coordinate_of_one_takes_the_direct_h0() { + let (halves, mut point, lambda, _) = layer(6, 77); + point[1] = FE::one(); + point[3] = FE::one(); + let eq = crate::eq::eq_evals(&point); + let claim = (0..1usize << 6).fold(FE::zero(), |acc, x| { + let [a, b, c, d] = [&halves[0][x], &halves[1][x], &halves[2][x], &halves[3][x]]; + acc + &eq[x] * (a * d + b * c + &lambda * &(c * d)) + }); + assert_eq!( + inverse_lefts(&point).iter().filter(|l| l.is_none()).count(), + 2, + "two rounds have no 1/(1 − u_j)" + ); + let mut a = DefaultTranscript::::new(b"gruen"); + let mut b = DefaultTranscript::::new(b"gruen"); + let want = today(&halves, &point, &lambda, 1, &mut a); + let got = host_rounds(halves, &point, &lambda, claim, 1, &mut b); + assert_eq!(values(&got), values(&want)); + } + + /// An input layer of `interactions` rounded up to a power of two, `rows` a + /// slot, the slot in the high bits: random fractions with the padding + /// written out as `pad_p / pad_q` (`0/1` for a real one), its point, `λ`, + /// the claim the level above makes of it, and how many cells of each upper + /// half are real. + fn input_layer( + interactions: usize, + rows: usize, + pad: (FE, FE), + seed: u64, + ) -> ([Vec; 4], Vec, FE, FE, usize) { + let full = interactions.next_power_of_two() * rows; + let m = full.trailing_zeros() as usize - 1; + let (mut halves, point, lambda, _) = layer(m, seed); + let half = 1usize << m; + let stored = interactions * rows - half; + halves[1][stored..].fill(pad.0); + halves[3][stored..].fill(pad.1); + let eq = crate::eq::eq_evals(&point); + let claim = (0..half).fold(FE::zero(), |acc, x| { + let [a, b, c, d] = [&halves[0][x], &halves[1][x], &halves[2][x], &halves[3][x]]; + acc + &eq[x] * (a * d + b * c + &lambda * &(c * d)) + }); + (halves, point, lambda, claim, stored) + } + + /// Today's rounds over the whole layer against the device algorithm over + /// its upper halves cut at the real fractions. + fn kept_against_today( + interactions: usize, + rows: usize, + pad: (FE, FE), + low: usize, + seed: u64, + ) -> ( + crate::gpu::LayerRounds, + crate::gpu::LayerRounds, + bool, + ) { + let (halves, point, lambda, claim, stored) = input_layer(interactions, rows, pad, seed); + let mut kept = halves.clone(); + kept[1].truncate(stored); + kept[3].truncate(stored); + let mut a = DefaultTranscript::::new(b"gruen"); + let mut b = DefaultTranscript::::new(b"gruen"); + let want = today(&halves, &point, &lambda, low, &mut a); + let got = host_rounds(kept, &point, &lambda, claim, low, &mut b); + let same_transcript = a.state() == b.state(); + (want, got, same_transcript) + } + + /// ★ An input layer kept without its padding — the upper halves stopping + /// at the real fractions, the rest read as `0/1` — sends today's rounds + /// over the whole layer, draws today's challenges and hands the tail + /// today's factors, at every split of card rounds and host tail. The + /// shapes put the padding's start anywhere from one cell into the upper + /// halves to one cell before their end, and from one fold before the slot + /// bits are spent to every round (`low = 0`). + #[test] + fn a_layer_kept_without_its_padding_sends_todays_rounds() { + let shapes = [ + (5usize, 1usize), // one real cell in each upper half + (7, 1), // one padding cell + (3, 2), + (5, 4), + (7, 8), + (20, 16), // a CPU table's slots, few rows + (9, 64), + (33, 4), + (1031, 1), // a Keccak-round table's slots + (2047, 1), + (1025, 2), + ]; + for (n, &(interactions, rows)) in shapes.iter().enumerate() { + let m = (interactions.next_power_of_two() * rows).trailing_zeros() as usize - 1; + for low in [0, 1, 2, 5, m] { + let what = format!("{interactions} interactions × {rows} rows (m {m}), low {low}"); + let padding = (FE::zero(), FE::one()); + let (want, got, same) = + kept_against_today(interactions, rows, padding, low, 0x5EED + n as u64); + assert_eq!( + values(&got), + values(&want), + "{what}: rounds or factors differ" + ); + assert_eq!(got.1, want.1, "{what}: the challenges differ"); + assert!(same, "{what}: the transcripts parted"); + } + } + } + + /// The same with `H(0)` summed on the card every round. + #[test] + fn a_kept_layer_with_a_direct_h0_sends_todays_rounds() { + force_gkr_gruen_direct_h0(true); + let outcome = std::panic::catch_unwind(|| { + for (interactions, rows) in [(5usize, 4usize), (1031, 1)] { + let (want, got, same) = + kept_against_today(interactions, rows, (FE::zero(), FE::one()), 1, 0xD1); + assert_eq!(values(&got), values(&want)); + assert!(same); + } + }); + force_gkr_gruen_direct_h0(false); + outcome.unwrap(); + } + + /// ⛔ The comparison above can fail: a layer whose padding is not `0/1` — + /// what a kept layer read with the wrong constant would stand for — sends + /// other rounds than the kept one. + #[test] + fn padding_that_is_not_zero_over_one_changes_the_rounds() { + for pad in [(FE::zero(), FE::zero()), (FE::one(), FE::one())] { + let (want, got, _) = kept_against_today(20, 16, pad, 2, 0xBAD); + assert_ne!( + values(&got).0, + values(&want).0, + "the padding's value must reach the rounds" + ); + } + } + + /// ⛔ The comparison above can fail: a claim off by one changes the rounds + /// that take `H(0)` from it. + #[test] + fn a_wrong_claim_changes_the_rounds() { + let (halves, point, lambda, claim) = layer(5, 5); + let mut a = DefaultTranscript::::new(b"gruen"); + let mut b = DefaultTranscript::::new(b"gruen"); + let want = today(&halves, &point, &lambda, 1, &mut a); + let got = host_rounds(halves, &point, &lambda, claim + FE::one(), 1, &mut b); + assert_ne!( + values(&got).0, + values(&want).0, + "the claim must reach the messages" + ); + } + + /// ★ The default: on unless `LAMBDA_VM_ARGUE_GKR_GRUEN=0` (FAST job 330). + #[test] + fn the_gruen_knob_is_on_unless_its_variable_is_zero() { + assert_eq!( + argue_gkr_gruen(), + not_off(std::env::var("LAMBDA_VM_ARGUE_GKR_GRUEN").ok().as_deref()) + ); + } + + #[test] + fn the_knobs_read_their_variables() { + assert!(!on_from(None)); + assert!(!on_from(Some("0"))); + assert!(!on_from(Some(""))); + assert!(on_from(Some("1"))); + // The main knob is on unless its variable is `0`. + assert!(not_off(None)); + assert!(not_off(Some("1"))); + assert!(!not_off(Some("0"))); + assert_eq!(tail_from(None), GRUEN_TAIL_DEFAULT); + assert_eq!(tail_from(Some("32")), 32); + assert_eq!(tail_from(Some("512")), 512); + assert_eq!(tail_from(Some("48")), GRUEN_TAIL_DEFAULT); + assert_eq!(tail_from(Some("1")), GRUEN_TAIL_DEFAULT); + assert_eq!(tail_from(Some("x")), GRUEN_TAIL_DEFAULT); + } +} diff --git a/crypto/multilinear/src/gkr_input_real.rs b/crypto/multilinear/src/gkr_input_real.rs new file mode 100644 index 000000000..efb439c80 --- /dev/null +++ b/crypto/multilinear/src/gkr_input_real.rs @@ -0,0 +1,312 @@ +//! A fraction tree's input layer kept on the card without its padding +//! (`LAMBDA_VM_ARGUE_INPUT_REAL`). +//! +//! # What the padding is, and where it sits +//! +//! A table's input layer is a fraction `p/q` for every interaction and row. +//! The interactions are rounded up to a power of two with the fraction `0/1`, +//! and the interaction slot is the high bits of a cell's index, so the padding +//! is the tail of the layer: `[interactions·rows, slots·rows)`. Fewer than +//! half the slots are padding — the count is rounded *up* — so the tail lies in +//! the upper halves `p_hi`, `q_hi` alone. On today's path it is written, folded +//! into the level above and read by the layer's own rounds like any other +//! cell: about a third of the input cells of a block. +//! +//! # What the knob changes +//! +//! On — the default; `0` writes the padding as before — a tree whose input +//! layer has padding writes only the real fractions, +//! folds the level above from them with the padding read as `0/1` (the fold a +//! tree that hands its layer back already runs), and **keeps** them. Its Gruen +//! rounds then read a cell past the stored run as `0/1` and write no folded +//! cell past it — a fold of two padding cells is the padding again — until the +//! rounds have bound the slot bits and the stored run covers the whole half +//! (`math_cuda::gkr::GruenLayer`). The constants are the limbs the padded +//! layer is written with, so every value the card reads, folds and sums is +//! raw-identical to today's: the same round sums, the same halves, the same +//! transcript and proof bytes. +//! +//! Today's layer session (`LAMBDA_VM_ARGUE_GKR_GRUEN=0`) reads every cell of +//! the halves: for it the tree writes the whole layer again, as a tree that +//! handed its layer back does. +//! +//! `check` also writes each kept input layer whole beside it, where the card +//! has room, runs today's Gruen rounds over that one in step, and compares +//! every round's sums and the halves handed to the host, limb for limb. A +//! layer that differs is counted and finished on the whole layer, so the proof +//! is today's either way. + +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; + +/// How a tree keeps its input layer on the card. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InputReal { + /// With its padding, as before. + Off, + /// Without it. The default. + On, + /// Without it, and compared round by round with the whole layer. + Check, +} + +/// The setting `LAMBDA_VM_ARGUE_INPUT_REAL` names: `0` off, unset, empty or +/// `1` on, `check` on and compared. Anything else is on, said once. +pub fn input_real_setting(value: Option<&str>) -> InputReal { + match value { + Some("0") => InputReal::Off, + None | Some("") | Some("1") => InputReal::On, + Some("check") => InputReal::Check, + Some(other) => { + eprintln!( + "[gkr] LAMBDA_VM_ARGUE_INPUT_REAL={other} is not 0, 1 or check; keeping each input layer \ + without its padding (the default)" + ); + InputReal::On + } + } +} + +static FORCED: AtomicU8 = AtomicU8::new(0); + +/// The setting in effect: a test's override, else the environment's, read +/// once with a banner. +pub fn argue_input_real() -> InputReal { + match FORCED.load(Ordering::Relaxed) { + 1 => InputReal::Off, + 2 => InputReal::On, + 3 => InputReal::Check, + _ => { + static SET: std::sync::OnceLock = std::sync::OnceLock::new(); + *SET.get_or_init(|| { + let set = input_real_setting( + std::env::var("LAMBDA_VM_ARGUE_INPUT_REAL").ok().as_deref(), + ); + eprintln!( + "★ ARGUE INPUT REAL: {}", + match set { + InputReal::Off => "off (each input layer written with its padding)", + InputReal::On => { + "on (the default: each input layer kept without its padding; \ + LAMBDA_VM_ARGUE_INPUT_REAL=0 writes the padding, =check also compares with the \ + whole layer)" + } + InputReal::Check => { + "check (each input layer kept without its padding and compared round by round \ + with the whole layer)" + } + } + ); + set + }) + } + } +} + +/// Overrides `LAMBDA_VM_ARGUE_INPUT_REAL` for the whole process — for a test +/// that proves every way in one binary. `None` restores the environment's. +#[doc(hidden)] +pub fn force_argue_input_real(setting: Option) { + FORCED.store( + match setting { + None => 0, + Some(InputReal::Off) => 1, + Some(InputReal::On) => 2, + Some(InputReal::Check) => 3, + }, + Ordering::Relaxed, + ); +} + +/// The setting in effect, for a harness's line. +pub fn input_real_in_effect() -> &'static str { + match argue_input_real() { + InputReal::Off => "off", + InputReal::On => "on", + InputReal::Check => "check", + } +} + +static PADDED_TREES: AtomicU64 = AtomicU64::new(0); +static KEPT_TREES: AtomicU64 = AtomicU64::new(0); +static CELLS_NOT_STORED: AtomicU64 = AtomicU64::new(0); +static DECLINES: AtomicU64 = AtomicU64::new(0); +static REBUILDS: AtomicU64 = AtomicU64::new(0); +static CHECKED: AtomicU64 = AtomicU64::new(0); +static DIFFERING: AtomicU64 = AtomicU64::new(0); +static CHECK_SKIPPED: AtomicU64 = AtomicU64::new(0); + +/// What the knob did this process, for a harness's line. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Counts { + /// Device trees whose input layer has padding, whatever the setting. + pub padded_trees: u64, + /// Of those, the ones kept without it. + pub kept_trees: u64, + /// The padding cells those trees did not store. + pub cells_not_stored: u64, + /// Trees the knob asked for that the card would not promise, or failed to + /// build: each took today's path instead. + pub declines: u64, + /// Kept input layers written whole again for today's layer session. + pub rebuilds: u64, + /// Under `check`: kept layers compared with the whole layer to the end. + pub checked: u64, + /// Of the layers compared, the ones that differed (finished on the whole + /// layer). + pub differing: u64, + /// Under `check`: kept layers the card had no room to compare. + pub check_skipped: u64, +} + +pub fn counts() -> Counts { + Counts { + padded_trees: PADDED_TREES.load(Ordering::Relaxed), + kept_trees: KEPT_TREES.load(Ordering::Relaxed), + cells_not_stored: CELLS_NOT_STORED.load(Ordering::Relaxed), + declines: DECLINES.load(Ordering::Relaxed), + rebuilds: REBUILDS.load(Ordering::Relaxed), + checked: CHECKED.load(Ordering::Relaxed), + differing: DIFFERING.load(Ordering::Relaxed), + check_skipped: CHECK_SKIPPED.load(Ordering::Relaxed), + } +} + +impl Counts { + /// The harness's line after its label. + pub fn line(&self) -> String { + format!( + "padded trees {} · kept without padding {} · cells not stored {} · declines {} · rewritten for the \ + old rounds {} · check compared {} · differing {} · skipped {}", + self.padded_trees, + self.kept_trees, + self.cells_not_stored, + self.declines, + self.rebuilds, + self.checked, + self.differing, + self.check_skipped + ) + } +} + +/// Zeroes every count, for a harness that reads one prove from a reused +/// process. +pub fn reset_counts() { + for counter in [ + &PADDED_TREES, + &KEPT_TREES, + &CELLS_NOT_STORED, + &DECLINES, + &REBUILDS, + &CHECKED, + &DIFFERING, + &CHECK_SKIPPED, + ] { + counter.store(0, Ordering::Relaxed); + } +} + +// The device driver's (`gpu::input_layer_tree_impl`, `gpu::GruenSession`): a +// build without a device reaches only the knob and the tests. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_padded_tree() { + PADDED_TREES.fetch_add(1, Ordering::Relaxed); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_kept(cells_not_stored: u64) { + KEPT_TREES.fetch_add(1, Ordering::Relaxed); + CELLS_NOT_STORED.fetch_add(cells_not_stored, Ordering::Relaxed); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_decline() { + DECLINES.fetch_add(1, Ordering::Relaxed); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_rebuild() { + REBUILDS.fetch_add(1, Ordering::Relaxed); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_checked(differed: bool) { + CHECKED.fetch_add(1, Ordering::Relaxed); + if differed { + DIFFERING.fetch_add(1, Ordering::Relaxed); + } +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_check_skipped() { + CHECK_SKIPPED.fetch_add(1, Ordering::Relaxed); +} + +thread_local! { + /// ⛔ A FAULT: while armed on a thread, a checked layer's first round's + /// sums from the kept layer get one added to a limb before they are + /// compared — what a kernel that read one cell wrong would hand back — so a + /// test can show the comparison sees it, counts it and finishes on the + /// whole layer. Per thread: a test's fault must not reach a proof running + /// beside it. + static CHECK_FAULT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[doc(hidden)] +pub fn force_check_fault(on: bool) { + CHECK_FAULT.with(|cell| cell.set(on)); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn check_fault() -> bool { + CHECK_FAULT.with(std::cell::Cell::get) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_knob_reads_its_variable() { + assert_eq!(input_real_setting(None), InputReal::On); + assert_eq!(input_real_setting(Some("")), InputReal::On); + assert_eq!(input_real_setting(Some("0")), InputReal::Off); + assert_eq!(input_real_setting(Some("1")), InputReal::On); + assert_eq!(input_real_setting(Some("check")), InputReal::Check); + assert_eq!(input_real_setting(Some("on")), InputReal::On); + assert_eq!(input_real_setting(Some("2")), InputReal::On); + } + + /// ★ The default keeps each input layer without its padding — the same + /// proof bytes, which the card tests show — and only `0` writes it. + #[test] + fn the_knob_is_on_unless_its_variable_is_zero() { + assert_eq!( + argue_input_real(), + input_real_setting(std::env::var("LAMBDA_VM_ARGUE_INPUT_REAL").ok().as_deref()) + ); + if std::env::var_os("LAMBDA_VM_ARGUE_INPUT_REAL").is_none() { + assert_eq!(argue_input_real(), InputReal::On); + assert_eq!(input_real_in_effect(), "on"); + } + } + + #[test] + fn the_counts_name_every_field_on_their_line() { + let counts = Counts { + padded_trees: 1, + kept_trees: 2, + cells_not_stored: 3, + declines: 4, + rebuilds: 5, + checked: 6, + differing: 7, + check_skipped: 8, + }; + assert_eq!( + counts.line(), + "padded trees 1 · kept without padding 2 · cells not stored 3 · declines 4 · rewritten for the old \ + rounds 5 · check compared 6 · differing 7 · skipped 8" + ); + } +} diff --git a/crypto/multilinear/src/gkr_lockstep.rs b/crypto/multilinear/src/gkr_lockstep.rs new file mode 100644 index 000000000..cab6f92da --- /dev/null +++ b/crypto/multilinear/src/gkr_lockstep.rs @@ -0,0 +1,1331 @@ +//! LogUp-GKR over several fraction trees at once, in **lockstep** (D-BATCH §2.2, +//! phase G): one sumcheck per step for every tree still descending. +//! +//! The trees stay separate and keep their own heights. What is shared is the +//! ladder: they are aligned at the OUTPUT, so step `i` proves layer `i + 1` of +//! every tree with more than `i` input variables, all against the same point. +//! Every challenge is shared, so the point every active tree sits at is the +//! same one, one `eq` table serves them all, and one sumcheck batches them with +//! a fresh `μ`: +//! +//! ```text +//! claim_i = Σ_a μ^{2a}·p̄_a + μ^{2a+1}·q̄_a +//! = Σ_x eq(P, x) · Σ_a [ μ^{2a}·(p_lo q_hi + p_hi q_lo) + μ^{2a+1}·q_lo q_hi ]_a(x) +//! ``` +//! +//! With one tree this is [`gkr::prove`](crate::gkr::prove) step for step, `μ` +//! in the place of its `λ`. A tree leaves the ladder at the step that reaches +//! its input layer, with its input claim at the point the ladder is at then — +//! which is what the caller settles against its trace. +//! +//! Soundness is SWIRL Thm 4.1.1 per step (every polynomial has `i` variables) +//! plus the GKR line through `lo`/`hi` at `c`: a false claim on any active tree +//! dooms the step, and the batching costs `(2|A_i| − 1)/|F|`. +//! +//! The bus outputs are NOT absorbed here: the caller absorbs them where the +//! format puts them (per bin, before its ladder). + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{ + Error, challenge_powers, + eq::{eq_eval, eq_mle}, + gkr::{FractionLayer, GkrClaim}, + mle::Mle, + poly::SumcheckPolynomial, + sumcheck::{self, SumcheckProof}, +}; + +/// One tree's four restricted values at the end of a step. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct TreeHalves { + pub p_lo: FieldElement, + pub p_hi: FieldElement, + pub q_lo: FieldElement, + pub q_hi: FieldElement, +} + +/// One step of the ladder: the shared sumcheck, and the halves of every tree +/// active in it, in the caller's tree order. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct LockstepLayer { + pub sumcheck: SumcheckProof, + pub halves: Vec>, +} + +/// The whole ladder, output end first: `max_T k_T` steps. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct LockstepProof { + pub layers: Vec>, +} + +/// Which trees step `i` proves: those with more than `i` input variables, in +/// the caller's order. Both sides call this. +pub fn active(input_vars: &[usize], step: usize) -> Vec { + (0..input_vars.len()) + .filter(|&t| input_vars[t] > step) + .collect() +} + +/// `Σ_a w_{2a}·(p_lo q_hi + p_hi q_lo) + w_{2a+1}·q_lo q_hi` over the trees' +/// values, laid out four a tree. +fn relation( + weights: &[FieldElement], + halves: &[FieldElement], +) -> FieldElement { + halves + .chunks(4) + .zip(weights.chunks(2)) + .fold(FieldElement::zero(), |acc, (v, w)| { + let numerator = &v[0] * &v[3] + &v[1] * &v[2]; + let denominator = &v[2] * &v[3]; + acc + &w[0] * numerator + &w[1] * denominator + }) +} + +/// The step's polynomial: `eq(P, ·)` and four halves per active tree. +struct LockstepRelation { + /// `[eq, (p_lo, p_hi, q_lo, q_hi) per tree]`. + polys: Vec>, + /// `μ` powers, two per tree. + weights: Vec>, +} + +impl SumcheckPolynomial for LockstepRelation { + fn num_vars(&self) -> usize { + self.polys[0].num_vars() + } + + fn degree(&self) -> usize { + 3 + } + + fn polys(&self) -> &[Mle] { + &self.polys + } + + fn combine(&self, v: &[FieldElement]) -> FieldElement { + &v[0] * relation(&self.weights, &v[1..]) + } + + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), Error> { + for p in &mut self.polys { + p.fix_first_variable_in_place(r)?; + } + Ok(()) + } +} + +/// `(1 − c)·lo + c·hi`. +fn line( + lo: &FieldElement, + hi: &FieldElement, + c: &FieldElement, +) -> FieldElement { + lo + c * &(hi - lo) +} + +/// One tree's side of one ladder step: its relation +/// `eq(P, ·)·(w_p·(p_lo q_hi + p_hi q_lo) + w_q·q_lo q_hi)` over layer `i + 1`, +/// stepped by the shared challenges. +pub trait TreeStep { + /// This round's polynomial at `1, 2, 3`. + fn round(&mut self) -> Result>, Error>; + /// Binds this round's variable to `r`. + fn bind(&mut self, r: &FieldElement) -> Result<(), Error>; + /// The four halves once every variable is bound. + fn halves(&self) -> Result, Error>; + /// Queues this round's work without waiting for it, where there is a + /// device to queue on: the ladder queues every active tree's round before + /// it reads any back, so a round costs one wait, not one a tree. + fn prefetch(&mut self) -> Result<(), Error> { + Ok(()) + } + /// The cube this step still has on a device; zero on the host. + fn device_cube(&self) -> usize { + 0 + } + /// Takes the step's factors to the host, from here on stepped there. + fn to_host(&mut self) -> Result<(), Error> { + Ok(()) + } +} + +/// A tree the ladder descends: where its layers are is its own business — on +/// the host, or on a device that steps them where they lie — and the ladder +/// cannot tell, which is what makes a device ladder the host's bytes. +pub trait LadderTree { + /// `k`: its input layer's variables. + fn input_vars(&self) -> usize; + /// The output fraction. + fn output(&self) -> (FieldElement, FieldElement); + /// Layer `layer`'s step (`layer ≥ 1`), at `point` (`layer − 1` + /// coordinates), weighted `[w_p, w_q]`, from the tree's claim `(p, q)` on + /// the layer above. + fn step( + &self, + layer: usize, + point: &[FieldElement], + weights: [FieldElement; 2], + claim: &(FieldElement, FieldElement), + ) -> Result + '_>, Error>; +} + +/// The step over a layer the host holds: one tree's [`LockstepRelation`]. +struct HostStep(LockstepRelation); + +impl HostStep +where + FieldElement: Send + Sync, +{ + fn over( + next: &FractionLayer, + point: &[FieldElement], + weights: [FieldElement; 2], + ) -> Result { + let half = next.p.len() / 2; + let mut polys = Vec::with_capacity(5); + polys.push(eq_mle(point)?); + for m in [&next.p, &next.q] { + polys.push(Mle::new(m.evals()[..half].to_vec())?); + polys.push(Mle::new(m.evals()[half..].to_vec())?); + } + Ok(Self(LockstepRelation { + polys, + weights: weights.to_vec(), + })) + } +} + +impl TreeStep for HostStep +where + FieldElement: Send + Sync, +{ + fn round(&mut self) -> Result>, Error> { + Ok(sumcheck::round_evaluations(&self.0, 3, false)) + } + + fn bind(&mut self, r: &FieldElement) -> Result<(), Error> { + self.0.fix_first_variable(r) + } + + fn halves(&self) -> Result, Error> { + let bound = |slot: usize| -> Result, Error> { + self.0.polys[slot] + .as_constant() + .cloned() + .ok_or(Error::NoVariablesLeft) + }; + Ok(TreeHalves { + p_lo: bound(1)?, + p_hi: bound(2)?, + q_lo: bound(3)?, + q_hi: bound(4)?, + }) + } +} + +/// A tree's layers held here, output first. +pub struct HostLayers<'a, F: IsField>(pub &'a [FractionLayer]); + +impl LadderTree for HostLayers<'_, F> +where + FieldElement: Send + Sync, +{ + fn input_vars(&self) -> usize { + self.0.len().saturating_sub(1) + } + + fn output(&self) -> (FieldElement, FieldElement) { + ( + self.0[0].p.evals()[0].clone(), + self.0[0].q.evals()[0].clone(), + ) + } + + fn step( + &self, + layer: usize, + point: &[FieldElement], + weights: [FieldElement; 2], + _claim: &(FieldElement, FieldElement), + ) -> Result + '_>, Error> { + let next = self.0.get(layer).ok_or(Error::NoVariablesLeft)?; + Ok(Box::new(HostStep::over(next, point, weights)?)) + } +} + +/// The step over a layer a device holds: its rounds on the card while the cube +/// is worth a launch, then the host's over the factors it hands back — today's +/// crossover ([`crate::HOST_CUBE_DIRECT`]). +struct DeviceStep { + session: Option, + host: Option>, + weights: [FieldElement; 2], +} + +impl DeviceStep +where + FieldElement: Send + Sync, +{ + /// The card's factors, folded so far, brought here: the step goes on on + /// the host's relation over them. + fn hand_over(&mut self) -> Result<(), Error> { + if let Some(session) = self.session.take() { + let factors = session.factors::()?; + if factors.len() != 5 { + return Err(Error::DeviceFailed { + stage: "layer factors", + }); + } + self.host = Some(HostStep(LockstepRelation { + polys: factors, + weights: self.weights.to_vec(), + })); + } + Ok(()) + } + + fn current(&mut self) -> Result<&mut dyn TreeStep, Error> { + match (self.session.as_mut(), self.host.as_mut()) { + (Some(session), _) => Ok(session as &mut dyn TreeStep), + (None, Some(host)) => Ok(host as &mut dyn TreeStep), + (None, None) => Err(Error::DeviceFailed { + stage: "layer session", + }), + } + } +} + +impl TreeStep for crate::gpu::LayerSession { + fn round(&mut self) -> Result>, Error> { + crate::gpu::LayerSession::round::(self) + } + + fn bind(&mut self, r: &FieldElement) -> Result<(), Error> { + self.fold(r) + } + + fn halves(&self) -> Result, Error> { + let factors = self.factors::()?; + let one = |slot: usize| -> Result, Error> { + factors + .get(slot) + .and_then(|f| f.as_constant().cloned()) + .ok_or(Error::NoVariablesLeft) + }; + Ok(TreeHalves { + p_lo: one(1)?, + p_hi: one(2)?, + q_lo: one(3)?, + q_hi: one(4)?, + }) + } +} + +impl TreeStep for DeviceStep +where + FieldElement: Send + Sync, +{ + fn round(&mut self) -> Result>, Error> { + self.current()?.round() + } + + fn bind(&mut self, r: &FieldElement) -> Result<(), Error> { + self.current()?.bind(r) + } + + fn halves(&self) -> Result, Error> { + match (&self.session, &self.host) { + (Some(session), _) => TreeStep::::halves(session), + (None, Some(host)) => host.halves(), + (None, None) => Err(Error::NoVariablesLeft), + } + } + + fn prefetch(&mut self) -> Result<(), Error> { + match self.session.as_mut() { + Some(session) => session.enqueue(), + None => Ok(()), + } + } + + fn device_cube(&self) -> usize { + self.session + .as_ref() + .map_or(0, |session| 1usize << session.num_vars()) + } + + fn to_host(&mut self) -> Result<(), Error> { + self.hand_over() + } +} + +/// What a [`GruenStep`] needs of the card: Gruen's rounds over one layer's +/// halves ([`crate::gpu::GruenSession`]). +trait GruenCard { + /// The rounds it runs, `J`. + fn rounds(&self) -> usize; + /// Queues the next round, folding by `previous`, without waiting for it. + fn enqueue(&mut self, previous: Option<&FieldElement>, want_h0: bool) -> Result<(), Error>; + /// The round's `H(1)`, `H(2)` — and `H(0)` with `want_h0`. + fn sums( + &mut self, + previous: Option<&FieldElement>, + want_h0: bool, + ) -> Result>, Error>; + /// After the last round: `p_lo, p_hi, q_lo, q_hi` bound to it. + fn finish(&mut self, previous: Option<&FieldElement>) -> Result>, Error>; +} + +impl GruenCard for crate::gpu::GruenSession { + fn rounds(&self) -> usize { + crate::gpu::GruenSession::rounds(self) + } + + fn enqueue(&mut self, previous: Option<&FieldElement>, want_h0: bool) -> Result<(), Error> { + crate::gpu::GruenSession::enqueue(self, previous, want_h0) + } + + fn sums( + &mut self, + previous: Option<&FieldElement>, + want_h0: bool, + ) -> Result>, Error> { + crate::gpu::GruenSession::sums(self, previous, want_h0) + } + + fn finish(&mut self, previous: Option<&FieldElement>) -> Result>, Error> { + crate::gpu::GruenSession::finish(self, previous) + } +} + +/// The step over a device layer on Gruen's rounds ([`crate::gkr_gruen`], on +/// with `LAMBDA_VM_ARGUE_GKR_GRUEN` as today's per-table layers are): the +/// tree's relation is `w_p·eq·(p_lo q_hi + p_hi q_lo + λ'·q_lo q_hi)` with +/// `λ' = w_q/w_p`, so the card sums its `H(1)`, `H(2)` under `λ'`, the tree's +/// own chain (from its claim `p + λ'·q`) makes the message a per-table layer +/// would send, and the step sends it times `w_p` — the field elements +/// [`HostStep`] sums. At the card's tail it hands the host today's five +/// factors, `eq` included, and steps on there. +struct GruenStep { + session: Option, + chain: crate::gkr_gruen::Chain, + /// The layer's point `u`, one coordinate a round. + point: Vec>, + /// `1/(1 − u_j)` for each card round; `None` sums `H(0)` on the card. + lefts: Vec>>, + /// The card rounds run so far. + round: usize, + /// This round's message, unscaled, until its challenge binds it. + message: Option>, + /// The last round's challenge, which the next card pass folds by. + previous: Option>, + host: Option>, + weights: [FieldElement; 2], +} + +/// `λ' = w_q/w_p`, or `None` when `w_p` has no inverse. +fn gruen_lambda(weights: &[FieldElement; 2]) -> Option> { + weights[0].inv().ok().map(|inverse| &weights[1] * &inverse) +} + +impl GruenStep +where + FieldElement: Send + Sync, +{ + /// `None` when the step cannot run this way — `w_p` with no inverse, or a + /// card that declines the session — and before anything moves, so the + /// caller opens today's session instead. + fn open( + device: &crate::gpu::DeviceTree, + layer: usize, + point: &[FieldElement], + weights: &[FieldElement; 2], + claim: &(FieldElement, FieldElement), + ) -> Result, Error> { + let Some(lambda) = gruen_lambda(weights) else { + return Ok(None); + }; + let tail = crate::gkr_gruen::gkr_gruen_tail(); + let Some(session) = device.gruen_session(layer, point, &lambda, tail) else { + return Ok(None); + }; + Self::over(session, point, weights, claim) + } +} + +impl> GruenStep +where + FieldElement: Send + Sync, +{ + /// The step over `session`, opened under `λ' = w_q/w_p` at `point`. + fn over( + session: C, + point: &[FieldElement], + weights: &[FieldElement; 2], + claim: &(FieldElement, FieldElement), + ) -> Result, Error> { + let Some(lambda) = gruen_lambda(weights) else { + return Ok(None); + }; + let Some(chain) = crate::gkr_gruen::Chain::new(&claim.0 + &lambda * &claim.1) else { + return Ok(None); + }; + let lefts = crate::gkr_gruen::inverse_lefts(&point[..session.rounds()]); + let mut step = Self { + session: Some(session), + chain, + point: point.to_vec(), + lefts, + round: 0, + message: None, + previous: None, + host: None, + weights: weights.clone(), + }; + if step.lefts.is_empty() { + step.hand_over()?; + } + Ok(Some(step)) + } + + /// After the card's last round: its halves, and `κ·eq(u_{J..})` for the + /// weight, as the host's relation. + fn hand_over(&mut self) -> Result<(), Error> { + let Some(mut session) = self.session.take() else { + return Ok(()); + }; + let halves = session.finish(self.previous.as_ref())?; + let rounds = self.lefts.len(); + let eq: Vec> = crate::eq::eq_evals(&self.point[rounds..]) + .iter() + .map(|v| self.chain.kappa() * v) + .collect(); + let mut polys = Vec::with_capacity(5); + polys.push(Mle::new(eq)?); + polys.extend(halves); + if polys.len() != 5 { + return Err(Error::DeviceFailed { + stage: "layer factors", + }); + } + crate::gkr_gruen::note_layer(rounds as u64, false); + self.host = Some(HostStep(LockstepRelation { + polys, + weights: self.weights.to_vec(), + })); + Ok(()) + } +} + +impl> TreeStep for GruenStep +where + FieldElement: Send + Sync, +{ + fn round(&mut self) -> Result>, Error> { + let Some(session) = self.session.as_mut() else { + return self.host.as_mut().ok_or(Error::NoVariablesLeft)?.round(); + }; + let left = self.lefts[self.round].as_ref(); + let sums = session.sums(self.previous.as_ref(), left.is_none())?; + if left.is_none() { + crate::gkr_gruen::note_direct_h0(); + } + let message = self + .chain + .message( + &self.point[self.round], + left, + &sums[0], + &sums[1], + sums.get(2).filter(|_| left.is_none()), + ) + .ok_or(Error::DeviceFailed { + stage: "gruen message", + })?; + let sent = message.sent.iter().map(|v| v * &self.weights[0]).collect(); + self.message = Some(message); + Ok(sent) + } + + fn bind(&mut self, r: &FieldElement) -> Result<(), Error> { + if self.session.is_none() { + return self.host.as_mut().ok_or(Error::NoVariablesLeft)?.bind(r); + } + let message = self.message.take().ok_or(Error::DeviceFailed { + stage: "gruen message", + })?; + self.chain.advance(&self.point[self.round], &message, r); + self.previous = Some(r.clone()); + self.round += 1; + if self.round == self.lefts.len() { + self.hand_over()?; + } + Ok(()) + } + + fn halves(&self) -> Result, Error> { + self.host.as_ref().ok_or(Error::NoVariablesLeft)?.halves() + } + + fn prefetch(&mut self) -> Result<(), Error> { + let want_h0 = self.lefts.get(self.round).is_some_and(Option::is_none); + match self.session.as_mut() { + Some(session) => session.enqueue(self.previous.as_ref(), want_h0), + None => Ok(()), + } + } + + // `device_cube` stays zero and `to_host` does nothing: a Gruen step leaves + // the card at its own tail (`LAMBDA_VM_ARGUE_GKR_GRUEN_TAIL`), the cube + // today's per-table layers leave it at. +} + +/// `eq·(w_p·(p_lo q_hi + p_hi q_lo) + w_q·q_lo q_hi)` as the device's program: +/// the batched step's relation for one tree. +fn tree_program( + weights: &[FieldElement; 2], +) -> Result, Error> { + use crate::program::Builder; + let mut b = Builder::::new(); + let eq = b.var(0); + let p_lo = b.var(1); + let p_hi = b.var(2); + let q_lo = b.var(3); + let q_hi = b.var(4); + let first = b.mul(p_lo, q_hi); + let second = b.mul(p_hi, q_lo); + let numerator = b.add(first, second); + let denominator = b.mul(q_lo, q_hi); + let wp = b.fixed(weights[0].clone()); + let wq = b.fixed(weights[1].clone()); + let a = b.mul(wp, numerator); + let c = b.mul(wq, denominator); + let sum = b.add(a, c); + let root = b.mul(eq, sum); + b.finish(root) +} + +/// A table's tree as the argue builds it: its levels here, or on a device. +impl LadderTree for crate::gkr::FractionTree +where + FieldElement: Send + Sync, +{ + fn input_vars(&self) -> usize { + self.num_layers().saturating_sub(1) + } + + fn output(&self) -> (FieldElement, FieldElement) { + crate::gkr::FractionTree::output(self) + } + + fn step( + &self, + layer: usize, + point: &[FieldElement], + weights: [FieldElement; 2], + claim: &(FieldElement, FieldElement), + ) -> Result + '_>, Error> { + let Some(device) = self.device() else { + return Ok(Box::new(HostStep::over(self.layer(layer), point, weights)?)); + }; + // A level near the output comes here whole, as today's prefix does: + // its rounds are over cubes a core walks in microseconds. Never the + // input layer, which a tree that gave it back writes again. + let prefix = crate::HOST_CUBE_DIRECT.trailing_zeros() as usize + 1; + if layer <= prefix && layer + 1 < self.num_layers() { + let (p, q) = device + .layer_to_host::(layer) + .ok_or(Error::DeviceFailed { + stage: "layer to host", + })?; + let next = FractionLayer::new(p, q)?; + return Ok(Box::new(HostStep::over(&next, point, weights)?)); + } + if crate::gkr_gruen::argue_gkr_gruen() + && let Some(step) = GruenStep::open(device, layer, point, &weights, claim)? + { + return Ok(Box::new(step)); + } + let program = tree_program(&weights)?; + let session = + device + .layer_session(layer, point, &program, 3) + .ok_or(Error::DeviceFailed { + stage: "layer session", + })?; + Ok(Box::new(DeviceStep { + session: Some(session), + host: None, + weights, + })) + } +} + +/// Proves every tree's layers in lockstep, on the host. +/// +/// `trees[t]` is tree `t`'s layers, output first (`trees[t][0]` has no +/// variables, the last is the input layer). Returns the ladder and each tree's +/// input-layer claim, in tree order. A tree with no variables at all is its +/// own input claim at the empty point and takes no step. +pub fn prove( + trees: &[&[FractionLayer]], + transcript: &mut T, +) -> Result<(LockstepProof, Vec>), Error> +where + F: IsField + 'static, + T: IsTranscript, + FieldElement: Send + Sync, +{ + for tree in trees { + for (i, layer) in tree.iter().enumerate() { + if layer.num_vars() != i { + return Err(Error::VariableCountMismatch { + expected: i, + got: layer.num_vars(), + }); + } + } + if tree.is_empty() { + return Err(Error::EmptyPolynomial); + } + } + let hosts: Vec> = trees.iter().map(|t| HostLayers(t)).collect(); + let trees: Vec<&dyn LadderTree> = hosts.iter().map(|t| t as &dyn LadderTree).collect(); + prove_trees(&trees, transcript) +} + +/// [`prove`] over trees wherever their layers are. +pub fn prove_trees( + trees: &[&dyn LadderTree], + transcript: &mut T, +) -> Result<(LockstepProof, Vec>), Error> +where + F: IsField + 'static, + T: IsTranscript, +{ + let input_vars: Vec = trees.iter().map(|t| t.input_vars()).collect(); + let steps = input_vars.iter().copied().max().unwrap_or(0); + + let mut claims: Vec<(FieldElement, FieldElement)> = + trees.iter().map(|tree| tree.output()).collect(); + let mut inputs: Vec>> = (0..trees.len()) + .map(|t| { + (input_vars[t] == 0).then(|| GkrClaim { + point: Vec::new(), + p: claims[t].0.clone(), + q: claims[t].1.clone(), + }) + }) + .collect(); + let mut point: Vec> = Vec::new(); + let mut layers = Vec::with_capacity(steps); + + for i in 0..steps { + let active = active(&input_vars, i); + let mu: FieldElement = transcript.sample_field_element(); + let weights = challenge_powers(&mu, 2 * active.len()); + let mut stepping: Vec + '_>> = active + .iter() + .zip(weights.chunks(2)) + .map(|(&t, w)| trees[t].step(i + 1, &point, [w[0].clone(), w[1].clone()], &claims[t])) + .collect::>()?; + + let mut rounds = Vec::with_capacity(i); + let mut z = Vec::with_capacity(i); + for _ in 0..i { + // The crossover on the step's SUMMED cube: while the active + // trees' device cubes together are worth a launch, they stay on + // the card; below it every one comes here — a batched host round + // then walks all of them at the cost of one. + let cube: usize = stepping.iter().map(|step| step.device_cube()).sum(); + if cube > 0 && cube <= crate::HOST_CUBE_DIRECT { + for step in stepping.iter_mut() { + step.to_host()?; + } + } + for step in stepping.iter_mut() { + step.prefetch()?; + } + let mut sent = vec![FieldElement::::zero(); 3]; + for step in stepping.iter_mut() { + let own = step.round()?; + if own.len() != 3 { + return Err(Error::RoundDegreeMismatch { + round: rounds.len(), + expected: 3, + got: own.len(), + }); + } + for (s, v) in sent.iter_mut().zip(own) { + *s += v; + } + } + for e in &sent { + transcript.append_field_element(e); + } + let r: FieldElement = transcript.sample_field_element(); + for step in stepping.iter_mut() { + step.bind(&r)?; + } + rounds.push(sumcheck::RoundProof { evaluations: sent }); + z.push(r); + } + + let mut halves = Vec::with_capacity(active.len()); + for step in &stepping { + let h = step.halves()?; + for v in [&h.p_lo, &h.p_hi, &h.q_lo, &h.q_hi] { + transcript.append_field_element(v); + } + halves.push(h); + } + drop(stepping); + let c: FieldElement = transcript.sample_field_element(); + point = std::iter::once(c.clone()).chain(z).collect(); + for (&t, h) in active.iter().zip(&halves) { + claims[t] = (line(&h.p_lo, &h.p_hi, &c), line(&h.q_lo, &h.q_hi, &c)); + if input_vars[t] == i + 1 { + inputs[t] = Some(GkrClaim { + point: point.clone(), + p: claims[t].0.clone(), + q: claims[t].1.clone(), + }); + } + } + layers.push(LockstepLayer { + sumcheck: SumcheckProof { rounds }, + halves, + }); + } + + let inputs = inputs + .into_iter() + .collect::>>() + .ok_or(Error::NoVariablesLeft)?; + Ok((LockstepProof { layers }, inputs)) +} + +/// Verifies a ladder against the trees' claimed outputs and their input +/// heights (`input_vars[t]` = `k_t`, from the statement, never the proof). +/// Returns each tree's input-layer claim, in tree order. +pub fn verify( + proof: &LockstepProof, + outputs: &[(FieldElement, FieldElement)], + input_vars: &[usize], + transcript: &mut T, +) -> Result>, Error> +where + F: IsField + 'static, + T: IsTranscript, +{ + verify_with(proof, outputs, input_vars, transcript, true) +} + +/// [`verify`], with the step residual's check switchable — `false` only in +/// the mutation test that shows the check is load-bearing. +#[doc(hidden)] +pub fn verify_with( + proof: &LockstepProof, + outputs: &[(FieldElement, FieldElement)], + input_vars: &[usize], + transcript: &mut T, + check_residual: bool, +) -> Result>, Error> +where + F: IsField + 'static, + T: IsTranscript, +{ + if outputs.len() != input_vars.len() { + return Err(Error::ArgueShapeMismatch { + part: "tree outputs", + }); + } + let steps = input_vars.iter().copied().max().unwrap_or(0); + if proof.layers.len() != steps { + return Err(Error::ArgueShapeMismatch { part: "ladder" }); + } + + let mut claims: Vec<(FieldElement, FieldElement)> = outputs.to_vec(); + let mut inputs: Vec>> = (0..outputs.len()) + .map(|t| { + (input_vars[t] == 0).then(|| GkrClaim { + point: Vec::new(), + p: claims[t].0.clone(), + q: claims[t].1.clone(), + }) + }) + .collect(); + let mut point: Vec> = Vec::new(); + + for (i, layer) in proof.layers.iter().enumerate() { + let active = active(input_vars, i); + if layer.halves.len() != active.len() { + return Err(Error::ArgueShapeMismatch { + part: "ladder step", + }); + } + let mu: FieldElement = transcript.sample_field_element(); + let weights = challenge_powers(&mu, 2 * active.len()); + let claimed = active + .iter() + .zip(weights.chunks(2)) + .fold(FieldElement::::zero(), |acc, (&t, w)| { + acc + &w[0] * &claims[t].0 + &w[1] * &claims[t].1 + }); + + let claim = sumcheck::verify(&layer.sumcheck, claimed, point.len(), 3, transcript)?; + + let values: Vec> = layer + .halves + .iter() + .flat_map(|h| { + [ + h.p_lo.clone(), + h.p_hi.clone(), + h.q_lo.clone(), + h.q_hi.clone(), + ] + }) + .collect(); + let expected = eq_eval(&point, &claim.point)? * relation(&weights, &values); + if check_residual && expected != claim.expected_evaluation { + return Err(Error::LayerRelationMismatch { layer: i }); + } + + for v in &values { + transcript.append_field_element(v); + } + let c: FieldElement = transcript.sample_field_element(); + point = std::iter::once(c.clone()).chain(claim.point).collect(); + for (&t, h) in active.iter().zip(&layer.halves) { + claims[t] = (line(&h.p_lo, &h.p_hi, &c), line(&h.q_lo, &h.q_hi, &c)); + if input_vars[t] == i + 1 { + inputs[t] = Some(GkrClaim { + point: point.clone(), + p: claims[t].0.clone(), + q: claims[t].1.clone(), + }); + } + } + } + + inputs + .into_iter() + .collect::>>() + .ok_or(Error::ArgueShapeMismatch { part: "ladder" }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::gkr::{self, FractionTree}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"gkr-lockstep-test") + } + + /// A random fraction layer of `2^num_vars` cells, reproducible from `seed`. + fn random_layer(num_vars: usize, seed: u64) -> FractionLayer { + let mut s = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut next = || { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + FE::from(s) + }; + let size = 1usize << num_vars; + let p: Vec = (0..size).map(|_| next()).collect(); + let q: Vec = (0..size).map(|_| next()).collect(); + FractionLayer::new(Mle::new(p).unwrap(), Mle::new(q).unwrap()).unwrap() + } + + /// Every layer, output first, by host folding. + fn layers(input: FractionLayer) -> Vec> { + let mut layers = vec![input]; + while layers.last().unwrap().num_vars() > 0 { + let next = layers.last().unwrap().fold().unwrap(); + layers.push(next); + } + layers.reverse(); + layers + } + + fn round_trip(heights: &[usize]) -> (Vec>>, LockstepProof) { + let trees: Vec>> = heights + .iter() + .enumerate() + .map(|(t, &k)| layers(random_layer(k, 17 + t as u64))) + .collect(); + let refs: Vec<&[FractionLayer]> = trees.iter().map(|t| t.as_slice()).collect(); + let mut prover = transcript(); + let (proof, claims) = prove(&refs, &mut prover).unwrap(); + + let outputs: Vec<(FE, FE)> = trees + .iter() + .map(|t| (t[0].p.evals()[0], t[0].q.evals()[0])) + .collect(); + let mut verifier = transcript(); + let got = verify(&proof, &outputs, heights, &mut verifier).unwrap(); + assert_eq!( + got, claims, + "the prover and the verifier reach the same claims" + ); + assert_eq!(prover.state(), verifier.state(), "and the same transcript"); + // Each claim is its tree's input layer at the claim point. + for (tree, claim) in trees.iter().zip(&claims) { + let input = tree.last().unwrap(); + assert_eq!(claim.point.len(), input.num_vars()); + assert_eq!(input.p.evaluate(&claim.point).unwrap(), claim.p); + assert_eq!(input.q.evaluate(&claim.point).unwrap(), claim.q); + } + (trees, proof) + } + + /// Trees of different heights, one of them without variables, share one + /// ladder; each leaves it with its own input claim. + #[test] + fn trees_of_different_heights_round_trip_in_lockstep() { + round_trip(&[5, 2, 0, 5, 3]); + round_trip(&[1]); + round_trip(&[0]); + round_trip(&[4, 4]); + } + + /// With one tree the ladder is today's GKR, message for message: `μ` is + /// drawn where `λ` was and plays its role. + #[test] + fn one_tree_is_todays_gkr() { + for k in 0..6 { + let input = random_layer(k, 99 + k as u64); + let tree = FractionTree::build(input.clone()).unwrap(); + let today = gkr::prove(&tree, &mut transcript()).unwrap(); + let layers = layers(input); + let (proof, claims) = prove(&[layers.as_slice()], &mut transcript()).unwrap(); + assert_eq!(claims, vec![today.claim.clone()], "k = {k}"); + assert_eq!(proof.layers.len(), today.proof.layers.len()); + for (ours, theirs) in proof.layers.iter().zip(&today.proof.layers) { + assert_eq!(ours.sumcheck, theirs.sumcheck, "k = {k}"); + let h = &ours.halves[0]; + assert_eq!( + [&h.p_lo, &h.p_hi, &h.q_lo, &h.q_hi], + [&theirs.p_lo, &theirs.p_hi, &theirs.q_lo, &theirs.q_hi] + ); + } + } + } + + /// One tree's halves wrong at a mid step of a multi-tree ladder: the step's + /// residual refuses it, and names the step. + #[test] + fn one_trees_halves_wrong_at_a_mid_step_are_refused() { + let heights = [4, 3, 4]; + let (trees, proof) = round_trip(&heights); + let outputs: Vec<(FE, FE)> = trees + .iter() + .map(|t| (t[0].p.evals()[0], t[0].q.evals()[0])) + .collect(); + for tree in 0..3 { + let mut bad = proof.clone(); + bad.layers[2].halves[tree].q_lo += FE::one(); + assert_eq!( + verify(&bad, &outputs, &heights, &mut transcript()).unwrap_err(), + Error::LayerRelationMismatch { layer: 2 }, + "tree {tree}" + ); + } + // A round value of the shared sumcheck, likewise. + let mut bad = proof.clone(); + bad.layers[3].sumcheck.rounds[1].evaluations[2] += FE::one(); + assert_eq!( + verify(&bad, &outputs, &heights, &mut transcript()).unwrap_err(), + Error::LayerRelationMismatch { layer: 3 } + ); + // A wrong output of one tree. + let mut wrong = outputs.clone(); + wrong[1].0 += FE::one(); + assert!(verify(&proof, &wrong, &heights, &mut transcript()).is_err()); + } + + /// The residual is load-bearing: made inert, the mid-step tamper above is + /// no longer refused at its step. + #[test] + fn the_residual_check_made_inert_lets_a_bad_step_through() { + let heights = [4, 3, 4]; + let (trees, proof) = round_trip(&heights); + let outputs: Vec<(FE, FE)> = trees + .iter() + .map(|t| (t[0].p.evals()[0], t[0].q.evals()[0])) + .collect(); + let mut bad = proof.clone(); + bad.layers[2].halves[1].q_lo += FE::one(); + assert!(verify_with(&bad, &outputs, &heights, &mut transcript(), false).is_ok()); + } + + /// The shape is the statement's: a ladder a step short, a step with a tree + /// missing, or heights the proof was not made for are refused. + #[test] + fn a_ladder_of_the_wrong_shape_is_refused() { + let heights = [4, 2]; + let (trees, proof) = round_trip(&heights); + let outputs: Vec<(FE, FE)> = trees + .iter() + .map(|t| (t[0].p.evals()[0], t[0].q.evals()[0])) + .collect(); + let mut short = proof.clone(); + short.layers.pop(); + assert!(verify(&short, &outputs, &heights, &mut transcript()).is_err()); + let mut missing = proof.clone(); + missing.layers[1].halves.pop(); + assert!(verify(&missing, &outputs, &heights, &mut transcript()).is_err()); + assert!(verify(&proof, &outputs, &[4, 3], &mut transcript()).is_err()); + assert!(verify(&proof, &outputs[..1], &heights, &mut transcript()).is_err()); + } + + /// Over the extension too, which is where the argue runs. + #[test] + fn the_ladder_runs_over_the_extension() { + let lift = |layer: FractionLayer| { + let up = |m: &Mle| { + Mle::new(m.evals().iter().map(|v| v.to_extension::()).collect()).unwrap() + }; + FractionLayer::new(up(&layer.p), up(&layer.q)).unwrap() + }; + let trees: Vec>> = [3usize, 1, 3] + .iter() + .enumerate() + .map(|(t, &k)| { + let mut layers = vec![lift(random_layer(k, t as u64))]; + while layers.last().unwrap().num_vars() > 0 { + let next = layers.last().unwrap().fold().unwrap(); + layers.push(next); + } + layers.reverse(); + layers + }) + .collect(); + let refs: Vec<&[FractionLayer]> = trees.iter().map(|t| t.as_slice()).collect(); + let mut prover = DefaultTranscript::::new(b"ext"); + let (proof, claims) = prove(&refs, &mut prover).unwrap(); + let outputs: Vec<_> = trees + .iter() + .map(|t| (t[0].p.evals()[0], t[0].q.evals()[0])) + .collect(); + let got = verify( + &proof, + &outputs, + &[3, 1, 3], + &mut DefaultTranscript::::new(b"ext"), + ) + .unwrap(); + for (a, b) in got.iter().zip(&claims) { + assert!(a.point == b.point && a.p == b.p && a.q == b.q); + } + assert_eq!(got.len(), claims.len()); + } + + /// The card's Gruen rounds on the host, value for value — the arithmetic + /// of `gkr_gruen::host_rounds`: the halves folded by the previous + /// challenge on the way in, `H` summed under the split weight. + struct HostGruen { + /// `p_lo, p_hi, q_lo, q_hi`, `2^m` cells each. + f: [Vec; 4], + m: usize, + low: usize, + e_lo: Vec, + e_hi: Vec>, + lambda: FE, + done: usize, + pending: Option>, + } + + impl HostGruen { + fn new(next: &FractionLayer, point: &[FE], lambda: FE, low: usize) -> Self { + let half = next.p.len() / 2; + let (p, q) = (next.p.evals(), next.q.evals()); + let m = point.len(); + let low = low.min(m); + let rounds = m - low; + Self { + f: [ + p[..half].to_vec(), + p[half..].to_vec(), + q[..half].to_vec(), + q[half..].to_vec(), + ], + m, + low, + e_lo: crate::eq::eq_evals(&point[rounds..]), + e_hi: (0..rounds) + .map(|j| crate::eq::eq_evals(&point[j + 1..rounds])) + .collect(), + lambda, + done: 0, + pending: None, + } + } + + /// Binds the last round's variable: `2·width` live cells to `width`. + fn fold(&mut self, previous: Option<&FE>, width: usize) { + if let Some(s) = previous { + for g in self.f.iter_mut() { + for x in 0..width { + g[x] = g[x] + s * &(g[x + width] - g[x]); + } + } + } + } + } + + impl GruenCard for HostGruen { + fn rounds(&self) -> usize { + self.m - self.low + } + + fn enqueue(&mut self, previous: Option<&FE>, want_h0: bool) -> Result<(), Error> { + if self.pending.is_some() { + return Ok(()); + } + let j = self.done; + let quarter = 1usize << (self.m - j - 1); + self.fold(previous, 2 * quarter); + let h = |v: [&FE; 4]| v[0] * v[3] + v[1] * v[2] + self.lambda * (v[2] * v[3]); + let mut sums = vec![FE::zero(); if want_h0 { 3 } else { 2 }]; + for x in 0..quarter { + let w = self.e_hi[j][x >> self.low] * self.e_lo[x & ((1 << self.low) - 1)]; + let lo: [&FE; 4] = std::array::from_fn(|k| &self.f[k][x]); + let hi: [&FE; 4] = std::array::from_fn(|k| &self.f[k][x + quarter]); + let two: [FE; 4] = std::array::from_fn(|k| hi[k] + hi[k] - lo[k]); + sums[0] += w * h(hi); + sums[1] += w * h([&two[0], &two[1], &two[2], &two[3]]); + if want_h0 { + sums[2] += w * h(lo); + } + } + self.pending = Some(sums); + self.done += 1; + Ok(()) + } + + fn sums(&mut self, previous: Option<&FE>, want_h0: bool) -> Result, Error> { + self.enqueue(previous, want_h0)?; + self.pending.take().ok_or(Error::NoVariablesLeft) + } + + fn finish(&mut self, previous: Option<&FE>) -> Result>, Error> { + let cells = 1usize << self.low; + self.fold(previous, cells); + self.f + .iter() + .map(|g| Mle::new(g[..cells].to_vec())) + .collect() + } + } + + /// A host tree whose steps run Gruen's rounds down to a cube of `2^low`, + /// as a device tree's steps do on the card. + struct GruenHostTree<'a> { + layers: &'a [FractionLayer], + low: usize, + } + + impl LadderTree for GruenHostTree<'_> { + fn input_vars(&self) -> usize { + self.layers.len() - 1 + } + + fn output(&self) -> (FE, FE) { + (self.layers[0].p.evals()[0], self.layers[0].q.evals()[0]) + } + + fn step( + &self, + layer: usize, + point: &[FE], + weights: [FE; 2], + claim: &(FE, FE), + ) -> Result + '_>, Error> { + let next = &self.layers[layer]; + let lambda = gruen_lambda(&weights).expect("μ ≠ 0 in these runs"); + let card = HostGruen::new(next, point, lambda, self.low); + let step = GruenStep::over(card, point, &weights, claim)?.expect("a Gruen step"); + Ok(Box::new(step)) + } + } + + /// ★ The ladder's Gruen steps send the host reference's bytes: every tree + /// at every step, with `w_p = μ^{2a}` scaling its own chain's messages, + /// from tails of one cell up to the whole layer, and through the rounds + /// that sum `H(0)` instead of deriving it. + #[test] + fn gruen_steps_send_the_host_references_bytes() { + let heights = [6usize, 3, 0, 6, 4, 1]; + let trees: Vec>> = heights + .iter() + .enumerate() + .map(|(t, &k)| layers(random_layer(k, 41 + t as u64))) + .collect(); + let refs: Vec<&[FractionLayer]> = trees.iter().map(|t| t.as_slice()).collect(); + let mut host = transcript(); + let (want, want_claims) = prove(&refs, &mut host).unwrap(); + for direct_h0 in [false, true] { + crate::gkr_gruen::force_gkr_gruen_direct_h0(direct_h0); + for low in [0usize, 1, 2, 6] { + let gruen: Vec> = trees + .iter() + .map(|layers| GruenHostTree { layers, low }) + .collect(); + let dyns: Vec<&dyn LadderTree> = + gruen.iter().map(|t| t as &dyn LadderTree).collect(); + let mut prover = transcript(); + let (got, claims) = prove_trees(&dyns, &mut prover).unwrap(); + assert!( + got == want && claims == want_claims, + "low {low}, direct H(0) {direct_h0}: the Gruen ladder moved a byte" + ); + assert_eq!(prover.state(), host.state()); + } + } + crate::gkr_gruen::force_gkr_gruen_direct_h0(false); + } +} diff --git a/crypto/multilinear/src/gpu.rs b/crypto/multilinear/src/gpu.rs new file mode 100644 index 000000000..392dbbfea --- /dev/null +++ b/crypto/multilinear/src/gpu.rs @@ -0,0 +1,7894 @@ +//! Device dispatch for the multilinear path. +//! +//! Every entry point here returns `None` when the device declines — a field +//! the kernels do not cover, a size below the launch threshold, a kill switch, +//! or any CUDA error — and the caller runs the host path. A dispatch that +//! succeeded is counted, so a bench can tell a GPU number from a CPU one +//! wearing its label. + +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Successful device commits of a stacked polynomial. +static COMMIT_CALLS: AtomicU64 = AtomicU64::new(0); +/// ★ Commits that asked the device and got nothing, and encoded on the host. +/// +/// The counter H4's arm needed and did not have. A device commit that declines +/// is INVISIBLE in every other number here: `COMMIT_CALLS` simply does not +/// rise, and a count that is merely lower than expected says nothing when the +/// expected count is itself derived. It matters because falling back is not a +/// slower way to do the same thing — `from_codeword` then retains a host +/// codeword and a host node array for the rest of the proof, so a card that +/// fills near the end of an epoch turns into gigabytes of host memory and a +/// utilisation figure that looks like a scheduling problem. +static HOST_FALLBACKS: AtomicU64 = AtomicU64::new(0); +/// ★ Commits the device was ASKED for and answered with an error — the part of +/// [`host_fallbacks`] that is not a policy decline (the size threshold, the +/// kill switch): a refused reservation (`CUDA_ERROR_OUT_OF_MEMORY`), a failed +/// allocation, launch or copy. Each one is also logged with its error. The +/// launch that can never succeed is why this exists: a WHIR base commit at +/// stack 27 launched a tile past CUDA's grid limit, every such commit fell back +/// to the host, and the error — the one thing that named the cause — was +/// dropped (`thoughts/zf/gap/fix/STRUCT.md` §9.2). +static COMMIT_ERRORS: AtomicU64 = AtomicU64::new(0); +/// ★ Merkle trees the device was ASKED to build over a HOST codeword (an +/// extension-field fold of a chain whose codeword is on the host, through +/// [`commit_tree_ext3`]) and answered with an error. The host builds that tree +/// instead. Kept apart from [`COMMIT_ERRORS`], which stays the part of +/// [`host_fallbacks`] that failed: such a tree is not a commit fallback. Each +/// one is also logged with its error. +static TREE_COMMIT_ERRORS: AtomicU64 = AtomicU64::new(0); +/// Sumchecks whose rounds ran on device. +static SUMCHECK_CALLS: AtomicU64 = AtomicU64::new(0); +/// Rounds within them, so a declined tail shows up. +static SUMCHECK_ROUNDS: AtomicU64 = AtomicU64::new(0); +/// Multilinear evaluations bound on device. +static EVALUATE_CALLS: AtomicU64 = AtomicU64::new(0); +/// Columns the claim reduce evaluated on the HOST, one at a time: the tables +/// the batched device evaluation did not take, at a height where a single +/// column stays here too ([`evaluates_on_device`]). The count +/// `LAMBDA_VM_ARGUE_DEVICE_COLUMNS` exists to move: with it off (`=0`) every +/// table under 2^16 rows lands here, however wide. +static HOST_EVALUATE_CALLS: AtomicU64 = AtomicU64::new(0); +/// Columns whose device value `LAMBDA_VM_ARGUE_XCHECK` recomputed on the host +/// and found equal — so a gate run can show the check ran, not only that +/// nothing failed. +static ARGUE_XCHECKS: AtomicU64 = AtomicU64::new(0); +/// Challenge tables the card built for the argument +/// (`LAMBDA_VM_ARGUE_DEVICE_TABLES`): the zerocheck's `eq` weights and the claim +/// reduce's shift tables and batched columns, each of which the host would +/// otherwise have built and uploaded. +static ARGUE_TABLES_ON_CARD: AtomicU64 = AtomicU64::new(0); +/// Of those, the ones `LAMBDA_VM_ARGUE_XCHECK` compared with the host's. +static ARGUE_TABLE_XCHECKS: AtomicU64 = AtomicU64::new(0); +/// Zerocheck sessions whose device rounds walked their program on demand +/// (`LAMBDA_VM_ARGUE_LEAN_PROGRAM`). +static LEAN_PROGRAMS: AtomicU64 = AtomicU64::new(0); +/// Of those, the ones `LAMBDA_VM_ARGUE_XCHECK` checked round by round against +/// today's program walking the same factors. +static LEAN_PROGRAM_XCHECKS: AtomicU64 = AtomicU64::new(0); +/// Fraction trees built and kept on device. +static TREE_CALLS: AtomicU64 = AtomicU64::new(0); +/// Tables whose factors were uploaded once and reused. +static FACTOR_CALLS: AtomicU64 = AtomicU64::new(0); +/// Openings whose two factors stayed on device across their groups. +static OPEN_CALLS: AtomicU64 = AtomicU64::new(0); +/// Of those, the ones that began in share form. +static LEAN_OPEN_CALLS: AtomicU64 = AtomicU64::new(0); +/// Of those, the ones whose rounds ran in tile form. +static LEAN_TILE_OPENS: AtomicU64 = AtomicU64::new(0); +/// Device folds that ran every level in one launch. +static FUSED_FOLD_CALLS: AtomicU64 = AtomicU64::new(0); +/// Folds of a codeword the device holds, in one launch or one per level: what +/// [`FUSED_FOLD_CALLS`] is a share of. +static RESIDENT_FOLD_CALLS: AtomicU64 = AtomicU64::new(0); +/// Successors encoded from their bound message instead of folded from the +/// codeword (the default; `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0` folds; D-SPEED +/// lever 5a). Not in [`RESIDENT_FOLD_CALLS`]: no fold ran. +static ENCODED_SUCCESSORS: AtomicU64 = AtomicU64::new(0); +/// ★ Successors the card was asked to encode — the encode on, the factors and +/// the codeword on the card — and declined: the session would not write its +/// tables, or the encode returned an error. The chain folds instead, which +/// proves the same bytes, so nothing else would name it: [`ENCODED_SUCCESSORS`] +/// only fails to rise. +static ENCODE_DECLINES: AtomicU64 = AtomicU64::new(0); +/// Round-0 openings of a coset-ordered tree served from the stack: each +/// queried coset evaluated from the polynomial, no codeword re-encoded. +static COSET_OPENINGS: AtomicU64 = AtomicU64::new(0); +/// ⚠ Coset-ordered round-0 commitments that re-encoded their codeword after +/// all: a plan the kernels do not take, the card declining, or a fold. Each is +/// today's cost paid where the lever was meant to save it. +static COSET_FALLBACKS: AtomicU64 = AtomicU64::new(0); +/// Coset-ordered round-0 successors encoded on the HOST, from the message the +/// host's factors bound: the card declined the opening (or the stack is under +/// its sumcheck threshold), and a round 0 on the card is never folded under +/// that order. The same bytes as the card's encode; a host NTT's time instead. +static COSET_HOST_ENCODES: AtomicU64 = AtomicU64::new(0); +/// Coset openings that read the stack where it lies in the store's buffer +/// (the default; `LAMBDA_VM_WHIR_COSET_INPLACE=0` assembles): no stack +/// assembled, its zero tiles folded unread. Among [`COSET_OPENINGS`]. +static COSET_INPLACE_OPENS: AtomicU64 = AtomicU64::new(0); +/// ⚠ Coset openings asked to read the stack in place whose parts the kernels +/// could not place: the stack assembled after all, today's cost. +static COSET_INPLACE_ASSEMBLED: AtomicU64 = AtomicU64::new(0); +/// The in-place openings' tiles no part touches (folded to zero unread), and +/// their mixed tiles (shorter columns, a column's end: read value by value), +/// summed over the openings. +static COSET_ZERO_TILES: AtomicU64 = AtomicU64::new(0); +static COSET_MIXED_TILES: AtomicU64 = AtomicU64::new(0); +/// ★ Stacked openings whose codeword the device holds but whose factors the +/// device did not build from the shares. +/// +/// The opening's twin of [`HOST_FALLBACKS`], and until this counter it had no +/// name either: `open_shared` declining sends the chain to `Factors::new`, +/// which assembles the stacked polynomial and its weight table on the host — a +/// base copy of the stack and an extension table three times its size — before +/// trying the device once more with the tables built. Nothing else rises or +/// falls when that happens: [`OPEN_CALLS`] only fails to rise, and an expected +/// count is itself derived. It matters most where the card is fullest, which +/// is where an opening runs without the turn its group asked for (see +/// `stacked_eval::StackedCommitment`). +static OPEN_HOST_FALLBACKS: AtomicU64 = AtomicU64::new(0); +/// ★ Grouped chains (`GroupFold::Indexed`, a run of two or more polynomials) +/// whose round 0 ran on the card: the polynomials' openings in lockstep, the +/// index rounds over their bound tables ([`GroupedFactors`]). +static GROUPED_CARD_CHAINS: AtomicU64 = AtomicU64::new(0); +/// ★ Grouped chains over codewords the card holds whose round-0 factors were +/// built on the HOST — the grouped twin of [`OPEN_HOST_FALLBACKS`], which each +/// of their polynomials also counts. A grouped chain over host codewords is +/// the host path, not a fallback, and counts in neither. +static GROUPED_HOST_CHAINS: AtomicU64 = AtomicU64::new(0); +/// Grouped round-0 successors folded on the card polynomial by polynomial and +/// combined there ([`combine_codewords`]): the path when the successor is not +/// encoded from the bound message. +static GROUPED_COMBINES: AtomicU64 = AtomicU64::new(0); +/// Groups that gave their room back when their commits ended. +static ROOM_PARKS: AtomicU64 = AtomicU64::new(0); +/// Group openings that took their room back. +static ROOM_TURNS: AtomicU64 = AtomicU64::new(0); +/// ★ Group openings whose room the card would not give back — see +/// [`take_turn`] for what the opening does then. +static ROOM_TURN_REFUSALS: AtomicU64 = AtomicU64::new(0); +/// ★ Revived groups whose openings' room the card would not promise: their +/// codewords then promise their own working sets one at a time +/// (`stacked_eval::StackedCommitment::revive`). +static ROOM_REVIVE_REFUSALS: AtomicU64 = AtomicU64::new(0); +/// ★ Groups whose commits' room the card would not promise: each codeword +/// then promises its own working set (`stacked_eval::StackedCommitment`). +static ROOM_COMMIT_REFUSALS: AtomicU64 = AtomicU64::new(0); +/// Rooms a group sized, for its commits or its openings' turn. +static ROOM_SIZINGS: AtomicU64 = AtomicU64::new(0); +/// Of those, the ones sized to the turn they cover rather than to a whole +/// base codeword (`LAMBDA_VM_NO_WHIR_ROOM_RESIZE`). +static ROOMS_TURN_SIZED: AtomicU64 = AtomicU64::new(0); +/// ★ GKR fraction trees whose whole-tree promise the budget refused on the +/// consume path, after refusing their carry: the host built that table's +/// factors and tree. See [`note_gkr_tree_refusal`]; each one is also a +/// `math_cuda::device::device_fallbacks`. +static GKR_TREE_REFUSALS: AtomicU64 = AtomicU64::new(0); +/// Fraction trees whose input layer was written from the base columns +/// (`LAMBDA_VM_ARGUE_GKR_INPUT`). +static INPUT_COLUMNS_TREES: AtomicU64 = AtomicU64::new(0); + +/// Fraction trees whose input layer was written from the base columns. +pub fn input_columns_trees() -> u64 { + INPUT_COLUMNS_TREES.load(Ordering::Relaxed) +} + +/// Tables whose factors were set up as base columns, with no lift +/// (`LAMBDA_VM_ARGUE_NO_LIFT`). +static COLUMN_FACTOR_TABLES: AtomicU64 = AtomicU64::new(0); + +pub fn column_factor_tables() -> u64 { + COLUMN_FACTOR_TABLES.load(Ordering::Relaxed) +} + +/// Tables the no-lift path lifted after all: the fused rounds declined and +/// today's rounds need the lifted factors. +static LATE_LIFTS: AtomicU64 = AtomicU64::new(0); + +pub fn late_lifts() -> u64 { + LATE_LIFTS.load(Ordering::Relaxed) +} + +pub(crate) fn note_late_lift() { + LATE_LIFTS.fetch_add(1, Ordering::Relaxed); + crate::whir_split::bump(&crate::whir_split::LATE_LIFT); +} + +/// Under `LAMBDA_VM_ARGUE_TREE_SYNC` (a diagnostic), wait for the stream's +/// kernels so the tree's split charges each part its own card time. +#[cfg(feature = "cuda")] +macro_rules! tree_sync { + ($stream:expr) => { + if crate::whir_split::tree_sync() { + let _ = $stream.synchronize(); + } + }; +} + +pub fn commit_calls() -> u64 { + COMMIT_CALLS.load(Ordering::Relaxed) +} + +pub fn host_fallbacks() -> u64 { + HOST_FALLBACKS.load(Ordering::Relaxed) +} + +pub fn commit_errors() -> u64 { + COMMIT_ERRORS.load(Ordering::Relaxed) +} + +pub fn tree_commit_errors() -> u64 { + TREE_COMMIT_ERRORS.load(Ordering::Relaxed) +} + +/// Device errors printed per counter per process; past it only the counter +/// moves. +const DEVICE_ERRORS_LOGGED: u64 = 16; + +/// Count one device error in `counter` and print the first +/// [`DEVICE_ERRORS_LOGGED`] of them. `describe` says what failed and where the +/// work goes instead; `kind` names the errors in the note on the last line +/// printed. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +fn note_device_error(counter: &AtomicU64, kind: &str, describe: impl FnOnce() -> String) { + let seen = counter.fetch_add(1, Ordering::Relaxed); + if seen < DEVICE_ERRORS_LOGGED { + let last = if seen + 1 == DEVICE_ERRORS_LOGGED { + format!(" (further {kind} are counted, not printed)") + } else { + String::new() + }; + eprintln!("[whir] {}{last}", describe()); + } +} + +/// Count and log a device commit that returned an error, before its caller +/// declines and the chain encodes on the host (where [`note_host_fallback`] +/// counts it too). +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +fn note_commit_error(what: &str, log_evals: usize, log_blowup: usize, err: &dyn core::fmt::Debug) { + note_device_error(&COMMIT_ERRORS, "commit errors", || { + format!( + "device commit ({what}) of 2^{log_evals} evaluations at blowup 2^{log_blowup} \ + failed: {err:?}; encoding on the host" + ) + }); +} + +/// Count and log a device tree over a host codeword that returned an error, +/// before its caller hashes the tree on the host. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +fn note_tree_commit_error(log_len: u32, log_folding: usize, err: &dyn core::fmt::Debug) { + note_device_error(&TREE_COMMIT_ERRORS, "tree commit errors", || { + format!( + "device commit (ext3 tree) of 2^{log_len} values in blocks of 2^{log_folding} \ + failed: {err:?}; hashing on the host" + ) + }); +} + +/// Called where a commit gives up on the device. Counts in non-cuda builds +/// too, where every commit takes that path and the number is the commit count. +pub(crate) fn note_host_fallback() { + HOST_FALLBACKS.fetch_add(1, Ordering::Relaxed); +} + +pub fn sumcheck_calls() -> u64 { + SUMCHECK_CALLS.load(Ordering::Relaxed) +} + +/// A device sumcheck another path ran (`gpu_fused`): counted as one call and +/// its rounds, as this file's own sessions are. +#[cfg(feature = "cuda")] +pub(crate) fn note_device_sumcheck(rounds: u64) { + SUMCHECK_CALLS.fetch_add(1, Ordering::Relaxed); + SUMCHECK_ROUNDS.fetch_add(rounds, Ordering::Relaxed); +} + +pub fn sumcheck_rounds() -> u64 { + SUMCHECK_ROUNDS.load(Ordering::Relaxed) +} + +pub fn evaluate_calls() -> u64 { + EVALUATE_CALLS.load(Ordering::Relaxed) +} + +pub fn host_evaluate_calls() -> u64 { + HOST_EVALUATE_CALLS.load(Ordering::Relaxed) +} + +pub fn argue_xchecks() -> u64 { + ARGUE_XCHECKS.load(Ordering::Relaxed) +} + +pub fn argue_tables_on_card() -> u64 { + ARGUE_TABLES_ON_CARD.load(Ordering::Relaxed) +} + +pub fn argue_table_xchecks() -> u64 { + ARGUE_TABLE_XCHECKS.load(Ordering::Relaxed) +} + +pub fn lean_programs() -> u64 { + LEAN_PROGRAMS.load(Ordering::Relaxed) +} + +pub fn lean_program_xchecks() -> u64 { + LEAN_PROGRAM_XCHECKS.load(Ordering::Relaxed) +} + +pub fn tree_calls() -> u64 { + TREE_CALLS.load(Ordering::Relaxed) +} + +pub fn factor_calls() -> u64 { + FACTOR_CALLS.load(Ordering::Relaxed) +} + +/// Stacked openings that ran their first rounds over the shares +/// ([`math_cuda::whir_open::LeanRound0`]) rather than materialising up front. +pub fn lean_open_calls() -> u64 { + LEAN_OPEN_CALLS.load(Ordering::Relaxed) +} + +/// Lean openings whose rounds ran in tile form — one pass over the message, +/// then tables a tile's size smaller ([`crate::lean_tiles`], +/// `LAMBDA_VM_WHIR_LEAN_TILES`) — rather than over every stack position. +pub fn lean_tile_opens() -> u64 { + LEAN_TILE_OPENS.load(Ordering::Relaxed) +} + +/// Rounds `LAMBDA_VM_WHIR_LEAN_TILES=check` ran both ways and compared, and +/// how many differed (a round that differed sent the share form's values). +#[cfg(feature = "cuda")] +pub fn lean_tile_checks() -> (u64, u64) { + math_cuda::whir_open::tile_check_counts() +} + +#[cfg(not(feature = "cuda"))] +pub fn lean_tile_checks() -> (u64, u64) { + (0, 0) +} + +/// Lean openings whose bound tables `LAMBDA_VM_WHIR_LEAN_TILES=check` wrote +/// both ways and compared, and the positions that differed (the share form's +/// tables are the ones kept). +#[cfg(feature = "cuda")] +pub fn lean_tile_table_checks() -> (u64, u64) { + math_cuda::whir_open::tile_table_check_counts() +} + +#[cfg(not(feature = "cuda"))] +pub fn lean_tile_table_checks() -> (u64, u64) { + (0, 0) +} + +/// The most one lean opening held on the card at once, by form — share, +/// tile, checking — counted from its own allocations +/// ([`math_cuda::whir_open::lean_peak_bytes`]). +#[cfg(feature = "cuda")] +pub fn lean_open_peaks() -> [u64; 3] { + math_cuda::whir_open::lean_peak_bytes() +} + +#[cfg(not(feature = "cuda"))] +pub fn lean_open_peaks() -> [u64; 3] { + [0; 3] +} + +/// Device folds that ran every level in one launch. +pub fn fused_fold_calls() -> u64 { + FUSED_FOLD_CALLS.load(Ordering::Relaxed) +} + +/// Folds of a codeword the device holds, fused or level by level. +pub fn resident_fold_calls() -> u64 { + RESIDENT_FOLD_CALLS.load(Ordering::Relaxed) +} + +/// Successors encoded from their bound message rather than folded — see +/// [`ENCODED_SUCCESSORS`]. +pub fn encoded_successors() -> u64 { + ENCODED_SUCCESSORS.load(Ordering::Relaxed) +} + +/// Successor encodes the card declined, the chain folding instead — see +/// [`ENCODE_DECLINES`]. Zero in a build without a device. +pub fn encode_declines() -> u64 { + ENCODE_DECLINES.load(Ordering::Relaxed) +} + +/// Called where the card declines a successor encode it was asked for: +/// counted, and said. +#[cfg(feature = "cuda")] +fn note_encode_declined(levels: usize) { + let count = ENCODE_DECLINES.fetch_add(1, Ordering::Relaxed) + 1; + eprintln!( + "[whir] the card DECLINED a successor encode ({levels}-level fold); the chain folds \ + instead; encode declines {count}" + ); +} + +/// Whether [`open_on_device`] declines as a card that will not take the +/// opening would, so a test can reach factors built on the host and KEPT there +/// over a device codeword: [`force_shared_open_declined`] alone sends the +/// opening to the host's tables, which [`open_on_device`] then uploads. +#[cfg(feature = "cuda")] +static DEVICE_OPEN_DECLINED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Makes [`open_on_device`] decline while `on`. Not for production callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_device_open_declined(on: bool) { + DEVICE_OPEN_DECLINED.store(on, Ordering::Relaxed); +} + +/// [`COSET_OPENINGS`]. +pub fn coset_openings() -> u64 { + COSET_OPENINGS.load(Ordering::Relaxed) +} + +/// [`COSET_FALLBACKS`]. +pub fn coset_fallbacks() -> u64 { + COSET_FALLBACKS.load(Ordering::Relaxed) +} + +#[cfg(feature = "cuda")] +fn note_coset_fallback() { + COSET_FALLBACKS.fetch_add(1, Ordering::Relaxed); +} + +/// [`COSET_HOST_ENCODES`]. +pub fn coset_host_encodes() -> u64 { + COSET_HOST_ENCODES.load(Ordering::Relaxed) +} + +/// Called where a coset-ordered round 0's successor is encoded on the host: +/// counted, and said. +pub(crate) fn note_coset_host_encode(levels: usize, num_vars: usize) { + let count = COSET_HOST_ENCODES.fetch_add(1, Ordering::Relaxed) + 1; + eprintln!( + "[whir] a coset round-0 successor encoded on the HOST from the bound message \ + ({levels}-level fold, message 2^{num_vars}); coset host encodes {count}" + ); +} + +/// Whether a coset-ordered round-0 commitment is revived over its stack (its +/// openings evaluated coset by coset) instead of re-encoded: the default under +/// the coset order; `LAMBDA_VM_WHIR_COSET_OPENINGS=0` or a test's override +/// re-encodes. The same bytes either way. `false` in a build without a device. +pub fn coset_openings_in_effect() -> bool { + match COSET_OPENINGS_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + cfg!(feature = "cuda") + && *ON.get_or_init(|| { + !matches!( + std::env::var("LAMBDA_VM_WHIR_COSET_OPENINGS") + .ok() + .as_deref() + .map(str::trim), + Some("0") + ) + }) + } + } +} + +static COSET_OPENINGS_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_WHIR_COSET_OPENINGS` for the whole process. `None` +/// restores the environment's setting. Not for production callers. +#[doc(hidden)] +pub fn force_coset_openings(on: Option) { + COSET_OPENINGS_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// [`COSET_INPLACE_OPENS`]. +pub fn coset_inplace_opens() -> u64 { + COSET_INPLACE_OPENS.load(Ordering::Relaxed) +} + +/// [`COSET_INPLACE_ASSEMBLED`]. +pub fn coset_inplace_assembled() -> u64 { + COSET_INPLACE_ASSEMBLED.load(Ordering::Relaxed) +} + +/// [`COSET_ZERO_TILES`] and [`COSET_MIXED_TILES`]. +pub fn coset_inplace_tiles() -> (u64, u64) { + ( + COSET_ZERO_TILES.load(Ordering::Relaxed), + COSET_MIXED_TILES.load(Ordering::Relaxed), + ) +} + +/// Whether a coset opening reads the stack where it lies instead of +/// assembling it unless told otherwise: [`coset_inplace_in_effect`]'s answer +/// with `LAMBDA_VM_WHIR_COSET_INPLACE` unset. +const COSET_INPLACE_DEFAULT: bool = true; + +/// Whether a coset opening reads the stack where it lies instead of +/// assembling it: [`COSET_INPLACE_DEFAULT`] (on); +/// `LAMBDA_VM_WHIR_COSET_INPLACE=0` assembles, `=1` reads in place, a test's +/// override wins. The same words either way. +pub fn coset_inplace_in_effect() -> bool { + match COSET_INPLACE_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + coset_inplace_from( + std::env::var("LAMBDA_VM_WHIR_COSET_INPLACE") + .ok() + .as_deref(), + ) + }) + } + } +} + +/// [`coset_inplace_in_effect`] for the variable's value `word`. +fn coset_inplace_from(word: Option<&str>) -> bool { + match word.map(str::trim) { + Some("0") => false, + Some("1") => true, + _ => COSET_INPLACE_DEFAULT, + } +} + +static COSET_INPLACE_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_WHIR_COSET_INPLACE` for the whole process. `None` +/// restores the environment's setting. Not for production callers. +#[doc(hidden)] +pub fn force_coset_inplace(on: Option) { + COSET_INPLACE_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Whether a chain's first successor is encoded from its bound message (the +/// default; `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0` or a test's override turns it +/// off). `false` in a build without a device. +pub fn encode_successor_in_effect() -> bool { + #[cfg(feature = "cuda")] + { + encode_successor() + } + #[cfg(not(feature = "cuda"))] + { + false + } +} + +/// The rounds a stacked opening runs over its shares, as this process reads +/// `LAMBDA_VM_WHIR_LEAN_ROUNDS` (or a test's override). `0` in a build without +/// a device, where no opening runs there. +pub fn lean_rounds_in_effect() -> usize { + #[cfg(feature = "cuda")] + { + lean_rounds() + } + #[cfg(not(feature = "cuda"))] + { + 0 + } +} + +/// Stacked openings over a device codeword whose factors were built on the +/// host — see [`OPEN_HOST_FALLBACKS`]. Zero in a build without a device, where +/// no codeword is on one. +pub fn open_host_fallbacks() -> u64 { + OPEN_HOST_FALLBACKS.load(Ordering::Relaxed) +} + +/// Called where a stacked opening over a device codeword builds its factors on +/// the host: counted, and said, because nothing else would show it. +pub(crate) fn note_open_host_fallback(n_stack: usize) { + let count = OPEN_HOST_FALLBACKS.fetch_add(1, Ordering::Relaxed) + 1; + eprintln!( + "[whir] opening factors built on the HOST over a device codeword (stack 2^{n_stack}); \ + open host fallbacks {count}" + ); +} + +pub fn open_calls() -> u64 { + OPEN_CALLS.load(Ordering::Relaxed) +} + +/// Grouped chains whose round 0 ran on the card — see [`GROUPED_CARD_CHAINS`]. +pub fn grouped_card_chains() -> u64 { + GROUPED_CARD_CHAINS.load(Ordering::Relaxed) +} + +/// Grouped chains over card codewords whose round-0 factors the host built — +/// see [`GROUPED_HOST_CHAINS`]. Zero in a build without a device. +pub fn grouped_host_chains() -> u64 { + GROUPED_HOST_CHAINS.load(Ordering::Relaxed) +} + +/// Grouped round-0 successors folded and combined on the card — see +/// [`GROUPED_COMBINES`]. +pub fn grouped_combined_successors() -> u64 { + GROUPED_COMBINES.load(Ordering::Relaxed) +} + +/// Called where a grouped chain over card codewords (held, or served from +/// their stack) builds its round-0 factors on the host: counted once as a +/// chain and once a polynomial +/// ([`note_open_host_fallback`]), and said. +pub(crate) fn note_grouped_host_fallback(polys: usize, n_stack: usize) { + let count = GROUPED_HOST_CHAINS.fetch_add(1, Ordering::Relaxed) + 1; + eprintln!( + "[whir] a grouped round 0 of {polys} polynomials built on the HOST over device \ + codewords (stack 2^{n_stack}); grouped host chains {count}" + ); + for _ in 0..polys { + note_open_host_fallback(n_stack); + } +} + +/// Groups that gave their room back when their commits ended. +pub fn room_parks() -> u64 { + ROOM_PARKS.load(Ordering::Relaxed) +} + +/// Group openings that took their room back. +pub fn room_turns() -> u64 { + ROOM_TURNS.load(Ordering::Relaxed) +} + +/// Group openings whose room the card would not give back. +pub fn room_turn_refusals() -> u64 { + ROOM_TURN_REFUSALS.load(Ordering::Relaxed) +} + +/// Revived groups whose openings' room the card would not promise. +pub fn room_revive_refusals() -> u64 { + ROOM_REVIVE_REFUSALS.load(Ordering::Relaxed) +} + +/// Called where a revive's room is refused. +pub(crate) fn note_revive_room_refused() { + ROOM_REVIVE_REFUSALS.fetch_add(1, Ordering::Relaxed); +} + +/// Groups whose commits' room the card would not promise. +pub fn room_commit_refusals() -> u64 { + ROOM_COMMIT_REFUSALS.load(Ordering::Relaxed) +} + +/// Called where a commit's room is refused. +pub(crate) fn note_commit_room_refused() { + ROOM_COMMIT_REFUSALS.fetch_add(1, Ordering::Relaxed); +} + +/// GKR fraction trees the host built because the card refused their promise. +pub fn gkr_tree_refusals() -> u64 { + GKR_TREE_REFUSALS.load(Ordering::Relaxed) +} + +/// Rooms a group sized, for its commits or its openings' turn. +pub fn room_sizings() -> u64 { + ROOM_SIZINGS.load(Ordering::Relaxed) +} + +/// Rooms sized to the turn they cover rather than to a whole base codeword. +pub fn rooms_turn_sized() -> u64 { + ROOMS_TURN_SIZED.load(Ordering::Relaxed) +} + +/// Called where a group sizes a room: `turn_sized` unless +/// `LAMBDA_VM_NO_WHIR_ROOM_RESIZE` has it promise a whole codeword. +pub(crate) fn note_room_sized(turn_sized: bool) { + ROOM_SIZINGS.fetch_add(1, Ordering::Relaxed); + if turn_sized { + ROOMS_TURN_SIZED.fetch_add(1, Ordering::Relaxed); + } +} + +pub fn reset_call_counters() { + COMMIT_CALLS.store(0, Ordering::Relaxed); + HOST_FALLBACKS.store(0, Ordering::Relaxed); + COMMIT_ERRORS.store(0, Ordering::Relaxed); + TREE_COMMIT_ERRORS.store(0, Ordering::Relaxed); + SUMCHECK_CALLS.store(0, Ordering::Relaxed); + SUMCHECK_ROUNDS.store(0, Ordering::Relaxed); + EVALUATE_CALLS.store(0, Ordering::Relaxed); + HOST_EVALUATE_CALLS.store(0, Ordering::Relaxed); + ARGUE_XCHECKS.store(0, Ordering::Relaxed); + ARGUE_TABLES_ON_CARD.store(0, Ordering::Relaxed); + ARGUE_TABLE_XCHECKS.store(0, Ordering::Relaxed); + LEAN_PROGRAMS.store(0, Ordering::Relaxed); + LEAN_PROGRAM_XCHECKS.store(0, Ordering::Relaxed); + TREE_CALLS.store(0, Ordering::Relaxed); + FACTOR_CALLS.store(0, Ordering::Relaxed); + OPEN_CALLS.store(0, Ordering::Relaxed); + LEAN_OPEN_CALLS.store(0, Ordering::Relaxed); + LEAN_TILE_OPENS.store(0, Ordering::Relaxed); + FUSED_FOLD_CALLS.store(0, Ordering::Relaxed); + RESIDENT_FOLD_CALLS.store(0, Ordering::Relaxed); + ENCODED_SUCCESSORS.store(0, Ordering::Relaxed); + ENCODE_DECLINES.store(0, Ordering::Relaxed); + COSET_OPENINGS.store(0, Ordering::Relaxed); + COSET_FALLBACKS.store(0, Ordering::Relaxed); + COSET_HOST_ENCODES.store(0, Ordering::Relaxed); + COSET_INPLACE_OPENS.store(0, Ordering::Relaxed); + COSET_INPLACE_ASSEMBLED.store(0, Ordering::Relaxed); + COSET_ZERO_TILES.store(0, Ordering::Relaxed); + COSET_MIXED_TILES.store(0, Ordering::Relaxed); + OPEN_HOST_FALLBACKS.store(0, Ordering::Relaxed); + GROUPED_CARD_CHAINS.store(0, Ordering::Relaxed); + GROUPED_HOST_CHAINS.store(0, Ordering::Relaxed); + GROUPED_COMBINES.store(0, Ordering::Relaxed); + ROOM_PARKS.store(0, Ordering::Relaxed); + ROOM_TURNS.store(0, Ordering::Relaxed); + ROOM_TURN_REFUSALS.store(0, Ordering::Relaxed); + ROOM_REVIVE_REFUSALS.store(0, Ordering::Relaxed); + ROOM_COMMIT_REFUSALS.store(0, Ordering::Relaxed); + ROOM_SIZINGS.store(0, Ordering::Relaxed); + ROOMS_TURN_SIZED.store(0, Ordering::Relaxed); + crate::gkr_input_real::reset_counts(); + #[cfg(feature = "cuda")] + math_cuda::round_slot::reset_counts(); +} + +/// A sumcheck's round proofs, the challenges they drew, and what every slot +/// was bound to — the factors are folded where they lie, so their values at +/// the sumcheck's point are already there when the rounds end. +/// What a resident sumcheck's rounds on device leave: the rounds, the point +/// they drew, and **every** factor — resident and not — as the last fold left +/// it. One value each when the device ran the cube out, a cube when it stopped +/// at the crossover for the caller to finish. +type ResidentRounds = ( + Vec>, + Vec>, + Vec>, +); + +type SumcheckRounds = ( + Vec>, + Vec>, + Vec>, +); + +/// What a grouped chain's round 0 hands back ([`GroupedFactors::rounds`]): its +/// messages, its challenges (the data ones, then the index's), and the one +/// opening every later round runs on. +type GroupedRounds = ( + Vec>, + Vec>, + OpeningFactors, +); + +/// Codeword size below which the host wins: the kernels are a dozen launches +/// and a round trip, and a small NTT finishes in host cache before that. +#[cfg(feature = "cuda")] +const COMMIT_THRESHOLD: usize = 1 << 16; + +/// Per query an authentication path, and the tree's Merkle cap. +pub(crate) type PathsAndCap = (Vec>, Vec<[u8; 32]>); + +/// Siblings on a full path of a tree over `leaves` leaves under `hash`: one +/// per level of a binary tree, three per level of an arity-4 one. +#[cfg(feature = "cuda")] +fn path_len(leaves: usize, hash: crate::whir_hash::DeviceHashKey) -> usize { + let arity = hash.arity(); + (arity - 1) * crypto::merkle_tree::cap::tree_levels(leaves.trailing_zeros() as usize, arity) +} + +/// A byte buffer of Merkle nodes, relabelled as nodes without copying. +/// +/// A tree over a stacked polynomial is hundreds of megabytes; chunking it into +/// arrays would copy all of it to change nothing but the type. +#[cfg(feature = "cuda")] +fn nodes_in_place(bytes: Vec) -> Option> { + if !bytes.len().is_multiple_of(32) || !bytes.capacity().is_multiple_of(32) { + return None; + } + // SAFETY: `[u8; 32]` has the alignment of `u8` and 32 times its size, so + // the allocation describes the same bytes either way. + let mut bytes = core::mem::ManuallyDrop::new(bytes); + Some(unsafe { + Vec::from_raw_parts( + bytes.as_mut_ptr() as *mut [u8; 32], + bytes.len() / 32, + bytes.capacity() / 32, + ) + }) +} + +/// Op tags the sumcheck kernel reads. MUST stay in sync with +/// `crypto/math-cuda/kernels/sumcheck.cu`. +pub mod op { + pub const FIXED: u32 = 0; + pub const VAR: u32 = 1; + pub const ADD: u32 = 2; + pub const SUB: u32 = 3; + pub const MUL: u32 = 4; + pub const NEG: u32 = 5; +} + +/// A program lowered for the device: the nodes (two u64 each, `op | a << 32` +/// then `b | res << 32`), the ext3 constants they read (three u64 each), the +/// slot file's width and the slot the root lands in. +#[derive(Clone, Debug)] +pub struct Lowered { + pub nodes: Vec, + pub consts: Vec, + pub num_slots: usize, + pub root_slot: u32, +} + +/// Slots the round kernel will hold per thread before the dispatch declines. +/// +/// The slot file is `slots * 24 * threads` bytes and the scratch budget is +/// fixed, so a wider program buys fewer threads. This is where that stops +/// being a trade: at 8192 live values a single block of 256 already wants +/// 48 MiB, and a launch of one block is not a launch. +/// +/// It is a cliff, not a dial. The real AIRs peak near a thousand — the widest +/// precompile lowers to 1036 — and a cap below that sends the tables with the +/// *most* work per row to the host, which is where they cost the most. +pub const MAX_SLOTS: usize = 8192; + +/// Cube size below which the host wins for a table of one or two factors: the +/// rounds are a launch and a round trip each, and a small cube fits in cache. +#[cfg(feature = "cuda")] +const SUMCHECK_THRESHOLD: usize = 1 << 12; + +/// Device bytes promised for as long as the returned handle lives. +/// +/// The caller is a structure whose parts are allocated and freed one at a time +/// — a group of commitments and the working set its commits and openings take +/// turns with — so what it promises is the turn, not the sum. +#[cfg(feature = "cuda")] +pub fn reserve_room(bytes: u64) -> Option { + math_cuda::device::reserve(bytes).map(DeviceRoom) +} + +#[cfg(not(feature = "cuda"))] +pub fn reserve_room(_bytes: u64) -> Option { + None +} + +/// Whether a group gives its room back between its last commit and its first +/// opening, and takes it again for the openings ([`take_turn`]). `true` unless +/// `LAMBDA_VM_NO_WHIR_ROOM_PARK` is set, which holds it from the commit to the +/// last opening as before — the A/B's control, off the same binary. +/// +/// Between the two runs the whole per-table argument, which reserves against +/// the same budget and never touches the room: the room is a promise for the +/// turns the commits and the openings take, and the argument takes none. Held +/// across it, it is a codeword's worth of budget the argument is refused. +pub(crate) fn room_park() -> bool { + match ROOM_PARK_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); + !*OFF.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_WHIR_ROOM_PARK").is_some()) + } + } +} + +static ROOM_PARK_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_NO_WHIR_ROOM_PARK` for the whole process — for a test +/// that commits and opens both ways in one binary. `None` restores the +/// environment's setting. Not for production callers. +#[doc(hidden)] +pub fn force_room_park(on: Option) { + ROOM_PARK_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Called where a group gives its room back after its commits. +pub(crate) fn note_room_parked() { + ROOM_PARKS.fetch_add(1, Ordering::Relaxed); +} + +/// Takes a parked group's room back for its openings: `bytes` promised until +/// the returned handle drops, or `None`, counted and said. +/// +/// ⛔ WHY IT CAN BE REFUSED. Anything that reserves between the group's last +/// commit and this call can have taken the bytes. In the production drivers +/// that is the argument of the same proof, whose reservations are all released +/// by the time its last table is argued — and the leaf layers the commits +/// retained, which `reserve` evicts on a miss, so a retained layer never +/// outranks a turn. A second proof on the same card in the same process is the +/// case that can hold the bytes past this point, and a test binary running in +/// parallel is one. +/// +/// ⛔ WHAT A REFUSAL DOES: the opening runs on the device anyway, UNPROMISED. +/// Its codewords are on the card with no host copy (`fold_held` has nowhere to +/// fold a device codeword the device declines) and the transcript has absorbed +/// their roots, so there is no host path to fall back to — declining here would +/// be failing the proof. What runs unpromised is one opening's working set: +/// with the lean opening and the fused fold, about a quarter of a codeword — +/// where before them every opening ran nearly three codewords past the room it +/// had. If the card then really is full, the factors fall to the host and +/// [`open_host_fallbacks`] counts them, or a fold fails with +/// [`crate::Error::DeviceFailed`] — both loud, neither silent. +pub(crate) fn take_turn(bytes: u64, polys: usize, n_stack: usize) -> Option { + if let Some(room) = reserve_room(bytes) { + ROOM_TURNS.fetch_add(1, Ordering::Relaxed); + return Some(room); + } + let count = ROOM_TURN_REFUSALS.fetch_add(1, Ordering::Relaxed) + 1; + eprintln!( + "[whir] opening turn REFUSED: {} MiB for a group of {polys} at 2^{n_stack}, {} — \ + opening on the device UNPROMISED; room turn refusals {count}", + bytes >> 20, + budget_headroom(), + ); + None +} + +/// The budget left to promise, for a refusal's line. +#[cfg(feature = "cuda")] +fn budget_headroom() -> String { + match math_cuda::device::backend() { + Ok(be) => format!( + "budget headroom {} MiB", + be.vram_budget_bytes().saturating_sub(be.reserved_bytes()) >> 20 + ), + Err(_) => "no device".to_string(), + } +} + +#[cfg(not(feature = "cuda"))] +fn budget_headroom() -> String { + "no device".to_string() +} + +/// Host bytes the card's pinned staging buffers hold +/// ([`math_cuda::device::pinned_host_bytes`]); 0 without a device. +#[cfg(feature = "cuda")] +pub fn pinned_host_bytes() -> usize { + math_cuda::device::pinned_host_bytes() +} + +#[cfg(not(feature = "cuda"))] +pub fn pinned_host_bytes() -> usize { + 0 +} + +/// Opens a window over the device ledger's peak +/// ([`math_cuda::device::reset_window_high_water`]); nothing without a device. +#[cfg(feature = "cuda")] +pub fn reset_reserved_window() { + math_cuda::device::reset_window_high_water(); +} + +#[cfg(not(feature = "cuda"))] +pub fn reset_reserved_window() {} + +/// The ledger's peak since [`reset_reserved_window`]; 0 without a device. +#[cfg(feature = "cuda")] +pub fn reserved_window_peak() -> u64 { + math_cuda::device::window_high_water() +} + +#[cfg(not(feature = "cuda"))] +pub fn reserved_window_peak() -> u64 { + 0 +} + +/// What the ledger has promised right now; 0 without a device. +#[cfg(feature = "cuda")] +pub fn ledger_reserved() -> u64 { + math_cuda::device::backend() + .map(|be| be.reserved_bytes()) + .unwrap_or(0) +} + +#[cfg(not(feature = "cuda"))] +pub fn ledger_reserved() -> u64 { + 0 +} + +/// Sets the device up ahead of its first use, without counting as a device +/// entry ([`math_cuda::device::set_up_backend`]); nothing without a device. +#[cfg(feature = "cuda")] +pub fn set_up_device() { + math_cuda::device::set_up_backend(); +} + +#[cfg(not(feature = "cuda"))] +pub fn set_up_device() {} + +/// What the ledger may promise in all; 0 without a device. +#[cfg(feature = "cuda")] +pub fn reserve_budget() -> u64 { + math_cuda::device::backend() + .map(|be| be.vram_budget_bytes()) + .unwrap_or(0) +} + +#[cfg(not(feature = "cuda"))] +pub fn reserve_budget() -> u64 { + 0 +} + +/// While it lives, this thread's device calls draw their streams from the +/// background pool ([`math_cuda::stream_class`]), when one exists; nothing +/// without a device. It belongs to the thread that took it. +#[cfg(feature = "cuda")] +pub struct BackgroundStreams(#[allow(dead_code)] math_cuda::stream_class::ClassGuard); + +#[cfg(not(feature = "cuda"))] +pub struct BackgroundStreams; + +/// Marks this thread's stream draws as background work until the guard drops. +#[cfg(feature = "cuda")] +pub fn background_streams() -> BackgroundStreams { + BackgroundStreams(math_cuda::stream_class::enter( + math_cuda::stream_class::StreamClass::Background, + )) +} + +#[cfg(not(feature = "cuda"))] +pub fn background_streams() -> BackgroundStreams { + BackgroundStreams +} + +/// While it lives, every background-class thread ([`background_streams`]) +/// reads back from the card through a pinned buffer with a blocking-sync event +/// wait ([`math_cuda::readback`]); nothing without a device. +#[cfg(feature = "cuda")] +pub struct PinnedReadbacks(#[allow(dead_code)] math_cuda::readback::PinnedReadbacks); + +#[cfg(not(feature = "cuda"))] +pub struct PinnedReadbacks; + +/// Turns pinned read-backs on until the guard drops. +#[cfg(feature = "cuda")] +pub fn pinned_readbacks() -> PinnedReadbacks { + PinnedReadbacks(math_cuda::readback::pinned()) +} + +#[cfg(not(feature = "cuda"))] +pub fn pinned_readbacks() -> PinnedReadbacks { + PinnedReadbacks +} + +/// Background-class read-backs so far, `(pinned, pageable)`, and the bytes +/// the pinned buffers hold now ([`math_cuda::readback`]); zeros without a +/// device. +#[cfg(feature = "cuda")] +pub fn background_readbacks() -> (u64, u64, u64) { + let (pinned, pageable) = math_cuda::readback::background_copies(); + (pinned, pageable, math_cuda::readback::pinned_bytes()) +} + +#[cfg(not(feature = "cuda"))] +pub fn background_readbacks() -> (u64, u64, u64) { + (0, 0, 0) +} + +/// Stream draws so far, `(foreground pool, background pool)` +/// ([`math_cuda::stream_class::draws`]); `(0, 0)` without a device. +#[cfg(feature = "cuda")] +pub fn stream_draws() -> (u64, u64) { + math_cuda::stream_class::draws() +} + +#[cfg(not(feature = "cuda"))] +pub fn stream_draws() -> (u64, u64) { + (0, 0) +} + +/// The argument's device fallbacks so far +/// ([`math_cuda::device::device_fallbacks`]); 0 without a device. +#[cfg(feature = "cuda")] +pub fn argue_device_fallbacks() -> u64 { + math_cuda::device::device_fallbacks() +} + +#[cfg(not(feature = "cuda"))] +pub fn argue_device_fallbacks() -> u64 { + 0 +} + +/// The card's drain-and-trims so far ([`math_cuda::device::trims`]): each +/// waits for every stream; 0 without a device. +#[cfg(feature = "cuda")] +pub fn device_trims() -> u64 { + math_cuda::device::trims() +} + +#[cfg(not(feature = "cuda"))] +pub fn device_trims() -> u64 { + 0 +} + +/// Whether a group's room is sized to the turns it covers — its commits', two +/// in flight, and one chain's opening, as the kernels that run them allocate — +/// rather than one base codeword. `true` unless `LAMBDA_VM_NO_WHIR_ROOM_RESIZE` +/// is set, which promises the codeword as before. +/// +/// One codeword was the right number once: two commits in flight at a fold of +/// four hold (half a codeword, less a node) each. At the production first fold +/// of six a commit holds a quarter, and an opening under the lean rounds and +/// the fused fold `23/64` at the default three rounds (about a fifth at six) — +/// where before them an opening held nearly four codewords against the one it +/// had promised. +pub(crate) fn room_resize() -> bool { + match ROOM_RESIZE_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); + !*OFF.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_WHIR_ROOM_RESIZE").is_some()) + } + } +} + +static ROOM_RESIZE_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_NO_WHIR_ROOM_RESIZE` for the whole process. `None` +/// restores the environment's setting. Not for production callers. +#[doc(hidden)] +pub fn force_room_resize(on: Option) { + ROOM_RESIZE_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// The two turns a stacked group's room covers, in bytes beside the codewords. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RoomTurns { + /// The commits', two in flight ([`math_cuda::whir::commit_transient_bytes`]). + pub commit: u64, + /// One chain's opening at a time, from the kernels that will run it + /// ([`math_cuda::whir_open::opening_transient_bytes`]) — a grouped + /// chain's whole run at once under `GroupFold::Indexed` + /// ([`math_cuda::whir_open::grouped_opening_transient_bytes`]). + pub open: u64, +} + +/// [`RoomTurns`] for a group laid out as `layout`, opened from resident +/// columns or not. Every stacked polynomial of a group has the same variable +/// count, so the widest share list and the most `eq` values over them bound +/// every chain's opening. +#[cfg(feature = "cuda")] +pub fn room_turns_for( + layout: &crate::stacking::StackedLayout, + config: &crate::whir_chain::ChainConfig, + staged: bool, +) -> RoomTurns { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + let n = layout.n_stack(); + let schedule = config.schedule(n); + let first_fold = schedule.first().copied().unwrap_or(0); + let next_fold = schedule.get(1).copied().unwrap_or(0); + let in_flight = layout.num_polys().min(2) as u64; + let commit = + in_flight * math_cuda::whir::commit_transient_bytes(n, config.log_blowup, first_fold); + // A share's `eq` halves are its point's high and low coordinates' tables, + // the low half `num_vars / 2` wide — counted per share, with no credit for + // the neighbours that share a point. + let mut shares = vec![0usize; layout.num_polys()]; + let mut eq_values = vec![0usize; layout.num_polys()]; + for place in layout.placements() { + shares[place.poly] += 1; + let lo_bits = place.num_vars / 2; + eq_values[place.poly] += (1usize << (place.num_vars - lo_bits)) + (1usize << lo_bits); + } + let slots = crate::whir_chain::opening_program::() + .ok() + .and_then(|program| lower(&program)) + .map_or(0, |lowered| lowered.num_slots); + let shape = math_cuda::whir_open::OpeningShape { + num_vars: n, + log_blowup: config.log_blowup, + first_fold, + next_fold, + lean_rounds: lean_rounds(), + fused: fused_fold(), + encoded: encode_successor(), + staged, + shares: shares.iter().copied().max().unwrap_or(0), + eq_values: eq_values.iter().copied().max().unwrap_or(0), + slots, + }; + // A grouped chain holds its whole run's openings at once in round 0: the + // widest run's turn, when it is wider than one. + let widest = crate::stacked_eval::index_chains(layout.num_polys(), config.format.group) + .iter() + .map(|run| run.len()) + .max() + .unwrap_or(1); + let open = math_cuda::whir_open::opening_transient_bytes(&shape).max( + math_cuda::whir_open::grouped_opening_transient_bytes( + &shape, + widest, + crate::whir_chain::index_vars(widest), + ), + ); + RoomTurns { commit, open } +} + +/// No device: nothing is ever promised, so nothing is sized. +#[cfg(not(feature = "cuda"))] +pub fn room_turns_for( + _layout: &crate::stacking::StackedLayout, + _config: &crate::whir_chain::ChainConfig, + _staged: bool, +) -> RoomTurns { + RoomTurns { commit: 0, open: 0 } +} + +/// What a device sumcheck that runs to the end hands back: the round proofs, +/// and the challenges the rounds were bound at. +/// +/// A named type rather than the tuple written out, because the tuple appears in +/// a return position wrapped in a `Result` and reads as punctuation there. The +/// sibling [`ResidentRounds`] carries a third member — the tables left folded +/// at the crossover — for the path that stops early. +#[cfg(feature = "cuda")] +type ClosedRounds = ( + Vec>, + Vec>, +); + +/// A promise held on someone else's behalf. Dropping it gives the room back. +/// +/// The field is never read, and that is the design: it is an RAII guard whose +/// `Drop` returns the reservation, so holding it IS the behaviour. Deleting it +/// to satisfy the lint would delete the promise. +#[cfg(feature = "cuda")] +#[derive(Debug)] +pub struct DeviceRoom(#[allow(dead_code)] math_cuda::device::DeviceReservation); + +/// A promise no device made. Never constructed. +#[cfg(not(feature = "cuda"))] +#[derive(Debug)] +pub struct DeviceRoom(std::convert::Infallible); + +impl DeviceRoom { + /// Gives the promise back now, rather than when its holder drops. + pub(crate) fn give_back(self) {} +} + +/// How much room handing the input layer back has to save before it is worth +/// doing. +/// +/// It cannot fail while it is being carried, and it can fail when it is asked +/// for again — with the transcript already moved and no host path left. A +/// table that saves a few megabytes buys a second chance to fail for nothing; +/// the widest precompiles save gigabytes, and for them it is the difference +/// between a device and the host. +#[cfg(feature = "cuda")] +const WORTH_HANDING_BACK: u64 = 1 << 30; + +/// [`WORTH_HANDING_BACK`], unless a test has set its own. +#[cfg(feature = "cuda")] +fn worth_handing_back() -> u64 { + match HAND_BACK_FOR_TESTS.load(Ordering::Relaxed) { + u64::MAX => WORTH_HANDING_BACK, + bytes => bytes, + } +} + +/// A test's hand-back threshold; `u64::MAX` is none. +static HAND_BACK_FOR_TESTS: AtomicU64 = AtomicU64::new(u64::MAX); + +/// Tests only: the room a tree's carry must save before it is handed back, +/// in place of [`WORTH_HANDING_BACK`]; `None` restores it. At `Some(0)` every +/// refused carry is handed back, which is the only way a table smaller than the +/// widest precompiles reaches the tree's whole-tree promise and its refusal. +/// Process-wide: a test that sets it runs alone. +#[doc(hidden)] +pub fn set_hand_back_threshold_for_tests(bytes: Option) { + HAND_BACK_FOR_TESTS.store(bytes.unwrap_or(u64::MAX), Ordering::Relaxed); +} + +/// Tests only: every tree built from here on hands its input layer back and +/// writes it again for its last sumcheck, as the widest precompiles do when the +/// card cannot carry it — so a test can reach the rewrite on any table. +/// Process-wide: a test that sets it runs alone. +static HAND_BACK_FORCED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +#[doc(hidden)] +pub fn force_hand_back_for_tests(on: bool) { + HAND_BACK_FORCED.store(on, Ordering::Relaxed); +} + +/// Cells — factors times rows — below which the host wins. +/// +/// The precompiles are short and very wide: a thousand factors over four +/// thousand rows is four million cells, and a gate that reads the row count +/// alone calls that small. It is not, and it is the shape where the host costs +/// the most, so what the gate has to read is the work. +#[cfg(feature = "cuda")] +const DEVICE_CELLS: usize = 1 << 12; + +/// A cube too short to fill a few warps is a round trip for nothing, however +/// wide the table is. +#[cfg(feature = "cuda")] +const MIN_CUBE: usize = 1 << 6; + +/// Whether a table of `width` factors over a cube of `len` is worth a device. +#[cfg(feature = "cuda")] +fn worth_the_device(width: usize, len: usize) -> bool { + len >= MIN_CUBE && width.saturating_mul(len) >= DEVICE_CELLS +} + +/// Assigns every step a slot, reusing the slot of a value whose last read has +/// passed. +/// +/// This is what makes the kernel possible at all: the precompile tables compile +/// to tens of thousands of steps, and a slot per step would be a megabyte of +/// scratch per thread. +pub fn lower(program: &crate::program::Program) -> Option +where + E: math::field::traits::IsField + 'static, +{ + use crate::program::Op; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + // The blob is ext3: the kernel reads three limbs per value, whether or not + // this particular program happens to hold a constant that would say so. + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + let steps = program.steps(); + // Last step that reads each value; the root is read by the caller, so it is + // never freed. + let mut last_use: Vec = vec![0; steps.len()]; + for (i, step) in steps.iter().enumerate() { + let mut mark = |operand: u32| last_use[operand as usize] = i; + match *step { + Op::Fixed(_) | Op::Var(_) => {} + Op::Neg(a) => mark(a), + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => { + mark(a); + mark(b); + } + } + } + last_use[program.root() as usize] = usize::MAX; + + let mut slot_of: Vec = vec![u32::MAX; steps.len()]; + let mut free: Vec = Vec::new(); + let mut num_slots = 0usize; + let mut nodes: Vec = Vec::with_capacity(steps.len() * 2); + let mut consts: Vec = Vec::new(); + + for (i, step) in steps.iter().enumerate() { + let (op, a, b, operands) = match *step { + Op::Fixed(ref value) => { + let at = consts.len() / 3; + consts.extend_from_slice(&ext3_raw(value)?); + (op::FIXED, at as u32, 0, [None, None]) + } + Op::Var(slot) => (op::VAR, slot, 0, [None, None]), + Op::Neg(a) => (op::NEG, slot_of[a as usize], 0, [Some(a), None]), + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => { + let tag = match *step { + Op::Add(..) => op::ADD, + Op::Sub(..) => op::SUB, + _ => op::MUL, + }; + ( + tag, + slot_of[a as usize], + slot_of[b as usize], + [Some(a), Some(b)], + ) + } + }; + // Freed before the result is allocated: the kernel loads both operands + // before it stores, so the result may take a slot this step frees. A + // value read twice — `x·x`, which is what a squaring compiles to — + // frees its slot once, or two later values would be handed the same + // one. + if let Some(x) = operands[0].filter(|x| last_use[*x as usize] == i) { + free.push(slot_of[x as usize]); + } + if let Some(y) = + operands[1].filter(|y| last_use[*y as usize] == i && operands[0] != Some(*y)) + { + free.push(slot_of[y as usize]); + } + let res = free.pop().unwrap_or_else(|| { + let slot = num_slots as u32; + num_slots += 1; + slot + }); + if num_slots > MAX_SLOTS { + return None; + } + slot_of[i] = res; + nodes.push(u64::from(op) | (u64::from(a) << 32)); + nodes.push(u64::from(b) | (u64::from(res) << 32)); + } + + Some(Lowered { + nodes, + consts, + num_slots, + root_slot: slot_of[program.root() as usize], + }) +} + +/// An ext3 element's three limbs, or `None` when `E` is not that field. +pub fn ext3_raw(value: &math::field::element::FieldElement) -> Option<[u64; 3]> +where + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + // SAFETY: `E == Ext3`, whose `FieldElement` is a transparent wrapper over + // three Goldilocks limbs, each transparent over its `u64`. + let limbs = unsafe { *(value as *const _ as *const [u64; 3]) }; + Some(limbs) +} + +/// A Goldilocks limb as the card left it, reduced to `[0, p)`. +/// +/// ★ The card's arithmetic leaves values anywhere in `[0, 2^64)`, and a proof +/// serializes the representative it holds: `FieldElement`'s rkyv form is the +/// raw `u64` (`math::field::element`), unlike its hashed and absorbed bytes, +/// which are canonical. Two kernels computing the same value by different +/// operations — the lean rounds and the materialised session, an encoded and +/// a folded successor — can leave different representatives, and the proof's +/// bytes would show which one ran. Every limb read back from the card is +/// reduced here, so they cannot. +#[inline] +pub(crate) fn canonical_limb(limb: u64) -> u64 { + use math::field::goldilocks::GOLDILOCKS_PRIME; + if limb >= GOLDILOCKS_PRIME { + limb - GOLDILOCKS_PRIME + } else { + limb + } +} + +/// Rebuilds an ext3 element from its three limbs, canonical +/// ([`canonical_limb`]). +pub fn ext3_from_raw(limbs: &[u64]) -> math::field::element::FieldElement +where + E: math::field::traits::IsField + 'static, +{ + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + + let value = FieldElement::::new([ + FieldElement::::from_raw(canonical_limb(limbs[0])), + FieldElement::::from_raw(canonical_limb(limbs[1])), + FieldElement::::from_raw(canonical_limb(limbs[2])), + ]); + // SAFETY: only called under a TypeId check that `E == Ext3`. + unsafe { core::mem::transmute_copy::, FieldElement>(&value) } +} + +/// Runs a batched sumcheck's rounds on device. +/// +/// `challenge` absorbs a round's evaluations and returns the challenge drawn +/// from them, which is the whole of the host's part: the transcript is +/// sequential by definition and stays where it is. +/// +/// The factors are consumed — they are left folded on device and dropped — so +/// this is for a caller that wants the rounds and the point, not the tables. +/// The prover's `g(0) + g(1)` self-check does not run on this path: it costs +/// the extra interpolation node the protocol exists to skip. +/// +/// `None` means the device declined **before the transcript moved**, and the +/// host path runs. `Some(Err)` means it failed after: the challenges are drawn, +/// the transcript cannot be rewound, and the proof fails rather than being +/// finished from a state the verifier will not reproduce. +#[cfg(feature = "cuda")] +pub(crate) fn prove_sumcheck( + polys: &[crate::mle::Mle], + program: &crate::program::Program, + degree: usize, + host_cube: usize, + challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, +) -> Option, crate::Error>> +where + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + let first = polys.first()?; + let num_vars = first.num_vars(); + if !worth_the_device(polys.len(), first.len()) || num_vars == 0 { + return None; + } + if degree == 0 || degree > math_cuda::sumcheck::MAX_NODES { + return None; + } + if polys.iter().any(|p| p.len() != first.len()) { + return None; + } + // A slot past the factor list would be an out-of-bounds device read, which + // no kernel can check for itself. + if program + .max_slot() + .is_some_and(|slot| slot as usize >= polys.len()) + { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_SUMCHECK").is_some()) { + return None; + } + let lowered = lower(program)?; + + // SAFETY: `E == Ext3` is established above, and its `FieldElement` is a + // transparent wrapper over three `u64` limbs — the layout the kernel reads. + let raw: Vec<&[u64]> = polys + .iter() + .map(|p| unsafe { + core::slice::from_raw_parts(p.evals().as_ptr() as *const u64, p.len() * 3) + }) + .collect(); + + let mut session = math_cuda::sumcheck::SumcheckSession::new( + &raw, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .ok()?; + + // Diagnostic hook: recompute each round on the host from the factors the + // device holds and stop at the first disagreement, naming the round. A + // device round that differs otherwise surfaces as a proof that does not + // verify, minutes and 55 tables later. + static XCHECK: std::sync::OnceLock = std::sync::OnceLock::new(); + let xcheck = *XCHECK.get_or_init(|| std::env::var_os("LAMBDA_VM_GPU_XCHECK").is_some()); + let reference = |session: &math_cuda::sumcheck::SumcheckSession| { + // A session over factors that were already there has no layout to read + // back, so there is nothing to rebuild the host round from. + if !xcheck || !session.can_download() { + return None; + } + let tables = session.download().expect("the device holds its factors"); + let factors: Vec> = tables + .iter() + .map(|table| { + crate::mle::Mle::new(table.chunks_exact(3).map(ext3_from_raw::).collect()) + }) + .collect::>() + .expect("the device holds power-of-two tables"); + Some( + crate::sumcheck::round_evaluations_for_program(&factors, program, degree) + .expect("the host round"), + ) + }; + + // The rounds stop where the cube reaches the crossover: past it a round is + // one thread walking the whole program, and a core here walks it far + // faster. The caller carries on from the factors this leaves folded. + let there = num_vars.saturating_sub(host_cube.max(1).trailing_zeros() as usize); + let outcome = run_rounds(&mut session, degree, there, challenge, reference); + let (rounds, challenges) = match outcome { + Ok(rounds) => rounds, + Err(error) => return Some(Err(error)), + }; + // Each factor as the last fold left it, read where it lies — which is the + // only thing a session over factors it does not own can say. + let Ok(values) = session.values() else { + return Some(Err(crate::Error::DeviceFailed { stage: "download" })); + }; + let folded: Result>, crate::Error> = values + .iter() + .map(|factor| { + crate::mle::Mle::new(factor.chunks_exact(3).map(ext3_from_raw::).collect()) + }) + .collect(); + let Ok(folded) = folded else { + return Some(Err(crate::Error::DeviceFailed { + stage: "folded tables", + })); + }; + SUMCHECK_CALLS.fetch_add(1, Ordering::Relaxed); + Some(Ok((rounds, challenges, folded))) +} + +/// The round loop: the device sums over the cube, the host draws the challenge +/// from what it sent, the device binds it. +/// +/// Past the first round the transcript has moved, so every failure here is an +/// error — there is no going back to the host path. +#[cfg(feature = "cuda")] +fn run_rounds( + session: &mut math_cuda::sumcheck::SumcheckSession, + degree: usize, + num_vars: usize, + challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + reference: impl FnMut( + &math_cuda::sumcheck::SumcheckSession, + ) -> Option>>, +) -> Result, crate::Error> +where + E: math::field::traits::IsField + 'static, +{ + run_rounds_timed(session, degree, num_vars, challenge, reference, None) +} + +/// Where a zerocheck's device rounds go on the base split's `ARGUE ZEROCHECK` +/// line: a big batch's split at [`whir_split::LATE_HALF`](crate::whir_split::LATE_HALF) +/// into early and late rounds, every other batch's together. +#[cfg(feature = "cuda")] +#[derive(Clone, Copy)] +enum ZerocheckRounds { + Big, + Other, +} + +#[cfg(feature = "cuda")] +impl ZerocheckRounds { + fn note(self, half: usize, start: Option) { + use crate::whir_split::{self, add_tick, bump}; + match self { + Self::Big if half >= whir_split::LATE_HALF => { + add_tick(&whir_split::ZC_BIG_EARLY, start); + bump(&whir_split::ZC_BIG_EARLY_ROUNDS); + } + Self::Big => { + add_tick(&whir_split::ZC_BIG_LATE, start); + bump(&whir_split::ZC_BIG_LATE_ROUNDS); + } + Self::Other => add_tick(&whir_split::ZC_OTHER, start), + } + } +} + +/// Today's program walking a zerocheck's factors beside the program on demand +/// (`LAMBDA_VM_ARGUE_XCHECK`): each round, the shadow's sums are taken before +/// the leader's round and compared with its answer before the challenge is +/// drawn. Inert without a shadow. +#[cfg(feature = "cuda")] +struct ShadowRounds { + shadow: std::cell::RefCell>, + /// The interpolation nodes, as the round loop sends them. + t: Vec, + /// The shadow's sums for the round under way; empty if its round failed. + expected: std::cell::RefCell>>, + round: std::cell::Cell, + /// The first round the two disagreed at. + parted: std::cell::Cell>, +} + +#[cfg(feature = "cuda")] +impl ShadowRounds { + fn new(shadow: Option, degree: usize) -> Option { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + let mut t = Vec::with_capacity(degree * 3); + for node in 1..=degree as u64 { + t.extend_from_slice(&ext3_raw( + &math::field::element::FieldElement::::from(node), + )?); + } + Some(Self { + shadow: std::cell::RefCell::new(shadow), + t, + expected: std::cell::RefCell::new(None), + round: std::cell::Cell::new(0), + parted: std::cell::Cell::new(None), + }) + } + + /// Before the leader's round: the shadow's, over the cube the leader has + /// left. Never a reference for the round loop to assert on. + fn walk( + &self, + leader: &math_cuda::sumcheck::SumcheckSession, + ) -> Option>> + where + E: math::field::traits::IsField + 'static, + { + if let Some(shadow) = self.shadow.borrow_mut().as_mut() { + shadow.follow(leader); + *self.expected.borrow_mut() = Some(shadow.round(&self.t).unwrap_or_default()); + } + None + } + + /// After it: the leader's answer against the shadow's sums. + fn compare(&self, evaluations: &[math::field::element::FieldElement]) + where + E: math::field::traits::IsField + 'static, + { + let round = self.round.replace(self.round.get() + 1); + let Some(sums) = self.expected.borrow_mut().take() else { + return; + }; + let same = sums.len() == evaluations.len() * 3 + && sums + .chunks_exact(3) + .zip(evaluations) + .all(|(sum, value)| ext3_from_raw::(sum) == *value); + if !same && self.parted.get().is_none() { + self.parted.set(Some(round)); + } + } + + fn parted(&self) -> Option { + self.parted.get() + } +} + +/// [`run_rounds`], each round — its kernels, its read-back, the challenge drawn +/// and its fold — timed onto the zerocheck's split when `zerocheck` says which. +#[cfg(feature = "cuda")] +fn run_rounds_timed( + session: &mut math_cuda::sumcheck::SumcheckSession, + degree: usize, + num_vars: usize, + mut challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + mut reference: impl FnMut( + &math_cuda::sumcheck::SumcheckSession, + ) -> Option>>, + zerocheck: Option, +) -> Result, crate::Error> +where + E: math::field::traits::IsField + 'static, +{ + use math::field::element::FieldElement; + + // The interpolation nodes are `1..=degree`: `g(0)` is not sent, the claim + // carried into the round fixes it. + let mut t = Vec::with_capacity(degree * 3); + for node in 1..=degree { + t.extend_from_slice(&ext3_raw(&FieldElement::::from(node as u64)).ok_or( + crate::Error::DeviceFailed { + stage: "interpolation node", + }, + )?); + } + + let failed = |stage| crate::Error::DeviceFailed { stage }; + let mut rounds = Vec::with_capacity(num_vars); + let mut challenges = Vec::with_capacity(num_vars); + for round in 0..num_vars { + let expected = reference(session); + let half = session.len() / 2; + let start = zerocheck.and_then(|_| crate::whir_split::tick()); + let sums = session.round(&t).map_err(|_| failed("round"))?; + let evaluations: Vec> = + sums.chunks_exact(3).map(ext3_from_raw::).collect(); + if let Some(expected) = expected { + assert_eq!( + evaluations, expected, + "the device round {round} differs from the host" + ); + } + let r = challenge(&evaluations); + let raw = ext3_raw(&r).ok_or_else(|| failed("challenge"))?; + session.fold(&raw).map_err(|_| failed("fold"))?; + if let Some(zerocheck) = zerocheck { + zerocheck.note(half, start); + } + rounds.push(crate::sumcheck::RoundProof { evaluations }); + challenges.push(r); + SUMCHECK_ROUNDS.fetch_add(1, Ordering::Relaxed); + } + Ok((rounds, challenges)) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn prove_sumcheck( + _polys: &[crate::mle::Mle], + _program: &crate::program::Program, + _degree: usize, + _host_cube: usize, + _challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, +) -> Option, crate::Error>> +where + E: math::field::traits::IsField + 'static, +{ + None +} + +/// The same sumcheck, with the trace's factors already on the device and only +/// the weight tables here. +/// +/// The rounds fold the resident factors where they lie, which spends them: a +/// table's argument runs one of these. +/// +/// `None` is a decline, and it is always before the first round — so the +/// caller can still build those factors here and run the host path from the +/// same transcript. Once the rounds start, a failure is a failure. +#[cfg(feature = "cuda")] +pub(crate) fn prove_sumcheck_resident( + resident: &DeviceFactors, + extra: &[WeightView<'_, E>], + program: &crate::program::Program, + degree: usize, + challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, +) -> Option, crate::Error>> +where + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + let len = resident.0.len(); + let num_vars = len.trailing_zeros() as usize; + if !worth_the_device(resident.0.width() + extra.len(), len) || num_vars == 0 { + return None; + } + if degree == 0 || degree > math_cuda::sumcheck::MAX_NODES { + return None; + } + if extra.iter().any(|weight| weight.cells() != len) { + return None; + } + // A slot past the factor list would be an out-of-bounds device read, which + // no kernel can check for itself. + let width = resident.0.width() + extra.len(); + if program + .max_slot() + .is_some_and(|slot| slot as usize >= width) + { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_SUMCHECK").is_some()) { + return None; + } + // A big batch's program on demand holds few enough values that its slot + // file is sized for every node from the first round. + let ZerocheckLowering { + run: lowered, + held, + big, + } = lower_for_zerocheck(program)?; + let lean = held.is_some(); + let spread = if lean { degree } else { 1 }; + + // A table goes up as it is; an `eq` weight goes up as its point, and the + // card builds it (`LAMBDA_VM_ARGUE_DEVICE_TABLES`). + let mut points = Vec::with_capacity(extra.len()); + for weight in extra { + points.push(match weight { + WeightView::Eq(point) => raw_point(point)?, + WeightView::Table(_) => Vec::new(), + }); + } + let raw: Vec> = extra + .iter() + .zip(&points) + .map(|(weight, point)| match weight { + // SAFETY: `E == Ext3` is established above, and its `FieldElement` + // is a transparent wrapper over three `u64` limbs — the layout the + // kernel reads. + WeightView::Table(table) => math_cuda::sumcheck::Extra::Table(unsafe { + core::slice::from_raw_parts(table.evals().as_ptr() as *const u64, table.len() * 3) + }), + WeightView::Eq(_) => math_cuda::sumcheck::Extra::Eq(point), + }) + .collect(); + let mut session = resident + .0 + .session_spread( + &raw, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + spread, + ) + .ok()?; + // Under the cross-check, today's program walks the same factors beside it, + // a round at a time: the block's identity gate for the schedule, as proof + // bytes cannot be one. A shadow the card has no room for declines here, + // before the first round, and is counted as unchecked. + let shadow = match &held { + Some(today) if argue_xcheck() => Some( + session + .shadow( + &today.nodes, + &today.consts, + today.num_slots, + today.root_slot, + ) + .ok()?, + ), + _ => None, + }; + let checked = shadow.is_some(); + let beside = ShadowRounds::new(shadow, degree)?; + if big { + crate::whir_split::bump(&crate::whir_split::ZC_BIG); + } + if lean { + LEAN_PROGRAMS.fetch_add(1, Ordering::Relaxed); + crate::whir_split::bump(&crate::whir_split::ZC_LEAN); + } + + // The weights the card built, by their slot in the session. + let built: Vec<(usize, &[math::field::element::FieldElement])> = extra + .iter() + .enumerate() + .filter_map(|(k, weight)| match weight { + WeightView::Eq(point) => Some((resident.0.width() + k, *point)), + WeightView::Table(_) => None, + }) + .collect(); + if let Some(&(slot, _)) = built.first() { + note_tables_on_card(built.len()); + if table_fault() == Some(TableFault::Eq) { + session.set_cell(slot, 0, &FAULT_CELL).ok()?; + } + } + // Nothing is absorbed until the first round, so a check that cannot read + // the card still declines; one that reads a wrong table fails the proof. + if argue_xcheck() { + for &(slot, point) in &built { + let card = session.factor(slot).ok()?; + if let Some(cell) = first_difference(&card, &crate::eq::eq_evals(point)) { + eprintln!( + "[argue] XCHECK: the card's eq weight in slot {slot} (2^{num_vars} cells) is not \ + the host's at cell {cell}" + ); + return Some(Err(crate::Error::DeviceFailed { stage: "eq weight" })); + } + note_tables_xchecked(1); + } + } + + // The rounds stop where the cube reaches the crossover: past it a round is + // one thread walking the whole program, and a core here walks it far + // faster. The caller finishes from the factors this hands back. + let there = num_vars.saturating_sub(crate::HOST_CUBE_COMPILED.trailing_zeros() as usize); + let mut challenge = challenge; + let outcome = run_rounds_timed( + &mut session, + degree, + there, + |evaluations| { + beside.compare(evaluations); + challenge(evaluations) + }, + |leader| beside.walk(leader), + Some(if big { + ZerocheckRounds::Big + } else { + ZerocheckRounds::Other + }), + ); + let (rounds, challenges) = match outcome { + Ok(rounds) => rounds, + Err(error) => return Some(Err(error)), + }; + if let Some(round) = beside.parted() { + eprintln!( + "[argue] XCHECK: a {num_vars}-variable zerocheck's rounds on demand ({} values a \ + thread) parted from today's program ({} values) at round {round}", + lowered.num_slots, + held.as_ref().map_or(0, |today| today.num_slots), + ); + return Some(Err(crate::Error::DeviceFailed { + stage: "program on demand", + })); + } + if checked { + LEAN_PROGRAM_XCHECKS.fetch_add(1, Ordering::Relaxed); + crate::whir_split::bump(&crate::whir_split::ZC_LEAN_XCHECKED); + } + // The rounds folded every factor where it lies, so reading them back is + // what spares the caller a pass over the trace to compute what the device + // already has — the values at the point, when the cube ran out here. + let Ok(values) = session.values() else { + return Some(Err(crate::Error::DeviceFailed { + stage: "factor values", + })); + }; + let factors: Result>, crate::Error> = values + .iter() + .map(|factor| { + crate::mle::Mle::new(factor.chunks_exact(3).map(ext3_from_raw::).collect()) + }) + .collect(); + let Ok(factors) = factors else { + return Some(Err(crate::Error::DeviceFailed { + stage: "factor values", + })); + }; + SUMCHECK_CALLS.fetch_add(1, Ordering::Relaxed); + Some(Ok((rounds, challenges, factors))) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn prove_sumcheck_resident( + _resident: &DeviceFactors, + _extra: &[WeightView<'_, E>], + _program: &crate::program::Program, + _degree: usize, + _challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, +) -> Option, crate::Error>> +where + E: math::field::traits::IsField + 'static, +{ + None +} + +/// The claim reduce's sumcheck with its tables built on the card +/// (`LAMBDA_VM_ARGUE_DEVICE_TABLES`): `eq(alpha)` once, each offset's shift +/// table a rotated copy of it, each offset's batched column out of the +/// epoch's resident columns (`math_cuda::sumcheck::reduce_session`). +/// +/// The gates are [`prove_sumcheck`]'s over the same `2 · offsets` tables, so the +/// card takes exactly the reductions it took with the tables uploaded, when +/// the table's columns are a resident run. The rounds are [`prove_sumcheck`]'s +/// too, down to the same crossover, and the tables come back folded to it for +/// the caller to finish. +/// +/// `None` is a decline before the transcript moved: the caller builds the +/// tables on the host, as before. +#[cfg(feature = "cuda")] +#[allow(clippy::too_many_arguments)] +pub(crate) fn prove_reduce_resident( + columns: &C, + sources: &[crate::claim_reduce::FactorSource], + weights: &[math::field::element::FieldElement], + offsets: &[usize], + alpha: &[math::field::element::FieldElement], + resident: Option<(&ResidentColumns, usize)>, + program: &crate::program::Program, + challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, +) -> Option, crate::Error>> +where + F: math::field::traits::IsField + math::field::traits::IsSubFieldOf + 'static, + E: math::field::traits::IsField + 'static, + C: crate::narrow::HostColumns + ?Sized, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + let (store, first) = resident?; + if !store.0.is_run(first, columns.width()) { + return None; + } + let num_vars = alpha.len(); + if num_vars == 0 || num_vars >= usize::BITS as usize { + return None; + } + let len = 1usize << num_vars; + if columns.rows() != Some(len) { + return None; + } + // The reduce's rule is a sum of kernel-times-column pairs: degree two. + let degree = 2; + let width = 2 * offsets.len(); + if offsets.is_empty() || !worth_the_device(width, len) { + return None; + } + if program + .max_slot() + .is_some_and(|slot| slot as usize >= width) + { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_SUMCHECK").is_some()) { + return None; + } + let lowered = lower(program)?; + let raw_alpha = raw_point(alpha)?; + // Per offset, every source reading at it: its column in the table's run + // and its batching weight — `claim_reduce::batched_column`'s terms. + let mut groups = Vec::with_capacity(offsets.len()); + for &offset in offsets { + let mut members = Vec::new(); + let mut member_weights = Vec::new(); + for (source, weight) in sources.iter().zip(weights) { + if source.offset == offset { + members.push(source.column as u64); + member_weights.extend_from_slice(&ext3_raw(weight)?); + } + } + groups.push((offset, members, member_weights)); + } + let mut session = math_cuda::sumcheck::reduce_session( + &store.0, + first, + columns.width(), + &raw_alpha, + &groups, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .ok()?; + note_tables_on_card(width); + match table_fault() { + Some(TableFault::Eq) => session.set_cell(0, 0, &FAULT_CELL).ok()?, + Some(TableFault::Batched) => session.set_cell(1, 0, &FAULT_CELL).ok()?, + None => {} + } + // Nothing is absorbed until the first round, so a check that cannot read + // the card still declines; one that reads a wrong table fails the proof. + if argue_xcheck() { + for (i, &offset) in offsets.iter().enumerate() { + let shift = crate::eq::shift_evals(alpha, offset); + let batched = match crate::claim_reduce::batched_column( + columns.host(), + sources, + weights, + offset, + num_vars, + ) { + Ok(batched) => batched, + Err(error) => return Some(Err(error)), + }; + for (slot, host, what) in [ + (2 * i, &shift[..], "shift table"), + (2 * i + 1, batched.evals(), "batched column"), + ] { + let card = session.factor(slot).ok()?; + if let Some(cell) = first_difference(&card, host) { + eprintln!( + "[argue] XCHECK: the card's {what} for offset {offset} (2^{num_vars} cells) \ + is not the host's at cell {cell}" + ); + return Some(Err(crate::Error::DeviceFailed { + stage: "reduce tables", + })); + } + note_tables_xchecked(1); + } + } + } + + // The same crossover `prove_sumcheck` stops at for the reduce's + // polynomial, whose rule the host walks through the program interpreter. + let there = num_vars.saturating_sub(crate::HOST_CUBE_COMPILED.trailing_zeros() as usize); + let outcome = run_rounds(&mut session, degree, there, challenge, |_| None); + let (rounds, challenges) = match outcome { + Ok(rounds) => rounds, + Err(error) => return Some(Err(error)), + }; + let Ok(values) = session.values() else { + return Some(Err(crate::Error::DeviceFailed { stage: "download" })); + }; + let folded: Result>, crate::Error> = values + .iter() + .map(|table| crate::mle::Mle::new(table.chunks_exact(3).map(ext3_from_raw::).collect())) + .collect(); + let Ok(folded) = folded else { + return Some(Err(crate::Error::DeviceFailed { + stage: "folded tables", + })); + }; + SUMCHECK_CALLS.fetch_add(1, Ordering::Relaxed); + Some(Ok((rounds, challenges, folded))) +} + +#[cfg(not(feature = "cuda"))] +#[allow(clippy::too_many_arguments)] +pub(crate) fn prove_reduce_resident( + _columns: &C, + _sources: &[crate::claim_reduce::FactorSource], + _weights: &[math::field::element::FieldElement], + _offsets: &[usize], + _alpha: &[math::field::element::FieldElement], + _resident: Option<(&ResidentColumns, usize)>, + _program: &crate::program::Program, + _challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, +) -> Option, crate::Error>> +where + F: math::field::traits::IsField + math::field::traits::IsSubFieldOf + 'static, + E: math::field::traits::IsField + 'static, + C: crate::narrow::HostColumns + ?Sized, +{ + None +} + +/// Codeword size below which the host fold wins: one launch per level plus the +/// round trip, against a pass a few cores finish in microseconds. +#[cfg(feature = "cuda")] +const FOLD_THRESHOLD: usize = 1 << 16; + +/// Folds a codeword `alphas.len()` times on device. +/// +/// `generator` is the fold domain's, and the domain squares with every level — +/// the kernel takes each level's inverse generator, which is this one's inverse +/// squared level by level. +#[cfg(feature = "cuda")] +pub(crate) fn fold_codeword_k( + codeword: &[math::field::element::FieldElement], + generator: &math::field::element::FieldElement, + alphas: &[math::field::element::FieldElement], +) -> Option>> +where + F: math::field::traits::IsField + 'static, + C: math::field::traits::IsField + 'static, + N: math::field::traits::IsField + 'static, +{ + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + let base = TypeId::of::() == TypeId::of::(); + if !base && TypeId::of::() != TypeId::of::() { + return None; + } + if alphas.is_empty() || codeword.len() < FOLD_THRESHOLD { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if HOST_WHIR_COMMIT_FORCED.load(Ordering::Relaxed) + || *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_WHIR_FOLD").is_some()) + { + return None; + } + + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let generator = unsafe { *(generator as *const _ as *const u64) }; + let generator = FieldElement::::from_raw(generator); + let two = FieldElement::::from(2u64); + let two_inv = *two.inv().ok()?.value(); + let mut g_inv = generator.inv().ok()?; + let mut g_invs = Vec::with_capacity(alphas.len()); + for _ in 0..alphas.len() { + g_invs.push(*g_inv.value()); + g_inv = g_inv.square(); + } + + let mut raw_alphas = Vec::with_capacity(alphas.len() * 3); + for alpha in alphas { + raw_alphas.extend_from_slice(&ext3_raw(alpha)?); + } + + // SAFETY: `C` is one of the two fields checked above, and both wrap their + // limbs transparently — one `u64` per base element, three per ext3. + let limbs = if base { 1 } else { 3 }; + let raw = unsafe { + core::slice::from_raw_parts(codeword.as_ptr() as *const u64, codeword.len() * limbs) + }; + let mut folded = if base { + math_cuda::whir::fold_codeword_base(raw, two_inv, &g_invs, &raw_alphas).ok()? + } else { + math_cuda::whir::fold_codeword_ext3(raw, two_inv, &g_invs, &raw_alphas).ok()? + }; + + // Canonical, as every value read back from the card ([`canonical_limb`]). + for limb in folded.iter_mut() { + *limb = canonical_limb(*limb); + } + // SAFETY: `N == Ext3`, three limbs per element and no drop glue, so the + // allocation changes type in place instead of being copied. + let mut folded = core::mem::ManuallyDrop::new(folded); + Some(unsafe { + Vec::from_raw_parts( + folded.as_mut_ptr() as *mut FieldElement, + folded.len() / 3, + folded.capacity() / 3, + ) + }) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn fold_codeword_k( + _codeword: &[math::field::element::FieldElement], + _generator: &math::field::element::FieldElement, + _alphas: &[math::field::element::FieldElement], +) -> Option>> +where + F: math::field::traits::IsField + 'static, + C: math::field::traits::IsField + 'static, + N: math::field::traits::IsField + 'static, +{ + None +} + +/// Merkle-commits an ext3 codeword's fold blocks on device, returning the tree +/// in the host node layout. A device error is logged and counted in +/// [`tree_commit_errors`] before the `None` that sends the caller to the host. +#[cfg(feature = "cuda")] +pub(crate) fn commit_tree_ext3( + codeword: &[math::field::element::FieldElement], + log_folding: usize, + hash: crate::whir_hash::DeviceHashKey, +) -> Option> +where + F: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + if codeword.len() < COMMIT_THRESHOLD || codeword.len() >> log_folding < 2 { + return None; + } + if whir_commit_declined() { + return None; + } + // No device is a decline, not an error (see `commit_parts`). + math_cuda::device::backend().ok()?; + // SAFETY: `F == Ext3`, three transparent `u64` limbs per element. + let raw = + unsafe { core::slice::from_raw_parts(codeword.as_ptr() as *const u64, codeword.len() * 3) }; + let nodes = match math_cuda::whir::commit_codeword_ext3(raw, log_folding, hash.into_math_cuda()) + { + Ok(nodes) => nodes, + Err(err) => { + note_tree_commit_error(codeword.len().trailing_zeros(), log_folding, &err); + return None; + } + }; + let nodes = nodes_in_place(nodes)?; + COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); + Some(nodes) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn commit_tree_ext3( + _codeword: &[math::field::element::FieldElement], + _log_folding: usize, + _hash: crate::whir_hash::DeviceHashKey, +) -> Option> +where + F: math::field::traits::IsField + 'static, +{ + None +} + +/// Table size below which the host evaluation wins: one launch per variable +/// against a couple of passes a few cores finish in microseconds. +#[cfg(feature = "cuda")] +const EVALUATE_THRESHOLD: usize = 1 << 16; + +/// Whether the claim reduce evaluates a resident table's columns on the card +/// once the TABLE holds [`EVALUATE_THRESHOLD`] cells, rather than only once +/// each column is that tall. On by default; `LAMBDA_VM_ARGUE_DEVICE_COLUMNS=0` +/// is the opt-out, and it is the old path exactly. +/// +/// The default was turned on by its A/B on the block (FAST, 2026-09-28): the +/// whole run 1.05 s faster, the columns the host walked 24,195 → 3,478, every +/// arm proved and verified on the record's identities. +/// +/// # Why the height is the wrong measure for a resident table +/// +/// The threshold was set for columns that had to be uploaded first, where a +/// short column's upload costs more than the host loop it replaces. Columns +/// the epoch already put on the card cost no upload, and the launches are one +/// per variable for the whole table, so what the host would spend is the +/// table's CELLS: the widest precompile — 1,480 columns of 2^15 rows — is a +/// 48-million-cell walk the host did one column after another, 263–303 ms of a +/// single thread per table, while the card reads it in well under a +/// millisecond (`thoughts/zf/gap2/fix2/D-GFS-NOTE.md` §1.2, G2). +/// +/// The values are the same either way — each is the column's multilinear +/// extension at the point, in exact arithmetic — so the proof is too. +pub fn argue_device_columns() -> bool { + match ARGUE_DEVICE_COLUMNS_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| env_not_off("LAMBDA_VM_ARGUE_DEVICE_COLUMNS")) + } + } +} + +static ARGUE_DEVICE_COLUMNS_FORCED: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_ARGUE_DEVICE_COLUMNS` for the whole process — for a +/// test that proves both ways in one binary. `None` restores the environment's +/// setting. Not for production callers. +#[doc(hidden)] +pub fn force_argue_device_columns(on: Option) { + ARGUE_DEVICE_COLUMNS_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Whether every column value the card computes for the claim reduce is +/// recomputed on the host and compared — `LAMBDA_VM_ARGUE_XCHECK=1`. A +/// diagnostic for an UNTIMED gate run: it puts the host walk back. +/// +/// ⛔ It is the block's identity gate, because proof bytes are not one. Two +/// proves of the same block never share their bytes: six table builders lay +/// their rows out in `HashMap` order (`prover/src/tables/eq.rs` and five +/// others), so every root and every challenge after it differs between two +/// processes whatever this file does. What can be compared is each value the +/// card produced against the host's, in the same process, at the moment it is +/// produced — which is this. +pub fn argue_xcheck() -> bool { + match ARGUE_XCHECK_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| env_on("LAMBDA_VM_ARGUE_XCHECK")) + } + } +} + +static ARGUE_XCHECK_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_ARGUE_XCHECK` for the whole process. `None` restores +/// the environment's setting. Not for production callers. +#[doc(hidden)] +pub fn force_argue_xcheck(on: Option) { + ARGUE_XCHECK_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// `name` set to anything but empty or `0`. +fn env_on(name: &str) -> bool { + std::env::var(name).is_ok_and(|v| !v.is_empty() && v != "0") +} + +/// `name` unset or set to anything but `0`: a knob that is on by default. +fn env_not_off(name: &str) -> bool { + not_off(std::env::var(name).ok().as_deref()) +} + +/// A default-on knob's reading of its variable: only `0` turns it off. +fn not_off(value: Option<&str>) -> bool { + value != Some("0") +} + +/// Whether the argument's challenge tables are built on the card: the +/// zerocheck's weights `eq(r)` and `eq(row)` from their points, and the claim +/// reduce's shift tables and batched columns from `alpha` and the resident +/// columns. On by default; `LAMBDA_VM_ARGUE_DEVICE_TABLES=0` is the opt-out, and +/// it is the old path exactly. +/// +/// The default was turned on by its A/B on the block (FAST, 2026-09-29): the +/// whole run 5.20 s faster, the argue 19.7 → 14.7 s, 1,364 tables built on the +/// card a run, every arm proved and verified on the record's identities, and +/// the cross-check arm — every card table compared with the host's — clean. +/// +/// # Why +/// +/// Off, both are host tables. They are built on the pool, and the ARGUE thread +/// waits in the join — 3.56 s of it on the head's trace, where the producer's +/// prep holds the pool at the same time — and then they cross as pageable +/// uploads, 20 GB a block (`thoughts/zf/gap2/fix2/I-GFS.md` §6). On the card +/// they are the same values, made from a few kilobytes: the points, the columns +/// each offset reads, and their weights. +pub fn argue_device_tables() -> bool { + match ARGUE_DEVICE_TABLES_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| env_not_off("LAMBDA_VM_ARGUE_DEVICE_TABLES")) + } + } +} + +static ARGUE_DEVICE_TABLES_FORCED: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_ARGUE_DEVICE_TABLES` for the whole process — for a test +/// that proves both ways in one binary. `None` restores the environment's +/// setting. Not for production callers. +#[doc(hidden)] +pub fn force_argue_device_tables(on: Option) { + ARGUE_DEVICE_TABLES_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Whether a table's GKR input layer is written straight from its resident +/// base columns (D-BATCH M1-2): one launch for every interaction, base × ext3, +/// where the old path runs two programs an interaction over the lifted factors. +/// The same cells, so the same tree and proof. On by default; +/// `LAMBDA_VM_ARGUE_GKR_INPUT=0` is the old path exactly. Read once, with a +/// banner. +/// +/// Turned on by its A/B on the block (FAST job 333, 4 + 4 arms): base −0.75 s, +/// the argue −0.97 s, the whole run −0.73 s, the tree region 1.98 → 1.11 s, +/// every arm on the record's program ids. +pub fn argue_gkr_input() -> bool { + match ARGUE_GKR_INPUT_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + let on = env_not_off("LAMBDA_VM_ARGUE_GKR_INPUT"); + eprintln!( + "★ ARGUE GKR INPUT: {}", + if on { + "from the base columns (the default; LAMBDA_VM_ARGUE_GKR_INPUT=0 is the lifted \ + factors)" + } else { + "from the lifted factors (LAMBDA_VM_ARGUE_GKR_INPUT=0)" + } + ); + on + }) + } + } +} + +static ARGUE_GKR_INPUT_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Whether a table's factors stay base columns on the card, with no lift +/// (D-BATCH M1-2, its second half). The input layer is written from the columns +/// ([`argue_gkr_input`], which this needs) and the fused zerocheck's first pass +/// reads them where they lie, so the `W·rows·24 B` lift is made only when the +/// old rounds need it. The same rounds and proof. On by default; +/// `LAMBDA_VM_ARGUE_NO_LIFT=0` lifts every table's factors as before. +/// +/// Turned on by its A/B on the block (FAST job 335, 4 + 4 arms over the input +/// from the columns): base −0.90 s, the whole run −1.55 s, the argue's +/// reserved peak −1.5 GiB, kept WHIR trees evicted 13 → 2 a run and the +/// openings' tree rebuilds 0.74 → 0.04 s, every arm on the record's program +/// ids. +pub fn argue_no_lift() -> bool { + match ARGUE_NO_LIFT_FORCED.load(Ordering::Relaxed) { + 1 => argue_gkr_input(), + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + let wanted = env_not_off("LAMBDA_VM_ARGUE_NO_LIFT"); + let input = argue_gkr_input(); + eprintln!( + "★ ARGUE NO LIFT: {}", + match (wanted, input) { + (true, true) => { + "on (the default, with the input from the columns; \ + LAMBDA_VM_ARGUE_NO_LIFT=0 lifts the factors)" + } + (false, _) => "off (LAMBDA_VM_ARGUE_NO_LIFT=0)", + (true, false) => "off (it needs the input from the columns)", + } + ); + wanted && input + }) + } + } +} + +static ARGUE_NO_LIFT_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_ARGUE_NO_LIFT` for the whole process; `None` restores it. +#[doc(hidden)] +pub fn force_argue_no_lift(on: Option) { + ARGUE_NO_LIFT_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Overrides `LAMBDA_VM_ARGUE_GKR_INPUT` for the whole process — for a test +/// that proves both ways in one binary. `None` restores the environment's. +#[doc(hidden)] +pub fn force_argue_gkr_input(on: Option) { + ARGUE_GKR_INPUT_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Whether a big batch's device rounds walk its program on demand +/// ([`Program::on_demand`](crate::program::Program::on_demand)), with the slot +/// file sized for every interpolation node from the first round. The same +/// round values either way. On by default; `LAMBDA_VM_ARGUE_LEAN_PROGRAM=0` is +/// the opt-out, and it is the old path exactly. +/// +/// The default was turned on by its A/B on the block (FAST, 2026-09-29): the +/// whole run 1.35 s faster, the big batches' early device rounds 1,527 → 442 ms +/// and the argue 1.43 s faster, every arm proved and verified on the record's +/// identities, and the cross-check arm — every big session's rounds compared, +/// round by round, with today's program over the same factors — clean. +/// +/// A batch is big when the values its lowered program holds a thread, more +/// than [`LEAN_ABOVE_SLOTS`], leave a round fewer than 64 k threads: the head's +/// widest ran its first rounds on 8,096 threads, at 94 ms a launch. What fills +/// that slot file is the order the batch was written in — every read held from +/// its first use to its last, every root and every interaction's side held +/// until the sum at the end. On demand the widest holds 207 values instead of +/// 2,763, and the four big batches 14 to 207 (`thoughts/zf/gap2/fix2/I-GFS.md` +/// §16). +pub fn argue_lean_program() -> bool { + match ARGUE_LEAN_PROGRAM_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| env_not_off("LAMBDA_VM_ARGUE_LEAN_PROGRAM")) + } + } +} + +static ARGUE_LEAN_PROGRAM_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_ARGUE_LEAN_PROGRAM` for the whole process — for a test +/// that proves both ways in one binary. `None` restores the environment's +/// setting. Not for production callers. +#[doc(hidden)] +pub fn force_argue_lean_program(on: Option) { + ARGUE_LEAN_PROGRAM_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// A batch whose lowered program holds more values a thread than this is +/// big: at 342 or more, the round kernel's slot budget leaves fewer than 64 k +/// threads (`math_cuda::sumcheck::thread_ceiling`; pinned by a test). The VM's +/// four big batches hold 859 to 2,763; the next widest, 117. +pub const LEAN_ABOVE_SLOTS: usize = 341; + +/// A test's own gate in place of [`LEAN_ABOVE_SLOTS`] (`u64::MAX` = none), +/// so a fixture whose batches are small can still take the lean path. +static LEAN_GATE_FORCED: AtomicU64 = AtomicU64::new(u64::MAX); + +#[doc(hidden)] +pub fn force_lean_program_gate(above_slots: Option) { + LEAN_GATE_FORCED.store( + above_slots.map_or(u64::MAX, |slots| slots as u64), + Ordering::Relaxed, + ); +} + +/// Whether a program holding `num_slots` values a thread is a big batch. +pub fn is_big_batch(num_slots: usize) -> bool { + match LEAN_GATE_FORCED.load(Ordering::Relaxed) { + u64::MAX => num_slots > LEAN_ABOVE_SLOTS, + forced => num_slots as u64 > forced, + } +} + +/// ⛔ A FAULT: while armed, every constant of a program run on demand is off by +/// one in its first limb — what a schedule that read wrong values would do — so +/// the checks after it can be shown to see it. Never armed outside a test. +static LEAN_PROGRAM_FAULT: core::sync::atomic::AtomicBool = + core::sync::atomic::AtomicBool::new(false); + +#[doc(hidden)] +pub fn force_lean_program_fault(on: bool) { + LEAN_PROGRAM_FAULT.store(on, Ordering::Relaxed); +} + +/// What a zerocheck's device rounds walk. +#[cfg(feature = "cuda")] +struct ZerocheckLowering { + /// The program the rounds run, lowered. + run: Lowered, + /// Today's lowering, when `run` is the same program on demand in its place. + held: Option, + /// Whether the batch is big ([`is_big_batch`]). + big: bool, +} + +/// `program` lowered — or, under `LAMBDA_VM_ARGUE_LEAN_PROGRAM` for a big batch, +/// the same program on demand, when that holds fewer values. +#[cfg(feature = "cuda")] +fn lower_for_zerocheck(program: &crate::program::Program) -> Option +where + E: math::field::traits::IsField + 'static, +{ + let today = lower(program)?; + let big = is_big_batch(today.num_slots); + let as_today = |today| ZerocheckLowering { + run: today, + held: None, + big, + }; + if !big || !argue_lean_program() { + return Some(as_today(today)); + } + let Some(mut demand) = lower(&program.on_demand()) else { + return Some(as_today(today)); + }; + if demand.num_slots >= today.num_slots { + return Some(as_today(today)); + } + if LEAN_PROGRAM_FAULT.load(Ordering::Relaxed) { + for constant in demand.consts.chunks_exact_mut(3) { + constant[0] = constant[0].wrapping_add(1); + } + } + Some(ZerocheckLowering { + run: demand, + held: Some(today), + big, + }) +} + +/// Which table the card built a fault corrupts ([`force_table_fault`]). +#[doc(hidden)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableFault { + /// The first table built from `eq`: the zerocheck's `eq(r)`, or the claim + /// reduce's first shift table. + Eq, + /// The claim reduce's first batched column. + Batched, +} + +/// ⛔ A FAULT: while armed, the first cell of the chosen table the card builds +/// is overwritten before the rounds read it — what a kernel that wrote one +/// wrong cell would leave. So the checks on the tables can be shown to fail. +/// Never armed outside a test. +static TABLE_FAULT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// What [`TABLE_FAULT`] writes: a value no table here holds in its first cell +/// but by a coincidence of one in 2^64. +#[cfg(feature = "cuda")] +const FAULT_CELL: [u64; 3] = [12345, 0, 0]; + +#[doc(hidden)] +pub fn force_table_fault(fault: Option) { + TABLE_FAULT.store( + match fault { + None => 0, + Some(TableFault::Eq) => 1, + Some(TableFault::Batched) => 2, + }, + Ordering::Relaxed, + ); +} + +#[cfg(feature = "cuda")] +fn table_fault() -> Option { + match TABLE_FAULT.load(Ordering::Relaxed) { + 1 => Some(TableFault::Eq), + 2 => Some(TableFault::Batched), + _ => None, + } +} + +/// Called where the card built `tables` of the argument's challenge tables. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +fn note_tables_on_card(tables: usize) { + ARGUE_TABLES_ON_CARD.fetch_add(tables as u64, Ordering::Relaxed); + crate::whir_split::bump_by(&crate::whir_split::TABLES_ON_CARD, tables as u64); +} + +/// Called where `LAMBDA_VM_ARGUE_XCHECK` found `tables` the card built equal to +/// the host's, cell for cell. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +fn note_tables_xchecked(tables: usize) { + ARGUE_TABLE_XCHECKS.fetch_add(tables as u64, Ordering::Relaxed); + crate::whir_split::bump_by(&crate::whir_split::TABLES_XCHECKED, tables as u64); +} + +/// A batch weight as the device is handed it: a table the host built, or the +/// point of an `eq` table the card builds (`LAMBDA_VM_ARGUE_DEVICE_TABLES`). +#[derive(Clone, Copy, Debug)] +pub(crate) enum WeightView<'a, E: math::field::traits::IsField> { + Table(&'a crate::mle::Mle), + Eq(&'a [math::field::element::FieldElement]), +} + +impl WeightView<'_, E> { + /// Cells the weight spans. + #[cfg_attr(not(feature = "cuda"), allow(dead_code))] + fn cells(&self) -> usize { + match self { + Self::Table(table) => table.len(), + Self::Eq(point) => 1usize << point.len(), + } + } +} + +/// The first cell at which a table the card built is not the host's. +#[cfg(feature = "cuda")] +fn first_difference( + card: &[u64], + host: &[math::field::element::FieldElement], +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + if card.len() != host.len() * 3 { + return Some(card.len().min(host.len() * 3) / 3); + } + card.chunks_exact(3) + .zip(host) + .position(|(cell, value)| ext3_from_raw::(cell) != *value) +} + +/// ⛔ A FAULT: while armed, the batched device evaluation hands back its first +/// column's value plus one — what a kernel that read one wrong cell would do. +/// +/// It exists so the checks that compare the card against the host can be shown +/// to fail: an identity test that passes whatever the card returns certifies +/// nothing. Never armed outside a test. Not for production callers. +static COLUMN_VALUE_FAULT: core::sync::atomic::AtomicBool = + core::sync::atomic::AtomicBool::new(false); + +#[doc(hidden)] +pub fn force_column_value_fault(on: bool) { + COLUMN_VALUE_FAULT.store(on, Ordering::Relaxed); +} + +/// Whether [`evaluate_mle`] sends a table of `len` evaluations to the device — +/// so a caller evaluating many one at a time can tell a host loop, which it may +/// spread over the pool, from device calls, which it must not. +#[cfg(feature = "cuda")] +pub(crate) fn evaluates_on_device(len: usize) -> bool { + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + len >= EVALUATE_THRESHOLD + && !*DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_MLE_EVAL").is_some()) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn evaluates_on_device(_len: usize) -> bool { + false +} + +/// Called where the claim reduce walks `columns` on the host, one at a time. +pub(crate) fn note_host_evaluations(columns: usize) { + HOST_EVALUATE_CALLS.fetch_add(columns as u64, Ordering::Relaxed); + crate::whir_split::bump_by(&crate::whir_split::COLUMNS_ON_HOST, columns as u64); +} + +/// Called where `LAMBDA_VM_ARGUE_XCHECK` found `columns` device values equal to +/// the host's. +pub(crate) fn note_xchecked(columns: usize) { + ARGUE_XCHECKS.fetch_add(columns as u64, Ordering::Relaxed); + crate::whir_split::bump_by(&crate::whir_split::COLUMNS_XCHECKED, columns as u64); +} + +/// Every base-field column's value at one point, folded together. +/// +/// The columns of a table are evaluated at the same point and one at a time +/// that is an upload and a launch per level each. Together it is one upload +/// and one launch per level for all of them. +#[cfg(feature = "cuda")] +pub(crate) fn evaluate_many_base( + columns: &C, + point: &[math::field::element::FieldElement], + resident: Option<(&ResidentColumns, usize)>, +) -> Option>> +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, + C: crate::narrow::HostColumns + ?Sized, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + let (Some(rows), width) = (columns.rows(), columns.width()) else { + return None; + }; + if width == 0 || point.is_empty() || rows != 1 << point.len() { + return None; + } + // Columns already on the card cost no upload, so under the knob a resident + // table is worth the launches once it has the cells; any other table still + // has to pay its upload, and is worth it only once each column is tall. + let resident_run = argue_device_columns() + && resident.is_some_and(|(store, first)| store.0.is_run(first, width)); + let size = if resident_run { + rows.saturating_mul(width) + } else { + rows + }; + if size < EVALUATE_THRESHOLD { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_MLE_EVAL").is_some()) { + return None; + } + + let mut raw_point = Vec::with_capacity(point.len() * 3); + for coordinate in point { + raw_point.extend_from_slice(&ext3_raw(coordinate)?); + } + let values = with_columns_at(resident, columns, |at| { + math_cuda::sumcheck::evaluate_many_base(at, &raw_point).ok() + }) + .flatten()?; + EVALUATE_CALLS.fetch_add(values.len() as u64, Ordering::Relaxed); + crate::whir_split::bump_by(&crate::whir_split::COLUMNS_ON_CARD, values.len() as u64); + let mut values: Vec> = + values.iter().map(|v| ext3_from_raw::(v)).collect(); + if COLUMN_VALUE_FAULT.load(Ordering::Relaxed) + && let Some(first) = values.first_mut() + { + *first += math::field::element::FieldElement::::one(); + } + Some(values) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn evaluate_many_base( + _columns: &C, + _point: &[math::field::element::FieldElement], + _resident: Option<(&ResidentColumns, usize)>, +) -> Option>> +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, + C: crate::narrow::HostColumns + ?Sized, +{ + None +} + +/// Store columns `first..first + width`, each at `point` (base columns, an +/// ext3 point), evaluated where they lie: a commit-time out-of-domain claim's +/// column values (`stacked_eval::ood_claims`). `None` when the store does not +/// hold them as one run spanning `point`, when the run has fewer than +/// `min_cells` cells, or when the card refuses: the caller evaluates them on +/// the host. +#[cfg(feature = "cuda")] +pub(crate) fn evaluate_resident_run( + store: &ResidentColumns, + first: usize, + width: usize, + point: &[math::field::element::FieldElement], + min_cells: usize, +) -> Option>> +where + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || point.is_empty() || width == 0 { + return None; + } + let rows = 1usize << point.len(); + if !store.0.is_run(first, width) + || store.0.span(first).1 != rows + || rows.saturating_mul(width) < min_cells + { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_MLE_EVAL").is_some()) { + return None; + } + // The point's `eq` table in two halves, a few KiB each: each column is then + // read once, at the bandwidth of its base words. + let lo_bits = point.len().min(EQ_DOT_LO_BITS); + let (high, low) = point.split_at(point.len() - lo_bits); + let raw_table = |coordinates: &[math::field::element::FieldElement]| -> Option> { + let mut raw = Vec::with_capacity(3 << coordinates.len()); + for value in crate::eq::eq_evals(coordinates) { + raw.extend_from_slice(&ext3_raw(&value)?); + } + Some(raw) + }; + let values = math_cuda::sumcheck::eq_dot_many_base( + &store.0, + first, + width, + &raw_table(high)?, + &raw_table(low)?, + lo_bits as u32, + ) + .ok()?; + EVALUATE_CALLS.fetch_add(values.len() as u64, Ordering::Relaxed); + Some(values.iter().map(|v| ext3_from_raw::(v)).collect()) +} + +/// The low half of [`evaluate_resident_run`]'s `eq` table, in variables: 2^10 +/// extension values, 24 KiB, read by every row of a column. +#[cfg(feature = "cuda")] +const EQ_DOT_LO_BITS: usize = 10; + +#[cfg(not(feature = "cuda"))] +pub(crate) fn evaluate_resident_run( + _store: &ResidentColumns, + _first: usize, + _width: usize, + _point: &[math::field::element::FieldElement], + _min_cells: usize, +) -> Option>> +where + E: math::field::traits::IsField + 'static, +{ + None +} + +/// A multilinear's value at `point`, bound variable by variable on device. +/// +/// `evals` may be base-field or ext3; the point is always ext3, which is what +/// a challenge is. +#[cfg(feature = "cuda")] +pub(crate) fn evaluate_mle( + evals: &[math::field::element::FieldElement], + point: &[math::field::element::FieldElement], +) -> Option> +where + C: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() { + return None; + } + let base = TypeId::of::() == TypeId::of::(); + if !base && TypeId::of::() != TypeId::of::() { + return None; + } + if point.is_empty() || evals.len() < EVALUATE_THRESHOLD || evals.len() != 1 << point.len() { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_MLE_EVAL").is_some()) { + return None; + } + + let mut raw_point = Vec::with_capacity(point.len() * 3); + for coordinate in point { + raw_point.extend_from_slice(&ext3_raw(coordinate)?); + } + // SAFETY: `C` is one of the two fields checked above, and both wrap their + // limbs transparently — one `u64` per base element, three per ext3. + let limbs = if base { 1 } else { 3 }; + let raw = + unsafe { core::slice::from_raw_parts(evals.as_ptr() as *const u64, evals.len() * limbs) }; + let value = if base { + math_cuda::sumcheck::evaluate_mle_base(raw, &raw_point).ok()? + } else { + math_cuda::sumcheck::evaluate_mle_ext3(raw, &raw_point).ok()? + }; + EVALUATE_CALLS.fetch_add(1, Ordering::Relaxed); + Some(ext3_from_raw::(&value)) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn evaluate_mle( + _evals: &[math::field::element::FieldElement], + _point: &[math::field::element::FieldElement], +) -> Option> +where + C: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + None +} + +/// Input-layer size below which the host tree wins: the levels are a launch +/// each and the fold is a pass a few cores finish in microseconds. +#[cfg(feature = "cuda")] +const TREE_THRESHOLD: usize = 1 << 14; + +/// A LogUp fraction tree the device holds. +/// +/// The levels are here; the input layer is here only if it was cheap to carry. +/// A tree built from resident factors gives it back after the first fold and +/// writes it again for its own sumcheck — see +/// [`input_layer_tree`](crate::gpu::input_layer_tree). +#[cfg(feature = "cuda")] +pub struct DeviceTree { + /// Taken when the input layer's sumcheck comes: every level above it is + /// spent by then, and letting them go is what makes room for it. + tree: std::sync::Mutex>, + /// Writes the input layer again, for a tree that did not keep it — or + /// kept it without its padding, for today's layer session, which reads + /// every cell. + #[allow(clippy::type_complexity)] + rebuild: Option Option + Send + Sync>>, + /// Whether the tree keeps its input layer without its padding + /// ([`crate::gkr_input_real`]): its Gruen rounds open the kept layer, and + /// only today's session writes it again. + input_truncated: bool, + num_layers: usize, + input_num_vars: usize, + /// The output fraction, read off the top level. + /// + /// Populated at build on the eager path (whoever asks may be asking after + /// the levels are gone). Left empty by the prefetch build so the fold + /// kernels stay in flight and are not waited on here; [`Self::output`] then + /// reads it from the retained tree on first access, at the consume site, + /// which is before any level is spent. + output: std::sync::OnceLock<([u64; 3], [u64; 3])>, + /// The room the whole thing promised itself, the handed-back layer + /// included. + _room: Option, +} + +#[cfg(feature = "cuda")] +impl std::fmt::Debug for DeviceTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeviceTree") + .field("layers", &self.num_layers) + .finish() + } +} + +/// A device tree the build declined to make. Never constructed. +#[cfg(not(feature = "cuda"))] +#[derive(Debug)] +pub struct DeviceTree(std::convert::Infallible); + +#[cfg(not(feature = "cuda"))] +impl DeviceTree { + pub(crate) fn num_layers(&self) -> usize { + match self.0 {} + } + + pub(crate) fn output( + &self, + ) -> Result< + ( + math::field::element::FieldElement, + math::field::element::FieldElement, + ), + crate::Error, + > + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn prove_layer( + &self, + _layer: usize, + _point: &[math::field::element::FieldElement], + _program: &crate::program::Program, + _degree: usize, + _tail: usize, + _challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Option, crate::Error>> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn prove_layer_gruen( + &self, + _layer: usize, + _point: &[math::field::element::FieldElement], + _lambda: &math::field::element::FieldElement, + _claim: math::field::element::FieldElement, + _tail: usize, + _today: Option<&crate::program::Program>, + _challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Option, crate::Error>> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn layer_to_host( + &self, + _layer: usize, + ) -> Option<(crate::mle::Mle, crate::mle::Mle)> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } +} + +/// What a layer's rounds on device leave: the rounds themselves, the point +/// they drew, and the five factors as the last fold left them. +/// +/// The factors are one value each when the device ran the layer out, and a +/// cube when it stopped at the crossover for the host to finish — the caller +/// carries on from them either way. +pub type LayerRounds = ( + Vec>, + Vec>, + Vec>, +); + +/// Builds the tree on device from an input layer, folding every level there. +#[cfg(feature = "cuda")] +pub(crate) fn build_tree(p: &crate::mle::Mle, q: &crate::mle::Mle) -> Option +where + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + if p.len() < TREE_THRESHOLD || p.len() != q.len() { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_GKR").is_some()) { + return None; + } + // SAFETY: `E == Ext3`, three transparent `u64` limbs per element. + let raw = |table: &crate::mle::Mle| unsafe { + core::slice::from_raw_parts(table.evals().as_ptr() as *const u64, table.len() * 3) + }; + let tree = math_cuda::gkr::DeviceFractionTree::build(raw(p), raw(q)).ok()?; + let output = tree.output().ok()?; + let num_layers = tree.num_layers(); + let input_num_vars = tree.layer_num_vars(num_layers - 1); + TREE_CALLS.fetch_add(1, Ordering::Relaxed); + // This one keeps its input layer: it was uploaded whole, so there is + // nothing cheaper to hand back. Eager: the output is read at build. + let output_cell = std::sync::OnceLock::new(); + let _ = output_cell.set(output); + Some(DeviceTree { + tree: std::sync::Mutex::new(Some(tree)), + rebuild: None, + input_truncated: false, + num_layers, + input_num_vars, + output: output_cell, + _room: None, + }) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn build_tree(_p: &crate::mle::Mle, _q: &crate::mle::Mle) -> Option +where + E: math::field::traits::IsField + 'static, +{ + None +} + +#[cfg(feature = "cuda")] +impl DeviceTree { + pub(crate) fn num_layers(&self) -> usize { + self.num_layers + } + + /// The output fraction, which is what says whether the bus balances. + /// + /// On the eager path the value was read at build and is returned straight. + /// On the prefetch path it was left for here: the top fraction is read off + /// the retained tree now — the one device sync the prefetch deferred — and + /// cached. This runs at the consume site, before GKR spends any level, so + /// the top is still there to read. + pub(crate) fn output( + &self, + ) -> Result< + ( + math::field::element::FieldElement, + math::field::element::FieldElement, + ), + crate::Error, + > + where + E: math::field::traits::IsField + 'static, + { + let (p, q) = self.materialized_output()?; + Ok((ext3_from_raw::(&p), ext3_from_raw::(&q))) + } + + /// The output limbs, reading them off the device on first access for a tree + /// the prefetch build left unread, then caching. `DeviceFailed` if the + /// retained tree is gone or the device read fails — the caller falls back + /// to a host build, and the transcript has not moved yet at this point. + fn materialized_output(&self) -> Result<([u64; 3], [u64; 3]), crate::Error> { + if let Some(v) = self.output.get() { + return Ok(*v); + } + let read = { + let held = self.tree.lock().map_err(|_| crate::Error::DeviceFailed { + stage: "gkr tree output", + })?; + let tree = held.as_ref().ok_or(crate::Error::DeviceFailed { + stage: "gkr tree output", + })?; + tree.output().map_err(|_| crate::Error::DeviceFailed { + stage: "gkr tree output", + })? + }; + let _ = self.output.set(read); + Ok(read) + } + + /// A level's halves back here, for the levels near the output that GKR + /// proves on the host. + pub(crate) fn layer_to_host( + &self, + layer: usize, + ) -> Option<(crate::mle::Mle, crate::mle::Mle)> + where + E: math::field::traits::IsField + 'static, + { + let held = self.tree.lock().ok()?; + let (p, q) = held.as_ref()?.layer_to_host(layer).ok()?; + let table = |raw: Vec| { + crate::mle::Mle::new(raw.chunks_exact(3).map(ext3_from_raw::).collect()).ok() + }; + Some((table(p)?, table(q)?)) + } + + /// One layer's sumcheck, folded in place: the rounds, the point they drew, + /// and the five factors as the last fold left them. + /// + /// The rounds stop once the cube is down to `tail`, which the caller + /// finishes here — below a few hundred indices a round is a launch and a + /// wait around a kernel with almost nothing to sum. `tail` of one runs the + /// layer out there, and the factors come back as one value each. + /// + /// The layer is spent afterwards, which is what makes the halves usable as + /// factors without copying them: GKR reads each layer once. + pub(crate) fn prove_layer( + &self, + layer: usize, + point: &[math::field::element::FieldElement], + program: &crate::program::Program, + degree: usize, + tail: usize, + challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Option, crate::Error>> + where + E: math::field::traits::IsField + 'static, + { + use crate::whir_split::{self, add_tick, tick}; + + let t = tick(); + let lowered = lower(program)?; + let mut raw_point = Vec::with_capacity(point.len() * 3); + for coordinate in point { + raw_point.extend_from_slice(&ext3_raw(coordinate)?); + } + add_tick(&whir_split::GKR_LOWER, t); + let input_layer = layer + 1 == self.num_layers; + let rebuilds = input_layer && self.rebuild.is_some(); + let t = tick(); + let session = if rebuilds { + // The one the tree gave back, or kept without its padding, which + // this session would read. Everything above it has been proved, + // so the levels go first and the layer is written where they were. + let num_vars = self.input_num_vars.checked_sub(1)?; + if num_vars * 3 != raw_point.len() { + return None; + } + drop(self.tree.lock().ok()?.take()); + let rebuilt = (self.rebuild.as_ref()?)()?; + if self.input_truncated { + crate::gkr_input_real::note_rebuild(); + } + rebuilt.sumcheck( + &raw_point, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + } else { + let held = self.tree.lock().ok()?; + let tree = held.as_ref()?; + if tree.layer_num_vars(layer).checked_sub(1)? * 3 != raw_point.len() { + return None; + } + tree.layer_sumcheck( + layer, + &raw_point, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + }; + if rebuilds { + add_tick(&whir_split::GKR_REBUILD, t); + whir_split::bump(&whir_split::GKR_REBUILDS); + } else { + add_tick(&whir_split::GKR_SESSION, t); + } + let num_vars = if input_layer { + self.input_num_vars.checked_sub(1)? + } else { + self.tree + .lock() + .ok()? + .as_ref()? + .layer_num_vars(layer) + .checked_sub(1)? + }; + let Ok(mut session) = session else { + return None; + }; + // What is left for the host: the rounds stop where the cube reaches it. + let here = tail.max(1).trailing_zeros() as usize; + let there = num_vars.saturating_sub(here); + + // Past here the transcript moves: the host path is no longer an option. + let failed = |stage| crate::Error::DeviceFailed { stage }; + let t = tick(); + let outcome = run_rounds(&mut session, degree, there, challenge, |_| None); + add_tick(&whir_split::GKR_ROUNDS, t); + let (rounds, challenges) = match outcome { + Ok(rounds) => rounds, + Err(error) => return Some(Err(error)), + }; + let t = tick(); + let Ok(values) = session.values() else { + return Some(Err(failed("layer values"))); + }; + add_tick(&whir_split::GKR_VALUES, t); + // Factor 0 is the weight; the four the layer reduces to follow. + if values.len() != 5 { + return Some(Err(failed("layer factors"))); + } + let t = tick(); + let factors: Result>, crate::Error> = values + .iter() + .map(|factor| { + crate::mle::Mle::new(factor.chunks_exact(3).map(ext3_from_raw::).collect()) + }) + .collect(); + let Ok(factors) = factors else { + return Some(Err(failed("layer factors"))); + }; + add_tick(&whir_split::GKR_FACTORS, t); + whir_split::bump(&whir_split::GKR_LAYERS); + SUMCHECK_CALLS.fetch_add(1, Ordering::Relaxed); + Some(Ok((rounds, challenges, factors))) + } +} + +#[cfg(feature = "cuda")] +impl DeviceTree { + /// [`prove_layer`](Self::prove_layer) with Gruen's rounds + /// ([`crate::gkr_gruen`]): the same rounds, challenges and factors, from a + /// pass that sums `H(1)` and `H(2)` and folds the previous challenge on the + /// way in. `claim` is `p + λ·q` of the level above, the value round 0's + /// `s(0) + s(1)` has to meet; `tail` the cube the host finishes from. + /// + /// With `today` (`LAMBDA_VM_ARGUE_GKR_GRUEN_XCHECK`), today's program walks + /// the same folded halves every round beside these, and the prove fails + /// where the two disagree — rounds or factors. + #[allow(clippy::too_many_arguments)] + pub(crate) fn prove_layer_gruen( + &self, + layer: usize, + point: &[math::field::element::FieldElement], + lambda: &math::field::element::FieldElement, + claim: math::field::element::FieldElement, + tail: usize, + today: Option<&crate::program::Program>, + mut challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Option, crate::Error>> + where + E: math::field::traits::IsField + 'static, + { + use crate::gkr_gruen::{self as gruen, Chain}; + use crate::whir_split::{self, add_tick, tick}; + use math::field::element::FieldElement; + + let t = tick(); + let mut raw_point = Vec::with_capacity(point.len() * 3); + for coordinate in point { + raw_point.extend_from_slice(&ext3_raw(coordinate)?); + } + let raw_lambda: [u64; 3] = ext3_raw(lambda)?; + let lowered = match today { + Some(program) => Some(lower(program)?), + None => None, + }; + let low = tail.max(1).trailing_zeros() as usize; + add_tick(&whir_split::GKR_LOWER, t); + let input_layer = layer + 1 == self.num_layers; + // A layer kept without its padding is opened where it lies; the + // cross-check's shadow declines it (today's session reads every cell). + let rebuilds = input_layer && self.rebuild.is_some() && !self.input_truncated; + let t = tick(); + let session = if rebuilds { + let num_vars = self.input_num_vars.checked_sub(1)?; + if num_vars * 3 != raw_point.len() { + return None; + } + drop(self.tree.lock().ok()?.take()); + let rebuilt = (self.rebuild.as_ref()?)()?; + rebuilt.gruen(&raw_point, low) + } else { + let held = self.tree.lock().ok()?; + let tree = held.as_ref()?; + if tree.layer_num_vars(layer).checked_sub(1)? * 3 != raw_point.len() { + return None; + } + tree.layer_gruen(layer, &raw_point, low) + }; + let Ok(mut session) = session else { + return None; + }; + session.use_slot(crate::round_slot::device_setting()); + // Today's session beside it, when the card has room for its `eq` table. + let shadow = match lowered.as_ref() { + Some(lowered) => { + let room = math_cuda::device::reserve(session.shadow_bytes(lowered.num_slots)); + let made = room.as_ref().and_then(|_| { + session + .shadow( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .ok() + }); + if made.is_none() { + gruen::note_xcheck_skipped(); + } + made.map(|shadow| (shadow, room)) + } + None => None, + }; + if rebuilds { + add_tick(&whir_split::GKR_REBUILD, t); + whir_split::bump(&whir_split::GKR_REBUILDS); + } else { + add_tick(&whir_split::GKR_SESSION, t); + } + let rounds_here = session.rounds(); + let lefts = gruen::inverse_lefts(&point[..rounds_here]); + let chain = Chain::new(claim); + let mut nodes = Vec::with_capacity(9); + for node in 1..=3u64 { + nodes.extend_from_slice(&ext3_raw(&FieldElement::::from(node))?); + } + + // Past here the transcript moves: the host path is no longer an option. + let failed = |stage| crate::Error::DeviceFailed { stage }; + let mut shadow = shadow; + let t = tick(); + let outcome = (|| -> Result<_, crate::Error> { + let mut chain = chain.ok_or(failed("gruen chain"))?; + let mut rounds = Vec::with_capacity(rounds_here); + let mut challenges: Vec> = Vec::with_capacity(rounds_here); + let mut previous: Option<[u64; 3]> = None; + for (j, left) in lefts.iter().enumerate() { + let want_h0 = left.is_none(); + let sums = session + .round(previous.as_ref(), &raw_lambda, want_h0) + .map_err(|_| failed("gruen round"))?; + let sum = |k: usize| ext3_from_raw::(&sums[k * 3..k * 3 + 3]); + let h0 = want_h0.then(|| sum(2)); + if want_h0 { + gruen::note_direct_h0(); + } + let mut message = chain + .message(&point[j], left.as_ref(), &sum(0), &sum(1), h0.as_ref()) + .ok_or(failed("gruen message"))?; + if j == 0 && gruen::gkr_gruen_fault() { + // `s(0) = claim − s(1)`: what the claim then implies. + message.sent[0] += FieldElement::::one(); + message.s0 = &message.s0 - FieldElement::::one(); + } + if let Some((shadow, _)) = shadow.as_mut() { + if let Some(r) = previous.as_ref() { + shadow + .fold_first(1, r) + .map_err(|_| failed("gruen shadow"))?; + } + let want: Vec> = shadow + .round(&nodes) + .map_err(|_| failed("gruen shadow"))? + .chunks_exact(3) + .map(ext3_from_raw::) + .collect(); + let parted = want != message.sent; // XCHECK-COMPARE + if parted { + return Err(failed("gkr gruen xcheck")); + } + } + let z = challenge(&message.sent); + chain.advance(&point[j], &message, &z); + previous = Some(ext3_raw(&z).ok_or(failed("challenge"))?); + rounds.push(crate::sumcheck::RoundProof { + evaluations: message.sent, + }); + challenges.push(z); + SUMCHECK_ROUNDS.fetch_add(1, Ordering::Relaxed); + } + Ok((rounds, challenges, chain, previous)) + })(); + add_tick(&whir_split::GKR_ROUNDS, t); + let (rounds, challenges, chain, previous) = match outcome { + Ok(outcome) => outcome, + Err(error) => return Some(Err(error)), + }; + let t = tick(); + let Ok(values) = session.finish(previous.as_ref()) else { + return Some(Err(failed("layer values"))); + }; + add_tick(&whir_split::GKR_VALUES, t); + let t = tick(); + let cells = 1usize << session.low(); + if values.len() != 4 * cells * 3 { + return Some(Err(failed("layer factors"))); + } + let e_lo = crate::eq::eq_evals(&point[rounds_here..]); + let mut factors = Vec::with_capacity(5); + let eq: Vec> = e_lo.iter().map(|v| chain.kappa() * v).collect(); + let Ok(eq) = crate::mle::Mle::new(eq) else { + return Some(Err(failed("layer factors"))); + }; + factors.push(eq); + for half in values.chunks_exact(cells * 3) { + let Ok(table) = + crate::mle::Mle::new(half.chunks_exact(3).map(ext3_from_raw::).collect()) + else { + return Some(Err(failed("layer factors"))); + }; + factors.push(table); + } + add_tick(&whir_split::GKR_FACTORS, t); + let checked = match shadow.as_mut() { + Some((shadow, _)) => { + if let Some(r) = previous.as_ref() + && shadow.fold_first(1, r).is_err() + { + return Some(Err(failed("gruen shadow"))); + } + let Ok(want) = shadow.values_gathered() else { + return Some(Err(failed("gruen shadow"))); + }; + let same = want.len() == 5 + && want.iter().zip(&factors).all(|(raw, table)| { + raw.len() == table.len() * 3 + && raw + .chunks_exact(3) + .zip(table.evals()) + .all(|(cell, value)| ext3_from_raw::(cell) == *value) + }); + if !same { + return Some(Err(failed("gkr gruen xcheck"))); + } + true + } + None => false, + }; + gruen::note_layer(rounds_here as u64, checked); + whir_split::bump(&whir_split::GKR_LAYERS); + whir_split::bump(&whir_split::GKR_GRUEN); + if checked { + whir_split::bump(&whir_split::GKR_GRUEN_XCHECKED); + } + SUMCHECK_CALLS.fetch_add(1, Ordering::Relaxed); + Some(Ok((rounds, challenges, factors))) + } +} + +/// One layer's sumcheck on the card, stepped a round at a time by its caller: +/// the batched argue's lockstep ladder (D-BATCH B-3), where every active tree's +/// round is summed on the host before the one shared challenge is drawn. +/// +/// The same session [`DeviceTree::prove_layer`] opens — over the layer where it +/// lies, the input layer written again for a tree that gave it back — with the +/// rounds left to the caller. +#[cfg(feature = "cuda")] +pub struct LayerSession { + session: math_cuda::sumcheck::SumcheckSession, + /// The interpolation nodes `1..=degree`, three u64 each. + nodes: Vec, + /// A round queued by [`Self::enqueue`] and not yet read back. + pending: Option, +} + +/// A layer session the build could not open. Never constructed. +#[cfg(not(feature = "cuda"))] +pub struct LayerSession(std::convert::Infallible); + +#[cfg(feature = "cuda")] +impl DeviceTree { + /// Opens layer `layer`'s sumcheck under `program` at `point`, or `None` + /// before anything moves when the card declines. + pub(crate) fn layer_session( + &self, + layer: usize, + point: &[math::field::element::FieldElement], + program: &crate::program::Program, + degree: usize, + ) -> Option + where + E: math::field::traits::IsField + 'static, + { + let lowered = lower(program)?; + let mut raw_point = Vec::with_capacity(point.len() * 3); + for coordinate in point { + raw_point.extend_from_slice(&ext3_raw(coordinate)?); + } + let mut nodes = Vec::with_capacity(degree * 3); + for node in 1..=degree as u64 { + nodes.extend_from_slice(&ext3_raw(&math::field::element::FieldElement::::from( + node, + ))?); + } + let input_layer = layer + 1 == self.num_layers; + // A layer kept without its padding is written whole again too: this + // session reads every cell. + let session = if input_layer && self.rebuild.is_some() { + let num_vars = self.input_num_vars.checked_sub(1)?; + if num_vars * 3 != raw_point.len() { + return None; + } + drop(self.tree.lock().ok()?.take()); + let rebuilt = (self.rebuild.as_ref()?)()?; + whir_split_bump_rebuild(); + if self.input_truncated { + crate::gkr_input_real::note_rebuild(); + } + rebuilt.sumcheck( + &raw_point, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + } else { + let held = self.tree.lock().ok()?; + let tree = held.as_ref()?; + if tree.layer_num_vars(layer).checked_sub(1)? * 3 != raw_point.len() { + return None; + } + tree.layer_sumcheck( + layer, + &raw_point, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + }; + let session = session.ok()?; + Some(LayerSession { + session, + nodes, + pending: None, + }) + } +} + +#[cfg(feature = "cuda")] +fn whir_split_bump_rebuild() { + crate::whir_split::bump(&crate::whir_split::GKR_REBUILDS); +} + +#[cfg(not(feature = "cuda"))] +impl DeviceTree { + pub(crate) fn layer_session( + &self, + _layer: usize, + _point: &[math::field::element::FieldElement], + _program: &crate::program::Program, + _degree: usize, + ) -> Option + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } +} + +#[cfg(feature = "cuda")] +impl LayerSession { + /// Variables left to bind. + pub(crate) fn num_vars(&self) -> usize { + self.session.len().trailing_zeros() as usize + } + + /// Queues this round's launches without waiting for them, so the ladder + /// can queue every active tree's before it reads any back. + pub(crate) fn enqueue(&mut self) -> Result<(), crate::Error> { + if self.pending.is_none() { + let num_t = self + .session + .round_enqueue(&self.nodes) + .map_err(|_| crate::Error::DeviceFailed { stage: "round" })?; + self.pending = Some(num_t); + } + Ok(()) + } + + /// This round's polynomial at `1..=degree`: the queued round read back, + /// or queued and read back now. + pub(crate) fn round( + &mut self, + ) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + self.enqueue()?; + let num_t = self.pending.take().unwrap_or(0); + let sums = self + .session + .round_collect(num_t) + .map_err(|_| crate::Error::DeviceFailed { stage: "round" })?; + SUMCHECK_ROUNDS.fetch_add(1, Ordering::Relaxed); + Ok(sums.chunks_exact(3).map(ext3_from_raw::).collect()) + } + + /// Binds the round's variable to `r` in every factor. + pub(crate) fn fold( + &mut self, + r: &math::field::element::FieldElement, + ) -> Result<(), crate::Error> + where + E: math::field::traits::IsField + 'static, + { + let raw = ext3_raw(r).ok_or(crate::Error::DeviceFailed { stage: "challenge" })?; + self.session + .fold(&raw) + .map_err(|_| crate::Error::DeviceFailed { stage: "fold" }) + } + + /// Every factor as the folds left it — the weight, then the four halves. + pub(crate) fn factors(&self) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + let values = self + .session + .values() + .map_err(|_| crate::Error::DeviceFailed { + stage: "layer values", + })?; + values + .iter() + .map(|factor| { + crate::mle::Mle::new(factor.chunks_exact(3).map(ext3_from_raw::).collect()) + }) + .collect() + } +} + +#[cfg(not(feature = "cuda"))] +impl LayerSession { + pub(crate) fn num_vars(&self) -> usize { + match self.0 {} + } + + pub(crate) fn enqueue(&mut self) -> Result<(), crate::Error> { + match self.0 {} + } + + pub(crate) fn round( + &mut self, + ) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn fold( + &mut self, + _r: &math::field::element::FieldElement, + ) -> Result<(), crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn factors(&self) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } +} + +/// One layer's Gruen rounds ([`crate::gkr_gruen`]) on the card, stepped a +/// round at a time by the batched argue's ladder: the card sums `H(1)`, `H(2)` +/// (and `H(0)` where asked) of `p_lo·q_hi + p_hi·q_lo + λ·q_lo·q_hi`, folding +/// the previous challenge on the way in, and the caller's chain makes the +/// messages. The session [`DeviceTree::prove_layer_gruen`] opens, with the +/// rounds left to the caller. +#[cfg(feature = "cuda")] +pub struct GruenSession { + layer: math_cuda::gkr::GruenLayer, + lambda: [u64; 3], + /// A round queued by [`Self::enqueue`] and not yet read back: its rows. + pending: Option, + /// Under `LAMBDA_VM_ARGUE_INPUT_REAL=check`, for an input layer kept + /// without its padding: the whole layer, written again, whose rounds run + /// in step and are compared with the kept one's. + whole: Option, + /// The whole layer's room, once a kept layer that differed finishes on it. + _held: Option, +} + +/// The whole input layer beside a kept one, and the room it was promised. +#[cfg(feature = "cuda")] +struct WholeLayer { + layer: math_cuda::gkr::GruenLayer, + _room: math_cuda::device::DeviceReservation, +} + +/// A Gruen session the build could not open. Never constructed. +#[cfg(not(feature = "cuda"))] +pub struct GruenSession(std::convert::Infallible); + +#[cfg(feature = "cuda")] +impl DeviceTree { + /// Opens layer `layer`'s Gruen rounds at `point` under `lambda`, at most + /// `log2(tail)` variables left to the host, or `None` before anything + /// moves when the card declines. + pub(crate) fn gruen_session( + &self, + layer: usize, + point: &[math::field::element::FieldElement], + lambda: &math::field::element::FieldElement, + tail: usize, + ) -> Option + where + E: math::field::traits::IsField + 'static, + { + let mut raw_point = Vec::with_capacity(point.len() * 3); + for coordinate in point { + raw_point.extend_from_slice(&ext3_raw(coordinate)?); + } + let lambda = ext3_raw(lambda)?; + let low = tail.max(1).trailing_zeros() as usize; + let input_layer = layer + 1 == self.num_layers; + // A layer kept without its padding is opened where it lies. + let session = if input_layer && self.rebuild.is_some() && !self.input_truncated { + let num_vars = self.input_num_vars.checked_sub(1)?; + if num_vars * 3 != raw_point.len() { + return None; + } + drop(self.tree.lock().ok()?.take()); + let rebuilt = (self.rebuild.as_ref()?)()?; + whir_split_bump_rebuild(); + rebuilt.gruen(&raw_point, low) + } else { + let held = self.tree.lock().ok()?; + let tree = held.as_ref()?; + if tree.layer_num_vars(layer).checked_sub(1)? * 3 != raw_point.len() { + return None; + } + tree.layer_gruen(layer, &raw_point, low) + }; + let mut layer = session.ok()?; + layer.use_slot(crate::round_slot::device_setting()); + let whole = (layer.truncated() + && crate::gkr_input_real::argue_input_real() + == crate::gkr_input_real::InputReal::Check) + .then(|| self.whole_beside(&raw_point, low)) + .flatten(); + Some(GruenSession { + layer, + lambda, + pending: None, + whole, + _held: None, + }) + } + + /// Under `check`: the input layer written whole again beside the kept one + /// and opened for today's Gruen rounds — or `None`, counted, when the card + /// has no room for it or the write fails, and the kept layer proves alone. + fn whole_beside(&self, raw_point: &[u64], low: usize) -> Option { + let skipped = || { + crate::gkr_input_real::note_check_skipped(); + None + }; + let Some(rebuild) = self.rebuild.as_ref() else { + return skipped(); + }; + // The layer, and the session's `eq` levels and partials beside it. + let m = self.input_num_vars.saturating_sub(1); + let (low, rounds) = (low.min(m), m - low.min(m)); + let bytes = (2u64 << self.input_num_vars) * 24 + + ((1u64 << low) + (1u64 << rounds) + 3 * 2048 + 3) * 24; + let Some(room) = math_cuda::device::reserve(bytes) else { + return skipped(); + }; + let Some(layer) = rebuild().and_then(|whole| whole.gruen(raw_point, low).ok()) else { + return skipped(); + }; + Some(WholeLayer { layer, _room: room }) + } +} + +#[cfg(not(feature = "cuda"))] +impl DeviceTree { + pub(crate) fn gruen_session( + &self, + _layer: usize, + _point: &[math::field::element::FieldElement], + _lambda: &math::field::element::FieldElement, + _tail: usize, + ) -> Option + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } +} + +#[cfg(feature = "cuda")] +impl GruenSession { + /// The rounds the card runs, `J`. + pub(crate) fn rounds(&self) -> usize { + self.layer.rounds() + } + + /// Queues the next round — folding by `previous`, the last round's + /// challenge — without waiting for it. + pub(crate) fn enqueue( + &mut self, + previous: Option<&math::field::element::FieldElement>, + want_h0: bool, + ) -> Result<(), crate::Error> + where + E: math::field::traits::IsField + 'static, + { + if self.pending.is_none() { + let failed = || crate::Error::DeviceFailed { + stage: "gruen round", + }; + let fold_by = match previous { + Some(r) => Some(ext3_raw(r).ok_or_else(failed)?), + None => None, + }; + let rows = self + .layer + .round_enqueue(fold_by.as_ref(), &self.lambda, want_h0) + .map_err(|_| failed())?; + if let Some(whole) = self.whole.as_mut() { + whole + .layer + .round_enqueue(fold_by.as_ref(), &self.lambda, want_h0) + .map_err(|_| failed())?; + } + self.pending = Some(rows); + } + Ok(()) + } + + /// The round's sums — `H(1)`, `H(2)`, then `H(0)` when asked — the queued + /// round read back, or queued and read back now. + pub(crate) fn sums( + &mut self, + previous: Option<&math::field::element::FieldElement>, + want_h0: bool, + ) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + self.enqueue(previous, want_h0)?; + let rows = self.pending.take().unwrap_or(0); + let failed = || crate::Error::DeviceFailed { + stage: "gruen round", + }; + let mut sums = self.layer.round_collect(rows).map_err(|_| failed())?; + if let Some(whole) = self.whole.take() { + let want = whole.layer.round_collect(rows).map_err(|_| failed())?; + if crate::gkr_input_real::check_fault() + && let Some(limb) = sums.first_mut() + { + *limb = limb.wrapping_add(1); + } + if sums == want { + self.whole = Some(whole); + } else { + // Counted, and the layer finished on the whole one: the + // proof is today's either way. + crate::gkr_input_real::note_checked(true); + let WholeLayer { layer, _room } = whole; + self.layer = layer; + self._held = Some(_room); + sums = want; + } + } + SUMCHECK_ROUNDS.fetch_add(1, Ordering::Relaxed); + Ok(sums.chunks_exact(3).map(ext3_from_raw::).collect()) + } + + /// After the card's last round (challenge `previous`, `None` if it ran + /// none): the four halves `p_lo, p_hi, q_lo, q_hi` over `2^L` cells. + pub(crate) fn finish( + &mut self, + previous: Option<&math::field::element::FieldElement>, + ) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + let failed = || crate::Error::DeviceFailed { + stage: "layer values", + }; + let fold_by = match previous { + Some(r) => Some(ext3_raw(r).ok_or_else(failed)?), + None => None, + }; + let mut values = self.layer.finish(fold_by.as_ref()).map_err(|_| failed())?; + if let Some(mut whole) = self.whole.take() { + let want = whole.layer.finish(fold_by.as_ref()).map_err(|_| failed())?; + let differed = values != want; + crate::gkr_input_real::note_checked(differed); + if differed { + values = want; + } + } + let cells = 1usize << self.layer.low(); + if values.len() != 4 * cells * 3 { + return Err(failed()); + } + values + .chunks_exact(cells * 3) + .map(|half| { + crate::mle::Mle::new(half.chunks_exact(3).map(ext3_from_raw::).collect()) + }) + .collect() + } +} + +#[cfg(not(feature = "cuda"))] +impl GruenSession { + pub(crate) fn rounds(&self) -> usize { + match self.0 {} + } + + pub(crate) fn enqueue( + &mut self, + _previous: Option<&math::field::element::FieldElement>, + _want_h0: bool, + ) -> Result<(), crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn sums( + &mut self, + _previous: Option<&math::field::element::FieldElement>, + _want_h0: bool, + ) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn finish( + &mut self, + _previous: Option<&math::field::element::FieldElement>, + ) -> Result>, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } +} + +/// The epoch's columns on the card, read by everything that would otherwise +/// upload its own copy of them. +#[cfg(feature = "cuda")] +pub struct ResidentColumns(math_cuda::columns::DeviceColumns); + +/// One that could not be made. Never constructed. +#[cfg(not(feature = "cuda"))] +pub struct ResidentColumns(std::convert::Infallible); + +#[cfg(feature = "cuda")] +impl ResidentColumns { + /// Every column back on the host, in upload order: a test's view of what + /// the card holds. + #[doc(hidden)] + pub fn download(&self) -> Option>> { + self.0.download().ok() + } +} + +impl std::fmt::Debug for ResidentColumns { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ResidentColumns") + } +} + +/// Puts every column of an epoch on the card, in the order given. +/// +/// `None` when there is no device or it will not promise the room, and then +/// each caller uploads what it needs as before. +#[cfg(feature = "cuda")] +pub fn upload_columns(columns: &[&crate::mle::Mle]) -> Option +where + F: math::field::traits::IsField + 'static, +{ + use math::field::goldilocks::GoldilocksField; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_COLUMNS").is_some()) { + return None; + } + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let raw: Vec<&[u64]> = columns + .iter() + .map(|column| unsafe { + core::slice::from_raw_parts(column.evals().as_ptr() as *const u64, column.len()) + }) + .collect(); + math_cuda::columns::DeviceColumns::upload(&raw).map(ResidentColumns) +} + +#[cfg(not(feature = "cuda"))] +pub fn upload_columns(_columns: &[&crate::mle::Mle]) -> Option +where + F: math::field::traits::IsField + 'static, +{ + None +} + +/// A table's columns as they go to the card: field elements, packed narrow +/// ([`crate::narrow::NarrowColumns`]) and widened there, or the records the +/// rows come from ([`crate::records`]) and filled there. +#[derive(Clone, Copy)] +pub enum TableColumns<'a, F: math::field::traits::IsField> { + Wide(&'a [crate::mle::Mle]), + Narrow(&'a crate::narrow::NarrowColumns), + Records(&'a crate::records::Records), +} + +/// A table's records as the card's fill reads them, before they go up: the +/// fill, and a CPU table's DECODE rows on the card ([`records_input`] borrows +/// both). +#[cfg(feature = "cuda")] +struct RecordsUp<'a> { + rows: usize, + cols: usize, + fill: crate::records::RecordsFill<'a>, + decode: Option>, + logs: Vec<&'a [math_cuda::fill::LogC]>, + registers: Vec<&'a [math_cuda::fill::RegRowC]>, + aligned: Vec<&'a [math_cuda::fill::AlignedRowC]>, + loads: Vec<&'a [math_cuda::fill::LoadOpC]>, + stores: Vec<&'a [math_cuda::fill::StoreOpC]>, + shifts: Vec<&'a [math_cuda::fill::ShiftOpC]>, + memws: Vec<&'a [math_cuda::fill::MemwOpC]>, +} + +#[cfg(feature = "cuda")] +impl<'a> RecordsUp<'a> { + /// `records`' pieces as the card's structs; `None` when a CPU table's + /// DECODE rows cannot go up. + fn of(records: &'a crate::records::Records) -> Option { + use crate::records::{ + AlignedRow, CpuLog, LoadOp, MemwOp, RecordsFill, RegisterRow, ShiftOp, StoreOp, + }; + use math_cuda::fill::{AlignedRowC, LoadOpC, LogC, MemwOpC, RegRowC, ShiftOpC, StoreOpC}; + const _: () = { + use std::mem::{align_of, offset_of, size_of}; + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(CpuLog, src2_val) == offset_of!(LogC, src2_val)); + assert!(offset_of!(CpuLog, dst_val) == offset_of!(LogC, dst_val)); + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(RegisterRow, old_ts_lo) == offset_of!(RegRowC, old_ts_lo)); + assert!(offset_of!(RegisterRow, address) == offset_of!(RegRowC, address)); + assert!(offset_of!(RegisterRow, is_read) == offset_of!(RegRowC, is_read)); + assert!(offset_of!(RegisterRow, pad) == offset_of!(RegRowC, pad)); + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(AlignedRow, old) == offset_of!(AlignedRowC, old)); + assert!(offset_of!(AlignedRow, width) == offset_of!(AlignedRowC, width)); + assert!(offset_of!(AlignedRow, is_read) == offset_of!(AlignedRowC, is_read)); + assert!(offset_of!(AlignedRow, is_register) == offset_of!(AlignedRowC, is_register)); + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(LoadOp, width) == offset_of!(LoadOpC, width)); + assert!(offset_of!(LoadOp, signed) == offset_of!(LoadOpC, is_signed)); + assert!(offset_of!(LoadOp, res) == offset_of!(LoadOpC, res)); + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(StoreOp, value) == offset_of!(StoreOpC, value)); + assert!(offset_of!(StoreOp, write2) == offset_of!(StoreOpC, write2)); + assert!(offset_of!(StoreOp, write8) == offset_of!(StoreOpC, write8)); + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(ShiftOp, in_halves) == offset_of!(ShiftOpC, in_halves)); + assert!(offset_of!(ShiftOp, shift) == offset_of!(ShiftOpC, shift)); + assert!(offset_of!(ShiftOp, signed) == offset_of!(ShiftOpC, is_signed)); + assert!(offset_of!(ShiftOp, word_instr) == offset_of!(ShiftOpC, word_instr)); + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(MemwOp, old_timestamp) == offset_of!(MemwOpC, old_timestamp)); + assert!(offset_of!(MemwOp, value) == offset_of!(MemwOpC, value)); + assert!(offset_of!(MemwOp, old) == offset_of!(MemwOpC, old)); + assert!(offset_of!(MemwOp, is_register) == offset_of!(MemwOpC, is_register)); + assert!(offset_of!(MemwOp, is_read) == offset_of!(MemwOpC, is_read)); + }; + /// `pieces` read as the card's struct `C`. + /// + /// SAFETY (the callers'): `R` and `C` are `#[repr(C)]` with the same + /// fields in the same order (asserted above), so the same size, + /// alignment and offsets. + unsafe fn cast<'p, R, C>(pieces: &[&'p [R]]) -> Vec<&'p [C]> { + pieces + .iter() + .map(|p| unsafe { core::slice::from_raw_parts(p.as_ptr() as *const C, p.len()) }) + .collect() + } + let fill = records.fill(); + let mut up = Self { + rows: records.rows(), + cols: records.cols(), + fill: RecordsFill::KeccakRnd { ops: &[], skip: 0 }, + decode: None, + logs: Vec::new(), + registers: Vec::new(), + aligned: Vec::new(), + loads: Vec::new(), + stores: Vec::new(), + shifts: Vec::new(), + memws: Vec::new(), + }; + match &fill { + RecordsFill::KeccakRnd { .. } => {} + RecordsFill::Cpu { logs, decode, .. } => { + up.decode = Some(decode.on_card()?); + // SAFETY: five `u64`s in the same order (asserted above). + up.logs = unsafe { cast::(logs) }; + } + // SAFETY: the same fields in the same order (asserted above). + RecordsFill::MemwR { rows } => { + up.registers = unsafe { cast::(rows) } + } + RecordsFill::MemwA { rows } => { + up.aligned = unsafe { cast::(rows) } + } + RecordsFill::Load { ops } => up.loads = unsafe { cast::(ops) }, + RecordsFill::Store { ops } => up.stores = unsafe { cast::(ops) }, + RecordsFill::Shift { ops } => up.shifts = unsafe { cast::(ops) }, + RecordsFill::Memw { ops } => up.memws = unsafe { cast::(ops) }, + } + up.fill = fill; + Some(up) + } +} + +/// A table's records as the card's fill takes them; `None` where the card +/// fills no table of that shape. +#[cfg(feature = "cuda")] +fn records_input<'a>(up: &'a RecordsUp<'a>) -> Option> { + use crate::records::{KeccakRndOp, RecordsFill}; + use math_cuda::fill::{KeccakOpInC, RecordsInput}; + const _: () = { + assert!(std::mem::size_of::() == std::mem::size_of::()); + assert!(std::mem::align_of::() == std::mem::align_of::()); + assert!( + std::mem::offset_of!(KeccakRndOp, input) == std::mem::offset_of!(KeccakOpInC, input) + ); + }; + let input = match &up.fill { + RecordsFill::KeccakRnd { ops, skip } => RecordsInput::KeccakRnd { + // SAFETY: both are `#[repr(C)]` with the same fields in the same + // order (a timestamp, then 25 lanes, all `u64`), so the same size, + // alignment and offsets (asserted above). + ops: unsafe { + core::slice::from_raw_parts(ops.as_ptr() as *const KeccakOpInC, ops.len()) + }, + skip: *skip, + rows: up.rows, + }, + // The CPU table writes its own timestamps: the first cycle is the + // records' (their digest and host fill), not the card's. + RecordsFill::Cpu { .. } => RecordsInput::Cpu { + logs: &up.logs, + decode: up.decode.as_deref()?, + rows: up.rows, + }, + RecordsFill::MemwR { .. } => RecordsInput::MemwR { + records: &up.registers, + rows: up.rows, + }, + RecordsFill::MemwA { .. } => RecordsInput::MemwA { + records: &up.aligned, + rows: up.rows, + }, + RecordsFill::Load { .. } => RecordsInput::Load { + records: &up.loads, + rows: up.rows, + }, + RecordsFill::Store { .. } => RecordsInput::Store { + records: &up.stores, + rows: up.rows, + }, + RecordsFill::Shift { .. } => RecordsInput::Shift { + records: &up.shifts, + rows: up.rows, + }, + RecordsFill::Memw { .. } => RecordsInput::Memw { + records: &up.memws, + rows: up.rows, + }, + }; + // A KECCAK_RND table's records are ops, 24 rows each, cut at `skip`. + let fits = input.cols() == up.cols + && (matches!(input, RecordsInput::KeccakRnd { .. }) || up.fill.len() <= up.rows); + fits.then_some(input) +} + +/// [`upload_columns`] of whole tables, each as it is held: a narrow one crosses +/// the bus at its packed bytes and is widened on the card into the run its +/// columns take, and one held as records crosses it at its records' bytes and +/// is filled there from them, so the store holds the same words at the same +/// places as if its field elements had been uploaded. `None` (the card +/// declines, and every reader takes the host's columns) also when a table's +/// records are not a table the card fills, or its fill refused them. +#[cfg(feature = "cuda")] +pub fn upload_tables(tables: &[TableColumns<'_, F>]) -> Option +where + F: math::field::traits::IsField + 'static, +{ + use math::field::goldilocks::GoldilocksField; + use math_cuda::columns::TableUpload; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_COLUMNS").is_some()) { + return None; + } + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let wide: Vec> = tables + .iter() + .map(|table| match table { + TableColumns::Wide(columns) => columns + .iter() + .map(|column| unsafe { + core::slice::from_raw_parts(column.evals().as_ptr() as *const u64, column.len()) + }) + .collect(), + TableColumns::Narrow(_) | TableColumns::Records(_) => Vec::new(), + }) + .collect(); + let ups: Vec>> = tables + .iter() + .map(|table| match table { + TableColumns::Records(records) => RecordsUp::of(records).map(Some), + TableColumns::Wide(_) | TableColumns::Narrow(_) => Some(None), + }) + .collect::>()?; + let uploads: Vec> = tables + .iter() + .zip(&wide) + .zip(&ups) + .map(|((table, raw), up)| { + Some(match (table, up) { + (TableColumns::Wide(_), _) => TableUpload::Wide(raw), + (TableColumns::Narrow(packed), _) => { + TableUpload::Narrow(math_cuda::narrow::NarrowInput::new( + packed.data(), + packed.offsets(), + packed.widths(), + packed.rows(), + packed.cols(), + )) + } + (TableColumns::Records(_), up) => { + TableUpload::Records(records_input(up.as_ref()?)?) + } + }) + }) + .collect::>()?; + let store = math_cuda::columns::DeviceColumns::upload_tables(&uploads)?; + crate::records::note_card_fills( + tables + .iter() + .filter(|t| matches!(t, TableColumns::Records(_))) + .count(), + ); + Some(ResidentColumns(store)) +} + +#[cfg(not(feature = "cuda"))] +pub fn upload_tables(_tables: &[TableColumns<'_, F>]) -> Option +where + F: math::field::traits::IsField + 'static, +{ + None +} + +/// The run of `width` columns from `first` in `store`, packed narrow on the +/// card ([`crate::narrow::NarrowColumns`]); only the packed bytes come back. +/// `None` when the card cannot (they are not a run, too many words for one +/// launch, or a device error) — the table then stays as it is held. +#[cfg(feature = "cuda")] +pub fn pack_resident( + store: &ResidentColumns, + first: usize, + width: usize, +) -> Option { + if width == 0 || first >= store.0.num_columns() { + return None; + } + let rows = store.0.span(first).1; + let (widths, data) = store.0.pack_run(first, width).ok()??; + crate::narrow::NarrowColumns::from_parts(rows, widths, data) +} + +#[cfg(not(feature = "cuda"))] +pub fn pack_resident( + store: &ResidentColumns, + _first: usize, + _width: usize, +) -> Option { + match store.0 {} +} + +/// `f` over where a table's columns are, for the entry points that take +/// either: the card's run when it holds one, or else the host's columns — +/// widened for this if they are held narrow, which only a table the card does +/// not hold pays. `None` when the host's are not one height. `F` must be +/// Goldilocks (the callers check). +#[cfg(feature = "cuda")] +fn with_columns_at( + resident: Option<(&ResidentColumns, usize)>, + columns: &C, + f: impl FnOnce(math_cuda::columns::Columns<'_>) -> R, +) -> Option +where + F: math::field::traits::IsField + 'static, + C: crate::narrow::HostColumns + ?Sized, +{ + let width = columns.width(); + if let Some((store, first)) = resident + && store.0.is_run(first, width) + { + return Some(f(math_cuda::columns::Columns::Device { + store: &store.0, + first, + width, + })); + } + columns.rows()?; + let host = columns.host(); + // SAFETY: `F == GoldilocksField` (the callers check), a transparent + // wrapper over one `u64` per element. + let raw: Vec<&[u64]> = host + .iter() + .map(|column| unsafe { + core::slice::from_raw_parts(column.evals().as_ptr() as *const u64, column.len()) + }) + .collect(); + Some(f(math_cuda::columns::Columns::Host(&raw))) +} + +/// A table's factors, uploaded once for everything that walks them. +#[cfg(feature = "cuda")] +pub struct DeviceFactors(math_cuda::sumcheck::DeviceFactors); + +#[cfg(feature = "cuda")] +impl std::fmt::Debug for DeviceFactors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeviceFactors") + .field("factors", &self.0.width()) + .field("cells", &self.0.len()) + .finish() + } +} + +#[cfg(feature = "cuda")] +impl DeviceFactors { + /// Under `LAMBDA_VM_ARGUE_TREE_SYNC` (a diagnostic), wait for the kernels + /// that built the factors, so the tree's split charges the lift its own + /// card time. + pub fn tree_sync(&self) { + tree_sync!(self.0.stream()); + } + + /// The card's handle, for the fused rounds (`gpu_fused`). + pub(crate) fn inner(&self) -> &math_cuda::sumcheck::DeviceFactors { + &self.0 + } + + /// The factors held. + pub fn width(&self) -> usize { + self.0.width() + } + + /// The cube they span. + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +/// Factors a build declined to upload. Never constructed. +#[cfg(not(feature = "cuda"))] +#[derive(Debug)] +pub struct DeviceFactors(std::convert::Infallible); + +#[cfg(not(feature = "cuda"))] +impl DeviceFactors { + pub fn tree_sync(&self) { + match self.0 {} + } +} + +/// A table's factors as its base columns on the card, with no lift (D-BATCH +/// M1-2, `LAMBDA_VM_ARGUE_NO_LIFT`): the fused zerocheck's rounds 0 and 1 read +/// a committed factor where the epoch's columns reside, and a public one from a +/// base copy. Keeps the columns alive for as long as it lives. +#[cfg(feature = "cuda")] +pub struct ColumnFactors { + inner: math_cuda::sumcheck::ColumnFactors, + _columns: std::sync::Arc, +} + +#[cfg(feature = "cuda")] +impl ColumnFactors { + pub(crate) fn view(&self) -> math_cuda::sumcheck::FactorView<'_> { + self.inner.view() + } +} + +#[cfg(feature = "cuda")] +impl std::fmt::Debug for ColumnFactors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ColumnFactors").finish() + } +} + +/// Column factors a build cannot make. Never constructed. +#[cfg(not(feature = "cuda"))] +#[derive(Debug)] +pub struct ColumnFactors(std::convert::Infallible); + +/// The table's factors as [`ColumnFactors`], or `None` — then the caller lifts +/// them as today. `None` unless every committed factor reads its column +/// unshifted from a resident run (a cyclic shift is not an address), every +/// public table is base-valued, and the device would take the lifted factors. +#[cfg(feature = "cuda")] +pub fn column_factors( + width: usize, + rows: usize, + kinds: &[crate::constraint_argument::FactorKind], + public: &[crate::mle::Mle], + resident: Option<(std::sync::Arc, usize)>, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + let (store, first) = resident?; + if width == 0 + || kinds.is_empty() + || !worth_the_device(kinds.len(), rows) + || !store.0.is_run(first, width) + { + return None; + } + // A public table's base values, when nothing sits above them. + let mut public_base: Vec> = Vec::new(); + for table in public { + let mut base = Vec::with_capacity(rows); + for value in table.evals() { + let [lo, hi1, hi2] = ext3_raw(value)?; + let canonical = ext3_raw(&math::field::element::FieldElement::::from(lo))?; + if [lo, hi1, hi2] != canonical { + return None; + } + base.push(lo); + } + public_base.push(base); + } + let mut slots = Vec::with_capacity(kinds.len()); + let mut next_public = 0usize; + for kind in kinds { + match kind.source() { + Some(source) if source.offset % rows == 0 && source.column < width => { + slots.push(math_cuda::sumcheck::ColumnSlot::Column(source.column)); + } + Some(_) => return None, + None => { + slots.push(math_cuda::sumcheck::ColumnSlot::Public( + public_base.get(next_public)?, + )); + next_public += 1; + } + } + } + let inner = { + let run = store.0.view(first, width); + math_cuda::sumcheck::ColumnFactors::new(&run, rows, &slots).ok()? + }; + COLUMN_FACTOR_TABLES.fetch_add(1, Ordering::Relaxed); + crate::whir_split::bump(&crate::whir_split::NO_LIFT); + Some(ColumnFactors { + inner, + _columns: store, + }) +} + +#[cfg(not(feature = "cuda"))] +pub fn column_factors( + _width: usize, + _rows: usize, + _kinds: &[crate::constraint_argument::FactorKind], + _public: &[crate::mle::Mle], + _resident: Option<(std::sync::Arc, usize)>, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + None +} + +/// A table's factors, built on the device out of its base columns. +/// +/// A committed factor is a column read at a frame-step offset and lifted, so +/// the columns are a third of what the factors are and the lift is a gather. +/// Building them there rather than here sends the trace instead of its lift — +/// on a real proof that is most of what crosses the bus — and the host never +/// holds the extension copy at all. +#[cfg(feature = "cuda")] +pub fn upload_factors_from_columns( + columns: &C, + kinds: &[crate::constraint_argument::FactorKind], + public: &[crate::mle::Mle], + resident: Option<(&ResidentColumns, usize)>, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, + C: crate::narrow::HostColumns + ?Sized, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + let (Some(rows), width) = (columns.rows(), columns.width()) else { + return None; + }; + if width == 0 || kinds.is_empty() || !worth_the_device(kinds.len(), rows) { + return None; + } + if public.iter().any(|table| table.len() != rows) { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_FACTORS").is_some()) { + return None; + } + // Three u64 per committed factor — where its column starts in the + // concatenated columns, its shift, and the slot it fills — and the public + // tables paired with theirs. + let mut plan = Vec::with_capacity(kinds.len() * 3); + let mut public_slots = Vec::new(); + let mut next_public = 0usize; + for (slot, kind) in kinds.iter().enumerate() { + match kind.source() { + Some(source) => { + if source.column >= width { + return None; + } + plan.push((source.column * rows) as u64); + plan.push((source.offset % rows) as u64); + plan.push(slot as u64); + } + None => { + public_slots.push((slot, public.get(next_public)?)); + next_public += 1; + } + } + } + if next_public != public.len() { + return None; + } + + // SAFETY: `E == Ext3`, wrapping its limbs transparently — three `u64` per + // element. + let raw_public: Vec<(usize, &[u64])> = public_slots + .iter() + .map(|(slot, table)| { + (*slot, unsafe { + core::slice::from_raw_parts(table.evals().as_ptr() as *const u64, table.len() * 3) + }) + }) + .collect(); + + let uploaded = with_columns_at(resident, columns, |at| { + math_cuda::sumcheck::DeviceFactors::from_columns(at, &plan, &raw_public, rows, kinds.len()) + .ok() + }) + .flatten()?; + FACTOR_CALLS.fetch_add(1, Ordering::Relaxed); + Some(DeviceFactors(uploaded)) +} + +#[cfg(not(feature = "cuda"))] +pub fn upload_factors_from_columns( + _columns: &C, + _kinds: &[crate::constraint_argument::FactorKind], + _public: &[crate::mle::Mle], + _resident: Option<(&ResidentColumns, usize)>, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, + C: crate::narrow::HostColumns + ?Sized, +{ + None +} + +/// The fraction tree's input layer, written by the device from the factors it +/// already holds. +/// +/// Each interaction contributes a `p` and a `q` slab of `rows`, in the order +/// [`logup::input_layer`](crate::logup::input_layer) lays them out. +/// +/// `padding` says whether the slots the interaction count is rounded up to are +/// written as the `0/1` they are. They are, for the layer that answers its own +/// sumcheck; they are not for the one that is only folded once, where they are +/// half the memory of the widest precompiles and the fold reads them as +/// constants instead. +#[cfg(feature = "cuda")] +fn write_input_layer( + factors: &DeviceFactors, + numerators: &[crate::program::Program], + denominators: &[crate::program::Program], + padding: bool, +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + use math::field::element::FieldElement; + + let rows = factors.0.len(); + let slots = numerators.len().next_power_of_two(); + let cells = if padding { + slots * rows + } else { + numerators.len() * rows + }; + let stream = factors.0.stream().clone(); + let mut p = math_cuda::device::alloc_zeros_or_trim::(&stream, cells * 3).ok()?; + let mut q = math_cuda::device::alloc_zeros_or_trim::(&stream, cells * 3).ok()?; + + for (i, (numerator, denominator)) in numerators.iter().zip(denominators).enumerate() { + for (program, out) in [(numerator, &mut p), (denominator, &mut q)] { + let t = crate::whir_split::tick(); + let lowered = lower(program)?; + crate::whir_split::add_tick(&crate::whir_split::TREE_LOWER, t); + factors + .0 + .map_program( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + out, + i * rows, + ) + .ok()?; + } + } + if padding { + // The padding interactions: numerator zero (already), denominator one. + let one = ext3_raw(&FieldElement::::one())?; + let tail = (slots - numerators.len()) * rows; + math_cuda::sumcheck::fill_ext3(&stream, &mut q, numerators.len() * rows, tail, &one) + .ok()?; + } + Some((p, q)) +} + +/// The tree over that input layer, built without carrying the padding. +/// +/// The layer is folded once and let go; when its own sumcheck comes — last, +/// with every level above it spent — it is written again, padding and all. +/// Writing it twice costs one pass of the interactions' programs; carrying it +/// costs the memory that decides whether the widest precompiles get a device +/// at all. +#[cfg(feature = "cuda")] +pub fn input_layer_tree( + factors: std::sync::Arc, + numerators: Vec>, + denominators: Vec>, +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + let rows = factors.0.len(); + let interactions = numerators.len(); + let stream = factors.0.stream().clone(); + input_layer_tree_impl( + stream, + rows, + interactions, + lifted_writer(factors, numerators, denominators), + Build::Eager, + ) +} + +/// Writes a tree's input layer, padded to a power of two or not — the call a +/// tree makes at its build and, when it gave the layer back, for its last +/// sumcheck. +#[cfg(feature = "cuda")] +type InputWriter = std::sync::Arc Option + Send + Sync>; + +/// The writer that runs each interaction's two programs over the lifted +/// factors (`write_input_layer`). +#[cfg(feature = "cuda")] +fn lifted_writer( + factors: std::sync::Arc, + numerators: Vec>, + denominators: Vec>, +) -> InputWriter +where + E: math::field::traits::IsField + 'static, +{ + std::sync::Arc::new(move |padding| { + write_input_layer(&factors, &numerators, &denominators, padding) + }) +} + +/// The tree over a table's input layer written straight from its base +/// columns (D-BATCH M1-2, `LAMBDA_VM_ARGUE_GKR_INPUT`): one launch writes +/// every interaction's two sides from the epoch's resident columns, where +/// [`input_layer_tree`] runs two programs an interaction over the lifted +/// factors. The same cells, so the same tree, output and GKR proof. +/// +/// `plan` is [`crate::logup::input_plan`]'s, over the table's run of `width` +/// columns from `first`. `None` when that run is not resident or the card +/// declines, before anything is written. +#[cfg(feature = "cuda")] +pub fn input_layer_tree_from_columns( + resident: std::sync::Arc, + first: usize, + width: usize, + rows: usize, + plan: &crate::logup::InputPlan, +) -> Option { + tree_from_columns(resident, first, width, rows, plan, Build::Eager) +} + +/// [`input_layer_tree_from_columns`] with the output fraction left unread, so +/// its kernels stay queued and the next tree's go in behind them +/// (`LAMBDA_VM_ARGUE_ROUND_SLOT`: a bin's trees without a wait each). The +/// output is read at the consume site, [`DeviceTree::output`]. A refused +/// promise still falls to the host, counted, as the eager build's does. +#[cfg(feature = "cuda")] +pub fn input_layer_tree_from_columns_deferred( + resident: std::sync::Arc, + first: usize, + width: usize, + rows: usize, + plan: &crate::logup::InputPlan, +) -> Option { + tree_from_columns(resident, first, width, rows, plan, Build::Deferred) +} + +#[cfg(feature = "cuda")] +fn tree_from_columns( + resident: std::sync::Arc, + first: usize, + width: usize, + rows: usize, + plan: &crate::logup::InputPlan, + build: Build, +) -> Option { + if !resident.0.is_run(first, width) || plan.interactions() == 0 { + return None; + } + let be = math_cuda::device::backend().ok()?; + let stream = be.next_stream(); + let uploaded = std::sync::Arc::new( + math_cuda::gkr::InputPlan::upload( + &stream, + &plan.side_start, + &plan.terms, + &plan.coeffs, + &plan.constants, + ) + .ok()?, + ); + let interactions = plan.interactions(); + let writer_stream = stream.clone(); + let write: InputWriter = std::sync::Arc::new(move |padding| { + let view = resident.0.view(first, width); + math_cuda::gkr::input_from_columns(&writer_stream, &view, rows, &uploaded, padding).ok() + }); + let tree = input_layer_tree_impl(stream, rows, interactions, write, build)?; + INPUT_COLUMNS_TREES.fetch_add(1, Ordering::Relaxed); + crate::whir_split::bump(&crate::whir_split::TREE_FROM_COLUMNS); + Some(tree) +} + +/// The prefetch sibling of [`input_layer_tree`]: builds the tree WITHOUT +/// reading its output fraction, so its fold kernels stay in flight and overlap +/// the caller's argue instead of being waited on here. The output is read at +/// the consume site (`DeviceTree::output`), by which time the kernels have run. +/// +/// `None` unless a second tree fits WITHOUT evicting the round-2 retained +/// layers: prefetch is subordinate to argue and to retention, never displacing +/// either. The headroom is the budget's free bytes (`vram_budget_bytes` less +/// what is already reserved, which INCLUDES retention), checked against the +/// carry peak so the build's own reservations then never need to evict. +#[cfg(feature = "cuda")] +pub fn input_layer_tree_deferred( + factors: std::sync::Arc, + numerators: Vec>, + denominators: Vec>, +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + let rows = factors.0.len(); + let slots = numerators.len().next_power_of_two(); + let full = slots * rows; + // The carry peak — also what `from_device` reserves internally — so a pass + // here means none of the build's reservations displaces anything. + let need = (4 * full) as u64 * 24; + let be = math_cuda::device::backend().ok()?; + if be.vram_budget_bytes().saturating_sub(be.reserved_bytes()) < need { + return None; + } + let interactions = numerators.len(); + let stream = factors.0.stream().clone(); + input_layer_tree_impl( + stream, + rows, + interactions, + lifted_writer(factors, numerators, denominators), + Build::Prefetch, + ) +} + +/// Count a GKR tree whose whole-tree promise the budget refused on the consume +/// path — in [`GKR_TREE_REFUSALS`] and in the argue surface's +/// `math_cuda::device::device_fallbacks` — and say so as it happens. +#[cfg(feature = "cuda")] +fn note_gkr_tree_refusal(bytes: u64, cells: usize) { + let count = GKR_TREE_REFUSALS.fetch_add(1, Ordering::Relaxed) + 1; + math_cuda::device::note_device_fallback(); + eprintln!( + "[gkr] tree promise REFUSED: {} MiB for {cells} cells, {} — the host builds this \ + table's factors and tree; gkr tree refusals {count}", + bytes >> 20, + budget_headroom(), + ); +} + +/// When a tree's output fraction is read, and whether a refusal is a host +/// fallback. +#[cfg(feature = "cuda")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Build { + /// Read at the build: the eager consume path. + Eager, + /// Left for the consume site, so a bin's trees are queued without a wait + /// each (`LAMBDA_VM_ARGUE_ROUND_SLOT`); a refusal still falls to the host. + Deferred, + /// Left for the consume site of a tree built ahead of its turn; a refusal + /// only means no prefetch. + Prefetch, +} + +/// `build` says when the output fraction is read: at once, or at the consume +/// site (the one device sync left to whoever asks first); everything else is +/// identical, so the eager caller is byte-for-byte the path it always was. +/// +/// A layer with padding is kept without it under `LAMBDA_VM_ARGUE_INPUT_REAL` +/// ([`crate::gkr_input_real`]) when the card promises that tree, and otherwise +/// built as before. +#[cfg(feature = "cuda")] +fn input_layer_tree_impl( + stream: std::sync::Arc, + rows: usize, + interactions: usize, + write: InputWriter, + build: Build, +) -> Option { + use crate::gkr_input_real::{self as input_real, InputReal}; + + let full = interactions.next_power_of_two() * rows; + let real = interactions * rows; + let padded = real < full; + let wanted = padded + && !HAND_BACK_FORCED.load(Ordering::Relaxed) + && input_real::argue_input_real() != InputReal::Off; + let kept = if wanted { + truncated_tree(&stream, real, full, &write, build) + } else { + None + }; + let tree = match kept { + Some(tree) => Some(tree), + None => { + let tree = carried_or_handed_back(stream, rows, interactions, write, build); + if wanted && tree.is_some() { + input_real::note_decline(); + } + tree + } + }; + if padded && tree.is_some() { + input_real::note_padded_tree(); + } + tree +} + +/// The tree over an input layer kept without its padding +/// ([`math_cuda::gkr::DeviceFractionTree::from_truncated_input`]): the real +/// fractions written, the level above folded from them with the padding read +/// as `0/1`, and kept for the layer's Gruen rounds. Today's layer session +/// writes the whole layer again (`rebuild`), as a tree that handed it back +/// does, inside the same promise. +/// +/// `None` before anything is written when the card will not promise it, and +/// whenever the build fails: the caller then builds today's tree. +#[cfg(feature = "cuda")] +fn truncated_tree( + stream: &std::sync::Arc, + real: usize, + full: usize, + write: &InputWriter, + build: Build, +) -> Option { + let room = math_cuda::device::reserve(math_cuda::gkr::truncated_peak_bytes(real, full))?; + let num_vars = full.trailing_zeros() as usize; + let t = crate::whir_split::tick(); + let (p, q) = write(false)?; + tree_sync!(stream); + crate::whir_split::add_tick(&crate::whir_split::TREE_WRITE, t); + let t = crate::whir_split::tick(); + let tree = math_cuda::gkr::DeviceFractionTree::from_truncated_input( + stream.clone(), + p, + q, + real, + num_vars, + ) + .ok()?; + tree_sync!(stream); + crate::whir_split::add_tick(&crate::whir_split::TREE_FOLD, t); + let output = std::sync::OnceLock::new(); + if build == Build::Eager { + let t = crate::whir_split::tick(); + let _ = output.set(tree.output().ok()?); + crate::whir_split::add_tick(&crate::whir_split::TREE_OUTPUT, t); + } + let num_layers = tree.num_layers(); + let (write, stream) = (write.clone(), stream.clone()); + let rebuild = move || { + let (p, q) = write(true)?; + Some(math_cuda::gkr::InputLayer::new( + stream.clone(), + p, + q, + num_vars, + )) + }; + crate::gkr_input_real::note_kept((full - real) as u64); + TREE_CALLS.fetch_add(1, Ordering::Relaxed); + Some(DeviceTree { + tree: std::sync::Mutex::new(Some(tree)), + rebuild: Some(Box::new(rebuild)), + input_truncated: true, + num_layers, + input_num_vars: num_vars, + output, + _room: Some(room), + }) +} + +/// Today's tree: the input layer carried with its padding, or handed back and +/// written again for its last sumcheck when the card cannot carry it. +#[cfg(feature = "cuda")] +fn carried_or_handed_back( + stream: std::sync::Arc, + rows: usize, + interactions: usize, + write: InputWriter, + build: Build, +) -> Option { + let slots = interactions.next_power_of_two(); + let full = slots * rows; + let real = interactions * rows; + + // Carrying the layer cannot fail late; handing it back can, and by then + // the transcript has moved and there is no host path left. So it is + // carried whenever the card has room for it, and handed back only when + // that is what the card cannot do **and** the difference is worth the + // second chance to fail — which is the widest precompiles, where it is + // gigabytes, and nothing else. + let eager = (4 * full) as u64 * 24; + let lazy = math_cuda::gkr::padded_peak_bytes(real, full); + let carried = + math_cuda::device::reserve(eager).filter(|_| !HAND_BACK_FORCED.load(Ordering::Relaxed)); + let hand_back = HAND_BACK_FORCED.load(Ordering::Relaxed); + if !hand_back && (carried.is_some() || eager - lazy < worth_handing_back()) { + drop(carried); + let t = crate::whir_split::tick(); + let (p, q) = write(true)?; + tree_sync!(stream); + crate::whir_split::add_tick(&crate::whir_split::TREE_WRITE, t); + let t = crate::whir_split::tick(); + let tree = math_cuda::gkr::DeviceFractionTree::from_device(stream.clone(), p, q).ok()?; + tree_sync!(stream); + crate::whir_split::add_tick(&crate::whir_split::TREE_FOLD, t); + let output = std::sync::OnceLock::new(); + if build == Build::Eager { + let t = crate::whir_split::tick(); + let _ = output.set(tree.output().ok()?); + crate::whir_split::add_tick(&crate::whir_split::TREE_OUTPUT, t); + } + let num_layers = tree.num_layers(); + let input_num_vars = tree.layer_num_vars(num_layers - 1); + TREE_CALLS.fetch_add(1, Ordering::Relaxed); + return Some(DeviceTree { + tree: std::sync::Mutex::new(Some(tree)), + rebuild: None, + input_truncated: false, + num_layers, + input_num_vars, + output, + _room: None, + }); + } + + // One promise for the whole tree, held past it: the layer handed back for + // the last sumcheck is part of the same structure. + // + // ⛔ ITS REFUSAL IS A HOST FALLBACK on the consume path: `logup::resident_tree` + // gets `None` and the table's factors and whole tree are built on the host. + // So it is counted, here where it is refused, unless this is a prefetch: a + // prefetch that is refused only means no prefetch, and the consume site asks + // again. + let promise = math_cuda::gkr::padded_peak_bytes(real, full); + let Some(room) = math_cuda::device::reserve(promise) else { + if build != Build::Prefetch { + note_gkr_tree_refusal(promise, full); + } + return None; + }; + + let t = crate::whir_split::tick(); + let (p, q) = write(false)?; + tree_sync!(stream); + crate::whir_split::add_tick(&crate::whir_split::TREE_WRITE, t); + let num_vars = full.trailing_zeros() as usize; + let t = crate::whir_split::tick(); + let tree = + math_cuda::gkr::DeviceFractionTree::from_padded_input(stream.clone(), p, q, real, num_vars) + .ok()?; + tree_sync!(stream); + crate::whir_split::add_tick(&crate::whir_split::TREE_FOLD, t); + let output = std::sync::OnceLock::new(); + if build == Build::Eager { + let t = crate::whir_split::tick(); + let _ = output.set(tree.output().ok()?); + crate::whir_split::add_tick(&crate::whir_split::TREE_OUTPUT, t); + } + let num_layers = tree.num_layers(); + + let rebuild = move || { + let (p, q) = write(true)?; + Some(math_cuda::gkr::InputLayer::new( + stream.clone(), + p, + q, + num_vars, + )) + }; + + TREE_CALLS.fetch_add(1, Ordering::Relaxed); + Some(DeviceTree { + tree: std::sync::Mutex::new(Some(tree)), + rebuild: Some(Box::new(rebuild)), + input_truncated: false, + num_layers, + input_num_vars: num_vars, + output, + _room: Some(room), + }) +} + +#[cfg(not(feature = "cuda"))] +pub fn input_layer_tree( + _factors: std::sync::Arc, + _numerators: Vec>, + _denominators: Vec>, +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + None +} + +#[cfg(not(feature = "cuda"))] +pub fn input_layer_tree_from_columns( + _resident: std::sync::Arc, + _first: usize, + _width: usize, + _rows: usize, + _plan: &crate::logup::InputPlan, +) -> Option { + None +} + +#[cfg(not(feature = "cuda"))] +pub fn input_layer_tree_from_columns_deferred( + _resident: std::sync::Arc, + _first: usize, + _width: usize, + _rows: usize, + _plan: &crate::logup::InputPlan, +) -> Option { + None +} + +#[cfg(not(feature = "cuda"))] +pub fn input_layer_tree_deferred( + _factors: std::sync::Arc, + _numerators: Vec>, + _denominators: Vec>, +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + None +} + +/// The opening's two factors on a device: the weight the chain carries and the +/// message it is opening, resident across groups of rounds. +#[cfg(feature = "cuda")] +pub struct OpeningFactors { + state: OpeningState, + lowered: Lowered, + /// A commit-time out-of-domain claim not yet in the weight + /// ([`crate::commit_ood::PendingOverlay`]): its share of each round is + /// added on the host until its table is spent or the first group of + /// rounds ends, and then what is left goes into the weight where it lies + /// ([`OpeningFactors::settle_overlay`]). + overlay: Option>, +} + +/// The field a pending overlay is held in on the card's side: the cubic +/// extension, which is the only one an opening runs on a card. +#[cfg(feature = "cuda")] +type OverlayField = math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; + +/// `value` read as an element of [`OverlayField`]; `E` must be that field. +#[cfg(feature = "cuda")] +fn to_overlay_field( + value: &math::field::element::FieldElement, +) -> Option> +where + E: math::field::traits::IsField + 'static, +{ + Some(ext3_from_raw::(&ext3_raw(value)?)) +} + +/// An [`crate::commit_ood::Overlay`] in [`OverlayField`], pending: `None` +/// when `E` is not that field. +#[cfg(feature = "cuda")] +fn pending_overlay( + overlay: &crate::commit_ood::Overlay<'_, E>, +) -> Option> +where + E: math::field::traits::IsField + 'static, +{ + let point: Vec<_> = overlay + .point + .iter() + .map(to_overlay_field) + .collect::>()?; + let partial: Vec<_> = overlay + .partial + .iter() + .map(to_overlay_field) + .collect::>()?; + Some(crate::commit_ood::PendingOverlay::new( + &crate::commit_ood::Overlay { + point: &point, + scale: to_overlay_field(&overlay.scale)?, + partial, + }, + )) +} + +/// The nodes `1..=degree` in [`OverlayField`], which a pending overlay's +/// round is evaluated at. +#[cfg(feature = "cuda")] +fn overlay_nodes(degree: usize) -> Vec> { + (1..=degree) + .map(|node| math::field::element::FieldElement::::from(node as u64)) + .collect() +} + +/// `evaluations += overlay's round at the nodes`, canonical as a card's are +/// ([`canonical_limb`]). No overlay adds nothing. +#[cfg(feature = "cuda")] +fn add_overlay_round( + overlay: Option<&crate::commit_ood::PendingOverlay>, + evaluations: &mut [math::field::element::FieldElement], +) -> Result<(), crate::Error> +where + E: math::field::traits::IsField + 'static, +{ + let Some(overlay) = overlay else { + return Ok(()); + }; + let failed = |stage| crate::Error::DeviceFailed { stage }; + let extra = overlay.round(&overlay_nodes(evaluations.len()))?; + for (value, extra) in evaluations.iter_mut().zip(extra) { + let sum = to_overlay_field(value).ok_or_else(|| failed("overlay round"))? + extra; + *value = ext3_from_raw::(&ext3_raw(&sum).ok_or_else(|| failed("overlay round"))?); + } + Ok(()) +} + +/// Binds a pending overlay's next variable at the round's challenge `r`. +#[cfg(feature = "cuda")] +fn bind_overlay( + overlay: Option<&mut crate::commit_ood::PendingOverlay>, + r: &math::field::element::FieldElement, +) -> Result<(), crate::Error> +where + E: math::field::traits::IsField + 'static, +{ + match overlay { + Some(overlay) => overlay.bind(&to_overlay_field(r).ok_or(crate::Error::DeviceFailed { + stage: "overlay challenge", + })?), + None => Ok(()), + } +} + +/// Writes what is left of `overlay` into `session`'s weight, which must be +/// bound at the overlay's challenges (`session.num_vars()` its point's +/// remaining variables). +#[cfg(feature = "cuda")] +fn write_overlay( + session: &math_cuda::whir_open::OpeningSession, + overlay: &crate::commit_ood::PendingOverlay, +) -> Result<(), crate::Error> { + let failed = |stage| crate::Error::DeviceFailed { stage }; + let (point, scale) = overlay.residual(); + if point.len() != session.num_vars() { + return Err(failed("overlay width")); + } + let raw = raw_point(point).ok_or_else(|| failed("overlay point"))?; + let scale = ext3_raw(&scale).ok_or_else(|| failed("overlay scale"))?; + session + .add_scaled_eq(&raw, &scale) + .map_err(|_| failed("overlay weight")) +} + +/// Where the factors are: still in share form for the first rounds, or +/// materialised. +#[cfg(feature = "cuda")] +enum OpeningState { + /// The first `rounds` rounds run over the shares + /// ([`math_cuda::whir_open::LeanRound0`]); the tables are written only + /// once they are bound. + Lean { + lean: Box, + rounds: usize, + }, + Session(math_cuda::whir_open::OpeningSession), + /// A materialisation failed, after the transcript had moved. + Gone, +} + +/// Rounds a stacked opening runs over its shares before materialising its +/// factors (`LAMBDA_VM_WHIR_LEAN_ROUNDS`, `0..=6`; default 3 since ULTRA 129, +/// [`math_cuda::whir_open::LEAN_ROUNDS_DEFAULT`] says why; `=6` is the way +/// back). `0` materialises at full width up front, which is the path before +/// this existed; every setting proves the same values. +/// +/// The factors at full width are the weight and the lifted message, `2·2^n` +/// extension values — 1.5× the committed base codeword, plus the base staging +/// while they are built — and they grow with the stack. Bound first, they are +/// `2^(n − rounds)` wide. +#[cfg(feature = "cuda")] +pub const LEAN_ROUNDS_DEFAULT: usize = math_cuda::whir_open::LEAN_ROUNDS_DEFAULT; + +#[cfg(feature = "cuda")] +static LEAN_ROUNDS_FORCED: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(usize::MAX); + +#[cfg(feature = "cuda")] +fn lean_rounds() -> usize { + let forced = LEAN_ROUNDS_FORCED.load(Ordering::Relaxed); + if forced != usize::MAX { + return forced; + } + static SET: std::sync::OnceLock = std::sync::OnceLock::new(); + *SET.get_or_init(|| { + math_cuda::whir_open::lean_rounds_setting( + std::env::var("LAMBDA_VM_WHIR_LEAN_ROUNDS").ok().as_deref(), + ) + }) +} + +/// Overrides [`LEAN_ROUNDS_DEFAULT`] / `LAMBDA_VM_WHIR_LEAN_ROUNDS` for the +/// whole process — for a parity test that proves both ways in one binary. +/// `None` restores the environment's setting. Not for production callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_lean_rounds(rounds: Option) { + LEAN_ROUNDS_FORCED.store(rounds.unwrap_or(usize::MAX), Ordering::Relaxed); +} + +/// How a lean opening's rounds are computed (`LAMBDA_VM_WHIR_LEAN_TILES`): +/// unset, from one pass over the message and tables a tile's size smaller +/// ([`crate::lean_tiles`]), the pass one block a tile +/// ([`math_cuda::whir_open::LEAN_TILES_DEFAULT`]); `1`, the same with the pass +/// one thread a tile; `0`, over every stack position in every round, as they +/// were before; `check`, both on every round, compared ([`lean_tile_checks`]). +/// The same values every way; a stack the tile form does not take runs over +/// every position. +#[cfg(feature = "cuda")] +pub use math_cuda::whir_open::LeanTiles; + +#[cfg(feature = "cuda")] +static LEAN_TILES_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +#[cfg(feature = "cuda")] +fn lean_tiles() -> LeanTiles { + match LEAN_TILES_FORCED.load(Ordering::Relaxed) { + 1 => LeanTiles::Off, + 2 => LeanTiles::On, + 3 => LeanTiles::Wide, + 4 => LeanTiles::Check, + _ => { + static SET: std::sync::OnceLock = std::sync::OnceLock::new(); + *SET.get_or_init(|| { + math_cuda::whir_open::lean_tiles_setting( + std::env::var("LAMBDA_VM_WHIR_LEAN_TILES").ok().as_deref(), + ) + }) + } + } +} + +/// Overrides `LAMBDA_VM_WHIR_LEAN_TILES` for the whole process — for a parity +/// test that proves every way in one binary. `None` restores the +/// environment's setting. Not for production callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_lean_tiles(tiles: Option) { + LEAN_TILES_FORCED.store( + match tiles { + None => 0, + Some(LeanTiles::Off) => 1, + Some(LeanTiles::On) => 2, + Some(LeanTiles::Wide) => 3, + Some(LeanTiles::Check) => 4, + }, + Ordering::Relaxed, + ); +} + +/// The setting in effect, for a harness's banner. +#[cfg(feature = "cuda")] +pub fn lean_tiles_in_effect() -> &'static str { + match lean_tiles() { + LeanTiles::Off => "off", + LeanTiles::On => "on", + LeanTiles::Wide => "wide", + LeanTiles::Check => "check", + } +} + +#[cfg(not(feature = "cuda"))] +pub fn lean_tiles_in_effect() -> &'static str { + "off" +} + +/// Whether a device fold runs every level in one launch +/// ([`math_cuda::whir::fold_resident_fused`]) or one launch per level. `true` +/// unless `LAMBDA_VM_NO_WHIR_FUSED_FOLD` is set; the two are raw-identical. +#[cfg(feature = "cuda")] +static FUSED_FOLD_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +#[cfg(feature = "cuda")] +fn fused_fold() -> bool { + match FUSED_FOLD_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); + !*OFF.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_WHIR_FUSED_FOLD").is_some()) + } + } +} + +/// Whether a chain's first successor is encoded from the message its rounds +/// bound rather than folded from the committed codeword +/// (`LAMBDA_VM_WHIR_ENCODE_SUCCESSOR`; on by default, +/// [`math_cuda::whir::ENCODE_SUCCESSOR_DEFAULT`] says why; `=0` folds). The +/// same values either way (`math_cuda::whir::encode_successor`): the fold +/// reads the `2^(n+blowup)` base codeword, the encode a `2^(n−k)` message. +#[cfg(feature = "cuda")] +fn encode_successor() -> bool { + match ENCODE_SUCCESSOR_FORCED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + math_cuda::whir::encode_successor_setting( + std::env::var("LAMBDA_VM_WHIR_ENCODE_SUCCESSOR") + .ok() + .as_deref(), + ) + }) + } + } +} + +#[cfg(feature = "cuda")] +static ENCODE_SUCCESSOR_FORCED: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Overrides `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR` for the whole process. `None` +/// restores the environment's setting. Not for production callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_encode_successor(on: Option) { + ENCODE_SUCCESSOR_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// The same override for the fused fold. Not for production callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_fused_fold(on: Option) { + FUSED_FOLD_FORCED.store( + match on { + Some(true) => 1, + Some(false) => 2, + None => 0, + }, + Ordering::Relaxed, + ); +} + +/// Whether every successor encode declines as a card that cannot would, so a +/// test can reach the path [`note_encode_declined`] counts. +#[cfg(feature = "cuda")] +static ENCODE_DECLINE_FORCED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Makes every successor encode the card is asked for decline while `on` — +/// for the test that the fold it falls to is counted. Not for production +/// callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_encode_decline(on: bool) { + ENCODE_DECLINE_FORCED.store(on, Ordering::Relaxed); +} + +/// Whether [`open_shared`] declines as a full card would make it, so a test can +/// reach the path [`note_open_host_fallback`] counts without filling one. +#[cfg(feature = "cuda")] +static SHARED_OPEN_DECLINED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Makes every stacked opening's device factors decline while `on` — for the +/// test that the host path they fall to is counted. Not for production callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_shared_open_declined(on: bool) { + SHARED_OPEN_DECLINED.store(on, Ordering::Relaxed); +} + +/// Whether the card declines every stacked commit, re-encode and successor +/// tree (`LAMBDA_VM_NO_GPU_WHIR_COMMIT`, or a test's +/// [`force_host_whir_commit`]): the codewords and their trees are then the +/// host's. +#[cfg(feature = "cuda")] +fn whir_commit_declined() -> bool { + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + HOST_WHIR_COMMIT_FORCED.load(Ordering::Relaxed) + || *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_WHIR_COMMIT").is_some()) +} + +#[cfg(feature = "cuda")] +static HOST_WHIR_COMMIT_FORCED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Makes every stacked commit, re-encode and successor tree decline while +/// `on`, as `LAMBDA_VM_NO_GPU_WHIR_COMMIT` does for a whole process, every fold +/// of a host codeword (`LAMBDA_VM_NO_GPU_WHIR_FOLD`), and every opening of +/// host-built factors (`LAMBDA_VM_NO_GPU_OPEN`) — for a parity test whose +/// reference arm proves on the host alone, in the same binary as its card arms; +/// with [`force_shared_open_declined`] nothing of a WHIR opening reaches the +/// card. Not for production callers. +#[doc(hidden)] +#[cfg(feature = "cuda")] +pub fn force_host_whir_commit(on: bool) { + HOST_WHIR_COMMIT_FORCED.store(on, Ordering::Relaxed); +} + +/// Factors a build declined to upload. Never constructed. +#[cfg(not(feature = "cuda"))] +pub struct OpeningFactors(std::convert::Infallible); + +/// Puts the chain's two factors on a device, lifting the message on the way. +/// +/// `program` is the rule the rounds evaluate — the product of the two — which +/// the host path runs as a closure. +#[cfg(feature = "cuda")] +pub(crate) fn open_on_device( + message: &crate::mle::Mle, + weight: &crate::mle::Mle, + program: &crate::program::Program, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + if message.len() != weight.len() || message.len() < SUMCHECK_THRESHOLD { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if HOST_WHIR_COMMIT_FORCED.load(Ordering::Relaxed) + || *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_OPEN").is_some()) + || DEVICE_OPEN_DECLINED.load(Ordering::Relaxed) + { + return None; + } + let lowered = lower(program)?; + + // SAFETY: the fields are the two checked above, each wrapping its limbs + // transparently — one `u64` per base element, three per ext3. + let raw_weight = unsafe { + core::slice::from_raw_parts(weight.evals().as_ptr() as *const u64, weight.len() * 3) + }; + let raw_message = unsafe { + core::slice::from_raw_parts(message.evals().as_ptr() as *const u64, message.len()) + }; + let session = math_cuda::whir_open::OpeningSession::new(raw_weight, raw_message).ok()?; + OPEN_CALLS.fetch_add(1, Ordering::Relaxed); + Some(OpeningFactors { + state: OpeningState::Session(session), + lowered, + overlay: None, + }) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn open_on_device( + _message: &crate::mle::Mle, + _weight: &crate::mle::Mle, + _program: &crate::program::Program, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + None +} + +#[cfg(feature = "cuda")] +impl OpeningFactors { + pub(crate) fn num_vars(&self) -> usize { + match &self.state { + OpeningState::Lean { lean, .. } => lean.num_vars(), + OpeningState::Session(session) => session.num_vars(), + OpeningState::Gone => 0, + } + } + + /// Writes the lean factors out, if they are still in share form, with a + /// pending overlay in the weight: what every reader of the tables after + /// the rounds takes. + fn materialize(&mut self) -> Result<&mut math_cuda::whir_open::OpeningSession, crate::Error> { + self.settle_overlay()?; + materialize_state(&mut self.state) + } + + /// ★ A pending overlay written into the weight: the factors materialised + /// at the variables bound so far — which are the overlay's, since every + /// round binds both — and what is left of it added there + /// ([`crate::commit_ood::PendingOverlay::residual`]). Nothing pending is a + /// no-op. + fn settle_overlay(&mut self) -> Result<(), crate::Error> { + if let Some(overlay) = self.overlay.take() { + write_overlay(materialize_state(&mut self.state)?, &overlay)?; + } + Ok(()) + } + + /// One group of rounds, with the host drawing each challenge. + /// + /// The factors are folded in place and stay for the next group, which is + /// the whole point: they are the width of a stacked polynomial. A lean + /// opening runs its first rounds over the shares and materialises when + /// they are spent or the group ends — whichever comes first, since what + /// follows a group (the out-of-domain answer, the weight's update) reads + /// the tables. + pub(crate) fn rounds( + &mut self, + group: usize, + degree: usize, + mut challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Result, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + use math::field::element::FieldElement; + + let failed = |stage| crate::Error::DeviceFailed { stage }; + let mut proofs = Vec::with_capacity(group); + let mut challenges = Vec::with_capacity(group); + let mut t = Vec::with_capacity(degree * 3); + for node in 1..=degree { + t.extend_from_slice( + &ext3_raw(&FieldElement::::from(node as u64)) + .ok_or_else(|| failed("interpolation node"))?, + ); + } + if let OpeningState::Lean { lean, rounds } = &mut self.state { + // The opening's rule is `w·f`, so its rounds are degree two: the + // lean kernel evaluates that product and nothing else. + if degree != 2 { + return Err(failed("lean opening degree")); + } + while proofs.len() < group && lean.bound() < *rounds && lean.num_vars() > 0 { + let sums = lean.round(&t).map_err(|_| failed("lean round"))?; + let mut evaluations: Vec> = + sums.chunks_exact(3).map(ext3_from_raw::).collect(); + add_overlay_round(self.overlay.as_ref(), &mut evaluations)?; + let r = challenge(&evaluations); + let raw = ext3_raw(&r).ok_or_else(|| failed("challenge"))?; + lean.bind(&raw); + bind_overlay(self.overlay.as_mut(), &r)?; + proofs.push(crate::sumcheck::RoundProof { evaluations }); + challenges.push(r); + SUMCHECK_ROUNDS.fetch_add(1, Ordering::Relaxed); + } + } + let mut left = group - proofs.len(); + // The materialised rounds a pending overlay still has a share of: one + // at a time, its share added before each challenge is drawn. + let shared = self + .overlay + .as_ref() + .map_or(0, crate::commit_ood::PendingOverlay::rounds_left) + .min(left); + if shared > 0 { + let session = materialize_state(&mut self.state)?; + let lowered = &self.lowered; + let mut rounds = session + .sumcheck( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .map_err(|_| failed("opening session"))?; + for _ in 0..shared { + let sums = rounds.round(&t).map_err(|_| failed("round"))?; + let mut evaluations: Vec> = + sums.chunks_exact(3).map(ext3_from_raw::).collect(); + add_overlay_round(self.overlay.as_ref(), &mut evaluations)?; + let r = challenge(&evaluations); + let raw = ext3_raw(&r).ok_or_else(|| failed("challenge"))?; + rounds.fold(&raw).map_err(|_| failed("fold"))?; + bind_overlay(self.overlay.as_mut(), &r)?; + proofs.push(crate::sumcheck::RoundProof { evaluations }); + challenges.push(r); + SUMCHECK_ROUNDS.fetch_add(1, Ordering::Relaxed); + } + drop(rounds); + session.bound(shared); + left -= shared; + } + // The overlay's table is spent or the group is over: what is left of + // it goes into the weight before any other round reads it, and before + // the caller reads the tables. + self.settle_overlay()?; + let session = materialize_state(&mut self.state)?; + let lowered = &self.lowered; + if left > 0 { + let mut rounds = session + .sumcheck( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .map_err(|_| failed("opening session"))?; + let (more, points) = run_rounds(&mut rounds, degree, left, challenge, |_| None)?; + session.bound(left); + proofs.extend(more); + challenges.extend(points); + } + SUMCHECK_CALLS.fetch_add(1, Ordering::Relaxed); + // The factors stay on device; the caller reads them through this + // handle, not through the tables it no longer has. + Ok((proofs, challenges, Vec::new())) + } + + /// The message's value at `point`, which the out-of-domain answer needs. + pub(crate) fn evaluate_message( + &mut self, + point: &[math::field::element::FieldElement], + ) -> Result, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + let raw = raw_point(point).ok_or(crate::Error::DeviceFailed { stage: "ood point" })?; + let value = self + .materialize()? + .evaluate_message(&raw) + .map_err(|_| crate::Error::DeviceFailed { stage: "ood value" })?; + Ok(ext3_from_raw::(&value)) + } + + /// `weight += gamma · eq(point, ·)`: the weight the next group carries. + pub(crate) fn add_scaled_eq( + &mut self, + point: &[math::field::element::FieldElement], + gamma: &math::field::element::FieldElement, + ) -> Result<(), crate::Error> + where + E: math::field::traits::IsField + 'static, + { + let failed = |stage| crate::Error::DeviceFailed { stage }; + let raw = raw_point(point).ok_or_else(|| failed("weight point"))?; + let scale = ext3_raw(gamma).ok_or_else(|| failed("weight scale"))?; + self.materialize()? + .add_scaled_eq(&raw, &scale) + .map_err(|_| failed("weight")) + } + + /// Device bytes the factors hold right now — the share-form working set + /// before the lean rounds are spent, the tables after. + pub fn device_bytes(&self) -> u64 { + match &self.state { + OpeningState::Lean { lean, .. } => lean.device_bytes(), + OpeningState::Session(session) => session.device_bytes(), + OpeningState::Gone => 0, + } + } + + /// Whether the factors are still in share form. + pub fn is_lean(&self) -> bool { + matches!(self.state, OpeningState::Lean { .. }) + } + + /// ★ The successor a `levels`-level fold of `codeword` would make, encoded + /// from the message these factors' rounds bound + /// ([`math_cuda::whir_open::OpeningSession::encode_successor`]), unless + /// `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0`. `None` sends the caller to the + /// fold: the knob off, or the card declining. + pub(crate) fn encode_successor( + &mut self, + codeword: &DeviceCodeword, + levels: usize, + ) -> Option { + if !encode_successor() { + return None; + } + let encoded = if ENCODE_DECLINE_FORCED.load(Ordering::Relaxed) { + None + } else { + self.materialize() + .ok() + .and_then(|session| session.encode_successor(&codeword.0, levels).ok()) + }; + match encoded { + Some(successor) => { + ENCODED_SUCCESSORS.fetch_add(1, Ordering::Relaxed); + Some(DeviceCodeword(successor)) + } + None => { + note_encode_declined(levels); + None + } + } + } + + /// [`Self::encode_successor`] of a round-0 codeword held as its stack + /// ([`CosetSource`]): the size and the room are the stack's. + pub(crate) fn encode_successor_from( + &mut self, + source: &CosetSource, + levels: usize, + ) -> Option { + if !encode_successor() { + return None; + } + let encoded = if ENCODE_DECLINE_FORCED.load(Ordering::Relaxed) { + None + } else { + self.materialize().ok().and_then(|session| { + session + .encode_successor_sized(source.0.elements(), source.0.room(), levels) + .ok() + }) + }; + match encoded { + Some(successor) => { + ENCODED_SUCCESSORS.fetch_add(1, Ordering::Relaxed); + Some(DeviceCodeword(successor)) + } + None => { + note_encode_declined(levels); + None + } + } + } +} + +/// A point as the limbs the kernels read. +#[cfg(feature = "cuda")] +fn raw_point(point: &[math::field::element::FieldElement]) -> Option> +where + E: math::field::traits::IsField + 'static, +{ + let mut raw = Vec::with_capacity(point.len() * 3); + for coordinate in point { + raw.extend_from_slice(&ext3_raw(coordinate)?); + } + Some(raw) +} + +#[cfg(not(feature = "cuda"))] +impl OpeningFactors { + pub(crate) fn num_vars(&self) -> usize { + match self.0 {} + } + + pub(crate) fn rounds( + &mut self, + _group: usize, + _degree: usize, + _challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Result, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn evaluate_message( + &mut self, + _point: &[math::field::element::FieldElement], + ) -> Result, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn add_scaled_eq( + &mut self, + _point: &[math::field::element::FieldElement], + _gamma: &math::field::element::FieldElement, + ) -> Result<(), crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn encode_successor( + &mut self, + _codeword: &DeviceCodeword, + _levels: usize, + ) -> Option { + match self.0 {} + } + + pub(crate) fn encode_successor_from( + &mut self, + _source: &CosetSource, + _levels: usize, + ) -> Option { + match self.0 {} + } +} + +/// ★ A grouped chain's round 0 on a card (`GroupFold::Indexed`, D-ONECHAIN +/// §2.5): one opening per polynomial of the run, in slot order, each in share +/// or tile form for its lean rounds and materialised after them as a +/// per-polynomial chain's is ([`open_shared`]). +/// +/// [`rounds`](Self::rounds) runs them in LOCKSTEP — each round's message the +/// sum of theirs, one challenge binding every one — then the index rounds over +/// their bound tables laid slot by slot +/// ([`math_cuda::whir_open::OpeningSession::gather_slots`]), and hands back +/// the one opening the chain carries on. The host's grouped round 0 +/// (`whir_chain::lockstep_rounds`) step for step: the same messages, the same +/// challenges, the same tables, so the same proof. +#[cfg(feature = "cuda")] +pub struct GroupedFactors { + parts: Vec, +} + +/// A grouped round 0 a build declined to put on a card. Never constructed. +#[cfg(not(feature = "cuda"))] +pub struct GroupedFactors(std::convert::Infallible); + +/// The openings of a grouped chain's polynomials on the card, or `None` when +/// the card declines any one of them — before any challenge is drawn, so the +/// caller builds the run's factors on the host instead (counted by +/// [`note_grouped_host_fallback`] when the codewords are on the card). +/// +/// `overlays[p]`, when given, is slot `p`'s commit-time overlay +/// ([`open_shared`]). +#[cfg(feature = "cuda")] +pub(crate) fn open_grouped( + messages: &[crate::whir_chain::Stacked<'_, F>], + shares: &[Vec>], + overlays: Option<&[crate::commit_ood::Overlay<'_, E>]>, + n_stack: usize, + program: &crate::program::Program, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + if messages.len() < 2 + || messages.len() != shares.len() + || overlays.is_some_and(|overlays| overlays.len() != messages.len()) + { + return None; + } + let parts = messages + .iter() + .zip(shares) + .enumerate() + .map(|(p, (message, shares))| { + open_shared( + message, + shares, + overlays.map(|overlays| &overlays[p]), + n_stack, + program, + ) + }) + .collect::>>()?; + Some(GroupedFactors { parts }) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn open_grouped( + _messages: &[crate::whir_chain::Stacked<'_, F>], + _shares: &[Vec>], + _overlays: Option<&[crate::commit_ood::Overlay<'_, E>]>, + _n_stack: usize, + _program: &crate::program::Program, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + None +} + +#[cfg(feature = "cuda")] +impl GroupedFactors { + /// Round 0 of the grouped chain: `data_rounds` rounds over the + /// polynomials in lockstep, then `index_vars` index rounds. Returns the + /// `data_rounds + index_vars` messages, the challenges (the data ones, + /// then `ρ`), and the opening every later round runs on — the run's + /// tables combined at `ρ`, `Σ_p eq(ρ, p)·(w_p, f_p)`, `2^(n − data_rounds)` + /// wide. + /// + /// A part runs its lean rounds while it has them and then its + /// materialised session's, each round one launch a part and one wait, + /// the message summed here. Past the first round the transcript has + /// moved, so every failure is an error. + pub(crate) fn rounds( + self, + data_rounds: usize, + index_vars: usize, + mut challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Result, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + use math::field::element::FieldElement; + + let failed = |stage| crate::Error::DeviceFailed { stage }; + let Self { mut parts } = self; + let lowered = parts + .first() + .map(|part| part.lowered.clone()) + .ok_or_else(|| failed("grouped round 0 without a polynomial"))?; + // The opening's rule is `w·f`: its rounds are degree two, sent at the + // nodes 1 and 2, as every opening's are. + let degree = 2; + let mut t = Vec::with_capacity(degree * 3); + for node in 1..=degree { + t.extend_from_slice( + &ext3_raw(&FieldElement::::from(node as u64)) + .ok_or_else(|| failed("interpolation node"))?, + ); + } + let mut sessions: Vec> = + parts.iter().map(|_| None).collect(); + let mut folded = vec![0usize; parts.len()]; + let mut proofs = Vec::with_capacity(data_rounds + index_vars); + let mut challenges = Vec::with_capacity(data_rounds + index_vars); + for _ in 0..data_rounds { + let mut sums = vec![FieldElement::::zero(); degree]; + for ((part, session), folds) in parts + .iter_mut() + .zip(sessions.iter_mut()) + .zip(folded.iter_mut()) + { + // A part's commit-time overlay whose table is spent goes into + // its weight before its next round: its session is let go, + // its tables told how far the rounds got, and a fresh session + // takes them from there. + if let Some(overlay) = part.overlay.take_if(|o| o.rounds_left() == 0) { + *session = None; + let tables = materialize_state(&mut part.state)?; + tables.bound(core::mem::take(folds)); + write_overlay(tables, &overlay)?; + } + let lean = match &mut part.state { + OpeningState::Lean { lean, rounds } + if lean.bound() < *rounds && lean.num_vars() > 0 => + { + Some(lean.round(&t).map_err(|_| failed("grouped lean round"))?) + } + _ => None, + }; + let raw = match lean { + Some(raw) => raw, + None => { + if session.is_none() { + let tables = materialize_state(&mut part.state)?; + *session = Some( + tables + .sumcheck( + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .map_err(|_| failed("grouped opening session"))?, + ); + } + session + .as_mut() + .ok_or_else(|| failed("grouped opening session"))? + .round(&t) + .map_err(|_| failed("grouped round"))? + } + }; + let mut part_sums: Vec> = + raw.chunks_exact(3).map(ext3_from_raw::).collect(); + add_overlay_round(part.overlay.as_ref(), &mut part_sums)?; + for (sum, value) in sums.iter_mut().zip(part_sums) { + *sum += value; + } + } + let r = challenge(&sums); + let raw = ext3_raw(&r).ok_or_else(|| failed("challenge"))?; + for ((part, session), folds) in parts + .iter_mut() + .zip(sessions.iter_mut()) + .zip(folded.iter_mut()) + { + match (session.as_mut(), &mut part.state) { + (Some(session), _) => { + session.fold(&raw).map_err(|_| failed("grouped fold"))?; + *folds += 1; + } + (None, OpeningState::Lean { lean, .. }) => lean.bind(&raw), + (None, _) => return Err(failed("grouped factors")), + } + bind_overlay(part.overlay.as_mut(), &r)?; + } + proofs.push(crate::sumcheck::RoundProof { evaluations: sums }); + challenges.push(r); + SUMCHECK_ROUNDS.fetch_add(1, Ordering::Relaxed); + } + // The sessions' clones of the tables go first, so the gather can + // write into the first part's own allocation. + drop(sessions); + let mut tables = Vec::with_capacity(parts.len()); + for (part, folds) in parts.into_iter().zip(folded) { + let mut state = part.state; + let session = materialize_state(&mut state)?; + session.bound(folds); + // What is left of a part's overlay, into its weight before the + // slots are gathered: the index rounds read the weights whole. + if let Some(overlay) = &part.overlay { + write_overlay(session, overlay)?; + } + match state { + OpeningState::Session(session) => tables.push(session), + _ => return Err(failed("grouped factors")), + } + } + let gathered = math_cuda::whir_open::OpeningSession::gather_slots(tables, index_vars) + .map_err(|_| failed("grouped slots"))?; + let mut combined = OpeningFactors { + state: OpeningState::Session(gathered), + lowered, + overlay: None, + }; + let (more, rho, _) = combined.rounds(index_vars, degree, &mut challenge)?; + proofs.extend(more); + challenges.extend(rho); + GROUPED_CARD_CHAINS.fetch_add(1, Ordering::Relaxed); + Ok((proofs, challenges, combined)) + } +} + +#[cfg(not(feature = "cuda"))] +impl GroupedFactors { + pub(crate) fn rounds( + self, + _data_rounds: usize, + _index_vars: usize, + _challenge: impl FnMut( + &[math::field::element::FieldElement], + ) -> math::field::element::FieldElement, + ) -> Result, crate::Error> + where + E: math::field::traits::IsField + 'static, + { + match self.0 {} + } +} + +/// `Σ_p scales[p]·parts[p]` on the card: a grouped round-0 successor from its +/// polynomials' folds ([`math_cuda::whir::combine_codewords`]). `None` when +/// the card declines, before anything is drawn from it. +#[cfg(feature = "cuda")] +pub(crate) fn combine_codewords( + parts: &[&DeviceCodeword], + scales: &[math::field::element::FieldElement], +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + let raw: Vec<[u64; 3]> = scales.iter().map(ext3_raw).collect::>()?; + let inner: Vec<&math_cuda::whir::DeviceCodeword> = parts.iter().map(|part| &part.0).collect(); + let combined = math_cuda::whir::combine_codewords(&inner, &raw).ok()?; + GROUPED_COMBINES.fetch_add(1, Ordering::Relaxed); + Some(DeviceCodeword(combined)) +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn combine_codewords( + parts: &[&DeviceCodeword], + _scales: &[math::field::element::FieldElement], +) -> Option +where + E: math::field::traits::IsField + 'static, +{ + match parts.first() { + Some(part) => match part.0 {}, + None => None, + } +} + +/// The same with the weight written on device from its shares: a stacked +/// polynomial's weight is as wide as the polynomial, and sending it would cost +/// more than the rounds that read it. +/// +/// `overlay`, a commit-time out-of-domain claim +/// ([`crate::commit_ood::Overlay`]), is never written at the stack's width: +/// its share of each of the first rounds is added on the host, from its +/// partial table, and what is left goes into the weight once the table is +/// spent or the first group of rounds ends, at `2^(n − bound)` +/// ([`OpeningFactors::settle_overlay`]). The lean rounds stop where its table +/// does. A table of no variables (a column filling the stack) is written at +/// once, here, before any challenge is drawn. +#[cfg(feature = "cuda")] +pub(crate) fn open_shared( + message: &crate::whir_chain::Stacked<'_, F>, + shares: &[crate::stacked_eval::WeightShare<'_, E>], + overlay: Option<&crate::commit_ood::Overlay<'_, E>>, + n_stack: usize, + program: &crate::program::Program, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + let len = 1usize << message.num_vars(); + if message.num_vars() != n_stack || len < SUMCHECK_THRESHOLD { + return None; + } + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_OPEN").is_some()) + || SHARED_OPEN_DECLINED.load(Ordering::Relaxed) + { + return None; + } + let lowered = lower(program)?; + let shares: Vec<(usize, Vec, [u64; 3])> = shares + .iter() + .map(|share| { + Some(( + share.offset, + raw_point(share.point)?, + ext3_raw(&share.scale)?, + )) + }) + .collect::>()?; + + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let raw = |table: &crate::mle::Mle| unsafe { + core::slice::from_raw_parts(table.evals().as_ptr() as *const u64, table.len()) + }; + // The columns go straight into the device buffer: what the host would + // assemble first is a copy of the bytes about to be sent. + if message + .parts + .iter() + .any(|(column, offset)| offset + column.rows() > len) + { + return None; + } + // Read on the host only when the card does not hold the columns: a + // narrow table's are widened there otherwise. + let host_parts: Vec<(&[u64], usize)> = match message.resident { + Some(_) => Vec::new(), + None => message + .parts + .iter() + .map(|(column, offset)| (raw(column.host()), *offset)) + .collect(), + }; + let overlay = match overlay { + Some(overlay) if overlay.point.len() == n_stack && !overlay.partial.is_empty() => { + Some(pending_overlay(overlay)?) + } + Some(_) => return None, + None => None, + }; + // ★ The first rounds over the shares, when the device takes them: the + // factors are then written at `2^(n − rounds)` instead of `2^n`. The same + // values either way; a stack the lean path cannot read materialises as + // before, before any challenge is drawn. + let rounds = lean_rounds() + .min(n_stack) + .min(overlay.as_ref().map_or(usize::MAX, |o| o.rounds_left())); + if rounds > 0 + && let Some(lean) = lean_opening(&shares, n_stack, message, &host_parts, rounds) + { + OPEN_CALLS.fetch_add(1, Ordering::Relaxed); + LEAN_OPEN_CALLS.fetch_add(1, Ordering::Relaxed); + return Some(OpeningFactors { + state: OpeningState::Lean { + lean: Box::new(lean), + rounds, + }, + lowered, + overlay, + }); + } + let session = match &message.resident { + Some((store, parts)) => math_cuda::whir_open::OpeningSession::from_shares_and_resident( + &shares, len, &store.0, parts, + ), + None => { + math_cuda::whir_open::OpeningSession::from_shares_and_parts(&shares, len, &host_parts) + } + } + .ok()?; + // An overlay with no rounds to take a share of goes in now, while a + // refusal can still send the opening to the host. + if let Some(overlay) = overlay.as_ref().filter(|o| o.rounds_left() == 0) { + write_overlay(&session, overlay).ok()?; + } + let overlay = overlay.filter(|o| o.rounds_left() > 0); + OPEN_CALLS.fetch_add(1, Ordering::Relaxed); + Some(OpeningFactors { + state: OpeningState::Session(session), + lowered, + overlay, + }) +} + +/// The factors as tables: written out of share form if they still are. Past +/// the first round the transcript has moved, so a failure here is final. +#[cfg(feature = "cuda")] +fn materialize_state( + state: &mut OpeningState, +) -> Result<&mut math_cuda::whir_open::OpeningSession, crate::Error> { + if let OpeningState::Lean { .. } = state { + let OpeningState::Lean { lean, .. } = core::mem::replace(state, OpeningState::Gone) else { + unreachable!("matched above") + }; + let session = (*lean) + .materialize() + .map_err(|_| crate::Error::DeviceFailed { + stage: "lean materialize", + })?; + *state = OpeningState::Session(session); + } + match state { + OpeningState::Session(session) => Ok(session), + _ => Err(crate::Error::DeviceFailed { + stage: "opening factors", + }), + } +} + +/// The lean opening of a stacked polynomial: its shares with their points' +/// `eq` tables in halves (computed here, a few kilobytes each), and the +/// message where it already is. `None` when the stack is not one it can read. +/// +/// Its `rounds` rounds run in tile form when the stack takes it +/// ([`crate::lean_tiles::plan`]), which is the default; in share form under +/// `LAMBDA_VM_WHIR_LEAN_TILES=0` and for a stack the tile form does not take. +#[cfg(feature = "cuda")] +fn lean_opening( + shares: &[(usize, Vec, [u64; 3])], + n_stack: usize, + message: &crate::whir_chain::Stacked<'_, F>, + host_parts: &[(&[u64], usize)], + rounds: usize, +) -> Option +where + F: math::field::traits::IsField + 'static, +{ + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + let source = || match &message.resident { + Some((store, parts)) => math_cuda::whir_open::LeanMessage::Resident { + store: &store.0, + parts, + }, + None => math_cuda::whir_open::LeanMessage::Parts(host_parts), + }; + let tiles = lean_tiles(); + if tiles != LeanTiles::Off + && let Some(plan) = crate::lean_tiles::plan(shares, n_stack, rounds) + { + let tile_shares: Vec = plan + .shares + .iter() + .map(|share| math_cuda::whir_open::LeanShare { + stack_offset: share.stack_offset, + num_vars: share.num_vars, + lo_bits: plan.tile_bits, + hi_at: share.hi_at, + lo_at: share.lo_at, + scale: share.scale, + }) + .collect(); + let spec = math_cuda::whir_open::TileSpec { + tile_bits: plan.tile_bits, + lean_rounds: rounds, + slots: plan.slots, + wide: tiles == LeanTiles::Wide, + check: tiles == LeanTiles::Check, + }; + // A decline here — the device's own reading of the stack, or a card + // that will not hold the form — opens in share form: no challenge + // has been drawn. + if let Ok(Some(lean)) = math_cuda::whir_open::LeanRound0::with_tiles( + &tile_shares, + &plan.eq, + n_stack, + source(), + Some(spec), + ) { + LEAN_TILE_OPENS.fetch_add(1, Ordering::Relaxed); + return Some(lean); + } + } + + // Sorted by offset: the map from a stack position to its column is a + // binary search over them. + let mut order: Vec = (0..shares.len()).collect(); + order.sort_by_key(|i| shares[*i].0); + let mut lean_shares = Vec::with_capacity(shares.len()); + let mut eq: Vec = Vec::new(); + // A table's columns share its point and sit next to each other: one pair + // of half tables per run of equal points. + let mut last: Option<(&[u64], usize, usize, usize)> = None; + for &i in &order { + let (offset, point, scale) = &shares[i]; + let num_vars = point.len() / 3; + let lo_bits = num_vars / 2; + let (hi_at, lo_at) = match last { + Some((p, lo, hi_at, lo_at)) if p == point.as_slice() && lo == lo_bits => (hi_at, lo_at), + _ => { + let coordinates: Vec> = + point.chunks_exact(3).map(ext3_from_raw::).collect(); + let (hi, lo) = coordinates.split_at(num_vars - lo_bits); + let hi_at = eq.len() / 3; + for value in crate::eq::eq_evals(hi) { + eq.extend_from_slice(&ext3_raw(&value)?); + } + let lo_at = eq.len() / 3; + for value in crate::eq::eq_evals(lo) { + eq.extend_from_slice(&ext3_raw(&value)?); + } + (hi_at, lo_at) + } + }; + last = Some((point.as_slice(), lo_bits, hi_at, lo_at)); + lean_shares.push(math_cuda::whir_open::LeanShare { + stack_offset: *offset, + num_vars, + lo_bits, + hi_at, + lo_at, + scale: *scale, + }); + } + math_cuda::whir_open::LeanRound0::new(&lean_shares, &eq, n_stack, source()) + .ok() + .flatten() +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn open_shared( + _message: &crate::whir_chain::Stacked<'_, F>, + _shares: &[crate::stacked_eval::WeightShare<'_, E>], + _overlay: Option<&crate::commit_ood::Overlay<'_, E>>, + _n_stack: usize, + _program: &crate::program::Program, +) -> Option +where + F: math::field::traits::IsField + 'static, + E: math::field::traits::IsField + 'static, +{ + None +} + +/// A round-0 stack the card holds, which a revived coset-ordered commitment +/// opens from: each queried coset evaluated from the polynomial instead of a +/// re-encoded codeword (`math_cuda::whir_coset`). +#[cfg(feature = "cuda")] +pub struct CosetSource(math_cuda::whir_coset::StackSlices); + +/// A stack a build without a device never holds. Never constructed. +#[cfg(not(feature = "cuda"))] +pub struct CosetSource(std::convert::Infallible); + +/// A coset source's openings: one tree over its gathered blocks (block `k` = +/// the leaves `[k·2^kept, (k+1)·2^kept)`, `leaves` in all with the padding), +/// each block's node's position among the kept nodes, and per query its block, +/// its slot in it and its raw values. +pub(crate) struct SourceOpenings { + pub nodes: Vec<[u8; 32]>, + pub leaves: usize, + pub positions: Vec, + pub slots: Vec<(usize, usize)>, + pub values: Vec>>, +} + +#[cfg(feature = "cuda")] +impl CosetSource { + /// The stack `parts` (`(column, offset)` into `store`) make. `None` when + /// the card declines. + pub(crate) fn from_resident( + store: &ResidentColumns, + parts: &[(usize, usize)], + num_vars: usize, + log_blowup: usize, + ) -> Option { + math_cuda::whir_coset::StackSlices::from_resident(&store.0, parts, num_vars, log_blowup) + .ok() + .map(Self) + } + + /// Values the codeword would have. + pub(crate) fn elements(&self) -> usize { + self.0.elements() + } + + /// The openings of `leaves` (leaf indices) under kept nodes of `2^kept` + /// leaves. `None` when the plan is not one the kernels take or the card + /// declines: the caller re-encodes ([`Self::encode`]). + pub(crate) fn openings( + &self, + log_folding: usize, + kept: usize, + leaves: &[usize], + hash: crate::whir_hash::DeviceHashKey, + ) -> Option> + where + F: math::field::traits::IsField + 'static, + { + let raw: Vec = leaves.iter().map(|&leaf| leaf as u64).collect(); + let plan = math_cuda::whir_coset::CosetPlan::new( + self.0.num_vars(), + self.0.log_blowup(), + log_folding, + kept, + &raw, + )?; + let in_place = coset_inplace_in_effect(); + let opened = math_cuda::whir_coset::coset_openings( + &self.0, + &plan, + &raw, + hash.into_math_cuda(), + in_place, + ) + .ok()?; + match opened.in_place { + Some(kinds) => { + COSET_INPLACE_OPENS.fetch_add(1, Ordering::Relaxed); + COSET_ZERO_TILES.fetch_add(kinds.zero, Ordering::Relaxed); + COSET_MIXED_TILES.fetch_add(kinds.mixed, Ordering::Relaxed); + } + None if in_place => { + COSET_INPLACE_ASSEMBLED.fetch_add(1, Ordering::Relaxed); + } + None => {} + } + let slots = raw + .iter() + .map(|&leaf| plan.slot_of(leaf)) + .collect::>>()?; + let block = 1usize << log_folding; + let values = values_from_raw::(&opened.values, true)?; + COSET_OPENINGS.fetch_add(1, Ordering::Relaxed); + Some(SourceOpenings { + nodes: nodes_in_place(opened.nodes)?, + leaves: opened.leaves, + positions: (0..plan.classes().len()) + .map(|q| plan.block_position(q)) + .collect(), + slots, + values: values.chunks_exact(block).map(<[_]>::to_vec).collect(), + }) + } + + /// The whole codeword, re-encoded: the fallback. Counted. + pub(crate) fn encode(&self) -> Option { + note_coset_fallback(); + self.0.encode(false).ok().map(DeviceCodeword) + } +} + +#[cfg(not(feature = "cuda"))] +impl CosetSource { + pub(crate) fn from_resident( + _store: &ResidentColumns, + _parts: &[(usize, usize)], + _num_vars: usize, + _log_blowup: usize, + ) -> Option { + None + } + + pub(crate) fn elements(&self) -> usize { + match self.0 {} + } + + pub(crate) fn openings( + &self, + _log_folding: usize, + _kept: usize, + _leaves: &[usize], + _hash: crate::whir_hash::DeviceHashKey, + ) -> Option> + where + F: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn encode(&self) -> Option { + match self.0 {} + } +} + +/// A codeword the device holds: the commit leaves one there and the chain +/// folds it there, so the array itself never crosses the bus. +#[cfg(feature = "cuda")] +pub struct DeviceCodeword(math_cuda::whir::DeviceCodeword); + +/// A codeword a commit declined to keep. Never constructed. +#[cfg(not(feature = "cuda"))] +pub struct DeviceCodeword(std::convert::Infallible); + +#[cfg(feature = "cuda")] +impl std::fmt::Debug for DeviceCodeword { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeviceCodeword") + .field("elements", &self.0.elements()) + .field("base", &self.0.is_base()) + .finish() + } +} + +#[cfg(not(feature = "cuda"))] +impl std::fmt::Debug for DeviceCodeword { + fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.0 {} + } +} + +/// The same, over a stacked polynomial given as the columns it is made of. +/// +/// The parts go straight into the device buffer, so what the host would have +/// assembled first — a copy of every byte about to be uploaded — is never +/// built. `parts` is `(column, offset in elements)`. +#[cfg(feature = "cuda")] +#[allow(clippy::too_many_arguments)] +pub(crate) fn commit_parts( + parts: &[(&dyn crate::narrow::HostColumn, usize)], + log_evals: usize, + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::whir_hash::DeviceHashKey, + leaves: crate::whir_commit::LeafOrder, +) -> Option<(DeviceCodeword, [u8; 32])> +where + F: math::field::traits::IsField + 'static, +{ + use math::field::goldilocks::GoldilocksField; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + if (1usize << log_evals) << log_blowup < COMMIT_THRESHOLD { + return None; + } + if whir_commit_declined() { + return None; + } + // A part past the end would be an out-of-bounds device write. + if parts + .iter() + .any(|(column, offset)| offset + column.rows() > (1usize << log_evals)) + { + return None; + } + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let raw: Vec<(&[u64], usize)> = parts + .iter() + .map(|(column, offset)| unsafe { + let column = column.host(); + ( + core::slice::from_raw_parts(column.evals().as_ptr() as *const u64, column.len()), + *offset, + ) + }) + .collect(); + // No device is a decline, not an error: the device is asked only once one + // exists (a GPU-less `cuda` build fails here on every commit). + math_cuda::device::backend().ok()?; + let (codeword, root) = match math_cuda::whir::commit_codeword_parts_placed( + &raw, + log_evals, + log_blowup, + log_folding, + transient, + hash.into_math_cuda(), + placement(leaves), + ) { + Ok(committed) => committed, + Err(err) => { + note_commit_error("parts", log_evals, log_blowup, &err); + return None; + } + }; + COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); + Some((DeviceCodeword(codeword), root)) +} + +/// The tree placement of a round-0 leaf order on the card. +#[cfg(feature = "cuda")] +fn placement(leaves: crate::whir_commit::LeafOrder) -> math_cuda::whir::LeafPlacement { + match leaves { + crate::whir_commit::LeafOrder::Natural => math_cuda::whir::LeafPlacement::Natural, + crate::whir_commit::LeafOrder::Coset => math_cuda::whir::LeafPlacement::Reversed, + } +} + +/// The same for parts the card already holds. +#[cfg(feature = "cuda")] +#[allow(clippy::too_many_arguments)] +pub(crate) fn commit_resident( + store: &ResidentColumns, + parts: &[(usize, usize)], + log_evals: usize, + log_blowup: usize, + log_folding: usize, + transient: bool, + hash: crate::whir_hash::DeviceHashKey, + leaves: crate::whir_commit::LeafOrder, +) -> Option<(DeviceCodeword, [u8; 32])> { + if (1usize << log_evals) << log_blowup < COMMIT_THRESHOLD { + return None; + } + if whir_commit_declined() { + return None; + } + // No device is a decline, not an error (see `commit_parts`). + math_cuda::device::backend().ok()?; + let (codeword, root) = match math_cuda::whir::commit_codeword_resident_placed( + &store.0, + parts, + log_evals, + log_blowup, + log_folding, + transient, + hash.into_math_cuda(), + placement(leaves), + ) { + Ok(committed) => committed, + Err(err) => { + note_commit_error("resident", log_evals, log_blowup, &err); + return None; + } + }; + COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); + Some((DeviceCodeword(codeword), root)) +} + +/// The codeword [`commit_resident`] would commit, with no tree built: what +/// the block prover's phase B rebuilds for an opening whose root it already +/// holds. Declines exactly where [`commit_resident`] does, so a stack committed +/// on the card is re-encoded on the card. +#[cfg(feature = "cuda")] +pub(crate) fn encode_resident( + store: &ResidentColumns, + parts: &[(usize, usize)], + log_evals: usize, + log_blowup: usize, + transient: bool, +) -> Option { + if (1usize << log_evals) << log_blowup < COMMIT_THRESHOLD { + return None; + } + if whir_commit_declined() { + return None; + } + math_cuda::device::backend().ok()?; + match math_cuda::whir::encode_codeword_resident( + &store.0, parts, log_evals, log_blowup, transient, + ) { + Ok(codeword) => Some(DeviceCodeword(codeword)), + Err(err) => { + note_commit_error("encode resident", log_evals, log_blowup, &err); + None + } + } +} + +/// [`encode_resident`] over columns the host holds, as [`commit_parts`]. +#[cfg(feature = "cuda")] +pub(crate) fn encode_parts( + parts: &[(&dyn crate::narrow::HostColumn, usize)], + log_evals: usize, + log_blowup: usize, + transient: bool, +) -> Option +where + F: math::field::traits::IsField + 'static, +{ + use math::field::goldilocks::GoldilocksField; + + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + if (1usize << log_evals) << log_blowup < COMMIT_THRESHOLD { + return None; + } + if whir_commit_declined() { + return None; + } + if parts + .iter() + .any(|(column, offset)| offset + column.rows() > (1usize << log_evals)) + { + return None; + } + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let raw: Vec<(&[u64], usize)> = parts + .iter() + .map(|(column, offset)| unsafe { + let column = column.host(); + ( + core::slice::from_raw_parts(column.evals().as_ptr() as *const u64, column.len()), + *offset, + ) + }) + .collect(); + math_cuda::device::backend().ok()?; + match math_cuda::whir::encode_codeword_parts(&raw, log_evals, log_blowup, transient) { + Ok(codeword) => Some(DeviceCodeword(codeword)), + Err(err) => { + note_commit_error("encode parts", log_evals, log_blowup, &err); + None + } + } +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn encode_resident( + _store: &ResidentColumns, + _parts: &[(usize, usize)], + _log_evals: usize, + _log_blowup: usize, + _transient: bool, +) -> Option { + None +} + +#[cfg(not(feature = "cuda"))] +pub(crate) fn encode_parts( + _parts: &[(&dyn crate::narrow::HostColumn, usize)], + _log_evals: usize, + _log_blowup: usize, + _transient: bool, +) -> Option +where + F: math::field::traits::IsField + 'static, +{ + None +} + +#[cfg(not(feature = "cuda"))] +#[allow(clippy::too_many_arguments)] +pub(crate) fn commit_resident( + _store: &ResidentColumns, + _parts: &[(usize, usize)], + _log_evals: usize, + _log_blowup: usize, + _log_folding: usize, + _transient: bool, + _hash: crate::whir_hash::DeviceHashKey, + _leaves: crate::whir_commit::LeafOrder, +) -> Option<(DeviceCodeword, [u8; 32])> { + None +} + +#[cfg(not(feature = "cuda"))] +#[allow(clippy::too_many_arguments)] +pub(crate) fn commit_parts( + _parts: &[(&dyn crate::narrow::HostColumn, usize)], + _log_evals: usize, + _log_blowup: usize, + _log_folding: usize, + _transient: bool, + _hash: crate::whir_hash::DeviceHashKey, + _leaves: crate::whir_commit::LeafOrder, +) -> Option<(DeviceCodeword, [u8; 32])> +where + F: math::field::traits::IsField + 'static, +{ + None +} + +#[cfg(feature = "cuda")] +impl DeviceCodeword { + pub(crate) fn elements(&self) -> usize { + self.0.elements() + } + + /// Folds it `alphas.len()` times, leaving the result on device too. + pub(crate) fn fold( + &self, + generator: &math::field::element::FieldElement, + alphas: &[math::field::element::FieldElement], + ) -> Option + where + F: math::field::traits::IsField + 'static, + N: math::field::traits::IsField + 'static, + { + let (two_inv, g_invs, raw_alphas) = fold_scalars(generator, alphas)?; + // ★ Every level in one launch when it fits: the level-by-level fold + // holds a 2.25× codeword transient beside the committed codeword, the + // fused one only its output. Raw-identical either way. + let folded = if fused_fold() && g_invs.len() <= math_cuda::whir::FUSED_MAX_FOLD { + let folded = + math_cuda::whir::fold_resident_fused(&self.0, two_inv, &g_invs, &raw_alphas) + .ok()?; + FUSED_FOLD_CALLS.fetch_add(1, Ordering::Relaxed); + folded + } else { + math_cuda::whir::fold_resident(&self.0, two_inv, &g_invs, &raw_alphas).ok()? + }; + RESIDENT_FOLD_CALLS.fetch_add(1, Ordering::Relaxed); + Some(Self(folded)) + } + + /// The root of the tree over its fold blocks. + /// + /// The tree is kept past this call, inside the codeword's promise and + /// evictable — whole by default, or its leaf layer under + /// `LFM_WHIR_WHOLE_TREES=0` — because the only other thing a proof wants + /// from it is a path per query, which [`paths`](Self::paths) reads when the + /// queries are known (see `math_cuda::whir::DeviceCodeword::commit`). + pub(crate) fn commit( + &self, + log_folding: usize, + hash: crate::whir_hash::DeviceHashKey, + ) -> Option<[u8; 32]> { + let root = self.0.commit(log_folding, hash.into_math_cuda()).ok()?; + COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); + Some(root) + } + + /// Waits for this codeword's stream (see + /// `math_cuda::whir::DeviceCodeword::synchronize`). + pub(crate) fn synchronize(&self) -> bool { + self.0.synchronize().is_ok() + } + + /// The tree's first `nodes` nodes (its top levels, root first), brought + /// home — what a commitment that lets its codeword go keeps. + pub(crate) fn top_nodes( + &self, + log_folding: usize, + nodes: usize, + hash: crate::whir_hash::DeviceHashKey, + ) -> Option> { + let bytes = self + .0 + .top_nodes_to_host(log_folding, hash.into_math_cuda(), nodes) + .ok()?; + nodes_in_place(bytes) + } + + /// One authentication path per index, against the same tree. + pub(crate) fn paths( + &self, + log_folding: usize, + indices: &[usize], + hash: crate::whir_hash::DeviceHashKey, + ) -> Option>> { + let leaves = self.0.elements() >> log_folding; + if indices.iter().any(|index| *index >= leaves) { + return None; + } + let positions: Vec = indices.iter().map(|index| *index as u32).collect(); + let bytes = self + .0 + .paths(log_folding, &positions, hash.into_math_cuda()) + .ok()?; + let nodes = nodes_in_place(bytes)?; + Some( + nodes + .chunks_exact(path_len(leaves, hash)) + .map(<[_]>::to_vec) + .collect(), + ) + } + + /// [`paths`](Self::paths) and the tree's Merkle cap at `cap_height` (its + /// `2^cap_height` nodes that height below the root, left to right), both + /// from ONE rebuild of the tree — so the cap and the paths are of the same + /// tree, whether its leaf layer was hashed or served from retention. + pub(crate) fn paths_and_cap( + &self, + log_folding: usize, + indices: &[usize], + cap_height: usize, + hash: crate::whir_hash::DeviceHashKey, + ) -> Option { + let leaves = self.0.elements() >> log_folding; + let depth = leaves.trailing_zeros() as usize; + let arity = hash.arity(); + if indices.iter().any(|index| *index >= leaves) + || cap_height > crypto::merkle_tree::cap::tree_levels(depth, arity) + { + return None; + } + let positions: Vec = indices.iter().map(|index| *index as u32).collect(); + let (bytes, cap_bytes) = self + .0 + .paths_and_cap(log_folding, &positions, cap_height, hash.into_math_cuda()) + .ok()?; + let nodes = nodes_in_place(bytes)?; + // `2^c` nodes (`cap_len` at arity 4): copied rather than reinterpreted + // in place, because a cap is a few hundred bytes and its allocation's + // capacity is not ours to vouch for. + let len = crypto::merkle_tree::cap::cap_len(depth, cap_height, arity)?; + if cap_bytes.len() != 32 * len { + return None; + } + let cap: Vec<[u8; 32]> = cap_bytes + .chunks_exact(32) + .map(|node| <[u8; 32]>::try_from(node).ok()) + .collect::>()?; + if cap.len() != len { + return None; + } + Some(( + nodes + .chunks_exact(path_len(leaves, hash)) + .map(<[_]>::to_vec) + .collect(), + cap, + )) + } + + /// The blocks `indices` open, gathered where they lie — one launch and one + /// copy back for the whole round. + pub(crate) fn cosets( + &self, + indices: &[usize], + num_leaves: usize, + block: usize, + ) -> Option>>> + where + F: math::field::traits::IsField + 'static, + { + let indices: Vec = indices.iter().map(|index| *index as u64).collect(); + let raw = self.0.cosets(&indices, num_leaves, block).ok()?; + let values = values_from_raw::(&raw, self.0.is_base())?; + Some(values.chunks_exact(block).map(<[_]>::to_vec).collect()) + } + + /// The subtrees of `blocks` (each `2^dropped` consecutive leaves), hashed + /// and built where the codeword lies: one tree over the gathered leaves in + /// the host node layout, the blocks' roots `dropped` levels above its + /// leaves (see `math_cuda::whir::DeviceCodeword::block_subtrees`). + pub(crate) fn block_subtrees( + &self, + log_folding: usize, + blocks: &[usize], + dropped: usize, + hash: crate::whir_hash::DeviceHashKey, + leaves: crate::whir_commit::LeafOrder, + ) -> Option> { + let blocks: Vec = blocks.iter().map(|block| *block as u64).collect(); + let bytes = self + .0 + .block_subtrees_placed( + log_folding, + &blocks, + dropped, + hash.into_math_cuda(), + placement(leaves), + ) + .ok()?; + nodes_in_place(bytes) + } + + /// Whether the tree this codeword builds puts its leaves in `leaves`. + pub(crate) fn placed_in(&self, leaves: crate::whir_commit::LeafOrder) -> bool { + self.0.placement() == placement(leaves) + } + + /// Its first value, which is what the last fold leaves behind. + pub(crate) fn first(&self) -> Option> + where + F: math::field::traits::IsField + 'static, + { + let raw = self.0.first().ok()?; + values_from_raw(&raw, false)?.into_iter().next() + } + + /// Every value, brought home (canonical): for a grouped round-0 successor + /// whose polynomials' folds did not all land on the card. + pub(crate) fn values(&self) -> Option>> + where + F: math::field::traits::IsField + 'static, + { + let raw = self.0.to_host().ok()?; + values_from_raw(&raw, self.0.is_base()) + } +} + +/// `two_inv`, each level's inverse generator, and the challenges as limbs. +#[cfg(feature = "cuda")] +fn fold_scalars( + generator: &math::field::element::FieldElement, + alphas: &[math::field::element::FieldElement], +) -> Option<(u64, Vec, Vec)> +where + F: math::field::traits::IsField + 'static, + N: math::field::traits::IsField + 'static, +{ + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { + return None; + } + if alphas.is_empty() { + return None; + } + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let generator = unsafe { *(generator as *const _ as *const u64) }; + let generator = FieldElement::::from_raw(generator); + let two_inv = *FieldElement::::from(2u64).inv().ok()?.value(); + let mut g_inv = generator.inv().ok()?; + let mut g_invs = Vec::with_capacity(alphas.len()); + for _ in 0..alphas.len() { + g_invs.push(*g_inv.value()); + g_inv = g_inv.square(); + } + let mut raw = Vec::with_capacity(alphas.len() * 3); + for alpha in alphas { + raw.extend_from_slice(&ext3_raw(alpha)?); + } + Some((two_inv, g_invs, raw)) +} + +/// Limbs as field elements, one `u64` each for a base codeword and three for +/// an extension one, canonical ([`canonical_limb`]). +#[cfg(feature = "cuda")] +fn values_from_raw(raw: &[u64], base: bool) -> Option>> +where + F: math::field::traits::IsField + 'static, +{ + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + use std::any::TypeId; + + if base { + if TypeId::of::() != TypeId::of::() { + return None; + } + return Some( + raw.iter() + .map(|limb| { + let value = FieldElement::::from_raw(canonical_limb(*limb)); + // SAFETY: `F == Gl`, checked above; same representation. + unsafe { + core::mem::transmute_copy::, FieldElement>(&value) + } + }) + .collect(), + ); + } + if TypeId::of::() != TypeId::of::() { + return None; + } + Some(raw.chunks_exact(3).map(ext3_from_raw::).collect()) +} + +#[cfg(not(feature = "cuda"))] +impl DeviceCodeword { + pub(crate) fn elements(&self) -> usize { + match self.0 {} + } + + pub(crate) fn top_nodes( + &self, + _log_folding: usize, + _nodes: usize, + _hash: crate::whir_hash::DeviceHashKey, + ) -> Option> { + match self.0 {} + } + + pub(crate) fn synchronize(&self) -> bool { + match self.0 {} + } + + pub(crate) fn fold( + &self, + _generator: &math::field::element::FieldElement, + _alphas: &[math::field::element::FieldElement], + ) -> Option + where + F: math::field::traits::IsField + 'static, + N: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn commit( + &self, + _log_folding: usize, + _hash: crate::whir_hash::DeviceHashKey, + ) -> Option<[u8; 32]> { + match self.0 {} + } + + pub(crate) fn paths( + &self, + _log_folding: usize, + _indices: &[usize], + _hash: crate::whir_hash::DeviceHashKey, + ) -> Option>> { + match self.0 {} + } + + pub(crate) fn paths_and_cap( + &self, + _log_folding: usize, + _indices: &[usize], + _cap_height: usize, + _hash: crate::whir_hash::DeviceHashKey, + ) -> Option { + match self.0 {} + } + + pub(crate) fn cosets( + &self, + _indices: &[usize], + _num_leaves: usize, + _block: usize, + ) -> Option>>> + where + F: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn block_subtrees( + &self, + _log_folding: usize, + _blocks: &[usize], + _dropped: usize, + _hash: crate::whir_hash::DeviceHashKey, + _leaves: crate::whir_commit::LeafOrder, + ) -> Option> { + match self.0 {} + } + + pub(crate) fn placed_in(&self, _leaves: crate::whir_commit::LeafOrder) -> bool { + match self.0 {} + } + + pub(crate) fn first(&self) -> Option> + where + F: math::field::traits::IsField + 'static, + { + match self.0 {} + } + + pub(crate) fn values(&self) -> Option>> + where + F: math::field::traits::IsField + 'static, + { + match self.0 {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::program::Builder; + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::{GOLDILOCKS_PRIME as P, GoldilocksField as Gl}; + + type FE = FieldElement; + + /// A coset opening reads the stack where it lies by default: unset (or + /// any other word) reads in place, `0` assembles, `1` reads in place; with + /// the variable unset the knob says so. + #[test] + fn a_coset_opening_reads_its_stack_in_place_by_default() { + assert!(coset_inplace_from(None), "unset: in place"); + assert!(!coset_inplace_from(Some("0")), "0: assembled"); + assert!(coset_inplace_from(Some(" 1 ")), "1: in place"); + assert!(coset_inplace_from(Some("yes")), "another word: the default"); + if std::env::var_os("LAMBDA_VM_WHIR_COSET_INPLACE").is_none() { + force_coset_inplace(None); + assert!(coset_inplace_in_effect()); + } + } + + /// ★ A value read back from the card is serialized canonically whatever + /// representative the card left: a limb `v + p` and the limb `v` give the + /// same proof bytes, and the bytes are `v`'s. Without the reduction the + /// rkyv bytes are the raw limb, which this checks first. + #[test] + fn a_non_canonical_limb_from_the_card_serializes_canonically() { + let raw = |value: &FE| { + rkyv::to_bytes::(value) + .unwrap() + .to_vec() + }; + let unreduced = FE::new([ + FieldElement::::from_raw(P + 5), + FieldElement::::from_raw(P), + FieldElement::::from_raw(7), + ]); + let reduced = FE::new([ + FieldElement::::from_raw(5), + FieldElement::::from_raw(0), + FieldElement::::from_raw(7), + ]); + assert_eq!(unreduced, reduced, "the same value"); + assert_ne!( + raw(&unreduced), + raw(&reduced), + "rkyv writes the representative" + ); + + let read = ext3_from_raw::(&[P + 5, P, 7]); + assert_eq!(raw(&read), raw(&reduced)); + assert_eq!((*read.value()).map(|limb| *limb.value()), [5, 0, 7]); + assert_eq!(canonical_limb(u64::MAX), u64::MAX - P); + assert_eq!(canonical_limb(P - 1), P - 1); + assert_eq!(canonical_limb(P), 0); + } + + /// The kernel's walk, in Rust: the same slot file, the same node encoding. + /// + /// This is what pins the lowering without a device — a slot freed too early + /// or an operand read from the wrong class shows up here as a wrong value, + /// not as a proof that does not verify an hour later. + fn run_lowered(lowered: &Lowered, values: &[FE]) -> FE { + let mut slots = vec![FE::zero(); lowered.num_slots]; + for node in lowered.nodes.chunks_exact(2) { + let op = (node[0] & 0xFFFF_FFFF) as u32; + let a = (node[0] >> 32) as u32 as usize; + let b = (node[1] & 0xFFFF_FFFF) as u32 as usize; + let res = (node[1] >> 32) as u32 as usize; + slots[res] = match op { + op::FIXED => ext3_from_raw::(&lowered.consts[a * 3..a * 3 + 3]), + op::VAR => values[a], + op::ADD => slots[a] + slots[b], + op::SUB => slots[a] - slots[b], + op::MUL => slots[a] * slots[b], + op::NEG => -slots[a], + _ => panic!("unknown op {op}"), + }; + } + slots[lowered.root_slot as usize] + } + + fn values(n: usize) -> Vec { + (0..n as u64) + .map(|i| { + FE::new([ + FieldElement::::from(i * 31 + 7), + FieldElement::::from(i * 17 + 2), + FieldElement::::from(i + 5), + ]) + }) + .collect() + } + + /// Every op, a constant, and a chain long enough that slots have to be + /// recycled. + fn sample_program() -> crate::program::Program { + let mut b = Builder::::new(); + let mut acc = b.var(0); + for slot in 1..6 { + let v = b.var(slot); + let doubled = b.add(v, v); + let scaled = b.mul(doubled, acc); + let shifted = b.sub(scaled, v); + acc = b.neg(shifted); + } + let seven = b.fixed(FE::from(7u64)); + let root = b.add(acc, seven); + b.finish(root).unwrap() + } + + /// `LAMBDA_VM_ARGUE_DEVICE_COLUMNS`, `LAMBDA_VM_ARGUE_DEVICE_TABLES` and + /// `LAMBDA_VM_ARGUE_LEAN_PROGRAM` are on by default: unset, empty or any value + /// reads on, and only `0` — the opt-out — reads off. + #[test] + fn only_zero_turns_a_default_on_knob_off() { + assert!(not_off(None), "unset is the default, on"); + assert!(not_off(Some("")), "empty is not the opt-out"); + assert!(not_off(Some("1"))); + assert!(!not_off(Some("0")), "`0` is the opt-out"); + } + + /// The GKR input knob reads its variable the default-on way, whatever the + /// environment sets. ⚠ No test in this binary forces the knob, which is + /// what makes the reading the environment's. + #[test] + fn the_gkr_input_knob_is_on_unless_its_variable_is_zero() { + let variable = std::env::var("LAMBDA_VM_ARGUE_GKR_INPUT").ok(); + assert_eq!(argue_gkr_input(), not_off(variable.as_deref())); + } + + /// No lift reads its variable the default-on way, and needs the input + /// from the columns. + #[test] + fn the_no_lift_knob_is_on_unless_either_variable_is_zero() { + let input = std::env::var("LAMBDA_VM_ARGUE_GKR_INPUT").ok(); + let variable = std::env::var("LAMBDA_VM_ARGUE_NO_LIFT").ok(); + assert_eq!( + argue_no_lift(), + not_off(variable.as_deref()) && not_off(input.as_deref()) + ); + } + + /// The tables knob reads its variable the default-on way, whatever the + /// environment sets. ⚠ No test in this binary forces the knob, which is + /// what makes the reading the environment's. + #[test] + fn the_tables_knob_is_on_unless_its_variable_is_zero() { + let variable = std::env::var("LAMBDA_VM_ARGUE_DEVICE_TABLES").ok(); + assert_eq!(argue_device_tables(), not_off(variable.as_deref())); + } + + /// The lean program's knob reads its variable the default-on way, whatever + /// the environment sets. ⚠ No test in this binary forces the knob, which is + /// what makes the reading the environment's. + #[test] + fn the_lean_program_knob_is_on_unless_its_variable_is_zero() { + let variable = std::env::var("LAMBDA_VM_ARGUE_LEAN_PROGRAM").ok(); + assert_eq!(argue_lean_program(), not_off(variable.as_deref())); + } + + #[test] + fn the_lowered_program_computes_what_the_program_does() { + let program = sample_program(); + let lowered = lower(&program).expect("lowers"); + let v = values(6); + let mut scratch = Vec::new(); + assert_eq!(run_lowered(&lowered, &v), program.eval(&v, &mut scratch)); + } + + /// A value read twice in the step that kills it — `x·x` — must not free its + /// slot twice, or two later values are handed the same one and the second + /// clobbers the first. Squarings are everywhere in a real constraint + /// program, so this is the shape that matters. + #[test] + fn a_value_read_twice_frees_its_slot_once() { + let mut b = Builder::::new(); + let mut acc = b.var(0); + // Each square kills its operand, and the sums below keep enough values + // live that a doubly-freed slot gets reused while it is still needed. + let mut squares = Vec::new(); + for slot in 1..8 { + let v = b.var(slot); + let squared = b.mul(v, v); + let with_acc = b.add(squared, acc); + squares.push(with_acc); + acc = b.mul(with_acc, with_acc); + } + squares.push(acc); + let root = b.sum(&squares); + let program = b.finish(root).unwrap(); + + let lowered = lower(&program).expect("lowers"); + let v = values(8); + let mut scratch = Vec::new(); + assert_eq!(run_lowered(&lowered, &v), program.eval(&v, &mut scratch)); + } + + /// The point of the slot file: a long chain of dead intermediates does not + /// widen it. + #[test] + fn slots_are_reused_once_a_value_is_dead() { + let lowered = lower(&sample_program()).expect("lowers"); + assert!( + lowered.num_slots < lowered.nodes.len() / 2, + "{} slots for {} steps is no reuse at all", + lowered.num_slots, + lowered.nodes.len() / 2 + ); + } + + /// A program wider than the slot file declines rather than asking a device + /// for scratch it cannot have. + #[test] + fn a_program_past_the_slot_ceiling_declines() { + let mut b = Builder::::new(); + // Every value stays live to the end, so the slots cannot be recycled. + let terms: Vec = (0..=MAX_SLOTS).map(|slot| b.var(slot)).collect(); + let root = b.sum(&terms); + let program = b.finish(root).unwrap(); + assert!(lower(&program).is_none()); + } + + /// The lowering of a program on demand — a read a step, the same values + /// in fewer slots — walks to the program's value, as the kernel walks it. + #[test] + fn a_lowered_program_on_demand_computes_what_the_program_does() { + let mut b = Builder::::new(); + let mut acc = b.var(0); + let mut squares = Vec::new(); + for slot in 1..8 { + let v = b.var(slot); + let squared = b.mul(v, v); + let with_acc = b.add(squared, acc); + squares.push(with_acc); + acc = b.mul(with_acc, with_acc); + } + squares.push(acc); + let root = b.sum(&squares); + let v = values(8); + let mut scratch = Vec::new(); + for program in [sample_program(), b.finish(root).unwrap()] { + let demand = lower(&program.on_demand()).expect("lowers"); + assert_eq!(run_lowered(&demand, &v), program.eval(&v, &mut scratch)); + } + } + + /// A batch is big past [`LEAN_ABOVE_SLOTS`] values a thread, unless a test + /// sets its own gate. + #[test] + fn the_big_batch_gate_reads_its_override() { + assert!(!is_big_batch(LEAN_ABOVE_SLOTS)); + assert!(is_big_batch(LEAN_ABOVE_SLOTS + 1)); + force_lean_program_gate(Some(0)); + let forced = (is_big_batch(0), is_big_batch(1)); + force_lean_program_gate(None); + assert_eq!(forced, (false, true), "a gate of 0 makes every program big"); + assert!(!is_big_batch(LEAN_ABOVE_SLOTS), "and `None` restores it"); + } + + /// ★ The gate is where the slot budget leaves a round fewer than 64 k + /// threads: 341 values a thread still run 65,600, 342 run 65,408. + #[cfg(feature = "cuda")] + #[test] + fn the_big_batch_gate_is_where_a_round_drops_below_64k_threads() { + use math_cuda::sumcheck::thread_ceiling; + assert!(thread_ceiling(LEAN_ABOVE_SLOTS) >= 1 << 16); + assert!(thread_ceiling(LEAN_ABOVE_SLOTS + 1) < 1 << 16); + } + + #[test] + fn a_field_the_kernel_does_not_cover_declines() { + let mut b = Builder::::new(); + let root = b.var(0); + let program = b.finish(root).unwrap(); + assert!(lower(&program).is_none()); + } + + /// ⛔ THE GKR TREE'S REFUSAL IS COUNTED AT ITS ONE SITE, and only there — + /// math-cuda's census of its own five, extended to this file. The argue + /// surface's device-fallback counter is bumped here once, inside + /// `note_gkr_tree_refusal`, which the whole-tree promise's refusal calls once. + /// The carry's refusal (speculative) and the prefetch's (no prefetch) must + /// not reach it: removing the call, or adding one, reddens this test by name. + /// The box test (`stark::multilinear_table`'s + /// `a_refused_gkr_tree_is_counted_and_the_host_builds_the_same_tree`) + /// drives the site. + #[test] + fn the_gkr_tree_refusal_is_counted_at_exactly_its_one_site() { + let source = include_str!("gpu.rs"); + let bumps = source + .matches(concat!("math_cuda::device::", "note_device_fallback()")) + .count(); + let notes = source + .matches(concat!("note_gkr_tree_refusal(", "promise, full)")) + .count(); + assert_eq!( + (bumps, notes), + (1, 1), + "the device-fallback counter must be bumped once in this file, in \ + `note_gkr_tree_refusal`, and that called once, at the whole-tree \ + promise's refusal (found {bumps} bumps, {notes} calls)" + ); + } +} diff --git a/crypto/multilinear/src/gpu_fused.rs b/crypto/multilinear/src/gpu_fused.rs new file mode 100644 index 000000000..28f504d9b --- /dev/null +++ b/crypto/multilinear/src/gpu_fused.rs @@ -0,0 +1,1201 @@ +//! D-ARGUE stage 1 on the card (`thoughts/zf/gap2/fix2/D-ARGUE.md` §4.2): a +//! table's zerocheck rounds with the bus as one column (S1-2), the weights +//! pulled out (S1-4), rounds 0 and 1 from one base-field pass (S1-5) and +//! integer nodes (S1-1) — every message today's. +//! +//! [`fused`](crate::fused) is the host reference and this is its device twin, +//! round for round: the same parts (`T`, `U`, `A`, `B`), the same messages out +//! of them ([`fused::message`]). The kernels are `math_cuda::argue_fused`'s. +//! +//! On by default for every table whose constraint part runs in the base field +//! — or only the committed widths `LAMBDA_VM_ARGUE_FUSED_WIDTHS` lists; +//! `LAMBDA_VM_ARGUE_FUSED=0` is the opt-out, and it is today's rounds exactly. +//! A table it declines runs today's rounds, counted. `LAMBDA_VM_ARGUE_FUSED_XCHECK=1` +//! runs today's rounds after it over the same factors with the same +//! challenges, refuses the prove at the first message that differs, times +//! both, and checks the grid's corners. +//! +//! The default was turned on by its A/B on the block (FAST jobs 274 and 270, +//! `thoughts/zf/gap2/fix2/I-ARGUE.md` §4.5–4.6): with integer nodes, the argue +//! 11.70 → 8.75 s and the whole run 2.80 s faster at 9e2728955, every arm +//! proved and verified on the record's identities, and the cross-check arm — +//! 390 fused tables, every message compared with today's — clean. + +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; + +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{fused::Constraints, logup::Interaction, program::Op}; + +/// The `ACC` op and its no-selector marker. MUST stay in sync with +/// `crypto/math-cuda/kernels/sumcheck.cu`. +pub mod op { + pub const ACC: u32 = 6; + pub const NO_SELECTOR: u32 = u32::MAX; +} + +/// What the fused rounds need beside the factors and the batch: the +/// constraint part in the base field, its batching powers, the bus, and the +/// two points. +pub struct FusedInput<'a, B: IsField, E: IsField> { + pub constraints: &'a Constraints, + pub betas: &'a [FieldElement], + pub interactions: &'a [Interaction], + /// The GKR input-layer claim's point; its last `num_vars` coordinates are `ρ`. + pub claim_point: &'a [FieldElement], + pub r: &'a [FieldElement], + /// The table's base columns on the card as factors, with no lift + /// (`LAMBDA_VM_ARGUE_NO_LIFT`, D-BATCH M1-2): the rounds read them when no + /// lifted factors were made. + pub columns: Option<&'a crate::gpu::ColumnFactors>, +} + +/// What the fused rounds read their factors through: the lifted factors, or +/// the base columns where they lie (D-BATCH M1-2). +#[cfg(feature = "cuda")] +#[derive(Clone, Copy)] +pub(crate) enum FusedSource<'a> { + Lifted(&'a crate::gpu::DeviceFactors), + Columns(&'a crate::gpu::ColumnFactors), +} + +#[cfg(feature = "cuda")] +impl<'a> FusedSource<'a> { + /// `(width, rows)`: the factors and the cube they span. + pub(crate) fn shape(self) -> (usize, usize) { + let view = self.view(); + (view.width, view.rows) + } + + fn view(self) -> math_cuda::sumcheck::FactorView<'a> { + match self { + Self::Lifted(factors) => factors.inner().view(), + Self::Columns(columns) => columns.view(), + } + } +} + +/// The constraint part lowered for both kernels: the base DAG with an `ACC` +/// step after each root (`acc += β[i]·root·selector`), slots reused past +/// their last read as [`crate::gpu::lower`] does. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LoweredFused { + pub nodes: Vec, + pub base_consts: Vec, + pub ext_consts: Vec, + pub num_slots: usize, +} + +/// A base-field element's canonical u64, or `None` off Goldilocks. +fn base_raw(value: &FieldElement) -> Option { + use math::field::goldilocks::GoldilocksField as Gl; + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + // SAFETY: `B == Gl`, checked above. + let gl: FieldElement = + unsafe { core::mem::transmute_copy::, FieldElement>(value) }; + Some(gl.canonical_u64()) +} + +/// Lowers `constraints` with its `ACC` steps. `None` off Goldilocks or past +/// [`crate::gpu::MAX_SLOTS`]. +pub fn lower_fused(constraints: &Constraints) -> Option { + enum Item { + Step(usize), + Acc(usize), + } + let steps = constraints.steps(); + let roots = constraints.roots(); + let selectors = constraints.selectors(); + // Each root's ACC right after the step that makes it. + let mut after: Vec> = vec![Vec::new(); steps.len()]; + for (i, &root) in roots.iter().enumerate() { + after.get_mut(root as usize)?.push(i); + } + let mut items = Vec::with_capacity(steps.len() + roots.len()); + for (s, accs) in after.iter().enumerate() { + items.push(Item::Step(s)); + items.extend(accs.iter().map(|&i| Item::Acc(i))); + } + // Each value's last reader, by item position. + let mut last_use = vec![0usize; steps.len()]; + for (at, item) in items.iter().enumerate() { + match *item { + Item::Step(s) => match steps[s] { + Op::Fixed(_) | Op::Var(_) => {} + Op::Neg(a) => last_use[a as usize] = at, + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => { + last_use[a as usize] = at; + last_use[b as usize] = at; + } + }, + Item::Acc(i) => last_use[roots[i] as usize] = at, + } + } + let mut slot_of = vec![u32::MAX; steps.len()]; + let mut free: Vec = Vec::new(); + let mut num_slots = 0usize; + let mut nodes = Vec::with_capacity(items.len() * 2); + let mut base_consts = Vec::new(); + let mut ext_consts = Vec::new(); + let push = |nodes: &mut Vec, op: u32, a: u32, b: u32, res: u32| { + nodes.push(u64::from(op) | (u64::from(a) << 32)); + nodes.push(u64::from(b) | (u64::from(res) << 32)); + }; + for (at, item) in items.iter().enumerate() { + match *item { + Item::Acc(i) => { + let value = slot_of[roots[i] as usize]; + let selector = match selectors[i] { + Some(slot) => u32::try_from(slot).ok()?, + None => op::NO_SELECTOR, + }; + push(&mut nodes, op::ACC, value, selector, i as u32); + if last_use[roots[i] as usize] == at { + free.push(value); + } + } + Item::Step(s) => { + let (tag, a, b, operands) = match steps[s] { + Op::Fixed(ref c) => { + let raw = base_raw(c)?; + let k = base_consts.len() as u32; + base_consts.push(raw); + ext_consts.extend_from_slice(&[raw, 0, 0]); + (crate::gpu::op::FIXED, k, 0, [None, None]) + } + Op::Var(slot) => (crate::gpu::op::VAR, slot, 0, [None, None]), + Op::Neg(a) => (crate::gpu::op::NEG, slot_of[a as usize], 0, [Some(a), None]), + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => { + let tag = match steps[s] { + Op::Add(..) => crate::gpu::op::ADD, + Op::Sub(..) => crate::gpu::op::SUB, + _ => crate::gpu::op::MUL, + }; + ( + tag, + slot_of[a as usize], + slot_of[b as usize], + [Some(a), Some(b)], + ) + } + }; + if let Some(x) = operands[0].filter(|x| last_use[*x as usize] == at) { + free.push(slot_of[x as usize]); + } + if let Some(y) = + operands[1].filter(|y| last_use[*y as usize] == at && operands[0] != Some(*y)) + { + free.push(slot_of[y as usize]); + } + let res = free.pop().unwrap_or_else(|| { + num_slots += 1; + (num_slots - 1) as u32 + }); + if num_slots > crate::gpu::MAX_SLOTS { + return None; + } + slot_of[s] = res; + push(&mut nodes, tag, a, b, res); + // A value nothing reads — a root is read by its ACC, and no + // step reads itself — frees its slot at once. + if last_use[s] == 0 { + free.push(res); + } + } + } + } + Some(LoweredFused { + nodes, + base_consts, + ext_consts, + num_slots: num_slots.max(1), + }) +} + +/// The kernels' walk of a lowered blob, on the host: `Σ β_i·sel_i·root_i` at +/// one row whose factors are `values` (base field), each step in the base +/// field and each `ACC` a base-by-extension multiply — what `zc_grid01` does at +/// a grid point. For tests: it is how a lowering is checked against the DAG +/// it came from without a card. +pub fn run_lowered_host( + lowered: &LoweredFused, + betas: &[FieldElement], + values: &[FieldElement], +) -> FieldElement +where + B: IsField + math::field::traits::IsSubFieldOf + 'static, + E: IsField, +{ + let mut slots = vec![FieldElement::::zero(); lowered.num_slots]; + let mut acc = FieldElement::::zero(); + for node in lowered.nodes.chunks_exact(2) { + let (tag, a) = ((node[0] & 0xFFFF_FFFF) as u32, (node[0] >> 32) as u32); + let (b, res) = ((node[1] & 0xFFFF_FFFF) as u32, (node[1] >> 32) as u32); + let v = |s: u32| slots[s as usize].clone(); + let value = match tag { + t if t == crate::gpu::op::FIXED => { + FieldElement::::from(lowered.base_consts[a as usize]) + } + t if t == crate::gpu::op::VAR => values[a as usize].clone(), + t if t == crate::gpu::op::ADD => v(a) + v(b), + t if t == crate::gpu::op::SUB => v(a) - v(b), + t if t == crate::gpu::op::MUL => v(a) * v(b), + t if t == crate::gpu::op::NEG => -v(a), + _ => { + let term = if b == op::NO_SELECTOR { + v(a) + } else { + v(a) * values[b as usize].clone() + }; + acc += &term * &betas[res as usize]; + continue; + } + }; + slots[res as usize] = value; + } + acc +} + +// ── knobs, counters, faults ────────────────────────────────────────────────── + +fn env_is(name: &str) -> bool { + std::env::var(name).is_ok_and(|v| v == "1" || v == "true") +} + +static FUSED_FORCED: AtomicU8 = AtomicU8::new(0); +static FUSED_XCHECK_FORCED: AtomicU8 = AtomicU8::new(0); + +fn forced(flag: &AtomicU8, env: impl FnOnce() -> bool) -> bool { + match flag.load(Ordering::Relaxed) { + 1 => false, + 2 => true, + _ => env(), + } +} + +fn store_forced(flag: &AtomicU8, on: Option) { + flag.store( + match on { + None => 0, + Some(false) => 1, + Some(true) => 2, + }, + Ordering::Relaxed, + ); +} + +/// Whether the zerocheck runs the fused rounds: on by default, +/// `LAMBDA_VM_ARGUE_FUSED=0` the opt-out. Read once, with a banner. +pub fn argue_fused() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + forced(&FUSED_FORCED, || { + *ON.get_or_init(|| { + let on = fused_from(std::env::var("LAMBDA_VM_ARGUE_FUSED").ok().as_deref()); + eprintln!( + "★ ARGUE FUSED: {}", + if on { + "on (the default; LAMBDA_VM_ARGUE_FUSED=0 is today's rounds)" + } else { + "off (LAMBDA_VM_ARGUE_FUSED=0)" + } + ); + on + }) + }) +} + +/// The knob's reading of its variable: only `0` turns it off. +fn fused_from(value: Option<&str>) -> bool { + value != Some("0") +} + +#[doc(hidden)] +pub fn force_argue_fused(on: Option) { + store_forced(&FUSED_FORCED, on); +} + +/// Whether a table of `width` committed columns is one the fused rounds take: +/// every table, or those `LAMBDA_VM_ARGUE_FUSED_WIDTHS` (comma-separated) names. +pub fn fused_takes_width(width: usize) -> bool { + static WIDTHS: std::sync::OnceLock>> = std::sync::OnceLock::new(); + WIDTHS + .get_or_init(|| { + std::env::var("LAMBDA_VM_ARGUE_FUSED_WIDTHS") + .ok() + .map(|list| { + list.split(',') + .filter_map(|w| w.trim().parse().ok()) + .collect() + }) + }) + .as_ref() + .is_none_or(|widths| widths.contains(&width)) +} + +/// Whether the fused rounds are cross-checked against today's +/// (`LAMBDA_VM_ARGUE_FUSED_XCHECK=1`; default off). Read once. +pub fn argue_fused_xcheck() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + forced(&FUSED_XCHECK_FORCED, || { + *ON.get_or_init(|| env_is("LAMBDA_VM_ARGUE_FUSED_XCHECK")) + }) +} + +#[doc(hidden)] +pub fn force_argue_fused_xcheck(on: Option) { + store_forced(&FUSED_XCHECK_FORCED, on); +} + +/// A program of at least this many steps keeps its rounds on the card past +/// the host's crossover (`LAMBDA_VM_ZC_DEVICE_TAIL`, on by default): the host walks it +/// through the interpreter once for every index pair and node of a round, +/// while a card round costs about one walk whatever the cube. +pub const DEVICE_TAIL_MIN_STEPS: usize = 1000; + +/// The cube such a program hands the host instead of today's 32: on an RTX +/// 5090 a KECCAK_RND round costs the card ≈ 0.5 ms and the host ≈ 0.24 ms an +/// index pair, so the card's round is the cheaper one down to a cube of 8 +/// (4 pairs) and the host's from a cube of 4 (2 pairs). +pub const DEVICE_TAIL_CUBE: usize = 1 << 2; + +static TAIL_FORCED: AtomicU8 = AtomicU8::new(0); +static TAIL_FORCED_CUBE: AtomicU64 = AtomicU64::new(0); + +/// Where the fused rounds hand a long program to the host +/// (`LAMBDA_VM_ZC_DEVICE_TAIL`; on by default, `=0` turns it off): `None` at +/// the host's crossover, or `(min_steps, cube)` — a program of at least +/// `min_steps` steps stops at `cube` instead. The same messages either way, so +/// the same proof. Read once, with a banner. +pub fn zc_device_tail() -> Option<(usize, usize)> { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + match TAIL_FORCED.load(Ordering::Relaxed) { + 1 => None, + 2 => Some((0, TAIL_FORCED_CUBE.load(Ordering::Relaxed) as usize)), + _ => ON + .get_or_init(|| { + let on = + device_tail_from(std::env::var("LAMBDA_VM_ZC_DEVICE_TAIL").ok().as_deref()); + eprintln!( + "★ ZC DEVICE TAIL: {}", + if on { + "on (the default: a program of ≥ 1000 steps keeps its rounds on the \ + card down to a cube of 4; LAMBDA_VM_ZC_DEVICE_TAIL=0 hands it to the \ + host at the crossover)" + } else { + "off (LAMBDA_VM_ZC_DEVICE_TAIL=0: every program's rounds go to the \ + host at the crossover)" + } + ); + on + }) + .then_some((DEVICE_TAIL_MIN_STEPS, DEVICE_TAIL_CUBE)), + } +} + +/// The knob's reading of its variable: empty, `0` or `off` turn it off; +/// anything else (unset included) leaves it on. +fn device_tail_from(value: Option<&str>) -> bool { + !matches!(value.map(str::trim), Some("" | "0" | "off")) +} + +/// Overrides the knob: `None` back to the environment, `Some(None)` off, +/// `Some(Some(cube))` every program, whatever its length, down to `cube` (a +/// power of two, at least 2) — a test's way to the card's smallest rounds on +/// short programs. +#[doc(hidden)] +pub fn force_zc_device_tail(mode: Option>) { + if let Some(Some(cube)) = mode { + assert!( + cube >= 2 && cube.is_power_of_two(), + "the host takes a cube of at least 2" + ); + TAIL_FORCED_CUBE.store(cube as u64, Ordering::Relaxed); + } + TAIL_FORCED.store( + match mode { + None => 0, + Some(None) => 1, + Some(Some(_)) => 2, + }, + Ordering::Relaxed, + ); +} + +/// Whether every fused table prints its timings (`LAMBDA_VM_ARGUE_FUSED_LOG=1`); +/// the cross-check prints them regardless. Read once. +pub fn argue_fused_log() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| env_is("LAMBDA_VM_ARGUE_FUSED_LOG")) +} + +/// ⛔ Faults and inert checks, for the negative controls and the mutations +/// that show each check is load-bearing. Never armed outside a test. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct FusedFaults { + /// The card's first bus coefficient off by one. + pub bus: bool, + /// Not a fault: the grid's corners summed into `T` rather than taken as + /// zero — what random columns, which break every AIR, need to send + /// today's messages. Overrides the corner check. + pub keep_corners: bool, + /// The cross-check compares nothing. + pub xcheck_inert: bool, + /// The corner check refuses nothing. + pub corners_inert: bool, +} + +thread_local! { + // Per thread, not per process: the fused rounds are on by default, so a + // fault armed for one test must not reach a proof another test runs beside + // it. A table's argument runs on the thread that calls it (the table loop + // is serial), which is the test's own. + static FAULTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Arms `faults` for the calling thread's proofs; `FusedFaults::default()` +/// disarms. +#[doc(hidden)] +pub fn force_fused_faults(faults: FusedFaults) { + FAULTS.with(|f| { + f.set( + u8::from(faults.bus) + | (u8::from(faults.keep_corners) << 1) + | (u8::from(faults.xcheck_inert) << 2) + | (u8::from(faults.corners_inert) << 3), + ) + }); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn faults() -> FusedFaults { + let f = FAULTS.with(std::cell::Cell::get); + FusedFaults { + bus: f & 1 != 0, + keep_corners: f & 2 != 0, + xcheck_inert: f & 4 != 0, + corners_inert: f & 8 != 0, + } +} + +static SESSIONS: AtomicU64 = AtomicU64::new(0); +static DECLINES: AtomicU64 = AtomicU64::new(0); +static XCHECKS: AtomicU64 = AtomicU64::new(0); +static CORNER_CHECKS: AtomicU64 = AtomicU64::new(0); + +/// Tables whose zerocheck ran the fused rounds. +pub fn fused_sessions() -> u64 { + SESSIONS.load(Ordering::Relaxed) +} + +/// Tables the fused rounds declined before round 0 (today's ran). +pub fn fused_declines() -> u64 { + DECLINES.load(Ordering::Relaxed) +} + +/// Fused tables whose every message today's rounds confirmed. +pub fn fused_xchecks() -> u64 { + XCHECKS.load(Ordering::Relaxed) +} + +/// Fused tables whose grid corners were computed and found zero. +pub fn fused_corner_checks() -> u64 { + CORNER_CHECKS.load(Ordering::Relaxed) +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_decline() { + DECLINES.fetch_add(1, Ordering::Relaxed); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_xcheck() { + XCHECKS.fetch_add(1, Ordering::Relaxed); +} + +#[doc(hidden)] +pub fn reset_fused_counters() { + for c in [&SESSIONS, &DECLINES, &XCHECKS, &CORNER_CHECKS] { + c.store(0, Ordering::Relaxed); + } +} + +/// One fused table's timings, for its log line. +#[derive(Clone, Copy, Debug, Default)] +pub struct FusedTimes { + pub grid: f64, + pub fold2: f64, + pub rounds: f64, + pub total: f64, + /// The table's signature on the log line (a committed table has no name + /// here): its roots, its bus column's terms, the constraint walk's slots. + pub roots: usize, + pub terms: usize, + pub slots: usize, +} + +static DEVICE_TAILS: AtomicU64 = AtomicU64::new(0); + +/// Fused tables whose rounds stayed on the card past today's crossover +/// (`LAMBDA_VM_ZC_DEVICE_TAIL`). +pub fn device_tail_sessions() -> u64 { + DEVICE_TAILS.load(Ordering::Relaxed) +} + +/// Where a thread's fused tables spent their later rounds since the last +/// [`take_tail`], short programs at `[0]` and those of at least +/// [`DEVICE_TAIL_MIN_STEPS`] steps at `[1]` (knob on or off): the batched +/// argue's log line under the split. +#[derive(Clone, Copy, Debug, Default)] +pub struct TailTimes { + /// Fused tables. + pub tables: [u32; 2], + /// Of them, those whose rounds stayed on the card past today's crossover. + pub on_card: [u32; 2], + /// The card's later rounds and their folds, seconds as the host saw them. + pub device: [f64; 2], + /// The rounds the host ran over what the card folded, and their seconds + /// (each round's evaluations and its fold). + pub host_rounds: [u32; 2], + pub host: [f64; 2], +} + +thread_local! { + static TAIL: std::cell::Cell = std::cell::Cell::new(TailTimes::default()); +} + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_tail(note: impl FnOnce(&mut TailTimes)) { + TAIL.with(|cell| { + let mut tail = cell.get(); + note(&mut tail); + cell.set(tail); + }); +} + +/// The calling thread's [`TailTimes`] since the last call, reset. +pub fn take_tail() -> TailTimes { + TAIL.with(|cell| cell.replace(TailTimes::default())) +} + +// ── the rounds ─────────────────────────────────────────────────────────────── + +/// The device rounds' result: the messages, the challenges, and every factor +/// — the trace's, then the two weights — as the last device fold left it. +#[cfg(feature = "cuda")] +pub(crate) type FusedRounds = ( + Vec>, + Vec>, + Vec>, + FusedTimes, +); + +/// A table's zerocheck rounds the fused way, down to the host crossover. +/// +/// `lambdas` are the batch's weights, `claim` its combined claim, `degree` +/// the batch's (`d_C + 1`). `challenge` absorbs a message and draws the next +/// challenge, as today's device rounds call it. +/// +/// `None` is a decline, always before the first message; `Some(Err)` after. +/// +/// The rounds are [`FusedStepper`]'s, driven here one after the other; the +/// batched argue drives the same stepper in lockstep with other tables'. +#[cfg(feature = "cuda")] +#[allow(clippy::too_many_arguments)] +pub(crate) fn prove_fused( + source: FusedSource<'_>, + input: &FusedInput<'_, B, E>, + lambdas: &[FieldElement], + claim: &FieldElement, + degree: usize, + check_corners: bool, + mut challenge: impl FnMut(&[FieldElement]) -> FieldElement, +) -> Option, crate::Error>> +where + B: IsField + math::field::traits::IsSubFieldOf + 'static, + E: IsField + 'static, + FieldElement: Send + Sync, +{ + let mut stepper = match FusedStepper::new(source, input, lambdas, claim, degree, check_corners)? + { + Ok(stepper) => stepper, + Err(error) => return Some(Err(error)), + }; + let there = stepper.device_rounds(); + let mut rounds = Vec::with_capacity(there); + let mut point = Vec::with_capacity(there); + for _ in 0..there { + let g = match stepper.message() { + Ok(g) => g, + Err(error) => return Some(Err(error)), + }; + let s = challenge(&g); + if let Err(error) = stepper.bind(&s) { + return Some(Err(error)); + } + rounds.push(crate::sumcheck::RoundProof { evaluations: g }); + point.push(s); + } + let (factors, times) = match stepper.into_factors() { + Ok(done) => done, + Err(error) => return Some(Err(error)), + }; + Some(Ok((rounds, point, factors, times))) +} + +/// A table's fused zerocheck rounds, stepped by its caller: the message of the +/// round under way, then the challenge to bind it with — so several tables' +/// rounds can share one challenge (the batched argue, D-BATCH B-3) and one +/// table's can run alone ([`prove_fused`]) with the same messages. +/// +/// Rounds 0 and 1 come from the grid pass (S1-5), the later ones from the +/// Gruen rounds (S1-4) until today's host crossover; [`Self::into_factors`] +/// then hands back every factor as the last fold left it, for the host tail. +#[cfg(feature = "cuda")] +#[doc(hidden)] +pub struct FusedStepper<'f, E: IsField> { + session: math_cuda::argue_fused::FusedZerocheck<'f>, + r: Vec>, + rho: Vec>, + d: usize, + big: FieldElement, + walked: Vec, + there: usize, + /// Today's crossover, which `there` passes under the device tail. + crossover: usize, + /// The lowered program's steps. + steps: usize, + t: Vec>>, + u: [[FieldElement; 2]; 2], + claim: FieldElement, + e_r: FieldElement, + e_rho: FieldElement, + /// The challenges bound so far. + bound: Vec>, + /// The message of the round under way, once asked for. + pending: Option>>, + /// A round queued on the card by [`Self::prefetch`], its rows unread. + queued: Option, + started: std::time::Instant, + times: FusedTimes, +} + +#[cfg(feature = "cuda")] +impl<'f, E> FusedStepper<'f, E> +where + E: IsField + 'static, + FieldElement: Send + Sync, +{ + /// Sets the session up and runs the grid pass. `None` is a decline, before + /// anything is sent; `Some(Err)` a trace the corner check refuses. + pub(crate) fn new( + source: FusedSource<'f>, + input: &FusedInput<'_, B, E>, + lambdas: &[FieldElement], + claim: &FieldElement, + degree: usize, + check_corners: bool, + ) -> Option> + where + B: IsField + math::field::traits::IsSubFieldOf + 'static, + { + use crate::fused::BusColumn; + use crate::gpu::{ext3_from_raw, ext3_raw}; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use std::time::Instant; + + if std::any::TypeId::of::() != std::any::TypeId::of::() || lambdas.len() != 3 { + return None; + } + let started = Instant::now(); + let device = source.view(); + let rows = device.rows; + let num_vars = rows.trailing_zeros() as usize; + let constraints = input.constraints; + let d = constraints.degree().max(1); + if degree != d + 1 || input.r.len() != num_vars || input.claim_point.len() < num_vars { + return None; + } + // Rounds 0 and 1 on the grid, then the later rounds down to where today's + // hand the cube to the host. + let there = num_vars.saturating_sub(crate::HOST_CUBE_COMPILED.trailing_zeros() as usize); + if there < 2 || input.betas.len() != constraints.roots().len() { + return None; + } + let lowered = lower_fused(constraints)?; + // A long program's rounds stay on the card past today's crossover + // (`LAMBDA_VM_ZC_DEVICE_TAIL`), except under the cross-check, which + // replays today's rounds and so stops where they do. + let crossover = there; + let steps = lowered.nodes.len() / 2; + let there = match zc_device_tail() { + Some((min_steps, cube)) if steps >= min_steps && !check_corners => num_vars + .saturating_sub(cube.trailing_zeros() as usize) + .max(crossover), + _ => crossover, + }; + let (interaction_point, rho) = input + .claim_point + .split_at(input.claim_point.len() - num_vars); + let mut bus = BusColumn::new( + input.interactions, + interaction_point, + &lambdas[1], + &lambdas[2], + ) + .ok()?; + if faults().bus + && let Some((slot, a)) = bus.terms().first().cloned() + { + let mut terms = bus.terms().to_vec(); + terms[0] = (slot, a + FieldElement::::one()); + bus = BusColumn::from_parts(bus.constant().clone(), terms); + } + let keep = faults().keep_corners; + let check_corners = check_corners && !keep; + let points: Vec<(u32, u32)> = (0..=d as u32) + .flat_map(|a| (0..=d as u32).map(move |b| (a, b))) + .filter(|&(a, b)| check_corners || keep || !(a < 2 && b < 2)) + .collect(); + let walked: Vec = core::iter::once(0).chain(2..=d as u32).collect(); + if points.len() > math_cuda::argue_fused::MAX_ROWS + || walked.len() + 3 > math_cuda::argue_fused::MAX_ROWS + { + return None; + } + + let raw = |v: &FieldElement| ext3_raw(v); + let flat = |vs: &[FieldElement]| -> Option> { + let mut out = Vec::with_capacity(vs.len() * 3); + for v in vs { + out.extend_from_slice(&raw(v)?); + } + Some(out) + }; + let betas = flat(input.betas)?; + let slots: Vec = bus + .terms() + .iter() + .map(|(slot, _)| u32::try_from(*slot).ok()) + .collect::>()?; + if slots.iter().any(|&s| s as usize >= device.width) + || constraints.reads().iter().any(|&s| s >= device.width) + { + return None; + } + let coeffs = flat( + &bus.terms() + .iter() + .map(|(_, a)| a.clone()) + .collect::>(), + )?; + let r_tail = flat(&input.r[2..])?; + let rho_tail = flat(&rho[2..])?; + let program = math_cuda::argue_fused::FusedProgram { + nodes: &lowered.nodes, + base_consts: &lowered.base_consts, + ext_consts: &lowered.ext_consts, + betas: &betas, + num_slots: lowered.num_slots, + }; + let fused_bus = math_cuda::argue_fused::FusedBus { + slots: &slots, + coeffs: &coeffs, + constant: raw(bus.constant())?, + }; + let mut session = match math_cuda::argue_fused::FusedZerocheck::new( + device, + program, + fused_bus, + &r_tail, + &rho_tail, + points.len(), + walked.len() + 2, + ) { + Ok(Some(session)) => session, + _ => return None, + }; + // No round slot here: through one the zerocheck's launches ran slower + // on whole blocks than with today's second launch, unlike the GKR + // rounds'. + + let from = |limbs: &[u64]| ext3_from_raw::(limbs); + let side = d + 1; + + // ── rounds 0 and 1: one base-field pass ── + let tick = Instant::now(); + let Ok((t_sums, u_sums, violation)) = session.grid(&points, keep) else { + return None; + }; + if check_corners { + if let Some(row) = violation + && !faults().corners_inert + { + eprintln!( + "[argue] FUSED XCHECK: the trace breaks its constraints at row {row} (2^{num_vars} rows)" + ); + return Some(Err(crate::Error::ConstraintViolated { row: row as usize })); + } + CORNER_CHECKS.fetch_add(1, Ordering::Relaxed); + } + let mut t = vec![vec![FieldElement::::zero(); side]; side]; + for (k, &(a, b)) in points.iter().enumerate() { + if a < 2 && b < 2 && !keep { + continue; + } + t[a as usize][b as usize] = from(&t_sums[k * 3..k * 3 + 3]); + } + let u: [[FieldElement; 2]; 2] = core::array::from_fn(|b| { + core::array::from_fn(|c| from(&u_sums[(2 * b + c) * 3..(2 * b + c) * 3 + 3])) + }); + let times = FusedTimes { + grid: tick.elapsed().as_secs_f64(), + roots: constraints.roots().len(), + terms: slots.len(), + slots: lowered.num_slots, + ..FusedTimes::default() + }; + Some(Ok(Self { + session, + r: input.r.to_vec(), + rho: rho.to_vec(), + d, + big: FieldElement::::from(degree as u64), + walked, + there, + crossover, + steps, + t, + u, + claim: claim.clone(), + e_r: FieldElement::one(), + e_rho: FieldElement::one(), + bound: Vec::with_capacity(there), + pending: None, + queued: None, + started, + times, + })) + } + + /// Rounds the card runs before the host takes the cube. + pub(crate) fn device_rounds(&self) -> usize { + self.there + } + + /// Variables this table has left to bind on the card. + pub(crate) fn rounds_left(&self) -> usize { + self.there - self.bound.len() + } + + /// The claim the round under way carries in. + pub(crate) fn claim(&self) -> &FieldElement { + &self.claim + } + + /// Queues the round under way's launches on the card, not waited for, so + /// the batched argue can queue every table's before it reads any back. + /// Rounds 0 and 1 come from the grid pass and queue nothing. + pub(crate) fn prefetch(&mut self) -> Result<(), crate::Error> { + let j = self.bound.len(); + if j >= 2 && j < self.there && self.pending.is_none() && self.queued.is_none() { + let rows = self.session.round_enqueue(&self.walked).map_err(|_| { + crate::Error::DeviceFailed { + stage: "fused round", + } + })?; + self.queued = Some(rows); + } + Ok(()) + } + + /// The round under way's message: its polynomial at `1..=d + 1`. + pub(crate) fn message(&mut self) -> Result>, crate::Error> { + use crate::fused; + use crate::gpu::ext3_from_raw; + let failed = |stage| crate::Error::DeviceFailed { stage }; + let j = self.bound.len(); + if j >= self.there { + return Err(failed("fused round")); + } + if let Some(g) = &self.pending { + return Ok(g.clone()); + } + let one = FieldElement::::one(); + let d = self.d; + let (r, rho) = (&self.r, &self.rho); + let g = match j { + 0 => { + let a0: Vec> = (0..=d) + .map(|a| (&one - &r[1]) * &self.t[a][0] + &r[1] * &self.t[a][1]) + .collect(); + let b0: [FieldElement; 2] = core::array::from_fn(|b| { + (&one - &rho[1]) * &self.u[b][0] + &rho[1] * &self.u[b][1] + }); + fused::message(&r[0], &rho[0], &self.e_r, &self.e_rho, &a0, &b0, &self.big) + } + 1 => { + let s0 = &self.bound[0]; + let a1: Vec> = (0..=d) + .map(|b| { + let column: Vec> = + (0..=d).map(|a| self.t[a][b].clone()).collect(); + crate::sumcheck::interpolate(&column, s0) + }) + .collect(); + let b1: [FieldElement; 2] = + core::array::from_fn(|c| (&one - s0) * &self.u[0][c] + s0 * &self.u[1][c]); + fused::message(&r[1], &rho[1], &self.e_r, &self.e_rho, &a1, &b1, &self.big) + } + _ => { + let tick = std::time::Instant::now(); + let from = |limbs: &[u64]| ext3_from_raw::(limbs); + let rows = match self.queued.take() { + Some(rows) => rows, + None => self + .session + .round_enqueue(&self.walked) + .map_err(|_| failed("fused round"))?, + }; + let sums = self + .session + .collect(rows) + .map_err(|_| failed("fused round"))?; + let mut a = vec![FieldElement::::zero(); d + 1]; + for (k, &node) in self.walked.iter().enumerate() { + a[node as usize] = from(&sums[k * 3..k * 3 + 3]); + } + let nb = self.walked.len(); + let b = [ + from(&sums[nb * 3..nb * 3 + 3]), + from(&sums[nb * 3 + 3..nb * 3 + 6]), + ]; + let (r_j, rho_j) = (&r[j], &rho[j]); + let divisor = &self.e_r * r_j; + let bus_part = &self.e_rho * ((&one - rho_j) * &b[0] + rho_j * &b[1]); + a[1] = match divisor.inv() { + Ok(inverse) => { + (&self.claim - &self.e_r * (&one - r_j) * &a[0] - bus_part) * inverse + } + // `E^r_j·r_j` vanished: the claim says nothing about `A(1)`, so + // it is walked. + Err(_) => match self.session.round(&[1]) { + Ok(direct) => from(&direct[0..3]), + Err(_) => return Err(failed("fused round")), + }, + }; + self.times.rounds += tick.elapsed().as_secs_f64(); + fused::message(r_j, rho_j, &self.e_r, &self.e_rho, &a, &b, &self.big) + } + }; + self.pending = Some(g.clone()); + Ok(g) + } + + /// Binds the round under way to `s`: the claim carried on, the weights' + /// prefix products, and the fold on the card. + pub(crate) fn bind(&mut self, s: &FieldElement) -> Result<(), crate::Error> { + use crate::fused; + use crate::gpu::ext3_raw; + let failed = |stage| crate::Error::DeviceFailed { stage }; + let j = self.bound.len(); + let g = self.pending.take().ok_or(failed("fused bind"))?; + let mut all = Vec::with_capacity(g.len() + 1); + all.push(&self.claim - &g[0]); + all.extend(g.iter().cloned()); + self.claim = crate::sumcheck::interpolate(&all, s); + self.e_r = &self.e_r * fused::eq1(&self.r[j], s); + self.e_rho = &self.e_rho * fused::eq1(&self.rho[j], s); + self.bound.push(s.clone()); + match j { + 0 => {} + 1 => { + // ── both folds at once ── + let tick = std::time::Instant::now(); + let (s0, s1) = (&self.bound[0], &self.bound[1]); + let mut w = [0u64; 12]; + for a in 0..2u64 { + for b in 0..2u64 { + let wab = fused::eq1(s0, &FieldElement::::from(a)) + * fused::eq1(s1, &FieldElement::::from(b)); + let at = (2 * a + b) as usize * 3; + let limbs = ext3_raw(&wab).ok_or(failed("fused weights"))?; + w[at..at + 3].copy_from_slice(&limbs); + } + } + self.session + .fold2(&w) + .map_err(|_| failed("fused double fold"))?; + self.times.fold2 = tick.elapsed().as_secs_f64(); + } + _ => { + let tick = std::time::Instant::now(); + let limbs = ext3_raw(s).ok_or(failed("fused challenge"))?; + self.session + .fold(&limbs, j + 1 < self.there) + .map_err(|_| failed("fused fold"))?; + self.times.rounds += tick.elapsed().as_secs_f64(); + } + } + Ok(()) + } + + /// What the host tail reads once the card's rounds are bound: the trace's + /// factors as folded, then the two weights folded by the same challenges — + /// `E^·` times `eq` of the rest. + pub(crate) fn into_factors( + mut self, + ) -> Result<(Vec>, FusedTimes), crate::Error> { + use crate::gpu::ext3_from_raw; + let failed = |stage| crate::Error::DeviceFailed { stage }; + if self.bound.len() != self.there { + return Err(failed("fused factor values")); + } + let values = self + .session + .values() + .map_err(|_| failed("fused factor values"))?; + let mut factors: Vec> = Vec::with_capacity(values.len() + 2); + for factor in &values { + factors.push( + crate::mle::Mle::new(factor.chunks_exact(3).map(ext3_from_raw::).collect()) + .map_err(|_| failed("fused factor values"))?, + ); + } + for (scale, tail) in [ + (&self.e_r, &self.r[self.there..]), + (&self.e_rho, &self.rho[self.there..]), + ] { + let table: Vec> = crate::eq::eq_evals(tail) + .into_iter() + .map(|v| v * scale) + .collect(); + factors.push(crate::mle::Mle::new(table).map_err(|_| failed("fused weights"))?); + } + SESSIONS.fetch_add(1, Ordering::Relaxed); + crate::gpu::note_device_sumcheck(self.there as u64); + self.times.total = self.started.elapsed().as_secs_f64(); + let past = self.there > self.crossover; + if past { + DEVICE_TAILS.fetch_add(1, Ordering::Relaxed); + } + let long = usize::from(self.long()); + let rounds = self.times.rounds; + note_tail(|tail| { + tail.tables[long] += 1; + tail.on_card[long] += u32::from(past); + tail.device[long] += rounds; + }); + Ok((factors, self.times)) + } + + /// Whether the program is one the device tail keeps on the card, knob on + /// or off: at least [`DEVICE_TAIL_MIN_STEPS`] steps. + pub(crate) fn long(&self) -> bool { + self.steps >= DEVICE_TAIL_MIN_STEPS + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as Gl; + + type FB = FieldElement; + + /// `x·y + 3` and `x − y` as two roots, the second selected by factor 2. + fn two_roots() -> Constraints { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + let steps: Vec> = vec![ + Op::Var(0), + Op::Var(1), + Op::Mul(0, 1), + Op::Fixed(FieldElement::::from(3u64)), + Op::Add(2, 3), + Op::Sub(0, 1), + ]; + Constraints::::from_extension(&steps, &[4, 5], &[None, Some(2)], 2).unwrap() + } + + /// Walks the lowered blob on the host the way `zc_grid01` does, in the base + /// field, and returns `Σ β_i·sel_i·root_i` with `β = (1, 10)`. + fn run(lowered: &LoweredFused, factors: &[u64]) -> u64 { + let mut slots = vec![FB::zero(); lowered.num_slots]; + let betas = [FB::from(1u64), FB::from(10u64)]; + let mut acc = FB::zero(); + for node in lowered.nodes.chunks_exact(2) { + let (op, a) = ((node[0] & 0xFFFF_FFFF) as u32, (node[0] >> 32) as u32); + let (b, res) = ((node[1] & 0xFFFF_FFFF) as u32, (node[1] >> 32) as u32); + let v = |s: u32| slots[s as usize]; + match op { + 0 => slots[res as usize] = FB::from(lowered.base_consts[a as usize]), + 1 => slots[res as usize] = FB::from(factors[a as usize]), + 2 => slots[res as usize] = v(a) + v(b), + 3 => slots[res as usize] = v(a) - v(b), + 4 => slots[res as usize] = v(a) * v(b), + 5 => slots[res as usize] = -v(a), + 6 => { + let sel = if b == op::NO_SELECTOR { + FB::one() + } else { + FB::from(factors[b as usize]) + }; + acc += betas[res as usize] * v(a) * sel; + } + _ => unreachable!(), + } + } + acc.canonical_u64() + } + + #[test] + fn the_lowered_blob_sums_the_selected_roots() { + let lowered = lower_fused(&two_roots()).unwrap(); + // 2 roots → 2 ACC nodes; 6 steps. + assert_eq!(lowered.nodes.len(), 2 * 8); + assert_eq!(lowered.base_consts, vec![3]); + assert_eq!(lowered.ext_consts, vec![3, 0, 0]); + for (x, y, s) in [(2u64, 5u64, 1u64), (7, 7, 0), (4, 9, 3)] { + let want = (FB::from(x) * FB::from(y) + FB::from(3u64)) + + FB::from(10u64) * (FB::from(x) - FB::from(y)) * FB::from(s); + assert_eq!( + run(&lowered, &[x, y, s]), + want.canonical_u64(), + "x={x} y={y} s={s}" + ); + } + } + + /// The default, pinned: on unless the variable says `0`. + #[test] + fn the_fused_rounds_are_on_unless_the_knob_says_0() { + assert!(fused_from(None)); + assert!(fused_from(Some("1"))); + assert!(fused_from(Some(""))); + assert!(!fused_from(Some("0"))); + } + + /// The device tail's default, pinned: on unless the variable says `0`, + /// `off` or nothing, and its rule's constants as frozen before the box + /// measured them. + #[test] + fn the_device_tail_is_on_unless_the_knob_says_0() { + assert!(device_tail_from(None)); + assert!(device_tail_from(Some("1"))); + assert!(device_tail_from(Some("true"))); + assert!(!device_tail_from(Some(""))); + assert!(!device_tail_from(Some("0"))); + assert!(!device_tail_from(Some(" 0 "))); + assert!(!device_tail_from(Some("off"))); + assert_eq!((DEVICE_TAIL_MIN_STEPS, DEVICE_TAIL_CUBE), (1000, 4)); + const { assert!(DEVICE_TAIL_CUBE < crate::HOST_CUBE_COMPILED) }; + } + + #[test] + fn a_slot_is_reused_once_its_last_reader_passed() { + let lowered = lower_fused(&two_roots()).unwrap(); + assert!(lowered.num_slots <= 4, "{} slots", lowered.num_slots); + } +} diff --git a/crypto/multilinear/src/lean_tiles.rs b/crypto/multilinear/src/lean_tiles.rs new file mode 100644 index 000000000..7a5af6538 --- /dev/null +++ b/crypto/multilinear/src/lean_tiles.rs @@ -0,0 +1,676 @@ +//! The plan of a stacked opening's lean rounds in TILE form. +//! +//! A stacked opening's weight is, inside each column, `scale · eq(z, row)`. +//! Cut the stack into tiles of `2^u` positions, `u` no more than the smallest +//! column's variables: columns sit at multiples of their own size, so a tile +//! lies inside one column or one gap, and there +//! +//! ```text +//! w(T, j) = ω(T) · eq(λ, j), ω(T) = scale · eq(z_hi, T − the column's first tile) +//! ``` +//! +//! with `λ` the point's last `u` coordinates and `z_hi` the rest. Columns whose +//! `λ` are equal are one CLASS and share one table. One pass over the message +//! then gives every tile's value at each class it can meet, and every lean +//! round runs on tables `2^u` times smaller than the stack — the same round +//! polynomials (`whir_tile_*` in `math-cuda/kernels/whir_fold.cu` writes the +//! identity out; [`tests`] checks it against the host prover's own rounds). +//! +//! This module is the host's part: which `u`, which classes, the tables, and +//! whether the form fits. It reads no device and is built without one. + +use std::collections::BTreeMap; + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + +/// The widest tile: `2^10` positions. A column of `2^20` or `2^21` rows splits +/// its point's table there today (`num_vars / 2`), so for those the two halves +/// are the ones the share form already builds. +pub const TILE_BITS_CAP: usize = 10; + +/// Slot tables the kernels have room for (`WHIR_TILE_MAX_SLOTS`). +pub const MAX_SLOTS: usize = 8; + +/// One column as the tile form reads it: its point's table split at the plan's +/// tile, the high half at `hi_at` and its class's low half at `lo_at`, in +/// extension elements into [`TilePlan::eq`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TileShare { + pub stack_offset: usize, + pub num_vars: usize, + pub hi_at: usize, + pub lo_at: usize, + pub scale: [u64; 3], +} + +/// A stacked opening's lean rounds, planned in tile form. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TilePlan { + /// A tile is `2^tile_bits` stack positions. + pub tile_bits: usize, + /// Classes in the stack: distinct tails of `tile_bits` coordinates. + pub classes: usize, + /// Slot tables to hold: the most classes a column of tiles can meet, + /// `min(classes, 2^lean_rounds)`. + pub slots: usize, + /// The columns, by stack offset. + pub shares: Vec, + /// Every `eq` half, three limbs an element. + pub eq: Vec, +} + +/// Device bytes the tile form holds through its rounds: a `u16` a tile, each +/// column of tiles' slots, the two slot tables at three limbs a tile and +/// slot, a challenge. `math_cuda::whir_open::TileSpec::device_bytes`, which a +/// `cuda` build's test holds this to. +pub const fn device_bytes( + n_stack: usize, + tile_bits: usize, + lean_rounds: usize, + slots: usize, +) -> u64 { + write_bytes(n_stack, tile_bits, lean_rounds, slots) + + ((24 * slots as u64) << (n_stack - tile_bits)) +} + +/// What it keeps beside the bound tables while it writes them: all of the +/// above but the message's slot tables (`TileSpec::write_bytes`). +pub const fn write_bytes( + n_stack: usize, + tile_bits: usize, + lean_rounds: usize, + slots: usize, +) -> u64 { + let tiles = 1u64 << (n_stack - tile_bits); + let columns = tiles >> lean_rounds; + let slots = slots as u64; + 2 * tiles + 8 * slots * columns + 24 * slots * tiles + 24 +} + +/// Whether the form fits a stack of `2^n_stack`: the rounds bind tile +/// variables only, the kernels have the slots, and the opening never holds +/// more than the share form does while it writes its tables — the position +/// map (a `u16` a position, which the tile form does not allocate) beside +/// the bound tables. Through the rounds all the form holds is no larger than +/// those tables; through the write what it keeps is no larger than the map. +pub const fn fits(n_stack: usize, tile_bits: usize, lean_rounds: usize, slots: usize) -> bool { + tile_bits <= n_stack + && lean_rounds >= 1 + && lean_rounds <= n_stack - tile_bits + && slots >= 1 + && slots <= MAX_SLOTS + && device_bytes(n_stack, tile_bits, lean_rounds, slots) <= 48u64 << (n_stack - lean_rounds) + && write_bytes(n_stack, tile_bits, lean_rounds, slots) <= 2u64 << n_stack +} + +/// The tile form of the opening whose weight is `shares` — `(stack offset, +/// point as limbs, scale)` per column — over `2^n_stack`, for `lean_rounds` +/// rounds. `None` when the stack does not take it: a column off its own +/// alignment, or so small (or the classes so many) that the form would hold +/// more than the tables written after it. +pub fn plan( + shares: &[(usize, Vec, [u64; 3])], + n_stack: usize, + lean_rounds: usize, +) -> Option { + if shares.is_empty() || lean_rounds == 0 { + return None; + } + let tile_bits = shares + .iter() + .map(|(_, point, _)| point.len() / 3) + .min()? + .min(TILE_BITS_CAP); + let mut order: Vec = (0..shares.len()).collect(); + order.sort_by_key(|i| shares[*i].0); + + let mut eq: Vec = Vec::new(); + let mut table = |coordinates: &[u64]| -> usize { + let point: Vec> = coordinates + .chunks_exact(3) + .map(crate::gpu::ext3_from_raw::) + .collect(); + let at = eq.len() / 3; + for value in crate::eq::eq_evals(&point) { + let [a, b, c] = value.value(); + eq.extend_from_slice(&[*a.value(), *b.value(), *c.value()]); + } + at + }; + // One table per distinct head and per distinct tail, limb for limb: a + // table's columns share both, and a class is a tail. + let mut heads: BTreeMap<&[u64], usize> = BTreeMap::new(); + let mut tails: BTreeMap<&[u64], usize> = BTreeMap::new(); + let mut planned = Vec::with_capacity(shares.len()); + for &i in &order { + let (offset, point, scale) = &shares[i]; + if !point.len().is_multiple_of(3) { + return None; + } + let num_vars = point.len() / 3; + if num_vars > n_stack || offset & ((1usize << num_vars) - 1) != 0 { + return None; + } + let (head, tail) = point.split_at((num_vars - tile_bits) * 3); + let hi_at = match heads.get(head) { + Some(at) => *at, + None => { + let at = table(head); + heads.insert(head, at); + at + } + }; + let lo_at = match tails.get(tail) { + Some(at) => *at, + None => { + let at = table(tail); + tails.insert(tail, at); + at + } + }; + planned.push(TileShare { + stack_offset: *offset, + num_vars, + hi_at, + lo_at, + scale: *scale, + }); + } + let classes = tails.len(); + let slots = classes.min(1usize.checked_shl(lean_rounds as u32)?); + if !fits(n_stack, tile_bits, lean_rounds, slots) { + return None; + } + Some(TilePlan { + tile_bits, + classes, + slots, + shares: planned, + eq, + }) +} + +#[cfg(test)] +mod tests { + //! The tile form against the host prover's own rounds: the weight written + //! out by [`weight_table`](crate::stacked_eval::weight_table), the message + //! lifted, [`sumcheck::prove_rounds`](crate::sumcheck::prove_rounds) over + //! their product — what a stacked opening does without a device — and the + //! same rounds over the plan's slot tables, on one transcript. A second + //! source for `whir_host_kat.cpp`'s check of the kernels, in another + //! language and with THIS planner, which the kernels' test does not use. + + use super::*; + use crate::mle::Mle; + use crate::poly::Composed; + use crate::stacked_eval::{WeightShare, weight_table}; + use crate::sumcheck; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as Gl; + + type FE = FieldElement; + type EE = FieldElement; + + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn ext(&mut self) -> EE { + EE::new([ + FE::from(self.next()), + FE::from(self.next()), + FE::from(self.next()), + ]) + } + } + + fn limbs(value: &EE) -> [u64; 3] { + crate::gpu::ext3_raw(value).expect("an ext3 element") + } + + fn from_limbs(eq: &[u64], at: usize) -> EE { + crate::gpu::ext3_from_raw::(&eq[at * 3..at * 3 + 3]) + } + + /// A stack of `2^n`: columns of at least `2^min_vars` rows at multiples of + /// their own size, with gaps, their points drawn from `tails` tails of + /// `tail_vars` coordinates (so that many classes at most) and a few heads + /// a height; `tall` puts a column of `2^(n−1)` first. + struct Stack { + n: usize, + columns: Vec<(usize, Vec, EE)>, + message: Vec, + } + + fn stack( + n: usize, + min_vars: usize, + tail_vars: usize, + tails: usize, + tall: bool, + seed: u64, + ) -> Stack { + let mut rng = Rng(seed | 1); + let tails: Vec> = (0..tails) + .map(|_| (0..tail_vars).map(|_| rng.ext()).collect()) + .collect(); + let mut heads: Vec>> = vec![Vec::new(); n + 1]; + let mut point = |m: usize, rng: &mut Rng| -> Vec { + let pick = (rng.next() % 3) as usize; + while heads[m].len() <= pick { + let head = (0..m - tail_vars).map(|_| rng.ext()).collect(); + heads[m].push(head); + } + let mut point = heads[m][pick].clone(); + point.extend( + tails[(rng.next() % tails.len() as u64) as usize] + .iter() + .cloned(), + ); + point + }; + let len = 1usize << n; + let (mut cursor, mut stop) = (0usize, len); + let mut columns = Vec::new(); + if tall { + let at = if rng.next() & 1 == 1 { len / 2 } else { 0 }; + columns.push((at, point(n - 1, &mut rng), rng.ext())); + if at == 0 { + cursor = len / 2; + } else { + stop = len / 2; + } + } + loop { + let top = (n - 2).max(min_vars); + let m = min_vars + (rng.next() as usize) % (top - min_vars + 1); + let size = 1usize << m; + let mut at = cursor.div_ceil(size) * size; + if rng.next().is_multiple_of(5) { + at += size; + } + if at + size > stop { + break; + } + columns.push((at, point(m, &mut rng), rng.ext())); + cursor = at + size; + } + let mut message = vec![FE::zero(); len]; + for (at, point, _) in &columns { + for cell in &mut message[*at..*at + (1usize << point.len())] { + *cell = FE::from(rng.next()); + } + } + Stack { + n, + columns, + message, + } + } + + /// The host prover's rounds over `(weight, message)`: the evaluations each + /// round sends, the challenges, and the two tables as the rounds leave + /// them. + fn host_rounds( + weight: Mle, + message: Mle, + rounds: usize, + ) -> (Vec>, Vec, Mle, Mle) { + let mut transcript = DefaultTranscript::::new(b"lean-tiles"); + let mut poly = Composed::new(vec![weight, message], |v: &[EE]| &v[0] * &v[1], 2).unwrap(); + let (proofs, challenges) = + sumcheck::prove_rounds(&mut poly, rounds, &mut transcript).unwrap(); + let mut parts = poly.into_polys(); + let message = parts.pop().unwrap(); + let weight = parts.pop().unwrap(); + ( + proofs.into_iter().map(|p| p.evaluations).collect(), + challenges, + weight, + message, + ) + } + + /// One stack: the plan's slot tables built here, their rounds on the same + /// transcript, and the bound weight rebuilt from the bound slot tables. + /// Returns the plan's `(classes, slots)`, or `None` when the stack does + /// not take the form. + fn check(s: &Stack, rounds: usize) -> Option<(usize, usize)> { + let raw: Vec<(usize, Vec, [u64; 3])> = s + .columns + .iter() + .map(|(at, point, scale)| (*at, point.iter().flat_map(limbs).collect(), limbs(scale))) + .collect(); + let plan = plan(&raw, s.n, rounds)?; + let u = plan.tile_bits; + let tiles = 1usize << (s.n - u); + let columns = tiles >> rounds; + + // Today: the weight as a table, the message lifted, the host's rounds. + let shares: Vec> = s + .columns + .iter() + .map(|(at, point, scale)| WeightShare { + offset: *at, + point, + scale: *scale, + }) + .collect(); + let weight = weight_table(&shares, s.n).unwrap(); + let message = + Mle::new(s.message.iter().map(|v| v.to_extension::()).collect()).unwrap(); + let (want, want_challenges, want_weight, _) = host_rounds(weight, message, rounds); + + // The tile form. Which share holds a tile, then each column of tiles' + // slots in the order its tiles meet their classes. + let share_of = |tile: usize| -> Option<&TileShare> { + plan.shares.iter().find(|share| { + let first = share.stack_offset >> u; + tile >= first && tile < first + (1usize << (share.num_vars - u)) + }) + }; + let mut slot_lo: Vec> = vec![Vec::new(); columns]; + let mut omega = vec![vec![EE::zero(); tiles]; plan.slots]; + let mut phi = vec![vec![EE::zero(); tiles]; plan.slots]; + for (column, slots) in slot_lo.iter_mut().enumerate() { + for q in 0..1usize << rounds { + let tile = q * columns + column; + let Some(share) = share_of(tile) else { + continue; + }; + let slot = match slots.iter().position(|lo| *lo == share.lo_at) { + Some(slot) => slot, + None => { + slots.push(share.lo_at); + slots.len() - 1 + } + }; + assert!( + slot < plan.slots, + "a column of tiles met more classes than slots" + ); + omega[slot][tile] = crate::gpu::ext3_from_raw::(&share.scale) + * from_limbs(&plan.eq, share.hi_at + tile - (share.stack_offset >> u)); + } + } + for tile in 0..tiles { + for (slot, lo_at) in slot_lo[tile % columns].iter().enumerate() { + phi[slot][tile] = (0..1usize << u) + .map(|j| &s.message[(tile << u) + j] * &from_limbs(&plan.eq, lo_at + j)) + .fold(EE::zero(), |acc, term| acc + term); + } + } + let slots = plan.slots; + let mut tables: Vec> = Vec::with_capacity(2 * slots); + for table in omega.into_iter().chain(phi) { + tables.push(Mle::new(table).unwrap()); + } + let mut transcript = DefaultTranscript::::new(b"lean-tiles"); + let mut poly = Composed::new( + tables, + move |v: &[EE]| (0..slots).fold(EE::zero(), |acc, k| acc + &v[k] * &v[slots + k]), + 2, + ) + .unwrap(); + let (proofs, challenges) = + sumcheck::prove_rounds(&mut poly, rounds, &mut transcript).unwrap(); + let got: Vec> = proofs.into_iter().map(|p| p.evaluations).collect(); + assert_eq!(got, want, "the tiles' round polynomials are the opening's"); + assert_eq!(challenges, want_challenges); + + // The weight the rounds leave, from the bound `Ω` and the classes' + // tables. + let bound = poly.into_polys(); + for (y, want) in want_weight.evals().iter().enumerate() { + let (tile, j) = (y >> u, y & ((1usize << u) - 1)); + let got = slot_lo[tile % columns].iter().enumerate().fold( + EE::zero(), + |acc, (slot, lo_at)| { + acc + &bound[slot].evals()[tile] * from_limbs(&plan.eq, lo_at + j) + }, + ); + assert_eq!(&got, want, "the bound weight at {y}"); + } + Some((plan.classes, plan.slots)) + } + + /// ★ The tile form's rounds are the host prover's, on stacks of one to + /// eight classes, with gaps and a column taller than the blocks the + /// rounds bind, for one to four rounds. + #[test] + fn the_tile_form_sends_the_host_provers_rounds() { + let mut by_slots = [0usize; MAX_SLOTS + 1]; + let mut planned = 0; + for trial in 0..96u64 { + let n = 11 + (trial % 3) as usize; // 11..13 + let tail_vars = 4 + (trial / 3 % 3) as usize; // the smallest column: 4..6 + let rounds = 1 + (trial / 9 % 4) as usize; // 1..4 + let tails = 1 + (trial % 9) as usize; + let s = stack( + n, + tail_vars, + tail_vars, + tails, + trial % 5 == 0, + 0x9E37_79B9 + trial, + ); + if let Some((_, slots)) = check(&s, rounds) { + by_slots[slots] += 1; + planned += 1; + } + } + println!("{planned} stacks planned; by slots {by_slots:?}"); + assert!(by_slots[1] > 0 && by_slots[2] > 0 && by_slots[4] > 0 && by_slots[8] > 0); + } + + /// A column shorter than the cap sets the tile, and columns of one height + /// in a table share both halves; the classes are the distinct tails. + #[test] + fn the_plan_cuts_at_the_smallest_column_and_names_a_class_by_its_tail() { + let mut rng = Rng(0xC0FFEE); + let table: Vec = (0..12).flat_map(|_| limbs(&rng.ext())).collect(); + let tail = table[(12 - 7) * 3..].to_vec(); + // Another height whose last seven coordinates are the table's. + let mut short: Vec = (0..2).flat_map(|_| limbs(&rng.ext())).collect(); + short.extend_from_slice(&tail); + let other: Vec = (0..7).flat_map(|_| limbs(&rng.ext())).collect(); + let scale = limbs(&rng.ext()); + let shares = vec![ + (0usize, table.clone(), scale), + (1 << 12, table.clone(), scale), + (1 << 13, short, scale), + ((1 << 13) + (1 << 9), other, scale), + ]; + let plan = plan(&shares, 16, 3).expect("the stack takes the form"); + assert_eq!(plan.tile_bits, 7, "the smallest column has seven variables"); + assert_eq!((plan.classes, plan.slots), (2, 2)); + assert_eq!(plan.shares[0].hi_at, plan.shares[1].hi_at); + assert_eq!(plan.shares[0].lo_at, plan.shares[1].lo_at); + assert_eq!( + plan.shares[2].lo_at, plan.shares[0].lo_at, + "one tail, one class" + ); + assert_ne!(plan.shares[2].hi_at, plan.shares[0].hi_at); + assert_ne!(plan.shares[3].lo_at, plan.shares[0].lo_at); + // 2^5 + 2^7 (the table), 2^2 (the short head), 2^0 + 2^7 (the other). + assert_eq!(plan.eq.len() / 3, 32 + 128 + 4 + 1 + 128); + + // Tall columns are cut at the cap. + let tall: Vec = (0..15).flat_map(|_| limbs(&rng.ext())).collect(); + let plan = super::plan(&[(0, tall, scale)], 16, 3).unwrap(); + assert_eq!((plan.tile_bits, plan.slots), (TILE_BITS_CAP, 1)); + assert_eq!(plan.eq.len() / 3, 32 + 1024); + } + + /// What does not take the form: a column off its own alignment, no column, + /// no round, and a stack whose form would hold more than the tables + /// written after it. + #[test] + fn the_plan_declines_what_the_form_cannot_hold() { + let mut rng = Rng(7); + let scale = limbs(&rng.ext()); + let mut point = + |vars: usize| -> Vec { (0..vars).flat_map(|_| limbs(&rng.ext())).collect() }; + assert!(plan(&[], 16, 3).is_none()); + assert!(plan(&[(0, point(8), scale)], 16, 0).is_none()); + assert!( + plan(&[(3 << 6, point(8), scale)], 16, 3).is_none(), + "off its alignment" + ); + // A column of two variables beside a tall one of its class: tiles of + // four positions, and the form at three rounds would be over twice the + // tables after it. + let tall = point(12); + let small = tall[10 * 3..].to_vec(); + let stack = [(0, tall.clone(), scale), (1 << 12, small, scale)]; + assert!(plan(&stack, 16, 3).is_none()); + // A column of three variables, one round: through the round the form + // is under the tables after it, but what the write keeps beside them + // would be more than the share form's position map. + let three = tall[9 * 3..].to_vec(); + let stack = [(0, tall.clone(), scale), (1 << 12, three, scale)]; + assert!(plan(&stack, 16, 1).is_none()); + assert!(device_bytes(16, 3, 1, 1) <= 48 << 15 && write_bytes(16, 3, 1, 1) > 2 << 16); + // Four variables fit, at one round and at three. + let four = tall[8 * 3..].to_vec(); + let stack = [(0, tall, scale), (1 << 12, four, scale)]; + for rounds in [1, 3] { + assert_eq!( + plan(&stack, 16, rounds).map(|p| (p.tile_bits, p.slots)), + Some((4, 1)) + ); + } + // Nine classes at four rounds are more slots than the kernels have. + let nine: Vec<_> = (0..9).map(|c| (c << 12, point(12), scale)).collect(); + assert!(plan(&nine, 16, 4).is_none()); + // At three rounds a column of tiles meets eight of them at most. + assert_eq!( + plan(&nine, 16, 3).map(|p| (p.classes, p.slots)), + Some((9, 8)) + ); + } + + /// The two stacks `tests/whir_lean_device.rs` proves on the card: the first, + /// with a column of eight rows and nine points, is declined; the second, no + /// column under `2^9` rows and four tables, is planned with four slots. + #[test] + fn the_device_tests_two_stacks_are_one_declined_and_one_planned() { + let stack = |heights: &[usize], tables: &[usize]| -> Vec<(usize, Vec, [u64; 3])> { + let layout = crate::stacking::StackedLayout::build(heights, 16).unwrap(); + layout + .placements() + .iter() + .zip(heights.iter().zip(tables)) + .map(|(place, (&h, &table))| { + let point = (0..h) + .flat_map(|j| limbs(&EE::from(101 + 13 * table as u64 + j as u64))) + .collect(); + (place.offset, point, [1, 0, 0]) + }) + .collect() + }; + let first = stack( + &[13, 13, 13, 12, 12, 10, 8, 6, 3], + &[0, 1, 2, 3, 4, 5, 6, 7, 8], + ); + assert!(plan(&first, 16, 3).is_none()); + let second = stack( + &[13, 13, 13, 12, 12, 12, 11, 9, 9], + &[0, 0, 0, 1, 1, 1, 2, 3, 3], + ); + let planned = plan(&second, 16, 3).expect("the second stack takes the form"); + assert_eq!( + (planned.tile_bits, planned.classes, planned.slots), + (9, 4, 4) + ); + assert!(plan(&second, 16, 6).is_some(), "and at six rounds"); + } + + /// ★ The opening's working set does not grow: wherever the form fits, all + /// it holds through the rounds is no larger than the tables written once + /// they are spent, and what it keeps through the write is no larger than + /// the share form's position map, which it never allocates. At the + /// production stack and three rounds: 6.4 MiB a slot against 768 MiB, and + /// 3.3 MiB a slot against 256 MiB. + #[test] + fn the_form_holds_no_more_than_the_tables_written_after_it() { + for n in [16usize, 22, 25, 27, 28] { + for lean_rounds in 1..=6usize { + for tile_bits in 0..=TILE_BITS_CAP.min(n) { + for slots in 1..=MAX_SLOTS { + if fits(n, tile_bits, lean_rounds, slots) { + let after = 48u64 << (n - lean_rounds); + assert!(device_bytes(n, tile_bits, lean_rounds, slots) <= after); + // Beside those tables, no more than the share + // form's position map. + assert!(write_bytes(n, tile_bits, lean_rounds, slots) <= 2u64 << n); + } + } + } + } + } + let mib = |bytes: u64| bytes as f64 / (1u64 << 20) as f64; + let after = 48u64 << (27 - 3); + assert_eq!(after, 768 << 20); + for (slots, want) in [(1usize, 6.4), (2, 12.5), (8, 49.3)] { + let held = device_bytes(27, 10, 3, slots); + println!( + "2^27, tiles of 2^10, three rounds, {slots} slot(s): {:.1} MiB", + mib(held) + ); + assert!((mib(held) - want).abs() < 0.1, "{} MiB", mib(held)); + assert!(fits(27, 10, 3, slots)); + // Through the write: `Ω`, the tile map and the slots, against the + // 256 MiB map the share form holds there. + let kept = write_bytes(27, 10, 3, slots); + assert!(kept * 8 < 2 << 27, "{} MiB", mib(kept)); + } + // The smallest tile that fits at the production stack and three + // rounds: 2^4 positions for one slot, 2^7 for eight. + let smallest = |slots| (0..=TILE_BITS_CAP).find(|u| fits(27, *u, 3, slots)); + assert_eq!( + (smallest(1), smallest(2), smallest(8)), + (Some(4), Some(5), Some(7)) + ); + } + + /// The device's own rule is this one. + #[cfg(feature = "cuda")] + #[test] + fn the_device_fits_what_the_plan_fits() { + assert_eq!(MAX_SLOTS, math_cuda::whir_open::TILE_MAX_SLOTS); + for n in [12usize, 20, 27] { + for lean_rounds in 0..=7usize { + for tile_bits in 0..=n { + for slots in 0..=MAX_SLOTS + 1 { + let spec = math_cuda::whir_open::TileSpec { + tile_bits, + lean_rounds, + slots, + wide: false, + check: false, + }; + assert_eq!( + spec.fits(n), + fits(n, tile_bits, lean_rounds, slots), + "2^{n}, tiles 2^{tile_bits}, {lean_rounds} rounds, {slots} slots" + ); + if spec.fits(n) { + assert_eq!( + spec.device_bytes(n), + device_bytes(n, tile_bits, lean_rounds, slots) + ); + } + } + } + } + } + } +} diff --git a/crypto/multilinear/src/lib.rs b/crypto/multilinear/src/lib.rs new file mode 100644 index 000000000..02947d1c2 --- /dev/null +++ b/crypto/multilinear/src/lib.rs @@ -0,0 +1,237 @@ +//! Multilinear machinery for a sumcheck-based proof system: extensions over the +//! Boolean hypercube, batched sumcheck, zerocheck, LogUp-GKR, stacking and +//! WHIR. +//! +//! Not wired into the prover. The codeword domain is a two-adic subgroup of the +//! base field; codeword values live in the extension. + +pub mod batch; +pub mod claim_reduce; +pub mod commit_ood; +pub mod constraint_argument; +pub mod eq; +pub mod front_loaded; +pub mod fused; +pub mod gkr; +pub mod gkr_gruen; +pub mod gkr_input_real; +pub mod gkr_lockstep; +pub mod gpu; +pub mod gpu_fused; +pub mod lean_tiles; +pub mod logup; +pub mod mle; +pub mod narrow; +pub mod poly; +pub mod program; +pub mod query_count; +pub mod records; +pub mod round_slot; +pub mod selector; +pub mod stacked_eval; +pub mod stacking; +pub mod sumcheck; +pub mod uneven; +pub mod uni_skip; +pub mod virtual_poly; +pub mod whir; +#[cfg(test)] +mod whir_cap_tests; +pub mod whir_chain; +pub mod whir_commit; +#[cfg(test)] +mod whir_commit_ood_tests; +#[cfg(test)] +mod whir_coset_tests; +pub mod whir_eval; +#[cfg(test)] +mod whir_group_tests; +pub mod whir_hash; +#[cfg(test)] +mod whir_p1_tests; +pub mod whir_round; +#[cfg(test)] +mod whir_run_claim_tests; +pub mod whir_split; +pub mod zerocheck; + +use math::field::{element::FieldElement, traits::IsField}; +use thiserror::Error; + +/// Below this a pass stays on the thread that asked for it: handing a slice to +/// the pool costs tens of microseconds whatever is in it, and a sumcheck's last +/// rounds are over a few hundred values. +#[cfg(feature = "parallel")] +pub(crate) const SERIAL_BELOW: usize = 1 << 12; + +/// The cube below which a sumcheck's rounds belong here, for a rule the host +/// evaluates directly: a device round costs the same whatever the cube — one +/// thread walks the whole rule at each index — and a host round is linear in it. +pub(crate) const HOST_CUBE_DIRECT: usize = 1 << 9; + +/// The same for a rule the host walks through the program interpreter, which +/// costs about ten times a multiplication written out per step while the device +/// round costs the same either way. +pub(crate) const HOST_CUBE_COMPILED: usize = 1 << 5; + +/// `[1, gamma, gamma^2, ..]` — the weights a batching challenge expands into. +/// +/// `pub` for the recursion emitter, which reproduces these weights inside the +/// field machine: a leg with no host function on the other side of the equals +/// sign can only be gated against a copy of itself, which is no evidence about +/// either. Every caller inside this crate reaches it through the batching +/// routines below. +pub fn challenge_powers(gamma: &FieldElement, count: usize) -> Vec> { + let mut acc = FieldElement::::one(); + (0..count) + .map(|_| { + let current = acc.clone(); + acc = &acc * gamma; + current + }) + .collect() +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum Error { + #[error("expected a power-of-two number of evaluations, got {0}")] + NotPowerOfTwo(usize), + #[error("expected {expected} variables, got {got}")] + VariableCountMismatch { expected: usize, got: usize }, + #[error("polynomial has no variables left to fold")] + NoVariablesLeft, + #[error("term references polynomial index {index}, but only {len} are registered")] + UnknownPolynomial { index: usize, len: usize }, + #[error("virtual polynomial has no terms")] + EmptyPolynomial, + #[error("round {round}: expected a degree-{expected} polynomial, got {got} evaluations")] + RoundDegreeMismatch { + round: usize, + expected: usize, + got: usize, + }, + #[error("proof has {got} rounds, expected {expected}")] + RoundCountMismatch { expected: usize, got: usize }, + #[error("final evaluation does not match the oracle: expected {expected}, got {got}")] + FinalEvaluationMismatch { expected: String, got: String }, + #[error("{exemptions} exempted steps exceed the {size}-step trace")] + TooManyExemptions { exemptions: usize, size: usize }, + #[error("layer {layer}: the sumcheck residual does not match the fraction-fold relation")] + LayerRelationMismatch { layer: usize }, + #[error("a column on {column_vars} variables does not fit a {n_stack}-variable stack")] + ColumnTallerThanStack { column_vars: usize, n_stack: usize }, + #[error("no subgroup of order 2^{l_skip} exists (field two-adicity is {two_adicity})")] + SkipDomainUnavailable { l_skip: usize, two_adicity: usize }, + #[error("{coefficients} coefficients do not fit a domain of {domain} points")] + CodewordTooShort { coefficients: usize, domain: usize }, + #[error("query index {index} is outside the {bound} committed leaves")] + QueryOutOfRange { index: usize, bound: usize }, + #[error("expected {expected} query openings, got {got}")] + QueryCountMismatch { expected: usize, got: usize }, + /// A chain's domain is not its code's: `2^(num_vars + log_blowup)` points + /// at the configuration's rate. A smaller one would take more words for + /// codewords. + #[error("the domain has 2^{got} points, the code 2^{expected}")] + DomainRateMismatch { expected: usize, got: usize }, + #[error("query {query}: the Merkle opening does not match the commitment")] + OpeningRejected { query: usize }, + /// A tree's Merkle cap, carried by its first opening, is the wrong length + /// or does not hash to the tree's root. + #[error("a Merkle cap does not authenticate against its root")] + CapRejected, + /// The prover could not cut its paths to the cap: a policy asked for a cap + /// taller than the tree, or a path had the wrong length. + #[error("could not embed the Merkle cap: {reason}")] + CapEmbedFailed { reason: &'static str }, + #[error("query {query}: the folded block does not match the committed successor")] + FoldInconsistent { query: usize }, + #[error("the folded codeword and the sumcheck disagree on the evaluation")] + EvaluationMismatch, + #[error("eq(z, alpha) vanished, leaving the evaluation unconstrained")] + DegenerateEvaluationPoint, + #[error("the out-of-domain point landed inside the evaluation domain")] + OodPointInDomain, + /// A stack's commit-time out-of-domain answers ([`commit_ood`]) are not one + /// per committed polynomial: missing or extra answers. + #[error("expected {expected} commit-time out-of-domain answers, got {got}")] + CommitOodCount { expected: usize, got: usize }, + #[error("no proof-of-work nonce found for {bits} bits")] + GrindingFailed { bits: u8 }, + #[error("a proof-of-work nonce does not carry the required {bits} bits")] + GrindingRejected { bits: u8 }, + #[error("a nonce the proof format does not carry is set")] + UnspentNonce, + #[error("the buses do not balance across the proof")] + BusImbalance, + #[error("the statements rebuilt from the factor values are not what the batch demands")] + BatchMismatch, + #[error("a shifted read is not the shift of the column it claims to shift")] + ShiftedReadMismatch, + #[error("column {column}: its claimed value does not match the commitment")] + ColumnOpeningRejected { column: usize }, + /// A device path that had already drawn a challenge cannot be retried on + /// the host: the transcript has moved. + #[error("the device failed mid-{stage}, after the transcript had moved")] + DeviceFailed { stage: &'static str }, + /// A spilled table's packed columns did not come back whole: the bytes + /// read are not the ones written, the read failed, or they do not have + /// the table's shape. The prover refuses rather than prove over other + /// columns. + #[error("table {table}: its spilled columns did not come back ({reason})")] + SpillFailed { table: usize, reason: &'static str }, + /// A commitment rebuilt from a kept tree top and a recomputed codeword: + /// the codeword's leaves under one kept node do not hash to it, so the + /// recomputation is not the committed codeword. + #[error("leaf block {block}: the recomputed codeword does not hash to the kept tree")] + RecomputedCodewordMismatch { block: usize }, + /// A constraint program's constant has an extension part, so the program + /// cannot run in the base field (`fused`). + #[error("step {step} of the constraint program is not a base-field constant")] + NotBaseField { step: usize }, + /// The trace breaks its constraints at a row the stage-1 rounds would have + /// taken as satisfied (`fused`'s corner check). + #[error("the trace violates its constraints at row {row}")] + ConstraintViolated { row: usize }, + /// A proof's argue variant is not the one the configuration names + /// ([`whir_chain::ArgueFormat`]): the prover and the verifier each refuse + /// to cross formats, since each reads the variant from its own config. + #[error("the proof's argue format is not the configured one")] + ArgueFormatMismatch, + /// A batched argue's shape (bins, trees, tables, reductions) is not the + /// one the verifier derives from the statement. + #[error("the batched argue's {part} does not have the shape the statement implies")] + ArgueShapeMismatch { part: &'static str }, + /// A grouped opening ([`whir_chain::GroupFold::Indexed`]) is not the shape + /// the verifier derives from its own layout and config: refused by name + /// before anything is indexed. + #[error("the grouped opening's shape is not the derived one: {0:?}")] + GroupedShape(whir_chain::GroupedShape), + /// A grouped round 0's block of round-0 tree `tree` at query `query` does + /// not open against that tree's own root (or cap). + #[error("query {query}: tree {tree}'s block does not open against its root")] + GroupedOpeningRejected { query: usize, tree: usize }, + /// The format's round-0 leaf order is not defined for the hash its trees + /// are built with: the coset order is Poseidon1's alone. Refused before a + /// tree is built or a proof is read. + #[error("the {order:?} round-0 leaf order is not defined under {hash}")] + LeafOrderUnsupported { + order: whir_commit::LeafOrder, + hash: &'static str, + }, + /// A round-0 commitment placed its leaves in another order than the + /// config it is proved under. + #[error("a round-0 commitment's leaves are in the {got:?} order, the config's in {expected:?}")] + LeafOrderMismatch { + expected: whir_commit::LeafOrder, + got: whir_commit::LeafOrder, + }, + /// Under the coset round-0 leaf order, a round-0 codeword on the card (or + /// held as its stack) whose first successor was not encoded from the + /// bound message: the prove is refused rather than folded, by cause. + #[error( + "the coset round-0 order needs the first successor encoded from the bound message: {cause:?}" + )] + CosetSuccessorNotEncoded { + cause: whir_chain::UnencodedSuccessor, + }, +} diff --git a/crypto/multilinear/src/logup.rs b/crypto/multilinear/src/logup.rs new file mode 100644 index 000000000..4c740d501 --- /dev/null +++ b/crypto/multilinear/src/logup.rs @@ -0,0 +1,869 @@ +//! A bus as a fraction tree over the trace, and the rules that settle its +//! input-layer claim against the trace's factors. +//! +//! A bus balances when `Σ ± mult / (z − fingerprint) = 0`. Both parts are +//! **affine** in the trace columns — a multiplicity is a linear combination of +//! flag columns, a fingerprint is limbs weighted by powers of two and bus +//! elements weighted by powers of a challenge — so neither is ever a column of +//! its own. [`FractionTree`](crate::gkr::FractionTree) proves the sum, and what +//! it leaves is one claim about the input layer. +//! +//! That claim reduces to **two rules**, whatever the number of interactions. +//! The input layer is indexed by `(interaction, row)` with the interaction in +//! the high variables, so the claim point splits and +//! `p(hi ‖ lo) = Σ_i eq(hi, i)·p_i(lo)`: the interaction weights are constants +//! the verifier computes, leaving a single affine combination of factors per +//! side. Both rules are degree 2 once the row weight is counted. + +use math::field::{element::FieldElement, traits::IsField}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{ + Error, + batch::Rule, + eq::eq_evals, + gkr::FractionLayer, + mle::Mle, + program::{Builder, Program}, +}; + +/// An affine expression over the sumcheck factors: `Σ c_j·f_{s_j} + k`. +#[derive(Clone, Debug)] +pub struct Affine { + terms: Vec<(usize, FieldElement)>, + constant: FieldElement, +} + +impl Affine { + /// `terms` pairs a factor slot with its coefficient. + pub fn new(terms: Vec<(usize, FieldElement)>, constant: FieldElement) -> Self { + Self { terms, constant } + } + + pub fn constant(value: FieldElement) -> Self { + Self::new(Vec::new(), value) + } + + /// A single factor read with coefficient one. + pub fn factor(slot: usize) -> Self { + Self::new(vec![(slot, FieldElement::one())], FieldElement::zero()) + } + + pub fn terms(&self) -> &[(usize, FieldElement)] { + &self.terms + } + + /// The `k` of `Σ c_j·f_{s_j} + k`. + /// + /// Read-only, and named `constant_term` rather than `constant` because that + /// name is already the CONSTRUCTOR above. A caller that emits this + /// expression as code — the WHIR recursion's in-guest verifier — needs the + /// coefficients and this, where [`Affine::evaluate`] gives it only the + /// answer. + pub fn constant_term(&self) -> &FieldElement { + &self.constant + } + + /// The expression's value, given every factor's value at a point. + pub fn evaluate(&self, values: &[FieldElement]) -> FieldElement { + self.terms + .iter() + .fold(self.constant.clone(), |acc, (slot, coefficient)| { + acc + coefficient * &values[*slot] + }) + } + + /// The expression as steps over the factor values, returning the step + /// holding it. + pub fn emit(&self, builder: &mut Builder) -> u32 { + if self.terms.is_empty() { + return builder.fixed(self.constant.clone()); + } + let terms: Vec<(u32, FieldElement)> = self + .terms + .iter() + .map(|(slot, coefficient)| (builder.var(*slot), coefficient.clone())) + .collect(); + let sum = builder.weighted_sum(&terms); + if self.constant == FieldElement::zero() { + return sum; + } + let constant = builder.fixed(self.constant.clone()); + builder.add(sum, constant) + } + + /// The expression's table over the cube. `factors` must not be empty: its + /// first entry sets the height. + pub fn table(&self, factors: &[Mle]) -> Result, Error> + where + FieldElement: Send + Sync, + { + let size = factors.first().ok_or(Error::EmptyPolynomial)?.len(); + let mut evals = vec![self.constant.clone(); size]; + for (slot, coefficient) in &self.terms { + let factor = factors.get(*slot).ok_or(Error::UnknownPolynomial { + index: *slot, + len: factors.len(), + })?; + if factor.len() != size { + return Err(Error::VariableCountMismatch { + expected: size.trailing_zeros() as usize, + got: factor.num_vars(), + }); + } + // One pass per term over the whole column. The bus builds one of + // these per interaction, so it is a hot loop on a real table. + #[cfg(feature = "parallel")] + evals + .par_iter_mut() + .zip(factor.evals().par_iter()) + .for_each(|(slot, value)| *slot += coefficient * value); + #[cfg(not(feature = "parallel"))] + for (slot, value) in evals.iter_mut().zip(factor.evals()) { + *slot += coefficient * value; + } + } + Mle::new(evals) + } +} + +/// One bus interaction on one table. +/// +/// `numerator` is the **signed** multiplicity — a receiver's sign is already in +/// its coefficients — and `denominator` is `z − fingerprint`. +#[derive(Clone, Debug)] +pub struct Interaction { + pub numerator: Affine, + pub denominator: Affine, +} + +impl Interaction { + pub fn new(numerator: Affine, denominator: Affine) -> Self { + Self { + numerator, + denominator, + } + } +} + +/// The number of variables the input layer spans: the rows plus the bits that +/// index the interaction. +pub fn input_layer_vars(interactions: usize, num_row_vars: usize) -> usize { + num_row_vars + interactions.next_power_of_two().trailing_zeros() as usize +} + +/// The fraction tree's input layer, indexed by `(interaction, row)` with the +/// interaction in the high variables. +/// +/// Interactions are padded up to a power of two with `0/1`, which the tree adds +/// without moving the sum. +pub fn input_layer( + interactions: &[Interaction], + factors: &[Mle], +) -> Result, Error> { + if interactions.is_empty() { + return Err(Error::EmptyPolynomial); + } + let size = factors.first().ok_or(Error::EmptyPolynomial)?.len(); + let slots = interactions.len().next_power_of_two(); + + let mut p = Vec::with_capacity(slots * size); + let mut q = Vec::with_capacity(slots * size); + for interaction in interactions { + p.extend(interaction.numerator.table(factors)?.into_evals()); + q.extend(interaction.denominator.table(factors)?.into_evals()); + } + for _ in interactions.len()..slots { + p.extend(std::iter::repeat_n(FieldElement::::zero(), size)); + q.extend(std::iter::repeat_n(FieldElement::::one(), size)); + } + + FractionLayer::new(Mle::new(p)?, Mle::new(q)?) +} + +/// A table's input layer as a plan over its base columns (D-BATCH M1-2): for +/// each interaction's two sides, its terms — where the column a factor reads +/// starts in the table's run of columns (`column · rows`) and its shift, mod +/// `rows` — with their ext3 coefficients, and the side's constant, all as raw +/// limbs for the card. +/// +/// Side `2i` is interaction `i`'s numerator, `2i + 1` its denominator; +/// `side_start[s] .. side_start[s + 1]` are side `s`'s terms. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InputPlan { + pub side_start: Vec, + pub terms: Vec, + pub coeffs: Vec, + pub constants: Vec, +} + +impl InputPlan { + pub fn interactions(&self) -> usize { + self.side_start.len() / 2 + } +} + +/// The plan for `interactions` over factors whose sources are `sources` (one +/// per slot: the committed column and shift a factor reads, `None` for a +/// public factor). `None` when a side reads a public factor, when a slot is +/// unknown, or when `E` is not the cubic extension the card works in. +pub fn input_plan( + interactions: &[Interaction], + sources: &[Option], + rows: usize, +) -> Option { + let mut plan = InputPlan { + side_start: vec![0], + terms: Vec::new(), + coeffs: Vec::new(), + constants: Vec::new(), + }; + for interaction in interactions { + for side in [&interaction.numerator, &interaction.denominator] { + for (slot, coefficient) in side.terms() { + let source = (*sources.get(*slot)?)?; + plan.terms.push((source.column * rows) as u64); + plan.terms.push((source.offset % rows.max(1)) as u64); + plan.coeffs + .extend_from_slice(&crate::gpu::ext3_raw(coefficient)?); + } + plan.constants + .extend_from_slice(&crate::gpu::ext3_raw(side.constant_term())?); + plan.side_start + .push(u32::try_from(plan.terms.len() / 2).ok()?); + } + } + Some(plan) +} + +/// The input layer and the tree above it, written straight from the table's +/// resident base columns ([`crate::gpu::input_layer_tree_from_columns`]), +/// under `LAMBDA_VM_ARGUE_GKR_INPUT`. `None` when the columns are not on the +/// card, a side reads a public factor, or the card declines — the caller then +/// takes today's path. +pub fn resident_tree_from_columns( + interactions: &[Interaction], + trace: &crate::constraint_argument::TraceData, +) -> Option> { + if interactions.is_empty() { + return None; + } + let (store, first) = trace.resident_shared()?; + if trace.num_columns() == 0 { + return None; + } + let rows = 1usize << trace.num_vars(); + let sources: Vec<_> = trace + .kinds() + .iter() + .map(crate::constraint_argument::FactorKind::source) + .collect(); + let plan = input_plan(interactions, &sources, rows)?; + let tree = + crate::gpu::input_layer_tree_from_columns(store, first, trace.num_columns(), rows, &plan)?; + crate::gkr::FractionTree::from_device(tree).ok() +} + +/// [`resident_tree_from_columns`] with its output fraction left unread +/// (`LAMBDA_VM_ARGUE_ROUND_SLOT`): a caller that builds several trees queues +/// them all before it waits for any, then finalizes each with +/// [`PrefetchedTree::into_tree`] — before anything is absorbed, so a tree whose +/// output cannot be read is built here instead. +pub fn resident_tree_from_columns_deferred( + interactions: &[Interaction], + trace: &crate::constraint_argument::TraceData, +) -> Option> { + if interactions.is_empty() { + return None; + } + let (store, first) = trace.resident_shared()?; + if trace.num_columns() == 0 { + return None; + } + let rows = 1usize << trace.num_vars(); + let sources: Vec<_> = trace + .kinds() + .iter() + .map(crate::constraint_argument::FactorKind::source) + .collect(); + let plan = input_plan(interactions, &sources, rows)?; + let device = crate::gpu::input_layer_tree_from_columns_deferred( + store, + first, + trace.num_columns(), + rows, + &plan, + )?; + Some(PrefetchedTree { + device, + _marker: core::marker::PhantomData, + }) +} + +/// The input layer and the tree above it, built where the factors already are. +/// +/// The layer is `interactions × rows` fractions — the biggest thing a table's +/// argument builds — and every one of its cells is an affine expression over +/// the factors, which is a program the device can run. +/// +/// `None` when the device declines; the caller then builds the layer here. +pub fn resident_tree( + interactions: &[Interaction], + factors: std::sync::Arc, +) -> Option> { + if interactions.is_empty() { + return None; + } + let emit = |side: &Affine| { + let mut builder = Builder::::new(); + let root = side.emit(&mut builder); + builder.finish(root).ok() + }; + let numerators: Vec> = interactions + .iter() + .map(|i| emit(&i.numerator)) + .collect::>()?; + let denominators: Vec> = interactions + .iter() + .map(|i| emit(&i.denominator)) + .collect::>()?; + + let tree = crate::gpu::input_layer_tree(factors, numerators, denominators)?; + crate::gkr::FractionTree::from_device(tree).ok() +} + +/// A device fraction tree built ahead of its turn, its output fraction left +/// unread so the fold kernels stay in flight — see [`resident_tree_deferred`]. +/// Finalized at the consume site with [`Self::into_tree`]. +pub struct PrefetchedTree { + device: crate::gpu::DeviceTree, + _marker: core::marker::PhantomData, +} + +impl PrefetchedTree { + /// Read the output fraction (the sync the prefetch deferred) and wrap the + /// tree for `prove`. `None` on a device read failure — the caller then + /// builds host-side; the transcript has not moved here, so that is sound. + pub fn into_tree(self) -> Option> { + crate::gkr::FractionTree::from_device(self.device).ok() + } +} + +/// The prefetch sibling of [`resident_tree`]: builds the device tree WITHOUT +/// reading its output, so the fold kernels overlap the caller's argue instead +/// of being waited on. `None` when the device declines (as [`resident_tree`]) +/// OR there is no non-evicting VRAM headroom for a second tree — prefetch is +/// subordinate to argue and to the round-2 retention, displacing neither. +pub fn resident_tree_deferred( + interactions: &[Interaction], + factors: std::sync::Arc, +) -> Option> { + if interactions.is_empty() { + return None; + } + let emit = |side: &Affine| { + let mut builder = Builder::::new(); + let root = side.emit(&mut builder); + builder.finish(root).ok() + }; + let numerators: Vec> = interactions + .iter() + .map(|i| emit(&i.numerator)) + .collect::>()?; + let denominators: Vec> = interactions + .iter() + .map(|i| emit(&i.denominator)) + .collect::>()?; + + let device = crate::gpu::input_layer_tree_deferred(factors, numerators, denominators)?; + Some(PrefetchedTree { + device, + _marker: core::marker::PhantomData, + }) +} + +/// What the batch needs to settle a bus's input-layer claim. +pub struct BusStatements<'a, E: IsField> { + pub numerator: Rule<'a, E>, + pub denominator: Rule<'a, E>, + /// The row half of the claim point — where the weight table belongs. + pub row_point: Vec>, +} + +/// Turns a GKR input-layer claim into two rules over the trace's factors. +/// +/// `weight` is the factor slot holding `eq(row_point, ·)`, which the caller +/// adds as a public factor. Both sides must build these from the same +/// interactions: they are the bus's structure, not proof data. +pub fn claim_statements<'a, E: IsField + 'static>( + interactions: &'a [Interaction], + claim_point: &[FieldElement], + num_row_vars: usize, + weight: usize, +) -> Result, Error> { + let expected = input_layer_vars(interactions.len(), num_row_vars); + if claim_point.len() != expected { + return Err(Error::VariableCountMismatch { + expected, + got: claim_point.len(), + }); + } + let (interaction_point, row_point) = claim_point.split_at(claim_point.len() - num_row_vars); + + // Weight per interaction; the tail belongs to the 0/1 padding slots, whose + // denominators are one and whose numerators vanish. + let weights = eq_evals(interaction_point); + let padding = weights[interactions.len()..] + .iter() + .fold(FieldElement::::zero(), |acc, w| acc + w); + let live = weights[..interactions.len()].to_vec(); + + // `Σ_i w_i · side_i(f)` (plus the padding, where it belongs), times the row + // weight — the shape both sides of the bus statement take. + let weighted = |sides: Vec<&Affine>, constant: Option>| { + let mut builder = Builder::::new(); + let mut terms: Vec<(u32, FieldElement)> = sides + .into_iter() + .zip(&live) + .map(|(side, w)| (side.emit(&mut builder), w.clone())) + .collect(); + if let Some(constant) = constant { + let step = builder.fixed(constant); + terms.push((step, FieldElement::one())); + } + let sum = builder.weighted_sum(&terms); + let row = builder.var(weight); + let root = builder.mul(row, sum); + builder.finish(root) + }; + + let numerator: Program = + weighted(interactions.iter().map(|i| &i.numerator).collect(), None)?; + let denominator: Program = weighted( + interactions.iter().map(|i| &i.denominator).collect(), + Some(padding), + )?; + let numerator = Rule::compiled(2, numerator); + let denominator = Rule::compiled(2, denominator); + + Ok(BusStatements { + numerator, + denominator, + row_point: row_point.to_vec(), + }) +} + +#[cfg(test)] +mod input_plan_tests { + use super::*; + use crate::claim_reduce::FactorSource; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + + type EE = FieldElement; + + fn next(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed + } + + fn ext(seed: &mut u64) -> EE { + EE::new([next(seed).into(), next(seed).into(), next(seed).into()]) + } + + /// What `gkr_input_from_columns` computes, on the host: the kernel's + /// arithmetic, cell by cell, from the plan's raw limbs. + fn run_plan( + plan: &InputPlan, + columns: &[u64], + rows: usize, + padding: bool, + ) -> (Vec, Vec) { + let interactions = plan.interactions(); + let slots = if padding { + interactions.next_power_of_two() + } else { + interactions + }; + let raw = |limbs: &[u64]| crate::gpu::ext3_from_raw::(limbs); + let (mut p, mut q) = (Vec::new(), Vec::new()); + for s in 0..slots { + for r in 0..rows { + if s >= interactions { + p.push(EE::zero()); + q.push(EE::one()); + continue; + } + let side = |which: usize| { + let mut acc = raw(&plan.constants[which * 3..which * 3 + 3]); + for t in plan.side_start[which] as usize..plan.side_start[which + 1] as usize { + let mut k = r + plan.terms[2 * t + 1] as usize; + if k >= rows { + k -= rows; + } + let value = + FieldElement::::from(columns[plan.terms[2 * t] as usize + k]); + acc += raw(&plan.coeffs[t * 3..t * 3 + 3]) * value.to_extension::(); + } + acc + }; + p.push(side(2 * s)); + q.push(side(2 * s + 1)); + } + } + (p, q) + } + + /// ★ The plan's cells are the lifted path's: for interactions over direct + /// and shifted columns, padded or not, the kernel's arithmetic on the plan + /// equals `input_layer` over the lifted factors, cell for cell. + #[test] + fn the_plan_writes_the_lifted_input_layer() { + let mut seed = 0x5EED_u64; + for (rows, width, count) in [(1usize, 2usize, 1usize), (8, 3, 3), (32, 5, 5), (64, 7, 9)] { + let columns: Vec> = (0..width) + .map(|_| { + (0..rows) + .map(|_| next(&mut seed) % 0xFFFF_FFFF_0000_0001) + .collect() + }) + .collect(); + // Every column direct, then two shifted views, then a public slot. + let mut sources: Vec> = + (0..width).map(|c| Some(FactorSource::direct(c))).collect(); + sources.push(Some(FactorSource::shifted(0, 1))); + sources.push(Some(FactorSource::shifted(width - 1, 3))); + sources.push(None); + let factors: Vec> = sources + .iter() + .map(|source| { + let evals = (0..rows) + .map(|r| match source { + Some(src) => FieldElement::::from( + columns[src.column][(r + src.offset) % rows], + ) + .to_extension::(), + None => EE::from(7u64), + }) + .collect(); + Mle::new(evals).unwrap() + }) + .collect(); + let committed = sources.len() - 1; + let affine = |seed: &mut u64| { + let terms = (0..(next(seed) % 4) as usize) + .map(|_| ((next(seed) as usize) % committed, ext(seed))) + .collect(); + Affine::new(terms, ext(seed)) + }; + let interactions: Vec> = (0..count) + .map(|_| Interaction::new(affine(&mut seed), affine(&mut seed))) + .collect(); + let plan = input_plan(&interactions, &sources, rows).expect("a plan over columns"); + let flat: Vec = columns.concat(); + let want = input_layer(&interactions, &factors).unwrap(); + let (p, q) = run_plan(&plan, &flat, rows, true); + assert_eq!(p, want.p.evals(), "rows {rows}: the numerators differ"); + assert_eq!(q, want.q.evals(), "rows {rows}: the denominators differ"); + let (p, _) = run_plan(&plan, &flat, rows, false); + assert_eq!(p.len(), count * rows, "unpadded: the real cells only"); + assert_eq!(p[..], want.p.evals()[..count * rows]); + } + } + + /// A side that reads a public factor has no plan: the caller keeps + /// today's path. + #[test] + fn a_public_factor_has_no_plan() { + let sources = vec![Some(FactorSource::direct(0)), None]; + let interactions = vec![Interaction::new( + Affine::factor(1), + Affine::constant(EE::one()), + )]; + assert_eq!(input_plan::(&interactions, &sources, 4), None); + let interactions = vec![Interaction::new( + Affine::factor(0), + Affine::constant(EE::one()), + )]; + assert!(input_plan::(&interactions, &sources, 4).is_some()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + batch, + eq::eq_mle, + gkr::{self, FractionTree}, + }; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"logup-test") + } + + /// Factor layout: the value, the two half-flags, and the row weight. + const VALUE: usize = 0; + const LOW: usize = 1; + const HIGH: usize = 2; + const WEIGHT: usize = 3; + + /// The low half sends each value and the high half receives the same one, + /// so the bus balances *because of the values*, not by construction. + fn columns(num_vars: usize) -> Vec> { + let size = 1usize << num_vars; + let half = size / 2; + let value: Vec = (0..size).map(|i| FE::from((i % half) as u64 + 1)).collect(); + let low: Vec = (0..size) + .map(|i| if i < half { FE::one() } else { FE::zero() }) + .collect(); + let high: Vec = low.iter().map(|v| FE::one() - v).collect(); + vec![ + Mle::new(value).unwrap(), + Mle::new(low).unwrap(), + Mle::new(high).unwrap(), + ] + } + + /// `+low / (z − alpha·value)` against `−high / (z − alpha·value)`. + fn balanced(z: FE, alpha: FE, _num_vars: usize) -> Vec> { + vec![ + Interaction::new(Affine::factor(LOW), Affine::new(vec![(VALUE, -alpha)], z)), + Interaction::new( + Affine::new(vec![(HIGH, -FE::one())], FE::zero()), + Affine::new(vec![(VALUE, -alpha)], z), + ), + ] + } + + #[test] + fn an_affine_expression_evaluates_and_tabulates_alike() { + let factors = columns(3); + let expr = Affine::new(vec![(VALUE, FE::from(7)), (LOW, FE::from(11))], FE::from(5)); + let table = expr.table(&factors).unwrap(); + + for i in 0..factors[0].len() { + let values: Vec = factors.iter().map(|f| f.evals()[i]).collect(); + assert_eq!(table.evals()[i], expr.evaluate(&values), "row {i}"); + } + } + + #[test] + fn the_input_layer_lays_interactions_out_row_by_row() { + let num_vars = 3; + let size = 1usize << num_vars; + let factors = columns(num_vars); + let interactions = balanced(FE::from(97), FE::from(31), num_vars); + let layer = input_layer(&interactions, &factors).unwrap(); + + assert_eq!(layer.num_vars(), input_layer_vars(2, num_vars)); + for (i, interaction) in interactions.iter().enumerate() { + let p = interaction.numerator.table(&factors).unwrap(); + let q = interaction.denominator.table(&factors).unwrap(); + for row in 0..size { + // The interaction sits in the high variables. + assert_eq!(layer.p.evals()[i * size + row], p.evals()[row]); + assert_eq!(layer.q.evals()[i * size + row], q.evals()[row]); + } + } + } + + #[test] + fn padding_slots_are_the_zero_fraction() { + let num_vars = 2; + let size = 1usize << num_vars; + let factors = columns(num_vars); + let z = FE::from(97); + let alpha = FE::from(31); + // Three interactions pad up to four slots. + let mut interactions = balanced(z, alpha, num_vars); + interactions.push(Interaction::new( + Affine::constant(FE::zero()), + Affine::constant(z), + )); + + let layer = input_layer(&interactions, &factors).unwrap(); + assert_eq!(layer.num_vars(), num_vars + 2); + for row in 0..size { + assert_eq!(layer.p.evals()[3 * size + row], FE::zero()); + assert_eq!(layer.q.evals()[3 * size + row], FE::one()); + } + } + + #[test] + fn a_balanced_bus_has_a_zero_output_numerator() { + let num_vars = 3; + let factors = columns(num_vars); + let interactions = balanced(FE::from(97), FE::from(31), num_vars); + let tree = FractionTree::build(input_layer(&interactions, &factors).unwrap()).unwrap(); + + let (p, q) = tree.output(); + assert_eq!(p, FE::zero()); + assert_ne!(q, FE::zero()); + } + + #[test] + fn an_unbalanced_bus_does_not() { + let num_vars = 3; + let factors = columns(num_vars); + let z = FE::from(97); + let alpha = FE::from(31); + // Only the send: nothing cancels it. + let interactions = vec![balanced(z, alpha, num_vars).remove(0)]; + let tree = FractionTree::build(input_layer(&interactions, &factors).unwrap()).unwrap(); + + assert_ne!(tree.output().0, FE::zero()); + } + + #[test] + fn a_value_received_that_was_never_sent_unbalances_the_bus() { + // The multisets are what balance, so moving one value apart is enough. + let num_vars = 3; + let mut factors = columns(num_vars); + let mut value = factors[0].evals().to_vec(); + value[5] += FE::one(); + factors[0] = Mle::new(value).unwrap(); + + let interactions = balanced(FE::from(97), FE::from(31), num_vars); + let tree = FractionTree::build(input_layer(&interactions, &factors).unwrap()).unwrap(); + assert_ne!(tree.output().0, FE::zero()); + } + + /// The identity the batch leans on: each rule sums over the row cube to the + /// claim the GKR handed back. + #[test] + fn the_input_claim_is_what_the_rules_sum_to() { + for num_vars in 1..=4usize { + let factors = columns(num_vars); + let interactions = balanced(FE::from(97), FE::from(31), num_vars); + let tree = FractionTree::build(input_layer(&interactions, &factors).unwrap()).unwrap(); + let out = gkr::prove(&tree, &mut transcript()).unwrap(); + + let statements = + claim_statements(&interactions, &out.claim.point, num_vars, WEIGHT).unwrap(); + let mut with_weight = factors.clone(); + with_weight.push(eq_mle(&statements.row_point).unwrap()); + + let sum = |rule: &Rule<'_, F>| { + (0..(1usize << num_vars)).fold(FE::zero(), |acc, i| { + let values: Vec = with_weight.iter().map(|f| f.evals()[i]).collect(); + acc + rule.apply(&values) + }) + }; + assert_eq!(sum(&statements.numerator), out.claim.p, "n={num_vars}"); + assert_eq!(sum(&statements.denominator), out.claim.q, "n={num_vars}"); + } + } + + /// Same identity with the padding in play, where a wrong padding weight + /// would show up in the denominator. + #[test] + fn the_claim_holds_with_padded_interactions() { + let num_vars = 3; + let factors = columns(num_vars); + let z = FE::from(97); + let alpha = FE::from(31); + let mut interactions = balanced(z, alpha, num_vars); + interactions.push(Interaction::new( + Affine::constant(FE::zero()), + Affine::constant(z), + )); + + let tree = FractionTree::build(input_layer(&interactions, &factors).unwrap()).unwrap(); + let out = gkr::prove(&tree, &mut transcript()).unwrap(); + let statements = + claim_statements(&interactions, &out.claim.point, num_vars, WEIGHT).unwrap(); + + let mut with_weight = factors; + with_weight.push(eq_mle(&statements.row_point).unwrap()); + let sum = |rule: &Rule<'_, F>| { + (0..(1usize << num_vars)).fold(FE::zero(), |acc, i| { + let values: Vec = with_weight.iter().map(|f| f.evals()[i]).collect(); + acc + rule.apply(&values) + }) + }; + assert_eq!(sum(&statements.numerator), out.claim.p); + assert_eq!(sum(&statements.denominator), out.claim.q); + } + + /// Two rules, one batched sumcheck, whatever the interaction count. + #[test] + fn the_bus_costs_two_rules_in_one_sumcheck() { + let num_vars = 3; + let factors = columns(num_vars); + let interactions = balanced(FE::from(97), FE::from(31), num_vars); + let tree = FractionTree::build(input_layer(&interactions, &factors).unwrap()).unwrap(); + let output = tree.output(); + + let mut prover = transcript(); + let out = gkr::prove(&tree, &mut prover).unwrap(); + let statements = + claim_statements(&interactions, &out.claim.point, num_vars, WEIGHT).unwrap(); + let mut with_weight = factors.clone(); + with_weight.push(eq_mle(&statements.row_point).unwrap()); + let claims = [out.claim.p, out.claim.q]; + + let (proof, point) = batch::prove( + with_weight.clone(), + vec![statements.numerator, statements.denominator], + &claims, + &mut prover, + ) + .unwrap(); + assert_eq!(proof.rounds.len(), num_vars); + + let mut verifier = transcript(); + let claim = gkr::verify(&out.proof, output, &mut verifier).unwrap(); + let statements = claim_statements(&interactions, &claim.point, num_vars, WEIGHT).unwrap(); + let row_point = statements.row_point.clone(); + + let checked = batch::verify( + &proof, + &[statements.numerator, statements.denominator], + &claims, + |at: &[FE]| { + // Only the trace factors travel; the weight is recomputed. + let mut values: Vec = factors + .iter() + .map(|f| f.evaluate(at)) + .collect::>()?; + values.push(crate::eq::eq_eval(&row_point, at)?); + Ok(values) + }, + num_vars, + &mut verifier, + ) + .unwrap(); + assert_eq!(checked, point); + } + + #[test] + fn a_claim_point_of_the_wrong_arity_is_rejected() { + let interactions = balanced(FE::from(97), FE::from(31), 3); + let result = claim_statements(&interactions, &[FE::one(); 3], 3, WEIGHT); + assert_eq!( + result.err(), + Some(Error::VariableCountMismatch { + expected: 4, + got: 3 + }) + ); + } + + #[test] + fn an_empty_bus_is_rejected() { + assert_eq!( + input_layer::(&[], &columns(3)).unwrap_err(), + Error::EmptyPolynomial + ); + } +} diff --git a/crypto/multilinear/src/mle.rs b/crypto/multilinear/src/mle.rs new file mode 100644 index 000000000..f6b05f65d --- /dev/null +++ b/crypto/multilinear/src/mle.rs @@ -0,0 +1,436 @@ +//! Multilinear extensions, held as their `2^n` hypercube evaluations. +//! +//! Index `i` is read with **variable 0 as the most significant bit**. Every +//! fold in this crate assumes that. + +use math::field::{ + element::FieldElement, + traits::{IsField, IsSubFieldOf}, +}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::Error; + +/// A multilinear polynomial held by its hypercube evaluations. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Mle { + evals: Vec>, + num_vars: usize, +} + +impl Mle { + /// Builds an MLE from `2^n` evaluations in hypercube order. + pub fn new(evals: Vec>) -> Result { + let len = evals.len(); + if !len.is_power_of_two() { + return Err(Error::NotPowerOfTwo(len)); + } + Ok(Self { + num_vars: len.trailing_zeros() as usize, + evals, + }) + } + + /// The constant polynomial on zero variables. + pub fn constant(value: FieldElement) -> Self { + Self { + evals: vec![value], + num_vars: 0, + } + } + + pub fn num_vars(&self) -> usize { + self.num_vars + } + + pub fn len(&self) -> usize { + self.evals.len() + } + + pub fn is_empty(&self) -> bool { + self.evals.is_empty() + } + + pub fn evals(&self) -> &[FieldElement] { + &self.evals + } + + pub fn into_evals(self) -> Vec> { + self.evals + } + + /// Fixes variable 0 to `r`, returning a polynomial on `n - 1` variables. + /// + /// `new[j] = (1 - r)·old[j] + r·old[j + 2^(n-1)]`, which is the multilinear + /// interpolation between the two halves of the table. + pub fn fix_first_variable(&self, r: &FieldElement) -> Result { + if self.num_vars == 0 { + return Err(Error::NoVariablesLeft); + } + let half = self.evals.len() / 2; + let evals = (0..half) + .map(|j| { + let lo = &self.evals[j]; + let hi = &self.evals[j + half]; + // lo + r·(hi - lo) — one multiplication instead of two. + lo + r * &(hi - lo) + }) + .collect(); + Ok(Self { + evals, + num_vars: self.num_vars - 1, + }) + } + + /// Fixes variable 0 in place. Same arithmetic as [`Self::fix_first_variable`], + /// but reuses the allocation — sumcheck folds once per round. + pub fn fix_first_variable_in_place(&mut self, r: &FieldElement) -> Result<(), Error> { + if self.num_vars == 0 { + return Err(Error::NoVariablesLeft); + } + let half = self.evals.len() / 2; + let (lo, hi) = self.evals.split_at_mut(half); + // Every index is independent, and the halves are disjoint slices, so the + // split is what lets the rows go out to the pool at all. + let fold = |(a, b): (&mut FieldElement, &FieldElement)| *a = &*a + r * &(b - &*a); + #[cfg(feature = "parallel")] + if half >= crate::SERIAL_BELOW { + lo.par_iter_mut().zip(hi.par_iter()).for_each(fold); + } else { + lo.iter_mut().zip(hi.iter()).for_each(fold); + } + #[cfg(not(feature = "parallel"))] + lo.iter_mut().zip(hi.iter()).for_each(fold); + self.evals.truncate(half); + self.num_vars -= 1; + Ok(()) + } + + /// Fixes the **last** variable to `r`, returning a polynomial on `n - 1` + /// variables. + /// + /// The last variable is the low bit of the index, so this pairs `2j` with + /// `2j + 1`. Codeword folding binds variables from this end, which is why + /// it exists alongside [`Self::fix_first_variable_in_place`]. + pub fn fix_last_variable_in_place(&mut self, r: &FieldElement) -> Result<(), Error> { + if self.num_vars == 0 { + return Err(Error::NoVariablesLeft); + } + let half = self.evals.len() / 2; + for j in 0..half { + let lo = self.evals[2 * j].clone(); + let delta = &self.evals[2 * j + 1] - &lo; + self.evals[j] = lo + r * δ + } + self.evals.truncate(half); + self.num_vars -= 1; + Ok(()) + } + + /// Evaluates the extension at an arbitrary point in `F^n`. + pub fn evaluate(&self, point: &[FieldElement]) -> Result, Error> { + if point.len() != self.num_vars { + return Err(Error::VariableCountMismatch { + expected: self.num_vars, + got: point.len(), + }); + } + Self::evaluate_at(&self.evals, point) + } + + /// The extension of `evals` at `point`, without owning an [`Mle`]. + /// + /// The first fold reads the slice and writes the half-size buffer the rest + /// fold in place, so the table is never copied at full width. A caller + /// holding a slice of a bigger table — a GKR layer's half, say — evaluates + /// it without materializing it at all. + pub fn evaluate_at( + evals: &[FieldElement], + point: &[FieldElement], + ) -> Result, Error> { + if evals.len() != 1usize << point.len() { + return Err(Error::VariableCountMismatch { + expected: point.len(), + got: evals.len().trailing_zeros() as usize, + }); + } + if let Some(value) = crate::gpu::evaluate_mle(evals, point) { + return Ok(value); + } + let Some((first, rest)) = point.split_first() else { + return Ok(evals[0].clone()); + }; + + let half = evals.len() / 2; + let (lo, hi) = evals.split_at(half); + let combine = |(l, h): (&FieldElement, &FieldElement)| l + first * &(h - l); + #[cfg(feature = "parallel")] + let mut current: Vec> = if half >= crate::SERIAL_BELOW { + lo.par_iter().zip(hi.par_iter()).map(combine).collect() + } else { + lo.iter().zip(hi.iter()).map(combine).collect() + }; + #[cfg(not(feature = "parallel"))] + let mut current: Vec> = lo.iter().zip(hi.iter()).map(combine).collect(); + + for r in rest { + let half = current.len() / 2; + let (lo, hi) = current.split_at_mut(half); + let fold = |(a, b): (&mut FieldElement, &FieldElement)| *a = &*a + r * &(b - &*a); + #[cfg(feature = "parallel")] + if half >= crate::SERIAL_BELOW { + lo.par_iter_mut().zip(hi.par_iter()).for_each(fold); + } else { + lo.iter_mut().zip(hi.iter()).for_each(fold); + } + #[cfg(not(feature = "parallel"))] + lo.iter_mut().zip(hi.iter()).for_each(fold); + current.truncate(half); + } + Ok(current.swap_remove(0)) + } + + /// The extension at a point in a **larger** field. + /// + /// A trace column lives in the base field while the challenges do not, so + /// this is how a committed column answers a claim: the first fold lifts, + /// the rest stay up. Lifting the whole table first would instead cost its + /// size times the extension degree. + pub fn evaluate_in(&self, point: &[FieldElement]) -> Result, Error> + where + F: IsSubFieldOf, + E: IsField + 'static, + { + if point.len() != self.num_vars { + return Err(Error::VariableCountMismatch { + expected: self.num_vars, + got: point.len(), + }); + } + if let Some(value) = crate::gpu::evaluate_mle(&self.evals, point) { + return Ok(value); + } + self.evaluate_in_on_host(point) + } + + /// [`evaluate_in`](Self::evaluate_in) with the device never asked: the + /// oracle a device value is checked against, whatever the table's size. + pub(crate) fn evaluate_in_on_host( + &self, + point: &[FieldElement], + ) -> Result, Error> + where + F: IsSubFieldOf, + E: IsField + 'static, + { + if point.len() != self.num_vars { + return Err(Error::VariableCountMismatch { + expected: self.num_vars, + got: point.len(), + }); + } + let Some((first, rest)) = point.split_first() else { + return Ok(self.evals[0].clone().to_extension::()); + }; + + let half = self.evals.len() / 2; + let mut current: Vec> = (0..half) + .map(|j| { + let lo = &self.evals[j]; + let hi = &self.evals[j + half]; + // The base element on the left: the only direction the tower + // gives. + lo.clone().to_extension::() + (hi - lo) * first + }) + .collect(); + + for r in rest { + let half = current.len() / 2; + for j in 0..half { + let delta = ¤t[j + half] - ¤t[j]; + current[j] = ¤t[j] + r * δ + } + current.truncate(half); + } + Ok(current.into_iter().next().expect("one value remains")) + } + + /// The single remaining evaluation, once every variable has been fixed. + pub fn as_constant(&self) -> Option<&FieldElement> { + (self.num_vars == 0).then(|| &self.evals[0]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + #[test] + fn rejects_non_power_of_two() { + let evals: Vec = (0..3).map(FE::from).collect(); + assert_eq!(Mle::new(evals).unwrap_err(), Error::NotPowerOfTwo(3)); + } + + #[test] + fn num_vars_is_log2_of_the_table() { + assert_eq!(mle(&[1]).num_vars(), 0); + assert_eq!(mle(&[1, 2]).num_vars(), 1); + assert_eq!(mle(&[1, 2, 3, 4]).num_vars(), 2); + assert_eq!(mle(&[0; 256]).num_vars(), 8); + } + + #[test] + fn agrees_with_the_table_on_hypercube_corners() { + // f(x0, x1) with x0 the high bit: [f(00), f(01), f(10), f(11)] + let f = mle(&[7, 11, 13, 17]); + for (i, expected) in [7u64, 11, 13, 17].iter().enumerate() { + let x0 = FE::from(((i >> 1) & 1) as u64); + let x1 = FE::from((i & 1) as u64); + assert_eq!(f.evaluate(&[x0, x1]).unwrap(), FE::from(*expected)); + } + } + + #[test] + fn is_multilinear_in_each_variable() { + // A multilinear polynomial is affine along every axis, so the midpoint + // evaluation is the average of the two endpoints. + let f = mle(&[7, 11, 13, 17]); + let two_inv = FE::from(2).inv().unwrap(); + let x1 = FE::from(5); + + let at_0 = f.evaluate(&[FE::zero(), x1]).unwrap(); + let at_1 = f.evaluate(&[FE::one(), x1]).unwrap(); + let at_mid = f.evaluate(&[two_inv, x1]).unwrap(); + + assert_eq!(at_mid, (at_0 + at_1) * two_inv); + } + + #[test] + fn fixing_a_variable_matches_evaluating_it() { + let f = mle(&[3, 5, 8, 13, 21, 34, 55, 89]); + let r = FE::from(42); + let folded = f.fix_first_variable(&r).unwrap(); + + assert_eq!(folded.num_vars(), 2); + for a in 0..2u64 { + for b in 0..2u64 { + let rest = [FE::from(a), FE::from(b)]; + let via_fold = folded.evaluate(&rest).unwrap(); + let direct = f.evaluate(&[r, FE::from(a), FE::from(b)]).unwrap(); + assert_eq!(via_fold, direct); + } + } + } + + #[test] + fn in_place_fold_matches_the_allocating_one() { + let f = mle(&[3, 5, 8, 13, 21, 34, 55, 89]); + let r = FE::from(9); + let expected = f.fix_first_variable(&r).unwrap(); + + let mut g = f; + g.fix_first_variable_in_place(&r).unwrap(); + assert_eq!(g, expected); + } + + #[test] + fn folding_every_variable_leaves_the_evaluation() { + let f = mle(&[3, 5, 8, 13]); + let point = [FE::from(6), FE::from(7)]; + let expected = f.evaluate(&point).unwrap(); + + let mut g = f; + for r in &point { + g.fix_first_variable_in_place(r).unwrap(); + } + assert_eq!(g.num_vars(), 0); + assert_eq!(g.as_constant().unwrap(), &expected); + } + + #[test] + fn folding_a_constant_is_an_error() { + let mut f = Mle::::constant(FE::from(4)); + assert_eq!( + f.fix_first_variable_in_place(&FE::from(1)).unwrap_err(), + Error::NoVariablesLeft + ); + } + + #[test] + fn evaluate_rejects_a_point_of_the_wrong_arity() { + let f = mle(&[1, 2, 3, 4]); + assert_eq!( + f.evaluate(&[FE::from(1)]).unwrap_err(), + Error::VariableCountMismatch { + expected: 2, + got: 1 + } + ); + } + + #[test] + fn fixing_the_last_variable_matches_evaluating_it() { + let f = mle(&[3, 5, 8, 13, 21, 34, 55, 89]); + let r = FE::from(23); + let mut folded = f.clone(); + folded.fix_last_variable_in_place(&r).unwrap(); + + assert_eq!(folded.num_vars(), 2); + for a in 0..2u64 { + for b in 0..2u64 { + let via_fold = folded.evaluate(&[FE::from(a), FE::from(b)]).unwrap(); + let direct = f.evaluate(&[FE::from(a), FE::from(b), r]).unwrap(); + assert_eq!(via_fold, direct, "a={a}, b={b}"); + } + } + } + + #[test] + fn evaluating_in_a_larger_field_matches_lifting_first() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + type ExtE = FieldElement; + + for num_vars in 0..=4usize { + let f = mle(&(0..(1u64 << num_vars)) + .map(|i| i.wrapping_mul(6364136223846793005) >> 13) + .collect::>()); + let point: Vec = (0..num_vars).map(|i| ExtE::from(101 + i as u64)).collect(); + + let lifted = + Mle::new(f.evals().iter().map(|v| v.to_extension::()).collect()).unwrap(); + + assert_eq!( + f.evaluate_in(&point).unwrap(), + lifted.evaluate(&point).unwrap(), + "num_vars={num_vars}" + ); + } + } + + #[test] + fn evaluating_in_the_same_field_is_evaluating() { + let f = mle(&[3, 5, 8, 13]); + let point = [FE::from(6), FE::from(7)]; + assert_eq!(f.evaluate_in(&point).unwrap(), f.evaluate(&point).unwrap()); + } + + #[test] + fn first_and_last_folds_bind_different_ends() { + let f = mle(&[1, 2, 3, 4]); + let r = FE::from(5); + let mut by_first = f.clone(); + by_first.fix_first_variable_in_place(&r).unwrap(); + let mut by_last = f; + by_last.fix_last_variable_in_place(&r).unwrap(); + assert_ne!(by_first, by_last); + } +} diff --git a/crypto/multilinear/src/narrow.rs b/crypto/multilinear/src/narrow.rs new file mode 100644 index 000000000..96bacabb4 --- /dev/null +++ b/crypto/multilinear/src/narrow.rs @@ -0,0 +1,512 @@ +//! Narrow storage of a table's committed columns: each column at the bytes its +//! words need. +//! +//! A trace's 64-bit words are mostly small (flags, bytes, 16-bit limbs, +//! timestamps under 2^32), so a block's committed cells need about two bytes +//! each instead of eight. [`NarrowColumns`] keeps the raw words of a table's +//! columns, each little-endian at the width of its largest word (1, 2, 4 or 8 +//! bytes) — the stark crate's `NarrowMain` layout, which `math_cuda::narrow` +//! packs and widens on the card — so widening gives back the same words bit +//! for bit and no proof byte can depend on it. +//! +//! The readers of a table's columns see them through [`HostColumns`] (a table) +//! and [`HostColumn`] (one column): the shape always, the field elements on +//! the host only when a reader there asks, which is what lets the columns stay +//! narrow on the host while the card reads them wide. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use math::field::{element::FieldElement, goldilocks::GoldilocksField, traits::IsField}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::mle::Mle; + +/// Narrow tables widened on the host, and their cells, since the process +/// started: a device path widens on the card, so on a block these count only +/// the tables a host path read ([`host_widens`]). +static HOST_WIDENS: AtomicU64 = AtomicU64::new(0); +static HOST_WIDEN_CELLS: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn note_host_widen(cells: usize) { + HOST_WIDENS.fetch_add(1, Ordering::Relaxed); + HOST_WIDEN_CELLS.fetch_add(cells as u64, Ordering::Relaxed); +} + +/// `(tables, cells)` of narrow columns widened on the host so far. +pub fn host_widens() -> (u64, u64) { + ( + HOST_WIDENS.load(Ordering::Relaxed), + HOST_WIDEN_CELLS.load(Ordering::Relaxed), + ) +} + +/// A table's columns at 1, 2, 4 or 8 bytes per cell (see the module docs). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NarrowColumns { + rows: usize, + /// Bytes per cell of each column. + widths: Vec, + /// Where each column starts in `data`. + offsets: Vec, + /// Column `c` is `rows × widths[c]` little-endian bytes from `offsets[c]`. + data: Vec, +} + +/// The bytes a word needs: 1, 2, 4 or 8. +fn width_of(max: u64) -> u8 { + match max { + 0..=0xff => 1, + 0x100..=0xffff => 2, + 0x1_0000..=0xffff_ffff => 4, + _ => 8, + } +} + +impl NarrowColumns { + /// Columns packed from their parts (the card's pack): `rows` rows, the + /// column widths, and the columns back to back. `None` when a width is not + /// 1, 2, 4 or 8 or the bytes are not `rows × Σ widths`. + pub fn from_parts(rows: usize, widths: Vec, data: Vec) -> Option { + if widths.iter().any(|w| ![1, 2, 4, 8].contains(w)) { + return None; + } + let mut offsets = Vec::with_capacity(widths.len()); + let mut total = 0usize; + for &w in &widths { + offsets.push(u64::try_from(total).ok()?); + total = total.checked_add(rows.checked_mul(w as usize)?)?; + } + (total == data.len()).then_some(Self { + rows, + widths, + offsets, + data, + }) + } + + /// Pack columns of raw words, every one `rows` long. `None` when they are + /// not all one height. + pub fn pack(columns: &[&[u64]]) -> Option { + let rows = columns.first().map_or(0, |c| c.len()); + if columns.iter().any(|c| c.len() != rows) { + return None; + } + let widths: Vec = columns + .iter() + .map(|c| width_of(c.iter().copied().max().unwrap_or(0))) + .collect(); + let mut data = Vec::with_capacity(rows * widths.iter().map(|&w| w as usize).sum::()); + for (column, &w) in columns.iter().zip(&widths) { + for word in column.iter() { + data.extend_from_slice(&word.to_le_bytes()[..w as usize]); + } + } + Self::from_parts(rows, widths, data) + } + + /// [`Self::pack`] of a row-major trace: the `cols`-wide rows of words + /// `row_major`, packed column by column (the stark crate's + /// `NarrowMain::pack`, a block of rows at a time so the block stays in + /// cache across the columns). On the caller's thread. `None` when `cols` + /// is zero or does not divide the words. + pub fn pack_row_major(row_major: &[u64], cols: usize) -> Option { + const BLOCK_ROWS: usize = 1 << 10; + if cols == 0 || !row_major.len().is_multiple_of(cols) { + return None; + } + let rows = row_major.len() / cols; + let mut max = vec![0u64; cols]; + for row in row_major.chunks_exact(cols) { + for (m, &v) in max.iter_mut().zip(row) { + *m = (*m).max(v); + } + } + let widths: Vec = max.into_iter().map(width_of).collect(); + let total: usize = widths.iter().map(|&w| rows * w as usize).sum(); + let mut data = vec![0u8; total]; + let mut columns: Vec<&mut [u8]> = Vec::with_capacity(cols); + let mut rest = data.as_mut_slice(); + for &w in &widths { + let (column, tail) = rest.split_at_mut(rows * w as usize); + columns.push(column); + rest = tail; + } + for first in (0..rows).step_by(BLOCK_ROWS) { + let last = (first + BLOCK_ROWS).min(rows); + let block = &row_major[first * cols..last * cols]; + for (c, (column, &w)) in columns.iter_mut().zip(&widths).enumerate() { + let words = block.iter().skip(c).step_by(cols); + let w = w as usize; + let out = &mut column[first * w..last * w]; + match w { + 1 => out.iter_mut().zip(words).for_each(|(o, &v)| *o = v as u8), + 2 => out + .chunks_exact_mut(2) + .zip(words) + .for_each(|(o, &v)| o.copy_from_slice(&(v as u16).to_le_bytes())), + 4 => out + .chunks_exact_mut(4) + .zip(words) + .for_each(|(o, &v)| o.copy_from_slice(&(v as u32).to_le_bytes())), + _ => out + .chunks_exact_mut(8) + .zip(words) + .for_each(|(o, &v)| o.copy_from_slice(&v.to_le_bytes())), + } + } + } + Self::from_parts(rows, widths, data) + } + + /// [`Self::pack`] of field columns. `None` unless `F` is Goldilocks, whose + /// elements are their raw words, or the columns are not one height. + pub fn pack_columns(columns: &[Mle]) -> Option { + let raw: Vec<&[u64]> = columns.iter().map(raw_words).collect::>()?; + Self::pack(&raw) + } + + /// Rows of each column. + pub fn rows(&self) -> usize { + self.rows + } + + /// How many columns. + pub fn cols(&self) -> usize { + self.widths.len() + } + + /// Bytes per cell of each column (1, 2, 4 or 8). + pub fn widths(&self) -> &[u8] { + &self.widths + } + + /// Where each column starts in [`Self::data`]. + pub fn offsets(&self) -> &[u64] { + &self.offsets + } + + /// The packed columns, back to back. + pub fn data(&self) -> &[u8] { + &self.data + } + + /// The parts, for a store to move elsewhere and back + /// ([`Self::from_parts`]): rows, widths and the bytes. The offsets follow + /// from the widths. + pub fn into_parts(self) -> (usize, Vec, Vec) { + (self.rows, self.widths, self.data) + } + + /// The word at `row` of column `c`. + pub fn get(&self, row: usize, c: usize) -> u64 { + let w = self.widths[c] as usize; + let at = self.offsets[c] as usize + row * w; + let mut word = [0u8; 8]; + word[..w].copy_from_slice(&self.data[at..at + w]); + u64::from_le_bytes(word) + } + + /// Writes `value` at `row` of column `c` when it fits the column's width; + /// `false` (nothing written) when it does not. + pub fn set(&mut self, row: usize, c: usize, value: u64) -> bool { + let w = self.widths[c] as usize; + if width_of(value) as usize > w { + return false; + } + let at = self.offsets[c] as usize + row * w; + self.data[at..at + w].copy_from_slice(&value.to_le_bytes()[..w]); + true + } + + /// Column `c`'s words. + pub fn column(&self, c: usize) -> Vec { + let w = self.widths[c] as usize; + let at = self.offsets[c] as usize; + self.data[at..at + self.rows * w] + .chunks_exact(w) + .map(|cell| { + let mut word = [0u8; 8]; + word[..w].copy_from_slice(cell); + u64::from_le_bytes(word) + }) + .collect() + } + + /// The columns as field elements, their words bit for bit. `None` unless + /// `F` is Goldilocks. + pub fn widen(&self) -> Option>> { + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + let one = |c: usize| -> Option> { + let mut words = core::mem::ManuallyDrop::new(self.column(c)); + // SAFETY: `F == GoldilocksField`, whose elements are transparent + // wrappers over one `u64`: the allocation describes the same words + // either way. + let evals = unsafe { + Vec::from_raw_parts( + words.as_mut_ptr() as *mut FieldElement, + words.len(), + words.capacity(), + ) + }; + Mle::new(evals).ok() + }; + #[cfg(feature = "parallel")] + return (0..self.cols()).into_par_iter().map(one).collect(); + #[cfg(not(feature = "parallel"))] + return (0..self.cols()).map(one).collect(); + } + + /// Narrow the first column wider than a byte by one width step in the map + /// (its offsets unchanged): a wrong width map, which widens to other words, + /// for the tests that check the prover refuses it. Returns whether a column + /// was there to narrow. + #[doc(hidden)] + pub fn fault_width_map(&mut self) -> bool { + match self.widths.iter_mut().find(|w| **w > 1) { + Some(w) => { + *w /= 2; + true + } + None => false, + } + } +} + +/// A column's raw words, when `F` is Goldilocks. +fn raw_words(column: &Mle) -> Option<&[u64]> { + if std::any::TypeId::of::() != std::any::TypeId::of::() { + return None; + } + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + Some(unsafe { + core::slice::from_raw_parts(column.evals().as_ptr() as *const u64, column.len()) + }) +} + +/// A table's base columns as a reader finds them: their shape always, their +/// field elements on the host on demand — widened then, if they are held +/// narrow ([`NarrowColumns`]). A device path reads the shape and the card's +/// copy; only a host path asks for [`host`](Self::host). +pub trait HostColumns: Sync { + /// How many columns. + fn width(&self) -> usize; + /// Rows of each column; `None` when there are none or they differ. + fn rows(&self) -> Option; + /// The columns on the host. + fn host(&self) -> &[Mle]; +} + +/// The height columns share, if they do. +fn common_rows(columns: &[Mle]) -> Option { + let rows = columns.first()?.len(); + columns.iter().all(|c| c.len() == rows).then_some(rows) +} + +impl HostColumns for [Mle] +where + FieldElement: Sync, +{ + fn width(&self) -> usize { + self.len() + } + + fn rows(&self) -> Option { + common_rows(self) + } + + fn host(&self) -> &[Mle] { + self + } +} + +impl HostColumns for Vec> +where + FieldElement: Sync, +{ + fn width(&self) -> usize { + self.len() + } + + fn rows(&self) -> Option { + common_rows(self) + } + + fn host(&self) -> &[Mle] { + self + } +} + +/// One column the same way: its height always, its field elements on demand. +/// What a stacked polynomial is made of (`whir_chain::Stacked`). +pub trait HostColumn: Sync { + /// Rows of the column. + fn rows(&self) -> usize; + /// The column on the host. + fn host(&self) -> &Mle; +} + +impl HostColumn for Mle +where + FieldElement: Sync, +{ + fn rows(&self) -> usize { + self.len() + } + + fn host(&self) -> &Mle { + self + } +} + +/// Column `index` of a table's [`HostColumns`]: widened with its table, if the +/// table is held narrow. +pub struct ColumnOf<'a, C: ?Sized> { + pub columns: &'a C, + pub index: usize, +} + +impl + ?Sized> HostColumn for ColumnOf<'_, C> { + fn rows(&self) -> usize { + self.columns.rows().unwrap_or(0) + } + + fn host(&self) -> &Mle { + &self.columns.host()[self.index] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as Gl; + + /// Every width boundary round trips through the host pack and widen, in + /// every position: 0, 2^8 − 1, 2^8, 2^16 − 1, 2^16, 2^32 − 1, 2^32, the + /// Goldilocks p − 1, a non-canonical word above p, and u64::MAX. + #[test] + fn narrow_columns_round_trip_at_every_width_boundary() { + let edges = [ + 0u64, + 0xff, + 0x100, + 0xffff, + 0x1_0000, + 0xffff_ffff, + 0x1_0000_0000, + 0xffff_ffff_0000_0000, + 0xffff_ffff_0000_0001, + u64::MAX, + ]; + let rows = 1usize << 9; + let columns: Vec> = edges + .iter() + .map(|&edge| { + (0..rows) + .map(|r| { + if r == rows - 1 { + edge + } else { + (r as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) % (edge / 2 + 1) + } + }) + .collect() + }) + .collect(); + let raw: Vec<&[u64]> = columns.iter().map(Vec::as_slice).collect(); + let narrow = NarrowColumns::pack(&raw).expect("one height"); + assert_eq!(narrow.widths(), &[1, 1, 2, 2, 4, 4, 8, 8, 8, 8]); + assert_eq!( + narrow.data().len(), + rows * narrow.widths().iter().map(|&w| w as usize).sum::() + ); + for (c, column) in columns.iter().enumerate() { + assert_eq!(&narrow.column(c), column, "column {c}"); + } + let wide = narrow.widen::().expect("Goldilocks"); + for (mle, column) in wide.iter().zip(&columns) { + let words: Vec = mle.evals().iter().map(|v| *v.value()).collect(); + assert_eq!(&words, column); + } + let again = + NarrowColumns::from_parts(rows, narrow.widths().to_vec(), narrow.data().to_vec()); + assert_eq!(again.as_ref(), Some(&narrow)); + } + + /// A width that is not 1, 2, 4 or 8, or bytes that are not the shape's, + /// make no packed columns. + #[test] + fn packed_parts_of_the_wrong_shape_are_refused() { + assert!(NarrowColumns::from_parts(4, vec![1, 3], vec![0; 16]).is_none()); + assert!(NarrowColumns::from_parts(4, vec![1, 2], vec![0; 11]).is_none()); + assert!(NarrowColumns::from_parts(4, vec![1, 2], vec![0; 12]).is_some()); + assert!(NarrowColumns::pack(&[&[1, 2], &[3]]).is_none()); + } + + /// A wrong width map widens to other words. + #[test] + fn a_wrong_width_map_widens_to_other_words() { + let column: Vec = (0..64u64).map(|r| r * 100_000).collect(); + let mut narrow = NarrowColumns::pack(&[&column]).expect("one height"); + assert_eq!(narrow.widths(), &[4]); + assert!(narrow.fault_width_map()); + assert_ne!(narrow.column(0), column); + } + + /// A row-major trace packs to what its columns pack to, at every width + /// boundary and across the pack's row blocks; a width that does not divide + /// the words packs nothing. + #[test] + fn a_row_major_trace_packs_as_its_columns() { + let edges = [ + 0u64, + 0xff, + 0x100, + 0xffff, + 0x1_0000, + 0xffff_ffff, + 0x1_0000_0000, + u64::MAX, + ]; + for rows in [1usize, 7, 1024, 1025, 3000] { + let columns: Vec> = edges + .iter() + .map(|&edge| { + (0..rows) + .map(|r| { + if r == rows / 2 { + edge + } else { + (r as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) % (edge / 2 + 1) + } + }) + .collect() + }) + .collect(); + let row_major: Vec = (0..rows) + .flat_map(|r| columns.iter().map(move |c| c[r])) + .collect(); + let raw: Vec<&[u64]> = columns.iter().map(Vec::as_slice).collect(); + assert_eq!( + NarrowColumns::pack_row_major(&row_major, edges.len()), + NarrowColumns::pack(&raw), + "{rows} rows" + ); + } + assert!(NarrowColumns::pack_row_major(&[1, 2, 3], 2).is_none()); + assert!(NarrowColumns::pack_row_major(&[1, 2], 0).is_none()); + } + + /// Only Goldilocks columns widen to field elements: their elements are + /// their raw words. + #[test] + fn only_goldilocks_columns_widen() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + let narrow = NarrowColumns::pack(&[&[1, 2, 3, 4]]).expect("one height"); + assert!(narrow.widen::().is_none()); + assert!(narrow.widen::().is_some()); + } +} diff --git a/crypto/multilinear/src/poly.rs b/crypto/multilinear/src/poly.rs new file mode 100644 index 000000000..f3117fb87 --- /dev/null +++ b/crypto/multilinear/src/poly.rs @@ -0,0 +1,424 @@ +//! What sumcheck needs from a polynomial: its multilinear factors plus a rule +//! for combining their values. Keeps a constraint DAG out of expanded form, +//! which would be exponential in the nesting depth. + +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{Error, mle::Mle, program::Program}; + +/// A polynomial over the hypercube, presented as multilinear factors plus a +/// rule for combining their values. +pub trait SumcheckPolynomial { + /// Variables left to bind. + fn num_vars(&self) -> usize; + + /// Total degree, which bounds each round polynomial's degree. + /// + /// Must be an upper bound: the verifier checks the round polynomial has + /// exactly `degree + 1` evaluations, so understating it rejects honest + /// proofs and overstating it only costs proof size. + fn degree(&self) -> usize; + + /// The multilinear factors, in the order [`combine`](Self::combine) indexes. + fn polys(&self) -> &[Mle]; + + /// The polynomial's value, given each factor's value at the same point. + fn combine(&self, values: &[FieldElement]) -> FieldElement; + + /// The same, with a scratch buffer the caller owns. The sumcheck calls this + /// once per cube index per interpolation node, so an implementation backed + /// by a program has nowhere to put its steps that is not the caller's. + fn combine_in( + &self, + values: &[FieldElement], + scratch: &mut Vec>, + ) -> FieldElement { + let _ = scratch; + self.combine(values) + } + + /// Binds variable 0 to `r` in every factor. + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), Error>; + + /// The program this polynomial is, when it can say: the description a + /// device runs in place of [`combine`](Self::combine). + /// + /// An implementation that offers one must also accept its factors back + /// through [`accept_folded`](Self::accept_folded) — the device binds them + /// there, and the sumcheck's contract is that the polynomial comes back + /// folded. + fn program(&self) -> Option<&Program> { + None + } + + /// The cube below which this polynomial's rounds stop going to a device. + /// + /// The default is for a rule the host runs through the program interpreter; + /// one that evaluates its rule directly crosses much later. + fn host_cube(&self) -> usize { + crate::HOST_CUBE_COMPILED + } + + /// Takes factors bound elsewhere, in the order [`polys`](Self::polys) + /// returns them. + fn accept_folded(&mut self, polys: Vec>) -> Result<(), Error> { + let _ = polys; + Err(Error::DeviceFailed { + stage: "write-back", + }) + } + + /// Value at hypercube index `i`. + fn eval_at_index(&self, i: usize) -> FieldElement { + let values: Vec> = + self.polys().iter().map(|p| p.evals()[i].clone()).collect(); + self.combine(&values) + } + + /// Sum over the whole hypercube. Reference implementation — the point of + /// sumcheck is to avoid paying this. + fn sum_over_hypercube(&self) -> FieldElement { + (0..(1usize << self.num_vars())) + .fold(FieldElement::zero(), |acc, i| acc + self.eval_at_index(i)) + } + + /// Value at an arbitrary point, extending every factor multilinearly. + fn evaluate(&self, point: &[FieldElement]) -> Result, Error> { + let values: Vec> = self + .polys() + .iter() + .map(|p| p.evaluate(point)) + .collect::>()?; + Ok(self.combine(&values)) + } +} + +/// Factors plus a closure that combines them. The closure holds no trace data, +/// so the verifier can carry the same one. +pub struct Composed { + polys: Vec>, + combine: C, + degree: usize, + num_vars: usize, + /// The same rule as straight-line code, when the caller can say what it is. + program: Option>, +} + +impl Composed +where + C: Fn(&[FieldElement]) -> FieldElement + 'static, +{ + /// `degree` must upper-bound the closure's total degree in the factors. + pub fn new(polys: Vec>, combine: C, degree: usize) -> Result { + let num_vars = polys.first().map(|p| p.num_vars()).unwrap_or(0); + for p in &polys { + if p.num_vars() != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got: p.num_vars(), + }); + } + } + Ok(Self { + polys, + combine, + degree, + num_vars, + program: None, + }) + } + + /// The same, saying which program the closure is. The two must agree: the + /// device runs the program and the host may run either. + pub fn with_program(mut self, program: Program) -> Self { + self.program = Some(program); + self + } +} + +impl Composed { + /// The factors, dropping the rule. + /// + /// A chained WHIR needs them back after each group of rounds: it rebuilds + /// its weight from the folded one, so it cannot keep the polynomial. + pub fn into_polys(self) -> Vec> { + self.polys + } +} + +impl SumcheckPolynomial for Composed +where + C: Fn(&[FieldElement]) -> FieldElement, +{ + fn num_vars(&self) -> usize { + self.num_vars + } + + fn degree(&self) -> usize { + self.degree + } + + fn polys(&self) -> &[Mle] { + &self.polys + } + + fn combine(&self, values: &[FieldElement]) -> FieldElement { + (self.combine)(values) + } + + fn combine_in( + &self, + values: &[FieldElement], + scratch: &mut Vec>, + ) -> FieldElement { + match &self.program { + Some(program) => program.eval(values, scratch), + None => (self.combine)(values), + } + } + + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), Error> { + for p in &mut self.polys { + p.fix_first_variable_in_place(r)?; + } + self.num_vars -= 1; + Ok(()) + } + + fn program(&self) -> Option<&Program> { + self.program.as_ref() + } + + fn accept_folded(&mut self, polys: Vec>) -> Result<(), Error> { + if polys.len() != self.polys.len() { + return Err(Error::VariableCountMismatch { + expected: self.polys.len(), + got: polys.len(), + }); + } + self.num_vars = polys.first().map(Mle::num_vars).unwrap_or(0); + self.polys = polys; + Ok(()) + } +} + +/// Multiplies another polynomial by `eq(r, ·)`, appended as one more factor. +/// +/// This is what turns a sumcheck into a zerocheck, and it works for any +/// underlying polynomial rather than only the sum-of-products one. +#[derive(Debug)] +pub struct EqScaled> { + inner: P, + /// `inner`'s factors followed by the `eq` table — the layout `combine` and + /// the sumcheck prover both index. + polys: Vec>, +} + +impl> EqScaled { + /// Wraps `inner` with the `eq(r, ·)` table. + pub fn new(inner: P, eq: Mle) -> Result { + if eq.num_vars() != inner.num_vars() { + return Err(Error::VariableCountMismatch { + expected: inner.num_vars(), + got: eq.num_vars(), + }); + } + let mut polys = inner.polys().to_vec(); + polys.push(eq); + Ok(Self { inner, polys }) + } + + pub fn into_inner(self) -> P { + self.inner + } +} + +impl> SumcheckPolynomial for EqScaled { + fn num_vars(&self) -> usize { + self.inner.num_vars() + } + + fn degree(&self) -> usize { + self.inner.degree() + 1 + } + + fn polys(&self) -> &[Mle] { + &self.polys + } + + fn combine(&self, values: &[FieldElement]) -> FieldElement { + let (inner_values, eq_value) = values.split_at(values.len() - 1); + self.inner.combine(inner_values) * &eq_value[0] + } + + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), Error> { + // `inner` keeps its own copies of the factors, so both views must fold. + self.inner.fix_first_variable(r)?; + for p in &mut self.polys { + p.fix_first_variable_in_place(r)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + eq::eq_mle, + virtual_poly::{Term, VirtualPolynomial}, + }; + + type FE = FieldElement; + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + fn inner() -> VirtualPolynomial { + VirtualPolynomial::new( + vec![mle(&[1, 2, 3, 4]), mle(&[5, 6, 7, 8])], + vec![Term::new(FE::from(2), vec![0, 1])], + ) + .unwrap() + } + + #[test] + fn eq_scaling_adds_one_to_the_degree() { + let r = vec![FE::from(3), FE::from(5)]; + let scaled = EqScaled::new(inner(), eq_mle(&r).unwrap()).unwrap(); + assert_eq!(inner().degree(), 2); + assert_eq!(scaled.degree(), 3); + } + + #[test] + fn eq_scaling_multiplies_pointwise_on_the_cube() { + let r = vec![FE::from(3), FE::from(5)]; + let eq = eq_mle(&r).unwrap(); + let base = inner(); + let scaled = EqScaled::new(inner(), eq.clone()).unwrap(); + + for i in 0..4 { + assert_eq!( + scaled.eval_at_index(i), + base.eval_at_index(i) * eq.evals()[i] + ); + } + } + + #[test] + fn eq_scaled_sum_of_a_multilinear_polynomial_is_its_extension_at_r() { + // Σ_x eq(r,x)·f(x) = f̃(r) — the defining property of eq, and it holds + // only when f is itself multilinear. + let linear = VirtualPolynomial::new( + vec![mle(&[1, 2, 3, 4])], + vec![Term::new(FE::from(2), vec![0])], + ) + .unwrap(); + let r = vec![FE::from(11), FE::from(13)]; + let scaled = EqScaled::new(linear.clone(), eq_mle(&r).unwrap()).unwrap(); + + assert_eq!(scaled.sum_over_hypercube(), linear.evaluate(&r).unwrap()); + } + + #[test] + fn above_degree_one_the_eq_weighted_sum_is_not_the_pointwise_product() { + // For a product of two columns, `evaluate` gives ã(r)·b̃(r) while the + // eq-weighted sum gives the multilinear extension of the *function* + // a·b. Those are different polynomials, and conflating them is an easy + // way to write a wrong soundness argument. + let r = vec![FE::from(11), FE::from(13)]; + let scaled = EqScaled::new(inner(), eq_mle(&r).unwrap()).unwrap(); + + assert_ne!(scaled.sum_over_hypercube(), inner().evaluate(&r).unwrap()); + } + + #[test] + fn folding_keeps_both_views_in_step() { + let r = vec![FE::from(2), FE::from(4)]; + let mut scaled = EqScaled::new(inner(), eq_mle(&r).unwrap()).unwrap(); + let challenge = FE::from(9); + let rest = FE::from(21); + + let expected = scaled.evaluate(&[challenge, rest]).unwrap(); + scaled.fix_first_variable(&challenge).unwrap(); + + assert_eq!(scaled.num_vars(), 1); + assert_eq!(scaled.evaluate(&[rest]).unwrap(), expected); + } + + #[test] + fn rejects_an_eq_table_of_the_wrong_arity() { + let r = vec![FE::from(3)]; + let err = EqScaled::new(inner(), eq_mle(&r).unwrap()).unwrap_err(); + assert_eq!( + err, + Error::VariableCountMismatch { + expected: 2, + got: 1 + } + ); + } + + #[test] + fn a_composed_polynomial_applies_its_closure() { + // 2ab + 3c, written as a closure instead of terms. + let polys = vec![ + mle(&[1, 2, 3, 4]), + mle(&[5, 6, 7, 8]), + mle(&[9, 10, 11, 12]), + ]; + let composed = Composed::new( + polys, + |v: &[FE]| FE::from(2) * v[0] * v[1] + FE::from(3) * v[2], + 2, + ) + .unwrap(); + + assert_eq!(composed.degree(), 2); + for i in 0..4 { + let (a, b, c) = ( + FE::from(1 + i as u64), + FE::from(5 + i as u64), + FE::from(9 + i as u64), + ); + assert_eq!( + composed.eval_at_index(i), + FE::from(2) * a * b + FE::from(3) * c + ); + } + } + + #[test] + fn a_composed_polynomial_matches_the_equivalent_terms() { + let polys = vec![mle(&[1, 2, 3, 4]), mle(&[5, 6, 7, 8])]; + let by_terms = + VirtualPolynomial::new(polys.clone(), vec![Term::new(FE::from(2), vec![0, 1])]) + .unwrap(); + let by_closure = Composed::new(polys, |v: &[FE]| FE::from(2) * v[0] * v[1], 2).unwrap(); + + let point = [FE::from(19), FE::from(23)]; + assert_eq!( + by_terms.evaluate(&point).unwrap(), + by_closure.evaluate(&point).unwrap() + ); + assert_eq!( + by_terms.sum_over_hypercube(), + by_closure.sum_over_hypercube() + ); + } + + #[test] + fn composed_rejects_factors_of_differing_arity() { + let result = Composed::new(vec![mle(&[1, 2]), mle(&[1, 2, 3, 4])], |v: &[FE]| v[0], 1); + assert!(matches!( + result.err(), + Some(Error::VariableCountMismatch { + expected: 1, + got: 2 + }) + )); + } +} diff --git a/crypto/multilinear/src/program.rs b/crypto/multilinear/src/program.rs new file mode 100644 index 000000000..40f988e29 --- /dev/null +++ b/crypto/multilinear/src/program.rs @@ -0,0 +1,776 @@ +//! A batch's rule as straight-line code over the factor values. +//! +//! A [`Rule`](crate::batch::Rule) is a closure, which is what makes a statement +//! easy to state and impossible to hand to a device. Every rule this crate +//! proves is nonetheless *data* — a compiled constraint program, an affine +//! expression over factor slots, a fixed formula — so a rule can carry the +//! program it is, and the sumcheck can run that instead: one description, run +//! by the host loop and by the kernel that replaces it. +//! +//! Values are whatever the sumcheck's field is. Two evaluators of the same +//! program may sum in different orders, which the field does not distinguish: +//! Goldilocks compares and serializes canonically, so equal values are equal +//! everywhere the protocol looks. + +use math::field::{element::FieldElement, traits::IsField}; + +use crate::Error; + +/// One step. Operands are indices of earlier steps; `Var` indexes the factor +/// values the sumcheck supplies. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Op { + Fixed(FieldElement), + Var(u32), + Add(u32, u32), + Sub(u32, u32), + Mul(u32, u32), + Neg(u32), +} + +/// Straight-line code and the step holding its value. +#[derive(Clone, Debug)] +pub struct Program { + steps: Vec>, + root: u32, +} + +impl Program { + pub fn steps(&self) -> &[Op] { + &self.steps + } + + pub fn root(&self) -> u32 { + self.root + } + + /// The program's value, given every factor's value at one point. + /// + /// `scratch` is the caller's, because this runs once per cube index per + /// interpolation node: a buffer of its own per call would be one heap + /// allocation per evaluation. + pub fn eval( + &self, + values: &[FieldElement], + scratch: &mut Vec>, + ) -> FieldElement { + scratch.clear(); + scratch.reserve(self.steps.len()); + for step in &self.steps { + let v = match *step { + Op::Fixed(ref c) => c.clone(), + Op::Var(i) => values[i as usize].clone(), + Op::Add(a, b) => &scratch[a as usize] + &scratch[b as usize], + Op::Sub(a, b) => &scratch[a as usize] - &scratch[b as usize], + Op::Mul(a, b) => &scratch[a as usize] * &scratch[b as usize], + Op::Neg(a) => -&scratch[a as usize], + }; + scratch.push(v); + } + scratch[self.root as usize].clone() + } + + /// The same program, emitted on demand: each step where the step that + /// uses it first needs it, the operand that holds more values evaluated + /// first, and every read — a factor's value or a constant — emitted again + /// at each use instead of once and held. + /// + /// A value lives from its step to its last use. [`simplify`](Self::simplify) + /// makes every read of a column or a constant one step, and a batch is + /// written piece by piece: every root, then their weighted sum; every + /// interaction's side, then theirs. So a batch holds each shared read from + /// its first use to its last, and each root or side until the sum at the + /// end. On the widest precompile's batch that is 2,763 values a thread + /// (`thoughts/zf/gap2/fix2/I-GFS.md` §15.5). On demand, a sum's terms are + /// added in as they are made, which is the running sum a lean builder + /// would write, and a read lives for one step. + /// + /// The order is the Sethi–Ullman one: of two operands, the one needing more + /// values goes first, so the other is held while less is live. A step is + /// the same operation on the same operands as before, so every value is + /// the same, to the limb. A reload costs a load, plus the extension to the + /// interpolation node for a factor. Nothing is simplified again, which + /// would merge the reads back. + pub fn on_demand(&self) -> Self { + let n = self.steps.len(); + let reads = |op: &Op| matches!(op, Op::Var(_) | Op::Fixed(_)); + // Values each step needs to be evaluated as a tree (a read, one). + let mut need = vec![0u32; n]; + for (k, op) in self.steps.iter().enumerate() { + need[k] = match *op { + Op::Var(_) | Op::Fixed(_) => 1, + Op::Neg(a) => need[a as usize], + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => { + let (x, y) = (need[a as usize], need[b as usize]); + if x == y { + x.saturating_add(1) + } else { + x.max(y) + } + } + }; + } + let mut steps: Vec> = Vec::with_capacity(n * 3 / 2); + // Where each old step landed, once it has; a read has no one place. + let mut at: Vec> = vec![None; n]; + let read = |steps: &mut Vec>, at: &[Option], k: u32| -> u32 { + match &self.steps[k as usize] { + Op::Var(i) => { + steps.push(Op::Var(*i)); + (steps.len() - 1) as u32 + } + Op::Fixed(c) => { + steps.push(Op::Fixed(c.clone())); + (steps.len() - 1) as u32 + } + _ => at[k as usize].expect("an operand is emitted before its user"), + } + }; + if reads(&self.steps[self.root as usize]) { + let root = read(&mut steps, &at, self.root); + return Self { steps, root }; + } + // A step, and whether its operands are already under way. + let mut stack: Vec<(u32, bool)> = vec![(self.root, false)]; + while let Some((k, expanded)) = stack.pop() { + if at[k as usize].is_some() { + continue; + } + let op = &self.steps[k as usize]; + let operands = match *op { + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => [Some(a), Some(b)], + Op::Neg(a) => [Some(a), None], + Op::Var(_) | Op::Fixed(_) => unreachable!("a read is emitted at its use"), + }; + if !expanded { + stack.push((k, true)); + let mut pending: Vec = operands + .into_iter() + .flatten() + .filter(|&o| !reads(&self.steps[o as usize]) && at[o as usize].is_none()) + .collect(); + // The one needing more goes first, so it goes on top; a tie + // keeps the written order. + if let [a, b] = pending[..] + && need[b as usize] > need[a as usize] + { + pending.swap(0, 1); + } + stack.extend(pending.into_iter().rev().map(|o| (o, false))); + continue; + } + let emitted = match *op { + Op::Add(a, b) => { + let (a, b) = (read(&mut steps, &at, a), read(&mut steps, &at, b)); + Op::Add(a, b) + } + Op::Sub(a, b) => { + let (a, b) = (read(&mut steps, &at, a), read(&mut steps, &at, b)); + Op::Sub(a, b) + } + Op::Mul(a, b) => { + let (a, b) = (read(&mut steps, &at, a), read(&mut steps, &at, b)); + Op::Mul(a, b) + } + Op::Neg(a) => Op::Neg(read(&mut steps, &at, a)), + Op::Var(_) | Op::Fixed(_) => unreachable!("a read is emitted at its use"), + }; + steps.push(emitted); + at[k as usize] = Some((steps.len() - 1) as u32); + } + let root = at[self.root as usize].expect("the root is emitted"); + Self { steps, root } + } + + /// The same program with each step emitted once and nothing dead in it. + /// + /// Straight-line code over a field: two steps with the same operator over + /// the same operands compute the same value, adding zero or multiplying by + /// one computes nothing, and a step the root does not reach is not + /// computed at all. A batch's program is spliced together out of pieces + /// that share structure — the same alpha powers, the same column read by + /// several interactions — so what this removes is not a mistake in any one + /// of them, it is the seam between them. + /// + /// The kernel walks this program once per cube index per interpolation + /// node, through a slot file in global memory, so a step removed here is + /// removed from every one of those walks. + pub fn simplify(&self) -> Self + where + FieldElement: PartialEq, + { + use std::collections::HashMap; + + let zero = FieldElement::::zero(); + let one = FieldElement::::one(); + let mut steps: Vec> = Vec::with_capacity(self.steps.len()); + // Where each old step ended up. + let mut moved: Vec = Vec::with_capacity(self.steps.len()); + // Steps that are not constants, keyed by what they compute. + let mut seen: HashMap<(u8, u32, u32), u32> = HashMap::new(); + // Constants, which have to be compared by value. + let mut constants: Vec<(u32, FieldElement)> = Vec::new(); + + let constant_of = |steps: &[Op], at: u32| match &steps[at as usize] { + Op::Fixed(c) => Some(c.clone()), + _ => None, + }; + + for step in &self.steps { + let emit = |steps: &mut Vec>, + seen: &mut HashMap<(u8, u32, u32), u32>, + constants: &mut Vec<(u32, FieldElement)>, + op: Op| + -> u32 { + if let Op::Fixed(value) = &op { + if let Some((at, _)) = constants.iter().find(|(_, c)| c == value) { + return *at; + } + steps.push(op.clone()); + let at = (steps.len() - 1) as u32; + constants.push((at, value.clone())); + return at; + } + // Addition and multiplication do not care which operand came + // first, so the key does not either. + let key = match op { + Op::Var(i) => (0, i, 0), + Op::Add(a, b) => (1, a.min(b), a.max(b)), + Op::Sub(a, b) => (2, a, b), + Op::Mul(a, b) => (3, a.min(b), a.max(b)), + Op::Neg(a) => (4, a, 0), + Op::Fixed(_) => unreachable!("handled above"), + }; + if let Some(at) = seen.get(&key) { + return *at; + } + steps.push(op); + let at = (steps.len() - 1) as u32; + seen.insert(key, at); + at + }; + + let at = match *step { + Op::Fixed(ref c) => { + emit(&mut steps, &mut seen, &mut constants, Op::Fixed(c.clone())) + } + Op::Var(i) => emit(&mut steps, &mut seen, &mut constants, Op::Var(i)), + Op::Neg(a) => { + let a = moved[a as usize]; + match constant_of(&steps, a) { + Some(c) => emit(&mut steps, &mut seen, &mut constants, Op::Fixed(-c)), + None => emit(&mut steps, &mut seen, &mut constants, Op::Neg(a)), + } + } + Op::Add(a, b) => { + let (a, b) = (moved[a as usize], moved[b as usize]); + match (constant_of(&steps, a), constant_of(&steps, b)) { + (Some(x), Some(y)) => { + emit(&mut steps, &mut seen, &mut constants, Op::Fixed(x + y)) + } + (Some(x), None) if x == zero => b, + (None, Some(y)) if y == zero => a, + _ => emit(&mut steps, &mut seen, &mut constants, Op::Add(a, b)), + } + } + Op::Sub(a, b) => { + let (a, b) = (moved[a as usize], moved[b as usize]); + match (constant_of(&steps, a), constant_of(&steps, b)) { + (Some(x), Some(y)) => { + emit(&mut steps, &mut seen, &mut constants, Op::Fixed(x - y)) + } + (None, Some(y)) if y == zero => a, + _ => emit(&mut steps, &mut seen, &mut constants, Op::Sub(a, b)), + } + } + Op::Mul(a, b) => { + let (a, b) = (moved[a as usize], moved[b as usize]); + match (constant_of(&steps, a), constant_of(&steps, b)) { + (Some(x), Some(y)) => { + emit(&mut steps, &mut seen, &mut constants, Op::Fixed(x * y)) + } + (Some(x), _) if x == zero => emit( + &mut steps, + &mut seen, + &mut constants, + Op::Fixed(zero.clone()), + ), + (_, Some(y)) if y == zero => emit( + &mut steps, + &mut seen, + &mut constants, + Op::Fixed(zero.clone()), + ), + (Some(x), None) if x == one => b, + (None, Some(y)) if y == one => a, + _ => emit(&mut steps, &mut seen, &mut constants, Op::Mul(a, b)), + } + } + }; + moved.push(at); + } + + let root = moved[self.root as usize]; + Self { steps, root }.prune() + } + + /// Drops the steps the root does not reach, renumbering the rest. + fn prune(self) -> Self { + let mut live = vec![false; self.steps.len()]; + live[self.root as usize] = true; + for i in (0..self.steps.len()).rev() { + if !live[i] { + continue; + } + match self.steps[i] { + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => { + live[a as usize] = true; + live[b as usize] = true; + } + Op::Neg(a) => live[a as usize] = true, + Op::Fixed(_) | Op::Var(_) => {} + } + } + let mut moved = vec![0u32; self.steps.len()]; + let mut steps = Vec::with_capacity(self.steps.len()); + for (i, step) in self.steps.into_iter().enumerate() { + if !live[i] { + continue; + } + let shifted = match step { + Op::Fixed(c) => Op::Fixed(c), + Op::Var(v) => Op::Var(v), + Op::Add(a, b) => Op::Add(moved[a as usize], moved[b as usize]), + Op::Sub(a, b) => Op::Sub(moved[a as usize], moved[b as usize]), + Op::Mul(a, b) => Op::Mul(moved[a as usize], moved[b as usize]), + Op::Neg(a) => Op::Neg(moved[a as usize]), + }; + steps.push(shifted); + moved[i] = (steps.len() - 1) as u32; + } + let root = moved[self.root as usize]; + Self { steps, root } + } + + /// The highest factor slot the program reads, or `None` when it reads none. + pub fn max_slot(&self) -> Option { + self.steps + .iter() + .filter_map(|step| match step { + Op::Var(i) => Some(*i), + _ => None, + }) + .max() + } +} + +/// Emits steps, handing back the index of each one. +#[derive(Debug, Default)] +pub struct Builder { + steps: Vec>, +} + +impl Builder { + pub fn new() -> Self { + Self { steps: Vec::new() } + } + + fn push(&mut self, op: Op) -> u32 { + self.steps.push(op); + (self.steps.len() - 1) as u32 + } + + pub fn fixed(&mut self, value: FieldElement) -> u32 { + self.push(Op::Fixed(value)) + } + + pub fn var(&mut self, slot: usize) -> u32 { + self.push(Op::Var(slot as u32)) + } + + pub fn add(&mut self, a: u32, b: u32) -> u32 { + self.push(Op::Add(a, b)) + } + + pub fn sub(&mut self, a: u32, b: u32) -> u32 { + self.push(Op::Sub(a, b)) + } + + pub fn mul(&mut self, a: u32, b: u32) -> u32 { + self.push(Op::Mul(a, b)) + } + + pub fn neg(&mut self, a: u32) -> u32 { + self.push(Op::Neg(a)) + } + + /// `Σ terms`, or a zero step when there are none. + pub fn sum(&mut self, terms: &[u32]) -> u32 { + match terms.split_first() { + None => self.fixed(FieldElement::zero()), + Some((&first, rest)) => rest.iter().fold(first, |acc, &t| self.add(acc, t)), + } + } + + /// `Σ coefficient_i · step_i`, folding a coefficient of one away. + pub fn weighted_sum(&mut self, terms: &[(u32, FieldElement)]) -> u32 { + let scaled: Vec = terms + .iter() + .map(|(step, coefficient)| { + if *coefficient == FieldElement::one() { + *step + } else { + let c = self.fixed(coefficient.clone()); + self.mul(c, *step) + } + }) + .collect(); + self.sum(&scaled) + } + + /// Copies `program`'s steps in, renumbering its operands, and returns where + /// its root landed. + pub fn splice(&mut self, program: &Program) -> u32 { + let base = self.steps.len() as u32; + for step in &program.steps { + let shifted = match *step { + Op::Fixed(ref c) => Op::Fixed(c.clone()), + Op::Var(i) => Op::Var(i), + Op::Add(a, b) => Op::Add(a + base, b + base), + Op::Sub(a, b) => Op::Sub(a + base, b + base), + Op::Mul(a, b) => Op::Mul(a + base, b + base), + Op::Neg(a) => Op::Neg(a + base), + }; + self.steps.push(shifted); + } + base + program.root + } + + pub fn len(&self) -> usize { + self.steps.len() + } + + pub fn is_empty(&self) -> bool { + self.steps.is_empty() + } + + /// The program computing `root`, with each step emitted once. + /// + /// A builder is written for whoever emits — a rule at a time, a piece at a + /// time — and the pieces overlap: the same alpha power, the same column, + /// the same difference. [`simplify`](Program::simplify) is run here because + /// this is where a program stops being written and starts being walked, + /// and it is walked once per cube index per interpolation node. + pub fn finish(self, root: u32) -> Result, Error> + where + FieldElement: PartialEq, + { + if root as usize >= self.steps.len() { + return Err(Error::UnknownPolynomial { + index: root as usize, + len: self.steps.len(), + }); + } + Ok(Program { + steps: self.steps, + root, + } + .simplify()) + } + + /// The program as it was emitted, step for step. + pub fn finish_verbatim(self, root: u32) -> Result, Error> { + if root as usize >= self.steps.len() { + return Err(Error::UnknownPolynomial { + index: root as usize, + len: self.steps.len(), + }); + } + Ok(Program { + steps: self.steps, + root, + }) + } +} + +#[cfg(test)] +mod simplify_tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + /// Every simplification has to be invisible to the evaluator: same value at + /// every point, fewer steps to get there. + fn agrees(program: &Program, slots: usize) { + let small = program.simplify(); + assert!( + small.steps().len() <= program.steps().len(), + "simplifying grew the program" + ); + let mut scratch = Vec::new(); + for seed in 0..6u64 { + let values: Vec = (0..slots) + .map(|i| FE::from((seed + 1) * (i as u64 + 3) + 7)) + .collect(); + let before = program.eval(&values, &mut scratch); + let after = small.eval(&values, &mut scratch); + assert_eq!(before, after, "seed {seed}"); + } + } + + /// A program on demand is the same program: every read becomes one step + /// per use, the root included when it is a read, a shared step is still + /// emitted once, and the value at every point is the same. + #[test] + fn a_program_on_demand_is_the_same_program() { + let mut b = Builder::::new(); + let x = b.var(0); + let y = b.var(1); + let seven = b.fixed(FE::from(7u64)); + let xy = b.mul(x, y); + let a = b.add(xy, seven); + let d = b.sub(a, x); + let n = b.neg(d); + let shared = b.mul(n, seven); + let root = b.mul(shared, shared); + let program = b.finish(root).unwrap(); + let demand = program.on_demand(); + let count = |p: &Program, read: bool| { + p.steps() + .iter() + .filter(|op| matches!(op, Op::Var(_) | Op::Fixed(_)) == read) + .count() + }; + // x twice, y once, seven twice: one read per use. The six other steps + // are six still: `shared` is used twice and made once. + assert_eq!((count(&program, true), count(&demand, true)), (3, 5)); + assert_eq!((count(&program, false), count(&demand, false)), (6, 6)); + let mut scratch = Vec::new(); + for seed in 0..6u64 { + let values: Vec = (0..2).map(|i| FE::from((seed + 1) * (i + 3) + 7)).collect(); + assert_eq!( + program.eval(&values, &mut scratch), + demand.eval(&values, &mut scratch), + "seed {seed}" + ); + } + // A program that is a single read is that read. + let mut b = Builder::::new(); + let only = b.var(3); + let read = b.finish(only).unwrap().on_demand(); + assert_eq!(read.steps(), &[Op::Var(3)]); + assert_eq!(read.root(), 0); + } + + /// ★ What the schedule is for: a weighted sum written after its terms + /// holds every term until the sum; on demand it holds the running sum and + /// the term being made. Eight products of two reads each, summed with + /// weights, as `weighted_sum` writes them after the terms. + #[test] + fn a_sum_on_demand_holds_one_term_at_a_time() { + let mut b = Builder::::new(); + let terms: Vec<(u32, FE)> = (0..8u32) + .map(|i| { + let (p, q) = (b.var(2 * i as usize), b.var(2 * i as usize + 1)); + (b.mul(p, q), FE::from(u64::from(i) + 2)) + }) + .collect(); + let root = b.weighted_sum(&terms); + let program = b.finish(root).unwrap(); + let demand = program.on_demand(); + let live = |p: &Program| { + // A value is live from its step to its last use. + let mut last = vec![0usize; p.steps().len()]; + for (k, op) in p.steps().iter().enumerate() { + match *op { + Op::Add(a, c) | Op::Sub(a, c) | Op::Mul(a, c) => { + last[a as usize] = k; + last[c as usize] = k; + } + Op::Neg(a) => last[a as usize] = k, + Op::Var(_) | Op::Fixed(_) => {} + } + } + last[p.root() as usize] = p.steps().len(); + (0..p.steps().len()) + .map(|k| (0..=k).filter(|&j| last[j] >= k).count()) + .max() + .unwrap() + }; + assert!( + live(&program) >= 8, + "written after its terms, the sum holds its eight terms: {}", + live(&program) + ); + assert!( + live(&demand) <= 4, + "on demand it holds the running sum, a term and its two reads: {}", + live(&demand) + ); + let values: Vec = (0..16u64).map(|i| FE::from(i * 5 + 1)).collect(); + let mut scratch = Vec::new(); + assert_eq!( + program.eval(&values, &mut scratch), + demand.eval(&values, &mut scratch) + ); + } + + #[test] + fn a_repeated_step_is_emitted_once() { + let mut b = Builder::::new(); + let x = b.var(0); + let y = b.var(1); + let first = b.mul(x, y); + // The same product again, and the same one with its operands swapped. + let second = b.mul(x, y); + let third = b.mul(y, x); + let sum = b.add(first, second); + let root = b.add(sum, third); + let program = b.finish_verbatim(root).unwrap(); + + agrees(&program, 2); + // one var, one var, one product, two sums + assert_eq!(program.simplify().steps().len(), 5); + } + + #[test] + fn constants_fold_and_units_disappear() { + let mut b = Builder::::new(); + let x = b.var(0); + let one = b.fixed(FE::one()); + let zero = b.fixed(FE::zero()); + let scaled = b.mul(x, one); + let shifted = b.add(scaled, zero); + let two = b.fixed(FE::from(2)); + let three = b.fixed(FE::from(3)); + let six = b.mul(two, three); + let root = b.add(shifted, six); + let program = b.finish_verbatim(root).unwrap(); + + agrees(&program, 1); + let small = program.simplify(); + // `x`, the constant six, and their sum. + assert_eq!(small.steps().len(), 3); + } + + #[test] + fn what_the_root_does_not_reach_is_dropped() { + let mut b = Builder::::new(); + let x = b.var(0); + let y = b.var(1); + let _dead = b.mul(x, y); + let root = b.add(x, x); + let program = b.finish_verbatim(root).unwrap(); + + agrees(&program, 2); + assert_eq!(program.simplify().steps().len(), 2); + } +} + +/// `Σ_i lambda_i · program_i`, the batch as one program. +pub fn combine( + programs: &[&Program], + lambdas: &[FieldElement], +) -> Result, Error> { + if programs.len() != lambdas.len() { + return Err(Error::VariableCountMismatch { + expected: programs.len(), + got: lambdas.len(), + }); + } + let mut builder = Builder::::new(); + let terms: Vec<(u32, FieldElement)> = programs + .iter() + .zip(lambdas) + .map(|(program, lambda)| (builder.splice(program), lambda.clone())) + .collect(); + let root = builder.weighted_sum(&terms); + builder.finish(root) +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn values(n: usize) -> Vec { + (0..n as u64).map(|i| FE::from(i * 7 + 3)).collect() + } + + #[test] + fn a_program_evaluates_its_root() { + // (f0 + f1) · f2 − 5 + let mut b = Builder::::new(); + let f0 = b.var(0); + let f1 = b.var(1); + let f2 = b.var(2); + let sum = b.add(f0, f1); + let product = b.mul(sum, f2); + let five = b.fixed(FE::from(5u64)); + let root = b.sub(product, five); + let program = b.finish(root).unwrap(); + + let v = values(3); + let mut scratch = Vec::new(); + assert_eq!( + program.eval(&v, &mut scratch), + (v[0] + v[1]) * v[2] - FE::from(5u64) + ); + assert_eq!(program.max_slot(), Some(2)); + } + + #[test] + fn a_spliced_program_keeps_its_meaning() { + let mut b = Builder::::new(); + let f0 = b.var(0); + let f1 = b.var(1); + let root = b.mul(f0, f1); + let inner = b.finish(root).unwrap(); + + let mut outer = Builder::::new(); + // A step before the splice, so the renumbering is not a no-op. + let f2 = outer.var(2); + let spliced = outer.splice(&inner); + let root = outer.add(spliced, f2); + let program = outer.finish(root).unwrap(); + + let v = values(3); + let mut scratch = Vec::new(); + assert_eq!(program.eval(&v, &mut scratch), v[0] * v[1] + v[2]); + } + + #[test] + fn a_combined_program_is_the_weighted_sum() { + let mut b = Builder::::new(); + let root = b.var(0); + let first = b.finish(root).unwrap(); + let mut b = Builder::::new(); + let root = b.var(1); + let second = b.finish(root).unwrap(); + + let lambdas = vec![FE::one(), FE::from(9u64)]; + let program = combine(&[&first, &second], &lambdas).unwrap(); + let v = values(2); + let mut scratch = Vec::new(); + assert_eq!(program.eval(&v, &mut scratch), v[0] + FE::from(9u64) * v[1]); + } + + #[test] + fn a_weighted_sum_of_nothing_is_zero() { + let mut b = Builder::::new(); + let root = b.weighted_sum(&[]); + let program = b.finish(root).unwrap(); + let mut scratch = Vec::new(); + assert_eq!(program.eval(&[], &mut scratch), FE::zero()); + } + + #[test] + fn a_root_past_the_end_is_rejected() { + let mut b = Builder::::new(); + b.var(0); + assert!(b.finish(7).is_err()); + } +} diff --git a/crypto/multilinear/src/query_count.rs b/crypto/multilinear/src/query_count.rs new file mode 100644 index 000000000..c7da37282 --- /dev/null +++ b/crypto/multilinear/src/query_count.rs @@ -0,0 +1,316 @@ +//! ★ The query count, derived in INTEGERS. +//! +//! `ChainConfig::with_security` computed `num_queries` with `f64` `sqrt`, +//! `log2` and `ceil`. That is fine on a host and a problem everywhere else this +//! protocol is going: a field-native verifier has no floating point, and a +//! prover and verifier that disagree by one query do not fail gracefully — the +//! transcript diverges and every later challenge is different. +//! +//! So the derivation lives here, in fixed-point integer arithmetic, and the +//! `f64` version survives only as the reference the tests enumerate against +//! over the whole parameter grid. +//! +//! # The formula, unchanged +//! +//! ```text +//! rate = 2^-log_blowup +//! proximity = 1 - sqrt(rate) - 1/300 (the Johnson bound) +//! bits_per_query = -log2(1 - proximity) +//! = -log2(sqrt(rate) + 1/300) +//! target = security_bits + log2(rounds) (a union bound over rounds) +//! num_queries = max(ceil(max(target - grind, 0) / bits_per_query), 1) +//! ``` +//! +//! ⚠ **This is not a soundness analysis and this module does not make it one.** +//! It is the conservative mirror of the parameters the univariate prover ships, +//! as `with_security`'s own doc says. Moving it to integers changes who can +//! evaluate it, not what it claims. +//! +//! # Precision, and why the answers are the same +//! +//! Everything is Q62 — 62 fractional bits in a `u128` — against `f64`'s 53 bits +//! of mantissa, so this form is strictly more precise than the one it replaces. +//! Where they could still differ is a grid point whose true ratio sits within a +//! rounding step of an integer, and `ceil` then goes either way. That is not +//! argued here, it is ENUMERATED: the grid test walks every point the protocol +//! can reach and fails on any disagreement. +//! +//! Q62 rather than Q64 for one concrete reason: the log2 loop squares its +//! running value, and a Q64 mantissa in `[1, 2)` squares to 130 bits, which a +//! `u128` does not hold. At Q62 the square fits with two bits to spare. + +/// Fractional bits in the fixed-point representation. +const FRAC: u32 = 62; +/// The value `1.0`. +const ONE: u128 = 1 << FRAC; + +/// Integer square root of a `u128`, by bit-by-bit restoring subtraction. +/// +/// Exact: returns `floor(sqrt(n))`. Written out rather than reached for through +/// a float, which is the thing this module exists to avoid. +fn isqrt(n: u128) -> u128 { + if n == 0 { + return 0; + } + // The largest power of four not exceeding `n`. + let mut bit: u128 = 1u128 << ((127 - n.leading_zeros()) & !1u32); + let mut rem = n; + let mut root: u128 = 0; + while bit != 0 { + if rem >= root + bit { + rem -= root + bit; + root = (root >> 1) + bit; + } else { + root >>= 1; + } + bit >>= 2; + } + root +} + +/// `log2(x / 2^FRAC)` in Q62, for `x > 0`. +/// +/// The integer part comes from the leading bit; the fraction from the classic +/// squaring loop — square the mantissa, and a result at or above two both emits +/// a one bit and halves the value. +fn log2_fixed(x: u128) -> i128 { + debug_assert!(x > 0, "log2 of zero is not a number this protocol uses"); + + // Normalise the mantissa into `[1, 2)`, i.e. `[2^FRAC, 2^(FRAC+1))`. + let bits = 128 - x.leading_zeros(); // position of the leading one, 1-based + let exponent = bits as i128 - 1 - FRAC as i128; + let mut mantissa = if exponent >= 0 { + x >> (exponent as u32) + } else { + x << ((-exponent) as u32) + }; + debug_assert!((ONE..ONE << 1).contains(&mantissa)); + + let mut fraction: u128 = 0; + let mut weight = ONE >> 1; + for _ in 0..FRAC { + // `mantissa` is in `[1, 2)`, so the square is in `[1, 4)` and fits. + mantissa = (mantissa * mantissa) >> FRAC; + if mantissa >= ONE << 1 { + mantissa >>= 1; + fraction |= weight; + } + weight >>= 1; + } + + (exponent << FRAC) + fraction as i128 +} + +/// `sqrt(2^-log_blowup)` in Q62. +/// +/// `sqrt(2^-b) * 2^62 = sqrt(2^(124 - b))`, so one integer square root does it +/// — exactly when `124 - b` is even, floored otherwise. +fn sqrt_rate(log_blowup: usize) -> u128 { + assert!( + log_blowup < 124, + "a rate of 2^-{log_blowup} is not a code this protocol can use" + ); + isqrt(1u128 << (124 - log_blowup as u32)) +} + +/// ★ Queries needed for `security_bits` under the Johnson bound, given the +/// round count and the proof of work spent on the query challenge. +/// +/// The integer twin of what `ChainConfig::with_security` used to compute in +/// `f64`, and the one the configuration now uses. +pub fn num_queries(log_blowup: usize, rounds: usize, security_bits: u8, grind_query: u8) -> usize { + // 1 - proximity = sqrt(rate) + 1/300. + let one_over_300 = ONE / 300; + let w = sqrt_rate(log_blowup) + one_over_300; + + // A rate whose `w` reached 1 would buy nothing per query. `log_blowup >= 1` + // keeps `w <= 0.708`. + assert!(w < ONE, "each query must buy a positive number of bits"); + let bits_per_query = -log2_fixed(w); + debug_assert!(bits_per_query > 0); + + let rounds = rounds.max(1); + let target = ((security_bits as i128) << FRAC) + log2_fixed((rounds as u128) << FRAC); + let left = (target - ((grind_query as i128) << FRAC)).max(0); + + // Both sides are Q62, so the ratio is a plain integer one. Written out + // rather than `div_ceil`, which is unstable for `i128`; both operands are + // known non-negative here (`left` is clamped, `bits_per_query` asserted + // positive), so the rounding has no sign case to get wrong. + let queries = (left + bits_per_query - 1) / bits_per_query; + (queries as usize).max(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The `f64` derivation this replaces, verbatim from `with_security` as it + /// stood — the reference, kept only so the integer form can be checked + /// against it. + fn f64_reference( + log_blowup: usize, + rounds: usize, + security_bits: u8, + grind_query: u8, + ) -> usize { + let rounds = rounds.max(1); + let rate = 1.0 / (1u64 << log_blowup) as f64; + let proximity = 1.0 - rate.sqrt() - 1.0 / 300.0; + let bits_per_query = -(1.0 - proximity).log2(); + + let target = security_bits as f64 + (rounds as f64).log2(); + let left = (target - grind_query as f64).max(0.0); + (left / bits_per_query).ceil().max(1.0) as usize + } + + /// ★★ The whole realistic grid, enumerated. Roughly 4.3 million points. + /// + /// Not a spot check and not an argument: the one way the two forms can + /// differ is a ratio landing within a rounding step of an integer, and the + /// only honest way to know whether that happens anywhere the protocol can + /// reach is to look at every point it can reach. + #[test] + fn the_integer_derivation_agrees_with_the_f64_reference() { + let mut checked = 0u64; + for log_blowup in 1..=4usize { + for rounds in 1..=64usize { + for security_bits in 0..=255u8 { + for grind_query in 0..=64u8 { + let got = num_queries(log_blowup, rounds, security_bits, grind_query); + let want = f64_reference(log_blowup, rounds, security_bits, grind_query); + assert_eq!( + got, want, + "blowup 2^{log_blowup}, {rounds} rounds, {security_bits} bits, \ + grind {grind_query}" + ); + checked += 1; + } + } + } + } + assert_eq!(checked, 4 * 64 * 256 * 65, "the grid must be fully walked"); + } + + /// ✓ The grid above is worth walking only if the answers vary across it. + #[test] + fn the_grid_is_not_one_answer_everywhere() { + let mut seen = std::collections::BTreeSet::new(); + for log_blowup in 1..=4usize { + for security_bits in [0u8, 64, 128, 255] { + for grind_query in [0u8, 20, 64] { + seen.insert(num_queries(log_blowup, 6, security_bits, grind_query)); + } + } + } + assert!( + seen.len() > 10, + "only {} distinct counts across the sample: the grid is degenerate", + seen.len() + ); + } + + /// ★ The shipped posture, pinned to its literal: blowup 4, 128 bits, 20 + /// bits of query grinding, one round — the same 110 the univariate + /// prover's own accounting gives. + #[test] + fn the_shipped_posture_is_110_at_one_round() { + assert_eq!(num_queries(2, 1, 128, 20), 110); + } + + /// ★ And 112 / 113 at the round counts a real proof reaches. + #[test] + fn the_shipped_posture_is_112_then_113_as_the_rounds_grow() { + for rounds in 4..=7 { + assert_eq!(num_queries(2, rounds, 128, 20), 112, "{rounds} rounds"); + } + for rounds in 8..=15 { + assert_eq!(num_queries(2, rounds, 128, 20), 113, "{rounds} rounds"); + } + } + + /// More grinding buys fewer queries, and a wider blowup buys more per + /// query — the two monotonicities the formula is supposed to have, checked + /// rather than assumed. + #[test] + fn the_count_moves_the_way_the_parameters_say_it_should() { + let at = |b, g| num_queries(b, 6, 128, g); + assert!( + at(2, 30) < at(2, 20), + "grinding must reduce the query count" + ); + assert!( + at(3, 20) < at(2, 20), + "a wider blowup must buy more per query" + ); + assert!(at(4, 20) < at(3, 20)); + for g in 0..64u8 { + assert!( + at(2, g + 1) <= at(2, g), + "the count must not rise with grinding at {g}" + ); + } + } + + /// The floor: a configuration that has already ground past its target still + /// checks one position. + #[test] + fn at_least_one_query_is_always_drawn() { + assert_eq!(num_queries(2, 1, 0, 64), 1); + assert_eq!(num_queries(2, 1, 10, 64), 1); + } + + /// `isqrt` against the definition: `r^2 <= n < (r+1)^2`. + #[test] + fn the_integer_square_root_is_the_floor_of_the_real_one() { + for n in [0u128, 1, 2, 3, 4, 5, 99, 100, 101, 1 << 40, (1 << 62) + 7] { + let r = isqrt(n); + assert!(r * r <= n, "isqrt({n}) = {r} is too large"); + assert!((r + 1).checked_mul(r + 1).is_none_or(|s| s > n)); + } + // The shapes `sqrt_rate` actually asks for. + for b in 1..=4u32 { + let n = 1u128 << (124 - b); + let r = isqrt(n); + assert!(r * r <= n && (r + 1) * (r + 1) > n, "blowup 2^{b}"); + } + } + + /// `log2_fixed` against `f64::log2` — a different algorithm for the same + /// number, to within the precision the fixed point carries. + #[test] + fn the_fixed_point_log2_agrees_with_the_floating_one() { + for v in [ + 1.0f64, + 1.5, + 2.0, + 3.0, + 7.0, + 64.0, + 0.5, + 0.25, + 0.1, + 0.708, + 1.0 / 300.0, + ] { + let x = (v * (ONE as f64)) as u128; + let got = log2_fixed(x) as f64 / ONE as f64; + let want = (x as f64 / ONE as f64).log2(); + assert!((got - want).abs() < 1e-15, "log2({v}): {got} vs {want}"); + } + } + + /// ✓ Exact on the powers of two, where the answer is an integer and any + /// drift in the squaring loop would show as a fraction. + #[test] + fn the_fixed_point_log2_is_exact_on_powers_of_two() { + for k in -30i32..=30 { + let x = if k >= 0 { ONE << k } else { ONE >> (-k) }; + assert_eq!( + log2_fixed(x), + (k as i128) << FRAC, + "log2(2^{k}) must be exactly {k}" + ); + } + } +} diff --git a/crypto/multilinear/src/records.rs b/crypto/multilinear/src/records.rs new file mode 100644 index 000000000..02c2e4cbd --- /dev/null +++ b/crypto/multilinear/src/records.rs @@ -0,0 +1,1207 @@ +//! A table's committed columns held as the records its rows come from, a small +//! fraction of the columns' bytes: the card fills the columns from them as it +//! uploads the table (`math_cuda::fill`), and a host reader that asks gets them +//! from the host fill the records also carry. +//! +//! Both fills write the same words, so the store a group's upload makes holds +//! the words the table's packed columns would have widened to, at the same +//! places ([`crate::gpu::upload_tables`]), and no proof byte depends on which +//! one ran. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::narrow::NarrowColumns; + +/// Tables filled from their records on the card and on the host since the +/// process started ([`fills`]). +static CARD_FILLS: AtomicU64 = AtomicU64::new(0); +static HOST_FILLS: AtomicU64 = AtomicU64::new(0); + +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn note_card_fills(tables: usize) { + CARD_FILLS.fetch_add(tables as u64, Ordering::Relaxed); +} + +pub(crate) fn note_host_fill() { + HOST_FILLS.fetch_add(1, Ordering::Relaxed); +} + +/// `(on the card, on the host)`: the tables filled from their records so far. +/// A block with a card fills each such table there, once a commit and once an +/// argument; a host fill is a reader that asked for the columns on the host. +pub fn fills() -> (u64, u64) { + ( + CARD_FILLS.load(Ordering::Relaxed), + HOST_FILLS.load(Ordering::Relaxed), + ) +} + +/// Under `LAMBDA_VM_CARD_UPLOAD_TIMES=1`, the group uploads' tables timed on +/// the card since the process started: +/// `((packed tables, copy and widen s), (records tables, copy and fill s))` +/// (`math_cuda::columns::upload_kernel_times`). Zeros otherwise, and in a +/// build without a card. +pub fn upload_times() -> ((u64, f64), (u64, f64)) { + #[cfg(feature = "cuda")] + { + math_cuda::columns::upload_kernel_times() + } + #[cfg(not(feature = "cuda"))] + { + ((0, 0.0), (0, 0.0)) + } +} + +/// One keccak permutation call as KECCAK_RND's fill reads it: the 24 rows of +/// its rounds are a function of these alone. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KeccakRndOp { + pub timestamp: u64, + pub input: [u64; 25], +} + +const _: () = { + assert!(std::mem::size_of::() == 208); + assert!(std::mem::offset_of!(KeccakRndOp, input) == 8); +}; + +/// One cycle's executor log as CPU's fill reads it (the executor's `Log`, +/// field for field): the pcs before and after, and the values it read and +/// wrote. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CpuLog { + pub current_pc: u64, + pub next_pc: u64, + pub src1_val: u64, + pub src2_val: u64, + pub dst_val: u64, +} + +/// One MEMW_R row as its fill reads it (the prover's register row, field for +/// field): `address` the register index, `old_ts_lo` the old timestamp's low +/// word, `is_read` 0 or 1, `pad` 0. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RegisterRow { + pub timestamp: u64, + pub val0: u32, + pub val1: u32, + pub old0: u32, + pub old1: u32, + pub old_ts_lo: u32, + pub address: u16, + pub is_read: u8, + pub pad: u8, +} + +/// One MEMW_A row as its fill reads it (the prover's aligned row, field for +/// field): the base address, timestamp and shared old timestamp, the value +/// and old value packed a word each (eight bytes for a memory access, the two +/// 32-bit halves for a register access), the width, and flags 0 or 1. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AlignedRow { + pub base_address: u64, + pub timestamp: u64, + pub old_timestamp: u64, + pub value: u64, + pub old: u64, + pub width: u8, + pub is_read: u8, + pub is_register: u8, +} + +/// One LOAD op as its fill reads it (the prover's load op, field for field): +/// the base address, the timestamp, the width, whether it sign-extends (0 or +/// 1) and the result's bytes. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LoadOp { + pub base_address: u64, + pub timestamp: u64, + pub width: u8, + pub signed: u8, + pub res: [u8; 8], +} + +/// One STORE op as its fill reads it (the prover's store op, field for field): +/// the base address, the timestamp, the value, and the width flags (0 or 1). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StoreOp { + pub base_address: u64, + pub timestamp: u64, + pub value: u64, + pub write2: u8, + pub write4: u8, + pub write8: u8, +} + +/// One SHIFT op as its fill reads it (the prover's shift op, field for +/// field): the full shift amount, the input as four 16-bit halves, the +/// amount's low byte, and flags 0 or 1. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ShiftOp { + pub shift_amount: u64, + pub in_halves: [u16; 4], + pub shift: u8, + pub direction: u8, + pub signed: u8, + pub word_instr: u8, +} + +/// One MEMW op as its fill reads it (the prover's MEMW op, field for field): +/// the base address, the timestamp, the eight old timestamps, the eight value +/// and old elements, and the flags and width. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MemwOp { + pub base_address: u64, + pub timestamp: u64, + pub old_timestamp: [u64; 8], + pub value: [u32; 8], + pub old: [u32; 8], + pub is_register: u8, + pub width: u8, + pub is_read: u8, +} + +/// One DECODE row as CPU's fill reads it: the pc, the extended immediate, +/// then the decoded fields as bytes (flags 0 or 1). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DecodeRow { + pub pc: u64, + pub imm: u64, + pub read_register1: u8, + pub read_register2: u8, + pub write_register: u8, + pub word_instr: u8, + pub alu: u8, + pub add: u8, + pub sub: u8, + pub memory: u8, + pub branch: u8, + pub ecall: u8, + pub rs1: u8, + pub rs2: u8, + pub rd: u8, + pub half_instruction_length: u8, + pub alu_flags: u8, + pub mem_flags: u8, +} + +/// A run of pcs 4 apart: pc `first_pc + 4k` is DECODE row `row + k` for +/// `k < len`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DecodeRun { + pub first_pc: u64, + pub row: u32, + pub len: u32, +} + +const _: () = { + use std::mem::{offset_of, size_of}; + assert!(size_of::() == 40); + assert!(offset_of!(CpuLog, dst_val) == 32); + assert!(size_of::() == 32); + assert!(offset_of!(RegisterRow, old_ts_lo) == 24); + assert!(offset_of!(RegisterRow, address) == 28); + assert!(offset_of!(RegisterRow, is_read) == 30); + assert!(offset_of!(RegisterRow, pad) == 31); + assert!(size_of::() == 32); + assert!(offset_of!(DecodeRow, read_register1) == 16); + assert!(offset_of!(DecodeRow, mem_flags) == 31); + assert!(size_of::() == 16); + assert!(offset_of!(DecodeRun, len) == 12); + assert!(size_of::() == 48); + assert!(offset_of!(AlignedRow, width) == 40); + assert!(offset_of!(AlignedRow, is_register) == 42); + assert!(size_of::() == 32); + assert!(offset_of!(LoadOp, signed) == 17); + assert!(offset_of!(LoadOp, res) == 18); + assert!(size_of::() == 32); + assert!(offset_of!(StoreOp, write8) == 26); + assert!(size_of::() == 24); + assert!(offset_of!(ShiftOp, word_instr) == 19); + assert!(size_of::() == 152); + assert!(offset_of!(MemwOp, is_read) == 146); +}; + +/// A program's DECODE rows as CPU's fill reads them, shared by every CPU +/// table of a prove: the entries in pc order and the runs of pcs 4 apart. On +/// a card it is uploaded once, at the first fill that reads it, and freed with +/// it. +pub struct DecodeRecords { + entries: Vec, + runs: Vec, + #[cfg(feature = "cuda")] + card: std::sync::Mutex>>, +} + +impl DecodeRecords { + /// The rows and runs of a program. Refused (`None`) unless there is a row, + /// the runs ascend without overlapping and every row they name is one. + pub fn new(entries: Vec, runs: Vec) -> Option { + if entries.is_empty() || runs.is_empty() { + return None; + } + let mut end_pc = 0u64; + for (k, run) in runs.iter().enumerate() { + let last = run + .first_pc + .checked_add(4 * u64::from(run.len.checked_sub(1)?))?; + if (k > 0 && run.first_pc <= end_pc) + || run.row as usize + run.len as usize > entries.len() + { + return None; + } + end_pc = last; + } + Some(Self { + entries, + runs, + #[cfg(feature = "cuda")] + card: std::sync::Mutex::new(None), + }) + } + + pub fn entries(&self) -> &[DecodeRow] { + &self.entries + } + + pub fn runs(&self) -> &[DecodeRun] { + &self.runs + } + + /// The DECODE row of `pc`, as CPU's fill finds it: the last run starting + /// at or below `pc`, if it holds it. + pub fn row(&self, pc: u64) -> Option { + let at = self.runs.partition_point(|run| run.first_pc <= pc); + let run = self.runs.get(at.checked_sub(1)?)?; + let offset = pc - run.first_pc; + (offset.is_multiple_of(4) && offset / 4 < u64::from(run.len)) + .then(|| run.row + (offset / 4) as u32) + } + + /// The rows on the card: uploaded at the first call, under a promise of + /// their bytes, and kept until this goes. `None` when the card declines. + #[cfg(feature = "cuda")] + pub(crate) fn on_card(&self) -> Option> { + use math_cuda::fill::{CardDecode, DecodeEntryC, DecodeRunC}; + const _: () = { + use std::mem::{align_of, size_of}; + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!( + std::mem::offset_of!(DecodeRow, mem_flags) + == std::mem::offset_of!(DecodeEntryC, mem_flags) + ); + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(std::mem::offset_of!(DecodeRun, len) == std::mem::offset_of!(DecodeRunC, len)); + }; + let mut held = self.card.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(card) = held.as_ref() { + return Some(Arc::clone(card)); + } + // SAFETY: each pair is `#[repr(C)]` with the same fields in the same + // order, so the same size, alignment and offsets (asserted above). + let (entries, runs) = unsafe { + ( + std::slice::from_raw_parts( + self.entries.as_ptr() as *const DecodeEntryC, + self.entries.len(), + ), + std::slice::from_raw_parts( + self.runs.as_ptr() as *const DecodeRunC, + self.runs.len(), + ), + ) + }; + let card = Arc::new(CardDecode::resident(entries, runs).ok()?); + *held = Some(Arc::clone(&card)); + Some(card) + } +} + +impl std::fmt::Debug for DecodeRecords { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "DecodeRecords {{ {} rows, {} runs }}", + self.entries.len(), + self.runs.len() + ) + } +} + +impl PartialEq for DecodeRecords { + fn eq(&self, other: &Self) -> bool { + self.entries == other.entries && self.runs == other.runs + } +} + +impl Eq for DecodeRecords {} + +/// What a card fill reads: the family's records and where its table starts in +/// what they make. A table's records may lie in pieces (the window parts a +/// chunk was cut from), read one after another. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RecordsFill<'a> { + /// KECCAK_RND's rows `[skip, skip + rows)` of the table `ops` make, 24 + /// rows an op; rows past the last op are zero. + KeccakRnd { ops: &'a [KeccakRndOp], skip: usize }, + /// CPU's rows: one a log of the pieces in order, log `i` the cycle + /// `first_cycle + i`, read with its row of `decode`; then the padding rows. + Cpu { + logs: Vec<&'a [CpuLog]>, + first_cycle: u64, + decode: &'a DecodeRecords, + }, + /// MEMW_R's rows: the pieces in order, then zero rows. + MemwR { rows: Vec<&'a [RegisterRow]> }, + /// MEMW_A's rows: the pieces in order, then zero rows. + MemwA { rows: Vec<&'a [AlignedRow]> }, + /// LOAD's rows: one an op of the pieces in order, then zero rows. + Load { ops: Vec<&'a [LoadOp]> }, + /// STORE's rows: one an op of the pieces in order, then zero rows. + Store { ops: Vec<&'a [StoreOp]> }, + /// SHIFT's rows: one an op of the pieces in order, then padding rows. + Shift { ops: Vec<&'a [ShiftOp]> }, + /// MEMW's rows: one an op of the pieces in order, then zero rows. + Memw { ops: Vec<&'a [MemwOp]> }, +} + +impl RecordsFill<'_> { + /// The records' bytes, as they cross the bus (a CPU table's DECODE rows + /// aside: one copy serves every CPU table). + pub fn bytes(&self) -> usize { + match self { + Self::KeccakRnd { ops, .. } => std::mem::size_of_val(*ops), + Self::Cpu { logs, .. } => logs.iter().map(|p| std::mem::size_of_val(*p)).sum(), + Self::MemwR { rows } => rows.iter().map(|p| std::mem::size_of_val(*p)).sum(), + Self::MemwA { rows } => rows.iter().map(|p| std::mem::size_of_val(*p)).sum(), + Self::Load { ops } => ops.iter().map(|p| std::mem::size_of_val(*p)).sum(), + Self::Store { ops } => ops.iter().map(|p| std::mem::size_of_val(*p)).sum(), + Self::Shift { ops } => ops.iter().map(|p| std::mem::size_of_val(*p)).sum(), + Self::Memw { ops } => ops.iter().map(|p| std::mem::size_of_val(*p)).sum(), + } + } + + /// The records' count: ops, logs or rows. + pub fn len(&self) -> usize { + match self { + Self::KeccakRnd { ops, .. } => ops.len(), + Self::Cpu { logs, .. } => logs.iter().map(|p| p.len()).sum(), + Self::MemwR { rows } => rows.iter().map(|p| p.len()).sum(), + Self::MemwA { rows } => rows.iter().map(|p| p.len()).sum(), + Self::Load { ops } => ops.iter().map(|p| p.len()).sum(), + Self::Store { ops } => ops.iter().map(|p| p.len()).sum(), + Self::Shift { ops } => ops.iter().map(|p| p.len()).sum(), + Self::Memw { ops } => ops.iter().map(|p| p.len()).sum(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The family, where its table starts, and the records' words, in order + /// (whatever pieces they lie in): what [`Records::digest`] reads. A CPU + /// table's DECODE rows are the program's, the same for every table: left + /// out. + fn words(&self, mut each: impl FnMut(u64)) { + match self { + Self::KeccakRnd { ops, skip } => { + each(0); + each(*skip as u64); + for op in ops.iter() { + each(op.timestamp); + op.input.iter().copied().for_each(&mut each); + } + } + Self::Cpu { + logs, first_cycle, .. + } => { + each(1); + each(*first_cycle); + for log in logs.iter().flat_map(|piece| piece.iter()) { + each(log.current_pc); + each(log.next_pc); + each(log.src1_val); + each(log.src2_val); + each(log.dst_val); + } + } + Self::MemwR { rows } => { + each(2); + for row in rows.iter().flat_map(|piece| piece.iter()) { + each(row.timestamp); + each(u64::from(row.val0) | u64::from(row.val1) << 32); + each(u64::from(row.old0) | u64::from(row.old1) << 32); + each( + u64::from(row.old_ts_lo) + | u64::from(row.address) << 32 + | u64::from(row.is_read) << 48 + | u64::from(row.pad) << 56, + ); + } + } + // The fields alone: these records end in padding. + Self::MemwA { rows } => { + each(3); + for row in rows.iter().flat_map(|piece| piece.iter()) { + each(row.base_address); + each(row.timestamp); + each(row.old_timestamp); + each(row.value); + each(row.old); + each( + u64::from(row.width) + | u64::from(row.is_read) << 8 + | u64::from(row.is_register) << 16, + ); + } + } + Self::Load { ops } => { + each(4); + for op in ops.iter().flat_map(|piece| piece.iter()) { + each(op.base_address); + each(op.timestamp); + each(u64::from(op.width) | u64::from(op.signed) << 8); + each(u64::from_le_bytes(op.res)); + } + } + Self::Store { ops } => { + each(5); + for op in ops.iter().flat_map(|piece| piece.iter()) { + each(op.base_address); + each(op.timestamp); + each(op.value); + each( + u64::from(op.write2) + | u64::from(op.write4) << 8 + | u64::from(op.write8) << 16, + ); + } + } + Self::Shift { ops } => { + each(6); + for op in ops.iter().flat_map(|piece| piece.iter()) { + each(op.shift_amount); + each( + op.in_halves + .iter() + .enumerate() + .fold(0, |w, (i, &h)| w | u64::from(h) << (16 * i)), + ); + each( + u64::from(op.shift) + | u64::from(op.direction) << 8 + | u64::from(op.signed) << 16 + | u64::from(op.word_instr) << 24, + ); + } + } + Self::Memw { ops } => { + each(7); + for op in ops.iter().flat_map(|piece| piece.iter()) { + each(op.base_address); + each(op.timestamp); + op.old_timestamp.iter().for_each(|&t| each(t)); + for pair in [&op.value, &op.old] { + for two in pair.chunks_exact(2) { + each(u64::from(two[0]) | u64::from(two[1]) << 32); + } + } + each( + u64::from(op.is_register) + | u64::from(op.width) << 8 + | u64::from(op.is_read) << 16, + ); + } + } + } + } +} + +/// What a records holder handed back to the OS as it let its records go +/// ([`TableRecords::return_pages`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PagesReturned { + /// Buffers this holder was the last to hold: their pages handed back (or + /// tried, see `failed`). + pub buffers: usize, + /// Buffers another holder still holds: left as they are. + pub shared: usize, + /// Bytes of whole pages handed back. + pub bytes: u64, + /// Buffers whose pages the OS would not take back (logged once). + pub failed: usize, +} + +impl std::ops::AddAssign for PagesReturned { + fn add_assign(&mut self, other: Self) { + self.buffers += other.buffers; + self.shared += other.shared; + self.bytes += other.bytes; + self.failed += other.failed; + } +} + +/// A table's records: its shape, what the card's fill reads, and the host's +/// fill of the same columns. +pub trait TableRecords: Send + Sync + std::fmt::Debug { + fn rows(&self) -> usize; + fn cols(&self) -> usize; + /// The records as the card's fill reads them. + fn fill(&self) -> RecordsFill<'_>; + /// The records' bytes ([`RecordsFill::bytes`]). + fn bytes(&self) -> usize { + self.fill().bytes() + } + + /// The leading columns the fill writes from the row index alone, as a + /// table's preprocessed columns are: a table held as records is laid out + /// only for an AIR with that many preprocessed columns. + fn row_index_columns(&self) -> usize { + 0 + } + /// The columns the host fill writes from the records, packed: the words + /// the card's fill writes. Only a host reader asks. `None` where the host + /// cannot (the table then reads as no columns, a proof the verifier + /// refuses). + fn host_fill(&self) -> Option; + + /// Hands the pages of the buffers this holder is the last to hold back to + /// the OS, as its records are let go: nothing reads them after, their + /// holder drops them next ([`Records::return_pages`]). Buffers another + /// holder still holds stay as they are. The default returns nothing. + fn return_pages(&mut self) -> PagesReturned { + PagesReturned::default() + } +} + +/// A table's records, shared: two are equal when they fill the same table +/// from the same records. +#[derive(Clone, Debug)] +pub struct Records(pub Arc); + +impl Records { + pub fn rows(&self) -> usize { + self.0.rows() + } + + pub fn cols(&self) -> usize { + self.0.cols() + } + + pub fn fill(&self) -> RecordsFill<'_> { + self.0.fill() + } + + /// The bytes the records take. + pub fn bytes(&self) -> usize { + self.0.bytes() + } + + /// [`TableRecords::row_index_columns`]. + pub fn row_index_columns(&self) -> usize { + self.0.row_index_columns() + } + + /// A 128-bit digest of the table's shape and records: a fast, + /// non-cryptographic check that records made again are these (the + /// packed columns' digest, `stark::regen::digest_of`, does the same for a + /// packed table). + pub fn digest(&self) -> [u64; 2] { + digest_of(self.0.as_ref()) + } + + pub fn host_fill(&self) -> Option { + self.0.host_fill() + } + + /// Lets the records go, first handing their buffers' pages back to the + /// OS where this is the records' last handle + /// ([`TableRecords::return_pages`]); another handle still holding them + /// returns nothing. + pub fn return_pages(mut self) -> PagesReturned { + match Arc::get_mut(&mut self.0) { + Some(records) => records.return_pages(), + None => PagesReturned::default(), + } + } +} + +/// [`Records::digest`] of records not (yet) shared: a regenerator that +/// rebuilt a table's records digests them before it fills the table from +/// them. +pub fn digest_of(records: &dyn TableRecords) -> [u64; 2] { + const K: [u64; 4] = [ + 0x9e37_79b9_7f4a_7c15, + 0xc2b2_ae3d_27d4_eb4f, + 0x1656_67b1_9e37_79f9, + 0x85eb_ca77_c2b2_ae63, + ]; + let mut lanes = [ + records.rows() as u64, + records.cols() as u64, + records.bytes() as u64, + 0x243f_6a88_85a3_08d3, + ]; + let mut i = 0usize; + records.fill().words(|w| { + let l = i % 4; + lanes[l] = (lanes[l] ^ w).wrapping_mul(K[l]).rotate_left(31); + i += 1; + }); + // MurmurHash3's finalizer, so every input bit reaches every output bit. + let fmix = |mut h: u64| { + h ^= h >> 33; + h = h.wrapping_mul(0xff51_afd7_ed55_8ccd); + h ^= h >> 33; + h = h.wrapping_mul(0xc4ce_b9fe_1a85_ec53); + h ^ (h >> 33) + }; + [ + fmix(lanes[0] ^ lanes[2].rotate_left(23)), + fmix(lanes[1] ^ lanes[3].rotate_left(41)), + ] +} + +impl PartialEq for Records { + fn eq(&self, other: &Self) -> bool { + self.rows() == other.rows() && self.cols() == other.cols() && self.fill() == other.fill() + } +} + +impl Eq for Records {} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug)] + struct Ops(Vec, usize, usize); + + impl TableRecords for Ops { + fn rows(&self) -> usize { + self.2 + } + + fn cols(&self) -> usize { + 1480 + } + + fn fill(&self) -> RecordsFill<'_> { + RecordsFill::KeccakRnd { + ops: &self.0, + skip: self.1, + } + } + + fn host_fill(&self) -> Option { + None + } + } + + fn ops(n: u64) -> Vec { + (0..n) + .map(|k| KeccakRndOp { + timestamp: 4 * k + 8, + input: std::array::from_fn(|i| k * 100 + i as u64), + }) + .collect() + } + + fn records(ops: Vec, skip: usize, rows: usize) -> Records { + Records(Arc::new(Ops(ops, skip, rows))) + } + + /// Records are equal, and digest alike, when they fill the same table from + /// the same ops; one word, the cut or the height apart, they are not. + #[test] + fn records_digest_their_table_and_every_word() { + let base = records(ops(3), 8, 32); + assert_eq!(base, records(ops(3), 8, 32)); + assert_eq!(base.digest(), records(ops(3), 8, 32).digest()); + assert_eq!(base.bytes(), 3 * 208); + let mut lane = ops(3); + lane[2].input[24] ^= 1; + let mut stamp = ops(3); + stamp[0].timestamp += 4; + for (what, other) in [ + ("a lane", records(lane, 8, 32)), + ("a timestamp", records(stamp, 8, 32)), + ("the cut", records(ops(3), 9, 32)), + ("the height", records(ops(3), 8, 64)), + ("an op fewer", records(ops(2), 8, 32)), + ] { + assert_ne!(base, other, "{what}"); + assert_ne!(base.digest(), other.digest(), "{what}"); + } + } + + fn decode() -> DecodeRecords { + let row = |pc| DecodeRow { + pc, + ..DecodeRow::default() + }; + DecodeRecords::new( + vec![row(0x1000), row(0x1004), row(0x2000), row(0x2008)], + vec![ + DecodeRun { + first_pc: 0x1000, + row: 0, + len: 2, + }, + DecodeRun { + first_pc: 0x2000, + row: 2, + len: 1, + }, + DecodeRun { + first_pc: 0x2008, + row: 3, + len: 1, + }, + ], + ) + .expect("a decode table") + } + + fn logs(n: u64) -> Vec { + (0..n) + .map(|i| CpuLog { + current_pc: 0x1000 + 4 * (i % 2), + next_pc: 0x1004, + src1_val: i, + src2_val: i.wrapping_mul(0x9e37_79b9), + dst_val: !i, + }) + .collect() + } + + /// Every pc a run holds finds its row; the pcs between and around the + /// runs find none. + #[test] + fn a_pc_finds_the_row_of_its_run() { + let decode = decode(); + for (pc, row) in [(0x1000, 0), (0x1004, 1), (0x2000, 2), (0x2008, 3)] { + assert_eq!(decode.row(pc), Some(row), "pc {pc:#x}"); + } + for pc in [0, 0xffc, 0x1002, 0x1008, 0x2004, 0x200c, u64::MAX] { + assert_eq!(decode.row(pc), None, "pc {pc:#x}"); + } + } + + /// DECODE rows are held only as a table the fill can search: a row, runs + /// that ascend without overlapping, each holding a pc, every row named one + /// of the table's. + #[test] + fn decode_rows_that_are_no_table_are_refused() { + let ok = decode(); + let entries = ok.entries().to_vec(); + let runs = ok.runs().to_vec(); + assert!(DecodeRecords::new(entries.clone(), runs.clone()).is_some()); + assert!(DecodeRecords::new(Vec::new(), runs.clone()).is_none()); + assert!(DecodeRecords::new(entries.clone(), Vec::new()).is_none()); + let mut empty_run = runs.clone(); + empty_run[1].len = 0; + assert!(DecodeRecords::new(entries.clone(), empty_run).is_none()); + let mut overlapping = runs.clone(); + overlapping[1].first_pc = 0x1004; + assert!(DecodeRecords::new(entries.clone(), overlapping).is_none()); + let mut descending = runs.clone(); + descending.swap(0, 1); + assert!(DecodeRecords::new(entries.clone(), descending).is_none()); + let mut past = runs; + past[2].row = 4; + assert!(DecodeRecords::new(entries, past).is_none()); + } + + /// CPU, MEMW_R or a row family's records in pieces. + #[derive(Debug)] + enum Pieces { + Cpu(Vec>, u64, Arc), + MemwR(Vec>), + MemwA(Vec>), + Load(Vec>), + Store(Vec>), + Shift(Vec>), + Memw(Vec>), + } + + impl TableRecords for Pieces { + fn rows(&self) -> usize { + 16 + } + + fn cols(&self) -> usize { + match self { + Self::Cpu(..) => 38, + Self::MemwR(_) => 10, + Self::MemwA(_) => 29, + Self::Load(_) => 18, + Self::Store(_) => 16, + Self::Shift(_) => 29, + Self::Memw(_) => 49, + } + } + + fn fill(&self) -> RecordsFill<'_> { + match self { + Self::Cpu(pieces, first_cycle, decode) => RecordsFill::Cpu { + logs: pieces.iter().map(Vec::as_slice).collect(), + first_cycle: *first_cycle, + decode, + }, + Self::MemwR(pieces) => RecordsFill::MemwR { + rows: pieces.iter().map(Vec::as_slice).collect(), + }, + Self::MemwA(pieces) => RecordsFill::MemwA { + rows: pieces.iter().map(Vec::as_slice).collect(), + }, + Self::Load(pieces) => RecordsFill::Load { + ops: pieces.iter().map(Vec::as_slice).collect(), + }, + Self::Store(pieces) => RecordsFill::Store { + ops: pieces.iter().map(Vec::as_slice).collect(), + }, + Self::Shift(pieces) => RecordsFill::Shift { + ops: pieces.iter().map(Vec::as_slice).collect(), + }, + Self::Memw(pieces) => RecordsFill::Memw { + ops: pieces.iter().map(Vec::as_slice).collect(), + }, + } + } + + fn host_fill(&self) -> Option { + None + } + } + + /// `records` cut into pieces at `cuts`. + fn cut(records: &[T], cuts: &[usize]) -> Vec> { + let mut pieces = Vec::new(); + let mut from = 0; + for &to in cuts.iter().chain([&records.len()]) { + pieces.push(records[from..to].to_vec()); + from = to; + } + pieces + } + + /// ★ A CPU or MEMW_R table's digest reads its records' words in order, + /// whatever pieces they lie in, and where the table starts: one bit moved + /// in any field of any record, another first cycle, or a record fewer, + /// moves it. + #[test] + fn the_records_digest_reads_every_field_and_not_the_pieces() { + let decode = Arc::new(decode()); + let logs = logs(9); + let cpu = |pieces: Vec>, first_cycle: u64| { + Records(Arc::new(Pieces::Cpu( + pieces, + first_cycle, + Arc::clone(&decode), + ))) + .digest() + }; + let whole = cpu(cut(&logs, &[]), 7); + assert_eq!(cpu(cut(&logs, &[4]), 7), whole); + assert_eq!(cpu(cut(&logs, &[1, 1, 8]), 7), whole); + assert_ne!(cpu(cut(&logs, &[]), 8), whole, "the first cycle"); + assert_ne!(cpu(cut(&logs[..8], &[]), 7), whole, "a record fewer"); + for at in 0..logs.len() { + for field in 0..5 { + let mut moved = logs.clone(); + let log = &mut moved[at]; + let word = match field { + 0 => &mut log.current_pc, + 1 => &mut log.next_pc, + 2 => &mut log.src1_val, + 3 => &mut log.src2_val, + _ => &mut log.dst_val, + }; + *word ^= 1 << (at % 64); + assert_ne!(cpu(cut(&moved, &[3]), 7), whole, "log {at}, field {field}"); + } + } + let rows: Vec = (0..5u32) + .map(|i| RegisterRow { + timestamp: 4 * u64::from(i) + 8, + val0: i, + val1: i + 1, + old0: i + 2, + old1: i + 3, + old_ts_lo: i + 4, + address: i as u16, + is_read: (i % 2) as u8, + pad: 0, + }) + .collect(); + let memw_r = |pieces| Records(Arc::new(Pieces::MemwR(pieces))).digest(); + let whole = memw_r(cut(&rows, &[])); + assert_eq!(memw_r(cut(&rows, &[2])), whole); + for at in 0..rows.len() { + for field in 0..9 { + let mut moved = rows.clone(); + let row = &mut moved[at]; + match field { + 0 => row.timestamp ^= 1 << 40, + 1 => row.val0 ^= 1, + 2 => row.val1 ^= 1 << 31, + 3 => row.old0 ^= 1, + 4 => row.old1 ^= 1 << 31, + 5 => row.old_ts_lo ^= 1, + 6 => row.address ^= 1 << 15, + 7 => row.is_read ^= 1, + _ => row.pad ^= 1, + } + assert_ne!(memw_r(cut(&moved, &[])), whole, "row {at}, field {field}"); + } + } + } + + /// ★ A MEMW_A or LOAD table's digest reads every field of every row, in + /// order, whatever pieces they lie in: one bit moved in any field (each + /// result byte apart), or a row fewer, moves it. + #[test] + fn the_row_records_digest_reads_every_field_and_not_the_pieces() { + let aligned: Vec = (0..5u64) + .map(|i| AlignedRow { + base_address: 0x8000 + 8 * i, + timestamp: 4 * i + 8, + old_timestamp: 4 * i + 1, + value: i.wrapping_mul(0x9e37_79b9_7f4a_7c15), + old: !i, + width: 8, + is_read: (i % 2) as u8, + is_register: (i % 3 == 0) as u8, + }) + .collect(); + let memw_a = |pieces| Records(Arc::new(Pieces::MemwA(pieces))).digest(); + let whole = memw_a(cut(&aligned, &[])); + assert_eq!(memw_a(cut(&aligned, &[2])), whole); + assert_eq!(memw_a(cut(&aligned, &[0, 1, 5])), whole); + assert_ne!(memw_a(cut(&aligned[..4], &[])), whole, "a row fewer"); + for at in 0..aligned.len() { + for field in 0..8 { + let mut moved = aligned.clone(); + let row = &mut moved[at]; + match field { + 0 => row.base_address ^= 1 << 63, + 1 => row.timestamp ^= 1, + 2 => row.old_timestamp ^= 1 << 32, + 3 => row.value ^= 1 << (at % 64), + 4 => row.old ^= 1 << 63, + 5 => row.width ^= 0x80, + 6 => row.is_read ^= 1, + _ => row.is_register ^= 1, + } + assert_ne!(memw_a(cut(&moved, &[3])), whole, "row {at}, field {field}"); + } + } + let ops: Vec = (0..5u64) + .map(|i| LoadOp { + base_address: 0x9000 + i, + timestamp: 4 * i + 8, + width: 1 << (i % 4), + signed: (i % 2) as u8, + res: (i * 0x0102_0304_0506_0708).to_le_bytes(), + }) + .collect(); + let load = |pieces| Records(Arc::new(Pieces::Load(pieces))).digest(); + let whole = load(cut(&ops, &[])); + assert_eq!(load(cut(&ops, &[4])), whole); + assert_ne!(load(cut(&ops[1..], &[])), whole, "an op fewer"); + for at in 0..ops.len() { + for field in 0..12 { + let mut moved = ops.clone(); + let op = &mut moved[at]; + match field { + 0 => op.base_address ^= 1 << 40, + 1 => op.timestamp ^= 1 << 63, + 2 => op.width ^= 0x80, + 3 => op.signed ^= 1, + byte => op.res[byte - 4] ^= 1 << (byte % 8), + } + assert_ne!(load(cut(&moved, &[1])), whole, "op {at}, field {field}"); + } + } + } + + /// ★ A STORE, SHIFT or MEMW table's digest reads every field of every op, in + /// order, whatever pieces they lie in: one bit moved in any field (each + /// flag, half, element and old timestamp apart), or an op fewer, moves it. + #[test] + fn the_op_records_digest_reads_every_field_and_not_the_pieces() { + let mut rng = 0x5EED_u64; + let mut next = move || { + rng = rng + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + rng ^ rng >> 29 + }; + // The digest of `ops` is the same however they are cut, and moves with + // an op fewer or any one field of any op flipped (`flip` flips field + // `k`, and says `false` past the last). + fn each_field( + ops: &[T], + digest: impl Fn(Vec>) -> [u64; 2], + flip: impl Fn(&mut T, usize) -> bool, + what: &str, + ) { + let whole = digest(cut(ops, &[])); + assert_eq!(digest(cut(ops, &[1])), whole, "{what}: the pieces"); + assert_eq!( + digest(cut(ops, &[0, 2, ops.len()])), + whole, + "{what}: the pieces" + ); + assert_ne!(digest(cut(&ops[1..], &[])), whole, "{what}: an op fewer"); + for at in 0..ops.len() { + let mut k = 0; + loop { + let mut moved = ops.to_vec(); + if !flip(&mut moved[at], k) { + break; + } + assert_ne!( + digest(cut(&moved, &[2])), + whole, + "{what}: op {at}, field {k}" + ); + k += 1; + } + assert!(k > 3, "{what}: {k} fields"); + } + } + let stores: Vec = (0..4u8) + .map(|i| StoreOp { + base_address: next(), + timestamp: next(), + value: next(), + write2: u8::from(i == 1), + write4: u8::from(i == 2), + write8: u8::from(i == 3), + }) + .collect(); + each_field( + &stores, + |p| Records(Arc::new(Pieces::Store(p))).digest(), + |op, k| { + match k { + 0 => op.base_address ^= 1 << 63, + 1 => op.timestamp ^= 1, + 2 => op.value ^= 1 << 40, + 3 => op.write2 ^= 1, + 4 => op.write4 ^= 1, + 5 => op.write8 ^= 1, + _ => return false, + } + true + }, + "STORE", + ); + let shifts: Vec = (0..4u8) + .map(|i| ShiftOp { + shift_amount: next(), + in_halves: [next() as u16, next() as u16, next() as u16, next() as u16], + shift: next() as u8, + direction: i & 1, + signed: (i >> 1) & 1, + word_instr: u8::from(i == 3), + }) + .collect(); + each_field( + &shifts, + |p| Records(Arc::new(Pieces::Shift(p))).digest(), + |op, k| { + match k { + 0 => op.shift_amount ^= 1 << 33, + 1..=4 => op.in_halves[k - 1] ^= 1 << 15, + 5 => op.shift ^= 0x80, + 6 => op.direction ^= 1, + 7 => op.signed ^= 1, + 8 => op.word_instr ^= 1, + _ => return false, + } + true + }, + "SHIFT", + ); + let memws: Vec = (0..4u8) + .map(|i| MemwOp { + base_address: next(), + timestamp: next(), + old_timestamp: std::array::from_fn(|_| next()), + value: std::array::from_fn(|_| next() as u32), + old: std::array::from_fn(|_| next() as u32), + is_register: i & 1, + width: 1 << i, + is_read: (i >> 1) & 1, + }) + .collect(); + each_field( + &memws, + |p| Records(Arc::new(Pieces::Memw(p))).digest(), + |op, k| { + match k { + 0 => op.base_address ^= 1, + 1 => op.timestamp ^= 1 << 63, + 2..=9 => op.old_timestamp[k - 2] ^= 1 << (k * 7), + 10..=17 => op.value[k - 10] ^= 1 << 31, + 18..=25 => op.old[k - 18] ^= 1, + 26 => op.is_register ^= 1, + 27 => op.width ^= 0x80, + 28 => op.is_read ^= 1, + _ => return false, + } + true + }, + "MEMW", + ); + } + + /// A fill's bytes and count are its pieces' summed. (Two fills compare + /// equal only when their pieces lie alike; their digests match however + /// the pieces lie.) + #[test] + fn a_fill_is_its_pieces_in_order() { + let decode = decode(); + let logs = logs(6); + let split = RecordsFill::Cpu { + logs: vec![&logs[..2], &logs[2..]], + first_cycle: 0, + decode: &decode, + }; + assert_eq!((split.len(), split.bytes()), (6, 240)); + let rows = [RegisterRow::default(); 3]; + let memw_r = RecordsFill::MemwR { + rows: vec![&rows[..1], &rows[1..]], + }; + assert_eq!((memw_r.len(), memw_r.bytes()), (3, 96)); + assert!(!memw_r.is_empty()); + assert!(RecordsFill::MemwR { rows: Vec::new() }.is_empty()); + let aligned = [AlignedRow::default(); 3]; + let memw_a = RecordsFill::MemwA { + rows: vec![&aligned[..2], &aligned[2..]], + }; + assert_eq!((memw_a.len(), memw_a.bytes()), (3, 144)); + let ops = [LoadOp::default(); 5]; + let load = RecordsFill::Load { + ops: vec![&ops[..1], &ops[1..3], &ops[3..]], + }; + assert_eq!((load.len(), load.bytes()), (5, 160)); + } +} diff --git a/crypto/multilinear/src/round_slot.rs b/crypto/multilinear/src/round_slot.rs new file mode 100644 index 000000000..06a8b46f2 --- /dev/null +++ b/crypto/multilinear/src/round_slot.rs @@ -0,0 +1,187 @@ +//! The argue's round trips made cheap (`LAMBDA_VM_ARGUE_ROUND_SLOT`). +//! +//! On — the default; `0` keeps today's round trips — a GKR layer's rounds on +//! the card take their sums from a slot of pinned host memory the round +//! kernel's last block writes (`math_cuda::round_slot`): no second launch to +//! reduce the block partials, no pageable read-back, a host spin on a sequence +//! word instead of a stream wait. And a bin's trees are built without waiting +//! for each one's output fraction: the outputs are read when the ladder +//! absorbs them. A table's fused zerocheck keeps today's round trips: through +//! a slot its launches ran slower on whole blocks. +//! +//! The sums are the limbs today's second launch writes, reduced by +//! `ext3_from_raw` as before, so the transcript and the proof bytes do not +//! move. `check` also runs today's second launch and read-back every round, +//! compares the two limb for limb, counts a difference and uses today's. + +use std::sync::atomic::{AtomicU8, Ordering}; + +/// How a round's sums come back. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RoundSlotSetting { + /// Today's second launch and read-back. + Off, + /// Through a slot. The default. + On, + /// Through a slot, and today's beside it, compared. + Check, +} + +/// The setting `LAMBDA_VM_ARGUE_ROUND_SLOT` names: `0` off, unset, empty or +/// `1` on, `check` on and compared. Anything else is on, said once — `wait`, +/// the next round queued behind a card-side wait, is not built. +pub fn round_slot_setting(value: Option<&str>) -> RoundSlotSetting { + match value { + Some("0") => RoundSlotSetting::Off, + None | Some("") | Some("1") => RoundSlotSetting::On, + Some("check") => RoundSlotSetting::Check, + Some(other) => { + eprintln!( + "[argue] LAMBDA_VM_ARGUE_ROUND_SLOT={other} is not 0, 1 or check; reading the GKR rounds' \ + sums from a slot (the default)" + ); + RoundSlotSetting::On + } + } +} + +static FORCED: AtomicU8 = AtomicU8::new(0); + +/// The setting in effect: a test's override, else the environment's, read +/// once with a banner. +pub fn argue_round_slot() -> RoundSlotSetting { + match FORCED.load(Ordering::Relaxed) { + 1 => RoundSlotSetting::Off, + 2 => RoundSlotSetting::On, + 3 => RoundSlotSetting::Check, + _ => { + static SET: std::sync::OnceLock = std::sync::OnceLock::new(); + *SET.get_or_init(|| { + let set = round_slot_setting( + std::env::var("LAMBDA_VM_ARGUE_ROUND_SLOT").ok().as_deref(), + ); + eprintln!( + "★ ARGUE ROUND SLOT: {}", + match set { + RoundSlotSetting::Off => { + "off (LAMBDA_VM_ARGUE_ROUND_SLOT=0: today's second launch and read-back every round)" + } + RoundSlotSetting::On => { + "on (the default; the GKR rounds' sums from a slot the round kernel's last block \ + writes, a bin's trees built without a wait each; =0 restores today's, =check \ + compares them)" + } + RoundSlotSetting::Check => { + "check (the rounds' sums from a slot and today's read-back, compared every round)" + } + } + ); + set + }) + } + } +} + +/// Overrides `LAMBDA_VM_ARGUE_ROUND_SLOT` for the whole process — for a test +/// that proves every way in one binary. `None` restores the environment's. +#[doc(hidden)] +pub fn force_argue_round_slot(setting: Option) { + FORCED.store( + match setting { + None => 0, + Some(RoundSlotSetting::Off) => 1, + Some(RoundSlotSetting::On) => 2, + Some(RoundSlotSetting::Check) => 3, + }, + Ordering::Relaxed, + ); +} + +/// The setting in effect, for a harness's line. +pub fn round_slot_in_effect() -> &'static str { + match argue_round_slot() { + RoundSlotSetting::Off => "off", + RoundSlotSetting::On => "on", + RoundSlotSetting::Check => "check", + } +} + +/// The setting as the card's sessions take it. +#[cfg(feature = "cuda")] +pub(crate) fn device_setting() -> math_cuda::round_slot::RoundSlot { + match argue_round_slot() { + RoundSlotSetting::Off => math_cuda::round_slot::RoundSlot::Off, + RoundSlotSetting::On => math_cuda::round_slot::RoundSlot::On, + RoundSlotSetting::Check => math_cuda::round_slot::RoundSlot::Check, + } +} + +/// What the slots did this process, for a harness's line after its label. +pub fn counts_line() -> String { + #[cfg(feature = "cuda")] + let c = math_cuda::round_slot::counts(); + #[cfg(feature = "cuda")] + let v = [ + c.slot_rounds, + c.fall_backs, + c.declines, + c.second_launches, + c.checked, + c.mismatches, + ]; + #[cfg(not(feature = "cuda"))] + let v = [0u64; 6]; + format_counts(v) +} + +fn format_counts(v: [u64; 6]) -> String { + format!( + "rounds through a slot {} · stream-wait fall-backs {} · declines {} · second launches {} · check \ + compared {} · mismatches {}", + v[0], v[1], v[2], v[3], v[4], v[5] + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_knob_reads_its_variable() { + assert_eq!(round_slot_setting(None), RoundSlotSetting::On); + assert_eq!(round_slot_setting(Some("")), RoundSlotSetting::On); + assert_eq!(round_slot_setting(Some("0")), RoundSlotSetting::Off); + assert_eq!(round_slot_setting(Some("1")), RoundSlotSetting::On); + assert_eq!(round_slot_setting(Some("check")), RoundSlotSetting::Check); + assert_eq!( + round_slot_setting(Some("wait")), + RoundSlotSetting::On, + "the queued round is not built" + ); + assert_eq!(round_slot_setting(Some("on")), RoundSlotSetting::On); + } + + /// ★ The default reads the GKR rounds' sums from a slot — the same proof + /// bytes, which the card tests show — and only `0` keeps today's round + /// trips. + #[test] + fn the_knob_is_on_unless_its_variable_is_zero() { + assert_eq!( + argue_round_slot(), + round_slot_setting(std::env::var("LAMBDA_VM_ARGUE_ROUND_SLOT").ok().as_deref()) + ); + if std::env::var_os("LAMBDA_VM_ARGUE_ROUND_SLOT").is_none() { + assert_eq!(argue_round_slot(), RoundSlotSetting::On); + assert_eq!(round_slot_in_effect(), "on"); + } + } + + #[test] + fn the_counts_name_every_field_on_their_line() { + assert_eq!( + format_counts([1, 2, 3, 4, 5, 6]), + "rounds through a slot 1 · stream-wait fall-backs 2 · declines 3 · second launches 4 · check \ + compared 5 · mismatches 6" + ); + } +} diff --git a/crypto/multilinear/src/selector.rs b/crypto/multilinear/src/selector.rs new file mode 100644 index 000000000..870050935 --- /dev/null +++ b/crypto/multilinear/src/selector.rs @@ -0,0 +1,210 @@ +//! Row selectors: the indicator of `index(x) < 2^n - end_exemptions`. +//! +//! A transition constraint reading the next step cannot hold on the last one. +//! Multiplying by a selector costs exactly one degree. + +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{Error, mle::Mle}; + +/// The indicator of `index(x) < 2^n − end_exemptions`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Selector { + pub end_exemptions: usize, +} + +impl Selector { + /// Applies on every step. + pub const ALL: Selector = Selector { end_exemptions: 0 }; + + pub const fn except_last(k: usize) -> Selector { + Selector { end_exemptions: k } + } + + /// True when this selector is the constant one and can be skipped. + pub fn is_trivial(&self) -> bool { + self.end_exemptions == 0 + } + + /// Number of steps the constraint applies to. + pub fn active_steps(&self, num_vars: usize) -> usize { + (1usize << num_vars).saturating_sub(self.end_exemptions) + } + + /// The selector's hypercube table. + pub fn table(&self, num_vars: usize) -> Result, Error> { + let size = 1usize << num_vars; + if self.end_exemptions > size { + return Err(Error::TooManyExemptions { + exemptions: self.end_exemptions, + size, + }); + } + let cutoff = size - self.end_exemptions; + let evals = (0..size) + .map(|i| { + if i < cutoff { + FieldElement::::one() + } else { + FieldElement::::zero() + } + }) + .collect(); + Mle::new(evals) + } + + /// The selector at an arbitrary point, in `O(num_vars)`. + /// + /// `s(x) = 1 − geq(x, cutoff)`, where `geq` is the multilinear indicator of + /// `index(x) >= cutoff`: either `x` matches `cutoff` bit for bit, or they + /// first differ at a position where `cutoff` has a zero and `x` a one. + pub fn evaluate( + &self, + point: &[FieldElement], + ) -> Result, Error> { + let num_vars = point.len(); + let size = 1usize << num_vars; + if self.end_exemptions > size { + return Err(Error::TooManyExemptions { + exemptions: self.end_exemptions, + size, + }); + } + if self.is_trivial() { + return Ok(FieldElement::one()); + } + let cutoff = size - self.end_exemptions; + if cutoff == 0 { + // Every step is exempt: the constraint applies nowhere. + return Ok(FieldElement::zero()); + } + + let one = FieldElement::::one(); + // Running product of "x agrees with cutoff on every earlier bit". + let mut prefix = one.clone(); + let mut geq = FieldElement::::zero(); + + for (i, x_i) in point.iter().enumerate() { + // Variable 0 is the most significant bit. + let bit = (cutoff >> (num_vars - 1 - i)) & 1; + if bit == 0 { + // x exceeds cutoff here: everything above matched, x_i = 1. + geq += &prefix * x_i; + prefix *= &one - x_i; + } else { + prefix *= x_i; + } + } + // The remaining prefix is the "x == cutoff" case. + Ok(one - (geq + prefix)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn corner(index: usize, num_vars: usize) -> Vec { + (0..num_vars) + .map(|i| FE::from(((index >> (num_vars - 1 - i)) & 1) as u64)) + .collect() + } + + #[test] + fn all_is_the_constant_one() { + let s = Selector::ALL; + assert!(s.is_trivial()); + let table = s.table::(3).unwrap(); + assert!(table.evals().iter().all(|v| *v == FE::one())); + assert_eq!(s.evaluate(&corner(5, 3)).unwrap(), FE::one()); + } + + #[test] + fn except_last_one_masks_only_the_final_step() { + let s = Selector::except_last(1); + let table = s.table::(3).unwrap(); + for i in 0..8 { + let expected = if i < 7 { FE::one() } else { FE::zero() }; + assert_eq!(table.evals()[i], expected, "step {i}"); + } + } + + #[test] + fn except_last_two_masks_the_final_pair() { + let s = Selector::except_last(2); + let table = s.table::(4).unwrap(); + for i in 0..16 { + let expected = if i < 14 { FE::one() } else { FE::zero() }; + assert_eq!(table.evals()[i], expected, "step {i}"); + } + } + + #[test] + fn closed_form_matches_the_table_on_every_corner() { + for num_vars in 1..=5usize { + let size = 1usize << num_vars; + for k in 0..=size { + let s = Selector::except_last(k); + let table = s.table::(num_vars).unwrap(); + for i in 0..size { + assert_eq!( + s.evaluate(&corner(i, num_vars)).unwrap(), + table.evals()[i], + "num_vars={num_vars}, k={k}, step={i}" + ); + } + } + } + } + + #[test] + fn closed_form_is_the_multilinear_extension_off_the_cube() { + // The verifier evaluates the closed form at a random point; it must be + // the same polynomial the prover's table extends to. + for k in [1usize, 2, 3, 5] { + let s = Selector::except_last(k); + let table = s.table::(4).unwrap(); + let point = vec![FE::from(9), FE::from(17), FE::from(3), FE::from(41)]; + assert_eq!( + s.evaluate(&point).unwrap(), + table.evaluate(&point).unwrap(), + "k={k}" + ); + } + } + + #[test] + fn exempting_everything_selects_nothing() { + let s = Selector::except_last(8); + let table = s.table::(3).unwrap(); + assert!(table.evals().iter().all(|v| *v == FE::zero())); + assert_eq!(s.evaluate(&corner(0, 3)).unwrap(), FE::zero()); + assert_eq!(s.evaluate(&[FE::from(7); 3]).unwrap(), FE::zero()); + } + + #[test] + fn active_step_count_matches_the_table() { + for k in 0..=8usize { + let s = Selector::except_last(k); + let table = s.table::(3).unwrap(); + let ones = table.evals().iter().filter(|v| **v == FE::one()).count(); + assert_eq!(ones, s.active_steps(3), "k={k}"); + } + } + + #[test] + fn more_exemptions_than_steps_is_an_error() { + let s = Selector::except_last(9); + assert_eq!( + s.table::(3).unwrap_err(), + Error::TooManyExemptions { + exemptions: 9, + size: 8 + } + ); + assert!(s.evaluate(&corner(0, 3)).is_err()); + } +} diff --git a/crypto/multilinear/src/stacked_eval.rs b/crypto/multilinear/src/stacked_eval.rs new file mode 100644 index 000000000..3ecde3185 --- /dev/null +++ b/crypto/multilinear/src/stacked_eval.rs @@ -0,0 +1,2207 @@ +//! Settling many column claims against **few** commitments. +//! +//! [`stacking`](crate::stacking) puts each column in a subcube of a shared +//! polynomial, so `column(z) = stacked(prefix ‖ z)` with no protocol. What is +//! left is a multi-point claim on one committed polynomial, and one weighted +//! sumcheck settles it: +//! +//! ```text +//! Σ_i gamma^i·column_i(z) = Σ_x [Σ_i gamma^i·eq(prefix_i ‖ z, x)]·stacked(x) +//! ``` +//! +//! The columns of one table share a height and — once the claim reduction has +//! run — a point, so the weight **factors**: a slot table over the prefix bits +//! times `eq(z, ·)` over the column's own variables. Building it is then one +//! pass over the stacked cube instead of one per column, which is what makes +//! this cheap enough to be worth doing. +//! +//! The claim is settled by [`whir_chain`](crate::whir_chain), so the stack pays +//! **one** round schedule and query set for the whole trace — where a +//! per-column path would pay one each. That is where stacking earns its keep; +//! with a single-round WHIR it only changed the shape. +//! +//! The stacked polynomials stay in the **base field**: a trace is base-field, +//! and nothing extension-valued touches it until the first fold. The claims and +//! the weight are in the extension, which is what the mixed evaluation is for. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{ + Error, challenge_powers, + eq::{eq_eval, eq_evals_into}, + mle::Mle, + narrow::HostColumn, + stacking::{Placement, StackedLayout}, + whir::Domain, + whir_chain::{ + self, ChainConfig, ChainProof, GroupFold, GroupedShape, MAX_INDEX_POLYS, + check_index_domain, eq_index, index_vars, + }, + whir_commit::{CodewordCommitment, Commitment}, + whir_hash::WhirHash, +}; + +/// The chains a stack of `num_polys` polynomials is opened by, each a range of +/// its polynomials: one each under [`GroupFold::PerPoly`]; under +/// [`GroupFold::Indexed`], consecutive runs of [`MAX_INDEX_POLYS`], the +/// remainder last (38 → nine runs of 4 and one of 2). +/// +/// ★ THE ONE SPLIT (REV-B2-JUDGE B1-1). A pure function of the verifier's own +/// layout and config, so the prover, the host verifier (groups and prepared +/// stacks alike) and the recursion's plan and emitter all cut the same chains, +/// and no chain binds more than `MAX_INDEX_VARS` index variables however wide +/// a statement makes a group: a single table stated at 2^27 rows stacks into +/// one polynomial per column (REV-B2-A §3), and a refusal would make that +/// honest block unprovable where the per-polynomial format proves it. +pub fn index_chains(num_polys: usize, group: GroupFold) -> Vec> { + let run = match group { + GroupFold::PerPoly => 1, + GroupFold::Indexed => MAX_INDEX_POLYS, + }; + (0..num_polys) + .step_by(run) + .map(|start| start..(start + run).min(num_polys)) + .collect() +} + +/// [`index_chains`] for `layout` under `config`, each grouped run's virtual +/// domain checked to exist ([`check_index_domain`]): what both sides run +/// before anything is opened or read. +fn layout_chains( + layout: &StackedLayout, + config: &ChainConfig, +) -> Result>, Error> { + let chains = index_chains(layout.num_polys(), config.format.group); + for run in chains.iter().filter(|run| run.len() >= 2) { + check_index_domain( + layout.n_stack(), + index_vars(run.len()), + config.log_blowup, + F::TWO_ADICITY as usize, + )?; + } + Ok(chains) +} + +/// The stacked polynomials, committed. Base-field, like the trace they hold. +pub struct StackedCommitment +where + FieldElement: AsBytes + Sync + Send, +{ + layout: StackedLayout, + commitments: Vec>, + domain: Domain, + /// The room the commits and the openings take turns with, promised once + /// for the whole group. + room: Room, +} + +/// What a group has promised the card for the working set its commits and its +/// openings take turns with. +enum Room { + /// Nothing to take back: each codeword promised its own working set, + /// because the card would not promise the group's when it committed — or + /// there is no card. + Own, + /// The group's room, held from its first commit to its last opening, across + /// the argument between them (`LAMBDA_VM_NO_WHIR_ROOM_PARK`). Never read: + /// holding it is the promise. + Held(#[allow(dead_code)] crate::gpu::DeviceRoom), + /// Given back when the commits ended; the openings take it again + /// ([`crate::gpu::take_turn`]). + Parked, +} + +/// Which of a group's turns a room is promised for. +#[derive(Clone, Copy)] +enum Turn { + Commit, + Open, + /// Held from the first commit to the last opening: the larger of the two. + Both, +} + +/// What a group's room promises for `turn`: the working set the kernels that +/// run it allocate beside the codewords ([`crate::gpu::room_turns_for`]), or +/// one base codeword whatever the turn under `LAMBDA_VM_NO_WHIR_ROOM_RESIZE`. +fn room_bytes(layout: &StackedLayout, config: &ChainConfig, staged: bool, turn: Turn) -> u64 { + let resize = crate::gpu::room_resize(); + crate::gpu::note_room_sized(resize); + if !resize { + return (1u64 << (layout.n_stack() + config.log_blowup)) * 8; + } + let turns = crate::gpu::room_turns_for(layout, config, staged); + match turn { + Turn::Commit => turns.commit, + Turn::Open => turns.open, + Turn::Both => turns.commit.max(turns.open), + } +} + +/// Where a group's columns sit in a resident store. +/// +/// `From(first)` is the contiguous run every group has always been — column `c` +/// of the group at store column `first + c`. `Map(store)` names each column's +/// store position, for a group whose columns are NOT one run of the store: a +/// stack that leaves each table's leading prefix out (the W-LFM's policy B), +/// while the store keeps every table's columns contiguous for its argument. +#[derive(Clone, Copy, Debug)] +pub enum ColumnsAt<'a> { + From(usize), + Map(&'a [usize]), +} + +impl ColumnsAt<'_> { + /// The store column holding the group's column `column`. + pub fn store_column(&self, column: usize) -> usize { + match self { + Self::From(first) => first + column, + Self::Map(map) => map[column], + } + } +} + +std::thread_local! { + /// Whether this thread's commits run each pair on threads of their own + /// ([`with_pairs_off_pool`]). + static PAIRS_OFF_POOL: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Runs `f` with this thread's [`StackedCommitment`] commits taking each pair +/// on this thread and one of its own (`on`), or on the global rayon pool, as +/// they always have (`off`). The commits, and so the roots, are the same. +/// +/// A caller outside the pool hands its pair to the pool's injector, which a +/// worker reads only once its own deque and every other worker's are empty: +/// beside a long parallel phase (the block's finish) the card waits for that. +/// A pair on its own threads waits for nothing but the card. +pub fn with_pairs_off_pool(on: bool, f: impl FnOnce() -> R) -> R { + struct Restore(bool); + impl Drop for Restore { + fn drop(&mut self) { + PAIRS_OFF_POOL.with(|c| c.set(self.0)); + } + } + let _restore = Restore(PAIRS_OFF_POOL.with(|c| c.replace(on))); + f() +} + +/// Commits `pair` (one or two polynomials) with `commit`, the second on a +/// thread of its own; in order, as the pool's `collect` would. +#[cfg(feature = "parallel")] +fn commit_pair_off_pool( + pair: &[P], + commit: &(dyn Fn(&P) -> Result + Sync), +) -> Result, Error> { + std::thread::scope(|scope| { + let second = pair.get(1).map(|p| scope.spawn(move || commit(p))); + let first = pair.first().map(commit); + let second = second.map(|h| { + h.join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }); + first.into_iter().chain(second).collect() + }) +} + +impl StackedCommitment +where + FieldElement: AsBytes + Sync + Send, +{ + /// Packs `columns` into `layout` and commits each stacked polynomial. + /// + /// Every stacked polynomial has `n_stack` variables, so they share one + /// evaluation domain. Takes the columns as they already are — copying them + /// into `Vec`s first would be one more resident copy of the whole trace. + pub fn commit>( + layout: StackedLayout, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, usize)>, + config: &ChainConfig, + ) -> Result { + Self::commit_mapped( + layout, + columns, + resident.map(|(store, first)| (store, ColumnsAt::From(first))), + config, + ) + } + + /// [`Self::commit`] with the columns' store positions named by a + /// [`ColumnsAt`] rather than one contiguous run. + pub fn commit_mapped>( + layout: StackedLayout, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, ColumnsAt<'_>)>, + config: &ChainConfig, + ) -> Result { + Self::commit_signalled(layout, columns, resident, config, &|| {}) + } + + /// [`Self::commit_mapped`], calling `on_room` once the group's room has + /// been asked of the card (promised or not) and before the first commit: + /// a caller that uploads beside the commit waits for it, so its upload + /// never takes the bytes the commits were about to be promised. + pub fn commit_signalled>( + layout: StackedLayout, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, ColumnsAt<'_>)>, + config: &ChainConfig, + on_room: &dyn Fn(), + ) -> Result { + if let Some((_, ColumnsAt::Map(map))) = resident + && map.len() != columns.len() + { + return Err(Error::QueryCountMismatch { + expected: columns.len(), + got: map.len(), + }); + } + // The stacked polynomials are not built here: each is its columns at + // their offsets, and both the commit and the opening write those + // straight into the device's buffer. Assembling a copy on the way is a + // pass over every byte about to be uploaded. + let sources: Vec> = (0..layout.num_polys()) + .map(|poly| whir_chain::Stacked { + parts: layout + .parts_of(poly) + .into_iter() + .map(|(column, offset)| (columns[column] as &dyn HostColumn, offset)) + .collect(), + resident: resident.map(|(store, at)| { + ( + store, + layout + .parts_of(poly) + .into_iter() + .map(|(column, offset)| (at.store_column(column), offset)) + .collect(), + ) + }), + num_vars: layout.n_stack(), + }) + .collect(); + // The commits and the openings take turns with the same working set — + // two commits in flight, then one opening at a time — so the group + // promises one turn's worth rather than every polynomial promising its + // own. For eight of them that is nine codewords of room instead of + // sixteen, and what the difference buys is the widest tables getting a + // device at all. If the card will not promise it, each commitment + // promises its own, which is the conservative accounting. + // + // ★ AND WHAT EACH COMMITMENT KEEPS OF ITS TREE IS IN THIS NUMBER, as a + // named term rather than as a surprise. A commitment that is opened + // keeps its whole tree by default — `(2·num_leaves − 1) × 32` ≈ + // `C · 2^(6−k)` bytes at fold width `k`, HALF a base codeword at the + // production `k = 4` — or, under `LFM_WHIR_WHOLE_TREES=0`, its leaf + // layer alone, `C · 2^(5−k)`, a quarter. So a group of `n` commitments + // holds up to `n · C/2` more than the codewords alone. It is not + // reserved here, because the width is not known until the tree is + // built: each codeword grows THIS reservation when it captures + // (`DeviceCodeword::capture_leaves`), and `grow` refuses without + // changing anything when the budget will not take it. A refused + // retention costs the leaf pass again and nothing else. + // + // ⚠ NOTHING of a tree may be held OUTSIDE this number. H4 kept whole + // node arrays outside any promise, and the card reached 96%, after + // which commits fell back to the host at ~1.5 GiB each. What is kept + // now is grown into the room and handed to the evictor for any request + // it can cover, so a request gets the budget it would get with nothing + // kept, and the worst case is the old rebuild. + // + // ★ HOW MUCH: the turn it is held for (`room_bytes`) — the commits' + // when it is given back before the openings, which then take their + // own, and the larger of the two when it is held throughout. + let park = crate::gpu::room_park(); + let room = if sources.is_empty() { + None + } else { + let turn = if park { Turn::Commit } else { Turn::Both }; + let room = + crate::gpu::reserve_room(room_bytes(&layout, config, resident.is_none(), turn)); + if room.is_none() && crate::gpu::reserve_budget() > 0 { + crate::gpu::note_commit_room_refused(); + } + room + }; + on_room(); + let transient = room.is_none(); + // A commit spends most of its wall time waiting on a device — the tree + // coming back — with the next polynomial's transform not yet launched. + // Committing in pairs covers that wait and no more: a third in flight + // has nothing left to hide behind, and each one holds a codeword and a + // tree on the device while it runs. Every polynomial has the same + // variable count, so they share a domain. + let commit = |poly: &whir_chain::Stacked<'_, F>| { + whir_chain::commit_stacked::(poly, config, transient) + }; + let mut domain = None; + let mut commitments = Vec::with_capacity(sources.len()); + #[cfg(feature = "parallel")] + let off_pool = PAIRS_OFF_POOL.with(std::cell::Cell::get); + for pair in sources.chunks(2) { + #[cfg(feature = "parallel")] + let built = if off_pool { + commit_pair_off_pool(pair, &commit)? + } else { + pair.par_iter() + .map(commit) + .collect::, Error>>()? + }; + #[cfg(not(feature = "parallel"))] + let built = pair.iter().map(commit).collect::, Error>>()?; + for (commitment, d) in built { + commitments.push(commitment); + domain = Some(d); + } + } + // ★ THE COMMITS ARE OVER, SO IS THEIR TURN. What runs next is the + // per-table argument — the whole of it, every table — and only then + // the openings; the argument reserves against the same budget and + // takes no turn of the group's. The codewords stay promised: they are + // what the openings open. The room goes back, and each opening takes + // it again (`prove`). + let room = match room { + None => Room::Own, + Some(room) if park => { + room.give_back(); + crate::gpu::note_room_parked(); + Room::Parked + } + Some(room) => Room::Held(room), + }; + Ok(Self { + layout, + commitments, + domain: domain.ok_or(Error::EmptyPolynomial)?, + room, + }) + } + + pub fn roots(&self) -> Vec { + self.commitments.iter().map(|c| c.root()).collect() + } + + /// Each stacked polynomial's commitment, for a test that reads its paths + /// on the card. + #[cfg(all(test, feature = "cuda"))] + pub(crate) fn commitments_mut(&mut self) -> &mut [CodewordCommitment] { + &mut self.commitments + } + + pub fn domain(&self) -> &Domain { + &self.domain + } + + pub fn layout(&self) -> &StackedLayout { + &self.layout + } + + /// Lets every codeword go — and the room the group promised the card — + /// keeping the roots and each tree's top (all but its bottom `drop` levels, + /// [`crate::whir_commit::TreeDrop`]) on the host. See + /// [`RetiredStack::revive`]. + pub fn retire( + self, + drop: crate::whir_commit::TreeDrop, + config: &ChainConfig, + ) -> Result, Error> { + // The group's room goes back here (`room: _` drops it); each codeword + // goes as its top comes home, so the card never holds more than it did + // at the commit. + let Self { + layout, + commitments, + domain, + room: _, + } = self; + let commitments = commitments + .into_iter() + .map(|commitment| { + commitment.retire( + drop, + config + .format + .cap_policy_at(crate::whir_hash::arity::()), + ) + }) + .collect::, Error>>()?; + Ok(RetiredStack { + layout, + commitments, + domain, + }) + } +} + +/// A group committed and then let go: its layout, its domain, and per stacked +/// polynomial the root and the top of its tree. +/// +/// ★ THE BLOCK PROVER'S HOLD BETWEEN ITS TWO PHASES. Every group is committed +/// before any challenge exists, and no card holds every group's codewords, so +/// each group gives its codewords back as soon as its roots are taken and +/// rebuilds them for its opening. What it keeps of a tree is the top: the +/// paths an opening sends are the whole tree's, byte for byte, because the +/// dropped levels are re-hashed from the rebuilt codeword and checked against +/// the kept node before a path leaves (`CodewordCommitment::retire`). +pub struct RetiredStack { + layout: StackedLayout, + commitments: Vec, + domain: Domain, +} + +impl RetiredStack +where + FieldElement: AsBytes + Sync + Send, +{ + pub fn roots(&self) -> Vec { + self.commitments.iter().map(|c| c.root()).collect() + } + + pub fn layout(&self) -> &StackedLayout { + &self.layout + } + + /// Host bytes kept of the trees. + pub fn tree_bytes(&self) -> usize { + self.commitments.iter().map(|c| c.tree_bytes()).sum() + } + + /// The group openable again: every codeword recomputed from `columns` — + /// the group's columns, in the order they were committed — with no tree + /// built. `resident` names them on the card as at the commit. + pub fn revive>( + self, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, ColumnsAt<'_>)>, + config: &ChainConfig, + ) -> Result, Error> { + if let Some((_, ColumnsAt::Map(map))) = resident + && map.len() != columns.len() + { + return Err(Error::QueryCountMismatch { + expected: columns.len(), + got: map.len(), + }); + } + let Self { + layout, + commitments: retired, + domain, + } = self; + // The openings' turn, held from here to the last opening: the codewords + // come back one at a time and the chain works beside each. The block + // prover can revive a group while the next group's argument holds its + // own reservations (its phase-B overlap); a room refused then is + // counted, and each codeword promises its own working set instead. + let room = if retired.is_empty() { + None + } else { + let room = crate::gpu::reserve_room(room_bytes( + &layout, + config, + resident.is_none(), + Turn::Open, + )); + if room.is_none() && crate::gpu::reserve_budget() > 0 { + crate::gpu::note_revive_room_refused(); + } + room + }; + let transient = room.is_none(); + // ★ A coset-ordered round 0 is revived over its stack, not re-encoded: + // each opening evaluates the cosets its queries fall in, a polynomial's + // own chain and every tree of a grouped run alike (both open a tree + // through `open_many_capped`). Only where the stack is resident. The + // successor must then be encoded from the bound message, a run's from + // its combined one; the chain refuses a fold + // (`Error::CosetSuccessorNotEncoded`). + let from_stack = config.format.leaves == crate::whir_commit::LeafOrder::Coset + && crate::gpu::coset_openings_in_effect(); + let mut commitments = Vec::with_capacity(retired.len()); + for (poly, retired) in retired.into_iter().enumerate() { + let source = whir_chain::Stacked { + parts: layout + .parts_of(poly) + .into_iter() + .map(|(column, offset)| (columns[column] as &dyn HostColumn, offset)) + .collect(), + resident: resident.map(|(store, at)| { + ( + store, + layout + .parts_of(poly) + .into_iter() + .map(|(column, offset)| (at.store_column(column), offset)) + .collect(), + ) + }), + num_vars: layout.n_stack(), + }; + let held = match (&source.resident, from_stack) { + (Some((store, parts)), true) => crate::gpu::CosetSource::from_resident( + store, + parts, + layout.n_stack(), + config.log_blowup, + ), + _ => None, + }; + let codeword = match held { + Some(held) => crate::whir_commit::Codeword::Source( + crate::whir_commit::SourceCodeword::new(held), + ), + None => whir_chain::encode_stacked::(&source, config, transient)?, + }; + commitments.push(retired.revive::(codeword)?); + } + Ok(StackedCommitment { + layout, + commitments, + domain, + room: match room { + Some(room) => Room::Held(room), + None => Room::Own, + }, + }) + } +} + +/// One weighted, chained evaluation proof per chain: per stacked polynomial, +/// or per run of them under [`GroupFold::Indexed`] ([`index_chains`]). +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct StackedProof { + pub polys: Vec>, +} + +impl StackedProof { + /// Codeword elements the openings carry, across every stacked polynomial. + pub fn opened_elements(&self) -> usize { + self.polys.iter().map(ChainProof::opened_elements).sum() + } +} + +/// Where each column is claimed. +/// +/// One table's sumcheck leaves every one of its columns at the **same** point, +/// which is [`Shared`](Self::Shared). Stacking several tables into one +/// commitment does not: each settles at its own, and the weight stops +/// factoring into slots times a single `eq`. +#[derive(Clone, Copy, Debug)] +pub enum Claimed<'a, E: IsField> { + Shared(&'a [FieldElement]), + PerColumn(&'a [Vec>]), +} + +impl Claimed<'_, E> { + fn point(&self, column: usize) -> Result<&[FieldElement], Error> { + match self { + Self::Shared(point) => Ok(point), + Self::PerColumn(points) => { + points + .get(column) + .map(Vec::as_slice) + .ok_or(Error::UnknownPolynomial { + index: column, + len: points.len(), + }) + } + } + } +} + +/// The columns one stacked polynomial holds, as `(column index, placement)`. +fn columns_in(layout: &StackedLayout, poly: usize) -> impl Iterator { + layout + .placements() + .iter() + .enumerate() + .filter(move |(_, place)| place.poly == poly) +} + +/// One column's share of a stacked polynomial's weight: the subcube it owns, +/// the point it is claimed at, and the scale it carries. +/// +/// The weight is the sum of these, and each lands in its own range — which is +/// what lets whoever builds it write the cells where they go, here or on a +/// device. +pub struct WeightShare<'a, E: IsField> { + pub offset: usize, + pub point: &'a [FieldElement], + pub scale: FieldElement, +} + +/// The shares of `poly`'s weight, one per column it holds. +fn weight_shares<'a, E: IsField>( + layout: &StackedLayout, + poly: usize, + points: &'a Claimed<'a, E>, + weights: &'a [FieldElement], +) -> Result>, Error> { + columns_in(layout, poly) + .map(|(column, place)| { + let point = points.point(column)?; + if point.len() != place.num_vars { + return Err(Error::VariableCountMismatch { + expected: place.num_vars, + got: point.len(), + }); + } + Ok(WeightShare { + offset: place.offset, + point, + scale: weights[column].clone(), + }) + }) + .collect() +} + +/// `Σ_i gamma^i·eq(prefix_i ‖ point_i, ·)` as a table. +/// +/// Each column owns a contiguous subcube of the stack, so its share of the +/// weight is written straight into that range and the gaps stay zero. Columns +/// of different heights cost nothing extra here — the range is just shorter. +pub fn weight_table( + shares: &[WeightShare<'_, E>], + n_stack: usize, +) -> Result, Error> +where + FieldElement: Send + Sync, +{ + let mut table = vec![FieldElement::::zero(); 1usize << n_stack]; + for share in shares { + let cells = 1usize << share.point.len(); + eq_evals_into( + share.point, + &share.scale, + &mut table[share.offset..share.offset + cells], + )?; + } + Mle::new(table) +} + +/// The same weight at an arbitrary point, in closed form. +/// +/// A column's prefix picks its subcube out of the stack, so its term is the +/// prefix indicator at the high variables times `eq(point_i, ·)` at the low +/// ones — and how many are "high" is the column's own business, which is what +/// lets heights differ. +fn weight_at( + layout: &StackedLayout, + poly: usize, + points: &Claimed<'_, E>, + weights: &[FieldElement], + at: &[FieldElement], +) -> Result, Error> { + if at.len() != layout.n_stack() { + return Err(Error::VariableCountMismatch { + expected: layout.n_stack(), + got: at.len(), + }); + } + let mut total = FieldElement::::zero(); + for (column, place) in columns_in(layout, poly) { + let corner: Vec> = place + .prefix_bits() + .into_iter() + .map(|b| { + if b { + FieldElement::::one() + } else { + FieldElement::::zero() + } + }) + .collect(); + let (high, low) = at.split_at(corner.len()); + total += &weights[column] * eq_eval(&corner, high)? * eq_eval(points.point(column)?, low)?; + } + Ok(total) +} + +/// A stack's commit-time point over the stack and every polynomial's scale, +/// as a chain's weight reads them ([`ood_scales`]). +type OodAt<'a, E> = (&'a [FieldElement], &'a [FieldElement]); + +/// A grouped chain's weight at its `n + m` challenges in binding order +/// (`k0` data, `m` index, the rest): `Σ_p eq(ρ, p)·weight_at_p` with each +/// polynomial's weight read at the `n` data challenges, `at[..k0] ‖ +/// at[k0+m..]` (REV-B2-JUDGE B1-3). The index sits after the column +/// selectors' first `k0` bits, so a selector longer than `k0` straddles it, +/// and only the data challenges reach `weight_at`. +/// +/// With `ood` (the commit-time point over the stack and every polynomial's +/// scale), each polynomial's weight also carries `scale_p·eq(point, data)` +/// ([`crate::commit_ood`]). +#[allow(clippy::too_many_arguments)] +fn grouped_weight_at( + layout: &StackedLayout, + run: std::ops::Range, + k0: usize, + points: &Claimed<'_, E>, + weights: &[FieldElement], + ood: Option>, + at: &[FieldElement], +) -> Result, Error> { + let m = index_vars(run.len()); + if at.len() != layout.n_stack() + m || k0 > layout.n_stack() { + return Err(Error::VariableCountMismatch { + expected: layout.n_stack() + m, + got: at.len(), + }); + } + let (head, rest) = at.split_at(k0); + let (rho, tail) = rest.split_at(m); + let data: Vec> = head.iter().chain(tail).cloned().collect(); + #[cfg(test)] + let fault = whir_chain::grouped_faults::get(); + #[cfg(test)] + let swapped: Vec> = rho.iter().rev().cloned().collect(); + #[cfg(test)] + let rho: &[FieldElement] = if fault == whir_chain::grouped_faults::Fault::SwapRhoInWeight { + &swapped + } else { + rho + }; + let ood_eq = match ood { + Some((point, _)) => Some(eq_eval(point, &data)?), + None => None, + }; + let mut total = FieldElement::::zero(); + for (p, poly) in run.clone().enumerate() { + let eq = eq_index(rho, p); + #[cfg(test)] + let eq = if fault == whir_chain::grouped_faults::Fault::DropIndexFactor { + FieldElement::::one() + } else { + eq + }; + let mut weight = weight_at(layout, poly, points, weights, &data)?; + if let (Some((_, scales)), Some(ood_eq)) = (ood, &ood_eq) { + weight += &scales[poly] * ood_eq; + } + total += eq * weight; + } + #[cfg(test)] + if fault == whir_chain::grouped_faults::Fault::AbsentSlotWeightOne { + for p in run.len()..(1usize << m) { + total += eq_index(rho, p); + } + } + Ok(total) +} + +/// The claimed sum for one stacked polynomial: its columns' values, batched. +fn claimed( + layout: &StackedLayout, + poly: usize, + values: &[FieldElement], + weights: &[FieldElement], +) -> Result, Error> { + Ok( + columns_in(layout, poly).fold(FieldElement::::zero(), |acc, (column, _)| { + acc + &weights[column] * &values[column] + }), + ) +} + +/// Where [`ood_claims`] found its column values: how many columns the card +/// evaluated in place and how many the host did. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct OodCounts { + pub on_card: usize, + pub on_host: usize, +} + +/// ★ A stack's commit-time out-of-domain claims as the prover holds them from +/// its commit to its opening (WHIR's commitment step, [`crate::commit_ood`]): +/// the point, every column's value at its share of it, and every stacked +/// polynomial's answer. +/// +/// The answers are what a transcript absorbs before the challenges the column +/// claims depend on; the column values are what the opening's device rounds +/// read the overlay's share from ([`crate::commit_ood::partial`]). +#[derive(Clone, Debug)] +pub struct OodClaims { + pub z0: FieldElement, + /// Column `c`'s value at the last `h_c` coordinates of + /// `ood_point(z0, n_stack)`, in placement order. + pub columns: Vec>, + /// `F_i(ood_point(z0, n_stack))`, one per stacked polynomial, in + /// commitment order. + pub answers: Vec>, +} + +/// ★ [`OodClaims`] at `z0` from the columns where they lie: each run of +/// equal-height columns the store holds side by side evaluated on the card in +/// place, in one call, and every other column on the host. The answers follow +/// from the column values ([`crate::commit_ood::answers`]). +pub fn ood_claims( + layout: &StackedLayout, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, ColumnsAt<'_>)>, + z0: &FieldElement, +) -> Result<(OodClaims, OodCounts), Error> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, + C: HostColumn, +{ + let placements = layout.placements(); + if columns.len() != placements.len() { + return Err(Error::QueryCountMismatch { + expected: placements.len(), + got: columns.len(), + }); + } + let point = whir_chain::ood_point(z0, layout.n_stack()); + let at = |c: usize| crate::commit_ood::column_point(layout, c, &point); + let mut values: Vec>> = vec![None; placements.len()]; + let mut counts = OodCounts::default(); + // Every run the card holds is read there, however small: a resident + // table may be held narrow or as records, and a host read of it would + // widen the whole table (and keep the copy) for a few columns' values. + if let Some((store, map)) = &resident { + let mut c = 0usize; + while c < placements.len() { + let height = placements[c].num_vars; + let first = map.store_column(c); + let mut end = c + 1; + while end < placements.len() + && placements[end].num_vars == height + && map.store_column(end) == first + (end - c) + { + end += 1; + } + if let Some(run) = crate::gpu::evaluate_resident_run(store, first, end - c, at(c), 0) { + counts.on_card += run.len(); + for (slot, value) in values[c..end].iter_mut().zip(run) { + *slot = Some(value); + } + } + c = end; + } + } + let missing: Vec = (0..placements.len()) + .filter(|&c| values[c].is_none()) + .collect(); + counts.on_host = missing.len(); + // ⚠ Each column's host copy is fetched HERE, on the calling thread, before + // the parallel map. A table held narrow or as records widens on its first + // host read, once, behind a `OnceLock`, and the widen is itself a parallel + // map. Fetched inside a job, the worker running the widen picks up a + // sibling job of the same table while it waits and re-enters the + // `OnceLock` it is initialising: the pool deadlocks + // (`a_narrow_tables_answers_end_and_match_its_widened_columns`). + let hosts: Vec<(usize, &Mle)> = missing.iter().map(|&c| (c, columns[c].host())).collect(); + let one = |&(c, column): &(usize, &Mle)| -> Result<(usize, FieldElement), Error> { + Ok((c, column.evaluate_in(at(c))?)) + }; + #[cfg(feature = "parallel")] + let host: Vec<(usize, FieldElement)> = + hosts.par_iter().map(one).collect::>()?; + #[cfg(not(feature = "parallel"))] + let host: Vec<(usize, FieldElement)> = hosts.iter().map(one).collect::>()?; + for (c, value) in host { + values[c] = Some(value); + } + let values: Vec> = values + .into_iter() + .enumerate() + .map(|(c, value)| { + value.ok_or(Error::UnknownPolynomial { + index: c, + len: placements.len(), + }) + }) + .collect::>()?; + let answers = crate::commit_ood::answers(layout, &point, &values)?; + Ok(( + OodClaims { + z0: z0.clone(), + columns: values, + answers, + }, + counts, + )) +} + +/// A stack's commit-time out-of-domain claims as the verifier reads them: the +/// point, and one answer per stacked polynomial in commitment order — absorbed +/// by the caller's transcript where they were given, before the challenges, +/// so NOT absorbed again here. +#[derive(Clone, Copy, Debug)] +pub struct CommitOod<'a, E: IsField> { + /// The point, as the univariate value [`whir_chain::ood_point`] lifts. + pub z0: &'a FieldElement, + pub answers: &'a [FieldElement], +} + +/// The commit-time claims' batching: after the column claims' challenge, one +/// more, `γ`, and polynomial `i`'s answer carried at `γ^(i+1)`. Distinct +/// powers, so a run's claims cannot cancel one another; none of them `1`, so +/// no answer merges into the column claims' first term. Returns the point over +/// the stack and the scales. Drawn on both sides from here. +fn ood_scales( + z0: &FieldElement, + layout: &StackedLayout, + transcript: &mut T, +) -> (Vec>, Vec>) +where + E: IsField + 'static, + T: IsTranscript, +{ + let gamma: FieldElement = transcript.sample_field_element(); + let mut scales = challenge_powers(&gamma, layout.num_polys() + 1); + scales.remove(0); + (whir_chain::ood_point(z0, layout.n_stack()), scales) +} + +/// Proves that every column takes its claimed value at the shared `point`. +/// +/// The claims are absorbed before the batching challenge, so the prover cannot +/// pick them after seeing it. +pub fn prove( + stacked: &StackedCommitment, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, usize)>, + point: &Claimed<'_, E>, + values: &[FieldElement], + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, + C: HostColumn, +{ + prove_mapped::( + stacked, + columns, + resident.map(|(store, first)| (store, ColumnsAt::From(first))), + point, + values, + config, + transcript, + ) +} + +/// [`prove`] with the columns' store positions named by a [`ColumnsAt`] — the +/// same map the group was committed with. +pub fn prove_mapped( + stacked: &StackedCommitment, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, ColumnsAt<'_>)>, + point: &Claimed<'_, E>, + values: &[FieldElement], + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, + C: HostColumn, +{ + prove_mapped_ood::( + stacked, columns, resident, point, values, None, config, transcript, + ) +} + +/// [`prove_mapped`] with the stack's commit-time out-of-domain claims +/// ([`OodClaims`]) batched into its chains: after the column claims' +/// challenge, `γ` ([`ood_scales`]), and each polynomial's chain proves its +/// column claims plus `γ^(i+1)·y_i` under its weight plus +/// `γ^(i+1)·eq(ood_point(z0), ·)` ([`crate::commit_ood`]). The answers were +/// absorbed where they were given, before the challenges; they are not +/// absorbed here. `None` is [`prove_mapped`], byte for byte. +#[allow(clippy::too_many_arguments)] +pub fn prove_mapped_ood( + stacked: &StackedCommitment, + columns: &[&C], + resident: Option<(&crate::gpu::ResidentColumns, ColumnsAt<'_>)>, + point: &Claimed<'_, E>, + values: &[FieldElement], + ood: Option<&OodClaims>, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, + C: HostColumn, +{ + if let Some((_, ColumnsAt::Map(map))) = resident + && map.len() != columns.len() + { + return Err(Error::QueryCountMismatch { + expected: columns.len(), + got: map.len(), + }); + } + let layout = &stacked.layout; + if values.len() != layout.placements().len() { + return Err(Error::QueryCountMismatch { + expected: layout.placements().len(), + got: values.len(), + }); + } + if let Some(claims) = ood + && (claims.answers.len() != layout.num_polys() + || claims.columns.len() != layout.placements().len()) + { + return Err(Error::CommitOodCount { + expected: layout.num_polys(), + got: claims.answers.len(), + }); + } + for value in values { + transcript.append_field_element(value); + } + let weights = challenge_powers(&transcript.sample_field_element(), values.len()); + let ood = ood.map(|claims| (claims, ood_scales(&claims.z0, layout, transcript))); + // Polynomial `i`'s overlay: its scale, and the table a card's first rounds + // read the overlay's share from. + let overlay_of = |i: usize| -> Result>, Error> { + let Some((claims, (ood_at, scales))) = &ood else { + return Ok(None); + }; + Ok(Some(crate::commit_ood::Overlay { + point: ood_at, + scale: scales[i].clone(), + partial: crate::commit_ood::partial( + layout, + i, + ood_at, + &claims.columns, + crate::commit_ood::partial_vars(layout, i), + )?, + })) + }; + + // ★ THE GROUP'S TURN, TAKEN BACK for as long as its openings run — one at a + // time, so one turn covers them all — and given back when they end. A + // refusal is counted and said, and the openings run anyway: see + // `take_turn` for why there is no other path. + let _turn = match &stacked.room { + Room::Parked => crate::gpu::take_turn( + room_bytes(layout, config, resident.is_none(), Turn::Open), + stacked.commitments.len(), + layout.n_stack(), + ), + Room::Own | Room::Held(_) => None, + }; + // The polynomial is its columns at their offsets, handed over as they + // are — see `StackedCommitment::commit`. + let poly_of = |i: usize| whir_chain::Stacked { + parts: layout + .parts_of(i) + .into_iter() + .map(|(column, offset)| (columns[column] as &dyn HostColumn, offset)) + .collect(), + resident: resident.map(|(store, at)| { + ( + store, + layout + .parts_of(i) + .into_iter() + .map(|(column, offset)| (at.store_column(column), offset)) + .collect(), + ) + }), + num_vars: layout.n_stack(), + }; + let chains = layout_chains::(layout, config)?; + let mut polys = Vec::with_capacity(chains.len()); + for run in chains { + if run.len() == 1 { + let i = run.start; + // The weight goes down as its shares: a device writes them into its + // own buffer, and the host materializes the table only if none does. + polys.push(whir_chain::prove_shared_ood::( + &poly_of(i), + &weight_shares(layout, i, point, &weights)?, + overlay_of(i)?.as_ref(), + layout.n_stack(), + &stacked.commitments[i], + &stacked.domain, + config, + transcript, + )?); + continue; + } + let sources: Vec> = run.clone().map(&poly_of).collect(); + let shares = run + .clone() + .map(|i| weight_shares(layout, i, point, &weights)) + .collect::, Error>>()?; + let overlays = run + .clone() + .map(&overlay_of) + .collect::>, Error>>()?; + let commitments: Vec<&CodewordCommitment> = + run.map(|i| &stacked.commitments[i]).collect(); + polys.push(whir_chain::prove_grouped_shared_ood::( + &sources, + &shares, + overlays.as_deref(), + layout.n_stack(), + &commitments, + &stacked.domain, + config, + transcript, + )?); + } + + Ok(StackedProof { polys }) +} + +/// Verifies the column claims against the stacked commitments. +/// +/// The layout is public and derived from the column heights, so it is not part +/// of the proof. A chain's refusal is reported as its first polynomial's +/// `ColumnOpeningRejected` (a shape refusal by its own name); the cause is +/// [`verify_traced`]'s. +#[allow(clippy::too_many_arguments)] +pub fn verify( + proof: &StackedProof, + layout: &StackedLayout, + roots: &[Commitment], + point: &Claimed<'_, E>, + values: &[FieldElement], + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + verify_traced::( + proof, layout, roots, point, values, domain, config, transcript, + ) + .map_err(Refusal::into_error) +} + +/// [`verify`] with the stack's commit-time out-of-domain claims +/// ([`CommitOod`]): after the column claims' challenge, `γ`; each chain's +/// claim gains `γ^(i+1)·y_i` and its weight `γ^(i+1)·eq(ood_point(z0), ·)`, +/// evaluated in closed form at the chain's final point. A count of answers +/// other than one per polynomial is refused before anything is absorbed. +/// `None` is [`verify`], byte for byte. +#[allow(clippy::too_many_arguments)] +pub fn verify_ood( + proof: &StackedProof, + layout: &StackedLayout, + roots: &[Commitment], + point: &Claimed<'_, E>, + values: &[FieldElement], + ood: Option>, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + verify_traced_ood::( + proof, layout, roots, point, values, ood, domain, config, transcript, + ) + .map_err(Refusal::into_error) +} + +/// Why [`verify_traced`] refused: before any chain ran, with the error +/// [`verify`] returns as it is, or in chain `chain` — whose first polynomial is +/// `poly` — with the chain verifier's own error, the check that refused. +#[derive(Debug, PartialEq, Eq)] +pub enum Refusal { + Before(Error), + Chain { + chain: usize, + poly: usize, + cause: Error, + }, +} + +impl Refusal { + /// The error [`verify`] reports: a shape refusal by its own name, any other + /// chain refusal as the chain's first polynomial's `ColumnOpeningRejected`. + pub fn into_error(self) -> Error { + match self { + Refusal::Before(e) => e, + Refusal::Chain { cause, poly, .. } => match cause { + Error::GroupedShape(_) => cause, + _ => Error::ColumnOpeningRejected { column: poly }, + }, + } + } +} + +/// [`verify`], its refusal carrying the cause ([`Refusal`]): which chain, and +/// the check inside it that refused. The verdict and the transcript are +/// [`verify`]'s. +#[allow(clippy::too_many_arguments)] +pub fn verify_traced( + proof: &StackedProof, + layout: &StackedLayout, + roots: &[Commitment], + point: &Claimed<'_, E>, + values: &[FieldElement], + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Refusal> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + verify_traced_ood::( + proof, layout, roots, point, values, None, domain, config, transcript, + ) +} + +/// [`verify_ood`], its refusal carrying the cause, as [`verify_traced`]'s. +#[allow(clippy::too_many_arguments)] +pub fn verify_traced_ood( + proof: &StackedProof, + layout: &StackedLayout, + roots: &[Commitment], + point: &Claimed<'_, E>, + values: &[FieldElement], + ood: Option>, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Refusal> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + if let Some(ood) = &ood + && ood.answers.len() != layout.num_polys() + { + return Err(Refusal::Before(Error::CommitOodCount { + expected: layout.num_polys(), + got: ood.answers.len(), + })); + } + if values.len() != layout.placements().len() { + return Err(Refusal::Before(Error::QueryCountMismatch { + expected: layout.placements().len(), + got: values.len(), + })); + } + // One root per polynomial, and — before anything is absorbed — one chain + // per run of the verifier's own split, never the proof's count. + let chains = layout_chains::(layout, config).map_err(Refusal::Before)?; + let per_poly = config.format.group == GroupFold::PerPoly; + if roots.len() != layout.num_polys() || (per_poly && proof.polys.len() != layout.num_polys()) { + return Err(Refusal::Before(Error::QueryCountMismatch { + expected: layout.num_polys(), + // Today's report under `PerPoly`; under `Indexed` the root count, + // the chain count being refused by name below. + got: if per_poly { + proof.polys.len().min(roots.len()) + } else { + roots.len() + }, + })); + } + if proof.polys.len() != chains.len() { + return Err(Refusal::Before(Error::GroupedShape( + GroupedShape::ChainCount { + expected: chains.len(), + got: proof.polys.len(), + }, + ))); + } + for value in values { + transcript.append_field_element(value); + } + let weights = challenge_powers(&transcript.sample_field_element(), values.len()); + let ood = ood.map(|ood| (ood.answers, ood_scales(ood.z0, layout, transcript))); + // Polynomial `i`'s answer, at its scale, on its chain's claim. + let ood_claim = |i: usize| match &ood { + Some((answers, (_, scales))) => &scales[i] * &answers[i], + None => FieldElement::::zero(), + }; + let ood_at = ood + .as_ref() + .map(|(_, (at, scales))| (at.as_slice(), scales.as_slice())); + + for (chain, (eval_proof, run)) in proof.polys.iter().zip(chains).enumerate() { + let refused = |cause: Error| Refusal::Chain { + chain, + poly: run.start, + cause, + }; + if run.len() == 1 { + let i = run.start; + whir_chain::verify_weighted::( + eval_proof, + &roots[i], + |at: &[FieldElement]| { + let mut weight = weight_at(layout, i, point, &weights, at)?; + if let Some((ood_point, scales)) = ood_at { + weight += &scales[i] * eq_eval(ood_point, at)?; + } + Ok(weight) + }, + claimed(layout, i, values, &weights).map_err(Refusal::Before)? + ood_claim(i), + layout.n_stack(), + domain, + config, + transcript, + ) + .map_err(refused)?; + continue; + } + // The run's claim is its polynomials' claims summed: one γ for the + // whole group, its powers indexed by the group's columns, so no two + // polynomials share an exponent (D-ONECHAIN §3.2, Lemma A). Their + // commit-time answers ride along at their own powers. + let mut claim = FieldElement::::zero(); + for i in run.clone() { + claim += claimed(layout, i, values, &weights).map_err(Refusal::Before)? + ood_claim(i); + } + let k0 = config + .schedule(layout.n_stack()) + .first() + .copied() + .unwrap_or(0); + whir_chain::verify_grouped::( + eval_proof, + &roots[run.clone()], + |at: &[FieldElement]| { + grouped_weight_at(layout, run.clone(), k0, point, &weights, ood_at, at) + }, + claim, + layout.n_stack(), + domain, + config, + transcript, + ) + .map_err(refused)?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{whir_chain::GrindBits, whir_hash::KeccakWhir}; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"stacked-eval-test") + } + + fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: crate::whir_chain::ChainFormat::DEFAULT, + } + } + + fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_add(seed).wrapping_mul(6364136223846793005) >> 11)) + .collect(), + ) + .unwrap() + } + + fn point(num_vars: usize) -> Vec { + (0..num_vars).map(|i| FE::from(101 + i as u64)).collect() + } + + /// `count` columns of one height, and the layout that packs them. + fn stack_of(count: usize, num_vars: usize, n_stack: usize) -> (StackedLayout, Vec>) { + let columns: Vec> = (0..count) + .map(|i| column(num_vars, 7 + i as u64 * 13)) + .collect(); + let layout = StackedLayout::build(&vec![num_vars; count], n_stack).unwrap(); + (layout, columns) + } + + fn values(columns: &[Mle], at: &[FE]) -> Vec { + columns.iter().map(|c| c.evaluate(at).unwrap()).collect() + } + + fn run( + layout: StackedLayout, + columns: &[Mle], + at: &[FE], + claimed: &[FE], + ) -> Result { + let stacked = StackedCommitment::::commit( + layout, + &crate::stacking::borrow(columns), + None, + &config(), + )?; + let roots = stacked.roots(); + let proof = prove( + &stacked, + &crate::stacking::borrow(columns), + None, + &Claimed::Shared(at), + claimed, + &config(), + &mut transcript(), + )?; + + verify::( + &proof, + stacked.layout(), + &roots, + &Claimed::Shared(at), + claimed, + stacked.domain(), + &config(), + &mut transcript(), + )?; + Ok(roots.len()) + } + + /// A stack committed, retired and revived, opened at `at`: the proof's + /// Debug text (every opened value and path). + fn opened_after_retire( + layout: StackedLayout, + committed: &[Mle], + revived_over: &[Mle], + at: &[FE], + claimed: &[FE], + drop_levels: usize, + ) -> Result { + opened_after_retire_under( + &config(), + layout, + committed, + revived_over, + at, + claimed, + drop_levels, + ) + } + + fn opened_after_retire_under( + config: &ChainConfig, + layout: StackedLayout, + committed: &[Mle], + revived_over: &[Mle], + at: &[FE], + claimed: &[FE], + drop_levels: usize, + ) -> Result { + let retired = StackedCommitment::::commit( + layout, + &crate::stacking::borrow(committed), + None, + config, + )? + .retire(crate::whir_commit::TreeDrop::uniform(drop_levels), config)?; + let revived = retired.revive::( + &crate::stacking::borrow(revived_over), + None, + config, + )?; + let proof = prove( + &revived, + &crate::stacking::borrow(revived_over), + None, + &Claimed::Shared(at), + claimed, + config, + &mut transcript(), + )?; + Ok(format!("{proof:?}")) + } + + /// ★ The block prover's hold: a stack that let its codewords go and kept + /// only the top of each tree opens to EXACTLY the proof of one that kept + /// everything — at every depth of the drop, including none and more than + /// the tree has — and the revived proof verifies against the first roots. + #[test] + fn a_retired_and_revived_stack_opens_to_the_same_bytes() { + let num_vars = 6; + let (layout, columns) = stack_of(5, num_vars, 8); + assert!(layout.num_polys() >= 2); + let at = point(num_vars); + let claimed = values(&columns, &at); + let kept = StackedCommitment::::commit( + layout.clone(), + &crate::stacking::borrow(&columns), + None, + &config(), + ) + .unwrap(); + let roots = kept.roots(); + let proof = prove( + &kept, + &crate::stacking::borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed, + &config(), + &mut transcript(), + ) + .unwrap(); + let want = format!("{proof:?}"); + for drop_levels in [0, 1, 3, 64] { + let got = opened_after_retire( + layout.clone(), + &columns, + &columns, + &at, + &claimed, + drop_levels, + ) + .unwrap(); + assert_eq!(got, want, "drop {drop_levels}: the revived opening differs"); + } + // The retired roots are the committed ones. + let retired = StackedCommitment::::commit( + layout.clone(), + &crate::stacking::borrow(&columns), + None, + &config(), + ) + .unwrap() + .retire(crate::whir_commit::TreeDrop::uniform(3), &config()) + .unwrap(); + assert_eq!(retired.roots(), roots); + assert!(retired.tree_bytes() > 0); + verify::( + &proof, + &layout, + &roots, + &Claimed::Shared(&at), + &claimed, + kept.domain(), + &config(), + &mut transcript(), + ) + .unwrap(); + } + + /// The same under a Merkle cap (production runs `whir_cap=auto`, height 3 + /// on the first round's tree): the owner's cap comes from the kept top. + #[test] + fn a_retired_stack_opens_to_the_same_bytes_under_a_cap() { + let mut capped = config(); + capped.format.cap = crate::whir_chain::CapPolicy::Fixed(2); + let num_vars = 6; + let (layout, columns) = stack_of(5, num_vars, 8); + let at = point(num_vars); + let claimed = values(&columns, &at); + let kept = StackedCommitment::::commit( + layout.clone(), + &crate::stacking::borrow(&columns), + None, + &capped, + ) + .unwrap(); + let want = format!( + "{:?}", + prove( + &kept, + &crate::stacking::borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed, + &capped, + &mut transcript(), + ) + .unwrap() + ); + for drop_levels in [0, 2, 3] { + let got = opened_after_retire_under( + &capped, + layout.clone(), + &columns, + &columns, + &at, + &claimed, + drop_levels, + ) + .unwrap(); + assert_eq!( + got, want, + "drop {drop_levels}: the capped revived opening differs" + ); + } + } + + /// The negative: revived over columns that are NOT the committed ones (one + /// value changed), the opening refuses rather than send paths the kept + /// tree does not authenticate. With no level dropped there is nothing to + /// re-hash, so the refusal needs at least one. + #[test] + fn a_stack_revived_over_other_columns_refuses_to_open() { + let num_vars = 6; + let (layout, columns) = stack_of(5, num_vars, 8); + let at = point(num_vars); + let claimed = values(&columns, &at); + let mut other = columns.clone(); + let mut evals = other[2].evals().to_vec(); + evals[5] += FE::one(); + other[2] = Mle::new(evals).unwrap(); + for drop_levels in [1, 3] { + let got = + opened_after_retire(layout.clone(), &columns, &other, &at, &claimed, drop_levels); + assert!( + matches!(got, Err(Error::RecomputedCodewordMismatch { .. })), + "drop {drop_levels}: {got:?}" + ); + } + } + + /// The kept-top paths re-hash their queried blocks in parallel: with many + /// queries over a tall stack (tens of distinct blocks per round), the + /// revived opening is still EXACTLY the kept one's, and a revive over other + /// columns still refuses. + #[test] + fn a_retired_stack_opens_to_the_same_bytes_over_many_queried_blocks() { + let many = ChainConfig { + num_queries: 48, + ..config() + }; + let num_vars = 10; + let (layout, columns) = stack_of(4, num_vars, 12); + let at = point(num_vars); + let claimed = values(&columns, &at); + let kept = StackedCommitment::::commit( + layout.clone(), + &crate::stacking::borrow(&columns), + None, + &many, + ) + .unwrap(); + let want = format!( + "{:?}", + prove( + &kept, + &crate::stacking::borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed, + &many, + &mut transcript(), + ) + .unwrap() + ); + let paths_before = crate::whir_commit::top_path_counts(); + for drop_levels in [3, 4] { + let got = opened_after_retire_under( + &many, + layout.clone(), + &columns, + &columns, + &at, + &claimed, + drop_levels, + ) + .unwrap(); + assert_eq!(got, want, "drop {drop_levels}: the revived opening differs"); + } + let paths_after = crate::whir_commit::top_path_counts(); + // Other tests may add to the process-wide counters; this one alone + // re-hashes at least 2 drops x 48 queries' blocks of 8 leaves. + assert!( + paths_after.1 - paths_before.1 >= 2 * 8 * 32, + "too few leaves re-hashed for the test to cover many blocks: {paths_before:?} -> {paths_after:?}" + ); + + let mut other = columns.clone(); + let mut evals = other[1].evals().to_vec(); + evals[300] += FE::one(); + other[1] = Mle::new(evals).unwrap(); + for drop_levels in [3, 4] { + let got = opened_after_retire_under( + &many, + layout.clone(), + &columns, + &other, + &at, + &claimed, + drop_levels, + ); + assert!( + matches!(got, Err(Error::RecomputedCodewordMismatch { .. })), + "drop {drop_levels}: {got:?}" + ); + } + } + + /// A pair committed off the pool is the pair the pool commits: the same roots, + /// polynomial by polynomial. The flag is this thread's, and is back as it was after + /// each call. + #[test] + fn pairs_off_the_pool_commit_the_same_roots() { + let off_pool = || PAIRS_OFF_POOL.with(std::cell::Cell::get); + // Six columns, two slots per stacked polynomial: three polynomials, a pair and + // one alone. + let roots = |off| { + with_pairs_off_pool(off, || { + let (layout, columns) = stack_of(6, 3, 4); + StackedCommitment::::commit( + layout, + &crate::stacking::borrow(&columns), + None, + &config(), + ) + .unwrap() + .roots() + }) + }; + let pool = roots(false); + assert_eq!(pool.len(), 3); + assert_eq!(roots(true), pool); + assert!(!off_pool()); + with_pairs_off_pool(true, || { + with_pairs_off_pool(false, || assert!(!off_pool())); + assert!(off_pool()); + }); + assert!(!off_pool()); + } + + /// Off the pool, a pair runs on its caller's thread and one of its own, never a + /// pool thread, in order; an error in either is the pair's. + #[cfg(feature = "parallel")] + #[test] + fn a_pair_off_the_pool_runs_on_threads_of_its_own() { + let caller = std::thread::current().id(); + let seen = std::sync::Mutex::new(Vec::new()); + let commit = |p: &u32| -> Result { + seen.lock().unwrap().push(( + *p, + std::thread::current().id(), + rayon::current_thread_index(), + )); + if *p == 9 { + Err(Error::EmptyPolynomial) + } else { + Ok(p * 10) + } + }; + assert_eq!( + commit_pair_off_pool(&[1, 2], &commit).unwrap(), + vec![10, 20] + ); + assert_eq!(commit_pair_off_pool(&[3], &commit).unwrap(), vec![30]); + assert!(commit_pair_off_pool(&[1, 9], &commit).is_err()); + assert!(commit_pair_off_pool(&[9, 1], &commit).is_err()); + let seen = seen.into_inner().unwrap(); + assert!(seen.iter().all(|&(_, _, pool)| pool.is_none()), "{seen:?}"); + let thread_of = |p| seen.iter().find(|s| s.0 == p).unwrap().1; + assert_eq!(thread_of(1), caller); + assert_ne!(thread_of(2), caller); + assert_eq!(thread_of(3), caller); + } + + /// The point of the module: many columns, one commitment, one opening. + #[test] + fn every_column_settles_against_one_commitment() { + let num_vars = 3; + let (layout, columns) = stack_of(6, num_vars, 6); + assert_eq!(layout.num_polys(), 1); + + let at = point(num_vars); + let claimed = values(&columns, &at); + assert_eq!(run(layout, &columns, &at, &claimed).unwrap(), 1); + } + + #[test] + fn columns_overflowing_the_stack_use_more_polynomials() { + let num_vars = 3; + // Two slots per stacked polynomial, so six columns need three. + let (layout, columns) = stack_of(6, num_vars, 4); + assert_eq!(layout.num_polys(), 3); + + let at = point(num_vars); + let claimed = values(&columns, &at); + assert_eq!(run(layout, &columns, &at, &claimed).unwrap(), 3); + } + + #[test] + fn unused_slots_do_not_disturb_the_claims() { + // Three columns in a four-slot stack: the padding is zero and carries no + // weight. + let num_vars = 3; + let (layout, columns) = stack_of(3, num_vars, 5); + assert_eq!(layout.num_polys(), 1); + + let at = point(num_vars); + let claimed = values(&columns, &at); + run(layout, &columns, &at, &claimed).unwrap(); + } + + #[test] + fn a_forged_column_value_is_rejected() { + let num_vars = 3; + let (layout, columns) = stack_of(4, num_vars, 5); + let at = point(num_vars); + let mut claimed = values(&columns, &at); + claimed[2] += FE::one(); + + assert!(run(layout, &columns, &at, &claimed).is_err()); + } + + #[test] + fn swapping_two_column_values_is_rejected() { + let num_vars = 3; + let (layout, columns) = stack_of(4, num_vars, 5); + let at = point(num_vars); + let mut claimed = values(&columns, &at); + claimed.swap(0, 1); + + assert!(run(layout, &columns, &at, &claimed).is_err()); + } + + /// Right values, wrong point: the weight pins where they were taken. + #[test] + fn values_from_another_point_are_rejected() { + let num_vars = 3; + let (layout, columns) = stack_of(4, num_vars, 5); + let at = point(num_vars); + let elsewhere: Vec = at.iter().map(|x| x + FE::one()).collect(); + let claimed = values(&columns, &elsewhere); + + assert!(run(layout, &columns, &at, &claimed).is_err()); + } + + /// The crux: the prover folds the weight's table and the verifier evaluates + /// its closed form, so they must be the same polynomial. + #[test] + fn the_weight_table_matches_its_closed_form() { + let num_vars = 2; + let n_stack = 4; + let (layout, _) = stack_of(3, num_vars, n_stack); + let at = point(num_vars); + let weights: Vec = (0..3).map(|i| FE::from(3 + i as u64)).collect(); + + let claimed = Claimed::Shared(&at); + let shares = weight_shares(&layout, 0, &claimed, &weights).unwrap(); + let table = weight_table(&shares, layout.n_stack()).unwrap(); + let off_cube: Vec = (0..n_stack).map(|i| FE::from(31 + i as u64)).collect(); + assert_eq!( + table.evaluate(&off_cube).unwrap(), + weight_at(&layout, 0, &Claimed::Shared(&at), &weights, &off_cube).unwrap() + ); + } + + /// And the weight really does read each column out of its own subcube. + #[test] + fn the_weight_picks_each_column_out_of_its_slot() { + let num_vars = 2; + let (layout, columns) = stack_of(3, num_vars, 4); + let at = point(num_vars); + let weights: Vec = (0..3).map(|i| FE::from(3 + i as u64)).collect(); + + let stacked = layout.stack(&crate::stacking::borrow(&columns)).unwrap(); + let claimed = Claimed::Shared(&at); + let shares = weight_shares(&layout, 0, &claimed, &weights).unwrap(); + let table = weight_table(&shares, layout.n_stack()).unwrap(); + // Σ_x w(x)·stacked(x) must be the batched column values. + let summed = table + .evals() + .iter() + .zip(stacked[0].evals()) + .fold(FE::zero(), |acc, (w, v)| acc + w * v); + let expected = values(&columns, &at) + .iter() + .zip(&weights) + .fold(FE::zero(), |acc, (v, w)| acc + v * w); + assert_eq!(summed, expected); + } + + #[test] + fn mixed_heights_in_one_stack_are_rejected() { + let layout = StackedLayout::build(&[3, 2], 5).unwrap(); + let weights = [FE::one(), FE::one()]; + assert!(matches!( + weight_shares(&layout, 0, &Claimed::Shared(&point(3)), &weights).err(), + Some(Error::VariableCountMismatch { .. }) + )); + } + + #[test] + fn a_value_per_column_is_required() { + let num_vars = 3; + let (layout, columns) = stack_of(4, num_vars, 5); + let at = point(num_vars); + let claimed = values(&columns, &at); + + let stacked = StackedCommitment::::commit( + layout, + &crate::stacking::borrow(&columns), + None, + &config(), + ) + .unwrap(); + assert!(matches!( + prove( + &stacked, + &crate::stacking::borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed[..3], + &config(), + &mut transcript() + ) + .err(), + Some(Error::QueryCountMismatch { + expected: 4, + got: 3 + }) + )); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let num_vars = 3; + let (layout, columns) = stack_of(4, num_vars, 5); + let at = point(num_vars); + let claimed = values(&columns, &at); + + let stacked = StackedCommitment::::commit( + layout, + &crate::stacking::borrow(&columns), + None, + &config(), + ) + .unwrap(); + let roots = stacked.roots(); + let proof = prove( + &stacked, + &crate::stacking::borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed, + &config(), + &mut transcript(), + ) + .unwrap(); + + let mut other = DefaultTranscript::::new(b"a-different-statement"); + assert!( + verify::( + &proof, + stacked.layout(), + &roots, + &Claimed::Shared(&at), + &claimed, + stacked.domain(), + &config(), + &mut other, + ) + .is_err() + ); + } + + /// The field tower in use: a base-field domain with extension-valued + /// columns. + #[test] + fn the_stack_settles_over_a_field_tower() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + type ExtE = FieldElement; + + let num_vars = 3; + let count = 5; + // The columns are base-field, like a trace; the point is not. + let columns: Vec> = (0..count) + .map(|i| { + Mle::new( + (0..(1u64 << num_vars)) + .map(|r| FE::from(r * 7 + 1 + i as u64 * 100)) + .collect(), + ) + .unwrap() + }) + .collect(); + let layout = StackedLayout::build(&vec![num_vars; count], 6).unwrap(); + let at: Vec = (0..num_vars).map(|i| ExtE::from(101 + i as u64)).collect(); + let claimed: Vec = columns + .iter() + .map(|c| c.evaluate_in(&at).unwrap()) + .collect(); + + let stacked = StackedCommitment::::commit( + layout, + &crate::stacking::borrow(&columns), + None, + &config(), + ) + .unwrap(); + let roots = stacked.roots(); + assert_eq!(roots.len(), 1); + + let mut prover = DefaultTranscript::::new(b"tower"); + let proof = prove::( + &stacked, + &crate::stacking::borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed, + &config(), + &mut prover, + ) + .unwrap(); + + let mut verifier = DefaultTranscript::::new(b"tower"); + verify::( + &proof, + stacked.layout(), + &roots, + &Claimed::Shared(&at), + &claimed, + stacked.domain(), + &config(), + &mut verifier, + ) + .unwrap(); + } + + /// Columns of **different heights**, each claimed at **its own point**, + /// settled against one commitment. + /// + /// This is what stacking separate tables together needs and what a shared + /// point cannot express: their sumchecks end wherever they end. The + /// weight stops factoring into slots times a single `eq`, so the check + /// that it still describes the same claim is the whole point of the test. + #[test] + fn columns_of_different_heights_settle_at_their_own_points() { + let columns = vec![column(3, 7), column(1, 11), column(2, 13)]; + let n_stack = 4; + let layout = StackedLayout::build(&[3, 1, 2], n_stack).unwrap(); + + // A point per column, each of that column's own width. + let points: Vec> = vec![ + vec![FE::from(2), FE::from(3), FE::from(5)], + vec![FE::from(7)], + vec![FE::from(11), FE::from(13)], + ]; + let claimed: Vec = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate(p).unwrap()) + .collect(); + + let stacked = StackedCommitment::::commit( + layout, + &crate::stacking::borrow(&columns), + None, + &config(), + ) + .unwrap(); + let roots = stacked.roots(); + let at = Claimed::PerColumn(&points); + + let proof = prove( + &stacked, + &crate::stacking::borrow(&columns), + None, + &at, + &claimed, + &config(), + &mut transcript(), + ) + .unwrap(); + verify::( + &proof, + stacked.layout(), + &roots, + &at, + &claimed, + stacked.domain(), + &config(), + &mut transcript(), + ) + .unwrap(); + + // And a wrong value for one column is rejected, so the per-point + // weight is really tying each claim to its own column. + let mut tampered = claimed.clone(); + tampered[1] += FE::one(); + let proof = prove( + &stacked, + &crate::stacking::borrow(&columns), + None, + &at, + &tampered, + &config(), + &mut transcript(), + ) + .unwrap(); + assert!( + verify::( + &proof, + stacked.layout(), + &roots, + &at, + &tampered, + stacked.domain(), + &config(), + &mut transcript(), + ) + .is_err() + ); + } +} diff --git a/crypto/multilinear/src/stacking.rs b/crypto/multilinear/src/stacking.rs new file mode 100644 index 000000000..5caa1d0ec --- /dev/null +++ b/crypto/multilinear/src/stacking.rs @@ -0,0 +1,421 @@ +//! Packs columns of differing heights into shared `2^n_stack` polynomials. +//! +//! Each column of height `2^m` sits at an offset that is a multiple of `2^m`, +//! making it a subcube: `column(z) = stacked(prefix_bits ‖ z)`, with no protocol +//! needed. The price is alignment padding; registering widest-first minimizes it. + +use math::field::{element::FieldElement, traits::IsField}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{Error, mle::Mle}; + +/// Where one column ended up. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Placement { + /// Index of the stacked polynomial holding this column. + pub poly: usize, + /// Start offset inside that polynomial. Always a multiple of `2^num_vars`. + pub offset: usize, + /// The column's own variable count; its height is `2^num_vars`. + pub num_vars: usize, + /// Variables of the stacked polynomial, `>= num_vars`. + pub n_stack: usize, +} + +impl Placement { + /// The high bits that select this column's subcube, most significant first. + pub fn prefix_bits(&self) -> Vec { + let prefix_len = self.n_stack - self.num_vars; + let index = self.offset >> self.num_vars; + (0..prefix_len) + .map(|i| (index >> (prefix_len - 1 - i)) & 1 == 1) + .collect() + } + + /// Lifts a point on this column's cube to a point on the stacked cube. + pub fn point_in_stacked( + &self, + point: &[FieldElement], + ) -> Result>, Error> { + if point.len() != self.num_vars { + return Err(Error::VariableCountMismatch { + expected: self.num_vars, + got: point.len(), + }); + } + let mut out: Vec> = self + .prefix_bits() + .into_iter() + .map(|b| { + if b { + FieldElement::::one() + } else { + FieldElement::::zero() + } + }) + .collect(); + out.extend_from_slice(point); + Ok(out) + } +} + +/// A fixed, public assignment of columns to stacked polynomials. +/// +/// Both sides derive the same layout from the same column heights, so it never +/// travels in a proof. +#[derive(Clone, Debug)] +pub struct StackedLayout { + n_stack: usize, + placements: Vec, + num_polys: usize, +} + +impl StackedLayout { + /// Places columns of the given heights, in the order given. + /// + /// A column that does not fit in the current polynomial starts a new one; a + /// column taller than the stacked cube is an error. + pub fn build(heights_log2: &[usize], n_stack: usize) -> Result { + let capacity = 1usize << n_stack; + let mut placements = Vec::with_capacity(heights_log2.len()); + let mut poly = 0usize; + let mut cursor = 0usize; + + for &m in heights_log2 { + if m > n_stack { + return Err(Error::ColumnTallerThanStack { + column_vars: m, + n_stack, + }); + } + let size = 1usize << m; + // Round the cursor up to this column's alignment. + let aligned = cursor.div_ceil(size) * size; + let (poly, offset) = if aligned + size > capacity { + poly += 1; + cursor = size; + (poly, 0) + } else { + cursor = aligned + size; + (poly, aligned) + }; + placements.push(Placement { + poly, + offset, + num_vars: m, + n_stack, + }); + } + + let num_polys = placements_poly_count(&placements); + Ok(Self { + n_stack, + placements, + num_polys, + }) + } + + pub fn n_stack(&self) -> usize { + self.n_stack + } + + pub fn num_polys(&self) -> usize { + self.num_polys + } + + pub fn placements(&self) -> &[Placement] { + &self.placements + } + + pub fn placement(&self, column: usize) -> Option<&Placement> { + self.placements.get(column) + } + + /// The columns that make up stacked polynomial `poly`, each with the offset + /// it sits at. + /// + /// What a stacked polynomial *is*, without building it: the commit writes + /// these straight into the device buffer instead of assembling a copy here + /// first, and the opening does the same. + pub fn parts_of(&self, poly: usize) -> Vec<(usize, usize)> { + self.placements + .iter() + .enumerate() + .filter(|(_, place)| place.poly == poly) + .map(|(column, place)| (column, place.offset)) + .collect() + } + + /// Cells that carry data, versus the `num_polys · 2^n_stack` committed. + pub fn occupancy(&self) -> (usize, usize) { + let used: usize = self.placements.iter().map(|p| 1usize << p.num_vars).sum(); + (used, self.num_polys * (1usize << self.n_stack)) + } + + /// Builds the stacked polynomials, zero-filling the padding. + /// + /// One polynomial per worker: they share no cells, and between the zeroing + /// and the copies this is gigabytes of memory traffic on a trace of any + /// size. + pub fn stack(&self, columns: &[&Mle]) -> Result>, Error> { + if columns.len() != self.placements.len() { + return Err(Error::VariableCountMismatch { + expected: self.placements.len(), + got: columns.len(), + }); + } + for (column, place) in columns.iter().zip(&self.placements) { + if column.len() != 1usize << place.num_vars { + return Err(Error::NotPowerOfTwo(column.len())); + } + } + let size = 1usize << self.n_stack; + // Which columns land in which polynomial, so a worker owning one + // polynomial has its whole list without scanning the placements. + let mut by_poly: Vec> = vec![Vec::new(); self.num_polys]; + for (index, place) in self.placements.iter().enumerate() { + by_poly[place.poly].push(index); + } + let fill = |members: &Vec| { + let mut buffer = vec![FieldElement::::zero(); size]; + for index in members { + let place = &self.placements[*index]; + let span = 1usize << place.num_vars; + buffer[place.offset..place.offset + span].clone_from_slice(columns[*index].evals()); + } + buffer + }; + #[cfg(feature = "parallel")] + let polys: Vec>> = by_poly.par_iter().map(fill).collect(); + #[cfg(not(feature = "parallel"))] + let polys: Vec>> = by_poly.iter().map(fill).collect(); + + polys.into_iter().map(Mle::new).collect() + } +} + +/// Every column, by reference — what [`stack`](StackedLayout::stack) reads. +pub fn borrow(columns: &[Mle]) -> Vec<&Mle> { + columns.iter().collect() +} + +fn placements_poly_count(placements: &[Placement]) -> usize { + placements.iter().map(|p| p.poly + 1).max().unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn column(len: usize, seed: u64) -> Mle { + Mle::new( + (0..len as u64) + .map(|i| FE::from(i.wrapping_mul(2654435761).wrapping_add(seed))) + .collect(), + ) + .unwrap() + } + + /// Heights in the shape our tables actually have, scaled down. + const REAL_SHAPE: [usize; 6] = [5, 5, 6, 6, 4, 0]; + + #[test] + fn every_offset_is_aligned_to_its_own_height() { + let layout = StackedLayout::build(&REAL_SHAPE, 8).unwrap(); + for place in layout.placements() { + assert_eq!( + place.offset % (1usize << place.num_vars), + 0, + "offset {} is not aligned to 2^{}", + place.offset, + place.num_vars + ); + } + } + + #[test] + fn placements_never_overlap() { + let layout = StackedLayout::build(&REAL_SHAPE, 8).unwrap(); + let mut occupied: Vec<(usize, usize, usize)> = layout + .placements() + .iter() + .map(|p| (p.poly, p.offset, p.offset + (1usize << p.num_vars))) + .collect(); + occupied.sort(); + for pair in occupied.windows(2) { + let (poly_a, _, end_a) = pair[0]; + let (poly_b, start_b, _) = pair[1]; + if poly_a == poly_b { + assert!(end_a <= start_b, "ranges overlap: {:?}", pair); + } + } + } + + #[test] + fn everything_fits_inside_the_stacked_cube() { + let n_stack = 8; + let layout = StackedLayout::build(&REAL_SHAPE, n_stack).unwrap(); + for place in layout.placements() { + assert!(place.offset + (1usize << place.num_vars) <= 1usize << n_stack); + } + } + + #[test] + fn a_column_taller_than_the_cube_is_rejected() { + let err = StackedLayout::build(&[3, 9], 8).unwrap_err(); + assert_eq!( + err, + Error::ColumnTallerThanStack { + column_vars: 9, + n_stack: 8 + } + ); + } + + #[test] + fn overflowing_columns_open_another_polynomial() { + // Four columns of 2^7 need two cubes of 2^8. + let layout = StackedLayout::build(&[7, 7, 7, 7], 8).unwrap(); + assert_eq!(layout.num_polys(), 2); + assert_eq!(layout.placement(0).unwrap().poly, 0); + assert_eq!(layout.placement(2).unwrap().poly, 1); + assert_eq!(layout.placement(2).unwrap().offset, 0); + } + + #[test] + fn stacked_cells_land_where_the_layout_says() { + let layout = StackedLayout::build(&REAL_SHAPE, 8).unwrap(); + let columns: Vec> = REAL_SHAPE + .iter() + .enumerate() + .map(|(i, &m)| column(1 << m, i as u64 + 1)) + .collect(); + let stacked = layout.stack(&borrow(&columns)).unwrap(); + + for (col_idx, place) in layout.placements().iter().enumerate() { + for (j, cell) in columns[col_idx].evals().iter().enumerate() { + assert_eq!( + stacked[place.poly].evals()[place.offset + j], + *cell, + "column {col_idx}, cell {j}" + ); + } + } + } + + #[test] + fn padding_is_zero() { + let layout = StackedLayout::build(&[3, 1], 5).unwrap(); + let columns = vec![column(8, 1), column(2, 2)]; + let stacked = layout.stack(&borrow(&columns)).unwrap(); + + let (used, committed) = layout.occupancy(); + assert_eq!(used, 10); + assert_eq!(committed, 32); + + let zeros = stacked[0] + .evals() + .iter() + .filter(|v| **v == FE::zero()) + .count(); + // The 22 padding cells are zero (a data cell could coincidentally be + // zero too, so this is a lower bound the layout must clear). + assert!(zeros >= 22, "expected at least 22 zeros, got {zeros}"); + } + + /// The property the whole construction exists for. + #[test] + fn a_column_evaluation_is_the_stacked_evaluation_at_the_lifted_point() { + let layout = StackedLayout::build(&REAL_SHAPE, 8).unwrap(); + let columns: Vec> = REAL_SHAPE + .iter() + .enumerate() + .map(|(i, &m)| column(1 << m, i as u64 + 1)) + .collect(); + let stacked = layout.stack(&borrow(&columns)).unwrap(); + + for (col_idx, place) in layout.placements().iter().enumerate() { + let mle = &columns[col_idx]; + // A point off the cube, where a wrong lift would show up. + let z: Vec = (0..place.num_vars) + .map(|k| FE::from(1000 + k as u64 + col_idx as u64)) + .collect(); + + let direct = mle.evaluate(&z).unwrap(); + let lifted = place.point_in_stacked(&z).unwrap(); + let via_stack = stacked[place.poly].evaluate(&lifted).unwrap(); + + assert_eq!(direct, via_stack, "column {col_idx}"); + } + } + + #[test] + fn the_lifted_point_has_the_stacked_arity() { + let layout = StackedLayout::build(&REAL_SHAPE, 8).unwrap(); + let place = layout.placement(4).unwrap(); + let z = vec![FE::from(3); place.num_vars]; + assert_eq!(place.point_in_stacked(&z).unwrap().len(), 8); + } + + #[test] + fn prefix_bits_are_the_offset_index_most_significant_first() { + // A 2^2 column at offset 20 in a 2^5 cube: 20/4 = 5 = 0b101. + let place = Placement { + poly: 0, + offset: 20, + num_vars: 2, + n_stack: 5, + }; + assert_eq!(place.prefix_bits(), vec![true, false, true]); + } + + #[test] + fn a_single_row_column_still_lifts() { + // HALT is one row: no variables of its own, all prefix. + let layout = StackedLayout::build(&[0], 4).unwrap(); + let columns = vec![Mle::new(vec![FE::from(42)]).unwrap()]; + let stacked = layout.stack(&borrow(&columns)).unwrap(); + let place = layout.placement(0).unwrap(); + + let lifted = place.point_in_stacked::(&[]).unwrap(); + assert_eq!(lifted.len(), 4); + assert_eq!(stacked[0].evaluate(&lifted).unwrap(), FE::from(42)); + } + + #[test] + fn lifting_a_point_of_the_wrong_arity_is_an_error() { + let layout = StackedLayout::build(&[3], 5).unwrap(); + let place = layout.placement(0).unwrap(); + assert_eq!( + place.point_in_stacked(&[FE::one()]).unwrap_err(), + Error::VariableCountMismatch { + expected: 3, + got: 1 + } + ); + } + + #[test] + fn widest_first_wastes_less_than_interleaved() { + // Same columns, two orders: alignment padding depends on the order. + let interleaved = StackedLayout::build(&[0, 4, 0, 4], 6).unwrap(); + let widest_first = StackedLayout::build(&[4, 4, 0, 0], 6).unwrap(); + + let end_of = |l: &StackedLayout| -> usize { + l.placements() + .iter() + .map(|p| p.offset + (1usize << p.num_vars)) + .max() + .unwrap() + }; + assert!( + end_of(&widest_first) < end_of(&interleaved), + "widest-first should pack tighter" + ); + } +} diff --git a/crypto/multilinear/src/sumcheck.rs b/crypto/multilinear/src/sumcheck.rs new file mode 100644 index 000000000..37a39c41c --- /dev/null +++ b/crypto/multilinear/src/sumcheck.rs @@ -0,0 +1,609 @@ +//! The sumcheck protocol, reducing `Σ_x f(x) = S` to one evaluation of `f`. +//! +//! Round polynomials travel as evaluations at `1, .., d`. `g(0)` is **not** +//! sent: `g(0) + g(1)` is the claim carried into the round, which fixes it. So +//! the prover skips a whole pass over the cube and the proof loses one field +//! element per round — and the rejection that used to happen per round now +//! happens **only** against the final claim, which [`verify`] returns and the +//! caller must discharge. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{element::FieldElement, traits::IsField}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{Error, mle::Mle, poly::SumcheckPolynomial}; + +/// One round: the round polynomial as evaluations at `1, .., degree`. +/// +/// `g(0)` is absent by construction — see the module docs. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct RoundProof { + pub evaluations: Vec>, +} + +/// A full sumcheck transcript. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct SumcheckProof { + pub rounds: Vec>, +} + +/// What the verifier is left holding: `f(point)` must equal `expected_evaluation`. +/// +/// Discharging this is the **only** place a sumcheck rejects, so dropping it +/// silently accepts anything. +#[must_use] +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SumcheckClaim { + pub point: Vec>, + pub expected_evaluation: FieldElement, +} + +/// Lagrange-interpolates values at `0, 1, .., d` and evaluates at `x`. +/// +/// Round polynomials are small, so the quadratic form is the cheap one. The +/// denominators depend only on how many nodes there are, so they go through one +/// batched inversion instead of one each — and the quadratic numerators are +/// kept rather than switching to the barycentric form, which would divide by +/// `x − x_i` and so need a special case for an `x` that lands on a node. +pub(crate) fn interpolate( + values: &[FieldElement], + x: &FieldElement, +) -> FieldElement { + let n = values.len(); + let node = |j: usize| FieldElement::::from(j as u64); + let others = |i: usize, at: &FieldElement| { + (0..n) + .filter(|&j| j != i) + .fold(FieldElement::::one(), |acc, j| acc * (at - node(j))) + }; + + let mut denominators: Vec> = (0..n).map(|i| others(i, &node(i))).collect(); + // The nodes are distinct, so none of them is zero. + FieldElement::inplace_batch_inverse(&mut denominators).expect("distinct interpolation nodes"); + + values + .iter() + .zip(&denominators) + .enumerate() + .fold(FieldElement::zero(), |acc, (i, (y_i, inv))| { + acc + y_i * others(i, x) * inv + }) +} + +/// Sums `f(r_0..r_{j-1}, t, rest)` over the remaining cube. +/// Sums `f(r_0..r_{j-1}, t, rest)` over the remaining cube. +/// +/// `t` runs over `1..=degree`, or `0..=degree` when `with_zero` — which only +/// the prover's debug self-check asks for. +/// +/// `poly` has already been folded on the earlier variables, so its first +/// variable is the one this round binds. +/// +/// One pass over the cube serves **every** `t`: each factor's `lo` and `hi` are +/// read once per index and the extensions come off `hi - lo`, rather than +/// re-reading the tables once per `t`. On a real trace the factors are hundreds +/// of megabytes, so the reads are the cost, not the arithmetic. +pub(crate) fn round_evaluations( + poly: &P, + degree: usize, + with_zero: bool, +) -> Vec> +where + F: IsField + 'static, + P: SumcheckPolynomial + Sync, + FieldElement: Send + Sync, +{ + let half = 1usize << (poly.num_vars() - 1); + let first = usize::from(!with_zero); + let steps: Vec> = (first..=degree) + .map(|t| FieldElement::::from(t as u64)) + .collect(); + + // A slice of the cube, summed independently. The rounds are the whole cost + // of the prover, and every index is independent, so this is where the cores + // go in. + let slice = |range: std::ops::Range| -> Vec> { + let width = poly.polys().len(); + // Buffers for the whole slice. Collecting a fresh `Vec` per cube index + // would be one heap allocation per index, which on a real trace is + // millions of them per round and dwarfs the arithmetic. + let mut totals = vec![FieldElement::::zero(); steps.len()]; + let mut lo = vec![FieldElement::::zero(); width]; + let mut hi = vec![FieldElement::::zero(); width]; + let mut delta = vec![FieldElement::::zero(); width]; + let mut values = vec![FieldElement::::zero(); width]; + // A compiled rule's steps go here, once for the whole slice. + let mut scratch = Vec::new(); + + for j in range { + for (k, p) in poly.polys().iter().enumerate() { + lo[k] = p.evals()[j].clone(); + hi[k] = p.evals()[j + half].clone(); + delta[k] = &hi[k] - &lo[k]; + } + for (total, t) in totals.iter_mut().zip(&steps) { + // `t = 0` and `t = 1` are the halves as they are, so they skip + // the multiplication entirely. + if t == &FieldElement::::zero() { + values.clone_from(&lo); + } else if t == &FieldElement::::one() { + values.clone_from(&hi); + } else { + for (v, (l, d)) in values.iter_mut().zip(lo.iter().zip(&delta)) { + *v = l + t * d; + } + } + *total += poly.combine_in(&values, &mut scratch); + } + } + totals + }; + + let add = |mut acc: Vec>, part: Vec>| { + for (a, p) in acc.iter_mut().zip(part) { + *a += p; + } + acc + }; + + #[cfg(feature = "parallel")] + { + if half < crate::SERIAL_BELOW { + return slice(0..half); + } + let chunk = half.div_ceil(rayon::current_num_threads().max(1)); + (0..half) + .into_par_iter() + .step_by(chunk) + .map(|start| slice(start..(start + chunk).min(half))) + .reduce(|| vec![FieldElement::::zero(); steps.len()], add) + } + #[cfg(not(feature = "parallel"))] + { + let _ = add; + slice(0..half) + } +} + +/// The round polynomial's evaluations at `1..=degree` for one compiled rule +/// over `factors`, through the host round loop. +/// +/// This is the reference the device round is checked against +/// (`math-cuda/tests/sumcheck.rs`): the comparison has to be against the loop +/// the prover actually runs, not a second copy of the formula. +pub fn round_evaluations_for_program( + factors: &[Mle], + program: &crate::program::Program, + degree: usize, +) -> Result>, Error> +where + F: IsField + 'static, + FieldElement: Send + Sync, +{ + let batched = crate::batch::Batched::new( + factors.to_vec(), + vec![crate::batch::Rule::compiled(degree, program.clone())], + vec![FieldElement::one()], + )?; + Ok(round_evaluations(&batched, degree, false)) +} + +/// Runs the prover, absorbing each round polynomial and drawing each challenge +/// from `transcript`. +/// +/// Returns the proof and the challenge point. The caller is responsible for +/// having absorbed the claimed sum and any statement binding beforehand. +pub fn prove( + mut poly: P, + transcript: &mut T, +) -> Result<(SumcheckProof, Vec>), Error> +where + F: IsField + 'static, + T: IsTranscript, + P: SumcheckPolynomial + Sync, + FieldElement: Send + Sync, +{ + let num_vars = poly.num_vars(); + let (rounds, challenges) = prove_rounds(&mut poly, num_vars, transcript)?; + Ok((SumcheckProof { rounds }, challenges)) +} + +/// A group of round polynomials and the challenges they drew. +pub type RoundGroup = (Vec>, Vec>); + +/// Runs `rounds` rounds, leaving `poly` folded on them. +/// +/// A chained WHIR interleaves: between groups of rounds it folds its codeword +/// and commits the successor, so it cannot run the whole sumcheck in one call. +pub fn prove_rounds( + poly: &mut P, + rounds: usize, + transcript: &mut T, +) -> Result, Error> +where + F: IsField + 'static, + T: IsTranscript, + P: SumcheckPolynomial + Sync, + FieldElement: Send + Sync, +{ + if rounds > poly.num_vars() { + return Err(Error::RoundCountMismatch { + expected: poly.num_vars(), + got: rounds, + }); + } + let degree = poly.degree().max(1); + + // The rounds on device when the polynomial says which program it is and the + // device takes it. Only a whole sumcheck: a group of rounds leaves tables + // the caller needs back, and downloading them between groups costs more + // than the rounds do. + let attempt = if rounds == poly.num_vars() { + match poly.program() { + Some(program) => { + let cube = poly.host_cube(); + crate::gpu::prove_sumcheck(poly.polys(), program, degree, cube, |evaluations| { + for e in evaluations { + transcript.append_field_element(e); + } + transcript.sample_field_element() + }) + } + None => None, + } + } else { + None + }; + let mut proofs = Vec::with_capacity(rounds); + let mut challenges = Vec::with_capacity(rounds); + // What a device ran of this, if it ran any. It stops where the cube stops + // being worth sending and hands the factors back folded, so what is left + // carries on below from exactly where it left off. + if let Some(outcome) = attempt { + let (device_proofs, device_challenges, folded) = outcome?; + poly.accept_folded(folded)?; + proofs = device_proofs; + challenges = device_challenges; + } + let rounds = rounds - proofs.len(); + // The identity the verifier now takes on faith. Checking it costs the pass + // over the cube the protocol exists to skip, so it runs in debug only — + // where it turns a silent prover bug into a local failure. + #[cfg(debug_assertions)] + let mut running: Option> = None; + + for _ in 0..rounds { + let all = round_evaluations(poly, degree, cfg!(debug_assertions)); + let sent = all[all.len() - degree..].to_vec(); + for e in &sent { + transcript.append_field_element(e); + } + let r = transcript.sample_field_element(); + + #[cfg(debug_assertions)] + { + let sum = &all[0] + &all[1]; + if let Some(expected) = &running { + debug_assert_eq!( + &sum, expected, + "sumcheck: g(0) + g(1) is not the claim carried into the round" + ); + } + running = Some(interpolate(&all, &r)); + } + + poly.fix_first_variable(&r)?; + proofs.push(RoundProof { evaluations: sent }); + challenges.push(r); + } + + Ok((proofs, challenges)) +} + +/// Checks every round against the running claim and returns the final claim. +/// +/// Verifying `claimed_sum` requires one more step the caller must perform: +/// obtain `f` at [`SumcheckClaim::point`] and compare it against +/// [`SumcheckClaim::expected_evaluation`]. +pub fn verify( + proof: &SumcheckProof, + claimed_sum: FieldElement, + num_vars: usize, + degree: usize, + transcript: &mut T, +) -> Result, Error> +where + F: IsField, + T: IsTranscript, +{ + if proof.rounds.len() != num_vars { + return Err(Error::RoundCountMismatch { + expected: num_vars, + got: proof.rounds.len(), + }); + } + verify_rounds(&proof.rounds, claimed_sum, degree, transcript) +} + +/// Verifies a group of rounds against a running claim. +/// +/// The returned claim carries this group's challenges and the claim it leaves, +/// which is what the next group starts from. Round indices in errors are +/// relative to the group. +pub fn verify_rounds( + rounds: &[RoundProof], + claimed_sum: FieldElement, + degree: usize, + transcript: &mut T, +) -> Result, Error> +where + F: IsField, + T: IsTranscript, +{ + let degree = degree.max(1); + + let mut current = claimed_sum; + let mut point = Vec::with_capacity(rounds.len()); + + for (round, r_proof) in rounds.iter().enumerate() { + if r_proof.evaluations.len() != degree { + return Err(Error::RoundDegreeMismatch { + round, + expected: degree, + got: r_proof.evaluations.len(), + }); + } + // `g(0)` is recovered rather than checked: `g(0) + g(1)` is the claim + // carried in. So nothing is rejected here, and everything rides on the + // claim this returns. + let mut all = Vec::with_capacity(degree + 1); + all.push(¤t - &r_proof.evaluations[0]); + all.extend(r_proof.evaluations.iter().cloned()); + + for e in &r_proof.evaluations { + transcript.append_field_element(e); + } + let r = transcript.sample_field_element(); + + current = interpolate(&all, &r); + point.push(r); + } + + Ok(SumcheckClaim { + point, + expected_evaluation: current, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + mle::Mle, + virtual_poly::{Term, VirtualPolynomial}, + }; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"sumcheck-test") + } + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + /// Pseudo-random table of `2^n` entries, deterministic across runs. + fn pseudo_mle(n: usize, seed: u64) -> Mle { + let vals: Vec = (0..(1u64 << n)) + .map(|i| (i.wrapping_mul(6364136223846793005).wrapping_add(seed)) >> 11) + .collect(); + mle(&vals) + } + + #[test] + fn interpolation_reproduces_the_nodes() { + let values: Vec = [3u64, 1, 4, 1].iter().map(|v| FE::from(*v)).collect(); + for (i, v) in values.iter().enumerate() { + assert_eq!(interpolate(&values, &FE::from(i as u64)), *v); + } + } + + #[test] + fn interpolation_of_a_line_is_affine() { + // g(t) = 5 + 2t sampled at 0,1 must give 5 + 2·7 at t = 7. + let values = vec![FE::from(5), FE::from(7)]; + assert_eq!(interpolate(&values, &FE::from(7)), FE::from(19)); + } + + #[test] + fn linear_polynomial_round_trips() { + let f = VirtualPolynomial::new(vec![pseudo_mle(4, 1)], vec![Term::single(0)]).unwrap(); + let claimed = f.sum_over_hypercube(); + let num_vars = f.num_vars(); + let degree = f.degree(); + + let (proof, _) = prove(f.clone(), &mut transcript()).unwrap(); + let claim = verify(&proof, claimed, num_vars, degree, &mut transcript()).unwrap(); + + assert_eq!(f.evaluate(&claim.point).unwrap(), claim.expected_evaluation); + } + + #[test] + fn product_of_three_polynomials_round_trips() { + let f = VirtualPolynomial::new( + vec![pseudo_mle(5, 1), pseudo_mle(5, 2), pseudo_mle(5, 3)], + vec![Term::new(FE::from(7), vec![0, 1, 2])], + ) + .unwrap(); + let claimed = f.sum_over_hypercube(); + let (num_vars, degree) = (f.num_vars(), f.degree()); + assert_eq!(degree, 3); + + let (proof, challenges) = prove(f.clone(), &mut transcript()).unwrap(); + let claim = verify(&proof, claimed, num_vars, degree, &mut transcript()).unwrap(); + + // Prover and verifier must derive the same challenges from the transcript. + assert_eq!(challenges, claim.point); + assert_eq!(f.evaluate(&claim.point).unwrap(), claim.expected_evaluation); + } + + #[test] + fn sum_of_terms_of_mixed_degree_round_trips() { + let f = VirtualPolynomial::new( + vec![pseudo_mle(4, 10), pseudo_mle(4, 20), pseudo_mle(4, 30)], + vec![ + Term::new(FE::from(2), vec![0, 1]), + Term::new(FE::from(3), vec![2]), + Term::new(FE::from(5), vec![]), + ], + ) + .unwrap(); + let claimed = f.sum_over_hypercube(); + let (num_vars, degree) = (f.num_vars(), f.degree()); + + let (proof, _) = prove(f.clone(), &mut transcript()).unwrap(); + let claim = verify(&proof, claimed, num_vars, degree, &mut transcript()).unwrap(); + + assert_eq!(f.evaluate(&claim.point).unwrap(), claim.expected_evaluation); + } + + #[test] + fn single_variable_round_trips() { + let f = VirtualPolynomial::new(vec![mle(&[3, 11])], vec![Term::single(0)]).unwrap(); + let claimed = f.sum_over_hypercube(); + assert_eq!(claimed, FE::from(14)); + + let (proof, _) = prove(f.clone(), &mut transcript()).unwrap(); + let claim = verify(&proof, claimed, 1, 1, &mut transcript()).unwrap(); + assert_eq!(f.evaluate(&claim.point).unwrap(), claim.expected_evaluation); + } + + /// `g(0)` is derived from the claim, so a wrong claim is not caught in the + /// round — it is caught by the residual, which stops being the + /// polynomial's value. Every caller discharges that; this is the property + /// they rely on. + #[test] + fn a_wrong_claimed_sum_corrupts_the_residual() { + let f = VirtualPolynomial::new(vec![pseudo_mle(3, 1)], vec![Term::single(0)]).unwrap(); + let claimed = f.sum_over_hypercube(); + let (proof, _) = prove(f.clone(), &mut transcript()).unwrap(); + + let honest = verify(&proof, claimed, 3, 1, &mut transcript()).unwrap(); + assert_eq!( + f.evaluate(&honest.point).unwrap(), + honest.expected_evaluation + ); + + let lied = verify(&proof, claimed + FE::one(), 3, 1, &mut transcript()).unwrap(); + assert_ne!(f.evaluate(&lied.point).unwrap(), lied.expected_evaluation); + } + + #[test] + fn a_tampered_round_polynomial_corrupts_the_residual() { + let f = VirtualPolynomial::new( + vec![pseudo_mle(4, 1), pseudo_mle(4, 2)], + vec![Term::new(FE::one(), vec![0, 1])], + ) + .unwrap(); + let claimed = f.sum_over_hypercube(); + let (mut proof, _) = prove(f.clone(), &mut transcript()).unwrap(); + + proof.rounds[2].evaluations[0] += FE::one(); + let claim = verify(&proof, claimed, 4, 2, &mut transcript()).unwrap(); + assert_ne!(f.evaluate(&claim.point).unwrap(), claim.expected_evaluation); + } + + #[test] + fn a_proof_with_the_wrong_round_count_is_rejected() { + let f = VirtualPolynomial::new(vec![pseudo_mle(3, 1)], vec![Term::single(0)]).unwrap(); + let claimed = f.sum_over_hypercube(); + let (mut proof, _) = prove(f, &mut transcript()).unwrap(); + proof.rounds.pop(); + + let err = verify(&proof, claimed, 3, 1, &mut transcript()).unwrap_err(); + assert_eq!( + err, + Error::RoundCountMismatch { + expected: 3, + got: 2 + } + ); + } + + #[test] + fn a_round_polynomial_of_the_wrong_degree_is_rejected() { + let f = VirtualPolynomial::new(vec![pseudo_mle(3, 1)], vec![Term::single(0)]).unwrap(); + let claimed = f.sum_over_hypercube(); + let (mut proof, _) = prove(f, &mut transcript()).unwrap(); + proof.rounds[0].evaluations.push(FE::from(1)); + + let err = verify(&proof, claimed, 3, 1, &mut transcript()).unwrap_err(); + assert!(matches!(err, Error::RoundDegreeMismatch { round: 0, .. })); + } + + #[test] + fn a_different_transcript_seed_yields_a_different_point() { + // The challenge point is bound to the statement, not just to the + // polynomial: proving the same claim under a different seed must land + // somewhere else. + let f = VirtualPolynomial::new(vec![pseudo_mle(4, 1)], vec![Term::single(0)]).unwrap(); + + let (_, a) = prove(f.clone(), &mut transcript()).unwrap(); + let mut other = DefaultTranscript::::new(b"a-different-statement"); + let (_, b) = prove(f, &mut other).unwrap(); + + assert_ne!(a, b); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + // Fiat-Shamir binding: the verifier redraws challenges, so a proof + // lifted onto a different statement stops matching. + let f = VirtualPolynomial::new(vec![pseudo_mle(4, 1)], vec![Term::single(0)]).unwrap(); + let claimed = f.sum_over_hypercube(); + let (proof, _) = prove(f.clone(), &mut transcript()).unwrap(); + + let honest = verify(&proof, claimed, 4, 1, &mut transcript()).unwrap(); + assert_eq!( + f.evaluate(&honest.point).unwrap(), + honest.expected_evaluation + ); + + // Replayed, the verifier redraws different challenges, so the residual + // stops describing the polynomial. + let mut other = DefaultTranscript::::new(b"a-different-statement"); + let replayed = verify(&proof, claimed, 4, 1, &mut other).unwrap(); + assert_ne!( + f.evaluate(&replayed.point).unwrap(), + replayed.expected_evaluation + ); + } +} diff --git a/crypto/multilinear/src/uneven.rs b/crypto/multilinear/src/uneven.rs new file mode 100644 index 000000000..7371d3af5 --- /dev/null +++ b/crypto/multilinear/src/uneven.rs @@ -0,0 +1,611 @@ +//! Sumchecks over cubes of **different heights**, proved as one. +//! +//! [`batch`](crate::batch) shares the *factors* between statements over one +//! cube. This shares the *rounds* between statements over cubes that are not +//! the same size: `k` of them on `n_1, .., n_k` variables become one sumcheck +//! of `n = max n_i` rounds instead of `k` sumchecks of `Σ n_i` rounds between +//! them. What that buys is not arithmetic — a round still sums over every +//! statement's cube — it is the **per-round cost**, which for this prover is a +//! kernel launch, a synchronization and a transcript step, and which one +//! measurement put at about half of what the rounds cost at all. +//! +//! ## How a short statement fits on the big cube +//! +//! Statement `i` is read on `n` variables as the function that **ignores the +//! last `n − n_i`**: its table with each entry repeated `2^{n−n_i}` times. +//! That extension is still multilinear and nothing is padded in memory — the +//! statement keeps its own cube and the repetition is only how the rounds count +//! it: +//! +//! - its sum over the big cube is `2^{n−n_i}` times its own, so its claim +//! enters the batch scaled by a constant the verifier computes; +//! - while it still has variables it contributes **its own round polynomial**, +//! at the cost of its own cube and no more; +//! - once they are bound it is a constant, contributing `2^{rest} · v_i` to +//! every interpolation node — **degree 0**, and all the exhausted statements +//! together are one accumulator, not one term each. +//! +//! Statements are walked tallest first, so the ones still binding variables are +//! a prefix and the exhausted ones a suffix: a cut point that advances, not a +//! test per statement per round. +//! +//! ## What the point means +//! +//! Every statement ends bound at a **prefix** of the one challenge point: +//! statement `i` at `r_1..r_{n_i}`, the tallest at all of `r`. So the identity +//! the verifier is left with carries no scale at all, +//! `G(r) = Σ λ^i f_i(r_1..r_{n_i})` — the scale is in the claim, not in the +//! point — and the openings a caller still has to discharge are nested rather +//! than unrelated. +//! +//! ## Soundness +//! +//! Two terms on top of one sumcheck's. The claims are absorbed before `λ` is +//! drawn, so a prover that lies about any of them needs `λ` to be a root of a +//! nonzero polynomial of degree `k − 1`: at most `(k−1)/|F|`. The sumcheck over +//! the combination costs the usual `d·n/|F|`, with `d` the largest statement's +//! degree. Total `(k − 1 + d·n)/|F|`. +//! +//! Over the degree-3 extension of Goldilocks that field has about `2^192` +//! elements, so for this prover's shape — some tens of statements, degree under +//! ten, cubes of at most `2^21` — the whole term is below `2^-180`. The +//! batching does not move the security level; what fixes it stays the query and +//! grinding side of the commitment scheme. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{ + Error, challenge_powers, + poly::SumcheckPolynomial, + sumcheck::{self, RoundProof, SumcheckProof, round_evaluations}, +}; + +/// `2^k` as a field element. +fn two_to(k: usize) -> FieldElement { + FieldElement::::from(2).pow(k as u64) +} + +/// Absorbs the statement — the heights, then the claims — and expands the +/// batching challenge. +/// +/// Prover and verifier both come through here so the order is written once. +/// The heights scale the claims, so they are bound rather than assumed: a +/// caller that has already bound them elsewhere pays two hashes for it. +fn absorb_statement( + heights: &[usize], + claims: &[FieldElement], + transcript: &mut T, +) -> Vec> +where + F: IsField, + T: IsTranscript, +{ + let mut bytes = Vec::with_capacity(heights.len() * 8); + for height in heights { + bytes.extend_from_slice(&(*height as u64).to_le_bytes()); + } + transcript.append_bytes(&bytes); + for claim in claims { + transcript.append_field_element(claim); + } + challenge_powers(&transcript.sample_field_element(), claims.len()) +} + +/// What the big cube's sum comes to, given each statement's own. +fn claimed_sum( + heights: &[usize], + claims: &[FieldElement], + lambdas: &[FieldElement], + n: usize, +) -> FieldElement { + heights + .iter() + .zip(claims) + .zip(lambdas) + .fold(FieldElement::zero(), |acc, ((height, claim), lambda)| { + acc + lambda * claim * two_to::(n - height) + }) +} + +/// Proves every statement in one sumcheck of `max num_vars` rounds. +/// +/// `claims[i]` is what `Σ_x f_i(x)` over statement `i`'s **own** cube must come +/// to. They are absorbed before the batching challenge, so the prover cannot +/// pick a statement after seeing it. +/// +/// Returns the proof and the whole challenge point. `parts` is left folded: +/// statement `i` is bound at `point[..n_i]`, which is where the caller reads +/// the factor values the verifier will ask it to justify. +pub fn prove( + parts: &mut [P], + claims: &[FieldElement], + transcript: &mut T, +) -> Result<(SumcheckProof, Vec>), Error> +where + F: IsField + 'static, + T: IsTranscript, + P: SumcheckPolynomial + Sync, + FieldElement: Send + Sync, +{ + if parts.is_empty() { + return Err(Error::EmptyPolynomial); + } + if claims.len() != parts.len() { + return Err(Error::VariableCountMismatch { + expected: parts.len(), + got: claims.len(), + }); + } + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let n = heights.iter().copied().max().unwrap_or(0); + let degree = parts.iter().map(|p| p.degree()).max().unwrap_or(1).max(1); + let lambdas = absorb_statement(&heights, claims, transcript); + + // Tallest first, so the statements still binding variables are a prefix and + // the exhausted ones a suffix. + let mut order: Vec = (0..parts.len()).collect(); + order.sort_by_key(|&i| std::cmp::Reverse(heights[i])); + // What a statement's own round polynomial is worth on the big cube. The + // powers of lambda stay in the caller's order, which is the order the + // verifier recombines in; this sort is only how the rounds walk. + let weights: Vec> = heights + .iter() + .zip(&lambdas) + .map(|(height, lambda)| lambda * two_to::(n - height)) + .collect(); + + // The identity the verifier takes on faith. Checking it costs the pass over + // the cube the protocol exists to skip, so it runs in debug only — and here + // it is what catches a wrong scale, which is otherwise a proof that fails + // to verify with nothing pointing at why. + #[cfg(debug_assertions)] + let mut running = claimed_sum(&heights, claims, &lambdas, n); + let with_zero = cfg!(debug_assertions); + let width = degree + usize::from(with_zero); + + let mut active = parts.len(); + // Σ λ^i · v_i over the statements that have run out of variables. + let mut tail = FieldElement::::zero(); + let mut rounds = Vec::with_capacity(n); + let mut challenges = Vec::with_capacity(n); + + for round in 0..n { + // Whatever finished binding last round joins the tail, once and for all. + while active > 0 && parts[order[active - 1]].num_vars() == 0 { + active -= 1; + let i = order[active]; + tail += &lambdas[i] * parts[i].eval_at_index(0); + } + + let mut all = vec![FieldElement::::zero(); width]; + for &i in &order[..active] { + let evaluations = round_evaluations(&parts[i], degree, with_zero); + for (slot, value) in all.iter_mut().zip(evaluations) { + *slot += &weights[i] * value; + } + } + // The exhausted ones are one constant for the whole batch: their + // combined value over the cube the rounds after this one still sum. + if tail != FieldElement::zero() { + let rest = &tail * two_to::(n - round - 1); + for slot in all.iter_mut() { + *slot += rest.clone(); + } + } + + let sent = all[width - degree..].to_vec(); + for e in &sent { + transcript.append_field_element(e); + } + let r = transcript.sample_field_element(); + + #[cfg(debug_assertions)] + { + debug_assert_eq!( + &all[0] + &all[1], + running, + "uneven sumcheck: g(0) + g(1) is not the claim carried into round {round}" + ); + running = sumcheck::interpolate(&all, &r); + } + + for &i in &order[..active] { + parts[i].fix_first_variable(&r)?; + } + rounds.push(RoundProof { evaluations: sent }); + challenges.push(r); + } + + Ok((SumcheckProof { rounds }, challenges)) +} + +/// Checks the batch against the value each statement takes at its own prefix of +/// the point. +/// +/// `values_at` is handed the whole point and returns one value per statement: +/// statement `i`'s at `point[..heights[i]]`. Where those values come from — +/// another sumcheck, a commitment opening — is the caller's, and so is +/// discharging them; this only checks that they rebuild what the rounds left. +/// +/// Returns the point. +pub fn verify( + proof: &SumcheckProof, + heights: &[usize], + claims: &[FieldElement], + degree: usize, + values_at: V, + transcript: &mut T, +) -> Result>, Error> +where + F: IsField, + T: IsTranscript, + V: FnOnce(&[FieldElement]) -> Result>, Error>, +{ + if heights.is_empty() { + return Err(Error::EmptyPolynomial); + } + if claims.len() != heights.len() { + return Err(Error::VariableCountMismatch { + expected: heights.len(), + got: claims.len(), + }); + } + let n = heights.iter().copied().max().unwrap_or(0); + let lambdas = absorb_statement(heights, claims, transcript); + let claimed = claimed_sum(heights, claims, &lambdas, n); + let claim = sumcheck::verify(proof, claimed, n, degree.max(1), transcript)?; + + let values = values_at(&claim.point)?; + if values.len() != heights.len() { + return Err(Error::VariableCountMismatch { + expected: heights.len(), + got: values.len(), + }); + } + // No scale here: the big-cube reading of statement `i` at `r` is its own + // extension at `r_1..r_{n_i}`, because the variables it ignores are the + // ones past that. The scale rode in the claim. + let rebuilt = lambdas + .iter() + .zip(&values) + .fold(FieldElement::::zero(), |acc, (lambda, value)| { + acc + lambda * value + }); + if rebuilt != claim.expected_evaluation { + return Err(Error::BatchMismatch); + } + + Ok(claim.point) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + mle::Mle, + poly::Composed, + virtual_poly::{Term, VirtualPolynomial}, + }; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"uneven-test") + } + + /// Pseudo-random table of `2^n` entries, deterministic across runs. + fn pseudo_mle(n: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << n)) + .map(|i| FE::from((i.wrapping_mul(6364136223846793005).wrapping_add(seed)) >> 11)) + .collect(), + ) + .unwrap() + } + + /// `2·a·b` over two tables of `num_vars` variables — degree 2. + fn part(num_vars: usize, seed: u64) -> VirtualPolynomial { + VirtualPolynomial::new( + vec![pseudo_mle(num_vars, seed), pseudo_mle(num_vars, seed + 1)], + vec![Term::new(FE::from(2), vec![0, 1])], + ) + .unwrap() + } + + /// A constant statement: no variables at all. + fn flat(value: u64) -> VirtualPolynomial { + VirtualPolynomial::new( + vec![Mle::new(vec![FE::from(value)]).unwrap()], + vec![Term::single(0)], + ) + .unwrap() + } + + fn claims_of(parts: &[VirtualPolynomial]) -> Vec { + parts.iter().map(|p| p.sum_over_hypercube()).collect() + } + + /// The verifier's side of the round trip: each statement's own extension at + /// its own prefix of the point. + fn values_at(parts: &[VirtualPolynomial], point: &[FE]) -> Result, Error> { + parts + .iter() + .map(|p| p.evaluate(&point[..p.num_vars()])) + .collect() + } + + /// The table read on `n` variables as the function ignoring the last + /// `n − num_vars`: each entry repeated. This is what the protocol claims a + /// short statement *is*, written out — only the tests pay for it. + fn repeated(table: &Mle, n: usize) -> Mle { + let shift = n - table.num_vars(); + Mle::new( + (0..(1usize << n)) + .map(|j| table.evals()[j >> shift]) + .collect(), + ) + .unwrap() + } + + #[test] + fn uneven_cubes_round_trip() { + // Deliberately not sorted: the batch orders its own walk. + let mut parts = vec![part(3, 1), part(5, 10), part(4, 20)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let originals = parts.clone(); + + let (proof, point) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + assert_eq!(proof.rounds.len(), 5); + + let verified = verify( + &proof, + &heights, + &claims, + 2, + |p| values_at(&originals, p), + &mut transcript(), + ) + .unwrap(); + assert_eq!(verified, point); + } + + #[test] + fn it_is_the_sumcheck_of_the_repeated_tables() { + // The whole construction, checked against writing the padding out: the + // batch must produce exactly the proof one plain sumcheck over the + // statements read on the big cube would. + let mut parts = vec![part(5, 10), part(3, 1), part(4, 20)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let n = 5; + + let (proof, point) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + + let mut other = transcript(); + let lambdas = absorb_statement(&heights, &claims, &mut other); + let polys: Vec> = [part(5, 10), part(3, 1), part(4, 20)] + .iter() + .flat_map(|p| p.polys().iter().map(|f| repeated(f, n)).collect::>()) + .collect(); + let combined = Composed::new( + polys, + move |v: &[FE]| { + (0..3).fold(FE::zero(), |acc, i| { + acc + lambdas[i] * FE::from(2) * v[2 * i] * v[2 * i + 1] + }) + }, + 2, + ) + .unwrap(); + let (expected, expected_point) = sumcheck::prove(combined, &mut other).unwrap(); + + assert_eq!(proof, expected); + assert_eq!(point, expected_point); + } + + #[test] + fn a_statement_with_no_variables_is_carried_as_a_constant() { + let mut parts = vec![part(4, 7), flat(31), part(2, 9)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + assert_eq!(heights[1], 0); + let claims = claims_of(&parts); + let originals = parts.clone(); + + let (proof, _) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + verify( + &proof, + &heights, + &claims, + 2, + |p| values_at(&originals, p), + &mut transcript(), + ) + .unwrap(); + } + + #[test] + fn statements_of_equal_height_round_trip() { + // The degenerate case the uneven machinery must not special-case wrong. + let mut parts = vec![part(4, 1), part(4, 5)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let originals = parts.clone(); + + let (proof, _) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + verify( + &proof, + &heights, + &claims, + 2, + |p| values_at(&originals, p), + &mut transcript(), + ) + .unwrap(); + } + + #[test] + fn statements_of_different_degree_round_trip() { + // The batch runs at the worst degree; the cheaper statement is simply + // sampled at more nodes than it needs. + let cubic = VirtualPolynomial::new( + vec![pseudo_mle(4, 1), pseudo_mle(4, 2), pseudo_mle(4, 3)], + vec![Term::new(FE::from(7), vec![0, 1, 2])], + ) + .unwrap(); + let linear = VirtualPolynomial::new(vec![pseudo_mle(2, 4)], vec![Term::single(0)]).unwrap(); + let mut parts = vec![cubic, linear]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let originals = parts.clone(); + + let (proof, _) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + assert_eq!(proof.rounds[0].evaluations.len(), 3); + verify( + &proof, + &heights, + &claims, + 3, + |p| values_at(&originals, p), + &mut transcript(), + ) + .unwrap(); + } + + #[test] + fn the_parts_come_back_bound_at_their_own_prefix() { + // What the prover reads off the folded statements has to be the very + // thing the verifier asks `values_at` for. + let mut parts = vec![part(5, 3), part(2, 4), part(4, 5)]; + let originals = parts.clone(); + + let claims = claims_of(&parts); + let (_, point) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + + for (folded, original) in parts.iter().zip(&originals) { + assert_eq!(folded.num_vars(), 0); + assert_eq!( + folded.eval_at_index(0), + original.evaluate(&point[..original.num_vars()]).unwrap() + ); + } + } + + #[test] + fn a_wrong_claim_is_rejected() { + let mut parts = vec![part(4, 1), part(2, 2)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let originals = parts.clone(); + let (proof, _) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + + // The short statement's claim is the one scaled by 2^{n−n_i}; getting + // that scale wrong is the mistake this construction invites, so the + // rejection has to come from the residual and not from a round. + let lied = vec![claims[0], claims[1] + FE::one()]; + let err = verify( + &proof, + &heights, + &lied, + 2, + |p| values_at(&originals, p), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::BatchMismatch); + } + + #[test] + fn a_tampered_round_is_rejected() { + let mut parts = vec![part(4, 1), part(3, 2)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let originals = parts.clone(); + let (mut proof, _) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + + proof.rounds[2].evaluations[0] += FE::one(); + let err = verify( + &proof, + &heights, + &claims, + 2, + |p| values_at(&originals, p), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::BatchMismatch); + } + + #[test] + fn a_value_from_the_wrong_prefix_is_rejected() { + // Reading a short statement at the *whole* point instead of its prefix + // is the other easy mistake, and it has to fail. + let mut parts = vec![part(4, 1), part(2, 2)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let originals = parts.clone(); + let (proof, _) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + + let err = verify( + &proof, + &heights, + &claims, + 2, + |p| { + Ok(vec![ + originals[0].evaluate(&p[..4]).unwrap(), + // The last two challenges, not the first two. + originals[1].evaluate(&p[2..]).unwrap(), + ]) + }, + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::BatchMismatch); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let mut parts = vec![part(4, 1), part(3, 2)]; + let heights: Vec = parts.iter().map(|p| p.num_vars()).collect(); + let claims = claims_of(&parts); + let originals = parts.clone(); + let (proof, _) = prove(&mut parts, &claims, &mut transcript()).unwrap(); + + let mut other = DefaultTranscript::::new(b"a-different-statement"); + let err = verify( + &proof, + &heights, + &claims, + 2, + |p| values_at(&originals, p), + &mut other, + ) + .unwrap_err(); + assert_eq!(err, Error::BatchMismatch); + } + + #[test] + fn a_claim_per_statement_is_required() { + let mut parts = vec![part(3, 1), part(2, 2)]; + let err = prove(&mut parts, &[FE::one()], &mut transcript()).unwrap_err(); + assert_eq!( + err, + Error::VariableCountMismatch { + expected: 2, + got: 1 + } + ); + } + + #[test] + fn an_empty_batch_is_rejected() { + let mut parts: Vec> = Vec::new(); + let err = prove(&mut parts, &[], &mut transcript()).unwrap_err(); + assert_eq!(err, Error::EmptyPolynomial); + } +} diff --git a/crypto/multilinear/src/uni_skip.rs b/crypto/multilinear/src/uni_skip.rs new file mode 100644 index 000000000..b9b4dd807 --- /dev/null +++ b/crypto/multilinear/src/uni_skip.rs @@ -0,0 +1,369 @@ +//! The univariate skip: the first `l_skip` Boolean variables become one variable +//! ranging over `D`, the multiplicative subgroup of order `2^l_skip`, so the +//! domain is a prism `D × {0,1}^n`. `eq_D` is the Lagrange kernel on `D`. + +use math::field::{ + element::FieldElement, + traits::{IsField, IsPrimeField}, +}; + +use crate::{Error, eq::eq_and_rot_eval, eq::eq_eval}; + +/// Successive squarings `x, x², x⁴, …`. +fn squarings(x: &FieldElement) -> impl Iterator> + use { + let mut current = x.clone(); + std::iter::from_fn(move || { + let out = current.clone(); + current = current.square(); + Some(out) + }) +} + +/// `2^{-l_skip}`, the normalizing factor every `eq_D` carries. +fn inv_two_pow(l_skip: usize) -> FieldElement { + let half = (FieldElement::::one() + FieldElement::::one()) + .inv() + .expect("2 is invertible in an odd-characteristic field"); + (0..l_skip).fold(FieldElement::::one(), |acc, _| acc * &half) +} + +/// The Lagrange kernel on `D`, the subgroup of order `2^l_skip`. On `D × D` it +/// is the equality indicator. +pub fn eq_uni( + l_skip: usize, + x: &FieldElement, + y: &FieldElement, +) -> FieldElement { + let one = FieldElement::::one(); + let mut res = one.clone(); + for (x_pow, y_pow) in squarings(x).zip(squarings(y)).take(l_skip) { + res = (&x_pow + &y_pow) * res + (&x_pow - &one) * (&y_pow - &one); + } + res * inv_two_pow::(l_skip) +} + +/// `eq_D(x, 1)`, which collapses to a product of `x^{2^i} + 1`. +pub fn eq_uni_at_one(l_skip: usize, x: &FieldElement) -> FieldElement { + let one = FieldElement::::one(); + let mut res = one.clone(); + for x_pow in squarings(x).take(l_skip) { + res *= x_pow + &one; + } + res * inv_two_pow::(l_skip) +} + +/// A univariate polynomial in coefficient form, lowest degree first. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UnivariatePoly(Vec>); + +impl UnivariatePoly { + pub fn new(coeffs: Vec>) -> Self { + Self(coeffs) + } + + pub fn coeffs(&self) -> &[FieldElement] { + &self.0 + } + + pub fn into_coeffs(self) -> Vec> { + self.0 + } + + /// Horner evaluation. + pub fn evaluate(&self, x: &FieldElement) -> FieldElement { + self.0 + .iter() + .rfold(FieldElement::::zero(), |acc, c| acc * x + c) + } +} + +/// `eq_D(x, ·)` as a polynomial in its second argument. +/// +/// `eq_D` is the Lagrange basis at `x`, which on `D` is the character sum +/// `eq_D(x, z) = 2^{-l} Σ_k x^k z^{N−k}`; reading that as coefficients in `z` +/// gives the reversed powers of `x`. +pub fn eq_uni_poly(l_skip: usize, x: &FieldElement) -> UnivariatePoly { + let n = 1usize << l_skip; + let n_inv = inv_two_pow::(l_skip); + + let mut coeffs = Vec::with_capacity(n); + let mut x_pow = x.clone(); + for _ in 0..n { + coeffs.push(&x_pow * &n_inv); + x_pow *= x; + } + coeffs.reverse(); + coeffs[0] = n_inv; + UnivariatePoly::new(coeffs) +} + +/// The prism analogue of `eq`: subgroup on the first coordinate, hypercube on +/// the rest. +pub fn eq_prism( + l_skip: usize, + x: &[FieldElement], + y: &[FieldElement], +) -> Result, Error> { + if x.is_empty() || y.is_empty() { + return Err(Error::VariableCountMismatch { + expected: 1, + got: 0, + }); + } + Ok(eq_uni(l_skip, &x[0], &y[0]) * eq_eval(&x[1..], &y[1..])?) +} + +/// The Möbius variant of `eq`, `∏_i ((1 − 2u_i)(1 − x_i) + u_i·x_i)`. +pub fn mobius_eq_eval( + u: &[FieldElement], + x: &[FieldElement], +) -> Result, Error> { + if u.len() != x.len() { + return Err(Error::VariableCountMismatch { + expected: u.len(), + got: x.len(), + }); + } + let one = FieldElement::::one(); + Ok(u.iter().zip(x).fold(one.clone(), |acc, (u_i, x_i)| { + let w0 = &one - u_i - u_i; + acc * (w0 * (&one - x_i) + u_i * x_i) + })) +} + +/// The rotation kernel on the prism. A step inside `D` multiplies by `ω`; only +/// at the end of `D` does it carry into the cube. `omega` must generate `D`. +pub fn rot_kernel_prism( + l_skip: usize, + omega: &FieldElement, + x: &[FieldElement], + y: &[FieldElement], +) -> Result, Error> { + if x.len() != y.len() || x.is_empty() { + return Err(Error::VariableCountMismatch { + expected: x.len().max(1), + got: y.len(), + }); + } + let (eq_cube, rot_cube) = eq_and_rot_eval(&x[1..], &y[1..])?; + let y0_omega = &y[0] * omega; + + Ok(eq_uni(l_skip, &x[0], &y0_omega) * &eq_cube + + eq_uni_at_one(l_skip, &x[0]) * eq_uni_at_one(l_skip, &y0_omega) * (rot_cube - &eq_cube)) +} + +/// Generator of `D`, the subgroup of order `2^l_skip`. +pub fn skip_domain_generator(l_skip: usize) -> Result, Error> +where + F: math::field::traits::IsFFTField + IsPrimeField, +{ + F::get_primitive_root_of_unity(l_skip as u64).map_err(|_| Error::SkipDomainUnavailable { + l_skip, + two_adicity: F::TWO_ADICITY as usize, + }) +} + +/// Every element of `D`, in the order `1, ω, ω², …`. +pub fn skip_domain(l_skip: usize) -> Result>, Error> +where + F: math::field::traits::IsFFTField + IsPrimeField, +{ + let omega = skip_domain_generator::(l_skip)?; + let mut out = Vec::with_capacity(1 << l_skip); + let mut current = FieldElement::::one(); + for _ in 0..(1usize << l_skip) { + out.push(current.clone()); + current *= ω + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + #[test] + fn eq_uni_is_the_indicator_on_the_subgroup() { + for l_skip in 1..=4usize { + let d = skip_domain::(l_skip).unwrap(); + assert_eq!(d.len(), 1 << l_skip); + for (i, z1) in d.iter().enumerate() { + for (j, z2) in d.iter().enumerate() { + let expected = if i == j { FE::one() } else { FE::zero() }; + assert_eq!(eq_uni(l_skip, z1, z2), expected, "l={l_skip}, {i} vs {j}"); + } + } + } + } + + #[test] + fn the_domain_is_a_subgroup_of_the_right_order() { + for l_skip in 1..=5usize { + let omega = skip_domain_generator::(l_skip).unwrap(); + let order = 1u64 << l_skip; + assert_eq!(omega.pow(order), FE::one(), "l={l_skip}"); + if l_skip > 0 { + assert_ne!(omega.pow(order / 2), FE::one(), "l={l_skip} not primitive"); + } + } + } + + #[test] + fn eq_uni_at_one_agrees_with_the_general_form() { + let one = FE::one(); + for l_skip in 0..=4usize { + for x in [FE::from(3), FE::from(17), FE::from(1)] { + assert_eq!( + eq_uni_at_one(l_skip, &x), + eq_uni(l_skip, &x, &one), + "l={l_skip}" + ); + } + } + } + + #[test] + fn eq_uni_is_symmetric() { + let (x, y) = (FE::from(11), FE::from(29)); + for l_skip in 0..=4usize { + assert_eq!(eq_uni(l_skip, &x, &y), eq_uni(l_skip, &y, &x)); + } + } + + #[test] + fn eq_uni_sums_to_one_over_the_domain() { + // Lagrange bases partition unity: Σ_{z ∈ D} eq_D(x, z) = 1. + for l_skip in 1..=4usize { + let d = skip_domain::(l_skip).unwrap(); + let x = FE::from(1234); + let total = d + .iter() + .fold(FE::zero(), |acc, z| acc + eq_uni(l_skip, &x, z)); + assert_eq!(total, FE::one(), "l={l_skip}"); + } + } + + #[test] + fn eq_uni_poly_is_eq_uni_in_its_second_argument() { + for l_skip in 1..=4usize { + let x = FE::from(97); + let poly = eq_uni_poly(l_skip, &x); + assert_eq!(poly.coeffs().len(), 1 << l_skip); + for z in [FE::from(2), FE::from(5), FE::from(1000)] { + assert_eq!(poly.evaluate(&z), eq_uni(l_skip, &x, &z), "l={l_skip}"); + } + } + } + + #[test] + fn horner_matches_the_written_out_polynomial() { + // 4 + 3x + 2x² + x³ + let poly = UnivariatePoly::new(vec![FE::from(4), FE::from(3), FE::from(2), FE::from(1)]); + let x = FE::from(5); + let expected = FE::from(4) + FE::from(3) * x + FE::from(2) * x * x + x * x * x; + assert_eq!(poly.evaluate(&x), expected); + } + + #[test] + fn a_zero_skip_leaves_the_plain_hypercube_eq() { + // With l_skip = 0 the subgroup is trivial and eq_D is the constant one, + // so the prism degenerates to the cube. + let x = vec![FE::from(3), FE::from(5), FE::from(7)]; + let y = vec![FE::from(11), FE::from(13), FE::from(17)]; + assert_eq!(eq_uni(0, &x[0], &y[0]), FE::one()); + assert_eq!( + eq_prism(0, &x, &y).unwrap(), + eq_eval(&x[1..], &y[1..]).unwrap() + ); + } + + #[test] + fn eq_prism_is_the_indicator_on_the_prism() { + let l_skip = 2; + let d = skip_domain::(l_skip).unwrap(); + let cube = [FE::zero(), FE::one()]; + + for (i, u) in d.iter().enumerate() { + for a in cube { + for (j, v) in d.iter().enumerate() { + for b in cube { + let x = [*u, a]; + let y = [*v, b]; + let expected = if i == j && a == b { + FE::one() + } else { + FE::zero() + }; + assert_eq!(eq_prism(l_skip, &x, &y).unwrap(), expected); + } + } + } + } + } + + #[test] + fn mobius_eq_matches_its_written_form() { + let u = vec![FE::from(3), FE::from(5)]; + let x = vec![FE::from(7), FE::from(11)]; + let one = FE::one(); + let expected = ((one - FE::from(3) - FE::from(3)) * (one - FE::from(7)) + + FE::from(3) * FE::from(7)) + * ((one - FE::from(5) - FE::from(5)) * (one - FE::from(11)) + + FE::from(5) * FE::from(11)); + assert_eq!(mobius_eq_eval(&u, &x).unwrap(), expected); + } + + #[test] + fn rot_on_the_prism_steps_within_the_domain() { + // Inside D a step is multiplication by omega, with the cube part fixed. + let l_skip = 2; + let omega = skip_domain_generator::(l_skip).unwrap(); + let d = skip_domain::(l_skip).unwrap(); + let cube_pt = [FE::zero(), FE::one()]; + + for (i, u) in d.iter().enumerate() { + let x: Vec = std::iter::once(*u).chain(cube_pt.iter().cloned()).collect(); + for (j, v) in d.iter().enumerate() { + let y: Vec = std::iter::once(*v).chain(cube_pt.iter().cloned()).collect(); + let got = rot_kernel_prism(l_skip, &omega, &x, &y).unwrap(); + // A step inside D keeps the cube point. The one exception is + // stepping off the last element of D, which must carry into the + // cube instead — so with the cube point held fixed it is zero. + let steps_within_d = i == j + 1; + let expected = if steps_within_d { + FE::one() + } else { + FE::zero() + }; + assert_eq!(got, expected, "u={i}, v={j}"); + } + } + } + + #[test] + fn rot_on_the_prism_carries_into_the_cube_at_the_domain_boundary() { + // Stepping off the last element of D advances the cube coordinate. + let l_skip = 2; + let omega = skip_domain_generator::(l_skip).unwrap(); + let d = skip_domain::(l_skip).unwrap(); + let last = *d.last().unwrap(); + let first = d[0]; + + // cube index 0 -> 1 on a single cube variable. + let y = vec![last, FE::zero()]; + let x = vec![first, FE::one()]; + assert_eq!(rot_kernel_prism(l_skip, &omega, &x, &y).unwrap(), FE::one()); + } + + #[test] + fn a_domain_beyond_two_adicity_is_rejected() { + let err = skip_domain_generator::(64).unwrap_err(); + assert!(matches!( + err, + Error::SkipDomainUnavailable { l_skip: 64, .. } + )); + } +} diff --git a/crypto/multilinear/src/virtual_poly.rs b/crypto/multilinear/src/virtual_poly.rs new file mode 100644 index 000000000..d380f9602 --- /dev/null +++ b/crypto/multilinear/src/virtual_poly.rs @@ -0,0 +1,244 @@ +//! A sum of products of multilinear polynomials — one implementation of +//! [`SumcheckPolynomial`](crate::poly::SumcheckPolynomial). + +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{Error, mle::Mle, poly::SumcheckPolynomial}; + +/// One monomial: `coefficient · Π_{i ∈ factors} poly[i]`. +/// +/// A term with no factors is a constant. Repeating an index raises that +/// polynomial to a power, which is how a squared column is expressed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Term { + pub coefficient: FieldElement, + pub factors: Vec, +} + +impl Term { + pub fn new(coefficient: FieldElement, factors: Vec) -> Self { + Self { + coefficient, + factors, + } + } + + /// A term that is just one polynomial, with coefficient one. + pub fn single(index: usize) -> Self { + Self::new(FieldElement::::one(), vec![index]) + } +} + +/// A sum of [`Term`]s over a shared set of multilinear polynomials. +#[derive(Clone, Debug)] +pub struct VirtualPolynomial { + polys: Vec>, + terms: Vec>, + num_vars: usize, +} + +impl VirtualPolynomial { + /// Builds the polynomial, checking that every factor resolves and that all + /// operands agree on the number of variables. + pub fn new(polys: Vec>, terms: Vec>) -> Result { + if terms.is_empty() { + return Err(Error::EmptyPolynomial); + } + let num_vars = polys.first().map(|p| p.num_vars()).unwrap_or(0); + for p in &polys { + if p.num_vars() != num_vars { + return Err(Error::VariableCountMismatch { + expected: num_vars, + got: p.num_vars(), + }); + } + } + for term in &terms { + for &index in &term.factors { + if index >= polys.len() { + return Err(Error::UnknownPolynomial { + index, + len: polys.len(), + }); + } + } + } + Ok(Self { + polys, + terms, + num_vars, + }) + } + + pub fn terms(&self) -> &[Term] { + &self.terms + } + + /// The value of each factor once every variable has been fixed. + pub fn factor_constants(&self) -> Option>> { + self.polys + .iter() + .map(|p| p.as_constant().cloned()) + .collect() + } +} + +impl SumcheckPolynomial for VirtualPolynomial { + fn num_vars(&self) -> usize { + self.num_vars + } + + /// The largest number of factors in any term. + fn degree(&self) -> usize { + self.terms + .iter() + .map(|t| t.factors.len()) + .max() + .unwrap_or(0) + } + + fn polys(&self) -> &[Mle] { + &self.polys + } + + fn combine(&self, values: &[FieldElement]) -> FieldElement { + self.terms.iter().fold(FieldElement::zero(), |acc, term| { + let product = term + .factors + .iter() + .fold(term.coefficient.clone(), |p, &f| p * &values[f]); + acc + product + }) + } + + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), Error> { + for p in &mut self.polys { + p.fix_first_variable_in_place(r)?; + } + self.num_vars -= 1; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + /// `f = 2·a·b + 3·c` on two variables. + fn sample() -> VirtualPolynomial { + let a = mle(&[1, 2, 3, 4]); + let b = mle(&[5, 6, 7, 8]); + let c = mle(&[9, 10, 11, 12]); + VirtualPolynomial::new( + vec![a, b, c], + vec![ + Term::new(FE::from(2), vec![0, 1]), + Term::new(FE::from(3), vec![2]), + ], + ) + .unwrap() + } + + #[test] + fn degree_is_the_longest_term() { + assert_eq!(sample().degree(), 2); + } + + #[test] + fn index_evaluation_applies_the_term_structure() { + let f = sample(); + for i in 0..4usize { + let (a, b, c) = ( + FE::from(1 + i as u64), + FE::from(5 + i as u64), + FE::from(9 + i as u64), + ); + let expected = FE::from(2) * a * b + FE::from(3) * c; + assert_eq!(f.eval_at_index(i), expected); + } + } + + #[test] + fn hypercube_sum_matches_the_terms() { + let f = sample(); + let expected = (0..4).fold(FE::zero(), |acc, i| acc + f.eval_at_index(i)); + assert_eq!(f.sum_over_hypercube(), expected); + } + + #[test] + fn evaluate_off_cube_matches_combine_of_factor_values() { + let f = sample(); + let point = [FE::from(17), FE::from(23)]; + let values: Vec = f + .polys() + .iter() + .map(|p| p.evaluate(&point).unwrap()) + .collect(); + assert_eq!(f.evaluate(&point).unwrap(), f.combine(&values)); + } + + #[test] + fn folding_agrees_with_evaluating_the_fixed_variable() { + let mut f = sample(); + let r = FE::from(31); + let rest = FE::from(37); + let expected = f.evaluate(&[r, rest]).unwrap(); + + f.fix_first_variable(&r).unwrap(); + assert_eq!(f.num_vars(), 1); + assert_eq!(f.evaluate(&[rest]).unwrap(), expected); + } + + #[test] + fn repeated_factor_is_a_power() { + let a = mle(&[2, 3, 5, 7]); + let f = VirtualPolynomial::new(vec![a], vec![Term::new(FE::one(), vec![0, 0])]).unwrap(); + assert_eq!(f.degree(), 2); + assert_eq!(f.eval_at_index(2), FE::from(25)); + } + + #[test] + fn constant_term_has_degree_zero() { + let f = VirtualPolynomial::new(vec![mle(&[1, 1])], vec![Term::new(FE::from(9), vec![])]) + .unwrap(); + assert_eq!(f.degree(), 0); + assert_eq!(f.eval_at_index(0), FE::from(9)); + assert_eq!(f.sum_over_hypercube(), FE::from(18)); + } + + #[test] + fn rejects_a_factor_that_does_not_resolve() { + let err = VirtualPolynomial::new(vec![mle(&[1, 2])], vec![Term::new(FE::one(), vec![3])]) + .unwrap_err(); + assert_eq!(err, Error::UnknownPolynomial { index: 3, len: 1 }); + } + + #[test] + fn rejects_operands_of_differing_arity() { + let err = VirtualPolynomial::new( + vec![mle(&[1, 2]), mle(&[1, 2, 3, 4])], + vec![Term::single(0)], + ) + .unwrap_err(); + assert_eq!( + err, + Error::VariableCountMismatch { + expected: 1, + got: 2 + } + ); + } + + #[test] + fn rejects_an_empty_term_list() { + let err = VirtualPolynomial::new(vec![mle(&[1, 2])], vec![]).unwrap_err(); + assert_eq!(err, Error::EmptyPolynomial); + } +} diff --git a/crypto/multilinear/src/whir.rs b/crypto/multilinear/src/whir.rs new file mode 100644 index 000000000..308408959 --- /dev/null +++ b/crypto/multilinear/src/whir.rs @@ -0,0 +1,610 @@ +//! Encoding a multilinear as a Reed-Solomon codeword, and folding it. +//! +//! The domain is a two-adic subgroup, so it lives in the base field `F`, while +//! codeword values live in `E`. Mixed products keep the base element on the +//! left, which is the only direction the field tower implements. +//! +//! The identity everything rests on: folding the codeword with `α` encodes the +//! polynomial with one variable fixed to `α`. [`lift_coefficients`] reverses the +//! coefficient index so that variable is the *first*, matching sumcheck. + +use math::field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, +}; + +#[cfg(not(feature = "parallel"))] +use math::fft::bowers_fft::bowers_fft_opt_fused; +#[cfg(feature = "parallel")] +use math::fft::bowers_fft::bowers_fft_opt_fused_parallel; +use math::fft::{bit_reversing::in_place_bit_reverse_permute, bowers_fft::LayerTwiddles}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{Error, mle::Mle}; + +/// The evaluation domain: a coset-free multiplicative subgroup of order `2^k`. +#[derive(Clone, Debug)] +pub struct Domain { + generator: FieldElement, + log_size: usize, +} + +impl Domain { + /// The subgroup of order `2^log_size`. + pub fn new(log_size: usize) -> Result { + let generator = F::get_primitive_root_of_unity(log_size as u64).map_err(|_| { + Error::SkipDomainUnavailable { + l_skip: log_size, + two_adicity: F::TWO_ADICITY as usize, + } + })?; + Ok(Self { + generator, + log_size, + }) + } + + /// The domain reached by squaring every element: order `2^(log_size - 1)`. + pub fn squared(&self) -> Result { + if self.log_size == 0 { + return Err(Error::NoVariablesLeft); + } + Ok(Self { + generator: self.generator.square(), + log_size: self.log_size - 1, + }) + } + + pub fn log_size(&self) -> usize { + self.log_size + } + + pub fn size(&self) -> usize { + 1usize << self.log_size + } + + pub fn generator(&self) -> &FieldElement { + &self.generator + } + + /// Every element, as `1, g, g², …`. + pub fn elements(&self) -> Vec> { + let mut out = Vec::with_capacity(self.size()); + let mut current = FieldElement::::one(); + for _ in 0..self.size() { + out.push(current.clone()); + current *= &self.generator; + } + out + } +} + +/// The multilinear's monomial coefficients, in hypercube index order. +/// +/// The inverse of the evaluation map: reading a multilinear's `2^m` hypercube +/// values as `Σ_S ĉ_S ∏_{i∈S} x_i`. Computed by the Möbius transform, in +/// `O(m·2^m)`. +pub fn monomial_coefficients(mle: &Mle) -> Vec> +where + FieldElement: Send + Sync, +{ + let mut coeffs = mle.evals().to_vec(); + let n = coeffs.len(); + let mut stride = 1; + while stride < n { + // Each block of `2·stride` is independent: its high half takes the low + // half away and nothing crosses the boundary. + let level = |chunk: &mut [FieldElement]| { + let (lo, hi) = chunk.split_at_mut(stride); + for (h, l) in hi.iter_mut().zip(lo.iter()) { + *h = &*h - l; + } + }; + #[cfg(feature = "parallel")] + coeffs.par_chunks_mut(stride * 2).for_each(level); + #[cfg(not(feature = "parallel"))] + coeffs.chunks_mut(stride * 2).for_each(level); + stride *= 2; + } + coeffs +} + +/// The univariate lift used by the folding argument. +/// +/// Reverses the coefficient index so variable 0 is the low bit, making one fold +/// bind the variable one sumcheck round binds. +pub fn lift_coefficients(mle: &Mle) -> Vec> +where + FieldElement: Send + Sync, +{ + let coeffs = monomial_coefficients(mle); + let num_vars = mle.num_vars(); + let gather = |i: usize| coeffs[reverse_bits(i, num_vars)].clone(); + #[cfg(feature = "parallel")] + return (0..coeffs.len()).into_par_iter().map(gather).collect(); + #[cfg(not(feature = "parallel"))] + return (0..coeffs.len()).map(gather).collect(); +} + +/// Reverses the low `width` bits of `index`. +/// +/// One instruction and a shift, not a loop over the bits: this runs once per +/// element of a codeword, so the difference is a pass over the whole thing. +fn reverse_bits(index: usize, width: usize) -> usize { + if width == 0 { + return 0; + } + index.reverse_bits() >> (usize::BITS as usize - width) +} + +/// Evaluates the univariate lift on every point of `domain`, in domain order: +/// `out[j] = F(g^j)`. +/// +/// One NTT, threaded under the `parallel` feature. The Bowers transform leaves +/// its output bit-reversed, and the ordering is load-bearing — +/// [`fold_codeword`] pairs `j` with `j + N/2` because `g^(j + N/2) = −g^j` — so +/// it is permuted back. +/// +/// The `Send + Sync` bounds are unconditional so the signature does not change +/// with the feature; the serial path does not need them, and every field this +/// crate is used with satisfies them. +pub fn encode( + coeffs: &[FieldElement], + domain: &Domain, +) -> Result>, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf, + E: IsField + Send + Sync, + FieldElement: Send + Sync, + FieldElement: Send + Sync, +{ + if coeffs.len() > domain.size() { + return Err(Error::CodewordTooShort { + coefficients: coeffs.len(), + domain: domain.size(), + }); + } + let mut values = coeffs.to_vec(); + values.resize(domain.size(), FieldElement::::zero()); + if domain.log_size == 0 { + return Ok(values); + } + + let twiddles = + LayerTwiddles::::new(domain.log_size as u64).ok_or(Error::SkipDomainUnavailable { + l_skip: domain.log_size, + two_adicity: F::TWO_ADICITY as usize, + })?; + #[cfg(feature = "parallel")] + bowers_fft_opt_fused_parallel::(&mut values, &twiddles) + .map_err(|_| Error::NotPowerOfTwo(domain.size()))?; + #[cfg(not(feature = "parallel"))] + bowers_fft_opt_fused::(&mut values, &twiddles) + .map_err(|_| Error::NotPowerOfTwo(domain.size()))?; + in_place_bit_reverse_permute(&mut values); + Ok(values) +} + +/// Folds a codeword once: `F_α = F₀ + α·F₁`, recovering the halves from `F(x)` +/// and `F(−x)`. `−x` is half a period away, so `j` pairs with `j + N/2`. +/// +/// The values go in over `C` and come out over `N`: a committed trace codeword +/// is base-field, and the first fold is what lifts it, since `α` is an +/// extension challenge. Later folds are `N → N`. +pub fn fold_codeword( + codeword: &[FieldElement], + domain: &Domain, + alpha: &FieldElement, +) -> Result>, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf, + C: IsField + IsSubFieldOf, + N: IsField, +{ + if codeword.len() != domain.size() { + return Err(Error::CodewordTooShort { + coefficients: codeword.len(), + domain: domain.size(), + }); + } + if domain.log_size == 0 { + return Err(Error::NoVariablesLeft); + } + let half = codeword.len() / 2; + let two_inv = (FieldElement::::one() + FieldElement::::one()) + .inv() + .expect("2 is invertible"); + // The odd half needs `1/g^j`, and those are the powers of `1/g` — so one + // inversion, not one per position. Folded together with `1/2` while we are + // at it, since they only ever appear as a product. + let step = domain + .generator() + .inv() + .expect("a domain generator is nonzero"); + // `odd_scale` walks the powers of `1/g`, which is a chain — so a chunk + // starts from `two_inv · step^start` and walks from there. That is what + // lets the fold go out to the pool at all: one `pow` per chunk instead of + // one inversion per position. + let fold_chunk = |start: usize, out: &mut Vec>, len: usize| { + let mut odd_scale = &two_inv * step.pow(start as u64); + for j in start..start + len { + let (a, b) = (&codeword[j], &codeword[j + half]); + let even = &two_inv * (a + b); + let odd = &odd_scale * (a - b); + // The base element on the left: the only direction the tower gives. + out.push(even + odd * alpha); + odd_scale *= &step; + } + }; + + #[cfg(feature = "parallel")] + { + const SERIAL_BELOW: usize = 1 << 12; + if half >= SERIAL_BELOW { + let chunk = half.div_ceil(rayon::current_num_threads().max(1)); + let parts: Vec>> = (0..half) + .into_par_iter() + .step_by(chunk) + .map(|start| { + let len = chunk.min(half - start); + let mut out = Vec::with_capacity(len); + fold_chunk(start, &mut out, len); + out + }) + .collect(); + return Ok(parts.concat()); + } + } + + let mut out = Vec::with_capacity(half); + fold_chunk(0, &mut out, half); + Ok(out) +} + +/// Folds `k` times, squaring the domain at each step. +/// +/// The first fold lifts `C` into `N`; the rest stay there. +pub fn fold_codeword_k( + codeword: &[FieldElement], + domain: &Domain, + alphas: &[FieldElement], +) -> Result<(Vec>, Domain), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf + 'static, + C: IsField + IsSubFieldOf + 'static, + N: IsField + 'static, +{ + // Every level of the group in one residency: the codeword is the biggest + // thing the proof moves, and the device halves it level by level without + // handing it back in between. + if let Some(folded) = crate::gpu::fold_codeword_k(codeword, domain.generator(), alphas) { + let mut folded_domain = domain.clone(); + for _ in alphas { + folded_domain = folded_domain.squared()?; + } + return Ok((folded, folded_domain)); + } + fold_codeword_k_on_host(codeword, domain, alphas) +} + +/// The same on the host, whatever a device would have done — the reference the +/// kernel is checked against. +pub fn fold_codeword_k_on_host( + codeword: &[FieldElement], + domain: &Domain, + alphas: &[FieldElement], +) -> Result<(Vec>, Domain), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf + 'static, + C: IsField + IsSubFieldOf + 'static, + N: IsField + 'static, +{ + let Some((first, rest)) = alphas.split_first() else { + let lifted = codeword + .iter() + .map(|v| v.clone().to_extension::()) + .collect(); + return Ok((lifted, domain.clone())); + }; + + let mut current = fold_codeword::(codeword, domain, first)?; + let mut current_domain = domain.squared()?; + for alpha in rest { + current = fold_codeword::(¤t, ¤t_domain, alpha)?; + current_domain = current_domain.squared()?; + } + Ok((current, current_domain)) +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + fn pseudo_mle(num_vars: usize, seed: u64) -> Mle { + let vals: Vec = (0..(1u64 << num_vars)) + .map(|i| (i.wrapping_mul(6364136223846793005).wrapping_add(seed)) >> 13) + .collect(); + mle(&vals) + } + + /// Evaluates the univariate lift directly, for comparison. + fn eval_univariate(coeffs: &[FE], x: &FE) -> FE { + coeffs.iter().rfold(FE::zero(), |acc, c| acc * x + c) + } + + /// The property a base-field commitment rests on: folding a base codeword + /// with an extension challenge gives what folding the lifted one would. + /// Without it, committing the trace over `F` would not be the same + /// statement as committing it over `E`. + #[test] + fn folding_a_base_codeword_matches_folding_the_lifted_one() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + type ExtE = FieldElement; + + for num_vars in 1..=4usize { + let f = pseudo_mle(num_vars, 3 * num_vars as u64); + let domain = Domain::::new(num_vars + 2).unwrap(); + let alphas: Vec = (0..num_vars).map(|i| ExtE::from(31 + i as u64)).collect(); + + // Left: a base codeword, folded with extension challenges. + let base = encode::(&lift_coefficients(&f), &domain).unwrap(); + let (from_base, _) = fold_codeword_k::(&base, &domain, &alphas).unwrap(); + + // Right: lift first, then fold. + let lifted = + Mle::new(f.evals().iter().map(|v| v.to_extension::()).collect()).unwrap(); + let extension = encode::(&lift_coefficients(&lifted), &domain).unwrap(); + let (from_extension, _) = + fold_codeword_k::(&extension, &domain, &alphas).unwrap(); + + assert_eq!(from_base, from_extension, "num_vars={num_vars}"); + } + } + + #[test] + fn the_domain_has_the_stated_order() { + for log_size in 1..=6usize { + let d = Domain::::new(log_size).unwrap(); + assert_eq!(d.size(), 1 << log_size); + assert_eq!(d.elements().len(), 1 << log_size); + assert_eq!(d.generator().pow(d.size() as u64), FE::one()); + } + } + + #[test] + fn squaring_the_domain_halves_it() { + let d = Domain::::new(4).unwrap(); + let sq = d.squared().unwrap(); + assert_eq!(sq.log_size(), 3); + // Squaring every element lands in the smaller subgroup, and covers it: + // each of its elements is hit exactly twice. + let smaller = sq.elements(); + let mut hits = vec![0usize; smaller.len()]; + for x in d.elements() { + let y = x.square(); + let at = smaller + .iter() + .position(|e| *e == y) + .expect("square left the subgroup"); + hits[at] += 1; + } + assert!(hits.iter().all(|h| *h == 2), "hits = {hits:?}"); + } + + #[test] + fn opposite_points_are_half_a_period_apart() { + // The pairing `fold_codeword` relies on: g^(j + N/2) = −g^j. + let d = Domain::::new(4).unwrap(); + let els = d.elements(); + let half = d.size() / 2; + for j in 0..half { + assert_eq!(els[j + half], -&els[j], "j={j}"); + } + } + + #[test] + fn monomial_coefficients_reproduce_the_evaluations() { + // Σ_S ĉ_S ∏_{i∈S} x_i must give back f on every corner. + let f = mle(&[7, 11, 13, 17, 19, 23, 29, 31]); + let coeffs = monomial_coefficients(&f); + let n = 3; + + for idx in 0..8usize { + // Variable 0 is the most significant bit of the index. + let x: Vec = (0..n) + .map(|i| FE::from(((idx >> (n - 1 - i)) & 1) as u64)) + .collect(); + let mut total = FE::zero(); + for (mask, c) in coeffs.iter().enumerate() { + let mut term = *c; + for (i, x_i) in x.iter().enumerate() { + if (mask >> (n - 1 - i)) & 1 == 1 { + term *= *x_i; + } + } + total += term; + } + assert_eq!(total, f.evals()[idx], "corner {idx}"); + } + } + + #[test] + fn encoding_evaluates_the_lift_on_the_domain() { + let f = pseudo_mle(3, 1); + let coeffs = monomial_coefficients(&f); + let domain = Domain::::new(5).unwrap(); + let codeword = encode(&coeffs, &domain).unwrap(); + + for (i, x) in domain.elements().iter().enumerate() { + assert_eq!(codeword[i], eval_univariate(&coeffs, x), "point {i}"); + } + } + + #[test] + fn a_codeword_longer_than_the_domain_is_rejected() { + let f = pseudo_mle(4, 1); + let coeffs = monomial_coefficients(&f); + let domain = Domain::::new(3).unwrap(); + assert!(matches!( + encode(&coeffs, &domain).unwrap_err(), + Error::CodewordTooShort { .. } + )); + } + + /// The identity the whole construction rests on. + #[test] + fn folding_commutes_with_fixing_a_variable() { + for num_vars in 1..=4usize { + for log_blowup in 1..=2usize { + let f = pseudo_mle(num_vars, 7 * num_vars as u64 + log_blowup as u64); + let domain = Domain::::new(num_vars + log_blowup).unwrap(); + let codeword = encode(&monomial_coefficients(&f), &domain).unwrap(); + + let alpha = FE::from(31 + num_vars as u64); + + // Left: fold the codeword. + let folded = fold_codeword(&codeword, &domain, &alpha).unwrap(); + + // Right: fix the last variable, then encode on the squared domain. + let mut fixed = f; + fixed.fix_last_variable_in_place(&alpha).unwrap(); + let smaller = domain.squared().unwrap(); + let re_encoded = encode(&monomial_coefficients(&fixed), &smaller).unwrap(); + + assert_eq!( + folded, re_encoded, + "num_vars={num_vars}, log_blowup={log_blowup}" + ); + } + } + } + + #[test] + fn folding_k_times_matches_fixing_k_variables() { + let num_vars = 4; + let log_blowup = 2; + let f = pseudo_mle(num_vars, 99); + let domain = Domain::::new(num_vars + log_blowup).unwrap(); + let codeword = encode(&monomial_coefficients(&f), &domain).unwrap(); + + let alphas = [FE::from(3), FE::from(5), FE::from(7)]; + let (folded, folded_domain) = fold_codeword_k(&codeword, &domain, &alphas).unwrap(); + + let mut fixed = f; + for alpha in &alphas { + fixed.fix_last_variable_in_place(alpha).unwrap(); + } + let re_encoded = encode(&monomial_coefficients(&fixed), &folded_domain).unwrap(); + + assert_eq!( + folded_domain.log_size(), + num_vars + log_blowup - alphas.len() + ); + assert_eq!(folded, re_encoded); + assert_eq!(fixed.num_vars(), num_vars - alphas.len()); + } + + #[test] + fn folding_preserves_the_rate() { + // The point of folding: the codeword and the message shrink together, + // so the code's rate is unchanged and the proximity test still applies. + let (num_vars, log_blowup) = (4usize, 2usize); + let f = pseudo_mle(num_vars, 5); + let domain = Domain::::new(num_vars + log_blowup).unwrap(); + let codeword = encode(&monomial_coefficients(&f), &domain).unwrap(); + + let alphas = [FE::from(11), FE::from(13)]; + let (folded, folded_domain) = fold_codeword_k(&codeword, &domain, &alphas).unwrap(); + + let message_len = 1usize << (num_vars - alphas.len()); + assert_eq!(folded.len(), folded_domain.size()); + assert_eq!(folded.len(), message_len << log_blowup); + } + + #[test] + fn folding_with_zero_keeps_only_the_even_part() { + // α = 0 selects F₀, which is f with its last variable set to 0 — the + // even-indexed hypercube values. + let f = mle(&[3, 5, 8, 13]); + let domain = Domain::::new(3).unwrap(); + let codeword = encode(&monomial_coefficients(&f), &domain).unwrap(); + let folded = fold_codeword(&codeword, &domain, &FE::zero()).unwrap(); + + let expected_mle = mle(&[3, 8]); // f(·, 0) + let expected = encode( + &monomial_coefficients(&expected_mle), + &domain.squared().unwrap(), + ) + .unwrap(); + assert_eq!(folded, expected); + } + + #[test] + fn folding_an_exhausted_domain_is_an_error() { + let domain = Domain::::new(0).unwrap(); + let codeword = vec![FE::from(4)]; + assert!(fold_codeword(&codeword, &domain, &FE::one()).is_err()); + assert!(domain.squared().is_err()); + } + + #[test] + fn bit_reversal_is_an_involution() { + for width in 0..=5usize { + for i in 0..(1usize << width) { + assert_eq!(reverse_bits(reverse_bits(i, width), width), i); + } + } + } + + /// The alignment the evaluation argument depends on: with the lift, + /// one fold binds the variable one sumcheck round binds. + #[test] + fn lifted_folding_binds_the_first_variable() { + for num_vars in 1..=4usize { + for log_blowup in 1..=2usize { + let f = pseudo_mle(num_vars, 3 * num_vars as u64 + log_blowup as u64); + let domain = Domain::::new(num_vars + log_blowup).unwrap(); + let codeword = encode(&lift_coefficients(&f), &domain).unwrap(); + + let alpha = FE::from(23 + num_vars as u64); + let folded = fold_codeword(&codeword, &domain, &alpha).unwrap(); + + let mut fixed = f; + fixed.fix_first_variable_in_place(&alpha).unwrap(); + let re_encoded = + encode(&lift_coefficients(&fixed), &domain.squared().unwrap()).unwrap(); + + assert_eq!(folded, re_encoded, "n={num_vars}, blowup={log_blowup}"); + } + } + } + + #[test] + fn folding_every_variable_leaves_the_evaluation() { + // Fold all the way and the codeword is the constant f(α), which is the + // value the evaluation argument compares against. + let (num_vars, log_blowup) = (4usize, 2usize); + let f = pseudo_mle(num_vars, 77); + let domain = Domain::::new(num_vars + log_blowup).unwrap(); + let codeword = encode(&lift_coefficients(&f), &domain).unwrap(); + + let alphas: Vec = (0..num_vars).map(|i| FE::from(5 + i as u64)).collect(); + let (folded, folded_domain) = fold_codeword_k(&codeword, &domain, &alphas).unwrap(); + + assert_eq!(folded_domain.log_size(), log_blowup); + let expected = f.evaluate(&alphas).unwrap(); + assert!( + folded.iter().all(|v| *v == expected), + "the fully folded codeword must be constant f(α)" + ); + } +} diff --git a/crypto/multilinear/src/whir_cap_tests.rs b/crypto/multilinear/src/whir_cap_tests.rs new file mode 100644 index 000000000..3378b702c --- /dev/null +++ b/crypto/multilinear/src/whir_cap_tests.rs @@ -0,0 +1,405 @@ +//! W1 — the Merkle cap on WHIR chains, end to end on the +//! host: every tree's paths stop `c` levels below its root, and the tree's cap +//! rides on its first opening in proof order (the owner path). +//! +//! Round-level fixtures that need the query positions (the unreached cap node +//! and the leaf forged from an internal node) live in `whir_round::tests`, +//! where the query draw is reachable. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as F, +}; + +use crate::{ + Error, + mle::Mle, + whir::Domain, + whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, ChainProof, ChainRound, GrindBits, RoundOpenings, + commit, prove, verify, + }, + whir_commit::Commitment, + whir_hash::{KeccakWhir, RpxWhir, WhirHash}, +}; + +type FE = FieldElement; +type EE = FieldElement; + +fn config(log_folding: usize, num_queries: usize, cap: CapPolicy) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding, + num_queries, + grind: GrindBits::default(), + format: ChainFormat { + cap, + ..ChainFormat::DEFAULT + }, + } +} + +struct Chain { + proof: ChainProof, + root: Commitment, + z: Vec, + y: EE, + domain: Domain, +} + +/// A base-field polynomial proved over the cubic tower, as production does: +/// round 0's blocks are base, every later one extension. +fn prove_chain(num_vars: usize, cfg: &ChainConfig, seed: u64) -> Chain { + let f = Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from((i.wrapping_add(seed)).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11)) + .collect(), + ) + .unwrap(); + let z: Vec = (0..num_vars) + .map(|i| EE::from(101 + 3 * i as u64 + seed)) + .collect(); + let y = f.evaluate_in(&z).unwrap(); + let (commitment, domain) = commit::(&f, cfg, true).unwrap(); + let proof = prove::( + &f, + &z, + &commitment, + &domain, + cfg, + &mut DefaultTranscript::::new(b"whir-cap"), + ) + .unwrap(); + Chain { + proof, + root: commitment.root(), + z, + y, + domain, + } +} + +fn check( + c: &Chain, + proof: &ChainProof, + cfg: &ChainConfig, +) -> Result<(), Error> { + verify::( + proof, + &c.root, + &c.z, + c.y, + &c.domain, + cfg, + &mut DefaultTranscript::::new(b"whir-cap"), + ) +} + +/// Path lengths of round `r`'s current and successor openings. +fn path_lens(round: &ChainRound) -> (Vec, Vec) { + match &round.openings { + RoundOpenings::Base(p) => ( + p.current + .iter() + .map(|o| o.proof.merkle_path.len()) + .collect(), + p.next.iter().map(|o| o.proof.merkle_path.len()).collect(), + ), + RoundOpenings::Extension(p) => ( + p.current + .iter() + .map(|o| o.proof.merkle_path.len()) + .collect(), + p.next.iter().map(|o| o.proof.merkle_path.len()).collect(), + ), + RoundOpenings::BaseGrouped(_) => panic!("these chains are per-polynomial"), + } +} + +fn current_path(round: &mut ChainRound, i: usize) -> &mut Vec { + match &mut round.openings { + RoundOpenings::Base(p) => &mut p.current[i].proof.merkle_path, + RoundOpenings::Extension(p) => &mut p.current[i].proof.merkle_path, + RoundOpenings::BaseGrouped(_) => panic!("these chains are per-polynomial"), + } +} + +fn next_path(round: &mut ChainRound, i: usize) -> &mut Vec { + match &mut round.openings { + RoundOpenings::Base(p) => &mut p.next[i].proof.merkle_path, + RoundOpenings::Extension(p) => &mut p.next[i].proof.merkle_path, + RoundOpenings::BaseGrouped(_) => panic!("these chains are per-polynomial"), + } +} + +/// Tree depths of a chain: tree `t` has `D_t − k_t` levels. +fn tree_depths(cfg: &ChainConfig, num_vars: usize) -> Vec { + let mut d = num_vars + cfg.log_blowup; + cfg.schedule(num_vars) + .iter() + .map(|k| { + d -= k; + d + }) + .collect() +} + +const SHAPES: [(usize, usize); 4] = [(6, 2), (5, 2), (3, 4), (9, 3)]; +const POLICIES: [CapPolicy; 4] = [ + CapPolicy::Fixed(1), + CapPolicy::Fixed(2), + CapPolicy::Fixed(3), + CapPolicy::Auto, +]; + +#[test] +fn tree_caps_follow_the_policy_and_are_zero_by_default() { + // The production chain: blowup 2, fold 4, 128 bits, uniform 20-bit grinds. + let mut cfg = ChainConfig::with_security(2, 4, 25, 128, GrindBits::uniform(20)); + assert_eq!(cfg.num_queries, 112); + assert_eq!(tree_depths(&cfg, 25), vec![23, 19, 15, 11, 7, 3, 2]); + assert_eq!(cfg.tree_caps(25), vec![0; 7], "the default caps nothing"); + cfg.format.cap = CapPolicy::Fixed(0); + assert_eq!(cfg.tree_caps(25), vec![0; 7]); + cfg.format.cap = CapPolicy::Auto; + // 112 and 224 openings: 3, clamped to the depth of the last tree. + assert_eq!(cfg.tree_caps(25), vec![3, 3, 3, 3, 3, 3, 2]); + cfg.format.cap = CapPolicy::Fixed(5); + assert_eq!(cfg.tree_caps(25), vec![5, 5, 5, 5, 5, 3, 2]); + + // Q = 3: tree 0 is opened 3 times (auto 0), every later tree 6 (auto 2). + let small = config(2, 3, CapPolicy::Auto); + assert_eq!(tree_depths(&small, 6), vec![6, 4, 2]); + assert_eq!(small.tree_caps(6), vec![0, 2, 2]); + // One round: the only tree is opened Q times. + let one = config(4, 25, CapPolicy::Auto); + assert_eq!(one.schedule(3), vec![3]); + assert_eq!( + one.tree_caps(3), + vec![2], + "25 openings -> 3, clamped to depth 2" + ); +} + +/// Round trips at every policy, one- and multi-round schedules, a remainder +/// last round, base and extension rounds, both hashes; every path carries +/// exactly `depth − c` siblings and the owner exactly `2^c` more. +#[test] +fn capped_chains_round_trip_with_the_owner_path_lengths() { + fn run() { + for (num_vars, k) in SHAPES { + for q in [3usize, 25] { + for policy in POLICIES { + let cfg = config(k, q, policy); + let caps = cfg.tree_caps(num_vars); + let depths = tree_depths(&cfg, num_vars); + let c = prove_chain::(num_vars, &cfg, 5); + let tag = format!("{} S={num_vars} k={k} Q={q} {policy}", H::NAME); + for (r, round) in c.proof.rounds.iter().enumerate() { + let (cur, nxt) = path_lens(round); + let owner = |t: usize, i: usize, owned: bool| { + depths[t] - caps[t] + + if owned && i == 0 && caps[t] > 0 { + 1 << caps[t] + } else { + 0 + } + }; + for (i, len) in cur.iter().enumerate() { + assert_eq!(*len, owner(r, i, r == 0), "{tag}: round {r} current {i}"); + } + for (i, len) in nxt.iter().enumerate() { + assert_eq!(*len, owner(r + 1, i, true), "{tag}: round {r} next {i}"); + } + } + check::(&c, &c.proof, &cfg).unwrap_or_else(|e| panic!("{tag}: {e:?}")); + } + } + } + } + run::(); + run::(); +} + +/// The default moves no byte: `Off` and `Fixed(0)` give the same archived +/// proof, and every path is the full depth. +#[test] +fn the_default_format_is_byte_identical_to_a_zero_cap() { + for (num_vars, k) in SHAPES { + let off = prove_chain::(num_vars, &config(k, 3, CapPolicy::Off), 1); + let zero = prove_chain::(num_vars, &config(k, 3, CapPolicy::Fixed(0)), 1); + let a = rkyv::to_bytes::(&off.proof).unwrap(); + let b = rkyv::to_bytes::(&zero.proof).unwrap(); + assert_eq!(a.as_slice(), b.as_slice(), "S={num_vars} k={k}"); + let cfg = config(k, 3, CapPolicy::Off); + let depths = tree_depths(&cfg, num_vars); + for (r, round) in off.proof.rounds.iter().enumerate() { + let (cur, nxt) = path_lens(round); + assert!(cur.iter().all(|l| *l == depths[r])); + assert!(nxt.iter().all(|l| *l == depths[r + 1])); + } + } +} + +/// The cap changes no transcript value. The same witness under +/// `Off`, `Fixed(3)` and `Auto` (no grinding, so the nonces are fixed) gives +/// the same sumchecks, roots, out-of-domain values, nonces and final value; +/// only the paths differ. +#[test] +fn the_cap_moves_no_transcript_value() { + for (num_vars, k) in SHAPES { + let base = prove_chain::(num_vars, &config(k, 25, CapPolicy::Off), 3); + for policy in [CapPolicy::Fixed(3), CapPolicy::Auto] { + let capped = prove_chain::(num_vars, &config(k, 25, policy), 3); + assert_eq!(capped.root, base.root); + assert_eq!(capped.proof.final_value, base.proof.final_value); + for (a, b) in capped.proof.rounds.iter().zip(&base.proof.rounds) { + assert_eq!(a.next_root, b.next_root); + assert_eq!(a.ood_value, b.ood_value); + assert_eq!(a.nonces, b.nonces); + let ev = |r: &ChainRound| -> Vec { + r.sumcheck + .iter() + .flat_map(|s| s.evaluations.clone()) + .collect() + }; + assert_eq!(ev(a), ev(b)); + } + } + } +} + +fn expect_err(c: &Chain, forged: &ChainProof, cfg: &ChainConfig, what: &str) -> Error { + match check::(c, forged, cfg) { + Ok(()) => panic!("{what}: the forgery verified"), + Err(e) => e, + } +} + +/// The tamper arm. The chain is `[2, 2, 2]` over 6 variables at `Fixed(2)`: +/// trees of depth 6, 4, 2, every one capped at 2. +#[test] +fn a_tampered_cap_or_owner_path_is_rejected() { + let cfg = config(2, 3, CapPolicy::Fixed(2)); + assert_eq!(cfg.tree_caps(6), vec![2, 2, 2]); + let c = prove_chain::(6, &cfg, 9); + check::(&c, &c.proof, &cfg).unwrap(); + let depths = [6usize, 4, 2]; + + // Tree 0's cap node: rides at the end of round 0's first current path. + for j in 0..4 { + let mut forged = c.proof.clone(); + current_path(&mut forged.rounds[0], 0)[depths[0] - 2 + j][5] ^= 1; + assert!(matches!( + expect_err(&c, &forged, &cfg, "tree-0 cap"), + Error::CapRejected + )); + } + // Tree t's cap node, t = 1, 2: rides on round t − 1's first successor path. + for (t, depth) in depths.iter().enumerate().skip(1) { + let mut forged = c.proof.clone(); + next_path(&mut forged.rounds[t - 1], 0)[depth - 2 + 1][0] ^= 1; + assert!(matches!( + expect_err(&c, &forged, &cfg, "tree-t cap"), + Error::CapRejected + )); + } + // Round t's first CURRENT opening carrying a cap as well: its path must + // be exactly depth − c, so a second cap for the same tree is refused. + for (t, depth) in depths.iter().enumerate().skip(1) { + let mut forged = c.proof.clone(); + let cap = next_path(&mut forged.rounds[t - 1], 0)[depth - 2..].to_vec(); + current_path(&mut forged.rounds[t], 0).extend(cap); + assert!(matches!( + expect_err(&c, &forged, &cfg, "a second cap on round t's current[0]"), + Error::OpeningRejected { query: 0 } + )); + } + // The owner path one short (a cap node dropped) and one long. + let mut forged = c.proof.clone(); + current_path(&mut forged.rounds[0], 0).pop(); + assert!(matches!( + expect_err(&c, &forged, &cfg, "owner short"), + Error::CapRejected + )); + let mut forged = c.proof.clone(); + let extra = current_path(&mut forged.rounds[0], 0)[0]; + current_path(&mut forged.rounds[0], 0).push(extra); + assert!(matches!( + expect_err(&c, &forged, &cfg, "owner long"), + Error::CapRejected + )); + // A non-owner path one node long, and one short. + let mut forged = c.proof.clone(); + let extra = current_path(&mut forged.rounds[0], 1)[0]; + current_path(&mut forged.rounds[0], 1).push(extra); + assert!(matches!( + expect_err(&c, &forged, &cfg, "non-owner long"), + Error::OpeningRejected { query: 1 } + )); + let mut forged = c.proof.clone(); + // Tree 1 (depth 4, two siblings below its cap), as round 0's successor. + assert!(next_path(&mut forged.rounds[0], 2).pop().is_some()); + assert!(matches!( + expect_err(&c, &forged, &cfg, "non-owner short"), + Error::OpeningRejected { query: 2 } + )); + // The cap moved from the first opening to the second. + let mut forged = c.proof.clone(); + let cap: Vec = current_path(&mut forged.rounds[0], 0) + .drain(depths[0] - 2..) + .collect(); + current_path(&mut forged.rounds[0], 1).extend(cap); + expect_err(&c, &forged, &cfg, "cap moved to query 1"); + // A sibling below the cap. + let mut forged = c.proof.clone(); + current_path(&mut forged.rounds[1], 2)[0][0] ^= 1; + assert!(matches!( + expect_err(&c, &forged, &cfg, "sibling"), + Error::OpeningRejected { query: 2 } + )); + + // A proof made under one policy is refused under another: the cap height + // is the verifier's constant, never read from the proof. + expect_err( + &c, + &c.proof, + &config(2, 3, CapPolicy::Fixed(1)), + "c=2 read as c=1", + ); + expect_err( + &c, + &c.proof, + &config(2, 3, CapPolicy::Fixed(3)), + "c=2 read as c=3", + ); + expect_err( + &c, + &c.proof, + &config(2, 3, CapPolicy::Off), + "c=2 read as off", + ); + let off = prove_chain::(6, &config(2, 3, CapPolicy::Off), 9); + expect_err(&off, &off.proof, &cfg, "off read as c=2"); +} + +/// A one-round chain has only the final round: tree 0's owner is its first +/// current opening there, and a flipped cap node is still refused. +#[test] +fn a_one_round_chain_carries_its_cap_on_the_final_openings() { + let cfg = config(4, 25, CapPolicy::Auto); + assert_eq!(cfg.tree_caps(3), vec![2]); + let c = prove_chain::(3, &cfg, 2); + assert_eq!(c.proof.rounds.len(), 1); + check::(&c, &c.proof, &cfg).unwrap(); + let mut forged = c.proof.clone(); + let path = current_path(&mut forged.rounds[0], 0); + // depth 2, cap 2: no siblings, four cap nodes. + assert_eq!(path.len(), 4); + path[3][7] ^= 1; + assert!(matches!( + check::(&c, &forged, &cfg), + Err(Error::CapRejected) + )); +} diff --git a/crypto/multilinear/src/whir_chain.rs b/crypto/multilinear/src/whir_chain.rs new file mode 100644 index 000000000..ba1bb11e5 --- /dev/null +++ b/crypto/multilinear/src/whir_chain.rs @@ -0,0 +1,4590 @@ +//! WHIR with its rounds chained: fold `k` variables at a time, committing the +//! successor codeword each round, so a query opens a block of `2^k` instead of +//! the whole message. +//! +//! The **committed** codeword is base-field, because a trace is. The extension +//! only enters at the first fold, so the largest structure in the proof — the +//! codeword and the Merkle leaves over it, `blowup` times the trace — costs a +//! third of what it would over the extension. That is why the first round's +//! openings have a different value type from every later one, and the type says +//! so rather than leaving it to a reader. +//! +//! [`whir_eval`](crate::whir_eval) is the one-round case, and there a block +//! *is* the message — which is what dominates its proof size and its verifier's +//! work. Here the sumcheck runs in groups of `k`: after each group the codeword +//! folds by `2^k`, the successor is committed **before** the queries are drawn, +//! and each query checks that folding a current block really lands on the +//! successor's value there. The last group has no successor — the message is a +//! constant by then, and that constant is sent. +//! +//! Each round also draws an **out-of-domain** point `z0` and takes the +//! successor's value there. Nothing about the domain constrains that value, so +//! answering it is what forces the committed word to be near a *single* +//! codeword rather than merely near the code — the in-domain queries alone +//! leave a prover room to be near several. The claim is folded into the weight +//! the next group carries, batched with a challenge, which is why the whole +//! thing runs on the weighted form: +//! +//! ```text +//! w_{r+1} = w_r(alpha_r, ·) + gamma_r·eq(ood_r, ·) +//! C_{r+1} = C_r' + gamma_r·y0_r +//! ``` +//! +//! Three challenges per round are worth redrawing for a cheating prover — the +//! folding randomness, the out-of-domain batching challenge and the query +//! positions — and [`GrindBits`] can put a [proof of work](crypto::grinding) +//! before each: retrying one costs `2^bits` hashes. Only the query grind raises +//! the proven minimum as placed. It sits right before the positions, which is +//! what lets a query count buy more soundness than its own bits. The folding +//! grind sits before the round's FIRST sumcheck message, so the first folding +//! challenge is redrawn by varying that message without grinding again. The +//! out-of-domain grind comes after the out-of-domain point, so the only +//! challenge it guards is the batching one, which has far more bits than the +//! target without any grind. So [`GrindBits::query_only`] loses no proven +//! bits, and with [`NonceLayout::Spent`] the proof carries one nonce a round. +//! +//! The claim is the weighted one, `Σ_x w(x)·f(x) = y`, so a stacked multi-point +//! claim chains just as an evaluation does. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +pub use crypto::merkle_tree::cap::CapPolicy; +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; + +use crate::{ + Error, + commit_ood::Overlay, + eq::{eq_eval, eq_mle}, + mle::Mle, + narrow::HostColumn, + poly::Composed, + sumcheck::{self, RoundProof as SumcheckRoundProof}, + whir::{Domain, encode, fold_codeword_k, lift_coefficients}, + whir_commit::{Codeword, CodewordCommitment, Commitment, fold_coset, verify_opening_capped}, + whir_hash::{DeviceHashKey, GrindingDigest, WhirHash}, + whir_round::{ + self, GroupedRoundProof, RoundCaps, RoundCommitments, RoundConfig, RoundProof, TreeCheck, + }, +}; + +pub use crate::whir_commit::LeafOrder; + +/// Why a round-0 successor was not encoded from its bound message — what +/// [`Error::CosetSuccessorNotEncoded`] names. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UnencodedSuccessor { + /// `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0` (or a test's override): the fold + /// was chosen. + KnobOff, + /// The opening's factors were built on the host — the card declined the + /// opening (`gpu::open_host_fallbacks`). Not returned by a prove: the + /// successor is then encoded on the host from the message those factors + /// bound (`gpu::coset_host_encodes`). + HostFactors, + /// The card was asked for the encode and declined it + /// (`gpu::encode_declines`). + CardDeclined, +} + +impl UnencodedSuccessor { + /// The cause, from whether the encode was on and where the factors were. + pub fn of(encode_on: bool, factors_on_card: bool) -> Self { + match (encode_on, factors_on_card) { + (false, _) => Self::KnobOff, + (true, false) => Self::HostFactors, + (true, true) => Self::CardDeclined, + } + } +} + +/// Refuses a format whose round-0 leaf order `H`'s trees do not take: the +/// coset order is defined for Poseidon1 trees alone — the production hash, +/// base and recursion alike — so under any other hash it is not a state a +/// prover or a verifier can reach. Called before a round-0 tree is built +/// ([`commit_stacked`]), before a chain is proved, and before a proof is read. +pub fn check_leaf_order(format: &ChainFormat) -> Result<(), Error> { + match format.leaves { + LeafOrder::Natural => Ok(()), + LeafOrder::Coset if H::DEVICE == DeviceHashKey::Poseidon1 => Ok(()), + order => Err(Error::LeafOrderUnsupported { + order, + hash: H::NAME, + }), + } +} + +/// `w(x)·f(x)`, the shape every group's sumcheck runs over. +fn product(values: &[FieldElement]) -> FieldElement { + &values[0] * &values[1] +} + +/// The multilinear point where the univariate lift is `z0`. +/// +/// [`lift_coefficients`] puts variable `j` on the monomial `X^(2^j)`, so +/// `F(z0) = f(z0, z0^2, z0^4, ..)`. +pub fn ood_point(z0: &FieldElement, num_vars: usize) -> Vec> { + let mut power = z0.clone(); + (0..num_vars) + .map(|_| { + let current = power.clone(); + power = power.square(); + current + }) + .collect() +} + +/// Rejects a point that landed inside `domain`, where it would constrain +/// nothing beyond the in-domain queries. +/// +/// Sampling from the transcript makes this negligible; it is checked rather +/// than assumed. +fn require_out_of_domain(z0: &FieldElement, domain: &Domain) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf, + E: IsField + Send + Sync + 'static, +{ + require_out_of_domain_of_size(z0, domain.log_size()) +} + +/// [`require_out_of_domain`] against the subgroup of order `2^log_size`: what +/// a commit-time point is checked against (`crate::commit_ood`), and every +/// smaller subgroup lies inside it. +pub fn require_out_of_domain_of_size( + z0: &FieldElement, + log_size: usize, +) -> Result<(), Error> { + let mut power = z0.clone(); + for _ in 0..log_size { + power = power.square(); + } + if power == FieldElement::::one() { + return Err(Error::OodPointInDomain); + } + Ok(()) +} + +/// Proof of work over the transcript state, absorbed before the next challenge. +/// +/// Retrying that challenge then costs `2^bits` hashes. Zero bits is a no-op, so +/// a caller that has not chosen its parameters yet pays nothing. +fn grind(transcript: &mut T, bits: u8) -> Result +where + E: IsField + Send + Sync + 'static, + T: IsTranscript, + H: WhirHash, +{ + if bits == 0 { + return Ok(0); + } + let nonce = + crypto::grinding::generate_nonce_maybe_gpu::>(&transcript.state(), bits) + .ok_or(Error::GrindingFailed { bits })?; + transcript.append_bytes(&nonce.to_be_bytes()); + Ok(nonce) +} + +/// The verifier's half: the nonce must pass against the same state, and it is +/// absorbed the same way. +fn check_grind(transcript: &mut T, bits: u8, nonce: u64) -> Result<(), Error> +where + E: IsField + Send + Sync + 'static, + T: IsTranscript, + H: WhirHash, +{ + if bits == 0 { + return Ok(()); + } + if !crypto::grinding::is_valid_nonce::>(&transcript.state(), nonce, bits) { + return Err(Error::GrindingRejected { bits }); + } + transcript.append_bytes(&nonce.to_be_bytes()); + Ok(()) +} + +/// A nonce slot the round does not spend: a grind of zero bits, or the last +/// round's out-of-domain slot. [`NonceLayout::Three`] carries it and never +/// reads it; [`NonceLayout::Spent`] has no such nonce, so the field standing in +/// for it must be 0. No transcript is touched either way. +fn check_unspent(layout: NonceLayout, nonce: u64) -> Result<(), Error> { + match layout { + NonceLayout::Spent if nonce != 0 => Err(Error::UnspentNonce), + _ => Ok(()), + } +} + +/// One nonce slot, as the layout has it: [`check_grind`] where the grind spends +/// bits, [`check_unspent`] where it does not. +fn check_nonce( + transcript: &mut T, + bits: u8, + nonce: u64, + layout: NonceLayout, +) -> Result<(), Error> +where + E: IsField + Send + Sync + 'static, + T: IsTranscript, + H: WhirHash, +{ + if bits == 0 { + return check_unspent(layout, nonce); + } + check_grind::(transcript, bits, nonce) +} + +/// `w + gamma·eq`, the weight the next group carries. +fn batch_weight( + w: &Mle, + eq: &Mle, + gamma: &FieldElement, +) -> Result, Error> { + Mle::new( + w.evals() + .iter() + .zip(eq.evals()) + .map(|(a, b)| a + gamma * b) + .collect(), + ) +} + +/// A stacked polynomial's weight built on the host: its shares' table, plus +/// the commit-time overlay when the opening carries one. +fn host_weight( + shares: &[crate::stacked_eval::WeightShare<'_, E>], + overlay: Option<&Overlay<'_, E>>, + n_stack: usize, +) -> Result, Error> +where + FieldElement: Send + Sync, +{ + let weight = crate::stacked_eval::weight_table(shares, n_stack)?; + match overlay { + Some(overlay) => overlay.add_to(&weight), + None => Ok(weight), + } +} + +/// Proof-of-work bits, in the three places a prover would want to redraw. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct GrindBits { + /// Before a group's folding challenges. + pub folding: u8, + /// Before the out-of-domain batching challenge. + pub ood: u8, + /// Before the query positions. + pub query: u8, +} + +impl GrindBits { + /// The same count in all three places. + pub const fn uniform(bits: u8) -> Self { + Self { + folding: bits, + ood: bits, + query: bits, + } + } + + /// Bits before the query positions only: the one grind that raises the + /// proven minimum as placed (module header). The query count reads `query` + /// alone, so it equals [`uniform`](Self::uniform)'s at the same bits. + pub const fn query_only(bits: u8) -> Self { + Self { + folding: 0, + ood: 0, + query: bits, + } + } +} + +/// Blowup, fold factor, query count and proof of work. +/// +/// `format` is the proof FORMAT ([`ChainFormat`]: the W1 cap and W2 fold +/// levers); its default is today's format. Like the rest of the config it is +/// a verifier-side constant, never read from a proof. The fold schedule is +/// absorbed into the statement through [`ChainConfig::fold_word`], whose value +/// at the default is `log_folding` itself (today's bytes); the rest of the +/// format is not absorbed (`push_config` binds it as `_`): absorbing it would +/// move every transcript at the default. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ChainConfig { + /// `log2` of the code's inverse rate. + pub log_blowup: usize, + /// Variables folded per round. The last round takes whatever is left. + pub log_folding: usize, + /// Positions checked per round. + pub num_queries: usize, + /// Proof of work before each redrawable challenge. + pub grind: GrindBits, + /// The chain's proof format. [`ChainFormat::DEFAULT`] = today. + pub format: ChainFormat, +} + +/// The proof-format levers of a WHIR chain. Grouped so a literal +/// `ChainConfig` names the format in one line (`format: ChainFormat::DEFAULT`) +/// and a lever added later touches this struct only. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct ChainFormat { + /// Merkle cap policy for the chain's commitment trees (W1). `Off` = today. + pub cap: CapPolicy, + /// Per-round fold schedule (W2). `Uniform` = today (`log_folding` every + /// round, the remainder last). + pub folds: WhirFolds, + /// How wide a stacked polynomial may get (S2). [`StackVars::LEGACY`] = + /// today (25). + pub stack: StackVars, + /// Which nonces a round carries (P2). [`NonceLayout::Three`] = today. + pub nonces: NonceLayout, + /// How a proof's tables argue their constraints and buses (D-BATCH). + /// [`ArgueFormat::PerTable`] = today. + pub argue: ArgueFormat, + /// The Merkle cap policy of a hash whose trees are 4-ary (Poseidon1, + /// D-WHIR-P1 D2), its heights in 4-ary levels: [`cap`](Self::cap) prices + /// binary caps only. Read at arity 4 alone, so it cannot move a binary + /// hash's bytes; `Off` = today. + pub arity4_cap: CapPolicy, + /// How a stack's polynomials are cut into chains (D-ONECHAIN, W-14). + /// [`GroupFold::PerPoly`] = today. + pub group: GroupFold, + /// Where the round-0 trees put their leaves. [`LeafOrder::Natural`] = + /// today. + pub leaves: LeafOrder, +} + +impl ChainFormat { + /// Today's format: every lever off. + pub const DEFAULT: Self = Self { + cap: CapPolicy::Off, + folds: WhirFolds::Uniform, + stack: StackVars::LEGACY, + nonces: NonceLayout::Three, + argue: ArgueFormat::PerTable, + arity4_cap: CapPolicy::Off, + group: GroupFold::PerPoly, + leaves: LeafOrder::Natural, + }; + + /// True when this is today's format (`Fixed(0)` counts as `Off`). + pub fn is_default(&self) -> bool { + self.cap.is_off() + && self.folds == WhirFolds::Uniform + && self.stack == StackVars::LEGACY + && self.nonces == NonceLayout::Three + && self.argue == ArgueFormat::PerTable + && self.arity4_cap.is_off() + && self.group == GroupFold::PerPoly + && self.leaves == LeafOrder::Natural + } + + /// The cap policy of trees with `arity` children per node: + /// [`cap`](Self::cap) at arity 2, [`arity4_cap`](Self::arity4_cap) at 4. + pub fn cap_policy_at(&self, arity: usize) -> CapPolicy { + if arity == 4 { + self.arity4_cap + } else { + self.cap + } + } +} + +/// How a proof's tables argue their constraints and buses (D-BATCH, +/// `thoughts/zf/gap2/fix2/D-BATCH.md` §2). +/// +/// [`PerTable`](Self::PerTable) is today's: each table runs its own LogUp-GKR, +/// its own constraint sumcheck and its own claim reduction. +/// [`Batched`](Self::Batched) runs one lockstep GKR per bin of tables, one +/// front-loaded constraint sumcheck over every table, and no claim reduction +/// for a table without shifted reads (`stark::multilinear_table::batched`). +/// +/// ★ A FORMAT VALUE, NOT A HINT. The two produce different proof types, and a +/// verifier refuses a proof argued under the other variant: it reads the +/// variant from its own config, never from a proof. The bin cap is part of +/// the format because the bins are part of the transcript. +/// +/// A verifier-side constant like the rest of [`ChainFormat`], and not absorbed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum ArgueFormat { + /// One argument per table. Today's format. + #[default] + PerTable, + /// One argument per proof, the GKR trees in bins of at most + /// `2^bin_log_cells` input cells each. + Batched { + /// `log2` of a bin's cap on `Σ 2^{k_T}`, `k_T` a tree's input + /// variables. A tree at or over the cap sits alone. + bin_log_cells: u8, + /// How a table's interactions are laid into trees: production + /// [`GkrPack::PRODUCTION`]; [`GkrPack::Off`] is one tree a table, its + /// interactions padded to a power of two. + pack: GkrPack, + }, +} + +impl ArgueFormat { + /// The production cap: `2^26` input cells a bin. D-BATCH §4.4 planned + /// `2^27` with `2^26` its fallback (§8.2); on the block's 15 epochs bins of + /// `2^27` overran the card (12 tree builds refused, FAST 347) and bins of + /// `2^26` refused none. Its trees are packed at + /// [`GkrPack::PRODUCTION`]. + pub const BATCHED: Self = Self::batched(26); + + /// The batched format at `bin_log_cells`, its trees packed at + /// [`GkrPack::PRODUCTION`]. + pub const fn batched(bin_log_cells: u8) -> Self { + Self::Batched { + bin_log_cells, + pack: GkrPack::PRODUCTION, + } + } +} + +/// How the batched argue lays a table's interactions into LogUp-GKR trees. +/// +/// [`Off`](Self::Off) is one tree a table over its interactions padded to a +/// power of two with `0/1`. [`Chunks`](Self::Chunks) cuts a table +/// whose padding costs more than the trees it would add into `c` trees of +/// `2^b` slots each, the chunks of its interactions in order, all in one bin: +/// they end at one point, and their claims combine at a fresh challenge into +/// the claim today's one tree would leave, so the constraint phase is +/// unchanged. Which tables are cut and how is a function of their shapes and +/// `tree_log_cells` alone. +/// +/// ★ A FORMAT VALUE: the trees, their bus outputs and their ladder halves are +/// part of the proof, so a verifier argues under its own value, never a +/// proof's. A verifier-side constant like the rest of [`ChainFormat`], and not +/// absorbed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum GkrPack { + /// One tree a table, the format before packing. + #[default] + Off, + /// A table cut into equal chunks where that saves cells: each extra tree + /// is priced at `2^tree_log_cells` input cells. + Chunks { + /// `log2` of the input cells one more tree is worth. + tree_log_cells: u8, + }, +} + +impl GkrPack { + /// The production packing: each extra tree priced at `2^22` input cells. + /// On a block's tables that cuts CPU, MEMW_A and SHIFT in 3 and KECCAK_RND + /// in 5. + pub const PRODUCTION: Self = Self::Chunks { tree_log_cells: 22 }; +} + +/// Which proof-of-work nonces a round's encoding carries (P2's format half). +/// +/// A round has three nonce slots, folding, out-of-domain and query, and spends +/// the ones whose grind has bits. At zero bits a grind searches, reads and +/// absorbs nothing, and the last round's out-of-domain slot has no grind at all +/// (there is no successor to batch). +/// +/// ★ WHY IT IS A FORMAT FIELD. Under [`Three`](Self::Three) an unspent slot is +/// carried and never read, so any value in it verifies: an unbound field. +/// [`GrindBits::query_only`] leaves two of those a round. [`Spent`](Self::Spent) +/// takes them out of the format. The in-guest encoding +/// (`prover::lfm::whir_chain`'s arena) has no word for an unspent nonce, so +/// there is nothing to set. The host [`RoundNonces`] keeps its three fields, so +/// both layouts share one proof type and `Three` keeps its bytes, and the +/// verifier refuses a nonzero value in a field the layout does not carry +/// ([`Error::UnspentNonce`]). +/// +/// A verifier-side constant like the rest of [`ChainFormat`], and not absorbed: +/// the grind bits, which decide which slots are spent, are (the statement's +/// grind trailer). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum NonceLayout { + /// Three nonces a round, whatever the bits. Today's format. + #[default] + Three, + /// Only the nonces the round spends: one a round under + /// [`GrindBits::query_only`]. + Spent, +} + +/// How wide one stacked polynomial may get, in variables: the stack cap (S2). +/// +/// The knob between proof size and peak memory, a resource tradeoff rather +/// than an optimization. Every extra variable halves the number of stacked +/// polynomials — and so the number of chains, which is most of a WHIR proof +/// and of the work of verifying one in-guest — and doubles the biggest single +/// allocation: a polynomial of `n` variables is encoded over +/// `2^(n + log_blowup)` field elements, 1 GiB in the base field at 25 and +/// blowup 4, 4 GiB at 27. +/// +/// ★ A FORMAT VALUE, NOT A HINT. A group's layout is a function of its shapes +/// and this cap, so the prover, the host verifier and the in-guest emitters +/// all build it from the config they are given, and a proof stacked under one +/// cap does not verify under another. Nothing reads it from a proof. +/// +/// Constructed only through [`StackVars::new`], so a cap of 0 or wider than +/// [`StackVars::WIDEST`] is not a value a config can hold. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct StackVars(u8); + +impl StackVars { + /// Today's cap: one 1 GiB base codeword. + pub const LEGACY: Self = Self(25); + + /// The widest cap a format may ask for. 27 is the widest measured to fit + /// a 32 GiB card (block 25368371, three runs: the argument's reserved peak + /// 24,218 MiB of a 25,688 MiB budget, the device peak at most 30,736 MiB); + /// at 28 the argument's reservations land 2–2.6 GiB over the budget in the + /// heaviest epochs. The device commit needs the NTT's grid split past 26. + pub const WIDEST: usize = 27; + + /// `None` outside `1..=WIDEST`. + pub const fn new(n: usize) -> Option { + if n >= 1 && n <= Self::WIDEST { + Some(Self(n as u8)) + } else { + None + } + } + + pub const fn get(self) -> usize { + self.0 as usize + } +} + +impl Default for StackVars { + fn default() -> Self { + Self::LEGACY + } +} + +/// See [`WHIR_CAP_IMPLEMENTED`]. S2's stack is in: the layouts the prover, the +/// host verifier and the in-guest emitters build, the device commit of a +/// 2^29-point codeword (the NTT and Möbius grid split) and the argument's +/// ledger (the rooms given back across it). +pub const WHIR_STACK_IMPLEMENTED: bool = true; + +/// Which WHIR format levers THIS build implements. A lever that is only +/// parsed must not be selectable (see `stark::proof::options:: +/// MERKLE_CAP_IMPLEMENTED`). Each flag is flipped in the commit that +/// makes the lever real. +/// +/// W1 (the Merkle cap) is real: host prover and verifier ([`ChainConfig:: +/// tree_caps`], the owner-path encoding), the device (`paths_and_cap`), and +/// the in-guest verifier and its cost model (`prover::lfm::whir_chain`). +pub const WHIR_CAP_IMPLEMENTED: bool = true; + +/// The widest fold any round of a chain may take. +/// +/// The stack is tested up to it and no further: the GPU commit/fold parity +/// (`math-cuda` `whir_commit`/`whir_fold`, k = 6) and the in-guest fold +/// emitter (`lfm::whir_fold_tests`, k = 5 and 6). `k0 = 7` loses on in-guest +/// instructions, so nothing above 6 is opened. +pub const MAX_FOLD: usize = 6; + +/// The per-round fold schedule of a chain (W2). +/// +/// ★ Why a FIRST fold and not a list. A config serves chains of every height +/// (`chain_config` takes the tallest stack, and each chain folds its own +/// `num_vars`), so a per-round list would have to say what a shorter chain +/// does with it. The lever is the first fold alone — +/// tree 0 is the only base-field tree, opened `Q` times rather than `2Q`, and +/// every variable it takes shortens every later tree — so the schedule is +/// "`k0`, then today's uniform walk", a function of `(k0, log_folding, +/// num_vars)` at every height. There is no DP. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum WhirFolds { + /// `log_folding` variables every round, the remainder last. Today's format. + #[default] + Uniform, + /// The first round folds `k0` variables (all of them when the chain has + /// fewer); every later round is today's walk: `log_folding`, the remainder + /// last. `first5` / `first6`. + First(FirstFold), +} + +/// See [`WHIR_CAP_IMPLEMENTED`]. W2 is in: the host chain, the statement word, +/// `agrees_with`, the production config, the GPU parity at k = 6 and the +/// in-guest gates at k = 5 and 6. +pub const WHIR_FOLDS_IMPLEMENTED: bool = true; + +/// A first-round fold, `1 ..= MAX_FOLD`. Constructed only through +/// [`FirstFold::new`], so a fold of 0 or wider than the tested stack is not a +/// value a config can hold. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct FirstFold(u8); + +impl FirstFold { + /// `None` outside `1..=MAX_FOLD`. + pub const fn new(k: usize) -> Option { + if k >= 1 && k <= MAX_FOLD { + Some(Self(k as u8)) + } else { + None + } + } + + pub const fn get(self) -> usize { + self.0 as usize + } +} + +/// The top bit of a non-uniform [`ChainConfig::fold_word`]. A uniform word is +/// `log_folding`, far below it, so no non-default word equals a default one. +pub const FOLD_WORD_TAG: u64 = 1 << 63; + +/// The bit of [`ChainConfig::fold_word`] that says [`GroupFold::Indexed`], +/// always beside [`FOLD_WORD_TAG`]. No other word sets it: a `First` word's +/// fields sit at bits 52+, 48 and the low nibble. +pub const FOLD_WORD_INDEXED: u64 = 1 << 47; + +/// The bit of [`ChainConfig::fold_word`] that says [`LeafOrder::Coset`], +/// always beside [`FOLD_WORD_TAG`]. No other word sets it: a `First` word's +/// fields sit at bits 52+, 48 and the low nibble, and [`FOLD_WORD_INDEXED`] +/// is bit 47. +pub const FOLD_WORD_COSET: u64 = 1 << 46; + +/// How a stack's polynomials are opened (D-ONECHAIN, the ledger's W-14). +/// +/// [`PerPoly`](Self::PerPoly) is today's: one chain per stacked polynomial. +/// [`Indexed`](Self::Indexed) cuts them into consecutive runs of at most +/// [`MAX_INDEX_POLYS`] (`stacked_eval::index_chains`) and opens a run of +/// `P' ≥ 2` with ONE chain at `n + m` variables, `m = ⌈log2 P'⌉` +/// ([`index_vars`]): +/// - round 0 runs `k0 + m` sumcheck rounds, the `m` index challenges `ρ` LAST; +/// - it opens the run's `P'` round-0 trees at ONE position set, combines each +/// query's blocks as `Σ_p eq(ρ, p)·block_p` ([`eq_index`]) and folds that as +/// today's `2^k0` block; +/// - rounds ≥ 1 are today's chain on the one successor. +/// +/// A run of one polynomial is today's chain exactly. The chain is today's at +/// `n + m` variables restricted to round-0 words whose absent slots are zero +/// (D-ONECHAIN §3.3, Lemma B), which needs `m ≤ 2` and `n + m + log_blowup` +/// within the field's two-adicity ([`check_index_domain`]). +/// +/// ★ A FORMAT VALUE, NOT A HINT. A verifier-side constant like the rest of +/// [`ChainFormat`], bound into the statement through [`ChainConfig::fold_word`] +/// ([`FOLD_WORD_INDEXED`]), and never read from a proof. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum GroupFold { + /// One chain per stacked polynomial. Today's format. + #[default] + PerPoly, + /// One chain per run of at most [`MAX_INDEX_POLYS`] polynomials. + Indexed, +} + +/// The most index variables a grouped chain binds. `m = 3` drops the +/// conjectured round-0 fold term below 128 bits (127.393 at a 2^27 stack, +/// REV-B2-JUDGE §5), and past it Lemma B's virtual domain stops existing. +pub const MAX_INDEX_VARS: usize = 2; + +/// The most polynomials one grouped chain opens: `2^MAX_INDEX_VARS`. +pub const MAX_INDEX_POLYS: usize = 1 << MAX_INDEX_VARS; + +/// `m = ⌈log2 P'⌉`: the index variables a chain over `polys` polynomials +/// binds. 0 for one polynomial, which is today's chain. +pub const fn index_vars(polys: usize) -> usize { + polys.next_power_of_two().trailing_zeros() as usize +} + +/// `eq(ρ, p)`, with `ρ_0` against the MOST significant of `p`'s `ρ.len()` +/// bits: index variable `y_0` is the high bit of the slot, as the index rounds +/// bind it (the run's tables concatenated in slot order, variable 0 the +/// highest). +/// +/// ★ THE ONE CONVENTION. The prover's factors after the index rounds, its +/// round-0 successor, the verifier's combine and the verifier's weight all +/// read it, so they cannot disagree (REV-B2-JUDGE B1-3). +pub fn eq_index(rho: &[FieldElement], p: usize) -> FieldElement { + let m = rho.len(); + rho.iter() + .enumerate() + .fold(FieldElement::::one(), |acc, (i, r)| { + if (p >> (m - 1 - i)) & 1 == 1 { + acc * r + } else { + acc * (FieldElement::::one() - r) + } + }) +} + +/// Refuses an index width whose virtual chain has no domain: Lemma B's +/// `H_{n+m+lb}` exists only when `n + m + log_blowup ≤` the field's two-adicity +/// (REV-B2-A C5). A verifier constant check, run before anything is read. +pub fn check_index_domain( + num_vars: usize, + index_vars: usize, + log_blowup: usize, + two_adicity: usize, +) -> Result<(), Error> { + if num_vars + index_vars + log_blowup > two_adicity { + return Err(Error::GroupedShape(GroupedShape::IndexDomain { + num_vars, + index_vars, + log_blowup, + two_adicity, + })); + } + Ok(()) +} + +/// Test only: verifier faults a grouped-chain test switches on, one at a time, +/// to show that each use of the index convention is load-bearing +/// (REV-B2-JUDGE B1-3): with any of them an honest proof must be refused. +/// Thread-local, so a test that sets one moves no other test's verifier. +#[cfg(test)] +pub(crate) mod grouped_faults { + /// Which piece of the verifier reads the index wrongly. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub(crate) enum Fault { + #[default] + None, + /// The round-0 combine reads `ρ` reversed (`ρ_0 ↔ ρ_1` at `m = 2`). + SwapRhoInCombine, + /// The final weight reads `ρ` reversed. + SwapRhoInWeight, + /// The final weight drops `eq(ρ, p)`: `Σ_p w_p` at the data point. + DropIndexFactor, + /// The final weight gives each absent slot `p ≥ P'` a weight of 1. + AbsentSlotWeightOne, + } + + std::thread_local! { + static FAULT: std::cell::Cell = const { std::cell::Cell::new(Fault::None) }; + } + + pub(crate) fn get() -> Fault { + FAULT.with(std::cell::Cell::get) + } + + /// Runs `f` under `fault`, restoring the previous one after. + pub(crate) fn with(fault: Fault, f: impl FnOnce() -> R) -> R { + struct Restore(Fault); + impl Drop for Restore { + fn drop(&mut self) { + FAULT.with(|c| c.set(self.0)); + } + } + let _restore = Restore(FAULT.with(|c| c.replace(fault))); + f() + } +} + +/// The shape a grouped opening must have, refused by name before any block is +/// indexed or zipped (REV-B2-JUDGE B1-2). A lenient count would let a prover +/// choose, after the positions are drawn, whether a slot reads its committed +/// block or nothing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GroupedShape { + /// A stacked proof's chain count is not the derived split's. + ChainCount { expected: usize, got: usize }, + /// A chain asked to open no polynomial, or more than [`MAX_INDEX_POLYS`]. + RunWidth { polys: usize }, + /// The virtual chain's domain does not exist ([`check_index_domain`]). + IndexDomain { + num_vars: usize, + index_vars: usize, + log_blowup: usize, + two_adicity: usize, + }, + /// Round 0 of a grouped chain carries other than `k0 + m` messages. + Round0Messages { expected: usize, got: usize }, + /// A round's openings are the grouped variant where they must not be, or + /// not it where they must be: round 0 of a chain over `P' ≥ 2` and nowhere + /// else. + Variant { round: usize }, + /// Other than `Q` queries. + Queries { expected: usize, got: usize }, + /// A query carries other than one block per round-0 tree. + BlocksPerQuery { + query: usize, + expected: usize, + got: usize, + }, + /// A block of other than `2^k0` values. + BlockWidth { + query: usize, + tree: usize, + expected: usize, + got: usize, + }, + /// Other than one successor block a query (none on a final round). + SuccessorBlocks { expected: usize, got: usize }, +} + +impl ChainConfig { + /// Parameters for a security target, in the **same regime the univariate + /// prover uses**: the Johnson bound, `proximity = 1 − √rate − 1/300`, so + /// each query buys `−log2(1 − proximity)` bits. Grinding on the query phase + /// is subtracted from what the queries have to buy, as it is there. + /// + /// Every round is spot-checked independently, so their errors add: the + /// target carries `log2(rounds)` of margin on top (a union bound). + /// + /// **What this is not.** WHIR's own analysis is per-round and gets to use + /// the out-of-domain point, which this ignores entirely. So it is the + /// conservative mirror of the parameters the repo already ships — read it as + /// "no weaker than the FRI ones next door" — and **not** a soundness proof + /// about this protocol. Doing that analysis is what would let the query + /// count come down. + pub fn with_security( + log_blowup: usize, + log_folding: usize, + num_vars: usize, + security_bits: u8, + grind: GrindBits, + ) -> Self { + Self::with_security_folds( + log_blowup, + log_folding, + WhirFolds::Uniform, + num_vars, + security_bits, + grind, + ) + } + + /// [`with_security`](Self::with_security) under a fold schedule. + /// + /// ★ THE ONE Q RULE. The query count is one number for every chain the + /// config serves, so the union bound is charged the WORST round count any + /// chain of at most `num_vars` variables has under `folds`. For `Uniform` + /// that is `ceil(num_vars / log_folding)` — today's count, exactly — and a + /// first fold `k0 >= log_folding` never has more rounds than that at any + /// height, so it never raises Q (`first5`/`first6` keep 112 at 25). + /// + /// The rate is `2^-log_blowup` in every round whatever the schedule (the + /// domain loses `k_r` bits as the message loses `k_r` variables), so the + /// per-query bits do not move; only `rounds` does. The disclaimer on + /// [`with_security`](Self::with_security) applies unchanged. + pub fn with_security_folds( + log_blowup: usize, + log_folding: usize, + folds: WhirFolds, + num_vars: usize, + security_bits: u8, + grind: GrindBits, + ) -> Self { + let mut config = Self { + log_blowup, + log_folding, + num_queries: 0, + grind, + format: ChainFormat { + folds, + ..ChainFormat::DEFAULT + }, + }; + let rounds = (1..=num_vars) + .map(|m| config.rounds(m)) + .max() + .unwrap_or(0) + .max(1); + // ★ Integers, not `f64`. The arithmetic and its provenance are in + // [`crate::query_count`]; what matters here is that the count a + // verifier has to reproduce no longer needs floating point to + // reproduce it, and that the answers did not move — the shipped + // posture's 110 / 112 / 113 are pinned in both places. + config.num_queries = + crate::query_count::num_queries(log_blowup, rounds, security_bits, grind.query); + config + } + + /// Rounds a chain of `num_vars` variables runs: `schedule(num_vars).len()`. + pub fn rounds(&self, num_vars: usize) -> usize { + self.schedule(num_vars).len() + } + + /// The statement's fold word: what the three host absorbs and the LFM's + /// `push_config` write where they wrote `log_folding`. + /// + /// - `Uniform` → `log_folding`: `4u64` at the default, today's bytes. + /// - `First(k0)` → `FOLD_WORD_TAG | log_folding << 52 | 1 << 48 | k0`: + /// a prefix encoding with a one-entry prefix (tail + /// `log_folding`, length 1, the fold in the low nibble). + /// + /// - [`GroupFold::Indexed`] → either word above with [`FOLD_WORD_TAG`] and + /// [`FOLD_WORD_INDEXED`] set, so a grouped statement shares no word with + /// a per-polynomial one under any fold schedule. + /// - [`LeafOrder::Coset`] → any word above with [`FOLD_WORD_TAG`] and + /// [`FOLD_WORD_COSET`] set: the round-0 trees' roots and paths follow the + /// order, so a statement under one order shares no word with the other. + /// + /// Every chain's schedule is a function of this word and its own + /// `num_vars`, which the statement already binds, so binding the word + /// binds every schedule — including the heights where two policies give + /// the same schedule (`first6` and `uniform4` at `num_vars <= 4`), where + /// only the word tells the proofs apart. + pub fn fold_word(&self) -> u64 { + let word = match self.format.folds { + WhirFolds::Uniform => self.log_folding as u64, + WhirFolds::First(k0) => { + FOLD_WORD_TAG + | ((self.log_folding as u64 & 0x7ff) << 52) + | (1 << 48) + | k0.get() as u64 + } + }; + let word = match self.format.group { + GroupFold::PerPoly => word, + GroupFold::Indexed => word | FOLD_WORD_TAG | FOLD_WORD_INDEXED, + }; + match self.format.leaves { + LeafOrder::Natural => word, + LeafOrder::Coset => word | FOLD_WORD_TAG | FOLD_WORD_COSET, + } + } + + /// The sumcheck messages each round of a chain over `num_vars`-variable + /// polynomials carries when it binds `index_vars` index variables: + /// [`schedule`](Self::schedule) with round 0 widened by them. + /// + /// ★ (k0, m, tail), NEVER `schedule(n + m)` (REV-B2-JUDGE J1). The blocks + /// round 0 opens are still `2^k0` wide — the index is folded by combining + /// the trees' blocks, not by a wider block — so the folds, the trees, their + /// caps, the round count and with it Q are the per-polynomial chain's. + /// `schedule(n + m)` would fold index variables in round 1, another + /// protocol; `First(k0 + m)` would open blocks no tree has. + pub fn index_schedule(&self, num_vars: usize, index_vars: usize) -> Vec { + let mut messages = self.schedule(num_vars); + if let Some(first) = messages.first_mut() { + *first += index_vars; + } + messages + } + + /// The Merkle cap height of each of the chain's `R` commitment trees, tree + /// `t` being the one round `t` opens as its current codeword (W1). + /// + /// Tree `t` has depth `D_t − k_t` (its leaves are round `t`'s domain + /// folded by that round's `k`) and is opened `Q` times when `t = 0` (round + /// 0's current openings) and `2Q` times after (round `t − 1`'s successor + /// openings and round `t`'s current ones, the last tree included). The + /// height is [`CapPolicy::height`] of those two public numbers, so the + /// prover, the host verifier and the in-guest emitter derive the same + /// heights from the config alone. All zero at the default. + pub fn tree_caps(&self, num_vars: usize) -> Vec { + self.tree_caps_at(num_vars, 2) + } + + /// [`tree_caps`](Self::tree_caps) for trees of `arity` children per node: + /// the policy at that arity ([`ChainFormat::cap_policy_at`]) over that + /// arity's levels ([`crate::whir_commit::tree_cap_height`]). At arity 2 it + /// is `tree_caps`; the chain's prover and verifier call it with their + /// hash's arity. + pub fn tree_caps_at(&self, num_vars: usize, arity: usize) -> Vec { + let policy = self.format.cap_policy_at(arity); + let mut domain_log = num_vars + self.log_blowup; + self.schedule(num_vars) + .iter() + .enumerate() + .map(|(t, &k)| { + domain_log -= k; + let openings = if t == 0 { + self.num_queries + } else { + 2 * self.num_queries + }; + crate::whir_commit::tree_cap_height(policy, openings, domain_log, arity) + }) + .collect() + } + + /// Variables folded in each round: `log_folding` until the remainder. + /// + /// Under [`WhirFolds::First`] the first round takes `k0` (or everything, + /// when there is less), and the rest is this same walk. + pub fn schedule(&self, num_vars: usize) -> Vec { + let step = self.log_folding.max(1); + let mut left = num_vars; + let mut out = Vec::new(); + if let WhirFolds::First(k0) = self.format.folds { + let take = k0.get().min(left); + if take > 0 { + out.push(take); + left -= take; + } + } + while left > 0 { + let take = step.min(left); + out.push(take); + left -= take; + } + out + } +} + +/// A round's proof-of-work nonces. +/// +/// Zero where the config asks for no bits, and the out-of-domain one is unused +/// on the last round. Under [`NonceLayout::Spent`] those unspent fields are not +/// part of the format and must stay zero; see there. +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct RoundNonces { + pub folding: u64, + pub ood: u64, + pub query: u64, +} + +/// The blocks one round opens. +/// +/// Only the first round's current codeword is base-field; every later one has +/// already been folded with an extension challenge. The successor is always in +/// the extension. +/// +/// ⚠ APPEND ONLY. A variant's serde index is its position, and rkyv lays the +/// enum out at its widest variant: [`BaseGrouped`](Self::BaseGrouped) is last +/// and, like the others, two `Vec`s, so neither encoding of a proof that does +/// not use it moves (REV-B2-JUDGE B1-6). +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub enum RoundOpenings { + Base(RoundProof), + Extension(RoundProof), + /// Round 0 of a [`GroupFold::Indexed`] chain over `P' ≥ 2` polynomials, + /// and nowhere else: per query one base block per round-0 tree, then the + /// successor block ([`GroupedRoundProof`]). + BaseGrouped(GroupedRoundProof), +} + +/// One round of the chain. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct ChainRound { + /// The sumcheck rounds this group consumed. + pub sumcheck: Vec>, + /// The successor's root. `None` on the last round, where the folded message + /// is a constant and is sent instead. + pub next_root: Option, + /// The successor's value at the out-of-domain point. `None` on the last + /// round: there is no successor left to be near a codeword. + pub ood_value: Option>, + pub nonces: RoundNonces, + pub openings: RoundOpenings, +} + +/// A chained proof that `Σ_x w(x)·f(x) = y`. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct ChainProof { + pub rounds: Vec>, + /// The constant the codeword folds to — the prover's claim for `f(α)`. + pub final_value: FieldElement, +} + +impl ChainProof { + /// Codeword elements the openings carry. What chaining is for. + pub fn opened_elements(&self) -> usize { + self.rounds + .iter() + .map(|r| match &r.openings { + RoundOpenings::Base(p) => block_size(&p.current) + block_size::(&p.next), + RoundOpenings::Extension(p) => { + block_size::(&p.current) + block_size::(&p.next) + } + RoundOpenings::BaseGrouped(p) => { + p.current.iter().map(|q| block_size(q)).sum::() + + block_size::(&p.next) + } + }) + .sum() + } +} + +fn block_size(openings: &[crate::whir_commit::CosetOpening]) -> usize { + openings.iter().map(|c| c.values.len()).sum() +} + +/// Commits `f`, blocked for the first round's fold. +/// +/// The codeword stays in the base field: nothing extension-valued has touched +/// the trace yet, and this is the biggest allocation in the proof. +/// Commits one polynomial's codeword. +/// +/// `transient` says whether this commitment has to promise the device room its +/// own commit and opening take — true when it stands alone, false when a +/// caller committing several has promised that turn once for all of them. +/// A polynomial to commit and open, which may not exist yet. +/// +/// A stacked polynomial is its columns written at their offsets and zeros in +/// between. Assembling it here is a pass over every byte the device is about to +/// read anyway, so the parts are handed over as they are and whoever needs a +/// whole one builds it — which on the device path is nobody. +pub struct Stacked<'a, F: IsField> { + /// `(column, offset in elements)`; a column's values are read on the host + /// only by a host path ([`HostColumn`]). + pub parts: Vec<(&'a dyn HostColumn, usize)>, + /// The same parts as `(index into the epoch's columns, offset)`, when the + /// card already holds them — then the scatter is a copy at device + /// bandwidth instead of the trace crossing the bus again. + pub resident: Option<(&'a crate::gpu::ResidentColumns, Vec<(usize, usize)>)>, + pub num_vars: usize, +} + +impl Stacked<'_, F> { + pub fn num_vars(&self) -> usize { + self.num_vars + } + + /// The polynomial itself, assembled here. Only the host paths ask — a + /// device writes the parts where they go and never sees a whole one. + pub fn assemble(&self) -> Result, Error> { + let mut buffer = vec![FieldElement::::zero(); 1usize << self.num_vars]; + for (column, offset) in &self.parts { + let column = column.host(); + buffer[*offset..*offset + column.len()].clone_from_slice(column.evals()); + } + Mle::new(buffer) + } +} + +/// A whole polynomial is a stacked one of a single part at offset zero. +pub fn commit( + f: &Mle, + config: &ChainConfig, + transient: bool, +) -> Result<(CodewordCommitment, Domain), Error> +where + F: IsFFTField + IsPrimeField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + commit_stacked( + &Stacked { + parts: vec![(f as &dyn HostColumn, 0)], + resident: None, + num_vars: f.num_vars(), + }, + config, + transient, + ) +} + +pub fn commit_stacked( + f: &Stacked<'_, F>, + config: &ChainConfig, + transient: bool, +) -> Result<(CodewordCommitment, Domain), Error> +where + F: IsFFTField + IsPrimeField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + check_leaf_order::(&config.format)?; + let num_vars = f.num_vars(); + let schedule = config.schedule(num_vars); + let first = schedule.first().copied().unwrap_or(0); + let domain = Domain::::new(num_vars + config.log_blowup)?; + // On a device the codeword stays there: the chain folds it and opens a + // handful of its values, and it is the biggest array the proof holds. + let leaves = config.format.leaves; + let attempt = match &f.resident { + Some((store, parts)) => crate::gpu::commit_resident( + store, + parts, + num_vars, + config.log_blowup, + first, + transient, + H::DEVICE, + leaves, + ), + None => crate::gpu::commit_parts( + &f.parts, + num_vars, + config.log_blowup, + first, + transient, + H::DEVICE, + leaves, + ), + }; + let commitment = match attempt { + Some((codeword, nodes)) => { + CodewordCommitment::from_device_in(codeword, nodes, first, leaves)? + } + // ⚠ COUNTED, because this arm is otherwise silent. The device declining + // is not a slower path to the same place: the codeword is assembled, + // lifted and encoded here, and the commitment then holds that codeword + // AND its node array on the host until the proof ends. An epoch that + // fills the card partway through lands here for every commit after, + // and the only visible symptoms are host memory and a utilisation + // figure — neither of which names the cause. + None => { + crate::gpu::note_host_fallback(); + let codeword = encode::(&lift_coefficients(&f.assemble()?), &domain)?; + match config.format.leaves { + LeafOrder::Natural => CodewordCommitment::from_codeword(codeword, first)?, + leaves => CodewordCommitment::from_codeword_on_host_in(codeword, first, leaves)?, + } + } + }; + Ok((commitment, domain)) +} + +/// The codeword [`commit_stacked`] commits, with no tree: on the card when the +/// card would have committed it, on the host otherwise. What a retired +/// commitment is revived over (`stacked_eval::RetiredStack::revive`); the +/// values are the committed ones because the transform is the same one. +pub fn encode_stacked( + f: &Stacked<'_, F>, + config: &ChainConfig, + transient: bool, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, +{ + let num_vars = f.num_vars(); + let attempt = match &f.resident { + Some((store, parts)) => { + crate::gpu::encode_resident(store, parts, num_vars, config.log_blowup, transient) + } + None => crate::gpu::encode_parts(&f.parts, num_vars, config.log_blowup, transient), + }; + match attempt { + Some(codeword) => { + // The transform is queued on the codeword's stream; the opening + // needs it done, and a stopwatch around this call should see it. + if !codeword.synchronize() { + return Err(Error::DeviceFailed { + stage: "re-encoding a retired codeword", + }); + } + Ok(Codeword::Device(codeword)) + } + None => { + crate::gpu::note_host_fallback(); + let domain = Domain::::new(num_vars + config.log_blowup)?; + Ok(Codeword::Host(encode::( + &lift_coefficients(&f.assemble()?), + &domain, + )?)) + } + } +} + +/// The product of a chain's two tables, which is what its rounds evaluate — +/// and what sizes a device session's slot file. +pub(crate) fn opening_program() -> Result, Error> { + let mut builder = crate::program::Builder::::new(); + let weight = builder.var(0); + let message = builder.var(1); + let root = builder.mul(weight, message); + builder.finish(root) +} + +/// The two tables a chain's rounds fold: the weight it carries and the message +/// it is opening. +/// +/// A group of rounds folds them, the group after that reads what is left, and +/// they are each the width of a stacked polynomial — so on a device they stay +/// there and only the round's few field elements cross the bus. The flow +/// around them is the same either way: this is the only thing that differs. +enum Factors, E: IsField> { + Host { + weight: Mle, + message: Mle, + field: core::marker::PhantomData, + }, + Device(crate::gpu::OpeningFactors), + /// Round 0 of a grouped chain, before it runs: each polynomial's + /// `(weight, message)` tables, on the host. Its first [`rounds`] + /// (Self::rounds) call runs them in lockstep, binds the index, and leaves + /// one [`Host`](Self::Host) pair behind ([`lockstep_rounds`]). + Lockstep { + parts: Vec<(Mle, Mle)>, + index_vars: usize, + }, + /// The same on a card: each polynomial's opening there. Its first + /// [`rounds`](Self::rounds) call runs them in lockstep, binds the index, + /// and leaves one [`Device`](Self::Device) opening behind + /// ([`crate::gpu::GroupedFactors::rounds`]) — the host's steps, so the + /// host's messages and tables. + DeviceLockstep { + parts: crate::gpu::GroupedFactors, + index_vars: usize, + num_vars: usize, + }, +} + +impl Factors +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, +{ + fn program() -> Result, Error> { + opening_program::() + } + + fn new(f: &Mle, weight: Mle) -> Result { + if let Some(device) = crate::gpu::open_on_device(f, &weight, &Self::program()?) { + return Ok(Self::Device(device)); + } + // The sumcheck's factors have to share a field, so the message is + // lifted here. The codeword is not, which is where the size is. + let message = Mle::new( + f.evals() + .iter() + .map(|v| v.clone().to_extension::()) + .collect(), + )?; + Ok(Self::Host { + weight, + message, + field: core::marker::PhantomData, + }) + } + + /// The same from a weight given as shares: on a device they are written + /// straight into its buffer, and the host builds the table only if none + /// takes them. + /// + /// `on_device` says whether the codeword being opened is on one. When it + /// is, the host building the tables is a fallback and is counted + /// ([`crate::gpu::open_host_fallbacks`]); when it is not, it is the path. + /// + /// `overlay`, a commit-time out-of-domain claim (`crate::commit_ood`), is + /// added to the weight: by the host as a table, by a card from its rounds + /// ([`crate::commit_ood::PendingOverlay`]). + fn from_shares( + f: &Stacked<'_, F>, + shares: &[crate::stacked_eval::WeightShare<'_, E>], + overlay: Option<&Overlay<'_, E>>, + n_stack: usize, + on_device: bool, + ) -> Result { + if let Some(device) = + crate::gpu::open_shared(f, shares, overlay, n_stack, &Self::program()?) + { + return Ok(Self::Device(device)); + } + if on_device { + crate::gpu::note_open_host_fallback(n_stack); + } + // Only here does a stacked polynomial have to exist on the host. + Self::new(&f.assemble()?, host_weight(shares, overlay, n_stack)?) + } + + fn num_vars(&self) -> usize { + match self { + Self::Host { message, .. } => message.num_vars(), + Self::Device(device) => device.num_vars(), + Self::Lockstep { parts, .. } => parts.first().map_or(0, |(_, f)| f.num_vars()), + Self::DeviceLockstep { num_vars, .. } => *num_vars, + } + } + + /// One group of `rounds` rounds, leaving the factors folded on them. + fn rounds>( + &mut self, + rounds: usize, + transcript: &mut T, + ) -> Result, Error> { + match self { + Self::Host { + weight, message, .. + } => { + let polys = vec![ + core::mem::replace(weight, Mle::new(vec![FieldElement::::zero()])?), + core::mem::replace(message, Mle::new(vec![FieldElement::::zero()])?), + ]; + let mut poly = Composed::new(polys, product::, 2)?; + let out = sumcheck::prove_rounds(&mut poly, rounds, transcript)?; + let mut parts = poly.into_polys(); + *message = parts.pop().expect("two factors"); + *weight = parts.pop().expect("two factors"); + Ok(out) + } + Self::Device(device) => { + let (proofs, challenges, _) = device.rounds(rounds, 2, |evaluations| { + for value in evaluations { + transcript.append_field_element(value); + } + transcript.sample_field_element() + })?; + Ok((proofs, challenges)) + } + Self::Lockstep { parts, index_vars } => { + let index_vars = *index_vars; + let (proofs, challenges, weight, message) = + lockstep_rounds(core::mem::take(parts), rounds, index_vars, transcript)?; + *self = Self::Host { + weight, + message, + field: core::marker::PhantomData, + }; + Ok((proofs, challenges)) + } + Self::DeviceLockstep { .. } => { + let Self::DeviceLockstep { + parts, index_vars, .. + } = core::mem::replace( + self, + Self::Lockstep { + parts: Vec::new(), + index_vars: 0, + }, + ) + else { + return Err(Error::NoVariablesLeft); + }; + let (proofs, challenges, combined) = + parts.rounds(rounds, index_vars, |evaluations| { + for value in evaluations { + transcript.append_field_element(value); + } + transcript.sample_field_element() + })?; + *self = Self::Device(combined); + Ok((proofs, challenges)) + } + } + } + + fn evaluate_message(&mut self, point: &[FieldElement]) -> Result, Error> { + match self { + Self::Host { message, .. } => message.evaluate(point), + Self::Device(device) => device.evaluate_message(point), + // Round 0's rounds turn lockstep factors into one pair before any + // out-of-domain point exists. + Self::Lockstep { .. } | Self::DeviceLockstep { .. } => Err(Error::NoVariablesLeft), + } + } + + /// Whether these factors live on the card (the only ones that encode). + fn on_card(&self) -> bool { + matches!(self, Self::Device(_)) + } + + /// The message these rounds bound, when the host holds it. + fn host_message(&self) -> Option<&Mle> { + match self { + Self::Host { message, .. } => Some(message), + _ => None, + } + } + + /// The successor a `levels`-level fold of `codeword` would make, encoded + /// from the message these rounds bound — on a device, when the factors and + /// the codeword are both there, unless `LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0`. + /// `None` is the fold's turn. + fn encode_successor( + &mut self, + codeword: &Codeword, + levels: usize, + ) -> Option> { + match (self, codeword) { + (Self::Device(device), Codeword::Device(held)) => { + device.encode_successor(held, levels).map(Codeword::Device) + } + (Self::Device(device), Codeword::Source(held)) => device + .encode_successor_from(held.source(), levels) + .map(Codeword::Device), + _ => None, + } + } + + /// `weight += gamma·eq(point, ·)`, the weight the next group carries. + fn add_scaled_eq( + &mut self, + point: &[FieldElement], + gamma: &FieldElement, + ) -> Result<(), Error> { + match self { + Self::Host { weight, .. } => { + *weight = batch_weight(weight, &eq_mle(point)?, gamma)?; + Ok(()) + } + Self::Device(device) => device.add_scaled_eq(point, gamma), + Self::Lockstep { .. } | Self::DeviceLockstep { .. } => Err(Error::NoVariablesLeft), + } + } +} + +/// Round 0 of a grouped chain's sumcheck, on the host (D-ONECHAIN §2.1, §2.5). +/// +/// - `rounds` data rounds over the run's `P'` pairs IN LOCKSTEP: each round's +/// message is the sum of theirs and one challenge binds them all, since +/// `Σ_y W·F` over the slots is `Σ_p w_p·f_p`. +/// - Then the `m = index_vars` index rounds over the bound tables, laid out +/// slot by slot (`y` the high variables, absent slots zero): `ρ`. +/// +/// Returns the `rounds + m` messages and challenges, and the pair the chain +/// carries on: `Σ_p eq(ρ, p)·(w_p, f_p)` at the data challenges, built with +/// [`eq_index`], the convention the verifier's combine and weight read. The +/// index rounds' own folded tables are that same pair when the conventions +/// agree, and are dropped; a disagreement would show as an honest proof that +/// does not verify. +#[allow(clippy::type_complexity)] +fn lockstep_rounds( + parts: Vec<(Mle, Mle)>, + rounds: usize, + index_vars: usize, + transcript: &mut T, +) -> Result< + ( + Vec>, + Vec>, + Mle, + Mle, + ), + Error, +> +where + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, + T: IsTranscript, +{ + use crate::poly::SumcheckPolynomial; + let mut polys = parts + .into_iter() + .map(|(weight, message)| Composed::new(vec![weight, message], product::, 2)) + .collect::, Error>>()?; + let mut proofs = Vec::with_capacity(rounds + index_vars); + let mut challenges = Vec::with_capacity(rounds + index_vars); + for _ in 0..rounds { + let mut sent = vec![FieldElement::::zero(); 2]; + for poly in &polys { + for (total, value) in sent + .iter_mut() + .zip(sumcheck::round_evaluations(poly, 2, false)) + { + *total += value; + } + } + for value in &sent { + transcript.append_field_element(value); + } + let r: FieldElement = transcript.sample_field_element(); + for poly in &mut polys { + poly.fix_first_variable(&r)?; + } + proofs.push(SumcheckRoundProof { evaluations: sent }); + challenges.push(r); + } + + let tables: Vec<(Mle, Mle)> = polys + .into_iter() + .map(|poly| { + let mut factors = poly.into_polys(); + match (factors.pop(), factors.pop()) { + (Some(message), Some(weight)) => Ok((weight, message)), + _ => Err(Error::EmptyPolynomial), + } + }) + .collect::>()?; + let width = tables.first().map_or(1, |(weight, _)| weight.len()); + let slots = 1usize << index_vars; + let mut weight = Vec::with_capacity(width * slots); + let mut message = Vec::with_capacity(width * slots); + for (w, f) in &tables { + weight.extend_from_slice(w.evals()); + message.extend_from_slice(f.evals()); + } + weight.resize(width * slots, FieldElement::::zero()); + message.resize(width * slots, FieldElement::::zero()); + let mut index = Composed::new(vec![Mle::new(weight)?, Mle::new(message)?], product::, 2)?; + let (index_proofs, rho) = sumcheck::prove_rounds(&mut index, index_vars, transcript)?; + drop(index); + + let eqs: Vec> = (0..tables.len()).map(|p| eq_index(&rho, p)).collect(); + let combine = |pick: fn(&(Mle, Mle)) -> &Mle| -> Result, Error> { + let mut out = vec![FieldElement::::zero(); width]; + for (table, eq) in tables.iter().zip(&eqs) { + for (acc, value) in out.iter_mut().zip(pick(table).evals()) { + *acc += eq * value; + } + } + Mle::new(out) + }; + let weight = combine(|t| &t.0)?; + let message = combine(|t| &t.1)?; + proofs.extend(index_proofs); + challenges.extend(rho); + Ok((proofs, challenges, weight, message)) +} + +/// What a round folds: the codeword and the commitment that answers for it. +/// +/// Only the first round's is base-field. Folding it with an extension challenge +/// is what lifts it, so every later round is `Extension`. +enum Current<'a, F: IsField + 'static, E: IsField + 'static, H: WhirHash> +where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + Base(&'a CodewordCommitment), + /// Round 0 of a grouped chain: the run's round-0 commitments, in slot + /// order. + Grouped(&'a [&'a CodewordCommitment]), + Extension(CodewordCommitment), +} + +/// Proves `f(z) = y`. +pub fn prove( + f: &Mle, + z: &[FieldElement], + commitment: &CodewordCommitment, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + prove_weighted::(f, eq_mle(z)?, commitment, domain, config, transcript) +} + +/// The same for a weight given as the shares of a stacked polynomial's +/// columns, which a device writes into its own buffer and the host +/// materializes only if none does. +#[allow(clippy::too_many_arguments)] +pub fn prove_shared( + f: &Stacked<'_, F>, + shares: &[crate::stacked_eval::WeightShare<'_, E>], + n_stack: usize, + commitment: &CodewordCommitment, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + prove_shared_ood::( + f, shares, None, n_stack, commitment, domain, config, transcript, + ) +} + +/// [`prove_shared`] with a commit-time out-of-domain claim on the weight +/// (`crate::commit_ood`): the chain proves its claim plus +/// `overlay.scale·F(overlay.point)` under the weight plus +/// `overlay.scale·eq(overlay.point, ·)`. `None` is [`prove_shared`]. +#[allow(clippy::too_many_arguments)] +pub fn prove_shared_ood( + f: &Stacked<'_, F>, + shares: &[crate::stacked_eval::WeightShare<'_, E>], + overlay: Option<&Overlay<'_, E>>, + n_stack: usize, + commitment: &CodewordCommitment, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + let factors = Factors::::from_shares( + f, + shares, + overlay, + n_stack, + commitment.codeword().on_device(), + )?; + prove_with_factors::( + f.num_vars(), + factors, + Current::Base(commitment), + domain, + config, + transcript, + ) +} + +/// One chain over a run of `P'` stacked polynomials ([`GroupFold::Indexed`], +/// D-ONECHAIN §2.1): `polys[p]` committed as `commitments[p]`, its weight +/// given as `shares[p]`. The claim is the sum of the polynomials' claims. +/// +/// Round 0 runs its data rounds over the `P'` pairs in lockstep and then the +/// `m` index rounds ([`lockstep_rounds`]), folds each round-0 codeword and +/// combines the folds with `eq(ρ, p)` into the one successor, and opens the +/// `P'` trees at one position set ([`whir_round::prove_grouped`]). Every later +/// round is today's. +/// +/// `2 ≤ P' ≤ MAX_INDEX_POLYS`; a run of one is [`prove_shared`]. Round 0 runs +/// on a card when it takes every polynomial's opening +/// ([`crate::gpu::open_grouped`]), on the host otherwise — counted as a +/// fallback when the codewords are on a card +/// ([`crate::gpu::note_grouped_host_fallback`]). The successor is encoded from +/// the combined bound message when the factors and codewords are on a card, +/// as a per-polynomial chain's is, and folded and combined otherwise +/// ([`fold_grouped`]). +#[allow(clippy::too_many_arguments)] +pub fn prove_grouped_shared( + polys: &[Stacked<'_, F>], + shares: &[Vec>], + n_stack: usize, + commitments: &[&CodewordCommitment], + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + prove_grouped_shared_ood::( + polys, + shares, + None, + n_stack, + commitments, + domain, + config, + transcript, + ) +} + +/// [`prove_grouped_shared`] with a commit-time out-of-domain claim on each +/// polynomial's weight, `overlays[p]` on slot `p` (`crate::commit_ood`): the +/// run's claim gains `Σ_p overlays[p].scale·F_p(point)`. `None` is +/// [`prove_grouped_shared`]. +#[allow(clippy::too_many_arguments)] +pub fn prove_grouped_shared_ood( + polys: &[Stacked<'_, F>], + shares: &[Vec>], + overlays: Option<&[Overlay<'_, E>]>, + n_stack: usize, + commitments: &[&CodewordCommitment], + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + let width = commitments.len(); + if !(2..=MAX_INDEX_POLYS).contains(&width) + || polys.len() != width + || shares.len() != width + || overlays.is_some_and(|overlays| overlays.len() != width) + { + return Err(Error::GroupedShape(GroupedShape::RunWidth { polys: width })); + } + check_index_domain( + n_stack, + index_vars(width), + config.log_blowup, + F::TWO_ADICITY as usize, + )?; + let overlay = |p: usize| overlays.map(|overlays| &overlays[p]); + let factors = match crate::gpu::open_grouped( + polys, + shares, + overlays, + n_stack, + &opening_program::()?, + ) { + Some(parts) => Factors::DeviceLockstep { + parts, + index_vars: index_vars(width), + num_vars: n_stack, + }, + None => { + // Counted over codewords the card holds or a stack it serves from: + // either way the card was meant to build these factors. + if commitments.iter().any(|c| c.codeword().on_device()) { + crate::gpu::note_grouped_host_fallback(width, n_stack); + } + // Only here do the run's stacked polynomials and weights exist on + // the host. + let parts = polys + .iter() + .zip(shares) + .enumerate() + .map(|(p, (f, shares))| -> Result<(Mle, Mle), Error> { + let message = Mle::new( + f.assemble()? + .evals() + .iter() + .map(|v| v.clone().to_extension::()) + .collect(), + )?; + Ok((host_weight(shares, overlay(p), n_stack)?, message)) + }) + .collect::, Error>>()?; + Factors::Lockstep { + parts, + index_vars: index_vars(width), + } + } + }; + prove_with_factors::( + n_stack, + factors, + Current::Grouped(commitments), + domain, + config, + transcript, + ) +} + +/// Proves `Σ_x w(x)·f(x) = y` for a weight the verifier can evaluate itself. +pub fn prove_weighted( + f: &Mle, + weight: Mle, + commitment: &CodewordCommitment, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + let factors = Factors::::new(f, weight)?; + prove_with_factors::( + f.num_vars(), + factors, + Current::Base(commitment), + domain, + config, + transcript, + ) +} + +/// The chain itself, over factors that are wherever they are. `current` is +/// round 0's commitment, or a grouped chain's commitments +/// ([`Current::Grouped`]); `num_vars` is a polynomial's, without the index. +fn prove_with_factors( + num_vars: usize, + mut factors: Factors, + mut current: Current<'_, F, E, H>, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + check_leaf_order::(&config.format)?; + let round0: &[&CodewordCommitment] = match ¤t { + Current::Base(held) => std::slice::from_ref(held), + Current::Grouped(held) => held, + Current::Extension(_) => &[], + }; + if let Some(held) = round0 + .iter() + .find(|held| held.leaves() != config.format.leaves) + { + return Err(Error::LeafOrderMismatch { + expected: config.format.leaves, + got: held.leaves(), + }); + } + // One chain, counted — arm F's identity is per chain, and a group can open + // several (`stacked_eval::prove` runs one per commitment). + crate::whir_split::bump(&crate::whir_split::CHAIN_COUNT); + let schedule = config.schedule(num_vars); + // One cap height per tree, in the hash's tree levels; all zero at the + // default format. + let caps = config.tree_caps_at(num_vars, crate::whir_hash::arity::()); + // The codeword comes out of the commitment rather than being encoded + // again: it is the same array, and the NTT is not cheap. + let mut current_domain = domain.clone(); + + let mut rounds = Vec::with_capacity(schedule.len()); + let mut final_value = FieldElement::::zero(); + + for (r, &k) in schedule.iter().enumerate() { + // The loop's OWN wall, so the six are measured against the thing that + // contains them rather than against `open_groups` two layers up. What + // sits between this and `open_groups` — `Factors::from_shares`, the + // weights, the stacked polys, the domain clone, the proof assembled + // after the last round — is reported as `setup_tail`, named rather + // than left as a gap for a tolerance to swallow. + let __wc_round = crate::whir_split::mark(); + // One per iteration, so arm F can derive the `open_many` calls this + // loop should have made (`2R − 1` per chain) from the run's own count. + crate::whir_split::bump(&crate::whir_split::ROUND_COUNT); + // ── the round's six slots, under `LAMBDA_VM_BASE_SPLIT=1` ── + // They partition the round, so `open_groups - Σ(six)` is loop overhead + // and nothing else. All three grinds share one slot: they are the same + // 20-bit search and a lever on one is a lever on all three. + let __wc_g = crate::whir_split::mark(); + let mut nonces = RoundNonces { + folding: grind::(transcript, config.grind.folding)?, + ..RoundNonces::default() + }; + crate::whir_split::add(&crate::whir_split::GRIND, __wc_g); + + let __wc_sc = crate::whir_split::mark(); + let (sumcheck_rounds, challenges) = factors.rounds(k, transcript)?; + crate::whir_split::add(&crate::whir_split::SUMCHECK, __wc_sc); + // A grouped round 0 also bound its index: the challenges past the + // first `k` are `ρ`, and only the first `k` fold a block. + let (alphas, rho) = challenges.split_at(k.min(challenges.len())); + + // The fold lands in the extension whichever field it started in, so + // the field is the only thing the two cases differ in — and a codeword + // the device holds is folded where it is. The first one, the widest, + // is encoded from the message the rounds just bound instead: the same + // codeword (`LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0` folds it). + let __wc_fold = crate::whir_split::mark(); + let (folded, folded_domain) = match ¤t { + Current::Base(held) => match factors.encode_successor(held.codeword(), k) { + Some(encoded) => { + let mut folded_domain = current_domain.clone(); + for _ in 0..k { + folded_domain = folded_domain.squared()?; + } + (encoded, folded_domain) + } + // ★ Under the coset order a round-0 codeword on the card, or + // held as its stack, is never folded: its successor is encoded + // from the bound message — on the host when the host built the + // factors — or the prove is refused, by cause. A codeword the + // host holds (a prover with no card) folds. + None if config.format.leaves == LeafOrder::Coset && held.codeword().on_device() => { + match factors.host_message() { + Some(message) if crate::gpu::encode_successor_in_effect() => { + encode_successor_on_host::(message, ¤t_domain, k)? + } + _ => { + return Err(Error::CosetSuccessorNotEncoded { + cause: UnencodedSuccessor::of( + crate::gpu::encode_successor_in_effect(), + factors.on_card(), + ), + }); + } + } + } + None => fold_held::(held.codeword(), ¤t_domain, alphas)?, + }, + Current::Grouped(held) => match held + .first() + .and_then(|first| factors.encode_successor(first.codeword(), k)) + { + // The run's combined bound message encoded: the successor + // `Σ_p eq(ρ, p)·Fold_α(C_p)` is, the fold and the encoding + // being linear and equal polynomial by polynomial. + Some(encoded) => { + let mut folded_domain = current_domain.clone(); + for _ in 0..k { + folded_domain = folded_domain.squared()?; + } + (encoded, folded_domain) + } + // ★ The same rule as a polynomial's own round 0: under the + // coset order a run whose round-0 codewords are on the card is + // never folded — its successor is encoded from the combined + // bound message, on the host when the host built the run's + // factors, or the prove is refused by cause (the knob off; the + // card declining the encode). A run whose codewords the host + // holds folds. + None if config.format.leaves == LeafOrder::Coset + && held.iter().any(|c| c.codeword().on_device()) => + { + match factors.host_message() { + Some(message) if crate::gpu::encode_successor_in_effect() => { + encode_successor_on_host::(message, ¤t_domain, k)? + } + _ => { + return Err(Error::CosetSuccessorNotEncoded { + cause: UnencodedSuccessor::of( + crate::gpu::encode_successor_in_effect(), + factors.on_card(), + ), + }); + } + } + } + None => fold_grouped::(held, ¤t_domain, alphas, rho)?, + }, + Current::Extension(held) => { + fold_held::(held.codeword(), ¤t_domain, alphas)? + } + }; + crate::whir_split::add(&crate::whir_split::FOLD, __wc_fold); + + // The successor is committed before the queries are drawn, so it cannot + // be chosen to match them. + let __wc_cf = crate::whir_split::mark(); + let next = match schedule.get(r + 1) { + Some(&next_k) => { + let next = commit_folded::(folded, next_k)?; + transcript.append_bytes(&next.root()); + Some(next) + } + None => { + final_value = first_value::(&folded)?; + transcript.append_field_element(&final_value); + None + } + }; + crate::whir_split::add(&crate::whir_split::COMMIT_FOLDED, __wc_cf); + let next_root = next.as_ref().map(|c| c.root()); + + // Out of domain: a value the queries cannot vouch for, so answering it + // pins the successor to one codeword. + // ⛔ TWO OOD WINDOWS, NOT ONE SPANNING WINDOW. The out-of-domain block + // contains a grind, and the grind belongs to GRIND — so a single + // window opened before it and closed after would count that time + // TWICE, once in each slot, and the six would sum to more than the + // `open_groups` wall that contains them. Arm E read exactly that on the + // first two real runs: a NEGATIVE remainder of -3.6s and then -4.0s. + // Slots that partition must not nest; these two abut the grind instead. + let ood_value = if next.is_some() { + let __wc_ood_a = crate::whir_split::mark(); + let z0: FieldElement = transcript.sample_field_element(); + require_out_of_domain::(&z0, &folded_domain)?; + let point = ood_point(&z0, factors.num_vars()); + let y0 = factors.evaluate_message(&point)?; + transcript.append_field_element(&y0); + crate::whir_split::add(&crate::whir_split::OOD, __wc_ood_a); + + let __wc_g2 = crate::whir_split::mark(); + nonces.ood = grind::(transcript, config.grind.ood)?; + crate::whir_split::add(&crate::whir_split::GRIND, __wc_g2); + + let __wc_ood_b = crate::whir_split::mark(); + let gamma: FieldElement = transcript.sample_field_element(); + factors.add_scaled_eq(&point, &gamma)?; + crate::whir_split::add(&crate::whir_split::OOD, __wc_ood_b); + Some(y0) + } else { + None + }; + + let __wc_g3 = crate::whir_split::mark(); + nonces.query = grind::(transcript, config.grind.query)?; + crate::whir_split::add(&crate::whir_split::GRIND, __wc_g3); + let round_config = RoundConfig { + num_queries: config.num_queries, + log_folding: k, + }; + let __wc_q = crate::whir_split::mark(); + // Tree `r` is opened under `caps[r]`, and carries its cap on its first + // opening in proof order: round 0's first current opening for tree 0, + // round `r − 1`'s first successor opening for every later tree. + let round_caps = RoundCaps { + current: caps[r], + current_owner: r == 0, + next: caps.get(r + 1).copied().unwrap_or(0), + }; + let openings = + match (¤t, &next) { + (Current::Base(held), Some(next)) => RoundOpenings::Base(whir_round::prove( + *held, + next, + &round_config, + round_caps, + transcript, + )?), + (Current::Base(held), None) => RoundOpenings::Base(final_openings::( + held, + &round_config, + round_caps, + transcript, + )?), + (Current::Extension(held), Some(next)) => RoundOpenings::Extension( + whir_round::prove(held, next, &round_config, round_caps, transcript)?, + ), + (Current::Extension(held), None) => { + RoundOpenings::Extension(final_openings::( + held, + &round_config, + round_caps, + transcript, + )?) + } + (Current::Grouped(held), next) => { + RoundOpenings::BaseGrouped(whir_round::prove_grouped( + held, + next.as_ref(), + &round_config, + round_caps, + transcript, + )?) + } + }; + crate::whir_split::add(&crate::whir_split::QUERIES, __wc_q); + + rounds.push(ChainRound { + sumcheck: sumcheck_rounds, + next_root, + ood_value, + nonces, + openings, + }); + crate::whir_split::add(&crate::whir_split::ROUND, __wc_round); + if let Some(next) = next { + current = Current::Extension(next); + } + current_domain = folded_domain; + } + + Ok(ChainProof { + rounds, + final_value, + }) +} + +/// Folds a codeword wherever it is, leaving the result where it was. +fn fold_held( + codeword: &Codeword, + domain: &Domain, + alphas: &[FieldElement], +) -> Result<(Codeword, Domain), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf + 'static, + C: IsField + IsSubFieldOf + 'static, + N: IsField + 'static, + FieldElement: AsBytes + Sync + Send, +{ + // A round-0 codeword held as its stack exists only under the coset + // order, whose round 0 refuses a fold before reaching here. + if codeword.source().is_some() { + return Err(Error::DeviceFailed { + stage: "folding a round-0 codeword held as its stack", + }); + } + if let Some(folded) = codeword + .device() + .and_then(|device| device.fold(domain.generator(), alphas)) + { + let mut folded_domain = domain.clone(); + for _ in alphas { + folded_domain = folded_domain.squared()?; + } + return Ok((Codeword::Device(folded), folded_domain)); + } + // A codeword the device holds has no copy here: if the device turned the + // fold down, this round cannot happen anywhere. + let values = codeword.host().ok_or(Error::DeviceFailed { + stage: "codeword fold", + })?; + let (folded, folded_domain) = fold_codeword_k::(values, domain, alphas)?; + Ok((Codeword::Host(folded), folded_domain)) +} + +/// ★ The successor of a coset-ordered round 0 whose factors the host built (the +/// card declined the opening): the message the rounds bound there, lifted and +/// encoded over the `levels`-times folded domain — the fold's values without +/// reading the codeword, so a round 0 held as its stack takes it too. A run's +/// message is its combined one, `Σ_p eq(ρ, p)·f_p` bound, so this is its +/// successor `Σ_p eq(ρ, p)·Fold_α(C_p)`. Counted and said. +fn encode_successor_on_host( + message: &Mle, + domain: &Domain, + levels: usize, +) -> Result<(Codeword, Domain), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, + FieldElement: Send + Sync, +{ + let mut folded_domain = domain.clone(); + for _ in 0..levels { + folded_domain = folded_domain.squared()?; + } + let values = encode::(&lift_coefficients(message), &folded_domain)?; + crate::gpu::note_coset_host_encode(levels, message.num_vars()); + Ok((Codeword::Host(values), folded_domain)) +} + +/// Round 0's successor in a grouped chain: each round-0 codeword folded by the +/// data challenges where it is, the folds combined with `eq(ρ, p)` +/// ([`eq_index`]) — on the card when every fold landed there +/// ([`crate::gpu::combine_codewords`]), on the host otherwise, a card fold +/// brought home first. Block `j` of it is the fold of `Σ_p eq(ρ, p)·block_p(j)`, +/// which is what the verifier checks (D-ONECHAIN §3.3, Lemma B item 5). +fn fold_grouped( + held: &[&CodewordCommitment], + domain: &Domain, + alphas: &[FieldElement], + rho: &[FieldElement], +) -> Result<(Codeword, Domain), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + H: WhirHash, +{ + let eqs: Vec> = (0..held.len()).map(|p| eq_index(rho, p)).collect(); + let mut folds = Vec::with_capacity(held.len()); + let mut folded_domain = domain.clone(); + for commitment in held { + let (folded, at) = fold_held::(commitment.codeword(), domain, alphas)?; + folds.push(folded); + folded_domain = at; + } + if let Some(on_card) = folds + .iter() + .map(Codeword::device) + .collect::>>() + && let Some(combined) = crate::gpu::combine_codewords(&on_card, &eqs) + { + return Ok((Codeword::Device(combined), folded_domain)); + } + let mut combined: Vec> = Vec::new(); + for (folded, eq) in folds.iter().zip(&eqs) { + let brought; + let values: &[FieldElement] = match folded { + Codeword::Host(values) => values, + Codeword::Device(device) => { + brought = device.values::().ok_or(Error::DeviceFailed { + stage: "a grouped round-0 fold brought home", + })?; + &brought + } + // A fold is held somewhere; only a round-0 commitment revived over + // its stack holds none, and a grouped run is never revived so. + Codeword::Source(_) => { + return Err(Error::DeviceFailed { + stage: "a grouped round-0 fold of a codeword not held", + }); + } + }; + if combined.is_empty() { + combined = values.iter().map(|v| eq * v).collect(); + } else { + for (acc, v) in combined.iter_mut().zip(values) { + *acc += eq * v; + } + } + } + Ok((Codeword::Host(combined), folded_domain)) +} + +/// Commits a folded codeword where it is. +fn commit_folded( + codeword: Codeword, + log_folding: usize, +) -> Result, Error> +where + N: IsField + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + match codeword { + Codeword::Host(values) => CodewordCommitment::from_codeword(values, log_folding), + Codeword::Device(device) => { + let nodes = device + .commit(log_folding, H::DEVICE) + .ok_or(Error::DeviceFailed { stage: "fold tree" })?; + CodewordCommitment::from_device(device, nodes, log_folding) + } + // A successor is always held. + Codeword::Source(_) => Err(Error::DeviceFailed { + stage: "committing a successor held as a stack", + }), + } +} + +/// The value the last fold leaves behind. +fn first_value(codeword: &Codeword) -> Result, Error> +where + N: IsField + 'static, +{ + match codeword { + Codeword::Host(values) => values.first().cloned().ok_or(Error::EmptyPolynomial), + Codeword::Device(device) => device.first().ok_or(Error::DeviceFailed { + stage: "final value", + }), + Codeword::Source(_) => Err(Error::DeviceFailed { + stage: "the final value of a codeword held as a stack", + }), + } +} + +/// The last round's openings: only the current codeword's blocks, since what +/// they fold to is the constant the prover sends. +/// +/// Mirrors [`whir_round`]'s query draw, so both sides sample the same +/// positions. +fn final_openings( + current: &CodewordCommitment, + config: &RoundConfig, + caps: RoundCaps, + transcript: &mut T, +) -> Result, Error> +where + C: IsField + 'static, + N: IsField, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + let __wq_sample = crate::whir_split::mark(); + let queries: Vec = (0..config.num_queries) + .map(|_| transcript.sample_u64(current.num_leaves() as u64) as usize) + .collect(); + crate::whir_split::add(&crate::whir_split::QUERY_SAMPLE, __wq_sample); + // ⛔ ONE `open_many` here, not two: the final round has no successor to + // open. That is the `− 1` in arm F's `2R − 1`. + Ok(RoundProof { + current: current.open_many_capped(&queries, caps.current, caps.current_owner)?, + next: Vec::new(), + }) +} + +/// Verifies `f(z) = y`. +pub fn verify( + proof: &ChainProof, + root: &Commitment, + z: &[FieldElement], + y: FieldElement, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + verify_weighted::( + proof, + root, + |alphas: &[FieldElement]| eq_eval(z, alphas), + y, + z.len(), + domain, + config, + transcript, + ) +} + +/// Verifies `Σ_x w(x)·f(x) = y`. +/// +/// `weight_at` is the weight's closed form, evaluated at the concatenation of +/// every round's challenges. +#[allow(clippy::too_many_arguments)] +pub fn verify_weighted<'a, F, E, T, W, H>( + proof: &'a ChainProof, + root: &'a Commitment, + weight_at: W, + y: FieldElement, + num_vars: usize, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + W: FnOnce(&[FieldElement]) -> Result, Error>, + H: WhirHash, +{ + verify_chain::( + proof, + core::slice::from_ref(root), + weight_at, + y, + num_vars, + domain, + config, + transcript, + ) +} + +/// Verifies ONE chain over a run of `roots.len() = P'` stacked polynomials +/// ([`GroupFold::Indexed`], D-ONECHAIN §2.6): `Σ_x Σ_p eq(y, p)·w_p(x)·f_p(x) +/// = y_claim`, round 0 over the `P'` trees at one position set. +/// +/// `num_vars` is a polynomial's, without the index. `weight_at` is the +/// weight's closed form at the chain's challenges in binding order, `n + m` of +/// them: round 0's `k0` data challenges, its `m` index challenges `ρ`, then +/// every later round's. A run of one is [`verify_weighted`], byte for byte. +#[allow(clippy::too_many_arguments)] +pub fn verify_grouped<'a, F, E, T, W, H>( + proof: &'a ChainProof, + roots: &'a [Commitment], + weight_at: W, + y: FieldElement, + num_vars: usize, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + W: FnOnce(&[FieldElement]) -> Result, Error>, + H: WhirHash, +{ + verify_chain::( + proof, roots, weight_at, y, num_vars, domain, config, transcript, + ) +} + +/// Where a round's openings may be which variant (REV-B2-JUDGE B1-2): +/// [`RoundOpenings::BaseGrouped`] exactly at round 0 of a chain over `P' ≥ 2` +/// (final or not), [`RoundOpenings::Base`] exactly at round 0 of a chain over +/// one polynomial, [`RoundOpenings::Extension`] after round 0. A one-polynomial +/// chain's refusals are today's. +fn check_variant( + r: usize, + grouped: bool, + openings: &RoundOpenings, +) -> Result<(), Error> { + match (openings, r == 0, grouped) { + (RoundOpenings::BaseGrouped(_), true, true) => Ok(()), + (RoundOpenings::BaseGrouped(_), _, _) | (RoundOpenings::Base(_), true, true) => { + Err(Error::GroupedShape(GroupedShape::Variant { round: r })) + } + // Only the first round's current codeword is base-field. Said here + // rather than left to a Merkle check failing on the byte width. + (RoundOpenings::Base(_), first, _) | (RoundOpenings::Extension(_), first, _) + if first != matches!(openings, RoundOpenings::Base(_)) => + { + Err(Error::RoundCountMismatch { + expected: 0, + got: r, + }) + } + _ => Ok(()), + } +} + +/// [`verify_weighted`] and [`verify_grouped`]: one chain over `roots.len()` +/// polynomials. At one polynomial every check and every transcript call is +/// [`verify_weighted`]'s as it was. +#[allow(clippy::too_many_arguments)] +fn verify_chain<'a, F, E, T, W, H>( + proof: &'a ChainProof, + roots: &'a [Commitment], + weight_at: W, + y: FieldElement, + num_vars: usize, + domain: &Domain, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + W: FnOnce(&[FieldElement]) -> Result, Error>, + H: WhirHash, +{ + check_leaf_order::(&config.format)?; + // The domain is the caller's: held to the configuration's rate here. + if domain.log_size() != num_vars + config.log_blowup { + return Err(Error::DomainRateMismatch { + expected: num_vars + config.log_blowup, + got: domain.log_size(), + }); + } + // The run's width, and with it the index, are the caller's derivation + // (`stacked_eval::index_chains`) — never the proof's. + let width = roots.len(); + let Some(root) = roots.first().filter(|_| width <= MAX_INDEX_POLYS) else { + return Err(Error::GroupedShape(GroupedShape::RunWidth { polys: width })); + }; + let grouped = width >= 2; + let index = index_vars(width); + if grouped { + check_index_domain(num_vars, index, config.log_blowup, F::TWO_ADICITY as usize)?; + } + // The folds — and so the trees, their caps and the round count — are the + // per-polynomial chain's; round 0 alone carries `m` more messages. + let schedule = config.schedule(num_vars); + let messages = config.index_schedule(num_vars, index); + if proof.rounds.len() != schedule.len() { + return Err(Error::RoundCountMismatch { + expected: schedule.len(), + got: proof.rounds.len(), + }); + } + + let mut claim = y; + let mut alphas: Vec> = Vec::with_capacity(num_vars + index); + // Tree 0 is authenticated by the cap round 0's first current opening + // carries; every later tree by the check the round that committed it + // returned. A verifier constant per tree, derived from the config and the + // hash alone. A grouped round 0 has `P'` trees 0, each today's tree 0. + let caps = config.tree_caps_at(num_vars, crate::whir_hash::arity::()); + // Which nonce slots the proof carries: a verifier constant too. + let layout = config.format.nonces; + // Round-0 trees put their leaves in the chain's order; every later tree is + // a successor's, natural. + let mut current: TreeCheck<'a> = TreeCheck::Owner { + root, + cap_height: caps[0], + leaves: config.format.leaves, + }; + let round0_trees: Vec> = if grouped { + roots + .iter() + .map(|root| TreeCheck::Owner { + root, + cap_height: caps[0], + leaves: config.format.leaves, + }) + .collect() + } else { + Vec::new() + }; + let mut current_domain = domain.clone(); + // Each round's out-of-domain claim, and how many variables were bound when + // it entered the weight — the challenges after that are where its `eq` + // lands. + let mut ood: Vec<(FieldElement, Vec>, usize)> = Vec::new(); + + for (r, (round, (&k, &sent))) in proof + .rounds + .iter() + .zip(schedule.iter().zip(&messages)) + .enumerate() + { + if round.sumcheck.len() != sent { + if grouped && r == 0 { + return Err(Error::GroupedShape(GroupedShape::Round0Messages { + expected: sent, + got: round.sumcheck.len(), + })); + } + return Err(Error::RoundCountMismatch { + expected: k, + got: round.sumcheck.len(), + }); + } + check_variant(r, grouped, &round.openings)?; + check_nonce::( + transcript, + config.grind.folding, + round.nonces.folding, + layout, + )?; + // The weight raises the degree of the plain `f` term to two. + let group = sumcheck::verify_rounds(&round.sumcheck, claim, 2, transcript)?; + claim = group.expected_evaluation; + // Round 0 of a grouped chain bound its index last: `ρ` folds no block. + let (data, rho) = group.point.split_at(k); + + let mut next_domain = current_domain.clone(); + for _ in 0..k { + next_domain = next_domain.squared()?; + } + let bound = alphas.len() + group.point.len(); + + let round_config = RoundConfig { + num_queries: config.num_queries, + log_folding: k, + }; + match ( + &round.next_root, + round.ood_value.as_ref(), + schedule.get(r + 1), + ) { + (Some(next_root), Some(y0), Some(&next_k)) => { + transcript.append_bytes(next_root); + + let z0: FieldElement = transcript.sample_field_element(); + require_out_of_domain::(&z0, &next_domain)?; + let point = ood_point(&z0, num_vars + index - bound); + transcript.append_field_element(y0); + + check_nonce::(transcript, config.grind.ood, round.nonces.ood, layout)?; + let gamma: FieldElement = transcript.sample_field_element(); + claim += &gamma * y0; + ood.push((gamma, point, bound)); + + check_nonce::(transcript, config.grind.query, round.nonces.query, layout)?; + let commitments = RoundCommitments { + current, + next_root, + next_num_leaves: next_domain.size() >> next_k, + next_cap_height: caps[r + 1], + }; + let next_check = match &round.openings { + RoundOpenings::Base(openings) => whir_round::verify::( + openings, + commitments, + ¤t_domain, + data, + &round_config, + transcript, + )?, + RoundOpenings::Extension(openings) => whir_round::verify::( + openings, + commitments, + ¤t_domain, + data, + &round_config, + transcript, + )?, + RoundOpenings::BaseGrouped(openings) => { + whir_round::verify_grouped::( + openings, + &round0_trees, + whir_round::GroupedEnd::Successor { + root: next_root, + num_leaves: next_domain.size() >> next_k, + cap_height: caps[r + 1], + }, + ¤t_domain, + data, + rho, + &round_config, + transcript, + )? + .ok_or(Error::GroupedShape( + GroupedShape::SuccessorBlocks { + expected: config.num_queries, + got: 0, + }, + ))? + } + }; + current = TreeCheck::Checked(next_check); + } + (None, None, None) => { + // No successor, so no out-of-domain grind: the slot is never + // spent. + check_unspent(layout, round.nonces.ood)?; + transcript.append_field_element(&proof.final_value); + check_nonce::(transcript, config.grind.query, round.nonces.query, layout)?; + match &round.openings { + RoundOpenings::Base(openings) => verify_final::( + openings, + current, + ¤t_domain, + data, + &round_config, + &proof.final_value, + transcript, + )?, + RoundOpenings::Extension(openings) => verify_final::( + openings, + current, + ¤t_domain, + data, + &round_config, + &proof.final_value, + transcript, + )?, + RoundOpenings::BaseGrouped(openings) => { + whir_round::verify_grouped::( + openings, + &round0_trees, + whir_round::GroupedEnd::Final(&proof.final_value), + ¤t_domain, + data, + rho, + &round_config, + transcript, + )?; + } + } + } + // A successor or an out-of-domain value where the schedule ends, or + // one missing where it does not. + _ => { + return Err(Error::RoundCountMismatch { + expected: schedule.len(), + got: r, + }); + } + } + + alphas.extend(group.point); + current_domain = next_domain; + } + + // The accumulated weight at the full challenge point: the caller's own, + // plus each round's batched `eq` over the challenges that came after it. + let mut weight = weight_at(&alphas)?; + for (gamma, point, bound) in &ood { + weight += gamma * eq_eval(point, &alphas[*bound..])?; + } + + // The wire: the sumcheck's residual is w(α)·f(α), and the fully folded + // codeword is f(α). Both must name the same value. + let required = weight + .inv() + .ok() + .map(|inv| claim * inv) + .ok_or(Error::DegenerateEvaluationPoint)?; + if proof.final_value != required { + return Err(Error::EvaluationMismatch); + } + + Ok(()) +} + +/// The last round: every queried block must fold to the constant that was sent. +fn verify_final<'a, F, C, N, T, H>( + openings: &'a RoundProof, + current: TreeCheck<'a>, + current_domain: &Domain, + alphas: &[FieldElement], + config: &RoundConfig, + final_value: &FieldElement, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf, + C: IsField + IsSubFieldOf + 'static, + N: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + if openings.current.len() != config.num_queries || !openings.next.is_empty() { + return Err(Error::QueryCountMismatch { + expected: config.num_queries, + got: openings.current.len(), + }); + } + let num_leaves = current_domain.size() >> config.log_folding; + let depth = num_leaves.trailing_zeros() as usize; + // The tree's check, built once from its first opening, after the count + // guard above, so indexing it never panics. With no openings there is nothing to check. + let Some(first) = openings.current.first() else { + return Ok(()); + }; + let (check, first_siblings) = current.open::(depth, first)?; + + for (i, opening) in openings.current.iter().enumerate() { + let q = transcript.sample_u64(num_leaves as u64) as usize; + let siblings = if i == 0 { + first_siblings + } else { + opening.proof.merkle_path.as_slice() + }; + let at = current.position(q, depth); + if !verify_opening_capped::(&check, at, opening, siblings) { + return Err(Error::OpeningRejected { query: i }); + } + if fold_coset::(&opening.values, current_domain, q, alphas)? != *final_value { + return Err(Error::FoldInconsistent { query: i }); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{eq::eq_evals, whir_eval, whir_hash::KeccakWhir}; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"whir-chain-test") + } + + /// The current codeword's blocks, whichever variant holds them. In these + /// tests `F` and `E` coincide, so both arms read the same. + fn current_blocks(openings: &RoundOpenings) -> &[crate::whir_commit::CosetOpening] { + match openings { + RoundOpenings::Base(p) => &p.current, + RoundOpenings::Extension(p) => &p.current, + RoundOpenings::BaseGrouped(_) => panic!("a per-polynomial chain"), + } + } + + fn current_blocks_mut( + openings: &mut RoundOpenings, + ) -> &mut [crate::whir_commit::CosetOpening] { + match openings { + RoundOpenings::Base(p) => &mut p.current, + RoundOpenings::Extension(p) => &mut p.current, + RoundOpenings::BaseGrouped(_) => panic!("a per-polynomial chain"), + } + } + + fn config(log_folding: usize) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding, + num_queries: 3, + grind: GrindBits::default(), + format: ChainFormat::DEFAULT, + } + } + + /// Deterministic pseudo-random values; the seed is mixed before the shift so + /// nearby seeds cannot collapse to the same polynomial. + fn pseudo_mle(num_vars: usize, seed: u64) -> Mle { + let vals: Vec = (0..(1u64 << num_vars)) + .map(|i| { + let mixed = i + .wrapping_add(seed) + .wrapping_mul(6364136223846793005) + .wrapping_add(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + FE::from(mixed >> 13) + }) + .collect(); + Mle::new(vals).unwrap() + } + + fn point(num_vars: usize) -> Vec { + (0..num_vars).map(|i| FE::from(101 + i as u64)).collect() + } + + fn run(num_vars: usize, log_folding: usize) -> Result, Error> { + let cfg = config(log_folding); + let f = pseudo_mle(num_vars, 11); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true)?; + let proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript())?; + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript(), + )?; + Ok(proof) + } + + /// ★ The verifier holds the caller's domain to the configuration's rate: + /// the honest proof against a domain one size smaller or one larger is + /// refused by name, before any opening is read; against its own it + /// verifies. + #[test] + fn a_domain_off_the_configured_rate_is_refused() { + let cfg = config(2); + let num_vars = 6; + let f = pseudo_mle(num_vars, 29); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + let rate = num_vars + cfg.log_blowup; + assert_eq!(domain.log_size(), rate); + let check = |domain: &Domain| { + verify::( + &proof, + &commitment.root(), + &z, + y, + domain, + &cfg, + &mut transcript(), + ) + }; + for log_size in [rate - 1, rate + 1] { + let other = Domain::::new(log_size).unwrap(); + assert_eq!( + check(&other), + Err(Error::DomainRateMismatch { + expected: rate, + got: log_size + }), + "a domain of 2^{log_size} points" + ); + } + assert_eq!(check(&domain), Ok(()), "its own domain"); + } + + #[test] + fn an_honest_evaluation_verifies() { + for num_vars in 1..=6usize { + for log_folding in 1..=3usize { + run(num_vars, log_folding) + .unwrap_or_else(|e| panic!("num_vars={num_vars}, k={log_folding}: {e:?}")); + } + } + } + + #[test] + fn the_query_count_follows_the_johnson_bound() { + // Blowup 4, 128 bits, 20 bits of query grinding, one round: the same + // 110 the univariate prover's own accounting gives. + let cfg = ChainConfig::with_security( + 2, + 4, + 4, + 128, + GrindBits { + query: 20, + ..GrindBits::default() + }, + ); + assert_eq!(cfg.num_queries, 110); + + // A wider blowup buys more per query. + let wider = ChainConfig::with_security( + 3, + 4, + 4, + 128, + GrindBits { + query: 20, + ..GrindBits::default() + }, + ); + assert!(wider.num_queries < cfg.num_queries); + + // More rounds cost a union bound, and grinding takes queries off. + let deeper = ChainConfig::with_security(2, 4, 64, 128, GrindBits::default()); + let ground = ChainConfig::with_security(2, 4, 64, 128, GrindBits::uniform(20)); + assert!(deeper.num_queries > cfg.num_queries); + assert!(ground.num_queries < deeper.num_queries); + } + + /// And the parameters it picks actually run. + #[test] + fn a_proof_at_the_chosen_parameters_verifies() { + let num_vars = 5; + // Small bits so the grind is instant; the query count is the real one. + let cfg = ChainConfig::with_security(2, 2, num_vars, 32, GrindBits::uniform(8)); + assert!(cfg.num_queries > 10); + + let (proof, root, domain, y) = prove_ground(num_vars, &cfg); + verify_ground(&proof, &root, &domain, y, &cfg, num_vars).unwrap(); + } + + #[test] + fn the_schedule_takes_the_remainder_last() { + assert_eq!(config(3).schedule(10), vec![3, 3, 3, 1]); + assert_eq!(config(3).schedule(9), vec![3, 3, 3]); + assert_eq!(config(4).schedule(3), vec![3]); + assert_eq!(config(1).schedule(3), vec![1, 1, 1]); + } + + // --------------------------------------------------------------- + // W2: the first-fold schedule. + // --------------------------------------------------------------- + + fn first(k0: usize, log_folding: usize) -> ChainConfig { + ChainConfig { + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(k0).unwrap()), + ..ChainFormat::DEFAULT + }, + ..config(log_folding) + } + } + + /// Today's `schedule` body, verbatim, as the reference the default must + /// reproduce. + fn uniform_reference(log_folding: usize, num_vars: usize) -> Vec { + let step = log_folding.max(1); + let mut left = num_vars; + let mut out = Vec::new(); + while left > 0 { + let take = step.min(left); + out.push(take); + left -= take; + } + out + } + + #[test] + fn the_schedule_is_uniform_by_default() { + for k in 1..=MAX_FOLD { + for n in 0..=40 { + assert_eq!( + config(k).schedule(n), + uniform_reference(k, n), + "k={k} n={n}" + ); + assert_eq!(config(k).rounds(n), uniform_reference(k, n).len()); + } + } + assert_eq!(ChainFormat::DEFAULT.folds, WhirFolds::Uniform); + assert_eq!(WhirFolds::default(), WhirFolds::Uniform); + } + + #[test] + fn with_security_folds_uniform_is_with_security() { + for k in 1..=MAX_FOLD { + for n in 0..=40 { + for grind in [GrindBits::default(), GrindBits::uniform(20)] { + assert_eq!( + ChainConfig::with_security_folds(2, k, WhirFolds::Uniform, n, 128, grind), + ChainConfig::with_security(2, k, n, 128, grind), + "k={k} n={n}" + ); + } + } + } + // And today's production numbers. + let today = ChainConfig::with_security(2, 4, 25, 128, GrindBits::uniform(20)); + assert_eq!((today.rounds(25), today.num_queries), (7, 112)); + } + + #[test] + fn the_default_fold_word_is_log_folding() { + for k in 1..=MAX_FOLD { + assert_eq!(config(k).fold_word(), k as u64); + } + assert_eq!(config(4).fold_word().to_le_bytes(), 4u64.to_le_bytes()); + } + + /// The first-fold schedules, by hand, and the clamp at small heights. + #[test] + fn the_first_fold_schedules() { + let (f5, f6) = (first(5, 4), first(6, 4)); + assert_eq!(f6.schedule(25), vec![6, 4, 4, 4, 4, 3]); + assert_eq!(f6.schedule(24), vec![6, 4, 4, 4, 4, 2]); + assert_eq!(f6.schedule(23), vec![6, 4, 4, 4, 4, 1]); + assert_eq!(f5.schedule(25), vec![5, 4, 4, 4, 4, 4]); + assert_eq!(f5.schedule(24), vec![5, 4, 4, 4, 4, 3]); + assert_eq!(f5.schedule(23), vec![5, 4, 4, 4, 4, 2]); + // The first round takes everything when there is less than k0. + assert_eq!(f6.schedule(0), Vec::::new()); + assert_eq!(f6.schedule(3), vec![3]); + assert_eq!(f6.schedule(6), vec![6]); + assert_eq!(f6.schedule(7), vec![6, 1]); + assert_eq!(f5.schedule(9), vec![5, 4]); + assert_eq!(f5.schedule(11), vec![5, 4, 2]); + assert_eq!(f6.schedule(9), vec![6, 3]); + // Every schedule covers exactly `n`, starts at min(k0, n), then walks + // today's uniform body over the rest; no fold exceeds MAX_FOLD. + for k0 in 1..=MAX_FOLD { + for k in 1..=4 { + let c = first(k0, k); + for n in 0..=40 { + let s = c.schedule(n); + assert_eq!(s.iter().sum::(), n); + assert!(s.iter().all(|&x| (1..=MAX_FOLD).contains(&x))); + if n > 0 { + assert_eq!(s[0], k0.min(n)); + assert_eq!(s[1..], uniform_reference(k, n - s[0])[..]); + } + } + } + } + } + + #[test] + fn a_first_fold_outside_the_tested_stack_is_unconstructible() { + assert!(FirstFold::new(0).is_none()); + assert!(FirstFold::new(MAX_FOLD + 1).is_none()); + assert!(FirstFold::new(64).is_none()); + for k in 1..=MAX_FOLD { + assert_eq!(FirstFold::new(k).unwrap().get(), k); + } + } + + /// S2: a stack cap of 0 or past the widest one measured to fit is not a + /// value a config can hold, and today's 25 is the default everywhere one is + /// spelled. + #[test] + fn a_stack_cap_outside_the_measured_range_is_unconstructible() { + assert!(StackVars::new(0).is_none()); + assert!(StackVars::new(StackVars::WIDEST + 1).is_none()); + assert!(StackVars::new(64).is_none()); + for n in 1..=StackVars::WIDEST { + assert_eq!(StackVars::new(n).unwrap().get(), n); + } + assert_eq!(StackVars::LEGACY.get(), 25); + assert_eq!(StackVars::default(), StackVars::LEGACY); + assert_eq!(ChainFormat::DEFAULT.stack, StackVars::LEGACY); + assert_eq!(ChainFormat::default(), ChainFormat::DEFAULT); + assert!(ChainFormat::DEFAULT.is_default()); + let wide = ChainFormat { + stack: StackVars::new(27).unwrap(), + ..ChainFormat::DEFAULT + }; + assert!(!wide.is_default(), "a wider stack is not today's format"); + } + + #[test] + fn every_fold_word_is_distinct_and_the_non_default_ones_are_tagged() { + // {Uniform, First(k0)} × {PerPoly, Indexed} (REV-B2-JUDGE B1-5): every + // word distinct, and every non-default one tagged. + let indexed = |mut c: ChainConfig| { + c.format.group = GroupFold::Indexed; + c + }; + let mut seen = std::collections::HashSet::new(); + for k in 1..=MAX_FOLD { + let uniform = config(k).fold_word(); + assert_eq!(uniform & FOLD_WORD_TAG, 0); + assert!(seen.insert(uniform)); + let w = indexed(config(k)).fold_word(); + assert_ne!(w & FOLD_WORD_TAG, 0, "k={k} uniform, indexed"); + assert_ne!(w & FOLD_WORD_INDEXED, 0, "k={k} uniform, indexed"); + assert!(seen.insert(w), "k={k} uniform, indexed: {w:#x} repeats"); + for k0 in 1..=MAX_FOLD { + let w = first(k0, k).fold_word(); + assert_ne!(w & FOLD_WORD_TAG, 0, "k={k} k0={k0}"); + assert_eq!(w & FOLD_WORD_INDEXED, 0, "k={k} k0={k0}"); + assert!(seen.insert(w), "k={k} k0={k0}: {w:#x} repeats"); + let w = indexed(first(k0, k)).fold_word(); + assert_ne!(w & FOLD_WORD_TAG, 0, "k={k} k0={k0} indexed"); + assert_ne!(w & FOLD_WORD_INDEXED, 0, "k={k} k0={k0} indexed"); + assert!(seen.insert(w), "k={k} k0={k0} indexed: {w:#x} repeats"); + } + } + // The production words, spelled out. + assert_eq!(first(6, 4).fold_word(), 0x8041_0000_0000_0006); + assert_eq!(first(5, 4).fold_word(), 0x8041_0000_0000_0005); + assert_eq!(indexed(first(6, 4)).fold_word(), 0x8041_8000_0000_0006); + assert_eq!(indexed(config(4)).fold_word(), 0x8000_8000_0000_0004); + } + + /// No accepted first fold raises the round count at any height, so the + /// union bound is never charged more and Q never rises; at the production + /// tallest (25) it is today's 112. + #[test] + fn a_first_fold_never_raises_the_query_count() { + let g = GrindBits::uniform(20); + for k0 in [5usize, 6] { + let folds = WhirFolds::First(FirstFold::new(k0).unwrap()); + for tallest in 1..=32 { + let today = ChainConfig::with_security(2, 4, tallest, 128, g); + let arm = ChainConfig::with_security_folds(2, 4, folds, tallest, 128, g); + assert_eq!(arm.format.folds, folds); + for m in 1..=tallest { + assert!(arm.rounds(m) <= today.rounds(m), "k0={k0} m={m}"); + } + assert!( + arm.num_queries <= today.num_queries, + "k0={k0} tallest={tallest}" + ); + } + let at25 = ChainConfig::with_security_folds(2, 4, folds, 25, 128, g); + assert_eq!((at25.rounds(25), at25.num_queries), (6, 112), "k0={k0}"); + } + // The rule charges the WORST height, not the tallest: a first fold + // narrower than the uniform one has more rounds at some shorter chain, + // and Q follows that one. + let narrow = ChainConfig::with_security_folds( + 2, + 4, + WhirFolds::First(FirstFold::new(1).unwrap()), + 8, + 128, + GrindBits::default(), + ); + let worst = (1..=8).map(|m| narrow.rounds(m)).max().unwrap(); + assert_eq!(worst, 3); + assert_eq!( + narrow.num_queries, + crate::query_count::num_queries(2, worst, 128, 0) + ); + } + + fn run_with(cfg: &ChainConfig, num_vars: usize) -> Result, Error> { + let f = pseudo_mle(num_vars, 13); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + let (commitment, domain) = commit::(&f, cfg, true)?; + let proof = + prove::(&f, &z, &commitment, &domain, cfg, &mut transcript())?; + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + cfg, + &mut transcript(), + )?; + Ok(proof) + } + + #[test] + fn a_first_fold_proof_verifies_and_opens_its_wide_block() { + for (k0, n) in [(5, 9), (5, 11), (6, 9), (6, 11), (6, 6), (6, 3)] { + let cfg = first(k0, 4); + let proof = run_with(&cfg, n).unwrap_or_else(|e| panic!("k0={k0} n={n}: {e:?}")); + assert_eq!(proof.rounds.len(), cfg.rounds(n)); + for (round, &k) in proof.rounds.iter().zip(&cfg.schedule(n)) { + assert_eq!(round.sumcheck.len(), k); + for opening in current_blocks(&round.openings) { + assert_eq!(opening.values.len(), 1 << k); + } + } + } + } + + #[test] + fn a_tampered_wide_base_block_is_rejected() { + let cfg = first(6, 4); + let num_vars = 11; + let f = pseudo_mle(num_vars, 83); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let honest = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + assert_eq!( + current_blocks(&honest.rounds[0].openings)[0].values.len(), + 64 + ); + // The last value of the 64-wide block, so the check must read all of it. + let mut proof = honest.clone(); + current_blocks_mut(&mut proof.rounds[0].openings)[0].values[63] += FE::one(); + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript(), + ) + .unwrap_err(); + assert!( + matches!( + err, + Error::OpeningRejected { .. } | Error::FoldInconsistent { .. } + ), + "{err:?}" + ); + } + + /// A proof made under one schedule is refused under another: the round + /// count (or the first round's sumcheck length) disagrees. + #[test] + fn a_first_fold_proof_is_refused_under_the_uniform_schedule() { + let num_vars = 11; + let (f6, uniform) = (first(6, 4), config(4)); + let f = pseudo_mle(num_vars, 13); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + let (commitment, domain) = commit::(&f, &f6, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, &f6, &mut transcript()) + .unwrap(); + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &uniform, + &mut transcript(), + ) + .unwrap_err(); + assert!(matches!(err, Error::RoundCountMismatch { .. }), "{err:?}"); + // And the other way round. + let (commitment, domain) = commit::(&f, &uniform, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, &uniform, &mut transcript()) + .unwrap(); + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &f6, + &mut transcript(), + ) + .unwrap_err(); + assert!(matches!(err, Error::RoundCountMismatch { .. }), "{err:?}"); + } + + /// The point of chaining: a query opens a block of `2^k`, not the message. + #[test] + fn a_block_is_the_fold_size_not_the_message() { + let num_vars = 6; + let log_folding = 2; + let proof = run(num_vars, log_folding).unwrap(); + + assert_eq!(proof.rounds.len(), 3); + for round in &proof.rounds { + for opening in current_blocks(&round.openings) { + assert_eq!(opening.values.len(), 1 << log_folding); + } + } + } + + /// And what that buys, against the one-round argument on the same + /// polynomial. + #[test] + fn chaining_opens_far_less_than_one_round_does() { + let num_vars = 8; + let cfg = config(2); + let f = pseudo_mle(num_vars, 13); + let z = point(num_vars); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let chained = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + + let one_round_cfg = whir_eval::EvalConfig { + log_blowup: cfg.log_blowup, + num_queries: cfg.num_queries, + }; + let (one_commitment, one_domain) = + whir_eval::commit::(&f, &one_round_cfg).unwrap(); + let one_round = whir_eval::prove::( + &f, + &z, + &one_commitment, + &one_domain, + &one_round_cfg, + &mut transcript(), + ) + .unwrap(); + + let one_round_elements: usize = one_round.openings.iter().map(|o| o.values.len()).sum(); + // One round sends `num_queries` blocks of the whole message; chaining + // sends small blocks, two per query per round. + assert_eq!(one_round_elements, 3 * (1 << num_vars)); + assert!( + chained.opened_elements() * 4 < one_round_elements, + "chained {} vs one round {one_round_elements}", + chained.opened_elements() + ); + } + + #[test] + fn a_false_evaluation_is_rejected() { + let cfg = config(2); + let num_vars = 5; + let f = pseudo_mle(num_vars, 17); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y + FE::one(), + &domain, + &cfg, + &mut transcript(), + ) + .is_err() + ); + } + + #[test] + fn a_forged_final_value_is_rejected() { + let cfg = config(2); + let num_vars = 5; + let f = pseudo_mle(num_vars, 19); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + proof.final_value += FE::one(); + + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript() + ) + .is_err() + ); + } + + #[test] + fn a_tampered_opening_in_a_middle_round_is_rejected() { + let cfg = config(2); + let num_vars = 6; + let f = pseudo_mle(num_vars, 23); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + assert!(proof.rounds.len() >= 3); + current_blocks_mut(&mut proof.rounds[1].openings)[0].values[0] += FE::one(); + + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript(), + ) + .unwrap_err(); + assert!(matches!(err, Error::OpeningRejected { .. })); + } + + /// The first round's blocks are hashed as base-field bytes, so tampering + /// with one has to be caught by that hash and not by a later fold. + #[test] + fn a_tampered_opening_in_the_base_round_is_rejected() { + let cfg = config(2); + let num_vars = 6; + let f = pseudo_mle(num_vars, 79); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + assert!(matches!(proof.rounds[0].openings, RoundOpenings::Base(_))); + current_blocks_mut(&mut proof.rounds[0].openings)[0].values[0] += FE::one(); + + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript(), + ) + .unwrap_err(); + assert!(matches!(err, Error::OpeningRejected { .. })); + } + + /// A prover who commits one polynomial and argues about another: the fold + /// check between consecutive codewords is what catches it. + #[test] + fn a_proof_about_a_different_polynomial_is_rejected() { + let cfg = config(2); + let num_vars = 5; + let f = pseudo_mle(num_vars, 29); + let g = pseudo_mle(num_vars, 31); + let z = point(num_vars); + + let (f_commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let proof = + prove::(&g, &z, &f_commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + + let err = verify::( + &proof, + &f_commitment.root(), + &z, + g.evaluate(&z).unwrap(), + &domain, + &cfg, + &mut transcript(), + ) + .unwrap_err(); + // The codeword now comes out of the commitment, so a prover cannot be + // inconsistent between the two: the lie has nowhere to go but the wire + // between the sumcheck and the folded value. + assert_eq!(err, Error::EvaluationMismatch); + } + + #[test] + fn a_round_missing_its_successor_is_rejected() { + let cfg = config(2); + let num_vars = 6; + let f = pseudo_mle(num_vars, 37); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + proof.rounds[0].next_root = None; + + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript() + ) + .is_err() + ); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let cfg = config(2); + let num_vars = 5; + let f = pseudo_mle(num_vars, 41); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + + let mut other = DefaultTranscript::::new(b"a-different-statement"); + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut other + ) + .is_err() + ); + } + + /// A weight that is not an `eq`: the shape a stacked multi-point claim + /// takes, chained the same way. + #[test] + fn a_two_point_weighted_claim_chains() { + let cfg = config(2); + let num_vars = 5; + let f = pseudo_mle(num_vars, 43); + let a = point(num_vars); + let b: Vec = a.iter().map(|x| x + FE::from(7)).collect(); + let gamma = FE::from(5); + + // w = eq(a, ·) + gamma·eq(b, ·) + let table: Vec = eq_evals(&a) + .into_iter() + .zip(eq_evals(&b)) + .map(|(p, q)| p + gamma * q) + .collect(); + let weight = Mle::new(table).unwrap(); + let y = f.evaluate(&a).unwrap() + gamma * f.evaluate(&b).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let proof = prove_weighted::( + &f, + weight, + &commitment, + &domain, + &cfg, + &mut transcript(), + ) + .unwrap(); + + verify_weighted::( + &proof, + &commitment.root(), + |at: &[FE]| Ok(eq_eval(&a, at)? + gamma * eq_eval(&b, at)?), + y, + num_vars, + &domain, + &cfg, + &mut transcript(), + ) + .unwrap(); + } + + /// The committed codeword is base-field, so only the first round's blocks + /// are — that is where the memory goes, and the type says which round is + /// which rather than leaving it implicit. + #[test] + fn only_the_first_round_opens_base_field_blocks() { + let proof = run(6, 2).unwrap(); + assert!(proof.rounds.len() >= 3); + assert!(matches!(proof.rounds[0].openings, RoundOpenings::Base(_))); + for round in &proof.rounds[1..] { + assert!(matches!(round.openings, RoundOpenings::Extension(_))); + } + } + + /// And the verifier says so: a first round presenting extension blocks is + /// refused outright, rather than left to fail on a byte-width mismatch + /// inside a hash. + #[test] + fn a_first_round_claiming_extension_blocks_is_rejected() { + let cfg = config(2); + let num_vars = 6; + let f = pseudo_mle(num_vars, 73); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + + // `F` and `E` coincide here, so the same blocks fit the other variant. + if let RoundOpenings::Base(openings) = proof.rounds[0].openings.clone() { + proof.rounds[0].openings = RoundOpenings::Extension(openings); + } + + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript() + ) + .is_err() + ); + } + + // --------------------------------------------------------------- + // Out of domain. + // --------------------------------------------------------------- + + /// The convention the out-of-domain claim rests on: the multilinear point + /// `(z, z^2, z^4, ..)` is where the univariate lift is `z`. Pinned against + /// the codeword itself, at every domain point. + #[test] + fn the_out_of_domain_point_is_where_the_lift_is_sampled() { + use crate::whir::{encode, lift_coefficients}; + + let num_vars = 4; + let f = pseudo_mle(num_vars, 53); + let domain = Domain::::new(num_vars + 2).unwrap(); + let codeword = encode::(&lift_coefficients(&f), &domain).unwrap(); + + let mut x = FE::one(); + for value in &codeword { + assert_eq!(*value, f.evaluate(&ood_point(&x, num_vars)).unwrap()); + x *= domain.generator(); + } + } + + /// And the property the whole round rests on: the folded codeword encodes + /// the message the sumcheck folded, in the same variable order. + #[test] + fn the_folded_codeword_encodes_the_folded_message() { + use crate::whir::{encode, lift_coefficients}; + + let num_vars = 4; + let f = pseudo_mle(num_vars, 59); + let domain = Domain::::new(num_vars + 2).unwrap(); + let codeword = encode::(&lift_coefficients(&f), &domain).unwrap(); + + let alphas = [FE::from(3), FE::from(5)]; + let (folded, folded_domain) = + crate::whir::fold_codeword_k::(&codeword, &domain, &alphas).unwrap(); + + let mut message = f; + for alpha in &alphas { + message.fix_first_variable_in_place(alpha).unwrap(); + } + let re_encoded = encode::(&lift_coefficients(&message), &folded_domain).unwrap(); + assert_eq!(folded, re_encoded); + } + + /// ★ The same at the production shape, through the prover's own steps: a + /// stacked base-field commitment at blowup 4, a first fold of six into the + /// cubic extension and folds of four after it, the message bound by the + /// chain's sumcheck on challenges from its transcript, the weight batched + /// with the out-of-domain term between rounds. Every round's successor — + /// the fold of the codeword it opens — is the bound message encoded on the + /// folded domain, byte for byte as the tree hashes it. So a successor can + /// be encoded from the `2^(n−k)` message instead of folded from the + /// `2^(n+blowup)` codeword (D-SPEED lever 5a). The domain is the subgroup + /// itself: there is no coset offset to match. + /// + /// Two near misses must differ, so the equality is not one any encoding + /// would show: the message encoded without the lift's bit reversal, and the + /// first round's variables bound from the other end. + #[test] + fn every_successor_is_the_bound_message_encoded_at_the_production_shape() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as E; + type EE = FieldElement; + let bytes = + |values: &[EE]| -> Vec { values.iter().flat_map(|v| v.as_bytes()).collect() }; + let lift = |m: &Mle| -> Mle { + Mle::new(m.evals().iter().map(|v| (*v).to_extension::()).collect()).unwrap() + }; + + let num_vars = 15; + let cfg = first(6, 4); + let schedule = cfg.schedule(num_vars); + assert_eq!(schedule, [6, 4, 4, 1], "the production schedule's shape"); + + // Columns of mixed heights where the production layout puts them. + let heights = [13usize, 13, 12, 11, 10, 8, 6, 3]; + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| pseudo_mle(h, 71 + 13 * i as u64)) + .collect(); + let layout = crate::stacking::StackedLayout::build(&heights, num_vars).unwrap(); + assert_eq!(layout.num_polys(), 1); + let stacked = Stacked { + parts: layout + .placements() + .iter() + .zip(&columns) + .map(|(place, column)| (column as &dyn HostColumn, place.offset)) + .collect(), + resident: None, + num_vars, + }; + let (commitment, domain) = commit_stacked::(&stacked, &cfg, true).unwrap(); + let base = commitment + .codeword() + .host() + .expect("no device: the host committed") + .to_vec(); + let message = stacked.assemble().unwrap(); + + let weight = Mle::new( + (0..1u64 << num_vars) + .map(|i| EE::from(i.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 7)) + .collect(), + ) + .unwrap(); + let mut factors = Factors::::new(&message, weight).unwrap(); + let mut transcript = DefaultTranscript::::new(b"whir-chain-5a"); + + let mut current: Vec = Vec::new(); + let mut current_domain = domain; + for (r, &k) in schedule.iter().enumerate() { + let (_, alphas) = factors.rounds(k, &mut transcript).unwrap(); + let (folded, folded_domain) = if r == 0 { + crate::whir::fold_codeword_k_on_host::(&base, ¤t_domain, &alphas) + } else { + crate::whir::fold_codeword_k_on_host::(¤t, ¤t_domain, &alphas) + } + .unwrap(); + let Factors::Host { message: bound, .. } = &factors else { + panic!("no device: the host bound the message"); + }; + let left: usize = num_vars - schedule[..=r].iter().sum::(); + assert_eq!(bound.num_vars(), left); + assert_eq!(folded.len(), 1 << (left + cfg.log_blowup)); + let encoded = encode::(&lift_coefficients(bound), &folded_domain).unwrap(); + assert_eq!( + bytes(&encoded), + bytes(&folded), + "round {r} (fold {k}): the encoded message is not the folded codeword" + ); + + if r == 0 { + let unreversed = + encode::(&crate::whir::monomial_coefficients(bound), &folded_domain) + .unwrap(); + assert_ne!( + bytes(&unreversed), + bytes(&folded), + "the lift's order matters" + ); + let mut other_end = lift(&message); + for alpha in &alphas { + other_end.fix_last_variable_in_place(alpha).unwrap(); + } + let wrong = encode::(&lift_coefficients(&other_end), &folded_domain).unwrap(); + assert_ne!(bytes(&wrong), bytes(&folded), "the binding order matters"); + } + + if schedule.get(r + 1).is_some() { + let z0: EE = transcript.sample_field_element(); + let point = ood_point(&z0, factors.num_vars()); + let y0 = factors.evaluate_message(&point).unwrap(); + transcript.append_field_element(&y0); + let gamma: EE = transcript.sample_field_element(); + factors.add_scaled_eq(&point, &gamma).unwrap(); + } + current = folded; + current_domain = folded_domain; + } + } + + /// ★ A coset round 0 whose factors the host built encodes its successor + /// there ([`encode_successor_on_host`]): the bytes of the host's fold, for a + /// polynomial's chain and for runs of 2, 3 and 4 (the run's combined bound + /// message), each call counted. The factors are the host's here because the + /// stack is under the card's sumcheck threshold, as in a test format. + #[test] + fn the_host_encoded_successor_is_the_host_fold_for_a_polynomial_and_a_run() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as E; + type EE = FieldElement; + let bytes = |c: &Codeword| -> Vec { + c.host() + .expect("on the host") + .iter() + .flat_map(|v| v.as_bytes()) + .collect() + }; + let num_vars = 11; + let cfg = first(6, 4); + let k = cfg.schedule(num_vars)[0]; + let weight = |seed: u64| -> Mle { + Mle::new( + (0..1u64 << num_vars) + .map(|i| EE::from((i + seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 7)) + .collect(), + ) + .unwrap() + }; + for width in 1..=MAX_INDEX_POLYS { + let polys: Vec> = (0..width) + .map(|p| pseudo_mle(num_vars, 29 + 17 * p as u64)) + .collect(); + let stacks: Vec> = polys + .iter() + .map(|f| Stacked { + parts: vec![(f as &dyn HostColumn, 0)], + resident: None, + num_vars, + }) + .collect(); + let committed: Vec<(CodewordCommitment, Domain)> = stacks + .iter() + .map(|s| commit_stacked::(s, &cfg, true).unwrap()) + .collect(); + let domain = committed[0].1.clone(); + let held: Vec<&CodewordCommitment> = + committed.iter().map(|(c, _)| c).collect(); + let index = if width == 1 { 0 } else { index_vars(width) }; + let mut factors = if width == 1 { + Factors::::new(&polys[0], weight(1)).unwrap() + } else { + Factors::::Lockstep { + parts: polys + .iter() + .enumerate() + .map(|(p, f)| { + let message = Mle::new( + f.evals().iter().map(|v| (*v).to_extension::()).collect(), + ) + .unwrap(); + (weight(1 + p as u64), message) + }) + .collect(), + index_vars: index, + } + }; + let mut transcript = DefaultTranscript::::new(b"whir-chain-host-encode"); + let (_, challenges) = factors.rounds(k, &mut transcript).unwrap(); + let (alphas, rho) = challenges.split_at(k); + assert_eq!(rho.len(), index, "width {width}: the index bound last"); + let (folded, folded_domain) = if width == 1 { + fold_held::(held[0].codeword(), &domain, alphas).unwrap() + } else { + fold_grouped::(&held, &domain, alphas, rho).unwrap() + }; + let before = crate::gpu::coset_host_encodes(); + let message = factors + .host_message() + .expect("under the threshold: the host bound the message"); + let (encoded, at) = encode_successor_on_host::(message, &domain, k).unwrap(); + assert!( + crate::gpu::coset_host_encodes() > before, + "width {width}: the host encode counted" + ); + assert_eq!(at.size(), folded_domain.size(), "width {width}: the domain"); + assert_eq!( + bytes(&encoded), + bytes(&folded), + "width {width}: the host encode is the fold" + ); + } + } + + /// The grouped round 0's successor — each round-0 codeword folded by the + /// data challenges, the folds combined with `eq(ρ, p)` ([`fold_grouped`]) + /// — is the encoding of the message the lockstep rounds leave bound + /// ([`lockstep_rounds`]), for runs of 2, 3 and 4, and so is every later + /// round's successor. So a card that holds the run's combined factors may + /// encode the successor from them instead of folding `P'` codewords. A + /// near miss must differ: `ρ` read from the other end, where two index + /// variables make the order matter. + #[test] + fn a_grouped_successor_is_the_runs_bound_message_encoded() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as E; + type EE = FieldElement; + let bytes = + |values: &[EE]| -> Vec { values.iter().flat_map(|v| v.as_bytes()).collect() }; + let num_vars = 11; + let cfg = first(6, 4); + let schedule = cfg.schedule(num_vars); + assert_eq!(schedule, [6, 4, 1]); + for width in 2..=MAX_INDEX_POLYS { + let polys: Vec> = (0..width) + .map(|p| pseudo_mle(num_vars, 71 + 13 * p as u64)) + .collect(); + let stacks: Vec> = polys + .iter() + .map(|f| Stacked { + parts: vec![(f as &dyn HostColumn, 0)], + resident: None, + num_vars, + }) + .collect(); + let committed: Vec<(CodewordCommitment, Domain)> = stacks + .iter() + .map(|s| commit_stacked::(s, &cfg, true).unwrap()) + .collect(); + let domain = committed[0].1.clone(); + let held: Vec<&CodewordCommitment> = + committed.iter().map(|(c, _)| c).collect(); + let parts: Vec<(Mle, Mle)> = polys + .iter() + .enumerate() + .map(|(p, f)| { + let weight = Mle::new( + (0..1u64 << num_vars) + .map(|i| { + EE::from( + (i + 1 + p as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 7, + ) + }) + .collect(), + ) + .unwrap(); + let message = + Mle::new(f.evals().iter().map(|v| (*v).to_extension::()).collect()) + .unwrap(); + (weight, message) + }) + .collect(); + let m = index_vars(width); + let mut factors = Factors::::Lockstep { + parts, + index_vars: m, + }; + let mut transcript = DefaultTranscript::::new(b"whir-chain-grouped-5a"); + let mut current: Vec = Vec::new(); + let mut current_domain = domain; + for (r, &k) in schedule.iter().enumerate() { + let (_, challenges) = factors.rounds(k, &mut transcript).unwrap(); + let (alphas, rho) = challenges.split_at(k.min(challenges.len())); + assert_eq!(rho.len(), if r == 0 { m } else { 0 }); + let (folded, folded_domain) = if r == 0 { + let (codeword, at) = + fold_grouped::(&held, ¤t_domain, alphas, rho) + .unwrap(); + (codeword.host().expect("a host fold").to_vec(), at) + } else { + crate::whir::fold_codeword_k_on_host::( + ¤t, + ¤t_domain, + alphas, + ) + .unwrap() + }; + let Factors::Host { message: bound, .. } = &factors else { + panic!("round 0 leaves one host pair"); + }; + let left = num_vars - schedule[..=r].iter().sum::(); + assert_eq!(bound.num_vars(), left); + assert_eq!(folded.len(), 1 << (left + cfg.log_blowup)); + let encoded = encode::(&lift_coefficients(bound), &folded_domain).unwrap(); + assert_eq!( + bytes(&encoded), + bytes(&folded), + "width {width} round {r}: the encoded message is not the successor" + ); + if r == 0 && m == 2 { + let swapped: Vec = rho.iter().rev().cloned().collect(); + let (other, _) = + fold_grouped::(&held, ¤t_domain, alphas, &swapped) + .unwrap(); + assert_ne!( + bytes(other.host().unwrap()), + bytes(&folded), + "width {width}: the index order matters" + ); + } + if schedule.get(r + 1).is_some() { + let z0: EE = transcript.sample_field_element(); + let point = ood_point(&z0, factors.num_vars()); + let y0 = factors.evaluate_message(&point).unwrap(); + transcript.append_field_element(&y0); + let gamma: EE = transcript.sample_field_element(); + factors.add_scaled_eq(&point, &gamma).unwrap(); + } + current = folded; + current_domain = folded_domain; + } + } + } + + #[test] + fn every_round_but_the_last_answers_out_of_domain() { + let proof = run(6, 2).unwrap(); + assert_eq!(proof.rounds.len(), 3); + for round in &proof.rounds[..2] { + assert!(round.ood_value.is_some()); + } + assert!(proof.rounds[2].ood_value.is_none()); + } + + /// The out-of-domain value is batched into the next group's claim, so + /// getting it wrong breaks the chain. + #[test] + fn a_forged_out_of_domain_value_is_rejected() { + let cfg = config(2); + let num_vars = 6; + let f = pseudo_mle(num_vars, 61); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + proof.rounds[0].ood_value = Some(proof.rounds[0].ood_value.unwrap() + FE::one()); + + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript() + ) + .is_err() + ); + } + + #[test] + fn a_round_missing_its_out_of_domain_value_is_rejected() { + let cfg = config(2); + let num_vars = 6; + let f = pseudo_mle(num_vars, 67); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); + proof.rounds[0].ood_value = None; + + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut transcript() + ) + .is_err() + ); + } + + #[test] + fn a_point_inside_the_domain_is_refused() { + // The generator of the folded domain is in it, so it must be rejected; + // a point outside must not be. + let domain = Domain::::new(5).unwrap(); + assert_eq!( + require_out_of_domain::(domain.generator(), &domain).err(), + Some(Error::OodPointInDomain) + ); + assert!(require_out_of_domain::(&FE::from(7), &domain).is_ok()); + } + + // --------------------------------------------------------------- + // Proof of work. + // --------------------------------------------------------------- + + /// Enough bits to be a real search, few enough to be instant. + fn ground(log_folding: usize) -> ChainConfig { + ChainConfig { + grind: GrindBits::uniform(8), + ..config(log_folding) + } + } + + fn prove_ground( + num_vars: usize, + cfg: &ChainConfig, + ) -> (ChainProof, Commitment, Domain, FE) { + let f = pseudo_mle(num_vars, 71); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + let (commitment, domain) = commit::(&f, cfg, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, cfg, &mut transcript()) + .unwrap(); + (proof, commitment.root(), domain, y) + } + + fn verify_ground( + proof: &ChainProof, + root: &Commitment, + domain: &Domain, + y: FE, + cfg: &ChainConfig, + num_vars: usize, + ) -> Result<(), Error> { + verify::( + proof, + root, + &point(num_vars), + y, + domain, + cfg, + &mut transcript(), + ) + } + + /// Grind checks in the verifier's order: `(the state the check read, the + /// nonce)`. + type GrindChecks = Vec<([u8; 32], u64)>; + + /// The verifier's transcript, logging every proof-of-work check it passes: + /// the state the check read and the nonce it absorbed right after. + /// `check_grind` is the verifier's only reader of `state()`, and it absorbs + /// the nonce next. + struct GrindLog { + inner: DefaultTranscript, + read: core::cell::Cell>, + checks: GrindChecks, + } + + impl IsTranscript for GrindLog { + fn append_field_element(&mut self, element: &FE) { + self.inner.append_field_element(element); + } + + fn append_bytes(&mut self, new_bytes: &[u8]) { + if let Some(state) = self.read.take() { + let nonce = new_bytes + .try_into() + .map(u64::from_be_bytes) + .expect("a passed grind check absorbs its eight-byte nonce next"); + self.checks.push((state, nonce)); + } + self.inner.append_bytes(new_bytes); + } + + fn state(&self) -> [u8; 32] { + let state = self.inner.state(); + self.read.set(Some(state)); + state + } + + fn sample_field_element(&mut self) -> FE { + self.inner.sample_field_element() + } + + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + self.inner.sample_u64(upper_bound) + } + } + + /// [`verify_ground`], with every grind check the verifier passed on the way. + fn verify_logged( + proof: &ChainProof, + root: &Commitment, + domain: &Domain, + y: FE, + cfg: &ChainConfig, + num_vars: usize, + ) -> (Result<(), Error>, GrindChecks) { + let mut log = GrindLog { + inner: transcript(), + read: Default::default(), + checks: Vec::new(), + }; + let verdict = + verify::(proof, root, &point(num_vars), y, domain, cfg, &mut log); + (verdict, log.checks) + } + + fn passes(state: &[u8; 32], nonce: u64, bits: u8) -> bool { + crypto::grinding::is_valid_nonce::>(state, nonce, bits) + } + + /// A forgery the check at `state` must refuse, whichever valid nonce the + /// search returned: the first of `nonce + 1, nonce + 2, …` that fails the + /// PoW there. `nonce + 1` alone passes with probability `2^-bits`, and a + /// forgery that passes is absorbed and moves the transcript, failing later + /// and elsewhere instead of at the check. + fn failing(state: &[u8; 32], nonce: u64, bits: u8) -> u64 { + (1..) + .map(|d| nonce.wrapping_add(d)) + .find(|&n| !passes(state, n, bits)) + .expect("a nonce that fails the PoW") + } + + /// Proves a uniformly ground chain, replaces the spent nonce of `slot` in + /// `round` with a forgery its check must refuse, and verifies. The log + /// holds three checks a round (folding, out-of-domain, query), so `round` + /// must not be the last, which has no out-of-domain check. + fn forge_uniform(round: usize, slot: &str) -> Result<(), Error> { + let num_vars = 6; + let cfg = ground(2); + let (mut proof, root, domain, y) = prove_ground(num_vars, &cfg); + let (verdict, checks) = verify_logged(&proof, &root, &domain, y, &cfg, num_vars); + verdict.expect("the honest proof must verify"); + assert!(round + 1 < proof.rounds.len(), "not the last round"); + let nonces = &mut proof.rounds[round].nonces; + let (at, field) = match slot { + "folding" => (0, &mut nonces.folding), + "ood" => (1, &mut nonces.ood), + _ => (2, &mut nonces.query), + }; + let (state, nonce) = checks[3 * round + at]; + assert_eq!(*field, nonce, "the logged check is round {round}'s {slot}"); + *field = failing(&state, nonce, 8); + verify_ground(&proof, &root, &domain, y, &cfg, num_vars) + } + + #[test] + fn a_ground_proof_verifies() { + let num_vars = 6; + let cfg = ground(2); + let (proof, root, domain, y) = prove_ground(num_vars, &cfg); + + // Every round grinds before its folding challenges and its queries, and + // before the out-of-domain challenge where there is one: the verifier + // checks exactly those nonces, in that order, and each passes the PoW + // at the state its check read. Not "non-zero": the search may return + // any valid nonce, 0 included (`find_any` under `parallel`). + let (verdict, checks) = verify_logged(&proof, &root, &domain, y, &cfg, num_vars); + verdict.unwrap(); + let last = proof.rounds.len() - 1; + let spent: Vec = proof + .rounds + .iter() + .enumerate() + .flat_map(|(r, round)| { + let n = round.nonces; + if r == last { + vec![n.folding, n.query] + } else { + vec![n.folding, n.ood, n.query] + } + }) + .collect(); + assert_eq!(checks.len(), 3 * proof.rounds.len() - 1); + assert_eq!( + checks.iter().map(|&(_, nonce)| nonce).collect::>(), + spent, + "one check a spent slot, in slot order" + ); + for (state, nonce) in &checks { + assert!(passes(state, *nonce, 8)); + } + assert_eq!( + proof.rounds[last].nonces.ood, 0, + "the last round has no out-of-domain grind" + ); + } + + #[test] + fn zero_bits_grinds_nothing() { + let proof = run(6, 2).unwrap(); + for round in &proof.rounds { + assert_eq!(round.nonces, RoundNonces::default()); + } + } + + #[test] + fn a_forged_folding_nonce_is_rejected() { + assert_eq!( + forge_uniform(1, "folding"), + Err(Error::GrindingRejected { bits: 8 }) + ); + } + + #[test] + fn a_forged_out_of_domain_nonce_is_rejected() { + assert_eq!( + forge_uniform(0, "ood"), + Err(Error::GrindingRejected { bits: 8 }) + ); + } + + #[test] + fn a_forged_query_nonce_is_rejected() { + assert_eq!( + forge_uniform(0, "query"), + Err(Error::GrindingRejected { bits: 8 }) + ); + } + + /// A verifier asking for bits the prover did not spend. + #[test] + fn a_proof_ground_for_fewer_bits_is_rejected() { + let num_vars = 6; + let (proof, root, domain, y) = prove_ground(num_vars, &ground(2)); + let demanding = ChainConfig { + grind: GrindBits::uniform(30), + ..config(2) + }; + + assert!(verify_ground(&proof, &root, &domain, y, &demanding, num_vars).is_err()); + } + + #[test] + fn the_bits_can_differ_per_place() { + let num_vars = 5; + let cfg = ChainConfig { + grind: GrindBits { + folding: 6, + ood: 0, + query: 8, + }, + ..config(2) + }; + let (proof, root, domain, y) = prove_ground(num_vars, &cfg); + assert_eq!(proof.rounds[0].nonces.ood, 0); + verify_ground(&proof, &root, &domain, y, &cfg, num_vars).unwrap(); + } + + // --------------------------------------------------------------- + // P2: a grind before the queries only, and the nonce layout. + // --------------------------------------------------------------- + + /// The query count reads the query grind alone, so dropping the other two + /// grinds leaves it where it was. + #[test] + fn a_query_only_grind_keeps_the_query_count() { + assert_eq!( + GrindBits::query_only(20), + GrindBits { + folding: 0, + ood: 0, + query: 20 + } + ); + for num_vars in [10, 25, 27] { + assert_eq!( + ChainConfig::with_security(2, 4, num_vars, 128, GrindBits::query_only(20)) + .num_queries, + ChainConfig::with_security(2, 4, num_vars, 128, GrindBits::uniform(20)).num_queries, + "num_vars {num_vars}" + ); + } + } + + /// Enough query bits to be a real search, under a nonce layout. + fn query_only(layout: NonceLayout) -> ChainConfig { + ChainConfig { + grind: GrindBits::query_only(8), + format: ChainFormat { + nonces: layout, + ..ChainFormat::DEFAULT + }, + ..config(2) + } + } + + /// ★ Under either layout a query-only chain grinds once a round, before its + /// queries: the other two nonces stay zero, and a wrong query nonce is + /// refused in every round. + #[test] + fn a_query_only_chain_checks_its_query_nonce_in_every_round() { + let num_vars = 9; + for layout in [NonceLayout::Three, NonceLayout::Spent] { + let cfg = query_only(layout); + let (proof, root, domain, y) = prove_ground(num_vars, &cfg); + assert!( + proof.rounds.len() > 2, + "a round that is neither first nor last" + ); + for round in &proof.rounds { + assert_eq!((round.nonces.folding, round.nonces.ood), (0, 0)); + } + // One check a round, the query's, each passing the PoW at the state + // it read. + let (verdict, checks) = verify_logged(&proof, &root, &domain, y, &cfg, num_vars); + verdict.unwrap(); + assert_eq!( + checks.iter().map(|&(_, nonce)| nonce).collect::>(), + proof + .rounds + .iter() + .map(|round| round.nonces.query) + .collect::>(), + "{layout:?}: one grind check a round, the query's" + ); + for (r, (state, nonce)) in checks.iter().enumerate() { + assert!(passes(state, *nonce, 8), "{layout:?}, round {r}"); + // A wrong query nonce, one that fails the PoW there, is refused + // by that round's check. + let mut forged = proof.clone(); + forged.rounds[r].nonces.query = failing(state, *nonce, 8); + assert_eq!( + verify_ground(&forged, &root, &domain, y, &cfg, num_vars), + Err(Error::GrindingRejected { bits: 8 }), + "{layout:?}, round {r}" + ); + } + } + } + + /// ★ A nonce the round does not spend is refused when set, under `Spent` + /// only. `Three` carries it unread: today's format, and the reason `Spent` + /// exists. Every unspent slot of a query-only chain, and the last round's + /// out-of-domain slot, which no config spends. + #[test] + fn an_unspent_nonce_is_refused_where_the_layout_does_not_carry_it() { + let num_vars = 9; + let (proof, root, domain, y) = prove_ground(num_vars, &query_only(NonceLayout::Spent)); + let rounds = proof.rounds.len(); + let mut refused = 0; + for r in 0..rounds { + for slot in ["folding", "ood"] { + let mut forged = proof.clone(); + match slot { + "folding" => forged.rounds[r].nonces.folding = 1, + _ => forged.rounds[r].nonces.ood = 1, + } + let under = |layout| { + verify_ground(&forged, &root, &domain, y, &query_only(layout), num_vars) + }; + assert_eq!( + under(NonceLayout::Spent), + Err(Error::UnspentNonce), + "round {r}, the {slot} slot" + ); + assert_eq!( + under(NonceLayout::Three), + Ok(()), + "round {r}, the {slot} slot: Three must not read it" + ); + refused += 1; + } + } + assert_eq!(refused, 2 * rounds, "two unspent slots a round"); + + // With every grind spent, the one unspent slot is the last round's + // out-of-domain nonce: refused under `Spent`, unread under `Three`. + let uniform = |layout| ChainConfig { + format: ChainFormat { + nonces: layout, + ..ChainFormat::DEFAULT + }, + ..ground(2) + }; + let (proof, root, domain, y) = prove_ground(num_vars, &uniform(NonceLayout::Spent)); + let mut forged = proof.clone(); + forged.rounds.last_mut().expect("a round").nonces.ood = 1; + let under = |layout| verify_ground(&forged, &root, &domain, y, &uniform(layout), num_vars); + assert_eq!(under(NonceLayout::Spent), Err(Error::UnspentNonce)); + assert_eq!(under(NonceLayout::Three), Ok(())); + } + + /// The field tower: a base-field domain with extension-valued codewords. + #[test] + fn the_chain_runs_over_a_field_tower() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + type ExtE = FieldElement; + + let cfg = config(2); + let num_vars = 5; + // The committed polynomial is base-field; only the point is not. + let f = Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i * 7 + 3)) + .collect(), + ) + .unwrap(); + let z: Vec = (0..num_vars).map(|i| ExtE::from(101 + i as u64)).collect(); + let y = f.evaluate_in(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let mut prover = DefaultTranscript::::new(b"tower"); + let proof = prove::(&f, &z, &commitment, &domain, &cfg, &mut prover) + .unwrap(); + + let mut verifier = DefaultTranscript::::new(b"tower"); + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut verifier, + ) + .unwrap(); + } +} diff --git a/crypto/multilinear/src/whir_commit.rs b/crypto/multilinear/src/whir_commit.rs new file mode 100644 index 000000000..5024961e8 --- /dev/null +++ b/crypto/multilinear/src/whir_commit.rs @@ -0,0 +1,1990 @@ +//! Committing a codeword by fold blocks, so each query is one Merkle opening. +//! +//! The pre-image of folded index `j` is the stride-`N/2^k` coset +//! `{ j, j + N/2^k, …, j + (2^k - 1)·N/2^k }`. + +use crypto::merkle_tree::{ + cap::{CapPolicy, CappedRoot, MAX_CAP_HEIGHT, embed_cap_arity, tree_levels}, + merkle::MerkleTree, + proof::Proof, + traits::IsMerkleTreeBackend, + utils::{level_offsets4, level_sizes4}, +}; +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{ + Error, + whir::Domain, + whir_hash::{KeccakWhir, WhirHash}, +}; + +/// 32-byte commitments, matching the rest of the prover. +/// +/// ★ **The width is the same for every [`WhirHash`]** — a keccak digest is 32 +/// bytes and an algebraic digest is four canonical Goldilocks felts, which is +/// also 32 bytes. That is what keeps a hash swap out of the proof format: every +/// type below and above this one keeps its layout, its rkyv derives and its +/// serialized length. +pub type Commitment = [u8; 32]; +type Backend = ::Backend; +type Tree = MerkleTree>; + +/// Where a chain's round-0 trees put their leaves. +/// +/// [`Natural`](Self::Natural) is today's: block `j` (the coset that folds onto +/// `j`) sits at tree position `j`. [`Coset`](Self::Coset) puts it at position +/// `bitrev_d(j)`, `d` the tree's depth. Then the leaves under any aligned +/// subtree of `2^m` positions are the blocks `j ≡ c (mod 2^(d−m))`, whose +/// values are the codeword on ONE coset of the domain: a prover can evaluate +/// them from the polynomial instead of re-encoding the whole codeword. +/// +/// Only a round-0 tree takes this order; a successor's tree is always natural. +/// A FORMAT VALUE and a verifier-side constant ([`crate::whir_chain::ChainFormat`]), +/// never read from a proof: the roots, the paths and the in-guest walk all +/// follow it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum LeafOrder { + /// Block `j` at position `j`. Today's format. + #[default] + Natural, + /// Block `j` at position `bitrev_d(j)`. + Coset, +} + +impl LeafOrder { + /// The tree position of block `index` in a tree of `2^depth` leaves. + /// + /// The map is its own inverse, so it also gives the block at a position. + /// An index past the tree is returned as it is, for the caller's range + /// check to refuse: reversing its low bits would land it inside. + pub fn position(self, index: usize, depth: usize) -> usize { + match self { + Self::Natural => index, + Self::Coset if depth == 0 || index >> depth != 0 => index, + Self::Coset => index.reverse_bits() >> (usize::BITS as usize - depth), + } + } +} + +/// A committed codeword and the tree needed to open it. +pub struct CodewordCommitment +where + FieldElement: AsBytes + Sync + Send, +{ + tree: Tree, + codeword: Codeword, + log_folding: usize, + log_domain_size: usize, + /// Where the tree puts its leaves: a round-0 tree's [`LeafOrder`], + /// natural for every other. + leaves: LeafOrder, + /// The top of the tree, when the commitment was retired and revived + /// ([`Self::retire`], [`RetiredCommitment::revive`]): the paths are then + /// read from it and from the leaves under one kept node, re-hashed from + /// the codeword. `None` on every commitment that kept its tree. + top: Option, +} + +/// Paths served from a kept tree top, and the leaves re-hashed on the host to +/// serve them — the whole cost of not keeping the bottom levels. A revived +/// commitment never builds a device tree (there is no code path for it), so +/// these two are all a phase-B opening spends on its first round's tree. +static TOP_PATH_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static TOP_LEAVES_REHASHED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +/// Nanoseconds the kept-top paths spent gathering their blocks from the +/// codeword, and re-hashing them on the host (the subtrees and their check). +static TOP_GATHER_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static TOP_REHASH_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// `BLOCK_WHIR_REHASH_SERIAL=1`: the kept-top paths re-hash their blocks one +/// at a time, as before they were re-hashed in parallel. A measurement knob +/// (the same bytes either way); read once. +#[cfg(feature = "parallel")] +fn rehash_serial() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + std::env::var("BLOCK_WHIR_REHASH_SERIAL").is_ok_and(|v| !v.is_empty() && v != "0") + }) +} + +/// `(seconds gathering, seconds re-hashing)` of the kept-top paths so far. +pub fn top_path_secs() -> (f64, f64) { + use std::sync::atomic::Ordering::Relaxed; + ( + TOP_GATHER_NANOS.load(Relaxed) as f64 / 1e9, + TOP_REHASH_NANOS.load(Relaxed) as f64 / 1e9, + ) +} + +/// The kept-top paths of codewords on the card, by route: re-hashed on the card +/// (the default), or on the host when the card's re-hash failed or +/// `LAMBDA_VM_TOP_PATHS_HOST=1` asked for it — the fallback, ≈ 16× today's host +/// cost at 8 dropped levels, which a block reports as a WARN line. +static TOP_DEVICE_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static TOP_DEVICE_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static TOP_HOST_FALLBACK_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static TOP_HOST_FALLBACK_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// `LAMBDA_VM_TOP_PATHS_HOST=1`: a codeword on the card has its kept-top paths +/// re-hashed on the host, as when the card's re-hash fails (the fallback's +/// measurement and its gate). Read once. +fn top_paths_on_host() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + std::env::var("LAMBDA_VM_TOP_PATHS_HOST").is_ok_and(|v| !v.is_empty() && v != "0") + }) +} + +/// `(calls, seconds)` of the kept-top paths of codewords on the card, re-hashed +/// on the card and on the host (the fallback): `((card calls, card s), (host +/// calls, host s))`. +pub fn top_path_routes() -> ((u64, f64), (u64, f64)) { + use std::sync::atomic::Ordering::Relaxed; + ( + ( + TOP_DEVICE_CALLS.load(Relaxed), + TOP_DEVICE_NANOS.load(Relaxed) as f64 / 1e9, + ), + ( + TOP_HOST_FALLBACK_CALLS.load(Relaxed), + TOP_HOST_FALLBACK_NANOS.load(Relaxed) as f64 / 1e9, + ), + ) +} + +/// `(open_many calls served from a kept top, leaves re-hashed for them)`. +pub fn top_path_counts() -> (u64, u64) { + use std::sync::atomic::Ordering::Relaxed; + ( + TOP_PATH_CALLS.load(Relaxed), + TOP_LEAVES_REHASHED.load(Relaxed), + ) +} + +/// How many bottom levels a retired commitment's tree drops +/// ([`CodewordCommitment::retire`]): `device` for a codeword the card holds, +/// whose dropped levels are re-hashed on the card when a path needs them, and +/// `host` for one on the host, re-hashed there. Each level dropped halves the +/// kept top and doubles the leaves a path re-hashes. Clamped per tree to its +/// depth less the tallest cap it may be asked for. +/// +/// ★ A PROVER-SIDE MEMORY CHOICE, NOT A FORMAT ONE (see [`TreeTop`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TreeDrop { + pub device: usize, + pub host: usize, +} + +impl TreeDrop { + /// The same depth wherever the codeword is. + pub const fn uniform(levels: usize) -> Self { + Self { + device: levels, + host: levels, + } + } + + /// The levels to drop from a tree `depth` deep whose tallest cap is + /// `tallest_cap`, its codeword on the card or not: never into the cap. + pub fn levels(self, on_device: bool, depth: usize, tallest_cap: usize) -> usize { + let wanted = if on_device { self.device } else { self.host }; + wanted.min(depth.saturating_sub(tallest_cap)) + } + + /// [`Self::levels`] for a tree of `arity` children per node over + /// `2^depth` leaves, `tallest_cap` counting that arity's levels. At arity + /// 4 the drop counts 4-ary levels, half the binary count, so a dropped + /// block keeps the leaves it has at arity 2 (8 → 4: 256 leaves); and it + /// stops where a kept node covers `4^dropped` whole leaves, which an odd + /// tree's two-node top does not. + pub fn levels_at( + self, + on_device: bool, + depth: usize, + tallest_cap: usize, + arity: usize, + ) -> usize { + if arity != 4 { + return self.levels(on_device, depth, tallest_cap); + } + let wanted = if on_device { self.device } else { self.host } / 2; + wanted + .min(depth / 2) + .min(tree_levels(depth, 4).saturating_sub(tallest_cap)) + } +} + +/// The cap height of one tree over `2^depth` leaves of `arity` children per +/// node, opened `openings` times under `policy`: `policy.height` over the +/// tree's levels (4-ary levels at arity 4), and at arity 4 no taller than a cap +/// of `2^MAX_CAP_HEIGHT` nodes. At arity 2 it is `policy.height(openings, +/// depth)`, today's. Prover and verifier both derive it from public numbers. +pub fn tree_cap_height(policy: CapPolicy, openings: usize, depth: usize, arity: usize) -> usize { + let c = policy.height(openings, tree_levels(depth, arity)); + if arity == 4 { + c.min(MAX_CAP_HEIGHT / 2) + } else { + c + } +} + +/// Nodes a kept top holds: the levels above the `dropped` bottom ones, root +/// first — `2^(depth − dropped + 1) − 1` at arity 2, the matching prefix of +/// the arity-4 layout (`MerkleTree`'s, top-down) at arity 4. +fn kept_top_nodes(num_leaves: usize, depth: usize, dropped: usize, arity: usize) -> usize { + if arity == 4 { + level_sizes4(num_leaves)[dropped..].iter().sum() + } else { + (1usize << (depth - dropped + 1)) - 1 + } +} + +impl std::fmt::Display for TreeDrop { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "card {} · host {}", self.device, self.host) + } +} + +/// What a retired commitment keeps of its tree: the heap prefix down to the +/// level whose nodes each cover `2^dropped` leaves (`4^dropped` at arity 4, +/// whose prefix is the arity-4 layout's top levels). +/// +/// ★ A PROVER-SIDE MEMORY CHOICE, NOT A FORMAT ONE. The paths it answers are +/// the paths of the whole tree, byte for byte: the levels it dropped are +/// rebuilt from the codeword's queried cosets, and the rebuilt node is checked +/// against the kept one before any path leaves. +#[derive(Clone, Debug)] +pub struct TreeTop { + /// `2^(depth − dropped + 1) − 1` nodes, root first (the host layout), or + /// the arity-4 layout's top levels. + nodes: Vec, + /// Levels dropped, in the tree's own levels (4-ary at arity 4). + dropped: usize, +} + +impl TreeTop { + /// Bytes held. + pub fn bytes(&self) -> usize { + self.nodes.len() * core::mem::size_of::() + } +} + +/// A commitment whose codeword was let go after its root was taken: the root, +/// the top of its tree and its shape. [`Self::revive`] makes it openable again +/// once the caller has recomputed the same codeword. +#[derive(Clone, Debug)] +pub struct RetiredCommitment { + root: Commitment, + top: TreeTop, + log_folding: usize, + log_domain_size: usize, + leaves: LeafOrder, +} + +impl RetiredCommitment { + pub fn root(&self) -> Commitment { + self.root + } + + /// Host bytes this keeps of the tree. + pub fn tree_bytes(&self) -> usize { + self.top.bytes() + } + + /// The bottom levels its tree dropped. + pub fn dropped(&self) -> usize { + self.top.dropped + } + + pub fn log_domain_size(&self) -> usize { + self.log_domain_size + } + + /// The commitment again, over `codeword` — which must be the committed + /// codeword. A wrong one is caught when a path is read: its leaves then do + /// not hash to the kept node ([`Error::RecomputedCodewordMismatch`]), so no + /// path of a codeword the root does not bind can leave. + pub fn revive( + self, + codeword: Codeword, + ) -> Result, Error> + where + FieldElement: AsBytes + Sync + Send, + { + if codeword.len() != 1usize << self.log_domain_size { + return Err(Error::CodewordTooShort { + coefficients: codeword.len(), + domain: 1usize << self.log_domain_size, + }); + } + Ok(CodewordCommitment { + tree: Tree::::from_root(self.root), + codeword, + log_folding: self.log_folding, + log_domain_size: self.log_domain_size, + leaves: self.leaves, + top: Some(self.top), + }) + } +} + +/// Where a commitment's codeword lives. +/// +/// A device commit leaves it there and the chain folds it there: it is the +/// biggest array the proof holds, and all the host needs of it is the handful +/// of values a query opens. +#[derive(Debug)] +pub enum Codeword { + Host(Vec>), + Device(crate::gpu::DeviceCodeword), + /// Not held: the stack it encodes ([`SourceCodeword`]). + Source(SourceCodeword), +} + +/// A round-0 codeword not held at all: the stack it encodes, from which each +/// opening evaluates the cosets its queries fall in (`crate::gpu::CosetSource`). +/// Only a commitment in the coset leaf order, revived for its openings, holds +/// one. +pub struct SourceCodeword { + source: crate::gpu::CosetSource, + /// The blocks the last paths gathered, by leaf index: an opening reads its + /// paths and its values in two calls, and both come from one evaluation. + opened: std::sync::Mutex>)>>, +} + +impl SourceCodeword { + pub fn new(source: crate::gpu::CosetSource) -> Self { + Self { + source, + opened: std::sync::Mutex::new(Vec::new()), + } + } + + pub(crate) fn source(&self) -> &crate::gpu::CosetSource { + &self.source + } + + fn keep(&self, blocks: Vec<(usize, Vec>)>) { + if let Ok(mut held) = self.opened.lock() { + *held = blocks; + } + } + + /// The kept blocks of `indices`, or `None` when the last paths did not + /// gather one of them. + fn opened(&self, indices: &[usize]) -> Option>>> { + let held = self.opened.lock().ok()?; + indices + .iter() + .map(|index| { + held.iter() + .find(|(leaf, _)| leaf == index) + .map(|(_, values)| values.clone()) + }) + .collect() + } +} + +impl std::fmt::Debug for SourceCodeword { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SourceCodeword") + .field("elements", &self.source.elements()) + .finish() + } +} + +impl Codeword { + pub fn len(&self) -> usize { + match self { + Self::Host(values) => values.len(), + Self::Device(device) => device.elements(), + Self::Source(source) => source.source.elements(), + } + } + + /// Whether the codeword's values come from a device: held there, or + /// evaluated there from the stack. + pub fn on_device(&self) -> bool { + !matches!(self, Self::Host(_)) + } + + /// The stack it is evaluated from, when it is not held. + pub fn source(&self) -> Option<&SourceCodeword> { + match self { + Self::Source(source) => Some(source), + _ => None, + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The values on the host, when they are there. + pub fn host(&self) -> Option<&[FieldElement]> { + match self { + Self::Host(values) => Some(values), + Self::Device(_) | Self::Source(_) => None, + } + } + + /// The device holding them, when one does. + pub fn device(&self) -> Option<&crate::gpu::DeviceCodeword> { + match self { + Self::Host(_) | Self::Source(_) => None, + Self::Device(device) => Some(device), + } + } +} + +/// One opened block, with its authentication path. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct CosetOpening { + /// The `2^k` codeword values, in coset order. + pub values: Vec>, + pub proof: Proof, +} + +/// A node's sibling in the host heap layout (root at 0, children `2i + 1` and +/// `2i + 2`): a right child's is the node before it, a left child's the one +/// after. +fn sibling(pos: usize) -> usize { + if pos.is_multiple_of(2) { + pos - 1 + } else { + pos + 1 + } +} + +/// The codeword positions that fold onto `index`. +/// +/// `log_folding` is `k`; the coset has `2^k` entries at stride `N / 2^k`. +pub fn coset_of(index: usize, log_domain_size: usize, log_folding: usize) -> Vec { + let stride = 1usize << (log_domain_size - log_folding); + (0..(1usize << log_folding)) + .map(|t| index + t * stride) + .collect() +} + +/// Hand-written: the Merkle tree behind it is not `Debug`, and its nodes are +/// not what a reader of this type wants to see anyway. +/// Where a codeword position lives in the committed blocks. +/// +/// Blocks are strided, so position `p` sits in leaf `p mod num_leaves` at slot +/// `p / num_leaves`. The inverse of [`coset_of`]. +pub fn leaf_and_slot(position: usize, num_leaves: usize) -> (usize, usize) { + (position % num_leaves, position / num_leaves) +} + +impl std::fmt::Debug for CodewordCommitment +where + FieldElement: AsBytes + Sync + Send, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CodewordCommitment") + .field("root", &self.root()) + .field("leaves", &self.num_leaves()) + .field("log_folding", &self.log_folding) + .field("log_domain_size", &self.log_domain_size) + .finish() + } +} + +impl CodewordCommitment +where + FieldElement: AsBytes + Sync + Send, +{ + /// Groups `codeword` into fold blocks and Merkle-commits them. + pub fn new(codeword: &[FieldElement], log_folding: usize) -> Result { + Self::from_codeword(codeword.to_vec(), log_folding) + } + + /// The same, taking the codeword. The commitment keeps it — the prover + /// folds from here rather than encoding a second time — so a caller that + /// has no further use for its own copy hands it over instead of paying for + /// a second one. + pub fn from_codeword( + codeword: Vec>, + log_folding: usize, + ) -> Result { + if !codeword.len().is_power_of_two() { + return Err(Error::NotPowerOfTwo(codeword.len())); + } + let log_domain_size = codeword.len().trailing_zeros() as usize; + if log_folding > log_domain_size { + return Err(Error::ColumnTallerThanStack { + column_vars: log_folding, + n_stack: log_domain_size, + }); + } + + if let Some(nodes) = crate::gpu::commit_tree_ext3(&codeword, log_folding, H::DEVICE) { + let tree = Tree::::from_precomputed_nodes(nodes).ok_or(Error::EmptyPolynomial)?; + return Ok(Self { + tree, + codeword: Codeword::Host(codeword), + log_folding, + log_domain_size, + leaves: LeafOrder::Natural, + top: None, + }); + } + Self::from_codeword_on_host(codeword, log_folding) + } + + /// The same with the leaves hashed here, whatever a device would have done + /// — the reference the kernel is checked against. + pub fn from_codeword_on_host( + codeword: Vec>, + log_folding: usize, + ) -> Result { + Self::from_codeword_on_host_in(codeword, log_folding, LeafOrder::Natural) + } + + /// [`from_codeword_on_host`](Self::from_codeword_on_host) with the leaves + /// placed in `leaves`' order: the hashes are the same, their positions are + /// not. + pub fn from_codeword_on_host_in( + codeword: Vec>, + log_folding: usize, + leaves: LeafOrder, + ) -> Result { + if !codeword.len().is_power_of_two() { + return Err(Error::NotPowerOfTwo(codeword.len())); + } + let log_domain_size = codeword.len().trailing_zeros() as usize; + if log_folding > log_domain_size { + return Err(Error::ColumnTallerThanStack { + column_vars: log_folding, + n_stack: log_domain_size, + }); + } + let num_leaves = 1usize << (log_domain_size - log_folding); + let block = 1usize << log_folding; + // One reused buffer per worker: a block is the leaf the backend hashes, + // and the coset is strided, so it has to be gathered somewhere. + let hash_leaf = |buffer: &mut Vec>, j: usize| { + buffer.clear(); + buffer.extend((0..block).map(|t| codeword[j + t * num_leaves].clone())); + Backend::::hash_data(buffer) + }; + #[cfg(feature = "parallel")] + let hashed: Vec<_> = (0..num_leaves) + .into_par_iter() + .map_init(|| Vec::with_capacity(block), hash_leaf) + .collect(); + #[cfg(not(feature = "parallel"))] + let hashed: Vec<_> = { + let mut buffer = Vec::with_capacity(block); + (0..num_leaves).map(|j| hash_leaf(&mut buffer, j)).collect() + }; + + // Position `p` holds block `π(p)`: the order is its own inverse. + let depth = log_domain_size - log_folding; + let hashed = match leaves { + LeafOrder::Natural => hashed, + LeafOrder::Coset => (0..num_leaves) + .map(|p| hashed[leaves.position(p, depth)]) + .collect(), + }; + let tree = Tree::::build_from_hashed_leaves(hashed).ok_or(Error::EmptyPolynomial)?; + Ok(Self { + tree, + codeword: Codeword::Host(codeword), + log_folding, + log_domain_size, + leaves, + top: None, + }) + } + + /// The same, with the codeword and the tree both already computed — a + /// device commit that handed the codeword back. The nodes carry no proof + /// of their own correctness, so the caller answers for the layout: + /// `2*num_leaves - 1` nodes, root first, leaves last. + pub fn from_precomputed( + codeword: Vec>, + nodes: Vec, + log_folding: usize, + ) -> Result { + if !codeword.len().is_power_of_two() { + return Err(Error::NotPowerOfTwo(codeword.len())); + } + let log_domain_size = codeword.len().trailing_zeros() as usize; + if log_folding > log_domain_size { + return Err(Error::ColumnTallerThanStack { + column_vars: log_folding, + n_stack: log_domain_size, + }); + } + let tree = Tree::::from_precomputed_nodes(nodes).ok_or(Error::EmptyPolynomial)?; + Ok(Self { + tree, + codeword: Codeword::Host(codeword), + log_folding, + log_domain_size, + leaves: LeafOrder::Natural, + top: None, + }) + } + + /// A commitment whose codeword stays on the device that built it. + /// + /// Only the root comes back. The tree is rebuilt there when the queries + /// are known — see [`paths`](Self::paths) — so this holds a root-only + /// tree and nothing else. + pub fn from_device( + codeword: crate::gpu::DeviceCodeword, + root: Commitment, + log_folding: usize, + ) -> Result { + Self::from_device_in(codeword, root, log_folding, LeafOrder::Natural) + } + + /// [`from_device`](Self::from_device) for a tree the card built with its + /// leaves in `leaves`' order. The codeword must say so: a tree it would + /// rebuild in another order is refused here rather than at its first path. + pub fn from_device_in( + codeword: crate::gpu::DeviceCodeword, + root: Commitment, + log_folding: usize, + leaves: LeafOrder, + ) -> Result { + if !codeword.placed_in(leaves) { + return Err(Error::DeviceFailed { + stage: "a device tree in another leaf order", + }); + } + let elements = codeword.elements(); + if !elements.is_power_of_two() { + return Err(Error::NotPowerOfTwo(elements)); + } + let log_domain_size = elements.trailing_zeros() as usize; + if log_folding > log_domain_size { + return Err(Error::ColumnTallerThanStack { + column_vars: log_folding, + n_stack: log_domain_size, + }); + } + Ok(Self { + tree: Tree::::from_root(root), + codeword: Codeword::Device(codeword), + log_folding, + log_domain_size, + leaves, + top: None, + }) + } + + pub fn root(&self) -> Commitment { + self.tree.root + } + + pub fn num_leaves(&self) -> usize { + 1usize << (self.log_domain_size - self.log_folding) + } + + /// `log2(num_leaves)`: the index bits, whatever the arity (siblings on a + /// full path at arity 2). + pub fn depth(&self) -> usize { + self.log_domain_size - self.log_folding + } + + /// Children per node of this commitment's tree. + pub fn arity() -> usize { + as IsMerkleTreeBackend>::ARITY + } + + /// The tree's levels: [`Self::depth`] at arity 2, `⌈depth / 2⌉` at arity + /// 4. Cap heights and kept tops count these. + pub fn levels(&self) -> usize { + tree_levels(self.depth(), Self::arity()) + } + + /// Siblings on a full authentication path: one per level at arity 2, + /// three per level at arity 4. + pub fn path_len(&self) -> usize { + (Self::arity() - 1) * self.levels() + } + + pub fn log_folding(&self) -> usize { + self.log_folding + } + + pub fn log_domain_size(&self) -> usize { + self.log_domain_size + } + + /// Where this commitment's tree puts its leaves. + pub fn leaves(&self) -> LeafOrder { + self.leaves + } + + /// The tree positions of blocks `indices`, in their order. + fn positions<'i>(&self, indices: &'i [usize]) -> std::borrow::Cow<'i, [usize]> { + match self.leaves { + LeafOrder::Natural => std::borrow::Cow::Borrowed(indices), + LeafOrder::Coset => std::borrow::Cow::Owned( + indices + .iter() + .map(|&index| self.leaves.position(index, self.depth())) + .collect(), + ), + } + } + + /// The committed codeword, in domain order. + /// + /// The prover folds from here rather than encoding a second time — on a + /// real trace that second NTT is the most expensive thing in the proof + /// after the sumcheck, and it computes something already in memory. + pub fn codeword(&self) -> &Codeword { + &self.codeword + } + + /// Lets the codeword go and keeps the root and the tree's top: every level + /// but the bottom `drop` levels for where the codeword is ([`TreeDrop`]) — + /// fewer when the tree is shallower, and never into the Merkle cap `cap` + /// may ask this tree for, which is read from the kept levels. What the + /// block prover holds between committing a group and opening it. `cap` is + /// the policy at this tree's arity (`ChainFormat::cap_policy_at`). + pub fn retire( + self, + drop: TreeDrop, + cap: crypto::merkle_tree::cap::CapPolicy, + ) -> Result { + if let Some(top) = self.top { + // Already a revived one: its top is what it keeps. + return Ok(RetiredCommitment { + root: self.tree.root, + top, + log_folding: self.log_folding, + log_domain_size: self.log_domain_size, + leaves: self.leaves, + }); + } + let depth = self.depth(); + let arity = Self::arity(); + // The tallest cap the policy gives a tree this deep, whatever its + // opening count (the height is monotone in it). + let tallest_cap = tree_cap_height(cap, usize::MAX, depth, arity); + let on_device = matches!(self.codeword, Codeword::Device(_)); + let dropped = drop.levels_at(on_device, depth, tallest_cap, arity); + let keep = kept_top_nodes(self.num_leaves(), depth, dropped, arity); + let nodes = + match &self.codeword { + Codeword::Device(device) => device + .top_nodes(self.log_folding, keep, H::DEVICE) + .ok_or(Error::DeviceFailed { + stage: "retiring a commitment", + })?, + Codeword::Host(_) => { + let all = self.tree.nodes(); + if all.len() < keep { + return Err(Error::DeviceFailed { + stage: "retiring a commitment with no host tree", + }); + } + all[..keep].to_vec() + } + // Held as its stack only once revived, and a revived one + // returned its kept top above. + Codeword::Source(_) => { + return Err(Error::DeviceFailed { + stage: "retiring a commitment held as its stack", + }); + } + }; + if nodes.len() != keep || nodes.first() != Some(&self.tree.root) { + return Err(Error::RecomputedCodewordMismatch { block: 0 }); + } + Ok(RetiredCommitment { + root: self.tree.root, + top: TreeTop { nodes, dropped }, + log_folding: self.log_folding, + log_domain_size: self.log_domain_size, + leaves: self.leaves, + }) + } + + /// The paths of tree positions `indices` from the kept top: each queried + /// block of `2^dropped` leaves is re-hashed — on the card when the + /// codeword is there ([`Self::card_top_paths`]), else, or when the card's + /// re-hash fails, on the host ([`Self::host_top_paths`]) — its subtree's + /// root is checked against the kept node, and the path continues through + /// the kept levels. Either route gives the same paths, byte for byte. + fn top_paths(&self, top: &TreeTop, indices: &[usize]) -> Result>, Error> { + use std::sync::atomic::Ordering::Relaxed; + let num_leaves = self.num_leaves(); + if let Some(&bad) = indices.iter().find(|index| **index >= num_leaves) { + return Err(Error::QueryOutOfRange { + index: bad, + bound: num_leaves, + }); + } + if let Codeword::Source(source) = &self.codeword { + let started = std::time::Instant::now(); + let paths = self.source_top_paths(source, top, indices); + TOP_DEVICE_CALLS.fetch_add(1, Relaxed); + TOP_DEVICE_NANOS.fetch_add(started.elapsed().as_nanos() as u64, Relaxed); + return paths; + } + let on_device = matches!(self.codeword, Codeword::Device(_)); + if on_device && !top_paths_on_host() { + let started = std::time::Instant::now(); + if let Some(paths) = self.card_top_paths(top, indices) { + TOP_DEVICE_CALLS.fetch_add(1, Relaxed); + TOP_DEVICE_NANOS.fetch_add(started.elapsed().as_nanos() as u64, Relaxed); + return paths; + } + } + let started = std::time::Instant::now(); + let paths = self.host_top_paths(top, indices); + if on_device { + TOP_HOST_FALLBACK_CALLS.fetch_add(1, Relaxed); + TOP_HOST_FALLBACK_NANOS.fetch_add(started.elapsed().as_nanos() as u64, Relaxed); + } + paths + } + + /// [`Self::top_paths`] on the card: the queried blocks' subtrees hashed and + /// built where the codeword lies (`DeviceCodeword::block_subtrees`), and + /// only their nodes brought home. `None` when the card could not (the + /// caller falls back to the host); a block whose rebuilt root is not the + /// kept node is refused, as on the host. + fn card_top_paths( + &self, + top: &TreeTop, + indices: &[usize], + ) -> Option>, Error>> { + let Codeword::Device(device) = &self.codeword else { + return None; + }; + self.card_top_paths_over(device, top, indices) + } + + /// [`Self::card_top_paths`] over `device`, this commitment's codeword + /// (or, for one held as its stack, the codeword re-encoded from it). + fn card_top_paths_over( + &self, + device: &crate::gpu::DeviceCodeword, + top: &TreeTop, + indices: &[usize], + ) -> Option>, Error>> { + let dropped = top.dropped; + let block_bits = self.block_bits(top); + let span = 1usize << block_bits; + let mut blocks: Vec = indices.iter().map(|index| index >> block_bits).collect(); + blocks.sort_unstable(); + blocks.dedup(); + let nodes = + device.block_subtrees(self.log_folding, &blocks, dropped, H::DEVICE, self.leaves)?; + let leaves = (blocks.len() << block_bits).next_power_of_two().max(2); + if nodes.len() != Self::gathered_nodes(leaves) { + return None; + } + TOP_PATH_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + TOP_LEAVES_REHASHED.fetch_add( + (blocks.len() << block_bits) as u64, + std::sync::atomic::Ordering::Relaxed, + ); + // Block `k` of the gathered leaves is the `k`-th queried block in + // position order. + let slots: Vec<(usize, usize)> = indices + .iter() + .map(|&index| { + let k = blocks + .binary_search(&(index >> block_bits)) + .expect("every queried block was gathered"); + (k, index & (span - 1)) + }) + .collect(); + Some(self.gathered_paths(top, &nodes, leaves, &blocks, indices, &slots)) + } + + /// A block's binary levels under its kept node: `dropped`, or twice that + /// at arity 4. + fn block_bits(&self, top: &TreeTop) -> usize { + if Self::arity() == 4 { + 2 * top.dropped + } else { + top.dropped + } + } + + /// Nodes of one tree over `leaves` gathered leaves in this arity's layout: + /// the binary heap, or the arity-4 levels top-down. + fn gathered_nodes(leaves: usize) -> usize { + if Self::arity() == 4 { + level_sizes4(leaves).iter().sum() + } else { + 2 * leaves - 1 + } + } + + /// The paths of tree positions `indices` from one tree over gathered + /// blocks (`nodes`, `leaves` leaves with the padding): gathered block `k` + /// is the kept block at position `blocks[k]`, and query `i` is leaf + /// `slots[i].1` of gathered block `slots[i].0`. Every gathered block's + /// root is checked against its kept node before a path leaves. + fn gathered_paths( + &self, + top: &TreeTop, + nodes: &[Commitment], + leaves: usize, + blocks: &[usize], + indices: &[usize], + slots: &[(usize, usize)], + ) -> Result>, Error> { + let arity = Self::arity(); + let dropped = top.dropped; + let block_bits = self.block_bits(top); + let span = 1usize << block_bits; + let sizes4 = level_sizes4(leaves); + let offsets4 = level_offsets4(&sizes4); + if nodes.len() != Self::gathered_nodes(leaves) { + return Err(Error::RecomputedCodewordMismatch { block: 0 }); + } + let frontier = self.frontier(top); + // The blocks are `span`-aligned among the gathered leaves, so block + // `k`'s root sits `dropped` levels up, `k`-th on its level. + let roots_at = if arity == 4 { + offsets4[dropped] + } else { + (leaves >> dropped) - 1 + }; + for (k, &block) in blocks.iter().enumerate() { + if top.nodes.get(frontier + block) != Some(&nodes[roots_at + k]) { + return Err(Error::RecomputedCodewordMismatch { block }); + } + } + let pad = Backend::::padding_node(); + indices + .iter() + .zip(slots) + .map(|(&index, &(k, slot))| { + let block = index >> block_bits; + if blocks.get(k) != Some(&block) || slot != index & (span - 1) { + return Err(Error::QueryOutOfRange { + index, + bound: self.num_leaves(), + }); + } + let mut merkle_path = Vec::with_capacity(self.path_len()); + let leaf = (k << block_bits) + slot; + if arity == 4 { + // Inside an aligned block every group is whole; the + // padding arm is the layout's rule, not a case reached. + let mut pos = leaf; + for level in 0..dropped { + let first = pos / 4 * 4; + for c in (first..first + 4).filter(|&c| c != pos) { + let node = if c < sizes4[level] { + Some(nodes[offsets4[level] + c]) + } else { + pad + }; + merkle_path + .push(node.ok_or(Error::RecomputedCodewordMismatch { block })?); + } + pos /= 4; + } + } else { + let mut pos = (leaves - 1) + leaf; + for _ in 0..dropped { + merkle_path.push(nodes[sibling(pos)]); + pos = (pos - 1) / 2; + } + } + self.push_kept_siblings(top, block, &mut merkle_path)?; + Ok(Proof { merkle_path }) + }) + .collect() + } + + /// [`Self::top_paths`] of a commitment held as its stack: each queried + /// coset evaluated from the polynomial and hashed on the card, its root + /// checked against the kept node, and the queried blocks' values kept for + /// the opening that asked. A plan the kernels do not take, or a card that + /// declines, re-encodes the codeword from the stack (counted) and takes + /// the card's re-hash over it. + fn source_top_paths( + &self, + source: &SourceCodeword, + top: &TreeTop, + indices: &[usize], + ) -> Result>, Error> { + let depth = self.depth(); + let block_bits = self.block_bits(top); + let wanted: Vec = indices + .iter() + .map(|&index| self.leaves.position(index, depth)) + .collect(); + if let Some(opened) = + source + .source + .openings::(self.log_folding, block_bits, &wanted, H::DEVICE) + { + TOP_PATH_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + TOP_LEAVES_REHASHED.fetch_add( + (opened.positions.len() << block_bits) as u64, + std::sync::atomic::Ordering::Relaxed, + ); + let paths = self.gathered_paths( + top, + &opened.nodes, + opened.leaves, + &opened.positions, + indices, + &opened.slots, + )?; + source.keep(wanted.into_iter().zip(opened.values).collect()); + return Ok(paths); + } + let device = source.source.encode().ok_or(Error::DeviceFailed { + stage: "re-encoding a round-0 codeword from its stack", + })?; + let paths = + self.card_top_paths_over(&device, top, indices) + .ok_or(Error::DeviceFailed { + stage: "re-hashing a re-encoded round-0 codeword", + })??; + let values = device + .cosets::(&wanted, self.num_leaves(), 1usize << self.log_folding) + .ok_or(Error::DeviceFailed { + stage: "gathering a re-encoded round-0 codeword", + })?; + source.keep(wanted.into_iter().zip(values).collect()); + Ok(paths) + } + + /// [`Self::top_paths`] on the host: the leaves of each queried block are + /// gathered from the codeword and hashed here. + fn host_top_paths( + &self, + top: &TreeTop, + indices: &[usize], + ) -> Result>, Error> { + let num_leaves = self.num_leaves(); + // A block is the `span` leaves under one kept frontier node: `2^d` at + // arity 2, `4^d` at arity 4. + let block_bits = if Self::arity() == 4 { + 2 * top.dropped + } else { + top.dropped + }; + let span = 1usize << block_bits; + let mut blocks: Vec = indices.iter().map(|index| index >> block_bits).collect(); + blocks.sort_unstable(); + blocks.dedup(); + let leaves: Vec = blocks + .iter() + .flat_map(|block| (block << block_bits)..((block + 1) << block_bits)) + .collect(); + let gathering = std::time::Instant::now(); + // A position's leaf is the block the order puts there. + let values = self.gather(&self.positions(&leaves))?; + TOP_GATHER_NANOS.fetch_add( + gathering.elapsed().as_nanos() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + let rehashing = std::time::Instant::now(); + TOP_PATH_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + TOP_LEAVES_REHASHED.fetch_add(leaves.len() as u64, std::sync::atomic::Ordering::Relaxed); + let frontier = self.frontier(top); + // Each queried block's subtree depends on its own leaves alone, so the + // blocks are re-hashed in parallel; the subtrees stay in block order + // and the first block (in that order) whose root is not the kept node + // is the one refused, as a serial walk would. + let rehash = |k: usize| -> Result, Error> { + let block = blocks[k]; + let hashed: Vec = values[k * span..(k + 1) * span] + .iter() + .map(Backend::::hash_data) + .collect(); + let subtree = + Tree::::build_from_hashed_leaves(hashed).ok_or(Error::EmptyPolynomial)?; + if top.nodes.get(frontier + block) != Some(&subtree.root) { + return Err(Error::RecomputedCodewordMismatch { block }); + } + Ok(subtree) + }; + #[cfg(feature = "parallel")] + let rehashed: Vec, Error>> = if rehash_serial() { + (0..blocks.len()).map(rehash).collect() + } else { + (0..blocks.len()).into_par_iter().map(rehash).collect() + }; + #[cfg(not(feature = "parallel"))] + let rehashed: Vec, Error>> = (0..blocks.len()).map(rehash).collect(); + let subtrees = rehashed.into_iter().collect::, _>>()?; + TOP_REHASH_NANOS.fetch_add( + rehashing.elapsed().as_nanos() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + indices + .iter() + .map(|&index| { + let block = index >> block_bits; + let at = blocks + .binary_search(&block) + .map_err(|_| Error::QueryOutOfRange { + index, + bound: num_leaves, + })?; + let mut merkle_path = subtrees[at] + .get_proof_by_pos(index & (span - 1)) + .ok_or(Error::QueryOutOfRange { + index, + bound: num_leaves, + })? + .merkle_path; + self.push_kept_siblings(top, block, &mut merkle_path)?; + Ok(Proof { merkle_path }) + }) + .collect() + } + + /// Where the kept top's frontier level (the nodes over the dropped + /// blocks) starts in its node vector. + fn frontier(&self, top: &TreeTop) -> usize { + if Self::arity() == 4 { + level_offsets4(&level_sizes4(self.num_leaves()))[top.dropped] + } else { + (1usize << (self.depth() - top.dropped)) - 1 + } + } + + /// The siblings of frontier node `block`'s path up the kept levels, read + /// from the top: one per level at arity 2; at arity 4 the three other + /// children of its group in child order, the padding digest where the + /// group is short (an odd tree's top), as `MerkleTree`'s own paths. + fn push_kept_siblings( + &self, + top: &TreeTop, + block: usize, + path: &mut Vec, + ) -> Result<(), Error> { + let missing = || Error::RecomputedCodewordMismatch { block }; + if Self::arity() != 4 { + let mut pos = self.frontier(top) + block; + while pos != 0 { + path.push(*top.nodes.get(sibling(pos)).ok_or_else(missing)?); + pos = (pos - 1) / 2; + } + return Ok(()); + } + let sizes = level_sizes4(self.num_leaves()); + let offsets = level_offsets4(&sizes); + let pad = Backend::::padding_node(); + let mut pos = block; + for level in top.dropped..sizes.len() - 1 { + let first = pos / 4 * 4; + for c in (first..first + 4).filter(|&c| c != pos) { + let node = if c < sizes[level] { + top.nodes.get(offsets[level] + c).copied() + } else { + pad + }; + path.push(node.ok_or_else(missing)?); + } + pos /= 4; + } + Ok(()) + } + + /// The fold blocks of `indices`, from wherever the codeword is. + fn gather(&self, indices: &[usize]) -> Result>>, Error> { + let num_leaves = self.num_leaves(); + match &self.codeword { + Codeword::Host(values) => Ok(indices + .iter() + .map(|index| { + coset_of(*index, self.log_domain_size, self.log_folding) + .into_iter() + .map(|p| values[p].clone()) + .collect() + }) + .collect()), + Codeword::Device(device) => device + .cosets(indices, num_leaves, 1usize << self.log_folding) + .ok_or(Error::QueryOutOfRange { + index: 0, + bound: num_leaves, + }), + Codeword::Source(source) => source.opened(indices).ok_or(Error::QueryOutOfRange { + index: 0, + bound: num_leaves, + }), + } + } + + /// Opens every block a round asks for. + /// + /// One call rather than one per query: on a device the blocks are gathered + /// in a single pass over the codeword, and a round asks for a hundred of + /// them. + pub fn open_many(&self, indices: &[usize]) -> Result>, Error> { + self.open_many_capped(indices, 0, false) + } + + /// [`open_many`](Self::open_many) under a Merkle cap of height + /// `cap_height` (the owner-path encoding, `crypto::merkle_tree::cap`). + /// + /// Every path is cut to `depth − cap_height` siblings. When `owner` is + /// set, this call's first opening is the tree's first opening in proof + /// order and carries the tree's cap (`2^cap_height` nodes) after its + /// siblings. At `cap_height = 0` this is exactly `open_many`, whatever + /// `owner` says. + /// + /// On a device the cap is read from the tree the paths are gathered from, + /// inside the same rebuild, so it costs no extra tree build. + pub fn open_many_capped( + &self, + indices: &[usize], + cap_height: usize, + owner: bool, + ) -> Result>, Error> { + let num_leaves = self.num_leaves(); + // ★ ONE CALL, ONE DEVICE TREE REBUILD. Counted rather than inferred: + // `whir_round::prove` opens the current commitment AND its successor, + // so a non-final round passes here twice, and `check_closure`'s arm F + // asserts the total against the rounds that produced it. + crate::whir_split::bump(&crate::whir_split::REBUILD_CALLS); + // ⛔ THE SPLIT IS HERE AND NOT INSIDE `paths()`. On the device arm + // `paths()` is a range check, ONE device call and a `map` into `Proof`, + // so splitting inside it would weigh the rebuild against host + // bookkeeping and read ~100% every time. What competes with the rebuild + // is the COSET GATHER, which is the next statement, not a nested one. + let __wq_tree = crate::whir_split::mark(); + let proofs = self.paths_capped(indices, cap_height, owner)?; + crate::whir_split::add(&crate::whir_split::TREE_REBUILD, __wq_tree); + + let block = 1usize << self.log_folding; + let __wq_gather = crate::whir_split::mark(); + let blocks: Vec>> = match &self.codeword { + Codeword::Host(values) => indices + .iter() + .map(|index| { + coset_of(*index, self.log_domain_size, self.log_folding) + .into_iter() + .map(|p| values[p].clone()) + .collect() + }) + .collect(), + Codeword::Device(device) => { + device + .cosets(indices, num_leaves, block) + .ok_or(Error::QueryOutOfRange { + index: 0, + bound: num_leaves, + })? + } + // The paths above evaluated and kept these blocks. + Codeword::Source(source) => source.opened(indices).ok_or(Error::QueryOutOfRange { + index: 0, + bound: num_leaves, + })?, + }; + + crate::whir_split::add(&crate::whir_split::COSET_GATHER, __wq_gather); + + let __wq_assemble = crate::whir_split::mark(); + let openings: Vec> = blocks + .into_iter() + .zip(proofs) + .map(|(values, proof)| CosetOpening { values, proof }) + .collect(); + crate::whir_split::add(&crate::whir_split::OPEN_ASSEMBLE, __wq_assemble); + Ok(openings) + } + + /// One authentication path per index, from wherever the tree is. + /// + /// A codeword the device kept has no tree here — only its root. The device + /// keeps the tree past the commit, inside the codeword's promise and + /// evictable (`math_cuda::whir`: whole by default, or its leaf layer under + /// `LFM_WHIR_WHOLE_TREES=0`), and rebuilds what an eviction took. Bringing + /// it home would cost ten times the rehash, because a pageable copy of half + /// a gigabyte is the slowest thing in the commit. What a proof wants of a + /// tree is a kilobyte per query. + fn paths(&self, indices: &[usize]) -> Result>, Error> { + self.paths_at(&self.positions(indices)) + } + + /// [`paths`](Self::paths) of tree positions. + fn paths_at(&self, indices: &[usize]) -> Result>, Error> { + if let Some(top) = &self.top { + return self.top_paths(top, indices); + } + let num_leaves = self.num_leaves(); + let out_of_range = |index: usize| Error::QueryOutOfRange { + index, + bound: num_leaves, + }; + match &self.codeword { + Codeword::Device(device) if !device.placed_in(self.leaves) => { + Err(Error::DeviceFailed { + stage: "a device tree in another leaf order", + }) + } + Codeword::Device(device) => { + if let Some(&bad) = indices.iter().find(|index| **index >= num_leaves) { + return Err(out_of_range(bad)); + } + // Past the range check there is one way to fail, and it is the + // device: the paths are read there — from the kept tree, or a + // tree rebuilt — because that is where the codeword is. + Ok(device + .paths(self.log_folding, indices, H::DEVICE) + .ok_or(Error::DeviceFailed { + stage: "opening paths", + })? + .into_iter() + .map(|merkle_path| Proof { merkle_path }) + .collect()) + } + Codeword::Host(_) => indices + .iter() + .map(|index| { + self.tree + .get_proof_by_pos(*index) + .ok_or_else(|| out_of_range(*index)) + }) + .collect(), + // Held as its stack only once revived, with its kept top. + Codeword::Source(_) => Err(Error::DeviceFailed { + stage: "a stack-held commitment with no kept top", + }), + } + } + + /// [`paths`](Self::paths), cut to the cap and, for the owner, with the + /// cap appended to the first path. + fn paths_capped( + &self, + indices: &[usize], + cap_height: usize, + owner: bool, + ) -> Result>, Error> { + self.paths_capped_at(&self.positions(indices), cap_height, owner) + } + + /// [`paths_capped`](Self::paths_capped) of tree positions. + fn paths_capped_at( + &self, + indices: &[usize], + cap_height: usize, + owner: bool, + ) -> Result>, Error> { + if cap_height == 0 { + return self.paths_at(indices); + } + let depth = self.depth(); + let arity = Self::arity(); + let levels = self.levels(); + if cap_height > levels { + return Err(Error::CapEmbedFailed { + reason: "cap taller than the tree", + }); + } + let embed_failed = |_: crypto::merkle_tree::cap::CapError| Error::CapEmbedFailed { + reason: "path or cap of the wrong length", + }; + let (mut proofs, cap) = if owner && let Some(top) = &self.top { + // The cap is kept whole when it sits in the kept levels. + if cap_height > levels - top.dropped { + return Err(Error::CapEmbedFailed { + reason: "cap below the kept top of a retired tree", + }); + } + let range = if arity == 4 { + // The real nodes of the level `cap_height` below the root. + let sizes = level_sizes4(self.num_leaves()); + let level = levels - cap_height; + let start = level_offsets4(&sizes)[level]; + start..start + sizes[level] + } else { + let start = (1usize << cap_height) - 1; + start..2 * start + 1 + }; + let cap = top + .nodes + .get(range) + .ok_or(Error::CapEmbedFailed { + reason: "cap below the kept top of a retired tree", + })? + .to_vec(); + (self.top_paths(top, indices)?, Some(cap)) + } else if owner { + match &self.codeword { + Codeword::Device(device) if !device.placed_in(self.leaves) => { + return Err(Error::DeviceFailed { + stage: "a device tree in another leaf order", + }); + } + Codeword::Device(device) => { + let num_leaves = self.num_leaves(); + if let Some(&bad) = indices.iter().find(|index| **index >= num_leaves) { + return Err(Error::QueryOutOfRange { + index: bad, + bound: num_leaves, + }); + } + // ONE rebuild: the cap comes from the tree the paths are + // gathered from. + let (paths, cap) = device + .paths_and_cap(self.log_folding, indices, cap_height, H::DEVICE) + .ok_or(Error::DeviceFailed { + stage: "opening paths and cap", + })?; + let proofs: Vec> = paths + .into_iter() + .map(|merkle_path| Proof { merkle_path }) + .collect(); + (proofs, Some(cap)) + } + Codeword::Host(_) => { + let cap = self.tree.cap(cap_height).ok_or(Error::CapEmbedFailed { + reason: "the host tree has no cap at this height", + })?; + (self.paths_at(indices)?, Some(cap)) + } + Codeword::Source(_) => { + return Err(Error::DeviceFailed { + stage: "a stack-held commitment with no kept top", + }); + } + } + } else { + (self.paths_at(indices)?, None) + }; + match cap { + Some(cap) => { + let mut refs: Vec<&mut Vec> = + proofs.iter_mut().map(|p| &mut p.merkle_path).collect(); + embed_cap_arity(&mut refs, depth, &cap, arity).map_err(embed_failed)?; + } + None if arity == 4 => { + // Every path whole (three siblings a level), then cut to the + // levels under the cap. + let full = self.path_len(); + let keep = 3 * (levels - cap_height); + for proof in &mut proofs { + if proof.merkle_path.len() != full { + return Err(Error::CapEmbedFailed { + reason: "path or cap of the wrong length", + }); + } + proof.merkle_path.truncate(keep); + } + } + None => { + for proof in &mut proofs { + proof + .truncate_to_cap(depth, cap_height) + .map_err(embed_failed)?; + } + } + } + Ok(proofs) + } + + /// Opens the block that folds onto `index`. + pub fn open(&self, index: usize) -> Result, Error> { + let num_leaves = self.num_leaves(); + let proof = self.paths(&[index])?.pop().ok_or(Error::QueryOutOfRange { + index, + bound: num_leaves, + })?; + let values = match &self.codeword { + Codeword::Host(values) => coset_of(index, self.log_domain_size, self.log_folding) + .into_iter() + .map(|p| values[p].clone()) + .collect(), + // A query opens `2^log_folding` values of an array that is not + // here: that is a gather, not a codeword coming back. + Codeword::Device(device) => device + .cosets(&[index], num_leaves, 1usize << self.log_folding) + .and_then(|blocks| blocks.into_iter().next()) + .ok_or(Error::QueryOutOfRange { + index, + bound: num_leaves, + })?, + // The path above evaluated and kept it. + Codeword::Source(source) => source + .opened(&[index]) + .and_then(|blocks| blocks.into_iter().next()) + .ok_or(Error::QueryOutOfRange { + index, + bound: num_leaves, + })?, + }; + Ok(CosetOpening { values, proof }) + } +} + +/// Checks an opening against a root, under `H`'s hash. +/// +/// `H` is explicit at every call site rather than defaulted, because a free +/// function's type parameter cannot carry a default and — more to the point — +/// because "which hash authenticated this path" is the whole content of the +/// call. A verifier reading a proof under the wrong `H` gets `false` here, not +/// a different-but-plausible answer. +/// +/// `depth` is the tree's depth (`log2` of its leaf count), a verifier +/// constant: the path must be exactly that long and `index < 2^depth`. A path +/// of any other length is refused before it is folded, so a leaf hash can +/// never be compared with an internal node. +pub fn verify_opening( + root: &Commitment, + depth: usize, + index: usize, + opening: &CosetOpening, +) -> bool +where + F: IsField + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + verify_opening_capped::( + &CappedRoot::uncapped(root, depth), + index, + opening, + &opening.proof.merkle_path, + ) +} + +/// Checks an opening against one tree's authenticated cap. +/// +/// `siblings` is the opening's path with any cap split off — the whole +/// `opening.proof.merkle_path` for every opening but a tree's owner, whose +/// siblings [`CappedRoot::from_owner`] returns. It must be exactly +/// `depth − c` long and fold `hash(values)` at `index` onto +/// `cap[index >> (depth − c)]`. At `c = 0` the cap is the root, and this is +/// [`verify_opening`]. +pub fn verify_opening_capped( + check: &CappedRoot<'_, Commitment>, + index: usize, + opening: &CosetOpening, + siblings: &[Commitment], +) -> bool +where + F: IsField + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + check.verify::>(siblings, index, Backend::::hash_data(&opening.values)) +} + +/// One level of a block's fold. +/// +/// Within the block the pair for slot `t` is `(t, t + half)`, mirroring the +/// global layout one level up. Each pair sits at its own domain point: slot `t` +/// is at `j + t·(N/L)`, so the points are `g^j·η^t` with `η` a primitive +/// `L`-th root of unity. Using one `x` for the whole level is only correct when +/// the block holds a single pair. +fn fold_block_level( + values: &[FieldElement], + domain: &Domain, + position: usize, + alpha: &FieldElement, +) -> Vec> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf, + A: IsField + IsSubFieldOf, + B: IsField, +{ + let two_inv = (FieldElement::::one() + FieldElement::::one()) + .inv() + .expect("2 is invertible"); + let half = values.len() / 2; + let eta = domain + .generator() + .pow((domain.size() / values.len()) as u64); + + let mut out = Vec::with_capacity(half); + let mut x = domain.generator().pow(position as u64); + for t in 0..half { + let (a, b) = (&values[t], &values[t + half]); + let even = &two_inv * (a + b); + let x_inv = x.inv().expect("domain elements are nonzero"); + let odd = (&two_inv * x_inv) * (a - b); + // The base element on the left: the only direction the tower gives. + out.push(even + odd * alpha); + x *= η + } + out +} + +/// Folds an opened block down to the single value it contributes. +/// +/// The verifier's local mirror of [`fold_codeword_k`](crate::whir::fold_codeword_k): +/// it never sees the whole codeword, only this block, and must reach the same +/// value the prover would have. +/// +/// The values go in over `C` and come out over `N`, matching the codeword fold: +/// the committed trace's blocks are base-field, and the first fold lifts them. +pub fn fold_coset( + values: &[FieldElement], + domain: &Domain, + index: usize, + alphas: &[FieldElement], +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf, + C: IsField + IsSubFieldOf, + N: IsField, +{ + if values.len() != 1usize << alphas.len() { + return Err(Error::CodewordTooShort { + coefficients: values.len(), + domain: 1usize << alphas.len(), + }); + } + let Some((first, rest)) = alphas.split_first() else { + return Ok(values[0].clone().to_extension::()); + }; + + let mut current_domain = domain.clone(); + // The block's own position within each successively squared domain. + let mut position = index; + + let mut current = fold_block_level::(values, ¤t_domain, position, first); + current_domain = current_domain.squared()?; + position %= current_domain.size(); + + for alpha in rest { + current = fold_block_level::(¤t, ¤t_domain, position, alpha); + current_domain = current_domain.squared()?; + position %= current_domain.size(); + } + + Ok(current.into_iter().next().expect("one value remains")) +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + mle::Mle, + whir::{encode, fold_codeword_k, monomial_coefficients}, + whir_hash::KeccakWhir, + }; + + type FE = FieldElement; + + fn pseudo_codeword(num_vars: usize, log_blowup: usize, seed: u64) -> (Vec, Domain) { + let vals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from((i.wrapping_mul(6364136223846793005).wrapping_add(seed)) >> 13)) + .collect(); + let f = Mle::new(vals).unwrap(); + let domain = Domain::::new(num_vars + log_blowup).unwrap(); + let cw = encode(&monomial_coefficients(&f), &domain).unwrap(); + (cw, domain) + } + + /// A retired tree drops its codeword's own depth of levels — the card's + /// or the host's — and never into the tallest cap it may be asked for: + /// a shallow tree keeps at least its cap. + #[test] + fn the_drop_follows_the_codeword_and_stops_at_the_cap() { + let drop = TreeDrop { device: 8, host: 4 }; + assert_eq!(drop.levels(true, 23, 3), 8); + assert_eq!(drop.levels(false, 23, 3), 4); + assert_eq!(drop.levels(true, 9, 3), 6, "clamped to depth − cap"); + assert_eq!(drop.levels(false, 5, 3), 2); + assert_eq!(drop.levels(true, 2, 3), 0, "a tree no deeper than its cap"); + assert_eq!(TreeDrop::uniform(5), TreeDrop { device: 5, host: 5 }); + assert_eq!(drop.to_string(), "card 8 · host 4"); + } + + /// The clamp on a real retire: a host tree 6 deep under the tallest auto + /// cap (3) asked to drop 8 drops 3, and its paths are the whole tree's. + #[test] + fn a_shallow_tree_drops_down_to_its_cap_and_keeps_its_paths() { + let (cw, _) = pseudo_codeword(6, 2, 11); + let kept = CodewordCommitment::::new(&cw, 2).unwrap(); + assert_eq!(kept.depth(), 6); + let indices = [0usize, 5, 17, 63]; + let want: Vec<_> = indices.iter().map(|&i| kept.paths(&[i]).unwrap()).collect(); + let cap = crypto::merkle_tree::cap::CapPolicy::Auto; + let retired = CodewordCommitment::::new(&cw, 2) + .unwrap() + .retire(TreeDrop::uniform(8), cap) + .unwrap(); + assert_eq!(retired.dropped(), 6 - cap.height(usize::MAX, 6)); + assert_eq!(retired.dropped(), 3); + let revived = retired.revive::(Codeword::Host(cw)).unwrap(); + let got: Vec<_> = indices + .iter() + .map(|&i| revived.paths(&[i]).unwrap()) + .collect(); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + } + + /// ★ The card's re-hash of a retired tree's queried blocks (box only, a + /// codeword big enough for the card): at 4, 6 and 8 dropped levels (2, 3 + /// and 4 four-ary ones under Poseidon1), under every hash, the card's paths + /// are the host's and the kept tree's, byte for byte; a kept node flipped + /// is refused on the card as on the host. + #[cfg(feature = "cuda")] + #[test] + fn the_card_rehashes_a_retired_trees_blocks_to_the_host_paths() { + fn check() { + use crate::stacked_eval::StackedCommitment; + use crate::stacking::StackedLayout; + let config = crate::whir_chain::ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: crate::whir_chain::GrindBits::default(), + format: crate::whir_chain::ChainFormat::DEFAULT, + }; + let num_vars = 14; + let columns: Vec> = (0..3u64) + .map(|c| { + Mle::new( + (0..1u64 << num_vars) + .map(|i| FE::from(i.wrapping_mul(0x9e37_79b9).wrapping_add(c) >> 7)) + .collect(), + ) + .unwrap() + }) + .collect(); + let layout = StackedLayout::build(&[num_vars; 3], num_vars).unwrap(); + let borrowed = crate::stacking::borrow(&columns); + let mut kept = + StackedCommitment::::commit(layout.clone(), &borrowed, None, &config) + .unwrap(); + let indices: Vec = (0..40usize) + .map(|q| (q * 2_654_435_761) % (1 << 14)) + .collect(); + let want: Vec>> = kept + .commitments_mut() + .iter() + .map(|c| c.paths(&indices).unwrap()) + .collect(); + for d in [4usize, 6, 8] { + let retired = + StackedCommitment::::commit(layout.clone(), &borrowed, None, &config) + .unwrap() + .retire(TreeDrop::uniform(d), &config) + .unwrap(); + let mut revived = retired.revive::(&borrowed, None, &config).unwrap(); + for (c, want) in revived.commitments_mut().iter_mut().zip(&want) { + assert!( + matches!(c.codeword, Codeword::Device(_)), + "the codeword is on the card" + ); + let top = c.top.clone().expect("a kept top"); + let four = CodewordCommitment::::arity() == 4; + assert_eq!(top.dropped, if four { d / 2 } else { d }); + let card = c + .card_top_paths(&top, &indices) + .expect("the card re-hashes") + .unwrap(); + let host = c.host_top_paths(&top, &indices).unwrap(); + assert_eq!( + format!("{card:?}"), + format!("{host:?}"), + "d {d}: card vs host" + ); + assert_eq!( + format!("{card:?}"), + format!("{want:?}"), + "d {d}: card vs kept" + ); + // A kept node under a queried block flipped: refused on both. + let frontier = c.frontier(&top); + let block_bits = if four { 2 * top.dropped } else { top.dropped }; + let mut bad = top.clone(); + bad.nodes[frontier + (indices[0] >> block_bits)][0] ^= 1; + assert!(matches!( + c.card_top_paths(&bad, &indices), + Some(Err(Error::RecomputedCodewordMismatch { .. })) + )); + assert!(matches!( + c.host_top_paths(&bad, &indices), + Err(Error::RecomputedCodewordMismatch { .. }) + )); + } + } + } + check::(); + check::(); + check::(); + } + + #[test] + fn a_coset_is_a_stride_of_the_domain() { + // 3 folds over a 32-point domain: stride 4, four entries. + assert_eq!(coset_of(0, 5, 2), vec![0, 8, 16, 24]); + assert_eq!(coset_of(3, 5, 2), vec![3, 11, 19, 27]); + assert_eq!(coset_of(1, 4, 1), vec![1, 9]); + } + + #[test] + fn leaves_cover_the_codeword_exactly_once() { + let (cw, _) = pseudo_codeword(3, 2, 1); + let commitment = CodewordCommitment::::new(&cw, 2).unwrap(); + assert_eq!(commitment.num_leaves(), cw.len() / 4); + + let mut seen = vec![0usize; cw.len()]; + for j in 0..commitment.num_leaves() { + for p in coset_of(j, commitment.log_domain_size(), 2) { + seen[p] += 1; + } + } + assert!(seen.iter().all(|c| *c == 1), "coverage = {seen:?}"); + } + + #[test] + fn an_opening_verifies_against_the_root() { + let (cw, _) = pseudo_codeword(3, 2, 7); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); + let root = commitment.root(); + + for j in 0..commitment.num_leaves() { + let opening = commitment.open(j).unwrap(); + assert_eq!(opening.values.len(), 2); + assert!( + verify_opening::(&root, commitment.depth(), j, &opening), + "leaf {j}" + ); + } + } + + #[test] + fn a_tampered_opening_is_rejected() { + let (cw, _) = pseudo_codeword(3, 2, 9); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); + let root = commitment.root(); + + let mut opening = commitment.open(2).unwrap(); + opening.values[0] += FE::one(); + assert!(!verify_opening::( + &root, + commitment.depth(), + 2, + &opening + )); + } + + #[test] + fn an_opening_does_not_verify_at_another_index() { + let (cw, _) = pseudo_codeword(3, 2, 11); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); + let root = commitment.root(); + let opening = commitment.open(2).unwrap(); + assert!(!verify_opening::( + &root, + commitment.depth(), + 3, + &opening + )); + } + + #[test] + fn a_query_beyond_the_leaves_is_an_error() { + let (cw, _) = pseudo_codeword(2, 1, 3); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); + let out = commitment.num_leaves(); + assert!(matches!( + commitment.open(out).unwrap_err(), + Error::QueryOutOfRange { .. } + )); + } + + /// What makes the query phase sound: the verifier folds only the block it + /// was given and lands on the same value the prover computed from the whole + /// codeword. + #[test] + fn folding_a_coset_matches_folding_the_whole_codeword() { + for k in 1..=3usize { + let (cw, domain) = pseudo_codeword(4, 2, 40 + k as u64); + let alphas: Vec = (0..k).map(|i| FE::from(13 + i as u64)).collect(); + + let (folded, _) = fold_codeword_k(&cw, &domain, &alphas).unwrap(); + let commitment = CodewordCommitment::::new(&cw, k).unwrap(); + + for (j, expected) in folded.iter().enumerate() { + let opening = commitment.open(j).unwrap(); + let local = fold_coset(&opening.values, &domain, j, &alphas).unwrap(); + assert_eq!(local, *expected, "k={k}, leaf={j}"); + } + } + } + + #[test] + fn folding_a_block_of_the_wrong_size_is_an_error() { + let (_, domain) = pseudo_codeword(3, 2, 1); + let values = vec![FE::one(); 3]; + assert!(matches!( + fold_coset(&values, &domain, 0, &[FE::one()]).unwrap_err(), + Error::CodewordTooShort { .. } + )); + } + + #[test] + fn folding_by_zero_returns_the_single_value() { + let (cw, domain) = pseudo_codeword(3, 2, 5); + let commitment = CodewordCommitment::::new(&cw, 0).unwrap(); + assert_eq!(commitment.num_leaves(), cw.len()); + let opening = commitment.open(6).unwrap(); + assert_eq!(fold_coset(&opening.values, &domain, 6, &[]).unwrap(), cw[6]); + } + + #[test] + fn a_codeword_that_is_not_a_power_of_two_is_rejected() { + let values = vec![FE::one(); 6]; + assert!(matches!( + CodewordCommitment::::new(&values, 1).unwrap_err(), + Error::NotPowerOfTwo(6) + )); + } + + #[test] + fn leaf_and_slot_inverts_the_coset_layout() { + let (log_domain, k) = (5usize, 2usize); + let num_leaves = 1usize << (log_domain - k); + for j in 0..num_leaves { + for (slot, position) in coset_of(j, log_domain, k).into_iter().enumerate() { + assert_eq!(leaf_and_slot(position, num_leaves), (j, slot)); + } + } + } + + #[test] + fn a_position_resolves_to_the_value_it_holds() { + let (cw, _) = pseudo_codeword(3, 2, 21); + let commitment = CodewordCommitment::::new(&cw, 2).unwrap(); + let num_leaves = commitment.num_leaves(); + + for (position, value) in cw.iter().enumerate() { + let (leaf, slot) = leaf_and_slot(position, num_leaves); + let opening = commitment.open(leaf).unwrap(); + assert_eq!(opening.values[slot], *value, "position {position}"); + } + } +} diff --git a/crypto/multilinear/src/whir_commit_ood_tests.rs b/crypto/multilinear/src/whir_commit_ood_tests.rs new file mode 100644 index 000000000..f57a1d73a --- /dev/null +++ b/crypto/multilinear/src/whir_commit_ood_tests.rs @@ -0,0 +1,349 @@ +//! A stack's commit-time out-of-domain claims ([`crate::commit_ood`]) through +//! the stacked opening: the answers are the stack's values at the point, an +//! opening carrying them verifies under both chain splits and both hashes the +//! block runs, and a false, moved, missing or extra answer is refused, as is a +//! verifier that leaves them out. + +use crypto::fiat_shamir::{default_transcript::DefaultTranscript, is_transcript::IsTranscript}; +use math::{ + field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as F, + }, + traits::AsBytes, +}; + +use crate::{ + Error, + mle::Mle, + stacked_eval::{ + Claimed, CommitOod, OodClaims, OodCounts, StackedCommitment, StackedProof, ood_claims, + prove_mapped_ood, verify_ood, + }, + stacking::{StackedLayout, borrow}, + whir_chain::{ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, WhirFolds}, + whir_commit::Commitment, + whir_hash::{KeccakWhir, P1Whir, WhirHash}, +}; + +type EE = FieldElement; + +/// Mixed heights with gaps between them, seven polynomials of ten variables: +/// under `Indexed` a run of four and a run of three. +const HEIGHTS: [usize; 14] = [8, 9, 7, 9, 6, 9, 9, 8, 5, 9, 9, 4, 9, 7]; +const N_STACK: usize = 10; + +fn config(group: GroupFold) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(3).unwrap()), + group, + ..ChainFormat::DEFAULT + }, + } +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FieldElement::::from( + i.wrapping_add(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +/// A stack committed, its claims answered at `z0`, and its opening proved +/// with them. +struct Opened { + layout: StackedLayout, + roots: Vec, + domain: crate::whir::Domain, + points: Vec>, + values: Vec, + claims: OodClaims, + counts: OodCounts, + proof: StackedProof, +} + +fn open(group: GroupFold, z0: &EE, transcript: fn() -> T) -> Opened +where + H: WhirHash, + T: IsTranscript, + EE: AsBytes, +{ + let cfg = config(group); + let columns: Vec> = HEIGHTS + .iter() + .enumerate() + .map(|(i, &h)| column(h, 5 + 31 * i as u64)) + .collect(); + let points: Vec> = HEIGHTS + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| EE::from(7 + 11 * i as u64 + 3 * j as u64)) + .collect() + }) + .collect(); + let values: Vec = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p).unwrap()) + .collect(); + let layout = StackedLayout::build(&HEIGHTS, N_STACK).unwrap(); + assert_eq!(layout.num_polys(), 7, "the stack this test means"); + let stacked = + StackedCommitment::::commit(layout.clone(), &borrow(&columns), None, &cfg).unwrap(); + let (claims, counts) = ood_claims::(&layout, &borrow(&columns), None, z0).unwrap(); + let proof = prove_mapped_ood::( + &stacked, + &borrow(&columns), + None, + &Claimed::PerColumn(&points), + &values, + Some(&claims), + &cfg, + &mut transcript(), + ) + .unwrap(); + Opened { + layout, + roots: stacked.roots(), + domain: stacked.domain().clone(), + points, + values, + claims, + counts, + proof, + } +} + +fn check( + o: &Opened, + group: GroupFold, + ood: Option>, + transcript: fn() -> T, +) -> Result<(), Error> +where + H: WhirHash, + T: IsTranscript, +{ + verify_ood::( + &o.proof, + &o.layout, + &o.roots, + &Claimed::PerColumn(&o.points), + &o.values, + ood, + &o.domain, + &config(group), + &mut transcript(), + ) +} + +fn keccak() -> DefaultTranscript { + DefaultTranscript::new(b"commit-ood") +} + +fn p1() -> ::Sponge { + P1Whir::sponge() +} + +/// Every refusal the claims owe, and the honest opening's acceptance, under +/// one hash and one split. +fn every_way(group: GroupFold, transcript: fn() -> T) +where + H: WhirHash, + T: IsTranscript, +{ + let z0 = EE::from(0x0005_eed0_f00d_u64); + let o = open::(group, &z0, transcript); + let answers = &o.claims.answers; + assert_eq!(answers.len(), o.layout.num_polys()); + assert_eq!( + o.counts, + OodCounts { + on_card: 0, + on_host: HEIGHTS.len() + } + ); + let with = + |answers: &[EE]| check::(&o, group, Some(CommitOod { z0: &z0, answers }), transcript); + with(answers).unwrap_or_else(|e| panic!("{group:?}: the honest answers: {e:?}")); + + for at in 0..answers.len() { + let mut wrong = answers.clone(); + wrong[at] += EE::one(); + assert!(with(&wrong).is_err(), "{group:?}: answer {at} false"); + } + // Two answers of one run (both splits put polynomials 0..4 in one). + let mut swapped = answers.clone(); + swapped.swap(1, 2); + assert_ne!(answers[1], answers[2]); + assert!(with(&swapped).is_err(), "{group:?}: two answers swapped"); + let n = answers.len(); + assert_eq!( + with(&answers[..n - 1]), + Err(Error::CommitOodCount { + expected: n, + got: n - 1 + }), + "{group:?}: an answer missing" + ); + let mut extra = answers.clone(); + extra.push(EE::one()); + assert_eq!( + with(&extra), + Err(Error::CommitOodCount { + expected: n, + got: n + 1 + }), + "{group:?}: an answer too many" + ); + let other = &z0 + EE::one(); + assert!( + check::( + &o, + group, + Some(CommitOod { + z0: &other, + answers, + }), + transcript, + ) + .is_err(), + "{group:?}: the answers read at another point" + ); + assert!( + check::(&o, group, None, transcript).is_err(), + "{group:?}: a verifier that leaves the answers out" + ); +} + +#[test] +fn an_opening_carries_its_answers_and_refuses_them_altered_per_polynomial() { + every_way::(GroupFold::PerPoly, keccak); + every_way::(GroupFold::PerPoly, p1); +} + +/// ★ The production split: a run's polynomials share one chain, each answer +/// at its own power of `γ`, so one answer cannot pay for another's. +#[test] +fn an_opening_carries_its_answers_and_refuses_them_altered_per_run() { + every_way::(GroupFold::Indexed, keccak); + every_way::(GroupFold::Indexed, p1); +} + +/// The answers are the stacked polynomials' values at the point, and a run's +/// answers cannot cancel: answers moved between two polynomials of a run by +/// equal and opposite amounts — their sum kept — are refused. +#[test] +fn a_runs_answers_cannot_trade_against_each_other() { + let z0 = EE::from(0x1234_5678_u64); + let o = open::(GroupFold::Indexed, &z0, p1); + let columns: Vec> = HEIGHTS + .iter() + .enumerate() + .map(|(i, &h)| column(h, 5 + 31 * i as u64)) + .collect(); + let stacked = o.layout.stack(&borrow(&columns)).unwrap(); + let point = crate::whir_chain::ood_point(&z0, N_STACK); + for (poly, answer) in o.claims.answers.iter().enumerate() { + assert_eq!( + *answer, + stacked[poly].evaluate_in(&point).unwrap(), + "poly {poly}" + ); + } + let mut traded = o.claims.answers.clone(); + let delta = EE::from(99u64); + traded[0] = &traded[0] + δ + traded[1] = &traded[1] - δ + assert!( + check::( + &o, + GroupFold::Indexed, + Some(CommitOod { + z0: &z0, + answers: &traded, + }), + p1, + ) + .is_err(), + "two answers of one run traded, their sum kept" + ); +} + +/// No answer is carried at power one. Column 0's claim rides at +/// `ρ^0 = 1`; were polynomial 0's answer carried at `γ^0 = 1` too, a prover +/// could move `Δ` from the answer onto column 0's value and keep every chain's +/// claim, proving the opening from the true polynomials. With `γ^(i+1)` the +/// moved pair changes the claim and the opening is refused. +#[test] +fn an_answer_cannot_trade_against_column_zero() { + for group in [GroupFold::PerPoly, GroupFold::Indexed] { + let z0 = EE::from(0x0bad_cafe_u64); + let o = open::(group, &z0, p1); + assert_eq!( + o.layout.placements()[0].poly, + 0, + "column 0 lies in polynomial 0" + ); + let columns: Vec> = HEIGHTS + .iter() + .enumerate() + .map(|(i, &h)| column(h, 5 + 31 * i as u64)) + .collect(); + let delta = EE::from(1234u64); + let mut values = o.values.clone(); + values[0] = &values[0] + δ + let mut claims = o.claims.clone(); + claims.answers[0] = &claims.answers[0] - δ + let cfg = config(group); + let stacked = + StackedCommitment::::commit(o.layout.clone(), &borrow(&columns), None, &cfg) + .unwrap(); + // The prover's chains run over the true polynomials; only the claims + // it states are moved. + let forged = prove_mapped_ood::( + &stacked, + &borrow(&columns), + None, + &Claimed::PerColumn(&o.points), + &values, + Some(&claims), + &cfg, + &mut p1(), + ); + let accepted = forged.is_ok_and(|proof| { + verify_ood::( + &proof, + &o.layout, + &o.roots, + &Claimed::PerColumn(&o.points), + &values, + Some(CommitOod { + z0: &z0, + answers: &claims.answers, + }), + &o.domain, + &cfg, + &mut p1(), + ) + .is_ok() + }); + assert!( + !accepted, + "{group:?}: answer 0 traded against column 0's value" + ); + } +} diff --git a/crypto/multilinear/src/whir_coset_tests.rs b/crypto/multilinear/src/whir_coset_tests.rs new file mode 100644 index 000000000..06f944398 --- /dev/null +++ b/crypto/multilinear/src/whir_coset_tests.rs @@ -0,0 +1,577 @@ +//! The coset leaf order ([`LeafOrder::Coset`]) on the host: a round-0 tree +//! places block `j` at position `bitrev(j)`, the prover's paths (whole trees +//! and kept tops) and the verifier read it there, and nothing else moves. The +//! order is Poseidon1's alone: under any other hash it is refused by name. +//! +//! The refusals across orders are the mutations that show each side's map is +//! load-bearing: a verifier that reads positions in the natural order (the +//! map dropped from the verifier) refuses a coset proof, and a prover whose +//! trees are natural (the map dropped from the prover) is refused by a coset +//! verifier. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::CapPolicy; +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as F, +}; + +use crate::{ + Error, + mle::Mle, + stacked_eval::{Claimed, Refusal, StackedCommitment, StackedProof, prove, verify_traced}, + stacking::{StackedLayout, borrow}, + whir_chain::{ + self, ChainConfig, ChainFormat, FOLD_WORD_COSET, FOLD_WORD_INDEXED, FOLD_WORD_TAG, + FirstFold, GrindBits, GroupFold, LeafOrder, UnencodedSuccessor, WhirFolds, + }, + whir_commit::{CodewordCommitment, Commitment, TreeDrop, verify_opening}, + whir_hash::{KeccakWhir, P1Whir, RpxWhir, WhirHash}, +}; + +type FE = FieldElement; +type EE = FieldElement; + +fn config( + first: Option, + log_folding: usize, + group: GroupFold, + leaves: LeafOrder, + cap: CapPolicy, +) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding, + num_queries: 9, + grind: GrindBits::default(), + format: ChainFormat { + folds: first.map_or(WhirFolds::Uniform, |k| { + WhirFolds::First(FirstFold::new(k).unwrap()) + }), + group, + leaves, + cap, + arity4_cap: cap, + ..ChainFormat::DEFAULT + }, + } +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_add(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11)) + .collect(), + ) + .unwrap() +} + +/// A stack, its claims, its roots and its proof. +struct Opened { + layout: StackedLayout, + roots: Vec, + points: Vec>, + values: Vec, + domain: crate::whir::Domain, + proof: StackedProof, +} + +fn sponge() -> H::Sponge { + let mut t = H::sponge(); + t.append_bytes(b"coset"); + t +} + +/// Commits the stack under `cfg`, retires its trees down to `drop` levels and +/// revives them when `drop` is given, and opens it. +fn open( + heights: &[usize], + n_stack: usize, + cfg: &ChainConfig, + drop: Option, +) -> Opened { + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 5 + 31 * i as u64)) + .collect(); + let points: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| EE::from(7 + 11 * i as u64 + 3 * j as u64)) + .collect() + }) + .collect(); + let values: Vec = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p).unwrap()) + .collect(); + let layout = StackedLayout::build(heights, n_stack).unwrap(); + let committed = + StackedCommitment::::commit(layout.clone(), &borrow(&columns), None, cfg).unwrap(); + let roots = committed.roots(); + let domain = committed.domain().clone(); + let stacked = match drop { + None => committed, + Some(drop) => committed + .retire(TreeDrop::uniform(drop), cfg) + .unwrap() + .revive::(&borrow(&columns), None, cfg) + .unwrap(), + }; + let proof = prove::( + &stacked, + &borrow(&columns), + None, + &Claimed::PerColumn(&points), + &values, + cfg, + &mut sponge::(), + ) + .unwrap(); + Opened { + layout, + roots, + points, + values, + domain, + proof, + } +} + +fn check( + o: &Opened, + proof: &StackedProof, + cfg: &ChainConfig, +) -> Result<(), Refusal> { + verify_traced::( + proof, + &o.layout, + &o.roots, + &Claimed::PerColumn(&o.points), + &o.values, + &o.domain, + cfg, + &mut sponge::(), + ) +} + +/// A refusal in a chain by a round-0 opening, per polynomial or grouped. +fn refused_by_an_opening(r: &Result<(), Refusal>) -> bool { + matches!( + r, + Err(Refusal::Chain { + cause: Error::OpeningRejected { .. } | Error::GroupedOpeningRejected { .. }, + .. + }) + ) +} + +/// The shapes every test below runs: a first fold of 6 over an 11-variable +/// stack (round 0 opens 64-wide blocks of a 2^7-leaf tree), a uniform chain, +/// and a chain whose round 0 is its last (its openings checked by the final +/// round's verifier). +const SHAPES: [(Option, usize, usize); 3] = + [(Some(6), 4, 11), (None, 3, 10), (Some(6), 4, 6)]; + +fn heights(n_stack: usize) -> Vec { + vec![n_stack, n_stack - 1, n_stack - 2, n_stack - 2, n_stack - 1] +} + +#[test] +fn the_coset_position_is_an_involution_that_keeps_an_index_past_the_tree() { + for depth in 0..=10usize { + let positions: Vec = (0..1usize << depth) + .map(|j| LeafOrder::Coset.position(j, depth)) + .collect(); + let mut sorted = positions.clone(); + sorted.sort_unstable(); + assert_eq!( + sorted, + (0..1usize << depth).collect::>(), + "a permutation" + ); + for (j, &p) in positions.iter().enumerate() { + assert_eq!(LeafOrder::Coset.position(p, depth), j, "its own inverse"); + assert_eq!(LeafOrder::Natural.position(j, depth), j); + } + let past = (1usize << depth) + 3; + assert_eq!( + LeafOrder::Coset.position(past, depth), + past, + "left for the range check" + ); + } + assert_eq!(LeafOrder::Coset.position(1, 3), 4); + assert_eq!(LeafOrder::Coset.position(3, 3), 6); + assert_eq!(LeafOrder::Coset.position(6, 5), 12); +} + +/// The coset tree holds block `j` at `bitrev(j)` and nowhere else: its opening +/// of `j` authenticates at that position, not at `j`; the natural tree's at +/// `j`. +#[test] +fn a_coset_tree_holds_block_j_at_its_reversed_position() { + type H = P1Whir; + let log_folding = 2; + let codeword: Vec = (0..1u64 << 9).map(|i| FE::from(i * i + 17)).collect(); + let natural = CodewordCommitment::::from_codeword_on_host_in( + codeword.clone(), + log_folding, + LeafOrder::Natural, + ) + .unwrap(); + let coset = CodewordCommitment::::from_codeword_on_host_in( + codeword, + log_folding, + LeafOrder::Coset, + ) + .unwrap(); + assert_ne!(natural.root(), coset.root(), "the order moves the root"); + assert_eq!(coset.leaves(), LeafOrder::Coset); + let depth = coset.depth(); + for j in [1usize, 2, 5, 6, 77, 100] { + let at = LeafOrder::Coset.position(j, depth); + assert_ne!(at, j, "a block whose position moves"); + let opened = coset.open(j).unwrap(); + assert_eq!( + opened.values, + natural.open(j).unwrap().values, + "the same block" + ); + assert!( + verify_opening::(&coset.root(), depth, at, &opened), + "block {j} at {at}" + ); + assert!( + !verify_opening::(&coset.root(), depth, j, &opened), + "block {j} not at {j}" + ); + assert!(verify_opening::( + &natural.root(), + depth, + j, + &natural.open(j).unwrap() + )); + } +} + +/// A coset stack verifies under the coset config, whole or retired to any +/// kept top (the paths re-hashed from the kept blocks), and the retired +/// proofs are the whole tree's, byte for byte. Per polynomial and grouped, +/// uncapped and capped. +#[test] +fn a_coset_stack_verifies_whole_or_retired() { + type H = P1Whir; + for (first, log_folding, n_stack) in SHAPES { + for group in [GroupFold::PerPoly, GroupFold::Indexed] { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + let cfg = config(first, log_folding, group, LeafOrder::Coset, cap); + let whole = open::(&heights(n_stack), n_stack, &cfg, None); + let label = format!("{first:?}/{log_folding}/{n_stack} {group:?} {cap:?}"); + assert_eq!(check::(&whole, &whole.proof, &cfg), Ok(()), "{label}"); + let want = format!("{:?}", whole.proof); + for drop in [1usize, 2, 3] { + let retired = open::(&heights(n_stack), n_stack, &cfg, Some(drop)); + assert_eq!(retired.roots, whole.roots, "{label} drop {drop}"); + assert_eq!(format!("{:?}", retired.proof), want, "{label} drop {drop}"); + } + } + } + } +} + +/// ★ Each side's map is load-bearing. A verifier reading the natural order +/// (the map dropped from the verifier) refuses a coset proof at a round-0 +/// opening; a coset verifier refuses a proof whose trees are natural (the map +/// dropped from the prover). The roots differ too, so the natural verifier is +/// handed the coset roots: only the position can refuse. +/// +/// On the two shapes whose round-0 trees have 2^7 and 2^9 leaves: a query +/// whose index is its own bit reversal opens the same leaf under either order +/// (as it should), and a chain whose nine queries all are has probability +/// 8^-9 there; on the 4-leaf tree of the one-round shape it is 2^-9 a chain. +#[test] +fn each_side_of_the_order_is_load_bearing() { + type H = P1Whir; + for (first, log_folding, n_stack) in SHAPES.into_iter().take(2) { + for group in [GroupFold::PerPoly, GroupFold::Indexed] { + let coset = config(first, log_folding, group, LeafOrder::Coset, CapPolicy::Off); + let natural = config( + first, + log_folding, + group, + LeafOrder::Natural, + CapPolicy::Off, + ); + let label = format!("{first:?}/{log_folding}/{n_stack} {group:?}"); + let made_coset = open::(&heights(n_stack), n_stack, &coset, None); + let r = check::(&made_coset, &made_coset.proof, &natural); + assert!( + refused_by_an_opening(&r), + "{label}: a natural verifier: {r:?}" + ); + let made_natural = open::(&heights(n_stack), n_stack, &natural, None); + assert_eq!( + check::(&made_natural, &made_natural.proof, &natural), + Ok(()), + "{label}" + ); + let r = check::(&made_natural, &made_natural.proof, &coset); + assert!( + refused_by_an_opening(&r), + "{label}: a coset verifier: {r:?}" + ); + } + } +} + +/// One sibling byte of a round-0 path moved: refused, by that opening. +#[test] +fn a_tampered_round0_sibling_is_refused_under_the_coset_order() { + for (first, log_folding, n_stack) in SHAPES { + let cfg = config( + first, + log_folding, + GroupFold::PerPoly, + LeafOrder::Coset, + CapPolicy::Off, + ); + let honest = open::(&heights(n_stack), n_stack, &cfg, None); + let mut proof = honest.proof.clone(); + let opening = match &mut proof.polys[0].rounds[0].openings { + crate::whir_chain::RoundOpenings::Base(round) => &mut round.current[1], + other => panic!("a per-polynomial round 0 opens base blocks: {other:?}"), + }; + opening.proof.merkle_path[0][7] ^= 1; + let r = check::(&honest, &proof, &cfg); + assert!(refused_by_an_opening(&r), "{first:?}/{n_stack}: {r:?}"); + } +} + +/// The order is part of the statement's fold word, on top of every other +/// word, and never part of the default's. +#[test] +fn the_coset_order_has_its_own_fold_word_and_is_not_the_default() { + assert!(ChainFormat::DEFAULT.is_default()); + assert_eq!(ChainFormat::DEFAULT.leaves, LeafOrder::Natural); + let mut words = Vec::new(); + for first in [None, Some(6)] { + for group in [GroupFold::PerPoly, GroupFold::Indexed] { + let natural = config(first, 4, group, LeafOrder::Natural, CapPolicy::Off); + let coset = config(first, 4, group, LeafOrder::Coset, CapPolicy::Off); + assert!(!coset.format.is_default()); + assert_eq!( + coset.fold_word(), + natural.fold_word() | FOLD_WORD_TAG | FOLD_WORD_COSET, + "{first:?} {group:?}" + ); + assert_eq!(natural.fold_word() & FOLD_WORD_COSET, 0); + words.push(natural.fold_word()); + words.push(coset.fold_word()); + } + } + assert_eq!( + config( + None, + 4, + GroupFold::PerPoly, + LeafOrder::Natural, + CapPolicy::Off + ) + .fold_word(), + 4 + ); + let mut distinct = words.clone(); + distinct.sort_unstable(); + distinct.dedup(); + assert_eq!(distinct.len(), words.len(), "eight formats, eight words"); + assert_ne!(FOLD_WORD_COSET, FOLD_WORD_INDEXED); +} + +/// ★ The coset order is Poseidon1's alone: under Keccak and RPX trees the +/// config is refused by name before a tree is built (a stack's commit, a +/// chain's commit) and before a proof is read (a stack's verifier, a chain's). +fn the_coset_order_is_refused_by_name() { + let cfg = config( + Some(6), + 4, + GroupFold::PerPoly, + LeafOrder::Coset, + CapPolicy::Off, + ); + let refused = || crate::Error::LeafOrderUnsupported { + order: LeafOrder::Coset, + hash: H::NAME, + }; + let columns: Vec> = (0..3).map(|i| column(10 - i, 9 + i as u64)).collect(); + let layout = StackedLayout::build(&[10, 9, 8], 11).unwrap(); + assert_eq!( + StackedCommitment::::commit(layout, &borrow(&columns), None, &cfg).err(), + Some(refused()), + "{}: a stack's commit", + H::NAME + ); + assert_eq!( + whir_chain::commit::(&columns[0], &cfg, true).err(), + Some(refused()), + "{}: a chain's commit", + H::NAME + ); + // A proof made where the order is defined, read under `H`. + let made = open::(&heights(11), 11, &cfg, None); + let r = check::(&made, &made.proof, &cfg); + assert!( + matches!(&r, Err(Refusal::Before(e) | Refusal::Chain { cause: e, .. }) if *e == refused()), + "{}: a stack's verifier: {r:?}", + H::NAME + ); + let chain = &made.proof.polys[0]; + let z: Vec = (0..11).map(|i| EE::from(3 + i as u64)).collect(); + assert_eq!( + whir_chain::verify::( + chain, + &made.roots[0], + &z, + EE::from(1), + &made.domain, + &cfg, + &mut sponge::(), + ) + .err(), + Some(refused()), + "{}: a chain's verifier", + H::NAME + ); + // The natural order stays every hash's. + let natural = config( + Some(6), + 4, + GroupFold::PerPoly, + LeafOrder::Natural, + CapPolicy::Off, + ); + assert!(whir_chain::commit::(&columns[0], &natural, true).is_ok()); +} + +#[test] +fn the_coset_order_is_refused_by_name_under_keccak() { + the_coset_order_is_refused_by_name::(); +} + +#[test] +fn the_coset_order_is_refused_by_name_under_rpx() { + the_coset_order_is_refused_by_name::(); +} + +/// A coset round 0 whose successor was not encoded is refused by cause: the +/// knob off whatever the factors, then factors on the host, then the card +/// declining. (The paths themselves need a card: `tests/whir_coset_device.rs`.) +#[test] +fn an_unencoded_successor_is_named_by_its_cause() { + assert_eq!( + UnencodedSuccessor::of(false, true), + UnencodedSuccessor::KnobOff + ); + assert_eq!( + UnencodedSuccessor::of(false, false), + UnencodedSuccessor::KnobOff + ); + assert_eq!( + UnencodedSuccessor::of(true, false), + UnencodedSuccessor::HostFactors + ); + assert_eq!( + UnencodedSuccessor::of(true, true), + UnencodedSuccessor::CardDeclined + ); +} + +/// A round-0 commitment placed in the natural order is not proved under a +/// coset config (nor the reverse): the prover refuses by name rather than +/// emit paths the verifier would read at other positions. +#[test] +fn a_commitment_in_another_order_is_not_proved() { + let coset = config( + Some(6), + 4, + GroupFold::PerPoly, + LeafOrder::Coset, + CapPolicy::Off, + ); + let natural = config( + Some(6), + 4, + GroupFold::PerPoly, + LeafOrder::Natural, + CapPolicy::Off, + ); + let f = column(11, 4); + let z: Vec = (0..11).map(|i| EE::from(5 + 2 * i as u64)).collect(); + for (committed_under, proved_under) in [(&natural, &coset), (&coset, &natural)] { + let (commitment, domain) = + whir_chain::commit::(&f, committed_under, true).unwrap(); + let r = whir_chain::prove::( + &f, + &z, + &commitment, + &domain, + proved_under, + &mut sponge::(), + ); + assert_eq!( + r.err(), + Some(crate::Error::LeafOrderMismatch { + expected: proved_under.format.leaves, + got: committed_under.format.leaves, + }) + ); + } +} + +/// `sample_u64_pow2`'s bits (low first, `depth` of them), reversed as the +/// leaf's `walk_bits` does at round 0, recomposed low first. +fn reversed_walk(q: usize, depth: usize) -> usize { + let bits: Vec = (0..depth).map(|t| (q >> t) & 1 == 1).collect(); + bits.iter() + .rev() + .enumerate() + .fold(0usize, |acc, (t, &b)| acc | (usize::from(b) << t)) +} + +/// ★ The host's coset position is the leaf walk's: the query's bits reversed +/// and recomposed low first, at every depth to 32 (the ends, the middle and +/// pseudo-random samples). And no alias: a bijection of `[0, 2^d)` at every +/// depth a round-0 tree of the block can have below 2^17 leaves, checked whole. +#[test] +fn the_coset_position_is_the_reversed_little_endian_walk() { + let mut x: u64 = 0x2545_F491_4F6C_DD1D; + for depth in 1..=32usize { + let top = (1usize << depth) - 1; + let mut samples = vec![0usize, 1, top, top >> 1, 1 << (depth - 1)]; + for _ in 0..64 { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + samples.push((x as usize) & top); + } + for q in samples { + assert_eq!( + LeafOrder::Coset.position(q, depth), + reversed_walk(q, depth), + "q {q} depth {depth}" + ); + } + } + for depth in 0..=16usize { + let mut seen = vec![false; 1 << depth]; + for q in 0..1usize << depth { + let p = LeafOrder::Coset.position(q, depth); + assert!(p < 1 << depth && !seen[p], "depth {depth}: {q} -> {p}"); + seen[p] = true; + } + } +} diff --git a/crypto/multilinear/src/whir_eval.rs b/crypto/multilinear/src/whir_eval.rs new file mode 100644 index 000000000..349fe2e6d --- /dev/null +++ b/crypto/multilinear/src/whir_eval.rs @@ -0,0 +1,547 @@ +//! Proving `Σ_x w(x)·f(x) = y` about a committed polynomial, which is what +//! settles the residual claims the other arguments hand back. +//! +//! The sumcheck produces the folding randomness; the fully folded codeword is +//! the constant `f(α)`, and the sumcheck's residual is `w(α)·f(α)`. Both must +//! name the same `f(α)`. +//! +//! An evaluation claim is the weight `w = eq(z, ·)`, which is what [`prove`] +//! and [`verify`] specialize to. A **stacked** claim — many columns packed into +//! one committed polynomial, each read from its own subcube — is a weight that +//! sums several of those, and settling it costs the same one sumcheck. That is +//! why the general form is the one implemented. +//! +//! One round, folding all the way down, so a block is the whole message — which +//! is what [`whir_chain`](crate::whir_chain) exists to fix. This module is the +//! degenerate one-round case of it, kept as the reference for the identity the +//! whole thing rests on. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; + +use crate::{ + Error, + eq::{eq_eval, eq_mle}, + mle::Mle, + poly::EqScaled, + sumcheck::{self, SumcheckProof}, + virtual_poly::{Term, VirtualPolynomial}, + whir::{Domain, encode, fold_codeword_k, lift_coefficients}, + whir_commit::{CodewordCommitment, Commitment, CosetOpening, fold_coset, verify_opening}, + whir_hash::WhirHash, +}; + +/// Blowup and query count. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EvalConfig { + /// `log2` of the code's inverse rate. + pub log_blowup: usize, + /// Codeword blocks checked against the fold. + pub num_queries: usize, +} + +/// A proof that a committed polynomial takes a claimed value at a point. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct EvalProof { + pub sumcheck: SumcheckProof, + /// The constant the codeword folds to — the prover's claim for `f(α)`. + pub final_value: FieldElement, + pub openings: Vec>, +} + +/// Commits to `f`, ready to answer evaluation claims. +pub fn commit( + f: &Mle, + config: &EvalConfig, +) -> Result<(CodewordCommitment, Domain), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + let domain = Domain::::new(f.num_vars() + config.log_blowup)?; + let codeword = encode::(&lift_coefficients(f), &domain)?; + // One block per fold target: folding all the way leaves `2^log_blowup`. + let commitment = CodewordCommitment::new(&codeword, f.num_vars())?; + Ok((commitment, domain)) +} + +/// `Σ_x w(x)·f(x)`, the sumcheck a weighted claim becomes. +fn weighted( + f: &Mle, + weight: Mle, +) -> Result>, Error> { + let inner = VirtualPolynomial::new(vec![f.clone()], vec![Term::single(0)])?; + EqScaled::new(inner, weight) +} + +/// Proves `f(z) = y`. +/// +/// The caller must have absorbed the commitment root and `z` into `transcript` +/// already; both sides must do the same. +pub fn prove( + f: &Mle, + z: &[FieldElement], + commitment: &CodewordCommitment, + domain: &Domain, + config: &EvalConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + prove_weighted::(f, eq_mle(z)?, commitment, domain, config, transcript) +} + +/// Proves `Σ_x w(x)·f(x) = y` for a weight the verifier can evaluate itself. +pub fn prove_weighted( + f: &Mle, + weight: Mle, + commitment: &CodewordCommitment, + domain: &Domain, + config: &EvalConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + let (sumcheck, alphas) = sumcheck::prove(weighted(f, weight)?, transcript)?; + + // From the commitment, not a second encoding: it is the same array, and a + // prover that folded a different one could not then answer the openings. + // This path builds its own commitment on the host, so the codeword is here. + let values = commitment + .codeword() + .host() + .ok_or(crate::Error::EmptyPolynomial)?; + let (folded, _) = fold_codeword_k::(values, domain, &alphas)?; + let final_value = folded[0].clone(); + transcript.append_field_element(&final_value); + + let queries = sample_queries(transcript, config.num_queries, commitment.num_leaves()); + let openings = queries + .into_iter() + .map(|q| commitment.open(q)) + .collect::>()?; + + Ok(EvalProof { + sumcheck, + final_value, + openings, + }) +} + +fn sample_queries(transcript: &mut T, num_queries: usize, bound: usize) -> Vec +where + F: IsField, + T: IsTranscript, +{ + (0..num_queries) + .map(|_| transcript.sample_u64(bound as u64) as usize) + .collect() +} + +/// Verifies `f(z) = y` against a commitment. +pub fn verify( + proof: &EvalProof, + root: &Commitment, + z: &[FieldElement], + y: FieldElement, + domain: &Domain, + config: &EvalConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + verify_weighted::( + proof, + root, + |alphas: &[FieldElement]| eq_eval(z, alphas), + y, + z.len(), + domain, + config, + transcript, + ) +} + +/// Verifies `Σ_x w(x)·f(x) = y`. +/// +/// `weight_at` is the weight's closed form; the verifier evaluates it at the +/// sumcheck point rather than holding its table. +#[allow(clippy::too_many_arguments)] +pub fn verify_weighted( + proof: &EvalProof, + root: &Commitment, + weight_at: W, + y: FieldElement, + num_vars: usize, + domain: &Domain, + config: &EvalConfig, + transcript: &mut T, +) -> Result<(), Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + W: FnOnce(&[FieldElement]) -> Result, Error>, + H: WhirHash, +{ + // The weight raises the degree of the plain `f` term to two. + let claim = sumcheck::verify(&proof.sumcheck, y, num_vars, 2, transcript)?; + let alphas = &claim.point; + + // The sumcheck's residual is w(α)·f(α); the verifier knows w. + let weight = weight_at(alphas)?; + let required = claim + .expected_evaluation + .clone() + .mul_by_inverse_of(&weight) + .ok_or(Error::DegenerateEvaluationPoint)?; + + transcript.append_field_element(&proof.final_value); + + // The wire: the folded codeword and the sumcheck must name the same f(α). + if proof.final_value != required { + return Err(Error::EvaluationMismatch); + } + + if proof.openings.len() != config.num_queries { + return Err(Error::QueryCountMismatch { + expected: config.num_queries, + got: proof.openings.len(), + }); + } + let num_leaves = domain.size() >> num_vars; + let queries = sample_queries(transcript, config.num_queries, num_leaves); + + for (i, (&q, opening)) in queries.iter().zip(&proof.openings).enumerate() { + if !verify_opening::(root, num_leaves.trailing_zeros() as usize, q, opening) { + return Err(Error::OpeningRejected { query: i }); + } + if fold_coset::(&opening.values, domain, q, alphas)? != proof.final_value { + return Err(Error::FoldInconsistent { query: i }); + } + } + + Ok(()) +} + +/// Small helper so the division reads as one step and the degenerate case is +/// explicit rather than a panic. +trait MulByInverse: Sized { + fn mul_by_inverse_of(self, other: &Self) -> Option; +} + +impl MulByInverse for FieldElement { + fn mul_by_inverse_of(self, other: &Self) -> Option { + other.inv().ok().map(|inv| self * inv) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::whir_hash::KeccakWhir; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"whir-eval-test") + } + + fn config() -> EvalConfig { + EvalConfig { + log_blowup: 2, + num_queries: 3, + } + } + + /// Deterministic pseudo-random values. The seed is mixed *before* the + /// shift — mixing it after would let nearby seeds collapse to the same + /// polynomial, which silently turns a negative test into a tautology. + fn pseudo_mle(num_vars: usize, seed: u64) -> Mle { + let vals: Vec = (0..(1u64 << num_vars)) + .map(|i| { + let mixed = i + .wrapping_add(seed) + .wrapping_mul(6364136223846793005) + .wrapping_add(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + FE::from(mixed >> 13) + }) + .collect(); + Mle::new(vals).unwrap() + } + + fn point(num_vars: usize) -> Vec { + (0..num_vars).map(|i| FE::from(101 + i as u64)).collect() + } + + #[test] + fn an_honest_evaluation_verifies() { + for num_vars in 1..=4usize { + let f = pseudo_mle(num_vars, 11); + let z = point(num_vars); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); + + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &config(), + &mut transcript(), + ) + .unwrap_or_else(|e| panic!("num_vars={num_vars}: {e:?}")); + } + } + + /// The point of the whole file: claiming a value the polynomial does not + /// take must fail, even though the prover runs the protocol honestly around + /// that lie. + #[test] + fn a_false_evaluation_is_rejected() { + let f = pseudo_mle(3, 13); + let z = point(3); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); + + let err = verify::( + &proof, + &commitment.root(), + &z, + y + FE::one(), + &domain, + &config(), + &mut transcript(), + ) + .unwrap_err(); + // The lie propagates into the residual, so it is the wire between the + // sumcheck and the folded codeword that breaks. + assert_eq!(err, Error::EvaluationMismatch); + } + + #[test] + fn a_forged_final_value_is_rejected() { + let f = pseudo_mle(3, 17); + let z = point(3); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let mut proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); + proof.final_value += FE::one(); + + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert_eq!(err, Error::EvaluationMismatch); + } + + #[test] + fn a_tampered_opening_is_rejected() { + let f = pseudo_mle(3, 19); + let z = point(3); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let mut proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); + proof.openings[0].values[0] += FE::one(); + + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert!(matches!(err, Error::OpeningRejected { query: 0 })); + } + + /// A prover who commits one polynomial and argues about another. + #[test] + fn a_proof_about_a_different_polynomial_is_rejected() { + let f = pseudo_mle(3, 23); + let g = pseudo_mle(3, 29); + let z = point(3); + + let (f_commitment, domain) = commit::(&f, &config()).unwrap(); + // Argue g's evaluation while presenting f's commitment. + let proof = prove::( + &g, + &z, + &f_commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); + + let err = verify::( + &proof, + &f_commitment.root(), + &z, + g.evaluate(&z).unwrap(), + &domain, + &config(), + &mut transcript(), + ) + .unwrap_err(); + // The codeword comes out of the commitment, so a prover cannot be + // inconsistent between the two: the lie surfaces on the wire between + // the sumcheck and the folded value. + assert_eq!(err, Error::EvaluationMismatch); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + let f = pseudo_mle(3, 31); + let z = point(3); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); + + let mut other = DefaultTranscript::::new(b"a-different-statement"); + assert!( + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &config(), + &mut other + ) + .is_err() + ); + } + + #[test] + fn a_proof_with_too_few_openings_is_rejected() { + let f = pseudo_mle(3, 37); + let z = point(3); + let y = f.evaluate(&z).unwrap(); + + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let mut proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); + proof.openings.pop(); + + let err = verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &config(), + &mut transcript(), + ) + .unwrap_err(); + assert!(matches!(err, Error::QueryCountMismatch { .. })); + } + + #[test] + fn the_commitment_has_one_block_per_fold_target() { + let f = pseudo_mle(4, 41); + let cfg = config(); + let (commitment, domain) = commit::(&f, &cfg).unwrap(); + assert_eq!(commitment.num_leaves(), 1 << cfg.log_blowup); + assert_eq!(domain.log_size(), 4 + cfg.log_blowup); + } +} diff --git a/crypto/multilinear/src/whir_group_tests.rs b/crypto/multilinear/src/whir_group_tests.rs new file mode 100644 index 000000000..0d68c8494 --- /dev/null +++ b/crypto/multilinear/src/whir_group_tests.rs @@ -0,0 +1,1311 @@ +//! D-ONECHAIN B1: one WHIR chain per run of a group's stacked polynomials +//! ([`GroupFold::Indexed`]) on the host, through the stacked opening the +//! block's groups use. The gates are REV-B2-JUDGE §4's B1-1 to B1-7, each named +//! where it is tested; the mutations it lists are run by hand against these. + +use crypto::fiat_shamir::{default_transcript::DefaultTranscript, is_transcript::IsTranscript}; +use crypto::merkle_tree::cap::CapPolicy; +use math::{ + field::{ + element::FieldElement, + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as F, + traits::{IsField, IsSubFieldOf}, + }, + traits::AsBytes, +}; + +use crate::{ + Error, + mle::Mle, + stacked_eval::{ + Claimed, Refusal, StackedCommitment, StackedProof, index_chains, prove, verify, + verify_traced, + }, + stacking::{StackedLayout, borrow}, + sumcheck, + whir::Domain, + whir_chain::{ + self, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, GroupedShape, + MAX_INDEX_POLYS, MAX_INDEX_VARS, RoundOpenings, WhirFolds, check_index_domain, eq_index, + grouped_faults::{self, Fault}, + index_vars, + }, + whir_commit::Commitment, + whir_hash::{KeccakWhir, P1Whir, WhirHash}, + whir_round::{GroupedRoundProof, RoundProof}, +}; + +type FE = FieldElement; +type EE = FieldElement; + +/// A chain config: round 0 folds `first` (`First`), or `log_folding` (uniform). +fn config( + first: Option, + log_folding: usize, + num_queries: usize, + group: GroupFold, +) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding, + num_queries, + grind: GrindBits::default(), + format: ChainFormat { + folds: first.map_or(WhirFolds::Uniform, |k| { + WhirFolds::First(FirstFold::new(k).unwrap()) + }), + group, + ..ChainFormat::DEFAULT + }, + } +} + +fn indexed(first: Option, log_folding: usize, num_queries: usize) -> ChainConfig { + config(first, log_folding, num_queries, GroupFold::Indexed) +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_add(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11)) + .collect(), + ) + .unwrap() +} + +/// A stack committed and opened, each column at a point of its own. +struct Opened { + layout: StackedLayout, + roots: Vec, + domain: Domain, + points: Vec>>, + values: Vec>, + proof: StackedProof, +} + +fn open( + heights: &[usize], + n_stack: usize, + cfg: &ChainConfig, + seed: u64, + transcript: fn() -> T, +) -> Result, Error> +where + F: IsSubFieldOf, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Send + Sync, + T: IsTranscript, + H: WhirHash, +{ + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, seed + 31 * i as u64)) + .collect(); + let points: Vec>> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| FieldElement::::from(7 + seed + 11 * i as u64 + 3 * j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p)) + .collect::, _>>()?; + let layout = StackedLayout::build(heights, n_stack)?; + let stacked = StackedCommitment::::commit(layout.clone(), &borrow(&columns), None, cfg)?; + let roots = stacked.roots(); + let proof = prove::( + &stacked, + &borrow(&columns), + None, + &Claimed::PerColumn(&points), + &values, + cfg, + &mut transcript(), + )?; + Ok(Opened { + layout, + roots, + domain: stacked.domain().clone(), + points, + values, + proof, + }) +} + +fn check( + o: &Opened, + proof: &StackedProof, + cfg: &ChainConfig, + transcript: fn() -> T, +) -> Result<(), Error> +where + F: IsSubFieldOf, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Send + Sync, + T: IsTranscript, + H: WhirHash, +{ + verify::( + proof, + &o.layout, + &o.roots, + &Claimed::PerColumn(&o.points), + &o.values, + &o.domain, + cfg, + &mut transcript(), + ) +} + +/// The tower production runs: base-field stacks, extension challenges. +fn tower() -> ::Sponge { + let mut t = KeccakWhir::sponge(); + t.append_bytes(b"onechain"); + t +} + +fn tower_p1() -> ::Sponge { + let mut t = P1Whir::sponge(); + t.append_bytes(b"onechain"); + t +} + +/// The base field alone, where a value-level forgery can keep a combination. +fn base() -> DefaultTranscript { + DefaultTranscript::::new(b"onechain-base") +} + +fn open_tower(heights: &[usize], n_stack: usize, cfg: &ChainConfig) -> Opened { + open::(heights, n_stack, cfg, 3, tower).unwrap() +} + +fn check_tower( + o: &Opened, + proof: &StackedProof, + cfg: &ChainConfig, +) -> Result<(), Error> { + check::(o, proof, cfg, tower) +} + +/// [`check`] with the refusal's cause: the chain, and the check inside it that +/// refused ([`verify_traced`]) — what a negative asserts, so it cannot pass on +/// another error. +fn trace( + o: &Opened, + proof: &StackedProof, + cfg: &ChainConfig, + transcript: fn() -> T, +) -> Result<(), Refusal> +where + F: IsSubFieldOf, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Send + Sync, + T: IsTranscript, + H: WhirHash, +{ + verify_traced::( + proof, + &o.layout, + &o.roots, + &Claimed::PerColumn(&o.points), + &o.values, + &o.domain, + cfg, + &mut transcript(), + ) +} + +fn trace_tower( + o: &Opened, + proof: &StackedProof, + cfg: &ChainConfig, +) -> Result<(), Refusal> { + trace::(o, proof, cfg, tower) +} + +/// Chain 0 (over polynomials 0..), refused by `cause`. +fn in_chain_0(cause: Error) -> Result<(), Refusal> { + Err(Refusal::Chain { + chain: 0, + poly: 0, + cause, + }) +} + +fn grouped(proof: &StackedProof, chain: usize) -> &GroupedRoundProof { + match &proof.polys[chain].rounds[0].openings { + RoundOpenings::BaseGrouped(g) => g, + _ => panic!("chain {chain} is not grouped"), + } +} + +fn grouped_mut(proof: &mut StackedProof, chain: usize) -> &mut GroupedRoundProof { + match &mut proof.polys[chain].rounds[0].openings { + RoundOpenings::BaseGrouped(g) => g, + _ => panic!("chain {chain} is not grouped"), + } +} + +/// The named shape a refusal carries. +fn shape(result: Result<(), Error>) -> GroupedShape { + match result { + Err(Error::GroupedShape(s)) => s, + other => panic!("expected a named shape refusal, got {other:?}"), + } +} + +/// Three 2^8 polynomials, one chain over all three (`m = 2`, slot 3 absent), +/// round 0 folding 6 data variables: 8 messages, blocks of 64. +const N3: usize = 8; + +fn p3() -> (Opened, ChainConfig) { + let cfg = indexed(Some(6), 4, 3); + let o = open_tower(&[N3; 3], N3, &cfg); + assert_eq!(o.layout.num_polys(), 3); + assert_eq!(o.proof.polys.len(), 1); + (o, cfg) +} + +// ───────────────────────────── B1-1: the split ───────────────────────────── + +/// REV-B2-JUDGE B1-1: consecutive runs of at most four, the remainder last; a +/// 38-polynomial singleton (CPU stated at 2^27) is nine runs of 4 and one of +/// 2, and nothing past `MAX_INDEX_VARS` index variables is ever cut. +#[test] +fn index_chains_cut_consecutive_runs_of_at_most_four() { + let widths = |n: usize| -> Vec { + index_chains(n, GroupFold::Indexed) + .iter() + .map(|r| r.len()) + .collect() + }; + assert_eq!(widths(38), [vec![4; 9], vec![2]].concat()); + assert_eq!(index_chains(5, GroupFold::Indexed), vec![0..4, 4..5]); + assert_eq!(index_chains(9, GroupFold::Indexed), vec![0..4, 4..8, 8..9]); + assert_eq!(index_chains(3, GroupFold::Indexed), vec![0..3]); + assert_eq!(MAX_INDEX_POLYS, 4); + for n in 0..=64 { + let chains = index_chains(n, GroupFold::Indexed); + let mut next = 0; + for (c, run) in chains.iter().enumerate() { + assert_eq!(run.start, next, "n={n}: consecutive"); + assert!((1..=MAX_INDEX_POLYS).contains(&run.len()), "n={n}"); + assert!(index_vars(run.len()) <= MAX_INDEX_VARS, "n={n}"); + if c + 1 < chains.len() { + assert_eq!(run.len(), MAX_INDEX_POLYS, "n={n}: only the last is short"); + } + next = run.end; + } + assert_eq!(next, n, "n={n}: every polynomial in one chain"); + assert_eq!( + index_chains(n, GroupFold::PerPoly), + (0..n).map(|i| i..i + 1).collect::>(), + "n={n}: per polynomial is today's" + ); + } +} + +/// REV-B2-JUDGE B1-1 (b): a singleton of 5 and of 9 stacked polynomials splits +/// into [4, 1] and [4, 4, 1] and verifies; the runs of 4 are grouped chains and +/// the run of 1 is today's. +#[test] +fn a_singleton_of_five_and_of_nine_split_and_verify() { + let cfg = indexed(None, 2, 3); + for (polys, want) in [(5usize, vec![4, 1]), (9, vec![4, 4, 1])] { + let o = open_tower(&vec![6; polys], 6, &cfg); + assert_eq!(o.layout.num_polys(), polys); + assert_eq!(o.proof.polys.len(), want.len()); + for (chain, &width) in want.iter().enumerate() { + let round0 = &o.proof.polys[chain].rounds[0]; + assert_eq!(round0.sumcheck.len(), 2 + index_vars(width)); + match (&round0.openings, width) { + (RoundOpenings::BaseGrouped(g), 4) => { + assert!(g.current.iter().all(|q| q.len() == 4)); + } + (RoundOpenings::Base(_), 1) => {} + _ => panic!("P={polys}: chain {chain} of width {width}"), + } + } + check_tower(&o, &o.proof, &cfg).unwrap_or_else(|e| panic!("P={polys}: {e:?}")); + } +} + +/// REV-B2-JUDGE B1-1 (c) and B1-2's chain count: a proof whose chains are not +/// the derived split is refused by name before anything is absorbed — one +/// chain over five, one chain fewer or one more — and a chain asked to open +/// five polynomials is refused before anything is read. +#[test] +fn a_chain_count_other_than_the_split_is_refused() { + let cfg = indexed(None, 2, 3); + let o5 = open_tower(&[6; 5], 6, &cfg); + let mut one = o5.proof.clone(); + one.polys.pop(); + assert_eq!( + shape(check_tower(&o5, &one, &cfg)), + GroupedShape::ChainCount { + expected: 2, + got: 1 + } + ); + let mut three = o5.proof.clone(); + three.polys.push(o5.proof.polys[1].clone()); + assert_eq!( + shape(check_tower(&o5, &three, &cfg)), + GroupedShape::ChainCount { + expected: 2, + got: 3 + } + ); + let o9 = open_tower(&[6; 9], 6, &cfg); + let mut short = o9.proof.clone(); + short.polys.remove(1); + assert_eq!( + shape(check_tower(&o9, &short, &cfg)), + GroupedShape::ChainCount { + expected: 3, + got: 2 + } + ); + // The chain verifier itself takes no run wider than four, nor an empty one. + for roots in [&o5.roots[..], &o5.roots[..0]] { + let refused = whir_chain::verify_grouped::( + &o5.proof.polys[0], + roots, + |_: &[EE]| Ok(EE::one()), + EE::zero(), + 6, + &o5.domain, + &cfg, + &mut tower(), + ); + assert_eq!( + shape(refused), + GroupedShape::RunWidth { polys: roots.len() } + ); + } +} + +/// REV-B2-JUDGE B1-1: the runtime refusal when the virtual chain's domain does +/// not exist, `n + m + log_blowup` past the two-adicity (REV-B2-A C5). +#[test] +fn an_index_past_the_two_adicity_is_refused() { + assert!(check_index_domain(27, 2, 2, 32).is_ok()); + assert!(check_index_domain(28, 2, 2, 32).is_ok()); + for (n, m, lb) in [(29, 2, 2), (27, 2, 4), (27, 6, 2), (32, 1, 0)] { + assert_eq!( + shape(check_index_domain(n, m, lb, 32)), + GroupedShape::IndexDomain { + num_vars: n, + index_vars: m, + log_blowup: lb, + two_adicity: 32 + } + ); + } +} + +/// REV-B2-JUDGE B1-1: that refusal is the stacked verifier's, on its own +/// layout and config, before anything is read: a three-polynomial opening +/// checked at a blowup where `n + m + log_blowup` passes the two-adicity is +/// refused by name. A per-polynomial verifier binds no index there. +#[test] +fn the_verifier_refuses_an_index_past_the_two_adicity_before_reading() { + let (o, cfg) = p3(); + let mut wide = cfg; + wide.log_blowup = 32 - N3 - 2 + 1; + assert_eq!( + shape(check_tower(&o, &o.proof, &wide)), + GroupedShape::IndexDomain { + num_vars: N3, + index_vars: 2, + log_blowup: wide.log_blowup, + two_adicity: 32 + } + ); + // A per-polynomial verifier binds no index there: what refuses the one + // grouped chain is its count against three polynomials. + wide.format.group = GroupFold::PerPoly; + assert_eq!( + check_tower(&o, &o.proof, &wide), + Err(Error::QueryCountMismatch { + expected: 3, + got: 1 + }) + ); +} + +// ─────────────────────── B1-2: exact shapes, by name ─────────────────────── + +/// REV-B2-JUDGE B1-2: one root per polynomial under `Indexed`, as today — a +/// root slice one short or one long is refused before anything is absorbed, +/// whatever the chain count. +#[test] +fn a_root_slice_other_than_one_per_polynomial_is_refused() { + let (o, cfg) = p3(); + let mut long = o.roots.clone(); + long.push(o.roots[0]); + for roots in [&o.roots[..2], &long[..]] { + let refused = verify::( + &o.proof, + &o.layout, + roots, + &Claimed::PerColumn(&o.points), + &o.values, + &o.domain, + &cfg, + &mut tower(), + ); + assert_eq!( + refused, + Err(Error::QueryCountMismatch { + expected: 3, + got: roots.len() + }) + ); + } +} + +/// REV-B2-JUDGE B1-2: per query exactly `P'` blocks — one fewer, or one in the +/// absent slot, is refused by name. +#[test] +fn a_query_with_other_than_one_block_per_tree_is_refused() { + let (o, cfg) = p3(); + let mut fewer = o.proof.clone(); + grouped_mut(&mut fewer, 0).current[1].pop(); + assert_eq!( + shape(check_tower(&o, &fewer, &cfg)), + GroupedShape::BlocksPerQuery { + query: 1, + expected: 3, + got: 2 + } + ); + let mut absent = o.proof.clone(); + let extra = grouped(&o.proof, 0).current[1][0].clone(); + grouped_mut(&mut absent, 0).current[1].push(extra); + assert_eq!( + shape(check_tower(&o, &absent, &cfg)), + GroupedShape::BlocksPerQuery { + query: 1, + expected: 3, + got: 4 + } + ); +} + +/// REV-B2-JUDGE B1-2: every block exactly `2^k0` values — not the virtual +/// `2^(k0+m)`, not `2^(k0−1)`, not one more. +#[test] +fn a_block_of_another_width_is_refused() { + let (o, cfg) = p3(); + for width in [1usize << 8, 1 << 5, (1 << 6) + 1] { + let mut proof = o.proof.clone(); + grouped_mut(&mut proof, 0).current[2][1] + .values + .resize(width, FE::one()); + assert_eq!( + shape(check_tower(&o, &proof, &cfg)), + GroupedShape::BlockWidth { + query: 2, + tree: 1, + expected: 64, + got: width + }, + "width {width}" + ); + } +} + +/// REV-B2-JUDGE B1-2: exactly one successor block a query, and `Q` queries. +#[test] +fn a_missing_successor_block_or_query_is_refused() { + let (o, cfg) = p3(); + let mut no_successor = o.proof.clone(); + grouped_mut(&mut no_successor, 0).next.pop(); + assert_eq!( + shape(check_tower(&o, &no_successor, &cfg)), + GroupedShape::SuccessorBlocks { + expected: 3, + got: 2 + } + ); + let mut fewer = o.proof.clone(); + let g = grouped_mut(&mut fewer, 0); + g.current.pop(); + g.next.pop(); + assert_eq!( + shape(check_tower(&o, &fewer, &cfg)), + GroupedShape::Queries { + expected: 3, + got: 2 + } + ); +} + +/// REV-B2-JUDGE B1-2: round 0 of a grouped chain carries exactly `k0 + m` +/// messages — not `k0` (6 instead of 8), not `k0 + m + 1`. +#[test] +fn round_zero_with_other_than_k0_plus_m_messages_is_refused() { + let (o, cfg) = p3(); + assert_eq!(o.proof.polys[0].rounds[0].sumcheck.len(), 8); + let mut six = o.proof.clone(); + six.polys[0].rounds[0].sumcheck.truncate(6); + assert_eq!( + shape(check_tower(&o, &six, &cfg)), + GroupedShape::Round0Messages { + expected: 8, + got: 6 + } + ); + let mut nine = o.proof.clone(); + let extra = nine.polys[0].rounds[0].sumcheck[0].clone(); + nine.polys[0].rounds[0].sumcheck.push(extra); + assert_eq!( + shape(check_tower(&o, &nine, &cfg)), + GroupedShape::Round0Messages { + expected: 8, + got: 9 + } + ); +} + +/// The per-polynomial variant built from a grouped round: tree 0's blocks. +fn as_base(g: &GroupedRoundProof) -> RoundOpenings { + RoundOpenings::Base(RoundProof { + current: g.current.iter().map(|q| q[0].clone()).collect(), + next: g.next.clone(), + }) +} + +/// A per-polynomial round 0 dressed as a grouped one of one tree. +fn as_grouped(openings: &RoundOpenings) -> RoundOpenings { + match openings { + RoundOpenings::Base(p) => RoundOpenings::BaseGrouped(GroupedRoundProof { + current: p.current.iter().map(|o| vec![o.clone()]).collect(), + next: p.next.clone(), + }), + _ => panic!("not a base round"), + } +} + +/// REV-B2-JUDGE B1-2: `BaseGrouped` exactly at round 0 of a grouped chain +/// (final or not). `Base` is refused there; `BaseGrouped` is refused at a +/// later round, under `PerPoly`, and on a chain over one polynomial. +#[test] +fn the_grouped_variant_is_refused_wherever_it_does_not_belong() { + let (o, cfg) = p3(); + // `Base` at a grouped round 0. + let mut base0 = o.proof.clone(); + base0.polys[0].rounds[0].openings = as_base(grouped(&o.proof, 0)); + assert_eq!( + shape(check_tower(&o, &base0, &cfg)), + GroupedShape::Variant { round: 0 } + ); + // `BaseGrouped` at round 1. + let mut late = o.proof.clone(); + late.polys[0].rounds[1].openings = RoundOpenings::BaseGrouped(grouped(&o.proof, 0).clone()); + assert_eq!( + shape(check_tower(&o, &late, &cfg)), + GroupedShape::Variant { round: 1 } + ); + // `BaseGrouped` under `PerPoly`, at today's round 0. + let per_poly = config(Some(6), 4, 3, GroupFold::PerPoly); + let today = open_tower(&[N3; 3], N3, &per_poly); + let mut dressed = today.proof.clone(); + dressed.polys[0].rounds[0].openings = as_grouped(&today.proof.polys[0].rounds[0].openings); + assert_eq!( + shape(check_tower(&today, &dressed, &per_poly)), + GroupedShape::Variant { round: 0 } + ); + // `BaseGrouped` on the singleton chain of an indexed split (P' = 1). + let cfg5 = indexed(None, 2, 3); + let o5 = open_tower(&[6; 5], 6, &cfg5); + let mut single = o5.proof.clone(); + single.polys[1].rounds[0].openings = as_grouped(&o5.proof.polys[1].rounds[0].openings); + assert_eq!( + shape(check_tower(&o5, &single, &cfg5)), + GroupedShape::Variant { round: 0 } + ); + // `Base` at a grouped round 0 that is also the last round (R = 1). + let one = indexed(Some(6), 4, 3); + let r1 = open_tower(&[3; 3], 3, &one); + assert_eq!(r1.proof.polys[0].rounds.len(), 1); + let mut base_final = r1.proof.clone(); + base_final.polys[0].rounds[0].openings = as_base(grouped(&r1.proof, 0)); + assert_eq!( + shape(check_tower(&r1, &base_final, &one)), + GroupedShape::Variant { round: 0 } + ); +} + +// ─────────────────── B1-3: one convention for the index ──────────────────── + +/// Columns of mixed heights at their own points, so a column's selector — its +/// prefix — runs past `k0` and straddles the index the chain inserts there. +const MIXED: [usize; 9] = [8, 3, 5, 8, 2, 8, 6, 4, 8]; + +/// REV-B2-JUDGE B1-3: mixed heights round-trip, with some column's selector +/// longer than `k0`, in a chain over three or four polynomials. +#[test] +fn mixed_heights_straddle_the_index_and_verify() { + let cfg = indexed(None, 2, 3); + let o = open_tower(&MIXED, 8, &cfg); + let k0 = cfg.schedule(8)[0]; + let runs = index_chains(o.layout.num_polys(), GroupFold::Indexed); + assert!( + runs.iter().any(|r| r.len() >= 3), + "a chain over three or more: {runs:?}" + ); + assert!( + o.layout + .placements() + .iter() + .any(|p| p.prefix_bits().len() > k0), + "some selector straddles the index" + ); + check_tower(&o, &o.proof, &cfg).unwrap(); +} + +/// REV-B2-JUDGE B1-3: each reading of the index is load-bearing. With `ρ` +/// swapped in the combine only, `ρ` swapped in the weight only, the index +/// factor dropped from the weight, or the absent slot given weight 1, the +/// honest proof is refused; with none of them it verifies. +#[test] +fn each_reading_of_the_index_is_load_bearing() { + let cfg = indexed(None, 2, 3); + let o = open_tower(&[3, 5, 8, 2, 4], 8, &cfg); + assert_eq!( + o.layout.num_polys(), + 3, + "one chain over three, slot 3 absent" + ); + grouped_faults::with(Fault::None, || check_tower(&o, &o.proof, &cfg)).unwrap(); + // The combine's reading refuses at round 0's first fold; the weight's at + // the final check. + for (fault, cause) in [ + ( + Fault::SwapRhoInCombine, + Error::FoldInconsistent { query: 0 }, + ), + (Fault::SwapRhoInWeight, Error::EvaluationMismatch), + (Fault::DropIndexFactor, Error::EvaluationMismatch), + (Fault::AbsentSlotWeightOne, Error::EvaluationMismatch), + ] { + let refused = grouped_faults::with(fault, || trace_tower(&o, &o.proof, &cfg)); + assert_eq!(refused, in_chain_0(cause), "{fault:?}"); + } + check_tower(&o, &o.proof, &cfg).unwrap(); +} + +/// `eq_index` is `eq(ρ, bits(p))` with `ρ_0` against the high bit, and sums to +/// one over the slots. +#[test] +fn eq_index_reads_rho_against_the_high_bit() { + let rho = [EE::from(5), EE::from(9)]; + let one = EE::one(); + assert_eq!(eq_index(&rho, 0), (&one - &rho[0]) * (&one - &rho[1])); + assert_eq!(eq_index(&rho, 1), (&one - &rho[0]) * &rho[1]); + assert_eq!(eq_index(&rho, 2), &rho[0] * (&one - &rho[1])); + assert_eq!(eq_index(&rho, 3), &rho[0] * &rho[1]); + let total = (0..4).fold(EE::zero(), |acc, p| acc + eq_index(&rho, p)); + assert_eq!(total, one); + assert_eq!(eq_index::(&[], 0), one); +} + +// ──────────────────────── B1-4: the grouped schedule ─────────────────────── + +/// REV-B2-JUDGE B1-4: `(k0, m, tail = schedule(n)[1..])` at the production +/// config — the round count, the folds and so the trees and their caps are +/// the per-polynomial chain's, `n + m` messages in all, Q 114 — and never +/// `schedule(n + m)`. +#[test] +fn the_grouped_schedule_widens_round_zero_only() { + let production = |group: GroupFold| { + let mut c = ChainConfig::with_security_folds( + 2, + 4, + WhirFolds::First(FirstFold::new(6).unwrap()), + 27, + 128, + GrindBits::query_only(18), + ); + c.format.group = group; + c + }; + let (per_poly, grouped) = ( + production(GroupFold::PerPoly), + production(GroupFold::Indexed), + ); + assert_eq!((per_poly.num_queries, grouped.num_queries), (114, 114)); + for n in 1..=27usize { + let folds = grouped.schedule(n); + assert_eq!(folds, per_poly.schedule(n), "n={n}: the folds"); + for m in 0..=MAX_INDEX_VARS { + let messages = grouped.index_schedule(n, m); + assert_eq!(messages.len(), per_poly.rounds(n), "n={n} m={m}: R"); + assert_eq!(messages.iter().sum::(), n + m, "n={n} m={m}"); + assert_eq!(messages[0], folds[0] + m, "n={n} m={m}"); + assert_eq!(messages[1..], folds[1..], "n={n} m={m}: the tail"); + } + for arity in [2, 4] { + assert_eq!( + grouped.tree_caps_at(n, arity), + per_poly.tree_caps_at(n, arity), + "n={n}: tree caps" + ); + } + } + assert_eq!(grouped.index_schedule(27, 2), vec![8, 4, 4, 4, 4, 4, 1]); + assert_ne!(grouped.index_schedule(27, 2), grouped.schedule(29)); + assert_eq!(grouped.schedule(29), vec![6, 4, 4, 4, 4, 4, 3]); +} + +/// REV-B2-JUDGE B1-4 (J2): a grouped chain whose round 0 is its last round +/// (`n ≤ k0`) round-trips for `P' = 2, 3, 4`; its round 0 opens no successor. +#[test] +fn grouped_chains_of_one_round_round_trip() { + let cfg = indexed(Some(6), 4, 3); + for polys in 2..=4usize { + let o = open_tower(&vec![3; polys], 3, &cfg); + assert_eq!(o.proof.polys.len(), 1); + let chain = &o.proof.polys[0]; + assert_eq!(chain.rounds.len(), 1, "P'={polys}: R = 1"); + assert_eq!(chain.rounds[0].sumcheck.len(), 3 + index_vars(polys)); + let g = grouped(&o.proof, 0); + assert!(g.next.is_empty()); + assert!(g.current.iter().all(|q| q.len() == polys)); + check_tower(&o, &o.proof, &cfg).unwrap_or_else(|e| panic!("P'={polys}: {e:?}")); + } +} + +// ───────────────────────── B1-5: the format word ─────────────────────────── + +/// REV-B2-JUDGE B1-5: a grouped proof is refused by a per-polynomial verifier +/// and the reverse. +#[test] +fn an_indexed_proof_and_a_per_poly_proof_refuse_each_others_verifier() { + let grouped_cfg = indexed(Some(6), 4, 3); + let per_poly = config(Some(6), 4, 3, GroupFold::PerPoly); + let g = open_tower(&[N3; 3], N3, &grouped_cfg); + let p = open_tower(&[N3; 3], N3, &per_poly); + assert_eq!(g.roots, p.roots, "the commitments are the same"); + check_tower(&g, &g.proof, &grouped_cfg).unwrap(); + check_tower(&p, &p.proof, &per_poly).unwrap(); + assert_eq!( + check_tower(&g, &g.proof, &per_poly), + Err(Error::QueryCountMismatch { + expected: 3, + got: 1 + }), + "one chain where a per-polynomial verifier counts three" + ); + assert_eq!( + shape(check_tower(&p, &p.proof, &grouped_cfg)), + GroupedShape::ChainCount { + expected: 1, + got: 3 + } + ); +} + +// ──────────────────────── B1-6: default bytes ───────────────────────────── + +/// REV-B2-JUDGE B1-6: the appended variant does not grow the archived enum, +/// so a proof without it keeps its rkyv layout. +#[test] +fn the_grouped_variant_does_not_grow_the_archived_enum() { + #[derive(rkyv::Archive)] + #[allow(dead_code)] + enum Before { + Base(RoundProof), + Extension(RoundProof), + } + assert_eq!( + size_of::>>(), + size_of::>() + ); + assert_eq!( + align_of::>>(), + align_of::>() + ); +} + +/// REV-B2-JUDGE B1-7: one polynomial under `Indexed` is today's chain, byte for +/// byte, and so is a P = 5 split's singleton chain against a lone polynomial. +#[test] +fn one_polynomial_under_indexed_is_todays_chain() { + let per_poly = config(Some(6), 4, 3, GroupFold::PerPoly); + let grouped_cfg = indexed(Some(6), 4, 3); + let today = open_tower(&[7], 7, &per_poly); + let one = open_tower(&[7], 7, &grouped_cfg); + assert_eq!(format!("{:?}", one.proof), format!("{:?}", today.proof)); + check_tower(&one, &one.proof, &grouped_cfg).unwrap(); +} + +// ──────────────────── B1-7: the note's vectors and negatives ────────────── + +/// REV-B2-JUDGE B1-7: round trips at stacks of 6 to 12 variables and 1 to 5 +/// polynomials, over pseudo-random layouts (tall columns, and short ones packed +/// among them). +#[test] +fn random_layouts_round_trip() { + let mut seed = 0x2545_f491_4f6c_dd1du64; + let mut next = |bound: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % bound + }; + for n in 6..=12usize { + let cfg = if n >= 7 { + indexed(Some(6), 4, 3) + } else { + indexed(None, 2, 3) + }; + for polys in 1..=5usize { + let mut shorts = Vec::new(); + let mut cells = 0usize; + for _ in 0..1 + next(4) { + let h = 1 + next(n as u64 - 1) as usize; + if cells + (1 << h) <= 1 << n { + cells += 1 << h; + shorts.push(h); + } + } + // Tallest first: powers of two in falling order pack with no + // alignment gap, so the shorts fill one polynomial. + shorts.sort_unstable_by(|a, b| b.cmp(a)); + let at = next(polys as u64) as usize; + let mut heights = vec![n; polys - 1]; + heights.splice(at..at, shorts); + let o = + open::(&heights, n, &cfg, n as u64 + 17 * polys as u64, tower) + .unwrap(); + assert_eq!(o.layout.num_polys(), polys, "n={n} heights={heights:?}"); + check_tower(&o, &o.proof, &cfg) + .unwrap_or_else(|e| panic!("n={n} P={polys} {heights:?}: {e:?}")); + } + } +} + +/// REV-B2-JUDGE B1-7: a tampered value in any slot `p` is refused by tree +/// `p`'s own opening at that query (and reported as the chain's). +#[test] +fn a_tampered_value_in_any_slot_is_refused() { + let (o, cfg) = p3(); + for p in 0..3 { + let mut proof = o.proof.clone(); + grouped_mut(&mut proof, 0).current[1][p].values[5] += FE::one(); + assert_eq!( + trace_tower(&o, &proof, &cfg), + in_chain_0(Error::GroupedOpeningRejected { query: 1, tree: p }), + "slot {p}" + ); + assert_eq!( + check_tower(&o, &proof, &cfg), + Err(Error::ColumnOpeningRejected { column: 0 }) + ); + } +} + +/// REV-B2-JUDGE B1-7: blocks swapped between trees, and one tree's block from +/// another position, are refused: each block opens against its own tree at the +/// query's one position. +#[test] +fn blocks_from_another_tree_or_position_are_refused() { + let (o, cfg) = p3(); + let mut swapped = o.proof.clone(); + grouped_mut(&mut swapped, 0).current[1].swap(0, 1); + assert_eq!( + trace_tower(&o, &swapped, &cfg), + in_chain_0(Error::GroupedOpeningRejected { query: 1, tree: 0 }), + "tree 0 reads tree 1's block" + ); + + let g = grouped(&o.proof, 0); + let other = (0..g.current.len()) + .find(|&j| g.current[j][2].values != g.current[1][2].values) + .expect("two queries at different positions"); + let mut moved = o.proof.clone(); + grouped_mut(&mut moved, 0).current[1][2] = g.current[other][2].clone(); + assert_eq!( + trace_tower(&o, &moved, &cfg), + in_chain_0(Error::GroupedOpeningRejected { query: 1, tree: 2 }), + "tree 2 at query {other}'s position" + ); +} + +/// Each tree's path is load-bearing on its own, not just through the fold: in +/// the base field a prover CAN change blocks 1 and 2 so that `Σ_p eq(ρ, p)· +/// block_p` — and so the fold — is unchanged; only tree 1's and tree 2's own +/// openings refuse it. (Over the tower the base blocks cannot absorb an +/// extension ratio, which is why the in-guest lift must stay pinned.) +#[test] +fn a_combination_preserving_forgery_is_refused_by_the_paths() { + let cfg = indexed(Some(6), 4, 3); + let o = open::(&[N3; 3], N3, &cfg, 5, base).unwrap(); + // `ρ`, replayed: the values, γ, then round 0's messages (no grind bits). + let mut t = base(); + for v in &o.values { + t.append_field_element(v); + } + let _gamma: FE = t.sample_field_element(); + let claim = + sumcheck::verify_rounds(&o.proof.polys[0].rounds[0].sumcheck, FE::zero(), 2, &mut t) + .unwrap(); + let k0 = cfg.schedule(N3)[0]; + let rho = &claim.point[k0..]; + let (e1, e2) = (eq_index(rho, 1), eq_index(rho, 2)); + let ratio = &e1 * e2.inv().unwrap(); + + let mut forged = o.proof.clone(); + let RoundOpenings::BaseGrouped(g) = &mut forged.polys[0].rounds[0].openings else { + panic!("grouped"); + }; + let honest = crate::whir_round::combine_blocks(&g.current[1], &[eq_index(rho, 0), e1, e2]); + for i in 0..g.current[1][1].values.len() { + let delta = FE::from(1 + i as u64); + g.current[1][1].values[i] += delta; + let kept = &g.current[1][2].values[i] - &delta * ∶ + g.current[1][2].values[i] = kept; + } + let kept = crate::whir_round::combine_blocks(&g.current[1], &[eq_index(rho, 0), e1, e2]); + assert_eq!(kept, honest, "the forgery keeps the combination"); + assert_eq!( + trace::(&o, &forged, &cfg, base), + in_chain_0(Error::GroupedOpeningRejected { query: 1, tree: 1 }), + "tree 1's own opening, not the fold" + ); + check::(&o, &o.proof, &cfg, base).unwrap(); +} + +/// The grouped opening under Poseidon1's 4-ary trees and their caps, as the +/// block's base runs it: each round-0 tree carries its own cap; the proof is +/// refused under another cap. +#[test] +fn a_grouped_opening_round_trips_under_p1_caps() { + for policy in [CapPolicy::Off, CapPolicy::Fixed(1)] { + let mut cfg = indexed(Some(6), 4, 3); + cfg.format.arity4_cap = policy; + for polys in [3usize, 5] { + let o = open::(&vec![N3; polys], N3, &cfg, 9, tower_p1).unwrap(); + check::(&o, &o.proof, &cfg, tower_p1) + .unwrap_or_else(|e| panic!("{policy} P={polys}: {e:?}")); + let mut other = cfg; + other.format.arity4_cap = if policy == CapPolicy::Off { + CapPolicy::Fixed(1) + } else { + CapPolicy::Off + }; + // Uncapped trees read under a cap have no cap to check; capped ones + // read uncapped walk the owner's cap words to no root. + let cause = if policy == CapPolicy::Off { + Error::CapRejected + } else { + Error::GroupedOpeningRejected { query: 0, tree: 0 } + }; + assert_eq!( + trace::(&o, &o.proof, &other, tower_p1), + in_chain_0(cause), + "{policy} P={polys}: read under {}", + other.format.arity4_cap + ); + } + } + // And a binary cap under KeccakWhir. + let mut cfg = indexed(Some(6), 4, 3); + cfg.format.cap = CapPolicy::Fixed(2); + let o = open_tower(&[N3; 4], N3, &cfg); + check_tower(&o, &o.proof, &cfg).unwrap(); +} + +// ─────────────────── from the code review (REV-OC-B) ─────────────────── + +/// The code review's M-a, host: each round-0 tree is bound to its OWN cap and +/// root. Under a binary cap, trees 0 and 1 trading the cap their owner paths +/// carry (query 0) is refused at tree 0's cap, `CapRejected`; under +/// Poseidon1's 4-ary cap, trading the whole owner paths likewise. +#[test] +fn round_zero_trees_cannot_trade_caps() { + let mut cfg = indexed(Some(6), 4, 3); + cfg.format.cap = CapPolicy::Fixed(2); + let o = open_tower(&[N3; 3], N3, &cfg); + check_tower(&o, &o.proof, &cfg).unwrap(); + let cap_nodes = 1usize << 2; + let mut traded = o.proof.clone(); + let g = grouped_mut(&mut traded, 0); + let len0 = g.current[0][0].proof.merkle_path.len(); + let len1 = g.current[0][1].proof.merkle_path.len(); + assert_eq!(len0, len1); + assert!(len0 >= cap_nodes, "the owner path carries the cap"); + let cap0: Vec<_> = g.current[0][0].proof.merkle_path[len0 - cap_nodes..].to_vec(); + let cap1: Vec<_> = g.current[0][1].proof.merkle_path[len1 - cap_nodes..].to_vec(); + assert_ne!(cap0, cap1, "two trees, two caps"); + g.current[0][0].proof.merkle_path[len0 - cap_nodes..].clone_from_slice(&cap1); + g.current[0][1].proof.merkle_path[len1 - cap_nodes..].clone_from_slice(&cap0); + assert_eq!( + trace_tower(&o, &traded, &cfg), + in_chain_0(Error::CapRejected), + "tree 0's cap is tree 1's" + ); + + let mut cfg = indexed(Some(6), 4, 3); + cfg.format.arity4_cap = CapPolicy::Fixed(1); + let o = open::(&[N3; 3], N3, &cfg, 9, tower_p1).unwrap(); + check::(&o, &o.proof, &cfg, tower_p1).unwrap(); + let mut traded = o.proof.clone(); + let g = grouped_mut(&mut traded, 0); + let path0 = g.current[0][0].proof.merkle_path.clone(); + g.current[0][0].proof.merkle_path = g.current[0][1].proof.merkle_path.clone(); + g.current[0][1].proof.merkle_path = path0; + assert_eq!( + trace::(&o, &traded, &cfg, tower_p1), + in_chain_0(Error::CapRejected), + "P1: tree 0's owner path is tree 1's" + ); +} + +/// The code review's M-b, host: how many successor blocks a grouped round 0 +/// carries is the verifier's schedule, never the proof's — a final round 0 +/// (R = 1) presenting successor blocks, and a non-final one presenting none, +/// are each refused by name before anything is opened. +#[test] +fn successor_blocks_follow_the_schedule_not_the_proof() { + let cfg = indexed(Some(6), 4, 3); + let (o, _) = p3(); + let r1 = open_tower(&[3; 3], 3, &cfg); + assert_eq!(r1.proof.polys[0].rounds.len(), 1, "R = 1"); + let mut extra = r1.proof.clone(); + grouped_mut(&mut extra, 0).next = grouped(&o.proof, 0).next.clone(); + assert_eq!( + shape(check_tower(&r1, &extra, &cfg)), + GroupedShape::SuccessorBlocks { + expected: 0, + got: 3 + } + ); + let mut none = o.proof.clone(); + grouped_mut(&mut none, 0).next.clear(); + assert_eq!( + shape(check_tower(&o, &none, &cfg)), + GroupedShape::SuccessorBlocks { + expected: 3, + got: 0 + } + ); +} + +// ─────────── each binding of a grouped round 0, on its own ─────────── + +/// [`open`] over the caller's columns, each column `i` claimed at the point +/// [`open`] gives it under `seed` — so two stacks that differ in one column +/// are claimed at the same points. +fn open_columns( + columns: &[Mle], + n_stack: usize, + cfg: &ChainConfig, + seed: u64, + transcript: fn() -> T, +) -> Opened +where + F: IsSubFieldOf, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Send + Sync, + T: IsTranscript, + H: WhirHash, +{ + let heights: Vec = columns.iter().map(Mle::num_vars).collect(); + let points: Vec>> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| FieldElement::::from(7 + seed + 11 * i as u64 + 3 * j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p)) + .collect::, _>>() + .unwrap(); + let layout = StackedLayout::build(&heights, n_stack).unwrap(); + let stacked = + StackedCommitment::::commit(layout.clone(), &borrow(columns), None, cfg).unwrap(); + let roots = stacked.roots(); + let proof = prove::( + &stacked, + &borrow(columns), + None, + &Claimed::PerColumn(&points), + &values, + cfg, + &mut transcript(), + ) + .unwrap(); + Opened { + layout, + roots, + domain: stacked.domain().clone(), + points, + values, + proof, + } +} + +/// EACH round-0 tree of a grouped chain is bound to the root at its own +/// position, its cap included — tree 0 as much as a later one. The forgery, +/// for every slot `p` in turn: slot `p` commits another polynomial, and the +/// proof is that stack's honest one (its own trees, caps and paths, +/// self-consistent), claimed against the stated roots, which differ from its +/// own at slot `p` alone. The claim is false — slot `p`'s values are not the +/// evaluations of the polynomial its stated root commits — and every check +/// but tree `p`'s binding to that root passes; trading two trees' caps +/// ([`round_zero_trees_cannot_trade_caps`]) breaks two bindings at once and +/// cannot show either alone. A capped tree refuses at its cap (`CapRejected`, +/// before any query is opened), an uncapped one at its first opening. +#[test] +fn a_tree_committing_another_polynomial_is_refused_at_its_own_root() { + fn case>( + cfg: &ChainConfig, + capped: bool, + transcript: fn() -> T, + ) { + let honest: Vec> = (0..3).map(|i| column(N3, 3 + 31 * i)).collect(); + for slot in 0..3 { + let mut other = honest.clone(); + other[slot] = column(N3, 1_000 + slot as u64); + let a = open_columns::(&honest, N3, cfg, 3, transcript); + let mut b = open_columns::(&other, N3, cfg, 3, transcript); + assert_eq!(a.points, b.points); + assert_ne!(a.values, b.values); + for p in 0..3 { + assert_eq!(a.roots[p] == b.roots[p], p != slot, "slot {slot}: root {p}"); + } + assert_eq!(trace::(&b, &b.proof, cfg, transcript), Ok(())); + b.roots = a.roots.clone(); + let cause = if capped { + Error::CapRejected + } else { + Error::GroupedOpeningRejected { + query: 0, + tree: slot, + } + }; + assert_eq!( + trace::(&b, &b.proof, cfg, transcript), + in_chain_0(cause), + "slot {slot}, capped {capped}" + ); + } + } + for cap in [CapPolicy::Fixed(2), CapPolicy::Off] { + let mut cfg = indexed(Some(6), 4, 3); + cfg.format.cap = cap; + case::(&cfg, cap != CapPolicy::Off, tower); + } + for cap in [CapPolicy::Fixed(1), CapPolicy::Off] { + let mut cfg = indexed(Some(6), 4, 3); + cfg.format.arity4_cap = cap; + case::(&cfg, cap != CapPolicy::Off, tower_p1); + } +} + +/// The grouped round 0's SUCCESSOR is authenticated by its own cap against the +/// root the round absorbed (`whir_round::verify_grouped`'s copy of the +/// per-polynomial round's check): a successor cap word that does not hash to +/// that root is refused at the cap, before any query's opening is read. +#[test] +fn a_grouped_successor_cap_off_its_root_is_refused_at_the_cap() { + let mut cfg = indexed(Some(6), 4, 3); + cfg.format.cap = CapPolicy::Fixed(1); + let caps = cfg.tree_caps_at(N3, 2); + assert!(caps[1] >= 1, "a capped successor: {caps:?}"); + let o = open_tower(&[N3; 3], N3, &cfg); + check_tower(&o, &o.proof, &cfg).unwrap(); + let mut forged = o.proof.clone(); + let owner = &mut grouped_mut(&mut forged, 0).next[0].proof.merkle_path; + let last = owner.len() - 1; + owner[last][0] ^= 1; + assert_eq!( + trace_tower(&o, &forged, &cfg), + in_chain_0(Error::CapRejected) + ); +} + +/// A grouped round 0 that is the chain's last round (R = 1) compares each +/// query's combined fold with the final value — its own check, ahead of the +/// final weight check. At a blowup of one the round-0 trees have one leaf, so +/// a moved final value moves no query: the openings stay valid and the fold is +/// what refuses (the final check alone would say `EvaluationMismatch`). +#[test] +fn a_one_round_grouped_chain_compares_its_folds_with_the_final_value() { + let mut cfg = indexed(Some(6), 4, 3); + cfg.log_blowup = 0; + let r1 = open_tower(&[3; 3], 3, &cfg); + assert_eq!(r1.proof.polys[0].rounds.len(), 1); + check_tower(&r1, &r1.proof, &cfg).unwrap(); + let mut forged = r1.proof.clone(); + forged.polys[0].final_value += EE::one(); + assert_eq!( + trace_tower(&r1, &forged, &cfg), + in_chain_0(Error::FoldInconsistent { query: 0 }) + ); +} + +// ──────────────── the base code's rate (LAMBDA_VM_ZF_WHIR_BLOWUP) ─────────────── + +/// The rate is a verifier constant under `Indexed` too. Runs of two, three and +/// four at blowup 2 round-trip; a verifier at blowup 4 refuses them over the +/// same roots, and a verifier at blowup 2 refuses a stack opened at blowup 4 +/// (every domain and tree depth differs by one); and at blowup 2 the two modes +/// refuse each other's proofs, as at blowup 4. +#[test] +fn an_indexed_stack_at_one_blowup_is_refused_at_the_other() { + let at = |log_blowup: usize, group: GroupFold| ChainConfig { + log_blowup, + ..config(Some(6), 4, 3, group) + }; + for polys in 2..=MAX_INDEX_POLYS { + let heights = vec![N3; polys]; + for (own, other) in [(1usize, 2usize), (2, 1)] { + let cfg = at(own, GroupFold::Indexed); + let o = open_tower(&heights, N3, &cfg); + assert_eq!(o.domain.log_size(), N3 + own, "{polys} polynomials"); + check_tower(&o, &o.proof, &cfg).unwrap_or_else(|e| { + panic!("{polys} polynomials at blowup 2^{own}: the control: {e:?}") + }); + let wrong = at(other, GroupFold::Indexed); + let domain = Domain::::new(N3 + other).unwrap(); + assert!( + verify::( + &o.proof, + &o.layout, + &o.roots, + &Claimed::PerColumn(&o.points), + &o.values, + &domain, + &wrong, + &mut tower(), + ) + .is_err(), + "{polys} polynomials at blowup 2^{own}: a verifier at 2^{other} must refuse it" + ); + } + let grouped = at(1, GroupFold::Indexed); + let per_poly = at(1, GroupFold::PerPoly); + let g = open_tower(&heights, N3, &grouped); + let p = open_tower(&heights, N3, &per_poly); + assert_eq!(g.roots, p.roots, "{polys}: the commitments are the same"); + assert!(check_tower(&g, &g.proof, &per_poly).is_err(), "{polys}"); + assert!(check_tower(&p, &p.proof, &grouped).is_err(), "{polys}"); + } +} diff --git a/crypto/multilinear/src/whir_hash.rs b/crypto/multilinear/src/whir_hash.rs new file mode 100644 index 000000000..657514405 --- /dev/null +++ b/crypto/multilinear/src/whir_hash.rs @@ -0,0 +1,371 @@ +//! ★ The hash the WHIR path runs on — the one name a proof's Merkle trees, its +//! Fiat-Shamir sponge and its proof-of-work all answer to. +//! +//! # Why one trait and not three parameters +//! +//! A WHIR proof has three hash consumers: the Merkle backend that builds its +//! roots, the transcript sponge that draws its challenges, and the grind that +//! gates each redrawable challenge. Parameterising them separately would make +//! the **half-flip** spellable — one hash's trees under another hash's sponge. +//! That configuration is self-consistent between prover and verifier, so it +//! verifies, so nothing fails; it is silent by construction, and the only thing +//! wrong with it is that no single name describes the proof. Here there is one +//! name to write, so there is nothing to assert against: the bad state is +//! unreachable rather than checked. +//! +//! # What a hash may NOT change +//! +//! [`Commitment`] stays `[u8; 32]` for every implementation. A keccak digest is +//! 32 bytes and an algebraic digest is four canonical Goldilocks felts, which is +//! also 32 bytes — so every proof type on this path (`ChainProof`, +//! `StackedProof`, `CosetOpening`, `Proof`, and `MultiProof` above +//! them) keeps its layout, its rkyv derives and its serialized length. **A hash +//! swap is not a proof-format change**, and `stacked_eval`'s +//! `the_two_hashes_serialize_to_the_same_length` is what holds that to it. +//! +//! # The device +//! +//! Under `cuda` the leaf and parent hashing happens in KERNELS, and the host +//! backend is only the label on the tree they built. So a configuration must +//! also name which kernel family the device has to run: +//! [`WhirHash::DEVICE`] is that name, handed down to `math-cuda`'s tree entry +//! points, which match on it exhaustively. A tree labelled `Self` was therefore +//! hashed by `Self`'s kernels or was not built on the device at all — never by +//! another hash's kernels wearing this name. +//! +//! The key is a type of this crate's own rather than `math_cuda::DeviceHash` +//! directly, because `math-cuda` is an optional dependency and the trait has to +//! exist on a build without it. [`DeviceHashKey::into_math_cuda`] is the bridge, +//! and it is total in both directions with the pairing asserted at compile time, +//! so the two enums cannot drift apart or be cross-wired. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::p1_transcript::{P1Transcript, P1TranscriptHash}; +use crypto::fiat_shamir::transcript_hash::{ + HasTranscriptHash, KeccakTranscriptHash, RpxTranscriptHash, TranscriptHash, +}; +use crypto::merkle_tree::backends::p1::P1BatchBackend; +use crypto::merkle_tree::backends::types::{BatchKeccak256Backend, BatchRpx256Backend}; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::{element::FieldElement, traits::IsField}; +use math::traits::AsBytes; + +use crate::whir_commit::Commitment; + +/// The digest a configuration grinds over: its transcript's hash, because the +/// grinding seed is `transcript.state()`. +pub type GrindingDigest = <::Transcript as TranscriptHash>::Digest; + +/// ★ Which kernel family the device must run for a configuration's trees. +/// +/// Mirrors `math_cuda::DeviceHash` and exists separately only so this trait +/// compiles without the optional `math-cuda` dependency. The two are kept in +/// step by [`DeviceHashKey::into_math_cuda`] plus the compile-time pairing +/// assertion beside it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DeviceHashKey { + /// Keccak-256 at both the leaf and the parent layer. + Keccak256, + /// RPX256 (XHash12) at both layers. + Rpx256, + /// ZisK's Poseidon1 (width 16, rate 12): the tagged leaf and 4-ary nodes. + Poseidon1, +} + +impl DeviceHashKey { + /// The key `math-cuda` dispatches on. + /// + /// Total, and exhaustive in both directions: a variant added on either side + /// without its twin is a compile error here rather than a silent + /// fallthrough to whichever hash happened to be first. + #[cfg(feature = "cuda")] + pub const fn into_math_cuda(self) -> math_cuda::DeviceHash { + match self { + Self::Keccak256 => math_cuda::DeviceHash::Keccak256, + Self::Rpx256 => math_cuda::DeviceHash::Rpx256, + Self::Poseidon1 => math_cuda::DeviceHash::Poseidon1, + } + } + + /// Children per node of a tree this key's kernels build: 4 under + /// Poseidon1, 2 under the others. + pub const fn arity(self) -> usize { + match self { + Self::Keccak256 | Self::Rpx256 => 2, + Self::Poseidon1 => 4, + } + } + + /// The name a tree built under this key may be called by. + pub const fn name(self) -> &'static str { + match self { + Self::Keccak256 => "keccak256", + Self::Rpx256 => "rpx256", + Self::Poseidon1 => "poseidon1-w16", + } + } +} + +/// ✓ The bridge is a bijection, checked at compile time rather than by reading +/// it: every key maps to the twin of the same name, and the names agree. +#[cfg(feature = "cuda")] +const _: () = { + const fn paired(key: DeviceHashKey, twin: math_cuda::DeviceHash) -> bool { + key.into_math_cuda() as u8 == twin as u8 + } + assert!(paired( + DeviceHashKey::Keccak256, + math_cuda::DeviceHash::Keccak256 + )); + assert!(paired(DeviceHashKey::Rpx256, math_cuda::DeviceHash::Rpx256)); + assert!(paired( + DeviceHashKey::Poseidon1, + math_cuda::DeviceHash::Poseidon1 + )); +}; + +/// ★ One WHIR hash configuration. +/// +/// Implementing it is the whole of adding a hash to this path: a unit struct, a +/// Merkle backend and a Fiat-Shamir configuration. Nothing in `whir_commit`, +/// `whir_round`, `whir_chain` or `stacked_eval` knows which one it has. +pub trait WhirHash: Copy + Clone + Default + Send + Sync + 'static { + /// The name a proof's roots may be called by — for banners, KATs and + /// diagnostics. Never absorbed into the transcript: the sponge IS this + /// hash, so two configurations' challenge streams diverge at the first + /// squeeze and a tag would separate nothing that is not already separate. + const NAME: &'static str; + + /// The kernel family the device must run for trees labelled `Self`. + /// + /// Not `#[cfg(feature = "cuda")]`: a configuration names its device hash on + /// every build, so a non-cuda build cannot define a configuration that would + /// have had nothing to dispatch on. + const DEVICE: DeviceHashKey; + + /// The Fiat-Shamir configuration this commitment hash is paired with — the + /// sponge's hash, and the one the proof-of-work grind computes over. + type Transcript: TranscriptHash; + + /// The Merkle backend: one leaf per fold block, 32-byte nodes. + type Backend: IsMerkleTreeBackend>> + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The block's Fiat–Shamir sponge over the cubic extension (the only + /// challenge field the block uses): the transcript a block proof's prover + /// and verifier start from ([`Self::sponge`]). Its configuration is + /// `Self::Transcript`, so a sponge of another hash cannot be named here. + /// Its `Default` is [`Self::sponge`]: a block's group draws its + /// commit-time out-of-domain point from a fresh one + /// (`stark::multilinear_block::commit_ood_point`). + type Sponge: IsTranscript + + HasTranscriptHash + + Clone + + Send + + Default; + + /// A fresh sponge: what a block proof's statement is absorbed into. + fn sponge() -> Self::Sponge; +} + +/// The keccak-256 configuration — the default everywhere on this path, and +/// byte-for-byte what PR #988 produces. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct KeccakWhir; + +impl WhirHash for KeccakWhir { + const NAME: &'static str = "keccak256"; + + const DEVICE: DeviceHashKey = DeviceHashKey::Keccak256; + + type Transcript = KeccakTranscriptHash; + + type Backend + = BatchKeccak256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Sponge = DefaultTranscript; + + fn sponge() -> Self::Sponge { + DefaultTranscript::new(&[]) + } +} + +/// ★ The RPX256 configuration — the algebraic hash, and the only reason this +/// trait exists. +/// +/// Slower than keccak on a host by a wide margin, and that is not a defect to +/// be fixed: the lever it pulls is elsewhere. A keccak-f[1600] costs roughly +/// 73,700 trace cells in a field-native verifier against RPX's 325, so a WHIR +/// proof verified inside another proof pays about 227x less for its hashing +/// under this configuration. A proof that will only ever be checked by a host +/// should use [`KeccakWhir`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct RpxWhir; + +impl WhirHash for RpxWhir { + const NAME: &'static str = "rpx256"; + + const DEVICE: DeviceHashKey = DeviceHashKey::Rpx256; + + type Transcript = RpxTranscriptHash; + + type Backend + = BatchRpx256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// `DefaultTranscript::new(&[])`: the transcript the block has always + /// started from under RPX, byte for byte. + type Sponge = DefaultTranscript; + + fn sponge() -> Self::Sponge { + DefaultTranscript::new(&[]) + } +} + +/// ★ ZisK's Poseidon1 configuration (D-WHIR-P1 §2.1): the tagged +/// `linear_hash` leaf over a fold block's felts, 4-ary trees with zero +/// padding, ZisK's field sponge ([`P1Transcript`]) and the width-8 grind. +/// +/// Its trees have four children per node (`Backend::ARITY = 4`): a path +/// carries three siblings per 4-ary level, and its caps count 4-ary levels +/// (`ChainFormat::arity4_cap`). Opt-in; RPX stays the default. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct P1Whir; + +impl WhirHash for P1Whir { + const NAME: &'static str = "poseidon1-w16"; + + const DEVICE: DeviceHashKey = DeviceHashKey::Poseidon1; + + type Transcript = P1TranscriptHash; + + type Backend + = P1BatchBackend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Sponge = P1Transcript; + + fn sponge() -> Self::Sponge { + P1Transcript::new() + } +} + +/// Children per node of `H`'s trees over `F` (2, or 4 under [`P1Whir`]). +pub fn arity() -> usize +where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + H: WhirHash, +{ + as IsMerkleTreeBackend>::ARITY +} + +/// ✓ Both configurations are INHABITED at the fields the prover actually +/// commits over — the base field for traces, the cubic extension for the folded +/// codewords — within one proof. +/// +/// A `WhirHash` impl that type-checks in isolation can still be unusable: the +/// associated type is generic over `F`, and the bound that matters is the one +/// the prover instantiates it at. This is that instantiation, as a compile-time +/// check rather than a comment claiming it holds. +const _: fn() = || { + fn assert_usable() + where + H::Backend: + IsMerkleTreeBackend, + H::Backend: + IsMerkleTreeBackend, + { + } + + assert_usable::(); + assert_usable::(); + assert_usable::(); +}; + +/// ✓ Each configuration's transcript is ITS OWN, not the other's. +/// +/// The half-flip is unspellable because one trait supplies both halves — but +/// only if the two impls actually name different transcripts. A copy-paste that +/// left `RpxWhir` on `KeccakTranscriptHash` would be exactly the silent +/// configuration this design exists to rule out, so it is made a compile error. +const _: fn() = || { + fn assert_same(_: core::marker::PhantomData<(T, T)>) {} + + assert_same::( + core::marker::PhantomData::<(KeccakTranscriptHash, ::Transcript)>, + ); + assert_same::( + core::marker::PhantomData::<(RpxTranscriptHash, ::Transcript)>, + ); + assert_same::( + core::marker::PhantomData::<(P1TranscriptHash, ::Transcript)>, + ); +}; + +/// ✓ A configuration and its device key answer to the SAME name. +/// +/// Both are string constants written by hand, so nothing but this stops +/// `RpxWhir::NAME` from saying `rpx256` while its kernels are filed under +/// `keccak256` — which is precisely the mislabelling the key exists to prevent, +/// reintroduced one level up. +const _: () = { + const fn same(a: &str, b: &str) -> bool { + let (a, b) = (a.as_bytes(), b.as_bytes()); + if a.len() != b.len() { + return false; + } + let mut i = 0; + while i < a.len() { + if a[i] != b[i] { + return false; + } + i += 1; + } + true + } + assert!(same(KeccakWhir::NAME, KeccakWhir::DEVICE.name())); + assert!(same(RpxWhir::NAME, RpxWhir::DEVICE.name())); + assert!(same(P1Whir::NAME, P1Whir::DEVICE.name())); +}; + +#[cfg(test)] +mod sponge_default_tests { + use super::{KeccakWhir, P1Whir, RpxWhir, WhirHash}; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + }; + + /// A default sponge is a fresh one: what a block group's commit-time point + /// is drawn from, so both must agree draw for draw. + fn same() { + let mut fresh = H::sponge(); + let mut default = H::Sponge::default(); + assert_eq!(fresh.state(), default.state()); + for t in [&mut fresh, &mut default] { + t.append_bytes(b"commit-time point"); + } + let a: FieldElement = fresh.sample_field_element(); + let b: FieldElement = default.sample_field_element(); + assert_eq!(a, b); + } + + #[test] + fn a_default_sponge_is_a_fresh_one() { + same::(); + same::(); + same::(); + } +} diff --git a/crypto/multilinear/src/whir_p1_tests.rs b/crypto/multilinear/src/whir_p1_tests.rs new file mode 100644 index 000000000..f99bea72a --- /dev/null +++ b/crypto/multilinear/src/whir_p1_tests.rs @@ -0,0 +1,712 @@ +//! D-WHIR-P1 S1: ZisK's Poseidon1 ([`P1Whir`]) on the WHIR chain, on the host. +//! Its trees are 4-ary: a path carries three siblings per 4-ary level, a cap +//! counts 4-ary levels (`ChainFormat::arity4_cap`), and an odd-depth tree's top +//! group is two real nodes and two padding digests. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::hash::poseidon1_stark::{Merkle4, linear_hash}; +use crypto::hash::rpx::digest_to_commitment; +use crypto::merkle_tree::cap::{CapPolicy, cap_len}; +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as F, +}; + +use crate::{ + Error, + mle::Mle, + whir::Domain, + whir_chain::{ + ChainConfig, ChainFormat, ChainProof, ChainRound, GrindBits, RoundOpenings, commit, prove, + verify, + }, + whir_commit::{ + Codeword, CodewordCommitment, Commitment, TreeDrop, tree_cap_height, verify_opening, + }, + whir_hash::{KeccakWhir, P1Whir, RpxWhir, WhirHash}, +}; + +type FE = FieldElement; +type EE = FieldElement; + +fn config(log_folding: usize, num_queries: usize, arity4_cap: CapPolicy) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding, + num_queries, + grind: GrindBits::default(), + format: ChainFormat { + arity4_cap, + ..ChainFormat::DEFAULT + }, + } +} + +/// The configuration's own sponge with a test tag absorbed: the transcript a +/// block proof under `H` starts from. +fn sponge() -> H::Sponge { + let mut t = H::sponge(); + t.append_bytes(b"whir-p1"); + t +} + +struct Chain { + proof: ChainProof, + root: Commitment, + z: Vec, + y: EE, + domain: Domain, +} + +fn poly(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from((i.wrapping_add(seed)).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11)) + .collect(), + ) + .unwrap() +} + +/// A base-field polynomial proved over the cubic tower, as production does. +fn prove_chain(num_vars: usize, cfg: &ChainConfig, seed: u64) -> Chain { + let f = poly(num_vars, seed); + let z: Vec = (0..num_vars) + .map(|i| EE::from(101 + 3 * i as u64 + seed)) + .collect(); + let y = f.evaluate_in(&z).unwrap(); + let (commitment, domain) = commit::(&f, cfg, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, cfg, &mut sponge::()).unwrap(); + Chain { + proof, + root: commitment.root(), + z, + y, + domain, + } +} + +/// `proof` against `c`'s statement under `V`'s hash and sponge. +fn check( + c: &Chain, + proof: &ChainProof, + cfg: &ChainConfig, +) -> Result<(), Error> { + verify::( + proof, + &c.root, + &c.z, + c.y, + &c.domain, + cfg, + &mut sponge::(), + ) +} + +fn path_lens(round: &ChainRound) -> (Vec, Vec) { + match &round.openings { + RoundOpenings::Base(p) => ( + p.current + .iter() + .map(|o| o.proof.merkle_path.len()) + .collect(), + p.next.iter().map(|o| o.proof.merkle_path.len()).collect(), + ), + RoundOpenings::Extension(p) => ( + p.current + .iter() + .map(|o| o.proof.merkle_path.len()) + .collect(), + p.next.iter().map(|o| o.proof.merkle_path.len()).collect(), + ), + RoundOpenings::BaseGrouped(_) => panic!("these chains are per-polynomial"), + } +} + +fn current_path(round: &mut ChainRound, i: usize) -> &mut Vec { + match &mut round.openings { + RoundOpenings::Base(p) => &mut p.current[i].proof.merkle_path, + RoundOpenings::Extension(p) => &mut p.current[i].proof.merkle_path, + RoundOpenings::BaseGrouped(_) => panic!("these chains are per-polynomial"), + } +} + +fn next_path(round: &mut ChainRound, i: usize) -> &mut Vec { + match &mut round.openings { + RoundOpenings::Base(p) => &mut p.next[i].proof.merkle_path, + RoundOpenings::Extension(p) => &mut p.next[i].proof.merkle_path, + RoundOpenings::BaseGrouped(_) => panic!("these chains are per-polynomial"), + } +} + +/// Binary depth (`log2` leaves) of each of a chain's trees. +fn tree_depths(cfg: &ChainConfig, num_vars: usize) -> Vec { + let mut d = num_vars + cfg.log_blowup; + cfg.schedule(num_vars) + .iter() + .map(|k| { + d -= k; + d + }) + .collect() +} + +const SHAPES: [(usize, usize); 4] = [(6, 2), (5, 2), (3, 4), (9, 3)]; +const POLICIES: [CapPolicy; 4] = [ + CapPolicy::Off, + CapPolicy::Fixed(1), + CapPolicy::Fixed(2), + CapPolicy::Fixed(3), +]; + +/// The production chain's trees at arity 4: depths 23 … 2 are 12 … 1 levels, +/// and `Fixed(2)` caps each at two levels (one on the one-level last tree). +/// At arity 2 the arity-4 policy is never read. +#[test] +fn tree_caps_at_arity_4_count_four_ary_levels() { + let mut cfg = ChainConfig::with_security(2, 4, 25, 128, GrindBits::uniform(20)); + assert_eq!(tree_depths(&cfg, 25), vec![23, 19, 15, 11, 7, 3, 2]); + cfg.format.arity4_cap = CapPolicy::Fixed(2); + assert_eq!(cfg.tree_caps_at(25, 4), vec![2, 2, 2, 2, 2, 2, 1]); + cfg.format.arity4_cap = CapPolicy::Fixed(9); + assert_eq!( + cfg.tree_caps_at(25, 4), + vec![8, 8, 8, 6, 4, 2, 1], + "clamped to the levels and to 2^MAX_CAP_HEIGHT nodes" + ); + cfg.format.arity4_cap = CapPolicy::Off; + assert_eq!(cfg.tree_caps_at(25, 4), vec![0; 7]); + for binary in [CapPolicy::Off, CapPolicy::Auto, CapPolicy::Fixed(5)] { + for four in [CapPolicy::Off, CapPolicy::Fixed(2)] { + cfg.format.cap = binary; + cfg.format.arity4_cap = four; + assert_eq!( + cfg.tree_caps_at(25, 2), + cfg.tree_caps(25), + "{binary} / {four}" + ); + let off = ChainFormat { + arity4_cap: CapPolicy::Off, + ..cfg.format + }; + assert_eq!( + cfg.tree_caps(25), + ChainConfig { format: off, ..cfg }.tree_caps(25), + "the arity-4 cap moves no binary height" + ); + } + } + assert_eq!(tree_cap_height(CapPolicy::Fixed(3), 10, 5, 2), 3); + assert_eq!(tree_cap_height(CapPolicy::Fixed(3), 10, 5, 4), 3); + assert_eq!(tree_cap_height(CapPolicy::Fixed(3), 10, 3, 4), 2); + assert_eq!(tree_cap_height(CapPolicy::Fixed(3), 0, 9, 4), 0); +} + +/// Round trips at every arity-4 cap, one- and multi-round schedules, odd and +/// even depths, base and extension rounds; every path carries exactly +/// `3(h − c)` siblings and the owner its `cap_len` cap nodes more. +#[test] +fn p1_chains_round_trip_with_the_owner_path_lengths() { + for (num_vars, k) in SHAPES { + for q in [3usize, 25] { + for policy in POLICIES { + let cfg = config(k, q, policy); + let caps = cfg.tree_caps_at(num_vars, 4); + let depths = tree_depths(&cfg, num_vars); + let c = prove_chain::(num_vars, &cfg, 5); + let tag = format!("S={num_vars} k={k} Q={q} {policy}"); + let len = |t: usize, i: usize, owned: bool| { + let walked = 3 * (depths[t].div_ceil(2) - caps[t]); + walked + + if owned && i == 0 && caps[t] > 0 { + cap_len(depths[t], caps[t], 4).unwrap() + } else { + 0 + } + }; + for (r, round) in c.proof.rounds.iter().enumerate() { + let (cur, nxt) = path_lens(round); + for (i, l) in cur.iter().enumerate() { + assert_eq!(*l, len(r, i, r == 0), "{tag}: round {r} current {i}"); + } + for (i, l) in nxt.iter().enumerate() { + assert_eq!(*l, len(r + 1, i, true), "{tag}: round {r} next {i}"); + } + } + check::(&c, &c.proof, &cfg).unwrap_or_else(|e| panic!("{tag}: {e:?}")); + } + } + } +} + +/// The arity-4 cap is inert under a binary hash: an RPX proof under +/// `arity4_cap = Fixed(2)` is the default format's, byte for byte. +#[test] +fn an_arity4_cap_moves_no_rpx_byte() { + for (num_vars, k) in SHAPES { + let a = prove_chain::(num_vars, &config(k, 3, CapPolicy::Off), 1); + let b = prove_chain::(num_vars, &config(k, 3, CapPolicy::Fixed(2)), 1); + assert_eq!( + rkyv::to_bytes::(&a.proof) + .unwrap() + .as_slice(), + rkyv::to_bytes::(&b.proof) + .unwrap() + .as_slice(), + "S={num_vars} k={k}" + ); + } +} + +/// The hashes do not cross-verify: a P1 proof is refused by an RPX or a +/// keccak verifier, and an RPX proof by a P1 verifier, whatever the cap. +#[test] +fn the_bases_do_not_cross_verify() { + for policy in [CapPolicy::Off, CapPolicy::Fixed(2)] { + let cfg = config(2, 3, policy); + let p1 = prove_chain::(6, &cfg, 4); + check::(&p1, &p1.proof, &cfg).unwrap(); + assert!(check::(&p1, &p1.proof, &cfg).is_err()); + assert!(check::(&p1, &p1.proof, &cfg).is_err()); + let rpx = prove_chain::(6, &cfg, 4); + check::(&rpx, &rpx.proof, &cfg).unwrap(); + assert!(check::(&rpx, &rpx.proof, &cfg).is_err()); + } +} + +fn expect_err(c: &Chain, forged: &ChainProof, cfg: &ChainConfig, what: &str) -> Error { + match check::(c, forged, cfg) { + Ok(()) => panic!("{what}: the forgery verified"), + Err(e) => e, + } +} + +/// The tamper arm, `[2, 2, 2]` over 6 variables at `Fixed(1)`: trees of +/// binary depth 6, 4, 2 (3, 2, 1 levels), each capped at one 4-ary level (a +/// cap of four nodes). +#[test] +fn a_tampered_p1_cap_or_path_is_rejected() { + let cfg = config(2, 3, CapPolicy::Fixed(1)); + assert_eq!(cfg.tree_caps_at(6, 4), vec![1, 1, 1]); + let c = prove_chain::(6, &cfg, 9); + check::(&c, &c.proof, &cfg).unwrap(); + // Walked siblings below the cap, per tree: 3 × (levels − 1). + let walked = [6usize, 3, 0]; + + // Every node of tree 0's cap (the end of round 0's first current path). + for j in 0..4 { + let mut forged = c.proof.clone(); + current_path(&mut forged.rounds[0], 0)[walked[0] + j][5] ^= 1; + assert!(matches!( + expect_err(&c, &forged, &cfg, "tree-0 cap"), + Error::CapRejected + )); + } + // Every node of tree t's cap, t = 1, 2 (round t − 1's first successor path). + for (t, w) in walked.iter().enumerate().skip(1) { + for j in 0..4 { + let mut forged = c.proof.clone(); + next_path(&mut forged.rounds[t - 1], 0)[w + j][0] ^= 1; + assert!(matches!( + expect_err(&c, &forged, &cfg, "tree-t cap"), + Error::CapRejected + )); + } + } + // Every sibling of a non-owner path below the cap. + for s in 0..walked[0] { + let mut forged = c.proof.clone(); + current_path(&mut forged.rounds[0], 1)[s][3] ^= 1; + assert!(matches!( + expect_err(&c, &forged, &cfg, "sibling"), + Error::OpeningRejected { query: 1 } + )); + } + // The owner path one short and one long. + let mut forged = c.proof.clone(); + current_path(&mut forged.rounds[0], 0).pop(); + assert!(matches!( + expect_err(&c, &forged, &cfg, "owner short"), + Error::CapRejected + )); + let mut forged = c.proof.clone(); + let extra = current_path(&mut forged.rounds[0], 0)[0]; + current_path(&mut forged.rounds[0], 0).push(extra); + assert!(matches!( + expect_err(&c, &forged, &cfg, "owner long"), + Error::CapRejected + )); + // A non-owner path one long, and one short. + let mut forged = c.proof.clone(); + let extra = current_path(&mut forged.rounds[0], 1)[0]; + current_path(&mut forged.rounds[0], 1).push(extra); + assert!(matches!( + expect_err(&c, &forged, &cfg, "non-owner long"), + Error::OpeningRejected { query: 1 } + )); + let mut forged = c.proof.clone(); + assert!(next_path(&mut forged.rounds[0], 2).pop().is_some()); + assert!(matches!( + expect_err(&c, &forged, &cfg, "non-owner short"), + Error::OpeningRejected { query: 2 } + )); + // The cap moved from the first opening to the second. + let mut forged = c.proof.clone(); + let cap: Vec = current_path(&mut forged.rounds[0], 0) + .drain(walked[0]..) + .collect(); + current_path(&mut forged.rounds[0], 1).extend(cap); + expect_err(&c, &forged, &cfg, "cap moved to query 1"); + + // The height is the verifier's constant, never read from the proof. + for other in [CapPolicy::Off, CapPolicy::Fixed(2)] { + expect_err(&c, &c.proof, &config(2, 3, other), "another arity-4 cap"); + } + let off = prove_chain::(6, &config(2, 3, CapPolicy::Off), 9); + expect_err(&off, &off.proof, &cfg, "off read as c=1"); +} + +fn codeword(log_len: usize, seed: u64) -> Vec { + (0..1u64 << log_len) + .map(|i| FE::from(i.wrapping_mul(0x9E37_79B9).wrapping_add(seed) >> 3)) + .collect() +} + +/// Every binary depth 1–9 at cap 0: every opening verifies with exactly +/// `3⌈d/2⌉` siblings, and none at an index past the tree. At an odd depth the +/// walk reaches the top group's two padding siblings: they are the zero digest, +/// and anything else there is refused. Here the honest root hashes zeros, so a +/// moved sibling misses it; but the root does not bind the padding in general +/// — a prover chooses the root, and may commit non-zero padding at the top. The +/// host's zero check (REV-P1-JUDGE F4) is what refuses that (covered by +/// `rp1a_uncapped_odd_top_padding_is_bound_on_the_host`), which a mutation +/// dropping it leaves this test green. +#[test] +fn every_depth_opens_and_an_odd_tops_padding_is_bound() { + for depth in 1..=9usize { + for k in [0usize, 2] { + let cw = codeword(depth + k, depth as u64); + let commitment = CodewordCommitment::::new(&cw, k).unwrap(); + assert_eq!(commitment.depth(), depth); + assert_eq!(commitment.levels(), depth.div_ceil(2)); + let root = commitment.root(); + for j in 0..commitment.num_leaves() { + let opening = commitment.open(j).unwrap(); + assert_eq!(opening.proof.merkle_path.len(), 3 * depth.div_ceil(2)); + assert!( + verify_opening::(&root, depth, j, &opening), + "d={depth} k={k} leaf {j}" + ); + assert!( + !verify_opening::( + &root, + depth, + j + commitment.num_leaves(), + &opening + ), + "d={depth} k={k}: an index past the tree" + ); + if depth % 2 == 1 { + let n = opening.proof.merkle_path.len(); + for at in [n - 2, n - 1] { + assert_eq!(opening.proof.merkle_path[at], [0u8; 32], "padding"); + let mut forged = opening.clone(); + forged.proof.merkle_path[at][31] ^= 1; + assert!( + !verify_opening::(&root, depth, j, &forged), + "d={depth} k={k} leaf {j}: a moved padding sibling {at}" + ); + } + } + } + } + } +} + +/// A P1 root is ZisK's tree: the tagged `linear_hash` of each fold block's +/// felts in coset order (an ext3 value's three coefficients in order), folded +/// by `Merkle4` with zero padding — at odd and even depths, base and ext3. +#[test] +fn a_p1_root_is_zisks_merkle4_over_the_blocks() { + for (log_len, k) in [(5usize, 2usize), (6, 2), (7, 4), (9, 3), (10, 4)] { + let cw = codeword(log_len, 3); + let commitment = CodewordCommitment::::new(&cw, k).unwrap(); + let n = commitment.num_leaves(); + let leaves: Vec<_> = (0..n) + .map(|j| { + let felts: Vec = (0..1usize << k).map(|t| cw[j + t * n]).collect(); + linear_hash(&felts) + }) + .collect(); + let want = digest_to_commitment(&Merkle4::new(&leaves).unwrap().root()); + assert_eq!(commitment.root(), want, "base log_len={log_len} k={k}"); + + let ext: Vec = cw + .iter() + .enumerate() + .map(|(i, v)| EE::new([*v, FE::from(i as u64), v.square()])) + .collect(); + let commitment = CodewordCommitment::::new(&ext, k).unwrap(); + let leaves: Vec<_> = (0..n) + .map(|j| { + let felts: Vec = (0..1usize << k) + .flat_map(|t| ext[j + t * n].value().to_vec()) + .collect(); + linear_hash(&felts) + }) + .collect(); + let want = digest_to_commitment(&Merkle4::new(&leaves).unwrap().root()); + assert_eq!(commitment.root(), want, "ext3 log_len={log_len} k={k}"); + } +} + +/// The drop at arity 4 counts 4-ary levels, half the binary request (the same +/// leaves a block), never past the frontier where a kept node covers `4^d` +/// whole leaves, and never into the tallest cap. +#[test] +fn the_arity4_drop_halves_the_request_and_stops_at_whole_groups() { + let drop = TreeDrop { device: 8, host: 4 }; + assert_eq!(drop.levels_at(true, 23, 2, 4), 4); + assert_eq!(drop.levels_at(false, 23, 2, 4), 2); + assert_eq!(drop.levels_at(true, 23, 3, 2), 8, "binary as before"); + assert_eq!( + drop.levels_at(true, 7, 0, 4), + 3, + "odd: the two-node top stays" + ); + assert_eq!(drop.levels_at(true, 8, 0, 4), 4, "even: down to the root"); + assert_eq!(drop.levels_at(true, 7, 2, 4), 2, "never into the cap"); + assert_eq!(drop.levels_at(true, 3, 0, 4), 1); + assert_eq!(drop.levels_at(true, 1, 0, 4), 0); +} + +/// A retired and revived P1 commitment serves the whole tree's paths byte for +/// byte, capped or not, at odd and even depths and every drop; the owner's cap +/// is read from the kept levels; and a wrong codeword is refused when a path +/// is read. +#[test] +fn a_retired_p1_commitment_serves_the_full_tree_paths() { + for (log_len, k) in [(9usize, 2usize), (10, 2), (12, 1)] { + let cw = codeword(log_len, 17); + let full = CodewordCommitment::::new(&cw, k).unwrap(); + let n = full.num_leaves(); + let indices: Vec = (0..12usize).map(|q| (q * 2_654_435_761) % n).collect(); + for policy in [CapPolicy::Off, CapPolicy::Fixed(1), CapPolicy::Fixed(2)] { + let c = tree_cap_height(policy, indices.len(), full.depth(), 4); + let want = full.open_many_capped(&indices, c, true).unwrap(); + for d in [0usize, 2, 3, 4, 6, 8] { + let tag = format!("log_len={log_len} k={k} {policy} drop {d}"); + let retired = CodewordCommitment::::new(&cw, k) + .unwrap() + .retire(TreeDrop::uniform(d), policy) + .unwrap(); + assert_eq!(retired.root(), full.root(), "{tag}"); + let revived = retired + .clone() + .revive::(Codeword::Host(cw.clone())) + .unwrap(); + let got = revived.open_many_capped(&indices, c, true).unwrap(); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "{tag}"); + let mut bad = cw.clone(); + for j in &indices { + bad[*j] += FE::one(); + } + let wrong = retired.revive::(Codeword::Host(bad)).unwrap(); + assert!( + matches!( + wrong.open_many_capped(&indices, c, true), + Err(Error::RecomputedCodewordMismatch { .. }) + ), + "{tag}: a wrong codeword" + ); + } + } + } +} + +/// A stacked opening under P1, as the block's groups use it: committed, +/// proved and verified over the extension; a stack retired and revived opens +/// to the same bytes at every drop and verifies against the committed roots; +/// and the same proof under another arity-4 cap or another hash is refused. +#[test] +fn a_p1_stacked_opening_round_trips_and_survives_a_retire() { + use crate::stacked_eval::{Claimed, StackedCommitment, prove, verify}; + use crate::stacking::{StackedLayout, borrow}; + + let num_vars = 6; + let columns: Vec> = (0..5u64).map(|i| poly(num_vars, 7 + 13 * i)).collect(); + let layout = StackedLayout::build(&[num_vars; 5], 8).unwrap(); + assert!(layout.num_polys() >= 2); + let at: Vec = (0..num_vars) + .map(|i| EE::new([FE::from(3 + i as u64), FE::from(5), FE::from(i as u64)])) + .collect(); + let claimed: Vec = columns + .iter() + .map(|c| c.evaluate_in(&at).unwrap()) + .collect(); + for policy in [CapPolicy::Off, CapPolicy::Fixed(1), CapPolicy::Fixed(2)] { + let cfg = config(2, 3, policy); + let kept = + StackedCommitment::::commit(layout.clone(), &borrow(&columns), None, &cfg) + .unwrap(); + let roots = kept.roots(); + let proof = prove( + &kept, + &borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed, + &cfg, + &mut sponge::(), + ) + .unwrap(); + let verified = |p: &crate::stacked_eval::StackedProof, cfg: &ChainConfig| { + verify::( + p, + &layout, + &roots, + &Claimed::Shared(&at), + &claimed, + kept.domain(), + cfg, + &mut sponge::(), + ) + }; + verified(&proof, &cfg).unwrap_or_else(|e| panic!("{policy}: {e:?}")); + let other = if policy == CapPolicy::Fixed(1) { + CapPolicy::Fixed(2) + } else { + CapPolicy::Fixed(1) + }; + assert!( + verified(&proof, &config(2, 3, other)).is_err(), + "{policy} read as {other}" + ); + assert!( + verify::( + &proof, + &layout, + &roots, + &Claimed::Shared(&at), + &claimed, + kept.domain(), + &cfg, + &mut sponge::(), + ) + .is_err(), + "{policy}: an RPX verifier" + ); + let want = format!("{proof:?}"); + for drop in [0usize, 1, 3, 8, 64] { + let retired = StackedCommitment::::commit( + layout.clone(), + &borrow(&columns), + None, + &cfg, + ) + .unwrap() + .retire(TreeDrop::uniform(drop), &cfg) + .unwrap(); + assert_eq!(retired.roots(), roots); + let revived = retired + .revive::(&borrow(&columns), None, &cfg) + .unwrap(); + let again = prove( + &revived, + &borrow(&columns), + None, + &Claimed::Shared(&at), + &claimed, + &cfg, + &mut sponge::(), + ) + .unwrap(); + assert_eq!(format!("{again:?}"), want, "{policy} drop {drop}"); + } + } +} + +/// ★ A WHIR P1 tree's internal node never opens as a leaf: the node one 4-ary +/// level above block 0, with the path from it upward (three siblings short), +/// lands on the cap at index 0 — only the capped check's exact length refuses +/// it. At odd and even depths, uncapped and capped. +#[test] +fn an_internal_node_never_opens_as_a_leaf() { + use crypto::merkle_tree::backends::p1::P1BatchBackend; + use crypto::merkle_tree::cap::{ + split_owner_path_arity, verify_merkle_path_to_cap_from_leaf_hash, + }; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + type B = P1BatchBackend; + + for (log_len, k) in [(9usize, 2usize), (10, 2), (12, 3)] { + let cw = codeword(log_len, 29); + let commitment = CodewordCommitment::::new(&cw, k).unwrap(); + let d = commitment.depth(); + let n = commitment.num_leaves(); + for c in [0usize, 1, 2] { + let tag = format!("d={d} c={c}"); + let owner = commitment + .open_many_capped(&[0], c, true) + .unwrap() + .remove(0); + let (siblings, cap) = + split_owner_path_arity(&owner.proof.merkle_path, d, c, 4).expect("an owner path"); + // At c = 0 the owner path carries no cap: the cap is the root. + let root = [commitment.root()]; + let cap = if c == 0 { &root[..] } else { cap }; + let leaf = B::hash_data(&owner.values); + assert!( + verify_merkle_path_to_cap_from_leaf_hash::(siblings, cap, d, 0, leaf), + "{tag}: the honest opening" + ); + let block0: Vec = (0..1usize << k).map(|t| cw[t * n]).collect(); + assert_eq!(owner.values, block0); + let parent = B::hash_four(&[leaf, siblings[0], siblings[1], siblings[2]]); + assert!( + !verify_merkle_path_to_cap_from_leaf_hash::(&siblings[3..], cap, d, 0, parent), + "{tag}: an internal node opened as leaf 0" + ); + } + } +} + +/// REV-P1W-A: a P1 chain verifies under its own arity-4 cap only. Proved at +/// `Fixed(2)` (the WHIR block's C2) over trees of odd and even depth, it is +/// refused under every other cap policy that yields another geometry and under +/// the binary hashes; `Off` and `Fixed(0)` are one geometry (the tag renders +/// both `C0`) and cross-verify, as they must. +#[test] +fn rp1wa_a_p1_chain_verifies_under_its_own_cap_only() { + for (num_vars, log_folding) in [(7usize, 2usize), (8, 3), (9, 2)] { + let c2 = config(log_folding, 5, CapPolicy::Fixed(2)); + let chain = prove_chain::(num_vars, &c2, 17); + check::(&chain, &chain.proof, &c2).unwrap(); + for other in [ + CapPolicy::Off, + CapPolicy::Fixed(0), + CapPolicy::Fixed(1), + CapPolicy::Fixed(3), + CapPolicy::Auto, + ] { + let cfg = config(log_folding, 5, other); + if cfg.tree_caps_at(num_vars, 4) == c2.tree_caps_at(num_vars, 4) { + continue; + } + assert!( + check::(&chain, &chain.proof, &cfg).is_err(), + "n {num_vars} k {log_folding}: a C2 proof verified under {other:?}" + ); + } + assert!(check::(&chain, &chain.proof, &c2).is_err()); + assert!(check::(&chain, &chain.proof, &c2).is_err()); + let off = config(log_folding, 5, CapPolicy::Off); + let uncapped = prove_chain::(num_vars, &off, 18); + check::( + &uncapped, + &uncapped.proof, + &config(log_folding, 5, CapPolicy::Fixed(0)), + ) + .unwrap(); + } +} diff --git a/crypto/multilinear/src/whir_round.rs b/crypto/multilinear/src/whir_round.rs new file mode 100644 index 000000000..1cba97468 --- /dev/null +++ b/crypto/multilinear/src/whir_round.rs @@ -0,0 +1,1187 @@ +//! One WHIR round: sample queries, open the current codeword's blocks, fold them +//! locally, and check they match the committed successor. +//! +//! The successor is committed before the queries are drawn, so it cannot be +//! chosen to match. Consistency holds only where the queries land. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::CappedRoot; +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; + +use crate::{ + Error, + whir::Domain, + whir_commit::{ + CodewordCommitment, Commitment, CosetOpening, LeafOrder, fold_coset, leaf_and_slot, + verify_opening_capped, + }, + whir_hash::WhirHash, +}; + +type Backend = ::Backend; + +/// How hard a round is to cheat. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RoundConfig { + /// Positions checked. Each one an inconsistent prover must get lucky on. + pub num_queries: usize, + /// `k`: the codeword folds by `2^k` and blocks hold `2^k` values. + pub log_folding: usize, +} + +/// How a tree's openings are authenticated in a round. +/// +/// A tree is authenticated ONCE: tree 0 by the +/// cap its first opening in round 0 carries, and tree `t ≥ 1` by the cap its +/// first opening as round `t − 1`'s SUCCESSOR carries. Round `t` then opens +/// tree `t` as its current tree against that stored check, and never re-reads +/// a cap from its own first opening. +#[derive(Clone, Copy, Debug)] +pub enum TreeCheck<'a> { + /// Authenticated in an earlier round. + Checked(CappedRoot<'a, Commitment>), + /// Owned by this round: its first opening carries its cap of height + /// `cap_height` (none at 0). `leaves` is where the tree puts its blocks: + /// the chain's [`LeafOrder`] for a round-0 tree, natural for a successor. + Owner { + root: &'a Commitment, + cap_height: usize, + leaves: LeafOrder, + }, +} + +impl<'a> TreeCheck<'a> { + /// The tree position of block `index` in this tree of `2^depth` leaves. + /// A tree checked in an earlier round is a successor's, in natural order. + pub(crate) fn position(&self, index: usize, depth: usize) -> usize { + match self { + TreeCheck::Checked(_) => index, + TreeCheck::Owner { leaves, .. } => leaves.position(index, depth), + } + } + + /// The tree's check and the siblings of `first`, the tree's first opening + /// in this round. + /// + /// An [`Owner`](Self::Owner) tree's cap is split off `first`'s path and + /// authenticated against the root here — once, before any opening of the + /// tree is checked against it. At `cap_height = 0` there is no cap: the + /// whole path is siblings, and the per-query check enforces its length. + /// A [`Checked`](Self::Checked) tree must have the depth this round + /// derives, and `first` carries no cap: its whole path is siblings. + pub(crate) fn open( + self, + depth: usize, + first: &'a CosetOpening, + ) -> Result<(CappedRoot<'a, Commitment>, &'a [Commitment]), Error> + where + C: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + H: WhirHash, + { + let path = first.proof.merkle_path.as_slice(); + match self { + TreeCheck::Checked(check) => { + if check.depth() != depth { + return Err(Error::CapRejected); + } + Ok((check, path)) + } + TreeCheck::Owner { + root, + cap_height: 0, + .. + } => Ok((CappedRoot::uncapped(root, depth), path)), + TreeCheck::Owner { + root, cap_height, .. + } => CappedRoot::from_owner::>(root, path, depth, cap_height) + .ok_or(Error::CapRejected), + } + } +} + +/// What the verifier already knows about the two codewords: how the current +/// tree is authenticated, the successor's root, how the successor was blocked, +/// and its cap height. +#[derive(Clone, Copy, Debug)] +pub struct RoundCommitments<'a> { + pub current: TreeCheck<'a>, + pub next_root: &'a Commitment, + /// Leaves in the successor's tree, needed to locate a position in it. + pub next_num_leaves: usize, + /// The successor tree's cap height. Its cap rides this round's first + /// successor opening, which is that tree's first opening in proof order. + pub next_cap_height: usize, +} + +/// The cap heights a round opens its two trees under (prover side). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RoundCaps { + /// The current tree's cap height. + pub current: usize, + /// True when this round's first current opening is the current tree's + /// first opening in proof order (round 0), and so carries its cap. + pub current_owner: bool, + /// The successor tree's cap height. The successor is always owned by the + /// round that commits it. + pub next: usize, +} + +/// The openings one round sends. +/// +/// The two value fields differ in the first round of a chain: the committed +/// codeword is base-field, because a trace is, while its successor has been +/// folded with an extension challenge. Later rounds have `C = N`. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct RoundProof { + /// Per query: the block of the current codeword that folds onto the query. + pub current: Vec>, + /// Per query: the successor block holding the folded value. + pub next: Vec>, +} + +/// The openings round 0 of a grouped chain sends (D-ONECHAIN §2.4): the run's +/// `P'` round-0 trees opened at ONE position set. +/// +/// Query-major: `current[i]` holds query `i`'s block of every tree, in tree +/// order, so the shape a verifier checks — `P'` blocks a query, none for an +/// absent slot — is the shape of each entry. Tree `p`'s cap rides +/// `current[0][p]`, its first opening in proof order. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct GroupedRoundProof { + /// Per query: one block per round-0 tree, in tree order. + pub current: Vec>>, + /// Per query: the successor block holding the folded value. Empty when + /// round 0 is the last round. + pub next: Vec>, +} + +/// Draws the query positions. Both sides run this on the same transcript. +/// +/// Positions index the *folded* codeword, which is also the block index of the +/// current one. +fn sample_queries(transcript: &mut T, num_queries: usize, bound: usize) -> Vec +where + F: IsField, + T: IsTranscript, +{ + (0..num_queries) + .map(|_| transcript.sample_u64(bound as u64) as usize) + .collect() +} + +/// Produces the openings for one round. +/// +/// `current` and `next` must already be committed, and `next` must be the fold +/// of `current` by `alphas` — [`verify`] is what checks that claim. +/// +/// `caps` is the Merkle cap each tree is opened under ([`RoundCaps`]); the +/// default is no cap on either. +pub fn prove( + current: &CodewordCommitment, + next: &CodewordCommitment, + config: &RoundConfig, + caps: RoundCaps, + transcript: &mut T, +) -> Result, Error> +where + C: IsField + 'static, + N: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + // The transcript squeezes that choose the indices, and the map onto the + // successor's leaves. Host work, and the only part of a round's openings + // that is not `open_many`. + let __wq_sample = crate::whir_split::mark(); + let queries = sample_queries(transcript, config.num_queries, current.num_leaves()); + + let leaves: Vec = queries + .iter() + .map(|q| leaf_and_slot(*q, next.num_leaves()).0) + .collect(); + crate::whir_split::add(&crate::whir_split::QUERY_SAMPLE, __wq_sample); + + Ok(RoundProof { + current: current.open_many_capped(&queries, caps.current, caps.current_owner)?, + next: next.open_many_capped(&leaves, caps.next, true)?, + }) +} + +/// Checks a round against the two commitments. +/// +/// Re-derives the queries from the transcript, so the prover could not have +/// chosen them. Returns the successor tree's authenticated check, which the +/// next round opens its current tree against. +/// +/// ⚠ ORDER. The opening counts are checked before any opening is indexed or +/// any cap is read, so a proof with too few openings is refused and never +/// panics. +pub fn verify<'a, F, C, N, T, H>( + proof: &'a RoundProof, + commitments: RoundCommitments<'a>, + domain: &Domain, + alphas: &[FieldElement], + config: &RoundConfig, + transcript: &mut T, +) -> Result, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + IsSubFieldOf, + C: IsField + IsSubFieldOf + 'static, + N: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + if alphas.len() != config.log_folding { + return Err(Error::VariableCountMismatch { + expected: config.log_folding, + got: alphas.len(), + }); + } + if proof.current.len() != config.num_queries || proof.next.len() != config.num_queries { + return Err(Error::QueryCountMismatch { + expected: config.num_queries, + got: proof.current.len().min(proof.next.len()), + }); + } + + let num_leaves = domain.size() >> config.log_folding; + let current_depth = num_leaves.trailing_zeros() as usize; + if !commitments.next_num_leaves.is_power_of_two() { + return Err(Error::NotPowerOfTwo(commitments.next_num_leaves)); + } + let next_depth = commitments.next_num_leaves.trailing_zeros() as usize; + + // Both trees' checks, each built once from the tree's first opening — + // after the count guard above, so index 0 exists (M2). With no openings + // (`num_queries == 0`) no cap exists either: `CapPolicy::height` is 0 for + // an unopened tree, and the successor's check is its bare root. + let (current_check, current_first, next_check, next_first) = + match (proof.current.first(), proof.next.first()) { + (Some(cur), Some(nxt)) => { + let (current_check, current_first) = + commitments.current.open::(current_depth, cur)?; + let (next_check, next_first) = TreeCheck::Owner { + root: commitments.next_root, + cap_height: commitments.next_cap_height, + leaves: LeafOrder::Natural, + } + .open::(next_depth, nxt)?; + (current_check, current_first, next_check, next_first) + } + _ => { + if commitments.next_cap_height != 0 { + return Err(Error::CapRejected); + } + return Ok(CappedRoot::uncapped(commitments.next_root, next_depth)); + } + }; + + let queries = sample_queries(transcript, config.num_queries, num_leaves); + + for (i, (&q, (cur, nxt))) in queries + .iter() + .zip(proof.current.iter().zip(&proof.next)) + .enumerate() + { + let (cur_siblings, nxt_siblings) = if i == 0 { + (current_first, next_first) + } else { + ( + cur.proof.merkle_path.as_slice(), + nxt.proof.merkle_path.as_slice(), + ) + }; + let at = commitments.current.position(q, current_depth); + if !verify_opening_capped::(¤t_check, at, cur, cur_siblings) { + return Err(Error::OpeningRejected { query: i }); + } + let (leaf, slot) = leaf_and_slot(q, commitments.next_num_leaves); + if !verify_opening_capped::(&next_check, leaf, nxt, nxt_siblings) { + return Err(Error::OpeningRejected { query: i }); + } + + let folded = fold_coset::(&cur.values, domain, q, alphas)?; + let claimed = nxt.values.get(slot).ok_or(Error::QueryOutOfRange { + index: slot, + bound: nxt.values.len(), + })?; + if folded != *claimed { + return Err(Error::FoldInconsistent { query: i }); + } + } + + Ok(next_check) +} + +/// Produces round 0's openings in a grouped chain: every round-0 tree opened +/// at ONE position set, drawn once, and the successor at the same positions' +/// leaves — or no successor block when round 0 is the last round. +/// +/// Each tree is owned by this round (round 0), so each carries its cap on its +/// first opening, `current[0][p]`. +pub fn prove_grouped( + current: &[&CodewordCommitment], + next: Option<&CodewordCommitment>, + config: &RoundConfig, + caps: RoundCaps, + transcript: &mut T, +) -> Result, Error> +where + C: IsField + 'static, + N: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + let first = current.first().ok_or(Error::GroupedShape( + crate::whir_chain::GroupedShape::RunWidth { polys: 0 }, + ))?; + let __wq_sample = crate::whir_split::mark(); + let queries = sample_queries(transcript, config.num_queries, first.num_leaves()); + let leaves: Vec = match next { + Some(next) => queries + .iter() + .map(|q| leaf_and_slot(*q, next.num_leaves()).0) + .collect(), + None => Vec::new(), + }; + crate::whir_split::add(&crate::whir_split::QUERY_SAMPLE, __wq_sample); + + let mut rows: Vec>> = (0..queries.len()) + .map(|_| Vec::with_capacity(current.len())) + .collect(); + // Each tree past the first is an `open_many` call a per-polynomial round 0 + // does not make: arm F's identity counts it. + crate::whir_split::bump_by( + &crate::whir_split::GROUPED_TREES, + current.len().saturating_sub(1) as u64, + ); + for tree in current { + let opened = tree.open_many_capped(&queries, caps.current, caps.current_owner)?; + for (row, opening) in rows.iter_mut().zip(opened) { + row.push(opening); + } + } + let next = match next { + Some(next) => next.open_many_capped(&leaves, caps.next, true)?, + None => Vec::new(), + }; + Ok(GroupedRoundProof { + current: rows, + next, + }) +} + +/// Where a grouped round 0's folds must land: the committed successor, or, +/// when round 0 is the chain's last round, the constant the prover sent. +#[derive(Debug)] +pub enum GroupedEnd<'a, N: IsField> { + Successor { + root: &'a Commitment, + /// Leaves in the successor's tree. + num_leaves: usize, + cap_height: usize, + }, + Final(&'a FieldElement), +} + +// By hand: a derive would ask `N: Copy`, and the variants hold references. +impl Clone for GroupedEnd<'_, N> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for GroupedEnd<'_, N> {} + +/// The shape a grouped round 0's openings must have, checked whole before any +/// of them is indexed, zipped or hashed (REV-B2-JUDGE B1-2): `queries` queries, +/// each exactly `trees` blocks of exactly `block` values, and `successors` +/// successor blocks (one a query, or none on a final round). +fn check_grouped_shape( + proof: &GroupedRoundProof, + trees: usize, + block: usize, + queries: usize, + successors: usize, +) -> Result<(), Error> { + use crate::whir_chain::GroupedShape; + if proof.current.len() != queries { + return Err(Error::GroupedShape(GroupedShape::Queries { + expected: queries, + got: proof.current.len(), + })); + } + if proof.next.len() != successors { + return Err(Error::GroupedShape(GroupedShape::SuccessorBlocks { + expected: successors, + got: proof.next.len(), + })); + } + for (query, blocks) in proof.current.iter().enumerate() { + if blocks.len() != trees { + return Err(Error::GroupedShape(GroupedShape::BlocksPerQuery { + query, + expected: trees, + got: blocks.len(), + })); + } + for (tree, opening) in blocks.iter().enumerate() { + if opening.values.len() != block { + return Err(Error::GroupedShape(GroupedShape::BlockWidth { + query, + tree, + expected: block, + got: opening.values.len(), + })); + } + } + } + Ok(()) +} + +/// The coefficients a grouped round 0 combines its trees' blocks with: +/// `eq(ρ, p)` for each tree `p` ([`crate::whir_chain::eq_index`]). Only the +/// `trees` present slots: an absent slot is zero and has no block. +pub(crate) fn combine_weights( + rho: &[FieldElement], + trees: usize, +) -> Vec> { + #[cfg(test)] + if crate::whir_chain::grouped_faults::get() + == crate::whir_chain::grouped_faults::Fault::SwapRhoInCombine + { + let swapped: Vec> = rho.iter().rev().cloned().collect(); + return (0..trees) + .map(|p| crate::whir_chain::eq_index(&swapped, p)) + .collect(); + } + (0..trees) + .map(|p| crate::whir_chain::eq_index(rho, p)) + .collect() +} + +/// `Σ_p eqs[p]·block_p`, value by value: the one `2^k0` block a grouped round +/// 0 folds. The base values are lifted here, and only from the blocks the +/// trees authenticate. Widths were checked before ([`check_grouped_shape`]). +pub(crate) fn combine_blocks( + blocks: &[CosetOpening], + eqs: &[FieldElement], +) -> Vec> +where + C: IsField + IsSubFieldOf, + N: IsField, +{ + let width = blocks.first().map_or(0, |b| b.values.len()); + (0..width) + .map(|t| { + blocks + .iter() + .zip(eqs) + .fold(FieldElement::::zero(), |acc, (block, eq)| { + acc + &block.values[t] * eq + }) + }) + .collect() +} + +/// Checks round 0 of a grouped chain against its `P'` round-0 trees. +/// +/// Re-derives ONE position set from the transcript; at each position every +/// tree's block must open against that tree's own root, and the blocks' +/// combination `Σ_p eq(ρ, p)·block_p`, folded by the data challenges, must be +/// the successor's value there (or the final constant). Returns the +/// successor tree's authenticated check, `None` on a final round. +/// +/// ⚠ ORDER. The whole shape is checked before any opening is indexed or any +/// cap is read ([`check_grouped_shape`]), so a short or padded proof is +/// refused by name and never panics or zips. +#[allow(clippy::too_many_arguments)] +pub fn verify_grouped<'a, F, E, T, H>( + proof: &'a GroupedRoundProof, + trees: &[TreeCheck<'a>], + end: GroupedEnd<'a, E>, + domain: &Domain, + alphas: &[FieldElement], + rho: &[FieldElement], + config: &RoundConfig, + transcript: &mut T, +) -> Result>, Error> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + H: WhirHash, +{ + use crate::whir_chain::{GroupedShape, MAX_INDEX_POLYS, index_vars}; + if !(2..=MAX_INDEX_POLYS).contains(&trees.len()) { + return Err(Error::GroupedShape(GroupedShape::RunWidth { + polys: trees.len(), + })); + } + if alphas.len() != config.log_folding { + return Err(Error::VariableCountMismatch { + expected: config.log_folding, + got: alphas.len(), + }); + } + if rho.len() != index_vars(trees.len()) { + return Err(Error::VariableCountMismatch { + expected: index_vars(trees.len()), + got: rho.len(), + }); + } + let successors = match end { + GroupedEnd::Successor { .. } => config.num_queries, + GroupedEnd::Final(_) => 0, + }; + check_grouped_shape( + proof, + trees.len(), + 1usize << config.log_folding, + config.num_queries, + successors, + )?; + + let num_leaves = domain.size() >> config.log_folding; + let current_depth = num_leaves.trailing_zeros() as usize; + let next_tree = match end { + GroupedEnd::Successor { + num_leaves, + root, + cap_height, + } => { + if !num_leaves.is_power_of_two() { + return Err(Error::NotPowerOfTwo(num_leaves)); + } + Some((root, num_leaves, cap_height)) + } + GroupedEnd::Final(_) => None, + }; + + // Every tree's check, built once from its first opening — after the shape + // guard above, so index 0 exists. With no queries nothing is opened, as in + // [`verify`]. + let Some(first) = proof.current.first() else { + return match next_tree { + Some((_, _, cap_height)) if cap_height != 0 => Err(Error::CapRejected), + Some((root, num_leaves, _)) => Ok(Some(CappedRoot::uncapped( + root, + num_leaves.trailing_zeros() as usize, + ))), + None => Ok(None), + }; + }; + let mut checks = Vec::with_capacity(trees.len()); + for (tree, opening) in trees.iter().zip(first) { + checks.push(tree.open::(current_depth, opening)?); + } + let next_check = match (next_tree, proof.next.first()) { + (Some((root, num_leaves, cap_height)), Some(opening)) => Some( + TreeCheck::Owner { + root, + cap_height, + leaves: LeafOrder::Natural, + } + .open::(num_leaves.trailing_zeros() as usize, opening)?, + ), + _ => None, + }; + + let queries = sample_queries(transcript, config.num_queries, num_leaves); + let eqs = combine_weights(rho, trees.len()); + + for (i, (&q, blocks)) in queries.iter().zip(&proof.current).enumerate() { + // One position for every tree: the oracle read is the run's tuple at + // block `q`, each component against its own root. + for (tree, (opening, (check, first_siblings))) in blocks.iter().zip(&checks).enumerate() { + let siblings = if i == 0 { + *first_siblings + } else { + opening.proof.merkle_path.as_slice() + }; + let at = trees[tree].position(q, current_depth); + if !verify_opening_capped::(check, at, opening, siblings) { + return Err(Error::GroupedOpeningRejected { query: i, tree }); + } + } + let folded = fold_coset::(&combine_blocks(blocks, &eqs), domain, q, alphas)?; + match (end, &next_check) { + (GroupedEnd::Final(value), _) => { + if folded != *value { + return Err(Error::FoldInconsistent { query: i }); + } + } + (GroupedEnd::Successor { num_leaves, .. }, Some((check, next_first))) => { + let nxt = &proof.next[i]; + let siblings = if i == 0 { + *next_first + } else { + nxt.proof.merkle_path.as_slice() + }; + let (leaf, slot) = leaf_and_slot(q, num_leaves); + if !verify_opening_capped::(check, leaf, nxt, siblings) { + return Err(Error::OpeningRejected { query: i }); + } + let claimed = nxt.values.get(slot).ok_or(Error::QueryOutOfRange { + index: slot, + bound: nxt.values.len(), + })?; + if folded != *claimed { + return Err(Error::FoldInconsistent { query: i }); + } + } + (GroupedEnd::Successor { .. }, None) => { + return Err(Error::GroupedShape(GroupedShape::SuccessorBlocks { + expected: config.num_queries, + got: proof.next.len(), + })); + } + } + } + + Ok(next_check.map(|(check, _)| check)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crypto::merkle_tree::{ + cap::verify_merkle_path_to_cap_from_leaf_hash, traits::IsMerkleTreeBackend, + }; + + use crate::{ + mle::Mle, + whir::{encode, fold_codeword_k, monomial_coefficients}, + whir_hash::KeccakWhir, + }; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"whir-round-test") + } + + struct Fixture { + current: CodewordCommitment, + next: CodewordCommitment, + domain: Domain, + alphas: Vec, + config: RoundConfig, + } + + /// An honest round: `next` really is `current` folded by `alphas`. + fn fixture(num_vars: usize, log_blowup: usize, k: usize) -> Fixture { + let vals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from((i.wrapping_mul(6364136223846793005).wrapping_add(7)) >> 13)) + .collect(); + let f = Mle::new(vals).unwrap(); + let domain = Domain::::new(num_vars + log_blowup).unwrap(); + let cw = encode(&monomial_coefficients(&f), &domain).unwrap(); + + let alphas: Vec = (0..k).map(|i| FE::from(17 + i as u64)).collect(); + let (folded, _) = fold_codeword_k(&cw, &domain, &alphas).unwrap(); + + Fixture { + current: CodewordCommitment::new(&cw, k).unwrap(), + next: CodewordCommitment::new(&folded, k.min(1)).unwrap(), + domain, + alphas, + config: RoundConfig { + num_queries: 4, + log_folding: k, + }, + } + } + + fn commitments<'a>( + fx: &Fixture, + current_root: &'a Commitment, + next_root: &'a Commitment, + ) -> RoundCommitments<'a> { + RoundCommitments { + current: TreeCheck::Owner { + root: current_root, + cap_height: 0, + leaves: LeafOrder::Natural, + }, + next_root, + next_num_leaves: fx.next.num_leaves(), + next_cap_height: 0, + } + } + + fn run(fx: &Fixture, proof: &RoundProof) -> Result<(), Error> { + let (current_root, next_root) = (fx.current.root(), fx.next.root()); + verify::( + proof, + commitments(fx, ¤t_root, &next_root), + &fx.domain, + &fx.alphas, + &fx.config, + &mut transcript(), + ) + .map(|_| ()) + } + + #[test] + fn an_honest_round_verifies() { + for k in 1..=3usize { + let mut fx = fixture(4, 2, k); + fx.config.num_queries = 4; + let proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + run(&fx, &proof).unwrap_or_else(|e| panic!("k={k}: {e:?}")); + } + } + + #[test] + fn the_prover_and_verifier_draw_the_same_queries() { + let fx = fixture(4, 2, 2); + let a = sample_queries::(&mut transcript(), 8, fx.current.num_leaves()); + let b = sample_queries::(&mut transcript(), 8, fx.current.num_leaves()); + assert_eq!(a, b); + assert!(a.iter().all(|q| *q < fx.current.num_leaves())); + } + + #[test] + fn a_tampered_current_opening_is_rejected() { + let fx = fixture(4, 2, 2); + let mut proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + proof.current[0].values[0] += FE::one(); + + assert!(matches!( + run(&fx, &proof).unwrap_err(), + Error::OpeningRejected { query: 0 } + )); + } + + #[test] + fn a_tampered_successor_opening_is_rejected() { + let fx = fixture(4, 2, 2); + let mut proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + proof.next[1].values[0] += FE::one(); + + assert!(matches!( + run(&fx, &proof).unwrap_err(), + Error::OpeningRejected { query: 1 } + )); + } + + /// The check the whole round exists for: a successor that is *validly + /// committed* but is not the fold of the current codeword. + #[test] + fn a_successor_that_is_not_the_fold_is_rejected() { + let mut fx = fixture(4, 2, 2); + + // Commit a different codeword, honestly. Every Merkle opening will + // verify; only the fold relation breaks. + let vals: Vec = (0..(1u64 << 2)).map(|i| FE::from(i + 100)).collect(); + let other = Mle::new(vals).unwrap(); + let other_domain = Domain::::new(2 + 2).unwrap(); + let other_cw = encode(&monomial_coefficients(&other), &other_domain).unwrap(); + fx.next = CodewordCommitment::new(&other_cw, 1).unwrap(); + + let proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + assert!(matches!( + run(&fx, &proof).unwrap_err(), + Error::FoldInconsistent { .. } + )); + } + + #[test] + fn the_wrong_folding_randomness_is_rejected() { + let mut fx = fixture(4, 2, 2); + let proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + + fx.alphas[0] += FE::one(); + assert!(matches!( + run(&fx, &proof).unwrap_err(), + Error::FoldInconsistent { .. } + )); + } + + #[test] + fn a_proof_replayed_under_another_transcript_is_rejected() { + // Queries are redrawn, so the openings no longer line up with them. + let fx = fixture(4, 2, 2); + let proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + run(&fx, &proof).unwrap(); + + let mut other = DefaultTranscript::::new(b"a-different-statement"); + let (current_root, next_root) = (fx.current.root(), fx.next.root()); + let result = verify::( + &proof, + commitments(&fx, ¤t_root, &next_root), + &fx.domain, + &fx.alphas, + &fx.config, + &mut other, + ); + assert!(result.is_err()); + } + + #[test] + fn a_proof_with_too_few_openings_is_rejected() { + let fx = fixture(4, 2, 2); + let mut proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + proof.current.pop(); + + assert!(matches!( + run(&fx, &proof).unwrap_err(), + Error::QueryCountMismatch { + expected: 4, + got: 3 + } + )); + } + + #[test] + fn randomness_of_the_wrong_arity_is_rejected() { + let mut fx = fixture(4, 2, 2); + let proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + fx.alphas.pop(); + + assert!(matches!( + run(&fx, &proof).unwrap_err(), + Error::VariableCountMismatch { + expected: 2, + got: 1 + } + )); + } + + #[test] + fn more_queries_check_more_positions() { + // The knob is real: it changes how many openings travel. + let mut fx = fixture(4, 2, 1); + fx.config.num_queries = 7; + let proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut transcript(), + ) + .unwrap(); + assert_eq!(proof.current.len(), 7); + run(&fx, &proof).unwrap(); + } + + // ---------------------------------------------------------------- caps + + type B = ::Backend; + + /// A round opened and checked under caps, both trees. + fn run_capped( + fx: &Fixture, + proof: &RoundProof, + caps: RoundCaps, + label: &[u8], + ) -> Result<(), Error> { + let (current_root, next_root) = (fx.current.root(), fx.next.root()); + verify::( + proof, + RoundCommitments { + current: TreeCheck::Owner { + root: ¤t_root, + cap_height: caps.current, + leaves: LeafOrder::Natural, + }, + next_root: &next_root, + next_num_leaves: fx.next.num_leaves(), + next_cap_height: caps.next, + }, + &fx.domain, + &fx.alphas, + &fx.config, + &mut DefaultTranscript::::new(label), + ) + .map(|_| ()) + } + + #[test] + fn a_capped_round_verifies_and_carries_its_caps_on_the_first_openings() { + // current: 32 leaves (depth 5); successor: 16 leaves (depth 4). + let mut fx = fixture(4, 2, 1); + fx.config.num_queries = 5; + for (c_cur, c_next) in [(0, 0), (1, 0), (0, 2), (3, 2), (5, 4)] { + let caps = RoundCaps { + current: c_cur, + current_owner: true, + next: c_next, + }; + let proof = prove(&fx.current, &fx.next, &fx.config, caps, &mut transcript()).unwrap(); + for (i, (cur, nxt)) in proof.current.iter().zip(&proof.next).enumerate() { + let own = |c: usize| if i == 0 && c > 0 { 1usize << c } else { 0 }; + assert_eq!(cur.proof.merkle_path.len(), 5 - c_cur + own(c_cur)); + assert_eq!(nxt.proof.merkle_path.len(), 4 - c_next + own(c_next)); + } + run_capped(&fx, &proof, caps, b"whir-round-test") + .unwrap_or_else(|e| panic!("caps ({c_cur}, {c_next}): {e:?}")); + } + } + + /// A cap node no query reaches, flipped. Every + /// per-query check still accepts against the forged cap — shown below — + /// so ONLY the cap-to-root check can refuse it. The error names it. + #[test] + fn a_cap_node_no_query_reaches_is_refused_by_the_cap_check_alone() { + let mut fx = fixture(4, 2, 1); + fx.config.num_queries = 2; + let (d_cur, c_cur, d_next, c_next) = (5usize, 3usize, 4usize, 2usize); + let caps = RoundCaps { + current: c_cur, + current_owner: true, + next: c_next, + }; + let proof = prove(&fx.current, &fx.next, &fx.config, caps, &mut transcript()).unwrap(); + run_capped(&fx, &proof, caps, b"whir-round-test").unwrap(); + let queries = sample_queries::(&mut transcript(), 2, fx.current.num_leaves()); + + // The current tree. + let reached: Vec = queries.iter().map(|q| q >> (d_cur - c_cur)).collect(); + let j = (0..1 << c_cur).find(|j| !reached.contains(j)).unwrap(); + let mut forged = proof.clone(); + forged.current[0].proof.merkle_path[d_cur - c_cur + j][0] ^= 1; + let cap = forged.current[0].proof.merkle_path[d_cur - c_cur..].to_vec(); + for (q, opening) in queries.iter().zip(&forged.current) { + let siblings = &opening.proof.merkle_path[..d_cur - c_cur]; + assert!( + verify_merkle_path_to_cap_from_leaf_hash::( + siblings, + &cap, + d_cur, + *q, + B::hash_data(&opening.values) + ), + "every query must still fold onto the forged cap, or this is not the fixture" + ); + } + assert!(matches!( + run_capped(&fx, &forged, caps, b"whir-round-test"), + Err(Error::CapRejected) + )); + + // The successor tree: leaf `q mod 16`. + let leaves: Vec = queries + .iter() + .map(|q| leaf_and_slot(*q, fx.next.num_leaves()).0) + .collect(); + let reached: Vec = leaves.iter().map(|l| l >> (d_next - c_next)).collect(); + let j = (0..1 << c_next).find(|j| !reached.contains(j)).unwrap(); + let mut forged = proof.clone(); + forged.next[0].proof.merkle_path[d_next - c_next + j][0] ^= 1; + let cap = forged.next[0].proof.merkle_path[d_next - c_next..].to_vec(); + for (leaf, opening) in leaves.iter().zip(&forged.next) { + let siblings = &opening.proof.merkle_path[..d_next - c_next]; + assert!(verify_merkle_path_to_cap_from_leaf_hash::( + siblings, + &cap, + d_next, + *leaf, + B::hash_data(&opening.values) + )); + } + assert!(matches!( + run_capped(&fx, &forged, caps, b"whir-round-test"), + Err(Error::CapRejected) + )); + } + + /// The WHIR analogue of the STARK's exact path-length check: a leaf forged from an + /// INTERNAL node. Under keccak a 64-byte block (eight base values at + /// `k = 3`) is a valid parent input, so values whose bytes are the level-1 + /// node's two children hash to that node, and a path one sibling short + /// then folds to the root — the raw fold accepts it (asserted). At index 0 + /// or all-ones the shifted index bits agree with the true ones, so the + /// fixture needs a statement whose first query lands there. + /// + /// Only the exact-length check can refuse it: without it the Merkle check + /// passes and the round fails later at the FOLD (`FoldInconsistent`), so + /// the `OpeningRejected { query: 0 }` this asserts is the length check's. + #[test] + fn a_leaf_forged_from_an_internal_node_is_refused_by_the_path_length_alone() { + const P: u64 = 0xFFFF_FFFF_0000_0001; + let fx = fixture(4, 2, 3); + let depth = fx.current.depth(); + let leaves = fx.current.num_leaves(); + assert_eq!((depth, leaves), (3, 8)); + let root = fx.current.root(); + + let (label, proof, forged) = (0u64..256) + .find_map(|i| { + let label = format!("m1a-{i}"); + let mut proof = prove( + &fx.current, + &fx.next, + &fx.config, + RoundCaps::default(), + &mut DefaultTranscript::::new(label.as_bytes()), + ) + .ok()?; + let q0 = sample_queries::( + &mut DefaultTranscript::::new(label.as_bytes()), + 1, + leaves, + )[0]; + if q0 != 0 && q0 != leaves - 1 { + return None; + } + let honest = &proof.current[0]; + let leaf = B::hash_data(&honest.values); + let s0 = honest.proof.merkle_path[0]; + let (l, r) = if q0 % 2 == 0 { (leaf, s0) } else { (s0, leaf) }; + let values: Vec = l + .iter() + .chain(r.iter()) + .copied() + .collect::>() + .chunks_exact(8) + .map(|c| u64::from_be_bytes(c.try_into().unwrap())) + .map(|v| (v < P).then(|| FE::from(v))) + .collect::>()?; + assert_eq!( + B::hash_data(&values), + B::hash_new_parent(&l, &r), + "the forged block must hash to the level-1 node" + ); + let forged = CosetOpening { + values, + proof: crypto::merkle_tree::proof::Proof { + merkle_path: honest.proof.merkle_path[1..].to_vec(), + }, + }; + assert!( + crypto::merkle_tree::proof::verify_merkle_path_from_leaf_hash::( + &forged.proof.merkle_path, + &root, + q0, + B::hash_data(&forged.values) + ), + "the raw fold must accept the short path, or the fixture tests nothing" + ); + assert!(!crate::whir_commit::verify_opening::( + &root, depth, q0, &forged + )); + proof.current[0] = forged.clone(); + Some((label, proof, forged)) + }) + .expect("a statement whose first query is 0 or all-ones"); + assert_eq!(forged.proof.merkle_path.len(), depth - 1); + assert!(matches!( + run_capped(&fx, &proof, RoundCaps::default(), label.as_bytes()), + Err(Error::OpeningRejected { query: 0 }) + )); + } + + /// M2: a proof with no openings at all is refused by the count guard, not + /// by a panic on the owner's index. + #[test] + fn a_capped_round_with_no_openings_is_refused_without_panicking() { + let mut fx = fixture(4, 2, 1); + fx.config.num_queries = 3; + let caps = RoundCaps { + current: 2, + current_owner: true, + next: 2, + }; + let mut proof = prove(&fx.current, &fx.next, &fx.config, caps, &mut transcript()).unwrap(); + proof.current.clear(); + proof.next.clear(); + assert!(matches!( + run_capped(&fx, &proof, caps, b"whir-round-test"), + Err(Error::QueryCountMismatch { .. }) + )); + } +} diff --git a/crypto/multilinear/src/whir_run_claim_tests.rs b/crypto/multilinear/src/whir_run_claim_tests.rs new file mode 100644 index 000000000..93cd7696d --- /dev/null +++ b/crypto/multilinear/src/whir_run_claim_tests.rs @@ -0,0 +1,108 @@ +//! The column claims of one grouped run: each column's batching power is +//! indexed in the whole group, so a claim cannot move between the run's +//! polynomials with their sum kept. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as F, +}; + +use crate::{ + mle::Mle, + stacked_eval::{Claimed, StackedCommitment, prove, verify}, + stacking::{StackedLayout, borrow}, + whir_chain::{ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, LeafOrder, WhirFolds}, + whir_hash::{P1Whir, WhirHash}, +}; + +type FE = FieldElement; +type EE = FieldElement; + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_add(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11)) + .collect(), + ) + .unwrap() +} + +fn sponge() -> ::Sponge { + let mut t = P1Whir::sponge(); + t.append_bytes(b"a grouped run's claims"); + t +} + +/// ★ A run of three polynomials (one column each) under the composed format +/// (one chain, coset order, blowup 2), proved over the TRUE polynomials for +/// claims that move one unit of value from polynomial 0's column to +/// polynomial 1's. The claims' sum is unchanged; only the column batching's +/// distinct exponents (each column's power indexed in the whole group) can +/// tell. Refused; the honest claims verify. +#[test] +fn a_claim_traded_between_a_runs_polynomials_is_refused() { + let n = 8; + let cfg = ChainConfig { + log_blowup: 1, + log_folding: 4, + num_queries: 9, + grind: GrindBits::default(), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + group: GroupFold::Indexed, + leaves: LeafOrder::Coset, + ..ChainFormat::DEFAULT + }, + }; + let columns: Vec> = (0..3).map(|i| column(n, 5 + 31 * i)).collect(); + let points: Vec> = (0..3u64) + .map(|i| { + (0..n as u64) + .map(|j| EE::from(7 + 11 * i + 3 * j)) + .collect() + }) + .collect(); + let honest: Vec = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p).unwrap()) + .collect(); + let layout = StackedLayout::build(&[n; 3], n).unwrap(); + assert_eq!(layout.num_polys(), 3); + let stacked = + StackedCommitment::::commit(layout.clone(), &borrow(&columns), None, &cfg) + .unwrap(); + let roots = stacked.roots(); + let run = |values: &[EE]| { + let proof = prove::( + &stacked, + &borrow(&columns), + None, + &Claimed::PerColumn(&points), + values, + &cfg, + &mut sponge(), + ) + .unwrap(); + assert_eq!(proof.polys.len(), 1, "one chain over the run of three"); + verify::( + &proof, + &layout, + &roots, + &Claimed::PerColumn(&points), + values, + stacked.domain(), + &cfg, + &mut sponge(), + ) + }; + assert_eq!(run(&honest), Ok(()), "the honest claims"); + let mut traded = honest.clone(); + traded[0] += EE::one(); + traded[1] = &traded[1] - EE::one(); + assert!( + run(&traded).is_err(), + "a claim traded between slots 0 and 1" + ); +} diff --git a/crypto/multilinear/src/whir_split.rs b/crypto/multilinear/src/whir_split.rs new file mode 100644 index 000000000..5e706b3df --- /dev/null +++ b/crypto/multilinear/src/whir_split.rs @@ -0,0 +1,2757 @@ +//! The WHIR base's stage breakdown, behind `LAMBDA_VM_BASE_SPLIT=1`. +//! +//! # Why this exists, and why the knob is not a new name +//! +//! The WHIR base proves the block's epochs and prints **one number** for all of +//! them — `base (WHIR): 15 epochs in 84.5s`, 57% of the block's wall with +//! nothing under it. There is not one timer, span or print between +//! `multilinear_continuation::prove_continuation` and the bottom of the chain, +//! so every optimisation round so far has moved that number without anyone +//! being able to say which part of it moved. +//! +//! The STARK base has had a breakdown for months, under `LAMBDA_VM_BASE_SPLIT`. +//! The LFM tree launcher exports that knob on the **WHIR** arm too — "byte +//! identical to the D-S exports" — and it reached nothing at all, because the +//! WHIR base does not go through `continuation::prove_continuation`. An inert +//! knob printed as if it mattered is worse than a missing one: the export is +//! the evidence a reader uses to believe the breakdown was taken. +//! +//! ⇒ this module reuses **the same knob name and the same line format**, so one +//! name means one thing on both pipelines. +//! +//! # Why it lives in `multilinear` and not beside the STARK instrument +//! +//! Not taste — reachability. The stages span four places: the producer +//! (`prover::continuation`), the epoch prover and the global stage +//! (`prover::multilinear_continuation`), the argument and the openings +//! (`stark::multilinear_table`), and the harness that reads them back. `prover` +//! depends on `stark`, `stark` depends on `multilinear`, and `multilinear` +//! depends on neither. This crate is the only one all four can reach, so the +//! helpers the STARK instrument keeps private to `prover::continuation` are not +//! an option for the two lower layers, whatever their visibility. +//! +//! # Phase walls and per-epoch records +//! +//! Two kinds of number, and mixing them is the way to misread the table: +//! +//! - **Stage walls** are measured on the thread that runs the stage, at stage +//! boundaries. Within one thread they do not overlap and they **partition** +//! that thread's epoch wall. +//! - **The producer and the prover run concurrently**, so their two sums do +//! NOT add up to the base wall and must never be added. What the pair says is +//! which of the two set the wall — see `handoff` in +//! `prover::continuation::for_each_epoch`. +//! +//! # Cost when disabled +//! +//! [`enabled`] is a `OnceLock` load and a predictable branch; [`mark`] +//! returns `None` and **no clock is read**, so a disabled run pays no +//! `Instant::now` at all. Every call site is at stage granularity — about a +//! dozen per epoch against a ~5.6 s epoch — so even enabled it is far below the +//! noise of what it measures. +//! +//! # ⚠ One prove at a time +//! +//! The inner slots are process-global, because they are written inside +//! `multi_prove` and read one layer up. Two `multi_prove` calls in flight would +//! mix their numbers. In the base that cannot happen — there is a single prover +//! thread — but the **same process** later proves the LFM tree with several +//! workers at once, so rather than assume the base is the only writer, +//! [`begin_prove`] counts concurrent proves and a record that saw one carries +//! `OVERLAPPED`. A mixed reading says so instead of looking clean. + +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Mutex, OnceLock}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{element::FieldElement, traits::IsField}; + +/// `LAMBDA_VM_BASE_SPLIT=1` (any non-empty value other than `0`) turns the +/// lines and the records on. +/// +/// The same spelling as `prover::continuation`'s own gate, deliberately: the +/// two are read in different crates and must not be able to disagree about what +/// the knob means. +pub fn enabled() -> bool { + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| match std::env::var("LAMBDA_VM_BASE_SPLIT") { + Ok(v) => !v.is_empty() && v != "0", + Err(_) => false, + }) +} + +/// Unix epoch seconds, for aligning a stage with an external GPU sampler. +/// +/// A duplicate of `stark::prove_split::epoch_secs` by necessity, not by +/// oversight: `multilinear` cannot reach `stark`. Kept byte-identical in +/// behaviour so a stamp from either instrument lands on the same timeline. +pub fn epoch_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or_default() +} + +/// Start a timed stage — `None`, and no clock read, when the knob is off. +#[inline] +pub fn mark() -> Option<(Instant, f64)> { + enabled().then(|| (Instant::now(), epoch_secs())) +} + +/// Close a stage opened by [`mark`], print its line, and return its seconds. +/// +/// The line is the STARK base's format, unchanged: +/// `BASE EPOCH {index}: {stage} {secs:.2}s t=[{t0:.3},{t1:.3}]`. +/// +/// The two wall-clock stamps are what let an external GPU sampler be sliced by +/// stage; the duration alone cannot place the stage on the sampler's timeline. +#[inline] +pub fn stage_done(index: u64, stage: &str, open: Option<(Instant, f64)>) -> f64 { + match open { + Some((start, t0)) => { + let secs = start.elapsed().as_secs_f64(); + // ⛔ The cross-epoch stage's index is `u64::MAX`, and printing it + // raw put `BASE EPOCH 18446744073709551615` in the log — read back + // from a real run, not imagined. It is unreadable and it defeats a + // parse that expects a small integer, so the sentinel is rendered + // by NAME. The record keeps the sentinel; only the line differs. + let who = if index == GLOBAL_INDEX { + "global".to_string() + } else { + index.to_string() + }; + println!( + "BASE EPOCH {who}: {stage} {secs:.2}s t=[{t0:.3},{:.3}]", + epoch_secs() + ); + secs + } + None => 0.0, + } +} + +// ── the inner slots: written inside `multi_prove`, read one layer up ──────── + +/// A nanosecond accumulator. Public so call sites name their slot as a constant +/// rather than passing an index. +#[derive(Debug)] +pub struct Slot(AtomicU64); + +impl Slot { + const fn new() -> Self { + Self(AtomicU64::new(0)) + } + /// Read and CLEAR. Reading a slot clears it, so a caller that drops the + /// value silently would hand this epoch's time to the next one. + fn take(&self) -> f64 { + self.0.swap(0, Ordering::Relaxed) as f64 / 1e9 + } +} + +/// An occurrence counter. Same discipline as [`Slot`] — process-global, and +/// cleared when read — but it counts CALLS, not nanoseconds. +/// +/// ★ It exists because a count survives a shape where a duration does not. On +/// the card-free fixture the query openings read `0.00` s, so no bound on a +/// TIME could ever see one of the four windows go missing there; the calls +/// still happen, so a count still can. That is what makes the gate's +/// real-wiring mutation able to fire at the shape the gate actually runs. +#[derive(Debug)] +pub struct Counter(AtomicU64); + +impl Counter { + const fn new() -> Self { + Self(AtomicU64::new(0)) + } + /// Read and CLEAR, for the reason [`Slot::take`] does. + fn take(&self) -> u64 { + self.0.swap(0, Ordering::Relaxed) + } +} + +/// Count one occurrence. Inert when the instrument is off, like [`mark`]. +#[inline] +pub fn bump(counter: &Counter) { + bump_by(counter, 1); +} + +/// Count `n` occurrences at once. Inert when the instrument is off. +#[inline] +pub fn bump_by(counter: &Counter, n: u64) { + if enabled() { + counter.0.fetch_add(n, Ordering::Relaxed); + } +} + +// ── the device ledger's peak per phase ────────────────────────────────────── + +/// The device reservation ledger's peak over one phase of one prove, in bytes. +/// Process-global and cleared when read, like [`Slot`]. +/// +/// ★ WHY PER PHASE. The run's one `reserved high-water` says how close the +/// process came to the budget and not when; a change that moves one phase's +/// reservations — a group giving its room back across the argument — is tested +/// by that phase's peak, epoch by epoch. The whole-run number is untouched: it +/// is a different atomic. +#[derive(Debug)] +pub struct Peak(AtomicU64); + +impl Peak { + const fn new() -> Self { + Self(AtomicU64::new(0)) + } + /// Read and CLEAR, for the reason [`Slot::take`] does. + fn take(&self) -> u64 { + self.0.swap(0, Ordering::Relaxed) + } +} + +/// `commit_grouped`: the epoch's columns onto the card and every group +/// committed. +pub static RESERVED_COMMIT: Peak = Peak::new(); +/// The per-table argument loop. +pub static RESERVED_ARGUE: Peak = Peak::new(); +/// The openings: every group's, then the prepared one. +pub static RESERVED_OPEN: Peak = Peak::new(); + +/// Opens a ledger window for a phase: its peak starts at what is promised now. +/// `false`, and nothing touched, when the instrument is off. +#[inline] +pub fn open_reserved() -> bool { + if !enabled() { + return false; + } + crate::gpu::reset_reserved_window(); + true +} + +/// Closes the window [`open_reserved`] opened, into `peak`. +#[inline] +pub fn close_reserved(peak: &Peak, opened: bool) { + if opened { + peak.0 + .store(crate::gpu::reserved_window_peak(), Ordering::Relaxed); + } +} + +/// `absorb_roots_and_challenge`: the roots into the transcript and the +/// challenge out. Host. +pub static CHALLENGE: Slot = Slot::new(); +/// The per-table argument: LogUp, the GKR input layer, the GKR prove, the +/// zerocheck and the constraint core. **Serial over the epoch's tables** — the +/// loop has no rayon and no `k` — so this is both a sum and a wall. +pub static ARGUE: Slot = Slot::new(); +/// The per-GROUP WHIR openings. Two per epoch: `epoch_groups(n)` is +/// `[n - 1, 1]`, so every epoch pays two full chains. +pub static OPEN_GROUPS: Slot = Slot::new(); +/// The out-of-band DECODE opening, one more chain on top of the two. +pub static OPEN_PREPARED: Slot = Slot::new(); + +// ── inside an opening: the WHIR chain's round loop ────────────────────────── +// +// `open_groups` is 43.8% of the base and 24.9% of the block's whole wall, and +// it was one number. These six PARTITION the round (`whir_chain.rs:715-795`), +// so `open_groups - Σ(six)` is loop overhead and nothing else. +// +// ⛔ They are taken at the END OF THE GROUP LOOP, before the prepared opening +// runs. The prepared chain writes the same slots, so a take placed after it +// would fold DECODE's chain into `open_groups` and arm E would close on a +// number that is not what it claims. + +/// All THREE 20-bit grinds a round pays: folding, out-of-domain, query. +pub static GRIND: Slot = Slot::new(); +/// `factors.rounds` — the opening sumcheck. +pub static SUMCHECK: Slot = Slot::new(); +/// `fold_held` — the codeword folded in place where it lies. +pub static FOLD: Slot = Slot::new(); +/// The FRESH Merkle commit of the successor, once per round, plus its root +/// absorb; or the final-value path on the last round. +pub static COMMIT_FOLDED: Slot = Slot::new(); +/// Out-of-domain: the sampled point, `evaluate_message`, `add_scaled_eq`. +/// Its grind is in [`GRIND`], not here. +pub static OOD: Slot = Slot::new(); +/// The query openings — `whir_round::prove` / `final_openings`, which REBUILD +/// the Merkle tree on device per query batch. +pub static QUERIES: Slot = Slot::new(); + +// ── inside the query openings: what `open_many` does, twice per round ─────── +// +// QUERIES is the largest slot in the chain and, like `open_groups` before it, +// one number. These four PARTITION it, and they are counted on EVERY +// `open_many` call — `whir_round::prove` opens the CURRENT commitment and the +// NEXT one (`whir_round.rs:104-105`), so a non-final round calls it twice and +// the final round once through `final_openings`. +// +// ⛔ THE BOUNDARY IS `open_many`, NOT `paths()`. On the device arm `paths()` is +// a range check, ONE device call, and a `map` that wraps each path in a +// `Proof`; splitting INSIDE it would put the rebuild against host bookkeeping +// over a hundred kilobyte-sized paths and read ~100% every time. The term that +// competes with the rebuild is the COSET GATHER, which sits beside `paths()` in +// `open_many` and would otherwise stay inside QUERIES as an unnamed remainder — +// the same shape as the setup gap the seventh slot was added to name. + +/// `sample_queries` / the `sample_u64` loop — the transcript squeezes that +/// choose the indices, and the `leaf_and_slot` map onto the successor. +pub static QUERY_SAMPLE: Slot = Slot::new(); +/// ★ `paths()` — where a device codeword's Merkle tree is REBUILT, because the +/// commitment kept only its root. This is round 3's whole question. +pub static TREE_REBUILD: Slot = Slot::new(); +/// `cosets()` — the query blocks gathered off the codeword where it lies. +pub static COSET_GATHER: Slot = Slot::new(); +/// The zip/map/collect that pairs each block with its path. +pub static OPEN_ASSEMBLE: Slot = Slot::new(); + +/// The ROUND LOOP's own wall, summed over rounds — the six measured against +/// their container rather than against `open_groups` directly. +/// +/// ★ WHY A SEVENTH NUMBER RATHER THAN A WIDER TOLERANCE. The six closed to +/// +1.4% on the laptop and +5.1% on the box, because what they leave out is +/// roughly FIXED per epoch while the window they sit in shrinks on a faster +/// machine. Widening the tolerance to admit 5% would have been the move that +/// makes a check unable to fail — and it would have been wrong about the cause, +/// because the gap is NOT round bookkeeping: +/// +/// `Factors::from_shares` runs in `prove_shared`, and `stacked_eval::prove` +/// builds the weights and the stacked polys — all inside `open_groups` and +/// OUTSIDE the round loop entirely. That is a setup phase, the same class of +/// miss as the out-of-domain grind, not a scattering of small gaps. +/// +/// So the round's wall is measured, and the two remainders are NAMED and +/// reported separately: `round_other` is the loop's own bookkeeping between +/// windows, `setup_tail` is everything outside the loop. The reading says which +/// of the two owns the gap; neither is inferred. +pub static ROUND: Slot = Slot::new(); + +/// Rounds the chain ran, summed over the group openings. +pub static ROUND_COUNT: Counter = Counter::new(); + +/// ★ CHAINS run in the group openings — NOT groups. +/// +/// ⛔ A group is not a chain. `stacked_eval::prove` runs one chain per +/// COMMITMENT in the stacked commitment (`stacked_eval.rs:367`, a loop over +/// `stacked.commitments`), so a single group can open several. Arm F's identity +/// is per chain, so it counts chains; deriving it from `groups` would have made +/// the arm red on the honest path the first time a group carried two +/// commitments — and a check that reddens honestly gets widened until it cannot +/// fail at all. +pub static CHAIN_COUNT: Counter = Counter::new(); + +/// ★ `open_many` calls — and therefore device tree REBUILDS, one per call. +/// +/// Pre-registered as an identity the record can check itself against: a chain +/// of `R` rounds calls `open_many` twice per non-final round and once in the +/// final one, so `2R − 1`. Summed over `g` chains that is +/// `2·Σ R − g = 2·round_count − groups`, which is [`check_closure`]'s arm F — +/// plus [`GROUPED_TREES`], a grouped round 0's trees past its first. +pub static REBUILD_CALLS: Counter = Counter::new(); + +/// Round-0 trees a grouped chain (`GroupFold::Indexed`) opens past the first: +/// `P' − 1` a chain over `P'` polynomials, each one `open_many` call more than +/// a per-polynomial chain's round 0 makes. Arm F adds them to `2R − 1`. +pub static GROUPED_TREES: Counter = Counter::new(); + +/// Columns the argument's claim reduces valued on the CARD, in one batched +/// evaluation per table — beside [`COLUMNS_ON_HOST`], the ones walked on the +/// host one at a time. The pair is the mechanism line of +/// `LAMBDA_VM_ARGUE_DEVICE_COLUMNS`: the knob moves columns from the second to +/// the first, and their sum is the prove's committed width either way — short +/// of it only by a tall table the card refused, whose columns then go to the +/// device one at a time and count in neither. +pub static COLUMNS_ON_CARD: Counter = Counter::new(); +/// See [`COLUMNS_ON_CARD`]. +pub static COLUMNS_ON_HOST: Counter = Counter::new(); +/// Of [`COLUMNS_ON_CARD`], the ones `LAMBDA_VM_ARGUE_XCHECK` recomputed on the +/// host and found equal. Zero on a run without the check, which is what says a +/// gate run's silence was a comparison rather than an absence of one. +pub static COLUMNS_XCHECKED: Counter = Counter::new(); + +/// Challenge tables the card built for the argument +/// (`LAMBDA_VM_ARGUE_DEVICE_TABLES`): the zerocheck's `eq` weights and the claim +/// reduce's shift tables and batched columns — each one the host would have +/// built on the pool and uploaded. Zero with the knob off: the mechanism line +/// of that knob's A/B. +pub static TABLES_ON_CARD: Counter = Counter::new(); +/// Of [`TABLES_ON_CARD`], the ones `LAMBDA_VM_ARGUE_XCHECK` compared with the +/// host's, cell for cell, and found equal. +pub static TABLES_XCHECKED: Counter = Counter::new(); + +// ── inside the argue: a device GKR layer, and the host work between two ───── +// +// The head's trace put 0.79 s of card idle between one GKR layer's read-back +// and the next layer's first kernel — 3,489 transitions, 227 µs each on +// average — and could not say whose it is. These split a device layer, in the +// order it runs, into the host's work before its rounds, the rounds, and the +// host's work after them. Only the layers a device runs are timed: a level GKR +// proves here from the start has no transition to split. +// +// ★ Timed with [`tick`], one clock read, not [`mark`]'s two: a prove opens +// these some hundred thousand times. + +/// The layer's `λ`, drawn from the transcript. +pub static GKR_LAMBDA: Slot = Slot::new(); +/// `program_for(λ)`: the layer relation written as a program. +pub static GKR_PROGRAM: Slot = Slot::new(); +/// The program lowered to the device's nodes and constants, and the point's +/// limbs. +pub static GKR_LOWER: Slot = Slot::new(); +/// The session set up on the card: the `eq` table over the point, the factor +/// pointers, the nodes and the round scratch. +pub static GKR_SESSION: Slot = Slot::new(); +/// The input layer written again for its own sumcheck by a tree that gave it +/// back, and that session set up — card work, once a table, kept out of +/// [`GKR_SESSION`] so the transitions are not charged for it. +pub static GKR_REBUILD: Slot = Slot::new(); +/// The rounds on the card, their transcript round trips included. +pub static GKR_ROUNDS: Slot = Slot::new(); +/// The factors read back at the crossover cube. +pub static GKR_VALUES: Slot = Slot::new(); +/// Those values made into the relation's five tables. +pub static GKR_FACTORS: Slot = Slot::new(); +/// The rounds the host finishes over the crossover cube, their transcript +/// included. +pub static GKR_TAIL: Slot = Slot::new(); +/// Of [`GKR_TAIL`], the transcript's share: the round messages absorbed and +/// the challenges drawn. +pub static GKR_TAIL_TRANSCRIPT: Slot = Slot::new(); +/// The layer's four values absorbed, `c` drawn and the next claim made. +pub static GKR_CLOSE: Slot = Slot::new(); +/// Layers a device ran. +pub static GKR_LAYERS: Counter = Counter::new(); +/// Of [`GKR_LAYERS`], the input layers written again ([`GKR_REBUILD`]). +pub static GKR_REBUILDS: Counter = Counter::new(); +/// Rounds the host finished after them. +pub static GKR_TAIL_ROUNDS: Counter = Counter::new(); +/// Of [`GKR_LAYERS`], the ones that ran Gruen's rounds +/// (`LAMBDA_VM_ARGUE_GKR_GRUEN`, D-ARGUE S1-3) — the mechanism line of that knob. +pub static GKR_GRUEN: Counter = Counter::new(); +/// Of [`GKR_GRUEN`], the ones `LAMBDA_VM_ARGUE_GKR_GRUEN_XCHECK` compared with +/// today's rounds and found equal. +pub static GKR_GRUEN_XCHECKED: Counter = Counter::new(); + +/// A prove's device GKR layers, split: seconds per region, in the order a +/// layer runs them, and the counts they are over. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct GkrSplit { + pub lambda: f64, + pub program: f64, + pub lower: f64, + pub session: f64, + pub rebuild: f64, + pub rounds: f64, + pub values: f64, + pub factors: f64, + pub tail: f64, + pub tail_transcript: f64, + pub close: f64, + pub layers: u64, + pub rebuilds: u64, + pub tail_rounds: u64, + pub gruen: u64, + pub gruen_xchecked: u64, +} + +impl GkrSplit { + /// Read and clear every GKR slot and counter. + fn take() -> Self { + Self { + lambda: GKR_LAMBDA.take(), + program: GKR_PROGRAM.take(), + lower: GKR_LOWER.take(), + session: GKR_SESSION.take(), + rebuild: GKR_REBUILD.take(), + rounds: GKR_ROUNDS.take(), + values: GKR_VALUES.take(), + factors: GKR_FACTORS.take(), + tail: GKR_TAIL.take(), + tail_transcript: GKR_TAIL_TRANSCRIPT.take(), + close: GKR_CLOSE.take(), + layers: GKR_LAYERS.take(), + rebuilds: GKR_REBUILDS.take(), + tail_rounds: GKR_TAIL_ROUNDS.take(), + gruen: GKR_GRUEN.take(), + gruen_xchecked: GKR_GRUEN_XCHECKED.take(), + } + } + + /// The host's work between two device layers: what follows one layer's + /// read-back — the factors, the tail, the close — and what precedes the + /// next one's rounds — `λ`, the program, the lowering, the session. The + /// rebuild, the rounds and the read-back are not in it. + pub fn between(&self) -> f64 { + self.lambda + + self.program + + self.lower + + self.session + + self.factors + + self.tail + + self.close + } +} + +// ── inside the argue: a table's work around its GKR layers and zerocheck ──── +// +// D-BATCH S0. The argue's "rest" — what is neither a device GKR layer nor a +// zerocheck round — was only known by subtraction (3.19 s a block at FAST job +// 292), and a batched argument removes some of it and keeps the rest. These +// split it per table, in the order `stark::multilinear_table::prove` runs, so +// the part batching removes is measured before it is built. Timed with +// [`tick`], as the GKR regions are; instrumentation only, no proof byte moves. + +/// The table's interactions, with `z` and `α` baked in. +pub static REST_INTERACTIONS: Slot = Slot::new(); +/// Its fraction tree: the factors lifted, the input layer written, every level +/// folded (and, built eagerly, the output read). +pub static REST_TREE: Slot = Slot::new(); +/// The bus output read off the tree and absorbed. +pub static REST_OUTPUT: Slot = Slot::new(); +/// `gkr::prove`'s wall: the device layers ([`GkrSplit`]) and the rest of it. +pub static REST_GKR: Slot = Slot::new(); +/// Of [`REST_GKR`], the host prefix: one level down and the folds above it. +pub static REST_PREFIX: Slot = Slot::new(); +/// The zerocheck point drawn, the bus statements, the rule, the weights and +/// the fused rounds' description. +pub static REST_SETUP: Slot = Slot::new(); +/// The zerocheck batch's wall ([`ZerocheckSplit`] splits its rounds). +pub static REST_BATCH: Slot = Slot::new(); +/// The committed factors' values at the batch's point. +pub static REST_VALUES: Slot = Slot::new(); +/// The claim reduce: the factor values absorbed, `γ′`, its sumcheck. +pub static REST_REDUCE: Slot = Slot::new(); +/// The columns evaluated at the reduced point and absorbed. +pub static REST_COLUMNS: Slot = Slot::new(); +/// The table's factors let go. +pub static REST_RELEASE: Slot = Slot::new(); +/// Tables argued. +pub static REST_TABLES: Counter = Counter::new(); + +// Of [`REST_TREE`], split — the factors lifted, the input layer's programs +// lowered and written, the levels folded, the output read. The card runs the +// first three asynchronously, so their host times say where the host waited, +// not what the card spent; under `LAMBDA_VM_ARGUE_TREE_SYNC=1` (a diagnostic, +// never a measurement of the wall) each part waits for its own kernels, and the +// parts then say what the card spent on each. + +/// The factors lifted from the columns into the extension. +pub static TREE_LIFT: Slot = Slot::new(); +/// The input layer's interaction programs lowered on the host. +pub static TREE_LOWER: Slot = Slot::new(); +/// The input layer written (the lowering included). +pub static TREE_WRITE: Slot = Slot::new(); +/// Every level folded up from it, the tree's promise included. +pub static TREE_FOLD: Slot = Slot::new(); +/// The output fraction read back. +pub static TREE_OUTPUT: Slot = Slot::new(); +/// Trees whose input layer was written from the base columns +/// (`LAMBDA_VM_ARGUE_GKR_INPUT`). +pub static TREE_FROM_COLUMNS: Counter = Counter::new(); +/// Tables whose factors stayed base columns, with no lift +/// (`LAMBDA_VM_ARGUE_NO_LIFT`), and of them the ones lifted after all. +pub static NO_LIFT: Counter = Counter::new(); +pub static LATE_LIFT: Counter = Counter::new(); + +/// Whether the tree's parts wait for their own kernels +/// (`LAMBDA_VM_ARGUE_TREE_SYNC`, with the split on): a diagnostic. +pub fn tree_sync() -> bool { + static ON: OnceLock = OnceLock::new(); + enabled() + && *ON.get_or_init(|| { + std::env::var("LAMBDA_VM_ARGUE_TREE_SYNC").is_ok_and(|v| !v.is_empty() && v != "0") + }) +} + +/// A prove's tree region, split: seconds per part. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct TreeSplit { + pub lift: f64, + pub lower: f64, + pub write: f64, + pub fold: f64, + pub output: f64, + pub from_columns: u64, + pub no_lift: u64, + pub late_lift: u64, +} + +impl TreeSplit { + fn take() -> Self { + Self { + lift: TREE_LIFT.take(), + lower: TREE_LOWER.take(), + write: TREE_WRITE.take(), + fold: TREE_FOLD.take(), + output: TREE_OUTPUT.take(), + from_columns: TREE_FROM_COLUMNS.take(), + no_lift: NO_LIFT.take(), + late_lift: LATE_LIFT.take(), + } + } +} + +/// A prove's per-table rest, split: seconds per region. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct RestSplit { + pub interactions: f64, + pub tree: f64, + pub output: f64, + pub gkr: f64, + pub prefix: f64, + pub setup: f64, + pub batch: f64, + pub values: f64, + pub reduce: f64, + pub columns: f64, + pub release: f64, + pub tables: u64, +} + +impl RestSplit { + fn take() -> Self { + Self { + interactions: REST_INTERACTIONS.take(), + tree: REST_TREE.take(), + output: REST_OUTPUT.take(), + gkr: REST_GKR.take(), + prefix: REST_PREFIX.take(), + setup: REST_SETUP.take(), + batch: REST_BATCH.take(), + values: REST_VALUES.take(), + reduce: REST_REDUCE.take(), + columns: REST_COLUMNS.take(), + release: REST_RELEASE.take(), + tables: REST_TABLES.take(), + } + } + + /// Every region of a table's `prove`: what the argue's wall should come to, + /// less the loop's own bookkeeping. + pub fn sum(&self) -> f64 { + self.interactions + + self.tree + + self.output + + self.gkr + + self.setup + + self.batch + + self.values + + self.reduce + + self.columns + + self.release + } +} + +/// One table's shape and its regions, for the census line D-BATCH's plan is +/// sized from: `n`, the committed columns, the factors and how many read a +/// shifted row, the interactions, the input layer's variables `k`, the +/// constraint degree and roots, and the widest bus message. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct AirCensus { + pub index: usize, + pub num_vars: usize, + pub columns: usize, + pub factors: usize, + pub shifted: usize, + pub interactions: usize, + pub input_vars: usize, + pub degree: usize, + pub roots: usize, + pub bus_len_max: usize, + /// Seconds: the tree, `gkr::prove`, and the constraint core (the batch, + /// the factor values, the reduce and its columns). + pub tree: f64, + pub gkr: f64, + pub core: f64, +} + +static AIRS: Mutex> = Mutex::new(Vec::new()); + +/// Note one table's census, numbered by its place in the prove's walk. Inert +/// when the instrument is off. +pub fn note_air(mut air: AirCensus) { + if !enabled() { + return; + } + if let Ok(mut held) = AIRS.lock() { + air.index = held.len(); + held.push(air); + } +} + +/// Seconds since `start`, added to `slot` — zero, and no clock read, when the +/// instrument is off. +#[inline] +pub fn lap(slot: &Slot, start: Option) -> f64 { + match start { + Some(t) => { + let nanos = t.elapsed().as_nanos() as u64; + slot.0.fetch_add(nanos, Ordering::Relaxed); + nanos as f64 * 1e-9 + } + None => 0.0, + } +} + +// ── inside the argue: a zerocheck's device rounds ─────────────────────────── +// +// A big batch holds so many values a thread that its first rounds run a few +// thousand threads (`gpu::LEAN_ABOVE_SLOTS`): the head's widest ran 8,096 at +// 94 ms a launch. `LAMBDA_VM_ARGUE_LEAN_PROGRAM` trades held values for steps, +// which buys threads in the early rounds and costs steps in the late ones, +// where a round is a handful of threads walking the program. These say which +// rounds paid. Timed with [`tick`], as the GKR regions are. + +/// A big batch's round is early while half its cube holds at least this many +/// indices — the head's widest batch had room for 8,096 threads, so its rounds +/// from here up ran short of them. +pub const LATE_HALF: usize = 1 << 13; +/// A big batch's early device rounds: the kernels, the read-back, the +/// challenge drawn and the fold. +pub static ZC_BIG_EARLY: Slot = Slot::new(); +/// A big batch's late device rounds, timed the same way. +pub static ZC_BIG_LATE: Slot = Slot::new(); +/// Every other batch's device rounds. +pub static ZC_OTHER: Slot = Slot::new(); +/// The rounds the host finishes over the crossover cube, every batch's, with +/// the factors taken in and their values read out. +pub static ZC_TAIL: Slot = Slot::new(); +/// Zerocheck sessions over a big batch. +pub static ZC_BIG: Counter = Counter::new(); +/// Of [`ZC_BIG`], the ones whose program ran on demand — the mechanism line of +/// `LAMBDA_VM_ARGUE_LEAN_PROGRAM`. +pub static ZC_LEAN: Counter = Counter::new(); +/// Of [`ZC_LEAN`], the ones `LAMBDA_VM_ARGUE_XCHECK` checked round by round +/// against today's program over the same factors, and found equal. +pub static ZC_LEAN_XCHECKED: Counter = Counter::new(); +/// The rounds in [`ZC_BIG_EARLY`]. +pub static ZC_BIG_EARLY_ROUNDS: Counter = Counter::new(); +/// The rounds in [`ZC_BIG_LATE`]. +pub static ZC_BIG_LATE_ROUNDS: Counter = Counter::new(); +/// Zerocheck sessions that ran D-ARGUE stage 1's fused rounds +/// (`gpu_fused`, `LAMBDA_VM_ARGUE_FUSED`), their device rounds' time, and the +/// sessions the fused rounds declined before round 0 (today's ran). +pub static ZC_FUSED: Counter = Counter::new(); +pub static ZC_FUSED_TIME: Slot = Slot::new(); +pub static ZC_FUSED_DECLINED: Counter = Counter::new(); + +/// A prove's zerocheck rounds, split: seconds per region and the counts they +/// are over. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct ZerocheckSplit { + pub big_early: f64, + pub big_late: f64, + pub other: f64, + pub tail: f64, + pub big: u64, + pub lean: u64, + pub xchecked: u64, + pub early_rounds: u64, + pub late_rounds: u64, + pub fused: u64, + pub fused_time: f64, + pub fused_declined: u64, +} + +impl ZerocheckSplit { + /// Read and clear every zerocheck slot and counter. + fn take() -> Self { + Self { + big_early: ZC_BIG_EARLY.take(), + big_late: ZC_BIG_LATE.take(), + other: ZC_OTHER.take(), + tail: ZC_TAIL.take(), + big: ZC_BIG.take(), + lean: ZC_LEAN.take(), + xchecked: ZC_LEAN_XCHECKED.take(), + early_rounds: ZC_BIG_EARLY_ROUNDS.take(), + late_rounds: ZC_BIG_LATE_ROUNDS.take(), + fused: ZC_FUSED.take(), + fused_time: ZC_FUSED_TIME.take(), + fused_declined: ZC_FUSED_DECLINED.take(), + } + } +} + +/// Start a region timed many times a prove: one clock read, where [`mark`] +/// takes two to place a stage on a sampler's timeline. `None`, and no clock +/// read, when the knob is off. +#[inline] +pub fn tick() -> Option { + enabled().then(Instant::now) +} + +/// Close a region opened by [`tick`] into `slot`. +#[inline] +pub fn add_tick(slot: &Slot, start: Option) { + if let Some(t) = start { + slot.0 + .fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed); + } +} + +/// A transcript that adds the time spent in it to a slot. +/// +/// It makes the same calls, in the same order, on the transcript it wraps, so +/// what it absorbs and draws is that transcript's: wrapping one changes no +/// challenge and no proof byte. +pub struct Timed<'a, T> { + inner: &'a mut T, + slot: &'a Slot, +} + +impl<'a, T> Timed<'a, T> { + pub fn new(inner: &'a mut T, slot: &'a Slot) -> Self { + Self { inner, slot } + } +} + +impl> IsTranscript for Timed<'_, T> { + fn append_field_element(&mut self, element: &FieldElement) { + let t = tick(); + self.inner.append_field_element(element); + add_tick(self.slot, t); + } + + fn append_bytes(&mut self, new_bytes: &[u8]) { + let t = tick(); + self.inner.append_bytes(new_bytes); + add_tick(self.slot, t); + } + + fn mark_statement_end(&mut self) { + self.inner.mark_statement_end(); + } + + fn state(&self) -> [u8; 32] { + self.inner.state() + } + + fn sample_field_element(&mut self) -> FieldElement { + let t = tick(); + let sampled = self.inner.sample_field_element(); + add_tick(self.slot, t); + sampled + } + + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + let t = tick(); + let sampled = self.inner.sample_u64(upper_bound); + add_tick(self.slot, t); + sampled + } +} + +/// Everything the chain parked at the group-loop boundary, awaiting the record. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct ChainSlots { + /// grind, sumcheck, fold, commit_folded, ood, queries. + pub six: [f64; 6], + /// The round loop's own wall, summed over rounds. + pub round_wall: f64, + /// query_sample, tree_rebuild, coset_gather, open_assemble. + pub queries: [f64; 4], + /// Chains run, rounds run, and `open_many` calls made, in the group + /// openings. + pub chain_count: u64, + pub round_count: u64, + pub rebuild_calls: u64, + /// Grouped round-0 trees past each chain's first ([`GROUPED_TREES`]). + pub grouped_trees: u64, +} + +/// The chain's slots, as taken at the group-loop boundary. +static CHAIN_AT_GROUPS: Mutex = Mutex::new(ChainSlots { + six: [0.0; 6], + round_wall: 0.0, + queries: [0.0; 4], + chain_count: 0, + round_count: 0, + rebuild_calls: 0, + grouped_trees: 0, +}); + +/// Park the chain's slots for the record being built one layer up. +pub fn note_chain(chain: ChainSlots) { + if !enabled() { + return; + } + if let Ok(mut held) = CHAIN_AT_GROUPS.lock() { + *held = chain; + } +} + +/// Read and clear every chain slot and counter, in record order. +pub fn take_chain() -> ChainSlots { + ChainSlots { + six: [ + GRIND.take(), + SUMCHECK.take(), + FOLD.take(), + COMMIT_FOLDED.take(), + OOD.take(), + QUERIES.take(), + ], + round_wall: ROUND.take(), + queries: [ + QUERY_SAMPLE.take(), + TREE_REBUILD.take(), + COSET_GATHER.take(), + OPEN_ASSEMBLE.take(), + ], + chain_count: CHAIN_COUNT.take(), + round_count: ROUND_COUNT.take(), + rebuild_calls: REBUILD_CALLS.take(), + grouped_trees: GROUPED_TREES.take(), + } +} + +/// Close a region opened by [`mark`] into `slot`, returning its seconds. +/// +/// It returns the value rather than making the caller read the clock again: +/// a second `elapsed()` for the same region measures a longer one, and the two +/// numbers would then disagree by the cost of the instrument itself. +#[inline] +pub fn add(slot: &Slot, start: Option<(Instant, f64)>) -> f64 { + match start { + Some((t, _)) => { + let nanos = t.elapsed().as_nanos() as u64; + slot.0.fetch_add(nanos, Ordering::Relaxed); + nanos as f64 / 1e9 + } + None => 0.0, + } +} + +/// How many groups the last argument opened, so the record carries the count +/// rather than a reader assuming `epoch_groups`' shape held. +static GROUPS: AtomicUsize = AtomicUsize::new(0); +/// The slowest single table of the current argument, by INDEX into the epoch's +/// table order. +static MAX_TABLE: Mutex> = Mutex::new(None); +/// The epoch's table names, in that same order. +/// +/// ⓘ Set one layer up, because the committed table does not carry a name — it +/// is a layout and a trace. The index is what the argument can observe; the +/// name is what a reader can act on, and only the caller holding the AIRs has +/// it. +static TABLE_NAMES: Mutex> = Mutex::new(Vec::new()); + +/// Note one table's argument time, keeping the maximum by index. +/// +/// ⛔ The SUM alone cannot choose a lever here. `argue` being large is +/// consistent with fifty even tables (where parallelism is the answer) and with +/// one dominating table (where it is not), and those want opposite fixes. +#[inline] +pub fn note_table(index: usize, secs: f64) { + if !enabled() { + return; + } + if let Ok(mut held) = MAX_TABLE.lock() + && held.as_ref().is_none_or(|(_, best)| secs > *best) + { + *held = Some((index, secs)); + } +} + +/// Name the epoch's tables, in the order the argument walks them. +pub fn set_table_names(names: Vec) { + if !enabled() { + return; + } + if let Ok(mut held) = TABLE_NAMES.lock() { + *held = names; + } +} + +/// Note how many groups this argument opened. +#[inline] +pub fn note_groups(n: usize) { + if enabled() { + GROUPS.store(n, Ordering::Relaxed); + } +} + +// ── the overlap falsifier ─────────────────────────────────────────────────── + +/// Proves inside `multi_prove` right now. +static IN_FLIGHT: AtomicUsize = AtomicUsize::new(0); +/// Set once if two proves were ever concurrent. Never cleared: one mixed +/// reading taints every later record, because a slot it polluted is only zeroed +/// by the take that reports it. +static OVERLAPPED: AtomicUsize = AtomicUsize::new(0); + +/// What [`begin_prove`] hands back. +/// +/// ⛔ It releases the count on DROP, not on the success path. `multi_prove` has +/// `?` early-returns, and a release that only ran when the prove succeeded +/// would leave the count stuck at one forever — every later record would then +/// be stamped `OVERLAPPED` by a prove that FAILED rather than by two that +/// overlapped. A false alarm on a falsifier is worse than no falsifier, because +/// it reads as evidence. +#[derive(Debug)] +pub struct ProveGuard(()); + +impl Drop for ProveGuard { + fn drop(&mut self) { + IN_FLIGHT.fetch_sub(1, Ordering::SeqCst); + } +} + +/// Open a prove. Cheap and inert when the knob is off. +pub fn begin_prove() -> Option { + if !enabled() { + return None; + } + if IN_FLIGHT.fetch_add(1, Ordering::SeqCst) + 1 > 1 { + OVERLAPPED.store(1, Ordering::Relaxed); + } + Some(ProveGuard(())) +} + +// ── the per-epoch records ─────────────────────────────────────────────────── + +/// One epoch's producer-side stages. The four **partition** `wall`, so the only +/// thing that can break `execute + collect + build + handoff == wall` is a +/// stage whose timer is missing. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ProducerSplit { + pub index: u64, + pub execute: f64, + pub collect: f64, + pub build: f64, + /// The blocking hand-off to the prover over an unbuffered channel. **This + /// is the backpressure**: large means the prover is the bottleneck, ~0 + /// means the producer is. + pub handoff: f64, + pub wall: f64, +} + +impl ProducerSplit { + /// What the four stages leave over. Named rather than left implicit — an + /// unattributed remainder is how a phase hides. + pub fn other(&self) -> f64 { + self.wall - self.execute - self.collect - self.build - self.handoff + } +} + +/// One epoch's prover-side stages, and the argument's own split inside `prove`. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ProverSplit { + /// The epoch index, or [`GLOBAL_INDEX`] for the cross-epoch stage. + pub index: u64, + pub prep: f64, + pub absorb: f64, + pub commit: f64, + pub prove: f64, + pub wall: f64, + pub challenge: f64, + pub argue: f64, + pub open_groups: f64, + pub open_prepared: f64, + pub groups: usize, + pub max_table: Option<(String, f64)>, + pub airs: usize, + pub overlapped: bool, + /// The chain's six, for the GROUP openings only, in the order + /// [`take_chain`] returns them: grind, sumcheck, fold, commit_folded, ood, + /// queries. The prepared opening keeps its wall and no breakdown — it is + /// 2.5% of the base, and six more fields would not move a ranking. + pub chain: [f64; 6], + /// The round loop's own wall, summed over rounds. The six live inside it; + /// everything else in `open_groups` lives outside it. + pub chain_round_wall: f64, + /// The four inside the QUERIES slot, in the order [`take_chain`] returns + /// them: query_sample, tree_rebuild, coset_gather, open_assemble. + pub queries: [f64; 4], + /// Chains and rounds the group openings ran, and `open_many` calls they + /// made — the three numbers arm F's identity is written over. + pub chain_count: u64, + pub round_count: u64, + pub rebuild_calls: u64, + /// Grouped round-0 trees past each chain's first ([`GROUPED_TREES`]): + /// each one more `open_many` call than `2R − 1` counts. + pub grouped_trees: u64, + /// The device ledger's peak over `commit_grouped`, the argument and the + /// openings, and the budget it is promised against — bytes, 0 without a + /// device ([`RESERVED_COMMIT`], [`RESERVED_ARGUE`], [`RESERVED_OPEN`]). + pub reserved_commit: u64, + pub reserved_argue: u64, + pub reserved_open: u64, + pub reserve_budget: u64, + /// Columns the claim reduces valued on the card, on the host, and checked + /// against the host — [`COLUMNS_ON_CARD`], [`COLUMNS_ON_HOST`], + /// [`COLUMNS_XCHECKED`] — and the two knobs they were valued under. + pub columns_on_card: u64, + pub columns_on_host: u64, + pub columns_xchecked: u64, + pub device_columns: bool, + pub xcheck: bool, + /// Challenge tables the card built, and checked against the host — + /// [`TABLES_ON_CARD`], [`TABLES_XCHECKED`] — and the knob they were built + /// under. + pub tables_on_card: u64, + pub tables_xchecked: u64, + pub device_tables: bool, + /// The device GKR layers, split — [`GkrSplit`]. + pub gkr: GkrSplit, + /// The zerocheck rounds, split — [`ZerocheckSplit`] — and the knob they + /// ran under (`LAMBDA_VM_ARGUE_LEAN_PROGRAM`). + pub zerocheck: ZerocheckSplit, + pub lean_program: bool, + /// The per-table rest, split — [`RestSplit`] — and each table's census. + pub rest: RestSplit, + /// The tree region, split — [`TreeSplit`] — and whether its parts waited + /// for their kernels. + pub tree: TreeSplit, + pub tree_sync: bool, + pub air_census: Vec, +} + +/// The six chain slots' names, in record order — so a message can name the one +/// that went missing instead of printing an index. +pub const CHAIN_NAMES: [&str; 6] = [ + "grind", + "sumcheck", + "fold", + "commit_folded", + "ood", + "queries", +]; + +/// The four query slots' names, in record order — same reason as +/// [`CHAIN_NAMES`]: a message names the slot that went missing. +pub const QUERY_NAMES: [&str; 4] = [ + "query_sample", + "tree_rebuild", + "coset_gather", + "open_assemble", +]; + +/// ★ The ABSOLUTE allowance arm E gives the four query slots, beside its +/// relative one. +/// +/// A purely relative bound cannot work at both shapes the gate runs. Card-free, +/// the fixture's query openings read `0.00` s — a couple of milliseconds — and +/// 3% of that is tens of microseconds, below the glue between the four windows +/// and below the clock's own resolution, so the bound would fire on the honest +/// path. At the block, QUERIES is about a second per epoch and this slack is +/// 0.2%, so any of the four going missing still trips it. +/// +/// ⇒ At the fixture the bound is inert BY DESIGN, and that is exactly why the +/// fixture's real-wiring mutation targets arm F's COUNT rather than a duration. +const QUERY_SLACK: f64 = 0.002; + +/// The index the cross-epoch global stage records under. It is the last thing +/// the base does and it is INSIDE the base's wall, so it belongs in the table — +/// but it is not an epoch and must not be averaged with them. +pub const GLOBAL_INDEX: u64 = u64::MAX; + +/// The index a W-LFM proof (an LFM recursion program proved by this prover, +/// pure WHIR) records under: NOT an epoch and not the base's, so its line reads +/// `WHIR PROVE SPLIT W-LFM` and no reader of the base's table can take it for +/// one of the base's. +pub const LFM_INDEX: u64 = u64::MAX - 1; + +impl ProverSplit { + /// What the four stages leave over. + pub fn other(&self) -> f64 { + self.wall - self.prep - self.absorb - self.commit - self.prove + } + /// What the four inner slots leave over inside `prove`. + pub fn prove_other(&self) -> f64 { + self.prove - self.challenge - self.argue - self.open_groups - self.open_prepared + } + /// The round loop's own bookkeeping: its wall, less the six windows inside + /// it. Must be ≥ 0 — a negative value means the six overlap or reach + /// outside the loop. + pub fn round_other(&self) -> f64 { + self.chain_round_wall - self.chain.iter().sum::() + } + /// Everything inside `open_groups` that is NOT the round loop: the factors + /// built from the weight shares, the stacked polys, the domain clone, the + /// proof assembled after the last round. Must be ≥ 0. + pub fn setup_tail(&self) -> f64 { + self.open_groups - self.chain_round_wall + } + /// What the six leave over inside `open_groups`, whatever its cause. Kept + /// because it is the number the first two defects showed up in. + pub fn chain_other(&self) -> f64 { + self.open_groups - self.chain.iter().sum::() + } + /// What the four leave over inside the QUERIES slot: the glue between the + /// windows `open_many` opens, plus `whir_round::prove`'s own frame. + pub fn queries_other(&self) -> f64 { + self.chain[5] - self.queries.iter().sum::() + } + /// ★ The `open_many` calls the group openings SHOULD have made, from the + /// chains and rounds they ran: `2R − 1` per chain, summed over the chains, + /// which is `2·ΣR − chains` — and one more per grouped round-0 tree past + /// its chain's first (`P' − 1` a grouped chain). + /// + /// ⛔ CHAINS, not groups: `stacked_eval::prove` runs one chain per + /// commitment, so a group can open several. Both numbers are counted by the + /// run, so this is a claim about the call structure that the run can refute + /// — not a constant retyped from a reading of the source. + pub fn derived_rebuild_calls(&self) -> i128 { + 2 * self.round_count as i128 - self.chain_count as i128 + self.grouped_trees as i128 + } + pub fn is_global(&self) -> bool { + self.index == GLOBAL_INDEX + } + /// Who the record is, as both of its lines name it: `#k` for epoch `k` + /// (0-based), `GLOBAL (in base)` for the cross-epoch stage, `W-LFM` for a + /// recursion proof proved by this prover ([`LFM_INDEX`]). + fn who(&self) -> String { + if self.is_global() { + "GLOBAL (in base)".to_string() + } else if self.index == LFM_INDEX { + "W-LFM".to_string() + } else { + format!("#{}", self.index) + } + } + /// The ledger line, whole MiB rounded down: + /// `RESERVED HW #k: commit X MiB · argue Y MiB · open Z MiB · budget B MiB`, + /// stamped `⛔OVERLAPPED` as the split line is when another prove ran. + pub fn reserved_line(&self) -> String { + format!( + "RESERVED HW {who}{tainted}: commit {commit} MiB · argue {argue} MiB · \ + open {open} MiB · budget {budget} MiB", + who = self.who(), + tainted = if self.overlapped { + " ⛔OVERLAPPED" + } else { + "" + }, + commit = self.reserved_commit >> 20, + argue = self.reserved_argue >> 20, + open = self.reserved_open >> 20, + budget = self.reserve_budget >> 20, + ) + } + /// The claim reduces' line, stamped as the split line is: + /// `ARGUE COLUMNS #k: on the card C · on the host H · xchecked X || + /// device columns on|off · xcheck on|off`. The knobs ride on the line + /// because the counts mean nothing without the setting they were taken + /// under. + pub fn columns_line(&self) -> String { + let on_off = |on: bool| if on { "on" } else { "off" }; + format!( + "ARGUE COLUMNS {who}{tainted}: on the card {card} · on the host {host} · \ + xchecked {checked} || device columns {knob} · xcheck {xcheck}", + who = self.who(), + tainted = if self.overlapped { + " ⛔OVERLAPPED" + } else { + "" + }, + card = self.columns_on_card, + host = self.columns_on_host, + checked = self.columns_xchecked, + knob = on_off(self.device_columns), + xcheck = on_off(self.xcheck), + ) + } + /// The challenge tables' line, stamped the same way: + /// `ARGUE TABLES #k: built on the card N · xchecked X || device tables + /// on|off · xcheck on|off`. + pub fn tables_line(&self) -> String { + let on_off = |on: bool| if on { "on" } else { "off" }; + format!( + "ARGUE TABLES {who}{tainted}: built on the card {card} · xchecked {checked} || \ + device tables {knob} · xcheck {xcheck}", + who = self.who(), + tainted = if self.overlapped { + " ⛔OVERLAPPED" + } else { + "" + }, + card = self.tables_on_card, + checked = self.tables_xchecked, + knob = on_off(self.device_tables), + xcheck = on_off(self.xcheck), + ) + } + /// The zerocheck line, milliseconds, stamped as the split line is: + /// `ARGUE ZEROCHECK #k: big sessions B (lean L · xchecked X) · early rounds + /// E · late rounds R || big early X · big late Y · other Z · host tail T + /// (ms) || lean program on|off · xcheck on|off || fused F (declined D) · M ms` + /// — the last part D-ARGUE stage 1's sessions and their device rounds' time. + pub fn zerocheck_line(&self) -> String { + let z = &self.zerocheck; + let ms = |secs: f64| secs * 1e3; + let on_off = |on: bool| if on { "on" } else { "off" }; + format!( + "ARGUE ZEROCHECK {who}{tainted}: big sessions {big} (lean {lean} · xchecked \ + {checked}) · early rounds {early} · late rounds {late} || big early {big_early:.2} · \ + big late {big_late:.2} · other {other:.2} · host tail {tail:.2} (ms) || lean \ + program {knob} · xcheck {xcheck} || fused {fused} (declined {declined}) · \ + {fused_ms:.2} ms", + who = self.who(), + tainted = if self.overlapped { + " ⛔OVERLAPPED" + } else { + "" + }, + big = z.big, + lean = z.lean, + checked = z.xchecked, + early = z.early_rounds, + late = z.late_rounds, + big_early = ms(z.big_early), + big_late = ms(z.big_late), + other = ms(z.other), + tail = ms(z.tail), + knob = on_off(self.lean_program), + xcheck = on_off(self.xcheck), + fused = z.fused, + declined = z.fused_declined, + fused_ms = ms(z.fused_time), + ) + } + /// The tree line: `ARGUE TREE #k: lift · write (lower) · fold · output · + /// of tree T (ms) || sync on|off` — [`TreeSplit`] beside the region it + /// splits. + pub fn tree_line(&self) -> String { + let t = &self.tree; + let ms = |secs: f64| secs * 1e3; + format!( + "ARGUE TREE {who}{tainted}: lift {lift:.2} · write {write:.2} (lower {lower:.2}) · fold \ + {fold:.2} · output {output:.2} · of tree {tree:.2} (ms) || sync {sync} || from the \ + columns {from_columns} · no lift {no_lift} (late {late_lift})", + who = self.who(), + tainted = if self.overlapped { + " ⛔OVERLAPPED" + } else { + "" + }, + lift = ms(t.lift), + write = ms(t.write), + lower = ms(t.lower), + fold = ms(t.fold), + output = ms(t.output), + tree = ms(self.rest.tree), + sync = if self.tree_sync { "on" } else { "off" }, + from_columns = t.from_columns, + no_lift = t.no_lift, + late_lift = t.late_lift, + ) + } + + /// The rest line, milliseconds, stamped as the split line is: + /// `ARGUE REST #k: tables T || interactions · tree · output · gkr (prefix) + /// · setup · batch · values · reduce · columns · release || Σ S of argue A + /// (ms)` — every region of a table's `prove`, summed over the epoch's + /// tables, against the argue's wall. + pub fn rest_line(&self) -> String { + let r = &self.rest; + let ms = |secs: f64| secs * 1e3; + format!( + "ARGUE REST {who}{tainted}: tables {tables} || interactions {interactions:.2} · tree \ + {tree:.2} · output {output:.2} · gkr {gkr:.2} (prefix {prefix:.2}) · setup {setup:.2} · \ + batch {batch:.2} · values {values:.2} · reduce {reduce:.2} · columns {columns:.2} · \ + release {release:.2} || Σ {sum:.2} of argue {argue:.2} (ms)", + who = self.who(), + tainted = if self.overlapped { + " ⛔OVERLAPPED" + } else { + "" + }, + tables = r.tables, + interactions = ms(r.interactions), + tree = ms(r.tree), + output = ms(r.output), + gkr = ms(r.gkr), + prefix = ms(r.prefix), + setup = ms(r.setup), + batch = ms(r.batch), + values = ms(r.values), + reduce = ms(r.reduce), + columns = ms(r.columns), + release = ms(r.release), + sum = ms(r.sum()), + argue = ms(self.argue), + ) + } + + /// One table's census line: + /// `ARGUE AIR #k t NAME: n N · cols W · factors F (shifted S) · interactions + /// I · k K · degree D · roots R · bus len L || tree · gkr · core (ms)`. + pub fn air_line(&self, air: &AirCensus, name: &str) -> String { + let ms = |secs: f64| secs * 1e3; + format!( + "ARGUE AIR {who} {at} {name}: n {n} · cols {cols} · factors {factors} (shifted {shifted}) \ + · interactions {interactions} · k {k} · degree {degree} · roots {roots} · bus len {bus} \ + || tree {tree:.2} · gkr {gkr:.2} · core {core:.2} (ms)", + who = self.who(), + at = air.index, + n = air.num_vars, + cols = air.columns, + factors = air.factors, + shifted = air.shifted, + interactions = air.interactions, + k = air.input_vars, + degree = air.degree, + roots = air.roots, + bus = air.bus_len_max, + tree = ms(air.tree), + gkr = ms(air.gkr), + core = ms(air.core), + ) + } + + /// The GKR line, milliseconds, stamped as the split line is: + /// `ARGUE GKR #k: layers on the card N (rebuilt R) · host tail rounds T || + /// between layers B ms, M µs a layer: lambda · program · lower · session · + /// factors · tail (transcript) · close || rebuild · rounds · values (ms) || + /// gruen G (xchecked X)` — the last part D-ARGUE S1-3's layers. + pub fn gkr_line(&self) -> String { + let g = &self.gkr; + let ms = |secs: f64| secs * 1e3; + let mean = if g.layers == 0 { + 0.0 + } else { + g.between() * 1e6 / g.layers as f64 + }; + format!( + "ARGUE GKR {who}{tainted}: layers on the card {layers} (rebuilt {rebuilt}) · host tail \ + rounds {tail_rounds} || between layers {between:.2} ms, {mean:.1} µs a layer: lambda \ + {lambda:.2} · program {program:.2} · lower {lower:.2} · session {session:.2} · \ + factors {factors:.2} · tail {tail:.2} (transcript {heard:.2}) · close {close:.2} || \ + rebuild {rebuild:.2} · rounds {rounds:.2} · values {values:.2} (ms) || gruen \ + {gruen} (xchecked {gruen_xchecked})", + who = self.who(), + tainted = if self.overlapped { + " ⛔OVERLAPPED" + } else { + "" + }, + layers = g.layers, + rebuilt = g.rebuilds, + tail_rounds = g.tail_rounds, + between = ms(g.between()), + lambda = ms(g.lambda), + program = ms(g.program), + lower = ms(g.lower), + session = ms(g.session), + factors = ms(g.factors), + tail = ms(g.tail), + heard = ms(g.tail_transcript), + close = ms(g.close), + rebuild = ms(g.rebuild), + rounds = ms(g.rounds), + values = ms(g.values), + gruen = g.gruen, + gruen_xchecked = g.gruen_xchecked, + ) + } +} + +static PRODUCER: Mutex> = Mutex::new(Vec::new()); +static PROVER: Mutex> = Mutex::new(Vec::new()); + +/// Record one epoch's producer stages. +pub fn push_producer(rec: ProducerSplit) { + if !enabled() { + return; + } + if let Ok(mut held) = PRODUCER.lock() { + held.push(rec); + } +} + +/// Close the prover's stages into a record: take the inner slots, print the +/// line, and store it. +/// +/// The line and the record come from the SAME values, so the shell's parse and +/// the harness's table cannot disagree about a number they both report. +pub fn push_prover(mut rec: ProverSplit) { + if !enabled() { + return; + } + rec.challenge = CHALLENGE.take(); + rec.argue = ARGUE.take(); + rec.open_groups = OPEN_GROUPS.take(); + rec.open_prepared = OPEN_PREPARED.take(); + rec.groups = GROUPS.swap(0, Ordering::Relaxed); + // ⓘ THE SLOTS ARE DISCARDED HERE, NOT READ. By this point they hold the + // PREPARED opening's chain — the group openings' were taken and parked at + // the end of the group loop, before DECODE's ran. Reading them now would + // put DECODE's chain under `open_groups`'s name. So: drop what they hold + // (which also keeps it out of the next epoch), then take the parked pair. + let _ = take_chain(); + let parked = CHAIN_AT_GROUPS + .lock() + .map(|mut h| std::mem::take(&mut *h)) + .unwrap_or_default(); + rec.chain = parked.six; + rec.chain_round_wall = parked.round_wall; + rec.queries = parked.queries; + rec.chain_count = parked.chain_count; + rec.round_count = parked.round_count; + rec.rebuild_calls = parked.rebuild_calls; + rec.grouped_trees = parked.grouped_trees; + let names = TABLE_NAMES + .lock() + .map(|mut h| std::mem::take(&mut *h)) + .unwrap_or_default(); + rec.rest = RestSplit::take(); + rec.tree = TreeSplit::take(); + rec.tree_sync = tree_sync(); + rec.air_census = AIRS + .lock() + .map(|mut h| std::mem::take(&mut *h)) + .unwrap_or_default(); + for air in &rec.air_census { + let name = names + .get(air.index) + .cloned() + .unwrap_or_else(|| format!("table#{}", air.index)); + println!("{}", rec.air_line(air, &name)); + } + rec.max_table = MAX_TABLE + .lock() + .ok() + .and_then(|mut h| h.take()) + .map(|(at, secs)| { + let name = names + .get(at) + .cloned() + .unwrap_or_else(|| format!("table#{at}")); + (name, secs) + }); + rec.overlapped = OVERLAPPED.load(Ordering::Relaxed) != 0; + rec.reserved_commit = RESERVED_COMMIT.take(); + rec.reserved_argue = RESERVED_ARGUE.take(); + rec.reserved_open = RESERVED_OPEN.take(); + rec.reserve_budget = crate::gpu::reserve_budget(); + rec.columns_on_card = COLUMNS_ON_CARD.take(); + rec.columns_on_host = COLUMNS_ON_HOST.take(); + rec.columns_xchecked = COLUMNS_XCHECKED.take(); + rec.device_columns = crate::gpu::argue_device_columns(); + rec.xcheck = crate::gpu::argue_xcheck(); + rec.tables_on_card = TABLES_ON_CARD.take(); + rec.tables_xchecked = TABLES_XCHECKED.take(); + rec.device_tables = crate::gpu::argue_device_tables(); + rec.gkr = GkrSplit::take(); + rec.zerocheck = ZerocheckSplit::take(); + rec.lean_program = crate::gpu::argue_lean_program(); + + let who = rec.who(); + let (max_name, max_secs) = rec + .max_table + .clone() + .unwrap_or_else(|| ("-".to_string(), 0.0)); + println!( + "WHIR PROVE SPLIT {who}{tainted}: airs {airs} · wall {wall:.2}s · \ + prep {prep:.2} · absorb {absorb:.3} · commit {commit:.2} · \ + prove {prove:.2} · other {other:.2} || inside[Σ] challenge {challenge:.3} · \ + argue {argue:.2} (max {max_name} {max_secs:.2}) · \ + open_groups {open_groups:.2} ({groups} groups) · \ + open_prepared {open_prepared:.2} · other {prove_other:.2} || \ + chain[Σ groups] grind {c0:.2} · sumcheck {c1:.2} · fold {c2:.2} · \ + commit_folded {c3:.2} · ood {c4:.2} · queries {c5:.2} || \ + round_wall {cw:.2} · round_other {c6:.2} · setup_tail {cst:.2} || \ + queries[Σ groups] query_sample {q0:.2} · tree_rebuild {q1:.2} · \ + coset_gather {q2:.2} · open_assemble {q3:.2} · queries_other {qo:.2} \ + || chains {chains} · rounds {rounds} · rebuild_calls {rebuilds} \ + (derived {rderiv})", + tainted = if rec.overlapped { " ⛔OVERLAPPED" } else { "" }, + airs = rec.airs, + wall = rec.wall, + prep = rec.prep, + absorb = rec.absorb, + commit = rec.commit, + prove = rec.prove, + other = rec.other(), + challenge = rec.challenge, + argue = rec.argue, + open_groups = rec.open_groups, + groups = rec.groups, + open_prepared = rec.open_prepared, + prove_other = rec.prove_other(), + c0 = rec.chain[0], + c1 = rec.chain[1], + c2 = rec.chain[2], + c3 = rec.chain[3], + c4 = rec.chain[4], + c5 = rec.chain[5], + cw = rec.chain_round_wall, + c6 = rec.round_other(), + cst = rec.setup_tail(), + q0 = rec.queries[0], + q1 = rec.queries[1], + q2 = rec.queries[2], + q3 = rec.queries[3], + qo = rec.queries_other(), + chains = rec.chain_count, + rounds = rec.round_count, + rebuilds = rec.rebuild_calls, + rderiv = rec.derived_rebuild_calls(), + ); + println!("{}", rec.reserved_line()); + println!("{}", rec.columns_line()); + println!("{}", rec.tables_line()); + println!("{}", rec.gkr_line()); + println!("{}", rec.zerocheck_line()); + println!("{}", rec.rest_line()); + println!("{}", rec.tree_line()); + + if let Ok(mut held) = PROVER.lock() { + held.push(rec); + } +} + +/// Take every record collected so far, clearing the stores. +/// +/// Clearing is what keeps a later phase — the LFM tree proves in the same +/// process — from being read as part of the base. +pub fn drain() -> (Vec, Vec) { + let producer = PRODUCER + .lock() + .map(|mut h| std::mem::take(&mut *h)) + .unwrap_or_default(); + let prover = PROVER + .lock() + .map(|mut h| std::mem::take(&mut *h)) + .unwrap_or_default(); + (producer, prover) +} + +/// Does the breakdown close? `Ok(())`, or the first failure spelled out. +/// +/// ★ A PURE FUNCTION OVER THE RECORDS, deliberately. The arms it runs are the +/// only reason the table can be quoted, so they have to be testable against +/// MANUFACTURED states — a state they must accept and states they must refuse — +/// rather than only against whatever a real run happens to produce. A check +/// that has never been seen to fail is not evidence. +/// +/// ⛔ IT DOES NOT CHECK `Σ producer + Σ prover == base`. The two run on +/// different threads at once, so that identity is false on a CORRECT +/// instrument: the base wall is epoch 0's preparation plus the proofs plus the +/// pipeline's waiting, while the sum double-counts every overlap. Asserting it +/// would redden the honest path, and a check that reddens honestly gets its +/// tolerance widened until it cannot fail at all. Arm D asserts the two +/// INEQUALITIES that are actually true of a two-thread pipeline. +pub fn check_closure( + producer: &[ProducerSplit], + prover: &[ProverSplit], + base_secs: f64, + tol: f64, +) -> Result<(), String> { + for r in prover { + let who = if r.is_global() { + "global".to_string() + } else { + format!("epoch {}", r.index) + }; + // Arm A: prep + absorb + commit + prove partition the prover's wall. + if r.other().abs() > tol * r.wall.max(1e-9) { + return Err(format!( + "arm A: {who}'s prover stages do not close — wall {:.3}s but \ + prep+absorb+commit+prove = {:.3}s, leaving {:.3}s ({:.1}%) \ + unattributed. A stage's timer is missing.", + r.wall, + r.wall - r.other(), + r.other(), + 100.0 * r.other() / r.wall.max(1e-9), + )); + } + // Arm B: the four inner slots partition `prove`. + if r.prove_other().abs() > tol * r.prove.max(1e-9) { + return Err(format!( + "arm B: {who}'s argument does not close — prove {:.3}s but \ + challenge+argue+open_groups+open_prepared = {:.3}s, leaving \ + {:.3}s ({:.1}%) unattributed inside `prove`.", + r.prove, + r.prove - r.prove_other(), + r.prove_other(), + 100.0 * r.prove_other() / r.prove.max(1e-9), + )); + } + // Arm E: the chain's two remainders must be NON-NEGATIVE. + // + // ⛔ NOT "the six sum to `open_groups`". With the round wall measured, + // `Σ(six) + round_other + setup_tail = open_groups` is an IDENTITY — + // the two remainders are defined as the differences — so asserting it + // would be a check that cannot fail, which is worse than no check. + // + // What can fail, and did twice: a remainder going NEGATIVE. That is not + // drift. It means the parts are not parts, and it has two causes — a + // window reaching outside what contains it, and windows that overlap + // and double-count. `round_other < 0` says the six overlap or escape + // the loop; `setup_tail < 0` says the loop's wall escapes + // `open_groups`. Both are structural errors in the instrument, and + // neither is reachable by construction. + let slots = || -> String { + CHAIN_NAMES + .iter() + .zip(r.chain.iter()) + .map(|(n, v)| format!("{n} {v:.3}")) + .collect::>() + .join(" · ") + }; + // ⛔ ORDER MATTERS, and the first draft had it wrong. CONTAINMENT + // failures are checked before the ACCOUNTING one: a loop wall that + // escapes its opening also leaves a huge positive `round_other`, so + // with the accounting check first it was reported as "a slot is not + // being added" — the wrong defect, named confidently. Structure first, + // then arithmetic. + if r.setup_tail() < -tol * r.open_groups.max(1e-9) { + return Err(format!( + "arm E: {who}'s round loop does not fit its opening — \ + open_groups is {:.3}s but the loop's wall is {:.3}s, leaving \ + {:.3}s. NEGATIVE, so the loop's window reaches outside the \ + opening that contains it. Slots: {}.", + r.open_groups, + r.chain_round_wall, + r.setup_tail(), + slots(), + )); + } + if r.round_other() < -tol * r.open_groups.max(1e-9) { + return Err(format!( + "arm E: {who}'s six do not fit the round loop — the loop's wall \ + is {:.3}s but the six inside it sum to {:.3}s, leaving \ + {:.3}s. NEGATIVE, so the six overlap each other or reach \ + outside the loop: they are not a partition of it. Slots: {}.", + r.chain_round_wall, + r.chain.iter().sum::(), + r.round_other(), + slots(), + )); + } + // ⛔⛔ AND AN UPPER BOUND, because dropping it cost the arm its power. + // Asserting only `>= 0` made a MISSING slot invisible: omit one of the + // six and Σ(six) shrinks, so `round_other = round_wall − Σ(six)` GROWS + // — positive, allowed, unseen. The gate proved it: the same mutation + // arm E caught before the round wall existed sailed through after it. + // + // The identity was not the thing to remove; asserting the identity was. + // `round_other` is the loop's own bookkeeping between windows and reads + // **0.00 on every record** on a correct instrument, so a few percent of + // the loop's wall is enormous headroom on the honest path AND trips on + // any omitted slot bigger than that. `setup_tail` keeps only `>= 0`: + // it is legitimately un-slotted work outside the loop, and bounding it + // would be asserting a size nobody measured. + if r.round_other() > tol * r.chain_round_wall.max(1e-9) { + let named: Vec = CHAIN_NAMES + .iter() + .zip(r.chain.iter()) + .map(|(n, v)| format!("{n} {v:.3}")) + .collect(); + return Err(format!( + "arm E: {who}'s six do not account for the round loop — the \ + loop's wall is {:.3}s but the six inside it sum to only \ + {:.3}s, leaving {:.3}s ({:.1}%) unattributed. The loop's own \ + bookkeeping is ~0 on a correct instrument, so a gap this size \ + is a SLOT THAT IS NOT BEING ADDED. Slots: {}.", + r.chain_round_wall, + r.chain.iter().sum::(), + r.round_other(), + 100.0 * r.round_other() / r.chain_round_wall.max(1e-9), + named.join(" · "), + )); + } + // Arm E, the query half: the four inside QUERIES, containment first. + let queries_secs = r.chain[5]; + let query_bound = tol * queries_secs.max(0.0) + QUERY_SLACK; + let query_slots = || -> String { + QUERY_NAMES + .iter() + .zip(r.queries.iter()) + .map(|(n, v)| format!("{n} {v:.4}")) + .collect::>() + .join(" · ") + }; + if r.queries_other() < -query_bound { + return Err(format!( + "arm E: {who}'s four query slots OVERRUN the openings that \ + contain them — QUERIES is {queries_secs:.3}s but the four sum \ + to {:.3}s, a NEGATIVE remainder of {:.3}s. A sum of parts \ + cannot exceed the whole that contains it: either a window \ + reaches outside `open_many` or two of them nest. Slots: {}.", + r.queries.iter().sum::(), + r.queries_other(), + query_slots(), + )); + } + if r.queries_other() > query_bound { + return Err(format!( + "arm E: {who}'s four do not account for the query openings — \ + QUERIES is {queries_secs:.3}s but the four inside it sum to \ + only {:.3}s, leaving {:.3}s ({:.1}%) unattributed. The glue \ + between the windows is ~0 on a correct instrument, so a gap \ + this size is A SLOT THAT IS NOT BEING ADDED. Slots: {}.", + r.queries.iter().sum::(), + r.queries_other(), + 100.0 * r.queries_other() / queries_secs.max(1e-9), + query_slots(), + )); + } + // Arm F: the rebuild count, against the rounds that produced it. + // + // `whir_round::prove` opens the CURRENT commitment and the NEXT one, + // and the last round opens only the current through `final_openings`, + // so a chain of R rounds makes `2R − 1` `open_many` calls — one device + // tree rebuild each — and g chains make `2·ΣR − g`. BOTH SIDES ARE + // COUNTED BY THE RUN; neither is a constant read off the source, so + // this is a claim about the call structure that the run can refute. + // + // ⭐ It is also the only arm here that can fire on the card-free + // fixture, where every duration in the chain's query half reads 0.00. + // ⛔ THE GUARD IS "EITHER SIDE IS NONZERO", NOT "THE ROUNDS ARE". + // Guarding on the rounds alone would make the arm blind to exactly one + // of the two omissions it exists to catch: drop the ROUND count and + // `round_count` is 0, the guard skips, and the missing counter is + // invisible. A record that genuinely ran no chain has BOTH at zero, and + // that is the only state this arm may pass over. + if (r.chain_count > 0 || r.round_count > 0 || r.rebuild_calls > 0) + && r.rebuild_calls as i128 != r.derived_rebuild_calls() + { + return Err(format!( + "arm F: {who}'s query openings made {} `open_many` calls, but \ + {} round(s) over {} chain(s) and {} grouped round-0 tree(s) \ + past their chains' first derive {} (2R − 1 per chain, plus one \ + a grouped tree). Either a call is not being counted, or the \ + chain no longer opens the current commitment and its \ + successor once each per round.", + r.rebuild_calls, + r.round_count, + r.chain_count, + r.grouped_trees, + r.derived_rebuild_calls(), + )); + } + } + // Arm C: execute + collect + build + handoff partition the producer's wall. + for r in producer { + if r.other().abs() > tol * r.wall.max(1e-9) { + return Err(format!( + "arm C: epoch {}'s producer stages do not close — wall {:.3}s \ + but execute+collect+build+handoff = {:.3}s, leaving {:.3}s \ + ({:.1}%) unattributed. A stage's timer is missing.", + r.index, + r.wall, + r.wall - r.other(), + r.other(), + 100.0 * r.other() / r.wall.max(1e-9), + )); + } + } + // Arm D: the two inequalities a two-thread pipeline really satisfies. + let p_wall: f64 = producer.iter().map(|r| r.wall).sum(); + let v_wall: f64 = prover.iter().map(|r| r.wall).sum(); + let busiest = p_wall.max(v_wall); + if busiest > base_secs * (1.0 + tol) { + return Err(format!( + "arm D: the busiest thread ({busiest:.2}s) exceeds the base wall \ + ({base_secs:.2}s) — a thread cannot take longer than the pipeline \ + that contains it.", + )); + } + if base_secs > (p_wall + v_wall) * (1.0 + tol) { + return Err(format!( + "arm D: the base wall ({base_secs:.2}s) exceeds the serial bound \ + ({:.2}s) — the pipeline took longer than running every stage one \ + after another, so time is being spent outside every stage.", + p_wall + v_wall, + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// ⛔ The disabled path must read no clock. Asserted through the only + /// observable it has: [`mark`] returns `None`, so [`add`] cannot move a + /// slot and [`stage_done`] cannot print. + /// + /// The knob is a process-wide `OnceLock` and the test binary does not set + /// it, so this is the state every other test in the crate runs under. + #[test] + fn disabled_is_inert() { + assert!( + !enabled(), + "the test binary must not set LAMBDA_VM_BASE_SPLIT" + ); + assert!(mark().is_none(), "a disabled mark must read no clock"); + assert_eq!( + add(&ARGUE, mark()), + 0.0, + "a disabled add must not move a slot" + ); + assert_eq!(ARGUE.take(), 0.0, "a disabled add must not move a slot"); + // ★ And the COUNTERS, which are the one thing here that is not a clock + // read: `bump` has to check the knob itself, because unlike `add` it + // takes no `Option` that a disabled `mark` could have emptied. + bump(&REBUILD_CALLS); + bump(&ROUND_COUNT); + bump(&CHAIN_COUNT); + assert_eq!( + (REBUILD_CALLS.take(), ROUND_COUNT.take(), CHAIN_COUNT.take()), + (0, 0, 0), + "a disabled bump must not move a counter" + ); + bump_by(&COLUMNS_ON_CARD, 5); + bump_by(&COLUMNS_ON_HOST, 7); + bump_by(&COLUMNS_XCHECKED, 11); + bump_by(&TABLES_ON_CARD, 13); + bump_by(&TABLES_XCHECKED, 17); + bump(&GKR_LAYERS); + bump(&GKR_REBUILDS); + bump_by(&GKR_TAIL_ROUNDS, 9); + bump(&ZC_BIG); + bump(&ZC_LEAN); + bump(&ZC_LEAN_XCHECKED); + bump(&ZC_BIG_EARLY_ROUNDS); + bump(&ZC_BIG_LATE_ROUNDS); + assert_eq!( + ( + COLUMNS_ON_CARD.take(), + COLUMNS_ON_HOST.take(), + COLUMNS_XCHECKED.take(), + TABLES_ON_CARD.take(), + TABLES_XCHECKED.take(), + GKR_LAYERS.take(), + GKR_REBUILDS.take(), + GKR_TAIL_ROUNDS.take() + ), + (0, 0, 0, 0, 0, 0, 0, 0), + "a disabled bump_by must not move a counter" + ); + // The GKR regions' one-read clock is inert the same way. + assert!(tick().is_none(), "a disabled tick must read no clock"); + add_tick(&GKR_TAIL, tick()); + assert_eq!( + GkrSplit::take(), + GkrSplit::default(), + "a disabled tick must not move a slot" + ); + add_tick(&ZC_BIG_EARLY, tick()); + add_tick(&ZC_TAIL, tick()); + assert_eq!( + ZerocheckSplit::take(), + ZerocheckSplit::default(), + "a disabled tick or bump must not move a zerocheck slot or counter" + ); + assert_eq!(stage_done(0, "execute", mark()), 0.0); + note_table(3, 9.0); + set_table_names(vec!["KECCAK".to_string()]); + note_groups(7); + push_producer(ProducerSplit { + index: 0, + wall: 1.0, + ..Default::default() + }); + push_prover(ProverSplit { + index: 0, + wall: 1.0, + ..Default::default() + }); + let (producer, prover) = drain(); + assert!( + producer.is_empty() && prover.is_empty(), + "disabled records nothing" + ); + // A ledger window is inert too: nothing opened, nothing stored. + let opened = open_reserved(); + assert!(!opened, "a disabled window must not open"); + close_reserved(&RESERVED_ARGUE, opened); + assert_eq!( + RESERVED_ARGUE.take(), + 0, + "a disabled window must not move a peak" + ); + } + + /// ★ The ledger line's shape is what the probe's readout parses: the same + /// `who` as the split line, whole MiB, the budget last, and the overlap + /// stamp where the split line puts it. + #[test] + fn the_ledger_line_names_its_prove_and_reads_whole_mib() { + let epoch = ProverSplit { + index: 3, + reserved_commit: (7 << 20) + 5, + reserved_argue: 22 << 30, + reserved_open: 1 << 20, + reserve_budget: 26_086 << 20, + ..Default::default() + }; + assert_eq!( + epoch.reserved_line(), + "RESERVED HW #3: commit 7 MiB · argue 22528 MiB · open 1 MiB · budget 26086 MiB" + ); + let global = ProverSplit { + index: GLOBAL_INDEX, + overlapped: true, + ..Default::default() + }; + assert_eq!( + global.reserved_line(), + "RESERVED HW GLOBAL (in base) ⛔OVERLAPPED: commit 0 MiB · argue 0 MiB · \ + open 0 MiB · budget 0 MiB" + ); + } + + /// The columns line carries its counts in a fixed order and the two knobs + /// they were taken under, with the split line's `who` and overlap stamp. + #[test] + fn the_columns_line_names_its_prove_and_its_knobs() { + let epoch = ProverSplit { + index: 4, + columns_on_card: 1480, + columns_on_host: 38, + columns_xchecked: 1480, + device_columns: true, + xcheck: true, + ..Default::default() + }; + assert_eq!( + epoch.columns_line(), + "ARGUE COLUMNS #4: on the card 1480 · on the host 38 · xchecked 1480 || \ + device columns on · xcheck on" + ); + let global = ProverSplit { + index: GLOBAL_INDEX, + overlapped: true, + columns_on_host: 9, + ..Default::default() + }; + assert_eq!( + global.columns_line(), + "ARGUE COLUMNS GLOBAL (in base) ⛔OVERLAPPED: on the card 0 · on the host 9 · \ + xchecked 0 || device columns off · xcheck off" + ); + } + + /// The tables line: its count, its checked count and its knob, stamped + /// like the split line. + #[test] + fn the_tables_line_names_its_prove_and_its_knob() { + let epoch = ProverSplit { + index: 2, + tables_on_card: 84, + tables_xchecked: 84, + device_tables: true, + xcheck: true, + ..Default::default() + }; + assert_eq!( + epoch.tables_line(), + "ARGUE TABLES #2: built on the card 84 · xchecked 84 || device tables on · xcheck on" + ); + let global = ProverSplit { + index: GLOBAL_INDEX, + overlapped: true, + ..Default::default() + }; + assert_eq!( + global.tables_line(), + "ARGUE TABLES GLOBAL (in base) ⛔OVERLAPPED: built on the card 0 · xchecked 0 || \ + device tables off · xcheck off" + ); + } + + /// The GKR line: its counts, the transition's sum and its mean a layer, + /// then every slot in milliseconds, stamped like the split line. The rebuild, + /// the rounds and the read-back stay out of the transition. + #[test] + fn the_gkr_line_names_its_prove_and_splits_a_transition() { + let epoch = ProverSplit { + index: 5, + gkr: GkrSplit { + lambda: 0.002, + program: 0.001, + lower: 0.003, + session: 0.010, + rebuild: 0.050, + rounds: 0.200, + values: 0.004, + factors: 0.0005, + tail: 0.030, + tail_transcript: 0.020, + close: 0.0035, + layers: 200, + rebuilds: 12, + tail_rounds: 1800, + gruen: 150, + gruen_xchecked: 140, + }, + ..Default::default() + }; + assert!((epoch.gkr.between() - 0.05).abs() < 1e-12); + assert_eq!( + epoch.gkr_line(), + "ARGUE GKR #5: layers on the card 200 (rebuilt 12) · host tail rounds 1800 || \ + between layers 50.00 ms, 250.0 µs a layer: lambda 2.00 · program 1.00 · lower 3.00 · \ + session 10.00 · factors 0.50 · tail 30.00 (transcript 20.00) · close 3.50 || \ + rebuild 50.00 · rounds 200.00 · values 4.00 (ms) || gruen 150 (xchecked 140)" + ); + let global = ProverSplit { + index: GLOBAL_INDEX, + overlapped: true, + ..Default::default() + }; + assert_eq!( + global.gkr_line(), + "ARGUE GKR GLOBAL (in base) ⛔OVERLAPPED: layers on the card 0 (rebuilt 0) · host tail \ + rounds 0 || between layers 0.00 ms, 0.0 µs a layer: lambda 0.00 · program 0.00 · \ + lower 0.00 · session 0.00 · factors 0.00 · tail 0.00 (transcript 0.00) · close 0.00 \ + || rebuild 0.00 · rounds 0.00 · values 0.00 (ms) || gruen 0 (xchecked 0)" + ); + } + + /// The rest line: every region in milliseconds, their sum against the + /// argue's wall; the census line: a table's shape and its three regions. + #[test] + fn the_rest_and_air_lines_split_a_table() { + let epoch = ProverSplit { + index: 4, + argue: 0.100, + rest: RestSplit { + interactions: 0.001, + tree: 0.010, + output: 0.0005, + gkr: 0.040, + prefix: 0.002, + setup: 0.003, + batch: 0.030, + values: 0.0005, + reduce: 0.008, + columns: 0.004, + release: 0.001, + tables: 22, + }, + ..Default::default() + }; + assert!((epoch.rest.sum() - 0.098).abs() < 1e-12); + assert_eq!( + epoch.rest_line(), + "ARGUE REST #4: tables 22 || interactions 1.00 · tree 10.00 · output 0.50 · gkr 40.00 \ + (prefix 2.00) · setup 3.00 · batch 30.00 · values 0.50 · reduce 8.00 · columns 4.00 · \ + release 1.00 || Σ 98.00 of argue 100.00 (ms)" + ); + let air = AirCensus { + index: 7, + num_vars: 16, + columns: 1480, + factors: 1482, + shifted: 0, + interactions: 1031, + input_vars: 27, + degree: 4, + roots: 140, + bus_len_max: 12, + tree: 0.020, + gkr: 0.150, + core: 0.300, + }; + assert_eq!( + epoch.air_line(&air, "KECCAK_RND"), + "ARGUE AIR #4 7 KECCAK_RND: n 16 · cols 1480 · factors 1482 (shifted 0) · interactions \ + 1031 · k 27 · degree 4 · roots 140 · bus len 12 || tree 20.00 · gkr 150.00 · core \ + 300.00 (ms)" + ); + } + + /// The tree line: the tree region's parts beside the region, and whether + /// they waited for their kernels. + #[test] + fn the_tree_line_splits_the_tree_region() { + let epoch = ProverSplit { + index: 3, + rest: RestSplit { + tree: 0.120, + ..Default::default() + }, + tree: TreeSplit { + lift: 0.010, + lower: 0.020, + write: 0.050, + fold: 0.040, + output: 0.001, + ..Default::default() + }, + ..Default::default() + }; + assert_eq!( + epoch.tree_line(), + "ARGUE TREE #3: lift 10.00 · write 50.00 (lower 20.00) · fold 40.00 · output 1.00 · of \ + tree 120.00 (ms) || sync off || from the columns 0 · no lift 0 (late 0)" + ); + } + + /// The zerocheck line: its session and round counts, then every slot in + /// milliseconds and the knob, stamped like the split line. + #[test] + fn the_zerocheck_line_names_its_prove_and_its_knob() { + let epoch = ProverSplit { + index: 3, + zerocheck: ZerocheckSplit { + big_early: 1.25, + big_late: 0.0405, + other: 0.5, + tail: 0.012, + big: 4, + lean: 4, + xchecked: 4, + early_rounds: 30, + late_rounds: 36, + fused: 21, + fused_time: 0.8, + fused_declined: 2, + }, + lean_program: true, + xcheck: true, + ..Default::default() + }; + assert_eq!( + epoch.zerocheck_line(), + "ARGUE ZEROCHECK #3: big sessions 4 (lean 4 · xchecked 4) · early rounds 30 · late \ + rounds 36 || big early 1250.00 · big late 40.50 · other 500.00 · host tail 12.00 \ + (ms) || lean program on · xcheck on || fused 21 (declined 2) · 800.00 ms" + ); + let global = ProverSplit { + index: GLOBAL_INDEX, + overlapped: true, + ..Default::default() + }; + assert_eq!( + global.zerocheck_line(), + "ARGUE ZEROCHECK GLOBAL (in base) ⛔OVERLAPPED: big sessions 0 (lean 0 · xchecked \ + 0) · early rounds 0 · late rounds 0 || big early 0.00 · big late 0.00 · other 0.00 · \ + host tail 0.00 (ms) || lean program off · xcheck off || fused 0 (declined 0) · 0.00 ms" + ); + } + + /// A region closed by `add_tick` lands in its slot in nanoseconds, and the + /// take reads it back in seconds and clears it. + #[test] + fn a_ticked_region_lands_in_its_slot() { + static PROBE: Slot = Slot::new(); + let started = Instant::now() + .checked_sub(std::time::Duration::from_millis(5)) + .expect("the clock is past its origin"); + add_tick(&PROBE, Some(started)); + let secs = PROBE.take(); + assert!( + (0.005..1.0).contains(&secs), + "five milliseconds ago, read {secs}" + ); + assert_eq!(PROBE.take(), 0.0, "the take clears the slot"); + } + + /// ★ The one piece of the GKR split on the proof's path: the tail's + /// transcript is wrapped to time its share. The wrapper must make the same + /// calls on the transcript it wraps — same state, same draws — or wrapping + /// it would change the proof it times. + #[test] + fn a_timed_transcript_draws_what_it_wraps() { + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + type FE = FieldElement; + let mut bare = DefaultTranscript::::new(b"timed"); + let mut held = DefaultTranscript::::new(b"timed"); + let drawn = |t: &mut dyn IsTranscript| { + let mut out = Vec::new(); + for k in 1..=5u64 { + t.append_field_element(&FE::from(k * 7)); + t.append_bytes(&k.to_le_bytes()); + t.mark_statement_end(); + out.push(t.sample_field_element()); + out.push(FE::from(t.sample_u64(1 << 20))); + } + out + }; + let from_bare = drawn(&mut bare); + let from_timed = drawn(&mut Timed::new(&mut held, &GKR_TAIL_TRANSCRIPT)); + assert_eq!( + from_bare, from_timed, + "the wrapper drew what the transcript draws" + ); + assert_eq!(bare.state(), held.state(), "and left it in the same state"); + assert_eq!( + GKR_TAIL_TRANSCRIPT.take(), + 0.0, + "no clock is read with the knob off" + ); + } + + /// The remainders are what catch a missing timer, so they must be the + /// arithmetic they claim and not a restatement of it. + #[test] + fn a_missing_stage_shows_up_in_the_remainder() { + let whole = ProducerSplit { + index: 0, + execute: 1.0, + collect: 2.0, + build: 3.0, + handoff: 4.0, + wall: 10.0, + }; + assert!( + whole.other().abs() < 1e-9, + "a complete partition leaves nothing over" + ); + + // The mutation the gate runs: one stage's timer omitted. The stage + // reads 0 and its time lands in the remainder, where the check sees it. + let missing = ProducerSplit { + build: 0.0, + ..whole.clone() + }; + assert!( + (missing.other() - 3.0).abs() < 1e-9, + "an omitted `build` must surface as 3.0s of remainder, got {}", + missing.other() + ); + + let p = ProverSplit { + index: 0, + prep: 1.0, + absorb: 0.5, + commit: 2.0, + prove: 6.5, + wall: 10.0, + challenge: 0.5, + argue: 4.0, + open_groups: 1.5, + open_prepared: 0.5, + ..Default::default() + }; + assert!(p.other().abs() < 1e-9); + assert!(p.prove_other().abs() < 1e-9); + assert!(!p.is_global()); + assert!( + ProverSplit { + index: GLOBAL_INDEX, + ..Default::default() + } + .is_global() + ); + } + + /// ⛔ The cross-epoch stage's index is `u64::MAX`. A real run printed + /// `BASE EPOCH 18446744073709551615` before this was fixed, so the + /// rendering is pinned rather than left to a reader to notice again. + #[test] + fn the_global_sentinel_is_rendered_by_name() { + assert_eq!(GLOBAL_INDEX, u64::MAX); + assert!( + !format!("{GLOBAL_INDEX}").contains("global"), + "the raw sentinel is what the line must NOT carry", + ); + assert!( + ProverSplit { + index: GLOBAL_INDEX, + ..Default::default() + } + .is_global() + ); + assert!( + !ProverSplit { + index: 0, + ..Default::default() + } + .is_global() + ); + } + + /// A realistic pair of records: the producer and the prover each close, and + /// the two threads overlap, so `Σ producer + Σ prover` is well above the + /// base wall and arm D's inequalities are the only true statements about it. + fn honest() -> (Vec, Vec, f64) { + let producer: Vec<_> = (0..3) + .map(|i| ProducerSplit { + index: i, + execute: 1.0, + collect: 0.5, + build: 1.5, + handoff: 2.0, + wall: 5.0, + }) + .collect(); + let prover: Vec<_> = (0..3) + .map(|i| ProverSplit { + index: i, + prep: 0.5, + absorb: 0.1, + commit: 1.4, + prove: 3.0, + wall: 5.0, + challenge: 0.1, + argue: 1.9, + open_groups: 0.9, + open_prepared: 0.1, + // The six partition `open_groups`: 0.3 + 0.2 + 0.1 + 0.1 + + // 0.05 + 0.05 = 0.8. + chain: [0.3, 0.2, 0.1, 0.1, 0.05, 0.05], + // ⛔ THE WALL MUST MODEL A CORRECT INSTRUMENT. The six sum to + // 0.80 and the loop's own bookkeeping reads ~0 on every real + // record, so the wall is 0.81 — 1.2% of remainder, inside the + // 3% bound. The first draft used 0.85 (5.9%) and the fixture + // itself tripped the bound it was written to test: a fixture + // that is not a correct instrument makes every arm meaningless. + // `open_groups` 0.90 leaves setup_tail 0.09, positive, which is + // where the un-slotted setup legitimately lives. + chain_round_wall: 0.81, + // The four partition the QUERIES slot (chain[5] = 0.05): + // 0.005 + 0.030 + 0.010 + 0.004 = 0.049, leaving 0.001 of + // glue — the same "~0 on a correct instrument" the round + // loop's own remainder reads. Same lesson as the round wall + // above: a fixture that is not a correct instrument makes + // every arm written against it meaningless. + queries: [0.005, 0.030, 0.010, 0.004], + // Two chains of six rounds: 2·12 − 2 = 22 `open_many` calls, + // which is arm F's identity satisfied rather than asserted. + // `groups` is carried too, and deliberately DIFFERENT from the + // chain count: a group can open several chains, and an identity + // written over groups would pass here by coincidence. + groups: 1, + chain_count: 2, + round_count: 12, + rebuild_calls: 22, + ..Default::default() + }) + .collect(); + // Two threads over three epochs: ~one epoch of preparation, then the + // proofs. Not 30s. + (producer, prover, 17.0) + } + + /// ⛔ THE STATE THE CHECK EXISTS TO ACCEPT. Run first, because an arm that + /// cannot pass is the failure mode that costs a box launch. + #[test] + fn closure_accepts_an_honest_run() { + let (producer, prover, base) = honest(); + assert_eq!(check_closure(&producer, &prover, base, 0.03), Ok(())); + } + + /// ★ AND THE STATES IT MUST REFUSE, one per arm, each a MANUFACTURED + /// omission of exactly the kind the box mutation will make. Each assertion + /// reads the arm's NAME out of the message: an arm that reddened for some + /// other reason would not be evidence that this arm works. + #[test] + fn closure_refuses_every_manufactured_omission() { + // Arm A: one prover stage's timer omitted. Its time becomes remainder. + let (producer, mut prover, base) = honest(); + prover[1].commit = 0.0; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm A:"), "expected arm A, got: {err}"); + assert!(err.contains("epoch 1"), "arm A must name the epoch: {err}"); + + // Arm A must name the GLOBAL stage as `global`, not as an epoch index. + let (producer, mut prover, base) = honest(); + prover.push(ProverSplit { + index: GLOBAL_INDEX, + prep: 0.1, + prove: 0.4, + wall: 2.0, + ..Default::default() + }); + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm A:"), "expected arm A, got: {err}"); + assert!( + err.contains("global"), + "arm A must name the global stage: {err}" + ); + + // Arm B: one INNER slot omitted. The four stages still close, so only + // arm B can catch it — which is why arm B exists. + let (producer, mut prover, base) = honest(); + prover[2].argue = 0.0; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm B:"), "expected arm B, got: {err}"); + assert!(err.contains("epoch 2"), "arm B must name the epoch: {err}"); + + // Arm C: one producer stage's timer omitted. + let (mut producer, prover, base) = honest(); + producer[0].handoff = 0.0; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm C:"), "expected arm C, got: {err}"); + assert!(err.contains("epoch 0"), "arm C must name the epoch: {err}"); + + // Arm E, failure 1: the six OVERLAP or escape the loop, so they sum to + // more than the loop's own wall. This is the shape both real defects + // took — a window secured at one edge, then two windows nesting around + // the same grind — and it is NEGATIVE, not drift. + let (producer, mut prover, base) = honest(); + prover[1].chain[0] = 0.60; // grind 0.30 -> 0.60: the six now exceed 0.85 + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm E:"), "expected arm E, got: {err}"); + assert!(err.contains("epoch 1"), "arm E must name the epoch: {err}"); + assert!( + err.contains("NEGATIVE"), + "arm E must say what a negative remainder means: {err}" + ); + assert!( + err.contains("grind 0.600"), + "arm E must print the six by NAME so the culprit is visible: {err}", + ); + + // Arm E, failure 2: the loop's wall escapes the opening that contains + // it. A different structural error, and only the second bound sees it. + let (producer, mut prover, base) = honest(); + prover[2].chain_round_wall = 1.50; // > open_groups 0.90 + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm E:"), "expected arm E, got: {err}"); + assert!(err.contains("epoch 2"), "{err}"); + assert!( + err.contains("reaches outside the opening"), + "the second bound must name its own failure: {err}", + ); + + // ⛔⛔ Arm E, failure 3: ONE OF THE SIX ZEROED, the round wall + // UNCHANGED — a slot whose timer is gone. This is the case the first + // version of the seventh slot could not see: Σ(six) shrinks, the + // remainder grows, and `>= 0` alone calls that fine. It is the whole + // reason `round_other` carries an upper bound. + let (producer, mut prover, base) = honest(); + prover[0].chain[0] = 0.0; // grind 0.30 gone; round_wall still 0.85 + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm E:"), "expected arm E, got: {err}"); + assert!(err.contains("epoch 0"), "{err}"); + assert!( + err.contains("SLOT THAT IS NOT BEING ADDED"), + "arm E must name the cause, not just the gap: {err}", + ); + assert!(err.contains("grind 0.000"), "and print the six: {err}"); + + // ⛔ AND THE CASE THAT MUST **NOT** REDDEN: a slot legitimately SMALL + // rather than missing. `queries` reads 0.00 on a card-free fixture + // because the codewords are tiny — a reading, not an error — and the + // round wall shrinks with it, so the remainder does not grow. + let (producer, mut prover, base) = honest(); + prover[0].chain[5] = 0.0; + prover[0].chain_round_wall = 0.76; // the wall loses it too + // ⛔ AND THE FOUR INSIDE IT GO WITH IT. This line was added when the + // query half of arm E reddened here on its first run: leaving the four + // at their honest values while zeroing the slot that CONTAINS them + // models four parts summing to more than their whole — an impossible + // instrument, and the very shape arm E exists to reject. The fixture + // was wrong, not the bound. Same lesson as `honest()`'s round wall. + prover[0].queries = [0.0; 4]; + assert_eq!( + check_closure(&producer, &prover, base, 0.03), + Ok(()), + "a slot that is genuinely small is a reading, not a missing timer", + ); + + // Arm D, first inequality: a thread claiming more than the pipeline. + let (producer, prover, _) = honest(); + let err = check_closure(&producer, &prover, 5.0, 0.03).unwrap_err(); + assert!(err.starts_with("arm D:"), "expected arm D, got: {err}"); + assert!(err.contains("busiest thread"), "{err}"); + + // Arm D, second: a base wall beyond the serial bound — time spent + // outside every stage. + let (producer, prover, _) = honest(); + let err = check_closure(&producer, &prover, 100.0, 0.03).unwrap_err(); + assert!(err.starts_with("arm D:"), "expected arm D, got: {err}"); + assert!(err.contains("serial bound"), "{err}"); + } + + /// ⛔ THE SECOND MUTATION THE GATE RUNS: the tolerance zeroed must redden a + /// run that is merely REALISTIC rather than exact. A real stage sum is + /// strictly below its wall — the wall's own clock reads bracket the stage + /// reads — so a check that still passed at tolerance zero would be reading + /// numbers that cannot have come from a measurement. + #[test] + fn a_zero_tolerance_refuses_a_realistic_run() { + let (mut producer, prover, base) = honest(); + // The instrument's own cost: a real wall is a hair longer than the sum + // of its parts, because the wall's clock reads bracket the stages'. + producer[0].wall += 0.002; + assert_eq!(check_closure(&producer, &prover, base, 0.03), Ok(())); + + // Arm C alone, with no prover records to reach first: the 2 ms of + // instrument cost is what a zeroed tolerance refuses. + let err = check_closure(&producer, &[], base, 0.0).unwrap_err(); + assert!( + err.starts_with("arm C:"), + "expected arm C at tol 0, got: {err}" + ); + assert!(err.contains("epoch 0"), "{err}"); + + // And on the WHOLE record set a zeroed tolerance still reddens — the + // arm that fires first is whichever record is walked first, which is + // why the isolated check above is the one that names arm C. + assert!( + check_closure(&producer, &prover, base, 0.0).is_err(), + "a zeroed tolerance must refuse a run carrying real timer cost", + ); + } + + /// ⛔ EVERY ONE OF THE FOUR, OMITTED IN TURN, IS SEEN — and named. + /// + /// The same discipline as the six: `check_closure` is a pure function over + /// records, so an omission can be FED to it rather than waited for. That + /// matters more here than it did for the six, because the shape the gate + /// runs card-free has every one of these durations at 0.00 and no bound on + /// a time could see anything there (see `QUERY_SLACK`). + #[test] + fn closure_refuses_every_omitted_query_slot() { + for (i, name) in QUERY_NAMES.iter().enumerate() { + let (producer, mut prover, base) = honest(); + let dropped = prover[1].queries[i]; + assert!( + dropped > 0.0, + "{name} must be nonzero in the fixture or \ + this case cannot fail" + ); + prover[1].queries[i] = 0.0; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!( + err.starts_with("arm E:"), + "expected arm E for {name}, got: {err}" + ); + assert!(err.contains("epoch 1"), "arm E must name the epoch: {err}"); + assert!( + err.contains("A SLOT THAT IS NOT BEING ADDED"), + "arm E must say what the gap means: {err}" + ); + assert!( + err.contains(&format!("{name} 0.0000")), + "arm E must print the four BY NAME so the culprit is visible: {err}" + ); + } + } + + /// ⛔ AND ITS TWIN, WHICH MUST NOT REDDEN: a genuinely tiny QUERIES with + /// the four tiny alongside it — the card-free fixture's own shape. + /// + /// Without this case the next lane meets a gate that reds on every + /// card-free run and widens the bound until it cannot fail at all. That is + /// the failure this pair exists to make impossible. + #[test] + fn closure_accepts_query_slots_that_are_small_because_the_work_was() { + let (producer, mut prover, base) = honest(); + for rec in prover.iter_mut() { + // QUERIES shrinks from 0.05 to 0.002, and the four shrink with it. + rec.chain[5] = 0.002; + rec.queries = [0.0002, 0.0012, 0.0004, 0.0001]; + // The six and the walls move together so the outer arms still hold. + rec.chain_round_wall = rec.chain.iter().sum::() + 0.001; + rec.open_groups = rec.chain_round_wall + 0.09; + rec.prove = rec.challenge + rec.argue + rec.open_groups + rec.open_prepared; + rec.wall = rec.prep + rec.absorb + rec.commit + rec.prove; + } + assert_eq!( + check_closure(&producer, &prover, base, 0.03), + Ok(()), + "a small QUERIES with small parts is an honest run, not a defect" + ); + } + + /// ⛔ A NEGATIVE remainder in the query half: the four cannot exceed the + /// slot that contains them, and it means nesting or a window reaching + /// outside `open_many` — the two defects arm E caught in the six. + #[test] + fn closure_refuses_query_slots_that_overrun_their_opening() { + let (producer, mut prover, base) = honest(); + // tree_rebuild alone made larger than the whole QUERIES slot. + prover[2].queries[1] = 0.09; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm E:"), "expected arm E, got: {err}"); + assert!(err.contains("epoch 2"), "arm E must name the epoch: {err}"); + assert!( + err.contains("NEGATIVE"), + "arm E must say what a negative remainder means: {err}" + ); + assert!( + err.contains("tree_rebuild 0.0900"), + "arm E must print the culprit by name: {err}" + ); + } + + /// ★ ARM F — the rebuild count against the rounds that produced it, and + /// the ONE arm that can fire where every duration reads 0.00. + #[test] + fn closure_refuses_a_rebuild_call_that_is_not_counted() { + let (producer, mut prover, base) = honest(); + prover[0].rebuild_calls -= 1; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm F:"), "expected arm F, got: {err}"); + assert!(err.contains("epoch 0"), "arm F must name the epoch: {err}"); + assert!( + err.contains("21") && err.contains("22"), + "arm F must print BOTH the counted and the derived number: {err}" + ); + assert!( + err.contains("2R − 1 per chain"), + "arm F must state the identity it is checking: {err}" + ); + } + + /// And arm F fires on the other side too — a chain that stopped opening + /// its successor would make FEWER calls per round, not more. + #[test] + fn closure_refuses_a_round_that_stopped_opening_its_successor() { + let (producer, mut prover, base) = honest(); + // 12 rounds over 2 groups that made only one call per round. + prover[1].rebuild_calls = 12; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm F:"), "expected arm F, got: {err}"); + assert!( + err.contains("opens the current commitment and its successor"), + "arm F must name the structure it assumes: {err}" + ); + } + + /// ★ AND THE OTHER OMISSION: the ROUND counter dropped while the calls + /// are still counted. Guarding arm F on `round_count > 0` alone would skip + /// this record entirely and the missing counter would be invisible — the + /// same blind spot the seventh slot opened in arm E, in a new place. + #[test] + fn closure_refuses_a_round_that_was_not_counted() { + let (producer, mut prover, base) = honest(); + prover[2].round_count = 0; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm F:"), "expected arm F, got: {err}"); + assert!(err.contains("epoch 2"), "arm F must name the epoch: {err}"); + assert!( + err.contains("22") && err.contains("0 round(s)"), + "arm F must print both counted numbers: {err}" + ); + } + + /// ★ AND THE THIRD OMISSION: the CHAIN counter dropped. The identity is + /// written over all three numbers, so each of them going missing is a + /// different wrong answer and each must be seen. + #[test] + fn closure_refuses_a_chain_that_was_not_counted() { + let (producer, mut prover, base) = honest(); + prover[1].chain_count = 0; + let err = check_closure(&producer, &prover, base, 0.03).unwrap_err(); + assert!(err.starts_with("arm F:"), "expected arm F, got: {err}"); + assert!( + err.contains("0 chain(s)") && err.contains("24"), + "arm F must print the chains it counted and what they derive: {err}" + ); + } + + /// ⛔ AND ARM F MUST NOT FIRE ON A RECORD THAT RAN NO CHAIN. An epoch that + /// opened no groups has no rounds and no calls, and `2·0 − 0 = 0` would + /// hold anyway — but a record with `groups` set and no chain at all would + /// read a derived `-groups`, which is not a defect in the run. + #[test] + fn closure_accepts_a_record_that_ran_no_chain() { + let (producer, mut prover, base) = honest(); + for rec in prover.iter_mut() { + rec.chain_count = 0; + rec.round_count = 0; + rec.rebuild_calls = 0; + rec.chain = [0.0; 6]; + rec.queries = [0.0; 4]; + rec.chain_round_wall = 0.0; + rec.open_groups = 0.9; + } + assert_eq!( + check_closure(&producer, &prover, base, 0.03), + Ok(()), + "no chain is not a broken chain" + ); + } +} diff --git a/crypto/multilinear/src/zerocheck.rs b/crypto/multilinear/src/zerocheck.rs new file mode 100644 index 000000000..15736dd8f --- /dev/null +++ b/crypto/multilinear/src/zerocheck.rs @@ -0,0 +1,259 @@ +//! `f` vanishes on the whole hypercube, via `Σ_x eq(r, x)·f(x) = 0`. +//! +//! The multilinear replacement for a quotient argument. `eq` adds one degree. +//! The residual claim about `f` is returned, not decided. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{element::FieldElement, traits::IsField}; + +use crate::{ + Error, + eq::eq_mle, + poly::{EqScaled, SumcheckPolynomial}, + sumcheck::{self, SumcheckProof}, +}; + +/// A zerocheck proof: the sumcheck transcript for `eq(r, ·)·C`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ZeroCheckProof { + pub sumcheck: SumcheckProof, +} + +/// What the verifier is left holding. +/// +/// `eq(r, point)` is computable by the verifier alone, so the residual claim is +/// entirely about `C` — [`constraint_evaluation`](Self::constraint_evaluation) +/// is the value the commitment scheme must confirm. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ZeroCheckClaim { + /// The zerocheck challenge `r`. + pub r: Vec>, + /// The sumcheck challenge point. + pub point: Vec>, + /// Required value of `eq(r, point)·C(point)`. + pub expected_evaluation: FieldElement, + /// `eq(r, point)`, recomputed by the verifier. + pub eq_at_point: FieldElement, +} + +impl ZeroCheckClaim { + /// The value `C(point)` must take, isolated from the `eq` factor. + /// + /// `None` when `eq(r, point)` is zero, which leaves `C(point)` + /// unconstrained by this claim. It happens only if the sumcheck challenges + /// land exactly on a cube corner other than `r`. + pub fn constraint_evaluation(&self) -> Option> { + self.eq_at_point + .inv() + .ok() + .map(|inv| &self.expected_evaluation * inv) + } +} + +/// What proving leaves the caller holding. +pub struct ZeroCheckOutput { + pub proof: ZeroCheckProof, + /// The zerocheck challenge. + pub r: Vec>, + /// The sumcheck challenge point — where the residual claim about the + /// constraint lives, and therefore where the commitment scheme must open. + pub point: Vec>, +} + +/// Proves that `constraint` vanishes on `{0,1}^n`. +/// +/// Draws `r` from the transcript first, so the prover cannot choose the +/// constraint after seeing it. +pub fn prove(constraint: P, transcript: &mut T) -> Result, Error> +where + F: IsField + 'static, + T: IsTranscript, + P: SumcheckPolynomial + Sync, + FieldElement: Send + Sync, +{ + let num_vars = constraint.num_vars(); + let r: Vec> = (0..num_vars) + .map(|_| transcript.sample_field_element()) + .collect(); + + let combined = EqScaled::new(constraint, eq_mle(&r)?)?; + let (sumcheck, point) = sumcheck::prove(combined, transcript)?; + Ok(ZeroCheckOutput { + proof: ZeroCheckProof { sumcheck }, + r, + point, + }) +} + +/// Verifies a zerocheck, returning the residual claim about `C`. +/// +/// `constraint_degree` is the degree of `C`; `eq` adds one on top. +pub fn verify( + proof: &ZeroCheckProof, + num_vars: usize, + constraint_degree: usize, + transcript: &mut T, +) -> Result, Error> +where + F: IsField + 'static, + T: IsTranscript, +{ + let r: Vec> = (0..num_vars) + .map(|_| transcript.sample_field_element()) + .collect(); + + // The claimed sum is zero — that is the whole statement. + let claim = sumcheck::verify( + &proof.sumcheck, + FieldElement::::zero(), + num_vars, + constraint_degree + 1, + transcript, + )?; + + let eq_at_point = crate::eq::eq_eval(&r, &claim.point)?; + Ok(ZeroCheckClaim { + r, + point: claim.point, + expected_evaluation: claim.expected_evaluation, + eq_at_point, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::goldilocks::GoldilocksField as F; + + use crate::{ + mle::Mle, + virtual_poly::{Term, VirtualPolynomial}, + }; + + type FE = FieldElement; + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"zerocheck-test") + } + + fn mle(vals: &[u64]) -> Mle { + Mle::new(vals.iter().map(|v| FE::from(*v)).collect()).unwrap() + } + + /// `C = a·b − c`, with `c` set to the product so it vanishes everywhere. + /// This is the shape of a real AIR constraint: a relation among columns. + fn satisfied_constraint(n: usize) -> VirtualPolynomial { + let size = 1usize << n; + let a: Vec = (0..size as u64).map(|i| i * 3 + 1).collect(); + let b: Vec = (0..size as u64).map(|i| i * 5 + 2).collect(); + let c: Vec = a.iter().zip(&b).map(|(x, y)| x * y).collect(); + VirtualPolynomial::new( + vec![mle(&a), mle(&b), mle(&c)], + vec![ + Term::new(FE::one(), vec![0, 1]), + Term::new(-FE::one(), vec![2]), + ], + ) + .unwrap() + } + + #[test] + fn a_satisfied_constraint_verifies() { + let c = satisfied_constraint(4); + assert_eq!(c.sum_over_hypercube(), FE::zero()); + + let out = prove(c.clone(), &mut transcript()).unwrap(); + let (proof, r_prover) = (out.proof, out.r); + let claim = verify(&proof, 4, c.degree(), &mut transcript()).unwrap(); + + assert_eq!(claim.r, r_prover); + // The residual claim must be exactly what C evaluates to there. + assert_eq!( + claim.constraint_evaluation().unwrap(), + c.evaluate(&claim.point).unwrap() + ); + } + + #[test] + fn the_residual_claim_factors_as_eq_times_c() { + let c = satisfied_constraint(3); + let proof = prove(c.clone(), &mut transcript()).unwrap().proof; + let claim = verify(&proof, 3, c.degree(), &mut transcript()).unwrap(); + + let c_at_point = c.evaluate(&claim.point).unwrap(); + assert_eq!(claim.eq_at_point * c_at_point, claim.expected_evaluation); + } + + #[test] + fn a_constraint_violated_in_one_row_is_rejected() { + let size = 8usize; + let a: Vec = (0..size as u64).map(|i| i * 3 + 1).collect(); + let b: Vec = (0..size as u64).map(|i| i * 5 + 2).collect(); + let mut c: Vec = a.iter().zip(&b).map(|(x, y)| x * y).collect(); + c[5] += 1; // one bad row + + let broken = VirtualPolynomial::new( + vec![mle(&a), mle(&b), mle(&c)], + vec![ + Term::new(FE::one(), vec![0, 1]), + Term::new(-FE::one(), vec![2]), + ], + ) + .unwrap(); + assert_ne!(broken.sum_over_hypercube(), FE::zero()); + + // The prover runs the protocol honestly on a false statement. `g(0)` is + // derived from the claim, so nothing is rejected in the round — the lie + // surfaces in the residual, which stops describing the constraint. + let proof = prove(broken.clone(), &mut transcript()).unwrap().proof; + let claim = verify(&proof, 3, broken.degree(), &mut transcript()).unwrap(); + assert_ne!( + claim.constraint_evaluation(), + Some(broken.evaluate(&claim.point).unwrap()), + "a violated constraint produced a consistent claim" + ); + } + + #[test] + fn a_nonzero_polynomial_that_happens_to_sum_to_zero_is_still_rejected() { + // Σ C = 0 but C is not identically zero: exactly the case a plain + // sumcheck-for-zero would miss and eq(r, ·) is there to catch. + let c = VirtualPolynomial::new( + vec![Mle::new(vec![FE::from(7), -FE::from(7)]).unwrap()], + vec![Term::single(0)], + ) + .unwrap(); + assert_eq!(c.sum_over_hypercube(), FE::zero()); + + let proof = prove(c.clone(), &mut transcript()).unwrap().proof; + let result = verify(&proof, 1, c.degree(), &mut transcript()); + + // The residual claim must be inconsistent with the real polynomial: + // that is where a non-vanishing constraint gets caught. + let claim = result.unwrap(); + assert_ne!( + claim.constraint_evaluation(), + Some(c.evaluate(&claim.point).unwrap()), + "a non-vanishing constraint produced a consistent claim" + ); + } + + #[test] + fn degree_accounts_for_the_eq_factor() { + let c = satisfied_constraint(3); + let proof = prove(c.clone(), &mut transcript()).unwrap().proof; + // C has degree 2; with eq the round polynomials are degree 3, so each + // carries three evaluations — `g(1), g(2), g(3)`, with `g(0)` derived. + assert_eq!(c.degree(), 2); + assert_eq!(proof.sumcheck.rounds[0].evaluations.len(), 3); + } + + #[test] + fn verifying_with_the_wrong_degree_is_rejected() { + let c = satisfied_constraint(3); + let proof = prove(c.clone(), &mut transcript()).unwrap().proof; + let err = verify(&proof, 3, c.degree() + 1, &mut transcript()).unwrap_err(); + assert!(matches!(err, Error::RoundDegreeMismatch { round: 0, .. })); + } +} diff --git a/crypto/multilinear/tests/argue_device_columns.rs b/crypto/multilinear/tests/argue_device_columns.rs new file mode 100644 index 000000000..215d58923 --- /dev/null +++ b/crypto/multilinear/tests/argue_device_columns.rs @@ -0,0 +1,402 @@ +//! The claim reduce's column values on the card, against the host they replace +//! (`LAMBDA_VM_ARGUE_DEVICE_COLUMNS`). +//! +//! ```text +//! cargo test --release -p multilinear --features cuda,parallel --test argue_device_columns -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. The arms switch process-wide overrides and read process-wide +//! counters, so every test takes one lock as well. +//! +//! The reference is the host's own `Mle::evaluate_in` at heights below 2^16, +//! where it never asks the device — so what is compared is the card against the +//! code it replaces, not against itself. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::Error; +use multilinear::claim_reduce::{self, FactorSource, ReduceProof}; +use multilinear::gpu; +use multilinear::mle::Mle; + +type FE = FieldElement; +type EE = FieldElement; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Every override this file sets, put back when the test ends — a failed +/// assertion included, so one red test cannot leave the next one armed. +struct Restore { + _serial: std::sync::MutexGuard<'static, ()>, +} + +impl Drop for Restore { + fn drop(&mut self) { + gpu::force_argue_device_columns(None); + gpu::force_argue_xcheck(None); + gpu::force_column_value_fault(false); + } +} + +fn serial() -> Restore { + Restore { + _serial: SERIAL.lock().unwrap_or_else(|e| e.into_inner()), + } +} + +/// A column with no structure a kernel could accidentally satisfy. +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(seed.wrapping_mul(6364136223846793005)) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +fn point(num_vars: usize, seed: u64) -> Vec { + (0..num_vars as u64) + .map(|i| { + EE::new([ + FE::from(seed + 3 * i + 1), + FE::from(2 * i + 7), + FE::from(seed * seed + i), + ]) + }) + .collect() +} + +/// The batched evaluation of a short table — read where it lies in a store it +/// shares with a taller table, and uploaded — against the host, column by +/// column: every height the knob moves onto the card, from one variable up, +/// and widths up to the widest precompile's and past it. +/// +/// The two device arms read the same cells with the same kernels, so they +/// must agree bit for bit; the host agrees with them as field elements. +#[test] +fn the_card_values_short_columns_as_the_host_does() { + let _serial = serial(); + math_cuda::device::backend().expect("needs a GPU"); + let shapes = [ + (1usize, 3usize), + (2, 7), + (3, 1), + (5, 2000), + (8, 256), + (10, 64), + (12, 1480), + (13, 5), + (15, 1480), + ]; + for (vars, width) in shapes { + let rows = 1usize << vars; + assert!(rows < 1 << 16, "the host reference must stay on the host"); + // A taller table first, so the run under test starts past column 0 of + // a store whose columns are not all one height — as an epoch's are. + let lead: Vec> = (0..2).map(|k| column(vars + 1, 900 + k)).collect(); + let table: Vec> = (0..width) + .map(|k| column(vars, 31 * k as u64 + 7)) + .collect(); + let raw = |c: &Mle| c.evals().iter().map(|v| *v.value()).collect::>(); + let stored: Vec> = lead.iter().chain(&table).map(raw).collect(); + let stored_ref: Vec<&[u64]> = stored.iter().map(Vec::as_slice).collect(); + let store = math_cuda::columns::DeviceColumns::upload(&stored_ref) + .expect("the card takes the store"); + assert!(store.is_run(lead.len(), width), "the table is a run"); + + let at = point(vars, vars as u64 * 101 + width as u64); + let raw_point: Vec = at.iter().flat_map(|c| gpu::ext3_raw(c).unwrap()).collect(); + let resident = math_cuda::sumcheck::evaluate_many_base( + math_cuda::columns::Columns::Device { + store: &store, + first: lead.len(), + width, + }, + &raw_point, + ) + .unwrap_or_else(|e| panic!("2^{vars} × {width}, resident: {e:?}")); + let uploaded = math_cuda::sumcheck::evaluate_many_base( + math_cuda::columns::Columns::Host(&stored_ref[lead.len()..]), + &raw_point, + ) + .unwrap_or_else(|e| panic!("2^{vars} × {width}, uploaded: {e:?}")); + assert_eq!(resident.len(), width); + assert_eq!( + resident, uploaded, + "2^{vars} × {width}: the resident and uploaded evaluations differ" + ); + for (k, (column, value)) in table.iter().zip(&resident).enumerate() { + let host = column.evaluate_in(&at).expect("the host evaluation"); + assert_eq!( + gpu::ext3_from_raw::(value), + host, + "2^{vars} × {width}: column {k} differs from the host" + ); + } + } +} + +/// A table's reduction: its columns, each read once unshifted and every third +/// one read a step ahead too, and the honest claims about them. +struct Case { + columns: Vec>, + sources: Vec, + values: Vec, + alpha: Vec, +} + +fn case(num_vars: usize, width: usize) -> Case { + let columns: Vec> = (0..width) + .map(|k| column(num_vars, 17 * k as u64 + 3)) + .collect(); + let sources: Vec = (0..width) + .map(FactorSource::direct) + .chain((0..width).step_by(3).map(|c| FactorSource::shifted(c, 1))) + .collect(); + let alpha = point(num_vars, 5); + let values = sources + .iter() + .map(|s| { + claim_reduce::materialize(&columns, s) + .unwrap() + .evaluate_in(&alpha) + .unwrap() + }) + .collect(); + Case { + columns, + sources, + values, + alpha, + } +} + +/// What one prove of a reduction left, and what it moved. +struct Reduced { + proof: ReduceProof, + point: Vec, + transcript: DefaultTranscript, + /// Columns the batched device evaluation returned, and columns the host + /// walked, during this prove. + on_card: u64, + on_host: u64, +} + +fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"argue-device-columns") +} + +fn reduce( + c: &Case, + resident: Option<(&gpu::ResidentColumns, usize)>, + on: bool, +) -> Result { + gpu::force_argue_device_columns(Some(on)); + let (card, host) = (gpu::evaluate_calls(), gpu::host_evaluate_calls()); + let mut transcript = transcript(); + let (proof, point) = claim_reduce::prove( + &c.columns, + &c.sources, + &c.values, + &c.alpha, + resident, + &mut transcript, + )?; + Ok(Reduced { + proof, + point, + transcript, + on_card: gpu::evaluate_calls() - card, + on_host: gpu::host_evaluate_calls() - host, + }) +} + +fn verify(c: &Case, proof: &ReduceProof) -> Result<(), Error> { + claim_reduce::verify( + proof, + &c.sources, + &c.values, + &c.alpha, + c.columns.len(), + &mut transcript(), + ) + .map(|_| ()) +} + +/// Puts `c`'s columns on the card behind a table of another height, and says +/// where they start — the layout an epoch's store has. +fn resident_store(c: &Case) -> (gpu::ResidentColumns, usize) { + let lead: Vec> = (0..3) + .map(|k| column(c.columns[0].num_vars() + 2, 500 + k)) + .collect(); + let all: Vec<&Mle> = lead.iter().chain(&c.columns).collect(); + ( + gpu::upload_columns(&all).expect("the card takes the columns"), + lead.len(), + ) +} + +/// The first index at which two proofs' column values differ, or `None`. +fn first_difference(a: &Reduced, b: &Reduced) -> Option { + let (x, y) = (&a.proof.column_values, &b.proof.column_values); + if x.len() != y.len() { + return Some(x.len().min(y.len())); + } + (0..x.len()).find(|&k| x[k] != y[k]) +} + +/// A reduction's round messages, compared value by value: the field marker +/// derives no `PartialEq`, so the proof struct's own `==` is not there to use. +fn rounds(r: &Reduced) -> Vec<&[EE]> { + r.proof + .sumcheck + .rounds + .iter() + .map(|round| round.evaluations.as_slice()) + .collect() +} + +/// ★ The stage's identity: a resident short table reduced with the knob off +/// (its columns walked on the host) and on (evaluated on the card) gives the +/// same proof, the same point and the same transcript — and each arm took the +/// path it is named for, so the comparison is not of a path with itself. +#[test] +fn a_resident_short_table_reduces_to_the_same_proof_on_the_card() { + let _serial = serial(); + // 64 columns of 2^10 rows: 2^16 cells, the knob's threshold exactly, in + // a table no column of which the default would put on the card. + let c = case(10, 64); + let (store, first) = resident_store(&c); + let resident = Some((&store, first)); + + let mut off = reduce(&c, resident, false).expect("the knob off proves"); + let mut on = reduce(&c, resident, true).expect("the knob on proves"); + + assert_eq!( + (off.on_card, off.on_host), + (0, 64), + "knob off: the host must walk every column" + ); + assert_eq!( + (on.on_card, on.on_host), + (64, 0), + "knob on: the card must take the whole table" + ); + assert_eq!( + first_difference(&off, &on), + None, + "a column value moved onto the card changed" + ); + assert_eq!(rounds(&off), rounds(&on), "the rounds moved"); + assert_eq!(off.point, on.point, "the reduced point moved"); + assert_eq!( + off.transcript.state(), + on.transcript.state(), + "the transcripts parted" + ); + assert_eq!( + off.transcript.sample_field_element(), + on.transcript.sample_field_element(), + "the next challenge moved" + ); + // Informational: rkyv writes raw limbs, and a value has two raw forms below + // 2^32 − 1, so equal values need not be equal rkyv bytes. The serde form — + // the transcript's, and the canonical one — is what is gated above. + let raw = |r: &Reduced| rkyv::to_bytes::(&r.proof).unwrap(); + let rkyv_equal = raw(&off).as_slice() == raw(&on).as_slice(); + eprintln!("argue device columns: rkyv bytes equal across the arms: {rkyv_equal}"); + verify(&c, &off.proof).expect("the knob-off proof verifies"); + verify(&c, &on.proof).expect("the knob-on proof verifies"); +} + +/// Where the knob stops: a resident table one column short of the threshold +/// stays on the host, and a table that is not resident stays there however +/// wide — it would have to be uploaded, which is what the height pays for. +#[test] +fn the_knob_takes_only_resident_tables_of_enough_cells() { + let _serial = serial(); + let short = case(10, 63); + let (store, first) = resident_store(&short); + let below = reduce(&short, Some((&store, first)), true).expect("proves"); + assert_eq!( + (below.on_card, below.on_host), + (0, 63), + "63 × 2^10 is under 2^16 cells: the host walks it" + ); + + let wide = case(10, 96); + let away = reduce(&wide, None, true).expect("proves"); + assert_eq!( + (away.on_card, away.on_host), + (0, 96), + "a table the card does not hold stays on the host" + ); + verify(&short, &below.proof).expect("verifies"); + verify(&wide, &away.proof).expect("verifies"); +} + +/// ⛔ THE NEGATIVE CONTROLS: the checks above and the block's cross-check can +/// each see a card value that is wrong. +/// +/// The fault adds one to the first column's device value. The identity test's +/// comparison must name that column, the proof must fail verification, and +/// `LAMBDA_VM_ARGUE_XCHECK` must refuse to prove at all — while, without the +/// fault, the cross-check proves and says how many columns it compared. +#[test] +fn a_wrong_card_value_is_seen_by_every_check() { + let _serial = serial(); + let c = case(10, 64); + let (store, first) = resident_store(&c); + let resident = Some((&store, first)); + let off = reduce(&c, resident, false).expect("the knob off proves"); + + gpu::force_column_value_fault(true); + let faulted = reduce(&c, resident, true).expect("the faulted arm proves"); + assert_eq!(faulted.on_card, 64, "the fault is on the card's path"); + assert_eq!( + first_difference(&off, &faulted), + Some(0), + "the identity comparison must name the corrupted column" + ); + assert_eq!( + verify(&c, &faulted.proof), + Err(Error::ShiftedReadMismatch), + "a corrupted column value must fail verification" + ); + + gpu::force_argue_xcheck(Some(true)); + let checked = gpu::argue_xchecks(); + assert_eq!( + reduce(&c, resident, true).err(), + Some(Error::DeviceFailed { + stage: "column evaluation" + }), + "the cross-check must refuse a card value the host disagrees with" + ); + assert_eq!( + gpu::argue_xchecks(), + checked, + "a refused table is not counted" + ); + + gpu::force_column_value_fault(false); + let clean = reduce(&c, resident, true).expect("the cross-checked arm proves"); + assert_eq!( + gpu::argue_xchecks() - checked, + 64, + "the cross-check must count every column it compared" + ); + assert_eq!(first_difference(&off, &clean), None); + verify(&c, &clean.proof).expect("the cross-checked proof verifies"); +} diff --git a/crypto/multilinear/tests/argue_device_tables.rs b/crypto/multilinear/tests/argue_device_tables.rs new file mode 100644 index 000000000..d43c0f32d --- /dev/null +++ b/crypto/multilinear/tests/argue_device_tables.rs @@ -0,0 +1,614 @@ +//! The argument's challenge tables on the card, against the host they replace +//! (`LAMBDA_VM_ARGUE_DEVICE_TABLES`): the zerocheck's `eq` weights, and the +//! claim reduce's shift tables and batched columns. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda,parallel --test argue_device_tables -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. The arms switch process-wide overrides and read process-wide +//! counters, so every test takes one lock as well. +#![cfg(feature = "cuda")] + +use std::sync::Arc; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::Error; +use multilinear::batch::{Rule, Weight}; +use multilinear::claim_reduce::{self, FactorSource}; +use multilinear::constraint_argument::{self, ConstraintCore, FactorKind, TraceData}; +use multilinear::eq::{eq_eval, eq_evals, eq_mle, shift_evals}; +use multilinear::gpu::{self, TableFault}; +use multilinear::mle::Mle; +use multilinear::program::Builder; + +type FE = FieldElement; +type EE = FieldElement; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Every override this file sets, put back when the test ends — a failed +/// assertion included, so one red test cannot leave the next one armed. +struct Restore { + _serial: std::sync::MutexGuard<'static, ()>, +} + +impl Drop for Restore { + fn drop(&mut self) { + gpu::force_argue_device_tables(None); + gpu::force_argue_device_columns(None); + gpu::force_argue_xcheck(None); + gpu::force_table_fault(None); + } +} + +fn serial() -> Restore { + Restore { + _serial: SERIAL.lock().unwrap_or_else(|e| e.into_inner()), + } +} + +/// A column with no structure a kernel could accidentally satisfy. +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(seed.wrapping_mul(6364136223846793005)) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +fn point(num_vars: usize, seed: u64) -> Vec { + (0..num_vars as u64) + .map(|i| { + EE::new([ + FE::from(seed + 3 * i + 1), + FE::from(2 * i + 7), + FE::from(seed * seed + i), + ]) + }) + .collect() +} + +fn raw(values: &[EE]) -> Vec { + values + .iter() + .flat_map(|v| gpu::ext3_raw(v).unwrap()) + .collect() +} + +fn ext(cells: &[u64]) -> Vec { + cells + .chunks_exact(3) + .map(gpu::ext3_from_raw::) + .collect() +} + +/// The program a session carries when a test only reads its tables: the +/// rounds never run, but a session is built with one. +fn any_program() -> gpu::Lowered { + let mut b = Builder::::new(); + let v = b.var(0); + gpu::lower(&b.finish(v).unwrap()).unwrap() +} + +/// The claim reduce's tables built on the card against the host's, table by +/// table: the shift table at every offset — the first one no shift, so +/// `eq(alpha)` is built in place, or a shift, so it is built apart; offsets +/// past the cube, which wrap — and the batched column of every offset, one +/// member and many, from a run that starts past column 0 of a mixed store. +#[test] +fn the_card_builds_the_reduce_tables_the_host_builds() { + let _serial = serial(); + math_cuda::device::backend().expect("needs a GPU"); + let lowered = any_program(); + let shapes: [(usize, usize, &[usize]); 5] = [ + (1, 2, &[0, 1]), + (6, 3, &[0, 1, 2]), + (10, 16, &[1, 3]), + (12, 40, &[0, 1, 2, 7, (1 << 12) + 3]), + (15, 64, &[0, 1]), + ]; + for (vars, width, offsets) in shapes { + let rows = 1usize << vars; + let lead: Vec> = (0..2).map(|k| column(vars + 1, 700 + k)).collect(); + let table: Vec> = (0..width) + .map(|k| column(vars, 13 * k as u64 + 5)) + .collect(); + let stored: Vec> = lead + .iter() + .chain(&table) + .map(|c| c.evals().iter().map(|v| *v.value()).collect()) + .collect(); + let stored_ref: Vec<&[u64]> = stored.iter().map(Vec::as_slice).collect(); + let store = math_cuda::columns::DeviceColumns::upload(&stored_ref) + .expect("the card takes the store"); + let alpha = point(vars, 29 + vars as u64); + + // Offset `i` reads every third column from `i`, each with a weight. + let members: Vec> = (0..offsets.len()) + .map(|i| { + (i..width) + .step_by(3) + .map(|c| (c, EE::from(1000 + 17 * c as u64 + i as u64))) + .collect() + }) + .collect(); + let groups: Vec<(usize, Vec, Vec)> = offsets + .iter() + .zip(&members) + .map(|(offset, members)| { + let columns = members.iter().map(|(c, _)| *c as u64).collect(); + let weights = members + .iter() + .flat_map(|(_, w)| gpu::ext3_raw(w).unwrap()) + .collect(); + (*offset, columns, weights) + }) + .collect(); + let session = math_cuda::sumcheck::reduce_session( + &store, + lead.len(), + width, + &raw(&alpha), + &groups, + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .unwrap_or_else(|e| panic!("2^{vars} × {width}: {e:?}")); + + for (i, (offset, members)) in offsets.iter().zip(&members).enumerate() { + assert_eq!( + ext(&session.factor(2 * i).unwrap()), + shift_evals(&alpha, *offset), + "2^{vars} × {width}: the shift table for offset {offset}" + ); + let batched: Vec = (0..rows) + .map(|y| { + members + .iter() + .fold(EE::zero(), |acc, (c, w)| acc + &table[*c].evals()[y] * w) + }) + .collect(); + assert_eq!( + ext(&session.factor(2 * i + 1).unwrap()), + batched, + "2^{vars} × {width}: the batched column for offset {offset}" + ); + } + } +} + +/// The zerocheck's weights built on the card from their points, beside a +/// weight uploaded as a table and the trace's own factors, which the building +/// must leave as they were. +#[test] +fn the_card_builds_the_zerocheck_weights_the_host_builds() { + let _serial = serial(); + math_cuda::device::backend().expect("needs a GPU"); + let lowered = any_program(); + for vars in [1usize, 5, 12, 16] { + let rows = 1usize << vars; + let factors: Vec> = (0..3u64) + .map(|k| { + raw(&(0..rows as u64) + .map(|i| EE::from(i * 31 + k)) + .collect::>()) + }) + .collect(); + let factor_refs: Vec<&[u64]> = factors.iter().map(Vec::as_slice).collect(); + let device = math_cuda::sumcheck::DeviceFactors::upload(&factor_refs) + .expect("the card takes the factors"); + let (a, b) = (point(vars, 3), point(vars, 7)); + let table: Vec = (0..rows as u64).map(|i| EE::from(i * i + 9)).collect(); + let (raw_a, raw_b, raw_table) = (raw(&a), raw(&b), raw(&table)); + let session = device + .session_with( + &[ + math_cuda::sumcheck::Extra::Eq(&raw_a), + math_cuda::sumcheck::Extra::Table(&raw_table), + math_cuda::sumcheck::Extra::Eq(&raw_b), + ], + &lowered.nodes, + &lowered.consts, + lowered.num_slots, + lowered.root_slot, + ) + .unwrap_or_else(|e| panic!("2^{vars}: {e:?}")); + for (k, factor) in factors.iter().enumerate() { + assert_eq!( + &session.factor(k).unwrap(), + factor, + "2^{vars}: trace factor {k}" + ); + } + assert_eq!( + ext(&session.factor(3).unwrap()), + eq_evals(&a), + "2^{vars}: eq(a)" + ); + assert_eq!( + ext(&session.factor(4).unwrap()), + table, + "2^{vars}: the table" + ); + assert_eq!( + ext(&session.factor(5).unwrap()), + eq_evals(&b), + "2^{vars}: eq(b)" + ); + } +} + +// --------------------------------------------------------------------------- +// The argument: a zerocheck, two claims on shifted reads, and the reduction. +// --------------------------------------------------------------------------- + +/// Rows `2^VARS`. Six columns of it are 6·2^14 cells, past the device-columns +/// threshold, so every knob here has something to move. +const VARS: usize = 14; +const WIDTH: usize = 6; +/// Factor slots: the six columns, column 3 a row ahead, column 4 two rows +/// ahead, then the two weights. +const AHEAD_1: usize = WIDTH; +const AHEAD_2: usize = WIDTH + 1; +const EQ_R: usize = WIDTH + 2; +const EQ_ROW: usize = WIDTH + 3; + +/// `c2 = c0 · c1` row by row — the zerocheck's constraint — and the rest free. +fn trace_columns() -> Vec> { + let (c0, c1) = (column(VARS, 3), column(VARS, 5)); + let c2 = Mle::new( + c0.evals() + .iter() + .zip(c1.evals()) + .map(|(a, b)| a * b) + .collect(), + ) + .unwrap(); + let mut columns = vec![c0, c1, c2]; + columns.extend((3..WIDTH as u64).map(|k| column(VARS, 40 + k))); + columns +} + +fn kinds() -> Vec { + (0..WIDTH) + .map(FactorKind::direct) + .chain([FactorKind::shifted(3, 1), FactorKind::shifted(4, 2)]) + .collect() +} + +/// The zerocheck `eq(r)·(c0·c1 − c2)` and the claims `eq(row)·c3[+1]` and +/// `eq(row)·c4[+2]`: as programs, which the device runs, or as closures, +/// which it cannot — so the zerocheck is turned down and its weights are built +/// on the host after all. +fn rules(compiled: bool) -> Vec> { + if !compiled { + return vec![ + Rule::new(3, |v: &[EE]| v[EQ_R] * (v[0] * v[1] - v[2])), + Rule::new(2, |v: &[EE]| v[EQ_ROW] * v[AHEAD_1]), + Rule::new(2, |v: &[EE]| v[EQ_ROW] * v[AHEAD_2]), + ]; + } + let zerocheck = { + let mut b = Builder::::new(); + let (w, x, y, z) = (b.var(EQ_R), b.var(0), b.var(1), b.var(2)); + let product = b.mul(x, y); + let constraint = b.sub(product, z); + let root = b.mul(w, constraint); + b.finish(root).unwrap() + }; + let claim = |slot: usize| { + let mut b = Builder::::new(); + let (w, v) = (b.var(EQ_ROW), b.var(slot)); + let root = b.mul(w, v); + b.finish(root).unwrap() + }; + vec![ + Rule::compiled(3, zerocheck), + Rule::compiled(2, claim(AHEAD_1)), + Rule::compiled(2, claim(AHEAD_2)), + ] +} + +fn r_point() -> Vec { + point(VARS, 41) +} + +fn row_point() -> Vec { + point(VARS, 43) +} + +fn claims(columns: &[Mle]) -> Vec { + let row = row_point(); + let ahead = |column, offset| { + claim_reduce::materialize(columns, &FactorSource::shifted(column, offset)) + .unwrap() + .evaluate_in(&row) + .unwrap() + }; + vec![EE::zero(), ahead(3, 1), ahead(4, 2)] +} + +fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"argue-device-tables") +} + +/// One arm: how it ran, and what it left. +#[derive(Clone, Copy)] +struct Arm { + /// The epoch's columns and the table's factors on the card. + resident: bool, + /// `LAMBDA_VM_ARGUE_DEVICE_TABLES`: the weights go as points. + tables: bool, + /// `LAMBDA_VM_ARGUE_DEVICE_COLUMNS`. + columns: bool, + /// Programs, which a device runs, or closures. + compiled: bool, +} + +struct Argued { + core: ConstraintCore, + point: Vec, + state: [u8; 32], + next: EE, + /// Challenge tables and column values the card made during the arm. + tables_on_card: u64, + columns_on_card: u64, +} + +fn argue(columns: &[Mle], arm: Arm) -> Result { + gpu::force_argue_device_tables(Some(arm.tables)); + gpu::force_argue_device_columns(Some(arm.columns)); + let mut trace = TraceData::new(columns.to_vec(), kinds(), Vec::new())?; + if arm.resident { + // A taller table first, so the run starts past column 0 of a store of + // mixed heights, as an epoch's does. + let lead: Vec> = (0..2).map(|k| column(VARS + 1, 300 + k)).collect(); + let all: Vec<&Mle> = lead.iter().chain(columns).collect(); + let store = gpu::upload_columns(&all).expect("the card takes the columns"); + trace.set_resident(Arc::new(store), lead.len()); + trace + .reside_from_columns() + .expect("the card takes the factors"); + } + let (r, row) = (r_point(), row_point()); + let weights = if arm.tables { + vec![Weight::Eq(r), Weight::Eq(row)] + } else { + vec![Weight::Table(eq_mle(&r)?), Weight::Table(eq_mle(&row)?)] + }; + let (tables, cards) = (gpu::argue_tables_on_card(), gpu::evaluate_calls()); + let mut t = transcript(); + let (core, point) = constraint_argument::prove_core::( + &trace, + weights, + rules(arm.compiled), + &claims(columns), + &mut t, + )?; + Ok(Argued { + core, + point, + state: t.state(), + next: t.sample_field_element(), + tables_on_card: gpu::argue_tables_on_card() - tables, + columns_on_card: gpu::evaluate_calls() - cards, + }) +} + +/// Every value the core carries, in order, each list led by its length — the +/// canonical content of its bytes. The field marker derives no `PartialEq`, +/// so the proof structs' own `==` is not there to use. +fn canonical(core: &ConstraintCore) -> Vec { + let mut out = Vec::new(); + let mut list = |values: &[EE]| { + out.push(EE::from(values.len() as u64)); + out.extend_from_slice(values); + }; + for round in &core.sumcheck.rounds { + list(&round.evaluations); + } + list(&core.factor_values); + for round in &core.reduce.sumcheck.rounds { + list(&round.evaluations); + } + list(&core.reduce.column_values); + out +} + +/// The verifier's side, whole: the batch against the closed-form weights, the +/// reduction, and — the commitment's half, done here directly — the column +/// values at the reduced point. +fn verify(columns: &[Mle], core: &ConstraintCore) -> Result<(), Error> { + let (r, row) = (r_point(), row_point()); + let reduced = constraint_argument::verify_core( + core, + &kinds(), + WIDTH, + VARS, + &rules(true), + &claims(columns), + |at: &[EE]| Ok(vec![eq_eval(&r, at)?, eq_eval(&row, at)?]), + &mut transcript(), + )?; + for (k, (column, value)) in columns.iter().zip(&reduced.column_values).enumerate() { + if column.evaluate_in(&reduced.point)? != *value { + return Err(Error::ColumnOpeningRejected { column: k }); + } + } + Ok(()) +} + +const TODAY: Arm = Arm { + resident: true, + tables: false, + columns: false, + compiled: true, +}; + +/// ★ The stage's identity. The same argument with its tables built on the +/// card or on the host — alone, beside the columns on the card, with the +/// zerocheck turned down (closure rules), with nothing resident — is the same +/// proof, point and transcript as today's, and verifies. And each arm took the +/// path it is named for, so no comparison is of a path with itself. +#[test] +fn the_argument_proves_the_same_bytes_with_its_tables_on_the_card() { + let _serial = serial(); + let columns = trace_columns(); + let today = argue(&columns, TODAY).expect("today's arm proves"); + verify(&columns, &today.core).expect("today's arm verifies"); + assert_eq!( + (today.tables_on_card, today.columns_on_card), + (0, 0), + "today: nothing on the card but the rounds" + ); + + // (arm, tables on the card, column values on the card) + let arms = [ + // The zerocheck's two weights and the reduce's three offsets × 2. + ( + Arm { + tables: true, + ..TODAY + }, + 8, + 0, + ), + ( + Arm { + columns: true, + ..TODAY + }, + 0, + 6, + ), + ( + Arm { + tables: true, + columns: true, + ..TODAY + }, + 8, + 6, + ), + // Closures: the zerocheck is turned down, and its weights are built + // on the host; the reduce's program is its own, so its tables are not. + ( + Arm { + tables: true, + compiled: false, + ..TODAY + }, + 6, + 0, + ), + // Nothing resident: every table on the host, as it always was. + ( + Arm { + tables: true, + columns: true, + resident: false, + ..TODAY + }, + 0, + 0, + ), + ]; + for (arm, tables, cards) in arms { + let what = format!( + "resident {} · tables {} · columns {} · compiled {}", + arm.resident, arm.tables, arm.columns, arm.compiled + ); + let argued = argue(&columns, arm).unwrap_or_else(|e| panic!("{what}: {e:?}")); + assert_eq!( + (argued.tables_on_card, argued.columns_on_card), + (tables, cards), + "{what}: the path taken (tables, columns on the card)" + ); + assert_eq!( + canonical(&argued.core), + canonical(&today.core), + "{what}: the proof moved" + ); + assert_eq!(argued.point, today.point, "{what}: the reduced point moved"); + assert_eq!(argued.state, today.state, "{what}: the transcripts parted"); + assert_eq!(argued.next, today.next, "{what}: the next challenge moved"); + verify(&columns, &argued.core).unwrap_or_else(|e| panic!("{what}: {e:?}")); + } +} + +/// ⛔ THE NEGATIVE CONTROLS: a table the card built wrong is seen by every +/// check — the identity, the verifier, and `LAMBDA_VM_ARGUE_XCHECK`, which +/// refuses to prove — and, with nothing wrong, the cross-check proves the +/// same bytes and says it compared every table. +#[test] +fn a_wrong_table_on_the_card_is_seen_by_every_check() { + let _serial = serial(); + let columns = trace_columns(); + let today = argue(&columns, TODAY).expect("today's arm proves"); + let on_card = Arm { + tables: true, + ..TODAY + }; + + for (fault, rejected) in [ + (TableFault::Eq, Error::BatchMismatch), + (TableFault::Batched, Error::ShiftedReadMismatch), + ] { + gpu::force_table_fault(Some(fault)); + let faulted = argue(&columns, on_card).unwrap_or_else(|e| panic!("{fault:?}: {e:?}")); + assert_eq!( + faulted.tables_on_card, 8, + "{fault:?}: the fault is on the card's path" + ); + assert_ne!( + canonical(&faulted.core), + canonical(&today.core), + "{fault:?}: the identity must see the corrupted table" + ); + assert_eq!( + verify(&columns, &faulted.core), + Err(rejected), + "{fault:?}: the verifier must reject the proof over a corrupted table" + ); + + gpu::force_argue_xcheck(Some(true)); + let stage = match fault { + TableFault::Eq => "eq weight", + TableFault::Batched => "reduce tables", + }; + assert_eq!( + argue(&columns, on_card).err(), + Some(Error::DeviceFailed { stage }), + "{fault:?}: the cross-check must refuse a table the host disagrees with" + ); + gpu::force_argue_xcheck(None); + gpu::force_table_fault(None); + } + + gpu::force_argue_xcheck(Some(true)); + let checked = gpu::argue_table_xchecks(); + let clean = argue(&columns, on_card).expect("the cross-checked arm proves"); + assert_eq!( + gpu::argue_table_xchecks() - checked, + 8, + "the cross-check must count every table it compared" + ); + assert_eq!(canonical(&clean.core), canonical(&today.core)); + assert_eq!(clean.state, today.state); + verify(&columns, &clean.core).expect("the cross-checked arm verifies"); +} diff --git a/crypto/multilinear/tests/argue_gkr_gruen.rs b/crypto/multilinear/tests/argue_gkr_gruen.rs new file mode 100644 index 000000000..c536aec85 --- /dev/null +++ b/crypto/multilinear/tests/argue_gkr_gruen.rs @@ -0,0 +1,271 @@ +//! A device GKR layer's rounds with Gruen's split against today's, on the card +//! (`LAMBDA_VM_ARGUE_GKR_GRUEN`, D-ARGUE S1-3). +//! +//! ```text +//! cargo test --release -p multilinear --features cuda,parallel --test argue_gkr_gruen -- --test-threads=1 +//! ``` +//! +//! Needs a GPU: only a layer a device holds runs the Gruen rounds. The arms +//! switch process-wide overrides and read process-wide counters, so every test +//! takes one lock as well. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::Error; +use multilinear::gkr::{self, FractionLayer, FractionTree, GkrOutput, GkrProof}; +use multilinear::gkr_gruen as gruen; +use multilinear::mle::Mle; + +type FE = FieldElement; +type EE = FieldElement; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Every override this file sets, put back when the test ends — a failed +/// assertion included, so one red test cannot leave the next one armed. +struct Restore { + _serial: std::sync::MutexGuard<'static, ()>, +} + +impl Drop for Restore { + fn drop(&mut self) { + gruen::force_argue_gkr_gruen(None); + gruen::force_argue_gkr_gruen_xcheck(None); + gruen::force_gkr_gruen_tail(None); + gruen::force_gkr_gruen_fault(false); + gruen::force_gkr_gruen_direct_h0(false); + } +} + +fn serial() -> Restore { + Restore { + _serial: SERIAL.lock().unwrap_or_else(|e| e.into_inner()), + } +} + +/// A fraction layer with no structure a kernel could accidentally satisfy. +fn input(num_vars: usize, seed: u64) -> FractionLayer { + let value = |i: u64, salt: u64| { + let x = i + .wrapping_add(seed) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(salt.wrapping_mul(6364136223846793005)); + EE::new([FE::from(x >> 11), FE::from(x >> 7), FE::from(x >> 3)]) + }; + let size = 1u64 << num_vars; + let p = (0..size).map(|i| value(i, 1)).collect(); + let q = (0..size).map(|i| value(i, 2)).collect(); + FractionLayer::new(Mle::new(p).unwrap(), Mle::new(q).unwrap()).unwrap() +} + +fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"argue-gkr-gruen") +} + +/// One prove of a fresh tree: the proof, the transcript after it, how long it +/// took, and how many layers ran Gruen's rounds and were cross-checked. +struct Arm { + output: Result, Error>, + transcript: DefaultTranscript, + on_card: bool, + gruen: u64, + xchecked: u64, + secs: f64, +} + +fn prove(layer: &FractionLayer, on: bool, xcheck: bool, tail: usize) -> Arm { + gruen::force_argue_gkr_gruen(Some(on)); + gruen::force_argue_gkr_gruen_xcheck(Some(xcheck)); + gruen::force_gkr_gruen_tail(Some(tail)); + // A device tree spends its layers, so every arm builds its own. + let tree = FractionTree::build(layer.clone()).unwrap(); + let (gruen_before, xchecked_before) = (gruen::gruen_layers(), gruen::gruen_xchecked()); + let mut t = transcript(); + let start = std::time::Instant::now(); + let output = gkr::prove(&tree, &mut t); + let secs = start.elapsed().as_secs_f64(); + Arm { + output, + transcript: t, + on_card: tree.device().is_some(), + gruen: gruen::gruen_layers() - gruen_before, + xchecked: gruen::gruen_xchecked() - xchecked_before, + secs, + } +} + +/// A proof's layers, comparable field element by field element: every round's +/// values and the four each layer is closed with. +#[allow(clippy::type_complexity)] +fn layers(proof: &GkrProof) -> Vec<(Vec>, [EE; 4])> { + proof + .layers + .iter() + .map(|layer| { + ( + layer + .sumcheck + .rounds + .iter() + .map(|round| round.evaluations.clone()) + .collect(), + [layer.p_lo, layer.p_hi, layer.q_lo, layer.q_hi], + ) + }) + .collect() +} + +fn verify(layer: &FractionLayer, proof: &GkrProof) -> Result<(), Error> { + let output = FractionTree::build(layer.clone())?.output(); + gkr::verify(proof, output, &mut transcript()).map(|_| ()) +} + +/// The two arms' proofs, claims, transcripts and next challenges are equal. +fn assert_same_proof(today: &mut Arm, arm: &mut Arm, what: &str) { + let (a, b) = (today.output.as_ref().unwrap(), arm.output.as_ref().unwrap()); + assert_eq!( + layers(&a.proof), + layers(&b.proof), + "{what}: a layer's rounds or closing values moved" + ); + assert_eq!( + a.claim.point, b.claim.point, + "{what}: the claim's point moved" + ); + assert_eq!( + (&a.claim.p, &a.claim.q), + (&b.claim.p, &b.claim.q), + "{what}: the claim moved" + ); + assert_eq!( + today.transcript.state(), + arm.transcript.state(), + "{what}: the transcripts parted" + ); + assert_eq!( + today.transcript.sample_field_element(), + arm.transcript.sample_field_element(), + "{what}: the next challenge moved" + ); +} + +/// ★ The stage's identity: a tree on the card proved with today's layer rounds +/// and with Gruen's gives the same proof — every round value, every layer's +/// four closing values, the claim it leaves — and the same transcript after +/// it, at every tail cube and with `H(0)` summed on the card or taken from the +/// claim. The Gruen arm ran every device layer that way; today's none. +#[test] +fn the_card_layers_prove_todays_tree_with_gruens_rounds() { + let _restore = serial(); + for (vars, seed) in [(14usize, 3u64), (16, 5), (20, 9)] { + let layer = input(vars, seed); + for tail in [512usize, 64, 32, 2] { + for direct in [false, true] { + if direct && tail != 64 { + continue; + } + let what = format!("2^{vars}, tail {tail}, direct H(0) {direct}"); + let mut today = prove(&layer, false, false, tail); + gruen::force_gkr_gruen_direct_h0(direct); + let mut arm = prove(&layer, true, false, tail); + gruen::force_gkr_gruen_direct_h0(false); + assert!( + today.on_card && arm.on_card, + "{what}: the tree is built on the card" + ); + assert_eq!(today.gruen, 0, "{what}: today's arm ran no Gruen layer"); + assert!( + arm.gruen > 0, + "{what}: the Gruen arm ran its layers on the card" + ); + assert_same_proof(&mut today, &mut arm, &what); + verify(&layer, &arm.output.as_ref().unwrap().proof) + .unwrap_or_else(|e| panic!("{what}: {e:?}")); + eprintln!( + "argue gkr gruen: {what}: {} layers, the same proof · today {:.1} ms, gruen {:.1} ms", + arm.gruen, + today.secs * 1e3, + arm.secs * 1e3 + ); + } + } + } +} + +/// The cross-check compares every Gruen layer with today's rounds on the same +/// halves, and proves the same proof while it does. +#[test] +fn the_cross_check_compares_every_gruen_layer() { + let _restore = serial(); + let layer = input(18, 11); + let mut today = prove(&layer, false, false, 64); + let mut crossed = prove(&layer, true, true, 64); + assert!(crossed.gruen > 0, "there are Gruen layers to compare"); + assert_eq!( + crossed.xchecked, crossed.gruen, + "every Gruen layer was cross-checked" + ); + assert_same_proof(&mut today, &mut crossed, "cross-checked"); + eprintln!( + "argue gkr gruen: {} layers cross-checked, the same proof", + crossed.xchecked + ); +} + +/// ⛔ THE NEGATIVE CONTROLS: with the fault armed, every Gruen layer adds one +/// to its first round's `s(1)`. The proof must then differ from today's and +/// fail at GKR's own layer check, and the cross-check must refuse to prove at +/// all. +#[test] +fn a_wrong_gruen_round_is_seen_by_every_check() { + let _restore = serial(); + let layer = input(16, 7); + let today = prove(&layer, false, false, 64); + let today_out = today.output.unwrap(); + + gruen::force_gkr_gruen_fault(true); + let faulted = prove(&layer, true, false, 64); + let crossed = prove(&layer, true, true, 64); + gruen::force_gkr_gruen_fault(false); + + let faulted_out = faulted.output.unwrap(); + assert!(faulted.gruen > 0, "the fault is on the Gruen path"); + assert_ne!( + layers(&faulted_out.proof), + layers(&today_out.proof), + "a wrong Gruen round must change the proof" + ); + let verdict = verify(&layer, &faulted_out.proof); + assert!( + matches!(verdict, Err(Error::LayerRelationMismatch { .. })), + "GKR's layer check must refuse it, got {verdict:?}" + ); + assert_eq!( + crossed.output.err(), + Some(Error::DeviceFailed { + stage: "gkr gruen xcheck" + }), + "the cross-check must refuse to prove over a wrong Gruen round" + ); + eprintln!("argue gkr gruen: the wrong round was refused ({verdict:?}) and by the cross-check"); +} + +/// Where the knob stops: a tree under the device's threshold is proved here +/// from the start, and runs no Gruen layer. +#[test] +fn the_knob_takes_only_layers_a_device_holds() { + let _restore = serial(); + let layer = input(10, 5); + let off = prove(&layer, false, false, 64); + let on = prove(&layer, true, false, 64); + assert!(!on.on_card, "a 2^10 tree stays on the host"); + assert_eq!(on.gruen, 0, "no layer ran on a device"); + assert_eq!( + layers(&off.output.unwrap().proof), + layers(&on.output.unwrap().proof) + ); +} diff --git a/crypto/multilinear/tests/argue_gkr_input.rs b/crypto/multilinear/tests/argue_gkr_input.rs new file mode 100644 index 000000000..9777c1517 --- /dev/null +++ b/crypto/multilinear/tests/argue_gkr_input.rs @@ -0,0 +1,211 @@ +//! A table's GKR input layer written from its resident base columns against +//! today's, from the lifted factors, on the card (`LAMBDA_VM_ARGUE_GKR_INPUT`, +//! D-BATCH M1-2). +//! +//! ```text +//! cargo test --release -p multilinear --features cuda,parallel --test argue_gkr_input -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. The tests switch process-wide overrides, so each takes one lock. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::claim_reduce::FactorSource; +use multilinear::constraint_argument::FactorKind; +use multilinear::gkr::{self, FractionTree, GkrProof}; +use multilinear::gpu; +use multilinear::logup::{self, Affine, Interaction}; +use multilinear::mle::Mle; + +type FE = FieldElement; +type EE = FieldElement; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +struct Restore { + _serial: std::sync::MutexGuard<'static, ()>, +} + +impl Drop for Restore { + fn drop(&mut self) { + gpu::force_hand_back_for_tests(false); + } +} + +fn serial() -> Restore { + Restore { + _serial: SERIAL.lock().unwrap_or_else(|e| e.into_inner()), + } +} + +fn next(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed +} + +/// A table of `width` random columns over `2^num_vars` rows, its factors (every +/// column, then two shifted views), and `count` interactions of up to five +/// terms over them. +struct Table { + columns: Vec>, + kinds: Vec, + interactions: Vec>, +} + +fn table(num_vars: usize, width: usize, count: usize, seed: u64) -> Table { + let mut seed = seed | 1; + let rows = 1usize << num_vars; + let columns: Vec> = (0..width) + .map(|_| Mle::new((0..rows).map(|_| FE::from(next(&mut seed) >> 1)).collect()).unwrap()) + .collect(); + let mut kinds: Vec = (0..width).map(FactorKind::direct).collect(); + kinds.push(FactorKind::shifted(0, 1)); + kinds.push(FactorKind::shifted(width - 1, 5)); + let slots = kinds.len(); + let ext = |seed: &mut u64| { + EE::new([ + FE::from(next(seed)), + FE::from(next(seed)), + FE::from(next(seed)), + ]) + }; + let affine = |seed: &mut u64| { + let terms = (0..1 + (next(seed) % 5) as usize) + .map(|_| ((next(seed) as usize) % slots, ext(seed))) + .collect(); + Affine::new(terms, ext(seed)) + }; + let interactions = (0..count) + .map(|_| Interaction::new(affine(&mut seed), affine(&mut seed))) + .collect(); + Table { + columns, + kinds, + interactions, + } +} + +#[allow(clippy::type_complexity)] +fn layers(proof: &GkrProof) -> Vec<(Vec>, [EE; 4])> { + proof + .layers + .iter() + .map(|layer| { + ( + layer + .sumcheck + .rounds + .iter() + .map(|r| r.evaluations.clone()) + .collect(), + [layer.p_lo, layer.p_hi, layer.q_lo, layer.q_hi], + ) + }) + .collect() +} + +/// A proved tree: its output, its proof and the transcript after it. +type Proved = ((EE, EE), GkrProof, DefaultTranscript); + +/// Proves a tree. +fn prove(tree: &FractionTree) -> Proved { + let mut t = DefaultTranscript::::new(b"argue-gkr-input"); + let out = gkr::prove(tree, &mut t).expect("the tree proves"); + (tree.output(), out.proof, t) +} + +/// Today's tree and the columns' tree over the same table, both proved. +fn both(t: &Table) -> (Proved, Proved) { + let refs: Vec<&Mle> = t.columns.iter().collect(); + let store = std::sync::Arc::new(gpu::upload_columns(&refs).expect("the columns reside")); + let factors = + gpu::upload_factors_from_columns::(&t.columns, &t.kinds, &[], Some((&store, 0))) + .expect("the factors lift"); + let today = + logup::resident_tree(&t.interactions, std::sync::Arc::new(factors)).expect("today's tree"); + let rows = t.columns[0].len(); + let sources: Vec> = t.kinds.iter().map(FactorKind::source).collect(); + let plan = logup::input_plan(&t.interactions, &sources, rows).expect("a plan"); + let before = gpu::input_columns_trees(); + let device = gpu::input_layer_tree_from_columns(store, 0, t.columns.len(), rows, &plan) + .expect("the columns' tree"); + assert_eq!(gpu::input_columns_trees(), before + 1, "counted"); + let columns = FractionTree::from_device(device).unwrap(); + (prove(&today), prove(&columns)) +} + +fn assert_same(t: &Table, what: &str) { + let ((out_a, proof_a, mut ta), (out_b, proof_b, mut tb)) = both(t); + assert_eq!(out_a, out_b, "{what}: the bus output moved"); + assert_eq!(layers(&proof_a), layers(&proof_b), "{what}: a layer moved"); + assert_eq!(ta.state(), tb.state(), "{what}: the transcripts parted"); + assert_eq!( + ta.sample_field_element(), + tb.sample_field_element(), + "{what}: the next challenge moved" + ); + eprintln!( + "argue gkr input: {what}: {} layers, the same proof", + proof_b.layers.len() + ); +} + +/// ★ The stage's identity: the tree over the input layer written from the +/// columns proves the same output, rounds and transcript as today's, at +/// several shapes, padded and not, carried and handed back (its rewrite). +#[test] +fn the_columns_write_todays_tree() { + let _restore = serial(); + for (num_vars, width, count, seed) in [ + (12usize, 6usize, 3usize, 1u64), + (14, 9, 8, 2), + (16, 20, 13, 3), + (10, 40, 64, 4), + ] { + let t = table(num_vars, width, count, seed); + assert_same( + &t, + &format!("2^{num_vars} rows, {width} columns, {count} interactions, carried"), + ); + gpu::force_hand_back_for_tests(true); + assert_same( + &t, + &format!("2^{num_vars} rows, {width} columns, {count} interactions, handed back"), + ); + gpu::force_hand_back_for_tests(false); + } +} + +/// ⛔ The comparison can fail: one interaction's constant off by one on the +/// columns' side changes the output and the proof. +#[test] +fn a_wrong_plan_changes_the_tree() { + let _restore = serial(); + let t = table(12, 6, 3, 9); + let refs: Vec<&Mle> = t.columns.iter().collect(); + let store = std::sync::Arc::new(gpu::upload_columns(&refs).expect("the columns reside")); + let factors = + gpu::upload_factors_from_columns::(&t.columns, &t.kinds, &[], Some((&store, 0))) + .expect("the factors lift"); + let today = + logup::resident_tree(&t.interactions, std::sync::Arc::new(factors)).expect("today's tree"); + let rows = t.columns[0].len(); + let sources: Vec> = t.kinds.iter().map(FactorKind::source).collect(); + let mut plan = logup::input_plan(&t.interactions, &sources, rows).expect("a plan"); + plan.constants[3] = plan.constants[3].wrapping_add(1); + let wrong = FractionTree::from_device( + gpu::input_layer_tree_from_columns(store, 0, t.columns.len(), rows, &plan).unwrap(), + ) + .unwrap(); + assert_ne!( + today.output(), + wrong.output(), + "a wrong denominator constant must move the output" + ); +} diff --git a/crypto/multilinear/tests/argue_gkr_truncated.rs b/crypto/multilinear/tests/argue_gkr_truncated.rs new file mode 100644 index 000000000..7b0904569 --- /dev/null +++ b/crypto/multilinear/tests/argue_gkr_truncated.rs @@ -0,0 +1,420 @@ +//! Tables' GKR input layers kept on the card without their padding against +//! today's trees (`LAMBDA_VM_ARGUE_INPUT_REAL`, `multilinear::gkr_input_real`). +//! +//! ```text +//! cargo test --release -p multilinear --features cuda,parallel --test argue_gkr_truncated -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. Each arm builds its trees from the same resident columns under +//! one setting and proves them — one table at a time (`gkr::prove`) and several +//! in lockstep (`gkr_lockstep::prove_trees`, the batched argue's ladder) — and +//! the proofs' bytes, the claims they leave and the transcripts after them are +//! compared with today's. The tests switch process-wide overrides and read +//! process-wide counters, so each takes one lock. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::claim_reduce::FactorSource; +use multilinear::constraint_argument::FactorKind; +use multilinear::gkr::{self, FractionTree}; +use multilinear::gkr_input_real::{self as input_real, InputReal}; +use multilinear::gkr_lockstep::{self as lockstep, LadderTree}; +use multilinear::gpu; +use multilinear::logup::{self, Affine, Interaction}; +use multilinear::mle::Mle; + +type FE = FieldElement; +type EE = FieldElement; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Every override this file sets, put back when the test ends — a failed +/// assertion included. +struct Restore { + _serial: std::sync::MutexGuard<'static, ()>, +} + +impl Drop for Restore { + fn drop(&mut self) { + input_real::force_argue_input_real(None); + input_real::force_check_fault(false); + multilinear::gkr_gruen::force_argue_gkr_gruen(None); + gpu::force_hand_back_for_tests(false); + gpu::set_hand_back_threshold_for_tests(None); + } +} + +fn serial() -> Restore { + Restore { + _serial: SERIAL.lock().unwrap_or_else(|e| e.into_inner()), + } +} + +fn next(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed +} + +/// A table of `width` random columns over `2^num_vars` rows, and `count` +/// interactions of up to five terms over them (two shifted views too). +struct Table { + columns: Vec>, + kinds: Vec, + interactions: Vec>, +} + +impl Table { + fn rows(&self) -> usize { + self.columns[0].len() + } + + fn real(&self) -> usize { + self.interactions.len() * self.rows() + } + + fn full(&self) -> usize { + self.interactions.len().next_power_of_two() * self.rows() + } +} + +fn table(num_vars: usize, width: usize, count: usize, seed: u64) -> Table { + let mut seed = seed | 1; + let rows = 1usize << num_vars; + let columns: Vec> = (0..width) + .map(|_| Mle::new((0..rows).map(|_| FE::from(next(&mut seed) >> 1)).collect()).unwrap()) + .collect(); + let mut kinds: Vec = (0..width).map(FactorKind::direct).collect(); + kinds.push(FactorKind::shifted(0, 1)); + kinds.push(FactorKind::shifted(width - 1, 5)); + let slots = kinds.len(); + let ext = |seed: &mut u64| { + EE::new([ + FE::from(next(seed)), + FE::from(next(seed)), + FE::from(next(seed)), + ]) + }; + let affine = |seed: &mut u64| { + let terms = (0..1 + (next(seed) % 5) as usize) + .map(|_| ((next(seed) as usize) % slots, ext(seed))) + .collect(); + Affine::new(terms, ext(seed)) + }; + let interactions = (0..count) + .map(|_| Interaction::new(affine(&mut seed), affine(&mut seed))) + .collect(); + Table { + columns, + kinds, + interactions, + } +} + +fn resident(t: &Table) -> std::sync::Arc { + let refs: Vec<&Mle> = t.columns.iter().collect(); + std::sync::Arc::new(gpu::upload_columns(&refs).expect("the columns reside")) +} + +/// The table's tree over its resident columns, its input layer written from +/// them under whatever setting is in force. +fn tree_over(t: &Table, store: std::sync::Arc) -> FractionTree { + let sources: Vec> = t.kinds.iter().map(FactorKind::source).collect(); + let plan = logup::input_plan(&t.interactions, &sources, t.rows()).expect("a plan"); + let device = gpu::input_layer_tree_from_columns(store, 0, t.columns.len(), t.rows(), &plan) + .expect("the card builds the tree"); + FractionTree::from_device(device).unwrap() +} + +fn device_tree(t: &Table) -> FractionTree { + tree_over(t, resident(t)) +} + +fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"argue-gkr-truncated") +} + +/// One table proved alone: the proof's bytes, the claim's values, and the +/// transcript's state and next challenge. +type Alone = (Vec, Vec, [u8; 32], EE); + +fn prove_alone(t: &Table) -> Alone { + let tree = device_tree(t); + let mut tr = transcript(); + let out = gkr::prove(&tree, &mut tr).expect("the tree proves"); + let bytes = rkyv::to_bytes::(&out.proof) + .unwrap() + .to_vec(); + let mut claim = out.claim.point.clone(); + claim.extend([out.claim.p, out.claim.q]); + let state = tr.state(); + (bytes, claim, state, tr.sample_field_element()) +} + +/// Several tables proved in lockstep: the ladder's bytes, every input claim, +/// and the transcript's state and next challenge. +fn prove_ladder(tables: &[Table]) -> Alone { + let trees: Vec> = tables.iter().map(device_tree).collect(); + let ladder: Vec<&dyn LadderTree> = + trees.iter().map(|t| t as &dyn LadderTree).collect(); + let mut tr = transcript(); + let (proof, claims) = lockstep::prove_trees(&ladder, &mut tr).expect("the ladder proves"); + let bytes = rkyv::to_bytes::(&proof) + .unwrap() + .to_vec(); + let mut values = Vec::new(); + for claim in claims { + values.extend(claim.point); + values.extend([claim.p, claim.q]); + } + // The verifier takes it, from the outputs and heights alone. + let outputs: Vec<(EE, EE)> = trees.iter().map(|t| t.output()).collect(); + let heights: Vec = trees.iter().map(|t| t.input_vars()).collect(); + lockstep::verify(&proof, &outputs, &heights, &mut transcript()).expect("the ladder verifies"); + let state = tr.state(); + (bytes, values, state, tr.sample_field_element()) +} + +fn assert_same(today: &Alone, arm: &Alone, what: &str) { + assert_eq!(today.0, arm.0, "{what}: the proof's bytes moved"); + assert_eq!(today.1, arm.1, "{what}: a claim moved"); + assert_eq!(today.2, arm.2, "{what}: the transcripts parted"); + assert_eq!(today.3, arm.3, "{what}: the next challenge moved"); +} + +/// The tables the tests prove: padded at a quarter, three eighths, a CPU-like +/// twelve of 32 and a Keccak-like 31 of 64; and one with no padding. +fn tables() -> Vec { + vec![ + table(12, 6, 3, 1), + table(13, 9, 5, 2), + table(11, 20, 20, 3), + table(12, 10, 8, 4), + table(10, 40, 33, 5), + ] +} + +/// ★ Each table proved alone, today's trees against kept ones: the same proof +/// bytes, claim and transcript, with Gruen's rounds (the default) and with +/// today's layer rounds (`LAMBDA_VM_ARGUE_GKR_GRUEN=0`), which write a kept +/// layer whole again. Every padded table was kept, and only those. +#[test] +fn kept_layers_prove_todays_tables() { + let _restore = serial(); + for gruen in [true, false] { + multilinear::gkr_gruen::force_argue_gkr_gruen(Some(gruen)); + for (n, t) in tables().iter().enumerate() { + let what = format!( + "table {n} ({} of {} slots × {} rows), gruen {gruen}", + t.interactions.len(), + t.interactions.len().next_power_of_two(), + t.rows() + ); + let padded = t.real() < t.full(); + input_real::force_argue_input_real(Some(InputReal::Off)); + let before = input_real::counts(); + let today = prove_alone(t); + let off = input_real::counts(); + assert_eq!( + off.kept_trees, before.kept_trees, + "{what}: off keeps nothing" + ); + assert_eq!( + off.padded_trees - before.padded_trees, + u64::from(padded), + "{what}: counted padded" + ); + + input_real::force_argue_input_real(Some(InputReal::On)); + let kept = prove_alone(t); + let on = input_real::counts(); + assert_eq!( + on.kept_trees - off.kept_trees, + u64::from(padded), + "{what}: kept iff padded" + ); + assert_eq!( + on.cells_not_stored - off.cells_not_stored, + (t.full() - t.real()) as u64, + "{what}: the padding's cells not stored" + ); + assert_eq!(on.declines, off.declines, "{what}: nothing declined"); + assert_eq!( + on.rebuilds - off.rebuilds, + u64::from(padded && !gruen), + "{what}: today's layer rounds write the kept layer whole again, and only they" + ); + assert_same(&today, &kept, &what); + eprintln!( + "argue gkr truncated: {what}: the same proof ({} bytes)", + kept.0.len() + ); + } + } +} + +/// ★ The batched argue's ladder over every table at once, today's trees +/// against kept ones: the same ladder bytes, input claims and transcript, and +/// the verifier takes both. +#[test] +fn kept_layers_prove_todays_ladder() { + let _restore = serial(); + let tables = tables(); + let padded = tables.iter().filter(|t| t.real() < t.full()).count() as u64; + input_real::force_argue_input_real(Some(InputReal::Off)); + let today = prove_ladder(&tables); + input_real::force_argue_input_real(Some(InputReal::On)); + let before = input_real::counts(); + let kept = prove_ladder(&tables); + let after = input_real::counts(); + assert_eq!( + after.kept_trees - before.kept_trees, + padded, + "every padded tree kept" + ); + assert_eq!( + after.rebuilds, before.rebuilds, + "no kept layer written whole again" + ); + assert_same(&today, &kept, "the ladder"); + eprintln!( + "argue gkr truncated: ladder of {} trees ({padded} kept): the same proof ({} bytes)", + tables.len(), + kept.0.len() + ); +} + +/// `check` runs the whole layer beside every kept one and compares every +/// round and the halves, limb for limb: every kept layer compared, none +/// differing, and the ladder is today's. +#[test] +fn the_check_compares_every_kept_layer() { + let _restore = serial(); + let tables = tables(); + let padded = tables.iter().filter(|t| t.real() < t.full()).count() as u64; + input_real::force_argue_input_real(Some(InputReal::Off)); + let today = prove_ladder(&tables); + input_real::force_argue_input_real(Some(InputReal::Check)); + let before = input_real::counts(); + let checked = prove_ladder(&tables); + let after = input_real::counts(); + assert_eq!( + after.kept_trees - before.kept_trees, + padded, + "every padded tree kept" + ); + assert_eq!( + after.checked - before.checked, + padded, + "every kept layer compared" + ); + assert_eq!(after.differing, before.differing, "none differed"); + assert_eq!(after.check_skipped, before.check_skipped, "none skipped"); + assert_same(&today, &checked, "the checked ladder"); +} + +/// ⛔ The check can fail: with the fault armed each kept layer's first sums +/// are one limb off. Every kept layer is then counted as differing and +/// finished on the whole layer — and the ladder is still today's. +#[test] +fn a_kept_layer_that_differs_is_counted_and_finished_whole() { + let _restore = serial(); + let tables = tables(); + let padded = tables.iter().filter(|t| t.real() < t.full()).count() as u64; + input_real::force_argue_input_real(Some(InputReal::Off)); + let today = prove_ladder(&tables); + input_real::force_argue_input_real(Some(InputReal::Check)); + input_real::force_check_fault(true); + let before = input_real::counts(); + let faulted = prove_ladder(&tables); + input_real::force_check_fault(false); + let after = input_real::counts(); + assert_eq!( + after.checked - before.checked, + padded, + "every kept layer compared" + ); + assert_eq!( + after.differing - before.differing, + padded, + "every one seen to differ" + ); + assert_same(&today, &faulted, "the ladder finished on the whole layers"); +} + +/// The room a kept tree promises: its layer and the levels above it +/// (`math_cuda::gkr::truncated_peak_bytes`), against today's carry of four +/// times the whole layer — the promise the argue's high water is made of. +#[test] +fn a_kept_tree_promises_its_layer_and_the_levels_above() { + let _restore = serial(); + let be = math_cuda::device::backend().expect("a device"); + let t = table(14, 12, 20, 7); + let (real, full) = (t.real(), t.full()); + // The columns reside first, so the promise is the tree's alone. + let store = resident(&t); + let promised = |setting| { + input_real::force_argue_input_real(Some(setting)); + let before = be.reserved_bytes(); + let tree = tree_over(&t, store.clone()); + let held = be.reserved_bytes() - before; + drop(tree); + held + }; + let today = promised(InputReal::Off); + let kept = promised(InputReal::On); + assert_eq!(today, (4 * full) as u64 * 24, "today's carry"); + assert_eq!( + kept, + math_cuda::gkr::truncated_peak_bytes(real, full), + "the kept tree's promise" + ); + eprintln!( + "argue gkr truncated: promise today {} MiB, kept {} MiB ({} of {} slots)", + today >> 20, + kept >> 20, + t.interactions.len(), + t.interactions.len().next_power_of_two() + ); +} + +/// ⛔ When the card will not promise the kept tree it builds today's — here +/// the tree that hands its layer back, which promises less — counts the +/// decline, and the proof is today's. +#[test] +fn a_kept_tree_the_card_declines_builds_todays() { + let _restore = serial(); + let be = math_cuda::device::backend().expect("a device"); + let t = table(14, 12, 20, 8); + let (real, full) = (t.real(), t.full()); + input_real::force_argue_input_real(Some(InputReal::Off)); + let today = prove_alone(&t); + + // Room for the handed-back tree and not for the kept one: they differ by + // `24·full` bytes. + gpu::set_hand_back_threshold_for_tests(Some(0)); + input_real::force_argue_input_real(Some(InputReal::On)); + let room = math_cuda::gkr::padded_peak_bytes(real, full) + (1 << 20); + assert!(room < math_cuda::gkr::truncated_peak_bytes(real, full)); + let store = resident(&t); + let before = input_real::counts(); + let free = be.vram_budget_bytes().saturating_sub(be.reserved_bytes()); + let held = math_cuda::device::reserve(free - room).expect("the rest of the budget"); + let tree = tree_over(&t, store); + drop(held); + let after = input_real::counts(); + assert_eq!(after.kept_trees, before.kept_trees, "not kept"); + assert_eq!(after.declines - before.declines, 1, "the decline counted"); + let mut tr = transcript(); + let out = gkr::prove(&tree, &mut tr).expect("today's tree proves"); + let bytes = rkyv::to_bytes::(&out.proof) + .unwrap() + .to_vec(); + assert_eq!(bytes, today.0, "the proof is today's"); + assert_eq!(tr.state(), today.2, "the transcript is today's"); +} diff --git a/crypto/multilinear/tests/argue_round_slot.rs b/crypto/multilinear/tests/argue_round_slot.rs new file mode 100644 index 000000000..ea124d1d3 --- /dev/null +++ b/crypto/multilinear/tests/argue_round_slot.rs @@ -0,0 +1,290 @@ +//! Tables' GKR trees proved with the rounds' sums through a slot against +//! today's (`LAMBDA_VM_ARGUE_ROUND_SLOT`, `multilinear::round_slot`), and a +//! bin's trees built without a wait each. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda,parallel --test argue_round_slot -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. Each arm builds its trees from the same resident columns and +//! proves them — one table at a time and several in lockstep (the batched +//! argue's ladder) — under one setting; the proofs' bytes, the claims and the +//! transcripts after them are compared with today's. The tests switch +//! process-wide overrides and read process-wide counters, so each takes one +//! lock. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::claim_reduce::FactorSource; +use multilinear::constraint_argument::FactorKind; +use multilinear::gkr::{self, FractionTree}; +use multilinear::gkr_lockstep::{self as lockstep, LadderTree}; +use multilinear::gpu; +use multilinear::logup::{self, Affine, Interaction}; +use multilinear::mle::Mle; +use multilinear::round_slot::{self as slot, RoundSlotSetting}; + +type FE = FieldElement; +type EE = FieldElement; + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +struct Restore { + _serial: std::sync::MutexGuard<'static, ()>, +} + +impl Drop for Restore { + fn drop(&mut self) { + slot::force_argue_round_slot(None); + multilinear::gkr_gruen::force_argue_gkr_gruen(None); + gpu::set_hand_back_threshold_for_tests(None); + } +} + +fn serial() -> Restore { + Restore { + _serial: SERIAL.lock().unwrap_or_else(|e| e.into_inner()), + } +} + +fn next(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed +} + +struct Table { + columns: Vec>, + kinds: Vec, + interactions: Vec>, +} + +impl Table { + fn rows(&self) -> usize { + self.columns[0].len() + } +} + +fn table(num_vars: usize, width: usize, count: usize, seed: u64) -> Table { + let mut seed = seed | 1; + let rows = 1usize << num_vars; + let columns: Vec> = (0..width) + .map(|_| Mle::new((0..rows).map(|_| FE::from(next(&mut seed) >> 1)).collect()).unwrap()) + .collect(); + let mut kinds: Vec = (0..width).map(FactorKind::direct).collect(); + kinds.push(FactorKind::shifted(0, 1)); + let slots = kinds.len(); + let ext = |seed: &mut u64| { + EE::new([ + FE::from(next(seed)), + FE::from(next(seed)), + FE::from(next(seed)), + ]) + }; + let affine = |seed: &mut u64| { + let terms = (0..1 + (next(seed) % 4) as usize) + .map(|_| ((next(seed) as usize) % slots, ext(seed))) + .collect(); + Affine::new(terms, ext(seed)) + }; + let interactions = (0..count) + .map(|_| Interaction::new(affine(&mut seed), affine(&mut seed))) + .collect(); + Table { + columns, + kinds, + interactions, + } +} + +fn resident(t: &Table) -> std::sync::Arc { + let refs: Vec<&Mle> = t.columns.iter().collect(); + std::sync::Arc::new(gpu::upload_columns(&refs).expect("the columns reside")) +} + +fn plan(t: &Table) -> logup::InputPlan { + let sources: Vec> = t.kinds.iter().map(FactorKind::source).collect(); + logup::input_plan(&t.interactions, &sources, t.rows()).expect("a plan") +} + +fn device_tree(t: &Table) -> FractionTree { + let device = + gpu::input_layer_tree_from_columns(resident(t), 0, t.columns.len(), t.rows(), &plan(t)) + .expect("the card builds the tree"); + FractionTree::from_device(device).unwrap() +} + +fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"argue-round-slot") +} + +/// A proof's bytes, its claims, and the transcript's state and next challenge. +type Proved = (Vec, Vec, [u8; 32], EE); + +fn prove_alone(t: &Table) -> Proved { + let tree = device_tree(t); + let mut tr = transcript(); + let out = gkr::prove(&tree, &mut tr).expect("the tree proves"); + let bytes = rkyv::to_bytes::(&out.proof) + .unwrap() + .to_vec(); + let mut claim = out.claim.point.clone(); + claim.extend([out.claim.p, out.claim.q]); + let state = tr.state(); + (bytes, claim, state, tr.sample_field_element()) +} + +fn prove_ladder(tables: &[Table]) -> Proved { + let trees: Vec> = tables.iter().map(device_tree).collect(); + let ladder: Vec<&dyn LadderTree> = + trees.iter().map(|t| t as &dyn LadderTree).collect(); + let mut tr = transcript(); + let (proof, claims) = lockstep::prove_trees(&ladder, &mut tr).expect("the ladder proves"); + let bytes = rkyv::to_bytes::(&proof) + .unwrap() + .to_vec(); + let mut values = Vec::new(); + for claim in claims { + values.extend(claim.point); + values.extend([claim.p, claim.q]); + } + let outputs: Vec<(EE, EE)> = trees.iter().map(|t| t.output()).collect(); + let heights: Vec = trees.iter().map(|t| t.input_vars()).collect(); + lockstep::verify(&proof, &outputs, &heights, &mut transcript()).expect("the ladder verifies"); + let state = tr.state(); + (bytes, values, state, tr.sample_field_element()) +} + +fn assert_same(today: &Proved, arm: &Proved, what: &str) { + assert_eq!(today.0, arm.0, "{what}: the proof's bytes moved"); + assert_eq!(today.1, arm.1, "{what}: a claim moved"); + assert_eq!(today.2, arm.2, "{what}: the transcripts parted"); + assert_eq!(today.3, arm.3, "{what}: the next challenge moved"); +} + +fn tables() -> Vec
{ + vec![ + table(12, 6, 3, 1), + table(13, 9, 5, 2), + table(11, 20, 20, 3), + table(12, 10, 8, 4), + ] +} + +/// ★ Proofs with the rounds' sums through a slot are today's, byte for byte — +/// each table alone and every table in lockstep — and under `check` every +/// round is compared and none differs. Under `on` the Gruen layers launch no +/// second launch. +#[test] +fn the_slot_proves_todays_bytes() { + let _restore = serial(); + let tables = tables(); + slot::force_argue_round_slot(Some(RoundSlotSetting::Off)); + let ladder_today = prove_ladder(&tables); + let alone_today: Vec = tables.iter().map(prove_alone).collect(); + for setting in [RoundSlotSetting::On, RoundSlotSetting::Check] { + slot::force_argue_round_slot(Some(setting)); + let before = math_cuda::round_slot::counts(); + let ladder = prove_ladder(&tables); + let alone: Vec = tables.iter().map(prove_alone).collect(); + let after = math_cuda::round_slot::counts(); + assert_same(&ladder_today, &ladder, &format!("{setting:?} ladder")); + for (n, (a, b)) in alone_today.iter().zip(&alone).enumerate() { + assert_same(a, b, &format!("{setting:?} table {n}")); + } + let rounds = after.slot_rounds - before.slot_rounds; + assert!(rounds > 0, "{setting:?}: rounds went through slots"); + assert_eq!( + after.declines, before.declines, + "{setting:?}: no slot declined" + ); + match setting { + RoundSlotSetting::On => assert_eq!( + after.second_launches, before.second_launches, + "on: no second launch" + ), + _ => { + assert_eq!( + after.checked - before.checked, + rounds, + "check: every round compared" + ); + assert_eq!(after.mismatches, before.mismatches, "check: none differed"); + } + } + eprintln!("argue round slot: {setting:?}: {rounds} rounds through slots, today's bytes"); + } +} + +/// The same with today's generic layer rounds (`LAMBDA_VM_ARGUE_GKR_GRUEN=0`): +/// no Gruen layer, so no slot; the bytes are today's. +#[test] +fn without_gruens_rounds_the_slot_changes_nothing() { + let _restore = serial(); + multilinear::gkr_gruen::force_argue_gkr_gruen(Some(false)); + let tables = tables(); + slot::force_argue_round_slot(Some(RoundSlotSetting::Off)); + let today = prove_ladder(&tables); + slot::force_argue_round_slot(Some(RoundSlotSetting::On)); + let arm = prove_ladder(&tables); + assert_same(&today, &arm, "generic ladder"); +} + +/// A tree built with its output left unread is today's tree: the same output +/// once read, the same proof. +#[test] +fn a_deferred_tree_is_todays_tree() { + let _restore = serial(); + for t in tables() { + let eager = device_tree(&t); + let deferred = gpu::input_layer_tree_from_columns_deferred( + resident(&t), + 0, + t.columns.len(), + t.rows(), + &plan(&t), + ) + .expect("the deferred tree"); + let deferred = FractionTree::from_device(deferred).unwrap(); + assert_eq!(eager.output(), deferred.output(), "the output fraction"); + let (mut a, mut b) = (transcript(), transcript()); + let pa = gkr::prove(&eager, &mut a).unwrap(); + let pb = gkr::prove(&deferred, &mut b).unwrap(); + assert_eq!( + rkyv::to_bytes::(&pa.proof) + .unwrap() + .to_vec(), + rkyv::to_bytes::(&pb.proof) + .unwrap() + .to_vec(), + "the proof" + ); + assert_eq!(a.state(), b.state()); + } +} + +/// ⛔ A deferred tree the card will not promise falls to the host and is +/// counted as a GKR tree refusal — unlike a prefetch's, it is the consume +/// path. +#[test] +fn a_deferred_trees_refusal_is_counted() { + let _restore = serial(); + let be = math_cuda::device::backend().expect("a device"); + let t = table(14, 12, 20, 9); + let store = resident(&t); + let p = plan(&t); + gpu::set_hand_back_threshold_for_tests(Some(0)); + let before = gpu::gkr_tree_refusals(); + let free = be.vram_budget_bytes().saturating_sub(be.reserved_bytes()); + let held = math_cuda::device::reserve(free).expect("the rest of the budget"); + let refused = + gpu::input_layer_tree_from_columns_deferred(store, 0, t.columns.len(), t.rows(), &p); + drop(held); + assert!(refused.is_none(), "with no room the card refuses the tree"); + assert_eq!(gpu::gkr_tree_refusals() - before, 1, "the refusal counted"); +} diff --git a/crypto/multilinear/tests/commit_error_counter.rs b/crypto/multilinear/tests/commit_error_counter.rs new file mode 100644 index 000000000..baf5b9750 --- /dev/null +++ b/crypto/multilinear/tests/commit_error_counter.rs @@ -0,0 +1,97 @@ +//! ★ A device commit that FAILS is counted apart from one the device declined, +//! and the chain still commits — on the host, to the same root. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test commit_error_counter -- --nocapture +//! ``` +//! +//! Needs a GPU. The failure is a real one, not a stand-in: the whole VRAM +//! ledger is reserved first, so the commit's own reservation is refused and the +//! device commit returns `CUDA_ERROR_OUT_OF_MEMORY` through the same arm a +//! launch error takes. That arm used to turn the error into a silent `None`: +//! at WHIR stack 27 a tile launched past CUDA's grid limit, every commit fell +//! back to the host, and only `host_fallbacks()` moved +//! (`thoughts/zf/gap/fix/STRUCT.md` §9.2). Both device entry points are +//! driven — a polynomial uploaded for the commit (`parts`) and one already on +//! the card (`resident`) — and each should print one +//! `[whir] device commit (..) ... failed` line. +//! +//! Its own integration binary so the process-wide counters belong to it alone +//! (see `host_fallback_counter.rs`). +#![cfg(feature = "cuda")] + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu::{ + commit_calls, commit_errors, host_fallbacks, reset_call_counters, upload_columns, +}; +use multilinear::mle::Mle; +use multilinear::whir_chain::{ + ChainConfig, ChainFormat, GrindBits, Stacked, commit, commit_stacked, +}; +use multilinear::whir_hash::RpxWhir; + +fn poly(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FieldElement::from(i.wrapping_mul(6364136223846793005).wrapping_add(seed))) + .collect(), + ) + .expect("power of two") +} + +/// `(device commits, host fallbacks, commit errors)`. +fn counts() -> (u64, u64, u64) { + (commit_calls(), host_fallbacks(), commit_errors()) +} + +#[test] +fn a_failed_device_commit_is_counted_as_an_error_and_committed_on_the_host() { + math_cuda::device::backend().expect("this test needs a GPU"); + let config = ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 3, + grind: GrindBits::default(), + format: ChainFormat::DEFAULT, + }; + // 2^16 evaluations at blowup 4: a 2^18 codeword, above the device commit + // threshold, so the device is asked. + let num_vars = 16; + let f = poly(num_vars, 5); + let store = upload_columns(&[&f]).expect("the column upload"); + let resident = Stacked { + parts: vec![(&f, 0)], + resident: Some((&store, vec![(0, 0)])), + num_vars, + }; + reset_call_counters(); + + // Every byte the ledger has left, held: each commit's reservation is refused. + let mut held = Vec::new(); + while let Some(reservation) = math_cuda::device::reserve(1 << 20) { + held.push(reservation); + } + let (parts_on_host, _) = commit::(&f, &config, true).expect("commit"); + assert_eq!(counts(), (0, 1, 1), "the uploaded commit's error"); + let (resident_on_host, _) = + commit_stacked::(&resident, &config, true).expect("commit"); + assert_eq!(counts(), (0, 2, 2), "the resident commit's error"); + assert!(parts_on_host.codeword().device().is_none()); + assert!(resident_on_host.codeword().device().is_none()); + drop(held); + + // The same commit with the ledger free runs on the device, to the same root. + let (on_device, _) = commit_stacked::(&resident, &config, true).expect("commit"); + assert_eq!(counts(), (1, 2, 2), "a device commit is not an error"); + assert!( + on_device.codeword().device().is_some(), + "the control commit must run on the card" + ); + assert_eq!(parts_on_host.root(), on_device.root()); + assert_eq!(resident_on_host.root(), on_device.root()); + + // Below the threshold the device is not asked: a fallback, not an error. + let _ = commit::(&poly(8, 9), &config, true).expect("commit"); + assert_eq!(counts(), (1, 3, 2), "a decline is not an error"); +} diff --git a/crypto/multilinear/tests/encode_decline_counter.rs b/crypto/multilinear/tests/encode_decline_counter.rs new file mode 100644 index 000000000..e9bc77ad3 --- /dev/null +++ b/crypto/multilinear/tests/encode_decline_counter.rs @@ -0,0 +1,167 @@ +//! A successor encode the card declines is COUNTED (`gpu::encode_declines`), +//! and the fold the chain falls to proves the same bytes — for a chain of one +//! polynomial and for a grouped chain over a run of three +//! (`GroupFold::Indexed`). The encode switched off by its knob is the fold by +//! choice, not a decline, and counts nothing. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test encode_decline_counter -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. One `#[test]`: the arms switch process-wide overrides and read +//! process-wide counters, and a parallel test would move both. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{self, Claimed, StackedCommitment}; +use multilinear::stacking::StackedLayout; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, WhirFolds, +}; +use multilinear::whir_hash::{P1Whir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 20, + grind: GrindBits::default(), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + group: GroupFold::Indexed, + ..ChainFormat::DEFAULT + }, + } +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_add(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11)) + .collect(), + ) + .unwrap() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Arm { + Encoded, + Declined, + KnobOff, +} + +/// Proves `polys` polynomials of `2^16` (one chain: per polynomial for one, +/// grouped for three) under `arm`; returns the bytes and what the encode and +/// decline counters moved by. +fn prove(polys: usize, arm: Arm) -> (Vec, u64, u64) { + gpu::force_encode_successor(if arm == Arm::KnobOff { + Some(false) + } else { + None + }); + gpu::force_encode_decline(arm == Arm::Declined); + let heights = vec![16usize; polys]; + let columns: Vec> = (0..polys).map(|i| column(16, 5 + 17 * i as u64)).collect(); + let points: Vec> = (0..polys) + .map(|i| { + (0..16) + .map(|j| EE::from(301 + 7 * i as u64 + j as u64)) + .collect() + }) + .collect(); + let values: Vec = columns + .iter() + .zip(&points) + .map(|(c, z)| c.evaluate_in(z).unwrap()) + .collect(); + let layout = StackedLayout::build(&heights, 16).unwrap(); + assert_eq!(layout.num_polys(), polys); + let borrowed = multilinear::stacking::borrow(&columns); + let sponge = || { + let mut t = P1Whir::sponge(); + t.append_bytes(b"encode-decline"); + t + }; + let (encoded, declined) = (gpu::encoded_successors(), gpu::encode_declines()); + let stacked = + StackedCommitment::::commit(layout.clone(), &borrowed, None, &config()).unwrap(); + let proof = stacked_eval::prove::( + &stacked, + &borrowed, + None, + &Claimed::PerColumn(&points), + &values, + &config(), + &mut sponge(), + ) + .unwrap(); + let moved = ( + gpu::encoded_successors() - encoded, + gpu::encode_declines() - declined, + ); + stacked_eval::verify::( + &proof, + &layout, + &stacked.roots(), + &Claimed::PerColumn(&points), + &values, + stacked.domain(), + &config(), + &mut sponge(), + ) + .unwrap_or_else(|e| panic!("{polys} polynomials · {arm:?}: the verifier refused: {e:?}")); + ( + rkyv::to_bytes::(&proof) + .unwrap() + .to_vec(), + moved.0, + moved.1, + ) +} + +#[test] +fn an_encode_the_card_declines_is_counted_and_proves_the_same_bytes() { + for polys in [1usize, 3] { + let (bytes, encoded, declined) = prove(polys, Arm::Encoded); + assert_eq!( + (encoded, declined), + (1, 0), + "{polys}: the one chain's successor encoded" + ); + let (folded, encoded, declined) = prove(polys, Arm::Declined); + assert_eq!( + (encoded, declined), + (0, 1), + "{polys}: the declined encode counted, once, and nothing encoded" + ); + assert!( + folded == bytes, + "{polys}: the fold proves the encode's bytes" + ); + let (chosen, encoded, declined) = prove(polys, Arm::KnobOff); + assert_eq!( + (encoded, declined), + (0, 0), + "{polys}: the knob's fold is not a decline" + ); + assert!( + chosen == bytes, + "{polys}: the knob's fold proves the same bytes" + ); + println!( + "ENCODE DECLINE {polys} polynomial(s): counted once, the fold the encode's {} bytes", + bytes.len() + ); + } + gpu::force_encode_decline(false); + gpu::force_encode_successor(None); +} diff --git a/crypto/multilinear/tests/host_fallback_counter.rs b/crypto/multilinear/tests/host_fallback_counter.rs new file mode 100644 index 000000000..ad4248228 --- /dev/null +++ b/crypto/multilinear/tests/host_fallback_counter.rs @@ -0,0 +1,85 @@ +//! ★ Every commit is accounted for on exactly one side: it ran on the device, +//! or it fell back to the host. Neither is silent. +//! +//! No GPU needed, and that is the point — without a card every commit takes the +//! fallback arm, so the arm this file is about is the one that always runs here. +//! +//! ```text +//! cargo test -p multilinear --test host_fallback_counter +//! ``` +//! +//! # Why this exists +//! +//! A device commit that declines does not report anything. `commit_calls()` +//! simply does not rise, and a count merely lower than expected says nothing +//! when the expected count is itself derived from the table census. That +//! silence hid a real regression: keeping a Merkle tree per commitment filled +//! the card, every commit after the ceiling encoded on the host instead, and +//! the only visible symptoms were host memory and a GPU-utilisation figure +//! that read like a scheduling problem. The arm was taken thousands of times +//! and no line of output named it. +//! +//! Falling back is not a slower route to the same place. `from_codeword` +//! retains the host codeword AND its node array for the rest of the proof, so +//! a card that fills partway through an epoch converts into gigabytes of host +//! memory that never comes back. +//! +//! # Why this assertion and not `host_fallbacks() == 3` +//! +//! Because that one would be true on this laptop and false on the box, and a +//! test whose truth depends on which machine ran it is not pinning anything. +//! The SUM is the invariant on both: three commits, three accounted for, +//! wherever they ran. It fails if the fallback arm forgets to count — the sum +//! reads 0 on a GPU-less host and short on a card, which is exactly the bug. +//! +//! Its own integration binary so the process-wide counters belong to it alone. +//! Read `crypto/math-cuda/tests/whir_tree_cache.rs` for what a global counter +//! costs when several tests share a binary. + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::mle::Mle; +use multilinear::whir_chain::{ChainConfig, GrindBits, commit}; +use multilinear::whir_hash::KeccakWhir; + +fn poly(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FieldElement::from(i.wrapping_mul(6364136223846793005).wrapping_add(seed))) + .collect(), + ) + .expect("power of two") +} + +#[test] +fn every_commit_is_counted_on_exactly_one_side() { + let config = ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + }; + + multilinear::gpu::reset_call_counters(); + assert_eq!( + multilinear::gpu::commit_calls() + multilinear::gpu::host_fallbacks(), + 0, + "the reset must clear both counters, or the count below is someone else's" + ); + + const COMMITS: u64 = 3; + for seed in 0..COMMITS { + let f = poly(8, seed * 7 + 1); + let _ = commit::(&f, &config, false).expect("commit"); + } + + let on_device = multilinear::gpu::commit_calls(); + let on_host = multilinear::gpu::host_fallbacks(); + assert_eq!( + on_device + on_host, + COMMITS, + "{COMMITS} commits were made; {on_device} on the device and {on_host} \ + on the host — an arm is silent" + ); +} diff --git a/crypto/multilinear/tests/narrow_device.rs b/crypto/multilinear/tests/narrow_device.rs new file mode 100644 index 000000000..2cd2397fa --- /dev/null +++ b/crypto/multilinear/tests/narrow_device.rs @@ -0,0 +1,146 @@ +//! Narrow storage on the card: the column-major widen, the pack of a resident +//! run, and a store uploaded from packed tables, each against the host. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda,parallel --test narrow_device -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. +#![cfg(feature = "cuda")] + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu::{self, TableColumns}; +use multilinear::mle::Mle; +use multilinear::narrow::NarrowColumns; + +/// Every width boundary: 0, 2^8 − 1, 2^8, 2^16 − 1, 2^16, 2^32 − 1, 2^32, the +/// Goldilocks p − 1, a non-canonical word above p, and u64::MAX. +const EDGES: [u64; 10] = [ + 0, + 0xff, + 0x100, + 0xffff, + 0x1_0000, + 0xffff_ffff, + 0x1_0000_0000, + 0xffff_ffff_0000_0000, + 0xffff_ffff_0000_0001, + u64::MAX, +]; + +/// One column per edge over `rows` rows, column `c` topping out at `EDGES[c]` +/// (in its last row), shifted by `salt` so two tables differ. +fn edge_columns(rows: usize, salt: u64) -> Vec> { + EDGES + .iter() + .map(|&edge| { + (0..rows) + .map(|r| { + if r == rows - 1 { + edge + } else { + ((r as u64) ^ salt).wrapping_mul(0x9e37_79b9_7f4a_7c15) % (edge / 2 + 1) + } + }) + .collect() + }) + .collect() +} + +fn mles(columns: &[Vec]) -> Vec> { + columns + .iter() + .map(|c| Mle::new(c.iter().map(|&w| FieldElement::::from_raw(w)).collect()).unwrap()) + .collect() +} + +fn raw(columns: &[Vec]) -> Vec<&[u64]> { + columns.iter().map(Vec::as_slice).collect() +} + +/// The card's column-major widen gives back every word of a host pack, and the +/// row-major widen is the same words transposed. +#[test] +fn the_card_widens_column_major_at_every_width_boundary() { + let rows = (1usize << 12) + 1024; + let columns = edge_columns(rows, 0); + let packed = NarrowColumns::pack(&raw(&columns)).unwrap(); + assert_eq!(packed.widths(), &[1, 1, 2, 2, 4, 4, 8, 8, 8, 8]); + let cols = packed.cols(); + let col_major = math_cuda::narrow::widen_col_major_to_host( + packed.data(), + packed.offsets(), + packed.widths(), + rows, + cols, + ) + .unwrap(); + assert_eq!(col_major, columns.concat()); + let row_major = math_cuda::narrow::widen_to_host( + packed.data(), + packed.offsets(), + packed.widths(), + rows, + cols, + ) + .unwrap(); + for (c, column) in columns.iter().enumerate() { + let back: Vec = row_major.iter().skip(c).step_by(cols).copied().collect(); + assert_eq!(&back, column, "column {c}"); + } +} + +/// The card packs each table's run of a resident store into the bytes the +/// host packs, for two tables of different heights in one store. +#[test] +fn the_card_packs_a_resident_run_as_the_host_does() { + let tall = edge_columns(1 << 13, 1); + let short = edge_columns(1 << 9, 2); + let (tall_mle, short_mle) = (mles(&tall), mles(&short)); + let all: Vec<&Mle> = tall_mle.iter().chain(&short_mle).collect(); + let store = gpu::upload_columns(&all).expect("the card takes the columns"); + for (first, columns) in [(0usize, &tall), (tall.len(), &short)] { + let card = gpu::pack_resident(&store, first, columns.len()).expect("packed"); + assert_eq!( + card, + NarrowColumns::pack(&raw(columns)).unwrap(), + "from {first}" + ); + } + // Not a run: a span across the two heights. + assert!(gpu::pack_resident(&store, tall.len() - 1, 2).is_none()); +} + +/// A store uploaded from packed tables holds, column for column, the words a +/// store uploaded wide holds, packed and wide tables mixed; a wrong width map +/// widens to other words. +#[test] +fn a_store_uploaded_narrow_holds_the_wide_words() { + let a = edge_columns(1 << 11, 3); + let b = edge_columns(1 << 10, 4); + let c = edge_columns(1 << 12, 5); + let (a_mle, b_mle, c_mle) = (mles(&a), mles(&b), mles(&c)); + let (a_packed, c_packed) = ( + NarrowColumns::pack(&raw(&a)).unwrap(), + NarrowColumns::pack(&raw(&c)).unwrap(), + ); + let mixed = gpu::upload_tables::(&[ + TableColumns::Narrow(&a_packed), + TableColumns::Wide(&b_mle), + TableColumns::Narrow(&c_packed), + ]) + .expect("the card takes the tables"); + let all: Vec<&Mle> = a_mle.iter().chain(&b_mle).chain(&c_mle).collect(); + let wide = gpu::upload_columns(&all).expect("the card takes the columns"); + let held = mixed.download().unwrap(); + assert_eq!(held, wide.download().unwrap()); + assert_eq!(held, [a.clone(), b, c].concat()); + // The run checks the readers make hold for the widened tables too. + assert!(gpu::pack_resident(&mixed, 0, a.len()).is_some()); + + let mut wrong = a_packed.clone(); + assert!(wrong.fault_width_map()); + let broken = gpu::upload_tables::(&[TableColumns::Narrow(&wrong)]).unwrap(); + assert_ne!(broken.download().unwrap(), a); +} diff --git a/crypto/multilinear/tests/tree_error_counter.rs b/crypto/multilinear/tests/tree_error_counter.rs new file mode 100644 index 000000000..d509927bb --- /dev/null +++ b/crypto/multilinear/tests/tree_error_counter.rs @@ -0,0 +1,80 @@ +//! ★ A device Merkle tree over a HOST codeword that FAILS is counted and logged, +//! and the host builds the same tree. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test tree_error_counter -- --nocapture +//! ``` +//! +//! Needs a GPU. `commit_tree_ext3` is where a chain whose codeword lives on the +//! host (after a commit fell back) hashes its extension-field folds. It used to +//! turn a device error into a silent `None`, which no counter saw: it is +//! neither a device commit nor a commit fallback. The failure here is a real +//! one. That path allocates without the VRAM ledger, so the card itself is +//! filled first, and the tree's first allocation returns +//! `CUDA_ERROR_OUT_OF_MEMORY`. The run should print one +//! `[whir] device commit (ext3 tree) ... failed` line. +//! +//! Its own integration binary: the counters are process-wide, and a full card +//! would fail anything running beside it. +#![cfg(feature = "cuda")] + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; +use math::field::goldilocks::GoldilocksField as Gl; +use multilinear::gpu::{commit_calls, commit_errors, reset_call_counters, tree_commit_errors}; +use multilinear::whir_commit::CodewordCommitment; +use multilinear::whir_hash::RpxWhir; + +/// An extension-field codeword with every limb in use. +fn codeword(log_len: u32) -> Vec> { + (0..1u64 << log_len) + .map(|i| { + let limb = |k: u64| FieldElement::::from(i.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ k); + FieldElement::::new([limb(1), limb(2), limb(3)]) + }) + .collect() +} + +/// `(device commits, tree commit errors, commit errors)`. +fn counts() -> (u64, u64, u64) { + (commit_calls(), tree_commit_errors(), commit_errors()) +} + +#[test] +fn a_failed_device_tree_is_counted_as_an_error_and_built_on_the_host() { + let be = math_cuda::device::backend().expect("this test needs a GPU"); + const LOG_FOLDING: usize = 4; + // 2^18 values (6 MiB): above the device threshold, so the device is asked. + let values = codeword(18); + reset_call_counters(); + + // The whole card held, down to the last MiB the pool will hand out. + let stream = be.next_stream(); + let mut held = Vec::new(); + let mut chunk = 1usize << 30; + while chunk >= 1 << 20 { + match stream.alloc_zeros::(chunk) { + Ok(buffer) => held.push(buffer), + Err(_) => chunk /= 2, + } + } + let held_mib: usize = held.iter().map(|buffer| buffer.len() >> 20).sum(); + println!("tree_error_counter: {held_mib} MiB of device memory held"); + let on_host = CodewordCommitment::::new(&values, LOG_FOLDING).expect("commit"); + assert_eq!(counts(), (0, 1, 0), "the tree's error, and nothing else"); + drop(held); + stream.synchronize().expect("the frees"); + // Back to the driver, so the next allocation does not depend on the pool + // handing another stream what this one freed. + be.trim_mempool_to(0); + + // The same tree with the card free is built on the device, to the same root. + let on_device = CodewordCommitment::::new(&values, LOG_FOLDING).expect("commit"); + assert_eq!(counts(), (1, 1, 0), "a device tree is not an error"); + assert_eq!(on_host.root(), on_device.root()); + + // Below the threshold the device is not asked: nothing is counted. + let _ = + CodewordCommitment::::new(&values[..1 << 10], LOG_FOLDING).expect("commit"); + assert_eq!(counts(), (1, 1, 0), "a decline is not an error"); +} diff --git a/crypto/multilinear/tests/whir_cap_device.rs b/crypto/multilinear/tests/whir_cap_device.rs new file mode 100644 index 000000000..b6a30ef91 --- /dev/null +++ b/crypto/multilinear/tests/whir_cap_device.rs @@ -0,0 +1,126 @@ +//! W1 end to end on the device: a WHIR chain whose codeword stays on the card +//! (commit, folds, and every opening's tree rebuilt there) proves, under the +//! `Auto` cap, the SAME bytes as the chain over a host-held codeword, and the +//! host verifier accepts it. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_cap_device +//! ``` +//! +//! Needs a GPU. The device path is asserted TAKEN (the first commitment's +//! codeword is on the card), so a card that declined could not turn this into +//! a host-against-host comparison. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::mle::Mle; +use multilinear::whir::{Domain, encode, lift_coefficients}; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, GrindBits, RoundOpenings, commit, prove, verify, +}; +use multilinear::whir_commit::CodewordCommitment; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn run(cap: CapPolicy) { + // 2^16 evaluations at blowup 4: a 2^18 codeword, above the device commit + // threshold, so the chain's first tree lives on the card. + let num_vars = 16; + let cfg = ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 25, + grind: GrindBits::default(), + format: ChainFormat { + cap, + ..ChainFormat::DEFAULT + }, + }; + let f = Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(6364136223846793005).wrapping_add(17) >> 11)) + .collect(), + ) + .unwrap(); + let z: Vec = (0..num_vars).map(|i| EE::from(301 + i as u64)).collect(); + let y = f.evaluate_in(&z).unwrap(); + let tag = format!("{} cap={cap}", H::NAME); + + let (device, domain) = commit::(&f, &cfg, true).unwrap(); + assert!( + device.codeword().device().is_some(), + "{tag}: the commit must have stayed on the card, or this compares the host with itself" + ); + let device_proof = prove::( + &f, + &z, + &device, + &domain, + &cfg, + &mut DefaultTranscript::::new(b"whir-cap-device"), + ) + .unwrap(); + + let host_domain = Domain::::new(num_vars + cfg.log_blowup).unwrap(); + let host = CodewordCommitment::::from_codeword_on_host( + encode::(&lift_coefficients(&f), &host_domain).unwrap(), + cfg.schedule(num_vars)[0], + ) + .unwrap(); + assert_eq!(host.root(), device.root(), "{tag}: roots"); + let host_proof = prove::( + &f, + &z, + &host, + &host_domain, + &cfg, + &mut DefaultTranscript::::new(b"whir-cap-device"), + ) + .unwrap(); + + let a = rkyv::to_bytes::(&device_proof).unwrap(); + let b = rkyv::to_bytes::(&host_proof).unwrap(); + assert_eq!( + a.as_slice(), + b.as_slice(), + "{tag}: the device chain must prove the host chain's bytes" + ); + + // The owner path of tree 0 carries its cap. + let caps = cfg.tree_caps(num_vars); + let depth0 = num_vars + cfg.log_blowup - cfg.schedule(num_vars)[0]; + if let RoundOpenings::Base(p) = &device_proof.rounds[0].openings { + let extra = if caps[0] > 0 { 1usize << caps[0] } else { 0 }; + assert_eq!( + p.current[0].proof.merkle_path.len(), + depth0 - caps[0] + extra + ); + assert_eq!(p.current[1].proof.merkle_path.len(), depth0 - caps[0]); + } else { + panic!("{tag}: round 0 opens base blocks"); + } + + verify::( + &device_proof, + &device.root(), + &z, + y, + &domain, + &cfg, + &mut DefaultTranscript::::new(b"whir-cap-device"), + ) + .unwrap_or_else(|e| panic!("{tag}: the host verifier refused the device proof: {e:?}")); +} + +#[test] +fn the_device_chain_proves_the_host_chains_bytes_under_the_cap() { + for cap in [CapPolicy::Off, CapPolicy::Auto, CapPolicy::Fixed(5)] { + run::(cap); + run::(cap); + } +} diff --git a/crypto/multilinear/tests/whir_commit_ood_device.rs b/crypto/multilinear/tests/whir_commit_ood_device.rs new file mode 100644 index 000000000..551ee5aba --- /dev/null +++ b/crypto/multilinear/tests/whir_commit_ood_device.rs @@ -0,0 +1,314 @@ +//! A stacked opening carrying its commit-time out-of-domain answers +//! (`multilinear::commit_ood`) with its round 0 on the card proves the SAME +//! bytes as the host alone, and the answers the card evaluates from resident +//! columns are the host's. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_commit_ood_device -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. The card never writes an answer's `γ·eq(point, ·)` at the +//! stack's width: its share of each of the first rounds is added on the host +//! from the commit-time table, and what is left goes into the weight once the +//! table is spent or the first group of rounds ends +//! (`gpu::OpeningFactors::settle_overlay`). Cases: columns short enough that +//! the table covers the whole first fold (the production shape: written at +//! `2^(n − k0)`); columns tall enough that it runs out mid-fold (written +//! between two session rounds, and between a grouped part's rounds); and a +//! column filling the stack (no table: written at once, at full width). Each +//! one chain a polynomial and one a run (`GroupFold::PerPoly`, `Indexed`). +//! Arms: the host (the reference); the defaults; the share form; no lean +//! rounds; six; every successor folded; resident columns; and the host +//! fallback, the card's codewords with the factors declined. One `#[test]`: +//! the arms switch process-wide overrides, and parallel tests would race them. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{ + self, Claimed, ColumnsAt, CommitOod, OodClaims, StackedCommitment, +}; +use multilinear::stacking::StackedLayout; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, WhirFolds, +}; +use multilinear::whir_hash::{P1Whir, RpxWhir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(seed.wrapping_mul(6364136223846793005)) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +/// First6 then uniform4, caps on every tree, no grind (a ground proof's nonce +/// is not reproducible across two searches). +fn config(group: GroupFold) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 20, + grind: GrindBits::default(), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + group, + ..ChainFormat::DEFAULT + }, + } +} + +struct Case { + name: &'static str, + layout: StackedLayout, + columns: Vec>, + points: Vec>, + values: Vec, + z0: EE, +} + +fn case_of(name: &'static str, heights: &[usize], polys: usize) -> Case { + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 7 + 31 * i as u64)) + .collect(); + let layout = StackedLayout::build(heights, 16).unwrap(); + assert_eq!(layout.num_polys(), polys, "{name}: the stack's polynomials"); + let points: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| EE::from(101 + 13 * (i / 4) as u64 + j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, z)| c.evaluate_in(z).unwrap()) + .collect(); + Case { + name, + layout, + columns, + points, + values, + z0: EE::from(0x5eed_f00d_u64), + } +} + +fn cases() -> Vec { + vec![ + // 2^10 columns: the table spans the whole first fold. + case_of("short columns", &[10; 5 * 64], 5), + // 2^13 columns: a three-variable table, spent mid-fold. + case_of("tall columns", &[13; 5 * 8], 5), + // A column filling the stack beside short ones: polynomial 0 has no + // table at all. + case_of("a full-height column", &[16, 10, 10, 10, 10], 2), + ] +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Arm { + Host, + Defaults, + ShareForm, + Materialised, + Lean(usize), + Folded, + Fallback, +} + +fn force(arm: Arm) { + gpu::force_host_whir_commit(arm == Arm::Host); + gpu::force_shared_open_declined(matches!(arm, Arm::Host | Arm::Fallback)); + gpu::force_encode_successor(match arm { + Arm::Folded => Some(false), + _ => None, + }); + gpu::force_lean_tiles(match arm { + Arm::ShareForm => Some(gpu::LeanTiles::Off), + _ => None, + }); + gpu::force_lean_rounds(match arm { + Arm::Materialised => Some(0), + Arm::Lean(rounds) => Some(rounds), + _ => None, + }); + gpu::force_fused_fold(None); +} + +fn release() { + gpu::force_host_whir_commit(false); + gpu::force_shared_open_declined(false); + gpu::force_encode_successor(None); + gpu::force_lean_tiles(None); + gpu::force_lean_rounds(None); + gpu::force_fused_fold(None); +} + +fn sponge() -> H::Sponge { + let mut t = H::sponge(); + t.append_bytes(b"whir-commit-ood-device"); + t +} + +/// The case's claims, from the host's columns. +fn host_claims(c: &Case) -> OodClaims { + stacked_eval::ood_claims::( + &c.layout, + &multilinear::stacking::borrow(&c.columns), + None, + &c.z0, + ) + .unwrap() + .0 +} + +/// One arm: commits, proves with the case's answers, has the host verifier +/// accept, and returns the proof's bytes and the roots. +fn prove_arm( + c: &Case, + group: GroupFold, + arm: Arm, + resident: Option<&gpu::ResidentColumns>, +) -> (Vec, Vec<[u8; 32]>) { + force(arm); + let tag = format!("{} · {group:?} · {} · {arm:?}", c.name, H::NAME); + let columns = multilinear::stacking::borrow(&c.columns); + let claims = host_claims(c); + let opens = gpu::open_calls(); + let stacked = StackedCommitment::::commit( + c.layout.clone(), + &columns, + resident.map(|s| (s, 0)), + &config(group), + ) + .unwrap(); + let proof = stacked_eval::prove_mapped_ood::( + &stacked, + &columns, + resident.map(|s| (s, ColumnsAt::From(0))), + &Claimed::PerColumn(&c.points), + &c.values, + Some(&claims), + &config(group), + &mut sponge::(), + ) + .unwrap_or_else(|e| panic!("{tag}: the prover refused: {e:?}")); + let opened = gpu::open_calls() - opens; + match arm { + Arm::Host => assert_eq!(opened, 0, "{tag}: an opening on the card"), + // A declined opening's host-built tables may still go to the card. + Arm::Fallback => {} + _ => assert!(opened > 0, "{tag}: no opening on the card"), + } + stacked_eval::verify_ood::( + &proof, + &c.layout, + &stacked.roots(), + &Claimed::PerColumn(&c.points), + &c.values, + Some(CommitOod { + z0: &c.z0, + answers: &claims.answers, + }), + stacked.domain(), + &config(group), + &mut sponge::(), + ) + .unwrap_or_else(|e| panic!("{tag}: the host verifier refused: {e:?}")); + let bytes = rkyv::to_bytes::(&proof) + .unwrap() + .to_vec(); + (bytes, stacked.roots()) +} + +fn every_arm(c: &Case, group: GroupFold, store: &gpu::ResidentColumns) { + let (host, host_roots) = prove_arm::(c, group, Arm::Host, None); + for arm in [ + Arm::Defaults, + Arm::ShareForm, + Arm::Materialised, + Arm::Lean(6), + Arm::Folded, + Arm::Fallback, + ] { + let (card, roots) = prove_arm::(c, group, arm, None); + assert_eq!(roots, host_roots, "{} · {group:?} · {arm:?}: roots", c.name); + assert!( + card == host, + "{} · {group:?} · {} · {arm:?}: the card must prove the host's bytes", + c.name, + H::NAME + ); + } + for arm in [Arm::Defaults, Arm::Lean(6)] { + let (card, _) = prove_arm::(c, group, arm, Some(store)); + assert!( + card == host, + "{} · {group:?} · {} · resident {arm:?}: the card must prove the host's bytes", + c.name, + H::NAME + ); + } + println!( + "COMMIT OOD PARITY {} · {group:?} · {}: every arm the host's {} bytes", + c.name, + H::NAME, + host.len() + ); +} + +#[test] +fn an_opening_with_its_answers_on_the_card_proves_the_hosts_bytes() { + for c in cases() { + let store = gpu::upload_columns(&multilinear::stacking::borrow(&c.columns)) + .expect("the card took the columns (needs a GPU)"); + // The answers from resident columns, evaluated in place: the host's. + release(); + let (card, counts) = stacked_eval::ood_claims::( + &c.layout, + &multilinear::stacking::borrow(&c.columns), + Some((&store, ColumnsAt::From(0))), + &c.z0, + ) + .unwrap(); + let host = host_claims(&c); + assert_eq!(card.answers, host.answers, "{}: the answers", c.name); + assert_eq!(card.columns, host.columns, "{}: the column values", c.name); + assert!( + counts.on_card > 0, + "{}: no column evaluated on the card", + c.name + ); + println!( + "COMMIT OOD ANSWERS {}: {} columns on the card, {} on the host", + c.name, counts.on_card, counts.on_host + ); + for group in [GroupFold::PerPoly, GroupFold::Indexed] { + every_arm::(&c, group, &store); + every_arm::(&c, group, &store); + } + } + release(); +} diff --git a/crypto/multilinear/tests/whir_coset_device.rs b/crypto/multilinear/tests/whir_coset_device.rs new file mode 100644 index 000000000..214dea3aa --- /dev/null +++ b/crypto/multilinear/tests/whir_coset_device.rs @@ -0,0 +1,553 @@ +//! The coset leaf order on the card: a round-0 tree the card commits in the +//! coset order is the host's, and a stack retired and revived over its +//! resident columns proves the whole tree's bytes whether its openings are +//! evaluated coset by coset from the stack or read from a re-encoded codeword. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_coset_device +//! ``` +//! +//! Needs a GPU. The device paths are asserted TAKEN (the commit stayed on the +//! card; the stack-held openings were served from the stack and nothing fell +//! back), so a card that declined could not turn this into a host-against-host +//! comparison. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::CapPolicy; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{Claimed, StackedCommitment, StackedProof, prove, verify}; +use multilinear::stacking::{StackedLayout, borrow}; +use multilinear::whir::{Domain, encode, lift_coefficients}; +use multilinear::whir_chain::{ + self, ChainConfig, ChainFormat, FirstFold, GrindBits, LeafOrder, UnencodedSuccessor, WhirFolds, +}; +use multilinear::whir_commit::{CodewordCommitment, TreeDrop}; +use multilinear::whir_hash::{P1Whir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn sponge() -> ::Sponge { + let mut t = P1Whir::sponge(); + t.append_bytes(b"whir-coset-device"); + t +} + +fn config(leaves: LeafOrder, cap: CapPolicy) -> ChainConfig { + config_at(leaves, cap, 2) +} + +/// [`config`] at blowup `2^log_blowup` (`LAMBDA_VM_ZF_WHIR_BLOWUP=2` is 1). +fn config_at(leaves: LeafOrder, cap: CapPolicy, log_blowup: usize) -> ChainConfig { + ChainConfig { + log_blowup, + log_folding: 4, + num_queries: 33, + grind: GrindBits::uniform(8), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + arity4_cap: cap, + leaves, + ..ChainFormat::DEFAULT + }, + } +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(6364136223846793005) + .wrapping_add(17) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +macro_rules! bytes { + ($proof:expr) => { + rkyv::to_bytes::($proof) + .unwrap() + .to_vec() + }; +} + +/// The tests flip process-wide knobs (the stack-held openings, the encoded +/// successor), so they run one at a time. +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// ★ A coset-ordered chain committed on the card proves the bytes of the +/// same chain over a host tree in the coset order, and the host verifier +/// accepts it; its root is not the natural tree's. +#[test] +fn a_coset_chain_on_the_card_proves_the_host_chains_bytes() { + let _serial = serial(); + coset_chain_on_the_card(2); +} + +/// ★ Blowup 2: the same, every tree a level shallower (its depth parity +/// flipped), at a 2^18 codeword as at blowup 4. +#[test] +fn a_coset_chain_at_blowup_two_on_the_card_proves_the_host_chains_bytes() { + let _serial = serial(); + coset_chain_on_the_card(1); +} + +/// [`a_coset_chain_on_the_card_proves_the_host_chains_bytes`] at blowup +/// `2^log_blowup`, over a `2^18` codeword (above the card's commit threshold). +fn coset_chain_on_the_card(log_blowup: usize) { + for cap in [CapPolicy::Off, CapPolicy::Fixed(2), CapPolicy::Auto] { + let num_vars = 18 - log_blowup; + let cfg = config_at(LeafOrder::Coset, cap, log_blowup); + let f = column(num_vars, 3); + let z: Vec = (0..num_vars).map(|i| EE::from(301 + i as u64)).collect(); + let y = f.evaluate_in(&z).unwrap(); + let (device, domain) = whir_chain::commit::(&f, &cfg, true).unwrap(); + assert!( + device.codeword().device().is_some(), + "{cap}: the commit must have stayed on the card" + ); + let host_domain = Domain::::new(num_vars + cfg.log_blowup).unwrap(); + let host = CodewordCommitment::::from_codeword_on_host_in( + encode::(&lift_coefficients(&f), &host_domain).unwrap(), + cfg.schedule(num_vars)[0], + LeafOrder::Coset, + ) + .unwrap(); + assert_eq!(host.root(), device.root(), "{cap}: roots"); + let (natural, _) = whir_chain::commit::( + &f, + &config_at(LeafOrder::Natural, cap, log_blowup), + true, + ) + .unwrap(); + assert_ne!( + natural.root(), + device.root(), + "{cap}: the order moves the root" + ); + let device_proof = + whir_chain::prove::(&f, &z, &device, &domain, &cfg, &mut sponge()) + .unwrap(); + let host_proof = whir_chain::prove::( + &f, + &z, + &host, + &host_domain, + &cfg, + &mut sponge(), + ) + .unwrap(); + assert_eq!( + bytes!(&device_proof), + bytes!(&host_proof), + "{cap}: proof bytes" + ); + whir_chain::verify::( + &device_proof, + &device.root(), + &z, + y, + &domain, + &cfg, + &mut sponge(), + ) + .unwrap_or_else(|e| panic!("{cap}: the verifier refused the card's proof: {e:?}")); + } +} + +/// A stack's roots and its proof. +type Proved = (Vec<[u8; 32]>, StackedProof); + +/// A stack committed over resident columns and opened: the whole tree's +/// proof, then the same stack retired to `drop` 4-ary levels and revived. +struct Stack { + layout: StackedLayout, + columns: Vec>, + points: Vec>, + values: Vec, +} + +impl Stack { + fn new(n_stack: usize) -> Self { + Self::with_heights( + n_stack, + &[n_stack, n_stack - 1, n_stack - 1, n_stack - 2, n_stack - 3], + ) + } + + /// Columns of `2^h` values for each `h` of `heights`, stacked in that + /// order. + fn with_heights(n_stack: usize, heights: &[usize]) -> Self { + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 11 + 7 * i as u64)) + .collect(); + let points: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| EE::from(5 + 13 * i as u64 + 3 * j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p).unwrap()) + .collect(); + Self { + layout: StackedLayout::build(heights, n_stack).unwrap(), + columns, + points, + values, + } + } + + /// The proof, whole (`drop` None) or retired and revived, under `cfg`. + fn prove( + &self, + cfg: &ChainConfig, + drop: Option, + ) -> (Vec<[u8; 32]>, StackedProof) { + self.try_prove(cfg, drop).unwrap() + } + + /// [`Self::prove`], its refusal returned. + fn try_prove( + &self, + cfg: &ChainConfig, + drop: Option, + ) -> Result { + let refs: Vec<&Mle> = self.columns.iter().collect(); + let store = multilinear::gpu::upload_columns(&refs).expect("resident columns (a GPU)"); + let committed = StackedCommitment::::commit( + self.layout.clone(), + &borrow(&self.columns), + Some((&store, 0)), + cfg, + )?; + let roots = committed.roots(); + let stacked = match drop { + None => committed, + Some(drop) => committed + .retire(TreeDrop::uniform(drop), cfg)? + .revive::( + &borrow(&self.columns), + Some((&store, multilinear::stacked_eval::ColumnsAt::From(0))), + cfg, + )?, + }; + let proof = prove::( + &stacked, + &borrow(&self.columns), + Some((&store, 0)), + &Claimed::PerColumn(&self.points), + &self.values, + cfg, + &mut sponge(), + )?; + Ok((roots, proof)) + } + + fn verify(&self, roots: &[[u8; 32]], proof: &StackedProof, cfg: &ChainConfig) { + let domain = Domain::::new(self.layout.n_stack() + cfg.log_blowup).unwrap(); + verify::( + proof, + &self.layout, + roots, + &Claimed::PerColumn(&self.points), + &self.values, + &domain, + cfg, + &mut sponge(), + ) + .unwrap_or_else(|e| panic!("the verifier refused a stack's proof: {e:?}")); + } +} + +/// ★ THE LEVER'S PARITY. A coset-ordered stack retired to a kept top of 16 or +/// 256 leaves and revived over its resident columns proves the whole tree's +/// bytes: with its openings evaluated coset by coset from the stack (served +/// from the stack, none fallen back; the stack assembled, and read where it +/// lies) and with them read from a re-encoded codeword. At stacks whose cosets fold 4 and 6 bits, uncapped and under the +/// production cap. +#[test] +fn a_revived_coset_stack_proves_the_whole_trees_bytes_from_the_stack_or_a_reencode() { + let _serial = serial(); + revived_coset_stack(2); +} + +/// ★ Blowup 2: the lever's parity at the rate's shapes — the kept top's cosets +/// are still `2^(k0 + 2·drop)` points, the codeword half as long. +#[test] +fn a_revived_coset_stack_at_blowup_two_proves_the_whole_trees_bytes_from_the_stack_or_a_reencode() { + let _serial = serial(); + revived_coset_stack(1); +} + +/// [`a_revived_coset_stack_proves_the_whole_trees_bytes_from_the_stack_or_a_reencode`] +/// at blowup `2^log_blowup`. +fn revived_coset_stack(log_blowup: usize) { + // Short columns among the tall: tiles that hold several columns, or a + // column and zeros, which the stack read in place reads value by value + // (a tall column last, so no tile reaches past the store's end). + let short = |n: usize| { + vec![ + n, + n - 1, + 3, + 5, + 4, + 7, + 6, + 3, + 8, + n - 2, + 3, + 3, + 6, + n - 3, + 5, + n - 3, + ] + }; + for (n_stack, cap, short_columns) in [ + (18usize, CapPolicy::Auto, false), + (18, CapPolicy::Off, false), + (20, CapPolicy::Auto, false), + (18, CapPolicy::Auto, true), + ] { + let cfg = config_at(LeafOrder::Coset, cap, log_blowup); + let stack = if short_columns { + Stack::with_heights(n_stack, &short(n_stack)) + } else { + Stack::new(n_stack) + }; + let (roots, whole) = stack.prove(&cfg, None); + stack.verify(&roots, &whole, &cfg); + for drop in [2usize, 4] { + let label = format!( + "2^{n_stack} {cap} drop {drop} blowup 2^{log_blowup} short columns {short_columns}" + ); + // From the stack assembled, then from the stack read where it + // lies (the third polynomial's zero tail folded unread). + for in_place in [false, true] { + let label = format!("{label} in place {in_place}"); + multilinear::gpu::force_coset_openings(Some(true)); + multilinear::gpu::force_coset_inplace(Some(in_place)); + let served = multilinear::gpu::coset_openings(); + let fell_back = multilinear::gpu::coset_fallbacks(); + let read_in_place = multilinear::gpu::coset_inplace_opens(); + let assembled = multilinear::gpu::coset_inplace_assembled(); + let (zero, _) = multilinear::gpu::coset_inplace_tiles(); + let (again, from_stack) = stack.prove(&cfg, Some(drop)); + assert_eq!(again, roots, "{label}: roots"); + let opened = multilinear::gpu::coset_openings() - served; + assert!( + opened > 0, + "{label}: the openings must have been served from the stack" + ); + assert_eq!( + multilinear::gpu::coset_fallbacks(), + fell_back, + "{label}: nothing re-encoded" + ); + assert_eq!( + multilinear::gpu::coset_inplace_opens() - read_in_place, + if in_place { opened } else { 0 }, + "{label}: every opening read the stack in place, or none" + ); + assert_eq!( + multilinear::gpu::coset_inplace_assembled(), + assembled, + "{label}: no stack assembled after all" + ); + assert_eq!( + multilinear::gpu::coset_inplace_tiles().0 > zero, + in_place, + "{label}: zero tiles folded unread only in place" + ); + assert_eq!( + bytes!(&from_stack), + bytes!(&whole), + "{label}: from the stack" + ); + } + multilinear::gpu::force_coset_inplace(None); + multilinear::gpu::force_coset_openings(Some(false)); + let served = multilinear::gpu::coset_openings(); + let (_, reencoded) = stack.prove(&cfg, Some(drop)); + assert_eq!( + multilinear::gpu::coset_openings(), + served, + "{label}: off is off" + ); + assert_eq!( + bytes!(&reencoded), + bytes!(&whole), + "{label}: from a re-encode" + ); + multilinear::gpu::force_coset_openings(None); + } + } +} + +/// ★ Under the coset order a successor that is not encoded from the bound +/// message refuses the prove, named by its cause, on each path — the knob off, +/// the card declining the encode — for a stack committed on the card and one +/// revived over its columns. Each path is shown taken by its own counter where +/// it has one. The natural order under the same forcing folds and proves its +/// whole tree's bytes. +/// +/// And the two fallbacks that still encode, each passing with the whole tree's +/// bytes and the host fallback counted: the factors kept on the host (the card +/// declined the opening: the shared one and the uploaded one both), whose +/// successor is encoded on the host from the message they bound (one host +/// encode a chain); and only the shared opening declined, whose host tables are +/// uploaded to the card, so the card encodes. +#[test] +fn an_unencoded_successor_refuses_the_coset_prove_by_cause() { + let _serial = serial(); + let coset = config(LeafOrder::Coset, CapPolicy::Auto); + let natural = config(LeafOrder::Natural, CapPolicy::Auto); + let stack = Stack::new(18); + let (_, natural_whole) = stack.prove(&natural, None); + type Force = fn(bool); + let paths: [(&str, Force, UnencodedSuccessor); 2] = [ + ( + "the knob off", + |on| multilinear::gpu::force_encode_successor(on.then_some(false)), + UnencodedSuccessor::KnobOff, + ), + ( + "the card declining the encode", + multilinear::gpu::force_encode_decline, + UnencodedSuccessor::CardDeclined, + ), + ]; + for (name, force, cause) in paths { + for drop in [None, Some(4)] { + let label = format!("{name}, drop {drop:?}"); + let (declines, host) = ( + multilinear::gpu::encode_declines(), + multilinear::gpu::open_host_fallbacks(), + ); + force(true); + let refused = stack.try_prove(&coset, drop); + let natural_proof = stack.try_prove(&natural, drop); + force(false); + assert_eq!( + refused.err(), + Some(multilinear::Error::CosetSuccessorNotEncoded { cause }), + "{label}" + ); + let (_, natural_proof) = + natural_proof.unwrap_or_else(|e| panic!("{label}: the natural order folds: {e:?}")); + assert_eq!( + bytes!(&natural_proof), + bytes!(&natural_whole), + "{label}: natural bytes" + ); + match cause { + UnencodedSuccessor::CardDeclined => assert!( + multilinear::gpu::encode_declines() > declines, + "{label}: the decline counted" + ), + UnencodedSuccessor::HostFactors | UnencodedSuccessor::KnobOff => {} + } + assert_eq!( + multilinear::gpu::open_host_fallbacks(), + host, + "{label}: the factors on the card" + ); + } + } + // Every forcing undone: the coset stack proves again, retired as whole. + let (_, again) = stack.prove(&coset, Some(4)); + let (_, whole) = stack.prove(&coset, None); + assert_eq!(bytes!(&again), bytes!(&whole)); + // The factors kept on the host: the successor is encoded there, from the + // message they bound, and the coset prove passes with the same bytes. + let chains = stack.layout.num_polys() as u64; + for drop in [None, Some(4)] { + let label = format!("the factors kept on the host, drop {drop:?}"); + let (host, encodes) = ( + multilinear::gpu::open_host_fallbacks(), + multilinear::gpu::coset_host_encodes(), + ); + multilinear::gpu::force_shared_open_declined(true); + multilinear::gpu::force_device_open_declined(true); + let encoded = stack.try_prove(&coset, drop); + multilinear::gpu::force_device_open_declined(false); + multilinear::gpu::force_shared_open_declined(false); + let (_, encoded) = + encoded.unwrap_or_else(|e| panic!("{label}: the host encodes the successor: {e:?}")); + assert!( + multilinear::gpu::open_host_fallbacks() > host, + "{label}: the host fallback counted" + ); + assert_eq!( + multilinear::gpu::coset_host_encodes() - encodes, + chains, + "{label}: one host encode a chain" + ); + assert_eq!( + bytes!(&encoded), + bytes!(&whole), + "{label}: the whole tree's bytes" + ); + } + // Only the shared opening declined: the host's tables go to the card, the + // successor is encoded, and the coset prove passes with the same bytes. + for drop in [None, Some(4)] { + let host = multilinear::gpu::open_host_fallbacks(); + multilinear::gpu::force_shared_open_declined(true); + let uploaded = stack.try_prove(&coset, drop); + multilinear::gpu::force_shared_open_declined(false); + let (_, uploaded) = uploaded + .unwrap_or_else(|e| panic!("drop {drop:?}: the uploaded factors encode: {e:?}")); + assert!( + multilinear::gpu::open_host_fallbacks() > host, + "drop {drop:?}: the host's tables were built (counted)" + ); + assert_eq!( + bytes!(&uploaded), + bytes!(&whole), + "drop {drop:?}: the same bytes" + ); + } +} + +/// The natural order on the card is untouched: a retired natural stack still +/// proves its whole tree's bytes, and nothing is served from a stack. +#[test] +fn a_revived_natural_stack_is_untouched() { + let _serial = serial(); + let cfg = config(LeafOrder::Natural, CapPolicy::Auto); + let stack = Stack::new(18); + let (roots, whole) = stack.prove(&cfg, None); + let served = multilinear::gpu::coset_openings(); + let (again, revived) = stack.prove(&cfg, Some(4)); + assert_eq!(again, roots); + assert_eq!(multilinear::gpu::coset_openings(), served); + assert_eq!(bytes!(&revived), bytes!(&whole)); + stack.verify(&roots, &whole, &cfg); +} diff --git a/crypto/multilinear/tests/whir_coset_group_device.rs b/crypto/multilinear/tests/whir_coset_group_device.rs new file mode 100644 index 000000000..d205e32a2 --- /dev/null +++ b/crypto/multilinear/tests/whir_coset_group_device.rs @@ -0,0 +1,352 @@ +//! The coset leaf order under one chain per run of a group's polynomials +//! (`LeafOrder::Coset` with `GroupFold::Indexed`) on the card: a grouped run +//! keeps its round-0 codewords on the card and its successor is ENCODED from +//! the run's combined bound message — the bytes of the same run proved on the +//! host alone, where the host folds — or the prove is refused, named by its +//! cause. No path folds a grouped run's card codewords under the coset order. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_coset_group_device -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. Each path is asserted taken by its counter. The tests flip +//! process-wide knobs, so they run one at a time. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::CapPolicy; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{Claimed, StackedCommitment, StackedProof, prove, verify}; +use multilinear::stacking::{StackedLayout, borrow}; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, LeafOrder, RoundOpenings, + UnencodedSuccessor, WhirFolds, +}; +use multilinear::whir_commit::TreeDrop; +use multilinear::whir_hash::{P1Whir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn sponge() -> ::Sponge { + let mut t = P1Whir::sponge(); + t.append_bytes(b"whir-coset-group-device"); + t +} + +/// First6 then uniform4, one chain per run of up to four, no grind (its nonce +/// is not reproducible across two searches). +fn config(leaves: LeafOrder) -> ChainConfig { + config_at(leaves, 2) +} + +/// [`config`] at blowup `2^log_blowup` (`LAMBDA_VM_ZF_WHIR_BLOWUP=2` is 1). +fn config_at(leaves: LeafOrder, log_blowup: usize) -> ChainConfig { + ChainConfig { + log_blowup, + log_folding: 4, + num_queries: 33, + grind: GrindBits::default(), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + arity4_cap: CapPolicy::Auto, + group: GroupFold::Indexed, + leaves, + ..ChainFormat::DEFAULT + }, + } +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(6364136223846793005) + .wrapping_add(17) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +fn bytes(proof: &StackedProof) -> Vec { + rkyv::to_bytes::(proof) + .unwrap() + .to_vec() +} + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// A stack of three polynomials over `2^18` — one grouped chain — committed +/// over resident columns and opened whole, or retired and revived. +struct Stack { + layout: StackedLayout, + columns: Vec>, + points: Vec>, + values: Vec, +} + +type Proved = (Vec<[u8; 32]>, StackedProof); + +impl Stack { + fn new() -> Self { + let heights = [18usize, 17, 17, 16, 15]; + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 11 + 7 * i as u64)) + .collect(); + let points: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| EE::from(5 + 13 * i as u64 + 3 * j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p).unwrap()) + .collect(); + let layout = StackedLayout::build(&heights, 18).unwrap(); + assert_eq!(layout.num_polys(), 3, "one run of three"); + Self { + layout, + columns, + points, + values, + } + } + + fn try_prove( + &self, + cfg: &ChainConfig, + drop: Option, + ) -> Result { + let refs: Vec<&Mle> = self.columns.iter().collect(); + let store = gpu::upload_columns(&refs).expect("resident columns (a GPU)"); + let committed = StackedCommitment::::commit( + self.layout.clone(), + &borrow(&self.columns), + Some((&store, 0)), + cfg, + )?; + let roots = committed.roots(); + let stacked = match drop { + None => committed, + Some(drop) => committed + .retire(TreeDrop::uniform(drop), cfg)? + .revive::( + &borrow(&self.columns), + Some((&store, multilinear::stacked_eval::ColumnsAt::From(0))), + cfg, + )?, + }; + let proof = prove::( + &stacked, + &borrow(&self.columns), + Some((&store, 0)), + &Claimed::PerColumn(&self.points), + &self.values, + cfg, + &mut sponge(), + )?; + Ok((roots, proof)) + } + + fn verify( + &self, + roots: &[[u8; 32]], + proof: &StackedProof, + cfg: &ChainConfig, + label: &str, + ) { + assert_eq!(proof.polys.len(), 1, "{label}: one chain"); + assert!( + matches!( + proof.polys[0].rounds[0].openings, + RoundOpenings::BaseGrouped(_) + ), + "{label}: a grouped round 0" + ); + let domain = Domain::::new(self.layout.n_stack() + cfg.log_blowup).unwrap(); + verify::( + proof, + &self.layout, + roots, + &Claimed::PerColumn(&self.points), + &self.values, + &domain, + cfg, + &mut sponge(), + ) + .unwrap_or_else(|e| panic!("{label}: the verifier refused: {e:?}")); + } +} + +/// ★ A grouped run under the coset order on the card — committed and opened +/// whole, and retired and revived (its codewords re-encoded on the card) — +/// encodes its successor and proves the bytes of the same run proved on the +/// host alone (the host's codewords, factors and fold). Its roots are not the +/// natural order's. +#[test] +fn a_grouped_coset_run_on_the_card_proves_the_host_runs_bytes() { + let _serial = serial(); + grouped_coset_run_on_the_card(2); +} + +/// ★ Blowup 2 — the composed format's round 0 (one chain a run, the coset +/// order, rate 1/2): the same run on the card encodes its successor and proves +/// the host run's bytes, whole and revived. +#[test] +fn a_grouped_coset_run_at_blowup_two_on_the_card_proves_the_host_runs_bytes() { + let _serial = serial(); + grouped_coset_run_on_the_card(1); +} + +/// [`a_grouped_coset_run_on_the_card_proves_the_host_runs_bytes`] at blowup +/// `2^log_blowup`. +fn grouped_coset_run_on_the_card(log_blowup: usize) { + let coset = config_at(LeafOrder::Coset, log_blowup); + let stack = Stack::new(); + gpu::force_host_whir_commit(true); + gpu::force_shared_open_declined(true); + let host = stack.try_prove(&coset, None); + gpu::force_shared_open_declined(false); + gpu::force_host_whir_commit(false); + let (host_roots, host_proof) = host.unwrap_or_else(|e| panic!("the host run: {e:?}")); + stack.verify(&host_roots, &host_proof, &coset, "the host run"); + for drop in [None, Some(2), Some(4)] { + let label = format!("drop {drop:?} blowup 2^{log_blowup}"); + let (encoded, chains) = (gpu::encoded_successors(), gpu::grouped_card_chains()); + let (roots, proof) = stack + .try_prove(&coset, drop) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + assert_eq!( + gpu::encoded_successors() - encoded, + 1, + "{label}: the successor encoded" + ); + assert_eq!( + gpu::grouped_card_chains() - chains, + 1, + "{label}: round 0 on the card" + ); + assert_eq!(roots, host_roots, "{label}: roots"); + stack.verify(&roots, &proof, &coset, &label); + assert!( + bytes(&proof) == bytes(&host_proof), + "{label}: the host run's bytes" + ); + } + let (natural_roots, _) = stack + .try_prove(&config_at(LeafOrder::Natural, log_blowup), None) + .unwrap(); + assert_ne!(natural_roots, host_roots, "the coset order moves the roots"); +} + +/// ★ Under the coset order a grouped run's successor that is not encoded +/// refuses the prove, named by its cause, on each path — the knob off, the card +/// declining the encode — committed and opened whole and revived. Each path is +/// shown taken by its counter where it has one. The natural order under the +/// same forcing folds and proves its unforced bytes. A run whose factors the +/// host built (the card declined an opening) encodes its successor on the host +/// and proves its unforced bytes. +#[test] +fn an_unencoded_grouped_successor_refuses_the_coset_prove_by_cause() { + let _serial = serial(); + let coset = config(LeafOrder::Coset); + let natural = config(LeafOrder::Natural); + let stack = Stack::new(); + let (_, natural_whole) = stack.try_prove(&natural, None).unwrap(); + type Force = fn(bool); + let paths: [(&str, Force, UnencodedSuccessor); 2] = [ + ( + "the knob off", + |on| gpu::force_encode_successor(on.then_some(false)), + UnencodedSuccessor::KnobOff, + ), + ( + "the card declining the encode", + gpu::force_encode_decline, + UnencodedSuccessor::CardDeclined, + ), + ]; + for (name, force, cause) in paths { + for drop in [None, Some(4)] { + let label = format!("{name}, drop {drop:?}"); + let (declines, host) = (gpu::encode_declines(), gpu::grouped_host_chains()); + force(true); + let refused = stack.try_prove(&coset, drop); + let natural_proof = stack.try_prove(&natural, drop); + force(false); + assert_eq!( + refused.err(), + Some(multilinear::Error::CosetSuccessorNotEncoded { cause }), + "{label}" + ); + let (_, natural_proof) = + natural_proof.unwrap_or_else(|e| panic!("{label}: the natural order folds: {e:?}")); + assert!( + bytes(&natural_proof) == bytes(&natural_whole), + "{label}: the natural order's bytes" + ); + match cause { + UnencodedSuccessor::CardDeclined => assert!( + gpu::encode_declines() > declines, + "{label}: the decline counted" + ), + UnencodedSuccessor::HostFactors | UnencodedSuccessor::KnobOff => {} + } + assert_eq!( + gpu::grouped_host_chains(), + host, + "{label}: the run's factors on the card" + ); + } + } + // Every forcing undone: the coset run proves again. + let (roots, again) = stack.try_prove(&coset, Some(4)).unwrap(); + stack.verify(&roots, &again, &coset, "after the forcings"); + // The run's factors on the host (the card declined the opening): its + // successor is encoded there, from the combined message the rounds bound, + // and the run proves the bytes it proves unforced. + let (_, whole) = stack.try_prove(&coset, None).unwrap(); + for drop in [None, Some(4)] { + let label = format!("the run's factors on the host, drop {drop:?}"); + let (host, encodes) = (gpu::grouped_host_chains(), gpu::coset_host_encodes()); + gpu::force_shared_open_declined(true); + let encoded = stack.try_prove(&coset, drop); + gpu::force_shared_open_declined(false); + let (roots, encoded) = + encoded.unwrap_or_else(|e| panic!("{label}: the host encodes the successor: {e:?}")); + assert!( + gpu::grouped_host_chains() > host, + "{label}: the grouped host fallback counted" + ); + assert_eq!( + gpu::coset_host_encodes() - encodes, + 1, + "{label}: one host encode, the run's" + ); + stack.verify(&roots, &encoded, &coset, &label); + assert!(bytes(&encoded) == bytes(&whole), "{label}: the run's bytes"); + } +} diff --git a/crypto/multilinear/tests/whir_coset_group_stack_device.rs b/crypto/multilinear/tests/whir_coset_group_stack_device.rs new file mode 100644 index 000000000..31edf71bd --- /dev/null +++ b/crypto/multilinear/tests/whir_coset_group_stack_device.rs @@ -0,0 +1,477 @@ +//! The coset leaf order under one chain per run (`LeafOrder::Coset` with +//! `GroupFold::Indexed`) on the card, a group retired and revived: every tree +//! of a run is held as its stack and served from it (each opening evaluates +//! the cosets its queries fall in), no round-0 codeword is re-encoded, and the +//! run's successor is encoded from its combined bound message — the bytes of +//! the same group proved on the host alone. A successor not encoded still +//! refuses the prove, by cause. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_coset_group_stack_device -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. Each path is asserted taken by its counter. The tests flip +//! process-wide knobs, so they run one at a time. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::CapPolicy; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{ + Claimed, ColumnsAt, StackedCommitment, StackedProof, index_chains, prove, verify, +}; +use multilinear::stacking::{StackedLayout, borrow}; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, LeafOrder, UnencodedSuccessor, + WhirFolds, +}; +use multilinear::whir_commit::TreeDrop; +use multilinear::whir_hash::{P1Whir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn sponge() -> ::Sponge { + let mut t = P1Whir::sponge(); + t.append_bytes(b"whir-coset-group-stack-device"); + t +} + +/// First6 then uniform4, one chain per run of up to four, no grind (its nonce +/// is not reproducible across two searches). +fn config(leaves: LeafOrder) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 33, + grind: GrindBits::default(), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + arity4_cap: CapPolicy::Auto, + group: GroupFold::Indexed, + leaves, + ..ChainFormat::DEFAULT + }, + } +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(6364136223846793005) + .wrapping_add(17) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +fn bytes(proof: &StackedProof) -> Vec { + rkyv::to_bytes::(proof) + .unwrap() + .to_vec() +} + +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +type Proved = (Vec<[u8; 32]>, StackedProof); + +/// A group's columns, stacked at `n_stack`, committed over resident columns +/// and opened whole or retired and revived. +struct Stack { + name: &'static str, + layout: StackedLayout, + columns: Vec>, + points: Vec>, + values: Vec, +} + +impl Stack { + fn new(name: &'static str, heights: &[usize], n_stack: usize) -> Self { + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 23 + 5 * i as u64)) + .collect(); + let points: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| EE::from(7 + 11 * i as u64 + 3 * j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p).unwrap()) + .collect(); + let layout = StackedLayout::build(heights, n_stack).unwrap(); + Self { + name, + layout, + columns, + points, + values, + } + } + + /// One run of three polynomials over `2^18`. + fn run_of_three() -> Self { + let stack = Self::new("a run of three", &[18, 17, 17, 16, 15], 18); + assert_eq!(stack.layout.num_polys(), 3, "one run of three"); + stack + } + + /// Five polynomials over `2^18`: a run of four and a polynomial alone. + fn runs_of_four_and_one() -> Self { + let stack = Self::new("runs [4, 1]", &[18; 5], 18); + assert_eq!(stack.layout.num_polys(), 5, "five polynomials"); + stack + } + + fn trees(&self) -> u64 { + self.layout.num_polys() as u64 + } + + fn chains(&self) -> u64 { + index_chains(self.layout.num_polys(), GroupFold::Indexed).len() as u64 + } + + fn grouped_runs(&self) -> u64 { + index_chains(self.layout.num_polys(), GroupFold::Indexed) + .iter() + .filter(|run| run.len() > 1) + .count() as u64 + } + + fn try_prove( + &self, + cfg: &ChainConfig, + drop: Option, + ) -> Result { + let refs: Vec<&Mle> = self.columns.iter().collect(); + let store = gpu::upload_columns(&refs).expect("resident columns (a GPU)"); + let committed = StackedCommitment::::commit( + self.layout.clone(), + &borrow(&self.columns), + Some((&store, 0)), + cfg, + )?; + let roots = committed.roots(); + let stacked = match drop { + None => committed, + Some(drop) => committed + .retire(TreeDrop::uniform(drop), cfg)? + .revive::( + &borrow(&self.columns), + Some((&store, ColumnsAt::From(0))), + cfg, + )?, + }; + let proof = prove::( + &stacked, + &borrow(&self.columns), + Some((&store, 0)), + &Claimed::PerColumn(&self.points), + &self.values, + cfg, + &mut sponge(), + )?; + Ok((roots, proof)) + } + + fn verify( + &self, + roots: &[[u8; 32]], + proof: &StackedProof, + cfg: &ChainConfig, + label: &str, + ) { + let domain = Domain::::new(self.layout.n_stack() + cfg.log_blowup).unwrap(); + verify::( + proof, + &self.layout, + roots, + &Claimed::PerColumn(&self.points), + &self.values, + &domain, + cfg, + &mut sponge(), + ) + .unwrap_or_else(|e| panic!("{label}: the verifier refused: {e:?}")); + } + + /// The same group proved on the host alone: host codewords, host factors, + /// host folds. + fn on_the_host(&self, cfg: &ChainConfig) -> Proved { + gpu::force_host_whir_commit(true); + gpu::force_shared_open_declined(true); + let host = self.try_prove(cfg, None); + gpu::force_shared_open_declined(false); + gpu::force_host_whir_commit(false); + host.unwrap_or_else(|e| panic!("{}: the host run: {e:?}", self.name)) + } +} + +/// The counters a revived group's openings move. +#[derive(Clone, Copy)] +struct Counts { + from_stack: u64, + reencoded: u64, + encoded: u64, + grouped: u64, +} + +impl Counts { + fn now() -> Self { + Self { + from_stack: gpu::coset_openings(), + reencoded: gpu::coset_fallbacks(), + encoded: gpu::encoded_successors(), + grouped: gpu::grouped_card_chains(), + } + } + + fn since(self) -> Self { + let now = Self::now(); + Self { + from_stack: now.from_stack - self.from_stack, + reencoded: now.reencoded - self.reencoded, + encoded: now.encoded - self.encoded, + grouped: now.grouped - self.grouped, + } + } +} + +/// ★ A group retired and revived under the coset order and one chain per run +/// serves EVERY tree from its stack — the openings counter rises by the +/// group's tree count and nothing is re-encoded — and encodes each chain's +/// successor, the run's from its combined bound message: the bytes of the same +/// group proved on the host alone, at two kept depths. Opened whole, its trees +/// are held codewords and the counter does not move. With the stack path +/// switched off the revived group re-encodes (the counter does not move) and +/// proves the same bytes. +#[test] +fn a_revived_grouped_coset_run_serves_every_tree_from_its_stack() { + let _serial = serial(); + let coset = config(LeafOrder::Coset); + for stack in [Stack::run_of_three(), Stack::runs_of_four_and_one()] { + let (host_roots, host_proof) = stack.on_the_host(&coset); + stack.verify(&host_roots, &host_proof, &coset, stack.name); + + let before = Counts::now(); + let (roots, whole) = stack + .try_prove(&coset, None) + .unwrap_or_else(|e| panic!("{}, whole: {e:?}", stack.name)); + let moved = before.since(); + assert_eq!(roots, host_roots, "{}, whole: roots", stack.name); + assert!( + bytes(&whole) == bytes(&host_proof), + "{}, whole: the host's bytes", + stack.name + ); + assert_eq!(moved.from_stack, 0, "{}, whole: held codewords", stack.name); + assert_eq!( + moved.encoded, + stack.chains(), + "{}, whole: every successor encoded", + stack.name + ); + + for drop in [2, 4] { + let label = format!("{}, revived at drop {drop}", stack.name); + let before = Counts::now(); + let (roots, proof) = stack + .try_prove(&coset, Some(drop)) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let moved = before.since(); + assert_eq!( + moved.from_stack, + stack.trees(), + "{label}: every tree from its stack" + ); + assert_eq!(moved.reencoded, 0, "{label}: nothing re-encoded"); + assert_eq!( + moved.encoded, + stack.chains(), + "{label}: every successor encoded" + ); + assert_eq!( + moved.grouped, + stack.grouped_runs(), + "{label}: the runs on the card" + ); + assert_eq!(roots, host_roots, "{label}: roots"); + stack.verify(&roots, &proof, &coset, &label); + assert!( + bytes(&proof) == bytes(&host_proof), + "{label}: the host's bytes" + ); + } + + let label = format!("{}, revived at drop 4, the stack path off", stack.name); + gpu::force_coset_openings(Some(false)); + let before = Counts::now(); + let reencoded = stack.try_prove(&coset, Some(4)); + let moved = before.since(); + gpu::force_coset_openings(None); + let (_, reencoded) = reencoded.unwrap_or_else(|e| panic!("{label}: {e:?}")); + assert_eq!(moved.from_stack, 0, "{label}: no tree from its stack"); + assert_eq!( + moved.encoded, + stack.chains(), + "{label}: every successor encoded" + ); + assert!( + bytes(&reencoded) == bytes(&host_proof), + "{label}: the host's bytes" + ); + } +} + +/// ★ Under the coset order a revived run's successor that is not encoded +/// refuses the prove, named by its cause, on each path — the knob off, the card +/// declining the encode (counted). The natural order under the same forcing +/// re-encodes, folds and proves its unforced bytes. +/// +/// A revived run whose factors the host built (the card declined the opening; +/// counted as a grouped host fallback, a tree held as its stack being a card +/// codeword) encodes its successor on the host from the combined message the +/// rounds bound, re-encodes nothing, and proves the bytes the run proves whole. +#[test] +fn a_revived_grouped_coset_run_refuses_an_unencoded_successor_by_cause() { + let _serial = serial(); + let coset = config(LeafOrder::Coset); + let natural = config(LeafOrder::Natural); + let stack = Stack::run_of_three(); + let (_, natural_whole) = stack.try_prove(&natural, None).unwrap(); + let (_, coset_whole) = stack.try_prove(&coset, None).unwrap(); + type Force = fn(bool); + let paths: [(&str, Force, UnencodedSuccessor); 2] = [ + ( + "the knob off", + |on| gpu::force_encode_successor(on.then_some(false)), + UnencodedSuccessor::KnobOff, + ), + ( + "the card declining the encode", + gpu::force_encode_decline, + UnencodedSuccessor::CardDeclined, + ), + ]; + for (name, force, cause) in paths { + let label = format!("{name}, revived at drop 4"); + let declines = gpu::encode_declines(); + let before = Counts::now(); + force(true); + let refused = stack.try_prove(&coset, Some(4)); + let natural_proof = stack.try_prove(&natural, Some(4)); + force(false); + let moved = before.since(); + assert_eq!( + refused.err(), + Some(multilinear::Error::CosetSuccessorNotEncoded { cause }), + "{label}" + ); + assert_eq!(moved.reencoded, 0, "{label}: nothing re-encoded"); + let (_, natural_proof) = + natural_proof.unwrap_or_else(|e| panic!("{label}: the natural order folds: {e:?}")); + assert!( + bytes(&natural_proof) == bytes(&natural_whole), + "{label}: the natural order's bytes" + ); + if cause == UnencodedSuccessor::CardDeclined { + assert!( + gpu::encode_declines() > declines, + "{label}: the decline counted" + ); + } + } + // The run's factors on the host: its successor is encoded there. + let label = "the run's factors on the host, revived at drop 4"; + let (host, encodes) = (gpu::grouped_host_chains(), gpu::coset_host_encodes()); + let before = Counts::now(); + gpu::force_shared_open_declined(true); + let encoded = stack.try_prove(&coset, Some(4)); + let natural_proof = stack.try_prove(&natural, Some(4)); + gpu::force_shared_open_declined(false); + let moved = before.since(); + let (roots, encoded) = + encoded.unwrap_or_else(|e| panic!("{label}: the host encodes the successor: {e:?}")); + assert!( + gpu::grouped_host_chains() >= host + 2, + "{label}: the grouped host fallback counted for the coset run and the natural one" + ); + assert_eq!( + gpu::coset_host_encodes() - encodes, + stack.chains(), + "{label}: one host encode a chain" + ); + assert_eq!(moved.reencoded, 0, "{label}: nothing re-encoded"); + stack.verify(&roots, &encoded, &coset, label); + assert!( + bytes(&encoded) == bytes(&coset_whole), + "{label}: the run's whole bytes" + ); + let (_, natural_proof) = + natural_proof.unwrap_or_else(|e| panic!("{label}: the natural order folds: {e:?}")); + assert!( + bytes(&natural_proof) == bytes(&natural_whole), + "{label}: the natural order's bytes" + ); + // Every forcing undone: the revived coset run proves again, from its stack. + let before = Counts::now(); + let (roots, again) = stack.try_prove(&coset, Some(4)).unwrap(); + assert_eq!( + before.since().from_stack, + stack.trees(), + "after the forcings: from the stack" + ); + stack.verify(&roots, &again, &coset, "after the forcings"); +} + +/// A revived group under the natural order and one chain per run is untouched: +/// no tree is served from a stack, and it proves the bytes it proves whole. +#[test] +fn a_revived_natural_grouped_run_is_untouched() { + let _serial = serial(); + let natural = config(LeafOrder::Natural); + for stack in [Stack::run_of_three(), Stack::runs_of_four_and_one()] { + let (_, whole) = stack.try_prove(&natural, None).unwrap(); + for drop in [2, 4] { + let label = format!("{}, natural, revived at drop {drop}", stack.name); + let before = Counts::now(); + let (roots, proof) = stack + .try_prove(&natural, Some(drop)) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + assert_eq!( + before.since().from_stack, + 0, + "{label}: no tree from a stack" + ); + stack.verify(&roots, &proof, &natural, &label); + assert!( + bytes(&proof) == bytes(&whole), + "{label}: the whole group's bytes" + ); + } + } +} diff --git a/crypto/multilinear/tests/whir_group_device.rs b/crypto/multilinear/tests/whir_group_device.rs new file mode 100644 index 000000000..b02c721a5 --- /dev/null +++ b/crypto/multilinear/tests/whir_group_device.rs @@ -0,0 +1,529 @@ +//! One WHIR chain per run of a group's stacked polynomials +//! (`GroupFold::Indexed`) with its round 0 on the card: the polynomials' +//! openings in lockstep, the index rounds over their bound tables laid slot by +//! slot, the successor encoded from the combined bound message (or each +//! codeword folded and the folds combined there) — proves the SAME bytes as +//! the host alone, codewords, folds, trees and factors all on the host. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_group_device -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. Cases: runs of 2, 3 and 4; a group of 5 and one of 9 split into +//! runs [4, 1] and [4, 4, 1] (the runs of one are today's chains); a run of 3 +//! whose columns' selectors straddle the index and whose polynomials take +//! different forms for their lean rounds (tile and share); and a 2^6 stack, +//! below every card threshold, where a card build proves on the host. Arms: the +//! defaults; the share form; the tile form checked against the share form on +//! every round; no lean rounds; four and six lean rounds; every successor +//! folded and combined on the card; resident columns; and the host fallback — +//! the card's codewords with the factors declined, which must be COUNTED. +//! Every path is asserted TAKEN in its arm by its counter, so a card that +//! declined could not turn a comparison into one of the host with itself. One +//! `#[test]`: the arms switch process-wide overrides, and parallel tests would +//! race them. Every case runs at blowup 4 and again at blowup 2 +//! (`LAMBDA_VM_ZF_WHIR_BLOWUP=2`), its lines then labelled ` · lb 1`: twelve +//! `GROUPED PARITY` lines a blowup, the blowup-4 ones as before. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{self, Claimed, StackedCommitment, index_chains}; +use multilinear::stacking::StackedLayout; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, RoundOpenings, WhirFolds, +}; +use multilinear::whir_hash::{P1Whir, RpxWhir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(seed.wrapping_mul(6364136223846793005)) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +/// First6 then uniform4, caps on every tree, one chain per run of up to four +/// polynomials, at blowup `2^log_blowup`; no grind, whose nonce is not +/// reproducible across two searches. +fn config(log_blowup: usize) -> ChainConfig { + ChainConfig { + log_blowup, + log_folding: 4, + num_queries: 20, + grind: GrindBits::default(), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + group: GroupFold::Indexed, + ..ChainFormat::DEFAULT + }, + } +} + +/// Columns stacked as `heights` gives them, each claimed at the point of its +/// height (a table's columns share one), and what the run split derives. +struct Case { + name: &'static str, + layout: StackedLayout, + columns: Vec>, + points: Vec>, + values: Vec, + /// Chains over two or more polynomials, the polynomials they open, and + /// every chain. + grouped: u64, + grouped_polys: u64, + chains: u64, + /// Whether the stack is wide enough for the card to take it at all. + on_card: bool, + /// The case's blowup, `log2`. + log_blowup: usize, +} + +fn case_of(name: &'static str, heights: &[usize], n_stack: usize, polys: usize) -> Case { + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 7 + 31 * i as u64)) + .collect(); + let layout = StackedLayout::build(heights, n_stack).unwrap(); + assert_eq!(layout.num_polys(), polys, "{name}: the stack's polynomials"); + let points: Vec> = heights + .iter() + .map(|&h| { + (0..h) + .map(|j| EE::from(101 + 13 * h as u64 + j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, z)| c.evaluate_in(z).unwrap()) + .collect(); + let runs = index_chains(polys, GroupFold::Indexed); + let grouped: Vec<_> = runs.iter().filter(|run| run.len() >= 2).collect(); + Case { + name, + layout, + columns, + points, + values, + grouped: grouped.len() as u64, + grouped_polys: grouped.iter().map(|run| run.len() as u64).sum(), + chains: runs.len() as u64, + on_card: n_stack >= 14, + log_blowup: 2, + } +} + +impl Case { + /// The case's name, tagged ` · lb 1` at blowup 2: the blowup-4 lines stay + /// the lines they were. + fn label(&self) -> String { + if self.log_blowup == 2 { + self.name.to_string() + } else { + format!("{} · lb {}", self.name, self.log_blowup) + } + } +} + +/// `c` at blowup `2^log_blowup`: the card takes a stack whose codeword reaches +/// its commit threshold (2^16 elements). Its label names the blowup off the +/// default (`run of 2 · lb 1`), so a read-out counts each blowup's parity lines. +fn at_blowup(c: Case, log_blowup: usize) -> Case { + let n_stack = c.layout.n_stack(); + Case { + on_card: n_stack + log_blowup >= 16, + log_blowup, + ..c + } +} + +fn cases() -> Vec { + vec![ + case_of("run of 2", &[16, 15, 15], 16, 2), + // Polynomial 2's columns run down to 2^3 rows: selectors of up to 13 + // bits, past the index after the first 6, and a stack the tile form + // declines while polynomials 0 and 1 take it. + case_of( + "run of 3, mixed forms", + &[ + 15, 14, 13, 12, 12, 16, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 3, + ], + 16, + 3, + ), + case_of("run of 4", &[16; 4], 16, 4), + case_of("runs [4, 1]", &[16; 5], 16, 5), + case_of("runs [4, 4, 1]", &[16; 9], 16, 9), + case_of("below the card", &[6, 6, 5, 5], 6, 3), + ] +} + +/// Which path an arm proves on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Arm { + /// Every codeword, fold, tree and factor on the host: the reference. + Host, + /// Nothing forced: three lean rounds in tile form where a stack takes it, + /// the fused fold, each first successor encoded from its bound message. + Defaults, + /// The lean rounds in share form (`LAMBDA_VM_WHIR_LEAN_TILES=0`). + ShareForm, + /// The tile form, every round and bound table compared with the share form + /// (`=check`). + TileCheck, + /// No lean rounds: every opening written at full width up front. + Materialised, + /// Four, then six lean rounds (`LAMBDA_VM_WHIR_LEAN_ROUNDS`). + Lean(usize), + /// Every successor folded (`LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0`): a grouped + /// one is each codeword folded and the folds combined on the card. + Folded, + /// The codewords on the card, every opening's factors declined there: the + /// host's grouped round 0, which must be counted. + Fallback, +} + +fn force(arm: Arm) { + gpu::force_host_whir_commit(arm == Arm::Host); + gpu::force_shared_open_declined(matches!(arm, Arm::Host | Arm::Fallback)); + gpu::force_encode_successor(match arm { + Arm::Folded => Some(false), + _ => None, + }); + gpu::force_lean_tiles(match arm { + Arm::ShareForm => Some(gpu::LeanTiles::Off), + Arm::TileCheck => Some(gpu::LeanTiles::Check), + _ => None, + }); + gpu::force_lean_rounds(match arm { + Arm::Materialised => Some(0), + Arm::Lean(rounds) => Some(rounds), + _ => None, + }); + gpu::force_fused_fold(None); +} + +fn release() { + gpu::force_host_whir_commit(false); + gpu::force_shared_open_declined(false); + gpu::force_encode_successor(None); + gpu::force_lean_tiles(None); + gpu::force_lean_rounds(None); + gpu::force_fused_fold(None); +} + +/// What the counters moved by over one proof. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Moved { + card_chains: u64, + host_chains: u64, + combined: u64, + encoded: u64, + opens: u64, + lean: u64, + tiles: u64, + open_fallbacks: u64, + commit_fallbacks: u64, + rounds_compared: u64, + rounds_differing: u64, + positions_differing: u64, +} + +fn counters() -> Moved { + let (compared, differing) = gpu::lean_tile_checks(); + let (_, positions) = gpu::lean_tile_table_checks(); + Moved { + card_chains: gpu::grouped_card_chains(), + host_chains: gpu::grouped_host_chains(), + combined: gpu::grouped_combined_successors(), + encoded: gpu::encoded_successors(), + opens: gpu::open_calls(), + lean: gpu::lean_open_calls(), + tiles: gpu::lean_tile_opens(), + open_fallbacks: gpu::open_host_fallbacks(), + commit_fallbacks: gpu::host_fallbacks(), + rounds_compared: compared, + rounds_differing: differing, + positions_differing: positions, + } +} + +fn since(before: Moved) -> Moved { + let now = counters(); + Moved { + card_chains: now.card_chains - before.card_chains, + host_chains: now.host_chains - before.host_chains, + combined: now.combined - before.combined, + encoded: now.encoded - before.encoded, + opens: now.opens - before.opens, + lean: now.lean - before.lean, + tiles: now.tiles - before.tiles, + open_fallbacks: now.open_fallbacks - before.open_fallbacks, + commit_fallbacks: now.commit_fallbacks - before.commit_fallbacks, + rounds_compared: now.rounds_compared - before.rounds_compared, + rounds_differing: now.rounds_differing - before.rounds_differing, + positions_differing: now.positions_differing - before.positions_differing, + } +} + +fn sponge() -> H::Sponge { + let mut t = H::sponge(); + t.append_bytes(b"whir-group-device"); + t +} + +/// One arm: commits, proves, has the host verifier accept, checks the arm's +/// counters, and returns the proof's bytes and the roots. +fn prove_arm( + c: &Case, + arm: Arm, + resident: Option<&gpu::ResidentColumns>, +) -> (Vec, Vec<[u8; 32]>) { + force(arm); + let tag = format!("{} · {} · {arm:?}", c.label(), H::NAME); + let columns = multilinear::stacking::borrow(&c.columns); + let before = counters(); + let stacked = StackedCommitment::::commit( + c.layout.clone(), + &columns, + resident.map(|s| (s, 0)), + &config(c.log_blowup), + ) + .unwrap(); + let proof = stacked_eval::prove::( + &stacked, + &columns, + resident.map(|s| (s, 0)), + &Claimed::PerColumn(&c.points), + &c.values, + &config(c.log_blowup), + &mut sponge::(), + ) + .unwrap_or_else(|e| panic!("{tag}: the prover refused: {e:?}")); + let moved = since(before); + stacked_eval::verify::( + &proof, + &c.layout, + &stacked.roots(), + &Claimed::PerColumn(&c.points), + &c.values, + stacked.domain(), + &config(c.log_blowup), + &mut sponge::(), + ) + .unwrap_or_else(|e| panic!("{tag}: the host verifier refused: {e:?}")); + + // The shape the split derives: one chain a run, the grouped variant at the + // grouped chains' round 0. + assert_eq!(proof.polys.len() as u64, c.chains, "{tag}: chains"); + let grouped = proof + .polys + .iter() + .filter(|chain| matches!(chain.rounds[0].openings, RoundOpenings::BaseGrouped(_))) + .count() as u64; + assert_eq!(grouped, c.grouped, "{tag}: grouped chains"); + + let card = c.on_card; + match arm { + Arm::Host => { + assert_eq!(moved.card_chains, 0, "{tag}: a grouped round 0 on the card"); + assert_eq!( + moved.host_chains, 0, + "{tag}: the host path counted as a fallback" + ); + assert_eq!(moved.opens, 0, "{tag}: an opening on the card"); + assert_eq!(moved.combined, 0, "{tag}: a combine on the card"); + assert_eq!(moved.encoded, 0, "{tag}: an encode on the card"); + } + Arm::Fallback if card => { + assert_eq!(moved.card_chains, 0, "{tag}: a grouped round 0 on the card"); + assert_eq!( + moved.host_chains, c.grouped, + "{tag}: every grouped round 0 counted as a host fallback" + ); + assert_eq!( + moved.open_fallbacks, + c.grouped_polys + (c.chains - c.grouped), + "{tag}: every polynomial's factors counted on the host" + ); + assert_eq!( + moved.combined, c.grouped, + "{tag}: every grouped successor folded and combined on the card" + ); + // A run of one falls back as today: its host-built tables go to + // the card whole, and its successor is encoded from them there. + assert_eq!( + moved.encoded, + c.chains - c.grouped, + "{tag}: successors encoded (the runs of one alone)" + ); + } + _ if card => { + assert_eq!( + moved.card_chains, c.grouped, + "{tag}: every grouped round 0 on the card" + ); + assert_eq!(moved.host_chains, 0, "{tag}: grouped host fallbacks"); + assert_eq!(moved.open_fallbacks, 0, "{tag}: openings on the host"); + assert_eq!(moved.commit_fallbacks, 0, "{tag}: commits on the host"); + assert_eq!( + moved.opens, + c.grouped_polys + (c.chains - c.grouped), + "{tag}: one card opening a polynomial" + ); + let (encoded, combined) = if arm == Arm::Folded { + (0, c.grouped) + } else { + (c.chains, 0) + }; + assert_eq!(moved.encoded, encoded, "{tag}: successors encoded"); + assert_eq!(moved.combined, combined, "{tag}: successors combined"); + let lean = if arm == Arm::Materialised { + 0 + } else { + moved.opens + }; + assert_eq!(moved.lean, lean, "{tag}: lean openings"); + if arm == Arm::ShareForm || arm == Arm::Materialised { + assert_eq!(moved.tiles, 0, "{tag}: an opening in tile form"); + } + if arm == Arm::TileCheck { + assert!(moved.tiles > 0, "{tag}: no opening in tile form"); + assert!(moved.rounds_compared > 0, "{tag}: no round compared"); + } + assert_eq!(moved.rounds_differing, 0, "{tag}: tile rounds differing"); + assert_eq!(moved.positions_differing, 0, "{tag}: tile tables differing"); + } + // Below the card's thresholds every arm is the host's path, and none + // of it is a fallback. + _ => { + assert_eq!(moved.card_chains, 0, "{tag}: a grouped round 0 on the card"); + assert_eq!( + moved.host_chains, 0, + "{tag}: the host path counted as a fallback" + ); + } + } + let bytes = rkyv::to_bytes::(&proof) + .unwrap() + .to_vec(); + (bytes, stacked.roots()) +} + +fn every_arm(c: &Case) { + let (host, host_roots) = prove_arm::(c, Arm::Host, None); + for arm in [ + Arm::Defaults, + Arm::ShareForm, + Arm::TileCheck, + Arm::Materialised, + Arm::Lean(4), + Arm::Lean(6), + Arm::Folded, + Arm::Fallback, + ] { + let (card, card_roots) = prove_arm::(c, arm, None); + assert_eq!( + card_roots, + host_roots, + "{} · {} · {arm:?}: roots", + c.name, + H::NAME + ); + assert!( + card == host, + "{} · {} · {arm:?}: the card must prove the host's bytes", + c.name, + H::NAME + ); + } + if c.on_card { + let store = gpu::upload_columns(&multilinear::stacking::borrow(&c.columns)) + .expect("the card took the columns (needs a GPU)"); + for arm in [Arm::Defaults, Arm::Lean(6), Arm::Folded] { + let (card, card_roots) = prove_arm::(c, arm, Some(&store)); + assert_eq!( + card_roots, + host_roots, + "{} · {} · resident {arm:?}: roots", + c.name, + H::NAME + ); + assert!( + card == host, + "{} · {} · resident {arm:?}: the card must prove the host's bytes", + c.name, + H::NAME + ); + } + } + println!( + "GROUPED PARITY {} · {}: {} chains ({} grouped over {} polynomials) · every arm the host's {} bytes", + c.label(), + H::NAME, + c.chains, + c.grouped, + c.grouped_polys, + host.len() + ); +} + +#[test] +fn a_grouped_chain_on_the_card_proves_the_hosts_bytes() { + let cases = cases(); + // Which of the mixed run's polynomials the tile form takes at three lean + // rounds: the first two, not the third. + let mixed = &cases[1]; + let takes: Vec = (0..mixed.layout.num_polys()) + .map(|poly| { + let raw: Vec<(usize, Vec, [u64; 3])> = mixed + .points + .iter() + .zip(mixed.layout.placements()) + .filter(|(_, place)| place.poly == poly) + .map(|(point, place)| { + let limbs = point + .iter() + .flat_map(|z| gpu::ext3_raw(z).unwrap()) + .collect(); + (place.offset, limbs, [1, 0, 0]) + }) + .collect(); + multilinear::lean_tiles::plan(&raw, 16, 3).is_some() + }) + .collect(); + assert_eq!( + takes, + [true, true, false], + "which polynomials the tile form takes" + ); + for c in &cases { + every_arm::(c); + every_arm::(c); + } + // Blowup 2: the same cases, every arm the host's bytes again. + for c in cases.into_iter().map(|c| at_blowup(c, 1)) { + every_arm::(&c); + every_arm::(&c); + } + release(); +} diff --git a/crypto/multilinear/tests/whir_lean_device.rs b/crypto/multilinear/tests/whir_lean_device.rs new file mode 100644 index 000000000..407bea1a6 --- /dev/null +++ b/crypto/multilinear/tests/whir_lean_device.rs @@ -0,0 +1,419 @@ +//! The two memory kernels end to end on the device: a stacked opening whose +//! first rounds run over its shares (the lean opening) and whose codeword +//! folds run every level in one launch proves the SAME bytes as today's +//! device path (factors materialised up front, one fold launch per level), and +//! the host verifier accepts both. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_lean_device +//! ``` +//! +//! Four arms: the path before the kernels (lean rounds 0, folds level by +//! level), six lean rounds with the fused fold (`LAMBDA_VM_WHIR_LEAN_ROUNDS=6`, +//! the way back), three lean rounds and the fused fold with every successor +//! folded (`LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0`, the way back), and the +//! defaults (three lean rounds, the fused fold, the first successor encoded +//! from the bound message instead of folded). Needs a GPU. Each new path is +//! asserted TAKEN in its arms and NOT in the others (the call counters), so a +//! card that declined could not turn this into a comparison of today's path +//! with itself. One `#[test]`: the arms switch process-wide overrides, and +//! parallel tests would race them. +//! +//! The tile form is the lean rounds' default, so the defaults arm runs it; the +//! arms before it force the share form. Three more arms run the lean rounds in +//! TILE form (`LAMBDA_VM_WHIR_LEAN_TILES=1`, `=wide`, `=check`) on a second +//! stack, one the form takes: the same bytes again, every opening counted in +//! tile form, and under `check` every round and every opening's bound tables +//! compared with the share form on the card, none different. On the first +//! stack — a column of eight rows, which the form declines — the same arms +//! prove today's bytes in share form. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{self, Claimed, StackedCommitment, StackedProof}; +use multilinear::stacking::StackedLayout; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, WhirFolds, +}; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(seed.wrapping_mul(6364136223846793005)) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +/// First6 then uniform4, caps on every tree, no grind: a ground proof's nonce +/// is not reproducible across two searches, and the grind is not what moved. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 20, + grind: GrindBits::default(), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + ..ChainFormat::DEFAULT + }, + } +} + +/// Columns of mixed heights in a `2^16` stack — above the device commit and +/// sumcheck thresholds — each claimed at its own point. +struct Case { + layout: StackedLayout, + columns: Vec>, + points: Vec>, + values: Vec, +} + +fn case() -> Case { + let heights = [13usize, 13, 13, 12, 12, 10, 8, 6, 3]; + case_of(&heights, &[0, 1, 2, 3, 4, 5, 6, 7, 8]) +} + +/// A stack the tile form takes: three tables of several columns each (their +/// columns share a point), no column under `2^9` rows. +fn tile_case() -> Case { + let heights = [13usize, 13, 13, 12, 12, 12, 11, 9, 9]; + case_of(&heights, &[0, 0, 0, 1, 1, 1, 2, 3, 3]) +} + +/// Columns of `heights` in one stack of `2^16`, column `i` claimed at the +/// point of table `tables[i]`. +fn case_of(heights: &[usize], tables: &[usize]) -> Case { + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 7 + 31 * i as u64)) + .collect(); + let layout = StackedLayout::build(heights, 16).unwrap(); + assert_eq!(layout.num_polys(), 1); + let points: Vec> = heights + .iter() + .zip(tables) + .map(|(&h, &table)| { + (0..h) + .map(|j| EE::from(101 + 13 * table as u64 + j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, z)| c.evaluate_in(z).unwrap()) + .collect(); + Case { + layout, + columns, + points, + values, + } +} + +/// Which path an arm proves on. +#[derive(Clone, Copy, Debug)] +enum Arm { + /// No lean rounds, every fold level by level: the path before the kernels. + Materialised, + /// Six lean rounds and the fused fold (`LAMBDA_VM_WHIR_LEAN_ROUNDS=6`). + Six, + /// Three lean rounds and the fused fold, every successor folded + /// (`LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0`). + Folded, + /// Nothing forced: the defaults, three lean rounds in tile form (a block a + /// tile), the fused fold and each chain's first successor encoded from its + /// bound message. + Defaults, + /// The defaults with the lean rounds' form forced + /// (`LAMBDA_VM_WHIR_LEAN_TILES`): tile form a thread or a block a tile, or + /// checked against the share form on every round. + Tiles(gpu::LeanTiles), +} + +/// One arm: the path before the kernels or the two kernels. +/// +/// `tiled` says whether the stack is one the tile form takes: its arms then +/// open every chain in tile form, and in share form otherwise. +fn prove_arm( + c: &Case, + arm: Arm, + resident: Option<&gpu::ResidentColumns>, + tiled: bool, +) -> (StackedProof, Vec<[u8; 32]>) { + let lean = !matches!(arm, Arm::Materialised); + gpu::force_encode_successor(match arm { + Arm::Defaults | Arm::Tiles(_) => None, + _ => Some(false), + }); + // The arms before the tile form force the share form; the defaults force + // nothing. + gpu::force_lean_tiles(match arm { + Arm::Tiles(tiles) => Some(tiles), + Arm::Defaults => None, + _ => Some(gpu::LeanTiles::Off), + }); + let default_tiles = std::env::var_os("LAMBDA_VM_WHIR_LEAN_TILES").is_none(); + match arm { + Arm::Materialised => { + gpu::force_lean_rounds(Some(0)); + gpu::force_fused_fold(Some(false)); + } + Arm::Six => { + gpu::force_lean_rounds(Some(6)); + gpu::force_fused_fold(Some(true)); + } + Arm::Folded | Arm::Defaults | Arm::Tiles(_) => { + gpu::force_lean_rounds(None); + gpu::force_fused_fold(None); + if std::env::var_os("LAMBDA_VM_WHIR_LEAN_ROUNDS").is_none() { + assert_eq!( + gpu::lean_rounds_in_effect(), + 3, + "the default is three lean rounds" + ); + } + if matches!(arm, Arm::Defaults | Arm::Tiles(_)) + && std::env::var_os("LAMBDA_VM_WHIR_ENCODE_SUCCESSOR").is_none() + { + assert!( + gpu::encode_successor_in_effect(), + "the default encodes the first successor" + ); + } + if matches!(arm, Arm::Defaults) && default_tiles { + assert_eq!( + gpu::lean_tiles_in_effect(), + "wide", + "the default runs the lean rounds in tile form, a block a tile" + ); + } + } + } + let columns = multilinear::stacking::borrow(&c.columns); + let stacked = StackedCommitment::::commit( + c.layout.clone(), + &columns, + resident.map(|store| (store, 0)), + &config(), + ) + .unwrap(); + let (lean_before, fused_before) = (gpu::lean_open_calls(), gpu::fused_fold_calls()); + let encoded_before = gpu::encoded_successors(); + let (opens_before, folds_before) = (gpu::open_calls(), gpu::resident_fold_calls()); + let (tiles_before, checks_before) = (gpu::lean_tile_opens(), gpu::lean_tile_checks()); + let tables_before = gpu::lean_tile_table_checks(); + let proof = stacked_eval::prove::( + &stacked, + &columns, + resident.map(|store| (store, 0)), + &Claimed::PerColumn(&c.points), + &c.values, + &config(), + &mut DefaultTranscript::::new(b"whir-lean-device"), + ) + .unwrap(); + let (lean_opens, fused_folds) = ( + gpu::lean_open_calls() - lean_before, + gpu::fused_fold_calls() - fused_before, + ); + // The totals the production log reads each share against: both arms open + // and fold on the device, so a share of zero is the path off, never a + // path that did not run. + let (opens, folds) = ( + gpu::open_calls() - opens_before, + gpu::resident_fold_calls() - folds_before, + ); + assert!(opens > 0, "{}: no opening ran on the device", H::NAME); + assert!(folds > 0, "{}: no fold ran on the device", H::NAME); + // Each chain's first successor, and only in its arm: one a chain, and the + // chain's later folds still fold. + assert_eq!( + gpu::encoded_successors() - encoded_before, + if matches!(arm, Arm::Defaults | Arm::Tiles(_)) && gpu::encode_successor_in_effect() { + opens + } else { + 0 + }, + "{} {arm:?}: successors encoded", + H::NAME + ); + // The tile form in its arms and at the defaults on a stack that takes it, + // every chain; never elsewhere. Under `check`, each chain's lean rounds + // compared with the share form, none different. + let tile_opens = gpu::lean_tile_opens() - tiles_before; + let (checked, mismatched) = gpu::lean_tile_checks(); + let checked = (checked - checks_before.0, mismatched - checks_before.1); + let (tables, positions) = gpu::lean_tile_table_checks(); + let tables = (tables - tables_before.0, positions - tables_before.1); + match arm { + Arm::Tiles(tiles) if tiled => { + assert_eq!( + tile_opens, + opens, + "{} {arm:?}: every opening in tile form", + H::NAME + ); + let (rounds, written) = if tiles == gpu::LeanTiles::Check { + (opens * gpu::lean_rounds_in_effect() as u64, opens) + } else { + (0, 0) + }; + assert_eq!( + checked, + (rounds, 0), + "{} {arm:?}: rounds compared, differing", + H::NAME + ); + assert_eq!( + tables, + (written, 0), + "{} {arm:?}: bound tables compared, positions differing", + H::NAME + ); + } + Arm::Defaults if tiled && default_tiles => { + assert_eq!( + tile_opens, + opens, + "{} {arm:?}: every opening in tile form by default", + H::NAME + ); + assert_eq!(checked, (0, 0), "{} {arm:?}: rounds compared", H::NAME); + assert_eq!(tables, (0, 0), "{} {arm:?}: tables compared", H::NAME); + } + // The environment chose the defaults' form: nothing to hold here. + Arm::Defaults if !default_tiles => {} + _ => { + assert_eq!( + tile_opens, + 0, + "{} {arm:?}: an opening in tile form", + H::NAME + ); + assert_eq!(checked, (0, 0), "{} {arm:?}: rounds compared", H::NAME); + assert_eq!(tables, (0, 0), "{} {arm:?}: tables compared", H::NAME); + } + } + if lean { + assert!( + lean_opens > 0, + "{}: the lean opening was not taken", + H::NAME + ); + assert!(fused_folds > 0, "{}: the fused fold was not taken", H::NAME); + assert_eq!( + (lean_opens, fused_folds), + (opens, folds), + "{}: every opening lean and every fold fused", + H::NAME + ); + } else { + assert_eq!( + lean_opens, + 0, + "{}: today's arm took the lean opening", + H::NAME + ); + assert_eq!( + fused_folds, + 0, + "{}: today's arm took the fused fold", + H::NAME + ); + } + stacked_eval::verify::( + &proof, + &c.layout, + &stacked.roots(), + &Claimed::PerColumn(&c.points), + &c.values, + stacked.domain(), + &config(), + &mut DefaultTranscript::::new(b"whir-lean-device"), + ) + .unwrap_or_else(|e| panic!("{}: the host verifier refused the proof: {e:?}", H::NAME)); + (proof, stacked.roots()) +} + +fn both_arms(c: &Case, resident: Option<&gpu::ResidentColumns>, tiled: bool) { + let (today, today_roots) = prove_arm::(c, Arm::Materialised, resident, tiled); + let today_bytes = rkyv::to_bytes::(&today).unwrap(); + for arm in [ + Arm::Six, + Arm::Folded, + Arm::Defaults, + Arm::Tiles(gpu::LeanTiles::On), + Arm::Tiles(gpu::LeanTiles::Wide), + Arm::Tiles(gpu::LeanTiles::Check), + ] { + let (lean, lean_roots) = prove_arm::(c, arm, resident, tiled); + assert_eq!(today_roots, lean_roots, "{} {arm:?}: roots", H::NAME); + assert_eq!( + rkyv::to_bytes::(&lean) + .unwrap() + .as_slice(), + today_bytes.as_slice(), + "{} ({}, {arm:?}): the kernels must prove today's bytes", + H::NAME, + if resident.is_some() { + "resident" + } else { + "parts" + }, + ); + } +} + +#[test] +fn the_memory_kernels_prove_todays_bytes() { + // The first stack has a column of eight rows, which the tile form + // declines; the second is one it takes. + for (c, tiled) in [(case(), false), (tile_case(), true)] { + let raw: Vec<(usize, Vec, [u64; 3])> = c + .points + .iter() + .zip(c.layout.placements()) + .map(|(point, place)| { + let limbs = point + .iter() + .flat_map(|z| gpu::ext3_raw(z).unwrap()) + .collect(); + (place.offset, limbs, [1, 0, 0]) + }) + .collect(); + assert_eq!( + multilinear::lean_tiles::plan(&raw, 16, 3).is_some(), + tiled, + "which stack the tile form takes" + ); + both_arms::(&c, None, tiled); + both_arms::(&c, None, tiled); + let store = gpu::upload_columns(&multilinear::stacking::borrow(&c.columns)) + .expect("the card took the columns (needs a GPU)"); + both_arms::(&c, Some(&store), tiled); + } + gpu::force_lean_rounds(None); + gpu::force_fused_fold(None); + gpu::force_encode_successor(None); + gpu::force_lean_tiles(None); +} diff --git a/crypto/multilinear/tests/whir_p1_device.rs b/crypto/multilinear/tests/whir_p1_device.rs new file mode 100644 index 000000000..bda993c00 --- /dev/null +++ b/crypto/multilinear/tests/whir_p1_device.rs @@ -0,0 +1,184 @@ +//! D-WHIR-P1 S2: ZisK's Poseidon1 WHIR on the card. A chain whose codeword +//! stays on the card (commit, folds, every opening's 4-ary tree rebuilt there) +//! proves the SAME bytes as the chain over a host-held codeword, at every +//! arity-4 cap, odd and even tree depths; and an ext3 codeword's device tree is +//! the host's, paths and caps included. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_p1_device +//! ``` +//! +//! Needs a GPU. The device path is asserted TAKEN, so a card that declined +//! could not turn this into a host-against-host comparison. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::{CapPolicy, cap_len}; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::mle::Mle; +use multilinear::whir::{Domain, encode, lift_coefficients}; +use multilinear::whir_chain::{ + ChainConfig, ChainFormat, GrindBits, RoundOpenings, commit, prove, verify, +}; +use multilinear::whir_commit::{CodewordCommitment, tree_cap_height}; +use multilinear::whir_hash::{P1Whir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; + +fn sponge() -> ::Sponge { + let mut t = P1Whir::sponge(); + t.append_bytes(b"whir-p1-device"); + t +} + +fn run(arity4_cap: CapPolicy, log_folding: usize) { + run_at(arity4_cap, log_folding, 2); +} + +/// [`run`] at blowup `2^log_blowup`. +fn run_at(arity4_cap: CapPolicy, log_folding: usize, log_blowup: usize) { + // A 2^18 codeword (2^16 evaluations at blowup 4, 2^17 at blowup 2), above + // the device commit threshold, so the chain's first tree lives on the card. + let num_vars = 18 - log_blowup; + let cfg = ChainConfig { + log_blowup, + log_folding, + num_queries: 25, + grind: GrindBits::uniform(8), + format: ChainFormat { + arity4_cap, + ..ChainFormat::DEFAULT + }, + }; + let f = Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(6364136223846793005).wrapping_add(17) >> 11)) + .collect(), + ) + .unwrap(); + let z: Vec = (0..num_vars).map(|i| EE::from(301 + i as u64)).collect(); + let y = f.evaluate_in(&z).unwrap(); + let tag = format!("k={log_folding} cap4={arity4_cap} blowup 2^{log_blowup}"); + + let (device, domain) = commit::(&f, &cfg, true).unwrap(); + assert!( + device.codeword().device().is_some(), + "{tag}: the commit must have stayed on the card" + ); + let device_proof = + prove::(&f, &z, &device, &domain, &cfg, &mut sponge()).unwrap(); + + let host_domain = Domain::::new(num_vars + cfg.log_blowup).unwrap(); + let host = CodewordCommitment::::from_codeword_on_host( + encode::(&lift_coefficients(&f), &host_domain).unwrap(), + cfg.schedule(num_vars)[0], + ) + .unwrap(); + assert_eq!(host.root(), device.root(), "{tag}: roots"); + let host_proof = + prove::(&f, &z, &host, &host_domain, &cfg, &mut sponge()).unwrap(); + assert_eq!( + rkyv::to_bytes::(&device_proof) + .unwrap() + .as_slice(), + rkyv::to_bytes::(&host_proof) + .unwrap() + .as_slice(), + "{tag}: the device chain must prove the host chain's bytes" + ); + + // Tree 0's paths: three siblings a 4-ary level below the cap, the owner + // carrying the cap's real nodes. + let caps = cfg.tree_caps_at(num_vars, 4); + let depth0 = num_vars + cfg.log_blowup - cfg.schedule(num_vars)[0]; + assert_eq!(caps[0], tree_cap_height(arity4_cap, 25, depth0, 4)); + let walked = 3 * (depth0.div_ceil(2) - caps[0]); + if let RoundOpenings::Base(p) = &device_proof.rounds[0].openings { + let extra = if caps[0] > 0 { + cap_len(depth0, caps[0], 4).unwrap() + } else { + 0 + }; + assert_eq!( + p.current[0].proof.merkle_path.len(), + walked + extra, + "{tag}" + ); + assert_eq!(p.current[1].proof.merkle_path.len(), walked, "{tag}"); + } else { + panic!("{tag}: round 0 opens base blocks"); + } + + verify::( + &device_proof, + &device.root(), + &z, + y, + &domain, + &cfg, + &mut sponge(), + ) + .unwrap_or_else(|e| panic!("{tag}: the host verifier refused the device proof: {e:?}")); +} + +/// Tree depths 14 (k = 4, even) and 15 (k = 3, odd: its top is two real nodes +/// and two padding digests), uncapped and at 4-ary caps 1 and 2. +#[test] +fn the_device_p1_chain_proves_the_host_chains_bytes() { + for cap in [CapPolicy::Off, CapPolicy::Fixed(1), CapPolicy::Fixed(2)] { + for k in [4usize, 3] { + run(cap, k); + } + } +} + +/// ★ Blowup 2 (`LAMBDA_VM_ZF_WHIR_BLOWUP=2`): the same bytes at the rate's +/// shapes — first folds of 6 (`first6`), 4 and 3, so tree 0's depth is 12, 14 +/// and 15 at 2^17 evaluations, both parities — uncapped and at the base's +/// arity-4 cap of 2. +#[test] +fn the_device_p1_chain_at_blowup_two_proves_the_host_chains_bytes() { + for cap in [CapPolicy::Off, CapPolicy::Fixed(2)] { + for k in [6usize, 4, 3] { + run_at(cap, k, 1); + } + } +} + +/// An ext3 codeword above the threshold: its device tree (the fold rounds' +/// `commit_tree_ext3`) is the host's — root, every queried path, and the +/// owner's cap — at odd and even depths. +#[test] +fn the_device_p1_ext3_tree_is_the_hosts() { + for (log_len, k) in [(18usize, 4usize), (18, 3)] { + let ext: Vec = (0..1u64 << log_len) + .map(|i| { + EE::new([ + FE::from(i.wrapping_mul(0x9E37_79B9) >> 3), + FE::from(i), + FE::from(i ^ 0x55), + ]) + }) + .collect(); + let before = multilinear::gpu::commit_calls(); + let device = CodewordCommitment::::from_codeword(ext.clone(), k).unwrap(); + assert!( + multilinear::gpu::commit_calls() > before, + "log_len={log_len} k={k}: the tree must have been built on the card" + ); + let host = CodewordCommitment::::from_codeword_on_host(ext, k).unwrap(); + assert_eq!(device.root(), host.root(), "log_len={log_len} k={k}"); + let n = host.num_leaves(); + let indices: Vec = (0..20usize).map(|q| (q * 2_654_435_761) % n).collect(); + for c in [0usize, 1, 2] { + assert_eq!( + format!("{:?}", device.open_many_capped(&indices, c, true).unwrap()), + format!("{:?}", host.open_many_capped(&indices, c, true).unwrap()), + "log_len={log_len} k={k} c={c}" + ); + } + } +} diff --git a/crypto/multilinear/tests/whir_room_device.rs b/crypto/multilinear/tests/whir_room_device.rs new file mode 100644 index 000000000..dccdd363b --- /dev/null +++ b/crypto/multilinear/tests/whir_room_device.rs @@ -0,0 +1,440 @@ +//! What a stacked group promises the card, end to end on the device. +//! +//! ```text +//! cargo test --release -p multilinear --features cuda --test whir_room_device -- --test-threads=1 +//! ``` +//! +//! Needs a GPU. Every arm is asserted by the counter of the path it is about, +//! in both directions, so a card that declined could not turn a comparison +//! into one of a path with itself. One `#[test]`: the arms switch process-wide +//! overrides and read the process-wide reservation total, and a parallel test +//! would move both. +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{self, Claimed, StackedCommitment, StackedProof}; +use multilinear::stacking::StackedLayout; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, WhirFolds, +}; +use multilinear::whir_hash::{RpxWhir, WhirHash}; + +type FE = FieldElement; +type EE = FieldElement; +type H = RpxWhir; + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| { + FE::from( + i.wrapping_add(seed) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(seed.wrapping_mul(6364136223846793005)) + >> 11, + ) + }) + .collect(), + ) + .unwrap() +} + +/// The production shape of a chain — first6 then uniform4, caps on every tree +/// — without the grind, whose nonce is not reproducible across two searches. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 20, + grind: GrindBits::default(), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + ..ChainFormat::DEFAULT + }, + } +} + +/// Columns of mixed heights, each claimed at its own point, spilling over +/// `polys` stacked polynomials of `2^n_stack` — above the device commit and +/// sumcheck thresholds. +struct Case { + layout: StackedLayout, + columns: Vec>, + points: Vec>, + values: Vec, +} + +fn case(n_stack: usize, polys: usize) -> Case { + // Each polynomial is filled by a column of `n_stack − 1` variables and the + // halves below it, so the layout is `polys` full stacks. + let mut heights = Vec::new(); + for _ in 0..polys { + heights.extend([n_stack - 1, n_stack - 2, n_stack - 3, n_stack - 4]); + heights.extend([n_stack - 5, n_stack - 6, n_stack - 7, n_stack - 7]); + } + let columns: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| column(h, 7 + 31 * i as u64)) + .collect(); + let layout = StackedLayout::build(&heights, n_stack).unwrap(); + assert_eq!(layout.num_polys(), polys); + let points: Vec> = heights + .iter() + .enumerate() + .map(|(i, &h)| { + (0..h) + .map(|j| EE::from(101 + 13 * i as u64 + j as u64)) + .collect() + }) + .collect(); + let values = columns + .iter() + .zip(&points) + .map(|(c, z)| c.evaluate_in(z).unwrap()) + .collect(); + Case { + layout, + columns, + points, + values, + } +} + +fn commit(c: &Case, resident: Option<&gpu::ResidentColumns>) -> StackedCommitment { + let columns = multilinear::stacking::borrow(&c.columns); + StackedCommitment::::commit( + c.layout.clone(), + &columns, + resident.map(|store| (store, 0)), + &config(), + ) + .unwrap() +} + +/// Opens `stacked` and checks the host verifier takes it. +fn open( + c: &Case, + stacked: &StackedCommitment, + resident: Option<&gpu::ResidentColumns>, +) -> StackedProof { + let columns = multilinear::stacking::borrow(&c.columns); + let proof = stacked_eval::prove::( + stacked, + &columns, + resident.map(|store| (store, 0)), + &Claimed::PerColumn(&c.points), + &c.values, + &config(), + &mut DefaultTranscript::::Transcript>::new(b"whir-room-device"), + ) + .unwrap(); + stacked_eval::verify::( + &proof, + &c.layout, + &stacked.roots(), + &Claimed::PerColumn(&c.points), + &c.values, + stacked.domain(), + &config(), + &mut DefaultTranscript::::Transcript>::new(b"whir-room-device"), + ) + .unwrap_or_else(|e| panic!("the host verifier refused the proof: {e:?}")); + proof +} + +fn bytes(proof: &StackedProof) -> Vec { + rkyv::to_bytes::(proof) + .unwrap() + .to_vec() +} + +/// Bytes promised across the process — what every `reserve` is checked against. +fn reserved() -> u64 { + math_cuda::device::backend() + .expect("needs a GPU") + .reserved_bytes() +} + +/// The room a group promised before any of this: one base codeword. +fn codeword_bytes(c: &Case) -> u64 { + (1u64 << (c.layout.n_stack() + config().log_blowup)) * 8 +} + +/// Counts of the room's three events since `from`. +fn room_counts(from: (u64, u64, u64)) -> (u64, u64, u64) { + ( + gpu::room_parks() - from.0, + gpu::room_turns() - from.1, + gpu::room_turn_refusals() - from.2, + ) +} + +fn room_marks() -> (u64, u64, u64) { + ( + gpu::room_parks(), + gpu::room_turns(), + gpu::room_turn_refusals(), + ) +} + +/// ★ The room goes back when the commits end and comes back for the openings. +/// +/// Both arms commit and open the same group; the park arm must hold EXACTLY +/// one room less than the held arm after its commits and after its openings, +/// give it back once and take it back once, and prove the same bytes. Every +/// other term — the codewords, the leaf layers they retain — is the same in +/// both, so the difference is the room and nothing else. +fn the_room_is_given_back_between_the_commits_and_the_openings(c: &Case, room: u64) { + let arm = |park: bool| { + gpu::force_room_park(Some(park)); + let marks = room_marks(); + let before = reserved(); + let stacked = commit(c, None); + let committed = reserved() - before; + let proof = open(c, &stacked, None); + let opened = reserved() - before; + let counts = room_counts(marks); + drop(stacked); + assert_eq!( + reserved(), + before, + "park {park}: the group gave back less than it promised" + ); + // The openings leave nothing promised behind them: every chain's + // folds, and the leaf layers those kept, went back with the chain. + assert_eq!( + opened, + committed, + "park {park}: the openings left {} B promised past their end", + opened.saturating_sub(committed) + ); + (bytes(&proof), committed, opened, counts) + }; + let (held, held_committed, held_opened, held_counts) = arm(false); + let (parked, park_committed, park_opened, park_counts) = arm(true); + gpu::force_room_park(None); + assert_eq!( + held_counts, + (0, 0, 0), + "the held arm gave its room back or took a turn" + ); + assert_eq!( + park_counts, + (1, 1, 0), + "the park arm must give its room back once and take it back once, unrefused" + ); + assert_eq!( + held_committed - park_committed, + room, + "between the commits and the openings the park arm must hold exactly one room less" + ); + assert_eq!( + held_opened - park_opened, + room, + "after the openings the park arm must have given its turn back" + ); + assert_eq!( + parked, held, + "where the room sits must not move a byte of the proof" + ); +} + +/// ⛔ A refused turn is counted, and the openings run on the device anyway — no +/// host fallback of either kind — proving the bytes a granted turn proves. +fn a_refused_turn_is_counted_and_still_opens_on_the_device(c: &Case) { + gpu::force_room_park(Some(true)); + let stacked = commit(c, None); + let granted = open(c, &stacked, None); + + // Empty the retention first — `reserve` evicts it on a miss, and a layer + // given back would let the turn through — then promise every byte the + // budget has left to nothing. + assert!(math_cuda::device::reserve(u64::MAX / 4).is_none()); + let be = math_cuda::device::backend().expect("needs a GPU"); + let filler = + math_cuda::device::reserve(be.vram_budget_bytes().saturating_sub(be.reserved_bytes())); + + let marks = room_marks(); + let (open_fallbacks, commit_fallbacks, lean) = ( + gpu::open_host_fallbacks(), + gpu::host_fallbacks(), + gpu::lean_open_calls(), + ); + let refused = open(c, &stacked, None); + let counts = room_counts(marks); + drop(filler); + gpu::force_room_park(None); + + assert_eq!( + counts, + (0, 0, 1), + "a full budget must refuse the turn, and count it" + ); + assert_eq!( + gpu::open_host_fallbacks() - open_fallbacks, + 0, + "a refused turn must not send the factors to the host" + ); + assert_eq!(gpu::host_fallbacks() - commit_fallbacks, 0); + assert_eq!( + gpu::lean_open_calls() - lean, + c.layout.num_polys() as u64, + "the refused group's openings must still have run on the device" + ); + assert_eq!( + bytes(&refused), + bytes(&granted), + "a refused turn must prove the bytes a granted one does" + ); +} + +/// An opening whose device factors decline over a codeword the device holds +/// builds them on the host, proves the same bytes, and is COUNTED — once per +/// chain, and never on the path that did not decline. +fn a_declined_opening_is_counted(c: &Case) { + let stacked = commit(c, None); + let polys = c.layout.num_polys() as u64; + + let before = gpu::open_host_fallbacks(); + let on_device = open(c, &stacked, None); + assert_eq!( + gpu::open_host_fallbacks() - before, + 0, + "an opening the device built counted as a host fallback" + ); + + gpu::force_shared_open_declined(true); + let before = gpu::open_host_fallbacks(); + let declined = open(c, &stacked, None); + gpu::force_shared_open_declined(false); + assert_eq!( + gpu::open_host_fallbacks() - before, + polys, + "every chain whose factors the device declined must be counted, once" + ); + assert_eq!( + bytes(&declined), + bytes(&on_device), + "the host-built factors must prove the device's bytes" + ); +} + +/// What `f` took from the device's memory pool at its peak, beyond what the +/// pool had handed out when it started. +fn peak_of(f: impl FnOnce() -> R) -> (R, u64) { + math_cuda::device::drain_and_trim().unwrap(); + let (before, _) = math_cuda::device::pool_used_bytes().unwrap(); + math_cuda::device::reset_pool_high_water().unwrap(); + let out = f(); + math_cuda::device::drain_and_trim().unwrap(); + let (_, high) = math_cuda::device::pool_used_bytes().unwrap(); + (out, high.max(before) - before) +} + +const MIB: f64 = 1024.0 * 1024.0; + +/// ★ THE RESIZE, MEASURED: a group's commits and its openings take no more from +/// the card than the rooms the resize promises for them — and those rooms are +/// smaller than the codeword the group used to promise. +/// +/// Read off the memory pool's own high-water mark, which sees every +/// allocation the turns make, not only the ones somebody reserved. The commits +/// leave their codewords and the leaf layers they retain behind, so theirs is +/// checked with those added; the openings leave nothing. Each bound is checked +/// from below too — by the one buffer the phase cannot avoid — so a probe that +/// read nothing could not pass. +fn a_groups_turns_hold_what_their_rooms_promise() { + let c = case(22, 2); + let polys = c.layout.num_polys() as u64; + let codeword = codeword_bytes(&c); + gpu::force_room_park(Some(true)); + gpu::force_room_resize(Some(true)); + let turns = gpu::room_turns_for(&c.layout, &config(), true); + + // The transform's twiddles are cached for the process the first time a + // domain is used, outside every promise: one commit first, so the + // measured one does not pay for them. + drop(commit(&c, None)); + + let (stacked, committed) = peak_of(|| commit(&c, None)); + let (_, opened) = peak_of(|| open(&c, &stacked, None)); + drop(stacked); + gpu::force_room_park(None); + gpu::force_room_resize(None); + + // At the production first fold of six a leaf layer is a sixteenth of the + // codeword, and the commits keep one each. + let layers = polys * codeword / 16; + let slack = |bytes: u64| bytes / 20 + (8 << 20); + println!( + "2^22 x{polys}: commit peak {:.1} MiB = {polys} codewords + layers + {:.1} MiB against a \ + {:.1} MiB turn; opening peak {:.1} MiB against a {:.1} MiB turn; codeword {:.1} MiB", + committed as f64 / MIB, + committed.saturating_sub(polys * codeword + layers) as f64 / MIB, + turns.commit as f64 / MIB, + opened as f64 / MIB, + turns.open as f64 / MIB, + codeword as f64 / MIB, + ); + assert!( + turns.commit < codeword && turns.open < codeword, + "the resized turns must be smaller than the codeword they replace" + ); + assert!( + committed <= polys * codeword + layers + turns.commit + slack(turns.commit), + "the commits took more than their codewords, their layers and their turn" + ); + assert!( + committed >= polys * codeword + (8 << c.layout.n_stack()), + "the probe missed the commits' coefficients beside their codewords" + ); + assert!( + opened <= turns.open + slack(turns.open), + "an opening took more than its turn" + ); + assert!( + opened >= math_cuda::whir::tree_bytes(c.layout.n_stack() + config().log_blowup - 6), + "the probe missed the tree the first round's paths are read from" + ); +} + +/// Rooms sized, and of those sized to their turn, since `from`. +fn sizings_since(from: (u64, u64)) -> (u64, u64) { + ( + gpu::room_sizings() - from.0, + gpu::rooms_turn_sized() - from.1, + ) +} + +#[test] +fn a_group_promises_the_card_what_its_turns_take() { + let c = case(16, 2); + a_declined_opening_is_counted(&c); + gpu::force_room_resize(Some(false)); + let from = (gpu::room_sizings(), gpu::rooms_turn_sized()); + the_room_is_given_back_between_the_commits_and_the_openings(&c, codeword_bytes(&c)); + let (sized, turn_sized) = sizings_since(from); + assert!( + sized > 0 && turn_sized == 0, + "resize off: {sized} rooms sized, {turn_sized} to their turn (want some, and none)" + ); + gpu::force_room_resize(Some(true)); + let turns = gpu::room_turns_for(&c.layout, &config(), true); + let from = (gpu::room_sizings(), gpu::rooms_turn_sized()); + the_room_is_given_back_between_the_commits_and_the_openings(&c, turns.commit.max(turns.open)); + let (sized, turn_sized) = sizings_since(from); + assert!( + sized > 0 && turn_sized == sized, + "resize on: {sized} rooms sized, {turn_sized} to their turn (want all)" + ); + a_refused_turn_is_counted_and_still_opens_on_the_device(&c); + gpu::force_room_resize(None); + a_groups_turns_hold_what_their_rooms_promise(); +} diff --git a/crypto/stark/Cargo.toml b/crypto/stark/Cargo.toml index c497949ed..b1017a277 100644 --- a/crypto/stark/Cargo.toml +++ b/crypto/stark/Cargo.toml @@ -15,6 +15,7 @@ math = { path = "../math", features = [ "rkyv", ] } crypto = { path = "../crypto", features = ["std", "serde", "rkyv"] } +multilinear = { path = "../multilinear" } thiserror = "1.0.38" log = "0.4.17" digest = "0.10.7" @@ -26,10 +27,6 @@ rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "byt # Parallelization crates rayon = { version = "1.8.0", optional = true } -memmap2 = { version = "0.9", optional = true } -tempfile = { version = "3", optional = true } -libc = { version = "0.2", optional = true } - # GPU backend for trace LDE — only linked when `cuda` is enabled. math-cuda = { path = "../math-cuda", optional = true } @@ -47,6 +44,9 @@ test-log = { version = "0.2.11", features = ["log"] } bincode = "1" rand = { version = "0.8.5", features = ["std"] } rand_chacha = "0.3.1" +# Digests of the default-format golden proofs (tests::zf_golden_tests); the +# version the `crypto` crate already links. +sha3 = "0.10.8" [features] test-utils = [] @@ -55,14 +55,13 @@ instruments = [] # This enab profile-markers = [] # Emits inlining-immune asm markers for guest step profiling debug-checks = [] # Enables validate_trace + bus balance report in prover parallel = ["dep:rayon", "crypto/parallel"] -cuda = ["dep:math-cuda"] +cuda = ["dep:math-cuda", "crypto/cuda"] test-cuda-faults = ["cuda", "math-cuda/test-faults"] # NVTX ranges for Nsight Systems: every instruments span (prover phases, # per-epoch marks) becomes a named timeline range. Pulls in `instruments` # so the span tree exists to mirror, and `cuda` for math-cuda's bindings. nvtx = ["cuda", "instruments", "math-cuda/nvtx"] wasm = ["dep:wasm-bindgen", "dep:serde-wasm-bindgen", "dep:web-sys"] -disk-spill = ["dep:memmap2", "dep:tempfile", "dep:libc", "crypto/disk-spill"] [package.metadata.wasm-pack.profile.dev] diff --git a/crypto/stark/benches/profile_prover.rs b/crypto/stark/benches/profile_prover.rs index f5438877e..a91de425b 100644 --- a/crypto/stark/benches/profile_prover.rs +++ b/crypto/stark/benches/profile_prover.rs @@ -22,6 +22,7 @@ fn main() { coset_offset: 3, grinding_factor: 0, fri_final_poly_log_degree: 7, + format: stark::proof::options::ProofFormat::DEFAULT, }; let num_columns = 16; diff --git a/crypto/stark/benches/prover_benchmark.rs b/crypto/stark/benches/prover_benchmark.rs index c152e7dbb..087019cc2 100644 --- a/crypto/stark/benches/prover_benchmark.rs +++ b/crypto/stark/benches/prover_benchmark.rs @@ -62,6 +62,7 @@ fn benchmark_proof_options() -> ProofOptions { coset_offset: 3, grinding_factor: 0, fri_final_poly_log_degree: 7, + format: stark::proof::options::ProofFormat::DEFAULT, } } diff --git a/crypto/stark/examples/examples_cli.rs b/crypto/stark/examples/examples_cli.rs index d8d20528b..3d5e87fa4 100644 --- a/crypto/stark/examples/examples_cli.rs +++ b/crypto/stark/examples/examples_cli.rs @@ -574,8 +574,7 @@ fn prove_multi_table_lookup() -> Result, String> { let multi_proof = Prover::::multi_prove( air_trace_pairs, &mut DefaultTranscript::::new(&[]), - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| format!("prove failed: {e:?}"))?; ser(&multi_proof) diff --git a/crypto/stark/src/commit_ood.rs b/crypto/stark/src/commit_ood.rs new file mode 100644 index 000000000..763bbf563 --- /dev/null +++ b/crypto/stark/src/commit_ood.rs @@ -0,0 +1,286 @@ +//! Commitment-time out-of-domain answers for a multi-table proof. +//! +//! In a multi-table proof every table's main trace is committed, then the +//! LogUp challenges `(z, α)` shared by all tables are drawn, then each table +//! runs its own DEEP/FRI on its own fork. A committed main trace is close to +//! several codewords at once (a list, in the Johnson regime), and without +//! anything absorbed before `(z, α)` but roots, a prover could pick each +//! table's codeword after seeing `(z, α)`: the one bus equation over all +//! tables would then range over the product of the lists. +//! +//! The bound entry points (`multi_prove_bound`, `multi_verify_views_bound`) +//! close that: after the last root, one point `z0` is drawn outside every +//! table's domains ([`draw_point`]); every table answers `t_c(z0)` for each +//! main column `c` that is not precomputed ([`answered_columns`]), every +//! answer is absorbed in the main transcript ([`absorb_answers`]), and only +//! then are `(z, α)` drawn. Two distinct codewords of one table differ in +//! some answered column, so they disagree at `z0` except with probability +//! `< n/|E|`: from `(z, α)` on, each table has one named codeword. +//! +//! Precomputed columns answer nothing: their root is the verifier's +//! constant, so the committed word is the codeword itself. Aux columns and +//! composition parts answer nothing: they are committed on the table's own +//! fork, after `(z, α)`, where every later challenge is per table. +//! +//! Each table's DEEP polynomial carries the answers inside each answered +//! column's existing quotient at `z`: with `c_c` the column's current-row +//! coefficient, its term becomes +//! +//! ```text +//! c_c·[(t_c(X) − t_c(z))/(X − z) − (t_c(X) − v_c)/(X − z0)] +//! = c_c·(z − z0)·(t_c(X) − I_c(X))/((X − z)(X − z0)), +//! ``` +//! +//! `I_c` the line through `(z, t_c(z))` and `(z0, v_c)`: one two-point +//! quotient per column, under the coefficient it already had. The batch keeps +//! its number of terms and its powers of `γ`; on the grid the answers' pole +//! takes `−c_c` ([`answer_coefficient`]), and a verifier folds the answered +//! columns' run of the current row once for both points. + +use core::ops::Range; + +use crypto::fiat_shamir::is_transcript::IsStarkTranscript; +use math::field::element::FieldElement; +use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; + +use crate::traits::AIR; + +/// One table's answers as its own rounds 2 to 4 take them. +#[derive(Clone, Copy, Debug)] +pub struct CommitOodTable<'a, E: IsField> { + /// The shared point `z0`. + pub point: &'a FieldElement, + /// The table's answers, one per [`answered_columns`] entry, in order. + pub answers: &'a [FieldElement], +} + +/// The main columns of `air` that answer at `z0`: every main column after +/// the precomputed ones. A verifier constant, read from the AIR alone. +pub fn answered_columns( + air: &dyn AIR, +) -> Range +where + F: IsFFTField + IsSubFieldOf + Send + Sync, + E: IsField + Send + Sync, +{ + let (main, _) = air.trace_layout(); + let precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + precomputed.min(main)..main +} + +/// The domain `z0` must avoid: the largest trace domain and the largest LDE +/// coset over the tables, `(trace_length, lde_length, coset_offset)`. Every +/// table's trace domain is a subgroup of the largest and every LDE coset (one +/// offset for all) a subset of the largest, so a point outside these two is +/// outside every table's. `None` when the tables do not share a coset offset +/// or there are none. +pub fn point_domain( + tables: impl IntoIterator, +) -> Option<(usize, usize, u64)> { + let mut out: Option<(usize, usize, u64)> = None; + for (trace_length, blowup, offset) in tables { + let lde = trace_length.checked_mul(blowup)?; + out = Some(match out { + None => (trace_length, lde, offset), + Some((t, l, o)) if o == offset => (t.max(trace_length), l.max(lde), o), + Some(_) => return None, + }); + } + out +} + +/// Draw `z0` from the main transcript, outside `domain` ([`point_domain`]). +pub fn draw_point( + transcript: &mut impl IsStarkTranscript, + domain: (usize, usize, u64), +) -> FieldElement +where + F: IsFFTField + IsSubFieldOf, + E: IsField, +{ + let (trace_length, lde_length, offset) = domain; + transcript.sample_z_ood_with_domain_params( + trace_length, + lde_length, + &FieldElement::::from(offset), + ) +} + +/// Absorb every table's answers, table order then column order: the step +/// between [`draw_point`] and the shared LogUp draws. +pub fn absorb_answers( + transcript: &mut impl IsStarkTranscript, + answers: &[Vec>], +) where + F: IsField + IsSubFieldOf, + E: IsField, +{ + for table in answers { + for answer in table { + transcript.append_field_element(answer); + } + } +} + +/// An answered column's coefficient at the answers' pole `z0`: minus its +/// coefficient at `z` (the current row's), so the two quotients form the +/// column's one two-point quotient. +pub fn answer_coefficient(current_row: &FieldElement) -> FieldElement { + FieldElement::::zero() - current_row +} + +/// Whether `z0` is a point a table's DEEP polynomial already has a quotient +/// at: `z·gʳ` for each of the `rows` rows of its OOD grid (`g` the trace +/// domain's generator), or `z^parts`, where its composition parts open. At +/// such a point a false answer's quotient could cancel against that one +/// (at `z0 = z` the column's two-point quotient degenerates), so a verifier +/// refuses the proof instead. +pub fn meets_deep_point( + z0: &FieldElement, + z: &FieldElement, + g: &FieldElement, + rows: usize, + parts: usize, +) -> bool +where + F: IsField + IsSubFieldOf, + E: IsField, +{ + let mut row_point = z.clone(); + for _ in 0..rows { + if &row_point == z0 { + return true; + } + row_point = g * &row_point; + } + &z.pow(parts) == z0 +} + +/// `t_c(point)` for each column `c` in `columns` of a row-major trace +/// (`width` columns) over the trace domain `⟨ω⟩` of `rows` rows, by the +/// barycentric formula `t(z) = (zⁿ − 1)/n · Σ_k t_k·ωᵏ/(z − ωᵏ)`. +/// +/// `None` when `point` lies in the domain (a zero denominator). +pub fn evaluate_trace_columns( + data: &[FieldElement], + width: usize, + columns: Range, + point: &FieldElement, +) -> Option>> +where + F: IsFFTField + IsSubFieldOf + Send + Sync, + E: IsField + Send + Sync, + FieldElement: Send + Sync, + FieldElement: Send + Sync, +{ + if columns.is_empty() { + return Some(Vec::new()); + } + if width == 0 || !data.len().is_multiple_of(width) { + return None; + } + let rows = data.len() / width; + if !rows.is_power_of_two() { + return None; + } + let omega = F::get_primitive_root_of_unity(rows.trailing_zeros() as u64).ok()?; + let mut powers = Vec::with_capacity(rows); + let mut w = FieldElement::::one(); + for _ in 0..rows { + powers.push(w.clone()); + w = &w * ω + } + let mut weights: Vec> = powers + .iter() + .map(|w| point - w.clone().to_extension::()) + .collect(); + FieldElement::inplace_batch_inverse(&mut weights).ok()?; + for (weight, w) in weights.iter_mut().zip(&powers) { + *weight = w * &*weight; + } + let n_inv = FieldElement::::from(rows as u64).inv().ok()?; + let factor = (point.pow(rows) - FieldElement::::one()) * n_inv.to_extension::(); + Some(crate::par::par_map_collect(columns, |c| { + let mut sum = FieldElement::::zero(); + for (k, weight) in weights.iter().enumerate() { + sum += &data[k * width + c] * weight; + } + &sum * &factor + })) +} + +/// Faults for the negative tests: what the prover sends and what both sides +/// fold, switched per thread around a prove and a verify. Never reachable +/// outside a test build. +#[cfg(any(test, feature = "test-utils"))] +pub mod faults { + use std::cell::Cell; + + /// One fault set. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub struct Faults { + /// The prover adds one to answer `index` of table `table` before it is + /// absorbed, and carries the false value into that table's DEEP. + pub false_answer: Option<(usize, usize)>, + /// Prover and verifier leave the `z0` quotients out of DEEP: the + /// answers are still sent and absorbed, but nothing checks them. + pub without_deep_term: bool, + /// The verifier's check that `z0` avoids every DEEP point sees `z` + /// in place of `z0`, as if the two draws had met. + pub z0_meets_z: bool, + } + + thread_local! { + static FAULTS: Cell = const { + Cell::new(Faults { + false_answer: None, + without_deep_term: false, + z0_meets_z: false, + }) + }; + } + + /// The faults set on this thread. + pub fn current() -> Faults { + FAULTS.with(|f| f.get()) + } + + /// Run `body` with `faults` set on this thread, restoring the previous + /// set afterwards. + pub fn with(faults: Faults, body: impl FnOnce() -> R) -> R { + let previous = FAULTS.with(|f| f.replace(faults)); + let out = body(); + FAULTS.with(|f| f.set(previous)); + out + } +} + +/// The `z0` the verifier's DEEP-point check sees: `z0` itself, outside a +/// test fault. +pub(crate) fn z0_for_point_check<'a, E: IsField>( + z0: &'a FieldElement, + z: &'a FieldElement, +) -> &'a FieldElement { + #[cfg(any(test, feature = "test-utils"))] + if faults::current().z0_meets_z { + return z; + } + let _ = z; + z0 +} + +/// Whether DEEP folds the `z0` quotients: always, outside a test fault. +pub(crate) fn deep_term_on() -> bool { + #[cfg(any(test, feature = "test-utils"))] + { + !faults::current().without_deep_term + } + #[cfg(not(any(test, feature = "test-utils")))] + { + true + } +} diff --git a/crypto/stark/src/commitment.rs b/crypto/stark/src/commitment.rs index d4a6dbdbe..9a4a1925b 100644 --- a/crypto/stark/src/commitment.rs +++ b/crypto/stark/src/commitment.rs @@ -33,7 +33,22 @@ use math::traits::{AsBytes, ByteConversion}; #[cfg(feature = "parallel")] use rayon::prelude::{IntoParallelIterator, ParallelIterator}; -use crate::config::{BatchedMerkleTree, BatchedMerkleTreeBackend, Commitment}; +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsStreamingLeafBackend; + +use crate::config::{ + BatchedMerkleTree, BatchedMerkleTreeBackend, Commitment, KeccakStarkHash, StarkHash, +}; + +/// The keccak batched leaf backend, named rather than aliased. +/// +/// The `keccak_leaves_*` helpers below exist to BE the keccak reference the +/// CUDA parity tests compare their kernels against. They used to reach it +/// through [`BatchedMerkleTreeBackend`], which was the same type while keccak +/// was the default — and stopped being at the P-a flip, at which point three +/// functions with `keccak` in their names would have computed BLAKE3 and the +/// GPU parity tests would have reported a hash change as a kernel bug. +type KeccakBatched = ::Batched; /// Number of consecutive (bit-reversed) rows packed into one Merkle leaf for the /// trace AND composition-polynomial commitments: the row-pair leaf the FRI @@ -49,12 +64,13 @@ pub const ROWS_PER_LEAF: usize = 2; /// exact leaf byte layout. This is the single code path behind both the per-row /// ([`keccak_leaves_bit_reversed`]) and per-row-pair /// ([`keccak_leaves_row_pair_bit_reversed`]) commitments. -pub fn keccak_leaves_bit_reversed_grouped( +pub fn leaves_bit_reversed_grouped( columns: &[Vec>], rows_per_leaf: usize, ) -> Vec where E: IsField, + B: IsStreamingLeafBackend, FieldElement: AsBytes + Sync + Send + ByteConversion, { if columns.is_empty() || columns[0].is_empty() { @@ -87,7 +103,7 @@ where offset += byte_len; } } - BatchedMerkleTreeBackend::::hash_bytes(buf) + >::hash_bytes(buf) }; // Per-thread buffer reuse (map_init) avoids millions of small allocations. @@ -106,6 +122,24 @@ where result } +/// [`leaves_bit_reversed_grouped`] at the keccak backend — the leaf hash the +/// CUDA keccak kernels and their parity tests mirror. +/// +/// ⚠ Since the P-a flip this is no longer the *default* leaf hash; it is the +/// keccak one, which is what the keccak kernels compute and therefore what +/// their parity tests need. For the production leaf hash of the current +/// configuration, use [`commit_bit_reversed`] or the alias directly. +pub fn keccak_leaves_bit_reversed_grouped( + columns: &[Vec>], + rows_per_leaf: usize, +) -> Vec +where + E: IsField + 'static, + FieldElement: AsBytes + Sync + Send + ByteConversion, +{ + leaves_bit_reversed_grouped::>(columns, rows_per_leaf) +} + /// Per-row Keccak-256 leaf hashes (one leaf per bit-reversed row). Thin wrapper /// over [`keccak_leaves_bit_reversed_grouped`] with `rows_per_leaf = 1`. /// @@ -114,10 +148,10 @@ where /// the GPU parity tests in dependent crates can compare the per-row code path. pub fn keccak_leaves_bit_reversed(columns: &[Vec>]) -> Vec where - E: IsField, + E: IsField + 'static, FieldElement: AsBytes + Sync + Send + ByteConversion, { - keccak_leaves_bit_reversed_grouped(columns, 1) + leaves_bit_reversed_grouped::>(columns, 1) } /// Per-row-pair Keccak-256 leaf hashes (leaf `i` hashes bit-reversed rows `2i`, @@ -125,10 +159,10 @@ where /// over [`keccak_leaves_bit_reversed_grouped`] with `rows_per_leaf = 2`. pub fn keccak_leaves_row_pair_bit_reversed(parts: &[Vec>]) -> Vec where - E: IsField, + E: IsField + 'static, FieldElement: AsBytes + Sync + Send + ByteConversion, { - keccak_leaves_bit_reversed_grouped(parts, 2) + leaves_bit_reversed_grouped::>(parts, 2) } /// Builds the Merkle tree committing to `columns`' bit-reversed, column-major LDE @@ -144,12 +178,30 @@ pub fn commit_bit_reversed( where E: IsField, FieldElement: AsBytes + Sync + Send + ByteConversion, +{ + commit_bit_reversed_with::>(columns, rows_per_leaf) +} + +/// [`commit_bit_reversed`] under an explicit leaf backend. +/// +/// The backend is a parameter rather than the fixed alias because the prover is +/// generic over its commitment configuration; `commit_bit_reversed` is this +/// function at the keccak backend, and is what every caller that commits a +/// fixed production table still uses. +pub fn commit_bit_reversed_with( + columns: &[Vec>], + rows_per_leaf: usize, +) -> Option<(MerkleTree, Commitment)> +where + E: IsField, + B: IsStreamingLeafBackend, + FieldElement: AsBytes + Sync + Send + ByteConversion, { if columns.is_empty() || columns[0].is_empty() { return None; } - let hashed_leaves = keccak_leaves_bit_reversed_grouped(columns, rows_per_leaf); - let tree = BatchedMerkleTree::::build_from_hashed_leaves(hashed_leaves)?; + let hashed_leaves = leaves_bit_reversed_grouped::(columns, rows_per_leaf); + let tree = MerkleTree::::build_from_hashed_leaves(hashed_leaves)?; let root = tree.root; Some((tree, root)) } diff --git a/crypto/stark/src/config.rs b/crypto/stark/src/config.rs index 50650e40a..951078179 100644 --- a/crypto/stark/src/config.rs +++ b/crypto/stark/src/config.rs @@ -1,24 +1,642 @@ +use crypto::fiat_shamir::p1_transcript::P1TranscriptHash; +use crypto::fiat_shamir::transcript_hash::{ + Blake3TranscriptHash, KeccakTranscriptHash, TranscriptHash, +}; +use crypto::merkle_tree::backends::p1::{P1BatchBackend, P1PairBackend}; +use crypto::merkle_tree::backends::types::{BatchBlake3Backend, PairBlake3Backend}; +use crypto::merkle_tree::cap::CapPolicy; use crypto::merkle_tree::{ - backends::types::{BatchKeccak256Backend, Keccak256Backend, PairKeccak256Backend}, + backends::types::{BatchKeccak256Backend, PairKeccak256Backend}, merkle::MerkleTree, + traits::{IsMerkleTreeBackend, IsStreamingLeafBackend}, }; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsField; +use math::traits::AsBytes; // Merkle Trees configuration // Security of both hashes should match -pub type FriMerkleTreeBackend = Keccak256Backend; -pub type FriMerkleTree = MerkleTree>; - // If using hashes with 256-bit security, commitment size should be 32 // If using hashes with 512-bit security, commitment size should be 64 // TODO: Commitment type should be obtained from MerkleTrees pub const COMMITMENT_SIZE: usize = 32; pub type Commitment = [u8; COMMITMENT_SIZE]; -pub type BatchedMerkleTreeBackend = BatchKeccak256Backend; +/// The default commitment configuration — what every unparameterized `Prover`, +/// `Verifier` and Merkle alias in this crate resolves to — on every build, +/// `cuda` included. +/// +/// A cuda build honours it because `gpu_lde`'s tree entry points dispatch on +/// the configuration: every backend they accept is a [`DeviceTreeBackend`], +/// whose `COMMITMENT_HASH` constant selects the device kernel family +/// (keccak or the Track G BLAKE3 kernels) at each leaf, level and FRI-layer +/// launch. The transcript and grinding follow the same configuration through +/// [`StarkHash::Transcript`], and the pairing asserts at the bottom of this +/// file keep a configuration's commitment and Fiat-Shamir hashes from ever +/// being edited apart. +pub type DefaultStarkHash = Blake3StarkHash; + +// Spelled as the concrete backends rather than as `::Batched`: the associated types carry `F: 'static`, which the +// alias would propagate into every caller that is merely generic over a field. +// The assertion at the bottom of this file is what keeps the two spellings the +// same type. +pub type BatchedMerkleTreeBackend = BatchBlake3Backend; pub type BatchedMerkleTree = MerkleTree>; +/// The keccak tree families, named rather than reached through the aliases. +/// +/// The CUDA keccak kernels compute keccak whatever the host default is, so +/// their parity tests need a CPU reference that says keccak rather than one +/// that says "whatever is default". While keccak WAS the default the aliases +/// served both purposes; the flip separated them, and a parity test left on an +/// alias would report a hash change as a kernel bug. +pub type KeccakBatchedMerkleTreeBackend = BatchKeccak256Backend; +pub type KeccakFriLayerMerkleTreeBackend = PairKeccak256Backend; +pub type KeccakFriLayerMerkleTree = MerkleTree>; + // FRI layer uses fixed-size pairs for efficiency (avoids Vec allocation per pair) -pub type FriLayerMerkleTreeBackend = PairKeccak256Backend; +pub type FriLayerMerkleTreeBackend = PairBlake3Backend; pub type FriLayerMerkleTree = MerkleTree>; + +/// A Merkle backend the GPU tree entry points can honour, naming its hash. +/// +/// `IsMerkleTreeBackend` is too weak a bound wherever the +/// backend does not actually do the hashing. The GPU tree entry points in +/// `gpu_lde` are exactly that case — they hash on the device and use `B` only +/// as the label on the host `MerkleTree` the roots are wrapped in, so any +/// 32-byte-node backend would compile there and hand back trees wearing a +/// name whose hash the kernels never computed. +/// +/// This trait closes that hole: `COMMITMENT_HASH` is the dispatch key +/// `gpu_lde` hands to `math-cuda`, which selects the kernel family at every +/// leaf, level and FRI-layer launch — or, for a hash whose kernels are not yet +/// ported (the algebraic three), aborts at that launch with `unimplemented!` +/// naming the hash. Either way a tree labelled `Self` was hashed by `Self`'s +/// kernels or was not built at all; what the trait rules out is the third +/// outcome, a tree built by another hash's kernels and labelled `Self`. +/// +/// Implementing it is therefore the reviewable statement of WHICH hash the +/// device must compute for this backend, not a claim that it already can. A +/// backend whose hash no [`CommitmentHash`] variant names has no true constant +/// to supply, so writing the impl is a deliberate false statement rather than +/// an omission nobody had to make. The algebraic impls live beside their +/// backends in `prover::lfm::algebraic_commit`. +pub trait DeviceTreeBackend: IsMerkleTreeBackend { + /// The hash the device kernels must compute for trees labelled `Self`. + const COMMITMENT_HASH: CommitmentHash; +} + +impl DeviceTreeBackend for BatchKeccak256Backend +where + Self: IsMerkleTreeBackend, +{ + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Keccak256; +} +impl DeviceTreeBackend for PairKeccak256Backend +where + Self: IsMerkleTreeBackend, +{ + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Keccak256; +} +impl DeviceTreeBackend for BatchBlake3Backend +where + Self: IsMerkleTreeBackend, +{ + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Blake3; +} +impl DeviceTreeBackend for PairBlake3Backend +where + Self: IsMerkleTreeBackend, +{ + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Blake3; +} +impl DeviceTreeBackend for P1BatchBackend +where + Self: IsMerkleTreeBackend, +{ + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Poseidon1; +} +impl DeviceTreeBackend for P1PairBackend +where + Self: IsMerkleTreeBackend, +{ + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Poseidon1; +} + +/// The hash every commitment this crate produces is built with. +/// +/// It is the machine-readable form of what the aliases above say in types, and +/// it exists so that code reasoning about *which hash is inside a root* can +/// match on it exhaustively rather than assert it in prose — see +/// `build_artifacts_with_hasher` in `prover/src/lfm/registry.rs`, whose +/// artifacts name a hash. Every such match is a place that has to be revisited +/// before this crate commits under a second hash; adding [`Self::Blake3`] broke +/// them, which is what that mechanism is for. +/// +/// ★ Under `cuda` every variant also needs a `math_cuda::DeviceHash` twin. +/// `gpu_lde`'s bridge is total in both directions and asserts the pairing at +/// compile time, so a variant added here without one is a build error naming +/// the gap, and the bridge cannot cross-pair two hashes. A twin whose kernels +/// are not yet ported is legal: its dispatch arms abort loudly. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CommitmentHash { + /// Keccak-256 at both the leaf and the parent layer. + Keccak256, + /// `Blake3Chain` at both the leaf and the parent layer — the single-chunk + /// BLAKE3 chain of PA-PLAN §1.7, at the round count `crypto`'s + /// `blake3-6round` feature selects. + /// + /// The round count is deliberately **not** a second variant. It is a + /// crate-global compile-time constant precisely so one build cannot produce + /// two hashes, so within a build there is nothing here to distinguish: a + /// proof's roots are named by this variant plus the build's feature set, + /// exactly as the `LFM_BLAKE3` chip's round count is. + Blake3, + /// RPX256 (XHash12) at both layers: RPO256's geometry and constants, a + /// different round schedule. Its backend lives in + /// `prover::lfm::algebraic_commit`, beside the machine's own chip. + Rpx256, + /// ZisK's Poseidon1 instance (Goldilocks, width 16, rate 12; `crypto::hash:: + /// poseidon1_stark`): ZisK's leaf hash and ARITY-4 trees. ⚠ An exploration + /// configuration (branch `p1/*`), not a candidate for landing. + Poseidon1, +} + +/// The hash behind [`Commitment`], [`BatchedMerkleTree`] and +/// [`FriLayerMerkleTree`]. Pinned to the aliases by the assertion below. +/// +/// ⚠ **This describes the DEFAULT configuration — the aliases — and nothing +/// else.** A prover can run under a configuration whose +/// [`StarkHash::COMMITMENT_HASH`] differs from this const and this const will +/// not know: it is a global, the configuration is per-type. Code that names the +/// hash inside a *particular* proof's roots must read `H::COMMITMENT_HASH` at +/// the call site; only code that names the hash of the aliases may read this. +/// `prover::lfm::registry` reads this one because its commit helpers are +/// hard-wired to the aliases — when they become generic over `H`, that read +/// moves with them (PA-PLAN §4.2). +/// +/// It is defined as [`DefaultStarkHash`]'s own constant rather than restated, so +/// the `cuda` fork cannot be taken here and forgotten there. +pub const COMMITMENT_HASH: CommitmentHash = ::COMMITMENT_HASH; + +/// One STARK commitment configuration: the Merkle backend families the +/// prover and verifier build trees with, named together so they cannot be +/// mixed, plus the [`CommitmentHash`] they all are. +/// +/// The three are separate families because they hash different leaf shapes, not +/// The two are separate families because they hash different leaf shapes, not +/// because they are different hashes: [`Self::Batched`] takes a whole row group, +/// [`Self::Pair`] a fixed FRI-layer pair. An implementation is expected to build +/// both on one hash — that is what [`Self::COMMITMENT_HASH`] asserts, and what +/// makes a proof's roots describable by a single name. +/// +/// Every member is generic over the field because the prover commits over both +/// the base field (main trace) and the extension (aux, composition, FRI) within +/// one proof, so the configuration cannot be pinned to one field. +/// +/// `Node` is deliberately **not** an associated type: it is [`Commitment`] for +/// every implementation. Keeping 32 bytes on the wire is what lets a +/// configuration change leave `StarkProof`'s fields and their rkyv derives +/// byte-identical — no format bump, no disturbance to the in-place verify path. +/// +/// # Invariant: the two families must agree on a two-element leaf +/// +/// `>::hash_data(&vec![a, b])` must equal `>::hash_data(&[a, b])`. +/// +/// This is load-bearing, not decorative. The prover builds FRI-layer trees with +/// [`Self::Pair`] (`fri/mod.rs`) and the verifier authenticates those same +/// openings with [`Self::Batched`] (`verify_fri_layer_openings`, which builds a +/// two-element `Vec`). Under keccak the two coincide — both stream the same +/// element bytes into one digest — which is why the split went unremarked while +/// there was only one configuration. A configuration whose families encode a +/// pair differently would reject every honest proof at its first FRI query. +/// +/// An implementation that cannot honour this must make the prover and verifier +/// agree on one family instead of implementing this trait and hoping. +pub trait StarkHash: Send + Sync + 'static { + /// The batched leaf backend: one leaf per row group, streamed. + /// + /// Under `cuda` this additionally has to be [`DeviceTreeBackend`]: + /// `gpu_lde`'s tree entries hash on the device with the kernel family the + /// backend's `COMMITMENT_HASH` names, and only *label* the result with + /// this type — the bound is what guarantees the label and the kernels + /// agree, at compile time. + #[cfg(feature = "cuda")] + type Batched: IsStreamingLeafBackend>> + + DeviceTreeBackend + + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The batched leaf backend: one leaf per row group, streamed. + #[cfg(not(feature = "cuda"))] + type Batched: IsStreamingLeafBackend>> + + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The FRI-layer backend: one leaf per fixed pair, no `Vec` per leaf. + /// + /// Under `cuda` this carries the same [`DeviceTreeBackend`] obligation + /// [`Self::Batched`] does, and for the same reason: `gpu_lde`'s FRI commit + /// drives the whole commit phase on device, hashing every layer tree with + /// the kernel family the backend's `COMMITMENT_HASH` names. + #[cfg(feature = "cuda")] + type Pair: IsMerkleTreeBackend; 2]> + + DeviceTreeBackend + + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The FRI-layer backend: one leaf per fixed pair, no `Vec` per leaf. + #[cfg(not(feature = "cuda"))] + type Pair: IsMerkleTreeBackend; 2]> + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The Fiat-Shamir configuration this commitment configuration is paired + /// with — the hash the transcript sponges on, and the one grinding's + /// proof-of-work computes over. + /// + /// Naming it here is what keeps a proof describable by one configuration. + /// The transcript object is still built by the caller and handed to + /// `multi_prove` / `multi_verify`, so this does not *force* the caller's + /// transcript to match; what it forces is that everything the prover and + /// verifier derive internally from the configuration — grinding — follows + /// this hash instead of a hard-wired one. + type Transcript: TranscriptHash; + + /// What both Merkle families hash with. The name a proof's roots may be + /// called by. + const COMMITMENT_HASH: CommitmentHash; + + /// Children per Merkle node in both families (`IsMerkleTreeBackend::ARITY`; + /// a configuration states it once, and its backends must agree). At 4 the + /// format's binary cap policy does not apply: the trees take + /// [`Self::ARITY4_CAP`] instead ([`effective_cap_policy`]). + const ARITY: usize = 2; + + /// The cap policy of an arity-4 configuration's trees, heights in 4-ary + /// levels (`crypto::merkle_tree::cap`'s arity-4 shape). The format's + /// `merkle_cap` prices binary caps, so an arity-4 configuration states its + /// own: a verifier constant of the configuration, never read from a proof + /// or the environment. Unused at arity 2. + const ARITY4_CAP: CapPolicy = CapPolicy::Off; +} + +/// The cap policy a configuration actually runs: the format's `merkle_cap` +/// (`format_cap`) at arity 2, the configuration's own +/// [`StarkHash::ARITY4_CAP`] at arity 4 (heights in 4-ary levels). A verifier +/// constant, from the configuration and the verifier's format, never from the +/// proof. +pub fn effective_cap_policy(format_cap: CapPolicy) -> CapPolicy { + TreeGeometry::of::().cap_policy(format_cap) +} + +/// A configuration's tree shape as a value — its arity and its own arity-4 +/// cap ([`StarkHash::ARITY`], [`StarkHash::ARITY4_CAP`]) — for code that lays +/// out a proof's trees without naming the configuration type (the in-guest +/// verifier's shapes). Taken from the configuration ([`Self::of`]), never +/// from a proof. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TreeGeometry { + /// Children per node: 2 or 4. + pub arity: usize, + /// The cap policy at arity 4, heights in 4-ary levels (unused at 2). + pub arity4_cap: CapPolicy, +} + +impl TreeGeometry { + /// Binary trees under the format's cap policy: every configuration but an + /// arity-4 one, and today's shapes. + pub const BINARY: Self = Self { + arity: 2, + arity4_cap: CapPolicy::Off, + }; + + /// The geometry of the configuration `H`. + pub const fn of() -> Self { + Self { + arity: H::ARITY, + arity4_cap: H::ARITY4_CAP, + } + } + + /// The cap policy these trees run under a format whose binary policy is + /// `format_cap` ([`effective_cap_policy`]). + pub fn cap_policy(self, format_cap: CapPolicy) -> CapPolicy { + if self.arity == 2 { + format_cap + } else { + self.arity4_cap + } + } +} + +/// The digest a configuration grinds over: its transcript's hash, because the +/// grinding seed is `transcript.state()`. +pub type GrindingDigest = <::Transcript as TranscriptHash>::Digest; + +/// ★ The Fiat-Shamir transcript of the default configuration — the one the +/// production prove and verify entry points must build. +/// +/// `DefaultTranscript`'s own type default is [`KeccakTranscriptHash`], and +/// that default is NOT the configuration's. The two coincided while keccak was +/// the only configuration, and a caller writing `DefaultTranscript::::new(..)` +/// after the flip gets a keccak sponge over BLAKE3 commitments — self-consistent +/// between prover and verifier, and therefore silent, but a half-flip: the +/// Fiat-Shamir hash would not have moved with the commitment hash. +/// +/// `multi_prove` / `multi_verify` take `impl IsStarkTranscript`, so the type +/// system cannot force this; naming the alias is what makes the production path +/// follow [`DefaultStarkHash`] instead of a `derive`'s default. +pub type DefaultStarkTranscript = crypto::fiat_shamir::default_transcript::DefaultTranscript< + F, + ::Transcript, +>; + +/// The keccak-256 configuration. +/// +/// Reachable by naming it: the keccak-pinned LFM instruments and the GPU +/// keccak A/B arms commit under it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeccakStarkHash; + +impl StarkHash for KeccakStarkHash { + type Batched + = BatchKeccak256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Pair + = PairKeccak256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Transcript = KeccakTranscriptHash; + + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Keccak256; +} + +/// The BLAKE3 configuration — `Blake3Chain` at both the leaf and the parent +/// layer, over the *same* two generic backends the keccak instance uses. +/// +/// Sharing those backends is what makes the two-element invariant above hold by +/// construction: `Batched::hash_data(&vec![a, b])` and `Pair::hash_data(&[a, b])` +/// serialize the same 16 bytes and hand them to the same digest, so there are +/// not two encodings to be shown equal. +/// `blake3_batched_and_pair_agree_on_a_two_element_leaf` pins it anyway, because +/// "holds by construction" is a claim about today's code and the invariant has +/// to survive tomorrow's. +/// +/// Separately, and for the parent layer rather than the leaf: a parent's message +/// is the two 32-byte children, and at 64 bytes `Blake3Chain` is a single BLAKE3 +/// compression in the framing the device kernels implement +/// (`crypto::hash::blake3::chain`, PA-PLAN §1.7 P2). +/// +/// # What selects it +/// +/// ★ **Everything, on every build.** This is [`DefaultStarkHash`]: every +/// `Prover` and `Verifier` alias resolves here, [`COMMITMENT_HASH`] names it, +/// and the transcript and grinding follow through [`StarkHash::Transcript`]. +/// The RV64 guest reaches it through the same aliases and hashes with the +/// `blake3_compress_6round` precompile; a cuda build reaches it through +/// `gpu_lde`'s [`DeviceTreeBackend`] dispatch onto the Track G BLAKE3 +/// kernels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Blake3StarkHash; + +impl StarkHash for Blake3StarkHash { + type Batched + = BatchBlake3Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Pair + = PairBlake3Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Transcript = Blake3TranscriptHash; + + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Blake3; +} + +/// [`Blake3StarkHash`]'s members are the BLAKE3 backends, not a mix. +/// +/// The two families exist because they hash different leaf *shapes*, not because +/// they are different hashes — that is the whole content of +/// [`StarkHash::COMMITMENT_HASH`] being one constant. Asserting it here means a +/// configuration assembled from one hash's batched backend and another's pair +/// backend fails to compile, rather than producing proofs whose roots no single +/// name describes. +const _: fn() = || { + fn assert_same(_: core::marker::PhantomData<(T, T)>) {} + + assert_same::>( + core::marker::PhantomData::<( + BatchBlake3Backend, + ::Batched, + )>, + ); + assert_same::>( + core::marker::PhantomData::<( + PairBlake3Backend, + ::Pair, + )>, + ); +}; + +/// ZisK's Poseidon1 (Goldilocks, width 16, rate 12) as a STARK commitment +/// configuration: the tagged leaf hash and 4-ary trees padded with zero +/// digests (`crypto::merkle_tree::backends::p1`), ZisK's transcript sponge and +/// its width-8 grinding (`crypto::fiat_shamir::p1_transcript`), every tree +/// capped at 4-ary height `CAP` (`0`: uncapped). +/// +/// The cap is part of the configuration's type, so a prover and a verifier +/// that name the same configuration agree on it by construction, and no +/// format field, environment variable or proof byte can move it. Callers name +/// one instance (the prover crate's `lfm::p1_commit::P1StarkHash`); the others +/// exist for the cap census and the tests that bind each height. +/// +/// The transcript object is still the caller's: prove and verify with +/// `crypto::fiat_shamir::p1_transcript::P1Transcript`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct P1StarkHashCapped; + +impl StarkHash for P1StarkHashCapped { + type Batched + = P1BatchBackend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Pair + = P1PairBackend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Transcript = P1TranscriptHash; + + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Poseidon1; + + const ARITY: usize = 4; + + const ARITY4_CAP: CapPolicy = CapPolicy::Fixed(CAP); +} + +/// The Poseidon1 configuration's arity is its backends', at both fields the +/// prover commits over, and its members name its own hash as their device key. +const _: () = { + type P1 = P1StarkHashCapped<0>; + assert!( as IsMerkleTreeBackend>::ARITY == P1::ARITY); + assert!( as IsMerkleTreeBackend>::ARITY == P1::ARITY); + assert!( + as IsMerkleTreeBackend>::ARITY + == P1::ARITY + ); + assert!( + as IsMerkleTreeBackend>::ARITY + == P1::ARITY + ); + assert!( + <::Batched as DeviceTreeBackend>::COMMITMENT_HASH as u8 + == CommitmentHash::Poseidon1 as u8 + ); + assert!( + <::Pair as DeviceTreeBackend>::COMMITMENT_HASH as u8 + == CommitmentHash::Poseidon1 as u8 + ); + assert!(matches!(P1::COMMITMENT_HASH, CommitmentHash::Poseidon1)); +}; + +/// The device marker's tie-in: each configuration's members carry the +/// constant that names the configuration's own hash, which is what makes the +/// [`DeviceTreeBackend`] impls above true statements rather than decoration — +/// `gpu_lde` dispatches device kernels on that constant, so a mismatch here +/// would be a GPU run hashing under a name the roots do not deserve. +/// +/// A `const` block rather than a never-called closure, so the value comparison +/// is evaluated at compile time and not merely type-checked. Discriminants are +/// compared because `PartialEq` is not `const`. The algebraic configurations +/// carry the same check beside their impls in `prover::lfm::algebraic_commit`. +const _: () = { + const fn assert_device_hash(expect: CommitmentHash) { + assert!( + B::COMMITMENT_HASH as u8 == expect as u8, + "a configuration's member must name the configuration's own hash as its device key" + ); + } + const fn assert_same(_: core::marker::PhantomData<(T, T)>) {} + + assert_device_hash::<::Batched>( + CommitmentHash::Keccak256, + ); + assert_device_hash::<::Pair>( + CommitmentHash::Keccak256, + ); + assert_device_hash::<::Batched>( + CommitmentHash::Blake3, + ); + assert_device_hash::<::Pair>( + CommitmentHash::Blake3, + ); + + // The aliases ARE [`DefaultStarkHash`]'s members, not a second opinion. + // They are spelled concretely for the lifetime reason noted at their + // definition, so this is the tie that makes the two spellings one type — + // and it is what fails if only one side of the `cuda` fork is edited. + assert_same::>( + core::marker::PhantomData::<( + BatchedMerkleTreeBackend, + ::Batched, + )>, + ); + assert_same::>( + core::marker::PhantomData::<( + FriLayerMerkleTreeBackend, + ::Pair, + )>, + ); +}; + +/// Stated positively: the default commits BLAKE3, on every build. +/// +/// The alias definitions make [`COMMITMENT_HASH`] follow [`DefaultStarkHash`] +/// automatically, so nothing above can *disagree* — what this catches is the +/// default being re-pointed without the blessed artifacts moving with it. +/// Every pinned root in this workspace (`LFM_REGISTRY`, +/// `static_zero_page_commitment`, the preprocessed table commitments) was +/// generated under this configuration. +const _: () = assert!(matches!(COMMITMENT_HASH, CommitmentHash::Blake3)); + +/// ★ The transcript follows the commitment hash — per configuration, by +/// assertion. +/// +/// `multi_prove` / `multi_verify` take `impl IsStarkTranscript`, so the type +/// system cannot force a caller's transcript to match its commitment +/// configuration; what CAN be forced is that each named configuration pairs +/// its commitment hash with its own family's Fiat-Shamir hash, so following +/// the configuration (as `DefaultStarkTranscript` does) can never produce the +/// half-flip — one family's sponge over the other family's roots, which is +/// self-consistent between prover and verifier and therefore silent. +const _: fn() = || { + fn assert_same(_: core::marker::PhantomData<(T, T)>) {} + + assert_same::( + core::marker::PhantomData::<( + Blake3TranscriptHash, + ::Transcript, + )>, + ); + assert_same::( + core::marker::PhantomData::<( + KeccakTranscriptHash, + ::Transcript, + )>, + ); + assert_same::( + core::marker::PhantomData::<( + P1TranscriptHash, + as StarkHash>::Transcript, + )>, + ); +}; +const _: () = assert!(matches!( + ::COMMITMENT_HASH, + CommitmentHash::Blake3 +)); +const _: () = assert!(matches!( + ::COMMITMENT_HASH, + CommitmentHash::Keccak256 +)); + +/// ★ The round-count lockstep, and the reason the default build must carry +/// `blake3-6round`. +/// +/// `BLAKE3_ROUNDS` is a crate-global compile-time constant, so one build cannot +/// produce two hashes — but two *builds* can, and every blessed constant in this +/// workspace was generated at six rounds. Without this assertion a build that +/// merely lost the feature would commit seven-round roots and fail later, at +/// drift-test time, with a mismatch that names no cause. Here it is a compile +/// error that names one. +/// +const _: () = assert!( + crypto::hash::blake3::BLAKE3_ROUNDS == crypto::hash::blake3::BLAKE3_SIX_ROUNDS, + "the default commitment configuration is BLAKE3, so this build must enable \ + `crypto/blake3-6round` (via `lambda-vm-prover/blake3-6round`, or the \ + default feature set): every blessed root was generated at six rounds" +); diff --git a/crypto/stark/src/constraint_ir/artifact.rs b/crypto/stark/src/constraint_ir/artifact.rs new file mode 100644 index 000000000..5a614ab04 --- /dev/null +++ b/crypto/stark/src/constraint_ir/artifact.rs @@ -0,0 +1,771 @@ +//! Build-time serialization of an AIR's transition constraints — "constraints +//! as data". +//! +//! [`DeviceProgram`](super::device::DeviceProgram) already flattens a captured +//! [`ConstraintProgram`] into POD arrays, but a program alone does not describe +//! an AIR's transition constraints: evaluating the roots is only part of the +//! job. A consumer also needs each constraint's ZEROFIER shape (which capture +//! discards) and the AIR's shape scalars. This module bundles all of it into one +//! serializable [`ConstraintArtifact`]. +//! +//! # Why the bundle is four things, not one +//! +//! Derived from what [`crate::verifier`] actually calls on an AIR, not from the +//! trait surface: +//! +//! 1. **The program** — `nodes` / `base_consts` / `ext_consts` / `roots` / +//! `num_base`, as a flat POD projection of the captured [`ConstraintProgram`] +//! ([`ArtifactNode`], node-id operands). Replaces `AIR::compute_transition`. +//! Note this is NOT [`DeviceProgram`]'s form — see [`ArtifactNode`] for why +//! the artifact keeps the liftable node-id form and re-lowers on demand. +//! 2. **Per-constraint metadata** — `{kind, end_exemptions}` per constraint. +//! Capture DISCARDS this: [`ConstraintProgram`] records each constraint's +//! root but not the row domain it applies to, and `end_exemptions` is what +//! picks the constraint's zerofier +//! (`AIR::transition_zerofier_evaluations_grouped` keys its dedup groups on +//! exactly this field). A program without it evaluates the right algebra +//! against the wrong divisor. +//! +//! **Production zerofiers are UNIFORM.** Measured, not assumed: every +//! production constraint across all 28 tables emits through `RowDomain::ALL` +//! — `RowDomain::except_last` appears only in `crate::examples` and in tests. +//! So `end_exemptions` is 0 everywhere and every table has exactly ONE +//! zerofier group. Two things follow. The GPU constraint path already +//! *requires* a uniform zerofier, so that precondition holds in fact rather +//! than by luck. And a consumer evaluating these constraints needs one +//! zerofier per AIR, not one per distinct exemption value — worth knowing +//! before speccing the general case defensively. +//! +//! The field is still carried, and is still load-bearing for anything that +//! is not a production VM table (the example AIRs use exemptions). It is +//! covered by `ExemptConstraints` in `artifact_tests`, deliberately, so that +//! "always zero in production" cannot decay into "never tested". +//! 3. **The AIR shape** — widths, step size, transition offsets, the next-row +//! column set (which decides the pruned `g·z` OOD opening), max bus elements. +//! 4. **The composition degree multiplier** — see +//! [`AirShape::composition_degree_multiplier`]. This one is easy to miss: it +//! lives in neither [`AirContext`](crate::context::AirContext) nor +//! [`ConstraintMeta`], only inside the `ConstraintSet` impl and the LogUp +//! layout, yet the verifier needs it to size the composition polynomial. +//! +//! # What is deliberately NOT in the bundle +//! +//! - **`ProofOptions`.** `AirContext` bundles the proof options in with the +//! shape scalars, but a captured program does not depend on them (pinned by +//! the blowup-invariance test in the prover's artifact suite). Storing them +//! would multiply the artifact count by the number of blowup factors for no +//! information gain, and would wrongly imply the constraints are +//! options-dependent. Options are supplied at AIR construction. +//! - **Trace length / epoch size.** No AIR constructor takes one, so the axis is +//! structurally absent; the only route by which it could reach the artifact is +//! `composition_poly_degree_bound(n)`, which the artifact stores divided +//! through by `n`. That division is sound only if the bound is exactly linear, +//! which `artifacts_are_invariant_across_trace_length` sweeps per table rather +//! than assuming. +//! - **The preprocessed COMMITMENT.** `AIR::precomputed_commitment` is a +//! blowup-dependent Merkle root, delivered by the existing static-commitment +//! mechanism. Only the `is_preprocessed` / `num_precomputed_columns` shape +//! flags are artifact material; putting the root here would reintroduce the +//! options dependence the previous point removes. +//! - **Boundary constraints.** `AIR::boundary_constraints` is a function of the +//! public inputs, not a static property of the AIR, so it is not data in the +//! sense this artifact means. Serializing it is a separate problem. +//! - **Derived scalars.** `has_aux_trace` and `num_auxiliary_rap_columns` are +//! pure functions of `trace_layout`; `num_transition_constraints` is +//! `roots.len()`. Storing a second copy only creates a way for the two to +//! disagree. +//! +//! # Guest safety +//! +//! [`ConstraintArtifact::capture`] CAPTURES — it calls +//! `AIR::constraint_program`, which hash-conses. It is a build-time entry point +//! and must never run in a guest. Everything else here (deserialize, +//! [`ConstraintArtifact::program`], [`ConstraintArtifact::validate_against`]) is +//! pure data handling and is guest-safe: that asymmetry is the whole point of +//! the artifact. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as GoldilocksExtension; +use math::field::goldilocks::GoldilocksField; + +use super::device::DeviceProgram; +use super::ir::{ConstraintProgram, Dim, Op}; +use crate::constraints::builder::{ConstraintMeta, RootKind}; +use crate::traits::AIR; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// Trace lengths used to probe `AIR::composition_poly_degree_bound`. Two of +/// them, so [`ConstraintArtifact::capture`] can check the bound really is linear +/// in the trace length rather than assuming it. +const DEGREE_PROBE_LEN: usize = 1 << 10; +const DEGREE_PROBE_LEN_2: usize = 1 << 11; + +// ============================================================================= +// The wire node +// ============================================================================= + +/// Node result is a base-field value. +pub const DIM_BASE: u32 = 0; +/// Node result is an extension-field value. +pub const DIM_EXT: u32 = 1; + +/// One serialized IR instruction: 16 bytes, `#[repr(C)]`. +/// +/// This is a POD projection of [`Op`] + its [`Dim`], NOT of +/// [`DeviceNode`](super::device::DeviceNode). The distinction is the whole +/// reason this type exists and is worth stating plainly. +/// +/// `DeviceNode` is the *lowered* form: its `a`/`b` are slot-encoded operand +/// words, uniform leaves and dead nodes have been eliminated, and it carries a +/// result slot instead of a dim. That lowering is LOSSY — there is no map back +/// to the [`ConstraintProgram`] it came from. An artifact that stored it could +/// not implement [`ConstraintArtifact::program`], which is the method every +/// consumer of this artifact actually uses. +/// +/// So the artifact stores the high-level form instead: `a`/`b` are **node ids** +/// (id `i` references only `< i`, the same invariant [`ConstraintProgram`] +/// carries), `dim` is [`DIM_BASE`] / [`DIM_EXT`], and the node list is dense — +/// one entry per `ConstraintProgram` node, in the same order. `program()` is its +/// exact inverse; the device blob is re-derived on demand by running main's own +/// [`DeviceProgram::lower`] over that lifted program, so the slot encoding is +/// never duplicated here and cannot drift from the prover's. +/// +/// The `OP_*` tags are shared with `device.rs` — those are stable and mean the +/// same thing in both forms; only the operand encoding differs. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ArtifactNode { + /// `OP_*` tag (shared with [`super::device`]). + pub op: u32, + /// Operand word 0: a node id for arithmetic ops, a table index for + /// constants/uniforms, packed [`Op::Var`] fields for `OP_VAR`. + pub a: u32, + /// Operand word 1: as `a`, where the op takes two operands. + pub b: u32, + /// [`DIM_BASE`] or [`DIM_EXT`] — this node's result dim. + pub dim: u32, +} + +// ============================================================================= +// Metadata +// ============================================================================= + +/// [`ConstraintMeta`] as plain serializable data. +/// +/// `kind` is encoded as a `u8` (see [`ArtifactMeta::KIND_BASE`] / +/// [`ArtifactMeta::KIND_EXT`]) rather than reusing [`RootKind`] so the wire +/// encoding is pinned independently of the in-memory enum. +#[derive(Clone, Copy, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ArtifactMeta { + /// Constraint index. Stored rather than implied by position so + /// [`ConstraintArtifact::validate_self`] can CHECK the dense-and-ordered + /// invariant instead of silently depending on it. + pub constraint_idx: u32, + /// [`ArtifactMeta::KIND_BASE`] or [`ArtifactMeta::KIND_EXT`]. + pub kind: u8, + /// Exempted rows at the end of the trace — the constraint's zerofier shape. + pub end_exemptions: u32, +} + +impl ArtifactMeta { + /// Base-field rooted constraint. + pub const KIND_BASE: u8 = 0; + /// Extension-field (LogUp) rooted constraint. + pub const KIND_EXT: u8 = 1; + + fn from_meta(m: &ConstraintMeta) -> Self { + // An exhaustive match, so adding a RootKind variant is a build error + // here rather than a silently wrong wire byte. + let kind = match m.kind { + RootKind::Base => Self::KIND_BASE, + RootKind::Ext => Self::KIND_EXT, + }; + Self { + constraint_idx: m.constraint_idx as u32, + kind, + end_exemptions: m.end_exemptions as u32, + } + } + + /// Back to a [`ConstraintMeta`]. Panics on an unknown `kind` byte — a + /// corrupt artifact must not silently become a base-field constraint. + pub fn to_meta(self) -> ConstraintMeta { + let kind = match self.kind { + Self::KIND_BASE => RootKind::Base, + Self::KIND_EXT => RootKind::Ext, + other => panic!("unknown ArtifactMeta kind byte {other}"), + }; + ConstraintMeta { + constraint_idx: self.constraint_idx as usize, + kind, + end_exemptions: self.end_exemptions as usize, + } + } +} + +// ============================================================================= +// Shape +// ============================================================================= + +/// An AIR's transition-constraint shape: everything the verifier reads off the +/// AIR that is neither the program nor per-constraint metadata. +#[derive(Clone, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct AirShape { + /// `AIR::step_size`. + pub step_size: u32, + /// `AIR::trace_layout().0` — main trace width. + pub main_width: u32, + /// `AIR::trace_layout().1` — aux trace width. + pub aux_width: u32, + /// `AirContext::transition_offsets` — the frame's row offsets. + pub transition_offsets: Vec, + /// `AIR::trace_ood_next_row_columns`, sorted and deduplicated: the + /// full-width `[main | aux]` columns opened at `g·z`. Every other column is + /// reconstructed as ZERO at the next row, so this set is soundness-critical. + pub next_row_columns: Vec, + /// `AIR::max_bus_elements` — decides the LogUp alpha-power count. + pub max_bus_elements: u32, + /// `AIR::has_trace_interaction`. + pub has_trace_interaction: bool, + /// `AIR::is_preprocessed`. + pub is_preprocessed: bool, + /// `AIR::num_precomputed_columns`. + pub num_precomputed_columns: u32, + /// `composition_poly_degree_bound(n) / n` — the trace-length-INDEPENDENT + /// part of the composition degree bound. + /// + /// Stored as this observable rather than as the underlying `max_degree` + /// because `max_degree` is not exposed on the `AIR` trait at all: it is + /// `max(ConstraintSet::max_degree(), logup_max_degree(layout))`, private to + /// the AIR's construction. The multiplier is what the verifier consumes, it + /// is directly measurable through the public trait, and it needs no new + /// trait method. + pub composition_degree_multiplier: u32, +} + +// ============================================================================= +// The artifact +// ============================================================================= + +/// A build-time-serializable bundle of one AIR's transition constraints. +/// +/// Produced by [`ConstraintArtifact::capture`] (build time, captures) and +/// consumed by [`ConstraintArtifact::program`] (guest-safe, pure data). +/// +/// # A trap for anyone optimizing a consumer of this program +/// +/// The node list is HASH-CONSED: structurally identical subexpressions share one +/// node, which is why the program is compact. That same sharing makes the +/// obvious peepholes UNSOUND if applied naively. +/// +/// Concretely, fusing `Add(Mul(a,b), c)` into a fused multiply-add is only valid +/// when the `Mul` has exactly ONE consumer. A shared `Mul` feeds several +/// parents, and fusing it into each would recompute it per parent — turning a +/// saving into a loss. A node named by [`Self::roots`] counts as a consumer too: +/// fusing it away deletes the value the quotient recombination reads. +/// +/// The rule generalizes to any rewrite that moves work into a consumer: compute +/// the fanout over `nodes` (plus `roots`) first and require it to be 1. The +/// property that makes this IR small is the property that makes rewriting it +/// hazardous, and the two are easy to reason about separately and get wrong +/// together. +#[derive(Clone, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ConstraintArtifact { + /// Topologically ordered flat instruction list (id `i` references only + /// `< i`) — see [`ArtifactNode`]. + pub nodes: Vec, + /// Base-field constant table, raw canonical limbs. + pub base_consts: Vec, + /// Extension-field constant table, raw canonical limbs. + pub ext_consts: Vec<[u64; 3]>, + /// Per-constraint root node ids. + pub roots: Vec, + /// Number of leading base-field-rooted constraints. + pub num_base: u32, + /// Idx-ordered, dense per-constraint metadata. + pub meta: Vec, + /// The AIR's shape scalars. + pub shape: AirShape, +} + +/// Why an artifact was rejected. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ArtifactError { + /// The artifact is internally inconsistent. + #[error("malformed constraint artifact: {0}")] + Malformed(String), + /// The artifact does not describe the AIR it was checked against. + #[error( + "constraint artifact does not match this AIR: {field} is {found} in the artifact but {expected} on the AIR" + )] + ShapeMismatch { + /// The disagreeing field. + field: &'static str, + /// The artifact's value. + found: String, + /// The AIR's value. + expected: String, + }, + /// Serialization or deserialization failed. + #[error("constraint artifact codec error: {0}")] + Codec(String), +} + +/// Compare one shape field, producing a [`ArtifactError::ShapeMismatch`]. +fn check_field(field: &'static str, found: T, expected: T) -> Result<(), ArtifactError> +where + T: PartialEq + core::fmt::Debug, +{ + if found == expected { + Ok(()) + } else { + Err(ArtifactError::ShapeMismatch { + field, + found: format!("{found:?}"), + expected: format!("{expected:?}"), + }) + } +} + +impl ConstraintArtifact { + /// Capture an AIR's constraints into a serializable artifact. + /// + /// BUILD TIME ONLY: this calls `AIR::constraint_program`, which hash-conses + /// the whole constraint body. Never call it from a verifier or a guest — + /// that is precisely what the artifact exists to avoid. + /// + /// # Panics + /// + /// If `composition_poly_degree_bound` is not exactly linear in the trace + /// length, since the artifact stores only the linear coefficient. Better a + /// loud failure at build time than an artifact that silently misstates the + /// composition bound. + pub fn capture(air: &A) -> Self + where + A: AIR + ?Sized, + { + use super::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, pack_var, + }; + + let prog = air.constraint_program(); + + // A 1:1 projection of the captured program — same node count, same + // order, operands left as node ids. Deliberately NOT + // `DeviceProgram::lower`: that is the slot-allocating lowering, and its + // output cannot be lifted back (see `ArtifactNode`). + let nodes: Vec = prog + .nodes + .iter() + .zip(prog.dims.iter()) + .map(|(op, dim)| { + let dim = match dim { + Dim::Base => DIM_BASE, + Dim::Ext => DIM_EXT, + }; + let (op, a, b) = match *op { + Op::ConstBase(idx) => (OP_CONST_BASE, idx, 0), + Op::ConstExt(idx) => (OP_CONST_EXT, idx, 0), + Op::Var { + main, + offset, + row, + col, + } => { + let (a, b) = pack_var(main, offset, row, col); + (OP_VAR, a, b) + } + Op::RapChallenge { idx } => (OP_RAP_CHALLENGE, idx as u32, 0), + Op::AlphaPow { idx } => (OP_ALPHA_POW, idx as u32, 0), + Op::TableOffset => (OP_TABLE_OFFSET, 0, 0), + Op::Add(a, b) => (OP_ADD, a, b), + Op::Sub(a, b) => (OP_SUB, a, b), + Op::Mul(a, b) => (OP_MUL, a, b), + Op::Neg(a) => (OP_NEG, a, 0), + Op::Embed(a) => (OP_EMBED, a, 0), + }; + ArtifactNode { op, a, b, dim } + }) + .collect(); + + let base_consts: Vec = prog.base_consts.iter().map(|c| *c.value()).collect(); + let ext_consts: Vec<[u64; 3]> = prog + .ext_consts + .iter() + .map(|x| { + let limbs = x.value(); + [*limbs[0].value(), *limbs[1].value(), *limbs[2].value()] + }) + .collect(); + + let (main_width, aux_width) = air.trace_layout(); + + let mut next_row_columns: Vec = air + .trace_ood_next_row_columns() + .into_iter() + .map(|c| c as u32) + .collect(); + next_row_columns.sort_unstable(); + next_row_columns.dedup(); + + // The composition bound is `n * k`; recover `k` and check linearity + // across two probe lengths rather than trusting one sample. + let b1 = air.composition_poly_degree_bound(DEGREE_PROBE_LEN); + let b2 = air.composition_poly_degree_bound(DEGREE_PROBE_LEN_2); + assert_eq!( + b1 % DEGREE_PROBE_LEN, + 0, + "composition_poly_degree_bound({DEGREE_PROBE_LEN}) = {b1} is not a multiple of the \ + trace length; the artifact cannot store it as a linear multiplier" + ); + let multiplier = b1 / DEGREE_PROBE_LEN; + assert_eq!( + b2, + multiplier * DEGREE_PROBE_LEN_2, + "composition_poly_degree_bound is not linear in the trace length ({b1} at \ + {DEGREE_PROBE_LEN}, {b2} at {DEGREE_PROBE_LEN_2}); the artifact's single \ + multiplier cannot represent it" + ); + + Self { + nodes, + base_consts, + ext_consts, + roots: prog.roots.clone(), + num_base: prog.num_base as u32, + meta: air + .constraints_meta() + .iter() + .map(ArtifactMeta::from_meta) + .collect(), + shape: AirShape { + step_size: air.step_size() as u32, + main_width: main_width as u32, + aux_width: aux_width as u32, + transition_offsets: air + .context() + .transition_offsets + .iter() + .map(|o| *o as u32) + .collect(), + next_row_columns, + max_bus_elements: air.max_bus_elements() as u32, + has_trace_interaction: air.has_trace_interaction(), + is_preprocessed: air.is_preprocessed(), + num_precomputed_columns: air.num_precomputed_columns() as u32, + composition_degree_multiplier: multiplier as u32, + }, + } + } + + /// The flat device form, produced by re-running the prover's own + /// [`DeviceProgram::lower`] over [`Self::program`]. + /// + /// Not a field copy: the artifact stores node ids, the device form stores + /// slots (see [`ArtifactNode`]). Re-lowering rather than storing the lowered + /// arrays is what keeps this blob identical to the one the prover and the + /// GPU path build from the same AIR — there is one lowering, not two. + /// + /// Guest-safe: `program()` is a POD walk and `lower()` is a liveness scan; + /// neither captures nor hashes. + pub fn device_program(&self) -> DeviceProgram { + DeviceProgram::lower(&self.program()) + } + + /// Lift back to a [`ConstraintProgram`] — the exact inverse of + /// [`DeviceProgram::lower`], so the generic CPU interpreters + /// ([`eval_program`](super::interp::eval_program) / + /// [`eval_program_verifier`](super::interp::eval_program_verifier)) can run + /// a deserialized artifact. + /// + /// Guest-safe: a linear walk over POD arrays, no capture and no hashing. + /// + /// # Panics + /// + /// On an unknown op tag or dim tag — a corrupt program must not evaluate to + /// something plausible. + pub fn program(&self) -> ConstraintProgram { + use super::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, unpack_var, + }; + + let mut nodes = Vec::with_capacity(self.nodes.len()); + let mut dims = Vec::with_capacity(self.nodes.len()); + + for (i, n) in self.nodes.iter().enumerate() { + let op = match n.op { + OP_CONST_BASE => Op::ConstBase(n.a), + OP_CONST_EXT => Op::ConstExt(n.a), + OP_VAR => { + let (main, offset, row, col) = unpack_var(n.a, n.b); + Op::Var { + main, + offset, + row, + col, + } + } + OP_RAP_CHALLENGE => Op::RapChallenge { idx: n.a as u16 }, + OP_ALPHA_POW => Op::AlphaPow { idx: n.a as u16 }, + OP_TABLE_OFFSET => Op::TableOffset, + OP_ADD => Op::Add(n.a, n.b), + OP_SUB => Op::Sub(n.a, n.b), + OP_MUL => Op::Mul(n.a, n.b), + OP_NEG => Op::Neg(n.a), + OP_EMBED => Op::Embed(n.a), + other => panic!("unknown op tag {other} at node {i}"), + }; + let dim = match n.dim { + DIM_BASE => Dim::Base, + DIM_EXT => Dim::Ext, + other => panic!("unknown dim tag {other} at node {i}"), + }; + nodes.push(op); + dims.push(dim); + } + + ConstraintProgram { + nodes, + dims, + base_consts: self + .base_consts + .iter() + .map(|c| FieldElement::::from_raw(*c)) + .collect(), + ext_consts: self + .ext_consts + .iter() + .map(|limbs| { + FieldElement::::from_raw([ + FieldElement::::from_raw(limbs[0]), + FieldElement::::from_raw(limbs[1]), + FieldElement::::from_raw(limbs[2]), + ]) + }) + .collect(), + roots: self.roots.clone(), + num_base: self.num_base as usize, + } + } + + /// The per-constraint metadata as the engine's own type. + pub fn constraints_meta(&self) -> Vec { + self.meta.iter().map(|m| m.to_meta()).collect() + } + + /// Internal consistency: the invariants a consumer would otherwise assume. + /// + /// Checks that node operands are topologically ordered and in range, that + /// constant/root indices are in range, and that the metadata list is dense, + /// idx-ordered and has its `Base` entries as a prefix of length `num_base`. + pub fn validate_self(&self) -> Result<(), ArtifactError> { + use super::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, + }; + let bad = |m: String| Err(ArtifactError::Malformed(m)); + + for (i, n) in self.nodes.iter().enumerate() { + if n.dim != DIM_BASE && n.dim != DIM_EXT { + return bad(format!("node {i} has unknown dim tag {}", n.dim)); + } + // Operand ids must reference strictly earlier nodes; constant and + // uniform indices must be in range for their tables. + let check_id = |x: u32| -> Result<(), ArtifactError> { + if (x as usize) < i { + Ok(()) + } else { + Err(ArtifactError::Malformed(format!( + "node {i} references node {x}, which is not strictly earlier" + ))) + } + }; + match n.op { + OP_CONST_BASE => { + if n.a as usize >= self.base_consts.len() { + return bad(format!( + "node {i} reads base_consts[{}] of {}", + n.a, + self.base_consts.len() + )); + } + } + OP_CONST_EXT => { + if n.a as usize >= self.ext_consts.len() { + return bad(format!( + "node {i} reads ext_consts[{}] of {}", + n.a, + self.ext_consts.len() + )); + } + } + // Var/challenge/alpha/table-offset index per-proof inputs whose + // lengths are not part of the artifact; range-checking them is + // the caller's job at evaluation time. + OP_VAR | OP_RAP_CHALLENGE | OP_ALPHA_POW | OP_TABLE_OFFSET => {} + OP_ADD | OP_SUB | OP_MUL => { + check_id(n.a)?; + check_id(n.b)?; + } + OP_NEG | OP_EMBED => check_id(n.a)?, + other => return bad(format!("node {i} has unknown op tag {other}")), + } + } + + if self.roots.len() != self.meta.len() { + return bad(format!( + "{} roots but {} metadata entries", + self.roots.len(), + self.meta.len() + )); + } + for (c, &root) in self.roots.iter().enumerate() { + if root as usize >= self.nodes.len() { + return bad(format!( + "constraint {c} roots at node {root} of {}", + self.nodes.len() + )); + } + } + + let num_base = self.num_base as usize; + if num_base > self.meta.len() { + return bad(format!( + "num_base {num_base} exceeds the {} constraints", + self.meta.len() + )); + } + for (i, m) in self.meta.iter().enumerate() { + if m.constraint_idx as usize != i { + return bad(format!( + "metadata entry {i} claims constraint_idx {}; the list must be dense and \ + idx-ordered", + m.constraint_idx + )); + } + let expected = if i < num_base { + ArtifactMeta::KIND_BASE + } else { + ArtifactMeta::KIND_EXT + }; + if m.kind != expected { + return bad(format!( + "constraint {i} has kind {} but num_base is {num_base}; Base entries must \ + form a prefix of exactly that length", + m.kind + )); + } + } + + Ok(()) + } + + /// Check that this artifact actually describes `air`. + /// + /// # What this proves, and what it does not + /// + /// It compares the SHAPE scalars and the per-constraint metadata — enough to + /// reject an artifact captured from a different AIR, or a stale artifact + /// from before a column was added or a constraint's exemptions changed. + /// + /// It does NOT prove the serialized program computes the same algebra as the + /// AIR's compiled folder: verifying that requires evaluating both, which + /// requires capture. An AIR edit that changes a constraint's arithmetic + /// without changing any width or exemption passes this check. The build-time + /// drift test is what covers that case, and it is not optional. + pub fn validate_against(&self, air: &A) -> Result<(), ArtifactError> + where + A: AIR + ?Sized, + { + self.validate_self()?; + + let (main_width, aux_width) = air.trace_layout(); + check_field("main_width", self.shape.main_width as usize, main_width)?; + check_field("aux_width", self.shape.aux_width as usize, aux_width)?; + check_field("step_size", self.shape.step_size as usize, air.step_size())?; + check_field( + "num_transition_constraints", + self.roots.len(), + air.context().num_transition_constraints, + )?; + check_field( + "num_base", + self.num_base as usize, + air.num_base_transition_constraints(), + )?; + check_field( + "max_bus_elements", + self.shape.max_bus_elements as usize, + air.max_bus_elements(), + )?; + check_field( + "has_trace_interaction", + self.shape.has_trace_interaction, + air.has_trace_interaction(), + )?; + check_field( + "is_preprocessed", + self.shape.is_preprocessed, + air.is_preprocessed(), + )?; + check_field( + "num_precomputed_columns", + self.shape.num_precomputed_columns as usize, + air.num_precomputed_columns(), + )?; + + let offsets: Vec = self + .shape + .transition_offsets + .iter() + .map(|o| *o as usize) + .collect(); + check_field( + "transition_offsets", + offsets, + air.context().transition_offsets.clone(), + )?; + + let mut declared = air.trace_ood_next_row_columns(); + declared.sort_unstable(); + declared.dedup(); + let stored: Vec = self + .shape + .next_row_columns + .iter() + .map(|c| *c as usize) + .collect(); + check_field("next_row_columns", stored, declared)?; + + check_field( + "composition_degree_multiplier", + self.shape.composition_degree_multiplier as usize, + air.composition_poly_degree_bound(DEGREE_PROBE_LEN) / DEGREE_PROBE_LEN, + )?; + + let air_meta: Vec = air.constraints_meta().to_vec(); + check_field("constraints_meta", self.constraints_meta(), air_meta)?; + + Ok(()) + } + + /// Serialize to the on-disk / in-guest byte form (rkyv, matching the + /// proof format's own encoding). + pub fn to_bytes(&self) -> Result, ArtifactError> { + rkyv::to_bytes::(self) + .map(|b| b.to_vec()) + .map_err(|e| ArtifactError::Codec(e.to_string())) + } + + /// Deserialize, then check internal consistency. Guest-safe. + pub fn from_bytes(bytes: &[u8]) -> Result { + let artifact = rkyv::from_bytes::(bytes) + .map_err(|e| ArtifactError::Codec(e.to_string()))?; + artifact.validate_self()?; + Ok(artifact) + } +} diff --git a/crypto/stark/src/constraint_ir/artifact_tests.rs b/crypto/stark/src/constraint_ir/artifact_tests.rs new file mode 100644 index 000000000..4df89aa9d --- /dev/null +++ b/crypto/stark/src/constraint_ir/artifact_tests.rs @@ -0,0 +1,394 @@ +//! Unit tests for [`ConstraintArtifact`]: the codec, the lift back to a +//! [`ConstraintProgram`], the self-consistency and shape checks, and the +//! pre-captured supply path. +//! +//! The per-table bit-exactness sweep over all 25 production AIRs lives in the +//! prover crate (`prover/src/tests/constraint_artifact_tests.rs`) — it needs the +//! production tables. What is here is what the production tables CANNOT cover: +//! +//! - **Nonzero `end_exemptions`.** Every production constraint applies to every +//! row (`RowDomain::ALL`); nothing under `prover/src` uses +//! `RowDomain::except_last`. So the production sweep would exercise the +//! artifact's zerofier metadata only in its all-zero case, which proves +//! nothing about the field that capture actually discards. The AIR below has +//! exemptions on purpose. +//! - **Rejection.** A suite of AIRs that all validate proves the checks accept; +//! it does not prove they can reject. The falsification tests here corrupt an +//! artifact in each way `validate_self` claims to catch and assert it does. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as GoldilocksExtension; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsField; + +use super::artifact::{ArtifactError, ArtifactMeta, ConstraintArtifact}; +use crate::constraints::builder::{ConstraintBuilder, ConstraintSet, RowDomain}; +use crate::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use crate::proof::options::GoldilocksCubicProofOptions; +use crate::traits::AIR; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +const NUM_COLS: usize = 4; + +/// A constraint set whose three constraints have DIFFERENT row domains, so the +/// artifact's `end_exemptions` is non-uniform and a bug that dropped, zeroed, or +/// permuted it would be visible. No production table does this today. +struct ExemptConstraints; + +impl ConstraintSet for ExemptConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + let m2 = b.main(0, 2); + let n0 = b.main(1, 0); + let n1 = b.main(1, 1); + + // c0: every row — a degree-2 product. + b.emit_base(0, m0.clone() * m1.clone() - m2.clone()); + // c1: skips the last row (reads the next row). + b.emit_base_rows(1, RowDomain::except_last(1), n0.clone() - m0.clone()); + // c2: skips the last two rows. + b.emit_base_rows(2, RowDomain::except_last(2), n1 - m1); + } +} + +fn options() -> crate::proof::options::ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// The exemption-bearing AIR, with no bus interactions so its constraints are +/// exactly the three above (no LogUp suffix). +fn exempt_air() -> AirWithBuses { + AirWithBuses::new( + NUM_COLS, + AuxiliaryTraceBuildData { + interactions: vec![], + }, + &options(), + 1, + ExemptConstraints, + ) + .with_name("EXEMPT") +} + +// ============================================================================= +// The zerofier metadata capture discards +// ============================================================================= + +#[test] +fn artifact_carries_the_end_exemptions_capture_discards() { + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + + // The three constraints' row domains, which the ConstraintProgram alone has + // no field for. + let exemptions: Vec = artifact.meta.iter().map(|m| m.end_exemptions).collect(); + assert_eq!( + exemptions, + vec![0, 1, 2], + "the artifact must preserve each constraint's row domain" + ); + assert_eq!(artifact.constraints_meta(), air.constraints_meta().to_vec()); + + // ... and it must survive the wire. + let bytes = artifact.to_bytes().expect("serialize"); + let back = ConstraintArtifact::from_bytes(&bytes).expect("deserialize"); + assert_eq!(back, artifact, "artifact must round-trip exactly"); + assert_eq!( + back.meta + .iter() + .map(|m| m.end_exemptions) + .collect::>(), + vec![0, 1, 2] + ); +} + +#[test] +fn zeroed_exemptions_are_rejected_against_the_air() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + assert!(artifact.validate_against(&air).is_ok()); + + // Drop the zerofier shapes — the exact damage that makes a serialized + // program evaluate the right algebra against the wrong divisor. + for m in &mut artifact.meta { + m.end_exemptions = 0; + } + let err = artifact + .validate_against(&air) + .expect_err("an artifact with the row domains flattened must be rejected"); + assert!( + matches!( + err, + ArtifactError::ShapeMismatch { + field: "constraints_meta", + .. + } + ), + "expected a constraints_meta mismatch, got {err:?}" + ); +} + +// ============================================================================= +// Lift / codec +// ============================================================================= + +#[test] +fn lift_is_the_inverse_of_lower() { + let air = exempt_air(); + let captured = air.constraint_program(); + let artifact = ConstraintArtifact::capture(&air); + let lifted = artifact.program(); + + assert_eq!( + lifted.nodes, captured.nodes, + "nodes must survive the round trip" + ); + assert_eq!( + lifted.dims, captured.dims, + "dims must survive the round trip" + ); + assert_eq!(lifted.roots, captured.roots); + assert_eq!(lifted.num_base, captured.num_base); + assert_eq!(lifted.base_consts, captured.base_consts); + assert_eq!(lifted.ext_consts, captured.ext_consts); + + // And through the wire, not just in memory. + let bytes = artifact.to_bytes().expect("serialize"); + let lifted2 = ConstraintArtifact::from_bytes(&bytes) + .expect("deserialize") + .program(); + assert_eq!(lifted2.nodes, captured.nodes); + assert_eq!(lifted2.dims, captured.dims); + assert_eq!(lifted2.base_consts, captured.base_consts); +} + +#[test] +fn constants_survive_as_exact_field_values() { + // Constants go out as raw limbs and come back through `from_raw`; a + // canonicalization slip there would change a constraint's arithmetic + // silently, so pin the values rather than only their count. + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + let lifted = artifact.program(); + for (i, (a, b)) in lifted + .base_consts + .iter() + .zip(air.constraint_program().base_consts.iter()) + .enumerate() + { + assert_eq!(a, b, "base_consts[{i}] changed value across the round trip"); + assert_eq!( + a.value(), + b.value(), + "base_consts[{i}] changed representation across the round trip" + ); + } +} + +// ============================================================================= +// Falsification: every check must be able to reject +// ============================================================================= + +#[test] +fn validate_self_rejects_a_forward_reference() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + // Point the last node's operand at itself: no longer topologically ordered, + // which an interpreter would read as an uninitialized value. + let last = artifact.nodes.len() - 1; + artifact.nodes[last].a = last as u32; + let err = artifact + .validate_self() + .expect_err("a self-referential node must be rejected"); + assert!(matches!(err, ArtifactError::Malformed(_)), "got {err:?}"); +} + +#[test] +fn validate_self_rejects_an_out_of_range_root() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + artifact.roots[0] = artifact.nodes.len() as u32; + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_self_rejects_an_out_of_range_constant() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + let n_base = artifact.base_consts.len() as u32; + let node = artifact + .nodes + .iter_mut() + .find(|n| n.op == super::device::OP_CONST_BASE) + .expect("the program reads at least one base constant"); + node.a = n_base; + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_self_rejects_a_non_prefix_base_kind() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + // All three constraints are base-rooted here, so flipping one to Ext breaks + // the "Base entries form a prefix of length num_base" invariant that + // `num_base_from_meta` relies on. + artifact.meta[0].kind = ArtifactMeta::KIND_EXT; + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_self_rejects_permuted_metadata() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + artifact.meta.swap(0, 2); + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_against_rejects_a_shape_change() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + assert!(artifact.validate_against(&air).is_ok()); + + artifact.shape.main_width += 1; + let err = artifact + .validate_against(&air) + .expect_err("a width change must be rejected"); + assert!( + matches!( + err, + ArtifactError::ShapeMismatch { + field: "main_width", + .. + } + ), + "got {err:?}" + ); +} + +#[test] +fn from_bytes_rejects_a_corrupt_artifact() { + // The codec must not hand back a structurally invalid artifact just because + // the bytes deserialized: `from_bytes` runs `validate_self`. + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + artifact.roots[0] = 9999; + let bytes = artifact.to_bytes().expect("serialize"); + assert!(matches!( + ConstraintArtifact::from_bytes(&bytes), + Err(ArtifactError::Malformed(_)) + )); +} + +// ============================================================================= +// The pre-captured supply path (the scoped verify-path unban) +// ============================================================================= + +#[test] +fn precaptured_is_none_even_after_a_capture() { + let air = exempt_air(); + assert!( + air.precaptured_constraint_program().is_none(), + "a freshly built AIR has no build-time program" + ); + + // Force a capture. This fills the AIR's OnceLock — but a captured program is + // NOT a build-time artifact, and the guest-safe accessor must keep saying so. + let _ = air.constraint_program(); + assert!( + air.precaptured_constraint_program().is_none(), + "a program the AIR captured at runtime must never be reported as pre-captured; \ + conflating the two would let a guest path believe capture had been avoided" + ); +} + +#[test] +fn supplying_a_program_short_circuits_capture() { + let program = ConstraintArtifact::capture(&exempt_air()).program(); + let air = exempt_air().with_precaptured(program); + + let supplied = air + .precaptured_constraint_program() + .expect("the supplied program must be visible"); + + // Pointer identity is the actual proof that no capture ran: a capture would + // have built a fresh program in the OnceLock and returned that instead. + assert!( + std::ptr::eq(air.constraint_program(), supplied), + "constraint_program() must hand back the supplied program itself, not a fresh capture" + ); +} + +#[test] +fn a_supplied_program_still_evaluates_correctly() { + // Supplying a program must not change what the AIR computes. + let program = ConstraintArtifact::capture(&exempt_air()).program(); + let air = exempt_air().with_precaptured(program); + let artifact = ConstraintArtifact::capture(&air); + assert!(artifact.validate_against(&air).is_ok()); + assert_eq!( + artifact.program().nodes, + exempt_air().constraint_program().nodes, + "a supplied program must be the same program the AIR would have captured" + ); +} + +#[test] +#[should_panic(expected = "roots")] +fn supplying_a_mismatched_program_panics() { + // A program for a different constraint count must not be installable. + let mut program = ConstraintArtifact::capture(&exempt_air()).program(); + program.roots.pop(); + let _ = exempt_air().with_precaptured(program); +} + +// ============================================================================= +// Degree bound +// ============================================================================= + +#[test] +fn composition_degree_multiplier_reproduces_the_bound() { + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + let k = artifact.shape.composition_degree_multiplier as usize; + assert!(k >= 1, "the multiplier must be positive"); + for log_n in [8usize, 12, 20] { + let n = 1usize << log_n; + assert_eq!( + air.composition_poly_degree_bound(n), + k * n, + "the stored multiplier must reproduce the AIR's own bound at n=2^{log_n}" + ); + } +} + +/// A sanity floor on the field type used for constants, so a field swap does not +/// silently reinterpret the artifact's raw limbs. +#[test] +fn base_constants_are_goldilocks_limbs() { + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + for &c in &artifact.base_consts { + let fe = FieldElement::::from_raw(c); + assert_eq!( + *fe.value(), + c, + "constant {c} is not a canonical Goldilocks limb" + ); + } +} diff --git a/crypto/stark/src/constraint_ir/device.rs b/crypto/stark/src/constraint_ir/device.rs index e4e170bfc..f86a1f311 100644 --- a/crypto/stark/src/constraint_ir/device.rs +++ b/crypto/stark/src/constraint_ir/device.rs @@ -114,7 +114,7 @@ pub const RES_EXT_BIT: u32 = 1 << 31; /// for root-pinned uniform leaves); `res` is the result slot with [`RES_EXT_BIT`] /// selecting the slot class. #[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct DeviceNode { pub op: u32, pub a: u32, diff --git a/crypto/stark/src/constraint_ir/gpu_interp.rs b/crypto/stark/src/constraint_ir/gpu_interp.rs index 5d5bde4a9..53b9c3a39 100644 --- a/crypto/stark/src/constraint_ir/gpu_interp.rs +++ b/crypto/stark/src/constraint_ir/gpu_interp.rs @@ -128,6 +128,9 @@ pub struct CompositionInputs<'a, F: IsField, E: IsField> { /// cached per domain) — resolved to device-resident columns via /// [`bzinv_device_handles`], so nothing LDE-sized crosses PCIe per dispatch. pub b_z_inv: &'a [std::sync::Arc>>], + /// Evaluate `H` only at every `row_stride`-th LDE row (a sub-coset of the + /// LDE coset); 1 is the whole coset. Must divide the LDE row count. + pub row_stride: usize, } pub(crate) type GoldilocksBZInv = std::sync::Arc>>; @@ -321,8 +324,9 @@ pub enum GpuComposition { } /// Fused composition-poly evaluation on the GPU: returns `H(row)` (host or -/// device-resident per `keep`), or `None` for non-Goldilocks towers (→ CPU -/// fallback). `H(row) = z_inv·Σβᵢ·Cᵢ + Σ_b z_b_inv·β_b·(trace_b − value_b)`, +/// device-resident per `keep`) at every `inputs.row_stride`-th LDE row, or +/// `None` for non-Goldilocks towers or a stride that does not divide `num_rows` +/// (→ CPU fallback). `H(row) = z_inv·Σβᵢ·Cᵢ + Σ_b z_b_inv·β_b·(trace_b − value_b)`, /// the uniform-zerofier accumulation of `evaluator::evaluate`. #[allow(clippy::too_many_arguments)] pub fn try_eval_composition_gpu( @@ -341,6 +345,9 @@ where F: IsField + 'static, E: IsField + 'static, { + if inputs.row_stride == 0 || !num_rows.is_multiple_of(inputs.row_stride) { + return None; + } let LoweredCall { lowered, rap, @@ -370,6 +377,7 @@ where b_value: &b_value, b_beta: &b_beta, b_z_inv: &b_z_inv, + row_stride: inputs.row_stride, }; let result = if keep { diff --git a/crypto/stark/src/constraint_ir/mod.rs b/crypto/stark/src/constraint_ir/mod.rs index 380f32d86..dfc1c13cb 100644 --- a/crypto/stark/src/constraint_ir/mod.rs +++ b/crypto/stark/src/constraint_ir/mod.rs @@ -19,12 +19,17 @@ //! - [`device`]: the concrete-Goldilocks flat lowering ([`DeviceProgram`]) for //! the GPU kernel, plus a CPU walker over that flat blob (the pre-GPU parity //! oracle). +//! - [`artifact`]: the build-time serializable bundle ([`ConstraintArtifact`]) +//! — the flat program PLUS the zerofier metadata capture discards and the +//! AIR's shape scalars, which is what "constraints as data" actually needs. //! +//! [`ConstraintArtifact`]: artifact::ConstraintArtifact //! [`ConstraintProgram`]: ir::ConstraintProgram //! [`Op`]: ir::Op //! [`Dim`]: ir::Dim //! [`DeviceProgram`]: device::DeviceProgram +pub mod artifact; pub mod builder; pub mod device; #[cfg(feature = "cuda")] @@ -32,9 +37,12 @@ pub mod gpu_interp; pub mod interp; pub mod ir; +#[cfg(test)] +mod artifact_tests; #[cfg(test)] mod tests; +pub use artifact::{AirShape, ArtifactError, ArtifactMeta, ArtifactNode, ConstraintArtifact}; pub use builder::{Expr, IrBuilder}; pub use device::{DeviceNode, DeviceProgram, eval_device_program}; pub use interp::{eval_program, eval_program_base, eval_program_verifier}; diff --git a/crypto/stark/src/constraints/builder.rs b/crypto/stark/src/constraints/builder.rs index 4554dd4ee..983c8ef3d 100644 --- a/crypto/stark/src/constraints/builder.rs +++ b/crypto/stark/src/constraints/builder.rs @@ -199,7 +199,7 @@ impl RowDomain { /// [`num_base_from_meta`]. Degree is intentionally absent: only the per-table /// max is consumed (by `composition_poly_degree_bound`), declared once via /// [`ConstraintSet::max_degree`]. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct ConstraintMeta { pub constraint_idx: usize, /// Base | Ext; Base entries MUST be a prefix. @@ -287,6 +287,65 @@ pub trait ConstraintSet: Send + Sync { } } +/// Why an emitted index set can be wrong in a way nothing else catches. +/// +/// [`EmitTracker`]'s duplicate assert is `#[cfg(debug_assertions)]`, and this +/// workspace declares no `[profile.release]` override — so under the house +/// convention of `cargo test --release` it is a no-op and a second +/// `emit_base(idx, …)` silently overwrites the first. Constraint *counts* do not +/// notice: a body that emits one index twice and another never still fills +/// `0..N` slots, so `NUM_CONSTRAINTS`, any hand-written predicted-count test, +/// and `assert_complete` all still pass while a constraint has been deleted. +/// +/// This is the check that does notice, and it runs wherever it is called +/// from — no `cfg`. It returns rather than panics so a caller on a proving or +/// verifying path can decide; the tests call it and assert. +/// +/// `meta` is what [`ConstraintSet::meta`] returns: one entry per `emit_*` call, +/// idx-sorted, duplicates included. +pub fn check_dense_index_set( + meta: &[ConstraintMeta], + num_constraints: usize, +) -> Result<(), String> { + if meta.len() != num_constraints { + return Err(format!( + "emitted {} constraints, declared {num_constraints}", + meta.len() + )); + } + // `meta` arrives idx-sorted, so a repeat is an equal neighbour and a gap is + // a jump. Reporting both by name beats reporting "not dense". + let mut duplicates = Vec::new(); + let mut missing = Vec::new(); + let mut expected = 0usize; + let mut prev: Option = None; + for m in meta { + if prev == Some(m.constraint_idx) { + duplicates.push(m.constraint_idx); + continue; + } + prev = Some(m.constraint_idx); + while expected < m.constraint_idx { + missing.push(expected); + expected += 1; + } + if m.constraint_idx == expected { + expected += 1; + } + } + while expected < num_constraints { + missing.push(expected); + expected += 1; + } + if duplicates.is_empty() && missing.is_empty() { + return Ok(()); + } + Err(format!( + "emitted index set is not exactly 0..{num_constraints}: \ + emitted twice {duplicates:?}, never emitted {missing:?}" + )) +} + /// A [`ConstraintSet`] with no transition constraints — for tables whose /// soundness rests entirely on their bus (LogUp) interactions (e.g. BITWISE, /// PAGE, REGISTER, the continuation GLOBAL_MEMORY / global L2G sub-tables). diff --git a/crypto/stark/src/constraints/evaluator.rs b/crypto/stark/src/constraints/evaluator.rs index 9d2fdc661..7e83e68ea 100644 --- a/crypto/stark/src/constraints/evaluator.rs +++ b/crypto/stark/src/constraints/evaluator.rs @@ -43,6 +43,7 @@ where num_transition: usize, offsets: &[usize], logup_table_offset: &FieldElement, + row_stride: usize, ) -> Vec> { let is_uniform = zerofier_data.is_uniform(); let num_base = air.num_base_transition_constraints(); @@ -64,11 +65,14 @@ where // buffers are row-major) — no per-row gather copy. // Per-row evaluation, shared by the parallel and sequential paths below: // borrow the rows, evaluate transition constraints, accumulate with zerofiers. - let eval_row = |i: usize, + // `j` indexes the output (and `boundary_evaluation`); the LDE row it + // evaluates is `j * row_stride` (the whole coset at stride 1). + let eval_row = |j: usize, boundary: FieldElement, transition_buf: &mut [FieldElement], base_buf: &mut [FieldElement]| -> FieldElement { + let i = j * row_stride; let rows = RowFrame::from_lde(lde_trace, i, offsets); let ctx = TransitionEvaluationContext::new_prover( @@ -174,6 +178,12 @@ where } } + /// The composition-polynomial evaluations `H` at every `row_stride`-th + /// point of the LDE coset (`lde_size / row_stride` values; the whole coset + /// at stride 1). A stride `s` evaluates on the sub-coset `g·⟨ωˢ⟩`, which is + /// what the degree-2 split needs when the blowup leaves room (see + /// `Prover::composition_eval_stride`). + #[allow(clippy::too_many_arguments)] pub(crate) fn evaluate( &self, air: &dyn AIR, @@ -182,11 +192,17 @@ where transition_coefficients: &[FieldElement], boundary_coefficients: &[FieldElement], rap_challenges: &[FieldElement], + row_stride: usize, ) -> Vec> where Field: 'static, FieldExtension: 'static, { + let lde_len = domain.lde_roots_of_unity_coset.len(); + assert!( + row_stride >= 1 && lde_len.is_multiple_of(row_stride), + "the composition row stride must divide the LDE size" + ); let boundary_constraints = &self.boundary_constraints; // Per-step inverse zerofier vectors, cached in the (process-shared) // domain: constraints sharing a step get the same Arc. @@ -215,6 +231,7 @@ where &zerofier_data, &boundary_zerofiers_inverse_evaluations, false, + row_stride, ) { // SAFETY: the TypeId gate established `FieldExtension == @@ -246,13 +263,14 @@ where // instead of pre-computing all boundary_polys_evaluations. // This eliminates N_constraints × LDE_size intermediate allocations. #[cfg(feature = "parallel")] - let boundary_eval_iter = (0..domain.lde_roots_of_unity_coset.len()).into_par_iter(); + let boundary_eval_iter = (0..lde_len / row_stride).into_par_iter(); #[cfg(not(feature = "parallel"))] - let boundary_eval_iter = 0..domain.lde_roots_of_unity_coset.len(); + let boundary_eval_iter = 0..lde_len / row_stride; let b_constraints = &boundary_constraints.constraints; let boundary_evaluation: Vec<_> = boundary_eval_iter - .map(|domain_index| { + .map(|j| { + let domain_index = j * row_stride; b_constraints .iter() .zip(boundary_coefficients) @@ -288,6 +306,7 @@ where num_transition, offsets, &self.logup_table_offset, + row_stride, ) } @@ -309,6 +328,7 @@ where zerofier_data: &ZerofierEvaluations, boundary_z_inv: &[std::sync::Arc>>], keep: bool, + row_stride: usize, ) -> Option where Field: 'static, @@ -362,6 +382,7 @@ where // column from the process-wide resident `GpuBaseVec` cache (no // flattened host copy of num_boundary × lde_size, no re-upload). b_z_inv: boundary_z_inv, + row_stride, }; let next_step = lde_trace.lde_step_size; // == blowup_factor (single-row steps) @@ -382,9 +403,11 @@ where } /// GPU composition path keeping `H` resident on device, for the on-device - /// degree-2 decomposition. `None` → the caller runs [`Self::evaluate`] - /// (the host path) instead. + /// degree-2 decomposition, at every `row_stride`-th LDE row (as + /// [`Self::evaluate`]). `None` → the caller runs [`Self::evaluate`] (the + /// host path) instead. #[cfg(feature = "cuda")] + #[allow(clippy::too_many_arguments)] pub(crate) fn evaluate_dev( &self, air: &dyn AIR, @@ -393,6 +416,7 @@ where transition_coefficients: &[FieldElement], boundary_coefficients: &[FieldElement], rap_challenges: &[FieldElement], + row_stride: usize, ) -> Option where Field: 'static, @@ -414,6 +438,7 @@ where &zerofier_data, &boundary_zerofiers_inverse_evaluations, true, + row_stride, )? { crate::constraint_ir::gpu_interp::GpuComposition::Dev(h) => Some(h), crate::constraint_ir::gpu_interp::GpuComposition::Host(_) => None, diff --git a/crypto/stark/src/device_set.rs b/crypto/stark/src/device_set.rs new file mode 100644 index 000000000..23d61ec44 --- /dev/null +++ b/crypto/stark/src/device_set.rs @@ -0,0 +1,496 @@ +//! The device working set of one table, term by term — and the pure admission +//! predicate over it. +//! +//! Two consumers read this model and must agree: the per-table scheduler's VRAM +//! throttle (`prover::VramGate`, which bounds the SUM of the tables proved +//! concurrently and runs on every build, GPU or not) and the GPU dispatch +//! layer's admission (`gpu_lde::admit`, which asks whether ONE table fits the +//! card at all). Keeping the arithmetic here, free of any `cuda` gate, is what +//! lets both read one model instead of each carrying a copy. +//! +//! Every term mirrors an allocation in `math_cuda`, with ONE LDE buffer, never +//! two: the column-major engine (`math_cuda::lde_cm`, the default) writes the +//! LDE in the layout the rounds read, and the legacy path (host-copy tables, +//! `LAMBDA_VM_LDE_LEGACY=1`) transposes its row-major LDE in place (#956); +//! `one_lde_buffer::vram_arm` measures the commit's three big terms and the doc on +//! `DEFAULT_MEMPOOL_RELEASE_THRESHOLD_BYTES` records the numbers. The model +//! carries no blanket safety factor: the card's admission budget is 80% of +//! device memory, and the 20% it leaves is what the context, the module code +//! and the retained pool live in. + +/// Bytes per Goldilocks element on device. +pub const BASE_BYTES: u64 = 8; + +/// Bytes per ext3 element on device — three adjacent base columns. +pub const EXT3_BYTES: u64 = 3 * BASE_BYTES; + +/// Bytes of one Merkle node. Every commitment hash the device dispatches on +/// emits a 32-byte digest — a four-felt Goldilocks digest is exactly 32 +/// canonical bytes — so the node buffer costs the same under every hash. +pub const MERKLE_NODE_BYTES: u64 = 32; + +/// Cap on the in-place transpose's device scratch, mirrored from +/// `math_cuda::lde::INPLACE_TRANSPOSE_SCRATCH_BYTES` (private there). The +/// admission wants a bound, not the block geometry. It also bounds the +/// column-major engine's scratch: one chunk of trace columns, at most 60% of +/// L2 (~58 MB on a 96 MB card). +pub const INPLACE_TRANSPOSE_SCRATCH_CAP_BYTES: u64 = 256 << 20; + +/// `(2 · leaves − 1) · 32` for the row-pair tree over `lde_size` rows. +pub const fn full_tree_bytes(lde_size: u64) -> u64 { + lde_size.saturating_sub(1).saturating_mul(MERKLE_NODE_BYTES) +} + +/// `(2 · leaves − 1) · 32` for the tree over `lde_size` rows with +/// `rows_per_leaf` rows per leaf: [`full_tree_bytes`] at 2 (today's row pair), +/// `(2 · lde − 1) · 32` — twice the leaves, about twice the bytes — at 1 (the +/// S2 one-row tree). +pub const fn tree_bytes_for(lde_size: u64, rows_per_leaf: u64) -> u64 { + if rows_per_leaf <= 1 { + lde_size + .saturating_mul(2) + .saturating_sub(1) + .saturating_mul(MERKLE_NODE_BYTES) + } else { + full_tree_bytes(lde_size) + } +} + +/// Bytes of `cols` ext3 columns over `rows` rows. +pub const fn ext3_bytes(rows: u64, cols: u64) -> u64 { + rows.saturating_mul(cols).saturating_mul(EXT3_BYTES) +} + +/// Bytes of `cols` base columns over `rows` rows. +pub const fn base_bytes(rows: u64, cols: u64) -> u64 { + rows.saturating_mul(cols).saturating_mul(BASE_BYTES) +} + +/// The device working set one fused row-major commit allocates, term by term +/// (`math_cuda::lde::coset_lde_row_major_inner`): ONE LDE buffer, the optional +/// trace-domain snapshot, the full Merkle node buffer, and the small scratch +/// (coset weights plus the capped transpose scratch). Both forms of the commit +/// fit it: the column-major engine's chunk scratch sits inside the transpose +/// cap the legacy form needs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CommitDeviceSet { + /// `lde_size · base_cols · 8`: the LDE — written column-major by the + /// engine, or row-major and transposed in place by the legacy path. + pub lde_bytes: u64, + /// `n · base_cols · 8`: the pre-NTT column-major snapshot the LogUp + /// fingerprint kernel reads in place (main commits only). + pub snapshot_bytes: u64, + /// `(2 · leaves − 1) · 32` with `leaves = lde_size / 2`: one full row-pair + /// tree. The preprocessed split path builds two, sequentially on one + /// stream — the precomputed tree is downloaded and freed before the + /// multiplicity tree is allocated — so one is the peak there too. + pub tree_bytes: u64, + /// Coset weights (`n · 8`) plus the transpose scratch cap, which also + /// bounds the engine's chunk scratch. + pub scratch_bytes: u64, +} + +impl CommitDeviceSet { + pub const fn total(&self) -> u64 { + self.lde_bytes + .saturating_add(self.snapshot_bytes) + .saturating_add(self.tree_bytes) + .saturating_add(self.scratch_bytes) + } +} + +/// Size one fused commit's device set. `base_cols` counts BASE-FIELD columns: +/// `m` for a base table, `3m` for an ext3 one (the ext3 row-major layout is +/// three adjacent base columns per element). `snapshot` is whether the +/// trace-domain column-major snapshot is retained (the main commits do, the +/// aux commits do not). +pub fn commit_device_set( + n: usize, + base_cols: usize, + blowup: usize, + snapshot: bool, +) -> CommitDeviceSet { + commit_device_set_rpl(n, base_cols, blowup, snapshot, 2) +} + +/// [`commit_device_set`] for a tree with `rows_per_leaf` rows per leaf (2 = +/// row pair, 1 = the S2 one-row tree, whose node buffer is twice as large). +pub fn commit_device_set_rpl( + n: usize, + base_cols: usize, + blowup: usize, + snapshot: bool, + rows_per_leaf: usize, +) -> CommitDeviceSet { + let n = n as u64; + let cols = base_cols as u64; + let lde = n.saturating_mul(blowup as u64); + CommitDeviceSet { + lde_bytes: base_bytes(lde, cols), + snapshot_bytes: if snapshot { base_bytes(n, cols) } else { 0 }, + tree_bytes: tree_bytes_for(lde, rows_per_leaf as u64), + scratch_bytes: n + .saturating_mul(BASE_BYTES) + .saturating_add(INPLACE_TRANSPOSE_SCRATCH_CAP_BYTES), + } +} + +/// The shape the rounds-2–4 model takes: what the AIR and the domain fix +/// before any device work starts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TableShape { + /// Trace rows (the interpolation domain). + pub n: usize, + pub blowup: usize, + /// Base-field main columns, preprocessed ones included. + pub main_cols: usize, + /// Ext3 aux (LogUp) columns. + pub aux_cols: usize, + /// Composition-polynomial parts: `composition_poly_degree_bound(n) / n`. + pub num_parts: usize, + /// OOD evaluation points per trace column: + /// `transition_offsets.len() · step_size`. + pub num_eval_points: usize, +} + +/// The device set of one table across rounds 2–4, term by term — what the +/// scheduler's throttle admits a table's fused task against. Everything the +/// main commit left resident stays counted (the LDE, its snapshot, its tree), +/// and each later round adds what it allocates on top: +/// +/// - R1 aux: the aux LDE (`lde · aux · 24`), the resident aux trace the LogUp +/// build left behind (`n · (aux + 1) · 24`, one extra column for the running +/// sum), and the aux tree; +/// - R2: `H` (`lde · 24`), the parts (`num_parts · lde · 24`) — `H` is still +/// alive while they are decomposed — and the parts tree; +/// - R3: the inverted denominators on the trace domain (`k · n · 24`); +/// - R4: the inverted denominators on the LDE (`(1 + k) · lde · 24`), the DEEP +/// codeword (`lde · 24`), the FRI layer chain (geometric, bounded by one more +/// codeword) and its trees (bounded by one full tree). +/// +/// Rounds do not overlap inside one table, so this is an upper bound on any +/// instant of the task, not a sum of the rounds' peaks; the counted R3/R4 +/// transients are small next to the resident LDEs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TableDeviceSet { + pub main: CommitDeviceSet, + pub aux_bytes: u64, + pub composition_bytes: u64, + pub deep_fri_bytes: u64, +} + +impl TableDeviceSet { + pub const fn total(&self) -> u64 { + self.main + .total() + .saturating_add(self.aux_bytes) + .saturating_add(self.composition_bytes) + .saturating_add(self.deep_fri_bytes) + } +} + +/// Size one table's rounds-2–4 device set for `shape`. +pub fn table_device_set(shape: TableShape) -> TableDeviceSet { + table_device_set_rpl(shape, 2) +} + +/// [`table_device_set`] for a table whose trace trees (main, aux, +/// composition) carry `rows_per_leaf` rows per leaf: at 1 (S2) every trace +/// tree is the one-row tree, about twice the node bytes. The FRI trees double +/// their bound too: under one row the chain starts at the LDE itself (the +/// input tree over the DEEP codeword, `lde / 2^{d_0}` leaves), so the layer +/// trees together hold fewer than `2 · lde` nodes, which is +/// [`tree_bytes_for`]`(lde, 1)`. +pub fn table_device_set_rpl(shape: TableShape, rows_per_leaf: usize) -> TableDeviceSet { + let TableShape { + n, + blowup, + main_cols, + aux_cols, + num_parts, + num_eval_points, + } = shape; + let main = commit_device_set_rpl(n, main_cols, blowup, true, rows_per_leaf); + let rpl = rows_per_leaf as u64; + let (n, k, aux, parts) = ( + n as u64, + num_eval_points as u64, + aux_cols as u64, + num_parts as u64, + ); + let lde = n.saturating_mul(blowup as u64); + let aux_bytes = if aux == 0 { + 0 + } else { + ext3_bytes(lde, aux) + .saturating_add(ext3_bytes(n, aux + 1)) + .saturating_add(tree_bytes_for(lde, rpl)) + }; + let composition_bytes = if parts == 0 { + 0 + } else { + ext3_bytes(lde, 1 + parts).saturating_add(tree_bytes_for(lde, rpl)) + }; + let deep_fri_bytes = ext3_bytes(n, k) + .saturating_add(ext3_bytes(lde, 1 + k)) + .saturating_add(ext3_bytes(lde, 2)) + .saturating_add(tree_bytes_for(lde, rpl)); + TableDeviceSet { + main, + aux_bytes, + composition_bytes, + deep_fri_bytes, + } +} + +// This module is the SIZE model and nothing else: what a stage puts on the +// card, so the gate can decide whether it fits. It is deliberately not the +// prover's table walk. The walk is a scheduling policy, keyed on a weight that +// is kept for the order it produces rather than for any byte it names, and it +// lives with the scheduler in `prover::table_walk_weight`. Sorting the walk by +// this model instead cost a measured 5.9 GiB of host peak at the q=20 wrap. + +/// What the admission predicate decided for one dispatch. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Admission { + /// No CUDA backend — no GPU, or cubins that would not load. The host path + /// is the only one; `math_cuda::device::backend` already warned once. A + /// GPU-less host is not the production pipeline, so this is not an abort. + NoDevice, + /// Below the launch-overhead floor: the host path is the faster one. + BelowFloor { lde_size: usize, floor: usize }, + /// Fits the card's admission budget. + Admitted { bytes: u64, budget: u64 }, + /// Does not fit the card even alone. + OverBudget { bytes: u64, budget: u64 }, +} + +impl Admission { + pub const fn is_admitted(&self) -> bool { + matches!(self, Admission::Admitted { .. }) + } +} + +/// The pure predicate, floor and budget supplied. The row floor is checked +/// first — a table below it never asks the device for anything, whatever its +/// width — then the bytes ceiling. +pub const fn admit_bytes(lde_size: usize, bytes: u64, floor: usize, budget: u64) -> Admission { + if lde_size < floor { + return Admission::BelowFloor { lde_size, floor }; + } + if bytes > budget { + return Admission::OverBudget { bytes, budget }; + } + Admission::Admitted { bytes, budget } +} + +/// The admission arithmetic, with the floor and the budget supplied: no +/// device, no backend, pure numbers. +#[cfg(test)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + /// `detect_vram_budget_bytes` on a 32 GiB card: 80% of the total. + const CARD_32_GIB_BUDGET: u64 = 32 * GIB / 5 * 4; + /// The dispatch layer's row floor (`gpu_lde::DEFAULT_GPU_LDE_THRESHOLD`). + const FLOOR: usize = 1 << 14; + + /// S2: a one-row tree has twice the leaves, so its node + /// buffer is `(2·lde − 1)·32` against the row pair's `(lde − 1)·32` — + /// +`lde·32` bytes per tree (128 MiB at an LDE of 2^22) — and + /// the table device set grows by that per trace tree plus the FRI bound; + /// the default (`rows_per_leaf = 2`) is the old model exactly. + #[test] + fn a_one_row_tree_doubles_the_node_buffer_and_the_default_is_unchanged() { + let lde: u64 = 1 << 22; + assert_eq!(tree_bytes_for(lde, 2), full_tree_bytes(lde)); + assert_eq!(tree_bytes_for(lde, 1), (2 * lde - 1) * MERKLE_NODE_BYTES); + assert_eq!(tree_bytes_for(lde, 1) - tree_bytes_for(lde, 2), lde * 32); + assert_eq!(tree_bytes_for(lde, 1) - tree_bytes_for(lde, 2), 128 << 20); + + let n = 1usize << 20; + assert_eq!( + commit_device_set_rpl(n, 49, 4, true, 2), + commit_device_set(n, 49, 4, true) + ); + let one = commit_device_set_rpl(n, 49, 4, true, 1); + assert_eq!(one.tree_bytes, tree_bytes_for(lde, 1)); + assert_eq!( + one.total() - commit_device_set(n, 49, 4, true).total(), + lde * 32 + ); + + let shape = TableShape { + n, + blowup: 4, + main_cols: 49, + aux_cols: 13, + num_parts: 2, + num_eval_points: 2, + }; + assert_eq!(table_device_set_rpl(shape, 2), table_device_set(shape)); + // Main, aux and composition trees, and the FRI tree bound: four + // node buffers, each `lde · 32` larger. + assert_eq!( + table_device_set_rpl(shape, 1).total() - table_device_set(shape).total(), + 4 * lde * 32 + ); + } + + /// The synthetic over-budget table: 2^22 rows x 612 columns at blowup 2. + /// Its LDE alone is 38.25 GiB; with the snapshot and the tree the commit's + /// device set is 57.9 GiB against a 25.6 GiB budget. + #[test] + fn the_over_budget_shape_is_over_budget() { + let n = 1usize << 22; + let set = commit_device_set(n, 612, 2, true); + assert_eq!(set.lde_bytes, (n as u64) * 2 * 612 * 8); + assert_eq!(set.snapshot_bytes, (n as u64) * 612 * 8); + assert_eq!(set.tree_bytes, ((n as u64) * 2 - 1) * 32); + assert!(set.lde_bytes > 38 * GIB && set.lde_bytes < 39 * GIB); + assert!(set.total() > 57 * GIB && set.total() < 58 * GIB); + assert!(matches!( + admit_bytes(n * 2, set.total(), FLOOR, CARD_32_GIB_BUDGET), + Admission::OverBudget { .. } + )); + } + + /// LFM_HASH under RPO — 2^21 rows x (436 value + 13 preprocessed) columns — + /// fits the card at blowup 2 with one LDE buffer and does not at blowup 4. + #[test] + fn lfm_hash_rpo_fits_at_blowup_2_and_not_at_4() { + let n = 1usize << 21; + let b2 = commit_device_set(n, 449, 2, true); + assert!(b2.total() > 21 * GIB && b2.total() < 22 * GIB, "{b2:?}"); + assert!(admit_bytes(n * 2, b2.total(), FLOOR, CARD_32_GIB_BUDGET).is_admitted()); + let b4 = commit_device_set(n, 449, 4, true); + assert!(b4.total() > 35 * GIB && b4.total() < 36 * GIB, "{b4:?}"); + assert!(matches!( + admit_bytes(n * 4, b4.total(), FLOOR, CARD_32_GIB_BUDGET), + Admission::OverBudget { .. } + )); + } + + /// The whole-table set of LFM_HASH under RPO at blowup 2 — 3 aux columns, + /// two parts, two eval points — is ~23 GiB: it proves alone inside the + /// budget, and nothing else proves beside it. The old throttle model + /// (two LDE buffers plus 256 B per LDE row) put the same table at + /// 28.3 GiB, over the budget it is actually under. + #[test] + fn lfm_hash_rpo_whole_table_set() { + let shape = TableShape { + n: 1 << 21, + blowup: 2, + main_cols: 449, + aux_cols: 3, + num_parts: 2, + num_eval_points: 2, + }; + let set = table_device_set(shape); + assert!(set.total() > 23 * GIB && set.total() < 24 * GIB, "{set:?}"); + assert!(set.total() <= CARD_32_GIB_BUDGET); + assert!(2 * set.total() > CARD_32_GIB_BUDGET); + let old_model = (1u64 << 22) * (449 * 8 + 3 * 24) * 2 + (1u64 << 22) * 256; + assert!(old_model > CARD_32_GIB_BUDGET); + } + + /// One LFM_BALU chunk of 2^22 rows (14 main, 2 aux, 2 parts, 2 points) is + /// ~4.9 GiB under this model — the R1 set is 1.8 GiB; the resident aux + /// trace, `H`, the denominators, DEEP and FRI add the rest — so five prove + /// concurrently inside the budget. The sizing behind + /// `prover::lfm::chunking::BaluChunking`, whose test calls this model. + #[test] + fn a_balu_chunk_is_five_gib() { + let set = table_device_set(TableShape { + n: 1 << 22, + blowup: 2, + main_cols: 14, + aux_cols: 2, + num_parts: 2, + num_eval_points: 2, + }); + assert!( + set.main.total() > GIB + GIB / 2 && set.main.total() < 2 * GIB, + "{set:?}" + ); + assert!( + set.total() > 4 * GIB + GIB / 2 && set.total() < 5 * GIB + GIB / 2, + "{set:?}" + ); + assert!(5 * set.total() <= CARD_32_GIB_BUDGET); + assert!(6 * set.total() > CARD_32_GIB_BUDGET); + } + + /// A table without aux or parts (d=1 with no lookups) counts only what it + /// allocates. + #[test] + fn absent_rounds_cost_nothing() { + let set = table_device_set(TableShape { + n: 1 << 16, + blowup: 2, + main_cols: 8, + aux_cols: 0, + num_parts: 0, + num_eval_points: 1, + }); + assert_eq!(set.aux_bytes, 0); + assert_eq!(set.composition_bytes, 0); + assert!(set.deep_fri_bytes > 0); + } + + /// The aux commit has no snapshot; its ext3 columns count as three base + /// columns each. + #[test] + fn aux_sets_have_no_snapshot() { + let set = commit_device_set(1 << 20, 3 * 3, 2, false); + assert_eq!(set.snapshot_bytes, 0); + assert_eq!(set.lde_bytes, ext3_bytes(1 << 21, 3)); + } + + /// The row floor is checked before the ceiling: a tiny table with an + /// absurd byte count is "too small", never "over budget" — it will not ask + /// the device for anything. + #[test] + fn the_floor_is_checked_before_the_budget() { + assert!(matches!( + admit_bytes(1 << 13, u64::MAX, FLOOR, CARD_32_GIB_BUDGET), + Admission::BelowFloor { .. } + )); + assert!(matches!( + admit_bytes(FLOOR, u64::MAX, FLOOR, CARD_32_GIB_BUDGET), + Admission::OverBudget { .. } + )); + } + + /// FRI re-derives admission at width 1: a narrow transient over a large + /// domain always clears the ceiling. The floor stays a row count, so it + /// does not degenerate there either. + #[test] + fn fri_at_width_one_never_degenerates() { + let n0 = 1usize << 24; + let bytes = ext3_bytes(n0 as u64, 1) + full_tree_bytes(n0 as u64); + assert!(admit_bytes(n0, bytes, FLOOR, CARD_32_GIB_BUDGET).is_admitted()); + } + + /// A budget of `u64::MAX` (query failed) makes the ceiling inert — the + /// floor alone decides, which is the pre-admission behaviour. + #[test] + fn an_unbounded_budget_is_inert() { + assert!(admit_bytes(1 << 20, u64::MAX - 1, FLOOR, u64::MAX).is_admitted()); + } + + /// The table's committed width is what the model takes: the row floor is + /// width-blind on purpose, the ceiling is not. + #[test] + fn width_moves_the_ceiling_not_the_floor() { + let n = 1usize << 21; + let narrow = commit_device_set(n, 4, 2, true); + let wide = commit_device_set(n, 612, 2, true); + assert!(admit_bytes(n * 2, narrow.total(), FLOOR, CARD_32_GIB_BUDGET).is_admitted()); + assert!(matches!( + admit_bytes(n * 2, wide.total(), FLOOR, CARD_32_GIB_BUDGET), + Admission::OverBudget { .. } + )); + } +} diff --git a/crypto/stark/src/domain.rs b/crypto/stark/src/domain.rs index 208d0ae58..490c18993 100644 --- a/crypto/stark/src/domain.rs +++ b/crypto/stark/src/domain.rs @@ -65,6 +65,13 @@ pub struct Domain { } impl Domain { + /// The bytes its two root tables hold (the values built on first use come + /// later and are not counted). + pub(crate) fn heap_bytes(&self) -> usize { + (self.lde_roots_of_unity_coset.capacity() + self.trace_roots_of_unity.capacity()) + * std::mem::size_of::>() + } + /// Builds the interpolation and LDE domains used by the prover. /// /// - Interpolation domain: the `trace_length` roots of unity (must be a power of 2). @@ -74,8 +81,17 @@ impl Domain { where A: AIR + ?Sized, { - let blowup_factor = air.options().blowup_factor as usize; - let coset_offset = FieldElement::from(air.options().coset_offset); + Self::from_options( + air.options().blowup_factor as usize, + air.options().coset_offset, + trace_length, + ) + } + + /// [`Domain::new`] from what it reads of the AIR: its blowup factor and + /// coset offset. + pub fn from_options(blowup_factor: usize, coset_offset: u64, trace_length: usize) -> Self { + let coset_offset = FieldElement::from(coset_offset); let root_order = trace_length.trailing_zeros(); let trace_primitive_root = F::get_primitive_root_of_unity(root_order as u64).unwrap(); let trace_roots_of_unity = get_powers_of_primitive_root_coset( @@ -130,6 +146,11 @@ impl Domain { v } + /// Whether [`Self::boundary_zerofier_inv`] has `step`'s inverse cached. + pub(crate) fn has_boundary_zerofier_inv(&self, step: usize) -> bool { + self.boundary_z_inv.lock().unwrap().contains_key(&step) + } + /// Barycentric OOD constants (round 3), computed once per domain. pub fn ood_constants(&self) -> &DomainConstants { self.ood_constants diff --git a/crypto/stark/src/examples/bus_permutation.rs b/crypto/stark/src/examples/bus_permutation.rs new file mode 100644 index 000000000..c4f54291f --- /dev/null +++ b/crypto/stark/src/examples/bus_permutation.rs @@ -0,0 +1,117 @@ +//! A self-balancing LogUp table built on [`AirWithBuses`] — the AIR type every +//! production VM table is — for tests that must prove on the DEVICE. +//! +//! The CUDA composition arm evaluates constraints from the AIR's captured IR +//! (`AIR::constraint_program`). `AirWithBuses` supplies it; the hand-written +//! example AIRs (`LogReadOnlyRAP`, `FibonacciRAP`, …) do not, and the trait's +//! default panics by design. A test whose trace crosses the GPU LDE threshold +//! therefore needs an AIR like this one, whatever it is testing. +//! +//! Layout: four main columns `a, b, c, d` with `b` a permutation of `a` and `d` +//! a permutation of `c`; four interactions (`a` sent and `b` received on one +//! bus, `c` sent and `d` received on another), so the aux trace has one +//! committed term column and the accumulated column. The table balances on its +//! own: its bus contribution is zero, which is what a single-table verify +//! expects. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; + +use crate::constraints::builder::EmptyConstraints; +use crate::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, Multiplicity, + NullBoundaryConstraintBuilder, Packing, +}; +use crate::proof::options::ProofOptions; +use crate::trace::TraceTable; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type FE = FieldElement; + +/// The AIR: no table constraints of its own, the LogUp ones from the framework. +pub type BusPermutationAir = + AirWithBuses; + +const BUS_AB: u64 = 1; +const BUS_CD: u64 = 2; + +/// The AIR under `options`. +pub fn bus_permutation_air(options: &ProofOptions) -> BusPermutationAir { + let one = |col: usize| Packing::Direct.columns(&[col]); + AirWithBuses::new( + 4, + AuxiliaryTraceBuildData { + interactions: vec![ + BusInteraction::sender(BUS_AB, Multiplicity::One, one(0)), + BusInteraction::receiver(BUS_AB, Multiplicity::One, one(1)), + BusInteraction::sender(BUS_CD, Multiplicity::One, one(2)), + BusInteraction::receiver(BUS_CD, Multiplicity::One, one(3)), + ], + }, + options, + 1, + EmptyConstraints, + ) +} + +/// A `rows`-row trace (`rows` a power of two, at least 2): `b` is `a` +/// reversed, `d` is `c` rotated by one row. +pub fn bus_permutation_trace(rows: usize) -> TraceTable { + assert!( + rows.is_power_of_two() && rows >= 2, + "rows must be a power of two ≥ 2" + ); + let a: Vec = (0..rows as u64).map(|i| FE::from(i + 1)).collect(); + let b: Vec = a.iter().rev().cloned().collect(); + let c: Vec = (0..rows as u64) + .map(|i| FE::from((i * 7919) % 4099 + 1)) + .collect(); + let d: Vec = (0..rows).map(|i| c[(i + 1) % rows]).collect(); + TraceTable::from_columns_main(vec![a, b, c, d], 1) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prover::{IsStarkProver, Prover}; + use crate::traits::AIR; + use crate::verifier::{IsStarkVerifier, Verifier}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + /// The two properties the device tests rely on: the AIR hands out a + /// constraint program (the CUDA composition arm's input), and an honest + /// trace proves and verifies as a single table (the bus balances to zero). + #[test] + fn the_bus_permutation_table_round_trips_and_has_a_constraint_program() { + let options = ProofOptions::default_test_options(); + let air = bus_permutation_air(&options); + assert!(!air.constraint_program().nodes.is_empty()); + let mut trace = bus_permutation_trace(64); + let proof = Prover::prove(&air, &mut trace, &(), &mut DefaultTranscript::::new(&[])) + .expect("an honest trace proves"); + assert!(proof.lde_trace_aux_merkle_root.is_some(), "a LogUp table"); + assert!(Verifier::verify( + &proof, + &air, + &mut DefaultTranscript::::new(&[]) + )); + } + + /// Non-vacuity of the balance: `d` no longer a permutation of `c` makes the + /// table's bus contribution non-zero, and the single-table verify refuses. + #[test] + fn an_unbalanced_bus_permutation_table_is_rejected() { + let options = ProofOptions::default_test_options(); + let air = bus_permutation_air(&options); + let mut trace = bus_permutation_trace(64); + trace.set_main(5, 3, FE::from(999_999u64)); + let rejected = + match Prover::prove(&air, &mut trace, &(), &mut DefaultTranscript::::new(&[])) { + Err(_) => true, + Ok(proof) => !Verifier::verify(&proof, &air, &mut DefaultTranscript::::new(&[])), + }; + assert!(rejected, "an unbalanced table must not verify"); + } +} diff --git a/crypto/stark/src/examples/mod.rs b/crypto/stark/src/examples/mod.rs index 770540e83..14c4d001b 100644 --- a/crypto/stark/src/examples/mod.rs +++ b/crypto/stark/src/examples/mod.rs @@ -1,3 +1,4 @@ +pub mod bus_permutation; pub mod dummy_air; pub mod fibonacci_2_cols_shifted; pub mod fibonacci_2_columns; diff --git a/crypto/stark/src/examples/read_only_memory_logup.rs b/crypto/stark/src/examples/read_only_memory_logup.rs index 9068e7276..ef4696201 100644 --- a/crypto/stark/src/examples/read_only_memory_logup.rs +++ b/crypto/stark/src/examples/read_only_memory_logup.rs @@ -3,13 +3,15 @@ //! use std::marker::PhantomData; +use std::sync::OnceLock; use crate::{ + constraint_ir::ConstraintProgram, constraints::{ boundary::{BoundaryConstraint, BoundaryConstraints}, builder::{ - ConstraintBuilder, ConstraintMeta, ConstraintSet, RowDomain, num_base_from_meta, - run_transition_prover, run_transition_verifier, + CaptureBuilder, ConstraintBuilder, ConstraintMeta, ConstraintSet, RowDomain, + num_base_from_meta, run_transition_prover, run_transition_verifier, }, }, context::AirContext, @@ -96,6 +98,11 @@ where { context: AirContext, meta: Vec, + /// The captured IR of [`LogReadOnlyRAPConstraints`], built on first use by + /// [`AIR::constraint_program`] (the CUDA composition arm needs it once main + /// and aux are device-resident). Same body as the folders, so the device + /// evaluates the same polynomials as the CPU path. + program: OnceLock>, phantom: PhantomData<(F, E)>, } @@ -148,6 +155,7 @@ where Self { context, meta, + program: OnceLock::new(), phantom: PhantomData, } } @@ -279,6 +287,16 @@ where num_base_from_meta(&ConstraintSet::::meta(&LogReadOnlyRAPConstraints)) } + fn constraint_program(&self) -> &ConstraintProgram { + // Prover/GPU/tests only (the verify path never calls this): capture + // the single constraint body once. + self.program.get_or_init(|| { + let mut cb = CaptureBuilder::::new(); + LogReadOnlyRAPConstraints.eval(&mut cb); + cb.finish(num_base_from_meta(&self.meta)).0 + }) + } + fn context(&self) -> &AirContext { &self.context } diff --git a/crypto/stark/src/fri/capture.rs b/crypto/stark/src/fri/capture.rs new file mode 100644 index 000000000..b875c954b --- /dev/null +++ b/crypto/stark/src/fri/capture.rs @@ -0,0 +1,75 @@ +//! Test-only capture of the verifier's FRI challenges and DEEP values, for the +//! exported test vectors (`tests/vectors/zf_fri`, the README's (d)): a vector +//! carries a proof AND the ζ, ι and DEEP values a correct verifier derives +//! from it, so the device prover and the in-guest verifier can check each stage separately. +//! +//! Compiled only for tests and the `test-utils` feature. Thread-local: the +//! host verifier is sequential on the calling thread, so [`capture`] sees +//! exactly the verification it wraps. + +use core::any::Any; +use core::cell::RefCell; +use std::vec::Vec; + +use math::field::element::FieldElement; +use math::field::traits::IsField; + +/// What one table's verification derived: ζ (every folding challenge), ι +/// (the query pair indices) and the DEEP values p₀(υ), p₀(−υ) per query. +#[derive(Clone, Debug)] +pub struct FriCapture { + pub zetas: Vec>, + pub iotas: Vec, + pub deep: Vec>, + pub deep_sym: Vec>, +} + +thread_local! { + static ACTIVE: RefCell>>> = const { RefCell::new(None) }; +} + +/// Run `f` (a verification) and return its result with one record per table +/// verified, in order. Records are `FriCapture` for the proof's extension +/// field; downcast with [`FriCapture::from_any`]. +pub fn capture(f: impl FnOnce() -> T) -> (T, Vec>) { + ACTIVE.with(|a| *a.borrow_mut() = Some(Vec::new())); + let out = f(); + let records = ACTIVE.with(|a| a.borrow_mut().take()).unwrap_or_default(); + (out, records) +} + +impl FriCapture { + pub fn from_any(record: &dyn Any) -> Option<&Self> { + record.downcast_ref::() + } +} + +/// Start a table's record with its challenges (called after the replay). +pub(crate) fn record_challenges(zetas: &[FieldElement], iotas: &[usize]) { + ACTIVE.with(|a| { + if let Some(records) = a.borrow_mut().as_mut() { + records.push(Box::new(FriCapture:: { + zetas: zetas.to_vec(), + iotas: iotas.to_vec(), + deep: Vec::new(), + deep_sym: Vec::new(), + })); + } + }); +} + +/// Add the DEEP values to the current table's record. +pub(crate) fn record_deep( + deep: &[FieldElement], + deep_sym: &[FieldElement], +) { + ACTIVE.with(|a| { + if let Some(records) = a.borrow_mut().as_mut() + && let Some(last) = records.last_mut() + && let Some(rec) = last.downcast_mut::>() + { + rec.deep = deep.to_vec(); + rec.deep_sym = deep_sym.to_vec(); + } + }); +} diff --git a/crypto/stark/src/fri/device_parity.rs b/crypto/stark/src/fri/device_parity.rs new file mode 100644 index 000000000..b7834ed53 --- /dev/null +++ b/crypto/stark/src/fri/device_parity.rs @@ -0,0 +1,443 @@ +//! Device-vs-host parity for the FRI commit and query phases under a proof +//! format's fold layout — the S3 group encoding and today's pair encoding. +//! +//! Compiled for `cuda` builds with tests or `test-utils`; every entry needs a +//! GPU and a lowered `LAMBDA_VM_GPU_LDE_THRESHOLD` (the device commit admits +//! only LDEs at or above it), so the callers are `#[ignore]`d box tests. The +//! stark crate instantiates them under Keccak and Blake3, the prover crate +//! under the production RPX pin (`tests::zf_rpx_device_tests`). +//! +//! What one [`fri_parity`] call pins, device against the host CPU loop +//! (`commit_phase_cpu_with_layout` / the host query walks) over one random +//! codeword and one transcript: +//! - the terminal coefficients, every committed layer's root, and (when the +//! device drained them) every layer's evaluations; +//! - the transcript after the commit phase (one more sampled element); +//! - per query (random pair indices plus both ends of the range) every +//! opened value — the whole group under the group encoding, the sibling +//! under the pair one — and every authentication path. + +use std::format; +use std::string::String; +use std::vec::Vec; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::CapPolicy; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; + +use crate::config::StarkHash; +use crate::fri::fri_commitment::FriLayer; +use crate::fri::fri_decommit::FriDecommitment; +use crate::fri::fri_functions::compute_coset_twiddles_inv; +use crate::fri::schedule::{FRI_SCHEDULE_DMAX, fri_chain_start, fri_schedule}; +use crate::fri::terminal::FriFoldLayout; +use crate::fri::vectors::splitmix64; +use crate::proof::options::{FriMode, FriScheduleOverride, OneRowMode, ProofFormat, ProofOptions}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; +type Ext = FieldElement; + +/// Uneven and extreme shapes the DP does not pick at production sizes but the +/// format admits (override): `d = 6` (DMAX, a 64-value group, a multi-block +/// leaf under every hash), and unequal neighbours (a fold-count off-by-one +/// between the commit and the pending folds shows only there). +pub const EXTRA_SHAPES: &[&[u8]] = &[ + // A lone 16-group layer: a DP schedule until the objective priced every + // emitted row, kept so the sweep's coverage does not shrink. + &[4], + &[6], + &[1, 6], + &[6, 1], + &[3, 1, 3], + &[1, 3], + &[2, 5, 1], + &[1, 1, 1], +]; + +/// Every distinct fold schedule the DP produces for an LDE of `2^B`, `B ≤ 23` +/// (the S3 chain from `B − 1`), at terminal logs 4 (the vectors), 9 (base +/// legs) and 10 (LFM proofs), 3 and 110 queries, cap off and auto; then +/// [`EXTRA_SHAPES`]. Empty schedules (nothing committed) are skipped: the +/// device commit declines them before sampling. +pub fn dp_shapes() -> Vec> { + let mut out: Vec> = Vec::new(); + for t in [4u32, 9, 10] { + for q in [3u64, 110] { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + for b in 2..=23u32 { + let b0 = fri_chain_start(b, false); + let s = fri_schedule(b0, t.min(b), q, cap, FRI_SCHEDULE_DMAX); + if !s.is_empty() && !out.contains(&s) { + out.push(s); + } + } + } + } + } + for s in EXTRA_SHAPES { + if !out.iter().any(|x| x.as_slice() == *s) { + out.push(s.to_vec()); + } + } + out +} + +/// Options for a group-encoded (dp) proof at blowup `2^blowup_log`, terminal +/// `k`, `queries` queries and cap policy `cap` (which the DP's objective +/// reads), with an optional explicit schedule. +pub fn dp_options( + blowup_log: u32, + k: u8, + queries: usize, + cap: CapPolicy, + schedule: Option<&[u8]>, +) -> ProofOptions { + ProofOptions { + blowup_factor: 1u8 << blowup_log, + fri_number_of_queries: queries, + coset_offset: 3, + grinding_factor: 0, + fri_final_poly_log_degree: k, + format: ProofFormat { + merkle_cap: cap, + fri_mode: FriMode::Dp, + fri_schedule_override: schedule + .map(|s| FriScheduleOverride::new(s).expect("override fits")), + ..ProofFormat::DEFAULT + }, + } +} + +/// Today's options (pair encoding) at the same shape parameters. +pub fn pair_options(blowup_log: u32, k: u8) -> ProofOptions { + ProofOptions { + blowup_factor: 1u8 << blowup_log, + fri_number_of_queries: 3, + coset_offset: 3, + grinding_factor: 0, + fri_final_poly_log_degree: k, + format: ProofFormat::DEFAULT, + } +} + +/// The smallest `(lde_log, options)` whose layout at blowup 2, `k = 1` (so a +/// terminal of 4) has exactly `schedule` as its committed folds. +pub fn smallest_case(schedule: &[u8]) -> (u32, ProofOptions) { + let sum: u32 = schedule.iter().map(|&d| u32::from(d)).sum(); + // b0 = lde_log − 1 = terminal_log + Σd, terminal_log = 1 + 1. + (sum + 3, dp_options(1, 1, 3, CapPolicy::Off, Some(schedule))) +} + +fn raw(v: &[Ext]) -> Vec<[u64; 3]> { + v.iter() + .map(|e| { + let c = e.value(); + [c[0].canonical(), c[1].canonical(), c[2].canonical()] + }) + .collect() +} + +/// Device-vs-host parity of the FRI commit and query phases under +/// `options`' fold layout for a random codeword of `2^lde_log` values. +/// `resident` keeps the device layers' evals resident only (the device-only +/// envelope's shape), so the device query phase gathers them on device. +/// +/// `options.format.one_row == On` runs the S2 layout: layer 0 is +/// the input tree committed from the codeword itself before any challenge, +/// and the query indexes range over the whole LDE (`Auto` is resolved per +/// table from an AIR, so it is not a codeword-level case: treated as off). +/// +/// `Err` names the first mismatch, or the device declining (threshold, +/// budget, a wiring gate) — never a silent pass. +pub fn fri_parity( + lde_log: u32, + options: &ProofOptions, + resident: bool, + seed: u64, +) -> Result { + let blowup_log = options.blowup_factor.trailing_zeros(); + let k = u32::from(options.fri_final_poly_log_degree); + let one_row = options.format.one_row == OneRowMode::On; + let layout = FriFoldLayout::for_options(lde_log, blowup_log, options, one_row) + .map_err(|e| format!("layout: {e}"))?; + let n = 1usize << lde_log; + let mut rng = seed; + let evals: Vec = (0..n) + .map(|_| { + Ext::new([ + Felt::from(splitmix64(&mut rng)), + Felt::from(splitmix64(&mut rng)), + Felt::from(splitmix64(&mut rng)), + ]) + }) + .collect(); + let offset = Felt::from(options.coset_offset); + let tw = compute_coset_twiddles_inv::(&offset, n); + let t0 = DefaultTranscript::::new(&seed.to_le_bytes()); + + let mut t_cpu = t0.clone(); + let (cpu_coeffs, cpu_layers) = crate::fri::commit_phase_cpu_with_layout::( + evals.clone(), + &mut t_cpu, + &offset, + n, + blowup_log, + k, + &layout, + &tw, + ); + + let mut t_gpu = t0.clone(); + let device = if resident { + crate::gpu_lde::try_fri_commit_gpu_resident::>( + &evals, &mut t_gpu, &offset, n, blowup_log, k, &layout, &tw, + ) + } else { + crate::gpu_lde::try_fri_commit_gpu::>( + &evals, &mut t_gpu, &offset, n, blowup_log, k, &layout, &tw, + ) + }; + let (gpu_coeffs, gpu_layers) = device.ok_or_else(|| { + format!( + "the device FRI commit declined at LDE 2^{lde_log} (schedule {:?}): \ + run with a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD <= {n}", + layout.schedule + ) + })?; + + let what = format!( + "LDE 2^{lde_log}, schedule {:?}, legacy {}, one_row {one_row}, resident {resident}", + layout.schedule, + layout.is_legacy() + ); + if gpu_coeffs != cpu_coeffs { + return Err(format!("{what}: terminal coefficients differ")); + } + if gpu_layers.len() != cpu_layers.len() || cpu_layers.len() != layout.num_committed { + return Err(format!( + "{what}: {} device layers, {} host layers", + gpu_layers.len(), + cpu_layers.len() + )); + } + for (j, (g, c)) in gpu_layers.iter().zip(&cpu_layers).enumerate() { + if g.merkle_tree.root != c.merkle_tree.root { + return Err(format!("{what}: layer {j} root differs")); + } + if g.gpu_tree.as_ref().map(|t| t.root) != Some(c.merkle_tree.root) { + return Err(format!("{what}: layer {j} resident tree root differs")); + } + if resident { + if !g.evaluation.is_empty() || g.gpu_evals.is_none() { + return Err(format!("{what}: layer {j} is not device-only")); + } + } else if raw(&g.evaluation) != raw(&c.evaluation) { + return Err(format!("{what}: layer {j} evaluations differ")); + } + } + let (after_cpu, after_gpu): (Ext, Ext) = + (t_cpu.sample_field_element(), t_gpu.sample_field_element()); + if after_cpu != after_gpu { + return Err(format!("{what}: the transcripts diverged")); + } + + // Queries: random indices and both ends of the range — pair indices below + // `N / 2`, or (one row) trace leaves over the whole LDE. + let bound = if one_row { n } else { n / 2 }; + let mut iotas: Vec = (0..40) + .map(|_| (splitmix64(&mut rng) % bound as u64) as usize) + .collect(); + iotas.push(0); + iotas.push(bound - 1); + let (cpu_q, gpu_q) = queries::(&cpu_layers, &gpu_layers, &iotas, &layout); + let gpu_q = gpu_q.ok_or_else(|| format!("{what}: the device query phase declined"))?; + for (q, (a, b)) in cpu_q.iter().zip(&gpu_q).enumerate() { + if raw(&a.layers_evaluations_sym) != raw(&b.layers_evaluations_sym) { + return Err(format!( + "{what}: query {q} (iota {}) values differ", + iotas[q] + )); + } + if a.layers_auth_paths.len() != b.layers_auth_paths.len() + || a.layers_auth_paths + .iter() + .zip(&b.layers_auth_paths) + .any(|(x, y)| x.merkle_path != y.merkle_path) + { + return Err(format!( + "{what}: query {q} (iota {}) paths differ", + iotas[q] + )); + } + if a.layers_evaluations_sym.len() != layout.opened_values_per_query() { + return Err(format!("{what}: query {q} opens the wrong value count")); + } + } + Ok(format!( + "{what}: {} layers, {} queries equal", + layout.num_committed, + iotas.len() + )) +} + +#[allow(clippy::type_complexity)] +fn queries( + cpu: &[FriLayer>], + gpu: &[FriLayer>], + iotas: &[usize], + layout: &FriFoldLayout, +) -> (Vec>, Option>>) { + if layout.is_legacy() { + // Host layers carry no device tree, so `query_phase` walks them on + // the host; the device layers take the device gather. + ( + crate::fri::query_phase::(cpu, iotas), + crate::gpu_lde::try_fri_query_phase_gpu::>(gpu, iotas), + ) + } else { + ( + crate::fri::query_phase_groups_host::(cpu, iotas, layout), + crate::gpu_lde::try_fri_query_phase_gpu_groups::>(gpu, iotas, layout), + ) + } +} + +/// One parity case: an LDE log and the options whose layout it runs under. +pub type Case = (u32, ProofOptions); + +/// The shape sweep: every [`dp_shapes`] entry at its [`smallest_case`]. +pub fn sweep_cases() -> Vec { + dp_shapes().iter().map(|s| smallest_case(s)).collect() +} + +/// Production sizes at the DP's own schedules: base legs (blowup 4, k = 7, +/// T = 9) at B = 14, 19, 21, 23 and an LFM-shaped proof (k = 8, T = 10) at +/// B = 22, 110 queries, cap auto (the objective the DP prices); and today's +/// pair encoding at B = 21. +pub fn production_cases() -> Vec { + let mut cases: Vec = [14u32, 19, 21, 23] + .iter() + .map(|&b| (b, dp_options(2, 7, 110, CapPolicy::Auto, None))) + .collect(); + cases.push((22, dp_options(2, 8, 110, CapPolicy::Auto, None))); + cases.push((21, pair_options(2, 7))); + cases +} + +/// Device-only layers (no host copy of any layer's evals, so the query phase +/// gathers the groups off the resident evals): uneven shapes, the DMAX group, +/// a DP schedule at B = 16, and today's encoding. +pub fn resident_cases() -> Vec { + let mut cases: Vec = [&[3u8, 1, 3][..], &[6, 1], &[1, 6]] + .iter() + .map(|s| smallest_case(s)) + .collect(); + cases.push((16, dp_options(2, 7, 110, CapPolicy::Auto, None))); + cases.push((14, pair_options(2, 7))); + cases +} + +/// Today's encoding through the layout-taking drive: one committed layer and +/// up, blowup 2 and 4. +pub fn legacy_cases() -> Vec { + let mut cases: Vec = [4u32, 5, 8, 12] + .iter() + .map(|&b| (b, pair_options(1, 1))) + .collect(); + cases.extend([10u32, 16].iter().map(|&b| (b, pair_options(2, 3)))); + cases +} + +/// `options` with one-row openings on (S2): the FRI chain starts at the LDE +/// itself and layer 0 is the input tree. +pub fn with_one_row(mut options: ProofOptions) -> ProofOptions { + options.format.one_row = OneRowMode::On; + options +} + +/// The smallest `(lde_log, options)` whose ONE-ROW layout at blowup 2, `k = 1` +/// (a terminal of 4) has exactly `schedule` as its committed folds (the chain +/// starts at the LDE, so one bit shorter than [`smallest_case`]). +pub fn smallest_one_row_case(schedule: &[u8]) -> (u32, ProofOptions) { + let sum: u32 = schedule.iter().map(|&d| u32::from(d)).sum(); + ( + sum + 2, + with_one_row(dp_options(1, 1, 3, CapPolicy::Off, Some(schedule))), + ) +} + +/// S2 on the device: every [`dp_shapes`] entry at its +/// [`smallest_one_row_case`] (d_0 = the input tree's group), today's pair +/// schedule with one-row openings (group encoding at d = 1) at blowup 2 and +/// 4, and production sizes at the DP's own one-row schedules (base legs at +/// B = 14, 19, 21, 23, an LFM-shaped B = 22; Q = 110, cap auto). +pub fn one_row_cases() -> Vec { + let mut cases: Vec = dp_shapes() + .iter() + .map(|s| smallest_one_row_case(s)) + .collect(); + cases.extend( + [4u32, 5, 8, 12] + .iter() + .map(|&b| (b, with_one_row(pair_options(1, 1)))), + ); + cases.extend( + [10u32, 16] + .iter() + .map(|&b| (b, with_one_row(pair_options(2, 3)))), + ); + cases.extend([14u32, 19, 21, 23].iter().map(|&b| { + ( + b, + with_one_row(dp_options(2, 7, 110, CapPolicy::Auto, None)), + ) + })); + cases.push(( + 22, + with_one_row(dp_options(2, 8, 110, CapPolicy::Auto, None)), + )); + cases +} + +/// S2 device-only layers: the input tree's evals ARE the resident codeword, +/// so the query phase gathers layer 0's groups off it. +pub fn one_row_resident_cases() -> Vec { + let mut cases: Vec = [&[3u8, 1, 3][..], &[6, 1], &[1, 6]] + .iter() + .map(|s| smallest_one_row_case(s)) + .collect(); + cases.push(( + 16, + with_one_row(dp_options(2, 7, 110, CapPolicy::Auto, None)), + )); + cases.push((14, with_one_row(pair_options(2, 7)))); + cases +} + +/// Run [`fri_parity`] over `cases` (seeds `seed_base + i`), printing one +/// `FRIDEV` line per case; `Err` lists every failing case. +pub fn run_cases( + name: &str, + cases: &[Case], + resident: bool, + seed_base: u64, +) -> Result> { + let mut failures = Vec::new(); + for (i, (b, opts)) in cases.iter().enumerate() { + match fri_parity::(*b, opts, resident, seed_base + i as u64) { + Ok(msg) => std::println!("FRIDEV {name} {msg}"), + Err(e) => failures.push(e), + } + } + if failures.is_empty() { + std::println!("FRIDEV {name}: {} cases equal", cases.len()); + Ok(cases.len()) + } else { + Err(failures) + } +} diff --git a/crypto/stark/src/fri/fri_commitment.rs b/crypto/stark/src/fri/fri_commitment.rs index 1c199441d..778527c82 100644 --- a/crypto/stark/src/fri/fri_commitment.rs +++ b/crypto/stark/src/fri/fri_commitment.rs @@ -4,7 +4,7 @@ use math::{ traits::AsBytes, }; -#[cfg_attr(not(feature = "disk-spill"), derive(Clone))] +#[derive(Clone)] pub struct FriLayer where F: IsField, diff --git a/crypto/stark/src/fri/group.rs b/crypto/stark/src/fri/group.rs new file mode 100644 index 000000000..310b202fd --- /dev/null +++ b/crypto/stark/src/fri/group.rs @@ -0,0 +1,232 @@ +//! Group-leaf FRI layers (S3): a committed layer of fold exponent `d` groups +//! `2^d` consecutive bit-reversed evaluations per leaf. +//! +//! # Why a group is a coset, and how it folds +//! +//! A layer of length `n = 2^b` on the coset `o_b·⟨ω_n⟩` stores, at position +//! `p`, the value at `o_b·ω_n^{br_b(p)}`. Bit reversal over `b` bits moves the +//! low `d` bits of `p = g·2^d + t` to the top, so the group of leaf `g` holds +//! the full fiber `x_g·⟨ω_{2^d}⟩` of `x ↦ x^{2^d}`, `x_g = o_b·ω_n^{br_{b−d}(g)}`, +//! in bit-reversed order (`point(g·2^d + t) = x_g·ω_{2^d}^{br_d(t)}`), and +//! `x_g^{2^d}` is the point at position `g` of the layer folded `d` times. +//! +//! The prover folds a committed layer with ONE challenge `ζ` as `d` successive +//! binary folds with `ζ, ζ², …, ζ^{2^{d−1}}` (the unchanged binary fold, so the +//! arity-`2^d` fold `2^d·Σ ζ^i f_i` of Haböck 2022/1216 eq. (3)). The verifier +//! runs the same `d` levels on the group alone ([`group_fold`]): at level `ℓ` +//! the pair `(2j, 2j+1)` sits at `(X, −X)`, +//! `X = x_g^{2^ℓ}·ω_{2^{d−ℓ}}^{br_{d−ℓ−1}(j)}`, and +//! `u'_j = (u_{2j} + u_{2j+1}) + ζ^{2^ℓ}·X⁻¹·(u_{2j} − u_{2j+1})`, exactly the +//! prover's `fold_evaluations_in_place` restricted to one fiber. +//! +//! # What a query checks per layer (the load-bearing checks) +//! +//! 1. the group is the leaf: hashed in full (`H::Batched` over the `2^d` +//! values) and authenticated against the layer root at `leaf = p >> d`, +//! with the exact path length; +//! 2. the slot check `group[p & (2^d − 1)] == v` — the round-consistency check +//! tying this layer to the value the previous fold produced; +//! 3. the group fold with `ζ_j` gives the value at `p >> d` of the next layer. +//! +//! Dropping 1 or 2 is a soundness break; `fri_group_tests` has a named test +//! that turns red for each (M1, M2). + +use crypto::merkle_tree::traits::IsStreamingLeafBackend; +use math::fft::bit_reversing::reverse_index; +use math::field::element::FieldElement; +use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; +use math::traits::AsBytes; + +use crate::config::Commitment; +use crate::fri::terminal::FriFoldLayout; +use crate::merkle_caps::TreeCheck; + +/// Verifier mutations for the load-bearing tests (M1, M2). Test builds only; +/// production has no switch. Thread-local: the host verifier is sequential, +/// so a test that sets one affects only its own verification. +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GroupMutation { + None, + /// M1: skip `group[slot] == v`. + SkipSlotCheck, + /// M2: skip the group's Merkle authentication. + SkipLeafAuth, +} + +#[cfg(test)] +thread_local! { + pub(crate) static GROUP_MUTATION: core::cell::Cell = + const { core::cell::Cell::new(GroupMutation::None) }; +} + +#[inline] +fn mutated(_m: u8) -> bool { + #[cfg(test)] + { + let m = match _m { + 1 => GroupMutation::SkipSlotCheck, + _ => GroupMutation::SkipLeafAuth, + }; + GROUP_MUTATION.with(|c| c.get() == m) + } + #[cfg(not(test))] + { + false + } +} + +/// `ω_{2^d}^t` for `t < 2^d`, `ω_{2^d}` the field's primitive `2^d`-th root — +/// the same root the LDE domain's `ω_N^{N/2^d}` is (both are powers of the +/// field's two-adic generator; `fri_group_tests::group_leaf_is_a_coset` checks +/// it against the prover's own domain). +pub(crate) fn roots_of_unity_table(d: u32) -> Option>> { + let w = F::get_primitive_root_of_unity(u64::from(d)).ok()?; + let n = 1usize << d; + let mut out = Vec::with_capacity(n); + let mut acc = FieldElement::::one(); + for _ in 0..n { + out.push(acc.clone()); + acc = &acc * &w; + } + Some(out) +} + +/// The group fold (see the module docs): `d = log2(group.len())` binary folds +/// of the `2^d` values of one leaf with `ζ, ζ², …`, given `x_g⁻¹` (the inverse +/// of the leaf's coset base) and `roots = roots_of_unity_table(d)`. Returns +/// the value at the leaf's position in the layer folded `d` times. +pub(crate) fn group_fold( + group: &[FieldElement], + zeta: &FieldElement, + x_g_inv: &FieldElement, + roots: &[FieldElement], +) -> FieldElement +where + F: IsField + IsSubFieldOf, + E: IsField, +{ + let n = group.len(); + debug_assert!(n.is_power_of_two() && roots.len() == n); + let d = n.trailing_zeros(); + let mut vals = group.to_vec(); + let mut xinv = x_g_inv.clone(); + let mut z = zeta.clone(); + for level in 0..d { + let half = vals.len() / 2; + // Pair j of this level: X⁻¹ = x_g^{−2^ℓ} · ω_{2^d}^{−2^ℓ·br_{d−ℓ−1}(j)}. + for j in 0..half { + let br = if half > 1 { + reverse_index(j, half as u64) + } else { + 0 + }; + // 2^ℓ·br < 2^{d−1} < n: already reduced. + let e = br << level; + let c = &roots[(n - e) % n]; + let x_inv_j = &xinv * c; + let lo = &vals[2 * j]; + let hi = &vals[2 * j + 1]; + let sum = lo + hi; + let diff = lo - hi; + vals[j] = &sum + &(&x_inv_j * &(&z * &diff)); + } + vals.truncate(half); + xinv = xinv.square(); + z = z.square(); + } + vals.swap_remove(0) +} + +/// The FRI checks of one query under a group-encoded layout (every format but +/// the legacy one): per committed layer `j`, the group is authenticated at +/// `leaf = p >> d_j` by `checks[j]` — the layer tree's check, built once per +/// tree at the layout's depth with its Merkle cap (`TreeCheck`; exact path +/// length `depth − c`, query 0 the cap's owner) — with path `paths(j)`, the +/// slot check `group[p & (2^{d_j} − 1)] == v` holds, and `v` becomes the group +/// fold with layer `j`'s challenge; finally `terminal[p] == v`. +/// +/// Layer `j`'s challenge is `zetas[j + 1]` for row pairs (`zetas[0]` drove the +/// uncommitted fold 0) and `zetas[j]` under one row (layer 0 is the committed +/// DEEP codeword, so no fold precedes it) — [`FriFoldLayout::num_zetas`]. +/// +/// * `v` / `y_inv`: the query's value at committed layer 0 and the inverse of +/// its point there (row pairs: fold 0 already applied by the caller; one +/// row: the DEEP value at `x_r` and `x_r⁻¹` — the layer-0 slot check is then +/// the input-slot check `group₀[slot] == DEEP(x_r)`); +/// * `query`: the query's position in proof order (query 0 is every capped +/// layer's owner opening); +/// * `iota`: the query's position in committed layer 0; +/// * `values`: the flat per-query group values (the proof's +/// `layers_evaluations_sym` under this encoding), length already checked by +/// the caller to be `layout.opened_values_per_query()`; +/// * `roots_tables[d]`: `roots_of_unity_table(d)` for every `d` in the schedule. +#[allow(clippy::too_many_arguments)] +pub(crate) fn verify_query_groups<'p, F, E, B>( + layout: &FriFoldLayout, + checks: &[TreeCheck<'_>], + query: usize, + paths: impl Fn(usize) -> &'p [Commitment], + values: &[FieldElement], + zetas: &[FieldElement], + iota: usize, + mut v: FieldElement, + mut y_inv: FieldElement, + terminal_codeword: &[FieldElement], + roots_tables: &[Vec>], +) -> bool +where + F: IsFFTField + IsSubFieldOf, + E: IsField, + FieldElement: AsBytes + Sync + Send, + B: IsStreamingLeafBackend, +{ + if checks.len() != layout.num_committed + || values.len() != layout.opened_values_per_query() + || zetas.len() != layout.num_zetas() + { + return false; + } + let zeta_offset = usize::from(!layout.one_row); + let mut index = iota; + let mut offset = 0usize; + let mut ok = true; + for (j, &d) in layout.schedule.iter().enumerate() { + let d = u32::from(d); + let n = 1usize << d; + let group = &values[offset..offset + n]; + offset += n; + let leaf = index >> d; + let slot = index & (n - 1); + + // (2) the slot check. + if group[slot] != v && !mutated(1) { + ok = false; + } + // (1) the group is the leaf, authenticated with the exact depth. + let leaf_hash = B::hash_data_from_slices(group, &[]); + if !checks[j].verify::(query, paths(j), leaf, leaf_hash) && !mutated(2) { + ok = false; + } + // (3) fold: x_g⁻¹ = y⁻¹ · ω_{2^d}^{br_d(slot)}. + let Some(table) = roots_tables.get(d as usize) else { + return false; + }; + if table.len() != n { + return false; + } + let br_slot = if n > 1 { + reverse_index(slot, n as u64) + } else { + 0 + }; + let x_g_inv = &y_inv * &table[br_slot]; + v = group_fold::(group, &zetas[j + zeta_offset], &x_g_inv, table); + for _ in 0..d { + y_inv = y_inv.square(); + } + index = leaf; + } + let terminal_ok = terminal_codeword.get(index).is_some_and(|t| &v == t); + ok & terminal_ok +} diff --git a/crypto/stark/src/fri/mod.rs b/crypto/stark/src/fri/mod.rs index 1f53b51cf..fb79b9d70 100644 --- a/crypto/stark/src/fri/mod.rs +++ b/crypto/stark/src/fri/mod.rs @@ -1,24 +1,44 @@ +#[cfg(any(test, feature = "test-utils"))] +pub mod capture; +#[cfg(all(feature = "cuda", any(test, feature = "test-utils")))] +pub mod device_parity; pub mod fri_commitment; pub mod fri_decommit; pub(crate) mod fri_functions; +pub(crate) mod group; +pub mod schedule; pub(crate) mod terminal; +#[cfg(any(test, feature = "test-utils"))] +pub mod vectors; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; +use crypto::merkle_tree::merkle::MerkleTree; use math::field::element::FieldElement; use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; use math::traits::AsBytes; -use crate::config::{FriLayerMerkleTree, FriLayerMerkleTreeBackend}; +use crate::config::StarkHash; +use crate::fri::terminal::FriFoldLayout; use self::fri_commitment::FriLayer; use self::fri_decommit::FriDecommitment; use self::fri_functions::{fold_evaluations_in_place, update_twiddles_in_place}; +#[cfg(feature = "parallel")] +use rayon::prelude::*; /// FRI commit phase from pre-computed bit-reversed evaluations, skipping the /// initial FFT. Stops folding when the remaining codeword encodes a polynomial /// of degree < 2^`final_poly_log_degree` with blowup 2^`blowup_log`, and /// returns the coefficient vector of that terminal polynomial. /// +/// Layer trees are built with `H::Pair` — the commitment configuration's +/// FRI-layer backend, the same `H` the caller's prover and verifier are +/// instantiated at. That is what makes the layer roots this returns +/// authenticable by [`crate::verifier::IsStarkVerifier::verify`], which +/// re-hashes each opened pair through `H::Batched`: the two are one hash by +/// [`StarkHash`]'s two-element invariant, so agreement is a property of naming +/// one configuration rather than of two call sites happening to match. +/// /// The `T: Clone` and `F/E: 'static` bounds are required by the cuda GPU /// fast path (`try_fri_commit_gpu` snapshots the transcript and TypeId- /// checks the field types). They are present unconditionally (including @@ -28,18 +48,82 @@ pub fn commit_phase_from_evaluations< F: IsFFTField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, T: IsStarkTranscript + Clone, + H: StarkHash, >( - mut evals: Vec>, + evals: Vec>, transcript: &mut T, coset_offset: &FieldElement, domain_size: usize, blowup_log: u32, final_poly_log_degree: u32, inv_twiddles: &[FieldElement], -) -> ( - Vec>, - Vec>>, -) +) -> (Vec>, Vec>>) +where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + // Today's layout: pair layers, the all-ones schedule. + let layout = FriFoldLayout::new( + evals.len().trailing_zeros(), + blowup_log, + final_poly_log_degree, + ); + commit_phase_with_layout::( + evals, + transcript, + coset_offset, + domain_size, + blowup_log, + final_poly_log_degree, + &layout, + inv_twiddles, + ) +} + +/// [`commit_phase_from_evaluations`] under an explicit fold layout (the proof +/// format's; see [`FriFoldLayout::for_options`]). +/// +/// Transcript, per committed layer `j` with fold exponent `d_j`: sample `ζ`, +/// fold `d_{j−1}` times with `ζ, ζ², …` (`d_{−1} = 1`: fold 0 is the binary +/// fold of the DEEP pair), commit the result with leaves of `2^{d_j}` values, +/// append the root. Then, when anything folds, sample the final `ζ` and fold +/// `d_last` times into the terminal codeword. At the all-ones schedule this is +/// exactly today's loop (sample, fold once, commit pairs, append). +/// +/// One-row layouts (S2): `d_{−1} = 0` — layer 0 is the DEEP codeword itself, +/// committed with groups of `2^{d_0}` and its root absorbed with NO challenge +/// before it; every later layer is "sample ζ, fold, commit, append" as above. +/// So `m` committed layers draw `m` challenges (the last one the final fold's), +/// against `m + 1` for row pairs. +/// +/// Leaves: the legacy encoding commits `[a, b]` pairs with `H::Pair`; the +/// group encoding hashes each `2^d`-value group with `H::Batched` (the two +/// agree on a two-element leaf, `StarkHash`'s invariant) and builds the tree +/// from those leaf hashes with `H::Pair`'s parent hash — the parent hash both +/// families share, which today's layer trees already rely on (built with +/// `H::Pair`, verified with `H::Batched`). +/// +/// The device arm (`try_fri_commit_gpu`) runs both encodings: today's loop for +/// the legacy one and its group twin otherwise — one-row layouts included, +/// whose input tree the device commits from the codeword before any +/// challenge. When it declines, the CPU loop +/// ([`commit_phase_cpu_with_layout`]) runs. +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +pub(crate) fn commit_phase_with_layout< + F: IsFFTField + IsSubFieldOf + 'static, + E: IsField + 'static + Send + Sync, + T: IsStarkTranscript + Clone, + H: StarkHash, +>( + evals: Vec>, + transcript: &mut T, + coset_offset: &FieldElement, + domain_size: usize, + blowup_log: u32, + final_poly_log_degree: u32, + layout: &FriFoldLayout, + inv_twiddles: &[FieldElement], +) -> (Vec>, Vec>>) where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, @@ -51,36 +135,70 @@ where // snapshots the transcript before mutating it so a mid-loop cudarc // error restores state and lets the CPU loop below run as if the GPU // had never been tried. + // Try the GPU early-termination FRI commit first. `try_fri_commit_gpu` + // drives the same commit phase on-device (Goldilocks + Ext3, above the + // LDE size threshold, and only when folding actually happens) and returns + // `Some` with the final-polynomial coefficients. It returns `None` on any + // precondition miss or cudarc error — restoring the transcript first — so + // the CPU path below then runs as if the GPU had never been tried. #[cfg(feature = "cuda")] - { - // Try the GPU early-termination FRI commit first. `try_fri_commit_gpu` - // drives the same commit phase on-device (Goldilocks + Ext3, above the - // LDE size threshold, and only when folding actually happens) and returns - // `Some` with the final-polynomial coefficients. It returns `None` on any - // precondition miss or cudarc error — restoring the transcript first — so - // the CPU path below then runs as if the GPU had never been tried. - if let Some(result) = crate::gpu_lde::try_fri_commit_gpu::( - &evals, - transcript, - coset_offset, - domain_size, - blowup_log, - final_poly_log_degree, - inv_twiddles, - ) { - return result; - } + if let Some(result) = crate::gpu_lde::try_fri_commit_gpu::>( + &evals, + transcript, + coset_offset, + domain_size, + blowup_log, + final_poly_log_degree, + layout, + inv_twiddles, + ) { + return result; } + commit_phase_cpu_with_layout::( + evals, + transcript, + coset_offset, + domain_size, + blowup_log, + final_poly_log_degree, + layout, + inv_twiddles, + ) +} +/// The CPU loop of [`commit_phase_with_layout`], with no device arm: what every +/// build runs when the device declines, and the host reference the device +/// parity tests compare against. +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +pub(crate) fn commit_phase_cpu_with_layout< + F: IsFFTField + IsSubFieldOf + 'static, + E: IsField + 'static + Send + Sync, + T: IsStarkTranscript + Clone, + H: StarkHash, +>( + mut evals: Vec>, + transcript: &mut T, + coset_offset: &FieldElement, + domain_size: usize, + blowup_log: u32, + final_poly_log_degree: u32, + layout: &FriFoldLayout, + inv_twiddles: &[FieldElement], +) -> (Vec>, Vec>>) +where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ debug_assert_eq!(evals.len(), domain_size); // Caller-enforced twiddle sizing (Domain::fri_inv_twiddles): the folding // loop below indexes `inv_twiddles[..len/2]` per layer. debug_assert_eq!(inv_twiddles.len(), evals.len() / 2); - // Fold layout, shared with the GPU prover and the verifier — see `FriFoldLayout`. - let layout = crate::fri::terminal::FriFoldLayout::new( - evals.len().trailing_zeros(), - blowup_log, - final_poly_log_degree, + // The fold layout, shared with the GPU prover and the verifier — see + // `FriFoldLayout`. It was built for this codeword's size. + let _ = (blowup_log, final_poly_log_degree); + debug_assert_eq!( + layout.total_folds, + evals.len().trailing_zeros() - layout.terminal_len.trailing_zeros() ); let num_committed = layout.num_committed; @@ -89,37 +207,51 @@ where let mut inv_twiddles = inv_twiddles.to_vec(); let mut fri_layer_list = Vec::with_capacity(num_committed); + // Folds still owed before the next commit: fold 0 is the binary fold of + // the DEEP pair, so one; after committing layer `j`, `d_j`. Under one-row + // openings (S2) the DEEP codeword itself is layer 0 (the input tree), so + // nothing is owed before it and its root is absorbed BEFORE the first + // folding challenge (a root absorbed after its challenge + // would let the prover pick the codeword after seeing it). + let mut pending: u32 = if layout.one_row { 0 } else { 1 }; + // Commit `num_committed` folded layers to the transcript. - for _ in 0..num_committed { - // <<<< Receive challenge 𝜁ₖ - let zeta = transcript.sample_field_element(); + for &d in &layout.schedule { + if pending > 0 { + // <<<< Receive challenge 𝜁ₖ + let zeta = transcript.sample_field_element(); - // Fold evaluations in-place (no FFT needed). - fold_evaluations_in_place(&mut evals, &zeta, &inv_twiddles); + // Fold `pending` times with 𝜁, 𝜁², … (evaluation form, no FFT). + fold_times(&mut evals, &zeta, pending, &mut inv_twiddles); + } - // Build the Merkle tree from consecutive pairs. - let leaves: Vec<[FieldElement; 2]> = evals - .chunks_exact(2) - .map(|chunk| [chunk[0].clone(), chunk[1].clone()]) - .collect(); - let merkle_tree = FriLayerMerkleTree::build(&leaves) - .expect("FRI commit: Merkle tree construction must succeed"); + let merkle_tree = if layout.is_legacy() { + // Build the Merkle tree from consecutive pairs. + let leaves: Vec<[FieldElement; 2]> = evals + .chunks_exact(2) + .map(|chunk| [chunk[0].clone(), chunk[1].clone()]) + .collect(); + MerkleTree::>::build(&leaves) + } else { + group_tree::(&evals, 1usize << d) + } + .expect("FRI commit: Merkle tree construction must succeed"); let root = merkle_tree.root; fri_layer_list.push(FriLayer::new(&evals, merkle_tree)); // >>>> Send commitment: [pₖ] transcript.append_bytes(&root); - // Update twiddles for the next level. - update_twiddles_in_place(&mut inv_twiddles); + pending = u32::from(d); } - // One final fold to reach the terminal codeword (size terminal_len), unless - // already there (total_folds == 0 means initial_len == terminal_len). + // The final folds to reach the terminal codeword (size terminal_len), + // unless already there (total_folds == 0 means initial_len == terminal_len; + // then `pending` is 0 under one row too, as the schedule is empty). if layout.total_folds > 0 { // <<<< Receive challenge: 𝜁_final let zeta = transcript.sample_field_element(); - fold_evaluations_in_place(&mut evals, &zeta, &inv_twiddles); + fold_times(&mut evals, &zeta, pending, &mut inv_twiddles); } debug_assert_eq!( evals.len(), @@ -150,8 +282,54 @@ where (final_poly_coeffs, fri_layer_list) } -pub fn query_phase( - fri_layers: &[FriLayer>], +/// `n` binary folds of `evals` with `ζ, ζ², …, ζ^{2^{n−1}}`, each followed by +/// the twiddle update for the halved domain. `n = 1` is one plain fold (the +/// trailing twiddle update only prepares a fold that may never come). +pub(crate) fn fold_times, E: IsField>( + evals: &mut Vec>, + zeta: &FieldElement, + n: u32, + inv_twiddles: &mut Vec>, +) { + let mut z = zeta.clone(); + for level in 0..n { + fold_evaluations_in_place(evals, &z, inv_twiddles); + update_twiddles_in_place(inv_twiddles); + if level + 1 < n { + z = z.square(); + } + } +} + +/// A group-leaf layer tree: leaf `g` = `H::Batched` over `evals[g·n .. (g+1)·n]`. +pub(crate) fn group_tree( + evals: &[FieldElement], + n: usize, +) -> Option>> +where + E: IsField + 'static + Send + Sync, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, +{ + use crypto::merkle_tree::traits::IsStreamingLeafBackend; + let hash = |g: &[FieldElement]| { + as IsStreamingLeafBackend>::hash_data_from_slices(g, &[]) + }; + #[cfg(feature = "parallel")] + let leaves: Vec<_> = evals.par_chunks_exact(n).map(hash).collect(); + #[cfg(not(feature = "parallel"))] + let leaves: Vec<_> = evals.chunks_exact(n).map(hash).collect(); + MerkleTree::>::build_from_hashed_leaves(leaves) +} + +/// Open every committed layer at each query index, producing one +/// [`FriDecommitment`] per query. +/// +/// Takes the layers [`commit_phase_from_evaluations`] built, so it is generic +/// over the same configuration `H`: the authentication paths it walks are only +/// meaningful against roots the verifier re-derives through `H`. +pub fn query_phase( + fri_layers: &[FriLayer>], iotas: &[usize], ) -> Vec> where @@ -161,7 +339,9 @@ where // layer trees stay resident from the GPU commit). Falls back to the host // walk below if any layer lacks a device tree. #[cfg(feature = "cuda")] - if let Some(decommits) = crate::gpu_lde::try_fri_query_phase_gpu::(fri_layers, iotas) { + if let Some(decommits) = + crate::gpu_lde::try_fri_query_phase_gpu::>(fri_layers, iotas) + { return decommits; } @@ -203,3 +383,67 @@ where .collect() } } + +/// [`query_phase`] under an explicit fold layout. The legacy encoding is +/// [`query_phase`] itself (device arm included); the group encoding opens, per +/// committed layer `j`, the whole group `evaluation[leaf·2^{d_j} ..][..2^{d_j}]` +/// (the query's own value included) and the path of +/// `leaf = p >> d_j`, then moves to `p >> d_j` — on the device when the layers +/// are device-resident (`try_fri_query_phase_gpu_groups`), else by the host +/// walk ([`query_phase_groups_host`]). +pub(crate) fn query_phase_with_layout( + fri_layers: &[FriLayer>], + iotas: &[usize], + layout: &FriFoldLayout, +) -> Vec> +where + FieldElement: AsBytes + Sync + Send, +{ + if layout.is_legacy() { + return query_phase::(fri_layers, iotas); + } + #[cfg(feature = "cuda")] + if let Some(decommits) = + crate::gpu_lde::try_fri_query_phase_gpu_groups::>(fri_layers, iotas, layout) + { + return decommits; + } + query_phase_groups_host::(fri_layers, iotas, layout) +} + +/// The host walk of [`query_phase_with_layout`]'s group encoding over host +/// layer trees (the device parity tests' reference). +pub(crate) fn query_phase_groups_host( + fri_layers: &[FriLayer>], + iotas: &[usize], + layout: &FriFoldLayout, +) -> Vec> +where + FieldElement: AsBytes + Sync + Send, +{ + debug_assert_eq!(fri_layers.len(), layout.num_committed); + iotas + .iter() + .map(|&iota| { + let mut values = Vec::with_capacity(layout.opened_values_per_query()); + let mut paths = Vec::with_capacity(fri_layers.len()); + let mut index = iota; + for (layer, &d) in fri_layers.iter().zip(&layout.schedule) { + let n = 1usize << d; + let leaf = index >> d; + values.extend_from_slice(&layer.evaluation[leaf * n..(leaf + 1) * n]); + paths.push( + layer + .merkle_tree + .get_proof_by_pos(leaf) + .expect("FRI query: leaf index within the layer tree"), + ); + index = leaf; + } + FriDecommitment { + layers_auth_paths: paths, + layers_evaluations_sym: values, + } + }) + .collect() +} diff --git a/crypto/stark/src/fri/schedule.rs b/crypto/stark/src/fri/schedule.rs new file mode 100644 index 000000000..d601934e3 --- /dev/null +++ b/crypto/stark/src/fri/schedule.rs @@ -0,0 +1,562 @@ +//! The FRI fold schedule: which fold exponents the committed FRI layers use. +//! +//! Today every committed FRI layer folds by 2 (a pair leaf). A fold schedule +//! `[d_1, .., d_m]` generalises that: committed layer `j` folds by `2^{d_j}` +//! (a group leaf of `2^{d_j}` extension values), and `Σ d_j` covers the bits +//! between the first committed layer and the terminal codeword. The all-ones +//! schedule is today's protocol exactly. +//! +//! The schedule is a **format constant**: the prover, the verifier and the +//! in-guest verifier must derive the same one from public shape parameters +//! only, never from a proof. So the dynamic program below is integer-only, +//! with a fixed tie rule, and its inputs are all public: +//! +//! * `b0` — log2 length of the first committed layer (`lde_log − 1` when fold 0 +//! is the uncommitted binary fold of the trace pair, `lde_log` when the DEEP +//! codeword itself is committed); +//! * `terminal_log` — log2 length of the terminal codeword; +//! * `num_queries` — FRI query count (every FRI tree is opened once per query); +//! * the active Merkle-cap policy ([`CapPolicy`]; `Off` caps nothing); +//! * `dmax` — the largest fold exponent the program may choose. +//! +//! # The objective: the cost law of every emitted row +//! +//! The DP minimises the in-guest verifier's price of the FRI leg under the +//! SAME cost-law weights the cap policy optimises ([`AUTO_WEIGHTS`], ns per +//! row from the node law 421 ns/instruction + 5.63 ns/cell and each chip's +//! committed width). Per query per committed layer it prices EVERY row the +//! in-guest group-layer emitter (`prover/src/lfm/fri.rs::emit_group_layer`) +//! and its opening's hints emit — [`fri_group_layer_rows`], at a tree of +//! `depth` levels (uncapped): +//! +//! ```text +//! leaf(d)·compress absorb the 2^d-value group leaf +//! + depth·(compress + select) the authentication walk (a Select and a compression per level) +//! + (2^d − 1)·select the slot mux picking the query's value out of the group +//! + 2·XALU the slot check (assert_eq_ext: esub + ediv) +//! + (2^d − 1)·fold the group fold: 2^d − 1 binary folds (5 XALU each) +//! + d·twiddle the twiddle chain: one base mul per fold level +//! + d·(select + BALU) x_g⁻¹ = y⁻¹·ω^{br(slot)}: a constant Select and a base mul per slot bit +//! + max(0, d − 2)·XALU + [d ≥ 2]·BALU fold-level scaling (emul_base per level of > 2 pairs; one base mul at 2 pairs) +//! + 8·BALU + 1·unpack the root compare (walked digest unpacked, four lowered asserts) +//! + 2^d·unpack + 2^d·hint the group's values: hinted, unpacked into the leaf +//! + packs(d)·unpack the leaf's 3·2^d felts packed four to a word (LFM_LANES rows) +//! + depth·hint the path's siblings +//! − cap_gain(Q, c(depth)) / Q − c·hint what the tree's cap saves, per query (0 without a cap): +//! the cap policy's own gain, plus the c sibling hints a +//! capped path does not carry +//! ``` +//! +//! `leaf(d) = max(1, ⌈3·2^d / 8⌉)` (an ext3 group at the RPX rate of 8 felts); +//! the digest is the production one-cell (algebraic) digest. Each row kind is +//! priced at one weight: `SELECT`, `LFM_HASH` (compress), `Unpack` and hint +//! at the cap policy's (a `Pack` is an `LFM_LANES` row, as an `Unpack` is, +//! and is priced like one), `XALU` at [`XALU_ROW_NS`], `BALU` at [`BALU_ROW_NS`]. +//! The in-guest emitter's tests pin "emitted rows == [`fri_group_layer_rows`]" kind by +//! kind against its emitter (`lfm::fri_group_tests`), capped and uncapped. +//! Costs are kept in units of `1/Q` ns so every term is an integer. + +use crypto::merkle_tree::cap::{AUTO_WEIGHTS, CapPolicy, CapWeights, cap_gain}; + +use crate::proof::options::{FriMode, FriScheduleOverride, ProofOptions}; + +/// Largest fold exponent the schedule may choose (a 64-value group leaf). +pub const FRI_SCHEDULE_DMAX: u32 = 6; + +/// Leaf absorption rate of the cost model, in base-field elements per +/// permutation (the RPX sponge rate). +pub const FRI_LEAF_RATE_FELTS: u64 = 8; + +/// Extension degree of the FRI codeword values. +pub const FRI_EXTENSION_DEGREE: u64 = 3; + +/// `XALU` rows of one binary FRI fold in-guest: `edsl::fri_fold` emits +/// `eadd, esub, emul, emul_base, eadd`. +pub const FRI_FOLD_XALU_ROWS: u64 = 5; + +/// `BALU` rows of one step of the twiddle chain in-guest (one base `mul`). +pub const FRI_TWIDDLE_BALU_ROWS: u64 = 1; + +/// `SELECT` rows of one two-way select of the slot mux (an ext value is one +/// cell, so one `Select` instruction). +pub const FRI_SLOT_SELECT_ROWS: u64 = 1; + +/// `XALU` rows of the slot check: `assert_eq_ext` lowers to an `esub` and an +/// `ediv` by zero. +pub const FRI_SLOT_ASSERT_XALU_ROWS: u64 = 2; + +/// `SELECT` rows of one slot bit of the `x_g` derivation (`x_g⁻¹ = +/// y⁻¹·ω_{2^d}^{br(slot)}`): the bit picks `1` or a constant. +pub const FRI_XG_SELECT_ROWS: u64 = 1; + +/// `BALU` rows of one slot bit of the `x_g` derivation (one base `mul`). +pub const FRI_XG_BALU_ROWS: u64 = 1; + +/// `BALU` rows of one opening's root (or cap-node) compare at the production +/// one-cell digest: four lowered `assert_eq`s, a `sub` and a `div` each. +pub const FRI_ROOT_COMPARE_BALU_ROWS: u64 = 8; + +/// `Unpack` rows of one opening's root compare: the walked digest's lanes (a +/// capped compare unpacks the muxed cap node too, which the cap's gain prices). +pub const FRI_ROOT_COMPARE_UNPACK_ROWS: u64 = 1; + +/// Felts one `Pack` row assembles into a word for the algebraic leaf sponge. +pub const FRI_LEAF_PACK_FELTS: u64 = 4; + +/// Cost-law price (ns) of one `XALU` row: 421 + 5.63 × 18 committed cells +/// (the `LFM_XALU` cliff in the census, `+18874368` cells per `2^20` rows). +pub const XALU_ROW_NS: u64 = 522; + +/// Cost-law price (ns) of one `BALU` row: 421 + 5.63 × 10 committed cells +/// (the `LFM_BALU` cliff, `+5242880` cells per `2^19` rows). +pub const BALU_ROW_NS: u64 = 477; + +/// The per-row prices the schedule DP weighs. `cap` is the cap policy's own +/// weights, so the two levers optimise one objective. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FriCostWeights { + /// Compression, select, unpack, hint and compare prices (the cap policy's). + pub cap: CapWeights, + /// One binary fold in-guest ([`FRI_FOLD_XALU_ROWS`] `XALU` rows). + pub fold: u64, + /// One step of the twiddle chain in-guest ([`FRI_TWIDDLE_BALU_ROWS`] `BALU` rows). + pub twiddle: u64, + /// One `XALU` row. + pub xalu: u64, + /// One `BALU` row. + pub balu: u64, +} + +/// The weights the schedule DP optimises. ⚠ A FORMAT CONSTANT: changing any of +/// them can change the schedule, and so the proofs, of every table under +/// `LAMBDA_VM_ZF_FRI=dp`. Pinned by `fri_schedule_tests`. +pub const FRI_COST_WEIGHTS: FriCostWeights = FriCostWeights { + cap: AUTO_WEIGHTS, + fold: FRI_FOLD_XALU_ROWS * XALU_ROW_NS, + twiddle: FRI_TWIDDLE_BALU_ROWS * BALU_ROW_NS, + xalu: XALU_ROW_NS, + balu: BALU_ROW_NS, +}; + +/// The rows one query's opening of one committed FRI layer emits in-guest, by +/// chip kind. `hashes` counts two-to-one compressions and leaf-absorption +/// permutations alike (both are `LFM_HASH` rows, priced `compress`); `packs` +/// and `unpacks` are both `LFM_LANES` rows, priced `unpack`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct FriLayerRows { + pub selects: u64, + pub xalu: u64, + pub balu: u64, + pub hashes: u64, + pub unpacks: u64, + pub packs: u64, + pub hints: u64, +} + +impl FriLayerRows { + /// The cost-law price (ns) of these rows under `weights`. + pub fn price(&self, weights: &FriCostWeights) -> u64 { + let w = &weights.cap; + [ + (self.selects, w.select), + (self.xalu, weights.xalu), + (self.balu, weights.balu), + (self.hashes, w.compress), + (self.unpacks, w.unpack), + (self.packs, w.unpack), + (self.hints, w.hint), + ] + .iter() + .fold(0u64, |acc, &(n, p)| acc.saturating_add(n.saturating_mul(p))) + } +} + +/// ★ The rows one query's opening of a GROUP layer (fold exponent `d`, tree +/// `depth` levels deep, cap height `cap_height`, clamped to the depth) emits +/// in-guest: `emit_group_layer` plus the opening's hints (see the module +/// docs; `lfm::fri_group_tests` pins these kind by kind against the emitter). +/// +/// A cap of height `c` walks `depth − c` levels, hints `depth − c` siblings, +/// muxes the cap node (`2^c − 1` selects) and unpacks it for the compare. +pub fn fri_group_layer_rows(d: u32, depth: u32, cap_height: u32) -> FriLayerRows { + let d = d.min(63); + let n = 1u64 << d; + let c = cap_height.min(depth).min(63); + let walk = u64::from(depth - c); + let cap_mux = (1u64 << c) - 1; + let d64 = u64::from(d); + FriLayerRows { + selects: (n - 1) * FRI_SLOT_SELECT_ROWS + walk + cap_mux + d64 * FRI_XG_SELECT_ROWS, + xalu: (n - 1) * FRI_FOLD_XALU_ROWS + FRI_SLOT_ASSERT_XALU_ROWS + d64.saturating_sub(2), + balu: d64 * FRI_TWIDDLE_BALU_ROWS + + d64 * FRI_XG_BALU_ROWS + + u64::from(d >= 2) + + FRI_ROOT_COMPARE_BALU_ROWS, + hashes: fri_leaf_blocks(d) + walk, + unpacks: n + FRI_ROOT_COMPARE_UNPACK_ROWS + u64::from(c > 0), + packs: fri_leaf_packs(d), + hints: n + walk, + } +} + +/// The rows one query's opening of a layer under TODAY's pair encoding +/// (`FriFormat::is_legacy`: one sibling value per layer, no slot check, no +/// `x_g`) emits in-guest (`prover/src/lfm/fri.rs::emit_pair_layer`): the +/// parity select, the pair leaf, the walk and compare, one squaring of the +/// point, one fold; hints: the sibling value and the path. +pub fn fri_pair_layer_rows(depth: u32, cap_height: u32) -> FriLayerRows { + let c = cap_height.min(depth).min(63); + let walk = u64::from(depth - c); + FriLayerRows { + selects: FRI_SLOT_SELECT_ROWS + walk + ((1u64 << c) - 1), + xalu: FRI_FOLD_XALU_ROWS, + balu: FRI_TWIDDLE_BALU_ROWS + FRI_ROOT_COMPARE_BALU_ROWS, + hashes: fri_leaf_blocks(1) + walk, + unpacks: 2 + FRI_ROOT_COMPARE_UNPACK_ROWS + u64::from(c > 0), + packs: fri_leaf_packs(1), + hints: 1 + walk, + } +} + +/// Log2 length of the first committed FRI layer for an LDE of `2^lde_log`. +/// +/// Row-pair openings (`one_row == false`) consume the first fold uncommitted, +/// so the chain starts at `lde_log − 1`; one-row openings commit the DEEP +/// codeword itself, so the chain starts at `lde_log`. +pub fn fri_chain_start(lde_log: u32, one_row: bool) -> u32 { + if one_row { + lde_log + } else { + lde_log.saturating_sub(1) + } +} + +/// `Pack` rows that assemble one group leaf's `3·2^d` felts into words (four +/// per word, the tail zero-padded) before the sponge absorbs them. +pub fn fri_leaf_packs(d: u32) -> u64 { + let felts = FRI_EXTENSION_DEGREE.saturating_mul(1u64.checked_shl(d).unwrap_or(u64::MAX)); + felts.div_ceil(FRI_LEAF_PACK_FELTS) +} + +/// Permutations to absorb one group leaf of `2^d` extension values. +pub fn fri_leaf_blocks(d: u32) -> u64 { + let felts = FRI_EXTENSION_DEGREE.saturating_mul(1u64.checked_shl(d).unwrap_or(u64::MAX)); + felts.div_ceil(FRI_LEAF_RATE_FELTS).max(1) +} + +/// `Q ×` the per-query cost-law price (ns) of one committed GROUP layer of +/// fold exponent `d` whose tree has `depth` levels (the layer is +/// `2^{depth + d}` values long), under `weights` and the cap policy `cap`: +/// every emitted row ([`fri_group_layer_rows`]) of the uncapped opening, minus +/// the cap's gain and the sibling hints the cap removes. See the module docs. +pub fn fri_layer_cost_q( + weights: &FriCostWeights, + d: u32, + depth: u32, + num_queries: u64, + cap: CapPolicy, +) -> u64 { + layer_cost_q( + weights, + &fri_group_layer_rows(d, depth, 0), + depth, + num_queries, + cap, + ) +} + +/// `Q ×` the per-query price of one committed layer under TODAY's pair +/// encoding ([`fri_pair_layer_rows`]), under `weights` and `cap`. +pub fn fri_pair_layer_cost_q( + weights: &FriCostWeights, + depth: u32, + num_queries: u64, + cap: CapPolicy, +) -> u64 { + layer_cost_q( + weights, + &fri_pair_layer_rows(depth, 0), + depth, + num_queries, + cap, + ) +} + +/// `Q × price(uncapped)` minus the cap's gain ([`cap_gain`], the cap policy's +/// own function) and the `c` sibling hints per query a capped path omits. In +/// i128, clamped into u64 (non-negative: a cap never saves more than the walk +/// it shortens, but the clamp keeps that a non-assumption). +fn layer_cost_q( + weights: &FriCostWeights, + uncapped: &FriLayerRows, + depth: u32, + num_queries: u64, + cap: CapPolicy, +) -> u64 { + let q = num_queries as i128; + let per_query = uncapped.price(weights) as i128; + let queries = usize::try_from(num_queries).unwrap_or(usize::MAX); + let c = cap.height(queries, depth as usize); + let hints_saved = q.saturating_mul(c as i128 * weights.cap.hint as i128); + let total = q.saturating_mul(per_query) - cap_gain(&weights.cap, queries, c) - hints_saved; + u64::try_from(total.max(0)).unwrap_or(u64::MAX) +} + +/// `Q ×` the per-query cost of an arbitrary schedule starting at `b0` under a +/// per-layer cost function `layer_cost_q(d, depth)`, or `None` if a fold +/// exponent is zero or the schedule folds past zero bits. +pub fn fri_schedule_cost_by( + b0: u32, + schedule: &[u8], + layer_cost_q: &dyn Fn(u32, u32) -> u64, +) -> Option { + let mut b = b0; + let mut cost = 0u64; + for &d in schedule { + let d = u32::from(d); + if d == 0 { + return None; + } + b = b.checked_sub(d)?; + cost = cost.saturating_add(layer_cost_q(d, b)); + } + Some(cost) +} + +/// [`fri_schedule_cost_by`] under the production objective +/// ([`FRI_COST_WEIGHTS`], [`fri_layer_cost_q`]) — group layers. +pub fn fri_schedule_cost_q( + b0: u32, + schedule: &[u8], + num_queries: u64, + cap: CapPolicy, +) -> Option { + fri_schedule_cost_by(b0, schedule, &|d, depth| { + fri_layer_cost_q(&FRI_COST_WEIGHTS, d, depth, num_queries, cap) + }) +} + +/// The optimum a schedule DP picks, with its cost. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FriScheduleChoice { + /// `Q ×` the per-query cost (see the module docs). + pub cost_q: u64, + /// Number of committed trees (`schedule.len()`). + pub trees: u32, + /// Fold exponents, first committed layer first. + pub schedule: Vec, +} + +/// The schedule DP over an arbitrary per-layer cost `layer_cost_q(d, depth)`: +/// +/// ```text +/// best(T) = (0, 0, []) +/// best(b > T) = min over d ∈ [1, min(dmax, b − T)] of +/// (layer_cost_q(d, b − d) + best(b − d).cost, best(b − d).trees + 1, [d] ++ best(b − d).sched) +/// ``` +/// +/// compared lexicographically on `(cost, trees)`; ties go to the smallest `d` +/// (the first reached). Equivalently, the result is the lexicographically +/// smallest schedule among the `(cost, trees)`-optimal ones. It lands exactly +/// on `terminal_log`: `Σ schedule == b0 − terminal_log`, and the schedule is +/// empty when `b0 ≤ terminal_log`. A `dmax` of 0 is treated as 1; `dmax` is +/// capped at 32. +pub fn fri_schedule_by( + b0: u32, + terminal_log: u32, + dmax: u32, + layer_cost_q: &dyn Fn(u32, u32) -> u64, +) -> FriScheduleChoice { + if b0 <= terminal_log { + return FriScheduleChoice { + cost_q: 0, + trees: 0, + schedule: Vec::new(), + }; + } + let dmax = dmax.clamp(1, 32); + let span = (b0 - terminal_log) as usize; + // best[i] = optimum from b = terminal_log + i down to the terminal, stored + // as (cost, trees, first fold exponent); the schedule is recovered by + // following the first exponents. + let mut best: Vec<(u64, u32, u32)> = Vec::with_capacity(span + 1); + best.push((0, 0, 0)); + for i in 1..=span { + let b = terminal_log + i as u32; + let mut cand: Option<(u64, u32, u32)> = None; + for d in 1..=dmax.min(i as u32) { + let (rest_cost, rest_trees, _) = best[i - d as usize]; + let cost = layer_cost_q(d, b - d).saturating_add(rest_cost); + let trees = rest_trees + 1; + // Strictly better only: ties keep the smaller `d` reached first. + if cand.is_none_or(|(c, t, _)| (cost, trees) < (c, t)) { + cand = Some((cost, trees, d)); + } + } + // `d = 1` is always admissible (i ≥ 1, dmax ≥ 1), so `cand` is set. + best.push(cand.unwrap_or((u64::MAX, u32::MAX, 1))); + } + let (cost_q, trees, _) = best[span]; + let mut schedule = Vec::with_capacity(trees as usize); + let mut i = span; + while i > 0 { + let d = best[i].2; + schedule.push(d as u8); + i -= d as usize; + } + FriScheduleChoice { + cost_q, + trees, + schedule, + } +} + +/// The production schedule DP: [`fri_schedule_by`] under the cost-law +/// objective ([`FRI_COST_WEIGHTS`]) with the cap policy `cap`. +pub fn fri_schedule_with_cost( + b0: u32, + terminal_log: u32, + num_queries: u64, + cap: CapPolicy, + dmax: u32, +) -> FriScheduleChoice { + fri_schedule_by(b0, terminal_log, dmax, &|d, depth| { + fri_layer_cost_q(&FRI_COST_WEIGHTS, d, depth, num_queries, cap) + }) +} + +/// The schedule of [`fri_schedule_with_cost`]. +pub fn fri_schedule( + b0: u32, + terminal_log: u32, + num_queries: u64, + cap: CapPolicy, + dmax: u32, +) -> Vec { + fri_schedule_with_cost(b0, terminal_log, num_queries, cap, dmax).schedule +} + +/// Today's schedule: every committed layer folds by 2. +pub fn legacy_fri_schedule(b0: u32, terminal_log: u32) -> Vec { + vec![1; b0.saturating_sub(terminal_log) as usize] +} + +/// Why a proof format cannot be laid out for a table. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FriFormatError { + /// The schedule override does not cover this table's committed folds + /// exactly, or has an exponent outside `1..=FRI_SCHEDULE_DMAX`. + ScheduleOverrideMismatch, +} + +impl core::fmt::Display for FriFormatError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::ScheduleOverrideMismatch => { + f.write_str("the FRI schedule override does not cover this table's committed folds") + } + } + } +} + +/// Everything the fold layout needs to know about the proof format, for one +/// table. All fields are verifier-side constants; none is ever read from a +/// proof. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FriFormat { + pub mode: FriMode, + /// The RESOLVED one-row choice for this table (never `Auto`). + pub one_row: bool, + /// FRI query count (the opening count of every FRI tree). + pub num_queries: u64, + /// The active Merkle-cap policy (an input of the DP: the cap changes each layer's path cost). + pub cap: CapPolicy, + /// An explicit schedule that replaces the DP's under [`FriMode::Dp`]. + pub schedule_override: Option, +} + +impl FriFormat { + /// Today's format: pair layers, row-pair openings. The query count and cap + /// policy are unused by the all-ones schedule. + pub const LEGACY: Self = Self { + mode: FriMode::Pair, + one_row: false, + num_queries: 0, + cap: CapPolicy::Off, + schedule_override: None, + }; + + /// The format of a table proved under `options` whose trace trees use + /// the RESOLVED leaf layout `one_row` (the table's + /// [`crate::leaf_layout::table_leaf_layout`]; `options.format.one_row` may + /// be `Auto`, which only the caller can resolve, from the AIR's widths). + pub fn from_options(options: &ProofOptions, one_row: bool) -> Self { + Self { + mode: options.format.fri_mode, + one_row, + num_queries: options.fri_number_of_queries as u64, + cap: options.format.merkle_cap, + schedule_override: options.format.fri_schedule_override, + } + } + + /// Whether the proof uses today's FRI encoding: one sibling value per + /// committed layer, pair leaves. True exactly for pair + /// layers with row-pair openings; any other format carries every layer's + /// full group, even where the schedule is all ones. Decided by the format, + /// never by the schedule's values. + pub fn is_legacy(&self) -> bool { + self.mode == FriMode::Pair && !self.one_row + } + + /// `Q ×` the per-query in-guest price of this table's whole FRI chain for + /// an LDE of `2^lde_log` folding to a terminal of `2^terminal_log`: every + /// committed layer of [`Self::schedule`] (group layers, or today's pair + /// layers when [`Self::is_legacy`]) plus, for row-pair openings, the + /// uncommitted fold 0 (one fold; the group encoding also squares the + /// point once into the first layer's `y⁻¹`, where the pair encoding + /// squares inside each layer). + pub fn chain_cost_q(&self, lde_log: u32, terminal_log: u32) -> u64 { + let w = &FRI_COST_WEIGHTS; + let (q, cap) = (self.num_queries, self.cap); + let b0 = fri_chain_start(lde_log, self.one_row); + let schedule = self.schedule(lde_log, terminal_log); + let layers = if self.is_legacy() { + fri_schedule_cost_by(b0, &schedule, &|_, depth| { + fri_pair_layer_cost_q(w, depth, q, cap) + }) + } else { + fri_schedule_cost_q(b0, &schedule, q, cap) + } + .unwrap_or(u64::MAX); + let fold0 = if !self.one_row && lde_log > terminal_log { + let per_query = if self.is_legacy() { + w.fold + } else { + w.fold + w.twiddle + }; + q.saturating_mul(per_query) + } else { + 0 + }; + layers.saturating_add(fold0) + } + + /// The committed-layer fold schedule for an LDE of `2^lde_log` folding to a + /// terminal of `2^terminal_log` (the override's, verbatim, when one is + /// set under `Dp`; the layout checks that it fits). + pub fn schedule(&self, lde_log: u32, terminal_log: u32) -> Vec { + let b0 = fri_chain_start(lde_log, self.one_row); + match (self.mode, self.schedule_override) { + (FriMode::Pair, _) => legacy_fri_schedule(b0, terminal_log), + (FriMode::Dp, Some(o)) => o.as_slice().to_vec(), + (FriMode::Dp, None) => fri_schedule( + b0, + terminal_log, + self.num_queries, + self.cap, + FRI_SCHEDULE_DMAX, + ), + } + } +} diff --git a/crypto/stark/src/fri/terminal.rs b/crypto/stark/src/fri/terminal.rs index 716fbcf3d..ac93790f9 100644 --- a/crypto/stark/src/fri/terminal.rs +++ b/crypto/stark/src/fri/terminal.rs @@ -9,6 +9,9 @@ use math::field::element::FieldElement; use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; use math::polynomial::Polynomial; +use crate::fri::schedule::{FRI_SCHEDULE_DMAX, FriFormat, FriFormatError}; +use crate::proof::options::ProofOptions; + /// The FRI early-termination fold layout. /// /// Derived identically by the CPU prover (`commit_phase_from_evaluations`), the @@ -16,11 +19,16 @@ use math::polynomial::Polynomial; /// Keeping the arithmetic in one place is load-bearing: the three callers must /// agree exactly or proofs fail to verify, and a CPU/GPU disagreement would /// surface only on GPU machines. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// +/// The committed layers follow a fold schedule (`crate::fri::schedule`): layer +/// `j` folds by `2^{schedule[j]}`. Today's layout ([`Self::new`]) is the +/// all-ones schedule, built through the same constructor as every other format. +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct FriFoldLayout { /// Folds from the LDE codeword down to the terminal codeword. pub(crate) total_folds: u32, - /// Committed (Merkle-rooted) FRI layers = `total_folds - 1`, or 0 when there + /// Committed (Merkle-rooted) FRI layers = `schedule.len()`. Row-pair + /// layout: `total_folds - 1` under the all-ones schedule, or 0 when there /// is no fold or only a single final fold. pub(crate) num_committed: usize, /// Terminal codeword length = `2^(blowup_log + effective_k)`. @@ -28,10 +36,24 @@ pub(crate) struct FriFoldLayout { /// Terminal polynomial log-degree bound actually used, `min(k, trace_bits)`. /// This is the verifier's `expected_k` and the prover's `effective_log_degree`. pub(crate) effective_k: u32, + /// Fold exponent of each committed layer, first committed layer first. + /// Invariant (checked by every constructor): `(one_row ? 0 : 1) + + /// Σ schedule == total_folds` whenever `total_folds >= 1`, and empty + /// otherwise; every entry is in `1..=FRI_SCHEDULE_DMAX`. + pub(crate) schedule: Vec, + /// Whether the DEEP codeword itself is committed (one-row openings): then + /// the chain starts at the LDE size and there is no uncommitted fold 0. + pub(crate) one_row: bool, + /// Today's FRI encoding ([`FriFormat::is_legacy`]): pair-leaf layer trees + /// and one sibling value per committed layer per query. `false` = group + /// leaves (`H::Batched` over `2^d` values) and the full group per layer — + /// decided by the format, never by the schedule's values, so a `Dp` + /// schedule that happens to be all ones still uses the group encoding. + pub(crate) legacy_encoding: bool, } impl FriFoldLayout { - /// Derive the layout from the LDE codeword size. + /// Today's layout, derived from the LDE codeword size. /// /// * `lde_log` — log2 of the LDE (deep-composition) codeword length. /// * `blowup_log` — log2 of the LDE blowup factor. @@ -42,16 +64,156 @@ impl FriFoldLayout { /// size for traces too small to fold that far (the `.min(lde_log)`). /// Computing `blowup_log + k` in `u32` (both small) sidesteps the /// `1 << (blowup_log + k)` overflow an out-of-range `k` would otherwise cause. + /// + /// This is [`Self::for_format`] at [`FriFormat::LEGACY`]: pair layers, + /// row-pair openings, the all-ones schedule. pub(crate) fn new(lde_log: u32, blowup_log: u32, k: u32) -> Self { let terminal_log = (blowup_log + k).min(lde_log); + let schedule = FriFormat::LEGACY.schedule(lde_log, terminal_log); + // The all-ones schedule covers the committed folds by construction. + Self::assemble(lde_log, blowup_log, terminal_log, false, schedule) + } + + /// The layout under an explicit proof format. `total_folds`, + /// `terminal_len` and `effective_k` do not depend on the format; only the + /// split of the folds into committed layers does. + /// + /// `None` only for a schedule override that does not cover the committed + /// folds exactly (the DP and the all-ones schedules land on the terminal + /// by construction). + pub(crate) fn for_format( + lde_log: u32, + blowup_log: u32, + k: u32, + fmt: &FriFormat, + ) -> Option { + let terminal_log = (blowup_log + k).min(lde_log); + let schedule = fmt.schedule(lde_log, terminal_log); + let mut layout = Self::assemble(lde_log, blowup_log, terminal_log, fmt.one_row, schedule); + layout.legacy_encoding = fmt.is_legacy(); + layout.schedule_is_consistent().then_some(layout) + } + + /// The layout of a table proved under `options` over an LDE of + /// `2^lde_log` with blowup `2^blowup_log`, whose trace trees use the + /// resolved leaf layout `one_row`: what the prover and the host verifier + /// both build. The format comes from `options` and the table's AIR — a + /// verifier-side constant — never from a proof. + pub(crate) fn for_options( + lde_log: u32, + blowup_log: u32, + options: &ProofOptions, + one_row: bool, + ) -> Result { + let fmt = FriFormat::from_options(options, one_row); + Self::for_format( + lde_log, + blowup_log, + u32::from(options.fri_final_poly_log_degree), + &fmt, + ) + .ok_or(FriFormatError::ScheduleOverrideMismatch) + } + + /// The layout for a caller-supplied schedule, or `None` if the schedule + /// does not cover exactly the committed folds (or has an exponent outside + /// `1..=FRI_SCHEDULE_DMAX`). The encoding is the group encoding unless + /// the schedule is today's (row pair, all ones), where it is legacy. + #[cfg(test)] + pub(crate) fn from_schedule( + lde_log: u32, + blowup_log: u32, + k: u32, + one_row: bool, + schedule: Vec, + ) -> Option { + let terminal_log = (blowup_log + k).min(lde_log); + let mut layout = Self::assemble(lde_log, blowup_log, terminal_log, one_row, schedule); + layout.legacy_encoding = !one_row && layout.schedule.iter().all(|&d| d == 1); + layout.schedule_is_consistent().then_some(layout) + } + + /// Whether this layout uses today's FRI encoding (see + /// [`Self::legacy_encoding`]). The device FRI arms branch on this: today's + /// pair loop, or its group-leaf twin. + pub(crate) fn is_legacy(&self) -> bool { + self.legacy_encoding + } + + /// Log2 length of committed layer `j` (0-based): the chain start minus the + /// bits the earlier committed layers consumed. + pub(crate) fn layer_log_len(&self, lde_log: u32, j: usize) -> u32 { + let consumed: u32 = self.schedule[..j].iter().map(|&d| u32::from(d)).sum(); + crate::fri::schedule::fri_chain_start(lde_log, self.one_row) - consumed + } + + /// Depth of committed layer `j`'s tree: its length over `2^{d_j}` leaves. + pub(crate) fn layer_depth(&self, lde_log: u32, j: usize) -> u32 { + self.layer_log_len(lde_log, j) - u32::from(self.schedule[j]) + } + + /// Folding challenges a proof of this layout draws: one per committed + /// layer plus the final fold's for row pairs (fold 0 consumes the first), + /// one per committed layer for one row (layer 0, the input tree, is + /// committed before any challenge); none when nothing folds. + pub(crate) fn num_zetas(&self) -> usize { + if self.total_folds == 0 { + 0 + } else { + self.num_committed + usize::from(!self.one_row) + } + } + + /// Depth of every committed layer's tree, in layer order. + pub(crate) fn layer_depths(&self, lde_log: u32) -> Vec { + (0..self.num_committed) + .map(|j| self.layer_depth(lde_log, j) as usize) + .collect() + } + + /// Opened values per query in the flat `layers_evaluations_sym` vector: + /// one per layer (legacy) or every layer's full group. + pub(crate) fn opened_values_per_query(&self) -> usize { + if self.legacy_encoding { + self.num_committed + } else { + self.schedule.iter().map(|&d| 1usize << d).sum() + } + } + + fn assemble( + lde_log: u32, + blowup_log: u32, + terminal_log: u32, + one_row: bool, + schedule: Vec, + ) -> Self { let total_folds = lde_log - terminal_log; Self { total_folds, - num_committed: total_folds.saturating_sub(1) as usize, + num_committed: schedule.len(), terminal_len: 1usize << terminal_log, effective_k: terminal_log - blowup_log, + schedule, + one_row, + legacy_encoding: !one_row, } } + + /// The constructor invariant (see [`Self::schedule`]). + fn schedule_is_consistent(&self) -> bool { + let entries_ok = self + .schedule + .iter() + .all(|&d| d >= 1 && u32::from(d) <= FRI_SCHEDULE_DMAX); + let covered: u64 = self.schedule.iter().map(|&d| u64::from(d)).sum(); + let expected = if self.total_folds == 0 { + 0 + } else { + u64::from(self.total_folds) - u64::from(!self.one_row) + }; + entries_ok && covered == expected + } } /// Prover side: given a FRI terminal codeword in **bit-reversed** order, diff --git a/crypto/stark/src/fri/vectors.rs b/crypto/stark/src/fri/vectors.rs new file mode 100644 index 000000000..0906f26a0 --- /dev/null +++ b/crypto/stark/src/fri/vectors.rs @@ -0,0 +1,620 @@ +//! The S3 test vectors the host prover exports ((a)–(d) in the README) for the +//! device prover and the in-guest verifier, checked in under +//! `crypto/stark/tests/vectors/zf_fri/` (see the README there). +//! +//! Compiled only for tests and the `test-utils` feature. Everything here is +//! deterministic: the KAT inputs come from [`splitmix64`], proofs are made at +//! `grinding_factor = 0`. `tests::zf_fri_vectors` (Keccak, Blake3) and the +//! prover crate's `tests::zf_rpx_vectors` (RPX) regenerate every file in memory +//! and require it byte-equal to the checked-in one. + +use std::fmt::Write as _; +use std::path::PathBuf; +use std::string::String; +use std::vec::Vec; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::merkle_tree::cap::CapPolicy; +use crypto::merkle_tree::traits::IsStreamingLeafBackend; +use math::fft::bit_reversing::reverse_index; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsFFTField; + +use crate::config::StarkHash; +use crate::examples::read_only_memory_logup::{ + LogReadOnlyPublicInputs, LogReadOnlyRAP, read_only_logup_trace, +}; +use crate::fri::capture::{FriCapture, capture}; +use crate::fri::fri_functions::compute_coset_twiddles_inv; +use crate::fri::group::{group_fold, roots_of_unity_table}; +use crate::fri::schedule::{ + FRI_COST_WEIGHTS, FRI_SCHEDULE_DMAX, fri_chain_start, fri_schedule_with_cost, +}; +use crate::fri::terminal::FriFoldLayout; +use crate::proof::options::{FriMode, FriScheduleOverride, ProofFormat, ProofOptions}; +use crate::prover::{GenericProver, IsStarkProver}; +use crate::trace::TraceTable; +use crate::traits::AIR; +use crate::verifier::{GenericVerifier, IsStarkVerifier}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; +type Ext = FieldElement; + +/// The vectors directory: `crypto/stark/tests/vectors/zf_fri`. +pub fn vectors_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/zf_fri") +} + +/// One vector file: its name in [`vectors_dir`] and its exact bytes. +pub struct VectorFile { + pub name: String, + pub bytes: Vec, +} + +/// Compare `files` with the checked-in ones (byte equality), or write them +/// when `write` is set. Returns the names that differ or are missing. +pub fn check_or_write(files: &[VectorFile], write: bool) -> Vec { + let dir = vectors_dir(); + let mut bad = Vec::new(); + for f in files { + let path = dir.join(&f.name); + if write { + std::fs::create_dir_all(&dir).expect("create the vectors directory"); + std::fs::write(&path, &f.bytes).expect("write a vector file"); + } else if std::fs::read(&path).ok().as_deref() != Some(f.bytes.as_slice()) { + bad.push(f.name.clone()); + } + } + bad +} + +/// SplitMix64: the KAT input generator (stated in the README so any consumer can +/// regenerate the inputs without this crate). +pub fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +/// An ext3 element from three SplitMix64 outputs, each reduced mod p. +pub(crate) fn next_ext(state: &mut u64) -> Ext { + Ext::new([ + Felt::from(splitmix64(state)), + Felt::from(splitmix64(state)), + Felt::from(splitmix64(state)), + ]) +} + +fn limbs(e: &Ext) -> [u64; 3] { + let v = e.value(); + [v[0].canonical(), v[1].canonical(), v[2].canonical()] +} + +fn ext_json(e: &Ext) -> String { + let [a, b, c] = limbs(e); + format!("[{a},{b},{c}]") +} + +fn exts_json(v: &[Ext]) -> String { + let items: Vec = v.iter().map(ext_json).collect(); + format!("[{}]", items.join(",")) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +// --------------------------------------------------------------------------- +// (a) schedules +// --------------------------------------------------------------------------- + +/// (a) The production fold schedules: T ∈ {4, 9, 10}, B = 6..=24, Q ∈ {3, 110}, +/// cap off / auto, the S3 chain (from B − 1) and the S2 chain (from B), each +/// with its cost-law cost (Q × ns) — the DP's output as the format constant it is. +pub fn schedules_json() -> VectorFile { + let mut s = String::from("{\n \"generator\": \"stark::fri::vectors::schedules_json\",\n"); + let w = FRI_COST_WEIGHTS; + let _ = writeln!( + s, + " \"weights_ns\": {{\"compress\": {}, \"select\": {}, \"unpack\": {}, \"hint\": {}, \"compare\": {}, \"fold\": {}, \"twiddle\": {}, \"xalu\": {}, \"balu\": {}}},", + w.cap.compress, + w.cap.select, + w.cap.unpack, + w.cap.hint, + w.cap.compare, + w.fold, + w.twiddle, + w.xalu, + w.balu + ); + let _ = writeln!(s, " \"dmax\": {FRI_SCHEDULE_DMAX},"); + s.push_str(" \"rows\": [\n"); + let mut rows = Vec::new(); + for t in [4u32, 9, 10] { + for q in [3u64, 110] { + for (cap_name, cap) in [("off", CapPolicy::Off), ("auto", CapPolicy::Auto)] { + for b in 6..=24u32 { + for (chain, one_row) in [("s3", false), ("s2", true)] { + let b0 = fri_chain_start(b, one_row); + let c = fri_schedule_with_cost(b0, t, q, cap, FRI_SCHEDULE_DMAX); + rows.push(format!( + " {{\"terminal_log\": {t}, \"queries\": {q}, \"cap\": \"{cap_name}\", \"lde_log\": {b}, \"chain\": \"{chain}\", \"b0\": {b0}, \"schedule\": {:?}, \"cost_q_ns\": {}}}", + c.schedule, c.cost_q + )); + } + } + } + } + } + s.push_str(&rows.join(",\n")); + s.push_str("\n ]\n}\n"); + VectorFile { + name: "a_schedules.json".into(), + bytes: s.into_bytes(), + } +} + +// --------------------------------------------------------------------------- +// (b) group-fold KATs +// --------------------------------------------------------------------------- + +/// The KAT codeword: `2^KAT_LOG` ext3 values from SplitMix64 seed +/// [`KAT_SEED`] (value i = three consecutive outputs), read as a bit-reversed +/// layer on the coset `3·⟨ω_{2^KAT_LOG}⟩`. +pub const KAT_LOG: u32 = 7; +pub const KAT_SEED: u64 = 0x5a46_4652_4933; + +pub fn kat_codeword() -> Vec { + let mut st = KAT_SEED; + (0..1usize << KAT_LOG).map(|_| next_ext(&mut st)).collect() +} + +/// ζ of the fold KAT for exponent `d`: SplitMix64 seeded `KAT_SEED + d`. +pub fn kat_zeta(d: u32) -> Ext { + let mut st = KAT_SEED + u64::from(d); + next_ext(&mut st) +} + +/// (b) For d = 1..=6: the KAT codeword folded d times with ζ, ζ², … (the +/// prover's commit loop), and the verifier's group fold of every group from +/// its slot-0 point (equal by construction; both listed so a device kernel +/// can be checked against either). +pub fn group_fold_json() -> VectorFile { + let o = Felt::from(3u64); + let n = 1usize << KAT_LOG; + let cw = kat_codeword(); + let w = F::get_primitive_root_of_unity(u64::from(KAT_LOG)).expect("root"); + let mut s = String::from("{\n \"generator\": \"stark::fri::vectors::group_fold_json\",\n"); + let _ = writeln!( + s, + " \"layer_log\": {KAT_LOG},\n \"coset_offset\": 3,\n \"codeword\": {},", + exts_json(&cw) + ); + s.push_str(" \"folds\": [\n"); + let mut items = Vec::new(); + for d in 1..=6u32 { + let zeta = kat_zeta(d); + let mut folded = cw.clone(); + let mut tw = compute_coset_twiddles_inv::(&o, n); + crate::fri::fold_times(&mut folded, &zeta, d, &mut tw); + let roots = roots_of_unity_table::(d).expect("table"); + let by_group: Vec = (0..n >> d) + .map(|g| { + // slot 0: y = x_g, so x_g⁻¹ = y⁻¹. + let y = &o * w.pow(reverse_index(g << d, n as u64) as u64); + group_fold::( + &cw[g << d..(g + 1) << d], + &zeta, + &y.inv().expect("nonzero"), + &roots, + ) + }) + .collect(); + assert_eq!( + folded, by_group, + "the prover's folds and the group fold agree" + ); + items.push(format!( + " {{\"d\": {d}, \"zeta\": {}, \"folded\": {}}}", + ext_json(&zeta), + exts_json(&folded) + )); + } + s.push_str(&items.join(",\n")); + s.push_str("\n ]\n}\n"); + VectorFile { + name: "b_group_folds.json".into(), + bytes: s.into_bytes(), + } +} + +// --------------------------------------------------------------------------- +// (c) group-leaf digests +// --------------------------------------------------------------------------- + +/// (c) Under hash `H` (named `hash_name`), for d = 1..=6: the leaf digest of +/// the KAT codeword's first group (`H::Batched` over its 2^d values) and the +/// root of the whole KAT codeword committed as a group-leaf layer tree. +pub fn leaf_digests_json(hash_name: &str) -> VectorFile { + let cw = kat_codeword(); + let mut s = format!( + "{{\n \"generator\": \"stark::fri::vectors::leaf_digests_json\",\n \"hash\": \"{hash_name}\",\n \"codeword\": \"b_group_folds.json codeword\",\n \"leaves\": [\n" + ); + let mut items = Vec::new(); + for d in 1..=6u32 { + let n = 1usize << d; + let leaf = + as IsStreamingLeafBackend>::hash_data_from_slices(&cw[..n], &[]); + let tree = crate::fri::group_tree::(&cw, n).expect("tree"); + items.push(format!( + " {{\"d\": {d}, \"first_leaf\": \"{}\", \"layer_root\": \"{}\"}}", + hex(&leaf), + hex(&tree.root) + )); + } + s.push_str(&items.join(",\n")); + s.push_str("\n ]\n}\n"); + VectorFile { + name: format!("c_leaf_digests_{hash_name}.json"), + bytes: s.into_bytes(), + } +} + +// --------------------------------------------------------------------------- +// (d) small proofs per format +// --------------------------------------------------------------------------- + +/// The (d) proof shape: `LogReadOnlyRAP` (ext3, one aux column), 2^10 rows, +/// blowup 4 (B = 12), k = 2, grinding 0, coset offset 3; Q = 3, or +/// [`CAPPED_QUERIES`] for the capped formats. +pub const PROOF_ROWS: usize = 1 << 10; + +/// The query count of the capped (d) formats: the `auto` cap policy caps a +/// tree opened at least 20 times at height 3, so a Q = 3 proof +/// carries no cap at all. +pub const CAPPED_QUERIES: usize = 20; + +pub fn proof_options(format: ProofFormat, queries: usize) -> ProofOptions { + ProofOptions { + blowup_factor: 4, + fri_number_of_queries: queries, + coset_offset: 3, + grinding_factor: 0, + fri_final_poly_log_degree: 2, + format, + } +} + +/// The formats of (d), with their query counts: `pair` (today), `dp` (the +/// DP's schedule) and `dp_3_1_3` (an explicit uneven schedule, to catch +/// fold-count bugs), all at Q = 3; and `cap_pair` / `cap_dp` (the `auto` Merkle +/// cap on every tree, with today's FRI and with the DP's schedule) at +/// Q = [`CAPPED_QUERIES`] — the combined S1 × S3 vector. +pub fn proof_formats() -> Vec<(&'static str, ProofFormat, usize)> { + let dp = ProofFormat { + fri_mode: FriMode::Dp, + ..ProofFormat::DEFAULT + }; + let cap = ProofFormat { + merkle_cap: CapPolicy::Auto, + ..ProofFormat::DEFAULT + }; + vec![ + ("pair", ProofFormat::DEFAULT, 3), + ("dp", dp, 3), + ( + "dp_3_1_3", + ProofFormat { + fri_schedule_override: FriScheduleOverride::new(&[3, 1, 3]), + ..dp + }, + 3, + ), + ("cap_pair", cap, CAPPED_QUERIES), + ( + "cap_dp", + ProofFormat { + fri_mode: FriMode::Dp, + ..cap + }, + CAPPED_QUERIES, + ), + ] +} + +fn logup_case( + format: ProofFormat, + queries: usize, +) -> ( + LogReadOnlyRAP, + TraceTable, + LogReadOnlyPublicInputs, +) { + let rows = PROOF_ROWS; + let addr: Vec = (0..rows).map(|i| Felt::from((i % 5) as u64 + 1)).collect(); + let val: Vec = (0..rows) + .map(|i| Felt::from(((i % 5) as u64 + 1) * 10)) + .collect(); + let trace: TraceTable = read_only_logup_trace(addr, val); + let cols = trace.columns_main(); + let pi = LogReadOnlyPublicInputs { + a0: cols[0][0], + v0: cols[1][0], + a_sorted_0: cols[2][0], + v_sorted_0: cols[3][0], + m0: cols[4][0], + }; + ( + LogReadOnlyRAP::::new(&proof_options(format, queries)), + trace, + pi, + ) +} + +/// (d) Under hash `H`: per format, the proof's rkyv bytes (`.rkyv`) and a JSON +/// with everything a verifier derives from it — layout, ζ, ι, DEEP values, +/// roots, terminal coefficients, and per query per layer the leaf, slot and +/// opened values. +pub fn proof_vectors(hash_name: &str) -> Vec { + let mut out = Vec::new(); + for (fmt_name, format, queries) in proof_formats() { + out.extend(proof_files::( + hash_name, fmt_name, format, queries, "d_proof", + )); + } + out +} + +/// The formats of (e) (S2): one-row openings with the pair schedule +/// (`one_row_pair`: all-ones groups, the input tree committed in pairs) and +/// with an explicit uneven schedule from the input tree (`one_row_3_2_1_2`, +/// `Σ = 8 = B − T`). +pub fn one_row_proof_formats() -> Vec<(&'static str, ProofFormat)> { + let on = ProofFormat { + one_row: crate::proof::options::OneRowMode::On, + ..ProofFormat::DEFAULT + }; + vec![ + ("one_row_pair", on), + ( + "one_row_3_2_1_2", + ProofFormat { + fri_mode: FriMode::Dp, + fri_schedule_override: FriScheduleOverride::new(&[3, 2, 1, 2]), + ..on + }, + ), + ] +} + +/// (e) The S2 proofs under hash `H`: the (d) shape (at `Q = 3`) proved with one-row +/// openings. Per query the JSON adds the trace leaf (`r`, a leaf index over +/// the whole LDE) and its path length (`log2(lde)`); `deep` is DEEP at the +/// ONE point `x_r` (there is no `deep_sym`), and layer 0 is the input tree +/// (the DEEP codeword itself; its root is `fri_roots[0]`, absorbed before the +/// first ζ, so `zetas` has one entry per layer). +pub fn one_row_proof_vectors(hash_name: &str) -> Vec { + let mut out = Vec::new(); + for (fmt_name, format) in one_row_proof_formats() { + out.extend(proof_files::(hash_name, fmt_name, format, 3, "e_proof")); + } + out +} + +/// (e) One-row trace-leaf digests under hash `H`: a KAT base matrix (16 rows × +/// 5 columns) and ext3 matrix (16 rows × 2 columns) from SplitMix64 seed +/// [`KAT_SEED`] + 100 / + 200, read as bit-reversed LDE columns, committed +/// with one row per leaf AND with row pairs (today's): every leaf digest and +/// the root of each. One row: leaf `i` = the row at bit-reversed position `i` +/// (columns in order, big-endian bytes / the `Batched` felt stream); row pair: +/// rows `2i`, `2i + 1`. +pub fn one_row_leaf_digests_json(hash_name: &str) -> VectorFile { + const ROWS: usize = 16; + let mut st = KAT_SEED + 100; + let base: Vec> = (0..5) + .map(|_| (0..ROWS).map(|_| Felt::from(splitmix64(&mut st))).collect()) + .collect(); + let mut st = KAT_SEED + 200; + let ext: Vec> = (0..2) + .map(|_| (0..ROWS).map(|_| next_ext(&mut st)).collect()) + .collect(); + let mut s = format!( + "{{\n \"generator\": \"stark::fri::vectors::one_row_leaf_digests_json\",\n \"hash\": \"{hash_name}\",\n \"rows\": {ROWS},\n" + ); + let base_json: Vec = base + .iter() + .map(|c| { + let v: Vec = c.iter().map(|x| x.canonical().to_string()).collect(); + format!("[{}]", v.join(",")) + }) + .collect(); + let _ = writeln!(s, " \"base_columns\": [{}],", base_json.join(",")); + let ext_cols: Vec = ext.iter().map(|c| exts_json(c)).collect(); + let _ = writeln!(s, " \"ext_columns\": [{}],", ext_cols.join(",")); + let mut items = Vec::new(); + for (layout_name, rows_per_leaf) in [("row", 1usize), ("row_pair", 2)] { + let b = crate::commitment::leaves_bit_reversed_grouped::>( + &base, + rows_per_leaf, + ); + let (_, b_root) = + crate::commitment::commit_bit_reversed_with::>(&base, rows_per_leaf) + .expect("base tree"); + let e = + crate::commitment::leaves_bit_reversed_grouped::>(&ext, rows_per_leaf); + let (_, e_root) = + crate::commitment::commit_bit_reversed_with::>(&ext, rows_per_leaf) + .expect("ext tree"); + let hexes = |v: &[crate::config::Commitment]| { + let h: Vec = v.iter().map(|x| format!("\"{}\"", hex(x))).collect(); + format!("[{}]", h.join(",")) + }; + items.push(format!( + " {{\"layout\": \"{layout_name}\", \"rows_per_leaf\": {rows_per_leaf}, \"base_leaves\": {}, \"base_root\": \"{}\", \"ext_leaves\": {}, \"ext_root\": \"{}\"}}", + hexes(&b), + hex(&b_root), + hexes(&e), + hex(&e_root) + )); + } + s.push_str(" \"layouts\": [\n"); + s.push_str(&items.join(",\n")); + s.push_str("\n ]\n}\n"); + VectorFile { + name: format!("e_leaf_digests_{hash_name}.json"), + bytes: s.into_bytes(), + } +} + +/// One proof's `{prefix}_{hash}_{format}.{json,rkyv}` pair (the (d) and (e) +/// files). The table's leaf layout is resolved as the prover and verifier +/// resolve it; the JSON keeps the (d) schema for row pairs byte for byte and +/// adds the one-row fields otherwise. +fn proof_files( + hash_name: &str, + fmt_name: &str, + format: ProofFormat, + queries: usize, + prefix: &str, +) -> Vec { + let mut out = Vec::new(); + { + let (air, mut trace, pi) = logup_case(format, queries); + let one_row = crate::leaf_layout::table_leaf_layout(&air, PROOF_ROWS).is_one_row(); + let proof = GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + let (ok, records) = capture(|| { + GenericVerifier::::verify( + &proof, + &air, + &mut DefaultTranscript::::new(&[]), + ) + }); + assert!(ok, "the {prefix} proof must verify"); + let rec = FriCapture::::from_any(records[0].as_ref()).expect("one ext3 record"); + let bytes = rkyv::to_bytes::(&proof) + .expect("rkyv") + .to_vec(); + let lde_log = PROOF_ROWS.trailing_zeros() + 2; + let layout = + FriFoldLayout::for_options(lde_log, 2, air.options(), one_row).expect("layout"); + let stem = format!("{prefix}_{hash_name}_{fmt_name}"); + + let mut s = format!( + "{{\n \"generator\": \"stark::fri::vectors::proof_vectors\",\n \"hash\": \"{hash_name}\",\n \"format\": \"{fmt_name}\",\n \"proof_rkyv\": \"{stem}.rkyv\",\n \"proof_rkyv_len\": {},\n", + bytes.len() + ); + let _ = writeln!( + s, + " \"air\": \"LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))\",\n \"trace_rows\": {PROOF_ROWS},\n \"lde_log\": {lde_log},\n \"blowup\": 4,\n \"fri_final_poly_log_degree\": 2,\n \"queries\": {queries},\n \"grinding_factor\": 0,\n \"coset_offset\": 3," + ); + if !format.merkle_cap.is_off() { + // The capped formats only (the Q = 3 files are unchanged): the + // policy and every tree's height, from the verifier's own + // `StarkCaps`. Each capped tree's cap rides at the end of query + // 0's path (the owner path), so that `path_len` is `D − c + 2^c`. + let caps = crate::merkle_caps::StarkCaps::for_options( + air.options(), + lde_log as usize, + one_row, + ) + .expect("caps"); + let _ = writeln!( + s, + " \"merkle_cap\": \"{}\",\n \"trace_tree_depth\": {},\n \"trace_cap\": {},\n \"fri_tree_depths\": {:?},\n \"fri_caps\": {:?},", + format.merkle_cap, caps.trace_depth, caps.trace, caps.fri_depths, caps.fri + ); + } + if one_row { + let _ = writeln!( + s, + " \"one_row\": true,\n \"query_bound\": {},\n \"trace_tree_depth\": {lde_log},", + 1u64 << lde_log + ); + } + let _ = writeln!( + s, + " \"legacy_encoding\": {},\n \"total_folds\": {},\n \"terminal_len\": {},\n \"schedule\": {:?},", + layout.is_legacy(), + layout.total_folds, + layout.terminal_len, + layout.schedule + ); + let roots: Vec = proof + .fri_layers_merkle_roots + .iter() + .map(|r| format!("\"{}\"", hex(r))) + .collect(); + let _ = writeln!(s, " \"fri_roots\": [{}],", roots.join(",")); + let _ = writeln!(s, " \"zetas\": {},", exts_json(&rec.zetas)); + let _ = writeln!( + s, + " \"terminal_coeffs\": {},", + exts_json(&proof.fri_final_poly_coeffs) + ); + s.push_str(" \"queries_detail\": [\n"); + let mut qs = Vec::new(); + for (qi, &iota) in rec.iotas.iter().enumerate() { + let dec = &proof.query_list[qi]; + let mut layers = Vec::new(); + let mut index = iota; + let mut off = 0usize; + for (j, &d) in layout.schedule.iter().enumerate() { + let (leaf, slot, n) = if layout.is_legacy() { + (index >> 1, index & 1, 1usize) + } else { + (index >> d, index & ((1 << d) - 1), 1usize << d) + }; + layers.push(format!( + "{{\"layer\": {j}, \"d\": {d}, \"position\": {index}, \"leaf\": {leaf}, \"slot\": {slot}, \"values\": {}, \"path_len\": {}}}", + exts_json(&dec.layers_evaluations_sym[off..off + n]), + dec.layers_auth_paths[j].merkle_path.len() + )); + off += n; + index = if layout.is_legacy() { + index >> 1 + } else { + index >> d + }; + } + if one_row { + let opening = &proof.deep_poly_openings[qi]; + qs.push(format!( + " {{\"iota\": {iota}, \"trace_leaf\": {iota}, \"trace_path_len\": {}, \"deep\": {}, \"terminal_position\": {index}, \"layers\": [{}]}}", + opening.main_trace_polys.proof.merkle_path.len(), + ext_json(&rec.deep[qi]), + layers.join(", ") + )); + } else { + qs.push(format!( + " {{\"iota\": {iota}, \"deep\": {}, \"deep_sym\": {}, \"terminal_position\": {index}, \"layers\": [{}]}}", + ext_json(&rec.deep[qi]), + ext_json(&rec.deep_sym[qi]), + layers.join(", ") + )); + } + } + s.push_str(&qs.join(",\n")); + s.push_str("\n ]\n}\n"); + out.push(VectorFile { + name: format!("{stem}.json"), + bytes: s.into_bytes(), + }); + out.push(VectorFile { + name: format!("{stem}.rkyv"), + bytes, + }); + } + out +} diff --git a/crypto/stark/src/gpu_lde.rs b/crypto/stark/src/gpu_lde.rs index 8782c6923..acb98895e 100644 --- a/crypto/stark/src/gpu_lde.rs +++ b/crypto/stark/src/gpu_lde.rs @@ -4,6 +4,21 @@ //! back to CPU for extension-field columns and small columns where kernel //! launch overhead dominates. Produces the same natural-order, non-canonical //! LDE evaluations as the CPU path. +//! +//! Admission is bytes-aware and the device path, once admitted, is the ONLY +//! path: see the "Bytes-aware admission" section. A table the device admits +//! proves on the device or the prove aborts with a diagnostic; host RAM is a +//! cache, not a compute path, and the one way back to a host commit is the +//! test-only switch [`TEST_ONLY_HOST_FALLBACK_ENV`]. +//! +//! The tree-building entries here are generic over a Merkle backend `B` that +//! they never call: the leaf and parent hashing happens in the `math-cuda` +//! kernels, and `B` only types the host `MerkleTree` the root is wrapped in. +//! `B` is therefore bound to [`DeviceTreeBackend`] rather than +//! `IsMerkleTreeBackend`: its `COMMITMENT_HASH` constant is the dispatch key +//! handed to `math-cuda` (selecting the keccak or the BLAKE3 kernel family at +//! every leaf, level and FRI-layer launch), so the label cannot disagree with +//! the kernel that produced the bytes. use core::mem::transmute_copy; use std::any::TypeId; @@ -20,7 +35,6 @@ use math_cuda::{CudaSlice, CudaStream}; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; use crypto::merkle_tree::merkle::MerkleTree; use crypto::merkle_tree::proof::Proof; -use crypto::merkle_tree::traits::IsMerkleTreeBackend; use math::field::element::FieldElement; use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; use math::field::goldilocks::GoldilocksField; @@ -29,12 +43,74 @@ use math::traits::AsBytes; #[cfg(feature = "parallel")] use rayon::prelude::{IndexedParallelIterator, ParallelIterator, ParallelSliceMut}; -use crate::config::{Commitment, FriLayerMerkleTreeBackend}; +use crate::config::{Commitment, CommitmentHash, DeviceTreeBackend}; use crate::domain::Domain; use crate::fri::fri_commitment::FriLayer; use crate::fri::fri_decommit::FriDecommitment; use crate::trace::LDETraceTable; +/// The `math_cuda` dispatch key for a commitment hash. +/// +/// Total over [`CommitmentHash`], so a variant added there without a +/// [`math_cuda::DeviceHash`] twin fails to compile HERE, naming the gap, rather +/// than at whatever `match` first meets it. Every tree entry point below +/// dispatches on the result, and `math-cuda` either has kernels for the key or +/// aborts loudly at the first launch — a build must never quietly commit under +/// a hash the configuration did not name (HASH-PINNING.md), so there is no +/// byte-hash fallback anywhere on this path. +const fn device_hash_for(hash: CommitmentHash) -> math_cuda::DeviceHash { + match hash { + CommitmentHash::Keccak256 => math_cuda::DeviceHash::Keccak256, + CommitmentHash::Blake3 => math_cuda::DeviceHash::Blake3, + CommitmentHash::Rpx256 => math_cuda::DeviceHash::Rpx256, + CommitmentHash::Poseidon1 => math_cuda::DeviceHash::Poseidon1, + } +} + +/// The inverse of [`device_hash_for`] — total over [`math_cuda::DeviceHash`], +/// so a device key no commitment hash names fails to compile here too. +const fn commitment_hash_for(hash: math_cuda::DeviceHash) -> CommitmentHash { + match hash { + math_cuda::DeviceHash::Keccak256 => CommitmentHash::Keccak256, + math_cuda::DeviceHash::Blake3 => CommitmentHash::Blake3, + math_cuda::DeviceHash::Rpx256 => CommitmentHash::Rpx256, + math_cuda::DeviceHash::Poseidon1 => CommitmentHash::Poseidon1, + } +} + +/// ★ Every [`CommitmentHash`] variant has exactly one [`math_cuda::DeviceHash`] +/// twin, and the pairing is not crossed. +/// +/// The two `match`es above being total already proves each side maps +/// somewhere; this proves the two maps are inverse to each other, which is what +/// rules out a mis-paired arm (`Rpx256 => DeviceHash::Poseidon1`) — the one +/// editing error that would hand a tree a name its kernels do not deserve. A +/// fifth variant fails `device_hash_for` first +/// and is added to this list with its arm. Discriminants are compared because +/// `PartialEq` is not `const`. +const _: () = { + const ALL: [CommitmentHash; 4] = [ + CommitmentHash::Keccak256, + CommitmentHash::Blake3, + CommitmentHash::Rpx256, + CommitmentHash::Poseidon1, + ]; + let mut i = 0; + while i < ALL.len() { + let back = commitment_hash_for(device_hash_for(ALL[i])); + assert!( + back as u8 == ALL[i] as u8, + "a CommitmentHash must round-trip through its DeviceHash twin" + ); + i += 1; + } +}; + +/// The `math_cuda` dispatch key for `B`'s hash. +pub(crate) fn device_hash_of() -> math_cuda::DeviceHash { + device_hash_for(B::COMMITMENT_HASH) +} + /// Break-even LDE size. For LDE sizes smaller than this, the CPU /// `coset_lde_full_expand` completes in a few hundred microseconds and the /// GPU's tens of kernel launches plus H2D/D2H round-trip is a net loss. The @@ -54,6 +130,13 @@ use crate::trace::LDETraceTable; /// it — see [`DEFAULT_DEVICE_ONLY_MIN_LDE`]. const DEFAULT_GPU_LDE_THRESHOLD: usize = 1 << 14; +/// Whether an LDE of `lde_size` rows clears the device commit's row floor — the +/// first of the admission rules, the one a caller can ask before building a +/// device input it would otherwise waste. +pub(crate) fn clears_device_floor(lde_size: usize) -> bool { + lde_size >= gpu_lde_threshold() +} + fn gpu_lde_threshold() -> usize { static CACHED: OnceLock = OnceLock::new(); *CACHED.get_or_init(|| { @@ -81,6 +164,10 @@ fn gpu_lde_threshold() -> usize { /// nothing to fall back to and aborts regardless of the host LDE. Lowering /// the commit threshold therefore widens that one abort site even though it /// leaves this envelope alone. +/// +/// Inside the envelope the contract is stricter still: a device-only table's +/// host recovery is refused in production ([`refuse_host_recovery`]), so a +/// runtime decline there is an abort with a diagnostic, never a slow prove. const DEFAULT_DEVICE_ONLY_MIN_LDE: usize = 1 << 19; fn gpu_device_only_threshold() -> usize { @@ -93,8 +180,294 @@ fn gpu_device_only_threshold() -> usize { }) } +// ============================================================================ +// Bytes-aware admission — the ONE predicate every dispatch site consults +// ============================================================================ +// +// Two rules decide whether a dispatch may ask the device for anything, and +// both live here so no site carries a private copy of either: +// +// - the row FLOOR (`gpu_lde_threshold`): below it the host path is the faster +// one — launch overhead dominates — and taking it is policy, not a fallback; +// - the bytes CEILING (the card's admission budget): what the site is about to +// allocate must fit the card. Width enters here and only here: a `2^21 × 449` +// table clears the floor at any width, and used to sail into an allocation +// failure that quietly became a host commit. +// +// What happens past admission is the other half of the contract. Host RAM is a +// cache, not a compute path: a table the device ADMITS proves on the device or +// the prove stops, loudly, with the shape, the bytes and the live VRAM in the +// message ([`abort_or_test_fallback`]). The only way back to a host commit is +// the test-only switch [`TEST_ONLY_HOST_FALLBACK_ENV`]. +// +// LOCKSTEP: [`admit`] is a pure function of `(lde_size, bytes)` and two +// process constants (the floor, the card's budget). R1 sizes its commit and +// asks; every handle-bearing site re-derives admission later from the same +// `lde_size` and its own, smaller, transient, and gets the same answer. Live +// free memory is deliberately NOT an input: it would let a table admitted at +// R1 be declined at R4, which for a device-only table is an abort with the +// work already done. FRI re-derives at width 1 (one ext3 column); the ceiling +// is an upper bound, so a narrow transient always clears it — a cells FLOOR +// would degenerate there, which is why the floor stays a row count. + +// The arithmetic — the device-set model and the pure predicate — lives in +// `crate::device_set`, free of the `cuda` gate, because the per-table +// scheduler's throttle reads the same model on every build. Re-exported here +// so the dispatch layer's callers keep one path. +use crate::device_set::BASE_BYTES; +pub use crate::device_set::{ + Admission, CommitDeviceSet, admit_bytes, commit_device_set, commit_device_set_rpl, ext3_bytes, + full_tree_bytes, tree_bytes_for, +}; + +/// The process predicate: `gpu_lde_threshold()` as the floor and the card's +/// admission budget ([`device_vram_budget_bytes`]: 80% of device memory, or +/// `LAMBDA_VM_VRAM_BUDGET_MB`) as the ceiling. Both are fixed for the life of +/// the process, so the same `(lde_size, bytes)` gets the same answer at R1 and +/// at every later re-derivation. +pub(crate) fn admit(lde_size: usize, bytes: u64) -> Admission { + match device_vram_budget_bytes() { + None => Admission::NoDevice, + Some(budget) => admit_bytes(lde_size, bytes, gpu_lde_threshold(), budget), + } +} + +/// One dispatch's identity for the diagnostics: which device stage, and the +/// shape it was sized for. The table's NAME is not known at this layer; the +/// prover's driver re-raises the panic payload with the message intact, and +/// its own `[gpu]` lines name the table. +#[derive(Clone, Copy, Debug)] +pub(crate) struct DispatchShape<'a> { + /// The AIR's name — the prover passes it down so the diagnostic names the + /// table at the site, not only in the re-raised payload. + pub table: &'a str, + pub what: &'static str, + pub n: usize, + pub base_cols: usize, + pub blowup: usize, +} + +/// Why a stage that must run on the device could not. +#[derive(Debug)] +pub(crate) enum DevicePathFailure { + OverBudget { bytes: u64, budget: u64 }, + DeviceError(String), +} + +fn gib(bytes: u64) -> f64 { + bytes as f64 / (1u64 << 30) as f64 +} + +/// Live `free / total` device memory for the diagnostics — the one place the +/// live number is read; admission itself never looks at it (LOCKSTEP). +fn live_vram_line() -> String { + match math_cuda::device::backend() + .ok() + .and_then(|be| be.device_mem_info()) + { + Some((free, total)) => format!( + "live VRAM free {:.2} GiB of {:.2} GiB", + gib(free), + gib(total) + ), + None => "live VRAM unavailable".to_string(), + } +} + +/// The live device posture every device-path failure reports: free/total VRAM +/// and the mempool release threshold. Shared by the dispatch layer's aborts and +/// the prover's clean errors so the two read the same line. +pub(crate) fn device_path_status() -> String { + format!("{}; {}", live_vram_line(), mempool_line()) +} + +fn mempool_line() -> String { + match math_cuda::device::mempool_release_threshold_bytes() { + u64::MAX => "mempool retains freed blocks (release threshold unset)".to_string(), + t => format!("mempool release threshold {} MiB", t >> 20), + } +} + +/// The diagnostic every abort and every test-only fallback prints: the stage +/// and shape, the device set term by term, the admission budget, the live +/// free/total VRAM and the mempool posture. +fn device_path_diagnostic( + shape: &DispatchShape<'_>, + set: Option<&CommitDeviceSet>, + failure: &DevicePathFailure, +) -> String { + let DispatchShape { + table, + what, + n, + base_cols, + blowup, + } = shape; + let reason = match failure { + DevicePathFailure::OverBudget { bytes, budget } => format!( + "over the VRAM admission budget: needs {bytes} B ({:.2} GiB), budget {budget} B ({:.2} GiB)", + gib(*bytes), + gib(*budget) + ), + DevicePathFailure::DeviceError(e) => format!("device error after admission: {e}"), + }; + let set_line = match set { + Some(s) => format!( + "; device set LDE {:.2} GiB + snapshot {:.2} GiB + tree {:.3} GiB + scratch {:.3} GiB = {:.2} GiB", + gib(s.lde_bytes), + gib(s.snapshot_bytes), + gib(s.tree_bytes), + gib(s.scratch_bytes), + gib(s.total()) + ), + None => String::new(), + }; + format!( + "table {table}: {what}: rows {n} x {base_cols} base cols @ blowup {blowup} (LDE {}); {reason}{set_line}; {}", + n.saturating_mul(*blowup), + device_path_status() + ) +} + +/// The test-only switch that turns the loud abort back into the old host +/// commit. Named so it cannot be read as a production knob; a banner is +/// printed when it is honoured. Production runs never set it. +pub const TEST_ONLY_HOST_FALLBACK_ENV: &str = "LAMBDA_VM_TEST_ONLY_HOST_FALLBACK"; + +/// Whether a device failure may fall back to host compute in this process. +/// `true` under [`TEST_ONLY_HOST_FALLBACK_ENV`], under the +/// [`gpu_force_downgrade`] test hook (whose whole purpose is to exercise the +/// host recovery), and in a `test-cuda-faults` build (the fault-injection +/// suite asserts on the recoveries). None of the three is a production +/// configuration; the banner says so once. +pub(crate) fn test_only_host_fallback() -> bool { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + let by_env = std::env::var(TEST_ONLY_HOST_FALLBACK_ENV).is_ok_and(|v| v != "0"); + let by_hook = gpu_force_downgrade(); + let by_feature = cfg!(feature = "test-cuda-faults"); + let on = by_env || by_hook || by_feature; + if on { + let why = if by_env { + TEST_ONLY_HOST_FALLBACK_ENV + } else if by_hook { + "LAMBDA_VM_GPU_FORCE_DOWNGRADE" + } else { + "feature test-cuda-faults" + }; + eprintln!( + "[gpu] TEST-ONLY host fallback ENABLED ({why}): a device failure falls back to \ + host compute instead of aborting the prove. Never a production configuration." + ); + } + on + }) +} + +/// Refuse the host path — or take it under the test-only switch. +/// +/// Returns only when [`test_only_host_fallback`] is set, after printing the +/// diagnostic tagged TEST-ONLY; the caller then returns `None` and its host +/// arm runs. Otherwise panics with the diagnostic: the prover's driver threads +/// catch and re-raise it on the calling thread (`run_admitted`), so the prove +/// stops with the reason instead of finishing hours later on the CPU. +pub(crate) fn abort_or_test_fallback( + shape: &DispatchShape<'_>, + set: Option<&CommitDeviceSet>, + failure: DevicePathFailure, +) { + let msg = device_path_diagnostic(shape, set, &failure); + if test_only_host_fallback() { + eprintln!("[gpu] TEST-ONLY host fallback: {msg}"); + return; + } + panic!("[gpu] ABORT: the device path is the production path and it is unavailable — {msg}"); +} + +/// R1 commit admission: `None` when the device is absent or the table is +/// below the floor (the host commit is the right one), `Some(())` when +/// admitted. Over budget is an abort — or, under the test-only switch, a +/// reported host commit. +fn admit_commit(lde_size: usize, shape: &DispatchShape<'_>, set: &CommitDeviceSet) -> Option<()> { + match admit(lde_size, set.total()) { + Admission::NoDevice | Admission::BelowFloor { .. } => None, + Admission::Admitted { .. } => Some(()), + Admission::OverBudget { bytes, budget } => { + abort_or_test_fallback( + shape, + Some(set), + DevicePathFailure::OverBudget { bytes, budget }, + ); + None + } + } +} + +/// [`admit_commit`] for an input that is ALREADY resident on device (the +/// LogUp aux build's `ResidentAux`): no row floor — the data is there, and a +/// decline would not be "take the faster host path" but "download it to +/// commit on the host" — only the bytes ceiling. +fn admit_resident_commit(shape: &DispatchShape<'_>, set: &CommitDeviceSet) -> Option<()> { + let budget = device_vram_budget_bytes()?; + match admit_bytes(usize::MAX, set.total(), 0, budget) { + Admission::OverBudget { bytes, budget } => { + abort_or_test_fallback( + shape, + Some(set), + DevicePathFailure::OverBudget { bytes, budget }, + ); + None + } + _ => Some(()), + } +} + +/// Admission for the R2–R4 transients (parts LDE, trees, DEEP, FRI, the +/// inverted denominators): `true` when admitted. A decline is the caller's +/// documented host arm; over budget is reported so it is never a silent one. +/// Whether that arm may run at all is decided downstream, where a device-only +/// table's recovery goes through [`refuse_host_recovery`]. +fn admit_transient(lde_size: usize, bytes: u64, what: &str) -> bool { + match admit(lde_size, bytes) { + Admission::Admitted { .. } => true, + Admission::NoDevice | Admission::BelowFloor { .. } => false, + Admission::OverBudget { bytes, budget } => { + eprintln!( + "[gpu] {what} declined at LDE {lde_size}: over the VRAM admission budget \ + (needs {bytes} B = {:.2} GiB, budget {budget} B = {:.2} GiB); the host arm runs", + gib(bytes), + gib(budget) + ); + false + } + } +} + +/// The gate on every "download the resident data and continue on the host" +/// recovery of a device-only table. In production that recovery IS the +/// failure to report: it aborts here with the shape and the live VRAM. Under +/// the test-only switch it prints the same line tagged TEST-ONLY and returns, +/// and the recovery proceeds. +fn refuse_host_recovery(what: &str, rows: usize, main_cols: usize, aux_cols: usize) { + let msg = format!( + "{what}: rows {rows} main cols {main_cols} aux cols {aux_cols}; {}", + device_path_status() + ); + if test_only_host_fallback() { + eprintln!("[gpu] TEST-ONLY host recovery: {msg}"); + return; + } + panic!( + "[gpu] ABORT: a device-only table would continue on the HOST (host RAM is a cache, \ + not a compute path) — {msg}" + ); +} + /// Test hook: decline the device R2 path unconditionally so device-only /// tables exercise the [`materialize_lde_trace_host`] recovery end to end. +/// Setting it also enables the test-only host fallback +/// ([`test_only_host_fallback`]) — the recovery it exists to exercise would +/// otherwise abort. pub(crate) fn gpu_force_downgrade() -> bool { static CACHED: OnceLock = OnceLock::new(); *CACHED.get_or_init(|| std::env::var("LAMBDA_VM_GPU_FORCE_DOWNGRADE").is_ok_and(|v| v != "0")) @@ -161,26 +534,49 @@ pub fn reset_all_gpu_call_counters() { GPU_BARY_CALLS.store(0, Ordering::Relaxed); GPU_COMP_POLY_TREE_CALLS.store(0, Ordering::Relaxed); GPU_DEEP_CALLS.store(0, Ordering::Relaxed); + GPU_FUSED_DEEP_CALLS.store(0, Ordering::Relaxed); GPU_FRI_CALLS.store(0, Ordering::Relaxed); GPU_BATCH_INVERT_CALLS.store(0, Ordering::Relaxed); GPU_LOGUP_CALLS.store(0, Ordering::Relaxed); GPU_COMPOSITION_CALLS.store(0, Ordering::Relaxed); GPU_OPENING_GATHER_CALLS.store(0, Ordering::Relaxed); + GPU_CAP_READ_CALLS.store(0, Ordering::Relaxed); GPU_DEVICE_ONLY_CALLS.store(0, Ordering::Relaxed); GPU_DEVICE_ONLY_DOWNGRADES.store(0, Ordering::Relaxed); GPU_RESIDENT_AUX_RETRIES.store(0, Ordering::Relaxed); GPU_RESIDENT_AUX_DOWNGRADES.store(0, Ordering::Relaxed); GPU_COMPOSITION_PARTS_DOWNLOADS.store(0, Ordering::Relaxed); - GPU_GRIND_CALLS.store(0, Ordering::Relaxed); + GPU_ONE_ROW_TREES.store(0, Ordering::Relaxed); + GPU_ONE_ROW_TREE_PEAK_BYTES.store(0, Ordering::Relaxed); + GPU_ONE_ROW_FRI_CALLS.store(0, Ordering::Relaxed); + #[cfg(feature = "cuda")] + crypto::grinding::reset_gpu_grind_calls(); } -/// Successful GPU proof-of-work grind dispatches — one per table whose round-4 -/// nonce search ran on device and produced a nonce that passed the host -/// validity check (a device miss or an invalid kernel result falls back to the -/// CPU search and is not counted). -pub(crate) static GPU_GRIND_CALLS: AtomicU64 = AtomicU64::new(0); +/// Successful GPU proof-of-work grind dispatches on the KECCAK kernel. Counted +/// by [`crypto::grinding`], which owns the dispatch both provers share. +/// +/// ⚠ Keccak only. A proof under an algebraic pin increments +/// [`gpu_grind_calls_rpx`] instead, and reading this one alone would report +/// ZERO for a run whose every grind reached the device. pub fn gpu_grind_calls() -> u64 { - GPU_GRIND_CALLS.load(Ordering::Relaxed) + #[cfg(feature = "cuda")] + return crypto::grinding::gpu_grind_calls(); + #[cfg(not(feature = "cuda"))] + return 0; +} + +/// Successful GPU proof-of-work grind dispatches on the RPX kernel. +/// +/// ★ This is the falsifier for the whole algebraic grind path. Both arms return +/// a valid nonce whether or not the device ran, so a zero here on a block proved +/// under the RPX pin means every grind fell to the host rayon search — ~2^20 RPX +/// permutations per table per epoch — with nothing failing and no other signal. +pub fn gpu_grind_calls_rpx() -> u64 { + #[cfg(feature = "cuda")] + return crypto::grinding::gpu_grind_calls_rpx(); + #[cfg(not(feature = "cuda"))] + return 0; } pub(crate) static GPU_EXTEND_HALVES_CALLS: AtomicU64 = AtomicU64::new(0); @@ -209,6 +605,19 @@ pub fn gpu_logup_calls() -> u64 { /// round-2 constraint evaluation took the fused on-device path (a failed attempt /// or a gate miss falls back to the CPU accumulation and is not counted). pub(crate) static GPU_COMPOSITION_CALLS: AtomicU64 = AtomicU64::new(0); + +/// The host↔card staging traffic since the process started, one line +/// (`math_cuda::device::staging_report`): for crates that log it without +/// depending on `math-cuda` themselves. +pub fn staging_report() -> String { + math_cuda::device::staging_report() +} + +/// `LAMBDA_VM_DEEP_HOISTED`'s line (`math_cuda::deep::deep_hoisted_report`). +pub fn deep_hoisted_report() -> String { + math_cuda::deep::deep_hoisted_report() +} + pub fn gpu_composition_calls() -> u64 { GPU_COMPOSITION_CALLS.load(Ordering::Relaxed) } @@ -224,6 +633,15 @@ pub fn gpu_opening_gather_calls() -> u64 { GPU_OPENING_GATHER_CALLS.load(Ordering::Relaxed) } +/// Merkle caps read off a device-resident tree ([`read_cap_dev`]) — one per +/// capped tree whose nodes live on the device (main, aux, composition, FRI +/// layer). Zero under the default format, where no tree is capped; under a +/// cap policy a device prove with this at zero never took the device arm. +pub(crate) static GPU_CAP_READ_CALLS: AtomicU64 = AtomicU64::new(0); +pub fn gpu_cap_read_calls() -> u64 { + GPU_CAP_READ_CALLS.load(Ordering::Relaxed) +} + /// Tables whose round-1 LDE was kept device-only (host trace D2H skipped) — the /// Stage-3 full-residency win. Incremented once per main trace that took the /// `device_only` path. Zero means every table kept its host copy (gate never @@ -288,15 +706,14 @@ pub(crate) fn device_only_disabled() -> bool { /// requires. /// /// If a precondition is nonetheless violated at runtime (mis-gate or -/// transient GPU error), what happens depends on the round. R2 and the R1 -/// resident-aux commit recover: they download what the host arms need (the -/// resident LDEs at R2, the resident aux trace plus the main LDE at R1), bump -/// their site's counter ([`GPU_DEVICE_ONLY_DOWNGRADES`] at R2, -/// [`GPU_RESIDENT_AUX_DOWNGRADES`] at R1) and continue host-backed — slower, -/// never wrong — aborting only when the resident handles cannot serve the -/// data. R3 and R4 have no such recovery: the R3 barycentric arms assert on -/// the buffer they are about to read and the R4 guards on `host_trace_empty`, -/// both failing loudly rather than reading an empty host trace. +/// transient GPU error), the table's recovery reaches +/// [`refuse_host_recovery`]: in production that is a loud abort with the shape +/// and the live VRAM — host RAM is a cache, not a compute path. Under the +/// test-only fallback ([`test_only_host_fallback`]) R2 downloads what the host +/// arms need (the resident LDEs), bumps [`GPU_DEVICE_ONLY_DOWNGRADES`] and +/// continues host-backed; the R1 resident-aux commit has no host recovery at +/// all (a decline after the drain-and-retry is a `ProvingError::DevicePath`), +/// and R3 and R4 assert on the buffer they are about to read. /// /// `zerofier_uniform` must be the R1-derived conservative form (all constraints /// share `end_exemptions == 0`), which implies `ZerofierEvaluations::is_uniform` @@ -381,7 +798,11 @@ where } let n = columns[0].len(); let lde_size = n.saturating_mul(blowup_factor); - if lde_size < gpu_lde_threshold() { + // One fresh LDE output per column on device. + let bytes = (lde_size as u64) + .saturating_mul(columns.len() as u64) + .saturating_mul(BASE_BYTES); + if !admit(lde_size, bytes).is_admitted() { return LayoutDispatch::Skip; } if TypeId::of::() != TypeId::of::() { @@ -409,7 +830,8 @@ where } let n = columns[0].len(); let lde_size = n.saturating_mul(blowup_factor); - if lde_size < gpu_lde_threshold() { + // One fresh ext3 LDE output per column on device. + if !admit(lde_size, ext3_bytes(lde_size as u64, columns.len() as u64)).is_admitted() { return LayoutDispatch::Skip; } if TypeId::of::() != TypeId::of::() { @@ -624,17 +1046,24 @@ where return None; } let n = h0.len(); - let blowup = 2; // extend_half_to_lde extends N → 2N always - let lde_size = n * blowup; - if lde_size < gpu_lde_threshold() { + // The halves extend N → the full LDE: blowup 2 when `H` was evaluated on + // the whole coset, the domain's blowup when on its 2N-point sub-coset. + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + if n == 0 || !lde_size.is_multiple_of(n) { return None; } + let blowup = lde_size / n; if TypeId::of::() != TypeId::of::() { return None; } if TypeId::of::() != TypeId::of::() { return None; } + // Two ext3 inputs staged and two ext3 LDE outputs allocated on device. + let bytes = ext3_bytes(lde_size as u64, 2).saturating_add(ext3_bytes(n as u64, 2)); + if !admit_transient(lde_size, bytes, "R2 extend-halves") { + return None; + } GPU_EXTEND_HALVES_CALLS.fetch_add(1, Ordering::Relaxed); // Weights are built from `g = domain.coset_offset` directly: the // CPU caller previously passed `g²` redundantly. See the @@ -693,10 +1122,11 @@ where } /// Shared admission gate for the device composition-parts producers: the tower -/// must be the Goldilocks/ext3 pair the kernels are written for, and the LDE must -/// be a power of two at or above the commit threshold. Returns the validated LDE -/// size so callers can derive from it. Kept in one place so a future condition -/// (a VRAM check, a tower widening) cannot land on only one of the d=1/d=2 arms. +/// must be the Goldilocks/ext3 pair the kernels are written for, the LDE must +/// be a power of two, and the two ext3 part slabs the decompose allocates must +/// clear [`admit`]. Returns the validated LDE size so callers can derive from +/// it. Kept in one place so a future condition (a tower widening) cannot land +/// on only one of the d=1/d=2 arms. fn dev_comp_parts_gate(num_rows: usize) -> Option where F: IsField + 'static, @@ -708,7 +1138,10 @@ where if TypeId::of::() != TypeId::of::() { return None; } - if num_rows < gpu_lde_threshold() || !num_rows.is_power_of_two() { + if !num_rows.is_power_of_two() { + return None; + } + if !admit_transient(num_rows, ext3_bytes(num_rows as u64, 2), "R2 decompose") { return None; } Some(num_rows) @@ -723,21 +1156,31 @@ where /// the evaluations are also drained to host for the fallback consumers; /// without it (device-only) the returned part Vecs are empty placeholders. /// `None` → the caller downloads `H` and runs the host decompose path. +/// +/// `row_stride` is the sub-coset stride `H` was evaluated at (`h.num_rows` +/// = `lde_size / row_stride`): the halves hold `h.num_rows / 2` points each and +/// are extended to `lde_size` with blowup `2 * row_stride`; `weights` must be +/// that extension's `g⁻ʲ/n`. Stride 1 is the whole-coset path (blowup 2). pub(crate) fn try_decompose_extend_d2_dev( h: &math_cuda::constraint_interp::GpuCompH, inv_2x: &std::sync::Arc>>, weights: &[FieldElement], want_host: bool, + row_stride: usize, ) -> Option<(Vec>>, math_cuda::lde::GpuLdeExt3)> where F: IsField + 'static, E: IsField + 'static, { - let lde_size = dev_comp_parts_gate::(h.num_rows)?; - let n = lde_size / 2; - if weights.len() != n || inv_2x.len() < n { + if row_stride == 0 { + return None; + } + let lde_size = dev_comp_parts_gate::(h.num_rows.checked_mul(row_stride)?)?; + let n = h.num_rows / 2; + if n == 0 || weights.len() != n || inv_2x.len() <= (n - 1) * row_stride { return None; } + let blowup = lde_size / n; // SAFETY: `F == GoldilocksField` (gated above); the Arc'd Vecs share layout. let inv_conc: &crate::constraint_ir::gpu_interp::GoldilocksBZInv = @@ -748,8 +1191,14 @@ where // SAFETY: F == Goldilocks; FieldElement is repr(transparent) over u64. let two_inv: u64 = unsafe { *(two_inv_fe.value() as *const _ as *const u64) }; - let (slabs, stream, n_dev) = - math_cuda::constraint_interp::decompose_d2_into_slabs(h, &inv_handle, two_inv).ok()?; + let (slabs, stream, n_dev) = math_cuda::constraint_interp::decompose_d2_into_slabs_strided( + h, + &inv_handle, + two_inv, + row_stride, + lde_size, + ) + .ok()?; debug_assert_eq!(n_dev, n); GPU_EXTEND_HALVES_CALLS.fetch_add(1, Ordering::Relaxed); @@ -765,7 +1214,7 @@ where slabs, 2, n, - 2, + blowup, weights_u64, None, ) @@ -787,7 +1236,7 @@ where slabs, 2, n, - 2, + blowup, weights_u64, Some(&mut outputs), ) @@ -850,7 +1299,11 @@ where pub(crate) fn download_comp_h_to_field( h: &math_cuda::constraint_interp::GpuCompH, ) -> Option>> { - let raw = math_cuda::constraint_interp::download_comp_h(h).ok()?; + let raw = if crate::prover::warm_prove_constants_enabled() { + math_cuda::constraint_interp::download_comp_h_staged(h).ok()? + } else { + math_cuda::constraint_interp::download_comp_h(h).ok()? + }; crate::constraint_ir::gpu_interp::ext3_u64_to_field::(&raw) } @@ -863,7 +1316,112 @@ pub fn gpu_leaf_hash_calls() -> u64 { /// Merkle → single D2H. Keeps the Merkle tree resident on device (in the /// handle's `.tree`); the returned host `MerkleTree` is root only, so query /// openings gather paths from the device tree via [`gather_proofs_dev`]. +#[allow(clippy::too_many_arguments)] +/// Commit ONE preprocessed column group on the device: coset LDE, leaf hash and +/// Merkle tree in a single fused call, returning only the root. +/// +/// # Why this exists +/// +/// `prover::lfm::registry::build_artifacts_with_hasher` commits every program's +/// instruction column groups on the HOST — interpolate, coset-expand, Merkle — +/// and lane P measured that at 12.6 s per epoch wrap and 22.0 s per interior +/// node, the largest phase of the recursion pipeline, while the card sat at +/// 1 MiB and 0%. The identical operation on the main trace already runs on the +/// device inside `r1_main_commit`. This is the entry point that lets the +/// artifact build reach it, and it is a `pub` wrapper rather than a visibility +/// change because the caller lives in another crate and should see one named +/// function, not the dispatch layer's internals. +/// +/// # What it returns, and what it does NOT +/// +/// The ROOT, and nothing else. The Merkle tree stays resident on the device +/// inside the handle, which is dropped here — the artifact build wants the root +/// (it goes into `lfm_program_id` and the AIR's declared commitment) and has no +/// use for the nodes. +/// +/// ⚠ That means this does NOT feed the process-wide precomputed-tree cache, and +/// deliberately so: `precomputed_tree_cache_put` takes a FULL host tree because +/// `multi_prove` opens against it, and a root-only tree in that cache would be a +/// tree that cannot answer a query. `multi_prove` builds and caches its own +/// precomputed tree on the device already (`try_expand_split_trees_row_major_keep` +/// with `build_precomputed`), so nothing here is lost by staying out of it. +/// +/// # The roots must match the host's, and a production invariant already says +/// they do +/// +/// The host commits with `commit_bit_reversed_with::<_, H::Batched>(evals, +/// ROWS_PER_LEAF)`. The device builds its leaves from its LDE (column-major under +/// the default engine, row-major on the legacy path), over the same bit-reversed +/// rows. These are already required to agree: `multi_prove` rebuilds the +/// precomputed tree on the device from the row-major main trace and REFUSES the +/// proof when its root differs from the one the AIR declares +/// (`ProvingError::PrecomputedCommitmentMismatch`), +/// and for an LFM proof the declared root is exactly what the host artifact build +/// produced. Every GPU recursion proof passes that check today. The six +/// `registry_drift_*` tests are the gate that keeps it true for this caller. +/// +/// `None` means the device declined — admission said no, the field is not +/// Goldilocks, or the shape is degenerate — and the caller must commit on the +/// host. That is the same contract every other entry point in this module has. +pub fn try_commit_row_major( + table: &str, + row_major: &[FieldElement], + rows: usize, + cols: usize, + blowup_factor: usize, + coset_offset: &FieldElement, +) -> Option +where + F: IsFFTField + 'static, + B: DeviceTreeBackend, +{ + try_commit_row_major_with::(table, row_major, rows, cols, blowup_factor, coset_offset, 2) +} + +/// [`try_commit_row_major`] with `rows_per_leaf` rows per Merkle leaf: 2 is +/// today's row pair, 1 the S2 one-row root (the host twin is +/// `commit_bit_reversed_with(.., rows_per_leaf)`). +pub fn try_commit_row_major_with( + table: &str, + row_major: &[FieldElement], + rows: usize, + cols: usize, + blowup_factor: usize, + coset_offset: &FieldElement, + rows_per_leaf: usize, +) -> Option +where + F: IsFFTField + 'static, + B: DeviceTreeBackend, +{ + if rows == 0 || cols == 0 || row_major.len() != rows * cols { + return None; + } + // ⛔ The weights come from the prover's own derivation, not a second copy — + // they carry the iFFT normalization, so a divergence here is a divergence in + // the roots. + let weights = crate::prover::coset_weights::(rows, coset_offset); + let (tree, _handle, _lde) = try_expand_leaf_and_tree_row_major_keep::( + table, + "LFM artifact prep commit", + row_major, + None, + rows, + cols, + blowup_factor, + &weights, + // The artifact build never reads the evaluations — only the root — so + // the row-major D2H is skipped entirely. + false, + rows_per_leaf, + )?; + Some(tree.root) +} + +#[allow(clippy::too_many_arguments)] pub(crate) fn try_expand_leaf_and_tree_row_major_keep( + table: &str, + what: &'static str, row_major: &[FieldElement], predev: Option<&math_cuda::CudaSlice>, n: usize, @@ -871,6 +1429,7 @@ pub(crate) fn try_expand_leaf_and_tree_row_major_keep( blowup_factor: usize, weights: &[FieldElement], retain_host_lde: bool, + rows_per_leaf: usize, ) -> Option<( MerkleTree, math_cuda::lde::GpuLdeBase, @@ -879,12 +1438,8 @@ pub(crate) fn try_expand_leaf_and_tree_row_major_keep( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: DeviceTreeBackend, { - let lde_size = n.saturating_mul(blowup_factor); - if lde_size < gpu_lde_threshold() { - return None; - } if TypeId::of::() != TypeId::of::() { return None; } @@ -894,6 +1449,20 @@ where if row_major.len() != n * m || m == 0 || n == 0 { return None; } + let lde_size = n.saturating_mul(blowup_factor); + // ⛔ `what` is a PARAMETER because this call now has two callers with two + // different stories. An abort diagnostic naming "R1 main commit" for an + // artifact-prep commit would send the reader to the prove, which is not + // where the failure is. + let shape = DispatchShape { + table, + what, + n, + base_cols: m, + blowup: blowup_factor, + }; + let set = commit_device_set_rpl(n, m, blowup_factor, true, rows_per_leaf); + admit_commit(lde_size, &shape, &set)?; let raw: &[u64] = unsafe { from_raw_parts(row_major.as_ptr() as *const u64, n * m) }; let weights_u64 = unsafe { weights_to_u64::(weights) }; @@ -901,19 +1470,32 @@ where GPU_LDE_CALLS.fetch_add(m as u64, Ordering::Relaxed); GPU_LEAF_HASH_CALLS.fetch_add(1, Ordering::Relaxed); GPU_MERKLE_TREE_CALLS.fetch_add(1, Ordering::Relaxed); + note_one_row_trees(rows_per_leaf, 1, set.tree_bytes); // The keep path keeps the Merkle tree resident on device (in `handle.tree`). // `retain_host_lde=false` additionally skips the row-major D2H (device-only). - let (handle, lde_u64) = math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep( + // Admitted means the device path is the only path: a failure here aborts. + let (handle, lde_u64) = match math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep_rpl( raw, predev, + device_hash_of::(), n, m, blowup_factor, &weights_u64, retain_host_lde, - ) - .ok()?; + rows_per_leaf, + ) { + Ok(v) => v, + Err(e) => { + abort_or_test_fallback( + &shape, + Some(&set), + DevicePathFailure::DeviceError(format!("{e:?}")), + ); + return None; + } + }; // Transmute Vec → Vec> (zero-copy, E == GoldilocksField). let lde_out: Vec> = unsafe { @@ -932,25 +1514,6 @@ where Some((tree, handle, lde_out)) } -/// Convert a GPU-built full node buffer (`(2*leaves - 1) * 32` bytes, inner -/// nodes first, root at offset 0, leaves at the tail) into a host -/// [`MerkleTree`], the exact layout `from_precomputed_nodes` expects. -fn tree_from_node_bytes(nodes: Vec) -> Option> -where - B: IsMerkleTreeBackend, -{ - debug_assert_eq!(nodes.len() % 32, 0); - let nodes: Vec<[u8; 32]> = nodes - .chunks_exact(32) - .map(|c| { - let mut n = [0u8; 32]; - n.copy_from_slice(c); - n - }) - .collect(); - MerkleTree::::from_precomputed_nodes(nodes) -} - /// Preprocessed-table variant of [`try_expand_leaf_and_tree_row_major_keep`]: /// one row-major GPU LDE of ALL columns plus TWO subset Merkle trees — the /// precomputed columns `[0, split_col)` and the multiplicity columns @@ -968,6 +1531,7 @@ where #[allow(clippy::type_complexity)] #[allow(clippy::too_many_arguments)] pub(crate) fn try_expand_split_trees_row_major_keep( + table: &str, row_major: &[FieldElement], predev: Option<&math_cuda::CudaSlice>, n: usize, @@ -977,6 +1541,7 @@ pub(crate) fn try_expand_split_trees_row_major_keep( split_col: usize, build_precomputed: bool, want_host: bool, + rows_per_leaf: usize, ) -> Option<( Option>, MerkleTree, @@ -986,12 +1551,8 @@ pub(crate) fn try_expand_split_trees_row_major_keep( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: DeviceTreeBackend, { - let lde_size = n.saturating_mul(blowup_factor); - if lde_size < gpu_lde_threshold() { - return None; - } if TypeId::of::() != TypeId::of::() { return None; } @@ -1004,6 +1565,16 @@ where if split_col == 0 || split_col >= m { return None; } + let lde_size = n.saturating_mul(blowup_factor); + let shape = DispatchShape { + table, + what: "R1 main commit (preprocessed split)", + n, + base_cols: m, + blowup: blowup_factor, + }; + let set = commit_device_set_rpl(n, m, blowup_factor, true, rows_per_leaf); + admit_commit(lde_size, &shape, &set)?; let raw: &[u64] = unsafe { from_raw_parts(row_major.as_ptr() as *const u64, n * m) }; let weights_u64 = unsafe { weights_to_u64::(weights) }; @@ -1011,10 +1582,13 @@ where GPU_LDE_CALLS.fetch_add(m as u64, Ordering::Relaxed); GPU_LEAF_HASH_CALLS.fetch_add(1 + build_precomputed as u64, Ordering::Relaxed); GPU_MERKLE_TREE_CALLS.fetch_add(1 + build_precomputed as u64, Ordering::Relaxed); + note_one_row_trees(rows_per_leaf, 1 + build_precomputed as u64, set.tree_bytes); - let (pre_nodes, handle, lde_u64) = math_cuda::lde::coset_lde_row_major_split_trees( + // Admitted means the device path is the only path: a failure here aborts. + let (pre_nodes, handle, lde_u64) = match math_cuda::lde::coset_lde_row_major_split_trees_rpl( raw, predev, + device_hash_of::(), n, m, blowup_factor, @@ -1022,11 +1596,24 @@ where split_col, build_precomputed, want_host, - ) - .ok()?; + rows_per_leaf, + ) { + Ok(v) => v, + Err(e) => { + abort_or_test_fallback( + &shape, + Some(&set), + DevicePathFailure::DeviceError(format!("{e:?}")), + ); + return None; + } + }; let pre_tree = match pre_nodes { - Some(nodes) => Some(tree_from_node_bytes::(nodes)?), + // A GPU-built full node buffer (`2*leaves - 1` nodes, inner nodes + // first, root at index 0, leaves at the tail): the exact layout + // `from_precomputed_nodes` expects. + Some(nodes) => Some(MerkleTree::::from_precomputed_nodes(nodes)?), None => None, }; // Mult tree resident in the handle: the host tree is root only and R4 @@ -1055,13 +1642,16 @@ where /// Row-major ext3 GPU path: single H2D → row-major NTT (m*3 base-field cols) → /// row-major Keccak → Merkle → single D2H → transpose to GpuLdeExt3 handle. /// Same optimization as the base-field path: no extract_columns, no CPU transpose. +#[allow(clippy::too_many_arguments)] pub(crate) fn try_expand_leaf_and_tree_ext3_row_major_keep( + table: &str, row_major: &[FieldElement], n: usize, m: usize, blowup_factor: usize, weights: &[FieldElement], retain_host_lde: bool, + rows_per_leaf: usize, ) -> Option<( MerkleTree, math_cuda::lde::GpuLdeExt3, @@ -1070,12 +1660,8 @@ pub(crate) fn try_expand_leaf_and_tree_ext3_row_major_keep( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: DeviceTreeBackend, { - let lde_size = n.saturating_mul(blowup_factor); - if lde_size < gpu_lde_threshold() { - return None; - } if TypeId::of::() != TypeId::of::() { return None; } @@ -1085,27 +1671,50 @@ where if row_major.len() != n * m || m == 0 || n == 0 { return None; } - // Fp3 = [u64; 3] in memory — reinterpret as flat u64 slice (m3 = m*3). let m3 = m * 3; + let lde_size = n.saturating_mul(blowup_factor); + let shape = DispatchShape { + table, + what: "R1 aux commit", + n, + base_cols: m3, + blowup: blowup_factor, + }; + let set = commit_device_set_rpl(n, m3, blowup_factor, false, rows_per_leaf); + admit_commit(lde_size, &shape, &set)?; + let raw: &[u64] = unsafe { from_raw_parts(row_major.as_ptr() as *const u64, n * m3) }; let weights_u64 = unsafe { weights_to_u64::(weights) }; GPU_LDE_CALLS.fetch_add((m * 3) as u64, Ordering::Relaxed); GPU_LEAF_HASH_CALLS.fetch_add(1, Ordering::Relaxed); GPU_MERKLE_TREE_CALLS.fetch_add(1, Ordering::Relaxed); + note_one_row_trees(rows_per_leaf, 1, set.tree_bytes); // The keep path keeps the Merkle tree resident on device (in `handle.tree`). // `retain_host_lde=false` additionally skips the row-major D2H (device-only). - let (handle, lde_u64) = math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep( + // Admitted means the device path is the only path: a failure here aborts. + let (handle, lde_u64) = match math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep_rpl( raw, + device_hash_of::(), n, m, blowup_factor, &weights_u64, retain_host_lde, - ) - .ok()?; + rows_per_leaf, + ) { + Ok(v) => v, + Err(e) => { + abort_or_test_fallback( + &shape, + Some(&set), + DevicePathFailure::DeviceError(format!("{e:?}")), + ); + return None; + } + }; // Transmute Vec → Vec> (zero-copy, E == Fp3 = [u64;3]). let lde_out: Vec> = unsafe { @@ -1182,6 +1791,41 @@ pub fn gpu_merkle_tree_calls() -> u64 { GPU_MERKLE_TREE_CALLS.load(Ordering::Relaxed) } +/// S2 one-row trees (`rows_per_leaf = 1`) the device built: trace trees (main, +/// the preprocessed split's subsets, aux plain and resident), composition +/// trees, and FRI input trees over the resident DEEP codeword. A one-row table +/// that fell back to the host leaves this unmoved, so the device-parity tests +/// assert on it (a fallback is a FAIL, never a pass). +static GPU_ONE_ROW_TREES: AtomicU64 = AtomicU64::new(0); +pub fn gpu_one_row_trees() -> u64 { + GPU_ONE_ROW_TREES.load(Ordering::Relaxed) +} + +/// The largest single one-row tree's node buffer the device was asked for, +/// in bytes (the admission's own term, `(2 · lde − 1) · 32`) — what a +/// one-row table adds over its row-pair twin, per tree, for the 0-fallback +/// gate. +static GPU_ONE_ROW_TREE_PEAK_BYTES: AtomicU64 = AtomicU64::new(0); +pub fn gpu_one_row_tree_peak_bytes() -> u64 { + GPU_ONE_ROW_TREE_PEAK_BYTES.load(Ordering::Relaxed) +} + +/// Device FRI commits under a one-row layout (S2): the input tree committed +/// from the DEEP codeword on device, then the group chain. +static GPU_ONE_ROW_FRI_CALLS: AtomicU64 = AtomicU64::new(0); +pub fn gpu_one_row_fri_calls() -> u64 { + GPU_ONE_ROW_FRI_CALLS.load(Ordering::Relaxed) +} + +/// Count `trees` one-row trees of `tree_bytes` node bytes each (no-op for +/// row pairs). +fn note_one_row_trees(rows_per_leaf: usize, trees: u64, tree_bytes: u64) { + if rows_per_leaf == 1 { + GPU_ONE_ROW_TREES.fetch_add(trees, Ordering::Relaxed); + GPU_ONE_ROW_TREE_PEAK_BYTES.fetch_max(tree_bytes, Ordering::Relaxed); + } +} + // ============================================================================ // PR-3: R2 composition-parts LDE + Merkle commit + R3 OOD barycentric // ============================================================================ @@ -1303,10 +1947,11 @@ where /// recomputes on CPU. pub(crate) fn try_build_comp_poly_tree_gpu( lde_parts: &[Vec>], + rows_per_leaf: usize, ) -> Option<(MerkleTree, math_cuda::lde::GpuMerkleTree)> where E: IsField + 'static, - B: IsMerkleTreeBackend, + B: DeviceTreeBackend, { if lde_parts.is_empty() { return None; @@ -1315,10 +1960,14 @@ where if !lde_size.is_power_of_two() || lde_size < 2 { return None; } - if lde_size < gpu_lde_threshold() { + if TypeId::of::() != TypeId::of::() { return None; } - if TypeId::of::() != TypeId::of::() { + // The parts are re-uploaded (`m` ext3 columns over the LDE) and one full + // tree (`rows_per_leaf` rows per leaf) is built. + let tree_bytes = tree_bytes_for(lde_size as u64, rows_per_leaf as u64); + let bytes = ext3_bytes(lde_size as u64, lde_parts.len() as u64).saturating_add(tree_bytes); + if !admit_transient(lde_size, bytes, "R2 composition tree") { return None; } // All parts must have the same LDE length. @@ -1339,12 +1988,35 @@ where // Keep the composition tree resident on device, so the whole tree copy to // host is eliminated. R4 composition openings gather paths from the device // tree (`gather_proofs_dev`); the returned host tree is root only. - let dev_tree = match math_cuda::merkle::build_comp_poly_tree_from_evals_ext3_keep(&raw_parts) { + let dev_tree = match match device_hash_of::() { + math_cuda::DeviceHash::Keccak256 => { + math_cuda::merkle::build_comp_poly_tree_from_evals_ext3_keep_rpl( + &raw_parts, + rows_per_leaf, + ) + } + math_cuda::DeviceHash::Blake3 => { + math_cuda::blake3::build_comp_poly_tree_from_evals_ext3_keep_rpl( + &raw_parts, + rows_per_leaf, + ) + } + math_cuda::DeviceHash::Rpx256 => { + math_cuda::rpx::build_comp_poly_tree_from_evals_ext3_keep_rpl(&raw_parts, rows_per_leaf) + } + math_cuda::DeviceHash::Poseidon1 => { + math_cuda::p1_stark::build_comp_poly_tree_from_evals_ext3_keep_rpl( + &raw_parts, + rows_per_leaf, + ) + } + } { Ok(t) => t, Err(_) => return None, }; - debug_assert_eq!(dev_tree.leaves_len, lde_size / 2); + debug_assert_eq!(dev_tree.leaves_len, lde_size / rows_per_leaf); GPU_COMP_POLY_TREE_CALLS.fetch_add(1, Ordering::Relaxed); + note_one_row_trees(rows_per_leaf, 1, tree_bytes); let host = MerkleTree::::from_root(dev_tree.root); Some((host, dev_tree)) } @@ -1354,29 +2026,69 @@ where /// host pack + H2D re-upload of data that is already on device. pub(crate) fn try_build_comp_poly_tree_gpu_from_dev( handle: &math_cuda::lde::GpuLdeExt3, + rows_per_leaf: usize, ) -> Option<(MerkleTree, math_cuda::lde::GpuMerkleTree)> where E: IsField + 'static, - B: IsMerkleTreeBackend, + B: DeviceTreeBackend, { if TypeId::of::() != TypeId::of::() { return None; } - if handle.m == 0 || !handle.lde_size.is_power_of_two() || handle.lde_size < gpu_lde_threshold() - { + if handle.m == 0 || !handle.lde_size.is_power_of_two() { + return None; + } + // Only the tree is fresh: the parts are already resident. + let tree_bytes = tree_bytes_for(handle.lde_size as u64, rows_per_leaf as u64); + if !admit_transient( + handle.lde_size, + tree_bytes, + "R2 composition tree (resident parts)", + ) { return None; } let be = math_cuda::device::backend().ok()?; let stream = be.next_stream(); handle.wait_ready_on(&stream).ok()?; - let dev_tree = math_cuda::merkle::build_comp_poly_tree_from_slabs_dev( - &stream, - handle.buf.as_ref(), - handle.m, - handle.lde_size, - ) + let dev_tree = match device_hash_of::() { + math_cuda::DeviceHash::Keccak256 => { + math_cuda::merkle::build_comp_poly_tree_from_slabs_dev_rpl( + &stream, + handle.buf.as_ref(), + handle.m, + handle.lde_size, + rows_per_leaf, + ) + } + math_cuda::DeviceHash::Blake3 => { + math_cuda::blake3::build_comp_poly_tree_from_slabs_dev_rpl( + &stream, + handle.buf.as_ref(), + handle.m, + handle.lde_size, + rows_per_leaf, + ) + } + math_cuda::DeviceHash::Rpx256 => math_cuda::rpx::build_comp_poly_tree_from_slabs_dev_rpl( + &stream, + handle.buf.as_ref(), + handle.m, + handle.lde_size, + rows_per_leaf, + ), + math_cuda::DeviceHash::Poseidon1 => { + math_cuda::p1_stark::build_comp_poly_tree_from_slabs_dev_rpl( + &stream, + handle.buf.as_ref(), + handle.m, + handle.lde_size, + rows_per_leaf, + ) + } + } .ok()?; GPU_COMP_POLY_TREE_CALLS.fetch_add(1, Ordering::Relaxed); + note_one_row_trees(rows_per_leaf, 1, tree_bytes); let host = MerkleTree::::from_root(dev_tree.root); Some((host, dev_tree)) } @@ -1488,6 +2200,38 @@ pub(crate) fn try_barycentric_base_on_handle_multi( z_pows: &[FieldElement], ctx: &R3DevContext, ) -> Option>>> +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + if !is_goldilocks_ext3_tower::() { + return None; + } + barycentric_base_multi_on_main::( + lde_trace.gpu_main()?, + row_stride, + coset_points_len, + coset_offset_pow_n, + n_inv, + g_n_inv, + z_pows, + ctx, + ) +} + +/// [`try_barycentric_base_on_handle_multi`] over a main LDE handle itself, +/// before any [`LDETraceTable`] holds it (the commitment-time answers). +#[allow(clippy::too_many_arguments)] +fn barycentric_base_multi_on_main( + main: &math_cuda::lde::GpuLdeBase, + row_stride: usize, + coset_points_len: usize, + coset_offset_pow_n: &FieldElement, + n_inv: &FieldElement, + g_n_inv: &FieldElement, + z_pows: &[FieldElement], + ctx: &R3DevContext, +) -> Option>>> where F: IsField + IsSubFieldOf + 'static, E: IsField + 'static, @@ -1499,7 +2243,6 @@ where if k_points == 0 || k_points > math_cuda::barycentric::BARY_MAX_EVAL_POINTS { return None; } - let main = lde_trace.gpu_main()?; let num_cols = main.m; if num_cols == 0 { return Some(vec![Vec::new(); k_points]); @@ -1543,6 +2286,42 @@ where ) } +/// Every main column of a table's resident LDE at `point`, on the card: the +/// R3 kernel over the trace-size coset subsample for one point. The +/// commitment-time answers' card arm (`commit_ood`). `None` when the shape +/// misses the kernel's thresholds or a call errs: the caller evaluates on the +/// host. +pub(crate) fn try_main_values_at_gpu( + main: &math_cuda::lde::GpuLdeBase, + domain: &crate::domain::Domain, + point: &FieldElement, +) -> Option>> +where + F: IsFFTField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let dc = domain.ood_constants(); + let n = domain.interpolation_domain_size; + let ctx = try_prep_r3_dev_context::(&dc.points, std::slice::from_ref(point), None)?; + let values = barycentric_base_multi_on_main::( + main, + domain.blowup_factor, + n, + &dc.offset_pow_n, + &dc.size_inv, + &dc.offset_pow_n_inv, + &[point.pow(n)], + &ctx, + )? + .pop()?; + COMMIT_OOD_GPU_CALLS.fetch_add(1, Ordering::Relaxed); + Some(values) +} + +/// Tables whose commitment-time answers came off the card +/// ([`try_main_values_at_gpu`]): the device test's proof that the card arm ran. +pub static COMMIT_OOD_GPU_CALLS: AtomicU64 = AtomicU64::new(0); + /// Aux (ext3) counterpart of [`try_barycentric_base_on_handle_multi`]. #[allow(clippy::too_many_arguments)] pub(crate) fn try_barycentric_ext3_on_handle_multi( @@ -1729,6 +2508,15 @@ pub fn gpu_deep_calls() -> u64 { GPU_DEEP_CALLS.load(Ordering::Relaxed) } +/// The share of [`gpu_deep_calls`] that took the fused kernel, which inverts +/// each row's denominators itself ([`try_deep_composition_gpu_fused_keep`]). +/// Zero under `LAMBDA_VM_DEEP_INV_LEGACY=1`; at the default a resident table +/// that stays at zero fell back to the buffered kernel and its inverse buffer. +pub(crate) static GPU_FUSED_DEEP_CALLS: AtomicU64 = AtomicU64::new(0); +pub fn gpu_fused_deep_calls() -> u64 { + GPU_FUSED_DEEP_CALLS.load(Ordering::Relaxed) +} + /// FRI commit-phase dispatch counter (one per successful commit, not per /// layer). Counts BOTH entry points, so a table whose device-resident attempt /// ([`try_fri_commit_gpu_from_dev`]) fails and then commits from host evals @@ -1753,28 +2541,25 @@ pub(crate) static GPU_BATCH_INVERT_CALLS: AtomicU64 = AtomicU64::new(0); /// ([`materialize_lde_trace_host`], the sole function that bumps this — /// entered from the R2 host evaluator, the R3 barycentric arms and the R4 /// DEEP host loop). Nonzero means the device-only gate cleared a table whose -/// downstream dispatch then declined at runtime — the table continued -/// host-backed, correct but slower. A count is either a gate miss (a static -/// condition worth mirroring into the gate) or a transient device decline -/// (VRAM pressure), which by definition cannot be gated out — see -/// [`materialize_lde_trace_host`]'s own note. The R1 resident-aux downgrade -/// is counted by [`GPU_RESIDENT_AUX_DOWNGRADES`] instead: it fires on tables -/// the gate never marked device-only, so summing the two would blame the gate -/// for declines it never made. +/// downstream dispatch then declined at runtime. Only reachable under the +/// test-only host fallback ([`test_only_host_fallback`]): in production the +/// recovery aborts instead. A count is either a gate miss (a static condition +/// worth mirroring into the gate) or a transient device decline (VRAM +/// pressure), which by definition cannot be gated out. The R1 resident-aux +/// downgrade is counted by [`GPU_RESIDENT_AUX_DOWNGRADES`] instead: it fires +/// on tables the gate never marked device-only, so summing the two would +/// blame the gate for declines it never made. pub(crate) static GPU_DEVICE_ONLY_DOWNGRADES: AtomicU64 = AtomicU64::new(0); pub fn gpu_device_only_downgrades() -> u64 { GPU_DEVICE_ONLY_DOWNGRADES.load(Ordering::Relaxed) } -/// R1 downgrades, and only those: times the resident aux trace was downloaded -/// so the aux commit could continue on the host arms, after the device aux LDE -/// declined and the drain-and-retry either did not run or declined again -/// ([`materialize_aux_trace_host`], the sole site that bumps this). Independent -/// of the device-only gate — the site is entered whenever `aux_resident()` is -/// set, whatever the gate said — so a table that was never device-only can land -/// here, and a nonzero value points at sustained VRAM pressure rather than a -/// gate miss. Read it against [`GPU_RESIDENT_AUX_RETRIES`]: retries alone mean -/// the drain absorbed the pressure, retries plus downgrades mean it did not. +/// R1 resident-aux downgrades. Retired: a resident aux LDE that declines after +/// the drain-and-retry is now a clean `ProvingError::DevicePath` in the prover +/// (host RAM is a cache, not a compute path), so nothing bumps this and it +/// reads zero. The accessor stays for the integration suite's assertion that +/// it IS zero. Read [`GPU_RESIDENT_AUX_RETRIES`] for the pressure signal: +/// retries mean the drain absorbed a transient decline. pub(crate) static GPU_RESIDENT_AUX_DOWNGRADES: AtomicU64 = AtomicU64::new(0); pub fn gpu_resident_aux_downgrades() -> u64 { GPU_RESIDENT_AUX_DOWNGRADES.load(Ordering::Relaxed) @@ -1806,15 +2591,20 @@ pub fn gpu_resident_aux_retries() -> u64 { /// Recover a device-only table for the host path: download the resident main /// and aux LDEs from their device handles into the host buffers and clear the /// device-only flag. A side whose host buffer is already populated (a mixed -/// state: one commit fell back to CPU while the other stayed device-only) is -/// kept as is — only the missing side is downloaded. The class-level safety -/// net under the device-only gate — a static predicate can never mirror every -/// reason a dynamic dispatch might decline (kernel eligibility, transient -/// errors, shapes a new workload brings), so any miss lands here and degrades -/// to a slower-but-correct CPU round instead of a hard abort. Returns false -/// (→ the caller's abort) when the resident handles cannot serve the data: a -/// missing handle or bound stream, a handle whose shape disagrees with the -/// trace, a failed download or sync, or a field tower with no CUDA lowering. +/// state: one side's dispatch declined below the device-only envelope while +/// the other stayed device-only) is kept as is — only the missing side is +/// downloaded. +/// +/// In production this recovery does not run: host RAM is a cache, not a +/// compute path, so a device-only table whose downstream dispatch declined is +/// a failure to report, and [`refuse_host_recovery`] aborts here with the +/// shape and the live VRAM. Under the test-only switch +/// ([`test_only_host_fallback`]) the download proceeds and the table continues +/// on the host arms — the fault-injection suites assert on exactly that. +/// Returns false (→ the caller's abort) when the resident handles cannot serve +/// the data: a missing handle or bound stream, a handle whose shape disagrees +/// with the trace, a failed download or sync, or a field tower with no CUDA +/// lowering. pub(crate) fn materialize_lde_trace_host( lde_trace: &mut crate::trace::LDETraceTable, ) -> bool @@ -1825,6 +2615,12 @@ where if !lde_trace.host_trace_empty() { return true; } + refuse_host_recovery( + "R2/R3/R4 host arm on a device-only trace", + lde_trace.num_rows(), + lde_trace.num_main_cols(), + lde_trace.num_aux_cols(), + ); if !is_goldilocks_ext3_tower::() { return false; } @@ -1872,7 +2668,7 @@ where } let (m, lde) = (h.m, h.lde_size); // Short download: degrade like the sibling paths - // (`download_main_lde_row_major`, `materialize_aux_trace_host`) + // (`download_main_lde_row_major`, `download_composition_parts_host`) // rather than panic on the slab slicing below. if slabs.len() != m * lde * 3 { return false; @@ -1981,49 +2777,6 @@ where }) } -/// R1 counterpart of [`materialize_lde_trace_host`]: download the resident -/// aux trace (already row-major ext3, matching the host layout) into the -/// trace's aux table, so the aux commit continues on the host arms when the -/// device aux LDE declines at runtime. -pub(crate) fn materialize_aux_trace_host(trace: &mut crate::trace::TraceTable) -> bool -where - F: IsField + IsSubFieldOf + 'static, - E: IsField + 'static, -{ - if !is_goldilocks_ext3_tower::() { - return false; - } - let (buf, rows, cols) = match trace.aux_resident.as_ref() { - Some(ra) => (ra.buf.clone(), ra.num_rows, ra.num_aux_cols), - None => return false, - }; - let Ok(be) = math_cuda::device::backend() else { - return false; - }; - let stream = be.next_stream(); - let Ok(raw) = stream.clone_dtoh(buf.as_ref()) else { - return false; - }; - if stream.synchronize().is_err() || raw.len() != rows * cols * 3 { - return false; - } - let data = u64_to_ext3_vec::(&raw); - trace.aux_table = crate::table::Table::new(data, cols); - trace.num_aux_columns = cols; - // The declined device LDE attempt can leave kernels enqueued on another - // stream still reading this buffer; its owning stream is long idle, so - // dropping here would complete the stream-ordered free immediately and - // the pool could hand the memory to a concurrent table's allocation - // while those kernels run. Drain the device before the drop — this is a - // rare recovery path. - if be.ctx.synchronize().is_err() { - return false; - } - trace.aux_resident = None; - GPU_RESIDENT_AUX_DOWNGRADES.fetch_add(1, Ordering::Relaxed); - true -} - /// Diagnostic: download a resident ext3 handle (3-slab layout) as per-column /// host Vecs. Used by the xcheck post-mortem to compare the committed R2 /// parts against a host recompute. @@ -2157,6 +2910,12 @@ where let Some(h) = lde_trace.gpu_composition_parts() else { return false; }; + refuse_host_recovery( + "host consumer of the composition parts on a device-only table", + h.lde_size, + lde_trace.num_main_cols(), + lde_trace.num_aux_cols(), + ); let Some(stream) = lde_trace.bound_stream() else { return false; }; @@ -2276,9 +3035,6 @@ where return None; } let lde_size = domain_size.checked_mul(blowup_factor)?; - if lde_size < gpu_lde_threshold() { - return None; - } if TypeId::of::() != TypeId::of::() { return None; } @@ -2286,6 +3042,12 @@ where return None; } let m = parts_coefs.len(); + // `m` ext3 coefficient inputs staged and `m` ext3 LDE parts kept resident. + let bytes = ext3_bytes(lde_size as u64, m as u64) + .saturating_add(ext3_bytes(domain_size as u64, m as u64)); + if !admit_transient(lde_size, bytes, "R2 parts LDE") { + return None; + } let mut weights_u64 = Vec::with_capacity(domain_size); let mut w = FieldElement::::one(); @@ -2354,10 +3116,12 @@ unsafe fn ext3_slice_to_u64(col: &[FieldElement]) -> &[u64] { /// The resident buffer is only borrowed: the device-input LDE copies it /// device-to-device into its own scratch, so `ra` stays valid afterwards. pub(crate) fn try_expand_leaf_and_tree_ext3_row_major_keep_dev( + table: &str, ra: &math_cuda::logup::ResidentAux, blowup_factor: usize, weights: &[FieldElement], retain_host_lde: bool, + rows_per_leaf: usize, ) -> Option<( MerkleTree, math_cuda::lde::GpuLdeExt3, @@ -2366,30 +3130,53 @@ pub(crate) fn try_expand_leaf_and_tree_ext3_row_major_keep_dev( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: DeviceTreeBackend, { if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() { return None; } + // No row floor: the aux trace is already on device. Only the bytes + // ceiling — a resident input that will not fit its own LDE is an abort. + let shape = DispatchShape { + table, + what: "R1 aux commit (resident)", + n: ra.num_rows, + base_cols: ra.num_aux_cols * 3, + blowup: blowup_factor, + }; + let set = commit_device_set_rpl( + ra.num_rows, + ra.num_aux_cols * 3, + blowup_factor, + false, + rows_per_leaf, + ); + admit_resident_commit(&shape, &set)?; let weights_u64 = unsafe { weights_to_u64::(weights) }; GPU_LDE_CALLS.fetch_add((ra.num_aux_cols * 3) as u64, Ordering::Relaxed); GPU_LEAF_HASH_CALLS.fetch_add(1, Ordering::Relaxed); GPU_MERKLE_TREE_CALLS.fetch_add(1, Ordering::Relaxed); + note_one_row_trees(rows_per_leaf, 1, set.tree_bytes); - let (handle, lde_u64) = math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep_dev( + let (handle, lde_u64) = math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep_dev_rpl( &ra.buf, + device_hash_of::(), ra.num_rows, ra.num_aux_cols, blowup_factor, &weights_u64, retain_host_lde, + rows_per_leaf, ) .inspect_err(|e| { - // Surface the swallowed driver error (e.g. OOM): the caller drains - // the device and retries, then downgrades the table to the host path. + // Surface the swallowed driver error (e.g. OOM): the caller drains the + // device and retries — a device-side recovery, which is why this is a + // decline and not an abort. If the retry declines too, the caller + // reports the failure (`ProvingError::DevicePath`); there is no host + // downgrade any more. eprintln!( "[gpu] resident aux LDE failed (rows={} cols={} blowup={}): {e:?}", ra.num_rows, ra.num_aux_cols, blowup_factor @@ -2474,9 +3261,6 @@ where } let main = lde_trace.gpu_main()?; let lde_size = main.lde_size; - if lde_size < gpu_lde_threshold() { - return None; - } if !lde_size.is_power_of_two() { return None; } @@ -2488,6 +3272,12 @@ where if h_ood.len() != num_parts { return None; } + // The DEEP codeword, plus — on the host arms — the parts and the inverted + // denominators staged onto the device (an upper bound over the three arms). + let bytes = ext3_bytes(lde_size as u64, (2 + num_parts + num_eval_points) as u64); + if !admit_transient(lde_size, bytes, "R4 DEEP") { + return None; + } if trace_ood_columns.len() != num_total_cols || trace_ood_columns.iter().any(|c| c.len() != num_eval_points) { @@ -2680,7 +3470,15 @@ where } let main = lde_trace.gpu_main()?; let lde_size = main.lde_size; - if lde_size < gpu_lde_threshold() || !lde_size.is_power_of_two() { + if !lde_size.is_power_of_two() { + return None; + } + // Everything is resident but the DEEP codeword itself. + if !admit_transient( + lde_size, + ext3_bytes(lde_size as u64, 1), + "R4 DEEP (resident)", + ) { return None; } let num_main = main.m; @@ -2742,6 +3540,121 @@ where Some(dw) } +/// [`try_deep_composition_gpu_keep`] with no inverted-denominators buffer, the +/// fully resident R4 DEEP's default path (see `math_cuda::deep_inv`): each row +/// inverts its own `1 + num_eval_points` denominators `x - z` in registers +/// ([`math_cuda::deep::deep_composition_ext3_fused_keep`]). +/// `coset_base` is the LDE coset and `z_scalars` the poles `[z^K, z·w^0, …]`, +/// the two inputs the buffered path hands `try_inv_denoms_dev_with_stream`; the +/// codeword is the buffered path's as field elements. `None` (→ the buffered +/// path) on any precondition miss, more points than the fused kernels take, or +/// a cudarc error. +#[allow(clippy::too_many_arguments)] +pub(crate) fn try_deep_composition_gpu_fused_keep( + lde_trace: &LDETraceTable, + parts_dev: &math_cuda::lde::GpuLdeExt3, + h_ood: &[FieldElement], + trace_ood_columns: &[Vec>], + composition_poly_gammas: &[FieldElement], + trace_terms_gammas: &[Vec>], + coset_base: &[FieldElement], + z_scalars: &[FieldElement], + num_eval_points: usize, +) -> Option +where + F: IsField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + if !is_goldilocks_ext3_tower::() { + return None; + } + if num_eval_points > math_cuda::deep::FUSED_MAX_EVAL_POINTS + || z_scalars.len() != 1 + num_eval_points + { + return None; + } + let main = lde_trace.gpu_main()?; + let lde_size = main.lde_size; + if !lde_size.is_power_of_two() || coset_base.len() != lde_size { + return None; + } + // The codeword and the coset points; there is no inverse buffer. + let bytes = ext3_bytes(lde_size as u64, 1).saturating_add((lde_size as u64) * BASE_BYTES); + if !admit_transient(lde_size, bytes, "R4 DEEP (fused)") { + return None; + } + let num_main = main.m; + let aux_handle = lde_trace.gpu_aux(); + let num_aux = aux_handle.map(|a| a.m).unwrap_or(0); + let num_total_cols = num_main + num_aux; + let num_parts = composition_poly_gammas.len(); + if h_ood.len() != num_parts { + return None; + } + if trace_ood_columns.len() != num_total_cols + || trace_ood_columns.iter().any(|c| c.len() != num_eval_points) + { + return None; + } + if trace_terms_gammas.len() != num_total_cols + || trace_terms_gammas + .iter() + .any(|c| c.len() != num_eval_points) + { + return None; + } + if parts_dev.m != num_parts || parts_dev.lde_size != lde_size { + return None; + } + + // Pack the small host scalars. SAFETY for ext3 transmutes: E == Ext3. + let h_ood_raw: &[u64] = unsafe { ext3_slice_to_u64::(h_ood) }; + let mut trace_ood_raw: Vec = Vec::with_capacity(num_total_cols * num_eval_points * 3); + for col in trace_ood_columns { + trace_ood_raw.extend_from_slice(unsafe { ext3_slice_to_u64::(col) }); + } + let gammas_h_raw: &[u64] = unsafe { ext3_slice_to_u64::(composition_poly_gammas) }; + let mut gammas_tr_raw: Vec = Vec::with_capacity(num_total_cols * num_eval_points * 3); + for col in trace_terms_gammas { + gammas_tr_raw.extend_from_slice(unsafe { ext3_slice_to_u64::(col) }); + } + let z_raw: &[u64] = unsafe { ext3_slice_to_u64::(z_scalars) }; + + // The table's session stream when it has one — the queue R3 and the + // buffered R4 path use — else a pool stream. + let stream = match lde_trace.bound_stream() { + Some(s) => s, + None => math_cuda::device::backend().ok()?.next_stream(), + }; + // SAFETY: F == Goldilocks per the tower check; FieldElement is + // #[repr(transparent)] over u64. + let coset_u64: &[u64] = unsafe { from_raw_parts(coset_base.as_ptr() as *const u64, lde_size) }; + let coset_dev = coset_points_device_handle(coset_u64, &stream)?; + + let dw = math_cuda::deep::deep_composition_ext3_fused_keep( + &stream, + main, + aux_handle, + parts_dev, + &coset_dev, + z_raw, + h_ood_raw, + &trace_ood_raw, + gammas_h_raw, + &gammas_tr_raw, + num_parts, + num_main, + num_aux, + num_eval_points, + 1, + lde_size, + ) + .ok()?; + GPU_DEEP_CALLS.fetch_add(1, Ordering::Relaxed); + GPU_FUSED_DEEP_CALLS.fetch_add(1, Ordering::Relaxed); + Some(dw) +} + /// Build `inv_denoms[k*n + i] = 1 / (lift(coset_base[i]) - z_scalars[k])` /// entirely on device. Used by both R3 OOD (n = trace_size, k_scalars = /// num_eval_points) and R4 DEEP (n = lde_size, k_scalars = 1 + @@ -2774,7 +3687,9 @@ where return None; } let total = n.checked_mul(k_scalars)?; - if total < gpu_lde_threshold() { + // The inverted denominators (`total` ext3) plus the coset points. + let bytes = ext3_bytes(total as u64, 1).saturating_add((n as u64).saturating_mul(BASE_BYTES)); + if !admit_transient(total, bytes, "batch invert") { return None; } @@ -2814,11 +3729,22 @@ fn coset_points_device_cache() CACHE.get_or_init(Default::default) } +/// Whether the device-resident copy of a host coset-points slice exists +/// already (what [`coset_points_device_handle`] would find without uploading). +pub(crate) fn coset_points_on_device(coset_u64: &[u64]) -> bool { + coset_u64.len() >= 2 + && coset_points_device_cache().lock().unwrap().contains_key(&( + coset_u64.len(), + coset_u64[0], + coset_u64[1], + )) +} + /// Resolve a host coset-points slice to its device-resident copy, uploading /// once per distinct coset. The first upload synchronizes its stream so the /// buffer is safe to read from any other stream afterwards. Returns `None` on /// upload failure (→ the caller's fallback). -fn coset_points_device_handle( +pub(crate) fn coset_points_device_handle( coset_u64: &[u64], stream: &Arc, ) -> Option>> { @@ -2912,14 +3838,27 @@ pub(crate) fn gather_proofs_dev( "gather_proofs_dev: position exceeds u32 range" ); let positions_u32: Vec = positions.iter().map(|&p| p as u32).collect(); - let bytes = math_cuda::merkle::gather_merkle_paths_dev( - &tree.nodes, - tree.leaves_len, - &positions_u32, - stream, - ) - .ok()?; - let depth = tree.leaves_len.trailing_zeros() as usize; + // A path is `depth` nodes: one sibling per binary level, or three per + // 4-ary level (`MerkleTree::get_proof_by_pos` at arity 4). + let (bytes, depth) = if tree.arity == math_cuda::p1_stark::ARITY { + let bytes = math_cuda::p1_stark::gather_paths_dev( + &tree.nodes, + tree.leaves_len, + &positions_u32, + stream, + ) + .ok()?; + (bytes, 3 * math_cuda::p1_stark::depth(tree.leaves_len)) + } else { + let bytes = math_cuda::merkle::gather_merkle_paths_dev( + &tree.nodes, + tree.leaves_len, + &positions_u32, + stream, + ) + .ok()?; + (bytes, tree.leaves_len.trailing_zeros() as usize) + }; debug_assert_eq!(bytes.len(), positions.len() * depth * 32); let mut proofs = Vec::with_capacity(positions.len()); for q in 0..positions.len() { @@ -2935,6 +3874,99 @@ pub(crate) fn gather_proofs_dev( Some(proofs) } +/// Read the height-`cap_height` Merkle cap of a device-resident tree: the +/// nodes `MerkleTree::cap` returns on the host tree, +/// byte for byte, since the device heap has the host layout. The R4 cap +/// post-pass calls it for every capped tree whose host tree is root-only. +/// +/// Fails closed with a message, never a panic: a cap taller than the tree or a +/// cudarc error is an `Err` the caller turns into a `ProvingError`. `stream` +/// is the stream the tree's own openings were gathered on (the table's bound +/// stream; a fresh backend stream for the FRI layers, as the FRI query phase +/// uses). +pub(crate) fn read_cap_dev( + tree: &math_cuda::lde::GpuMerkleTree, + cap_height: usize, + stream: &Arc, +) -> Result, String> { + if !tree.leaves_len.is_power_of_two() { + return Err(format!( + "device tree has {} leaves, not a power of two", + tree.leaves_len + )); + } + // A 4-ary tree (top-down level layout, `math_cuda::p1_stark`): the real + // nodes of the level `cap_height` 4-ary levels below the root, the last + // level of the node buffer's top `cap_height + 1` levels. + if tree.arity == math_cuda::p1_stark::ARITY { + let sizes = math_cuda::p1_stark::level_sizes(tree.leaves_len); + let depth4 = sizes.len() - 1; + if cap_height > depth4 { + return Err(format!( + "cap height {cap_height} exceeds the arity-4 device tree's {depth4} levels" + )); + } + let prefix = math_cuda::p1_stark::top_levels_nodes(tree.leaves_len, cap_height + 1); + if tree.nodes.len() < prefix * 32 { + return Err(format!( + "device node buffer of {} bytes is too short for a height-{cap_height} cap", + tree.nodes.len() + )); + } + // The cap level is the prefix's tail: copy just its bytes, on the + // stream the tree's openings were gathered on. + let len = sizes[depth4 - cap_height]; + let bytes = stream + .clone_dtoh(&tree.nodes.slice((prefix - len) * 32..prefix * 32)) + .map_err(|e| format!("cudarc: {e:?}"))?; + stream.synchronize().map_err(|e| format!("cudarc: {e:?}"))?; + let cap: Vec = bytes + .chunks_exact(32) + .map(|c| { + let mut node: Commitment = [0u8; 32]; + node.copy_from_slice(c); + node + }) + .collect(); + GPU_CAP_READ_CALLS.fetch_add(1, Ordering::Relaxed); + return Ok(cap); + } + if tree.arity != 2 { + return Err(format!("a cap of an arity-{} device tree", tree.arity)); + } + let depth = tree.leaves_len.trailing_zeros() as usize; + if cap_height > depth { + return Err(format!( + "cap height {cap_height} exceeds the device tree depth {depth}" + )); + } + if tree.nodes.len() < ((2usize << cap_height) - 1) * 32 { + return Err(format!( + "device node buffer of {} bytes is too short for a height-{cap_height} cap", + tree.nodes.len() + )); + } + let bytes = math_cuda::merkle::read_cap_dev(&tree.nodes, tree.leaves_len, cap_height, stream) + .map_err(|e| format!("cudarc: {e:?}"))?; + let cap: Vec = bytes + .chunks_exact(32) + .map(|c| { + let mut node: Commitment = [0u8; 32]; + node.copy_from_slice(c); + node + }) + .collect(); + if cap.len() != 1 << cap_height { + return Err(format!( + "device cap read returned {} nodes, expected {}", + cap.len(), + 1usize << cap_height + )); + } + GPU_CAP_READ_CALLS.fetch_add(1, Ordering::Relaxed); + Ok(cap) +} + /// R3 OOD device-side context: bundles the inverted denominators, the /// coset_points upload (used by every barycentric kernel for this batch), /// and the stream so producer + consumers serialize naturally. Hoisting @@ -2976,7 +4008,9 @@ where return None; } let total = n.checked_mul(k_scalars)?; - if total < gpu_lde_threshold() { + // The inverted denominators (`total` ext3) plus the coset points. + let bytes = ext3_bytes(total as u64, 1).saturating_add((n as u64).saturating_mul(BASE_BYTES)); + if !admit_transient(total, bytes, "R3 context") { return None; } @@ -3024,25 +4058,95 @@ where /// a byte-identical pre-GPU transcript state and produces the same proof /// it would have produced had the GPU never been tried. This requires the /// concrete transcript type to support snapshot semantics via `Clone`. -#[allow(clippy::type_complexity)] -pub(crate) fn try_fri_commit_gpu( +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +pub(crate) fn try_fri_commit_gpu( evals: &[FieldElement], transcript: &mut T, coset_offset: &FieldElement, domain_size: usize, blowup_log: u32, final_poly_log_degree: u32, + layout: &crate::fri::terminal::FriFoldLayout, inv_twiddles: &[FieldElement], -) -> Option<( - Vec>, - Vec>>, -)> +) -> Option<(Vec>, Vec>)> +where + F: IsFFTField + IsField + IsSubFieldOf + 'static, + E: IsField + 'static + Send + Sync, + FieldElement: AsBytes, + FieldElement: AsBytes, + T: IsStarkTranscript + Clone, + B: DeviceTreeBackend, +{ + // Host-evals entry: the caller works with host copies, keep draining them. + try_fri_commit_gpu_evals::( + evals, + transcript, + coset_offset, + domain_size, + blowup_log, + final_poly_log_degree, + layout, + inv_twiddles, + true, + ) +} + +/// [`try_fri_commit_gpu`] with the layers' host copies optional: `want_host = +/// false` keeps each committed layer's evals resident only (`gpu_evals`), the +/// shape the device-only envelope produces — so the device query phase's +/// resident gathers can be tested from host evals. +#[cfg(any(test, feature = "test-utils"))] +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +pub(crate) fn try_fri_commit_gpu_resident( + evals: &[FieldElement], + transcript: &mut T, + coset_offset: &FieldElement, + domain_size: usize, + blowup_log: u32, + final_poly_log_degree: u32, + layout: &crate::fri::terminal::FriFoldLayout, + inv_twiddles: &[FieldElement], +) -> Option<(Vec>, Vec>)> +where + F: IsFFTField + IsField + IsSubFieldOf + 'static, + E: IsField + 'static + Send + Sync, + FieldElement: AsBytes, + FieldElement: AsBytes, + T: IsStarkTranscript + Clone, + B: DeviceTreeBackend, +{ + try_fri_commit_gpu_evals::( + evals, + transcript, + coset_offset, + domain_size, + blowup_log, + final_poly_log_degree, + layout, + inv_twiddles, + false, + ) +} + +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +fn try_fri_commit_gpu_evals( + evals: &[FieldElement], + transcript: &mut T, + coset_offset: &FieldElement, + domain_size: usize, + blowup_log: u32, + final_poly_log_degree: u32, + layout: &crate::fri::terminal::FriFoldLayout, + inv_twiddles: &[FieldElement], + want_host: bool, +) -> Option<(Vec>, Vec>)> where F: IsFFTField + IsField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, FieldElement: AsBytes, FieldElement: AsBytes, T: IsStarkTranscript + Clone, + B: DeviceTreeBackend, { // GPU drives the early-termination FRI commit phase, mirroring // `commit_phase_from_evaluations`: for each committed layer (sample zeta, @@ -3058,7 +4162,14 @@ where if n0 != domain_size || !n0.is_power_of_two() || n0 < 2 { return None; } - if n0 < gpu_lde_threshold() { + // The evals upload, the geometric layer chain (bounded by one more + // codeword) and the layer trees (bounded by one full tree; under one row + // the chain starts at the codeword itself, so by the one-row bound). + let bytes = ext3_bytes(n0 as u64, 2).saturating_add(tree_bytes_for( + n0 as u64, + if layout.one_row { 1 } else { 2 }, + )); + if !admit_transient(n0, bytes, "R4 FRI commit") { return None; } // Mismatched twiddles would panic inside `FriCommitState::new`; gate here @@ -3082,43 +4193,47 @@ where // SAFETY: E == Ext3; FieldElement backing is [u64; 3]. let evals_u64: &[u64] = unsafe { ext3_slice_to_u64::(evals) }; - let state = match math_cuda::fri::FriCommitState::new(evals_u64, &inv_tw_u64, n0) { + let state = match math_cuda::fri::FriCommitState::new( + evals_u64, + &inv_tw_u64, + n0, + device_hash_of::(), + ) { Ok(s) => s, Err(_) => return None, }; - // Host-evals entry: the caller works with host copies, keep draining them. - fri_commit_gpu_drive( + fri_commit_gpu_drive::( state, transcript, coset_offset, n0, blowup_log, final_poly_log_degree, - true, + layout, + want_host, ) } /// [`try_fri_commit_gpu`] entered from a device-resident DEEP codeword /// (already in FRI order): no evals H2D at all. -#[allow(clippy::type_complexity)] -pub(crate) fn try_fri_commit_gpu_from_dev( +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +pub(crate) fn try_fri_commit_gpu_from_dev( codeword: math_cuda::deep::GpuDeepCodeword, transcript: &mut T, coset_offset: &FieldElement, blowup_log: u32, final_poly_log_degree: u32, + layout: &crate::fri::terminal::FriFoldLayout, inv_twiddles: &[FieldElement], want_host: bool, -) -> Option<( - Vec>, - Vec>>, -)> +) -> Option<(Vec>, Vec>)> where F: IsFFTField + IsField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, FieldElement: AsBytes, FieldElement: AsBytes, T: IsStarkTranscript + Clone, + B: DeviceTreeBackend, { if TypeId::of::() != TypeId::of::() { return None; @@ -3127,7 +4242,17 @@ where return None; } let n0 = codeword.n; - if !n0.is_power_of_two() || n0 < 2 || n0 < gpu_lde_threshold() { + if !n0.is_power_of_two() || n0 < 2 { + return None; + } + // The layer chain and its trees; the codeword is already resident. Under + // one row the input tree is built over the codeword itself (the one-row + // tree bound covers it and every later layer). + let bytes = ext3_bytes(n0 as u64, 1).saturating_add(tree_bytes_for( + n0 as u64, + if layout.one_row { 1 } else { 2 }, + )); + if !admit_transient(n0, bytes, "R4 FRI commit (resident)") { return None; } // Mismatched twiddles would panic inside `FriCommitState::new_dev`; @@ -3141,44 +4266,54 @@ where let v: u64 = unsafe { *(t.value() as *const _ as *const u64) }; inv_tw_u64.push(v); } - let state = match math_cuda::fri::FriCommitState::new_dev(codeword, &inv_tw_u64) { - Ok(s) => s, - Err(_) => return None, - }; - fri_commit_gpu_drive( + let state = + match math_cuda::fri::FriCommitState::new_dev(codeword, &inv_tw_u64, device_hash_of::()) + { + Ok(s) => s, + Err(_) => return None, + }; + fri_commit_gpu_drive::( state, transcript, coset_offset, n0, blowup_log, final_poly_log_degree, + layout, want_host, ) } -/// The shared FRI commit loop over an initialized device state: per committed -/// layer sample ζ, fold + commit on device, D2H root/evals; then the terminal -/// fold and CPU coefficient extraction. Restores the transcript and returns -/// `None` on any mid-loop cudarc failure so the CPU path reruns cleanly. -#[allow(clippy::type_complexity)] -fn fri_commit_gpu_drive( +/// The shared FRI commit loop over an initialized device state, under the +/// proof format's fold `layout` (the caller's, built for this codeword by +/// [`crate::fri::terminal::FriFoldLayout::for_options`]): per committed layer +/// sample ζ, fold + commit on device, D2H root/evals; then the terminal folds +/// and CPU coefficient extraction. Restores the transcript and returns `None` +/// on any mid-loop cudarc failure so the CPU path reruns cleanly. +/// +/// The legacy encoding runs today's loop (one fold and a pair-leaf commit per +/// layer); the group encoding runs [`fri_commit_gpu_drive_groups`], the device +/// twin of `commit_phase_with_layout`'s pending-fold loop — one-row layouts +/// (S2) included, whose layer 0 is the input tree committed from the codeword +/// itself with no challenge before it. +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +fn fri_commit_gpu_drive( mut state: math_cuda::fri::FriCommitState, transcript: &mut T, coset_offset: &FieldElement, n0: usize, blowup_log: u32, final_poly_log_degree: u32, + layout: &crate::fri::terminal::FriFoldLayout, want_host: bool, -) -> Option<( - Vec>, - Vec>>, -)> +) -> Option<(Vec>, Vec>)> where F: IsFFTField + IsField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, FieldElement: AsBytes, FieldElement: AsBytes, T: IsStarkTranscript + Clone, + B: DeviceTreeBackend, { // The unsafe zeta reads below reinterpret `FieldElement` as 3 u64: // every caller gates the tower, but assert here so a future caller with @@ -3195,12 +4330,17 @@ where // produced had this dispatch never been called. let transcript_snapshot = transcript.clone(); - // Fold layout, shared with the CPU prover and the verifier — see `FriFoldLayout`. - let layout = crate::fri::terminal::FriFoldLayout::new( - n0.trailing_zeros(), - blowup_log, - final_poly_log_degree, - ); + // Fold layout, shared with the CPU prover and the verifier — see + // `FriFoldLayout`. It must be this codeword's: a layout built for another + // size degrades to the CPU path instead of committing a wrong chain. + if layout.terminal_len == 0 + || n0 + .trailing_zeros() + .checked_sub(layout.terminal_len.trailing_zeros()) + != Some(layout.total_folds) + { + return None; + } // The GPU path only runs above gpu_lde_threshold(). Two cases fall back to // the CPU path (which handles both correctly): tiny clamped traces // (total_folds == 0), and terminal_len == 1 (blowup_log + k == 0), whose @@ -3209,9 +4349,30 @@ where if layout.total_folds == 0 || layout.terminal_len < 2 { return None; } + // One-row layouts are never the legacy encoding (FriFoldLayout: one row ⇒ + // group encoding), so they always take the group drive below. + debug_assert!(!layout.one_row || !layout.is_legacy()); + if !layout.is_legacy() { + return fri_commit_gpu_drive_groups::( + state, + transcript, + transcript_snapshot, + coset_offset, + layout, + want_host, + ); + } + // Today's encoding: the layout is today's (the all-ones schedule). + debug_assert_eq!( + *layout, + crate::fri::terminal::FriFoldLayout::new( + n0.trailing_zeros(), + blowup_log, + final_poly_log_degree + ) + ); let num_committed = layout.num_committed; - let mut fri_layer_list: Vec>> = - Vec::with_capacity(num_committed); + let mut fri_layer_list: Vec> = Vec::with_capacity(num_committed); for _layer_idx in 0..num_committed { // <<<< Receive challenge zeta_k @@ -3236,7 +4397,7 @@ where .map(|v| u64_to_ext3_vec::(&v)) .unwrap_or_default(); let root = dev_tree.root; - let merkle_tree = MerkleTree::>::from_root(root); + let merkle_tree = MerkleTree::::from_root(root); // Retain the device evals only when no host copy exists (device-only): // with a host copy the query phase reads it, and the retained buffer // would be ~24 bytes/LDE-row of dead VRAM per table. @@ -3288,6 +4449,120 @@ where Some((final_poly_coeffs, fri_layer_list)) } +/// The raw limbs of `ζ, ζ², …, ζ^{2^{n−1}}`: the challenges of `n` successive +/// binary folds, squared on the host exactly as the CPU loop's `fold_times` +/// squares them. +fn zeta_powers_raw(zeta: &FieldElement, n: u32) -> Vec<[u64; 3]> { + let mut out = Vec::with_capacity(n as usize); + let mut z = zeta.clone(); + for level in 0..n { + // SAFETY: E == Ext3 (asserted by the drive before any call); its + // backing is [u64; 3]. + let p = &z as *const FieldElement as *const u64; + out.push(unsafe { [*p, *p.add(1), *p.add(2)] }); + if level + 1 < n { + z = z.square(); + } + } + out +} + +/// The group-encoding (S3) device commit loop: the device twin of +/// [`crate::fri::commit_phase_with_layout`]'s pending-fold loop. Per committed +/// layer `j` with exponent `d_j`: sample ζ, fold `d_{j−1}` times on device with +/// `ζ, ζ², …` (`d_{−1} = 1`, the binary fold 0 of the DEEP pair), commit the +/// result with leaves of `2^{d_j}` consecutive values, append the root; then +/// sample the final ζ and fold `d_last` times into the terminal codeword. +/// +/// One-row layouts (S2): `d_{−1} = 0` — layer 0 is the INPUT TREE, the resident +/// DEEP codeword itself committed with groups of `2^{d_0}` (a zero-fold group +/// commit, the group kernels), its root appended with NO challenge +/// sampled before it (the CPU loop's `pending = 0`); every later +/// layer is as above. +/// Transcript order, ζ powers, fold arithmetic and leaf bytes are the CPU +/// loop's, so the two produce the same proof (the parity tests pin it). +#[allow(clippy::type_complexity)] +fn fri_commit_gpu_drive_groups( + mut state: math_cuda::fri::FriCommitState, + transcript: &mut T, + transcript_snapshot: T, + coset_offset: &FieldElement, + layout: &crate::fri::terminal::FriFoldLayout, + want_host: bool, +) -> Option<(Vec>, Vec>)> +where + F: IsFFTField + IsField + IsSubFieldOf + 'static, + E: IsField + 'static + Send + Sync, + FieldElement: AsBytes, + FieldElement: AsBytes, + T: IsStarkTranscript + Clone, + B: DeviceTreeBackend, +{ + let mut fri_layer_list: Vec> = Vec::with_capacity(layout.num_committed); + // Folds owed before the next commit: fold 0 is the binary fold of the DEEP + // pair, so one; after committing layer `j`, `d_j`. Under one row nothing + // is owed before the input tree, and no challenge is drawn for it. + let mut pending: u32 = if layout.one_row { 0 } else { 1 }; + for &d in &layout.schedule { + let powers = if pending > 0 { + // <<<< Receive challenge zeta_j + let zeta: FieldElement = transcript.sample_field_element(); + zeta_powers_raw(&zeta, pending) + } else { + Vec::new() + }; + let (layer_evals_u64, evals_dev, dev_tree) = + match state.fold_and_commit_group(&powers, u32::from(d), want_host) { + Ok(v) => v, + Err(_) => { + *transcript = transcript_snapshot; + return None; + } + }; + let evaluation = layer_evals_u64 + .map(|v| u64_to_ext3_vec::(&v)) + .unwrap_or_default(); + let root = dev_tree.root; + fri_layer_list.push(FriLayer { + evaluation, + merkle_tree: MerkleTree::::from_root(root), + gpu_tree: Some(dev_tree), + gpu_evals: (!want_host).then_some(evals_dev), + }); + // >>>> Send commitment: [p_j] + transcript.append_bytes(&root); + pending = u32::from(d); + } + + // The final folds into the terminal codeword (total_folds > 0 here). + let zeta_final: FieldElement = transcript.sample_field_element(); + let terminal_evals_u64 = match state.fold_to_host(&zeta_powers_raw(&zeta_final, pending)) { + Ok(v) => v, + Err(_) => { + *transcript = transcript_snapshot; + return None; + } + }; + debug_assert_eq!(terminal_evals_u64.len(), layout.terminal_len * 3); + let terminal_codeword = u64_to_ext3_vec::(&terminal_evals_u64); + let terminal_offset = coset_offset.pow(1u64 << layout.total_folds); + let final_poly_coeffs = crate::fri::terminal::coeffs_from_terminal_codeword::( + &terminal_codeword, + &terminal_offset, + layout.effective_k, + ); + // >>>> Send the final polynomial coefficients. + for c in &final_poly_coeffs { + transcript.append_field_element(c); + } + + GPU_FRI_CALLS.fetch_add(1, Ordering::Relaxed); + if layout.one_row { + GPU_ONE_ROW_FRI_CALLS.fetch_add(1, Ordering::Relaxed); + } + Some((final_poly_coeffs, fri_layer_list)) +} + /// GPU FRI query phase: gather each layer's paths on device instead of walking /// host trees. For layer `l` and query `iota` the opened position is /// `(iota >> l) >> 1`, matching [`crate::fri::query_phase`]. Paths for all @@ -3296,13 +4571,14 @@ where /// /// Returns None when there are no layers or the layers are host trees (CPU /// commit), so the caller falls back to the host walk. -pub(crate) fn try_fri_query_phase_gpu( - fri_layers: &[FriLayer>], +pub(crate) fn try_fri_query_phase_gpu( + fri_layers: &[FriLayer], iotas: &[usize], ) -> Option>> where E: IsField + 'static, FieldElement: AsBytes + Sync + Send, + B: DeviceTreeBackend, { if fri_layers.is_empty() { return None; @@ -3392,6 +4668,186 @@ where Some(decommits) } +/// GPU FRI query phase for the group encoding (S3): the device twin of +/// [`crate::fri::query_phase_with_layout`]'s group branch. Per committed layer +/// `j` and query at position `p` the opened leaf is `p >> d_j`, its path is +/// gathered on device (one batched call per layer), and the opened values are +/// the whole group `[leaf·2^{d_j}, (leaf+1)·2^{d_j})` — read from the host evals +/// when the commit drained them, else one batched device gather per layer off +/// the resident evals; then `p ← p >> d_j`. +/// +/// Returns `None` when there are no layers or the layers are host trees (CPU +/// commit), so the caller takes the host walk. Resident layers have root-only +/// host trees, so a failed gather there is a hard abort, as in +/// [`try_fri_query_phase_gpu`]. +pub(crate) fn try_fri_query_phase_gpu_groups( + fri_layers: &[FriLayer], + iotas: &[usize], + layout: &crate::fri::terminal::FriFoldLayout, +) -> Option>> +where + E: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + B: DeviceTreeBackend, +{ + if fri_layers.is_empty() { + return None; + } + let first_resident = fri_layers[0].gpu_tree.is_some(); + debug_assert!( + fri_layers + .iter() + .all(|l| l.gpu_tree.is_some() == first_resident), + "FRI layer residency must be all or nothing" + ); + if !first_resident { + return None; + } + assert_eq!( + fri_layers.len(), + layout.schedule.len(), + "one committed FRI layer per schedule entry" + ); + let stream = math_cuda::device::backend() + .expect("cuda backend for device-resident FRI query") + .next_stream(); + + // Per query, the position at each committed layer. + let positions: Vec> = iotas + .iter() + .map(|&iota| { + let mut p = iota; + layout + .schedule + .iter() + .map(|&d| { + let here = p; + p >>= d; + here + }) + .collect() + }) + .collect(); + + let mut per_layer_proofs: Vec>> = Vec::with_capacity(fri_layers.len()); + let mut per_layer_groups: Vec>>> = + Vec::with_capacity(fri_layers.len()); + for (j, (layer, &d)) in fri_layers.iter().zip(&layout.schedule).enumerate() { + let tree = layer + .gpu_tree + .as_ref() + .expect("FRI layers are device-resident as a group"); + let leaves: Vec = positions.iter().map(|p| p[j] >> d).collect(); + per_layer_proofs.push( + gather_proofs_dev(tree, &leaves, &stream) + .expect("device FRI-layer gather failed; resident tree has no host fallback"), + ); + per_layer_groups.push(if layer.evaluation.is_empty() { + let evals_dev = layer + .gpu_evals + .as_ref() + .expect("device-only FRI layer without resident evals"); + let n = 1usize << d; + let group_positions: Vec = leaves + .iter() + .flat_map(|&leaf| (leaf * n..(leaf + 1) * n).map(|x| x as u32)) + .collect(); + let raw = math_cuda::fri::gather_ext3_at(evals_dev, &group_positions, &stream) + .expect("device FRI group gather failed; no host fallback"); + Some( + crate::constraint_ir::gpu_interp::ext3_u64_to_field::(&raw) + .expect("resident FRI evals are Goldilocks ext3"), + ) + } else { + None + }); + } + + let values_per_query = layout.opened_values_per_query(); + let decommits = positions + .iter() + .enumerate() + .map(|(q, pos)| { + let mut values = Vec::with_capacity(values_per_query); + let mut paths = Vec::with_capacity(fri_layers.len()); + for (j, (layer, &d)) in fri_layers.iter().zip(&layout.schedule).enumerate() { + let n = 1usize << d; + match &per_layer_groups[j] { + Some(g) => values.extend_from_slice(&g[q * n..(q + 1) * n]), + None => { + let leaf = pos[j] >> d; + values.extend_from_slice(&layer.evaluation[leaf * n..(leaf + 1) * n]); + } + } + paths.push(per_layer_proofs[j][q].clone()); + } + FriDecommitment { + layers_auth_paths: paths, + layers_evaluations_sym: values, + } + }) + .collect(); + Some(decommits) +} + +/// The abort itself, on a real device. `LAMBDA_VM_VRAM_BUDGET_MB` is read once +/// at backend init, so this test runs in its own process with the budget +/// lowered to 1 GiB — the shape is then over budget on any card while its host +/// input stays at 1.2 GiB: +/// +/// ```text +/// LAMBDA_VM_VRAM_BUDGET_MB=1024 cargo test -p stark --release --features cuda \ +/// --lib gpu_lde::admission_box_tests -- --ignored --nocapture +/// ``` +#[cfg(all(test, feature = "cuda"))] +mod admission_box_tests { + use super::*; + use crate::config::BatchedMerkleTreeBackend; + + type F = GoldilocksField; + type Fp = FieldElement; + + /// 2^18 rows x 612 columns at blowup 2: a 2.4 GiB LDE, 3.7 GiB device set + /// against the 1 GiB budget the command line sets. Must abort with the + /// over-budget diagnostic, never commit on the host. + #[test] + #[ignore = "requires GPU and LAMBDA_VM_VRAM_BUDGET_MB=1024 in its own process"] + #[should_panic(expected = "over the VRAM admission budget")] + fn an_over_budget_commit_aborts_instead_of_committing_on_the_host() { + let n: usize = 1 << 18; + let m: usize = 612; + let blowup: usize = 2; + let budget = device_vram_budget_bytes().expect("cuda backend"); + assert!( + budget <= 2 << 30, + "this test needs LAMBDA_VM_VRAM_BUDGET_MB=1024 set before the backend initialises \ + (budget is {budget} B)" + ); + let data: Vec = (0..n * m).map(|i| Fp::from(i as u64)).collect(); + let weights: Vec = (0..n).map(|i| Fp::from(i as u64 + 1)).collect(); + let committed = try_expand_leaf_and_tree_row_major_keep::>( + "admission_box_test", + "admission box", + &data, + None, + n, + m, + blowup, + &weights, + true, + 2, + ); + panic!( + "the over-budget commit returned {} instead of aborting", + if committed.is_some() { + "a device handle" + } else { + "None (a silent host commit)" + } + ); + } +} + /// GPU↔CPU parity for the preprocessed split-tree commit path. Requires the /// `cuda` feature and a visible GPU (skipped otherwise via the dispatch gate /// returning `None` — asserted here, so a silent skip fails the test). @@ -3437,7 +4893,17 @@ mod split_tree_tests { let (pre_tree, mult_tree, handle, lde) = try_expand_split_trees_row_major_keep::>( - &data, None, n, m, blowup, &weights, split, true, true, + "split_tree_test", + &data, + None, + n, + m, + blowup, + &weights, + split, + true, + true, + 2, ) .expect("GPU split path must engage above the threshold"); let pre_tree = pre_tree.expect("precomputed tree was requested"); diff --git a/crypto/stark/src/grinding.rs b/crypto/stark/src/grinding.rs index b3642c656..cdec76eda 100644 --- a/crypto/stark/src/grinding.rs +++ b/crypto/stark/src/grinding.rs @@ -1,155 +1,20 @@ -use crypto::hash::platform_keccak::PlatformKeccak256 as Keccak256; -use digest::Digest; -#[cfg(feature = "parallel")] -use rayon::prelude::{IntoParallelIterator, ParallelIterator}; - -const PREFIX: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xed]; - -/// Checks if the bit-string `Hash(Hash(prefix || seed || grinding_factor) || nonce)` -/// has at least `grinding_factor` zeros to the left. -/// `prefix` is the bit-string `0x123456789abcded` -/// -/// # Parameters -/// -/// * `seed`: the input seed, -/// * `nonce`: the value to be tested, -/// * `grinding_factor`: the number of leading zeros needed; must be in `1..=64`. -/// -/// # Returns -/// -/// `true` if the number of leading zeros is at least `grinding_factor`, and `false` otherwise. -pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool { - debug_assert!( - (1..=64).contains(&grinding_factor), - "grinding_factor must be in 1..=64, got {grinding_factor}" - ); - let inner_hash = get_inner_hash(seed, grinding_factor); - let limit = 1 << (64 - grinding_factor); - is_valid_nonce_for_inner_hash(&inner_hash, nonce, limit) -} - -/// Performs grinding, returning a new nonce for the proof. -/// The nonce generated is such that: -/// Hash(Hash(prefix || seed || grinding_factor) || nonce) has at least `grinding_factor` zeros -/// to the left. -/// `prefix` is the bit-string `0x123456789abcded` -/// -/// # Parameters -/// -/// * `seed`: the input seed, -/// * `grinding_factor`: the number of leading zeros needed; must be in `1..=64`. -/// -/// # Returns -/// -/// A `nonce` satisfying the required condition. -pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option { - debug_assert!( - (1..=64).contains(&grinding_factor), - "grinding_factor must be in 1..=64, got {grinding_factor}" - ); - let inner_hash = get_inner_hash(seed, grinding_factor); - let limit = 1 << (64 - grinding_factor); - - #[cfg(not(feature = "parallel"))] - return (0..u64::MAX).find(|&candidate_nonce| { - is_valid_nonce_for_inner_hash(&inner_hash, candidate_nonce, limit) - }); - - #[cfg(feature = "parallel")] - return (0..u64::MAX).into_par_iter().find_any(|&candidate_nonce| { - is_valid_nonce_for_inner_hash(&inner_hash, candidate_nonce, limit) - }); -} - -/// Checks if the leftmost 8 bytes of `Hash(inner_hash || candidate_nonce)` are less than `limit` -/// when interpreted as `u64`. -#[inline(always)] -fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool { - // Tag this finalize as grinding so a verify-hash metric can report it apart - // (see `crypto::hash_metrics`); no-op unless the `hash-metrics` feature is on. - crypto::hash_metrics::count_grinding(); - let mut data = [0; 40]; - data[..32].copy_from_slice(inner_hash); - data[32..].copy_from_slice(&candidate_nonce.to_be_bytes()); - - let digest = Keccak256::digest(data); - - let seed_head = u64::from_be_bytes(digest[..8].try_into().unwrap()); - seed_head < limit -} - -/// Returns the bit-string constructed as -/// Hash(prefix || seed || grinding_factor) -/// `prefix` is the bit-string `0x123456789abcded` -fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] { - // Grinding finalize (see `crypto::hash_metrics`); no-op unless enabled. - crypto::hash_metrics::count_grinding(); - let mut inner_data = [0u8; 41]; - inner_data[0..8].copy_from_slice(&PREFIX); - inner_data[8..40].copy_from_slice(seed); - inner_data[40] = grinding_factor; - - let digest = Keccak256::digest(inner_data); - digest[..32].try_into().unwrap() -} - -/// The inner hash as the four little-endian u64 lanes Keccak absorbs it into — -/// the form the device nonce search takes as input. -/// -/// The GPU dispatch and its test both go through here rather than each doing -/// their own byte-to-lane conversion: a second copy would let this one drift -/// (`from_le_bytes` → `from_be_bytes` reads identically at a glance) with every -/// test still green, while at runtime `is_valid_nonce` rejected every device -/// nonce and the search silently sat on the CPU fallback forever. -pub fn inner_hash_lanes(seed: &[u8; 32], grinding_factor: u8) -> [u64; 4] { - let inner_hash = get_inner_hash(seed, grinding_factor); - core::array::from_fn(|i| u64::from_le_bytes(inner_hash[i * 8..i * 8 + 8].try_into().unwrap())) -} - -/// Grind on the GPU when a CUDA backend is up, falling back to the CPU search -/// otherwise (or on any device error). Which valid nonce comes back depends on -/// the arm: the device search returns the smallest in the range it scanned, -/// while the CPU's `find_any` returns an arbitrary one. Neither is a contract — -/// the verifier accepts any nonce passing `is_valid_nonce`, and nothing -/// downstream depends on the choice. The heavy per-table-per-epoch -/// ~2^grinding_factor hashing is the prover's dominant CPU cost, so this moves -/// it off the 16 cores onto the idle GPU. -#[cfg(feature = "cuda")] -pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option { - debug_assert!( - (1..=64).contains(&grinding_factor), - "grinding_factor must be in 1..=64, got {grinding_factor}" - ); - // Kill switch (presence-based, matching `LAMBDA_VM_NO_GPU_LOGUP`): - // `LAMBDA_VM_NO_GPU_GRIND` forces the CPU search — a production escape hatch - // and fallback-path coverage. Cached; read once. - static GPU_DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - if *GPU_DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_GRIND").is_some()) { - return generate_nonce(seed, grinding_factor); - } - let inner_lanes = inner_hash_lanes(seed, grinding_factor); - if let Some(nonce) = math_cuda::grinding::generate_nonce_gpu(&inner_lanes, grinding_factor) { - // Validate unconditionally (one host hash against the ~2^grinding_factor - // device search): a kernel/driver defect must degrade to the CPU search, - // never append an unverifiable nonce to the transcript. This runs in - // release too — the cost is negligible next to the grind it replaces. - if is_valid_nonce(seed, nonce, grinding_factor) { - crate::gpu_lde::GPU_GRIND_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - return Some(nonce); - } - // eprintln, not log::warn: the CLI initialises env_logger with no - // default filter, so a warn-level line is invisible unless RUST_LOG is - // set — and this is the only signal that the kernel has started - // returning garbage and the feature has silently reverted to the CPU - // search. Matches the `[gpu]` prefix the other device-decline paths use. - eprintln!( - "[gpu] grind returned an invalid nonce ({nonce}); falling back to the CPU search" - ); - } - generate_nonce(seed, grinding_factor) -} - -#[cfg(not(feature = "cuda"))] -pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option { - generate_nonce(seed, grinding_factor) -} +//! The grinding primitive and its device dispatch, re-exported so existing +//! call sites read unchanged. Both live in [`crypto::grinding`], which the +//! multilinear prover can also reach. +//! +//! ⚠ The primitive is generic over its hash and carries **no default** — a +//! proof-of-work hash that defaulted would silently keep grinding on keccak for +//! a configuration that had moved everything else. This crate's univariate +//! prover and verifier are keccak throughout, so they name +//! [`StarkGrindingDigest`] at each call site: one token, but it is a statement +//! rather than an omission, and it is the thing that has to change if this path +//! ever gains a hash parameter of its own. + +pub use crypto::grinding::{ + DeviceGrindKey, GrindDigest, generate_nonce, generate_nonce_maybe_gpu, inner_hash_felts, + inner_hash_lanes, is_valid_nonce, +}; + +/// The hash the univariate prover and verifier grind with: keccak-256, the same +/// hash their Merkle trees and their transcript use. +pub type StarkGrindingDigest = crypto::hash::platform_keccak::PlatformKeccak256; diff --git a/crypto/stark/src/leaf_layout.rs b/crypto/stark/src/leaf_layout.rs new file mode 100644 index 000000000..a2134d87a --- /dev/null +++ b/crypto/stark/src/leaf_layout.rs @@ -0,0 +1,335 @@ +//! The trace-tree leaf layout of one table's proof (S2). +//! +//! Today every trace, precomputed, aux and composition tree commits one LDE +//! row PAIR per leaf (`commitment::ROWS_PER_LEAF = 2`): leaf `i` hashes the +//! bit-reversed rows `2i` and `2i + 1`, the points `x` and `−x`, and a query +//! opens that pair to rebuild the DEEP pair for the uncommitted FRI fold 0. +//! +//! Under one-row openings ([`LeafLayout::Row`]) every such tree commits ONE row +//! per leaf, the DEEP codeword itself is committed as FRI layer 0 (the "input +//! tree"), a query index ranges over the whole LDE (`r ∈ [0, N)`, bound `N`), +//! and the verifier computes DEEP at the one point `x_r` and checks it against +//! the input group's slot. +//! +//! The layout is decided PER TABLE by the proof format +//! ([`crate::proof::options::OneRowMode`]): `Off` = row pairs, `On` = one row, +//! `Auto` = whichever [`one_row_is_cheaper`] prices lower for this table's +//! committed widths and LDE size. Every input is AIR metadata or the trace +//! length the verifier already trusts for the FRI layout; none is read from +//! the proof's bytes. A proof may therefore mix layouts across tables, and +//! each table's layout is a verifier-side constant. +//! +//! [`LeafLayout::query_rows`] is the ONE place a query index becomes LDE rows: +//! every opening site, prover and verifier, goes through it. + +use crypto::merkle_tree::cap::{CapPolicy, cap_gain}; +use math::fft::bit_reversing::reverse_index; +use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; + +use crate::fri::schedule::{FRI_COST_WEIGHTS, FriFormat}; +use crate::proof::options::{OneRowMode, ProofOptions}; +use crate::traits::AIR; + +/// How many LDE rows one trace-tree leaf holds. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum LeafLayout { + /// Two bit-reversed rows per leaf, `(x, −x)`. Today's layout. + #[default] + RowPair, + /// One bit-reversed row per leaf (S2). + Row, +} + +impl LeafLayout { + /// `Row` iff `one_row`. + pub const fn from_one_row(one_row: bool) -> Self { + if one_row { Self::Row } else { Self::RowPair } + } + + /// Whether this is the one-row layout. + pub const fn is_one_row(self) -> bool { + matches!(self, Self::Row) + } + + /// Rows per leaf: 2 (today, [`crate::commitment::ROWS_PER_LEAF`]) or 1. + pub const fn rows_per_leaf(self) -> usize { + match self { + Self::RowPair => crate::commitment::ROWS_PER_LEAF, + Self::Row => 1, + } + } + + /// The exclusive bound of a query index over an LDE of `lde_len` points: + /// a leaf index, so `lde / 2` for row pairs and `lde` for one row + /// (under one row `r` must be uniform over ALL of `D₀`). + pub fn query_bound(self, lde_len: u64) -> u64 { + match self { + Self::RowPair => lde_len >> 1, + #[cfg(test)] + Self::Row + if M3_PAIR_BOUND_AT_LDE.load(core::sync::atomic::Ordering::SeqCst) == lde_len => + { + lde_len >> 1 + } + Self::Row => lde_len, + } + } + + /// Depth of a trace tree over an LDE of `2^lde_log` rows: one level per + /// bit of the leaf index (`log2(lde) − 1` for row pairs, `log2(lde)` for + /// one row; 0 when the leaf hash is the root). + pub const fn tree_depth(self, lde_log: usize) -> usize { + match self { + Self::RowPair => lde_log.saturating_sub(1), + Self::Row => lde_log, + } + } + + /// The LDE storage rows (natural-order indices into the LDE columns) that + /// query `q` opens: `(row, Some(sym_row))` for a row pair — the rows at + /// bit-reversed positions `2q` and `2q + 1`, the points `x` and `−x` — + /// and `(row, None)` for one row, the row at bit-reversed position `q`. + /// + /// The single site where a query index becomes rows. + pub fn query_rows(self, q: usize, lde_len: usize) -> (usize, Option) { + let n = lde_len as u64; + match self { + Self::RowPair => (reverse_index(q * 2, n), Some(reverse_index(q * 2 + 1, n))), + Self::Row => (reverse_index(q, n), None), + } + } +} + +/// Mutation M3, test builds only: sample one-row query indexes +/// over the row-pair bound `N / 2` — for an LDE of exactly this many points +/// (0 = off). Prover and verifier both read it, so a mutated proof still +/// verifies; only `one_row_tests`' bound test sees the bias, which is what +/// makes that test load-bearing. Process-global (the prover samples on worker +/// threads) and keyed by the LDE size, so it touches only the M3 test's own +/// shape (an LDE no other one-row test uses), never a concurrent test's proof. +#[cfg(test)] +pub(crate) static M3_PAIR_BOUND_AT_LDE: core::sync::atomic::AtomicU64 = + core::sync::atomic::AtomicU64::new(0); + +/// The committed widths of one table, in base-field elements per LDE row, per +/// tree. `0` = the tree does not exist. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TableWidths { + /// The precomputed tree (preprocessed tables only). + pub precomputed: u64, + /// The main tree (every main column, or the multiplicities of a + /// preprocessed table). + pub main: u64, + /// The aux tree. + pub aux: u64, + /// The composition tree (every part). + pub composition: u64, + /// `XALU` rows of ONE in-guest DEEP point for this table + /// ([`deep_point_xalu_rows`]); row pairs evaluate DEEP at two points, one + /// row at one. + pub deep_point_rows: u64, +} + +/// `XALU` rows the in-guest verifier emits for DEEP at ONE query point +/// (`prover/src/lfm/deep.rs::emit_deep_point`), for a table whose DEEP +/// reconstruction folds `num_surviving` trace openings (the pruned OOD grid, +/// [`crate::ood::OodLayout::num_surviving`]) over `num_eval_points` OOD rows +/// and `num_parts` composition parts: +/// +/// ```text +/// per OOD row r: (|cols_r| − 1) Horner steps + [r ≥ 1] block scale +/// + numerator esub + denominator esub + ediv + (emul | emul_add) +/// parts: (P − 1) Horner steps + emul + esub + esub + ediv + emul_add +/// total: num_surviving + 4·E + P + 3 +/// ``` +/// +/// One `XALU` row per opened value plus a per-point constant; the prover +/// crate's `lfm::fri_group_tests` pins it against the emitter. +pub const fn deep_point_xalu_rows(num_surviving: u64, num_eval_points: u64, num_parts: u64) -> u64 { + num_surviving + 4 * num_eval_points + num_parts + 3 +} + +impl TableWidths { + /// The widths of `air`'s trees for a trace of `trace_length` rows. Main + /// and precomputed columns are base-field elements; aux columns and + /// composition parts are `FieldExtension` elements, each + /// `ext_degree::()` base elements wide. + pub fn of( + air: &dyn AIR, + trace_length: usize, + ) -> Self + where + F: IsFFTField + IsSubFieldOf + Send + Sync, + E: IsField + Send + Sync, + { + let precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + let main = air.trace_layout().0.saturating_sub(precomputed); + let aux = air.num_auxiliary_rap_columns(); + let parts = if trace_length == 0 { + 0 + } else { + air.composition_poly_degree_bound(trace_length) / trace_length + }; + let ext = ext_degree::(); + let ctx = air.context(); + let num_eval_points = ctx.transition_offsets.len() * air.step_size(); + let ood = crate::ood::OodLayout::new( + ctx.trace_columns, + num_eval_points, + air.step_size(), + air.trace_ood_next_row_columns(), + ); + Self { + precomputed: precomputed as u64, + main: main as u64, + aux: (aux as u64).saturating_mul(ext), + composition: (parts as u64).saturating_mul(ext), + deep_point_rows: deep_point_xalu_rows( + ood.num_surviving() as u64, + num_eval_points as u64, + parts as u64, + ), + } + } +} + +/// Base-field elements per `E` element (3 for the Goldilocks cubic +/// extension, 1 when `E = F`). +fn ext_degree() -> u64 { + let f = core::mem::size_of::().max(1); + let e = core::mem::size_of::(); + (e / f).max(1) as u64 +} + +/// `Q ×` the per-query cost-law price of opening one trace tree whose leaf +/// holds `felts` base elements and whose tree is `depth` deep, under `cap`: +/// the leaf absorption, the walk (a compression and a select per level), and +/// minus what the tree's cap saves — the terms and weights of the FRI schedule +/// objective ([`crate::fri::schedule`]), applied to a trace tree. +pub fn trace_tree_cost_q(felts: u64, depth: u32, num_queries: u64, cap: CapPolicy) -> u64 { + if felts == 0 { + return 0; + } + let w = &FRI_COST_WEIGHTS.cap; + let blocks = felts + .div_ceil(crate::fri::schedule::FRI_LEAF_RATE_FELTS) + .max(1) as i128; + let per_query = + blocks * w.compress as i128 + i128::from(depth) * (w.compress as i128 + w.select as i128); + let queries = usize::try_from(num_queries).unwrap_or(usize::MAX); + let c = cap.height(queries, depth as usize); + let total = (num_queries as i128).saturating_mul(per_query) - cap_gain(w, queries, c); + u64::try_from(total.max(0)).unwrap_or(u64::MAX) +} + +/// `Q ×` the per-query price of one table's openings (every trace tree plus +/// the FRI chain) under `one_row`, for an LDE of `2^lde_log` rows with blowup +/// `2^blowup_log` and terminal log-degree `k`, under `options`' FRI mode, cap +/// policy and query count. +/// +/// Row pairs: every tree's leaf holds two rows and is `lde_log − 1` deep, the +/// FRI chain starts at `lde_log − 1` with the uncommitted fold 0 +/// ([`FriFormat::chain_cost_q`]), and DEEP is evaluated at TWO points (`υ`, +/// `−υ`). One row: every leaf holds one row and is `lde_log` deep, the FRI +/// chain (layer 0 = the committed DEEP codeword) starts at `lde_log`, and DEEP +/// is evaluated at ONE point. A DEEP point costs +/// [`TableWidths::deep_point_rows`] `XALU` rows. +pub fn table_openings_cost_q( + widths: &TableWidths, + options: &ProofOptions, + lde_log: u32, + blowup_log: u32, + one_row: bool, +) -> u64 { + let q = options.fri_number_of_queries as u64; + let cap = options.format.merkle_cap; + let layout = LeafLayout::from_one_row(one_row); + let rows = layout.rows_per_leaf() as u64; + let depth = layout.tree_depth(lde_log as usize) as u32; + let trees = [ + widths.precomputed, + widths.main, + widths.aux, + widths.composition, + ] + .iter() + .map(|&w| trace_tree_cost_q(w.saturating_mul(rows), depth, q, cap)) + .fold(0u64, u64::saturating_add); + + let terminal_log = (blowup_log + u32::from(options.fri_final_poly_log_degree)).min(lde_log); + let fmt = FriFormat { + mode: options.format.fri_mode, + one_row, + num_queries: q, + cap, + schedule_override: options.format.fri_schedule_override, + }; + let chain = fmt.chain_cost_q(lde_log, terminal_log); + let deep_points: u64 = if one_row { 1 } else { 2 }; + let deep = q + .saturating_mul(deep_points) + .saturating_mul(widths.deep_point_rows) + .saturating_mul(FRI_COST_WEIGHTS.xalu); + trees.saturating_add(chain).saturating_add(deep) +} + +/// The per-table `auto` rule: one row iff it is STRICTLY cheaper than row pairs +/// under [`table_openings_cost_q`] (a tie keeps today's layout). +pub fn one_row_is_cheaper( + widths: &TableWidths, + options: &ProofOptions, + lde_log: u32, + blowup_log: u32, +) -> bool { + table_openings_cost_q(widths, options, lde_log, blowup_log, true) + < table_openings_cost_q(widths, options, lde_log, blowup_log, false) +} + +/// The leaf layout of a table with committed `widths` over an LDE of +/// `2^lde_log` rows (blowup `2^blowup_log`) under `options`' format. +pub fn resolve_leaf_layout( + widths: &TableWidths, + options: &ProofOptions, + lde_log: u32, + blowup_log: u32, +) -> LeafLayout { + match options.format.one_row { + OneRowMode::Off => LeafLayout::RowPair, + OneRowMode::On => LeafLayout::Row, + OneRowMode::Auto => { + LeafLayout::from_one_row(one_row_is_cheaper(widths, options, lde_log, blowup_log)) + } + } +} + +/// ★ The leaf layout of `air`'s proof over a trace of `trace_length` rows — +/// what the prover and the host verifier both call. The format comes from +/// `air.options()` (a verifier-side constant), the widths from the AIR, and +/// the length from the trace (the verifier's `proof.trace_length()`, the same +/// value its FRI layout already trusts). +pub fn table_leaf_layout( + air: &dyn AIR, + trace_length: usize, +) -> LeafLayout +where + F: IsFFTField + IsSubFieldOf + Send + Sync, + E: IsField + Send + Sync, +{ + let options = air.options(); + if options.format.one_row == OneRowMode::Off { + return LeafLayout::RowPair; + } + let blowup = options.blowup_factor as usize; + let lde_log = (trace_length.saturating_mul(blowup)).trailing_zeros(); + let blowup_log = blowup.trailing_zeros(); + resolve_leaf_layout( + &TableWidths::of(air, trace_length), + options, + lde_log, + blowup_log, + ) +} diff --git a/crypto/stark/src/lib.rs b/crypto/stark/src/lib.rs index 6f8e7c82e..c79b64733 100644 --- a/crypto/stark/src/lib.rs +++ b/crypto/stark/src/lib.rs @@ -1,15 +1,12 @@ -// `StorageMode::Disk` uses `memmap2`, which does not build on wasm32. -// Fail at the crate root rather than as a transitive memmap2 error. -#[cfg(all(target_arch = "wasm32", feature = "disk-spill"))] -compile_error!("the `disk-spill` feature requires memmap2, which does not compile on wasm32"); - #[cfg(feature = "debug-checks")] pub mod bus_debug; +pub mod commit_ood; pub mod commitment; pub mod constraint_ir; pub mod constraints; pub mod context; pub mod debug; +pub mod device_set; pub mod domain; #[cfg(any(test, feature = "test-utils"))] pub mod examples; @@ -20,17 +17,30 @@ pub mod gpu_lde; pub mod grinding; #[cfg(feature = "instruments")] pub mod instruments; +pub mod leaf_layout; #[cfg(feature = "cuda")] pub mod logup_gpu; pub mod lookup; +pub mod merkle_caps; +pub mod multilinear_air; +pub mod multilinear_block; +pub mod multilinear_logup; +pub mod multilinear_table; +// `narrow` is #1013's packed-trace form, imported byte-identical (#1013 is +// its one source of truth). Some of its parts serve #1013's prover only. +#[allow(dead_code)] +pub mod narrow; pub mod ood; pub(crate) mod par; pub mod profile_markers; pub mod proof; +pub mod prove_split; pub mod prover; pub mod r4_denoms; -#[cfg(feature = "disk-spill")] -pub mod storage_mode; +pub mod regen; +pub mod residency_mode; +#[cfg(all(feature = "cuda", any(test, feature = "test-utils")))] +pub mod s2_device_parity; pub mod table; pub mod trace; pub mod traits; diff --git a/crypto/stark/src/lookup.rs b/crypto/stark/src/lookup.rs index ceda5417a..772719e1b 100644 --- a/crypto/stark/src/lookup.rs +++ b/crypto/stark/src/lookup.rs @@ -540,6 +540,18 @@ pub enum LinearTerm { Constant(i64), } +impl LinearTerm { + /// The main column this term reads, if it reads one. + pub(crate) fn columns_read(&self, out: &mut Vec) { + match self { + LinearTerm::Column { column, .. } | LinearTerm::ColumnUnsigned { column, .. } => { + out.push(*column) + } + LinearTerm::Constant(_) => {} + } + } +} + /// A value that contributes to the bus fingerprint. /// /// A `BusValue` produces 1, 2, or 4 bus elements for the fingerprint depending @@ -691,6 +703,21 @@ impl BusValue { /// /// # Returns /// Vector of combined bus elements (length = num_bus_elements()) + /// The main columns this value reads, appended to `out`. + pub(crate) fn columns_read(&self, out: &mut Vec) { + match self { + BusValue::Packed { + start_column, + packing, + } => out.extend(*start_column..*start_column + packing.num_columns()), + BusValue::Linear(terms) => { + for term in terms { + term.columns_read(out); + } + } + } + } + pub fn combine_from FieldElement>( &self, get_column: F, @@ -810,6 +837,107 @@ impl BusValue { /// table's base-field transition constraints, and the framework appends the /// LogUp constraints (generated from [`Self::logup`]) after them. One body /// serves the compiled prover folder, the verifier folder, and IR capture. +/// A preprocessed table's commitment, computed when someone asks for it. +/// +/// The univariate path compares it against the proof's root and so always +/// does; the multilinear one has no separate root and checks the claimed +/// openings against [`precomputed_columns`] instead, so on that path nobody +/// ever asks. On a real program the ones that are not compiled-in constants — +/// the ELF's data pages and its instruction table — are an LDE and a Merkle +/// tree each, and there are two dozen of them. +/// +/// [`precomputed_columns`]: crate::traits::AIR::precomputed_columns +/// +/// # One root per leaf layout (S2) +/// +/// The root depends on the trace trees' leaf layout +/// ([`crate::leaf_layout::LeafLayout`]), so a commitment carries a separate, +/// separately cached source for the one-row layout. [`get`](Self::get) is +/// today's (row-pair) root, unchanged; [`get_for`](Self::get_for) serves +/// either and returns `None` for a layout this commitment has no source for +/// (the prover then refuses and the verifier rejects; never a silent recompute). +#[derive(Clone)] +pub struct LazyCommitment { + value: std::sync::Arc>, + #[allow(clippy::type_complexity)] + build: std::sync::Arc crate::config::Commitment + Send + Sync>, + /// The one-row root: `None` = no source (every constructor but + /// [`with_one_row`](Self::with_one_row)). + #[allow(clippy::type_complexity)] + one_row: Option<( + std::sync::Arc>>, + std::sync::Arc Option + Send + Sync>, + )>, +} + +impl std::fmt::Debug for LazyCommitment { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LazyCommitment") + .field("computed", &self.value.get().is_some()) + .field("one_row_source", &self.one_row.is_some()) + .finish() + } +} + +impl LazyCommitment { + /// One that is already known — a compiled-in constant, or a caller's. + pub fn ready(value: crate::config::Commitment) -> Self { + let cell = std::sync::OnceLock::new(); + let _ = cell.set(value); + Self { + value: std::sync::Arc::new(cell), + build: std::sync::Arc::new(|| [0u8; 32]), + one_row: None, + } + } + + /// One that costs something, computed on the first [`get`](Self::get) and + /// shared by every clone of the AIR from then on. + pub fn deferred(build: impl Fn() -> crate::config::Commitment + Send + Sync + 'static) -> Self { + Self { + value: std::sync::Arc::new(std::sync::OnceLock::new()), + build: std::sync::Arc::new(build), + one_row: None, + } + } + + /// This commitment plus a source for the ONE-ROW layout's root, computed + /// on the first [`get_for`](Self::get_for)`(Row)` and cached like the + /// row-pair one. The source returns `None` when it has no root for that + /// layout (e.g. a static table with no one-row entry): a hard miss, never + /// a fallback to the row-pair root. + pub fn with_one_row( + mut self, + build: impl Fn() -> Option + Send + Sync + 'static, + ) -> Self { + self.one_row = Some(( + std::sync::Arc::new(std::sync::OnceLock::new()), + std::sync::Arc::new(build), + )); + self + } + + /// Today's (row-pair) root. + pub fn get(&self) -> crate::config::Commitment { + *self.value.get_or_init(|| (self.build)()) + } + + /// The root under `layout`; `None` when this commitment has no source for + /// it. + pub fn get_for( + &self, + layout: crate::leaf_layout::LeafLayout, + ) -> Option { + match layout { + crate::leaf_layout::LeafLayout::RowPair => Some(self.get()), + crate::leaf_layout::LeafLayout::Row => { + let (cell, build) = self.one_row.as_ref()?; + *cell.get_or_init(|| build()) + } + } + } +} + pub struct AirWithBuses< F: IsFFTField + IsSubFieldOf + IsPrimeField + Send + Sync, E: IsField + Send + Sync, @@ -838,12 +966,27 @@ pub struct AirWithBuses< /// program (16-25K nodes on the big tables) per epoch/shard instance. constraint_program: std::sync::OnceLock>>, + /// A build-time program supplied via [`Self::with_precaptured`], if any. + /// + /// Kept separate from `constraint_program` rather than pre-filling that + /// `OnceLock`: `precaptured_constraint_program()` must answer "was one + /// SUPPLIED", not "has one been materialized by any means". Sharing the + /// cell would make a capture triggered by an earlier prover call look like + /// a build-time artifact. + precaptured_program: Option>, auxiliary_trace_build_data: AuxiliaryTraceBuildData, boundary_constraint_builder: PhantomData<(B, PI)>, - /// Commitment to precomputed columns (if this is a preprocessed table) - preprocessed_commitment: Option, + /// Commitment to precomputed columns (if this is a preprocessed table), + /// computed on demand — see [`LazyCommitment`]. + preprocessed_commitment: Option, /// Number of precomputed columns (columns 0..n are precomputed, rest are multiplicities) num_precomputed_cols: Option, + /// Builds the precomputed columns on demand. Only the multilinear path asks + /// for them, and only on the verifying side, so they are generated rather + /// than carried — BITWISE's are 2^20 rows. + #[allow(clippy::type_complexity)] + precomputed_columns: + Option Vec>> + Send + Sync>>, /// Optional name for debug output (per-table bus sum tracking) name: Option, /// Maximum number of bus elements across all interactions. @@ -874,10 +1017,12 @@ impl< meta: self.meta.clone(), num_base: self.num_base, constraint_program: self.constraint_program.clone(), + precaptured_program: self.precaptured_program.clone(), auxiliary_trace_build_data: self.auxiliary_trace_build_data.clone(), boundary_constraint_builder: PhantomData, - preprocessed_commitment: self.preprocessed_commitment, + preprocessed_commitment: self.preprocessed_commitment.clone(), num_precomputed_cols: self.num_precomputed_cols, + precomputed_columns: self.precomputed_columns.clone(), name: self.name.clone(), max_bus_elements: self.max_bus_elements, } @@ -960,10 +1105,12 @@ impl< meta, num_base, constraint_program: std::sync::OnceLock::new(), + precaptured_program: None, auxiliary_trace_build_data, boundary_constraint_builder: PhantomData, preprocessed_commitment: None, num_precomputed_cols: None, + precomputed_columns: None, name: None, max_bus_elements, } @@ -986,15 +1133,103 @@ impl< /// .with_preprocessed(bitwise::preprocessed_commitment(), bitwise::NUM_PRECOMPUTED_COLS); /// ``` pub fn with_preprocessed( - mut self, + self, commitment: crate::config::Commitment, num_precomputed_cols: usize, + ) -> Self { + self.with_lazy_preprocessed(LazyCommitment::ready(commitment), num_precomputed_cols) + } + + /// The same for a commitment nobody may end up needing. + pub fn with_lazy_preprocessed( + mut self, + commitment: LazyCommitment, + num_precomputed_cols: usize, ) -> Self { self.preprocessed_commitment = Some(commitment); self.num_precomputed_cols = Some(num_precomputed_cols); self } + /// Give this AIR's preprocessed commitment a ONE-ROW (S2) root: `root` is + /// what [`AIR::precomputed_commitment_for`](crate::traits::AIR::precomputed_commitment_for) + /// returns for [`LeafLayout::Row`](crate::leaf_layout::LeafLayout::Row) + /// (`None` = a hard miss). A no-op on an AIR that is not preprocessed. + pub fn with_one_row_commitment(mut self, root: Option) -> Self { + if let Some(c) = self.preprocessed_commitment.take() { + self.preprocessed_commitment = Some(c.with_one_row(move || root)); + } + self + } + + /// Supply a constraint program captured at BUILD time, so this AIR never + /// has to capture one. + /// + /// This is the guest-safe half of the constraint-program story: with a + /// program supplied, both [`AIR::constraint_program`] and + /// [`AIR::precaptured_constraint_program`] hand it back without running the + /// hash-consing capture, which is what makes a constraint program usable on + /// a verify/recursion path at all. + /// + /// The caller is responsible for the program actually being this AIR's. + /// [`ConstraintArtifact::validate_against`] rejects the shape-level + /// mismatches (wrong table, stale widths, changed exemptions); it cannot + /// detect an edit that changes a constraint's arithmetic without changing + /// any shape, which is what the build-time drift test is for. + /// + /// [`ConstraintArtifact::validate_against`]: + /// crate::constraint_ir::ConstraintArtifact::validate_against + pub fn with_precaptured( + mut self, + program: crate::constraint_ir::ConstraintProgram, + ) -> Self { + assert_eq!( + program.roots.len(), + self.meta.len(), + "pre-captured program has {} roots but this AIR has {} transition constraints", + program.roots.len(), + self.meta.len() + ); + assert_eq!( + program.num_base, self.num_base, + "pre-captured program declares num_base {} but this AIR has {}", + program.num_base, self.num_base + ); + self.precaptured_program = Some(program); + self + } + + /// The same, plus a generator for the columns themselves. + /// + /// Needed by the multilinear path, which checks the claimed openings of the + /// precomputed columns instead of comparing a commitment. Without it that + /// path cannot tell a real preprocessed table from a forged one. + pub fn with_preprocessed_columns( + self, + commitment: crate::config::Commitment, + num_precomputed_cols: usize, + columns: std::sync::Arc Vec>> + Send + Sync>, + ) -> Self { + self.with_lazy_preprocessed_columns( + LazyCommitment::ready(commitment), + num_precomputed_cols, + columns, + ) + } + + /// The same with the commitment deferred: the multilinear path checks the + /// columns and never forces it. + pub fn with_lazy_preprocessed_columns( + self, + commitment: LazyCommitment, + num_precomputed_cols: usize, + columns: std::sync::Arc Vec>> + Send + Sync>, + ) -> Self { + let mut air = self.with_lazy_preprocessed(commitment, num_precomputed_cols); + air.precomputed_columns = Some(columns); + air + } + /// Set a debug name for this AIR (for per-table bus sum tracking). /// /// When set, debug output will show bus sums prefixed with this name, @@ -1060,15 +1295,21 @@ where self.max_bus_elements } - fn composition_poly_degree_bound(&self, trace_length: usize) -> usize { - // Only the per-table MAX degree is consumed. Base constraints declare it - // once via `ConstraintSet::max_degree()`; the framework's LogUp - // constraints contribute their own known max (batched terms degree 3, - // accumulator `1 + absorbed`). - let max_degree = self - .constraint_set + fn bus_interactions(&self) -> &[BusInteraction] { + &self.auxiliary_trace_build_data.interactions + } + + fn max_constraint_degree(&self) -> usize { + // Base constraints declare their max once via `ConstraintSet::max_degree()`; + // the framework's LogUp constraints contribute their own known max + // (batched terms degree 3, accumulator `1 + absorbed`). + self.constraint_set .max_degree() - .max(logup_max_degree(&self.logup)); + .max(logup_max_degree(&self.logup)) + } + + fn composition_poly_degree_bound(&self, trace_length: usize) -> usize { + let max_degree = self.max_constraint_degree(); // The composition polynomial is the constraint QUOTIENT H = Σ βᵢ·Cᵢ/Zᵢ. Its degree is // deg(Cᵢ) − deg(Zᵢ) = (max_degree−1)·N − max_degree + eᵢ, so with the end-exemptions // eᵢ < max_degree (the max-degree LogUp constraints have eᵢ = 0) it fits in @@ -1124,9 +1365,15 @@ where fn constraint_program( &self, ) -> &crate::constraint_ir::ConstraintProgram { - // Lazily captured once (prover/GPU/tests only — the verify path never - // calls this). Runs the table set AND the LogUp emission through one - // CaptureBuilder, matching the folder emission order/indexing exactly. + // A build-time program, if one was supplied, short-circuits capture + // entirely. + if let Some(prog) = &self.precaptured_program { + return prog; + } + // Otherwise lazily captured once (prover/GPU/tests only — the verify + // path never calls this). Runs the table set AND the LogUp emission + // through one CaptureBuilder, matching the folder emission + // order/indexing exactly. self.constraint_program .get_or_init(|| { let mut cb = crate::constraints::builder::CaptureBuilder::::new(); @@ -1138,6 +1385,14 @@ where .as_ref() } + fn precaptured_constraint_program( + &self, + ) -> Option<&crate::constraint_ir::ConstraintProgram> { + // Deliberately NOT `constraint_program.get()`: only a program supplied + // at build time counts, never one a prover run happened to capture. + self.precaptured_program.as_ref() + } + fn build_auxiliary_trace( &self, trace: &mut TraceTable, @@ -1182,8 +1437,8 @@ where // Without `parallel`: sequential over pairs, sequential over rows. let interactions = &self.auxiliary_trace_build_data.interactions; - // GPU-resident aux build (Goldilocks + ext3, not disk-spill, not - // debug-checks): build the aux columns on device and keep them resident + // GPU-resident aux build (Goldilocks + ext3, not debug-checks): + // build the aux columns on device and keep them resident // for the aux LDE (no term-column download). Returns the table // contribution; the host set_aux + CPU accumulate below are skipped. #[cfg(all(feature = "cuda", not(feature = "debug-checks")))] @@ -1353,7 +1608,29 @@ where } fn precomputed_commitment(&self) -> crate::config::Commitment { - self.preprocessed_commitment.unwrap_or([0u8; 32]) + self.preprocessed_commitment + .as_ref() + .map(LazyCommitment::get) + .unwrap_or([0u8; 32]) + } + + fn precomputed_commitment_for( + &self, + layout: crate::leaf_layout::LeafLayout, + ) -> Option { + match &self.preprocessed_commitment { + Some(c) => c.get_for(layout), + // Not preprocessed: the row-pair answer is the trait's zero root + // (never compared); there is no one-row root to give. + None => (!layout.is_one_row()).then_some([0u8; 32]), + } + } + + fn precomputed_columns(&self) -> Vec>> { + self.precomputed_columns + .as_ref() + .map(|build| build()) + .unwrap_or_default() } } @@ -1411,10 +1688,25 @@ pub enum Multiplicity { } impl Multiplicity { + /// The main columns this expression reads, appended to `out`. + pub(crate) fn columns_read(&self, out: &mut Vec) { + match self { + Multiplicity::One => {} + Multiplicity::Column(col) | Multiplicity::Negated(col) => out.push(*col), + Multiplicity::Sum(a, b) | Multiplicity::Diff(a, b) => out.extend([*a, *b]), + Multiplicity::Sum3(a, b, c) => out.extend([*a, *b, *c]), + Multiplicity::Linear(terms) => { + for term in terms { + term.columns_read(out); + } + } + } + } + /// Evaluate the multiplicity expression to a field element. `get_col(i)` /// must return the value of main column `i` at the row being evaluated. #[inline] - fn evaluate_with(&self, get_col: G) -> FieldElement + pub(crate) fn evaluate_with(&self, get_col: G) -> FieldElement where F: IsField, G: Fn(usize) -> FieldElement, @@ -1490,6 +1782,23 @@ pub struct BusInteraction { } impl BusInteraction { + /// The main columns this interaction reads — its multiplicity's and its + /// values' — sorted and deduplicated. + /// + /// Everything here is affine in the columns, so a column nobody reads has + /// coefficient zero: this is what a caller recovering that affine form has + /// to look at, and the rest of the table is not worth asking about. + pub fn columns_read(&self) -> Vec { + let mut columns = Vec::new(); + self.multiplicity.columns_read(&mut columns); + for value in &self.values { + value.columns_read(&mut columns); + } + columns.sort_unstable(); + columns.dedup(); + columns + } + /// Creates a new table interaction. /// /// # Arguments @@ -2481,12 +2790,12 @@ mod logup_single_source_tests { for i in 0..n_rows { let mut row_sum = Fp3::zero(); for col in &term_columns { - row_sum = row_sum + &col[i]; + row_sum += col[i]; } let acc_i = *trace.get_aux(i, acc_col_idx); let acc_next = *trace.get_aux((i + 1) % n_rows, acc_col_idx); - let lhs = (acc_next - acc_i) * &n_fe; - let rhs = row_sum * &n_fe - &l; + let lhs = (acc_next - acc_i) * n_fe; + let rhs = row_sum * n_fe - l; assert_eq!(lhs, rhs, "forward circular recurrence broken at row {i}"); } } diff --git a/crypto/stark/src/merkle_caps.rs b/crypto/stark/src/merkle_caps.rs new file mode 100644 index 000000000..3a6222ed9 --- /dev/null +++ b/crypto/stark/src/merkle_caps.rs @@ -0,0 +1,291 @@ +//! Merkle caps of a univariate STARK proof (lever S1). +//! +//! Every tree of a proof is opened once per query: the trace trees (main, +//! precomputed, aux), the composition tree and each committed FRI layer. Under +//! a cap policy a tree of depth `D` gets a height-`c` cap +//! (`CapPolicy::height(num_queries, D)`); its `2^c` cap nodes ride at the end +//! of the authentication path of the tree's FIRST opening in proof order (the +//! "owner path"), and every path of that tree is cut to `D − c` siblings. +//! +//! [`StarkCaps`] is the one place the heights are computed from public shape +//! data (the policy, the query count, `log2(lde)` and the committed FRI layer +//! count). The prover embeds with it and the verifier checks with it, so the +//! split point of every path is a verifier constant, never read from a proof. +//! +//! [`TreeCheck`] is the verifier's per-tree check: built ONCE per tree (the +//! owner path's length and its cap-to-root check), then used for every query. +//! At `c = 0` it never touches the owner opening and is exactly the uncapped +//! exact-length check, so the legacy format verifies the same bytes. + +use crypto::merkle_tree::cap::{CapPolicy, CappedRoot}; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; + +use crate::config::Commitment; + +/// The cap height of every tree of one table's proof. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StarkCaps { + /// Depth of the trace, precomputed, aux and composition trees. + pub trace_depth: usize, + /// Cap height of those four trees (they share depth and opening count). + pub trace: usize, + /// Depth of committed FRI layer `i`. + pub fri_depths: Vec, + /// Cap height of committed FRI layer `i`. + pub fri: Vec, +} + +impl StarkCaps { + /// Depth of the trace, precomputed, aux and composition trees: a leaf is a + /// row PAIR, so `lde / 2` leaves and `log2(lde) − 1` levels (0 for a + /// two-point LDE, where the leaf hash is the root). + pub fn trace_tree_depth(lde_log: usize) -> usize { + lde_log.saturating_sub(1) + } + + /// Depth of committed FRI layer `i`: it holds `lde / 2^(i+1)` values in + /// pair leaves, so `log2(lde) − i − 2` levels. + pub fn fri_layer_depth(lde_log: usize, layer: usize) -> usize { + lde_log.saturating_sub(layer + 2) + } + + /// The heights for a proof with `num_queries` queries over an LDE of + /// `2^lde_log` points and `num_committed` committed FRI layers of today's + /// PAIR layout (layer `i` is `log2(lde) − i − 2` deep). Every tree is + /// opened `num_queries` times. A proof under a fold schedule + /// (`fri = dp`) has other layer depths: use [`Self::for_options`]. + pub fn new( + policy: CapPolicy, + num_queries: usize, + lde_log: usize, + num_committed: usize, + ) -> Self { + let trace_depth = Self::trace_tree_depth(lde_log); + let fri_depths: Vec = (0..num_committed) + .map(|i| Self::fri_layer_depth(lde_log, i)) + .collect(); + let fri = fri_depths + .iter() + .map(|&d| policy.height(num_queries, d)) + .collect(); + Self { + trace_depth, + trace: policy.height(num_queries, trace_depth), + fri_depths, + fri, + } + } + + /// The heights of a table proved under `options` over an LDE of + /// `2^lde_log` points, with the table's RESOLVED leaf layout `one_row` + /// (the same resolution `FriFoldLayout::for_options` takes): the trace + /// trees are the layout's depth and the committed FRI layers are the ones + /// the proof format's fold layout commits, so under a fold schedule + /// (`fri = dp`) layer `j` is `layer_depth(j)` deep, not `log2(lde) − j − 2`. + /// + /// This is the one public entry point the in-guest verifier checks its own + /// cap heights against (row pairs only there: `one_row = false`). `Err` + /// for a format that cannot be laid out. + pub fn for_options( + options: &crate::proof::options::ProofOptions, + lde_log: usize, + one_row: bool, + ) -> Result { + Self::for_options_with( + options, + lde_log, + one_row, + crate::config::TreeGeometry::BINARY, + ) + } + + /// [`Self::for_options`] for trees of `geometry` (a configuration's arity + /// and arity-4 cap, `TreeGeometry::of::()`): the heights the prover and + /// the host verifier of that configuration lay out. At + /// `TreeGeometry::BINARY` it is [`Self::for_options`]. + pub fn for_options_with( + options: &crate::proof::options::ProofOptions, + lde_log: usize, + one_row: bool, + geometry: crate::config::TreeGeometry, + ) -> Result { + let blowup_log = (options.blowup_factor as u32).trailing_zeros(); + let layout = crate::fri::terminal::FriFoldLayout::for_options( + lde_log as u32, + blowup_log, + options, + one_row, + )?; + Ok(Self::from_layout_arity( + geometry.cap_policy(options.format.merkle_cap), + options.fri_number_of_queries, + lde_log, + &layout, + geometry.arity, + )) + } + + /// The heights over an explicit FRI fold layout (the prover's and the + /// verifier's route: each holds the layout it built from the options). + /// The trace trees take the layout's leaf layout (row pairs: `log2(lde) − + /// 1`; one row: `log2(lde)`) and committed layer `j` is + /// `layout.layer_depth(lde_log, j)` deep: `log2(lde) − j − 2` under the + /// all-ones row-pair schedule (so this is [`Self::new`] there), the group + /// tree's depth under any other. For trees of `arity` children per node + /// the depths stay binary (`log2(leaves)`) and the heights count that + /// arity's levels ([`Self::with_depths_arity`]). + pub(crate) fn from_layout_arity( + policy: CapPolicy, + num_queries: usize, + lde_log: usize, + layout: &crate::fri::terminal::FriFoldLayout, + arity: usize, + ) -> Self { + Self::with_depths_arity( + policy, + num_queries, + crate::leaf_layout::LeafLayout::from_one_row(layout.one_row).tree_depth(lde_log), + layout.layer_depths(lde_log as u32), + arity, + ) + } + + /// The cap height of one tree over `2^depth` leaves of `arity` children + /// per node, opened `num_queries` times: `policy.height` over the tree's + /// levels, and at arity 4 no taller than a cap of `2^MAX_CAP_HEIGHT` + /// nodes. Prover and verifier both call this. + pub fn tree_cap_height( + policy: CapPolicy, + num_queries: usize, + depth: usize, + arity: usize, + ) -> usize { + let levels = crypto::merkle_tree::cap::tree_levels(depth, arity); + let c = policy.height(num_queries, levels); + if arity == 4 { + c.min(crypto::merkle_tree::cap::MAX_CAP_HEIGHT / 2) + } else { + c + } + } + + /// The heights for trace trees of depth `trace_depth` and committed FRI + /// layers of depths `fri_depths`, every tree opened `num_queries` times. + /// + /// The general form of [`Self::new`], for any leaf layout and FRI + /// schedule: the caller passes the depths its layout implies + /// ([`crate::leaf_layout::LeafLayout::tree_depth`] and the FRI layout's + /// per-layer depths). At row pairs and the all-ones schedule those are + /// exactly [`Self::new`]'s. + pub fn with_depths( + policy: CapPolicy, + num_queries: usize, + trace_depth: usize, + fri_depths: Vec, + ) -> Self { + Self::with_depths_arity(policy, num_queries, trace_depth, fri_depths, 2) + } + + /// [`Self::with_depths`] for trees of `arity` children per node + /// ([`Self::tree_cap_height`]). At arity 2 it is exactly + /// [`Self::with_depths`]. + pub fn with_depths_arity( + policy: CapPolicy, + num_queries: usize, + trace_depth: usize, + fri_depths: Vec, + arity: usize, + ) -> Self { + let fri = fri_depths + .iter() + .map(|&d| Self::tree_cap_height(policy, num_queries, d, arity)) + .collect(); + Self { + trace_depth, + trace: Self::tree_cap_height(policy, num_queries, trace_depth, arity), + fri_depths, + fri, + } + } + + /// True when some tree has a cap (`c > 0`). + pub fn any(&self) -> bool { + self.trace > 0 || self.fri.iter().any(|&c| c > 0) + } +} + +/// The verifier's check for one tree: its authenticated cap, plus the owner +/// opening's own siblings when the tree is capped. +#[derive(Clone, Copy, Debug)] +pub struct TreeCheck<'a> { + capped: CappedRoot<'a, Commitment>, + /// `Some` iff `c > 0`: the owner path minus its cap. Query 0 of this tree + /// is checked with these siblings. + owner_siblings: Option<&'a [Commitment]>, +} + +impl<'a> TreeCheck<'a> { + /// Build the check of one tree of depth `depth` and cap height + /// `cap_height` against `root`. + /// + /// At `c = 0` the owner opening is never read (`owner_path` is not + /// called): the check is the exact-length full-path check, and the default + /// format touches no index a count guard has not covered. At `c > 0`, + /// `owner_path` must return the tree's first opening's path (`None` when + /// the proof has none, which rejects); its length must be exactly + /// `D − c + 2^c` (at arity 4, `3(⌈D/2⌉ − c)` + the cap's + /// `crypto::merkle_tree::cap::cap_len`) and its cap must hash to `root`. + pub fn build>( + root: &'a Commitment, + depth: usize, + cap_height: usize, + owner_path: impl FnOnce() -> Option<&'a [Commitment]>, + ) -> Option { + if cap_height == 0 { + return Some(Self { + capped: CappedRoot::uncapped(root, depth), + owner_siblings: None, + }); + } + let (capped, siblings) = + CappedRoot::from_owner::(root, owner_path()?, depth, cap_height)?; + Some(Self { + capped, + owner_siblings: Some(siblings), + }) + } + + /// Check query `query`'s opening of this tree: `path` as the proof carries + /// it, the transcript's leaf `index`, and the leaf hash of the opened + /// values. Query 0 of a capped tree is the owner: its siblings are the + /// owner path minus the cap (split once in [`build`](Self::build)); every + /// other query's path must be exactly `D − c` long. + pub fn verify>( + &self, + query: usize, + path: &[Commitment], + index: usize, + leaf_hash: Commitment, + ) -> bool { + let siblings = match (query, self.owner_siblings) { + (0, Some(owner)) => owner, + _ => path, + }; + self.capped.verify::(siblings, index, leaf_hash) + } + + pub fn cap_height(&self) -> usize { + self.capped.cap_height() + } +} + +/// The checks of every tree of one table's proof. +#[derive(Clone, Debug)] +pub struct TableTreeChecks<'a> { + pub main: TreeCheck<'a>, + pub precomputed: Option>, + pub aux: Option>, + pub composition: TreeCheck<'a>, + /// One per committed FRI layer, in layer order. + pub fri: Vec>, +} diff --git a/crypto/stark/src/multilinear_air.rs b/crypto/stark/src/multilinear_air.rs new file mode 100644 index 000000000..f40f1109c --- /dev/null +++ b/crypto/stark/src/multilinear_air.rs @@ -0,0 +1,1889 @@ +//! Bridges an AIR's constraint IR to the hypercube: one committed column per +//! distinct `(main, col)` trace read, `offset` as a cyclic shift *view* of that +//! column, and row domains as [`Selector`]s. +//! +//! Nothing here has to be taken on trust. A shifted factor gets no commitment +//! of its own — [`claim_reduce`] binds its value to the column it shifts — and +//! a selector gets none either: it is a *public* factor, which the verifier +//! evaluates through [`Selector::evaluate`] rather than reading out of a +//! commitment. Only the trace columns are committed, and they stay in the base +//! field they are; the sumcheck's factors are lifted for it. + +use math::field::{ + element::FieldElement, + traits::{IsField, IsSubFieldOf}, +}; +use multilinear::{ + Error as MlError, + claim_reduce::{self, FactorSource}, + constraint_argument::FactorKind, + mle::Mle, + poly::SumcheckPolynomial, + program::{Builder, Op as ProgramOp, Program}, + selector::Selector, +}; +use std::collections::BTreeMap; +use std::marker::PhantomData; + +use crate::constraint_ir::ir::{ConstraintProgram, Op}; +use crate::constraints::builder::ConstraintMeta; + +/// Identifies a trace read: main-vs-aux, frame-step offset, column. +/// +/// `row` is not part of the key — every table in this VM reads row 0 of each +/// frame step, which the IR interpreter asserts. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct LeafKey { + pub main: bool, + pub offset: u8, + pub col: u16, +} + +/// Identifies a committed column: the same read with its offset dropped. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct ColumnKey { + pub main: bool, + pub col: u16, +} + +impl LeafKey { + pub fn column(&self) -> ColumnKey { + ColumnKey { + main: self.main, + col: self.col, + } + } +} + +/// Which column each sumcheck factor reads and how many steps ahead — the +/// structure of [`TraceLeaves`] with none of the trace in it. +/// +/// One slot per distinct `(main, offset, col)` read, one column per distinct +/// `(main, col)`. A read at offset `k` is a **view**: the shifted table the +/// sumcheck folds is derived from the column, and the column is the only thing +/// that gets committed. +/// +/// The verifier holds one of these: every slot assignment comes from the +/// program, so both sides derive the same one and only the columns' values are +/// missing. Building it lives here once — if the two sides laid out slots +/// separately they could disagree, and every claim would be about the wrong +/// table. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LeafLayout { + /// Leaf -> factor slot, kept ordered so the layout is deterministic. + index: BTreeMap, + /// Column key -> index into the columns. + column_index: BTreeMap, + /// Column index -> which column it is, so materialization order is + /// explicit rather than a side effect of discovery order. + column_keys: Vec, + /// One entry per factor, in factor order. + sources: Vec, + num_vars: usize, +} + +impl LeafLayout { + /// Records every trace read in `program`. + pub fn build(program: &ConstraintProgram, num_vars: usize) -> Self + where + F: IsField, + E: IsField, + { + Self::build_live(program, &vec![true; program.nodes.len()], num_vars) + } + + /// The same, restricted to the reads a [`live_nodes`] mask keeps — so a + /// table's dropped LogUp constraints do not drag their auxiliary columns + /// in. + pub fn build_live( + program: &ConstraintProgram, + live: &[bool], + num_vars: usize, + ) -> Self + where + F: IsField, + E: IsField, + { + Self::build_live_over(program, live, num_vars, 0) + } + + /// The same, with the table's first `num_main_columns` registered up front. + /// + /// Which columns a table commits, and in **what order**, is then a function + /// of the table and not of the AIR that reads it: two AIRs over the same + /// trace — an epoch's local-to-global bookend and the cross-epoch one, say + /// — commit the same polynomial, which is the only way one proof can say to + /// another that it committed that table. + pub fn build_live_over( + program: &ConstraintProgram, + live: &[bool], + num_vars: usize, + num_main_columns: usize, + ) -> Self + where + F: IsField, + E: IsField, + { + let mut layout = Self { + index: BTreeMap::new(), + column_index: BTreeMap::new(), + column_keys: Vec::new(), + sources: Vec::new(), + num_vars, + }; + + for col in 0..num_main_columns { + layout.register_main(col as u16); + } + + for (id, op) in program.nodes.iter().enumerate() { + if !live.get(id).copied().unwrap_or(false) { + continue; + } + let Op::Var { + main, offset, col, .. + } = *op + else { + continue; + }; + layout.record(LeafKey { main, offset, col }); + } + + layout + } + + /// Assigns `key` a factor slot, and its column an index if this is the + /// first read of it. Returns the slot, which an already-recorded key keeps. + fn record(&mut self, key: LeafKey) -> usize { + if let Some(&slot) = self.index.get(&key) { + return slot; + } + let column = match self.column_index.get(&key.column()) { + Some(&i) => i, + None => { + let i = self.column_keys.len(); + self.column_keys.push(key.column()); + self.column_index.insert(key.column(), i); + i + } + }; + let slot = self.sources.len(); + self.index.insert(key, slot); + self.sources + .push(FactorSource::shifted(column, key.offset as usize)); + slot + } + + /// Registers main column `col`, read unshifted, as a factor — returning the + /// slot it already had if the constraints read it too. + /// + /// A bus interaction reads columns the constraints may not, and the two + /// must share one factor when they overlap: that is what makes a column + /// read by a constraint and by a fingerprint fold once. + pub fn register_main(&mut self, col: u16) -> usize { + self.record(LeafKey { + main: true, + offset: 0, + col, + }) + } + + pub fn num_vars(&self) -> usize { + self.num_vars + } + + /// The columns to commit, in the order they must be materialized. + pub fn column_keys(&self) -> &[ColumnKey] { + &self.column_keys + } + + /// How many columns get committed. + pub fn num_columns(&self) -> usize { + self.column_keys.len() + } + + /// What each factor reads. + pub fn sources(&self) -> &[FactorSource] { + &self.sources + } + + /// Number of sumcheck factors. + pub fn len(&self) -> usize { + self.sources.len() + } + + pub fn is_empty(&self) -> bool { + self.sources.is_empty() + } + + /// Factor slot of a trace read. + pub fn index_of(&self, key: &LeafKey) -> Option { + self.index.get(key).copied() + } + + /// Column slot of a trace read. + pub fn column_of(&self, key: &ColumnKey) -> Option { + self.column_index.get(key).copied() + } + + /// Fills the layout in, one call per column in [`column_keys`] order. + /// + /// [`column_keys`]: Self::column_keys + pub fn materialize( + self, + mut column: impl FnMut(ColumnKey) -> Vec>, + ) -> Result, MlError> { + let size = 1usize << self.num_vars; + let mut columns = Vec::with_capacity(self.column_keys.len()); + for &key in &self.column_keys { + let values = column(key); + if values.len() != size { + return Err(MlError::NotPowerOfTwo(values.len())); + } + columns.push(Mle::new(values)?); + } + Ok(TraceLeaves { + layout: self, + columns, + }) + } +} + +/// A [`LeafLayout`] with its columns materialized. +/// +/// `V` is the columns' value field: the trace's, which is the base one. +#[derive(Clone, Debug)] +pub struct TraceLeaves { + pub(crate) layout: LeafLayout, + pub(crate) columns: Vec>, +} + +impl TraceLeaves { + /// Materializes one MLE per distinct column in `program` and records the + /// offset each factor reads it at. + /// + /// `main_column` and `aux_column` return a column's values indexed by step; + /// both must return exactly `2^num_vars` entries. + pub fn build( + program: &ConstraintProgram, + num_vars: usize, + main_column: impl FnMut(u16) -> Vec>, + aux_column: impl FnMut(u16) -> Vec>, + ) -> Result + where + F: IsField + 'static, + E: IsField + 'static, + { + Self::build_live( + program, + &vec![true; program.nodes.len()], + num_vars, + main_column, + aux_column, + ) + } + + /// The same, restricted to the reads a [`live_nodes`] mask keeps. + pub fn build_live( + program: &ConstraintProgram, + live: &[bool], + num_vars: usize, + mut main_column: impl FnMut(u16) -> Vec>, + mut aux_column: impl FnMut(u16) -> Vec>, + ) -> Result + where + F: IsField + 'static, + E: IsField + 'static, + { + LeafLayout::build_live(program, live, num_vars).materialize(|key| { + if key.main { + main_column(key.col) + } else { + aux_column(key.col) + } + }) + } + + /// The slot assignment alone. + pub fn layout(&self) -> &LeafLayout { + &self.layout + } + + pub fn num_vars(&self) -> usize { + self.layout.num_vars + } + + /// The columns to commit. + pub fn columns(&self) -> &[Mle] { + &self.columns + } + + /// What each factor reads. + pub fn sources(&self) -> &[FactorSource] { + &self.layout.sources + } + + /// The factor tables, every shifted read materialized. + pub fn factors(&self) -> Result>, MlError> { + self.layout + .sources + .iter() + .map(|s| claim_reduce::materialize(&self.columns, s)) + .collect() + } + + /// Number of sumcheck factors. + pub fn len(&self) -> usize { + self.layout.len() + } + + pub fn is_empty(&self) -> bool { + self.layout.is_empty() + } + + /// Factor slot of a trace read. + pub fn index_of(&self, key: &LeafKey) -> Option { + self.layout.index_of(key) + } + + /// Column slot of a trace read. + pub fn column_of(&self, key: &ColumnKey) -> Option { + self.layout.column_of(key) + } + + /// Registers main column `col`, read unshifted, as a factor — returning the + /// slot it already had if the constraints read it too. `values` is only + /// consulted the first time the column appears. + pub fn register_main( + &mut self, + col: u16, + values: impl FnOnce() -> Vec>, + ) -> Result { + let slot = self.layout.register_main(col); + if self.layout.column_keys.len() > self.columns.len() { + let values = values(); + if values.len() != 1usize << self.layout.num_vars { + return Err(MlError::NotPowerOfTwo(values.len())); + } + self.columns.push(Mle::new(values)?); + } + Ok(slot) + } +} + +/// Uniform values an IR program may read: LogUp challenges, alpha powers and +/// the table offset. Constant across the whole trace, so they are scalars here +/// rather than polynomials. +#[derive(Clone, Debug)] +pub struct Uniforms { + pub rap_challenges: Vec>, + pub logup_alpha_powers: Vec>, + pub logup_table_offset: FieldElement, +} + +/// Hand-written so the field itself need not be `Default`. +impl Default for Uniforms { + fn default() -> Self { + Self { + rap_challenges: Vec::new(), + logup_alpha_powers: Vec::new(), + logup_table_offset: FieldElement::zero(), + } + } +} + +/// The nodes reachable from `roots`, marked in one reverse pass — the node list +/// is topologically ordered, so an operand always has a lower id. +/// +/// A real table's program carries its LogUp constraints after the base prefix. +/// The multilinear path replaces those with a bus, so their roots are dropped — +/// and their subtrees are a large part of the DAG and read auxiliary columns +/// and challenges this path does not have. Filtering by reachability is what +/// keeps them out. +pub fn live_nodes( + program: &ConstraintProgram, + roots: &[u32], +) -> Vec { + let mut live = vec![false; program.nodes.len()]; + for &root in roots { + live[root as usize] = true; + } + for (id, op) in program.nodes.iter().enumerate().rev() { + if !live[id] { + continue; + } + match *op { + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => { + live[a as usize] = true; + live[b as usize] = true; + } + Op::Neg(a) | Op::Embed(a) => live[a as usize] = true, + _ => {} + } + } + live +} + +/// Compiles the subgraph the roots reach into straight-line steps. +/// +/// Runs once, at build time, so the evaluator never touches the `BTreeMap`, a +/// dead node, or a slot it does not need. Returns the steps and, per root, the +/// step holding its value. +fn compile( + program: &ConstraintProgram, + live: &[bool], + leaf_index: &BTreeMap, + uniforms: &Uniforms, + roots: &[u32], +) -> Result<(Vec>, Vec), MlError> +where + F: IsSubFieldOf, + E: IsField, +{ + let mut steps = Vec::new(); + // Node id -> the step holding its value. `Embed` maps to its operand's, + // which is why this is a mapping and not just a running count. + let mut at = vec![u32::MAX; program.nodes.len()]; + let step_of = |at: &[u32], id: u32| -> Result { + let slot = at[id as usize]; + if slot == u32::MAX { + // An operand of a live node is live, so this cannot happen unless + // the node list stopped being topologically ordered. + return Err(MlError::UnknownPolynomial { + index: id as usize, + len: program.nodes.len(), + }); + } + Ok(slot) + }; + + for (id, op) in program.nodes.iter().enumerate() { + if !live.get(id).copied().unwrap_or(false) { + continue; + } + let step = match *op { + Op::ConstBase(idx) => Step::Fixed( + program.base_consts[idx as usize] + .clone() + .to_extension::(), + ), + Op::ConstExt(idx) => Step::Fixed(program.ext_consts[idx as usize].clone()), + Op::RapChallenge { idx } => Step::Fixed(uniforms.rap_challenges[idx as usize].clone()), + Op::AlphaPow { idx } => Step::Fixed(uniforms.logup_alpha_powers[idx as usize].clone()), + Op::TableOffset => Step::Fixed(uniforms.logup_table_offset.clone()), + Op::Var { + main, offset, col, .. + } => { + let key = LeafKey { main, offset, col }; + let slot = *leaf_index + .get(&key) + .expect("every Var leaf was materialized at build time"); + Step::Var(slot as u32) + } + Op::Add(a, b) => Step::Add(step_of(&at, a)?, step_of(&at, b)?), + Op::Sub(a, b) => Step::Sub(step_of(&at, a)?, step_of(&at, b)?), + Op::Mul(a, b) => Step::Mul(step_of(&at, a)?, step_of(&at, b)?), + Op::Neg(a) => Step::Neg(step_of(&at, a)?), + // A no-op on values: alias the operand rather than copy it. + Op::Embed(a) => { + at[id] = step_of(&at, a)?; + continue; + } + }; + at[id] = steps.len() as u32; + steps.push(step); + } + + let root_steps = roots + .iter() + .map(|&r| step_of(&at, r)) + .collect::, _>>()?; + Ok((steps, root_steps)) +} + +/// `beta^i` for `i` in `0..n` — the coefficients that batch a program's roots +/// into one constraint. +/// +/// Kept out of [`IrShape`] on purpose: `beta` is drawn once the trace is +/// committed, and a shape is built before that. +pub fn beta_powers(beta: &FieldElement, n: usize) -> Vec> { + let mut powers = Vec::with_capacity(n); + let mut acc = FieldElement::::one(); + for _ in 0..n { + powers.push(acc.clone()); + acc *= beta; + } + powers +} + +/// Per-node degree of an IR program, in the trace variables. +/// +/// Constants and uniforms are degree 0, a trace read is degree 1, products add +/// and sums take the max. Used to size the sumcheck round polynomials. +fn node_degrees(program: &ConstraintProgram) -> Vec { + let mut deg: Vec = Vec::with_capacity(program.nodes.len()); + for op in &program.nodes { + let d = match *op { + Op::ConstBase(_) + | Op::ConstExt(_) + | Op::RapChallenge { .. } + | Op::AlphaPow { .. } + | Op::TableOffset => 0, + Op::Var { .. } => 1, + Op::Add(a, b) | Op::Sub(a, b) => deg[a as usize].max(deg[b as usize]), + Op::Mul(a, b) => deg[a as usize] + deg[b as usize], + Op::Neg(a) | Op::Embed(a) => deg[a as usize], + }; + deg.push(d); + } + deg +} + +/// An AIR's constraints as one polynomial: `C = Σ_i beta^i · s_i(x) · C_i(x)`. +/// +/// Factors are the trace leaves followed by one table per distinct non-trivial +/// selector, in that order. +pub struct IrPolynomial { + shape: IrShape, + beta_powers: Vec>, + polys: Vec>, + layout: CommitLayout, +} + +/// What the argument needs from an [`IrPolynomial`]: the columns to commit, +/// where every factor's table comes from, and the public tables in factor +/// order. +/// +/// The columns are the trace, so base-field; the public tables are computed and +/// live where the challenges do. +#[derive(Clone, Debug)] +pub struct CommitLayout { + pub columns: Vec>, + pub kinds: Vec, + pub public_tables: Vec>, +} + +/// The constraint's *structure*, with no trace data in it. +/// +/// [`combine`](Self::combine) turns factor values into the batched constraint, +/// and needs nothing but this — so the verifier can hold one and rebuild +/// `C(point)` from values it learned through the commitment scheme, without +/// ever seeing a column. +#[derive(Clone)] +/// Owned: once the program is compiled there is nothing left to borrow from +/// it, and a shape that borrows nothing is one less lifetime to thread. +pub struct IrShape { + /// Compiling folds the base field away — constants are lifted once — so + /// nothing here holds an `F`, but the shape is still the shape of a program + /// over the tower and says so. + field: PhantomData, + /// The program compiled to straight-line steps. + steps: Vec>, + /// Each batched root's step, in `roots` order. + root_steps: Vec, + /// Roots to batch, in order. + roots: Vec, + /// Index into the factor list of each root's selector, or `None` when it + /// applies on every step and the multiplication can be skipped. + selector_of_root: Vec>, + /// The public factors, in factor order — what the verifier recomputes. + public_selectors: Vec, + degree: usize, + num_vars: usize, +} + +/// One step of the compiled constraint program. +/// +/// The IR is a DAG over every node the AIR emits, and evaluating it used to +/// mean walking that whole list per hypercube index: a `BTreeMap` probe for +/// each variable read, a pushed zero for each node the selected roots do not +/// reach, and a buffer the width of the entire program — tens of thousands of +/// extension elements on the precompile tables. This is the reachable subgraph +/// renumbered so operands are dense indices, resolved once. +/// +/// `Embed` leaves no step: it aliases its operand's. +#[derive(Clone, Debug)] +enum Step { + /// Known before the trace: a constant or a uniform. + Fixed(FieldElement), + /// A factor value, by slot. + Var(u32), + Add(u32, u32), + Sub(u32, u32), + Mul(u32, u32), + Neg(u32), +} + +impl IrPolynomial +where + F: IsSubFieldOf + 'static, + E: IsField + 'static, +{ + /// Batches every constraint in `program`, taking each one's row domain from + /// `meta`. + pub fn new( + program: &ConstraintProgram, + leaves: TraceLeaves, + uniforms: Uniforms, + beta: FieldElement, + meta: &[ConstraintMeta], + ) -> Result { + let roots: Vec = program.roots.clone(); + let selectors: Vec = meta + .iter() + .map(|m| Selector::except_last(m.end_exemptions)) + .collect(); + Self::with_roots(program, leaves, uniforms, beta, roots, &selectors) + } + + /// Batches every constraint with no row exemptions at all. + /// + /// Only correct for AIRs whose constraints really do hold on every step, + /// wrap included. + pub fn new_unselected( + program: &ConstraintProgram, + leaves: TraceLeaves, + uniforms: Uniforms, + beta: FieldElement, + ) -> Result { + let roots: Vec = program.roots.clone(); + let selectors = vec![Selector::ALL; roots.len()]; + Self::with_roots(program, leaves, uniforms, beta, roots, &selectors) + } + + /// Batches the listed roots with the matching selectors. + pub fn with_roots( + program: &ConstraintProgram, + leaves: TraceLeaves, + uniforms: Uniforms, + beta: FieldElement, + roots: Vec, + selectors: &[Selector], + ) -> Result { + let TraceLeaves { layout, columns } = leaves; + let (shape, kinds) = IrShape::build(program, &layout, uniforms, roots, selectors)?; + let beta_powers = beta_powers(&beta, shape.num_roots()); + let public_tables = shape.public_tables()?; + + let mut public = public_tables.iter().cloned(); + let polys = kinds + .iter() + .map(|kind| match kind { + // The sumcheck's factors share a field, so a base view is + // lifted for it. + FactorKind::Committed(s) => { + let view = claim_reduce::materialize(&columns, s)?; + Mle::new( + view.evals() + .iter() + .map(|v| v.clone().to_extension::()) + .collect(), + ) + } + FactorKind::Public => public.next().ok_or(MlError::EmptyPolynomial), + }) + .collect::, _>>()?; + + Ok(Self { + shape, + beta_powers, + polys, + layout: CommitLayout { + columns, + kinds, + public_tables, + }, + }) + } + + /// The structure alone, for the verifier. + pub fn shape(&self) -> &IrShape { + &self.shape + } + + /// The columns to commit and, for each sumcheck factor, where its table + /// comes from. Selectors are public factors, so they are not columns. + pub fn committed(&self) -> (&[Mle], &[FactorKind]) { + (&self.layout.columns, &self.layout.kinds) + } + + /// The whole layout, dropping the factor tables — the argument rebuilds + /// those from the columns and the public tables. + pub fn into_layout(self) -> CommitLayout { + self.layout + } + + /// The structure and the layout together, dropping the factor tables. + pub fn into_shape_and_layout(self) -> (IrShape, CommitLayout) { + (self.shape, self.layout) + } +} + +impl IrShape +where + F: IsSubFieldOf, + E: IsField + 'static, +{ + /// The batched constraint's structure, from the program and a slot + /// assignment alone. + /// + /// Returns the factor kinds alongside it: the trace leaves in slot order, + /// then one public factor per distinct non-trivial selector. **The verifier + /// builds this too** — the whole point of taking a [`LeafLayout`] rather + /// than a materialized [`TraceLeaves`] — so the two sides cannot disagree + /// on what a factor slot means. + pub fn build( + program: &ConstraintProgram, + leaves: &LeafLayout, + uniforms: Uniforms, + roots: Vec, + selectors: &[Selector], + ) -> Result<(Self, Vec), MlError> { + if selectors.len() != roots.len() { + return Err(MlError::VariableCountMismatch { + expected: roots.len(), + got: selectors.len(), + }); + } + let num_vars = leaves.num_vars; + let degrees = node_degrees(program); + let live = live_nodes(program, &roots); + + let mut kinds: Vec = leaves + .sources + .iter() + .copied() + .map(FactorKind::Committed) + .collect(); + + // One table per distinct non-trivial selector, shared across roots, and + // public: the verifier evaluates it in closed form instead of reading + // it out of a commitment. + let mut selector_slot: BTreeMap = BTreeMap::new(); + let mut selector_of_root = Vec::with_capacity(roots.len()); + let mut public_selectors = Vec::new(); + for selector in selectors { + if selector.is_trivial() { + selector_of_root.push(None); + continue; + } + let slot = match selector_slot.get(&selector.end_exemptions) { + Some(&i) => i, + None => { + let factor = kinds.len(); + kinds.push(FactorKind::Public); + public_selectors.push(*selector); + selector_slot.insert(selector.end_exemptions, factor); + factor + } + }; + selector_of_root.push(Some(slot)); + } + + // A selector is multilinear, so it costs exactly one degree. + let degree = roots + .iter() + .zip(selectors) + .map(|(&r, sel)| degrees[r as usize] + usize::from(!sel.is_trivial())) + .max() + .unwrap_or(0); + + let (steps, root_steps) = compile(program, &live, &leaves.index, &uniforms, &roots)?; + + Ok(( + Self { + field: PhantomData, + steps, + root_steps, + roots, + selector_of_root, + public_selectors, + degree, + num_vars, + }, + kinds, + )) + } + + /// The public factors' selectors, in the order they appear in the kinds. + /// + /// [`public_values`](Self::public_values) evaluates these and hands back + /// the answers; a caller that is EMITTING that evaluation as code needs the + /// selectors themselves, and [`public_tables`](Self::public_tables) is not + /// a substitute — it builds a `2^num_vars` table per selector, which is the + /// pass the closed form exists to avoid. + pub fn public_selectors(&self) -> &[Selector] { + &self.public_selectors + } + + /// The public factors' tables, in the order they appear in the kinds. + /// + /// Only the prover needs them: the verifier evaluates the same selectors in + /// closed form through [`public_values`](Self::public_values), which is why + /// building the structure does not build these. + pub fn public_tables(&self) -> Result>, MlError> { + self.public_selectors + .iter() + .map(|s| s.table(self.num_vars)) + .collect() + } +} + +impl IrShape +where + F: IsSubFieldOf, + E: IsField + 'static, +{ + pub fn degree(&self) -> usize { + self.degree + } + + pub fn num_vars(&self) -> usize { + self.num_vars + } + + /// How many roots are batched — the length `beta_powers` must have. + pub fn num_roots(&self) -> usize { + self.roots.len() + } + + /// The compiled DAG, in `multilinear::program`'s vocabulary. + /// + /// `Step` and `program::Op` are the same six variants over the same dense + /// operand numbering, so this is that conversion — stated once, and in the + /// crate that owns `Step`, rather than a second spelling of the DAG in + /// every consumer that has to walk it. `Step` itself stays private. + /// + /// Read together with [`root_steps`](Self::root_steps) and + /// [`selector_of_root`](Self::selector_of_root) by the recursion emitter, + /// which rebuilds [`combine`](Self::combine) with the batching coefficients + /// as RUNTIME wires: `program` interns them as `Fixed` steps + /// (`Builder::weighted_sum`), which is correct for a host that already has + /// `beta` and wrong for a verifier that draws it from a transcript. + pub fn steps_as_ops(&self) -> Vec> { + self.steps + .iter() + .map(|step| match *step { + Step::Fixed(ref c) => ProgramOp::Fixed(c.clone()), + Step::Var(i) => ProgramOp::Var(i), + Step::Add(a, b) => ProgramOp::Add(a, b), + Step::Sub(a, b) => ProgramOp::Sub(a, b), + Step::Mul(a, b) => ProgramOp::Mul(a, b), + Step::Neg(a) => ProgramOp::Neg(a), + }) + .collect() + } + + /// Each batched root's step index, in `roots` order — the steps + /// [`combine`](Self::combine) reads out of the DAG. + pub fn root_steps(&self) -> &[u32] { + &self.root_steps + } + + /// Each batched root's selector factor, or `None` where the root applies on + /// every step and the multiplication is skipped. + pub fn selector_of_root(&self) -> &[Option] { + &self.selector_of_root + } + + /// The batched constraint, given each factor's value at a point. + /// + /// `beta_powers` batches the roots and is **not** part of the structure: + /// the challenge behind it is drawn after the trace is committed, which a + /// layout built before there is a transcript cannot know. Size it with + /// [`num_roots`](Self::num_roots). + /// The zerocheck rule as straight-line code over the factor values: + /// [`combine`](Self::combine) times the weight at factor slot `weight`. + /// + /// The steps go in first and unchanged, so a step's index is its operand + /// number; the batched roots, their selectors and the weight follow. + pub fn program( + &self, + beta_powers: &[FieldElement], + weight: usize, + ) -> Result, MlError> { + if beta_powers.len() != self.roots.len() { + return Err(MlError::VariableCountMismatch { + expected: self.roots.len(), + got: beta_powers.len(), + }); + } + let mut builder = Builder::::new(); + for step in &self.steps { + let emitted = match *step { + Step::Fixed(ref c) => builder.fixed(c.clone()), + Step::Var(i) => builder.var(i as usize), + Step::Add(a, b) => builder.add(a, b), + Step::Sub(a, b) => builder.sub(a, b), + Step::Mul(a, b) => builder.mul(a, b), + Step::Neg(a) => builder.neg(a), + }; + debug_assert_eq!( + emitted as usize, + builder.len() - 1, + "a step's index is its operand number" + ); + } + let terms: Vec<(u32, FieldElement)> = self + .root_steps + .iter() + .zip(beta_powers) + .zip(&self.selector_of_root) + .map(|((&root, beta), selector)| { + let term = match selector { + Some(slot) => { + let s = builder.var(*slot); + builder.mul(root, s) + } + None => root, + }; + (term, beta.clone()) + }) + .collect(); + let sum = builder.weighted_sum(&terms); + let w = builder.var(weight); + let root = builder.mul(w, sum); + builder.finish(root) + } + + /// [`program`](Self::program) with each root folded into the running sum + /// as soon as its step is computed, rather than all of them at the end. + /// + /// The same polynomial, `weight · Σ β^i · sel_i · root_i`, summed in step + /// order rather than root order. What changes is how long a root stays + /// live. The sum at the end keeps every root's value until the last step, + /// so the program's live set — the slot file a device round keeps per + /// thread — is at least its number of roots, and on the widest precompile + /// that file is tens of kilobytes a thread: it caps a round's threads and + /// does not stay in cache (`thoughts/zf/gap2/fix2/I-GFS.md` §15). Folded in + /// early, a root dies at its term. + pub fn program_lean( + &self, + beta_powers: &[FieldElement], + weight: usize, + ) -> Result, MlError> { + if beta_powers.len() != self.roots.len() { + return Err(MlError::VariableCountMismatch { + expected: self.roots.len(), + got: beta_powers.len(), + }); + } + // The roots each step completes, so their terms follow it directly. + let mut completes: Vec> = vec![Vec::new(); self.steps.len()]; + for (root, &step) in self.root_steps.iter().enumerate() { + completes[step as usize].push(root); + } + let mut builder = Builder::::new(); + // Where each step landed: the terms between the steps move them. + let mut node: Vec = Vec::with_capacity(self.steps.len()); + let mut sum: Option = None; + for (index, step) in self.steps.iter().enumerate() { + let emitted = match *step { + Step::Fixed(ref c) => builder.fixed(c.clone()), + Step::Var(i) => builder.var(i as usize), + Step::Add(a, b) => builder.add(node[a as usize], node[b as usize]), + Step::Sub(a, b) => builder.sub(node[a as usize], node[b as usize]), + Step::Mul(a, b) => builder.mul(node[a as usize], node[b as usize]), + Step::Neg(a) => builder.neg(node[a as usize]), + }; + node.push(emitted); + for &root in &completes[index] { + let mut term = emitted; + if let Some(slot) = self.selector_of_root[root] { + let s = builder.var(slot); + term = builder.mul(term, s); + } + if beta_powers[root] != FieldElement::one() { + let c = builder.fixed(beta_powers[root].clone()); + term = builder.mul(c, term); + } + sum = Some(match sum { + Some(acc) => builder.add(acc, term), + None => term, + }); + } + } + let sum = match sum { + Some(sum) => sum, + None => builder.fixed(FieldElement::zero()), + }; + let w = builder.var(weight); + let root = builder.mul(w, sum); + builder.finish(root) + } + + pub fn combine( + &self, + beta_powers: &[FieldElement], + values: &[FieldElement], + ) -> FieldElement { + let nodes = self.run(values); + self.root_steps + .iter() + .zip(beta_powers) + .zip(&self.selector_of_root) + .fold(FieldElement::zero(), |acc, ((&root, beta_pow), sel)| { + let mut term = &nodes[root as usize] * beta_pow; + if let Some(slot) = sel { + term *= &values[*slot]; + } + acc + term + }) + } + + /// The public factors' values at `point`, in factor order. + /// + /// A selector is structure, not data, so the verifier computes it here + /// rather than believing a commitment for it. + pub fn public_values( + &self, + point: &[FieldElement], + ) -> Result>, MlError> { + self.public_selectors + .iter() + .map(|s| s.evaluate(point)) + .collect() + } + + /// Runs the DAG with each trace leaf taking the supplied value. + fn run(&self, values: &[FieldElement]) -> Vec> { + let mut out: Vec> = Vec::with_capacity(self.steps.len()); + for step in &self.steps { + let v = match *step { + Step::Fixed(ref c) => c.clone(), + Step::Var(i) => values[i as usize].clone(), + Step::Add(a, b) => &out[a as usize] + &out[b as usize], + Step::Sub(a, b) => &out[a as usize] - &out[b as usize], + Step::Mul(a, b) => &out[a as usize] * &out[b as usize], + Step::Neg(a) => -&out[a as usize], + }; + out.push(v); + } + out + } +} + +impl SumcheckPolynomial for IrPolynomial +where + F: IsSubFieldOf, + E: IsField + 'static, +{ + fn num_vars(&self) -> usize { + self.shape.num_vars + } + + fn degree(&self) -> usize { + self.shape.degree + } + + fn polys(&self) -> &[Mle] { + &self.polys + } + + fn combine(&self, values: &[FieldElement]) -> FieldElement { + self.shape.combine(&self.beta_powers, values) + } + + fn fix_first_variable(&mut self, r: &FieldElement) -> Result<(), MlError> { + for p in &mut self.polys { + p.fix_first_variable_in_place(r)?; + } + self.shape.num_vars -= 1; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + use math::field::goldilocks::GoldilocksField as Fp; + use multilinear::zerocheck; + + use crate::constraints::builder::{ + CaptureBuilder, ConstraintBuilder, ConstraintSet, RowDomain, + }; + + type ExtE = FieldElement; + /// The trace's own field: a column is base, only the challenges are not. + type BaseE = FieldElement; + + const COL_A: usize = 0; + const COL_B: usize = 1; + const COL_C: usize = 2; + + /// Two constraints in the shape real tables use: + /// + /// - idx 0, degree 1, reads the **next** step: `next(a) − a − b` + /// - idx 1, degree 2, current step only: `a·a − c` + /// + /// Unlike the Fibonacci examples these hold cyclically, so no wrap-around + /// exemption is needed — selectors are not modelled yet (see module docs). + struct SampleSet; + + impl ConstraintSet for SampleSet { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, COL_A); + let b_col = b.main(0, COL_B); + let a_next = b.main(1, COL_A); + b.emit_base(0, a_next - a.clone() - b_col); + + let c = b.main(0, COL_C); + b.emit_base(1, a.clone() * a - c); + } + } + + fn program() -> ConstraintProgram { + let mut cb = CaptureBuilder::::new(); + SampleSet.eval(&mut cb); + cb.finish(2).0 + } + + /// Columns satisfying both constraints on every step, wrap included. + fn satisfying_columns(num_vars: usize) -> [Vec; 3] { + let size = 1usize << num_vars; + let a: Vec = (0..size as u64) + .map(|i| BaseE::from(i.wrapping_mul(7).wrapping_add(3))) + .collect(); + // b is the cyclic forward difference, so `next(a) − a − b` vanishes + // including across the wrap. + let b: Vec = (0..size).map(|i| a[(i + 1) % size] - a[i]).collect(); + let c: Vec = a.iter().map(|x| x * x).collect(); + [a, b, c] + } + + fn leaves_from(columns: &[Vec; 3], num_vars: usize) -> TraceLeaves { + let prog = program(); + TraceLeaves::build( + &prog, + num_vars, + |col| columns[col as usize].clone(), + |_| unreachable!("this set has no aux reads"), + ) + .unwrap() + } + + fn transcript() -> DefaultTranscript { + DefaultTranscript::::new(b"multilinear-air-test") + } + + #[test] + fn one_leaf_per_distinct_trace_read() { + let num_vars = 4; + let leaves = leaves_from(&satisfying_columns(num_vars), num_vars); + // a@0, b@0, a@1, c@0 — the next-step read of `a` is its own factor. + assert_eq!(leaves.len(), 4); + assert!( + leaves + .index_of(&LeafKey { + main: true, + offset: 1, + col: COL_A as u16 + }) + .is_some() + ); + } + + #[test] + fn an_offset_leaf_is_a_shift_view_of_one_committed_column() { + let num_vars = 3; + let columns = satisfying_columns(num_vars); + let leaves = leaves_from(&columns, num_vars); + let factors = leaves.factors().unwrap(); + let size = 1usize << num_vars; + + let cur = leaves + .index_of(&LeafKey { + main: true, + offset: 0, + col: COL_A as u16, + }) + .unwrap(); + let next = leaves + .index_of(&LeafKey { + main: true, + offset: 1, + col: COL_A as u16, + }) + .unwrap(); + + // Both factors read the same committed column. + assert_eq!(leaves.sources()[cur].column, leaves.sources()[next].column); + assert_eq!(leaves.sources()[cur].offset, 0); + assert_eq!(leaves.sources()[next].offset, 1); + + for s in 0..size { + assert_eq!(factors[cur].evals()[s], columns[COL_A][s]); + assert_eq!( + factors[next].evals()[s], + columns[COL_A][(s + 1) % size], + "step {s}" + ); + } + } + + #[test] + fn a_next_step_read_adds_no_commitment() { + // Four factors — a@0, b@0, a@1, c@0 — over three columns: the next-step + // read of `a` rides on `a`'s commitment instead of taking its own. + let num_vars = 3; + let leaves = leaves_from(&satisfying_columns(num_vars), num_vars); + assert_eq!(leaves.len(), 4); + assert_eq!(leaves.columns().len(), 3); + } + + #[test] + fn registering_a_column_the_constraints_read_reuses_its_factor() { + let num_vars = 3; + let columns = satisfying_columns(num_vars); + let mut leaves = leaves_from(&columns, num_vars); + let before = (leaves.len(), leaves.columns().len()); + + let slot = leaves + .register_main(COL_A as u16, || unreachable!("already a factor")) + .unwrap(); + + assert_eq!( + slot, + leaves + .index_of(&LeafKey { + main: true, + offset: 0, + col: COL_A as u16 + }) + .unwrap() + ); + assert_eq!((leaves.len(), leaves.columns().len()), before); + } + + #[test] + fn registering_a_column_no_constraint_reads_adds_one_of_each() { + // A bus can read a column the constraints ignore, and then it does need + // a commitment. + let num_vars = 3; + let columns = satisfying_columns(num_vars); + let mut leaves = leaves_from(&columns, num_vars); + let (factors, committed) = (leaves.len(), leaves.columns().len()); + + let bus_column: Vec = (0..(1u64 << num_vars)).map(BaseE::from).collect(); + let slot = leaves.register_main(9, || bus_column.clone()).unwrap(); + + assert_eq!(slot, factors); + assert_eq!(leaves.len(), factors + 1); + assert_eq!(leaves.columns().len(), committed + 1); + assert_eq!(leaves.factors().unwrap()[slot].evals(), &bus_column[..]); + } + + /// Reads column 0 only at the next step, so its column is committed with no + /// unshifted factor of its own. + struct AheadOnlySet; + + impl ConstraintSet for AheadOnlySet { + fn max_degree(&self) -> usize { + 1 + } + + fn eval>(&self, b: &mut B) { + let next = b.main(1, COL_A); + let other = b.main(0, COL_B); + b.emit_base(0, next - other); + } + } + + #[test] + fn registering_a_column_only_read_ahead_reuses_its_commitment() { + let num_vars = 3; + let columns = satisfying_columns(num_vars); + let mut cb = CaptureBuilder::::new(); + AheadOnlySet.eval(&mut cb); + let prog = cb.finish(1).0; + + let mut leaves = TraceLeaves::build( + &prog, + num_vars, + |col| columns[col as usize].clone(), + |_| unreachable!("no aux reads"), + ) + .unwrap(); + // Two factors — A ahead, B here — over two columns. + assert_eq!((leaves.len(), leaves.columns().len()), (2, 2)); + + let slot = leaves + .register_main(COL_A as u16, || unreachable!("already committed")) + .unwrap(); + + // A new factor, but no new commitment: the shifted read already put + // column A in. + assert_eq!(slot, 2); + assert_eq!((leaves.len(), leaves.columns().len()), (3, 2)); + assert_eq!(leaves.factors().unwrap()[slot].evals(), &columns[COL_A][..]); + } + + #[test] + fn a_registered_column_of_the_wrong_height_is_rejected() { + let num_vars = 3; + let columns = satisfying_columns(num_vars); + let mut leaves = leaves_from(&columns, num_vars); + assert_eq!( + leaves + .register_main(9, || vec![BaseE::zero(); 3]) + .unwrap_err(), + MlError::NotPowerOfTwo(3) + ); + } + + #[test] + fn degree_comes_from_the_dag() { + let num_vars = 3; + let prog = program(); + let leaves = leaves_from(&satisfying_columns(num_vars), num_vars); + let poly = IrPolynomial::new_unselected(&prog, leaves, Uniforms::default(), ExtE::from(5)) + .unwrap(); + // `a·a − c` is the degree-2 constraint; batching does not raise it. + assert_eq!(poly.degree(), 2); + } + + #[test] + fn a_satisfying_trace_zerochecks() { + let num_vars = 5; + let prog = program(); + let leaves = leaves_from(&satisfying_columns(num_vars), num_vars); + let poly = IrPolynomial::new_unselected(&prog, leaves, Uniforms::default(), ExtE::from(5)) + .unwrap(); + + // The batched constraint really is zero on every step. + assert_eq!(poly.sum_over_hypercube(), ExtE::zero()); + let degree = poly.degree(); + + let proof = zerocheck::prove(poly, &mut transcript()).unwrap().proof; + let claim = zerocheck::verify(&proof, num_vars, degree, &mut transcript()).unwrap(); + + // Rebuild the polynomial to check the residual claim against it. + let leaves = leaves_from(&satisfying_columns(num_vars), num_vars); + let poly = IrPolynomial::new_unselected(&prog, leaves, Uniforms::default(), ExtE::from(5)) + .unwrap(); + assert_eq!( + claim.constraint_evaluation().unwrap(), + poly.evaluate(&claim.point).unwrap() + ); + } + + #[test] + fn a_single_broken_step_is_rejected() { + let num_vars = 5; + let prog = program(); + let mut columns = satisfying_columns(num_vars); + columns[COL_C][9] += BaseE::one(); + + let build = || { + IrPolynomial::new_unselected( + &prog, + leaves_from(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + ) + .unwrap() + }; + let poly = build(); + assert_ne!(poly.sum_over_hypercube(), ExtE::zero()); + let degree = poly.degree(); + + // `g(0)` is derived from the claim, so the lie surfaces in the residual + // rather than inside a round — and discharging that is the caller's. + let proof = zerocheck::prove(poly, &mut transcript()).unwrap().proof; + let claim = zerocheck::verify(&proof, num_vars, degree, &mut transcript()).unwrap(); + assert_ne!( + claim.constraint_evaluation(), + Some(build().evaluate(&claim.point).unwrap()), + "a violated constraint produced a consistent claim" + ); + } + + #[test] + fn violations_that_cancel_in_the_sum_are_still_caught() { + // Perturbing a[3] breaks `next(a) − a − b` twice: at step 2 the + // next-step read is one too high, at step 3 the current-step read is. + // The two violations are equal and opposite, so the plain sum over the + // cube stays zero — this is exactly the case eq(r, ·) exists to catch, + // and it is reachable from an ordinary AIR, not just a contrived one. + let num_vars = 4; + let prog = program(); + let mut columns = satisfying_columns(num_vars); + columns[COL_A][3] += BaseE::one(); + columns[COL_C][3] = columns[COL_A][3] * columns[COL_A][3]; // keep idx 1 satisfied + + let build = |cols: &[Vec; 3]| { + IrPolynomial::new_unselected( + &prog, + leaves_from(cols, num_vars), + Uniforms::default(), + ExtE::from(5), + ) + .unwrap() + }; + + let poly = build(&columns); + assert_eq!( + poly.sum_over_hypercube(), + ExtE::zero(), + "the violations were expected to cancel" + ); + // But the constraint is genuinely nonzero somewhere. + let nonzero_steps = (0..(1usize << num_vars)) + .filter(|&i| poly.eval_at_index(i) != ExtE::zero()) + .count(); + assert_eq!(nonzero_steps, 2); + + let degree = poly.degree(); + let proof = zerocheck::prove(poly, &mut transcript()).unwrap().proof; + + match zerocheck::verify(&proof, num_vars, degree, &mut transcript()) { + Err(_) => {} + Ok(claim) => { + // If the rounds happened to line up, the residual claim must + // still disagree with the real polynomial. + assert_ne!( + claim.constraint_evaluation(), + Some(build(&columns).evaluate(&claim.point).unwrap()), + "a violated constraint produced a consistent claim" + ); + } + } + } + + #[test] + fn batching_covers_every_constraint() { + // With only the degree-2 root selected, a violation of the degree-1 + // one must go unnoticed — which is what makes the batched version's + // rejection meaningful. + let num_vars = 4; + let prog = program(); + let mut columns = satisfying_columns(num_vars); + columns[COL_B][2] += BaseE::one(); // breaks idx 0 only + + let only_second = IrPolynomial::with_roots( + &prog, + leaves_from(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + vec![prog.roots[1]], + &[Selector::ALL], + ) + .unwrap(); + assert_eq!(only_second.sum_over_hypercube(), ExtE::zero()); + + let batched = IrPolynomial::new_unselected( + &prog, + leaves_from(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + ) + .unwrap(); + assert_ne!(batched.sum_over_hypercube(), ExtE::zero()); + } + + // --------------------------------------------------------------- + // Selectors: an AIR whose transition constraint cannot hold on the + // wrap-around step, which is the shape every real table has. + // --------------------------------------------------------------- + + /// The 2-column Fibonacci recurrence, verbatim from + /// `examples::fibonacci_2_columns`: both constraints read the next step + /// and therefore carry one end exemption. + struct FibSet; + + impl ConstraintSet for FibSet { + fn max_degree(&self) -> usize { + 1 + } + + fn eval>(&self, b: &mut B) { + let s0_0 = b.main(0, 0); + let s0_1 = b.main(0, 1); + let s1_0 = b.main(1, 0); + let s1_1 = b.main(1, 1); + + b.emit_base_rows( + 0, + RowDomain::except_last(1), + s1_0.clone() - s0_0 - s0_1.clone(), + ); + b.emit_base_rows(1, RowDomain::except_last(1), s1_1 - s0_1 - s1_0); + } + } + + fn fib_program() -> (ConstraintProgram, Vec) { + let mut cb = CaptureBuilder::::new(); + FibSet.eval(&mut cb); + let prog = cb.finish(2).0; + let meta = vec![ + ConstraintMeta::base(0).with_end_exemptions(1), + ConstraintMeta::base(1).with_end_exemptions(1), + ]; + (prog, meta) + } + + /// A genuine Fibonacci trace: the recurrence holds on every step except + /// the wrap, exactly where the exemption applies. + fn fib_columns(num_vars: usize) -> [Vec; 2] { + let size = 1usize << num_vars; + let mut c0 = vec![BaseE::one()]; + let mut c1 = vec![BaseE::one()]; + for i in 1..size { + // s0_{i} = s0_{i-1} + s1_{i-1}; s1_{i} = s1_{i-1} + s0_{i} + let next0 = c0[i - 1] + c1[i - 1]; + let next1 = c1[i - 1] + next0; + c0.push(next0); + c1.push(next1); + } + [c0, c1] + } + + fn fib_leaves(columns: &[Vec; 2], num_vars: usize) -> TraceLeaves { + let (prog, _) = fib_program(); + TraceLeaves::build( + &prog, + num_vars, + |col| columns[col as usize].clone(), + |_| unreachable!("no aux reads"), + ) + .unwrap() + } + + #[test] + fn without_a_selector_the_wrap_step_breaks_fibonacci() { + // Establishes that the exemption is load-bearing: unselected, the + // constraint is violated precisely at the wrap. + let num_vars = 4; + let (prog, _) = fib_program(); + let columns = fib_columns(num_vars); + let poly = IrPolynomial::new_unselected( + &prog, + fib_leaves(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + ) + .unwrap(); + + let last = (1usize << num_vars) - 1; + for i in 0..last { + assert_eq!(poly.eval_at_index(i), ExtE::zero(), "step {i}"); + } + assert_ne!(poly.eval_at_index(last), ExtE::zero(), "wrap step"); + } + + #[test] + fn with_the_selector_a_real_fibonacci_air_zerochecks() { + let num_vars = 5; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + + let build = || { + IrPolynomial::new( + &prog, + fib_leaves(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + &meta, + ) + .unwrap() + }; + + let poly = build(); + // The selector masks the wrap, so the batched constraint vanishes + // on the whole cube. + for i in 0..(1usize << num_vars) { + assert_eq!(poly.eval_at_index(i), ExtE::zero(), "step {i}"); + } + let degree = poly.degree(); + + let proof = zerocheck::prove(poly, &mut transcript()).unwrap().proof; + let claim = zerocheck::verify(&proof, num_vars, degree, &mut transcript()).unwrap(); + assert_eq!( + claim.constraint_evaluation().unwrap(), + build().evaluate(&claim.point).unwrap() + ); + } + + #[test] + fn the_selector_costs_exactly_one_degree() { + let num_vars = 4; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + + let unselected = IrPolynomial::new_unselected( + &prog, + fib_leaves(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + ) + .unwrap(); + let selected = IrPolynomial::new( + &prog, + fib_leaves(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + &meta, + ) + .unwrap(); + + assert_eq!(unselected.degree(), 1); + assert_eq!(selected.degree(), 2); + } + + #[test] + fn a_selector_does_not_hide_a_violation_inside_the_active_range() { + let num_vars = 5; + let (prog, meta) = fib_program(); + let mut columns = fib_columns(num_vars); + columns[0][7] += BaseE::one(); + + let build = || { + IrPolynomial::new( + &prog, + fib_leaves(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + &meta, + ) + .unwrap() + }; + let poly = build(); + let degree = poly.degree(); + + let proof = zerocheck::prove(poly, &mut transcript()).unwrap().proof; + let claim = zerocheck::verify(&proof, num_vars, degree, &mut transcript()).unwrap(); + assert_ne!( + claim.constraint_evaluation(), + Some(build().evaluate(&claim.point).unwrap()), + "a violated constraint produced a consistent claim" + ); + } + + #[test] + fn the_fibonacci_air_commits_only_its_two_columns() { + // Five factors: both columns at the current and the next step, plus the + // shared selector. Two commitments: the next-step reads are views of + // the columns, and the selector is public. + let num_vars = 3; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + let poly = IrPolynomial::new( + &prog, + fib_leaves(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + &meta, + ) + .unwrap(); + + let (committed, kinds) = poly.committed(); + assert_eq!(kinds.len(), 5); + assert_eq!(committed.len(), 2); + assert_eq!( + kinds.iter().filter(|k| **k == FactorKind::Public).count(), + 1 + ); + } + + #[test] + fn the_verifier_recomputes_the_selector_it_never_receives() { + // The public value the argument uses is the selector's closed form, and + // it is the same table the prover folded. + let num_vars = 3; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + let poly = IrPolynomial::new( + &prog, + fib_leaves(&columns, num_vars), + Uniforms::default(), + ExtE::from(5), + &meta, + ) + .unwrap(); + + let shape = poly.shape().clone(); + let point: Vec = (0..num_vars).map(|i| ExtE::from(31 + i as u64)).collect(); + let public = shape.public_values(&point).unwrap(); + assert_eq!(public.len(), 1); + + let table = poly.into_layout().public_tables.pop().unwrap(); + assert_eq!(public[0], table.evaluate(&point).unwrap()); + } + + #[test] + fn roots_sharing_an_exemption_share_one_selector_table() { + // Both Fibonacci constraints exempt one step, so exactly one selector + // table is materialized on top of the four trace leaves. + let num_vars = 3; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + let leaves = fib_leaves(&columns, num_vars); + let num_leaves = leaves.len(); + + let poly = + IrPolynomial::new(&prog, leaves, Uniforms::default(), ExtE::from(5), &meta).unwrap(); + assert_eq!(poly.polys().len(), num_leaves + 1); + } + + /// The compiled rule must agree with the walk it replaces: same roots, + /// same beta powers, same selectors, times the weight. + #[test] + fn the_zerocheck_program_agrees_with_the_walk() { + let num_vars = 3; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + let leaves = fib_leaves(&columns, num_vars); + let poly = + IrPolynomial::new(&prog, leaves, Uniforms::default(), ExtE::from(5), &meta).unwrap(); + let shape = poly.shape().clone(); + let betas = beta_powers(&ExtE::from(5), shape.num_roots()); + + // One more factor than the shape reads: the last is the weight. + let width = poly.polys().len(); + let values: Vec = (0..=width) + .map(|i| ExtE::from((i as u64).wrapping_mul(2654435761) + 11)) + .collect(); + let program = shape.program(&betas, width).unwrap(); + let mut scratch = Vec::new(); + assert_eq!( + program.eval(&values, &mut scratch), + values[width] * shape.combine(&betas, &values[..width]) + ); + } + + /// The lean program is the same polynomial, and it holds fewer values at + /// once: its roots die at their terms instead of at the end. + #[test] + fn the_lean_zerocheck_program_is_the_same_rule() { + let num_vars = 3; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + let leaves = fib_leaves(&columns, num_vars); + let poly = + IrPolynomial::new(&prog, leaves, Uniforms::default(), ExtE::from(5), &meta).unwrap(); + let shape = poly.shape().clone(); + let betas = beta_powers(&ExtE::from(5), shape.num_roots()); + let width = poly.polys().len(); + let values: Vec = (0..=width) + .map(|i| ExtE::from((i as u64).wrapping_mul(2654435761) + 11)) + .collect(); + let today = shape.program(&betas, width).unwrap(); + let lean = shape.program_lean(&betas, width).unwrap(); + let mut scratch = Vec::new(); + assert_eq!( + lean.eval(&values, &mut scratch), + today.eval(&values, &mut scratch) + ); + assert!(shape.program_lean(&[ExtE::one()], width).is_err()); + } + + /// A beta power per root, or the rule would batch the wrong number of them. + #[test] + fn a_program_with_the_wrong_beta_count_is_rejected() { + let num_vars = 3; + let (prog, meta) = fib_program(); + let columns = fib_columns(num_vars); + let leaves = fib_leaves(&columns, num_vars); + let poly = + IrPolynomial::new(&prog, leaves, Uniforms::default(), ExtE::from(5), &meta).unwrap(); + let shape = poly.shape().clone(); + assert!(shape.program(&[ExtE::one()], poly.polys().len()).is_err()); + } + + // --------------------------------------------------------------- + // The whole argument over a real captured AIR. + // --------------------------------------------------------------- + + /// Runs the full argument over the captured AIR and returns the verdict. + /// + /// The `next`-step read is a view of the column it shifts: it gets no + /// commitment, and the shift kernel binds its value to that column. + fn argue_fib(columns: &[Vec; 2], num_vars: usize) -> Result<(), multilinear::Error> { + use multilinear::{ + constraint_argument::{self, CommittedTrace, TraceClaim}, + whir_chain::{ChainConfig, GrindBits}, + whir_hash::KeccakWhir, + }; + + let (prog, meta) = fib_program(); + let leaves = TraceLeaves::build( + &prog, + num_vars, + |col| columns[col as usize].clone(), + |_| unreachable!("no aux reads"), + )?; + let poly = IrPolynomial::new(&prog, leaves, Uniforms::default(), ExtE::from(5), &meta)?; + let degree = poly.degree(); + let shape = poly.shape().clone(); + let layout = poly.into_layout(); + + let config = ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + }; + // Domain in the base field, columns in the degree-3 extension. + let n_stack = constraint_argument::one_stack(num_vars, layout.columns.len()); + let trace = CommittedTrace::::commit_views( + layout.columns, + layout.kinds, + layout.public_tables, + n_stack, + &config, + )?; + let roots = trace.roots(); + let betas = beta_powers(&ExtE::from(5), shape.num_roots()); + + let mut prover_transcript = DefaultTranscript::::new(b"air-argument"); + let proof = constraint_argument::prove::( + &trace, + |v: &[ExtE]| shape.combine(&betas, v), + degree, + &config, + &mut prover_transcript, + )?; + + let mut verifier_transcript = DefaultTranscript::::new(b"air-argument"); + constraint_argument::verify::( + &proof, + TraceClaim { + roots: &roots, + kinds: trace.kinds(), + layout: trace.layout(), + domain: trace.domain(), + num_vars, + }, + |v: &[ExtE]| shape.combine(&betas, v), + |point: &[ExtE]| shape.public_values(point), + degree, + &config, + &mut verifier_transcript, + ) + } + + /// The composition this whole crate exists for: an AIR captured by the same + /// builder the prover uses, zerochecked over the hypercube, every factor + /// settled against a WHIR commitment. + #[test] + fn a_real_air_argues_end_to_end_against_commitments() { + let num_vars = 4; + let columns = fib_columns(num_vars); + argue_fib(&columns, num_vars).unwrap(); + } + + #[test] + fn a_real_air_with_a_broken_row_is_rejected_end_to_end() { + let num_vars = 4; + let mut columns = fib_columns(num_vars); + columns[1][6] += BaseE::one(); + assert!(argue_fib(&columns, num_vars).is_err()); + } +} diff --git a/crypto/stark/src/multilinear_block.rs b/crypto/stark/src/multilinear_block.rs new file mode 100644 index 000000000..a088999c4 --- /dev/null +++ b/crypto/stark/src/multilinear_block.rs @@ -0,0 +1,4710 @@ +//! One multilinear proof for a whole block, proved in two phases. +//! +//! The block's tables — every chunk of every table type, run at full height — +//! are split into **groups**: contiguous runs in table order, each as many +//! tables as fit a few stacked polynomials. That is the unit a card holds. The +//! proof then goes: +//! +//! 1. **Phase A (commit).** Group by group: the columns go up, the stack is +//! committed, the roots are taken, and the codewords go — only the top of +//! each tree stays, on the host ([`RetiredStack`]). No card holds a block's +//! codewords, and every root must exist before the first challenge. Each +//! group then answers its commit-time out-of-domain point +//! ([`commit_ood_point`]) from its columns, still where its commit read +//! them. +//! 2. **The roots block.** Every root, in group order, into the transcript; +//! then every commit-time answer; then `(z, α, β)` once, shared by every +//! table of the block ([`absorb_roots_answers_and_challenge`]). The +//! transcript's state after the draw is the block's `S_post`. Each group's +//! opening batches its answers into its chains (`stacked_eval::OodClaims`). +//! 3. **Phase B (prove), per group `g`, on the fork `S_post ‖ g`.** The group's +//! columns go up again; each of its tables is argued (today's +//! [`prove`]) — or, under [`ArgueFormat::Batched`], the group's tables are +//! argued together ([`batched::prove_argue`]: a lockstep GKR per bin and one +//! constraint sumcheck for the group); the group's codewords are recomputed +//! — no hash, the tree's top is kept — and its stack is opened at the +//! tables' points ([`stacked_eval::prove`]). +//! +//! Between the phases a group's columns can be held **narrow** ([`Narrowing`]): +//! packed on the card right after the group's commit, each column at the bytes +//! its words need, and widened on the card again when phase B uploads the group. +//! The words come back bit for bit, so the proof is the same byte for byte; a +//! widening that went wrong is refused when the opening reads a path from the +//! kept tree top (the recomputed codeword does not hash to it). +//! +//! The tables whose leading preprocessed columns are settled out of band are +//! stacked per group ([`BlockPrepared`]): each such stack's derived roots are +//! absorbed after the groups' roots, and its opening proved on its group's fork +//! after the group's own opening. +//! +//! The verifier mirrors it and checks the bus balance ONCE, over every table of +//! the block: the LogUp challenges are shared, so the block's fractions sum to +//! what the statement owes exactly as one monolithic proof's do. +//! +//! # Why a fork per group is sound +//! +//! Each fork starts from `S_post`, which binds the statement and every root of +//! the block, and then the group's index; everything the group's argument and +//! opening draw is a function of that and of the group's own messages. This is +//! the STARK prover's per-table fork (`crypto/stark/src/prover.rs`, `append(idx)`) +//! at the granularity of a group. A prover cannot move a group's messages to +//! another group (the index is absorbed), cannot choose a root after a +//! challenge (all are absorbed before `z`), and cannot mix columns of two +//! groups (each opening is against its own group's roots, sliced by the +//! verifier's own layouts). +//! +//! ★ THE GROUPS ARE THE VERIFIER'S, never the proof's: [`block_groups`] is a +//! function of the tables' shapes and two format constants, and both sides call +//! it. +//! +//! # Why every committed word answers before `(z, α, β)` +//! +//! WHIR's commitment ends with an out-of-domain point after the root and the +//! committed word's value there, answered; the answer names the one codeword +//! the word stands for, and the opening carries it into its chain's first +//! claim (`multilinear::commit_ood`). The block's shared challenges come +//! after every answer, so each argues about one codeword per committed word. +//! Here the point is each group's own ([`commit_ood_point`]), drawn the moment +//! its roots exist, so phase A answers while the group's columns are on the +//! card; every answer still enters the main transcript before `(z, α, β)`. +//! Prepared roots need none: the verifier derives them, so each is a codeword. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::time::Instant; + +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; +use multilinear::{ + Error as MlError, + mle::Mle, + narrow::ColumnOf, + stacked_eval::{ + self, Claimed, ColumnsAt, CommitOod, OodClaims, OodCounts, RetiredStack, StackedCommitment, + StackedProof, + }, + stacking::StackedLayout, + whir::Domain, + whir_chain::{ArgueFormat, ChainConfig, StackVars}, + whir_commit::Commitment, + whir_hash::WhirHash, +}; + +use crate::regen::{RegenError, RegenProducer, RegenSlot, RegenWindow}; + +use crate::multilinear_table::{ + CommittedTable, MultiProof, TableProof, TableStatement, absorb_roots, + batched::{self, BatchedArgue, ProverFaults, VerifierChecks, Where}, + check_preprocessed, contribution, global_layout, prove, verify, +}; + +/// How the block's tables split into groups: contiguous runs in table order, +/// each as many tables as its stack holds in at most `max_polys` polynomials +/// under the stack cap. A table that alone needs more is a group of its own. +/// +/// `shapes` is `(committed columns, height in variables)` per table. Returns +/// the size of each group, in order; they sum to `shapes.len()`. +pub fn block_groups( + shapes: &[(usize, usize)], + cap: StackVars, + max_polys: usize, +) -> Result, MlError> { + let mut sizes = Vec::new(); + let mut start = 0usize; + for end in 1..=shapes.len() { + let polys = global_layout(&shapes[start..end], cap)?.num_polys(); + if polys > max_polys && end - start > 1 { + sizes.push(end - 1 - start); + start = end - 1; + } + } + if start < shapes.len() { + sizes.push(shapes.len() - start); + } + Ok(sizes) +} + +/// What a group cost, for the readout. Seconds of wall time on the prover's +/// thread. +#[derive(Clone, Debug, Default)] +pub struct GroupStamps { + pub tables: usize, + pub polys: usize, + pub cells: usize, + /// Phase A: the wait for the group's tables (a streamed producer), the + /// columns up, the commit, the roots, the tops home. + pub wait_a: f64, + pub upload_a: f64, + /// Of `upload_a`, the seconds the committer waited for: all of it unless + /// the upload ran beside the previous group's commit, when only what + /// outlasted that commit is paid (`upload_a − upload_paid` is hidden). + pub upload_paid: f64, + pub commit: f64, + pub retire: f64, + /// Phase B: the columns up again, the tables' arguments, the codewords + /// recomputed, the opening. + pub upload_b: f64, + pub argue: f64, + pub encode: f64, + pub open: f64, + /// Phase B: when the group's turn began and ended, seconds since phase B + /// began (its columns back, up, argued, opened and let go). Under + /// [`PhaseBOverlap::On`] a group's turn ends once the next group's + /// argument does, so the turns overlap. + pub start_b: f64, + pub end_b: f64, + /// Host bytes kept of the group's trees between the phases. + pub tree_bytes: usize, + /// Phase B: the bytes of records its tables held as their records, let go + /// with the group after its openings (under a spill). + pub records_released: usize, + /// Of those records, what their pages handed back to the OS came to + /// ([`RecordsReturner`]), and the seconds the handing back took on its + /// thread. + pub records_pages: multilinear::records::PagesReturned, + pub records_return_secs: f64, + /// The device ledger's peak promise through the group's argue — the + /// argue's own and the next group's columns uploaded beside it, and under + /// [`PhaseBOverlap::On`] the previous group's openings beside it too + /// (`transition_reserved`). Bytes; 0 without a device. + pub argue_reserved: u64, + /// The ledger's promise just before the group's argue (its own columns + /// on the card), and the ledger's peak through its revive and openings + /// (the next group's columns, uploaded beside the argue, included), with + /// what the revive itself added (the openings' room). Bytes; 0 without a + /// device. Phase B's overlap reads them: group g's openings beside group + /// g + 1's argue would hold about `open_reserved[g] + argue_reserved[g+1] + /// − argue_base[g+1]`. When the openings ran beside the next group's + /// argue, the two share one window: `open_reserved` is that window's peak + /// (the next group's `transition_reserved`), and `open_room` is 0. + pub argue_base: u64, + pub open_reserved: u64, + pub open_room: u64, + /// Of the group's openings (`encode` + `open`), the seconds phase B's + /// driver waited for: all of them when it ran them itself; beside the next + /// group's argue ([`PhaseBOverlap::On`]), what outlasted that argue and + /// the upload beside it. + pub open_paid: f64, + /// The group's openings ran beside the next group's argue. + pub overlapped: bool, + /// Under [`PhaseBOverlap::On`], the group's openings ran alone because an + /// earlier transition saw the card refuse a promise. + pub serial_after_refusal: bool, + /// The ledger's peak through the group's argue with the previous group's + /// openings beside it (and the next group's upload); 0 when no openings + /// ran beside it. Bytes. + pub transition_reserved: u64, + /// Stream draws while the group's openings ran, `(foreground pool, + /// background pool)`. Beside an argue the foreground count holds the + /// argue's draws too; alone, only the openings'. + pub open_draws: (u64, u64), + /// Phase B's driver thread over the group's turn on it (from the turn's + /// start to the next turn's), `(CPU ns, run-queue delay ns)` from its + /// schedstat, when [`BlockCommitted::with_sched_stamps`] asks for them + /// and the file reads; `None` otherwise. + pub sched_b: Option<(u64, u64)>, + /// Phase A's ledger: the promise just before the group's commit (its own + /// columns on the card, and the previous group's while it packs) and the + /// peak through the commit (the next group's columns, uploaded beside it, + /// included). Bytes; 0 without a device. + pub commit_base: u64, + pub commit_reserved: u64, + /// The group's columns were to go up beside the previous group's commit + /// and the ledger refused them; they went up after it. + pub ahead_refused: bool, + /// When the group's commit ended, seconds since phase A started. + pub committed_at: f64, + /// Phase A: the group's tables packed narrow after the commit + /// ([`Narrowing`]) — the seconds the committer waited for the pack, the + /// packer's own seconds (beside the next group's upload), how many tables, + /// their cells and the packed bytes. + pub pack: f64, + pub pack_busy: f64, + pub packed_tables: usize, + pub packed_cells: usize, + pub packed_bytes: usize, + /// Phase A: the group's commit-time out-of-domain answers + /// ([`commit_ood_point`]) — the seconds on the committer's thread, and how + /// many columns the card evaluated in place and the host did. + pub ood: f64, + pub ood_counts: OodCounts, +} + +/// The label that opens a group's commit-time out-of-domain transcript +/// ([`commit_ood_point`]). 32 bytes, so the prefix ends on a felt boundary. +pub const COMMIT_OOD_LABEL: &[u8; 32] = b"LAMBDAVM_WHIR_BLOCK_COMMIT_OOD_1"; + +/// The bytes a group's commit-time out-of-domain transcript opens with, in +/// ONE append: [`COMMIT_OOD_LABEL`], then the group's index, its stack's +/// variables, its polynomial count, the code's `log_blowup` and the chain's +/// fold word ([`ChainConfig::fold_word`]), each a little-endian `u64` — 72 +/// bytes, nine whole felts. The index separates the groups; the rest fixes the +/// code a root is read under, so no root is read under another code with the +/// same point. The in-guest leaf absorbs the same bytes as program constants. +pub fn commit_ood_prefix(group: usize, layout: &StackedLayout, config: &ChainConfig) -> Vec { + let mut bytes = COMMIT_OOD_LABEL.to_vec(); + for word in [ + group as u64, + layout.n_stack() as u64, + layout.num_polys() as u64, + config.log_blowup as u64, + config.fold_word(), + ] { + bytes.extend_from_slice(&word.to_le_bytes()); + } + bytes +} + +/// ★★ WHERE GROUP `g`'s COMMIT-TIME OUT-OF-DOMAIN POINT COMES FROM — the only +/// place, called by phase A and by the verifier. +/// +/// A fresh transcript over [`commit_ood_prefix`] and the group's roots, in +/// order; the point is its first draw, refused inside the group's evaluation +/// domain (`2^(n_stack + log_blowup)`; every smaller subgroup lies inside it). +/// +/// # Why the group's own transcript, and not the main one +/// +/// The answers must come after the group's roots and before `(z, α, β)`. This +/// point exists the moment the group's commit ends, while its columns are on +/// the card: phase A answers there, at the cost of one evaluation, instead of +/// a second pass over every column once phase A is over. What the answer +/// rules out — two codewords near one committed word agreeing at the point — +/// depends only on that word and its code, both fixed by this input; the main +/// transcript binds the statement, every root and every answer before +/// `(z, α, β)`, where every equation that spans groups is drawn. +pub fn commit_ood_point( + group: usize, + layout: &StackedLayout, + roots: &[Commitment], + config: &ChainConfig, +) -> Result, MlError> +where + E: IsField + 'static, + T: crypto::fiat_shamir::is_transcript::IsTranscript + Default, +{ + if roots.len() != layout.num_polys() { + return Err(MlError::QueryCountMismatch { + expected: layout.num_polys(), + got: roots.len(), + }); + } + let mut transcript = T::default(); + transcript.append_bytes(&commit_ood_prefix(group, layout, config)); + for root in roots { + transcript.append_bytes(root); + } + let z0: FieldElement = transcript.sample_field_element(); + multilinear::whir_chain::require_out_of_domain_of_size( + &z0, + layout.n_stack() + config.log_blowup, + )?; + Ok(z0) +} + +/// ★ The block's roots block: [`absorb_roots_and_answers`], then `(z, α, β)`. +pub fn absorb_roots_answers_and_challenge( + transcript: &mut T, + carried: &[Commitment], + derived: &[Commitment], + answers: &[FieldElement], +) -> (FieldElement, FieldElement, FieldElement) +where + E: IsField + 'static, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + absorb_roots_and_answers::(transcript, carried, derived, answers); + ( + transcript.sample_field_element(), + transcript.sample_field_element(), + transcript.sample_field_element(), + ) +} + +/// The block's roots, then every carried polynomial's commit-time answer in +/// root order: everything a challenge drawn next is bound to. The prover, the +/// verifier and every replay of the roots block (the COMMIT bus's probe) call +/// this, and the in-guest leaf's front mirrors it. +/// +/// ⚠ The answers go AFTER every root — derived ones included — and BEFORE any +/// draw. An answer absorbed after `z` binds nothing, and no round trip can see +/// that: prover and verifier would agree on the wrong order. +/// `the_roots_block_binds_every_answer_before_the_first_draw` checks the +/// order against a transcript spelled out by hand. +pub fn absorb_roots_and_answers( + transcript: &mut T, + carried: &[Commitment], + derived: &[Commitment], + answers: &[FieldElement], +) where + E: IsField + 'static, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + absorb_roots::(transcript, carried, derived); + for answer in answers { + transcript.append_field_element(answer); + } +} + +/// How a block holds its committed columns between phase A and phase B. +/// +/// The proof is the same byte for byte under every choice: a narrow table +/// keeps its raw words, each column at the bytes its largest needs (1, 2, 4 or +/// 8), and the card widens them back into the store phase B reads. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Narrowing { + /// As field elements, eight bytes a cell. + #[default] + Wide, + /// Packed on the card from the group's columns there, after the group's + /// commit; only the packed bytes come back. A table of fewer than + /// `min_cells` cells, or one the card did not hold, stays wide. + Card { min_cells: usize }, + /// Packed on the host after the group's commit, every table: a test's, or + /// a host without a card. The same packed bytes as [`Self::Card`]. + Host, +} + +impl Narrowing { + /// Production: on the card, every table of [`NARROW_MIN_CELLS`] or more. + pub const CARD: Self = Self::Card { + min_cells: NARROW_MIN_CELLS, + }; +} + +/// Production's smallest table packed narrow: under it a table's eight bytes +/// a cell are little, and packing it costs the card two round trips. +pub const NARROW_MIN_CELLS: usize = 1 << 16; + +/// Where a block's committed columns are on the host as they move, in bytes: +/// a memory log's terms (the block prover's `LAMBDA_VM_BLOCK_MEMLOG`), read +/// by its sampler while the block proves. It counts and changes nothing. +pub struct BlockMem { + /// The group phase A is uploading and committing, eight bytes a cell. + pub committing: AtomicUsize, + /// The next group, taken and uploaded beside that commit (uploading + /// ahead), eight bytes a cell. + pub ahead: AtomicUsize, + /// Committed groups still held wide while their packers run, until their + /// packed tables are installed. + pub packing_wide: AtomicUsize, + /// What finished packers hold and nothing installed yet. + pub packed_ready: AtomicUsize, + /// Committed tables as held: packed, and eight bytes a cell. + pub held_narrow: AtomicUsize, + pub held_wide: AtomicUsize, + /// The groups' kept tree tops. + pub tree_tops: AtomicUsize, + mark: Box, +} + +impl BlockMem { + /// Every term at zero; `mark` is called at each event: a group committed, + /// a group's packed tables installed, a phase-B group's end. + pub fn new(mark: impl Fn(&str) + Send + Sync + 'static) -> Self { + Self { + committing: AtomicUsize::new(0), + ahead: AtomicUsize::new(0), + packing_wide: AtomicUsize::new(0), + packed_ready: AtomicUsize::new(0), + held_narrow: AtomicUsize::new(0), + held_wide: AtomicUsize::new(0), + tree_tops: AtomicUsize::new(0), + mark: Box::new(mark), + } + } + + fn mark(&self, label: &str) { + (self.mark)(label); + } + + /// Counts `tables` as held from here on, packed or wide. + fn hold<'t, 'a: 't, F, E>(&self, tables: impl IntoIterator>) + where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + for table in tables.into_iter().filter(|t| !t.is_spilled()) { + match table.compact_bytes() { + Some(bytes) => self.held_narrow.fetch_add(bytes, Relaxed), + None => self.held_wide.fetch_add(wide_bytes(table), Relaxed), + }; + } + } + + /// Stops counting `tables` as held ([`Self::hold`]'s inverse). + fn release<'t, 'a: 't, F, E>( + &self, + tables: impl IntoIterator>, + ) where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + // A table whose columns are out left the count when they went: a + // drop, or phase B letting them go after its group. + for table in tables.into_iter().filter(|t| !t.is_spilled()) { + match table.compact_bytes() { + Some(bytes) => self.held_narrow.fetch_sub(bytes, Relaxed), + None => self.held_wide.fetch_sub(wide_bytes(table), Relaxed), + }; + } + } +} + +/// Whether a committed table of `bytes` packed bytes is to leave the host, +/// given the packed bytes kept so far and the main cells committed so far +/// (this table's included): `(kept, cells, bytes)` ([`BlockSpill::new`]). +pub type SpillWanted = dyn Fn(u64, u64, u64) -> bool + Send + Sync; + +/// What a block moves off the host while phase A holds its committed tables. +/// With live regeneration ([`Self::with_regen`]) a table phase B can rebuild +/// is parked as its group is installed, and leaves the host as a drop +/// ([`BlockRegen`]) once the policy (`wanted`) first wants a table out; every +/// other table stays in its group. Phase B brings each group's parked and +/// dropped tables back before their upload, and lets each group's packed +/// columns go after its opening. +#[derive(Clone)] +pub struct BlockSpill { + /// The policy's choice for one table. + pub wanted: Arc, + /// Live regeneration, when it is on. + pub regen: Option, + /// Packed bytes of the committed tables kept, and main cells committed. + kept: Arc, + cells: Arc, +} + +/// A committed table's packed columns parked for live regeneration: held here +/// until it drops them, and read back from the host or from their +/// regeneration slot before the group's upload (`restore_group`). +struct Parked { + len: u64, + /// The main cells committed once it was, its own included + /// ([`BlockRegen::keep_cells`]). + cells: u64, + /// Its class and rank when phase B can rebuild it ([`BlockRegen`]). + rank: Option<(usize, u64)>, + state: std::sync::Mutex, +} + +enum ParkedState { + Resident(multilinear::narrow::NarrowColumns), + /// A table held as the records its rows come from + /// ([`multilinear::records`]): live regeneration drops it as them, into a + /// records slot ([`RegenWindow::slot_for_records`]). + ResidentRecords(multilinear::records::Records), + /// Dropped after its digest: phase B's regenerator deposits it again. + Dropped(RegenSlot), + /// Taken and not put back: phase B refuses the table. + Lost, +} + +impl Parked { + /// The slot it was dropped into, once dropped. + fn dropped(&self) -> Option { + match &*self.state.lock().unwrap_or_else(|e| e.into_inner()) { + ParkedState::Dropped(slot) => Some(slot.clone()), + _ => None, + } + } + + /// The class the columns were dropped in, when they were. + fn dropped_class(&self) -> Option { + self.dropped() + .and_then(|_| self.rank.map(|(class, _)| class)) + } +} + +impl BlockSpill { + /// Deciding with `wanted`; nothing leaves the host until live + /// regeneration is on ([`Self::with_regen`]). + pub fn new(wanted: Arc) -> Self { + Self { + wanted, + regen: None, + kept: Arc::new(std::sync::atomic::AtomicU64::new(0)), + cells: Arc::new(std::sync::atomic::AtomicU64::new(0)), + } + } + + /// Live regeneration on: a table `regen` can rebuild leaves the host as a + /// drop ([`BlockRegen`]). + pub fn with_regen(mut self, regen: BlockRegen) -> Self { + self.regen = Some(regen); + self + } + + /// The walk's hand-off: arms live regeneration ([`BlockRegen::arm`]), so + /// every parked table phase B can rebuild is dropped. Partial drops + /// ([`BlockRegen::partial`]) drop back parked tables, in rank order, only + /// until `need` bytes are on their way. `false` without live regeneration. + pub fn hand_off(&self, need: u64) -> bool { + let Some(regen) = &self.regen else { + return false; + }; + regen.arm_at(self.cells.load(Relaxed)); + // Partial drops hand back only what the hand-off asks for. + if regen.is_partial() { + regen.drop_back(need); + } + true + } + + /// Phase A's end: joins live regeneration's drops. + fn close_hand_off(&self) { + if let Some(regen) = &self.regen { + regen.close_drops(); + } + } +} + +/// Live regeneration on a block (D-WHIR-NODISK §2.1). `droppable(t)` is the +/// rank of the table at block index `t` when phase B can rebuild its columns +/// (a streamed chunk: its place in the hand-out), `None` otherwise. Phase A +/// drops such a table instead of keeping it: its packed columns +/// leave the host after their digest, on a thread of their own, and a +/// [`RegenSlot`] in `window` stands for them; phase B waits on the slot before +/// the table's group needs it (`restore_group`), and a deposit that is not +/// the dropped columns refuses the table there, before any device work, with +/// the kept tree top's check behind the digest. +/// +/// Under `always` every droppable table is dropped (the byte-identity test +/// mode). Otherwise nothing drops until the policy first moves a table off +/// the host (or the walk's hand-off): that arms it, every parked droppable +/// table is dropped back in rank order, and every later one is dropped. A +/// block that never arms drops nothing. +#[derive(Clone)] +pub struct BlockRegen { + droppable: Arc Option<(usize, u64)> + Send + Sync>, + /// One window a class: each class's regenerator deposits, and phase B + /// takes, in that class's rank order. + windows: Vec>, + always: bool, + /// Each table is dropped only when the policy wants it out, armed or not + /// ([`Self::partial`]). + partial: bool, + /// Partial drops' keep budget at arming, in main cells committed + /// ([`Self::keep_cells`]). + keep_cells: Option, + /// Partial drops' keep budget relative to the arming point: this many main + /// cells below those committed at arming ([`Self::keep_below`]). + keep_below: Option, + started: Instant, + inner: Arc>, + /// Wakes [`Self::await_rank`] at each decision and when the decisions + /// close. + decided: Arc, +} + +/// A droppable table's fate, as a regenerator started before phase A's end +/// needs it ([`BlockRegen::await_rank`]). +#[derive(Clone)] +pub enum RankDecision { + /// Dropped: its slot, which the regenerator fills. + Dropped(RegenSlot), + /// Not dropped, and never will be: held by its group's class + /// ([`drop_one_class`]), no longer on the host, or phase A ended without + /// dropping it. + Kept, + /// Its class's window closed (the prove ended or failed) before the table + /// was decided. + Stopped, +} + +#[derive(Default)] +struct RegenState { + /// When it armed, seconds after it was made. + armed: Option, + /// At arming: the main cells committed, when known; and the candidates + /// parked by then (tables, bytes), before the keep budget's drop-back. + armed_cells: Option, + armed_kept: (usize, u64), + /// What the keep budget dropped back at arming (tables, bytes). + keep_back: (usize, u64), + /// The budget arming used, in main cells committed, when it had one. + budget: Option, + /// Each window's first producer, kept for its regenerator. + producers: Vec>, + /// Parked droppable tables, for drop-back at arming. + candidates: Vec>, + tx: Option>, + thread: Option>, + /// Every drop, in the order made: its class and rank, its slot, its bytes + /// and whether drop-back made it. + dropped: Vec<((usize, u64), RegenSlot, u64, bool)>, + /// Drops that found their table no longer held on the host. + refused_late: usize, + /// Partial drops: the candidates were settled ([`BlockRegen::settle`]), + /// and a table not dropped from now on is kept at once. + settled: bool, + /// Tables phase B could rebuild that stay, their group dropping another + /// class ([`drop_one_class`]). + held: usize, + /// A memory log, which counts each drop out of the held bytes. + mem: Option>, + /// Each decided table by class and rank: its slot when dropped; the + /// ranks decided to stay; and whether the decisions are over (phase A's + /// end: an undecided table then stays). + slots: std::collections::HashMap<(usize, u64), RegenSlot>, + kept: std::collections::HashSet<(usize, u64)>, + decisions_closed: bool, + /// Test only: a pause before each drop of this class, so a regenerator + /// started early meets undecided tables. + drop_delay: Option<(usize, std::time::Duration)>, +} + +/// A table for the drop thread, and whether drop-back sent it. +struct DropJob { + parked: Arc, + back: bool, +} + +/// What live regeneration dropped ([`BlockRegen::report`]). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DropReport { + pub armed: Option, + pub tables: usize, + pub bytes: u64, + pub back_tables: usize, + pub back_bytes: u64, + pub refused_late: usize, + /// By class: the tables and bytes dropped. + pub by_class: Vec<(usize, u64)>, + /// Tables phase B could rebuild kept on the host because their group + /// dropped another class ([`drop_one_class`]). + pub held: usize, + /// At arming: the main cells committed, when known, and the candidates + /// parked by then (tables, bytes) before the keep budget's drop-back. + pub armed_cells: Option, + pub armed_kept: (usize, u64), + /// The keep budget ([`BlockRegen::keep_cells`]) and what it dropped back + /// at arming (tables, bytes). + pub keep_cells: Option, + pub keep_back: (usize, u64), + /// The budget relative to the arming point ([`BlockRegen::keep_below`]), + /// and the budget arming used, in main cells committed. + pub keep_below: Option, + pub budget: Option, + /// Of `tables` and `bytes`, those dropped as the records their rows come + /// from (their records' bytes). + pub records: (usize, u64), +} + +/// One class's regeneration for phase B ([`BlockRegen::into_plan`]): its +/// window, its dropped tables' ranks and slots in the window's order, and the +/// window's first producer for its regenerator. +pub struct ClassPlan { + pub window: Arc, + pub dropped: Vec<(u64, RegenSlot)>, + pub producer: Option, +} + +impl BlockRegen { + /// Live regeneration over `droppable` (a table's class and rank, when + /// phase B can rebuild it), a class's slots paced by a window of its + /// `aheads` bytes ([`RegenWindow::new`]). Its drop thread starts here; + /// phase A's end joins it. + pub fn new( + droppable: Arc Option<(usize, u64)> + Send + Sync>, + aheads: &[u64], + always: bool, + ) -> Self { + let (windows, producers): (Vec<_>, Vec<_>) = aheads + .iter() + .map(|&ahead| { + let (window, producer) = RegenWindow::new(ahead); + (window, Some(producer)) + }) + .unzip(); + let inner = Arc::new(std::sync::Mutex::new(RegenState { + producers, + ..RegenState::default() + })); + let (tx, rx) = std::sync::mpsc::channel::(); + let decided = Arc::new(std::sync::Condvar::new()); + let (windows_in, inner_in, decided_in) = + (windows.clone(), Arc::clone(&inner), Arc::clone(&decided)); + let thread = std::thread::Builder::new() + .name("block-regen-drop".to_string()) + .spawn(move || { + for job in rx { + let rank = job.parked.rank; + let (mem, delay) = { + let state = inner_in.lock().unwrap_or_else(|e| e.into_inner()); + (state.mem.clone(), state.drop_delay) + }; + if let (Some((class, pause)), Some((c, _))) = (delay, rank) + && class == c + { + std::thread::sleep(pause); + } + let dropped = rank.and_then(|(class, rank)| { + let window = windows_in.get(class)?; + drop_parked(window, &job.parked, rank, mem.as_deref()) + }); + let mut state = inner_in.lock().unwrap_or_else(|e| e.into_inner()); + match (dropped, rank) { + (Some(slot), Some(rank)) => { + state.slots.insert(rank, slot.clone()); + state.dropped.push((rank, slot, job.parked.len, job.back)) + } + (None, Some(rank)) => { + state.kept.insert(rank); + state.refused_late += 1 + } + _ => state.refused_late += 1, + } + decided_in.notify_all(); + } + }); + { + let mut state = inner.lock().unwrap_or_else(|e| e.into_inner()); + match thread { + Ok(thread) => { + state.tx = Some(tx); + state.thread = Some(thread); + } + // No drop thread: nothing drops, and every table stays as the + // policy leaves it. + Err(_) => drop(tx), + } + } + Self { + droppable, + windows, + always, + partial: false, + keep_cells: None, + keep_below: None, + started: Instant::now(), + inner, + decided, + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, RegenState> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// The class and rank of the table at block index `index`, when it may + /// be dropped (a class with no window may not). + pub fn rank_of(&self, index: usize) -> Option<(usize, u64)> { + (self.droppable)(index).filter(|&(class, _)| class < self.windows.len()) + } + + /// Each class's window, the dropped tables' slots in it. + pub fn windows(&self) -> &[Arc] { + &self.windows + } + + /// Whether it armed. + pub fn is_armed(&self) -> bool { + self.lock().armed.is_some() + } + + /// Whether every droppable table is dropped. + pub fn always(&self) -> bool { + self.always + } + + /// Partial drops: arming drops back nothing and does not make later tables + /// drop. Each committed table is dropped only when the policy wants it out + /// as it is installed ([`Self::drop_one`]), and the parked ones only as far + /// as a hand-off asks ([`Self::drop_back`]). The rest stay on the host. + pub fn partial(mut self) -> Self { + self.partial = true; + self + } + + /// Whether drops are partial ([`Self::partial`]). + pub fn is_partial(&self) -> bool { + self.partial + } + + /// Partial drops' keep budget: at arming, the candidates committed past + /// `cells` main cells (the block's cells committed once each was, its own + /// included) are dropped back, in rank order, and only those; the others + /// stay candidates as before. What arming keeps then no longer follows how + /// far phase A got when the policy first wanted a table out. `None` (the + /// default): arming drops back nothing. Nothing for drops that are not + /// partial. + pub fn keep_cells(mut self, cells: Option) -> Self { + self.keep_cells = cells; + self + } + + /// Partial drops' keep budget relative to the arming point: `cells` main + /// cells below those committed when it arms ([`Self::arm_at`]), so the + /// candidates committed in the last `cells` before arming are dropped + /// back, in rank order. Where the arming point moves (the host's reading at + /// the forecast's crossing), the budget moves with it. With + /// [`Self::keep_cells`] too, the lower of the two holds. Arming without the + /// cells committed ([`Self::arm`]) uses only the absolute budget. `None` + /// (the default): no relative budget. Nothing for drops that are not + /// partial. + pub fn keep_below(mut self, cells: Option) -> Self { + self.keep_below = cells; + self + } + + /// Arm: from now on every droppable table is dropped, and every parked one + /// is dropped back, in rank order; partial drops ([`Self::partial`]) note + /// when, and drop back only what their keep budget asks + /// ([`Self::keep_cells`]). Returns whether this call armed. + pub fn arm(&self) -> bool { + self.arm_with(None) + } + + /// [`Self::arm`] with the main cells committed by then, which the drop + /// report keeps. + pub fn arm_at(&self, cells: u64) -> bool { + self.arm_with(Some(cells)) + } + + fn arm_with(&self, cells: Option) -> bool { + let mut state = self.lock(); + if state.armed.is_some() { + return false; + } + state.armed = Some(self.started.elapsed().as_secs_f64()); + state.armed_cells = cells; + if self.partial { + state.armed_kept = ( + state.candidates.len(), + state.candidates.iter().map(|p| p.len).sum(), + ); + let below = self + .keep_below + .zip(cells) + .map(|(below, at)| at.saturating_sub(below)); + state.budget = match (self.keep_cells, below) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; + if let Some(budget) = state.budget { + let (mut past, within): (Vec<_>, Vec<_>) = std::mem::take(&mut state.candidates) + .into_iter() + .partition(|p| p.cells > budget); + past.sort_by_key(|p| p.rank); + state.keep_back = (past.len(), past.iter().map(|p| p.len).sum()); + state.candidates = within; + for parked in past { + Self::queue(&state, parked, true); + } + } + return true; + } + let mut candidates = std::mem::take(&mut state.candidates); + candidates.sort_by_key(|p| p.rank); + for parked in candidates { + Self::queue(&state, parked, true); + } + true + } + + /// A memory log for the drops to count in. + fn set_mem(&self, mem: Option>) { + self.lock().mem = mem; + } + + /// A parked droppable table: dropped now when armed (not partial) or + /// `always`, else a drop-back candidate. + fn parked(&self, parked: Arc) { + let mut state = self.lock(); + if self.always || (state.armed.is_some() && !self.partial) { + Self::queue(&state, parked, false); + } else if self.partial && state.settled { + if let Some(rank) = parked.rank { + state.kept.insert(rank); + } + drop(state); + self.decided.notify_all(); + } else { + state.candidates.push(parked); + } + } + + /// Partial drops: every parked candidate kept from now on, and every table + /// not dropped later kept as it is parked, so a regenerator started before + /// phase A's end learns each table's fate as it comes + /// ([`Self::drops_all_from_now`]). Called once the last drop-back is past + /// (the walk's end, after its hand-off). Nothing for drops that are not + /// partial. + pub fn settle(&self) { + if !self.partial { + return; + } + let mut state = self.lock(); + let candidates = std::mem::take(&mut state.candidates); + for parked in candidates { + if let Some(rank) = parked.rank { + state.kept.insert(rank); + } + } + state.settled = true; + drop(state); + self.decided.notify_all(); + } + + /// A parked droppable table the policy wants out: dropped now. + fn drop_one(&self, parked: Arc) { + Self::queue(&self.lock(), parked, false); + } + + /// Partial drops: the parked candidates dropped back, in rank order, until + /// `need` bytes are on their way (all of them for `u64::MAX`). Returns the + /// bytes it sent. + pub fn drop_back(&self, need: u64) -> u64 { + let mut state = self.lock(); + let mut candidates = std::mem::take(&mut state.candidates); + candidates.sort_by_key(|p| p.rank); + let mut sent = 0u64; + let mut rest = Vec::new(); + for parked in candidates { + if sent < need { + sent = sent.saturating_add(parked.len); + Self::queue(&state, parked, true); + } else { + rest.push(parked); + } + } + state.candidates = rest; + sent + } + + fn queue(state: &RegenState, parked: Arc, back: bool) { + if let Some(tx) = &state.tx { + let _ = tx.send(DropJob { parked, back }); + } + } + + /// No more drops: the drop thread finishes what is queued and ends, and + /// every table not dropped by then stays ([`RankDecision::Kept`]). + fn close_drops(&self) { + let thread = { + let mut state = self.lock(); + state.tx = None; + state.thread.take() + }; + if let Some(thread) = thread { + let _ = thread.join(); + } + self.lock().decisions_closed = true; + self.decided.notify_all(); + } + + /// Tables kept because their group dropped another class. + fn held(&self, ranks: &[(usize, u64)]) { + let mut state = self.lock(); + state.held += ranks.len(); + state.kept.extend(ranks.iter().copied()); + drop(state); + self.decided.notify_all(); + } + + /// Whether the fate of every table phase B can rebuild that phase A still + /// commits is known as it is installed: dropped (armed, or `always`), or + /// under partial drops once settled ([`Self::settle`]), dropped or kept. A + /// regenerator may then start before phase A's end and take each table's + /// fate from [`Self::await_rank`]. + pub fn drops_all_from_now(&self) -> bool { + let state = self.lock(); + self.always || (state.armed.is_some() && !self.partial) || (self.partial && state.settled) + } + + /// Class `class`'s first producer, for a regenerator that starts before + /// phase A's end ([`Self::into_plan`] then hands out none for the class). + pub fn take_producer(&self, class: usize) -> Option { + self.lock().producers.get_mut(class).and_then(Option::take) + } + + /// The fate of class `class`'s table of rank `rank`, waiting for it: + /// dropped (its slot), kept, or `Stopped` once the class's window + /// closes. Returns too how long it waited. A table phase A never dropped + /// is kept once phase A's end closes the decisions; nothing is taken as + /// kept before that, so a regenerator never skips a table that drops + /// later. + pub fn await_rank(&self, class: usize, rank: u64) -> (RankDecision, std::time::Duration) { + let started = Instant::now(); + let window = self.windows.get(class).cloned(); + let mut state = self.lock(); + loop { + if let Some(slot) = state.slots.get(&(class, rank)) { + return (RankDecision::Dropped(slot.clone()), started.elapsed()); + } + if state.kept.contains(&(class, rank)) || state.decisions_closed { + return (RankDecision::Kept, started.elapsed()); + } + if window.as_ref().is_none_or(|w| w.is_closed()) { + return (RankDecision::Stopped, started.elapsed()); + } + // A closed window does not wake this condvar: poll for it. + state = self + .decided + .wait_timeout(state, std::time::Duration::from_millis(50)) + .unwrap_or_else(|e| e.into_inner()) + .0; + } + } + + /// Whether phase A's end has closed the decisions. + pub fn decisions_closed(&self) -> bool { + self.lock().decisions_closed + } + + /// Test only: pause before each drop of class `class`, so a regenerator + /// started early meets undecided tables. + #[doc(hidden)] + pub fn delay_drops(&self, class: usize, pause: std::time::Duration) { + self.lock().drop_delay = Some((class, pause)); + } + + /// What it dropped. + pub fn report(&self) -> DropReport { + let state = self.lock(); + let back = state.dropped.iter().filter(|d| d.3); + let mut by_class = vec![(0usize, 0u64); self.windows.len()]; + for ((class, _), _, bytes, _) in &state.dropped { + if let Some(c) = by_class.get_mut(*class) { + c.0 += 1; + c.1 += bytes; + } + } + let records = state.dropped.iter().filter(|d| d.1.is_records()); + DropReport { + armed: state.armed, + tables: state.dropped.len(), + bytes: state.dropped.iter().map(|d| d.2).sum(), + back_tables: back.clone().count(), + back_bytes: back.map(|d| d.2).sum(), + refused_late: state.refused_late, + by_class, + held: state.held, + armed_cells: state.armed_cells, + armed_kept: state.armed_kept, + keep_cells: self.keep_cells.filter(|_| self.partial), + keep_back: state.keep_back, + keep_below: self.keep_below.filter(|_| self.partial), + budget: state.budget, + records: (records.clone().count(), records.map(|d| d.2).sum()), + } + } + + /// Phase B's regeneration, a class each ([`ClassPlan`]): a class that + /// dropped nothing has no dropped tables and no producer (it went with + /// the plan, so its window's slots, none, need no regenerator). + pub fn into_plan(self) -> Vec { + let mut state = self.lock(); + let producers = std::mem::take(&mut state.producers); + let mut plans: Vec = self + .windows + .iter() + .zip(producers) + .map(|(window, producer)| ClassPlan { + window: Arc::clone(window), + dropped: Vec::new(), + producer, + }) + .collect(); + for ((class, rank), slot, _, _) in &state.dropped { + if let Some(plan) = plans.get_mut(*class) { + plan.dropped.push((*rank, slot.clone())); + } + } + for plan in &mut plans { + plan.dropped.sort_by_key(|(_, slot)| slot.order_key()); + if plan.dropped.is_empty() { + plan.producer = None; + } + } + plans + } +} + +/// Drops a parked table's packed columns: they leave `parked` under its lock, +/// are digested into a slot of `window` outside it, are freed, and the slot +/// takes their place. `None`, and nothing changed, when the table is no +/// longer held on the host. +fn drop_parked( + window: &Arc, + parked: &Parked, + rank: u64, + mem: Option<&BlockMem>, +) -> Option { + let held = { + let mut state = parked.state.lock().unwrap_or_else(|e| e.into_inner()); + match std::mem::replace(&mut *state, ParkedState::Lost) { + held @ (ParkedState::Resident(_) | ParkedState::ResidentRecords(_)) => held, + other => { + *state = other; + return None; + } + } + }; + // Records are digested as they lie; their slot takes the packed columns + // the regenerator makes from records it rebuilt. + let (slot, len) = match &held { + ParkedState::Resident(packed) => (window.slot(packed, rank), packed.data().len()), + ParkedState::ResidentRecords(records) => { + (window.slot_for_records(records, rank), records.bytes()) + } + _ => return None, + }; + drop(held); + *parked.state.lock().unwrap_or_else(|e| e.into_inner()) = ParkedState::Dropped(slot.clone()); + if let Some(mem) = mem { + mem.held_narrow.fetch_sub(len, Relaxed); + } + Some(slot) +} + +/// Considers a group's narrow tables for the spill, `first` being the +/// first's index; where each table taken out of its group went goes in `out`. +/// Every table counts toward the cells committed. A table phase B can rebuild +/// is parked ([`Parked`]) and dropped ([`BlockRegen`]): at once under +/// `always`, once the policy first wants a table out (which arms it), and back +/// at arming while parked; it counts toward the bytes kept only while it is +/// not dropped. Every other table stays in its group and counts toward the +/// bytes kept. +fn spill_group( + spill: &BlockSpill, + first: usize, + tables: &mut [CommittedTable<'_, F, E>], + out: &mut [Option>], +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + // A group's tables drop in one class only ([`drop_one_class`]). A table + // held as its records drops as them, and is kept where it has no rank. + let mut ranks: Vec> = (0..tables.len()) + .map(|k| { + tables[k].compact_bytes()?; + spill.regen.as_ref()?.rank_of(first + k) + }) + .collect(); + let before = ranks.clone(); + drop_one_class(&mut ranks); + if let Some(regen) = &spill.regen { + let held: Vec<(usize, u64)> = before + .iter() + .zip(&ranks) + .filter_map(|(b, a)| b.filter(|_| a.is_none())) + .collect(); + if !held.is_empty() { + regen.held(&held); + } + } + for (k, (table, slot)) in tables.iter_mut().zip(out.iter_mut()).enumerate() { + let table_cells = (table.num_committed_columns() as u64) << table.num_vars(); + let cells = spill.cells.fetch_add(table_cells, Relaxed) + table_cells; + let Some(len) = table.compact_bytes() else { + continue; + }; + let rank = ranks[k]; + // Only a table phase B can rebuild leaves its group. + let (Some(regen), Some(_)) = (&spill.regen, rank) else { + spill.kept.fetch_add(len as u64, Relaxed); + continue; + }; + let failed = || MlError::SpillFailed { + table: first + k, + reason: "its packed parts did not move out of the table", + }; + let held = if table.records().is_some() { + ParkedState::ResidentRecords(table.take_records_for_spill().ok_or_else(failed)?) + } else { + ParkedState::Resident(table.take_narrow_for_spill().ok_or_else(failed)?) + }; + let parked = Arc::new(Parked { + len: len as u64, + cells, + rank, + state: std::sync::Mutex::new(held), + }); + *slot = Some(Arc::clone(&parked)); + let wanted = !regen.always() && (spill.wanted)(spill.kept.load(Relaxed), cells, len as u64); + if wanted { + regen.arm_at(cells); + } + if regen.is_partial() && !regen.always() { + // Out only while the policy wants it out; else kept, and a + // candidate for a hand-off's drop-back. + if wanted { + regen.drop_one(parked); + } else { + spill.kept.fetch_add(len as u64, Relaxed); + regen.parked(parked); + } + continue; + } + if !regen.always() && !regen.is_armed() { + spill.kept.fetch_add(len as u64, Relaxed); + } + regen.parked(parked); + } + Ok(()) +} + +/// Brings `tables`' columns back before their upload, `first` being the +/// first's index: a parked table's from the host, a dropped table's from its +/// regeneration slot, waiting for its deposit. Refused when a deposit is not +/// what was dropped. +fn restore_group( + first: usize, + tables: &mut [CommittedTable<'_, F, E>], + out: &mut [Option>], + mem: Option<&BlockMem>, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + for (k, (table, slot)) in tables.iter_mut().zip(out.iter_mut()).enumerate() { + let Some(parked) = slot.take() else { + continue; + }; + let index = first + k; + let state = std::mem::replace( + &mut *parked.state.lock().unwrap_or_else(|e| e.into_inner()), + ParkedState::Lost, + ); + match state { + ParkedState::Resident(packed) => { + if !table.restore_narrow(packed) { + return Err(MlError::SpillFailed { + table: index, + reason: "the parked columns do not have the table's shape", + }); + } + } + ParkedState::ResidentRecords(records) => { + if !table.restore_records(records) { + return Err(MlError::SpillFailed { + table: index, + reason: "the parked records do not have the table's shape", + }); + } + } + ParkedState::Dropped(regen) => { + regen.wait(); + let packed = regen.take().map_err(|e| MlError::SpillFailed { + table: index, + reason: match e { + RegenError::Mismatch => "the regenerated columns are not the ones dropped", + RegenError::Failed(_) => "the regenerator did not bring its columns back", + RegenError::Closed(_) => { + "the regeneration window closed before its columns came back" + } + }, + })?; + let len = packed.data().len(); + if !table.restore_narrow(packed) { + return Err(MlError::SpillFailed { + table: index, + reason: "the regenerated columns do not have the table's shape", + }); + } + if let Some(mem) = mem { + mem.held_narrow.fetch_add(len, Relaxed); + } + } + // The table stays out, and the group's check refuses it. + ParkedState::Lost => {} + } + } + Ok(()) +} + +/// A table's committed columns as held on the host: packed, as records, or +/// eight bytes a cell. +fn held_bytes(table: &CommittedTable<'_, F, E>) -> usize +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + table.compact_bytes().unwrap_or_else(|| wide_bytes(table)) +} + +/// A group's columns put on the card for its commit: a group of wide tables +/// as their field elements, one with a table held narrow (built packed) or as +/// records as [`upload_group`] puts it — packed and widened on the card, or +/// filled there from the records. +fn upload_for_commit(group: &[CommittedTable<'_, F, E>]) -> Store +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + if group.iter().any(|t| t.compact_bytes().is_some()) { + return upload_group(group); + } + let columns: Vec<&Mle> = group.iter().flat_map(|t| t.columns()).collect(); + multilinear::gpu::upload_columns(&columns).map(Arc::new) +} + +/// A table's committed columns at eight bytes a cell. +fn wide_bytes(table: &CommittedTable<'_, F, E>) -> usize +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + (table.num_committed_columns() << table.num_vars()) * core::mem::size_of::>() +} + +/// A group's PREPARED opening in a block: one commitment, which both sides +/// derive from the program, over the leading preprocessed columns of the +/// group's prepared tables, opened on the group's fork after the group's own +/// opening — each table's columns at that table's point. +/// +/// It is the epoch's prepared opening ([`crate::multilinear_table::Prepared`]) +/// per group: its roots are DERIVED by the verifier and absorbed after the group +/// roots, before `z`; the columns stay in the group's stack too, and the opening +/// proves the derived commitment takes the values each table's own argument +/// settled on, at that table's point. One stack a group pays one chain for all +/// of its prepared tables. +pub struct BlockPrepared<'a, F, H> +where + F: IsFFTField + IsPrimeField + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + /// The group whose fork opens it. + pub group: usize, + /// The tables it settles, in stack order: each table's position in the + /// proof's table order and the length of its leading prefix. + pub tables: Vec<(usize, usize)>, + pub commitment: &'a StackedCommitment, + /// The tables' prefixes, concatenated in `tables`' order. + pub columns: &'a [&'a Mle], +} + +/// What the verifier settles a [`BlockPrepared`] opening against. +pub struct BlockPreparedCheck<'a, F> +where + F: IsFFTField + IsPrimeField + 'static, +{ + pub group: usize, + /// As [`BlockPrepared::tables`]: the prefixes `check_preprocessed` skips. + pub tables: Vec<(usize, usize)>, + /// Derived from the program by the verifier, never read from the proof. + pub roots: &'a [Commitment], + pub layout: &'a StackedLayout, + pub domain: &'a Domain, +} + +/// The prepared openings' shape, checked the same way by both sides: at most one +/// per group, in group order; each settles distinct tables of its own group in +/// increasing order, each prefix at least one column and at most `max_prefix` of +/// the table's; and the stack holds exactly those columns (`columns`, when the +/// caller knows them). +/// One prepared opening's shape: its group, its `(table, prefix)` list, and the +/// stack's column count when the caller knows it. +type PreparedEntry<'a> = (usize, &'a [(usize, usize)], Option); + +fn prepared_shape( + entries: &[PreparedEntry<'_>], + starts: &[usize], + num_tables: usize, + max_prefix: &dyn Fn(usize) -> usize, +) -> Result<(), MlError> { + let mut last_group: Option = None; + for &(group, tables, columns) in entries { + let range = starts + .get(group) + .copied() + .zip(starts.get(group + 1).copied().or(Some(num_tables))) + .ok_or(MlError::UnknownPolynomial { + index: group, + len: starts.len(), + })?; + if last_group.is_some_and(|l| group <= l) || tables.is_empty() { + return Err(MlError::UnknownPolynomial { + index: group, + len: starts.len(), + }); + } + last_group = Some(group); + let mut last: Option = None; + for &(table, n) in tables { + if table < range.0 || table >= range.1 || last.is_some_and(|l| table <= l) { + return Err(MlError::UnknownPolynomial { + index: table, + len: num_tables, + }); + } + if n == 0 || n > max_prefix(table) { + return Err(MlError::QueryCountMismatch { + expected: max_prefix(table), + got: n, + }); + } + last = Some(table); + } + let total: usize = tables.iter().map(|&(_, n)| n).sum(); + if columns.is_some_and(|c| c != total) { + return Err(MlError::QueryCountMismatch { + expected: total, + got: columns.unwrap_or(0), + }); + } + } + Ok(()) +} + +/// A group's prepared claims: per table of `tables`, its prefix's columns at the +/// table's point (one point per column) and the values its argument settled on. +/// `firsts` are the group's tables' first columns, `start` its first table. +#[allow(clippy::type_complexity)] +fn prepared_claims( + tables: &[(usize, usize)], + start: usize, + firsts: &[usize], + points: &[Vec>], + values: &[FieldElement], +) -> (Vec>>, Vec>) { + let mut at_points = Vec::new(); + let mut at_values = Vec::new(); + for &(table, n) in tables { + let first = firsts[table - start]; + at_points.extend_from_slice(&points[first..first + n]); + at_values.extend_from_slice(&values[first..first + n]); + } + (at_points, at_values) +} + +/// How a test makes the prover open a group's prepared stack wrongly, for the +/// verifier to refuse. +#[doc(hidden)] +#[derive(Clone, Debug, Default)] +pub struct PreparedDeviation { + /// Which opening, by its index in the `prepared` list. + pub open: usize, + /// `(table, other)`: open `table`'s block at `other`'s point (both positions + /// in the proof's table order, in the opening's group). + pub at_point_of: Vec<(usize, usize)>, + /// Claim every value as the committed column evaluated at its claimed point, + /// so the opening is internally consistent and only the verifier's binding + /// of it to the tables (and to its derived roots) refuses it. + pub consistent: bool, +} + +/// How a test makes the prover argue a group wrongly under the batched argue, +/// or argue every group on the host, for the card-vs-host byte gate. The +/// production prover takes [`ArgueDeviation::default`]: no fault, the card. +#[doc(hidden)] +#[derive(Clone, Copy, Debug)] +pub struct ArgueDeviation { + /// The group whose batched argue takes `faults`. + pub group: Option, + pub faults: ProverFaults, + /// Where every group's batched argue runs. + pub at: Where, +} + +impl Default for ArgueDeviation { + fn default() -> Self { + Self { + group: None, + faults: ProverFaults::default(), + at: Where::Device, + } + } +} + +/// Whether phase B opens each group beside the next group's argue +/// ([`BlockCommitted::with_phase_b_overlap`]). The proof is the same byte for +/// byte either way: each group proves on its own fork `S_post ‖ g`, so only +/// when its openings run moves. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum PhaseBOverlap { + /// One group at a time: its argue, then its openings. + #[default] + Off, + /// A group's openings on a thread of their own, drawing the card's + /// background streams, while the driver argues the next group; at most one + /// group's openings in flight, the last group's opened alone. After a + /// transition in which the card refused a promise, the rest of phase B goes + /// one group at a time. + On, + /// As [`Self::On`], and the openings' thread reads back from the card + /// through a pinned buffer with a blocking-sync event wait + /// ([`multilinear::gpu::pinned_readbacks`]): its read-backs leave the + /// driver's pageable path, where the argument's waited on them. The + /// argument's own read-backs are unchanged. + Pinned, +} + +/// How a test disturbs phase B's overlap ([`PhaseBOverlap::On`]): each failure +/// must come back as the prove's error — never a panic, never another proof — +/// and a refusal must turn the rest of phase B serial. Places are positions in +/// phase B's order. The production prover takes the default: none. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Default)] +pub struct OverlapFaults { + /// The openings of the group at this place fail, on whichever thread. + pub open_fails: Option, + /// The argue of the group at this place fails. + pub argue_fails: Option, + /// The opening thread of the group at this place panics (test builds). + pub open_panics: Option, + /// The opening thread waits this long before each group's openings. + pub open_stall: Option, + /// The transition that argues the group at this place reads as one in + /// which the card refused a promise. + pub refusal_at: Option, +} + +/// Every count of a promise the card refused, or of a wait on every stream at +/// once, summed: a transition of phase B's overlap in which it rose turns the +/// rest of phase B serial. The revive's room and an opening's turn refused, +/// the GKR trees and other argue work the card refused, and drain-and-trims. +/// Openings whose factors were built on the host are not among them: a small +/// stack declines the card by policy, with no refusal behind it. Each is 0 +/// without a device. +fn phase_b_refusals() -> u64 { + use multilinear::gpu; + [ + gpu::room_revive_refusals(), + gpu::room_turn_refusals(), + gpu::gkr_tree_refusals(), + gpu::argue_device_fallbacks(), + gpu::device_trims(), + ] + .into_iter() + .fold(0u64, u64::saturating_add) +} + +/// What phase B proves: the [`MultiProof`] (its `tables` empty under the +/// batched argue), the groups' batched argues (one per group under the batched +/// argue, none under the per-table one), the prepared openings in the +/// `prepared` list's order, and every group's stamps. +/// +/// The last element is every carried polynomial's commit-time out-of-domain +/// answer, in root order ([`commit_ood_point`]). +pub type BlockProved = ( + MultiProof, + Vec>, + Vec>, + Vec, + Vec>, +); + +/// The block after phase A: every table, the groups' roots and the tops of +/// their trees. The codewords are gone. +pub struct BlockCommitted<'a, F, E> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + tables: Vec>, + sizes: Vec, + groups: Vec>, + roots: Vec, + stamps: Vec, + /// A memory log's terms, counted through phase B as well. + mem: Option>, + /// The spill, when one is on, and each table's slot in it. + spill: Option, + /// Each table taken out of its group in phase A: parked, or dropped. + spilled: Vec>>, + /// Phase B's overlap ([`Self::with_phase_b_overlap`]), and a test's faults + /// in it. + overlap: PhaseBOverlap, + overlap_faults: OverlapFaults, + /// Phase B stamps its driver thread's schedstat a group turn + /// ([`Self::with_sched_stamps`]). + sched: bool, + /// Each group's commit-time out-of-domain claims, from its commit. + ood: Vec>, + /// A test's fault ([`Self::fault_commit_ood`]): no chain carries the + /// answers. + ood_unbound: bool, +} + +/// A test's fault in a block's commit-time answers +/// ([`BlockCommitted::fault_commit_ood`]). +#[doc(hidden)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OodFault { + /// The first answer made false, and carried by its chain as given. + False, + /// The first answer made false, and no chain carrying any answer: a + /// prover whose answers bind nothing. + FalseUnbound, +} + +impl<'a, F, E> BlockCommitted<'a, F, E> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + /// Phase A: commits each group in turn and lets its codewords go, keeping + /// each tree but its bottom `drop_levels` levels on the host. + /// + /// `sizes` must be [`block_groups`]'s over the same shapes and the + /// config's stack cap, or the verifier rebuilds other stacks. + /// + /// `T` is the block's transcript: each group's commit-time point is drawn + /// from a fresh one ([`commit_ood_point`]), as its commit ends. + pub fn commit( + tables: Vec>, + sizes: &[usize], + config: &ChainConfig, + drop_levels: multilinear::whir_commit::TreeDrop, + narrow: Narrowing, + upload_ahead: bool, + ) -> Result + where + H: WhirHash, + T: crypto::fiat_shamir::is_transcript::IsTranscript + Default, + { + if sizes.iter().sum::() != tables.len() { + return Err(MlError::QueryCountMismatch { + expected: tables.len(), + got: sizes.iter().sum(), + }); + } + let mut tables = tables.into_iter(); + let groups = sizes + .iter() + .map(|&size| tables.by_ref().take(size).collect::>()) + .collect::>(); + Self::commit_streamed::(groups, sizes, config, drop_levels, narrow, upload_ahead) + } + + /// [`Self::commit`] over groups handed over one at a time, in group order — + /// by a producer still preparing the later ones while the card commits + /// the earlier. Each group's wait for its tables is stamped (`wait_a`). A + /// producer that stops early leaves fewer groups than `sizes` names, which + /// is refused. + pub fn commit_streamed( + groups: impl IntoIterator>>, + sizes: &[usize], + config: &ChainConfig, + drop_levels: multilinear::whir_commit::TreeDrop, + narrow: Narrowing, + upload_ahead: bool, + ) -> Result + where + H: WhirHash, + T: crypto::fiat_shamir::is_transcript::IsTranscript + Default, + { + let block = Self::commit_groups::( + groups.into_iter().take(sizes.len()), + config, + drop_levels, + narrow, + upload_ahead, + )?; + if block.sizes != sizes { + return Err(MlError::QueryCountMismatch { + expected: sizes.len(), + got: block.sizes.len(), + }); + } + Ok(block) + } + + /// Phase A over whatever groups arrive, in arrival order, until the + /// producer stops: the groups (and so their sizes) are the prover's, and the + /// statement carries them. Each group's wait for its tables is stamped, and + /// each group is held as `narrow` says once committed. + /// + /// `upload_ahead`: the next group is taken, and its columns put on the card, + /// while this group commits (on a thread of its own) — once the commit has + /// asked the card for its room, so the upload only takes what is left; a + /// store the ledger refuses is uploaded after the commit, as without it. + /// The commits, their order and their bytes are the same either way. + pub fn commit_groups( + groups: impl IntoIterator>>, + config: &ChainConfig, + drop_levels: multilinear::whir_commit::TreeDrop, + narrow: Narrowing, + upload_ahead: bool, + ) -> Result + where + H: WhirHash, + T: crypto::fiat_shamir::is_transcript::IsTranscript + Default, + { + Self::commit_groups_logged::( + groups, + config, + drop_levels, + narrow, + upload_ahead, + None, + None, + ) + } + + /// [`Self::commit_groups`], counting where the columns are in `mem` (a + /// memory log, through phase B too) as they move. The commits are the same. + pub fn commit_groups_logged( + groups: impl IntoIterator>>, + config: &ChainConfig, + drop_levels: multilinear::whir_commit::TreeDrop, + narrow: Narrowing, + upload_ahead: bool, + mem: Option>, + spill: Option, + ) -> Result + where + H: WhirHash, + T: crypto::fiat_shamir::is_transcript::IsTranscript + Default, + { + let mut tables = Vec::new(); + let mut ood = Vec::new(); + let mut spilled: Vec>> = Vec::new(); + let mut sizes = Vec::new(); + let mut retired_groups = Vec::new(); + if let Some(regen) = spill.as_ref().and_then(|spill| spill.regen.as_ref()) { + regen.set_mem(mem.clone()); + } + let mut roots = Vec::new(); + let mut stamps = Vec::new(); + let mut incoming = groups.into_iter(); + let started = Instant::now(); + // The previous group's pack on the card, running beside this group's + // upload (or, uploading ahead, beside this group's commit). + let mut packing: Option = None; + // The next group, taken and uploaded beside this group's commit. + let mut ahead: Option> = None; + let mut exhausted = false; + loop { + let (mut group, mut stamp, store, wide) = match ahead.take() { + Some(next) => { + // A memory log: the group taken ahead is now the one in + // its commit. + if let Some(mem) = &mem { + mem.ahead.fetch_sub(next.wide, Relaxed); + mem.committing.fetch_add(next.wide, Relaxed); + } + let mut stamp = GroupStamps { + tables: next.group.len(), + wait_a: next.wait_paid, + upload_a: next.upload, + upload_paid: next.upload_paid, + ..Default::default() + }; + let store = match next.store { + Some(store) => Some(store), + None => { + // Refused beside the commit (or no card): up now, + // with the commit's room given back. + stamp.ahead_refused = multilinear::gpu::reserve_budget() > 0; + let t = Instant::now(); + let store = upload_for_commit(&next.group); + let secs = t.elapsed().as_secs_f64(); + stamp.upload_a += secs; + stamp.upload_paid += secs; + store + } + }; + (next.group, stamp, store, next.wide) + } + None => { + if exhausted { + break; + } + let waited = Instant::now(); + let Some(group) = incoming.next() else { + break; + }; + let mut stamp = GroupStamps { + tables: group.len(), + wait_a: waited.elapsed().as_secs_f64(), + ..Default::default() + }; + let wide = mem.as_ref().map_or(0, |mem| { + let held = group.iter().map(held_bytes).sum(); + mem.committing.fetch_add(held, Relaxed); + held + }); + let t = Instant::now(); + // Held in an `Arc` like phase B's: once the tree tops are + // home it goes to the group's packer, or is dropped. + let store = upload_for_commit(&group); + stamp.upload_a = t.elapsed().as_secs_f64(); + stamp.upload_paid = stamp.upload_a; + (group, stamp, store, wide) + } + }; + let size = group.len(); + if size == 0 { + return Err(MlError::QueryCountMismatch { + expected: 1, + got: 0, + }); + } + sizes.push(size); + let group_index = sizes.len() - 1; + if !upload_ahead && let Some(packed) = packing.take() { + install_and_spill( + packed, + &mut tables, + &mut spilled, + &mut stamps, + &sizes, + mem.as_deref(), + spill.as_ref(), + )?; + } + let shapes: Vec<(usize, usize)> = group + .iter() + .map(|t| (t.num_committed_columns(), t.num_vars())) + .collect(); + stamp.cells = shapes.iter().map(|&(w, n)| w << n).sum(); + let layout = global_layout(&shapes, config.format.stack)?; + stamp.polys = layout.num_polys(); + stamp.commit_base = multilinear::gpu::ledger_reserved(); + multilinear::gpu::reset_reserved_window(); + // Handles, not field elements: a table held narrow is read on the + // host only by a host fallback. + let handles = group_columns(&group); + let columns: Vec<&ColumnOf<'_, _>> = handles.iter().collect(); + let resident = store.as_ref().map(|store| (&**store, 0)); + let committed = if upload_ahead { + let (committed, next) = std::thread::scope(|scope| { + let (room_tx, room_rx) = std::sync::mpsc::channel::<()>(); + let columns = &columns; + let committer = scope.spawn(move || { + let signal = move || { + let _ = room_tx.send(()); + }; + let committed = commit_and_retire::( + group_index, + layout, + columns, + resident, + config, + drop_levels, + &signal, + ); + (committed, Instant::now()) + }); + // Beside the commit: the next group, then its columns once + // the commit has asked for its room (or ended). + let next = incoming.next(); + let arrived = Instant::now(); + let next = next.map(|group| { + let wide = mem.as_ref().map_or(0, |mem| { + let held = group.iter().map(held_bytes).sum(); + mem.ahead.fetch_add(held, Relaxed); + held + }); + let _ = room_rx.recv(); + let t = Instant::now(); + let store = upload_for_commit(&group); + ( + group, + store, + wide, + t.elapsed().as_secs_f64(), + Instant::now(), + ) + }); + let (committed, commit_end) = committer + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)); + let next = next.map(|(group, store, wide, upload, uploaded)| { + // What the committer waited for once its commit ended: + // the group's arrival, then the rest of its upload — + // never more than the upload (a commit that ended before + // the upload began paid all of it, and no more). + let ready = arrived.max(commit_end); + Ahead { + group, + store, + wide, + upload, + wait_paid: arrived.saturating_duration_since(commit_end).as_secs_f64(), + upload_paid: uploaded + .saturating_duration_since(ready) + .as_secs_f64() + .min(upload), + } + }); + (committed, next) + }); + exhausted = next.is_none(); + ahead = next; + committed + } else { + commit_and_retire::( + group_index, + layout, + &columns, + resident, + config, + drop_levels, + &|| {}, + ) + }; + let committed = committed?; + drop(columns); + drop(handles); + roots.extend(committed.roots); + let retired = committed.retired; + stamp.commit = committed.commit; + stamp.retire = committed.retire; + stamp.ood = committed.ood_secs; + stamp.ood_counts = committed.ood_counts; + ood.push(committed.ood); + stamp.commit_reserved = multilinear::gpu::reserved_window_peak(); + stamp.tree_bytes = retired.tree_bytes(); + stamp.committed_at = started.elapsed().as_secs_f64(); + if upload_ahead && let Some(packed) = packing.take() { + install_and_spill( + packed, + &mut tables, + &mut spilled, + &mut stamps, + &sizes, + mem.as_deref(), + spill.as_ref(), + )?; + } + // Which tables were committed wide (a packer's, from here on). + let was_wide: Vec = group.iter().map(|t| t.compact_bytes().is_none()).collect(); + packing = + narrow_group(&mut group, store, narrow, &mut stamp, mem.clone()).map(|handle| { + Packing { + first_table: tables.len(), + group: stamps.len(), + wide: group + .iter() + .zip(&was_wide) + .filter(|&(_, &w)| w) + .map(|(t, _)| wide_bytes(t)) + .sum(), + was_wide: was_wide.clone(), + handle, + } + }); + if let Some(mem) = &mem { + mem.committing.fetch_sub(wide, Relaxed); + mem.tree_tops.fetch_add(stamp.tree_bytes, Relaxed); + match &packing { + Some(packed) => { + mem.packing_wide.fetch_add(packed.wide, Relaxed); + // The tables committed narrow are held as they are. + mem.hold( + group + .iter() + .zip(&was_wide) + .filter(|&(_, &w)| !w) + .map(|(t, _)| t), + ); + } + None => mem.hold(group.iter()), + } + mem.mark(&format!("group {} committed", stamps.len())); + } + retired_groups.push(retired); + stamps.push(stamp); + let first = tables.len(); + tables.extend(group); + spilled.resize(tables.len(), None); + // A group no packer took is held as it is from here on. + if packing.is_none() + && let Some(spill) = &spill + { + spill_group(spill, first, &mut tables[first..], &mut spilled[first..])?; + } + } + if let Some(packed) = packing.take() { + install_and_spill( + packed, + &mut tables, + &mut spilled, + &mut stamps, + &sizes, + mem.as_deref(), + spill.as_ref(), + )?; + } + // Phase A's end: live regeneration's drops are joined. + if let Some(spill) = &spill { + spill.close_hand_off(); + } + Ok(Self { + tables, + sizes, + groups: retired_groups, + roots, + stamps, + mem, + spill, + spilled, + overlap: PhaseBOverlap::Off, + overlap_faults: OverlapFaults::default(), + sched: false, + ood, + ood_unbound: false, + }) + } + + /// Phase B reads its driver thread's CPU time and run-queue delay from + /// `/proc/thread-self/schedstat` at each group turn ([`GroupStamps::sched_b`]); + /// off unless set here. A report: the proof is the same. + pub fn with_sched_stamps(mut self, on: bool) -> Self { + self.sched = on; + self + } + + /// Phase B opens each group beside the next group's argue, or not + /// ([`PhaseBOverlap`]); `Off` unless set here. The proof is the same. + pub fn with_phase_b_overlap(mut self, overlap: PhaseBOverlap) -> Self { + self.overlap = overlap; + self + } + + /// A test's faults in phase B's overlap ([`OverlapFaults`]). + #[doc(hidden)] + pub fn with_overlap_faults(mut self, faults: OverlapFaults) -> Self { + self.overlap_faults = faults; + self + } + + /// Every group's roots, in group order — what the transcript absorbs. + pub fn roots(&self) -> &[Commitment] { + &self.roots + } + + pub fn sizes(&self) -> &[usize] { + &self.sizes + } + + pub fn tables(&self) -> &[CommittedTable<'a, F, E>] { + &self.tables + } + + /// Phase A's stamps, one per group. + pub fn stamps(&self) -> &[GroupStamps] { + &self.stamps + } + + /// By group, the class of the tables it dropped for phase B to rebuild + /// ([`BlockRegen`]), or `None`: what [`interleaved_order`] orders by. + /// Refused with the first group that dropped tables of two classes + /// ([`one_class`]): phase B could not take it in both classes' orders. + pub fn rebuilt_classes(&self) -> Result>, usize> { + let mut at = 0usize; + self.sizes + .iter() + .enumerate() + .map(|(g, &size)| { + let classes = self.spilled[at..at + size] + .iter() + .flatten() + .map(|parked| parked.dropped_class()); + at += size; + one_class(classes).ok_or(g) + }) + .collect() + } + + /// Live regeneration, when phase A ran with it. + pub fn regen(&self) -> Option<&BlockRegen> { + self.spill.as_ref().and_then(|spill| spill.regen.as_ref()) + } + + /// A test's fault in the commit-time answers ([`OodFault`]): the first + /// group's first answer made false — absorbed and sent as it is — and, + /// under [`OodFault::FalseUnbound`], every group's answers left out of its + /// chains. A false answer its chain carries is refused by the chain; one + /// no chain carries only by a verifier that batches the answers + /// (`VerifierChecks::commit_ood`). Returns whether there was an answer. + #[doc(hidden)] + pub fn fault_commit_ood(&mut self, fault: OodFault) -> bool { + let Some(first) = self + .ood + .first_mut() + .and_then(|claims| claims.answers.first_mut()) + else { + return false; + }; + *first += FieldElement::::one(); + self.ood_unbound = fault == OodFault::FalseUnbound; + true + } + + /// A test's fault: the first narrow table's width map made wrong + /// ([`multilinear::narrow::NarrowColumns::fault_width_map`]), so phase B + /// widens other words than were committed. Returns whether a table was + /// there to break. + #[doc(hidden)] + pub fn fault_narrow_width_map(&mut self) -> bool { + self.tables + .iter_mut() + .filter_map(|table| table.narrow_mut()) + .any(|packed| packed.fault_width_map()) + } +} + +/// The next group, taken and uploaded beside the current group's commit +/// ([`BlockCommitted::commit_groups`]'s `upload_ahead`): its tables, its store +/// (`None` when the ledger refused it, or there is no card), the upload's +/// seconds, and what of the wait and of the upload outlasted the commit. +struct Ahead<'a, F, E> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + group: Vec>, + store: Store, + /// Its columns at eight bytes a cell (counted only for a memory log). + wide: usize, + upload: f64, + wait_paid: f64, + upload_paid: f64, +} + +/// A group committed and retired ([`commit_and_retire`]). +struct CommittedGroup { + roots: Vec, + retired: RetiredStack, + /// The commit's and the retire's seconds. + commit: f64, + retire: f64, + /// Its commit-time out-of-domain claims, their seconds and where their + /// columns were evaluated. + ood: OodClaims, + ood_secs: f64, + ood_counts: OodCounts, +} + +/// Group `group` committed and retired, and its commit-time out-of-domain +/// point answered ([`commit_ood_point`]) from its columns while they are still +/// where the commit read them — on the card, in place, when `resident` holds +/// them: after the retire, so the codewords' room is back, and before the +/// store goes to the packer. `on_room` as +/// [`StackedCommitment::commit_signalled`]. +#[allow(clippy::too_many_arguments)] +fn commit_and_retire( + group: usize, + layout: StackedLayout, + columns: &[&C], + resident: Option<(&multilinear::gpu::ResidentColumns, usize)>, + config: &ChainConfig, + drop_levels: multilinear::whir_commit::TreeDrop, + on_room: &dyn Fn(), +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + T: crypto::fiat_shamir::is_transcript::IsTranscript + Default, + C: multilinear::narrow::HostColumn, + FieldElement: AsBytes + Sync + Send, + FieldElement: Send + Sync, +{ + let t = Instant::now(); + let stacked = multilinear::stacked_eval::with_pairs_off_pool(commit_pairs_off_pool(), || { + StackedCommitment::::commit_signalled( + layout, + columns, + resident.map(|(store, first)| (store, ColumnsAt::From(first))), + config, + on_room, + ) + })?; + let roots = stacked.roots(); + let commit = t.elapsed().as_secs_f64(); + let t = Instant::now(); + let retired = stacked.retire(drop_levels, config)?; + let retire = t.elapsed().as_secs_f64(); + let t = Instant::now(); + let z0 = commit_ood_point::(group, retired.layout(), &roots, config)?; + let (ood, ood_counts) = stacked_eval::ood_claims::( + retired.layout(), + columns, + resident.map(|(store, first)| (store, ColumnsAt::From(first))), + &z0, + )?; + Ok(CommittedGroup { + roots, + retired, + commit, + retire, + ood, + ood_secs: t.elapsed().as_secs_f64(), + ood_counts, + }) +} + +/// The calling thread's CPU time and run-queue delay so far, in ns, from +/// `/proc/thread-self/schedstat` (Linux); `None` where the file does not read. +/// A kernel without scheduler statistics prints zeros. +fn thread_schedstat() -> Option<(u64, u64)> { + parse_schedstat(&std::fs::read_to_string("/proc/thread-self/schedstat").ok()?) +} + +/// A schedstat line's first two fields: CPU ns and run-queue delay ns. +fn parse_schedstat(text: &str) -> Option<(u64, u64)> { + let mut fields = text.split_whitespace().map(|w| w.parse::().ok()); + Some((fields.next()??, fields.next()??)) +} + +/// `LAMBDA_VM_BLOCK_COMMIT_OFF_POOL`: phase A commits each group's pairs of +/// polynomials on the committer's own threads (unset or anything but `0`, the +/// default), or on the global rayon pool (`0`, the opt-out). The roots are the +/// same either way. The committer is not a pool thread, so on the pool its pair +/// waits in the injector behind every job the finish's generators queue +/// (rayon-core's `find_work`: own deque, then stealing, then injected jobs), +/// and the card waits with it: at the median block one commit took 2.7–2.9 s +/// against 0.68 s off the pool (ULTRA 096). Read once. +pub fn commit_pairs_off_pool() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + commit_pairs_off_pool_from( + std::env::var("LAMBDA_VM_BLOCK_COMMIT_OFF_POOL") + .ok() + .as_deref(), + ) + }) +} + +/// [`commit_pairs_off_pool`]'s reading of its variable. +fn commit_pairs_off_pool_from(value: Option<&str>) -> bool { + value.is_none_or(|v| v.trim() != "0") +} + +/// What a packer hands back: one packed table or none per table of its group, +/// and its own seconds. +type Packed = (Vec>, f64); + +/// A group's tables being packed on the card, on a thread of their own: +/// where the group's tables start in the block, its stamp, and the packer. +struct Packing { + first_table: usize, + group: usize, + /// The group's columns committed wide, at eight bytes a cell, and which + /// tables those are (counted only for a memory log). + wide: usize, + was_wide: Vec, + handle: std::thread::JoinHandle, +} + +impl Packing { + /// Waits for the packer and holds each table it packed narrow from here + /// on. A packer that failed leaves its tables wide. + fn install( + self, + tables: &mut [CommittedTable<'_, F, E>], + stamps: &mut [GroupStamps], + mem: Option<&BlockMem>, + ) where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + let t = Instant::now(); + let joined = self.handle.join(); + let stamp = &mut stamps[self.group]; + stamp.pack += t.elapsed().as_secs_f64(); + let installed = match joined { + Ok((packed, busy)) => { + stamp.pack_busy = busy; + let ready: usize = packed.iter().flatten().map(|p| p.data().len()).sum(); + for (table, packed) in tables[self.first_table..].iter_mut().zip(packed) { + let Some(packed) = packed else { + continue; + }; + let bytes = packed.data().len(); + if table.install_narrow(packed) { + stamp.packed_tables += 1; + stamp.packed_cells += table.num_committed_columns() << table.num_vars(); + stamp.packed_bytes += bytes; + } + } + ready + } + Err(_) => 0, + }; + if let Some(mem) = mem { + mem.packed_ready.fetch_sub(installed, Relaxed); + mem.packing_wide.fetch_sub(self.wide, Relaxed); + mem.hold( + tables[self.first_table..] + .iter() + .zip(&self.was_wide) + .filter(|&(_, &w)| w) + .map(|(t, _)| t), + ); + mem.mark(&format!("group {} installed", self.group)); + } + } +} + +/// Installs a packer's tables ([`Packing::install`]), then hands its group to +/// the spill when one is on. +fn install_and_spill( + packed: Packing, + tables: &mut [CommittedTable<'_, F, E>], + spilled: &mut [Option>], + stamps: &mut [GroupStamps], + sizes: &[usize], + mem: Option<&BlockMem>, + spill: Option<&BlockSpill>, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + let (g, first) = (packed.group, packed.first_table); + packed.install(tables, stamps, mem); + let Some(spill) = spill else { + return Ok(()); + }; + let end = first + sizes[g]; + spill_group( + spill, + first, + &mut tables[first..end], + &mut spilled[first..end], + ) +} + +/// Holds a just-committed group's tables as `narrow` says. On the card, each +/// table of at least `min_cells` cells is packed from its run in `store` by a +/// thread of its own, returned for the caller to [`Packing::install`] once the +/// next group's columns are up: the committer is phase A's critical path, and +/// the pack's download overlaps that upload. On the host, every table is packed +/// here. A table that cannot be packed stays as it is. +fn narrow_group( + group: &mut [CommittedTable<'_, F, E>], + store: Store, + narrow: Narrowing, + stamp: &mut GroupStamps, + mem: Option>, +) -> Option> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + match (narrow, store) { + (Narrowing::Wide, _) | (Narrowing::Card { .. }, None) => None, + (Narrowing::Card { min_cells }, Some(store)) => { + // Each table's run in the store, if it is to be packed. + let mut first = 0usize; + let runs: Vec> = group + .iter() + .map(|table| { + let width = table.num_committed_columns(); + // A table built packed is held narrow already, and one + // held as records keeps them. + let run = (table.compact_bytes().is_none() + && (width << table.num_vars()) >= min_cells) + .then_some((first, width)); + first += width; + run + }) + .collect(); + Some(std::thread::spawn(move || { + let t = Instant::now(); + let packed: Vec> = runs + .into_iter() + .map(|run| { + run.and_then(|(first, width)| { + multilinear::gpu::pack_resident(&store, first, width) + }) + }) + .collect(); + if let Some(mem) = mem { + let ready = packed.iter().flatten().map(|p| p.data().len()).sum(); + mem.packed_ready.fetch_add(ready, Relaxed); + } + (packed, t.elapsed().as_secs_f64()) + })) + } + (Narrowing::Host, _) => { + let t = Instant::now(); + for table in group.iter_mut() { + if table.pack_on_host() { + stamp.packed_tables += 1; + stamp.packed_cells += table.num_committed_columns() << table.num_vars(); + stamp.packed_bytes += table.narrow().map_or(0, |packed| packed.data().len()); + } + } + stamp.pack = t.elapsed().as_secs_f64(); + stamp.pack_busy = stamp.pack; + None + } + } +} + +/// A group's columns on the card, shared by its tables. +type Store = Option>; + +/// Uploads a group's columns, in table order — a narrow table's packed, and +/// widened on the card; `None` when the card declines (every reader then takes +/// the host copy, widening a narrow table's there). +fn upload_group(group: &[CommittedTable<'_, F, E>]) -> Store +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + let tables: Vec> = + group.iter().map(|t| t.upload_view()).collect(); + multilinear::gpu::upload_tables(&tables).map(Arc::new) +} + +/// A group's columns in table order, each read on the host only by a path +/// that asks ([`multilinear::narrow::HostColumn`]): the revive and the opening take them so, and +/// the card holds them. +fn group_columns<'t, F, E>( + group: &'t [CommittedTable<'_, F, E>], +) -> Vec>> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + group + .iter() + .flat_map(|table| { + (0..table.num_committed_columns()).map(move |index| ColumnOf { + columns: table.trace(), + index, + }) + }) + .collect() +} + +/// The fork a group proves on: `S_post` then the group's index. +fn group_fork(transcript: &T, group: usize) -> T +where + E: IsField + 'static, + T: crypto::fiat_shamir::is_transcript::IsTranscript + Clone, +{ + let mut fork = transcript.clone(); + fork.append_bytes(&(group as u64).to_le_bytes()); + fork +} + +/// Phase B and the proof: the roots block on `transcript`, then each group on +/// its own fork. Consumes the block — each group's host columns and kept tree +/// top are released as its opening ends. Returns the proof and every group's +/// stamps (phase A's, completed by phase B's). +/// +/// The proof is a [`MultiProof`] — the roots, a table proof per table in +/// order, an opening per group — read under the forked schedule by +/// [`block_verify`] and by nothing else. +pub fn block_prove( + committed: BlockCommitted<'_, F, E>, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(MultiProof, Vec), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript> + + Clone + + Send, +{ + // The batched argue's messages are not a `MultiProof`'s: its callers take + // them from `block_prove_on_forks`. + if config.format.argue != ArgueFormat::PerTable { + return Err(MlError::ArgueFormatMismatch); + } + let (proof, _, _, stamps, _) = block_prove_on_forks::( + committed, + config, + transcript, + &[], + &[], + &|g| g, + &ArgueDeviation::default(), + )?; + Ok((proof, stamps)) +} + +/// [`block_prove`] with the tables' prepared openings, and group `g` proved on +/// the fork of index `fork_of(g)`, under the config's argue format +/// ([`BlockProved`]). +/// +/// Only a test proves on a fork map other than the identity, or with an +/// [`ArgueDeviation`] other than the default: it is how a proof whose groups +/// sit on the wrong forks, or argue wrongly, is built, for the verifier to +/// refuse. +#[doc(hidden)] +pub fn block_prove_on_forks( + committed: BlockCommitted<'_, F, E>, + config: &ChainConfig, + transcript: &mut T, + prepared: &[BlockPrepared<'_, F, H>], + deviations: &[PreparedDeviation], + fork_of: &dyn Fn(usize) -> usize, + argue: &ArgueDeviation, +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript> + + Clone + + Send, +{ + block_prove_on_forks_observed::( + committed, + config, + transcript, + prepared, + deviations, + fork_of, + argue, + &|_| {}, + ) +} + +/// One group's share of the proof, the moment its opening is done: what a +/// consumer of the finished groups (the block tree's first leaf) reads before +/// the rest of phase B ends. Every field is a part of the proof as +/// [`block_prove_on_forks`] returns it. +pub struct GroupOpened<'a, F: IsField, E: IsField> { + pub group: usize, + /// Every group's root, as the proof carries them. + pub roots: &'a [Commitment], + /// Every carried polynomial's commit-time out-of-domain answer, in root + /// order, as the proof carries them. + pub commit_ood: &'a [FieldElement], + /// The group's batched argue (the batched format), or `None`. + pub argue: Option<&'a BatchedArgue>, + /// The group's tables' proofs, in table order (the per-table format; + /// empty under the batched one). + pub tables: &'a [TableProof], + /// The group's opening. + pub opening: &'a StackedProof, + /// The group's prepared opening, when the group carries one. + pub prepared: Option<&'a StackedProof>, +} + +/// [`block_prove_on_forks`], calling `on_group` after each group's opening +/// with that group's share of the proof ([`GroupOpened`]). The observer reads; +/// the proof is the same with any observer. +#[allow(clippy::too_many_arguments)] +#[doc(hidden)] +pub fn block_prove_on_forks_observed( + committed: BlockCommitted<'_, F, E>, + config: &ChainConfig, + transcript: &mut T, + prepared: &[BlockPrepared<'_, F, H>], + deviations: &[PreparedDeviation], + fork_of: &dyn Fn(usize) -> usize, + argue: &ArgueDeviation, + on_group: &(dyn Fn(GroupOpened<'_, F, E>) + Sync), +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript> + + Clone + + Send, +{ + let order: Vec = (0..committed.sizes.len()).collect(); + block_prove_in_order::( + committed, config, transcript, prepared, deviations, fork_of, argue, on_group, &order, None, + ) +} + +/// Phase B's re-ranking hook ([`block_prove_in_order`]): at each group's turn +/// `p` it is handed phase B's whole order as it stands (`order[p]` is the group +/// whose turn it is) and may return the groups after it, `order[p + 1..]`, in +/// another order; `None` keeps them. The proof is the same in any order, so a +/// caller may decide the rest of the order from what it learns while phase B +/// runs (the recursion's leaves, `W3_LEAF_FIRST`). +pub type PhaseBReorder<'a> = dyn Fn(usize, &[usize]) -> Option> + Sync + 'a; + +/// The groups that hold a table whose columns phase B rebuilds rather than +/// reads back (`rebuilt`, by group) last, each set in group order: an order +/// for [`block_prove_in_order`] that gives the rebuilding the other groups' +/// phase B as a head start. With nothing rebuilt it is the group order. +pub fn rebuilt_last(rebuilt: &[bool]) -> Vec { + (0..rebuilt.len()) + .filter(|&g| !rebuilt[g]) + .chain((0..rebuilt.len()).filter(|&g| rebuilt[g])) + .collect() +} + +/// The ranks of a group's tables that drop, `ranks` holding each table's rank +/// in its class (`None` for one no regenerator rebuilds): those of the class of +/// its first droppable table only. Phase B takes each class's groups in that +/// class's rank order, and a group of two classes would have to come last in +/// one and first in the other; the other class's tables in it stay. +fn drop_one_class(ranks: &mut [Option<(usize, u64)>]) { + let class = ranks.iter().flatten().next().map(|&(class, _)| class); + for rank in ranks { + if rank.is_some_and(|(c, _)| Some(c) != class) { + *rank = None; + } + } +} + +/// The one class among a group's tables' dropped classes (`None` for a table +/// not dropped), `Some(None)` when none dropped, `None` when two classes did. +fn one_class(classes: impl IntoIterator>) -> Option> { + let mut one = None; + for class in classes.into_iter().flatten() { + match one { + Some(c) if c != class => return None, + _ => one = Some(class), + } + } + Some(one) +} + +/// Phase B's order with two classes rebuilt (D-WHIR-NODISK N3): the groups +/// with nothing dropped first, in group order (they need no regenerator, and +/// give every regenerator a head start); then the rebuilt groups, each class +/// in group order (its rank order), the classes interleaved so each class's +/// share of the `weights` taken so far stays at its share of the whole. Each +/// regenerator then has to keep up with only its share of phase B's pace. With +/// one class it is [`rebuilt_last`]. +pub fn interleaved_order(classes: &[Option], weights: &[u64]) -> Vec { + let mut order: Vec = (0..classes.len()) + .filter(|&g| classes[g].is_none()) + .collect(); + let n = classes.iter().flatten().max().map_or(0, |&c| c + 1); + let lists: Vec> = (0..n) + .map(|c| { + (0..classes.len()) + .filter(|&g| classes[g] == Some(c)) + .collect() + }) + .collect(); + let weight = |g: usize| weights.get(g).copied().unwrap_or(0).max(1) as u128; + let totals: Vec = lists + .iter() + .map(|l| l.iter().map(|&g| weight(g)).sum()) + .collect(); + let mut next = vec![0usize; n]; + let mut taken = vec![0u128; n]; + while (0..n).any(|c| next[c] < lists[c].len()) { + // The class furthest behind its share: taken_c / total_c the least + // (compared without division), ties to the lower class. + let Some(c) = (0..n) + .filter(|&c| next[c] < lists[c].len()) + .min_by(|&a, &b| (taken[a] * totals[b]).cmp(&(taken[b] * totals[a]))) + else { + break; + }; + let g = lists[c][next[c]]; + next[c] += 1; + taken[c] += weight(g); + order.push(g); + } + order +} + +/// A group argued and not yet opened: what its openings read, all of it its +/// own — its fork, its kept tree tops, its columns on the card and the claims +/// its argue settled on — and its share of the proof so far. +struct Argued +where + F: IsFFTField + IsPrimeField + 'static, + E: IsField, +{ + group: usize, + /// Its place in phase B's order. + place: usize, + /// Its first table, and how many it has. + at: usize, + size: usize, + fork: T, + /// Taken by its openings. + retired: Option>, + store: Store, + tree_tops: usize, + points: Vec>>, + values: Vec>, + proofs: Vec>, + argue: Option>, + /// Its commit-time out-of-domain claims, batched into its opening + /// (`None` only under a test's [`OodFault::FalseUnbound`]). + ood: Option>, +} + +/// A group's openings done: its opening, its prepared one when it has one, +/// and what they cost. +struct Opened { + opening: StackedProof, + prepared: Option>, + encode: f64, + open: f64, + /// What the revive added to the ledger, and the ledger's peak through the + /// openings: read only when the openings had the ledger's window to + /// themselves (0 and `None` otherwise). + open_room: u64, + open_reserved: Option, + /// Stream draws while they ran, `(foreground pool, background pool)`. + draws: (u64, u64), +} + +/// A group's openings on its own fork: its stack revived over its columns (on +/// the card when its store holds them), opened at the points and values its +/// argue settled on, then its prepared stack when it has one. Returns them, +/// and the revived stack for the caller to let go. +/// +/// It reads the group's own tables, fork, claims and kept tree tops, nothing +/// another group proves, so phase B may run it on a thread of its own beside +/// the next group's argue ([`PhaseBOverlap::On`]): nothing it calls may read a +/// setting the driver's thread holds per thread. `own_window`: the ledger's +/// window is the openings' alone (one group at a time), so it is opened here +/// and its peak read. `fails`: a test's failed opening. +#[allow(clippy::too_many_arguments, clippy::type_complexity)] +fn open_group( + argued: &mut Argued, + group: &[CommittedTable<'_, F, E>], + prepared: &[BlockPrepared<'_, F, H>], + deviations: &[PreparedDeviation], + config: &ChainConfig, + own_window: bool, + fails: bool, +) -> Result<(Opened, StackedCommitment), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + if fails { + return Err(MlError::DeviceFailed { + stage: "a test's failed opening", + }); + } + let retired = argued.retired.take().ok_or(MlError::QueryCountMismatch { + expected: 1, + got: 0, + })?; + let draws = multilinear::gpu::stream_draws(); + let handles = group_columns(group); + let columns: Vec<&ColumnOf<'_, _>> = handles.iter().collect(); + let store = argued.store.as_ref(); + let t = Instant::now(); + if own_window { + multilinear::gpu::reset_reserved_window(); + } + let before_revive = own_window.then(multilinear::gpu::ledger_reserved); + let stacked = retired.revive::( + &columns, + store.map(|store| (&**store, ColumnsAt::From(0))), + config, + )?; + let encode = t.elapsed().as_secs_f64(); + let open_room = before_revive.map_or(0, |before| { + multilinear::gpu::ledger_reserved().saturating_sub(before) + }); + let t = Instant::now(); + let opening = stacked_eval::prove_mapped_ood::( + &stacked, + &columns, + store.map(|store| (&**store, ColumnsAt::From(0))), + &Claimed::PerColumn(&argued.points), + &argued.values, + argued.ood.as_ref(), + config, + &mut argued.fork, + )?; + // The group's prepared tables, in table order: each opened at its own + // point for its own prefix, on this fork. + let (at, points, values) = (argued.at, &argued.points, &argued.values); + let firsts: Vec = group + .iter() + .scan(0usize, |first, table| { + let at = *first; + *first += table.num_committed_columns(); + Some(at) + }) + .collect(); + let prepared_opening = match prepared.iter().position(|p| p.group == argued.group) { + Some(k) => { + let p = &prepared[k]; + let (mut at_points, mut at_values) = + prepared_claims(&p.tables, at, &firsts, points, values); + if let Some(d) = deviations.iter().find(|d| d.open == k) { + // Each block's columns sit at its table's place in the stack. + let mut column = 0usize; + for &(table, n) in &p.tables { + if let Some(&(_, other)) = d.at_point_of.iter().find(|(t, _)| *t == table) { + let local = other.checked_sub(at).filter(|&l| l < firsts.len()).ok_or( + MlError::UnknownPolynomial { + index: other, + len: firsts.len(), + }, + )?; + for point in &mut at_points[column..column + n] { + *point = points[firsts[local]].clone(); + } + } + column += n; + } + if d.consistent { + at_values = p + .columns + .iter() + .zip(&at_points) + .map(|(c, point)| c.evaluate_in(point)) + .collect::>()?; + } + } + Some(stacked_eval::prove::( + p.commitment, + p.columns, + None, + &Claimed::PerColumn(&at_points), + &at_values, + config, + &mut argued.fork, + )?) + } + None => None, + }; + let open = t.elapsed().as_secs_f64(); + let open_reserved = own_window.then(multilinear::gpu::reserved_window_peak); + let after = multilinear::gpu::stream_draws(); + Ok(( + Opened { + opening, + prepared: prepared_opening, + encode, + open, + open_room, + open_reserved, + draws: ( + after.0.saturating_sub(draws.0), + after.1.saturating_sub(draws.1), + ), + }, + stacked, + )) +} + +/// Phase B's finished groups' records, let go off the driver thread +/// ([`release_group`]): one thread of its own hands each table's records' +/// pages back to the OS ([`multilinear::records::Records::return_pages`]) and +/// frees them, so phase B neither waits on that nor on the frees. Joined before +/// phase B returns ([`Self::finish`]), and when dropped, so a prove's time and +/// its stamps hold all of it. +struct RecordsReturner { + tx: Option>, + thread: Option>>, +} + +impl RecordsReturner { + /// Its thread, started; without one (the spawn refused) records are + /// dropped where they are let go, nothing handed back. + fn start() -> Self { + let (tx, rx) = std::sync::mpsc::channel::<(usize, multilinear::records::Records)>(); + let thread = std::thread::Builder::new() + .name("records-return".to_string()) + .spawn(move || { + rx.into_iter() + .map(|(g, records)| { + let t = Instant::now(); + let pages = records.return_pages(); + (g, pages, t.elapsed().as_secs_f64()) + }) + .collect() + }) + .ok(); + Self { + tx: thread.is_some().then_some(tx), + thread, + } + } + + /// Group `g`'s records, handed to the thread (dropped here without one). + fn take(&self, g: usize, records: multilinear::records::Records) { + if let Some(tx) = &self.tx { + // A thread that is gone hands the records back in the error, and + // they drop here. + let _ = tx.send((g, records)); + } + } + + /// Waits for the thread to finish what it was handed, and adds what each + /// group's records came to to its stamps. + fn finish(mut self, stamps: &mut [GroupStamps]) { + self.tx = None; + let Some(thread) = self.thread.take() else { + return; + }; + for (g, pages, secs) in thread.join().unwrap_or_default() { + if let Some(stamp) = stamps.get_mut(g) { + stamp.records_pages += pages; + stamp.records_return_secs += secs; + } + } + } +} + +impl Drop for RecordsReturner { + fn drop(&mut self) { + self.tx = None; + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +/// Lets a group go once its openings are done: its tables' columns off the +/// card and out of their widened form — under a spill its packed columns too, +/// and the records of a table held as them, which nothing reads after its +/// openings — its store, and its kept tree tops out of the memory log. The +/// bytes of records let go; the records themselves go to `returner`, which +/// hands their pages back to the OS off this thread. +fn release_group( + group: &mut [CommittedTable<'_, F, E>], + g: usize, + store: Store, + tree_tops: usize, + spilling: bool, + returner: Option<&RecordsReturner>, + mem: Option<&BlockMem>, +) -> usize +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + let mut records_released = 0; + for table in group.iter_mut() { + table.clear_resident(); + table.drop_widened(); + // While a spill is on, a finished group's packed columns go, and so do + // the records of a table held as them: nothing reads either after its + // opening. + if !spilling { + continue; + } + let released = match table.take_narrow_for_spill() { + Some(packed) => Some(packed.data().len()), + None => table.take_records_for_spill().map(|records| { + let bytes = records.bytes(); + records_released += bytes; + match returner { + Some(returner) => returner.take(g, records), + None => drop(records), + } + bytes + }), + }; + if let (Some(bytes), Some(mem)) = (released, mem) { + mem.held_narrow.fetch_sub(bytes, Relaxed); + } + } + drop(store); + if let Some(mem) = mem { + mem.tree_tops.fetch_sub(tree_tops, Relaxed); + mem.mark(&format!("phase B group {g} end")); + } + records_released +} + +/// [`block_prove_on_forks_observed`] with phase B taking the groups in +/// `order`, a permutation of their indices. Each group proves on its own fork +/// `S_post ‖ g` and nothing one group proves enters another's transcript, so +/// the proof — each group's share, assembled in group order — is the same in +/// any order; only when each group's columns are needed moves. The next +/// group's columns go up beside this group's argue in `order` too. An `order` +/// that is not a permutation of the groups is refused. +/// +/// Under [`PhaseBOverlap::On`] ([`BlockCommitted::with_phase_b_overlap`]) each +/// group's openings run on a thread of their own beside the next group's +/// argue, which keeps each fork's own sequence — argue, openings, prepared +/// opening — so the proof is the same again. `on_group` is then called on that +/// thread, as each group's openings end, in `order` all the same. +/// +/// `reorder`, when given, may re-rank the groups still to come at each turn, +/// before the next group's columns go up ([`PhaseBReorder`]); a re-ranking that +/// is not a permutation of those groups is refused. +#[allow(clippy::too_many_arguments)] +#[doc(hidden)] +pub fn block_prove_in_order( + committed: BlockCommitted<'_, F, E>, + config: &ChainConfig, + transcript: &mut T, + prepared: &[BlockPrepared<'_, F, H>], + deviations: &[PreparedDeviation], + fork_of: &dyn Fn(usize) -> usize, + argue: &ArgueDeviation, + on_group: &(dyn Fn(GroupOpened<'_, F, E>) + Sync), + order: &[usize], + reorder: Option<&PhaseBReorder<'_>>, +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript> + + Clone + + Send, +{ + let BlockCommitted { + mut tables, + sizes, + groups, + roots, + mut stamps, + mem, + spill, + mut spilled, + overlap, + overlap_faults, + sched, + ood, + ood_unbound, + } = committed; + let n = sizes.len(); + let mut seen = vec![false; n]; + if order.len() != n + || !order + .iter() + .all(|&g| g < n && !std::mem::replace(&mut seen[g], true)) + { + return Err(MlError::QueryCountMismatch { + expected: n, + got: order.len(), + }); + } + let starts: Vec = sizes + .iter() + .scan(0usize, |at, &size| { + let start = *at; + *at += size; + Some(start) + }) + .collect(); + let entries: Vec> = prepared + .iter() + .map(|p| (p.group, p.tables.as_slice(), Some(p.columns.len()))) + .collect(); + prepared_shape(&entries, &starts, tables.len(), &|t| { + tables[t].num_committed_columns() + })?; + if ood.len() != n { + return Err(MlError::CommitOodCount { + expected: n, + got: ood.len(), + }); + } + let entered = Instant::now(); + let derived: Vec = prepared.iter().flat_map(|p| p.commitment.roots()).collect(); + let answers: Vec> = ood + .iter() + .flat_map(|claims| claims.answers.iter().cloned()) + .collect(); + let (z, alpha, beta) = + absorb_roots_answers_and_challenge::(transcript, &roots, &derived, &answers); + let mut ood: Vec>> = ood.into_iter().map(Some).collect(); + + // Each group's share, by group, assembled in group order at the end. + let mut retired: Vec>> = groups.into_iter().map(Some).collect(); + let mut group_tables: Vec>> = (0..n).map(|_| Vec::new()).collect(); + let mut group_argue: Vec>> = (0..n).map(|_| None).collect(); + let mut group_opening: Vec>> = (0..n).map(|_| None).collect(); + let mut group_prepared: Vec>> = (0..n).map(|_| None).collect(); + // The next group's columns, uploaded during this group's argument — the + // card idles through the argument's host glue, and a group's store is a + // few GiB beside the argument's working set, not beside its codewords. + let mut pre_uploaded: Option = None; + // The finished groups' records go to a thread of their own (under a spill, + // the only time phase B lets them go). + let returner = spill.is_some().then(RecordsReturner::start); + // Under the overlap: the group argued last, whose openings run beside the + // next group's argue; and whether a transition saw the card refuse a + // promise, after which every group opens alone. + let overlapping = overlap != PhaseBOverlap::Off; + // Under `Pinned`, the openings' thread (a background-class one) reads back + // through pinned staging; this thread, the argument's, does not. + let _pinned = (overlap == PhaseBOverlap::Pinned).then(multilinear::gpu::pinned_readbacks); + let mut prev: Option> = None; + let mut latched = false; + // The driver thread's schedstat at the start of the turn under way, and + // its group: each turn's share is closed at the next turn's start. + let mut turn_sched: Option<((u64, u64), usize)> = None; + // The order as it stands: `reorder` may re-rank what is still to come. + let mut order: Vec = order.to_vec(); + for p in 0..n { + let g = order[p]; + stamps[g].start_b = entered.elapsed().as_secs_f64(); + if sched { + let now = thread_schedstat(); + if let (Some((then, h)), Some(now)) = (turn_sched, now) { + stamps[h].sched_b = + Some((now.0.saturating_sub(then.0), now.1.saturating_sub(then.1))); + } + turn_sched = now.map(|now| (now, g)); + } + let (at, size) = (starts[g], sizes[g]); + // The groups still to come, re-ranked before the next one is chosen + // (and before its columns go up): only a permutation of them. + if let Some(rest) = reorder.and_then(|reorder| reorder(p, &order)) { + let mut was = order[p + 1..].to_vec(); + let mut now = rest.clone(); + was.sort_unstable(); + now.sort_unstable(); + if was != now { + return Err(MlError::QueryCountMismatch { + expected: n - p - 1, + got: rest.len(), + }); + } + order[p + 1..].copy_from_slice(&rest); + } + let next_g = order.get(p + 1).copied(); + let mut fork = group_fork::(transcript, fork_of(g)); + if spill.is_some() { + // This group's columns and the next's, back before their uploads + // (the next one's goes up beside this group's argue). + for (first, len) in + std::iter::once((at, size)).chain(next_g.map(|next| (starts[next], sizes[next]))) + { + restore_group( + first, + &mut tables[first..first + len], + &mut spilled[first..first + len], + mem.as_deref(), + )?; + // No reader of these groups meets a table whose columns are + // still out: one whose slot did not come back is refused here, + // before its upload. + if let Some(k) = tables[first..first + len] + .iter() + .position(|t| t.is_spilled()) + { + return Err(MlError::SpillFailed { + table: first + k, + reason: "its columns are still spilled when its group is read", + }); + } + } + } + let retired = retired[g].take().ok_or(MlError::QueryCountMismatch { + expected: n, + got: order.len(), + })?; + + let t = Instant::now(); + let store = match pre_uploaded.take() { + Some(store) => store, + None => upload_group(&tables[at..at + size]), + }; + if let Some(store) = &store { + let mut first = 0usize; + for table in tables[at..at + size].iter_mut() { + table.set_resident(store.clone(), first); + first += table.num_committed_columns(); + } + } + stamps[g].upload_b += t.elapsed().as_secs_f64(); + + // The group's tables, one after another, each leaving its columns + // claimed at its own point — or, batched, all of them in one argue — + // with the next group's upload beside them, and under the overlap the + // previous group's openings too. + let t = Instant::now(); + stamps[g].argue_base = multilinear::gpu::ledger_reserved(); + multilinear::gpu::reset_reserved_window(); + let refusals_before = prev.as_ref().map(|_| phase_b_refusals()); + let mut points: Vec>> = Vec::new(); + let mut values: Vec> = Vec::new(); + let mut proofs: Vec> = Vec::new(); + let mut batched_argue: Option> = None; + let all: &[CommittedTable<'_, F, E>] = &tables; + let group_ref = &all[at..at + size]; + let next = next_g.map(|next| &all[starts[next]..starts[next] + sizes[next]]); + let roots_ref = &roots; + let answers_ref = &answers; + let (argue_result, argue_end, next_store, joined, opened, open_paid) = + std::thread::scope(|scope| { + let uploader = next.map(|next| scope.spawn(move || upload_group(next))); + let opener = prev.as_mut().map(|earlier| { + let its_tables = &all[earlier.at..earlier.at + earlier.size]; + scope.spawn(move || -> Result, MlError> { + // Every stream this thread draws is a background one, + // so the argue's kernels are dispatched first. + let _background = multilinear::gpu::background_streams(); + if let Some(stall) = overlap_faults.open_stall { + std::thread::sleep(stall); + } + #[cfg(any(test, feature = "test-utils"))] + if overlap_faults.open_panics == Some(earlier.place) { + panic!("a test's opening thread panics"); + } + let fails = overlap_faults.open_fails == Some(earlier.place); + let (opened, stacked) = open_group::( + earlier, its_tables, prepared, deviations, config, false, fails, + )?; + on_group(GroupOpened { + group: earlier.group, + roots: roots_ref, + commit_ood: answers_ref, + argue: earlier.argue.as_ref(), + tables: &earlier.proofs, + opening: &opened.opening, + prepared: opened.prepared.as_ref(), + }); + drop(stacked); + Ok(opened) + }) + }); + let argue_result = (|| -> Result<(), MlError> { + if overlap_faults.argue_fails == Some(p) { + return Err(MlError::DeviceFailed { + stage: "a test's failed argue", + }); + } + match config.format.argue { + ArgueFormat::PerTable => { + for table in group_ref.iter() { + let (proof, point) = + prove(table, &z, &alpha, &beta, &mut fork, None)?; + for _ in 0..table.num_committed_columns() { + points.push(point.clone()); + } + values + .extend(proof.constraint.reduce.column_values.iter().cloned()); + proofs.push(proof); + } + } + ArgueFormat::Batched { + bin_log_cells, + pack, + } => { + let faults = match argue.group { + Some(faulty) if faulty == g => argue.faults, + _ => ProverFaults::default(), + }; + let (proof, reduced) = batched::prove_argue( + group_ref, + &z, + &alpha, + &beta, + bin_log_cells, + pack, + &mut fork, + faults, + argue.at, + )?; + for (table, claim) in group_ref.iter().zip(reduced) { + for _ in 0..table.num_committed_columns() { + points.push(claim.point.clone()); + } + values.extend(claim.column_values); + } + batched_argue = Some(proof); + } + } + Ok(()) + })(); + let argue_end = Instant::now(); + let next_store = uploader.map(|handle| handle.join()); + let joined = argue_end.elapsed().as_secs_f64(); + // What of the previous group's openings outlasted the argue + // and the upload: the overlap's cost on this thread. + let waited = Instant::now(); + let opened = opener.map(|handle| handle.join()); + ( + argue_result, + argue_end, + next_store, + joined, + opened, + waited.elapsed().as_secs_f64(), + ) + }); + // The previous group's openings answer first: their error is the one + // one group at a time would have met first. A panic on their thread + // is refused, never resumed. + let opened = match opened { + None => None, + Some(Ok(Ok(opened))) => Some(opened), + Some(Ok(Err(e))) => return Err(e), + Some(Err(_)) => { + return Err(MlError::DeviceFailed { + stage: "the previous group's openings", + }); + } + }; + argue_result?; + stamps[g].argue = argue_end.saturating_duration_since(t).as_secs_f64(); + stamps[g].argue_reserved = multilinear::gpu::reserved_window_peak(); + if let (Some(next_store), Some(next)) = (next_store, next_g) { + // The wait for the upload after the argument ended is the next + // group's upload cost; the rest of it hid behind this argument. + stamps[next].upload_b += joined; + pre_uploaded = Some(next_store.map_err(|_| MlError::DeviceFailed { + stage: "uploading the next group's columns", + })?); + } + if let (Some(opened), Some(earlier)) = (opened, prev.take()) { + // The previous group's openings ran beside this argue: one window + // on the ledger for both. + let h = earlier.group; + let window = stamps[g].argue_reserved; + stamps[g].transition_reserved = window; + let s = &mut stamps[h]; + s.encode = opened.encode; + s.open = opened.open; + s.open_room = opened.open_room; + s.open_reserved = window; + s.open_paid = open_paid; + s.overlapped = true; + s.open_draws = opened.draws; + group_tables[h] = earlier.proofs; + group_argue[h] = earlier.argue; + group_opening[h] = Some(opened.opening); + group_prepared[h] = opened.prepared; + stamps[h].records_released = release_group( + &mut tables[earlier.at..earlier.at + earlier.size], + h, + earlier.store, + earlier.tree_tops, + spill.is_some(), + returner.as_ref(), + mem.as_deref(), + ); + stamps[h].end_b = entered.elapsed().as_secs_f64(); + // A transition in which the card refused a promise turns the rest + // of phase B serial: the work went where it goes without the + // overlap, and no later transition risks it again. + if refusals_before.is_some_and(|before| phase_b_refusals() > before) + || overlap_faults.refusal_at == Some(p) + { + latched = true; + } + } + + let tree_tops = retired.tree_bytes(); + let mut argued = Argued { + group: g, + place: p, + at, + size, + fork, + retired: Some(retired), + store, + tree_tops, + points, + values, + proofs, + argue: batched_argue, + ood: Some(ood[g].take().ok_or(MlError::QueryCountMismatch { + expected: n, + got: order.len(), + })?) + .filter(|_| !ood_unbound), + }; + if overlapping && !latched && next_g.is_some() { + // Opened beside the next group's argue. + prev = Some(argued); + continue; + } + stamps[g].serial_after_refusal = overlapping && latched && next_g.is_some(); + let paid = Instant::now(); + let (opened, stacked) = open_group::( + &mut argued, + &tables[at..at + size], + prepared, + deviations, + config, + true, + overlap_faults.open_fails == Some(p), + )?; + stamps[g].encode = opened.encode; + stamps[g].open_room = opened.open_room; + stamps[g].open = opened.open; + stamps[g].open_reserved = opened.open_reserved.unwrap_or(0); + stamps[g].open_paid = paid.elapsed().as_secs_f64(); + stamps[g].open_draws = opened.draws; + on_group(GroupOpened { + group: g, + roots: &roots, + commit_ood: &answers, + argue: argued.argue.as_ref(), + tables: &argued.proofs, + opening: &opened.opening, + prepared: opened.prepared.as_ref(), + }); + group_tables[g] = argued.proofs; + group_argue[g] = argued.argue; + group_opening[g] = Some(opened.opening); + group_prepared[g] = opened.prepared; + drop(stacked); + stamps[g].records_released = release_group( + &mut tables[at..at + size], + g, + argued.store, + argued.tree_tops, + spill.is_some(), + returner.as_ref(), + mem.as_deref(), + ); + stamps[g].end_b = entered.elapsed().as_secs_f64(); + } + if sched && let (Some((then, h)), Some(now)) = (turn_sched, thread_schedstat()) { + stamps[h].sched_b = Some((now.0.saturating_sub(then.0), now.1.saturating_sub(then.1))); + } + // Every finished group's records handed back before phase B returns. + if let Some(returner) = returner { + returner.finish(&mut stamps); + } + // The tables go with this function: a memory log stops holding them. + if let Some(mem) = &mem { + mem.release(tables.iter()); + } + let openings = group_opening + .into_iter() + .map(|opening| { + opening.ok_or(MlError::QueryCountMismatch { + expected: n, + got: order.len(), + }) + }) + .collect::, _>>()?; + Ok(( + MultiProof { + roots, + tables: group_tables.into_iter().flatten().collect(), + columns: openings, + preprocessed: None, + }, + group_argue.into_iter().flatten().collect(), + group_prepared.into_iter().flatten().collect(), + stamps, + answers, + )) +} + +/// Verifies a [`block_prove`] proof: the roots block, each group on its fork, +/// and the bus balance over every table of the block against `expected`. +/// +/// `layouts`/`domains` are one per group, rebuilt by the caller from the +/// shapes and `sizes` ([`block_groups`]) — never from the proof. `prepared` are +/// the tables' prepared checks the caller derived from the program, and +/// `prepared_openings` the proof's openings of them, one each, in that order. +/// `commit_ood` is the proof's commit-time answers ([`block_verify_with`]). +#[allow(clippy::too_many_arguments)] +pub fn block_verify( + proof: &MultiProof, + commit_ood: &[FieldElement], + prepared_openings: &[StackedProof], + prepared: &[BlockPreparedCheck<'_, F>], + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + expected: &FieldElement, + config: &ChainConfig, + transcript: &mut T, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript> + + Clone + + Default, +{ + block_verify_with::( + proof, + commit_ood, + &[], + prepared_openings, + prepared, + statements, + layouts, + domains, + sizes, + expected, + config, + transcript, + false, + VerifierChecks::ALL, + ) +} + +/// [`block_verify`] under either argue format, the config's: per table, the +/// proof's `tables`; batched, `argues`, one per group, each read on its +/// group's fork ([`batched::verify_argue`]). The other format's messages must +/// be absent. +/// +/// The prepared openings are left unchecked when `skip_prepared` — their +/// prefixes still skipped by `check_preprocessed` — and the batched argue's +/// checks and the bus balance are switched by `checks`. Both are mutations: a +/// test shows each check is what refuses its forgery. Production passes +/// `false` and [`VerifierChecks::ALL`]. +/// +/// `commit_ood` is every carried polynomial's commit-time out-of-domain +/// answer, in root order: exactly one per root — none missing, none extra — +/// absorbed after the roots and before `(z, α, β)`, and each batched into its +/// chain against its group's point, which the verifier draws from the proof's +/// roots and its own layout ([`commit_ood_point`]). There is no format without +/// them. +#[doc(hidden)] +#[allow(clippy::too_many_arguments)] +pub fn block_verify_with( + proof: &MultiProof, + commit_ood: &[FieldElement], + argues: &[BatchedArgue], + prepared_openings: &[StackedProof], + prepared: &[BlockPreparedCheck<'_, F>], + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + expected: &FieldElement, + config: &ChainConfig, + transcript: &mut T, + skip_prepared: bool, + checks: VerifierChecks, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript> + + Clone + + Default, +{ + // The format is the verifier's config's, never the proof's: that format's + // messages present in full, the other's absent. + let (table_proofs, group_argues) = match config.format.argue { + ArgueFormat::PerTable => (statements.len(), 0), + ArgueFormat::Batched { .. } => (0, sizes.len()), + }; + if proof.tables.len() != table_proofs || argues.len() != group_argues { + return Err(MlError::ArgueFormatMismatch); + } + if layouts.len() != sizes.len() + || domains.len() != sizes.len() + || proof.columns.len() != sizes.len() + || sizes.iter().sum::() != statements.len() + { + return Err(MlError::QueryCountMismatch { + expected: sizes.len(), + got: proof.columns.len(), + }); + } + // Every root the proof carries is one some group opens: a root no layout + // claims would be absorbed and bound to nothing. + let polys: usize = layouts.iter().map(StackedLayout::num_polys).sum(); + if polys != proof.roots.len() { + return Err(MlError::QueryCountMismatch { + expected: polys, + got: proof.roots.len(), + }); + } + // One commit-time answer per carried root, by the verifier's own layouts. + if commit_ood.len() != polys { + return Err(MlError::CommitOodCount { + expected: polys, + got: commit_ood.len(), + }); + } + if proof.preprocessed.is_some() { + return Err(MlError::QueryCountMismatch { + expected: 0, + got: 1, + }); + } + // One opening per group with a prepared stack, each table settled at most + // once and by its own group, each prefix within the table's preprocessed + // columns — never more than it has, which would skip a check nothing + // replaced — and the stack holding exactly those columns. + if prepared_openings.len() != prepared.len() { + return Err(MlError::QueryCountMismatch { + expected: prepared.len(), + got: prepared_openings.len(), + }); + } + let starts: Vec = sizes + .iter() + .scan(0usize, |at, &size| { + let start = *at; + *at += size; + Some(start) + }) + .collect(); + let entries: Vec> = prepared + .iter() + .map(|p| { + ( + p.group, + p.tables.as_slice(), + Some(p.layout.placements().len()), + ) + }) + .collect(); + prepared_shape(&entries, &starts, statements.len(), &|t| { + statements[t].num_preprocessed + })?; + let mut settled = vec![0usize; statements.len()]; + for p in prepared { + for &(table, n) in &p.tables { + settled[table] = n; + } + } + let derived: Vec = prepared + .iter() + .flat_map(|p| p.roots.iter().copied()) + .collect(); + let (z, alpha, beta) = + absorb_roots_answers_and_challenge::(transcript, &proof.roots, &derived, commit_ood); + + let mut balance = FieldElement::::zero(); + let mut statement_at = 0usize; + let mut root_at = 0usize; + for (g, (((opening, layout), domain), &size)) in proof + .columns + .iter() + .zip(layouts) + .zip(domains) + .zip(sizes) + .enumerate() + { + let mut fork = group_fork::(transcript, g); + let mut points: Vec>> = Vec::new(); + let mut values: Vec> = Vec::new(); + let group_statements = &statements[statement_at..statement_at + size]; + let group_settled = &settled[statement_at..statement_at + size]; + match config.format.argue { + ArgueFormat::PerTable => { + for ((table, statement), &settled) in proof.tables + [statement_at..statement_at + size] + .iter() + .zip(group_statements) + .zip(group_settled) + { + let (output, reduced) = + verify(table, *statement, &z, &alpha, &beta, &mut fork, settled)?; + balance += contribution(&output).ok_or(MlError::BusImbalance)?; + for _ in 0..statement.slot_of.len() { + points.push(reduced.point.clone()); + } + values.extend(reduced.column_values); + } + } + ArgueFormat::Batched { + bin_log_cells, + pack, + } => { + // The group's bins and trees come from its statements and the + // verifier's cap and packing, never from the proof. + let argue = &argues[g]; + let reduced = batched::verify_argue( + argue, + group_statements, + bin_log_cells, + pack, + &z, + &alpha, + &beta, + &mut fork, + checks, + )?; + for output in &argue.bus_outputs { + balance += contribution(output).ok_or(MlError::BusImbalance)?; + } + for ((statement, reduced), &settled) in + group_statements.iter().zip(reduced).zip(group_settled) + { + if checks.preprocessed { + check_preprocessed(*statement, &reduced, settled)?; + } + for _ in 0..statement.slot_of.len() { + points.push(reduced.point.clone()); + } + values.extend(reduced.column_values); + } + } + } + let roots = &proof.roots[root_at..root_at + layout.num_polys()]; + // The group's point, from its own roots and the verifier's layout. + let z0 = commit_ood_point::(g, layout, roots, config)?; + let ood = checks.commit_ood.then_some(CommitOod { + z0: &z0, + answers: &commit_ood[root_at..root_at + layout.num_polys()], + }); + stacked_eval::verify_ood::( + opening, + layout, + roots, + &Claimed::PerColumn(&points), + &values, + ood, + domain, + config, + &mut fork, + )?; + // The group's prepared stack, on this fork: the derived commitment must + // take the values each table settled on, at that table's point. + if let Some(k) = prepared + .iter() + .position(|p| p.group == g) + .filter(|_| !skip_prepared) + { + let firsts: Vec = statements[statement_at..statement_at + size] + .iter() + .scan(0usize, |first, s| { + let at = *first; + *first += s.slot_of.len(); + Some(at) + }) + .collect(); + let p = &prepared[k]; + let (at_points, at_values) = + prepared_claims(&p.tables, statement_at, &firsts, &points, &values); + stacked_eval::verify::( + &prepared_openings[k], + p.layout, + p.roots, + &Claimed::PerColumn(&at_points), + &at_values, + p.domain, + config, + &mut fork, + )?; + } + statement_at += size; + root_at += layout.num_polys(); + } + if checks.balance && balance != *expected { + return Err(MlError::BusImbalance); + } + Ok(()) +} + +#[cfg(test)] +mod commit_ood_point_tests { + use super::{ + COMMIT_OOD_LABEL, absorb_roots_answers_and_challenge, commit_ood_point, commit_ood_prefix, + }; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + }; + use multilinear::{ + stacking::StackedLayout, + whir_chain::{ChainConfig, ChainFormat, GrindBits, GroupFold}, + whir_hash::{P1Whir, WhirHash}, + }; + + type T = ::Sponge; + type EE = FieldElement; + + fn config(log_blowup: usize, group: GroupFold) -> ChainConfig { + ChainConfig { + log_blowup, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: ChainFormat { + group, + ..ChainFormat::DEFAULT + }, + } + } + + /// The prefix is nine whole felts: the label, then five little-endian + /// words, in the order the in-guest leaf absorbs them as constants. + #[test] + fn the_prefix_is_the_label_and_five_words() { + let layout = StackedLayout::build(&[4, 4, 4], 5).unwrap(); + let config = config(1, GroupFold::Indexed); + let prefix = commit_ood_prefix(3, &layout, &config); + assert_eq!(prefix.len(), 72); + assert_eq!(&prefix[..32], COMMIT_OOD_LABEL); + let word = + |i: usize| u64::from_le_bytes(prefix[32 + 8 * i..40 + 8 * i].try_into().unwrap()); + assert_eq!( + [word(0), word(1), word(2), word(3), word(4)], + [3, 5, layout.num_polys() as u64, 1, config.fold_word()] + ); + } + + /// ★ The point is a function of the group's index, its stack's shape, the + /// code and every one of its roots. + #[test] + fn the_point_moves_with_the_group_its_code_and_each_root() { + let layout = StackedLayout::build(&[4, 4, 4], 5).unwrap(); + assert_eq!(layout.num_polys(), 2); + let roots = [[7u8; 32], [9u8; 32]]; + let base_config = config(1, GroupFold::Indexed); + let at = |g: usize, layout: &StackedLayout, roots: &[[u8; 32]], config: &ChainConfig| { + commit_ood_point::(g, layout, roots, config).unwrap() + }; + let base: EE = at(0, &layout, &roots, &base_config); + assert_eq!(base, at(0, &layout, &roots, &base_config), "deterministic"); + assert_ne!( + base, + at(1, &layout, &roots, &base_config), + "the group's index" + ); + let wider = StackedLayout::build(&[4, 4, 4, 4, 4], 6).unwrap(); + assert_eq!(wider.num_polys(), 2); + assert_ne!( + base, + at(0, &wider, &roots, &base_config), + "the stack's variables" + ); + assert_ne!( + base, + at(0, &layout, &roots, &config(2, GroupFold::Indexed)), + "the blowup" + ); + assert_ne!( + base, + at(0, &layout, &roots, &config(1, GroupFold::PerPoly)), + "the fold word" + ); + for k in 0..roots.len() { + let mut moved = roots; + moved[k][31] ^= 1; + assert_ne!(base, at(0, &layout, &moved, &base_config), "root {k}"); + } + assert!( + commit_ood_point::(0, &layout, &roots[..1], &base_config).is_err(), + "a root per polynomial" + ); + } + + /// ★ The roots block, spelled out: every carried root, every derived root, + /// every answer, THEN the three draws. Each answer moves the first draw, + /// and an answer absorbed after the draws would not — the placement no + /// round trip can see, since both sides would share it. + #[test] + fn the_roots_block_binds_every_answer_before_the_first_draw() { + let carried = [[1u8; 32], [2u8; 32], [3u8; 32]]; + let derived = [[4u8; 32]]; + let answers: Vec = (0..carried.len() as u64) + .map(|i| EE::from(1000 + i)) + .collect(); + let draws = |answers: &[EE]| { + let mut transcript = T::default(); + absorb_roots_answers_and_challenge::( + &mut transcript, + &carried, + &derived, + answers, + ) + }; + let (z, alpha, beta) = draws(&answers); + + let mut by_hand = T::default(); + for root in carried.iter().chain(&derived) { + by_hand.append_bytes(root); + } + for answer in &answers { + by_hand.append_field_element(answer); + } + let spelled: [EE; 3] = core::array::from_fn(|_| by_hand.sample_field_element()); + assert_eq!([z, alpha, beta], spelled); + + for k in 0..answers.len() { + let mut moved = answers.clone(); + moved[k] += EE::one(); + assert_ne!(draws(&moved).0, z, "answer {k} moves z"); + } + let mut late = T::default(); + for root in carried.iter().chain(&derived) { + late.append_bytes(root); + } + let late_z: EE = late.sample_field_element(); + assert_ne!(late_z, z, "answers absorbed after the draw bind nothing"); + } +} + +#[cfg(test)] +mod spill_tests { + use super::commit_pairs_off_pool_from; + use multilinear::narrow::NarrowColumns; + + /// The default commits the pairs off the pool; `0` is the opt-out to the pool. + #[test] + fn the_commit_pairs_go_off_the_pool_by_default_and_zero_opts_out() { + assert!(commit_pairs_off_pool_from(None)); + assert!(commit_pairs_off_pool_from(Some("1"))); + assert!(commit_pairs_off_pool_from(Some("yes"))); + assert!(!commit_pairs_off_pool_from(Some("0"))); + assert!(!commit_pairs_off_pool_from(Some(" 0 "))); + } + + /// ★ Under a spill, a finished group lets go of its tables' records as it + /// does of their packed columns: the table is out (spilled), the records' + /// last handle is gone, and the bytes come back as the group's records + /// released; packed columns go too but are not records. With a returner the + /// records go to its thread, which hands their pages back and frees them + /// before it finishes, and what they came to lands in the group's stamps. + /// With no spill, nothing is taken and nothing counts. + #[test] + fn a_finished_group_lets_go_of_its_records_under_a_spill() { + use super::{GroupStamps, RecordsReturner, release_group}; + use crate::examples::multi_table_lookup::new_cpu_air_with_lookup; + use crate::multilinear_air::Uniforms; + use crate::multilinear_table::{CommittedTable, TableLayout}; + use crate::proof::options::ProofOptions; + use crate::traits::AIR; + use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, + }; + use multilinear::records::{ + KeccakRndOp, PagesReturned, Records, RecordsFill, TableRecords, + }; + use std::sync::Arc; + + // What these records say they handed back. + const RETURNED: PagesReturned = PagesReturned { + buffers: 1, + shared: 2, + bytes: 4096, + failed: 0, + }; + // Records freed so far (a handle kept to watch them would keep the + // returner from being their last holder). + static FREED: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + #[derive(Debug)] + struct Ops(Vec); + impl Drop for Ops { + fn drop(&mut self) { + FREED.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + impl TableRecords for Ops { + fn rows(&self) -> usize { + 8 + } + fn cols(&self) -> usize { + 5 + } + fn fill(&self) -> RecordsFill<'_> { + RecordsFill::KeccakRnd { + ops: &self.0, + skip: 0, + } + } + fn host_fill(&self) -> Option { + None + } + fn return_pages(&mut self) -> PagesReturned { + RETURNED + } + } + let air = new_cpu_air_with_lookup(&ProofOptions::default_test_options()); + let layout = || { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + 5, + 3, + Uniforms::default(), + ) + .expect("a layout") + }; + let ops = || { + Records(Arc::new(Ops(vec![ + KeccakRndOp { + timestamp: 3, + input: [7; 25], + }; + 2 + ]))) + }; + let bytes = ops().bytes(); + assert!(bytes > 0); + let held = || CommittedTable::from_records(layout(), ops()).expect("a table"); + let packed = || { + let mut table = CommittedTable::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + 5, + 3, + Uniforms::default(), + |col| { + (0..8u64) + .map(|r| FieldElement::from(r * 5 + u64::from(col))) + .collect() + }, + ) + .expect("a table"); + assert!(table.pack_on_host(), "packs"); + table + }; + + // No spill: the records stay where they are. + let mut group = vec![held()]; + assert_eq!(release_group(&mut group, 0, None, 0, false, None, None), 0); + assert!(group[0].records().is_some() && !group[0].is_spilled()); + drop(group); + + // A spill with no returner: the records go here, with their last + // handle, and are counted; the packed table goes uncounted. + let mut group = vec![packed(), held()]; + let freed = FREED.load(std::sync::atomic::Ordering::SeqCst); + assert_eq!( + release_group(&mut group, 0, None, 0, true, None, None), + bytes + ); + assert!(group.iter().all(|t| t.is_spilled()), "both tables let go"); + assert_eq!( + FREED.load(std::sync::atomic::Ordering::SeqCst), + freed + 1, + "freed here" + ); + + // With a returner: its thread takes them, the last handle, and hands + // their pages back and frees them before it finishes; the group's + // stamps say so. + let mut group = vec![packed(), held()]; + let freed = FREED.load(std::sync::atomic::Ordering::SeqCst); + let returner = RecordsReturner::start(); + assert_eq!( + release_group(&mut group, 1, None, 0, true, Some(&returner), None), + bytes + ); + assert!(group.iter().all(|t| t.is_spilled()), "both tables let go"); + let mut stamps = vec![GroupStamps::default(); 2]; + returner.finish(&mut stamps); + assert_eq!( + FREED.load(std::sync::atomic::Ordering::SeqCst), + freed + 1, + "freed there" + ); + assert_eq!(stamps[1].records_pages, RETURNED, "group 1's records"); + assert_eq!(stamps[0].records_pages, PagesReturned::default()); + } + + /// A schedstat line reads as its CPU and run-queue delay nanoseconds; a + /// line that is not one reads as nothing. + #[test] + fn a_schedstat_line_reads_as_cpu_and_run_queue_delay() { + use super::parse_schedstat; + assert_eq!(parse_schedstat("123456 7890 42\n"), Some((123456, 7890))); + assert_eq!(parse_schedstat("0 0 0\n"), Some((0, 0))); + assert_eq!(parse_schedstat("123456\n"), None); + assert_eq!(parse_schedstat("cpu wait\n"), None); + assert_eq!(parse_schedstat(""), None); + } + + /// Two classes rebuilt (N3): the plain groups first, then each class in + /// group order, interleaved by its share of the weights; one class is + /// `rebuilt_last`. + #[test] + fn interleaved_classes_keep_their_share_and_their_order() { + use super::interleaved_order; + let classes = [None, Some(0), Some(0), Some(1), None, Some(1), Some(0)]; + assert_eq!( + interleaved_order(&classes, &[1; 7]), + vec![0, 4, 1, 3, 2, 5, 6] + ); + // A heavy rest group waits until the streamed class has caught up. + assert_eq!( + interleaved_order(&[Some(0), Some(0), Some(1), Some(0)], &[1, 1, 3, 1]), + vec![0, 2, 1, 3] + ); + // One class: the groups with nothing dropped, then it (rebuilt_last). + let one = [true, true, false, true, false]; + let as_classes: Vec> = one.iter().map(|&r| r.then_some(0)).collect(); + assert_eq!( + interleaved_order(&as_classes, &[5; 5]), + super::rebuilt_last(&one) + ); + assert_eq!(interleaved_order(&[None; 3], &[]), vec![0, 1, 2]); + // Any mix: a permutation, the plain groups first, each class ascending. + let mut seed = 0x9e37_79b9_u64; + for _ in 0..200 { + let n = 1 + (seed % 40) as usize; + let classes: Vec> = (0..n) + .map(|i| { + seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + match (seed >> 33) % 3 { + 0 => None, + c => Some(c as usize - 1), + } + .filter(|_| i < n) + }) + .collect(); + let weights: Vec = (0..n).map(|i| 1 + (i as u64 * 7919) % 13).collect(); + let order = interleaved_order(&classes, &weights); + let mut sorted = order.clone(); + sorted.sort_unstable(); + assert_eq!(sorted, (0..n).collect::>()); + let plain = classes.iter().filter(|c| c.is_none()).count(); + assert!(order[..plain].iter().all(|&g| classes[g].is_none())); + for c in 0..2 { + let seq: Vec = order + .iter() + .copied() + .filter(|&g| classes[g] == Some(c)) + .collect(); + assert!(seq.windows(2).all(|w| w[0] < w[1]), "class {c} ascending"); + } + } + } + + /// One class a group (N3): a group's tables drop in the class of its first + /// droppable one only, and phase B refuses a group that dropped two. + #[test] + fn a_group_drops_in_the_class_of_its_first_droppable_table() { + use super::{drop_one_class, one_class}; + let mut ranks = [None, Some((0, 7)), Some((1, 0)), Some((0, 8)), Some((1, 1))]; + drop_one_class(&mut ranks); + assert_eq!(ranks, [None, Some((0, 7)), None, Some((0, 8)), None]); + let mut ranks = [Some((1, 3)), None, Some((0, 9)), Some((1, 4))]; + drop_one_class(&mut ranks); + assert_eq!(ranks, [Some((1, 3)), None, None, Some((1, 4))]); + let mut ranks = [None, None]; + drop_one_class(&mut ranks); + assert_eq!(ranks, [None, None]); + + assert_eq!(one_class([None, Some(1), None, Some(1)]), Some(Some(1))); + assert_eq!(one_class([None, None]), Some(None)); + assert_eq!(one_class([]), Some(None)); + assert_eq!(one_class([Some(0), None, Some(1)]), None); + } + + /// The decision feed a regenerator started before phase A's end reads + /// (N4c): a table dropped after the wait began is handed over with its + /// slot; a held table is kept; a table never dropped is kept only once + /// phase A's end closes the decisions; and a waiter on an undecided table + /// returns `Stopped` as soon as its class's window closes (a failed or + /// ended prove), never hanging. + #[test] + fn a_regenerator_started_early_waits_for_each_tables_fate() { + use super::{BlockRegen, Parked, ParkedState, RankDecision}; + use std::sync::Arc; + use std::time::Duration; + let words: Vec = (0..2 * 1024u64).map(|i| i * 7919 % 50_000).collect(); + let packed = || NarrowColumns::pack_row_major(&words, 2).expect("packs"); + let parked = |class: usize, rank: u64| { + Arc::new(Parked { + len: packed().data().len() as u64, + cells: 0, + rank: Some((class, rank)), + state: std::sync::Mutex::new(ParkedState::Resident(packed())), + }) + }; + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30, 1 << 30], true); + let _producers = (regen.take_producer(0), regen.take_producer(1)); + assert!(regen.drops_all_from_now()); + // Dropped after the wait began: handed over with its slot. + regen.delay_drops(1, Duration::from_millis(300)); + let waiter = { + let regen = regen.clone(); + std::thread::spawn(move || regen.await_rank(1, 7)) + }; + std::thread::sleep(Duration::from_millis(50)); + regen.parked(parked(1, 7)); + let (decision, waited) = waiter.join().expect("joins"); + assert!(matches!(decision, RankDecision::Dropped(ref slot) if slot.rank() == 7)); + assert!(waited >= Duration::from_millis(200), "waited {waited:?}"); + // Held by its group's class: kept at once. + regen.held(&[(1, 9)]); + assert!(matches!(regen.await_rank(1, 9).0, RankDecision::Kept)); + // Never dropped: undecided while phase A runs, kept once it ends. + let waiter = { + let regen = regen.clone(); + std::thread::spawn(move || regen.await_rank(1, 11)) + }; + std::thread::sleep(Duration::from_millis(150)); + assert!(!waiter.is_finished(), "kept before phase A's end"); + regen.close_drops(); + assert!(matches!( + waiter.join().expect("joins").0, + RankDecision::Kept + )); + assert!(regen.decisions_closed()); + // Liveness: a waiter on an undecided table of a class whose window + // closes (the prove failed) returns at once, Stopped. + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30, 1 << 30], true); + let waiter = { + let regen = regen.clone(); + std::thread::spawn(move || regen.await_rank(1, 3)) + }; + std::thread::sleep(Duration::from_millis(100)); + assert!(!waiter.is_finished()); + regen.windows()[1].close("the prove failed"); + let (decision, waited) = waiter.join().expect("joins"); + assert!(matches!(decision, RankDecision::Stopped)); + assert!(waited < Duration::from_secs(2), "waited {waited:?}"); + } + + /// Partial drops: arming drops back nothing and later tables are not + /// dropped on their own; a table the policy wants out drops, and a + /// hand-off's drop-back takes the parked candidates in rank order only + /// until its bytes are on their way. Once settled, what stays is kept, so + /// a regenerator started early learns every fate as it comes. + #[test] + fn partial_drops_take_only_what_is_asked() { + use super::{BlockRegen, Parked, ParkedState, RankDecision}; + use std::sync::Arc; + let words: Vec = (0..2 * 1024u64).map(|i| i * 7919 % 50_000).collect(); + let packed = || NarrowColumns::pack_row_major(&words, 2).expect("packs"); + let len = packed().data().len() as u64; + let parked = |rank: u64| { + Arc::new(Parked { + len, + cells: 0, + rank: Some((0, rank)), + state: std::sync::Mutex::new(ParkedState::Resident(packed())), + }) + }; + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], false).partial(); + assert!(regen.is_partial()); + let _producer = regen.take_producer(0); + // Parked before and after arming: candidates either way. + regen.parked(parked(0)); + regen.parked(parked(3)); + assert!(regen.arm()); + assert!(!regen.arm(), "arms once"); + assert!(!regen.drops_all_from_now()); + regen.parked(parked(1)); + regen.parked(parked(2)); + // The policy wants this one out. + regen.drop_one(parked(4)); + // A hand-off asking for one table and a byte: the two lowest ranks. + assert_eq!(regen.drop_back(len + 1), 2 * len); + regen.close_drops(); + let report = regen.report(); + assert_eq!(report.tables, 3, "{report:?}"); + assert_eq!(report.back_tables, 2, "{report:?}"); + for rank in [0, 1, 4] { + assert!( + matches!(regen.await_rank(0, rank).0, RankDecision::Dropped(_)), + "rank {rank} dropped" + ); + } + for rank in [2, 3] { + assert!( + matches!(regen.await_rank(0, rank).0, RankDecision::Kept), + "rank {rank} kept" + ); + } + // Asked for exactly one table's bytes: that table and not the next; + // asked for none: nothing. + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], false).partial(); + let _producer = regen.take_producer(0); + regen.parked(parked(0)); + regen.parked(parked(1)); + assert!(regen.arm()); + assert_eq!(regen.drop_back(0), 0, "nothing asked"); + assert_eq!(regen.drop_back(len), len, "one table's bytes"); + regen.close_drops(); + assert!(matches!(regen.await_rank(0, 0).0, RankDecision::Dropped(_))); + assert!(matches!(regen.await_rank(0, 1).0, RankDecision::Kept)); + // Settled: the candidates left are kept, and so is a table parked + // after, at once (a waiter returns before phase A's end); their fates + // are known from now on. + let fate = |regen: &BlockRegen, rank: u64| { + let regen = regen.clone(); + let waiter = std::thread::spawn(move || regen.await_rank(0, rank).0); + let t = std::time::Instant::now(); + while !waiter.is_finished() { + assert!( + t.elapsed() < std::time::Duration::from_secs(5), + "rank {rank}'s fate is not known" + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + waiter.join().expect("joins") + }; + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], false).partial(); + let _producer = regen.take_producer(0); + regen.parked(parked(0)); + assert!(regen.arm()); + assert!(!regen.drops_all_from_now(), "not before it settles"); + regen.settle(); + assert!(regen.drops_all_from_now()); + assert!(matches!(fate(®en, 0), RankDecision::Kept)); + regen.parked(parked(1)); + assert!(matches!(fate(®en, 1), RankDecision::Kept)); + regen.drop_one(parked(2)); + assert!(matches!(fate(®en, 2), RankDecision::Dropped(_))); + assert_eq!(regen.drop_back(u64::MAX), 0, "nothing left to drop back"); + regen.close_drops(); + // Not partial: arming drops back every candidate and every later one. + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], false); + let _producer = regen.take_producer(0); + regen.parked(parked(0)); + assert!(regen.arm()); + assert!(regen.drops_all_from_now()); + regen.parked(parked(1)); + regen.close_drops(); + assert_eq!(regen.report().tables, 2); + } + + /// The keep budget's two halves at arming ([`super::BlockRegen::keep_cells`]), + /// for the same candidates parked out of rank order (cells as committed, + /// ranks not): a budget at or above every candidate's cells drops nothing + /// more than no budget (same drops, same fates, every kept table's bytes + /// still on the host), and a budget below some drops back exactly the + /// candidates committed past it, in rank order, at arming only. + #[test] + fn the_keep_budget_drops_back_only_what_was_committed_past_it() { + use super::{BlockRegen, Parked, ParkedState, RankDecision}; + use std::sync::Arc; + let words: Vec = (0..2 * 1024u64).map(|i| i * 7919 % 50_000).collect(); + let packed = || NarrowColumns::pack_row_major(&words, 2).expect("packs"); + let len = packed().data().len() as u64; + let parked = |rank: u64, cells: u64| { + Arc::new(Parked { + len, + cells, + rank: Some((0, rank)), + state: std::sync::Mutex::new(ParkedState::Resident(packed())), + }) + }; + // (rank, cells committed once it was): committed in cells order, + // ranks shuffled. + let committed = [(5, 10), (1, 20), (7, 30), (3, 40), (0, 50), (6, 60)]; + // Parks them, arms at 60 cells, parks one more past every budget + // below, lets the policy drop one and a hand-off drop back one table's + // bytes; returns the regen, the parked tables and the drop thread's + // order. + let run = |budget: Option| { + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], false) + .partial() + .keep_cells(budget); + let _producer = regen.take_producer(0); + let tables: Vec<_> = committed.iter().map(|&(r, c)| parked(r, c)).collect(); + for table in &tables { + regen.parked(Arc::clone(table)); + } + assert!(regen.arm_at(60)); + let late = parked(2, 70); + regen.parked(Arc::clone(&late)); + regen.drop_one(parked(9, 80)); + let back = regen.drop_back(1); + regen.close_drops(); + let order: Vec<(u64, bool)> = regen + .lock() + .dropped + .iter() + .map(|((_, rank), _, _, back)| (*rank, *back)) + .collect(); + (regen, tables, late, back, order) + }; + let fates = |regen: &BlockRegen| -> Vec { + (0..10) + .map(|rank| matches!(regen.await_rank(0, rank).0, RankDecision::Dropped(_))) + .collect() + }; + + // No budget, and budgets at and above the last candidate's cells: the + // same drops (the policy's rank 9, the hand-off's lowest rank 0), the + // same fates, nothing dropped back at arming, and every kept table's + // columns still the ones parked. + let (none, _, _, none_back, none_order) = run(None); + assert_eq!(none_order, vec![(9, false), (0, true)]); + for budget in [60, 61, u64::MAX] { + let (regen, tables, late, back, order) = run(Some(budget)); + assert_eq!(order, none_order, "budget {budget}"); + assert_eq!(back, none_back, "budget {budget}"); + assert_eq!(fates(®en), fates(&none), "budget {budget}"); + let (report, plain) = (regen.report(), none.report()); + assert_eq!(report.keep_back, (0, 0), "budget {budget}"); + assert_eq!(report.keep_cells, Some(budget)); + assert_eq!( + ( + report.tables, + report.bytes, + report.back_tables, + report.back_bytes + ), + ( + plain.tables, + plain.bytes, + plain.back_tables, + plain.back_bytes + ), + "budget {budget}" + ); + assert_eq!( + (report.armed_cells, report.armed_kept), + (Some(60), (6, 6 * len)) + ); + for table in tables.iter().chain([&late]) { + if table.rank == Some((0, 0)) { + continue; + } + let state = table.state.lock().unwrap_or_else(|e| e.into_inner()); + assert!( + matches!(&*state, ParkedState::Resident(p) if p.data() == packed().data()), + "budget {budget}: rank {:?} left the host", + table.rank + ); + } + } + assert_eq!(none.report().keep_back, (0, 0)); + assert_eq!(none.report().keep_cells, None); + + // A budget of 30 cells: the candidates committed past it (cells 40, 50, + // 60: ranks 3, 0, 6) are dropped back at arming in rank order, before + // anything else; the one at the budget (rank 7, cells 30) stays, and so + // does the late one parked after arming, past the budget (rank 2). The + // hand-off then takes the lowest rank left (1). + let (regen, tables, late, back, order) = run(Some(30)); + assert_eq!( + order, + vec![(0, true), (3, true), (6, true), (9, false), (1, true)] + ); + assert_eq!(back, len); + let report = regen.report(); + assert_eq!(report.keep_back, (3, 3 * len), "{report:?}"); + assert_eq!(report.armed_kept, (6, 6 * len)); + let dropped: Vec = (0..10) + .filter(|&rank| matches!(regen.await_rank(0, rank).0, RankDecision::Dropped(_))) + .collect(); + assert_eq!(dropped, vec![0, 1, 3, 6, 9]); + for table in tables.iter().chain([&late]) { + let kept = matches!( + &*table.state.lock().unwrap_or_else(|e| e.into_inner()), + ParkedState::Resident(_) + ); + let rank = table.rank.map(|(_, r)| r).unwrap_or_default(); + assert_eq!(kept, !dropped.contains(&rank), "rank {rank}"); + } + + // Not partial: the budget is not used, and arming drops back every + // candidate as before. + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], false).keep_cells(Some(30)); + let _producer = regen.take_producer(0); + for &(r, c) in &committed { + regen.parked(parked(r, c)); + } + assert!(regen.arm_at(60)); + regen.close_drops(); + let report = regen.report(); + assert_eq!( + (report.tables, report.keep_back, report.keep_cells), + (6, (0, 0), None) + ); + } + + /// The keep budget relative to the arming point + /// ([`super::BlockRegen::keep_below`]): arming at 60 cells with 25 below + /// drops back exactly the candidates committed past 35, in rank order, and + /// only at arming; with an absolute budget too the lower one holds; a + /// margin past the arming point drops back every candidate; arming without + /// the cells committed uses only the absolute budget; and none is today's. + #[test] + fn the_relative_keep_budget_drops_back_what_was_committed_in_its_last_cells() { + use super::{BlockRegen, Parked, ParkedState, RankDecision}; + use std::sync::Arc; + let words: Vec = (0..2 * 1024u64).map(|i| i * 7919 % 50_000).collect(); + let packed = || NarrowColumns::pack_row_major(&words, 2).expect("packs"); + let len = packed().data().len() as u64; + let parked = |rank: u64, cells: u64| { + Arc::new(Parked { + len, + cells, + rank: Some((0, rank)), + state: std::sync::Mutex::new(ParkedState::Resident(packed())), + }) + }; + let committed = [(5, 10), (1, 20), (7, 30), (3, 40), (0, 50), (6, 60)]; + // Parks them, arms (at 60 cells when `at` is set, else without the + // cells), parks one more past every budget; returns the drop thread's + // order and the report. + let run = |absolute: Option, below: Option, at: bool| { + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], false) + .partial() + .keep_cells(absolute) + .keep_below(below); + let _producer = regen.take_producer(0); + for &(r, c) in &committed { + regen.parked(parked(r, c)); + } + assert!(if at { regen.arm_at(60) } else { regen.arm() }); + regen.parked(parked(2, 70)); + regen.close_drops(); + let order: Vec = regen + .lock() + .dropped + .iter() + .map(|((_, rank), _, _, back)| { + assert!(*back, "a drop-back"); + *rank + }) + .collect(); + let kept: Vec = (0..10) + .filter(|&r| matches!(regen.await_rank(0, r).0, RankDecision::Kept)) + .collect(); + (order, kept, regen.report()) + }; + // 25 below 60: past 35 are cells 40, 50, 60 (ranks 3, 0, 6), in rank + // order; cells 30 stays, and so does the table parked after arming. + let (order, kept, report) = run(None, Some(25), true); + assert_eq!(order, vec![0, 3, 6]); + assert_eq!(kept, vec![1, 2, 4, 5, 7, 8, 9]); + assert_eq!( + (report.budget, report.keep_below, report.keep_back), + (Some(35), Some(25), (3, 3 * len)) + ); + // With an absolute 45 too, the lower (35) holds; with an absolute 25, + // that one does (past 25: cells 30 as well). + assert_eq!(run(Some(45), Some(25), true).0, vec![0, 3, 6]); + assert_eq!(run(Some(25), Some(25), true).0, vec![0, 3, 6, 7]); + // A margin at or past the arming point drops back every candidate. + assert_eq!(run(None, Some(60), true).0, vec![0, 1, 3, 5, 6, 7]); + assert_eq!(run(None, Some(1_000), true).0, vec![0, 1, 3, 5, 6, 7]); + // Without the cells committed, only the absolute budget. + assert_eq!(run(None, Some(25), false).0, Vec::::new()); + assert_eq!(run(Some(45), Some(25), false).0, vec![0, 6]); + // None: arming drops back nothing. + let (order, _, report) = run(None, None, true); + assert!(order.is_empty()); + assert_eq!((report.budget, report.keep_below), (None, None)); + } + + /// ★ A parked table held as its records drops as them: its slot keeps the + /// records' digest and shape and takes only a checked deposit, the drop + /// report counts it among the records, and a records table not dropped + /// leaves its parking as it came. + #[test] + fn a_parked_records_table_drops_as_its_records() { + use super::{BlockRegen, Parked, ParkedState, RankDecision, drop_parked}; + use multilinear::records::{KeccakRndOp, Records, RecordsFill, TableRecords}; + use std::sync::Arc; + + #[derive(Debug)] + struct Ops(Vec); + impl TableRecords for Ops { + fn rows(&self) -> usize { + 1024 + } + fn cols(&self) -> usize { + 2 + } + fn fill(&self) -> RecordsFill<'_> { + RecordsFill::KeccakRnd { + ops: &self.0, + skip: 0, + } + } + fn host_fill(&self) -> Option { + None + } + } + let records = |seed: u64| { + Records(Arc::new(Ops(vec![ + KeccakRndOp { + timestamp: seed, + input: [seed; 25], + }; + 3 + ]))) + }; + let held = records(5); + let parked = Arc::new(Parked { + len: held.bytes() as u64, + cells: 0, + rank: Some((0, 4)), + state: std::sync::Mutex::new(ParkedState::ResidentRecords(held.clone())), + }); + let regen = BlockRegen::new(Arc::new(|_| None), &[1 << 30], true); + let _producer = regen.take_producer(0); + regen.parked(Arc::clone(&parked)); + let RankDecision::Dropped(slot) = regen.await_rank(0, 4).0 else { + panic!("dropped"); + }; + assert!(slot.is_records()); + assert_eq!( + (slot.digest(), slot.rows(), slot.cols()), + (held.digest(), 1024, 2) + ); + assert!(parked.dropped().is_some(), "its slot in its place"); + let words: Vec = (0..2 * 1024u64).map(|i| i * 7919 % 50_000).collect(); + let packed = NarrowColumns::pack_row_major(&words, 2).expect("packs"); + slot.deposit_checked(packed, records(5).digest()) + .expect("columns from the same records"); + regen.close_drops(); + let report = regen.report(); + assert_eq!( + (report.tables, report.records), + (1, (1, held.bytes() as u64)) + ); + // Not dropped: a second drop of the same parking finds nothing held. + let window = ®en.windows()[0]; + assert!( + drop_parked(window, &parked, 4, None).is_none(), + "dropped once" + ); + } + + /// The rebuilt groups go last, each set in group order; nothing rebuilt is + /// the group order. + #[test] + fn rebuilt_groups_go_last() { + assert_eq!( + super::rebuilt_last(&[true, true, false, true, false]), + vec![2, 4, 0, 1, 3] + ); + assert_eq!(super::rebuilt_last(&[false; 4]), vec![0, 1, 2, 3]); + assert_eq!(super::rebuilt_last(&[true; 3]), vec![0, 1, 2]); + } +} diff --git a/crypto/stark/src/multilinear_logup.rs b/crypto/stark/src/multilinear_logup.rs new file mode 100644 index 000000000..5667618a1 --- /dev/null +++ b/crypto/stark/src/multilinear_logup.rs @@ -0,0 +1,730 @@ +//! Translates a table's bus interactions into the affine form +//! [`multilinear::logup`] works over. +//! +//! A multiplicity and a fingerprint are both **linear** in the main +//! columns, so each interaction is two affine expressions over the sumcheck's +//! factors and neither needs a column of its own. The coefficients are found by +//! **probing the real evaluators** — the value on an all-zero row is the +//! constant, the value on a basis row is that column's coefficient — rather +//! than re-deriving the packing formulas, which is what keeps this from +//! drifting away from what the prover computes. + +use math::field::{element::FieldElement, traits::IsField}; +use multilinear::{ + Error as MlError, + logup::{Affine, Interaction}, +}; + +use crate::lookup::BusInteraction; + +/// The interaction's fingerprint from a row of column values: the bus id at +/// `alpha^0`, then each bus element at the next power. +/// +/// `alpha_powers[i]` must be `alpha^i`, up to the widest interaction. +fn fingerprint_at( + interaction: &BusInteraction, + alpha_powers: &[FieldElement], + column: &dyn Fn(usize) -> FieldElement, +) -> FieldElement { + let mut fingerprint = FieldElement::::from(interaction.bus_id); + let mut power = 1; + for value in &interaction.values { + for element in value.combine_from(column) { + fingerprint += &alpha_powers[power] * element; + power += 1; + } + } + fingerprint +} + +/// Recovers the affine form of a linear function of the main columns. +/// +/// `slot_of` maps a main column to the factor that reads it, and is only asked +/// about columns that turn out to have a nonzero coefficient — so a table's +/// unread columns never need a factor. +/// +/// `candidates` are the columns worth asking about, in ascending order: the +/// coefficient is read by evaluating at each basis vector, so the cost is one +/// pass over the expression per candidate. Handing it the whole table turns +/// that into `interactions × columns`, which for a precompile with a thousand +/// of each is millions of passes for a handful of nonzero terms. +fn probe( + candidates: &[usize], + slot_of: &mut S, + eval: impl Fn(&dyn Fn(usize) -> FieldElement) -> FieldElement, +) -> Result, MlError> +where + E: IsField + 'static, + S: FnMut(usize) -> Result, +{ + let constant = eval(&|_| FieldElement::::zero()); + let mut terms = Vec::new(); + for &column in candidates { + let coefficient = eval(&|i| { + if i == column { + FieldElement::::one() + } else { + FieldElement::::zero() + } + }) - &constant; + if coefficient == FieldElement::zero() { + continue; + } + terms.push((slot_of(column)?, coefficient)); + } + Ok(Affine::new(terms, constant)) +} + +/// Every interaction as a signed numerator over `z − fingerprint`. +/// +/// `z` and `alpha` are the LogUp challenges, in the same roles the univariate +/// prover gives them. `slot_of` is asked for the factor reading each main +/// column an interaction actually uses, so a caller can register factors +/// lazily. +pub fn interactions( + buses: &[BusInteraction], + num_main_columns: usize, + z: &FieldElement, + alpha: &FieldElement, + mut slot_of: S, +) -> Result>, MlError> +where + E: IsField + 'static, + S: FnMut(usize) -> Result, +{ + let width = buses + .iter() + .map(BusInteraction::num_bus_elements) + .max() + .unwrap_or(0); + let mut alpha_powers = Vec::with_capacity(width + 1); + let mut power = FieldElement::::one(); + for _ in 0..=width { + alpha_powers.push(power.clone()); + power *= alpha; + } + + buses + .iter() + .map(|bus| { + // What this interaction reads, and nothing else: a column it never + // touches has coefficient zero, and asking costs a pass over the + // expression. + let candidates: Vec = bus + .columns_read() + .into_iter() + .filter(|&column| column < num_main_columns) + .collect(); + let numerator = probe(&candidates, &mut slot_of, |column| { + let value = bus.multiplicity.evaluate_with(column); + if bus.is_sender { value } else { -value } + })?; + let denominator = probe(&candidates, &mut slot_of, |column| { + z - fingerprint_at(bus, &alpha_powers, column) + })?; + Ok(Interaction::new(numerator, denominator)) + }) + .collect() +} + +/// The same recovery for a VECTOR of linear functions probed together. +/// +/// One pass per candidate recovers every component's coefficient at once. That +/// is what [`interaction_shapes`] needs: a fingerprint's bus elements all come +/// out of ONE `combine_from` walk, so probing them one at a time would repeat +/// that walk once per element and turn a pass per candidate into `elements` +/// passes per candidate. +/// +/// `slot_of` is asked about a column that is nonzero in ANY component, once — +/// [`probe`]'s laziness widened to the vector. +/// +/// ⚠ This duplicates [`probe`]'s dozen lines rather than replacing it. +/// Expressing `probe` as a one-component call here would allocate a `Vec` per +/// eval, and eval is the thing [`interactions`] is written to run as few times +/// as possible. The two are held together by a test instead: +/// `the_vector_probe_is_the_scalar_one_component_by_component`. +fn probe_many( + candidates: &[usize], + slot_of: &mut S, + eval: impl Fn(&dyn Fn(usize) -> FieldElement) -> Vec>, +) -> Result>, MlError> +where + E: IsField + 'static, + S: FnMut(usize) -> Result, +{ + let constants = eval(&|_| FieldElement::::zero()); + let mut terms: Vec)>> = vec![Vec::new(); constants.len()]; + for &column in candidates { + let at_basis = eval(&|i| { + if i == column { + FieldElement::::one() + } else { + FieldElement::::zero() + } + }); + // A contract on `eval`, not a runtime condition: the one producer is a + // fingerprint's bus elements, whose count is a property of the + // interaction and not of the columns it is asked about. Stated as an + // assert rather than an error so a future producer that breaks it says + // so here instead of silently dropping a component. + assert_eq!( + at_basis.len(), + constants.len(), + "a probed vector must have the same length at every point" + ); + let coefficients: Vec> = at_basis + .iter() + .zip(&constants) + .map(|(value, constant)| value - constant) + .collect(); + if coefficients.iter().all(|c| *c == FieldElement::::zero()) { + continue; + } + let slot = slot_of(column)?; + for (component, coefficient) in coefficients.into_iter().enumerate() { + if coefficient == FieldElement::zero() { + continue; + } + terms[component].push((slot, coefficient)); + } + } + + Ok(terms + .into_iter() + .zip(constants) + .map(|(terms, constant)| Affine::new(terms, constant)) + .collect()) +} + +/// One interaction's affine structure with the LogUp challenges factored OUT. +/// +/// [`interactions`] fuses `z` and the powers of `alpha` into the denominator's +/// coefficients, which is right for a verifier that has drawn them and wrong +/// for one that is EMITTING code to be run later: the recursion's in-guest +/// verifier knows the bus at emit time and the challenges only at run time. +/// This is the same bus, cut the other way. +/// +/// ```text +/// numerator_i = the multiplicity, signed (no challenge at all) +/// denominator_i = z − bus_id − Σ_p alpha^{p+1}·elements[p] +/// ``` +/// +/// ★ The numerator is not challenge-dependent in any part: `interactions` +/// builds it from `multiplicity.evaluate_with` alone, so it is repeated here +/// verbatim and a caller may intern its coefficients as constants. +pub struct InteractionShape { + /// Exactly what [`interactions`] builds, at any `z` and `alpha`. + pub numerator: Affine, + /// The fingerprint's `alpha^0` term. + pub bus_id: FieldElement, + /// The bus elements in fingerprint order: `elements[p]` rides + /// `alpha^{p+1}`, matching `fingerprint_at`'s `power` counter. + pub elements: Vec>, +} + +/// Every interaction's affine structure, challenge-free. +/// +/// The same `probe`-the-real-evaluators discipline [`interactions`] uses, and +/// the same `num_main_columns` filter: a column past the main width is PINNED +/// TO ZERO in the recovered affine rather than carried, because it is never a +/// probe candidate. +/// +/// ⚠ `slot_of` is asked about a column whose coefficient is structurally +/// nonzero. [`interactions`] asks about a column whose coefficient is nonzero +/// AT ITS ALPHA, which is the same set except on a measure-zero choice of +/// alpha where the powers cancel; the two therefore agree in value always and +/// in term list with overwhelming probability. The test below asserts both. +pub fn interaction_shapes( + buses: &[BusInteraction], + num_main_columns: usize, + mut slot_of: S, +) -> Result>, MlError> +where + E: IsField + 'static, + S: FnMut(usize) -> Result, +{ + buses + .iter() + .map(|bus| { + let candidates: Vec = bus + .columns_read() + .into_iter() + .filter(|&column| column < num_main_columns) + .collect(); + let numerator = probe(&candidates, &mut slot_of, |column| { + let value = bus.multiplicity.evaluate_with(column); + if bus.is_sender { value } else { -value } + })?; + let elements = probe_many(&candidates, &mut slot_of, |column| { + bus.values + .iter() + .flat_map(|value| value.combine_from(column)) + .collect() + })?; + Ok(InteractionShape { + numerator, + bus_id: FieldElement::::from(bus.bus_id), + elements, + }) + }) + .collect() +} + +/// The identity map: factor `i` reads main column `i`. +/// +/// What a table with no constraints of its own uses, where the factor list is +/// just its columns. +pub fn columns_as_factors(column: usize) -> Result { + Ok(column) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, + }; + use multilinear::{ + batch::Rule, + eq::eq_mle, + gkr::{self, FractionTree}, + logup::{self, input_layer}, + mle::Mle, + }; + + use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, + }; + use crate::proof::options::ProofOptions; + use crate::traits::AIR; + + type FE = FieldElement; + type ExtE = FieldElement; + + /// The LogUp challenges. Fixed here; the protocol draws them after the + /// trace is committed. + fn challenges() -> (ExtE, ExtE) { + (ExtE::from(0x9E37_79B9u64), ExtE::from(31u64)) + } + + fn base(values: &[u64]) -> Vec { + values.iter().map(|v| FE::from(*v)).collect() + } + + /// One factor per main column, lifted into the extension. + fn factors(columns: &[Vec]) -> Vec> { + columns + .iter() + .map(|c| { + Mle::new( + c.iter() + .map(|v| v.to_extension::()) + .collect::>(), + ) + .unwrap() + }) + .collect() + } + + /// The CPU trace from the multi-table completeness test: eight rows + /// dispatching four additions and four multiplications. + fn cpu_columns() -> Vec> { + vec![ + base(&[1, 0, 1, 0, 1, 1, 0, 0]), // add flag + base(&[0, 1, 0, 1, 0, 0, 1, 1]), // mul flag + base(&[1, 2, 3, 4, 5, 6, 7, 8]), // a + base(&[10, 20, 30, 40, 50, 60, 70, 80]), // b + base(&[11, 40, 33, 160, 55, 66, 490, 640]), // c + ] + } + + /// The additions the CPU dispatched, each received once. + fn add_columns() -> Vec> { + vec![ + base(&[1, 3, 5, 6]), + base(&[10, 30, 50, 60]), + base(&[11, 33, 55, 66]), + base(&[1, 1, 1, 1]), + ] + } + + /// The multiplications, likewise. + fn mul_columns() -> Vec> { + vec![ + base(&[2, 4, 7, 8]), + base(&[20, 40, 70, 80]), + base(&[40, 160, 490, 640]), + base(&[1, 1, 1, 1]), + ] + } + + /// `p/q` of a table's whole bus contribution. + fn contribution(interactions: &[Interaction], columns: &[Vec]) -> ExtE { + let tree = + FractionTree::build(input_layer(interactions, &factors(columns)).unwrap()).unwrap(); + let (p, q) = tree.output(); + p * q.inv().unwrap() + } + + fn cpu_interactions() -> Vec> { + let (z, alpha) = challenges(); + let options = ProofOptions::default_test_options(); + let air = new_cpu_air_with_lookup(&options); + interactions(air.bus_interactions(), 5, &z, &alpha, columns_as_factors).unwrap() + } + + fn add_interactions() -> Vec> { + let (z, alpha) = challenges(); + let options = ProofOptions::default_test_options(); + let air = new_add_air_with_lookup(&options); + interactions(air.bus_interactions(), 4, &z, &alpha, columns_as_factors).unwrap() + } + + fn mul_interactions() -> Vec> { + let (z, alpha) = challenges(); + let options = ProofOptions::default_test_options(); + let air = new_mul_air_with_lookup(&options); + interactions(air.bus_interactions(), 4, &z, &alpha, columns_as_factors).unwrap() + } + + /// The affine form must agree with the formula the univariate prover uses, + /// worked out by hand for this packing: `Direct.columns(&[2, 3, 4])` puts + /// each column in its own bus element, so the fingerprint is + /// `bus_id + alpha·a + alpha²·b + alpha³·c`. + #[test] + fn the_affine_form_matches_the_fingerprint_by_hand() { + let (z, alpha) = challenges(); + let columns = cpu_columns(); + let cpu = cpu_interactions(); + let factors = factors(&columns); + assert_eq!(cpu.len(), 2); + + for (bus_id, interaction) in cpu.iter().enumerate() { + for row in 0..columns[0].len() { + let values: Vec = factors.iter().map(|f| f.evals()[row]).collect(); + let a = values[2]; + let b = values[3]; + let c = values[4]; + let fingerprint = ExtE::from(bus_id as u64) + + alpha * a + + alpha * alpha * b + + alpha * alpha * alpha * c; + + assert_eq!( + interaction.denominator.evaluate(&values), + z - fingerprint, + "bus {bus_id}, row {row}" + ); + // Both CPU interactions are senders, reading their flag column. + assert_eq!( + interaction.numerator.evaluate(&values), + values[bus_id], + "bus {bus_id}, row {row}" + ); + } + } + } + + /// A receiver's sign is in its numerator, so the same fingerprint appears + /// on both sides of the bus with opposite multiplicity. + #[test] + fn a_receiver_carries_the_negative_multiplicity() { + let columns = add_columns(); + let factors = factors(&columns); + let add = add_interactions(); + assert_eq!(add.len(), 1); + + for row in 0..columns[0].len() { + let values: Vec = factors.iter().map(|f| f.evals()[row]).collect(); + assert_eq!(add[0].numerator.evaluate(&values), -values[3]); + } + } + + /// The milestone: the buses of a real three-table example balance when + /// computed entirely through the multilinear path — one fraction tree per + /// table, over tables of different heights. + #[test] + fn three_real_tables_balance_across_their_buses() { + let total = contribution(&cpu_interactions(), &cpu_columns()) + + contribution(&add_interactions(), &add_columns()) + + contribution(&mul_interactions(), &mul_columns()); + assert_eq!(total, ExtE::zero()); + } + + #[test] + fn a_receive_that_never_happened_unbalances_the_bus() { + let mut columns = add_columns(); + columns[3][2] = FE::zero(); // one addition goes unreceived + + let total = contribution(&cpu_interactions(), &cpu_columns()) + + contribution(&add_interactions(), &columns) + + contribution(&mul_interactions(), &mul_columns()); + assert_ne!(total, ExtE::zero()); + } + + #[test] + fn a_value_that_was_never_sent_unbalances_the_bus() { + let mut columns = add_columns(); + columns[2][1] += FE::one(); // the received sum is not the one dispatched + + let total = contribution(&cpu_interactions(), &cpu_columns()) + + contribution(&add_interactions(), &columns) + + contribution(&mul_interactions(), &mul_columns()); + assert_ne!(total, ExtE::zero()); + } + + /// And the claim GKR leaves about a real table's input layer is what two + /// rules over that table's columns sum to — the whole point of keeping the + /// numerator and denominator out of the commitment. + #[test] + fn a_real_tables_input_claim_reduces_to_two_rules() { + let columns = cpu_columns(); + let cpu = cpu_interactions(); + let factors = factors(&columns); + let num_row_vars = factors[0].num_vars(); + + let tree = FractionTree::build(input_layer(&cpu, &factors).unwrap()).unwrap(); + let mut transcript = DefaultTranscript::::new(b"logup-bridge"); + let out = gkr::prove(&tree, &mut transcript).unwrap(); + + let weight = factors.len(); + let statements = + logup::claim_statements(&cpu, &out.claim.point, num_row_vars, weight).unwrap(); + let mut with_weight = factors; + with_weight.push(eq_mle(&statements.row_point).unwrap()); + + let sum = |rule: &Rule<'_, Ext>| { + (0..(1usize << num_row_vars)).fold(ExtE::zero(), |acc, i| { + let values: Vec = with_weight.iter().map(|f| f.evals()[i]).collect(); + acc + rule.apply(&values) + }) + }; + assert_eq!(sum(&statements.numerator), out.claim.p); + assert_eq!(sum(&statements.denominator), out.claim.q); + } + + /// Deterministic extension values, one per main column. + fn column_values(seed: u64, count: usize) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..count) + .map(|_| { + let (a, b, c) = (next(), next(), next()); + ExtE::new([a, b, c]) + }) + .collect() + } + + /// The affine recovery over a VECTOR agrees, component by component, with + /// the scalar one it is written beside. + /// + /// The two are separate code because `probe` sits on a path whose cost is + /// the number of eval calls, and expressing it as a one-component + /// `probe_many` would allocate a `Vec` per call. This test is what holds + /// them together: a drift in either fires here. + #[test] + fn the_vector_probe_is_the_scalar_one_component_by_component() { + let options = ProofOptions::default_test_options(); + let air = new_cpu_air_with_lookup(&options); + let mut components = 0; + for bus in air.bus_interactions() { + let candidates: Vec = bus + .columns_read() + .into_iter() + .filter(|&column| column < 5) + .collect(); + let mut slot_of = columns_as_factors; + let together: Vec> = probe_many(&candidates, &mut slot_of, |column| { + bus.values + .iter() + .flat_map(|value| value.combine_from(column)) + .collect() + }) + .unwrap(); + assert!( + !together.is_empty(), + "the interaction must have bus elements for this to compare anything" + ); + for (index, component) in together.iter().enumerate() { + let mut slot_of = columns_as_factors; + let alone: Affine = probe(&candidates, &mut slot_of, |column| { + bus.values + .iter() + .flat_map(|value| value.combine_from(column)) + .nth(index) + .expect("the element index is inside the interaction") + }) + .unwrap(); + assert_eq!(component.terms(), alone.terms(), "element {index}'s terms"); + assert_eq!( + component.constant_term(), + alone.constant_term(), + "element {index}'s constant" + ); + components += 1; + } + } + assert_eq!(components, 6, "two interactions of three bus elements each"); + } + + /// ★ The challenge-free shapes rebuild what `interactions` fuses. + /// + /// This is the whole contract: a caller holding the bus at emit time and + /// the challenges only at run time can put them back together and land on + /// what the verifier would have computed. A wrong alpha offset, a reversed + /// element order, a dropped bus id, a lost receiver sign or a dropped + /// out-of-range filter each break it — and the challenges are varied + /// because an identity that only holds at one `alpha` is not one. + #[test] + fn the_shapes_recombine_into_the_fused_interactions() { + let options = ProofOptions::default_test_options(); + let cpu = new_cpu_air_with_lookup(&options); + let add = new_add_air_with_lookup(&options); + let mul = new_mul_air_with_lookup(&options); + let cases: Vec<(&str, &[BusInteraction], usize)> = vec![ + ("cpu", cpu.bus_interactions(), 5), + ("add", add.bus_interactions(), 4), + ("mul", mul.bus_interactions(), 4), + // ⚠ The CPU bus reads columns 0..=4; declaring only three main + // columns is what exercises the `column < num_main_columns` + // filter, which pins columns 3 and 4 to ZERO on BOTH sides rather + // than carrying them. Without this case the filter is a line no + // gate reaches. + ("cpu narrowed to 3 main columns", cpu.bus_interactions(), 3), + ]; + for (name, buses, width) in cases { + let mut slot_of = columns_as_factors; + let shapes: Vec> = + interaction_shapes(buses, width, &mut slot_of).unwrap(); + assert_eq!(shapes.len(), buses.len(), "{name}"); + + for (index, (z, alpha)) in [ + (ExtE::from(7u64), ExtE::from(11u64)), + (ExtE::from(0x9E37_79B9u64), ExtE::from(31u64)), + ( + ExtE::new([FE::from(3u64), FE::from(5u64), FE::from(9u64)]), + ExtE::new([FE::from(2u64), FE::from(0u64), FE::from(4u64)]), + ), + ] + .into_iter() + .enumerate() + { + let fused: Vec> = + interactions(buses, width, &z, &alpha, columns_as_factors).unwrap(); + let values = column_values(0xB0_5A11 + index as u64, width); + + for (i, (fused_i, shape)) in fused.iter().zip(&shapes).enumerate() { + assert_eq!( + fused_i.numerator.evaluate(&values), + shape.numerator.evaluate(&values), + "{name} interaction {i}: the numerator reads no challenge and must be identical" + ); + + // `fingerprint_at`'s own loop: the bus id at alpha^0, then + // each element at the next power. + let mut power = alpha; + let mut fingerprint = shape.bus_id; + for element in &shape.elements { + fingerprint += &power * element.evaluate(&values); + power = &power * α + } + assert_eq!( + fused_i.denominator.evaluate(&values), + &z - &fingerprint, + "{name} interaction {i} at challenge set {index}" + ); + + // The same statement about the TERM LIST, which the value + // comparison cannot make: the slots the fused affine keeps + // are exactly the slots some element reads. ⚠ True with + // overwhelming probability rather than always — a fused + // coefficient is a polynomial in alpha and could vanish at + // a particular one — so this is a check on these + // challenges, not a theorem. + let mut fused_slots: Vec = fused_i + .denominator + .terms() + .iter() + .map(|(s, _)| *s) + .collect(); + fused_slots.sort_unstable(); + let mut element_slots: Vec = shape + .elements + .iter() + .flat_map(|e| e.terms().iter().map(|(s, _)| *s)) + .collect(); + element_slots.sort_unstable(); + element_slots.dedup(); + assert_eq!(fused_slots, element_slots, "{name} interaction {i}'s slots"); + } + } + } + } + + /// The element ORDER and the sign, worked out by hand for this bus. + /// + /// `Packing::Direct.columns(&[2, 3, 4])` gives each column its own bus + /// element, so the CPU sender's fingerprint is + /// `bus_id + alpha·c2 + alpha²·c3 + alpha³·c4` — the same hand derivation + /// the fused test above this one makes, stated against the pieces rather + /// than against the sum, which is where an off-by-one in the power would + /// otherwise hide behind a matching total. + #[test] + fn the_elements_are_the_columns_in_fingerprint_order() { + let options = ProofOptions::default_test_options(); + let cpu = new_cpu_air_with_lookup(&options); + let mut slot_of = columns_as_factors; + let shapes: Vec> = + interaction_shapes(cpu.bus_interactions(), 5, &mut slot_of).unwrap(); + assert_eq!(shapes.len(), 2); + for (i, shape) in shapes.iter().enumerate() { + // A sender's numerator is its multiplicity, unnegated: column `i` + // with coefficient one and no constant. + assert_eq!(shape.numerator.terms(), &[(i, ExtE::one())]); + assert_eq!(shape.numerator.constant_term(), &ExtE::zero()); + assert_eq!(shape.elements.len(), 3); + for (p, element) in shape.elements.iter().enumerate() { + assert_eq!( + element.terms(), + &[(p + 2, ExtE::one())], + "element {p} of interaction {i} reads column {}", + p + 2 + ); + assert_eq!(element.constant_term(), &ExtE::zero()); + } + } + + // A RECEIVER's numerator carries the sign, which the senders above + // cannot show. + let add = new_add_air_with_lookup(&options); + let mut slot_of = columns_as_factors; + let received: Vec> = + interaction_shapes(add.bus_interactions(), 4, &mut slot_of).unwrap(); + assert_eq!(received[0].numerator.terms(), &[(3, -ExtE::one())]); + } + + /// Columns no interaction reads never need a factor: the probe only asks + /// about the ones with a nonzero coefficient. + #[test] + fn unread_columns_are_never_asked_for() { + let (z, alpha) = challenges(); + let options = ProofOptions::default_test_options(); + let air = new_mul_air_with_lookup(&options); + // The MUL table reads columns 0, 1, 2 and 3 — so widen it and check the + // extra column is not demanded. + let result = interactions(air.bus_interactions(), 6, &z, &alpha, |column| { + if column >= 4 { + panic!("column {column} is not read by any interaction"); + } + Ok(column) + }); + assert!(result.is_ok()); + } +} diff --git a/crypto/stark/src/multilinear_table.rs b/crypto/stark/src/multilinear_table.rs new file mode 100644 index 000000000..fde55bd7e --- /dev/null +++ b/crypto/stark/src/multilinear_table.rs @@ -0,0 +1,4494 @@ +//! The argument for one table of this VM: its own constraints zerochecked, its +//! buses proved by LogUp-GKR, and both discharged in **one** sumcheck against +//! one commitment per column. +//! +//! Where the univariate prover commits auxiliary columns and proves LogUp with +//! transition constraints, this proves it with a fraction tree. So a table's +//! program is used in two parts: `roots[..num_base]` — the table's own +//! constraints — are zerochecked, and the LogUp roots after them are +//! **dropped**, the bus replacing them. Dropping them drops every auxiliary +//! read along with them, which is why only main columns are committed here. +//! +//! The bus balance is not a table's own business: its contribution is a +//! fraction, and only the sum over every table in the proof has to vanish. + +use math::{ + field::{ + element::FieldElement, + traits::{IsFFTField, IsField, IsPrimeField, IsSubFieldOf}, + }, + traits::AsBytes, +}; +use std::sync::Arc; + +use multilinear::{ + Error as MlError, + batch::{Rule, Weight}, + claim_reduce, + constraint_argument::{self, ConstraintCore, FactorKind, TraceData}, + eq::{eq_eval, eq_mle}, + gkr::{self, FractionTree, GkrProof}, + logup, + mle::Mle, + stacked_eval::{self, Claimed, StackedCommitment, StackedProof}, + stacking::StackedLayout, + whir::Domain, + whir_chain::{ArgueFormat, ChainConfig, StackVars}, + whir_commit::Commitment, + whir_hash::{KeccakWhir, WhirHash}, +}; + +use crate::constraint_ir::ir::ConstraintProgram; +use crate::constraints::builder::ConstraintMeta; +use crate::lookup::BusInteraction; +use crate::multilinear_air; +use crate::multilinear_air::{ColumnKey, IrShape, LeafLayout, Uniforms, live_nodes}; +use crate::multilinear_logup; +use multilinear::selector::Selector; + +pub mod batched; +pub use batched::{ + ArguePlan, ArgueShape, BatchedArgue, BatchedMultiProof, Chunking, argue_plan, + argue_plan_packed, chunking, multi_prove_batched, multi_verify_batched, + multi_verify_batched_settled, +}; + +/// A table's structure, with no trace in it: what the factors are, what gets +/// committed, and over what domain. +/// +/// **Both sides build one.** Everything here comes from the program, the +/// constraint metadata, the bus interactions and the shape of the trace — never +/// from its values — so a verifier holding the same arguments derives the same +/// factor slots, the same stack and the same domain. Only the commitment roots +/// come out of the proof, and [`statement`](Self::statement) is where they join. +/// +/// Keeping the construction in one place is the point: if the two sides laid +/// out factors separately they could assign a column a different slot, and +/// every claim would then be about a different table than the one committed. +pub struct TableLayout<'a, F, E> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf, + E: IsField, +{ + shape: IrShape, + interactions: &'a [BusInteraction], + leaves: LeafLayout, + /// Main column -> the factor that reads it unshifted. + slot_of: Vec, + kinds: Vec, + num_vars: usize, +} + +impl<'a, F, E> TableLayout<'a, F, E> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf, + E: IsField + 'static, +{ + /// Lays out a table's factors and its stack. + /// + /// Every main column is committed, read or not, **in index order** — the + /// trace is the trace, a bus reads columns the constraints may not, and two + /// AIRs over the same table have to commit the same polynomial. + /// + /// `uniforms` is normally [`Uniforms::default`]: a table's own constraint + /// set is base-rooted, so it reads no challenge. + #[allow(clippy::too_many_arguments)] + pub fn new( + program: &'a ConstraintProgram, + meta: &[ConstraintMeta], + interactions: &'a [BusInteraction], + num_main_columns: usize, + num_vars: usize, + uniforms: Uniforms, + ) -> Result { + if interactions.is_empty() { + // A table with no bus is just `constraint_argument::prove`. + return Err(MlError::EmptyPolynomial); + } + if meta.len() < program.num_base { + return Err(MlError::VariableCountMismatch { + expected: program.num_base, + got: meta.len(), + }); + } + + let roots = program.roots[..program.num_base].to_vec(); + let live = live_nodes(program, &roots); + // The main columns are registered first, in index order, so what a + // table commits and in what order is a function of the table alone — + // not of which of them this AIR's constraints happen to read first. + let mut leaves = LeafLayout::build_live_over(program, &live, num_vars, num_main_columns); + + let slot_of: Vec = (0..num_main_columns) + .map(|col| leaves.register_main(col as u16)) + .collect(); + + let selectors: Vec = meta[..program.num_base] + .iter() + .map(|m| Selector::except_last(m.end_exemptions)) + .collect(); + let (shape, kinds) = IrShape::build(program, &leaves, uniforms, roots, &selectors)?; + + Ok(Self { + shape, + interactions, + leaves, + slot_of, + kinds, + num_vars, + }) + } + + /// The statement this layout describes, with no preprocessed columns to + /// check. Purely structural: the commitment roots are not in it, because + /// they come out of the proof. + pub fn statement(&self) -> TableStatement<'_, F, E> { + self.statement_with_preprocessed(&[]) + } + + /// The same, with the table's preprocessed columns for the verifier to + /// check the claimed openings against — see [`TableStatement::preprocessed`]. + /// + /// The table's preprocessed count is the number of columns handed in: the + /// verifier holds a copy of every one of them. + pub fn statement_with_preprocessed<'s>( + &'s self, + preprocessed: &'s [Mle], + ) -> TableStatement<'s, F, E> { + TableStatement { + shape: &self.shape, + interactions: self.interactions, + slot_of: &self.slot_of, + preprocessed, + num_preprocessed: preprocessed.len(), + kinds: &self.kinds, + num_vars: self.num_vars, + } + } + + /// The statement of a table whose leading `count` columns are preprocessed + /// and of which the verifier holds NO copy: a prepared opening must settle + /// all `count` of them, and [`multi_verify`] refuses the table otherwise. + /// + /// ⛔ `count` IS THE AIR'S, never the length of a column list. An AIR that + /// declares its preprocessed columns by count alone (`with_preprocessed`) + /// has no column builder, so its `precomputed_columns()` is empty; a + /// statement built from that list would say "nothing to check" about a + /// table whose program columns are exactly what binds the proof to its + /// program. Taking the count from the AIR is what makes an unsettled prefix + /// a refusal instead of an unchecked column. + pub fn statement_with_prepared_prefix(&self, count: usize) -> TableStatement<'_, F, E> { + TableStatement { + shape: &self.shape, + interactions: self.interactions, + slot_of: &self.slot_of, + preprocessed: &[], + num_preprocessed: count, + kinds: &self.kinds, + num_vars: self.num_vars, + } + } + + pub fn shape(&self) -> &IrShape { + &self.shape + } + + /// Main column -> the factor that reads it unshifted. + pub fn slot_of(&self) -> &[usize] { + &self.slot_of + } + + pub fn kinds(&self) -> &[FactorKind] { + &self.kinds + } + + pub fn num_vars(&self) -> usize { + self.num_vars + } + + /// How many columns get committed — main columns only. + pub fn num_columns(&self) -> usize { + self.leaves.num_columns() + } + + /// The columns to commit, in the order they must be materialized. + pub fn column_keys(&self) -> &[ColumnKey] { + self.leaves.column_keys() + } +} + +/// One table's structure and its trace, with **no commitment of its own**. +/// +/// A table used to carry its own, so a proof of N tables paid N openings — and +/// the opening is almost all of a proof. Here the tables hand their columns to +/// [`CommittedTables`], which commits every one of them together. +pub struct CommittedTable<'a, F, E> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + layout: TableLayout<'a, F, E>, + trace: TraceData, +} + +impl<'a, F, E> CommittedTable<'a, F, E> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + /// Lays out the factors and materializes every main column. + /// + /// `main_column` returns a column's values by step, `2^num_vars` of them. + /// Auxiliary columns are never asked for: the only constraints that read + /// them are the LogUp ones, and the bus replaces those. + pub fn new( + program: &'a ConstraintProgram, + meta: &[ConstraintMeta], + interactions: &'a [BusInteraction], + num_main_columns: usize, + num_vars: usize, + uniforms: Uniforms, + main_column: impl FnMut(u16) -> Vec>, + ) -> Result { + let layout = TableLayout::new( + program, + meta, + interactions, + num_main_columns, + num_vars, + uniforms, + )?; + Self::from_layout(layout, main_column) + } + + /// The same against a layout already built — the same one the verifier will + /// rebuild, so the statement and the trace cannot describe different + /// tables. + /// Each column is asked for exactly once, in [`column_keys`] order, so a + /// caller holding its own copy may hand it over rather than clone it — + /// which on a real trace is the difference between one copy of it and two. + /// + /// [`column_keys`]: TableLayout::column_keys + pub fn from_layout( + layout: TableLayout<'a, F, E>, + mut main_column: impl FnMut(u16) -> Vec>, + ) -> Result { + debug_assert!( + { + let mut seen: Vec = layout.column_keys().iter().map(|k| k.col).collect(); + seen.sort_unstable(); + let asked = seen.len(); + seen.dedup(); + seen.len() == asked + }, + "a caller may move its columns in, so each one must be asked for once", + ); + let size = 1usize << layout.num_vars(); + let mut columns = Vec::with_capacity(layout.num_columns()); + for key in layout.column_keys() { + assert!( + key.main, + "the bus replaces every constraint that reads an auxiliary column" + ); + let values = main_column(key.col); + if values.len() != size { + return Err(MlError::NotPowerOfTwo(values.len())); + } + columns.push(Mle::new(values)?); + } + + let trace = TraceData::new( + columns, + layout.kinds().to_vec(), + layout.shape().public_tables()?, + )?; + Ok(Self { layout, trace }) + } + + /// [`Self::from_layout`] over columns held narrow from the start: `packed` + /// holds the layout's columns in [`column_keys`] order, each the table's + /// main column of that index — so only a layout whose keys are the main + /// columns `0..n` in order takes it. + /// + /// [`column_keys`]: TableLayout::column_keys + pub fn from_narrow( + layout: TableLayout<'a, F, E>, + packed: multilinear::narrow::NarrowColumns, + ) -> Result { + let in_order = layout + .column_keys() + .iter() + .enumerate() + .all(|(i, key)| key.main && key.col as usize == i); + if !in_order || layout.num_columns() != packed.cols() { + return Err(MlError::QueryCountMismatch { + expected: layout.num_columns(), + got: packed.cols(), + }); + } + if packed.rows() != 1usize << layout.num_vars() { + return Err(MlError::NotPowerOfTwo(packed.rows())); + } + let trace = TraceData::new_narrow( + packed, + layout.kinds().to_vec(), + layout.shape().public_tables()?, + )?; + Ok(Self { layout, trace }) + } + + /// [`Self::from_narrow`] over columns held as the records their rows come + /// from ([`multilinear::records`]): the card fills them as the group + /// uploads them. The same layouts only. + pub fn from_records( + layout: TableLayout<'a, F, E>, + records: multilinear::records::Records, + ) -> Result { + let in_order = layout + .column_keys() + .iter() + .enumerate() + .all(|(i, key)| key.main && key.col as usize == i); + if !in_order || layout.num_columns() != records.cols() { + return Err(MlError::QueryCountMismatch { + expected: layout.num_columns(), + got: records.cols(), + }); + } + if records.rows() != 1usize << layout.num_vars() { + return Err(MlError::NotPowerOfTwo(records.rows())); + } + let trace = TraceData::new_records( + records, + layout.kinds().to_vec(), + layout.shape().public_tables()?, + )?; + Ok(Self { layout, trace }) + } + + /// The structure alone — what the verifier holds. + pub fn layout(&self) -> &TableLayout<'a, F, E> { + &self.layout + } + + /// This table's statement. + pub fn statement(&self) -> TableStatement<'_, F, E> { + self.layout.statement() + } + + pub fn kinds(&self) -> &[FactorKind] { + self.trace.kinds() + } + + pub fn shape(&self) -> &IrShape { + self.layout.shape() + } + + /// Main column -> the factor that reads it unshifted. + pub fn slot_of(&self) -> &[usize] { + self.layout.slot_of() + } + + pub fn num_vars(&self) -> usize { + self.trace.num_vars() + } + + /// How many columns get committed — main columns only. + pub fn num_committed_columns(&self) -> usize { + self.trace.num_columns() + } + + /// Points the table's columns at a device copy of them (see + /// [`TraceData::set_resident`]) — the block prover's per-group store. + pub(crate) fn set_resident( + &mut self, + store: Arc, + first: usize, + ) { + self.trace.set_resident(store, first); + } + + /// Lets the device copy go; the host columns stay. + pub(crate) fn clear_resident(&mut self) { + self.trace.clear_resident(); + } + + /// The committed columns themselves, on the host — widened there if the + /// table holds them narrow (see [`TraceData::columns`]). The opening needs + /// them: a stacked polynomial is these at their offsets, and it is never + /// assembled. + pub fn columns(&self) -> &[Mle] { + self.trace.columns() + } + + /// The table's trace: its columns' shape and, on demand, their values + /// ([`multilinear::narrow::HostColumns`]). + pub fn trace(&self) -> &TraceData { + &self.trace + } + + /// The columns as they go to the card: their field elements, or packed. + pub(crate) fn upload_view(&self) -> multilinear::gpu::TableColumns<'_, F> { + match (self.trace.narrow(), self.trace.records()) { + (Some(packed), _) => multilinear::gpu::TableColumns::Narrow(packed), + (None, Some(records)) => multilinear::gpu::TableColumns::Records(records), + (None, None) => multilinear::gpu::TableColumns::Wide(self.trace.columns()), + } + } + + /// Holds the columns as `packed` from here on (see + /// [`TraceData::install_narrow`]). + pub(crate) fn install_narrow(&mut self, packed: multilinear::narrow::NarrowColumns) -> bool { + self.trace.install_narrow(packed) + } + + /// Hands the packed columns to a spill store (see + /// [`TraceData::take_narrow_for_spill`]). + pub(crate) fn take_narrow_for_spill(&mut self) -> Option { + self.trace.take_narrow_for_spill() + } + + /// Brings spilled columns back (see [`TraceData::restore_narrow`]). + pub(crate) fn restore_narrow(&mut self, packed: multilinear::narrow::NarrowColumns) -> bool { + self.trace.restore_narrow(packed) + } + + /// The records out, for the spill (see [`TraceData::take_records_for_spill`]). + pub(crate) fn take_records_for_spill(&mut self) -> Option { + self.trace.take_records_for_spill() + } + + /// Puts spilled records back (see [`TraceData::restore_records`]). + pub(crate) fn restore_records(&mut self, records: multilinear::records::Records) -> bool { + self.trace.restore_records(records) + } + + /// Whether the packed columns are out in a spill store's slot. + pub fn is_spilled(&self) -> bool { + self.trace.is_spilled() + } + + /// Packs the columns narrow on the host (see [`TraceData::pack_on_host`]). + pub(crate) fn pack_on_host(&mut self) -> bool { + self.trace.pack_on_host() + } + + /// Lets a host widening of narrow columns go (see + /// [`TraceData::drop_widened`]). + pub(crate) fn drop_widened(&mut self) { + self.trace.drop_widened(); + } + + /// The packed columns, when the table holds them narrow. + pub fn narrow(&self) -> Option<&multilinear::narrow::NarrowColumns> { + self.trace.narrow() + } + + /// The records the columns are filled from, when the table holds them so. + pub fn records(&self) -> Option<&multilinear::records::Records> { + self.trace.records() + } + + /// The bytes the table's columns take on the host as it holds them, when + /// that is less than eight a cell: packed, or as records. + pub fn compact_bytes(&self) -> Option { + match (self.trace.narrow(), self.trace.records()) { + (Some(packed), _) => Some(packed.data().len()), + (None, Some(records)) => Some(records.bytes()), + (None, None) => None, + } + } + + /// A test's hook: the packed columns, to break them. + #[doc(hidden)] + pub fn narrow_mut(&mut self) -> Option<&mut multilinear::narrow::NarrowColumns> { + self.trace.narrow_mut() + } +} + +/// Every table's columns, committed **once**. +/// +/// The opening is almost the whole proof, and a commitment costs one however +/// many columns it holds, so N tables committing separately pay N times for +/// what one commitment settles. Here the columns of every table go into one +/// stack, each keeps the height it has, and one opening answers all of them — +/// which is what [`Claimed::PerColumn`] exists for, since each table's sumcheck +/// ends at its own point. +/// +/// [`Claimed::PerColumn`]: multilinear::stacked_eval::Claimed::PerColumn +pub struct CommittedTables<'a, F, E, H = KeccakWhir> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + tables: Vec>, + /// The epoch's columns on the card, alive as long as the tables that read + /// them. `None` when there is no device or it would not promise the room. + store: Option>, + /// The stacks the tables are committed in, in table order. One is the usual + /// case; more than one exists so a table can have a commitment of its own — + /// which is what binds the same table across two proofs, since a table has + /// no root of its own when it shares a stack. + groups: Vec>, + /// How many tables each group holds, in order. + sizes: Vec, + roots: Vec, + /// Per table, how many LEADING columns the group stacks leave out — the + /// prefix a prepared opening settles instead (policy B). Empty when every + /// column is in its group's stack, which is every proof but a W-LFM one + /// under policy B. + settled: Vec, +} + +/// Where each table's columns start in the global column order. +/// +/// Both sides derive it from the table shapes alone, so a claim about table +/// `i`'s column `c` means the same thing to each. +pub fn column_offsets(widths: &[usize]) -> Vec { + widths + .iter() + .scan(0usize, |at, w| { + let start = *at; + *at += w; + Some(start) + }) + .collect() +} + +/// The stack every table's columns share, from their shapes and the format's +/// stack cap alone. +/// +/// `shapes` is `(columns, height in variables)` per table, in table order. The +/// heights differ and that is fine: a column takes the subcube it needs and the +/// next one starts after it, so stacking wastes less than each table rounding +/// up to its own power of two. +/// +/// Columns that do not fit spill into another polynomial at `cap` — see +/// [`StackVars`], which is what stops one proof-sized allocation. The cap is +/// the config's (`ChainConfig::format.stack`), so the prover and the verifier +/// stack under the same one, and never a proof's. +pub fn global_layout(shapes: &[(usize, usize)], cap: StackVars) -> Result { + let heights: Vec = shapes + .iter() + .flat_map(|&(width, num_vars)| std::iter::repeat_n(num_vars, width)) + .collect(); + StackedLayout::build(&heights, stack_height(shapes, cap)) +} + +/// The height, in variables, [`global_layout`] stacks `shapes` at under `cap`: +/// the smallest power of two that holds every column, at most `cap`, and never +/// narrower than the tallest column, or it would not fit at all. +/// +/// ★ Monotone in the shapes: a group's height is at most that of any shape +/// list containing it, so every group of a proof stacks at most as tall as all +/// of its shapes stacked together would. +pub fn stack_height(shapes: &[(usize, usize)], cap: StackVars) -> usize { + let cells: usize = shapes + .iter() + .map(|&(width, num_vars)| width << num_vars) + .sum(); + let want = cells.next_power_of_two().trailing_zeros() as usize; + let tallest = shapes + .iter() + .filter(|&&(width, _)| width > 0) + .map(|&(_, num_vars)| num_vars) + .max() + .unwrap_or(0); + want.min(cap.get()).max(tallest) +} + +/// The store column of every column a group stacks when each table's leading +/// `settled[k]` columns are left out: table `k`'s columns `settled[k]..width`, +/// which sit at `firsts[k] + settled[k]..firsts[k] + width` in a store that +/// keeps every table's columns contiguous. +fn settled_column_map(widths: &[usize], firsts: &[usize], settled: &[usize]) -> Vec { + widths + .iter() + .zip(firsts) + .zip(settled) + .flat_map(|((&width, &first), &skip)| (first + skip)..(first + width)) + .collect() +} + +/// A group's claims with each table's leading `settled[k]` columns left out, +/// gathered from the global per-column lists: the points and values the group's +/// stack opens under policy B. `starts[k]` is table `k`'s first global column. +fn unsettled_claims( + widths: &[usize], + starts: &[usize], + settled: &[usize], + points: &[Vec>], + values: &[FieldElement], +) -> Result, MlError> { + let mut at_points = Vec::new(); + let mut at_values = Vec::new(); + for ((&width, &start), &skip) in widths.iter().zip(starts).zip(settled) { + let range = start + skip..start + width; + at_points.extend( + points + .get(range.clone()) + .ok_or(MlError::QueryCountMismatch { + expected: start + width, + got: points.len(), + })? + .iter() + .cloned(), + ); + at_values.extend( + values + .get(range) + .ok_or(MlError::QueryCountMismatch { + expected: start + width, + got: values.len(), + })? + .iter() + .cloned(), + ); + } + Ok((at_points, at_values)) +} + +/// One layout per group, from the shapes, how the tables are split and the +/// stack cap. +pub fn global_layouts( + shapes: &[(usize, usize)], + sizes: &[usize], + cap: StackVars, +) -> Result, MlError> { + if sizes.iter().sum::() != shapes.len() { + return Err(MlError::QueryCountMismatch { + expected: shapes.len(), + got: sizes.iter().sum(), + }); + } + let mut layouts = Vec::with_capacity(sizes.len()); + let mut at = 0usize; + for &size in sizes { + layouts.push(global_layout(&shapes[at..at + size], cap)?); + at += size; + } + Ok(layouts) +} + +impl<'a, F, E, H> CommittedTables<'a, F, E, H> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + /// Commits every table's columns into one stack. + pub fn commit( + tables: Vec>, + config: &ChainConfig, + ) -> Result { + let sizes = [tables.len()]; + Self::commit_grouped(tables, &sizes, config) + } + + /// The same, with the tables split across several stacks. + /// + /// `sizes` is how many tables each group takes, in order, and must cover + /// them all. A group of one is a table with a commitment of its own: the + /// only way to say "this is the same table" to another proof, since a table + /// that shares a stack has no root to compare. + /// + /// Everything else is unchanged — the tables are argued in one transcript + /// against one set of roots, and each group is opened once. + pub fn commit_grouped( + tables: Vec>, + sizes: &[usize], + config: &ChainConfig, + ) -> Result { + Self::commit_grouped_settled(tables, sizes, config, &[]) + } + + /// [`Self::commit_grouped`] with each table's leading `settled[t]` columns + /// LEFT OUT of its group's stack — policy B of a W-LFM proof, where a + /// table's preprocessed prefix is committed only in the prepared stack and + /// its claims are settled only by the prepared opening. + /// + /// ★ The argument still reads every column: the tables' factors, their GKR + /// input and their reduction are over the whole trace, so the resident + /// store keeps each table's columns contiguous as before, and a group's + /// stack reads its own columns out of it through a column map. + /// + /// `settled` empty is [`Self::commit_grouped`] exactly. [`multi_prove`] + /// refuses a proof whose prepared opening does not settle exactly these + /// prefixes, since a column in neither stack would be bound by nothing. + pub fn commit_grouped_settled( + mut tables: Vec>, + sizes: &[usize], + config: &ChainConfig, + settled: &[usize], + ) -> Result { + if sizes.iter().sum::() != tables.len() { + return Err(MlError::QueryCountMismatch { + expected: tables.len(), + got: sizes.iter().sum(), + }); + } + if !settled.is_empty() { + if settled.len() != tables.len() { + return Err(MlError::QueryCountMismatch { + expected: tables.len(), + got: settled.len(), + }); + } + for (table, &count) in tables.iter().zip(settled) { + if count >= table.num_committed_columns() { + return Err(MlError::QueryCountMismatch { + expected: table.num_committed_columns(), + got: count, + }); + } + } + } + let settled_of = |table: usize| settled.get(table).copied().unwrap_or(0); + // The device ledger's peak over the whole commit, under + // `LAMBDA_VM_BASE_SPLIT=1` — `RESERVED HW`'s `commit`. + let __rw_commit = multilinear::whir_split::open_reserved(); + // The epoch's columns, on the card once. Four things read them — the + // commitment, the sumcheck's factors, the evaluation at the reduction + // point and the opening's message — and each used to upload its own + // copy. + let all: Vec<&Mle> = tables.iter().flat_map(|t| t.columns()).collect(); + let store = multilinear::gpu::upload_columns(&all).map(Arc::new); + // Where each table's columns start in it. + let mut firsts = Vec::with_capacity(tables.len()); + let mut column_at = 0usize; + for table in &tables { + firsts.push(column_at); + column_at += table.num_committed_columns(); + } + drop(all); + + let mut groups = Vec::with_capacity(sizes.len()); + let mut roots = Vec::new(); + let mut at = 0usize; + for &size in sizes { + let group = &tables[at..at + size]; + let shapes: Vec<(usize, usize)> = group + .iter() + .enumerate() + .map(|(k, t)| (t.num_committed_columns() - settled_of(at + k), t.num_vars())) + .collect(); + let layout = global_layout(&shapes, config.format.stack)?; + // By reference: the stack copies every column into its own buffer, + // and the trace holds the originals for the rest of the proof. + let columns: Vec<&Mle> = group + .iter() + .enumerate() + .flat_map(|(k, t)| &t.columns()[settled_of(at + k)..]) + .collect(); + let stacked = if settled.is_empty() { + StackedCommitment::::commit( + layout, + &columns, + store.as_ref().map(|store| (&**store, firsts[at])), + config, + )? + } else { + let widths: Vec = group.iter().map(|t| t.num_committed_columns()).collect(); + let map = + settled_column_map(&widths, &firsts[at..at + size], &settled[at..at + size]); + StackedCommitment::::commit_mapped( + layout, + &columns, + store + .as_ref() + .map(|store| (&**store, stacked_eval::ColumnsAt::Map(&map))), + config, + )? + }; + roots.extend(stacked.roots()); + groups.push(stacked); + at += size; + } + // Each table points at its own run, so its factors and its reduction + // read them where they lie. + if let Some(store) = &store { + for (table, first) in tables.iter_mut().zip(&firsts) { + table.trace.set_resident(store.clone(), *first); + } + } + multilinear::whir_split::close_reserved( + &multilinear::whir_split::RESERVED_COMMIT, + __rw_commit, + ); + Ok(Self { + tables, + store, + groups, + sizes: sizes.to_vec(), + roots, + settled: settled.to_vec(), + }) + } + + pub fn tables(&self) -> &[CommittedTable<'a, F, E>] { + &self.tables + } + + /// Per table, the leading columns the group stacks leave out — empty when + /// every column is in its stack. + pub fn settled(&self) -> &[usize] { + &self.settled + } + + /// Every group's roots, in order — what the transcript absorbs. + pub fn roots(&self) -> &[Commitment] { + &self.roots + } + + /// How many tables each group holds. + pub fn sizes(&self) -> &[usize] { + &self.sizes + } + + /// The stacks, one per group. + pub fn groups(&self) -> &[StackedCommitment] { + &self.groups + } +} + +/// What the verifier holds: the table's structure and what was committed. +pub struct TableStatement<'a, F: IsFFTField + IsPrimeField, E: IsField> { + pub shape: &'a IrShape, + pub interactions: &'a [BusInteraction], + /// Main column -> the factor that reads it unshifted. + pub slot_of: &'a [usize], + /// The table's **preprocessed** columns, `0..n` of the main trace: the ones + /// the program determines and the verifier can therefore recompute. + /// + /// Empty on the proving side, and empty for a table that has none. The + /// commitment only says the prover stayed consistent with what it + /// committed; these are what say it committed the right thing. + pub preprocessed: &'a [Mle], + /// How many of the table's leading main columns ARE preprocessed. + /// + /// Equal to `preprocessed.len()` whenever the verifier holds the columns + /// ([`TableLayout::statement_with_preprocessed`]). Larger when it holds none + /// of them ([`TableLayout::statement_with_prepared_prefix`]), and then a + /// prepared opening must settle exactly this many, or the table is refused: + /// every preprocessed column is either opened out of band or recomputed + /// here, never neither. + pub num_preprocessed: usize, + pub kinds: &'a [FactorKind], + pub num_vars: usize, +} + +/// Every field is a reference or a length, so copying is free — spelled out +/// rather than derived, which would demand `F: Copy` and `E: Copy` of the field +/// markers. +impl Clone for TableStatement<'_, F, E> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for TableStatement<'_, F, E> {} + +/// A table's argument. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct TableProof { + pub gkr: GkrProof, + /// The bus's output fraction. A table's own contribution need not vanish — + /// the balance is over every table in the proof — so both halves travel and + /// checking the sum is the caller's. Lying about them yields an input-layer + /// claim the trace does not answer, so nothing has to be taken on trust. + pub bus_output: (FieldElement, FieldElement), + /// The table's argument up to its columns' claims. The opening that + /// settles them is [`MultiProof::columns`], shared with every other table. + pub constraint: ConstraintCore, +} + +/// ★★★ THE ROOTS BLOCK: every root into the transcript, then the three shared +/// challenges — used by `multi_prove`, by `multi_verify`, AND by every caller +/// that REPLAYS the block on a fork of the transcript. +/// +/// # Why it is one function +/// +/// The two sides must absorb the same roots in the same order and draw the same +/// challenges after them. Written twice, they can drift — and a drift here is +/// silent in the worst way, because a consistently wrong order still verifies: +/// prover and verifier agree with each other and disagree with the +/// specification. Sharing the code makes that particular disagreement +/// unspellable rather than merely tested for. +/// +/// # ⛔ The replays are callers too, and forgetting one is how this broke +/// +/// A verifier that needs `z` and `alpha` BEFORE `multi_verify` runs — to compute +/// the COMMIT bus's counterparty, which is a function of them — replays this +/// block on a clone of the transcript. Such a replay is a third side of the same +/// agreement, and it is public for exactly that reason: when the block grew the +/// derived root, the two call sites inside this module grew with it and a +/// hand-rolled replay in the prover crate did not, so the counterparty was +/// computed at challenges no table had been checked at and every epoch that +/// published output failed on `BusImbalance`. The epochs that published nothing +/// could not see it, because the counterparty is zero there without reading +/// either challenge. +/// +/// So: never spell this loop out again. Call [`absorb_roots`], pass the same +/// `derived` list the verification will be handed, and draw what you consume. +/// +/// # The order, and why `derived` is last +/// +/// The carried roots — the ones the proof actually contains — go in first, then +/// any derived root the verifier computed for itself. "Last" means last among +/// the roots, NOT after the challenges: a root absorbed after `z` is +/// indistinguishable IN `z` from a root never absorbed at all, so it would bind +/// nothing while looking wired from every counter and every byte gate. +/// +/// That is the one placement no round-trip can catch, which is why +/// `the_roots_block_binds_the_derived_root_to_the_first_challenge` compares this +/// against an independently built transcript rather than against the other side. +pub fn absorb_roots_and_challenge( + transcript: &mut T, + carried: &[Commitment], + derived: &[Commitment], +) -> (FieldElement, FieldElement, FieldElement) +where + E: IsField + 'static, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + absorb_roots::(transcript, carried, derived); + ( + transcript.sample_field_element(), + transcript.sample_field_element(), + transcript.sample_field_element(), + ) +} + +/// The half of the roots block that can DRIFT: which roots, in which order. +/// +/// Split out because a replay needs this half and not the other. The order is a +/// shared fact and is shared here; how many challenges are drawn afterwards is +/// the caller's own business, because a replay works on a fork it throws away +/// and only the first two challenges ever leave it. +/// +/// ⚠ A challenge nobody reads is neither free nor invisible. Each is a sponge +/// squeeze; `hash_metrics` counts squeezes on every transcript instance, a +/// clone included; and the output buffer hands out four candidates per squeeze, +/// so an unread draw moves a pinned squeeze count by an amount that depends on +/// where the buffer happened to be. A replay draws exactly what it consumes, +/// and the order it draws in is checked against +/// [`absorb_roots_and_challenge`] by a test rather than by a comment. +pub fn absorb_roots(transcript: &mut T, carried: &[Commitment], derived: &[Commitment]) +where + E: IsField + 'static, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + for root in carried { + transcript.append_bytes(root); + } + for root in derived { + transcript.append_bytes(root); + } +} + +/// Where one stacked column of a prepared commitment is settled: a table, and +/// **which** of that table's preprocessed columns. +/// +/// ★ THE TABLE INDEX IS WHAT GENERALISED; THE COLUMN INDEX IS WHAT KEEPS IT +/// HONEST. DECODE's prepared commitment covers ONE table's leading five columns. +/// The cross-epoch genesis stack covers one table PER DENSE PAGE, each settled +/// at its own reduced point — so the opening needed a table LIST, which is the +/// change. The column index rides along because a list of tables alone cannot +/// say WHICH columns of each, and the contract those columns must satisfy is +/// exact: see [`prefix_at`]. +/// +/// ⚠ A CROSS-EPOCH PAGE'S STACK CARRIES BOTH ITS COLUMNS, and that is a cost +/// paid to avoid changing a contract. A GLOBAL_MEMORY page presents +/// `[OFFSET, INIT]`; only INIT is worth an opening, since OFFSET is the identity +/// ramp whose extension costs `num_vars − 1` rows to check outright. But +/// `check_preprocessed` skips a PREFIX, and `{1}` is not one — so the stack +/// takes `{0, 1}`, the page loses its ramp, and `settled_out_of_band` stays a +/// count. On the block that is 51 rows lost against 282 gained, about 230 on a +/// hybrid costing order `10^5`. +/// +/// ⚠ BOTH FIELDS ARE PART OF THE STATEMENT, NOT HINTS. The opening binds this +/// column to ONE table's reduced point, and a prover who could aim a pinned +/// column at another table's point — or at another of the same table's columns +/// — would be settling it against challenges it was never bound to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PreparedColumn { + /// Which table's reduced point this column is settled at. + pub table: usize, + /// Which of that table's PREPROCESSED columns it is, by index. + pub column: usize, +} + +/// Where `table`'s prepared columns sit in `at`, and how many there are — +/// `None` when the commitment does not cover that table at all. +/// +/// ⛔⛔ **THE PREFIX CONTRACT, CHECKED RATHER THAN ASSUMED.** +/// [`check_preprocessed`] settles a table's FIRST `n` preprocessed columns: +/// it iterates `.skip(settled_out_of_band)`, and the in-guest mirror does the +/// same with `&columns[plan.settled..]`. So a commitment covering a table's +/// columns `{1}` — or `{0, 2}` — is INEXPRESSIBLE, and a caller that built one +/// would have the host skip the wrong columns while the opening settled others, +/// with every value gate still green. +/// +/// A leading COUNT could not express that defect because it could not describe +/// the intent either. [`PreparedColumn`] can, so this is where the two meet: +/// the entries for one table must be CONTIGUOUS in `at` and must be exactly +/// `0..n`, and anything else is an error rather than a silent reinterpretation. +/// +/// ⚠ CONTIGUITY IS PART OF IT, not a tidiness rule. The opening's claimed +/// values are taken as the run `values[start .. start + n]` of that table's own +/// columns, so entries interleaved with another table's would settle this +/// table's commitment against a slice that is not its own. +fn prefix_at(at: &[PreparedColumn], table: usize) -> Result, MlError> { + let Some(first) = at.iter().position(|c| c.table == table) else { + return Ok(None); + }; + let n = at[first..].iter().take_while(|c| c.table == table).count(); + // Contiguous: nothing belonging to this table sits outside that run. + if at.iter().filter(|c| c.table == table).count() != n { + return Err(MlError::QueryCountMismatch { + expected: n, + got: at.iter().filter(|c| c.table == table).count(), + }); + } + // And the run is the prefix `0..n`, in order. + for (offset, entry) in at[first..first + n].iter().enumerate() { + if entry.column != offset { + return Err(MlError::UnknownPolynomial { + index: entry.column, + len: n, + }); + } + } + Ok(Some((first, n))) +} + +/// One table's leading `columns` preprocessed columns — what a prepared +/// commitment over a single table's prefix covers, which is DECODE's case. +/// +/// Returned OWNED because [`Prepared`] borrows the slice and a self-referential +/// struct cannot hand one out; the caller holds it for as long as the opening, +/// exactly as it already holds the borrowed column references. +pub fn leading_columns(table: usize, columns: usize) -> Vec { + (0..columns) + .map(|column| PreparedColumn { table, column }) + .collect() +} + +/// What the verifier needs to settle a prepared commitment it derived itself. +/// +/// ⚠ `roots` are DERIVED — recomputed from the ELF by the verifier — never read +/// from the proof. A root taken from the proof would be a value absorbed before +/// it was checked. +pub struct PreparedCheck<'a, F> +where + F: IsFFTField + IsPrimeField + 'static, +{ + /// Recomputed from the ELF, not carried in the proof. + pub roots: &'a [Commitment], + pub layout: &'a StackedLayout, + pub domain: &'a Domain, + /// Where each stacked column is settled, parallel to the stack's columns. + pub at: &'a [PreparedColumn], +} + +/// A commitment built OUTSIDE this proof, opened at the points of the tables +/// whose preprocessed columns it covers. +/// +/// DECODE's five preprocessed columns are ELF-derived: the same bytes in every +/// epoch of every run of that program. Committing them per epoch and then +/// re-evaluating their MLEs per epoch is work that depends on nothing the epoch +/// chose. This carries a commitment built once per ELF, so the epoch pays an +/// opening instead of five 2^20 folds. +/// +/// ★ ONE STACK, SEVERAL TABLES, SEVERAL POINTS — and no new machinery for it. +/// The cross-epoch proof's genesis pages are one table each, so their INIT +/// columns are claimed at as many different reduced points as there are dense +/// pages. [`stacked_eval::Claimed::PerColumn`] has always resolved the point +/// per column — it is what every group opening in [`multi_prove`] already uses +/// — so the generality was in the opening all along and only this struct named +/// a single table. See [`PreparedColumn`]. +pub struct Prepared<'a, F, H> +where + F: IsFFTField + IsPrimeField + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, +{ + /// Built once per ELF, outside this proof. + pub commitment: &'a StackedCommitment, + /// The columns it was committed over, in that order. + pub columns: &'a [&'a Mle], + /// Where each of those columns is settled, in the same order. + pub at: &'a [PreparedColumn], +} + +/// Every table's argument, and the **one** opening that settles all of them. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct MultiProof { + /// One per stacked polynomial. The verifier absorbs these before any + /// challenge is drawn, so a prover cannot choose a commitment after seeing + /// one — and carrying them here is what makes the proof self-contained. + pub roots: Vec, + pub tables: Vec>, + /// Every table's columns, at each one's own point — one opening per + /// commitment group, in group order. + pub columns: Vec>, + /// ★★ The out-of-band preprocessed opening, when the proof carries one. + /// + /// DECODE's five preprocessed columns are ELF-derived, so their commitment + /// is a function of the program alone and is built ONCE per ELF rather than + /// per epoch. This opens it at DECODE's own reduced point, and the verifier + /// compares the opened values against the ones DECODE's table settled on — + /// which replaces re-evaluating five 2^20 MLEs on every epoch. + /// + /// ⚠ Its ROOT IS NOT IN [`roots`](Self::roots), and that is deliberate. The + /// verifier derives the root from the ELF it already holds and absorbs the + /// DERIVED value; a root carried here would be a second copy that a reader + /// assumes is checked, and a value that has not been checked must never + /// reach the transcript. `roots` therefore keeps exactly the group roots it + /// always had, and the group indexing that slices it is untouched. + pub preprocessed: Option>, +} + +/// The table's share of the bus, `p/q`. +/// +/// `None` when the denominator vanished, which a random challenge makes +/// negligible. +pub fn contribution( + output: &(FieldElement, FieldElement), +) -> Option> { + output.1.inv().ok().map(|inv| &output.0 * inv) +} + +/// The factor slots: the trace's factors, then `eq(r, ·)` for the zerocheck and +/// `eq(row, ·)` for the bus's two claims. +/// +/// Public because a verifier written OUTSIDE this crate — the WHIR recursion's +/// in-guest one — has to index the same two slots, and a second spelling of a +/// convention both sides must agree on is exactly the kind of drift nothing +/// would catch. +pub fn weight_slots(num_trace_factors: usize) -> (usize, usize) { + (num_trace_factors, num_trace_factors + 1) +} + +/// Proves the table: its constraints vanish and its bus sums to what the proof +/// says, in one sumcheck. +/// +/// Stops at the claims about the table's columns and hands back the point they +/// are claimed at; the caller settles every table's claims against the shared +/// stack in one opening. +/// +/// `z` and `alpha` are the LogUp challenges, **shared across every table** in a +/// multi-table proof. The caller must have absorbed the commitment roots and +/// drawn them, identically on both sides. +pub fn prove( + table: &CommittedTable<'_, F, E>, + z: &FieldElement, + alpha: &FieldElement, + beta: &FieldElement, + transcript: &mut T, + prebuilt: Option>, +) -> Result<(TableProof, Vec>), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + use multilinear::whir_split::{self as split, lap, tick}; + + let t = tick(); + let interactions = multilinear_logup::interactions( + table.layout.interactions, + table.slot_of().len(), + z, + alpha, + |col| slot(table.slot_of(), col), + )?; + lap(&split::REST_INTERACTIONS, t); + + // The input layer reads the trace's factors. On a device they stay there + // for the sumcheck too — they are the biggest thing the argument holds — + // and the layer is written where they are; on the host they are + // materialized here, used and dropped. + // + // `prebuilt` is a tree the pipeline built during the previous table's argue + // (its output read here, at the consume site). Prefetch never changes WHAT + // is built — same factors, same (z, alpha, beta) — only when, so a prebuilt + // tree yields byte-for-byte the same proof as building it here now. + // The tree where the factors reside, the lift timed on its own. + // Under `LAMBDA_VM_ARGUE_GKR_INPUT` (D-BATCH M1-2) the input layer is + // written from the resident base columns; the factors are still lifted, for + // the zerocheck. + // With no lift (`LAMBDA_VM_ARGUE_NO_LIFT`, M1-2's second half) a tree written + // from the columns needs no factors at all: they are lifted later only if + // the zerocheck needs them. + let no_lift = multilinear::gpu::argue_no_lift(); + let resident_tree = || { + if no_lift + && table.trace.device_factors().is_none() + && let Some(tree) = logup::resident_tree_from_columns(&interactions, &table.trace) + { + return Some(tree); + } + let lift = tick(); + let resident = table.trace.reside_from_columns(); + if let Some(factors) = resident.as_ref() { + factors.tree_sync(); + } + lap(&split::TREE_LIFT, lift); + let from_columns = (multilinear::gpu::argue_gkr_input() && resident.is_some()) + .then(|| logup::resident_tree_from_columns(&interactions, &table.trace)) + .flatten(); + from_columns + .or_else(|| resident.and_then(|resident| logup::resident_tree(&interactions, resident))) + }; + let t = tick(); + let tree = match prebuilt { + Some(tree) => tree, + None => match resident_tree() { + Some(tree) => tree, + // No device took them, so the host builds what it needs: the + // factors, used here and by the sumcheck that follows. + None => { + let factors = table.trace.factors()?; + FractionTree::build(logup::input_layer(&interactions, &factors)?)? + } + }, + }; + let tree_secs = lap(&split::REST_TREE, t); + let t = tick(); + let bus_output = tree.output(); + transcript.append_field_element(&bus_output.0); + transcript.append_field_element(&bus_output.1); + lap(&split::REST_OUTPUT, t); + let t = tick(); + let gkr_out = gkr::prove(&tree, transcript)?; + let gkr_secs = lap(&split::REST_GKR, t); + + let t = tick(); + let num_vars = table.num_vars(); + let r: Vec> = (0..num_vars) + .map(|_| transcript.sample_field_element()) + .collect(); + argue_capture::record(table, &interactions, &gkr_out.claim, &r, beta); + + let (weight_r, weight_z) = weight_slots(table.kinds().len()); + let bus = logup::claim_statements(&interactions, &gkr_out.claim.point, num_vars, weight_z)?; + + let shape = table.shape(); + let betas = multilinear_air::beta_powers(beta, shape.num_roots()); + let zerocheck = Rule::compiled(shape.degree() + 1, shape.program(&betas, weight_r)?); + + // Under `LAMBDA_VM_ARGUE_DEVICE_TABLES` the weights go as their points and + // the card builds them where the zerocheck folds them; the host builds them + // only if the card turns the rounds down. + let weights = if multilinear::gpu::argue_device_tables() { + vec![Weight::Eq(r.clone()), Weight::Eq(bus.row_point.clone())] + } else { + vec![ + Weight::Table(eq_mle(&r)?), + Weight::Table(eq_mle(&bus.row_point)?), + ] + }; + // D-ARGUE stage 1 (`LAMBDA_VM_ARGUE_FUSED`, on by default; `=0` is today's + // rounds): the same batch, described for the fused rounds — its constraint + // part in the base field. A table whose part is not (a constant with an + // extension limb) runs today's rounds. + let constraints = (multilinear::gpu_fused::argue_fused() + && multilinear::gpu_fused::fused_takes_width(table.num_committed_columns())) + .then(|| { + multilinear::fused::Constraints::::from_extension( + &shape.steps_as_ops(), + shape.root_steps(), + shape.selector_of_root(), + shape.degree(), + ) + .ok() + }) + .flatten(); + // With no lift, the fused rounds read the base columns; a table they cannot + // read that way (a shifted factor, a public table above the base field, no + // fused description) is lifted here, as it would have been. + let column_factors = if no_lift && table.trace.device_factors().is_none() { + let made = constraints + .as_ref() + .and_then(|_| table.trace.column_factors()); + if made.is_none() { + let _ = table.trace.reside_from_columns(); + } + made + } else { + None + }; + let fused = constraints + .as_ref() + .map(|constraints| multilinear::gpu_fused::FusedInput { + constraints, + betas: &betas, + interactions: &interactions, + claim_point: &gkr_out.claim.point, + r: &r, + columns: column_factors.as_ref(), + }); + lap(&split::REST_SETUP, t); + let t = tick(); + let (constraint, point) = constraint_argument::prove_core_with::( + &table.trace, + weights, + vec![zerocheck, bus.numerator, bus.denominator], + &[ + FieldElement::zero(), + gkr_out.claim.p.clone(), + gkr_out.claim.q.clone(), + ], + fused, + transcript, + )?; + // The sumcheck folded the factors where they lay, so they are spent — and + // the table outlives its own argument. Letting go of them here is what + // keeps a proof from holding every table's at once. + let core_secs = t.map_or(0.0, |t| t.elapsed().as_secs_f64()); + let t = tick(); + table.trace.release_device(); + lap(&split::REST_RELEASE, t); + split::bump(&split::REST_TABLES); + if split::enabled() { + split::note_air(split::AirCensus { + // Its position in the prove's walk: `note_air` numbers it. + index: 0, + num_vars, + columns: table.num_committed_columns(), + factors: table.kinds().len(), + shifted: table + .kinds() + .iter() + .filter(|kind| kind.source().is_some_and(|source| source.offset != 0)) + .count(), + interactions: interactions.len(), + input_vars: logup::input_layer_vars(interactions.len(), num_vars), + degree: shape.degree(), + roots: shape.num_roots(), + bus_len_max: table + .layout + .interactions + .iter() + .map(BusInteraction::num_bus_elements) + .max() + .unwrap_or(0), + tree: tree_secs, + gkr: gkr_secs, + core: core_secs, + }); + } + + Ok(( + TableProof { + gkr: gkr_out.proof, + bus_output, + constraint, + }, + point, + )) +} +/// ⛔ TEST-ONLY: a table's argue inputs, recorded by [`prove`] while a test has +/// armed the table's width — so the stage-1 reference +/// (`multilinear::fused`, `thoughts/zf/gap2/fix2/D-ARGUE.md` §4.1(b)) can be run +/// on a real trace that satisfies its AIR, which random columns never do. +/// +/// Off unless armed, and then one atomic load a table. It records; it never +/// changes what `prove` does. +#[doc(hidden)] +pub mod argue_capture { + use super::*; + use std::any::Any; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + static ARMED: AtomicBool = AtomicBool::new(false); + static WIDTHS: Mutex> = Mutex::new(Vec::new()); + static CAPTURED: Mutex>> = Mutex::new(Vec::new()); + + /// One table's argue as [`prove`] held it once the zerocheck point was + /// drawn: its committed columns and factor kinds, its constraint shape and + /// bus, the GKR input-layer claim, the zerocheck point and `beta`. + pub struct Captured { + pub columns: Vec>, + pub kinds: Vec, + pub shape: IrShape, + pub interactions: Vec>, + pub claim: gkr::GkrClaim, + pub r: Vec>, + pub beta: FieldElement, + } + + /// Records every table with this many committed columns from now on; an + /// empty list disarms. + pub fn arm(widths: &[usize]) { + if let Ok(mut armed) = WIDTHS.lock() { + *armed = widths.to_vec(); + } + ARMED.store(!widths.is_empty(), Ordering::Relaxed); + } + + /// What has been recorded, in the order `prove` ran; the record empties. + pub fn take() -> Vec> { + CAPTURED + .lock() + .map(|mut captured| std::mem::take(&mut *captured)) + .unwrap_or_default() + } + + pub(super) fn record( + table: &CommittedTable<'_, F, E>, + interactions: &[logup::Interaction], + claim: &gkr::GkrClaim, + r: &[FieldElement], + beta: &FieldElement, + ) where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + if !ARMED.load(Ordering::Relaxed) { + return; + } + let width = table.num_committed_columns(); + if !WIDTHS.lock().is_ok_and(|armed| armed.contains(&width)) { + return; + } + let captured = Captured { + columns: table.columns().to_vec(), + kinds: table.kinds().to_vec(), + shape: table.shape().clone(), + interactions: interactions.to_vec(), + claim: gkr::GkrClaim { + point: claim.point.clone(), + p: claim.p.clone(), + q: claim.q.clone(), + }, + r: r.to_vec(), + beta: beta.clone(), + }; + if let Ok(mut record) = CAPTURED.lock() { + record.push(Box::new(captured)); + } + } +} + +/// What verifying one table leaves for the caller: its share of the bus, and +/// the claims its columns are left at. +pub type TableVerdict = ( + (FieldElement, FieldElement), + claim_reduce::ReducedClaim, +); + +/// Verifies the table and returns its bus output together with the claims its +/// columns are left at, for the caller to sum and to settle against the shared +/// stack. +pub fn verify( + proof: &TableProof, + statement: TableStatement< + '_, + impl IsFFTField + IsPrimeField + IsSubFieldOf + Sync + 'static, + E, + >, + z: &FieldElement, + alpha: &FieldElement, + beta: &FieldElement, + transcript: &mut T, + settled_out_of_band: usize, +) -> Result, MlError> +where + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + let interactions = multilinear_logup::interactions( + statement.interactions, + statement.slot_of.len(), + z, + alpha, + |col| slot(statement.slot_of, col), + )?; + + transcript.append_field_element(&proof.bus_output.0); + transcript.append_field_element(&proof.bus_output.1); + let gkr_claim = gkr::verify(&proof.gkr, proof.bus_output.clone(), transcript)?; + + let num_vars = statement.num_vars; + let r: Vec> = (0..num_vars) + .map(|_| transcript.sample_field_element()) + .collect(); + + let (weight_r, weight_z) = weight_slots(statement.kinds.len()); + let bus = logup::claim_statements(&interactions, &gkr_claim.point, num_vars, weight_z)?; + let row_point = bus.row_point.clone(); + + let shape = statement.shape; + let betas = multilinear_air::beta_powers(beta, shape.num_roots()); + let zerocheck = Rule::compiled(shape.degree() + 1, shape.program(&betas, weight_r)?); + + let reduced = constraint_argument::verify_core( + &proof.constraint, + statement.kinds, + statement.slot_of.len(), + num_vars, + &[zerocheck, bus.numerator, bus.denominator], + &[ + FieldElement::zero(), + gkr_claim.p.clone(), + gkr_claim.q.clone(), + ], + |at: &[FieldElement]| { + // The selectors, then the two weight tables. + let mut values = shape.public_values(at)?; + values.push(eq_eval(&r, at)?); + values.push(eq_eval(&row_point, at)?); + Ok(values) + }, + transcript, + )?; + + check_preprocessed(statement, &reduced, settled_out_of_band)?; + + Ok((proof.bus_output.clone(), reduced)) +} + +/// Checks the table's preprocessed columns against what the proof claims for +/// them at the reduced point. +/// +/// The commitment binds the prover to the columns it committed, not to the +/// *right* ones — nothing else in the argument says a preprocessed table is the +/// one the program implies. So the verifier evaluates its own copy at the point +/// the proof settled on and demands the same value. Costs one pass over each +/// such column, which is what recomputing a preprocessed commitment costs on +/// the univariate side. +/// ★★ `settled_out_of_band` is how many of this table's LEADING preprocessed +/// columns a PREPARED OPENING already settled, and it is the whole saving: those +/// columns are tied to an ELF-derived commitment by an opening at this very +/// point, so evaluating their MLEs here would prove the same thing a second time +/// at `5 * 2^20` folds an epoch. +/// +/// ⚠ A LEADING COUNT AND NOT A SET, deliberately, and a prepared commitment is +/// shaped to fit it rather than the other way round. A cross-epoch genesis page +/// presents `[OFFSET, INIT]` and only INIT is worth an opening — but `{1}` is +/// not a prefix, so the stack carries BOTH of that page's columns and settles +/// `2`. The page loses its OFFSET ramp, which on the block is 51 rows against +/// the 282 the three extra stack columns cost: about 230 rows on a hybrid +/// costing order `10^5`, in exchange for this contract not moving. See +/// [`prefix_at`], which is where a commitment that cannot be expressed as a +/// prefix is refused rather than silently reinterpreted. +/// +/// ⚠ It must be driven by the same `PreparedCheck` value that drives the +/// opening, never by a flag a caller sets on its own — otherwise it is a switch +/// that turns off a check with nothing put in its place. The caller derives both +/// from one `at` slice; see `multi_verify`. +pub(crate) fn check_preprocessed( + statement: TableStatement<'_, F, E>, + reduced: &claim_reduce::ReducedClaim, + settled_out_of_band: usize, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField + 'static, +{ + let count = statement.num_preprocessed; + if settled_out_of_band > count { + return Err(MlError::QueryCountMismatch { + expected: count, + got: settled_out_of_band, + }); + } + // ⛔ THE COUNT TRAP, CLOSED. Columns `settled_out_of_band..count` are + // preprocessed and the opening did not settle them, so this function must + // recompute every one; a verifier holding fewer copies than that would skip + // them in the loop below and leave the program's own columns bound by + // NOTHING. That is a refusal, never a pass. + if settled_out_of_band < count && statement.preprocessed.len() < count { + return Err(MlError::QueryCountMismatch { + expected: count, + got: settled_out_of_band.max(statement.preprocessed.len()), + }); + } + for (col, column) in statement + .preprocessed + .iter() + .enumerate() + .skip(settled_out_of_band) + { + let factor = slot(statement.slot_of, col)?; + // A preprocessed column is read unshifted by construction: `TableLayout` + // registers every main column that way. Anything else means the two + // sides disagree about the layout, which is not a claim to compare. + let source = statement + .kinds + .get(factor) + .and_then(FactorKind::source) + .filter(|s| s.offset == 0) + .ok_or(MlError::UnknownPolynomial { + index: factor, + len: statement.kinds.len(), + })?; + let claimed = + reduced + .column_values + .get(source.column) + .ok_or(MlError::UnknownPolynomial { + index: source.column, + len: reduced.column_values.len(), + })?; + if column.evaluate_in(&reduced.point)? != *claimed { + return Err(MlError::EvaluationMismatch); + } + } + Ok(()) +} + +/// `at` as validated `(table, count)` runs, in stack order. +/// +/// ★ ONE WALK DOES BOTH JOBS: it is where the prefix and contiguity contracts +/// of [`prefix_at`] are enforced, and it is where the stack's COLUMN ORDER is +/// pinned. `at` must visit each table once, in the order the stack's columns +/// were committed, so run `k` of the commitment settles table `k`'s claims. A +/// reordering would settle one table's columns against another's commitment +/// while every individual value gate stayed green — the failure the AIR set's +/// single-derivation rule exists to prevent, one level down. +fn prepared_runs(at: &[PreparedColumn]) -> Result, MlError> { + let mut runs = Vec::new(); + let mut i = 0usize; + while i < at.len() { + let table = at[i].table; + let (first, n) = prefix_at(at, table)?.ok_or(MlError::UnknownPolynomial { + index: table, + len: at.len(), + })?; + // The walk is in `at` order, so a table's run must begin where we are. + // A table seen twice reaches here with `first` behind `i`. + if first != i { + return Err(MlError::QueryCountMismatch { + expected: i, + got: first, + }); + } + runs.push((table, n)); + i += n; + } + Ok(runs) +} + +/// What [`prepared_claims`] hands back: the point each settled column is claimed +/// at, and the value it is claimed to take there, both in stack order. +/// +/// ⚠ NAMED ONLY TO KEEP THAT SIGNATURE READABLE, and it carries no bound — a +/// bound on a type alias is not enforced, and `FieldElement`'s own is checked +/// at every use. That is the form `sumcheck::RoundGroup` and +/// `batch::ResidentProof` already take in this workspace. +type PreparedClaims = (Vec>>, Vec>); + +/// The points and claimed values a prepared opening is settled against, gathered +/// in stack order. +/// +/// ★ A GATHER AND NOT ONE SLICE, which is the difference a multi-table prepared +/// commitment makes. A single table's prepared prefix is one contiguous run of +/// the global column order — `&points[at..at + width]`, as it always was. A +/// stack spanning three genesis pages is THREE such runs at three different +/// tables' offsets, and no single slice names them. +/// +/// ⚠ Each run is still CONTIGUOUS and still that table's own leading columns, +/// which is what keeps [`check_preprocessed`]'s prefix contract intact: the +/// columns this settles are exactly the columns that check skips. +/// +/// ⚠ The values are the ones THOSE TABLES' OWN arguments settled on. That is +/// what makes the opening a check on the pinned columns rather than on a second +/// copy of them, and it is why no separate equality is asserted anywhere. +fn prepared_claims( + runs: &[(usize, usize)], + points: &[Vec>], + values: &[FieldElement], +) -> Result, MlError> { + let mut at_points = Vec::new(); + let mut at_values = Vec::new(); + for &(start, n) in runs { + let end = start + n; + at_points.extend( + points + .get(start..end) + .ok_or(MlError::QueryCountMismatch { + expected: end, + got: points.len(), + })? + .iter() + .cloned(), + ); + at_values.extend( + values + .get(start..end) + .ok_or(MlError::QueryCountMismatch { + expected: end, + got: values.len(), + })? + .iter() + .cloned(), + ); + } + Ok((at_points, at_values)) +} + +/// Proves every table in one transcript, against one commitment. +/// +/// The LogUp challenges are drawn **once**, after the commitment roots are +/// absorbed: sharing them is what lets one table's send be another's receive, +/// and absorbing the roots first is what stops a prover from choosing a bus +/// after seeing them. +/// +/// Each table's sumcheck leaves its columns claimed at a point of its own. +/// Those go into **one** opening at the end, which is what makes a proof of +/// many tables cost about what a proof of one does. +pub fn multi_prove( + committed: &CommittedTables<'_, F, E, H>, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + // ★★ The transcript's hash must BE the configuration's. Not "should": a + // caller that passes a keccak transcript under an RPX `H` does not compile. + // + // `DefaultTranscript`'s hash parameter has a default, so `DefaultTranscript::` + // is a keccak transcript that looks like it names no hash. Every call site + // wrote exactly that, and the RPX configuration therefore ran an RPX Merkle + // backend, an RPX grind and a KECCAK sponge — through four measured A/Bs, + // with no instrument disagreeing, because nothing failed: the proofs were + // valid and the arms did differ from each other. + // + // Requiring each site to NAME a hash would not have caught it; a site can + // name the wrong one. An equality the compiler checks is what makes the + // half-configured arm unspellable, and it costs the several hundred STARK + // call sites nothing, because they are not generic over `H`. + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + // ⚠ The prepared root is NOT added to `MultiProof::roots`. The verifier + // derives it from the ELF and absorbs the derived value; a copy carried in + // the proof would be a field a reader assumes is checked. + // `StackedCommitment::roots` builds a fresh `Vec`, so it is bound here + // rather than borrowed from a temporary. + // ── the argument's split, under `LAMBDA_VM_BASE_SPLIT=1` ────────────── + // The guard counts concurrent proves so a record that could have mixed two + // says so; it releases on DROP, which is what the `?`s below need. + let _split = multilinear::whir_split::begin_prove(); + + // The per-table argue is `ArgueFormat::PerTable`'s; a batched config's + // proofs come from `batched::multi_prove_batched`. + if config.format.argue != ArgueFormat::PerTable { + return Err(MlError::ArgueFormatMismatch); + } + check_settled_prefixes(committed, prepared.as_ref())?; + + let prepared_roots: Vec = prepared + .as_ref() + .map(|p| p.commitment.roots()) + .unwrap_or_default(); + let __sp_challenge = multilinear::whir_split::mark(); + let (z, alpha, beta) = + absorb_roots_and_challenge::(transcript, committed.roots(), &prepared_roots); + multilinear::whir_split::add(&multilinear::whir_split::CHALLENGE, __sp_challenge); + + let mut tables = Vec::with_capacity(committed.tables().len()); + // One point and one claimed value per **column**, in the global column + // order the stack was built in. + let mut points: Vec>> = Vec::new(); + let mut values: Vec> = Vec::new(); + // Where each table's columns start in the global column order. A prepared + // commitment may span several tables, so every start is kept rather than + // the one a single-table opening needed. + let mut table_starts = Vec::with_capacity(committed.tables().len()); + // The device ledger's peak over the argument alone — `RESERVED HW`'s + // `argue`. It opens on what the commits left promised (the codewords, the + // layers they kept, and whatever room a group still holds) and rises by + // what the tables reserve. + let __rw_argue = multilinear::whir_split::open_reserved(); + for (__sp_at, table) in committed.tables().iter().enumerate() { + table_starts.push(points.len()); + // ⛔ SERIAL, and the instrument says so rather than a reader inferring + // it: this loop has no rayon and no `k`, so `ARGUE` is a WALL as well + // as a sum. The per-table maximum is kept beside it because the sum + // alone cannot tell fifty even tables from one that dominates, and + // those two want opposite levers. + let __sp_table = multilinear::whir_split::mark(); + let (proof, point) = prove(table, &z, &alpha, &beta, transcript, None)?; + let __sp_secs = multilinear::whir_split::add(&multilinear::whir_split::ARGUE, __sp_table); + multilinear::whir_split::note_table(__sp_at, __sp_secs); + for _ in 0..table.num_committed_columns() { + points.push(point.clone()); + } + values.extend(proof.constraint.reduce.column_values.iter().cloned()); + tables.push(proof); + } + multilinear::whir_split::close_reserved(&multilinear::whir_split::RESERVED_ARGUE, __rw_argue); + + let (columns, preprocessed) = prove_openings( + committed, + config, + transcript, + prepared, + &points, + &values, + &table_starts, + )?; + + Ok(MultiProof { + roots: committed.roots().to_vec(), + tables, + columns, + preprocessed, + }) +} + +/// Verifies every table **and the bus balance across them**, then settles every +/// column against the one commitment. +/// +/// A table's own contribution need not vanish, and neither does the sum: a bus +/// whose counterparty is the statement rather than another table leaves a +/// residue, so the caller says what it owes. For this VM that is the COMMIT +/// bus carrying the program's public output — `expected` is zero exactly when +/// the program outputs nothing. +#[allow(clippy::too_many_arguments)] +pub fn multi_verify( + proof: &MultiProof, + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + expected: &FieldElement, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + multi_verify_settled::( + proof, statements, layouts, domains, sizes, expected, config, transcript, prepared, false, + ) +} + +/// [`multi_verify`], with `exclude_settled` saying whether the group stacks +/// LEAVE OUT each table's prefix the prepared check settles (policy B, the +/// proofs of [`CommittedTables::commit_grouped_settled`]). +/// +/// The exclusion is the verifier's own FORMAT decision, never a proof's: the +/// caller builds `layouts` from the value-only shapes under it. It turns no +/// check off — a prefix left out of the stacks is settled by the prepared +/// opening, which is then its only binding, exactly as it is DECODE's — and a +/// proof made under the other policy fails the group openings, whose layouts +/// then describe different stacks. +#[allow(clippy::too_many_arguments)] +pub fn multi_verify_settled( + proof: &MultiProof, + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + expected: &FieldElement, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, + exclude_settled: bool, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + // ★★ The transcript's hash must BE the configuration's. Not "should": a + // caller that passes a keccak transcript under an RPX `H` does not compile. + // + // `DefaultTranscript`'s hash parameter has a default, so `DefaultTranscript::` + // is a keccak transcript that looks like it names no hash. Every call site + // wrote exactly that, and the RPX configuration therefore ran an RPX Merkle + // backend, an RPX grind and a KECCAK sponge — through four measured A/Bs, + // with no instrument disagreeing, because nothing failed: the proofs were + // valid and the arms did differ from each other. + // + // Requiring each site to NAME a hash would not have caught it; a site can + // name the wrong one. An equality the compiler checks is what makes the + // half-configured arm unspellable, and it costs the several hundred STARK + // call sites nothing, because they are not generic over `H`. + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + // The variant is the verifier's config's, never the proof's: a verifier + // configured for the batched argue takes no per-table proof. + if config.format.argue != ArgueFormat::PerTable { + return Err(MlError::ArgueFormatMismatch); + } + if proof.tables.len() != statements.len() { + return Err(MlError::QueryCountMismatch { + expected: statements.len(), + got: proof.tables.len(), + }); + } + if layouts.len() != sizes.len() + || domains.len() != sizes.len() + || proof.columns.len() != sizes.len() + || sizes.iter().sum::() != statements.len() + { + return Err(MlError::QueryCountMismatch { + expected: sizes.len(), + got: proof.columns.len(), + }); + } + let (z, alpha, beta) = absorb_roots_and_challenge::( + transcript, + &proof.roots, + prepared.as_ref().map(|p| p.roots).unwrap_or(&[]), + ); + + let mut balance = FieldElement::::zero(); + let mut points: Vec>> = Vec::new(); + let mut values: Vec> = Vec::new(); + // Where each table's columns start in the global column order. A prepared + // commitment may span several tables, so every start is kept. + let mut table_starts = Vec::with_capacity(statements.len()); + // How many of each table's leading columns the prepared opening settles. + let mut settled_counts = Vec::with_capacity(statements.len()); + for (index, (table, statement)) in proof.tables.iter().zip(statements).enumerate() { + table_starts.push(points.len()); + let settled = settled_count(prepared.as_ref(), index, statement)?; + settled_counts.push(settled); + let (output, reduced) = verify(table, *statement, &z, &alpha, &beta, transcript, settled)?; + balance += contribution(&output).ok_or(MlError::BusImbalance)?; + for _ in 0..statement.slot_of.len() { + points.push(reduced.point.clone()); + } + values.extend(reduced.column_values); + } + if balance != *expected { + return Err(MlError::BusImbalance); + } + + verify_openings::( + &proof.roots, + &proof.columns, + proof.preprocessed.as_ref(), + statements, + layouts, + domains, + sizes, + config, + transcript, + prepared, + exclude_settled, + &points, + &values, + &table_starts, + &settled_counts, + ) +} + +/// ⛔ A PREFIX LEFT OUT OF THE STACKS MUST BE SETTLED BY THE OPENING — the +/// prover's check, shared by both argue formats. +fn check_settled_prefixes( + committed: &CommittedTables<'_, F, E, H>, + prepared: Option<&Prepared<'_, F, H>>, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + // ⛔ A PREFIX LEFT OUT OF THE STACKS MUST BE SETTLED BY THE OPENING, table + // for table and count for count: a column in neither stack is bound by + // nothing, and a prover must never hand out the proof that says so. + if !committed.settled.is_empty() { + let at = prepared.map(|p| p.at).ok_or(MlError::QueryCountMismatch { + expected: 1, + got: 0, + })?; + for (table, &count) in committed.settled.iter().enumerate() { + let opened = prefix_at(at, table)?.map(|(_, n)| n).unwrap_or(0); + if opened != count { + return Err(MlError::QueryCountMismatch { + expected: count, + got: opened, + }); + } + } + } + + Ok(()) +} + +/// What [`prove_openings`] hands back: every group's opening, then the +/// prepared one. +type Openings = (Vec>, Option>); + +/// Every group's opening and the prepared one, at the claims an argue left: +/// one point and one value per column in the global column order, and where +/// each table's columns start. Shared by both argue formats, so the openings +/// are one spelling whatever argued the claims. +fn prove_openings( + committed: &CommittedTables<'_, F, E, H>, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, + points: &[Vec>], + values: &[FieldElement], + table_starts: &[usize], +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + // One opening per group, over that group's columns. The points and values + // are in the global column order, so a group takes the slice its tables + // span. + let mut columns = Vec::with_capacity(committed.groups().len()); + multilinear::whir_split::note_groups(committed.groups().len()); + // The ledger's peak over every opening — the groups' and the prepared + // one — as `RESERVED HW`'s `open`. + let __rw_open = multilinear::whir_split::open_reserved(); + // ⛔ CLEARED AT THE OPENING OF THE WINDOW, not merely read at its close. + // The six are process-global accumulators, so whatever ran the chain + // earlier in this process is still sitting in them; reading at the end + // alone attributes that to this group loop. The first fixture run showed + // it as a NEGATIVE remainder — the six summed to 16.40s inside an + // `open_groups` of 12.77s, which is the one arithmetic a sum of parts + // cannot produce honestly. Clearing here makes "group openings only" a + // property of the window rather than an assumption about callers. + let _ = multilinear::whir_split::take_chain(); + let __sp_groups = multilinear::whir_split::mark(); + let mut table_at = 0usize; + let mut column_at = 0usize; + for (group, &size) in committed.groups().iter().zip(committed.sizes()) { + let width: usize = committed.tables()[table_at..table_at + size] + .iter() + .map(|t| t.num_committed_columns()) + .sum(); + if committed.settled.is_empty() { + // The same columns, in the same order, the group was committed over. + let group_columns: Vec<&Mle> = committed.tables()[table_at..table_at + size] + .iter() + .flat_map(|t| t.trace.columns()) + .collect(); + columns.push(stacked_eval::prove::( + group, + &group_columns, + committed.store.as_ref().map(|store| (&**store, column_at)), + &Claimed::PerColumn(&points[column_at..column_at + width]), + &values[column_at..column_at + width], + config, + transcript, + )?); + } else { + // Policy B: the group's stack holds each table's columns past its + // settled prefix, and opens exactly those claims; the prefix's are + // the prepared opening's. + let tables = &committed.tables()[table_at..table_at + size]; + let skip = &committed.settled[table_at..table_at + size]; + let starts = &table_starts[table_at..table_at + size]; + let widths: Vec = tables.iter().map(|t| t.num_committed_columns()).collect(); + let group_columns: Vec<&Mle> = tables + .iter() + .zip(skip) + .flat_map(|(t, &s)| &t.trace.columns()[s..]) + .collect(); + let (at_points, at_values) = unsettled_claims(&widths, starts, skip, points, values)?; + let map = settled_column_map(&widths, starts, skip); + columns.push(stacked_eval::prove_mapped::( + group, + &group_columns, + committed + .store + .as_ref() + .map(|store| (&**store, stacked_eval::ColumnsAt::Map(&map))), + &Claimed::PerColumn(&at_points), + &at_values, + config, + transcript, + )?); + } + table_at += size; + column_at += width; + } + + // The out-of-band opening, each column at the reduced point of the table + // whose preprocessed column it is. `PerColumn` because those points are in + // general different — a cross-epoch genesis page is a table of its own — and + // settling each column at its own table's point is what makes the opened + // values comparable to that table's own claimed values. + // + // ⚠ A single-table prepared commitment is the special case where every one + // of these points is the same one, and it produces the same weight shares + // and therefore the same bytes as the `Shared` form it replaces: + // `Claimed::point` hands back that one point for every column under either + // variant, and `Claimed` reaches nothing but the weight. + multilinear::whir_split::add(&multilinear::whir_split::OPEN_GROUPS, __sp_groups); + // ⛔ THE SIX ARE TAKEN HERE AND NOWHERE LATER. The prepared opening below + // runs the same chain and writes the same slots, so a take placed after it + // would fold DECODE's chain into `open_groups` — and arm E would close, on + // a number that is not what its name says. This is the only point at which + // the slots hold the GROUP openings and nothing else. + multilinear::whir_split::note_chain(multilinear::whir_split::take_chain()); + + let __sp_prepared = multilinear::whir_split::mark(); + let preprocessed = match prepared { + Some(prepared) => { + if prepared.at.len() != prepared.columns.len() { + return Err(MlError::QueryCountMismatch { + expected: prepared.columns.len(), + got: prepared.at.len(), + }); + } + let runs: Vec<(usize, usize)> = prepared_runs(prepared.at)? + .into_iter() + .map(|(table, n)| { + let start = *table_starts.get(table).ok_or(MlError::UnknownPolynomial { + index: table, + len: table_starts.len(), + })?; + Ok((start, n)) + }) + .collect::>()?; + let (at_points, at_values) = prepared_claims(&runs, points, values)?; + Some(stacked_eval::prove::( + prepared.commitment, + prepared.columns, + None, + &Claimed::PerColumn(&at_points), + &at_values, + config, + transcript, + )?) + } + None => None, + }; + multilinear::whir_split::add(&multilinear::whir_split::OPEN_PREPARED, __sp_prepared); + multilinear::whir_split::close_reserved(&multilinear::whir_split::RESERVED_OPEN, __rw_open); + Ok((columns, preprocessed)) +} + +/// How many of table `index`'s leading columns the prepared opening settles. +fn settled_count( + prepared: Option<&PreparedCheck<'_, F>>, + index: usize, + statement: &TableStatement<'_, F, E>, +) -> Result +where + F: IsFFTField + IsPrimeField + 'static, + E: IsField, +{ + // ★ ONE VALUE drives both halves: the count the opening settles is the + // count `check_preprocessed` skips, and both come off the same `at` + // slice. A second, independent knob would be a way to switch off a check + // with nothing in its place. + // + // ⚠ `prefix_at` is what makes the COUNT safe here. `at` names a column + // per stacked column, so it CAN describe a non-prefix that this count + // could not express — and `prefix_at` refuses that rather than letting + // the count silently reinterpret it as `0..n`. + let settled = match prepared { + Some(p) => prefix_at(p.at, index)?.map(|(_, n)| n).unwrap_or(0), + None => 0, + }; + // ⚠ THE ASSERT THAT MATTERS. The opening covers `settled` of this + // table's columns; skipping more than that would drop a preprocessed + // check nothing replaced. + // + // ★ Against the table's preprocessed COUNT, not the length of the + // column copies the verifier holds: a statement built with + // `statement_with_prepared_prefix` holds none, and an honest proof + // whose opening settles its whole prefix must pass here. The other + // half — a prefix nothing settles — is `check_preprocessed`'s refusal. + if settled > statement.num_preprocessed { + return Err(MlError::QueryCountMismatch { + expected: statement.num_preprocessed, + got: settled, + }); + } + Ok(settled) +} + +/// Every group's opening and the prepared one, verified against the claims an +/// argue left. Shared by both argue formats. +#[allow(clippy::too_many_arguments)] +fn verify_openings( + all_roots: &[Commitment], + columns: &[StackedProof], + preprocessed: Option<&StackedProof>, + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, + exclude_settled: bool, + points: &[Vec>], + values: &[FieldElement], + table_starts: &[usize], + settled_counts: &[usize], +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + // Each group settles its own columns against its own roots, in the order + // the prover opened them. + let mut statement_at = 0usize; + let mut column_at = 0usize; + let mut root_at = 0usize; + for (((opening, layout), domain), &size) in columns.iter().zip(layouts).zip(domains).zip(sizes) + { + let width: usize = statements[statement_at..statement_at + size] + .iter() + .map(|s| s.slot_of.len()) + .sum(); + let roots = all_roots.get(root_at..root_at + layout.num_polys()).ok_or( + MlError::QueryCountMismatch { + expected: root_at + layout.num_polys(), + got: all_roots.len(), + }, + )?; + if exclude_settled { + // Policy B: the group's stack holds each table's columns past the + // prefix the prepared opening settles, and these are its claims. + let widths: Vec = statements[statement_at..statement_at + size] + .iter() + .map(|s| s.slot_of.len()) + .collect(); + let (at_points, at_values) = unsettled_claims( + &widths, + &table_starts[statement_at..statement_at + size], + &settled_counts[statement_at..statement_at + size], + points, + values, + )?; + stacked_eval::verify::( + opening, + layout, + roots, + &Claimed::PerColumn(&at_points), + &at_values, + domain, + config, + transcript, + )?; + } else { + stacked_eval::verify::( + opening, + layout, + roots, + &Claimed::PerColumn(&points[column_at..column_at + width]), + &values[column_at..column_at + width], + domain, + config, + transcript, + )?; + } + statement_at += size; + column_at += width; + root_at += layout.num_polys(); + } + + // ★★★ THE PREPARED OPENING, and check (d) with it. + // + // (d) is NOT a separate assertion here, and that is deliberate. The values + // handed to `stacked_eval::verify` are the ones EACH COLUMN'S OWN TABLE + // settled on, gathered at that table's point — so the opening has to prove + // the pinned commitment takes exactly those values at exactly those points. + // Two copies of the same columns are tied by the check that already exists + // rather than by an equality someone has to remember to write. + // + // The alternative shape — verify the opening against its own claimed + // values, then assert those equal the tables' — is one line longer and one + // line forgettable. This one cannot be omitted without deleting the call. + if let Some(prepared) = prepared { + let opening = preprocessed.ok_or(MlError::QueryCountMismatch { + expected: 1, + got: 0, + })?; + let runs: Vec<(usize, usize)> = prepared_runs(prepared.at)? + .into_iter() + .map(|(table, n)| { + let start = *table_starts.get(table).ok_or(MlError::UnknownPolynomial { + index: table, + len: table_starts.len(), + })?; + Ok((start, n)) + }) + .collect::>()?; + let (at_points, at_values) = prepared_claims(&runs, points, values)?; + stacked_eval::verify::( + opening, + prepared.layout, + prepared.roots, + &Claimed::PerColumn(&at_points), + &at_values, + prepared.domain, + config, + transcript, + )?; + } + Ok(()) +} + +fn slot(slots: &[usize], column: usize) -> Result { + slots + .get(column) + .copied() + .ok_or(MlError::UnknownPolynomial { + index: column, + len: slots.len(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, + }; + + use multilinear::whir_chain::GrindBits; + + use crate::constraints::builder::{ConstraintBuilder, ConstraintSet, EmptyConstraints}; + use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, + }; + use crate::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; + use crate::proof::options::ProofOptions; + use crate::traits::AIR; + + type FE = FieldElement; + type ExtE = FieldElement; + type Air = AirWithBuses; + + fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + } + } + + /// The ADD table's own constraint: every row really is an addition. + struct AddConstraints; + + impl ConstraintSet for AddConstraints { + fn max_degree(&self) -> usize { + 1 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, 0); + let addend = b.main(0, 1); + let sum = b.main(0, 2); + b.emit_base(0, a + addend - sum); + } + } + + /// The MUL table's, one degree higher. + struct MulConstraints; + + impl ConstraintSet for MulConstraints { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, 0); + let factor = b.main(0, 1); + let product = b.main(0, 2); + b.emit_base(0, a * factor - product); + } + } + + /// The same tables the multi-table completeness test proves, but with the + /// lookup tables now constraining their own rows too. + fn airs() -> ( + Air, + Air, + Air, + ) { + let options = ProofOptions::default_test_options(); + let cpu = new_cpu_air_with_lookup(&options); + let add = AirWithBuses::new( + 4, + AuxiliaryTraceBuildData { + interactions: new_add_air_with_lookup(&options) + .bus_interactions() + .to_vec(), + }, + &options, + 1, + AddConstraints, + ); + let mul = AirWithBuses::new( + 4, + AuxiliaryTraceBuildData { + interactions: new_mul_air_with_lookup(&options) + .bus_interactions() + .to_vec(), + }, + &options, + 1, + MulConstraints, + ); + (cpu, add, mul) + } + + fn base(values: &[u64]) -> Vec { + values.iter().map(|v| FE::from(*v)).collect() + } + + fn cpu_columns() -> Vec> { + vec![ + base(&[1, 0, 1, 0, 1, 1, 0, 0]), + base(&[0, 1, 0, 1, 0, 0, 1, 1]), + base(&[1, 2, 3, 4, 5, 6, 7, 8]), + base(&[10, 20, 30, 40, 50, 60, 70, 80]), + base(&[11, 40, 33, 160, 55, 66, 490, 640]), + ] + } + + fn add_columns() -> Vec> { + vec![ + base(&[1, 3, 5, 6]), + base(&[10, 30, 50, 60]), + base(&[11, 33, 55, 66]), + base(&[1, 1, 1, 1]), + ] + } + + fn mul_columns() -> Vec> { + vec![ + base(&[2, 4, 7, 8]), + base(&[20, 40, 70, 80]), + base(&[40, 160, 490, 640]), + base(&[1, 1, 1, 1]), + ] + } + + fn table<'a, CS: ConstraintSet>( + air: &'a Air, + columns: &[Vec], + ) -> Result, MlError> { + let num_vars = columns[0].len().trailing_zeros() as usize; + // The trace goes in as it is: base-field. + let lifted = columns.to_vec(); + CommittedTable::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + columns.len(), + num_vars, + Uniforms::default(), + |col| lifted[col as usize].clone(), + ) + } + + /// Proves and verifies all three tables through the multi-table entry + /// points, which check the bus balance themselves. + fn argue( + cpu_cols: Vec>, + add_cols: Vec>, + mul_cols: Vec>, + ) -> Result<(), MlError> { + let (cpu_air, add_air, mul_air) = airs(); + let committed = CommittedTables::<_, _, KeccakWhir>::commit( + vec![ + table(&cpu_air, &cpu_cols)?, + table(&add_air, &add_cols)?, + table(&mul_air, &mul_cols)?, + ], + &config(), + )?; + + let mut prover = DefaultTranscript::::new(b"multilinear-table"); + let proof = multi_prove(&committed, &config(), &mut prover, None)?; + + // Three tables of different heights, and **one** commitment with one + // opening for all of them. + assert_eq!(proof.roots.len(), 1); + assert_eq!(proof.columns.len(), 1); + assert_eq!(proof.columns[0].polys.len(), 1); + for (table, table_proof) in committed.tables().iter().zip(&proof.tables) { + // Three statements — the constraint and the bus's two claims — in + // one pass over the table's rows. + assert_eq!( + table_proof.constraint.sumcheck.rounds.len(), + table.num_vars() + ); + } + + let statements: Vec> = + committed.tables().iter().map(|t| t.statement()).collect(); + + let mut verifier = DefaultTranscript::::new(b"multilinear-table"); + multi_verify::<_, _, _, KeccakWhir>( + &proof, + &statements, + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &ExtE::zero(), + &config(), + &mut verifier, + None, + ) + } + + /// The composition this whole port is aimed at, on the repo's own + /// multi-table example: three tables of different heights, each with its + /// own constraints *and* its buses, every one argued in a single sumcheck + /// against one commitment per trace — and the buses balance, checked by the + /// verifier rather than by the caller. + /// ★★★ THE ROOTS BLOCK BINDS THE DERIVED ROOT TO THE FIRST CHALLENGE. + /// + /// This is the guard for the one placement nothing else catches. Step 2's + /// prover test sees a root that is never absorbed, and its sibling sees one + /// aimed at the wrong table — but a root absorbed AFTER `z` left both green, + /// because `alpha` and `beta` still move and the proof still differs. + /// + /// A round-trip cannot close it either: prover and verifier call the same + /// block, so a consistently late absorb verifies on both sides. The two + /// halves agree with each other and disagree with the specification. + /// + /// So this compares against an INDEPENDENTLY BUILT transcript rather than + /// against the other side — the shape every check on this branch that held + /// has in common. The reference absorbs carried-then-derived and draws three + /// challenges; the production block must produce the same three. + /// + /// ⚠ The first challenge is the one that matters. Moving the absorb after + /// `z` leaves `alpha` and `beta` correct, so a test comparing only the later + /// two would pass on exactly the mutation this exists to catch. + #[test] + fn the_roots_block_binds_the_derived_root_to_the_first_challenge() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let carried: Vec = vec![[0x11; 32], [0x22; 32]]; + let derived: Vec = vec![[0xAB; 32]]; + + // The specification, built by hand: carried roots, then the derived one, + // then the three challenges. + let mut reference = DefaultTranscript::::new(b"roots-block"); + for root in carried.iter().chain(derived.iter()) { + reference.append_bytes(root); + } + let want = ( + reference.sample_field_element(), + reference.sample_field_element(), + reference.sample_field_element(), + ); + + let mut got_transcript = DefaultTranscript::::new(b"roots-block"); + let got = absorb_roots_and_challenge::(&mut got_transcript, &carried, &derived); + + assert_eq!( + got.0, want.0, + "the FIRST challenge does not match a transcript that absorbed the \ + derived root before drawing it — the root is absorbed late, or not \ + at all, and binds nothing" + ); + assert_eq!(got.1, want.1, "alpha diverged"); + assert_eq!(got.2, want.2, "beta diverged"); + + // …and the reference really is sensitive to the placement, or the + // assertions above hold for a reason that is not the ordering. + let mut late = DefaultTranscript::::new(b"roots-block"); + for root in &carried { + late.append_bytes(root); + } + let late_z: ExtE = late.sample_field_element(); + assert_ne!( + late_z, want.0, + "drawing before the derived root gives the same first challenge as \ + drawing after it — this test cannot see the placement" + ); + } + + #[test] + fn three_tables_argue_and_their_buses_balance() { + argue(cpu_columns(), add_columns(), mul_columns()).unwrap(); + } + + /// The same three tables at a height where the claim reduce's + /// columns-on-the-card knob (`LAMBDA_VM_ARGUE_DEVICE_COLUMNS`) has work to + /// move: CPU is 2^14 rows of 5 columns and ADD 2^14 of 4 — exactly 2^16 + /// cells, the threshold — so a card values both once they are resident, + /// while MUL, 2^13 of 4, stays on the host either way. Every column is + /// under 2^16 rows, so with the knob off the host walks all thirteen. + /// + /// Half the CPU's rows add and half multiply, each met by one receive; ADD + /// is padded with rows that add up (0 + 0 = 0) and receive nothing. + fn tall_columns() -> [Vec>; 3] { + let (adds, muls) = (1u64 << 13, 1u64 << 13); + let mut cpu = vec![Vec::new(); 5]; + let mut add = vec![Vec::new(); 4]; + let mut mul = vec![Vec::new(); 4]; + let push = |table: &mut Vec>, row: &[FE]| { + assert_eq!(table.len(), row.len(), "a row of the table's width"); + for (column, value) in table.iter_mut().zip(row) { + column.push(*value); + } + }; + for i in 0..adds { + let (a, b) = (FE::from(i + 1), FE::from(3 * i + 7)); + let c = a + b; + push(&mut cpu, &[FE::one(), FE::zero(), a, b, c]); + push(&mut add, &[a, b, c, FE::one()]); + } + for _ in 0..adds { + push(&mut add, &[FE::zero(); 4]); + } + for i in 0..muls { + let (a, b) = (FE::from(i + 2), FE::from(5 * i + 3)); + let c = a * b; + push(&mut cpu, &[FE::zero(), FE::one(), a, b, c]); + push(&mut mul, &[a, b, c, FE::one()]); + } + [cpu, add, mul] + } + + /// The tests below switch process-wide overrides — some arm a fault — so + /// they never run at the same time. + static ARGUE_OVERRIDES: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// The three AIRs [`airs`] builds, borrowed as one. + type ThreeAirs = ( + Air, + Air, + Air, + ); + + /// One arm of a knob's A/B: the commitment it proved against, the proof, + /// the transcript after it, how many columns the card and the host valued + /// at the reduction points, and how many challenge tables the card built, + /// while it ran. + struct TallArm<'a> { + committed: CommittedTables<'a, Fp, Ext, KeccakWhir>, + proof: MultiProof, + transcript: DefaultTranscript, + on_card: u64, + on_host: u64, + tables_on_card: u64, + /// Zerocheck sessions whose rounds walked their program on demand, and + /// the ones the cross-check walked today's program beside. + lean_programs: u64, + lean_program_xchecks: u64, + /// Zerocheck sessions that ran D-ARGUE stage 1's fused rounds. + fused: u64, + /// Device GKR layers that ran Gruen's rounds (D-ARGUE S1-3). + gruen: u64, + /// Fraction trees whose input layer was written from the base columns. + input: u64, + /// Tables whose zerocheck read the base columns with no lift. + nolift: u64, + } + + /// The argue knobs an arm runs under: `LAMBDA_VM_ARGUE_DEVICE_COLUMNS`, + /// `LAMBDA_VM_ARGUE_DEVICE_TABLES`, `LAMBDA_VM_ARGUE_LEAN_PROGRAM`. + /// + /// Under `program` every batch counts as big, so these small tables take + /// the path the VM's four big batches do + /// ([`the_tall_batches_hold_fewer_values_on_demand`]). + /// + /// `fused` is D-ARGUE stage 1 (`LAMBDA_VM_ARGUE_FUSED` with + /// `LAMBDA_VM_ARGUE_INT_NODES`, on by default in production): off, the + /// zerocheck takes today's rounds — the path every other knob here acts on. + #[derive(Clone, Copy, Debug, Default)] + struct Knobs { + columns: bool, + tables: bool, + program: bool, + fused: bool, + gruen: bool, + input: bool, + nolift: bool, + } + + const TODAY: Knobs = Knobs { + columns: false, + tables: false, + program: false, + fused: false, + gruen: false, + input: false, + nolift: false, + }; + + /// Commits the tall tables afresh and proves them under `knobs`. + fn prove_tall<'a>( + airs: &'a ThreeAirs, + columns: &[Vec>; 3], + knobs: Knobs, + ) -> TallArm<'a> { + try_prove_tall(airs, columns, knobs).unwrap_or_else(|e| panic!("{knobs:?}: {e:?}")) + } + + /// [`prove_tall`], a refused proof handed back rather than a panic. + fn try_prove_tall<'a>( + airs: &'a ThreeAirs, + columns: &[Vec>; 3], + knobs: Knobs, + ) -> Result, MlError> { + let committed = CommittedTables::<_, _, KeccakWhir>::commit( + vec![ + table(&airs.0, &columns[0]).unwrap(), + table(&airs.1, &columns[1]).unwrap(), + table(&airs.2, &columns[2]).unwrap(), + ], + &config(), + ) + .unwrap(); + multilinear::gpu::force_argue_device_columns(Some(knobs.columns)); + multilinear::gpu::force_argue_device_tables(Some(knobs.tables)); + multilinear::gpu::force_argue_lean_program(Some(knobs.program)); + multilinear::gpu::force_lean_program_gate(knobs.program.then_some(0)); + multilinear::gpu_fused::force_argue_fused(Some(knobs.fused)); + #[cfg(feature = "cuda")] + math_cuda::sumcheck::force_int_nodes(Some(knobs.fused)); + multilinear::gkr_gruen::force_argue_gkr_gruen(Some(knobs.gruen)); + multilinear::gpu::force_argue_gkr_input(Some(knobs.input)); + multilinear::gpu::force_argue_no_lift(Some(knobs.nolift)); + let fused = multilinear::gpu_fused::fused_sessions(); + let gruen = multilinear::gkr_gruen::gruen_layers(); + let input = multilinear::gpu::input_columns_trees(); + let nolift = multilinear::gpu::column_factor_tables(); + let (card, host, built) = ( + multilinear::gpu::evaluate_calls(), + multilinear::gpu::host_evaluate_calls(), + multilinear::gpu::argue_tables_on_card(), + ); + let (demand, demand_checked) = ( + multilinear::gpu::lean_programs(), + multilinear::gpu::lean_program_xchecks(), + ); + let mut transcript = DefaultTranscript::::new(b"multilinear-table"); + let proof = multi_prove(&committed, &config(), &mut transcript, None); + let arm = proof.map(|proof| TallArm { + committed, + proof, + transcript, + on_card: multilinear::gpu::evaluate_calls() - card, + on_host: multilinear::gpu::host_evaluate_calls() - host, + tables_on_card: multilinear::gpu::argue_tables_on_card() - built, + lean_programs: multilinear::gpu::lean_programs() - demand, + lean_program_xchecks: multilinear::gpu::lean_program_xchecks() - demand_checked, + fused: multilinear::gpu_fused::fused_sessions() - fused, + gruen: multilinear::gkr_gruen::gruen_layers() - gruen, + input: multilinear::gpu::input_columns_trees() - input, + nolift: multilinear::gpu::column_factor_tables() - nolift, + }); + multilinear::gkr_gruen::force_argue_gkr_gruen(None); + multilinear::gpu::force_argue_gkr_input(None); + multilinear::gpu::force_argue_no_lift(None); + multilinear::gpu_fused::force_argue_fused(None); + #[cfg(feature = "cuda")] + math_cuda::sumcheck::force_int_nodes(None); + multilinear::gpu::force_argue_device_columns(None); + multilinear::gpu::force_argue_device_tables(None); + multilinear::gpu::force_argue_lean_program(None); + multilinear::gpu::force_lean_program_gate(None); + arm + } + + fn verify_tall( + committed: &CommittedTables<'_, Fp, Ext, KeccakWhir>, + proof: &MultiProof, + ) -> Result<(), MlError> { + let statements: Vec> = + committed.tables().iter().map(|t| t.statement()).collect(); + let mut verifier = DefaultTranscript::::new(b"multilinear-table"); + multi_verify::<_, _, _, KeccakWhir>( + proof, + &statements, + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &ExtE::zero(), + &config(), + &mut verifier, + None, + ) + } + + /// The first table whose argument's canonical bytes differ between two + /// proofs — the name a failed identity gives — or `None`. + fn first_table_that_differs(a: &MultiProof, b: &MultiProof) -> Option { + let bytes = |p: &MultiProof, k: usize| { + p.tables.get(k).map(|t| bincode::serialize(t).unwrap()) + }; + (0..a.tables.len().max(b.tables.len())).find(|&k| bytes(a, k) != bytes(b, k)) + } + + /// Whether a device took this process's columns: `false` in a build or on + /// a machine without one. + fn a_device() -> bool { + multilinear::gpu::reserve_budget() > 0 + } + + /// ★ `LAMBDA_VM_ARGUE_DEVICE_COLUMNS` moves no byte of the proof. The tall + /// tables proved with the knob off and on give the same canonical bytes, + /// table by table and whole, the same transcript after them and the same + /// next challenge, and both verify. + /// + /// On a device the arms are also shown to have taken different paths — + /// the card valued 9 columns with the knob on and none with it off — so the + /// comparison is not of a path with itself. Without one, what is compared + /// is the host walk against the same walk spread over the pool (under + /// `parallel`). ⚠ Run it alone (`--exact`): the counters are process-wide. + /// + /// ```text + /// cargo test --release -p stark --features cuda,parallel,multilinear/cuda,multilinear/parallel \ + /// --lib -- multilinear_table::tests::the_argument_proves_the_same_bytes_with_its_columns_on_the_card --exact + /// ``` + #[test] + fn the_argument_proves_the_same_bytes_with_its_columns_on_the_card() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + let mut off = prove_tall(&airs, &columns, TODAY); + let mut on = prove_tall( + &airs, + &columns, + Knobs { + columns: true, + ..TODAY + }, + ); + + if a_device() { + assert_eq!( + (off.on_card, off.on_host), + (0, 13), + "knob off: the host must value all 13 columns (card, host)" + ); + assert_eq!( + (on.on_card, on.on_host), + (9, 4), + "knob on: the card must value CPU's 5 and ADD's 4 (card, host)" + ); + eprintln!( + "argue device columns: the card valued {} columns", + on.on_card + ); + } else { + assert_eq!( + (off.on_card, on.on_card), + (0, 0), + "no device valued a column" + ); + eprintln!("argue device columns: no device; the host walk compared with itself"); + } + assert_eq!( + first_table_that_differs(&off.proof, &on.proof), + None, + "a table's argument changed with its columns on the card" + ); + assert_eq!( + bincode::serialize(&off.proof).unwrap(), + bincode::serialize(&on.proof).unwrap(), + "the proofs' canonical bytes differ" + ); + assert_eq!( + off.transcript.state(), + on.transcript.state(), + "the transcripts parted" + ); + assert_eq!( + off.transcript.sample_field_element(), + on.transcript.sample_field_element(), + "the next challenge moved" + ); + verify_tall(&off.committed, &off.proof).expect("the knob-off proof verifies"); + verify_tall(&on.committed, &on.proof).expect("the knob-on proof verifies"); + } + + /// ⛔ The identity above can fail. With the fault armed the card hands back + /// its first column's value plus one; the knob-on proof must then differ + /// from the knob-off one at CPU — the first table the card values — and + /// must not verify. Needs a device: there is no card value to corrupt + /// without one, and it says so rather than passing. + #[test] + fn a_wrong_card_value_changes_the_argument_and_fails_it() { + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + if !a_device() { + eprintln!("argue device columns: SKIPPED, no device to corrupt a value on"); + return; + } + let off = prove_tall(&airs, &columns, TODAY); + multilinear::gpu::force_column_value_fault(true); + let faulted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + prove_tall( + &airs, + &columns, + Knobs { + columns: true, + ..TODAY + }, + ) + })); + multilinear::gpu::force_column_value_fault(false); + let faulted = faulted.expect("the faulted arm proves"); + assert_eq!(faulted.on_card, 9, "the fault is on the card's path"); + assert_eq!( + first_table_that_differs(&off.proof, &faulted.proof), + Some(0), + "the identity must name CPU, the first table the card valued" + ); + assert_eq!( + verify_tall(&faulted.committed, &faulted.proof), + Err(MlError::ShiftedReadMismatch), + "a corrupted column value must fail the claim reduce's check" + ); + } + + /// ★ `LAMBDA_VM_ARGUE_DEVICE_TABLES` moves no byte of the proof, alone or + /// with `LAMBDA_VM_ARGUE_DEVICE_COLUMNS`: the same canonical bytes table by + /// table and whole, the same transcript and next challenge, and each proof + /// verifies. + /// + /// On a device the arms are shown to take their own paths: the card builds + /// 12 tables with the knob on — per table, the zerocheck's two `eq` weights + /// and the reduce's shift table and batched column at the one offset these + /// AIRs read — and none with it off. ⚠ Run it alone (`--exact`): the + /// counters are process-wide. + #[test] + fn the_argument_proves_the_same_bytes_with_its_tables_on_the_card() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + let mut today = prove_tall(&airs, &columns, TODAY); + verify_tall(&today.committed, &today.proof).expect("today's proof verifies"); + let today_state = today.transcript.state(); + let today_next = today.transcript.sample_field_element(); + let device = a_device(); + for (on, tables) in [(false, true), (true, true)] { + let what = format!("columns {on}, tables {tables}"); + let mut arm = prove_tall( + &airs, + &columns, + Knobs { + columns: on, + tables, + ..TODAY + }, + ); + if device { + assert_eq!( + (today.tables_on_card, arm.tables_on_card), + (0, 12), + "{what}: the tables the card built (today, this arm)" + ); + let columns_moved = if on { (9, 4) } else { (0, 13) }; + assert_eq!( + (arm.on_card, arm.on_host), + columns_moved, + "{what}: the column values (card, host)" + ); + eprintln!( + "argue device tables: {what}: the card built {} tables", + arm.tables_on_card + ); + } else { + assert_eq!(arm.tables_on_card, 0, "{what}: no device built a table"); + eprintln!( + "argue device tables: no device; the host's tables compared with themselves" + ); + } + assert_eq!( + first_table_that_differs(&today.proof, &arm.proof), + None, + "{what}: a table's argument changed" + ); + assert_eq!( + bincode::serialize(&today.proof).unwrap(), + bincode::serialize(&arm.proof).unwrap(), + "{what}: the proofs' canonical bytes differ" + ); + assert_eq!( + today_state, + arm.transcript.state(), + "{what}: the transcripts parted" + ); + assert_eq!( + today_next, + arm.transcript.sample_field_element(), + "{what}: the next challenge moved" + ); + verify_tall(&arm.committed, &arm.proof).unwrap_or_else(|e| panic!("{what}: {e:?}")); + } + } + + /// ⛔ The identity above can fail. With a fault armed, the first cell of + /// tables the card builds is overwritten: under `Eq`, the zerocheck's `eq(r)` + /// weight and the reduce's offset-0 `eq(α)` table; under `Batched`, the + /// reduce's offset-0 batched column. The proof must then differ from + /// today's at CPU, the first table argued, and must not verify. + /// + /// ★ Here both faults are refused by the reduce (`ShiftedReadMismatch`), and + /// the test pins why. CPU has no constraints of its own + /// (`EmptyConstraints`), so its zerocheck rule is `eq(r)` times an empty sum, + /// which the program builder makes the constant zero: a wrong `eq(r)` moves + /// no round. CPU's GKR, zerocheck and factor values stay today's; its reduce + /// is what the fault reaches, and the reduce's final check is the first to + /// see it. A zerocheck weight fault under a table with constraints is + /// refused as `BatchMismatch`, which multilinear's `argue_device_tables` + /// matrix pins. + /// + /// The control comes first: the same arm without a fault is today's proof, + /// and it verifies. So what the faulted arms show is the fault's doing. + /// Needs a device, and says so rather than passing without one. + #[test] + fn a_wrong_table_on_the_card_changes_the_argument_and_fails_it() { + use multilinear::gpu::TableFault; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + if !a_device() { + eprintln!("argue device tables: SKIPPED, no device to corrupt a table on"); + return; + } + let today = prove_tall(&airs, &columns, TODAY); + // CPU's argument, part by part: what precedes its reduce, and the reduce. + let cpu = |arm: &TallArm<'_>| { + let table = &arm.proof.tables[0]; + let before = [ + bincode::serialize(&table.gkr).unwrap(), + bincode::serialize(&table.constraint.sumcheck).unwrap(), + bincode::serialize(&table.constraint.factor_values).unwrap(), + ]; + ( + before, + bincode::serialize(&table.constraint.reduce).unwrap(), + ) + }; + let (today_before, today_reduce) = cpu(&today); + + let clean = prove_tall( + &airs, + &columns, + Knobs { + tables: true, + ..TODAY + }, + ); + assert_eq!( + clean.tables_on_card, 12, + "the control is on the card's path" + ); + assert_eq!( + first_table_that_differs(&today.proof, &clean.proof), + None, + "without a fault the card's tables are the host's" + ); + verify_tall(&clean.committed, &clean.proof).expect("without a fault the proof verifies"); + + for fault in [TableFault::Eq, TableFault::Batched] { + multilinear::gpu::force_table_fault(Some(fault)); + let faulted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + prove_tall( + &airs, + &columns, + Knobs { + tables: true, + ..TODAY + }, + ) + })); + multilinear::gpu::force_table_fault(None); + let faulted = faulted.unwrap_or_else(|_| panic!("{fault:?}: the faulted arm proves")); + assert_eq!( + faulted.tables_on_card, 12, + "{fault:?}: the fault is on the card's path" + ); + assert_eq!( + first_table_that_differs(&today.proof, &faulted.proof), + Some(0), + "{fault:?}: the identity must name CPU, the first table argued" + ); + let (before, reduce) = cpu(&faulted); + assert_eq!( + before, today_before, + "{fault:?}: CPU's GKR, zerocheck and factor values are today's (its zerocheck is \ + eq(r) times an empty sum)" + ); + assert_ne!( + reduce, today_reduce, + "{fault:?}: the fault reaches CPU's reduce" + ); + assert_eq!( + verify_tall(&faulted.committed, &faulted.proof), + Err(MlError::ShiftedReadMismatch), + "{fault:?}: the reduce's final check refuses the corrupted table" + ); + } + eprintln!( + "argue device tables: both faults refused by the reduce, CPU's zerocheck untouched" + ); + } + + /// The CPU table's leading columns a count-only AIR would declare + /// preprocessed in the tests below: the two selectors and the first + /// operand, which together say WHICH operations the program runs in which + /// order. + const CPU_PREFIX: usize = 3; + + /// The CPU trace with its rows reversed — a DIFFERENT program (its leading + /// columns are not the pinned ones) that still proves: the bus is a + /// multiset, so the lookups balance in any row order. It is exactly the + /// forgery a preprocessed commitment exists to refuse. + fn reversed_cpu_columns() -> Vec> { + cpu_columns() + .into_iter() + .map(|mut column| { + column.reverse(); + column + }) + .collect() + } + + /// How the verifier describes the CPU table's preprocessed prefix. + #[derive(Clone, Copy)] + enum PrefixStatement { + /// Today's `statement()`: no copies, so a count of zero. + CountZero, + /// `statement_with_prepared_prefix(count)`: the AIR's count, no copies. + PreparedPrefix(usize), + } + + /// The three tables argued with `cpu_cols` committed, the prepared stack + /// built over the PINNED program columns (`cpu_columns()`'s first + /// `opened` columns), and the verifier describing the CPU prefix as + /// `statement`. + /// + /// `prove_opening` says whether the prover carries the prepared opening; + /// `verify_check` whether the verifier settles one. + fn argue_prepared( + cpu_cols: Vec>, + opened: usize, + prove_opening: bool, + statement: PrefixStatement, + verify_check: bool, + ) -> Result<(), MlError> { + let (cpu_air, add_air, mul_air) = airs(); + let committed = CommittedTables::<_, _, KeccakWhir>::commit( + vec![ + table(&cpu_air, &cpu_cols)?, + table(&add_air, &add_columns())?, + table(&mul_air, &mul_columns())?, + ], + &config(), + )?; + + // The program's columns, committed outside the proof: the ones the + // verifier trusts, whatever the prover put in its trace. + let pinned: Vec> = cpu_columns()[..opened] + .iter() + .map(|column| Mle::new(column.clone())) + .collect::>()?; + let num_vars = pinned[0].num_vars(); + let layout = global_layout(&[(pinned.len(), num_vars)], config().format.stack)?; + let borrowed = multilinear::stacking::borrow(&pinned); + let prepared_commitment = + StackedCommitment::::commit(layout, &borrowed, None, &config())?; + let at = leading_columns(0, opened); + + let mut prover = DefaultTranscript::::new(b"multilinear-table"); + let proof = multi_prove( + &committed, + &config(), + &mut prover, + prove_opening.then_some(Prepared { + commitment: &prepared_commitment, + columns: &borrowed, + at: &at, + }), + )?; + + let layouts: Vec<&TableLayout<'_, Fp, Ext>> = + committed.tables().iter().map(|t| t.layout()).collect(); + let statements: Vec> = layouts + .iter() + .enumerate() + .map(|(index, layout)| match (index, statement) { + (0, PrefixStatement::PreparedPrefix(count)) => { + layout.statement_with_prepared_prefix(count) + } + _ => layout.statement(), + }) + .collect(); + let roots = prepared_commitment.roots(); + let mut verifier = DefaultTranscript::::new(b"multilinear-table"); + multi_verify::<_, _, _, KeccakWhir>( + &proof, + &statements, + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &ExtE::zero(), + &config(), + &mut verifier, + verify_check.then_some(PreparedCheck { + roots: &roots, + layout: prepared_commitment.layout(), + domain: prepared_commitment.domain(), + at: &at, + }), + ) + } + + /// ⛔ THE TRAP, SHOWN RATHER THAN DESCRIBED: a table whose preprocessed + /// columns the statement counts as ZERO binds nothing. The reversed CPU + /// trace is a different program and it VERIFIES — which is what an LFM + /// table (count-only `with_preprocessed`, so `precomputed_columns()` is + /// empty) pushed through `statement()` would do for any forged program. + /// + /// This is the control for the two refusals below: it proves the forgery is + /// a valid proof in every respect but the program, so their failures are + /// the count's doing and nothing else's. + #[test] + fn a_prefix_counted_as_zero_binds_no_program() { + argue_prepared( + reversed_cpu_columns(), + CPU_PREFIX, + false, + PrefixStatement::CountZero, + false, + ) + .expect("the forged program verifies when its prefix is counted as zero — the trap"); + } + + /// The trap closed, first direction: the AIR's count with NO prepared + /// opening to settle it is refused, the forged program with it. + #[test] + fn a_counted_prefix_nothing_settles_is_refused() { + assert!( + argue_prepared( + reversed_cpu_columns(), + CPU_PREFIX, + false, + PrefixStatement::PreparedPrefix(CPU_PREFIX), + false, + ) + .is_err(), + "a preprocessed prefix neither opened nor recomputed must be refused" + ); + // …and the honest program too: without an opening nothing binds it + // either, so a verifier that accepted this would accept the forgery. + assert!( + argue_prepared( + cpu_columns(), + CPU_PREFIX, + false, + PrefixStatement::PreparedPrefix(CPU_PREFIX), + false, + ) + .is_err(), + "an unsettled prefix is refused whatever the trace holds" + ); + } + + /// The trap closed, second direction: an HONEST proof whose prepared opening + /// settles the whole counted prefix is accepted. Under the old bound + /// (`settled > preprocessed.len()`, the copies' length) this proof was + /// rejected for carrying the opening that binds it. + #[test] + fn an_honest_prefix_settled_by_its_opening_is_accepted() { + argue_prepared( + cpu_columns(), + CPU_PREFIX, + true, + PrefixStatement::PreparedPrefix(CPU_PREFIX), + true, + ) + .expect("an honest proof whose opening settles the counted prefix verifies"); + // The old statement shape, count zero, against the same honest proof: + // the settled count exceeds it and the proof is refused. Kept as the + // record of why the count had to move off the copies' length. + assert!( + argue_prepared( + cpu_columns(), + CPU_PREFIX, + true, + PrefixStatement::CountZero, + true, + ) + .is_err(), + "a statement counting zero preprocessed columns cannot take a settled prefix" + ); + } + + /// The opening is what binds the program: the forged trace with the + /// prepared opening over the PINNED columns is refused, because the values + /// the forged table settled on are not what the pinned commitment takes at + /// that point. + #[test] + fn a_forged_prefix_is_refused_by_the_prepared_opening() { + let refused = argue_prepared( + reversed_cpu_columns(), + CPU_PREFIX, + true, + PrefixStatement::PreparedPrefix(CPU_PREFIX), + true, + ); + assert!( + refused.is_err(), + "the forged program must not open the pinned commitment" + ); + } + + /// Policy B: the three tables proved with the CPU's prefix LEFT OUT of the + /// main stack (`commit_grouped_settled`), its claims settled only by the + /// prepared opening over the pinned columns. `prove_b` / `verify_b` say + /// which policy each side runs; the verifier rebuilds its layout from the + /// shapes under its own policy, never from the prover's commitment. + fn argue_policy( + cpu_cols: Vec>, + prove_b: bool, + verify_b: bool, + settled_by_prover: usize, + ) -> Result<(), MlError> { + let (cpu_air, add_air, mul_air) = airs(); + let tables = vec![ + table(&cpu_air, &cpu_cols)?, + table(&add_air, &add_columns())?, + table(&mul_air, &mul_columns())?, + ]; + let sizes = [3usize]; + let settled = if prove_b { + vec![settled_by_prover, 0, 0] + } else { + Vec::new() + }; + let committed = CommittedTables::<_, _, KeccakWhir>::commit_grouped_settled( + tables, + &sizes, + &config(), + &settled, + )?; + let pinned: Vec> = cpu_columns()[..CPU_PREFIX] + .iter() + .map(|column| Mle::new(column.clone())) + .collect::>()?; + let num_vars = pinned[0].num_vars(); + let layout = global_layout(&[(pinned.len(), num_vars)], config().format.stack)?; + let borrowed = multilinear::stacking::borrow(&pinned); + let prepared_commitment = + StackedCommitment::::commit(layout, &borrowed, None, &config())?; + let at = leading_columns(0, CPU_PREFIX); + + let mut prover = DefaultTranscript::::new(b"multilinear-table"); + let proof = multi_prove( + &committed, + &config(), + &mut prover, + Some(Prepared { + commitment: &prepared_commitment, + columns: &borrowed, + at: &at, + }), + )?; + + let statements: Vec> = committed + .tables() + .iter() + .enumerate() + .map(|(index, t)| { + if index == 0 { + t.layout().statement_with_prepared_prefix(CPU_PREFIX) + } else { + t.layout().statement() + } + }) + .collect(); + // The verifier's own stack, from the shapes under ITS policy. + let shapes: Vec<(usize, usize)> = three_shapes() + .into_iter() + .enumerate() + .map(|(index, (width, n))| { + if verify_b && index == 0 { + (width - CPU_PREFIX, n) + } else { + (width, n) + } + }) + .collect(); + let group_layout = global_layout(&shapes, config().format.stack)?; + let domain = Domain::::new(group_layout.n_stack() + config().log_blowup)?; + let roots = prepared_commitment.roots(); + let mut verifier = DefaultTranscript::::new(b"multilinear-table"); + multi_verify_settled::<_, _, _, KeccakWhir>( + &proof, + &statements, + std::slice::from_ref(&group_layout), + std::slice::from_ref(&domain), + &sizes, + &ExtE::zero(), + &config(), + &mut verifier, + Some(PreparedCheck { + roots: &roots, + layout: prepared_commitment.layout(), + domain: prepared_commitment.domain(), + at: &at, + }), + verify_b, + ) + } + + /// ★ Policy B round-trips: the prefix out of the main stack, settled by the + /// prepared opening alone — and the main stack is narrower for it. + #[test] + fn policy_b_settles_the_prefix_only_in_the_prepared_stack() { + argue_policy(cpu_columns(), true, true, CPU_PREFIX) + .expect("an honest policy-B proof verifies under policy B"); + // Policy A still round-trips through the same entry points. + argue_policy(cpu_columns(), false, false, 0) + .expect("an honest policy-A proof verifies under policy A"); + } + + /// Under policy B the prepared opening is the prefix's ONLY binding, and it + /// refuses the forged program. + #[test] + fn policy_b_refuses_a_forged_prefix() { + assert!( + argue_policy(reversed_cpu_columns(), true, true, CPU_PREFIX).is_err(), + "the forged prefix must not open the pinned stack under policy B" + ); + } + + /// The policy is the verifier's: a B proof read as A, and an A proof read + /// as B, are both refused — their group openings describe other stacks. + #[test] + fn a_proof_read_under_the_other_policy_is_refused() { + assert!( + argue_policy(cpu_columns(), true, false, CPU_PREFIX).is_err(), + "a policy-B proof must not verify under policy A" + ); + assert!( + argue_policy(cpu_columns(), false, true, 0).is_err(), + "a policy-A proof must not verify under policy B" + ); + } + + /// ⛔ A prover whose prepared opening does not settle exactly the prefix it + /// left out of the stacks is refused before it hands anything out: a column + /// in neither stack would be bound by nothing. + #[test] + fn a_left_out_prefix_the_opening_does_not_settle_is_refused_by_the_prover() { + assert!( + argue_policy(cpu_columns(), true, true, CPU_PREFIX - 1).is_err(), + "the prover must refuse a stack exclusion its opening does not match" + ); + } + + /// An opening that settles only part of the counted prefix leaves the rest + /// bound by nothing, and is refused like no opening at all. + #[test] + fn a_prefix_settled_only_in_part_is_refused() { + assert!( + argue_prepared( + cpu_columns(), + CPU_PREFIX - 1, + true, + PrefixStatement::PreparedPrefix(CPU_PREFIX), + true, + ) + .is_err(), + "a column past the settled run, with no copy to recompute, must be refused" + ); + } + + /// ★ D-ARGUE stage 1 (the fused zerocheck with integer nodes — on by default + /// in production) moves no byte of the proof, alone or with the other argue + /// knobs on: the same canonical bytes, table by table and whole, the same + /// transcript and next challenge, and every proof verifies. + /// + /// On a device the arms are shown to take their own paths: off, no session + /// runs the fused rounds; on, the tall tables do (each is 2^13 or 2^14 rows, + /// past the fused rounds' 2^7). ⚠ Run it alone (`--exact`): the counters are + /// process-wide. + #[test] + fn the_argument_proves_the_same_bytes_with_the_fused_rounds() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + let mut today = prove_tall(&airs, &columns, TODAY); + verify_tall(&today.committed, &today.proof).expect("today's proof verifies"); + let today_state = today.transcript.state(); + let today_next = today.transcript.sample_field_element(); + let device = a_device(); + assert_eq!(today.fused, 0, "today runs no fused session"); + let fused = Knobs { + fused: true, + ..TODAY + }; + let all = Knobs { + columns: true, + tables: true, + fused: true, + ..TODAY + }; + for knobs in [fused, all] { + let mut arm = prove_tall(&airs, &columns, knobs); + if device { + assert!(arm.fused > 0, "{knobs:?}: the card ran the fused rounds"); + eprintln!("argue fused: {knobs:?}: {} fused sessions", arm.fused); + } else { + assert_eq!(arm.fused, 0, "{knobs:?}: no device, no fused session"); + eprintln!("argue fused: no device; today's rounds ran"); + } + assert_eq!( + first_table_that_differs(&today.proof, &arm.proof), + None, + "{knobs:?}: a table's argument changed" + ); + assert_eq!( + bincode::serialize(&today.proof).unwrap(), + bincode::serialize(&arm.proof).unwrap(), + "{knobs:?}: the proofs' canonical bytes differ" + ); + assert_eq!( + today_state, + arm.transcript.state(), + "{knobs:?}: the transcripts parted" + ); + assert_eq!( + today_next, + arm.transcript.sample_field_element(), + "{knobs:?}: the next challenge moved" + ); + verify_tall(&arm.committed, &arm.proof).unwrap_or_else(|e| panic!("{knobs:?}: {e:?}")); + } + } + + /// ⛔ The identity above can fail. With the fused rounds' bus coefficient + /// off by one on the card, the proof must differ from today's at CPU, the + /// first table argued, and must not verify. Needs a device, and says so + /// rather than passing without one. + #[test] + fn a_wrong_fused_bus_coefficient_changes_the_argument_and_fails_it() { + use multilinear::gpu_fused::{FusedFaults, force_fused_faults}; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + if !a_device() { + eprintln!("argue fused: SKIPPED, no device to corrupt a coefficient on"); + return; + } + let today = prove_tall(&airs, &columns, TODAY); + force_fused_faults(FusedFaults { + bus: true, + ..FusedFaults::default() + }); + let faulted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + prove_tall( + &airs, + &columns, + Knobs { + fused: true, + ..TODAY + }, + ) + })); + force_fused_faults(FusedFaults::default()); + let faulted = faulted.expect("the faulted arm proves"); + assert!(faulted.fused > 0, "the fault is on the fused path"); + assert_eq!( + first_table_that_differs(&today.proof, &faulted.proof), + Some(0), + "the identity must name CPU, the first table argued" + ); + let verdict = verify_tall(&faulted.committed, &faulted.proof); + assert!( + verdict.is_err(), + "a proof over a wrong bus coefficient must not verify, got {verdict:?}" + ); + eprintln!("argue fused: the wrong coefficient was refused: {verdict:?}"); + } + + /// ★ `LAMBDA_VM_ARGUE_GKR_GRUEN` (D-ARGUE S1-3) moves no byte of the proof, + /// alone, beside the fused zerocheck (the production default), and with + /// every other argue knob: the same canonical bytes, table by table and + /// whole, the same transcript and next challenge, and every proof verifies. + /// + /// On a device the arms are shown to take their own paths: off, no GKR + /// layer runs Gruen's rounds; on, CPU's and ADD's card layers do. ⚠ Run it + /// alone (`--exact`): the counters are process-wide. + #[test] + fn the_argument_proves_the_same_bytes_with_the_gkr_gruen_rounds() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + let device = a_device(); + for (base, arms) in [ + ( + TODAY, + vec![ + Knobs { + gruen: true, + ..TODAY + }, + Knobs { + columns: true, + tables: true, + gruen: true, + ..TODAY + }, + ], + ), + ( + Knobs { + fused: true, + ..TODAY + }, + vec![Knobs { + columns: true, + tables: true, + fused: true, + gruen: true, + ..TODAY + }], + ), + ] { + let mut today = prove_tall(&airs, &columns, base); + verify_tall(&today.committed, &today.proof).expect("today's proof verifies"); + assert_eq!(today.gruen, 0, "{base:?}: no Gruen layer"); + let today_state = today.transcript.state(); + let today_next = today.transcript.sample_field_element(); + for knobs in arms { + let mut arm = prove_tall(&airs, &columns, knobs); + if device { + assert!( + arm.gruen > 0, + "{knobs:?}: the card layers ran Gruen's rounds" + ); + eprintln!("argue gkr gruen: {knobs:?}: {} Gruen layers", arm.gruen); + } else { + assert_eq!(arm.gruen, 0, "{knobs:?}: no device, no Gruen layer"); + eprintln!("argue gkr gruen: no device; today's layer rounds ran"); + } + assert_eq!( + first_table_that_differs(&today.proof, &arm.proof), + None, + "{knobs:?}: a table's argument changed" + ); + assert_eq!( + bincode::serialize(&today.proof).unwrap(), + bincode::serialize(&arm.proof).unwrap(), + "{knobs:?}: the proofs' canonical bytes differ" + ); + assert_eq!( + today_state, + arm.transcript.state(), + "{knobs:?}: the transcripts parted" + ); + assert_eq!( + today_next, + arm.transcript.sample_field_element(), + "{knobs:?}: the next challenge moved" + ); + verify_tall(&arm.committed, &arm.proof) + .unwrap_or_else(|e| panic!("{knobs:?}: {e:?}")); + } + } + } + + /// ★ `LAMBDA_VM_ARGUE_GKR_INPUT` (D-BATCH M1-2) moves no byte of the proof, + /// alone and with the production defaults beside it (the fused zerocheck and + /// the Gruen layers): the same canonical bytes, table by table and whole, the + /// same transcript and next challenge, and every proof verifies. On a device + /// the arms show their trees written from the columns. ⚠ Run it alone. + #[test] + fn the_argument_proves_the_same_bytes_with_the_input_from_columns() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + let device = a_device(); + let production = Knobs { + columns: true, + tables: true, + fused: true, + gruen: true, + ..TODAY + }; + for (base, arm) in [ + ( + TODAY, + Knobs { + input: true, + ..TODAY + }, + ), + ( + production, + Knobs { + input: true, + ..production + }, + ), + ( + production, + Knobs { + input: true, + nolift: true, + ..production + }, + ), + ] { + let mut today = prove_tall(&airs, &columns, base); + verify_tall(&today.committed, &today.proof).expect("today's proof verifies"); + assert_eq!(today.input, 0, "{base:?}: no tree from the columns"); + let today_state = today.transcript.state(); + let today_next = today.transcript.sample_field_element(); + let mut got = prove_tall(&airs, &columns, arm); + if device { + assert!( + got.input > 0, + "{arm:?}: the trees were written from the columns" + ); + if arm.nolift { + assert!( + got.nolift > 0, + "{arm:?}: the zerocheck read the columns, no lift" + ); + } + eprintln!( + "argue gkr input: {arm:?}: {} trees from the columns, {} tables with no lift", + got.input, got.nolift + ); + } else { + assert_eq!(got.input, 0, "{arm:?}: no device, no tree from the columns"); + eprintln!("argue gkr input: no device; today's trees"); + } + assert_eq!( + first_table_that_differs(&today.proof, &got.proof), + None, + "{arm:?}: a table's argument changed" + ); + assert_eq!( + bincode::serialize(&today.proof).unwrap(), + bincode::serialize(&got.proof).unwrap(), + "{arm:?}: the proofs' canonical bytes differ" + ); + assert_eq!( + today_state, + got.transcript.state(), + "{arm:?}: the transcripts parted" + ); + assert_eq!( + today_next, + got.transcript.sample_field_element(), + "{arm:?}: the next challenge moved" + ); + verify_tall(&got.committed, &got.proof).unwrap_or_else(|e| panic!("{arm:?}: {e:?}")); + } + } + + /// ⛔ The identity above can fail. With the fault armed, every Gruen layer + /// adds one to its first round's `s(1)`. The proof must then differ from + /// today's at CPU, the first table argued, and be refused by GKR's layer + /// check. Needs a device, and says so rather than passing without one. + #[test] + fn a_wrong_gkr_gruen_round_changes_the_argument_and_fails_it() { + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + if !a_device() { + eprintln!("argue gkr gruen: SKIPPED, no device to corrupt a round on"); + return; + } + let today = prove_tall(&airs, &columns, TODAY); + multilinear::gkr_gruen::force_gkr_gruen_fault(true); + let faulted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + prove_tall( + &airs, + &columns, + Knobs { + gruen: true, + ..TODAY + }, + ) + })); + multilinear::gkr_gruen::force_gkr_gruen_fault(false); + let faulted = faulted.expect("the faulted arm proves"); + assert!(faulted.gruen > 0, "the fault is on the Gruen path"); + assert_eq!( + first_table_that_differs(&today.proof, &faulted.proof), + Some(0), + "the identity must name CPU, the first table argued" + ); + let verdict = verify_tall(&faulted.committed, &faulted.proof); + assert!( + matches!(verdict, Err(MlError::LayerRelationMismatch { .. })), + "GKR's layer check must refuse it, got {verdict:?}" + ); + eprintln!("argue gkr gruen: the wrong round was refused: {verdict:?}"); + } + + /// A tall table's zerocheck batch as [`prove`] builds it — the constraint + /// rule beside the bus's two, combined — at arbitrary challenges: the + /// program a device's rounds walk. + fn tall_batch(table: &CommittedTable<'_, Fp, Ext>) -> multilinear::program::Program { + let interactions = multilinear_logup::interactions( + table.layout.interactions, + table.slot_of().len(), + &ExtE::from(0x2a_u64), + &ExtE::from(0x3b_u64), + |col| slot(table.slot_of(), col), + ) + .unwrap(); + let num_vars = table.num_vars(); + let claim: Vec = (0..logup::input_layer_vars(interactions.len(), num_vars) as u64) + .map(|i| ExtE::from(i * 7 + 3)) + .collect(); + let (weight_r, weight_z) = weight_slots(table.kinds().len()); + let bus = logup::claim_statements(&interactions, &claim, num_vars, weight_z).unwrap(); + let shape = table.shape(); + let betas = multilinear_air::beta_powers(&ExtE::from(0x5eed_u64), shape.num_roots()); + let zerocheck = shape.program(&betas, weight_r).unwrap(); + multilinear::program::combine( + &[ + &zerocheck, + bus.numerator.program().unwrap(), + bus.denominator.program().unwrap(), + ], + &[ExtE::from(1u64), ExtE::from(5u64), ExtE::from(25u64)], + ) + .unwrap() + } + + /// ★ Every tall table's zerocheck batch holds fewer values a thread on + /// demand, so under the knob — with every batch counted big — each takes + /// the path the VM's big batches take: the identity below compares two + /// paths, not one path with itself. + #[test] + fn the_tall_batches_hold_fewer_values_on_demand() { + let airs = airs(); + let columns = tall_columns(); + let tables = [ + table(&airs.0, &columns[0]).unwrap(), + table(&airs.1, &columns[1]).unwrap(), + table(&airs.2, &columns[2]).unwrap(), + ]; + for (k, table) in tables.iter().enumerate() { + let batch = tall_batch(table); + let today = multilinear::gpu::lower(&batch).expect("today's lowers"); + let demand = multilinear::gpu::lower(&batch.on_demand()).expect("on demand lowers"); + assert!( + demand.num_slots < today.num_slots, + "table {k}: on demand holds {} values a thread, today {}", + demand.num_slots, + today.num_slots + ); + } + } + + /// ★ `LAMBDA_VM_ARGUE_LEAN_PROGRAM` moves no byte of the proof, alone or with + /// every other argue knob: the same canonical bytes, transcript and next + /// challenge, and every proof verifies. + /// + /// On a device the arms are shown to take their own paths: off, no + /// zerocheck walks its program on demand; on, all three tables' do (every + /// batch counted big). ⚠ Run it alone (`--exact`): the counters are + /// process-wide. + #[test] + fn the_argument_proves_the_same_bytes_with_its_programs_on_demand() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + let mut today = prove_tall(&airs, &columns, TODAY); + verify_tall(&today.committed, &today.proof).expect("today's proof verifies"); + let today_state = today.transcript.state(); + let today_next = today.transcript.sample_field_element(); + let device = a_device(); + assert_eq!(today.lean_programs, 0, "today no program runs on demand"); + // Every knob of today's rounds: the fused rounds take the zerocheck + // away from the program these knobs act on, so they stay off here. + let every = Knobs { + columns: true, + tables: true, + program: true, + fused: false, + gruen: false, + input: false, + nolift: false, + }; + for knobs in [ + Knobs { + program: true, + ..TODAY + }, + every, + ] { + let mut arm = prove_tall(&airs, &columns, knobs); + if device { + assert_eq!( + arm.lean_programs, 3, + "{knobs:?}: the three zerochecks' rounds on demand" + ); + eprintln!( + "argue lean program: {knobs:?}: {} zerochecks on demand", + arm.lean_programs + ); + } else { + assert_eq!( + arm.lean_programs, 0, + "{knobs:?}: no device, no rounds on demand" + ); + eprintln!("argue lean program: no device; no rounds to run on demand"); + } + assert_eq!( + first_table_that_differs(&today.proof, &arm.proof), + None, + "{knobs:?}: a table's argument changed" + ); + assert_eq!( + bincode::serialize(&today.proof).unwrap(), + bincode::serialize(&arm.proof).unwrap(), + "{knobs:?}: the proofs' canonical bytes differ" + ); + assert_eq!( + today_state, + arm.transcript.state(), + "{knobs:?}: the transcripts parted" + ); + assert_eq!( + today_next, + arm.transcript.sample_field_element(), + "{knobs:?}: the next challenge moved" + ); + verify_tall(&arm.committed, &arm.proof).unwrap_or_else(|e| panic!("{knobs:?}: {e:?}")); + } + } + + /// ⛔ The identity above can fail, and both checks behind it see it. With + /// the fault armed, every constant of a program on demand is off by one. + /// + /// - The control first: the same arm without the fault, under the + /// cross-check, is today's proof, verifies, and had every session checked. + /// - Faulted: the proof differs from today's at CPU, the first table argued, + /// and the zerocheck's final check refuses it (`BatchMismatch`). + /// - Faulted under `LAMBDA_VM_ARGUE_XCHECK`: today's program walking the + /// same factors disagrees at the first round, and the prove is refused + /// there (`DeviceFailed`), before a proof exists. + /// + /// Needs a device, and says so rather than passing without one. + #[test] + fn a_wrong_program_on_demand_changes_the_argument_and_fails_it() { + let _overrides = ARGUE_OVERRIDES.lock().unwrap_or_else(|e| e.into_inner()); + let airs = airs(); + let columns = tall_columns(); + if !a_device() { + eprintln!("argue lean program: SKIPPED, no device to corrupt a program on"); + return; + } + let lean = Knobs { + program: true, + ..TODAY + }; + let today = prove_tall(&airs, &columns, TODAY); + + multilinear::gpu::force_argue_xcheck(Some(true)); + let control = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + try_prove_tall(&airs, &columns, lean) + })); + multilinear::gpu::force_argue_xcheck(None); + let control = control + .expect("the control proves") + .expect("the control is not refused"); + assert_eq!( + (control.lean_programs, control.lean_program_xchecks), + (3, 3), + "the control is on the path, every session checked" + ); + assert_eq!( + first_table_that_differs(&today.proof, &control.proof), + None, + "without the fault the rounds on demand are today's" + ); + verify_tall(&control.committed, &control.proof).expect("without the fault it verifies"); + + multilinear::gpu::force_lean_program_fault(true); + let faulted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + prove_tall(&airs, &columns, lean) + })); + multilinear::gpu::force_argue_xcheck(Some(true)); + let refused = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + try_prove_tall(&airs, &columns, lean).map(|_| ()) + })); + multilinear::gpu::force_argue_xcheck(None); + multilinear::gpu::force_lean_program_fault(false); + + let faulted = faulted.expect("the faulted arm proves"); + assert_eq!(faulted.lean_programs, 3, "the fault is on the path"); + assert_eq!( + first_table_that_differs(&today.proof, &faulted.proof), + Some(0), + "the identity must name CPU, the first table argued" + ); + assert_eq!( + verify_tall(&faulted.committed, &faulted.proof), + Err(MlError::BatchMismatch), + "the zerocheck's final check must refuse it" + ); + assert_eq!( + refused.expect("the cross-checked arm returns"), + Err(MlError::DeviceFailed { + stage: "program on demand" + }), + "the cross-check must refuse it before a proof exists" + ); + eprintln!( + "argue lean program: the wrong program was refused, by the verifier and the xcheck" + ); + } + + /// [`config`] with its stack cap at `cap`. + fn config_at(cap: usize) -> ChainConfig { + ChainConfig { + format: multilinear::whir_chain::ChainFormat { + stack: StackVars::new(cap).expect("a legal cap"), + ..multilinear::whir_chain::ChainFormat::DEFAULT + }, + ..config() + } + } + + /// The three tables' shapes, as `commit_grouped` reads them. + fn three_shapes() -> Vec<(usize, usize)> { + [cpu_columns(), add_columns(), mul_columns()] + .iter() + .map(|columns| (columns.len(), columns[0].len().trailing_zeros() as usize)) + .collect() + } + + /// Proves the three tables stacked under `prove_cap`, then verifies under + /// `verify_cap` with every layout and domain REBUILT from the shapes and the + /// verifier's own config — as the production verifier does + /// (`prover::multilinear_prove::stacks`), never taken from the proof or + /// from the prover's commitment. Returns the proof's root count. + fn argue_across(prove_cap: usize, verify_cap: usize) -> Result { + let (cpu_air, add_air, mul_air) = airs(); + let prover_config = config_at(prove_cap); + let committed = CommittedTables::<_, _, KeccakWhir>::commit( + vec![ + table(&cpu_air, &cpu_columns())?, + table(&add_air, &add_columns())?, + table(&mul_air, &mul_columns())?, + ], + &prover_config, + )?; + let mut prover = DefaultTranscript::::new(b"multilinear-table"); + let proof = multi_prove(&committed, &prover_config, &mut prover, None)?; + + let verifier_config = config_at(verify_cap); + let layouts = global_layouts( + &three_shapes(), + committed.sizes(), + verifier_config.format.stack, + )?; + let domains = layouts + .iter() + .map(|layout| Domain::::new(layout.n_stack() + verifier_config.log_blowup)) + .collect::, _>>()?; + let statements: Vec> = + committed.tables().iter().map(|t| t.statement()).collect(); + let mut verifier = DefaultTranscript::::new(b"multilinear-table"); + multi_verify::<_, _, _, KeccakWhir>( + &proof, + &statements, + &layouts, + &domains, + committed.sizes(), + &ExtE::zero(), + &verifier_config, + &mut verifier, + None, + )?; + Ok(proof.roots.len()) + } + + /// ★ S2: THE STACK IS A FORMAT CONSTANT OF BOTH SIDES. The three tables hold + /// 72 cells: one polynomial of 7 variables under a cap of 7, three of 5 under + /// a cap of 5. Each proof verifies under its own cap — the honest control, + /// first — and is REFUSED under the other, in both directions, as an error: + /// a panic fails this test. The code path of 25 against 27, at a size a + /// laptop proves. + #[test] + fn a_proof_stacked_under_one_cap_is_refused_under_another() { + let polys = |cap: usize| { + global_layout(&three_shapes(), StackVars::new(cap).unwrap()) + .unwrap() + .num_polys() + }; + assert_eq!( + (polys(7), polys(5)), + (1, 3), + "the caps must stack differently" + ); + assert_eq!(argue_across(7, 7).unwrap(), 1, "the honest control at 7"); + assert_eq!(argue_across(5, 5).unwrap(), 3, "the honest control at 5"); + assert!( + argue_across(7, 5).is_err(), + "a proof stacked under 7 verified under 5" + ); + assert!( + argue_across(5, 7).is_err(), + "a proof stacked under 5 verified under 7" + ); + } + + /// `stack_height` is `global_layout`'s height, and no group of a shape list + /// stacks taller than the whole list: what lets the query count charge all + /// of a proof's shapes and never undercharge one of its groups. + #[test] + fn the_stack_height_is_the_layouts_and_bounds_every_group() { + let shapes = [(5, 3), (4, 2), (4, 2), (0, 9), (16, 4), (2, 6)]; + for cap in [1, 4, 5, 6, 7, 8, 9, 25, StackVars::WIDEST] { + let cap = StackVars::new(cap).unwrap(); + let all = stack_height(&shapes, cap); + assert_eq!( + global_layout(&shapes, cap).unwrap().n_stack(), + all, + "{cap:?}" + ); + for start in 0..shapes.len() { + for end in start + 1..=shapes.len() { + assert!( + stack_height(&shapes[start..end], cap) <= all, + "{cap:?}: shapes[{start}..{end}] stack taller than all of them" + ); + } + } + } + // The empty column does not count toward the tallest. + assert_eq!(stack_height(&[(0, 9), (1, 2)], StackVars::LEGACY), 2); + } + + #[test] + fn a_row_that_is_not_an_addition_is_rejected() { + let mut columns = add_columns(); + columns[2][2] += FE::one(); + assert!(argue(cpu_columns(), columns, mul_columns()).is_err()); + } + + #[test] + fn a_row_that_is_not_a_product_is_rejected() { + let mut columns = mul_columns(); + columns[2][1] += FE::one(); + assert!(argue(cpu_columns(), add_columns(), columns).is_err()); + } + + /// Every table verifies on its own and the proof still fails: the balance + /// is the sum, and that is what a missing receive breaks. + #[test] + fn a_receive_that_never_happened_leaves_the_bus_unbalanced() { + let mut columns = add_columns(); + columns[3][2] = FE::zero(); + assert_eq!( + argue(cpu_columns(), columns, mul_columns()).unwrap_err(), + MlError::BusImbalance + ); + } + + /// A row that still adds up, but is not the one the CPU dispatched: the + /// table's own constraint holds and the bus does not. What makes this + /// meaningful is that the fingerprint is read off the *committed* columns. + #[test] + fn a_coherent_row_the_cpu_never_dispatched_unbalances_the_bus() { + let mut columns = add_columns(); + columns[0][0] = FE::from(2); + columns[1][0] = FE::from(9); // 2 + 9 = 11, still an addition + + assert_eq!( + argue(cpu_columns(), columns, mul_columns()).unwrap_err(), + MlError::BusImbalance + ); + } + + /// The LogUp roots really are being dropped: the program has more + /// constraints than the base prefix, and none of the auxiliary columns they + /// read gets committed. + #[test] + fn only_main_columns_are_committed() { + let (cpu_air, add_air, mul_air) = airs(); + + for (air_roots, num_base, aux_width, table, main_width) in [ + { + let table = table(&cpu_air, &cpu_columns()).unwrap(); + let program = cpu_air.constraint_program(); + ( + program.roots.len(), + program.num_base, + cpu_air.trace_layout().1, + table, + 5, + ) + }, + { + let table = table(&add_air, &add_columns()).unwrap(); + let program = add_air.constraint_program(); + ( + program.roots.len(), + program.num_base, + add_air.trace_layout().1, + table, + 4, + ) + }, + { + let table = table(&mul_air, &mul_columns()).unwrap(); + let program = mul_air.constraint_program(); + ( + program.roots.len(), + program.num_base, + mul_air.trace_layout().1, + table, + 4, + ) + }, + ] { + assert!( + air_roots > num_base, + "the fixture must have LogUp constraints to drop" + ); + assert!( + aux_width > 0, + "the univariate path commits auxiliary columns" + ); + assert_eq!(table.num_committed_columns(), main_width); + } + } + + /// The multilinear query count is the univariate prover's own accounting, in + /// the same regime — so these parameters are no weaker than the FRI ones the + /// repo already ships. One round, so no union-bound margin. + #[test] + fn the_query_count_matches_the_univariate_provers() { + use crate::proof::options::GoldilocksCubicProofOptions; + use multilinear::whir_chain::ChainConfig; + + let univariate = GoldilocksCubicProofOptions::with_params(4, 128, 20).unwrap(); + let multilinear = ChainConfig::with_security( + 2, + 4, + 4, + 128, + GrindBits { + query: 20, + ..GrindBits::default() + }, + ); + + assert_eq!( + multilinear.num_queries, univariate.fri_number_of_queries, + "the two regimes must agree" + ); + assert_eq!(multilinear.grind.query, univariate.grinding_factor); + } + + #[test] + fn a_table_with_no_bus_is_not_this_argument() { + let options = ProofOptions::default_test_options(); + let air: Air = AirWithBuses::new( + 4, + AuxiliaryTraceBuildData { + interactions: Vec::new(), + }, + &options, + 1, + AddConstraints, + ); + assert_eq!( + table(&air, &add_columns()).err(), + Some(MlError::EmptyPolynomial) + ); + } + + /// An ADD table of `2^12` rows: wide enough that its factors go to the card + /// (factors × rows over the device's cell floor), with its buses' interactions + /// at `(z, alpha) = (7, 11)`. + #[cfg(feature = "cuda")] + fn resident_add_table<'a>( + air: &'a Air, + ) -> ( + CommittedTable<'a, Fp, Ext>, + Vec>, + ) { + let rows = 1u64 << 12; + let columns = vec![ + (0..rows).map(FE::from).collect::>(), + (0..rows).map(|i| FE::from(2 * i)).collect(), + (0..rows).map(|i| FE::from(3 * i)).collect(), + vec![FE::one(); rows as usize], + ]; + let t = table(air, &columns).expect("the ADD table builds"); + let interactions = multilinear_logup::interactions( + t.layout.interactions, + t.slot_of().len(), + &ExtE::from(7u64), + &ExtE::from(11u64), + |col| slot(t.slot_of(), col), + ) + .expect("the interactions build"); + (t, interactions) + } + + /// ★★ A REFUSED GKR TREE IS COUNTED, AND THE HOST BUILDS THE SAME TREE. + /// + /// The tree's whole-tree promise (`multilinear::gpu::input_layer_tree_impl`) + /// is the one argue-side refusal that moves work to the host. It is reached + /// only when the tree's carry is refused too and handing it back saves enough + /// (a gigabyte, which only the widest precompiles do), so the test hands every + /// carry back (`set_hand_back_threshold_for_tests(Some(0))`) and leaves the + /// budget nothing to promise, by holding the rest of it itself. + /// + /// - The control, at the normal budget: the card builds the tree; nothing is + /// counted. + /// - A prefetch refused the same way is not a fallback (the consume site asks + /// again): nothing is counted. + /// - The consume path refused: one GKR tree refusal, one argue-side device + /// fallback, and the host's tree has the card's output fraction. + /// + /// ⚠ Run it with `--features cuda,multilinear/cuda`: this crate's `cuda` does + /// not turn on multilinear's, and without it the factors never go to the card. + #[cfg(feature = "cuda")] + #[test] + #[ignore = "the card, run alone, with multilinear/cuda: the budget and the counts are process-wide"] + fn a_refused_gkr_tree_is_counted_and_the_host_builds_the_same_tree() { + use math_cuda::device::{backend, device_fallbacks, reserve}; + use multilinear::gpu::{gkr_tree_refusals, set_hand_back_threshold_for_tests}; + + let (_, add_air, _) = airs(); + let (t, interactions) = resident_add_table(&add_air); + let resident = t.trace.reside_from_columns().expect( + "an ADD table of 2^12 rows goes to the card — only with multilinear's `cuda` \ + feature on (`--features cuda,multilinear/cuda`)", + ); + let counts = || (gkr_tree_refusals(), device_fallbacks()); + let at_start = counts(); + + // The control: the normal budget. + let on_card = logup::resident_tree(&interactions, resident.clone()) + .expect("at the normal budget the card builds the tree"); + assert_eq!(counts(), at_start, "a tree the card built counts nothing"); + + // The budget left to promise, taken; every carry handed back. + set_hand_back_threshold_for_tests(Some(0)); + let be = backend().expect("the card"); + let rest = be.vram_budget_bytes().saturating_sub(be.reserved_bytes()); + let held = reserve(rest).expect("the rest of the budget"); + + let prefetched = logup::resident_tree_deferred(&interactions, resident.clone()); + let after_prefetch = counts(); + let refused = logup::resident_tree(&interactions, resident); + let after_refusal = counts(); + drop(held); + set_hand_back_threshold_for_tests(None); + + assert!(prefetched.is_none(), "with no room a prefetch is declined"); + assert_eq!( + after_prefetch, at_start, + "a refused prefetch is not a fallback: the consume site asks again" + ); + assert!( + refused.is_none(), + "with no room the card must refuse the tree" + ); + assert_eq!( + after_refusal, + (at_start.0 + 1, at_start.1 + 1), + "the refused tree is one GKR tree refusal and one argue-side device fallback" + ); + + // The path the refusal hands the table to: the host's own tree. + let factors = t.trace.factors().expect("the host builds the factors"); + let on_host = FractionTree::build( + logup::input_layer(&interactions, &factors).expect("the host builds the layer"), + ) + .expect("the host builds the tree"); + assert_eq!( + on_host.output(), + on_card.output(), + "the host's tree has the card's output fraction" + ); + } +} diff --git a/crypto/stark/src/multilinear_table/batched.rs b/crypto/stark/src/multilinear_table/batched.rs new file mode 100644 index 000000000..9c037afa4 --- /dev/null +++ b/crypto/stark/src/multilinear_table/batched.rs @@ -0,0 +1,1559 @@ +//! ★ THE BATCHED ARGUE (`ArgueFormat::Batched`, D-BATCH §2): one argument for +//! every table of a proof, where [`multi_prove`](super::multi_prove) runs one +//! per table. +//! +//! The roots and `(z, α, β)` are today's. Then, in the transcript's order: +//! +//! - **G**, per bin of [`argue_plan`]: the bin's bus outputs absorbed, then one +//! LogUp-GKR ladder over its trees in lockstep +//! ([`multilinear::gkr_lockstep`]). Each tree leaves with its input claim +//! `(p̃_T, q̃_T)` at a point whose last `n_T` coordinates are the row point +//! `ρ_T`, as today. The bus balance is checked once, over every table. +//! - **C**: a zerocheck point `ξ` of `n_max` coordinates, the claims `p̃_T, q̃_T` +//! absorbed per table, one `λ`, and ONE front-loaded sumcheck +//! ([`multilinear::front_loaded`]) of `Σ_t λ^{3t}·f_t`, with today's per-table +//! rule `f_t = eq(ξ_{(statement: &TableStatement<'_, F, E>) -> Self + where + F: IsFFTField + IsPrimeField + IsSubFieldOf, + E: IsField + 'static, + { + Self { + num_vars: statement.num_vars, + input_vars: logup::input_layer_vars(statement.interactions.len(), statement.num_vars), + degree: (statement.shape.degree() + 1).max(2), + shifted: statement + .kinds + .iter() + .filter_map(FactorKind::source) + .any(|source| source.offset != 0), + interactions: statement.interactions.len(), + } + } +} + +/// How a table's interactions are laid into trees: `count` trees of +/// `2^slot_vars` slots, chunk `j` holding interactions +/// `[j·2^slot_vars, (j + 1)·2^slot_vars)` in order, the last padded `0/1`. +/// `count == 1` is today's one tree. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Chunking { + /// `b`: the bits that index a slot within a chunk. + pub slot_vars: usize, + /// `c`: the chunks. + pub count: usize, +} + +impl Chunking { + /// Today's: one tree over every interaction. + pub fn whole(shape: &ArgueShape) -> Self { + Self { + slot_vars: shape.input_vars - shape.num_vars, + count: 1, + } + } + + /// `ℓ`: the bits that index a chunk, `log2` of `count` rounded up to a + /// power of two — the coordinates of the challenge that combines the + /// chunks' claims. `slot_vars + combine_vars` is the table's `k_T − n_T`. + pub fn combine_vars(&self) -> usize { + self.count.next_power_of_two().trailing_zeros() as usize + } + + /// The input cells of the table's trees together. + fn cells(&self, shape: &ArgueShape) -> u128 { + (self.count as u128) << (shape.num_vars + self.slot_vars).min(120) + } + + /// Chunk `j`'s interactions. + pub fn range(&self, j: usize, interactions: usize) -> std::ops::Range { + let start = (j << self.slot_vars).min(interactions); + start..((j + 1) << self.slot_vars).min(interactions) + } +} + +/// ★ How `pack` lays a table out, one function for every side. +/// +/// [`GkrPack::Off`]: [`Chunking::whole`]. [`GkrPack::Chunks`]: over +/// `b ∈ [0, B]` (`2^B` today's slots), `c = ⌈m / 2^b⌉` chunks, the `b` that +/// minimises `c·2^{n+b} + (c − 1)·2^{tree_log_cells}` — the cells, and each +/// extra tree at its price — ties to the larger `b`, so `b = B` (today's tree) +/// unless cutting saves. +pub fn chunking(shape: &ArgueShape, pack: GkrPack) -> Chunking { + let whole = Chunking::whole(shape); + let GkrPack::Chunks { tree_log_cells } = pack else { + return whole; + }; + if shape.interactions < 2 { + return whole; + } + let tree = 1u128 << tree_log_cells.min(120); + let cost = |c: &Chunking| c.cells(shape) + (c.count as u128 - 1) * tree; + let mut best = whole; + for slot_vars in (0..whole.slot_vars).rev() { + let candidate = Chunking { + slot_vars, + count: shape.interactions.div_ceil(1 << slot_vars), + }; + if cost(&candidate) < cost(&best) { + best = candidate; + } + } + // The chunks and the all-padding ones after them fill today's slots + // exactly, so the combined claim sits at a point of today's length. + if best.slot_vars + best.combine_vars() != whole.slot_vars { + return whole; + } + best +} + +/// The public schedule of a batched argue. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArguePlan { + pub shapes: Vec, + /// Each table's trees ([`chunking`]); one tree a table under + /// [`GkrPack::Off`]. + pub chunks: Vec, + /// The bins, in the order their ladders run; each in table order, a + /// table's chunks adjacent and in order. + pub bins: Vec>, + /// `n_max`: the constraint sumcheck's rounds. + pub num_vars: usize, + /// `D_max`: its round degree. + pub degree: usize, +} + +impl ArguePlan { + /// Every table's trees together: the bus outputs a proof carries. + pub fn trees(&self) -> usize { + self.chunks.iter().map(|c| c.count).sum() + } + + /// Where table `t`'s first tree sits among the bus outputs (table order, + /// then chunk order). + pub fn first_tree(&self, t: usize) -> usize { + self.chunks[..t].iter().map(|c| c.count).sum() + } + + /// The input variables of each of table `t`'s trees. + pub fn tree_vars(&self, t: usize) -> usize { + self.shapes[t].num_vars + self.chunks[t].slot_vars + } +} + +/// [`argue_plan_packed`] with one tree a table: today's plan. +pub fn argue_plan(shapes: &[ArgueShape], bin_log_cells: u8) -> ArguePlan { + argue_plan_packed(shapes, bin_log_cells, GkrPack::Off) +} + +/// ★★ THE PLAN, one function for every side (the `absorb_roots` lesson: two +/// spellings of a public ordering drift silently). +/// +/// Each table's trees are [`chunking`]'s under `pack`, one unit of +/// `c·2^{n_T+b}` input cells (`2^{k_T}` under [`GkrPack::Off`]). Bins: units +/// sorted by `(cells desc, index asc)`, first-fit-decreasing under +/// `Σ cells ≤ 2^{bin_log_cells}`; a unit at or over the cap sits alone. Bins +/// run in the order they open, each in table order. +pub fn argue_plan_packed(shapes: &[ArgueShape], bin_log_cells: u8, pack: GkrPack) -> ArguePlan { + let cap = 1u128 << bin_log_cells.min(127); + let chunks: Vec = shapes.iter().map(|shape| chunking(shape, pack)).collect(); + let unit = |t: usize| chunks[t].cells(&shapes[t]); + let mut order: Vec = (0..shapes.len()).collect(); + order.sort_by(|&a, &b| unit(b).cmp(&unit(a)).then(a.cmp(&b))); + // (tables, cells used, open to more) + let mut bins: Vec<(Vec, u128, bool)> = Vec::new(); + for t in order { + let cells = unit(t); + if cells >= cap { + bins.push((vec![t], cells, false)); + continue; + } + match bins + .iter_mut() + .find(|(_, used, open)| *open && used + cells <= cap) + { + Some((tables, used, _)) => { + tables.push(t); + *used += cells; + } + None => bins.push((vec![t], cells, true)), + } + } + ArguePlan { + shapes: shapes.to_vec(), + chunks, + bins: bins + .into_iter() + .map(|(mut tables, _, _)| { + tables.sort_unstable(); + tables + }) + .collect(), + num_vars: shapes.iter().map(|s| s.num_vars).max().unwrap_or(0), + degree: shapes.iter().map(|s| s.degree).max().unwrap_or(2), + } +} + +/// The batched argue's messages. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct BatchedArgue { + /// Every tree's bus output fraction, in table order: one a table, or — + /// under a [`GkrPack::Chunks`] format — one for each chunk of a cut table, + /// in chunk order. + pub bus_outputs: Vec<(FieldElement, FieldElement)>, + /// One ladder per bin, in plan order. + pub gkr: Vec>, + /// The front-loaded constraint sumcheck: `n_max` rounds of `D_max` values. + pub constraint: SumcheckProof, + /// Per table, its committed factors' values at `r_{>>, + /// Per table, its claim reduction — present exactly for a table with a + /// shifted read. + pub reduces: Vec>>, +} + +/// A proof under the batched argue: [`MultiProof`] with the argue batched. +/// +/// ⚠ A TYPE OF ITS OWN, not a variant inside [`MultiProof`], so today's proofs +/// keep their bytes: the byte gates serialize `MultiProof` through rkyv, which +/// writes a discriminant for an enum. The openings are the same code and the +/// same fields. +#[derive( + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct BatchedMultiProof { + pub roots: Vec, + pub argue: BatchedArgue, + pub columns: Vec>, + pub preprocessed: Option>, +} + +/// ⛔ Faults a NEGATIVE test arms in the prover, each producing a proof the +/// verifier must refuse. [`multi_prove_batched`] arms none. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Default)] +pub struct ProverFaults { + /// This table's zerocheck weight takes the SUFFIX of `ξ`. + pub suffix_xi: Option, + /// This table's cross-table weight is `λ^{3t+1}`, not `λ^{3t}`. + pub wrong_weight: Option, + /// This table's `p̃` is absorbed off by one. + pub tamper_claim: Option, + /// This table is padded the `2^Δ` way, not by `Π X_j`. + pub scaled_padding: Option, + /// The prover plans its bins under this cap instead of the config's. + pub bin_log_cells: Option, + /// This table's bus output numerator is replaced by the value + /// that makes `Σ p/q = 0` over every table, absorbed and sent as if true, + /// while its ladder runs on the TRUE tree (only step 0's residual sees it). + /// A cut table's first chunk carries the forgery. + pub balance_with: Option, + /// The prover lays its trees out under this packing instead of the + /// config's. + pub pack: Option, + /// This cut table's first two chunks trade their interactions: honest + /// trees, outputs and ladder, over the chunks in the wrong order. + pub swap_chunks: Option, +} + +/// Which of the batched verifier's checks run. Every one, except in the +/// mutation tests that show each is load-bearing. +#[doc(hidden)] +#[derive(Clone, Copy, Debug)] +pub struct VerifierChecks { + /// Each ladder step's residual. + pub residual: bool, + /// The constraint sumcheck's final equation. + pub constraint: bool, + /// `Σ p_T/q_T == expected`. + pub balance: bool, + /// A reduction present exactly for a table with a shifted read. + pub reduce_shape: bool, + /// The preprocessed columns at `r_{( + interactions: &[Interaction], + trace: &TraceData, +) -> Result>, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, +{ + let factors = trace.factors()?; + let mut layers = vec![logup::input_layer(interactions, &factors)?]; + while layers.last().expect("non-empty").num_vars() > 0 { + let next = layers.last().expect("non-empty").fold()?; + layers.push(next); + } + layers.reverse(); + Ok(layers) +} + +/// Each of a table's trees' interactions under `chunks`: its own list when it +/// is one tree, else each chunk's in order. A last chunk too short to take +/// `2^b` slots by itself is padded with `0/1` interactions, which are the +/// padding's own cells, so its tree is the `2^b`-slot tree. +fn chunk_lists<'a, E: IsField + 'static>( + interactions: &'a [Interaction], + chunks: &Chunking, +) -> Vec]>> { + use std::borrow::Cow; + if chunks.count == 1 { + return vec![Cow::Borrowed(interactions)]; + } + let slots = 1usize << chunks.slot_vars; + (0..chunks.count) + .map(|j| { + let list = &interactions[chunks.range(j, interactions.len())]; + if list.len().next_power_of_two() == slots { + return Cow::Borrowed(list); + } + let padding = Interaction::new( + Affine::constant(FieldElement::zero()), + Affine::constant(FieldElement::one()), + ); + let mut padded = list.to_vec(); + padded.resize(slots / 2 + 1, padding); + Cow::Owned(padded) + }) + .collect() +} + +/// The table a tree belongs to, `tree` counted in table order then chunk +/// order. +fn table_of_tree(plan: &ArguePlan, tree: usize) -> usize { + let mut first = 0; + for (t, chunks) in plan.chunks.iter().enumerate() { + first += chunks.count; + if tree < first { + return t; + } + } + plan.chunks.len() +} + +/// A table's GKR claim from its trees' input claims: its one tree's, or — a +/// cut table — the chunks' combined at `s'`, drawn here, once the chunks' last +/// line challenge has fixed their claims ([`combine_chunk_claims`]). +fn table_claim( + chunk_claims: &[GkrClaim], + chunks: &Chunking, + transcript: &mut T, +) -> Result, MlError> +where + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + if chunk_claims.len() != chunks.count { + return Err(MlError::ArgueShapeMismatch { part: "chunks" }); + } + if chunks.count == 1 { + return Ok(chunk_claims[0].clone()); + } + let combine: Vec> = (0..chunks.combine_vars()) + .map(|_| transcript.sample_field_element()) + .collect(); + combine_chunk_claims(chunk_claims, &combine) +} + +/// ★ A cut table's claim: its `c` chunks' claims, all at one point `P`, +/// combined at `s'` (`ℓ` coordinates, `c ≤ 2^ℓ`) into the claim of today's one +/// tree at `s' ‖ P`: +/// +/// - `p̃ = Σ_{j( + chunk_claims: &[GkrClaim], + combine: &[FieldElement], +) -> Result, MlError> +where + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, +{ + let weights = multilinear::eq::eq_evals(combine); + let first = chunk_claims + .first() + .ok_or(MlError::ArgueShapeMismatch { part: "chunks" })?; + if chunk_claims.len() > weights.len() + || chunk_claims.iter().any(|claim| claim.point != first.point) + { + return Err(MlError::ArgueShapeMismatch { part: "chunks" }); + } + let mut p = FieldElement::::zero(); + let mut q = FieldElement::::zero(); + for (claim, w) in chunk_claims.iter().zip(&weights) { + p += w * &claim.p; + q += w * &claim.q; + } + for w in &weights[chunk_claims.len()..] { + q += w.clone(); + } + Ok(GkrClaim { + point: combine.iter().chain(&first.point).cloned().collect(), + p, + q, + }) +} + +/// A table's column claims when every committed factor reads its column +/// unshifted: each column's claim is its factor's value. A column read twice +/// must be claimed the same both times, and every column must be read. +fn direct_column_values( + kinds: &[FactorKind], + factor_values: &[FieldElement], + num_columns: usize, +) -> Result>, MlError> { + let mut columns: Vec>> = vec![None; num_columns]; + let mut values = factor_values.iter(); + for source in kinds.iter().filter_map(FactorKind::source) { + let value = values.next().ok_or(MlError::ArgueShapeMismatch { + part: "factor values", + })?; + if source.offset != 0 { + return Err(MlError::ArgueShapeMismatch { part: "reduction" }); + } + let slot = columns + .get_mut(source.column) + .ok_or(MlError::UnknownPolynomial { + index: source.column, + len: num_columns, + })?; + match slot { + Some(claimed) if claimed != value => return Err(MlError::EvaluationMismatch), + Some(_) => {} + None => *slot = Some(value.clone()), + } + } + if values.next().is_some() { + return Err(MlError::ArgueShapeMismatch { + part: "factor values", + }); + } + columns + .into_iter() + .collect::>>() + .ok_or(MlError::ArgueShapeMismatch { + part: "column claims", + }) +} + +/// A table's rules and its bus's row point. +type TableRules<'a, E> = (Vec>, Vec>); + +/// A table's three rules, today's: the zerocheck at `weight_r`, the bus's two +/// at `weight_z`, the bus claimed at `claim_point`. +fn table_rules<'a, F, E>( + statement: &TableStatement<'_, F, E>, + interactions: &'a [Interaction], + claim_point: &[FieldElement], + beta: &FieldElement, +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf, + E: IsField + 'static, +{ + let (weight_r, weight_z) = weight_slots(statement.kinds.len()); + let bus = logup::claim_statements(interactions, claim_point, statement.num_vars, weight_z)?; + let shape = statement.shape; + let betas = multilinear_air::beta_powers(beta, shape.num_roots()); + let zerocheck = Rule::compiled(shape.degree() + 1, shape.program(&betas, weight_r)?); + Ok(( + vec![zerocheck, bus.numerator, bus.denominator], + bus.row_point, + )) +} + +/// `λ^{3t}` per table — table `t`'s share of the batch. +fn table_weights(lambda: &FieldElement, tables: usize) -> Vec> { + let cube = lambda * lambda * lambda; + multilinear::challenge_powers(&cube, tables) +} + +/// Proves every table under the batched argue, against one commitment. +pub fn multi_prove_batched( + committed: &CommittedTables<'_, F, E, H>, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + prove_batched_on( + committed, + config, + transcript, + prepared, + ProverFaults::default(), + Where::Device, + ) +} + +/// Where the batched prover runs its trees and rounds. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Where { + /// Every tree and every round here: the host reference. + Host, + /// On the card where it takes them — the trees where the factors are, the + /// ladder's layers stepped there, the fused zerocheck rounds — and here + /// where it declines. The same bytes as [`Where::Host`]. + Device, +} + +/// [`multi_prove_batched`] as the host reference, with faults a negative test +/// arms. +#[doc(hidden)] +pub fn prove_batched_with( + committed: &CommittedTables<'_, F, E, H>, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, + faults: ProverFaults, +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + prove_batched_on(committed, config, transcript, prepared, faults, Where::Host) +} + +/// A bin's tree, where it was built. +enum BinTree { + Host(Vec>), + Built(Box>), +} + +/// A bin's tree as its build left it: built, or queued on the card with its +/// output unread (`LAMBDA_VM_ARGUE_ROUND_SLOT`), finalized once every tree of +/// the bin is queued. +enum Pending { + Ready(BinTree), + Queued(multilinear::logup::PrefetchedTree), +} + +/// [`multi_prove_batched`] on the host or the card, with faults. +#[doc(hidden)] +pub fn prove_batched_on( + committed: &CommittedTables<'_, F, E, H>, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, + faults: ProverFaults, + at: Where, +) -> Result, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + let _split = multilinear::whir_split::begin_prove(); + let ArgueFormat::Batched { + bin_log_cells, + pack, + } = config.format.argue + else { + return Err(MlError::ArgueFormatMismatch); + }; + check_settled_prefixes(committed, prepared.as_ref())?; + + let prepared_roots: Vec = prepared + .as_ref() + .map(|p| p.commitment.roots()) + .unwrap_or_default(); + let __sp_challenge = multilinear::whir_split::mark(); + let (z, alpha, beta) = + absorb_roots_and_challenge::(transcript, committed.roots(), &prepared_roots); + multilinear::whir_split::add(&multilinear::whir_split::CHALLENGE, __sp_challenge); + + let tables = committed.tables(); + // The base split's `argue` and `RESERVED HW`'s, as `multi_prove` keeps + // them — the whole batched argue as one window (it has no per-table loop). + let __rw_argue = multilinear::whir_split::open_reserved(); + let __sp_argue = multilinear::whir_split::mark(); + let argued = prove_argue::( + tables, + &z, + &alpha, + &beta, + bin_log_cells, + pack, + transcript, + faults, + at, + ); + let __sp_secs = multilinear::whir_split::add(&multilinear::whir_split::ARGUE, __sp_argue); + multilinear::whir_split::note_table(0, __sp_secs); + multilinear::whir_split::close_reserved(&multilinear::whir_split::RESERVED_ARGUE, __rw_argue); + let (argue, reduced) = argued?; + let mut points: Vec>> = Vec::new(); + let mut values: Vec> = Vec::new(); + let mut table_starts = Vec::with_capacity(tables.len()); + for (table, claim) in tables.iter().zip(reduced) { + table_starts.push(points.len()); + for _ in 0..table.num_committed_columns() { + points.push(claim.point.clone()); + } + values.extend(claim.column_values); + } + + // ── O: the openings, as the per-table format's ──────────────────────── + let (columns, preprocessed) = prove_openings( + committed, + config, + transcript, + prepared, + &points, + &values, + &table_starts, + )?; + + Ok(BatchedMultiProof { + roots: committed.roots().to_vec(), + argue, + columns, + preprocessed, + }) +} + +/// ★ The batched argue itself, from the shared challenges to every table's +/// column claims: phases G, C and Rd of the module docs. The roots block +/// before it and the openings after it are the caller's. +/// +/// `pub` for the in-guest emitter's gate, which argues a few tables with no +/// commitment at all, as the per-table leg's gate calls `prove`. +#[doc(hidden)] +#[allow(clippy::too_many_arguments)] +pub fn prove_argue( + tables: &[CommittedTable<'_, F, E>], + z: &FieldElement, + alpha: &FieldElement, + beta: &FieldElement, + bin_log_cells: u8, + pack: GkrPack, + transcript: &mut T, + faults: ProverFaults, + at: Where, +) -> Result<(BatchedArgue, Vec>), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + let beta = beta.clone(); + let statements: Vec> = tables.iter().map(|t| t.statement()).collect(); + let shapes: Vec = statements.iter().map(ArgueShape::of).collect(); + let plan = argue_plan_packed( + &shapes, + faults.bin_log_cells.unwrap_or(bin_log_cells), + faults.pack.unwrap_or(pack), + ); + let interactions: Vec>> = tables + .iter() + .map(|table| { + multilinear_logup::interactions( + table.layout.interactions, + table.slot_of().len(), + z, + alpha, + |col| slot(table.slot_of(), col), + ) + }) + .collect::>()?; + // Each tree's interactions, table order then chunk order: a whole table's + // own list, or a chunk of it. + let mut lists: Vec]>> = Vec::with_capacity(plan.trees()); + for (t, chunks) in plan.chunks.iter().enumerate() { + let mut table_lists = chunk_lists(&interactions[t], chunks); + if faults.swap_chunks == Some(t) && table_lists.len() > 1 { + table_lists.swap(0, 1); + } + lists.extend(table_lists); + } + + // The fault `balance_with`: the numerator that balances the bus, from every true + // output. + let fake_output: Option<(usize, FieldElement)> = match faults.balance_with { + None => None, + Some(target) => { + let mut outputs = Vec::with_capacity(lists.len()); + for (i, list) in lists.iter().enumerate() { + let t = table_of_tree(&plan, i); + let layers = host_tree(list, &tables[t].trace)?; + outputs.push(gkr_lockstep::LadderTree::output(&gkr_lockstep::HostLayers( + &layers, + ))); + } + let target = plan.first_tree(target); + let mut rest = FieldElement::::zero(); + for (i, (p, q)) in outputs.iter().enumerate() { + if i != target { + rest += p * q.inv().expect("q != 0"); + } + } + Some((target, -(rest * &outputs[target].1))) + } + }; + let mut split = ArgueSplit::default(); + let gruen_layers = multilinear::gkr_gruen::gruen_layers(); + // ── G: a ladder per bin, its outputs absorbed just before it ────────── + let mut bus_outputs: Vec, FieldElement)>> = vec![None; plan.trees()]; + let mut claims: Vec>> = vec![None; tables.len()]; + let mut gkr = Vec::with_capacity(plan.bins.len()); + // Under the round slot the columns' trees of a bin are queued one after + // another and their outputs read once all are (each tree has its stream). + let queue = multilinear::round_slot::argue_round_slot() + != multilinear::round_slot::RoundSlotSetting::Off; + let host_built = |t: usize, list: &[Interaction]| -> Result, MlError> { + let factors = tables[t].trace.factors()?; + Ok(BinTree::Built(Box::new( + multilinear::gkr::FractionTree::build(logup::input_layer(list, &factors)?)?, + ))) + }; + for bin in &plan.bins { + // The bin's trees: each table's chunks, in order. + let members: Vec<(usize, usize)> = bin + .iter() + .flat_map(|&t| { + let first = plan.first_tree(t); + (0..plan.chunks[t].count).map(move |j| (t, first + j)) + }) + .collect(); + let pending: Vec> = members + .iter() + .map(|&(t, i)| -> Result, MlError> { + let (trace, interactions) = (&tables[t].trace, &lists[i][..]); + if at == Where::Host { + return Ok(Pending::Ready(BinTree::Host(host_tree( + interactions, + trace, + )?))); + } + let columns_tree = || { + if queue { + logup::resident_tree_from_columns_deferred(interactions, trace) + .map(Pending::Queued) + } else { + logup::resident_tree_from_columns(interactions, trace) + .map(|tree| Pending::Ready(BinTree::Built(Box::new(tree)))) + } + }; + // Where today's argue builds it: written from the base columns + // with no lift (`LAMBDA_VM_ARGUE_NO_LIFT`), else from the + // lifted factors on the card, else here — and a big host tree + // still folds on a device. The same tree every way. + let started = std::time::Instant::now(); + let on_card = (multilinear::gpu::argue_no_lift() + && trace.device_factors().is_none()) + .then(columns_tree) + .flatten() + .or_else(|| { + let resident = trace.reside_from_columns(); + let from_columns = (multilinear::gpu::argue_gkr_input() && resident.is_some()) + .then(columns_tree) + .flatten(); + from_columns.or_else(|| { + resident + .and_then(|resident| logup::resident_tree(interactions, resident)) + .map(|tree| Pending::Ready(BinTree::Built(Box::new(tree)))) + }) + }); + let tree = match on_card { + Some(tree) => tree, + None => Pending::Ready(host_built(t, interactions)?), + }; + split.trees += started.elapsed().as_secs_f64(); + Ok(tree) + }) + .collect::>()?; + // The queued trees' outputs, now that the bin's every tree is queued; + // one that cannot be read is built here, before anything is absorbed. + let started = std::time::Instant::now(); + let trees: Vec> = pending + .into_iter() + .zip(&members) + .map(|(tree, &(t, i))| match tree { + Pending::Ready(tree) => Ok(tree), + Pending::Queued(queued) => match queued.into_tree() { + Some(tree) => Ok(BinTree::Built(Box::new(tree))), + None => host_built(t, &lists[i]), + }, + }) + .collect::>()?; + split.trees += started.elapsed().as_secs_f64(); + let hosts: Vec>> = trees + .iter() + .map(|tree| match tree { + BinTree::Host(layers) => Some(gkr_lockstep::HostLayers(layers)), + BinTree::Built(_) => None, + }) + .collect(); + let refs: Vec<&dyn gkr_lockstep::LadderTree> = trees + .iter() + .zip(&hosts) + .map(|(tree, host)| match (tree, host) { + (_, Some(host)) => host as &dyn gkr_lockstep::LadderTree, + (BinTree::Built(tree), None) => &**tree as &dyn gkr_lockstep::LadderTree, + (BinTree::Host(_), None) => unreachable!("a host tree has its layers"), + }) + .collect(); + for (&(_, i), tree) in members.iter().zip(&refs) { + let mut output = tree.output(); + if let Some((target, p)) = &fake_output + && *target == i + { + output.0 = p.clone(); + } + transcript.append_field_element(&output.0); + transcript.append_field_element(&output.1); + bus_outputs[i] = Some(output); + } + let started = std::time::Instant::now(); + let (ladder, inputs) = gkr_lockstep::prove_trees(&refs, transcript)?; + split.ladders += started.elapsed().as_secs_f64(); + let inputs: Vec> = inputs; + let mut at_tree = 0; + for &t in bin { + let count = plan.chunks[t].count; + let chunk_claims = &inputs[at_tree..at_tree + count]; + at_tree += count; + claims[t] = Some(table_claim(chunk_claims, &plan.chunks[t], transcript)?); + } + gkr.push(ladder); + } + let bus_outputs: Vec<(FieldElement, FieldElement)> = bus_outputs + .into_iter() + .collect::>() + .ok_or(MlError::ArgueShapeMismatch { part: "bins" })?; + let claims: Vec> = claims + .into_iter() + .collect::>() + .ok_or(MlError::ArgueShapeMismatch { part: "bins" })?; + + // ── C: one front-loaded sumcheck over every table ──────────────────── + let xi: Vec> = (0..plan.num_vars) + .map(|_| transcript.sample_field_element()) + .collect(); + for (t, claim) in claims.iter().enumerate() { + let mut p = claim.p.clone(); + if faults.tamper_claim == Some(t) { + p += FieldElement::::one(); + } + transcript.append_field_element(&p); + transcript.append_field_element(&claim.q); + } + let lambda: FieldElement = transcript.sample_field_element(); + let mut weights = table_weights(&lambda, tables.len()); + if let Some(t) = faults.wrong_weight { + weights[t] = &weights[t] * λ + } + let lambdas = vec![FieldElement::one(), lambda.clone(), &lambda * &lambda]; + let lambda2 = &lambda * λ + let started = std::time::Instant::now(); + // What each table's fused rounds read, held for as long as the rounds do: + // the base columns on the card with no lift (`LAMBDA_VM_ARGUE_NO_LIFT`), the + // lifted factors for a table they cannot read that way (lifted here, as the + // per-table argue lifts it), or nothing, on the host. + let mut constraints: Vec>> = + Vec::with_capacity(tables.len()); + let mut columns: Vec> = + Vec::with_capacity(tables.len()); + for table in tables { + let shape = table.shape(); + let made = (at == Where::Device + && multilinear::gpu_fused::argue_fused() + && multilinear::gpu_fused::fused_takes_width(table.num_committed_columns())) + .then(|| { + multilinear::fused::Constraints::::from_extension( + &shape.steps_as_ops(), + shape.root_steps(), + shape.selector_of_root(), + shape.degree(), + ) + .ok() + }) + .flatten(); + let column = (at == Where::Device + && multilinear::gpu::argue_no_lift() + && table.trace.device_factors().is_none()) + .then(|| made.as_ref().and_then(|_| table.trace.column_factors())) + .flatten(); + if at == Where::Device && made.is_some() && column.is_none() { + let _ = table.trace.reside_from_columns(); + } + constraints.push(made); + columns.push(column); + } + let device: Vec>> = tables + .iter() + .map(|table| match at { + Where::Host => None, + Where::Device => table.trace.device_factors(), + }) + .collect(); + let mut polys = Vec::with_capacity(tables.len()); + for (t, table) in tables.iter().enumerate() { + let n = table.num_vars(); + let (rules, row_point) = + table_rules(&statements[t], &interactions[t], &claims[t].point, &beta)?; + let xi_t = if faults.suffix_xi == Some(t) { + &xi[plan.num_vars - n..] + } else { + &xi[..n] + }; + let betas = multilinear_air::beta_powers(&beta, table.shape().num_roots()); + let fused = constraints[t] + .as_ref() + .map(|constraints| multilinear::gpu_fused::FusedInput { + constraints, + betas: &betas, + interactions: &interactions[t], + claim_point: &claims[t].point, + r: xi_t, + columns: columns[t].as_ref(), + }); + let claim = &lambda * &claims[t].p + &lambda2 * &claims[t].q; + polys.push(front_loaded::TableRounds::new( + rules, + lambdas.clone(), + &claim, + [xi_t, &row_point], + n, + device[t].as_deref(), + columns[t].as_ref(), + fused, + || table.trace.factors(), + )?); + } + split.zc_setup += started.elapsed().as_secs_f64(); + // What an earlier argue on this thread left, so the line below is this one's. + multilinear::gpu_fused::take_tail(); + let started = std::time::Instant::now(); + let (constraint, point) = front_loaded::prove_with( + &mut polys, + &weights, + plan.degree, + transcript, + PaddingFault { + scaled: faults.scaled_padding, + }, + )?; + split.zc_rounds += started.elapsed().as_secs_f64(); + let tail = multilinear::gpu_fused::take_tail(); + let mut factor_values = Vec::with_capacity(tables.len()); + for (table, poly) in tables.iter().zip(&polys) { + let values = table + .kinds() + .iter() + .zip(poly.polys()) + .filter(|(kind, _)| kind.source().is_some()) + .map(|(_, factor)| factor.as_constant().cloned()) + .collect::>>() + .ok_or(MlError::NoVariablesLeft)?; + factor_values.push(values); + } + drop(polys); + drop(device); + drop(columns); + for table in tables { + table.trace.release_device(); + } + for values in &factor_values { + for value in values { + transcript.append_field_element(value); + } + } + + // ── Rd: a shifted table's reduction; the others read their columns ──── + let started = std::time::Instant::now(); + let mut reduces = Vec::with_capacity(tables.len()); + let mut reduced = Vec::with_capacity(tables.len()); + for (t, table) in tables.iter().enumerate() { + let at = &point[..table.num_vars()]; + let (column_point, column_values, reduce) = if plan.shapes[t].shifted { + let sources: Vec = table + .kinds() + .iter() + .filter_map(FactorKind::source) + .collect(); + let (reduce, reduced_point) = claim_reduce::prove::( + &table.trace, + &sources, + &factor_values[t], + at, + table.trace.resident(), + transcript, + )?; + (reduced_point, reduce.column_values.clone(), Some(reduce)) + } else { + let column_values = direct_column_values( + table.kinds(), + &factor_values[t], + table.num_committed_columns(), + )?; + (at.to_vec(), column_values, None) + }; + reduced.push(claim_reduce::ReducedClaim { + point: column_point, + column_values, + }); + reduces.push(reduce); + } + split.reduce += started.elapsed().as_secs_f64(); + if multilinear::whir_split::enabled() { + // The Gruen count is the process's: an argue running beside this one + // adds its layers too. + eprintln!( + "BATCHED ARGUE SPLIT: tables {} · bins {} · trees {:.3}s · ladders {:.3}s · zc setup \ + {:.3}s · zc rounds {:.3}s · reduce {:.3}s · Gruen layers (process) +{}", + tables.len(), + plan.bins.len(), + split.trees, + split.ladders, + split.zc_setup, + split.zc_rounds, + split.reduce, + multilinear::gkr_gruen::gruen_layers() - gruen_layers + ); + // The fused tables' later rounds by program length: the card's, and the + // host's over what the card folded. + eprintln!( + "BATCHED ARGUE ZC TAIL: long tables {} (past the crossover {}) · card rounds {:.3}s · \ + host rounds {} in {:.3}s | short tables {} ({}) · card rounds {:.3}s · host rounds {} in {:.3}s", + tail.tables[1], + tail.on_card[1], + tail.device[1], + tail.host_rounds[1], + tail.host[1], + tail.tables[0], + tail.on_card[0], + tail.device[0], + tail.host_rounds[0], + tail.host[0], + ); + // The inputs of D-BATCH §3.2's error terms, measured on this argue. + eprintln!( + "BATCHED ARGUE BITS INPUTS: tables {} · trees a bin max {} · log2 cells {:.2} · \ + len_max {} · N_C max {} · D_max {} · n_max {}", + tables.len(), + plan.bins.iter().map(Vec::len).max().unwrap_or(0), + plan.shapes + .iter() + .map(|shape| (shape.input_vars as f64).exp2()) + .sum::() + .log2(), + statements + .iter() + .flat_map(|s| s.interactions.iter().map(BusInteraction::num_bus_elements)) + .max() + .unwrap_or(0), + statements + .iter() + .map(|s| s.shape.num_roots()) + .max() + .unwrap_or(0), + plan.degree, + plan.num_vars + ); + // The trees the ladders ran: one a table, or a cut table's chunks. + eprintln!( + "BATCHED ARGUE TREES: tables {} · trees {} · cut {} · tree log2 cells {:.2}", + tables.len(), + plan.trees(), + plan.chunks.iter().filter(|c| c.count > 1).count(), + (0..tables.len()) + .map(|t| plan.chunks[t].count as f64 * (plan.tree_vars(t) as f64).exp2()) + .sum::() + .log2() + ); + } + Ok(( + BatchedArgue { + bus_outputs, + gkr, + constraint, + factor_values, + reduces, + }, + reduced, + )) +} + +/// Where a batched argue's time goes, printed under `LAMBDA_VM_BASE_SPLIT=1`. +#[derive(Default)] +struct ArgueSplit { + trees: f64, + ladders: f64, + zc_setup: f64, + zc_rounds: f64, + reduce: f64, +} + +/// Verifies a batched proof: every table, the bus balance across them, and +/// every column against the one commitment. See [`multi_verify`]. +#[allow(clippy::too_many_arguments)] +pub fn multi_verify_batched( + proof: &BatchedMultiProof, + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + expected: &FieldElement, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + multi_verify_batched_settled::( + proof, statements, layouts, domains, sizes, expected, config, transcript, prepared, false, + ) +} + +/// [`multi_verify_batched`] with `exclude_settled`, as +/// [`multi_verify_settled`](super::multi_verify_settled). +#[allow(clippy::too_many_arguments)] +pub fn multi_verify_batched_settled( + proof: &BatchedMultiProof, + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + expected: &FieldElement, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, + exclude_settled: bool, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + verify_batched_with::( + proof, + statements, + layouts, + domains, + sizes, + expected, + config, + transcript, + prepared, + exclude_settled, + VerifierChecks::ALL, + ) +} + +/// [`multi_verify_batched_settled`], with each new check switchable for the +/// mutation tests. +#[doc(hidden)] +#[allow(clippy::too_many_arguments)] +pub fn verify_batched_with( + proof: &BatchedMultiProof, + statements: &[TableStatement<'_, F, E>], + layouts: &[StackedLayout], + domains: &[Domain], + sizes: &[usize], + expected: &FieldElement, + config: &ChainConfig, + transcript: &mut T, + prepared: Option>, + exclude_settled: bool, + checks: VerifierChecks, +) -> Result<(), MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + H: WhirHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, +{ + // The variant and the bin cap are the verifier's config's, never the + // proof's. + let ArgueFormat::Batched { + bin_log_cells, + pack, + } = config.format.argue + else { + return Err(MlError::ArgueFormatMismatch); + }; + let argue = &proof.argue; + let n = statements.len(); + if argue.factor_values.len() != n || argue.reduces.len() != n { + return Err(MlError::ArgueShapeMismatch { part: "tables" }); + } + if layouts.len() != sizes.len() + || domains.len() != sizes.len() + || proof.columns.len() != sizes.len() + || sizes.iter().sum::() != n + { + return Err(MlError::QueryCountMismatch { + expected: sizes.len(), + got: proof.columns.len(), + }); + } + let (z, alpha, beta) = absorb_roots_and_challenge::( + transcript, + &proof.roots, + prepared.as_ref().map(|p| p.roots).unwrap_or(&[]), + ); + + let reduced_claims = verify_argue::( + argue, + statements, + bin_log_cells, + pack, + &z, + &alpha, + &beta, + transcript, + checks, + )?; + let mut balance = FieldElement::::zero(); + for output in &argue.bus_outputs { + balance += contribution(output).ok_or(MlError::BusImbalance)?; + } + if checks.balance && balance != *expected { + return Err(MlError::BusImbalance); + } + + let mut points: Vec>> = Vec::new(); + let mut values: Vec> = Vec::new(); + let mut table_starts = Vec::with_capacity(n); + let mut settled_counts = Vec::with_capacity(n); + for (t, (statement, reduced)) in statements.iter().zip(reduced_claims).enumerate() { + table_starts.push(points.len()); + let settled = settled_count(prepared.as_ref(), t, statement)?; + settled_counts.push(settled); + if checks.preprocessed { + check_preprocessed(*statement, &reduced, settled)?; + } + for _ in 0..statement.slot_of.len() { + points.push(reduced.point.clone()); + } + values.extend(reduced.column_values); + } + + // ── O ───────────────────────────────────────────────────────────────── + verify_openings::( + &proof.roots, + &proof.columns, + proof.preprocessed.as_ref(), + statements, + layouts, + domains, + sizes, + config, + transcript, + prepared, + exclude_settled, + &points, + &values, + &table_starts, + &settled_counts, + ) +} + +/// ★ The batched argue's verifying half, from the shared challenges to every +/// table's column claims (phases G, C, Rd). The bus balance, the preprocessed +/// checks and the openings are the caller's. +/// +/// `pub` for the in-guest emitter's gate, which checks a leg against it. +#[doc(hidden)] +#[allow(clippy::too_many_arguments)] +pub fn verify_argue( + argue: &BatchedArgue, + statements: &[TableStatement<'_, F, E>], + bin_log_cells: u8, + pack: GkrPack, + z: &FieldElement, + alpha: &FieldElement, + beta: &FieldElement, + transcript: &mut T, + checks: VerifierChecks, +) -> Result>, MlError> +where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + T: crypto::fiat_shamir::is_transcript::IsTranscript, +{ + let n = statements.len(); + let beta = beta.clone(); + let shapes: Vec = statements.iter().map(ArgueShape::of).collect(); + let plan = argue_plan_packed(&shapes, bin_log_cells, pack); + // One bus output a tree: a table's, or each of a cut table's chunks'. + if argue.bus_outputs.len() != plan.trees() + || argue.factor_values.len() != n + || argue.reduces.len() != n + { + return Err(MlError::ArgueShapeMismatch { part: "tables" }); + } + if argue.gkr.len() != plan.bins.len() { + return Err(MlError::ArgueShapeMismatch { part: "bins" }); + } + let interactions: Vec>> = statements + .iter() + .map(|statement| { + multilinear_logup::interactions( + statement.interactions, + statement.slot_of.len(), + z, + alpha, + |col| slot(statement.slot_of, col), + ) + }) + .collect::>()?; + + // ── G ───────────────────────────────────────────────────────────────── + let mut claims: Vec>> = vec![None; n]; + for (bin, ladder) in plan.bins.iter().zip(&argue.gkr) { + let mut outputs = Vec::with_capacity(bin.len()); + let mut heights = Vec::with_capacity(bin.len()); + for &t in bin { + let first = plan.first_tree(t); + for output in &argue.bus_outputs[first..first + plan.chunks[t].count] { + transcript.append_field_element(&output.0); + transcript.append_field_element(&output.1); + outputs.push(output.clone()); + heights.push(plan.tree_vars(t)); + } + } + let inputs = + gkr_lockstep::verify_with(ladder, &outputs, &heights, transcript, checks.residual)?; + let mut at_tree = 0; + for &t in bin { + let count = plan.chunks[t].count; + let chunk_claims = inputs + .get(at_tree..at_tree + count) + .ok_or(MlError::ArgueShapeMismatch { part: "bins" })?; + at_tree += count; + claims[t] = Some(table_claim(chunk_claims, &plan.chunks[t], transcript)?); + } + } + let claims: Vec> = claims + .into_iter() + .collect::>() + .ok_or(MlError::ArgueShapeMismatch { part: "bins" })?; + + // ── C ───────────────────────────────────────────────────────────────── + let xi: Vec> = (0..plan.num_vars) + .map(|_| transcript.sample_field_element()) + .collect(); + for claim in &claims { + transcript.append_field_element(&claim.p); + transcript.append_field_element(&claim.q); + } + let lambda: FieldElement = transcript.sample_field_element(); + let weights = table_weights(&lambda, n); + let lambda2 = &lambda * λ + let claimed = claims + .iter() + .zip(&weights) + .fold(FieldElement::::zero(), |acc, (claim, w)| { + acc + w * (&lambda * &claim.p + &lambda2 * &claim.q) + }); + let sumcheck = sumcheck::verify( + &argue.constraint, + claimed, + plan.num_vars, + plan.degree, + transcript, + )?; + let point = &sumcheck.point; + let mut rebuilt = FieldElement::::zero(); + for (t, statement) in statements.iter().enumerate() { + let num_vars = statement.num_vars; + let at = &point[..num_vars]; + let (rules, row_point) = table_rules(statement, &interactions[t], &claims[t].point, &beta)?; + let mut public = statement.shape.public_values(at)?; + public.push(eq_eval(&xi[..num_vars], at)?); + public.push(eq_eval(&row_point, at)?); + let values = constraint_argument::weave(statement.kinds, &argue.factor_values[t], &public)?; + let value = rules[0].apply(&values) + + &lambda * rules[1].apply(&values) + + &lambda2 * rules[2].apply(&values); + rebuilt += &weights[t] * value * front_loaded::padding_product(point, num_vars); + } + if checks.constraint && rebuilt != sumcheck.expected_evaluation { + return Err(MlError::BatchMismatch); + } + for values in &argue.factor_values { + for value in values { + transcript.append_field_element(value); + } + } + + // ── Rd ──────────────────────────────────────────────────────────────── + let mut reduced_claims = Vec::with_capacity(n); + for (t, statement) in statements.iter().enumerate() { + let at = &point[..statement.num_vars]; + let factor_values = &argue.factor_values[t]; + let reduced = match (plan.shapes[t].shifted, &argue.reduces[t]) { + (true, Some(reduce)) => { + let sources: Vec = statement + .kinds + .iter() + .filter_map(FactorKind::source) + .collect(); + claim_reduce::verify( + reduce, + &sources, + factor_values, + at, + statement.slot_of.len(), + transcript, + )? + } + (false, None) => claim_reduce::ReducedClaim { + point: at.to_vec(), + column_values: direct_column_values( + statement.kinds, + factor_values, + statement.slot_of.len(), + )?, + }, + (false, Some(_)) if !checks.reduce_shape => claim_reduce::ReducedClaim { + point: at.to_vec(), + column_values: direct_column_values( + statement.kinds, + factor_values, + statement.slot_of.len(), + )?, + }, + _ => return Err(MlError::ArgueShapeMismatch { part: "reduction" }), + }; + reduced_claims.push(reduced); + } + Ok(reduced_claims) +} + +#[cfg(test)] +mod tests; diff --git a/crypto/stark/src/multilinear_table/batched/tests.rs b/crypto/stark/src/multilinear_table/batched/tests.rs new file mode 100644 index 000000000..b7ec31431 --- /dev/null +++ b/crypto/stark/src/multilinear_table/batched/tests.rs @@ -0,0 +1,1366 @@ +//! The batched argue (D-BATCH B-2) on the repo's multi-table example: round +//! trips across heights and bins, every negative D-BATCH §7 names, and the +//! mutation tests that show each new verifier check is load-bearing. + +use super::*; +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; +use multilinear::whir_chain::{ChainFormat, GrindBits}; + +use crate::constraints::builder::{ConstraintBuilder, ConstraintSet, EmptyConstraints, RowDomain}; +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use crate::proof::options::ProofOptions; +use crate::traits::AIR; + +type FE = FieldElement; +type ExtE = FieldElement; +type Air = AirWithBuses; +type Proof = BatchedMultiProof; + +const SEED: &[u8] = b"batched-argue"; + +mod pack; + +fn config(argue: ArgueFormat) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: ChainFormat { + argue, + ..ChainFormat::DEFAULT + }, + } +} + +fn batched(bin_log_cells: u8) -> ChainConfig { + config(ArgueFormat::batched(bin_log_cells)) +} + +/// The ADD table's own constraint: every row really is an addition. +struct AddConstraints; + +impl ConstraintSet for AddConstraints { + fn max_degree(&self) -> usize { + 1 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, 0); + let addend = b.main(0, 1); + let sum = b.main(0, 2); + b.emit_base(0, a + addend - sum); + } +} + +/// The MUL table's, one degree higher. +struct MulConstraints; + +impl ConstraintSet for MulConstraints { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, 0); + let factor = b.main(0, 1); + let product = b.main(0, 2); + b.emit_base(0, a * factor - product); + } +} + +/// The MUL table with a transition too — its multiplicity is the same on +/// every row but the last — so a factor reads the NEXT row: a shifted table, +/// which keeps its claim reduction. +struct ShiftedMulConstraints; + +impl ConstraintSet for ShiftedMulConstraints { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, 0); + let factor = b.main(0, 1); + let product = b.main(0, 2); + b.emit_base(0, a * factor - product); + let mult = b.main(0, 3); + let next = b.main(1, 3); + b.emit_base_rows(1, RowDomain::except_last(1), next - mult); + } +} + +struct Airs { + cpu: Air, + add: Air, + mul: Air, + shifted: Air, +} + +fn airs() -> Airs { + let options = ProofOptions::default_test_options(); + let with = |interactions: &[BusInteraction]| AuxiliaryTraceBuildData { + interactions: interactions.to_vec(), + }; + let add_bus = new_add_air_with_lookup(&options); + let mul_bus = new_mul_air_with_lookup(&options); + Airs { + cpu: new_cpu_air_with_lookup(&options), + add: AirWithBuses::new( + 4, + with(add_bus.bus_interactions()), + &options, + 1, + AddConstraints, + ), + mul: AirWithBuses::new( + 4, + with(mul_bus.bus_interactions()), + &options, + 1, + MulConstraints, + ), + shifted: AirWithBuses::new( + 4, + with(mul_bus.bus_interactions()), + &options, + 1, + ShiftedMulConstraints, + ), + } +} + +/// CPU `2^(a+1)` rows (half add, half multiply), ADD `2^(a+pad)` (its adds, +/// then zero rows that add up and receive nothing), MUL `2^a`. +fn columns(a: usize, pad: usize) -> [Vec>; 3] { + let count = 1u64 << a; + let mut cpu = vec![Vec::new(); 5]; + let mut add = vec![Vec::new(); 4]; + let mut mul = vec![Vec::new(); 4]; + let push = |table: &mut Vec>, row: &[FE]| { + for (column, value) in table.iter_mut().zip(row) { + column.push(*value); + } + }; + for i in 0..count { + let (x, y) = (FE::from(i + 1), FE::from(3 * i + 7)); + push(&mut cpu, &[FE::one(), FE::zero(), x, y, x + y]); + push(&mut add, &[x, y, x + y, FE::one()]); + } + for _ in count..(count << pad) { + push(&mut add, &[FE::zero(); 4]); + } + for i in 0..count { + let (x, y) = (FE::from(i + 2), FE::from(5 * i + 3)); + push(&mut cpu, &[FE::zero(), FE::one(), x, y, x * y]); + push(&mut mul, &[x, y, x * y, FE::one()]); + } + [cpu, add, mul] +} + +fn table<'a, CS: ConstraintSet>( + air: &'a Air, + columns: &[Vec], +) -> CommittedTable<'a, Fp, Ext> { + let num_vars = columns[0].len().trailing_zeros() as usize; + let owned = columns.to_vec(); + CommittedTable::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + columns.len(), + num_vars, + Uniforms::default(), + |col| owned[col as usize].clone(), + ) + .expect("a table") +} + +/// CPU, ADD and MUL (or the shifted MUL) over `columns(a, pad)`. +fn tables<'a>( + airs: &'a Airs, + cols: &[Vec>; 3], + shifted: bool, +) -> Vec> { + vec![ + table(&airs.cpu, &cols[0]), + table(&airs.add, &cols[1]), + if shifted { + table(&airs.shifted, &cols[2]) + } else { + table(&airs.mul, &cols[2]) + }, + ] +} + +fn commit<'a>( + tables: Vec>, + cfg: &ChainConfig, +) -> CommittedTables<'a, Fp, Ext, KeccakWhir> { + CommittedTables::commit(tables, cfg).expect("committed") +} + +fn prove( + committed: &CommittedTables<'_, Fp, Ext, KeccakWhir>, + cfg: &ChainConfig, + faults: ProverFaults, +) -> Proof { + let mut transcript = DefaultTranscript::::new(SEED); + prove_batched_with(committed, cfg, &mut transcript, None, faults).expect("the prover runs") +} + +fn verify_under( + committed: &CommittedTables<'_, Fp, Ext, KeccakWhir>, + statements: &[TableStatement<'_, Fp, Ext>], + proof: &Proof, + cfg: &ChainConfig, + checks: VerifierChecks, +) -> Result<(), MlError> { + let mut transcript = DefaultTranscript::::new(SEED); + verify_batched_with::<_, _, _, KeccakWhir>( + proof, + statements, + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &ExtE::zero(), + cfg, + &mut transcript, + None, + false, + checks, + ) +} + +fn verify( + committed: &CommittedTables<'_, Fp, Ext, KeccakWhir>, + proof: &Proof, + cfg: &ChainConfig, +) -> Result<(), MlError> { + let statements: Vec<_> = committed.tables().iter().map(|t| t.statement()).collect(); + verify_under(committed, &statements, proof, cfg, VerifierChecks::ALL) +} + +fn verify_with_checks( + committed: &CommittedTables<'_, Fp, Ext, KeccakWhir>, + proof: &Proof, + cfg: &ChainConfig, + checks: VerifierChecks, +) -> Result<(), MlError> { + let statements: Vec<_> = committed.tables().iter().map(|t| t.statement()).collect(); + verify_under(committed, &statements, proof, cfg, checks) +} + +/// A proof of `columns(a, pad)` under `cap`, and its verdict. +fn round_trip(a: usize, pad: usize, cap: u8, shifted: bool) -> Result<(), MlError> { + let airs = airs(); + let cfg = batched(cap); + let committed = commit(tables(&airs, &columns(a, pad), shifted), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + verify(&committed, &proof, &cfg) +} + +fn shape(num_vars: usize, input_vars: usize) -> ArgueShape { + ArgueShape { + num_vars, + input_vars, + degree: 2, + shifted: false, + interactions: 1 << (input_vars - num_vars), + } +} + +// ─── the plan ───────────────────────────────────────────────────────────── + +/// First-fit-decreasing by `k`, ties by index; a table at or over the cap +/// alone; bins in the order they open, each in table order. +#[test] +fn the_plan_packs_first_fit_decreasing() { + let shapes = [ + shape(3, 4), // 16 cells + shape(5, 6), // 64 + shape(1, 2), // 4 + shape(4, 5), // 32 + shape(2, 3), // 8 + shape(8, 10), // over the cap + shape(3, 5), // 32 + ]; + let plan = argue_plan(&shapes, 6); // cap 64 + // Order: 5 (alone), 1 (64 fills a bin), 3 (32), 6 (32 joins 3), 0 (16), + // 4 (8 joins 0), 2 (4 joins 0). + assert_eq!(plan.bins, vec![vec![5], vec![1], vec![3, 6], vec![0, 2, 4]]); + assert_eq!(plan.num_vars, 8); + assert_eq!(plan.degree, 2); + // A roomy cap puts every table in one bin. + assert_eq!( + argue_plan(&shapes, 27).bins, + vec![vec![0, 1, 2, 3, 4, 5, 6]] + ); + // A cap of one cell: every table alone, largest first. + assert_eq!( + argue_plan(&shapes, 0).bins, + vec![ + vec![5], + vec![1], + vec![3], + vec![6], + vec![0], + vec![4], + vec![2] + ] + ); +} + +/// The example's shapes, read off the statements: CPU `k = n + 1` (two +/// interactions), ADD and MUL `k = n` (one). +#[test] +fn the_shapes_come_from_the_statement() { + let airs = airs(); + let committed = commit(tables(&airs, &columns(3, 1), true), &batched(27)); + let shapes: Vec = committed + .tables() + .iter() + .map(|t| ArgueShape::of(&t.statement())) + .collect(); + assert_eq!( + shapes, + vec![ + ArgueShape { + num_vars: 4, + input_vars: 5, + degree: 2, + shifted: false, + interactions: 2 + }, + ArgueShape { + num_vars: 4, + input_vars: 4, + degree: 2, + shifted: false, + interactions: 1 + }, + ArgueShape { + num_vars: 3, + input_vars: 3, + degree: 3, + shifted: true, + interactions: 1 + }, + ] + ); +} + +// ─── round trips ───────────────────────────────────────────────────────── + +/// ★ Three tables of different heights, virtually padded to the tallest, in +/// one bin, in bins of one or two, and with every table alone. +#[test] +fn tables_of_different_heights_round_trip_across_bins() { + for (a, pad) in [(2, 0), (3, 1), (3, 2), (1, 3)] { + for cap in [27u8, 5, 4, 3, 2, 0] { + round_trip(a, pad, cap, false) + .unwrap_or_else(|e| panic!("a={a} pad={pad} cap={cap}: {e:?}")); + } + } +} + +/// One-row tables (`n_T = 0`: ADD and MUL at `a = 0`), a rootless one (CPU), +/// and a table alone in its bin. +#[test] +fn edge_shapes_round_trip() { + round_trip(0, 0, 27, false).expect("two one-row tables and a two-row one"); + round_trip(0, 0, 0, false).expect("each alone"); + round_trip(0, 3, 27, false).expect("a one-row MUL beside taller tables"); +} + +/// A table with a shifted read keeps its claim reduction, at `r_{(airs: &'a Airs) -> (CommittedTables<'a, Fp, Ext, KeccakWhir>, Proof, ChainConfig) { + let cfg = batched(27); + let committed = commit(tables(airs, &columns(3, 1), false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + verify(&committed, &proof, &cfg).expect("the untampered proof verifies"); + (committed, proof, cfg) +} + +fn tampered(tamper: impl FnOnce(&mut Proof)) -> Result<(), MlError> { + let airs = airs(); + let (committed, mut proof, cfg) = fixture(&airs); + tamper(&mut proof); + verify(&committed, &proof, &cfg) +} + +#[test] +fn a_wrong_bus_output_is_refused() { + for t in 0..3 { + assert!( + tampered(|p| p.argue.bus_outputs[t].0 += ExtE::one()).is_err(), + "table {t}'s numerator" + ); + assert!( + tampered(|p| p.argue.bus_outputs[t].1 += ExtE::one()).is_err(), + "table {t}'s denominator" + ); + } +} + +#[test] +fn a_missing_bus_output_is_refused() { + assert_eq!( + tampered(|p| { + p.argue.bus_outputs.pop(); + }), + Err(MlError::ArgueShapeMismatch { part: "tables" }) + ); +} + +/// One tree's halves wrong at a mid step of the three-tree bin: that step's +/// residual refuses it, and names the step. +#[test] +fn one_trees_halves_wrong_at_a_mid_step_are_refused() { + for tree in 0..3 { + assert_eq!( + tampered(|p| p.argue.gkr[0].layers[2].halves[tree].p_hi += ExtE::one()), + Err(MlError::LayerRelationMismatch { layer: 2 }), + "tree {tree}" + ); + } + assert_eq!( + tampered(|p| { + p.argue.gkr[0].layers[2].halves.pop(); + }), + Err(MlError::ArgueShapeMismatch { + part: "ladder step" + }) + ); +} + +#[test] +fn a_wrong_round_value_is_refused() { + assert_eq!( + tampered(|p| p.argue.gkr[0].layers[3].sumcheck.rounds[1].evaluations[0] += ExtE::one()), + Err(MlError::LayerRelationMismatch { layer: 3 }), + "a GKR round" + ); + assert_eq!( + tampered(|p| p.argue.constraint.rounds[1].evaluations[2] += ExtE::one()), + Err(MlError::BatchMismatch), + "a constraint round" + ); +} + +#[test] +fn one_tables_factor_value_wrong_is_refused() { + for t in 0..3 { + assert_eq!( + tampered(|p| p.argue.factor_values[t][1] += ExtE::one()), + Err(MlError::BatchMismatch), + "table {t}" + ); + } + assert_eq!( + tampered(|p| { + p.argue.factor_values[1].pop(); + }), + Err(MlError::QueryCountMismatch { + expected: 4, + got: 3 + }), + "a factor value short" + ); +} + +/// ADD and MUL have the same shape; their messages swapped are refused. +#[test] +fn swapped_tables_are_refused() { + let airs = airs(); + let cfg = batched(27); + // Same heights, so the two are interchangeable in shape. + let committed = commit(tables(&airs, &columns(3, 0), false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + verify(&committed, &proof, &cfg).expect("honest"); + let mut outputs = proof.clone(); + outputs.argue.bus_outputs.swap(1, 2); + assert!(verify(&committed, &outputs, &cfg).is_err(), "bus outputs"); + let mut values = proof.clone(); + values.argue.factor_values.swap(1, 2); + assert!(verify(&committed, &values, &cfg).is_err(), "factor values"); + // And a verifier handed the statements in another order. + let mut statements: Vec<_> = committed.tables().iter().map(|t| t.statement()).collect(); + statements.swap(1, 2); + assert!( + verify_under(&committed, &statements, &proof, &cfg, VerifierChecks::ALL).is_err(), + "statements" + ); +} + +/// A reduction present for an unshifted table, or absent for a shifted one. +#[test] +fn a_reduction_where_the_format_has_none_is_refused() { + let airs = airs(); + let cfg = batched(27); + let committed = commit(tables(&airs, &columns(3, 1), true), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + let reduce = proof.argue.reduces[2].clone().expect("the shifted table's"); + let mut extra = proof.clone(); + extra.argue.reduces[1] = Some(reduce); + assert_eq!( + verify(&committed, &extra, &cfg), + Err(MlError::ArgueShapeMismatch { part: "reduction" }) + ); + let mut missing = proof.clone(); + missing.argue.reduces[2] = None; + assert_eq!( + verify(&committed, &missing, &cfg), + Err(MlError::ArgueShapeMismatch { part: "reduction" }) + ); +} + +// ─── negatives: provers that break the format ──────────────────────────── + +fn faulted(faults: ProverFaults) -> Result<(), MlError> { + let airs = airs(); + let cfg = batched(27); + let committed = commit(tables(&airs, &columns(3, 1), false), &cfg); + let proof = prove(&committed, &cfg, faults); + verify(&committed, &proof, &cfg) +} + +/// A prover padding a short table the `2^Δ` way instead of by `Π X_j`. +#[test] +fn a_table_padded_by_scaling_is_refused() { + // MUL (n = 3) is the one shorter than `n_max = 4`. + assert_eq!( + faulted(ProverFaults { + scaled_padding: Some(2), + ..Default::default() + }), + Err(MlError::BatchMismatch) + ); +} + +/// A table's zerocheck weighted by the suffix of `ξ`, not its prefix. +#[test] +fn a_suffix_of_xi_is_refused() { + assert_eq!( + faulted(ProverFaults { + suffix_xi: Some(2), + ..Default::default() + }), + Err(MlError::BatchMismatch) + ); +} + +/// A wrong cross-table weight, and a table claim absorbed wrong. +#[test] +fn a_wrong_weight_or_claim_is_refused() { + for t in 0..3 { + assert_eq!( + faulted(ProverFaults { + wrong_weight: Some(t), + ..Default::default() + }), + Err(MlError::BatchMismatch), + "weight of table {t}" + ); + assert_eq!( + faulted(ProverFaults { + tamper_claim: Some(t), + ..Default::default() + }), + Err(MlError::BatchMismatch), + "claim of table {t}" + ); + } +} + +/// A prover that bins under another cap argues a different partition. +#[test] +fn a_different_bin_partition_is_refused() { + for cap in [4u8, 0] { + assert!( + faulted(ProverFaults { + bin_log_cells: Some(cap), + ..Default::default() + }) + .is_err(), + "cap {cap}" + ); + } +} + +/// A trace breaking one constraint of a small table, its bus still balanced: +/// only the constraint sumcheck's final equation can see it. +#[test] +fn a_violated_constraint_inside_a_bin_is_refused() { + let airs = airs(); + let cfg = batched(27); + let mut cols = columns(3, 1); + // Row 2 of ADD, and the CPU's send of it, both claim a wrong sum (ADD's + // `c` is column 2, the CPU's column 4; the columns are column-major). + cols[1][2][2] += FE::one(); + cols[0][4][2] += FE::one(); + let committed = commit(tables(&airs, &cols, false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + assert_eq!( + verify(&committed, &proof, &cfg), + Err(MlError::BatchMismatch) + ); +} + +/// One multiplicity off: the bus no longer balances. +#[test] +fn one_bus_multiplicity_off_is_refused() { + let airs = airs(); + let cfg = batched(27); + let mut cols = columns(3, 1); + cols[2][3][1] += FE::one(); + let committed = commit(tables(&airs, &cols, false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + assert_eq!(verify(&committed, &proof, &cfg), Err(MlError::BusImbalance)); +} + +/// A verifier holding a preprocessed column the proof did not commit. +#[test] +fn a_wrong_preprocessed_column_is_refused() { + let airs = airs(); + let cfg = batched(27); + let cols = columns(3, 1); + let committed = commit(tables(&airs, &cols, false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + let right = vec![Mle::new(cols[1][0].clone()).unwrap()]; + let mut bad = cols[1][0].clone(); + bad[5] += FE::one(); + let wrong = vec![Mle::new(bad).unwrap()]; + let with = |pre: &[Mle]| -> Result<(), MlError> { + let tables = committed.tables(); + let statements = vec![ + tables[0].statement(), + tables[1].layout().statement_with_preprocessed(pre), + tables[2].statement(), + ]; + verify_under(&committed, &statements, &proof, &cfg, VerifierChecks::ALL) + }; + with(&right).expect("the right copy agrees"); + assert_eq!(with(&wrong), Err(MlError::EvaluationMismatch)); +} + +// ─── the format is the verifier's ──────────────────────────────────────── + +/// Neither format's prover nor verifier takes the other's config. +#[test] +fn a_proof_under_the_other_format_is_refused() { + let airs = airs(); + let cols = columns(2, 1); + let per_table = config(ArgueFormat::PerTable); + let batch = batched(27); + + let committed = commit(tables(&airs, &cols, false), &per_table); + let mut t = DefaultTranscript::::new(SEED); + assert_eq!( + multi_prove_batched(&committed, &per_table, &mut t, None).unwrap_err(), + MlError::ArgueFormatMismatch, + "the batched prover under a per-table config" + ); + let mut t = DefaultTranscript::::new(SEED); + assert_eq!( + multi_prove(&committed, &batch, &mut t, None).unwrap_err(), + MlError::ArgueFormatMismatch, + "the per-table prover under a batched config" + ); + + // A per-table proof, under a verifier configured for the batched argue. + let mut t = DefaultTranscript::::new(SEED); + let today = multi_prove(&committed, &per_table, &mut t, None).unwrap(); + let statements: Vec<_> = committed.tables().iter().map(|t| t.statement()).collect(); + let run_today = |cfg: &ChainConfig| { + let mut t = DefaultTranscript::::new(SEED); + multi_verify::<_, _, _, KeccakWhir>( + &today, + &statements, + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &ExtE::zero(), + cfg, + &mut t, + None, + ) + }; + run_today(&per_table).expect("today's proof verifies today"); + assert_eq!(run_today(&batch), Err(MlError::ArgueFormatMismatch)); + + // A batched proof, under a verifier configured for today's argue. + let committed = commit(tables(&airs, &cols, false), &batch); + let proof = prove(&committed, &batch, ProverFaults::default()); + verify(&committed, &proof, &batch).expect("verifies batched"); + assert_eq!( + verify(&committed, &proof, &per_table), + Err(MlError::ArgueFormatMismatch) + ); +} + +/// ★ Default-off: today's format is `PerTable`, and the per-table proof of +/// the example is what it was — the batched module moved no byte of it. +#[test] +fn the_default_format_is_per_table() { + assert_eq!(ChainFormat::DEFAULT.argue, ArgueFormat::PerTable); + assert!(ChainFormat::DEFAULT.is_default()); + assert!( + !ChainFormat { + argue: ArgueFormat::BATCHED, + ..ChainFormat::DEFAULT + } + .is_default() + ); + assert_eq!(ArgueFormat::default(), ArgueFormat::PerTable); +} + +// ─── mutations: each new check made inert lets its negative through ────── + +fn inert(checks: VerifierChecks, tamper: impl FnOnce(&mut Proof)) -> Result<(), MlError> { + let airs = airs(); + let (committed, mut proof, cfg) = fixture(&airs); + tamper(&mut proof); + verify_with_checks(&committed, &proof, &cfg, checks) +} + +#[test] +fn the_ladder_residual_is_load_bearing() { + let checks = VerifierChecks { + residual: false, + ..VerifierChecks::ALL + }; + assert_ne!( + inert(checks, |p| p.argue.gkr[0].layers[2].halves[1].p_hi += + ExtE::one()), + Err(MlError::LayerRelationMismatch { layer: 2 }), + "made inert, the step no longer refuses its own tamper" + ); +} + +#[test] +fn the_constraint_equation_is_load_bearing() { + let airs = airs(); + let cfg = batched(27); + let mut cols = columns(3, 1); + cols[1][2][2] += FE::one(); + cols[0][4][2] += FE::one(); + let committed = commit(tables(&airs, &cols, false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + let checks = VerifierChecks { + constraint: false, + ..VerifierChecks::ALL + }; + assert_eq!( + verify_with_checks(&committed, &proof, &cfg, checks), + Ok(()), + "made inert, a violated constraint is accepted" + ); +} + +#[test] +fn the_bus_balance_is_load_bearing() { + let airs = airs(); + let cfg = batched(27); + let mut cols = columns(3, 1); + cols[2][3][1] += FE::one(); + let committed = commit(tables(&airs, &cols, false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + let checks = VerifierChecks { + balance: false, + ..VerifierChecks::ALL + }; + assert_eq!( + verify_with_checks(&committed, &proof, &cfg, checks), + Ok(()), + "made inert, an unbalanced bus is accepted" + ); +} + +#[test] +fn the_reduction_shape_is_load_bearing() { + let airs = airs(); + let cfg = batched(27); + let committed = commit(tables(&airs, &columns(3, 1), true), &cfg); + let mut proof = prove(&committed, &cfg, ProverFaults::default()); + proof.argue.reduces[1] = proof.argue.reduces[2].clone(); + let checks = VerifierChecks { + reduce_shape: false, + ..VerifierChecks::ALL + }; + assert_eq!( + verify_with_checks(&committed, &proof, &cfg, checks), + Ok(()), + "made inert, a reduction the format has no place for is accepted" + ); +} + +#[test] +fn the_preprocessed_check_is_load_bearing() { + let airs = airs(); + let cfg = batched(27); + let cols = columns(3, 1); + let committed = commit(tables(&airs, &cols, false), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + let mut bad = cols[1][0].clone(); + bad[5] += FE::one(); + let wrong = vec![Mle::new(bad).unwrap()]; + let tables = committed.tables(); + let statements = vec![ + tables[0].statement(), + tables[1].layout().statement_with_preprocessed(&wrong), + tables[2].statement(), + ]; + let checks = VerifierChecks { + preprocessed: false, + ..VerifierChecks::ALL + }; + assert_eq!( + verify_under(&committed, &statements, &proof, &cfg, checks), + Ok(()), + "made inert, a wrong preprocessed column is accepted" + ); +} + +// ─── B-3: the card proves the host reference's bytes ───────────────────── + +/// Whether a device promises any room (the parent module's probe). +fn a_device() -> bool { + multilinear::gpu::reserve_budget() > 0 +} + +/// Whether this process holds a card, by math-cuda's own probe — whatever +/// `multilinear` was built with, unlike [`a_device`], which is `false` in a +/// build whose `multilinear` has no `cuda`. +fn a_card() -> bool { + #[cfg(feature = "cuda")] + { + math_cuda::device::backend().is_ok() + } + #[cfg(not(feature = "cuda"))] + { + false + } +} + +/// A card in the process that the argue's device paths cannot reach — a stark +/// build without `multilinear/cuda` — would pass every card test's byte checks +/// on the host's fallbacks: each card test refuses it here, not a read-out. +fn a_present_card_is_reachable() { + if a_card() { + assert!( + a_device(), + "a card is present but the batched argue's device paths are not built \ + (stark's `cuda` does not enable `multilinear/cuda`): it would never be used" + ); + } +} + +/// Held by every test here that sets a process-wide argue switch +/// (`force_argue_*`) or asserts on a process-wide device counter, so no two of +/// them overlap: one test's forced switch moves the other's rounds. (The +/// round slot's On arm, run while another test had Gruen's rounds off, put its +/// GKR rounds on the generic layer path — none through a slot — while both +/// tests' launches landed in the one counter.) +fn card_switches() -> std::sync::MutexGuard<'static, ()> { + static SWITCHES: std::sync::Mutex<()> = std::sync::Mutex::new(()); + SWITCHES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// ★ D-BATCH B-3's gate: the prover on the card ([`Where::Device`]) proves the +/// host reference's canonical bytes — every tree, ladder step, round and value +/// — with the transcript in the same state after, and the proof verifies. +/// +/// The tables are tall enough for the card to take them: CPU 2^13 rows (a tree +/// of 2^14 input cells, the device's tree threshold), ADD 2^13, MUL 2^12, all +/// past the fused rounds' 2^7. On a device the fused sessions and the ladder's +/// device rounds are counted, so the comparison is not of the host with +/// itself; without one it is the host reference against the device path's +/// host fallbacks, and says so. +/// +/// ```text +/// cargo test --release -p stark --features cuda,multilinear/cuda --lib -- \ +/// multilinear_table::batched::tests::the_card_proves_the_host_references_bytes --exact --nocapture +/// ``` +#[test] +fn the_card_proves_the_host_references_bytes() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let _switches = card_switches(); + a_present_card_is_reachable(); + let airs = airs(); + for (a, pad, cap) in [(12usize, 1usize, 27u8), (12, 1, 14), (3, 1, 27)] { + let cfg = batched(cap); + let cols = columns(a, pad); + let run = |at: Where| { + let committed = commit(tables(&airs, &cols, false), &cfg); + let fused = multilinear::gpu_fused::fused_sessions(); + let rounds = multilinear::gpu::sumcheck_rounds(); + let gruen = multilinear::gkr_gruen::gruen_layers(); + let mut transcript = DefaultTranscript::::new(SEED); + let proof = prove_batched_on( + &committed, + &cfg, + &mut transcript, + None, + ProverFaults::default(), + at, + ) + .unwrap_or_else(|e| panic!("{at:?}: {e:?}")); + verify(&committed, &proof, &cfg).unwrap_or_else(|e| panic!("{at:?}: {e:?}")); + let fused = multilinear::gpu_fused::fused_sessions() - fused; + let rounds = multilinear::gpu::sumcheck_rounds() - rounds; + let gruen = multilinear::gkr_gruen::gruen_layers() - gruen; + ( + bincode::serialize(&proof).expect("bytes"), + transcript.state(), + transcript.sample_field_element(), + fused, + rounds, + gruen, + ) + }; + let host = run(Where::Host); + let card = run(Where::Device); + // The ladder on today's generic layer rounds, not Gruen's: the same + // bytes again. (Its Gruen count is not asserted: the switch and the + // counter are the process's, and a test beside this one moves both.) + multilinear::gkr_gruen::force_argue_gkr_gruen(Some(false)); + let generic = run(Where::Device); + multilinear::gkr_gruen::force_argue_gkr_gruen(None); + assert!( + host.0 == generic.0, + "a={a} cap={cap}: the generic ladder's bytes differ" + ); + // A refusal on the card is the host's rounds, never another proof: the + // fused rounds switched off leave the ladder on the card and the + // constraint rounds here. + multilinear::gpu_fused::force_argue_fused(Some(false)); + let declined = run(Where::Device); + multilinear::gpu_fused::force_argue_fused(None); + assert_eq!(declined.3, 0, "fused rounds off: no fused session"); + assert!( + host.0 == declined.0, + "a={a} cap={cap}: the declined card's bytes differ" + ); + assert_eq!(host.3, 0, "the host reference ran no fused session"); + eprintln!( + "batched argue B-3 a={a} pad={pad} cap={cap}: device {} fused sessions, {} device rounds, {} \ + Gruen ladder layers; {} bytes", + card.3, + card.4, + card.5, + card.0.len() + ); + if a_device() && a >= 7 { + assert!(card.3 > 0, "a={a}: the card ran fused sessions"); + assert!(card.4 > 0, "a={a}: the card ran rounds"); + assert!(card.5 > 0, "a={a}: the ladder ran Gruen's rounds"); + } + assert!( + host.0 == card.0, + "a={a} cap={cap}: the canonical bytes differ" + ); + assert_eq!(host.1, card.1, "a={a} cap={cap}: the transcripts parted"); + assert_eq!(host.2, card.2, "a={a} cap={cap}: the next challenge moved"); + } +} + +/// ★ The batched argue on the card with the rounds' sums through a slot +/// (`LAMBDA_VM_ARGUE_ROUND_SLOT`: the GKR ladder's rounds, and a bin's trees +/// queued before their outputs are read) proves the host reference's bytes, +/// with the transcript in the same state after; under `check` every round is +/// compared with today's and none differs. Each GKR round through a slot is +/// one second launch fewer than today's on the card; the fused zerocheck's +/// rounds keep theirs. +/// +/// ```text +/// cargo test --release -p stark --features cuda,multilinear/cuda --lib -- \ +/// multilinear_table::batched::tests::the_round_slot_proves_the_host_references_bytes --exact --nocapture +/// ``` +#[test] +fn the_round_slot_proves_the_host_references_bytes() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + use multilinear::round_slot::{self as slot, RoundSlotSetting}; + + let _switches = card_switches(); + a_present_card_is_reachable(); + let airs = airs(); + let cfg = batched(27); + let cols = columns(12, 1); + let run = |at: Where, setting: RoundSlotSetting| { + slot::force_argue_round_slot(Some(setting)); + let committed = commit(tables(&airs, &cols, false), &cfg); + let mut transcript = DefaultTranscript::::new(SEED); + let proof = prove_batched_on( + &committed, + &cfg, + &mut transcript, + None, + ProverFaults::default(), + at, + ) + .unwrap_or_else(|e| panic!("{at:?} {setting:?}: {e:?}")); + verify(&committed, &proof, &cfg).unwrap_or_else(|e| panic!("{at:?} {setting:?}: {e:?}")); + ( + bincode::serialize(&proof).expect("bytes"), + transcript.state(), + transcript.sample_field_element(), + ) + }; + let host = run(Where::Host, RoundSlotSetting::Off); + // Today's on the card: every round's sums by a second launch. + let before = slot_counts(); + let today = run(Where::Device, RoundSlotSetting::Off); + let after = slot_counts(); + slot::force_argue_round_slot(None); + assert!(host.0 == today.0, "Off: the canonical bytes differ"); + assert_eq!(host.1, today.1, "Off: the transcripts parted"); + let today_seconds = after[3] - before[3]; + assert_eq!(after[0], before[0], "off: no round through a slot"); + for setting in [RoundSlotSetting::On, RoundSlotSetting::Check] { + let before = slot_counts(); + let card = run(Where::Device, setting); + let after = slot_counts(); + slot::force_argue_round_slot(None); + assert!(host.0 == card.0, "{setting:?}: the canonical bytes differ"); + assert_eq!(host.1, card.1, "{setting:?}: the transcripts parted"); + assert_eq!(host.2, card.2, "{setting:?}: the next challenge moved"); + let rounds = after[0] - before[0]; + let seconds = after[3] - before[3]; + eprintln!( + "batched argue, round slot {setting:?}: {rounds} rounds through slots, {seconds} second \ + launches ({today_seconds} today), {} compared, {} differing; {} bytes", + after[4] - before[4], + after[5] - before[5], + card.0.len() + ); + if a_device() { + assert!( + rounds > 0, + "{setting:?}: the card's rounds went through slots" + ); + assert_eq!(after[2], before[2], "{setting:?}: no slot declined"); + if setting == RoundSlotSetting::On { + assert_eq!( + seconds + rounds, + today_seconds, + "on: each GKR round through a slot is one second launch fewer" + ); + assert!( + seconds > 0, + "on: the fused zerocheck kept today's second launch" + ); + } else { + assert_eq!( + seconds, today_seconds, + "check: today's second launch beside every round" + ); + assert_eq!(after[4] - before[4], rounds, "check: every round compared"); + assert_eq!(after[5], before[5], "check: none differed"); + } + } + } +} + +/// The round slots' counters: rounds, fall-backs, declines, second launches, +/// rounds compared, differing. +fn slot_counts() -> [u64; 6] { + #[cfg(feature = "cuda")] + { + let c = math_cuda::round_slot::counts(); + [ + c.slot_rounds, + c.fall_backs, + c.declines, + c.second_launches, + c.checked, + c.mismatches, + ] + } + #[cfg(not(feature = "cuda"))] + [0; 6] +} + +// ─── The ladder residual is load-bearing for a FALSE statement ── + +/// An unbalanced bus (one MUL multiplicity off), table 1's output numerator +/// replaced by the value that balances it, its ladder run on the true tree: +/// refused by step 0's residual alone, and accepted with that one check off. +#[test] +fn a_balancing_output_is_refused_by_the_residual_alone() { + let airs = airs(); + let cfg = batched(27); + let mut cols = columns(3, 1); + cols[2][3][1] += FE::one(); + let committed = commit(tables(&airs, &cols, false), &cfg); + let honest = prove(&committed, &cfg, ProverFaults::default()); + assert_eq!( + verify(&committed, &honest, &cfg), + Err(MlError::BusImbalance), + "premise: the trace's bus does not balance" + ); + let forged = prove( + &committed, + &cfg, + ProverFaults { + balance_with: Some(1), + ..ProverFaults::default() + }, + ); + assert_eq!( + verify(&committed, &forged, &cfg), + Err(MlError::LayerRelationMismatch { layer: 0 }), + "every check on: refused at step 0" + ); + let checks = VerifierChecks { + residual: false, + ..VerifierChecks::ALL + }; + assert_eq!( + verify_with_checks(&committed, &forged, &cfg, checks), + Ok(()), + "residual off: an unbalanced bus is accepted" + ); +} + +/// ★ Table `t`'s share of the batch is weighted `λ^{3t}`, so its three rules +/// (zerocheck, numerator, denominator) sit at `λ^{3t}`, `λ^{3t+1}`, `λ^{3t+2}`: +/// distinct powers across every table. At `λ^t` table 1's zerocheck would +/// share table 0's numerator power, and a slip to it on both host sides passes +/// every other host test. +#[test] +fn each_tables_share_is_weighted_lambda_cubed_to_its_index() { + let lambda = ExtE::from(3u64) + ExtE::from(5u64) * ExtE::from(7u64); + let weights = table_weights(&lambda, 6); + assert_eq!(weights.len(), 6); + let mut power = ExtE::one(); + for (t, weight) in weights.iter().enumerate() { + assert_eq!(*weight, power, "table {t}: λ^{}", 3 * t); + power = &power * &lambda * &lambda * λ + } +} + +/// The device tail (`LAMBDA_VM_ZC_DEVICE_TAIL`, forced here on every program +/// down to a cube of 4, the rule's, and of 2, the card's smallest) proves the +/// host reference's bytes: the same proof, the transcript in the same state +/// after. On a device the tables kept on the card past today's crossover are +/// counted, so the comparison is not of the path with itself; without one it +/// is the host fallback against itself, and says so. +/// +/// ```text +/// cargo test --release -p stark --features cuda,multilinear/cuda,parallel --lib -- \ +/// multilinear_table::batched::tests::the_device_tail_proves_the_host_references_bytes --exact --nocapture +/// ``` +#[test] +fn the_device_tail_proves_the_host_references_bytes() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let airs = airs(); + for (a, pad, cap) in [(12usize, 1usize, 27u8), (12, 1, 14), (3, 1, 27)] { + let cfg = batched(cap); + let cols = columns(a, pad); + let run = |at: Where, cube: Option| { + multilinear::gpu_fused::force_zc_device_tail(Some(cube)); + let committed = commit(tables(&airs, &cols, false), &cfg); + let past = multilinear::gpu_fused::device_tail_sessions(); + let mut transcript = DefaultTranscript::::new(SEED); + let proof = prove_batched_on( + &committed, + &cfg, + &mut transcript, + None, + ProverFaults::default(), + at, + ) + .unwrap_or_else(|e| panic!("{at:?} cube {cube:?}: {e:?}")); + let past = multilinear::gpu_fused::device_tail_sessions() - past; + multilinear::gpu_fused::force_zc_device_tail(None); + verify(&committed, &proof, &cfg) + .unwrap_or_else(|e| panic!("{at:?} cube {cube:?}: {e:?}")); + ( + bincode::serialize(&proof).expect("bytes"), + transcript.state(), + transcript.sample_field_element(), + past, + ) + }; + let host = run(Where::Host, None); + let today = run(Where::Device, None); + let rule = run( + Where::Device, + Some(multilinear::gpu_fused::DEVICE_TAIL_CUBE), + ); + let edge = run(Where::Device, Some(2)); + for (name, other) in [ + ("today's crossover", &today), + ("a cube of 4", &rule), + ("a cube of 2", &edge), + ] { + assert!( + host.0 == other.0, + "a={a} cap={cap}: the card handing the host {name}, the bytes differ" + ); + assert_eq!(host.1, other.1, "a={a} cap={cap}: {name}, transcript state"); + assert_eq!(host.2, other.2, "a={a} cap={cap}: {name}, next challenge"); + } + assert_eq!( + host.3, 0, + "a={a} cap={cap}: the host keeps nothing on a card" + ); + assert_eq!( + today.3, 0, + "a={a} cap={cap}: off keeps no table past the crossover" + ); + eprintln!( + "batched argue device tail a={a} pad={pad} cap={cap}: {} tables past the crossover to a cube of 4, \ + {} to 2; {} bytes", + rule.3, + edge.3, + host.0.len() + ); + if a_device() && a >= 7 { + assert!(rule.3 > 0, "a={a}: the card kept tables to a cube of 4"); + assert!(edge.3 > 0, "a={a}: the card kept tables to a cube of 2"); + } + } +} + +// ─── today's bytes, pinned ─────────────────────────────────────────────── + +/// The fixtures the byte pin proves: `(a, pad, cap, shifted)` — one bin, bins +/// of one or two, every table alone, and a shifted table. +const PINNED_FIXTURES: [(usize, usize, u8, bool); 5] = [ + (3, 1, 27, false), + (3, 1, 3, false), + (2, 2, 0, true), + (3, 1, 27, true), + (4, 0, 4, false), +]; + +/// sha3-256 of each fixture's proof (bincode) and of the transcript's state +/// after it, at `exp/b3-base` 6c1a9faf6, before the GKR slot packing existed. +const PINNED_BYTES: [(&str, &str); 5] = [ + ( + "6b7cc0374530dec0b94d875e622c91349b5dcfb15dbcdb891723a7187eb3a01f", + "fb8f20b189c038b6e454c9dcf59a821cbc76776d84d9860577e1c382a0d15a96", + ), + ( + "443395b19d148a85e888d1821c8ff90a77730e21535266be82335a8d612775cb", + "ca9f88ca1ea1ae1c41fa9332e774ecbb942fd2c0220789947ba1dd6861a55679", + ), + ( + "51b85f24fd81bd7cba55e25cec9c75dc09e303459deb4fc8f2cdd56ad4e3e8be", + "72b62ed6d0d28958d578151122fc38915d41b5c6c49c30afd2222490b0d3478e", + ), + ( + "65e0e7271c241142f5ebe0b36f740a8b5c56d1dedb7f0f432950a2f178b5ddda", + "8aed6355bd25659a434bde34d7a373a4c16ee60b187c248a7a46202608e81c58", + ), + ( + "fe10a6ff42ac078912eac0e827c14cd887b560e50e376c1be3a8cd872287c6fa", + "67ecdc39e3f83542383bb45c9142a0a1d4c583741db0ba112478829c7abc84ef", + ), +]; + +fn sha3_hex(bytes: &[u8]) -> String { + use sha3::{Digest, Sha3_256}; + Sha3_256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// ★ The default batched argue proves today's bytes: every fixture's proof and +/// the transcript after it hash to what they hashed to before any format +/// lever of the GKR's slots was written. A lever left at its default must +/// keep this green. +#[test] +fn the_default_batched_argue_proves_todays_bytes() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let airs = airs(); + let mut got = Vec::new(); + for ((a, pad, cap, shifted), (proof_pin, state_pin)) in PINNED_FIXTURES.iter().zip(PINNED_BYTES) + { + let cfg = batched(*cap); + let committed = commit(tables(&airs, &columns(*a, *pad), *shifted), &cfg); + let mut transcript = DefaultTranscript::::new(SEED); + let proof = prove_batched_with( + &committed, + &cfg, + &mut transcript, + None, + ProverFaults::default(), + ) + .expect("the prover runs"); + verify(&committed, &proof, &cfg).expect("verifies"); + let bytes = bincode::serialize(&proof).expect("bytes"); + let (got_proof, got_state) = (sha3_hex(&bytes), sha3_hex(&transcript.state())); + eprintln!( + "BATCHED PIN a={a} pad={pad} cap={cap} shifted={shifted}: proof {got_proof} · state {got_state} · {} bytes", + bytes.len() + ); + got.push(( + format!("a={a} pad={pad} cap={cap} shifted={shifted}"), + (got_proof, proof_pin), + (got_state, state_pin), + )); + } + for (fixture, (proof, proof_pin), (state, state_pin)) in got { + assert_eq!(proof, proof_pin, "{fixture}: the proof's bytes"); + assert_eq!(state, state_pin, "{fixture}: the transcript"); + } +} diff --git a/crypto/stark/src/multilinear_table/batched/tests/pack.rs b/crypto/stark/src/multilinear_table/batched/tests/pack.rs new file mode 100644 index 000000000..a00380fa0 --- /dev/null +++ b/crypto/stark/src/multilinear_table/batched/tests/pack.rs @@ -0,0 +1,718 @@ +//! The GKR slot packing (`GkrPack::Chunks`): +//! the chunking rule, the combined claim, round trips of cut tables, the +//! tampers it must refuse, and that neither format's verifier takes the +//! other's proof. + +use super::*; +use multilinear::mle::Mle; + +use crate::lookup::{BusInteraction, Multiplicity, Packing}; + +fn packed(bin_log_cells: u8, tree_log_cells: u8) -> ChainConfig { + config(ArgueFormat::Batched { + bin_log_cells, + pack: GkrPack::Chunks { tree_log_cells }, + }) +} + +/// The batched format at `bin_log_cells` with one tree a table. +fn unpacked(bin_log_cells: u8) -> ChainConfig { + config(ArgueFormat::Batched { + bin_log_cells, + pack: GkrPack::Off, + }) +} + +fn shape_of(interactions: usize, num_vars: usize) -> ArgueShape { + ArgueShape { + num_vars, + input_vars: multilinear::logup::input_layer_vars(interactions, num_vars), + degree: 2, + shifted: false, + interactions, + } +} + +fn cut(interactions: usize, num_vars: usize, tree_log_cells: u8) -> (usize, usize) { + let chunks = chunking( + &shape_of(interactions, num_vars), + GkrPack::Chunks { tree_log_cells }, + ); + (chunks.slot_vars, chunks.count) +} + +// ─── the rule ──────────────────────────────────────────────────────────── + +/// ★ Over every interaction count to 2,100, every height to 24 and a spread of +/// prices: the chunks cover the interactions with less than a chunk to spare, +/// they and the all-padding chunks after them fill today's slots exactly (so +/// the combined claim has today's length), a cut saves at its own price, and +/// `Off` is today's one tree. +#[test] +fn the_chunking_rule_keeps_its_invariants() { + for m in 1..=2100usize { + for n in [0usize, 1, 3, 7, 16, 20, 21, 24] { + let shape = shape_of(m, n); + let whole = Chunking::whole(&shape); + assert_eq!(chunking(&shape, GkrPack::Off), whole); + assert_eq!(whole.count, 1); + for tau in [0u8, 2, 8, 16, 21, 22, 23, 30, 40] { + let c = chunking( + &shape, + GkrPack::Chunks { + tree_log_cells: tau, + }, + ); + let b = c.slot_vars; + assert!(c.count << b >= m, "m={m} n={n} τ={tau}: the chunks cover"); + assert!( + (c.count - 1) << b < m, + "m={m} n={n} τ={tau}: no empty chunk" + ); + assert_eq!( + b + c.combine_vars(), + whole.slot_vars, + "m={m} n={n} τ={tau}: today's slots" + ); + if c.count > 1 { + let cost = |c: &Chunking| { + ((c.count as u128) << (n + c.slot_vars)) + + (c.count as u128 - 1) * (1u128 << tau) + }; + assert!(cost(&c) < cost(&whole), "m={m} n={n} τ={tau}: a cut saves"); + assert!(c.count >= 3, "m={m} n={n} τ={tau}: two chunks never save"); + } + } + } + } +} + +/// The production setting's cuts on the base's tables (their measured +/// shapes): τ = 22 cuts CPU, MEMW_A and SHIFT in three and KECCAK_RND in five, +/// and nothing whose padding is cheaper than the trees; τ = 23 cuts only +/// KECCAK_RND, in three (five ties with three there, and ties go to fewer +/// trees). +#[test] +fn the_production_settings_cut_the_named_tables() { + let tables = [ + ("CPU", 20usize, 21usize, (3usize, 3usize), None), + ("KECCAK_RND", 1031, 16, (8, 5), Some((9, 3))), + ("MEMW_A", 20, 21, (3, 3), None), + ("SHIFT", 18, 21, (3, 3), None), + ("MEMW_R", 7, 21, (3, 1), None), + ("LT", 9, 20, (4, 1), None), + ("STORE", 10, 21, (4, 1), None), + ("LOAD", 5, 21, (3, 1), None), + ("MEMW", 26, 21, (5, 1), None), + ("ECDAS", 388, 17, (9, 1), None), + ("KECCAK", 134, 17, (8, 1), None), + ("PAGE", 3, 18, (2, 1), None), + ]; + for (name, m, n, at22, at23) in tables { + assert_eq!(cut(m, n, 22), at22, "{name} at τ = 22"); + let whole = (shape_of(m, n).input_vars - n, 1); + assert_eq!(cut(m, n, 23), at23.unwrap_or(whole), "{name} at τ = 23"); + } +} + +/// ★ At `Off` the prover's trees are today's: one a table, each over the +/// table's own interaction list (borrowed, the same slice), in table order — +/// so the device path builds, queues and steps exactly what it did before. +#[test] +fn the_off_layout_is_one_borrowed_tree_a_table() { + let airs = airs(); + let fans = fan_airs(); + let fan_cols = fan_cols(); + let cfg = unpacked(27); + let committed = commit(packed_tables(&airs, &fans, &fan_cols), &cfg); + let plan = plan_of(&committed, &cfg); + assert_eq!(plan.trees(), committed.tables().len()); + let z = ExtE::from(7u64); + let alpha = ExtE::from(11u64); + for (t, table) in committed.tables().iter().enumerate() { + assert_eq!(plan.first_tree(t), t); + assert_eq!(table_of_tree(&plan, t), t); + let statement = table.statement(); + let interactions = multilinear_logup::interactions( + statement.interactions, + statement.slot_of.len(), + &z, + &alpha, + |col| slot(statement.slot_of, col), + ) + .expect("interactions"); + let lists = chunk_lists(&interactions, &plan.chunks[t]); + assert_eq!(lists.len(), 1, "table {t}: one tree"); + match &lists[0] { + std::borrow::Cow::Borrowed(list) => assert!( + std::ptr::eq(list.as_ptr(), interactions.as_ptr()) + && list.len() == interactions.len(), + "table {t}: the table's own list" + ), + std::borrow::Cow::Owned(_) => panic!("table {t}: a copied list at Off"), + } + } +} + +/// At a price no cut can beat, the packed plan is today's plan. +#[test] +fn a_prohibitive_price_plans_todays_bins() { + let shapes: Vec = [(20, 21), (7, 21), (1031, 16), (9, 20), (3, 18), (5, 21)] + .iter() + .map(|&(m, n)| shape_of(m, n)) + .collect(); + for cap in [26u8, 27, 20, 0] { + assert_eq!( + argue_plan_packed(&shapes, cap, GkrPack::Chunks { tree_log_cells: 40 }), + argue_plan(&shapes, cap), + "cap {cap}" + ); + } + // And a cut table's unit is its chunks' cells: CPU in three chunks of + // 2^24 no longer sits alone under 2^26, so a MEMW_R joins it. + let plan = argue_plan_packed(&shapes, 26, GkrPack::Chunks { tree_log_cells: 22 }); + assert_eq!( + plan.chunks[0], + Chunking { + slot_vars: 3, + count: 3 + } + ); + assert!(plan.bins.contains(&vec![0, 1]), "{:?}", plan.bins); +} + +// ─── the combined claim ────────────────────────────────────────────────── + +fn next(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed +} + +fn ext(seed: &mut u64) -> ExtE { + ExtE::new([ + FE::from(next(seed)), + FE::from(next(seed)), + FE::from(next(seed)), + ]) +} + +/// ★ The chunks' claims at one point, combined at `s'`, are the padded +/// layer's multilinear extension at `s' ‖ P`: today's claim. Every count of +/// chunks, so the all-padding chunks' `q ≡ 1` term is exercised too. +#[test] +fn combined_chunk_claims_are_the_padded_layers_extension() { + let mut seed = 0x9AC4_u64; + for (m, n, b) in [ + (5usize, 2usize, 0usize), + (7, 1, 0), + (9, 2, 2), + (11, 3, 2), + (20, 2, 3), + (6, 0, 1), + ] { + let slots = m.next_power_of_two(); + let rows = 1usize << n; + // The padded layer: interaction-major, `0/1` past `m`. + let (mut p, mut q) = (Vec::new(), Vec::new()); + for i in 0..slots { + for _ in 0..rows { + if i < m { + p.push(ext(&mut seed)); + q.push(ext(&mut seed)); + } else { + p.push(ExtE::zero()); + q.push(ExtE::one()); + } + } + } + let chunk = (1usize << b) * rows; + let count = m.div_ceil(1 << b); + let point: Vec = (0..b + n).map(|_| ext(&mut seed)).collect(); + let claims: Vec> = (0..count) + .map(|j| GkrClaim { + point: point.clone(), + p: Mle::new(p[j * chunk..(j + 1) * chunk].to_vec()) + .unwrap() + .evaluate(&point) + .unwrap(), + q: Mle::new(q[j * chunk..(j + 1) * chunk].to_vec()) + .unwrap() + .evaluate(&point) + .unwrap(), + }) + .collect(); + let ell = count.next_power_of_two().trailing_zeros() as usize; + assert_eq!(b + ell, slots.trailing_zeros() as usize, "m={m} b={b}"); + let combine: Vec = (0..ell).map(|_| ext(&mut seed)).collect(); + let claim = combine_chunk_claims(&claims, &combine).expect("combines"); + let at: Vec = combine.iter().chain(&point).cloned().collect(); + assert_eq!(claim.point, at, "m={m} b={b}: the point"); + assert_eq!( + claim.p, + Mle::new(p.clone()).unwrap().evaluate(&at).unwrap(), + "m={m} b={b}: p̃" + ); + assert_eq!( + claim.q, + Mle::new(q.clone()).unwrap().evaluate(&at).unwrap(), + "m={m} b={b}: q̃" + ); + // Chunks at two points do not combine. + let mut apart = claims.clone(); + apart[count - 1].point[0] += ExtE::one(); + assert!( + matches!( + combine_chunk_claims(&apart, &combine), + Err(MlError::ArgueShapeMismatch { part: "chunks" }) + ), + "m={m} b={b}: chunks at two points" + ); + } +} + +// ─── round trips ───────────────────────────────────────────────────────── + +/// A table of `m` interactions over one column `x` that balances by itself: +/// pair `k` sends `x` on bus `100 + k` and receives it back, and an odd last +/// one sends `x` on bus 200 with the zero column for multiplicity. Every +/// interaction is a different fraction, so chunks in the wrong order are +/// another statement. +fn fan_air(m: usize) -> Air { + let options = ProofOptions::default_test_options(); + let interactions: Vec = (0..m) + .map(|i| { + let x = Packing::Direct.columns(&[0]); + if i + 1 == m && m % 2 == 1 { + BusInteraction::sender(200u64, Multiplicity::Column(1), x) + } else if i % 2 == 0 { + BusInteraction::sender(100 + (i / 2) as u64, Multiplicity::One, x) + } else { + BusInteraction::receiver(100 + (i / 2) as u64, Multiplicity::One, x) + } + }) + .collect(); + AirWithBuses::new( + 2, + AuxiliaryTraceBuildData { interactions }, + &options, + 1, + EmptyConstraints, + ) +} + +/// `x` distinct on every row, the zero column beside it. +fn fan_columns(num_vars: usize, salt: u64) -> Vec> { + let rows = 1u64 << num_vars; + vec![ + (0..rows) + .map(|r| FE::from(salt * 1000 + 3 * r + 1)) + .collect(), + vec![FE::zero(); rows as usize], + ] +} + +/// The fan tables of a run: `(interactions, height)`. +const FANS: [(usize, usize); 4] = [(5, 3), (7, 5), (9, 3), (11, 3)]; + +fn fan_airs() -> Vec> { + FANS.iter().map(|&(m, _)| fan_air(m)).collect() +} + +/// CPU, ADD and MUL over `columns(3, 1)`, then the fans. +fn packed_tables<'a>( + airs: &'a Airs, + fans: &'a [Air], + fan_cols: &[Vec>], +) -> Vec> { + let cols = columns(3, 1); + let mut all = tables(airs, &cols, false); + for (air, cols) in fans.iter().zip(fan_cols) { + all.push(table(air, cols)); + } + all +} + +fn fan_cols() -> Vec>> { + FANS.iter() + .enumerate() + .map(|(i, &(_, n))| fan_columns(n, i as u64 + 1)) + .collect() +} + +fn plan_of(committed: &CommittedTables<'_, Fp, Ext, KeccakWhir>, cfg: &ChainConfig) -> ArguePlan { + let ArgueFormat::Batched { + bin_log_cells, + pack, + } = cfg.format.argue + else { + unreachable!("a batched config") + }; + let shapes: Vec = committed + .tables() + .iter() + .map(|t| ArgueShape::of(&t.statement())) + .collect(); + argue_plan_packed(&shapes, bin_log_cells, pack) +} + +/// ★ Cut tables round trip — in one bin, in small bins, every table alone — +/// with one bus output a tree. At τ = 2 every fan is cut: FAN(5) and FAN(9) +/// in chunks of two whose last holds one interaction (padded to its slots), +/// FAN(7) in seven chunks of one, FAN(11) in three of four; each with +/// all-padding chunks after its last. +#[test] +fn cut_tables_round_trip_across_bins() { + let airs = airs(); + let fans = fan_airs(); + let fan_cols = fan_cols(); + for cap in [27u8, 6, 4, 0] { + let cfg = packed(cap, 2); + let committed = commit(packed_tables(&airs, &fans, &fan_cols), &cfg); + let plan = plan_of(&committed, &cfg); + let cuts: Vec<(usize, usize)> = + plan.chunks.iter().map(|c| (c.slot_vars, c.count)).collect(); + assert_eq!( + cuts, + vec![(1, 1), (0, 1), (0, 1), (1, 3), (0, 7), (1, 5), (2, 3)], + "cap {cap}: every fan cut" + ); + let proof = prove(&committed, &cfg, ProverFaults::default()); + assert_eq!(proof.argue.bus_outputs.len(), plan.trees()); + assert_eq!(plan.trees(), 3 + 3 + 7 + 5 + 3); + verify(&committed, &proof, &cfg).unwrap_or_else(|e| panic!("cap {cap}: {e:?}")); + } +} + +/// The fixture the packed tamper tests share: one bin, the fans cut at τ = 2. +fn packed_fixture<'a>( + airs: &'a Airs, + fans: &'a [Air], + fan_cols: &[Vec>], +) -> (CommittedTables<'a, Fp, Ext, KeccakWhir>, Proof, ChainConfig) { + let cfg = packed(27, 2); + let committed = commit(packed_tables(airs, fans, fan_cols), &cfg); + let proof = prove(&committed, &cfg, ProverFaults::default()); + verify(&committed, &proof, &cfg).expect("the untampered packed proof verifies"); + (committed, proof, cfg) +} + +fn packed_tampered(tamper: impl FnOnce(&mut Proof)) -> Result<(), MlError> { + let airs = airs(); + let fans = fan_airs(); + let fan_cols = fan_cols(); + let (committed, mut proof, cfg) = packed_fixture(&airs, &fans, &fan_cols); + tamper(&mut proof); + verify(&committed, &proof, &cfg) +} + +fn packed_faulted(faults: ProverFaults) -> Result<(), MlError> { + let airs = airs(); + let fans = fan_airs(); + let fan_cols = fan_cols(); + let cfg = packed(27, 2); + let committed = commit(packed_tables(&airs, &fans, &fan_cols), &cfg); + let proof = prove(&committed, &cfg, faults); + verify(&committed, &proof, &cfg) +} + +// ─── neither verifier takes the other's proof ──────────────────────────── + +/// ★ A packed proof is refused by the unpacked verifier, and the unpacked proof by the +/// packed verifier: the format is the verifier's constant, never the proof's. +#[test] +fn neither_format_verifies_the_others_proof() { + let airs = airs(); + let fans = fan_airs(); + let fan_cols = fan_cols(); + let today = unpacked(27); + let pack = packed(27, 2); + let committed = commit(packed_tables(&airs, &fans, &fan_cols), &today); + let todays = prove(&committed, &today, ProverFaults::default()); + let packs = prove(&committed, &pack, ProverFaults::default()); + verify(&committed, &todays, &today).expect("the unpacked proof, the unpacked verifier"); + verify(&committed, &packs, &pack).expect("the packed proof, the packed verifier"); + assert_eq!( + verify(&committed, &packs, &today), + Err(MlError::ArgueShapeMismatch { part: "tables" }), + "the unpacked verifier refuses a packed proof" + ); + assert_eq!( + verify(&committed, &todays, &pack), + Err(MlError::ArgueShapeMismatch { part: "tables" }), + "the packed verifier refuses the unpacked proof" + ); + // Another price that cuts the same tables another way is another format. + let other = packed(27, 0); + assert_ne!(plan_of(&committed, &other), plan_of(&committed, &pack)); + assert!( + verify(&committed, &packs, &other).is_err(), + "τ = 0 refuses τ = 2's proof" + ); +} + +/// Where no table is cut, the packed proof is the unpacked proof, byte for byte: +/// one protocol instance, so either verifier takes it. +#[test] +fn with_nothing_cut_the_packed_proof_is_todays() { + let airs = airs(); + let fans = fan_airs(); + let fan_cols = fan_cols(); + let today = unpacked(27); + let uncut = packed(27, 40); + let committed = commit(packed_tables(&airs, &fans, &fan_cols), &today); + assert!( + plan_of(&committed, &uncut) + .chunks + .iter() + .all(|c| c.count == 1) + ); + let todays = prove(&committed, &today, ProverFaults::default()); + let uncuts = prove(&committed, &uncut, ProverFaults::default()); + assert!( + bincode::serialize(&todays).unwrap() == bincode::serialize(&uncuts).unwrap(), + "the same bytes" + ); + verify(&committed, &uncuts, &today).expect("the unpacked verifier"); +} + +// ─── tampers under the packing ─────────────────────────────────────────── + +/// A wrong count of trees: a bus output short or extra, or a proof laid out +/// under another price. +#[test] +fn a_wrong_tree_count_is_refused() { + assert_eq!( + packed_tampered(|p| { + p.argue.bus_outputs.pop(); + }), + Err(MlError::ArgueShapeMismatch { part: "tables" }) + ); + assert_eq!( + packed_tampered(|p| { + let first = p.argue.bus_outputs[0]; + p.argue.bus_outputs.push(first); + }), + Err(MlError::ArgueShapeMismatch { part: "tables" }) + ); + for tree_log_cells in [0u8, 40] { + assert!( + packed_faulted(ProverFaults { + pack: Some(GkrPack::Chunks { tree_log_cells }), + ..ProverFaults::default() + }) + .is_err(), + "the prover cut at τ = {tree_log_cells}" + ); + } + assert!( + packed_faulted(ProverFaults { + pack: Some(GkrPack::Off), + ..ProverFaults::default() + }) + .is_err(), + "the prover did not cut" + ); +} + +/// A wrong claimed sum: any chunk's output fraction off by one, either side. +#[test] +fn a_wrong_chunk_output_is_refused() { + let trees = 3 + 3 + 7 + 5 + 3; + for i in 0..trees { + assert!( + packed_tampered(|p| p.argue.bus_outputs[i].0 += ExtE::one()).is_err(), + "tree {i}'s numerator" + ); + assert!( + packed_tampered(|p| p.argue.bus_outputs[i].1 += ExtE::one()).is_err(), + "tree {i}'s denominator" + ); + } +} + +/// Two chunks' outputs swapped in the proof, or the prover's trees built over +/// its chunks in the wrong order (honest trees, a balanced bus, an honest +/// ladder): the claim then is another layer's, and the constraint phase +/// refuses it. +#[test] +fn swapped_chunks_are_refused() { + assert!( + packed_tampered(|p| p.argue.bus_outputs.swap(3, 4)).is_err(), + "two chunks' outputs swapped" + ); + for t in 3..7 { + assert_eq!( + packed_faulted(ProverFaults { + swap_chunks: Some(t), + ..ProverFaults::default() + }), + Err(MlError::BatchMismatch), + "table {t}'s first two chunks swapped" + ); + } +} + +/// A chunk's halves wrong at a mid step: that step's residual names it. +#[test] +fn one_chunks_halves_wrong_are_refused() { + let airs = airs(); + let fans = fan_airs(); + let fan_cols = fan_cols(); + let (_, proof, _) = packed_fixture(&airs, &fans, &fan_cols); + let step = 1; + let active = proof.argue.gkr[0].layers[step].halves.len(); + assert!(active > 7, "the bin's step {step} steps {active} trees"); + for tree in 0..active { + assert_eq!( + packed_tampered(|p| p.argue.gkr[0].layers[step].halves[tree].q_lo += ExtE::one()), + Err(MlError::LayerRelationMismatch { layer: step }), + "tree {tree}" + ); + } +} + +/// A forged balance on a cut table's first chunk (its numerator the value +/// that makes the bus sum to zero, the ladder run on its true tree): refused +/// by step 0's residual, and accepted with that check off — the residual is +/// as load-bearing under the packing as without it. +#[test] +fn a_balancing_chunk_output_is_refused_by_the_residual_alone() { + let airs = airs(); + let fans = fan_airs(); + let mut fan_cols = fan_cols(); + let cfg = packed(27, 2); + // FAN(7)'s odd sender reads the zero column: one row of it set to one + // unbalances the bus. + fan_cols[1][1][2] = FE::one(); + let committed = commit(packed_tables(&airs, &fans, &fan_cols), &cfg); + let honest = prove(&committed, &cfg, ProverFaults::default()); + assert_eq!( + verify(&committed, &honest, &cfg), + Err(MlError::BusImbalance), + "premise: the trace's bus does not balance" + ); + let forged = prove( + &committed, + &cfg, + ProverFaults { + balance_with: Some(4), + ..ProverFaults::default() + }, + ); + assert_eq!( + verify(&committed, &forged, &cfg), + Err(MlError::LayerRelationMismatch { layer: 0 }) + ); + let checks = VerifierChecks { + residual: false, + ..VerifierChecks::ALL + }; + assert_eq!( + verify_with_checks(&committed, &forged, &cfg, checks), + Ok(()) + ); +} + +/// A cut table's claim absorbed off by one is refused, as an uncut table's. +#[test] +fn a_tampered_combined_claim_is_refused() { + for t in [0, 3, 6] { + assert!( + packed_faulted(ProverFaults { + tamper_claim: Some(t), + ..ProverFaults::default() + }) + .is_err(), + "table {t}" + ); + } +} + +// ─── the card ──────────────────────────────────────────────────────────── + +/// ★ The card proves the host reference's bytes under the packing, with the +/// transcript in the same state after: tables tall enough for the card's +/// trees (CPU 2^13 rows, the fans 2^13 and 2^15), every fan cut at τ = 12 — +/// a last chunk padded to its slots among them — in one bin and in bins of +/// 2^14 cells. Without a device it is the host against the device path's +/// host fallbacks. +/// +/// ```text +/// cargo test --release -p stark --features cuda,multilinear/cuda --lib -- \ +/// multilinear_table::batched::tests::pack::the_card_proves_the_host_references_packed_bytes --exact --nocapture +/// ``` +#[test] +fn the_card_proves_the_host_references_packed_bytes() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + const TALL: [(usize, usize); 4] = [(5, 13), (7, 15), (9, 13), (11, 13)]; + let _switches = card_switches(); + a_present_card_is_reachable(); + let airs = airs(); + let fans: Vec> = TALL.iter().map(|&(m, _)| fan_air(m)).collect(); + let fan_cols: Vec>> = TALL + .iter() + .enumerate() + .map(|(i, &(_, n))| fan_columns(n, i as u64 + 1)) + .collect(); + let cols = columns(12, 1); + for cap in [27u8, 14] { + let cfg = packed(cap, 12); + let run = |at: Where| { + let mut all = tables(&airs, &cols, false); + for (air, cols) in fans.iter().zip(&fan_cols) { + all.push(table(air, cols)); + } + let committed = commit(all, &cfg); + let plan = plan_of(&committed, &cfg); + assert_eq!( + plan.chunks[3..] + .iter() + .map(|c| (c.slot_vars, c.count)) + .collect::>(), + vec![(1, 3), (0, 7), (1, 5), (2, 3)], + "every fan cut" + ); + let trees = multilinear::gpu::tree_calls(); + let mut transcript = DefaultTranscript::::new(SEED); + let proof = prove_batched_on( + &committed, + &cfg, + &mut transcript, + None, + ProverFaults::default(), + at, + ) + .unwrap_or_else(|e| panic!("{at:?}: {e:?}")); + verify(&committed, &proof, &cfg).unwrap_or_else(|e| panic!("{at:?}: {e:?}")); + ( + bincode::serialize(&proof).expect("bytes"), + transcript.state(), + transcript.sample_field_element(), + multilinear::gpu::tree_calls() - trees, + ) + }; + let host = run(Where::Host); + let card = run(Where::Device); + eprintln!( + "packed argue cap={cap}: {} bytes, card {}, {} trees built on it", + host.0.len(), + if a_device() { + "present" + } else { + "absent (host fallbacks)" + }, + card.3 + ); + // (The counter is the process's: a test beside this one that holds no + // switch lock can move it, so only the card's side is asserted.) + if a_device() { + assert!(card.3 > 0, "cap={cap}: the card built trees"); + } + assert!(host.0 == card.0, "cap={cap}: the canonical bytes differ"); + assert_eq!(host.1, card.1, "cap={cap}: the transcripts parted"); + assert_eq!(host.2, card.2, "cap={cap}: the next challenge moved"); + } +} diff --git a/crypto/stark/src/narrow.rs b/crypto/stark/src/narrow.rs new file mode 100644 index 000000000..f9f3882c4 --- /dev/null +++ b/crypto/stark/src/narrow.rs @@ -0,0 +1,858 @@ +//! Narrow storage of a main trace: each column at the bytes its values need. +//! +//! A trace's 64-bit words are mostly small (flags, bytes, 16-bit limbs, +//! timestamps under 2^32): on the no-epoch record block the main cells need +//! 2.0 bytes on average instead of 8. [`NarrowMain`] keeps the raw words of a +//! row-major main trace column by column, each column little-endian at the +//! width of its largest word (1, 2, 4 or 8 bytes), so widening gives back the +//! same words bit for bit: no proof byte can depend on it. + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +/// Rows per block of the widen pass: each block reads and writes every column +/// once, so the row-major side is written sequentially. +const BLOCK_ROWS: usize = 1 << 12; + +/// Rows per block of the pack pass: a block's words (a few hundred KiB) stay in +/// cache while each column takes its strided share. +const PACK_BLOCK_ROWS: usize = 1 << 10; + +/// A main trace at 1, 2, 4 or 8 bytes per cell (see the module docs). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NarrowMain { + rows: usize, + /// Bytes per cell of each column. + widths: Vec, + /// Where each column starts in `data`. + offsets: Vec, + /// Column-major: column `c` is `rows * widths[c]` little-endian bytes from + /// `offsets[c]`. + data: Vec, +} + +/// The bytes a word needs: 1, 2, 4 or 8. +fn width_of(max: u64) -> u8 { + match max { + 0..=0xff => 1, + 0x100..=0xffff => 2, + 0x1_0000..=0xffff_ffff => 4, + _ => 8, + } +} + +impl NarrowMain { + /// Pack the `cols`-wide row-major words `row_major`. `cols` must divide its + /// length (a trace of `rows * cols` cells). + /// + /// Runs on the caller's thread: the block packs each streamed instance on + /// its committer, and a pack spread over the shared pool there competed + /// with the executor and the walk (FAST 505). + pub fn pack(row_major: &[u64], cols: usize) -> Self { + let rows = row_major.len().checked_div(cols).unwrap_or(0); + debug_assert_eq!(rows * cols, row_major.len()); + let row_major = &row_major[..rows * cols]; + + let mut max = vec![0u64; cols]; + for row in row_major.chunks_exact(cols.max(1)) { + for (m, &v) in max.iter_mut().zip(row) { + *m = (*m).max(v); + } + } + let widths: Vec = max.into_iter().map(width_of).collect(); + let mut offsets = Vec::with_capacity(cols); + let mut total = 0usize; + for &w in &widths { + offsets.push(total); + total += rows * w as usize; + } + let mut data = vec![0u8; total]; + + // Each column's slice, written a block of rows at a time so the + // block's rows stay in cache across the columns. + let mut columns: Vec<&mut [u8]> = Vec::with_capacity(cols); + let mut rest = data.as_mut_slice(); + for &w in &widths { + let (column, tail) = rest.split_at_mut(rows * w as usize); + columns.push(column); + rest = tail; + } + for first in (0..rows).step_by(PACK_BLOCK_ROWS) { + let last = (first + PACK_BLOCK_ROWS).min(rows); + let block = &row_major[first * cols..last * cols]; + for (c, (column, &w)) in columns.iter_mut().zip(&widths).enumerate() { + let words = block.iter().skip(c).step_by(cols); + let w = w as usize; + let out = &mut column[first * w..last * w]; + match w { + 1 => out.iter_mut().zip(words).for_each(|(o, &v)| *o = v as u8), + 2 => out + .chunks_exact_mut(2) + .zip(words) + .for_each(|(o, &v)| o.copy_from_slice(&(v as u16).to_le_bytes())), + 4 => out + .chunks_exact_mut(4) + .zip(words) + .for_each(|(o, &v)| o.copy_from_slice(&(v as u32).to_le_bytes())), + _ => out + .chunks_exact_mut(8) + .zip(words) + .for_each(|(o, &v)| o.copy_from_slice(&v.to_le_bytes())), + } + } + } + + Self { + rows, + widths, + offsets, + data, + } + } + + /// A packed trace from its parts (the device pack's output): `rows` rows, + /// the column widths, and the columns back to back. `None` when a width is + /// not 1, 2, 4 or 8 or the bytes are not `rows × Σ widths`. + pub fn from_parts(rows: usize, widths: Vec, data: Vec) -> Option { + if widths.iter().any(|w| ![1, 2, 4, 8].contains(w)) { + return None; + } + let mut offsets = Vec::with_capacity(widths.len()); + let mut total = 0usize; + for &w in &widths { + offsets.push(total); + total = total.checked_add(rows.checked_mul(w as usize)?)?; + } + (total == data.len()).then_some(Self { + rows, + widths, + offsets, + data, + }) + } + + /// The rows, the column widths and the packed bytes, for a holder of + /// another type with this layout (`multilinear::narrow::NarrowColumns`); + /// [`Self::from_parts`] puts them back together. + pub fn into_parts(self) -> (usize, Vec, Vec) { + (self.rows, self.widths, self.data) + } + + /// Rows of the trace. + pub fn rows(&self) -> usize { + self.rows + } + + /// Columns of the trace. + pub fn cols(&self) -> usize { + self.widths.len() + } + + /// Bytes per cell of each column (1, 2, 4 or 8). + pub fn widths(&self) -> &[u8] { + &self.widths + } + + /// Where each column starts in [`Self::data`]. + pub fn offsets(&self) -> &[usize] { + &self.offsets + } + + /// The packed columns (see [`NarrowMain`]'s fields). + pub fn data(&self) -> &[u8] { + &self.data + } + + /// Column `c`'s words. + pub fn column(&self, c: usize) -> Vec { + let w = self.widths[c] as usize; + let bytes = &self.data[self.offsets[c]..self.offsets[c] + self.rows * w]; + bytes + .chunks_exact(w) + .map(|cell| { + let mut word = [0u8; 8]; + word[..w].copy_from_slice(cell); + u64::from_le_bytes(word) + }) + .collect() + } + + /// A 128-bit digest of the packed trace (its shape, widths and bytes): a + /// fast, non-cryptographic check that a trace built again is this one. A + /// prover bug is the threat, not an adversary. + pub fn digest(&self) -> [u64; 2] { + digest_parts(self.rows, &self.widths, &self.data) + } + + /// Flip the low bit of the first packed byte: a wrong word, for the tests + /// that check a bad widen is refused. + #[cfg(any(test, feature = "test-utils"))] + pub fn flip_first_bit(&mut self) { + if let Some(b) = self.data.first_mut() { + *b ^= 1; + } + } + + /// Write the row-major words into `out` (`rows * cols` of them). + pub fn widen_into(&self, out: &mut [u64]) { + let cols = self.cols(); + assert_eq!(out.len(), self.rows * cols, "widen_into: output size"); + if cols == 0 { + return; + } + let widen_block = |(b, block): (usize, &mut [u64])| { + let first = b * BLOCK_ROWS; + for (c, (&w, &off)) in self.widths.iter().zip(&self.offsets).enumerate() { + let w = w as usize; + let src = &self.data[off + first * w..]; + for (r, row) in block.chunks_exact_mut(cols).enumerate() { + let mut word = [0u8; 8]; + word[..w].copy_from_slice(&src[r * w..(r + 1) * w]); + row[c] = u64::from_le_bytes(word); + } + } + }; + #[cfg(feature = "parallel")] + out.par_chunks_mut(BLOCK_ROWS * cols) + .enumerate() + .for_each(widen_block); + #[cfg(not(feature = "parallel"))] + out.chunks_mut(BLOCK_ROWS * cols) + .enumerate() + .for_each(widen_block); + } +} + +/// [`NarrowMain::digest`] of a packed trace held as its parts (the spill +/// store digests the bytes on its writer thread). +pub(crate) fn digest_parts(rows: usize, widths: &[u8], data: &[u8]) -> [u64; 2] { + const K: [u64; 4] = [ + 0x9e37_79b9_7f4a_7c15, + 0xc2b2_ae3d_27d4_eb4f, + 0x1656_67b1_9e37_79f9, + 0x85eb_ca77_c2b2_ae63, + ]; + let mix = |acc: u64, w: u64, k: u64| (acc ^ w).wrapping_mul(k).rotate_left(31); + let mut lanes = [ + rows as u64, + widths.len() as u64, + data.len() as u64, + 0x243f_6a88_85a3_08d3, + ]; + for (i, &w) in widths.iter().enumerate() { + lanes[i % 4] = mix(lanes[i % 4], u64::from(w), K[i % 4]); + } + let mut blocks = data.chunks_exact(32); + for block in &mut blocks { + for (l, word) in block.chunks_exact(8).enumerate() { + let w = u64::from_le_bytes(word.try_into().expect("8 bytes")); + lanes[l] = mix(lanes[l], w, K[l]); + } + } + let mut tail = [0u8; 32]; + tail[..blocks.remainder().len()].copy_from_slice(blocks.remainder()); + for (l, word) in tail.chunks_exact(8).enumerate() { + let w = u64::from_le_bytes(word.try_into().expect("8 bytes")); + lanes[l] = mix(lanes[l], w, K[l]); + } + // MurmurHash3's finalizer, so every input bit reaches every output bit. + let fmix = |mut h: u64| { + h ^= h >> 33; + h = h.wrapping_mul(0xff51_afd7_ed55_8ccd); + h ^= h >> 33; + h = h.wrapping_mul(0xc4ce_b9fe_1a85_ec53); + h ^ (h >> 33) + }; + [ + fmix(lanes[0] ^ lanes[2].rotate_left(23)), + fmix(lanes[1] ^ lanes[3].rotate_left(41)), + ] +} + +/// A [`NarrowMain`] built a block of rows at a time, so a generator never holds +/// a whole 64-bit trace: each pushed block is packed into its columns at once, +/// and a column whose block needs more bytes than it had so far is re-encoded +/// at the wider width. Rows never pushed are zero. The result is the +/// [`NarrowMain::pack`] of the whole trace, byte for byte. +pub struct NarrowBuilder { + rows: usize, + cols: usize, + pushed: usize, + widths: Vec, + /// Column-major, one buffer per column at its width so far. + columns: Vec>, +} + +impl NarrowBuilder { + /// A builder for a `rows` × `cols` trace. + pub fn new(rows: usize, cols: usize) -> Self { + Self { + rows, + cols, + pushed: 0, + widths: vec![1; cols], + columns: (0..cols).map(|_| Vec::with_capacity(rows)).collect(), + } + } + + /// Append whole rows, row-major (`cols` words each). Panics past `rows`. + pub fn push_rows(&mut self, row_major: &[u64]) { + let cols = self.cols.max(1); + assert_eq!(row_major.len() % cols, 0, "push_rows: whole rows only"); + let n = row_major.len() / cols; + assert!( + self.pushed + n <= self.rows, + "push_rows: past the trace's rows" + ); + for c in 0..self.cols { + let words = row_major.iter().skip(c).step_by(cols); + let need = width_of(words.clone().copied().max().unwrap_or(0)); + if need > self.widths[c] { + self.columns[c] = rewiden(&self.columns[c], self.widths[c], need, self.rows); + self.widths[c] = need; + } + let column = &mut self.columns[c]; + match self.widths[c] { + 1 => column.extend(words.map(|&v| v as u8)), + 2 => words.for_each(|&v| column.extend_from_slice(&(v as u16).to_le_bytes())), + 4 => words.for_each(|&v| column.extend_from_slice(&(v as u32).to_le_bytes())), + _ => words.for_each(|&v| column.extend_from_slice(&v.to_le_bytes())), + } + } + self.pushed += n; + } + + /// The packed trace: the rows pushed, then zero rows up to `rows`. + pub fn finish(self) -> NarrowMain { + let rows = self.rows; + let mut offsets = Vec::with_capacity(self.cols); + let total: usize = self.widths.iter().map(|&w| rows * w as usize).sum(); + let mut data = Vec::with_capacity(total); + for (column, &w) in self.columns.into_iter().zip(&self.widths) { + offsets.push(data.len()); + data.extend_from_slice(&column); + data.resize(data.len() + (rows - self.pushed) * w as usize, 0); + } + NarrowMain { + rows, + widths: self.widths, + offsets, + data, + } + } +} + +/// `bytes`, cells of `from` bytes each, re-encoded at `to` bytes each (little +/// endian, so each cell is zero-extended), with room for `rows` cells. +fn rewiden(bytes: &[u8], from: u8, to: u8, rows: usize) -> Vec { + let (from, to) = (from as usize, to as usize); + let mut out = Vec::with_capacity(rows * to); + for cell in bytes.chunks_exact(from) { + out.extend_from_slice(cell); + out.resize(out.len() + (to - from), 0); + } + out +} + +/// A [`NarrowMain`] written a cell at a time straight into its packed columns, +/// so a generator never holds a 64-bit trace, not even a block of one. +/// +/// The caller picks each column's width up front: a guess, such as the widths +/// the table's earlier traces needed. The writer stores each word's low bytes +/// at its column's width and keeps the OR of every word written to each +/// column, which has the bit length of the column's largest word. When every +/// word fit, [`Self::finish`] gives the [`NarrowMain::pack`] of the same words, +/// byte for byte: a column given more bytes than it needed is narrowed in +/// place. When a word did not fit, the bytes written are not the trace, and +/// `finish` returns the widths the trace needs instead, for the caller to write +/// it again at those (which cannot miss: they hold every word written). +/// +/// Cells never written are zero. A cell written twice keeps its last word, but +/// both words count toward its column's width: writers set each cell once. +pub struct NarrowWriter { + rows: usize, + columns: Vec, + data: Vec, +} + +/// A [`NarrowWriter`] column: where it starts in the data, its width, and the +/// OR of every word written to it. +#[derive(Clone, Copy)] +struct WriterColumn { + offset: usize, + width: usize, + seen: u64, +} + +/// The fewest bytes of 1, 2, 4 and 8 that hold `w` bytes. +fn valid_width(w: u8) -> usize { + match w { + 0 | 1 => 1, + 2 => 2, + 3 | 4 => 4, + _ => 8, + } +} + +impl NarrowWriter { + /// A writer for a trace of `rows` rows and `widths.len()` columns, column + /// `c` at `widths[c]` bytes (rounded up to 1, 2, 4 or 8). + pub fn new(rows: usize, widths: &[u8]) -> Self { + let mut columns = Vec::with_capacity(widths.len()); + let mut total = 0usize; + for &w in widths { + let width = valid_width(w); + columns.push(WriterColumn { + offset: total, + width, + seen: 0, + }); + total += rows * width; + } + Self { + rows, + columns, + data: vec![0u8; total], + } + } + + /// Rows of the trace. + pub fn rows(&self) -> usize { + self.rows + } + + /// Columns of the trace. + pub fn cols(&self) -> usize { + self.columns.len() + } + + /// Write `word` into row `row` of column `col`: its low bytes, at the + /// column's width (see [`NarrowWriter`] for a word that does not fit). + #[inline] + pub fn set(&mut self, row: usize, col: usize, word: u64) { + debug_assert!(row < self.rows, "NarrowWriter: row {row} of {}", self.rows); + let column = &mut self.columns[col]; + column.seen |= word; + let (width, at) = (column.width, column.offset + row * column.width); + match width { + 1 => self.data[at] = word as u8, + 2 => self.data[at..at + 2].copy_from_slice(&(word as u16).to_le_bytes()), + 4 => self.data[at..at + 4].copy_from_slice(&(word as u32).to_le_bytes()), + _ => self.data[at..at + 8].copy_from_slice(&word.to_le_bytes()), + } + } + + /// The widths the words written so far need: what [`Self::finish`] gives + /// the trace when they all fit. + pub fn needed_widths(&self) -> Vec { + self.columns.iter().map(|c| width_of(c.seen)).collect() + } + + /// The widths the columns are written at. + pub fn widths(&self) -> Vec { + self.columns.iter().map(|c| c.width as u8).collect() + } + + /// The packed trace, every column narrowed to the bytes its words need; or, + /// when a word did not fit its column, `Err` with the widths the trace + /// needs (each at least the width given, for a column that missed). + pub fn finish(self) -> Result> { + let need = self.needed_widths(); + if need + .iter() + .zip(&self.columns) + .any(|(&n, c)| n as usize > c.width) + { + return Err(need); + } + let Self { + rows, + columns, + mut data, + } = self; + // Columns move only toward the front and only narrow, so moving them + // in order never overwrites a byte not yet moved. + let mut offsets = Vec::with_capacity(columns.len()); + let mut at = 0usize; + for (&n, c) in need.iter().zip(&columns) { + let n = n as usize; + offsets.push(at); + if at != c.offset || n != c.width { + move_column(&mut data, rows, c.offset, c.width, at, n); + } + at += rows * n; + } + if at != data.len() { + data.truncate(at); + data.shrink_to_fit(); + } + let narrow = NarrowMain { + rows, + widths: need, + offsets, + data, + }; + #[cfg(debug_assertions)] + debug_assert_eq!( + narrow.widths, + (0..narrow.cols()) + .map(|c| width_of(narrow.column(c).into_iter().fold(0, |m, v| m | v))) + .collect::>(), + "NarrowWriter: a column's width is not its largest word's (a cell written twice?)" + ); + #[cfg(any(test, feature = "test-utils"))] + let narrow = mutation::apply(narrow); + Ok(narrow) + } +} + +/// Move a column of `rows` cells at `from` (`w` bytes each) to `to` (`n` bytes +/// each, `n <= w`), keeping each cell's low `n` bytes. `to <= from`, so the +/// cells are moved front to back. +fn move_column(data: &mut [u8], rows: usize, from: usize, w: usize, to: usize, n: usize) { + debug_assert!(to <= from && n <= w); + if n == w { + data.copy_within(from..from + rows * w, to); + return; + } + fn narrow( + data: &mut [u8], + rows: usize, + from: usize, + to: usize, + ) { + for r in 0..rows { + let mut cell = [0u8; W]; + cell.copy_from_slice(&data[from + r * W..from + (r + 1) * W]); + data[to + r * N..to + (r + 1) * N].copy_from_slice(&cell[..N]); + } + } + match (w, n) { + (2, 1) => narrow::<2, 1>(data, rows, from, to), + (4, 1) => narrow::<4, 1>(data, rows, from, to), + (4, 2) => narrow::<4, 2>(data, rows, from, to), + (8, 1) => narrow::<8, 1>(data, rows, from, to), + (8, 2) => narrow::<8, 2>(data, rows, from, to), + _ => narrow::<8, 4>(data, rows, from, to), + } +} + +/// Test-only faults in [`NarrowWriter::finish`]'s output, per thread: one +/// column given a byte more than it needs, or one column's bytes shifted by a +/// byte. The gates that compare a written trace with the packed 64-bit one +/// must catch both. +#[cfg(any(test, feature = "test-utils"))] +pub mod mutation { + use super::NarrowMain; + use std::cell::Cell; + + /// A fault [`set`] arms. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Mutation { + /// Column `c` at the next width up (a 1-byte column at 2 bytes): the + /// words are the same, the bytes are not. + WidenColumn(usize), + /// Column `c`'s bytes rotated by one: a column read one byte off. + ShiftColumn(usize), + } + + thread_local! { + static ARMED: Cell> = const { Cell::new(None) }; + } + + /// Arm (or with `None`, disarm) a fault for every trace this thread's + /// writers finish. + pub fn set(m: Option) { + ARMED.with(|a| a.set(m)); + } + + pub(super) fn apply(narrow: NarrowMain) -> NarrowMain { + let Some(m) = ARMED.with(Cell::get) else { + return narrow; + }; + let cols = narrow.cols(); + match m { + Mutation::WidenColumn(c) if c < cols && narrow.widths[c] < 8 => { + let mut widths = narrow.widths.clone(); + widths[c] *= 2; + let mut data = Vec::new(); + for (k, &w) in widths.iter().enumerate() { + for v in narrow.column(k) { + data.extend_from_slice(&v.to_le_bytes()[..w as usize]); + } + } + NarrowMain::from_parts(narrow.rows, widths, data).unwrap_or(narrow) + } + Mutation::ShiftColumn(c) if c < cols && narrow.rows > 1 => { + let mut narrow = narrow; + let (off, len) = (narrow.offsets[c], narrow.rows * narrow.widths[c] as usize); + narrow.data[off..off + len].rotate_left(1); + narrow + } + _ => narrow, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every width boundary, in every position, round trips: 0, 2^8 − 1, + /// 2^8, 2^16 − 1, 2^16, 2^32 − 1, 2^32, the Goldilocks p − 1 and + /// u64::MAX, over a row count that is not a multiple of the block. + #[test] + fn narrow_round_trips_at_every_width_boundary() { + let edges = [ + 0u64, + 0xff, + 0x100, + 0xffff, + 0x1_0000, + 0xffff_ffff, + 0x1_0000_0000, + 0xffff_ffff_0000_0000, + u64::MAX, + ]; + let cols = edges.len() + 1; + let rows = BLOCK_ROWS + 3; + let data: Vec = (0..rows * cols) + .map(|i| { + let (r, c) = (i / cols, i % cols); + // Column c < 9 tops out at edges[c] (in the last row); the + // last column is all zeros. + if c == edges.len() { + 0 + } else if r == rows - 1 { + edges[c] + } else { + (r as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) % (edges[c] / 2 + 1) + } + }) + .collect(); + let narrow = NarrowMain::pack(&data, cols); + assert_eq!(narrow.widths(), &[1, 1, 2, 2, 4, 4, 8, 8, 8, 1]); + let mut back = vec![0u64; data.len()]; + narrow.widen_into(&mut back); + assert_eq!(back, data); + for c in 0..cols { + let column: Vec = data.iter().skip(c).step_by(cols).copied().collect(); + assert_eq!(narrow.column(c), column, "column {c}"); + } + assert_eq!( + narrow.data().len(), + rows * narrow.widths().iter().map(|&w| w as usize).sum::() + ); + } + + /// Built a block at a time — with columns that need more bytes only in a + /// later block, and zero rows never pushed — the trace packs to the bytes + /// [`NarrowMain::pack`] gives the whole trace. + #[test] + fn a_trace_built_a_block_at_a_time_packs_as_a_whole() { + let cols = 6; + let rows = 3 * 1000 + 37; + let pushed = rows - 37; + let data: Vec = (0..rows * cols) + .map(|i| { + let (r, c) = (i / cols, i % cols); + if r >= pushed { + return 0; + } + let x = (r as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) >> (8 * c as u32); + match c { + // Grows from 1 byte to 2 in the second block, to 8 in the last. + 0 if r < 1000 => x & 0xff, + 0 if r < 2000 => x & 0xffff, + 0 => x, + 1 => x & 0xff, + 2 => 0, + 3 if r == pushed - 1 => u64::MAX, + 3 => x & 0xff, + _ => x & 0xffff_ffff, + } + }) + .collect(); + let whole = NarrowMain::pack(&data, cols); + for block in [1, 7, 1000, 4096] { + let mut builder = NarrowBuilder::new(rows, cols); + for part in data[..pushed * cols].chunks(block * cols) { + builder.push_rows(part); + } + assert_eq!(builder.finish(), whole, "blocks of {block} rows"); + } + assert_eq!(whole.widths(), &[8, 1, 1, 8, 4, 4]); + } + + /// The digest sees every packed byte and the shape: equal traces digest + /// alike, and one flipped bit anywhere (head, middle, the tail past the + /// last 32-byte block) or another row count changes it. + #[test] + fn the_digest_sees_every_byte() { + let cols = 3; + let words: Vec = (0..(1000 * cols) as u64).map(|i| i * 7 % 300).collect(); + let narrow = NarrowMain::pack(&words, cols); + assert_eq!(narrow.digest(), NarrowMain::pack(&words, cols).digest()); + assert!( + !narrow.data().len().is_multiple_of(32), + "a tail past the last block" + ); + for at in [0, narrow.data().len() / 2, narrow.data().len() - 1] { + let mut bent = narrow.clone(); + bent.data[at] ^= 1; + assert_ne!(bent.digest(), narrow.digest(), "byte {at}"); + } + let fewer = NarrowMain::pack(&words[..999 * cols], cols); + assert_ne!(fewer.digest(), narrow.digest()); + } + + /// Words with every column width, some cells never written (rows past + /// `written`): column c tops out at a different byte boundary, in a row + /// that is not the first. + fn words_for_the_writer(rows: usize, written: usize) -> (Vec, usize) { + let tops = [ + 0u64, + 1, + 0xff, + 0x100, + 0xffff, + 0x1_0000, + 0xffff_ffff, + 0x1_0000_0000, + u64::MAX, + ]; + let cols = tops.len(); + let words = (0..rows * cols) + .map(|i| { + let (r, c) = (i / cols, i % cols); + if r >= written { + 0 + } else if r == written / 2 { + tops[c] + } else { + (r as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) % (tops[c] / 2 + 1) + } + }) + .collect(); + (words, cols) + } + + /// Write `words` (row-major, `cols` wide) into a writer at `widths`, in + /// row order or column by column from the last row. + fn write(words: &[u64], cols: usize, widths: &[u8], scrambled: bool) -> NarrowWriter { + let rows = words.len() / cols; + let mut w = NarrowWriter::new(rows, widths); + let mut cell = |r: usize, c: usize| { + if words[r * cols + c] != 0 { + w.set(r, c, words[r * cols + c]); + } + }; + if scrambled { + for c in 0..cols { + for r in (0..rows).rev() { + cell(r, c); + } + } + } else { + for r in 0..rows { + for c in 0..cols { + cell(r, c); + } + } + } + w + } + + /// ★ Written a cell at a time — at the widths the columns need, at more + /// bytes than they need (each column narrowed in place, every width pair), + /// at a mix, in any order, with cells never written — the trace finishes + /// to the bytes [`NarrowMain::pack`] gives it. At fewer bytes than a column + /// needs it does not finish: it names the widths, and written again at + /// those it finishes to the same bytes. + #[test] + fn a_trace_written_cell_by_cell_packs_as_a_whole() { + let rows = 3 * PACK_BLOCK_ROWS + 5; + let (words, cols) = words_for_the_writer(rows, rows - 7); + let whole = NarrowMain::pack(&words, cols); + assert_eq!(whole.widths(), &[1, 1, 1, 2, 2, 4, 4, 8, 8]); + let exact = whole.widths().to_vec(); + let mixed: Vec = exact + .iter() + .enumerate() + .map(|(c, &w)| if c % 2 == 0 { 8 } else { w }) + .collect(); + let at_least = |floor: u8| exact.iter().map(|&w| w.max(floor)).collect::>(); + for widths in [at_least(1), at_least(2), at_least(4), at_least(8), mixed] { + for scrambled in [false, true] { + let written = write(&words, cols, &widths, scrambled); + assert_eq!(written.needed_widths(), exact); + assert_eq!( + written.finish().ok(), + Some(whole.clone()), + "{widths:?}, scrambled {scrambled}" + ); + } + } + // Too narrow for every column past the first three: refused with the + // widths, then written at those. + let missed = write(&words, cols, &[1; 9], false).finish(); + assert_eq!(missed.as_ref().err(), Some(&exact)); + let again = write(&words, cols, &missed.unwrap_err(), true); + assert_eq!(again.finish().ok(), Some(whole)); + } + + /// A widths list that is not 1, 2, 4 or 8 is rounded up; a trace never + /// written, of no rows or of no columns, finishes empty. + #[test] + fn the_writer_rounds_widths_up_and_finishes_empty_traces() { + let w = NarrowWriter::new(10, &[0, 3, 5, 9]); + assert_eq!(w.widths(), [1, 4, 8, 8]); + assert_eq!(w.finish().ok(), Some(NarrowMain::pack(&[0; 40], 4))); + assert_eq!( + NarrowWriter::new(0, &[2, 4]).finish().ok(), + Some(NarrowMain::pack(&[], 2)) + ); + assert_eq!( + NarrowWriter::new(5, &[]).finish().ok(), + NarrowMain::from_parts(5, Vec::new(), Vec::new()) + ); + } + + /// The test faults change the bytes: a column one width up widens to the + /// same words, a shifted column does not. + #[test] + fn the_writer_faults_change_the_bytes() { + let rows = 100; + let (words, cols) = words_for_the_writer(rows, rows); + let whole = NarrowMain::pack(&words, cols); + for (m, same_words) in [ + (mutation::Mutation::WidenColumn(3), true), + (mutation::Mutation::ShiftColumn(5), false), + ] { + mutation::set(Some(m)); + let bent = write(&words, cols, whole.widths(), false).finish(); + mutation::set(None); + let bent = bent.expect("the words fit"); + assert_ne!(bent, whole, "{m:?}"); + assert_ne!(bent.digest(), whole.digest(), "{m:?}"); + let mut back = vec![0u64; words.len()]; + bent.widen_into(&mut back); + assert_eq!(back == words, same_words, "{m:?}"); + } + assert_eq!( + write(&words, cols, whole.widths(), false).finish().ok(), + Some(whole) + ); + } + + /// An empty trace packs to nothing and widens to nothing. + #[test] + fn an_empty_trace_packs_to_nothing() { + let narrow = NarrowMain::pack(&[], 3); + assert_eq!( + (narrow.rows(), narrow.cols(), narrow.data().len()), + (0, 3, 0) + ); + narrow.widen_into(&mut []); + } +} diff --git a/crypto/stark/src/ood.rs b/crypto/stark/src/ood.rs index d5f17d159..a171a7faa 100644 --- a/crypto/stark/src/ood.rs +++ b/crypto/stark/src/ood.rs @@ -223,6 +223,11 @@ impl OodLayout { } } + /// Rows in the full OOD grid: one DEEP point `z·gʳ` per row. + pub fn num_eval_points(&self) -> usize { + self.num_eval_points + } + /// Rows per offset block. pub fn step_size(&self) -> usize { self.step_size diff --git a/crypto/stark/src/par.rs b/crypto/stark/src/par.rs index cee693e3f..42f0729bd 100644 --- a/crypto/stark/src/par.rs +++ b/crypto/stark/src/par.rs @@ -72,23 +72,3 @@ pub(crate) fn par_for_each_mut(slice: &mut [T], f: impl Fn(&mut T) + Sy slice.iter_mut().for_each(f); } } - -/// Run `f(&mut item)` for each element of `slice`, short-circuiting on the -/// first `Err`. Parallel when `feature = "parallel"`, sequential otherwise. -// Only called from `disk-spill`-gated paths; keep it available without warning -// when that feature is off. -#[cfg_attr(not(feature = "disk-spill"), allow(dead_code))] -pub(crate) fn par_try_for_each_mut( - slice: &mut [T], - f: impl Fn(&mut T) -> Result<(), E> + Sync + Send, -) -> Result<(), E> { - #[cfg(feature = "parallel")] - { - use rayon::prelude::*; - slice.par_iter_mut().try_for_each(f) - } - #[cfg(not(feature = "parallel"))] - { - slice.iter_mut().try_for_each(f) - } -} diff --git a/crypto/stark/src/proof/options.rs b/crypto/stark/src/proof/options.rs index 15e2c8909..0e3e90f45 100644 --- a/crypto/stark/src/proof/options.rs +++ b/crypto/stark/src/proof/options.rs @@ -1,4 +1,7 @@ use core::fmt; +use core::str::FromStr; + +pub use crypto::merkle_tree::cap::CapPolicy; #[cfg(feature = "wasm")] use wasm_bindgen::prelude::wasm_bindgen; @@ -39,6 +42,19 @@ impl fmt::Display for ProofOptionsError { /// - `coset_offset`: the offset for the coset /// - `grinding_factor`: the number of leading zeros that we want for the Hash(hash || nonce) /// - `fri_final_poly_log_degree`: log2 degree bound at which FRI terminates folding +/// - `format`: the proof FORMAT ([`ProofFormat`], the ZF proof-format levers). +/// Its default is the legacy format (every lever off), byte for byte. +/// +/// # The format is not serialized +/// +/// `format` is skipped by serde and rkyv (and restored to its default on +/// deserialize), so a serialized `ProofOptions` has exactly the bytes it had +/// before the field existed. Nothing repo-wide was found to serialize a +/// `ProofOptions` into pinned bytes (the one by-value holder, `AirContext`, +/// derives neither), and skipping it makes that true by construction rather +/// than by search. The format is a verifier-side constant: it comes from the +/// code that builds the options, never from bytes a prover supplied — a +/// proof never carries it. #[cfg_attr(feature = "wasm", wasm_bindgen)] #[derive( Clone, @@ -58,9 +74,304 @@ pub struct ProofOptions { /// polynomial has degree < 2^fri_final_poly_log_degree; the prover sends those /// 2^k coefficients instead of folding to a constant. pub fri_final_poly_log_degree: u8, + /// The proof format. [`ProofFormat::DEFAULT`] = the legacy format (the + /// production format is stamped on by the prover crate). Not serialized. + #[serde(skip)] + #[rkyv(with = rkyv::with::Skip)] + #[cfg_attr(feature = "wasm", wasm_bindgen(skip))] + pub format: ProofFormat, +} + +/// A base proof's commitment geometry: which hash commits it and, for an +/// arity-4 hash, the height of its Merkle caps in 4-ary levels (the binary cap +/// policies price binary trees only). The WHIR block path dispatches on it +/// (`prover::block_whir`): a verifier-side constant from the caller, never +/// read from a proof or the environment, never serialized. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct BaseFormat { + pub hash: crate::config::CommitmentHash, + pub arity4_cap: CapPolicy, +} + +impl BaseFormat { + /// RPX256 over binary trees (the format's binary cap applies): today. + pub const RPX: Self = Self { + hash: crate::config::CommitmentHash::Rpx256, + arity4_cap: CapPolicy::Off, + }; + + /// ZisK's Poseidon1 over 4-ary trees, the WHIR block's: caps at 4-ary + /// height 2 (D-WHIR-P1 D2, tag `C2`). + pub const P1_WHIR: Self = Self { + hash: crate::config::CommitmentHash::Poseidon1, + arity4_cap: CapPolicy::Fixed(2), + }; +} + +impl Default for BaseFormat { + fn default() -> Self { + Self::RPX + } +} + +/// The proof-format levers of a univariate STARK proof. Grouped so a literal +/// `ProofOptions` names the format in one line (`format: ProofFormat::DEFAULT`) +/// and a lever added later touches this struct only. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct ProofFormat { + /// Merkle cap policy for every tree of the proof (S1). `Off` = today. + pub merkle_cap: CapPolicy, + /// FRI fold schedule of the committed layers (S3). `Pair` = today. + pub fri_mode: FriMode, + /// One-row trace openings with a committed FRI input (S2). `Off` = today. + pub one_row: OneRowMode, + /// An explicit committed-layer fold schedule that replaces the DP's under + /// [`FriMode::Dp`] (ignored under [`FriMode::Pair`]). `None` = the DP. + /// + /// A TEST HOOK: it lets round-trip tests prove and verify schedules the DP + /// never picks (unequal neighbouring exponents such as `[1, 3]`, the only + /// shape that catches a fold-count off-by-one). No knob sets it — the + /// `ZF FORMAT` parser always leaves it `None` — and like every format + /// field it is a verifier-side constant, never read from a proof. A + /// schedule that does not cover the table's committed folds exactly is a + /// proving error and a verification failure, never a silent fallback. + pub fri_schedule_override: Option, + /// Whether an AIR-set builder that assembles one proof from several chips + /// (the LFM recursion machine, `prover::lfm::airs`) puts the chips of equal + /// padded height into one AIR, side by side. `false` = today: one AIR a + /// chip. + /// + /// A LAYOUT choice of the caller's AIR set, carried here so it travels with + /// the options every artifact build, prove and verify is handed. The STARK + /// prover and verifier never read it: a merged AIR is an ordinary AIR to + /// them, and no tree or FRI encoding depends on it. The one reader in this + /// crate is [`Self::is_legacy`] (it is a lever, so a format with it on is not + /// the legacy format — which makes a caller that requires the legacy format, + /// such as the RV64 recursion guard, refuse it). Like every format field it + /// is a verifier-side constant, never read from a proof. + pub merge_equal_height_tables: bool, +} + +impl ProofFormat { + /// This crate's default: every lever off, i.e. [`Self::LEGACY`]. + /// + /// ⚠ NOT the production format. The prover crate's + /// `zf_format::ZfFormat::DEFAULT` (the measured configuration) is stamped + /// onto the options at the production sites; a `ProofOptions` built here + /// without a format, or deserialized (the format is not serialized), is + /// the legacy format. + pub const DEFAULT: Self = Self::LEGACY; + + /// The legacy format: every lever off. The only format the RV64 + /// recursion guest verifies. + pub const LEGACY: Self = Self { + merkle_cap: CapPolicy::Off, + fri_mode: FriMode::Pair, + one_row: OneRowMode::Off, + fri_schedule_override: None, + merge_equal_height_tables: false, + }; + + /// True when this is this crate's default format, [`Self::LEGACY`] + /// (`Fixed(0)` counts as `Off`). + pub fn is_default(&self) -> bool { + self.is_legacy() + } + + /// True when every lever is off (`Fixed(0)` counts as `Off`): the proof + /// this produces is the legacy format, byte for byte. + pub fn is_legacy(&self) -> bool { + self.merkle_cap.is_off() + && self.fri_mode == FriMode::Pair + && self.one_row == OneRowMode::Off + && self.fri_schedule_override.is_none() + && !self.merge_equal_height_tables + } +} + +/// Longest schedule a [`FriScheduleOverride`] holds. +pub const FRI_SCHEDULE_OVERRIDE_MAX: usize = 32; + +/// An explicit FRI fold schedule (see [`ProofFormat::fri_schedule_override`]): +/// the fold exponent of each committed layer, first committed layer first. +/// Fixed capacity so [`ProofFormat`] stays `Copy`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct FriScheduleOverride { + len: u8, + exponents: [u8; FRI_SCHEDULE_OVERRIDE_MAX], +} + +impl FriScheduleOverride { + /// `None` if `schedule` is longer than [`FRI_SCHEDULE_OVERRIDE_MAX`]. The + /// exponents themselves are validated where the layout is built (each in + /// `1..=FRI_SCHEDULE_DMAX`, summing to the table's committed folds). + pub fn new(schedule: &[u8]) -> Option { + if schedule.len() > FRI_SCHEDULE_OVERRIDE_MAX { + return None; + } + let mut exponents = [0u8; FRI_SCHEDULE_OVERRIDE_MAX]; + exponents[..schedule.len()].copy_from_slice(schedule); + Some(Self { + len: schedule.len() as u8, + exponents, + }) + } + + /// The schedule. + pub fn as_slice(&self) -> &[u8] { + &self.exponents[..self.len as usize] + } +} + +/// How the committed FRI layers fold (S3). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum FriMode { + /// One binary fold per committed layer, pair leaves. Today's format. + #[default] + Pair, + /// Folds of `2^d` per committed layer, `d` chosen by the verifier-side DP. + Dp, +} + +impl FriMode { + /// The knob spelling (`LAMBDA_VM_ZF_FRI`). + pub const fn name(self) -> &'static str { + match self { + Self::Pair => "pair", + Self::Dp => "dp", + } + } +} + +impl fmt::Display for FriMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } } +impl FromStr for FriMode { + type Err = (); + fn from_str(s: &str) -> Result { + match s { + "pair" => Ok(Self::Pair), + "dp" => Ok(Self::Dp), + _ => Err(()), + } + } +} + +/// Whether the trace trees commit one LDE row per leaf (S2). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum OneRowMode { + /// Row-pair leaves, the DEEP pair rebuilt from trace openings. Today's format. + #[default] + Off, + /// One-row leaves and a committed FRI-input tree for every table. + On, + /// Per table, whichever the cost model prefers from the AIR's widths. + Auto, +} + +impl OneRowMode { + /// The knob spelling (`LAMBDA_VM_ZF_ONE_ROW`). + pub const fn name(self) -> &'static str { + match self { + Self::Off => "0", + Self::On => "1", + Self::Auto => "auto", + } + } +} + +impl fmt::Display for OneRowMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +impl FromStr for OneRowMode { + type Err = (); + fn from_str(s: &str) -> Result { + match s { + "0" => Ok(Self::Off), + "1" => Ok(Self::On), + "auto" => Ok(Self::Auto), + _ => Err(()), + } + } +} + +/// Which format levers THIS build implements. A lever that is only parsed — +/// its field exists so the option structs and the `ZF FORMAT` banner stay +/// stable before the lever lands — must not be selectable, or a run +/// could print a non-default format and prove the default one. Each flag +/// is flipped in the commit that makes the lever real. +/// +/// The Merkle cap is real on the host and device STARK provers, the host +/// verifier and the LFM in-guest STARK verifier +/// (`lfm::merkle_cap::CapCells`, one caps arena per sub-proof), on pair and on +/// group-leaf (`Dp`) FRI layers alike. The RV64 recursion guest stays +/// legacy-only: its archived verifier refuses any other format. +pub const MERKLE_CAP_IMPLEMENTED: bool = true; + +/// `FriMode::Dp` (S3) is implemented on the prover paths and the host verifier: +/// - the CPU prover (group-leaf layer commits, the scheduled folds, group +/// openings) and the host verifier (`multi_verify` / `multi_verify_archived`); +/// - on a `cuda` build the device FRI arms (DEEP→FRI on device, the device +/// layer commit, the device query gather) run both encodings: the group +/// loop (`gpu_lde::fri_commit_gpu_drive_groups`, +/// `math_cuda::fri::FriCommitState::fold_and_commit_group`) is the CPU +/// loop's device twin, byte for byte (`tests::zf_fri_device_tests`). +/// +/// - the in-guest (LFM) STARK verifier (G1 + G2): `lfm::fri::FriShape` takes +/// the same schedule, and the emitter verifies group layers (slot check, +/// group leaf, group fold), so an LFM wrap or node verifies a `Dp` proof. +/// +/// NOT implemented: the RV64 recursion guest (legacy-only; it +/// refuses a non-legacy format). +pub const FRI_MODE_IMPLEMENTED: bool = true; + +/// `OneRowMode::{On, Auto}` (S2) is implemented on the prover (CPU and +/// device) and the host verifier: +/// - the CPU prover (one-row trace, precomputed, aux and composition trees; +/// the DEEP codeword committed as FRI layer 0 before the first challenge; +/// query indexes over the whole LDE; one-row openings) and the host +/// verifier (`multi_verify` / `multi_verify_archived`), with the per-table +/// `Auto` rule (`crate::leaf_layout`); +/// - the preprocessed roots: static one-row twins at blowup 4 +/// (`STATIC_BLOWUP_FACTORS_ONE_ROW` in the prover crate), every computed +/// root at run time, the LFM artifacts' one-row roots and the registry +/// policy (a one-row format never reads `LFM_REGISTRY`); a table with no +/// root for its layout is a proving error and a verifier reject, never a recompute +/// — e.g. `one_row = 1` at blowup 2, 8 or 16 fails on BITWISE; +/// - the device: one-row trees for the fused main commit, +/// the preprocessed split, the aux commits (host input and resident) and the +/// composition tree, device openings at row `r`, the LFM artifact commit, +/// and the input tree committed from the resident DEEP codeword before the +/// first challenge — each proof byte-identical to the CPU one; a one-row +/// table may be device-only like a row-pair one, and under `Auto` one proof +/// mixes both layouts on the device. +/// +/// NOT implemented: the in-guest (LFM) verifier of a one-row proof (an emitter +/// asked for one refuses at emit time, `lfm::fri::FriShape::from_options`), +/// and the RV64 recursion guest (legacy-only). A block run under +/// `LAMBDA_VM_ZF_ONE_ROW` therefore proves and host-verifies its STARK and +/// LFM proofs but cannot recurse over one-row STARK proofs yet. +pub const ONE_ROW_IMPLEMENTED: bool = true; + impl ProofOptions { + /// True when every format field is at this crate's default (the legacy + /// format): the proof this produces is the legacy format, byte for + /// byte. + pub fn has_default_format(&self) -> bool { + self.format.is_default() + } + + /// True when every lever is off: [`ProofFormat::LEGACY`]. + pub fn has_legacy_format(&self) -> bool { + self.format.is_legacy() + } + /// Default proof options used for testing purposes. /// These options should never be used in production. pub fn default_test_options() -> Self { @@ -70,6 +381,7 @@ impl ProofOptions { coset_offset: 3, grinding_factor: 1, fri_final_poly_log_degree: DEFAULT_FRI_FINAL_POLY_LOG_DEGREE, + format: ProofFormat::DEFAULT, } } } @@ -130,6 +442,7 @@ impl GoldilocksCubicProofOptions { coset_offset: 3, grinding_factor, fri_final_poly_log_degree: DEFAULT_FRI_FINAL_POLY_LOG_DEGREE, + format: ProofFormat::DEFAULT, }) } } diff --git a/crypto/stark/src/prove_split.rs b/crypto/stark/src/prove_split.rs new file mode 100644 index 000000000..cdfa0fe1d --- /dev/null +++ b/crypto/stark/src/prove_split.rs @@ -0,0 +1,353 @@ +//! One `PROVE SPLIT:` line per `multi_prove`, behind `LFM_PROVE_SPLIT=1`. +//! +//! # Why this exists next to `instruments` +//! +//! [`crate::instruments`] already carries a finer breakdown of the same prove — +//! and it is compiled out unless `--features instruments` is named. That is the +//! wrong shape for this question. The number we need to attribute is the block +//! record, and the record is produced by a binary that does NOT enable the +//! feature; a figure taken from a differently-compiled binary answers a +//! different run. ⇒ this module is **always compiled and runtime-gated**, so the +//! split can be read off the record build itself, with the knob unset for the +//! record and set for the profile. +//! +//! It does not replace `instruments`: the spans there nest and reconstruct a +//! tree, which this does not attempt. This is one flat line per prove. +//! +//! # What the numbers mean +//! +//! Two kinds of field, and mixing them is the way to misread the line: +//! +//! - **Phase walls** (`prepass`, `main_commit`, `absorb`, `fused`) are measured +//! on the calling thread at phase boundaries. They do not overlap and they sum +//! to the prove. +//! - **Per-table sums** (everything inside `fused`) are accumulated from up to +//! `table_parallelism(num_airs)` driver threads. They are **worker-seconds, +//! not wall**, and their total runs up to `k` times over `fused`. +//! +//! The line says which is which by printing the per-table group under a +//! `tables[Σ]` prefix. +//! +//! # Cost when disabled +//! +//! [`enabled`] is a `OnceLock` load and a predictable branch; [`mark`] +//! returns `None` and no `Instant` is read. Every call site is at phase +//! granularity — tens of calls per table per prove, never inside a per-row loop +//! — so even enabled the instrumentation is far below the noise of the phases it +//! measures. +//! +//! # ⚠ One prove at a time +//! +//! The accumulators are process-global (the fused region's drivers are rayon +//! workers, so a thread-local cannot see them). Two `multi_prove` calls in +//! flight would therefore mix their numbers. On the LFM tree that cannot happen +//! — `lfm::device_permit` holds the card across `multi_prove` — and in the base +//! there is a single prover thread. Rather than assume it, [`begin`] counts +//! concurrent proves and the printed line carries a loud `OVERLAPPED` marker if +//! one was ever seen, so a mixed reading says so instead of looking clean. + +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +/// `LFM_PROVE_SPLIT=1` (any non-empty value other than `0`) turns the line on. +pub fn enabled() -> bool { + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| match std::env::var("LFM_PROVE_SPLIT") { + Ok(v) => !v.is_empty() && v != "0", + Err(_) => false, + }) +} + +/// Unix epoch seconds, for aligning a phase with an external sampler +/// (`nvidia-smi --query-gpu=... --format=csv` carries wall-clock timestamps). +/// +/// Printed rather than a process-relative offset on purpose: a relative stamp +/// needs the reader to know the process start, and the two logs being aligned +/// are written by different processes. +pub fn epoch_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or_default() +} + +/// A nanosecond accumulator. Public so call sites name their slot as a +/// constant rather than passing an index. +#[derive(Debug)] +pub struct Slot(AtomicU64); + +impl Slot { + const fn new() -> Self { + Self(AtomicU64::new(0)) + } + fn take(&self) -> f64 { + self.0.swap(0, Ordering::Relaxed) as f64 / 1e9 + } +} + +/// Start a timed region — `None`, and no clock read, when the knob is off. +#[inline] +pub fn mark() -> Option { + enabled().then(Instant::now) +} + +/// Close a region opened by [`mark`] into `slot`. +#[inline] +pub fn add(slot: &Slot, start: Option) { + if let Some(t) = start { + slot.0 + .fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed); + } +} + +// ── phase walls, one per prove, measured on the calling thread ────────────── +/// Domains, twiddles, walk orders and the VRAM gate. +pub static PREPASS: Slot = Slot::new(); +/// Round 1 main commits: the admitted device region, every table. +pub static MAIN_COMMIT: Slot = Slot::new(); +/// Host-only: the sequential absorption of the main roots into the transcript, +/// plus the LogUp challenge sampling that follows it. The one Fiat-Shamir +/// barrier between the main commits and the fused region. +pub static MAIN_ABSORB: Slot = Slot::new(); +/// The fused per-table region's wall: aux build, aux commit, rounds 2-4. +pub static FUSED: Slot = Slot::new(); + +// ── per-table sums inside `fused`: worker-seconds across `k` drivers ──────── +/// LogUp auxiliary trace construction. +pub static AUX_BUILD: Slot = Slot::new(); +/// Auxiliary trace LDE + Merkle commit. +pub static AUX_COMMIT: Slot = Slot::new(); +/// `build_round1` and the bus-contribution absorption. +pub static R1_ASSEMBLE: Slot = Slot::new(); +/// Round 2: constraint evaluation over the LDE. +pub static R2_CONSTRAINTS: Slot = Slot::new(); +/// Round 2: composition-polynomial decompose/extend. +pub static R2_DECOMPOSE: Slot = Slot::new(); +/// Round 2: composition-polynomial Merkle commit. +pub static R2_COMMIT: Slot = Slot::new(); +/// Round 3: out-of-domain evaluation. +pub static R3_OOD: Slot = Slot::new(); +/// Host-only: absorbing the OOD values into the table's transcript fork. +pub static R3_ABSORB: Slot = Slot::new(); +/// Round 4: DEEP composition + the FRI commit phase. +pub static R4_DEEP_FRI: Slot = Slot::new(); +/// Round 4: the proof-of-work grind. +/// +/// ⛔ READ THIS PER TABLE, NOT PER CALL — it is a Σ like every slot under the +/// `tables[Σ]` prefix, and a per-call threshold read against it is meaningless. +/// The scale that settles which arm ran: one 2^20 grind costs **~105 ms/table** +/// on the host at BLAKE3's 100 ns, and **~5,830 s/table** at RPX's measured +/// 5,561 ns. Single-digit milliseconds is the device. +/// +/// ⇒ and the line no longer makes anyone infer it: `on device` below counts the +/// device dispatches this call actually made. +pub static R4_GRIND: Slot = Slot::new(); +/// Round 4: query sampling, the FRI query phase and the DEEP openings. +pub static R4_QUERIES: Slot = Slot::new(); + +/// [`device_grinds_now`] as of the last report, so each line carries the device +/// grinds of ITS OWN call rather than the process total. +static GRIND_GPU_SEEN: AtomicU64 = AtomicU64::new(0); + +/// Device grinds so far on EITHER arm — the process total a line takes its +/// `on device` delta of. +/// +/// ⛔ It read the keccak arm's counter alone, which an RPX configuration never +/// moves: RPX device grinds count in `gpu_grind_calls_rpx`. So under RPX every +/// line read `0/airs on device` while every table ground on the card — the +/// WHIR block's root printed `0/11` beside the harness's own count of 11 RPX +/// device grinds (job 160). `prover/tests/rpx_grind_device.rs` reads this +/// around one RPX device grind. +pub fn device_grinds_now() -> u64 { + #[cfg(feature = "cuda")] + return crate::gpu_lde::gpu_grind_calls() + crate::gpu_lde::gpu_grind_calls_rpx(); + #[cfg(not(feature = "cuda"))] + return 0; +} + +/// This call's share of a monotone process-wide counter: read the previous mark +/// and replace it in ONE swap, then subtract. +/// +/// ⛔ Its own first draft swapped and then LOADED the same cell, which returns +/// the value just stored and makes the delta identically zero — a counter that +/// reads 0 on every line, which is exactly what "the device arm never fired" +/// looks like. A check that cannot fail is worse than no check. The test below +/// is written so that bug fails it. +#[inline] +fn grind_delta(seen: &AtomicU64, now: u64) -> u64 { + now.saturating_sub(seen.swap(now, Ordering::Relaxed)) +} +/// Proves that have started; the sequence number the line carries. +static SEQ: AtomicUsize = AtomicUsize::new(0); +/// Proves inside `multi_prove` right now — the overlap falsifier. +static IN_FLIGHT: AtomicUsize = AtomicUsize::new(0); +/// Set once if two proves were ever concurrent. Never cleared: one mixed +/// reading taints every later line, because a slot it polluted is only zeroed +/// by the print that reports it. +static OVERLAPPED: AtomicUsize = AtomicUsize::new(0); + +/// What [`begin`] hands back, so [`report`] can price the whole prove without +/// the caller threading a second timer. +/// +/// ⛔ It releases the concurrency count on DROP, not in [`report`]. `multi_prove` +/// has `?` early-returns, and a release that only ran on the success path would +/// leave the count stuck at one forever — every later line would then be stamped +/// `OVERLAPPED` by a prove that failed rather than by two that overlapped. A +/// false alarm on the falsifier is worse than no falsifier, because it reads as +/// evidence. +#[derive(Debug)] +pub struct ProveMark { + seq: usize, + start: Instant, + start_epoch: f64, +} + +impl Drop for ProveMark { + fn drop(&mut self) { + IN_FLIGHT.fetch_sub(1, Ordering::SeqCst); + } +} + +/// Open a prove. Cheap and inert when the knob is off. +pub fn begin() -> Option { + if !enabled() { + return None; + } + if IN_FLIGHT.fetch_add(1, Ordering::SeqCst) + 1 > 1 { + OVERLAPPED.store(1, Ordering::Relaxed); + } + Some(ProveMark { + seq: SEQ.fetch_add(1, Ordering::Relaxed), + start: Instant::now(), + start_epoch: epoch_secs(), + }) +} + +/// Close a prove and return its line, or `None` when the knob is off. +/// +/// Reading a slot CLEARS it, so a caller that drops the line silently would +/// hand this prove's time to the next one. There is exactly one caller and it +/// prints. +pub fn report(m: Option, num_airs: usize, total_rows: usize) -> Option { + let m = m?; + let wall = m.start.elapsed().as_secs_f64(); + let end_epoch = epoch_secs(); + + let prepass = PREPASS.take(); + let main_commit = MAIN_COMMIT.take(); + let absorb = MAIN_ABSORB.take(); + let fused = FUSED.take(); + + let aux_build = AUX_BUILD.take(); + let aux_commit = AUX_COMMIT.take(); + let r1_assemble = R1_ASSEMBLE.take(); + let r2_constraints = R2_CONSTRAINTS.take(); + let r2_decompose = R2_DECOMPOSE.take(); + let r2_commit = R2_COMMIT.take(); + let r3_ood = R3_OOD.take(); + let r3_absorb = R3_ABSORB.take(); + let r4_deep_fri = R4_DEEP_FRI.take(); + let r4_grind = R4_GRIND.take(); + let r4_queries = R4_QUERIES.take(); + // ★ THE GRIND'S ARM, COUNTED RATHER THAN INFERRED. The two device counters + // count one per table whose round-4 nonce search ran on device AND passed + // the host validity check — a device miss falls back to the CPU search and + // is NOT counted. So `n/airs on device` reading n = airs is the device arm + // firing on every table, and anything less names exactly how many fell + // back. ONE path, so the helper and its mark are live on every build: a + // non-cuda build simply never grinds on device, and `0/airs` is the true + // reading there rather than a cfg-ed-out field. + let grind_gpu = grind_delta(&GRIND_GPU_SEEN, device_grinds_now()); + + let table_sum = aux_build + + aux_commit + + r1_assemble + + r2_constraints + + r2_decompose + + r2_commit + + r3_ood + + r3_absorb + + r4_deep_fri + + r4_grind + + r4_queries; + // What the four phase walls leave over: the sequential prep between them + // (transcript forking, the per-table slot vectors, the shape and weight + // walks that are not inside `prepass`). Named rather than left implicit — + // an unattributed remainder is how a phase hides. + let other = (wall - prepass - main_commit - absorb - fused).max(0.0); + + Some(format!( + "PROVE SPLIT #{seq}{tainted}: airs {num_airs} · rows {total_rows} · \ + wall {wall:.2}s · t=[{t0:.3},{t1:.3}] · \ + prepass {prepass:.2} · main_commit {main_commit:.2} · absorb {absorb:.3} · \ + fused {fused:.2} · other {other:.2} || \ + tables[Σ] aux_build {aux_build:.2} · aux_commit {aux_commit:.2} · \ + r1_assemble {r1_assemble:.2} · r2_constraints {r2_constraints:.2} · \ + r2_decompose {r2_decompose:.2} · r2_commit {r2_commit:.2} · \ + r3_ood {r3_ood:.2} · r3_absorb {r3_absorb:.3} · r4_deep_fri {r4_deep_fri:.2} · \ + r4_grind {r4_grind:.2} ({grind_gpu}/{num_airs} on device) · \ + r4_queries {r4_queries:.2} · Σ {table_sum:.2}", + seq = m.seq, + tainted = if OVERLAPPED.load(Ordering::Relaxed) == 0 { + "" + } else { + " ⛔OVERLAPPED" + }, + t0 = m.start_epoch, + t1 = end_epoch, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// ⛔ The disabled path must read no clock. Asserted through the only + /// observable it has: `mark` returns `None`, so `add` cannot move a slot. + /// + /// The knob is a process-wide `OnceLock` and the test binary does not set + /// it, so this is the state every other test in the crate runs under too. + #[test] + fn disabled_is_inert() { + assert!(!enabled(), "the test binary must not set LFM_PROVE_SPLIT"); + static S: Slot = Slot::new(); + let t = mark(); + assert!(t.is_none(), "no clock is read when the knob is off"); + add(&S, t); + assert_eq!(S.0.load(Ordering::Relaxed), 0, "and no slot moved"); + assert!(begin().is_none(), "and no prove is opened"); + assert!(report(None, 3, 1024).is_none(), "and no line is produced"); + } + + /// ⛔ THE DELTA MUST REPORT THIS CALL, NOT ZERO AND NOT THE PROCESS TOTAL. + /// + /// Written against the bug it had: a swap followed by a load of the same + /// cell returns what was just stored, so the delta is identically zero and + /// every line reads "0 on device" whatever the device did. The second + /// assertion is the one that fires on it. + #[test] + fn the_grind_delta_is_per_call_and_a_swap_then_load_would_fail_it() { + static SEEN: AtomicU64 = AtomicU64::new(0); + assert_eq!(grind_delta(&SEEN, 27), 27, "the first call sees all of it"); + assert_eq!(grind_delta(&SEEN, 41), 14, "the second sees only its own"); + assert_eq!(grind_delta(&SEEN, 41), 0, "and a call that ground nothing"); + assert_eq!(grind_delta(&SEEN, 7), 0, "a reset counter never underflows"); + } + + /// A slot accumulates across threads and `take` CLEARS it — the property + /// that makes one line describe one prove rather than every prove so far. + #[test] + fn a_slot_sums_across_threads_and_clears_on_take() { + static S: Slot = Slot::new(); + // Written directly rather than through `mark`, which is off in this + // binary by design (see `disabled_is_inert`). + std::thread::scope(|sc| { + for _ in 0..4 { + sc.spawn(|| S.0.fetch_add(250_000_000, Ordering::Relaxed)); + } + }); + assert!((S.take() - 1.0).abs() < 1e-9, "four workers, 0.25s each"); + assert_eq!(S.take(), 0.0, "and the second read finds it cleared"); + } +} diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index faf512a72..058abc148 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -1,5 +1,6 @@ use std::any::Any; use std::marker::PhantomData; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; #[cfg(feature = "instruments")] use std::time::{Duration, Instant}; @@ -12,7 +13,6 @@ use math::fft::two_half_fft::TwoHalfTwiddles; use log::info; use math::field::traits::{IsField, IsSubFieldOf}; -use math::spill_safe::SpillSafe; use math::traits::AsBytes; use math::{ field::{element::FieldElement, traits::IsFFTField}, @@ -25,14 +25,14 @@ use rayon::prelude::{IntoParallelIterator, ParallelIterator}; #[cfg(feature = "debug-checks")] use crate::debug::validate_trace; use crate::fri; +use crate::leaf_layout::LeafLayout; use crate::lookup::LOGUP_NUM_CHALLENGES; use crate::proof::stark::{DeepPolynomialOpenings, PolynomialOpenings}; -#[cfg(feature = "disk-spill")] -use crate::storage_mode::StorageMode; +use crate::residency_mode::ResidencyMode; use crate::table::Table; use crate::trace::LDETraceTable; -use super::config::{BatchedMerkleTree, BatchedMerkleTreeBackend, Commitment}; +use super::config::{Commitment, DefaultStarkHash, StarkHash}; use super::constraints::evaluator::ConstraintEvaluator; use super::domain::Domain; use super::fri::fri_decommit::FriDecommitment; @@ -41,11 +41,36 @@ use super::lookup::BusPublicInputs; use super::proof::stark::{DeepPolynomialOpening, MultiProof, StarkProof}; use super::trace::TraceTable; use super::traits::AIR; -#[cfg(feature = "cuda")] +use crypto::merkle_tree::merkle::MerkleTree; use crypto::merkle_tree::proof::Proof; +use crypto::merkle_tree::traits::{IsMerkleTreeBackend, IsStreamingLeafBackend}; pub use crate::commitment::{keccak_leaves_bit_reversed, keccak_leaves_row_pair_bit_reversed}; +/// The DEEP trace poles of a grid of `num_eval_points` points: `z·gᵏ` for +/// every frame point, then the commitment-time answers' pole when there is +/// one (`crate::commit_ood`), which is the grid's last point. +fn trace_poles( + z: &FieldElement, + extra_pole: Option<&FieldElement>, + primitive_root: &FieldElement, + num_eval_points: usize, +) -> Vec> +where + F: IsSubFieldOf, + E: IsField, +{ + let frame_points = num_eval_points.saturating_sub(usize::from(extra_pole.is_some())); + let mut poles = Vec::with_capacity(num_eval_points); + let mut current = z.clone(); + for _ in 0..frame_points { + poles.push(current.clone()); + current = primitive_root * ¤t; + } + poles.extend(extra_pole.cloned()); + poles +} + /// A triple of (AIR, TraceTable, PublicInputs) for proving. type AirTracePair<'a, Field, FieldExtension, PI> = ( &'a dyn AIR, @@ -53,20 +78,34 @@ type AirTracePair<'a, Field, FieldExtension, PI> = ( &'a PI, ); -/// A default STARK prover implementing `IsStarkProver`. -pub struct Prover< +/// A default STARK prover implementing `IsStarkProver`, generic over the +/// commitment configuration `H`. +/// +/// `H` rides on the concrete type rather than defaulting on the trait: a +/// defaulted trait parameter would be uninferable at a bare +/// `Prover::multi_prove(..)` call, whereas an alias pins it. That is what keeps +/// every existing call site resolving unchanged — see [`Prover`]. +pub struct GenericProver< Field: IsSubFieldOf + IsFFTField + Send + Sync, FieldExtension: Send + Sync + IsField, PI, + H, > { - p: PhantomData<(Field, FieldExtension, PI)>, + p: PhantomData<(Field, FieldExtension, PI, H)>, } +/// The production prover: [`GenericProver`] at the default commitment +/// configuration — BLAKE3 off `cuda`, keccak under it. See +/// [`DefaultStarkHash`](crate::config::DefaultStarkHash). +pub type Prover = + GenericProver; + impl< Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, FieldExtension: Send + Sync + IsField + 'static, PI, -> IsStarkProver for Prover + H: StarkHash, +> IsStarkProver for GenericProver where FieldElement: math::traits::ByteConversion, FieldElement: math::traits::ByteConversion, @@ -84,15 +123,90 @@ pub enum ProvingError { /// proof an honest verifier always rejects — fail fast on the prover side /// with a localized error instead. PrecomputedCommitmentMismatch, - /// I/O failure while spilling prover state (traces, LDE, Merkle trees) to disk: - /// out of disk space, fd exhaustion, or mmap failure. - #[cfg(feature = "disk-spill")] - DiskSpill(String), + /// The AIR has no preprocessed commitment for the table's leaf layout + /// (S2: a one-row layout whose static root was never generated). A hard + /// error, never a silent recompute: proving on would either + /// take the other layout's root — a proof every verifier rejects — or + /// rebuild a whole preprocessed LDE and tree behind the operator's back. + PrecomputedCommitmentMissing(String), /// An internal FFT/LDE computation failed (e.g. domain size exceeds the /// field's two-adicity, or a degenerate coset offset). Distinct from /// `WrongParameter` because the cause is internal prover machinery, not a /// caller-supplied parameter. Carries the underlying `FFTError`'s message. Fft(String), + /// The device path is the production path and it was unavailable for a + /// table after its device-side recovery ran: a resident aux LDE that + /// declined again after the drain-and-retry. Host RAM is a cache, not a + /// compute path, so there is no host arm to continue on; the message names + /// the table, the shape and the live device posture. + DevicePath(String), + /// Under a thread's device cap ([`cap_device_bytes`]), a table whose device + /// set alone is over it: refused before any device work, so the caller can + /// prove it later without the cap. `bytes` is the table's larger device + /// set (its main commit's or its rounds'). + OverDeviceCap { + table: String, + bytes: u64, + cap: u64, + }, +} + +thread_local! { + /// This thread's device cap: what `multi_prove` admits its tables against + /// instead of the card's whole budget. `None` (the default) is the budget. + static DEVICE_CAP: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Cap the device bytes this thread's `multi_prove` admits its tables against +/// at `bytes` until the guard drops, which puts the previous cap back. For a +/// proof run beside another on the card: it holds a reservation of `bytes` and +/// must stay inside it. A table whose device set alone is over the cap is +/// refused ([`ProvingError::OverDeviceCap`]) before any device work, rather +/// than admitted alone past it as an uncapped prove admits it. +pub fn cap_device_bytes(bytes: u64) -> DeviceCapGuard { + let previous = DEVICE_CAP.with(|c| c.replace(Some(bytes))); + DeviceCapGuard { + previous, + _this_thread: std::marker::PhantomData, + } +} + +/// This thread's device cap, if one is set ([`cap_device_bytes`]). +pub fn device_cap() -> Option { + DEVICE_CAP.with(std::cell::Cell::get) +} + +/// Puts the thread's previous device cap back when it drops. Not `Send`: the +/// cap belongs to the thread that set it. +pub struct DeviceCapGuard { + previous: Option, + _this_thread: std::marker::PhantomData<*const ()>, +} + +impl Drop for DeviceCapGuard { + fn drop(&mut self) { + DEVICE_CAP.with(|c| c.set(self.previous)); + } +} + +/// The first table whose device set (the larger of its two phases') is over +/// `cap`, refused by name. +fn refuse_over_cap(names: &[String], phases: [&[u64]; 2], cap: u64) -> Result<(), ProvingError> { + for (idx, name) in names.iter().enumerate() { + let bytes = phases + .iter() + .filter_map(|p| p.get(idx).copied()) + .max() + .unwrap_or(0); + if bytes > cap { + return Err(ProvingError::OverDeviceCap { + table: name.clone(), + bytes, + cap, + }); + } + } + Ok(()) } impl From for ProvingError { @@ -106,28 +220,28 @@ impl From for ProvingError { /// separate Merkle tree over their precomputed columns, hence the optional /// `precomputed_tree`/`precomputed_root` pair and the `num_precomputed_cols` /// index used when opening positions. -pub(crate) struct TableCommit +pub(crate) struct TableCommit where - FieldElement: AsBytes, + FieldElement: AsBytes + Sync + Send, { /// Merkle tree over the trace columns (multiplicities only for preprocessed tables). - pub(crate) tree: Arc>, + pub(crate) tree: Arc>>, /// Root of `tree`. pub(crate) root: Commitment, /// Preprocessed tables only: Merkle tree over precomputed columns. - pub(crate) precomputed_tree: Option>>, + pub(crate) precomputed_tree: Option>>>, /// Preprocessed tables only: root of `precomputed_tree`. pub(crate) precomputed_root: Option, /// Preprocessed tables only: number of precomputed columns. Zero otherwise. pub(crate) num_precomputed_cols: usize, } -impl TableCommit +impl TableCommit where - FieldElement: AsBytes, + FieldElement: AsBytes + Sync + Send, { /// Build a `TableCommit` for a plain (non-preprocessed) table. - fn plain(tree: BatchedMerkleTree, root: Commitment) -> Self { + fn plain(tree: MerkleTree>, root: Commitment) -> Self { Self { tree: Arc::new(tree), root, @@ -141,9 +255,9 @@ where /// arrives as an `Arc` because it may be shared from the process-wide /// cache (see [`precomputed_tree_cache_get`]). fn preprocessed( - tree: BatchedMerkleTree, + tree: MerkleTree>, root: Commitment, - precomputed_tree: Arc>, + precomputed_tree: Arc>>, precomputed_root: Commitment, num_precomputed_cols: usize, ) -> Self { @@ -180,53 +294,254 @@ where /// same DECODE/BITWISE/range tables once per epoch — those trees are /// execution-independent; only the multiplicity columns change per run. /// Type-erased so one static serves every field instantiation. -fn precomputed_tree_cache() --> &'static Mutex>> { - static CACHE: OnceLock< - Mutex>>, - > = OnceLock::new(); +/// +/// # ⚠ It is CAPPED in recursion, and why +/// +/// The cache was built for the BASE, where the same execution-independent +/// DECODE/BITWISE/range trees recur every epoch and hit. ✓ Measured on the +/// block tree (2026-09-14): the recursion inserts **7–9 program-dependent trees +/// per proof** whose roots never recur — a child's label is a program constant, +/// so every node proves a distinct program — while still hitting ~35% on the +/// shared tables. Unbounded, that reached **357 entries at ~99.7 MiB each = +/// essentially ALL of the 34.2 GiB of live host allocation at the end of a +/// block**, and it is what made the interior's host peaks rise while the levels +/// shrank. +/// +/// ⇒ [`PRECOMPUTED_TREE_CACHE_CAP_ENV`] bounds it, LRU by recency of use. +/// ✓ Eviction is **semantically free**: the doc above is the argument — the +/// lookup key IS the root a rebuild would be checked against, so a hit needs no +/// re-verification and a miss is only a rebuild. Nothing a proof commits to can +/// move. +/// +/// The value carries a recency tick beside the tree. The cap is small (tens), so +/// the O(n) scan for the least-recently-used entry costs less than any ordering +/// structure would. +type PrecomputedTreeMap = + std::collections::HashMap)>; + +/// The cache key: the root AND the trees' rows per leaf (S2). The root alone +/// already differs between leaf layouts (a one-row leaf hashes other bytes), +/// so two layouts cannot alias; the layout is in the key anyway so that +/// argument is not a hash-collision argument. +type PrecomputedTreeKey = (Commitment, usize); + +fn precomputed_tree_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); CACHE.get_or_init(|| Mutex::new(std::collections::HashMap::new())) } -fn precomputed_tree_cache_get( +/// Entries to keep. **Unset = unbounded = the behaviour before the cap +/// existed**, so a control arm is the same binary with the knob absent. +pub const PRECOMPUTED_TREE_CACHE_CAP_ENV: &str = "LFM_PRECOMPUTED_TREE_CACHE_CAP"; + +/// The cap, read once. `None` = unbounded. A value of `0` is treated as unset +/// rather than as "cache nothing": a zero-size cache would evict on every insert +/// and turn every lookup into a miss, which is a configuration nobody wants and +/// a typo everybody makes. +fn precomputed_tree_cache_cap() -> Option { + static CAP: OnceLock> = OnceLock::new(); + *CAP.get_or_init(|| { + std::env::var(PRECOMPUTED_TREE_CACHE_CAP_ENV) + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&n| n > 0) + }) +} + +/// Monotone recency clock. Bumped on every hit and every insert, so "least +/// recently USED" means used, not merely inserted — which is the whole point: +/// the shared tables that hit must survive eviction of the program-dependent +/// ones that never do. +static PRECOMPUTED_TREE_TICK: AtomicU64 = AtomicU64::new(0); + +/// Evictions performed, for the line that reports the cache. ★ The falsifier's +/// own counter: a run whose MISSES rise with the cap in place has a cap below +/// its working set, and these two numbers are what say so. +static PRECOMPUTED_TREE_EVICTIONS: AtomicU64 = AtomicU64::new(0); + +/// `(entries, hits, misses, evictions)` for the cache. +pub fn precomputed_tree_cache_stats() -> (usize, u64, u64, u64) { + let (hits, misses) = precomputed_tree_cache_hit_miss(); + ( + precomputed_tree_cache_entries(), + hits, + misses, + PRECOMPUTED_TREE_EVICTIONS.load(Ordering::Relaxed), + ) +} + +/// Insert into `map`, evicting the least recently used entry while the map is +/// over `cap`. +/// +/// ⛔ Split out from [`precomputed_tree_cache_put`] and taking `cap` as an +/// ARGUMENT so the tests can exercise a real cap. The live cap is a process-wide +/// `OnceLock` read from the environment; a test that could only reach it through +/// that would exercise the UNBOUNDED path and pass whatever the eviction did — +/// a check that cannot fail. +/// Returns the keys it evicted. +fn precomputed_tree_insert_capped( + map: &mut PrecomputedTreeMap, + root: PrecomputedTreeKey, + tree: Arc, + cap: Option, +) -> Vec { + let tick = PRECOMPUTED_TREE_TICK.fetch_add(1, Ordering::Relaxed); + map.insert(root, (tick, tree)); + let mut evicted = Vec::new(); + let Some(cap) = cap else { return evicted }; + while map.len() > cap { + // The cap is tens of entries, so this scan is cheaper than maintaining + // an order. `expect` is unreachable: the loop condition implies len > 0. + let lru = map + .iter() + .min_by_key(|(_, (t, _))| *t) + .map(|(k, _)| *k) + .expect("a map with len > cap >= 1 is non-empty"); + map.remove(&lru); + evicted.push(lru); + PRECOMPUTED_TREE_EVICTIONS.fetch_add(1, Ordering::Relaxed); + } + evicted +} + +/// ★★ HOW MANY DISTINCT SHAPES THE TWO PROCESS-GLOBAL CACHES HOLD. +/// +/// Both are insert-only — ✓ nothing removes from either — so **the entry count +/// IS the cumulative miss count**, and the pair is the falsifier for a specific +/// hypothesis: that the interior's rising host floor is these caches rather than +/// retained children. Measured, `allocated` rises 19.3 → 28.9 → 31.5 → 32.7 → +/// 33.7 → 34.2 GiB across levels holding 19 → 10 → 5 → 3 → 2 → 1 children, and +/// neither "all children freed" nor "none freed" reproduces that shape, while a +/// monotone insert-only shape-keyed accumulator does. +/// +/// ⇒ **Entries plateauing while `allocated` keeps rising REFUTES the cache +/// hypothesis.** Entries and `allocated` rising together, with both increments +/// shrinking, supports it. Either way the reading is a subtraction, not an +/// argument. +/// +/// ⓘ `crate::tests::domain_cache_stats` already counts hits and misses and +/// cannot answer this: it is `#[cfg(test)]` on THIS crate, so it is compiled out +/// whenever `stark` is a dependency — which is every LFM run. These are always +/// compiled. +pub fn domain_twiddle_cache_entries() -> usize { + domain_twiddle_cache() + .lock() + .map(|c| c.len()) + .unwrap_or(usize::MAX) +} + +/// Companion to [`domain_twiddle_cache_entries`], for the precomputed Merkle +/// trees. Keyed by root, so one entry per distinct preprocessed table shape. +pub fn precomputed_tree_cache_entries() -> usize { + precomputed_tree_cache() + .lock() + .map(|c| c.len()) + .unwrap_or(usize::MAX) +} + +/// ★ WHETHER THE TREE CACHE EVER HITS — the reading that decides what it is. +/// +/// Its own doc says it was built for the BASE's repeating DECODE/BITWISE/range +/// trees, which are execution-independent. ⚠ In RECURSION the key is the +/// precomputed ROOT and every level proves N DISTINCT programs (a child's label +/// is a program constant), so the program-dependent tables would MISS on every +/// proof forever — one insert per proof per table, never hitting, never evicted. +/// +/// That is a claim about hit BEHAVIOUR, not about size, so counting hits and +/// misses tests it directly where an entry count or a byte total only proxies +/// it. Near-zero hits in the tree levels ⇒ the cache is a proof-count-linear +/// accumulator and eviction at harvest is a lever; a high hit rate ⇒ it is not, +/// and the interior's rising floor is something else. +static PRECOMPUTED_TREE_HITS: AtomicU64 = AtomicU64::new(0); +static PRECOMPUTED_TREE_MISSES: AtomicU64 = AtomicU64::new(0); + +/// The node bytes of each tree the cache holds now, by key: inserts add, the +/// cap's evictions remove ([`precomputed_tree_cache_live_bytes`]). +fn precomputed_tree_live() -> &'static Mutex> { + static LIVE: OnceLock>> = + OnceLock::new(); + LIVE.get_or_init(|| Mutex::new(std::collections::HashMap::new())) +} + +/// The node bytes the precomputed-tree cache holds now (a measurement). +pub fn precomputed_tree_cache_live_bytes() -> u64 { + precomputed_tree_live() + .lock() + .map(|m| m.values().sum()) + .unwrap_or(0) +} + +/// `(hits, misses)` on the precomputed-tree cache since the process started. +pub fn precomputed_tree_cache_hit_miss() -> (u64, u64) { + ( + PRECOMPUTED_TREE_HITS.load(Ordering::Relaxed), + PRECOMPUTED_TREE_MISSES.load(Ordering::Relaxed), + ) +} + +pub(crate) fn precomputed_tree_cache_get( root: &Commitment, -) -> Option>> -where - FieldElement: AsBytes, -{ - let cache = precomputed_tree_cache().lock().unwrap(); - cache + rows_per_leaf: usize, +) -> Option>> { + let root = &(*root, rows_per_leaf); + let mut cache = precomputed_tree_cache().lock().unwrap(); + let out = cache .get(root) - .cloned() - .and_then(|any| any.downcast::>().ok()) + .map(|(_, any)| Arc::clone(any)) + .and_then(|any| any.downcast::>().ok()); + // ★ A HIT REFRESHES RECENCY. Without this the cap would evict by insertion + // order, which throws away exactly the shared tables that keep hitting and + // keeps the program-dependent ones that never will. + if out.is_some() + && let Some(slot) = cache.get_mut(root) + { + slot.0 = PRECOMPUTED_TREE_TICK.fetch_add(1, Ordering::Relaxed); + } + // ⛔ Counted on the DOWNCAST result, not on the map lookup: a key that is + // present but holds another backend's tree is a miss to the caller, and + // counting the lookup would report a hit the caller never got. + if out.is_some() { + PRECOMPUTED_TREE_HITS.fetch_add(1, Ordering::Relaxed); + } else { + PRECOMPUTED_TREE_MISSES.fetch_add(1, Ordering::Relaxed); + } + out } -fn precomputed_tree_cache_put( +pub(crate) fn precomputed_tree_cache_put( root: Commitment, - tree: Arc>, -) where - FieldElement: AsBytes, -{ - precomputed_tree_cache() - .lock() - .unwrap() - .insert(root, tree as Arc); + rows_per_leaf: usize, + tree: Arc>, +) { + let bytes = std::mem::size_of_val(tree.nodes()) as u64; + let evicted = precomputed_tree_insert_capped( + &mut precomputed_tree_cache().lock().unwrap(), + (root, rows_per_leaf), + tree as Arc, + precomputed_tree_cache_cap(), + ); + let mut live = precomputed_tree_live().lock().unwrap(); + for key in evicted { + live.remove(&key); + } + live.insert((root, rows_per_leaf), bytes); } /// A container for the results of the first round of the STARK Prove protocol. -pub(crate) struct Round1 +pub(crate) struct Round1 where - Field: IsSubFieldOf + IsFFTField, - FieldExtension: IsField, - FieldElement: AsBytes, - FieldElement: AsBytes, + Field: IsSubFieldOf + IsFFTField + 'static, + FieldExtension: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { /// The table of evaluations over the LDE of the main and auxiliary trace tables. pub(crate) lde_trace: LDETraceTable, /// Commitment to the main trace. - pub(crate) main: TableCommit, + pub(crate) main: TableCommit, /// Commitment to the auxiliary (RAP) trace, if any. - pub(crate) aux: Option>, + pub(crate) aux: Option>, /// The challenges of the RAP round. pub(crate) rap_challenges: Vec>, /// Bus interaction public inputs (initial and final aux column values). @@ -237,25 +552,26 @@ where /// and (under cuda) the optional device LDE buffer kept alive for downstream /// rounds when the R1 fused GPU pipeline ran. #[cfg(feature = "cuda")] -type MainCommitTuple = ( - TableCommit, +type MainCommitTuple = ( + TableCommit, (Vec>, usize), Option, ); #[cfg(not(feature = "cuda"))] -type MainCommitTuple = (TableCommit, (Vec>, usize)); +type MainCommitTuple = (TableCommit, (Vec>, usize)); /// Round 1 commitment artifacts — Merkle trees, roots, challenges, and bus inputs. /// Borrowed (not consumed) when building `Round1`. -pub(crate) struct Round1Commitments +pub(crate) struct Round1Commitments where - Field: IsFFTField + IsSubFieldOf, - FieldExtension: IsField, - FieldElement: AsBytes, - FieldElement: AsBytes, + Field: IsFFTField + IsSubFieldOf + 'static, + FieldExtension: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { - main: TableCommit, - aux: Option>, + main: TableCommit, + aux: Option>, rap_challenges: Vec>, bus_public_inputs: Option>, } @@ -287,12 +603,26 @@ struct Lde { gpu_aux: Option, } -impl Round1Commitments +/// A table's Round-1 main LDE, held between the main commit and the table's +/// fused task. +/// +/// `Dropped` is the `ResidencyMode::RecomputeLde` state. It carries no buffer +/// at all, so a consumer added between Round 1 and the fused task cannot read +/// empty data believing it is an LDE — it has to handle the recompute arm or +/// fail to compile. That is the loud guard for the one real risk in dropping +/// the buffer: a retention point the audit missed. +enum MainLdeSlot { + Retained((Vec>, usize)), + Dropped { num_cols: usize }, +} + +impl Round1Commitments where - Field: IsFFTField + IsSubFieldOf + Send + Sync, - FieldExtension: IsField + Send + Sync, - FieldElement: AsBytes, - FieldElement: AsBytes, + Field: IsFFTField + IsSubFieldOf + Send + Sync + 'static, + FieldExtension: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { /// Build a `Round1` by consuming a `Lde` and borrowing commitment data. /// The `TableCommit::share` calls are cheap — only bump Arc refcounts. @@ -301,7 +631,7 @@ where lde: Lde, step_size: usize, blowup_factor: usize, - ) -> Round1 { + ) -> Round1 { let (main_data, num_main_cols) = lde.main; let (aux_data, num_aux_cols) = lde.aux; @@ -397,12 +727,22 @@ pub(crate) struct LdeTwiddles { /// `two_half_fwd` size-`n·blowup` forward. two_half_inv: TwoHalfTwiddles, two_half_fwd: TwoHalfTwiddles, + /// Size-`n` FORWARD set, built lazily — only the batched phase-4 coset + /// evaluation wants it (a full LDE's forward set is size `n·blowup`). + two_half_fwd_n: OnceLock>, coset_weights: Vec>, /// Composition half-extension cache, initialized only when the degree-2 /// decomposition path actually runs on CPU. composition: OnceLock>, /// `1/(2·g·ωⁱ)` for the degree-2 quotient decomposition — see [`Self::inv_2x`]. inv_2x: OnceLock>>>, + /// Weights `g⁻ʲ/n` for extending the two halves from `n` points when `H` + /// was evaluated on the `2n`-point sub-coset — see + /// [`Self::composition_subcoset_weights`]. + composition_subcoset_weights: OnceLock>>, + /// Size-`n` inverse twiddles for that extension on the host (the device + /// path needs only the weights), built on first host use. + composition_subcoset_inv: OnceLock>, } pub(crate) struct CompositionLdeTwiddles { @@ -414,27 +754,33 @@ pub(crate) struct CompositionLdeTwiddles { weights: Vec>, } +/// Composition half-extension weights `g⁻ʲ / half_size`, `j < half_size`. The +/// constraint-quotient halves live on a g²-coset of size `half_size`; the +/// unnormalized iFFT yields `half_size·cⱼ·(g²)ʲ` and these weights turn that into +/// `cⱼ·gʲ` for the forward FFT onto the g-coset. Shared by the whole-coset +/// halves (`half_size = lde/2`) and the sub-coset halves (`half_size = n`). +fn composition_half_weights( + half_size: usize, + offset: &FieldElement, +) -> Vec> { + let half_size_fe = FieldElement::::from(half_size as u64); + let inv_half_size_offset = (&half_size_fe * offset) + .inv() + .expect("half_size and coset offset are non-zero"); + let half_size_inv = offset * &inv_half_size_offset; + let offset_inv = &half_size_fe * &inv_half_size_offset; + let mut w = Vec::with_capacity(half_size); + let mut cur = half_size_inv; + for _ in 0..half_size { + w.push(cur.clone()); + cur = &cur * &offset_inv; + } + w +} + impl CompositionLdeTwiddles { fn new(half_size: usize, offset: &FieldElement) -> Self { - // Composition half-extension weights: g⁻ʲ / half_size. The constraint- - // quotient halves live on the g²-coset of size `half_size`; the unnormalized - // iFFT yields `n·cⱼ·(g²)ʲ` and these weights turn that into `cⱼ·gʲ` for the - // forward FFT onto the g-coset. - let half_size_fe = FieldElement::::from(half_size as u64); - let inv_half_size_offset = (&half_size_fe * offset) - .inv() - .expect("half_size and coset offset are non-zero"); - let half_size_inv = offset * &inv_half_size_offset; - let offset_inv = &half_size_fe * &inv_half_size_offset; - let weights = { - let mut w = Vec::with_capacity(half_size); - let mut cur = half_size_inv; - for _ in 0..half_size { - w.push(cur.clone()); - cur = &cur * &offset_inv; - } - w - }; + let weights = composition_half_weights(half_size, offset); Self { inv: LayerTwiddles::::new_inverse(half_size.trailing_zeros() as u64) @@ -446,25 +792,49 @@ impl CompositionLdeTwiddles { } } +/// `[n_inv, n_inv·g, n_inv·g², …, n_inv·g^(n−1)]` — the iFFT normalization folded +/// together with the coset generator's powers, which is what the row-major LDE +/// (host and device alike) multiplies a column by before the forward transform. +/// +/// ⛔ ONE DERIVATION, because it now has two readers. [`LdeTwiddles::new`] builds +/// a table's weights for the prove, and +/// [`gpu_lde::try_commit_row_major`](crate::gpu_lde::try_commit_row_major) builds +/// them for a preprocessed group committed outside any prove. Written inline in +/// both, a change to the normalization would move one path's roots and not the +/// other's — and the two are required to agree bit for bit, since a proof +/// declares the root the artifact build produced. +pub(crate) fn coset_weights( + domain_size: usize, + offset: &FieldElement, +) -> Vec> { + let domain_size_inv = FieldElement::::from(domain_size as u64) + .inv() + .expect("domain_size is a power of two"); + let mut w = Vec::with_capacity(domain_size); + let mut offset_power = domain_size_inv; + for _ in 0..domain_size { + w.push(offset_power.clone()); + offset_power = offset * &offset_power; + } + w +} + impl LdeTwiddles { + /// The bytes its production tables hold once built: the two two-half sets + /// and the coset weights (the sets built on first use come later and are + /// not counted). + pub(crate) fn heap_bytes(&self) -> usize { + self.two_half_inv.heap_bytes() + + self.two_half_fwd.heap_bytes() + + self.coset_weights.capacity() * std::mem::size_of::>() + } + /// Construct twiddles and coset weights for a domain of the given size and blowup factor. pub(crate) fn new(domain: &Domain) -> Self { let domain_size = domain.interpolation_domain_size; let lde_size = domain_size * domain.blowup_factor; - let domain_size_inv = FieldElement::::from(domain_size as u64) - .inv() - .expect("domain_size is power of two"); - let offset = &domain.coset_offset; - let coset_weights = { - let mut w = Vec::with_capacity(domain_size); - let mut offset_power = domain_size_inv; - for _ in 0..domain_size { - w.push(offset_power.clone()); - offset_power = offset * &offset_power; - } - w - }; + let coset_weights = coset_weights(domain_size, &domain.coset_offset); Self { #[cfg(any(test, feature = "test-utils", feature = "debug-checks"))] @@ -477,12 +847,23 @@ impl LdeTwiddles { .expect("valid inverse two-half twiddles"), two_half_fwd: TwoHalfTwiddles::::new(lde_size.trailing_zeros() as usize, false) .expect("valid forward two-half twiddles"), + two_half_fwd_n: OnceLock::new(), coset_weights, composition: OnceLock::new(), inv_2x: OnceLock::new(), + composition_subcoset_weights: OnceLock::new(), + composition_subcoset_inv: OnceLock::new(), } } + /// The size-`n` forward set for the phase-4 coset evaluation, built once. + pub(crate) fn fwd_n(&self, domain_size: usize) -> &TwoHalfTwiddles { + self.two_half_fwd_n.get_or_init(|| { + TwoHalfTwiddles::::new(domain_size.trailing_zeros() as usize, false) + .expect("valid size-n forward two-half twiddles") + }) + } + fn composition(&self, domain: &Domain) -> &CompositionLdeTwiddles { let lde_size = domain.interpolation_domain_size * domain.blowup_factor; let half_size = lde_size / 2; @@ -514,6 +895,130 @@ impl LdeTwiddles { Arc::new(inv) }) } + + /// `g⁻ʲ/n` (`n` = trace length) for extending the degree-2 halves from the + /// `n` points the sub-coset split yields to the whole LDE (see + /// [`composition_eval_stride`]). Sequential to build; pre-filled with the + /// rest of the domain's caches. + fn composition_subcoset_weights(&self, domain: &Domain) -> &[FieldElement] { + self.composition_subcoset_weights.get_or_init(|| { + composition_half_weights(domain.interpolation_domain_size, &domain.coset_offset) + }) + } + + /// The size-`n` inverse twiddles the HOST sub-coset extension needs (the + /// forward set is the whole-coset extension's, of the same LDE size). + fn composition_subcoset_inv(&self, domain: &Domain) -> &LayerTwiddles { + self.composition_subcoset_inv.get_or_init(|| { + LayerTwiddles::::new_inverse(domain.interpolation_domain_size.trailing_zeros() as u64) + .expect("valid composition sub-coset inverse twiddles") + }) + } +} + +/// Tables whose composition was evaluated on the sub-coset (any stride > 1), +/// process-wide — so a test can tell the path ran, not only that bytes held. +static COMPOSITION_SUBCOSET_TABLES: AtomicU64 = AtomicU64::new(0); + +/// How many tables so far had their composition evaluated on the sub-coset. +pub fn composition_subcoset_tables() -> u64 { + COMPOSITION_SUBCOSET_TABLES.load(Ordering::Relaxed) +} + +/// Runtime override of the sub-coset composition evaluation: 0 = follow the +/// environment, 1 = forced off, 2 = forced on. +static COMPOSITION_SUBCOSET_OVERRIDE: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(0); + +/// Force the sub-coset composition evaluation on (`Some(true)`), off +/// (`Some(false)`), or back to the environment's choice (`None`). It changes +/// how the prover computes the composition parts, never their values: the +/// proof bytes are the same either way. +pub fn set_composition_subcoset(on: Option) { + let v = match on { + None => 0, + Some(false) => 1, + Some(true) => 2, + }; + COMPOSITION_SUBCOSET_OVERRIDE.store(v, Ordering::Relaxed); +} + +/// Whether a degree-2 composition is evaluated on the `2n`-point sub-coset +/// instead of the whole LDE coset. On by default; +/// `LAMBDA_VM_COMPOSITION_SUBCOSET=0` turns it off (an A/B knob — the proof +/// bytes do not depend on it). +pub(crate) fn composition_subcoset_enabled() -> bool { + match COMPOSITION_SUBCOSET_OVERRIDE.load(Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + static ENV: OnceLock = OnceLock::new(); + *ENV.get_or_init(|| { + std::env::var("LAMBDA_VM_COMPOSITION_SUBCOSET") + .map(|v| v != "0") + .unwrap_or(true) + }) +} + +/// The LDE-row stride the composition polynomial `H` is evaluated at. +/// +/// `H` has degree `< parts·n` and is committed as `parts` pieces of degree +/// `< n`. With two parts, `H(x) = H₀(x²) + x·H₁(x²)` and the split is pointwise +/// over pairs `(x, −x)`, so it needs `H` only on a coset that is closed under +/// negation and whose squares number `n`: the `2n`-point sub-coset `g·⟨ωˢ⟩`, +/// `s = blowup/2` (`ω` the LDE generator; `−1 = ω^(lde/2)` lies in `⟨ωˢ⟩`). +/// `H₀`/`H₁` then come out on the `n` points `g²·⟨ω²ˢ⟩`, which determine them, +/// and extend to the LDE exactly as from `2n` points — the same polynomials at +/// the same points, so the same parts and the same proof. The constraint walk +/// runs on `2n` rows instead of `blowup·n`. +/// +/// Returns 1 (the whole coset, the original path) for any other part count, +/// for blowup 2 (where the sub-coset IS the coset), or with the knob off. +pub(crate) fn composition_eval_stride(blowup_factor: usize, number_of_parts: usize) -> usize { + if number_of_parts == 2 + && blowup_factor >= 4 + && blowup_factor.is_power_of_two() + && composition_subcoset_enabled() + { + blowup_factor / 2 + } else { + 1 + } +} + +/// Frees a main trace a [`crate::trace::MainDevProducer`] built, once the R1 +/// main commit that read it is done with it. +/// +/// The commit read the buffer on its own stream, and dropping a `CudaSlice` +/// frees it on the stream that allocated it — with cudarc's event tracking off +/// (`math_cuda::device`), nothing orders that free after the read. So the +/// producer's stream first waits for the commit's `ready` event, which follows +/// every read of its input. A handle without one was drained to the host before +/// it was returned (the read is long done); no handle means the commit declined +/// or failed, and the whole device is waited for (rare). +#[cfg(feature = "cuda")] +fn release_produced_after( + produced: Option>, + handle: Option<&math_cuda::lde::GpuLdeBase>, +) { + let Some(buf) = produced else { return }; + match handle { + Some(h) => { + if let Some(ev) = &h.ready + && buf.stream().wait(ev.event()).is_err() + && let Ok(be) = math_cuda::device::backend() + { + let _ = be.ctx.synchronize(); + } + } + None => { + if let Ok(be) = math_cuda::device::backend() { + let _ = be.ctx.synchronize(); + } + } + } + drop(buf); } /// Process-wide `Domain` + `LdeTwiddles` cache keyed by @@ -537,30 +1042,351 @@ fn domain_twiddle_cache() -> &'static std::sync::Mutex< CACHE.get_or_init(Default::default) } -fn domain_and_twiddles(air: &A, trace_length: usize) -> (Arc>, Arc>) +pub(crate) fn domain_and_twiddles( + air: &A, + trace_length: usize, +) -> (Arc>, Arc>) where F: IsFFTField + 'static, FieldElement: Send + Sync, A: AIR + ?Sized, +{ + let (d, t, _) = cached_domain_and_twiddles::( + air.options().blowup_factor as usize, + air.options().coset_offset, + trace_length, + ); + (d, t) +} + +/// Runtime override of [`warm_prove_constants_enabled`]: 0 = the environment's +/// choice, 1 = off, 2 = on. +static WARM_PROVE_CONSTANTS_OVERRIDE: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(0); + +/// Force [`warm_prove_constants_enabled`] on, off, or back to the environment. +pub fn set_warm_prove_constants(on: Option) { + let v = match on { + None => 0, + Some(false) => 1, + Some(true) => 2, + }; + WARM_PROVE_CONSTANTS_OVERRIDE.store(v, Ordering::Relaxed); +} + +/// Whether a prove's first-use costs are taken out of its card window: the +/// per-domain constants a table's prove builds on first use warmed beforehand +/// ([`warm_prove_constants`]), and the composition drain routed through the +/// staging pairs rather than the growing pinned slab. Neither changes a value: +/// the proof bytes are the same. On by default; +/// `LAMBDA_VM_WARM_PROVE_CONSTANTS=0` turns both off (an A/B knob). +pub fn warm_prove_constants_enabled() -> bool { + match WARM_PROVE_CONSTANTS_OVERRIDE.load(Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + static ENV: OnceLock = OnceLock::new(); + *ENV.get_or_init(|| { + std::env::var("LAMBDA_VM_WARM_PROVE_CONSTANTS") + .map(|v| v != "0") + .unwrap_or(true) + }) +} + +/// Constants [`warm_prove_constants`] built or uploaded so far, process-wide. +static WARMED_CONSTANTS: AtomicU64 = AtomicU64::new(0); + +/// How many constants [`warm_prove_constants`] built or uploaded so far. +pub fn warmed_prove_constants() -> u64 { + WARMED_CONSTANTS.load(Ordering::Relaxed) +} + +/// Builds, before a prove takes the card, the per-domain constants its tables +/// would otherwise build on first use inside it: each boundary constraint's +/// zerofier inverse over the LDE (an LDE-size host batch inversion), and — on +/// the card, for tables that clear the device floor — their resident copies, +/// the degree-2 split's `1/(2x)` table and the coset points the R3 / commitment +/// time contexts read. Each is cached for the process exactly as the prove +/// caches it, so the prove finds it there and no value changes. +/// +/// `tables` is each table's AIR, row count and public inputs. The boundary +/// constraints' rows are read off the AIR with zero challenges: a constraint's +/// row is structural (the challenges only move its value). The tables run side +/// by side on scoped threads. Call it from a plain thread, not a rayon worker. +pub fn warm_prove_constants( + tables: &[( + &dyn AIR, + usize, + &PI, + )], +) where + F: IsFFTField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, + PI: Sync, +{ + std::thread::scope(|scope| { + for &(air, rows, pub_inputs) in tables { + if rows < 2 || !rows.is_power_of_two() { + continue; + } + scope.spawn(move || warm_table_constants::(air, rows, pub_inputs)); + } + }); +} + +fn warm_table_constants( + air: &dyn AIR, + rows: usize, + pub_inputs: &PI, +) where + F: IsFFTField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, +{ + let (domain, twiddles) = domain_and_twiddles::(air, rows); + let challenges = vec![FieldElement::::zero(); LOGUP_NUM_CHALLENGES]; + let mut steps: Vec = air + .boundary_constraints(pub_inputs, &challenges, None, rows) + .constraints + .iter() + .map(|bc| bc.step) + .collect(); + steps.sort_unstable(); + steps.dedup(); + #[cfg_attr(not(feature = "cuda"), allow(unused_variables))] + let z_invs: Vec>>> = steps + .iter() + .map(|&step| domain.boundary_zerofier_inv(step)) + .collect(); + WARMED_CONSTANTS.fetch_add(z_invs.len() as u64, Ordering::Relaxed); + #[cfg(not(feature = "cuda"))] + let _ = twiddles; + #[cfg(feature = "cuda")] + { + let lde = rows * domain.blowup_factor; + if !crate::gpu_lde::is_goldilocks_ext3_tower::() + || !crate::gpu_lde::clears_device_floor(lde) + { + return; + } + // SAFETY: `F == GoldilocksField` (the tower check above); the `Arc`'d + // Vecs share their layout with the concrete ones. + let conc = + |v: &Arc>>| -> crate::constraint_ir::gpu_interp::GoldilocksBZInv { + unsafe { + (*(v as *const _ as *const crate::constraint_ir::gpu_interp::GoldilocksBZInv)) + .clone() + } + }; + let mut warmed = 0u64; + for z in &z_invs { + warmed += + crate::constraint_ir::gpu_interp::base_vec_device_handle(&conc(z)).is_some() as u64; + } + if air.composition_poly_degree_bound(rows) / rows == 2 { + warmed += crate::constraint_ir::gpu_interp::base_vec_device_handle(&conc( + twiddles.inv_2x(&domain), + )) + .is_some() as u64; + } + let points = &domain.ood_constants().points; + // SAFETY: as above, `FieldElement` is `u64`. + let points_u64: &[u64] = + unsafe { std::slice::from_raw_parts(points.as_ptr() as *const u64, points.len()) }; + if let Ok(be) = math_cuda::device::backend() { + let stream = be.next_stream(); + warmed += + crate::gpu_lde::coset_points_device_handle(points_u64, &stream).is_some() as u64; + } + WARMED_CONSTANTS.fetch_add(warmed, Ordering::Relaxed); + } +} + +/// Whether every table's boundary-zerofier inverses are in the cache already: +/// what [`warm_prove_constants`] would find without building anything. A table +/// whose domain the cache does not hold yet is not warm. Builds nothing. +pub fn prove_constants_warm( + tables: &[( + &dyn AIR, + usize, + &PI, + )], +) -> bool +where + F: IsFFTField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, +{ + type Entry = (Arc>, Arc>); + tables.iter().all(|&(air, rows, pub_inputs)| { + if rows < 2 || !rows.is_power_of_two() { + return true; + } + let key = ( + std::any::TypeId::of::(), + rows, + air.options().blowup_factor as usize, + air.options().coset_offset, + ); + let domain = domain_twiddle_cache() + .lock() + .unwrap() + .get(&key) + .and_then(|b| b.downcast_ref::>()) + .map(|e| e.0.clone()); + let Some(domain) = domain else { + return false; + }; + let challenges = vec![FieldElement::::zero(); LOGUP_NUM_CHALLENGES]; + air.boundary_constraints(pub_inputs, &challenges, None, rows) + .constraints + .iter() + .all(|bc| domain.has_boundary_zerofier_inv(bc.step)) + }) +} + +/// Makes the card's staging pairs a later transfer would make on first use, +/// until `want` exist (`math_cuda::device::prewarm_staging_pairs`); returns +/// how many this call made. Zero without a card. +pub fn prewarm_staging_pairs(want: usize) -> usize { + #[cfg(feature = "cuda")] + { + math_cuda::device::prewarm_staging_pairs(want).unwrap_or(0) + } + #[cfg(not(feature = "cuda"))] + { + let _ = want; + 0 + } +} + +/// LDE cosets [`warm_deep_coset_points`] put on the card so far, process-wide. +static WARMED_DEEP_COSETS: AtomicU64 = AtomicU64::new(0); + +/// How many LDE cosets [`warm_deep_coset_points`] put on the card so far. +pub fn warmed_deep_cosets() -> u64 { + WARMED_DEEP_COSETS.load(Ordering::Relaxed) +} + +/// Puts on the card, before a prove takes it, each table's whole LDE coset — +/// the points the R4 DEEP reads (`coset_points_device_handle`), which the +/// first prove of a table length otherwise uploads from pageable memory inside +/// its card window (64 MiB at 2^21 rows, 128 MiB at 2^22, at blowup 4). The +/// copy is the one the DEEP caches for the process, so the DEEP finds it and +/// no value changes. Only for the tables the device DEEP takes (Goldilocks with +/// its cubic extension, an LDE over the device floor); a no-op without a card. +/// Returns how many cosets this call uploaded (one already on the card counts +/// nothing). Call it from a plain thread, not a rayon worker. +pub fn warm_deep_coset_points( + tables: &[( + &dyn AIR, + usize, + &PI, + )], +) -> u64 +where + F: IsFFTField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: Send + Sync, +{ + #[cfg(not(feature = "cuda"))] + { + let _ = tables; + 0 + } + #[cfg(feature = "cuda")] + { + if !crate::gpu_lde::is_goldilocks_ext3_tower::() { + return 0; + } + let Ok(be) = math_cuda::device::backend() else { + return 0; + }; + let mut warmed = 0u64; + for &(air, rows, _) in tables { + if rows < 2 || !rows.is_power_of_two() { + continue; + } + let (domain, _) = domain_and_twiddles::(air, rows); + let coset = &domain.lde_roots_of_unity_coset; + if !crate::gpu_lde::clears_device_floor(coset.len()) { + continue; + } + // SAFETY: `F == GoldilocksField` (the tower check above), a + // transparent wrapper over `u64`. + let coset_u64: &[u64] = + unsafe { std::slice::from_raw_parts(coset.as_ptr() as *const u64, coset.len()) }; + if crate::gpu_lde::coset_points_on_device(coset_u64) { + continue; + } + let stream = be.next_stream(); + warmed += + crate::gpu_lde::coset_points_device_handle(coset_u64, &stream).is_some() as u64; + } + WARMED_DEEP_COSETS.fetch_add(warmed, Ordering::Relaxed); + warmed + } +} + +/// Fills the process-wide domain and twiddle cache for a trace of +/// `trace_length` rows under `blowup_factor` and `coset_offset` — what the first +/// prove of that shape builds before its first round — so a prove that comes +/// later finds it there. The bytes the new entry holds when it built it; +/// `None` when it was already there, or for a length no trace has (zero, or +/// not a power of two). Call it from a plain thread, not a rayon worker (see +/// the pre-fill below). +pub fn warm_domain_twiddles( + blowup_factor: usize, + coset_offset: u64, + trace_length: usize, +) -> Option +where + F: IsFFTField + 'static, + FieldElement: Send + Sync, +{ + if !trace_length.is_power_of_two() { + return None; + } + let (domain, twiddles, built) = + cached_domain_and_twiddles::(blowup_factor, coset_offset, trace_length); + built.then(|| (domain.heap_bytes() + twiddles.heap_bytes()) as u64) +} + +/// [`domain_and_twiddles`] by the three values its entry is keyed by, and +/// whether this call built the entry. +fn cached_domain_and_twiddles( + blowup_factor: usize, + coset_offset: u64, + trace_length: usize, +) -> (Arc>, Arc>, bool) +where + F: IsFFTField + 'static, + FieldElement: Send + Sync, { type Entry = (Arc>, Arc>); let key = ( std::any::TypeId::of::(), trace_length, - air.options().blowup_factor as usize, - air.options().coset_offset, + blowup_factor, + coset_offset, ); { let cache = domain_twiddle_cache().lock().unwrap(); if let Some(e) = cache.get(&key).and_then(|b| b.downcast_ref::>()) { #[cfg(test)] crate::tests::domain_cache_stats::record(true); - return e.clone(); + return (e.0.clone(), e.1.clone(), false); } } #[cfg(test)] crate::tests::domain_cache_stats::record(false); - let d = Arc::new(Domain::new(air, trace_length)); + let d = Arc::new(Domain::from_options( + blowup_factor, + coset_offset, + trace_length, + )); let t = Arc::new(LdeTwiddles::new(&d)); // Pre-fill every lazy domain-derived cache from this setup thread, so no // rayon worker ever runs — or blocks waiting on — an initializer @@ -570,19 +1396,21 @@ where let _ = d.fri_inv_twiddles(); let _ = t.composition(&d); let _ = t.inv_2x(&d); + if composition_eval_stride(blowup_factor, 2) > 1 { + let _ = t.composition_subcoset_weights(&d); + } let mut cache = domain_twiddle_cache().lock().unwrap(); // Re-check under the lock: concurrent misses both build, and using the // loser would pin ITS per-instance vectors in the pointer-keyed device // caches for the process lifetime, duplicating VRAM. The winner stays. if let Some(e) = cache.get(&key).and_then(|b| b.downcast_ref::>()) { - return e.clone(); + return (e.0.clone(), e.1.clone(), false); } cache.insert(key, Box::new((d.clone(), t.clone()))); - (d, t) + (d, t, true) } -/// Explicit `TABLE_PARALLELISM` override, honoured by both `k` values below so -/// setting it pins the scheduler and the storage estimate to the same number. +/// Explicit `TABLE_PARALLELISM` override of the scheduler's `k` below. #[cfg(feature = "parallel")] fn parallelism_override() -> Option { std::env::var("TABLE_PARALLELISM") @@ -590,7 +1418,7 @@ fn parallelism_override() -> Option { .and_then(|s| s.parse().ok()) } -#[cfg(feature = "parallel")] +#[cfg(all(feature = "parallel", not(feature = "cuda")))] fn host_cores() -> usize { std::thread::available_parallelism() .map(|n| n.get()) @@ -637,61 +1465,6 @@ pub fn table_parallelism(num_airs: usize) -> usize { } } -/// How many tables' rounds 2-4 transients the *RAM* estimate assumes are alive -/// at once (`auto_storage::peak_bytes` sums the transient bytes of the top-k -/// tables, and `decide` turns that into RAM vs Disk). -/// -/// Deliberately not `table_parallelism(num_airs)`. That is a ceiling, not a -/// bound: on a `cuda` build what actually limits how many tables are in flight -/// is `VramGate`'s byte budget, which this host-side estimate cannot see. -/// Feeding an unbounded count in here would sum *every* table's transients — -/// on many-PAGE shapes that inflates the estimate by up to +44 % (512 PAGE -/// tables at blowup 4) and would spill proofs to disk that fit in RAM. On the -/// shapes that reach this path today (~21 tables, one PAGE table) the top-k sum -/// has all but saturated, so this value and `num_airs` agree to well under 1 %. -/// -/// Kept at exactly the value it had when the scheduler shared it, so splitting -/// the two does not move any storage decision. -/// -/// TODO: derive this from a byte budget rather than a table count, so it -/// tracks what `VramGate` admits instead of standing in for it. -pub fn storage_estimate_parallelism() -> usize { - #[cfg(feature = "parallel")] - { - parallelism_override().unwrap_or_else(|| { - #[cfg(feature = "cuda")] - { - (host_cores() * 2 / 3).max(1) - } - #[cfg(not(feature = "cuda"))] - { - (host_cores() / 3).max(1) - } - }) - } - #[cfg(not(feature = "parallel"))] - { - 1 - } -} - -/// Heuristic peak device bytes for one table: co-resident LDE columns plus the -/// resident Merkle trees, with a scratch factor for NTT and leaf transients. A -/// deliberate over estimate for a safety ceiling, not a precise allocator. Pass -/// aux_cols == 0 when the aux LDE is not yet resident (R1 main commit). -fn estimate_table_vram_bytes(main_cols: usize, aux_cols: usize, lde_size: usize) -> u64 { - const BYTES_PER_BASE: u64 = 8; - const EXT3_BYTES: u64 = 24; - const SCRATCH_FACTOR: u64 = 2; - const RESIDENT_TREE_BYTES_PER_LDE: u64 = 256; - let lde = lde_size as u64; - let per_row = (main_cols as u64).saturating_mul(BYTES_PER_BASE) - + (aux_cols as u64).saturating_mul(EXT3_BYTES); - let lde_term = lde.saturating_mul(per_row).saturating_mul(SCRATCH_FACTOR); - let tree_term = lde.saturating_mul(RESIDENT_TREE_BYTES_PER_LDE); - lde_term.saturating_add(tree_term) -} - /// Byte-budget admission gate for concurrently proven tables. `acquire` /// blocks until the requested bytes fit under the budget, releasing on /// permit drop. An oversized request is admitted alone (when nothing else @@ -742,15 +1515,18 @@ impl Drop for VramPermit<'_> { } /// Run `task` once per table index on `workers` OS driver threads, admitting -/// each index through `gate` with its estimated bytes. `order` fixes the -/// start order (heaviest table first, so the long pole starts early and small -/// tables fill around it — the fixed chunks this replaces made every table -/// wait for the slowest of its chunk). Returns one slot per original index. +/// each index through `gate` with its estimated bytes. The two array arguments +/// are deliberately independent: `estimates` is what the gate spends (the +/// device set, `crate::device_set`), while `order` is the caller's walk — a +/// scheduling policy keyed on [`table_walk_weight`], heaviest first, so the +/// long pole starts early and small tables fill around it. Returns one slot +/// per original index. fn run_admitted( order: &[usize], estimates: &[u64], gate: &VramGate, workers: usize, + label: impl Fn(usize) -> String + Sync, task: impl Fn(usize) -> T + Sync, ) -> Vec> { let results: Vec>> = estimates @@ -758,46 +1534,238 @@ fn run_admitted( .map(|_| std::sync::Mutex::new(None)) .collect(); let cursor = std::sync::atomic::AtomicUsize::new(0); + // ★ The first worker panic, kept so it can be re-raised on THIS thread. + // + // ⚠ Without this, a panic inside `task` is destroyed rather than reported. + // `std::thread::scope` propagates by panicking at the scope call with the + // fixed string "a scoped thread panicked", which names neither the cause nor + // its location; and the worker's own message does not survive either, + // because libtest installs a GLOBAL panic hook that suppresses the default + // output and files the message against the test thread it is capturing for — + // a spawned thread matches no test, so the message is dropped on the floor + // rather than merely misfiled. ✓ VERIFIED by grepping a full box run's raw + // log, stderr included: not one worker message appeared. + // + // The cost of that was eleven anonymous failures in one suite run. Catching + // the payload and re-raising it here puts the real message back inside the + // failing test's own block. + let first_panic: std::sync::Mutex>> = + std::sync::Mutex::new(None); + // The drivers draw their streams in the spawner's class: a proof run in the + // background stays there on every table's thread. + #[cfg(feature = "cuda")] + let class = math_cuda::stream_class::current(); + // A poisoned lock is itself a panic we are mid-way through reporting, so + // read through the poison rather than panicking about it and losing the + // message a second time. + let taken = |m: &std::sync::Mutex>>| { + m.lock().unwrap_or_else(|e| e.into_inner()).is_some() + }; std::thread::scope(|scope| { for _ in 0..workers.max(1).min(order.len().max(1)) { scope.spawn(|| { + #[cfg(feature = "cuda")] + let _class = math_cuda::stream_class::enter(class); loop { let pos = cursor.fetch_add(1, std::sync::atomic::Ordering::Relaxed); if pos >= order.len() { return; } + // ⚖ Stop pulling work once a sibling has failed. The result + // is discarded either way, so this only declines to spend + // cores on it; the previous code let every remaining index + // run to completion before the scope re-panicked. + if taken(&first_panic) { + return; + } let idx = order[pos]; let permit = gate.acquire(estimates[idx]); - let out = task(idx); - *results[idx].lock().unwrap() = Some(out); + let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| task(idx))); + // Released explicitly: the catch means unwinding no longer + // drops it for us, and a leaked permit would deadlock every + // remaining worker on the gate. drop(permit); + match out { + Ok(v) => *results[idx].lock().unwrap() = Some(v), + Err(payload) => { + // The worker's message names the stage and the + // shape; the driver knows which table it was. + let payload = name_panic_payload(payload, &label(idx)); + let mut slot = first_panic.lock().unwrap_or_else(|e| e.into_inner()); + if slot.is_none() { + *slot = Some(payload); + } + return; + } + } } }); } }); + if let Some(payload) = first_panic.into_inner().unwrap_or_else(|e| e.into_inner()) { + // ⚠ `resume_unwind` deliberately does NOT re-run the panic hook: the + // hook already ran in the worker, and running it again would print the + // same panic twice. The consequence is that the reported LOCATION is the + // harness's rather than the original `panic!` site — the message + // survives, the line number does not. That is the trade, and it is worth + // making: a named cause without a line beats a line without a cause. + std::panic::resume_unwind(payload); + } results .into_iter() .map(|m| m.into_inner().unwrap()) .collect() } -/// Table indices sorted heaviest-first by estimate. -fn heaviest_first(estimates: &[u64]) -> Vec { - let mut order: Vec = (0..estimates.len()).collect(); - order.sort_by_key(|&i| std::cmp::Reverse(estimates[i])); +/// [`run_admitted`] under a gate of `budget`, for the tests of what its +/// drivers inherit. +#[cfg(all(test, feature = "cuda"))] +pub(crate) fn run_admitted_for_tests( + order: &[usize], + estimates: &[u64], + budget: u64, + workers: usize, + task: impl Fn(usize) -> T + Sync, +) -> Vec> { + run_admitted( + order, + estimates, + &VramGate::new(budget), + workers, + |idx| format!("table {idx}"), + task, + ) +} + +/// Prefix a string panic payload with the table's name so the re-raised +/// message says which table failed; a payload that is not a string is passed +/// through unchanged. +fn name_panic_payload( + payload: Box, + table: &str, +) -> Box { + let message = payload.downcast_ref::().cloned().or_else(|| { + payload + .downcast_ref::<&'static str>() + .map(|m| (*m).to_string()) + }); + match message { + Some(m) => Box::new(format!("table {table}: {m}")), + None => payload, + } +} + +/// The sort weight for the table walk. A SCHEDULING policy, not a size model — +/// see [`crate::device_set`] for the bytes anything is admitted against. +/// +/// The walk is the order the per-table drivers *start* tables in. At +/// `TABLE_PARALLELISM=1` it cannot change what is resident on the device (one +/// table at a time, the gate never blocks). What it does change is the order +/// the host allocator sees the per-table arenas in. Measured on the q=20 wrap +/// (2^22, blowup 4, RTX 5090 box, 2026-09-07/08), `/usr/bin/time -v` max RSS, +/// proof bytes identical on every row: +/// +/// | walk | build | `MALLOC_MMAP_THRESHOLD_` | max RSS | +/// |------|-------|--------------------------|---------| +/// | this weight | pre-#964 tip | default | 47,307,284 kB = 45.1 GiB | +/// | this weight | diagnostic, device-set gate | default | 47,365,192 kB = 45.2 GiB | +/// | this weight | this one | default | 53,456,648 kB = 51.0 GiB | +/// | the device-set model's order | this one | default | 53,472,980 kB = 51.0 GiB | +/// | a fused-phase host-transient order | this one | default | 53,453,344 kB = 51.0 GiB | +/// | this weight | this one | 1 MiB | 46,053,164 kB = 43.9 GiB | +/// | this weight | pre-#964 tip | 1 MiB | 46,054,788 kB = 43.9 GiB | +/// +/// Read the first three rows together: the SAME order measures 45.2 GiB in one +/// build and 51.0 GiB in this one. **The order is a correlate, not a cause.** +/// The mechanism is glibc arena retention — the test harness installs no +/// `#[global_allocator]`, and a freed multi-gibibyte buffer is returned to the +/// OS only when the arena top can be trimmed, which depends on what was +/// allocated above it. The walk order is one input to that layout; the +/// allocations made around it are another. The last two rows settle what is +/// being measured: forcing every allocation of a megabyte or more to be mapped +/// and unmapped directly collapses a 5.9 GiB spread to 1.6 MB, so none of it +/// was ever working set. (At q=41 the orders are indistinguishable, 98.5 +/// against 98.6 GiB.) +/// +/// ⇒ **The durable fix is the allocator, not this weight.** Lane S is landing +/// it as jemalloc in the test harness; until then the effect is reproducible +/// with `MALLOC_MMAP_THRESHOLD_=1048576`, which removes it outright and takes +/// 1.2 GiB off the good arm as well. The MEMORY saving is large and comes with +/// a control: ≈6 GiB at q=20 and ≈13 GiB at q=41 from a high-retention start, +/// ≈1.2 GiB from a low-retention one. +/// +/// The TIME effect is a different matter and is NOT established. In the paired +/// q=20 comparison above the low-retention arm is ~2.5% slower (135.8/136.2 s +/// against 139.1/140.3 s, two runs each, consistent sign) — but that does not +/// carry to the allocator flag, whose four measured cells read −1.6%, +1.4%, +/// −0.1% and +2.8%, one run each and no consistent sign, three of them inside +/// what a single run resolves. Quote the memory saving; do not quote a time +/// cost without paired repeats per shape. +/// +/// Until the allocator fix lands, this order is kept because it is the one the +/// prover had before #964 — not because reordering is a lever, since nothing in +/// this file controls the layout that decides the number. +/// +/// Its arithmetic is inherited verbatim from the VRAM estimate the prover +/// sorted by before the device-set model, and it is deliberately NOT re-read as +/// a byte count: the factor of two assumed the second LDE buffer that #956's +/// in-place transpose removed, and the flat 256 B per LDE row stands in for a +/// tree whose real width depends on the digest. Changing these numbers changes +/// the schedule, so any change needs a wrap measurement, not an argument about +/// bytes. +/// +/// Pass `aux_cols == 0` for the R1 main-commit walk and the AIR's aux width for +/// the fused rounds walk: the two phases weigh the tables differently, and they +/// always have. +fn table_walk_weight(main_cols: usize, aux_cols: usize, lde_size: usize) -> u64 { + const BASE_WEIGHT: u64 = 8; + const EXT3_WEIGHT: u64 = 24; + const WIDTH_WEIGHT: u64 = 2; + const PER_LDE_ROW_WEIGHT: u64 = 256; + let lde = lde_size as u64; + let per_row = (main_cols as u64).saturating_mul(BASE_WEIGHT) + + (aux_cols as u64).saturating_mul(EXT3_WEIGHT); + let width_term = lde.saturating_mul(per_row).saturating_mul(WIDTH_WEIGHT); + let row_term = lde.saturating_mul(PER_LDE_ROW_WEIGHT); + width_term.saturating_add(row_term) +} + +/// Table indices sorted heaviest-first by weight. `sort_by_key` is stable, so +/// tables that weigh the same keep their registry order. +fn heaviest_first(weights: &[u64]) -> Vec { + let mut order: Vec = (0..weights.len()).collect(); + order.sort_by_key(|&i| std::cmp::Reverse(weights[i])); order } +/// One line naming the walk a phase took, heaviest first. Printed once per +/// phase per prove: the walk is a measured choice (see [`table_walk_weight`]), +/// so a run that moves host peak has to be able to say which order it took. +fn describe_walk(order: &[usize], weights: &[u64], names: &[String]) -> String { + order + .iter() + .map(|&i| { + format!( + "{}={:.2}GiB", + names[i], + weights[i] as f64 / (1u64 << 30) as f64 + ) + }) + .collect::>() + .join(" ") +} + /// A container for the results of the second round of the STARK Prove protocol. -pub(crate) struct Round2 +pub(crate) struct Round2 where - F: IsField, - FieldElement: AsBytes, + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { /// Evaluations of the composition polynomial parts over the LDE domain. pub(crate) lde_composition_poly_evaluations: Vec>>, /// The Merkle tree built to compute the commitment to the composition polynomial parts. - pub(crate) composition_poly_merkle_tree: BatchedMerkleTree, + pub(crate) composition_poly_merkle_tree: MerkleTree>, /// The commitment to the composition polynomial parts. pub(crate) composition_poly_root: Commitment, /// The composition Merkle tree kept resident on device (when the R2 GPU tree @@ -808,12 +1776,33 @@ where pub(crate) gpu_composition_tree: Option, } +/// The composition-polynomial parts, before any commitment is taken over them. +/// +/// Returned by [`IsStarkProver::compute_composition_parts`], which round 2 and +/// the batched prover share. The device handle rides along rather than being +/// installed on the `Round1` inside, because the two callers install it at +/// different points: round 2 folds it into the table's own LDE session, the +/// batched prover keeps every table's parts alive only until they have been +/// absorbed into the epoch's MMCS. +pub(crate) struct CompositionParts +where + FieldElement: AsBytes + Sync + Send, +{ + pub(crate) parts: Vec>>, + #[cfg(feature = "cuda")] + pub(crate) gpu_parts: Option, + #[cfg(feature = "instruments")] + pub(crate) constraints_dur: Duration, + #[cfg(feature = "instruments")] + pub(crate) fft_dur: Duration, +} + /// A container for the results of the third round of the STARK Prove protocol. pub(crate) struct Round3 { /// Evaluations of the trace polynomials, main and auxiliary, at the out-of-domain challenge. - trace_ood_evaluations: Table, + pub(crate) trace_ood_evaluations: Table, /// Evaluations of the composition polynomial parts at the out-of-domain challenge. - composition_poly_parts_ood_evaluation: Vec>, + pub(crate) composition_poly_parts_ood_evaluation: Vec>, } /// A container for the results of the fourth round of the STARK Prove protocol. @@ -868,6 +1857,7 @@ pub trait IsStarkProver< Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, FieldExtension: Send + Sync + IsField + 'static, PI, + H: StarkHash, > where FieldElement: math::traits::ByteConversion, FieldElement: math::traits::ByteConversion, @@ -880,7 +1870,7 @@ pub trait IsStarkProver< fn commit_rows_bit_reversed( data: &[FieldElement], num_cols: usize, - ) -> Option<(BatchedMerkleTree, Commitment)> + ) -> Option<(MerkleTree>, Commitment)> where FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, E: IsField, @@ -897,7 +1887,43 @@ pub trait IsStarkProver< num_cols: usize, col_start: usize, col_end: usize, - ) -> Option<(BatchedMerkleTree, Commitment)> + ) -> Option<(MerkleTree>, Commitment)> + where + FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, + E: IsField, + { + Self::commit_rows_bit_reversed_subset_with( + data, + num_cols, + col_start, + col_end, + crate::commitment::ROWS_PER_LEAF, + ) + } + + /// [`Self::commit_rows_bit_reversed`] with `rows_per_leaf` rows per leaf + /// (the table's [`LeafLayout`]): 2 = today's row pairs, 1 = one row (S2). + fn commit_rows_bit_reversed_with( + data: &[FieldElement], + num_cols: usize, + rows_per_leaf: usize, + ) -> Option<(MerkleTree>, Commitment)> + where + FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, + E: IsField, + { + Self::commit_rows_bit_reversed_subset_with(data, num_cols, 0, num_cols, rows_per_leaf) + } + + /// [`Self::commit_rows_bit_reversed_subset`] with `rows_per_leaf` rows per + /// leaf: leaf `i` hashes the bit-reversed rows `R·i .. R·i + R − 1`. + fn commit_rows_bit_reversed_subset_with( + data: &[FieldElement], + num_cols: usize, + col_start: usize, + col_end: usize, + rows_per_leaf: usize, + ) -> Option<(MerkleTree>, Commitment)> where FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, E: IsField, @@ -918,17 +1944,19 @@ pub trait IsStarkProver< "num_rows must be a power of two for reverse_index" ); - // Local alias for the canonical constant, used several times below. - const ROWS_PER_LEAF: usize = crate::commitment::ROWS_PER_LEAF; - let num_leaves = num_rows / ROWS_PER_LEAF; + debug_assert!(rows_per_leaf == 1 || rows_per_leaf == 2); + if rows_per_leaf == 0 || !num_rows.is_multiple_of(rows_per_leaf) { + return None; + } + let num_leaves = num_rows / rows_per_leaf; let subset_cols = col_end - col_start; let byte_len = as ByteConversion>::BYTE_LEN; - let leaf_bytes = ROWS_PER_LEAF * subset_cols * byte_len; + let leaf_bytes = rows_per_leaf * subset_cols * byte_len; let hash_leaf = |buf: &mut [u8], leaf_idx: usize| -> Commitment { let mut offset = 0; - for k in 0..ROWS_PER_LEAF { - let br_idx = reverse_index(ROWS_PER_LEAF * leaf_idx + k, num_rows as u64); + for k in 0..rows_per_leaf { + let br_idx = reverse_index(rows_per_leaf * leaf_idx + k, num_rows as u64); let row_start = br_idx * num_cols; let row = &data[row_start + col_start..row_start + col_end]; for elem in row.iter() { @@ -936,7 +1964,7 @@ pub trait IsStarkProver< offset += byte_len; } } - BatchedMerkleTreeBackend::::hash_bytes(buf) + as IsStreamingLeafBackend>::hash_bytes(buf) }; #[cfg(feature = "parallel")] @@ -955,7 +1983,7 @@ pub trait IsStarkProver< .collect() }; - let tree = BatchedMerkleTree::::build_from_hashed_leaves(hashed_leaves)?; + let tree = MerkleTree::>::build_from_hashed_leaves(hashed_leaves)?; let root = tree.root; Some((tree, root)) } @@ -972,6 +2000,27 @@ pub trait IsStarkProver< air: &impl AIR, num_precomputed_cols: usize, ) -> Option + where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + Self::compute_precomputed_commitment_for_testing_with( + trace, + air, + num_precomputed_cols, + LeafLayout::RowPair, + ) + } + + /// [`Self::compute_precomputed_commitment_for_testing`] under an explicit + /// leaf layout (S2's one-row root of the same columns). + #[cfg(any(test, feature = "test-utils"))] + fn compute_precomputed_commitment_for_testing_with( + trace: &TraceTable, + air: &impl AIR, + num_precomputed_cols: usize, + layout: LeafLayout, + ) -> Option where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, @@ -982,8 +2031,10 @@ pub trait IsStarkProver< let twiddles = LdeTwiddles::new(&domain); let evals = Self::compute_lde_from_columns_cached::(&precomputed, &domain, &twiddles); - let (_, commitment) = - crate::commitment::commit_bit_reversed(&evals, crate::commitment::ROWS_PER_LEAF)?; + let (_, commitment) = crate::commitment::commit_bit_reversed_with::< + Field, + H::Batched, + >(&evals, layout.rows_per_leaf())?; Some(commitment) } @@ -1127,27 +2178,39 @@ pub trait IsStarkProver< /// /// `precomputed`: if present, the leading `num_cols` columns are committed /// as a separate Merkle tree (the precomputed split for preprocessed - /// tables) and the root is checked against the AIR-hardcoded commitment. - #[allow(clippy::type_complexity)] + /// tables) and the root is checked against the AIR-hardcoded commitment + /// OF `layout`. `table` is the AIR's name, for the device diagnostics. + /// + /// `layout` is the table's trace-tree leaf layout: every arm (the fused + /// and split device commits and the CPU one) builds its trees with + /// `layout.rows_per_leaf()` rows per leaf, so a one-row table (S2) commits + /// on the device like a row-pair one. + #[allow(clippy::type_complexity, clippy::too_many_arguments)] fn commit_main_trace( + #[cfg_attr(not(feature = "cuda"), allow(unused_variables))] table: &str, trace: &TraceTable, domain: &Domain, twiddles: &LdeTwiddles, precomputed: Option<(Commitment, usize)>, + layout: LeafLayout, #[cfg(feature = "cuda")] device_only: bool, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, - ) -> Result, ProvingError> + #[cfg_attr(not(feature = "cuda"), allow(unused_variables))] residency: ResidencyMode, + ) -> Result, ProvingError> where FieldElement: AsBytes, FieldElement: AsBytes, { - let lde_size = domain.interpolation_domain_size * domain.blowup_factor; - // Fused GPU path (cuda only): row-major NTT — single H2D from the // already-row-major trace, no column extraction, no transpose. // Falls back to CPU if GPU path returns None. + // + // `RecomputeLde` skips both device paths: the LDE it drops after this + // commit is recomputed on the host, so the buffer the tree was built + // from must be the host one. The mode is for CPU proving and forces + // the host path per table. + let rows_per_leaf = layout.rows_per_leaf(); #[cfg(feature = "cuda")] - if precomputed.is_none() { + if precomputed.is_none() && !residency.recomputes_main_lde() { let (trace_slice, num_cols) = trace.main_data_row_major(); let n = if num_cols > 0 { trace_slice.len() / num_cols @@ -1156,21 +2219,30 @@ pub trait IsStarkProver< }; #[cfg(feature = "instruments")] let t_sub = Instant::now(); + // A table that builds its trace on the card uploads fewer bytes; + // the buffer lives until the commit has copied it in. + let produced = crate::gpu_lde::clears_device_floor(n * domain.blowup_factor) + .then(|| trace.produce_main_dev()) + .flatten(); if let Some((tree, handle, main_data)) = crate::gpu_lde::try_expand_leaf_and_tree_row_major_keep::< Field, Field, - BatchedMerkleTreeBackend, + H::Batched, >( + table, + "R1 main commit", trace_slice, - trace.main_rowmajor_dev(), + trace.main_rowmajor_dev().or(produced.as_ref()), n, num_cols, domain.blowup_factor, &twiddles.coset_weights, !device_only, + rows_per_leaf, ) { + release_produced_after(produced, Some(&handle)); #[cfg(feature = "instruments")] let main_lde_dur = t_sub.elapsed(); let root = tree.root; @@ -1189,6 +2261,7 @@ pub trait IsStarkProver< Some(handle), )); } + release_produced_after(produced, None); } // Fused GPU split path for preprocessed tables (cuda only): one @@ -1201,30 +2274,40 @@ pub trait IsStarkProver< // are gathered on device. The handle keeps the LDE device-resident for // the downstream GPU rounds. #[cfg(feature = "cuda")] - if let Some((expected_precomputed_root, num_precomputed)) = precomputed { + if let Some((expected_precomputed_root, num_precomputed)) = precomputed + && !residency.recomputes_main_lde() + { let (trace_slice, num_cols) = trace.main_data_row_major(); let n = if num_cols > 0 { trace_slice.len() / num_cols } else { 0 }; - #[cfg(feature = "disk-spill")] - let cache_ok = storage_mode != StorageMode::Disk; - #[cfg(not(feature = "disk-spill"))] let cache_ok = true; let cached_pre = cache_ok - .then(|| precomputed_tree_cache_get::(&expected_precomputed_root)) + .then(|| { + precomputed_tree_cache_get::>( + &expected_precomputed_root, + rows_per_leaf, + ) + }) .flatten(); #[cfg(feature = "instruments")] let t_sub = Instant::now(); + // As in the plain arm: the trace built on the card when the table + // carries a producer. + let produced = crate::gpu_lde::clears_device_floor(n * domain.blowup_factor) + .then(|| trace.produce_main_dev()) + .flatten(); if let Some((pre_tree, mult_tree, handle, main_data)) = crate::gpu_lde::try_expand_split_trees_row_major_keep::< Field, Field, - BatchedMerkleTreeBackend, + H::Batched, >( + table, trace_slice, - trace.main_rowmajor_dev(), + trace.main_rowmajor_dev().or(produced.as_ref()), n, num_cols, domain.blowup_factor, @@ -1232,8 +2315,10 @@ pub trait IsStarkProver< num_precomputed, cached_pre.is_none(), !device_only, + rows_per_leaf, ) { + release_produced_after(produced, Some(&handle)); #[cfg(feature = "instruments")] crate::instruments::accum_r1_main(t_sub.elapsed(), std::time::Duration::ZERO); if device_only { @@ -1250,12 +2335,11 @@ pub trait IsStarkProver< if tree.root != expected_precomputed_root { return Err(ProvingError::PrecomputedCommitmentMismatch); } - #[cfg(feature = "disk-spill")] - Self::spill_tree(&mut tree, storage_mode, "precomputed Merkle tree")?; let tree = Arc::new(tree); if cache_ok { - precomputed_tree_cache_put::( + precomputed_tree_cache_put::>( expected_precomputed_root, + rows_per_leaf, Arc::clone(&tree), ); } @@ -1264,8 +2348,6 @@ pub trait IsStarkProver< }; #[allow(unused_mut)] let mut mult_tree = mult_tree; - #[cfg(feature = "disk-spill")] - Self::spill_tree(&mut mult_tree, storage_mode, "mult Merkle tree")?; let mult_root = mult_tree.root; let commit = TableCommit::preprocessed( mult_tree, @@ -1277,34 +2359,17 @@ pub trait IsStarkProver< return Ok((commit, (main_data, num_cols), Some(handle))); } // GPU split path declined (size threshold / tower) → CPU path below. + release_produced_after(produced, None); } // CPU path: the trace `Table` is already row-major, so copy it directly // (one memcpy — no transpose) and expand in place with the cache-blocked // batched two-half FFT. Row-major end-to-end: no LDE-size transpose, - // contiguous Merkle leaves. - let (trace_data, total_cols) = trace.main_data_row_major(); - - #[cfg(feature = "instruments")] - let t_sub = Instant::now(); - - let mut main_data: Vec> = Vec::with_capacity(lde_size * total_cols); - main_data.extend_from_slice(trace_data); - - #[cfg(feature = "disk-spill")] - if storage_mode == StorageMode::Disk { - trace.main_table.advise_drop_cache(); - } - - Polynomial::>::coset_lde_full_expand_row_major::( - &mut main_data, - total_cols, - domain.blowup_factor, - &twiddles.coset_weights, - &twiddles.two_half_inv, - &twiddles.two_half_fwd, - ) - .expect("row-major coset LDE expansion"); + // contiguous Merkle leaves. + #[cfg(feature = "instruments")] + let t_sub = Instant::now(); + + let (main_data, total_cols) = Self::expand_main_lde_row_major(trace, domain, twiddles); #[cfg(feature = "instruments")] let main_lde_dur = t_sub.elapsed(); @@ -1315,10 +2380,9 @@ pub trait IsStarkProver< let commit = match precomputed { None => { #[allow(unused_mut)] - let (mut tree, root) = Self::commit_rows_bit_reversed(&main_data, total_cols) - .ok_or(ProvingError::EmptyCommitment)?; - #[cfg(feature = "disk-spill")] - Self::spill_tree(&mut tree, storage_mode, "main Merkle tree")?; + let (mut tree, root) = + Self::commit_rows_bit_reversed_with(&main_data, total_cols, rows_per_leaf) + .ok_or(ProvingError::EmptyCommitment)?; TableCommit::plain(tree, root) } Some((expected_precomputed_root, num_precomputed)) => { @@ -1326,14 +2390,15 @@ pub trait IsStarkProver< // precomputed-columns tree is a pure function of (content, // domain) already pinned by `expected_precomputed_root`, so it // is reused from the process cache when this exact commitment - // was built before — across epochs and across proves. Bypassed - // in disk-spill Disk mode, where trees are spilled (mutated). - #[cfg(feature = "disk-spill")] - let cache_ok = storage_mode != StorageMode::Disk; - #[cfg(not(feature = "disk-spill"))] + // was built before — across epochs and across proves. let cache_ok = true; let precomputed_tree = match cache_ok - .then(|| precomputed_tree_cache_get::(&expected_precomputed_root)) + .then(|| { + precomputed_tree_cache_get::>( + &expected_precomputed_root, + rows_per_leaf, + ) + }) .flatten() { // Cache key == the root a rebuild would be verified @@ -1341,22 +2406,22 @@ pub trait IsStarkProver< Some(tree) => tree, None => { #[allow(unused_mut)] - let (mut tree, root) = Self::commit_rows_bit_reversed_subset( + let (mut tree, root) = Self::commit_rows_bit_reversed_subset_with( &main_data, total_cols, 0, num_precomputed, + rows_per_leaf, ) .ok_or(ProvingError::EmptyCommitment)?; if root != expected_precomputed_root { return Err(ProvingError::PrecomputedCommitmentMismatch); } - #[cfg(feature = "disk-spill")] - Self::spill_tree(&mut tree, storage_mode, "precomputed Merkle tree")?; let tree = Arc::new(tree); if cache_ok { - precomputed_tree_cache_put::( + precomputed_tree_cache_put::>( expected_precomputed_root, + rows_per_leaf, Arc::clone(&tree), ); } @@ -1364,15 +2429,14 @@ pub trait IsStarkProver< } }; #[allow(unused_mut)] - let (mut mult_tree, mult_root) = Self::commit_rows_bit_reversed_subset( + let (mut mult_tree, mult_root) = Self::commit_rows_bit_reversed_subset_with( &main_data, total_cols, num_precomputed, total_cols, + rows_per_leaf, ) .ok_or(ProvingError::EmptyCommitment)?; - #[cfg(feature = "disk-spill")] - Self::spill_tree(&mut mult_tree, storage_mode, "mult Merkle tree")?; TableCommit::preprocessed( mult_tree, mult_root, @@ -1392,24 +2456,115 @@ pub trait IsStarkProver< Ok((commit, (main_data, total_cols))) } - /// Spill a committed Merkle tree to disk when `storage_mode` is `Disk`, - /// tagging any I/O error with `label`. No-op otherwise. Shared by every commit - /// site (main / preprocessed split / aux). - #[cfg(feature = "disk-spill")] - fn spill_tree( - tree: &mut BatchedMerkleTree, - storage_mode: StorageMode, - label: &str, - ) -> Result<(), ProvingError> - where - C: IsField, - FieldElement: AsBytes + Sync + Send, - { - if storage_mode == StorageMode::Disk { - tree.spill_nodes_to_disk() - .map_err(|e| ProvingError::DiskSpill(format!("{label}: {e}")))?; - } - Ok(()) + /// Expand a table's main trace to its coset LDE, row-major, without + /// building any Merkle tree. + /// + /// The Round-1 CPU commit and the `ResidencyMode::RecomputeLde` recompute + /// both go through here, which is what makes the recomputed buffer + /// bit-identical to the one the tree was built from — identical by + /// construction rather than by argument. The twiddles are process-cached, + /// so the second call re-runs the NTT over the same inputs. + fn expand_main_lde_row_major( + trace: &TraceTable, + domain: &Domain, + twiddles: &LdeTwiddles, + ) -> (Vec>, usize) { + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + let (trace_data, total_cols) = trace.main_data_row_major(); + + let mut main_data: Vec> = Vec::with_capacity(lde_size * total_cols); + main_data.extend_from_slice(trace_data); + + Polynomial::>::coset_lde_full_expand_row_major::( + &mut main_data, + total_cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .expect("row-major coset LDE expansion"); + + (main_data, total_cols) + } + + /// The MAIN trace's size-`n` coset evaluation, row-major — the stride- + /// `blowup` subsample of [`Self::expand_main_lde_row_major`]'s output, + /// computed directly: iFFT(n) → coset weights → FFT(n), about 37% of a + /// full expansion's work and a quarter of its bytes. Values are + /// bit-identical to the subsample (exact modular arithmetic; both compute + /// the same DFT), which is what the batched phase 4 reads and ALL it + /// reads. + fn expand_main_coset_eval_row_major( + trace: &TraceTable, + domain: &Domain, + twiddles: &LdeTwiddles, + ) -> (Vec>, usize) { + let (trace_data, total_cols) = trace.main_data_row_major(); + let mut main_data: Vec> = Vec::with_capacity(trace_data.len()); + main_data.extend_from_slice(trace_data); + Polynomial::>::coset_lde_full_expand_row_major::( + &mut main_data, + total_cols, + 1, + &twiddles.coset_weights, + &twiddles.two_half_inv, + twiddles.fwd_n(domain.interpolation_domain_size), + ) + .expect("row-major coset evaluation"); + (main_data, total_cols) + } + + /// The AUX counterpart of [`Self::expand_main_coset_eval_row_major`]. + fn expand_aux_coset_eval_row_major( + trace: &TraceTable, + domain: &Domain, + twiddles: &LdeTwiddles, + ) -> (Vec>, usize) { + let (trace_data, total_cols) = trace.aux_data_row_major(); + let mut aux_data: Vec> = Vec::with_capacity(trace_data.len()); + aux_data.extend_from_slice(trace_data); + Polynomial::>::coset_lde_full_expand_row_major::( + &mut aux_data, + total_cols, + 1, + &twiddles.coset_weights, + &twiddles.two_half_inv, + twiddles.fwd_n(domain.interpolation_domain_size), + ) + .expect("row-major aux coset evaluation"); + (aux_data, total_cols) + } + + /// Expand a table's auxiliary trace to its coset LDE, row-major, without + /// building any Merkle tree — the aux counterpart of + /// [`Self::expand_main_lde_row_major`], and extracted for the same reason: + /// the batched prover rebuilds this buffer once per phase instead of + /// retaining it, and a second expansion written elsewhere would be a second + /// encoding of the committed one. + fn expand_aux_lde_row_major( + trace: &TraceTable, + domain: &Domain, + twiddles: &LdeTwiddles, + ) -> (Vec>, usize) { + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + let (trace_data, total_cols) = trace.aux_data_row_major(); + + let mut aux_data: Vec> = + Vec::with_capacity(lde_size * total_cols); + aux_data.extend_from_slice(trace_data); + + Polynomial::>::coset_lde_full_expand_row_major::( + &mut aux_data, + total_cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .expect("row-major aux coset LDE expansion"); + + (aux_data, total_cols) } /// Recompute Round1 from the trace, reusing the Merkle trees stored in commitments. @@ -1421,9 +2576,9 @@ pub trait IsStarkProver< air: &dyn AIR, trace: &TraceTable, domain: &Domain, - commitment: &Round1Commitments, + commitment: &Round1Commitments, twiddles: &LdeTwiddles, - ) -> Result, ProvingError> + ) -> Result, ProvingError> where FieldElement: AsBytes, FieldElement: AsBytes, @@ -1497,7 +2652,7 @@ pub trait IsStarkProver< #[cfg(feature = "debug-checks")] fn run_debug_checks( pair_cells: &[std::sync::Mutex>], - commitments: &[Round1Commitments], + commitments: &[Round1Commitments], domains: &[Arc>], twiddle_caches: &[Arc>], ) where @@ -1505,7 +2660,7 @@ pub trait IsStarkProver< FieldElement: AsBytes, PI: Send + Sync + Clone, { - let mut temp_results: Vec> = + let mut temp_results: Vec> = Vec::with_capacity(pair_cells.len()); for ((cell, commitment), (domain, twiddles)) in pair_cells .iter() @@ -1553,6 +2708,9 @@ pub trait IsStarkProver< /// R2 producer and the `xcheck` mirror so the two cannot drift. `want_host` gates /// the d=2 host drain only — d=1 tables are never device-only, so they always /// keep their host part. + /// + /// `row_stride` is the stride `h_dev` was evaluated at + /// ([`composition_eval_stride`]); only the d=2 split takes a stride above 1. #[cfg(feature = "cuda")] fn decompose_comp_h_dev( number_of_parts: usize, @@ -1560,11 +2718,15 @@ pub trait IsStarkProver< domain: &Domain, twiddles: &LdeTwiddles, want_host: bool, + row_stride: usize, ) -> Option<( Vec>>, math_cuda::lde::GpuLdeExt3, )> { if number_of_parts == 1 { + if row_stride != 1 { + return None; + } // d=1 is never device-only (`device_only_for`'s degree gate admits only // d=2), so the single part is always kept on host — `want_host` must // hold, and the d=1 helper ignores it by design. @@ -1582,11 +2744,17 @@ pub trait IsStarkProver< ); crate::gpu_lde::try_comp_h_to_slabs_dev::(h_dev) } else { + let weights: &[FieldElement] = if row_stride == 1 { + &twiddles.composition(domain).weights + } else { + twiddles.composition_subcoset_weights(domain) + }; crate::gpu_lde::try_decompose_extend_d2_dev::( h_dev, twiddles.inv_2x(domain), - &twiddles.composition(domain).weights, + weights, want_host, + row_stride, ) } } @@ -1602,10 +2770,18 @@ pub trait IsStarkProver< /// /// On the LDE coset {g·ω^i | i=0..2N-1}, we have -g·ω^i = g·ω^{i+N} /// since ω^N = -1 for a 2N-th root of unity ω. + /// + /// With `row_stride` s > 1, `constraint_evaluations` holds `H` on the + /// sub-coset of every s-th LDE point only ([`composition_eval_stride`]). The + /// same pairing holds there (the sub-coset is closed under negation), the + /// point i is the LDE point i·s (so 1/(2x) is `inv_2x[i·s]`), and the halves + /// come out on fewer points that still determine them; the extension then + /// runs at the domain's blowup instead of 2. The parts are the same. fn decompose_and_extend_d2( constraint_evaluations: &[FieldElement], domain: &Domain, twiddles: &LdeTwiddles, + row_stride: usize, ) -> Vec>> where FieldElement: AsBytes + Sync + Send, @@ -1614,11 +2790,16 @@ pub trait IsStarkProver< let two_n = constraint_evaluations.len(); let n = two_n / 2; debug_assert_eq!(two_n, n * 2); + debug_assert_eq!( + two_n * row_stride, + domain.interpolation_domain_size * domain.blowup_factor, + "H must cover the LDE coset at the given stride" + ); // Step 1: 1/(2·g·ω^i) for i=0..N-1, cached once per domain in the // shared twiddles (base field — mixed F×E multiplication below). let inv_2x = twiddles.inv_2x(domain); - debug_assert_eq!(inv_2x.len(), n); + debug_assert_eq!(inv_2x.len(), n * row_stride); // Step 2: Pointwise decomposition. // H₀((g·ω^i)²) = (evals[i] + evals[i+N]) / 2 @@ -1630,11 +2811,11 @@ pub trait IsStarkProver< let sum = &constraint_evaluations[i] + &constraint_evaluations[i + n]; let diff = &constraint_evaluations[i] - &constraint_evaluations[i + n]; // F × E → E (base field scalar on left for mixed multiplication) - (&two_inv * &sum, &inv_2x[i] * &diff) + (&two_inv * &sum, &inv_2x[i * row_stride] * &diff) }); - // Step 3: Extend each part from n evals on the g²-coset to 2n evals on the - // g-coset (the full LDE domain). + // Step 3: Extend each part from n evals on the g²-coset to the g-coset + // (the full LDE domain). // GPU fast path: batch both halves into one ext3 LDE call. Requires // `cuda` feature and a qualifying size. Falls through to CPU when not. @@ -1646,49 +2827,70 @@ pub trait IsStarkProver< } let composition_twiddles = twiddles.composition(domain); + let (weights, inv): (&[FieldElement], &LayerTwiddles) = if row_stride == 1 { + (&composition_twiddles.weights, &composition_twiddles.inv) + } else { + ( + twiddles.composition_subcoset_weights(domain), + twiddles.composition_subcoset_inv(domain), + ) + }; + let blowup = 2 * row_stride; + let fwd = &composition_twiddles.fwd; let (lde_h0, lde_h1) = crate::par::join( - || Self::extend_half_to_lde(&h0_evals, composition_twiddles), - || Self::extend_half_to_lde(&h1_evals, composition_twiddles), + || Self::extend_half_to_lde(&h0_evals, blowup, weights, inv, fwd), + || Self::extend_half_to_lde(&h1_evals, blowup, weights, inv, fwd), ); vec![lde_h0, lde_h1] } - /// Extend `half_evals` — `n = lde_size/2` evaluations of a degree-`], - twiddles: &CompositionLdeTwiddles, + blowup: usize, + weights: &[FieldElement], + inv: &LayerTwiddles, + fwd: &LayerTwiddles, ) -> Vec> where FieldElement: AsBytes, FieldElement: AsBytes, { - debug_assert_eq!(half_evals.len(), twiddles.weights.len()); - Polynomial::coset_lde_full::( - half_evals, - 2, - &twiddles.weights, - &twiddles.inv, - &twiddles.fwd, - ) - .expect("coset extension") + debug_assert_eq!(half_evals.len(), weights.len()); + Polynomial::coset_lde_full::(half_evals, blowup, weights, inv, fwd) + .expect("coset extension") } - /// Returns the result of the second round of the STARK Prove protocol. - fn round_2_compute_composition_polynomial( + /// The evaluations of the composition-polynomial parts over the LDE domain, + /// and nothing else — no commitment. + /// + /// Split out of round 2's commitment step. The arm selection + /// (`number_of_parts` 1 / 2 / d>2, the device paths and their fallbacks) is + /// intricate enough that a second copy of it would drift, so producing the + /// parts and committing them are separate functions. + #[allow(clippy::too_many_arguments)] + fn compute_composition_parts( air: &dyn AIR, pub_inputs: &PI, domain: &Domain, twiddles: &LdeTwiddles, - round_1_result: &mut Round1, + // `&mut` for the device-only recovery below: when the host evaluator is + // reached on a device-resident trace, the LDEs are downloaded back into + // the host buffers in place rather than aborting the table. + lde_trace: &mut LDETraceTable, + rap_challenges: &[FieldElement], + bus_public_inputs: Option<&BusPublicInputs>, transition_coefficients: &[FieldElement], boundary_coefficients: &[FieldElement], - ) -> Result, ProvingError> + ) -> Result, ProvingError> where FieldElement: AsBytes, FieldElement: AsBytes, @@ -1698,12 +2900,19 @@ pub trait IsStarkProver< let evaluator = ConstraintEvaluator::new( air, pub_inputs, - &round_1_result.rap_challenges, - round_1_result.bus_public_inputs.as_ref(), + rap_challenges, + bus_public_inputs, trace_length, ); let number_of_parts = air.composition_poly_degree_bound(trace_length) / trace_length; + // Evaluate `H` on the sub-coset when the split allows it (two parts, + // blowup ≥ 4): the same parts from fewer constraint evaluations. + let row_stride = composition_eval_stride(domain.blowup_factor, number_of_parts); + if row_stride > 1 { + COMPOSITION_SUBCOSET_TABLES.fetch_add(1, Ordering::Relaxed); + } + let __ps_c = crate::prove_split::mark(); #[cfg(feature = "instruments")] let t_sub = Instant::now(); #[cfg(feature = "cuda")] @@ -1738,13 +2947,14 @@ pub trait IsStarkProver< let _r2_serial_guard = crate::gpu_lde::r2_serialize_guard(); if let Some(h_dev) = evaluator.evaluate_dev( air, - &round_1_result.lde_trace, + lde_trace, domain, transition_coefficients, boundary_coefficients, - &round_1_result.rap_challenges, + rap_challenges, + row_stride, ) { - let want_host = !round_1_result.lde_trace.host_trace_empty(); + let want_host = !lde_trace.host_trace_empty(); // num_parts==1 de-interleaves `H` (the single part); num_parts==2 // runs the degree-2 quotient split. Both keep the parts resident. match Self::decompose_comp_h_dev( @@ -1753,6 +2963,7 @@ pub trait IsStarkProver< domain, twiddles, want_host, + row_stride, ) { Some((parts, handle)) => { gpu_composition_parts = Some(handle); @@ -1772,12 +2983,14 @@ pub trait IsStarkProver< precomputed_parts = Some(if number_of_parts == 1 { vec![h] } else { - Self::decompose_and_extend_d2(&h, domain, twiddles) + Self::decompose_and_extend_d2(&h, domain, twiddles, row_stride) }); } #[cfg(not(feature = "cuda"))] let precomputed_parts: Option>>> = None; + crate::prove_split::add(&crate::prove_split::R2_CONSTRAINTS, __ps_c); + let __ps_d = crate::prove_split::mark(); #[cfg(feature = "instruments")] let constraints_dur = t_sub.elapsed(); #[cfg(feature = "instruments")] @@ -1786,18 +2999,20 @@ pub trait IsStarkProver< // Every arm below runs the HOST evaluator, which reads `get_main` / // `get_aux`. Under device-only those buffers are intentionally empty, // so landing here means the device decompose AND the `H` download both - // failed. The gate is a static predicate and cannot mirror every - // dynamic decline, so recover rather than abort: download the resident - // LDEs into the host buffers (which also clears the device-only flag) - // and let the host arms run — slower for this table, never wrong. The - // assert is left for the case where the handles themselves cannot - // serve the data, so that failure carries the device-only contract's - // message rather than a bare index-out-of-bounds from somewhere inside - // the evaluator. + // failed. In production that is the failure to report — host RAM is a + // cache, not a compute path — and `materialize_lde_trace_host` aborts + // with the shape and the live VRAM before returning. Only under the + // test-only host fallback (`LAMBDA_VM_TEST_ONLY_HOST_FALLBACK`, the + // `LAMBDA_VM_GPU_FORCE_DOWNGRADE` hook or the `test-cuda-faults` + // feature) does it download the resident LDEs into the host buffers + // (clearing the device-only flag) and let the host arms run; the + // `gpu_force_downgrade` binary asserts on exactly that. The assert is + // left for the case where the handles themselves cannot serve the data, + // so that failure carries the device-only contract's message rather + // than a bare index-out-of-bounds from somewhere inside the evaluator. #[cfg(feature = "cuda")] - if precomputed_parts.is_none() && round_1_result.lde_trace.host_trace_empty() { - let recovered = - crate::gpu_lde::materialize_lde_trace_host(&mut round_1_result.lde_trace); + if precomputed_parts.is_none() && lde_trace.host_trace_empty() { + let recovered = crate::gpu_lde::materialize_lde_trace_host(lde_trace); if recovered { // Rare by design; the name tells which condition the gate is // missing so it can be mirrored as an optimization. @@ -1817,13 +3032,12 @@ pub trait IsStarkProver< air.name(), trace_length, number_of_parts, - round_1_result.lde_trace.num_main_cols(), - round_1_result.lde_trace.num_aux_cols(), + lde_trace.num_main_cols(), + lde_trace.num_aux_cols(), ); } - #[cfg_attr(not(feature = "cuda"), allow(unused_mut))] - let mut lde_composition_poly_parts_evaluations = if let Some(parts) = precomputed_parts { + let lde_composition_poly_parts_evaluations = if let Some(parts) = precomputed_parts { parts } else if number_of_parts == 2 { // Direct quotient decomposition: avoid full-size iFFT by algebraically @@ -1833,32 +3047,35 @@ pub trait IsStarkProver< // On the LDE coset {g·ω^i}, we have -g·ω^i = g·ω^{i+N} since ω^N = -1. let constraint_evaluations = evaluator.evaluate( air, - &round_1_result.lde_trace, + lde_trace, domain, transition_coefficients, boundary_coefficients, - &round_1_result.rap_challenges, + rap_challenges, + row_stride, ); - Self::decompose_and_extend_d2(&constraint_evaluations, domain, twiddles) + Self::decompose_and_extend_d2(&constraint_evaluations, domain, twiddles, row_stride) } else if number_of_parts == 1 { // Degree bound equals trace length: constraint evals are the LDE directly. vec![evaluator.evaluate( air, - &round_1_result.lde_trace, + lde_trace, domain, transition_coefficients, boundary_coefficients, - &round_1_result.rap_challenges, + rap_challenges, + 1, )] } else { // Fallback for any future AIR with d > 2. let constraint_evaluations = evaluator.evaluate( air, - &round_1_result.lde_trace, + lde_trace, domain, transition_coefficients, boundary_coefficients, - &round_1_result.rap_challenges, + rap_challenges, + 1, ); let composition_poly = Polynomial::interpolate_offset_fft(&constraint_evaluations, &domain.coset_offset)?; @@ -1906,9 +3123,57 @@ pub trait IsStarkProver< cpu_eval()? }; + crate::prove_split::add(&crate::prove_split::R2_DECOMPOSE, __ps_d); #[cfg(feature = "instruments")] let fft_dur = t_sub.elapsed(); + Ok(CompositionParts { + parts: lde_composition_poly_parts_evaluations, + #[cfg(feature = "cuda")] + gpu_parts: gpu_composition_parts, + #[cfg(feature = "instruments")] + constraints_dur, + #[cfg(feature = "instruments")] + fft_dur, + }) + } + + /// Returns the result of the second round of the STARK Prove protocol. + fn round_2_compute_composition_polynomial( + air: &dyn AIR, + pub_inputs: &PI, + domain: &Domain, + twiddles: &LdeTwiddles, + round_1_result: &mut Round1, + transition_coefficients: &[FieldElement], + boundary_coefficients: &[FieldElement], + ) -> Result, ProvingError> + where + FieldElement: AsBytes, + FieldElement: AsBytes, + { + let computed = Self::compute_composition_parts( + air, + pub_inputs, + domain, + twiddles, + &mut round_1_result.lde_trace, + &round_1_result.rap_challenges, + round_1_result.bus_public_inputs.as_ref(), + transition_coefficients, + boundary_coefficients, + )?; + // `mut` for the device-only recovery in the commit arm below, which + // repopulates these from the resident parts handle. + #[cfg_attr(not(feature = "cuda"), allow(unused_mut))] + let mut lde_composition_poly_parts_evaluations = computed.parts; + #[cfg(feature = "cuda")] + let gpu_composition_parts = computed.gpu_parts; + #[cfg(feature = "instruments")] + let constraints_dur = computed.constraints_dur; + #[cfg(feature = "instruments")] + let fft_dur = computed.fft_dur; + // Fold the R2 device composition parts handle into the session // (resident R2 to R4) before the commit: the tree build below, its // recovery, R3 OOD, R4 DEEP and the openings all read it from the @@ -1918,8 +3183,13 @@ pub trait IsStarkProver< round_1_result.lde_trace.set_gpu_composition_parts(handle); } + let __ps_r2c = crate::prove_split::mark(); #[cfg(feature = "instruments")] let t_sub = Instant::now(); + // The table's leaf layout (S2): the composition tree, device or host, + // carries `leaf_layout.rows_per_leaf()` rows per leaf. + let leaf_layout = + crate::leaf_layout::table_leaf_layout(air, domain.interpolation_domain_size); // GPU fast path for the comp-poly Merkle commit: hash straight from // the resident parts handle when R2 kept one (no host pack + H2D // re-upload); otherwise wrap the host eval Vecs. Either way the tree @@ -1934,14 +3204,17 @@ pub trait IsStarkProver< .and_then(|h| { crate::gpu_lde::try_build_comp_poly_tree_gpu_from_dev::< FieldExtension, - BatchedMerkleTreeBackend, - >(h) + H::Batched, + >(h, leaf_layout.rows_per_leaf()) }) .or_else(|| { crate::gpu_lde::try_build_comp_poly_tree_gpu::< FieldExtension, - BatchedMerkleTreeBackend, - >(&lde_composition_poly_parts_evaluations) + H::Batched, + >( + &lde_composition_poly_parts_evaluations, + leaf_layout.rows_per_leaf(), + ) }) { Some((host_tree, dev_tree)) => { let root = host_tree.root; @@ -1967,9 +3240,12 @@ pub trait IsStarkProver< on a device-only table and the resident parts handle \ could not be downloaded" ); - let (tree, root) = crate::commitment::commit_bit_reversed( + let (tree, root) = crate::commitment::commit_bit_reversed_with::< + FieldExtension, + H::Batched, + >( &lde_composition_poly_parts_evaluations, - crate::commitment::ROWS_PER_LEAF, + leaf_layout.rows_per_leaf(), ) .ok_or(ProvingError::EmptyCommitment)?; (tree, root, None) @@ -1977,11 +3253,12 @@ pub trait IsStarkProver< }; #[cfg(not(feature = "cuda"))] let (composition_poly_merkle_tree, composition_poly_root) = - crate::commitment::commit_bit_reversed( + crate::commitment::commit_bit_reversed_with::>( &lde_composition_poly_parts_evaluations, - crate::commitment::ROWS_PER_LEAF, + leaf_layout.rows_per_leaf(), ) .ok_or(ProvingError::EmptyCommitment)?; + crate::prove_split::add(&crate::prove_split::R2_COMMIT, __ps_r2c); #[cfg(feature = "instruments")] let merkle_dur = t_sub.elapsed(); @@ -2001,15 +3278,18 @@ pub trait IsStarkProver< fn round_3_evaluate_polynomials_in_out_of_domain_element( air: &dyn AIR, domain: &Domain, - round_1_result: &mut Round1, - round_2_result: &mut Round2, + // Both `&mut` for the device-only recoveries below: the parts OOD arm + // repopulates the host part evals from the resident handle, and the + // trace OOD reads through a trace that may have to be materialized. + lde_trace: &mut LDETraceTable, + composition_parts: &mut [Vec>], z: &FieldElement, ) -> Round3 where FieldElement: AsBytes, FieldElement: AsBytes, { - let num_parts = round_2_result.lde_composition_poly_evaluations.len(); + let num_parts = composition_parts.len(); let z_power = z.pow(num_parts); let domain_size = domain.interpolation_domain_size; let blowup_factor = domain.blowup_factor; @@ -2025,8 +3305,7 @@ pub trait IsStarkProver< // the host stride-extract and the sequential CPU fold per part. #[cfg(feature = "cuda")] let gpu_parts_ood: Option>> = - round_1_result - .lde_trace + lde_trace .gpu_composition_parts() .and_then(|parts_dev| { let dispatch = |inv_host: &[FieldElement], @@ -2046,7 +3325,7 @@ pub trait IsStarkProver< match crate::gpu_lde::try_prep_r3_dev_context::( &dc.points, std::slice::from_ref(&z_power), - round_1_result.lde_trace.bound_stream(), + lde_trace.bound_stream(), ) { Some(ctx) => dispatch(&[], Some((&ctx, 0))), // Below the dev-context threshold (single eval point): @@ -2072,8 +3351,8 @@ pub trait IsStarkProver< #[cfg(feature = "cuda")] { let recovered = crate::gpu_lde::materialize_composition_parts_host( - &round_1_result.lde_trace, - &mut round_2_result.lde_composition_poly_evaluations, + lde_trace, + composition_parts, ); assert!( recovered, @@ -2084,8 +3363,7 @@ pub trait IsStarkProver< } let comp_inv_denoms = math::polynomial::barycentric_inv_denoms(&z_power, &dc.points); - round_2_result - .lde_composition_poly_evaluations + composition_parts .iter() .map(|lde_evals| { // Extract trace-size evaluations (stride = blowup_factor) @@ -2108,7 +3386,7 @@ pub trait IsStarkProver< // === Trace polynomials: barycentric evaluation via LDE === let trace_ood_evaluations = crate::trace::get_trace_evaluations_from_lde( - &mut round_1_result.lde_trace, + lde_trace, domain, z, &air.context().transition_offsets, @@ -2140,25 +3418,44 @@ pub trait IsStarkProver< } /// Returns the result of the fourth round of the STARK Prove protocol. + #[allow(clippy::too_many_arguments)] fn round_4_compute_and_run_fri_on_the_deep_composition_polynomial( air: &dyn AIR, domain: &Domain, - round_1_result: &mut Round1, - round_2_result: &mut Round2, + round_1_result: &mut Round1, + round_2_result: &mut Round2, round_3_result: &Round3, z: &FieldElement, + commit_ood: Option>, transcript: &mut (impl IsStarkTranscript + Clone), - ) -> Round4 + ) -> Result, ProvingError> where FieldElement: AsBytes, FieldElement: AsBytes, { + // The FRI fold layout of this table's proof format (a verifier-side + // constant built from the options, the same call the verifier makes). + // A format this build cannot lay out is refused here, before anything + // enters the transcript. + let leaf_layout = + crate::leaf_layout::table_leaf_layout(air, domain.interpolation_domain_size); + let fri_layout = crate::fri::terminal::FriFoldLayout::for_options( + domain.lde_roots_of_unity_coset.len().trailing_zeros(), + domain.blowup_factor.trailing_zeros(), + air.options(), + leaf_layout.is_one_row(), + ) + .map_err(|e| ProvingError::WrongParameter(format!("FRI format: {e}")))?; + let coset_offset_u64 = air.context().proof_options.coset_offset; let coset_offset = FieldElement::::from(coset_offset_u64); let gamma = transcript.sample_field_element(); - let n_terms_composition_poly = round_2_result.lde_composition_poly_evaluations.len(); + // The parts under their shared name; round 4 still holds the whole + // `Round2` because the openings need its Merkle tree. + let composition_parts = &round_2_result.lde_composition_poly_evaluations; + let n_terms_composition_poly = composition_parts.len(); // g·z pruning: only the current-row block (all columns) plus the masked // next-row columns get an opening / DEEP coefficient. let layout = Self::ood_layout(air); @@ -2176,51 +3473,99 @@ pub trait IsStarkProver< // Rectangular W×num_eval_points grid with the sampled powers at surviving // positions and zeros at pruned next-row positions, so the DEEP loop // below (and the GPU path) stay unchanged — zero-coefficient terms vanish. - let trace_term_coeffs = layout.build_trace_term_coeffs(&trace_term_powers); + #[allow(unused_mut)] + let mut trace_term_coeffs = layout.build_trace_term_coeffs(&trace_term_powers); // <<<< Receive challenges: 𝛾ⱼ, 𝛾ⱼ' let gammas = deep_composition_coefficients; + // The commitment-time answers' pole (`crate::commit_ood`): one more + // point of the rectangular grid, with minus the column's current-row + // coefficient and OOD value v_i on the i-th answered column, zero + // everywhere else, so every DEEP arm below folds each answered + // column's two-point quotient unchanged. + let mut trace_ood_columns = round_3_result.trace_ood_evaluations.columns(); + let extra_pole = match commit_ood { + Some(ood) => { + let columns = crate::commit_ood::answered_columns(air); + if ood.answers.len() != columns.len() { + return Err(ProvingError::WrongParameter(format!( + "commitment-time answers: {} for {} answered columns", + ood.answers.len(), + columns.len() + ))); + } + for (col, (coeff_col, ood_col)) in trace_term_coeffs + .iter_mut() + .zip(trace_ood_columns.iter_mut()) + .enumerate() + { + match col.checked_sub(columns.start) { + Some(i) if columns.contains(&col) => { + let at_z0 = crate::commit_ood::answer_coefficient(&coeff_col[0]); + coeff_col.push(at_z0); + ood_col.push(ood.answers[i].clone()); + } + _ => { + coeff_col.push(FieldElement::zero()); + ood_col.push(FieldElement::zero()); + } + } + } + Some(ood.point) + } + None => None, + }; + let domain_size = domain.lde_roots_of_unity_coset.len(); // Fully device-resident DEEP → FRI: the codeword is computed, bit- // reversed, and folded on device without crossing PCIe. On any miss // (gates, cudarc failure — the FRI driver restores the transcript) // the host path below recomputes DEEP through its own arms. + let __ps_df = crate::prove_split::mark(); #[cfg(feature = "instruments")] let t_sub = Instant::now(); + // Device FRI implements the pair and group (S3) encodings and the + // one-row layout (S2), whose input tree is committed from the resident + // codeword before the first challenge. #[cfg(feature = "cuda")] - let precomputed_fri = Self::try_compute_deep_dev( - &round_1_result.lde_trace, - round_2_result, - round_3_result, - z, - domain, - &domain.trace_primitive_root, - &gammas, - &trace_term_coeffs, - ) - .and_then(|dw| { - crate::gpu_lde::try_fri_commit_gpu_from_dev( - dw, - transcript, - &coset_offset, - domain.blowup_factor.trailing_zeros(), - air.options().fri_final_poly_log_degree as u32, - domain.fri_inv_twiddles(), - !round_1_result.lde_trace.host_trace_empty(), + let precomputed_fri = { + Self::try_compute_deep_dev( + &round_1_result.lde_trace, + composition_parts, + round_3_result, + &trace_ood_columns, + z, + extra_pole, + domain, + &domain.trace_primitive_root, + &gammas, + &trace_term_coeffs, ) - }); + .and_then(|dw| { + crate::gpu_lde::try_fri_commit_gpu_from_dev::< + Field, + FieldExtension, + _, + H::Pair, + >( + dw, + transcript, + &coset_offset, + domain.blowup_factor.trailing_zeros(), + air.options().fri_final_poly_log_degree as u32, + &fri_layout, + domain.fri_inv_twiddles(), + !round_1_result.lde_trace.host_trace_empty(), + ) + }) + }; #[cfg(not(feature = "cuda"))] #[allow(clippy::type_complexity)] let precomputed_fri: Option<( Vec>, - Vec< - crate::fri::fri_commitment::FriLayer< - FieldExtension, - crate::config::FriLayerMerkleTreeBackend, - >, - >, + Vec>>, )> = None; #[cfg(feature = "instruments")] let mut other_dur_1 = t_sub.elapsed(); @@ -2237,9 +3582,11 @@ pub trait IsStarkProver< let t_sub = Instant::now(); let deep_evals = Self::compute_deep_composition_poly_evaluations( &mut round_1_result.lde_trace, - round_2_result, + &mut round_2_result.lde_composition_poly_evaluations, round_3_result, + &trace_ood_columns, z, + extra_pole, domain, &domain.trace_primitive_root, &gammas, @@ -2264,13 +3611,14 @@ pub trait IsStarkProver< // FRI commit phase from pre-computed evaluations #[cfg(feature = "instruments")] let t_sub = Instant::now(); - let res = fri::commit_phase_from_evaluations( + let res = fri::commit_phase_with_layout::( lde_evals, transcript, &coset_offset, domain_size, domain.blowup_factor.trailing_zeros(), air.options().fri_final_poly_log_degree as u32, + &fri_layout, domain.fri_inv_twiddles(), ); #[cfg(feature = "instruments")] @@ -2280,31 +3628,87 @@ pub trait IsStarkProver< res }; + crate::prove_split::add(&crate::prove_split::R4_DEEP_FRI, __ps_df); + let __ps_g = crate::prove_split::mark(); // grinding: generate nonce and append it to the transcript #[cfg(feature = "instruments")] let t_sub = Instant::now(); + // `grinding_factor`, not `security_bits`: proof-of-work is a prover cost + // multiplier, and the two are not interchangeable names for one number. let grinding_factor = air.context().proof_options.grinding_factor; let mut nonce = None; if grinding_factor > 0 { + // ★ `H`'s own digest, never a fixed one: the block path proves + // under a separately pinned configuration, and a hard-coded + // `StarkGrindingDigest` here would grind on keccak for a proof that + // had moved everything else. Which device arm that digest takes is + // the digest's own `GrindDigest::DEVICE_GRIND`, so there is no list + // anywhere that a new hash can be missing from. let nonce_value = - grinding::generate_nonce_maybe_gpu(&transcript.state(), grinding_factor) - .expect("nonce not found"); + grinding::generate_nonce_maybe_gpu::>( + &transcript.state(), + grinding_factor, + ) + .expect("nonce not found"); transcript.append_bytes(&nonce_value.to_be_bytes()); nonce = Some(nonce_value); } + crate::prove_split::add(&crate::prove_split::R4_GRIND, __ps_g); + let __ps_q = crate::prove_split::mark(); let number_of_queries = air.options().fri_number_of_queries; - let iotas = Self::sample_query_indexes(number_of_queries, domain, transcript); + let iotas = Self::sample_query_indexes(number_of_queries, domain, leaf_layout, transcript); - let query_list = fri::query_phase(&fri_layers, &iotas); + let mut query_list = + fri::query_phase_with_layout::(&fri_layers, &iotas, &fri_layout); let fri_layers_merkle_roots: Vec<_> = fri_layers .iter() .map(|layer| layer.merkle_tree.root) .collect(); - let deep_poly_openings = - Self::open_deep_composition_poly(domain, round_1_result, round_2_result, &iotas); + let mut deep_poly_openings = Self::open_deep_composition_poly( + domain, + round_1_result, + round_2_result, + &iotas, + leaf_layout, + ); + + // Merkle caps: a post-pass over the finished + // openings. The heights are the verifier's (`StarkCaps`, public shape + // only); nothing is absorbed, so the transcript is the uncapped one. + // At the default format every height is 0 and this is skipped. + // + // Every depth is the layout's: the trace trees' from the table's leaf + // layout (row pairs `log2(lde) − 1`, one row `log2(lde)`) and each FRI + // layer's from the fold layout (a group tree under a fold schedule), + // so a capped `fri = dp` or one-row proof caps the trees it committed. + let caps = crate::merkle_caps::StarkCaps::from_layout_arity( + crate::config::effective_cap_policy::(air.options().format.merkle_cap), + number_of_queries, + domain_size.trailing_zeros() as usize, + &fri_layout, + H::ARITY, + ); + if caps.fri.len() != fri_layers.len() { + return Err(ProvingError::WrongParameter(format!( + "Merkle cap: the FRI layout commits {} layers, the prover built {}", + caps.fri.len(), + fri_layers.len() + ))); + } + if caps.any() { + Self::embed_stark_caps( + &caps, + round_1_result, + round_2_result, + &fri_layers, + &mut deep_poly_openings, + &mut query_list, + )?; + } + crate::prove_split::add(&crate::prove_split::R4_QUERIES, __ps_q); #[cfg(feature = "instruments")] { @@ -2312,23 +3716,237 @@ pub trait IsStarkProver< crate::instruments::store_r4_sub(r4_fft_dur, r4_merkle_dur, other_dur_1, queries_dur); } - Round4 { - fri_final_poly_coeffs, - fri_layers_merkle_roots, - deep_poly_openings, - query_list, - nonce, + Ok(Round4 { + fri_final_poly_coeffs, + fri_layers_merkle_roots, + deep_poly_openings, + query_list, + nonce, + }) + } + + /// Embed every capped tree's cap into its owner path and cut every path of + /// that tree to `depth − c` siblings. + /// + /// Per tree: read the cap (the host tree's heap slice; see + /// [`Self::tree_cap`] for a device-resident tree), then + /// [`embed_cap`](crypto::merkle_tree::cap::embed_cap) over the tree's + /// paths in proof order, so query 0 is the owner. Every path must be the + /// full `depth` long (checked), so a tree whose depth disagrees with the + /// verifier's constant fails here instead of producing a proof the + /// verifier rejects. + fn embed_stark_caps( + caps: &crate::merkle_caps::StarkCaps, + round_1_result: &Round1, + round_2_result: &Round2, + fri_layers: &[crate::fri::fri_commitment::FriLayer< + FieldExtension, + H::Pair, + >], + deep_poly_openings: &mut [DeepPolynomialOpening], + query_list: &mut [FriDecommitment], + ) -> Result<(), ProvingError> + where + FieldElement: AsBytes, + FieldElement: AsBytes, + { + fn embed<'p>( + paths: impl Iterator>>, + depth: usize, + cap: &[Commitment], + what: &str, + arity: usize, + ) -> Result<(), ProvingError> { + let mut paths: Vec<&mut Vec> = + paths.collect::>().ok_or_else(|| { + ProvingError::WrongParameter(format!( + "Merkle cap: an opening of the {what} tree is missing" + )) + })?; + crypto::merkle_tree::cap::embed_cap_arity(&mut paths, depth, cap, arity).map_err(|e| { + ProvingError::WrongParameter(format!("Merkle cap of the {what} tree: {e}")) + }) + } + + // The device arm of `tree_cap`: read the cap off the resident tree on + // `stream`. `None` when the tree is not device-resident. + #[cfg(feature = "cuda")] + fn dev<'t>( + tree: Option<&'t math_cuda::lde::GpuMerkleTree>, + stream: impl FnOnce() -> Option> + 't, + ) -> impl FnOnce(usize) -> Option, String>> + 't { + move |c| { + tree.map(|tree| { + let stream = stream().ok_or("no CUDA stream for the device cap read")?; + crate::gpu_lde::read_cap_dev(tree, c, &stream) + }) + } + } + #[cfg(feature = "cuda")] + let lde_trace = &round_1_result.lde_trace; + + let (depth, c) = (caps.trace_depth, caps.trace); + if c > 0 { + #[cfg(feature = "cuda")] + let main_dev = dev(lde_trace.gpu_main().and_then(|h| h.tree.as_ref()), || { + lde_trace.bound_stream() + }); + #[cfg(not(feature = "cuda"))] + let main_dev = |_| None; + let main_cap = Self::tree_cap(&round_1_result.main.tree, depth, c, "main", main_dev)?; + embed( + deep_poly_openings + .iter_mut() + .map(|o| Some(&mut o.main_trace_polys.proof.merkle_path)), + depth, + &main_cap, + "main", + H::ARITY, + )?; + if let Some(tree) = round_1_result.main.precomputed_tree.as_ref() { + // Always a full host tree (the process-wide cache; its openings + // walk it on the host too), so there is no device arm. + let cap = Self::tree_cap(tree, depth, c, "precomputed", |_| None)?; + embed( + deep_poly_openings.iter_mut().map(|o| { + o.precomputed_trace_polys + .as_mut() + .map(|p| &mut p.proof.merkle_path) + }), + depth, + &cap, + "precomputed", + H::ARITY, + )?; + } + if let Some(aux) = round_1_result.aux.as_ref() { + #[cfg(feature = "cuda")] + let aux_dev = dev(lde_trace.gpu_aux().and_then(|h| h.tree.as_ref()), || { + lde_trace.bound_stream() + }); + #[cfg(not(feature = "cuda"))] + let aux_dev = |_| None; + let cap = Self::tree_cap(&aux.tree, depth, c, "aux", aux_dev)?; + embed( + deep_poly_openings + .iter_mut() + .map(|o| o.aux_trace_polys.as_mut().map(|p| &mut p.proof.merkle_path)), + depth, + &cap, + "aux", + H::ARITY, + )?; + } + #[cfg(feature = "cuda")] + let comp_dev = dev(round_2_result.gpu_composition_tree.as_ref(), || { + lde_trace.bound_stream() + }); + #[cfg(not(feature = "cuda"))] + let comp_dev = |_| None; + let cap = Self::tree_cap( + &round_2_result.composition_poly_merkle_tree, + depth, + c, + "composition", + comp_dev, + )?; + embed( + deep_poly_openings + .iter_mut() + .map(|o| Some(&mut o.composition_poly.proof.merkle_path)), + depth, + &cap, + "composition", + H::ARITY, + )?; + } + + for (i, layer) in fri_layers.iter().enumerate() { + let (depth, c) = (caps.fri_depths[i], caps.fri[i]); + if c == 0 { + continue; + } + let what = format!("FRI layer {i}"); + // A fresh backend stream, as the device FRI query phase reads the + // same resident layer trees (`try_fri_query_phase_gpu`). + #[cfg(feature = "cuda")] + let layer_dev = dev(layer.gpu_tree.as_ref(), || { + math_cuda::device::backend().ok().map(|b| b.next_stream()) + }); + #[cfg(not(feature = "cuda"))] + let layer_dev = |_| None; + let cap = Self::tree_cap(&layer.merkle_tree, depth, c, &what, layer_dev)?; + embed( + query_list + .iter_mut() + .map(|q| q.layers_auth_paths.get_mut(i).map(|p| &mut p.merkle_path)), + depth, + &cap, + &what, + H::ARITY, + )?; + } + Ok(()) + } + + /// The height-`c` cap of one tree of depth `depth`. + /// + /// A full host tree serves it from its heap (`MerkleTree::cap`), after + /// checking the tree's depth is the verifier's. A root-only + /// host tree means the nodes are device-resident: `device(c)` reads the + /// cap off the resident tree, and `None` from it (no resident tree) is a + /// hard error naming the tree — never a skipped cap, which would ship + /// full-length paths the verifier rejects with no pointer to the cause. + fn tree_cap( + host: &MerkleTree, + depth: usize, + c: usize, + what: &str, + device: impl FnOnce(usize) -> Option, String>>, + ) -> Result, ProvingError> + where + B: IsMerkleTreeBackend, + { + if !host.is_root_only() { + // `MerkleTree::depth` counts the tree's own levels (4-ary at arity 4). + let levels = crypto::merkle_tree::cap::tree_levels(depth, B::ARITY); + if host.depth() != Some(levels) { + return Err(ProvingError::WrongParameter(format!( + "Merkle cap: the {what} tree has {:?} levels, the format expects {levels} \ + (depth {depth})", + host.depth() + ))); + } + return host.cap(c).ok_or_else(|| { + ProvingError::WrongParameter(format!( + "Merkle cap: height {c} does not fit the {what} tree (depth {depth})" + )) + }); + } + match device(c) { + Some(Ok(cap)) => Ok(cap), + Some(Err(e)) => Err(ProvingError::DevicePath(format!( + "Merkle cap: reading the height-{c} cap of the device-resident {what} tree \ + failed: {e}" + ))), + None => Err(ProvingError::DevicePath(format!( + "Merkle cap: the {what} tree is device-resident (its host tree is root-only) \ + and no device cap read is wired for it" + ))), } } + /// The query indexes: trace-tree leaf indexes, uniform below + /// [`LeafLayout::query_bound`] (`lde / 2` today, `lde` under one row). fn sample_query_indexes( number_of_queries: usize, domain: &Domain, + leaf_layout: LeafLayout, transcript: &mut impl IsStarkTranscript, ) -> Vec { - let domain_size = domain.lde_roots_of_unity_coset.len() as u64; + let bound = leaf_layout.query_bound(domain.lde_roots_of_unity_coset.len() as u64); (0..number_of_queries) - .map(|_| (transcript.sample_u64(domain_size >> 1)) as usize) + .map(|_| (transcript.sample_u64(bound)) as usize) .collect::>() } @@ -2349,9 +3967,11 @@ pub trait IsStarkProver< #[allow(clippy::too_many_arguments)] fn try_compute_deep_dev( lde_trace: &LDETraceTable, - round_2_result: &Round2, + composition_parts: &[Vec>], round_3_result: &Round3, + trace_ood_columns: &[Vec>], z: &FieldElement, + extra_pole: Option<&FieldElement>, domain: &Domain, primitive_root: &FieldElement, composition_poly_gammas: &[FieldElement], @@ -2362,21 +3982,34 @@ pub trait IsStarkProver< FieldElement: AsBytes, { let parts_dev = lde_trace.gpu_composition_parts()?; - let num_parts = round_2_result.lde_composition_poly_evaluations.len(); + let num_parts = composition_parts.len(); let z_power = z.pow(num_parts); let num_eval_points = if trace_terms_gammas.is_empty() { 0 } else { trace_terms_gammas[0].len() }; - let mut z_shifted = Vec::with_capacity(num_eval_points); - let mut current_z = z.clone(); - for _ in 0..num_eval_points { - z_shifted.push(current_z.clone()); - current_z = primitive_root * ¤t_z; - } + let z_shifted = trace_poles(z, extra_pole, primitive_root, num_eval_points); let z_scalars: Vec> = core::iter::once(z_power).chain(z_shifted).collect(); + // The rows invert their own denominators, so there is no inverse + // buffer; `LAMBDA_VM_DEEP_INV_LEGACY=1` keeps the buffered kernel. + if math_cuda::deep_inv::rowwise_enabled() + && let Some(dw) = + crate::gpu_lde::try_deep_composition_gpu_fused_keep::( + lde_trace, + parts_dev, + &round_3_result.composition_poly_parts_ood_evaluation, + trace_ood_columns, + composition_poly_gammas, + trace_terms_gammas, + &domain.lde_roots_of_unity_coset, + &z_scalars, + num_eval_points, + ) + { + return Some(dw); + } let (inv_dev, stream) = crate::gpu_lde::try_inv_denoms_dev_with_stream::( &domain.lde_roots_of_unity_coset, @@ -2388,7 +4021,7 @@ pub trait IsStarkProver< lde_trace, parts_dev, &round_3_result.composition_poly_parts_ood_evaluation, - &round_3_result.trace_ood_evaluations.columns(), + trace_ood_columns, composition_poly_gammas, trace_terms_gammas, (&inv_dev, &stream), @@ -2398,10 +4031,15 @@ pub trait IsStarkProver< #[allow(clippy::too_many_arguments)] fn compute_deep_composition_poly_evaluations( + // Both `&mut` for the device-only recovery in the host DEEP loop: the + // trace and the part evals are downloaded back in place there rather + // than aborting the table. lde_trace: &mut LDETraceTable, - round_2_result: &mut Round2, + composition_parts: &mut [Vec>], round_3_result: &Round3, + trace_ood_columns: &[Vec>], z: &FieldElement, + extra_pole: Option<&FieldElement>, domain: &Domain, primitive_root: &FieldElement, composition_poly_gammas: &[FieldElement], @@ -2411,7 +4049,7 @@ pub trait IsStarkProver< FieldElement: AsBytes, FieldElement: AsBytes, { - let num_parts = round_2_result.lde_composition_poly_evaluations.len(); + let num_parts = composition_parts.len(); let z_power = z.pow(num_parts); // pole for H terms // Number of evaluation points per trace column (= transition_offsets.len() * step_size) @@ -2421,13 +4059,8 @@ pub trait IsStarkProver< trace_terms_gammas[0].len() }; - // Trace poles: z_shifted[k] = primitive_root^k * z for k = 0..num_eval_points - let mut z_shifted = Vec::with_capacity(num_eval_points); - let mut current_z = z.clone(); - for _ in 0..num_eval_points { - z_shifted.push(current_z.clone()); - current_z = primitive_root * ¤t_z; - } + // Trace poles: z_shifted[k] = primitive_root^k * z, then the answers' pole. + let z_shifted = trace_poles(z, extra_pole, primitive_root, num_eval_points); // Number of main and aux columns in the LDE trace let num_main_cols = lde_trace.num_main_cols(); @@ -2436,7 +4069,6 @@ pub trait IsStarkProver< // OOD evaluations let h_ood = &round_3_result.composition_poly_parts_ood_evaluation; - let trace_ood_columns = round_3_result.trace_ood_evaluations.columns(); let num_total_cols = num_main_cols + num_aux_cols; // Fully device-resident GPU fast path: build inv_denoms on device @@ -2460,9 +4092,9 @@ pub trait IsStarkProver< crate::gpu_lde::try_deep_composition_gpu::( lde_trace, lde_trace.gpu_composition_parts(), - &round_2_result.lde_composition_poly_evaluations, + composition_parts, h_ood, - &trace_ood_columns, + trace_ood_columns, composition_poly_gammas, trace_terms_gammas, &[], @@ -2496,9 +4128,9 @@ pub trait IsStarkProver< crate::gpu_lde::try_deep_composition_gpu::( lde_trace, lde_trace.gpu_composition_parts(), - &round_2_result.lde_composition_poly_evaluations, + composition_parts, h_ood, - &trace_ood_columns, + trace_ood_columns, composition_poly_gammas, trace_terms_gammas, &denoms, @@ -2526,10 +4158,8 @@ pub trait IsStarkProver< downloaded" ); } - let parts_recovered = crate::gpu_lde::materialize_composition_parts_host( - lde_trace, - &mut round_2_result.lde_composition_poly_evaluations, - ); + let parts_recovered = + crate::gpu_lde::materialize_composition_parts_host(lde_trace, composition_parts); assert!( parts_recovered, "R4 DEEP composition fell back to the host part evals on a \ @@ -2580,7 +4210,7 @@ pub trait IsStarkProver< // H terms for j in 0..num_parts { - let h_j_val = &round_2_result.lde_composition_poly_evaluations[j][i]; + let h_j_val = &composition_parts[j][i]; let h_j_ood = &h_ood[j]; result += &composition_poly_gammas[j] * (h_j_val - h_j_ood) * &inv_h[i]; } @@ -2607,9 +4237,10 @@ pub trait IsStarkProver< /// at the domain value corresponding to the FRI query challenge `index` and its symmetric /// element. fn open_composition_poly( - composition_poly_merkle_tree: &BatchedMerkleTree, + composition_poly_merkle_tree: &MerkleTree>, lde_composition_poly_evaluations: &[Vec>], index: usize, + leaf_layout: LeafLayout, ) -> PolynomialOpenings where FieldElement: AsBytes + Sync + Send, @@ -2618,28 +4249,39 @@ pub trait IsStarkProver< let proof = composition_poly_merkle_tree .get_proof_by_pos(index) .expect("FRI query index in bounds"); + Self::composition_opening_from_proof( + proof, + lde_composition_poly_evaluations, + index, + leaf_layout, + ) + } - let lde_composition_poly_parts_evaluation: Vec<_> = lde_composition_poly_evaluations - .iter() - .flat_map(|part| { - vec![ - part[reverse_index(index * 2, part.len() as u64)].clone(), - part[reverse_index(index * 2 + 1, part.len() as u64)].clone(), - ] - }) - .collect(); - + /// The composition parts' values at query `index` (the rows + /// [`LeafLayout::query_rows`] names) with an already-built Merkle proof: + /// both rows for a row pair, the one row (and an empty `evaluations_sym`) + /// for one row. + fn composition_opening_from_proof( + proof: Proof, + lde_composition_poly_evaluations: &[Vec>], + index: usize, + leaf_layout: LeafLayout, + ) -> PolynomialOpenings + where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + let rows = + |part: &Vec>| leaf_layout.query_rows(index, part.len()); PolynomialOpenings { proof, - evaluations: lde_composition_poly_parts_evaluation - .clone() - .into_iter() - .step_by(2) + evaluations: lde_composition_poly_evaluations + .iter() + .map(|part| part[rows(part).0].clone()) .collect(), - evaluations_sym: lde_composition_poly_parts_evaluation - .into_iter() - .skip(1) - .step_by(2) + evaluations_sym: lde_composition_poly_evaluations + .iter() + .filter_map(|part| rows(part).1.map(|r| part[r].clone())) .collect(), } } @@ -2647,40 +4289,25 @@ pub trait IsStarkProver< /// Like [`Self::open_composition_poly`] but uses a Merkle proof already /// gathered from the resident device composition tree /// ([`crate::gpu_lde::gather_proofs_dev`]) instead of walking a host tree. - /// Row-pair leaf: one proof at position `index` authenticates both rows. + /// One proof at position `index` authenticates the leaf: both rows of a + /// row pair, or the one row (S2). #[cfg(feature = "cuda")] fn open_composition_poly_with_proof( proof: Proof, lde_composition_poly_evaluations: &[Vec>], index: usize, + leaf_layout: LeafLayout, ) -> PolynomialOpenings where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, { - let lde_composition_poly_parts_evaluation: Vec<_> = lde_composition_poly_evaluations - .iter() - .flat_map(|part| { - vec![ - part[reverse_index(index * 2, part.len() as u64)].clone(), - part[reverse_index(index * 2 + 1, part.len() as u64)].clone(), - ] - }) - .collect(); - - PolynomialOpenings { + Self::composition_opening_from_proof( proof, - evaluations: lde_composition_poly_parts_evaluation - .clone() - .into_iter() - .step_by(2) - .collect(), - evaluations_sym: lde_composition_poly_parts_evaluation - .into_iter() - .skip(1) - .step_by(2) - .collect(), - } + lde_composition_poly_evaluations, + index, + leaf_layout, + ) } /// Computes values and validity proofs of the evaluations of trace polynomials at @@ -2689,8 +4316,9 @@ pub trait IsStarkProver< /// storage (full main row, ranged main row, or aux row). fn open_polys_with( domain: &Domain, - tree: &BatchedMerkleTree, + tree: &MerkleTree>, challenge: usize, + leaf_layout: LeafLayout, gather: G, ) -> PolynomialOpenings where @@ -2698,29 +4326,33 @@ pub trait IsStarkProver< FieldElement: AsBytes + Sync + Send, G: Fn(usize) -> Vec>, { - let domain_size = domain.lde_roots_of_unity_coset.len() as u64; - // Rows `2·challenge` and `2·challenge+1` are committed together as the - // single leaf at position `challenge`; one Merkle path authenticates both - // the queried row and its symmetric counterpart. + // Row pairs: rows `2·challenge` and `2·challenge+1` are committed + // together as the single leaf at position `challenge`; one Merkle path + // authenticates both the queried row and its symmetric counterpart. + // One row: the leaf at `challenge` is the row alone, and there is no + // symmetric row. + let (row, sym) = leaf_layout.query_rows(challenge, domain.lde_roots_of_unity_coset.len()); PolynomialOpenings { proof: tree .get_proof_by_pos(challenge) .expect("FRI query index in bounds"), - evaluations: gather(reverse_index(challenge * 2, domain_size)), - evaluations_sym: gather(reverse_index(challenge * 2 + 1, domain_size)), + evaluations: gather(row), + evaluations_sym: sym.map(&gather).unwrap_or_default(), } } /// Like [`Self::open_polys_with`], but uses a Merkle proof already gathered /// from the resident device tree (see [`crate::gpu_lde::gather_proofs_dev`]) - /// instead of walking a host tree. Row-pair leaf: one proof at position - /// `challenge` authenticates both the queried row and its symmetric - /// counterpart. Evaluations still come from the host LDE columns via `gather`. + /// instead of walking a host tree. One proof at position `challenge` + /// authenticates the leaf: the queried row and its symmetric counterpart + /// (row pair), or the one row (S2). Evaluations still come from the host + /// LDE columns via `gather`. #[cfg(feature = "cuda")] fn open_polys_with_proofs( domain: &Domain, proof: Proof, challenge: usize, + leaf_layout: LeafLayout, gather: G, ) -> PolynomialOpenings where @@ -2728,11 +4360,11 @@ pub trait IsStarkProver< FieldElement: AsBytes + Sync + Send, G: Fn(usize) -> Vec>, { - let domain_size = domain.lde_roots_of_unity_coset.len() as u64; + let (row, sym) = leaf_layout.query_rows(challenge, domain.lde_roots_of_unity_coset.len()); PolynomialOpenings { proof, - evaluations: gather(reverse_index(challenge * 2, domain_size)), - evaluations_sym: gather(reverse_index(challenge * 2 + 1, domain_size)), + evaluations: gather(row), + evaluations_sym: sym.map(&gather).unwrap_or_default(), } } @@ -2753,17 +4385,37 @@ pub trait IsStarkProver< } } - /// Slice out query `qi`'s even/odd row (each `ncols` field elements) from the - /// row-major device gather `[even(q0), odd(q0), even(q1), odd(q1), ...]`. + /// The LDE rows the device gathers for `queries`, in query order: per + /// query the rows [`LeafLayout::query_rows`] names — `[row, sym]` for a + /// row pair, `[row]` for one row (S2). [`Self::device_rows`] slices the + /// gather back per query. + #[cfg(feature = "cuda")] + fn device_query_rows(queries: &[usize], lde_len: usize, leaf_layout: LeafLayout) -> Vec { + queries + .iter() + .flat_map(|&c| { + let (row, sym) = leaf_layout.query_rows(c, lde_len); + core::iter::once(row as u32).chain(sym.map(|r| r as u32)) + }) + .collect() + } + + /// Slice out query `qi`'s rows (each `ncols` field elements) from the + /// row-major device gather of [`Self::device_query_rows`]: `(row, sym)` + /// for a row pair (`[row(q0), sym(q0), row(q1), sym(q1), ...]`), `(row, + /// [])` for one row (`[row(q0), row(q1), ...]`). #[cfg(feature = "cuda")] - fn device_row_pair( + fn device_rows( vals: &[FieldElement], qi: usize, ncols: usize, + leaf_layout: LeafLayout, ) -> (Vec>, Vec>) { - let even = vals[(2 * qi) * ncols..(2 * qi + 1) * ncols].to_vec(); - let odd = vals[(2 * qi + 1) * ncols..(2 * qi + 2) * ncols].to_vec(); - (even, odd) + let per = leaf_layout.rows_per_leaf(); + let at = |k: usize| vals[(per * qi + k) * ncols..(per * qi + k + 1) * ncols].to_vec(); + let row = at(0); + let sym = if per == 2 { at(1) } else { Vec::new() }; + (row, sym) } /// Gather every query's row-pair off a device-resident LDE (a small D2H of @@ -2819,12 +4471,13 @@ pub trait IsStarkProver< lde_trace: &LDETraceTable, dev_proofs: Option<&Vec>>, dev_values: Option<&Vec>>, - tree: &BatchedMerkleTree, + tree: &MerkleTree>, qi: usize, challenge: usize, ncols: usize, col_range: std::ops::Range, what: &str, + leaf_layout: LeafLayout, gather: G, ) -> PolynomialOpenings where @@ -2846,7 +4499,7 @@ pub trait IsStarkProver< !tree.is_root_only(), "R4 {what} opening fell back to a root-only host tree (nodes device-resident)" ); - return Self::open_polys_with(domain, tree, challenge, gather); + return Self::open_polys_with(domain, tree, challenge, leaf_layout, gather); }; let proof = proofs[qi].clone(); let Some(dev_vals) = dev_values else { @@ -2856,10 +4509,16 @@ pub trait IsStarkProver< !lde_trace.host_trace_empty(), "R4 {what} opening fell back to the host gather, but it is device-only (empty)" ); - return Self::open_polys_with_proofs(domain, proof, challenge, gather); + return Self::open_polys_with_proofs(domain, proof, challenge, leaf_layout, gather); + }; + let (even, odd) = Self::device_rows(dev_vals, qi, ncols, leaf_layout); + // `odd` is empty for one row (no symmetric row). + let odd = if odd.is_empty() { + odd + } else { + odd[col_range.clone()].to_vec() }; - let (even, odd) = Self::device_row_pair(dev_vals, qi, ncols); - let (even, odd) = (even[col_range.clone()].to_vec(), odd[col_range].to_vec()); + let even = even[col_range].to_vec(); // Cross-check the device gather against the host LDE. Skipped under // device-only (host trace empty): the gather was proven bit-identical // while the host copy was resident, and there is nothing to check @@ -2867,9 +4526,8 @@ pub trait IsStarkProver< // --release, and gather failure modes — stride/offset/layout — are // systematic, so one query catches them); debug checks every query. if (cfg!(debug_assertions) || qi == 0) && !lde_trace.host_trace_empty() { - let domain_size = domain.lde_roots_of_unity_coset.len() as u64; - let r_even = reverse_index(challenge * 2, domain_size); - let r_odd = reverse_index(challenge * 2 + 1, domain_size); + let domain_size = domain.lde_roots_of_unity_coset.len(); + let (r_even, r_odd) = leaf_layout.query_rows(challenge, domain_size); assert_eq!( even, gather(r_even), @@ -2877,19 +4535,21 @@ pub trait IsStarkProver< ); assert_eq!( odd, - gather(r_odd), + r_odd.map(&gather).unwrap_or_default(), "device {what}-row gather mismatch (odd), query {qi}" ); } Self::open_polys_from_values(proof, even, odd) } - /// Open the deep composition polynomial on a list of indexes and their symmetric elements. + /// Open the deep composition polynomial on a list of indexes and their + /// symmetric elements (row pairs) or at the indexes alone (one row, S2). fn open_deep_composition_poly( domain: &Domain, - round_1_result: &Round1, - round_2_result: &Round2, + round_1_result: &Round1, + round_2_result: &Round2, indexes_to_open: &[usize], + leaf_layout: LeafLayout, ) -> DeepPolynomialOpenings where FieldElement: AsBytes, @@ -2897,28 +4557,24 @@ pub trait IsStarkProver< { let mut openings = Vec::with_capacity(indexes_to_open.len()); + let composition_parts = &round_2_result.lde_composition_poly_evaluations; let lde_trace = &round_1_result.lde_trace; let main_commit = &round_1_result.main; let is_preprocessed = main_commit.is_preprocessed(); let num_precomputed_cols = main_commit.num_precomputed_cols; let total_cols = lde_trace.num_main_cols(); - // Row-pair LDE positions for every query, `[even(q0), odd(q0), ...]`. - // Each query opens the leaf at `challenge`, which pairs LDE rows + // The LDE rows of every query's leaf: `[row(q0), sym(q0), ...]` for + // row pairs — the leaf at `challenge` pairs LDE rows // `reverse_index(2·challenge)` (the queried point) and - // `reverse_index(2·challenge+1)` (its symmetric `-x` point). + // `reverse_index(2·challenge+1)` (its symmetric `-x` point) — and + // `[row(q0), row(q1), ...]` for one row (S2), the leaf at `challenge` + // being the row `reverse_index(challenge)` alone. #[cfg(feature = "cuda")] let domain_size = domain.lde_roots_of_unity_coset.len() as u64; #[cfg(feature = "cuda")] - let query_rows: Vec = indexes_to_open - .iter() - .flat_map(|&c| { - [ - reverse_index(c * 2, domain_size) as u32, - reverse_index(c * 2 + 1, domain_size) as u32, - ] - }) - .collect(); + let query_rows: Vec = + Self::device_query_rows(indexes_to_open, domain_size as usize, leaf_layout); // R4 trace proofs from the resident device trees, gathered in one batch // over all query positions instead of walking the host trees (byte @@ -2939,7 +4595,7 @@ pub trait IsStarkProver< let stream = lde_trace .bound_stream() .expect("bound stream for device-resident main-tree opening"); - // Row-pair leaves: one proof per query at position `challenge`. + // One proof per query at leaf `challenge` (either layout). crate::gpu_lde::gather_proofs_dev(tree, indexes_to_open, &stream) .expect("device main-tree gather failed; resident tree has no host fallback") }); @@ -2954,13 +4610,14 @@ pub trait IsStarkProver< let stream = lde_trace .bound_stream() .expect("bound stream for device-resident aux-tree opening"); - // Row-pair leaves: one proof per query at position `challenge`. + // One proof per query at leaf `challenge` (either layout). crate::gpu_lde::gather_proofs_dev(tree, indexes_to_open, &stream) .expect("device aux-tree gather failed; resident tree has no host fallback") }); - // Composition tree: openings open a single position `index` (row pair - // leaf), so gather one proof per query challenge from the device tree. + // Composition tree: openings open a single position `index` (a row + // pair or one-row leaf), so gather one proof per query challenge from + // the device tree. #[cfg(feature = "cuda")] let comp_dev_proofs: Option>> = round_2_result.gpu_composition_tree.as_ref().map(|tree| { @@ -3031,7 +4688,7 @@ pub trait IsStarkProver< let comp_num_parts = lde_trace .gpu_composition_parts() .map(|h| h.m) - .unwrap_or_else(|| round_2_result.lde_composition_poly_evaluations.len()); + .unwrap_or_else(|| composition_parts.len()); #[cfg(feature = "cuda")] let comp_dev_values: Option>> = comp_dev_proofs.as_ref().and_then(|_| { @@ -3076,13 +4733,14 @@ pub trait IsStarkProver< total_cols, num_precomputed_cols..total_cols, "multiplicity", + leaf_layout, |row| { lde_trace.gather_main_row_range(row, num_precomputed_cols, total_cols) }, ) } #[cfg(not(feature = "cuda"))] - Self::open_polys_with(domain, &main_commit.tree, *index, |row| { + Self::open_polys_with(domain, &main_commit.tree, *index, leaf_layout, |row| { lde_trace.gather_main_row_range(row, num_precomputed_cols, total_cols) }) } else { @@ -3099,12 +4757,13 @@ pub trait IsStarkProver< total_cols, 0..total_cols, "main", + leaf_layout, |row| lde_trace.gather_main_row(row), ) } #[cfg(not(feature = "cuda"))] { - Self::open_polys_with(domain, &main_commit.tree, *index, |row| { + Self::open_polys_with(domain, &main_commit.tree, *index, leaf_layout, |row| { lde_trace.gather_main_row(row) }) } @@ -3120,17 +4779,20 @@ pub trait IsStarkProver< { match main_dev_values.as_ref() { Some(vals) => { - let (even, odd) = Self::device_row_pair(vals, qi, total_cols); - let (even, odd) = ( - even[..num_precomputed_cols].to_vec(), - odd[..num_precomputed_cols].to_vec(), - ); + let (even, odd) = Self::device_rows(vals, qi, total_cols, leaf_layout); + let even = even[..num_precomputed_cols].to_vec(); + // Empty for one row (no symmetric row). + let odd = if odd.is_empty() { + odd + } else { + odd[..num_precomputed_cols].to_vec() + }; // Query 0 stays a release canary, same rationale // as `open_trace_polys_device`. if (cfg!(debug_assertions) || qi == 0) && !lde_trace.host_trace_empty() { - let r_even = reverse_index(*index * 2, domain_size); - let r_odd = reverse_index(*index * 2 + 1, domain_size); + let (r_even, r_odd) = + leaf_layout.query_rows(*index, domain_size as usize); assert_eq!( even, lde_trace.gather_main_row_range( @@ -3142,7 +4804,13 @@ pub trait IsStarkProver< ); assert_eq!( odd, - lde_trace.gather_main_row_range(r_odd, 0, num_precomputed_cols), + r_odd + .map(|r| lde_trace.gather_main_row_range( + r, + 0, + num_precomputed_cols + )) + .unwrap_or_default(), "device precomputed-row gather mismatch (odd), query {qi}" ); } @@ -3159,14 +4827,14 @@ pub trait IsStarkProver< "R4 precomputed opening fell back to the host gather, \ but it is device-only (empty)" ); - Self::open_polys_with(domain, tree, *index, |row| { + Self::open_polys_with(domain, tree, *index, leaf_layout, |row| { lde_trace.gather_main_row_range(row, 0, num_precomputed_cols) }) } } } #[cfg(not(feature = "cuda"))] - Self::open_polys_with(domain, tree, *index, |row| { + Self::open_polys_with(domain, tree, *index, leaf_layout, |row| { lde_trace.gather_main_row_range(row, 0, num_precomputed_cols) }) }); @@ -3176,7 +4844,8 @@ pub trait IsStarkProver< { match (&comp_dev_proofs, &comp_dev_values) { (Some(proofs), Some(vals)) => { - let (even, odd) = Self::device_row_pair(vals, qi, comp_num_parts); + let (even, odd) = + Self::device_rows(vals, qi, comp_num_parts, leaf_layout); // Cross-check against the host part evals while // they are still resident (absent under full // residency, where the gather is the only source). @@ -3190,8 +4859,9 @@ pub trait IsStarkProver< { let expected = Self::open_composition_poly_with_proof( proofs[qi].clone(), - &round_2_result.lde_composition_poly_evaluations, + composition_parts, *index, + leaf_layout, ); assert_eq!( even, expected.evaluations, @@ -3219,14 +4889,16 @@ pub trait IsStarkProver< ); Self::open_composition_poly_with_proof( proofs[qi].clone(), - &round_2_result.lde_composition_poly_evaluations, + composition_parts, *index, + leaf_layout, ) } _ => Self::open_composition_poly( &round_2_result.composition_poly_merkle_tree, - &round_2_result.lde_composition_poly_evaluations, + composition_parts, *index, + leaf_layout, ), } } @@ -3234,8 +4906,9 @@ pub trait IsStarkProver< { Self::open_composition_poly( &round_2_result.composition_poly_merkle_tree, - &round_2_result.lde_composition_poly_evaluations, + composition_parts, *index, + leaf_layout, ) } }; @@ -3254,12 +4927,13 @@ pub trait IsStarkProver< lde_trace.num_aux_cols(), 0..lde_trace.num_aux_cols(), "aux", + leaf_layout, |row| lde_trace.gather_aux_row(row), ) } #[cfg(not(feature = "cuda"))] { - Self::open_polys_with(domain, &aux.tree, *index, |row| { + Self::open_polys_with(domain, &aux.tree, *index, leaf_layout, |row| { lde_trace.gather_aux_row(row) }) } @@ -3297,9 +4971,9 @@ pub trait IsStarkProver< /// /// The transcript must be safely initialized before passing it to this method. fn multi_prove( - #[allow(unused_mut)] mut air_trace_pairs: Vec>, + air_trace_pairs: Vec>, transcript: &mut (impl IsStarkTranscript + Clone + Send), - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + residency: ResidencyMode, ) -> Result, ProvingError> where FieldElement: AsBytes, @@ -3307,10 +4981,76 @@ pub trait IsStarkProver< PI: Send + Sync + Clone, Field: Copy + 'static, FieldExtension: Copy + 'static, - ::BaseType: SpillSafe, - ::BaseType: SpillSafe, + { + Self::multi_prove_with(air_trace_pairs, transcript, residency, false) + .map(|(proof, _)| proof) + } + + /// [`Self::multi_prove`] with commitment-time out-of-domain answers + /// ([`crate::commit_ood`]): after the last main root, `z0` is drawn and + /// every table's non-precomputed main columns answer at it, all absorbed + /// before the shared LogUp challenges, and each table's DEEP polynomial + /// carries its answers. Returns the proof and every table's answers, which + /// the verifier takes beside it ([`crate::verifier::IsStarkVerifier::multi_verify_views_bound`]). + #[allow(clippy::type_complexity)] + fn multi_prove_bound( + air_trace_pairs: Vec>, + transcript: &mut (impl IsStarkTranscript + Clone + Send), + residency: ResidencyMode, + ) -> Result< + ( + MultiProof, + Vec>>, + ), + ProvingError, + > + where + FieldElement: AsBytes, + FieldElement: AsBytes, + PI: Send + Sync + Clone, + Field: Copy + 'static, + FieldExtension: Copy + 'static, + { + let (proof, answers) = + Self::multi_prove_with(air_trace_pairs, transcript, residency, true)?; + Ok((proof, answers.unwrap_or_default())) + } + + /// The one implementation of [`Self::multi_prove`] (`bind` false) and + /// [`Self::multi_prove_bound`] (`bind` true; the answers come back). + #[allow(clippy::type_complexity)] + #[doc(hidden)] + fn multi_prove_with( + #[allow(unused_mut)] mut air_trace_pairs: Vec>, + transcript: &mut (impl IsStarkTranscript + Clone + Send), + residency: ResidencyMode, + bind: bool, + ) -> Result< + ( + MultiProof, + Option>>>, + ), + ProvingError, + > + where + FieldElement: AsBytes, + FieldElement: AsBytes, + PI: Send + Sync + Clone, + Field: Copy + 'static, + FieldExtension: Copy + 'static, { info!("Started proof generation..."); + let __ps = crate::prove_split::begin(); + + // `debug-checks` reconstructs every table's Round 1 from retained state + // between the aux and rounds stages, so the recompute mode's dropped + // buffers have no meaning there. Forcing `Retain` keeps the debug build + // checking what it always checked. + #[cfg(feature = "debug-checks")] + let residency = { + let _ = residency; + ResidencyMode::Retain + }; #[cfg(feature = "instruments")] crate::instruments::reset_all(); @@ -3333,6 +5073,7 @@ pub trait IsStarkProver< #[cfg(feature = "instruments")] let __sp = crate::instruments::span("r1_prepass"); + let __ps_prepass = crate::prove_split::mark(); let mut domains = Vec::with_capacity(num_airs); let mut twiddle_caches: Vec>> = Vec::with_capacity(num_airs); @@ -3341,6 +5082,13 @@ pub trait IsStarkProver< domains.push(domain); twiddle_caches.push(twiddles); } + // Each table's trace-tree leaf layout (S2): a verifier-side constant + // from the AIR's format and widths and the trace length — the call + // the verifier makes with the proof's trace length. + let leaf_layouts: Vec = air_trace_pairs + .iter() + .map(|(air, trace, _)| crate::leaf_layout::table_leaf_layout(*air, trace.num_rows())) + .collect(); let k = table_parallelism(num_airs); @@ -3355,6 +5103,9 @@ pub trait IsStarkProver< .unwrap_or(u64::MAX); #[cfg(not(feature = "cuda"))] let vram_budget = u64::MAX; + // A proof beside another on the card admits against its own cap. + let cap = device_cap(); + let vram_budget = cap.map_or(vram_budget, |cap| vram_budget.min(cap)); // NOTE: an earlier revision published prove-wide pinned-staging size // hints here so worker slabs allocated once at final size. Measured on @@ -3366,28 +5117,85 @@ pub trait IsStarkProver< let vram_gate = VramGate::new(vram_budget); - // R1 main commit: only the main LDE and its Merkle scratch are resident, - // so the aux columns add nothing to this phase's working set. - let main_estimates: Vec = air_trace_pairs + // The shapes the AIR and the domain fix, read once: the device-set + // estimates the gate admits against derive from them + // (`crate::device_set`). The walk order does NOT — see + // `table_walk_weight`. + let table_shapes: Vec = air_trace_pairs .iter() .enumerate() - .map(|(idx, (_, trace, _))| { - let lde_size = domains[idx].interpolation_domain_size * domains[idx].blowup_factor; - estimate_table_vram_bytes(trace.num_main_columns, 0, lde_size) + .map(|(idx, (air, trace, _))| { + let domain = &domains[idx]; + let n = domain.interpolation_domain_size; + let (_, aux_cols) = air.trace_layout(); + crate::device_set::TableShape { + n, + blowup: domain.blowup_factor, + main_cols: trace.num_main_columns, + aux_cols, + num_parts: air.composition_poly_degree_bound(n) / n, + // The answers' pole is one more DEEP point. + num_eval_points: air.context().transition_offsets.len() * air.step_size() + + usize::from(bind), + } }) .collect(); - // Spill main traces to mmap before Round 1 LDE. - #[cfg(feature = "disk-spill")] - if storage_mode == StorageMode::Disk { - crate::par::par_try_for_each_mut(&mut air_trace_pairs, |(_, trace, _)| { - trace - .main_table - .spill_to_disk() - .map_err(|e| ProvingError::DiskSpill(format!("early main: {e}"))) - })?; + // R1 main commit: the fused commit's device set — one LDE buffer, the + // trace snapshot, the tree and the scratch — the same model the + // dispatch layer admits the commit against. + let main_estimates: Vec = table_shapes + .iter() + .zip(&leaf_layouts) + .map(|(s, l)| { + crate::device_set::commit_device_set_rpl( + s.n, + s.main_cols, + s.blowup, + true, + l.rows_per_leaf(), + ) + .total() + }) + .collect(); + + // The AIR names, for the driver threads' panic payloads: a device abort + // names its stage and shape, the driver adds which table. + let table_names: Vec = air_trace_pairs + .iter() + .map(|(air, _, _)| air.name().to_string()) + .collect(); + + // The R1 walk: the main commit's weight, aux width zero because the aux + // columns are not resident yet in this phase. Keyed on + // `table_walk_weight`, NOT on `main_estimates` — the estimates above + // are what the gate spends, this is the schedule, and the two are no + // longer the same function. The weight's doc carries the measurement + // that makes this the order rather than any other. + let main_walk_weights: Vec = table_shapes + .iter() + .map(|s| table_walk_weight(s.main_cols, 0, s.n * s.blowup)) + .collect(); + let main_walk_order = heaviest_first(&main_walk_weights); + // Under a cap, every table's device set in both phases is held to it + // before the first commit: a table over it is refused here, by name, + // with nothing on the card yet. + if let Some(cap) = cap { + let rounds: Vec = table_shapes + .iter() + .zip(&leaf_layouts) + .map(|(s, l)| { + crate::device_set::table_device_set_rpl(*s, l.rows_per_leaf()).total() + }) + .collect(); + refuse_over_cap(&table_names, [&main_estimates, &rounds], cap)?; } + eprintln!( + "[prover] table walk R1 (walk weight, largest first): {}", + describe_walk(&main_walk_order, &main_walk_weights, &table_names) + ); + crate::prove_split::add(&crate::prove_split::PREPASS, __ps_prepass); #[cfg(feature = "instruments")] drop(__sp); #[cfg(feature = "instruments")] @@ -3408,8 +5216,8 @@ pub trait IsStarkProver< #[cfg(feature = "instruments")] let __sp = crate::instruments::span("r1_main_commit"); - let mut main_commits: Vec> = Vec::with_capacity(num_airs); - let mut main_ldes: Vec<(Vec>, usize)> = Vec::with_capacity(num_airs); + let mut main_commits: Vec> = Vec::with_capacity(num_airs); + let mut main_ldes: Vec> = Vec::with_capacity(num_airs); // Optional device-side LDE handle per table, populated only when the // R1 fused GPU pipeline produced one. Pairing is by index: this vector // is moved into the per-table `gpu_main_cells` mutex slots below, and @@ -3423,37 +5231,55 @@ pub trait IsStarkProver< // the transcript only needs the roots absorbed in index order, done // sequentially below once every commit completed — the one ordering // Fiat-Shamir requires before sampling the shared challenges. + let __ps_mc = crate::prove_split::mark(); let main_results = run_admitted( - &heaviest_first(&main_estimates), + &main_walk_order, &main_estimates, &vram_gate, k, + |idx| table_names[idx].clone(), |idx| { let (air, trace, _) = &air_trace_pairs[idx]; let domain = &domains[idx]; let twiddles = &twiddle_caches[idx]; - let precomputed = air - .is_preprocessed() - .then(|| (air.precomputed_commitment(), air.num_precomputed_columns())); + let layout = leaf_layouts[idx]; + // The root of THIS layout; a layout the AIR has no root for is + // refused here, before anything is committed. + let precomputed = if air.is_preprocessed() { + let root = air.precomputed_commitment_for(layout).ok_or_else(|| { + ProvingError::PrecomputedCommitmentMissing(format!( + "table {}: no precomputed commitment for the {layout:?} leaf layout", + air.name() + )) + })?; + Some((root, air.num_precomputed_columns())) + } else { + None + }; // Stage-3 device-only gate: when it holds, `commit_main_trace` - // keeps the R1 LDE device-resident and skips the host D2H. + // keeps the R1 LDE device-resident and skips the host D2H. A + // one-row table (S2) is no exception: its device trees and + // openings follow its leaf layout. #[cfg(feature = "cuda")] let device_only = Self::device_only_for(*air, domain); Self::commit_main_trace( + air.name(), *trace, domain, twiddles, precomputed, + layout, #[cfg(feature = "cuda")] device_only, - #[cfg(feature = "disk-spill")] - storage_mode, + residency, ) }, ); + crate::prove_split::add(&crate::prove_split::MAIN_COMMIT, __ps_mc); + let __ps_abs = crate::prove_split::mark(); for result in main_results { let result = result.expect("run_admitted fills every slot"); #[cfg(feature = "cuda")] @@ -3465,7 +5291,15 @@ pub trait IsStarkProver< } transcript.append_bytes(&commit.root); main_commits.push(commit); - main_ldes.push(cached_main); + // The root is in the transcript; that is all Fiat-Shamir asks of + // this phase. Under `RecomputeLde` the buffer it was built from + // dies here and the table's fused task rebuilds it from the trace. + main_ldes.push(match residency { + ResidencyMode::Retain => MainLdeSlot::Retained(cached_main), + ResidencyMode::RecomputeLde => MainLdeSlot::Dropped { + num_cols: cached_main.1, + }, + }); #[cfg(feature = "cuda")] main_gpu_handles.push(gpu_main); } @@ -3479,6 +5313,57 @@ pub trait IsStarkProver< heap_snaps.push(s); } + // ===================================================================== + // Round 1: commitment-time answers (bound proofs only) + // ===================================================================== + // After every root and before the shared challenges: `z0`, then every + // table's answers, absorbed in table order (`crate::commit_ood`). + + let commit_ood: Option<( + FieldElement, + Vec>>, + )> = if bind { + let domain = crate::commit_ood::point_domain(air_trace_pairs.iter().zip(&domains).map( + |((air, _, _), d)| { + ( + d.interpolation_domain_size, + d.blowup_factor, + air.context().proof_options.coset_offset, + ) + }, + )) + .ok_or_else(|| { + ProvingError::WrongParameter( + "commitment-time answers: the tables do not share one coset offset".to_string(), + ) + })?; + let z0 = crate::commit_ood::draw_point::(transcript, domain); + #[allow(unused_mut)] + let mut answers = Vec::with_capacity(num_airs); + for (idx, (air, trace, _)) in air_trace_pairs.iter().enumerate() { + answers.push(Self::commit_ood_answers( + *air, + trace, + &domains[idx], + #[cfg(feature = "cuda")] + main_gpu_handles[idx].as_ref(), + &z0, + )?); + } + #[cfg(any(test, feature = "test-utils"))] + if let Some((table, index)) = crate::commit_ood::faults::current().false_answer + && let Some(answer) = answers.get_mut(table).and_then(|a| a.get_mut(index)) + { + *answer = &*answer + FieldElement::::one(); + } + crate::commit_ood::absorb_answers::(transcript, &answers); + Some((z0, answers)) + } else { + None + }; + // Read once, on this thread: the per-table chains below run on workers. + let commit_ood_deep = crate::commit_ood::deep_term_on(); + // ===================================================================== // Round 1: Sample shared LogUp challenges // ===================================================================== @@ -3490,6 +5375,7 @@ pub trait IsStarkProver< } else { Vec::new() }; + crate::prove_split::add(&crate::prove_split::MAIN_ABSORB, __ps_abs); // ===================================================================== // Aux build + aux commit + Rounds 2-4: fused per table @@ -3503,10 +5389,11 @@ pub trait IsStarkProver< // driver chains all three for its table, so tables never wait on a // phase barrier); only this sequential prep runs here. - // Disk-spill needs the aux columns in the host trace to spill them, so - // disable the GPU-resident aux build (it would keep them device-only). - #[cfg(all(feature = "cuda", feature = "disk-spill"))] - if storage_mode == StorageMode::Disk { + // `RecomputeLde` already forced the main commit onto the host path; + // keeping the aux build there too makes the mode wholly host-side, which + // is what its aux release at the end of each fused task acts on. + #[cfg(feature = "cuda")] + if residency.recomputes_main_lde() { for (_, trace, _) in air_trace_pairs.iter_mut() { trace.set_resident_aux_ok(false); } @@ -3542,13 +5429,13 @@ pub trait IsStarkProver< // so the handle stays inside its own table's task and never needs a // separate handle vector. #[cfg(feature = "cuda")] - type AuxResult = ( - Option>, + type AuxResult = ( + Option>, (Vec>, usize), Option, ); #[cfg(not(feature = "cuda"))] - type AuxResult = (Option>, (Vec>, usize)); + type AuxResult = (Option>, (Vec>, usize)); // R1 aux commit and rounds 2 to 4 share the peak working set: the main // and aux LDEs are co-resident, plus the composition and Merkle // transients (in the scratch factor). The aux width comes from the AIR @@ -3556,13 +5443,24 @@ pub trait IsStarkProver< let peak_estimates: Vec = air_trace_pairs .iter() .enumerate() - .map(|(idx, (air, trace, _))| { - let lde_size = domains[idx].interpolation_domain_size * domains[idx].blowup_factor; - let (_, aux_cols) = air.trace_layout(); - estimate_table_vram_bytes(trace.num_main_columns, aux_cols, lde_size) + .map(|(idx, _)| { + crate::device_set::table_device_set_rpl( + table_shapes[idx], + leaf_layouts[idx].rows_per_leaf(), + ) + .total() }) .collect(); + // The fused phase's own walk, separate from R1's because the aux + // columns are resident here and so carry weight. Keyed on + // `table_walk_weight`, not on `peak_estimates`: the estimates feed the + // gate, the weight fixes the schedule. + let peak_walk_weights: Vec = table_shapes + .iter() + .map(|s| table_walk_weight(s.main_cols, s.aux_cols, s.n * s.blowup)) + .collect(); + // Per-table slots for the fused chain: each driver takes or locks only // its own index, so every mutex is uncontended by construction. let pair_cells: Vec>> = @@ -3570,14 +5468,11 @@ pub trait IsStarkProver< .into_iter() .map(std::sync::Mutex::new) .collect(); - let main_commit_cells: Vec>>> = main_commits + let main_commit_cells: Vec>>> = main_commits .into_iter() .map(|c| std::sync::Mutex::new(Some(c))) .collect(); - #[allow(clippy::type_complexity)] - let main_lde_cells: Vec< - std::sync::Mutex>, usize)>>, - > = main_ldes + let main_lde_cells: Vec>>> = main_ldes .into_iter() .map(|l| std::sync::Mutex::new(Some(l))) .collect(); @@ -3602,7 +5497,7 @@ pub trait IsStarkProver< #[allow(clippy::type_complexity)] let aux_stage = |idx: usize| -> Result< ( - Round1Commitments, + Round1Commitments, Lde, ), ProvingError, @@ -3612,6 +5507,7 @@ pub trait IsStarkProver< let domain = &domains[idx]; let twiddles = &twiddle_caches[idx]; + let __ps_ab = crate::prove_split::mark(); #[cfg(feature = "instruments")] let __sp = crate::instruments::span("r1_aux_build_table"); let bus_public_inputs = if air.has_aux_trace() { @@ -3631,39 +5527,37 @@ pub trait IsStarkProver< handle.trace_rows = 0; } } - #[cfg(feature = "disk-spill")] - if storage_mode == StorageMode::Disk && air.has_aux_trace() { - trace - .spill_aux_to_disk() - .map_err(|e| ProvingError::DiskSpill(format!("aux trace: {e}")))?; - } #[cfg(feature = "instruments")] drop(__sp); + crate::prove_split::add(&crate::prove_split::AUX_BUILD, __ps_ab); + let __ps_ac = crate::prove_split::mark(); #[cfg(feature = "instruments")] let __sp = crate::instruments::span("r1_aux_commit_table"); - let aux_full: AuxResult = - (|| -> Result, ProvingError> { + let aux_full: AuxResult = + (|| -> Result, ProvingError> { if air.has_aux_trace() { let lde_size = domain.interpolation_domain_size * domain.blowup_factor; // Device-only for the aux commit: the main commit's // gate AND a produced main device handle. The aux side // may be MORE conservative than main (never less) — if - // the GPU main commit declined and fell back to CPU, - // skipping the aux D2H here would leave a device-only - // trace with no main handle to serve it. + // the GPU main commit declined below the floor and + // committed on the host, skipping the aux D2H here would + // leave a device-only trace with no main handle to serve + // it. + let layout = leaf_layouts[idx]; #[cfg(feature = "cuda")] - let mut device_only = Self::device_only_for(*air, domain) + let device_only = Self::device_only_for(*air, domain) && gpu_main_cells[idx].lock().unwrap().is_some(); // Resident GPU path: aux columns already on device (from // the resident LogUp aux build) — LDE straight from device // memory, no upload, no host column extraction. When the // resident build fired the host aux trace is empty, so a - // device LDE failure downloads the resident aux trace and - // continues on the host arms below (falling through as-is - // would commit a zero aux trace). + // device LDE failure gets one drain-and-retry and is then + // a clean error (falling through as-is would commit a + // zero aux trace). #[cfg(feature = "cuda")] if trace.aux_resident().is_some() { #[cfg(feature = "instruments")] @@ -3673,12 +5567,14 @@ pub trait IsStarkProver< crate::gpu_lde::try_expand_leaf_and_tree_ext3_row_major_keep_dev::< Field, FieldExtension, - BatchedMerkleTreeBackend, + H::Batched, >( + air.name(), ra, domain.blowup_factor, &twiddles.coset_weights, !device_only, + layout.rows_per_leaf(), ) }; let mut expanded = expand(trace.aux_resident().expect("checked above")); @@ -3710,67 +5606,24 @@ pub trait IsStarkProver< Some(handle), )); } - // The device aux LDE declined at runtime (transient - // VRAM pressure, usually) and there is no host aux - // trace to fall back to. Same class as the R2 - // downgrade: download the resident aux trace — and - // the main LDE if this table was device-only — and - // continue fully host-backed on the arms below. - let mut recovered = crate::gpu_lde::materialize_aux_trace_host(*trace); - // Once the aux download lands, the host aux trace is - // populated: a later failure is the main-LDE - // download's, and the error has to name that step - // instead of claiming an empty aux trace. - let aux_recovered = recovered; - if recovered && device_only { - let mut cell = main_lde_cells[idx].lock().unwrap(); - if let Some((data, _)) = cell.as_mut() - && data.is_empty() - && trace.num_main_columns > 0 - { - recovered = match ( - gpu_main_cells[idx].lock().unwrap().as_ref(), - math_cuda::device::backend(), - ) { - (Some(h), Ok(be)) => { - match crate::gpu_lde::download_main_lde_row_major::( - h, - &be.next_stream(), - ) { - Some(v) => { - *data = v; - true - } - None => false, - } - } - _ => false, - }; - } - } - if !recovered { - return Err(ProvingError::Fft( - if aux_recovered { - "resident aux LDE declined; the aux trace was recovered \ - but the main-LDE download failed" - } else { - "resident aux LDE declined and the aux-trace download \ - recovery failed" - } - .to_string(), - )); - } - eprintln!( - "[gpu] resident-aux downgrade: table={} rows={} \ - (device aux LDE declined; continuing on host)", + // The device aux LDE declined twice — before and + // after a device drain. There is no host aux trace, + // and host RAM is a cache, not a compute path: this + // is the failure to report, not a downgrade. + return Err(ProvingError::DevicePath(format!( + "table {}: resident aux LDE declined after the drain-and-retry \ + (rows={} aux_cols={} blowup={}); {}", air.name(), trace.num_rows(), - ); - device_only = false; + num_cols, + domain.blowup_factor, + crate::gpu_lde::device_path_status(), + ))); } // Fused GPU path (cuda only): row-major ext3 NTT — single - // H2D, no column extraction, no CPU transpose. + // H2D, no column extraction, no CPU transpose. The tree + // follows the table's leaf layout. #[cfg(feature = "cuda")] { let (trace_slice, num_cols) = trace.aux_data_row_major(); @@ -3785,14 +5638,16 @@ pub trait IsStarkProver< crate::gpu_lde::try_expand_leaf_and_tree_ext3_row_major_keep::< Field, FieldExtension, - BatchedMerkleTreeBackend, + H::Batched, >( + air.name(), trace_slice, n, num_cols, domain.blowup_factor, &twiddles.coset_weights, !device_only, + layout.rows_per_leaf(), ) { #[cfg(feature = "instruments")] @@ -3811,40 +5666,26 @@ pub trait IsStarkProver< // CPU path: copy the already-row-major aux trace directly // (one memcpy — no transpose) and expand with the // cache-blocked batched two-half FFT. - let (trace_data, total_cols) = trace.aux_data_row_major(); - #[cfg(feature = "instruments")] let t_sub = Instant::now(); - let mut aux_data: Vec> = - Vec::with_capacity(lde_size * total_cols); - aux_data.extend_from_slice(trace_data); - - #[cfg(feature = "disk-spill")] - if storage_mode == StorageMode::Disk { - trace.aux_table.advise_drop_cache(); - } - - Polynomial::>::coset_lde_full_expand_row_major::( - &mut aux_data, - total_cols, - domain.blowup_factor, - &twiddles.coset_weights, - &twiddles.two_half_inv, - &twiddles.two_half_fwd, - ) - .expect("row-major aux coset LDE expansion"); + let _ = lde_size; + let (aux_data, total_cols) = + Self::expand_aux_lde_row_major(trace, domain, twiddles); + #[allow(unused_mut)] + let mut aux_data = aux_data; #[cfg(feature = "instruments")] let aux_lde_dur = t_sub.elapsed(); #[cfg(feature = "instruments")] let t_sub = Instant::now(); #[allow(unused_mut)] - let (mut tree, root) = - Self::commit_rows_bit_reversed(&aux_data, total_cols) - .ok_or(ProvingError::EmptyCommitment)?; - #[cfg(feature = "disk-spill")] - Self::spill_tree(&mut tree, storage_mode, "aux Merkle tree")?; + let (mut tree, root) = Self::commit_rows_bit_reversed_with( + &aux_data, + total_cols, + layout.rows_per_leaf(), + ) + .ok_or(ProvingError::EmptyCommitment)?; let commit = TableCommit::plain(tree, root); #[cfg(feature = "instruments")] crate::instruments::accum_r1_aux(aux_lde_dur, t_sub.elapsed()); @@ -3868,6 +5709,8 @@ pub trait IsStarkProver< } #[cfg(feature = "instruments")] drop(__sp); + crate::prove_split::add(&crate::prove_split::AUX_COMMIT, __ps_ac); + let __ps_r1a = crate::prove_split::mark(); #[cfg(feature = "cuda")] let (aux_commit, cached_aux, gpu_aux) = aux_full; @@ -3878,11 +5721,27 @@ pub trait IsStarkProver< .unwrap() .take() .expect("main commit consumed once per table"); - let main_lde = main_lde_cells[idx] + let main_lde = match main_lde_cells[idx] .lock() .unwrap() .take() - .expect("main lde consumed once per table"); + .expect("main lde consumed once per table") + { + MainLdeSlot::Retained(lde) => lde, + // The Merkle tree was kept, so this is one forward NTT and no + // re-hashing: the root openings are checked against is still + // the root Round 1 absorbed. The buffer dies with this task. + MainLdeSlot::Dropped { num_cols } => { + #[cfg(feature = "instruments")] + let __sp_recompute = crate::instruments::span("r1_main_lde_recompute_table"); + let recomputed = Self::expand_main_lde_row_major(&**trace, domain, twiddles); + assert_eq!( + recomputed.1, num_cols, + "recomputed main LDE width must match the committed one" + ); + recomputed + } + }; #[cfg(feature = "cuda")] let gpu_main = gpu_main_cells[idx].lock().unwrap().take(); let commitment = Round1Commitments { @@ -3903,16 +5762,17 @@ pub trait IsStarkProver< main: main_lde, aux: cached_aux, }; + crate::prove_split::add(&crate::prove_split::R1_ASSEMBLE, __ps_r1a); Ok((commitment, lde)) }; // Fused chain, stage 2: Round1 from the cached LDE (consumed by value, // no recomputation) → rounds 2-4 against the table's transcript fork. let rounds_stage = |idx: usize, - commitment: Round1Commitments, + commitment: Round1Commitments, lde: Lde| -> Result, ProvingError> { - let pair = pair_cells[idx].lock().unwrap(); + let mut pair = pair_cells[idx].lock().unwrap(); let (air, trace, pub_inputs) = &*pair; let _ = trace; // used by instruments let domain = &domains[idx]; @@ -3922,6 +5782,7 @@ pub trait IsStarkProver< #[cfg(feature = "instruments")] let table_start = Instant::now(); + let __ps_r1b = crate::prove_split::mark(); let mut round_1_result = commitment.build_round1(lde, air.step_size(), domain.blowup_factor); @@ -3929,7 +5790,16 @@ pub trait IsStarkProver< if let Some(ref bpi) = round_1_result.bus_public_inputs { tguard.append_field_element(&bpi.table_contribution); } - + crate::prove_split::add(&crate::prove_split::R1_ASSEMBLE, __ps_r1b); + + let table_ood = + commit_ood + .as_ref() + .filter(|_| commit_ood_deep) + .map(|(point, answers)| crate::commit_ood::CommitOodTable { + point, + answers: &answers[idx], + }); let proof = Self::prove_rounds_2_to_4( *air, *pub_inputs, @@ -3937,6 +5807,7 @@ pub trait IsStarkProver< &mut *tguard, domain, &twiddle_caches[idx], + table_ood, )?; #[cfg(feature = "instruments")] @@ -3949,6 +5820,15 @@ pub trait IsStarkProver< sub_ops, )); } + + // Phase B of the recompute contract: this table's proof exists, so + // its aux columns are dead weight for the rest of the prove. They + // live in the caller's trace and would otherwise survive to the end + // of `multi_prove` — the second-largest per-table retention after + // the main LDE. + if residency.recomputes_main_lde() { + pair.1.release_aux_columns(); + } Ok(proof) }; @@ -3964,24 +5844,43 @@ pub trait IsStarkProver< #[cfg(feature = "instruments")] let __sp = crate::instruments::span("rounds_2to4"); - let peak_order = heaviest_first(&peak_estimates); + let __ps_fused = crate::prove_split::mark(); + let peak_order = heaviest_first(&peak_walk_weights); + eprintln!( + "[prover] table walk rounds 2-4 (walk weight, largest first): {}", + describe_walk(&peak_order, &peak_walk_weights, &table_names) + ); // One fused task per table: while a heavy table works through a // host-bound stretch, the others' GPU stages fill the device. The // shared transcript is untouched past this point (each fork is // per-table), so any order is sound; proofs are drained in index order. #[cfg(not(feature = "debug-checks"))] - let table_results = run_admitted(&peak_order, &peak_estimates, &vram_gate, k, |idx| { - let (commitment, lde) = aux_stage(idx)?; - rounds_stage(idx, commitment, lde) - }); + let table_results = run_admitted( + &peak_order, + &peak_estimates, + &vram_gate, + k, + |idx| table_names[idx].clone(), + |idx| { + let (commitment, lde) = aux_stage(idx)?; + rounds_stage(idx, commitment, lde) + }, + ); // debug-checks needs every table's commitments and traces between the // aux and rounds stages (cross-table bus balance), so it splits the // fused chain into two admitted passes around the check. #[cfg(feature = "debug-checks")] let table_results = { - let aux_outs = run_admitted(&peak_order, &peak_estimates, &vram_gate, k, aux_stage); + let aux_outs = run_admitted( + &peak_order, + &peak_estimates, + &vram_gate, + k, + |idx| table_names[idx].clone(), + aux_stage, + ); let mut commitments = Vec::with_capacity(num_airs); let mut ldes = Vec::with_capacity(num_airs); for out in aux_outs { @@ -3994,7 +5893,7 @@ pub trait IsStarkProver< let staged: Vec< std::sync::Mutex< Option<( - Round1Commitments, + Round1Commitments, Lde, )>, >, @@ -4003,12 +5902,20 @@ pub trait IsStarkProver< .zip(ldes) .map(|p| std::sync::Mutex::new(Some(p))) .collect(); - run_admitted(&peak_order, &peak_estimates, &vram_gate, k, |idx| { - let (c, l) = staged[idx].lock().unwrap().take().unwrap(); - rounds_stage(idx, c, l) - }) + run_admitted( + &peak_order, + &peak_estimates, + &vram_gate, + k, + |idx| table_names[idx].clone(), + |idx| { + let (c, l) = staged[idx].lock().unwrap().take().unwrap(); + rounds_stage(idx, c, l) + }, + ) }; + crate::prove_split::add(&crate::prove_split::FUSED, __ps_fused); let mut proofs = Vec::with_capacity(num_airs); for result in table_results { proofs.push(result.expect("run_admitted fills every slot")?); @@ -4031,7 +5938,53 @@ pub trait IsStarkProver< }); } - Ok(MultiProof { proofs }) + if let Some(line) = crate::prove_split::report( + __ps, + num_airs, + domains.iter().map(|d| d.interpolation_domain_size).sum(), + ) { + println!("{line}"); + } + + Ok(( + MultiProof { proofs }, + commit_ood.map(|(_, answers)| answers), + )) + } + + /// One table's commitment-time answers at `z0`: `t_c(z0)` for every + /// [`crate::commit_ood::answered_columns`] column. On the card from the + /// resident main LDE when the table has one, else on the host from the + /// trace; both evaluate the same polynomial, so the values are equal. + fn commit_ood_answers( + air: &dyn AIR, + trace: &TraceTable, + domain: &Domain, + #[cfg(feature = "cuda")] gpu_main: Option<&math_cuda::lde::GpuLdeBase>, + z0: &FieldElement, + ) -> Result>, ProvingError> + where + Field: 'static, + FieldExtension: 'static, + { + let columns = crate::commit_ood::answered_columns(air); + let (data, width) = trace.main_data_row_major(); + #[cfg(feature = "cuda")] + if let Some(handle) = gpu_main + && let Some(values) = + crate::gpu_lde::try_main_values_at_gpu::(handle, domain, z0) + && values.len() == width + { + return Ok(values[columns].to_vec()); + } + #[cfg(not(feature = "cuda"))] + let _ = domain; + crate::commit_ood::evaluate_trace_columns(data, width, columns, z0).ok_or_else(|| { + ProvingError::WrongParameter(format!( + "table {}: the commitment-time point lies in the trace domain", + air.name() + )) + }) } /// Generate a STARK proof for a single AIR/trace. @@ -4048,17 +6001,10 @@ pub trait IsStarkProver< PI: Send + Sync + Clone, Field: Copy + 'static, FieldExtension: Copy + 'static, - ::BaseType: SpillSafe, - ::BaseType: SpillSafe, { let air_trace_pairs = vec![(air, trace, pub_inputs)]; - Self::multi_prove( - air_trace_pairs, - transcript, - #[cfg(feature = "disk-spill")] - StorageMode::Ram, - ) - .map(|mut multi_proof| multi_proof.proofs.remove(0)) + Self::multi_prove(air_trace_pairs, transcript, ResidencyMode::Retain) + .map(|mut multi_proof| multi_proof.proofs.remove(0)) } // TODO: propagate errors instead of unwrap() in open_deep_composition_poly and FRI operations @@ -4222,10 +6168,10 @@ pub trait IsStarkProver< pub_inputs: &PI, domain: &Domain, twiddles: &LdeTwiddles, - round_1_result: &mut Round1, + round_1_result: &mut Round1, transition_coefficients: &[FieldElement], boundary_coefficients: &[FieldElement], - round_2_result: &Round2, + round_2_result: &Round2, round_3_result: &Round3, z: &FieldElement, ) where @@ -4250,6 +6196,11 @@ pub trait IsStarkProver< round_1_result.bus_public_inputs.as_ref(), trace_length, ); + // num_parts==1: `H` IS the single part (no host decompose); num_parts==2: + // the degree-2 split, at the producer's stride. Mirrors the R2 producer so + // the compare is apples-to-apples. + let number_of_parts = air.composition_poly_degree_bound(trace_length) / trace_length; + let row_stride = composition_eval_stride(domain.blowup_factor, number_of_parts); let host_h = evaluator.evaluate( air, &round_1_result.lde_trace, @@ -4257,14 +6208,12 @@ pub trait IsStarkProver< transition_coefficients, boundary_coefficients, &round_1_result.rap_challenges, + row_stride, ); - // num_parts==1: `H` IS the single part (no host decompose); num_parts==2: - // the degree-2 split. Mirrors the R2 producer so the compare is apples-to-apples. - let number_of_parts = air.composition_poly_degree_bound(trace_length) / trace_length; let host_parts = if number_of_parts == 1 { vec![host_h] } else { - Self::decompose_and_extend_d2(&host_h, domain, twiddles) + Self::decompose_and_extend_d2(&host_h, domain, twiddles, row_stride) }; let device_parts: Option>>> = if round_2_result .lde_composition_poly_evaluations @@ -4336,10 +6285,18 @@ pub trait IsStarkProver< transition_coefficients, boundary_coefficients, &round_1_result.rap_challenges, + row_stride, ) .and_then(|h_dev| { - Self::decompose_comp_h_dev(number_of_parts, &h_dev, domain, twiddles, true) - .map(|(parts, _handle)| parts) + Self::decompose_comp_h_dev( + number_of_parts, + &h_dev, + domain, + twiddles, + true, + row_stride, + ) + .map(|(parts, _handle)| parts) }); let rerun_verdict = match &rerun { None => "device rerun declined".to_string(), @@ -4433,10 +6390,11 @@ pub trait IsStarkProver< fn prove_rounds_2_to_4( air: &dyn AIR, pub_inputs: &PI, - round_1_result: &mut Round1, + round_1_result: &mut Round1, transcript: &mut (impl IsStarkTranscript + Clone), domain: &Domain, twiddles: &LdeTwiddles, + commit_ood: Option>, ) -> Result, ProvingError> where FieldElement: AsBytes, @@ -4496,15 +6454,17 @@ pub trait IsStarkProver< &domain.trace_roots_of_unity, ); + let __ps_ood = crate::prove_split::mark(); #[cfg(feature = "instruments")] let t_r3 = Instant::now(); let round_3_result = Self::round_3_evaluate_polynomials_in_out_of_domain_element( air, domain, - round_1_result, - &mut round_2_result, + &mut round_1_result.lde_trace, + &mut round_2_result.lde_composition_poly_evaluations, &z, ); + crate::prove_split::add(&crate::prove_split::R3_OOD, __ps_ood); #[cfg(feature = "instruments")] let round_3_dur = t_r3.elapsed(); @@ -4544,6 +6504,7 @@ pub trait IsStarkProver< // the current-row block (all columns) and the pruned next-row block // (masked columns only), and absorb only the surviving values — the // verifier absorbs the identical two blocks in the same order. + let __ps_oa = crate::prove_split::mark(); let (ood_block0, ood_block1) = Self::ood_layout(air).split_full(&round_3_result.trace_ood_evaluations); for block in [&ood_block0, &ood_block1] { @@ -4558,6 +6519,7 @@ pub trait IsStarkProver< for element in round_3_result.composition_poly_parts_ood_evaluation.iter() { transcript.append_field_element(element); } + crate::prove_split::add(&crate::prove_split::R3_ABSORB, __ps_oa); // =================================== // ==========| Round 4 |========== @@ -4573,8 +6535,9 @@ pub trait IsStarkProver< &mut round_2_result, &round_3_result, &z, + commit_ood, transcript, - ); + )?; #[cfg(feature = "instruments")] { @@ -4740,3 +6703,206 @@ fn print_bus_balance_report( } } } + +#[cfg(test)] +mod walk_tests { + use super::{heaviest_first, table_walk_weight}; + + /// The weight is `2·lde·(8·main + 24·aux) + 256·lde`. These constants are a + /// schedule, not a size, so this test exists to make a change to them a + /// deliberate act that comes with a wrap measurement. + #[test] + fn the_walk_weight_is_pinned() { + let lde = 1usize << 22; + assert_eq!( + table_walk_weight(25, 3, lde), + (lde as u64) * (2 * (8 * 25 + 24 * 3) + 256) + ); + assert_eq!(table_walk_weight(1, 1, 16), 16 * (2 * (8 + 24) + 256)); + // No floor: the per-LDE-row term scales with height, so two narrow + // tables of different heights never tie. The device-set model's 256 MiB + // scratch floor tied six small tables at the wrap, and that tie is + // where the two walks first diverge. + assert!(table_walk_weight(4, 1, 1 << 16) > table_walk_weight(4, 1, 1 << 15)); + } + + /// The two phases weigh differently: R1 passes `aux_cols == 0` because the + /// aux columns are not resident yet, the fused phase passes the AIR's aux + /// width. A table that is narrow in main and wide in aux therefore moves + /// between the two walks. + #[test] + fn the_two_phases_can_walk_differently() { + let lde = 1usize << 20; + // (main, aux) per table: the second is aux-heavy, the first main-heavy. + let main_walk: Vec = [(60usize, 1usize), (10, 30)] + .iter() + .map(|&(m, _)| table_walk_weight(m, 0, lde)) + .collect(); + let fused_walk: Vec = [(60usize, 1usize), (10, 30)] + .iter() + .map(|&(m, a)| table_walk_weight(m, a, lde)) + .collect(); + assert_eq!(heaviest_first(&main_walk), vec![0, 1]); + assert_eq!(heaviest_first(&fused_walk), vec![1, 0]); + } + + /// Ties keep registry order, so equal-weight tables walk in the order the + /// registry lists them and the walk is reproducible run to run. + #[test] + fn ties_keep_registry_order() { + assert_eq!(heaviest_first(&[5, 9, 5, 9]), vec![1, 3, 0, 2]); + } +} + +#[cfg(test)] +mod precomputed_tree_cache_tests { + use super::*; + use crate::config::COMMITMENT_SIZE; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + /// A backend with no hashing at all: the tests here are about the CACHE's + /// eviction, not about Merkle construction, and a real hasher would only + /// make them slower and their failures harder to read. + #[derive(Debug)] + struct TestBackend; + impl IsMerkleTreeBackend for TestBackend { + type Node = u64; + type Data = u64; + fn hash_data(leaf: &u64) -> u64 { + *leaf + } + fn hash_new_parent(a: &u64, b: &u64) -> u64 { + a.wrapping_add(*b) + } + } + + fn root(n: u8) -> PrecomputedTreeKey { + let mut c = [0u8; COMMITMENT_SIZE]; + c[0] = n; + (c, crate::commitment::ROWS_PER_LEAF) + } + fn tree(n: u64) -> Arc> { + Arc::new(MerkleTree::::build(&[n, n + 1]).expect("two leaves build a tree")) + } + fn erased(n: u64) -> Arc { + tree(n) as Arc + } + fn keys(m: &PrecomputedTreeMap) -> Vec { + let mut k: Vec = m.keys().map(|(c, _)| c[0]).collect(); + k.sort_unstable(); + k + } + + /// ⛔ THE CAP EVICTS THE LEAST RECENTLY USED, NOT THE OLDEST INSERTED. + /// + /// Written against the bug it would otherwise have: evicting by insertion + /// order throws away exactly the shared tables that keep hitting and keeps + /// the program-dependent ones that never will — the opposite of the point. + /// Key 1 is TOUCHED after 2 and 3 land, so insertion order would evict it + /// and recency must not. + #[test] + fn the_cap_evicts_by_recency_of_use_not_by_insertion_order() { + let mut m = PrecomputedTreeMap::new(); + let cap = Some(3); + for n in 1..=3u8 { + precomputed_tree_insert_capped(&mut m, root(n), erased(n as u64), cap); + } + assert_eq!(keys(&m), vec![1, 2, 3]); + + // Touch 1, the way a hit does. + m.get_mut(&root(1)).expect("1 is present").0 = + PRECOMPUTED_TREE_TICK.fetch_add(1, Ordering::Relaxed); + + precomputed_tree_insert_capped(&mut m, root(4), erased(4), cap); + assert_eq!( + keys(&m), + vec![1, 3, 4], + "★ 2 is the least recently USED and must be the one evicted; \ + seeing 1 gone means eviction is by insertion order" + ); + assert_eq!(m.len(), 3, "the cap holds"); + } + + /// The property the cap exists to preserve: a key that keeps being used + /// survives unrelated traffic, as long as the working set fits. + #[test] + fn a_repeatedly_used_key_survives_unrelated_inserts_under_the_cap() { + let mut m = PrecomputedTreeMap::new(); + let cap = Some(4); + precomputed_tree_insert_capped(&mut m, root(99), erased(99), cap); + for n in 1..=12u8 { + precomputed_tree_insert_capped(&mut m, root(n), erased(n as u64), cap); + m.get_mut(&root(99)).expect("99 is still present").0 = + PRECOMPUTED_TREE_TICK.fetch_add(1, Ordering::Relaxed); + } + assert!( + m.contains_key(&root(99)), + "the hot key was evicted despite being used between every insert" + ); + assert_eq!(m.len(), 4, "and the cap still holds"); + } + + /// ⛔ A hit must hand back the SAME allocation, not an equal one: the whole + /// saving is not rebuilding the tree. + #[test] + fn a_hit_returns_the_identical_tree() { + let t = tree(7); + let mut m = PrecomputedTreeMap::new(); + precomputed_tree_insert_capped( + &mut m, + root(7), + Arc::clone(&t) as Arc, + Some(2), + ); + let got = m + .get(&root(7)) + .map(|(_, any)| Arc::clone(any)) + .and_then(|any| any.downcast::>().ok()) + .expect("the entry is present and is this backend's tree"); + assert!( + Arc::ptr_eq(&t, &got), + "a hit returned a different allocation" + ); + } + + /// ⛔ THE CONTROL ARM. Unset, the cap must leave the map exactly as it was + /// before the cap existed — otherwise every A/B on this knob compares two + /// changed things. + #[test] + fn no_cap_leaves_the_map_unbounded() { + let mut m = PrecomputedTreeMap::new(); + for n in 0..=200u8 { + precomputed_tree_insert_capped(&mut m, root(n), erased(n as u64), None); + } + assert_eq!(m.len(), 201, "an unset cap must not evict anything"); + } + + /// The leaf layout is part of the key (S2): one root under two layouts is + /// two entries, never a hit on the other layout's tree. + #[test] + fn the_leaf_layout_is_part_of_the_key() { + let mut m = PrecomputedTreeMap::new(); + let (c, _) = root(5); + precomputed_tree_insert_capped(&mut m, (c, 2), erased(1), None); + precomputed_tree_insert_capped(&mut m, (c, 1), erased(2), None); + assert_eq!(m.len(), 2); + let got = |k: &PrecomputedTreeKey| { + m.get(k) + .and_then(|(_, any)| Arc::clone(any).downcast::>().ok()) + .map(|t| t.root) + }; + assert_ne!(got(&(c, 2)), got(&(c, 1))); + } + + /// ⓘ `0` is read as UNSET, not as "cache nothing" — a zero-size cache would + /// miss on every lookup, which is a typo nobody means to make. + #[test] + fn a_zero_cap_is_read_as_unset() { + // The live knob is a process-wide OnceLock, so this asserts the parse + // rule the knob applies rather than the knob itself. + let parse = |v: &str| v.parse::().ok().filter(|&n| n > 0); + assert_eq!(parse("0"), None); + assert_eq!(parse("64"), Some(64)); + assert_eq!(parse("notanumber"), None); + } +} diff --git a/crypto/stark/src/regen.rs b/crypto/stark/src/regen.rs new file mode 100644 index 000000000..109bc88af --- /dev/null +++ b/crypto/stark/src/regen.rs @@ -0,0 +1,1172 @@ +//! A block's packed tables dropped after their group's commit and built again +//! for their group in phase B (D-REGEN §2, D-WHIR-NODISK §2.1). +//! +//! A block can rebuild some of its committed tables from the run itself (the +//! streamed chunks: a replay of the windows they span). Such a table need not +//! stay on the host between its group's commit and its group's turn in phase +//! B: its packed columns +//! ([`multilinear::narrow::NarrowColumns`]) are dropped and a caller's +//! regenerator deposits them again in phase B. +//! +//! - A [`RegenWindow`] holds the dropped tables' [`RegenSlot`]s and paces the +//! regenerator by rank: a deposit of rank `r` goes once the slots from the +//! frontier (the lowest rank not yet taken or failed) up to `r` hold at most +//! `ahead` bytes, deposited or not; the frontier itself always goes. So the +//! regenerator runs at most a window ahead of the groups that take them, in +//! rank order, and its generators may finish out of order inside it: the +//! window is reserved from the frontier, so later ranks can never fill it +//! while the rank a taker waits on cannot deposit (R-REGEN R1). A pacer may +//! resize the window as it runs ([`RegenWindow::set_ahead`]): a larger one +//! admits the depositors it held, a smaller one holds the next deposits and +//! never one already in. +//! - A deposit is checked against the shape and the digest taken when the +//! table was dropped ([`digest_of`], [`crate::narrow::NarrowMain`]'s digest of +//! the same parts); another table fails its slot, and the prove refuses that table +//! before any of its device work. The digest is not cryptographic: the +//! threat is a bug, not an adversary, and the kept-top check and the verifier +//! stay behind it. +//! - Nothing waits forever. The window is created with its first +//! [`RegenProducer`]; once every producer is dropped (the regenerator ended, +//! panicked, or never started), every slot not deposited fails. Closing the +//! window ([`RegenWindow::close`], the prove's end) wakes every waiter and +//! every depositor. A take never blocks: its taker waited for the slot first. +//! +//! - A table held as the records its rows come from +//! ([`multilinear::records`]) has no packed columns on the host to digest. +//! Its slot ([`RegenWindow::slot_for_records`]) keeps the records' digest +//! and shape instead, and its deposit +//! ([`RegenSlot::deposit_checked`]) is the packed columns the regenerator +//! made from records it rebuilt, checked by those records' digest and the +//! columns' shape. What is checked is the input the columns follow from. +//! +//! #1013's `stark::regen` (814a4b656), over the block's committed form instead +//! of a trace's `NarrowMain`: the window, its pacing and its failure semantics +//! are the same. Nothing is dropped unless a caller opens a window and drops +//! tables into it. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::Instant; + +use multilinear::narrow::NarrowColumns; + +const GIB: f64 = (1u64 << 30) as f64; + +/// The 128-bit digest of packed columns: [`crate::narrow::NarrowMain::digest`] +/// of the same rows, widths and bytes. A fast, non-cryptographic check that +/// columns built again are these. +pub fn digest_of(packed: &NarrowColumns) -> [u64; 2] { + crate::narrow::digest_parts(packed.rows(), packed.widths(), packed.data()) +} + +/// A poisoned lock here is a panic already being reported elsewhere; the +/// state behind it stays consistent (every transition is one assignment). +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Why a dropped table did not come back. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RegenError { + /// The deposited columns are not the ones dropped: another shape or digest. + Mismatch, + /// The regenerator could not build them, or stopped before it did. + Failed(String), + /// The window was closed before they came back (the prove ended or stopped). + Closed(String), +} + +impl std::fmt::Display for RegenError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RegenError::Mismatch => write!(f, "the regenerated columns are not the ones dropped"), + RegenError::Failed(why) => write!(f, "not regenerated: {why}"), + RegenError::Closed(why) => write!(f, "the regeneration window closed: {why}"), + } + } +} + +/// Dropped packed columns' shape and digest, or dropped records' shape and +/// digest: what their deposit must be. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Want { + rows: usize, + /// The packed columns' widths; for records, empty (their deposit's widths + /// are the host fill's, known once it is made). + widths: Vec, + /// The bytes dropped: the packed columns', or the records'. + len: usize, + digest: [u64; 2], + /// `Some(columns)` for a table dropped as its records. + records: Option, +} + +impl Want { + fn of(packed: &NarrowColumns) -> Self { + Self { + rows: packed.rows(), + widths: packed.widths().to_vec(), + len: packed.data().len(), + digest: digest_of(packed), + records: None, + } + } + + fn of_records(records: &multilinear::records::Records) -> Self { + Self { + rows: records.rows(), + widths: Vec::new(), + len: records.bytes(), + digest: records.digest(), + records: Some(records.cols()), + } + } + + /// The shape alone: rows, column widths and bytes for packed columns; + /// rows and columns for records. Always checked, so a deposit can never be + /// widened into a buffer of another size. + fn same_shape(&self, packed: &NarrowColumns) -> bool { + match self.records { + None => { + self.rows == packed.rows() + && self.widths == packed.widths() + && self.len == packed.data().len() + } + Some(cols) => self.rows == packed.rows() && cols == packed.cols(), + } + } +} + +enum SlotState { + Waiting, + Ready(NarrowColumns), + Failed(RegenError), + Taken, +} + +struct Slot { + rank: u64, + /// The bytes it reserves: what was dropped until its deposit, then the + /// deposit's. + len: u64, + state: SlotState, + /// Set once the state leaves `Waiting`: the lock-free half of + /// [`RegenSlot::is_ready`]. + settled: Arc, +} + +#[derive(Default)] +struct Shared { + slots: Vec, + /// The slots not yet taken or failed, by (rank, id), with their bytes: + /// the first is the frontier, and the window is reserved from it. + live: BTreeMap<(u64, usize), u64>, + /// Bytes deposited and not taken, and the most at once. + parked: u64, + high_water: u64, + /// Producers alive; once the last one is dropped, every waiting slot fails. + producers: usize, + /// Why the window was closed, once it is. + closed: Option, + deposits: u64, + mismatches: u64, + taken: u64, + /// Seconds depositors waited for room, and takers for their columns. + deposit_wait_ns: u64, + take_wait_ns: u64, +} + +impl Shared { + /// What a reader of slot `id` sees now: its columns, its failure, or + /// nothing yet (`None`). + fn settled(&self, id: usize) -> Option> { + match &self.slots[id].state { + SlotState::Ready(_) => Some(Ok(())), + SlotState::Failed(e) => Some(Err(e.clone())), + SlotState::Taken => Some(Err(RegenError::Failed( + "its columns were already taken".to_string(), + ))), + SlotState::Waiting => { + if let Some(why) = &self.closed { + Some(Err(RegenError::Closed(why.clone()))) + } else if self.producers == 0 { + Some(Err(RegenError::Failed( + "the regenerator stopped before it".to_string(), + ))) + } else { + None + } + } + } + } + + /// Slot `id` leaves `Waiting` for `state` (a failed slot is consumed: + /// it leaves the window's reservation). + fn settle(&mut self, id: usize, state: SlotState) { + let slot = &mut self.slots[id]; + if matches!(state, SlotState::Failed(_)) { + self.live.remove(&(slot.rank, id)); + } + slot.state = state; + slot.settled.store(true, Ordering::Release); + } + + /// Whether slot `key` may deposit now: it is the frontier, or the slots + /// from the frontier up to it reserve at most `ahead` bytes. + fn admits(&self, key: (u64, usize), ahead: u64) -> bool { + if self.live.keys().next() == Some(&key) { + return true; + } + let mut reserved = 0u64; + for (&k, &len) in &self.live { + if k > key { + break; + } + reserved = reserved.saturating_add(len); + if reserved > ahead { + return false; + } + } + true + } +} + +/// The dropped tables of one prove and the bytes deposited ahead of the +/// groups that take them (see the module docs). +pub struct RegenWindow { + shared: Mutex, + changed: Condvar, + /// The bytes reserved from the frontier, and the most it has been. + ahead: AtomicU64, + most_ahead: AtomicU64, + /// The lock-free halves of a waiting slot's readiness: the window closed, + /// and every producer gone. + closed: AtomicBool, + gone: AtomicBool, + /// Test only: deposits checked on their shape alone, so the checks behind + /// the digest can be tested. + #[cfg(any(test, feature = "test-utils"))] + verify: AtomicBool, +} + +impl RegenWindow { + /// A window that reserves at most `ahead` bytes from its frontier (the + /// frontier, however large, always goes), and its first producer: the + /// caller hands it to the regenerator, so a regenerator that never + /// starts, ends early or panics fails every slot it did not deposit. + pub fn new(ahead: u64) -> (Arc, RegenProducer) { + let window = Arc::new(Self { + shared: Mutex::new(Shared { + producers: 1, + ..Shared::default() + }), + changed: Condvar::new(), + ahead: AtomicU64::new(ahead), + most_ahead: AtomicU64::new(ahead), + closed: AtomicBool::new(false), + gone: AtomicBool::new(false), + #[cfg(any(test, feature = "test-utils"))] + verify: AtomicBool::new(true), + }); + let producer = RegenProducer { + window: Arc::clone(&window), + }; + (window, producer) + } + + /// Another producer, alive as long as it is held. None once every + /// producer is gone (the slots not deposited have failed already). + pub fn producer(self: &Arc) -> Option { + let mut shared = lock(&self.shared); + if shared.producers == 0 { + return None; + } + shared.producers += 1; + Some(RegenProducer { + window: Arc::clone(self), + }) + } + + /// A slot for `packed`, about to be dropped: `rank` orders it among the + /// window's slots (the regenerator deposits in rank order, and phase B + /// takes them in it). + pub fn slot(self: &Arc, packed: &NarrowColumns, rank: u64) -> RegenSlot { + // The digest first: concurrent drops must not queue on the window's + // lock behind it (R-REGEN N1). + let want = Want::of(packed); + let settled = Arc::new(AtomicBool::new(false)); + let mut shared = lock(&self.shared); + let id = shared.slots.len(); + shared.slots.push(Slot { + rank, + len: packed.data().len() as u64, + state: SlotState::Waiting, + settled: Arc::clone(&settled), + }); + shared.live.insert((rank, id), packed.data().len() as u64); + RegenSlot { + window: Arc::clone(self), + id, + rank, + want, + settled, + } + } + + /// A slot for a table held as `records`, about to be dropped: it keeps + /// their digest and shape, and reserves their bytes until its deposit + /// ([`RegenSlot::deposit_checked`]), then the deposit's. `rank` orders it + /// as for [`Self::slot`]. + pub fn slot_for_records( + self: &Arc, + records: &multilinear::records::Records, + rank: u64, + ) -> RegenSlot { + let want = Want::of_records(records); + let len = want.len as u64; + let settled = Arc::new(AtomicBool::new(false)); + let mut shared = lock(&self.shared); + let id = shared.slots.len(); + shared.slots.push(Slot { + rank, + len, + state: SlotState::Waiting, + settled: Arc::clone(&settled), + }); + shared.live.insert((rank, id), len); + RegenSlot { + window: Arc::clone(self), + id, + rank, + want, + settled, + } + } + + /// The bytes reserved from the frontier now. + pub fn ahead(&self) -> u64 { + self.ahead.load(Ordering::Acquire) + } + + /// Reserve `ahead` bytes from the frontier from now on (a pacer's + /// reading): a larger window admits the depositors it held at once; a + /// smaller one holds the next deposits, never one already in. Set under + /// the window's lock, so a depositor that has just been refused cannot + /// miss the wake-up. + pub fn set_ahead(&self, ahead: u64) { + let shared = lock(&self.shared); + self.ahead.store(ahead, Ordering::Release); + self.most_ahead.fetch_max(ahead, Ordering::AcqRel); + drop(shared); + self.changed.notify_all(); + } + + /// Bytes deposited and not yet taken. + pub fn parked(&self) -> u64 { + lock(&self.shared).parked + } + + /// Close: every slot not deposited fails with `why`, every waiter and + /// every depositor returns. The first reason stays. + pub fn close(&self, why: &str) { + let mut shared = lock(&self.shared); + if shared.closed.is_none() { + shared.closed = Some(why.to_string()); + } + self.closed.store(true, Ordering::Release); + drop(shared); + self.changed.notify_all(); + } + + /// Whether it is closed. + pub fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + + /// Test only: check deposits on their shape alone (`false`), so wrong + /// columns of the right shape reach the checks behind the digest. + #[cfg(any(test, feature = "test-utils"))] + pub fn set_verify(&self, on: bool) { + self.verify.store(on, Ordering::SeqCst); + } + + fn verifies(&self) -> bool { + #[cfg(any(test, feature = "test-utils"))] + return self.verify.load(Ordering::SeqCst); + #[cfg(not(any(test, feature = "test-utils")))] + true + } + + /// One line for the prove's log. + pub fn report(&self) -> String { + let shared = lock(&self.shared); + format!( + "{} slots · {} deposited · {} taken · {} mismatches · {} never deposited · window \ + {:.2} GiB, high-water {:.2} GiB · depositors waited {:.2} s · takers waited {:.2} s · \ + window most {:.2} GiB", + shared.slots.len(), + shared.deposits, + shared.taken, + shared.mismatches, + shared + .slots + .iter() + .filter(|s| matches!(s.state, SlotState::Waiting)) + .count(), + self.ahead() as f64 / GIB, + shared.high_water as f64 / GIB, + shared.deposit_wait_ns as f64 / 1e9, + shared.take_wait_ns as f64 / 1e9, + self.most_ahead.load(Ordering::Acquire) as f64 / GIB, + ) + } +} + +/// A regenerator's hold on a window: while one is alive, a slot not +/// deposited waits for it; when the last is dropped, those slots fail. +pub struct RegenProducer { + window: Arc, +} + +impl Drop for RegenProducer { + fn drop(&mut self) { + let mut shared = lock(&self.window.shared); + shared.producers = shared.producers.saturating_sub(1); + if shared.producers == 0 { + self.window.gone.store(true, Ordering::Release); + } + drop(shared); + self.window.changed.notify_all(); + } +} + +/// Dropped packed columns' place in their window: the dropped table holds +/// one, the regenerator a clone. Clones are the same slot. +#[derive(Clone)] +pub struct RegenSlot { + window: Arc, + id: usize, + rank: u64, + want: Want, + settled: Arc, +} + +impl std::fmt::Debug for RegenSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RegenSlot") + .field("id", &self.id) + .field("rank", &self.rank) + .field("rows", &self.want.rows) + .field("cols", &self.cols()) + .field("bytes", &self.want.len) + .finish_non_exhaustive() + } +} + +/// Two slots are equal when they are one slot, or when the columns they stand +/// for have the same shape, length and digest. +impl PartialEq for RegenSlot { + fn eq(&self, other: &Self) -> bool { + (Arc::ptr_eq(&self.window, &other.window) && self.id == other.id) || self.want == other.want + } +} + +impl Eq for RegenSlot {} + +impl RegenSlot { + /// Its place in the order the regenerator deposits and phase B takes. + pub fn rank(&self) -> u64 { + self.rank + } + + /// Its key in the window's order: (rank, then the order slots were made + /// in), the order the window reserves in, so equal ranks cannot cross. + pub fn order_key(&self) -> (u64, usize) { + (self.rank, self.id) + } + + /// Rows of the dropped columns. + pub fn rows(&self) -> usize { + self.want.rows + } + + /// Columns dropped. + pub fn cols(&self) -> usize { + self.want.records.unwrap_or(self.want.widths.len()) + } + + /// Packed bytes dropped. + pub fn len(&self) -> usize { + self.want.len + } + + /// Whether the dropped columns had no bytes. + pub fn is_empty(&self) -> bool { + self.want.len == 0 + } + + /// The digest taken when the columns were dropped: the packed columns', + /// or the records' ([`Self::is_records`]). + pub fn digest(&self) -> [u64; 2] { + self.want.digest + } + + /// Whether the table was dropped as its records + /// ([`RegenWindow::slot_for_records`]): its deposit is + /// [`Self::deposit_checked`]. + pub fn is_records(&self) -> bool { + self.want.records.is_some() + } + + /// Its window. + pub fn window(&self) -> &Arc { + &self.window + } + + /// Put the regenerated columns in the slot: checked against the dropped + /// ones' shape and digest, then held until their group takes them. It + /// waits first until the window admits its rank (see the module docs). + /// Columns that are not the dropped ones fail the slot (their table is + /// refused) and return [`RegenError::Mismatch`]; a closed window returns + /// [`RegenError::Closed`]; a slot already settled refuses a second deposit. + pub fn deposit(&self, packed: NarrowColumns) -> Result<(), RegenError> { + // A table dropped as its records is checked by the records its + // columns were made from. + let matches = !self.is_records() + && if self.window.verifies() { + Want::of(&packed) == self.want + } else { + self.want.same_shape(&packed) + }; + self.settle_deposit(packed, matches) + } + + /// [`Self::deposit`] for a table dropped as its records: `packed` made by + /// the host fill from records whose digest is `records_digest`, checked + /// against the dropped records' digest and against their shape. Anything + /// else fails the slot with [`RegenError::Mismatch`], as does this call on + /// a slot of packed columns. + pub fn deposit_checked( + &self, + packed: NarrowColumns, + records_digest: [u64; 2], + ) -> Result<(), RegenError> { + let matches = self.is_records() + && self.want.same_shape(&packed) + && (!self.window.verifies() || records_digest == self.want.digest); + self.settle_deposit(packed, matches) + } + + /// A deposit whose check gave `matches`: held once the window admits its + /// rank, or the slot failed. + fn settle_deposit(&self, packed: NarrowColumns, matches: bool) -> Result<(), RegenError> { + let window = &self.window; + let t = Instant::now(); + let mut shared = lock(&window.shared); + if !matches!(shared.slots[self.id].state, SlotState::Waiting) { + return Err(RegenError::Failed("deposited twice".to_string())); + } + if !matches { + shared.settle(self.id, SlotState::Failed(RegenError::Mismatch)); + shared.mismatches += 1; + drop(shared); + window.changed.notify_all(); + return Err(RegenError::Mismatch); + } + let len = packed.data().len() as u64; + let key = (self.rank, self.id); + loop { + if let Some(why) = &shared.closed { + let why = why.clone(); + shared.deposit_wait_ns += t.elapsed().as_nanos() as u64; + return Err(RegenError::Closed(why)); + } + if !matches!(shared.slots[self.id].state, SlotState::Waiting) { + return Err(RegenError::Failed("settled while it waited".to_string())); + } + if shared.admits(key, window.ahead()) { + break; + } + shared = window + .changed + .wait(shared) + .unwrap_or_else(|e| e.into_inner()); + } + shared.deposit_wait_ns += t.elapsed().as_nanos() as u64; + // A records slot reserved the records' bytes: from now on, the + // deposit's. + shared.slots[self.id].len = len; + if let Some(reserved) = shared.live.get_mut(&key) { + *reserved = len; + } + shared.settle(self.id, SlotState::Ready(packed)); + shared.parked += len; + shared.high_water = shared.high_water.max(shared.parked); + shared.deposits += 1; + drop(shared); + window.changed.notify_all(); + Ok(()) + } + + /// The regenerator gives this slot up: its table is refused with `why`. + pub fn fail(&self, why: &str) { + let mut shared = lock(&self.window.shared); + if matches!(shared.slots[self.id].state, SlotState::Waiting) { + shared.settle( + self.id, + SlotState::Failed(RegenError::Failed(why.to_string())), + ); + } + drop(shared); + self.window.changed.notify_all(); + } + + /// Whether [`Self::wait`] would return at once: the columns are in, or + /// they will never be. Lock-free. + pub fn is_ready(&self) -> bool { + self.settled.load(Ordering::Acquire) + || self.window.closed.load(Ordering::Acquire) + || self.window.gone.load(Ordering::Acquire) + } + + /// Block until the columns are in, or they will never be. + pub fn wait(&self) { + let t = Instant::now(); + let mut shared = lock(&self.window.shared); + while shared.settled(self.id).is_none() { + shared = self + .window + .changed + .wait(shared) + .unwrap_or_else(|e| e.into_inner()); + } + shared.take_wait_ns += t.elapsed().as_nanos() as u64; + } + + /// The regenerated columns, for their one reader, without blocking: the + /// reader waited for the slot first ([`Self::wait`]). A slot not deposited + /// yet is an error (and settles failed, so a late deposit is refused rather + /// than parked for nobody); the bytes leave the window, which makes room + /// for the next deposit. + pub fn take(&self) -> Result { + let mut shared = lock(&self.window.shared); + let state = std::mem::replace(&mut shared.slots[self.id].state, SlotState::Taken); + match state { + SlotState::Ready(packed) => { + let len = shared.slots[self.id].len; + shared.parked = shared.parked.saturating_sub(len); + shared.live.remove(&(self.rank, self.id)); + shared.taken += 1; + drop(shared); + self.window.changed.notify_all(); + Ok(packed) + } + SlotState::Failed(e) => { + shared.slots[self.id].state = SlotState::Failed(e.clone()); + Err(e) + } + SlotState::Taken => Err(RegenError::Failed( + "its columns were already taken".to_string(), + )), + SlotState::Waiting => { + shared.slots[self.id].state = SlotState::Waiting; + let e = match shared.settled(self.id) { + Some(Err(e)) => e, + _ => RegenError::Failed("not deposited when its group took it".to_string()), + }; + shared.settle(self.id, SlotState::Failed(e.clone())); + drop(shared); + self.window.changed.notify_all(); + Err(e) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc; + use std::time::Duration; + + /// Packed columns of `rows` × 3 whose words depend on `seed`. + fn packed(rows: usize, seed: u64) -> NarrowColumns { + let words: Vec = (0..rows as u64 * 3) + .map(|i| (i ^ seed).wrapping_mul(0x9e37_79b9) % 70_000) + .collect(); + NarrowColumns::pack_row_major(&words, 3).expect("packs") + } + + /// `p` with the low bit of its first packed byte flipped: one wrong word, + /// the same shape. + fn bent(p: &NarrowColumns) -> NarrowColumns { + let (rows, widths, mut data) = p.clone().into_parts(); + data[0] ^= 1; + NarrowColumns::from_parts(rows, widths, data).expect("the same shape") + } + + /// The thread's result within `secs`, or a failure naming the hang. + fn within( + secs: u64, + what: &str, + f: impl FnOnce() -> T + Send + 'static, + ) -> T { + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(secs)) + .unwrap_or_else(|_| panic!("{what}: hung")) + } + + /// ★ The digest is [`crate::narrow::NarrowMain`]'s: packed columns and a + /// `NarrowMain` of the same parts digest alike, and one flipped bit, one + /// row fewer or another width map changes it. + #[test] + fn the_digest_is_narrow_mains() { + let a = packed(100, 1); + let (rows, widths, data) = a.clone().into_parts(); + let main = crate::narrow::NarrowMain::from_parts(rows, widths, data).expect("converts"); + assert_eq!(digest_of(&a), main.digest()); + assert_ne!(digest_of(&a), digest_of(&bent(&a))); + assert_ne!(digest_of(&a), digest_of(&packed(99, 1))); + let (rows, mut widths, data) = a.clone().into_parts(); + widths.swap(0, 2); + if widths != a.widths() { + let other = NarrowColumns::from_parts(rows, widths, data).expect("same bytes"); + assert_ne!(digest_of(&a), digest_of(&other)); + } + } + + /// The columns deposited are the columns taken, bit for bit, and their + /// bytes leave the window. + #[test] + fn deposited_columns_are_taken_bit_for_bit() { + let (window, _producer) = RegenWindow::new(1 << 30); + let a = packed(100, 1); + let slot = window.slot(&a, 0); + assert!(!slot.is_ready()); + slot.deposit(a.clone()).unwrap(); + assert!(slot.is_ready()); + assert_eq!(slot.take().unwrap(), a); + assert!(slot.take().is_err(), "one reader"); + let report = window.report(); + assert!( + report.contains("1 deposited · 1 taken · 0 mismatches"), + "{report}" + ); + } + + /// ★ A deposit that is not the dropped columns — another word, another row + /// count — fails its slot: the deposit and the taker both see `Mismatch`, + /// and a second deposit is refused. + #[test] + fn other_columns_fail_their_slot() { + let (window, _producer) = RegenWindow::new(1 << 30); + let a = packed(100, 1); + for other in [packed(100, 2), packed(99, 1)] { + let slot = window.slot(&a, 0); + assert_eq!(slot.deposit(other), Err(RegenError::Mismatch)); + assert!(slot.is_ready(), "a failed slot is settled"); + assert_eq!(slot.take(), Err(RegenError::Mismatch)); + assert!(matches!( + slot.deposit(a.clone()), + Err(RegenError::Failed(_)) + )); + } + let slot = window.slot(&a, 1); + assert_eq!(slot.deposit(bent(&a)), Err(RegenError::Mismatch)); + } + + /// ★ A regenerator that stops — its producer dropped, as when its thread + /// ends or panics — wakes every taker waiting on a slot it never + /// deposited, which fails; what it deposited is still taken. + #[test] + fn a_dead_regenerator_fails_what_it_did_not_deposit() { + let (window, producer) = RegenWindow::new(1 << 30); + let a = packed(64, 1); + let b = packed(64, 2); + let slot_a = window.slot(&a, 0); + let slot_b = window.slot(&b, 1); + let waiter = { + let slot_b = slot_b.clone(); + std::thread::spawn(move || { + slot_b.wait(); + slot_b.take() + }) + }; + std::thread::sleep(Duration::from_millis(50)); + let regenerator = std::thread::spawn(move || { + let _producer = producer; + slot_a.deposit(a).unwrap(); + panic!("the regenerator dies before its second table"); + }); + assert!(regenerator.join().is_err()); + let got = within(10, "the taker of a slot never deposited", move || { + waiter.join().unwrap() + }); + assert!(matches!(got, Err(RegenError::Failed(_))), "{got:?}"); + assert!(window.report().contains("1 deposited")); + } + + /// ★ The window paces by rank from its frontier: with room for one table, + /// ranks 1 and 2 wait while rank 0 is parked, rank 1 goes once rank 0 is + /// taken, rank 2 once rank 1 is; a frontier larger than the window goes + /// alone. + #[test] + fn a_deposit_waits_for_room_in_the_window() { + let t: Vec = (1..=3).map(|seed| packed(64, seed)).collect(); + let (window, _producer) = RegenWindow::new(t[0].data().len() as u64); + let slots: Vec = (0..3).map(|r| window.slot(&t[r], r as u64)).collect(); + slots[0].deposit(t[0].clone()).unwrap(); + let (tx, rx) = mpsc::channel(); + let mut depositors = Vec::new(); + for r in [2, 1] { + let (slot, columns, tx) = (slots[r].clone(), t[r].clone(), tx.clone()); + depositors.push(std::thread::spawn(move || { + let out = slot.deposit(columns); + tx.send(r).unwrap(); + out + })); + std::thread::sleep(Duration::from_millis(50)); + } + let quiet = Duration::from_millis(200); + assert!(rx.recv_timeout(quiet).is_err(), "deposited past the window"); + assert_eq!(slots[0].take().unwrap(), t[0]); + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)), + Ok(1), + "rank 1, the frontier" + ); + assert!(rx.recv_timeout(quiet).is_err(), "rank 2 past the window"); + assert_eq!(slots[1].take().unwrap(), t[1]); + assert_eq!(rx.recv_timeout(Duration::from_secs(5)), Ok(2)); + for d in depositors { + d.join().unwrap().unwrap(); + } + assert_eq!(slots[2].take().unwrap(), t[2]); + let (small, _p) = RegenWindow::new(1); + let big = packed(1000, 3); + small.slot(&big, 0).deposit(big).unwrap(); + } + + /// A larger window admits the depositor a smaller one held, at once; made + /// smaller again it holds the next deposit, and keeps what is in. + #[test] + fn a_resized_window_admits_and_holds() { + let t: Vec = (1..=3).map(|seed| packed(64, seed)).collect(); + let one = t[0].data().len() as u64; + let (window, _producer) = RegenWindow::new(one); + let slots: Vec = (0..3).map(|r| window.slot(&t[r], r as u64)).collect(); + slots[0].deposit(t[0].clone()).unwrap(); + let (tx, rx) = mpsc::channel(); + let (slot, columns, tx1) = (slots[1].clone(), t[1].clone(), tx.clone()); + let held = std::thread::spawn(move || { + let out = slot.deposit(columns); + tx1.send(1).unwrap(); + out + }); + let quiet = Duration::from_millis(200); + assert!(rx.recv_timeout(quiet).is_err(), "deposited past the window"); + window.set_ahead(3 * one); + assert_eq!(window.ahead(), 3 * one); + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)), + Ok(1), + "admitted once larger" + ); + held.join().unwrap().unwrap(); + assert_eq!(window.parked(), 2 * one); + window.set_ahead(one); + let (slot, columns) = (slots[2].clone(), t[2].clone()); + let next = std::thread::spawn(move || { + let out = slot.deposit(columns); + tx.send(2).unwrap(); + out + }); + assert!( + rx.recv_timeout(quiet).is_err(), + "deposited past the smaller window" + ); + assert_eq!(window.parked(), 2 * one, "what was in stays"); + assert_eq!(slots[0].take().unwrap(), t[0]); + assert_eq!(slots[1].take().unwrap(), t[1]); + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)), + Ok(2), + "the frontier goes" + ); + next.join().unwrap().unwrap(); + assert!( + window + .report() + .ends_with(&format!("window most {:.2} GiB", 3.0 * one as f64 / GIB)) + ); + } + + /// The window stays a bound for a regenerator that deposits in rank order + /// with no taker: it deposits what the window reserves, then waits. + #[test] + fn an_in_order_regenerator_stays_inside_the_window() { + let t: Vec = (0..6).map(|seed| packed(64, seed)).collect(); + let len = t[0].data().len() as u64; + let (window, _producer) = RegenWindow::new(2 * len); + let slots: Vec = (0..6).map(|r| window.slot(&t[r], r as u64)).collect(); + let (tx, rx) = mpsc::channel(); + let regenerator = { + let (slots, t) = (slots.clone(), t.clone()); + std::thread::spawn(move || { + for (slot, columns) in slots.iter().zip(t) { + if slot.deposit(columns).is_err() { + return; + } + tx.send(()).unwrap(); + } + }) + }; + for _ in 0..2 { + rx.recv_timeout(Duration::from_secs(5)) + .expect("inside the window"); + } + assert!( + rx.recv_timeout(Duration::from_millis(200)).is_err(), + "past the window" + ); + window.close("done"); + regenerator.join().unwrap(); + } + + /// ★ Generators that take their jobs in rank order (a FIFO dispatch, as a + /// regenerator hands them out) and finish them out of order never + /// deadlock a taker that takes in rank order: with three generators, the + /// later a rank the sooner its generation ends, inside a window of one + /// table, every rank comes back. + #[test] + fn out_of_order_depositors_never_deadlock_an_in_order_taker() { + let t: Vec = (0..9).map(|seed| packed(64, seed)).collect(); + let (window, producer) = RegenWindow::new(t[0].data().len() as u64); + let slots: Vec = (0..9).map(|r| window.slot(&t[r], r as u64)).collect(); + let next = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut generators = Vec::new(); + for _ in 0..3 { + let token = window.producer().expect("alive"); + let (slots, t, next) = (slots.clone(), t.clone(), Arc::clone(&next)); + generators.push(std::thread::spawn(move || { + let _token = token; + loop { + let r = next.fetch_add(1, Ordering::SeqCst); + if r >= 9 { + return; + } + std::thread::sleep(Duration::from_millis(5 * (9 - r as u64))); + let _ = slots[r].deposit(t[r].clone()); + } + })); + } + drop(producer); + let taken = within(20, "an in-order taker", move || { + (0..9) + .map(|r| { + slots[r].wait(); + slots[r].take() + }) + .collect::>() + }); + for g in generators { + g.join().unwrap(); + } + for (r, got) in taken.into_iter().enumerate() { + assert_eq!(got.as_ref(), Ok(&t[r]), "rank {r}"); + } + } + + /// ★ Closing the window wakes a depositor waiting for room and a taker + /// waiting for its columns: both return `Closed`, and nothing hangs. + #[test] + fn closing_the_window_wakes_every_waiter() { + let a = packed(64, 1); + let b = packed(64, 2); + let c = packed(64, 3); + let (window, _producer) = RegenWindow::new(a.data().len() as u64); + let slot_a = window.slot(&a, 0); + let slot_b = window.slot(&b, 1); + let slot_c = window.slot(&c, 2); + slot_a.deposit(a).unwrap(); + // Rank 2 waits for room (rank 1 is the lowest, never deposited); the + // taker of rank 1 waits for it. + let depositor = std::thread::spawn(move || slot_c.deposit(c)); + let taker = std::thread::spawn(move || { + slot_b.wait(); + slot_b.take() + }); + std::thread::sleep(Duration::from_millis(100)); + window.close("the prove ended"); + let (dep, take) = within(10, "a closed window's waiters", move || { + (depositor.join().unwrap(), taker.join().unwrap()) + }); + assert!(matches!(dep, Err(RegenError::Closed(_))), "{dep:?}"); + assert!(matches!(take, Err(RegenError::Closed(_))), "{take:?}"); + assert!(window.is_closed()); + drop(b); + } + + /// ★ A take never blocks: before its deposit it is an error, the slot + /// settles failed, and the late deposit is refused instead of parked for + /// a reader that is gone. + #[test] + fn a_take_before_the_deposit_is_an_error_not_a_wait() { + let (window, _producer) = RegenWindow::new(1 << 30); + let a = packed(16, 1); + let slot = window.slot(&a, 0); + let got = within(5, "a take before the deposit", { + let slot = slot.clone(); + move || slot.take() + }); + assert!(matches!(got, Err(RegenError::Failed(_))), "{got:?}"); + assert!(slot.is_ready()); + assert!(slot.deposit(a).is_err()); + assert!(window.report().contains("0 deposited")); + } + + /// Readiness without the lock: settled by a deposit, by a failure, by the + /// window closing, or by its last producer leaving. + #[test] + fn readiness_is_read_without_the_lock() { + let a = packed(16, 1); + let (window, producer) = RegenWindow::new(1 << 30); + let (s0, s1, s2) = (window.slot(&a, 0), window.slot(&a, 1), window.slot(&a, 2)); + assert!(!s0.is_ready() && !s1.is_ready() && !s2.is_ready()); + s0.deposit(a.clone()).unwrap(); + s1.fail("no"); + assert!(s0.is_ready() && s1.is_ready() && !s2.is_ready()); + drop(producer); + assert!(s2.is_ready(), "its last producer left"); + assert!( + window.producer().is_none(), + "no producer after the last one left" + ); + let (closed, _p) = RegenWindow::new(1 << 30); + let s = closed.slot(&a, 0); + closed.close("done"); + assert!(s.is_ready() && closed.is_closed()); + } + + /// Test hook: with the digest off wrong columns of the right shape are + /// deposited (the checks behind the digest then have to catch them); a + /// wrong shape is refused all the same. + #[test] + fn the_shape_is_checked_with_the_digest_off() { + let (window, _producer) = RegenWindow::new(1 << 30); + window.set_verify(false); + let a = packed(16, 1); + window.slot(&a, 0).deposit(bent(&a)).unwrap(); + assert_eq!( + window.slot(&a, 1).deposit(packed(15, 1)), + Err(RegenError::Mismatch) + ); + } + + /// A slot given up fails with the regenerator's reason. + #[test] + fn a_slot_given_up_fails() { + let (window, _producer) = RegenWindow::new(1 << 30); + let a = packed(10, 1); + let slot = window.slot(&a, 0); + slot.fail("the walk refused window 3"); + slot.wait(); + assert_eq!( + slot.take(), + Err(RegenError::Failed("the walk refused window 3".to_string())) + ); + } + + /// Records of a table of `rows` × 3: KECCAK_RND-shaped ops whose words + /// depend on `seed` (only their shape and digest matter here). + #[derive(Debug)] + struct Shaped { + rows: usize, + ops: Vec, + } + + impl multilinear::records::TableRecords for Shaped { + fn rows(&self) -> usize { + self.rows + } + + fn cols(&self) -> usize { + 3 + } + + fn fill(&self) -> multilinear::records::RecordsFill<'_> { + multilinear::records::RecordsFill::KeccakRnd { + ops: &self.ops, + skip: 0, + } + } + + fn host_fill(&self) -> Option { + None + } + } + + fn records(rows: usize, seed: u64) -> multilinear::records::Records { + let ops = (0..4) + .map(|k| multilinear::records::KeccakRndOp { + timestamp: seed + k, + input: [seed; 25], + }) + .collect(); + multilinear::records::Records(Arc::new(Shaped { rows, ops })) + } + + /// ★ A table dropped as its records takes back the packed columns made + /// from records with the same digest, of the dropped table's rows and + /// columns, and only through the checked deposit: records with another + /// digest, columns of another shape, a plain deposit, or a checked one on + /// a slot of packed columns all fail their slot. The window reserves the + /// records' bytes until the deposit, then the deposit's. + #[test] + fn a_records_slot_takes_columns_checked_by_their_records() { + let (window, _producer) = RegenWindow::new(1 << 30); + let held = records(100, 7); + let columns = packed(100, 1); + let slot = window.slot_for_records(&held, 0); + assert!(slot.is_records()); + assert_eq!(slot.digest(), held.digest()); + assert_eq!( + (slot.rows(), slot.cols(), slot.len()), + (100, 3, held.bytes()) + ); + slot.deposit_checked(columns.clone(), records(100, 7).digest()) + .expect("the same records' columns"); + assert_eq!(window.parked(), columns.data().len() as u64); + assert_eq!(slot.take().expect("taken"), columns); + assert_eq!(window.parked(), 0); + for (what, deposit) in [ + ("other records", (packed(100, 1), records(100, 8).digest())), + ("another height", (packed(99, 1), held.digest())), + ] { + let slot = window.slot_for_records(&held, 1); + assert_eq!( + slot.deposit_checked(deposit.0, deposit.1), + Err(RegenError::Mismatch), + "{what}" + ); + assert_eq!(slot.take(), Err(RegenError::Mismatch), "{what}"); + } + let slot = window.slot_for_records(&held, 2); + assert_eq!( + slot.deposit(packed(100, 1)), + Err(RegenError::Mismatch), + "a plain deposit" + ); + let plain = window.slot(&columns, 3); + assert!(!plain.is_records()); + assert_eq!( + plain.deposit_checked(columns.clone(), held.digest()), + Err(RegenError::Mismatch), + "a checked deposit on packed columns' slot" + ); + } +} diff --git a/crypto/stark/src/residency_mode.rs b/crypto/stark/src/residency_mode.rs new file mode 100644 index 000000000..52885e703 --- /dev/null +++ b/crypto/stark/src/residency_mode.rs @@ -0,0 +1,36 @@ +/// Whether Round 1 keeps every table's main LDE resident until its fused task +/// runs, or drops it after the commit and recomputes it inside the task. +/// +/// Fiat-Shamir requires the main *roots* to be absorbed before the shared LogUp +/// challenges are sampled; it says nothing about the LDE buffers, so keeping +/// them is a performance choice. `Retain` makes it; `RecomputeLde` trades one +/// extra forward NTT per table for turning an `O(N)` retention into an +/// `O(table_parallelism)` transient. The Merkle tree is kept either way, so a +/// recompute never re-hashes and the root that entered the transcript stays the +/// root openings are checked against. +/// +/// The choice is invisible to the proof: same roots, same transcript order, +/// same proof bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ResidencyMode { + /// Keep every main LDE from its Round-1 commit until its table's fused + /// task consumes it. + #[default] + Retain, + /// Drop each main LDE once its root is absorbed and recompute it from the + /// still-resident trace at the top of the table's fused task. + /// + /// Also releases each table's aux columns from the caller-owned + /// `TraceTable` when that table's proof is complete — a documented part of + /// this mode's contract, since it mutates caller-visible state. Callers + /// that read a trace's aux columns after `multi_prove` returns must use + /// `Retain`. + RecomputeLde, +} + +impl ResidencyMode { + /// True when main LDEs are dropped after Round 1 and recomputed on demand. + pub fn recomputes_main_lde(self) -> bool { + matches!(self, Self::RecomputeLde) + } +} diff --git a/crypto/stark/src/s2_device_parity.rs b/crypto/stark/src/s2_device_parity.rs new file mode 100644 index 000000000..bcd96c08d --- /dev/null +++ b/crypto/stark/src/s2_device_parity.rs @@ -0,0 +1,631 @@ +//! Device-vs-host parity for S2's one-row trees and openings. +//! +//! Compiled for `cuda` builds with tests or `test-utils`; every entry needs a +//! GPU, so the callers are `#[ignore]`d box tests. The stark crate instantiates +//! them under Keccak and Blake3 (`tests::zf_s2_device_tests`), the prover crate +//! under the production RPX pin (`tests::zf_rpx_device_tests`). +//! +//! Each entry builds a tree on the device at `rows_per_leaf` 1 (and, as the +//! control, 2) and pins against the host commit over the SAME evaluations: +//! - the root, and the device tree's leaf count (`lde / rows_per_leaf`); +//! - the authentication path of scattered leaves and both ends, gathered off +//! the resident tree (`gather_proofs_dev`, the production opening path), +//! against the host tree's; +//! - where the entry keeps an LDE handle, the device row gather at the rows a +//! query opens (`LeafLayout::query_rows`), against the host rows; +//! - that the one-row root differs from the row-pair root (a device path that +//! ignored the layout would equal it). +//! +//! The LDE itself is parity-pinned by the existing fused-commit tests, so the +//! host reference consumes the evaluations the device returned: this isolates +//! the leaf layout and the tree. +//! +//! Every entry returns `Err` when the device declines (threshold, budget), so a +//! host fallback is a failure, never a pass. + +use std::format; +use std::string::String; +use std::sync::Arc; +use std::vec; +use std::vec::Vec; + +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; + +use crate::config::{Commitment, StarkHash}; +use crate::fri::vectors::splitmix64; +use crate::leaf_layout::LeafLayout; +use crate::prover::{GenericProver, IsStarkProver}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; +type Ext = FieldElement; +type P = GenericProver; + +const LAYOUTS: [LeafLayout; 2] = [LeafLayout::Row, LeafLayout::RowPair]; + +fn base_values(count: usize, seed: &mut u64) -> Vec { + (0..count).map(|_| Felt::from(splitmix64(seed))).collect() +} + +fn ext_values(count: usize, seed: &mut u64) -> Vec { + (0..count) + .map(|_| { + Ext::new([ + Felt::from(splitmix64(seed)), + Felt::from(splitmix64(seed)), + Felt::from(splitmix64(seed)), + ]) + }) + .collect() +} + +/// Leaves to open: both ends, their neighbours and a spread of random ones. +fn open_positions(num_leaves: usize, seed: &mut u64) -> Vec { + let mut p = vec![0, 1, num_leaves / 2, num_leaves - 2, num_leaves - 1]; + p.extend((0..16).map(|_| (splitmix64(seed) % num_leaves as u64) as usize)); + p +} + +/// The resident device tree against the host tree over the same leaves: the +/// root, the leaf count, and every opened path gathered on device. +fn check_tree( + what: &str, + dev: &math_cuda::lde::GpuMerkleTree, + host: &MerkleTree, + host_root: &Commitment, + num_leaves: usize, + seed: &mut u64, +) -> Result<(), String> +where + B: IsMerkleTreeBackend, +{ + if dev.root != *host_root { + return Err(format!("{what}: device root differs from the host root")); + } + if dev.leaves_len != num_leaves { + return Err(format!( + "{what}: device tree has {} leaves, the layout needs {num_leaves}", + dev.leaves_len + )); + } + let stream = math_cuda::device::backend() + .map_err(|e| format!("{what}: no cuda backend: {e:?}"))? + .next_stream(); + let positions = open_positions(num_leaves, seed); + let proofs = crate::gpu_lde::gather_proofs_dev(dev, &positions, &stream) + .ok_or_else(|| format!("{what}: the device path gather failed"))?; + for (pos, proof) in positions.iter().zip(&proofs) { + let want = host + .get_proof_by_pos(*pos) + .ok_or_else(|| format!("{what}: host tree has no leaf {pos}"))?; + if proof.merkle_path != want.merkle_path { + return Err(format!("{what}: the path of leaf {pos} differs")); + } + } + Ok(()) +} + +/// Leaf count of a tree over `lde` rows under `layout`. +fn leaves_of(lde: usize, layout: LeafLayout) -> usize { + lde / layout.rows_per_leaf() +} + +/// The fused main commit (`try_expand_leaf_and_tree_row_major_keep`, the R1 +/// main arm and the LFM artifact commit) over a random `n × m` base trace at +/// `blowup`, at one row and row pairs: tree parity, plus the device row gather +/// at the one-row query rows (the R4 main opening values). +pub fn main_tree_parity( + n: usize, + m: usize, + blowup: usize, + seed: u64, +) -> Result { + let mut rng = seed; + let data = base_values(n * m, &mut rng); + let weights = base_values(n, &mut rng); + let lde_len = n * blowup; + let mut roots = Vec::new(); + for layout in LAYOUTS { + let what = format!("main {n}x{m} blowup {blowup} {layout:?}"); + let (tree, handle, lde) = + crate::gpu_lde::try_expand_leaf_and_tree_row_major_keep::>( + "s2_device_parity", + "S2 main parity", + &data, + None, + n, + m, + blowup, + &weights, + true, + layout.rows_per_leaf(), + ) + .ok_or_else(|| format!("{what}: the device commit declined"))?; + let (host, host_root) = + P::::commit_rows_bit_reversed_with(&lde, m, layout.rows_per_leaf()) + .ok_or_else(|| format!("{what}: host commit failed"))?; + if tree.root != host_root { + return Err(format!("{what}: returned root-only tree differs")); + } + let dev = handle + .tree + .as_ref() + .ok_or_else(|| format!("{what}: no resident tree"))?; + check_tree( + &what, + dev, + &host, + &host_root, + leaves_of(lde_len, layout), + &mut rng, + )?; + // The R4 opening values: the rows a query opens, off the resident LDE. + let queries = open_positions(leaves_of(lde_len, layout), &mut rng); + let rows: Vec = queries + .iter() + .flat_map(|&q| { + let (row, sym) = layout.query_rows(q, lde_len); + core::iter::once(row as u32).chain(sym.map(|r| r as u32)) + }) + .collect(); + let stream = math_cuda::device::backend() + .map_err(|e| format!("{what}: {e:?}"))? + .next_stream(); + let got = math_cuda::barycentric::gather_rows_base_on_device(&handle, &rows, &stream) + .map_err(|e| format!("{what}: device row gather failed: {e:?}"))?; + for (i, &r) in rows.iter().enumerate() { + let want: Vec = lde[r as usize * m..(r as usize + 1) * m] + .iter() + .map(|x| x.canonical()) + .collect(); + let have: Vec = got[i * m..(i + 1) * m] + .iter() + .map(|&x| Felt::from(x).canonical()) + .collect(); + if have != want { + return Err(format!("{what}: device row gather differs at LDE row {r}")); + } + } + roots.push(host_root); + } + if roots[0] == roots[1] { + return Err(format!( + "main {n}x{m}: the one-row root equals the row-pair root" + )); + } + Ok(format!( + "main {n}x{m} blowup {blowup}: one-row and row-pair trees equal the host, \ + one-row tree {lde_len} leaves" + )) +} + +/// The preprocessed split commit (`try_expand_split_trees_row_major_keep`): +/// the precomputed tree (full host tree) and the multiplicity tree (resident) +/// at one row and row pairs, against the host subset commits. +pub fn split_tree_parity( + n: usize, + m: usize, + split: usize, + blowup: usize, + seed: u64, +) -> Result { + let mut rng = seed; + let data = base_values(n * m, &mut rng); + let weights = base_values(n, &mut rng); + let lde_len = n * blowup; + let mut roots = Vec::new(); + for layout in LAYOUTS { + let rpl = layout.rows_per_leaf(); + let what = format!("split {n}x{m} at {split} blowup {blowup} {layout:?}"); + let (pre, mult, handle, lde) = + crate::gpu_lde::try_expand_split_trees_row_major_keep::>( + "s2_device_parity", + &data, + None, + n, + m, + blowup, + &weights, + split, + true, + true, + rpl, + ) + .ok_or_else(|| format!("{what}: the device commit declined"))?; + let pre = pre.ok_or_else(|| format!("{what}: no precomputed tree"))?; + let (host_pre, host_pre_root) = + P::::commit_rows_bit_reversed_subset_with(&lde, m, 0, split, rpl) + .ok_or_else(|| format!("{what}: host precomputed commit failed"))?; + let (host_mult, host_mult_root) = + P::::commit_rows_bit_reversed_subset_with(&lde, m, split, m, rpl) + .ok_or_else(|| format!("{what}: host multiplicity commit failed"))?; + if pre.root != host_pre_root { + return Err(format!("{what}: precomputed root differs")); + } + let num_leaves = leaves_of(lde_len, layout); + for pos in open_positions(num_leaves, &mut rng) { + if pre.get_proof_by_pos(pos).map(|p| p.merkle_path) + != host_pre.get_proof_by_pos(pos).map(|p| p.merkle_path) + { + return Err(format!("{what}: precomputed path of leaf {pos} differs")); + } + } + if mult.root != host_mult_root { + return Err(format!("{what}: multiplicity root differs")); + } + let dev = handle + .tree + .as_ref() + .ok_or_else(|| format!("{what}: no resident tree"))?; + check_tree( + &what, + dev, + &host_mult, + &host_mult_root, + num_leaves, + &mut rng, + )?; + roots.push(host_pre_root); + } + if roots[0] == roots[1] { + return Err(format!( + "split {n}x{m}: the one-row root equals the row-pair root" + )); + } + Ok(format!( + "split {n}x{m} at {split} blowup {blowup}: both subset trees equal the host at both layouts" + )) +} + +/// The aux commits: the fused ext3 commit from a host trace +/// (`try_expand_leaf_and_tree_ext3_row_major_keep`) and from a resident aux +/// trace (`..._keep_dev`, the LogUp aux path), at one row and row pairs; the +/// two must agree with each other and with the host commit, and the device +/// ext3 row gather must return the rows a query opens. +pub fn aux_tree_parity( + n: usize, + m: usize, + blowup: usize, + seed: u64, +) -> Result { + let mut rng = seed; + let data = ext_values(n * m, &mut rng); + let weights = base_values(n, &mut rng); + let lde_len = n * blowup; + let raw: Vec = data + .iter() + .flat_map(|x| x.value().iter().map(|c| c.canonical()).collect::>()) + .collect(); + let mut roots = Vec::new(); + for layout in LAYOUTS { + let rpl = layout.rows_per_leaf(); + let what = format!("aux {n}x{m} blowup {blowup} {layout:?}"); + let (tree, handle, lde) = crate::gpu_lde::try_expand_leaf_and_tree_ext3_row_major_keep::< + F, + E, + H::Batched, + >( + "s2_device_parity", &data, n, m, blowup, &weights, true, rpl + ) + .ok_or_else(|| format!("{what}: the device commit declined"))?; + let (host, host_root) = P::::commit_rows_bit_reversed_with(&lde, m, rpl) + .ok_or_else(|| format!("{what}: host commit failed"))?; + if tree.root != host_root { + return Err(format!("{what}: returned root-only tree differs")); + } + let dev = handle + .tree + .as_ref() + .ok_or_else(|| format!("{what}: no resident tree"))?; + check_tree( + &what, + dev, + &host, + &host_root, + leaves_of(lde_len, layout), + &mut rng, + )?; + + // The resident arm over the same trace (uploaded as the LogUp build + // would leave it: row-major ext3). + let be = math_cuda::device::backend().map_err(|e| format!("{what}: {e:?}"))?; + let stream = be.next_stream(); + let buf = stream + .clone_htod(&raw) + .map_err(|e| format!("{what}: upload failed: {e:?}"))?; + stream.synchronize().map_err(|e| format!("{what}: {e:?}"))?; + let ra = math_cuda::logup::ResidentAux { + buf: Arc::new(buf), + num_aux_cols: m, + num_rows: n, + table_contribution: [0; 3], + }; + let (rtree, rhandle, _) = + crate::gpu_lde::try_expand_leaf_and_tree_ext3_row_major_keep_dev::>( + "s2_device_parity", + &ra, + blowup, + &weights, + true, + rpl, + ) + .ok_or_else(|| format!("{what}: the resident aux commit declined"))?; + if rtree.root != host_root { + return Err(format!( + "{what}: the resident aux root differs from the host root" + )); + } + let rdev = rhandle + .tree + .as_ref() + .ok_or_else(|| format!("{what}: no resident aux tree"))?; + check_tree( + &format!("{what} (resident)"), + rdev, + &host, + &host_root, + leaves_of(lde_len, layout), + &mut rng, + )?; + + let queries = open_positions(leaves_of(lde_len, layout), &mut rng); + let rows: Vec = queries + .iter() + .flat_map(|&q| { + let (row, sym) = layout.query_rows(q, lde_len); + core::iter::once(row as u32).chain(sym.map(|r| r as u32)) + }) + .collect(); + let got = math_cuda::barycentric::gather_rows_ext3_on_device(&handle, &rows, &stream) + .map_err(|e| format!("{what}: device ext3 row gather failed: {e:?}"))?; + let got = crate::constraint_ir::gpu_interp::ext3_u64_to_field::(&got) + .ok_or_else(|| format!("{what}: gather is not ext3"))?; + for (i, &r) in rows.iter().enumerate() { + if got[i * m..(i + 1) * m] != lde[r as usize * m..(r as usize + 1) * m] { + return Err(format!( + "{what}: device ext3 row gather differs at LDE row {r}" + )); + } + } + roots.push(host_root); + } + if roots[0] == roots[1] { + return Err(format!( + "aux {n}x{m}: the one-row root equals the row-pair root" + )); + } + Ok(format!( + "aux {n}x{m} blowup {blowup}: host-input and resident trees equal the host at both layouts" + )) +} + +/// The composition trees: from host part evaluations +/// (`try_build_comp_poly_tree_gpu`) and from resident part slabs +/// (`try_build_comp_poly_tree_gpu_from_dev`), at one row and row pairs, +/// against `commit_bit_reversed_with` over the parts; and the device gather of +/// the parts at a query's rows. +pub fn composition_tree_parity( + lde_len: usize, + parts: usize, + seed: u64, +) -> Result { + let mut rng = seed; + let evals: Vec> = (0..parts).map(|_| ext_values(lde_len, &mut rng)).collect(); + // The resident layout: part `c` component `k` is the slab `(c·3 + k)`. + let mut slabs = vec![0u64; 3 * parts * lde_len]; + for (c, part) in evals.iter().enumerate() { + for (r, x) in part.iter().enumerate() { + for (k, comp) in x.value().iter().enumerate() { + slabs[(c * 3 + k) * lde_len + r] = comp.canonical(); + } + } + } + let be = math_cuda::device::backend().map_err(|e| format!("composition: {e:?}"))?; + let stream = be.next_stream(); + let buf = stream + .clone_htod(&slabs) + .map_err(|e| format!("composition: upload failed: {e:?}"))?; + stream + .synchronize() + .map_err(|e| format!("composition: {e:?}"))?; + let handle = math_cuda::lde::GpuLdeExt3 { + buf: Arc::new(buf), + m: parts, + lde_size: lde_len, + tree: None, + ready: None, + }; + let mut roots = Vec::new(); + for layout in LAYOUTS { + let rpl = layout.rows_per_leaf(); + let what = format!("composition lde {lde_len} parts {parts} {layout:?}"); + let (host, host_root) = + crate::commitment::commit_bit_reversed_with::>(&evals, rpl) + .ok_or_else(|| format!("{what}: host commit failed"))?; + let (tree, dev) = + crate::gpu_lde::try_build_comp_poly_tree_gpu::>(&evals, rpl) + .ok_or_else(|| format!("{what}: the device tree (host parts) declined"))?; + if tree.root != host_root { + return Err(format!("{what}: returned root-only tree differs")); + } + check_tree( + &what, + &dev, + &host, + &host_root, + leaves_of(lde_len, layout), + &mut rng, + )?; + let (rtree, rdev) = + crate::gpu_lde::try_build_comp_poly_tree_gpu_from_dev::>(&handle, rpl) + .ok_or_else(|| format!("{what}: the device tree (resident parts) declined"))?; + if rtree.root != host_root { + return Err(format!("{what}: the resident-parts root differs")); + } + check_tree( + &format!("{what} (resident parts)"), + &rdev, + &host, + &host_root, + leaves_of(lde_len, layout), + &mut rng, + )?; + // The R4 composition opening values off the resident parts. + let queries = open_positions(leaves_of(lde_len, layout), &mut rng); + let rows: Vec = queries + .iter() + .flat_map(|&q| { + let (row, sym) = layout.query_rows(q, lde_len); + core::iter::once(row as u32).chain(sym.map(|r| r as u32)) + }) + .collect(); + let got = math_cuda::barycentric::gather_rows_ext3_on_device(&handle, &rows, &stream) + .map_err(|e| format!("{what}: device parts gather failed: {e:?}"))?; + let got = crate::constraint_ir::gpu_interp::ext3_u64_to_field::(&got) + .ok_or_else(|| format!("{what}: gather is not ext3"))?; + for (i, &r) in rows.iter().enumerate() { + let want: Vec = evals.iter().map(|p| p[r as usize]).collect(); + if got[i * parts..(i + 1) * parts] != want[..] { + return Err(format!( + "{what}: device parts gather differs at LDE row {r}" + )); + } + } + roots.push(host_root); + } + if roots[0] == roots[1] { + return Err(format!( + "composition lde {lde_len}: the one-row root equals the row-pair root" + )); + } + Ok(format!( + "composition lde {lde_len} parts {parts}: host-parts and resident-parts trees equal the host at both layouts" + )) +} + +/// The (e) leaf-digest KAT (`fri::vectors::one_row_leaf_digests_json`): the +/// same 16-row base (5 columns) and ext3 (2 columns) matrices, hashed by the +/// device row-major leaf kernels at one row and row pairs, against the CPU +/// leaves the checked-in `e_leaf_digests_{hash}.json` was generated from. +pub fn leaf_digest_parity() -> Result { + const ROWS: usize = 16; + let mut st = crate::fri::vectors::KAT_SEED + 100; + let base: Vec> = (0..5) + .map(|_| (0..ROWS).map(|_| Felt::from(splitmix64(&mut st))).collect()) + .collect(); + let mut st = crate::fri::vectors::KAT_SEED + 200; + let ext: Vec> = (0..2) + .map(|_| { + (0..ROWS) + .map(|_| crate::fri::vectors::next_ext(&mut st)) + .collect() + }) + .collect(); + // Row-major u64 views (an ext3 element = three consecutive u64). + let base_rm: Vec = (0..ROWS) + .flat_map(|r| base.iter().map(move |c| c[r].canonical())) + .collect(); + let ext_rm: Vec = (0..ROWS) + .flat_map(|r| { + ext.iter().flat_map(move |c| { + c[r].value() + .iter() + .map(|x| x.canonical()) + .collect::>() + }) + }) + .collect(); + let hash = crate::gpu_lde::device_hash_of::>(); + for layout in LAYOUTS { + let rpl = layout.rows_per_leaf(); + let want_b = crate::commitment::leaves_bit_reversed_grouped::>(&base, rpl); + let want_e = crate::commitment::leaves_bit_reversed_grouped::>(&ext, rpl); + let got_b = math_cuda::lde::row_major_leaves(hash, &base_rm, 5, 0, 5, ROWS, rpl) + .map_err(|e| format!("leaf KAT base {layout:?}: {e:?}"))?; + let got_e = math_cuda::lde::row_major_leaves(hash, &ext_rm, 6, 0, 6, ROWS, rpl) + .map_err(|e| format!("leaf KAT ext3 {layout:?}: {e:?}"))?; + let flat = |v: &[Commitment]| v.iter().flatten().copied().collect::>(); + if got_b != flat(&want_b) { + return Err(format!( + "leaf KAT base {layout:?}: device leaves differ from the CPU" + )); + } + if got_e != flat(&want_e) { + return Err(format!( + "leaf KAT ext3 {layout:?}: device leaves differ from the CPU" + )); + } + } + Ok(String::from( + "the (e) leaf-digest KAT: device leaves equal the CPU at both layouts", + )) +} + +/// Every tree entry at the shapes the box runs: narrow and wide, blowup 2 and +/// 4, the LDE floor (2^14) and a production-sized 2^20 LDE. Prints one +/// `S2DEV` line per case and a summary line; `Err` lists every failure. +pub fn run_tree_parity(name: &str) -> Result> { + let mut results: Vec> = vec![leaf_digest_parity::()]; + for (i, &(n, m, blowup)) in [ + (1usize << 12, 1usize, 4usize), + (1 << 13, 20, 2), + (1 << 12, 134, 4), + (1 << 18, 7, 4), + ] + .iter() + .enumerate() + { + results.push(main_tree_parity::(n, m, blowup, 0x5230_0000 + i as u64)); + } + for (i, &(n, m, split, blowup)) in [(1usize << 12, 5usize, 2usize, 4usize), (1 << 18, 9, 4, 4)] + .iter() + .enumerate() + { + results.push(split_tree_parity::( + n, + m, + split, + blowup, + 0x5231_0000 + i as u64, + )); + } + for (i, &(n, m, blowup)) in [ + (1usize << 12, 1usize, 4usize), + (1 << 13, 13, 2), + (1 << 18, 5, 4), + ] + .iter() + .enumerate() + { + results.push(aux_tree_parity::(n, m, blowup, 0x5232_0000 + i as u64)); + } + for (i, &(lde, parts)) in [(1usize << 14, 1usize), (1 << 14, 2), (1 << 20, 2)] + .iter() + .enumerate() + { + results.push(composition_tree_parity::( + lde, + parts, + 0x5233_0000 + i as u64, + )); + } + let total = results.len(); + let mut failures = Vec::new(); + for r in results { + match r { + Ok(msg) => std::println!("S2DEV {name} {msg}"), + Err(e) => failures.push(e), + } + } + if failures.is_empty() { + std::println!("S2DEV {name}: {total} tree cases equal"); + Ok(total) + } else { + Err(failures) + } +} diff --git a/crypto/stark/src/storage_mode.rs b/crypto/stark/src/storage_mode.rs deleted file mode 100644 index 0ba4b32b5..000000000 --- a/crypto/stark/src/storage_mode.rs +++ /dev/null @@ -1,8 +0,0 @@ -/// Storage backend for intermediate prover state: `Ram` (heap) or `Disk` (mmap). -/// Disk trades wall time for peak RAM. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum StorageMode { - #[default] - Ram, - Disk, -} diff --git a/crypto/stark/src/table.rs b/crypto/stark/src/table.rs index 238c4fcfb..15d72b175 100644 --- a/crypto/stark/src/table.rs +++ b/crypto/stark/src/table.rs @@ -1,198 +1,36 @@ -#[cfg(feature = "disk-spill")] -use crypto::mmap_util::spill_slice_to_mmap; use math::field::{ element::FieldElement, traits::{IsField, IsSubFieldOf}, }; -#[cfg(feature = "disk-spill")] -use math::spill_safe::SpillSafe; #[cfg(feature = "parallel")] use rayon::prelude::*; -/// Mmap-backed storage for a spilled Table. -/// -/// Access goes through pointer arithmetic on the mmap, matching the -/// original `data[row * width + col]` layout. -#[cfg(feature = "disk-spill")] -pub(crate) struct TableMmapBacking { - mmap: memmap2::Mmap, - /// Number of columns per row. - width: usize, - /// Number of rows. - height: usize, - /// Size in bytes of a single element. - elem_size: usize, -} - -#[cfg(feature = "disk-spill")] -impl std::fmt::Debug for TableMmapBacking { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TableMmapBacking") - .field("width", &self.width) - .field("height", &self.height) - .field("elem_size", &self.elem_size) - .finish() - } -} - /// A two-dimensional Table holding field elements, arranged in a row-major order. /// This is the basic underlying data structure used for any two-dimensional component in the /// the STARK protocol implementation, such as the `TraceTable` and the `EvaluationFrame`. /// Since this struct is a representation of a two-dimensional table, all rows should have the same /// length. -#[derive(Default, Debug, serde::Deserialize)] -#[cfg_attr( - not(feature = "disk-spill"), - derive( - Clone, - PartialEq, - Eq, - serde::Serialize, - rkyv::Archive, - rkyv::Serialize, - rkyv::Deserialize - ) +#[derive( + Default, + Debug, + serde::Deserialize, + Clone, + PartialEq, + Eq, + serde::Serialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, )] #[serde(bound = "")] pub struct Table { - /// Row-major backing store. Crate-private: external callers must go through - /// the spill-safe accessors (`get`/`get_row`/`set`) rather than indexing the - /// raw buffer, which bypasses the disk-spill mmap backing. + /// Row-major backing store. Crate-private: external callers go through the + /// accessors (`get`/`get_row`/`set`). pub(crate) data: Vec>, pub width: usize, pub height: usize, - #[cfg(feature = "disk-spill")] - #[serde(skip)] - pub(crate) mmap_backing: Option, -} - -#[cfg(feature = "disk-spill")] -impl serde::Serialize for Table -where - FieldElement: serde::Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - use serde::ser::SerializeStruct; - let mut s = serializer.serialize_struct("Table", 3)?; - if self.mmap_backing.is_some() { - s.serialize_field("data", &MmapDataSeq(self))?; - } else { - s.serialize_field("data", &self.data)?; - } - s.serialize_field("width", &self.width)?; - s.serialize_field("height", &self.height)?; - s.end() - } -} - -#[cfg(feature = "disk-spill")] -struct MmapDataSeq<'a, F: IsField>(&'a Table); - -#[cfg(feature = "disk-spill")] -impl serde::Serialize for MmapDataSeq<'_, F> -where - FieldElement: serde::Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - use serde::ser::SerializeSeq; - let table = self.0; - let mut seq = serializer.serialize_seq(Some(table.width * table.height))?; - for r in 0..table.height { - for elem in table.get_row(r) { - seq.serialize_element(elem)?; - } - } - seq.end() - } -} - -// Manual rkyv impl under disk-spill: the derive can't handle `mmap_backing`, -// and serialization must read through `row_major_data()` so a spilled table -// archives its mmap contents (deserializing always yields an unspilled table). -// The archived layout matches what the derive generates without disk-spill, so -// both configurations produce byte-identical archives. -#[cfg(feature = "disk-spill")] -mod archived_table { - use super::{FieldElement, IsField, Table}; - use math::field::element::ArchivedFieldElement; - use rkyv::rancor::Fallible; - use rkyv::ser::{Allocator, Writer}; - use rkyv::vec::{ArchivedVec, VecResolver}; - use rkyv::{Archive, Deserialize, Place, Portable, Serialize}; - - #[derive(Portable, rkyv::bytecheck::CheckBytes)] - #[bytecheck(crate = rkyv::bytecheck)] - #[repr(C)] - pub struct ArchivedTable - where - F::BaseType: Archive, - { - pub data: ArchivedVec>, - pub width: rkyv::primitive::ArchivedUsize, - pub height: rkyv::primitive::ArchivedUsize, - } - - pub struct TableResolver { - data: VecResolver, - } - - impl Archive for Table - where - F::BaseType: Archive, - { - type Archived = ArchivedTable; - type Resolver = TableResolver; - - fn resolve(&self, resolver: Self::Resolver, out: Place) { - rkyv::munge::munge!(let ArchivedTable { data, width, height } = out); - ArchivedVec::resolve_from_len(self.width * self.height, resolver.data, data); - self.width.resolve((), width); - self.height.resolve((), height); - } - } - - impl Serialize for Table - where - F::BaseType: Archive, - FieldElement: Serialize, - S: Fallible + Allocator + Writer + ?Sized, - { - fn serialize(&self, serializer: &mut S) -> Result { - Ok(TableResolver { - data: ArchivedVec::serialize_from_slice(self.row_major_data(), serializer)?, - }) - } - } - - impl Deserialize, D> for ArchivedTable - where - F::BaseType: Archive, - ArchivedFieldElement: Deserialize, D>, - D: Fallible + ?Sized, - { - fn deserialize(&self, deserializer: &mut D) -> Result, D::Error> { - // Element-by-element rather than `self.data.deserialize(...)`: - // `ArchivedVec`'s blanket `Deserialize` impl needs a - // `DeserializeUnsized` bound this crate doesn't otherwise use, - // while the per-element bound below is already satisfied. - let data = self - .data - .iter() - .map(|elem| elem.deserialize(deserializer)) - .collect::, _>>()?; - Ok(Table { - data, - width: self.width.to_native() as usize, - height: self.height.to_native() as usize, - mmap_backing: None, - }) - } - } } -#[cfg(feature = "disk-spill")] -pub use archived_table::ArchivedTable; - /// Read API over an rkyv-archived [`Table`], used by the verifier to consume /// the out-of-domain evaluations straight from the proof buffer. On /// little-endian targets the element data is viewed in place with no copy. @@ -237,54 +75,6 @@ where } } -/// Cloning a spilled table copies its mmap bytes into a fresh heap `Vec` -/// and returns an unspilled clone. -#[cfg(feature = "disk-spill")] -impl Clone for Table { - fn clone(&self) -> Self { - if self.mmap_backing.is_some() { - let mut data = Vec::with_capacity(self.width * self.height); - for row in 0..self.height { - for col in 0..self.width { - data.push(self.get(row, col).clone()); - } - } - return Self { - data, - width: self.width, - height: self.height, - mmap_backing: None, - }; - } - Self { - data: self.data.clone(), - width: self.width, - height: self.height, - mmap_backing: None, - } - } -} - -#[cfg(feature = "disk-spill")] -impl PartialEq for Table { - fn eq(&self, other: &Self) -> bool { - if self.width != other.width || self.height != other.height { - return false; - } - for row in 0..self.height { - for col in 0..self.width { - if self.get(row, col) != other.get(row, col) { - return false; - } - } - } - true - } -} - -#[cfg(feature = "disk-spill")] -impl Eq for Table {} - impl Table { /// Crates a new Table instance from a one-dimensional array in row major order /// and the intended width of the table. @@ -295,8 +85,6 @@ impl Table { data: Vec::new(), width, height: 0, - #[cfg(feature = "disk-spill")] - mmap_backing: None, }; } @@ -308,8 +96,6 @@ impl Table { data, width, height, - #[cfg(feature = "disk-spill")] - mmap_backing: None, } } @@ -337,52 +123,18 @@ impl Table { /// Given a row index, returns a reference to that row as a slice of field elements. pub fn get_row(&self, row_idx: usize) -> &[FieldElement] { - #[cfg(feature = "disk-spill")] - if let Some(ref backing) = self.mmap_backing { - // Ensures the unsafe block's read stays within the mmap. - assert!( - row_idx < backing.height, - "Table::get_row out of bounds: row={row_idx}, height={}", - backing.height - ); - let offset = row_idx * backing.width * backing.elem_size; - // SAFETY: spill_to_disk writes the table in row-major layout, so - // width elements at this offset are contiguous. FieldElement - // is #[repr(transparent)] over F::BaseType, and spill_to_disk - // requires F::BaseType: SpillSafe (no padding, all bit patterns - // valid). - return unsafe { - std::slice::from_raw_parts( - backing.mmap.as_ptr().add(offset) as *const FieldElement, - backing.width, - ) - }; - } let row_offset = row_idx * self.width; &self.data[row_offset..row_offset + self.width] } - /// Full row-major data as a contiguous slice, reading the mmap when spilled. + /// Full row-major data as a contiguous slice. pub fn row_major_data(&self) -> &[FieldElement] { - #[cfg(feature = "disk-spill")] - if let Some(ref backing) = self.mmap_backing { - // SAFETY: same contract as get_row — spill_to_disk writes row-major and - // FieldElement is #[repr(transparent)] over F::BaseType: SpillSafe. - return unsafe { - std::slice::from_raw_parts( - backing.mmap.as_ptr() as *const FieldElement, - backing.height * backing.width, - ) - }; - } &self.data } /// `true` iff the backing data holds exactly `width × height` elements — /// the invariant `get_row` indexing relies on. Owned counterpart to - /// `ArchivedTable::dimensions_consistent`, reading the length through - /// `row_major_data()` so a disk-spilled table (whose `data` Vec is emptied) - /// reports its true mmap-backed length. + /// `ArchivedTable::dimensions_consistent`. #[inline] pub fn dimensions_consistent(&self) -> bool { self.width @@ -393,13 +145,76 @@ impl Table { /// Returns a vector of vectors of field elements representing the table /// columns pub fn columns(&self) -> Vec>> { - (0..self.width) - .map(|col_idx| { - (0..self.height) - .map(|row_idx| self.get(row_idx, col_idx).clone()) - .collect() - }) - .collect() + // One column per worker: the table is row-major, so this is a strided + // read of the whole trace and the widest thing between the executor and + // the commitment. + let column = |col_idx: usize| -> Vec> { + (0..self.height) + .map(|row_idx| self.get(row_idx, col_idx).clone()) + .collect() + }; + #[cfg(feature = "parallel")] + return (0..self.width).into_par_iter().map(column).collect(); + #[cfg(not(feature = "parallel"))] + return (0..self.width).map(column).collect(); + } + + /// [`Self::columns`] by tiles: row blocks in parallel, and within a block + /// a band of columns at a time, so each row's cache lines are read once + /// rather than once per column. The same columns, in the same order. + /// + /// `columns` reads the row-major table at a stride of one row per element + /// on every column's worker, which pulls a whole cache line for each value: + /// on a block's traces that is most of the transposition's time, and its + /// memory traffic is what the columns' upload then competes with. + pub fn columns_blocked(&self) -> Vec>> { + let (width, height) = (self.width, self.height); + if width == 0 || height == 0 || self.data.len() != width * height { + return self.columns(); + } + const ROWS: usize = 1024; + const BAND: usize = 64; + let mut columns: Vec>> = + (0..width).map(|_| Vec::with_capacity(height)).collect(); + // Addresses, not pointers, so the closure is `Sync`; each block writes + // rows `r0..r1` of every column, disjoint from every other block. + let starts: Vec = columns + .iter_mut() + .map(|column| column.as_mut_ptr() as usize) + .collect(); + let data = &self.data; + let block = |b: usize| { + let r0 = b * ROWS; + let r1 = (r0 + ROWS).min(height); + let mut c0 = 0; + while c0 < width { + let c1 = (c0 + BAND).min(width); + for row in r0..r1 { + let values = &data[row * width + c0..row * width + c1]; + for (offset, value) in values.iter().enumerate() { + // SAFETY: `starts[c]` is column `c`'s buffer, `height` + // elements of capacity; `row < height`, and no other + // block writes this row. + unsafe { + (starts[c0 + offset] as *mut FieldElement) + .add(row) + .write(value.clone()); + } + } + } + c0 = c1; + } + }; + let blocks = height.div_ceil(ROWS); + #[cfg(feature = "parallel")] + (0..blocks).into_par_iter().for_each(block); + #[cfg(not(feature = "parallel"))] + (0..blocks).for_each(block); + for column in &mut columns { + // SAFETY: every row of every column was written above. + unsafe { column.set_len(height) }; + } + columns } /// Extract columns as owned vectors, with each allocated at `capacity`. @@ -425,88 +240,14 @@ impl Table { /// Given row and column indexes, returns the stored field element in that position of the table. pub fn get(&self, row: usize, col: usize) -> &FieldElement { - #[cfg(feature = "disk-spill")] - if let Some(ref backing) = self.mmap_backing { - // Ensures the unsafe block's read stays within the mmap. - assert!( - row < backing.height && col < backing.width, - "Table::get out of bounds: row={row}, col={col}, height={}, width={}", - backing.height, - backing.width - ); - // Row-major layout: offset = (row * width + col) * elem_size - let offset = (row * backing.width + col) * backing.elem_size; - // SAFETY: FieldElement is #[repr(transparent)] over F::BaseType. - // The mmap is page-aligned and elements are contiguously packed. - // The data was written from identical types on the same machine, - // and spill_to_disk requires F::BaseType: SpillSafe (no padding, - // all bit patterns valid). - return unsafe { &*(backing.mmap.as_ptr().add(offset) as *const FieldElement) }; - } let idx = row * self.width + col; &self.data[idx] } pub fn set(&mut self, row: usize, col: usize, value: FieldElement) { - #[cfg(feature = "disk-spill")] - assert!( - self.mmap_backing.is_none(), - "Table::set on a spilled table — backing mmap is read-only" - ); let idx = row * self.width + col; self.data[idx] = value; } - - /// Spill the table's row-major data to a temp file and mmap it back. - /// Frees the heap `data` Vec while preserving access through - /// [`Self::get`], [`Self::get_row`], and [`Self::columns`]. - /// - /// No-op if the table is empty or already spilled. - #[cfg(feature = "disk-spill")] - pub fn spill_to_disk(&mut self) -> std::io::Result<()> - where - F: Copy + 'static, - F::BaseType: SpillSafe, - { - if self.data.is_empty() || self.mmap_backing.is_some() { - return Ok(()); - } - - let mmap = spill_slice_to_mmap(&self.data)?; - self.mmap_backing = Some(TableMmapBacking { - mmap, - width: self.width, - height: self.height, - elem_size: size_of::>(), - }); - self.data = Vec::new(); - - Ok(()) - } - - /// Hint the kernel to drop mmap pages from the page cache. - /// Call after reading spilled data into pool buffers so the same - /// data doesn't occupy RAM in both places. - /// - /// Reliable on Linux for clean file-backed mappings; on other Unix - /// (macOS/BSD) the hint may be a no-op. No-op on non-Unix targets. - #[cfg(all(feature = "disk-spill", unix))] - pub fn advise_drop_cache(&self) { - if let Some(ref backing) = self.mmap_backing { - // SAFETY: pointer and length are from a valid mmap. - // MADV_DONTNEED is advisory and cannot cause UB. - unsafe { - libc::madvise( - backing.mmap.as_ptr() as *mut libc::c_void, - backing.mmap.len(), - libc::MADV_DONTNEED, - ); - } - } - } - - #[cfg(all(feature = "disk-spill", not(unix)))] - pub fn advise_drop_cache(&self) {} } /// A view of a contiguous subset of rows of a table. diff --git a/crypto/stark/src/test_utils.rs b/crypto/stark/src/test_utils.rs index f5cd19f80..881a2b46b 100644 --- a/crypto/stark/src/test_utils.rs +++ b/crypto/stark/src/test_utils.rs @@ -7,7 +7,6 @@ use crate::traits::AIR; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; use math::field::element::FieldElement; use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; -use math::spill_safe::SpillSafe; use math::traits::{AsBytes, ByteConversion}; type AirTracePair<'a, Field, FieldExtension, PI> = ( @@ -26,13 +25,10 @@ where PI: Send + Sync + Clone, FieldElement: AsBytes + ByteConversion, FieldElement: AsBytes + ByteConversion, - ::BaseType: SpillSafe, - ::BaseType: SpillSafe, { Prover::::multi_prove( air_trace_pairs, transcript, - #[cfg(feature = "disk-spill")] - crate::storage_mode::StorageMode::Ram, + crate::residency_mode::ResidencyMode::Retain, ) } diff --git a/crypto/stark/src/tests/blake3_stark_roundtrip_tests.rs b/crypto/stark/src/tests/blake3_stark_roundtrip_tests.rs new file mode 100644 index 000000000..9013c51c5 --- /dev/null +++ b/crypto/stark/src/tests/blake3_stark_roundtrip_tests.rs @@ -0,0 +1,322 @@ +//! A full STARK prove → verify under [`Blake3StarkHash`], and the FRI-layer +//! evidence that makes it work. +//! +//! The whole commitment path is named by one configuration: the prover builds +//! FRI layer trees with `H::Pair` and the verifier authenticates those openings +//! with `H::Batched`, which are one hash by [`StarkHash`]'s two-element +//! invariant. A configuration that broke that agreement would reject every +//! honest proof at its first FRI query, so these tests are what says it holds +//! for a hash other than the default. +//! +//! Everything here is `cfg(not(cuda))`, because [`Blake3StarkHash`] is: under +//! `cuda` the tree entries hash on the device with the keccak kernels and only +//! label the result, so there is no second configuration to name. +#![cfg(not(feature = "cuda"))] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; +use crate::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::StarkProof; +use crate::prover::{GenericProver, IsStarkProver}; +use crate::traits::AIR; +use crate::verifier::{GenericVerifier, IsStarkVerifier}; + +type F = GoldilocksField; +type FE = FieldElement; +type PI = SimpleAdditionPublicInputs; + +type Prove = GenericProver; +type Verify = GenericVerifier; + +/// 1024 rows: `trace_bits = 10` against the default `k = 7`, so FRI actually +/// folds and commits layers. A trace that terminates immediately would make +/// every assertion below vacuous, which is why the layer count is asserted. +const TRACE_ROWS: usize = 1024; + +fn air_and_inputs() -> (SimpleAdditionAIR, PI) { + let proof_options = ProofOptions::default_test_options(); + let air = SimpleAdditionAIR::::new(&proof_options); + let pub_inputs = SimpleAdditionPublicInputs { + a: FE::from(1u64), + b: FE::from(2u64), + }; + (air, pub_inputs) +} + +fn prove_with(air: &SimpleAdditionAIR, pub_inputs: &PI) -> StarkProof { + let mut trace = simple_addition_trace::(TRACE_ROWS); + Prove::::prove( + air, + &mut trace, + pub_inputs, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed") +} + +/// ★ The Stage-2 oracle: a real STARK proves and verifies end to end under the +/// BLAKE3 configuration. +/// +/// This is a **same-reference** claim — this build's prover and this build's +/// verifier agree — and that is exactly what it is for. It makes no +/// cross-version claim; `scripts/cross_verify_vm.sh` is what covers the default +/// keccak path across refs. +#[test] +fn a_blake3_stark_proof_verifies() { + let (air, pub_inputs) = air_and_inputs(); + let proof = prove_with::(&air, &pub_inputs); + + // Non-vacuity: the proof must actually contain committed FRI layers, or it + // would verify without ever exercising the trees this stage threads `H` + // through. + assert!( + !proof.fri_layers_merkle_roots.is_empty(), + "the test trace must fold; otherwise this proves nothing about fri/" + ); + + assert!( + Verify::::verify(&proof, &air, &mut DefaultTranscript::::new(&[])), + "an honest BLAKE3-committed proof must verify under the BLAKE3 verifier" + ); +} + +/// HONEST-PATH CONTROL: the keccak configuration still round-trips. +/// +/// The refactor rewrote the code path the default prover runs through; this +/// says it still proves and verifies. Its stronger sibling is the cross-ref +/// gate, which checks the actual proof BYTES did not move. +#[test] +fn the_keccak_stark_proof_still_verifies() { + let (air, pub_inputs) = air_and_inputs(); + let proof = prove_with::(&air, &pub_inputs); + + assert!(!proof.fri_layers_merkle_roots.is_empty()); + assert!( + Verify::::verify(&proof, &air, &mut DefaultTranscript::::new(&[])), + "an honest keccak-committed proof must verify under the keccak verifier" + ); +} + +/// FALSIFICATION: tampering a FRI layer root must break verification. +/// +/// A positive round trip alone cannot distinguish "the verifier checks the FRI +/// layer openings" from "the verifier reached the end without looking". This is +/// aimed at the precise bytes this stage changed the producer of. +#[test] +fn a_tampered_blake3_fri_layer_root_is_rejected() { + let (air, pub_inputs) = air_and_inputs(); + let honest = prove_with::(&air, &pub_inputs); + + for layer in 0..honest.fri_layers_merkle_roots.len() { + let mut tampered = honest.clone(); + tampered.fri_layers_merkle_roots[layer][0] ^= 1; + assert!( + !Verify::::verify( + &tampered, + &air, + &mut DefaultTranscript::::new(&[]) + ), + "a proof with FRI layer root {layer} flipped must be rejected" + ); + } +} + +/// FALSIFICATION: a tampered FRI layer *opening* must break verification. +/// +/// The root tamper above also moves every challenge drawn after it, so it would +/// be caught by a verifier that only replayed Fiat-Shamir. This one leaves the +/// transcript untouched and corrupts the authenticated value instead, so only +/// the Merkle check can catch it — and that check is `H::Batched` against a tree +/// the prover built with `H::Pair`. +#[test] +fn a_tampered_blake3_fri_layer_opening_is_rejected() { + let (air, pub_inputs) = air_and_inputs(); + let honest = prove_with::(&air, &pub_inputs); + + assert!( + !honest.query_list.is_empty() && !honest.query_list[0].layers_evaluations_sym.is_empty(), + "the test proof must carry FRI query openings" + ); + + let mut tampered = honest.clone(); + tampered.query_list[0].layers_evaluations_sym[0] += FE::one(); + assert!( + !Verify::::verify(&tampered, &air, &mut DefaultTranscript::::new(&[])), + "a tampered FRI symmetric evaluation must fail its Merkle authentication" + ); +} + +/// ★ CONTROL — the stark-proof-level analog of +/// `the_blake3_and_keccak_configurations_commit_differently`. +/// +/// Without this, every test above would pass just as well if `Blake3StarkHash` +/// still resolved to the keccak backends, or if `fri/` had kept building keccak +/// layer trees under a BLAKE3 `H`. Both proofs are over the same trace with the +/// same transcript seed, so nothing but the commitment hash can move these +/// bytes — and each verifier must reject the other configuration's proof. +#[test] +fn the_two_configurations_produce_mutually_unverifiable_proofs() { + let (air, pub_inputs) = air_and_inputs(); + let blake3_proof = prove_with::(&air, &pub_inputs); + let keccak_proof = prove_with::(&air, &pub_inputs); + + assert_ne!( + blake3_proof.lde_trace_main_merkle_root, keccak_proof.lde_trace_main_merkle_root, + "the two configurations must commit the same trace to different roots" + ); + assert_ne!( + blake3_proof.fri_layers_merkle_roots, keccak_proof.fri_layers_merkle_roots, + "the two configurations must commit FRI layers to different roots" + ); + + assert!( + !Verify::::verify( + &blake3_proof, + &air, + &mut DefaultTranscript::::new(&[]) + ), + "the keccak verifier must reject a BLAKE3-committed proof" + ); + assert!( + !Verify::::verify( + &keccak_proof, + &air, + &mut DefaultTranscript::::new(&[]) + ), + "the BLAKE3 verifier must reject a keccak-committed proof" + ); +} + +/// ★ The FRI layer tree IS the configuration's tree over that layer's +/// evaluations — checked directly, at both configurations. +/// +/// The round trip above says prover and verifier agree; it does not say *which* +/// hash they agree on, and a `commit_phase_from_evaluations` that ignored `H` +/// and used keccak for both would still round-trip under a keccak verifier. +/// This rebuilds each committed layer's tree from the layer's own evaluations +/// with `H::Pair` and demands the roots match, so the threading is pinned at the +/// producer rather than inferred from the consumer. +#[test] +fn fri_layer_trees_are_built_with_the_configurations_pair_backend() { + use crate::fri::commit_phase_from_evaluations; + use crate::fri::fri_functions::compute_coset_twiddles_inv; + + /// Returns each committed layer's root, and layer 0's folded codeword. + fn check( + offset: &FE, + len: usize, + blowup_log: u32, + k: u32, + ) -> (Vec<[u8; 32]>, Vec) { + let codeword: Vec = (0..len as u64).map(|i| FE::from(i * 7 + 1)).collect(); + let inv_twiddles = compute_coset_twiddles_inv::(offset, len); + let mut transcript = DefaultTranscript::::new(&[]); + let (_coeffs, layers) = commit_phase_from_evaluations::( + codeword, + &mut transcript, + offset, + len, + blowup_log, + k, + &inv_twiddles, + ); + assert!(!layers.is_empty(), "the input must fold"); + + for (i, layer) in layers.iter().enumerate() { + let leaves: Vec<[FE; 2]> = layer + .evaluation + .chunks_exact(2) + .map(|c| [c[0], c[1]]) + .collect(); + let rebuilt = MerkleTree::>::build(&leaves).expect("rebuild layer tree"); + assert_eq!( + rebuilt.root, layer.merkle_tree.root, + "layer {i}'s committed root must be the H::Pair tree over its own evaluations" + ); + } + ( + layers.iter().map(|l| l.merkle_tree.root).collect(), + layers[0].evaluation.clone(), + ) + } + + let offset = FE::from(3u64); + let (len, blowup_log, k) = (1usize << 10, 1u32, 5u32); + + let (keccak_roots, keccak_layer0) = check::(&offset, len, blowup_log, k); + let (blake3_roots, blake3_layer0) = check::(&offset, len, blowup_log, k); + + // ζ₀ is drawn before anything is appended, so both configurations fold the + // same input with the same challenge and layer 0's codeword is identical. + // Checked rather than argued, because it is what makes the root comparison + // below mean "the hash differs" instead of "the input differs". + assert_eq!( + keccak_layer0, blake3_layer0, + "layer 0 must fold identically under both configurations" + ); + assert_ne!( + keccak_roots[0], blake3_roots[0], + "over one identical codeword, the layer root must differ only because \ + the hash does" + ); +} + +/// ★ The whole BLAKE3 configuration at once: BLAKE3 commitments, a BLAKE3 +/// Fiat-Shamir transcript, and BLAKE3 grinding, proving and verifying. +/// +/// The tests above run the BLAKE3 commitment configuration against a *keccak* +/// transcript, which is a legitimate configuration but not the destination. +/// This is the destination minus the guest leg: nothing keccak is left on +/// either side except what the AIR itself does. +/// +/// `default_test_options` sets `grinding_factor: 1`, so the proof carries a +/// nonce and the verifier re-checks it — through `GrindingDigest`, which for +/// this configuration is `Blake3Chain`. A grinding port that had been left +/// hard-wired to keccak would fail here, because prover and verifier would +/// disagree about which work the nonce satisfies. +#[test] +fn the_full_blake3_configuration_proves_and_verifies_with_grinding() { + use crypto::fiat_shamir::default_transcript::Blake3Transcript; + + let (air, pub_inputs) = air_and_inputs(); + assert_eq!( + air.options().grinding_factor, + 1, + "this test is about the grinding path; it must be on" + ); + + let mut trace = simple_addition_trace::(TRACE_ROWS); + let proof = Prove::::prove( + &air, + &mut trace, + &pub_inputs, + &mut Blake3Transcript::::new(&[]), + ) + .expect("proving under the full BLAKE3 configuration must succeed"); + + assert!( + proof.nonce.is_some(), + "grinding is on, so the proof must carry a nonce" + ); + assert!(!proof.fri_layers_merkle_roots.is_empty()); + + assert!( + Verify::::verify(&proof, &air, &mut Blake3Transcript::::new(&[])), + "an honest all-BLAKE3 proof must verify" + ); + + // FALSIFICATION: the transcript is part of the configuration. A keccak + // transcript replaying a BLAKE3-transcripted proof derives different + // challenges and must reject. + assert!( + !Verify::::verify(&proof, &air, &mut DefaultTranscript::::new(&[])), + "verifying with the wrong transcript hash must reject" + ); +} diff --git a/crypto/stark/src/tests/block_commit_ood_tests.rs b/crypto/stark/src/tests/block_commit_ood_tests.rs new file mode 100644 index 000000000..326113fb3 --- /dev/null +++ b/crypto/stark/src/tests/block_commit_ood_tests.rs @@ -0,0 +1,160 @@ +//! A block's commit-time out-of-domain answers (`multilinear_block::commit_ood_point`) +//! on the small block of [`super::phase_b_overlap_tests`]: every carried +//! polynomial answers and the proof verifies, under either argue format and +//! either chain split; an answer altered, missing, extra or moved to another +//! group is refused; a false answer its chain carries is refused by the +//! chain; and a false answer no chain carries is refused only because the +//! verifier batches the answers into the chains. + +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, +}; +use multilinear::{ + Error as MlError, + whir_chain::{ArgueFormat, GroupFold}, +}; + +use super::phase_b_overlap_tests::{ + ARGUES, SIZES, config_for, in_order, prove_block, verify_block, +}; +use crate::multilinear_block::{BlockProved, OodFault, OverlapFaults, PhaseBOverlap}; +use crate::multilinear_table::batched::VerifierChecks; + +type Fp = math::field::goldilocks::GoldilocksField; + +const GROUPS: [GroupFold; 2] = [GroupFold::PerPoly, GroupFold::Indexed]; + +fn proved(argue: ArgueFormat, group: GroupFold, fault: Option) -> BlockProved { + let (proved, _) = prove_block( + &config_for(argue, group), + PhaseBOverlap::Off, + &in_order(), + OverlapFaults::default(), + &|block| { + if let Some(fault) = fault { + assert!(block.fault_commit_ood(fault), "an answer to make false"); + } + }, + ); + proved.unwrap_or_else(|e| panic!("{argue:?} {group:?}: {e:?}")) +} + +fn check( + argue: ArgueFormat, + group: GroupFold, + proved: &BlockProved, + answers: &[FieldElement], +) -> Result<(), MlError> { + verify_block( + &config_for(argue, group), + proved, + answers, + VerifierChecks::ALL, + ) +} + +/// ★ Every carried polynomial answers — one per root, in root order — and the +/// block verifies with them, under each argue format and each chain split. +/// The proof carries polynomials enough that `Indexed` cuts a run of two or +/// more somewhere, so a grouped chain carries answers. +#[test] +fn every_carried_polynomial_answers_and_the_block_verifies() { + for argue in ARGUES { + for group in GROUPS { + let proved = proved(argue, group, None); + assert_eq!(proved.4.len(), proved.0.roots.len(), "{argue:?} {group:?}"); + assert_eq!(proved.3.len(), SIZES.len()); + check(argue, group, &proved, &proved.4) + .unwrap_or_else(|e| panic!("{argue:?} {group:?}: {e:?}")); + } + } +} + +/// ★ An answer altered, missing, extra, or swapped with another group's is +/// refused: the count before anything is absorbed, the rest at the +/// challenges they moved or at the chain that carries them. +#[test] +fn an_altered_missing_extra_or_moved_answer_is_refused() { + for argue in ARGUES { + for group in GROUPS { + let proved = proved(argue, group, None); + let answers = &proved.4; + let n = answers.len(); + assert!(n >= 3, "the block carries a few polynomials"); + for at in [0, n / 2, n - 1] { + let mut altered = answers.clone(); + altered[at] += FieldElement::::one(); + assert!( + check(argue, group, &proved, &altered).is_err(), + "{argue:?} {group:?}: answer {at} altered" + ); + } + assert_eq!( + check(argue, group, &proved, &answers[..n - 1]), + Err(MlError::CommitOodCount { + expected: n, + got: n - 1 + }), + "{argue:?} {group:?}: an answer missing" + ); + let mut extra = answers.clone(); + extra.push(FieldElement::::one()); + assert_eq!( + check(argue, group, &proved, &extra), + Err(MlError::CommitOodCount { + expected: n, + got: n + 1 + }), + "{argue:?} {group:?}: an answer too many" + ); + let mut moved = answers.clone(); + moved.swap(0, n - 1); + assert_ne!(answers[0], answers[n - 1]); + assert!( + check(argue, group, &proved, &moved).is_err(), + "{argue:?} {group:?}: the first group's answer swapped with the last's" + ); + } + } +} + +/// A prover that answers falsely — and absorbs, sends and carries the false +/// answer, so every challenge is the one it drew — is refused by the chain +/// that carries it. +#[test] +fn a_false_answer_its_chain_carries_is_refused() { + for argue in ARGUES { + for group in GROUPS { + let proved = proved(argue, group, Some(OodFault::False)); + let refused = check(argue, group, &proved, &proved.4); + assert!( + matches!(refused, Err(MlError::ColumnOpeningRejected { .. })), + "{argue:?} {group:?}: {refused:?}" + ); + } + } +} + +/// ★ Mutation: the batching is what binds an answer. A prover whose false +/// answer no chain carries is refused by the verifier; the same verifier with +/// the answers left out of its chains (`VerifierChecks::commit_ood` off) +/// accepts it — the answers are then absorbed and counted, and bind nothing. +#[test] +fn the_batching_is_what_refuses_an_unbound_false_answer() { + for argue in ARGUES { + for group in GROUPS { + let proved = proved(argue, group, Some(OodFault::FalseUnbound)); + assert!( + check(argue, group, &proved, &proved.4).is_err(), + "{argue:?} {group:?}: refused" + ); + let unbatched = VerifierChecks { + commit_ood: false, + ..VerifierChecks::ALL + }; + verify_block(&config_for(argue, group), &proved, &proved.4, unbatched).unwrap_or_else( + |e| panic!("{argue:?} {group:?}: a verifier that does not batch refused: {e:?}"), + ); + } + } +} diff --git a/crypto/stark/src/tests/cap_fri_matrix_tests.rs b/crypto/stark/src/tests/cap_fri_matrix_tests.rs new file mode 100644 index 000000000..f4278816a --- /dev/null +++ b/crypto/stark/src/tests/cap_fri_matrix_tests.rs @@ -0,0 +1,267 @@ +//! Merkle caps (S1) composed with group-leaf FRI layers (S3) on the host path: +//! a round-trip matrix {cap off, fixed, auto} × {pair, dp, +//! dp with an uneven override}, at a query count where `auto` caps (Q ≥ 20). +//! +//! Under a fold schedule a committed FRI layer is a GROUP tree whose depth is +//! the layout's (`FriFoldLayout::layer_depth`), not today's +//! `log2(lde) − j − 2`. The cap of each layer is taken at that depth, by the +//! prover (`StarkCaps::from_layout` in round 4) and by the verifier (the +//! per-tree `TreeCheck` the group path authenticates with). These tests pin: +//! - every cell of the matrix proves and verifies, owned and archived; +//! - every FRI layer's paths have the capped shape at the LAYOUT's depth, +//! computed here independently from the schedule; +//! - every cap node of a capped group layer is bound, and an unreached one is +//! rejected by the cap-to-root check alone (on a group tree); +//! - a proof made under one (cap, fri) format fails under the others. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::merkle_tree::cap::{CapPolicy, verify_cap}; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +use crate::config::{Commitment, DefaultStarkHash, StarkHash}; +use crate::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use crate::fri::capture::{FriCapture, capture}; +use crate::fri::schedule::FriFormat; +use crate::merkle_caps::StarkCaps; +use crate::proof::options::{FriMode, FriScheduleOverride, ProofFormat, ProofOptions}; +use crate::proof::stark::{MultiProof, StarkProof}; +use crate::prover::{IsStarkProver, Prover}; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +type F = GoldilocksField; +type FE = FieldElement; +type PI = SimpleAdditionPublicInputs; +type Proof = StarkProof; +/// The leaf backend the FRI layer trees are verified with (the FRI values +/// live in the proof's extension, which is `F` for this AIR). +type Leaf = ::Batched; + +/// 1024 rows at blowup 2 with `k = 2`: LDE 2^11, terminal 2^3, so the +/// committed chain covers 10 → 3 (seven bits). +const ROWS: usize = 1024; +const LDE_LOG: u32 = 11; +const TERMINAL_LOG: u32 = 3; +/// `auto` caps at height 3 from 20 openings on. +const QUERIES: usize = 24; + +fn options(cap: CapPolicy, fri: FriMode, over: Option<&[u8]>, queries: usize) -> ProofOptions { + let mut o = ProofOptions::default_test_options(); + o.blowup_factor = 2; + o.fri_number_of_queries = queries; + o.grinding_factor = 0; + o.fri_final_poly_log_degree = 2; + o.format = ProofFormat { + merkle_cap: cap, + fri_mode: fri, + fri_schedule_override: over.and_then(FriScheduleOverride::new), + ..ProofFormat::DEFAULT + }; + o +} + +fn prove(opts: &ProofOptions) -> (SimpleAdditionAIR, Proof) { + let air = SimpleAdditionAIR::::new(opts); + let pub_inputs = SimpleAdditionPublicInputs { + a: FE::from(1u64), + b: FE::from(2u64), + }; + let mut trace = simple_addition_trace::(ROWS); + let proof = Prover::prove( + &air, + &mut trace, + &pub_inputs, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof) +} + +fn verifies(air: &SimpleAdditionAIR, proof: &Proof) -> bool { + Verifier::verify(proof, air, &mut DefaultTranscript::::new(&[])) +} + +fn verifies_archived(air: &SimpleAdditionAIR, proof: &Proof) -> bool { + let multi = MultiProof { + proofs: vec![proof.clone()], + }; + let bytes = rkyv::to_bytes::(&multi).unwrap(); + let archived = rkyv::access::< + crate::proof::stark::ArchivedMultiProof, + rkyv::rancor::Error, + >(&bytes) + .unwrap(); + let airs: Vec<&dyn AIR> = vec![air]; + Verifier::multi_verify_archived( + &airs, + archived, + &mut DefaultTranscript::::new(&[]), + &FE::zero(), + ) +} + +/// The committed layers' tree depths, from the schedule alone: the chain +/// starts at `lde_log − 1` and layer `j`'s tree is its length over `2^{d_j}` +/// leaves. Independent of `FriFoldLayout::layer_depth`. +fn schedule_depths(opts: &ProofOptions) -> (Vec, Vec) { + let schedule = FriFormat::from_options(opts, false).schedule(LDE_LOG, TERMINAL_LOG); + let mut b = LDE_LOG as usize - 1; + let depths = schedule + .iter() + .map(|&d| { + b -= d as usize; + b + }) + .collect(); + (schedule, depths) +} + +fn fri_paths(proof: &Proof, layer: usize) -> Vec<&Vec> { + proof + .query_list + .iter() + .map(|q| &q.layers_auth_paths[layer].merkle_path) + .collect() +} + +const FORMATS: [(&str, FriMode, Option<&[u8]>); 3] = [ + ("pair", FriMode::Pair, None), + ("dp", FriMode::Dp, None), + ("dp [3,1,3]", FriMode::Dp, Some(&[3, 1, 3])), +]; + +#[test] +fn the_cap_and_fri_matrix_round_trips_owned_and_archived() { + for cap in [CapPolicy::Off, CapPolicy::Fixed(2), CapPolicy::Auto] { + for (name, fri, over) in FORMATS { + let opts = options(cap, fri, over, QUERIES); + let (air, proof) = prove(&opts); + let (schedule, depths) = schedule_depths(&opts); + let caps = StarkCaps::for_options(&opts, LDE_LOG as usize, false).expect("layout"); + assert_eq!( + caps.fri_depths, depths, + "cap={cap} fri={name}: the caps' FRI depths are the layout's" + ); + assert_eq!(proof.fri_layers_merkle_roots.len(), schedule.len()); + for (j, root) in proof.fri_layers_merkle_roots.iter().enumerate() { + let (d, c) = (depths[j], caps.fri[j]); + assert_eq!(c, cap.height(QUERIES, d), "cap={cap} fri={name} layer {j}"); + let paths = fri_paths(&proof, j); + let owner = if c == 0 { d } else { d - c + (1 << c) }; + assert_eq!( + paths[0].len(), + owner, + "cap={cap} fri={name} layer {j} owner" + ); + for p in &paths[1..] { + assert_eq!(p.len(), d - c, "cap={cap} fri={name} layer {j}"); + } + if c > 0 { + assert!( + verify_cap::(&paths[0][d - c..], root, c), + "cap={cap} fri={name} layer {j}: the owner's cap hashes to the root" + ); + } + } + if cap != CapPolicy::Off { + assert!( + caps.fri.iter().any(|&c| c > 0), + "cap={cap} fri={name}: some FRI layer must be capped" + ); + } + assert!(verifies(&air, &proof), "cap={cap} fri={name}"); + assert!( + verifies_archived(&air, &proof), + "cap={cap} fri={name}: archived" + ); + } + } +} + +#[test] +fn every_cap_node_of_a_capped_group_layer_is_bound() { + let opts = options(CapPolicy::Auto, FriMode::Dp, Some(&[3, 1, 3]), QUERIES); + let (air, honest) = prove(&opts); + assert!(verifies(&air, &honest)); + let (_, depths) = schedule_depths(&opts); + // Layer 0 folds by 8 (depth 7) and layer 2 by 8 (depth 3): both capped at 3. + for j in [0usize, 2] { + let (d, c) = (depths[j], CapPolicy::Auto.height(QUERIES, depths[j])); + assert_eq!(c, 3, "layer {j}"); + for k in 0..(1usize << c) { + let mut bad = honest.clone(); + bad.query_list[0].layers_auth_paths[j].merkle_path[d - c + k][5] ^= 1; + assert!( + !verifies(&air, &bad) && !verifies_archived(&air, &bad), + "group layer {j}: cap node {k} flipped" + ); + } + // A later query's path, cut at the cap (layer 2's tree is all cap: + // depth 3 at c = 3, so its paths are empty). + assert_eq!( + honest.query_list[7].layers_auth_paths[j].merkle_path.len(), + d - c + ); + if d > c { + let mut bad = honest.clone(); + bad.query_list[7].layers_auth_paths[j].merkle_path[0][0] ^= 1; + assert!(!verifies(&air, &bad), "group layer {j}: query 7 sibling"); + } + } +} + +/// An unreached cap node on a group tree: with three queries and a height-3 cap on +/// FRI layer 0, at least five of its eight cap nodes are reached by no query. +/// Flipping one leaves every per-query fold untouched (each still lands on its +/// own cap node), so only the cap-to-root check of the group layer's +/// `TreeCheck` rejects the proof. +#[test] +fn an_unreached_cap_node_of_a_group_layer_is_rejected_by_the_cap_to_root_check_alone() { + let opts = options(CapPolicy::Fixed(3), FriMode::Dp, Some(&[3, 1, 3]), 3); + let (air, honest) = prove(&opts); + let (ok, records) = + capture(|| Verifier::verify(&honest, &air, &mut DefaultTranscript::::new(&[]))); + assert!(ok); + let rec = FriCapture::::from_any(records[0].as_ref()).expect("one record"); + let (schedule, depths) = schedule_depths(&opts); + let (d0, d, c) = (schedule[0] as usize, depths[0], 3usize); + // Layer 0's leaf is `iota >> d0`; its cap node is `leaf >> (d − c)`. + let reached: Vec = rec.iotas.iter().map(|i| (i >> d0) >> (d - c)).collect(); + let unreached: Vec = (0..8).filter(|k| !reached.contains(k)).collect(); + assert!(unreached.len() >= 5, "3 queries reach at most 3 of 8 nodes"); + for k in unreached { + let mut bad = honest.clone(); + bad.query_list[0].layers_auth_paths[0].merkle_path[d - c + k][3] ^= 1; + assert!(!verifies(&air, &bad), "unreached cap node {k}"); + assert!( + !verifies_archived(&air, &bad), + "unreached cap node {k}: archived" + ); + } +} + +/// The (cap, fri) format is a verifier constant: a proof made under one fails +/// under every other cell of the matrix. +#[test] +fn a_proof_made_under_one_cap_and_fri_format_fails_under_another() { + let cells = [ + (CapPolicy::Off, FriMode::Pair), + (CapPolicy::Auto, FriMode::Pair), + (CapPolicy::Off, FriMode::Dp), + (CapPolicy::Auto, FriMode::Dp), + ]; + for (i, &(cap, fri)) in cells.iter().enumerate() { + let (_, proof) = prove(&options(cap, fri, None, QUERIES)); + for (k, &(cap_v, fri_v)) in cells.iter().enumerate() { + let air = SimpleAdditionAIR::::new(&options(cap_v, fri_v, None, QUERIES)); + assert_eq!( + verifies(&air, &proof), + i == k, + "proved at ({cap}, {fri:?}), verified at ({cap_v}, {fri_v:?})" + ); + } + } +} diff --git a/crypto/stark/src/tests/commit_ood_device_tests.rs b/crypto/stark/src/tests/commit_ood_device_tests.rs new file mode 100644 index 000000000..d27bf88d1 --- /dev/null +++ b/crypto/stark/src/tests/commit_ood_device_tests.rs @@ -0,0 +1,172 @@ +//! Commitment-time answers on the card: the answers a bound prove takes off +//! the resident main LDE equal the host's barycentric evaluation of the same +//! columns, the proof verifies with them, and a false answer is refused. Box +//! only (`--features cuda`, ignored); it fails loudly when the card arm did not +//! run, so it cannot pass by falling back to the host: +//! +//! ```text +//! LAMBDA_VM_GPU_LDE_THRESHOLD=2 LAMBDA_VM_GPU_BARY_THRESHOLD=2 \ +//! cargo test -p stark --release --features cuda --lib \ +//! tests::commit_ood_device_tests -- --ignored --test-threads=1 +//! ``` + +use std::sync::atomic::Ordering; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField, goldilocks::GoldilocksField, +}; + +use crate::commit_ood::faults::{self, Faults}; +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::MultiProof; +use crate::proof::view::MultiProofView; +use crate::prover::{IsStarkProver, Prover}; +use crate::residency_mode::ResidencyMode; +use crate::trace::TraceTable; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type FE = FieldElement; +type EE = FieldElement; + +/// The CPU/ADD/MUL instance at `2^log_n` operations of each kind: the CPU +/// table sends every ADD then every MUL, and each receiving table holds its +/// triples once. +fn traces(log_n: u32) -> (TraceTable, TraceTable, TraceTable) { + let n = 1usize << log_n; + let a: Vec = (0..n as u64).map(|i| FE::from(3 * i + 1)).collect(); + let b: Vec = (0..n as u64).map(|i| FE::from(7 * i + 2)).collect(); + let sum: Vec = a.iter().zip(&b).map(|(x, y)| x + y).collect(); + let prod: Vec = a.iter().zip(&b).map(|(x, y)| x * y).collect(); + let ones = vec![FE::one(); n]; + let zeros = vec![FE::zero(); n]; + let cat = |x: &[FE], y: &[FE]| [x, y].concat(); + let cpu = TraceTable::from_columns_main( + vec![ + cat(&ones, &zeros), + cat(&zeros, &ones), + cat(&a, &a), + cat(&b, &b), + cat(&sum, &prod), + ], + 1, + ); + let add = TraceTable::from_columns_main(vec![a.clone(), b.clone(), sum, ones.clone()], 1); + let mul = TraceTable::from_columns_main(vec![a, b, prod, ones], 1); + (cpu, add, mul) +} + +fn options() -> ProofOptions { + ProofOptions::default_test_options() +} + +/// A bound proof, its answers, and the traces it proved. +type Proved = (MultiProof, Vec>, [TraceTable; 3]); + +fn prove(log_n: u32, faults: Faults) -> Proved { + let (mut cpu, mut add, mut mul) = traces(log_n); + let opts = options(); + let (cpu_air, add_air, mul_air) = ( + new_cpu_air_with_lookup(&opts), + new_add_air_with_lookup(&opts), + new_mul_air_with_lookup(&opts), + ); + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu, &()), + (&add_air, &mut add, &()), + (&mul_air, &mut mul, &()), + ]; + let (proof, answers) = faults::with(faults, || { + Prover::multi_prove_bound( + pairs, + &mut DefaultTranscript::::new(&[]), + ResidencyMode::Retain, + ) + }) + .expect("the instance proves"); + (proof, answers, [cpu, add, mul]) +} + +fn verifies(proof: &MultiProof, answers: &[Vec]) -> bool { + let opts = options(); + let (cpu_air, add_air, mul_air) = ( + new_cpu_air_with_lookup(&opts), + new_add_air_with_lookup(&opts), + new_mul_air_with_lookup(&opts), + ); + let airs: Vec<&dyn AIR> = + vec![&cpu_air, &add_air, &mul_air]; + Verifier::multi_verify_views_bound( + &airs, + MultiProofView::Owned(proof), + answers, + &mut DefaultTranscript::::new(&[]), + &FieldElement::zero(), + ) +} + +/// ★ The card's answers are the host's, the proof verifies, and the card arm +/// ran for every table (`COMMIT OOD DEVICE` line: tables off the card). +#[test] +#[ignore = "box: needs a GPU"] +fn the_cards_answers_are_the_hosts_and_the_proof_verifies() { + for log_n in [14u32, 16] { + let before = crate::gpu_lde::COMMIT_OOD_GPU_CALLS.load(Ordering::Relaxed); + let (proof, answers, traces) = prove(log_n, Faults::default()); + let on_card = crate::gpu_lde::COMMIT_OOD_GPU_CALLS.load(Ordering::Relaxed) - before; + println!("COMMIT OOD DEVICE log_n {log_n}: {on_card} of 3 tables off the card"); + assert_eq!(on_card, 3, "every table's answers came off the card"); + // `z0` again, as the verifier draws it. + let mut t = DefaultTranscript::::new(&[]); + for p in &proof.proofs { + crypto::fiat_shamir::is_transcript::IsTranscript::append_bytes( + &mut t, + &p.lde_trace_main_merkle_root, + ); + } + let rows = 2usize << log_n; + let z0 = crate::commit_ood::draw_point::( + &mut t, + ( + rows, + rows * options().blowup_factor as usize, + options().coset_offset, + ), + ); + for (table, (trace, answers)) in traces.iter().zip(&answers).enumerate() { + let (data, width) = trace.main_data_row_major(); + let host = crate::commit_ood::evaluate_trace_columns(data, width, 0..width, &z0) + .expect("z0 is out of the domain"); + assert_eq!(*answers, host, "log_n {log_n} table {table}"); + } + assert!(verifies(&proof, &answers), "log_n {log_n}"); + println!("COMMIT OOD DEVICE log_n {log_n}: answers equal the host's, proof verifies"); + } +} + +/// A false answer carried into DEEP on the card's path is refused. +#[test] +#[ignore = "box: needs a GPU"] +fn a_false_answer_on_the_cards_path_is_refused() { + let (proof, answers, _) = prove( + 14, + Faults { + false_answer: Some((0, 2)), + without_deep_term: false, + z0_meets_z: false, + }, + ); + assert!(!verifies(&proof, &answers)); + println!("COMMIT OOD DEVICE false answer: refused"); +} diff --git a/crypto/stark/src/tests/commit_ood_tests.rs b/crypto/stark/src/tests/commit_ood_tests.rs new file mode 100644 index 000000000..5e039f218 --- /dev/null +++ b/crypto/stark/src/tests/commit_ood_tests.rs @@ -0,0 +1,339 @@ +//! Commitment-time out-of-domain answers on a multi-table proof +//! (`crate::commit_ood`), on the bus-balanced CPU/ADD/MUL instance of +//! [`super::residency_mode_tests`]: a bound proof verifies with its answers; +//! Phase A absorbs every answer between the roots and the shared LogUp +//! challenges; an answer altered, missing, extra or moved is refused; a false +//! answer the prover carries into its DEEP polynomial is refused; and with the +//! answers' DEEP quotients left out on both sides that same false answer is +//! accepted, so the quotients are what refuses it. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::{IsStarkTranscript, IsTranscript}; +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField, goldilocks::GoldilocksField, +}; + +use crate::commit_ood::faults::{self, Faults}; +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::MultiProof; +use crate::proof::view::MultiProofView; +use crate::prover::{IsStarkProver, Prover}; +use crate::residency_mode::ResidencyMode; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type EE = FieldElement; +type Answers = Vec>; + +/// Blowup 4 at 40 queries and no grinding: a false answer's DEEP quotient is +/// far from every codeword, so FRI misses it with probability below 2^-80, +/// and the proof is deterministic. +fn options() -> ProofOptions { + ProofOptions { + blowup_factor: 4, + fri_number_of_queries: 40, + grinding_factor: 0, + ..ProofOptions::default_test_options() + } +} + +/// The instance proved bound under `faults`, its answers beside it. +fn prove_bound(faults: Faults) -> (MultiProof, Answers) { + let (mut cpu_trace, mut add_trace, mut mul_trace) = super::residency_mode_tests::traces(); + let opts = options(); + let cpu_air = new_cpu_air_with_lookup(&opts); + let add_air = new_add_air_with_lookup(&opts); + let mul_air = new_mul_air_with_lookup(&opts); + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu_trace, &()), + (&add_air, &mut add_trace, &()), + (&mul_air, &mut mul_trace, &()), + ]; + faults::with(faults, || { + Prover::multi_prove_bound( + pairs, + &mut DefaultTranscript::::new(&[]), + ResidencyMode::Retain, + ) + }) + .expect("the instance proves") +} + +fn verify_bound(proof: &MultiProof, answers: &[Vec], faults: Faults) -> bool { + let opts = options(); + let cpu_air = new_cpu_air_with_lookup(&opts); + let add_air = new_add_air_with_lookup(&opts); + let mul_air = new_mul_air_with_lookup(&opts); + let airs: Vec<&dyn AIR> = + vec![&cpu_air, &add_air, &mul_air]; + faults::with(faults, || { + Verifier::multi_verify_views_bound( + &airs, + MultiProofView::Owned(proof), + answers, + &mut DefaultTranscript::::new(&[]), + &FieldElement::zero(), + ) + }) +} + +/// ★ Every table answers once per main column (none of these AIRs is +/// preprocessed), and the bound proof verifies with its answers — and only +/// with the bound verifier: the plain one replays another transcript. +#[test] +fn a_bound_proof_verifies_with_its_answers() { + let (proof, answers) = prove_bound(Faults::default()); + let widths: Vec = answers.iter().map(Vec::len).collect(); + assert_eq!(widths, vec![5, 4, 4], "one answer per main column"); + assert!(verify_bound(&proof, &answers, Faults::default())); + + let opts = options(); + let cpu_air = new_cpu_air_with_lookup(&opts); + let add_air = new_add_air_with_lookup(&opts); + let mul_air = new_mul_air_with_lookup(&opts); + let airs: Vec<&dyn AIR> = + vec![&cpu_air, &add_air, &mul_air]; + assert!( + !Verifier::multi_verify( + &airs, + &proof, + &mut DefaultTranscript::::new(&[]), + &FieldElement::zero(), + ), + "a bound proof is not a plain one" + ); +} + +/// The answers are `t_c(z0)`: each one equals its column's interpolant at the +/// `z0` Phase A draws, evaluated independently here by Lagrange over the +/// table's trace domain. +#[test] +fn every_answer_is_its_column_at_the_drawn_point() { + let (proof, answers) = prove_bound(Faults::default()); + let mut t = DefaultTranscript::::new(&[]); + for p in &proof.proofs { + t.append_bytes(&p.lde_trace_main_merkle_root); + } + let z0: EE = t.sample_z_ood_with_domain_params( + 8, + 8 * options().blowup_factor as usize, + &FieldElement::::from(options().coset_offset), + ); + let (cpu, add, mul) = super::residency_mode_tests::traces(); + for (table, (trace, answers)) in [cpu, add, mul].iter().zip(&answers).enumerate() { + let (data, width) = trace.main_data_row_major(); + let rows = data.len() / width; + let omega = ::get_primitive_root_of_unity( + rows.trailing_zeros() as u64, + ) + .unwrap(); + let points: Vec> = (0..rows).map(|k| omega.pow(k)).collect(); + for (c, answer) in answers.iter().enumerate() { + let mut value = EE::zero(); + for k in 0..rows { + let mut basis = EE::one(); + for (j, xj) in points.iter().enumerate() { + if j != k { + let num = &z0 - &(*xj).to_extension::(); + let den = (&points[k] - xj).inv().unwrap(); + basis = basis * num * den.to_extension::(); + } + } + value += basis * data[k * width + c].to_extension::(); + } + assert_eq!(*answer, value, "table {table} column {c}"); + } + } +} + +/// ★ Phase A spelled out: every root, `z0`, every answer, THEN the shared +/// LogUp challenges. Each answer moves the first challenge, and the same +/// draw with the answers left out (or absorbed after it) is another value — +/// the placement no round trip can see, since prover and verifier would +/// share it. +#[test] +fn phase_a_binds_every_answer_before_the_shared_challenges() { + let (proof, answers) = prove_bound(Faults::default()); + let opts = options(); + let cpu_air = new_cpu_air_with_lookup(&opts); + let add_air = new_add_air_with_lookup(&opts); + let mul_air = new_mul_air_with_lookup(&opts); + let airs: Vec<&dyn AIR> = + vec![&cpu_air, &add_air, &mul_air]; + let challenges = |answers: &[Vec]| { + Verifier::replay_phase_a_bound( + &airs, + MultiProofView::Owned(&proof), + answers, + &mut DefaultTranscript::::new(&[]), + ) + .expect("the replay accepts the counts") + }; + let drawn = challenges(&answers); + + let mut by_hand = DefaultTranscript::::new(&[]); + for p in &proof.proofs { + by_hand.append_bytes(&p.lde_trace_main_merkle_root); + } + let lde = 8 * opts.blowup_factor as usize; + let offset = FieldElement::::from(opts.coset_offset); + let _z0: EE = by_hand.sample_z_ood_with_domain_params(8, lde, &offset); + for answer in answers.iter().flatten() { + by_hand.append_field_element(answer); + } + let spelled: Vec = (0..2).map(|_| by_hand.sample_field_element()).collect(); + assert_eq!(drawn, spelled); + + for (t, table) in answers.iter().enumerate() { + for i in 0..table.len() { + let mut moved = answers.clone(); + moved[t][i] += EE::one(); + assert_ne!( + challenges(&moved)[0], + drawn[0], + "table {t} answer {i} moves z" + ); + } + } + let mut late = DefaultTranscript::::new(&[]); + for p in &proof.proofs { + late.append_bytes(&p.lde_trace_main_merkle_root); + } + let _z0: EE = late.sample_z_ood_with_domain_params(8, lde, &offset); + let late_z: EE = late.sample_field_element(); + assert_ne!( + late_z, drawn[0], + "answers absorbed after the draw bind nothing" + ); +} + +/// An answer altered, a table's answers short or long, a table's answers +/// missing or extra, or two tables' answers swapped: each refused. +#[test] +fn an_altered_missing_extra_or_moved_answer_is_refused() { + let (proof, answers) = prove_bound(Faults::default()); + let refused = |answers: Answers, what: &str| { + assert!( + !verify_bound(&proof, &answers, Faults::default()), + "{what} must be refused" + ); + }; + for (t, table) in answers.iter().enumerate() { + for i in 0..table.len() { + let mut altered = answers.clone(); + altered[t][i] += EE::one(); + refused(altered, &format!("table {t} answer {i} altered")); + } + let mut short = answers.clone(); + short[t].pop(); + refused(short, &format!("table {t} one answer short")); + let mut long = answers.clone(); + long[t].push(EE::zero()); + refused(long, &format!("table {t} one answer long")); + } + refused(answers[..2].to_vec(), "a table's answers missing"); + let mut extra = answers.clone(); + extra.push(Vec::new()); + refused(extra, "an extra table of answers"); + let mut swapped = answers.clone(); + swapped.swap(1, 2); + refused(swapped, "two tables' answers swapped"); + refused(Vec::new(), "no answers at all"); +} + +/// A false answer the prover absorbs AND carries into its DEEP polynomial (a +/// consistent proof of the false value) is refused: the DEEP quotient at `z0` +/// is not low degree. +#[test] +fn a_false_answer_its_deep_carries_is_refused() { + for (table, index) in [(0, 0), (0, 4), (1, 2), (2, 3)] { + let fault = Faults { + false_answer: Some((table, index)), + without_deep_term: false, + z0_meets_z: false, + }; + let (proof, answers) = prove_bound(fault); + assert!( + !verify_bound(&proof, &answers, Faults::default()), + "a false answer at table {table} column {index} must be refused" + ); + } +} + +/// ★ Load-bearing: with the answers' DEEP quotients left out on both sides, +/// the same false answer is ACCEPTED. So the quotients are what refuses it in +/// [`a_false_answer_its_deep_carries_is_refused`], not some other check. +#[test] +fn the_deep_quotients_are_what_refuse_a_false_answer() { + let fault = Faults { + false_answer: Some((1, 2)), + without_deep_term: true, + z0_meets_z: false, + }; + let (proof, answers) = prove_bound(fault); + assert!( + verify_bound(&proof, &answers, fault), + "without the quotients nothing checks an answer" + ); + assert!( + !verify_bound(&proof, &answers, Faults::default()), + "with them the false answer is refused" + ); +} + +/// The faults are per thread and off by default: a bound proof proved and +/// verified with none set is the production path. +#[test] +fn no_fault_is_set_by_default() { + assert_eq!(faults::current(), Faults::default()); + let (proof, answers) = prove_bound(Faults::default()); + assert!(verify_bound(&proof, &answers, Faults::default())); +} + +/// `z0` meets `z·gʳ` for each OOD row and `z^P`, and no other point. +#[test] +fn the_point_check_finds_every_deep_point() { + use math::field::traits::IsFFTField; + let g = F::get_primitive_root_of_unity(3).expect("a root of order 8"); + let z = EE::new([ + FieldElement::from(7u64), + FieldElement::from(11u64), + FieldElement::from(13u64), + ]); + let meets = |z0: &EE| crate::commit_ood::meets_deep_point(z0, &z, &g, 2, 3); + assert!(meets(&z), "z"); + assert!(meets(&(&g * &z)), "g·z"); + assert!(meets(&z.pow(3usize)), "z^P"); + assert!( + !meets(&(&g * &(&g * &z))), + "g²·z is no DEEP point at two rows" + ); + assert!(!meets(&z.pow(2usize)), "z² is no DEEP point at three parts"); + assert!(!meets(&(&z + EE::one())), "an unrelated point"); +} + +/// A `z0` that meets a DEEP point is refused: the same honest proof passes, +/// and fails once the verifier's point check sees `z` in place of `z0`. +#[test] +fn a_z0_that_meets_a_deep_point_is_refused() { + let (proof, answers) = prove_bound(Faults::default()); + assert!(verify_bound(&proof, &answers, Faults::default())); + let fault = Faults { + z0_meets_z: true, + ..Faults::default() + }; + assert!( + !verify_bound(&proof, &answers, fault), + "z0 = z must be refused" + ); +} diff --git a/crypto/stark/src/tests/commitment_tests.rs b/crypto/stark/src/tests/commitment_tests.rs index f1684112b..eb696c11e 100644 --- a/crypto/stark/src/tests/commitment_tests.rs +++ b/crypto/stark/src/tests/commitment_tests.rs @@ -6,8 +6,11 @@ use crate::commitment::{ ROWS_PER_LEAF, commit_bit_reversed, keccak_leaves_bit_reversed, keccak_leaves_bit_reversed_grouped, keccak_leaves_row_pair_bit_reversed, + leaves_bit_reversed_grouped, +}; +use crate::config::{ + BatchedMerkleTree, BatchedMerkleTreeBackend, Commitment, KeccakStarkHash, StarkHash, }; -use crate::config::{BatchedMerkleTree, BatchedMerkleTreeBackend, Commitment}; use math::fft::bit_reversing::reverse_index; use math::field::{element::FieldElement, goldilocks::GoldilocksField}; use math::traits::ByteConversion; @@ -27,7 +30,10 @@ fn sample_columns() -> Vec> { /// each row, big-endian), hashed once with the same backend the prover uses. /// Structurally separate from the production `map_init` loop, so a transposed /// row/column order or a wrong bit-reversal in production fails this check. -fn expected_leaf(columns: &[Vec], rows_per_leaf: usize, leaf_idx: usize) -> Commitment { +fn expected_leaf(columns: &[Vec], rows_per_leaf: usize, leaf_idx: usize) -> Commitment +where + B: crypto::merkle_tree::traits::IsStreamingLeafBackend, +{ let num_rows = columns[0].len(); let byte_len = ::BYTE_LEN; let mut buf = vec![0u8; rows_per_leaf * columns.len() * byte_len]; @@ -39,24 +45,40 @@ fn expected_leaf(columns: &[Vec], rows_per_leaf: usize, leaf_idx: usize) - offset += byte_len; } } - BatchedMerkleTreeBackend::::hash_bytes(&buf) + B::hash_bytes(&buf) } +/// The keccak batched backend, named. `keccak_leaves_*` is the CUDA parity +/// reference and computes keccak whatever the default is (P-a Stage 6), so a +/// test of THOSE helpers must say keccak on both sides. +type KeccakBatched = ::Batched; + #[test] fn grouped_leaves_match_documented_layout_for_r1_and_r2() { let columns = sample_columns(); let num_rows = columns[0].len(); for &rows_per_leaf in &[1usize, 2usize] { - let leaves = keccak_leaves_bit_reversed_grouped(&columns, rows_per_leaf); + // Both configurations, because the layout is what is being pinned and + // the layout is hash-independent: the production one is what the + // verifier must match, the keccak one is what the CUDA kernels mirror. + let leaves = + leaves_bit_reversed_grouped::>(&columns, rows_per_leaf); + let keccak_leaves = keccak_leaves_bit_reversed_grouped(&columns, rows_per_leaf); assert_eq!( leaves.len(), num_rows / rows_per_leaf, "leaf count for rows_per_leaf={rows_per_leaf}" ); + assert_eq!(keccak_leaves.len(), leaves.len()); for (i, leaf) in leaves.iter().enumerate() { + assert_eq!( + keccak_leaves[i], + expected_leaf::(&columns, rows_per_leaf, i), + "keccak leaf {i} for rows_per_leaf={rows_per_leaf}" + ); assert_eq!( *leaf, - expected_leaf(&columns, rows_per_leaf, i), + expected_leaf::>(&columns, rows_per_leaf, i), "leaf {i} for rows_per_leaf={rows_per_leaf}" ); } @@ -79,7 +101,10 @@ fn wrappers_agree_with_grouped() { #[test] fn commit_root_matches_tree_built_over_leaves() { let columns = sample_columns(); - let leaves = keccak_leaves_bit_reversed_grouped(&columns, ROWS_PER_LEAF); + // The production leaves, not the keccak ones: `commit_bit_reversed` follows + // the aliases, so both sides have to. + let leaves = + leaves_bit_reversed_grouped::>(&columns, ROWS_PER_LEAF); let tree = BatchedMerkleTree::::build_from_hashed_leaves(leaves).unwrap(); let (_, root) = commit_bit_reversed(&columns, ROWS_PER_LEAF).unwrap(); assert_eq!(root, tree.root); @@ -94,3 +119,163 @@ fn empty_and_zero_row_inputs_short_circuit() { assert!(keccak_leaves_bit_reversed_grouped(&zero_rows, ROWS_PER_LEAF).is_empty()); assert!(commit_bit_reversed(&zero_rows, ROWS_PER_LEAF).is_none()); } + +/// ★ The [`StarkHash`] two-element-leaf invariant, for the keccak configuration. +/// +/// The prover commits FRI layers with `Pair` and the verifier authenticates +/// those openings with `Batched` (`verify_fri_layer_openings` builds a +/// two-element `Vec`). Nothing in the type system makes those agree — this is +/// what says they do, so a second configuration that breaks it fails here +/// rather than by rejecting every honest proof at its first FRI query. +#[test] +fn batched_and_pair_agree_on_a_two_element_leaf() { + use crate::config::{KeccakStarkHash, StarkHash}; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + type Batched = ::Batched; + type Pair = ::Pair; + + for (a, b) in [(0u64, 1u64), (7, 7), (u64::MAX - 1, 12345)] { + let (x, y) = (Felt::from(a), Felt::from(b)); + assert_eq!( + ::hash_data(&vec![x, y]), + ::hash_data(&[x, y]), + "Batched and Pair must hash the pair ({a}, {b}) identically" + ); + } +} + +/// ★ The same invariant for the BLAKE3 configuration. +/// +/// Under `Blake3StarkHash` it holds by construction rather than by coincidence: +/// both families are the same generic backend over `Blake3Chain`, and a +/// two-element leaf is 16 bytes, one block, one compression. This is here +/// because "holds by construction" is an argument about today's code, and the +/// invariant has to survive tomorrow's. +#[cfg(not(feature = "cuda"))] +#[test] +fn blake3_batched_and_pair_agree_on_a_two_element_leaf() { + use crate::config::{Blake3StarkHash, StarkHash}; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + type Batched = ::Batched; + type Pair = ::Pair; + + for (a, b) in [(0u64, 1u64), (7, 7), (u64::MAX - 1, 12345)] { + let (x, y) = (Felt::from(a), Felt::from(b)); + assert_eq!( + ::hash_data(&vec![x, y]), + ::hash_data(&[x, y]), + "Batched and Pair must hash the pair ({a}, {b}) identically" + ); + } +} + +/// The two configurations are actually different hashes. +/// +/// Without this, every BLAKE3 test in this file would pass just as well if the +/// blake3 aliases had been left pointing at the keccak backends — which is a +/// realistic way for a type alias change to be wrong, and one that nothing else +/// here would notice. +#[cfg(not(feature = "cuda"))] +#[test] +fn the_blake3_and_keccak_configurations_commit_differently() { + use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + let leaf: Vec = (0..5u64).map(Felt::from).collect(); + assert_ne!( + <::Batched as IsMerkleTreeBackend>::hash_data(&leaf), + <::Batched as IsMerkleTreeBackend>::hash_data(&leaf), + ); +} + +/// A commit → open → verify round trip over the BLAKE3 configuration, through +/// the production commitment path. +/// +/// `commit_bit_reversed_with` is the entry point the prover's main trace commit +/// uses, and it is already backend-generic, so this exercises the real leaf +/// serialization (bit-reversed, row-grouped, big-endian) and the real tree +/// construction — not a hand-rolled tree over the backend. +/// +/// # What this does and does not establish +/// +/// It is a **same-reference** check: this build's prover and this build's +/// verifier agree, over the commitment layer. It makes no cross-version claim. +/// It is also deliberately narrow — the commitment layer alone, isolated from +/// FRI, the transcript and the AIR. The full prove→verify under this +/// configuration lives in `tests::blake3_stark_roundtrip_tests`; when both fail +/// together the fault is here, and this test says where. +#[cfg(not(feature = "cuda"))] +#[test] +fn blake3_commitments_open_and_verify() { + use crate::commitment::{commit_bit_reversed_with, leaves_bit_reversed_grouped}; + use crate::config::{Blake3StarkHash, StarkHash}; + use crypto::merkle_tree::merkle::MerkleTree; + use crypto::merkle_tree::proof::verify_merkle_path_from_leaf_hash; + + type Batched = ::Batched; + + let columns: Vec> = (0..4u64) + .map(|c| (0..16u64).map(|r| Felt::from(1000 * c + r + 1)).collect()) + .collect(); + + let (tree, root) = + commit_bit_reversed_with::(&columns, ROWS_PER_LEAF).expect("commit"); + let leaves = leaves_bit_reversed_grouped::(&columns, ROWS_PER_LEAF); + assert_eq!(root, tree.root); + assert_eq!(leaves.len(), 16 / ROWS_PER_LEAF); + + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree + .get_proof_by_pos(i) + .expect("proof for an in-range leaf"); + assert!( + verify_merkle_path_from_leaf_hash::(&proof.merkle_path, &root, i, *leaf), + "the honest opening of leaf {i} must verify" + ); + + // NEGATIVE CONTROL: the same path must not authenticate a different + // leaf, or "verify" above would be measuring nothing. + let other = leaves[(i + 1) % leaves.len()]; + assert!( + !verify_merkle_path_from_leaf_hash::(&proof.merkle_path, &root, i, other), + "a path must not authenticate a leaf it is not for, at {i}" + ); + } + + // And the root is the tree the leaves make, built independently. + let rebuilt = MerkleTree::::build_from_hashed_leaves(leaves).expect("rebuild"); + assert_eq!(rebuilt.root, root); +} + +/// The streaming routes and the owned-`Data` route are the same leaf. +#[test] +fn streaming_leaf_routes_match_hash_data() { + use crate::config::{KeccakStarkHash, StarkHash}; + use crypto::merkle_tree::traits::{IsMerkleTreeBackend, IsStreamingLeafBackend}; + + type Batched = ::Batched; + + let row: Vec = (0..5u64).map(Felt::from).collect(); + let (left, right) = row.split_at(2); + + let owned = ::hash_data(&row); + assert_eq!( + owned, + >::hash_data_from_slices(left, right), + "hash_data_from_slices must equal hash_data on the concatenation" + ); + + let mut buf = Vec::new(); + for e in &row { + let mut b = [0u8; 8]; + e.write_bytes_be(&mut b); + buf.extend_from_slice(&b); + } + assert_eq!( + owned, + >::hash_bytes(&buf), + "hash_bytes must equal hash_data on the elements those bytes encode" + ); +} diff --git a/crypto/stark/src/tests/constraint_index_tests.rs b/crypto/stark/src/tests/constraint_index_tests.rs new file mode 100644 index 000000000..d2140ec2b --- /dev/null +++ b/crypto/stark/src/tests/constraint_index_tests.rs @@ -0,0 +1,126 @@ +//! `check_dense_index_set` — the release-visible guard against a constraint +//! body emitting one index twice and another never. +//! +//! These tests are the guard's own honest control. A checker that never fires +//! would pass every "the real chip is fine" assertion in the workspace, so the +//! first thing established here is that it fires — on the exact shape that +//! motivated it (`COMMIT.md` §1.4.4 H1: a widened lane loop overrunning into +//! the pins that follow it) and on the degenerate cases either side. + +use crate::constraints::builder::{ + ConstraintBuilder, ConstraintMeta, ConstraintSet, RootKind, check_dense_index_set, +}; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; + +fn base(constraint_idx: usize) -> ConstraintMeta { + ConstraintMeta { + constraint_idx, + kind: RootKind::Base, + end_exemptions: 0, + } +} + +/// A body shaped like the hazard: `lanes` lane identities starting at 6, +/// then 8 unused-output pins starting at `pin_base`, then one tail constraint. +/// At `lanes = 8, pin_base = 14` the blocks abut exactly; widening `lanes` to +/// 12 without moving `pin_base` makes 14..17 collide. +struct LaneBody { + lanes: usize, + pin_base: usize, + tail: usize, +} + +impl ConstraintSet for LaneBody { + fn eval>(&self, b: &mut B) { + for lane in 0..self.lanes { + b.emit_base(6 + lane, b.zero()); + } + for j in 0..8 { + b.emit_base(self.pin_base + j, b.zero()); + } + for i in 0..6 { + b.emit_base(i, b.zero()); + } + b.emit_base(self.tail, b.zero()); + } +} + +/// ★ The H1 shape: widening the lane block over the pins that follow it. +/// +/// The count is unchanged — the body still emits 8+8+6+1 = 23 constraints into +/// 23 declared slots — which is exactly why `NUM_CONSTRAINTS`, a predicted-count +/// test, and `assert_complete` all miss it. Four lane identities are silently +/// overwritten and nothing else notices. +#[test] +fn the_widened_lane_block_collides_and_the_checker_says_so() { + let healthy = LaneBody { + lanes: 8, + pin_base: 14, + tail: 22, + }; + check_dense_index_set(&healthy.meta(), 23).expect("the un-widened body is dense"); + + let widened = LaneBody { + lanes: 12, + pin_base: 14, + tail: 22, + }; + // Same declared count — the collision is invisible to counting. + assert_eq!(widened.meta().len(), 27); + let err = check_dense_index_set(&widened.meta(), 27) + .expect_err("a lane block overrunning the pins must be caught"); + assert!( + err.contains("emitted twice [14, 15, 16, 17]"), + "the four colliding indices must be named, got: {err}" + ); + assert!( + err.contains("never emitted [23, 24, 25, 26]"), + "the slots left unwritten must be named, got: {err}" + ); +} + +/// A repeat with no compensating gap is still a repeat. +#[test] +fn a_plain_duplicate_is_caught() { + let meta = vec![base(0), base(1), base(1), base(2)]; + let err = check_dense_index_set(&meta, 4).expect_err("1 emitted twice"); + assert!(err.contains("emitted twice [1]"), "got: {err}"); + assert!(err.contains("never emitted [3]"), "got: {err}"); +} + +/// A hole with no compensating duplicate cannot keep the count, so it surfaces +/// as a count mismatch — the half `assert_complete` used to cover. Worth +/// pinning: it is the reason a gap alone is the *easy* failure, and why H1 +/// (which pairs a gap with a duplicate and so keeps the count) is the hard one. +#[test] +fn a_gap_without_a_duplicate_shows_up_as_a_count_mismatch() { + let meta = vec![base(0), base(2), base(3), base(4)]; + let err = check_dense_index_set(&meta, 5).expect_err("1 never emitted"); + assert!( + err.contains("emitted 4 constraints, declared 5"), + "got: {err}" + ); +} + +/// Wrong total is reported as wrong total, not as a confusing index list. +#[test] +fn a_count_mismatch_is_reported_plainly() { + let meta = vec![base(0), base(1)]; + let err = check_dense_index_set(&meta, 3).expect_err("2 != 3"); + assert!( + err.contains("emitted 2 constraints, declared 3"), + "got: {err}" + ); +} + +/// The honest control: a dense set passes, including the empty one. +#[test] +fn a_dense_set_passes() { + check_dense_index_set(&[], 0).expect("the empty body is dense"); + let meta: Vec<_> = (0..64).map(base).collect(); + check_dense_index_set(&meta, 64).expect("0..64 with no repeats is dense"); +} diff --git a/crypto/stark/src/tests/device_cap_tests.rs b/crypto/stark/src/tests/device_cap_tests.rs new file mode 100644 index 000000000..2a1f7a249 --- /dev/null +++ b/crypto/stark/src/tests/device_cap_tests.rs @@ -0,0 +1,150 @@ +//! A thread's device cap ([`crate::prover::cap_device_bytes`]): a prove under a +//! cap below a table's device set is refused by name before its first commit, +//! and a cap every table fits under changes no byte of the proof. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField, goldilocks::GoldilocksField, +}; + +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::MultiProof; +use crate::prover::{IsStarkProver, Prover, ProvingError, cap_device_bytes, device_cap}; +use crate::residency_mode::ResidencyMode; +use crate::traits::AIR; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; + +fn options() -> ProofOptions { + ProofOptions { + grinding_factor: 0, + ..ProofOptions::default_test_options() + } +} + +/// The CPU/ADD/MUL instance proved on `transcript`, under this thread's cap. +fn prove_on(transcript: &mut DefaultTranscript) -> Result, ProvingError> { + let (mut cpu, mut add, mut mul) = super::residency_mode_tests::traces(); + let opts = options(); + let (cpu_air, add_air, mul_air) = ( + new_cpu_air_with_lookup(&opts), + new_add_air_with_lookup(&opts), + new_mul_air_with_lookup(&opts), + ); + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu, &()), + (&add_air, &mut add, &()), + (&mul_air, &mut mul, &()), + ]; + Prover::multi_prove(pairs, transcript, ResidencyMode::Retain) +} + +/// ★ Under a cap below every table's device set the prove is refused, naming +/// the first table in index order with its bytes and the cap, and the +/// transcript has absorbed nothing: no root was committed. Without the +/// pre-check the gate admits an oversized table alone and the prove succeeds, +/// so this test is red with it removed. +#[test] +fn a_prove_under_a_cap_below_a_table_is_refused_before_its_first_commit() { + let mut transcript = DefaultTranscript::::new(&[]); + let before = transcript.state(); + let refused = { + let _cap = cap_device_bytes(1); + prove_on(&mut transcript) + }; + match refused { + Err(ProvingError::OverDeviceCap { table, bytes, cap }) => { + let cpu = new_cpu_air_with_lookup(&options()); + assert_eq!(table, cpu.name(), "the first table over the cap is named"); + assert_eq!(cap, 1); + assert!(bytes > 1, "its device set is the bytes reported: {bytes}"); + } + other => panic!( + "a prove under a one-byte cap must be refused: {:?}", + other.err() + ), + } + assert_eq!( + transcript.state(), + before, + "refused before any commit: the transcript absorbed nothing" + ); + assert_eq!(device_cap(), None, "the guard put the uncapped state back"); +} + +/// A cap every table fits under proves the same bytes as no cap. +#[test] +fn a_cap_every_table_fits_under_changes_no_byte() { + let open = prove_on(&mut DefaultTranscript::::new(&[])).expect("proves uncapped"); + let capped = { + let _cap = cap_device_bytes(u64::MAX / 2); + prove_on(&mut DefaultTranscript::::new(&[])).expect("proves under a wide cap") + }; + let bytes = |p: &MultiProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + assert_eq!(bytes(&open), bytes(&capped)); +} + +/// The guard is the thread's and nests: the inner cap is undone, the outer +/// one kept, and another thread never sees either. +#[test] +fn the_cap_is_the_threads_and_restored_by_its_guard() { + assert_eq!(device_cap(), None); + { + let _outer = cap_device_bytes(10 << 30); + { + let _inner = cap_device_bytes(1 << 30); + assert_eq!(device_cap(), Some(1 << 30)); + } + assert_eq!(device_cap(), Some(10 << 30)); + let elsewhere = std::thread::spawn(device_cap) + .join() + .expect("a plain thread"); + assert_eq!(elsewhere, None, "the cap is this thread's alone"); + } + assert_eq!(device_cap(), None); +} + +/// ★ The per-table drivers draw their streams in their spawner's class, so a +/// proof marked background stays on the background pool on every table's +/// thread. Thread-local state only: no card is touched. Red if `run_admitted` +/// stops copying the class into its workers. +#[cfg(feature = "cuda")] +#[test] +fn the_table_drivers_take_their_spawners_stream_class() { + use math_cuda::stream_class::{StreamClass, current, enter}; + let classes = |spawner: StreamClass| -> Vec { + let _class = enter(spawner); + let gate_budget = u64::MAX; + let order: Vec = (0..6).collect(); + let estimates = vec![1u64; 6]; + crate::prover::run_admitted_for_tests(&order, &estimates, gate_budget, 3, |_| current()) + .into_iter() + .map(|c| c.expect("every slot is filled")) + .collect() + }; + assert!( + classes(StreamClass::Background) + .iter() + .all(|&c| c == StreamClass::Background), + "a background prove's drivers are background" + ); + assert!( + classes(StreamClass::Foreground) + .iter() + .all(|&c| c == StreamClass::Foreground), + "a foreground prove's drivers are foreground" + ); +} diff --git a/crypto/stark/src/tests/fri_group_tests.rs b/crypto/stark/src/tests/fri_group_tests.rs new file mode 100644 index 000000000..b0ac56418 --- /dev/null +++ b/crypto/stark/src/tests/fri_group_tests.rs @@ -0,0 +1,618 @@ +//! S3 (group-leaf FRI layers) on the CPU prover and host verifier: the +//! round trips U4–U6, the tamper tests T1–T3, the load-bearing mutations M1–M2 and the +//! differential of the group path at the all-ones schedule against the legacy +//! path. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::fft::bit_reversing::{in_place_bit_reverse_permute, reverse_index}; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsFFTField; +use math::polynomial::Polynomial; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; + +use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; +use crate::fri::fri_functions::compute_coset_twiddles_inv; +use crate::fri::group::{ + GROUP_MUTATION, GroupMutation, group_fold, roots_of_unity_table, verify_query_groups, +}; +use crate::fri::terminal::{FriFoldLayout, terminal_codeword_from_coeffs}; +use crate::fri::{commit_phase_with_layout, fold_times, query_phase_with_layout}; +use crate::merkle_caps::TreeCheck; +use crate::proof::options::{FriMode, FriScheduleOverride, ProofFormat}; +use crate::traits::AIR; + +use super::zf_golden_tests::{ + fingerprint, golden_options, prove_logup, prove_multi, prove_simple_addition, verify_logup, + verify_multi, verify_simple_addition, +}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; +type Ext = FieldElement; + +fn rand_ext(rng: &mut ChaCha20Rng) -> Ext { + Ext::new([ + Felt::from(rng.r#gen::()), + Felt::from(rng.r#gen::()), + Felt::from(rng.r#gen::()), + ]) +} + +/// The point at position `p` of a bit-reversed layer of length `2^b` on the +/// coset `o·⟨ω_{2^b}⟩`. +fn point(o: &Felt, b: u32, p: usize) -> Felt { + let w = F::get_primitive_root_of_unity(u64::from(b)).unwrap(); + o * w.pow(reverse_index(p, 1u64 << b) as u64) +} + +/// A bit-reversed coset codeword of a random ext3 polynomial with `num_coeffs` +/// coefficients over `2^b` points; returns (codeword, coefficients). +fn random_codeword( + rng: &mut ChaCha20Rng, + b: u32, + num_coeffs: usize, + o: &Felt, +) -> (Vec, Vec) { + let coeffs: Vec = (0..num_coeffs).map(|_| rand_ext(rng)).collect(); + let poly = Polynomial::new(&coeffs); + let n = 1usize << b; + let mut cw = Polynomial::evaluate_offset_fft::( + &poly, + n / num_coeffs.next_power_of_two(), + Some(num_coeffs), + o, + ) + .expect("fft"); + assert_eq!(cw.len(), n); + in_place_bit_reverse_permute(&mut cw); + (cw, coeffs) +} + +fn dp_with(schedule: Option<&[u8]>) -> ProofFormat { + ProofFormat { + fri_mode: FriMode::Dp, + fri_schedule_override: schedule.map(|s| FriScheduleOverride::new(s).unwrap()), + ..ProofFormat::DEFAULT + } +} + +// --------------------------------------------------------------------------- +// U5: a group leaf is a coset, and its base folds to the next layer's point. +// --------------------------------------------------------------------------- + +#[test] +fn group_leaf_is_a_coset() { + let o = Felt::from(3u64); + for b in 1..=10u32 { + for d in 1..=b.min(6) { + let roots = roots_of_unity_table::(d).unwrap(); + // The table's root is the layer domain's ω_{2^b}^{2^{b−d}}. + let w_b = F::get_primitive_root_of_unity(u64::from(b)).unwrap(); + assert_eq!(roots[1], w_b.pow(1u64 << (b - d)), "b={b} d={d}"); + let o_next = o.pow(1u64 << d); + for g in 0..(1usize << (b - d)) { + let x_g = &o * w_b.pow(reverse_index(g, 1u64 << (b - d)) as u64); + for t in 0..(1usize << d) { + let want = &x_g * &roots[reverse_index(t, 1u64 << d)]; + assert_eq!(point(&o, b, (g << d) + t), want, "b={b} d={d} g={g} t={t}"); + } + // x_g^{2^d} is position g of the layer folded d times. + assert_eq!( + x_g.pow(1u64 << d), + point(&o_next, b - d, g), + "b={b} d={d} g={g}" + ); + } + } + } +} + +// --------------------------------------------------------------------------- +// U4: the group fold = d binary folds with ζ, ζ², … = 2^d·Σ ζ^i f_i. +// --------------------------------------------------------------------------- + +#[test] +fn group_fold_equals_d_binary_folds() { + let mut rng = ChaCha20Rng::seed_from_u64(0x5334); + let o = Felt::from(3u64); + let b = 9u32; + let n = 1usize << b; + let (codeword, coeffs) = random_codeword(&mut rng, b, n, &o); + for d in 1..=6u32 { + let zeta = rand_ext(&mut rng); + // Prover: d binary folds with ζ^{2^ℓ} (`fold_times`, the commit loop's). + let mut folded = codeword.clone(); + let mut tw = compute_coset_twiddles_inv::(&o, n); + fold_times(&mut folded, &zeta, d, &mut tw); + assert_eq!(folded.len(), n >> d); + + let roots = roots_of_unity_table::(d).unwrap(); + let o_next = o.pow(1u64 << d); + let two_d = Felt::from(1u64 << d); + for g in 0..(n >> d) { + let group = &codeword[g << d..(g + 1) << d]; + // Verifier: the group fold from ANY slot's point gives the same value. + for s in 0..(1usize << d) { + let y_inv = point(&o, b, (g << d) + s).inv().unwrap(); + let x_g_inv = &y_inv * &roots[reverse_index(s, 1u64 << d)]; + assert_eq!( + group_fold::(group, &zeta, &x_g_inv, &roots), + folded[g], + "d={d} g={g} slot={s}" + ); + } + // The polynomial identity: 2^d · Σ_i ζ^i f_i(Y), f(X) = Σ X^i f_i(X^{2^d}). + let y = point(&o_next, b - d, g); + let mut acc = Ext::zero(); + let mut zp = Ext::one(); + for i in 0..(1usize << d) { + let mut fi = Ext::zero(); + let mut yp = Felt::one(); + for k in (i..n).step_by(1 << d) { + fi += &yp * &coeffs[k]; + yp = &yp * &y; + } + acc += &zp * &fi; + zp = &zp * ζ + } + assert_eq!(folded[g], &two_d * &acc, "d={d} g={g}: 2^d·Σζ^i f_i"); + } + } +} + +// --------------------------------------------------------------------------- +// FRI-level harness (M1, M2): commit a codeword, open queries, verify with the +// same group checks the host verifier runs. +// --------------------------------------------------------------------------- + +struct FriRun { + layout: FriFoldLayout, + lde_log: u32, + roots: Vec<[u8; 32]>, + zetas: Vec, + coeffs: Vec, + decommitments: Vec>, + iotas: Vec, + terminal_offset: Felt, +} + +/// FRI over `committed` (what the prover commits); the transcript is replayed +/// to recover ζ. `lde_log` 10, blowup 4 (log 2), k 1: the chain covers 9 → 3, +/// schedule `[3, 1, 2]`. +fn fri_run(committed: &[Ext], o: &Felt) -> FriRun { + let lde_log = 10u32; + let n = 1usize << lde_log; + assert_eq!(committed.len(), n); + let layout = FriFoldLayout::from_schedule(lde_log, 2, 1, false, vec![3, 1, 2]).unwrap(); + assert!(!layout.is_legacy()); + let tw = compute_coset_twiddles_inv::(o, n); + let mut transcript = DefaultTranscript::::new(&[7]); + let (coeffs, layers) = commit_phase_with_layout::( + committed.to_vec(), + &mut transcript, + o, + n, + 2, + 1, + &layout, + &tw, + ); + let roots: Vec<[u8; 32]> = layers.iter().map(|l| l.merkle_tree.root).collect(); + let mut replay = DefaultTranscript::::new(&[7]); + let mut zetas = Vec::new(); + for r in &roots { + zetas.push(replay.sample_field_element()); + replay.append_bytes(r); + } + zetas.push(replay.sample_field_element()); + let iotas: Vec = (0..n / 2).step_by(37).collect(); + let decommitments = query_phase_with_layout::(&layers, &iotas, &layout); + FriRun { + terminal_offset: o.pow(1u64 << layout.total_folds), + layout, + lde_log, + roots, + zetas, + coeffs, + decommitments, + iotas, + } +} + +/// Verify every query of `run` with DEEP values read from `deep` (the +/// codeword the VERIFIER believes in, bit-reversed). +fn fri_accepts(run: &FriRun, deep: &[Ext], o: &Felt) -> bool { + let terminal = terminal_codeword_from_coeffs::( + &run.coeffs, + &run.terminal_offset, + run.layout.terminal_len, + ); + let tables: Vec> = (0..=6) + .map(|d| roots_of_unity_table::(d).unwrap()) + .collect(); + // One uncapped check per layer tree, at the layout's group-tree depth. + let checks: Vec> = run + .roots + .iter() + .enumerate() + .map(|(j, root)| { + let depth = run.layout.layer_depth(run.lde_log, j) as usize; + TreeCheck::build::>(root, depth, 0, || None).unwrap() + }) + .collect(); + run.iotas + .iter() + .zip(&run.decommitments) + .all(|(&iota, dec)| { + let x = point(o, run.lde_log, 2 * iota); + let x_inv = x.inv().unwrap(); + let (p0, p0s) = (&deep[2 * iota], &deep[2 * iota + 1]); + let v = (p0 + p0s) + &x_inv * &run.zetas[0] * (p0 - p0s); + verify_query_groups::>( + &run.layout, + &checks, + 0, + |j| dec.layers_auth_paths[j].merkle_path.as_slice(), + &dec.layers_evaluations_sym, + &run.zetas, + iota, + v, + x_inv.square(), + &terminal, + &tables, + ) + }) +} + +fn with_mutation(m: GroupMutation, f: impl FnOnce() -> T) -> T { + GROUP_MUTATION.with(|c| c.set(m)); + let out = f(); + GROUP_MUTATION.with(|c| c.set(GroupMutation::None)); + out +} + +/// A low-degree ext3 codeword on LDE 2^10, blowup 4 (256 coefficients). +fn low_degree(seed: u64, o: &Felt) -> Vec { + let mut rng = ChaCha20Rng::seed_from_u64(seed); + random_codeword(&mut rng, 10, 256, o).0 +} + +#[test] +fn honest_fri_run_is_accepted() { + let o = Felt::from(3u64); + let p0 = low_degree(1, &o); + let run = fri_run::(&p0, &o); + assert_eq!(run.roots.len(), 3); + assert!(fri_accepts::(&run, &p0, &o)); + let run = fri_run::(&p0, &o); + assert!(fri_accepts::(&run, &p0, &o)); +} + +/// M1 — the slot check is load-bearing. A prover commits FRI for +/// `p₀ + c` (still low degree, so every layer and the terminal are +/// consistent) while the trace openings say `p₀`: only `group[slot] == v` at +/// the first committed layer ties FRI to the DEEP value. With it the forgery +/// is rejected; with it skipped (the mutation) it is ACCEPTED. +#[test] +fn m1_the_slot_check_is_load_bearing() { + let o = Felt::from(3u64); + let p0 = low_degree(2, &o); + let c = Ext::new([Felt::from(5u64), Felt::from(6u64), Felt::from(7u64)]); + let shifted: Vec = p0.iter().map(|v| v + &c).collect(); + let run = fri_run::(&shifted, &o); + assert!( + fri_accepts::(&run, &shifted, &o), + "control: FRI of p0 + c is honest for p0 + c" + ); + assert!( + !fri_accepts::(&run, &p0, &o), + "the slot check must reject" + ); + assert!( + with_mutation(GroupMutation::SkipSlotCheck, || fri_accepts::< + KeccakStarkHash, + >(&run, &p0, &o)), + "without the slot check the forgery is accepted (the check is load-bearing)" + ); +} + +/// M2 — the group's Merkle authentication is load-bearing. Replacing a layer +/// root (with the challenges kept) leaves the fold chain consistent; only the +/// authentication of the group against the root rejects it. +#[test] +fn m2_the_group_authentication_is_load_bearing() { + let o = Felt::from(3u64); + let p0 = low_degree(3, &o); + let mut run = fri_run::(&p0, &o); + run.roots[1] = [0xAB; 32]; + assert!( + !fri_accepts::(&run, &p0, &o), + "authentication must reject" + ); + assert!( + with_mutation( + GroupMutation::SkipLeafAuth, + || fri_accepts::(&run, &p0, &o) + ), + "without authentication the foreign root is accepted (the check is load-bearing)" + ); +} + +// --------------------------------------------------------------------------- +// U6: prove / verify round trips at fri = dp. +// --------------------------------------------------------------------------- + +/// SimpleAddition at `rows`, blowup `blowup`, k = 1, under `format`: returns +/// the proof's committed-layer count and the values per query, after asserting +/// it verifies. +fn round_trip_simple(rows: usize, blowup: u8, format: ProofFormat) -> (usize, usize) { + let o = golden_options(blowup, 1, 9, format); + let (air, proof) = prove_simple_addition::(rows, &o); + assert!( + verify_simple_addition::(&air, &proof), + "rows {rows} blowup {blowup} {format:?}: an honest proof must verify" + ); + let layers = proof.fri_layers_merkle_roots.len(); + let values = proof.query_list[0].layers_evaluations_sym.len(); + for q in &proof.query_list { + assert_eq!(q.layers_auth_paths.len(), layers); + assert_eq!(q.layers_evaluations_sym.len(), values); + } + (layers, values) +} + +#[test] +fn dp_round_trips_at_every_fold_count() { + // k = 1: total_folds = log2(rows) + blowup_log − (blowup_log + 1). + for blowup in [2u8, 4] { + for log_rows in 1..=10u32 { + let rows = 1usize << log_rows; + let (layers, values) = + round_trip_simple::(rows, blowup, dp_with(None)); + let lde_log = log_rows + blowup.trailing_zeros(); + let o = golden_options(blowup, 1, 9, dp_with(None)); + let l = + FriFoldLayout::for_options(lde_log, blowup.trailing_zeros(), &o, false).unwrap(); + assert_eq!(layers, l.num_committed, "rows {rows}"); + assert_eq!(values, l.opened_values_per_query(), "rows {rows}"); + } + } + // A shape where the DP picks a non-trivial schedule is exercised. + let o = golden_options(4, 1, 9, dp_with(None)); + let l = FriFoldLayout::for_options(12, 2, &o, false).unwrap(); + assert!( + l.schedule.iter().any(|&d| d > 1), + "schedule {:?}", + l.schedule + ); +} + +#[test] +fn dp_round_trips_under_explicit_schedules() { + // rows 2^9, blowup 4, k 1: lde_log 11, chain from 10 to T = 3: 7 bits. + for sched in [ + &[1u8, 3, 3][..], + &[3, 1, 3], + &[2, 1, 2, 2], + &[1, 1, 1, 1, 1, 1, 1], + &[6, 1], + &[1, 6], + &[4, 3], + ] { + for blake in [false, true] { + let (layers, values) = if blake { + round_trip_simple::(512, 4, dp_with(Some(sched))) + } else { + round_trip_simple::(512, 4, dp_with(Some(sched))) + }; + assert_eq!(layers, sched.len(), "{sched:?}"); + assert_eq!( + values, + sched.iter().map(|&d| 1usize << d).sum::(), + "{sched:?}" + ); + } + } + // An override that does not fit is a proving error, never a fallback. + let o = golden_options(4, 1, 9, dp_with(Some(&[3, 1]))); + let air = crate::examples::simple_addition::SimpleAdditionAIR::::new(&o); + let mut trace = crate::examples::simple_addition::simple_addition_trace::(512); + let pi = crate::examples::simple_addition::SimpleAdditionPublicInputs { + a: Felt::from(1u64), + b: Felt::from(2u64), + }; + use crate::prover::IsStarkProver; + let res = crate::prover::GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ); + assert!( + res.is_err(), + "a schedule that does not cover the folds must be refused" + ); +} + +#[test] +fn dp_round_trips_ext3_aux_and_multi_table() { + for (rows, blowup) in [(16usize, 2u8), (128, 4), (512, 2)] { + let lde_log = rows.trailing_zeros() + blowup.trailing_zeros(); + // Committed chain: from lde_log − 1 down to T = blowup_log + k (k = 1). + let span = (lde_log - 1 - (blowup.trailing_zeros() + 1)) as u8; + // An uneven explicit schedule where there is room for one. + let explicit = if span >= 3 { + vec![2u8, span - 2] + } else { + vec![span] + }; + for format in [dp_with(None), dp_with(Some(&explicit))] { + let o = golden_options(blowup, 1, 7, format); + let (air, proof, _) = prove_logup::(rows, &o); + assert!( + verify_logup::(&air, &proof), + "logup rows {rows} {format:?}" + ); + let (air, proof, _) = prove_logup::(rows, &o); + assert!( + verify_logup::(&air, &proof), + "logup keccak rows {rows}" + ); + } + } + let o = golden_options(2, 1, 6, dp_with(None)); + let multi = prove_multi::(&o); + assert!(verify_multi::(&o, &multi)); + assert!( + multi + .proofs + .iter() + .any(|p| !p.fri_layers_merkle_roots.is_empty()) + ); +} + +/// The format is a verifier-side constant: a dp proof does not verify under +/// pair options, nor a pair proof under dp options. +#[test] +fn the_format_is_a_verifier_constant() { + let dp = golden_options(4, 1, 9, dp_with(None)); + let pair = golden_options(4, 1, 9, ProofFormat::DEFAULT); + let (_, dp_proof) = prove_simple_addition::(1024, &dp); + let (_, pair_proof) = prove_simple_addition::(1024, &pair); + let dp_air = crate::examples::simple_addition::SimpleAdditionAIR::::new(&dp); + let pair_air = crate::examples::simple_addition::SimpleAdditionAIR::::new(&pair); + assert!(verify_simple_addition::( + &dp_air, &dp_proof + )); + assert!(verify_simple_addition::( + &pair_air, + &pair_proof + )); + assert!(!verify_simple_addition::( + &pair_air, &dp_proof + )); + assert!(!verify_simple_addition::( + &dp_air, + &pair_proof + )); +} + +// --------------------------------------------------------------------------- +// The group path at the all-ones schedule vs the legacy path. +// --------------------------------------------------------------------------- + +/// Proving under `dp` with an all-ones schedule runs the GROUP code path (group +/// trees via `H::Batched`, full-group encoding, the group verifier) where the +/// legacy format runs the pair path. Every root, the terminal polynomial, every +/// trace/composition opening and every FRI path must be identical (so ζ and ι +/// are too), and each two-value group must be exactly the legacy pair: the +/// legacy sibling is the group entry that is not the query's own value. +#[test] +fn generic_path_at_all_ones_equals_legacy() { + for blowup in [2u8, 4] { + let rows = 256usize; + let lde_log = rows.trailing_zeros() + blowup.trailing_zeros(); + let span = (lde_log - 1 - (blowup.trailing_zeros() + 1)) as usize; + let ones = vec![1u8; span]; + let legacy_o = golden_options(blowup, 1, 9, ProofFormat::DEFAULT); + let group_o = golden_options(blowup, 1, 9, dp_with(Some(&ones))); + let (_, legacy, _) = prove_logup::(rows, &legacy_o); + let (air, group, _) = prove_logup::(rows, &group_o); + assert!(verify_logup::(&air, &group)); + + assert_eq!( + legacy.fri_layers_merkle_roots, + group.fri_layers_merkle_roots + ); + assert_eq!(legacy.fri_final_poly_coeffs, group.fri_final_poly_coeffs); + let a = fingerprint!(&legacy); + let b = fingerprint!(&group); + assert_eq!( + a.openings, b.openings, + "trace/composition openings (so every ι) equal" + ); + assert_eq!(a.fri_roots, b.fri_roots); + assert_ne!( + a.proof, b.proof, + "the encodings differ (full groups vs siblings)" + ); + for (lq, gq) in legacy.query_list.iter().zip(&group.query_list) { + assert_eq!(lq.layers_auth_paths.len(), span); + for j in 0..span { + assert_eq!( + lq.layers_auth_paths[j].merkle_path, gq.layers_auth_paths[j].merkle_path, + "layer {j}: same leaf, same path" + ); + let pair = &gq.layers_evaluations_sym[2 * j..2 * j + 2]; + let sym = &lq.layers_evaluations_sym[j]; + assert!( + pair.contains(sym), + "layer {j}: the legacy sibling is in the group" + ); + } + } + } +} + +// --------------------------------------------------------------------------- +// T1–T3: tamper tests on a dp proof with non-trivial groups. +// --------------------------------------------------------------------------- + +#[test] +fn tampering_any_fri_value_path_or_root_is_rejected() { + let format = dp_with(Some(&[3, 2, 2])); + let o = golden_options(4, 1, 5, format); + let (air, honest, _) = prove_logup::(512, &o); + assert!(verify_logup::(&air, &honest)); + let values = honest.query_list[0].layers_evaluations_sym.len(); + assert_eq!(values, 8 + 4 + 4); + let bump = Ext::new([Felt::one(), Felt::zero(), Felt::zero()]); + + // T1/T2: every value of query 0's groups (the slot value and every other). + for i in 0..values { + let mut p = honest.clone(); + p.query_list[0].layers_evaluations_sym[i] += bump; + assert!( + !verify_logup::(&air, &p), + "value {i} tampered" + ); + } + // A value of the LAST query too. + let last = honest.query_list.len() - 1; + let mut p = honest.clone(); + p.query_list[last].layers_evaluations_sym[values - 1] += bump; + assert!(!verify_logup::(&air, &p)); + // One path sibling per layer. + for j in 0..3 { + let mut p = honest.clone(); + p.query_list[0].layers_auth_paths[j].merkle_path[0][0] ^= 1; + assert!(!verify_logup::(&air, &p), "layer {j} path"); + } + // A layer root. + for j in 0..3 { + let mut p = honest.clone(); + p.fri_layers_merkle_roots[j][5] ^= 1; + assert!(!verify_logup::(&air, &p), "layer {j} root"); + } + // T3: the flat value vector one short / one long (checked before the loop, + // so neither panics). + let mut p = honest.clone(); + p.query_list[0].layers_evaluations_sym.pop(); + assert!(!verify_logup::(&air, &p)); + let mut p = honest.clone(); + p.query_list[0].layers_evaluations_sym.push(Ext::zero()); + assert!(!verify_logup::(&air, &p)); + // A path of the wrong length (the exact depth is a verifier constant). + let mut p = honest.clone(); + p.query_list[0].layers_auth_paths[1].merkle_path.pop(); + assert!(!verify_logup::(&air, &p)); + // One layer too few. + let mut p = honest.clone(); + p.fri_layers_merkle_roots.pop(); + assert!(!verify_logup::(&air, &p)); +} diff --git a/crypto/stark/src/tests/fri_schedule_tests.rs b/crypto/stark/src/tests/fri_schedule_tests.rs new file mode 100644 index 000000000..a29a7aa0a --- /dev/null +++ b/crypto/stark/src/tests/fri_schedule_tests.rs @@ -0,0 +1,1298 @@ +//! Tests for the FRI fold schedule (`crate::fri::schedule`) and the generalised +//! `FriFoldLayout` (U1–U3). +//! +//! Two objectives appear here. The PRODUCTION one is the cost law +//! (`FRI_COST_WEIGHTS`): U1 pins its schedules as the Rust DP computes them, U2 +//! checks it against brute force. The design model's PERMUTATION objective +//! (Merkle permutations per layer only) is kept as a second instance of the generic DP +//! (`fri_schedule_by`), pinned against an independently computed table: it shows the DP +//! machinery reproduces an independent model exactly, and documents how far +//! the two objectives' schedules differ. + +use crate::fri::schedule::{ + BALU_ROW_NS, FRI_COST_WEIGHTS, FRI_FOLD_XALU_ROWS, FRI_SCHEDULE_DMAX, FriFormat, + FriFormatError, XALU_ROW_NS, fri_chain_start, fri_group_layer_rows, fri_layer_cost_q, + fri_leaf_blocks, fri_pair_layer_cost_q, fri_pair_layer_rows, fri_schedule, fri_schedule_by, + fri_schedule_cost_by, fri_schedule_cost_q, fri_schedule_with_cost, legacy_fri_schedule, +}; +use crate::fri::terminal::FriFoldLayout; +use crate::proof::options::{ + CapPolicy, FriMode, FriScheduleOverride, OneRowMode, ProofFormat, ProofOptions, +}; +use crypto::merkle_tree::cap::{AUTO_WEIGHTS, cap_gain}; + +const Q: u64 = 110; + +// --------------------------------------------------------------------------- +// Cap-height functions for the permutation objective. +// --------------------------------------------------------------------------- + +/// A cap-height function that caps nothing. +fn no_cap(_depth: u32) -> u32 { + 0 +} + +/// The cap rule the design model's table was computed with: +/// `c = argmax_{0 ≤ c ≤ depth} (Q·c − (2^c − 1))`, ties to the smaller `c`. +fn cap_design_model(depth: u32) -> u32 { + let (mut best, mut best_c) = (0i64, 0u32); + for c in 0..=depth.min(62) { + let v = Q as i64 * i64::from(c) - ((1i64 << c) - 1); + if v > best { + (best, best_c) = (v, c); + } + } + best_c +} + +/// The adopted policy: every FRI tree is opened once per query. +fn cap_auto(depth: u32) -> u32 { + CapPolicy::Auto.height(Q as usize, depth as usize) as u32 +} + +#[test] +fn cap_auto_heights_match_cap_md() { + // The `CapPolicy::Auto` table, at a depth large enough not to clamp. + for (openings, want) in [(1, 0), (3, 0), (4, 2), (19, 2), (20, 3), (110, 3), (224, 3)] { + assert_eq!( + CapPolicy::Auto.height(openings, 20), + want, + "openings {openings}" + ); + } + // Clamped to depth. + for depth in 0..8 { + assert_eq!(cap_auto(depth), depth.min(3), "depth {depth}"); + } + // The design model's rule reaches 7 at Q = 110. + assert_eq!(cap_design_model(20), 7); + assert_eq!(cap_design_model(5), 5); +} + +/// The design model's per-layer cost, `Q ×` permutations: `Q·leaf(d) + Q·(depth − +/// c) + 2^c − 1`. +fn perm_layer_q(d: u32, depth: u32, q: u64, cap: &dyn Fn(u32) -> u32) -> u64 { + let c = cap(depth).min(depth); + q * fri_leaf_blocks(d) + q * u64::from(depth - c) + (1u64 << c) - 1 +} + +fn perm_schedule( + b0: u32, + t: u32, + cap: &dyn Fn(u32) -> u32, +) -> crate::fri::schedule::FriScheduleChoice { + fri_schedule_by(b0, t, FRI_SCHEDULE_DMAX, &|d, depth| { + perm_layer_q(d, depth, Q, cap) + }) +} + +fn perm_cost(b0: u32, schedule: &[u8], cap: &dyn Fn(u32) -> u32) -> Option { + fri_schedule_cost_by(b0, schedule, &|d, depth| perm_layer_q(d, depth, Q, cap)) +} + +// --------------------------------------------------------------------------- +// Cost-model primitives. +// --------------------------------------------------------------------------- + +#[test] +fn leaf_blocks() { + // ⌈3·2^d / 8⌉, at least 1. + let want = [1u64, 1, 2, 3, 6, 12, 24]; + for (d, w) in want.iter().enumerate() { + assert_eq!(fri_leaf_blocks(d as u32), *w, "d = {d}"); + } +} + +/// The objective's weights are a format constant: the cap +/// policy's weights plus the in-guest XALU and BALU row prices (a fold is 5 +/// XALU rows, a twiddle one BALU row). +#[test] +fn cost_weights_are_pinned() { + assert_eq!(FRI_COST_WEIGHTS.cap, AUTO_WEIGHTS); + assert_eq!( + ( + FRI_COST_WEIGHTS.cap.compress, + FRI_COST_WEIGHTS.cap.select, + FRI_COST_WEIGHTS.cap.unpack, + FRI_COST_WEIGHTS.cap.hint, + FRI_COST_WEIGHTS.cap.compare + ), + (2251, 567, 528, 460, 3789) + ); + assert_eq!( + (FRI_FOLD_XALU_ROWS, XALU_ROW_NS, BALU_ROW_NS), + (5, 522, 477) + ); + assert_eq!( + (FRI_COST_WEIGHTS.fold, FRI_COST_WEIGHTS.twiddle), + (2610, 477) + ); + assert_eq!( + (FRI_COST_WEIGHTS.xalu, FRI_COST_WEIGHTS.balu), + (XALU_ROW_NS, BALU_ROW_NS) + ); + assert_eq!( + FRI_COST_WEIGHTS.fold, + FRI_FOLD_XALU_ROWS * FRI_COST_WEIGHTS.xalu + ); + assert_eq!(FRI_COST_WEIGHTS.twiddle, FRI_COST_WEIGHTS.balu); + // The node cost law, 421 ns/instruction + 5.63 ns/cell, at the committed + // widths (XALU 18, BALU 10 cells), rounded to the nearest ns. + assert_eq!(((421.0f64 + 5.63 * 18.0).round()) as u64, XALU_ROW_NS); + assert_eq!(((421.0f64 + 5.63 * 10.0).round()) as u64, BALU_ROW_NS); +} + +/// One layer's cost written out by hand (every emitted row). +#[test] +fn layer_cost_by_hand() { + // d = 3, depth 10, no cap: + // leaf 3·2251 + walk 10·(2251+567) + slot mux 7·567 + folds 7·2610 + // + twiddles 3·477 + x_g 3·(567+477) + scaling 1·522 + 1·477 + // + slot assert 2·522 + compare 8·477 + 528 + values 8·(528+460) + // + leaf packs 6·528 + siblings 10·460. + let per_query = 3 * 2251 + + 10 * (2251 + 567) + + 7 * 567 + + 7 * 2610 + + 3 * 477 + + 3 * (567 + 477) + + 522 + + 477 + + 2 * 522 + + 8 * 477 + + 528 + + 8 * (528 + 460) + + 6 * 528 + + 10 * 460; + assert_eq!( + fri_layer_cost_q(&FRI_COST_WEIGHTS, 3, 10, Q, CapPolicy::Off), + Q * per_query + ); + // Auto cap at Q = 110 is c = 3 on a 10-deep tree: minus its gain and the + // three sibling hints per query the capped path omits. + let gain = cap_gain(&AUTO_WEIGHTS, 110, 3); + assert!(gain > 0); + assert_eq!( + fri_layer_cost_q(&FRI_COST_WEIGHTS, 3, 10, Q, CapPolicy::Auto), + Q * per_query - gain as u64 - Q * 3 * 460 + ); + // d = 1 at depth 0 under the group encoding: leaf, mux select, fold, + // twiddle, x_g select + mul, slot assert, compare, 2 values, 2 packs. + assert_eq!( + fri_layer_cost_q(&FRI_COST_WEIGHTS, 1, 0, 1, CapPolicy::Off), + 2251 + 567 + + 2610 + + 477 + + (567 + 477) + + 2 * 522 + + (8 * 477 + 528) + + 2 * (528 + 460) + + 2 * 528 + ); + // Today's pair layer at depth 0: parity select, leaf, fold, squaring, + // compare, two unpacks, two packs and one hinted sibling value. + assert_eq!( + fri_pair_layer_cost_q(&FRI_COST_WEIGHTS, 0, 1, CapPolicy::Off), + 567 + 2251 + 2610 + 477 + (8 * 477 + 528) + 2 * 528 + 2 * 528 + 460 + ); +} + +/// The row model's kinds at `d = 1..=6`, written out (the in-guest tests pin +/// the same numbers against the emitter, `lfm::fri_group_tests`). +#[test] +fn group_layer_rows_by_hand() { + // (d, selects, XALU, BALU, hashes, unpacks, packs, hints) at depth 2, + // uncapped. + let want = [ + (1u32, 4u64, 7u64, 10u64, 3u64, 3u64, 2u64, 4u64), + (2, 7, 17, 13, 4, 5, 3, 6), + (3, 12, 38, 15, 5, 9, 6, 10), + (4, 21, 79, 17, 8, 17, 12, 18), + (5, 38, 160, 19, 14, 33, 24, 34), + (6, 71, 321, 21, 26, 65, 48, 66), + ]; + for (d, sel, xalu, balu, hashes, unpacks, packs, hints) in want { + let r = fri_group_layer_rows(d, 2, 0); + assert_eq!( + ( + r.selects, r.xalu, r.balu, r.hashes, r.unpacks, r.packs, r.hints + ), + (sel, xalu, balu, hashes, unpacks, packs, hints), + "d = {d}" + ); + } + // A cap of c on the same tree: c fewer walk levels and sibling hints, + // 2^c − 1 cap-mux selects, one more unpack. + let r = fri_group_layer_rows(3, 2, 2); + assert_eq!( + (r.selects, r.hashes, r.unpacks, r.hints), + (12 - 2 + 3, 5 - 2, 9 + 1, 10 - 2) + ); + // The cap height is clamped to the depth. + assert_eq!(fri_group_layer_rows(3, 2, 9), fri_group_layer_rows(3, 2, 2)); + let p = fri_pair_layer_rows(2, 0); + assert_eq!( + ( + p.selects, p.xalu, p.balu, p.hashes, p.unpacks, p.packs, p.hints + ), + (3, 5, 9, 3, 3, 2, 3) + ); +} + +#[test] +fn schedule_cost_rejects_malformed_schedules() { + let cap = CapPolicy::Off; + assert_eq!(fri_schedule_cost_q(10, &[], Q, cap), Some(0)); + assert_eq!(fri_schedule_cost_q(10, &[0, 1], Q, cap), None); + assert_eq!(fri_schedule_cost_q(3, &[2, 2], Q, cap), None); + assert!(fri_schedule_cost_q(4, &[2, 2], Q, cap).is_some()); +} + +// --------------------------------------------------------------------------- +// The design model (permutation objective): its schedule table, reproduced. +// --------------------------------------------------------------------------- + +/// (B, today, S3 from B−1, S2+S3 from B); each entry = (cost·Q, schedule). +/// Generated by an independent Python reproduction of the design model in exact +/// integer units, and cross-checked against a second independent +/// implementation for the OFF and MODEL caps (cost / 110). +type Row = ( + u32, + (u64, &'static [u8]), + (u64, &'static [u8]), + (u64, &'static [u8]), +); + +/// T = 9, cap = OFF: (B, today, S3, S2+S3), each (cost·Q, schedule). +const PIN_T9_CAP_OFF: &[Row] = &[ + (6, (0, &[]), (0, &[]), (0, &[])), + (7, (0, &[]), (0, &[]), (0, &[])), + (8, (0, &[]), (0, &[]), (0, &[])), + (9, (0, &[]), (0, &[]), (0, &[])), + (10, (0, &[]), (0, &[]), (1100, &[1])), + (11, (1100, &[1]), (1100, &[1]), (1210, &[2])), + (12, (2310, &[1, 1]), (1210, &[2]), (1320, &[3])), + (13, (3630, &[1, 1, 1]), (1320, &[3]), (1650, &[4])), + (14, (5060, &[1, 1, 1, 1]), (1650, &[4]), (2310, &[5])), + (15, (6600, &[1, 1, 1, 1, 1]), (2310, &[5]), (2970, &[3, 3])), + ( + 16, + (8250, &[1, 1, 1, 1, 1, 1]), + (2970, &[3, 3]), + (3300, &[4, 3]), + ), + ( + 17, + (10010, &[1, 1, 1, 1, 1, 1, 1]), + (3300, &[4, 3]), + (3740, &[4, 4]), + ), + ( + 18, + (11880, &[1, 1, 1, 1, 1, 1, 1, 1]), + (3740, &[4, 4]), + (4400, &[5, 4]), + ), + ( + 19, + (13860, &[1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4400, &[5, 4]), + (5170, &[5, 5]), + ), + ( + 20, + (15950, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5170, &[5, 5]), + (5720, &[4, 4, 3]), + ), + ( + 21, + (18150, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5720, &[4, 4, 3]), + (6270, &[4, 4, 4]), + ), + ( + 22, + (20460, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (6270, &[4, 4, 4]), + (6930, &[5, 4, 4]), + ), + ( + 23, + (22880, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (6930, &[5, 4, 4]), + (7700, &[5, 5, 4]), + ), + ( + 24, + (25410, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (7700, &[5, 5, 4]), + (8580, &[5, 5, 5]), + ), +]; +/// T = 10, cap = OFF: (B, today, S3, S2+S3), each (cost·Q, schedule). +const PIN_T10_CAP_OFF: &[Row] = &[ + (6, (0, &[]), (0, &[]), (0, &[])), + (7, (0, &[]), (0, &[]), (0, &[])), + (8, (0, &[]), (0, &[]), (0, &[])), + (9, (0, &[]), (0, &[]), (0, &[])), + (10, (0, &[]), (0, &[]), (0, &[])), + (11, (0, &[]), (0, &[]), (1210, &[1])), + (12, (1210, &[1]), (1210, &[1]), (1320, &[2])), + (13, (2530, &[1, 1]), (1320, &[2]), (1430, &[3])), + (14, (3960, &[1, 1, 1]), (1430, &[3]), (1760, &[4])), + (15, (5500, &[1, 1, 1, 1]), (1760, &[4]), (2420, &[5])), + (16, (7150, &[1, 1, 1, 1, 1]), (2420, &[5]), (3190, &[3, 3])), + ( + 17, + (8910, &[1, 1, 1, 1, 1, 1]), + (3190, &[3, 3]), + (3520, &[4, 3]), + ), + ( + 18, + (10780, &[1, 1, 1, 1, 1, 1, 1]), + (3520, &[4, 3]), + (3960, &[4, 4]), + ), + ( + 19, + (12760, &[1, 1, 1, 1, 1, 1, 1, 1]), + (3960, &[4, 4]), + (4620, &[5, 4]), + ), + ( + 20, + (14850, &[1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4620, &[5, 4]), + (5390, &[5, 5]), + ), + ( + 21, + (17050, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5390, &[5, 5]), + (6050, &[4, 4, 3]), + ), + ( + 22, + (19360, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (6050, &[4, 4, 3]), + (6600, &[4, 4, 4]), + ), + ( + 23, + (21780, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (6600, &[4, 4, 4]), + (7260, &[5, 4, 4]), + ), + ( + 24, + (24310, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (7260, &[5, 4, 4]), + (8030, &[5, 5, 4]), + ), +]; +/// T = 9, cap = MODEL: (B, today, S3, S2+S3), each (cost·Q, schedule). +const PIN_T9_CAP_MODEL: &[Row] = &[ + (6, (0, &[]), (0, &[]), (0, &[])), + (7, (0, &[]), (0, &[]), (0, &[])), + (8, (0, &[]), (0, &[]), (0, &[])), + (9, (0, &[]), (0, &[]), (0, &[])), + (10, (0, &[]), (0, &[]), (457, &[1])), + (11, (457, &[1]), (457, &[1]), (567, &[2])), + (12, (1024, &[1, 1]), (567, &[2]), (677, &[3])), + (13, (1701, &[1, 1, 1]), (677, &[3]), (1007, &[4])), + (14, (2488, &[1, 1, 1, 1]), (1007, &[4]), (1464, &[3, 2])), + ( + 15, + (3385, &[1, 1, 1, 1, 1]), + (1464, &[3, 2]), + (1684, &[3, 3]), + ), + ( + 16, + (4392, &[1, 1, 1, 1, 1, 1]), + (1684, &[3, 3]), + (2014, &[4, 3]), + ), + ( + 17, + (5509, &[1, 1, 1, 1, 1, 1, 1]), + (2014, &[4, 3]), + (2454, &[4, 4]), + ), + ( + 18, + (6736, &[1, 1, 1, 1, 1, 1, 1, 1]), + (2454, &[4, 4]), + (3021, &[3, 3, 3]), + ), + ( + 19, + (8073, &[1, 1, 1, 1, 1, 1, 1, 1, 1]), + (3021, &[3, 3, 3]), + (3351, &[4, 3, 3]), + ), + ( + 20, + (9520, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (3351, &[4, 3, 3]), + (3791, &[4, 4, 3]), + ), + ( + 21, + (11077, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (3791, &[4, 4, 3]), + (4341, &[4, 4, 4]), + ), + ( + 22, + (12744, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4341, &[4, 4, 4]), + (5001, &[5, 4, 4]), + ), + ( + 23, + (14521, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5001, &[5, 4, 4]), + (5458, &[4, 4, 3, 3]), + ), + ( + 24, + (16408, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5458, &[4, 4, 3, 3]), + (6008, &[4, 4, 4, 3]), + ), +]; +/// T = 10, cap = MODEL: (B, today, S3, S2+S3), each (cost·Q, schedule). +const PIN_T10_CAP_MODEL: &[Row] = &[ + (6, (0, &[]), (0, &[]), (0, &[])), + (7, (0, &[]), (0, &[]), (0, &[])), + (8, (0, &[]), (0, &[]), (0, &[])), + (9, (0, &[]), (0, &[]), (0, &[])), + (10, (0, &[]), (0, &[]), (0, &[])), + (11, (0, &[]), (0, &[]), (567, &[1])), + (12, (567, &[1]), (567, &[1]), (677, &[2])), + (13, (1244, &[1, 1]), (677, &[2]), (787, &[3])), + (14, (2031, &[1, 1, 1]), (787, &[3]), (1117, &[4])), + (15, (2928, &[1, 1, 1, 1]), (1117, &[4]), (1684, &[3, 2])), + ( + 16, + (3935, &[1, 1, 1, 1, 1]), + (1684, &[3, 2]), + (1904, &[3, 3]), + ), + ( + 17, + (5052, &[1, 1, 1, 1, 1, 1]), + (1904, &[3, 3]), + (2234, &[4, 3]), + ), + ( + 18, + (6279, &[1, 1, 1, 1, 1, 1, 1]), + (2234, &[4, 3]), + (2674, &[4, 4]), + ), + ( + 19, + (7616, &[1, 1, 1, 1, 1, 1, 1, 1]), + (2674, &[4, 4]), + (3334, &[5, 4]), + ), + ( + 20, + (9063, &[1, 1, 1, 1, 1, 1, 1, 1, 1]), + (3334, &[5, 4]), + (3681, &[4, 3, 3]), + ), + ( + 21, + (10620, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (3681, &[4, 3, 3]), + (4121, &[4, 4, 3]), + ), + ( + 22, + (12287, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4121, &[4, 4, 3]), + (4671, &[4, 4, 4]), + ), + ( + 23, + (14064, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4671, &[4, 4, 4]), + (5331, &[5, 4, 4]), + ), + ( + 24, + (15951, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5331, &[5, 4, 4]), + (5898, &[4, 4, 3, 3]), + ), +]; +/// T = 9, cap = AUTO: (B, today, S3, S2+S3), each (cost·Q, schedule). +const PIN_T9_CAP_AUTO: &[Row] = &[ + (6, (0, &[]), (0, &[]), (0, &[])), + (7, (0, &[]), (0, &[]), (0, &[])), + (8, (0, &[]), (0, &[]), (0, &[])), + (9, (0, &[]), (0, &[]), (0, &[])), + (10, (0, &[]), (0, &[]), (777, &[1])), + (11, (777, &[1]), (777, &[1]), (887, &[2])), + (12, (1664, &[1, 1]), (887, &[2]), (997, &[3])), + (13, (2661, &[1, 1, 1]), (997, &[3]), (1327, &[4])), + (14, (3768, &[1, 1, 1, 1]), (1327, &[4]), (1987, &[5])), + (15, (4985, &[1, 1, 1, 1, 1]), (1987, &[5]), (2324, &[3, 3])), + ( + 16, + (6312, &[1, 1, 1, 1, 1, 1]), + (2324, &[3, 3]), + (2654, &[4, 3]), + ), + ( + 17, + (7749, &[1, 1, 1, 1, 1, 1, 1]), + (2654, &[4, 3]), + (3094, &[4, 4]), + ), + ( + 18, + (9296, &[1, 1, 1, 1, 1, 1, 1, 1]), + (3094, &[4, 4]), + (3754, &[5, 4]), + ), + ( + 19, + (10953, &[1, 1, 1, 1, 1, 1, 1, 1, 1]), + (3754, &[5, 4]), + (4311, &[4, 3, 3]), + ), + ( + 20, + (12720, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4311, &[4, 3, 3]), + (4751, &[4, 4, 3]), + ), + ( + 21, + (14597, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4751, &[4, 4, 3]), + (5301, &[4, 4, 4]), + ), + ( + 22, + (16584, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5301, &[4, 4, 4]), + (5961, &[5, 4, 4]), + ), + ( + 23, + (18681, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5961, &[5, 4, 4]), + (6731, &[5, 5, 4]), + ), + ( + 24, + (20888, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (6731, &[5, 5, 4]), + (7288, &[4, 4, 4, 3]), + ), +]; +/// T = 10, cap = AUTO: (B, today, S3, S2+S3), each (cost·Q, schedule). +const PIN_T10_CAP_AUTO: &[Row] = &[ + (6, (0, &[]), (0, &[]), (0, &[])), + (7, (0, &[]), (0, &[]), (0, &[])), + (8, (0, &[]), (0, &[]), (0, &[])), + (9, (0, &[]), (0, &[]), (0, &[])), + (10, (0, &[]), (0, &[]), (0, &[])), + (11, (0, &[]), (0, &[]), (887, &[1])), + (12, (887, &[1]), (887, &[1]), (997, &[2])), + (13, (1884, &[1, 1]), (997, &[2]), (1107, &[3])), + (14, (2991, &[1, 1, 1]), (1107, &[3]), (1437, &[4])), + (15, (4208, &[1, 1, 1, 1]), (1437, &[4]), (2097, &[5])), + (16, (5535, &[1, 1, 1, 1, 1]), (2097, &[5]), (2544, &[3, 3])), + ( + 17, + (6972, &[1, 1, 1, 1, 1, 1]), + (2544, &[3, 3]), + (2874, &[4, 3]), + ), + ( + 18, + (8519, &[1, 1, 1, 1, 1, 1, 1]), + (2874, &[4, 3]), + (3314, &[4, 4]), + ), + ( + 19, + (10176, &[1, 1, 1, 1, 1, 1, 1, 1]), + (3314, &[4, 4]), + (3974, &[5, 4]), + ), + ( + 20, + (11943, &[1, 1, 1, 1, 1, 1, 1, 1, 1]), + (3974, &[5, 4]), + (4641, &[4, 3, 3]), + ), + ( + 21, + (13820, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (4641, &[4, 3, 3]), + (5081, &[4, 4, 3]), + ), + ( + 22, + (15807, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5081, &[4, 4, 3]), + (5631, &[4, 4, 4]), + ), + ( + 23, + (17904, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (5631, &[4, 4, 4]), + (6291, &[5, 4, 4]), + ), + ( + 24, + (20111, &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + (6291, &[5, 4, 4]), + (7061, &[5, 5, 4]), + ), +]; + +fn check_pin(name: &str, terminal_log: u32, cap: &dyn Fn(u32) -> u32, rows: &[Row]) { + assert_eq!(rows.len(), 19, "{name}: B = 6..=24"); + for &(b, (today_q, today), (s3_q, s3), (s2_q, s2)) in rows { + let ctx = format!("{name} B={b}"); + // today: the all-ones chain from B − 1 (no committed layer when B − 1 ≤ T). + let b0 = fri_chain_start(b, false); + assert_eq!(legacy_fri_schedule(b0, terminal_log), today, "{ctx} today"); + assert_eq!(perm_cost(b0, today, cap), Some(today_q), "{ctx} today cost"); + // S3: the DP from B − 1. + let got = perm_schedule(b0, terminal_log, cap); + assert_eq!(got.schedule, s3, "{ctx} S3 schedule"); + assert_eq!(got.cost_q, s3_q, "{ctx} S3 cost"); + assert_eq!(got.trees as usize, s3.len(), "{ctx} S3 trees"); + // S2+S3: the DP from B (the DEEP codeword is committed). + let b0 = fri_chain_start(b, true); + let got = perm_schedule(b0, terminal_log, cap); + assert_eq!(got.schedule, s2, "{ctx} S2+S3 schedule"); + assert_eq!(got.cost_q, s2_q, "{ctx} S2+S3 cost"); + } +} + +#[test] +fn design_model_reproduces_the_fri_md_table() { + check_pin("T9 cap off", 9, &no_cap, PIN_T9_CAP_OFF); + check_pin("T10 cap off", 10, &no_cap, PIN_T10_CAP_OFF); + check_pin("T9 cap model", 9, &cap_design_model, PIN_T9_CAP_MODEL); + check_pin("T10 cap model", 10, &cap_design_model, PIN_T10_CAP_MODEL); + check_pin("T9 cap auto", 9, &cap_auto, PIN_T9_CAP_AUTO); + check_pin("T10 cap auto", 10, &cap_auto, PIN_T10_CAP_AUTO); +} + +/// Spot checks tying the design-model pins to the model's printed table +/// (costs there are per query, i.e. cost·Q / 110, rounded to two decimals). +#[test] +fn design_model_pins_match_fri_md_table() { + let per_query = |cost_q: u64| (cost_q as f64 / Q as f64 * 100.0).round() / 100.0; + let today = legacy_fri_schedule(20, 9); + assert_eq!(per_query(perm_cost(20, &today, &no_cap).unwrap()), 165.0); + assert_eq!( + per_query(perm_cost(20, &today, &cap_design_model).unwrap()), + 100.70 + ); + let c = perm_schedule(20, 9, &no_cap); + assert_eq!((per_query(c.cost_q), c.schedule), (52.0, vec![4, 4, 3])); + let c = perm_schedule(20, 9, &cap_design_model); + assert_eq!((per_query(c.cost_q), c.schedule), (34.46, vec![4, 4, 3])); + let c = perm_schedule(21, 9, &cap_design_model); + assert_eq!((per_query(c.cost_q), c.schedule), (39.46, vec![4, 4, 4])); + assert_eq!(perm_schedule(18, 9, &no_cap).schedule, vec![5, 4]); + assert_eq!( + perm_schedule(18, 9, &cap_design_model).schedule, + vec![3, 3, 3] + ); + let c = perm_schedule(20, 10, &cap_design_model); + assert_eq!((per_query(c.cost_q), c.schedule), (33.46, vec![4, 3, 3])); + let c = perm_schedule(21, 10, &cap_design_model); + assert_eq!((per_query(c.cost_q), c.schedule), (37.46, vec![4, 4, 3])); +} + +// --------------------------------------------------------------------------- +// U1: the PRODUCTION schedules (cost-law objective), pinned from the Rust DP. +// The schedule is a format constant: a change here is a format change. +// --------------------------------------------------------------------------- + +/// (B, S3 schedule from B − 1, S2+S3 schedule from B) for B = 6..=24. +type CostRow = (u32, &'static [u8], &'static [u8]); + +/// Generated by `print_cost_law_schedule_table` (below, `--ignored`) from the +/// Rust DP, with every emitted row priced. The whole table was cross-checked +/// against an independent Python reproduction of the objective: identical. +/// An independent derivation of the cost law gives [2,2] / [3,3,3] / +/// [3,3,3,2] / [3,3,3,3,2] at B = 14 / 19 / 21 / 24, T = 9 — the Auto rows. +const PIN_COST_T9_CAP_OFF: &[CostRow] = &[ + (6, &[], &[]), + (7, &[], &[]), + (8, &[], &[]), + (9, &[], &[]), + (10, &[], &[1]), + (11, &[1], &[2]), + (12, &[2], &[3]), + (13, &[3], &[2, 2]), + (14, &[2, 2], &[3, 2]), + (15, &[3, 2], &[3, 3]), + (16, &[3, 3], &[3, 2, 2]), + (17, &[3, 2, 2], &[3, 3, 2]), + (18, &[3, 3, 2], &[3, 3, 3]), + (19, &[3, 3, 3], &[4, 3, 3]), + (20, &[4, 3, 3], &[3, 3, 3, 2]), + (21, &[3, 3, 3, 2], &[3, 3, 3, 3]), + (22, &[3, 3, 3, 3], &[4, 3, 3, 3]), + (23, &[4, 3, 3, 3], &[3, 3, 3, 3, 2]), + (24, &[3, 3, 3, 3, 2], &[3, 3, 3, 3, 3]), +]; +const PIN_COST_T10_CAP_OFF: &[CostRow] = &[ + (6, &[], &[]), + (7, &[], &[]), + (8, &[], &[]), + (9, &[], &[]), + (10, &[], &[]), + (11, &[], &[1]), + (12, &[1], &[2]), + (13, &[2], &[3]), + (14, &[3], &[2, 2]), + (15, &[2, 2], &[3, 2]), + (16, &[3, 2], &[3, 3]), + (17, &[3, 3], &[4, 3]), + (18, &[4, 3], &[3, 3, 2]), + (19, &[3, 3, 2], &[3, 3, 3]), + (20, &[3, 3, 3], &[4, 3, 3]), + (21, &[4, 3, 3], &[3, 3, 3, 2]), + (22, &[3, 3, 3, 2], &[3, 3, 3, 3]), + (23, &[3, 3, 3, 3], &[4, 3, 3, 3]), + (24, &[4, 3, 3, 3], &[3, 3, 3, 3, 2]), +]; +const PIN_COST_T9_CAP_AUTO: &[CostRow] = &[ + (6, &[], &[]), + (7, &[], &[]), + (8, &[], &[]), + (9, &[], &[]), + (10, &[], &[1]), + (11, &[1], &[2]), + (12, &[2], &[3]), + (13, &[3], &[2, 2]), + (14, &[2, 2], &[3, 2]), + (15, &[3, 2], &[3, 3]), + (16, &[3, 3], &[3, 2, 2]), + (17, &[3, 2, 2], &[3, 3, 2]), + (18, &[3, 3, 2], &[3, 3, 3]), + (19, &[3, 3, 3], &[3, 3, 2, 2]), + (20, &[3, 3, 2, 2], &[3, 3, 3, 2]), + (21, &[3, 3, 3, 2], &[3, 3, 3, 3]), + (22, &[3, 3, 3, 3], &[4, 3, 3, 3]), + (23, &[4, 3, 3, 3], &[3, 3, 3, 3, 2]), + (24, &[3, 3, 3, 3, 2], &[3, 3, 3, 3, 3]), +]; +const PIN_COST_T10_CAP_AUTO: &[CostRow] = &[ + (6, &[], &[]), + (7, &[], &[]), + (8, &[], &[]), + (9, &[], &[]), + (10, &[], &[]), + (11, &[], &[1]), + (12, &[1], &[2]), + (13, &[2], &[3]), + (14, &[3], &[2, 2]), + (15, &[2, 2], &[3, 2]), + (16, &[3, 2], &[3, 3]), + (17, &[3, 3], &[3, 2, 2]), + (18, &[3, 2, 2], &[3, 3, 2]), + (19, &[3, 3, 2], &[3, 3, 3]), + (20, &[3, 3, 3], &[4, 3, 3]), + (21, &[4, 3, 3], &[3, 3, 3, 2]), + (22, &[3, 3, 3, 2], &[3, 3, 3, 3]), + (23, &[3, 3, 3, 3], &[4, 3, 3, 3]), + (24, &[4, 3, 3, 3], &[3, 3, 3, 3, 2]), +]; + +fn check_cost_pin(name: &str, terminal_log: u32, cap: CapPolicy, rows: &[CostRow]) { + assert_eq!(rows.len(), 19, "{name}: B = 6..=24"); + for &(b, s3, s2) in rows { + let ctx = format!("{name} B={b}"); + let got = fri_schedule( + fri_chain_start(b, false), + terminal_log, + Q, + cap, + FRI_SCHEDULE_DMAX, + ); + assert_eq!(got, s3, "{ctx} S3"); + let got = fri_schedule( + fri_chain_start(b, true), + terminal_log, + Q, + cap, + FRI_SCHEDULE_DMAX, + ); + assert_eq!(got, s2, "{ctx} S2+S3"); + } +} + +#[test] +fn schedule_pinned_table() { + check_cost_pin("T9 cap off", 9, CapPolicy::Off, PIN_COST_T9_CAP_OFF); + check_cost_pin("T10 cap off", 10, CapPolicy::Off, PIN_COST_T10_CAP_OFF); + check_cost_pin("T9 cap auto", 9, CapPolicy::Auto, PIN_COST_T9_CAP_AUTO); + check_cost_pin("T10 cap auto", 10, CapPolicy::Auto, PIN_COST_T10_CAP_AUTO); +} + +/// Prints the U1 table in the `fri_schedule_cost_pins.rs` format. Run with +/// `-- --ignored --nocapture` to regenerate after a DELIBERATE objective change. +#[test] +#[ignore = "generator for fri_schedule_cost_pins.rs"] +fn print_cost_law_schedule_table() { + for (name, t, cap) in [ + ("T9_CAP_OFF", 9, CapPolicy::Off), + ("T10_CAP_OFF", 10, CapPolicy::Off), + ("T9_CAP_AUTO", 9, CapPolicy::Auto), + ("T10_CAP_AUTO", 10, CapPolicy::Auto), + ] { + println!("const PIN_COST_{name}_DATA: [CostRow; 19] = ["); + for b in 6..=24u32 { + let s3 = fri_schedule(fri_chain_start(b, false), t, Q, cap, FRI_SCHEDULE_DMAX); + let s2 = fri_schedule(fri_chain_start(b, true), t, Q, cap, FRI_SCHEDULE_DMAX); + println!(" ({b}, &{s3:?}, &{s2:?}),"); + } + println!("];"); + } +} + +/// B = 21, T = 9, no cap, S3: every candidate schedule's cost by hand, so the +/// pinned optimum is shown to be one, not merely reproduced. +#[test] +fn cost_law_b21_by_hand() { + let layer = |d: u64, depth: u64| -> u64 { + let leaf = (3u64 << d).div_ceil(8).max(1); + let n = 1u64 << d; + let g = n - 1; + let extras = d * (567 + 477) + + d.saturating_sub(2) * 522 + + u64::from(d >= 2) * 477 + + 2 * 522 + + 8 * 477 + + 528 + + n * (528 + 460) + + (3 * n).div_ceil(4) * 528 + + depth * 460; + Q * (leaf * 2251 + depth * (2251 + 567) + g * (567 + 2610) + d * 477 + extras) + }; + let cost = |sched: &[u64]| { + let mut b = 20u64; + sched + .iter() + .map(|&d| { + b -= d; + layer(d, b) + }) + .sum::() + }; + let got = fri_schedule_with_cost(20, 9, Q, CapPolicy::Off, FRI_SCHEDULE_DMAX); + let as_u64: Vec = got.schedule.iter().map(|&d| u64::from(d)).collect(); + assert_eq!(got.cost_q, cost(&as_u64)); + // It beats the permutation objective's choice and today's. + assert!(got.cost_q <= cost(&[4, 4, 3])); + assert!(got.cost_q < cost(&[1; 11])); +} + +// --------------------------------------------------------------------------- +// U2: brute-force optimality for b₀ ≤ 16, under the production objective. +// --------------------------------------------------------------------------- + +/// Independent oracle for one layer's cost-law price (the module docs' +/// formula, written out again from the CAPPED rows priced directly plus the +/// cap's once-per-tree cost, rather than the uncapped rows minus the gain). +fn oracle_layer_q(d: u32, depth: u32, q: u64, cap: CapPolicy) -> u64 { + let (wc, ws, wu, wh, wq) = (2251i128, 567i128, 528i128, 460i128, 3789i128); + let (xalu, balu) = (522i128, 477i128); + let leaf = i128::from((3u64 << d).div_ceil(8).max(1) as u32); + let n = 1i128 << d; + let d = i128::from(d); + let c = cap.height(q as usize, depth as usize) as i128; + let walk = i128::from(depth) - c; + let cap_nodes = 1i128 << c; + let selects = (n - 1) + walk + (cap_nodes - 1) + d; + let xalus = 5 * (n - 1) + 2 + (d - 2).max(0); + let balus = d + d + i128::from(d >= 2) + 8; + let hashes = leaf + walk; + let unpacks = n + 1 + i128::from(c > 0); + let packs = (3 * n + 3) / 4; + let hints = n + walk; + let per_query = selects * ws + + xalus * xalu + + balus * balu + + hashes * wc + + (unpacks + packs) * wu + + hints * wh; + let per_tree = if c == 0 { + 0 + } else { + (cap_nodes - 1) * wc + cap_nodes * wh + wq + }; + (q as i128 * per_query + per_tree) as u64 +} + +/// Every composition of `b0 − t` into parts in `1..=dmax`, with its cost; +/// returns the minimum under (cost, trees, schedule) lexicographic order. +fn brute_force(b0: u32, t: u32, q: u64, cap: CapPolicy, dmax: u32) -> (u64, Vec) { + struct Search { + t: u32, + q: u64, + cap: CapPolicy, + dmax: u32, + prefix: Vec, + best: Option<(u64, usize, Vec)>, + } + impl Search { + fn walk(&mut self, b: u32, cost: u64) { + if b == self.t { + let cand = (cost, self.prefix.len(), self.prefix.clone()); + if self.best.as_ref().is_none_or(|cur| cand < *cur) { + self.best = Some(cand); + } + return; + } + for d in 1..=self.dmax.min(b - self.t) { + self.prefix.push(d as u8); + let c = cost + oracle_layer_q(d, b - d, self.q, self.cap); + self.walk(b - d, c); + self.prefix.pop(); + } + } + } + let mut s = Search { + t, + q, + cap, + dmax, + prefix: Vec::new(), + best: None, + }; + s.walk(b0, 0); + let (cost, _, sched) = s.best.expect("at least the empty / all-ones composition"); + (cost, sched) +} + +#[test] +fn dp_is_optimal() { + let caps = [ + CapPolicy::Off, + CapPolicy::Auto, + CapPolicy::Fixed(2), + CapPolicy::Fixed(5), + ]; + let mut checked = 0u32; + for cap in caps { + for q in [1u64, 3, 110] { + for dmax in [1u32, 2, 3, FRI_SCHEDULE_DMAX] { + for b0 in 0..=16u32 { + for t in 0..=b0 { + let got = fri_schedule_with_cost(b0, t, q, cap, dmax); + let (cost, sched) = brute_force(b0, t, q, cap, dmax); + let ctx = format!("cap={cap:?} q={q} dmax={dmax} b0={b0} t={t}"); + assert_eq!(got.cost_q, cost, "{ctx}: cost"); + // The tie rule makes the optimum unique: the smallest + // (trees, schedule) among the cost-optimal ones. + assert_eq!(got.schedule, sched, "{ctx}: schedule"); + assert_eq!(got.trees as usize, sched.len(), "{ctx}: trees"); + assert_eq!( + fri_schedule_cost_q(b0, &got.schedule, q, cap), + Some(got.cost_q), + "{ctx}: cost of the schedule" + ); + let sum: u32 = got.schedule.iter().map(|&d| u32::from(d)).sum(); + assert_eq!(sum, b0 - t, "{ctx}: lands on the terminal"); + assert!( + got.schedule + .iter() + .all(|&d| (1..=dmax).contains(&u32::from(d))) + ); + checked += 1; + } + // Above the terminal: nothing to commit. + for t in b0 + 1..=b0 + 2 { + assert!(fri_schedule(b0, t, q, cap, dmax).is_empty()); + } + } + } + } + } + assert_eq!(checked, 4 * 3 * 4 * (17 * 18 / 2)); +} + +#[test] +fn dmax_one_is_the_legacy_schedule() { + for b0 in 0..=30u32 { + for t in 0..=31u32 { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + assert_eq!(fri_schedule(b0, t, Q, cap, 1), legacy_fri_schedule(b0, t)); + // dmax = 0 is treated as 1. + assert_eq!(fri_schedule(b0, t, Q, cap, 0), legacy_fri_schedule(b0, t)); + } + } + } +} + +// --------------------------------------------------------------------------- +// U3: the legacy constructor is unchanged. +// --------------------------------------------------------------------------- + +/// `FriFoldLayout::new` as it was before the schedule existed (terminal.rs @ +/// 5d0b0a41a), copied verbatim: (total_folds, num_committed, terminal_len, +/// effective_k). +fn old_layout(lde_log: u32, blowup_log: u32, k: u32) -> (u32, usize, usize, u32) { + let terminal_log = (blowup_log + k).min(lde_log); + let total_folds = lde_log - terminal_log; + ( + total_folds, + total_folds.saturating_sub(1) as usize, + 1usize << terminal_log, + terminal_log - blowup_log, + ) +} + +fn dp_format(cap: CapPolicy) -> FriFormat { + FriFormat { + mode: FriMode::Dp, + one_row: false, + num_queries: Q, + cap, + schedule_override: None, + } +} + +#[test] +fn legacy_layout_equals_old_layout() { + // one_row = true is exercised through the schedule arithmetic only (the + // layout is still well defined); the prover refuses it (S2 not built). + let dp_formats: Vec = [false, true] + .into_iter() + .flat_map(|one_row| { + [CapPolicy::Off, CapPolicy::Auto].map(|cap| FriFormat { + one_row, + ..dp_format(cap) + }) + }) + .collect(); + let mut checked = 0u32; + for blowup_log in 1..=4u32 { + // The LDE is at least the blowup (trace length ≥ 1). + for lde_log in blowup_log..=30u32 { + for k in 0..=10u32 { + let ctx = format!("lde_log={lde_log} blowup_log={blowup_log} k={k}"); + let (total_folds, num_committed, terminal_len, effective_k) = + old_layout(lde_log, blowup_log, k); + let new = FriFoldLayout::new(lde_log, blowup_log, k); + assert_eq!(new.total_folds, total_folds, "{ctx}"); + assert_eq!(new.num_committed, num_committed, "{ctx}"); + assert_eq!(new.terminal_len, terminal_len, "{ctx}"); + assert_eq!(new.effective_k, effective_k, "{ctx}"); + assert_eq!(new.schedule, vec![1u8; num_committed], "{ctx}"); + assert!(!new.one_row, "{ctx}"); + assert!(new.is_legacy(), "{ctx}"); + assert_eq!(new.opened_values_per_query(), num_committed, "{ctx}"); + + // Pair mode ignores the query count, the cap policy and any + // schedule override. + for cap in [CapPolicy::Off, CapPolicy::Auto] { + let pair = FriFormat { + mode: FriMode::Pair, + schedule_override: FriScheduleOverride::new(&[3, 1]), + ..dp_format(cap) + }; + assert_eq!( + FriFoldLayout::for_format(lde_log, blowup_log, k, &pair), + Some(new.clone()), + "{ctx}" + ); + } + assert_eq!( + FriFoldLayout::from_schedule( + lde_log, + blowup_log, + k, + false, + new.schedule.clone() + ), + Some(new.clone()), + "{ctx}" + ); + + // Any format moves only the split of the folds into committed + // layers (and, off the legacy format, the encoding). + for fmt in &dp_formats { + let l = FriFoldLayout::for_format(lde_log, blowup_log, k, fmt) + .expect("the DP lands on the terminal"); + assert_eq!( + (l.total_folds, l.terminal_len, l.effective_k, l.one_row), + (total_folds, terminal_len, effective_k, fmt.one_row), + "{ctx} {fmt:?}" + ); + assert!( + !l.is_legacy(), + "{ctx} {fmt:?}: Dp is never the legacy encoding" + ); + assert_eq!(l.num_committed, l.schedule.len(), "{ctx} {fmt:?}"); + let covered: u32 = l.schedule.iter().map(|&d| u32::from(d)).sum(); + let expected = match (total_folds, fmt.one_row) { + (0, _) => 0, + (n, true) => n, + (n, false) => n - 1, + }; + assert_eq!(covered, expected, "{ctx} {fmt:?}"); + assert_eq!( + l.opened_values_per_query(), + l.schedule.iter().map(|&d| 1usize << d).sum::(), + "{ctx} {fmt:?}" + ); + for j in 0..l.num_committed { + let d = u32::from(l.schedule[j]); + assert_eq!( + l.layer_depth(lde_log, j) + d, + l.layer_log_len(lde_log, j), + "{ctx} {fmt:?} layer {j}" + ); + } + } + checked += 1; + } + } + } + assert_eq!(checked, 11 * (30 + 29 + 28 + 27)); +} + +#[test] +fn from_schedule_rejects_a_schedule_that_does_not_cover_the_folds() { + // lde_log 20, blowup 2, k 7: total_folds 11, row-pair chain covers 10 bits. + assert!(FriFoldLayout::from_schedule(20, 2, 7, false, vec![4, 4, 2]).is_some()); + assert!(FriFoldLayout::from_schedule(20, 2, 7, false, vec![4, 4, 3]).is_none()); + assert!(FriFoldLayout::from_schedule(20, 2, 7, false, vec![4, 4, 1]).is_none()); + // One-row: the chain covers all 11. + assert!(FriFoldLayout::from_schedule(20, 2, 7, true, vec![4, 4, 3]).is_some()); + assert!(FriFoldLayout::from_schedule(20, 2, 7, true, vec![4, 4, 2]).is_none()); + // Zero and over-DMAX exponents. + assert!(FriFoldLayout::from_schedule(20, 2, 7, false, vec![0, 5, 5]).is_none()); + assert!(FriFoldLayout::from_schedule(20, 2, 7, false, vec![7, 3]).is_none()); + // No fold: only the empty schedule. + assert!(FriFoldLayout::from_schedule(8, 2, 7, false, vec![]).is_some()); + assert!(FriFoldLayout::from_schedule(8, 2, 7, true, vec![1]).is_none()); + // One fold, row pair: no committed layer. + assert!(FriFoldLayout::from_schedule(10, 2, 7, false, vec![]).is_some()); + assert!(FriFoldLayout::from_schedule(10, 2, 7, false, vec![1]).is_none()); + assert!(FriFoldLayout::from_schedule(10, 2, 7, true, vec![1]).is_some()); +} + +// --------------------------------------------------------------------------- +// The layout from `ProofOptions` (what the prover and verifier build). +// --------------------------------------------------------------------------- + +fn options_with(format: ProofFormat) -> ProofOptions { + ProofOptions { + format, + ..ProofOptions::default_test_options() + } +} + +#[test] +fn layout_from_options() { + // Default format: today's layout. + let o = options_with(ProofFormat::DEFAULT); + let k = u32::from(o.fri_final_poly_log_degree); + assert_eq!( + FriFoldLayout::for_options(20, 1, &o, false), + Ok(FriFoldLayout::new(20, 1, k)) + ); + // Dp: the DP's schedule under the options' query count and cap. + let o = options_with(ProofFormat { + fri_mode: FriMode::Dp, + ..ProofFormat::DEFAULT + }); + let l = FriFoldLayout::for_options(20, 1, &o, false).unwrap(); + let t = (1 + k).min(20); + assert_eq!( + l.schedule, + fri_schedule( + 19, + t, + o.fri_number_of_queries as u64, + CapPolicy::Off, + FRI_SCHEDULE_DMAX + ) + ); + assert!(!l.is_legacy()); + // An override that fits is taken verbatim; one that does not is an error. + let span = 19 - t; + let mut fit = vec![1u8; span as usize - 3]; + fit.insert(0, 3); + let o = options_with(ProofFormat { + fri_mode: FriMode::Dp, + fri_schedule_override: FriScheduleOverride::new(&fit), + ..ProofFormat::DEFAULT + }); + assert_eq!( + FriFoldLayout::for_options(20, 1, &o, false) + .unwrap() + .schedule, + fit + ); + let o = options_with(ProofFormat { + fri_mode: FriMode::Dp, + fri_schedule_override: FriScheduleOverride::new(&[3, 1]), + ..ProofFormat::DEFAULT + }); + assert_eq!( + FriFoldLayout::for_options(20, 1, &o, false), + Err(FriFormatError::ScheduleOverrideMismatch) + ); + // An all-ones override under Dp keeps the GROUP encoding. + let o = options_with(ProofFormat { + fri_mode: FriMode::Dp, + fri_schedule_override: FriScheduleOverride::new(&vec![1u8; span as usize]), + ..ProofFormat::DEFAULT + }); + let l = FriFoldLayout::for_options(20, 1, &o, false).unwrap(); + assert_eq!(l.schedule, vec![1u8; span as usize]); + assert!(!l.is_legacy()); + // One row (S2): the chain starts at the LDE size, the encoding is the + // group one even at fri = pair, and the all-ones schedule covers every + // fold (no uncommitted fold 0). + for one_row in [OneRowMode::On, OneRowMode::Auto] { + let o = options_with(ProofFormat { + one_row, + ..ProofFormat::DEFAULT + }); + let l = FriFoldLayout::for_options(20, 1, &o, true).unwrap(); + assert!(l.one_row && !l.is_legacy()); + assert_eq!(l.schedule, vec![1u8; (20 - t) as usize]); + assert_eq!(l.num_committed as u32, l.total_folds); + assert_eq!(l.num_zetas(), l.num_committed); + assert_eq!( + l.layer_depth(20, 0), + 19, + "the input tree: 2^20 values in pairs" + ); + // The same options at a resolved row-pair layout: today's. + assert_eq!( + FriFoldLayout::for_options(20, 1, &o, false), + Ok(FriFoldLayout::new(20, 1, k)) + ); + } + // An override longer than the fixed capacity is refused at construction. + assert!(FriScheduleOverride::new(&[1u8; 33]).is_none()); + assert_eq!( + FriScheduleOverride::new(&[2, 1]).unwrap().as_slice(), + &[2, 1] + ); +} diff --git a/crypto/stark/src/tests/fri_tests.rs b/crypto/stark/src/tests/fri_tests.rs index 5b599886b..b9f8de565 100644 --- a/crypto/stark/src/tests/fri_tests.rs +++ b/crypto/stark/src/tests/fri_tests.rs @@ -142,6 +142,7 @@ fn test_eval_fold_matches_coeff_fold() { /// reconstructed terminal codeword at the query's terminal-layer position. #[test] fn test_commit_phase_early_termination_roundtrip() { + use crate::config::KeccakStarkHash; use crate::fri::fri_functions::update_twiddles_in_place; use crate::fri::terminal::terminal_codeword_from_coeffs; use crate::fri::{commit_phase_from_evaluations, query_phase}; @@ -176,7 +177,7 @@ fn test_commit_phase_early_termination_roundtrip() { let mut transcript = DefaultTranscript::::new(&[]); let inv_twiddles = crate::fri::fri_functions::compute_coset_twiddles_inv::(&offset, initial_len); - let (final_poly_coeffs, fri_layers) = commit_phase_from_evaluations::( + let (final_poly_coeffs, fri_layers) = commit_phase_from_evaluations::( codeword.clone(), &mut transcript, &offset, @@ -199,7 +200,7 @@ fn test_commit_phase_early_termination_roundtrip() { // query_phase must still work against the committed layers. let iotas = vec![0usize, 1, 5, 17, 30]; - let _decommitments = query_phase(&fri_layers, &iotas); + let _decommitments = query_phase::(&fri_layers, &iotas); // ---- Reconstruct terminal codeword from the emitted coefficients ---- let terminal_len = (1usize << blowup_log) << final_poly_log_degree; // 8 diff --git a/crypto/stark/src/tests/grinding_tests.rs b/crypto/stark/src/tests/grinding_tests.rs index 49c47e81f..53f79fd6d 100644 --- a/crypto/stark/src/tests/grinding_tests.rs +++ b/crypto/stark/src/tests/grinding_tests.rs @@ -1,5 +1,10 @@ +use crate::config::{GrindingDigest, KeccakStarkHash}; use crate::grinding::is_valid_nonce; +/// The default configuration's grinding hash. These vectors were computed +/// against keccak-256, so they name it rather than following any alias. +type Keccak = GrindingDigest; + #[test] fn test_invalid_nonce_grinding_factor_6() { // This setting produces a hash with 5 leading zeros, therefore not enough for grinding @@ -10,7 +15,7 @@ fn test_invalid_nonce_grinding_factor_6() { ]; let nonce = 4; let grinding_factor = 6; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -23,7 +28,7 @@ fn test_invalid_nonce_grinding_factor_9() { ]; let nonce = 287; let grinding_factor = 9; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -34,7 +39,7 @@ fn test_is_valid_nonce_grinding_factor_10() { ]; let nonce = 0x5ba; let grinding_factor = 10; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -45,7 +50,7 @@ fn test_is_valid_nonce_grinding_factor_20() { ]; let nonce = 0x2c5db8; let grinding_factor = 20; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -59,7 +64,7 @@ fn test_invalid_nonce_grinding_factor_19() { ]; let nonce = 0x2c5db8; let grinding_factor = 19; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -70,7 +75,7 @@ fn test_is_valid_nonce_grinding_factor_30() { ]; let nonce = 0x1ae839e1; let grinding_factor = 30; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -81,5 +86,60 @@ fn test_is_valid_nonce_grinding_factor_33() { ]; let nonce = 0x4cc3123f; let grinding_factor = 33; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); +} + +// ========================================================================= +// The BLAKE3 configuration's proof of work. +// +// Grinding is a type substitution — same two one-block hashes, same 32-byte +// seed and digest — so what needs pinning is that the substitution actually +// happened and that the two configurations do not accept each other's work. +// ========================================================================= + +/// The digest the BLAKE3 configuration grinds over. Gated with the tests below +/// because `Blake3StarkHash` does not exist under `cuda` — the device kernels +/// are keccak-only, so there is no second configuration to name there. +#[cfg(not(feature = "cuda"))] +type Blake3 = GrindingDigest; + +/// ★ Honest path: a nonce ground under BLAKE3 satisfies the BLAKE3 check. +/// +/// At `grinding_factor = 20` a nonce passes by chance with probability 2⁻²⁰, so +/// the cross-hash rejection below is a real control rather than a coin flip. +#[cfg(not(feature = "cuda"))] +#[test] +fn a_blake3_ground_nonce_satisfies_the_blake3_check() { + let seed = [0x5au8; 32]; + let factor = 20; + + let nonce = crate::grinding::generate_nonce::(&seed, factor) + .expect("a nonce exists at this factor"); + assert!( + is_valid_nonce::(&seed, nonce, factor), + "the nonce grinding found must satisfy the check it was ground against" + ); + + // FALSIFICATION: work done against one hash is not work against the other. + // Without this, `generate_nonce::` could still be computing keccak + // and every assertion above would hold. + assert!( + !is_valid_nonce::(&seed, nonce, factor), + "a BLAKE3-ground nonce must not satisfy the keccak check" + ); +} + +/// The two configurations compute different work on identical inputs. +#[cfg(not(feature = "cuda"))] +#[test] +fn the_two_configurations_grind_different_work() { + let seed = [0x11u8; 32]; + let factor = 16; + + let blake3 = crate::grinding::generate_nonce::(&seed, factor).expect("blake3 nonce"); + let keccak = crate::grinding::generate_nonce::(&seed, factor).expect("keccak nonce"); + assert_ne!( + blake3, keccak, + "the same seed under two hashes must not grind to the same nonce" + ); } diff --git a/crypto/stark/src/tests/log_read_only_program_tests.rs b/crypto/stark/src/tests/log_read_only_program_tests.rs new file mode 100644 index 000000000..08e389bb7 --- /dev/null +++ b/crypto/stark/src/tests/log_read_only_program_tests.rs @@ -0,0 +1,151 @@ +//! `LogReadOnlyRAP` carries a constraint program. +//! +//! The CUDA composition arm evaluates `AIR::constraint_program()` once main +//! and aux are device-resident; `LogReadOnlyRAP` (the AIR of the checked-in +//! S3/S2 proof vectors) had none, so every device-proved vector test panicked +//! before it compared a byte. The program is captured from the SAME +//! `LogReadOnlyRAPConstraints` body the CPU folders run, so the device +//! composes the same polynomials and the vector bytes cannot move. +//! +//! These CPU tests pin that equality on random two-row frames three ways: +//! the prover folder (the CPU prover's hot path) == the captured program under +//! the generic interpreter == the lowered device program under its host model +//! (`eval_device_program`, the CPU model of the GPU kernel). + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as E; +use math::field::goldilocks::GoldilocksField as F; + +use crate::constraint_ir::{DeviceProgram, eval_device_program, eval_program}; +use crate::examples::read_only_memory_logup::LogReadOnlyRAP; +use crate::frame::Frame; +use crate::proof::options::ProofOptions; +use crate::table::TableView; +use crate::traits::{AIR, TransitionEvaluationContext}; + +type Felt = FieldElement; +type Ext = FieldElement; + +struct SplitMix64(u64); +impl SplitMix64 { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn fp(&mut self) -> Felt { + Felt::from(self.next_u64()) + } + fn ext(&mut self) -> Ext { + Ext::from_raw([self.fp(), self.fp(), self.fp()]) + } +} + +fn limbs(x: &Ext) -> [u64; 3] { + let v = x.value(); + [v[0].canonical(), v[1].canonical(), v[2].canonical()] +} + +fn from_limbs(l: [u64; 3]) -> Ext { + Ext::from_raw([Felt::from(l[0]), Felt::from(l[1]), Felt::from(l[2])]) +} + +fn air() -> LogReadOnlyRAP { + LogReadOnlyRAP::::new(&ProofOptions::default_test_options()) +} + +#[test] +fn the_log_read_only_program_has_the_air_shape() { + let air = air(); + let prog = air.constraint_program(); + assert_eq!(prog.roots.len(), air.num_transition_constraints()); + assert_eq!(prog.num_base, air.num_base_transition_constraints()); + assert_eq!(prog.num_base, 2, "continuity and single-value are base"); + // Cached: a second call hands back the same program. + assert!(std::ptr::eq(prog, air.constraint_program())); +} + +#[test] +fn the_log_read_only_program_equals_the_prover_folder_and_the_device_model() { + let air = air(); + let prog = air.constraint_program(); + let dev = DeviceProgram::lower(prog); + let n = air.num_transition_constraints(); + let nb = air.num_base_transition_constraints(); + let (main_w, aux_w) = air.trace_layout(); + + let mut rng = SplitMix64(0x1F1C_D2D2_0000_0001); + for trial in 0..500 { + let main: Vec> = (0..2) + .map(|_| (0..main_w).map(|_| rng.fp()).collect()) + .collect(); + let aux: Vec> = (0..2) + .map(|_| (0..aux_w).map(|_| rng.ext()).collect()) + .collect(); + let rap = vec![rng.ext(), rng.ext()]; + let alphas: Vec = Vec::new(); + let offset = Ext::zero(); + + let steps: Vec> = main + .iter() + .zip(aux.iter()) + .map(|(m, a)| TableView::::new(vec![m.clone()], vec![a.clone()])) + .collect(); + let frame = Frame::::new(steps); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &rap, &alphas, &offset); + + // The CPU prover's path. + let mut folder_base = vec![Felt::zero(); nb]; + let mut folder_ext = vec![Ext::zero(); n]; + air.compute_transition_prover(&ctx, &mut folder_base, &mut folder_ext); + + // The captured program, generic interpreter. + let mut interp_base = vec![Felt::zero(); nb]; + let mut interp_ext = vec![Ext::zero(); n]; + eval_program(prog, &ctx, &mut interp_base, &mut interp_ext); + assert_eq!(folder_base, interp_base, "base constraints, trial {trial}"); + assert_eq!( + folder_ext[nb..], + interp_ext[nb..], + "ext constraints, trial {trial}" + ); + + // The lowered device program, host model of the GPU kernel. + let main_raw: Vec> = main + .iter() + .map(|r| r.iter().map(|x| x.canonical()).collect()) + .collect(); + let aux_raw: Vec> = + aux.iter().map(|r| r.iter().map(limbs).collect()).collect(); + let rap_raw: Vec<[u64; 3]> = rap.iter().map(limbs).collect(); + let mut base_dev = vec![0u64; nb]; + let mut ext_dev = vec![[0u64; 3]; n]; + eval_device_program( + &dev, + &main_raw, + &aux_raw, + &rap_raw, + &[], + limbs(&offset), + &mut base_dev, + &mut ext_dev, + ); + for c in 0..nb { + assert_eq!( + Felt::from(base_dev[c]), + folder_base[c], + "device base {c}, trial {trial}" + ); + } + for c in nb..n { + assert_eq!( + from_limbs(ext_dev[c]), + folder_ext[c], + "device ext {c}, trial {trial}" + ); + } + } +} diff --git a/crypto/stark/src/tests/merkle_cap_tests.rs b/crypto/stark/src/tests/merkle_cap_tests.rs new file mode 100644 index 000000000..d00d4ca70 --- /dev/null +++ b/crypto/stark/src/tests/merkle_cap_tests.rs @@ -0,0 +1,831 @@ +//! Merkle caps on univariate STARK proofs (lever S1). +//! +//! Every tree of a proof — main, precomputed, aux, composition, each committed +//! FRI layer — gets a height-`c` cap under a cap policy. The cap rides at the +//! end of the tree's first opening (the owner path); every path is cut to +//! `D − c` siblings. These tests pin: +//! - round trips at every policy, over the owned and the archived (rkyv) path; +//! - the default (`Off`) is byte-identical to a zero-height policy; +//! - the transcript does not move: an `Off` and an `Auto` proof of one witness +//! differ only in their Merkle paths; +//! - tampers of every tree class, of the owner split, and of the policy; +//! - load-bearing checks at the verifier level: an unreached cap node that only the +//! cap-to-root check rejects, and an internal node passed off as a leaf that +//! only the exact-length check rejects. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::merkle_tree::cap::{CapPolicy, verify_cap}; +use crypto::merkle_tree::proof::verify_merkle_path_from_leaf_hash; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +use crate::config::{Commitment, DefaultStarkHash, StarkHash}; +use crate::domain::new_verifier_domain; +use crate::examples::fibonacci_2_columns::compute_trace; +use crate::examples::fibonacci_rap::{FibonacciRAP, FibonacciRAPPublicInputs, fibonacci_rap_trace}; +use crate::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use crate::examples::simple_fibonacci::FibonacciPublicInputs; +use crate::merkle_caps::StarkCaps; +use crate::proof::options::ProofOptions; +use crate::proof::stark::{MultiProof, StarkProof}; +use crate::proof::view::StarkProofView; +use crate::prover::{IsStarkProver, Prover}; +use crate::tests::opening_width_tests::FibonacciSplitAIR; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +type F = GoldilocksField; +type FE = FieldElement; +type PI = SimpleAdditionPublicInputs; +type Proof = StarkProof; +/// The leaf backend the default prover commits the trace trees with. +type Leaf = ::Batched; + +/// 1024 rows at blowup 2: trace trees 10 deep, 2 committed FRI layers. +const ROWS: usize = 1024; + +fn options(policy: CapPolicy, queries: usize, blowup: u8) -> ProofOptions { + let mut o = ProofOptions::default_test_options(); + o.blowup_factor = blowup; + o.fri_number_of_queries = queries; + // Grinding off: the nonce is then absent, and two proofs of one witness + // are comparable byte for byte. + o.grinding_factor = 0; + o.format.merkle_cap = policy; + o +} + +fn prove(opts: &ProofOptions) -> (SimpleAdditionAIR, Proof) { + let air = SimpleAdditionAIR::::new(opts); + let pub_inputs = SimpleAdditionPublicInputs { + a: FE::from(1u64), + b: FE::from(2u64), + }; + let mut trace = simple_addition_trace::(ROWS); + let proof = Prover::prove( + &air, + &mut trace, + &pub_inputs, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof) +} + +fn verifies(air: &SimpleAdditionAIR, proof: &Proof) -> bool { + Verifier::verify(proof, air, &mut DefaultTranscript::::new(&[])) +} + +/// The same proof over the wire: rkyv, then `multi_verify_archived` (the +/// read-in-place path host continuation verification uses). +fn verifies_archived(air: &SimpleAdditionAIR, proof: &Proof) -> bool { + let multi = MultiProof { + proofs: vec![proof.clone()], + }; + let bytes = rkyv::to_bytes::(&multi).unwrap(); + let archived = rkyv::access::< + crate::proof::stark::ArchivedMultiProof, + rkyv::rancor::Error, + >(&bytes) + .unwrap(); + let airs: Vec<&dyn AIR> = vec![air]; + Verifier::multi_verify_archived( + &airs, + archived, + &mut DefaultTranscript::::new(&[]), + &FE::zero(), + ) +} + +fn caps_of(air: &SimpleAdditionAIR, proof: &Proof) -> StarkCaps { + let o = air.options(); + StarkCaps::new( + o.format.merkle_cap, + o.fri_number_of_queries, + (o.blowup_factor as usize * proof.trace_length).trailing_zeros() as usize, + proof.fri_layers_merkle_roots.len(), + ) +} + +/// Every path of one tree: the owner carries `D − c + 2^c` nodes (the cap at +/// its end, hashing to `root`), every other opening `D − c`. +fn assert_tree_shape(paths: &[&Vec], root: &Commitment, depth: usize, c: usize) { + let owner_len = if c == 0 { depth } else { depth - c + (1 << c) }; + assert_eq!(paths[0].len(), owner_len, "owner path, D={depth} c={c}"); + for (q, p) in paths.iter().enumerate().skip(1) { + assert_eq!(p.len(), depth - c, "query {q}, D={depth} c={c}"); + } + if c > 0 { + assert!( + verify_cap::(&paths[0][depth - c..], root, c), + "the owner's cap must hash to the root" + ); + } +} + +fn assert_proof_shape(air: &SimpleAdditionAIR, proof: &Proof) { + let caps = caps_of(air, proof); + let main: Vec<_> = proof + .deep_poly_openings + .iter() + .map(|o| &o.main_trace_polys.proof.merkle_path) + .collect(); + assert_tree_shape( + &main, + &proof.lde_trace_main_merkle_root, + caps.trace_depth, + caps.trace, + ); + let comp: Vec<_> = proof + .deep_poly_openings + .iter() + .map(|o| &o.composition_poly.proof.merkle_path) + .collect(); + assert_tree_shape( + &comp, + &proof.composition_poly_root, + caps.trace_depth, + caps.trace, + ); + for (i, root) in proof.fri_layers_merkle_roots.iter().enumerate() { + let layer: Vec<_> = proof + .query_list + .iter() + .map(|q| &q.layers_auth_paths[i].merkle_path) + .collect(); + assert_tree_shape(&layer, root, caps.fri_depths[i], caps.fri[i]); + } +} + +// ------------------------------------------------------------------ round trips + +#[test] +fn every_policy_round_trips_owned_and_archived() { + for blowup in [2u8, 4] { + for (policy, queries) in [ + (CapPolicy::Fixed(1), 3), + (CapPolicy::Fixed(2), 3), + (CapPolicy::Fixed(3), 3), + (CapPolicy::Fixed(4), 3), + (CapPolicy::Auto, 3), + (CapPolicy::Auto, 8), + (CapPolicy::Auto, 30), + ] { + let (air, proof) = prove(&options(policy, queries, blowup)); + assert!( + proof.fri_layers_merkle_roots.len() >= 2, + "the FRI arm must commit layers" + ); + let caps = caps_of(&air, &proof); + if policy != CapPolicy::Auto || queries >= 4 { + assert!(caps.any(), "{policy} Q={queries}: some tree must be capped"); + } + assert_proof_shape(&air, &proof); + assert!( + verifies(&air, &proof), + "{policy} Q={queries} blowup {blowup}" + ); + assert!( + verifies_archived(&air, &proof), + "{policy} Q={queries} blowup {blowup}: archived" + ); + } + } +} + +/// A preprocessed table (precomputed + main trees) and a RAP table (main + aux +/// trees) round-trip under a cap, and a cap node flip in each of those trees is +/// rejected. +#[test] +fn preprocessed_and_aux_trees_are_capped() { + // Preprocessed: 1 precomputed column, 1 main column, 1024 rows. + let opts = options(CapPolicy::Fixed(3), 3, 2); + let mut trace = compute_trace([FE::one(), FE::one()], ROWS); + let reference = FibonacciSplitAIR::::honest(&opts, None); + let commitment = Prover::compute_precomputed_commitment_for_testing(&trace, &reference, 1) + .expect("precomputed commitment"); + let air = FibonacciSplitAIR::::preprocessed_declaring(&opts, None, 1, commitment); + let pi = FibonacciPublicInputs { + a0: FE::one(), + a1: FE::one(), + }; + let proof = + Prover::prove(&air, &mut trace, &pi, &mut DefaultTranscript::::new(&[])).expect("prove"); + let verify = |p: &StarkProof>| { + Verifier::verify(p, &air, &mut DefaultTranscript::::new(&[])) + }; + assert!(verify(&proof), "capped preprocessed proof"); + let depth = 10; + let pre = proof.deep_poly_openings[0] + .precomputed_trace_polys + .as_ref() + .expect("precomputed opening"); + assert_eq!(pre.proof.merkle_path.len(), depth - 3 + 8); + for k in 0..8 { + let mut bad = proof.clone(); + bad.deep_poly_openings[0] + .precomputed_trace_polys + .as_mut() + .unwrap() + .proof + .merkle_path[depth - 3 + k][0] ^= 1; + assert!(!verify(&bad), "precomputed cap node {k}"); + } + + // RAP: 2 main + 1 aux column, 16 steps (the AIR's constraints are fixed to + // 16 steps), capped at 3. + let opts = options(CapPolicy::Fixed(3), 3, 2); + let mut trace = fibonacci_rap_trace([FE::one(), FE::one()], 16); + let air = FibonacciRAP::::new(&opts); + let pi = FibonacciRAPPublicInputs { + steps: 16, + a0: FE::one(), + a1: FE::one(), + }; + let proof = + Prover::prove(&air, &mut trace, &pi, &mut DefaultTranscript::::new(&[])).expect("prove"); + let verify = |p: &StarkProof>| { + Verifier::verify(p, &air, &mut DefaultTranscript::::new(&[])) + }; + assert!(verify(&proof), "capped RAP proof"); + let depth = StarkCaps::trace_tree_depth((2 * proof.trace_length).trailing_zeros() as usize); + assert!(depth >= 3); + let aux = proof.deep_poly_openings[0] + .aux_trace_polys + .as_ref() + .expect("aux opening"); + assert_eq!(aux.proof.merkle_path.len(), depth - 3 + 8); + assert_eq!( + proof.deep_poly_openings[1] + .aux_trace_polys + .as_ref() + .unwrap() + .proof + .merkle_path + .len(), + depth - 3 + ); + for k in 0..8 { + let mut bad = proof.clone(); + bad.deep_poly_openings[0] + .aux_trace_polys + .as_mut() + .unwrap() + .proof + .merkle_path[depth - 3 + k][5] ^= 0x40; + assert!(!verify(&bad), "aux cap node {k}"); + } +} + +// ------------------------------------------------------------ default identity + +/// `Off`, `Fixed(0)` and a policy whose every height is 0 (`Auto` at 3 +/// queries) produce the same bytes: the default format is unchanged. +#[test] +fn a_zero_height_policy_is_byte_identical_to_off() { + let bytes = |policy| { + let (air, proof) = prove(&options(policy, 3, 2)); + assert!(!caps_of(&air, &proof).any()); + rkyv::to_bytes::(&proof) + .unwrap() + .to_vec() + }; + let off = bytes(CapPolicy::Off); + assert_eq!(off, bytes(CapPolicy::Fixed(0))); + assert_eq!(off, bytes(CapPolicy::Auto)); +} + +/// The transcript does not change under a cap. One witness +/// proved at `Off` and at `Auto` (grinding off) gives equal roots, OOD values, +/// FRI final coefficients, nonces and opened values; only the Merkle paths +/// differ, and each capped path is exactly its full path cut to `D − c`, with +/// the cap appended on the owner. +#[test] +fn the_transcript_is_the_same_with_and_without_a_cap() { + let (_, off) = prove(&options(CapPolicy::Off, 30, 2)); + let (air, on) = prove(&options(CapPolicy::Auto, 30, 2)); + let caps = caps_of(&air, &on); + assert_eq!(caps.trace, 3); + assert!(caps.fri.iter().all(|&c| c == 3)); + + assert_eq!( + off.lde_trace_main_merkle_root, + on.lde_trace_main_merkle_root + ); + assert_eq!(off.lde_trace_aux_merkle_root, on.lde_trace_aux_merkle_root); + assert_eq!( + off.lde_trace_precomputed_merkle_root, + on.lde_trace_precomputed_merkle_root + ); + assert_eq!(off.composition_poly_root, on.composition_poly_root); + assert_eq!(off.fri_layers_merkle_roots, on.fri_layers_merkle_roots); + assert_eq!(off.trace_ood_evaluations, on.trace_ood_evaluations); + assert_eq!( + off.trace_ood_next_evaluations, + on.trace_ood_next_evaluations + ); + assert_eq!( + off.composition_poly_parts_ood_evaluation, + on.composition_poly_parts_ood_evaluation + ); + assert_eq!(off.fri_final_poly_coeffs, on.fri_final_poly_coeffs); + assert_eq!(off.nonce, on.nonce); + assert_eq!(off.trace_length, on.trace_length); + + let cut = |full: &Vec, capped: &Vec, q: usize, d: usize, c: usize| { + assert_eq!(full.len(), d); + assert_eq!(&capped[..d - c], &full[..d - c], "query {q}: the siblings"); + let tail = if q == 0 { 1usize << c } else { 0 }; + assert_eq!(capped.len(), d - c + tail, "query {q}"); + }; + let d = caps.trace_depth; + for (q, (a, b)) in off + .deep_poly_openings + .iter() + .zip(&on.deep_poly_openings) + .enumerate() + { + assert_eq!( + a.main_trace_polys.evaluations, + b.main_trace_polys.evaluations + ); + assert_eq!( + a.main_trace_polys.evaluations_sym, + b.main_trace_polys.evaluations_sym + ); + assert_eq!( + a.composition_poly.evaluations, + b.composition_poly.evaluations + ); + assert_eq!( + a.composition_poly.evaluations_sym, + b.composition_poly.evaluations_sym + ); + cut( + &a.main_trace_polys.proof.merkle_path, + &b.main_trace_polys.proof.merkle_path, + q, + d, + 3, + ); + cut( + &a.composition_poly.proof.merkle_path, + &b.composition_poly.proof.merkle_path, + q, + d, + 3, + ); + } + for (q, (a, b)) in off.query_list.iter().zip(&on.query_list).enumerate() { + assert_eq!(a.layers_evaluations_sym, b.layers_evaluations_sym); + for i in 0..caps.fri.len() { + cut( + &a.layers_auth_paths[i].merkle_path, + &b.layers_auth_paths[i].merkle_path, + q, + caps.fri_depths[i], + caps.fri[i], + ); + } + } +} + +// --------------------------------------------------------------------- tampers + +type PathOf = fn(&mut Proof) -> &mut Vec; +type PathFn = dyn Fn(&mut Proof) -> &mut Vec; + +fn main_path(q: usize) -> impl Fn(&mut Proof) -> &mut Vec { + move |p| &mut p.deep_poly_openings[q].main_trace_polys.proof.merkle_path +} +fn comp_path(q: usize) -> impl Fn(&mut Proof) -> &mut Vec { + move |p| &mut p.deep_poly_openings[q].composition_poly.proof.merkle_path +} +fn fri_path(q: usize, layer: usize) -> impl Fn(&mut Proof) -> &mut Vec { + move |p| &mut p.query_list[q].layers_auth_paths[layer].merkle_path +} + +fn rejected_after( + air: &SimpleAdditionAIR, + honest: &Proof, + tamper: impl FnOnce(&mut Proof), +) -> bool { + let mut p = honest.clone(); + tamper(&mut p); + !verifies(air, &p) && !verifies_archived(air, &p) +} + +#[test] +fn every_cap_node_of_every_tree_class_is_bound() { + let (air, honest) = prove(&options(CapPolicy::Auto, 30, 2)); + assert!(verifies(&air, &honest)); + let caps = caps_of(&air, &honest); + let last = honest.fri_layers_merkle_roots.len() - 1; + let trees: Vec<(&str, Box, usize, usize)> = vec![ + ("main", Box::new(main_path(0)), caps.trace_depth, caps.trace), + ( + "composition", + Box::new(comp_path(0)), + caps.trace_depth, + caps.trace, + ), + ( + "FRI layer 0", + Box::new(fri_path(0, 0)), + caps.fri_depths[0], + caps.fri[0], + ), + ( + "last FRI layer", + Box::new(fri_path(0, last)), + caps.fri_depths[last], + caps.fri[last], + ), + ]; + for (what, path_of, d, c) in &trees { + assert_eq!(*c, 3, "{what}"); + for k in 0..(1usize << c) { + assert!( + rejected_after(&air, &honest, |p| path_of(p)[d - c + k][7] ^= 1), + "{what}: cap node {k} flipped" + ); + } + } +} + +#[test] +fn a_path_node_of_a_later_query_is_bound() { + let (air, honest) = prove(&options(CapPolicy::Auto, 30, 2)); + let paths: [(&str, PathOf); 3] = [ + ("main", |p| { + &mut p.deep_poly_openings[5].main_trace_polys.proof.merkle_path + }), + ("composition", |p| { + &mut p.deep_poly_openings[5].composition_poly.proof.merkle_path + }), + ("FRI layer 1", |p| { + &mut p.query_list[5].layers_auth_paths[1].merkle_path + }), + ]; + for (what, path_of) in paths { + let len = path_of(&mut honest.clone()).len(); + for k in 0..len { + assert!( + rejected_after(&air, &honest, |p| path_of(p)[k][0] ^= 0x10), + "{what}: query 5 node {k}" + ); + } + } +} + +#[test] +fn the_owner_split_is_exact() { + let (air, honest) = prove(&options(CapPolicy::Auto, 30, 2)); + // The owner path one node short (the last cap node dropped) or long. + assert!(rejected_after(&air, &honest, |p| { + main_path(0)(p).pop(); + })); + assert!(rejected_after(&air, &honest, |p| { + let path = main_path(0)(p); + path.push(path[0]); + })); + // A non-owner path carrying the cap too. + assert!(rejected_after(&air, &honest, |p| { + let cap: Vec = main_path(0)(p)[7..].to_vec(); + main_path(1)(p).extend(cap); + })); + // The cap moved from query 0 to query 1. + assert!(rejected_after(&air, &honest, |p| { + let cap: Vec = main_path(0)(p).split_off(7); + main_path(1)(p).extend(cap); + })); + // The same for a FRI layer. + assert!(rejected_after(&air, &honest, |p| { + let path = fri_path(0, 0); + let d = path(p).len() - 8; + let cap: Vec = path(p).split_off(d); + fri_path(1, 0)(p).extend(cap); + })); +} + +/// The cap height is a verifier constant: a proof made under one policy fails +/// under any other, in both directions. +#[test] +fn a_proof_made_under_one_policy_fails_under_another() { + let (_, fixed3) = prove(&options(CapPolicy::Fixed(3), 30, 2)); + let (air_off, off) = prove(&options(CapPolicy::Off, 30, 2)); + let air_at = |policy| SimpleAdditionAIR::::new(&options(policy, 30, 2)); + assert!(verifies(&air_at(CapPolicy::Fixed(3)), &fixed3)); + assert!(!verifies(&air_at(CapPolicy::Fixed(2)), &fixed3)); + assert!(!verifies(&air_at(CapPolicy::Fixed(4)), &fixed3)); + assert!(!verifies(&air_off, &fixed3)); + assert!(!verifies(&air_at(CapPolicy::Auto), &off)); + assert!(verifies(&air_off, &off)); +} + +// ---------------------------------------------------- M1 at the verifier level + +/// The transcript's index of query `q` of the main tree, recovered from its +/// capped opening (the only index whose fold lands on the cap), and the cap +/// node it reaches. +fn main_query_index(proof: &Proof, q: usize, d: usize, c: usize) -> (usize, usize) { + let owner = &proof.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path; + let cap = &owner[d - c..]; + let opening = &proof.deep_poly_openings[q].main_trace_polys; + let siblings = &opening.proof.merkle_path[..d - c]; + let leaf = Leaf::hash_data_from_slices(&opening.evaluations, &opening.evaluations_sym); + let hits: Vec = (0..1usize << d) + .filter(|&i| { + verify_merkle_path_from_leaf_hash::(siblings, &cap[i >> (d - c)], i, leaf) + }) + .collect(); + assert_eq!( + hits.len(), + 1, + "query {q}: exactly one index folds onto the cap" + ); + (hits[0], hits[0] >> (d - c)) +} + +/// M1(b): with 3 queries and a height-3 cap, at least 5 of the 8 main-tree cap +/// nodes are reached by no query. Flipping one leaves every per-query check +/// green (each query still folds onto its own, unchanged, cap node), so only +/// the cap-to-root check rejects the proof. Deleting `verify_cap` from +/// `CappedRoot::from_owner` makes this test fail. +#[test] +fn an_unreached_cap_node_is_rejected_by_the_cap_to_root_check_alone() { + let (air, honest) = prove(&options(CapPolicy::Fixed(3), 3, 2)); + let (d, c) = (10, 3); + assert!(verifies(&air, &honest)); + let reached: Vec = (0..3) + .map(|q| main_query_index(&honest, q, d, c).1) + .collect(); + let unreached: Vec = (0..8).filter(|k| !reached.contains(k)).collect(); + assert!(unreached.len() >= 5, "3 queries reach at most 3 of 8 nodes"); + for k in unreached { + let mut bad = honest.clone(); + main_path(0)(&mut bad)[d - c + k][3] ^= 1; + // Precondition: the per-query folds are untouched by the flip. + for (q, &node) in reached.iter().enumerate() { + assert_eq!(main_query_index(&bad, q, d, c).1, node); + } + assert!(!verifies(&air, &bad), "unreached cap node {k}"); + assert!( + !verifies_archived(&air, &bad), + "unreached cap node {k}: archived" + ); + } +} + +/// M1(a): the verifier's own per-tree check (`table_tree_checks`) refuses the +/// real internal node one level above a queried leaf, presented as a leaf hash +/// with the path from that node up — which the length-agnostic fold accepts. +/// Only the exact-length check stands between the two; deleting it from the +/// cap primitive makes this test fail. Run at the default (`c = 0`) and +/// under a cap. +#[test] +fn an_internal_node_passed_as_a_leaf_is_rejected_by_the_length_check_alone() { + for policy in [CapPolicy::Off, CapPolicy::Fixed(3)] { + let (air, proof) = prove(&options(policy, 3, 2)); + let c = if policy == CapPolicy::Off { 0 } else { 3 }; + let d = 10; + let view = StarkProofView::Owned(&proof); + let domain = new_verifier_domain(&air, proof.trace_length); + let checks = Verifier::table_tree_checks(&air, view, &domain).expect("honest shape"); + for q in 1..3 { + let iota = if c == 0 { + // Uncapped: fold against the root directly. + let opening = &proof.deep_poly_openings[q].main_trace_polys; + let leaf = + Leaf::hash_data_from_slices(&opening.evaluations, &opening.evaluations_sym); + (0..1usize << d) + .find(|&i| { + verify_merkle_path_from_leaf_hash::( + &opening.proof.merkle_path, + &proof.lde_trace_main_merkle_root, + i, + leaf, + ) + }) + .expect("the honest index") + } else { + main_query_index(&proof, q, d, c).0 + }; + let opening = &proof.deep_poly_openings[q].main_trace_polys; + let path = &opening.proof.merkle_path; + let leaf = Leaf::hash_data_from_slices(&opening.evaluations, &opening.evaluations_sym); + // The real node one level up, and the position it sits at. + let node = if iota & 1 == 0 { + Leaf::hash_new_parent(&leaf, &path[0]) + } else { + Leaf::hash_new_parent(&path[0], &leaf) + }; + let forged = &path[1..]; + let target = if c == 0 { + proof.lde_trace_main_merkle_root + } else { + proof.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path[d - c + (iota >> (d - c))] + }; + assert!( + verify_merkle_path_from_leaf_hash::(forged, &target, iota >> 1, node), + "{policy} q={q}: precondition, the fold alone accepts the forgery" + ); + assert!( + !checks.main.verify::(q, forged, iota >> 1, node), + "{policy} q={q}: an internal node passed for a leaf" + ); + // The honest opening passes the same check. + assert!(checks.main.verify::(q, path, iota, leaf)); + } + } +} + +// ------------------------------------------------------------- device trees + +/// A device-resident tree (a root-only host tree) whose cap has +/// no device read is a hard `Err` naming the tree — never a skipped cap, which +/// would ship full-length paths the verifier rejects with no pointer to the +/// cause. And a device read that fails is an `Err` too, not a panic. +#[test] +fn a_device_resident_tree_without_a_cap_read_is_an_error() { + use crate::prover::ProvingError; + use crypto::merkle_tree::merkle::MerkleTree; + type P = Prover; + let root_only = MerkleTree::::from_root([7u8; 32]); + match

>::tree_cap( + &root_only, + 10, + 3, + "main", + |_| None, + ) { + Err(ProvingError::DevicePath(msg)) => { + assert!( + msg.contains("main") && msg.contains("device-resident"), + "{msg}" + ) + } + other => panic!("expected a DevicePath error, got {other:?}"), + } + match

>::tree_cap( + &root_only, + 10, + 3, + "FRI layer 2", + |_| Some(Err("cudarc said no".to_string())), + ) { + Err(ProvingError::DevicePath(msg)) => { + assert!( + msg.contains("FRI layer 2") && msg.contains("cudarc said no"), + "{msg}" + ) + } + other => panic!("expected a DevicePath error, got {other:?}"), + } + // A host tree whose depth is not the format's is refused too. + let data: Vec> = (0..16u64) + .map(|i| vec![FE::from(i), FE::from(i + 1)]) + .collect(); + let host = MerkleTree::::build(&data).expect("tree"); + assert!( +

>::tree_cap(&host, 5, 2, "aux", |_| None) + .is_err() + ); + let cap = +

>::tree_cap(&host, 4, 2, "aux", |_| None) + .expect("a full host tree serves its cap"); + assert!(verify_cap::(&cap, &host.root, 2)); +} + +/// The device cap read on a real device (box only; `--features cuda -- --ignored`): a LogUp +/// table over the cubic extension, big enough that its main, aux, +/// composition and FRI trees are committed on the device (host trees +/// root-only), proved under `Auto` at 30 queries. The caps must come off the +/// device (`gpu_cap_read_calls` moves), the proof must verify owned and +/// archived, and against an `Off` proof of the same witness the transcript is +/// unchanged and every capped path is the full device-gathered path cut to +/// `D − c` (the cap on the owner). +#[cfg(feature = "cuda")] +#[test] +#[ignore = "requires a GPU; run with --features cuda -- --ignored"] +fn device_trees_serve_their_caps() { + // An `AirWithBuses` table: the device composition arm needs the AIR's + // constraint program, which the hand-written example AIRs do not supply + // (`LogReadOnlyRAP` here panicked in `constraint_program` on the box). + use crate::examples::bus_permutation::{bus_permutation_air, bus_permutation_trace}; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as E; + type Pi = (); + + let rows = 1usize << 14; + let prove_at = |policy| { + let opts = options(policy, 30, 2); + let mut trace = bus_permutation_trace(rows); + let air = bus_permutation_air(&opts); + let proof = Prover::prove(&air, &mut trace, &(), &mut DefaultTranscript::::new(&[])) + .expect("prove"); + (air, proof) + }; + + let (_, off) = prove_at(CapPolicy::Off); + let before = crate::gpu_lde::gpu_cap_read_calls(); + let (air, on) = prove_at(CapPolicy::Auto); + let reads = crate::gpu_lde::gpu_cap_read_calls() - before; + println!("CAPDEV device cap reads: {reads}"); + assert!( + reads > 0, + "no cap came off the device: the trees were host trees, the test proves nothing" + ); + assert!( + Verifier::verify(&on, &air, &mut DefaultTranscript::::new(&[])), + "a device-proved capped proof must verify" + ); + let multi = MultiProof { + proofs: vec![on.clone()], + }; + let bytes = rkyv::to_bytes::(&multi).unwrap(); + let archived = rkyv::access::< + crate::proof::stark::ArchivedMultiProof, + rkyv::rancor::Error, + >(&bytes) + .unwrap(); + let airs: Vec<&dyn AIR> = vec![&air]; + assert!(Verifier::multi_verify_archived( + &airs, + archived, + &mut DefaultTranscript::::new(&[]), + &FieldElement::::zero(), + )); + + assert_eq!( + off.lde_trace_main_merkle_root, + on.lde_trace_main_merkle_root + ); + assert_eq!(off.lde_trace_aux_merkle_root, on.lde_trace_aux_merkle_root); + assert_eq!(off.composition_poly_root, on.composition_poly_root); + assert_eq!(off.fri_layers_merkle_roots, on.fri_layers_merkle_roots); + assert_eq!(off.fri_final_poly_coeffs, on.fri_final_poly_coeffs); + let lde_log = (2 * on.trace_length).trailing_zeros() as usize; + let caps = StarkCaps::new( + CapPolicy::Auto, + 30, + lde_log, + on.fri_layers_merkle_roots.len(), + ); + assert_eq!(caps.trace, 3); + let check = |full: &Vec, capped: &Vec, q: usize, d: usize, c: usize| { + assert_eq!(full.len(), d, "query {q}: full path"); + assert_eq!(&capped[..d - c], &full[..d - c], "query {q}: siblings"); + assert_eq!( + capped.len(), + d - c + if q == 0 { 1 << c } else { 0 }, + "query {q}" + ); + }; + let d = caps.trace_depth; + for (q, (a, b)) in off + .deep_poly_openings + .iter() + .zip(&on.deep_poly_openings) + .enumerate() + { + check( + &a.main_trace_polys.proof.merkle_path, + &b.main_trace_polys.proof.merkle_path, + q, + d, + 3, + ); + check( + &a.composition_poly.proof.merkle_path, + &b.composition_poly.proof.merkle_path, + q, + d, + 3, + ); + let (aa, bb) = ( + a.aux_trace_polys.as_ref().unwrap(), + b.aux_trace_polys.as_ref().unwrap(), + ); + check(&aa.proof.merkle_path, &bb.proof.merkle_path, q, d, 3); + } + for (q, (a, b)) in off.query_list.iter().zip(&on.query_list).enumerate() { + for i in 0..caps.fri.len() { + check( + &a.layers_auth_paths[i].merkle_path, + &b.layers_auth_paths[i].merkle_path, + q, + caps.fri_depths[i], + caps.fri[i], + ); + } + } +} diff --git a/crypto/stark/src/tests/mod.rs b/crypto/stark/src/tests/mod.rs index 468a4cd3c..e121af519 100644 --- a/crypto/stark/src/tests/mod.rs +++ b/crypto/stark/src/tests/mod.rs @@ -1,19 +1,44 @@ pub mod air_tests; pub mod aux_opening_width_tests; +pub mod blake3_stark_roundtrip_tests; +pub mod block_commit_ood_tests; #[cfg(feature = "debug-checks")] pub mod bus_debug_tests; pub mod bus_tests; +pub mod cap_fri_matrix_tests; +#[cfg(feature = "cuda")] +pub mod commit_ood_device_tests; +pub mod commit_ood_tests; pub mod commitment_tests; +pub mod constraint_index_tests; +pub mod device_cap_tests; pub mod domain_cache_stats; +pub mod fri_group_tests; +pub mod fri_schedule_tests; pub mod fri_tests; pub mod grinding_tests; +pub mod log_read_only_program_tests; +pub mod merkle_cap_tests; +pub mod one_row_tests; pub mod opening_width_tests; +pub mod path_length_tests; +pub mod phase_b_overlap_tests; pub mod proof_options_tests; pub mod prove_verify_roundtrip_tests; pub mod prover_tests; +pub mod residency_mode_tests; pub mod row_pair_opening_tests; pub mod small_trace_tests; -#[cfg(feature = "disk-spill")] -pub mod table_disk_spill_tests; +#[cfg(feature = "cuda")] +pub mod subcoset_device_tests; +pub mod table_transpose_tests; pub mod terminal_tests; pub mod trace_test_helpers; +#[cfg(feature = "cuda")] +pub mod warm_device_tests; +#[cfg(feature = "cuda")] +pub mod zf_fri_device_tests; +pub mod zf_fri_vectors; +pub mod zf_golden_tests; +#[cfg(feature = "cuda")] +pub mod zf_s2_device_tests; diff --git a/crypto/stark/src/tests/one_row_tests.rs b/crypto/stark/src/tests/one_row_tests.rs new file mode 100644 index 000000000..ab5f384cd --- /dev/null +++ b/crypto/stark/src/tests/one_row_tests.rs @@ -0,0 +1,981 @@ +//! S2 (one-row trace openings with a committed FRI input) on the CPU prover +//! and host verifier: the round trip U6 at one_row, the tamper +//! tests T4–T6, the load-bearing mutation M3, the transcript-order KAT, the +//! per-table `auto` rule, the preprocessed-root +//! miss (a hard error, never a recompute) and the cap × FRI × one-row matrix. + +use std::sync::Mutex; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::merkle_tree::cap::CapPolicy; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsFFTField; + +use crate::config::{Blake3StarkHash, KeccakStarkHash}; +use crate::examples::fibonacci_2_columns::compute_trace; +use crate::examples::simple_fibonacci::FibonacciPublicInputs; +use crate::fri::capture::{FriCapture, capture}; +use crate::fri::fri_functions::compute_coset_twiddles_inv; +use crate::fri::schedule::FRI_COST_WEIGHTS; +use crate::fri::terminal::FriFoldLayout; +use crate::fri::{commit_phase_with_layout, fold_times}; +use crate::leaf_layout::{ + LeafLayout, M3_PAIR_BOUND_AT_LDE, TableWidths, deep_point_xalu_rows, resolve_leaf_layout, + table_leaf_layout, table_openings_cost_q, +}; +use crate::proof::options::{FriMode, FriScheduleOverride, OneRowMode, ProofFormat, ProofOptions}; +use crate::proof::stark::MultiProof; +use crate::prover::{IsStarkProver, Prover, ProvingError}; +use crate::tests::opening_width_tests::FibonacciSplitAIR; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +use super::zf_golden_tests::{ + golden_options, prove_logup, prove_multi, prove_simple_addition, verify_logup, verify_multi, + verify_simple_addition, +}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; +type Ext = FieldElement; + +/// Serialises the tests that flip the process-global M3 switch (see +/// `leaf_layout::M3_PAIR_BOUND_AT_LDE`). +static M3_LOCK: Mutex<()> = Mutex::new(()); + +fn fmt(one_row: OneRowMode, fri_mode: FriMode, schedule: Option<&[u8]>) -> ProofFormat { + ProofFormat { + one_row, + fri_mode, + fri_schedule_override: schedule.map(|s| FriScheduleOverride::new(s).unwrap()), + ..ProofFormat::DEFAULT + } +} + +fn on(fri_mode: FriMode) -> ProofFormat { + fmt(OneRowMode::On, fri_mode, None) +} + +// --------------------------------------------------------------------------- +// The layout helper: one place a query becomes rows. +// --------------------------------------------------------------------------- + +#[test] +fn query_rows_bounds_and_depths() { + use math::fft::bit_reversing::reverse_index; + for lde_log in 1..=12u32 { + let n = 1usize << lde_log; + assert_eq!(LeafLayout::RowPair.query_bound(n as u64), (n / 2) as u64); + assert_eq!(LeafLayout::Row.query_bound(n as u64), n as u64); + assert_eq!( + LeafLayout::RowPair.tree_depth(lde_log as usize), + lde_log as usize - 1 + ); + assert_eq!( + LeafLayout::Row.tree_depth(lde_log as usize), + lde_log as usize + ); + for q in 0..n / 2 { + assert_eq!( + LeafLayout::RowPair.query_rows(q, n), + ( + reverse_index(2 * q, n as u64), + Some(reverse_index(2 * q + 1, n as u64)) + ) + ); + } + for r in 0..n { + assert_eq!( + LeafLayout::Row.query_rows(r, n), + (reverse_index(r, n as u64), None) + ); + } + } +} + +/// No stray `2·iota(+1)` row arithmetic outside the helper in +/// the opening code of the prover and the verifier (the legacy FRI +/// zero-fold terminal check, which indexes the TERMINAL codeword by the pair, +/// is the one named exception). +#[test] +fn every_opening_site_goes_through_query_rows() { + let prover = include_str!("../prover.rs"); + let verifier = include_str!("../verifier.rs"); + for (name, src) in [("prover.rs", prover), ("verifier.rs", verifier)] { + for (i, line) in src.lines().enumerate() { + let code = line.split("//").next().unwrap_or(""); + let pairish = code.contains("* 2 + 1") || code.contains("*2+1"); + let terminal = code.contains(".get(iota * 2 + 1)"); + let point_helper = code.contains("let raw = iota * 2"); + assert!( + !pairish || terminal || point_helper, + "{name}:{}: row-pair arithmetic outside LeafLayout::query_rows: {line}", + i + 1 + ); + } + } +} + +// --------------------------------------------------------------------------- +// U6: round trips at one_row, every fold count, pair and dp FRI. +// --------------------------------------------------------------------------- + +fn check_shape_simple( + proof: &crate::proof::stark::StarkProof< + F, + F, + crate::examples::simple_addition::SimpleAdditionPublicInputs, + >, + lde_log: u32, + layout: &FriFoldLayout, +) { + assert_eq!(proof.fri_layers_merkle_roots.len(), layout.num_committed); + for (q, dec) in proof.query_list.iter().zip(&proof.deep_poly_openings) { + assert!(dec.main_trace_polys.evaluations_sym.is_empty()); + assert!(dec.composition_poly.evaluations_sym.is_empty()); + assert_eq!( + dec.main_trace_polys.proof.merkle_path.len(), + lde_log as usize + ); + assert_eq!( + dec.composition_poly.proof.merkle_path.len(), + lde_log as usize + ); + assert_eq!( + q.layers_evaluations_sym.len(), + layout.opened_values_per_query() + ); + } +} + +#[test] +fn one_row_round_trips_at_every_fold_count() { + // k = 1: total_folds = log2(rows) + blowup_log − (blowup_log + 1). + for blowup in [2u8, 4] { + for log_rows in 1..=10u32 { + for mode in [FriMode::Pair, FriMode::Dp] { + let rows = 1usize << log_rows; + let o = golden_options(blowup, 1, 9, on(mode)); + let (air, proof) = prove_simple_addition::(rows, &o); + assert!( + verify_simple_addition::(&air, &proof), + "rows {rows} blowup {blowup} {mode:?}" + ); + let lde_log = log_rows + blowup.trailing_zeros(); + let l = + FriFoldLayout::for_options(lde_log, blowup.trailing_zeros(), &o, true).unwrap(); + check_shape_simple(&proof, lde_log, &l); + if l.total_folds > 0 { + // The input tree is layer 0: one more committed layer + // than the row-pair chain has under the pair schedule. + assert_eq!( + l.schedule.iter().map(|&d| u32::from(d)).sum::(), + l.total_folds + ); + } + } + } + } +} + +#[test] +fn one_row_round_trips_under_explicit_schedules() { + // rows 2^9, blowup 4, k 1: lde_log 11, chain from 11 to T = 3: 8 bits. + for sched in [ + &[1u8, 3, 4][..], + &[3, 1, 3, 1], + &[2, 1, 2, 2, 1], + &[1, 1, 1, 1, 1, 1, 1, 1], + &[6, 2], + &[1, 6, 1], + &[4, 4], + ] { + let o = golden_options(4, 1, 9, fmt(OneRowMode::On, FriMode::Dp, Some(sched))); + let (air, proof) = prove_simple_addition::(512, &o); + assert!( + verify_simple_addition::(&air, &proof), + "{sched:?}" + ); + assert_eq!(proof.fri_layers_merkle_roots.len(), sched.len()); + assert_eq!( + proof.query_list[0].layers_evaluations_sym.len(), + sched.iter().map(|&d| 1usize << d).sum::() + ); + } + // An override that fits the row-pair chain (7 bits) but not the one-row + // chain (8 bits) is a proving error under one row, never a fallback. + let o = golden_options(4, 1, 9, fmt(OneRowMode::On, FriMode::Dp, Some(&[3, 4]))); + let air = crate::examples::simple_addition::SimpleAdditionAIR::::new(&o); + let mut trace = crate::examples::simple_addition::simple_addition_trace::(512); + let pi = crate::examples::simple_addition::SimpleAdditionPublicInputs { + a: Felt::from(1u64), + b: Felt::from(2u64), + }; + assert!( + crate::prover::GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ) + .is_err() + ); +} + +#[test] +fn one_row_round_trips_ext3_aux_and_multi_table() { + for (rows, blowup) in [(4usize, 2u8), (16, 2), (128, 4), (512, 2)] { + for format in [on(FriMode::Pair), on(FriMode::Dp)] { + let o = golden_options(blowup, 1, 7, format); + let (air, proof, _) = prove_logup::(rows, &o); + assert!(verify_logup::(&air, &proof), "rows {rows}"); + let lde_log = rows.trailing_zeros() + blowup.trailing_zeros(); + for dec in &proof.deep_poly_openings { + let aux = dec.aux_trace_polys.as_ref().expect("aux opening"); + assert!(aux.evaluations_sym.is_empty()); + assert_eq!(aux.proof.merkle_path.len(), lde_log as usize); + } + let (air, proof, _) = prove_logup::(rows, &o); + assert!( + verify_logup::(&air, &proof), + "keccak rows {rows}" + ); + } + } + for format in [ + on(FriMode::Pair), + on(FriMode::Dp), + fmt(OneRowMode::Auto, FriMode::Dp, None), + ] { + let o = golden_options(2, 1, 6, format); + let multi = prove_multi::(&o); + assert!(verify_multi::(&o, &multi), "{format:?}"); + } +} + +/// The archived (rkyv, read-in-place) verifier path verifies a one-row proof +/// too: the proof structs did not change, only the encoding of their vectors. +#[test] +fn one_row_verifies_archived() { + let o = golden_options(4, 2, 5, on(FriMode::Dp)); + let (air, proof) = prove_simple_addition::(256, &o); + let multi = MultiProof { + proofs: vec![proof.clone()], + }; + let bytes = rkyv::to_bytes::(&multi).unwrap(); + type Pi = crate::examples::simple_addition::SimpleAdditionPublicInputs; + let archived = rkyv::access::< + crate::proof::stark::ArchivedMultiProof, + rkyv::rancor::Error, + >(&bytes) + .unwrap(); + let airs: Vec<&dyn AIR> = vec![&air]; + assert!( + crate::verifier::GenericVerifier::::multi_verify_archived( + &airs, + archived, + &mut DefaultTranscript::::new(&[]), + &Felt::zero(), + ) + ); +} + +/// The layout is a verifier-side constant: a one-row proof does not verify +/// under row-pair options, nor a row-pair proof under one-row options. +#[test] +fn the_layout_is_a_verifier_constant() { + let one = golden_options(4, 1, 9, on(FriMode::Pair)); + let pair = golden_options(4, 1, 9, ProofFormat::DEFAULT); + let (one_air, one_proof) = prove_simple_addition::(1024, &one); + let (pair_air, pair_proof) = prove_simple_addition::(1024, &pair); + assert!(verify_simple_addition::( + &one_air, &one_proof + )); + assert!(verify_simple_addition::( + &pair_air, + &pair_proof + )); + assert!(!verify_simple_addition::( + &pair_air, &one_proof + )); + assert!(!verify_simple_addition::( + &one_air, + &pair_proof + )); +} + +// --------------------------------------------------------------------------- +// T4–T6: tamper tests on a one-row proof. +// --------------------------------------------------------------------------- + +#[test] +fn tampering_a_one_row_proof_is_rejected() { + let o = golden_options(4, 1, 5, fmt(OneRowMode::On, FriMode::Dp, Some(&[3, 2, 3]))); + let (air, honest, _) = prove_logup::(512, &o); + assert!(verify_logup::(&air, &honest)); + let bump = Ext::new([Felt::one(), Felt::zero(), Felt::zero()]); + let values = honest.query_list[0].layers_evaluations_sym.len(); + assert_eq!(values, 8 + 4 + 8); + + // T4: every value of query 0's input group (layer 0), the slot included — + // the input-slot check `group₀[slot] == DEEP(x_r)` and the group hash. + for i in 0..8 { + let mut p = honest.clone(); + p.query_list[0].layers_evaluations_sym[i] += bump; + assert!( + !verify_logup::(&air, &p), + "input group value {i}" + ); + } + // The input tree's root and a sibling of its path. + let mut p = honest.clone(); + p.fri_layers_merkle_roots[0][3] ^= 1; + assert!(!verify_logup::(&air, &p), "input root"); + let mut p = honest.clone(); + p.query_list[0].layers_auth_paths[0].merkle_path[0][0] ^= 1; + assert!(!verify_logup::(&air, &p), "input path"); + + // T5: a non-empty `evaluations_sym` under one row, for each tree — even + // one holding the honest value of the symmetric row. + let mut p = honest.clone(); + p.deep_poly_openings[0].main_trace_polys.evaluations_sym = + p.deep_poly_openings[0].main_trace_polys.evaluations.clone(); + assert!(!verify_logup::(&air, &p), "main sym"); + let mut p = honest.clone(); + p.deep_poly_openings[0].composition_poly.evaluations_sym = + p.deep_poly_openings[0].composition_poly.evaluations.clone(); + assert!( + !verify_logup::(&air, &p), + "composition sym" + ); + let mut p = honest.clone(); + let aux = p.deep_poly_openings[1].aux_trace_polys.as_mut().unwrap(); + aux.evaluations_sym = aux.evaluations.clone(); + assert!(!verify_logup::(&air, &p), "aux sym"); + + // T6: a trace value, an aux value and a composition value of one opening + // (each moves DEEP(x_r) and the leaf hash). + let mut p = honest.clone(); + p.deep_poly_openings[2].main_trace_polys.evaluations[0] += Felt::one(); + assert!(!verify_logup::(&air, &p), "main value"); + let mut p = honest.clone(); + p.deep_poly_openings[2] + .aux_trace_polys + .as_mut() + .unwrap() + .evaluations[0] += bump; + assert!(!verify_logup::(&air, &p), "aux value"); + let mut p = honest.clone(); + p.deep_poly_openings[2].composition_poly.evaluations[0] += bump; + assert!( + !verify_logup::(&air, &p), + "composition value" + ); + // A trace path one level short (the row-pair depth) and one long. + let mut p = honest.clone(); + p.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path + .pop(); + assert!( + !verify_logup::(&air, &p), + "short trace path" + ); + let mut p = honest.clone(); + p.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path + .push([0u8; 32]); + assert!( + !verify_logup::(&air, &p), + "long trace path" + ); + // The flat group vector one short / one long. + let mut p = honest.clone(); + p.query_list[0].layers_evaluations_sym.pop(); + assert!(!verify_logup::(&air, &p)); + let mut p = honest.clone(); + p.query_list[0].layers_evaluations_sym.push(Ext::zero()); + assert!(!verify_logup::(&air, &p)); + // A missing input layer. + let mut p = honest.clone(); + p.fri_layers_merkle_roots.remove(0); + assert!(!verify_logup::(&air, &p)); +} + +/// Zero folds under one row (`B ≤ T`): no layer, no challenge; the terminal +/// codeword IS the DEEP codeword and `terminal[r] == DEEP(x_r)` is the check. +#[test] +fn one_row_zero_fold_case() { + // rows 4, blowup 2, k 2: T = min(1 + 2, 3) = 3 = lde_log → no fold. + let o = golden_options(2, 2, 5, on(FriMode::Pair)); + let (air, proof) = prove_simple_addition::(4, &o); + assert!(verify_simple_addition::(&air, &proof)); + assert!(proof.fri_layers_merkle_roots.is_empty()); + assert_eq!(proof.fri_final_poly_coeffs.len(), 4); + let mut p = proof.clone(); + p.fri_final_poly_coeffs[1] += Felt::one(); + assert!(!verify_simple_addition::(&air, &p)); + let mut p = proof.clone(); + p.deep_poly_openings[0].main_trace_polys.evaluations[1] += Felt::one(); + assert!(!verify_simple_addition::(&air, &p)); +} + +// --------------------------------------------------------------------------- +// Soundness: r is uniform over ALL of D₀. M3 shows the test that says so +// is load-bearing. +// --------------------------------------------------------------------------- + +/// The query indexes the verifier draws for a one-row SimpleAddition proof of +/// `rows` rows at blowup 2 with `queries` queries (and whether it verifies). +fn one_row_iotas(rows: usize, queries: usize) -> (Vec, bool) { + let o = golden_options(2, 1, queries, on(FriMode::Pair)); + let (air, proof) = prove_simple_addition::(rows, &o); + let (ok, records) = capture(|| verify_simple_addition::(&air, &proof)); + let rec = FriCapture::::from_any(records[0].as_ref()).expect("one record"); + (rec.iotas.clone(), ok) +} + +/// The M3 shape: 4096 rows at blowup 2, an LDE of 8192 points no other +/// one-row test proves at (the mutation is keyed by it), and 64 queries: all +/// 64 indexes below `N / 2` has probability 2⁻⁶⁴ under the right bound; the +/// pair bound makes it certain. +const M3_ROWS: usize = 4096; +const M3_LDE: usize = 2 * M3_ROWS; + +fn upper_half_reached(iotas: &[usize], lde: usize) -> bool { + iotas.iter().any(|&r| r >= lde / 2) && iotas.iter().all(|&r| r < lde) +} + +#[test] +fn one_row_query_indexes_cover_the_whole_lde() { + let _g = M3_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let (iotas, ok) = one_row_iotas(M3_ROWS, 64); + assert!(ok); + assert!(upper_half_reached(&iotas, M3_LDE), "iotas {iotas:?}"); +} + +/// M3: sample r over N/2 under one row. Prover and verifier agree on the +/// mutated bound, so the proof still VERIFIES — the bias is invisible to +/// verification, and only the bound test catches it. +#[test] +fn m3_the_query_bound_test_is_load_bearing() { + let _g = M3_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + M3_PAIR_BOUND_AT_LDE.store(M3_LDE as u64, std::sync::atomic::Ordering::SeqCst); + let (iotas, ok) = one_row_iotas(M3_ROWS, 64); + M3_PAIR_BOUND_AT_LDE.store(0, std::sync::atomic::Ordering::SeqCst); + assert!(ok, "the mutated proof still verifies (both sides mutated)"); + assert!( + !upper_half_reached(&iotas, M3_LDE), + "under the mutation the bound test must fail" + ); +} + +// --------------------------------------------------------------------------- +// Soundness: the input root is absorbed before ζ₀ (transcript KAT). +// --------------------------------------------------------------------------- + +#[test] +fn input_root_is_absorbed_before_the_first_challenge() { + use crate::fri::group::roots_of_unity_table; + let o = Felt::from(3u64); + let lde_log = 10u32; + let n = 1usize << lde_log; + // A low-degree ext3 codeword (256 coefficients, blowup 4), bit-reversed. + let coeffs: Vec = (0..256u64) + .map(|i| Ext::new([Felt::from(i + 1), Felt::from(3 * i), Felt::from(7)])) + .collect(); + let poly = math::polynomial::Polynomial::new(&coeffs); + let mut cw = + math::polynomial::Polynomial::evaluate_offset_fft::(&poly, 4, Some(256), &o).unwrap(); + math::fft::bit_reversing::in_place_bit_reverse_permute(&mut cw); + // One row, schedule [2, 3, 3] from 10 to T = 2 + 0 = 2. + let layout = FriFoldLayout::from_schedule(lde_log, 2, 0, true, vec![2, 3, 3]).unwrap(); + let tw = compute_coset_twiddles_inv::(&o, n); + let mut t = DefaultTranscript::::new(&[9]); + let (_coeffs, layers) = commit_phase_with_layout::( + cw.clone(), + &mut t, + &o, + n, + 2, + 0, + &layout, + &tw, + ); + assert_eq!(layers.len(), 3); + assert_eq!( + layers[0].evaluation, cw, + "layer 0 is the DEEP codeword itself" + ); + + // The right order: root₀, then ζ₀. Folding layer 0 with that ζ₀ gives + // exactly the committed layer 1. + let mut right = DefaultTranscript::::new(&[9]); + right.append_bytes(&layers[0].merkle_tree.root); + let zeta0 = right.sample_field_element(); + let mut folded = cw.clone(); + let mut tw2 = tw.clone(); + fold_times(&mut folded, &zeta0, 2, &mut tw2); + assert_eq!(folded, layers[1].evaluation, "ζ₀ was drawn after root₀"); + + // The wrong order (ζ₀ before root₀) gives another challenge and another + // layer 1. + let mut wrong = DefaultTranscript::::new(&[9]); + let zeta_wrong = wrong.sample_field_element(); + assert_ne!(zeta_wrong, zeta0); + let mut folded = cw.clone(); + let mut tw2 = tw.clone(); + fold_times(&mut folded, &zeta_wrong, 2, &mut tw2); + assert_ne!(folded, layers[1].evaluation); + let _ = roots_of_unity_table::(1); +} + +/// M1 at the input tree: the input-slot check `group₀[slot] == DEEP(x_r)` is +/// what ties FRI to the trace openings under one row. A prover commits (and +/// folds) the input codeword `p₀ + c` — still low degree, so every layer and +/// the terminal are consistent — while DEEP(x_r) from the openings is `p₀`. +/// With the check the forgery is rejected; with it skipped (the mutation) it +/// is ACCEPTED. +#[test] +fn m1_the_input_slot_check_is_load_bearing() { + use crate::fri::group::{ + GROUP_MUTATION, GroupMutation, roots_of_unity_table, verify_query_groups, + }; + use crate::fri::query_phase_with_layout; + use crate::fri::terminal::terminal_codeword_from_coeffs; + use crate::merkle_caps::TreeCheck; + use math::fft::bit_reversing::{in_place_bit_reverse_permute, reverse_index}; + type H = KeccakStarkHash; + + let o = Felt::from(3u64); + let lde_log = 10u32; + let n = 1usize << lde_log; + let coeffs: Vec = (0..256u64) + .map(|i| Ext::new([Felt::from(i + 5), Felt::from(i * i), Felt::from(11)])) + .collect(); + let poly = math::polynomial::Polynomial::new(&coeffs); + let mut p0 = + math::polynomial::Polynomial::evaluate_offset_fft::(&poly, 4, Some(256), &o).unwrap(); + in_place_bit_reverse_permute(&mut p0); + let c = Ext::new([Felt::from(5u64), Felt::from(6u64), Felt::from(7u64)]); + let shifted: Vec = p0.iter().map(|v| v + &c).collect(); + + // One row, schedule [3, 2, 3] from 10 to T = 2 + 0. + let layout = FriFoldLayout::from_schedule(lde_log, 2, 0, true, vec![3, 2, 3]).unwrap(); + let tw = compute_coset_twiddles_inv::(&o, n); + let mut t = DefaultTranscript::::new(&[5]); + let (tcoeffs, layers) = + commit_phase_with_layout::(shifted, &mut t, &o, n, 2, 0, &layout, &tw); + let roots: Vec<[u8; 32]> = layers.iter().map(|l| l.merkle_tree.root).collect(); + // Replay: root₀ first, then (ζ, root) per later layer, then the final ζ. + let mut replay = DefaultTranscript::::new(&[5]); + let mut zetas = Vec::new(); + for (j, r) in roots.iter().enumerate() { + if j > 0 { + zetas.push(replay.sample_field_element()); + } + replay.append_bytes(r); + } + zetas.push(replay.sample_field_element()); + assert_eq!(zetas.len(), layout.num_zetas()); + let queries: Vec = (0..n).step_by(53).collect(); + let decs = query_phase_with_layout::(&layers, &queries, &layout); + let terminal = terminal_codeword_from_coeffs::( + &tcoeffs, + &o.pow(1u64 << layout.total_folds), + layout.terminal_len, + ); + let tables: Vec> = (0..=6) + .map(|d| roots_of_unity_table::(d).unwrap()) + .collect(); + let checks: Vec> = roots + .iter() + .enumerate() + .map(|(j, root)| { + TreeCheck::build::<::Batched>( + root, + layout.layer_depth(lde_log, j) as usize, + 0, + || None, + ) + .unwrap() + }) + .collect(); + let accepts = |deep: &[Ext]| { + queries.iter().zip(&decs).all(|(&r, dec)| { + let w = F::get_primitive_root_of_unity(u64::from(lde_log)).unwrap(); + let x_r = &o * w.pow(reverse_index(r, n as u64) as u64); + verify_query_groups::::Batched>( + &layout, + &checks, + 1, + |j| dec.layers_auth_paths[j].merkle_path.as_slice(), + &dec.layers_evaluations_sym, + &zetas, + r, + deep[r], + x_r.inv().unwrap(), + &terminal, + &tables, + ) + }) + }; + let shifted_again: Vec = p0.iter().map(|v| v + &c).collect(); + assert!(accepts(&shifted_again), "control: honest for p0 + c"); + assert!(!accepts(&p0), "the input-slot check must reject"); + GROUP_MUTATION.with(|m| m.set(GroupMutation::SkipSlotCheck)); + let mutated = accepts(&p0); + GROUP_MUTATION.with(|m| m.set(GroupMutation::None)); + assert!( + mutated, + "without the input-slot check the forgery is accepted (the check is load-bearing)" + ); +} + +// --------------------------------------------------------------------------- +// Preprocessed tables: one-row roots, and a miss is an error (never a recompute). +// --------------------------------------------------------------------------- + +#[test] +fn one_row_preprocessed_table_and_a_missing_root() { + let opts = golden_options(2, 1, 5, on(FriMode::Pair)); + let mut trace = compute_trace([Felt::one(), Felt::one()], 256); + let reference = FibonacciSplitAIR::::honest(&opts, None); + let pair_root = Prover::compute_precomputed_commitment_for_testing(&trace, &reference, 1) + .expect("row-pair root"); + let row_root = Prover::compute_precomputed_commitment_for_testing_with( + &trace, + &reference, + 1, + LeafLayout::Row, + ) + .expect("one-row root"); + assert_ne!( + pair_root, row_root, + "the two layouts commit different bytes" + ); + let pi = FibonacciPublicInputs { + a0: Felt::one(), + a1: Felt::one(), + }; + + // With both roots: proves and verifies, and the proof carries the ROW root. + let air = FibonacciSplitAIR::::preprocessed_declaring(&opts, None, 1, pair_root) + .with_one_row_commitment(row_root); + let proof = + Prover::prove(&air, &mut trace, &pi, &mut DefaultTranscript::::new(&[])).expect("prove"); + assert_eq!(proof.lde_trace_precomputed_merkle_root, Some(row_root)); + assert!(Verifier::verify( + &proof, + &air, + &mut DefaultTranscript::::new(&[]) + )); + + // The same AIR without a one-row root: the prover refuses with an Err + // (no panic, no recompute) and the verifier rejects the honest proof. + let bare = FibonacciSplitAIR::::preprocessed_declaring(&opts, None, 1, pair_root); + let mut trace2 = compute_trace([Felt::one(), Felt::one()], 256); + match Prover::prove( + &bare, + &mut trace2, + &pi, + &mut DefaultTranscript::::new(&[]), + ) { + Err(ProvingError::PrecomputedCommitmentMissing(_)) => {} + other => panic!( + "expected PrecomputedCommitmentMissing, got {:?}", + other.map(|_| ()) + ), + } + assert!(!Verifier::verify( + &proof, + &bare, + &mut DefaultTranscript::::new(&[]) + )); + + // A wrong one-row root: the prover's rebuilt tree disagrees. + let wrong = FibonacciSplitAIR::::preprocessed_declaring(&opts, None, 1, pair_root) + .with_one_row_commitment(pair_root); + let mut trace3 = compute_trace([Felt::one(), Felt::one()], 256); + assert!(matches!( + Prover::prove( + &wrong, + &mut trace3, + &pi, + &mut DefaultTranscript::::new(&[]) + ), + Err(ProvingError::PrecomputedCommitmentMismatch) + )); +} + +// --------------------------------------------------------------------------- +// The per-table `auto` rule. +// --------------------------------------------------------------------------- + +fn opts_q(q: usize, one_row: OneRowMode, fri: FriMode, cap: CapPolicy) -> ProofOptions { + let mut o = golden_options(4, 7, q, fmt(one_row, fri, None)); + o.format.merkle_cap = cap; + o +} + +/// The rule is the cost comparison, strictly: one row iff cheaper. +#[test] +fn auto_is_the_strict_cost_comparison() { + for fri in [FriMode::Pair, FriMode::Dp] { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + let o = opts_q(110, OneRowMode::Auto, fri, cap); + for lde_log in 4..=24u32 { + for main in [1u64, 4, 8, 30, 120, 400] { + for aux in [0u64, 3, 30, 120] { + let w = TableWidths { + precomputed: 0, + main, + aux, + composition: 6, + deep_point_rows: deep_point_xalu_rows(main + 2 * aux, 2, 2), + }; + let row = table_openings_cost_q(&w, &o, lde_log, 2, true); + let pair = table_openings_cost_q(&w, &o, lde_log, 2, false); + assert_eq!( + resolve_leaf_layout(&w, &o, lde_log, 2), + LeafLayout::from_one_row(row < pair), + "fri {fri:?} cap {cap:?} B {lde_log} main {main} aux {aux}" + ); + } + } + } + } + } + // Off and On ignore the costs. + let w = TableWidths { + precomputed: 0, + main: 1, + aux: 0, + composition: 3, + deep_point_rows: deep_point_xalu_rows(1, 1, 1), + }; + for lde_log in 4..=24 { + let off = opts_q(110, OneRowMode::Off, FriMode::Pair, CapPolicy::Off); + let on = opts_q(110, OneRowMode::On, FriMode::Pair, CapPolicy::Off); + assert_eq!( + resolve_leaf_layout(&w, &off, lde_log, 2), + LeafLayout::RowPair + ); + assert_eq!(resolve_leaf_layout(&w, &on, lde_log, 2), LeafLayout::Row); + } +} + +/// The DEEP term of the pinned cases: `E = 2` OOD rows (a current and a next +/// row; every case has aux columns, whose LogUp accumulators read the next +/// row), every column opened at the current row and the aux columns at the +/// next (an ASSUMED window: the real one is each AIR's +/// `trace_ood_next_row_columns`), `parts` composition parts. +fn pinned_deep_rows(pre: u64, main: u64, aux: u64, parts: u64) -> u64 { + deep_point_xalu_rows(pre + main + aux + aux, 2, parts) +} + +/// ⚠ A FORMAT PIN: `auto`'s choice for a set of production-like shapes (Q = +/// 110, blowup 4, k = 7, cap auto, fri dp). Wide tables go one-row, narrow +/// tall ones stay row pairs. Any change to the cost function or its weights +/// that moves one of these is a format change. The widths are illustrative +/// (MEMW 49 main / 13 aux; the others are round +/// numbers), not a census. With every emitted FRI row priced and DEEP at two +/// points vs one, two choices are one row only because of the DEEP term: the +/// MEMW-like case (one row by 5.5%, see `auto_choices_margins`) and the narrow short +/// preprocessed one (its LDE is already terminal, so no FRI layer separates +/// the layouts and the second DEEP point decides). +#[test] +fn auto_choices_are_pinned() { + let o = opts_q(110, OneRowMode::Auto, FriMode::Dp, CapPolicy::Auto); + // (name, B, precomputed, main, aux ext columns, composition parts, one row?) + let cases: &[(&str, u32, u64, u64, u64, u64, bool)] = &[ + ("wide keccak-like", 16, 0, 2600, 40, 2, true), + ("wide, short", 12, 0, 400, 20, 2, true), + ("narrow tall, preprocessed", 22, 12, 4, 2, 2, false), + ("narrow short, preprocessed", 7, 8, 1, 1, 2, true), + ("memw-like", 21, 0, 49, 13, 2, MEMW_LIKE_ONE_ROW), + ("cpu-like", 21, 0, 74, 20, 2, true), + ]; + let mut got = Vec::new(); + for &(name, b, pre, main, aux, parts, _) in cases { + got.push(( + name, + resolve_leaf_layout(&pinned_widths(pre, main, aux, parts), &o, b, 2).is_one_row(), + )); + } + let want: Vec<_> = cases.iter().map(|c| (c.0, c.6)).collect(); + assert_eq!(got, want); +} + +/// The MEMW-like case's pinned choice (see `auto_choices_are_pinned`). +const MEMW_LIKE_ONE_ROW: bool = true; + +fn pinned_widths(pre: u64, main: u64, aux: u64, parts: u64) -> TableWidths { + TableWidths { + precomputed: pre, + main, + aux: aux * 3, + composition: parts * 3, + deep_point_rows: pinned_deep_rows(pre, main, aux, parts), + } +} + +/// Prints each pinned case's two prices (`-- --nocapture`), and pins how much +/// of the one-row saving the DEEP term is at the two edge cases. +#[test] +fn auto_choices_margins() { + let o = opts_q(110, OneRowMode::Auto, FriMode::Dp, CapPolicy::Auto); + for (name, pre, main, aux, parts) in [ + ("memw-like", 0u64, 49u64, 13u64, 2u64), + ("cpu-like", 0, 74, 20, 2), + ] { + let w = pinned_widths(pre, main, aux, parts); + let row = table_openings_cost_q(&w, &o, 21, 2, true); + let pair = table_openings_cost_q(&w, &o, 21, 2, false); + let deep_point = 110 * w.deep_point_rows * FRI_COST_WEIGHTS.xalu; + println!( + " {name}: row {row} pair {pair} (x Q ns; one DEEP point {deep_point}; row/pair {:.4})", + row as f64 / pair as f64 + ); + assert!(row < pair, "{name} goes one row"); + // Without the DEEP point one row saves, the MEMW-like case would stay + // row pairs: the term decides it. + if name == "memw-like" { + assert!(row + deep_point >= pair, "{name}: the DEEP term decides"); + } + } +} + +/// DEEP costs two points under row pairs and one under one row, +/// each [`TableWidths::deep_point_rows`] XALU rows per query — and nothing +/// else in the price depends on it. +#[test] +fn the_deep_term_is_two_points_vs_one() { + for fri in [FriMode::Pair, FriMode::Dp] { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + let o = opts_q(110, OneRowMode::Auto, fri, cap); + let base = pinned_widths(0, 49, 13, 2); + let none = TableWidths { + deep_point_rows: 0, + ..base + }; + let point = 110 * base.deep_point_rows * FRI_COST_WEIGHTS.xalu; + for lde_log in 8..=22u32 { + for (one_row, points) in [(true, 1u64), (false, 2)] { + assert_eq!( + table_openings_cost_q(&base, &o, lde_log, 2, one_row), + table_openings_cost_q(&none, &o, lde_log, 2, one_row) + points * point, + "fri {fri:?} cap {cap:?} B {lde_log} one_row {one_row}" + ); + } + } + } + } + // The formula: one XALU row per surviving opening, plus 4 per OOD row, + // one per part and 3. + assert_eq!(deep_point_xalu_rows(100, 2, 2), 100 + 8 + 2 + 3); +} + +/// `auto` resolves per table from the AIR, and the prover and the verifier +/// resolve identically (one function, `table_leaf_layout`); a multi-table +/// proof can mix layouts. +#[test] +fn auto_resolves_per_table_from_the_air() { + let o = golden_options(2, 1, 6, fmt(OneRowMode::Auto, FriMode::Dp, None)); + let air = crate::examples::simple_addition::SimpleAdditionAIR::::new(&o); + for log_rows in 1..=20 { + let rows = 1usize << log_rows; + let w = TableWidths::of(&air, rows); + assert_eq!(w.main, air.trace_layout().0 as u64); + assert_eq!( + table_leaf_layout(&air, rows), + resolve_leaf_layout(&w, &o, log_rows + 1, 1) + ); + } + // At the default format every AIR is row pairs, whatever its widths. + let d = golden_options(2, 1, 6, ProofFormat::DEFAULT); + let air = crate::examples::simple_addition::SimpleAdditionAIR::::new(&d); + assert_eq!(table_leaf_layout(&air, 1 << 20), LeafLayout::RowPair); +} + +// --------------------------------------------------------------------------- +// The format matrix: {cap off, auto} × {pair, dp} × {0, 1, auto}, Q ≥ 20. +// --------------------------------------------------------------------------- + +#[test] +fn cap_fri_one_row_matrix_round_trips() { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + for fri in [FriMode::Pair, FriMode::Dp] { + for one_row in [OneRowMode::Off, OneRowMode::On, OneRowMode::Auto] { + let mut o = golden_options(4, 1, 24, fmt(one_row, fri, None)); + o.format.merkle_cap = cap; + let (air, proof, _) = prove_logup::(256, &o); + assert!( + verify_logup::(&air, &proof), + "cap {cap:?} fri {fri:?} one_row {one_row:?}" + ); + let (air, proof) = prove_simple_addition::(1024, &o); + assert!( + verify_simple_addition::(&air, &proof), + "simple cap {cap:?} fri {fri:?} one_row {one_row:?}" + ); + if cap == CapPolicy::Auto { + // Q = 24 ≥ 20: every tree deeper than 3 carries a cap of 3 + // at the end of query 0's path. + let lde_log = 12usize; + let layout = table_leaf_layout(&air, 1024); + let d = layout.tree_depth(lde_log); + assert_eq!( + proof.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path + .len(), + d - 3 + 8 + ); + assert_eq!( + proof.deep_poly_openings[1] + .main_trace_polys + .proof + .merkle_path + .len(), + d - 3 + ); + } + } + } + } +} + +/// The FRI layout the verifier builds for a table resolves the SAME layout the +/// prover used, for the base-field and the extension-field AIRs alike. +#[test] +fn widths_of_an_extension_air() { + let o = golden_options(2, 1, 6, on(FriMode::Pair)); + let air = crate::examples::read_only_memory_logup::LogReadOnlyRAP::::new(&o); + let w = TableWidths::of(&air, 64); + assert_eq!(w.aux, 3 * air.num_auxiliary_rap_columns() as u64); + assert_eq!(w.main, air.trace_layout().0 as u64); + assert!(w.composition.is_multiple_of(3) && w.composition > 0); + // The DEEP term from the AIR's own OOD layout (the verifier's reading). + let e = air.context().transition_offsets.len() * air.step_size(); + let ood = crate::ood::OodLayout::new( + air.context().trace_columns, + e, + air.step_size(), + air.trace_ood_next_row_columns(), + ); + assert_eq!( + w.deep_point_rows, + deep_point_xalu_rows(ood.num_surviving() as u64, e as u64, w.composition / 3) + ); + assert!(w.deep_point_rows > ood.num_surviving() as u64); + let _ = ::TWO_ADICITY; +} diff --git a/crypto/stark/src/tests/opening_width_tests.rs b/crypto/stark/src/tests/opening_width_tests.rs index db5764220..8d86da29a 100644 --- a/crypto/stark/src/tests/opening_width_tests.rs +++ b/crypto/stark/src/tests/opening_width_tests.rs @@ -71,12 +71,15 @@ pub struct FibonacciSplitAIR { out: Option>, precomputed_columns: usize, precomputed_commitment: Commitment, + /// The one-row (S2) root of the same precomputed columns; `None` = the + /// AIR has none (the one-row prover must refuse, never recompute). + precomputed_commitment_row: Option, phantom: PhantomData, } impl FibonacciSplitAIR { /// The AIR as the verifier sees it: plain, non-preprocessed. - fn honest(proof_options: &ProofOptions, out: Option>) -> Self { + pub(crate) fn honest(proof_options: &ProofOptions, out: Option>) -> Self { let mut air = ::new(proof_options); air.out = out; air @@ -96,7 +99,7 @@ impl FibonacciSplitAIR { /// Handing the verifier a different count than the prover used is how the /// hook-free test below reaches the precomputed term of the guard: both /// sides still absorb the same commitment, so the transcripts agree. - fn preprocessed_declaring( + pub(crate) fn preprocessed_declaring( proof_options: &ProofOptions, out: Option>, precomputed_columns: usize, @@ -107,6 +110,12 @@ impl FibonacciSplitAIR { air.precomputed_commitment = commitment; air } + + /// This AIR with a one-row (S2) precomputed root as well. + pub(crate) fn with_one_row_commitment(mut self, commitment: Commitment) -> Self { + self.precomputed_commitment_row = Some(commitment); + self + } } impl AIR for FibonacciSplitAIR @@ -135,6 +144,7 @@ where out: None, precomputed_columns: 0, precomputed_commitment: [0u8; 32], + precomputed_commitment_row: None, phantom: PhantomData, } } @@ -213,6 +223,16 @@ where fn precomputed_commitment(&self) -> Commitment { self.precomputed_commitment } + + fn precomputed_commitment_for( + &self, + layout: crate::leaf_layout::LeafLayout, + ) -> Option { + match layout { + crate::leaf_layout::LeafLayout::RowPair => Some(self.precomputed_commitment), + crate::leaf_layout::LeafLayout::Row => self.precomputed_commitment_row, + } + } } fn pub_inputs() -> FibonacciPublicInputs { diff --git a/crypto/stark/src/tests/path_length_tests.rs b/crypto/stark/src/tests/path_length_tests.rs new file mode 100644 index 000000000..7634d3bff --- /dev/null +++ b/crypto/stark/src/tests/path_length_tests.rs @@ -0,0 +1,142 @@ +//! Exact authentication-path lengths at the default format. +//! +//! Every tree's depth is a verifier constant: `log2(lde) − 1` for the trace, +//! precomputed, aux and composition trees (a leaf is a row pair), and +//! `log2(lde) − i − 2` for committed FRI layer `i` (pair leaves over +//! `lde / 2^(i+1)` values). The verifier requires the exact length, so a leaf +//! hash is never compared with an internal node. These tests pin that honest proofs meet +//! the lengths exactly and that a path one node short or long is rejected, for +//! each tree class the verifier walks. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +use crate::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::StarkProof; +use crate::prover::{IsStarkProver, Prover}; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +type F = GoldilocksField; +type FE = FieldElement; +type PI = SimpleAdditionPublicInputs; + +/// 1024 rows at blowup 2: `lde = 2048`, so the trace trees are 10 deep and +/// FRI commits layers (final degree 2^7 < 1024). +const TRACE_ROWS: usize = 1024; +const LDE_LOG: usize = 11; + +fn prove() -> (SimpleAdditionAIR, StarkProof) { + let options = ProofOptions::default_test_options(); + let air = SimpleAdditionAIR::::new(&options); + let pub_inputs = SimpleAdditionPublicInputs { + a: FE::from(1u64), + b: FE::from(2u64), + }; + let mut trace = simple_addition_trace::(TRACE_ROWS); + let proof = Prover::prove( + &air, + &mut trace, + &pub_inputs, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof) +} + +fn verifies(air: &SimpleAdditionAIR, proof: &StarkProof) -> bool { + Verifier::verify(proof, air, &mut DefaultTranscript::::new(&[])) +} + +#[test] +fn honest_paths_have_exactly_the_verifier_depths_and_verify() { + let (air, proof) = prove(); + assert_eq!( + air.options().blowup_factor as usize * TRACE_ROWS, + 1 << LDE_LOG + ); + assert!( + !proof.fri_layers_merkle_roots.is_empty(), + "the trace must fold, or the FRI arm is vacuous" + ); + for opening in &proof.deep_poly_openings { + assert_eq!( + opening.main_trace_polys.proof.merkle_path.len(), + LDE_LOG - 1 + ); + assert_eq!( + opening.composition_poly.proof.merkle_path.len(), + LDE_LOG - 1 + ); + } + for query in &proof.query_list { + for (i, path) in query.layers_auth_paths.iter().enumerate() { + assert_eq!(path.merkle_path.len(), LDE_LOG - i - 2, "layer {i}"); + } + } + assert!(verifies(&air, &proof), "an honest proof must verify"); +} + +/// One node short, one node long: both rejected. +fn assert_both_lengths_rejected( + air: &SimpleAdditionAIR, + honest: &StarkProof, + what: &str, + path_of: impl Fn(&mut StarkProof) -> &mut Vec<[u8; 32]>, +) { + let mut short = honest.clone(); + let path = path_of(&mut short); + assert!(!path.is_empty(), "{what}: precondition, a non-empty path"); + path.pop(); + assert!( + !verifies(air, &short), + "{what}: a path one node short must be rejected" + ); + + let mut long = honest.clone(); + let path = path_of(&mut long); + let extra = path[0]; + path.push(extra); + assert!( + !verifies(air, &long), + "{what}: a path one node long must be rejected" + ); +} + +#[test] +fn a_main_trace_path_of_the_wrong_length_is_rejected() { + let (air, honest) = prove(); + assert_both_lengths_rejected(&air, &honest, "main, query 0", |p| { + &mut p.deep_poly_openings[0].main_trace_polys.proof.merkle_path + }); + let last = honest.deep_poly_openings.len() - 1; + assert_both_lengths_rejected(&air, &honest, "main, last query", move |p| { + &mut p.deep_poly_openings[last] + .main_trace_polys + .proof + .merkle_path + }); +} + +#[test] +fn a_composition_path_of_the_wrong_length_is_rejected() { + let (air, honest) = prove(); + assert_both_lengths_rejected(&air, &honest, "composition, query 0", |p| { + &mut p.deep_poly_openings[0].composition_poly.proof.merkle_path + }); +} + +#[test] +fn a_fri_layer_path_of_the_wrong_length_is_rejected() { + let (air, honest) = prove(); + let layers = honest.fri_layers_merkle_roots.len(); + for layer in [0, layers - 1] { + assert_both_lengths_rejected(&air, &honest, &format!("FRI layer {layer}"), move |p| { + &mut p.query_list[0].layers_auth_paths[layer].merkle_path + }); + } +} diff --git a/crypto/stark/src/tests/phase_b_overlap_tests.rs b/crypto/stark/src/tests/phase_b_overlap_tests.rs new file mode 100644 index 000000000..147189c2d --- /dev/null +++ b/crypto/stark/src/tests/phase_b_overlap_tests.rs @@ -0,0 +1,815 @@ +//! Phase B's overlap ([`PhaseBOverlap`]) on a small block of the multi-table +//! example: each group's openings run beside the next group's argue and the +//! proof is the one phase B proves one group at a time, byte for byte, in any +//! order and under either argue format; the observer hears the same groups, +//! in the same order, with the same shares; a failure on either thread is the +//! prove's error, never a panic; a refusal turns the rest of phase B serial. +//! +//! The block and its verifier here are shared with +//! [`super::block_commit_ood_tests`]. + +use std::sync::Mutex; + +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; +use multilinear::{ + Error as MlError, + mle::Mle, + stacked_eval::StackedCommitment, + whir::Domain, + whir_chain::{ArgueFormat, ChainConfig, ChainFormat, GrindBits, GroupFold}, + whir_commit::TreeDrop, + whir_hash::{KeccakWhir, WhirHash}, +}; + +use crate::constraints::builder::EmptyConstraints; +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::lookup::{AirWithBuses, NullBoundaryConstraintBuilder}; +use crate::multilinear_air::Uniforms; +use crate::multilinear_block::{ + self, ArgueDeviation, BlockCommitted, BlockPrepared, BlockPreparedCheck, BlockProved, + GroupOpened, GroupStamps, Narrowing, OverlapFaults, PhaseBOverlap, PhaseBReorder, +}; +use crate::multilinear_table::{ + CommittedTable, TableLayout, TableStatement, batched::VerifierChecks, global_layout, + global_layouts, +}; +use crate::proof::options::ProofOptions; +use crate::traits::AIR; + +type FE = FieldElement; +type Air = AirWithBuses; +pub(super) type H = KeccakWhir; + +/// The example's three tables (CPU, ADD, MUL), this many times over: the bus +/// balances in each copy, so it balances over the block. +const COPIES: usize = 3; + +/// Tables a group: CPU | ADD MUL | CPU | ADD | MUL | CPU ADD | MUL. +pub(super) const SIZES: [usize; 7] = [1, 2, 1, 1, 1, 2, 1]; + +/// The prepared stacks: a group and its tables' prefixes, `(table, columns)`. +const PREPARED: [(usize, &[(usize, usize)]); 2] = [(1, &[(1, 1), (2, 2)]), (5, &[(6, 2), (7, 1)])]; + +fn config(argue: ArgueFormat) -> ChainConfig { + config_for(argue, GroupFold::PerPoly) +} + +/// The block's config under `argue`, each group's stacked polynomials opened +/// per `group`. +pub(super) fn config_for(argue: ArgueFormat, group: GroupFold) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: ChainFormat { + argue, + group, + ..ChainFormat::DEFAULT + }, + } +} + +fn airs() -> [Air; 3] { + let options = ProofOptions::default_test_options(); + [ + new_cpu_air_with_lookup(&options), + new_add_air_with_lookup(&options), + new_mul_air_with_lookup(&options), + ] +} + +fn base(values: &[u64]) -> Vec { + values.iter().map(|v| FE::from(*v)).collect() +} + +/// CPU (8 rows), ADD and MUL (4 rows each), as the multi-table example has +/// them. +fn columns() -> [Vec>; 3] { + [ + vec![ + base(&[1, 0, 1, 0, 1, 1, 0, 0]), + base(&[0, 1, 0, 1, 0, 0, 1, 1]), + base(&[1, 2, 3, 4, 5, 6, 7, 8]), + base(&[10, 20, 30, 40, 50, 60, 70, 80]), + base(&[11, 40, 33, 160, 55, 66, 490, 640]), + ], + vec![ + base(&[1, 3, 5, 6]), + base(&[10, 30, 50, 60]), + base(&[11, 33, 55, 66]), + base(&[1, 1, 1, 1]), + ], + vec![ + base(&[2, 4, 7, 8]), + base(&[20, 40, 70, 80]), + base(&[40, 160, 490, 640]), + base(&[1, 1, 1, 1]), + ], + ] +} + +/// Table `t` of the block: which of the three. +fn kind(t: usize) -> usize { + t % 3 +} + +/// Table `t`'s height in variables. +fn num_vars(t: usize) -> usize { + columns()[kind(t)][0].len().trailing_zeros() as usize +} + +fn tables(airs: &[Air; 3]) -> Vec> { + let columns = columns(); + (0..3 * COPIES) + .map(|t| { + let (air, cols) = (&airs[kind(t)], &columns[kind(t)]); + CommittedTable::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + cols.len(), + cols[0].len().trailing_zeros() as usize, + Uniforms::default(), + |col| cols[col as usize].clone(), + ) + .expect("a table") + }) + .collect() +} + +/// Each prepared stack's columns: its tables' prefixes, in its order. +fn prepared_columns() -> Vec>> { + let columns = columns(); + PREPARED + .iter() + .map(|(_, tables)| { + tables + .iter() + .flat_map(|&(t, n)| columns[kind(t)][..n].iter()) + .map(|values| Mle::new(values.clone()).expect("a column")) + .collect() + }) + .collect() +} + +/// Each prepared stack committed over its columns. +fn prepared_commitments( + columns: &[Vec>], + config: &ChainConfig, +) -> Vec> { + PREPARED + .iter() + .zip(columns) + .map(|((_, tables), columns)| { + let shapes: Vec<(usize, usize)> = + tables.iter().map(|&(t, n)| (n, num_vars(t))).collect(); + let layout = global_layout(&shapes, config.format.stack).expect("a layout"); + StackedCommitment::::commit( + layout, + &multilinear::stacking::borrow(columns), + None, + config, + ) + .expect("a prepared commitment") + }) + .collect() +} + +/// One group's share as the observer was handed it, serialized. +type Heard = (usize, Vec); + +/// The block proved under `overlap`, its groups taken in `order`, with a +/// test's `faults`: the proof and every group's stamps, or the prove's error, +/// and what the observer heard. +fn prove( + argue: ArgueFormat, + overlap: PhaseBOverlap, + order: &[usize], + faults: OverlapFaults, +) -> (Result, MlError>, Vec) { + prove_block(&config(argue), overlap, order, faults, &|_| {}) +} + +/// [`prove`] under `config`, phase A's block handed to `fault` before phase B +/// (a test's fault in it, or nothing). +pub(super) fn prove_block( + config: &ChainConfig, + overlap: PhaseBOverlap, + order: &[usize], + faults: OverlapFaults, + fault: &dyn Fn(&mut BlockCommitted<'_, Fp, Ext>), +) -> (Result, MlError>, Vec) { + prove_block_reordered(config, overlap, order, faults, fault, None) +} + +/// [`prove_block`] with phase B's re-ranking hook ([`PhaseBReorder`]). +fn prove_block_reordered( + config: &ChainConfig, + overlap: PhaseBOverlap, + order: &[usize], + faults: OverlapFaults, + fault: &dyn Fn(&mut BlockCommitted<'_, Fp, Ext>), + reorder: Option<&PhaseBReorder<'_>>, +) -> (Result, MlError>, Vec) { + let airs = airs(); + let config = *config; + let columns = prepared_columns(); + let commitments = prepared_commitments(&columns, &config); + let borrowed: Vec>> = columns + .iter() + .map(|c| multilinear::stacking::borrow(c)) + .collect(); + let prepared: Vec> = PREPARED + .iter() + .zip(&commitments) + .zip(&borrowed) + .map(|(((group, tables), commitment), columns)| BlockPrepared { + group: *group, + tables: tables.to_vec(), + commitment, + columns, + }) + .collect(); + let mut block = BlockCommitted::commit::::Sponge>( + tables(&airs), + &SIZES, + &config, + TreeDrop::uniform(1), + Narrowing::Wide, + false, + ) + .expect("phase A") + .with_phase_b_overlap(overlap) + .with_overlap_faults(faults); + fault(&mut block); + let heard: Mutex> = Mutex::new(Vec::new()); + let on_group = |opened: GroupOpened<'_, Fp, Ext>| { + let share = bincode::serialize(&( + opened.roots, + opened.commit_ood, + opened.argue, + opened.tables, + opened.opening, + opened.prepared, + )) + .expect("serialize"); + heard + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push((opened.group, share)); + }; + let mut transcript = H::sponge(); + let proved = multilinear_block::block_prove_in_order::( + block, + &config, + &mut transcript, + &prepared, + &[], + &|g| g, + &ArgueDeviation::default(), + &on_group, + order, + reorder, + ); + ( + proved, + heard.into_inner().unwrap_or_else(|e| e.into_inner()), + ) +} + +pub(super) fn bytes(proved: &BlockProved) -> Vec { + bincode::serialize(&(&proved.0, &proved.1, &proved.2, &proved.4)).expect("serialize") +} + +/// The verifier's reading of `proved`, against the shapes and the split it +/// rebuilds itself and the prepared stacks' roots. +fn verify(argue: ArgueFormat, proved: &BlockProved) -> Result<(), MlError> { + verify_block(&config(argue), proved, &proved.4, VerifierChecks::ALL) +} + +/// [`verify`] under `config`, reading `commit_ood` as the proof's commit-time +/// answers, with the verifier's `checks`. +pub(super) fn verify_block( + config: &ChainConfig, + proved: &BlockProved, + commit_ood: &[FieldElement], + checks: VerifierChecks, +) -> Result<(), MlError> { + let airs = airs(); + let config = *config; + let columns = columns(); + let prepared_at = |t: usize| { + PREPARED + .iter() + .flat_map(|(_, tables)| tables.iter()) + .find(|&&(table, _)| table == t) + .map(|&(_, n)| n) + }; + let layouts: Vec> = (0..3 * COPIES) + .map(|t| { + let (air, cols) = (&airs[kind(t)], &columns[kind(t)]); + TableLayout::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + cols.len(), + cols[0].len().trailing_zeros() as usize, + Uniforms::default(), + ) + .expect("a layout") + }) + .collect(); + let statements: Vec> = layouts + .iter() + .enumerate() + .map(|(t, layout)| match prepared_at(t) { + Some(n) => layout.statement_with_prepared_prefix(n), + None => layout.statement(), + }) + .collect(); + let shapes: Vec<(usize, usize)> = (0..3 * COPIES) + .map(|t| { + let cols = &columns[kind(t)]; + (cols.len(), cols[0].len().trailing_zeros() as usize) + }) + .collect(); + let stacks = global_layouts(&shapes, &SIZES, config.format.stack).expect("the stacks"); + let domains: Vec> = stacks + .iter() + .map(|layout| Domain::new(layout.n_stack() + config.log_blowup).expect("a domain")) + .collect(); + let prepared = prepared_columns(); + let commitments = prepared_commitments(&prepared, &config); + let roots: Vec> = + commitments.iter().map(StackedCommitment::roots).collect(); + let prepared_checks: Vec> = PREPARED + .iter() + .zip(&commitments) + .zip(&roots) + .map( + |(((group, tables), commitment), roots)| BlockPreparedCheck { + group: *group, + tables: tables.to_vec(), + roots, + layout: commitment.layout(), + domain: commitment.domain(), + }, + ) + .collect(); + let mut transcript = H::sponge(); + multilinear_block::block_verify_with::( + &proved.0, + commit_ood, + &proved.1, + &proved.2, + &prepared_checks, + &statements, + &stacks, + &domains, + &SIZES, + &FieldElement::::zero(), + &config, + &mut transcript, + false, + checks, + ) +} + +pub(super) fn in_order() -> Vec { + (0..SIZES.len()).collect() +} + +pub(super) const ARGUES: [ArgueFormat; 2] = [ArgueFormat::PerTable, ArgueFormat::BATCHED]; + +/// The block's proof in group order under each of [`ARGUES`], SHA3-256 of its +/// serialized bytes with its commit-time answers, as phase B proves it one +/// group at a time: re-pinned when every block proof gained those answers +/// (before them, at 76a950dc2, phase B proved the bytes the overlap was first +/// pinned against). One group at a time still proves these bytes. +const BEFORE_THE_OVERLAP: [&str; 2] = [ + "fef46d1082e57b1c835f97af901a8d6d6b5d8104e33f0d577af8482580284563", + "0fe883d9d0241ddf41a7eda3eece5adbe2b7be4af2440a6f248357cca5a3bec0", +]; + +fn digest(proved: &BlockProved) -> String { + use sha3::{Digest, Sha3_256}; + Sha3_256::digest(bytes(proved)) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +fn stamps(proved: &BlockProved) -> &[GroupStamps] { + &proved.3 +} + +/// ★ The overlap moves no byte: under either argue format, with each group's +/// openings beside the next group's argue, the groups taken in order, +/// reversed, or odd ones first, and with the openings' read-backs through +/// pinned staging ([`PhaseBOverlap::Pinned`]), the proof is the one phase B +/// proves one group at a time in group order — the bytes it proved before the +/// overlap existed ([`BEFORE_THE_OVERLAP`]) — and it verifies. Every group but the last one +/// taken opened beside an argue; the last opened alone. +#[test] +fn the_overlap_proves_the_bytes_of_one_group_at_a_time() { + let n = SIZES.len(); + let reversed: Vec = (0..n).rev().collect(); + let odd_first: Vec = (1..n).step_by(2).chain((0..n).step_by(2)).collect(); + for (argue, before) in ARGUES.into_iter().zip(BEFORE_THE_OVERLAP) { + let (serial, _) = prove( + argue, + PhaseBOverlap::Off, + &in_order(), + OverlapFaults::default(), + ); + let serial = serial.expect("one group at a time"); + assert_eq!(digest(&serial), before, "{argue:?}: one group at a time"); + verify(argue, &serial).expect("one group at a time verifies"); + for (what, order, overlap) in [ + ("in order", in_order(), PhaseBOverlap::On), + ("reversed", reversed.clone(), PhaseBOverlap::On), + ("odd first", odd_first.clone(), PhaseBOverlap::On), + ("in order, pinned", in_order(), PhaseBOverlap::Pinned), + ("reversed, pinned", reversed.clone(), PhaseBOverlap::Pinned), + ] { + let (proved, _) = prove(argue, overlap, &order, OverlapFaults::default()); + let proved = proved.unwrap_or_else(|e| panic!("{argue:?} {what}: {e:?}")); + assert_eq!( + bytes(&proved), + bytes(&serial), + "{argue:?} {what}: the proof" + ); + verify(argue, &proved).unwrap_or_else(|e| panic!("{argue:?} {what}: {e:?}")); + let s = stamps(&proved); + let last = *order.last().expect("a group"); + for &g in &order { + assert_eq!(s[g].overlapped, g != last, "{argue:?} {what}: group {g}"); + assert!(!s[g].serial_after_refusal, "{argue:?} {what}: group {g}"); + } + // Each transition's window is stamped on the group it argued: + // every group taken after the first (0 without a device). + assert_eq!( + s[order[0]].transition_reserved, 0, + "{argue:?} {what}: the first argue ran alone" + ); + } + } +} + +/// Off, phase B is one group at a time: no group opens beside an argue, none +/// is counted serial after a refusal, and the driver pays every opening. +#[test] +fn with_the_overlap_off_every_group_opens_alone() { + let (proved, _) = prove( + ArgueFormat::PerTable, + PhaseBOverlap::Off, + &in_order(), + OverlapFaults::default(), + ); + let proved = proved.expect("prove"); + for (g, s) in stamps(&proved).iter().enumerate() { + assert!(!s.overlapped, "group {g}"); + assert!(!s.serial_after_refusal, "group {g}"); + assert_eq!(s.transition_reserved, 0, "group {g}"); + assert!(s.open_paid >= s.open, "group {g}: paid {s:?}"); + } +} + +/// The observer hears each group once, in phase B's order, with its share of +/// the proof — the same shares with the overlap on and off — and each share +/// is the proof's. +#[test] +fn the_observer_hears_the_same_shares_in_order() { + let n = SIZES.len(); + let reversed: Vec = (0..n).rev().collect(); + for argue in ARGUES { + let (serial, heard_serial) = prove( + argue, + PhaseBOverlap::Off, + &reversed, + OverlapFaults::default(), + ); + let serial = serial.expect("prove"); + let (proved, heard) = prove( + argue, + PhaseBOverlap::On, + &reversed, + OverlapFaults::default(), + ); + proved.expect("prove"); + let groups: Vec = heard.iter().map(|(g, _)| *g).collect(); + assert_eq!(groups, reversed, "{argue:?}: the order heard"); + assert_eq!(heard, heard_serial, "{argue:?}: the shares heard"); + // Each share as the proof carries it. + let (proof, argues, prepared, _, answers) = &serial; + let mut table = 0usize; + let mut first_table = vec![0usize; n]; + for (g, &size) in SIZES.iter().enumerate() { + first_table[g] = table; + table += size; + } + for (g, share) in &heard { + let tables = match argue { + ArgueFormat::PerTable => { + &proof.tables[first_table[*g]..first_table[*g] + SIZES[*g]] + } + ArgueFormat::Batched { .. } => &proof.tables[..0], + }; + let k = PREPARED.iter().position(|(group, _)| group == g); + let expected = bincode::serialize(&( + &proof.roots, + answers, + argues.get(*g).filter(|_| argue != ArgueFormat::PerTable), + tables, + &proof.columns[*g], + k.map(|k| &prepared[k]), + )) + .expect("serialize"); + assert_eq!(share, &expected, "{argue:?}: group {g}'s share"); + } + } +} + +/// ★ A failure is the prove's error, never a panic: an opening that fails +/// beside an argue, an argue that fails beside an opening (the opening +/// thread joined first), both in one transition (the opening's, the earlier +/// group's), and a panic on the opening thread (refused, not resumed). The +/// failed opening is the same error with the overlap off. +#[test] +fn a_failure_beside_an_argue_is_the_proves_error() { + let failed_opening = MlError::DeviceFailed { + stage: "a test's failed opening", + }; + let failed_argue = MlError::DeviceFailed { + stage: "a test's failed argue", + }; + let panicked = MlError::DeviceFailed { + stage: "the previous group's openings", + }; + for (what, overlap, faults, expected, heard_groups) in [ + ( + "an opening fails", + PhaseBOverlap::On, + OverlapFaults { + open_fails: Some(1), + ..OverlapFaults::default() + }, + &failed_opening, + vec![0], + ), + ( + "an opening fails, one at a time", + PhaseBOverlap::Off, + OverlapFaults { + open_fails: Some(1), + ..OverlapFaults::default() + }, + &failed_opening, + vec![0], + ), + ( + "an argue fails", + PhaseBOverlap::On, + OverlapFaults { + argue_fails: Some(3), + ..OverlapFaults::default() + }, + &failed_argue, + vec![0, 1, 2], + ), + ( + "both, in one transition", + PhaseBOverlap::On, + OverlapFaults { + open_fails: Some(2), + argue_fails: Some(3), + ..OverlapFaults::default() + }, + &failed_opening, + vec![0, 1], + ), + ( + "the opening thread panics", + PhaseBOverlap::On, + OverlapFaults { + open_panics: Some(4), + ..OverlapFaults::default() + }, + &panicked, + vec![0, 1, 2, 3], + ), + ] { + let (proved, heard) = prove(ArgueFormat::PerTable, overlap, &in_order(), faults); + let err = proved.expect_err(what); + assert_eq!(format!("{err:?}"), format!("{expected:?}"), "{what}"); + // The groups opened before the failure were heard, and no other. + let heard: Vec = heard.iter().map(|(g, _)| *g).collect(); + assert_eq!(heard, heard_groups, "{what}"); + } +} + +/// An opening thread slower than the argue beside it moves no byte: what the +/// driver waits for is paid, at least the stall less that argue and the +/// upload beside it. +#[test] +fn a_stalled_opening_thread_is_paid_and_moves_no_byte() { + let stall = std::time::Duration::from_millis(150); + let (serial, _) = prove( + ArgueFormat::PerTable, + PhaseBOverlap::Off, + &in_order(), + OverlapFaults::default(), + ); + let (proved, _) = prove( + ArgueFormat::PerTable, + PhaseBOverlap::On, + &in_order(), + OverlapFaults { + open_stall: Some(stall), + ..OverlapFaults::default() + }, + ); + let (serial, proved) = (serial.expect("prove"), proved.expect("prove")); + assert_eq!(bytes(&proved), bytes(&serial)); + let s = stamps(&proved); + let order = in_order(); + for w in order.windows(2) { + let (h, g) = (w[0], w[1]); + assert!(s[h].overlapped, "group {h}"); + // The opening thread starts as the argue does, so the stall ends + // within the argue, the upload's wait after it (no card here: none) + // and what the driver paid. + let covered = s[h].open_paid + s[g].argue + 0.05; + assert!( + covered >= stall.as_secs_f64(), + "group {h}: paid {:.3} s beside an argue of {:.3} s", + s[h].open_paid, + s[g].argue + ); + } +} + +/// ★ A refusal turns the rest of phase B serial: the transition that sees one +/// keeps its overlap, every later group but the last opens alone and is +/// counted, and the proof is the same. +#[test] +fn a_refusal_turns_the_rest_of_phase_b_serial() { + let n = SIZES.len(); + let (serial, _) = prove( + ArgueFormat::PerTable, + PhaseBOverlap::Off, + &in_order(), + OverlapFaults::default(), + ); + let (proved, _) = prove( + ArgueFormat::PerTable, + PhaseBOverlap::On, + &in_order(), + OverlapFaults { + refusal_at: Some(2), + ..OverlapFaults::default() + }, + ); + let (serial, proved) = (serial.expect("prove"), proved.expect("prove")); + assert_eq!(bytes(&proved), bytes(&serial)); + let s = stamps(&proved); + // Groups 0 and 1 opened beside the argues at places 1 and 2; the refusal + // at place 2 sends groups 2..n-1 to the driver, the last one as always. + let overlapped: Vec = (0..n).filter(|&g| s[g].overlapped).collect(); + let serial_after: Vec = (0..n).filter(|&g| s[g].serial_after_refusal).collect(); + assert_eq!(overlapped, vec![0, 1]); + assert_eq!(serial_after, (2..n - 1).collect::>()); +} + +/// The order phase B takes under `reorder`, from group order: each turn's +/// re-ranking applied as the prover applies it. +fn taken_under(reorder: &PhaseBReorder<'_>) -> Vec { + let mut order = in_order(); + for p in 0..order.len() { + if let Some(rest) = reorder(p, &order) { + order[p + 1..].copy_from_slice(&rest); + } + } + order +} + +/// ★ Re-ranking the groups still to come while phase B runs moves no byte: +/// under either argue format and with the overlap off, on or pinned, a hook +/// that reverses what is left at one turn, at every turn, or swaps the last +/// two late proves the bytes of one group at a time in group order (the pinned +/// [`BEFORE_THE_OVERLAP`]) and verifies — and phase B did take the re-ranked +/// order (the observer hears it), so the hook is not a no-op. +#[test] +fn a_re_ranked_phase_b_proves_the_bytes_of_group_order() { + let n = SIZES.len(); + let at_turn_one = |p: usize, order: &[usize]| -> Option> { + (p == 1).then(|| order[p + 1..].iter().rev().copied().collect()) + }; + let every_turn = + |p: usize, order: &[usize]| Some(order[p + 1..].iter().rev().copied().collect()); + let late_swap = |p: usize, order: &[usize]| -> Option> { + (p + 3 == order.len()).then(|| { + let mut rest = order[p + 1..].to_vec(); + rest.swap(0, 1); + rest + }) + }; + let hooks: [(&str, &PhaseBReorder<'_>); 3] = [ + ("reversed at turn one", &at_turn_one), + ("reversed at every turn", &every_turn), + ("the last two swapped late", &late_swap), + ]; + for (argue, before) in ARGUES.into_iter().zip(BEFORE_THE_OVERLAP) { + for (what, hook) in hooks { + let taken = taken_under(hook); + assert_ne!(taken, in_order(), "{what}: the hook moves a group"); + for overlap in [PhaseBOverlap::Off, PhaseBOverlap::On, PhaseBOverlap::Pinned] { + let (proved, heard) = prove_block_reordered( + &config(argue), + overlap, + &in_order(), + OverlapFaults::default(), + &|_| {}, + Some(hook), + ); + let proved = + proved.unwrap_or_else(|e| panic!("{argue:?} {what} {overlap:?}: {e:?}")); + assert_eq!(digest(&proved), before, "{argue:?} {what} {overlap:?}"); + verify(argue, &proved) + .unwrap_or_else(|e| panic!("{argue:?} {what} {overlap:?}: {e:?}")); + let groups: Vec = heard.iter().map(|(g, _)| *g).collect(); + assert_eq!( + groups, taken, + "{argue:?} {what} {overlap:?}: the order taken" + ); + assert_eq!(groups.len(), n); + } + } + } +} + +/// A re-ranking that is not a permutation of the groups still to come — one +/// named twice, one left out, one already taken, one extra, one that does not +/// exist — is the prove's error, never a panic, with the overlap off, on or +/// pinned. +#[test] +fn a_re_ranking_that_is_not_a_permutation_is_refused() { + let n = SIZES.len(); + let twice = |p: usize, order: &[usize]| -> Option> { + (p == 1).then(|| { + let mut rest = order[p + 1..].to_vec(); + rest[1] = rest[0]; + rest + }) + }; + let left_out = |p: usize, order: &[usize]| -> Option> { + (p == 1).then(|| order[p + 2..].to_vec()) + }; + let taken = |p: usize, order: &[usize]| -> Option> { + (p == 1).then(|| { + let mut rest = order[p + 1..].to_vec(); + rest[0] = order[p]; + rest + }) + }; + let extra = |p: usize, order: &[usize]| -> Option> { + (p == 1).then(|| { + let mut rest = order[p + 1..].to_vec(); + rest.push(order[0]); + rest + }) + }; + let foreign = |p: usize, order: &[usize]| -> Option> { + (p == 1).then(|| { + let mut rest = order[p + 1..].to_vec(); + rest[0] = n; + rest + }) + }; + let hooks: [(&str, &PhaseBReorder<'_>); 5] = [ + ("a group twice", &twice), + ("a group left out", &left_out), + ("a group already taken", &taken), + ("a group extra", &extra), + ("no such group", &foreign), + ]; + for (what, hook) in hooks { + for overlap in [PhaseBOverlap::Off, PhaseBOverlap::On, PhaseBOverlap::Pinned] { + let (proved, _) = prove_block_reordered( + &config(ArgueFormat::PerTable), + overlap, + &in_order(), + OverlapFaults::default(), + &|_| {}, + Some(hook), + ); + assert!(proved.is_err(), "{what} {overlap:?}: proved"); + } + } +} diff --git a/crypto/stark/src/tests/prover_tests.rs b/crypto/stark/src/tests/prover_tests.rs index 1fe37f8a2..e930edbd8 100644 --- a/crypto/stark/src/tests/prover_tests.rs +++ b/crypto/stark/src/tests/prover_tests.rs @@ -72,6 +72,7 @@ fn test_domain_constructor() { coset_offset, grinding_factor, fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, }; let domain = Domain::new( @@ -163,6 +164,7 @@ fn barycentric_trace_eval_matches_horner_trace_eval() { coset_offset, grinding_factor: 0, fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, }; let air = simple_fibonacci::FibonacciAIR::::new(&proof_options); @@ -235,6 +237,7 @@ fn test_decompose_and_extend_d2_matches_original() { coset_offset: 3, grinding_factor: 0, fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, }; // We need an AIR with composition_poly_degree_bound = 2 * trace_length. @@ -276,6 +279,7 @@ fn test_decompose_and_extend_d2_matches_original() { &constraint_evaluations, &domain, &twiddles, + 1, ); #[cfg(not(feature = "cuda"))] assert!(twiddles.has_composition_cache()); @@ -289,6 +293,88 @@ fn test_decompose_and_extend_d2_matches_original() { } } +/// The sub-coset split: with blowup ≥ 4, `H` evaluated only at every +/// `blowup/2`-th LDE point (the `2n`-point sub-coset) must decompose and extend +/// to exactly the parts the whole-coset split produces, and both must equal the +/// iFFT + `break_in_parts` + LDE reference. This is the identity the prover's +/// sub-coset composition evaluation rests on (same parts ⇒ same proof bytes). +#[test] +fn decompose_and_extend_d2_subcoset_matches_whole_coset() { + for (n, blowup_factor) in [(16usize, 4usize), (32, 4), (16, 8)] { + let lde_size = n * blowup_factor; + let proof_options = ProofOptions { + blowup_factor: blowup_factor as u8, + fri_number_of_queries: 1, + coset_offset: 7, + grinding_factor: 0, + fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, + }; + let air = QuadraticAIR::::new(&proof_options); + let domain = Domain::new(&air, n); + assert_eq!(domain.lde_roots_of_unity_coset.len(), lde_size); + + // H of degree < 2n, evaluated on the whole LDE coset. + let coeffs: Vec = (0..2 * n) + .map(|i| FieldElement::from((i as u64) * 0x9E37_79B9 + 11)) + .collect(); + let h_poly = Polynomial::new(&coeffs); + let full: Vec = domain + .lde_roots_of_unity_coset + .iter() + .map(|x| h_poly.evaluate(x)) + .collect(); + + // Reference: interpolate, break in two parts, evaluate each on the LDE. + let reference: Vec> = h_poly + .break_in_parts(2) + .iter() + .map(|part| { + evaluate_polynomial_on_lde_domain(part, blowup_factor, n, &domain.coset_offset) + .expect("LDE evaluation failed") + }) + .collect(); + + let twiddles = LdeTwiddles::new(&domain); + let whole = Prover::::decompose_and_extend_d2( + &full, &domain, &twiddles, 1, + ); + let stride = blowup_factor / 2; + let sub: Vec = full.iter().step_by(stride).cloned().collect(); + assert_eq!(sub.len(), 2 * n); + let subcoset = Prover::::decompose_and_extend_d2( + &sub, &domain, &twiddles, stride, + ); + + assert_eq!( + whole, reference, + "whole-coset split != reference (n={n}, b={blowup_factor})" + ); + assert_eq!( + subcoset, reference, + "sub-coset split != reference (n={n}, b={blowup_factor})" + ); + } +} + +/// The stride rule: the sub-coset is taken only for two parts at blowup ≥ 4, +/// at `blowup/2`; every other shape keeps the whole coset (stride 1). +#[test] +fn composition_eval_stride_rule() { + use crate::prover::composition_eval_stride; + // Read under the default (environment) setting, which these tests never + // override: on unless `LAMBDA_VM_COMPOSITION_SUBCOSET=0`. + if !crate::prover::composition_subcoset_enabled() { + return; + } + assert_eq!(composition_eval_stride(4, 2), 2); + assert_eq!(composition_eval_stride(8, 2), 4); + assert_eq!(composition_eval_stride(2, 2), 1); + assert_eq!(composition_eval_stride(4, 1), 1); + assert_eq!(composition_eval_stride(4, 3), 1); + assert_eq!(composition_eval_stride(16, 4), 1); +} + /// Test that the domain cache 3-tuple key `(trace_length, blowup, coset_offset)` correctly /// distinguishes AIRs that share the same `(trace_length, blowup)` but differ in /// `coset_offset`. Both AIRs must get their own `Domain` and the resulting proofs must @@ -301,6 +387,7 @@ fn test_multi_prove_mixed_coset_offsets() { coset_offset: 3, grinding_factor: 1, fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, }; let proof_options_7 = ProofOptions { blowup_factor: 2, @@ -308,6 +395,7 @@ fn test_multi_prove_mixed_coset_offsets() { coset_offset: 7, grinding_factor: 1, fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, }; // Both AIRs have the same trace length and blowup, but different coset offsets. @@ -373,6 +461,7 @@ fn test_multi_prove_dedups_shared_domain_params() { coset_offset: 3, grinding_factor: 1, fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, }; let mut trace_1 = simple_fibonacci::fibonacci_trace([Felt::from(1), Felt::from(1)], 8); @@ -436,6 +525,68 @@ fn test_multi_prove_dedups_shared_domain_params() { ); } +/// ★ A warmed domain is the entry a later prove's lookup finds, and it is the +/// domain a cold build gives, value for value; a length no trace has warms +/// nothing. The key (blowup 2, coset offset 7, 32 rows) is this test's alone. +#[test] +fn a_warmed_domain_is_the_one_a_prove_finds() { + use crate::prover::{domain_and_twiddles, warm_domain_twiddles}; + let options = ProofOptions { + blowup_factor: 2, + fri_number_of_queries: 3, + coset_offset: 7, + grinding_factor: 1, + fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, + }; + let air = FibonacciAIR::::new(&options); + assert_eq!( + warm_domain_twiddles::(2, 7, 0), + None, + "no rows" + ); + assert_eq!( + warm_domain_twiddles::(2, 7, 24), + None, + "not a power of two" + ); + // The entry's tables: the LDE coset (n·blowup) and trace roots (n), the + // two-half sets (n and n·blowup) and the coset weights (n), 8 bytes each. + assert_eq!( + warm_domain_twiddles::(2, 7, 32), + Some(32 * (2 * 2 + 3) * 8), + "built" + ); + assert_eq!( + warm_domain_twiddles::(2, 7, 32), + None, + "already there" + ); + + domain_cache_stats::reset(); + let (first, _) = domain_and_twiddles(&air, 32); + let (again, _) = domain_and_twiddles(&air, 32); + assert_eq!( + domain_cache_stats::get(), + (2, 0), + "both lookups hit the warmed entry" + ); + assert!(std::sync::Arc::ptr_eq(&first, &again)); + + let cold = Domain::new(&air, 32); + assert_eq!( + first.lde_roots_of_unity_coset, + cold.lde_roots_of_unity_coset + ); + assert_eq!(first.trace_roots_of_unity, cold.trace_roots_of_unity); + assert_eq!(first.trace_primitive_root, cold.trace_primitive_root); + assert_eq!(first.coset_offset, cold.coset_offset); + assert_eq!( + (first.blowup_factor, first.interpolation_domain_size), + (cold.blowup_factor, cold.interpolation_domain_size) + ); +} + /// Differential test for the DEEP composition polynomial "direct 2N" evaluation. /// /// After this PR, `compute_deep_composition_poly_evaluations` evaluates the DEEP @@ -463,6 +614,7 @@ fn test_deep_poly_direct_2n_matches_interpolate_fft_extend() { coset_offset: 3, grinding_factor: 0, fri_final_poly_log_degree: 7, + format: crate::proof::options::ProofFormat::DEFAULT, }; let air = QuadraticAIR::::new(&proof_options); diff --git a/crypto/stark/src/tests/residency_mode_tests.rs b/crypto/stark/src/tests/residency_mode_tests.rs new file mode 100644 index 000000000..f00af4edd --- /dev/null +++ b/crypto/stark/src/tests/residency_mode_tests.rs @@ -0,0 +1,253 @@ +//! `ResidencyMode` equivalence: dropping each main LDE after Round 1 and +//! recomputing it inside the table's fused task changes nothing a verifier can +//! see. +//! +//! The oracle is the whole proof, not just the roots. Comparing serialized +//! proof bytes is normally avoided — a committed golden blob turns every +//! legitimate format change into a test failure — but there is no golden blob +//! here: both sides are produced in this process from the same traces, and the +//! only difference between them is the mode. That makes byte equality the +//! sharpest available statement of "invisible to the proof", and it is the same +//! oracle the closed streaming-prover PR (#647) used for the same change. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField, goldilocks::GoldilocksField, +}; + +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::MultiProof; +use crate::prover::{IsStarkProver, Prover}; +use crate::residency_mode::ResidencyMode; +use crate::trace::TraceTable; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type FE = FieldElement; + +/// The bus-balanced CPU/ADD/MUL instance from the completeness tests. Rebuilt +/// per prove because `multi_prove` writes the LogUp aux columns into the caller's +/// traces — and under `RecomputeLde` frees them again. +pub(super) fn traces() -> (TraceTable, TraceTable, TraceTable) { + let cpu = TraceTable::from_columns_main( + vec![ + vec![ + FE::one(), + FE::zero(), + FE::one(), + FE::zero(), + FE::one(), + FE::one(), + FE::zero(), + FE::zero(), + ], + vec![ + FE::zero(), + FE::one(), + FE::zero(), + FE::one(), + FE::zero(), + FE::zero(), + FE::one(), + FE::one(), + ], + (1..=8).map(FE::from).collect(), + (1..=8).map(|i| FE::from(i * 10)).collect(), + vec![ + FE::from(11), + FE::from(40), + FE::from(33), + FE::from(160), + FE::from(55), + FE::from(66), + FE::from(490), + FE::from(640), + ], + ], + 1, + ); + let add = TraceTable::from_columns_main( + vec![ + vec![FE::from(1), FE::from(3), FE::from(5), FE::from(6)], + vec![FE::from(10), FE::from(30), FE::from(50), FE::from(60)], + vec![FE::from(11), FE::from(33), FE::from(55), FE::from(66)], + vec![FE::one(); 4], + ], + 1, + ); + let mul = TraceTable::from_columns_main( + vec![ + vec![FE::from(2), FE::from(4), FE::from(7), FE::from(8)], + vec![FE::from(20), FE::from(40), FE::from(70), FE::from(80)], + vec![FE::from(40), FE::from(160), FE::from(490), FE::from(640)], + vec![FE::one(); 4], + ], + 1, + ); + (cpu, add, mul) +} + +/// The test options with grinding off. Under `parallel` the nonce search is +/// rayon's `find_any`, so two proves of the same instance can return different +/// valid nonces, and the nonce is absorbed before the queries are drawn: a +/// ground proof is not byte-reproducible, whatever the residency mode (the +/// reason `zf_golden_tests` proves at `grinding_factor = 0` too). The mode acts +/// on the main LDE, which the grind never reads, so nothing it could move is +/// lost. +fn test_options() -> ProofOptions { + ProofOptions { + grinding_factor: 0, + ..ProofOptions::default_test_options() + } +} + +fn prove_under(residency: ResidencyMode) -> MultiProof { + let (mut cpu_trace, mut add_trace, mut mul_trace) = traces(); + let proof_options = test_options(); + let cpu_air = new_cpu_air_with_lookup(&proof_options); + let add_air = new_add_air_with_lookup(&proof_options); + let mul_air = new_mul_air_with_lookup(&proof_options); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu_trace, &()), + (&add_air, &mut add_trace, &()), + (&mul_air, &mut mul_trace, &()), + ]; + + Prover::multi_prove(pairs, &mut DefaultTranscript::::new(&[]), residency).unwrap() +} + +fn verifies(proof: &MultiProof) -> bool { + let proof_options = test_options(); + let cpu_air = new_cpu_air_with_lookup(&proof_options); + let add_air = new_add_air_with_lookup(&proof_options); + let mul_air = new_mul_air_with_lookup(&proof_options); + let airs: Vec<&dyn AIR> = + vec![&cpu_air, &add_air, &mul_air]; + Verifier::multi_verify( + &airs, + proof, + &mut DefaultTranscript::::new(&[]), + &FieldElement::zero(), + ) +} + +/// Every commitment root is unchanged. This is the load-bearing half: the root +/// that entered the transcript at Round 1 is the root the recomputed LDE's +/// openings are checked against, so if the recompute produced different values +/// the openings would be answered against a tree that no longer matches them. +#[test_log::test] +fn recompute_lde_preserves_every_commitment_root() { + let retained = prove_under(ResidencyMode::Retain); + let recomputed = prove_under(ResidencyMode::RecomputeLde); + + assert_eq!(retained.proofs.len(), recomputed.proofs.len()); + for (idx, (a, b)) in retained + .proofs + .iter() + .zip(recomputed.proofs.iter()) + .enumerate() + { + assert_eq!( + a.lde_trace_main_merkle_root, b.lde_trace_main_merkle_root, + "table {idx}: main root moved" + ); + assert_eq!( + a.lde_trace_aux_merkle_root, b.lde_trace_aux_merkle_root, + "table {idx}: aux root moved" + ); + assert_eq!( + a.lde_trace_precomputed_merkle_root, b.lde_trace_precomputed_merkle_root, + "table {idx}: precomputed root moved" + ); + assert_eq!( + a.composition_poly_root, b.composition_poly_root, + "table {idx}: composition root moved" + ); + assert_eq!( + a.fri_layers_merkle_roots, b.fri_layers_merkle_roots, + "table {idx}: FRI roots moved" + ); + } +} + +/// The whole proof, byte for byte — openings, FRI decommitments and all +/// (proved without grinding: see [`test_options`]). +#[test_log::test] +fn recompute_lde_produces_byte_identical_proofs() { + let retained = bincode::serialize(&prove_under(ResidencyMode::Retain)).unwrap(); + let recomputed = bincode::serialize(&prove_under(ResidencyMode::RecomputeLde)).unwrap(); + assert_eq!( + retained.len(), + recomputed.len(), + "proof size moved between residency modes" + ); + assert!( + retained == recomputed, + "proof bytes moved between residency modes" + ); +} + +/// A proof made under `RecomputeLde` verifies with the standard verifier. Near +/// tautological given the byte equality above — and that is the point: the mode +/// has no wire presence for a verifier to know about. +#[test_log::test] +fn recompute_lde_proofs_verify() { + assert!(verifies(&prove_under(ResidencyMode::RecomputeLde))); + assert!(verifies(&prove_under(ResidencyMode::Retain))); +} + +/// The caller-visible half of the `RecomputeLde` contract: the aux columns +/// `multi_prove` wrote into the caller's traces are gone when it returns, and +/// under `Retain` they are still there. Pins the documented difference so a +/// caller that needs the aux columns after proving finds out here. +#[test_log::test] +fn recompute_lde_releases_aux_columns_and_retain_keeps_them() { + for (residency, expect_aux_rows) in [ + (ResidencyMode::Retain, true), + (ResidencyMode::RecomputeLde, false), + ] { + let (mut cpu_trace, mut add_trace, mut mul_trace) = traces(); + let proof_options = ProofOptions::default_test_options(); + let cpu_air = new_cpu_air_with_lookup(&proof_options); + let add_air = new_add_air_with_lookup(&proof_options); + let mul_air = new_mul_air_with_lookup(&proof_options); + + { + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu_trace, &()), + (&add_air, &mut add_trace, &()), + (&mul_air, &mut mul_trace, &()), + ]; + Prover::multi_prove(pairs, &mut DefaultTranscript::::new(&[]), residency).unwrap(); + } + + for (name, trace) in [ + ("cpu", &cpu_trace), + ("add", &add_trace), + ("mul", &mul_trace), + ] { + let has_rows = trace.aux_table.height > 0; + assert_eq!( + has_rows, expect_aux_rows, + "{name} trace aux residency wrong under {residency:?}" + ); + // The declared width survives either way — only the data is freed. + assert!(trace.aux_table.width > 0, "{name} lost its aux width"); + } + } +} diff --git a/crypto/stark/src/tests/subcoset_device_tests.rs b/crypto/stark/src/tests/subcoset_device_tests.rs new file mode 100644 index 000000000..e3bfbadd4 --- /dev/null +++ b/crypto/stark/src/tests/subcoset_device_tests.rs @@ -0,0 +1,85 @@ +//! The sub-coset composition evaluation on the card: a two-part LogUp table at +//! blowup 4, proved with the evaluation forced off and forced on, must give the +//! same proof bytes, and the forced-on prove must have run the device +//! composition kernel and the device split + extension at stride 2 (so the test +//! cannot pass by falling back to the host). Box only (`--features cuda`, +//! ignored; the counters are process-global, hence one test thread): +//! +//! ```text +//! cargo test -p stark --release --features cuda --lib \ +//! tests::subcoset_device_tests -- --ignored --test-threads=1 +//! ``` + +use crate::config::Blake3StarkHash; +use crate::gpu_lde::{gpu_composition_calls, gpu_device_only_calls, gpu_extend_halves_calls}; +use crate::proof::options::ProofFormat; +use crate::prover::{composition_subcoset_tables, set_composition_subcoset}; +use crate::tests::zf_golden_tests::{golden_options, prove_logup, sha3_hex, verify_logup}; + +struct Restore; +impl Drop for Restore { + fn drop(&mut self) { + set_composition_subcoset(None); + } +} + +fn counters() -> (u64, u64, u64, u64) { + ( + composition_subcoset_tables(), + gpu_composition_calls(), + gpu_extend_halves_calls(), + gpu_device_only_calls(), + ) +} + +/// 2^13 rows (LDE 2^15: the card commits and keeps a host copy) and 2^17 rows +/// (LDE 2^19: the device-only envelope). +#[test] +#[ignore = "card: proves on the GPU; run on the box with --ignored --test-threads=1"] +fn subcoset_composition_on_the_card_is_byte_identical() { + let _restore = Restore; + for log_rows in [13u32, 17] { + let rows = 1usize << log_rows; + let o = golden_options(4, 1, 7, ProofFormat::DEFAULT); + + set_composition_subcoset(Some(false)); + let (air_off, off, _) = prove_logup::(rows, &o); + + set_composition_subcoset(Some(true)); + let before = counters(); + let (air_on, on, _) = prove_logup::(rows, &o); + let after = counters(); + set_composition_subcoset(None); + + eprintln!( + "SUBCOSET CARD rows=2^{log_rows}: subcoset tables +{} · composition kernels +{} · \ + device split+extend +{} · device-only tables +{}", + after.0 - before.0, + after.1 - before.1, + after.2 - before.2, + after.3 - before.3, + ); + assert!( + after.0 > before.0, + "rows 2^{log_rows}: sub-coset path not taken" + ); + assert!( + after.1 > before.1, + "rows 2^{log_rows}: the device composition kernel did not run" + ); + assert!( + after.2 > before.2, + "rows 2^{log_rows}: the device split + extension did not run" + ); + + assert!(verify_logup::(&air_off, &off)); + assert!(verify_logup::(&air_on, &on)); + let off_bytes = rkyv::to_bytes::(&off).expect("rkyv"); + let on_bytes = rkyv::to_bytes::(&on).expect("rkyv"); + assert_eq!( + sha3_hex(&on_bytes), + sha3_hex(&off_bytes), + "rows 2^{log_rows}: sub-coset proof bytes differ from the whole-coset proof on the card" + ); + } +} diff --git a/crypto/stark/src/tests/table_disk_spill_tests.rs b/crypto/stark/src/tests/table_disk_spill_tests.rs deleted file mode 100644 index 3a1ec8d56..000000000 --- a/crypto/stark/src/tests/table_disk_spill_tests.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::table::Table; -use math::field::goldilocks::GoldilocksField; - -type F = GoldilocksField; - -#[test] -fn test_table_spill_roundtrip() { - let width = 4; - let height = 8; - let data: Vec> = (0..width * height) - .map(|i| math::field::element::FieldElement::::from(i as u64)) - .collect(); - - let mut table = Table::new(data.clone(), width); - assert!(table.mmap_backing.is_none()); - - // Snapshot values before spill - let pre_spill: Vec>> = (0..height) - .map(|r| (0..width).map(|c| *table.get(r, c)).collect()) - .collect(); - - table.spill_to_disk().expect("spill_to_disk failed"); - assert!(table.mmap_backing.is_some()); - assert!( - table.data.is_empty(), - "heap data should be freed after spill" - ); - - // Verify get() returns the same values - for (r, pre_row) in pre_spill.iter().enumerate() { - for (c, pre_val) in pre_row.iter().enumerate() { - assert_eq!(table.get(r, c), pre_val, "mismatch at ({r}, {c})"); - } - } - - // Verify get_row() returns the same values - for (r, pre_row) in pre_spill.iter().enumerate() { - let row = table.get_row(r); - assert_eq!(row.len(), width); - for (c, pre_val) in pre_row.iter().enumerate() { - assert_eq!(&row[c], pre_val, "get_row mismatch at ({r}, {c})"); - } - } -} - -#[test] -fn test_table_spill_empty_is_noop() { - let mut table = Table::::new(Vec::new(), 0); - table - .spill_to_disk() - .expect("spill_to_disk on empty table failed"); - assert!(table.mmap_backing.is_none()); -} - -#[test] -fn test_table_spill_idempotent() { - let data: Vec> = (0..16) - .map(|i| math::field::element::FieldElement::::from(i as u64)) - .collect(); - let mut table = Table::new(data, 4); - - table.spill_to_disk().expect("first spill failed"); - assert!(table.mmap_backing.is_some()); - - table.spill_to_disk().expect("second spill should be no-op"); - assert!(table.mmap_backing.is_some()); - - // Still readable - assert_eq!( - table.get(0, 0), - &math::field::element::FieldElement::::from(0u64) - ); - assert_eq!( - table.get(3, 3), - &math::field::element::FieldElement::::from(15u64) - ); -} - -#[test] -fn test_clone_spilled_table_materializes_to_heap() { - let width = 4; - let height = 8; - let data: Vec> = (0..width * height) - .map(|i| math::field::element::FieldElement::::from(i as u64)) - .collect(); - - let mut table = Table::new(data, width); - table.spill_to_disk().expect("spill_to_disk failed"); - assert!(table.mmap_backing.is_some()); - - let cloned = table.clone(); - assert!(cloned.mmap_backing.is_none(), "clone should not be spilled"); - assert_eq!(cloned.width, width); - assert_eq!(cloned.height, height); - assert_eq!(cloned, table, "clone must equal source element-wise"); -} - -#[test] -fn test_serialize_spilled_table_matches_unspilled() { - let width = 4; - let height = 8; - let data: Vec> = (0..width * height) - .map(|i| math::field::element::FieldElement::::from(i as u64)) - .collect(); - - let unspilled = Table::new(data.clone(), width); - let unspilled_bytes = bincode::serialize(&unspilled).expect("serialize unspilled"); - - let mut spilled = Table::new(data, width); - spilled.spill_to_disk().expect("spill_to_disk failed"); - let spilled_bytes = bincode::serialize(&spilled).expect("serialize spilled"); - - assert_eq!( - spilled_bytes, unspilled_bytes, - "spilled and unspilled tables must serialize to identical bytes" - ); - - let restored: Table = - bincode::deserialize(&spilled_bytes).expect("deserialize spilled bytes"); - assert!(restored.mmap_backing.is_none()); - assert_eq!(restored, unspilled); -} diff --git a/crypto/stark/src/tests/table_transpose_tests.rs b/crypto/stark/src/tests/table_transpose_tests.rs new file mode 100644 index 000000000..36202a630 --- /dev/null +++ b/crypto/stark/src/tests/table_transpose_tests.rs @@ -0,0 +1,33 @@ +//! `Table::columns_blocked` is `Table::columns`, tile by tile. + +use crate::table::Table; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +type FE = FieldElement; + +fn table(width: usize, height: usize) -> Table { + let data: Vec = (0..width * height) + .map(|i| FE::from((i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 7)) + .collect(); + Table::new(data, width) +} + +/// Every shape a block has — one column, fewer columns than a band, bands and a +/// remainder, a KECCAK_RND-wide table — at heights below, at and past a row +/// block, including a partial last block. +#[test] +fn the_blocked_transposition_is_the_transposition() { + for width in [1, 3, 64, 70, 1480] { + for height in [1, 4, 1024, 1500, 4096] { + let t = table(width, height); + assert_eq!(t.columns_blocked(), t.columns(), "{width} x {height}"); + } + } +} + +#[test] +fn an_empty_table_has_no_columns() { + let t = Table::::new(Vec::new(), 0); + assert!(t.columns_blocked().is_empty()); +} diff --git a/crypto/stark/src/tests/warm_device_tests.rs b/crypto/stark/src/tests/warm_device_tests.rs new file mode 100644 index 000000000..cd11f7ffa --- /dev/null +++ b/crypto/stark/src/tests/warm_device_tests.rs @@ -0,0 +1,124 @@ +//! The prove-constants warm-up and the staged composition drain on the card. +//! Box only (`--features cuda`, ignored; process-global counters, one thread): +//! +//! ```text +//! cargo test -p stark --release --features cuda --lib tests::warm_device_tests \ +//! -- --ignored --test-threads=1 +//! ``` + +use crate::config::Blake3StarkHash; +use crate::examples::read_only_memory_logup::LogReadOnlyRAP; +use crate::proof::options::ProofFormat; +use crate::prover::{set_warm_prove_constants, warm_prove_constants, warmed_prove_constants}; +use crate::tests::zf_golden_tests::{golden_options, prove_logup, sha3_hex, verify_logup}; +use crate::traits::AIR; + +struct Restore; +impl Drop for Restore { + fn drop(&mut self) { + set_warm_prove_constants(None); + } +} + +/// The staged drain returns the slab drain's words, raw. +#[test] +#[ignore = "card: drains a resident H both ways; run on the box with --ignored"] +fn the_staged_composition_drain_matches_the_slab_drain() { + let rows = (1usize << 21) + 5; // 48 MiB and change: two staged chunks and a short tail + let words: Vec = (0..rows as u64 * 3) + .map(|i| i.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ 0x5555) + .collect(); + let h = math_cuda::constraint_interp::comp_h_from_host_interleaved(&words, rows).expect("card"); + let slab = math_cuda::constraint_interp::download_comp_h(&h).expect("slab drain"); + let staged = math_cuda::constraint_interp::download_comp_h_staged(&h).expect("staged drain"); + assert_eq!(slab, words); + assert_eq!(staged, words); +} + +/// A two-part LogUp table at blowup 4 over the device floor (2^17 rows: the +/// device-only envelope), proved with the warm-up off, then warmed and proved +/// with it on: the same bytes, both verify, and the warm-up ran over the +/// table's constants (boundary zerofier inverse, their card copies, the +/// split's table, the coset points). +#[test] +#[ignore = "card: proves on the GPU; run on the box with --ignored --test-threads=1"] +fn a_warmed_prove_is_byte_identical() { + let _restore = Restore; + let rows = 1usize << 17; + let o = golden_options(4, 1, 7, ProofFormat::DEFAULT); + set_warm_prove_constants(Some(false)); + let (air_off, off, pi) = prove_logup::(rows, &o); + + set_warm_prove_constants(Some(true)); + let air = LogReadOnlyRAP::< + math::field::goldilocks::GoldilocksField, + math::field::extensions_goldilocks::Degree3GoldilocksExtensionField, + >::new(&o); + let before = warmed_prove_constants(); + let tables: Vec<( + &dyn AIR, + usize, + &_, + )> = vec![(&air, rows, &pi)]; + warm_prove_constants(&tables); + let warmed = warmed_prove_constants() - before; + let (air_on, on, _) = prove_logup::(rows, &o); + set_warm_prove_constants(None); + println!("WARM DEVICE: {warmed} constants warmed"); + assert!( + warmed >= 4, + "boundary inverse, its card copy, the split's table, the coset points" + ); + + assert!(verify_logup::(&air_off, &off)); + assert!(verify_logup::(&air_on, &on)); + let b = |p| sha3_hex(&rkyv::to_bytes::(p).expect("rkyv")); + assert_eq!(b(&on), b(&off), "the warm-up moved a proof byte"); +} + +/// The DEEP's LDE coset warmed before a prove: one upload for a length the card +/// has not seen under that offset, none on the second call. A prove after it +/// verifies and repeats byte for byte, and a prove whose DEEP found its coset in +/// the cache (the second under another offset) has the bytes of the one whose +/// DEEP uploaded it: the warm-up only fills that cache. +#[test] +#[ignore = "card: proves on the GPU; run on the box with --ignored --test-threads=1"] +fn the_deep_coset_warm_uploads_once_and_moves_no_byte() { + use crate::prover::{warm_deep_coset_points, warmed_deep_cosets}; + type Air = LogReadOnlyRAP< + math::field::goldilocks::GoldilocksField, + math::field::extensions_goldilocks::Degree3GoldilocksExtensionField, + >; + let rows = 1usize << 17; + // Offsets no other test proves under, so neither coset is on the card yet. + let mut cold_o = golden_options(4, 1, 7, ProofFormat::DEFAULT); + cold_o.coset_offset = 23; + let mut o = cold_o.clone(); + o.coset_offset = 29; + let (air_cold, cold, pi) = prove_logup::(rows, &cold_o); + let (_, cold_again, _) = prove_logup::(rows, &cold_o); + + let air = Air::new(&o); + let tables: Vec<( + &dyn AIR, + usize, + &_, + )> = vec![(&air, rows, &pi)]; + let before = warmed_deep_cosets(); + assert_eq!(warm_deep_coset_points(&tables), 1, "one coset uploaded"); + assert_eq!(warm_deep_coset_points(&tables), 0, "then found"); + assert_eq!(warmed_deep_cosets() - before, 1); + let (air_w, warm, _) = prove_logup::(rows, &o); + let (_, warm_again, _) = prove_logup::(rows, &o); + + assert!(verify_logup::(&air_cold, &cold)); + assert!(verify_logup::(&air_w, &warm)); + let b = |p| sha3_hex(&rkyv::to_bytes::(p).expect("rkyv")); + assert_eq!( + b(&cold), + b(&cold_again), + "a cached coset moved a proof byte" + ); + assert_eq!(b(&warm), b(&warm_again), "a warmed prove does not repeat"); + assert_ne!(b(&warm), b(&cold), "the two offsets prove differently"); +} diff --git a/crypto/stark/src/tests/zf_fri_device_tests.rs b/crypto/stark/src/tests/zf_fri_device_tests.rs new file mode 100644 index 000000000..8dda1e81e --- /dev/null +++ b/crypto/stark/src/tests/zf_fri_device_tests.rs @@ -0,0 +1,167 @@ +//! S3 on the device: the device FRI commit and +//! query phases against the host CPU loop, under Keccak and Blake3 (the RPX +//! twins live in the prover crate's `tests::zf_rpx_device_tests`). +//! +//! Every `#[ignore]`d test here needs a GPU and a lowered +//! `LAMBDA_VM_GPU_LDE_THRESHOLD`; each one fails loudly when the device path +//! does not run (a declined commit is an `Err`, a vector proof must move the +//! device FRI counter), so none can pass by falling back to the host: +//! +//! ```text +//! LAMBDA_VM_GPU_LDE_THRESHOLD=2 cargo test -p stark --release --features cuda \ +//! --lib tests::zf_fri_device_tests::parity_ -- --ignored +//! LAMBDA_VM_GPU_LDE_THRESHOLD=1024 cargo test -p stark --release --features cuda \ +//! --lib tests::zf_fri_device_tests::proved_vectors_equal_the_cpu_bytes \ +//! -- --ignored --exact --test-threads=1 +//! ``` +//! +//! `dp_shapes_are_pinned` needs no GPU (it only computes the shape list). + +use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; +use crate::fri::device_parity::{ + Case, dp_shapes, legacy_cases, production_cases, resident_cases, run_cases, sweep_cases, +}; + +/// The shapes the parity sweep covers: every distinct DP schedule for +/// `B ≤ 23` (T ∈ {4, 9, 10}, Q ∈ {3, 110}, cap off/auto) plus the extras. +/// Pinned so the box run's pre-registered count means something; a DP change +/// that moves this list is a format change and re-pins it deliberately. +#[test] +fn dp_shapes_are_pinned() { + let shapes = dp_shapes(); + let expected: &[&[u8]] = PINNED_SHAPES; + assert_eq!( + shapes, + expected.iter().map(|s| s.to_vec()).collect::>(), + "the DP's schedule set moved" + ); +} + +const PINNED_SHAPES: &[&[u8]] = &[ + // The DP's own (21, in first-appearance order). + &[1], + &[2], + &[3], + &[2, 2], + &[3, 2], + &[3, 3], + &[3, 2, 2], + &[3, 3, 2], + &[3, 3, 3], + &[3, 3, 2, 2], + &[3, 3, 3, 2], + &[3, 3, 3, 3], + &[3, 3, 3, 2, 2], + &[3, 3, 3, 3, 2], + &[3, 3, 3, 3, 3], + &[3, 3, 3, 3, 2, 2], + &[3, 3, 3, 3, 3, 2], + &[3, 3, 3, 3, 3, 3], + &[4, 3, 3], + &[4, 3, 3, 3], + &[4, 3], + // EXTRA_SHAPES (8). + &[4], + &[6], + &[1, 6], + &[6, 1], + &[3, 1, 3], + &[1, 3], + &[2, 5, 1], + &[1, 1, 1], +]; + +fn check(name: &str, cases: &[Case], resident: bool, seed: u64) { + if let Err(failures) = run_cases::(name, cases, resident, seed) { + panic!("{name}: {failures:#?}"); + } +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_every_dp_shape_keccak() { + check::("keccak", &sweep_cases(), false, 0x5a46_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_every_dp_shape_blake3() { + check::("blake3", &sweep_cases(), false, 0x5a46_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_production_sizes_keccak() { + check::("keccak", &production_cases(), false, 0x5a47_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_production_sizes_blake3() { + check::("blake3", &production_cases(), false, 0x5a47_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_resident_layers_keccak() { + check::("keccak", &resident_cases(), true, 0x5a48_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_resident_layers_blake3() { + check::("blake3", &resident_cases(), true, 0x5a48_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_legacy_encoding_keccak() { + check::("keccak", &legacy_cases(), false, 0x5a49_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_legacy_encoding_blake3() { + check::("blake3", &legacy_cases(), false, 0x5a49_0000); +} + +/// The (d) vector proofs (the README's (d): `pair`, `dp`, `dp_3_1_3`, and the +/// Merkle-capped `cap_pair`, `cap_dp` at Q = 20) proved on +/// the device path — LDE 4096, so `LAMBDA_VM_GPU_LDE_THRESHOLD` must be at +/// most 4096 — are byte-identical to the checked-in CPU-proved files (rkyv +/// bytes and the verifier-derived JSON), under Keccak and Blake3. The device +/// FRI counter must move once per proof, so a host fallback fails the test. +/// Run alone (`--exact --test-threads=1`): the counter is process-wide. +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD<=4096; run alone with --features cuda -- --ignored --exact --test-threads=1"] +fn proved_vectors_equal_the_cpu_bytes() { + use crate::fri::vectors::{check_or_write, proof_vectors}; + let before = crate::gpu_lde::gpu_fri_calls(); + let comp_before = crate::gpu_lde::gpu_composition_calls(); + let mut files = proof_vectors::("keccak"); + files.extend(proof_vectors::("blake3")); + let device_commits = crate::gpu_lde::gpu_fri_calls() - before; + let compositions = crate::gpu_lde::gpu_composition_calls() - comp_before; + println!( + "FRIDEV vector proofs: {} files, {device_commits} device FRI commits, {compositions} device compositions", + files.len() + ); + // Five (d) formats (pair, dp, dp_3_1_3 at Q = 3; cap_pair, cap_dp at + // Q = 20) x two hashes, two files and one FRI commit per proof. + assert_eq!(files.len(), 2 * 5 * 2); + assert_eq!( + device_commits, 10, + "every vector proof must take the device FRI commit (lower LAMBDA_VM_GPU_LDE_THRESHOLD)" + ); + // Every proof composes on the device (the AIR's constraint program); a + // host composition would not be counted here. + assert_eq!( + compositions, 10, + "every vector proof must compose on the device ({compositions} device compositions)" + ); + let bad = check_or_write(&files, false); + assert!( + bad.is_empty(), + "device-proved vectors differ from the checked-in CPU bytes: {bad:?}" + ); +} diff --git a/crypto/stark/src/tests/zf_fri_vectors.rs b/crypto/stark/src/tests/zf_fri_vectors.rs new file mode 100644 index 000000000..0561e63aa --- /dev/null +++ b/crypto/stark/src/tests/zf_fri_vectors.rs @@ -0,0 +1,46 @@ +//! The exported S3 and S2 vectors ((a)–(e) in the README) under Keccak and Blake3 are +//! current: regenerated in memory and byte-equal to the checked-in files in +//! `crypto/stark/tests/vectors/zf_fri/` (the RPX files: the prover crate's +//! `tests::zf_rpx_vectors`). Regenerate after a deliberate format change: +//! `cargo test -p stark --lib zf_fri_vectors::write_vectors -- --ignored`. + +use crate::config::{Blake3StarkHash, KeccakStarkHash}; +use crate::fri::vectors::{ + VectorFile, check_or_write, group_fold_json, leaf_digests_json, one_row_leaf_digests_json, + one_row_proof_vectors, proof_vectors, schedules_json, +}; + +fn all() -> Vec { + let mut v = vec![ + schedules_json(), + group_fold_json(), + leaf_digests_json::("keccak"), + leaf_digests_json::("blake3"), + ]; + v.extend(proof_vectors::("keccak")); + v.extend(proof_vectors::("blake3")); + // (e) S2. + v.push(one_row_leaf_digests_json::("keccak")); + v.push(one_row_leaf_digests_json::("blake3")); + v.extend(one_row_proof_vectors::("keccak")); + v.extend(one_row_proof_vectors::("blake3")); + v +} + +#[test] +fn vectors_are_current() { + let files = all(); + assert_eq!(files.len(), 4 + 2 * 5 * 2 + 2 + 2 * 2 * 2); + let bad = check_or_write(&files, false); + assert!( + bad.is_empty(), + "stale or missing vector files {bad:?}; regenerate with \ + `cargo test -p stark --lib zf_fri_vectors::write_vectors -- --ignored`" + ); +} + +#[test] +#[ignore = "writes crypto/stark/tests/vectors/zf_fri"] +fn write_vectors() { + assert!(check_or_write(&all(), true).is_empty()); +} diff --git a/crypto/stark/src/tests/zf_golden_tests.rs b/crypto/stark/src/tests/zf_golden_tests.rs new file mode 100644 index 000000000..4fe36f0b0 --- /dev/null +++ b/crypto/stark/src/tests/zf_golden_tests.rs @@ -0,0 +1,469 @@ +//! Default-format golden proofs: the bytes today's prover emits, +//! pinned, so a format lever that claims "the default is byte-identical" is +//! checked against the prover's own output rather than against a round trip +//! (a drifted prover still accepts its own proofs). +//! +//! Proof bytes are reproducible only without grinding (the nonce search is a +//! parallel `find_any`), so every case proves at `grinding_factor = 0`. Each +//! case pins the SHA3-256 of the proof's rkyv bytes (the wire format of +//! record) and, so that a failure says WHERE the drift is, separately the +//! digests of: the FRI layer roots, the terminal coefficients, the per-query FRI +//! decommitments, and the trace/composition openings (plus the layer count). +//! ζ and ι are not in a proof; a ζ drift moves every later layer root and the +//! terminal coefficients, an ι drift moves the decommitment and opening digests. +//! +//! Coverage: both byte hashes this crate owns (Keccak, Blake3; RPX is pinned +//! the same way in the prover crate, `tests::zf_rpx_golden_tests`), blowup 2 and +//! 4, a base-field AIR (`SimpleAddition`, E = F) and an extension-field AIR with +//! an aux trace (`LogReadOnlyRAP`, E = F³), `total_folds` ∈ {0, 1, 2, ≥ 3}, and +//! one multi-table bus proof (CPU/ADD/MUL, `multi_prove`). +//! +//! Generated at the default format BEFORE any S3 prover code existed (the +//! schedule DP alone changes no prover path). Regenerate only for a deliberate format change: +//! `cargo test -p stark --lib zf_golden_tests::print_goldens -- --ignored --nocapture`. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use sha3::{Digest, Sha3_256}; + +use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::examples::read_only_memory_logup::{ + LogReadOnlyPublicInputs, LogReadOnlyRAP, read_only_logup_trace, +}; +use crate::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use crate::proof::options::{ProofFormat, ProofOptions}; +use crate::proof::stark::{MultiProof, StarkProof}; +use crate::prover::{GenericProver, IsStarkProver}; +use crate::residency_mode::ResidencyMode; +use crate::trace::TraceTable; +use crate::traits::AIR; +use crate::verifier::{GenericVerifier, IsStarkVerifier}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; + +/// Test options at the DEFAULT format with grinding off. `k` is the terminal +/// log-degree, so `total_folds = log2(rows) − k` whenever that is ≥ 0. +pub(crate) fn golden_options( + blowup: u8, + k: u8, + queries: usize, + format: ProofFormat, +) -> ProofOptions { + ProofOptions { + blowup_factor: blowup, + fri_number_of_queries: queries, + coset_offset: 3, + grinding_factor: 0, + fri_final_poly_log_degree: k, + format, + } +} + +pub(crate) fn sha3_hex(bytes: &[u8]) -> String { + let d = Sha3_256::digest(bytes); + d.iter().map(|b| format!("{b:02x}")).collect() +} + +/// The pinned digests of one proof, as one line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Fingerprint { + pub proof: String, + pub fri_roots: String, + pub num_fri_roots: usize, + pub coeffs: String, + pub queries: String, + pub openings: String, +} + +impl Fingerprint { + pub(crate) fn line(&self) -> String { + format!( + "proof {} roots[{}] {} coeffs {} queries {} openings {}", + self.proof, + self.num_fri_roots, + self.fri_roots, + self.coeffs, + self.queries, + self.openings + ) + } +} + +/// The [`Fingerprint`] of any `StarkProof` (a macro: the rkyv serializer +/// bounds of a generic `StarkProof` are not worth spelling out). +macro_rules! fingerprint { + ($proof:expr) => {{ + let proof = $proof; + let rk = |bytes: Result| { + $crate::tests::zf_golden_tests::sha3_hex(&bytes.expect("rkyv")) + }; + $crate::tests::zf_golden_tests::Fingerprint { + proof: rk(rkyv::to_bytes::(proof)), + fri_roots: $crate::tests::zf_golden_tests::sha3_hex( + &proof.fri_layers_merkle_roots.concat(), + ), + num_fri_roots: proof.fri_layers_merkle_roots.len(), + coeffs: rk(rkyv::to_bytes::( + &proof.fri_final_poly_coeffs, + )), + queries: rk(rkyv::to_bytes::(&proof.query_list)), + openings: rk(rkyv::to_bytes::( + &proof.deep_poly_openings, + )), + } + }}; +} +pub(crate) use fingerprint; + +// --------------------------------------------------------------------------- +// The cases +// --------------------------------------------------------------------------- + +/// `SimpleAddition` (E = F) under hash `H`. +pub(crate) fn prove_simple_addition( + rows: usize, + options: &ProofOptions, +) -> ( + SimpleAdditionAIR, + StarkProof>, +) { + let air = SimpleAdditionAIR::::new(options); + let pi = SimpleAdditionPublicInputs { + a: Felt::from(1u64), + b: Felt::from(2u64), + }; + let mut trace = simple_addition_trace::(rows); + let proof = GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof) +} + +pub(crate) fn verify_simple_addition( + air: &SimpleAdditionAIR, + proof: &StarkProof>, +) -> bool { + GenericVerifier::::verify(proof, air, &mut DefaultTranscript::::new(&[])) +} + +/// A continuous read-only memory over addresses 1..=5, `rows` reads. +fn logup_reads(rows: usize) -> (Vec, Vec) { + let addr: Vec = (0..rows).map(|i| Felt::from((i % 5) as u64 + 1)).collect(); + let val: Vec = (0..rows) + .map(|i| Felt::from(((i % 5) as u64 + 1) * 10)) + .collect(); + (addr, val) +} + +/// `LogReadOnlyRAP` (E = F³, one aux column) under hash `H`. +/// An AIR, its proof and its public inputs. +pub(crate) type LogupCase = ( + LogReadOnlyRAP, + StarkProof>, + LogReadOnlyPublicInputs, +); + +pub(crate) fn prove_logup(rows: usize, options: &ProofOptions) -> LogupCase { + let (addr, val) = logup_reads(rows); + let mut trace: TraceTable = read_only_logup_trace(addr, val); + let cols = trace.columns_main(); + let pi = LogReadOnlyPublicInputs { + a0: cols[0][0], + v0: cols[1][0], + a_sorted_0: cols[2][0], + v_sorted_0: cols[3][0], + m0: cols[4][0], + }; + let air = LogReadOnlyRAP::::new(options); + let proof = GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof, pi) +} + +pub(crate) fn verify_logup( + air: &LogReadOnlyRAP, + proof: &StarkProof>, +) -> bool { + GenericVerifier::::verify(proof, air, &mut DefaultTranscript::::new(&[])) +} + +/// The CPU/ADD/MUL bus instance (`residency_mode_tests`) under hash `H`. +pub(crate) fn prove_multi(options: &ProofOptions) -> MultiProof { + let (mut cpu_trace, mut add_trace, mut mul_trace) = super::residency_mode_tests::traces(); + let cpu_air = new_cpu_air_with_lookup(options); + let add_air = new_add_air_with_lookup(options); + let mul_air = new_mul_air_with_lookup(options); + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu_trace, &()), + (&add_air, &mut add_trace, &()), + (&mul_air, &mut mul_trace, &()), + ]; + GenericProver::::multi_prove( + pairs, + &mut DefaultTranscript::::new(&[]), + ResidencyMode::default(), + ) + .expect("proving must succeed") +} + +pub(crate) fn verify_multi( + options: &ProofOptions, + proof: &MultiProof, +) -> bool { + let cpu_air = new_cpu_air_with_lookup(options); + let add_air = new_add_air_with_lookup(options); + let mul_air = new_mul_air_with_lookup(options); + let airs: Vec<&dyn AIR> = + vec![&cpu_air, &add_air, &mul_air]; + GenericVerifier::::multi_verify( + &airs, + proof, + &mut DefaultTranscript::::new(&[]), + &FieldElement::zero(), + ) +} + +/// Every golden case: (name, fingerprint line) computed now. +fn compute_goldens() -> Vec<(String, String)> { + let mut out = Vec::new(); + let d = ProofFormat::DEFAULT; + // SimpleAddition, k = 2: total_folds = log2(rows) − 2. + for (hash, rows, blowup) in [ + ("keccak", 4usize, 2u8), // total_folds 0 + ("keccak", 8, 2), // 1 + ("keccak", 16, 4), // 2 + ("keccak", 256, 2), // 6 + ("blake3", 8, 4), // 1 + ("blake3", 64, 4), // 4 + ] { + let o = golden_options(blowup, 2, 5, d); + let (air, proof) = match hash { + "keccak" => prove_simple_addition::(rows, &o), + _ => prove_simple_addition::(rows, &o), + }; + let ok = match hash { + "keccak" => verify_simple_addition::(&air, &proof), + _ => verify_simple_addition::(&air, &proof), + }; + assert!(ok, "golden case must verify"); + out.push(( + format!("simple_addition/{hash}/rows{rows}/blowup{blowup}"), + fingerprint!(&proof).line(), + )); + } + // LogReadOnlyRAP (ext3 + aux), k = 1. + for (hash, rows, blowup) in [ + ("blake3", 16usize, 2u8), // total_folds 3 + ("blake3", 128, 4), // 6 + ("keccak", 32, 4), // 4 + ] { + let o = golden_options(blowup, 1, 7, d); + let (air, proof, _) = match hash { + "keccak" => prove_logup::(rows, &o), + _ => prove_logup::(rows, &o), + }; + let ok = match hash { + "keccak" => verify_logup::(&air, &proof), + _ => verify_logup::(&air, &proof), + }; + assert!(ok, "golden case must verify"); + out.push(( + format!("logup/{hash}/rows{rows}/blowup{blowup}"), + fingerprint!(&proof).line(), + )); + } + // Multi-table bus proof, k = 1 (CPU 8 rows, ADD/MUL 4 rows). + let o = golden_options(2, 1, 6, d); + let multi = prove_multi::(&o); + assert!(verify_multi::(&o, &multi)); + let bytes = rkyv::to_bytes::(&multi).expect("rkyv"); + let mut line = format!("multi {}", sha3_hex(&bytes)); + for (i, p) in multi.proofs.iter().enumerate() { + line.push_str(&format!(" | table{i} {}", fingerprint!(p).line())); + } + out.push(("multi/blake3/blowup2".to_string(), line)); + out +} + +/// Pinned at the default format (see the module docs). +const GOLDENS: &[(&str, &str)] = &[ + ( + "simple_addition/keccak/rows4/blowup2", + "proof d72bff5491a61ff5a58c4677dbcf1a2daa17953ea1b976113e21a24b54bec6b7 roots[0] a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a coeffs 283be3dea88b9f9fc3012a6ec6f4dd9452c63f49bf5030c0119426aeca2e2ead queries 52d34b9f6d30aaf0b5bc5a4c5cb5c99909fcc4018af897ee04b9ff4beb69bc0c openings bf85016f5d66788f79a6f55ec69d9829d8f2afad2a792e9432d2e2bc6e85a391", + ), + ( + "simple_addition/keccak/rows8/blowup2", + "proof df6437ee9bbbabb2d22dddfc8ac6888388328454cb6772bc52bd91d33e9c961e roots[0] a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a coeffs 1fa5e702cc4464b2fccd2c2ed05f9544aae1c1f81d6e63ec2907fa1225ee1309 queries 52d34b9f6d30aaf0b5bc5a4c5cb5c99909fcc4018af897ee04b9ff4beb69bc0c openings 02e305d0d828960f22c1c2d46dc01c2cdb04488a7cdaf66c97310061ffb34817", + ), + ( + "simple_addition/keccak/rows16/blowup4", + "proof f513e31eca509f0eb72429e007724d368fa3466007c9bbd3cb0c1a4e62d0231b roots[1] 1b8a5d9014a32fd18488405253b1b382fd94681299ce888d2054b413dc9ffd4e coeffs 6763e71de233097b3c8a2563c8fc958f7dd8258c015d4602ef1be5c5f3d7d1ed queries 3b62f8adbf53c30c0a06b8b8e04c1c655e776ea7193a6eac577e1612a564db09 openings d3e7a59c53cfba8ec96b4014ddeff9c395b6a2b12f0ad4e49139f824de65241c", + ), + ( + "simple_addition/keccak/rows256/blowup2", + "proof 0d21b0b5d2405473c93d17df09db5d3c771f87014e46e6aa26d9f2ee1f20caa8 roots[5] d274efdb442a2c9dec26ace00aadf90e47bb33c46cafc818d35aa2a44025ab82 coeffs 8ac737818bedf94a381b4b10f650dee2d300024d7a0bb9fa5f8200406c293971 queries 247e2e8c14f12cf87975bb5aabf89d7b4cb046e21d889bd736e3c9ab3b3891b3 openings cf8f7514ffd6b2df685f2cbcfaf5e5e37c60868b10c9c9186bd66b26267e2e7c", + ), + ( + "simple_addition/blake3/rows8/blowup4", + "proof d6ebfa4b14b5e9bd6239c7a31e17a146e6995ade0b3101847df515176fe563ba roots[0] a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a coeffs 307d4a0b258158554a0f243871f919fcf4031414b3ecbdd39f287d2d31cba197 queries 52d34b9f6d30aaf0b5bc5a4c5cb5c99909fcc4018af897ee04b9ff4beb69bc0c openings 4632d0d62386964b4e3566b3af2271c6b0800ffe89894d39ddf020eaa5258695", + ), + ( + "simple_addition/blake3/rows64/blowup4", + "proof 8caa0c1311ee2092d1a85c5e2fe6923466ebd8ed89edca123614d4127f1e674b roots[3] 9a7b1bf916dd5bdaee9ce8727a51131c055bcde12f780d48681cb6d51c543691 coeffs 9629f564cf5f72bc3b17b1c88d5864beb200da36ab5c0b2fea2f45905dc8b3f8 queries 0633449ec687a0c183ef547ee538d8620a51f3b3a0d42f570f9c50f5c3e13125 openings 24345624f78a948e8e006559792233857529c9612c603280be9e2779c50f7643", + ), + ( + "logup/blake3/rows16/blowup2", + "proof 51b186c5c4c958d4de0c1359e32d6568780c0631c9b5e7289aa3bd1363997e88 roots[2] d6bad767d9d70cbd82c59d6ee0d13f4415e7ac4046ea2d606ebc305a6bac61ce coeffs ba8e4c0e358be69f2d39a246318d31ccf796086042c469ac060702006e79d62f queries 818f9de07a7ea34f7b1794aa0a62a77fd4c5d700e349c33aba6be5033b93a9d4 openings a7045f424c3096a7554d78e6ff535700eaa8b6af269bcbd6939b56f93bf8391d", + ), + ( + "logup/blake3/rows128/blowup4", + "proof bea6e98f19c49e75401bcc9c21c73dbf3bf9576e5ce80a09a48f9dd5b70fa3ec roots[5] ceec0e7bc2ca1c16410c2222b96ce6d45d3e4573f54eb25378625bba375c4804 coeffs 17bb0241af6871c3359af6a56981dd842c667f6f1ca87227bda570f93c9a566b queries 1d84c98fdb64d057b0af98d2be55dd2c611cfc3cca5fea9abf1b5c1e86aaebc7 openings 4374ff4fab6a7bc892f8769661b6ad5abba2ac4979931c296f4b3d857b547369", + ), + ( + "logup/keccak/rows32/blowup4", + "proof 2caf234c6858994f7a1910bb95b465c9ab7b5fc675f489f3e3fdc818b5d54798 roots[3] 92a3704249af017157fb755e570d4b07f73c151392119f4a8ffd1d96e2511123 coeffs c85284e687629ec7d505bb0723f10324896d4e7ebd05eb6aa3dd57459b5af0f6 queries 67dbdc533f359c0b48f3b14e704e086ac17d785bec9cd53b9891147ce1f925a9 openings ede54d2b9998a60133ac0cf63c88d5b47a487082899becea17711c0e7db55283", + ), + ( + "multi/blake3/blowup2", + "multi eec51cb0e5701209e4709f89fb72b3af2dbc3038f2c80a185a2c74391ebffad5 | table0 proof 1079e69c4d47411814b05c5a7cc960d1c93846a3a873381c0184fa8273300cc3 roots[1] 1c7023dfeb09e6cc2ec141f6ec83e04f95e036ab1ae54a468248adde46a60d79 coeffs cb86ea5b8fe22227a96ad9d6ca4f68cb3bcd07825956d3ab804a98182304bfc8 queries 853284b820c6409aa20eab2a60863f4d1456deed856a8e9baadda3ecc754e7a3 openings e862a605d77e405c43605a4db1c05f49fefba0bda4d722d1dca33440ecf677ba | table1 proof 759ca54156cf6132fe61bd99f21528789bd60a21c190b3ffdc4caea20351f16d roots[0] a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a coeffs bd928b7e79613fa867a51d21ce6af7c18c3bd07aae44c0a83460486f50be9cfb queries c50b6659101c4ff74629092f4030534eec067bfaa650f3048807c4f2bba7ca72 openings 764b653d05d0cc14328bd94b8454420df2611a7d3465375b80235bfdb93f70ad | table2 proof b276a32877699f6e9e105ae05bd3bbfe1416d088187ceb22e710f968b9e5efd9 roots[0] a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a coeffs 1aef1aa9a22d64128d1469e69aa38886572f485071c6d6b046dd4f8bca60beb6 queries c50b6659101c4ff74629092f4030534eec067bfaa650f3048807c4f2bba7ca72 openings 9a7740f93e490a668f8f16dd9d4b00b79cab3e4561a85eecca03c8b5eafc13f6", + ), +]; + +#[test] +fn default_format_goldens_are_byte_identical() { + let got = compute_goldens(); + assert_eq!(got.len(), GOLDENS.len(), "one pin per case"); + for ((name, line), (pin_name, pin_line)) in got.iter().zip(GOLDENS) { + assert_eq!(name, pin_name); + assert_eq!( + line, pin_line, + "{name}: the default-format proof moved (a field whose digest differs is where)" + ); + } +} + +/// The sub-coset composition evaluation (two parts at blowup ≥ 4: `H` on +/// `2n` points instead of the whole LDE) must not move a byte: the same case +/// proved with it forced off and forced on gives the same rkyv bytes, both +/// verify, and the "on" prove really took the sub-coset path. The pinned +/// goldens above run under the default (on); this compares the two paths +/// directly, so it also holds if the default changes. +#[test] +fn subcoset_composition_is_byte_identical_to_whole_coset() { + use crate::prover::{composition_subcoset_tables, set_composition_subcoset}; + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + set_composition_subcoset(None); + } + } + let _restore = Restore; + + for (rows, blowup) in [(128usize, 4u8), (64, 8)] { + let o = golden_options(blowup, 1, 7, ProofFormat::DEFAULT); + + set_composition_subcoset(Some(false)); + let (air_off, off, _) = prove_logup::(rows, &o); + + set_composition_subcoset(Some(true)); + let before = composition_subcoset_tables(); + let (air_on, on, _) = prove_logup::(rows, &o); + assert!( + composition_subcoset_tables() > before, + "rows{rows}/blowup{blowup}: the forced-on prove never took the sub-coset path" + ); + set_composition_subcoset(None); + + assert!(verify_logup::(&air_off, &off)); + assert!(verify_logup::(&air_on, &on)); + let off_bytes = rkyv::to_bytes::(&off).expect("rkyv"); + let on_bytes = rkyv::to_bytes::(&on).expect("rkyv"); + assert_eq!( + sha3_hex(&on_bytes), + sha3_hex(&off_bytes), + "rows{rows}/blowup{blowup}: sub-coset proof bytes differ from the whole-coset proof" + ); + } + + // The multi-table bus proof (CPU/ADD/MUL, LogUp, one transcript) at blowup 4. + let o = golden_options(4, 1, 6, ProofFormat::DEFAULT); + set_composition_subcoset(Some(false)); + let off = prove_multi::(&o); + set_composition_subcoset(Some(true)); + let before = composition_subcoset_tables(); + let on = prove_multi::(&o); + assert!( + composition_subcoset_tables() > before, + "multi/blowup4: the forced-on prove never took the sub-coset path" + ); + set_composition_subcoset(None); + assert!(verify_multi::(&o, &off)); + assert!(verify_multi::(&o, &on)); + let off_bytes = rkyv::to_bytes::(&off).expect("rkyv"); + let on_bytes = rkyv::to_bytes::(&on).expect("rkyv"); + assert_eq!( + sha3_hex(&on_bytes), + sha3_hex(&off_bytes), + "multi/blowup4: sub-coset proof bytes differ from the whole-coset proof" + ); +} + +/// Prints `GOLDENS`. Run only for a deliberate format change. +#[test] +#[ignore = "generator for GOLDENS"] +fn print_goldens() { + println!("const GOLDENS: &[(&str, &str)] = &["); + for (name, line) in compute_goldens() { + println!(" (\n \"{name}\",\n \"{line}\",\n ),"); + } + println!("];"); +} + +/// The prove-constants warm-up (host side) builds the boundary zerofier +/// inverses a prove would build on first use, and a prove after it has the +/// bytes of one without it. +#[test] +fn a_warmed_prove_has_the_unwarmed_bytes() { + use crate::prover::{warm_prove_constants, warmed_prove_constants}; + let o = golden_options(4, 1, 7, ProofFormat::DEFAULT); + let rows = 1usize << 9; + let (_, cold, pi) = prove_logup::(rows, &o); + let air = LogReadOnlyRAP::::new(&o); + let before = warmed_prove_constants(); + let tables: Vec<( + &dyn AIR, + usize, + &_, + )> = vec![(&air, rows, &pi)]; + warm_prove_constants(&tables); + assert!( + warmed_prove_constants() > before, + "a LogUp table has a boundary constraint" + ); + let (air_w, warm, _) = prove_logup::(rows, &o); + assert!(verify_logup::(&air_w, &warm)); + let b = |p| sha3_hex(&rkyv::to_bytes::(p).expect("rkyv")); + assert_eq!(b(&cold), b(&warm)); +} diff --git a/crypto/stark/src/tests/zf_s2_device_tests.rs b/crypto/stark/src/tests/zf_s2_device_tests.rs new file mode 100644 index 000000000..97bbbfb26 --- /dev/null +++ b/crypto/stark/src/tests/zf_s2_device_tests.rs @@ -0,0 +1,134 @@ +//! S2 on the device: one-row trees and +//! openings, and the committed input tree from the DEEP codeword, against the +//! host CPU paths, under Keccak and Blake3 (the RPX twins live in the prover +//! crate's `tests::zf_rpx_device_tests`). +//! +//! Every test here needs a GPU and fails loudly when the device path does not +//! run (a declined commit is an `Err`, a vector proof must move the one-row +//! device counters), so none can pass by falling back to the host: +//! +//! ```text +//! cargo test -p stark --release --features cuda --lib \ +//! tests::zf_s2_device_tests::trees_ -- --ignored +//! LAMBDA_VM_GPU_LDE_THRESHOLD=2 cargo test -p stark --release --features cuda --lib \ +//! tests::zf_s2_device_tests::fri_ -- --ignored +//! LAMBDA_VM_GPU_LDE_THRESHOLD=1024 cargo test -p stark --release --features cuda --lib \ +//! tests::zf_s2_device_tests::proved_one_row_vectors_equal_the_cpu_bytes \ +//! -- --ignored --exact --test-threads=1 +//! ``` + +use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; +use crate::fri::device_parity::{Case, one_row_cases, one_row_resident_cases, run_cases}; +use crate::s2_device_parity::run_tree_parity; + +fn trees(name: &str) { + if let Err(failures) = run_tree_parity::(name) { + panic!("{name}: {failures:#?}"); + } +} + +fn fri(name: &str, cases: &[Case], resident: bool, seed: u64) { + if let Err(failures) = run_cases::(name, cases, resident, seed) { + panic!("{name}: {failures:#?}"); + } +} + +/// The one-row case list is pinned by count, so the box run's pre-registered +/// `FRIDEV … cases equal` lines mean something (29 DP shapes + 6 pair-mode + 5 +/// production; 5 resident). +#[test] +fn one_row_case_lists_are_pinned() { + assert_eq!(one_row_cases().len(), 29 + 6 + 5); + assert_eq!(one_row_resident_cases().len(), 5); + for (lde_log, o) in one_row_cases().iter().chain(&one_row_resident_cases()) { + assert_eq!( + o.format.one_row, + crate::proof::options::OneRowMode::On, + "LDE 2^{lde_log}: a one-row case without one-row openings" + ); + } +} + +#[test] +#[ignore = "requires a GPU; run with --features cuda -- --ignored"] +fn trees_one_row_keccak() { + trees::("keccak"); +} + +#[test] +#[ignore = "requires a GPU; run with --features cuda -- --ignored"] +fn trees_one_row_blake3() { + trees::("blake3"); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn fri_one_row_keccak() { + fri::("keccak", &one_row_cases(), false, 0x5234_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn fri_one_row_blake3() { + fri::("blake3", &one_row_cases(), false, 0x5234_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn fri_one_row_resident_keccak() { + fri::("keccak", &one_row_resident_cases(), true, 0x5235_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn fri_one_row_resident_blake3() { + fri::("blake3", &one_row_resident_cases(), true, 0x5235_0000); +} + +/// The (e) vector proofs (the README's (e): `one_row_pair` and +/// `one_row_3_2_1_2`, LDE 4096, Q = 3, grinding 0) proved on the device path +/// are byte-identical to the checked-in CPU-proved files (rkyv bytes and the +/// verifier-derived JSON), under Keccak and Blake3. Each proof must take the +/// one-row device FRI commit once (the input tree off the DEEP codeword) and +/// build its main, aux and composition trees one-row on the device (at least +/// three one-row device trees per proof), so a host fallback fails the test. +/// Run alone (`--exact --test-threads=1`): the counters are process-wide. +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD<=4096; run alone with --features cuda -- --ignored --exact --test-threads=1"] +fn proved_one_row_vectors_equal_the_cpu_bytes() { + use crate::fri::vectors::{check_or_write, one_row_proof_vectors}; + let fri_before = crate::gpu_lde::gpu_one_row_fri_calls(); + let trees_before = crate::gpu_lde::gpu_one_row_trees(); + let comp_before = crate::gpu_lde::gpu_composition_calls(); + let mut files = one_row_proof_vectors::("keccak"); + files.extend(one_row_proof_vectors::("blake3")); + let fri_commits = crate::gpu_lde::gpu_one_row_fri_calls() - fri_before; + let trees = crate::gpu_lde::gpu_one_row_trees() - trees_before; + let compositions = crate::gpu_lde::gpu_composition_calls() - comp_before; + println!( + "S2DEV vector proofs: {} files, {fri_commits} one-row device FRI commits, {trees} one-row device trees, {compositions} device compositions", + files.len() + ); + // Two (e) formats x two hashes, two files per proof. + assert_eq!(files.len(), 2 * 2 * 2); + assert_eq!( + fri_commits, 4, + "every one-row vector proof must take the device FRI commit (lower LAMBDA_VM_GPU_LDE_THRESHOLD)" + ); + assert!( + trees >= 3 * 4, + "every one-row vector proof must build its main, aux and composition trees on the device \ + ({trees} one-row device trees for 4 proofs)" + ); + // Every proof composes on the device (the AIR's constraint program); a + // host composition would not be counted here. + assert_eq!( + compositions, 4, + "every one-row vector proof must compose on the device ({compositions} device compositions)" + ); + let bad = check_or_write(&files, false); + assert!( + bad.is_empty(), + "device-proved one-row vectors differ from the checked-in CPU bytes: {bad:?}" + ); +} diff --git a/crypto/stark/src/trace.rs b/crypto/stark/src/trace.rs index f953faac8..d1efc1893 100644 --- a/crypto/stark/src/trace.rs +++ b/crypto/stark/src/trace.rs @@ -3,8 +3,6 @@ use crate::table::Table; use math::field::traits::{IsField, IsSubFieldOf}; use math::field::{element::FieldElement, traits::IsFFTField}; use math::polynomial::barycentric_inv_denoms; -#[cfg(feature = "disk-spill")] -use math::spill_safe::SpillSafe; #[cfg(feature = "parallel")] use rayon::prelude::{ IndexedParallelIterator, IntoParallelIterator, ParallelIterator, ParallelSliceMut, @@ -35,8 +33,8 @@ where /// round-trip. None on the CPU / download path. #[cfg(feature = "cuda")] pub(crate) aux_resident: Option, - /// Whether the GPU-resident aux build is allowed (false under disk-spill, - /// which needs the aux columns in the host trace to spill them). + /// Whether the GPU-resident aux build is allowed (false under + /// `RecomputeLde`, which keeps the aux build on the host). #[cfg(feature = "cuda")] pub(crate) resident_aux_ok: bool, /// Trace-domain main columns kept resident on device from the R1 main LDE @@ -51,6 +49,113 @@ where /// instead of paying the H2D inside its chain. #[cfg(feature = "cuda")] pub(crate) main_rowmajor_dev: Option, + /// Builds the row-major main trace on the device from fewer bytes than the + /// whole trace (see [`MainDevProducer`]); the R1 main commit calls it, inside + /// the prove, when nothing was pre-uploaded. + #[cfg(feature = "cuda")] + pub(crate) main_dev_producer: Option, + /// The main trace packed at the bytes its columns need + /// ([`Self::pack_main_narrow`]), column by column: `main_table` then holds + /// no data, only its width and height. Only the WHIR block prover packs a + /// trace, and it takes the packed columns as they are. + pub(crate) narrow_main: Option, + /// The main trace held as the records its rows come from + /// ([`Self::from_records_main`]): `main_table` then holds no data, only + /// its width and height. Only the WHIR block prover takes such a trace; + /// the card fills its columns from the records. + pub(crate) records_main: Option, +} + +/// A table's row-major main trace built on the device, called with the host +/// row-major words and the trace's `(rows, columns)`: returns the whole trace on +/// the card, word for word what the host buffer holds, or `None` (the commit +/// then uploads the host buffer as before). It exists so a table whose columns +/// are mostly a function of a few can upload the few and derive the rest on +/// the card; the host trace stays complete, so every host reader is unchanged. +/// The returned buffer must be ready for any stream (its producing stream +/// synchronized). +#[cfg(feature = "cuda")] +#[derive(Clone)] +pub struct MainDevProducer( + #[allow(clippy::type_complexity)] + pub std::sync::Arc< + dyn Fn(&[u64], usize, usize) -> Option> + Send + Sync, + >, +); + +#[cfg(feature = "cuda")] +impl core::fmt::Debug for MainDevProducer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MainDevProducer").finish_non_exhaustive() + } +} + +#[cfg(feature = "cuda")] +impl MainDevProducer { + /// The producer for a table whose row is one width-16 Poseidon1 + /// permutation's witness: `prep_w` preprocessed words, the sixteen input + /// lanes, then from `rounds_off` the `x³`/`x⁷` of every S-boxed lane of every + /// round, for the first `real_rows` rows (padding rows hold zeros past their + /// inputs). It uploads the leading `prep_w + 16` words of each row and + /// derives the rest on the card (`math_cuda::fill::lfm_p1w16_socket_rows`), + /// word for word the host fill's. Declines (`None`) a trace of another + /// width or with more real rows than rows. + pub fn poseidon1_w16_rows( + real_rows: usize, + prep_w: usize, + width: usize, + rounds_off: usize, + ) -> Self { + Self(std::sync::Arc::new( + move |words: &[u64], rows: usize, cols: usize| { + const LANES: usize = 16; + let narrow_w = prep_w + LANES; + if cols != width || real_rows > rows || narrow_w > width || rounds_off < narrow_w { + return None; + } + let mut narrow = vec![0u64; rows * narrow_w]; + #[cfg(feature = "parallel")] + { + use rayon::prelude::{ParallelSlice, ParallelSliceMut}; + narrow + .par_chunks_mut(narrow_w) + .zip(words.par_chunks(width)) + .for_each(|(n, w)| n.copy_from_slice(&w[..narrow_w])); + } + #[cfg(not(feature = "parallel"))] + for (n, w) in narrow.chunks_mut(narrow_w).zip(words.chunks(width)) { + n.copy_from_slice(&w[..narrow_w]); + } + math_cuda::fill::lfm_p1w16_socket_rows( + &narrow, rows, real_rows, prep_w, width, rounds_off, + ) + .ok() + }, + )) + } +} + +/// Excluded from logical trace equality, like [`PreUploadedMainTrace`]: it +/// produces data the host trace already holds. +#[cfg(feature = "cuda")] +impl PartialEq for MainDevProducer { + fn eq(&self, _other: &Self) -> bool { + true + } +} + +#[cfg(feature = "cuda")] +impl Eq for MainDevProducer {} + +/// Tables whose R1 main commit took its trace from a [`MainDevProducer`]. +#[cfg(feature = "cuda")] +pub(crate) static MAIN_DEV_PRODUCER_CALLS: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +/// How many R1 main commits so far took their trace from a device producer. +#[cfg(feature = "cuda")] +pub fn main_dev_producer_calls() -> u64 { + MAIN_DEV_PRODUCER_CALLS.load(std::sync::atomic::Ordering::Relaxed) } /// Device-resident row-major main trace, pre-uploaded ahead of the prove. @@ -188,6 +293,10 @@ where main_trace_dev: None, #[cfg(feature = "cuda")] main_rowmajor_dev: None, + #[cfg(feature = "cuda")] + main_dev_producer: None, + narrow_main: None, + records_main: None, } } @@ -218,6 +327,10 @@ where main_trace_dev: None, #[cfg(feature = "cuda")] main_rowmajor_dev: None, + #[cfg(feature = "cuda")] + main_dev_producer: None, + narrow_main: None, + records_main: None, } } @@ -241,6 +354,10 @@ where main_trace_dev: None, #[cfg(feature = "cuda")] main_rowmajor_dev: None, + #[cfg(feature = "cuda")] + main_dev_producer: None, + narrow_main: None, + records_main: None, } } @@ -248,6 +365,110 @@ where self.main_table.height } + /// Packs the main trace at the bytes its columns need + /// ([`multilinear::narrow::NarrowColumns::pack_row_major`]) and frees the + /// 64-bit copy; the raw words come back bit for bit. Only a Goldilocks + /// trace held whole in memory packs: `false` leaves the trace as it was. + pub fn pack_main_narrow(&mut self) -> bool + where + F: 'static, + { + if self.narrow_main.is_some() { + return true; + } + let table = &self.main_table; + if std::any::TypeId::of::() + != std::any::TypeId::of::() + || table.width == 0 + || table.data.len() != table.width * table.height + { + return false; + } + // SAFETY: `F == GoldilocksField`, a transparent wrapper over `u64`. + let words = unsafe { + core::slice::from_raw_parts(table.data.as_ptr() as *const u64, table.data.len()) + }; + let Some(packed) = multilinear::narrow::NarrowColumns::pack_row_major(words, table.width) + else { + return false; + }; + self.main_table.data = Vec::new(); + self.narrow_main = Some(packed); + true + } + + /// A trace held packed from the start: `packed`'s rows and columns, with + /// no 64-bit copy ever made (the prover's G-pack). `Err` gives `packed` + /// back where a trace cannot be held packed, as [`Self::pack_main_narrow`] + /// refuses it: not Goldilocks, or no columns. + pub fn try_from_narrow_main( + packed: multilinear::narrow::NarrowColumns, + step_size: usize, + ) -> Result + where + F: 'static, + { + if std::any::TypeId::of::() + != std::any::TypeId::of::() + || packed.cols() == 0 + { + return Err(packed); + } + let mut trace = Self::new_main(Vec::new(), packed.cols(), step_size); + trace.main_table.height = packed.rows(); + trace.narrow_main = Some(packed); + Ok(trace) + } + + /// A trace held as the records its rows come from: `records`' rows and + /// columns, with no column of words made on the host. `Err` gives + /// `records` back where a trace cannot be held so: not Goldilocks, or no + /// columns. + pub fn try_from_records_main( + records: multilinear::records::Records, + step_size: usize, + ) -> Result + where + F: 'static, + { + if std::any::TypeId::of::() + != std::any::TypeId::of::() + || records.cols() == 0 + { + return Err(records); + } + let mut trace = Self::new_main(Vec::new(), records.cols(), step_size); + trace.main_table.height = records.rows(); + trace.records_main = Some(records); + Ok(trace) + } + + /// The records the main trace is held as, when it is. + pub fn records_main(&self) -> Option<&multilinear::records::Records> { + self.records_main.as_ref() + } + + /// Takes the records out, leaving the table's shape. + pub fn take_records_main(&mut self) -> Option { + self.records_main.take() + } + + /// The packed main trace, when [`Self::pack_main_narrow`] packed it. + pub fn narrow_main(&self) -> Option<&multilinear::narrow::NarrowColumns> { + self.narrow_main.as_ref() + } + + /// The packed main trace, to edit in place (a cell rewritten within its + /// column's width). + pub fn narrow_main_mut(&mut self) -> Option<&mut multilinear::narrow::NarrowColumns> { + self.narrow_main.as_mut() + } + + /// Takes the packed main trace out, leaving the table's shape. + pub fn take_narrow_main(&mut self) -> Option { + self.narrow_main.take() + } + /// Store the resident (pre-LDE) LogUp aux columns, threaded to the aux commit. #[cfg(feature = "cuda")] pub fn set_aux_resident(&mut self, ra: math_cuda::logup::ResidentAux) { @@ -260,13 +481,13 @@ where self.aux_resident.as_ref() } - /// Whether the GPU-resident aux build is allowed (false under disk-spill). + /// Whether the GPU-resident aux build is allowed. #[cfg(feature = "cuda")] pub fn resident_aux_ok(&self) -> bool { self.resident_aux_ok } - /// Disable the GPU-resident aux build (host trace needed, e.g. disk-spill). + /// Disable the GPU-resident aux build (host trace needed). #[cfg(feature = "cuda")] pub fn set_resident_aux_ok(&mut self, ok: bool) { self.resident_aux_ok = ok; @@ -306,6 +527,58 @@ where self.main_rowmajor_dev.as_ref().map(|p| p.buf.as_ref()) } + /// Attach a [`MainDevProducer`]: the R1 main commit builds this table's + /// row-major trace on the card with it instead of uploading the host buffer. + #[cfg(feature = "cuda")] + pub fn set_main_dev_producer(&mut self, producer: MainDevProducer) { + self.main_dev_producer = Some(producer); + } + + /// Whether a [`MainDevProducer`] is attached. + #[cfg(feature = "cuda")] + pub fn has_main_dev_producer(&self) -> bool { + self.main_dev_producer.is_some() + } + + /// Detach the [`MainDevProducer`]: what a caller that edits the host trace + /// after it was built must do, or the card would rebuild the unedited rows. + #[cfg(feature = "cuda")] + pub fn clear_main_dev_producer(&mut self) { + self.main_dev_producer = None; + } + + /// The row-major main trace on the card from the attached producer, when + /// there is one, nothing was pre-uploaded, and the host trace is held whole + /// as 64-bit words. `None` means the commit uploads the host buffer. + #[cfg(feature = "cuda")] + pub(crate) fn produce_main_dev(&self) -> Option> + where + F: 'static, + { + let producer = self.main_dev_producer.as_ref()?; + if self.main_rowmajor_dev.is_some() + || std::any::TypeId::of::() + != std::any::TypeId::of::() + { + return None; + } + let table = &self.main_table; + if table.width == 0 || table.data.len() != table.width * table.height { + return None; + } + // SAFETY: `F == GoldilocksField` (checked above), `#[repr(transparent)]` + // over `u64`. + let words = unsafe { + core::slice::from_raw_parts(table.data.as_ptr() as *const u64, table.data.len()) + }; + let buf = (producer.0)(words, table.height, table.width)?; + if buf.len() != words.len() { + return None; + } + MAIN_DEV_PRODUCER_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Some(buf) + } + /// Drop the pre-uploaded row-major trace. Its only consumer is the R1 main /// commit, so the prover clears it alongside `clear_main_trace_dev` to /// reclaim the VRAM before the aux-commit + DEEP/FRI peak. @@ -314,6 +587,22 @@ where self.main_rowmajor_dev = None; } + /// Free the auxiliary columns, keeping the declared aux width. + /// + /// Called by `multi_prove` under `ResidencyMode::RecomputeLde` once a + /// table's proof exists: `allocate_aux_table` writes the LogUp columns into + /// this caller-owned trace and nothing reads them afterwards, so under that + /// mode they are released rather than carried to the end of the prove. + /// Callers that do read a trace's aux columns after proving must use + /// `ResidencyMode::Retain`. + pub fn release_aux_columns(&mut self) { + self.aux_table = Table::new(Vec::new(), self.aux_table.width); + #[cfg(feature = "cuda")] + { + self.aux_resident = None; + } + } + pub fn num_steps(&self) -> usize { debug_assert!(self.main_table.height.is_multiple_of(self.step_size)); self.main_table.height / self.step_size @@ -364,26 +653,6 @@ where self.num_aux_columns = num_aux_columns; } - /// Write main trace data to a temp file and free the in-memory vector. - /// Accessors read from the mmap after this call. - #[cfg(feature = "disk-spill")] - pub fn spill_main_to_disk(&mut self) -> std::io::Result<()> - where - F: Copy + 'static, - F::BaseType: SpillSafe, - { - self.main_table.spill_to_disk() - } - - #[cfg(feature = "disk-spill")] - pub fn spill_aux_to_disk(&mut self) -> std::io::Result<()> - where - E: Copy + 'static, - E::BaseType: SpillSafe, - { - self.aux_table.spill_to_disk() - } - /// Extract main columns as owned vectors, each allocated at `capacity`. /// Pass the LDE size so downstream FFT expansion is in-place. pub fn extract_columns_main(&self, capacity: usize) -> Vec>> { @@ -841,7 +1110,11 @@ where E: IsField + 'static, { let n = domain.interpolation_domain_size; - let bf = domain.blowup_factor; + // The read stride is the TABLE's own blowup, not the domain's: for every + // existing caller the two coincide (the table was expanded at the + // domain's blowup), and the batched phase 4 hands a blowup-1 table that + // IS the stride subsample already — same values, a quarter the buffer. + let bf = lde_trace.blowup_factor; let num_main_cols = lde_trace.num_main_cols(); let num_aux_cols = lde_trace.num_aux_cols(); let table_width = num_main_cols + num_aux_cols; diff --git a/crypto/stark/src/traits.rs b/crypto/stark/src/traits.rs index 0aec97a2a..fcb65fc11 100644 --- a/crypto/stark/src/traits.rs +++ b/crypto/stark/src/traits.rs @@ -7,8 +7,10 @@ use math::field::{ }; use crate::{ - constraint_ir::ConstraintProgram, constraints::builder::ConstraintMeta, domain::Domain, - lookup::BusPublicInputs, + constraint_ir::ConstraintProgram, + constraints::builder::ConstraintMeta, + domain::Domain, + lookup::{BusInteraction, BusPublicInputs}, }; use super::{ @@ -171,6 +173,20 @@ pub trait AIR: Send + Sync { 0 } + /// The table's bus interactions, in declaration order. + /// + /// Shape-only consumers (profiling, cost models) read this; the proving + /// path consumes the `LogUpLayout` built from the same list. + fn bus_interactions(&self) -> &[BusInteraction] { + &[] + } + + /// Highest degree among this table's transition constraints, counting both + /// the table's own constraints and the framework-emitted LogUp ones. + fn max_constraint_degree(&self) -> usize { + 1 + } + /// Returns true if this AIR has preprocessed (precomputed) columns. /// /// Preprocessed tables have columns that are fully deterministic and known @@ -194,6 +210,35 @@ pub trait AIR: Send + Sync { [0u8; 32] } + /// The hardcoded commitment to the precomputed columns under the trace + /// trees' leaf `layout` (S2). The root depends on the layout (a one-row + /// leaf hashes different bytes), so each layout has its own trust anchor. + /// + /// `None` = this AIR has no root for `layout`: the prover refuses to prove + /// and the verifier rejects (never a silent recompute, never + /// the other layout's root). The default serves today's layout only. + /// Only meaningful if `is_preprocessed()` returns true. + fn precomputed_commitment_for( + &self, + layout: crate::leaf_layout::LeafLayout, + ) -> Option { + match layout { + crate::leaf_layout::LeafLayout::RowPair => Some(self.precomputed_commitment()), + crate::leaf_layout::LeafLayout::Row => None, + } + } + + /// The precomputed columns themselves, `0..num_precomputed_columns()`. + /// + /// Empty unless `is_preprocessed()`. The univariate path never needs these + /// — it compares [`precomputed_commitment`](Self::precomputed_commitment) + /// against the proof's root — but the multilinear one has no separate root + /// to compare, so it checks the claimed openings against these directly. + /// Generating them is the same work recomputing that commitment costs. + fn precomputed_columns(&self) -> Vec>> { + Vec::new() + } + fn num_auxiliary_rap_columns(&self) -> usize { self.trace_layout().1 } @@ -270,17 +315,48 @@ pub trait AIR: Send + Sync { /// prefix (its length is `num_base_transition_constraints()`). fn constraints_meta(&self) -> &[ConstraintMeta]; - /// The lazily captured flat IR ([`ConstraintProgram`]) of every transition - /// constraint, for the CPU interpreter and the GPU kernel. + /// The flat IR ([`ConstraintProgram`]) of every transition constraint, for + /// the CPU interpreter and the GPU kernel — captured on demand unless a + /// pre-captured program was supplied (see + /// [`Self::precaptured_constraint_program`]). /// - /// GUEST-SAFETY: capture hash-conses, so the verify/recursion path must - /// NEVER call this — only the prover, GPU lowering, and tests do. The - /// default panics precisely so any accidental verify-path use is caught; - /// AIRs that support capture override it with a cached (`OnceLock`) build. + /// GUEST-SAFETY: this MAY capture, and capture hash-conses, so the + /// verify/recursion path must never call it — only the prover, GPU + /// lowering, and tests do. The default panics precisely so any accidental + /// verify-path use is caught; AIRs that support capture override it with a + /// cached (`OnceLock`) build. + /// + /// The prohibition is on CAPTURE, not on constraint programs as such: a + /// program serialized at build time is ordinary data, and consuming one is + /// allowed anywhere. That path is + /// [`Self::precaptured_constraint_program`]. fn constraint_program(&self) -> &ConstraintProgram { unimplemented!("constraint_program is not available for this AIR") } + /// A pre-captured constraint program supplied at build time, if this AIR was + /// given one; `None` otherwise. + /// + /// GUEST-SAFETY: unlike [`Self::constraint_program`], this NEVER captures + /// under any circumstance — it is a borrow of data handed to the AIR at + /// construction, so it is safe on the verify/recursion path. That is the + /// entire distinction between the two methods, and it is why they are + /// separate rather than one method with a flag: an accidental verify-path + /// call to the capturing one still hits the panic above. + /// + /// `None` is not an error — it means nobody supplied an artifact, and the + /// caller must fall back to the compiled folder. A caller that needs a + /// program on a guest path must treat `None` as fatal itself, because + /// falling back to `constraint_program()` there would reintroduce capture. + /// + /// See [`ConstraintArtifact`](crate::constraint_ir::ConstraintArtifact) for + /// the serialized form and for what validating one does and does not prove. + fn precaptured_constraint_program( + &self, + ) -> Option<&ConstraintProgram> { + None + } + fn boundary_constraints( &self, pub_inputs: &Self::PublicInputs, diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index 44add9c21..1bd824f5d 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -1,10 +1,12 @@ use super::{ - config::BatchedMerkleTreeBackend, + config::{DefaultStarkHash, StarkHash}, domain::VerifierDomain, grinding, proof::stark::StarkProof, traits::{AIR, TransitionEvaluationContext}, }; +use crate::leaf_layout::LeafLayout; +use crate::merkle_caps::{StarkCaps, TableTreeChecks, TreeCheck}; pub use crate::proof::view::PiDeserializer; use crate::{ config::Commitment, @@ -18,7 +20,8 @@ use crate::{ table::Table, }; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; -use crypto::merkle_tree::proof::{verify_merkle_path, verify_merkle_path_from_leaf_hash}; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use crypto::merkle_tree::traits::IsStreamingLeafBackend; #[cfg(not(feature = "test_fiat_shamir"))] use log::error; #[cfg(feature = "debug-checks")] @@ -36,20 +39,35 @@ use std::marker::PhantomData; #[cfg(feature = "instruments")] use std::time::Instant; -/// A default STARK verifier implementing `IsStarkVerifier`. -pub struct Verifier< +/// A default STARK verifier implementing `IsStarkVerifier`, generic over the +/// commitment configuration `H`. Mirrors `GenericProver`: `H` rides on the +/// concrete type so [`Verifier`] pins it and existing call sites resolve +/// unchanged. +pub struct GenericVerifier< Field: IsSubFieldOf + IsFFTField + Send + Sync, FieldExtension: Send + Sync + IsField, PI, + H, > { - phantom: PhantomData<(Field, FieldExtension, PI)>, + phantom: PhantomData<(Field, FieldExtension, PI, H)>, } +/// The production verifier: [`GenericVerifier`] at the default commitment +/// configuration. +/// +/// Naming [`DefaultStarkHash`] rather than a fixed hash is what lets the RV64 +/// recursion guest verify a BLAKE3-committed proof: `verify_and_attest_blob` and +/// `verify_continuation_and_attest` take only a `&ProofOptions`, so this alias is +/// the only thing that selects the configuration they reach. +pub type Verifier = + GenericVerifier; + impl< - Field: IsSubFieldOf + IsFFTField + Send + Sync, - FieldExtension: IsField + Send + Sync, + Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, + FieldExtension: IsField + Send + Sync + 'static, PI, -> IsStarkVerifier for Verifier + H: StarkHash, +> IsStarkVerifier for GenericVerifier where Field::BaseType: math::field::element::NativeArchived, FieldExtension::BaseType: math::field::element::NativeArchived, @@ -82,6 +100,24 @@ where pub rap_challenges: Vec>, /// The seed used to verify the proof-of-work nonce. pub grinding_seed: [u8; 32], + /// The commitment-time answers' DEEP term, on a bound proof + /// (`crate::commit_ood`). + pub commit_ood: Option>, +} + +/// One table's commitment-time answers as its DEEP reconstruction folds them: +/// `Σ_i coeffs[i]·(t_{first+i}(x) − v_i)/(x − point)`, which with each +/// answered column's quotient at `z` forms its two-point quotient. +pub struct CommitOodDeep { + /// `z0`. + pub point: FieldElement, + /// The first answered column in `precomputed ‖ main ‖ aux` order. + pub first_col: usize, + /// Minus each answered column's current-row coefficient + /// (`crate::commit_ood::answer_coefficient`). + pub coeffs: Vec>, + /// `Σ_i coeffs[i]·v_i`, the same for every query. + pub answers_sum: FieldElement, } pub type DeepPolynomialEvaluations = (Vec>, Vec>); @@ -94,15 +130,15 @@ where { /// `ood_row_sum[row] = sum_col trace_term_coeffs[col][row] * ood(row, col)`, /// over the reconstructed full OOD grid (g·z-pruned positions are zero). - ood_row_sum: Vec>, + pub(crate) ood_row_sum: Vec>, /// Width of the reconstructed full OOD grid (= full trace width). - ood_width: usize, - /// Derived from `proof.composition_poly_parts_ood_evaluation().len()`. - number_of_parts: usize, + pub(crate) ood_width: usize, + /// Derived from the proof's `composition_poly_parts_ood_evaluation` length. + pub(crate) number_of_parts: usize, /// `challenges.z.pow(number_of_parts)`. - z_pow: FieldElement, + pub(crate) z_pow: FieldElement, /// `sum_j composition_poly_parts_ood_evaluation[j] * challenges.gammas[j]`. - h_sum_zpow: FieldElement, + pub(crate) h_sum_zpow: FieldElement, } // The verifier reads proofs in place from their rkyv archive; archived field @@ -121,26 +157,43 @@ compile_error!("the zero-copy STARK verifier requires a little-endian target"); /// are thin entry points that build the matching view and share every /// downstream check — no serialization, no duplicated logic. pub trait IsStarkVerifier< - Field: IsSubFieldOf + IsFFTField + Send + Sync, - FieldExtension: Send + Sync + IsField, + Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, + FieldExtension: Send + Sync + IsField + 'static, PI, + H: StarkHash, > where Field::BaseType: math::field::element::NativeArchived, FieldExtension::BaseType: math::field::element::NativeArchived, PI: rkyv::Archive + Clone, ::Archived: rkyv::Deserialize, { + /// The query indexes: leaf indexes of the trace trees, uniform below + /// [`LeafLayout::query_bound`] — `lde / 2` (a row PAIR) today, `lde` under + /// one-row openings, where each index is one point of `D₀` (soundness: + /// sampling a pair and opening one of its points would bias `x₀`). fn sample_query_indexes( number_of_queries: usize, domain: &VerifierDomain, + leaf_layout: LeafLayout, transcript: &mut impl IsStarkTranscript, ) -> Vec { - let domain_size = domain.lde_length as u64; + let bound = leaf_layout.query_bound(domain.lde_length as u64); (0..number_of_queries) - .map(|_| (transcript.sample_u64(domain_size >> 1)) as usize) + .map(|_| (transcript.sample_u64(bound)) as usize) .collect::>() } + /// The trace-tree leaf layout of `air`'s proof over `trace_length` rows + /// (row pairs, or one row under S2): a verifier-side constant from the + /// AIR's options and widths and the trace length the FRI layout already + /// trusts ([`crate::leaf_layout::table_leaf_layout`]). + fn leaf_layout( + air: &dyn AIR, + trace_length: usize, + ) -> LeafLayout { + crate::leaf_layout::table_leaf_layout(air, trace_length) + } + /// The pruned-OOD layout for this AIR — the single place in the verifier that /// reads the shape metadata (`trace_columns`, `step_size`, the /// transition-offset count, and the next-row column set). Everything that used @@ -236,6 +289,16 @@ pub trait IsStarkVerifier< None => return false, }; let expected_aux = air.num_auxiliary_rap_columns(); + // The symmetric slot exists only for row pairs: a one-row leaf holds + // the queried row alone, so every `evaluations_sym` must be EMPTY (a + // non-empty one would be hashed into the leaf and read by nothing). + let one_row = Self::leaf_layout(air, proof.trace_length()).is_one_row(); + let sym = |n: usize| if one_row { 0 } else { n }; + let (sym_precomputed, sym_main, sym_aux) = ( + sym(expected_precomputed), + sym(expected_main), + sym(expected_aux), + ); if proof.deep_poly_openings_len() < num_queries { return false; @@ -255,17 +318,25 @@ pub trait IsStarkVerifier< let main = opening.main_trace_polys(); precomputed == expected_precomputed - && precomputed_sym == expected_precomputed + && precomputed_sym == sym_precomputed && main.evaluations().len() == expected_main - && main.evaluations_sym().len() == expected_main + && main.evaluations_sym().len() == sym_main && aux == expected_aux - && aux_sym == expected_aux + && aux_sym == sym_aux + && (!one_row || opening.composition_poly().evaluations_sym().is_empty()) }) } + /// The three proof-derived inputs are passed as plain data rather than read + /// off a `StarkProofView`, so the check does not require its caller to hold + /// one. One constraint check, whatever the caller reads it from. + #[allow(clippy::too_many_arguments)] fn step_2_verify_claimed_composition_polynomial( air: &dyn AIR, - proof: StarkProofView<'_, Field, FieldExtension, PI>, + trace_length: usize, + bus_table_contribution: Option>, + ood_current_row: &[FieldElement], + composition_parts_ood: &[FieldElement], public_inputs: &PI, domain: &VerifierDomain, challenges: &Challenges, @@ -280,11 +351,10 @@ pub trait IsStarkVerifier< crate::profile_markers::step_marker::< { crate::profile_markers::STEP_VERIFY_CLAIMED_COMPOSITION_POLYNOMIAL }, >(); - let trace_length = proof.trace_length(); // Owned `BusPublicInputs` (just the table contribution L — one field // element) reconstructed for the AIR boundary call. - let bus_public_inputs = proof - .bus_table_contribution() + let bus_public_inputs = bus_table_contribution + .clone() .map(BusPublicInputs::from_contribution); let boundary_constraints = air.boundary_constraints( @@ -309,8 +379,7 @@ pub trait IsStarkVerifier< .collect(); let main_trace_width = air.trace_layout().0; - let trace_ood_evaluations = proof.trace_ood_evaluations(); - let ood_row = trace_ood_evaluations.get_row(0); + let ood_row = ood_current_row; let (boundary_c_i_evaluations_num, mut boundary_c_i_evaluations_den): ( Vec>, @@ -352,8 +421,8 @@ pub trait IsStarkVerifier< // aux count; reject instead of underflowing. The current-row block keeps // the full trace width even under g·z pruning, so this still yields the // main width. - let num_main_trace_columns = match trace_ood_evaluations - .width() + let num_main_trace_columns = match ood_current_row + .len() .checked_sub(air.num_auxiliary_rap_columns()) { Some(n) => n, @@ -370,7 +439,7 @@ pub trait IsStarkVerifier< Vec::new() }; - let logup_table_offset = match proof.bus_table_contribution() { + let logup_table_offset = match bus_table_contribution { Some(contribution) => { let n = FieldElement::::from(trace_length as u64); match n.inv() { @@ -418,8 +487,7 @@ pub trait IsStarkVerifier< let composition_poly_ood_evaluation = &boundary_quotient_ood_evaluation + transition_c_i_evaluations_sum; - let composition_poly_claimed_ood_evaluation = proof - .composition_poly_parts_ood_evaluation() + let composition_poly_claimed_ood_evaluation = composition_parts_ood .iter() .rev() .fold(FieldElement::zero(), |acc, coeff| { @@ -436,21 +504,32 @@ pub trait IsStarkVerifier< /// arithmetic as the CPU and GPU provers; drift between them would break all /// proofs. `VerifierDomain.lde_length` is the codeword size and /// `lde_length / trace_length` the blowup factor. + /// + /// The proof FORMAT (fold schedule, encoding) comes from `air.options()` — + /// a verifier-side constant, never read from the proof. `None` when the + /// format cannot be laid out for this table (a one-row mode, or a schedule + /// override that does not fit): the proof is then rejected. // `FriFoldLayout` is a crate-internal helper type returned from a default method // of this public trait; the exposure is intentional (internal helper). #[allow(private_interfaces)] fn fri_termination_params( air: &dyn AIR, domain: &VerifierDomain, - ) -> crate::fri::terminal::FriFoldLayout { - let k = air.options().fri_final_poly_log_degree as u32; + ) -> Option { let blowup_log = (domain.lde_length / domain.trace_length).trailing_zeros(); - crate::fri::terminal::FriFoldLayout::new(domain.lde_length.trailing_zeros(), blowup_log, k) + crate::fri::terminal::FriFoldLayout::for_options( + domain.lde_length.trailing_zeros(), + blowup_log, + air.options(), + Self::leaf_layout(air, domain.trace_length).is_one_row(), + ) + .ok() } /// Reconstructs the Deep composition polynomial evaluations at the challenge indices values using the provided /// openings of the trace polynomials and the composition polynomial parts. It then uses these to verify that the /// FRI decommitments are valid and correspond to the Deep composition polynomial. + #[allow(clippy::too_many_arguments)] fn step_3_verify_fri( air: &dyn AIR, proof: StarkProofView<'_, Field, FieldExtension, PI>, @@ -462,31 +541,41 @@ pub trait IsStarkVerifier< ood_full: &Table, next_row_cols: &[usize], step_size: usize, + // The per-tree Merkle checks (`table_tree_checks`); this step reads the + // committed FRI layers' ones. + checks: &TableTreeChecks<'_>, ) -> bool where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, { crate::profile_markers::step_marker::<{ crate::profile_markers::STEP_VERIFY_FRI }>(); + // ---- Reconstruct the FRI terminal codeword from the final-poly coeffs ---- + // The prover folds the deep composition codeword down to a terminal + // codeword of length `terminal_len = 2^(blowup_log + effective_k)` and sends + // the `2^effective_k` coefficients of the low-degree polynomial it encodes. + let Some(layout) = Self::fri_termination_params(air, domain) else { + return false; + }; + let num_committed = layout.num_committed; + // Row pairs: DEEP at `x` and `−x` per query. One row: DEEP at the one + // point `x_r` (the sym vector comes back empty). + let leaf_layout = LeafLayout::from_one_row(layout.one_row); let (deep_poly_evaluations, deep_poly_evaluations_sym) = - match Self::reconstruct_deep_composition_poly_evaluations_for_all_queries( + match Self::reconstruct_deep_composition_poly_evaluations_for_layout( challenges, domain, proof, ood_full, next_row_cols, step_size, + leaf_layout, ) { Some(pair) => pair, None => return false, }; - - // ---- Reconstruct the FRI terminal codeword from the final-poly coeffs ---- - // The prover folds the deep composition codeword down to a terminal - // codeword of length `terminal_len = 2^(blowup_log + effective_k)` and sends - // the `2^effective_k` coefficients of the low-degree polynomial it encodes. - let layout = Self::fri_termination_params(air, domain); - let num_committed = layout.num_committed; + #[cfg(any(test, feature = "test-utils"))] + crate::fri::capture::record_deep(&deep_poly_evaluations, &deep_poly_evaluations_sym); // Structural check: number of committed FRI layers must equal // `num_committed` (zero when no fold or a single final fold happened). @@ -506,14 +595,22 @@ pub trait IsStarkVerifier< // iterations and accept the query vacuously) or padded (making the loop // skip the terminal low-degree check), bypassing FRI entirely. This length // check is the only thing that pins them, so it must run before the loop. + // Opened values per query: one sibling per layer under the legacy + // encoding, every layer's full group otherwise (a format constant). + let values_per_query = layout.opened_values_per_query(); if (0..proof.query_list_len()).any(|i| { let decommitment = proof.query(i); decommitment.layers_auth_paths_len() != num_committed - || decommitment.layers_evaluations_sym().len() != num_committed + || decommitment.layers_evaluations_sym().len() != values_per_query }) { return false; } + // One check per committed layer, built with the same layer count. + if checks.fri.len() != num_committed { + return false; + } + let terminal_offset = domain.coset_offset.pow(1u64 << layout.total_folds); let terminal_codeword = crate::fri::terminal::terminal_codeword_from_coeffs::( @@ -526,13 +623,47 @@ pub trait IsStarkVerifier< let mut evaluation_point_inverse = challenges .iotas .iter() - .map(|iota| Self::query_challenge_to_evaluation_point(*iota, false, domain)) + .map(|iota| Self::query_point(leaf_layout, *iota, domain)) .collect::>>(); // Any zero evaluation point means a malformed query index, reject. if FieldElement::inplace_batch_inverse(&mut evaluation_point_inverse).is_err() { return false; } + if !layout.is_legacy() { + // Group encoding (S3): the ω_{2^d} tables once, then every query. + let mut roots_tables: Vec>> = Vec::new(); + for &d in &layout.schedule { + let d = d as usize; + if roots_tables.len() <= d { + roots_tables.resize(d + 1, Vec::new()); + } + if roots_tables[d].is_empty() { + match crate::fri::group::roots_of_unity_table::(d as u32) { + Some(t) => roots_tables[d] = t, + None => return false, + } + } + } + return (0..challenges.iotas.len()) + .zip(evaluation_point_inverse) + .all(|(i, eval)| { + Self::verify_query_groups( + &layout, + &checks.fri, + i, + &challenges.zetas, + challenges.iotas[i], + proof.query(i), + eval, + &deep_poly_evaluations[i], + deep_poly_evaluations_sym.get(i), + &terminal_codeword, + &roots_tables, + ) + }); + } + (0..challenges.iotas.len()) .zip(evaluation_point_inverse) .all(|(i, eval)| { @@ -545,10 +676,23 @@ pub trait IsStarkVerifier< &deep_poly_evaluations[i], &deep_poly_evaluations_sym[i], &terminal_codeword, + &checks.fri, + i, ) }) } + /// The LDE-coset point query `q` opens first under `leaf_layout`: the row + /// at bit-reversed position `2q` for row pairs (as + /// [`Self::query_challenge_to_evaluation_point`]), `q` for one row. + fn query_point( + leaf_layout: LeafLayout, + q: usize, + domain: &VerifierDomain, + ) -> FieldElement { + domain.lde_coset_element(leaf_layout.query_rows(q, domain.lde_length).0) + } + /// Returns the field element element of the domain `domain` corresponding to the given FRI query index challenge `iota`. /// Returns the LDE-coset element for FRI query challenge `iota`. The /// `sym` flag picks the symmetric counterpart (`iota*2+1`) instead of the @@ -566,37 +710,39 @@ pub trait IsStarkVerifier< /// (`2·iota`, `2·iota+1`) is committed as the single leaf at position `iota`, /// so one Merkle path authenticates both `evaluations` (the row) and /// `evaluations_sym` (its symmetric). Same layout used for trace and composition. + /// + /// `check` is the tree's [`TreeCheck`], built once per tree: it fixes the + /// exact path length (`log2(lde) − 1 − c`, a verifier constant) and, for a + /// capped tree, the authenticated cap the path folds onto. `query` is the + /// opening's position in proof order (query 0 is a capped tree's owner). fn verify_opening_pair( opening: PolynomialOpeningsView<'_, E>, - root: &Commitment, + check: &TreeCheck<'_>, + query: usize, iota: usize, ) -> bool where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, - E: IsField, + E: IsField + 'static, E::BaseType: math::field::element::NativeArchived, Field: IsSubFieldOf, { // Two-slice leaf hash: the committed leaf is `evaluations ‖ evaluations_sym`, // hashed without allocating the concatenation (see `hash_data_from_slices`). - let leaf_hash = BatchedMerkleTreeBackend::::hash_data_from_slices( + let leaf_hash = as IsStreamingLeafBackend>::hash_data_from_slices( opening.evaluations(), opening.evaluations_sym(), ); - verify_merkle_path_from_leaf_hash::>( - opening.merkle_path(), - root, - iota, - leaf_hash, - ) + check.verify::>(query, opening.merkle_path(), iota, leaf_hash) } /// Verify opening Open(tⱼ(D_LDE), 𝜐) and Open(tⱼ(D_LDE), -𝜐) for all trace polynomials tⱼ, /// where 𝜐 and -𝜐 are the elements corresponding to the index challenge `iota`. fn verify_trace_openings( - proof: StarkProofView<'_, Field, FieldExtension, PI>, deep_poly_openings: DeepPolynomialOpeningView<'_, Field, FieldExtension>, + checks: &TableTreeChecks<'_>, + query: usize, iota: usize, ) -> bool where @@ -606,11 +752,13 @@ pub trait IsStarkVerifier< // Main trace (multiplicities for preprocessed, full trace for normal). let mut ok = Self::verify_opening_pair::( deep_poly_openings.main_trace_polys(), - proof.lde_trace_main_merkle_root(), + &checks.main, + query, iota, ); - // Precomputed trace (preprocessed tables only). Mismatched presence: + // Precomputed trace (preprocessed tables only). The check exists iff the + // proof carries a precomputed root (`table_tree_checks`). Mismatched presence: // `(Some(root), None)` and any `(None, Some(opening))` carrying at least // one column are rejected upstream by `trace_opening_widths_well_formed` // (which pins the precomputed opening width to the AIR — zero for a @@ -621,10 +769,12 @@ pub trait IsStarkVerifier< // only site that rejects that shape, and the check keeps the function // self-contained. ok &= match ( - proof.lde_trace_precomputed_merkle_root(), + checks.precomputed.as_ref(), deep_poly_openings.precomputed_trace_polys(), ) { - (Some(root), Some(opening)) => Self::verify_opening_pair::(opening, root, iota), + (Some(check), Some(opening)) => { + Self::verify_opening_pair::(opening, check, query, iota) + } (None, None) => true, _ => false, }; @@ -636,12 +786,9 @@ pub trait IsStarkVerifier< // aux tree got to choose them after seeing `z`/`alpha` // (`tests::aux_opening_width_tests`). The width is pinned upstream by // `trace_opening_widths_well_formed`; do not re-derive it from the proof. - ok &= match ( - proof.lde_trace_aux_merkle_root(), - deep_poly_openings.aux_trace_polys(), - ) { - (Some(root), Some(opening)) => { - Self::verify_opening_pair::(opening, root, iota) + ok &= match (checks.aux.as_ref(), deep_poly_openings.aux_trace_polys()) { + (Some(check), Some(opening)) => { + Self::verify_opening_pair::(opening, check, query, iota) } (None, None) => true, _ => false, @@ -654,8 +801,9 @@ pub trait IsStarkVerifier< /// polynomial, where 𝜐 and -𝜐 are the elements corresponding to the index challenge `iota`. fn verify_composition_poly_opening( deep_poly_openings: DeepPolynomialOpeningView<'_, Field, FieldExtension>, - composition_poly_merkle_root: &Commitment, - iota: &usize, + check: &TreeCheck<'_>, + query: usize, + iota: usize, ) -> bool where FieldElement: AsBytes + Sync + Send, @@ -663,15 +811,17 @@ pub trait IsStarkVerifier< { let composition_poly = deep_poly_openings.composition_poly(); // Two-slice leaf hash of `evaluations ‖ evaluations_sym`, no concat alloc. - let leaf_hash = BatchedMerkleTreeBackend::::hash_data_from_slices( + let leaf_hash = as IsStreamingLeafBackend< + FieldExtension, + >>::hash_data_from_slices( composition_poly.evaluations(), composition_poly.evaluations_sym(), ); - verify_merkle_path_from_leaf_hash::>( + check.verify::>( + query, composition_poly.merkle_path(), - composition_poly_merkle_root, - *iota, + iota, leaf_hash, ) } @@ -682,6 +832,7 @@ pub trait IsStarkVerifier< fn step_4_verify_trace_and_composition_openings( proof: StarkProofView<'_, Field, FieldExtension, PI>, challenges: &Challenges, + checks: &TableTreeChecks<'_>, ) -> bool where FieldElement: AsBytes + Sync + Send, @@ -692,19 +843,126 @@ pub trait IsStarkVerifier< >(); // `step_3_verify_fri` (which runs before this) already rejects proofs // whose `deep_poly_openings` is shorter than `challenges.iotas`. - challenges.iotas.iter().enumerate().all(|(i, iota_n)| { + challenges.iotas.iter().enumerate().all(|(i, &iota_n)| { let deep_poly_opening = proof.deep_poly_opening(i); - Self::verify_composition_poly_opening( - deep_poly_opening, - proof.composition_poly_root(), - iota_n, - ) && Self::verify_trace_openings(proof, deep_poly_opening, *iota_n) + Self::verify_composition_poly_opening(deep_poly_opening, &checks.composition, i, iota_n) + && Self::verify_trace_openings(deep_poly_opening, checks, i, iota_n) + }) + } + + /// The per-tree Merkle checks of one table's proof, built ONCE per tree + /// before any query is verified. + /// + /// Every depth and cap height is a verifier constant ([`StarkCaps`], from + /// the AIR's options and the LDE size): the trace, precomputed, aux and + /// composition trees are `log2(lde) − 1` deep, committed FRI layer `i` is + /// the fold layout's `layer_depth(i)` deep (`log2(lde) − i − 2` under the + /// all-ones schedule, the group tree's depth under any other). Every authentication path must be exactly + /// `depth − c` long, at `c = 0` too: a path of any other length would be + /// folded and compared with the root, letting an internal node pass as a leaf. + /// + /// A capped tree (`c > 0`) reads its owner opening — query 0's path — here, + /// splits off the cap and checks it hashes to the root. That read is safe + /// by construction: the caller runs this only after the + /// `query_list_len` / `trace_opening_widths_well_formed` count guards, and + /// every access below is a length-checked `get`, so a proof with no + /// openings, too few FRI layers, or a missing aux/precomputed opening + /// rejects (`None`) and never panics. At `c = 0` (the default format) no + /// opening is read at all. + /// + /// The FRI layer count is `fri_termination_params(..).num_committed`; a + /// proof with a different number of layer roots is rejected here as in + /// `step_3_verify_fri`. + fn table_tree_checks<'a>( + air: &dyn AIR, + proof: StarkProofView<'a, Field, FieldExtension, PI>, + domain: &VerifierDomain, + ) -> Option> + where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + let options = air.options(); + // A format this verifier cannot lay out rejects here, as in step 3. + let layout = Self::fri_termination_params(air, domain)?; + let num_committed = layout.num_committed; + let lde_log = domain.lde_length.trailing_zeros() as usize; + // Every depth from the table's layouts (row pairs: `log2(lde) − 1`; + // one row: `log2(lde)`; a group tree per FRI layer under a fold + // schedule) — at the default exactly `StarkCaps::new`'s. + let caps = StarkCaps::from_layout_arity( + crate::config::effective_cap_policy::(options.format.merkle_cap), + options.fri_number_of_queries, + lde_log, + &layout, + H::ARITY, + ); + let fri_roots = proof.fri_layers_merkle_roots(); + if fri_roots.len() != num_committed { + return None; + } + // The owner opening: query 0, read only for a capped tree and only + // once it is known to exist. + let owner = || (proof.deep_poly_openings_len() > 0).then(|| proof.deep_poly_opening(0)); + let (d, c) = (caps.trace_depth, caps.trace); + + let main = TreeCheck::build::>( + proof.lde_trace_main_merkle_root(), + d, + c, + || owner().map(|o| o.main_trace_polys().merkle_path()), + )?; + let precomputed = match proof.lde_trace_precomputed_merkle_root() { + Some(root) => Some(TreeCheck::build::>(root, d, c, || { + owner()?.precomputed_trace_polys().map(|p| p.merkle_path()) + })?), + None => None, + }; + let aux = match proof.lde_trace_aux_merkle_root() { + Some(root) => Some(TreeCheck::build::>( + root, + d, + c, + || owner()?.aux_trace_polys().map(|p| p.merkle_path()), + )?), + None => None, + }; + let composition = TreeCheck::build::>( + proof.composition_poly_root(), + d, + c, + || owner().map(|o| o.composition_poly().merkle_path()), + )?; + let fri = fri_roots + .iter() + .enumerate() + .map(|(i, root)| { + TreeCheck::build::>( + root, + caps.fri_depths[i], + caps.fri[i], + || { + (proof.query_list_len() > 0) + .then(|| proof.query(0)) + .filter(|q| q.layers_auth_paths_len() > i) + .map(|q| q.layer_auth_path(i)) + }, + ) + }) + .collect::>>()?; + Some(TableTreeChecks { + main, + precomputed, + aux, + composition, + fri, }) } /// Verifies the openings of a fold polynomial of an inner layer of FRI. fn verify_fri_layer_openings( - merkle_root: &Commitment, + check: &TreeCheck<'_>, + query: usize, auth_path_sym: &[Commitment], evaluation: &FieldElement, evaluation_sym: &FieldElement, @@ -720,11 +978,80 @@ pub trait IsStarkVerifier< vec![evaluation.clone(), evaluation_sym.clone()] }; - verify_merkle_path::>( + check.verify::>( + query, auth_path_sym, - merkle_root, iota >> 1, - &evaluations, + as IsMerkleTreeBackend>::hash_data(&evaluations), + ) + } + + /// Verify a single FRI query under the group encoding (S3 and S2; any + /// format but the legacy one), then [`crate::fri::group::verify_query_groups`] + /// for the committed layers and the terminal check. + /// + /// * Row pairs (`p0_eval_sym = Some`): fold 0 from the DEEP pair as today; + /// the zero-fold case is the legacy one (no layer, no challenge, both + /// points checked against the terminal codeword). + /// * One row (`p0_eval_sym = None`, S2): layer 0 IS the committed DEEP + /// codeword, so the query's value there is `DEEP(x_r)` itself and the + /// layer-0 slot check is the input-slot check `group₀[slot] == DEEP(x_r)`. + /// With nothing to fold the terminal codeword is the DEEP + /// codeword and `terminal[r] == DEEP(x_r)` is the whole check. + // Crate-internal layout type on a default method, as `fri_termination_params`. + #[allow(clippy::too_many_arguments, private_interfaces)] + fn verify_query_groups( + layout: &crate::fri::terminal::FriFoldLayout, + // One per committed layer (`table_tree_checks`, at the layout's group + // tree depths), and this query's position in proof order (query 0 is + // every capped layer's owner). + fri_checks: &[TreeCheck<'_>], + query: usize, + zetas: &[FieldElement], + iota: usize, + fri_decommitment: FriDecommitmentView<'_, FieldExtension>, + evaluation_point_inv: FieldElement, + p0_eval: &FieldElement, + p0_eval_sym: Option<&FieldElement>, + terminal_codeword: &[FieldElement], + roots_tables: &[Vec>], + ) -> bool + where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + // The encoding of the DEEP value(s) must match the layout: a + // one-row layout has no symmetric value, a row-pair one needs it. + let (v, y_inv) = match (layout.one_row, p0_eval_sym) { + (true, None) => (p0_eval.clone(), evaluation_point_inv), + (false, Some(p0_eval_sym)) => { + if zetas.is_empty() { + return terminal_codeword + .get(iota * 2) + .is_some_and(|t| p0_eval == t) + && terminal_codeword + .get(iota * 2 + 1) + .is_some_and(|t| p0_eval_sym == t); + } + // Fold 0 (binary, uncommitted) consumes the DEEP pair: p₁(𝜐²). + let v = (p0_eval + p0_eval_sym) + + &evaluation_point_inv * &zetas[0] * (p0_eval - p0_eval_sym); + (v, evaluation_point_inv.square()) + } + _ => return false, + }; + crate::fri::group::verify_query_groups::>( + layout, + fri_checks, + query, + |j| fri_decommitment.layer_auth_path(j), + fri_decommitment.layers_evaluations_sym(), + zetas, + iota, + v, + y_inv, + terminal_codeword, + roots_tables, ) } @@ -746,6 +1073,10 @@ pub trait IsStarkVerifier< deep_composition_evaluation: &FieldElement, deep_composition_evaluation_sym: &FieldElement, terminal_codeword: &[FieldElement], + // One per committed layer (`table_tree_checks`), and this query's + // position in proof order (query 0 is every capped layer's owner). + fri_checks: &[TreeCheck<'_>], + query: usize, ) -> bool where FieldElement: AsBytes + Sync + Send, @@ -786,19 +1117,20 @@ pub trait IsStarkVerifier< // previous iteration), then obtain pᵢ₊₁(𝜐^(2ⁱ⁺¹)). When there are no // committed layers (`total_folds == 1`, a single final fold) this fold is // empty and `v`/`index` already hold the terminal-layer value/position. - let openings_ok = fri_layers_merkle_roots + let openings_ok = fri_checks .iter() .zip(fri_decommitment.layers_evaluations_sym()) .zip(evaluation_point_vec) .enumerate() .fold( true, - |result, (i, ((merkle_root, evaluation_sym), evaluation_point_inv))| { + |result, (i, ((check, evaluation_sym), evaluation_point_inv))| { // Verify opening Open(pᵢ(Dₖ), −𝜐^(2ⁱ)) and Open(pᵢ(Dₖ), 𝜐^(2ⁱ)). // `v` is pᵢ(𝜐^(2ⁱ)). // `evaluation_sym` is pᵢ(−𝜐^(2ⁱ)). let openings_ok = Self::verify_fri_layer_openings( - merkle_root, + check, + query, fri_decommitment.layer_auth_path(i), &v, evaluation_sym, @@ -836,12 +1168,36 @@ pub trait IsStarkVerifier< /// elsewhere), not from `proof.trace_ood_evaluations()` which now carries /// only the current-row block. Pruned positions are zero in both the grid /// and `trace_term_coeffs`, so next rows sum only the window columns. + /// The per-proof invariant terms, from a `StarkProofView`. + /// + /// A thin delegate to [`Self::query_invariant_deep_terms_from_parts`], kept + /// at this name and signature because it is public API with callers outside + /// this crate (`prover/src/lfm/`'s constraint and join oracles). The batched + /// epoch verifier calls the slice-taking form instead — one OOD walk, two + /// entry points, rather than a second copy of it. fn compute_query_invariant_deep_terms( challenges: &Challenges, proof: StarkProofView<'_, Field, FieldExtension, PI>, ood_full: &Table, next_row_cols: &[usize], step_size: usize, + ) -> Option> { + Self::query_invariant_deep_terms_from_parts( + challenges, + proof.composition_poly_parts_ood_evaluation(), + ood_full, + next_row_cols, + step_size, + ) + } + + /// `composition_parts_ood` is the proof's `Hᵢ(z^N)` values. + fn query_invariant_deep_terms_from_parts( + challenges: &Challenges, + composition_parts_ood: &[FieldElement], + ood_full: &Table, + next_row_cols: &[usize], + step_size: usize, ) -> Option> { let ood_evaluations_table_height = ood_full.height; let ood_evaluations_table_width = ood_full.width; @@ -874,7 +1230,6 @@ pub trait IsStarkVerifier< ood_row_sum.push(sum); } - let composition_parts_ood = proof.composition_poly_parts_ood_evaluation(); let number_of_parts = composition_parts_ood.len(); let z_pow = challenges.z.pow(number_of_parts); @@ -905,6 +1260,31 @@ pub trait IsStarkVerifier< ood_full: &Table, next_row_cols: &[usize], step_size: usize, + ) -> Option> { + Self::reconstruct_deep_composition_poly_evaluations_for_layout( + challenges, + domain, + proof, + ood_full, + next_row_cols, + step_size, + LeafLayout::RowPair, + ) + } + + /// [`Self::reconstruct_deep_composition_poly_evaluations_for_all_queries`] + /// under a leaf layout: for row pairs, DEEP at each query's two points + /// (`x`, `−x`); for one row, DEEP at the query's one point `x_r` and an + /// EMPTY symmetric vector (the openings carry no symmetric row). + #[allow(clippy::too_many_arguments)] + fn reconstruct_deep_composition_poly_evaluations_for_layout( + challenges: &Challenges, + domain: &VerifierDomain, + proof: StarkProofView<'_, Field, FieldExtension, PI>, + ood_full: &Table, + next_row_cols: &[usize], + step_size: usize, + leaf_layout: LeafLayout, ) -> Option> { let num_queries = challenges.iotas.len(); @@ -928,14 +1308,42 @@ pub trait IsStarkVerifier< let primitive_root = &Field::get_primitive_root_of_unity(domain.root_order as u64) .expect("verifier domain root_order is a valid power of two"); - let query_invariant_terms = Self::compute_query_invariant_deep_terms( + let query_invariant_terms = Self::query_invariant_deep_terms_from_parts( challenges, - proof, + proof.composition_poly_parts_ood_evaluation(), ood_full, next_row_cols, step_size, )?; + if leaf_layout.is_one_row() { + for (i, r) in challenges.iotas.iter().enumerate() { + let opening = proof.deep_poly_opening(i); + let lde_precomputed: &[FieldElement] = opening + .precomputed_trace_polys() + .map(|p| p.evaluations()) + .unwrap_or(&[]); + let lde_aux: &[FieldElement] = opening + .aux_trace_polys() + .map(|a| a.evaluations()) + .unwrap_or(&[]); + let point = Self::query_point(leaf_layout, *r, domain); + deep_poly_evaluations.push(Self::reconstruct_deep_composition_poly_evaluation_at( + &point, + primitive_root, + challenges, + &query_invariant_terms, + next_row_cols, + step_size, + lde_precomputed, + opening.main_trace_polys().evaluations(), + lde_aux, + opening.composition_poly().evaluations(), + )?); + } + return Some((deep_poly_evaluations, deep_poly_evaluations_sym)); + } + for (i, iota) in challenges.iotas.iter().enumerate() { let opening = proof.deep_poly_opening(i); @@ -992,6 +1400,102 @@ pub trait IsStarkVerifier< Some((deep_poly_evaluations, deep_poly_evaluations_sym)) } + /// The deep composition polynomial at ONE point (one-row openings, S2): + /// the same terms as [`Self::reconstruct_deep_composition_poly_evaluation_pair`] + /// at `evaluation_point` alone, with the same panic guards (a malformed + /// width, a zero denominator → `None`). + #[allow(clippy::too_many_arguments)] + fn reconstruct_deep_composition_poly_evaluation_at( + evaluation_point: &FieldElement, + primitive_root: &FieldElement, + challenges: &Challenges, + query_invariant_terms: &QueryInvariantDeepTerms, + next_row_cols: &[usize], + step_size: usize, + lde_trace_precomputed_evaluations: &[FieldElement], + lde_trace_main_evaluations: &[FieldElement], + lde_trace_aux_evaluations: &[FieldElement], + lde_composition_poly_parts_evaluation: &[FieldElement], + ) -> Option> { + let height = query_invariant_terms.ood_row_sum.len(); + let width = query_invariant_terms.ood_width; + let trace_term_coeffs = &challenges.trace_term_coeffs; + let num_precomputed = lde_trace_precomputed_evaluations.len(); + let num_base = num_precomputed + lde_trace_main_evaluations.len(); + let base_at = |col: usize| -> &FieldElement { + if col < num_precomputed { + &lde_trace_precomputed_evaluations[col] + } else { + &lde_trace_main_evaluations[col - num_precomputed] + } + }; + if num_base + lde_trace_aux_evaluations.len() != width { + return None; + } + + let mut denoms = Vec::with_capacity(height); + let mut current_z = challenges.z.clone(); + for _ in 0..height { + denoms.push(evaluation_point - ¤t_z); + current_z = primitive_root * ¤t_z; + } + FieldElement::inplace_batch_inverse(&mut denoms).ok()?; + + let mut trace_term = FieldElement::::zero(); + for (row_idx, denom) in denoms.iter().enumerate() { + let ood_row_sum = &query_invariant_terms.ood_row_sum[row_idx]; + let mut base_row_sum = FieldElement::::zero(); + let mut add = |col_idx: usize, coeff: &FieldElement| { + if col_idx < num_base { + base_row_sum += base_at(col_idx) * coeff; + } else { + base_row_sum += coeff * &lde_trace_aux_evaluations[col_idx - num_base]; + } + }; + if row_idx < step_size { + for (col_idx, coeff_col) in trace_term_coeffs.iter().enumerate() { + add(col_idx, &coeff_col[row_idx]); + } + } else { + for &col_idx in next_row_cols { + add(col_idx, &trace_term_coeffs[col_idx][row_idx]); + } + } + trace_term += denom * &(&base_row_sum - ood_row_sum); + } + + // The commitment-time answers' quotients (`crate::commit_ood`). + if let Some(ood) = &challenges.commit_ood { + let denom = (evaluation_point - &ood.point).inv().ok()?; + let mut sum = FieldElement::::zero(); + for (i, coeff) in ood.coeffs.iter().enumerate() { + let col = ood.first_col + i; + if col >= num_base { + return None; + } + sum += base_at(col) * coeff; + } + trace_term += denom * (sum - &ood.answers_sum); + } + + let number_of_parts = query_invariant_terms.number_of_parts; + if lde_composition_poly_parts_evaluation.len() != number_of_parts { + return None; + } + let denom_composition = (evaluation_point - &query_invariant_terms.z_pow) + .inv() + .ok()?; + let mut h_sum = FieldElement::::zero(); + for (h, gamma) in lde_composition_poly_parts_evaluation + .iter() + .zip(&challenges.gammas) + { + h_sum += h * gamma; + } + let h_terms = (&h_sum - &query_invariant_terms.h_sum_zpow) * denom_composition; + Some(trace_term + h_terms) + } + /// Reconstructs the deep composition polynomial evaluation at a query's /// point and its symmetric counterpart together. Rewriting the per-element /// trace term `coeff*(base-ood)*denom` as `denom*(coeff*base - coeff*ood)` @@ -1126,6 +1630,28 @@ pub trait IsStarkVerifier< trace_term_sym += &denoms_trace_sym[row_idx] * &(&base_row_sum_sym - ood_row_sum); } + // The commitment-time answers' quotients (`crate::commit_ood`), at + // both points. + if let Some(ood) = &challenges.commit_ood { + let mut denoms = [ + evaluation_point - &ood.point, + evaluation_point_sym - &ood.point, + ]; + FieldElement::inplace_batch_inverse(&mut denoms).ok()?; + let mut sum = FieldElement::::zero(); + let mut sum_sym = FieldElement::::zero(); + for (i, coeff) in ood.coeffs.iter().enumerate() { + let col = ood.first_col + i; + if col >= num_base { + return None; + } + sum += base_at(col) * coeff; + sum_sym += base_at_sym(col) * coeff; + } + trace_term += &denoms[0] * (sum - &ood.answers_sum); + trace_term_sym += &denoms[1] * (sum_sym - &ood.answers_sum); + } + let number_of_parts = query_invariant_terms.number_of_parts; // Also rejects a per-query opening length that disagrees with the // proof-level `number_of_parts`, not just a regular/symmetric mismatch. @@ -1222,6 +1748,127 @@ pub trait IsStarkVerifier< transcript: &mut (impl IsStarkTranscript + Clone), expected_bus_balance: &FieldElement, ) -> bool + where + Field: 'p, + FieldExtension: 'p, + PI: 'p, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + Self::multi_verify_views_with(airs, proofs, None, transcript, expected_bus_balance) + } + + /// Verifies a proof made by `multi_prove_bound`, with every table's + /// commitment-time answers (`crate::commit_ood`): the answer counts are + /// the AIRs', checked before anything is absorbed; `z0` is drawn after the + /// last root and every answer absorbed before the shared LogUp challenges; + /// each table's DEEP polynomial carries its answers. + fn multi_verify_views_bound<'p>( + airs: &[&dyn AIR], + proofs: impl ProofViewSource<'p, Field, FieldExtension, PI>, + commit_ood: &[Vec>], + transcript: &mut (impl IsStarkTranscript + Clone), + expected_bus_balance: &FieldElement, + ) -> bool + where + Field: 'p, + FieldExtension: 'p, + PI: 'p, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + Self::multi_verify_views_with( + airs, + proofs, + Some(commit_ood), + transcript, + expected_bus_balance, + ) + } + + /// The commitment-time answers' step of Phase A, after every root: the + /// counts checked against the AIRs, `z0` drawn, every answer absorbed. + /// Returns `z0`, or `None` (reject) on a count or a coset offset that + /// disagrees. One function for the verifier and every replay of its Phase A. + fn absorb_commit_ood<'p>( + airs: &[&dyn AIR], + proofs: &impl ProofViewSource<'p, Field, FieldExtension, PI>, + commit_ood: &[Vec>], + transcript: &mut impl IsStarkTranscript, + ) -> Option> + where + Field: 'p, + FieldExtension: 'p, + PI: 'p, + { + if commit_ood.len() != airs.len() || proofs.view_len() != airs.len() { + return None; + } + let mut shapes = Vec::with_capacity(airs.len()); + for ((air, proof), answers) in airs.iter().zip(proofs.view_iter()).zip(commit_ood) { + if answers.len() != crate::commit_ood::answered_columns(*air).len() { + error!( + "{} commitment-time answers for {} answered columns", + answers.len(), + crate::commit_ood::answered_columns(*air).len() + ); + return None; + } + shapes.push(( + proof.trace_length(), + air.options().blowup_factor as usize, + air.options().coset_offset, + )); + } + let domain = crate::commit_ood::point_domain(shapes)?; + let z0 = crate::commit_ood::draw_point::(transcript, domain); + crate::commit_ood::absorb_answers::(transcript, commit_ood); + Some(z0) + } + + /// Phase A of a bound proof replayed for its shared LogUp challenges: + /// every table's preprocessed root (the AIR's, at its leaf layout) and main + /// root, then [`Self::absorb_commit_ood`], then the challenges. `None` = + /// reject (a layout with no preprocessed root, a count that disagrees). + fn replay_phase_a_bound<'p>( + airs: &[&dyn AIR], + proofs: impl ProofViewSource<'p, Field, FieldExtension, PI>, + commit_ood: &[Vec>], + transcript: &mut impl IsStarkTranscript, + ) -> Option>> + where + Field: 'p, + FieldExtension: 'p, + PI: 'p, + { + if airs.len() != proofs.view_len() { + return None; + } + for (air, proof) in airs.iter().zip(proofs.view_iter()) { + if air.is_preprocessed() { + let layout = Self::leaf_layout(*air, proof.trace_length()); + transcript.append_bytes(&air.precomputed_commitment_for(layout)?); + } + transcript.append_bytes(proof.lde_trace_main_merkle_root()); + } + Self::absorb_commit_ood(airs, &proofs, commit_ood, transcript)?; + Some( + (0..LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect(), + ) + } + + /// The one implementation of [`Self::multi_verify_views`] (`commit_ood` + /// `None`) and [`Self::multi_verify_views_bound`]. + #[doc(hidden)] + fn multi_verify_views_with<'p>( + airs: &[&dyn AIR], + proofs: impl ProofViewSource<'p, Field, FieldExtension, PI>, + commit_ood: Option<&[Vec>]>, + transcript: &mut (impl IsStarkTranscript + Clone), + expected_bus_balance: &FieldElement, + ) -> bool where Field: 'p, FieldExtension: 'p, @@ -1274,7 +1921,16 @@ pub trait IsStarkVerifier< if air.is_preprocessed() { // Preprocessed table: VERIFY precomputed commitment matches hardcoded. // This is the critical soundness check - ensures prover used correct precomputed values. - let expected_precomputed = air.precomputed_commitment(); + // The root of THIS table's leaf layout (a verifier constant); + // a layout the AIR has no root for rejects (never a recompute). + let layout = Self::leaf_layout(*air, trace_length); + let Some(expected_precomputed) = air.precomputed_commitment_for(layout) else { + error!( + "Preprocessed table {idx}: no precomputed commitment for the {layout:?} \ + leaf layout" + ); + return false; + }; match proof.lde_trace_precomputed_merkle_root() { Some(actual) if *actual == expected_precomputed => { // OK - commitment matches hardcoded @@ -1303,6 +1959,17 @@ pub trait IsStarkVerifier< } } + // The commitment-time answers (bound proofs): after every root, before + // the shared challenges. + let z0 = match commit_ood { + Some(answers) => match Self::absorb_commit_ood(airs, &proofs, answers, transcript) { + Some(z0) => Some(z0), + None => return false, + }, + None => None, + }; + let deep_term = crate::commit_ood::deep_term_on(); + // ===================================================================== // Round 1, Phase B: Sample shared LogUp challenges // ===================================================================== @@ -1373,12 +2040,22 @@ pub trait IsStarkVerifier< }; // Rounds 2-4: verify + let table_ood = match (&z0, commit_ood) { + (Some(point), Some(answers)) if deep_term => { + Some(crate::commit_ood::CommitOodTable { + point, + answers: &answers[idx], + }) + } + _ => None, + }; if !Self::verify_rounds_2_to_4( *air, proof, &public_inputs, &mut table_transcript, lookup_challenges.clone(), + table_ood, ) { error!( "Table {} failed verify_rounds_2_to_4 (num_constraints={}, trace_cols={})", @@ -1455,6 +2132,38 @@ pub trait IsStarkVerifier< rap_challenges: Vec>, layout: &crate::ood::OodLayout, ) -> Challenges + where + FieldElement: AsBytes, + FieldElement: AsBytes, + { + Self::replay_rounds_after_round_1_bound( + air, + proof, + public_inputs, + domain, + transcript, + rap_challenges, + layout, + None, + ) + } + + /// [`Self::replay_rounds_after_round_1`] for a table of a bound proof: the + /// same draws, and the DEEP coefficients of its answers (minus each + /// answered column's current-row one, `crate::commit_ood`) in + /// `Challenges::commit_ood`. The answer count is + /// the caller's to have checked. + #[allow(clippy::too_many_arguments)] + fn replay_rounds_after_round_1_bound( + air: &dyn AIR, + proof: StarkProofView<'_, Field, FieldExtension, PI>, + public_inputs: &PI, + domain: &VerifierDomain, + transcript: &mut impl IsStarkTranscript, + rap_challenges: Vec>, + layout: &crate::ood::OodLayout, + commit_ood: Option>, + ) -> Challenges where FieldElement: AsBytes, FieldElement: AsBytes, @@ -1548,24 +2257,57 @@ pub trait IsStarkVerifier< // <<<< Receive challenges: 𝛾ⱼ, 𝛾ⱼ' let gammas = deep_composition_coefficients; - // FRI commit phase + // Each answered column's coefficient at z0 is minus its current-row + // one: the column's two quotients form its one two-point quotient + // (`crate::commit_ood`). + let commit_ood = commit_ood.map(|ood| { + let columns = crate::commit_ood::answered_columns(air); + let coeffs: Vec<_> = columns + .clone() + .map(|c| crate::commit_ood::answer_coefficient(&trace_term_coeffs[c][0])) + .collect(); + let answers_sum = coeffs + .iter() + .zip(ood.answers) + .fold(FieldElement::::zero(), |acc, (c, v)| { + acc + c * v + }); + CommitOodDeep { + point: ood.point.clone(), + first_col: columns.start, + coeffs, + answers_sum, + } + }); + + // FRI commit phase. Under one-row openings (S2) the first root is the + // input tree (the DEEP codeword itself), absorbed BEFORE any folding + // challenge; every other root follows its challenge as today. + let leaf_layout = Self::leaf_layout(air, trace_length); let merkle_roots = proof.fri_layers_merkle_roots(); - let mut zetas = merkle_roots - .iter() - .map(|root| { + let mut zetas = Vec::with_capacity(merkle_roots.len() + 1); + for (j, root) in merkle_roots.iter().enumerate() { + if !(leaf_layout.is_one_row() && j == 0) { // >>>> Send challenge 𝜁ₖ - let element = transcript.sample_field_element(); - // <<<< Receive commitment: [pₖ] (the first one is [p₀]) - transcript.append_bytes(root); - element - }) - .collect::>>(); + zetas.push(transcript.sample_field_element()); + } + // <<<< Receive commitment: [pₖ] (the first one is [p₀]) + transcript.append_bytes(root); + } // The prover only samples the final-fold challenge when the codeword // actually folds past the committed layers. For tiny traces (the clamp // case) no fold happens, so no challenge is drawn. This must mirror the // prover's `commit_phase_from_evaluations` exactly. - let total_folds = Self::fri_termination_params(air, domain).total_folds; + // `total_folds` does not depend on the format (only its split into + // committed layers does), so the replay reads it from today's layout; + // a format the verifier cannot lay out is rejected in step 3. + let total_folds = crate::fri::terminal::FriFoldLayout::new( + domain.lde_length.trailing_zeros(), + (domain.lde_length / domain.trace_length).trailing_zeros(), + u32::from(air.options().fri_final_poly_log_degree), + ) + .total_folds; // >>>> Send final-fold challenge 𝜁_final (only when folding occurs) if total_folds > 0 { @@ -1591,7 +2333,7 @@ pub trait IsStarkVerifier< // FRI query phase // <<<< Send challenges 𝜄ₛ (iota_s) let number_of_queries = air.options().fri_number_of_queries; - let iotas = Self::sample_query_indexes(number_of_queries, domain, transcript); + let iotas = Self::sample_query_indexes(number_of_queries, domain, leaf_layout, transcript); Challenges { z, @@ -1603,6 +2345,7 @@ pub trait IsStarkVerifier< iotas, rap_challenges, grinding_seed, + commit_ood, } } @@ -1613,6 +2356,7 @@ pub trait IsStarkVerifier< public_inputs: &PI, transcript: &mut impl IsStarkTranscript, rap_challenges: Vec>, + commit_ood: Option>, ) -> bool where FieldElement: AsBytes + Sync + Send, @@ -1640,6 +2384,17 @@ pub trait IsStarkVerifier< return false; } + // The per-tree Merkle checks, built once per tree and only now: after + // the two count guards above, so a capped tree's owner opening (query + // 0) is known to exist before it is read. A capped + // tree's cap is authenticated against its root here; at the default + // format this reads no opening at all. + let Some(tree_checks) = Self::table_tree_checks(air, proof, &domain) else { + #[cfg(not(feature = "test_fiat_shamir"))] + error!("Merkle cap or path shape does not match the proof format"); + return false; + }; + // The pruned-OOD layout, read from the AIR once and shared by the round-4 // challenge replay, the block-shape guard, the single grid reconstruction, // and both verify steps below — one reconstruction instead of the previous @@ -1651,7 +2406,7 @@ pub trait IsStarkVerifier< #[cfg(feature = "instruments")] let timer1 = Instant::now(); - let challenges = Self::replay_rounds_after_round_1( + let challenges = Self::replay_rounds_after_round_1_bound( air, proof, public_inputs, @@ -1659,13 +2414,38 @@ pub trait IsStarkVerifier< transcript, rap_challenges, &layout, + commit_ood, ); + #[cfg(any(test, feature = "test-utils"))] + crate::fri::capture::record_challenges(&challenges.zetas, &challenges.iotas); + + // A bound proof's answers are checked by their quotients at `z0`; at a + // point where DEEP already has a quotient (`z·gʳ`, `z^P`) a false + // answer could cancel, so such a draw is refused. + if let Some(ood) = &challenges.commit_ood { + let z0 = crate::commit_ood::z0_for_point_check(&ood.point, &challenges.z); + if crate::commit_ood::meets_deep_point( + z0, + &challenges.z, + &domain.trace_primitive_root, + layout.num_eval_points(), + proof.composition_poly_parts_ood_evaluation().len(), + ) { + #[cfg(not(feature = "test_fiat_shamir"))] + error!("The commitment-time point meets a DEEP point"); + return false; + } + } // verify grinding let grinding_factor = air.context().proof_options.grinding_factor; if grinding_factor > 0 { let nonce_is_valid = proof.nonce().is_some_and(|nonce_value| { - grinding::is_valid_nonce(&challenges.grinding_seed, nonce_value, grinding_factor) + grinding::is_valid_nonce::>( + &challenges.grinding_seed, + nonce_value, + grinding_factor, + ) }); if !nonce_is_valid { @@ -1710,7 +2490,10 @@ pub trait IsStarkVerifier< if !Self::step_2_verify_claimed_composition_polynomial( air, - proof, + proof.trace_length(), + proof.bus_table_contribution(), + proof.trace_ood_evaluations().get_row(0), + proof.composition_poly_parts_ood_evaluation(), public_inputs, &domain, &challenges, @@ -1739,6 +2522,7 @@ pub trait IsStarkVerifier< &ood_full, layout.next_row_cols(), layout.step_size(), + &tree_checks, ) { #[cfg(not(feature = "test_fiat_shamir"))] error!("FRI verification failed"); @@ -1756,7 +2540,7 @@ pub trait IsStarkVerifier< let timer4 = Instant::now(); #[allow(clippy::let_and_return)] - if !Self::step_4_verify_trace_and_composition_openings(proof, &challenges) { + if !Self::step_4_verify_trace_and_composition_openings(proof, &challenges, &tree_checks) { #[cfg(not(feature = "test_fiat_shamir"))] error!("DEEP Composition Polynomial verification failed"); return false; diff --git a/crypto/stark/tests/gpu_constraint_interp.rs b/crypto/stark/tests/gpu_constraint_interp.rs index eef21953a..c5ad6888a 100644 --- a/crypto/stark/tests/gpu_constraint_interp.rs +++ b/crypto/stark/tests/gpu_constraint_interp.rs @@ -483,6 +483,7 @@ fn check_composition(prog: &ConstraintProgram, label: &str, seed: u64) b_value: &b_value, b_beta: &b_beta, b_z_inv: &b_z_inv, + row_stride: 1, }; let gpu = match try_eval_composition_gpu( prog, &main, &aux, &rap, &alpha, &offset, NEXT_STEP, NUM_ROWS, &inputs, false, @@ -568,6 +569,63 @@ fn check_composition(prog: &ConstraintProgram, label: &str, seed: u64) dev_raw, gpu, "[{label}] evaluate_dev (keep=true) H != host-drained H, seed {seed:#x}" ); + + // Sub-coset evaluation: at stride 2 the kernel must produce exactly the + // full evaluation's even rows — frame, zerofier and boundary reads all at + // the LDE row — in both the host-drained and the device-resident mode. + const STRIDE: usize = 2; + let strided_inputs = CompositionInputs { + row_stride: STRIDE, + ..inputs + }; + let strided = match try_eval_composition_gpu( + prog, + &main, + &aux, + &rap, + &alpha, + &offset, + NEXT_STEP, + NUM_ROWS, + &strided_inputs, + false, + ) { + Some(stark::constraint_ir::gpu_interp::GpuComposition::Host(raw)) => raw, + _ => panic!("[{label}] strided GPU composition path must engage (host mode)"), + }; + let even_rows: Vec = (0..NUM_ROWS / STRIDE) + .flat_map(|j| gpu[j * STRIDE * 3..j * STRIDE * 3 + 3].iter().copied()) + .collect(); + assert_eq!( + strided, even_rows, + "[{label}] stride-{STRIDE} H != the full H's even rows, seed {seed:#x}" + ); + let strided_dev = match try_eval_composition_gpu( + prog, + &main, + &aux, + &rap, + &alpha, + &offset, + NEXT_STEP, + NUM_ROWS, + &strided_inputs, + true, + ) { + Some(stark::constraint_ir::gpu_interp::GpuComposition::Dev(h)) => h, + _ => panic!("[{label}] strided GPU composition Dev (keep) path must engage"), + }; + assert_eq!( + strided_dev.num_rows, + NUM_ROWS / STRIDE, + "[{label}] strided H rows" + ); + let strided_dev_raw = math_cuda::constraint_interp::download_comp_h(&strided_dev) + .unwrap_or_else(|e| panic!("[{label}] download_comp_h (strided) failed: {e:?}")); + assert_eq!( + strided_dev_raw, even_rows, + "[{label}] strided device-resident H != the full H's even rows, seed {seed:#x}" + ); } #[test] diff --git a/crypto/stark/tests/r4_denoms_parity.rs b/crypto/stark/tests/r4_denoms_parity.rs index ad8284103..c16991691 100644 --- a/crypto/stark/tests/r4_denoms_parity.rs +++ b/crypto/stark/tests/r4_denoms_parity.rs @@ -1,7 +1,9 @@ //! R4 DEEP inverse-denominator parity: GPU `compute_and_invert_denoms_ext3_dev` -//! (with `DenomSign::XMinusZ`, the convention used by the prover's R4 DEEP -//! fast path) must match the CPU helper `build_r4_inv_denoms_cpu` that the -//! prover's CPU fallback also calls into. +//! (with `DenomSign::XMinusZ`, the convention of the prover's buffered R4 DEEP +//! path) must match the CPU helper `build_r4_inv_denoms_cpu` that the prover's +//! CPU fallback also calls into. By default the call takes the row-wise kernel, +//! under `LAMBDA_VM_DEEP_INV_LEGACY=1` the scan (`math_cuda::deep_inv`); run +//! it under both. //! //! Pins the three-copy fragility flagged in PR review: kernel construction, //! CPU fallback in prover.rs, and any test references must all be the same. diff --git a/crypto/stark/tests/vectors/zf_fri/README.md b/crypto/stark/tests/vectors/zf_fri/README.md new file mode 100644 index 000000000..10539a060 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/README.md @@ -0,0 +1,143 @@ +# S3 and S2 FRI vectors (group-leaf FRI layers, one-row openings) + +Test vectors for the S3 proof-format lever (`LAMBDA_VM_ZF_FRI=dp`, +`ProofFormat.fri_mode = FriMode::Dp`): committed FRI layer `j` folds by +`2^{d_j}` and commits groups of `2^{d_j}` consecutive values per leaf. They are +the oracle for the device prover (group-leaf commits, multi-fold kernels, query +gathers) and the in-guest verifier (group folds, group-leaf walks, slot checks). + +Every file is generated by `crypto/stark/src/fri/vectors.rs` and checked by a +test that regenerates it in memory and requires it byte-equal to this copy: + +| files | test (fails if stale) | regenerate | +|---|---|---| +| `a_*`, `b_*`, `c_*_keccak`, `c_*_blake3`, `d_*_keccak_*`, `d_*_blake3_*`, `e_*_keccak*`, `e_*_blake3*` | `cargo test -p stark --lib zf_fri_vectors::vectors_are_current` | `cargo test -p stark --lib zf_fri_vectors::write_vectors -- --ignored` | +| `c_*_rpx`, `d_*_rpx_*`, `e_*_rpx*` | `cargo test -p lambda-vm-prover --lib tests::zf_rpx_vectors::rpx_vectors_are_current` | `cargo test -p lambda-vm-prover --lib tests::zf_rpx_vectors::write_vectors -- --ignored` | + +Regenerate only for a deliberate format change (the schedule DP, its weights, +the fold, the leaf encoding): a stale file means the format moved. + +## Conventions + +- Field: Goldilocks `p = 2^64 − 2^32 + 1`. An extension element is `[c0, c1, c2]` + (canonical `u64` limbs) of `Degree3GoldilocksExtensionField`. +- A layer of length `n = 2^b` on the coset `o·⟨ω_n⟩` is stored in + **bit-reversed order**: position `p` holds `f(o·ω_n^{br_b(p)})`, where + `ω_n = F::get_primitive_root_of_unity(b)` (the LDE domain's root). +- Binary fold (unchanged, `fri_functions::fold_evaluations_in_place`): for the + pair at positions `(2j, 2j+1)` = points `(x_j, −x_j)`, + `out[j] = (lo + hi) + x_j⁻¹·ζ·(lo − hi)` (no ½: the terminal polynomial + absorbs the `2^{total_folds}`). Then the coset squares: offset `o → o²`. +- **Group fold** of exponent `d` with challenge `ζ`: `d` binary folds with + `ζ, ζ², ζ⁴, …, ζ^{2^{d−1}}`. It equals `2^d·Σ_{i<2^d} ζ^i f_i(Y)` for + `f(X) = Σ X^i f_i(X^{2^d})`. +- **Group leaf** `g` of a layer with exponent `d` = the values at positions + `g·2^d .. (g+1)·2^d` (the fiber `x_g·⟨ω_{2^d}⟩`, bit-reversed), hashed as ONE + leaf with the configuration's `Batched` leaf backend over those `2^d` ext + values in position order (`hash_data_from_slices(group, [])`). Parents use + the configuration's parent hash (as every STARK tree). At `d = 1` this is + exactly today's pair leaf (the `StarkHash` two-element invariant). +- A query with pair index `ι` (sampled as today, `sample_u64(lde/2)`) sits at + position `p_1 = ι` of committed layer 1 (fold 0 is the uncommitted binary + fold of the DEEP pair `p₀(υ), p₀(−υ)`, `υ` = LDE point at position `2ι`). + At committed layer `j`: `leaf = p_j >> d_j`, `slot = p_j & (2^{d_j} − 1)`, + `p_{j+1} = p_j >> d_j`. The terminal position is `ι >> Σ d_j`. +- Checks per layer: the group hashes to the leaf and authenticates at `leaf` + with a path of exactly `layer_log_len − d_j` siblings; `group[slot] == v` + (the value the previous fold produced); `v ← group fold with ζ_{j+1}`, + using `x_g⁻¹ = y⁻¹·ω_{2^d}^{br_d(slot)}` (`y` = the query's point at this + layer). Finally `terminal[p] == v`. +- Transcript (unchanged in form): `γ` → per committed layer: sample `ζ_j`, + append `root_{j+1}` → sample the final `ζ` (if anything folds) → terminal + coefficients → grinding nonce → `ι`s. `zetas` has `layers + 1` entries. +- Proof encoding: under `dp` the flat `layers_evaluations_sym` of each query + carries every layer's FULL group (`Σ 2^{d_j}` values, the query's own value + included); under `pair` (today) one sibling per layer. + +## Files + +**(a) `a_schedules.json`** — the fold-schedule DP (`fri::schedule::fri_schedule`) +at terminal logs `T ∈ {4, 9, 10}`, queries `Q ∈ {3, 110}`, cap `off`/`auto`, +LDE log `B = 6..24`, chains `s3` (from `b0 = B − 1`, row-pair openings) and +`s2` (from `b0 = B`, for S2 later). `cost_q_ns` is `Q ×` the per-query +cost-law price (`weights_ns` in the file header). Production: +base legs `T = 9`, LFM proofs `T = 10`, `Q = 110`. + +**(b) `b_group_folds.json`** — the KAT codeword: `2^7` ext values on the coset +`3·⟨ω_128⟩` (bit-reversed). Value `i` = three consecutive SplitMix64 outputs +(each reduced mod p) from state `0x5a4646524933` (`fri::vectors::splitmix64`: +`s += 0x9e3779b97f4a7c15; z = s; z = (z ^ z>>30)·0xbf58476d1ce4e5b9; +z = (z ^ z>>27)·0x94d049bb133111eb; out = z ^ z>>31`). For `d = 1..6`: +`zeta` (SplitMix64 from state `0x5a4646524933 + d`) and `folded`, the codeword +after `d` folds (length `2^{7−d}`, bit-reversed on the coset `3^{2^d}·⟨ω⟩`). +The generator asserts the verifier's group fold of every group reproduces it. + +**(c) `c_leaf_digests_{keccak,blake3,rpx}.json`** — for `d = 1..6`: the leaf +digest of the KAT codeword's first group (values `0 .. 2^d`), and the root of +the whole KAT codeword committed as a group-leaf layer tree (`2^{7−d}` leaves). +Digests are the 32-byte node encoding, hex. + +**(d) `d_proof_{keccak,blake3,rpx}_{pair,dp,dp_3_1_3,cap_pair,cap_dp}.{json,rkyv}`** — one small +proof per format: `LogReadOnlyRAP` (one aux column), +`2^10` rows of reads `(i % 5 + 1, 10·(i % 5 + 1))`, blowup 4 (so `B = 12`), +`fri_final_poly_log_degree = 2` (`T = 4`), 3 queries (20 for the `cap_*` +formats), grinding 0, coset +offset 3, proved with `DefaultTranscript::new(&[])` by `GenericProver<…, H>`. +`.rkyv` is the proof's rkyv bytes (`StarkProof`, the wire format of record). +The JSON has the layout (`schedule`, `legacy_encoding`, `total_folds`, +`terminal_len`), the FRI `fri_roots`, all `zetas` and `terminal_coeffs`, and +per query `iota`, the DEEP pair (`deep` = p₀(υ), `deep_sym` = p₀(−υ)), +`terminal_position`, and per layer `position`, `leaf`, `slot`, the opened +`values` (the full group under dp; the single sibling under pair) and the +authentication `path_len`. Formats: `pair` (today, all-ones schedule), +`dp` (the DP's schedule at `Q = 3`, cap off: `[3, 2, 2]`), `dp_3_1_3` (an +explicit uneven schedule via the test hook `fri_schedule_override`: unequal +neighbouring exponents are what catch a fold-count off-by-one), and the +Merkle-cap pair: `cap_pair` (`LAMBDA_VM_ZF_CAP=auto`, today's +FRI) and `cap_dp` (`auto` cap and the DP's schedule), at `Q = 20` so that +`auto` caps every tree at height 3. Their JSON adds `merkle_cap`, +`trace_tree_depth`, `trace_cap`, `fri_tree_depths` and `fri_caps` (the +verifier's `StarkCaps`); each capped tree's `2^c` cap nodes ride at the end of +query 0's authentication path (the owner path, `D − c + 2^c` nodes; every +other query carries `D − c`). The cap changes no transcript value: `cap_dp`'s +roots and `zetas` equal `dp`'s. + +**(e) S2 — one-row openings with a committed FRI input.** + +`e_leaf_digests_{keccak,blake3,rpx}.json` — one-row trace-tree leaves: a KAT +base matrix (16 rows × 5 columns, SplitMix64 from `KAT_SEED + 100`, one output +per value, reduced mod p) and an ext3 matrix (16 rows × 2 columns, from +`KAT_SEED + 200`, three outputs per value), each read as bit-reversed LDE +columns and committed at `rows_per_leaf = 1` (leaf `i` = the row at +bit-reversed position `i`) and, for comparison, at `rows_per_leaf = 2` (today: +rows `2i`, `2i + 1`). Every leaf digest and both roots per layout. A leaf +hashes the row's values column by column (`leaves_bit_reversed_grouped`, the +same stream the verifier's `hash_data_from_slices(evaluations, [])` hashes). + +`e_proof_{keccak,blake3,rpx}_{one_row_pair,one_row_3_2_1_2}.{json,rkyv}` — the +(d) proof shape (same AIR, trace, blowup 4, `B = 12`, `T = 4`, `Q = 3`, +grinding 0) proved with `ProofFormat.one_row = On`: +- every trace, aux and composition tree commits ONE row per leaf and is + `B = 12` deep (`trace_tree_depth`); a query index `r` is uniform over the + whole LDE (`query_bound = 4096`, not 2048) and opens leaf `r` of every trace + tree (`trace_leaf`, `trace_path_len`); openings carry no symmetric row; +- `deep` is DEEP at the ONE point `x_r` = the LDE point at bit-reversed + position `r` (there is no `deep_sym`); +- FRI layer 0 is the INPUT tree: the DEEP codeword itself (`2^12` values, + bit-reversed), committed with groups of `2^{d_0}` values; its root is + `fri_roots[0]` and is absorbed BEFORE the first folding challenge. Transcript: + `γ` → append `root_0` → per later layer: sample `ζ`, append its root → sample + the final `ζ` → coefficients → nonce → `r`s. So `zetas` has one entry per + layer (layer `j` folds with `zetas[j]`), against `layers + 1` for row pairs; +- per layer `j`: `position = r >> Σ_{i> d_j`, + `slot = position & (2^{d_j} − 1)`; layer 0's slot check is the input-slot + check `group₀[slot] == deep`; the terminal position is `r >> Σ d_j`. +- Formats: `one_row_pair` (fri = pair: the all-ones schedule from `B`, eight + pair layers, group encoding), `one_row_3_2_1_2` (an explicit uneven schedule, + `Σ = 8 = B − T`). + +## Not here yet + +- A capped vector under one-row openings (the cap composes with one_row on + the host, `one_row_tests::cap_fri_one_row_matrix_round_trips`, but only the + row-pair `cap_pair` / `cap_dp` files are exported). diff --git a/crypto/stark/tests/vectors/zf_fri/a_schedules.json b/crypto/stark/tests/vectors/zf_fri/a_schedules.json new file mode 100644 index 000000000..38e9fdf84 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/a_schedules.json @@ -0,0 +1,463 @@ +{ + "generator": "stark::fri::vectors::schedules_json", + "weights_ns": {"compress": 2251, "select": 567, "unpack": 528, "hint": 460, "compare": 3789, "fold": 2610, "twiddle": 477, "xalu": 522, "balu": 477}, + "dmax": 6, + "rows": [ + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [1], "cost_q_ns": 85443}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [2], "cost_q_ns": 124764}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [2], "cost_q_ns": 124764}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [3], "cost_q_ns": 192378}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [3], "cost_q_ns": 192378}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [2, 2], "cost_q_ns": 269196}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [2, 2], "cost_q_ns": 269196}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [3, 2], "cost_q_ns": 336810}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [3, 2], "cost_q_ns": 336810}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [3, 3], "cost_q_ns": 414258}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [3, 3], "cost_q_ns": 414258}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 500910}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 500910}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 578358}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 578358}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 665640}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 665640}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 762126}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 762126}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 849408}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 849408}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 946524}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 946524}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 1052844}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 1052844}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1149960}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1149960}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 1256910}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 1256910}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 1373064}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 1373064}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 1480014}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 1480014}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 1596798}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 1596798}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 1722786}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 1722786}, + {"terminal_log": 4, "queries": 3, "cap": "off", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3, 3, 2], "cost_q_ns": 1839570}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [1], "cost_q_ns": 85443}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [2], "cost_q_ns": 124764}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [2], "cost_q_ns": 124764}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [3], "cost_q_ns": 192378}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [3], "cost_q_ns": 192378}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [2, 2], "cost_q_ns": 269196}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [2, 2], "cost_q_ns": 269196}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [3, 2], "cost_q_ns": 336810}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [3, 2], "cost_q_ns": 336810}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [3, 3], "cost_q_ns": 414258}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [3, 3], "cost_q_ns": 414258}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 500910}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 500910}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 578358}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 578358}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 665640}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 665640}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 762126}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 762126}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 849408}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 849408}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 946524}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 946524}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 1052844}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 1052844}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1149960}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1149960}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 1256910}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 1256910}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 1373064}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 1373064}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 1480014}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 1480014}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 1596798}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 1596798}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 1722786}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 1722786}, + {"terminal_log": 4, "queries": 3, "cap": "auto", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3, 3, 2], "cost_q_ns": 1839570}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [1], "cost_q_ns": 3132910}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [2], "cost_q_ns": 4574680}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [2], "cost_q_ns": 4574680}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [3], "cost_q_ns": 7053860}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [3], "cost_q_ns": 7053860}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [2, 2], "cost_q_ns": 9870520}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [2, 2], "cost_q_ns": 9870520}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [3, 2], "cost_q_ns": 12349700}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [3, 2], "cost_q_ns": 12349700}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [3, 3], "cost_q_ns": 15189460}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [3, 3], "cost_q_ns": 15189460}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 18366700}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 18366700}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 21206460}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 21206460}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 24406800}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 24406800}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 27944620}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 27944620}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 31144960}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 31144960}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 34705880}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 34705880}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 38604280}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 38604280}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 42165200}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 42165200}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 46086700}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 46086700}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 50345680}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 50345680}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 54267180}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 54267180}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 58549260}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 58549260}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 63168820}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 63168820}, + {"terminal_log": 4, "queries": 110, "cap": "off", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3, 3, 2], "cost_q_ns": 67450900}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [1], "cost_q_ns": 2569066}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [2], "cost_q_ns": 4010836}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [2], "cost_q_ns": 4010836}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [3], "cost_q_ns": 6490016}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [3], "cost_q_ns": 6490016}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [2, 2], "cost_q_ns": 8742832}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [2, 2], "cost_q_ns": 8742832}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [3, 2], "cost_q_ns": 11222012}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [3, 2], "cost_q_ns": 11222012}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [3, 3], "cost_q_ns": 14061772}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [3, 3], "cost_q_ns": 14061772}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 16675168}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [3, 2, 2], "cost_q_ns": 16675168}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 19514928}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3, 3, 2], "cost_q_ns": 19514928}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 22715268}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3, 3, 3], "cost_q_ns": 22715268}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 25689244}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 3, 2, 2], "cost_q_ns": 25689244}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 28889584}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3, 3, 2], "cost_q_ns": 28889584}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 32450504}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3, 3, 3], "cost_q_ns": 32450504}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 35785060}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 3, 2, 2], "cost_q_ns": 35785060}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 39345980}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 39345980}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 43267480}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 43267480}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 46962616}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 3, 2, 2], "cost_q_ns": 46962616}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 50884116}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3, 3, 2], "cost_q_ns": 50884116}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 55166196}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3, 3, 3], "cost_q_ns": 55166196}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 59221912}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 3, 2, 2], "cost_q_ns": 59221912}, + {"terminal_log": 4, "queries": 110, "cap": "auto", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3, 3, 2], "cost_q_ns": 63503992}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [1], "cost_q_ns": 134613}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [1], "cost_q_ns": 134613}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [2], "cost_q_ns": 173934}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [2], "cost_q_ns": 173934}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3], "cost_q_ns": 241548}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3], "cost_q_ns": 241548}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [2, 2], "cost_q_ns": 367536}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [2, 2], "cost_q_ns": 367536}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 2], "cost_q_ns": 435150}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 2], "cost_q_ns": 435150}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3], "cost_q_ns": 512598}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3], "cost_q_ns": 512598}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 648420}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 648420}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 725868}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 725868}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 813150}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 813150}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [4, 3, 3], "cost_q_ns": 949002}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [4, 3, 3], "cost_q_ns": 949002}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 1046088}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 1046088}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 1143204}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 1143204}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 1279056}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 1279056}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1395810}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1395810}, + {"terminal_log": 9, "queries": 3, "cap": "off", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 1502760}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [1], "cost_q_ns": 134613}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [1], "cost_q_ns": 134613}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [2], "cost_q_ns": 173934}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [2], "cost_q_ns": 173934}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3], "cost_q_ns": 241548}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3], "cost_q_ns": 241548}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [2, 2], "cost_q_ns": 367536}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [2, 2], "cost_q_ns": 367536}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 2], "cost_q_ns": 435150}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 2], "cost_q_ns": 435150}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3], "cost_q_ns": 512598}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3], "cost_q_ns": 512598}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 648420}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 648420}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 725868}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 725868}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 813150}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 813150}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [4, 3, 3], "cost_q_ns": 949002}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [4, 3, 3], "cost_q_ns": 949002}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 1046088}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 1046088}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 1143204}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 1143204}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 1279056}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 1279056}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1395810}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1395810}, + {"terminal_log": 9, "queries": 3, "cap": "auto", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 1502760}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [1], "cost_q_ns": 4935810}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [1], "cost_q_ns": 4935810}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [2], "cost_q_ns": 6377580}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [2], "cost_q_ns": 6377580}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3], "cost_q_ns": 8856760}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3], "cost_q_ns": 8856760}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [2, 2], "cost_q_ns": 13476320}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [2, 2], "cost_q_ns": 13476320}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 2], "cost_q_ns": 15955500}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 2], "cost_q_ns": 15955500}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3], "cost_q_ns": 18795260}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3], "cost_q_ns": 18795260}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 23775400}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 23775400}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 26615160}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 26615160}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 29815500}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 29815500}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [4, 3, 3], "cost_q_ns": 34796740}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [4, 3, 3], "cost_q_ns": 34796740}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 38356560}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 38356560}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 41917480}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 41917480}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 46898720}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 46898720}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 51179700}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 51179700}, + {"terminal_log": 9, "queries": 110, "cap": "off", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 55101200}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [1], "cost_q_ns": 4371966}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [1], "cost_q_ns": 4371966}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [2], "cost_q_ns": 5813736}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [2], "cost_q_ns": 5813736}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [3], "cost_q_ns": 8292916}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [3], "cost_q_ns": 8292916}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [2, 2], "cost_q_ns": 12348632}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [2, 2], "cost_q_ns": 12348632}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [3, 2], "cost_q_ns": 14827812}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [3, 2], "cost_q_ns": 14827812}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 3], "cost_q_ns": 17667572}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 3], "cost_q_ns": 17667572}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 22083868}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 2, 2], "cost_q_ns": 22083868}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 24923628}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 3, 2], "cost_q_ns": 24923628}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 28123968}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 3], "cost_q_ns": 28123968}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 2, 2], "cost_q_ns": 32900844}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 2, 2], "cost_q_ns": 32900844}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 36101184}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [3, 3, 3, 2], "cost_q_ns": 36101184}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 39662104}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 3], "cost_q_ns": 39662104}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 44643344}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [4, 3, 3, 3], "cost_q_ns": 44643344}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 48360480}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 48360480}, + {"terminal_log": 9, "queries": 110, "cap": "auto", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 3], "cost_q_ns": 52281980}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [1], "cost_q_ns": 144447}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [1], "cost_q_ns": 144447}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [2], "cost_q_ns": 183768}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [2], "cost_q_ns": 183768}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3], "cost_q_ns": 251382}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3], "cost_q_ns": 251382}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [2, 2], "cost_q_ns": 387204}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [2, 2], "cost_q_ns": 387204}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 2], "cost_q_ns": 454818}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 2], "cost_q_ns": 454818}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3], "cost_q_ns": 532266}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3], "cost_q_ns": 532266}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [4, 3], "cost_q_ns": 668118}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [4, 3], "cost_q_ns": 668118}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 755370}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 755370}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 842652}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 842652}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 978504}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 978504}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 1085424}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 1085424}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 1182540}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 1182540}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 1318392}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 1318392}, + {"terminal_log": 10, "queries": 3, "cap": "off", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1444980}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [1], "cost_q_ns": 144447}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [1], "cost_q_ns": 144447}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [2], "cost_q_ns": 183768}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [2], "cost_q_ns": 183768}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3], "cost_q_ns": 251382}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3], "cost_q_ns": 251382}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [2, 2], "cost_q_ns": 387204}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [2, 2], "cost_q_ns": 387204}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 2], "cost_q_ns": 454818}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 2], "cost_q_ns": 454818}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3], "cost_q_ns": 532266}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3], "cost_q_ns": 532266}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [4, 3], "cost_q_ns": 668118}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [4, 3], "cost_q_ns": 668118}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 755370}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 755370}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 842652}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 842652}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 978504}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 978504}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 1085424}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 1085424}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 1182540}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 1182540}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 1318392}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 1318392}, + {"terminal_log": 10, "queries": 3, "cap": "auto", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 1444980}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [1], "cost_q_ns": 5296390}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [1], "cost_q_ns": 5296390}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [2], "cost_q_ns": 6738160}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [2], "cost_q_ns": 6738160}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3], "cost_q_ns": 9217340}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3], "cost_q_ns": 9217340}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [2, 2], "cost_q_ns": 14197480}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [2, 2], "cost_q_ns": 14197480}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 2], "cost_q_ns": 16676660}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 2], "cost_q_ns": 16676660}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3], "cost_q_ns": 19516420}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3], "cost_q_ns": 19516420}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [4, 3], "cost_q_ns": 24497660}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [4, 3], "cost_q_ns": 24497660}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 27696900}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 27696900}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 30897240}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 30897240}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 35878480}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 35878480}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 39798880}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 39798880}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 43359800}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 43359800}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 48341040}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 48341040}, + {"terminal_log": 10, "queries": 110, "cap": "off", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 52982600}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 6, "chain": "s3", "b0": 5, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 6, "chain": "s2", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 7, "chain": "s3", "b0": 6, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 7, "chain": "s2", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 8, "chain": "s3", "b0": 7, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 8, "chain": "s2", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 9, "chain": "s3", "b0": 8, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 9, "chain": "s2", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 10, "chain": "s3", "b0": 9, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 10, "chain": "s2", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 11, "chain": "s3", "b0": 10, "schedule": [], "cost_q_ns": 0}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 11, "chain": "s2", "b0": 11, "schedule": [1], "cost_q_ns": 4732546}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 12, "chain": "s3", "b0": 11, "schedule": [1], "cost_q_ns": 4732546}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 12, "chain": "s2", "b0": 12, "schedule": [2], "cost_q_ns": 6174316}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 13, "chain": "s3", "b0": 12, "schedule": [2], "cost_q_ns": 6174316}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 13, "chain": "s2", "b0": 13, "schedule": [3], "cost_q_ns": 8653496}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 14, "chain": "s3", "b0": 13, "schedule": [3], "cost_q_ns": 8653496}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 14, "chain": "s2", "b0": 14, "schedule": [2, 2], "cost_q_ns": 13069792}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 15, "chain": "s3", "b0": 14, "schedule": [2, 2], "cost_q_ns": 13069792}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 15, "chain": "s2", "b0": 15, "schedule": [3, 2], "cost_q_ns": 15548972}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 16, "chain": "s3", "b0": 15, "schedule": [3, 2], "cost_q_ns": 15548972}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 16, "chain": "s2", "b0": 16, "schedule": [3, 3], "cost_q_ns": 18388732}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 17, "chain": "s3", "b0": 16, "schedule": [3, 3], "cost_q_ns": 18388732}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 17, "chain": "s2", "b0": 17, "schedule": [3, 2, 2], "cost_q_ns": 23165608}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 18, "chain": "s3", "b0": 17, "schedule": [3, 2, 2], "cost_q_ns": 23165608}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 18, "chain": "s2", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 26005368}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 19, "chain": "s3", "b0": 18, "schedule": [3, 3, 2], "cost_q_ns": 26005368}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 19, "chain": "s2", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 29205708}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 20, "chain": "s3", "b0": 19, "schedule": [3, 3, 3], "cost_q_ns": 29205708}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 20, "chain": "s2", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 34186948}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 21, "chain": "s3", "b0": 20, "schedule": [4, 3, 3], "cost_q_ns": 34186948}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 21, "chain": "s2", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 37543504}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 22, "chain": "s3", "b0": 21, "schedule": [3, 3, 3, 2], "cost_q_ns": 37543504}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 22, "chain": "s2", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 41104424}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 23, "chain": "s3", "b0": 22, "schedule": [3, 3, 3, 3], "cost_q_ns": 41104424}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 23, "chain": "s2", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 46085664}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 24, "chain": "s3", "b0": 23, "schedule": [4, 3, 3, 3], "cost_q_ns": 46085664}, + {"terminal_log": 10, "queries": 110, "cap": "auto", "lde_log": 24, "chain": "s2", "b0": 24, "schedule": [3, 3, 3, 3, 2], "cost_q_ns": 50163380} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/b_group_folds.json b/crypto/stark/tests/vectors/zf_fri/b_group_folds.json new file mode 100644 index 000000000..c596389b8 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/b_group_folds.json @@ -0,0 +1,14 @@ +{ + "generator": "stark::fri::vectors::group_fold_json", + "layer_log": 7, + "coset_offset": 3, + "codeword": [[114248373298330572,9950395923948974552,18292651019826168847],[11264014397209282797,17011101826315946882,7833410878953924418],[5067401236789812724,11181820620780326085,3913229743111987323],[6878663313357031824,4948368221433110479,4512378869763498949],[7300424359380325951,2629022478318430713,2352490085885205544],[3345914654228628712,17857800835092815061,4926885334099255041],[13961999760601160978,1108987075972034959,9069506549887126910],[3666212158788092907,15984312022772250051,13786698764199474274],[18233060593890754966,10696033068098814113,17116650682688893057],[1890999107425815346,13272904155828474695,15745756932851453243],[2002649893950066027,10673426297906541463,5573335164912747759],[14252812568771729760,8622463572814113680,16610631961562864340],[12657518525956593423,456049067459718558,6504017001053832],[7829250107108091587,10926092578370219313,15367852261670847390],[4283296287331528321,15615269286480435397,9412341781204946411],[17483792075638818644,17606390452909413141,16358299522213780360],[15538447972185760193,614856646744234365,14918316599210172173],[13828376987499133466,7148424706229729581,18158364113742349760],[13317215957353694181,9553347083671652872,5030147550646492824],[18336842288707666825,40487482472454760,5308286461165749931],[10834483197013613298,2298902883281893103,9942650691919595847],[11525892593642558393,9577750050426263731,2069904186241791718],[12406808437096063139,18073864409508715723,9230946790196975681],[3616632367654660274,10085665700693958697,16234982436400875270],[11917376668558775704,1901622792059622932,4968714500333547882],[4392135747981402671,5045545397172485329,15159213888871909626],[16061598631806973090,16901853911822821748,2267463094830397080],[4727640163684891461,9538126371028842686,9262479336654913939],[10304857519585917067,8918162236608948478,10808080315658145629],[13297830096390482799,4157291057334423461,15367988991022445219],[4469526150015381313,17786059701133525583,3006562354700561941],[9644214213943849393,657307829680962020,8280808660067407727],[6478239413655729580,18326407550008754007,11164300005736105706],[7471085016665548590,4020868342565639949,12446775137750043372],[11263844806142541136,8163614071841666446,504867900502982859],[2159692865036309446,5561796258936018645,4890282858963284249],[2258498590121909142,5188649223164923014,6006554213473999361],[16573169297305620351,7930659280834829778,3035332598973915968],[17592510683391522494,10588162808097688243,2750314556565551745],[1715278030362642140,17922857671664280100,17783735148214017483],[16877780672179297763,13713324251228354245,3672315061421186130],[11957769149749177020,7483809133384536686,13037449287285835030],[6060158857796037968,9236857535693839353,12225811918673374846],[6771836030305042448,5598588681544690427,736358855645767628],[14779764761098060676,9663318929005065160,7646712406485710697],[10352335144026281330,17034102847782780786,15015319655928055430],[1633346440070508358,18371146929211533278,13712502342528757365],[6697173780674409885,9465979684532956528,9986298627803261808],[1293703708762286999,15474728880582396103,11846671819987630109],[16286799938735909326,14556742360048391013,4621869127488820336],[3218948681922449986,221947035006398726,18253852596682421145],[5664709069538661127,4659564549683834986,9161648188075992511],[4114697847890739171,11027906093491171326,5104702896789353783],[5359187956347809939,4089103937352895916,2940850950576450587],[17399476639146650887,5016017632778311111,7271973075312826604],[8550063129321630329,5052407800723282653,9652683408301892866],[14047321464110593862,10155502225800659400,7644371686383087249],[2944575710255990900,8979321450340805175,1650833521640647570],[8476354023472637773,11675470599792868270,3889936884919286636],[4576427959904738712,13458256969793770315,13656500986899329602],[3314515815155456785,18041964041297850883,5747108416396853662],[16339498612485786868,4924956705534843858,18134066455875036650],[13147536215094553671,15997716004885695733,285745296538690591],[14255722797880109791,7210927213576476088,17587677203842504678],[11860626404695384951,8746280467222948941,2592292097704094105],[5717377429282337416,15899322474938656172,9164145311561944609],[8790455045054538216,8200372301439147858,4268466785537731197],[16115353753741389192,16439022717181043398,349125038085494325],[16242984037949079700,2000284139079405798,1102724556130734343],[17534186906134655633,12270638421670546342,2117549142856690160],[8558441615229845283,590680743110320488,846899180246940639],[8542947849373777247,8239462877891411699,13543070908409877225],[3806515466354321427,16677093996613821429,16921085645841743272],[4047188426270378454,10565945368133880359,17594120901052187483],[7520285678261358264,486417986208464051,211120145804645863],[2334355826641841630,10207668950452587969,9309038626488644891],[5508103946177442353,15433466943821065373,13938026508879038839],[10691340797952801443,13249035363727977704,16549098586767055074],[3189700400573926573,8505018310981494158,14989412664038149386],[1971071720876537511,8957196981073330551,3168195186127451837],[3370211075940519112,18221121112327787210,6388629410921300698],[13825718672353564576,2504067456039422864,14533168195148200715],[5098493520843201029,8869953106041465055,8122558809048000182],[18430351289255880067,8351103019978092501,11781980863743298682],[9373833362978281396,844921688433531872,18324413337107299243],[13485327368693733374,10691899904332665196,13229367546756152914],[1258912480864571676,2071340549663549730,6758277227815971576],[9549818200437251075,3405125360623851175,16942008793677491431],[13433015808893608304,9222727569003177390,14687583678862150856],[4894349561543165168,10011853121155194320,9907724893228480846],[12041518508022051244,14979367150570215901,3947232093288738331],[11640383997489081340,12156286191378664878,3386069373993561222],[14532281185964301339,17117192326887343454,12633381051772056545],[17985763565350791897,14532411921854015955,4139666522539931465],[10357060237166899055,10250685368132853801,4328663504089949328],[4264850046336035568,15127401253622843536,222690904877854589],[103201182986371004,7320118757475979880,984802483944950906],[4914621342385792092,4235556697395691596,2043652815146446309],[4532406341643755749,14046023185726959179,3407252364328999400],[15570319685166093974,11598076819033158060,10721222638522050524],[11649937100850667658,17866319465093320968,142299381593219063],[11567600299112655079,14716338034173449979,4226948256144437849],[17648997948271597061,4104103481584022969,9423885717708494854],[1800923050060833384,16555506371273926041,498679432809679599],[12532533693534476990,10541261484153048609,2029747857339337056],[17700837854248686544,8203001322382349030,15904685394162363036],[16467223518425325962,7471482565975648931,4501907999120368192],[13531965789846280913,3600794695377887032,7838235768256633532],[686248293220456248,4108049970362787107,6235191269358902257],[12274949027939456860,15170327482863637391,14551777247649584285],[4935455115743580892,17399757865092689306,12479638661441169770],[1141676659857718415,15295011150279022529,5183519654745430110],[14342582113481762098,9747082398264244127,7785589445197325825],[10731773423360016215,7909315320595870046,9263859799229246894],[16219555816209372043,14587867657169784888,13701890000248380413],[10735217865192115671,18281444572297642773,14564753480394576404],[4083176687777968338,9886723294778689857,2639615648938171605],[11804598711561790479,14108740349310438693,16344856194162855154],[12511529987086828495,15492173228579281126,7332520940589871870],[8790290147053500537,15807794493185089187,8793068148887118703],[3376003944858192249,13739302494003574116,62782776568845328],[16790760525790734333,16991944191494589086,14551119359498268396],[16553479249857101463,13158060559356436102,12798557079200239492],[9074427498021082773,2169770807462520334,3610202083964275218],[506760479544331739,11162253383698480455,836801623611651750],[5626003773148940461,13711907462553979908,3200744853949011781],[8991193990761346695,13002673796626326441,8147380006804403512],[15640945099500338290,13502092422384284689,2357524241674917606]], + "folds": [ + {"d": 1, "zeta": [5982986140143379172,8639098839961006196,9106571855708565181], "folded": [[13041342392770451398,1367539920642605247,12923439445535733123],[15111846465231705251,10869098118386980757,2110572119091924200],[1743889628524016675,9605940437084964891,5870648398677220037],[6785059710301850641,14432698623008569523,5117724888539462515],[6129637404252570793,15785473664722944987,15376383444291444540],[2830472080319199117,6916797001546272848,1139387193878398347],[854234618140711067,12865338332827840657,7683341066234270028],[8556763960483054921,14226068309370485405,8805068317203179907],[11652032324272791418,17067109057057488352,2155226401905577343],[7378977493906685189,7396070752922181160,14809630492126546402],[11221476835047076052,5552290674014561395,12839848385284537238],[17031632340357046723,18418693201929269666,13281360691649993105],[14947913625120789885,8869931497227681431,6516962090810790237],[4326895494534626222,210001341783279907,15836165149567730137],[4565156726448738349,1579099250129172856,11598873810743137497],[5081435166149576077,17735684136426917354,6036322550169833380],[3968018905876962370,9352856551674136594,256398433064191051],[5820988155798552494,16230330221979405516,5442468488737945834],[15049537498785237890,13862744868893397470,14510812512146738273],[2768819311689065150,11555171857964503456,9951892334960521639],[5497873220595924588,17485581362478882607,10739818946094945949],[12392623511920994647,10343193041454038320,11232463150930599175],[4625036141519292479,13478581817394022221,8433652285864766485],[8687067248403931348,14109209674648261023,8070097095869626139],[12511484557452774393,1615115631906611225,16808972079402084194],[12670112237463559315,16792075638721230118,4649929775577578039],[13963863236547405391,1737948292933472334,3462705389007925732],[13961623756411541908,17272945038285068896,13053563848099968430],[8542033251067556043,3943859879856553609,1365063928448331070],[2728051694180779145,10148083909313286564,3223039336597638137],[9969089259162820539,17201853852727321770,17280168509961302772],[3881414877058380274,16498435535752619897,15169721157991250091],[12282716117858167167,17479863209036482076,6900099660425552793],[5363085088357372880,12899192158122355397,13941244711242550925],[11954766328831456765,9114697862560243873,4708816652260351582],[4661049181802151049,4160451092099917117,15731571421871714693],[10394212062234425622,13543302894196970343,7268132449793459504],[14805027189806919077,6120844744139351106,8713141837178140203],[14364702613876332182,279691163524163244,3233426636605062982],[17638057925580708952,11880443875791088541,12663664333800944631],[2747387356761608808,6128024573264497456,1906647192604679604],[18142783074158655012,6930998950528696333,5966402208739676722],[5228353224625780153,11776267822025310956,7789010923833663159],[9198934511149137839,18297187364872279600,1100924458194065309],[15830556237832765901,15481187844668569608,5745954936691172836],[2463446672805614173,17654813086499958284,6627425475604431075],[9744974175376768951,6003907175064199467,3019863289756215214],[1105660295481585495,2846670207270484742,13893045392786235471],[8033640395713514621,2058287371987793338,16306955591035758260],[16718195256006779193,12845025146677090529,15507111131880041003],[17450061014428799367,8932849712635572444,1399273411695530237],[15314353830639146281,1617556098418450755,9680944244630765095],[7866028852159950947,5854581393547112260,1209047338584429762],[16498824236346505226,5525498508776325306,16356508675665315508],[2932115578077162425,14823813025467406358,8601995247003571190],[17627413824927498465,10363734279082421342,9882196638918242818],[8646412436728893485,4209536846449535861,5211710103494064584],[16883885278496934465,14398134661515437444,8320406202240514852],[1749674726038026642,10561302741404371186,1715480365650746116],[10098522374844915378,804272538400298735,11436350855199609294],[2242210078462043777,12856281862345183327,2934427225584611373],[8999664123015327012,3103813813738847199,6919160343947595269],[16973230894532108938,8182849463983648185,4979542969698307285],[1968658373866109019,4665310856168057167,8922308844881140926]]}, + {"d": 2, "zeta": [8277629894573820398,11987396219901710093,14286132552350819815], "folded": [[10659459074290418840,11404377731597685792,6412680166265662870],[18049568505121810974,10282830458869143188,10391546737972078780],[5253731277067850711,17621783193604886697,13132862192775322516],[11410018894194817656,16637267147743673871,15691689860434995800],[15506678364970685080,14861538350404551244,2837968110277602935],[2375804235161086170,8739888673911906050,4578176649759128810],[11442767291103846881,17342277446275794116,12662010138142028184],[14193068335114990813,13299033286367041493,8752101333196968920],[8620817047384791266,772804132056888022,3013625826044251668],[17723345225477209951,14603399632728913506,13406293657603104633],[4812604701894865410,3562163093929977386,15780981544278663169],[15273963609109427358,11988463685517847983,9902509355838126711],[4774731828063032978,17552776897472460989,2880706347154895482],[5738180859617894461,3267965271741550920,17196156652613304319],[6718748369866396894,13753766810220425783,14436843765103965982],[5096396145492219879,8332076396846918178,16515416299487658941],[12756397208828506185,14529843240352940945,1630979625933120288],[10236252437520398367,6121362081952229925,9675066952179694897],[12876998450208355897,5602539663494599642,16419361351446313034],[17633560925173793293,10491185305687269016,17609214207103544675],[9732322475272580838,2343921293067816168,11514857875151018473],[13301737884115171269,6768114510583834753,8437194746277735446],[7268976342321786428,6451174973850490262,17039742468036691953],[14197904117477439251,3256501541063215422,6078363490531782656],[18436670341339182065,4740965983665404202,13097850020662306658],[11643420520201444238,13894047914357275925,14847322032688363344],[6172254798990809302,9944297163286485901,16114164081446284441],[8126667779381531756,4335237177165305128,17759934364874626226],[2592542309881532259,3824543554027954290,7149789287312016189],[4722752886837845508,12270618432482714981,11136091373728017350],[15757453116814468536,4821251310523727862,11444859270510782763],[9887722687648353215,1752436496823926870,8599976724602573013]]}, + {"d": 3, "zeta": [1552802620964980016,1023686708859178672,17199325712939756230], "folded": [[14918904049632525799,12846102155294064811,12915689463213974410],[13752095238017906920,5561348862729856162,8163198169507984328],[11958998197578919994,1714262070066328704,13055202499057967831],[12579394395986170933,15453469291495055084,12511065676009529992],[5909815762699180536,325949289899580405,7602105144051417564],[8766981089440784624,12849391202710350407,10656754887032826357],[9043352336673692541,8082743422328089822,17620199666704121177],[1547345145325217649,17789026623191553517,13155450911809985342],[10770466821207224048,17638400349644439368,17680774930596894643],[3195520839440654360,358031439555466513,14635703796729568535],[13997904731947774862,2254971366004956008,4134927195987875200],[9792028107437876456,3202983791892939142,13683282596889965866],[5928029347378441967,7321247732850393678,12853557145597527353],[15655424873278218072,9543164650639120857,11400932202697208831],[1071396197694857509,11588664618481638829,6683877201390805789],[12171679035502060770,4545108039692003627,314780921239473147]]}, + {"d": 4, "zeta": [8615088116925916633,14390088107058038174,2021072648982338043], "folded": [[7052712868241756772,4553468042549522103,3196731637763074741],[16344025613025748553,452293191141481183,8204830158779577480],[18369870030235551668,5080227102498455366,5796020548436058409],[16981890415956580037,2793261484183937351,12700002093053910932],[9237817075302126092,16582740101697295386,3495992442642679478],[2649989405081990475,18422077614247848350,17579251581382283805],[11281358868900100483,6371375087829651837,15893368395499180654],[11309332093392525358,11076419593701924186,12957435685350473463]]}, + {"d": 5, "zeta": [12638890986933725064,6233988671304140119,10906738775467990202], "folded": [[14938993435336039587,8433402340287360947,1444885065781874699],[5846601836415060987,9442110707393776817,7424757494850568828],[6809934921619048346,230609055236823753,4480150124686349040],[313437947384391868,7391389585330885817,9866105847013411666]]}, + {"d": 6, "zeta": [16246432198723013017,7005472786384078398,3590918350879141987], "folded": [[11082178205324756851,2117653779716520051,16365930981660702508],[315019225942101281,11078153742178048043,5070729680691113956]]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_blake3.json b/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_blake3.json new file mode 100644 index 000000000..689daaff2 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_blake3.json @@ -0,0 +1,13 @@ +{ + "generator": "stark::fri::vectors::leaf_digests_json", + "hash": "blake3", + "codeword": "b_group_folds.json codeword", + "leaves": [ + {"d": 1, "first_leaf": "3e94099701015ba4b517b2da241e742be08d6247bc8863a07484aa134dd3c0b2", "layer_root": "ab6a1860afd95f056e99f36162ee961aec77ee8e14c4206b024c02f63a5a7f1e"}, + {"d": 2, "first_leaf": "8a3a4601f36fa9c390e9d690a87c2c32ca8d039becd87497f2c7e6aeadda7a3f", "layer_root": "c5e6e011fd76656fa059479df53c33122b79894e95471ffab51dd6b6dedeb9a2"}, + {"d": 3, "first_leaf": "b1ea4061930ffd7af7ea20c263e8c41e3cfc46202480b54945b302759e4fb8a9", "layer_root": "1f638946ead0713ced39f29587cb25f308f43a2db7443fe4fc12ee5574e2e930"}, + {"d": 4, "first_leaf": "126e67d8deb4860e5a065a64b28319be8db0441a397dc716b2cc63b14dafacdc", "layer_root": "225d49f0dd2fc63dabf168f21825dd49dc08958ee89a3a1841db1dc2ab9e5fd7"}, + {"d": 5, "first_leaf": "a420dd3555ee2d35e54094b728abb93ef1e441426c0fdac244a88f0e733d7438", "layer_root": "f6c327d2325176f17f157c40f345251b788f37377c65ea3d5eaa3f72e2af68d3"}, + {"d": 6, "first_leaf": "3a1ad10b88c75a26d86a8898e00519c7bce95870d7d2f7be716d179393a124e7", "layer_root": "0f42251b7c5b0e388e761f3a35b319c455f9539adb59f7e725959b91ad0e3f5a"} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_keccak.json b/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_keccak.json new file mode 100644 index 000000000..6012eebbc --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_keccak.json @@ -0,0 +1,13 @@ +{ + "generator": "stark::fri::vectors::leaf_digests_json", + "hash": "keccak", + "codeword": "b_group_folds.json codeword", + "leaves": [ + {"d": 1, "first_leaf": "b4356e59d9d0129ac1baeec1c40de2a0fcf45567e9e1e8a360d745f6778ce760", "layer_root": "99a27b756e7788b5f0fafd9ac8db8be0550e52b3acced0493e4afc8c9a8569c4"}, + {"d": 2, "first_leaf": "5e4e45941b9f874cd7a656bdc58ba55813f699836033cf2e05c2b45217e86ecb", "layer_root": "84735cf04432cd12c5f1b0d44b01765d65b6d1d442e4fa38d5e4567e57cbd40e"}, + {"d": 3, "first_leaf": "e592afb01c57ba10be32ea80ed280731d8aff2c07d13ca46f582d95ca0883710", "layer_root": "82365c8c49ea1f57c6ac51159d7f50ca46e05d5ac71bd04494bc86280ac19392"}, + {"d": 4, "first_leaf": "db81fea22e4d8a7a71f5a888328f5e1cd879b78b68c77b607beeaa32a26dec44", "layer_root": "4d720f49bb8997718342a6dd2c6d1c15a50165098e7a71b0fa797cb3b4a76bce"}, + {"d": 5, "first_leaf": "21c121a439c36321972e249c2aa67dfe06d681db973e1f9c264f1cfb71f2ff59", "layer_root": "12119631efb91a67936a88bc1d7d6ceff2a035b33cc82d63ee153b720ece6d63"}, + {"d": 6, "first_leaf": "3c89c987acd720a42d8ab1a61f200c6d5f57571fc5407adb81c9438e3c508ebd", "layer_root": "1be1ad81c0b977048465555718a8d8d5c5116339c1ccd09cf1de51ac4f82917c"} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_rpx.json b/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_rpx.json new file mode 100644 index 000000000..81af400ee --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/c_leaf_digests_rpx.json @@ -0,0 +1,13 @@ +{ + "generator": "stark::fri::vectors::leaf_digests_json", + "hash": "rpx", + "codeword": "b_group_folds.json codeword", + "leaves": [ + {"d": 1, "first_leaf": "c4e19fef0dcdc71c226bf8697094739acde92ea9d2259c1a3048ffcd9df670a1", "layer_root": "60d83fb6ab5a61b5009a1f668a918ba75fe21eb0a0b428055ccd2f827468ac24"}, + {"d": 2, "first_leaf": "6b783ab0c5d8a708a5834c5f6b3696c93447e470a702402f708d1dbc4e666de3", "layer_root": "e8b68715ea877da8ea07c99c43777b891b75105961e8fefde6b6ea9cb0292d82"}, + {"d": 3, "first_leaf": "5b3726fbd3b5ff5e7310f7e1e8023b595ca1e311d8bc1e398e5c236bd0d07fd7", "layer_root": "573e5bb92f63bf95684a5599c71f931371f096e829eaafe764ee2604037bfae2"}, + {"d": 4, "first_leaf": "4237eaa85f846fbe832443ca5ba31fccd041d1c2d1536f6726bc220c9583138a", "layer_root": "1d73919647290db8d8fa0031d93dbba43f40c097383c4adee0f1a59780004b0a"}, + {"d": 5, "first_leaf": "37fc7cc3a71608b4595d31c80f3914ba81ec8bdaec6c42f9bae72b6a3a9b972f", "layer_root": "c853b5880db08c3b1581c1f285fa1065beb62507d11e18db2167b17a29b60d49"}, + {"d": 6, "first_leaf": "6854823ddedf06afd546f97064760b19a811a8e4a9f026cabdfa1d17e1f971bb", "layer_root": "122803d141311339cbef2d4483906f0f548de2db1e58b6696aacbe21675a2595"} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_dp.json b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_dp.json new file mode 100644 index 000000000..1c15b0b37 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_dp.json @@ -0,0 +1,49 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "blake3", + "format": "cap_dp", + "proof_rkyv": "d_proof_blake3_cap_dp.rkyv", + "proof_rkyv_len": 41480, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 20, + "grinding_factor": 0, + "coset_offset": 3, + "merkle_cap": "auto", + "trace_tree_depth": 11, + "trace_cap": 3, + "fri_tree_depths": [8, 6, 4], + "fri_caps": [3, 3, 3], + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 2], + "fri_roots": ["f5660c4333b6e611e901e87422b2c4270acfad24f631d9429831d76b54566517","0cd55ebeb840e8373096d7b45d7a99eb5f2ae89c0b700d618f5bc5e5cf7d8cac","eaed7db0665b821d2c690ad7c99f3fb3b28d4b7d3c6167be86e2033300fc3997"], + "zetas": [[10771210179622817679,127754635188287825,9592161990157892076],[2334636387541570725,4598538304975584359,12240732424901763132],[10166376440375277554,17046118386894069248,10115891851528829537],[1897382409627266702,6084356605560232122,6323818535469693028]], + "terminal_coeffs": [[5380735102582769720,14770520085343157731,17397790325610342738],[14177126935727096750,9878484623770692025,8126381307417814598],[13709110344626157024,14960543001611777495,13121995131109452668],[3543500174378306775,245990784589754978,17448449264639928647]], + "queries_detail": [ + {"iota": 975, "deep": [10762173397373278909,6238205991322615201,16902290430091080608], "deep_sym": [6360510169239840515,6098314158097471188,14374437857455028609], "terminal_position": 7, "layers": [{"layer": 0, "d": 3, "position": 975, "leaf": 121, "slot": 7, "values": [[6358380543480134188,15233297943481819331,7802884743754435287],[1561050269828331995,17056437354980486597,15953089478678981981],[9902832432796422005,16144145762744260912,2452053501664136327],[4059115819907980750,13622239393642520008,10742588081843022005],[15097201520147174657,13343912955236085051,15486897778505591961],[5267077696719908108,9464151356603282791,2320496645939400341],[7693143272069720023,17017576095617663956,6992176147234983386],[14242806506751524709,5590906401091982117,17893550137403079326]], "path_len": 13}, {"layer": 1, "d": 2, "position": 121, "leaf": 30, "slot": 1, "values": [[10075702410473013791,4716385726265313391,11609542590222699029],[7845467542947739937,12387174233603512715,18200019442985323599],[2909241347398984484,5399916910453173204,9233450494253184011],[4113688591137365584,8099000987494976281,456105366810755859]], "path_len": 11}, {"layer": 2, "d": 2, "position": 30, "leaf": 7, "slot": 2, "values": [[4720003309196990551,4178739593595040029,10411467881539262427],[17765747612192294497,14700500661535383226,13695608480138073067],[1009102207610472007,14305427383432249940,4131500240907956149],[1496298963912337677,3644564800854218514,5628674521221557415]], "path_len": 9}]}, + {"iota": 1979, "deep": [1491025643980379174,12685070184352261704,7728318385342818721], "deep_sym": [16450052277900778758,13025974084100681593,284476606938439535], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 1979, "leaf": 247, "slot": 3, "values": [[17193942932342370980,12145038434561791480,11576353455053528850],[8379852527282925844,8601836539934055187,15168362594496179266],[2592359005135839197,5381418174777500780,3896499938254073228],[4822332081114310830,2565510977245490832,11907903767149013019],[13285130888391799040,1041585227093494597,2304343024112292721],[11261822772218442511,17634015746885931060,3497632743424289560],[10018483860357178719,5407737136598053306,11038589506904123227],[5386981123451134746,15435103548476985734,14493978820809476719]], "path_len": 5}, {"layer": 1, "d": 2, "position": 247, "leaf": 61, "slot": 3, "values": [[16716297285756313772,4980386954389515672,7587525751122803270],[14964533687760416685,13917074670739361960,17216570321581969515],[3750454710447327212,10631398170004764756,11015661283393845074],[10023782474286288464,1360821076844805368,12167231288721989489]], "path_len": 3}, {"layer": 2, "d": 2, "position": 61, "leaf": 15, "slot": 1, "values": [[6218711529479866098,15946762367783592827,15439960344592751968],[13215040631267974591,7742956115152970799,9370753578504439872],[8789152590505293503,2216348788422376564,6920254999236655340],[702787882218669028,13764292175754416497,529455276546629621]], "path_len": 1}]}, + {"iota": 196, "deep": [12996823082221702228,15546436838001982955,12765624607176451818], "deep_sym": [15258355026629750431,13813035249993486719,14742394712154322144], "terminal_position": 1, "layers": [{"layer": 0, "d": 3, "position": 196, "leaf": 24, "slot": 4, "values": [[5990450875556600463,11483765027548679992,15569727055914393856],[15481225486576456763,6990687647448126227,17377462139297815371],[17823918767605952564,15408822346409695669,7766449979365244357],[10521282185009747890,10208004641233698759,9793502955518714526],[15989513811586569514,8058942065900320453,2895357368330783138],[12375312625287684078,17725753712340770465,14836143340177261245],[6076100021308446097,6158543389568230761,12040165962685290590],[12787237553493716673,6399667255546886261,10645808993947995036]], "path_len": 5}, {"layer": 1, "d": 2, "position": 24, "leaf": 6, "slot": 0, "values": [[3955724155215551649,4903104037739106676,8341497941364977219],[1366207781672916929,4653995748612992444,1985491773491753288],[12110069387491220579,3462604155932552453,12998143760810181545],[12022604030106382896,9872253098559971229,8337931158086605882]], "path_len": 3}, {"layer": 2, "d": 2, "position": 6, "leaf": 1, "slot": 2, "values": [[13763808492525116750,17048718856988468120,4007208303874274964],[9747356233047801788,12353956971904007004,16715523138327251637],[17215889828796017128,10441867300762877602,5539458827215095394],[16874552336957454881,3298743664346751734,7613930184659830057]], "path_len": 1}]}, + {"iota": 1203, "deep": [15752534939543042330,15583402344956286664,18180929469912608205], "deep_sym": [6004675541515862623,15700646189296511672,11249127824744232770], "terminal_position": 9, "layers": [{"layer": 0, "d": 3, "position": 1203, "leaf": 150, "slot": 3, "values": [[10874562829332116737,16119632175137550595,5814025373320402574],[537192166862156226,1226209784871864696,10142087228779911497],[13928959866231520713,15456751127415083532,8693947028410900381],[3828009832111636345,5111308859100454226,13658206209512386784],[6969249891844518361,15061980941965197861,2988715140401985875],[4920000362291503630,4798949236132645012,14846074530401444401],[11403031433147724274,14457081909743702709,9196868723010592467],[8002035308229413454,14341041264576097070,4407237699945452429]], "path_len": 5}, {"layer": 1, "d": 2, "position": 150, "leaf": 37, "slot": 2, "values": [[3024954652785290495,9666301642784843613,6945943976099679195],[12418472477586847230,17196060582103289666,2393471149779994062],[4625263396386879378,12619942520976382752,7834949255201727970],[5650597075990240870,17238954492838515108,1158458673914035833]], "path_len": 3}, {"layer": 2, "d": 2, "position": 37, "leaf": 9, "slot": 1, "values": [[15079255711491555218,12300854889741332788,10589502824497300729],[11478058193422450763,12570729264925447255,11149230142755138274],[8007098169052196112,15752356817250027978,12944679080360575431],[17096915696030076215,2520825861781781734,8689682692440434782]], "path_len": 1}]}, + {"iota": 991, "deep": [8681993120970919677,2087072008066845857,11711806762509319738], "deep_sym": [7491938478046119788,14436381252314064679,2021960428839489614], "terminal_position": 7, "layers": [{"layer": 0, "d": 3, "position": 991, "leaf": 123, "slot": 7, "values": [[241286883786860593,7693402300417727673,16090435993444967711],[17935640868834497012,8620962675063390145,15113657813157556306],[15781246783767794142,4964978784804618058,15015775329041983920],[13016971764566756554,2018125833213814590,14318788237881126017],[10152329590681864501,3050038139514453021,1800033870197117892],[13712181320069854702,12383945862328316760,7877046573358953983],[16434444243828717809,5651549530951913294,4761546699101515128],[2254401359065396503,11395830875147205648,10277023618508617127]], "path_len": 5}, {"layer": 1, "d": 2, "position": 123, "leaf": 30, "slot": 3, "values": [[10075702410473013791,4716385726265313391,11609542590222699029],[7845467542947739937,12387174233603512715,18200019442985323599],[2909241347398984484,5399916910453173204,9233450494253184011],[4113688591137365584,8099000987494976281,456105366810755859]], "path_len": 3}, {"layer": 2, "d": 2, "position": 30, "leaf": 7, "slot": 2, "values": [[4720003309196990551,4178739593595040029,10411467881539262427],[17765747612192294497,14700500661535383226,13695608480138073067],[1009102207610472007,14305427383432249940,4131500240907956149],[1496298963912337677,3644564800854218514,5628674521221557415]], "path_len": 1}]}, + {"iota": 730, "deep": [3749017813721115261,14557058401905093868,6250134219782047991], "deep_sym": [10973933773174417008,12152588244373773982,7254759842174161858], "terminal_position": 5, "layers": [{"layer": 0, "d": 3, "position": 730, "leaf": 91, "slot": 2, "values": [[13625437253679645360,4781346700753413259,9739480830032267559],[10343180462388971404,14390237986214676488,1233992481629191112],[7094279693781689175,2117205439232100477,1821367394658403345],[10297720058835310480,1175273423588874691,13535140666215718820],[17216420973199492163,7368341788859375110,13031855082791043411],[14966444954846038540,15465064230605228385,11469894714541073913],[2621509644171994588,10617636738579191725,16179719487663085109],[6977255124091830661,18103993031165746594,17724514913003277706]], "path_len": 5}, {"layer": 1, "d": 2, "position": 91, "leaf": 22, "slot": 3, "values": [[5144674147078997594,5491908906114697312,3844920803894566132],[5020900329166536587,7144686597885242940,16786875069484486943],[12993512973289219318,7358410183833446838,13935406277168409330],[17784522352072343142,9563050577201338945,352692387285700454]], "path_len": 3}, {"layer": 2, "d": 2, "position": 22, "leaf": 5, "slot": 2, "values": [[6011936006421691404,10675680366181426359,16583790013325704227],[12401360982478626076,2729294664100958931,4483876789092781890],[7256982332670895249,18228230382919437261,7873183635821042676],[15333325283421136752,3228834115588111137,8250758386838693228]], "path_len": 1}]}, + {"iota": 1461, "deep": [5143682188678502351,16221457536374172264,13900721353390447660], "deep_sym": [753761722491209855,4901966673579435005,10628686203083643451], "terminal_position": 11, "layers": [{"layer": 0, "d": 3, "position": 1461, "leaf": 182, "slot": 5, "values": [[9078356149970439678,12742462076713984472,8334953287620001223],[6232363074738132968,9592569284571747908,17454801279480835743],[12665951647893564170,16631753895531546839,18037466198834859557],[9379129987218727754,2990857268642168597,3455448609799108188],[2306338561029461238,9231856230271642991,5092529416795310001],[11856654519610033437,18143211246415090160,2467191868091550359],[14215724434611581339,4721597587453395495,8407334378937024772],[174624353941079833,16048126312610653415,3173888900307413954]], "path_len": 5}, {"layer": 1, "d": 2, "position": 182, "leaf": 45, "slot": 2, "values": [[755410960438700629,2805302518756125053,17531434065929643718],[3071543226078084354,9909734270156800125,15970672385847214190],[10754777501829526062,8163465937849620718,17841447578808981118],[9283169039803580114,4638181989398948167,6627029688357333091]], "path_len": 3}, {"layer": 2, "d": 2, "position": 45, "leaf": 11, "slot": 1, "values": [[5980468832933785485,14242050719805404593,6439206589369129064],[15179786468695181372,4377693513116986124,1322832299243542576],[13360246428180627504,4483599704657039337,1136060911200590793],[16615211484878415091,13032718370113144279,16130260341719337467]], "path_len": 1}]}, + {"iota": 1956, "deep": [13012147546762863285,17378967107273509838,16785009961307145384], "deep_sym": [13570520044671590068,13173769417357444649,15377036488763418784], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 1956, "leaf": 244, "slot": 4, "values": [[1139695083250640099,6281616195601480811,15831019522963135362],[18292945331627888905,3930898106053175575,14706941153128633931],[4811231360020697715,485782268072192698,165464771280611802],[4877247552542499443,3192777604604862923,6319355463364641086],[15125822918693992437,17537798162189493620,1629697386871100514],[13796269321411676444,1854432270020465144,6973461315370024637],[15795021451379651796,7736292815854993306,5175720845279510537],[3000276692915542192,11250424266363190971,590499797331551276]], "path_len": 5}, {"layer": 1, "d": 2, "position": 244, "leaf": 61, "slot": 0, "values": [[16716297285756313772,4980386954389515672,7587525751122803270],[14964533687760416685,13917074670739361960,17216570321581969515],[3750454710447327212,10631398170004764756,11015661283393845074],[10023782474286288464,1360821076844805368,12167231288721989489]], "path_len": 3}, {"layer": 2, "d": 2, "position": 61, "leaf": 15, "slot": 1, "values": [[6218711529479866098,15946762367783592827,15439960344592751968],[13215040631267974591,7742956115152970799,9370753578504439872],[8789152590505293503,2216348788422376564,6920254999236655340],[702787882218669028,13764292175754416497,529455276546629621]], "path_len": 1}]}, + {"iota": 1148, "deep": [559191688237983931,4204147968037837581,13155536057586080738], "deep_sym": [4401246871529010816,13593062267599263121,2999155291195094619], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1148, "leaf": 143, "slot": 4, "values": [[11039635021177097374,4477374690065111236,15665506636854122333],[6156104977645698326,6926748032535851306,11511066775905706022],[8580444569339206543,2026670230496760715,16952652587434159221],[14258287573194214551,3410911056137109245,17066913606006937522],[12939165465098362074,7509018497879618565,11429597422765499875],[7515506259952081009,5672153501166785178,12639132181321448068],[13980620889037711513,14136365014306635812,3631194797964167943],[9895737230979731583,14206155153721926339,7744139835103824986]], "path_len": 5}, {"layer": 1, "d": 2, "position": 143, "leaf": 35, "slot": 3, "values": [[1057482104613101351,12077553649510848318,12985024014551316629],[400287114595203204,11726952929487123048,12602932319028339522],[13399556945017055043,10426691419037754163,8459530740165622424],[15320314632502273335,16540722097958614548,2745741974161055219]], "path_len": 3}, {"layer": 2, "d": 2, "position": 35, "leaf": 8, "slot": 3, "values": [[2361770517565856606,18392804438032607863,14698819959800212628],[16937432840950086279,13504488964490434884,5737219975509076724],[16200627196689697959,5428662221761898729,14790338823710953763],[5136352044889977961,7465012862093234680,14298704360569780639]], "path_len": 1}]}, + {"iota": 1097, "deep": [9911322248523663148,7449350727945815226,7912850267400926375], "deep_sym": [4610277628482265293,7796822887877965851,16588608283361741697], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1097, "leaf": 137, "slot": 1, "values": [[16579933164913246961,14938927767270908477,12243013016946014882],[1211652563999613683,3322115237433417792,16033653635443512602],[12523441763998198184,12506679984508601708,2645930596723636519],[11351023897894651361,14829732713595927367,14888122988734751436],[6927428004267178384,5040625818333788974,1120890039336808382],[8903235146386526066,4628324794344608477,258520791444110126],[11261972428415464801,4725869900107112620,14674247944396481252],[14305402251855101376,10312372909744114657,3393540230751075403]], "path_len": 5}, {"layer": 1, "d": 2, "position": 137, "leaf": 34, "slot": 1, "values": [[12497579461873370340,17223221246127914665,1592668878267302670],[14187438249429787970,18179245312246498387,10554628983733290057],[2179539185730340905,2135162114646552181,7035059331050521551],[5465159271385692206,5923081554081629690,1937189932867117535]], "path_len": 3}, {"layer": 2, "d": 2, "position": 34, "leaf": 8, "slot": 2, "values": [[2361770517565856606,18392804438032607863,14698819959800212628],[16937432840950086279,13504488964490434884,5737219975509076724],[16200627196689697959,5428662221761898729,14790338823710953763],[5136352044889977961,7465012862093234680,14298704360569780639]], "path_len": 1}]}, + {"iota": 893, "deep": [14869975709152106679,1921980599777105357,12641264178656378660], "deep_sym": [4324020330371914544,15450910801133054761,11596191006253436286], "terminal_position": 6, "layers": [{"layer": 0, "d": 3, "position": 893, "leaf": 111, "slot": 5, "values": [[7364517996926882784,4423842349740890359,4094086245325002393],[78915192562221491,14174672544353666956,16649158135147014355],[17270498933821331212,17933816006483508343,9680389171947044434],[17637018130336130973,15277620556589607194,10715099103759978271],[7046591968540024718,11745653068892231709,4936447358022281253],[8846985800004343385,8976869429746572050,13004655854294770678],[135177480516526320,3881210905077568264,5572941719046141216],[14535766980758331393,15803350649698253723,12689112861429696910]], "path_len": 5}, {"layer": 1, "d": 2, "position": 111, "leaf": 27, "slot": 3, "values": [[6396069085956288173,15903209904786184991,6201948201142607279],[16683133169849730866,9587936157017377155,6431682643504102182],[4650709611847295533,4352442522449825086,12605080154257239873],[6067851984933547871,1248514420175494616,11918932032981867635]], "path_len": 3}, {"layer": 2, "d": 2, "position": 27, "leaf": 6, "slot": 3, "values": [[18054014190726286609,5230501828626636123,4663425874675860736],[3106421522097749857,13707080887749262922,9451679288545940912],[749343417051452695,17957036162478834582,2396924204100391866],[7003646066458667451,7182864086926233463,6926954008656726935]], "path_len": 1}]}, + {"iota": 1611, "deep": [10283771577448452308,6311775650455490315,10011119081764825290], "deep_sym": [2942709850112994258,15777379869206527462,16026469347544900843], "terminal_position": 12, "layers": [{"layer": 0, "d": 3, "position": 1611, "leaf": 201, "slot": 3, "values": [[15034096618106703532,16266067412950314637,6984842303440981460],[11574209943803058102,14368413300772335990,83483935827968805],[15002499834400737064,18125927580513201844,14083158362275701452],[2794306545247555542,12893554998288301299,6896983749182260428],[17387138325837804041,5534198072980561863,17681730334834591590],[1433277771010912265,7385939762528684302,5987338178976194148],[10817868126959669011,11169882412867946117,6174821245334077769],[6672868139619767727,15414026211813473718,12731116147464688792]], "path_len": 5}, {"layer": 1, "d": 2, "position": 201, "leaf": 50, "slot": 1, "values": [[4371265254077541125,15847370424923923597,9559539041216336520],[11627574753275232018,8594664059292053846,7068185601245487025],[8147614893575946269,4951488265475391522,2682227487549181930],[14284256686954366294,3438993162828147454,12704539693506731230]], "path_len": 3}, {"layer": 2, "d": 2, "position": 50, "leaf": 12, "slot": 2, "values": [[7732520432572237241,13165516863377634804,7705769214984174459],[9141973685089497517,2604347535429187405,11005539496063051989],[7573637287085966786,9264725481276609229,1916875510556010185],[5085817792127901346,9883478890715282684,12164815580791611471]], "path_len": 1}]}, + {"iota": 1315, "deep": [4868878336503280648,5502497972773445066,13818167565514677736], "deep_sym": [14213966109955828538,16788446689672845476,13145565271098668723], "terminal_position": 10, "layers": [{"layer": 0, "d": 3, "position": 1315, "leaf": 164, "slot": 3, "values": [[6376909581141271245,1241339294886889724,16399006528135619687],[17670280822265450577,11175947999744963222,6580869690978006091],[7637614296323359836,5078512359894114442,11077878261128887197],[6481499981089297500,12508414171445953510,221573720616415350],[2027678399645124930,17032205516310667683,8433264502054998557],[11784019934107793864,16587931275900712549,4206781342497882241],[3314194782870738512,5273174711638861445,9661562963755791982],[7181141736115860532,1833310872594558499,9936079683014406622]], "path_len": 5}, {"layer": 1, "d": 2, "position": 164, "leaf": 41, "slot": 0, "values": [[14261929297497037883,12784405026888296033,10988981792043533678],[1267904154492123740,15997048429724618067,9199126858793819891],[2141794565460227736,2222704669356598473,485299134004841347],[1392975148673254767,7775679193829347867,1952883468569488832]], "path_len": 3}, {"layer": 2, "d": 2, "position": 41, "leaf": 10, "slot": 1, "values": [[15744827006673985450,11711997518029820360,526067923678103240],[9361848992962370566,5820347163924234588,15609654412100488927],[12467756039166196733,1959095310809430573,10751774919655598094],[12317929042402698177,8299237154342900303,376015377231772288]], "path_len": 1}]}, + {"iota": 347, "deep": [5652495261376918233,11003491597239608592,8555109965912264684], "deep_sym": [11701311397388666708,14684683370759232903,4789515469099634998], "terminal_position": 2, "layers": [{"layer": 0, "d": 3, "position": 347, "leaf": 43, "slot": 3, "values": [[2279034670790862856,3918865091040307633,14883222214352393554],[9955061263535213790,1461571236494044152,11490165111605523433],[11370145577179544,7636842533978780720,12565305142199480505],[18246018928058218312,2899480678750643106,359559756082944587],[8261422293642933083,14348141907525272774,2827499312888751533],[15265874781544945251,15758119172686786126,14020258214772758245],[7692428087112738499,16089983222131207652,5190297970971948095],[6288015372171692919,3560619850227336675,3503535371271013628]], "path_len": 5}, {"layer": 1, "d": 2, "position": 43, "leaf": 10, "slot": 3, "values": [[8082651957698097349,11080401022154114235,13225618016723934942],[12967900839345856436,10245135173744964460,13882328420034803705],[7308376203592554799,2200255532820098891,3144650772598452114],[16979859653628826157,11957020649260092048,4154152077335082351]], "path_len": 3}, {"layer": 2, "d": 2, "position": 10, "leaf": 2, "slot": 2, "values": [[626631545137480777,14040093596076234059,11563086371519029480],[14166743346901113445,1584993346684759291,6073191583192600810],[17835536105487204093,4751491938112621960,5358807637082779310],[10015740991204856004,10917778466779632243,17376279851693605463]], "path_len": 1}]}, + {"iota": 7, "deep": [10729628599147983865,11262284762986532334,12067698941208999342], "deep_sym": [13793692426065840049,17804767925899894057,12522649863195261323], "terminal_position": 0, "layers": [{"layer": 0, "d": 3, "position": 7, "leaf": 0, "slot": 7, "values": [[15397680014036791351,1884054888443124705,10356418525983122034],[3574327614819495940,9311713712635394509,9592445444305087428],[3995997584691044082,2898147102620050906,1534897124393607006],[12010016124993134544,11373335453832742986,15988031694065613139],[8586139949312340825,7786086002367368963,15297348463510698442],[11932348748044977641,6067544001558133969,5132720332181670008],[7282236280819674652,15971785141886602896,15580216555727892132],[17515904616771614042,17626757208160268926,6170444558336859025]], "path_len": 5}, {"layer": 1, "d": 2, "position": 0, "leaf": 0, "slot": 0, "values": [[18419289037790398515,4119860990809229123,8532257483528117106],[2441301968378033486,11031829990918540139,9356755462377865367],[2444772718654314503,10338519488613625025,4057680024245573228],[9688440605635587702,993932466441680669,10433316942822491330]], "path_len": 3}, {"layer": 2, "d": 2, "position": 0, "leaf": 0, "slot": 0, "values": [[6889909335419354700,6796959293305346289,14415195110069695447],[18373991401276132678,11560808149746317105,10847324815709489832],[16208114084891951396,4178287177882041172,2800573986594485795],[4690326594433345688,7539141686653911826,11409842170078198974]], "path_len": 1}]}, + {"iota": 1833, "deep": [15091594336359056927,17731773756750016012,3951308870671530023], "deep_sym": [11288885646087766633,14279114384306327058,17497384530855239826], "terminal_position": 14, "layers": [{"layer": 0, "d": 3, "position": 1833, "leaf": 229, "slot": 1, "values": [[5835838119563522966,13527318661581226992,8988488912368153474],[7825002203746412915,7157868495818377610,391097759177990040],[15505006135794333734,11222088631013402988,7599709940706104976],[17400545490854520115,8691678530155684831,2366389859525368701],[13271538683612303742,9468247552162442421,9765400918464248667],[75309224696716840,15170015511120189248,3729324586155886010],[6124965313206069075,15411539721238865905,16260001499293568983],[3120690321051812524,4122134073004025220,4196588213607274638]], "path_len": 5}, {"layer": 1, "d": 2, "position": 229, "leaf": 57, "slot": 1, "values": [[16190254108336593334,153294746140690160,9788877368221835751],[5557446448754715381,12694267056985157970,2800513960783386007],[15681186669330942481,6671086479859882339,17225476729331239755],[17856735751311346283,17121906477052463042,17696748984568817182]], "path_len": 3}, {"layer": 2, "d": 2, "position": 57, "leaf": 14, "slot": 1, "values": [[984203213469929437,4224343177582388484,9572623288142859938],[16056684969230873251,10652539299959440313,12750365249972731935],[6580582599927055682,3943061636956573218,9060032440671433186],[17342194358321478128,894315335249473662,4301831733190433175]], "path_len": 1}]}, + {"iota": 1893, "deep": [9502080685171718273,11900261630676834135,11105421041413284470], "deep_sym": [7000977341138268486,6102080310951961489,13938957829736479714], "terminal_position": 14, "layers": [{"layer": 0, "d": 3, "position": 1893, "leaf": 236, "slot": 5, "values": [[5812351387423155422,8111386547126011924,6341489365517505745],[14564580661283786368,11910756739879885116,14124074572254428382],[11785855844471819792,6567606644225299130,8592427053884858150],[16039001077420822027,14876600592689496834,4177020490289287383],[12573539672230359958,10736543775311885948,8222790836481073619],[13429784704579008119,16778130495890976308,3410075892657910004],[1667848710464024897,3131118275609095446,13086150401349088028],[17315040022593921497,10611946554924693795,14092060639864510202]], "path_len": 5}, {"layer": 1, "d": 2, "position": 236, "leaf": 59, "slot": 0, "values": [[5034510331297990362,8916005402476503296,3316722436455100847],[4322052901144351236,5052993851804539011,7660734087728392519],[10109600378082773260,16588247839764178304,12656780741746346580],[10741868519284278895,5468237293959312321,8792679859332184847]], "path_len": 3}, {"layer": 2, "d": 2, "position": 59, "leaf": 14, "slot": 3, "values": [[984203213469929437,4224343177582388484,9572623288142859938],[16056684969230873251,10652539299959440313,12750365249972731935],[6580582599927055682,3943061636956573218,9060032440671433186],[17342194358321478128,894315335249473662,4301831733190433175]], "path_len": 1}]}, + {"iota": 1006, "deep": [278565245816208269,10403413944694705412,18373543770665442480], "deep_sym": [5782750051717883566,9869349874627987641,5162411754453845894], "terminal_position": 7, "layers": [{"layer": 0, "d": 3, "position": 1006, "leaf": 125, "slot": 6, "values": [[424531153078822075,14437669868520738150,6405119015988337644],[11781162730973500861,10727531823436307995,16141001035929866190],[16031527700303044792,10858195524075015505,5243643158770699539],[176124510599809593,14363838589718254776,15251157617241324835],[6771402597407673287,8335611809952380981,1074070984178883283],[3055397484829669688,4726642450162797422,1224456793439981816],[8394748372127108582,6409876248154059481,18188302789923254571],[5509435528613056072,579645458909143161,14213312078695342130]], "path_len": 5}, {"layer": 1, "d": 2, "position": 125, "leaf": 31, "slot": 1, "values": [[10361947016727152801,9865134566086214370,1855324780858693853],[378975246974756134,11395599957458128578,15469914222301991129],[9353484865522397870,949259615002670803,12259663185538090699],[15448519710699047426,10685872832601579041,2583763360548785823]], "path_len": 3}, {"layer": 2, "d": 2, "position": 31, "leaf": 7, "slot": 3, "values": [[4720003309196990551,4178739593595040029,10411467881539262427],[17765747612192294497,14700500661535383226,13695608480138073067],[1009102207610472007,14305427383432249940,4131500240907956149],[1496298963912337677,3644564800854218514,5628674521221557415]], "path_len": 1}]}, + {"iota": 1700, "deep": [1410836301490999973,1302595678191022288,179613372424907210], "deep_sym": [12728179003292154660,10911574655256566401,13521656637148951812], "terminal_position": 13, "layers": [{"layer": 0, "d": 3, "position": 1700, "leaf": 212, "slot": 4, "values": [[13559371558166416798,3015153850462235040,6169493881691645384],[13186179764720193276,10761686304300220634,6573602884002212893],[14971197062009843562,4024604816664565086,13338302287007886789],[17452603893950420306,13885944451748366985,2828010835887818953],[2716040662584989146,13533080688972263783,7945319097121604891],[16210715316971838963,13545031526832441319,17252233972962118392],[16894810944015893775,6179082506926514775,10260076086525229951],[6880444503718828723,7798036362196708901,5135439215824977126]], "path_len": 5}, {"layer": 1, "d": 2, "position": 212, "leaf": 53, "slot": 0, "values": [[11921998034465242009,15803801940516939286,5425014481155423121],[14339889849134608607,3519911207241293731,1012333810442413988],[8313961635410473445,4840142476262538653,10140604888347464359],[12239251471703462122,11212894982547623503,4964774848319983656]], "path_len": 3}, {"layer": 2, "d": 2, "position": 53, "leaf": 13, "slot": 1, "values": [[17097230081346561873,9416095663746874971,16909476699859173651],[5954831163834882122,10648165262835477953,16923660589352447836],[16560227121839913403,12539513539081302255,14451257014363123269],[17396501188837888843,6509268572137540082,14614123686429091845]], "path_len": 1}]}, + {"iota": 516, "deep": [5960966010798800123,2846205985179489762,6865158036389153804], "deep_sym": [16774012547545437690,10867764373143372640,12104783288707649791], "terminal_position": 4, "layers": [{"layer": 0, "d": 3, "position": 516, "leaf": 64, "slot": 4, "values": [[17750384411855288273,1741652024793611125,15489881843544541870],[9786778802115784024,14048855153868086834,10298495367053368114],[14192416002951717457,10195523734556928612,6591176491061048897],[15079918591651446981,15575218939349755148,15828430898221110407],[5072774290398093241,18130906298164818449,300258120769860962],[15503009350805253650,3867086856121895573,14538628658269385269],[3336112409534602303,17408497384008958408,3796420379159471748],[11021938962937693750,8369798889504894647,12334916565960248080]], "path_len": 5}, {"layer": 1, "d": 2, "position": 64, "leaf": 16, "slot": 0, "values": [[6139920174700566034,15308264411679642375,8162232957374836514],[4046575768078141090,4755179760145096589,2066516433627498940],[17931094972437772834,2869413196283560188,8007455432109443910],[17654138861816481141,10102921129471255319,16147744345843337964]], "path_len": 3}, {"layer": 2, "d": 2, "position": 16, "leaf": 4, "slot": 0, "values": [[2330219872305505679,2524558845414778148,13832449164595042239],[12707191272162393673,14207086513203844784,9008382796051335778],[7379323499145744541,74149321890311351,6771439343933968571],[2213691850474575489,841181090964414977,2633191934983112982]], "path_len": 1}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_dp.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_dp.rkyv new file mode 100644 index 000000000..017dfddb3 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_dp.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_pair.json b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_pair.json new file mode 100644 index 000000000..cf73a0ae8 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_pair.json @@ -0,0 +1,49 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "blake3", + "format": "cap_pair", + "proof_rkyv": "d_proof_blake3_cap_pair.rkyv", + "proof_rkyv_len": 51752, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 20, + "grinding_factor": 0, + "coset_offset": 3, + "merkle_cap": "auto", + "trace_tree_depth": 11, + "trace_cap": 3, + "fri_tree_depths": [10, 9, 8, 7, 6, 5, 4], + "fri_caps": [3, 3, 3, 3, 3, 3, 3], + "legacy_encoding": true, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["2fc983d7a9f8dba305332d7c27f44e2afc07aa1fe014ad8a85ca9bc36844a992","a01be93f245807d30fde826335bc7dd8bfbf9fb0545769f85da9a6ee2b564301","4cc8bdb5c5d436e8b5cc91aa230573d630e9eb8a908f2650f9d4598e3d731b65","82de5d8f879bb994fc9573a6c3706b71adaf8b236e17ff2047e44bfa64f1e480","779e1acea2c1b391312f39412312a9a6bcb8cf5a66e1954d2b9d340a74a74bd0","191d0d55f0bf47af108196ca3b2c067e667807e78ae6681a814b3be0a8f4e445","878efaffc3ca3b900bd232b64cdd6a142203e2ef0ff1cda3f52ec590b776f071"], + "zetas": [[10771210179622817679,127754635188287825,9592161990157892076],[339236561547217708,14515476371055385421,3041135081988152589],[7430745936816588155,8998042728974583901,11515773416551488605],[7956826836586454026,8667292109104632665,2851244499340860067],[16324173539864659489,11301157219502799655,18016560099956879839],[4272458413724263223,15273501817168123109,13432776003642703715],[18153136978195245525,4668271491129789573,15852649611975035906],[13206066232974685659,15811531208029248608,9742874826372310642]], + "terminal_coeffs": [[11908419985256297049,6320124696091700849,10477651950916658009],[16710003718284845920,14728440137509904251,12073313539240356766],[15142905694919717110,8656948196775444897,1363513317241862160],[2198207007945388790,2708142890943514224,17003186495140238478]], + "queries_detail": [ + {"iota": 1803, "deep": [15272426180920759111,5106447191221278975,14792296330971372023], "deep_sym": [10304415851256192438,7276545599604954905,12402529132092837573], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1803, "leaf": 901, "slot": 1, "values": [[3388282554933400969,3823175679916949076,2787870681482871753]], "path_len": 15}, {"layer": 1, "d": 1, "position": 901, "leaf": 450, "slot": 1, "values": [[17941501397892820289,9746070180589186316,4483120140038292319]], "path_len": 14}, {"layer": 2, "d": 1, "position": 450, "leaf": 225, "slot": 0, "values": [[1518365534971821388,8220153128022570539,11364526563819683345]], "path_len": 13}, {"layer": 3, "d": 1, "position": 225, "leaf": 112, "slot": 1, "values": [[10971480354833343982,3135816652628770915,6720283715471365573]], "path_len": 12}, {"layer": 4, "d": 1, "position": 112, "leaf": 56, "slot": 0, "values": [[5709537754561370510,10236832031319039769,1874314679153150939]], "path_len": 11}, {"layer": 5, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[2528382099678622252,12218130109821183716,1136296192569704372]], "path_len": 10}, {"layer": 6, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[7786343213267754545,6056155651923690370,13889151246865202821]], "path_len": 9}]}, + {"iota": 474, "deep": [4642716204571870719,13791353321977000304,9948795077040124575], "deep_sym": [10074490863165540107,1346332627183725457,15559140971681542809], "terminal_position": 3, "layers": [{"layer": 0, "d": 1, "position": 474, "leaf": 237, "slot": 0, "values": [[9803582068471756145,5326669840186105035,7793279955894935834]], "path_len": 7}, {"layer": 1, "d": 1, "position": 237, "leaf": 118, "slot": 1, "values": [[18172681946601424763,4149543359487769368,2150741210857753378]], "path_len": 6}, {"layer": 2, "d": 1, "position": 118, "leaf": 59, "slot": 0, "values": [[10054490575191079786,12193424298068301071,8417982262482120641]], "path_len": 5}, {"layer": 3, "d": 1, "position": 59, "leaf": 29, "slot": 1, "values": [[15947123418199701165,18407774728151935281,3292539258646734529]], "path_len": 4}, {"layer": 4, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[3117529940052833834,10473131964376682009,2083760568833245811]], "path_len": 3}, {"layer": 5, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[7713732352748329805,3614247649513246873,888672929281612740]], "path_len": 2}, {"layer": 6, "d": 1, "position": 7, "leaf": 3, "slot": 1, "values": [[12707951974262439387,10229714375229842447,6079425424868885692]], "path_len": 1}]}, + {"iota": 1018, "deep": [15882578000804364217,17570699945731153943,17271573467219472049], "deep_sym": [11776097457111120055,8466990234121688300,9890187330955688279], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 1018, "leaf": 509, "slot": 0, "values": [[11811277273608028663,11557005519804590428,6147111063572348711]], "path_len": 7}, {"layer": 1, "d": 1, "position": 509, "leaf": 254, "slot": 1, "values": [[7939867388228535545,10890533442334944369,9121900079366539214]], "path_len": 6}, {"layer": 2, "d": 1, "position": 254, "leaf": 127, "slot": 0, "values": [[432090123891713462,15713972828822391493,11186642764496342828]], "path_len": 5}, {"layer": 3, "d": 1, "position": 127, "leaf": 63, "slot": 1, "values": [[1293317382852890727,12037476111710244625,8463877166491912968]], "path_len": 4}, {"layer": 4, "d": 1, "position": 63, "leaf": 31, "slot": 1, "values": [[14282184254670867115,4596231514897671604,8263298406545493773]], "path_len": 3}, {"layer": 5, "d": 1, "position": 31, "leaf": 15, "slot": 1, "values": [[4047442787689190383,14125010312351736105,11271388519733766106]], "path_len": 2}, {"layer": 6, "d": 1, "position": 15, "leaf": 7, "slot": 1, "values": [[5728710831141537085,4396671778989160837,8614177465654515251]], "path_len": 1}]}, + {"iota": 1013, "deep": [16539758758549291992,4800579245526150018,16962061393147898641], "deep_sym": [3988521226579075591,6521018656605876229,8515646311306923282], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 1013, "leaf": 506, "slot": 1, "values": [[5710159636982614636,13770330871767670923,10497539398468115473]], "path_len": 7}, {"layer": 1, "d": 1, "position": 506, "leaf": 253, "slot": 0, "values": [[6144097270558028455,4496949226190935449,516924673664036100]], "path_len": 6}, {"layer": 2, "d": 1, "position": 253, "leaf": 126, "slot": 1, "values": [[15656447957273490722,18139085581736996306,16157908077113276289]], "path_len": 5}, {"layer": 3, "d": 1, "position": 126, "leaf": 63, "slot": 0, "values": [[2914172444221939731,10755256182453994313,12172760769612416178]], "path_len": 4}, {"layer": 4, "d": 1, "position": 63, "leaf": 31, "slot": 1, "values": [[14282184254670867115,4596231514897671604,8263298406545493773]], "path_len": 3}, {"layer": 5, "d": 1, "position": 31, "leaf": 15, "slot": 1, "values": [[4047442787689190383,14125010312351736105,11271388519733766106]], "path_len": 2}, {"layer": 6, "d": 1, "position": 15, "leaf": 7, "slot": 1, "values": [[5728710831141537085,4396671778989160837,8614177465654515251]], "path_len": 1}]}, + {"iota": 493, "deep": [17552019816042641949,18232228328537735996,17721947593967519347], "deep_sym": [11476893994278310325,10597338058207765344,1336796915425294807], "terminal_position": 3, "layers": [{"layer": 0, "d": 1, "position": 493, "leaf": 246, "slot": 1, "values": [[18246023938477235888,16844208616248807001,7488877263795102693]], "path_len": 7}, {"layer": 1, "d": 1, "position": 246, "leaf": 123, "slot": 0, "values": [[5457193399828553874,6465888369873413124,13473464106011068418]], "path_len": 6}, {"layer": 2, "d": 1, "position": 123, "leaf": 61, "slot": 1, "values": [[14501885818421213754,2404493437532211906,18327053541415153599]], "path_len": 5}, {"layer": 3, "d": 1, "position": 61, "leaf": 30, "slot": 1, "values": [[10253919672763263534,13085255767145830067,3363478501517189514]], "path_len": 4}, {"layer": 4, "d": 1, "position": 30, "leaf": 15, "slot": 0, "values": [[15108152339623035561,7941003559796812370,11287785174720768629]], "path_len": 3}, {"layer": 5, "d": 1, "position": 15, "leaf": 7, "slot": 1, "values": [[9325774833112564040,15477717807642519431,6899846380415881059]], "path_len": 2}, {"layer": 6, "d": 1, "position": 7, "leaf": 3, "slot": 1, "values": [[12707951974262439387,10229714375229842447,6079425424868885692]], "path_len": 1}]}, + {"iota": 1295, "deep": [2907714514381034844,1070164129957410707,17697561092214226008], "deep_sym": [10898207006015057880,4855160508758814138,8376350505293817877], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1295, "leaf": 647, "slot": 1, "values": [[18237373031964124495,12933656975114878376,8124190660344903209]], "path_len": 7}, {"layer": 1, "d": 1, "position": 647, "leaf": 323, "slot": 1, "values": [[15160245734528398038,2223725890827758023,1845689963201887565]], "path_len": 6}, {"layer": 2, "d": 1, "position": 323, "leaf": 161, "slot": 1, "values": [[3632524267806718007,17868686869941004595,2494655627500700339]], "path_len": 5}, {"layer": 3, "d": 1, "position": 161, "leaf": 80, "slot": 1, "values": [[2718920637662991361,14524410770564791347,12758797938498319947]], "path_len": 4}, {"layer": 4, "d": 1, "position": 80, "leaf": 40, "slot": 0, "values": [[4287120775860448141,16640217501487511425,11969890935976958528]], "path_len": 3}, {"layer": 5, "d": 1, "position": 40, "leaf": 20, "slot": 0, "values": [[17219987829003584375,1216251437672055770,14903611013931497919]], "path_len": 2}, {"layer": 6, "d": 1, "position": 20, "leaf": 10, "slot": 0, "values": [[14693608784504098272,9224706871872850546,6936163159577250052]], "path_len": 1}]}, + {"iota": 1692, "deep": [4675075042609955645,15761568838215639339,882232805073275850], "deep_sym": [333372144900086791,18010920076537972462,14717091746069690415], "terminal_position": 13, "layers": [{"layer": 0, "d": 1, "position": 1692, "leaf": 846, "slot": 0, "values": [[17431861332287145374,14475947496191718268,14296768809856075377]], "path_len": 7}, {"layer": 1, "d": 1, "position": 846, "leaf": 423, "slot": 0, "values": [[9642845874839210936,17308379870883033292,15084137183703271672]], "path_len": 6}, {"layer": 2, "d": 1, "position": 423, "leaf": 211, "slot": 1, "values": [[1604327021956893871,10821501388643107257,2099354348106496350]], "path_len": 5}, {"layer": 3, "d": 1, "position": 211, "leaf": 105, "slot": 1, "values": [[15537394326487831857,3289175237492248753,4061069590219069813]], "path_len": 4}, {"layer": 4, "d": 1, "position": 105, "leaf": 52, "slot": 1, "values": [[1158672035570015579,7704491263593106797,189694936280376845]], "path_len": 3}, {"layer": 5, "d": 1, "position": 52, "leaf": 26, "slot": 0, "values": [[2975349498257900987,8948160026774816179,8738746312419937635]], "path_len": 2}, {"layer": 6, "d": 1, "position": 26, "leaf": 13, "slot": 0, "values": [[12634357439633813059,12837117735719628106,9594443338063320145]], "path_len": 1}]}, + {"iota": 1926, "deep": [13364378999009406176,7107425205883074344,8183456523235029556], "deep_sym": [1636435348441089579,9805704461425670937,17419258600423616040], "terminal_position": 15, "layers": [{"layer": 0, "d": 1, "position": 1926, "leaf": 963, "slot": 0, "values": [[17308510152305227176,14537029458183837062,3374534082981073645]], "path_len": 7}, {"layer": 1, "d": 1, "position": 963, "leaf": 481, "slot": 1, "values": [[161046960075229799,15081533890008098182,8599524986623667909]], "path_len": 6}, {"layer": 2, "d": 1, "position": 481, "leaf": 240, "slot": 1, "values": [[13523182842471850292,10023096669923855615,13861557035808485449]], "path_len": 5}, {"layer": 3, "d": 1, "position": 240, "leaf": 120, "slot": 0, "values": [[67725569644445261,2660555931585193200,8656214761037478724]], "path_len": 4}, {"layer": 4, "d": 1, "position": 120, "leaf": 60, "slot": 0, "values": [[10483311268993759898,9100235776648696140,9649517420196481515]], "path_len": 3}, {"layer": 5, "d": 1, "position": 60, "leaf": 30, "slot": 0, "values": [[2870598012461783835,7827094732606362988,7524303457911972645]], "path_len": 2}, {"layer": 6, "d": 1, "position": 30, "leaf": 15, "slot": 0, "values": [[2109067029401775981,11881535591595910816,1594111885893068151]], "path_len": 1}]}, + {"iota": 618, "deep": [17392264061216667070,3799528080943413629,11407018515159519420], "deep_sym": [3459201148312316640,12192741989538265398,16701705627877756079], "terminal_position": 4, "layers": [{"layer": 0, "d": 1, "position": 618, "leaf": 309, "slot": 0, "values": [[6155497286543365943,6823874926095585913,8109198684617396853]], "path_len": 7}, {"layer": 1, "d": 1, "position": 309, "leaf": 154, "slot": 1, "values": [[9833782480490414257,17217623423597120964,8843356381310409960]], "path_len": 6}, {"layer": 2, "d": 1, "position": 154, "leaf": 77, "slot": 0, "values": [[2396282023618047124,5843578103453680384,12531555769585716035]], "path_len": 5}, {"layer": 3, "d": 1, "position": 77, "leaf": 38, "slot": 1, "values": [[2315015608088301356,11152573585322960408,1223035286229753124]], "path_len": 4}, {"layer": 4, "d": 1, "position": 38, "leaf": 19, "slot": 0, "values": [[9500029461340804850,2485568652985992693,10933239535849742952]], "path_len": 3}, {"layer": 5, "d": 1, "position": 19, "leaf": 9, "slot": 1, "values": [[3881399978602743849,14035452762582637444,9964587495007914030]], "path_len": 2}, {"layer": 6, "d": 1, "position": 9, "leaf": 4, "slot": 1, "values": [[18383902680402169760,11946899516366108295,4125966378742256522]], "path_len": 1}]}, + {"iota": 159, "deep": [10604883523135079191,13796358429189892774,13519594221824462179], "deep_sym": [6022801154996096846,1980184131037388799,13442421833478239751], "terminal_position": 1, "layers": [{"layer": 0, "d": 1, "position": 159, "leaf": 79, "slot": 1, "values": [[12670835830594729766,10775639007232550859,16555875757668836110]], "path_len": 7}, {"layer": 1, "d": 1, "position": 79, "leaf": 39, "slot": 1, "values": [[17928879845407862406,10337347125997172892,6291144022483388143]], "path_len": 6}, {"layer": 2, "d": 1, "position": 39, "leaf": 19, "slot": 1, "values": [[5290573282698993256,628791798829218958,8102047498213541200]], "path_len": 5}, {"layer": 3, "d": 1, "position": 19, "leaf": 9, "slot": 1, "values": [[14292313220916080740,6970106785239825796,6583204468013840496]], "path_len": 4}, {"layer": 4, "d": 1, "position": 9, "leaf": 4, "slot": 1, "values": [[7445378306973294798,9606155811494332299,13941390530992223098]], "path_len": 3}, {"layer": 5, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[9701830235220841955,11701359494893213657,2335524941435328481]], "path_len": 2}, {"layer": 6, "d": 1, "position": 2, "leaf": 1, "slot": 0, "values": [[9845217540333963803,15463176285714913034,4644416336041292088]], "path_len": 1}]}, + {"iota": 912, "deep": [12054840391066048689,3601865202668571108,3892056646934199431], "deep_sym": [9125201337264190457,16207456649903069037,12880097491155594601], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 912, "leaf": 456, "slot": 0, "values": [[2265230700843108553,17593609161732194274,15239094127825087222]], "path_len": 7}, {"layer": 1, "d": 1, "position": 456, "leaf": 228, "slot": 0, "values": [[16642757887568897673,12667640433037803234,3193412306162004453]], "path_len": 6}, {"layer": 2, "d": 1, "position": 228, "leaf": 114, "slot": 0, "values": [[13884150065805737304,15437553345903357492,13214756750175066224]], "path_len": 5}, {"layer": 3, "d": 1, "position": 114, "leaf": 57, "slot": 0, "values": [[11896301986580569790,4500452152629603038,11035825169199843994]], "path_len": 4}, {"layer": 4, "d": 1, "position": 57, "leaf": 28, "slot": 1, "values": [[1012511382225873659,15136067568327604338,60333472435469092]], "path_len": 3}, {"layer": 5, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[14606336359276572532,5481281866123149881,5272344691855823540]], "path_len": 2}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[4057849095198852267,11892082119611012572,10198841277104067590]], "path_len": 1}]}, + {"iota": 28, "deep": [8259937475034448900,5883617888360704157,13149344576568937951], "deep_sym": [15432868084718982482,15561794222120679975,9170450966169669817], "terminal_position": 0, "layers": [{"layer": 0, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[9242926421875877150,16140238903834851021,13531888265072917881]], "path_len": 7}, {"layer": 1, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[13133029412005626438,798748005302048131,7332222029041754348]], "path_len": 6}, {"layer": 2, "d": 1, "position": 7, "leaf": 3, "slot": 1, "values": [[7796322037150240186,9577838651761349973,16298966752722802932]], "path_len": 5}, {"layer": 3, "d": 1, "position": 3, "leaf": 1, "slot": 1, "values": [[4038697707906694945,2235019174556978195,15759609321762126709]], "path_len": 4}, {"layer": 4, "d": 1, "position": 1, "leaf": 0, "slot": 1, "values": [[14091206233662491151,9403345652412347071,15921683191799735380]], "path_len": 3}, {"layer": 5, "d": 1, "position": 0, "leaf": 0, "slot": 0, "values": [[13879050027862793750,770925931399928084,9161288321845839824]], "path_len": 2}, {"layer": 6, "d": 1, "position": 0, "leaf": 0, "slot": 0, "values": [[8305487573804742677,4779453755826958897,5416814198222686333]], "path_len": 1}]}, + {"iota": 76, "deep": [10920452299577596995,3320775998846492791,6849000225488292243], "deep_sym": [11707565976056054812,18291447239090459750,10377317711770383279], "terminal_position": 0, "layers": [{"layer": 0, "d": 1, "position": 76, "leaf": 38, "slot": 0, "values": [[15441279527880237572,16195703840399892975,4758470874282941851]], "path_len": 7}, {"layer": 1, "d": 1, "position": 38, "leaf": 19, "slot": 0, "values": [[18288674810117416457,5942604063344825953,4067135455677183442]], "path_len": 6}, {"layer": 2, "d": 1, "position": 19, "leaf": 9, "slot": 1, "values": [[3243931976597857557,14219957437372274067,4958358972487943150]], "path_len": 5}, {"layer": 3, "d": 1, "position": 9, "leaf": 4, "slot": 1, "values": [[6405106249199677879,10131651475896546166,13979395597501489022]], "path_len": 4}, {"layer": 4, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[15443546284012971304,2993634536769373283,2481852760355668878]], "path_len": 3}, {"layer": 5, "d": 1, "position": 2, "leaf": 1, "slot": 0, "values": [[3684463403749072493,11162129148197513032,14132169035515477628]], "path_len": 2}, {"layer": 6, "d": 1, "position": 1, "leaf": 0, "slot": 1, "values": [[18015477955632229458,1590473352291154340,1284851958473884494]], "path_len": 1}]}, + {"iota": 1379, "deep": [2671572208018711864,17938506104726169260,699081865341519986], "deep_sym": [1467098100466440450,9829414836620083259,10592222250402520868], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1379, "leaf": 689, "slot": 1, "values": [[14575599383640259218,8800880267594736215,13241772630433177501]], "path_len": 7}, {"layer": 1, "d": 1, "position": 689, "leaf": 344, "slot": 1, "values": [[1068698510447171772,13785811637582432776,16021446183151759205]], "path_len": 6}, {"layer": 2, "d": 1, "position": 344, "leaf": 172, "slot": 0, "values": [[927602389973061550,5073606438314814579,9869150991189971161]], "path_len": 5}, {"layer": 3, "d": 1, "position": 172, "leaf": 86, "slot": 0, "values": [[2535000398961545073,13768963704791829580,7399445715925562080]], "path_len": 4}, {"layer": 4, "d": 1, "position": 86, "leaf": 43, "slot": 0, "values": [[7943116901400220395,6371902111188986077,18264207171394083113]], "path_len": 3}, {"layer": 5, "d": 1, "position": 43, "leaf": 21, "slot": 1, "values": [[1152140764835361081,13898656790604124651,211509517521939152]], "path_len": 2}, {"layer": 6, "d": 1, "position": 21, "leaf": 10, "slot": 1, "values": [[2898297057086247105,8938194155188493164,3922699838444447823]], "path_len": 1}]}, + {"iota": 432, "deep": [15168271348910525142,14863991650496712335,564399768222653450], "deep_sym": [14646540013786724593,627304134268139825,7693608058989799918], "terminal_position": 3, "layers": [{"layer": 0, "d": 1, "position": 432, "leaf": 216, "slot": 0, "values": [[16945056781763873103,4147654905260621537,13070939633098599226]], "path_len": 7}, {"layer": 1, "d": 1, "position": 216, "leaf": 108, "slot": 0, "values": [[8711713063120759004,17111841543407718986,7409132053022715737]], "path_len": 6}, {"layer": 2, "d": 1, "position": 108, "leaf": 54, "slot": 0, "values": [[335306033794264840,3472897435351186058,4033837016810566210]], "path_len": 5}, {"layer": 3, "d": 1, "position": 54, "leaf": 27, "slot": 0, "values": [[7651149011250772184,9302481777241286440,9218801175418545890]], "path_len": 4}, {"layer": 4, "d": 1, "position": 27, "leaf": 13, "slot": 1, "values": [[6273227611468200768,2351133109562315987,1227054803639332633]], "path_len": 3}, {"layer": 5, "d": 1, "position": 13, "leaf": 6, "slot": 1, "values": [[6241052350319128640,8850536336423185920,15942149741970688151]], "path_len": 2}, {"layer": 6, "d": 1, "position": 6, "leaf": 3, "slot": 0, "values": [[9090592503262467466,8815962496836678004,4005637083568909720]], "path_len": 1}]}, + {"iota": 1032, "deep": [17851351312830372904,17617180244584954975,6387515032357752484], "deep_sym": [16265563244185578973,3252040868362477919,11341903784051896692], "terminal_position": 8, "layers": [{"layer": 0, "d": 1, "position": 1032, "leaf": 516, "slot": 0, "values": [[14093802161954164449,2714211322275968535,16260835473628978242]], "path_len": 7}, {"layer": 1, "d": 1, "position": 516, "leaf": 258, "slot": 0, "values": [[18009724111979905813,17719079489240485487,11317947430945448557]], "path_len": 6}, {"layer": 2, "d": 1, "position": 258, "leaf": 129, "slot": 0, "values": [[12575320568206317040,12724817382033075473,17343707112876650515]], "path_len": 5}, {"layer": 3, "d": 1, "position": 129, "leaf": 64, "slot": 1, "values": [[2468983175696377286,8123992486804046219,14998166175444204309]], "path_len": 4}, {"layer": 4, "d": 1, "position": 64, "leaf": 32, "slot": 0, "values": [[7088268260308644145,12018111465420124058,11385959261406757704]], "path_len": 3}, {"layer": 5, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[4971290189600716302,5514894773276055215,7098701689719309312]], "path_len": 2}, {"layer": 6, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[7935000193983825264,5386745176623997326,2689120037326345474]], "path_len": 1}]}, + {"iota": 526, "deep": [7523797964145835221,2350256745822342772,7495064266374662697], "deep_sym": [12443822411346913512,588359790533263303,15431128093166141371], "terminal_position": 4, "layers": [{"layer": 0, "d": 1, "position": 526, "leaf": 263, "slot": 0, "values": [[2778619449088084568,6700130457648041594,9906573914237896132]], "path_len": 7}, {"layer": 1, "d": 1, "position": 263, "leaf": 131, "slot": 1, "values": [[16517878130852354667,8206981990154853455,13534764593092745799]], "path_len": 6}, {"layer": 2, "d": 1, "position": 131, "leaf": 65, "slot": 1, "values": [[13861028275067157678,7774492653685014689,10405536925574021168]], "path_len": 5}, {"layer": 3, "d": 1, "position": 65, "leaf": 32, "slot": 1, "values": [[7637396778244847013,7916728431030253874,4151452972168897929]], "path_len": 4}, {"layer": 4, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[11389250702043873935,10610216047882982229,14335557185298532304]], "path_len": 3}, {"layer": 5, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[1827481070999240844,7018596498770465949,12683874916808890167]], "path_len": 2}, {"layer": 6, "d": 1, "position": 8, "leaf": 4, "slot": 0, "values": [[17263492077473791592,13925874335233374149,14333243844514772893]], "path_len": 1}]}, + {"iota": 929, "deep": [7209442172898103651,5078432396478215691,3359681651317667922], "deep_sym": [278577782173243961,2648608839113898350,14362352946213059321], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 929, "leaf": 464, "slot": 1, "values": [[742727014119487962,9415944698655895699,15901341720432948056]], "path_len": 7}, {"layer": 1, "d": 1, "position": 464, "leaf": 232, "slot": 0, "values": [[11887246137253338911,17471030057314561898,834604778466298478]], "path_len": 6}, {"layer": 2, "d": 1, "position": 232, "leaf": 116, "slot": 0, "values": [[2389324468080173975,14177420689333058099,3158032004849467555]], "path_len": 5}, {"layer": 3, "d": 1, "position": 116, "leaf": 58, "slot": 0, "values": [[11619263787659055028,17318414171778481053,10437342890333779898]], "path_len": 4}, {"layer": 4, "d": 1, "position": 58, "leaf": 29, "slot": 0, "values": [[3959405585248829475,10255018570515570189,7622928700457320856]], "path_len": 3}, {"layer": 5, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[14583500637061974005,3413989937573586254,7710889154554862151]], "path_len": 2}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[4057849095198852267,11892082119611012572,10198841277104067590]], "path_len": 1}]}, + {"iota": 147, "deep": [13192043606205898537,9469661584107672084,6315405556618292305], "deep_sym": [7642096387211884471,13644445525206900663,12919905892136611553], "terminal_position": 1, "layers": [{"layer": 0, "d": 1, "position": 147, "leaf": 73, "slot": 1, "values": [[5875338751136714530,7695228435662657364,5324885464763079482]], "path_len": 7}, {"layer": 1, "d": 1, "position": 73, "leaf": 36, "slot": 1, "values": [[12651956221626411650,9637806698413362635,7129211185802919037]], "path_len": 6}, {"layer": 2, "d": 1, "position": 36, "leaf": 18, "slot": 0, "values": [[6789928023159332903,8447447108763088136,17040077659500093070]], "path_len": 5}, {"layer": 3, "d": 1, "position": 18, "leaf": 9, "slot": 0, "values": [[15277897623289562052,8717440867787183431,5485634594337188074]], "path_len": 4}, {"layer": 4, "d": 1, "position": 9, "leaf": 4, "slot": 1, "values": [[7445378306973294798,9606155811494332299,13941390530992223098]], "path_len": 3}, {"layer": 5, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[9701830235220841955,11701359494893213657,2335524941435328481]], "path_len": 2}, {"layer": 6, "d": 1, "position": 2, "leaf": 1, "slot": 0, "values": [[9845217540333963803,15463176285714913034,4644416336041292088]], "path_len": 1}]}, + {"iota": 1839, "deep": [16378093384372275214,13015330254633996760,18205668585575265052], "deep_sym": [7837357816293187253,1010587672969474122,7673637645687776024], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1839, "leaf": 919, "slot": 1, "values": [[6124965313206069075,15411539721238865905,16260001499293568983]], "path_len": 7}, {"layer": 1, "d": 1, "position": 919, "leaf": 459, "slot": 1, "values": [[18150944016374367121,13957147611188270834,13296917984821951100]], "path_len": 6}, {"layer": 2, "d": 1, "position": 459, "leaf": 229, "slot": 1, "values": [[9313854673763300966,13537150988150843141,7627873754102692779]], "path_len": 5}, {"layer": 3, "d": 1, "position": 229, "leaf": 114, "slot": 1, "values": [[3800674278793119171,12770808243975540919,3800349180536817313]], "path_len": 4}, {"layer": 4, "d": 1, "position": 114, "leaf": 57, "slot": 0, "values": [[15928772226463361524,4352428081022084788,2641890282906734261]], "path_len": 3}, {"layer": 5, "d": 1, "position": 57, "leaf": 28, "slot": 1, "values": [[13689188977929574562,4258131260759228984,9939109160367792422]], "path_len": 2}, {"layer": 6, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[7786343213267754545,6056155651923690370,13889151246865202821]], "path_len": 1}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_pair.rkyv new file mode 100644 index 000000000..f0b663c10 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_cap_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp.json b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp.json new file mode 100644 index 000000000..aae1dee82 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "blake3", + "format": "dp", + "proof_rkyv": "d_proof_blake3_dp.rkyv", + "proof_rkyv_len": 8488, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 2], + "fri_roots": ["f5660c4333b6e611e901e87422b2c4270acfad24f631d9429831d76b54566517","0cd55ebeb840e8373096d7b45d7a99eb5f2ae89c0b700d618f5bc5e5cf7d8cac","eaed7db0665b821d2c690ad7c99f3fb3b28d4b7d3c6167be86e2033300fc3997"], + "zetas": [[10771210179622817679,127754635188287825,9592161990157892076],[2334636387541570725,4598538304975584359,12240732424901763132],[10166376440375277554,17046118386894069248,10115891851528829537],[1897382409627266702,6084356605560232122,6323818535469693028]], + "terminal_coeffs": [[5380735102582769720,14770520085343157731,17397790325610342738],[14177126935727096750,9878484623770692025,8126381307417814598],[13709110344626157024,14960543001611777495,13121995131109452668],[3543500174378306775,245990784589754978,17448449264639928647]], + "queries_detail": [ + {"iota": 975, "deep": [10762173397373278909,6238205991322615201,16902290430091080608], "deep_sym": [6360510169239840515,6098314158097471188,14374437857455028609], "terminal_position": 7, "layers": [{"layer": 0, "d": 3, "position": 975, "leaf": 121, "slot": 7, "values": [[6358380543480134188,15233297943481819331,7802884743754435287],[1561050269828331995,17056437354980486597,15953089478678981981],[9902832432796422005,16144145762744260912,2452053501664136327],[4059115819907980750,13622239393642520008,10742588081843022005],[15097201520147174657,13343912955236085051,15486897778505591961],[5267077696719908108,9464151356603282791,2320496645939400341],[7693143272069720023,17017576095617663956,6992176147234983386],[14242806506751524709,5590906401091982117,17893550137403079326]], "path_len": 8}, {"layer": 1, "d": 2, "position": 121, "leaf": 30, "slot": 1, "values": [[10075702410473013791,4716385726265313391,11609542590222699029],[7845467542947739937,12387174233603512715,18200019442985323599],[2909241347398984484,5399916910453173204,9233450494253184011],[4113688591137365584,8099000987494976281,456105366810755859]], "path_len": 6}, {"layer": 2, "d": 2, "position": 30, "leaf": 7, "slot": 2, "values": [[4720003309196990551,4178739593595040029,10411467881539262427],[17765747612192294497,14700500661535383226,13695608480138073067],[1009102207610472007,14305427383432249940,4131500240907956149],[1496298963912337677,3644564800854218514,5628674521221557415]], "path_len": 4}]}, + {"iota": 1979, "deep": [1491025643980379174,12685070184352261704,7728318385342818721], "deep_sym": [16450052277900778758,13025974084100681593,284476606938439535], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 1979, "leaf": 247, "slot": 3, "values": [[17193942932342370980,12145038434561791480,11576353455053528850],[8379852527282925844,8601836539934055187,15168362594496179266],[2592359005135839197,5381418174777500780,3896499938254073228],[4822332081114310830,2565510977245490832,11907903767149013019],[13285130888391799040,1041585227093494597,2304343024112292721],[11261822772218442511,17634015746885931060,3497632743424289560],[10018483860357178719,5407737136598053306,11038589506904123227],[5386981123451134746,15435103548476985734,14493978820809476719]], "path_len": 8}, {"layer": 1, "d": 2, "position": 247, "leaf": 61, "slot": 3, "values": [[16716297285756313772,4980386954389515672,7587525751122803270],[14964533687760416685,13917074670739361960,17216570321581969515],[3750454710447327212,10631398170004764756,11015661283393845074],[10023782474286288464,1360821076844805368,12167231288721989489]], "path_len": 6}, {"layer": 2, "d": 2, "position": 61, "leaf": 15, "slot": 1, "values": [[6218711529479866098,15946762367783592827,15439960344592751968],[13215040631267974591,7742956115152970799,9370753578504439872],[8789152590505293503,2216348788422376564,6920254999236655340],[702787882218669028,13764292175754416497,529455276546629621]], "path_len": 4}]}, + {"iota": 196, "deep": [12996823082221702228,15546436838001982955,12765624607176451818], "deep_sym": [15258355026629750431,13813035249993486719,14742394712154322144], "terminal_position": 1, "layers": [{"layer": 0, "d": 3, "position": 196, "leaf": 24, "slot": 4, "values": [[5990450875556600463,11483765027548679992,15569727055914393856],[15481225486576456763,6990687647448126227,17377462139297815371],[17823918767605952564,15408822346409695669,7766449979365244357],[10521282185009747890,10208004641233698759,9793502955518714526],[15989513811586569514,8058942065900320453,2895357368330783138],[12375312625287684078,17725753712340770465,14836143340177261245],[6076100021308446097,6158543389568230761,12040165962685290590],[12787237553493716673,6399667255546886261,10645808993947995036]], "path_len": 8}, {"layer": 1, "d": 2, "position": 24, "leaf": 6, "slot": 0, "values": [[3955724155215551649,4903104037739106676,8341497941364977219],[1366207781672916929,4653995748612992444,1985491773491753288],[12110069387491220579,3462604155932552453,12998143760810181545],[12022604030106382896,9872253098559971229,8337931158086605882]], "path_len": 6}, {"layer": 2, "d": 2, "position": 6, "leaf": 1, "slot": 2, "values": [[13763808492525116750,17048718856988468120,4007208303874274964],[9747356233047801788,12353956971904007004,16715523138327251637],[17215889828796017128,10441867300762877602,5539458827215095394],[16874552336957454881,3298743664346751734,7613930184659830057]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp.rkyv new file mode 100644 index 000000000..af336fd71 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp_3_1_3.json b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp_3_1_3.json new file mode 100644 index 000000000..5b2af63ec --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp_3_1_3.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "blake3", + "format": "dp_3_1_3", + "proof_rkyv": "d_proof_blake3_dp_3_1_3.rkyv", + "proof_rkyv_len": 8728, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 1, 3], + "fri_roots": ["f5660c4333b6e611e901e87422b2c4270acfad24f631d9429831d76b54566517","8f8497f1cc45d51572bb910d068306f0bfd0786175356863f7f078d4cdf4e6ba","559c0a403325bab03ff3d994f3f4bbb8bc5a5273ce96b19cfa3b3ce702096f79"], + "zetas": [[10771210179622817679,127754635188287825,9592161990157892076],[2334636387541570725,4598538304975584359,12240732424901763132],[1967510421138513926,14736776382609640613,1460789662965777522],[16830093636709684605,14842995202055722516,3914963795294019471]], + "terminal_coeffs": [[3331264495832828347,7185866664642756789,6843402643577021942],[10027383757451337894,9264328518027569750,618424366955010939],[8692021868639966632,14854503284949996751,16299654113095994525],[16358064282920315685,8517693515046071617,8244358201180006711]], + "queries_detail": [ + {"iota": 100, "deep": [15761165101399351880,9293892056917660698,13909387281546150815], "deep_sym": [18215445829741639441,18204034434822136671,3853351857208611712], "terminal_position": 0, "layers": [{"layer": 0, "d": 3, "position": 100, "leaf": 12, "slot": 4, "values": [[2664088888686076698,9363645576360161584,1094240937101656707],[2795005918076354239,7750515319260536800,16289535278568684750],[7828826467911276845,1956586071935190428,15259431314043971928],[7822913789832758503,9918120603487277767,12026915655054942547],[888266083852283926,5096366371443196859,15117111606629506813],[4632446112795977254,70591550309874064,2974240486951077444],[3263397032290487763,10334174333608144873,7959568761616464988],[3552368110883502870,10051530687251218433,1225942363435614861]], "path_len": 8}, {"layer": 1, "d": 1, "position": 12, "leaf": 6, "slot": 0, "values": [[15737476631448817853,12900508163676467004,1839840965848663120],[1924436232008221847,6241001949124084469,16747179164658090748]], "path_len": 7}, {"layer": 2, "d": 3, "position": 6, "leaf": 0, "slot": 6, "values": [[15641448562265420777,2088983744824930344,11939682928497748793],[736521859268441061,8342215319580479488,8897739578889100429],[2233749865829555840,7685479678190027881,7165376174653672864],[16112495396145500837,15368970251974031937,8711572005387377201],[10829235607656718845,15945783778988408022,6451476517960690463],[13004491108608034011,5760372999051763783,10788993322566782709],[867253151713547845,15943107042851967891,4251287764631266192],[15157411414371223400,10098736874879334226,12514777895591726498]], "path_len": 4}]}, + {"iota": 1086, "deep": [15608288023485154616,2200294644984832055,4541546713926039807], "deep_sym": [3079430625393683639,12119948572157099431,4967034397857673556], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1086, "leaf": 135, "slot": 6, "values": [[5064229977681024521,9535269840323459819,2145244677072055460],[125607644455879710,14506413514331503134,7702126688776868511],[12678478928085254970,4540996268990745755,14388757028488235130],[18401672303609607515,7662131759563109282,7660269693766476361],[12595928190669188977,1618490482153422770,8961555872666895518],[247968980481965489,17983378050067665214,4220288566350421147],[10455930571559153168,8083836495281389550,11606628425775345710],[6321202569249989985,13520813269599138260,9359837729431782857]], "path_len": 8}, {"layer": 1, "d": 1, "position": 135, "leaf": 67, "slot": 1, "values": [[9987387063165574134,10270232291887854585,7103905701655407856],[6084978948665007904,10991990918095191655,5571913575443446873]], "path_len": 7}, {"layer": 2, "d": 3, "position": 67, "leaf": 8, "slot": 3, "values": [[13495453314946811061,2343295106864069961,6620630672964439277],[10552127783637139629,3020177997249405451,7832607995418649242],[8174254051324615715,17948814487092268693,16221369621163145135],[15430193703939101357,9596189644948662926,6169730862756697992],[17705515929508746677,7532640930868421021,1201111313798040228],[4684151188798651217,1435893368118583475,17905397822166319127],[158248900632187591,7681052109159395530,11617423390658273269],[13135522550772402387,13249575094922739378,5732136554014601552]], "path_len": 4}]}, + {"iota": 53, "deep": [9529224066667257075,18203512621861259113,8451301895307292472], "deep_sym": [4759052315125095300,2509792134269440990,9502699147162584206], "terminal_position": 0, "layers": [{"layer": 0, "d": 3, "position": 53, "leaf": 6, "slot": 5, "values": [[6834949326104870826,11079296222041095752,10563898283978097910],[18016976558759513078,10014951360803540706,7578552812701626973],[7308646796660453164,16827564718672090573,3430734446912554275],[9366807165793692103,18212281866171309151,15746885233303123394],[12216624691318647156,17593898404433611439,4395950489549474014],[9621925250936315282,2015707641277960720,9413688144023119266],[13994221524705245228,2626300897160867166,5634372390616255857],[5369690143385904477,4543638345355920554,12365449364504512872]], "path_len": 8}, {"layer": 1, "d": 1, "position": 6, "leaf": 3, "slot": 0, "values": [[14698668472270914105,16176977995025063834,18253104268403247552],[5896354434492429616,9697870880870630454,4609762693976853786]], "path_len": 7}, {"layer": 2, "d": 3, "position": 3, "leaf": 0, "slot": 3, "values": [[15641448562265420777,2088983744824930344,11939682928497748793],[736521859268441061,8342215319580479488,8897739578889100429],[2233749865829555840,7685479678190027881,7165376174653672864],[16112495396145500837,15368970251974031937,8711572005387377201],[10829235607656718845,15945783778988408022,6451476517960690463],[13004491108608034011,5760372999051763783,10788993322566782709],[867253151713547845,15943107042851967891,4251287764631266192],[15157411414371223400,10098736874879334226,12514777895591726498]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp_3_1_3.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp_3_1_3.rkyv new file mode 100644 index 000000000..225fc63b3 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_dp_3_1_3.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_pair.json b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_pair.json new file mode 100644 index 000000000..6484a9238 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_pair.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "blake3", + "format": "pair", + "proof_rkyv": "d_proof_blake3_pair.rkyv", + "proof_rkyv_len": 11136, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": true, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["2fc983d7a9f8dba305332d7c27f44e2afc07aa1fe014ad8a85ca9bc36844a992","a01be93f245807d30fde826335bc7dd8bfbf9fb0545769f85da9a6ee2b564301","4cc8bdb5c5d436e8b5cc91aa230573d630e9eb8a908f2650f9d4598e3d731b65","82de5d8f879bb994fc9573a6c3706b71adaf8b236e17ff2047e44bfa64f1e480","779e1acea2c1b391312f39412312a9a6bcb8cf5a66e1954d2b9d340a74a74bd0","191d0d55f0bf47af108196ca3b2c067e667807e78ae6681a814b3be0a8f4e445","878efaffc3ca3b900bd232b64cdd6a142203e2ef0ff1cda3f52ec590b776f071"], + "zetas": [[10771210179622817679,127754635188287825,9592161990157892076],[339236561547217708,14515476371055385421,3041135081988152589],[7430745936816588155,8998042728974583901,11515773416551488605],[7956826836586454026,8667292109104632665,2851244499340860067],[16324173539864659489,11301157219502799655,18016560099956879839],[4272458413724263223,15273501817168123109,13432776003642703715],[18153136978195245525,4668271491129789573,15852649611975035906],[13206066232974685659,15811531208029248608,9742874826372310642]], + "terminal_coeffs": [[11908419985256297049,6320124696091700849,10477651950916658009],[16710003718284845920,14728440137509904251,12073313539240356766],[15142905694919717110,8656948196775444897,1363513317241862160],[2198207007945388790,2708142890943514224,17003186495140238478]], + "queries_detail": [ + {"iota": 1803, "deep": [15272426180920759111,5106447191221278975,14792296330971372023], "deep_sym": [10304415851256192438,7276545599604954905,12402529132092837573], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1803, "leaf": 901, "slot": 1, "values": [[3388282554933400969,3823175679916949076,2787870681482871753]], "path_len": 10}, {"layer": 1, "d": 1, "position": 901, "leaf": 450, "slot": 1, "values": [[17941501397892820289,9746070180589186316,4483120140038292319]], "path_len": 9}, {"layer": 2, "d": 1, "position": 450, "leaf": 225, "slot": 0, "values": [[1518365534971821388,8220153128022570539,11364526563819683345]], "path_len": 8}, {"layer": 3, "d": 1, "position": 225, "leaf": 112, "slot": 1, "values": [[10971480354833343982,3135816652628770915,6720283715471365573]], "path_len": 7}, {"layer": 4, "d": 1, "position": 112, "leaf": 56, "slot": 0, "values": [[5709537754561370510,10236832031319039769,1874314679153150939]], "path_len": 6}, {"layer": 5, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[2528382099678622252,12218130109821183716,1136296192569704372]], "path_len": 5}, {"layer": 6, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[7786343213267754545,6056155651923690370,13889151246865202821]], "path_len": 4}]}, + {"iota": 474, "deep": [4642716204571870719,13791353321977000304,9948795077040124575], "deep_sym": [10074490863165540107,1346332627183725457,15559140971681542809], "terminal_position": 3, "layers": [{"layer": 0, "d": 1, "position": 474, "leaf": 237, "slot": 0, "values": [[9803582068471756145,5326669840186105035,7793279955894935834]], "path_len": 10}, {"layer": 1, "d": 1, "position": 237, "leaf": 118, "slot": 1, "values": [[18172681946601424763,4149543359487769368,2150741210857753378]], "path_len": 9}, {"layer": 2, "d": 1, "position": 118, "leaf": 59, "slot": 0, "values": [[10054490575191079786,12193424298068301071,8417982262482120641]], "path_len": 8}, {"layer": 3, "d": 1, "position": 59, "leaf": 29, "slot": 1, "values": [[15947123418199701165,18407774728151935281,3292539258646734529]], "path_len": 7}, {"layer": 4, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[3117529940052833834,10473131964376682009,2083760568833245811]], "path_len": 6}, {"layer": 5, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[7713732352748329805,3614247649513246873,888672929281612740]], "path_len": 5}, {"layer": 6, "d": 1, "position": 7, "leaf": 3, "slot": 1, "values": [[12707951974262439387,10229714375229842447,6079425424868885692]], "path_len": 4}]}, + {"iota": 1018, "deep": [15882578000804364217,17570699945731153943,17271573467219472049], "deep_sym": [11776097457111120055,8466990234121688300,9890187330955688279], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 1018, "leaf": 509, "slot": 0, "values": [[11811277273608028663,11557005519804590428,6147111063572348711]], "path_len": 10}, {"layer": 1, "d": 1, "position": 509, "leaf": 254, "slot": 1, "values": [[7939867388228535545,10890533442334944369,9121900079366539214]], "path_len": 9}, {"layer": 2, "d": 1, "position": 254, "leaf": 127, "slot": 0, "values": [[432090123891713462,15713972828822391493,11186642764496342828]], "path_len": 8}, {"layer": 3, "d": 1, "position": 127, "leaf": 63, "slot": 1, "values": [[1293317382852890727,12037476111710244625,8463877166491912968]], "path_len": 7}, {"layer": 4, "d": 1, "position": 63, "leaf": 31, "slot": 1, "values": [[14282184254670867115,4596231514897671604,8263298406545493773]], "path_len": 6}, {"layer": 5, "d": 1, "position": 31, "leaf": 15, "slot": 1, "values": [[4047442787689190383,14125010312351736105,11271388519733766106]], "path_len": 5}, {"layer": 6, "d": 1, "position": 15, "leaf": 7, "slot": 1, "values": [[5728710831141537085,4396671778989160837,8614177465654515251]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_pair.rkyv new file mode 100644 index 000000000..3aae6b0f8 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_blake3_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_dp.json b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_dp.json new file mode 100644 index 000000000..e51187926 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_dp.json @@ -0,0 +1,49 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "keccak", + "format": "cap_dp", + "proof_rkyv": "d_proof_keccak_cap_dp.rkyv", + "proof_rkyv_len": 41480, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 20, + "grinding_factor": 0, + "coset_offset": 3, + "merkle_cap": "auto", + "trace_tree_depth": 11, + "trace_cap": 3, + "fri_tree_depths": [8, 6, 4], + "fri_caps": [3, 3, 3], + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 2], + "fri_roots": ["24ad3d0e98f4bed6edf18ec793157a4b40d412b869a719de1c98a970fab00072","f9fe88a494b4bb9e5ca08fc1c1a7a3ef4624b75d0ad3812e39dc65d9020f91cf","7ecf9321963996332fa2eb0464dd78c32efb4185fbc866687520f36de7764131"], + "zetas": [[5019159632337129269,238091556992722228,5532889084085155677],[12296403571495774788,9626523507187974856,1515890197535251952],[9104931154505306807,6806930774857449431,13982536486847686418],[17943736705802395901,4283444887199783601,5105112647180456117]], + "terminal_coeffs": [[18046538310705593629,16035634115336395623,14269474772235161333],[13449754012599068599,8932449597508197521,3279495531022860796],[11948801525571458678,1807139812678879355,3178944376615033389],[11072843192958306056,3667138469373329065,14070513692562577743]], + "queries_detail": [ + {"iota": 1277, "deep": [112612903969624832,13540544077113206977,891744669294414204], "deep_sym": [10665692780904921752,13891743997545272459,5218021303841191956], "terminal_position": 9, "layers": [{"layer": 0, "d": 3, "position": 1277, "leaf": 159, "slot": 5, "values": [[2726840187197314970,4641373057133563422,18254905628294267124],[7726190489312153580,9907582621009652564,13704195924065075984],[4913854101341609490,7059003433635310965,6314417828660586086],[15329881112488297229,17154024704563340256,10996086559637584958],[8931011741374043492,14857842271836329185,1962274052252210912],[11335680486698327443,1217890136881310458,6960827381617415085],[11843024897136178316,4050095544328259531,1109189699536974526],[15465183123903289453,8756197396528546255,3770807126986676922]], "path_len": 13}, {"layer": 1, "d": 2, "position": 159, "leaf": 39, "slot": 3, "values": [[9337955700188682368,10005268201090501927,17075626829468745589],[12775344777395238092,12444988312381492194,18162313775388685340],[12834611725527989937,3931095319124210104,7011958104454824522],[18278843176886412077,2091177023787081796,10712499925409758781]], "path_len": 11}, {"layer": 2, "d": 2, "position": 39, "leaf": 9, "slot": 3, "values": [[12663636275089871938,1342734324200714786,13647156802297113741],[16459715330172958447,16246821789525783433,13803231028510688298],[4472259574895772221,15705768718567917064,4738154395575758232],[10184880754128237084,3408521813484574087,14812129773919197844]], "path_len": 9}]}, + {"iota": 1793, "deep": [8057175728474570347,4157164488656378128,15766577891820220836], "deep_sym": [6733030217476856996,3149008183846048310,5846868056871306014], "terminal_position": 14, "layers": [{"layer": 0, "d": 3, "position": 1793, "leaf": 224, "slot": 1, "values": [[5863889590658237167,8803207495494391631,488510412724115696],[7367902939689275966,5399515143439789253,13537028177165637670],[13057594490447211533,12028941489541574294,10245716700381823303],[3536160573392264847,13647402147340435120,9933763201558099138],[17705005489971962397,16100850492966888022,3356205035428066804],[4834413239841014089,10648175143241294336,14941339194282038433],[11268069224352915944,7397295511095760171,650865519941991105],[17810125080425025549,8252558882871738031,1603536863803495337]], "path_len": 5}, {"layer": 1, "d": 2, "position": 224, "leaf": 56, "slot": 0, "values": [[14576290996393278046,4099269296443923918,13962179114375143747],[8539012341704406611,14597685688217769420,16489000745330409389],[4928849784411569862,5656061150696874101,18052668495466081830],[10699077948197816254,7120867110842505641,2470038313983831606]], "path_len": 3}, {"layer": 2, "d": 2, "position": 56, "leaf": 14, "slot": 0, "values": [[7743892560805052942,10417724895478695146,9242061460595868046],[5215961958705134614,3646588380176163324,11215186743127464548],[18165082680919870330,16446510594026310692,9931249060720003450],[16761232818563256745,6094664995607304883,14831579372437454855]], "path_len": 1}]}, + {"iota": 1422, "deep": [4336444987633806031,42270359695066150,811124501724833250], "deep_sym": [6040322601513087150,2232031154133685564,13268270931765776955], "terminal_position": 11, "layers": [{"layer": 0, "d": 3, "position": 1422, "leaf": 177, "slot": 6, "values": [[5773020228763106950,12181432689259931341,2904380769668095371],[5378436167230488318,1136926564836430281,11025181981762941864],[9767397875213699870,16391873535337268069,9544088588384136146],[50456808154105685,7570275210936766391,3076092320148066703],[6519022523009943654,14501422860440411207,16766709789063948727],[1942043923567112082,9396051082847161748,4275006641168421309],[802135155222683305,8086721014210384187,5276472197522953276],[14963808776084644130,11822327586546991308,9902819457375193080]], "path_len": 5}, {"layer": 1, "d": 2, "position": 177, "leaf": 44, "slot": 1, "values": [[8007252311096823131,15451587500561065094,5200475833640745404],[12419273660987748398,619789569423171010,4299596803633862803],[14489492344890871493,14652990201720622453,5263973935492910147],[12959859661416681018,3696933911172366326,18087484035368403648]], "path_len": 3}, {"layer": 2, "d": 2, "position": 44, "leaf": 11, "slot": 0, "values": [[5556470237486890782,15738386834927433074,12010912686098111476],[1997630762550526785,7678738670208248417,194037932413528879],[7091997090193394797,1911130281305530368,8017953523793910594],[1805231709436512031,5522280617529416910,4194339594951184587]], "path_len": 1}]}, + {"iota": 375, "deep": [15896026706216286558,850738027208789055,10723639171146949965], "deep_sym": [3646273813276642761,2241354158260357600,16568479433952979865], "terminal_position": 2, "layers": [{"layer": 0, "d": 3, "position": 375, "leaf": 46, "slot": 7, "values": [[15387740746947457604,13589966986149538764,14032877163179148290],[13708225577102587055,13441481804739059493,8991395910890718293],[10414453853530795658,15349599265905617934,9809656972176258562],[13902200104895237219,8980123068515626615,17539094337944877822],[18430868960253363076,10447808943170909240,152854829625985981],[9586311282657541841,11963572983397487123,15423990981040471599],[8970700645449704597,438050523969751745,14841344594967333290],[54692778579574998,13160092994989063843,16827559681417886436]], "path_len": 5}, {"layer": 1, "d": 2, "position": 46, "leaf": 11, "slot": 2, "values": [[10482637989529479610,11046049345170984111,16611894623477967708],[17524538985625388400,3561755534548238727,12316349888672143316],[18242276635503340986,2708203015724362019,5591280696094092762],[6520070575385940768,12335650810165131772,14579374211771389454]], "path_len": 3}, {"layer": 2, "d": 2, "position": 11, "leaf": 2, "slot": 3, "values": [[7120644982042697520,4877211885227668924,17408917161652445778],[10118435938790577823,5444915436844861924,11542569338792872830],[157835802203199106,17140088994041704553,3978505728255765620],[14623016493704661158,10884525528551030981,1014977751818009979]], "path_len": 1}]}, + {"iota": 1948, "deep": [7087937129631102186,10324887181174666606,10384177212640173098], "deep_sym": [4839108783067615636,16148817134865123179,14412947888881988463], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 1948, "leaf": 243, "slot": 4, "values": [[4384664434476341418,4109755903884089713,448848679048843676],[502577095427712111,1950110303512630307,2083947009798347444],[13138990615390871461,16250161617582919998,9900591626273979634],[8185137308944158747,11670104219697834910,4945466864899594101],[8246417461024835979,11435835163928684321,12443831336801837744],[14595002883778732256,3549608309749680403,9674226137969631683],[2538087213630520003,9729540931449938752,14042293170573545],[17478515448816995850,3847752760399403901,1279474917978074079]], "path_len": 5}, {"layer": 1, "d": 2, "position": 243, "leaf": 60, "slot": 3, "values": [[11375860614512743705,12159552691638095048,1748577297743065506],[10427635604250192507,12188377611915858990,3305939877387789858],[5427071017873339313,12464684671386940995,2097029895587251782],[5305312732620219072,1636602398709148096,9970069024630049051]], "path_len": 3}, {"layer": 2, "d": 2, "position": 60, "leaf": 15, "slot": 0, "values": [[16083923945252889347,14007201087796689962,9760068277015825398],[13908085417078852557,6940036306507102360,12381427201366074798],[17699679852100348611,14554545952620420176,15525485196160550265],[3320877808744335198,16258308082275368886,16540132530644722341]], "path_len": 1}]}, + {"iota": 1966, "deep": [2737464584824995267,14918683733418229385,7196299024600452041], "deep_sym": [18370036556571597981,1830788502375962850,10934794948443612497], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 1966, "leaf": 245, "slot": 6, "values": [[13671960172398405357,17855080896113870507,9364618645616150458],[17414960211254073904,14038003153076967146,14805527776041656840],[6673455254303213924,17167822529254968482,6338703889954748273],[817936940740959714,13256059908467474472,4760748120011860765],[16094771459087701234,653221362292145817,15615540196276244089],[16454431268210406849,9591779889310584976,14000251994194405957],[16611363856511027300,5707903787206279761,3321883520519277902],[17140913678660095447,17794956949402229996,1697394557625431157]], "path_len": 5}, {"layer": 1, "d": 2, "position": 245, "leaf": 61, "slot": 1, "values": [[14919330226061001544,7003661483458825842,4258396156973505816],[1749975506220495240,14492956044198192519,2252471877392573853],[10278317655546946839,2718392541606326429,7276295292726873270],[6742827957320313011,989165657890668514,10537792207525539025]], "path_len": 3}, {"layer": 2, "d": 2, "position": 61, "leaf": 15, "slot": 1, "values": [[16083923945252889347,14007201087796689962,9760068277015825398],[13908085417078852557,6940036306507102360,12381427201366074798],[17699679852100348611,14554545952620420176,15525485196160550265],[3320877808744335198,16258308082275368886,16540132530644722341]], "path_len": 1}]}, + {"iota": 1057, "deep": [288064885883882665,10890893893344733623,4388390573256310456], "deep_sym": [77447489977792414,17008457381051219699,8203417707137847288], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1057, "leaf": 132, "slot": 1, "values": [[14602002417936541588,2060133397734822291,3872298068558041525],[16644565893843641095,6451828667327073839,5933463085769248929],[3093170179341351174,16475110652552847864,4116847028819369612],[7455182260930115447,9256567728239627584,10404107984679348218],[462894392829998510,12545489967056621090,9716019685493474827],[5141239682319935027,12773795560778526585,11627811492010847833],[16920552392970673286,12668306819870810712,13126806710171727182],[8142891815470815835,9396478658697097228,12628929348098678933]], "path_len": 5}, {"layer": 1, "d": 2, "position": 132, "leaf": 33, "slot": 0, "values": [[6762027819238736502,1994817665773640169,12308975809811156353],[430727574191951338,6595710101744551501,9974788267850124432],[10843514598889494219,10953618560938278742,4109801507007785601],[14297383862207660850,4695764767127474585,15066808880087009327]], "path_len": 3}, {"layer": 2, "d": 2, "position": 33, "leaf": 8, "slot": 1, "values": [[11758409758221297221,11034734048298619314,14549108557531584252],[8435201558561332969,8784175351312936877,16456903953760264513],[15792142755751630193,13249956046916067676,3979910583159107465],[10326060926810026901,6731236006246528468,9917621895543682894]], "path_len": 1}]}, + {"iota": 1649, "deep": [16113310391862898080,3750040675083148501,18062999865878766004], "deep_sym": [14118473159812533224,8837649250340416771,4486451966545117618], "terminal_position": 12, "layers": [{"layer": 0, "d": 3, "position": 1649, "leaf": 206, "slot": 1, "values": [[23773808765211251,2724987214266210270,10363574896205763514],[1063447907708073502,16993201402556864939,6216290352654158679],[13902131533558810019,8219699955683922369,5524029970275705845],[9663368849740022629,6328211828585780341,16135481082943213812],[13309480709205613563,13201964663308045097,15012894330470548828],[7952710722453649074,13361594031191679724,2551826062919144587],[2190871277041262405,14972906233227189104,13243835470767713833],[16257674481805524096,12933708557913758180,17717332288188416810]], "path_len": 5}, {"layer": 1, "d": 2, "position": 206, "leaf": 51, "slot": 2, "values": [[5012420639169771645,8546165005815674507,12115096682090388454],[1313249665614203061,11495183200823441072,7479279848115399142],[8563943629740366687,3146950596701191193,16306076291276600083],[11544006396789333386,7474851117767078503,14971448517887911011]], "path_len": 3}, {"layer": 2, "d": 2, "position": 51, "leaf": 12, "slot": 3, "values": [[1545572393754741841,11690420781243035314,9918423316728445502],[11627290908117424199,6051342473574127770,9134970903777327367],[8604029722356773913,15305165503694345878,7930243008777305646],[7998741184655164490,4603484846085525689,5714420271927771819]], "path_len": 1}]}, + {"iota": 1030, "deep": [14373800732940166725,18430465542586899752,3776694530884561141], "deep_sym": [7463758950313777458,5528294336202769938,10635152219191787810], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1030, "leaf": 128, "slot": 6, "values": [[5780659269726964115,15217834712966871504,15901731355578075927],[3191285795483457442,13926700417060571460,14330808147944740432],[13795075764902119338,6281162078196555170,6359184787760896428],[10558246965775800911,3982131670900263214,14050992306146354219],[2516726934071551986,3136147731612096946,14474217385552349651],[14293016964834423850,14204722566823234558,349167635719564975],[13167235163881118400,12264236181888999124,16205070808867739650],[11049080656783136961,10255632632760224697,17530910159094919662]], "path_len": 5}, {"layer": 1, "d": 2, "position": 128, "leaf": 32, "slot": 0, "values": [[1603596091347416547,1685572709416834966,14131289672992784683],[16630113299352830035,4130651607240740485,3953453261651720240],[12833920182305139813,5357683431524468303,17512826818408810918],[10196527414059609335,2811204653129765191,3987710025744114355]], "path_len": 3}, {"layer": 2, "d": 2, "position": 32, "leaf": 8, "slot": 0, "values": [[11758409758221297221,11034734048298619314,14549108557531584252],[8435201558561332969,8784175351312936877,16456903953760264513],[15792142755751630193,13249956046916067676,3979910583159107465],[10326060926810026901,6731236006246528468,9917621895543682894]], "path_len": 1}]}, + {"iota": 282, "deep": [4605030828346954542,10107141483819085453,16340363278551917239], "deep_sym": [3153592424307667625,10703831073067631050,3420493377982575506], "terminal_position": 2, "layers": [{"layer": 0, "d": 3, "position": 282, "leaf": 35, "slot": 2, "values": [[16351637314632867303,10550715376056936204,389777731531914906],[5423756822712339682,3207707814678981159,14554301294689068832],[9522977257558972583,1237680911860807845,5581983543124725091],[18155213215714573372,15753220935196394414,15413435704086206572],[17467711387504357770,12918792077398046747,13177021707668672818],[10384275476170871726,17189159971389164274,4738120100226236541],[2016889463784941813,15496084267640314674,4377476686201105459],[11044917781013069171,5759042811678180522,8832858058718560393]], "path_len": 5}, {"layer": 1, "d": 2, "position": 35, "leaf": 8, "slot": 3, "values": [[3773095292795911748,296981318361970991,3709827473597750432],[9579271102479213590,9301751476218455612,2459672023739543089],[16953705104003922674,10560831490120226302,12377739364455719677],[17933522243182263073,9435594104570083871,15300526250364715780]], "path_len": 3}, {"layer": 2, "d": 2, "position": 8, "leaf": 2, "slot": 0, "values": [[7120644982042697520,4877211885227668924,17408917161652445778],[10118435938790577823,5444915436844861924,11542569338792872830],[157835802203199106,17140088994041704553,3978505728255765620],[14623016493704661158,10884525528551030981,1014977751818009979]], "path_len": 1}]}, + {"iota": 1941, "deep": [9440125841541173544,15658990514951940362,9945303899609953144], "deep_sym": [2216207227382168010,1020647313760285429,17493691611512355197], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 1941, "leaf": 242, "slot": 5, "values": [[1844732848412660250,4602035078406182240,18209111516444714084],[1788494049217962277,7927061198967915720,4534818919883673538],[16061940833143770472,10191170167737465651,16698057848562098452],[13677653368054703338,2015726882419666285,1181093536994502902],[13518204986550635413,17541492429993064125,12452400951551991409],[6813945670717812094,4826602370772664987,2554812707018042294],[6452100949112115734,2943030424053276291,18381591199498086949],[16888214740738548397,9551065487491227380,3842611139925254562]], "path_len": 5}, {"layer": 1, "d": 2, "position": 242, "leaf": 60, "slot": 2, "values": [[11375860614512743705,12159552691638095048,1748577297743065506],[10427635604250192507,12188377611915858990,3305939877387789858],[5427071017873339313,12464684671386940995,2097029895587251782],[5305312732620219072,1636602398709148096,9970069024630049051]], "path_len": 3}, {"layer": 2, "d": 2, "position": 60, "leaf": 15, "slot": 0, "values": [[16083923945252889347,14007201087796689962,9760068277015825398],[13908085417078852557,6940036306507102360,12381427201366074798],[17699679852100348611,14554545952620420176,15525485196160550265],[3320877808744335198,16258308082275368886,16540132530644722341]], "path_len": 1}]}, + {"iota": 728, "deep": [13153405618339387285,12087014183745369343,7112360601826025321], "deep_sym": [6731259745018423611,2389294347923787576,8183626695115640159], "terminal_position": 5, "layers": [{"layer": 0, "d": 3, "position": 728, "leaf": 91, "slot": 0, "values": [[4548103059600255688,13178230568439787764,3041699253933608412],[4457817450677572724,7643960018359720291,12243487772429543665],[12640410128497954957,17663777625894295479,16121535441378525388],[5867807913173865683,15979217689236923364,1430096963540407779],[14787506865643198218,5829300918005392054,7263572657707494447],[10220333170775435288,10906980106043563821,9691813031194362081],[9122624667692505876,8929466534834669539,1812126731322561044],[18283235548136170389,15471761571546707919,12216120490171808248]], "path_len": 5}, {"layer": 1, "d": 2, "position": 91, "leaf": 22, "slot": 3, "values": [[16496262089917335101,16848297095433487741,3667071413045078735],[4701350522645074997,6137785806070879134,6782802506336928605],[17016004655611056354,4792727985778759853,17352960297403153524],[3506807141796159313,3285735666660479462,10512393962352017174]], "path_len": 3}, {"layer": 2, "d": 2, "position": 22, "leaf": 5, "slot": 2, "values": [[7191562821665302979,16440975359403640357,805687569336425751],[99473289543227136,7658450705401826751,1775111614621733606],[3938272754229468231,13802545678870326785,10553018281064249974],[5837029391332631245,3021110496026535726,17844415609475040928]], "path_len": 1}]}, + {"iota": 1480, "deep": [9421862934060914690,1734270806357642127,233875232532598714], "deep_sym": [18117459245225845776,647221298613096286,10640037095774592376], "terminal_position": 11, "layers": [{"layer": 0, "d": 3, "position": 1480, "leaf": 185, "slot": 0, "values": [[13198700555948538666,664034775197733983,17957513155068905625],[2048503957988366328,9846960232985635947,13572178912083388628],[16233241902330938996,8784515565338695415,18070637485116033776],[18303603161390467571,17564789126088883463,9181958520784428546],[1097773023732610587,1700095744687457468,13923523307874957244],[6452537530029647754,760433398807241482,8147876834946939403],[2552498477488711372,6077095896422145430,3139202260933438747],[8831062329706968515,1468344329142906895,4596941720011238461]], "path_len": 5}, {"layer": 1, "d": 2, "position": 185, "leaf": 46, "slot": 1, "values": [[11479654887262673782,16879657354497066704,13493609808166566252],[9227553114284958970,268322353749174939,15955048230739207146],[15251812456265475774,4859854400533217022,17327573994810098785],[5937937289982444485,17224265450150211107,14220481255969979395]], "path_len": 3}, {"layer": 2, "d": 2, "position": 46, "leaf": 11, "slot": 2, "values": [[5556470237486890782,15738386834927433074,12010912686098111476],[1997630762550526785,7678738670208248417,194037932413528879],[7091997090193394797,1911130281305530368,8017953523793910594],[1805231709436512031,5522280617529416910,4194339594951184587]], "path_len": 1}]}, + {"iota": 290, "deep": [14021209778763882017,13506671047373952976,13392681072178172499], "deep_sym": [11086591086110335323,259671578182683537,18045322574659161142], "terminal_position": 2, "layers": [{"layer": 0, "d": 3, "position": 290, "leaf": 36, "slot": 2, "values": [[7261845420047701776,11246462243179677035,7702811478875914330],[2501725647556803489,1263325614023439244,618889748304473690],[14900655738542057542,17891011431003616839,17575788078977955023],[9759952812872127999,2959573722831254786,11962249271050766536],[8837517616108370065,8909991979831591490,14572195230786394145],[3413523119546850389,10719675103647747609,10771843755320173053],[16068304745558270030,15349103431007750612,15888609516370901269],[12163776512755042098,8738806965462894074,2933481754801473916]], "path_len": 5}, {"layer": 1, "d": 2, "position": 36, "leaf": 9, "slot": 0, "values": [[9665548463325373214,13945489773253237666,11177605171166059499],[5997090037924425191,17204975125146622508,503183837706051239],[2316026578557707879,16320620904918007181,17497979472925238826],[13879387828100917678,3163214737154467295,4711748701448570178]], "path_len": 3}, {"layer": 2, "d": 2, "position": 9, "leaf": 2, "slot": 1, "values": [[7120644982042697520,4877211885227668924,17408917161652445778],[10118435938790577823,5444915436844861924,11542569338792872830],[157835802203199106,17140088994041704553,3978505728255765620],[14623016493704661158,10884525528551030981,1014977751818009979]], "path_len": 1}]}, + {"iota": 661, "deep": [6854765970215394823,2556446177852765810,15642530413576195024], "deep_sym": [11839673598506880492,12298724903716565425,17867957635740925997], "terminal_position": 5, "layers": [{"layer": 0, "d": 3, "position": 661, "leaf": 82, "slot": 5, "values": [[7496922916330212072,12338475639414754372,17434511279595495729],[12245166466801611098,11133729606825314606,16531306043264518426],[564339237851429615,5203048796060508002,4430341536129038167],[5147401255457999382,9931580009307097849,8744002423214609744],[10491831961067514093,18103125205776967329,8175823406417691246],[11064366642819303520,13175449112774951959,15394722024297982774],[9273040857889525487,8156191332710451250,6058357687441996617],[15172984517813136604,572230973827870853,2578811246222104697]], "path_len": 5}, {"layer": 1, "d": 2, "position": 82, "leaf": 20, "slot": 2, "values": [[13594173089496239018,16757524473770480938,2194449842399469486],[7693636871554533860,18228379298759503779,7255998013993585144],[6003272190433350033,16403086524442062476,3157252260914019799],[6560541927535846002,8658991620623781167,52645927622895508]], "path_len": 3}, {"layer": 2, "d": 2, "position": 20, "leaf": 5, "slot": 0, "values": [[7191562821665302979,16440975359403640357,805687569336425751],[99473289543227136,7658450705401826751,1775111614621733606],[3938272754229468231,13802545678870326785,10553018281064249974],[5837029391332631245,3021110496026535726,17844415609475040928]], "path_len": 1}]}, + {"iota": 1763, "deep": [16676350833658941157,2584692057630204480,7582912906866648961], "deep_sym": [4095891502055755185,13892816713210296784,7051523027024881040], "terminal_position": 13, "layers": [{"layer": 0, "d": 3, "position": 1763, "leaf": 220, "slot": 3, "values": [[16326594147024038562,9586321346049704049,16274016159125623699],[9684549243598334032,17409814664793682726,12541896833276119385],[17641608400031098675,2253078537857552057,15121822299814424285],[7632477760011931609,17323557877642753933,6270578313288524675],[8888728255851738386,5448327313361066048,8657660858889133604],[8906225553342660315,4039383537123350997,4361862025659933030],[12059143965033852868,12077620965086179016,1717778924363437527],[8645643362090899969,2453455417848760733,6065353499642801339]], "path_len": 5}, {"layer": 1, "d": 2, "position": 220, "leaf": 55, "slot": 0, "values": [[11167732534386473640,3688647619183319927,6771087108562321391],[2756854838210769859,11731540172240008448,14994311570837748550],[12460925440629396509,8895390108817935301,4433031690687849429],[6736179714496515355,8860053820523315526,1155063141492321859]], "path_len": 3}, {"layer": 2, "d": 2, "position": 55, "leaf": 13, "slot": 3, "values": [[1950472547524004311,4558894411088780101,2566116983780004090],[9850772263745460440,12240919944799806857,9423785085799841200],[10115571518972942631,891027316670140863,3962163664572049993],[12458991981252355554,11912590518631871559,4112952399935969276]], "path_len": 1}]}, + {"iota": 1459, "deep": [8866747437831929030,10111830668002232419,9891926968670703598], "deep_sym": [14950462972542314436,11068150428568742217,4363920990391254045], "terminal_position": 11, "layers": [{"layer": 0, "d": 3, "position": 1459, "leaf": 182, "slot": 3, "values": [[5621156612807134869,3345956687826194262,13719133172692197722],[187232746531771179,17246540586741393957,2113581492916353628],[14537862810343895723,14215448634865111212,12991827929331658789],[13073895064759706803,17778673011867563448,452451749060463813],[12846679264035621120,12178187075908759317,12064065106831945187],[4579685478921958669,14774426740464025944,55055029294679824],[6099726242109184639,7488918690483691710,7049115936021752350],[2224517884119604090,17691735657694722276,9579643693807667617]], "path_len": 5}, {"layer": 1, "d": 2, "position": 182, "leaf": 45, "slot": 2, "values": [[8456382716454272480,16645421597608797660,12030709434883295243],[3372347214817990474,12018637425931002418,18138096918129559976],[11036828035571675075,10829007173575275474,15479641102643849163],[8865544767290468476,14597804511677549154,10856945379225943026]], "path_len": 3}, {"layer": 2, "d": 2, "position": 45, "leaf": 11, "slot": 1, "values": [[5556470237486890782,15738386834927433074,12010912686098111476],[1997630762550526785,7678738670208248417,194037932413528879],[7091997090193394797,1911130281305530368,8017953523793910594],[1805231709436512031,5522280617529416910,4194339594951184587]], "path_len": 1}]}, + {"iota": 114, "deep": [11751211446885482280,8012913465173039444,4107968611842971166], "deep_sym": [10488146978631146249,3467633423044049359,13479118611390879923], "terminal_position": 0, "layers": [{"layer": 0, "d": 3, "position": 114, "leaf": 14, "slot": 2, "values": [[5054091260429375997,2372563079791414841,15961965775627314875],[8208101533109755006,10960112208848981436,12546368827738177063],[17241176247833241026,11720761989984833555,8771774514348889502],[13410213695129425655,11767807066594055802,1657611495470542750],[4104538544930139773,8324811138881401032,10951935808416008988],[16337892790583063462,6895512917021839800,332909730861766138],[2519452891579026128,15222959152639598375,15658823423618261612],[1296293383294420664,16252829718733227685,4547694876070093619]], "path_len": 5}, {"layer": 1, "d": 2, "position": 14, "leaf": 3, "slot": 2, "values": [[15136756194127393062,4698044514178552355,12949198668179909142],[4091359350279459743,14319222623939706000,12219427317438862304],[4269573043818774567,16386562989658503068,5783719340279165],[4551029118358661196,3011374291792230268,2530559486119546592]], "path_len": 3}, {"layer": 2, "d": 2, "position": 3, "leaf": 0, "slot": 3, "values": [[1246705822755947889,8914108151194633966,4522266899253302396],[16201135612349464620,13859003806625765459,17984848598039352151],[6334738368622223838,6843337439306526304,18123371975397451841],[14227174061878079563,9112226576955262019,10815074815388628783]], "path_len": 1}]}, + {"iota": 544, "deep": [11057980555532760461,11586349582617303100,16497605342895638788], "deep_sym": [18061532605418420298,17071096387002494484,1394440083059761068], "terminal_position": 4, "layers": [{"layer": 0, "d": 3, "position": 544, "leaf": 68, "slot": 0, "values": [[10681629651480504680,6876253625368506226,10472659439291431599],[17785494285214182646,1804332925713511638,7153802427684694427],[1750264208377628993,820549178969237976,16390984533398973123],[10670530517867835066,12282871405054959092,12927285669478093049],[5546919256863979013,14443394136083095146,17564043265733484750],[14358571524397172434,7614147518147971039,4972517443731385986],[17394577959486964371,12156225653284998877,4867008229487794604],[10055672574215541953,9093457710633016265,15886120991533784050]], "path_len": 5}, {"layer": 1, "d": 2, "position": 68, "leaf": 17, "slot": 0, "values": [[17555740993009699460,13499094199330296378,12313673954514069309],[18266186885237466730,8332842579765698647,12229600321596010012],[7494822478802989443,1646007509889999451,14055340666938528550],[15299999973184419556,7901614965414837111,11004543585603055469]], "path_len": 3}, {"layer": 2, "d": 2, "position": 17, "leaf": 4, "slot": 1, "values": [[12308156264265450342,14529545468385582654,10306409771655370083],[5744127407541261138,609091809805764668,11357141328347006033],[9685222367075524996,4627985111548410376,17296675071807714573],[16230217316348512832,4224924582692012386,17776784353682712436]], "path_len": 1}]}, + {"iota": 1095, "deep": [13793916523207725911,4108451676966047413,1462634368654358881], "deep_sym": [7823026204548128064,13411246497390018434,15618839968645625409], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1095, "leaf": 136, "slot": 7, "values": [[11736401263141382284,10580128431826960097,5565915715799547472],[18003563539250073891,6570617972736388495,8214297766385341820],[1530694407674999126,16275703648402757513,16025522073522358616],[13906516806776928060,5099232614834454594,17068868816848297917],[47484327520214282,16540557461423049181,10993471889508224336],[16847149291275321641,17089824238747236703,17143855242932676043],[16202409188065483980,7855976757770989585,15943194525433908185],[5724927034599390525,383950499348991303,17140274434618449862]], "path_len": 5}, {"layer": 1, "d": 2, "position": 136, "leaf": 34, "slot": 0, "values": [[14341369317333296234,14417614635979218844,8846520178864173288],[1541780884121598895,9871339961103297156,10854594137206876826],[547616547450581964,11549891556971613671,7197857705597676318],[3423261488439137215,6479960370124087383,17627165654203595745]], "path_len": 3}, {"layer": 2, "d": 2, "position": 34, "leaf": 8, "slot": 2, "values": [[11758409758221297221,11034734048298619314,14549108557531584252],[8435201558561332969,8784175351312936877,16456903953760264513],[15792142755751630193,13249956046916067676,3979910583159107465],[10326060926810026901,6731236006246528468,9917621895543682894]], "path_len": 1}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_dp.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_dp.rkyv new file mode 100644 index 000000000..38608334e Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_dp.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_pair.json b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_pair.json new file mode 100644 index 000000000..d49fb6ee2 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_pair.json @@ -0,0 +1,49 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "keccak", + "format": "cap_pair", + "proof_rkyv": "d_proof_keccak_cap_pair.rkyv", + "proof_rkyv_len": 51752, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 20, + "grinding_factor": 0, + "coset_offset": 3, + "merkle_cap": "auto", + "trace_tree_depth": 11, + "trace_cap": 3, + "fri_tree_depths": [10, 9, 8, 7, 6, 5, 4], + "fri_caps": [3, 3, 3, 3, 3, 3, 3], + "legacy_encoding": true, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["ae4c32d62674232b0ba6d27505a79e0dc351cbef2c7bf981f7e6071e033b1452","fdab08009575f071a9cdb78934264925e0a3c15831bffb8e296c2591d89776f8","b4b063fcffce6bb444a2b9bf7c738bf8ef4465da63ce003dd7d5140381a04ac8","a7f6a30a8015cee5d8733324dd6d4be66fe4eebba7bfca7b22f8bbb5af303ef1","076d8140a23a10b4647daca20621eeb312110446791252fd479afe5d8bfbbde3","bcb71db9203da0dc14f0c83aaa660fb81bc4a5e75bb907a59cbe36950991d3a5","df52656e7ae59323992a70632af97caa3f59f00dd5e21b8777b2510bd3d222f8"], + "zetas": [[5019159632337129269,238091556992722228,5532889084085155677],[11401249367489891504,3463462679569597100,4274808243399237651],[8939167920209768920,2181998912923045116,13686372517593792052],[17506395411273156879,11867889290151972542,11407542419953424413],[12104105903477959461,15124137694392601173,12282310738917257185],[1926491111051272611,2535199797145028677,910132886595075560],[8723582983141910029,6422360606508862377,12863861130764823176],[7140187269295878849,18092345223848887536,1238341624802517565]], + "terminal_coeffs": [[9675119329879772776,14801841314017838067,18236548730038982274],[14126988372849377104,17362610904962048507,2997281616627556854],[12732028867745254654,13981984175972346630,2203858718614623478],[15602387493645224223,15059227250182045714,17573228172572152503]], + "queries_detail": [ + {"iota": 1377, "deep": [7455843768244639387,9743762440574029878,4101673830513504298], "deep_sym": [3511192201323992326,7425862771688426773,15803627959215119946], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1377, "leaf": 688, "slot": 1, "values": [[4904023103714634616,2269339872048973425,18202589099877576301]], "path_len": 15}, {"layer": 1, "d": 1, "position": 688, "leaf": 344, "slot": 0, "values": [[15552158325687108952,4967392357145231790,8955886657214556570]], "path_len": 14}, {"layer": 2, "d": 1, "position": 344, "leaf": 172, "slot": 0, "values": [[2612792798869544968,5468544103442595665,3584530796466538409]], "path_len": 13}, {"layer": 3, "d": 1, "position": 172, "leaf": 86, "slot": 0, "values": [[17892726695253162907,11675528495509671187,15693589065103820991]], "path_len": 12}, {"layer": 4, "d": 1, "position": 86, "leaf": 43, "slot": 0, "values": [[14820453473679253910,2296745045332094341,7018948464228315015]], "path_len": 11}, {"layer": 5, "d": 1, "position": 43, "leaf": 21, "slot": 1, "values": [[8878572242937089178,9511332576208914366,533065969276291189]], "path_len": 10}, {"layer": 6, "d": 1, "position": 21, "leaf": 10, "slot": 1, "values": [[329830151858172684,8816779408413419857,10745160516112719560]], "path_len": 9}]}, + {"iota": 1361, "deep": [4424386105649019747,2750120983721151361,6541960356959252561], "deep_sym": [2651307476190031573,2146706624393544526,7968881411570009805], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1361, "leaf": 680, "slot": 1, "values": [[10340133767590356363,16890795849631589605,4349300871495131212]], "path_len": 7}, {"layer": 1, "d": 1, "position": 680, "leaf": 340, "slot": 0, "values": [[16007861022951537862,1205624391150308005,9614344438830586704]], "path_len": 6}, {"layer": 2, "d": 1, "position": 340, "leaf": 170, "slot": 0, "values": [[5160827061158288284,12918447257018839138,7766567275162096996]], "path_len": 5}, {"layer": 3, "d": 1, "position": 170, "leaf": 85, "slot": 0, "values": [[6754672244016872340,5555744013098268221,7002442437311308123]], "path_len": 4}, {"layer": 4, "d": 1, "position": 85, "leaf": 42, "slot": 1, "values": [[4787433690686736449,16041398203469231226,7447353953408244230]], "path_len": 3}, {"layer": 5, "d": 1, "position": 42, "leaf": 21, "slot": 0, "values": [[7528210498357234213,5262471822230748745,1619393323132032449]], "path_len": 2}, {"layer": 6, "d": 1, "position": 21, "leaf": 10, "slot": 1, "values": [[329830151858172684,8816779408413419857,10745160516112719560]], "path_len": 1}]}, + {"iota": 1885, "deep": [16644821497740984244,7192193719577633592,567497027096456459], "deep_sym": [3328772084659598265,11972720802068272473,14505810264201907862], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1885, "leaf": 942, "slot": 1, "values": [[12364109274760885259,18414168185378424120,201814257603045233]], "path_len": 7}, {"layer": 1, "d": 1, "position": 942, "leaf": 471, "slot": 0, "values": [[7736303778366316429,1787490843314858765,7883070884957703536]], "path_len": 6}, {"layer": 2, "d": 1, "position": 471, "leaf": 235, "slot": 1, "values": [[15968265871928534159,10969654878936738885,1434479089693489220]], "path_len": 5}, {"layer": 3, "d": 1, "position": 235, "leaf": 117, "slot": 1, "values": [[14556752700637506643,2348388154040563259,6914534242512885631]], "path_len": 4}, {"layer": 4, "d": 1, "position": 117, "leaf": 58, "slot": 1, "values": [[5904884676539004901,13961094313008801215,10931051894898429435]], "path_len": 3}, {"layer": 5, "d": 1, "position": 58, "leaf": 29, "slot": 0, "values": [[6697622104848356345,9323096160160053871,1649543559850819773]], "path_len": 2}, {"layer": 6, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[2254094792868780472,5656680908759260325,8864344245400962516]], "path_len": 1}]}, + {"iota": 1744, "deep": [11452590330265941625,16465601636710775034,17872016418185674598], "deep_sym": [16550931405044858895,2842618607274450166,13945529987242443515], "terminal_position": 13, "layers": [{"layer": 0, "d": 1, "position": 1744, "leaf": 872, "slot": 0, "values": [[5840528715108031929,11251160953820502305,1847324356710588644]], "path_len": 7}, {"layer": 1, "d": 1, "position": 872, "leaf": 436, "slot": 0, "values": [[11441586800024912458,16897683070247569656,12936863636324987699]], "path_len": 6}, {"layer": 2, "d": 1, "position": 436, "leaf": 218, "slot": 0, "values": [[2931056859857396180,6542035642962382244,9584081401290887397]], "path_len": 5}, {"layer": 3, "d": 1, "position": 218, "leaf": 109, "slot": 0, "values": [[2750067571950691581,8917446378041792268,5959056670684938469]], "path_len": 4}, {"layer": 4, "d": 1, "position": 109, "leaf": 54, "slot": 1, "values": [[18190979118219497085,3777885519368171867,5693206000720286197]], "path_len": 3}, {"layer": 5, "d": 1, "position": 54, "leaf": 27, "slot": 0, "values": [[8706481269396914430,6674339489760451417,12658234950971102773]], "path_len": 2}, {"layer": 6, "d": 1, "position": 27, "leaf": 13, "slot": 1, "values": [[17485199390895953445,5012845572731004092,17089930098592877424]], "path_len": 1}]}, + {"iota": 210, "deep": [14307124538962133335,5367173567221739243,16181558281583666348], "deep_sym": [7477631851561761913,4032606894656421959,618632229077000541], "terminal_position": 1, "layers": [{"layer": 0, "d": 1, "position": 210, "leaf": 105, "slot": 0, "values": [[18137304087224859738,3711500441386140844,17907712343369021266]], "path_len": 7}, {"layer": 1, "d": 1, "position": 105, "leaf": 52, "slot": 1, "values": [[688129786007139312,16273523563578800001,6402715237464307222]], "path_len": 6}, {"layer": 2, "d": 1, "position": 52, "leaf": 26, "slot": 0, "values": [[15119722078793073875,9100397834406702866,8005263254883193710]], "path_len": 5}, {"layer": 3, "d": 1, "position": 26, "leaf": 13, "slot": 0, "values": [[12501950698698707958,9389678858044410090,2604431415729322136]], "path_len": 4}, {"layer": 4, "d": 1, "position": 13, "leaf": 6, "slot": 1, "values": [[12402545306693990231,6163162517672281727,1650688300675708627]], "path_len": 3}, {"layer": 5, "d": 1, "position": 6, "leaf": 3, "slot": 0, "values": [[4077053280000033629,46248355743088296,6931998335990998662]], "path_len": 2}, {"layer": 6, "d": 1, "position": 3, "leaf": 1, "slot": 1, "values": [[5059575420996256821,4190436121309789696,1054706479073574679]], "path_len": 1}]}, + {"iota": 284, "deep": [1593779888553658402,13684409011714935906,6988894231470032034], "deep_sym": [13681972226048710479,5087922577534541004,17738039277407157018], "terminal_position": 2, "layers": [{"layer": 0, "d": 1, "position": 284, "leaf": 142, "slot": 0, "values": [[10384275476170871726,17189159971389164274,4738120100226236541]], "path_len": 7}, {"layer": 1, "d": 1, "position": 142, "leaf": 71, "slot": 0, "values": [[5092224361370880150,5913027675357865702,3574493885316113157]], "path_len": 6}, {"layer": 2, "d": 1, "position": 71, "leaf": 35, "slot": 1, "values": [[3864201566255052169,10153589149463947626,15614707352052562337]], "path_len": 5}, {"layer": 3, "d": 1, "position": 35, "leaf": 17, "slot": 1, "values": [[13392845361354827382,7360065332806830307,13439209107509686589]], "path_len": 4}, {"layer": 4, "d": 1, "position": 17, "leaf": 8, "slot": 1, "values": [[3035440457560863603,17670638245452663662,3020459914118654763]], "path_len": 3}, {"layer": 5, "d": 1, "position": 8, "leaf": 4, "slot": 0, "values": [[5173582849907788165,17699447796082674486,18297160988347655497]], "path_len": 2}, {"layer": 6, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[7952914794860711365,1036989991297879170,13371197571151367681]], "path_len": 1}]}, + {"iota": 149, "deep": [573286711455923633,5855910878508172803,91428610298637857], "deep_sym": [2816550618177864130,6352906390970817013,7650975019200287906], "terminal_position": 1, "layers": [{"layer": 0, "d": 1, "position": 149, "leaf": 74, "slot": 1, "values": [[11233585344845411115,1959242180628250960,7104254617117422527]], "path_len": 7}, {"layer": 1, "d": 1, "position": 74, "leaf": 37, "slot": 0, "values": [[2484328013310643083,16375687057542596532,6992210163284325896]], "path_len": 6}, {"layer": 2, "d": 1, "position": 37, "leaf": 18, "slot": 1, "values": [[14240110153299640841,11864535742232373216,8820317589108825390]], "path_len": 5}, {"layer": 3, "d": 1, "position": 18, "leaf": 9, "slot": 0, "values": [[4219795048204406341,10081909315224922351,8536329541575263506]], "path_len": 4}, {"layer": 4, "d": 1, "position": 9, "leaf": 4, "slot": 1, "values": [[15818073039788991153,13592893004124091169,10505519961286136356]], "path_len": 3}, {"layer": 5, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[8969390140574186381,12119251962002215102,1342086715066738699]], "path_len": 2}, {"layer": 6, "d": 1, "position": 2, "leaf": 1, "slot": 0, "values": [[7282793269070874198,5967657170308489624,3490988850839770011]], "path_len": 1}]}, + {"iota": 281, "deep": [7039177929252956250,8051104077967999964,14732764528071854743], "deep_sym": [15313172973719342947,7854227470716335803,2735174912562240866], "terminal_position": 2, "layers": [{"layer": 0, "d": 1, "position": 281, "leaf": 140, "slot": 1, "values": [[16351637314632867303,10550715376056936204,389777731531914906]], "path_len": 7}, {"layer": 1, "d": 1, "position": 140, "leaf": 70, "slot": 0, "values": [[966404594873295819,15537343227084575831,7601432475360510279]], "path_len": 6}, {"layer": 2, "d": 1, "position": 70, "leaf": 35, "slot": 0, "values": [[7818355896976204163,14890152152511730349,12023921930726425947]], "path_len": 5}, {"layer": 3, "d": 1, "position": 35, "leaf": 17, "slot": 1, "values": [[13392845361354827382,7360065332806830307,13439209107509686589]], "path_len": 4}, {"layer": 4, "d": 1, "position": 17, "leaf": 8, "slot": 1, "values": [[3035440457560863603,17670638245452663662,3020459914118654763]], "path_len": 3}, {"layer": 5, "d": 1, "position": 8, "leaf": 4, "slot": 0, "values": [[5173582849907788165,17699447796082674486,18297160988347655497]], "path_len": 2}, {"layer": 6, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[7952914794860711365,1036989991297879170,13371197571151367681]], "path_len": 1}]}, + {"iota": 1972, "deep": [13972310113770535822,5552131090533568835,8748308349705042723], "deep_sym": [12904991763452026590,2045517226935572658,4927886687011235995], "terminal_position": 15, "layers": [{"layer": 0, "d": 1, "position": 1972, "leaf": 986, "slot": 0, "values": [[103123668545543070,621540249427176615,5870517844992589529]], "path_len": 7}, {"layer": 1, "d": 1, "position": 986, "leaf": 493, "slot": 0, "values": [[8303741178446572413,11118598719810183172,15827861596787744863]], "path_len": 6}, {"layer": 2, "d": 1, "position": 493, "leaf": 246, "slot": 1, "values": [[5268687713648529238,18141485527344485068,2311193777804771088]], "path_len": 5}, {"layer": 3, "d": 1, "position": 246, "leaf": 123, "slot": 0, "values": [[16925572875524888510,10859497315051053091,16558322538563635674]], "path_len": 4}, {"layer": 4, "d": 1, "position": 123, "leaf": 61, "slot": 1, "values": [[11539635759505059868,15834597176730063772,7414351026055757950]], "path_len": 3}, {"layer": 5, "d": 1, "position": 61, "leaf": 30, "slot": 1, "values": [[17088097359308541830,16348344533700096317,12062715773603923136]], "path_len": 2}, {"layer": 6, "d": 1, "position": 30, "leaf": 15, "slot": 0, "values": [[3243369265885820103,2781729126563128823,6192776688734542888]], "path_len": 1}]}, + {"iota": 525, "deep": [17094259693227384751,12469668007350821596,16165213632326518833], "deep_sym": [4824386483891070766,4297530548260959791,3323344396656680919], "terminal_position": 4, "layers": [{"layer": 0, "d": 1, "position": 525, "leaf": 262, "slot": 1, "values": [[637338097538477161,5920215863388397200,3152375791077944359]], "path_len": 7}, {"layer": 1, "d": 1, "position": 262, "leaf": 131, "slot": 0, "values": [[10024172190768150414,18093095236811875938,3690458710772253406]], "path_len": 6}, {"layer": 2, "d": 1, "position": 131, "leaf": 65, "slot": 1, "values": [[2283998104138345537,6246288698769373908,18207218848997438649]], "path_len": 5}, {"layer": 3, "d": 1, "position": 65, "leaf": 32, "slot": 1, "values": [[608973827866390398,3083718923908497492,4092593950150801479]], "path_len": 4}, {"layer": 4, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[5533161650558229887,6822717597707274535,10938914320750008584]], "path_len": 3}, {"layer": 5, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[16833823024521921629,5309715217326987516,15179334130005053137]], "path_len": 2}, {"layer": 6, "d": 1, "position": 8, "leaf": 4, "slot": 0, "values": [[16996637738703906945,8756168288594149033,9169518720773082560]], "path_len": 1}]}, + {"iota": 1610, "deep": [17498517330427110815,10324701463545824868,7265055644247617298], "deep_sym": [13004031424029418383,10647612169997720653,5854165218274484100], "terminal_position": 12, "layers": [{"layer": 0, "d": 1, "position": 1610, "leaf": 805, "slot": 0, "values": [[10812029475504267638,16720837822975707610,4489770116780798833]], "path_len": 7}, {"layer": 1, "d": 1, "position": 805, "leaf": 402, "slot": 1, "values": [[12805265948321776124,8836568045853207748,1905085313949096243]], "path_len": 6}, {"layer": 2, "d": 1, "position": 402, "leaf": 201, "slot": 0, "values": [[12693966393771267041,8348939428433425034,14079667903116984336]], "path_len": 5}, {"layer": 3, "d": 1, "position": 201, "leaf": 100, "slot": 1, "values": [[2226945069041744409,3011529255908565018,3820982554989890820]], "path_len": 4}, {"layer": 4, "d": 1, "position": 100, "leaf": 50, "slot": 0, "values": [[9043526195385487228,1384729029931888705,4930313006869208923]], "path_len": 3}, {"layer": 5, "d": 1, "position": 50, "leaf": 25, "slot": 0, "values": [[5834801524537936096,12279959006427881864,14659167700684975065]], "path_len": 2}, {"layer": 6, "d": 1, "position": 25, "leaf": 12, "slot": 1, "values": [[8161617527135559126,6297769416866581594,3173676877728375344]], "path_len": 1}]}, + {"iota": 742, "deep": [13303682450365250715,18180497031344687707,13387718062770299003], "deep_sym": [8019771175969538382,16505142083751995154,3756467025059499914], "terminal_position": 5, "layers": [{"layer": 0, "d": 1, "position": 742, "leaf": 371, "slot": 0, "values": [[7715330600617454806,263816730109342565,6982233398900496069]], "path_len": 7}, {"layer": 1, "d": 1, "position": 371, "leaf": 185, "slot": 1, "values": [[955661343416646194,3810245907731434521,18162941425162857639]], "path_len": 6}, {"layer": 2, "d": 1, "position": 185, "leaf": 92, "slot": 1, "values": [[11268010522668285064,3695687107814936352,17155735713986570760]], "path_len": 5}, {"layer": 3, "d": 1, "position": 92, "leaf": 46, "slot": 0, "values": [[10648387502898115835,13677350527157965004,13646005978346329332]], "path_len": 4}, {"layer": 4, "d": 1, "position": 46, "leaf": 23, "slot": 0, "values": [[2169193262335748111,4583523388905316755,11663405555188998011]], "path_len": 3}, {"layer": 5, "d": 1, "position": 23, "leaf": 11, "slot": 1, "values": [[2441722207610391689,3667110974786560880,16709615656122872703]], "path_len": 2}, {"layer": 6, "d": 1, "position": 11, "leaf": 5, "slot": 1, "values": [[8841621684899426418,6810113607590278534,4970473809851222151]], "path_len": 1}]}, + {"iota": 1559, "deep": [13187342739415041022,4483312787545931366,1258939318303884106], "deep_sym": [9013614428963284821,13873874726042830330,16401517306744501335], "terminal_position": 12, "layers": [{"layer": 0, "d": 1, "position": 1559, "leaf": 779, "slot": 1, "values": [[14279429615183321578,16868429224720157241,7715359029040314952]], "path_len": 7}, {"layer": 1, "d": 1, "position": 779, "leaf": 389, "slot": 1, "values": [[1300114829801264936,1368022253127134748,491228609246606049]], "path_len": 6}, {"layer": 2, "d": 1, "position": 389, "leaf": 194, "slot": 1, "values": [[12689291513986525901,17681993845909985885,7114229310549916761]], "path_len": 5}, {"layer": 3, "d": 1, "position": 194, "leaf": 97, "slot": 0, "values": [[9043297265158582254,5816561522348004322,12262412645634531093]], "path_len": 4}, {"layer": 4, "d": 1, "position": 97, "leaf": 48, "slot": 1, "values": [[12968352116263480832,10876007460350245110,8248432895412464350]], "path_len": 3}, {"layer": 5, "d": 1, "position": 48, "leaf": 24, "slot": 0, "values": [[2319650304318305615,1936107267691353544,1508962945379790603]], "path_len": 2}, {"layer": 6, "d": 1, "position": 24, "leaf": 12, "slot": 0, "values": [[5368639923100606835,8118461821476130869,15151552861996072850]], "path_len": 1}]}, + {"iota": 1876, "deep": [2188958272630719718,16739216132014462061,15969720682844330480], "deep_sym": [17625020173931802981,1432206676468061533,9266034828665035492], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1876, "leaf": 938, "slot": 0, "values": [[11088999406359806620,13387672636599280160,17389860177775088876]], "path_len": 7}, {"layer": 1, "d": 1, "position": 938, "leaf": 469, "slot": 0, "values": [[15424340190559890880,18056666780987163338,4717633278725151722]], "path_len": 6}, {"layer": 2, "d": 1, "position": 469, "leaf": 234, "slot": 1, "values": [[11159565469336764502,17022137387877916585,14189315808586600062]], "path_len": 5}, {"layer": 3, "d": 1, "position": 234, "leaf": 117, "slot": 0, "values": [[14827270393572676639,10638371894509469964,11247843226701229209]], "path_len": 4}, {"layer": 4, "d": 1, "position": 117, "leaf": 58, "slot": 1, "values": [[5904884676539004901,13961094313008801215,10931051894898429435]], "path_len": 3}, {"layer": 5, "d": 1, "position": 58, "leaf": 29, "slot": 0, "values": [[6697622104848356345,9323096160160053871,1649543559850819773]], "path_len": 2}, {"layer": 6, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[2254094792868780472,5656680908759260325,8864344245400962516]], "path_len": 1}]}, + {"iota": 902, "deep": [17300476179493052956,5814189479075607576,9616298647385021830], "deep_sym": [1907324078291216890,11818855514989027498,16227487509439123490], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 902, "leaf": 451, "slot": 0, "values": [[1388622092030634984,14956937837334565805,9515911371808271297]], "path_len": 7}, {"layer": 1, "d": 1, "position": 451, "leaf": 225, "slot": 1, "values": [[4488968937607451421,17097333351519751000,684929099341931785]], "path_len": 6}, {"layer": 2, "d": 1, "position": 225, "leaf": 112, "slot": 1, "values": [[10204113238986370967,15052308195105135188,7343687815021226533]], "path_len": 5}, {"layer": 3, "d": 1, "position": 112, "leaf": 56, "slot": 0, "values": [[3262852310309194876,9232449904487550350,2876133383225467975]], "path_len": 4}, {"layer": 4, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[1322899532053186713,3220390191406884778,2557975581575432098]], "path_len": 3}, {"layer": 5, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[15593906472738996186,8290296308377011327,15323376687127304814]], "path_len": 2}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[12138138523651517612,14970545128614267257,13317972550568896498]], "path_len": 1}]}, + {"iota": 81, "deep": [7129660006418579837,5464112336437590821,16893227050504408018], "deep_sym": [10320409399989363056,2525461309991914337,5154789638308425332], "terminal_position": 0, "layers": [{"layer": 0, "d": 1, "position": 81, "leaf": 40, "slot": 1, "values": [[110229392569089403,12259696148007863211,3042644096845288748]], "path_len": 7}, {"layer": 1, "d": 1, "position": 40, "leaf": 20, "slot": 0, "values": [[8122893273598551502,14924910766491787263,6043356495912840217]], "path_len": 6}, {"layer": 2, "d": 1, "position": 20, "leaf": 10, "slot": 0, "values": [[1534937080669066257,13951054226642220235,1121681024521608867]], "path_len": 5}, {"layer": 3, "d": 1, "position": 10, "leaf": 5, "slot": 0, "values": [[17177952673650405035,1346144544956907988,9345414829703491079]], "path_len": 4}, {"layer": 4, "d": 1, "position": 5, "leaf": 2, "slot": 1, "values": [[210777378017248551,9127223126794160358,13638356578759501263]], "path_len": 3}, {"layer": 5, "d": 1, "position": 2, "leaf": 1, "slot": 0, "values": [[9219093612718247019,18210320365635570152,10887580690615904962]], "path_len": 2}, {"layer": 6, "d": 1, "position": 1, "leaf": 0, "slot": 1, "values": [[7953909299506274224,10647255018552488450,4686578840673990309]], "path_len": 1}]}, + {"iota": 526, "deep": [5717674392817305871,2534117920242909263,4358061539679503173], "deep_sym": [14563289310680755610,7841726989291354564,3706700768991955750], "terminal_position": 4, "layers": [{"layer": 0, "d": 1, "position": 526, "leaf": 263, "slot": 0, "values": [[5070813189403071397,17169903868570646682,7095625026664290666]], "path_len": 7}, {"layer": 1, "d": 1, "position": 263, "leaf": 131, "slot": 1, "values": [[16492098086237622833,9637699063752431944,3804902190530425641]], "path_len": 6}, {"layer": 2, "d": 1, "position": 131, "leaf": 65, "slot": 1, "values": [[2283998104138345537,6246288698769373908,18207218848997438649]], "path_len": 5}, {"layer": 3, "d": 1, "position": 65, "leaf": 32, "slot": 1, "values": [[608973827866390398,3083718923908497492,4092593950150801479]], "path_len": 4}, {"layer": 4, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[5533161650558229887,6822717597707274535,10938914320750008584]], "path_len": 3}, {"layer": 5, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[16833823024521921629,5309715217326987516,15179334130005053137]], "path_len": 2}, {"layer": 6, "d": 1, "position": 8, "leaf": 4, "slot": 0, "values": [[16996637738703906945,8756168288594149033,9169518720773082560]], "path_len": 1}]}, + {"iota": 1373, "deep": [12933527252844961314,13482483241538752965,12319601543423549241], "deep_sym": [7596263983079696846,9948320041500637225,10695456770784677056], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1373, "leaf": 686, "slot": 1, "values": [[2894892426971301864,18226940770443630715,2545440710177696719]], "path_len": 7}, {"layer": 1, "d": 1, "position": 686, "leaf": 343, "slot": 0, "values": [[6721789835500909538,12623796400853595278,17710357262450931617]], "path_len": 6}, {"layer": 2, "d": 1, "position": 343, "leaf": 171, "slot": 1, "values": [[4748393949877499829,18082733519044814220,4403730337012059288]], "path_len": 5}, {"layer": 3, "d": 1, "position": 171, "leaf": 85, "slot": 1, "values": [[13768096748957837555,16920999351439661381,15314456781383790071]], "path_len": 4}, {"layer": 4, "d": 1, "position": 85, "leaf": 42, "slot": 1, "values": [[4787433690686736449,16041398203469231226,7447353953408244230]], "path_len": 3}, {"layer": 5, "d": 1, "position": 42, "leaf": 21, "slot": 0, "values": [[7528210498357234213,5262471822230748745,1619393323132032449]], "path_len": 2}, {"layer": 6, "d": 1, "position": 21, "leaf": 10, "slot": 1, "values": [[329830151858172684,8816779408413419857,10745160516112719560]], "path_len": 1}]}, + {"iota": 290, "deep": [14021209778763882017,13506671047373952976,13392681072178172499], "deep_sym": [11086591086110335323,259671578182683537,18045322574659161142], "terminal_position": 2, "layers": [{"layer": 0, "d": 1, "position": 290, "leaf": 145, "slot": 0, "values": [[9759952812872127999,2959573722831254786,11962249271050766536]], "path_len": 7}, {"layer": 1, "d": 1, "position": 145, "leaf": 72, "slot": 1, "values": [[3160384506142212126,11129016019287199417,2497436678283294653]], "path_len": 6}, {"layer": 2, "d": 1, "position": 72, "leaf": 36, "slot": 0, "values": [[6632052039655398745,13544374544161250673,17324736033988297484]], "path_len": 5}, {"layer": 3, "d": 1, "position": 36, "leaf": 18, "slot": 0, "values": [[5636138478357741260,17945430522797296925,14722846344237536296]], "path_len": 4}, {"layer": 4, "d": 1, "position": 18, "leaf": 9, "slot": 0, "values": [[8112443770414550527,8363541473308682635,5818072564448079451]], "path_len": 3}, {"layer": 5, "d": 1, "position": 9, "leaf": 4, "slot": 1, "values": [[3973680665106131834,17762603807804556840,3909451547858769903]], "path_len": 2}, {"layer": 6, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[7952914794860711365,1036989991297879170,13371197571151367681]], "path_len": 1}]}, + {"iota": 1327, "deep": [4372229306796237183,4031141783655548937,5828194788736158720], "deep_sym": [2119907010727716769,8907448833006125972,8989670646794729154], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1327, "leaf": 663, "slot": 1, "values": [[9966484036733354412,13353043178793754619,15948320617124935973]], "path_len": 7}, {"layer": 1, "d": 1, "position": 663, "leaf": 331, "slot": 1, "values": [[12013446190091167856,12342764258757427021,4505252478189751377]], "path_len": 6}, {"layer": 2, "d": 1, "position": 331, "leaf": 165, "slot": 1, "values": [[9916273341643963839,15768417062469268654,2948508297908526741]], "path_len": 5}, {"layer": 3, "d": 1, "position": 165, "leaf": 82, "slot": 1, "values": [[5704875491456947424,5058415357100718396,10805003747125458482]], "path_len": 4}, {"layer": 4, "d": 1, "position": 82, "leaf": 41, "slot": 0, "values": [[17782440572252320235,8930515870775211256,12299752851325740463]], "path_len": 3}, {"layer": 5, "d": 1, "position": 41, "leaf": 20, "slot": 1, "values": [[5421279213360062408,4617814356361337913,11799045697810412882]], "path_len": 2}, {"layer": 6, "d": 1, "position": 20, "leaf": 10, "slot": 0, "values": [[5755772604009881655,14090912597746712379,5736120848822458413]], "path_len": 1}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_pair.rkyv new file mode 100644 index 000000000..9a7335edf Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_cap_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp.json b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp.json new file mode 100644 index 000000000..039fee98f --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "keccak", + "format": "dp", + "proof_rkyv": "d_proof_keccak_dp.rkyv", + "proof_rkyv_len": 8488, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 2], + "fri_roots": ["24ad3d0e98f4bed6edf18ec793157a4b40d412b869a719de1c98a970fab00072","f9fe88a494b4bb9e5ca08fc1c1a7a3ef4624b75d0ad3812e39dc65d9020f91cf","7ecf9321963996332fa2eb0464dd78c32efb4185fbc866687520f36de7764131"], + "zetas": [[5019159632337129269,238091556992722228,5532889084085155677],[12296403571495774788,9626523507187974856,1515890197535251952],[9104931154505306807,6806930774857449431,13982536486847686418],[17943736705802395901,4283444887199783601,5105112647180456117]], + "terminal_coeffs": [[18046538310705593629,16035634115336395623,14269474772235161333],[13449754012599068599,8932449597508197521,3279495531022860796],[11948801525571458678,1807139812678879355,3178944376615033389],[11072843192958306056,3667138469373329065,14070513692562577743]], + "queries_detail": [ + {"iota": 1277, "deep": [112612903969624832,13540544077113206977,891744669294414204], "deep_sym": [10665692780904921752,13891743997545272459,5218021303841191956], "terminal_position": 9, "layers": [{"layer": 0, "d": 3, "position": 1277, "leaf": 159, "slot": 5, "values": [[2726840187197314970,4641373057133563422,18254905628294267124],[7726190489312153580,9907582621009652564,13704195924065075984],[4913854101341609490,7059003433635310965,6314417828660586086],[15329881112488297229,17154024704563340256,10996086559637584958],[8931011741374043492,14857842271836329185,1962274052252210912],[11335680486698327443,1217890136881310458,6960827381617415085],[11843024897136178316,4050095544328259531,1109189699536974526],[15465183123903289453,8756197396528546255,3770807126986676922]], "path_len": 8}, {"layer": 1, "d": 2, "position": 159, "leaf": 39, "slot": 3, "values": [[9337955700188682368,10005268201090501927,17075626829468745589],[12775344777395238092,12444988312381492194,18162313775388685340],[12834611725527989937,3931095319124210104,7011958104454824522],[18278843176886412077,2091177023787081796,10712499925409758781]], "path_len": 6}, {"layer": 2, "d": 2, "position": 39, "leaf": 9, "slot": 3, "values": [[12663636275089871938,1342734324200714786,13647156802297113741],[16459715330172958447,16246821789525783433,13803231028510688298],[4472259574895772221,15705768718567917064,4738154395575758232],[10184880754128237084,3408521813484574087,14812129773919197844]], "path_len": 4}]}, + {"iota": 1793, "deep": [8057175728474570347,4157164488656378128,15766577891820220836], "deep_sym": [6733030217476856996,3149008183846048310,5846868056871306014], "terminal_position": 14, "layers": [{"layer": 0, "d": 3, "position": 1793, "leaf": 224, "slot": 1, "values": [[5863889590658237167,8803207495494391631,488510412724115696],[7367902939689275966,5399515143439789253,13537028177165637670],[13057594490447211533,12028941489541574294,10245716700381823303],[3536160573392264847,13647402147340435120,9933763201558099138],[17705005489971962397,16100850492966888022,3356205035428066804],[4834413239841014089,10648175143241294336,14941339194282038433],[11268069224352915944,7397295511095760171,650865519941991105],[17810125080425025549,8252558882871738031,1603536863803495337]], "path_len": 8}, {"layer": 1, "d": 2, "position": 224, "leaf": 56, "slot": 0, "values": [[14576290996393278046,4099269296443923918,13962179114375143747],[8539012341704406611,14597685688217769420,16489000745330409389],[4928849784411569862,5656061150696874101,18052668495466081830],[10699077948197816254,7120867110842505641,2470038313983831606]], "path_len": 6}, {"layer": 2, "d": 2, "position": 56, "leaf": 14, "slot": 0, "values": [[7743892560805052942,10417724895478695146,9242061460595868046],[5215961958705134614,3646588380176163324,11215186743127464548],[18165082680919870330,16446510594026310692,9931249060720003450],[16761232818563256745,6094664995607304883,14831579372437454855]], "path_len": 4}]}, + {"iota": 1422, "deep": [4336444987633806031,42270359695066150,811124501724833250], "deep_sym": [6040322601513087150,2232031154133685564,13268270931765776955], "terminal_position": 11, "layers": [{"layer": 0, "d": 3, "position": 1422, "leaf": 177, "slot": 6, "values": [[5773020228763106950,12181432689259931341,2904380769668095371],[5378436167230488318,1136926564836430281,11025181981762941864],[9767397875213699870,16391873535337268069,9544088588384136146],[50456808154105685,7570275210936766391,3076092320148066703],[6519022523009943654,14501422860440411207,16766709789063948727],[1942043923567112082,9396051082847161748,4275006641168421309],[802135155222683305,8086721014210384187,5276472197522953276],[14963808776084644130,11822327586546991308,9902819457375193080]], "path_len": 8}, {"layer": 1, "d": 2, "position": 177, "leaf": 44, "slot": 1, "values": [[8007252311096823131,15451587500561065094,5200475833640745404],[12419273660987748398,619789569423171010,4299596803633862803],[14489492344890871493,14652990201720622453,5263973935492910147],[12959859661416681018,3696933911172366326,18087484035368403648]], "path_len": 6}, {"layer": 2, "d": 2, "position": 44, "leaf": 11, "slot": 0, "values": [[5556470237486890782,15738386834927433074,12010912686098111476],[1997630762550526785,7678738670208248417,194037932413528879],[7091997090193394797,1911130281305530368,8017953523793910594],[1805231709436512031,5522280617529416910,4194339594951184587]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp.rkyv new file mode 100644 index 000000000..000c26f6c Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp_3_1_3.json b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp_3_1_3.json new file mode 100644 index 000000000..fa967427f --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp_3_1_3.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "keccak", + "format": "dp_3_1_3", + "proof_rkyv": "d_proof_keccak_dp_3_1_3.rkyv", + "proof_rkyv_len": 8728, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 1, 3], + "fri_roots": ["24ad3d0e98f4bed6edf18ec793157a4b40d412b869a719de1c98a970fab00072","871e7c3df8ccc3c64730c863f6a2fa787fd06fae0f2a2e11ebf5abf3de3a2226","ade304313a4dfb3665231a85ee8c14f736c819109c36d1638f3fea9d06d68e1b"], + "zetas": [[5019159632337129269,238091556992722228,5532889084085155677],[12296403571495774788,9626523507187974856,1515890197535251952],[14523967509618755178,7981048790673426368,4165831416408514802],[15590192218200992447,15146425979047601999,2279642302822794473]], + "terminal_coeffs": [[1388079943033759100,11009361148285669739,18003680692828365043],[17237532129585750347,1517078761242656685,16199054381564524094],[1938844203267517687,9877447212410491634,12968975246239323922],[4666008817029420275,15603102928404590219,2901983394916486066]], + "queries_detail": [ + {"iota": 1101, "deep": [13444408416110140567,15620040802225407588,15223031624530673992], "deep_sym": [290005468330435364,4114526459923533080,2460530217353380230], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1101, "leaf": 137, "slot": 5, "values": [[7571539965246193760,12259342495260382898,7053829414421388370],[3226322966601049319,14317828094417592584,54137258033536103],[14021848107088228906,2919431011366834709,10068585320682235218],[2856107074513493232,15548865223592133260,2006005570476947405],[15658026536969269856,1554422422590567528,10731091312921677367],[16481563759951422033,12119679792493024127,5422020975449206720],[13826052861493771289,4614039571553346032,6654350834419205173],[1115307290735749016,14730692960078495598,8274405822262668945]], "path_len": 8}, {"layer": 1, "d": 1, "position": 137, "leaf": 68, "slot": 1, "values": [[14341369317333296234,14417614635979218844,8846520178864173288],[1541780884121598895,9871339961103297156,10854594137206876826]], "path_len": 7}, {"layer": 2, "d": 3, "position": 68, "leaf": 8, "slot": 4, "values": [[17340337315510687533,16466850183023390197,16170822732462346370],[13577065484001180074,10789052881678110757,1984289784499658738],[10322313144195499275,9573474260847586333,18360532062725495499],[13517606960732314903,5924632567214305012,6118505914147684536],[542347889760713356,9784404433359539161,7414969705090267463],[12066365013085694995,5956563561476960593,14866809534596581635],[4561576524370871036,10849591696019172330,6961009597333354300],[3866908127244773168,15669605755659572510,9331312028362224237]], "path_len": 4}]}, + {"iota": 685, "deep": [4925031604963438677,830417094021520731,15862979152598215734], "deep_sym": [6436035638622990139,2908039237793753059,14976610380852847077], "terminal_position": 5, "layers": [{"layer": 0, "d": 3, "position": 685, "leaf": 85, "slot": 5, "values": [[2711194398170199905,17428802182914895478,635293871302441912],[3653691656041130802,10425319869671607000,8515362419986619290],[9754395736388531158,5271124400817841404,13920649595609625458],[3615836389795967869,8175211513136783109,7424862389176434620],[5088724134008406528,2287923670496620078,11530128097074748850],[14785342658225808385,15047767963588960580,14000936891403725028],[12926575319413478177,154624349820291247,6857332647761830396],[5171891210798910511,5038297187202472504,2006656822716967144]], "path_len": 8}, {"layer": 1, "d": 1, "position": 85, "leaf": 42, "slot": 1, "values": [[13540904436659318567,17336738776988905048,17276842271454845243],[11189914825747541931,6819516493339643473,9190484781628012885]], "path_len": 7}, {"layer": 2, "d": 3, "position": 42, "leaf": 5, "slot": 2, "values": [[31446593612320238,17773679528145884633,5637457551076632612],[6257322271130649695,16400515433097762086,6346320073297491491],[15047501166121929445,1523283072139552980,16645225185759966672],[11124874806014753988,1879122302621705061,2303532870366114852],[17332404032257580601,17636788358461353225,1675268583132812138],[11429077391219906462,4620602044590711909,15393150759290484458],[13671100188568527934,12564621711691339017,7611452210500384787],[1397326939955468944,5653632621592529379,4992247048829424615]], "path_len": 4}]}, + {"iota": 1249, "deep": [9946431352694562883,2008734157674376412,16548416984122237506], "deep_sym": [16096242337339141933,11323512440171655616,10360796140662398409], "terminal_position": 9, "layers": [{"layer": 0, "d": 3, "position": 1249, "leaf": 156, "slot": 1, "values": [[6978142119154416518,8547442860041728708,14970652831834600826],[13824685731161489176,17885767568970612304,2534572619566806965],[10060589250654053507,6130325395888609220,6951027455020094700],[13364481942665285269,7033035359754839223,453294621666011599],[9238061526093088533,4650807553901898659,2478698084496860348],[18049237297669172639,7829189454966868378,9410917270666579147],[6566615427776827262,15491421233653898723,5991626663109222605],[3511981306102162279,2482433176886809990,15432684550774174243]], "path_len": 8}, {"layer": 1, "d": 1, "position": 156, "leaf": 78, "slot": 0, "values": [[9337955700188682368,10005268201090501927,17075626829468745589],[12775344777395238092,12444988312381492194,18162313775388685340]], "path_len": 7}, {"layer": 2, "d": 3, "position": 78, "leaf": 9, "slot": 6, "values": [[10776393641929202433,971877452076755765,14157900628235863657],[12650629188205370781,1368692753999156063,4809609548511919935],[6843606874063648474,4015927221886946244,11248907885089940182],[4735026784363667383,3345527530080963457,5763353545005297393],[6069373268525402579,13721627381486427074,16411807685985695596],[8299354347455469646,3094329301555192808,2288817730528401204],[8418242767604578714,4171510248854052332,14012470415711676305],[4277858526620869279,6369201857160850861,7422271882544911562]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp_3_1_3.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp_3_1_3.rkyv new file mode 100644 index 000000000..4f49e1749 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_dp_3_1_3.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_pair.json b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_pair.json new file mode 100644 index 000000000..062eaeca6 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_pair.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "keccak", + "format": "pair", + "proof_rkyv": "d_proof_keccak_pair.rkyv", + "proof_rkyv_len": 11136, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": true, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["ae4c32d62674232b0ba6d27505a79e0dc351cbef2c7bf981f7e6071e033b1452","fdab08009575f071a9cdb78934264925e0a3c15831bffb8e296c2591d89776f8","b4b063fcffce6bb444a2b9bf7c738bf8ef4465da63ce003dd7d5140381a04ac8","a7f6a30a8015cee5d8733324dd6d4be66fe4eebba7bfca7b22f8bbb5af303ef1","076d8140a23a10b4647daca20621eeb312110446791252fd479afe5d8bfbbde3","bcb71db9203da0dc14f0c83aaa660fb81bc4a5e75bb907a59cbe36950991d3a5","df52656e7ae59323992a70632af97caa3f59f00dd5e21b8777b2510bd3d222f8"], + "zetas": [[5019159632337129269,238091556992722228,5532889084085155677],[11401249367489891504,3463462679569597100,4274808243399237651],[8939167920209768920,2181998912923045116,13686372517593792052],[17506395411273156879,11867889290151972542,11407542419953424413],[12104105903477959461,15124137694392601173,12282310738917257185],[1926491111051272611,2535199797145028677,910132886595075560],[8723582983141910029,6422360606508862377,12863861130764823176],[7140187269295878849,18092345223848887536,1238341624802517565]], + "terminal_coeffs": [[9675119329879772776,14801841314017838067,18236548730038982274],[14126988372849377104,17362610904962048507,2997281616627556854],[12732028867745254654,13981984175972346630,2203858718614623478],[15602387493645224223,15059227250182045714,17573228172572152503]], + "queries_detail": [ + {"iota": 1377, "deep": [7455843768244639387,9743762440574029878,4101673830513504298], "deep_sym": [3511192201323992326,7425862771688426773,15803627959215119946], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1377, "leaf": 688, "slot": 1, "values": [[4904023103714634616,2269339872048973425,18202589099877576301]], "path_len": 10}, {"layer": 1, "d": 1, "position": 688, "leaf": 344, "slot": 0, "values": [[15552158325687108952,4967392357145231790,8955886657214556570]], "path_len": 9}, {"layer": 2, "d": 1, "position": 344, "leaf": 172, "slot": 0, "values": [[2612792798869544968,5468544103442595665,3584530796466538409]], "path_len": 8}, {"layer": 3, "d": 1, "position": 172, "leaf": 86, "slot": 0, "values": [[17892726695253162907,11675528495509671187,15693589065103820991]], "path_len": 7}, {"layer": 4, "d": 1, "position": 86, "leaf": 43, "slot": 0, "values": [[14820453473679253910,2296745045332094341,7018948464228315015]], "path_len": 6}, {"layer": 5, "d": 1, "position": 43, "leaf": 21, "slot": 1, "values": [[8878572242937089178,9511332576208914366,533065969276291189]], "path_len": 5}, {"layer": 6, "d": 1, "position": 21, "leaf": 10, "slot": 1, "values": [[329830151858172684,8816779408413419857,10745160516112719560]], "path_len": 4}]}, + {"iota": 1361, "deep": [4424386105649019747,2750120983721151361,6541960356959252561], "deep_sym": [2651307476190031573,2146706624393544526,7968881411570009805], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1361, "leaf": 680, "slot": 1, "values": [[10340133767590356363,16890795849631589605,4349300871495131212]], "path_len": 10}, {"layer": 1, "d": 1, "position": 680, "leaf": 340, "slot": 0, "values": [[16007861022951537862,1205624391150308005,9614344438830586704]], "path_len": 9}, {"layer": 2, "d": 1, "position": 340, "leaf": 170, "slot": 0, "values": [[5160827061158288284,12918447257018839138,7766567275162096996]], "path_len": 8}, {"layer": 3, "d": 1, "position": 170, "leaf": 85, "slot": 0, "values": [[6754672244016872340,5555744013098268221,7002442437311308123]], "path_len": 7}, {"layer": 4, "d": 1, "position": 85, "leaf": 42, "slot": 1, "values": [[4787433690686736449,16041398203469231226,7447353953408244230]], "path_len": 6}, {"layer": 5, "d": 1, "position": 42, "leaf": 21, "slot": 0, "values": [[7528210498357234213,5262471822230748745,1619393323132032449]], "path_len": 5}, {"layer": 6, "d": 1, "position": 21, "leaf": 10, "slot": 1, "values": [[329830151858172684,8816779408413419857,10745160516112719560]], "path_len": 4}]}, + {"iota": 1885, "deep": [16644821497740984244,7192193719577633592,567497027096456459], "deep_sym": [3328772084659598265,11972720802068272473,14505810264201907862], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1885, "leaf": 942, "slot": 1, "values": [[12364109274760885259,18414168185378424120,201814257603045233]], "path_len": 10}, {"layer": 1, "d": 1, "position": 942, "leaf": 471, "slot": 0, "values": [[7736303778366316429,1787490843314858765,7883070884957703536]], "path_len": 9}, {"layer": 2, "d": 1, "position": 471, "leaf": 235, "slot": 1, "values": [[15968265871928534159,10969654878936738885,1434479089693489220]], "path_len": 8}, {"layer": 3, "d": 1, "position": 235, "leaf": 117, "slot": 1, "values": [[14556752700637506643,2348388154040563259,6914534242512885631]], "path_len": 7}, {"layer": 4, "d": 1, "position": 117, "leaf": 58, "slot": 1, "values": [[5904884676539004901,13961094313008801215,10931051894898429435]], "path_len": 6}, {"layer": 5, "d": 1, "position": 58, "leaf": 29, "slot": 0, "values": [[6697622104848356345,9323096160160053871,1649543559850819773]], "path_len": 5}, {"layer": 6, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[2254094792868780472,5656680908759260325,8864344245400962516]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_pair.rkyv new file mode 100644 index 000000000..df5753048 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_keccak_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_dp.json b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_dp.json new file mode 100644 index 000000000..27ea5af30 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_dp.json @@ -0,0 +1,49 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "rpx", + "format": "cap_dp", + "proof_rkyv": "d_proof_rpx_cap_dp.rkyv", + "proof_rkyv_len": 41480, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 20, + "grinding_factor": 0, + "coset_offset": 3, + "merkle_cap": "auto", + "trace_tree_depth": 11, + "trace_cap": 3, + "fri_tree_depths": [8, 6, 4], + "fri_caps": [3, 3, 3], + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 2], + "fri_roots": ["b8cd71d876dd084c3fba58b6a1b1788b09ee71b58bd373a3b0c32da0c88abac4","8abd4354863193215d65af7903d5f53018e4b2076a5d160d8197c0df719c33dc","5ba58234c6bd1059d39526597538aa208c0fce55ac284c9b1abcbfc4bc87fcb7"], + "zetas": [[4735330965523630181,1034630526833404286,12017969954712239940],[7889074366333103969,4290811767201827376,14455537773263474986],[16567739822379498242,5753162788299774204,5950576806486104926],[3148119476643166323,15342831354566172589,16163821384909909157]], + "terminal_coeffs": [[12646447477222048401,12937374675136009352,16549558038379651479],[2564516689604577222,14255657332782844950,7303342851315364550],[7904019038462202246,10880807545931735486,15264205294200432227],[13482626913175767796,15717304750858041741,4892518987751974292]], + "queries_detail": [ + {"iota": 1095, "deep": [15404367171170966026,18436452028196411499,5024906168965672354], "deep_sym": [16265809848131463264,11495087467666035873,7459148639182883977], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1095, "leaf": 136, "slot": 7, "values": [[2406839404446874184,14378864393598774400,9244658727562446653],[7310068857272658177,6690242714355677003,16014750232870151724],[16918409907732086886,2943891056625643727,17633501031830476314],[12754819164990375128,11705998079299817355,11740835587370543910],[2273041869480575555,1772390107814740151,15057029391149023100],[11526866775207687538,5051892843389067913,18351320525206389605],[12623868928666452372,4025975981633944670,16736429527117892317],[3360740073930066434,11793243838384529871,16278212037669344741]], "path_len": 13}, {"layer": 1, "d": 2, "position": 136, "leaf": 34, "slot": 0, "values": [[12541072756489582885,2370820915463632688,2423396583266204055],[14905864220509753211,15950579206739519424,15082498997069827244],[9459878356316854591,1270075853426673136,4472301856924467933],[14393624207528009097,9675335233598348594,16693550050532620583]], "path_len": 11}, {"layer": 2, "d": 2, "position": 34, "leaf": 8, "slot": 2, "values": [[16818112822979374356,10274787015391993318,15763705279771580830],[7191284429859224732,4809043185931564649,14671147736651195233],[2802946927799549417,8037886970914238609,9324105614581397069],[720806501774538325,13075390526153910697,6384491353149171884]], "path_len": 9}]}, + {"iota": 1336, "deep": [9782001122439711942,9555857402003070809,5596777784231506518], "deep_sym": [2420617880218855450,11218452639813192342,8178398275584052689], "terminal_position": 10, "layers": [{"layer": 0, "d": 3, "position": 1336, "leaf": 167, "slot": 0, "values": [[2041956337797731597,4112831764093230891,11432379156394499095],[8144473764935818890,6978818592127372038,13240419817221723331],[4510585753595495978,7259665793084123234,5963491368598554412],[12449613922291734239,18143330045847154993,11555008710041017422],[13423943835517857770,6610089783383402616,14732195742916433439],[3080128605461466890,10236270832537313262,8257530437238638301],[8362553719204526827,2153485092467020843,4632353622181731171],[11789751946472833749,6000336259605467995,973022182701680872]], "path_len": 5}, {"layer": 1, "d": 2, "position": 167, "leaf": 41, "slot": 3, "values": [[1806710437233960703,865918887752352625,6749716608005253549],[4630423819292356457,16185182827820005660,8859790871695731864],[642687296268297522,15932233592480046187,1035347428796527533],[1571642650814870704,5246859957097853911,1771889554939298762]], "path_len": 3}, {"layer": 2, "d": 2, "position": 41, "leaf": 10, "slot": 1, "values": [[16605815559724387167,7600882892259287443,6994922477906460043],[14201299358752116799,5717899003132557091,9025489995620184926],[15619332420018523266,1595985739793856288,4969668978550259454],[6667880876193245735,12155122404735786091,14288219524612442946]], "path_len": 1}]}, + {"iota": 396, "deep": [1195167017398997224,3684677677763618554,1602181459315078555], "deep_sym": [17092653866053864012,10319696574527941179,65705194922387228], "terminal_position": 3, "layers": [{"layer": 0, "d": 3, "position": 396, "leaf": 49, "slot": 4, "values": [[11065792751948336436,6945821278266161605,695451384357543318],[8408178489419937465,2838485655880223095,8492969326019934396],[265145725283343233,10180163860108398826,11843491620723569992],[15123282416936963659,17530049459658255167,8917537469248528646],[739600072118952183,889941540565197657,4394371451414923712],[15901449078141666097,9754018756689853016,12057470623441911843],[16131822616193081053,4052660922016275605,6765793682960198828],[9270165410843091007,10965847749993617830,14798583832773956809]], "path_len": 5}, {"layer": 1, "d": 2, "position": 49, "leaf": 12, "slot": 1, "values": [[3895026366660990149,2600803056632231256,4321642801809818286],[12836403324608808788,172846604945358907,7659991926477877030],[90956911091737129,4468391866397480991,9108435893769413449],[15315665520481292732,6371916974269842650,7526540926329622480]], "path_len": 3}, {"layer": 2, "d": 2, "position": 12, "leaf": 3, "slot": 0, "values": [[11401453414757530315,353357581965599259,15287411324344444169],[16460794964378913634,42195047452164617,805857385603866297],[8773174365564475115,10384145495722058796,3379422144880863992],[11394111484266117948,1699290548295696723,1801926384528077999]], "path_len": 1}]}, + {"iota": 464, "deep": [13315577128423449762,7252275752688912750,14779099846109420810], "deep_sym": [16753143537545762856,11973892546591768447,2290866283218361113], "terminal_position": 3, "layers": [{"layer": 0, "d": 3, "position": 464, "leaf": 58, "slot": 0, "values": [[11098928215659943301,10386996362178863142,1067926555475592947],[11535820862256123319,6058445108874161607,7587035286616875938],[4239169084106685387,14808332345509319384,4043547374147254999],[15526762616350004649,16244669597538508259,10278101662101015121],[18035363729350438005,12990733754043379238,8273599682664364735],[13621563928537378285,9657104990759654583,5190489215411308914],[10206630974422042578,733815345948759775,16325705866146830934],[15972587451892223286,5888285155558528998,8471935757408025741]], "path_len": 5}, {"layer": 1, "d": 2, "position": 58, "leaf": 14, "slot": 2, "values": [[6148049423775918455,14589867543249973290,3056806138608021102],[12775873754809006869,2779646161953595826,7537613261740435835],[14650203784677039238,2948257343798866283,2842367134151335097],[9538295325872013162,18409431940839324820,18140461979228732677]], "path_len": 3}, {"layer": 2, "d": 2, "position": 14, "leaf": 3, "slot": 2, "values": [[11401453414757530315,353357581965599259,15287411324344444169],[16460794964378913634,42195047452164617,805857385603866297],[8773174365564475115,10384145495722058796,3379422144880863992],[11394111484266117948,1699290548295696723,1801926384528077999]], "path_len": 1}]}, + {"iota": 772, "deep": [15575460507847184623,16637149665265295724,5265665106595265083], "deep_sym": [4196865330670975011,8511959952317818220,4486734497171560361], "terminal_position": 6, "layers": [{"layer": 0, "d": 3, "position": 772, "leaf": 96, "slot": 4, "values": [[2178356221295347642,1554040541670965436,12680044827998501479],[13087366873272454428,9093557071516503454,5636907849804883550],[14754354684262512615,5714243598007047435,13773896977707185793],[7705776266726168682,7258853463238437662,3752966942983826169],[1682970084775361702,9078299675771697083,15031900324424974662],[6155448351863329101,9050375465121490777,18189771998535169449],[18305285849261724818,13292852586514925182,4155242995629021457],[13189484825275779588,10465389948129098426,7519584792001068661]], "path_len": 5}, {"layer": 1, "d": 2, "position": 96, "leaf": 24, "slot": 0, "values": [[6688927116788273172,6301558512541473027,2984598334581139578],[10065444334709453301,1598632954335847018,7577084245944080724],[5802245441082115461,16683794194713280100,3262824299805359566],[6787132102759609571,15294244265503303220,18325636619723595824]], "path_len": 3}, {"layer": 2, "d": 2, "position": 24, "leaf": 6, "slot": 0, "values": [[18149104366334536105,18153758938222784692,13286440057442018956],[15775709367588068380,793261547229653412,4802668313348493294],[15397241215398330711,14917271561697322841,5468323818766131144],[17983203460117045278,11467095688805271950,3173804581379279974]], "path_len": 1}]}, + {"iota": 2024, "deep": [17862126616266007631,39298754798513079,16960535725074283747], "deep_sym": [14855693456863274664,9126881790095326223,12348213703764840419], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 2024, "leaf": 253, "slot": 0, "values": [[248680714655802291,8287422482732093021,11512168046999728330],[1698550105716295035,4359650213628285538,15696348528790637165],[10225623802443553751,7412205301699440524,10936257831384578972],[5580328202384677588,3511693997310828484,4032541176270544725],[1502477949338677178,8937518693682245174,2377119977838417508],[16831417672874055750,8516735277199927274,7151182767156521586],[13172991599883336198,3614626671380230259,4994228220289388971],[5126695922719234861,6460266335430374719,13754234627137778040]], "path_len": 5}, {"layer": 1, "d": 2, "position": 253, "leaf": 63, "slot": 1, "values": [[12671436740885090106,4712797108775092170,16379995986181438746],[13487967099277239218,3917591648834860403,5185034935052652064],[7260034507286567562,15934756997500426999,8176082139629178738],[3799004422275886064,1804703660742419729,790511295811445306]], "path_len": 3}, {"layer": 2, "d": 2, "position": 63, "leaf": 15, "slot": 3, "values": [[13804487110218064471,11031393271821788048,12102032588536669622],[15933072575736243346,16812635495079777057,16914031897057335688],[10355406612524080793,5018917751865171386,5486395490041162930],[10611084620214441847,16088658798861853787,13676265492939844660]], "path_len": 1}]}, + {"iota": 1018, "deep": [9666497629706741416,2711014432294742997,5338993119085585124], "deep_sym": [17591789704411965224,8334765678444352322,12136050117161229927], "terminal_position": 7, "layers": [{"layer": 0, "d": 3, "position": 1018, "leaf": 127, "slot": 2, "values": [[15915610210880015156,654178370987434333,9001484692176855369],[16701406999377053209,14900983005331860275,15608324566254649573],[16800180945129655421,16239098360085105416,746862932917931344],[7135860444559045688,4368596734678544347,10009304364373728046],[7211051365822209500,7053194884746004028,9886256935976347858],[17030910034289132002,1075234149160835061,6072641001730603910],[9112216685035143354,8790352404567562935,1598562387253550849],[4263051900018733301,16370941081911055892,703039766805563670]], "path_len": 5}, {"layer": 1, "d": 2, "position": 127, "leaf": 31, "slot": 3, "values": [[893701164733485859,10783705128469438790,6430592977790142885],[9273314110616551886,3191912665801675716,4156207855013590725],[7766475431522936470,13966589519538816880,5903906557995018826],[17064858841826728263,13953088988948658971,4173474481815629242]], "path_len": 3}, {"layer": 2, "d": 2, "position": 31, "leaf": 7, "slot": 3, "values": [[1033049072983578823,10867488755001030385,3516800898677287794],[16923583511889176879,17224454678017056146,10546138564032550635],[244008804088196093,5255888736598227352,5009267593252073460],[7173875125936270454,543113462009931580,6960567853046668448]], "path_len": 1}]}, + {"iota": 434, "deep": [804929999337509138,1468933884987952182,7645286452403436813], "deep_sym": [12619978267720755695,13772173429398559183,11854116844548204019], "terminal_position": 3, "layers": [{"layer": 0, "d": 3, "position": 434, "leaf": 54, "slot": 2, "values": [[8986955191011758421,10726725355813200951,10473372829063026604],[9071506389801486070,8509869025281211871,2644477472432484436],[13964968611822928574,13285675260824773453,11748096510209525604],[2310215663300437810,8171542782131568893,9768443869295330051],[17498954884778899589,13217673593679621957,9626919390121158837],[5978707022097593593,16199170155804328509,15948541937847602294],[4647558760010050621,1464615084228594593,14590764956234869493],[3513037597053252531,16880048805136258473,3524654951292347810]], "path_len": 5}, {"layer": 1, "d": 2, "position": 54, "leaf": 13, "slot": 2, "values": [[4400625513865422810,6743756406446708109,16537227040912844421],[11049544044019279635,2177374879725984586,2701240540680480214],[10262723708506140141,9571236840615945782,17624993559481847384],[4590038678235841403,4251101814116480663,1187654653540111335]], "path_len": 3}, {"layer": 2, "d": 2, "position": 13, "leaf": 3, "slot": 1, "values": [[11401453414757530315,353357581965599259,15287411324344444169],[16460794964378913634,42195047452164617,805857385603866297],[8773174365564475115,10384145495722058796,3379422144880863992],[11394111484266117948,1699290548295696723,1801926384528077999]], "path_len": 1}]}, + {"iota": 1406, "deep": [13230221430615035516,10813863012752583972,2318008317718129139], "deep_sym": [6106910834220921197,6523099607877010761,6593979421588875795], "terminal_position": 10, "layers": [{"layer": 0, "d": 3, "position": 1406, "leaf": 175, "slot": 6, "values": [[323195911353260228,17263245316893654565,11463257750186360671],[7320445494663096340,14237138155561682560,3110675865172535763],[6713680053659494631,2032538538940427656,1057113045797355331],[8631725061358052832,423633208027496564,4662527007038932496],[18132757568820259228,2663534511082111563,8199429364722786367],[6933883104364360942,2929190753787481714,13046068366344658748],[14590650123346818081,6366762199596637381,12314754550654698852],[9574093927666969281,2563022925074734877,7105053068735183895]], "path_len": 5}, {"layer": 1, "d": 2, "position": 175, "leaf": 43, "slot": 3, "values": [[7377752767619274477,14960320299150954633,1656998823700909017],[13909380617930782423,1665610621370769977,4853928730766007756],[15420917467817398658,12095262020142939256,15586101709279441247],[545594005296785681,7981396123147073003,7300637563208043684]], "path_len": 3}, {"layer": 2, "d": 2, "position": 43, "leaf": 10, "slot": 3, "values": [[16605815559724387167,7600882892259287443,6994922477906460043],[14201299358752116799,5717899003132557091,9025489995620184926],[15619332420018523266,1595985739793856288,4969668978550259454],[6667880876193245735,12155122404735786091,14288219524612442946]], "path_len": 1}]}, + {"iota": 851, "deep": [5069211856273159308,15738204765827836414,7803687198427520729], "deep_sym": [3365555321677271314,7183365741688524927,7992791527747308157], "terminal_position": 6, "layers": [{"layer": 0, "d": 3, "position": 851, "leaf": 106, "slot": 3, "values": [[8185664597084646010,7199396461353948815,4290149667551161223],[10553626672500095754,16880963759787771784,5047385498810058389],[5128343943071886226,9397445922439232244,17537431850446381343],[12456949827564107349,1794586338037375500,10510426836892300002],[4392351958428368786,3828249484931527240,4409399853039764489],[17886905826175496683,11715164881135227845,717170915527280937],[4601525578578854078,12081089708058704727,3314088772912454648],[12218840982635270313,8071433161511740522,13958090671284912090]], "path_len": 5}, {"layer": 1, "d": 2, "position": 106, "leaf": 26, "slot": 2, "values": [[9301234740979162646,2753907425628220151,17889478520006217760],[6274786885341819980,6331824999788377843,3265708602123659790],[14009363662878866218,8048778356133517596,14181916097130251870],[11109783065294107,6529788703768046412,16921505187996595492]], "path_len": 3}, {"layer": 2, "d": 2, "position": 26, "leaf": 6, "slot": 2, "values": [[18149104366334536105,18153758938222784692,13286440057442018956],[15775709367588068380,793261547229653412,4802668313348493294],[15397241215398330711,14917271561697322841,5468323818766131144],[17983203460117045278,11467095688805271950,3173804581379279974]], "path_len": 1}]}, + {"iota": 1619, "deep": [1408276854512374808,8900300285643770305,6610355828502006211], "deep_sym": [14513375217811742076,16874657744221025801,1601542188964114380], "terminal_position": 12, "layers": [{"layer": 0, "d": 3, "position": 1619, "leaf": 202, "slot": 3, "values": [[16405036730644138202,13573262514280719051,8197665227336875941],[4810183087946276285,10961977203566478230,5769377055311518319],[1039569578885082534,10799768036753555823,5824281002988763926],[4663399330769730164,11219583712814900217,2591059508733142074],[3710383079295111760,7862687122903785212,8819873747000791656],[12731875014052926425,16826320984485484741,5314929152696311448],[6472125744179394964,9646881788806417866,18320151023544004885],[13413754152113967178,16805826508014926467,11562774648789987451]], "path_len": 5}, {"layer": 1, "d": 2, "position": 202, "leaf": 50, "slot": 2, "values": [[9746517607667315512,5356526412011744366,378649477001740238],[15819241902267462633,10827020772783032148,12295143323093252537],[16740218466659950335,11322925347085219479,12626480753167162603],[8928078663829289801,17970166156916115780,14458074476181583109]], "path_len": 3}, {"layer": 2, "d": 2, "position": 50, "leaf": 12, "slot": 2, "values": [[5464005601434038088,9623393316986421227,10756173414515652631],[15629554433951570575,5891404871893161211,7794506514771656936],[7015985412422436116,5314828133090116385,3339720643265786168],[7095733908393958914,18058755147947351090,12396916271309078570]], "path_len": 1}]}, + {"iota": 734, "deep": [6726635026919291185,6114134777381380747,3969234766899550501], "deep_sym": [11610741361144710308,8022256674750058278,4492303683849342962], "terminal_position": 5, "layers": [{"layer": 0, "d": 3, "position": 734, "leaf": 91, "slot": 6, "values": [[1776024911715557280,9535820208453597638,11375092966262227432],[4695891893436478717,14290340289102254735,10729924849860397325],[18204714348454572237,9512367534659701392,6022391662543991831],[10220654977012995561,9581140563703575642,537552342188447673],[17761397183816365641,2144580924868674917,16884137666549458392],[16018528100941444706,8139525193477316658,9580421776092478677],[1122027982641871908,15563930121846823177,4497287494922049376],[16980848242871090598,16990717267573275839,1474503480350023461]], "path_len": 5}, {"layer": 1, "d": 2, "position": 91, "leaf": 22, "slot": 3, "values": [[18181522475995947445,8128317690483459736,15340725558620520241],[17934575813731174648,3157137049601440032,9620131208779528051],[1301762668122567517,15591669752359556141,10373328281998565515],[17301133920439871298,14423637770918509274,11879195659894573040]], "path_len": 3}, {"layer": 2, "d": 2, "position": 22, "leaf": 5, "slot": 2, "values": [[6504414617537595573,6354919752630017286,15662046576375417059],[18190904080162926060,10799053964643538786,2642404754410762933],[7534862575300724521,10620461630712972371,7610325415054858437],[7083822503393396084,3502344503227274059,4024386299932845131]], "path_len": 1}]}, + {"iota": 1584, "deep": [145172723069761214,6132204174748908772,1251322044684973154], "deep_sym": [16371791428722529123,13517035650984980097,18308470790125148470], "terminal_position": 12, "layers": [{"layer": 0, "d": 3, "position": 1584, "leaf": 198, "slot": 0, "values": [[8084017772650914005,12302362108596853331,14892860500875362423],[16700768433721502832,174528624483998797,3598044338018230632],[17169989850796931758,8694474674286330181,5058940594902287935],[4759200886165615660,14047932516327578328,13086455261210927405],[17230655055940252911,10079960231862789693,1594365425215384720],[12450224610855411435,6793022022333426780,3810635609165506132],[7869220775064086658,3175255887204963156,13219361725689881080],[10624982050920163244,12102988984123124133,8621395626272779527]], "path_len": 5}, {"layer": 1, "d": 2, "position": 198, "leaf": 49, "slot": 2, "values": [[3935900619226349263,1242728730961115505,5259089442116211009],[3425851199438451350,2149634388463401711,2660624055263107619],[17389106764791954576,12349010740432980330,10945228020128263917],[15743952789761425276,1575275858179672379,4552729722665455521]], "path_len": 3}, {"layer": 2, "d": 2, "position": 49, "leaf": 12, "slot": 1, "values": [[5464005601434038088,9623393316986421227,10756173414515652631],[15629554433951570575,5891404871893161211,7794506514771656936],[7015985412422436116,5314828133090116385,3339720643265786168],[7095733908393958914,18058755147947351090,12396916271309078570]], "path_len": 1}]}, + {"iota": 1108, "deep": [8291018792785974071,13313098309166568094,8280431289888479898], "deep_sym": [14797191581087246444,943987639731003521,7102902153805509629], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1108, "leaf": 138, "slot": 4, "values": [[10747780665270753536,14925858981123565357,14776533201595099206],[17020063819487208823,16331022987088535628,398851099833709126],[3905999722443358219,13174069017335826404,615555513443939273],[7848731174735099969,16244771701549524850,3504209930058176715],[16177707419406544780,5336690175849378686,14295699186837538335],[14276398242879772669,10348977498094124152,380521701586907949],[15354946610280677012,14044707150946569948,17571854873593814494],[12389508910263015150,16025161443698577861,4003448375894115960]], "path_len": 5}, {"layer": 1, "d": 2, "position": 138, "leaf": 34, "slot": 2, "values": [[12541072756489582885,2370820915463632688,2423396583266204055],[14905864220509753211,15950579206739519424,15082498997069827244],[9459878356316854591,1270075853426673136,4472301856924467933],[14393624207528009097,9675335233598348594,16693550050532620583]], "path_len": 3}, {"layer": 2, "d": 2, "position": 34, "leaf": 8, "slot": 2, "values": [[16818112822979374356,10274787015391993318,15763705279771580830],[7191284429859224732,4809043185931564649,14671147736651195233],[2802946927799549417,8037886970914238609,9324105614581397069],[720806501774538325,13075390526153910697,6384491353149171884]], "path_len": 1}]}, + {"iota": 1115, "deep": [6618438615200267976,4919274826576345125,4625388421371444138], "deep_sym": [9246029957586231281,17197016168898005971,16581617646226338233], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1115, "leaf": 139, "slot": 3, "values": [[17001607609996340984,13454749039895100314,3715210367178856146],[6048616277769393765,1359761812099145862,8711385731818791115],[3932895076288661499,17258907193956398903,11400500372312523510],[12054074072422017896,3208762764990612220,5562583925758922572],[13314015859018059125,7621965631850760423,11054467258190560956],[18038525678620396892,3154653838113744463,15429304576725024961],[15961703551772895141,9711979530086833226,6731603843790161535],[11068648140989596682,4028169998411965337,14241442597618133873]], "path_len": 5}, {"layer": 1, "d": 2, "position": 139, "leaf": 34, "slot": 3, "values": [[12541072756489582885,2370820915463632688,2423396583266204055],[14905864220509753211,15950579206739519424,15082498997069827244],[9459878356316854591,1270075853426673136,4472301856924467933],[14393624207528009097,9675335233598348594,16693550050532620583]], "path_len": 3}, {"layer": 2, "d": 2, "position": 34, "leaf": 8, "slot": 2, "values": [[16818112822979374356,10274787015391993318,15763705279771580830],[7191284429859224732,4809043185931564649,14671147736651195233],[2802946927799549417,8037886970914238609,9324105614581397069],[720806501774538325,13075390526153910697,6384491353149171884]], "path_len": 1}]}, + {"iota": 1531, "deep": [9728795865480759962,10841781668352654296,10796525908360120488], "deep_sym": [14058290127965822562,13000425636675524513,14945972201157220666], "terminal_position": 11, "layers": [{"layer": 0, "d": 3, "position": 1531, "leaf": 191, "slot": 3, "values": [[948368160510490102,8544727733740695269,10749830773933086533],[10329255878178745701,16086143103855673178,14825016990397793196],[9788573768211855305,1974783517146606653,7321026158536110064],[18390256767309975591,14744604900512807878,5586461636310614090],[17243629565902491525,5069704867446951940,15263602100351989401],[12555155414279618008,13856182192732480001,6153453073174628912],[348446857428337296,6537149763573446604,3301475865544646631],[8211285100903112410,13242877615395271090,5151960547612505286]], "path_len": 5}, {"layer": 1, "d": 2, "position": 191, "leaf": 47, "slot": 3, "values": [[9004309183751256646,15053749661805259059,13402872281440665070],[11923832368540281318,9591949487276575382,3941126143583950984],[10524734310662209668,18323881505728001248,6372778122146690452],[1004243276605533431,3591193302962306504,1633219252869578500]], "path_len": 3}, {"layer": 2, "d": 2, "position": 47, "leaf": 11, "slot": 3, "values": [[13252749493197899059,7111335394328122694,16103957492148269244],[15670868136619302923,10379423668602200154,74901909926248704],[10169946223356276752,13455008041475535374,5339865456753800954],[591266454968139898,799088598745050263,7430921739329049493]], "path_len": 1}]}, + {"iota": 460, "deep": [3946463641621599873,5430918586547480310,12399672165657981355], "deep_sym": [817130378730303002,18226081220269208290,10040121201250819828], "terminal_position": 3, "layers": [{"layer": 0, "d": 3, "position": 460, "leaf": 57, "slot": 4, "values": [[15528130404920832646,5215522714446081493,12271422738351742797],[6863031435719765099,8964286762487153093,3348744363473493650],[2929074367356164008,2538292072132929796,7267988012732332696],[16186424673451190780,12421741952053121662,7403524188509867498],[9913385481138587183,566253416801528561,4689471667889074830],[5238292704539165876,5207577595457270053,15319933152776760516],[988335866269748584,12704285650474491375,13454789435901030277],[5181234302079931035,16724170191637402584,9857139164589527058]], "path_len": 5}, {"layer": 1, "d": 2, "position": 57, "leaf": 14, "slot": 1, "values": [[6148049423775918455,14589867543249973290,3056806138608021102],[12775873754809006869,2779646161953595826,7537613261740435835],[14650203784677039238,2948257343798866283,2842367134151335097],[9538295325872013162,18409431940839324820,18140461979228732677]], "path_len": 3}, {"layer": 2, "d": 2, "position": 14, "leaf": 3, "slot": 2, "values": [[11401453414757530315,353357581965599259,15287411324344444169],[16460794964378913634,42195047452164617,805857385603866297],[8773174365564475115,10384145495722058796,3379422144880863992],[11394111484266117948,1699290548295696723,1801926384528077999]], "path_len": 1}]}, + {"iota": 21, "deep": [16751469336709747935,13034541952351682476,362160150163509523], "deep_sym": [13613547504591209906,17994713379913002098,12951980278075054529], "terminal_position": 0, "layers": [{"layer": 0, "d": 3, "position": 21, "leaf": 2, "slot": 5, "values": [[12229915140224745050,11914636115907519215,15454813039040787602],[11004608921862797938,2786446545964014409,11215296566347994755],[14247823219081093028,7678319371580752545,4364550269713641097],[4950501297074424090,3777394133863232678,3952696221507450235],[17403688435427697952,14643967294446729293,938567040701683587],[1347531407523399298,12307613284931727666,2603176576665318112],[17569334922626660479,15457741679174967682,2884291902000028503],[2613590051935242029,9966289619120623030,13359190776901017615]], "path_len": 5}, {"layer": 1, "d": 2, "position": 2, "leaf": 0, "slot": 2, "values": [[14271557498503267540,5604443783523955218,2419070109913866744],[5057225870813679079,6878920305747299227,7904171198740585785],[11637960830172701537,9246719637937614202,1888778944505873394],[14752417623798547784,12864596472359031008,17470784308582478716]], "path_len": 3}, {"layer": 2, "d": 2, "position": 0, "leaf": 0, "slot": 0, "values": [[16711837865871043867,7230876273111376341,3914710442871488734],[9427846744532214811,5523266006034183126,10995783862421668556],[6894989026912389225,5249821206546415796,2490306299428115481],[2435848346614272276,13191746087021246298,17137928531725839726]], "path_len": 1}]}, + {"iota": 1190, "deep": [10564639282863785564,8648928983583492162,1854090593019502747], "deep_sym": [14736828121689239151,13061678655855235343,595509014249069299], "terminal_position": 9, "layers": [{"layer": 0, "d": 3, "position": 1190, "leaf": 148, "slot": 6, "values": [[1912108142230074757,18120753521788756304,4744797074824315095],[7622973140663785730,3218517023600596379,12933172925186170580],[6856954486901287670,10911063608587332337,10495005156059272273],[3630161301090547043,9976280925750132817,14959174963198538503],[15849533635970265205,7931782409217351652,1711881977559092105],[4244786941653413432,611712464118926980,10550338065376625928],[4261656037681919894,13899927091162554149,4496214128402436523],[4758719564465786030,6242494762166384866,6922579272962812326]], "path_len": 5}, {"layer": 1, "d": 2, "position": 148, "leaf": 37, "slot": 0, "values": [[1720604404904246207,4456310064256824594,16023405838743839253],[12095426019459384804,3545515663528391901,13847932260916760557],[14481945053830228832,10032951144866827271,18307948857941228918],[9568183540619999479,6075525606092921498,9812470079212029532]], "path_len": 3}, {"layer": 2, "d": 2, "position": 37, "leaf": 9, "slot": 1, "values": [[4938759970149737351,5713750335509713787,7704473464848629755],[12178860564334633133,4505589151486207207,11019837447191491161],[759847948334892703,2198096190256262147,2258375761380806592],[8544214346083083966,2594628239276646728,10573244010893722073]], "path_len": 1}]}, + {"iota": 540, "deep": [13790780674944227860,7489226687845769437,18266880323214702999], "deep_sym": [4078467403041968014,11722414815152663777,6384647940485904195], "terminal_position": 4, "layers": [{"layer": 0, "d": 3, "position": 540, "leaf": 67, "slot": 4, "values": [[16038763184741478660,4860630134492478394,5827712014215397494],[8602567303568281597,6070847271420277083,4594582598770358286],[5742595377141191612,11862514134763543219,7170856629784425895],[3215088790510035073,3705899893413116883,13687575627121706169],[11324792041998818675,16480222931634912753,9694814253569073593],[7554074958443565868,10066082608075392955,3858077481277024958],[12771829728786218223,15277647901280526420,2963318663981532012],[10528238661142981081,7489180081743875754,5379717373430647217]], "path_len": 5}, {"layer": 1, "d": 2, "position": 67, "leaf": 16, "slot": 3, "values": [[7108793073240269897,13667909718173120897,14121100863859163201],[17341774714088985140,15724372809747486471,226187607144172421],[9549280667125660671,2332354502197878932,9627352183216333391],[4789373132786799347,10303671649960281535,1512145240231698617]], "path_len": 3}, {"layer": 2, "d": 2, "position": 16, "leaf": 4, "slot": 0, "values": [[13859184395281262686,3799190906283224991,18307316685430313213],[10009945053196490616,8817414741194159404,9457261446137636070],[8676620017703582792,17038654695824559897,8064727157295768002],[641207936648328848,6763031934725076030,4948104159318021490]], "path_len": 1}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_dp.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_dp.rkyv new file mode 100644 index 000000000..6ec585737 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_dp.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_pair.json b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_pair.json new file mode 100644 index 000000000..e003a144d --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_pair.json @@ -0,0 +1,49 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "rpx", + "format": "cap_pair", + "proof_rkyv": "d_proof_rpx_cap_pair.rkyv", + "proof_rkyv_len": 51752, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 20, + "grinding_factor": 0, + "coset_offset": 3, + "merkle_cap": "auto", + "trace_tree_depth": 11, + "trace_cap": 3, + "fri_tree_depths": [10, 9, 8, 7, 6, 5, 4], + "fri_caps": [3, 3, 3, 3, 3, 3, 3], + "legacy_encoding": true, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["a5cb9815a33628e2d9aee1104d2210e8066854dd38976a0143c7b0638504f9f3","5af88af0782998624ef05869a90b8e35e82a85d7da5c899ce3a0d8d1872a1ab0","97ae9eedee1a1b29b7eb30478eb3b8f118b84f3bd43e6352fbdac68314723730","2f2da5b7622cc6eb49677363305e2626fd79387dc9acaae9202b655cb84f1f8a","9fef7049e45b8f097dee76ba5feadc10268bb51f63e9336dfafc3ac54e3e3e38","c261958da4b13c4ce5dcf9cb19d42cf09808cb932dea3b9e69bdb7577f308167","2aa133f65dd602f13ab5b2961d2b96cc3f6daf6f106e9783939c69716b9d2ab9"], + "zetas": [[4735330965523630181,1034630526833404286,12017969954712239940],[16665743570319646148,16897879252278531211,10291861723093761662],[10619368815145924427,1493089516910409884,14431758427697423319],[9552622148858278301,6488516694070886107,4893272118711353122],[2783515638190829017,9945112524553572548,15631202162117197821],[17815262798501899446,18394714429174366312,8636369618480887460],[11856812424473920575,10766055906630249609,2957922871886357218],[8171264462115707646,14626134561370321125,16568024845886241762]], + "terminal_coeffs": [[3714161951696662500,2595793324825979078,3565379477475041685],[4424026265791135346,6558459514194683116,753777937513084834],[7727091673734829526,10561609288187203284,15868472042909273283],[7864830287677944053,6520068215425390864,4795411284951093801]], + "queries_detail": [ + {"iota": 907, "deep": [3040718383397280274,11956941222209451830,13732184933327600162], "deep_sym": [14770993481378414249,10902020514396191223,13023915065759495237], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 907, "leaf": 453, "slot": 1, "values": [[10738449088863093738,17001367698800175667,872083450534898496]], "path_len": 15}, {"layer": 1, "d": 1, "position": 453, "leaf": 226, "slot": 1, "values": [[8949331791643690447,11945875389884147400,4863937416666523377]], "path_len": 14}, {"layer": 2, "d": 1, "position": 226, "leaf": 113, "slot": 0, "values": [[6531312029425646452,8614572489917572820,5793377902239563396]], "path_len": 13}, {"layer": 3, "d": 1, "position": 113, "leaf": 56, "slot": 1, "values": [[13245888542464671401,9490131273601039377,5473879260557931677]], "path_len": 12}, {"layer": 4, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[967364531071989975,2293195037664380504,8237422110493151214]], "path_len": 11}, {"layer": 5, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[3068323341190833160,2495604132745403317,440675983381132721]], "path_len": 10}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[6235735872747817365,1215148853071865642,1291260655626560448]], "path_len": 9}]}, + {"iota": 327, "deep": [6648971487643812783,3504019026315364339,5553986533175031714], "deep_sym": [4847162544338375188,3760822623165876653,18385909206353757824], "terminal_position": 2, "layers": [{"layer": 0, "d": 1, "position": 327, "leaf": 163, "slot": 1, "values": [[3102706233933768041,2061658593148860966,2937765886239263664]], "path_len": 7}, {"layer": 1, "d": 1, "position": 163, "leaf": 81, "slot": 1, "values": [[5351203088071617904,5190285090595573022,679443460868787999]], "path_len": 6}, {"layer": 2, "d": 1, "position": 81, "leaf": 40, "slot": 1, "values": [[6813072834546256573,17560727769766864490,11355003854934910309]], "path_len": 5}, {"layer": 3, "d": 1, "position": 40, "leaf": 20, "slot": 0, "values": [[4276057153471798645,16589259977369031850,10213981092871167197]], "path_len": 4}, {"layer": 4, "d": 1, "position": 20, "leaf": 10, "slot": 0, "values": [[6724049492496023179,14516868345363052182,5463760551208548401]], "path_len": 3}, {"layer": 5, "d": 1, "position": 10, "leaf": 5, "slot": 0, "values": [[872892273747685357,17097460053185137664,7011811299561387579]], "path_len": 2}, {"layer": 6, "d": 1, "position": 5, "leaf": 2, "slot": 1, "values": [[6972245892337738186,7308248010937532343,15369312713036268331]], "path_len": 1}]}, + {"iota": 1055, "deep": [817615283715329005,16733328116567315164,13240493922581948560], "deep_sym": [1731008489953378402,3915464684156498806,11643985226885705726], "terminal_position": 8, "layers": [{"layer": 0, "d": 1, "position": 1055, "leaf": 527, "slot": 1, "values": [[7812814834329037929,5936428735545781918,6769737470751296857]], "path_len": 7}, {"layer": 1, "d": 1, "position": 527, "leaf": 263, "slot": 1, "values": [[17350970259606761374,14937434176606905201,2436331477479557398]], "path_len": 6}, {"layer": 2, "d": 1, "position": 263, "leaf": 131, "slot": 1, "values": [[11807655768007569086,16744789612728260734,4620195842063357801]], "path_len": 5}, {"layer": 3, "d": 1, "position": 131, "leaf": 65, "slot": 1, "values": [[13354553806338021504,6368498475783127995,1444223788932639250]], "path_len": 4}, {"layer": 4, "d": 1, "position": 65, "leaf": 32, "slot": 1, "values": [[2595125021894609724,3173778131451323870,2008467966233623536]], "path_len": 3}, {"layer": 5, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[734813548736948502,6040967620827257108,7735869121954590664]], "path_len": 2}, {"layer": 6, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[6896385518190873683,9696907498203743737,11255624105341837683]], "path_len": 1}]}, + {"iota": 490, "deep": [5333555309022376619,6497848330298835332,16563034812639328791], "deep_sym": [1540291644563635092,1259808496858708141,6436795295029442319], "terminal_position": 3, "layers": [{"layer": 0, "d": 1, "position": 490, "leaf": 245, "slot": 0, "values": [[16165997071554694371,7809042381953747683,14845781732875929587]], "path_len": 7}, {"layer": 1, "d": 1, "position": 245, "leaf": 122, "slot": 1, "values": [[4202361345781088339,2870932090735936211,12730858377474507808]], "path_len": 6}, {"layer": 2, "d": 1, "position": 122, "leaf": 61, "slot": 0, "values": [[4487121043982363903,7222683238575461495,4659152637789648093]], "path_len": 5}, {"layer": 3, "d": 1, "position": 61, "leaf": 30, "slot": 1, "values": [[9216116296820106680,16579878421713900505,13631642496076283964]], "path_len": 4}, {"layer": 4, "d": 1, "position": 30, "leaf": 15, "slot": 0, "values": [[8296754296575280656,4566791758227172626,8560225543529906602]], "path_len": 3}, {"layer": 5, "d": 1, "position": 15, "leaf": 7, "slot": 1, "values": [[16151268240327736551,6959880367891302202,12923693381147279704]], "path_len": 2}, {"layer": 6, "d": 1, "position": 7, "leaf": 3, "slot": 1, "values": [[2041773572074858086,8235519873000800075,6801272137943839603]], "path_len": 1}]}, + {"iota": 1293, "deep": [13202019215294755661,792069985274517970,1837919879426916066], "deep_sym": [11004593316219637619,11457286750131782961,4287723567360687560], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1293, "leaf": 646, "slot": 1, "values": [[11542857627666095499,5331048150248723898,10911724239825767773]], "path_len": 7}, {"layer": 1, "d": 1, "position": 646, "leaf": 323, "slot": 0, "values": [[4284204476542245753,10807736373362441887,8541842606839334792]], "path_len": 6}, {"layer": 2, "d": 1, "position": 323, "leaf": 161, "slot": 1, "values": [[6049964432242287280,10740098497842851829,11762819817772120687]], "path_len": 5}, {"layer": 3, "d": 1, "position": 161, "leaf": 80, "slot": 1, "values": [[3659481731940555489,7978114488151616121,2434855029340771432]], "path_len": 4}, {"layer": 4, "d": 1, "position": 80, "leaf": 40, "slot": 0, "values": [[3062799608155256757,10298993203696925632,4135505462794541032]], "path_len": 3}, {"layer": 5, "d": 1, "position": 40, "leaf": 20, "slot": 0, "values": [[208230508203853506,15333826890764774037,18057329976005815313]], "path_len": 2}, {"layer": 6, "d": 1, "position": 20, "leaf": 10, "slot": 0, "values": [[8675393632150954143,13193766732232908346,11058925871258175295]], "path_len": 1}]}, + {"iota": 1232, "deep": [5100849131875730764,13311523601572949287,17754933639302584926], "deep_sym": [9125629300697698907,6092338751183093721,4621944178520612389], "terminal_position": 9, "layers": [{"layer": 0, "d": 1, "position": 1232, "leaf": 616, "slot": 0, "values": [[16018690189368362260,420945639044652702,6800220696365653435]], "path_len": 7}, {"layer": 1, "d": 1, "position": 616, "leaf": 308, "slot": 0, "values": [[423722432265633255,17388955584979232176,11808809124110976521]], "path_len": 6}, {"layer": 2, "d": 1, "position": 308, "leaf": 154, "slot": 0, "values": [[6224050008368218659,7793526065195312542,7754584313398499093]], "path_len": 5}, {"layer": 3, "d": 1, "position": 154, "leaf": 77, "slot": 0, "values": [[5353496693137660222,1400459084141815741,12865696614160780188]], "path_len": 4}, {"layer": 4, "d": 1, "position": 77, "leaf": 38, "slot": 1, "values": [[10248192692058468149,13403323599212523055,1065945654017210721]], "path_len": 3}, {"layer": 5, "d": 1, "position": 38, "leaf": 19, "slot": 0, "values": [[15747202659571070477,11582933239615491642,9692129310833850457]], "path_len": 2}, {"layer": 6, "d": 1, "position": 19, "leaf": 9, "slot": 1, "values": [[7595395067547975575,4679686797858417144,7459990344248030932]], "path_len": 1}]}, + {"iota": 906, "deep": [16987545712744995801,9134272306172516667,6154404976421516758], "deep_sym": [1567212379715838741,18001184530175578021,18020286302555685985], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 906, "leaf": 453, "slot": 0, "values": [[208147472821821449,11920382692700401347,10451650462477063379]], "path_len": 7}, {"layer": 1, "d": 1, "position": 453, "leaf": 226, "slot": 1, "values": [[8949331791643690447,11945875389884147400,4863937416666523377]], "path_len": 6}, {"layer": 2, "d": 1, "position": 226, "leaf": 113, "slot": 0, "values": [[6531312029425646452,8614572489917572820,5793377902239563396]], "path_len": 5}, {"layer": 3, "d": 1, "position": 113, "leaf": 56, "slot": 1, "values": [[13245888542464671401,9490131273601039377,5473879260557931677]], "path_len": 4}, {"layer": 4, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[967364531071989975,2293195037664380504,8237422110493151214]], "path_len": 3}, {"layer": 5, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[3068323341190833160,2495604132745403317,440675983381132721]], "path_len": 2}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[6235735872747817365,1215148853071865642,1291260655626560448]], "path_len": 1}]}, + {"iota": 1445, "deep": [8238996569358525839,7400933341107165980,13539245176576736813], "deep_sym": [16723622915527146625,4997619439840450854,16643934399247561766], "terminal_position": 11, "layers": [{"layer": 0, "d": 1, "position": 1445, "leaf": 722, "slot": 1, "values": [[748244684854860367,15581535348713203250,14488124003887067430]], "path_len": 7}, {"layer": 1, "d": 1, "position": 722, "leaf": 361, "slot": 0, "values": [[12029438690276759030,14110935376777698328,2444834449659203005]], "path_len": 6}, {"layer": 2, "d": 1, "position": 361, "leaf": 180, "slot": 1, "values": [[1344770080555719020,4450866369841668538,8535818524384001641]], "path_len": 5}, {"layer": 3, "d": 1, "position": 180, "leaf": 90, "slot": 0, "values": [[1627076855653398376,3342203343274309383,17248029193346284789]], "path_len": 4}, {"layer": 4, "d": 1, "position": 90, "leaf": 45, "slot": 0, "values": [[9174377671867584593,4419776172399299719,8816228065868848859]], "path_len": 3}, {"layer": 5, "d": 1, "position": 45, "leaf": 22, "slot": 1, "values": [[11828230151300071203,11852837209955541876,18387241096789405769]], "path_len": 2}, {"layer": 6, "d": 1, "position": 22, "leaf": 11, "slot": 0, "values": [[4497056867330769201,5556595583679787664,7599353405711249964]], "path_len": 1}]}, + {"iota": 1900, "deep": [7431512126338372394,12901884170471295656,13229107773857763089], "deep_sym": [4305251512907028158,11796135359754093295,10529446823649361544], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1900, "leaf": 950, "slot": 0, "values": [[10597607009932306054,2041180774756384802,15073779384058885182]], "path_len": 7}, {"layer": 1, "d": 1, "position": 950, "leaf": 475, "slot": 0, "values": [[7594923001698684643,17769333214797688037,9184058274024548434]], "path_len": 6}, {"layer": 2, "d": 1, "position": 475, "leaf": 237, "slot": 1, "values": [[6570785028429573968,7305051340909591677,17087044582461523085]], "path_len": 5}, {"layer": 3, "d": 1, "position": 237, "leaf": 118, "slot": 1, "values": [[15847483128935587866,9558607112153220780,8383580498378797538]], "path_len": 4}, {"layer": 4, "d": 1, "position": 118, "leaf": 59, "slot": 0, "values": [[13707884290003644050,6351837470214772186,5687335870977301326]], "path_len": 3}, {"layer": 5, "d": 1, "position": 59, "leaf": 29, "slot": 1, "values": [[2614229400037681378,4408538218630961289,11581585520222664939]], "path_len": 2}, {"layer": 6, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[3713361366251623804,5215610397924481652,3743305604543961985]], "path_len": 1}]}, + {"iota": 716, "deep": [3889282557667010008,16658559808392896075,4479839851402389919], "deep_sym": [16572265183460544525,3676808044400424795,6222338751362850681], "terminal_position": 5, "layers": [{"layer": 0, "d": 1, "position": 716, "leaf": 358, "slot": 0, "values": [[12116550705477708982,16724229524281302287,5666602575093991096]], "path_len": 7}, {"layer": 1, "d": 1, "position": 358, "leaf": 179, "slot": 0, "values": [[8804090231151493253,18093309713086984478,15588806257211634066]], "path_len": 6}, {"layer": 2, "d": 1, "position": 179, "leaf": 89, "slot": 1, "values": [[10899689051865181242,10286430280999561071,10374738856475973108]], "path_len": 5}, {"layer": 3, "d": 1, "position": 89, "leaf": 44, "slot": 1, "values": [[15859176201743508538,9966478424929065552,17216980535016118127]], "path_len": 4}, {"layer": 4, "d": 1, "position": 44, "leaf": 22, "slot": 0, "values": [[11294734992695880275,17781633920126111425,6521795554173378949]], "path_len": 3}, {"layer": 5, "d": 1, "position": 22, "leaf": 11, "slot": 0, "values": [[12562602703896107334,12802966902777675319,12362156114430019523]], "path_len": 2}, {"layer": 6, "d": 1, "position": 11, "leaf": 5, "slot": 1, "values": [[7956105874520444819,17097517896355943076,9000902050689723599]], "path_len": 1}]}, + {"iota": 1338, "deep": [12916742567028440091,13262759860843631686,6490219764970839891], "deep_sym": [4390440514850516016,11679559744418381099,6473519317075923322], "terminal_position": 10, "layers": [{"layer": 0, "d": 1, "position": 1338, "leaf": 669, "slot": 0, "values": [[12449613922291734239,18143330045847154993,11555008710041017422]], "path_len": 7}, {"layer": 1, "d": 1, "position": 669, "leaf": 334, "slot": 1, "values": [[15840889759225825365,16154054991495270570,8346715740435189744]], "path_len": 6}, {"layer": 2, "d": 1, "position": 334, "leaf": 167, "slot": 0, "values": [[15446200573881484727,493204182891896018,5570847731959122629]], "path_len": 5}, {"layer": 3, "d": 1, "position": 167, "leaf": 83, "slot": 1, "values": [[11306093278808513099,5005574932225287058,3712866700044915855]], "path_len": 4}, {"layer": 4, "d": 1, "position": 83, "leaf": 41, "slot": 1, "values": [[9898632816009731525,11460701178569989294,5910734408848020819]], "path_len": 3}, {"layer": 5, "d": 1, "position": 41, "leaf": 20, "slot": 1, "values": [[13823564168872677235,12873871450576728205,10000039197109786678]], "path_len": 2}, {"layer": 6, "d": 1, "position": 20, "leaf": 10, "slot": 0, "values": [[8675393632150954143,13193766732232908346,11058925871258175295]], "path_len": 1}]}, + {"iota": 316, "deep": [9809389796479279253,2609724777592635053,6358204434308916543], "deep_sym": [2090293030578564485,17046018365802373505,710220348793320602], "terminal_position": 2, "layers": [{"layer": 0, "d": 1, "position": 316, "leaf": 158, "slot": 0, "values": [[14340795179817377614,13720843975494266939,16347076499677306763]], "path_len": 7}, {"layer": 1, "d": 1, "position": 158, "leaf": 79, "slot": 0, "values": [[13483727959324373428,17135481261001400981,5488797648533180440]], "path_len": 6}, {"layer": 2, "d": 1, "position": 79, "leaf": 39, "slot": 1, "values": [[6265626914156287390,16951504734271861328,13384183869976926336]], "path_len": 5}, {"layer": 3, "d": 1, "position": 39, "leaf": 19, "slot": 1, "values": [[2974214004291265886,10871188085049143848,7536558763607470101]], "path_len": 4}, {"layer": 4, "d": 1, "position": 19, "leaf": 9, "slot": 1, "values": [[1709687672422415859,7196711959144469574,4032055591279181426]], "path_len": 3}, {"layer": 5, "d": 1, "position": 9, "leaf": 4, "slot": 1, "values": [[2292456446077332702,16811939064872409390,9436216556421672528]], "path_len": 2}, {"layer": 6, "d": 1, "position": 4, "leaf": 2, "slot": 0, "values": [[8757596275494264689,4441538513452859255,12082179960950577135]], "path_len": 1}]}, + {"iota": 242, "deep": [18073784541193312396,15542186237964955282,555832555083137815], "deep_sym": [14622721919178152000,11799583864430949889,14403748362265045306], "terminal_position": 1, "layers": [{"layer": 0, "d": 1, "position": 242, "leaf": 121, "slot": 0, "values": [[7347252276027670505,14027197946541438400,14584497917100139601]], "path_len": 7}, {"layer": 1, "d": 1, "position": 121, "leaf": 60, "slot": 1, "values": [[8302613772554768521,13092101283703885902,15794297093428991085]], "path_len": 6}, {"layer": 2, "d": 1, "position": 60, "leaf": 30, "slot": 0, "values": [[12473978045343060063,10453235614622229329,3311550628499108388]], "path_len": 5}, {"layer": 3, "d": 1, "position": 30, "leaf": 15, "slot": 0, "values": [[15262068716790978691,9813059787786197463,17791554441717694526]], "path_len": 4}, {"layer": 4, "d": 1, "position": 15, "leaf": 7, "slot": 1, "values": [[9382801202838763478,521562019286824498,8986832934377767675]], "path_len": 3}, {"layer": 5, "d": 1, "position": 7, "leaf": 3, "slot": 1, "values": [[8128307510882286524,10821318725892370323,13877665066000822352]], "path_len": 2}, {"layer": 6, "d": 1, "position": 3, "leaf": 1, "slot": 1, "values": [[4491081093077012764,4892525095907921320,4263981023199701481]], "path_len": 1}]}, + {"iota": 948, "deep": [12443299642485287037,11141614935052770991,3956088967515912652], "deep_sym": [11552668708672716993,7022530386125334965,11013481679675578715], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 948, "leaf": 474, "slot": 0, "values": [[3313509122448078552,6828550099476726627,16726730109080172301]], "path_len": 7}, {"layer": 1, "d": 1, "position": 474, "leaf": 237, "slot": 0, "values": [[6469422069387281069,5851964510015292197,6102876639432844707]], "path_len": 6}, {"layer": 2, "d": 1, "position": 237, "leaf": 118, "slot": 1, "values": [[8504627830733515497,3203577139713973580,17102368772612150880]], "path_len": 5}, {"layer": 3, "d": 1, "position": 118, "leaf": 59, "slot": 0, "values": [[16130295155283771788,17235604904183598091,4187471025053003634]], "path_len": 4}, {"layer": 4, "d": 1, "position": 59, "leaf": 29, "slot": 1, "values": [[3962084612706899730,4714968921900557097,709166559014599792]], "path_len": 3}, {"layer": 5, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[8060973025226066530,6562800565017872823,17062166073441934934]], "path_len": 2}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[6235735872747817365,1215148853071865642,1291260655626560448]], "path_len": 1}]}, + {"iota": 955, "deep": [9748633164615405044,15200968922465510431,4328658426123658909], "deep_sym": [1010012317306963274,17571607453822902460,6536507881715918866], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 955, "leaf": 477, "slot": 1, "values": [[1238885797271121897,17869821964957541322,51871560878855088]], "path_len": 7}, {"layer": 1, "d": 1, "position": 477, "leaf": 238, "slot": 1, "values": [[11770897055806471365,12170177732376960747,365830011230340707]], "path_len": 6}, {"layer": 2, "d": 1, "position": 238, "leaf": 119, "slot": 0, "values": [[5243897450647853820,3709982785023675266,15992418649835130718]], "path_len": 5}, {"layer": 3, "d": 1, "position": 119, "leaf": 59, "slot": 1, "values": [[331311268110073093,7642796023034543418,16183959937845547380]], "path_len": 4}, {"layer": 4, "d": 1, "position": 59, "leaf": 29, "slot": 1, "values": [[3962084612706899730,4714968921900557097,709166559014599792]], "path_len": 3}, {"layer": 5, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[8060973025226066530,6562800565017872823,17062166073441934934]], "path_len": 2}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[6235735872747817365,1215148853071865642,1291260655626560448]], "path_len": 1}]}, + {"iota": 425, "deep": [5220382834730846224,11698870976303831066,12172341881198518479], "deep_sym": [10761357836357599593,17991906410856106278,11940257533055125249], "terminal_position": 3, "layers": [{"layer": 0, "d": 1, "position": 425, "leaf": 212, "slot": 1, "values": [[18297288576489106507,17090009343077141516,1372897608266065964]], "path_len": 7}, {"layer": 1, "d": 1, "position": 212, "leaf": 106, "slot": 0, "values": [[13641345983585634201,12358551939098946232,12614312160614023414]], "path_len": 6}, {"layer": 2, "d": 1, "position": 106, "leaf": 53, "slot": 0, "values": [[13491220669646918991,3519918468530596688,1717073988549802829]], "path_len": 5}, {"layer": 3, "d": 1, "position": 53, "leaf": 26, "slot": 1, "values": [[12453130284583355935,1672709573116503708,6358973158760603860]], "path_len": 4}, {"layer": 4, "d": 1, "position": 26, "leaf": 13, "slot": 0, "values": [[13023521510587738275,6244946759545711611,2933419860181065892]], "path_len": 3}, {"layer": 5, "d": 1, "position": 13, "leaf": 6, "slot": 1, "values": [[8746702812610050947,17055089583592083208,10622082825270590110]], "path_len": 2}, {"layer": 6, "d": 1, "position": 6, "leaf": 3, "slot": 0, "values": [[3528697806224570571,16712965396148344850,11633688770159473649]], "path_len": 1}]}, + {"iota": 1024, "deep": [8467191801933543488,2838244879671958413,253003732133238029], "deep_sym": [8674660436931916443,2580563650129586025,4381248104485517596], "terminal_position": 8, "layers": [{"layer": 0, "d": 1, "position": 1024, "leaf": 512, "slot": 0, "values": [[552255252847067932,7290890199123029178,15598943983249374509]], "path_len": 7}, {"layer": 1, "d": 1, "position": 512, "leaf": 256, "slot": 0, "values": [[16489740470231076529,12150094973064418068,3809331140355786413]], "path_len": 6}, {"layer": 2, "d": 1, "position": 256, "leaf": 128, "slot": 0, "values": [[6692803711311527429,9433701761439709710,11396497374487473788]], "path_len": 5}, {"layer": 3, "d": 1, "position": 128, "leaf": 64, "slot": 0, "values": [[2851110493503553692,17864469541465912512,14423609184050031580]], "path_len": 4}, {"layer": 4, "d": 1, "position": 64, "leaf": 32, "slot": 0, "values": [[1878384626655112620,13903070236496566397,8873806899472591798]], "path_len": 3}, {"layer": 5, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[734813548736948502,6040967620827257108,7735869121954590664]], "path_len": 2}, {"layer": 6, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[6896385518190873683,9696907498203743737,11255624105341837683]], "path_len": 1}]}, + {"iota": 1865, "deep": [4212103210510884997,7035972176942140753,17306526335631249262], "deep_sym": [5983969155960921170,7716709446605112430,3507511199173005913], "terminal_position": 14, "layers": [{"layer": 0, "d": 1, "position": 1865, "leaf": 932, "slot": 1, "values": [[5072085790887243848,7949766786871583993,15324686885973668545]], "path_len": 7}, {"layer": 1, "d": 1, "position": 932, "leaf": 466, "slot": 0, "values": [[8531984238557370083,9886895154897844227,7323311592408787298]], "path_len": 6}, {"layer": 2, "d": 1, "position": 466, "leaf": 233, "slot": 0, "values": [[2262375630065310858,8475167051901754791,4082565711443850205]], "path_len": 5}, {"layer": 3, "d": 1, "position": 233, "leaf": 116, "slot": 1, "values": [[12487752698509265952,9985264885850011293,7003863281393010368]], "path_len": 4}, {"layer": 4, "d": 1, "position": 116, "leaf": 58, "slot": 0, "values": [[16471209112474711913,15774800313780824510,5119409826517069109]], "path_len": 3}, {"layer": 5, "d": 1, "position": 58, "leaf": 29, "slot": 0, "values": [[11193987865309744766,16476775623635910867,11903106571698127077]], "path_len": 2}, {"layer": 6, "d": 1, "position": 29, "leaf": 14, "slot": 1, "values": [[3713361366251623804,5215610397924481652,3743305604543961985]], "path_len": 1}]}, + {"iota": 520, "deep": [3002569401286487231,2687094515823569584,10351005430430287996], "deep_sym": [5657188868044694823,1228961647863731570,2179735891322572965], "terminal_position": 4, "layers": [{"layer": 0, "d": 1, "position": 520, "leaf": 260, "slot": 0, "values": [[6120686149557313247,9185726802812345144,11610026896189521524]], "path_len": 7}, {"layer": 1, "d": 1, "position": 260, "leaf": 130, "slot": 0, "values": [[17876923936705498700,16905214363602848937,13102287876002453738]], "path_len": 6}, {"layer": 2, "d": 1, "position": 130, "leaf": 65, "slot": 0, "values": [[1998095284741463700,706089076349940846,6176734019944784667]], "path_len": 5}, {"layer": 3, "d": 1, "position": 65, "leaf": 32, "slot": 1, "values": [[15588686557797525038,6358534243003599342,16114922279370500069]], "path_len": 4}, {"layer": 4, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[357439278952587411,537701478644505544,1654234603253554058]], "path_len": 3}, {"layer": 5, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[9198749334315201944,7219853247166450147,146514115183690041]], "path_len": 2}, {"layer": 6, "d": 1, "position": 8, "leaf": 4, "slot": 0, "values": [[5723305765232985900,4398833039615894329,17495563933725145151]], "path_len": 1}]}, + {"iota": 517, "deep": [14456110041334564788,13307720518286312411,14751228950079222761], "deep_sym": [13824372239230008108,620968933146506970,13902742496082168769], "terminal_position": 4, "layers": [{"layer": 0, "d": 1, "position": 517, "leaf": 258, "slot": 1, "values": [[3023625249063251227,17529527820976626583,16522680750687216278]], "path_len": 7}, {"layer": 1, "d": 1, "position": 258, "leaf": 129, "slot": 0, "values": [[3498053157125553740,17589795835218554013,18370974423581492306]], "path_len": 6}, {"layer": 2, "d": 1, "position": 129, "leaf": 64, "slot": 1, "values": [[753327007082907383,14955370069137255706,10625147434646339336]], "path_len": 5}, {"layer": 3, "d": 1, "position": 64, "leaf": 32, "slot": 0, "values": [[7989894481772335406,16901511694245139737,2822325316258093644]], "path_len": 4}, {"layer": 4, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[357439278952587411,537701478644505544,1654234603253554058]], "path_len": 3}, {"layer": 5, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[9198749334315201944,7219853247166450147,146514115183690041]], "path_len": 2}, {"layer": 6, "d": 1, "position": 8, "leaf": 4, "slot": 0, "values": [[5723305765232985900,4398833039615894329,17495563933725145151]], "path_len": 1}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_pair.rkyv new file mode 100644 index 000000000..dadf62917 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_cap_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp.json b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp.json new file mode 100644 index 000000000..93eb41b19 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "rpx", + "format": "dp", + "proof_rkyv": "d_proof_rpx_dp.rkyv", + "proof_rkyv_len": 8488, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 2], + "fri_roots": ["b8cd71d876dd084c3fba58b6a1b1788b09ee71b58bd373a3b0c32da0c88abac4","8abd4354863193215d65af7903d5f53018e4b2076a5d160d8197c0df719c33dc","5ba58234c6bd1059d39526597538aa208c0fce55ac284c9b1abcbfc4bc87fcb7"], + "zetas": [[4735330965523630181,1034630526833404286,12017969954712239940],[7889074366333103969,4290811767201827376,14455537773263474986],[16567739822379498242,5753162788299774204,5950576806486104926],[3148119476643166323,15342831354566172589,16163821384909909157]], + "terminal_coeffs": [[12646447477222048401,12937374675136009352,16549558038379651479],[2564516689604577222,14255657332782844950,7303342851315364550],[7904019038462202246,10880807545931735486,15264205294200432227],[13482626913175767796,15717304750858041741,4892518987751974292]], + "queries_detail": [ + {"iota": 1095, "deep": [15404367171170966026,18436452028196411499,5024906168965672354], "deep_sym": [16265809848131463264,11495087467666035873,7459148639182883977], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1095, "leaf": 136, "slot": 7, "values": [[2406839404446874184,14378864393598774400,9244658727562446653],[7310068857272658177,6690242714355677003,16014750232870151724],[16918409907732086886,2943891056625643727,17633501031830476314],[12754819164990375128,11705998079299817355,11740835587370543910],[2273041869480575555,1772390107814740151,15057029391149023100],[11526866775207687538,5051892843389067913,18351320525206389605],[12623868928666452372,4025975981633944670,16736429527117892317],[3360740073930066434,11793243838384529871,16278212037669344741]], "path_len": 8}, {"layer": 1, "d": 2, "position": 136, "leaf": 34, "slot": 0, "values": [[12541072756489582885,2370820915463632688,2423396583266204055],[14905864220509753211,15950579206739519424,15082498997069827244],[9459878356316854591,1270075853426673136,4472301856924467933],[14393624207528009097,9675335233598348594,16693550050532620583]], "path_len": 6}, {"layer": 2, "d": 2, "position": 34, "leaf": 8, "slot": 2, "values": [[16818112822979374356,10274787015391993318,15763705279771580830],[7191284429859224732,4809043185931564649,14671147736651195233],[2802946927799549417,8037886970914238609,9324105614581397069],[720806501774538325,13075390526153910697,6384491353149171884]], "path_len": 4}]}, + {"iota": 1336, "deep": [9782001122439711942,9555857402003070809,5596777784231506518], "deep_sym": [2420617880218855450,11218452639813192342,8178398275584052689], "terminal_position": 10, "layers": [{"layer": 0, "d": 3, "position": 1336, "leaf": 167, "slot": 0, "values": [[2041956337797731597,4112831764093230891,11432379156394499095],[8144473764935818890,6978818592127372038,13240419817221723331],[4510585753595495978,7259665793084123234,5963491368598554412],[12449613922291734239,18143330045847154993,11555008710041017422],[13423943835517857770,6610089783383402616,14732195742916433439],[3080128605461466890,10236270832537313262,8257530437238638301],[8362553719204526827,2153485092467020843,4632353622181731171],[11789751946472833749,6000336259605467995,973022182701680872]], "path_len": 8}, {"layer": 1, "d": 2, "position": 167, "leaf": 41, "slot": 3, "values": [[1806710437233960703,865918887752352625,6749716608005253549],[4630423819292356457,16185182827820005660,8859790871695731864],[642687296268297522,15932233592480046187,1035347428796527533],[1571642650814870704,5246859957097853911,1771889554939298762]], "path_len": 6}, {"layer": 2, "d": 2, "position": 41, "leaf": 10, "slot": 1, "values": [[16605815559724387167,7600882892259287443,6994922477906460043],[14201299358752116799,5717899003132557091,9025489995620184926],[15619332420018523266,1595985739793856288,4969668978550259454],[6667880876193245735,12155122404735786091,14288219524612442946]], "path_len": 4}]}, + {"iota": 396, "deep": [1195167017398997224,3684677677763618554,1602181459315078555], "deep_sym": [17092653866053864012,10319696574527941179,65705194922387228], "terminal_position": 3, "layers": [{"layer": 0, "d": 3, "position": 396, "leaf": 49, "slot": 4, "values": [[11065792751948336436,6945821278266161605,695451384357543318],[8408178489419937465,2838485655880223095,8492969326019934396],[265145725283343233,10180163860108398826,11843491620723569992],[15123282416936963659,17530049459658255167,8917537469248528646],[739600072118952183,889941540565197657,4394371451414923712],[15901449078141666097,9754018756689853016,12057470623441911843],[16131822616193081053,4052660922016275605,6765793682960198828],[9270165410843091007,10965847749993617830,14798583832773956809]], "path_len": 8}, {"layer": 1, "d": 2, "position": 49, "leaf": 12, "slot": 1, "values": [[3895026366660990149,2600803056632231256,4321642801809818286],[12836403324608808788,172846604945358907,7659991926477877030],[90956911091737129,4468391866397480991,9108435893769413449],[15315665520481292732,6371916974269842650,7526540926329622480]], "path_len": 6}, {"layer": 2, "d": 2, "position": 12, "leaf": 3, "slot": 0, "values": [[11401453414757530315,353357581965599259,15287411324344444169],[16460794964378913634,42195047452164617,805857385603866297],[8773174365564475115,10384145495722058796,3379422144880863992],[11394111484266117948,1699290548295696723,1801926384528077999]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp.rkyv new file mode 100644 index 000000000..ce6c014db Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp_3_1_3.json b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp_3_1_3.json new file mode 100644 index 000000000..3a6256410 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp_3_1_3.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "rpx", + "format": "dp_3_1_3", + "proof_rkyv": "d_proof_rpx_dp_3_1_3.rkyv", + "proof_rkyv_len": 8728, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 1, 3], + "fri_roots": ["b8cd71d876dd084c3fba58b6a1b1788b09ee71b58bd373a3b0c32da0c88abac4","4e3dc7f5aa11f888402fd6360c83de194bdb5f026d1e18197efa932467225837","c697839c9a24596be7c0905e049be88be18edded924fafa042fa6c04aff41539"], + "zetas": [[4735330965523630181,1034630526833404286,12017969954712239940],[7889074366333103969,4290811767201827376,14455537773263474986],[7770302924502205803,17599204623846053939,9971594608079091649],[17586252287694023458,7259590326536071696,12289082985476495883]], + "terminal_coeffs": [[6545599878510160802,16048981941637217531,11388709074500405832],[14061425267190733018,15002089986646168363,58669014784357566],[2413176590794729875,18383285226207373965,3934831234105776640],[2976777462567838906,972764554709234443,8590380189090386920]], + "queries_detail": [ + {"iota": 1053, "deep": [10250707639143879807,8370264329266959220,3378560773770382438], "deep_sym": [9932815673506821976,2964873329768126194,1815148923933943615], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 1053, "leaf": 131, "slot": 5, "values": [[8206566942588720809,17872933398873708985,7115180618693784557],[2678263910421709698,8346434880095868658,6107710828755343595],[16155660615844807708,15139547910119346356,10456400642597445409],[6862225755362317604,8468614376293384197,4418093324826000636],[3436445875439159867,9162846014864517992,17418766279869328368],[4884572017146877890,13142482661507438973,8397595055179396044],[7812814834329037929,5936428735545781918,6769737470751296857],[5571634424827863459,16952340246853650937,3860405205017447442]], "path_len": 8}, {"layer": 1, "d": 1, "position": 131, "leaf": 65, "slot": 1, "values": [[13279332016024174713,10180249446915101956,14037762681904687442],[629567766354029343,2428986310818076928,3442626561722631918]], "path_len": 7}, {"layer": 2, "d": 3, "position": 65, "leaf": 8, "slot": 1, "values": [[16234327972113548284,7197673276934127279,8012846128365704540],[15705483126643608833,9148138369265927751,4497445583271196018],[1116871917513432828,7754822209782634050,13152119519303694369],[8099659913026938434,5657455326385259056,15838886403916165803],[8176479475952277130,2228622485820353743,2845101904427108813],[10292683523489835292,10247814894946347631,2644590758149436851],[14525911450532087857,17942252987826072121,6801450836348820530],[10671464302529786982,7450404385797873183,9037411470647583838]], "path_len": 4}]}, + {"iota": 567, "deep": [18047729633876989466,99374521245134966,3360754488684044331], "deep_sym": [17516618015903139100,2951436536334618682,17980489157745383386], "terminal_position": 4, "layers": [{"layer": 0, "d": 3, "position": 567, "leaf": 70, "slot": 7, "values": [[6245239563054698545,10953784134743681631,309117201705107312],[10147311537216613353,7920342408185388334,2703788232899065126],[3635161026520856384,6954959520249125012,13529324898537327155],[15764061797087379466,9370872654131914299,6335165267128049318],[10994028427916486458,13040415860252061487,6960268057679038591],[10815095027707255667,18279067243625263691,11050218513605064717],[6181460592069893286,7846926160755689348,13639874225392218244],[7537553748832786394,11392017698094394877,103694717963243848]], "path_len": 8}, {"layer": 1, "d": 1, "position": 70, "leaf": 35, "slot": 0, "values": [[10356989277157925581,7734667612467985055,16277606814930031695],[11515250190753871632,14360030790213387188,16280773358479821956]], "path_len": 7}, {"layer": 2, "d": 3, "position": 35, "leaf": 4, "slot": 3, "values": [[7986871652212847729,3216947561173799887,17389602311097823189],[2840685283826001252,3345163626551462189,757499744147822538],[15701632955855902379,16763676821054664820,13807560596838550641],[2742910096593592820,8353559143048342553,9954138356989099170],[9847969385057154419,12680646407006050555,18056330072996240036],[14446742634554659401,12873011047125567117,32390744878969203],[13758404333410532093,17090097282290779556,16627657403379939801],[8324004754530806517,15962588376622656722,6193420264960613083]], "path_len": 4}]}, + {"iota": 1526, "deep": [2567025427785041582,8000187002850677269,3827534577909862613], "deep_sym": [16629321827697767665,13697610440565804273,59291125760329351], "terminal_position": 11, "layers": [{"layer": 0, "d": 3, "position": 1526, "leaf": 190, "slot": 6, "values": [[18367964984216933062,5660898099715863344,12091825699828272785],[4064797189050673755,8758312521479053195,17586433145799905667],[11042109590396741557,13453998188520539681,4026691058011471979],[6920737729535120217,5791562277528914497,1738885481870857403],[17162616217057319708,14564053023082606491,7572870302558855093],[12956855366818227103,17813714031214072642,12230382703550138851],[757677278244590014,1785069472838724489,3241296500485712017],[3086114952743418053,15349864503276037083,11277820017340792738]], "path_len": 8}, {"layer": 1, "d": 1, "position": 190, "leaf": 95, "slot": 0, "values": [[10524734310662209668,18323881505728001248,6372778122146690452],[1004243276605533431,3591193302962306504,1633219252869578500]], "path_len": 7}, {"layer": 2, "d": 3, "position": 95, "leaf": 11, "slot": 7, "values": [[5240087236059509009,1172869790317804758,5089515052485497818],[10355844810202830688,17925465335816866521,8351483564904242640],[4079809391339080498,15892637900298544667,7778229385415383189],[17801563449072120160,11734094917552760905,11750172894724870474],[16550070760801625399,11956680567406663251,11737023057057024488],[440034897161641969,8320818327047863761,5881652704612506137],[8980928091881739289,807661344907923994,2585567908456372771],[13123983852650476319,5128554601153007764,13742590666612112409]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp_3_1_3.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp_3_1_3.rkyv new file mode 100644 index 000000000..e57464d91 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_dp_3_1_3.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_pair.json b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_pair.json new file mode 100644 index 000000000..af3751548 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_pair.json @@ -0,0 +1,27 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "rpx", + "format": "pair", + "proof_rkyv": "d_proof_rpx_pair.rkyv", + "proof_rkyv_len": 11136, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "legacy_encoding": true, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["a5cb9815a33628e2d9aee1104d2210e8066854dd38976a0143c7b0638504f9f3","5af88af0782998624ef05869a90b8e35e82a85d7da5c899ce3a0d8d1872a1ab0","97ae9eedee1a1b29b7eb30478eb3b8f118b84f3bd43e6352fbdac68314723730","2f2da5b7622cc6eb49677363305e2626fd79387dc9acaae9202b655cb84f1f8a","9fef7049e45b8f097dee76ba5feadc10268bb51f63e9336dfafc3ac54e3e3e38","c261958da4b13c4ce5dcf9cb19d42cf09808cb932dea3b9e69bdb7577f308167","2aa133f65dd602f13ab5b2961d2b96cc3f6daf6f106e9783939c69716b9d2ab9"], + "zetas": [[4735330965523630181,1034630526833404286,12017969954712239940],[16665743570319646148,16897879252278531211,10291861723093761662],[10619368815145924427,1493089516910409884,14431758427697423319],[9552622148858278301,6488516694070886107,4893272118711353122],[2783515638190829017,9945112524553572548,15631202162117197821],[17815262798501899446,18394714429174366312,8636369618480887460],[11856812424473920575,10766055906630249609,2957922871886357218],[8171264462115707646,14626134561370321125,16568024845886241762]], + "terminal_coeffs": [[3714161951696662500,2595793324825979078,3565379477475041685],[4424026265791135346,6558459514194683116,753777937513084834],[7727091673734829526,10561609288187203284,15868472042909273283],[7864830287677944053,6520068215425390864,4795411284951093801]], + "queries_detail": [ + {"iota": 907, "deep": [3040718383397280274,11956941222209451830,13732184933327600162], "deep_sym": [14770993481378414249,10902020514396191223,13023915065759495237], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 907, "leaf": 453, "slot": 1, "values": [[10738449088863093738,17001367698800175667,872083450534898496]], "path_len": 10}, {"layer": 1, "d": 1, "position": 453, "leaf": 226, "slot": 1, "values": [[8949331791643690447,11945875389884147400,4863937416666523377]], "path_len": 9}, {"layer": 2, "d": 1, "position": 226, "leaf": 113, "slot": 0, "values": [[6531312029425646452,8614572489917572820,5793377902239563396]], "path_len": 8}, {"layer": 3, "d": 1, "position": 113, "leaf": 56, "slot": 1, "values": [[13245888542464671401,9490131273601039377,5473879260557931677]], "path_len": 7}, {"layer": 4, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[967364531071989975,2293195037664380504,8237422110493151214]], "path_len": 6}, {"layer": 5, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[3068323341190833160,2495604132745403317,440675983381132721]], "path_len": 5}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[6235735872747817365,1215148853071865642,1291260655626560448]], "path_len": 4}]}, + {"iota": 327, "deep": [6648971487643812783,3504019026315364339,5553986533175031714], "deep_sym": [4847162544338375188,3760822623165876653,18385909206353757824], "terminal_position": 2, "layers": [{"layer": 0, "d": 1, "position": 327, "leaf": 163, "slot": 1, "values": [[3102706233933768041,2061658593148860966,2937765886239263664]], "path_len": 10}, {"layer": 1, "d": 1, "position": 163, "leaf": 81, "slot": 1, "values": [[5351203088071617904,5190285090595573022,679443460868787999]], "path_len": 9}, {"layer": 2, "d": 1, "position": 81, "leaf": 40, "slot": 1, "values": [[6813072834546256573,17560727769766864490,11355003854934910309]], "path_len": 8}, {"layer": 3, "d": 1, "position": 40, "leaf": 20, "slot": 0, "values": [[4276057153471798645,16589259977369031850,10213981092871167197]], "path_len": 7}, {"layer": 4, "d": 1, "position": 20, "leaf": 10, "slot": 0, "values": [[6724049492496023179,14516868345363052182,5463760551208548401]], "path_len": 6}, {"layer": 5, "d": 1, "position": 10, "leaf": 5, "slot": 0, "values": [[872892273747685357,17097460053185137664,7011811299561387579]], "path_len": 5}, {"layer": 6, "d": 1, "position": 5, "leaf": 2, "slot": 1, "values": [[6972245892337738186,7308248010937532343,15369312713036268331]], "path_len": 4}]}, + {"iota": 1055, "deep": [817615283715329005,16733328116567315164,13240493922581948560], "deep_sym": [1731008489953378402,3915464684156498806,11643985226885705726], "terminal_position": 8, "layers": [{"layer": 0, "d": 1, "position": 1055, "leaf": 527, "slot": 1, "values": [[7812814834329037929,5936428735545781918,6769737470751296857]], "path_len": 10}, {"layer": 1, "d": 1, "position": 527, "leaf": 263, "slot": 1, "values": [[17350970259606761374,14937434176606905201,2436331477479557398]], "path_len": 9}, {"layer": 2, "d": 1, "position": 263, "leaf": 131, "slot": 1, "values": [[11807655768007569086,16744789612728260734,4620195842063357801]], "path_len": 8}, {"layer": 3, "d": 1, "position": 131, "leaf": 65, "slot": 1, "values": [[13354553806338021504,6368498475783127995,1444223788932639250]], "path_len": 7}, {"layer": 4, "d": 1, "position": 65, "leaf": 32, "slot": 1, "values": [[2595125021894609724,3173778131451323870,2008467966233623536]], "path_len": 6}, {"layer": 5, "d": 1, "position": 32, "leaf": 16, "slot": 0, "values": [[734813548736948502,6040967620827257108,7735869121954590664]], "path_len": 5}, {"layer": 6, "d": 1, "position": 16, "leaf": 8, "slot": 0, "values": [[6896385518190873683,9696907498203743737,11255624105341837683]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_pair.rkyv new file mode 100644 index 000000000..1bd3d5358 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/d_proof_rpx_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_blake3.json b/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_blake3.json new file mode 100644 index 000000000..da5dd975f --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_blake3.json @@ -0,0 +1,11 @@ +{ + "generator": "stark::fri::vectors::one_row_leaf_digests_json", + "hash": "blake3", + "rows": 16, + "base_columns": [[14950669930584181769,11380843527670038249,14170810701887864585,12657459883858543732,11080332778200492175,4152169804438290561,12191777403133591725,10801486430904554830,4417364748388562854,4379087181347593436,2580857809684985080,7673975303685132775,13322131507302669334,9040154351058314042,14264724532560863887,13962096292449051454],[18243414257841422358,1304221489434139653,4524329391722882702,18400865441867651612,8077364911250023428,594594441077591684,2534844611324517698,1969697784587826307,13838274770865440410,17810497879044384847,17948609656845876769,2245783734148948709,3359004654947870420,7611350254961757910,2256523342594777630,5184348059537790602],[3333165186168681317,10992969103574531539,10875599246434246438,4202797048359915902,13708589652114080127,8862588509040537726,5062794908899136299,16177654723492523013,782810894950674176,16085627345094361018,11968464090099871210,10878072172278744852,2776239942643392900,1706434847502238813,1553514265852765581,10755221291880268160],[15185130643288894846,2969650219458132482,10730508005208922807,6538486738868699860,13637771772236929810,2559123444577356896,18126217652353331113,5721278983068996567,9421049141588406289,3056349041078578205,1927015489752515349,16193479730068331852,7962887402148557259,18122082188664764562,4432334024656166286,14439109430197722085],[12869403534454369847,444100742738500988,5149525751578300798,16074201275155691844,18094321804223741766,9704695991314911754,11529325637956947874,5462031299392823211,12784861903617806249,15621907776666625844,6514538806212006718,16075501809475733688,11901509892253068338,3954885611778170505,14288373624468033718,5293929189132021115]], + "ext_columns": [[[207727902132756252,11173563745377630981,7306909256194215961],[3383316449733693245,14112212308402603625,16675907919222413400],[11091225657268605779,2260900423939991720,9458175385801186643],[6403564405749070118,14462018993348769223,1663236480835094319],[10760925658949673415,6256953096125034850,6374165608116273133],[7525097355787171930,8065360669614765403,15671833331641072930],[8387767360670315620,12721973472388740613,12037449270884550397],[12780009495799128023,18254530395830801598,17124580712984908689],[7170091433976859457,2918423366040466885,12162269374600581905],[16638539687531051970,2633731385302464777,9274096096546535786],[15186392598723191854,4370449889476143518,10080202853130152767],[10247876770105988134,17464801317635882529,17998396633050378591],[13977131749530808397,7521738060361358462,1158021110493825475],[17395948259724017503,18208524233454958027,9357130278945496078],[12514637307887469569,8173084814001755783,16874068347906640087],[13076889950263576212,9681825774613785687,14728844907461535493]],[[1703295679615235702,13608405329556208281,13586959445987754067],[13509429031623984718,13028166630131220703,12842497139504455345],[12483125829912424503,359627891118073558,14115869743926542122],[206993782868978585,1945608048083412892,16924920981352735495],[10460072476710726221,6746467623189781681,718200883831581176],[17283619850490311477,15509599890076648547,9393695392791290257],[14045616078604790859,3033230427237039184,13069887780656089759],[12119710066391128062,9603138251095584760,12526281507633415864],[11646594443330237117,10431672579314833108,2414794606147947405],[7226819219904749,13787705273014176717,2174618065661578531],[11184428261634409861,17041311285406150036,9151670609840952406],[4504922362140931424,3374239225141597561,8705297669257511518],[12618315602974981246,12028487155674968288,14243761148199342949],[11397300516573109633,2515281435755958282,9596751895950036808],[1977998875513216148,3757846768502258754,9275101095842896217],[12145146597496230120,1463483070281991503,5369856519452762238]]], + "layouts": [ + {"layout": "row", "rows_per_leaf": 1, "base_leaves": ["c2ae1f207f4add26960a874712a5161f3e7a0a8a02b2fd6b613b97b4a0fc8c24","3f627fcbdd872e6d88c03d9f395575fa3b3eea394010620ff65d9722689a598e","bb3ed2b53c1f771e3bd9c5acddf0b1c8755b058e41d9b649e8ff0021d366dd1c","808da1a70572374969157c4afda12b3f52583dd427dafc3741b4d9ba07eece1b","d01b246953e878de6a9b410436ebf1e76ee3b22e1c03524c587dae6687b0170f","15af76f38031f18d8b1c8361710081971d319539f9bbd471451b83b88d2fff1f","a1ec624c309ec5cbc49834b1564c4682c3bc9dc541145eff3b86894b72efe181","a31686ba3cc471d239920544d9d0a0530aa44307dcea70f7153305fc89f92a12","f0249e04f045a3e8532fd6fed462d1b3fcf83d3701a44b91abbb44642f1ae604","ecb609906aed5a3016b0a9387d37a4e62030c174507ca681c26eaa62fce2d186","cb46989ec3547d9848b76ac33ba51159d15c5419932deb3cf42d8ea3e8bffb3c","804bbf310e4150eab3986958be7207e67c99883e0334da005535cd8cd7e12734","02c125aa450ca69c2af813165a1fecf7d2be257d03e07004be0007feb7709a6b","9d3756f8ecb33d589f6f47ec016b3b84f5c306d320fba9dc05ea3884800c4974","2458aff19dd89b72a29ec102231dbf0cdab82928a9b35ac70a6642d253292429","591ef32c4b694c118ae2515d4a4c6bc0130e337be7262235fc7e8909a1265de4"], "base_root": "b460868e5b8e9f7e1bcc9d9761ba1f985ff2ab56d2b86d938eafef633ef0736a", "ext_leaves": ["5cb7148f5b3bd7a78c4e7f145222de7f889406c0ba7e472760fcbbb527c953c1","055ba11f6b9a6b6a8f87f6d9e114e1b8568c6c806372f25961b10a7b61f2623a","d058badb4b77e7678d127b38fab95f08b6c9a69e91457598a7befaeab56f45d7","10ea36f5886e3bbbb44dd85e688a3d4df098e9855824191429beeed571c3ecc3","8f82f9e303ce6654b137e7c4eea8923c345eb60f829ea086fbdb448d0f895f8a","224fbba90b02b3f4ba70f6b525fe0e125496f29b1bd1219fa6389b01d76eb0ab","a80cc0208b85255bb3fc7db907f7c099717c8359cc6d7447507415b26d925ccf","ba058dbd45ff6a0c56d6863f0db9ee3027f614ab272f29af577144d432ae9034","0c5f1927bbf3ebb9de0d8f7d86930e7a9ea113463d9857069a1900835ab24b8a","8a9c5cfe653b949eddb01b2a3aef15e60c7f4a3305721dd399007ca53cbf2a39","4ccfa171a65d37af70f753f8890a89b71697b0a4c7e12dfe629639579674b47a","48add12b795515f30db197468f3e933158f3da39442c92e1314e16840eb5d886","1ba6c68722fb7b8d37a119c6f89142e284bb1a64ceedff9f93e3563a799721e4","c06fda11a91ced90b28bee4c98ae0d17986eba3e06aacd3404014fc763963d26","09670b114cce12e38257d0d91ecf952f43b89531cf2465e4ada56974a1294646","eed259595561389703fafee890c5efa3fd55f12a5f6d7e56f840595afda51652"], "ext_root": "624475b85934f5abe03978164a25aa1a14c518fcd393ed0fc11f407b956ea4ad"}, + {"layout": "row_pair", "rows_per_leaf": 2, "base_leaves": ["9f7e2bcfc6268ad6b644aa70a6dd99fbceb797191a821d5e3e6b3ab62ca41c02","bf01cfebfb776782174a75e80826318dad32a6480378f84747861930f066e585","5f2a6e2e9899df9318108f0a23ed4f71fe0655479c19ee044e725c621d1e5782","fce457696f147a49d959e7031fbc58febd75380bc25f10c53f574b92f171fb38","01bf0eeb8d9551c210e0a09fd46312785ba0391a41c568ad21b76f55ce051d3d","0ebe04379d102820eeace432f6195c2d1110318cacf8e93ce57de2b8677220ac","0c036dff3607011f632ac774c4df64387cd28f88797c1ea9e9473e814d9d4c21","966ff57fe712ae13733cd13031a88e692b70ba8a5f897d25c39f060aefc87f53"], "base_root": "8bb46242b852482ed7068636fcee4b72b98aa1bed07ab497f772872f4f3a7daf", "ext_leaves": ["e413ef1157c371bea24606d9acff1976e6a84f7161a435010a774fa80ca62424","7aa5945d700777d3764a218cfb5c9f74e8268694e3bad8707f25ccb740c4a03e","213996c3e0303bffb30e1d3f7849cfec2ee2c04cae38f598c6ad4e8f31a54961","4b75d22f0b7abd02ccb5038f19c60e265e1029f8ea0dd577b4b3014f96c60d72","533e2b13491d03acdc84546116e5d8ff773c072682c2b86f360b4ab6c7874176","3b76337a827ab8a20edcf5770025783e0e30a47dec1595cf4fadc7811e37caf0","fb0d3a529500753ec91fadf14f843a828ad56531e1935147e385b0e96c3e2f42","95e6d95c63db1bcd7a6df701931f28e13916b9451908a29db4d7837508e611f9"], "ext_root": "a7a60a908813bc33dc23a595b9f9a3e8c476f4fce108b35caad8e3c730e7fc31"} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_keccak.json b/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_keccak.json new file mode 100644 index 000000000..d7a4b101f --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_keccak.json @@ -0,0 +1,11 @@ +{ + "generator": "stark::fri::vectors::one_row_leaf_digests_json", + "hash": "keccak", + "rows": 16, + "base_columns": [[14950669930584181769,11380843527670038249,14170810701887864585,12657459883858543732,11080332778200492175,4152169804438290561,12191777403133591725,10801486430904554830,4417364748388562854,4379087181347593436,2580857809684985080,7673975303685132775,13322131507302669334,9040154351058314042,14264724532560863887,13962096292449051454],[18243414257841422358,1304221489434139653,4524329391722882702,18400865441867651612,8077364911250023428,594594441077591684,2534844611324517698,1969697784587826307,13838274770865440410,17810497879044384847,17948609656845876769,2245783734148948709,3359004654947870420,7611350254961757910,2256523342594777630,5184348059537790602],[3333165186168681317,10992969103574531539,10875599246434246438,4202797048359915902,13708589652114080127,8862588509040537726,5062794908899136299,16177654723492523013,782810894950674176,16085627345094361018,11968464090099871210,10878072172278744852,2776239942643392900,1706434847502238813,1553514265852765581,10755221291880268160],[15185130643288894846,2969650219458132482,10730508005208922807,6538486738868699860,13637771772236929810,2559123444577356896,18126217652353331113,5721278983068996567,9421049141588406289,3056349041078578205,1927015489752515349,16193479730068331852,7962887402148557259,18122082188664764562,4432334024656166286,14439109430197722085],[12869403534454369847,444100742738500988,5149525751578300798,16074201275155691844,18094321804223741766,9704695991314911754,11529325637956947874,5462031299392823211,12784861903617806249,15621907776666625844,6514538806212006718,16075501809475733688,11901509892253068338,3954885611778170505,14288373624468033718,5293929189132021115]], + "ext_columns": [[[207727902132756252,11173563745377630981,7306909256194215961],[3383316449733693245,14112212308402603625,16675907919222413400],[11091225657268605779,2260900423939991720,9458175385801186643],[6403564405749070118,14462018993348769223,1663236480835094319],[10760925658949673415,6256953096125034850,6374165608116273133],[7525097355787171930,8065360669614765403,15671833331641072930],[8387767360670315620,12721973472388740613,12037449270884550397],[12780009495799128023,18254530395830801598,17124580712984908689],[7170091433976859457,2918423366040466885,12162269374600581905],[16638539687531051970,2633731385302464777,9274096096546535786],[15186392598723191854,4370449889476143518,10080202853130152767],[10247876770105988134,17464801317635882529,17998396633050378591],[13977131749530808397,7521738060361358462,1158021110493825475],[17395948259724017503,18208524233454958027,9357130278945496078],[12514637307887469569,8173084814001755783,16874068347906640087],[13076889950263576212,9681825774613785687,14728844907461535493]],[[1703295679615235702,13608405329556208281,13586959445987754067],[13509429031623984718,13028166630131220703,12842497139504455345],[12483125829912424503,359627891118073558,14115869743926542122],[206993782868978585,1945608048083412892,16924920981352735495],[10460072476710726221,6746467623189781681,718200883831581176],[17283619850490311477,15509599890076648547,9393695392791290257],[14045616078604790859,3033230427237039184,13069887780656089759],[12119710066391128062,9603138251095584760,12526281507633415864],[11646594443330237117,10431672579314833108,2414794606147947405],[7226819219904749,13787705273014176717,2174618065661578531],[11184428261634409861,17041311285406150036,9151670609840952406],[4504922362140931424,3374239225141597561,8705297669257511518],[12618315602974981246,12028487155674968288,14243761148199342949],[11397300516573109633,2515281435755958282,9596751895950036808],[1977998875513216148,3757846768502258754,9275101095842896217],[12145146597496230120,1463483070281991503,5369856519452762238]]], + "layouts": [ + {"layout": "row", "rows_per_leaf": 1, "base_leaves": ["b145d5dfcfba3e9fbc6fc0f2b22f1f09cface989a60271c974c7527775d70966","d174efcf6b40ca002c952a477dd2dd314560bf6a67f399383d25896fea933476","f3c366529bba2c38ca7ae96915d677b148fd17e4e8384fcedd0b1b9b8fc00669","d62a245394266645006b118ea9512ef37a8b5ee505b261b700282d01d705479d","c7ca499150cab681ff24b6c26b5a581c88c81e80fcbddd96ed610c4e6efca518","18f9a71059056718abe5ce1c90837753e3fa546b3b902d6a4926e474b8606a51","e5e7234ad258993e0afd4ae7b3e65825bcc87bef7136545e398b20407da7df78","4f8c5ea8ceb6d90b8cfdcafd2cc7a03ef6a67ff3cb14a3a1aa926aeb75e42c95","02cd5ceb9d233c6737c77ff6cb74102447cf6d3bbfd55b1891a8bcef4a362823","8dc989a34870bc7dbe1e590062db5a176ee4aa8c0ac1d33f864cbe68df135c9f","3be46427672143f60911dc082387e7c47a9e7b0b96447c74d949e6ecd24dd71d","b921b40b675814f03cfc6e4b6c432a0d71595341774fc9201c87e11d821b3be7","5a2ebfc42f679e888bcc9c841d4aaebeeed53d01dab0c4edc461579b1d9e9690","f08ffa382d4ee6de4b2b0d1d44febc30d214c2a60ff801e462ca64f324142b18","5a7540037162d28a86465459c79a644708307875c7c2932855dfd23729447fd1","185933343be20010c846441b1a7a195f3486e35572294e35ce86793d43ed097b"], "base_root": "3c3af9866450b1fc3d15b65113f318ecf11cd7edff613bf7bcfbca08fec9a4a9", "ext_leaves": ["148a2ad5111bf100ce8db96c8fe59d5b92f379a1db44af511db1146957e9f371","c0a40f4420e1e64c87dad9812ff3178934daccfb3d918606e8c84a33c82904f8","f274452120f914f4e92ae73d567e4a57d5eeea968fd544166e98b20de91c6a7a","9fa6d9e3a8bec3c2c55df2883be794c76ba185a1986a1103e9f66cc79840a296","e3aca190939b9c1df8aec098f803fc9b6d2047d4c1f7c6dca7c6c3dba1aab124","61afea46b802e6e048a3a5a9b6b3fb631bb24871ce510fb411a4c60d6cd214c3","7c40f52149bb6e91631efcc3b719a55ecb234bcf1401e5f4ba3974db13dde47b","46a994cf8c9d16d65eb5d200157b3271019a1d7673b630ed6efd86e7ae1b256e","ba7b579ba82456a7be452a5c0003a10542ffd41e27c3e13ac1d0ab1ff18968bc","1e2dcbcfa40a77880dfe33836b0627a5654d4ccca4972deb4919a760406d4e62","9b2b4f60a42eb415ae22044c5ec829e4ef3d11860e8001cf656733ee863e2988","ac706956eb2e8e10329cac607f0fd73dacbe265adf48f37a3869fcdc86df0659","6b9deb036a2ff21a5cffea8094a086aac4d1d8d49df630bc90ea452348a63847","9195181ef2a3ead35296e1accf80c54586517b0001e3419cdfc5761d4ca3d0db","e03605e10a09a8cc5c6c2c08f2dad3a9d10f3e636b203c67d1cc863692de183a","4335332d20d3ef2a29aab8786890d3f73f218dc8ff00788316b4c60e4474f0ec"], "ext_root": "eec6dc06c211f42f6f29af14bbc2ad32e51d4ff0c679d6dd47a53682310ccb5e"}, + {"layout": "row_pair", "rows_per_leaf": 2, "base_leaves": ["a36317970aaf00a83fe6135cfe78616cf49bd3722b64b69b708c5f2d6abf88ab","45865e8afc7ac83bbf0735c2ce56982e5af254ed5259e92588bc51c4cc4b3cdf","07ed9afd31cc009980d8b13bae81eb47bc3da4e385f2c27ec1fa767b25e8afbb","2be6e6fb26e8d9d1673ad2ca4a6ceaa750fe2e18a15f30009d796cac2303fe20","29cdb49d1e49b05bfa2cd22da89b5797227600d68009ecaa30d82ff93e61a996","f398e56c618e09676a414306a23c602ef6017aa0a3d3fc12f35c8323f6ea1f73","2becd8729d468c25db0dce34bb5646b5bdd23d63fca063bb3edfe330bfc20dba","19a0abb354cfbcd2dc44abb85aefa038969b9286e3699b5a1ff90bb5d1d6e120"], "base_root": "d147ee4422cef57eb47727eafd4d78486626c66279439806f708264f4be74534", "ext_leaves": ["e8efb646b7299d1862c3ebdfc2819842509758766679b687b01ee66aec2c5b37","3038c9b8d756d3c7517c535ed23821e40a2e5615378c9025a5f5fc15acf98d9f","8b89774cf9b005e98093d0a75735b7751787812f0a2cde4d649636fcd529dc81","f1a5c88fe103293b22937693d160520e6499db51725d8bdf133975067f9781ab","ab33959ac5998722fb8f51d00727592c409eabcb2db5280b2e7a3861ec2fecc6","79dd017b1547ea8140601a45064c6f3f2e485ac4966bee5a03fba69f82f4216a","af6d4e37be309fd771da76b81ecb6ce3dd1a5bd25dc051fef98aabb9dc649f35","3811a544683432e9cbf78da718ddc0e09608b81fb9ea1e44666dc9a38593a2be"], "ext_root": "90c35a8d73cbccd81153f8086d835f7a6521fee7f66800a48fb4d63495a4b5bf"} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_rpx.json b/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_rpx.json new file mode 100644 index 000000000..0e1709d4d --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_leaf_digests_rpx.json @@ -0,0 +1,11 @@ +{ + "generator": "stark::fri::vectors::one_row_leaf_digests_json", + "hash": "rpx", + "rows": 16, + "base_columns": [[14950669930584181769,11380843527670038249,14170810701887864585,12657459883858543732,11080332778200492175,4152169804438290561,12191777403133591725,10801486430904554830,4417364748388562854,4379087181347593436,2580857809684985080,7673975303685132775,13322131507302669334,9040154351058314042,14264724532560863887,13962096292449051454],[18243414257841422358,1304221489434139653,4524329391722882702,18400865441867651612,8077364911250023428,594594441077591684,2534844611324517698,1969697784587826307,13838274770865440410,17810497879044384847,17948609656845876769,2245783734148948709,3359004654947870420,7611350254961757910,2256523342594777630,5184348059537790602],[3333165186168681317,10992969103574531539,10875599246434246438,4202797048359915902,13708589652114080127,8862588509040537726,5062794908899136299,16177654723492523013,782810894950674176,16085627345094361018,11968464090099871210,10878072172278744852,2776239942643392900,1706434847502238813,1553514265852765581,10755221291880268160],[15185130643288894846,2969650219458132482,10730508005208922807,6538486738868699860,13637771772236929810,2559123444577356896,18126217652353331113,5721278983068996567,9421049141588406289,3056349041078578205,1927015489752515349,16193479730068331852,7962887402148557259,18122082188664764562,4432334024656166286,14439109430197722085],[12869403534454369847,444100742738500988,5149525751578300798,16074201275155691844,18094321804223741766,9704695991314911754,11529325637956947874,5462031299392823211,12784861903617806249,15621907776666625844,6514538806212006718,16075501809475733688,11901509892253068338,3954885611778170505,14288373624468033718,5293929189132021115]], + "ext_columns": [[[207727902132756252,11173563745377630981,7306909256194215961],[3383316449733693245,14112212308402603625,16675907919222413400],[11091225657268605779,2260900423939991720,9458175385801186643],[6403564405749070118,14462018993348769223,1663236480835094319],[10760925658949673415,6256953096125034850,6374165608116273133],[7525097355787171930,8065360669614765403,15671833331641072930],[8387767360670315620,12721973472388740613,12037449270884550397],[12780009495799128023,18254530395830801598,17124580712984908689],[7170091433976859457,2918423366040466885,12162269374600581905],[16638539687531051970,2633731385302464777,9274096096546535786],[15186392598723191854,4370449889476143518,10080202853130152767],[10247876770105988134,17464801317635882529,17998396633050378591],[13977131749530808397,7521738060361358462,1158021110493825475],[17395948259724017503,18208524233454958027,9357130278945496078],[12514637307887469569,8173084814001755783,16874068347906640087],[13076889950263576212,9681825774613785687,14728844907461535493]],[[1703295679615235702,13608405329556208281,13586959445987754067],[13509429031623984718,13028166630131220703,12842497139504455345],[12483125829912424503,359627891118073558,14115869743926542122],[206993782868978585,1945608048083412892,16924920981352735495],[10460072476710726221,6746467623189781681,718200883831581176],[17283619850490311477,15509599890076648547,9393695392791290257],[14045616078604790859,3033230427237039184,13069887780656089759],[12119710066391128062,9603138251095584760,12526281507633415864],[11646594443330237117,10431672579314833108,2414794606147947405],[7226819219904749,13787705273014176717,2174618065661578531],[11184428261634409861,17041311285406150036,9151670609840952406],[4504922362140931424,3374239225141597561,8705297669257511518],[12618315602974981246,12028487155674968288,14243761148199342949],[11397300516573109633,2515281435755958282,9596751895950036808],[1977998875513216148,3757846768502258754,9275101095842896217],[12145146597496230120,1463483070281991503,5369856519452762238]]], + "layouts": [ + {"layout": "row", "rows_per_leaf": 1, "base_leaves": ["7a656aa379ece6ab0d5023d5fcd1c5017daaa9516e633411dccf7ee7866c6af0","232b79335b3332a541edf79e0df1cec14debacb48c087cde110e0c5a51cf8651","479953e1ea3119218d15dbf0e79b26f6a877763d3010b0898519cf62ff281d67","ea4f255dfe92205d9e211022f0427898debdc7d6f26c765d52af22a36c719e62","df4ca202bd7a3ac32af9b915695123eb863377f28c3030f64059913358266b4a","9680d16e917410c10be2fbf05c40e8321bc2bb94edf7feb9d6445b3fb77e7fb3","8f586b1d423e031a3361f5ef3e5228d17bbce14ce417fe384e4bd9a87532db18","74c1164da23920abdae2e977117a40f897b8fcfb194e65af9043c67428d0190e","e083f6db031e946a2bacb0e144955efe90fa1753b9c4e51b7cb5cb6d78ba979a","a95e210e54add5c9bd71881fec66ccecd55a5e1a62c77c63ab19c87c9141e65f","e96fb82d9895224c11bba9a827e202218a1a6e057c41c2bd77e67a11fde061ce","601a1ceb3e882454002ff63c661ca542180888d618c3e6c2fa97697d7b9b1f80","8e45674da156a10920975d5b5d37e8a5ab3e4dbfeb21655ccfb88c48ff7ffae3","8bd8e04ab8499789bf74604dc68a89e629fa4b243f2d570d0d8ddb146c54fceb","db1d1debba5659edb8b922fbd6632df4c0b54811fd25b78bdcc68d1322289db6","fb688db3bb190dfb5e6f1f8d62ba967ad8feb333940b910e117010dc7252eb29"], "base_root": "650703b5195d641807f1115b2941afc8e6badd197b4720f7e5cc726f80f0e94b", "ext_leaves": ["e419504fcdaac941688c9b129a10ef93e071239523bfa6c5a15bb7cddd89fdd7","6e7c6e1dc231c3071d4bcfa74715fd4d15050c985255964ccc9d7ca89700c975","789bf008db122ef16929cbff7f511cecf808a7fc4845e748544362d4d83270ec","091ecc0d98d7284afc23ee5a52e0bd539be406a5e46ac80120e2d13a39aba975","79c18d4b78159536942793f03b88fbd0f93ec7842705a0d3bdeae4bdc43fbdfa","ecdba206bca9c613692ac36df568afd4af0dddbdc9742749eeab253fb130a836","208c517175582dae45602ed704f994d80a2cc6abbd60a1a9f726d82f2c73088a","5a89d4ac4fd8d53f9f1393c60fa534d80d86a41a07236fe31e34b1c68dbc0e6e","cd8f955ad87fb3e020db7bd504e83e85cc191c07decc1870326130fa581a6a35","ecf26ee0dd73b146ca161c832bc372dbc141e5e25b5d2061bbfc14240603bf99","4fa40c537344477c2efeecb2ae31e6a7a5f99db28d225f5b7dee4fcec5f17ee9","b172009224988abff945032a0662c4d9e8e238162f44b5a320cacb7097b0436c","fc0da71090b7ee174679277c91bd812dd9c397dbbb712ac208c439414ada87d3","85e8820f2db905bc31195a9b7ed4a1df71c480b3551645e0416fa4d9c73170ec","4cfaacbdae71da58fe2e60a023c725fab89d4a6964181f7188502c7f7f0169e6","d17030d4f63c802ce32188cbfaed596a3e94c01524da10afaaab38f022618444"], "ext_root": "469ec5bc2e06bd3ad746e058f0f735191581b91b5797082ebdcc520fd32cfece"}, + {"layout": "row_pair", "rows_per_leaf": 2, "base_leaves": ["f379cb35b3ab7aa83b17d05c7d78222697c27af406322bca15d11095f7318624","098fa8a23134de8b9d29b5aa7549f82226a280ba82fc5018c00df22ea2b19a87","5542d40fd569abae399fe4bafe1136422ad298740727a4e34bfaeebfa71aa6a4","d22ff0692ced201a9b9a5a27d7145b9bd37681b73e2c3052e4209c11f9650f88","9bf40a6d5e9eadd8bceee2b929ef60ee6b308b7104dcac4a98d33c5543839e19","c23ad117cd81006a27ffd1106761d2362c43ce5f7f59dd3bf94e26c2f8c36b6b","50ddaa72336de98afa901d94e8f018ffe47d9324bc657504a5a108d233e3d081","c69ad71b949c67c8f904107de5f2f0e5ed17f6eb963f5e271675cbc86b81013a"], "base_root": "5672122216f801a68d91b66ce4ed4116409df94443816318e495ed50d8ca5c9b", "ext_leaves": ["75f0801e118d4e5da1ee9fb58d326c8b8d8bfa5c7bc8e46847b4604c6e06a70d","bd71b862db923fd053d5f0919533edf3d988c080028df66b5d00d62227aa9833","68714ecd51fa4ceec480c49176202da940b6edf3ddb9cc0233ac39e0218de228","0fd68f5c137f1eff95fa07414cacc2a55a8de82f93aa07374f33c6c79b030403","e74ab3f854ffcd180d98c2abe8f295cd41b16c977d59b1118ee7757173bebbd9","e2cc8e459b6afa6c681abd691f9b458ad49bd599220aa645e4ae0dbb0e9fc79d","6b0465b2b2058547912be3efc8a7fda4154b32d1c1416ed6117c99f9cef23aaa","41708a29c8127bb725993061c29acdca304ad24618b18260cba34c2e42d7ee42"], "ext_root": "239707df45a54d232d9a70936d777527fd096305a0871e83292c5f3b2b5101bc"} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_3_2_1_2.json b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_3_2_1_2.json new file mode 100644 index 000000000..366f49906 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_3_2_1_2.json @@ -0,0 +1,30 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "blake3", + "format": "one_row_3_2_1_2", + "proof_rkyv": "e_proof_blake3_one_row_3_2_1_2.rkyv", + "proof_rkyv_len": 9432, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "one_row": true, + "query_bound": 4096, + "trace_tree_depth": 12, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 1, 2], + "fri_roots": ["631d2cc3b49a8af4dfe3f98daf6aed283292da1952fa3955b5f7f08b0addbdcd","bdd4fec81d28bac7cd36e7ee047215e8e4e103a6bbaf6283b7451634d9ad7b13","da5a923543dd35c5ca8583a9e7ab6628943515c416c5b40f7d4cc001287c0f21","57dcb115bfa2c9d5d0b40f9d0095f35f4748f9f7b775fe1c691ec543de2f9991"], + "zetas": [[15303203179608116932,15332629808348820381,2738315253515061193],[363681095822826847,9352367609937442414,5055796092274921848],[415111747713879034,9715309884249926995,17241085662786188492],[11355540028731819133,14059481269354791517,9429735590016017728]], + "terminal_coeffs": [[10406129371342019884,6637860243315701565,17413526420005229991],[10816053847836238497,6095471629599537880,14691285080173723159],[5086529536450569695,3715400737876031162,8188649170801691210],[2322353628246006421,538626804931903626,11216069688036902183]], + "queries_detail": [ + {"iota": 1456, "trace_leaf": 1456, "trace_path_len": 12, "deep": [194573393572430413,575541586100696177,17185197193417606866], "terminal_position": 5, "layers": [{"layer": 0, "d": 3, "position": 1456, "leaf": 182, "slot": 0, "values": [[194573393572430413,575541586100696177,17185197193417606866],[15167111713629212848,3371841055974538903,389590522308985887],[9856468522289727427,1141024469855737413,8823427854447807630],[10019267653830064531,7797126114468478428,3383543947467966788],[16420762772661263866,17352027881895391520,1058807058912222348],[16025551977927421603,5271612813891409186,3946984757956381153],[167401832213904072,11333883202427868807,9026386292192864724],[13734568471847334025,15888679784224105042,16695142320317115780]], "path_len": 9}, {"layer": 1, "d": 2, "position": 182, "leaf": 45, "slot": 2, "values": [[7985111730242223468,13043427922133413739,15165458788387177067],[11920096695007353835,4322196391270192330,6331521415760836415],[17528524193106433492,16056325209359722539,13396748850679192187],[11059400933671702759,3929660671090461634,17528699381798378854]], "path_len": 7}, {"layer": 2, "d": 1, "position": 45, "leaf": 22, "slot": 1, "values": [[11036601194838964653,13091553256446016225,17725342463852698195],[17165721511809460325,16005781762499650030,6120337115418879894]], "path_len": 6}, {"layer": 3, "d": 2, "position": 22, "leaf": 5, "slot": 2, "values": [[10080803378345683631,13581445870922208388,112183069305103648],[274796791223342208,11333819441498540010,18272391722137453186],[935894502491640394,8069316797606646829,2723518402895760156],[5241309797487768752,1497453635557166066,4753616983058012740]], "path_len": 4}]}, + {"iota": 2121, "trace_leaf": 2121, "trace_path_len": 12, "deep": [5932154655850286336,14843004070290831321,5490619261809314267], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 2121, "leaf": 265, "slot": 1, "values": [[9752620111750347501,17004819712705738208,10174864346111711421],[5932154655850286336,14843004070290831321,5490619261809314267],[12457881330515312702,9438882645062572601,2477304330124838740],[13342612810388937410,1955529733589402555,17637786938873078095],[11463360995704355283,5397845986356162493,5459989775793774225],[17202488491312627995,4756487032691328612,7189238154931068032],[12446893681390783201,13814247764991316483,10749934184693828068],[17656899509184221667,3230105340003095516,1646950555390068296]], "path_len": 9}, {"layer": 1, "d": 2, "position": 265, "leaf": 66, "slot": 1, "values": [[143943179255856116,4798827311101200581,11778189320573623738],[11960653567770059154,15137154786077885916,3444065176719318769],[15345592422504941107,13385472108954667751,16279989344517567620],[294297353456909017,2378799116308021259,5821554510368416071]], "path_len": 7}, {"layer": 2, "d": 1, "position": 66, "leaf": 33, "slot": 0, "values": [[4366185996301223320,18130458512453329325,12321842910238819391],[13152781073289974129,16384943335603405623,14986113949777996567]], "path_len": 6}, {"layer": 3, "d": 2, "position": 33, "leaf": 8, "slot": 1, "values": [[15524180633480904874,14071768287211384567,1927601050817459981],[13921811725908965900,9784889492536125216,6245093131103517056],[12348990429102907453,7378250459311877760,16884998755649302425],[3694751754196132007,6208476045974134093,12303339167180175644]], "path_len": 4}]}, + {"iota": 1748, "trace_leaf": 1748, "trace_path_len": 12, "deep": [9397218555520598655,18184148177222589316,16018724279546536410], "terminal_position": 6, "layers": [{"layer": 0, "d": 3, "position": 1748, "leaf": 218, "slot": 4, "values": [[17724856226765146592,10082092572657001926,13645259543255264275],[863625578904618075,10400287656113463467,16528818973977431892],[11599916912452455840,352475967223050044,13498709060109220207],[18319345150264424015,15100223399046302655,14937551297828451863],[9397218555520598655,18184148177222589316,16018724279546536410],[14436493135967899386,7768390391746153895,1993950119548277638],[6389622230992030034,9873739093623680361,12974578811051650909],[12122986716699513979,3853691390967758280,3045013771093848114]], "path_len": 9}, {"layer": 1, "d": 2, "position": 218, "leaf": 54, "slot": 2, "values": [[6707930231302068305,6289296625208072513,6228121725933507311],[3402815304253234050,7931680117825231650,5904690421620213191],[3811509250261400570,10486550803692321562,6467245992250408341],[18434400026770783222,3810674857839593215,178699979830025126]], "path_len": 7}, {"layer": 2, "d": 1, "position": 54, "leaf": 27, "slot": 0, "values": [[11497692889911500389,4956481265465714259,11985015358430959406],[2966973332759365868,15603791491812407637,10069156031838418994]], "path_len": 6}, {"layer": 3, "d": 2, "position": 27, "leaf": 6, "slot": 3, "values": [[3392282136864393874,10616718363163724573,17205136376367452557],[16827435575822246951,2730373153072397240,9528833467876665279],[16245708365531335642,8964017129754350926,7380809409241229694],[1307506453050297819,16732696313432695426,12187551015507047031]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_3_2_1_2.rkyv b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_3_2_1_2.rkyv new file mode 100644 index 000000000..3d83f778c Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_3_2_1_2.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_pair.json b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_pair.json new file mode 100644 index 000000000..b737d89d5 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_pair.json @@ -0,0 +1,30 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "blake3", + "format": "one_row_pair", + "proof_rkyv": "e_proof_blake3_one_row_pair.rkyv", + "proof_rkyv_len": 12872, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "one_row": true, + "query_bound": 4096, + "trace_tree_depth": 12, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["ffcda5bca5d29a901db3846a0a5fa7a118262e25e4728bb756f5c3e049d81721","10eb5cc9eeee08e57157087e948c84c60527a46ceeb0d2089ddf045d779a2413","af88804bc30f486661c073c2e6ff7a5a8d21afe8a0770f1158bb1b25d181b7f8","42cee7233e12c7eb7632517a3f13a5cb95766d730eecbdd40639ed8967584469","5ed6e25331051300afac1ab13117757d4e3527a7bf5ee9b7c76446fb48f7bde0","5ab17f3d3712ad338e104eb211a723c069b954ac84e050cc42c484cc749d914e","4142f22a168743d89b8a6bb8e5faf9ebd05d4942673a1cbffe2cc0275fcec5a8","1943156b542514107b16253cbb66bb7c511207bff9832d934520c4f998e343fe"], + "zetas": [[12610513987238684980,2656594610096053963,9788033839815437623],[8202322326541763249,4638671715876570408,1548089539524959966],[5418912639886591044,14436703782498228938,13150451438376090423],[16129413131115373670,12116921447145899673,1789434332460341265],[5229606843698452721,2538028647533910313,2087788943472497651],[7882549724414645682,15998438433459111387,17172254213527133673],[15797561344767407336,7451312555724701235,9353079321439324766],[2630306677520305007,18443316671340290298,698233072451755557]], + "terminal_coeffs": [[8166688057727980294,3527098508508475499,3579219994840971435],[13088957070833237586,2843452289971455650,13931698865628860263],[10280665272663538462,16957920478250150732,16525456308843497239],[3490947975253976115,15530748444489164043,8363654267677576632]], + "queries_detail": [ + {"iota": 908, "trace_leaf": 908, "trace_path_len": 12, "deep": [8337040516664561595,3591184463121466475,5880410691370975250], "terminal_position": 3, "layers": [{"layer": 0, "d": 1, "position": 908, "leaf": 454, "slot": 0, "values": [[8337040516664561595,3591184463121466475,5880410691370975250],[9942079076094822757,15207253271776161812,17271810815973175383]], "path_len": 11}, {"layer": 1, "d": 1, "position": 454, "leaf": 227, "slot": 0, "values": [[6500024495234768669,14790536704385357656,10970653425198349506],[3037118830635648665,12049339759137376056,2058818805291034312]], "path_len": 10}, {"layer": 2, "d": 1, "position": 227, "leaf": 113, "slot": 1, "values": [[12998320690946467439,3143873445886610368,8364194405026736828],[12303592634730705070,7969752057750737167,7711769752711670497]], "path_len": 9}, {"layer": 3, "d": 1, "position": 113, "leaf": 56, "slot": 1, "values": [[6149215606200672108,12230310026488736926,6564511547928486060],[14404622591513458299,7898366117446141210,15470602245685106535]], "path_len": 8}, {"layer": 4, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[9258841562081245909,5303464181200090505,15395838452030486667],[17983690631426950677,9281380336221551905,4689865579398814961]], "path_len": 7}, {"layer": 5, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[16752007877985850286,14449418999911030214,18103323226226493927],[15685253434469102082,1229693862459337190,2229360339491735221]], "path_len": 6}, {"layer": 6, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[13212989330237734024,11511715167835460658,3579953756865186267],[11683874874595907750,17023168257014162671,7683877784938300551]], "path_len": 5}, {"layer": 7, "d": 1, "position": 7, "leaf": 3, "slot": 1, "values": [[3673055120577650390,17761862347260528145,15879867737729884886],[15701614389839158101,16196947250955156894,9204580243531569325]], "path_len": 4}]}, + {"iota": 1667, "trace_leaf": 1667, "trace_path_len": 12, "deep": [13636529597871698334,13678607474726528147,9848862240480814905], "terminal_position": 6, "layers": [{"layer": 0, "d": 1, "position": 1667, "leaf": 833, "slot": 1, "values": [[10220852481278625157,14071873960699589269,828872414573626613],[13636529597871698334,13678607474726528147,9848862240480814905]], "path_len": 11}, {"layer": 1, "d": 1, "position": 833, "leaf": 416, "slot": 1, "values": [[16761011709352981912,10851157093617305415,3734351154823449094],[2178317076330816469,2215320630266717566,10026952272033988103]], "path_len": 10}, {"layer": 2, "d": 1, "position": 416, "leaf": 208, "slot": 0, "values": [[11153389165091966496,8933652262252062319,4639213090021931600],[9036890528007069229,9889438455510066330,7755295326886188573]], "path_len": 9}, {"layer": 3, "d": 1, "position": 208, "leaf": 104, "slot": 0, "values": [[15798430107321119679,7399920888497789797,3851469018050691970],[12265152089375660269,12464326470257268915,13750970462859745799]], "path_len": 8}, {"layer": 4, "d": 1, "position": 104, "leaf": 52, "slot": 0, "values": [[10197137544449174329,2666341861104716732,8994168763272311938],[2467333217759550862,15961397925655894924,3556723982947593317]], "path_len": 7}, {"layer": 5, "d": 1, "position": 52, "leaf": 26, "slot": 0, "values": [[13808654807413576808,12906200930280508316,18118829275719952794],[11445909634129708166,5679871913112149183,1062655856487114970]], "path_len": 6}, {"layer": 6, "d": 1, "position": 26, "leaf": 13, "slot": 0, "values": [[16433333226992445962,4282547907874224073,5296795991017901170],[3650660750394594762,12394385974057356737,16977602210444761993]], "path_len": 5}, {"layer": 7, "d": 1, "position": 13, "leaf": 6, "slot": 1, "values": [[14131047591148505700,8704142041558711804,15425037308105431711],[12808742107704184070,12949984306744884736,12983161641959877635]], "path_len": 4}]}, + {"iota": 2556, "trace_leaf": 2556, "trace_path_len": 12, "deep": [14961774084741758894,17724548465625463449,407937579721329606], "terminal_position": 9, "layers": [{"layer": 0, "d": 1, "position": 2556, "leaf": 1278, "slot": 0, "values": [[14961774084741758894,17724548465625463449,407937579721329606],[10956977985548048024,1607572610044263675,9234012651162083785]], "path_len": 11}, {"layer": 1, "d": 1, "position": 1278, "leaf": 639, "slot": 0, "values": [[12579967482476121137,8013459107863289407,903732426692410311],[10959159311742709132,3796355782169593955,16416941109827413153]], "path_len": 10}, {"layer": 2, "d": 1, "position": 639, "leaf": 319, "slot": 1, "values": [[14145955780063915793,223976980533745232,4941025582364757750],[979968522591148177,17376083620876130881,8934103765040324984]], "path_len": 9}, {"layer": 3, "d": 1, "position": 319, "leaf": 159, "slot": 1, "values": [[93790095226020304,5037292919527109604,5200685977454171447],[1267192859330294723,15876287822629290895,5670415768498196749]], "path_len": 8}, {"layer": 4, "d": 1, "position": 159, "leaf": 79, "slot": 1, "values": [[17463990687529665122,194968129408922552,2783831484230524712],[5763012266564981842,7685162751697513212,18392797312610438655]], "path_len": 7}, {"layer": 5, "d": 1, "position": 79, "leaf": 39, "slot": 1, "values": [[6387031204678005052,5691431782676540765,17705074806211579330],[4488774873052044926,1720374853552626477,12277685967489572963]], "path_len": 6}, {"layer": 6, "d": 1, "position": 39, "leaf": 19, "slot": 1, "values": [[11996208840794354830,17068718129454278781,14889985865285000741],[12959561901970052932,6028487744399307154,12374012229986672980]], "path_len": 5}, {"layer": 7, "d": 1, "position": 19, "leaf": 9, "slot": 1, "values": [[15922681132257262015,4190699033194233110,8485117899075106073],[9473060730087051793,12084821640297482348,1315857464550192679]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_pair.rkyv new file mode 100644 index 000000000..2f9915aac Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/e_proof_blake3_one_row_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_3_2_1_2.json b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_3_2_1_2.json new file mode 100644 index 000000000..287ca4ba2 --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_3_2_1_2.json @@ -0,0 +1,30 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "keccak", + "format": "one_row_3_2_1_2", + "proof_rkyv": "e_proof_keccak_one_row_3_2_1_2.rkyv", + "proof_rkyv_len": 9432, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "one_row": true, + "query_bound": 4096, + "trace_tree_depth": 12, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 1, 2], + "fri_roots": ["0573d33cd3831d78041a924b5fb8e99c6e7869182059338fc15f7a4733ee0a85","90290b9eb0e2d969efcdf2aaf485507286ffb07d43f6820471f7f23cd188a131","006f08a7f15cb5500f5cfd9b0a63cff4740c9bfc5568651611a59584e2131c4c","5969363fdc7f452e555c8fe81f1067568f2db56604cd9252b1863428573ed9ef"], + "zetas": [[614569478871447995,15598463788603497943,12360548214323292500],[2749684798805164550,11842624514060754293,11317727074124763797],[11378158352117685781,13883522662539884430,10300089410576297656],[18245032832565177967,14493753044579952825,7681180882173639262]], + "terminal_coeffs": [[16982082107160915382,2282929924407988814,10086391487892448182],[15345308450015855078,1512152681275637452,17871467084486542926],[15729379371492426567,14327971657269957480,15709819461529248088],[2510735358381904920,10777919307354146356,849522161725247283]], + "queries_detail": [ + {"iota": 3747, "trace_leaf": 3747, "trace_path_len": 12, "deep": [13162161543905520345,16222340621118398767,14812861061492486552], "terminal_position": 14, "layers": [{"layer": 0, "d": 3, "position": 3747, "leaf": 468, "slot": 3, "values": [[11939368898252254820,7966298589702067435,13923464541084449620],[16519000565047420203,14800180017840624901,2473854278768394879],[18353344654355725185,7598655595952838514,485174952313354076],[13162161543905520345,16222340621118398767,14812861061492486552],[516438490002139474,1312461157064834419,11778937658407907710],[7512883462459603712,11575140191108900564,9299557270393767791],[13927402591382951840,14922292783994652432,2270976895176414228],[13126156821235988227,13558411403710322322,15584318315517571146]], "path_len": 9}, {"layer": 1, "d": 2, "position": 468, "leaf": 117, "slot": 0, "values": [[9741722024427321673,15417388380850289838,8511550521789513588],[17090311002612650370,17871609573955562961,15251904324037649227],[3481455838415491242,12644378374962812916,4629625775108086156],[2520177331257733270,5248631378196257018,17601467731237825492]], "path_len": 7}, {"layer": 2, "d": 1, "position": 117, "leaf": 58, "slot": 1, "values": [[1171841683027868450,3516707735676916944,8408424371334170472],[13055852338994778977,6161935832821634597,17625442510433625962]], "path_len": 6}, {"layer": 3, "d": 2, "position": 58, "leaf": 14, "slot": 2, "values": [[2041002906107072756,9720053546030065860,555422539991461512],[8649597796105717676,9887359070425505677,7903904779785967548],[12514932492420530080,11822836074630855773,12630954109616877078],[13178656397698138192,14330519614295969207,1587797339945367521]], "path_len": 4}]}, + {"iota": 3932, "trace_leaf": 3932, "trace_path_len": 12, "deep": [11997983309011693619,14832433174498138681,3972042518256438394], "terminal_position": 15, "layers": [{"layer": 0, "d": 3, "position": 3932, "leaf": 491, "slot": 4, "values": [[593841392894287482,3244515950377860767,17235984492213527070],[13357798639288218697,1603406640572299408,1355476450824903574],[14365125491994659199,17578081687479202067,2333721695909416367],[14797892649259416114,17329545340641227275,9090376908372138884],[11997983309011693619,14832433174498138681,3972042518256438394],[2404507422347028884,3461603417214790720,13793889970733275211],[16958760744710710146,8223175518268331369,6936434935236651275],[11435013313213504764,13514836260626656026,3988205428095959867]], "path_len": 9}, {"layer": 1, "d": 2, "position": 491, "leaf": 122, "slot": 3, "values": [[6441239995333214949,2008557044640809954,6302267122927451198],[5755366049690739100,5940433195631773921,10073298633835659656],[10355852173748934975,375915313542881553,9614581595079369722],[9838682547908915799,13076759558720065724,15972429628786102473]], "path_len": 7}, {"layer": 2, "d": 1, "position": 122, "leaf": 61, "slot": 0, "values": [[9046291822940486809,15904882642864702632,9243802697598860090],[637086985604128903,11984196675483089673,10105544773484042559]], "path_len": 6}, {"layer": 3, "d": 2, "position": 61, "leaf": 15, "slot": 1, "values": [[13705054946293609652,10520758388559636598,1843839319895042140],[8984595321379749268,957464648666588723,5697566634900201733],[6979165281995072482,14556780583710113004,18196894460862786007],[5854409060872630881,11874995298834561657,18087563126396428879]], "path_len": 4}]}, + {"iota": 2157, "trace_leaf": 2157, "trace_path_len": 12, "deep": [17241446171198887870,9089374642387650352,4362693261445459131], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 2157, "leaf": 269, "slot": 5, "values": [[18072602033057449462,791493680086261727,5006389479070283833],[13982827640424845255,1912086835758784968,9468131248488756799],[15669337530775031072,10768501467625042840,11838115817859132235],[9770512334600625737,12989387495815999473,1158167208066671977],[17046020249111576087,1346189118933924257,14161816367472112208],[17241446171198887870,9089374642387650352,4362693261445459131],[8007161180953444347,4632969092460641420,18079743432785932568],[18364944787622309257,6409634255995638389,17237001282452441669]], "path_len": 9}, {"layer": 1, "d": 2, "position": 269, "leaf": 67, "slot": 1, "values": [[659414522409282892,14050932206177175844,18208880827946597537],[4678902988385856119,18195843974585314494,806802659805655072],[10558643965950283455,12223164169372974730,8716989064208992085],[14026387295497206822,2716121227615181969,6807619021643285971]], "path_len": 7}, {"layer": 2, "d": 1, "position": 67, "leaf": 33, "slot": 1, "values": [[9471244475049593593,17798865937770683050,14180813714250709557],[14814757386327982232,16484305865918755940,12556180948754871812]], "path_len": 6}, {"layer": 3, "d": 2, "position": 33, "leaf": 8, "slot": 1, "values": [[10573339342448139864,13245993176403407621,8769913239944605419],[16805842258116935004,8741300544834758532,7091541834846627245],[6988904484043719279,18021125333806579078,8922319143029332726],[7354216556376030521,16944997308615623648,6846330711616610055]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_3_2_1_2.rkyv b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_3_2_1_2.rkyv new file mode 100644 index 000000000..df7b44bd1 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_3_2_1_2.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_pair.json b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_pair.json new file mode 100644 index 000000000..cc2aa5ecb --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_pair.json @@ -0,0 +1,30 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "keccak", + "format": "one_row_pair", + "proof_rkyv": "e_proof_keccak_one_row_pair.rkyv", + "proof_rkyv_len": 12872, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "one_row": true, + "query_bound": 4096, + "trace_tree_depth": 12, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["eb1586bcc3bde284f3670ea0fc65169877abd7992d5dcffac57e32cdfc0ba5f2","30b219ab4e8860d4f3c872e81f11240bc60e921a204b888f53f4348dd589c2ed","ec7372f78de6460bb087801ee033a25fb3c016625d1acb946c1e96e0a55d29a8","8f88290226e193b7acc7e388a2a127e9d692f577fb235912df6eb2ed3d726bf5","87f44cf42799f46c75c2cec1cf06b6e9af17948837dd4cfa03de3cce38ebcb96","90a1b58155d4e93a0beec06f1b1fa52e9fe4935083e0ae7314cc9f3be42a4af4","6e7269bd4c2931aee21f1ecd16dbce5485e7d78a83fbd62f34b7f12e0d014bd3","6de72339fcca67b8bbd95f83392ba7e72e6e417d8f2d76cf219d2e4c0229561b"], + "zetas": [[5395733614478478870,11359234539595361029,9468765475367811309],[4691404022056334253,17758983664188144681,8299164536713330578],[5571467312536115048,5474225967440588377,2237621988735074941],[16344818043268592932,10747181205106830409,3900555077883816039],[13194324152105116145,9291931726475171977,5358827732935644598],[3745110952868976217,4893045201053783273,1326274809970111783],[15086693329382369694,10223940530923563648,9474084692470291838],[2414133768868227156,15636351245895875877,1237127212595351297]], + "terminal_coeffs": [[6924181524295252559,1631579396016903923,6572868464790892487],[15080983657215378178,5352747742150251534,1033168116145250917],[9161197465518332209,1467001063992449915,13256635128719367890],[11794544518733838288,17573472610644282767,9381019061455135878]], + "queries_detail": [ + {"iota": 762, "trace_leaf": 762, "trace_path_len": 12, "deep": [10094068370361461326,18298575760673302723,167760047989915880], "terminal_position": 2, "layers": [{"layer": 0, "d": 1, "position": 762, "leaf": 381, "slot": 0, "values": [[10094068370361461326,18298575760673302723,167760047989915880],[5459334766786537305,7232075403155012048,2842130299185158477]], "path_len": 11}, {"layer": 1, "d": 1, "position": 381, "leaf": 190, "slot": 1, "values": [[6567901125757655267,17890067315322448035,14925681233931409419],[18235131754497127385,4333965804878517668,15984067725454874720]], "path_len": 10}, {"layer": 2, "d": 1, "position": 190, "leaf": 95, "slot": 0, "values": [[14137450043918885600,2486776962123393416,10502354082803621117],[15828920042687943811,16961409959142313627,2460745610550574849]], "path_len": 9}, {"layer": 3, "d": 1, "position": 95, "leaf": 47, "slot": 1, "values": [[10227001240940270478,17070186034540605583,17207116355067825115],[10861036229411731283,2596952122795351504,8962616386048239227]], "path_len": 8}, {"layer": 4, "d": 1, "position": 47, "leaf": 23, "slot": 1, "values": [[14899232665400030951,11822327205407897399,12852772973679258007],[67353967484070222,7344793933182310052,13538204803111544355]], "path_len": 7}, {"layer": 5, "d": 1, "position": 23, "leaf": 11, "slot": 1, "values": [[1048029576799362121,15752469893941610794,15074550518954149367],[4813813954182030731,14827624934013545034,14558705574433931634]], "path_len": 6}, {"layer": 6, "d": 1, "position": 11, "leaf": 5, "slot": 1, "values": [[2992753296415936463,1047880712958114015,588465930372032317],[2822506618795001014,3554454054060556307,3722117492988577995]], "path_len": 5}, {"layer": 7, "d": 1, "position": 5, "leaf": 2, "slot": 1, "values": [[14523993385875431820,10849282782304498204,17992197523634908884],[7510878449777876572,17576379251232901406,3916410497425810328]], "path_len": 4}]}, + {"iota": 1814, "trace_leaf": 1814, "trace_path_len": 12, "deep": [3486173205532075028,13501479726213628195,7711397338049492637], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 1814, "leaf": 907, "slot": 0, "values": [[3486173205532075028,13501479726213628195,7711397338049492637],[2724410617343204033,17041886471397262471,10048053216853072532]], "path_len": 11}, {"layer": 1, "d": 1, "position": 907, "leaf": 453, "slot": 1, "values": [[15218919491708099717,6265300493536016072,17374044107043655971],[15414362704001559067,5910203742828375847,18174508871703012260]], "path_len": 10}, {"layer": 2, "d": 1, "position": 453, "leaf": 226, "slot": 1, "values": [[8297598642705710651,8896841861319899434,4271968726705068454],[14232960524765109061,1751219308888705154,1611213564303633947]], "path_len": 9}, {"layer": 3, "d": 1, "position": 226, "leaf": 113, "slot": 0, "values": [[10187672769979218181,6883190422014197230,17630315930747985708],[11543340710353323292,2579691248118751998,10788465591631670544]], "path_len": 8}, {"layer": 4, "d": 1, "position": 113, "leaf": 56, "slot": 1, "values": [[3255185536445046756,801363096168136444,2144739588122306839],[2881737247016614189,16267857870605242701,2892508963994520231]], "path_len": 7}, {"layer": 5, "d": 1, "position": 56, "leaf": 28, "slot": 0, "values": [[10005708081878913971,16883002466271960497,15819714690417286842],[8979443613673943889,446485870763451531,7396295827609540705]], "path_len": 6}, {"layer": 6, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[11530783784575839535,16830773339017628756,13263531435704047809],[3000166235818018639,10723029199016018551,4802476952484817780]], "path_len": 5}, {"layer": 7, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[15108810641842900353,9017572136933373745,827650886629197094],[8879386830232170817,4440284036099816188,756978902737227389]], "path_len": 4}]}, + {"iota": 1574, "trace_leaf": 1574, "trace_path_len": 12, "deep": [2436382877645926339,12231969268009871513,13861361779801063289], "terminal_position": 6, "layers": [{"layer": 0, "d": 1, "position": 1574, "leaf": 787, "slot": 0, "values": [[2436382877645926339,12231969268009871513,13861361779801063289],[2290948958860766437,9846444826915849467,11266926617187786854]], "path_len": 11}, {"layer": 1, "d": 1, "position": 787, "leaf": 393, "slot": 1, "values": [[3052277439643912161,790057607007591807,1104643025786218165],[13383466910132863239,8751924119524353407,1366177720789238310]], "path_len": 10}, {"layer": 2, "d": 1, "position": 393, "leaf": 196, "slot": 1, "values": [[9149083653236277356,15817005062217202304,1581999755255560507],[18317007025956390544,15382505965948419620,1288943076662677473]], "path_len": 9}, {"layer": 3, "d": 1, "position": 196, "leaf": 98, "slot": 0, "values": [[2435561994978144773,9972848500245771018,15897565491963037741],[16346129647770001039,4519376622261929280,16372045926229684526]], "path_len": 8}, {"layer": 4, "d": 1, "position": 98, "leaf": 49, "slot": 0, "values": [[2655294817086169804,3003897080814048378,15077680701138897251],[13861152057247201528,1734797488345216608,10118591602100077410]], "path_len": 7}, {"layer": 5, "d": 1, "position": 49, "leaf": 24, "slot": 1, "values": [[9208453110434430430,10253020393070553579,3519126129019616401],[8333278291328356074,2249190254359800375,15583977317131110273]], "path_len": 6}, {"layer": 6, "d": 1, "position": 24, "leaf": 12, "slot": 0, "values": [[6821195066155593029,7962639569178694042,7256273041640925963],[6951321742118564316,3964363899377279367,6170583504292385774]], "path_len": 5}, {"layer": 7, "d": 1, "position": 12, "leaf": 6, "slot": 0, "values": [[527725121383626666,16173092967399922496,4843509717089046499],[12008023089913322763,2738286001211741840,9628318339839965077]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_pair.rkyv new file mode 100644 index 000000000..917d6a823 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/e_proof_keccak_one_row_pair.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_3_2_1_2.json b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_3_2_1_2.json new file mode 100644 index 000000000..d9db0388e --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_3_2_1_2.json @@ -0,0 +1,30 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "rpx", + "format": "one_row_3_2_1_2", + "proof_rkyv": "e_proof_rpx_one_row_3_2_1_2.rkyv", + "proof_rkyv_len": 9432, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "one_row": true, + "query_bound": 4096, + "trace_tree_depth": 12, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [3, 2, 1, 2], + "fri_roots": ["c8c6a3857a1e9d7c2f61fd46746e2bb3535cdb9f63b6e22d3ec7f9a5e383a2f2","95f4386479d6d55922c93cfdea9675fb5f65b0c2321b74dffb788cf03ae2c53c","ebdfff240e34a91a0648ee7466adb199067ae4715f31db95ed0776b6213f786f","9ba65129b5413142af80b633ecb5a1d37f6c8816c451709ae71b0674281831a5"], + "zetas": [[2072553990002038672,3152733128493477078,11602711029815742146],[15459197400690133497,11842684412984680192,16710139345478133626],[13216241016891166733,2969942746573330626,6504928145990413379],[2081893305540362911,6699593400215758319,13627062838661458060]], + "terminal_coeffs": [[8917127793609234202,3818436251448018868,9153923638574958298],[41372134856129893,14445791458964858478,6916624050008194800],[1369532866486613187,17310202425868976552,5527516346235608556],[113871941917206458,5117289324887501201,17934737032884348866]], + "queries_detail": [ + {"iota": 2205, "trace_leaf": 2205, "trace_path_len": 12, "deep": [6857190562724640788,4007496433083125706,17921103436030748622], "terminal_position": 8, "layers": [{"layer": 0, "d": 3, "position": 2205, "leaf": 275, "slot": 5, "values": [[4602638682387835976,8451276510867598838,14669621850686772327],[8046089631358719438,6000422636414950401,17791408447453155495],[13224227945885332137,4022497221569521121,14132968543576776612],[8366093996352563224,10586245183137297300,16590897440396536726],[10577642219811204609,8132865527707081930,3794842370168730599],[6857190562724640788,4007496433083125706,17921103436030748622],[17771010997934343328,9579289656828118766,11375662375469437872],[11752688824513319943,11813227599571567056,203875144659324433]], "path_len": 9}, {"layer": 1, "d": 2, "position": 275, "leaf": 68, "slot": 3, "values": [[14027603398985142442,14931974495511084269,13169342190365263967],[10729570682734190201,13360886974794249550,12166794907646215730],[17505870873946782225,1272114538253616029,7221576044859230217],[14680968125359200025,1834120669802456377,7634397345903233497]], "path_len": 7}, {"layer": 2, "d": 1, "position": 68, "leaf": 34, "slot": 0, "values": [[1755336120330909687,4738723403799425508,16526304371652197035],[6198070429371798043,9875400842722205776,4978023505035734507]], "path_len": 6}, {"layer": 3, "d": 2, "position": 34, "leaf": 8, "slot": 2, "values": [[12432865959921923654,8411549371093338152,4907757842896396886],[16829062543972548747,4595223385839320908,17306133878441923256],[9516230404474158645,184282845086571435,16960709705922000024],[3121559606125253722,16789510687716757680,2036697317195379873]], "path_len": 4}]}, + {"iota": 752, "trace_leaf": 752, "trace_path_len": 12, "deep": [1916903000875823615,16736707246874285829,9564656102773943099], "terminal_position": 2, "layers": [{"layer": 0, "d": 3, "position": 752, "leaf": 94, "slot": 0, "values": [[1916903000875823615,16736707246874285829,9564656102773943099],[11453173613332389795,4145797392594904289,7876548709028745774],[4615854952402447642,14543475126162204942,12910463926362207121],[7593405292236047148,8724292599272739129,17322028385419128472],[12625625798216679042,18063690514277702530,14755773946776919650],[4912295306283908672,14347175193378890942,7951297300128137473],[12063826769130634695,7054630009345082543,10321569615007428609],[14137661524220596099,3152399428801735017,10004540932738062053]], "path_len": 9}, {"layer": 1, "d": 2, "position": 94, "leaf": 23, "slot": 2, "values": [[8811625971723314382,7552532437654286077,16441080950187543807],[17484288545086663,16256622829149716703,8461021173188657701],[6252273343596476642,11321822468352368566,9211443333124516620],[8039005244383157612,6950435551192179736,13659146476691124801]], "path_len": 7}, {"layer": 2, "d": 1, "position": 23, "leaf": 11, "slot": 1, "values": [[18056204561276561229,14835128819825911563,5046426107340521641],[5893548925016901766,4760873787913692676,18439055921004270113]], "path_len": 6}, {"layer": 3, "d": 2, "position": 11, "leaf": 2, "slot": 3, "values": [[10904235002716272775,9975472978635657080,14964204635917748194],[5305060826180553417,12937376069285077497,3287539991567464608],[8719348652353813625,994707129833621565,11914955711098119318],[17889548190955188583,5660707692435915686,3267735454667947369]], "path_len": 4}]}, + {"iota": 760, "trace_leaf": 760, "trace_path_len": 12, "deep": [18267711020647082997,13248031096375226280,188428981296028522], "terminal_position": 2, "layers": [{"layer": 0, "d": 3, "position": 760, "leaf": 95, "slot": 0, "values": [[18267711020647082997,13248031096375226280,188428981296028522],[12182114973803986625,9335268974613975205,10075971023962334808],[12163018125168083220,2817955096360236833,15999670448454868863],[8495492306395569540,4095933714799065998,13368466763250358722],[8006825421091369573,5803517742005105201,1788791879629939833],[1820353059050006357,5346090292370034386,2193165956274050915],[802007586120352552,16206387393817464715,4632934878557695217],[7783716627999292698,9167226174152632427,17535031471772053546]], "path_len": 9}, {"layer": 1, "d": 2, "position": 95, "leaf": 23, "slot": 3, "values": [[8811625971723314382,7552532437654286077,16441080950187543807],[17484288545086663,16256622829149716703,8461021173188657701],[6252273343596476642,11321822468352368566,9211443333124516620],[8039005244383157612,6950435551192179736,13659146476691124801]], "path_len": 7}, {"layer": 2, "d": 1, "position": 23, "leaf": 11, "slot": 1, "values": [[18056204561276561229,14835128819825911563,5046426107340521641],[5893548925016901766,4760873787913692676,18439055921004270113]], "path_len": 6}, {"layer": 3, "d": 2, "position": 11, "leaf": 2, "slot": 3, "values": [[10904235002716272775,9975472978635657080,14964204635917748194],[5305060826180553417,12937376069285077497,3287539991567464608],[8719348652353813625,994707129833621565,11914955711098119318],[17889548190955188583,5660707692435915686,3267735454667947369]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_3_2_1_2.rkyv b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_3_2_1_2.rkyv new file mode 100644 index 000000000..b73dc5fb5 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_3_2_1_2.rkyv differ diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_pair.json b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_pair.json new file mode 100644 index 000000000..859468b1a --- /dev/null +++ b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_pair.json @@ -0,0 +1,30 @@ +{ + "generator": "stark::fri::vectors::proof_vectors", + "hash": "rpx", + "format": "one_row_pair", + "proof_rkyv": "e_proof_rpx_one_row_pair.rkyv", + "proof_rkyv_len": 12872, + "air": "LogReadOnlyRAP, reads (i % 5 + 1, 10·(i % 5 + 1))", + "trace_rows": 1024, + "lde_log": 12, + "blowup": 4, + "fri_final_poly_log_degree": 2, + "queries": 3, + "grinding_factor": 0, + "coset_offset": 3, + "one_row": true, + "query_bound": 4096, + "trace_tree_depth": 12, + "legacy_encoding": false, + "total_folds": 8, + "terminal_len": 16, + "schedule": [1, 1, 1, 1, 1, 1, 1, 1], + "fri_roots": ["77e5d86a0702fe0c50672508977a685b2bfdae2cd941fad392c8294731c9e422","7258024a9819630328acea419c9a638b25bdbdee4bd724583300d7d1848f5519","7ab08ab352939e6e9b968fce340beb441ed26f4d1f179b03e74183a4503d1239","0d41af1406e7f515bad385e1fdcc4d28446fe84ab66f3fec873810cbec793c33","81e8a52594b1153a62d1ef60f5b10785044c11977864f8598b7bb7e677a383f5","458ef527b1574a69063b821685e35402eb11527965e9a8afbfe27d1c055a706d","ed0dac83bdc4727544d623f801c76a700690e70738888ddd5b58ce4ec98aad16","b87b6976a6466e4ba2a542e3e401ab0dc8ad488df17e4e95f35413220c779215"], + "zetas": [[11356657239044866106,13376194269418604399,16611853721418243315],[10366391614236701735,449823887536073129,4380920291293489693],[11922106070388417666,2861436225089580234,7941726727098846953],[6173349043357729189,13221589175675104657,12541630541605756582],[9571725825048407501,8781332429625970728,14051607987466410322],[271455075676084364,8661011714467229856,11921036807597295366],[15420589626999451879,2381820146514785423,464796940378704146],[9630091406616888336,14072038474898281586,2889648399077434883]], + "terminal_coeffs": [[9252102389862037098,11485521080712742978,15330746307211460791],[2408142744451238759,16015006922876515991,8289145647860378560],[3822746191229327094,13873058238083039505,4675575401224124965],[17237282230694182751,10651742490541135044,7065525042114859165]], + "queries_detail": [ + {"iota": 1490, "trace_leaf": 1490, "trace_path_len": 12, "deep": [2932413113042791435,11894382985923903979,9485622643104290083], "terminal_position": 5, "layers": [{"layer": 0, "d": 1, "position": 1490, "leaf": 745, "slot": 0, "values": [[2932413113042791435,11894382985923903979,9485622643104290083],[5617561700813657648,9150526953144641040,5053509220889776834]], "path_len": 11}, {"layer": 1, "d": 1, "position": 745, "leaf": 372, "slot": 1, "values": [[17441972696003475220,9388283442433602320,4390345575578824170],[5133627800531223425,11914988783658563617,14937632697871883584]], "path_len": 10}, {"layer": 2, "d": 1, "position": 372, "leaf": 186, "slot": 0, "values": [[8133041217017086758,1820646995207011242,11484845985140404156],[915012823390639608,17635150337718406583,5780426687318592651]], "path_len": 9}, {"layer": 3, "d": 1, "position": 186, "leaf": 93, "slot": 0, "values": [[10340667171407563529,1547279552216815127,6915765815699207391],[3154596510699264083,818314717512897513,17101621809342283826]], "path_len": 8}, {"layer": 4, "d": 1, "position": 93, "leaf": 46, "slot": 1, "values": [[1886552692492201400,5423010926098300279,15295999817653895971],[1768335035148138120,7519399748343183899,9919504600048365073]], "path_len": 7}, {"layer": 5, "d": 1, "position": 46, "leaf": 23, "slot": 0, "values": [[11863182194811651752,13735783487106290491,14167096383978964230],[17804173782191017059,1101561508558957260,5395509579711619792]], "path_len": 6}, {"layer": 6, "d": 1, "position": 23, "leaf": 11, "slot": 1, "values": [[14734565326035031259,17231263762901376046,10981121081937478029],[1400117789794109009,2476100258917768647,7408475937295308469]], "path_len": 5}, {"layer": 7, "d": 1, "position": 11, "leaf": 5, "slot": 1, "values": [[8182145593033563858,4641852677457838421,8561738423708568284],[17635728374138200680,13543955077887316863,1304709511706523848]], "path_len": 4}]}, + {"iota": 1846, "trace_leaf": 1846, "trace_path_len": 12, "deep": [2284487697263572951,17954063421123594597,12734531888209557865], "terminal_position": 7, "layers": [{"layer": 0, "d": 1, "position": 1846, "leaf": 923, "slot": 0, "values": [[2284487697263572951,17954063421123594597,12734531888209557865],[470910654823987329,4406533214267543541,17643437692058643171]], "path_len": 11}, {"layer": 1, "d": 1, "position": 923, "leaf": 461, "slot": 1, "values": [[13111132197997166221,11524508225265192094,12264127282465523178],[12430851011513078654,15525325890315442870,9377900677550954376]], "path_len": 10}, {"layer": 2, "d": 1, "position": 461, "leaf": 230, "slot": 1, "values": [[12783785382459763502,8900401989734896668,2244166906061745250],[10849887253553603615,4793736872544089110,6359342210468001257]], "path_len": 9}, {"layer": 3, "d": 1, "position": 230, "leaf": 115, "slot": 0, "values": [[8588935302892386269,8551761774913690890,4520450724077434230],[18374304396057544181,18384245929379916003,7814889479906253757]], "path_len": 8}, {"layer": 4, "d": 1, "position": 115, "leaf": 57, "slot": 1, "values": [[5267360283434351791,15392776900246357898,4943372451034957132],[15726203054000898387,8814074561425959376,6200283473451399643]], "path_len": 7}, {"layer": 5, "d": 1, "position": 57, "leaf": 28, "slot": 1, "values": [[8964694237736672537,2508030093710076938,2895555912020756696],[1316945196546810175,9540203727253325223,3428063813032906148]], "path_len": 6}, {"layer": 6, "d": 1, "position": 28, "leaf": 14, "slot": 0, "values": [[5827834535601424539,2126802115856515298,18004136256491770090],[9108720695214937294,5361393747591213366,673049126834026586]], "path_len": 5}, {"layer": 7, "d": 1, "position": 14, "leaf": 7, "slot": 0, "values": [[5890571997869690666,10800464999013389735,9378580896967811195],[704334176429400473,7326528124909318680,5250527536912250307]], "path_len": 4}]}, + {"iota": 3542, "trace_leaf": 3542, "trace_path_len": 12, "deep": [11685830694059955100,12253131036674908176,5644896282847145797], "terminal_position": 13, "layers": [{"layer": 0, "d": 1, "position": 3542, "leaf": 1771, "slot": 0, "values": [[11685830694059955100,12253131036674908176,5644896282847145797],[6796601152078455260,4829064821059167185,8422718924143860404]], "path_len": 11}, {"layer": 1, "d": 1, "position": 1771, "leaf": 885, "slot": 1, "values": [[10348593929766088319,600276997320945892,11551193361309397901],[4497425865068046782,7711618918678573010,3670607890048107603]], "path_len": 10}, {"layer": 2, "d": 1, "position": 885, "leaf": 442, "slot": 1, "values": [[4549713374957844263,14882952107271375459,15888844724609212573],[11867586896494500539,5382822032643071953,2547573499666355760]], "path_len": 9}, {"layer": 3, "d": 1, "position": 442, "leaf": 221, "slot": 0, "values": [[179249684856242416,8237726877005160673,11377156407621955226],[16338343630712151739,4547151014003636050,545748730564614338]], "path_len": 8}, {"layer": 4, "d": 1, "position": 221, "leaf": 110, "slot": 1, "values": [[17445710063408886615,3577256811508087571,13704116923286529205],[16582893718205853404,6964390291711440128,11944101285969035096]], "path_len": 7}, {"layer": 5, "d": 1, "position": 110, "leaf": 55, "slot": 0, "values": [[13599868760913890866,8792505734133014168,15701834533643325861],[17796919121337590328,5161833031535532699,595353643473481312]], "path_len": 6}, {"layer": 6, "d": 1, "position": 55, "leaf": 27, "slot": 1, "values": [[10120435360324233608,4278971934281013570,4137924372760735235],[16590085914178287331,864859356312877269,16418927990479178093]], "path_len": 5}, {"layer": 7, "d": 1, "position": 27, "leaf": 13, "slot": 1, "values": [[16002614920667576628,9559377662140985339,2909311776212759241],[4909556401234084390,18251274463717352852,444518797285025356]], "path_len": 4}]} + ] +} diff --git a/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_pair.rkyv b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_pair.rkyv new file mode 100644 index 000000000..666fd43a2 Binary files /dev/null and b/crypto/stark/tests/vectors/zf_fri/e_proof_rpx_one_row_pair.rkyv differ diff --git a/docs/continuations_design.md b/docs/continuations_design.md index 755a9dcb3..623aff173 100644 --- a/docs/continuations_design.md +++ b/docs/continuations_design.md @@ -594,11 +594,11 @@ becomes an **explicit verifier action**: - **Reconstruct the output** by concatenating the per-epoch commit slices (each commit-bus-bound, contiguous via the x254 chain). - The verifier also `validate()`s `table_counts` — which since per-epoch table skipping - means *only* that CPU and MEMW_R are present and that no accelerator, and no IS_WH, - claims more than one table (for IS_WH that cap is what bounds its multiplicity sum), - not that every table is there — and never trusts a prover-supplied - page config (continuation epochs have none — PAGE is skipped under the L2G - bookend, so `page_configs` is always empty). + means *only* that CPU and MEMW_R are present and that no unchunked chip claims more + than one table (the six accelerators, BLAKE3 and IS_WH; for IS_WH that cap is what + bounds its multiplicity sum), not that every table is there — and + never trusts a prover-supplied page config (continuation epochs have none — PAGE is + skipped under the L2G bookend, so `page_configs` is always empty). A single `build_epoch_airs` helper builds the AIR set identically on both sides, so prove and verify cannot diverge. diff --git a/docs/general_flow.md b/docs/general_flow.md index 155a96ead..0e9fade94 100644 --- a/docs/general_flow.md +++ b/docs/general_flow.md @@ -21,7 +21,7 @@ For a deeper dive into each component see the [proof system overview](./cryptogr ## Accelerated memory operations -`memcpy` is accelerated: `lambda-vm-syscalls` exports it under its standard, unmangled C name, so both explicit calls and the copies the compiler emits implicitly (struct moves, slice copies, `Vec` growth) reach the DMA ecall with no guest source changes. Behaviour is identical to the C function for every input, including `n == 0` and any alignment of `dest`, `src` or `n`. `memmove` and `memset` are accelerated too, through the same chip: `memmove` reuses the copy ecall and chunks backwards when the ranges overlap, and `memset` is expressed as a *propagating* copy — the stub seeds eight bytes with ordinary stores and then calls the accelerator with `dst = seed_end, src = seed_start`, which the chip runs with the read/write timestamp order inverted so the seed walks across the range. Fills shorter than sixteen bytes take a plain store loop instead. `memcmp` is not accelerated and falls back to the toolchain's `compiler-builtins` definition. +`memcpy` is accelerated: `lambda-vm-syscalls` exports it under its standard, unmangled C name, so both explicit calls and the copies the compiler emits implicitly (struct moves, slice copies, `Vec` growth) reach the DMA ecall with no guest source changes. Behaviour is identical to the C function for every input, including `n == 0` and any alignment of `dest`, `src` or `n`. `memmove` and `memset` are accelerated too, through the same chip: `memmove` reuses the copy ecall and chunks backwards when the ranges overlap, and `memset` is expressed as a *propagating* copy — the stub seeds eight bytes with ordinary stores and then calls the accelerator with `dst = seed_end, src = seed_start`, which the chip runs with the read/write timestamp order inverted so the seed walks across the range. Fills shorter than sixteen bytes take a plain store loop instead. `memcmp` is not accelerated, but `lambda-vm-syscalls` replaces the toolchain's byte-at-a-time `compiler-builtins` definition with one that compares a doubleword at a time and returns the same values. **Observability.** `cli execute --cycles` reports `Dma calls`, `Dma bytes` and `Dma rows`. The call line confirms the accelerator is engaged at all; the byte and row lines are the cost, since the guest stub chunks one `memcpy` into as many ecalls as it needs and each ecall adds one row per chunk, where a chunk is eight bytes or one (see the alignment note below). There is no terminal row: the chain stops on the row that moves the last bytes, so a zero-length call is the only one whose single row moves nothing. `Dma rows` is the raw row count the copies contribute, before the DMA trace is padded to a power-of-two height — for a guest with few copies the padded table is larger than the reported figure. diff --git a/executor/programs/asm/dense_data_page_touch.s b/executor/programs/asm/dense_data_page_touch.s new file mode 100644 index 000000000..de6024749 --- /dev/null +++ b/executor/programs/asm/dense_data_page_touch.s @@ -0,0 +1,60 @@ + .data + .align 3 +# `data_page_touch`, but on a page whose genesis is DENSE. +# +# That fixture's .data page holds one .dword, so its INIT column has 112 nonzero +# entries of 262,144. It passes the first half of the routing rule and the +# second half refuses it, so it exercises the SPARSE genesis route and leaves +# the prepared route with no table at fixture scale. +# +# THE RULE IT IS SIZED AGAINST, in `prover/src/continuation.rs` +# (`genesis_stack_plan`), read at 84a02b8c4. It has two parts and this fixture has +# to clear BOTH: +# +# 1. per page: `18 + 18*S` against `marginal_stacked_rows(18, n_fixed)`, the +# rows one more carried page adds to the prepared leg. That marginal is a +# FORM evaluated at the run's own bracket — 101 at one genesis page, 111 at +# the block's thirty, 125 at four thousand — so the entry threshold is five +# or six nonzero bytes and NOT a fixed number to quote here. +# 2. per candidate SET: its total savings against `PREPARED_LEG_ROWS`, the +# 175,066 rows the stacked chain costs once, however many pages ride it. +# This is the part the 112-entry fixture fails: 1,923 saved rows do not buy +# a 175,066-row chain. +# +# This one surrounds the touched cell with non-zero bytes so the page it lives +# on clears part 2 outright, whatever offset the linker puts it at. +# +# ⚠ WHY THE FILL IS ON BOTH SIDES. The counter has to be on a DENSE page, and +# where .data starts inside its page is the linker's business. With `n` bytes +# before and `n` after, the counter's own page holds at least `n` of them +# whatever its offset: a counter at the start of a page keeps the trailing fill, +# one at the end keeps the leading fill, and one in the middle keeps both. A +# single trailing fill would leave a counter near the end of a page with almost +# none of it. +# +# 32 KiB a side puts the floor at 32,768 nonzero entries. That is four orders +# past part 1 at any bracket, and its savings are `18 + 18*32,768 - marginal`, +# about 589,700 rows: 3.4x the chain part 2 weighs them against. The fixture is +# not sized to just barely qualify, which matters because the marginal is a form +# that moves with the run's page count and the chain's cost moves with the +# proof's posture. + .fill 32768, 1, 0xA5 +counter: + .dword 0x123456789ABCDEF0 + .fill 32768, 1, 0x5A + + .text + .attribute 5, "rv64i2p1" + .globl main +main: + # The same five instructions `data_page_touch` runs: load, mutate, store + # back a static global, so the page is genuinely ELF-backed and the cell + # crosses an epoch boundary. Only the page's genesis differs. + la t0, counter # 1: t0 = &counter + ld t1, 0(t0) # 2: t1 = counter (0x123456789ABCDEF0) + addi t1, t1, 1 # 3: t1 += 1 + sd t1, 0(t0) # 4: counter = t1 + + li a0, 0 + li a7, 93 + ecall # 5: Halt diff --git a/executor/programs/asm/test_blake3.s b/executor/programs/asm/test_blake3.s new file mode 100644 index 000000000..d066f4645 --- /dev/null +++ b/executor/programs/asm/test_blake3.s @@ -0,0 +1,60 @@ + .attribute 5, "rv64i2p1_m2p0_zmmul1p0" + .globl main +main: + # 176 bytes on the stack for the BLAKE3 state region (22 x u64): + # h[4 dwords] | m[8] | t[1] | block_len,flags[1] | out[8]. + addi sp, sp, -176 + + # Deterministic non-zero seed over the 14 input dwords: dword[k] = k + 1. + # (t therefore = 13, block_len = 14, flags = 0 — arbitrary but fixed.) + mv t0, sp + li t1, 1 + li t2, 15 +.Linit_loop: + sd t1, 0(t0) + addi t0, t0, 8 + addi t1, t1, 1 + bne t1, t2, .Linit_loop + + # First compression. + # a0 = pointer to the 176-byte region (8-aligned) + # a7 = syscall number (u64::MAX - 2 = -3) + mv a0, sp + li a7, -3 + ecall + + # Chain: copy out (8 dwords at sp+112) over m (8 dwords at sp+32), so the + # second call consumes the first call's output AND its out-region write has + # non-zero previous content. + li t1, 0 +.Lcopy_loop: + slli t2, t1, 3 + addi t3, sp, 112 + add t3, t3, t2 + ld t4, 0(t3) + addi t3, sp, 32 + add t3, t3, t2 + sd t4, 0(t3) + addi t1, t1, 1 + li t2, 8 + bne t1, t2, .Lcopy_loop + + # Second compression. + mv a0, sp + li a7, -3 + ecall + + # Commit the final 64-byte output. + li a0, 1 + addi a1, sp, 112 + li a2, 64 + li a7, 64 + ecall + + # Restore stack and halt. + addi sp, sp, 176 + li a0, 0 + li a7, 93 + ecall +.Lfunc_end0: + .size main, .Lfunc_end0-main diff --git a/executor/programs/asm/test_blake3_absorb.s b/executor/programs/asm/test_blake3_absorb.s new file mode 100644 index 000000000..dbe14e5b2 --- /dev/null +++ b/executor/programs/asm/test_blake3_absorb.s @@ -0,0 +1,79 @@ + .attribute 5, "rv64i2p1_m2p0_zmmul1p0" + .globl main +main: + # 256 bytes on the stack, in two disjoint regions the absorb ABI requires: + # sp+0 .. sp+64 the 64-byte control region (cv_in | cv_out) + # sp+64 .. sp+256 three 64-byte message blocks, read in place + # Both are 8-aligned because sp is. + addi sp, sp, -256 + + # cv_in = dwords 1..4 at sp+0. + mv t0, sp + li t1, 1 + li t2, 5 +.Lcv_loop: + sd t1, 0(t0) + addi t0, t0, 8 + addi t1, t1, 1 + bne t1, t2, .Lcv_loop + + # Message = dwords 100..123 at sp+64 (24 dwords = 3 blocks). + addi t0, sp, 64 + li t1, 100 + li t2, 124 +.Lmsg_loop: + sd t1, 0(t0) + addi t0, t0, 8 + addi t1, t1, 1 + bne t1, t2, .Lmsg_loop + + # Absorb all three blocks in ONE ecall — a group of 3 compression rows plus + # its END row. + # a0 = control region, a1 = message, a2 = num_blocks, a3 = first_flags, + # a7 = syscall number (u64::MAX - 3 = -4). + mv a0, sp + addi a1, sp, 64 + li a2, 3 + li a3, 1 + li a7, -4 + ecall + + # Chain into a SECOND absorb: copy cv_out (sp+32) over cv_in (sp+0). The + # second ecall's END row then writes cv_out where the first one already + # wrote, so its Memw `old` is non-zero rather than fresh memory. + li t1, 0 +.Lchain_loop: + slli t2, t1, 3 + addi t3, sp, 32 + add t3, t3, t2 + ld t4, 0(t3) + mv t3, sp + add t3, t3, t2 + sd t4, 0(t3) + addi t1, t1, 1 + li t2, 4 + bne t1, t2, .Lchain_loop + + # The degenerate group: one block, interior flags. Re-reads the message's + # second block, so the same addresses are touched at a second timestamp. + mv a0, sp + addi a1, sp, 128 + li a2, 1 + li a3, 0 + li a7, -4 + ecall + + # Commit the 32-byte chaining value. + li a0, 1 + addi a1, sp, 32 + li a2, 32 + li a7, 64 + ecall + + # Restore stack and halt. + addi sp, sp, 256 + li a0, 0 + li a7, 93 + ecall +.Lfunc_end0: + .size main, .Lfunc_end0-main diff --git a/executor/programs/asm/test_copy_ecalls.s b/executor/programs/asm/test_copy_ecalls.s new file mode 100644 index 000000000..e44a95045 --- /dev/null +++ b/executor/programs/asm/test_copy_ecalls.s @@ -0,0 +1,74 @@ + .attribute 5, "rv64i2p1" + .globl main +main: + # Every row shape the COPY table has, 17 rows in all, so a cut at a few rows + # leaves a chain across a table boundary and the whole run pads to 32: + # memcpy of 21 bytes 2 wide rows + 5 one-byte rows + # memset of 8 + 11 bytes 1 wide row + 3 one-byte rows (the seed is 8 sb) + # memcpy of 0 bytes the zero-length row + # memcpy of 2 bytes 2 one-byte rows, onto its own source + 1 + # write of 3 bytes 3 one-byte rows into the COMMIT domain + # memcpy is ecall -30, memset -31, write 64; x10 = dst, x11 = src, x12 = n. + addi sp, sp, -128 + + # A source pattern at sp+0..24. + addi t0, zero, 0x11 + sb t0, 0(sp) + addi t0, zero, 0x22 + sb t0, 1(sp) + addi t0, zero, 0x33 + sb t0, 2(sp) + addi t0, zero, 0x44 + sb t0, 9(sp) + addi t0, zero, 0x55 + sb t0, 20(sp) + + # memcpy(sp+40, sp, 21) + addi a0, sp, 40 + mv a1, sp + addi a2, zero, 21 + addi a7, zero, -30 + ecall + + # memset(sp+64, 0x5A, 19): the seed, then the copy from seed to seed + 8 + addi t0, zero, 0x5A + sb t0, 64(sp) + sb t0, 65(sp) + sb t0, 66(sp) + sb t0, 67(sp) + sb t0, 68(sp) + sb t0, 69(sp) + sb t0, 70(sp) + sb t0, 71(sp) + addi a0, sp, 72 + addi a1, sp, 64 + addi a2, zero, 11 + addi a7, zero, -31 + ecall + + # memcpy(sp+100, sp+8, 0) + addi a0, sp, 100 + addi a1, sp, 8 + addi a2, zero, 0 + addi a7, zero, -30 + ecall + + # memcpy(sp+2, sp+1, 2): overlapping, the snapshot reads before it writes + addi a0, sp, 2 + addi a1, sp, 1 + addi a2, zero, 2 + addi a7, zero, -30 + ecall + + # write(1, sp+40, 3): commits 0x11 0x22 0x33 + addi a0, zero, 1 + addi a1, sp, 40 + addi a2, zero, 3 + addi a7, zero, 64 + ecall + + # Halt + addi sp, sp, 128 + addi a0, zero, 0 + addi a7, zero, 93 + ecall diff --git a/executor/programs/asm/test_copy_overlaps.s b/executor/programs/asm/test_copy_overlaps.s new file mode 100644 index 000000000..34fcfad2a --- /dev/null +++ b/executor/programs/asm/test_copy_overlaps.s @@ -0,0 +1,75 @@ + .attribute 5, "rv64i2p1" + .globl main +main: + # The COPY calls whose row split and overlap decide which bytes end at the + # call's timestamp and which one after it: + # memcpy of 16 bytes onto its own source + 5 2 wide rows, forward overlap + # memcpy of 13 bytes onto its own source - 3 1 wide + 5 one-byte rows, + # backward overlap (a memmove) + # memset of 8 + 13 bytes 1 wide + 5 one-byte rows + # memset of 8 + 0 bytes the zero-length row + # memcpy of 7 bytes 7 one-byte rows + # memcpy is ecall -30, memset -31; x10 = dst, x11 = src, x12 = n. + addi sp, sp, -128 + + # A source pattern at sp+0..20. + addi t0, zero, 0x11 + sb t0, 0(sp) + addi t0, zero, 0x22 + sb t0, 3(sp) + addi t0, zero, 0x33 + sb t0, 7(sp) + addi t0, zero, 0x44 + sb t0, 12(sp) + addi t0, zero, 0x55 + sb t0, 15(sp) + + # memcpy(sp+5, sp, 16): the destination overlaps the source's tail + addi a0, sp, 5 + mv a1, sp + addi a2, zero, 16 + addi a7, zero, -30 + ecall + + # memcpy(sp+1, sp+4, 13): the destination overlaps the source's head + addi a0, sp, 1 + addi a1, sp, 4 + addi a2, zero, 13 + addi a7, zero, -30 + ecall + + # memset(sp+64, 0xA5, 21): the seed, then the copy from seed to seed + 8 + addi t0, zero, 0xA5 + sb t0, 64(sp) + sb t0, 65(sp) + sb t0, 66(sp) + sb t0, 67(sp) + sb t0, 68(sp) + sb t0, 69(sp) + sb t0, 70(sp) + sb t0, 71(sp) + addi a0, sp, 72 + addi a1, sp, 64 + addi a2, zero, 13 + addi a7, zero, -31 + ecall + + # memset(sp+96, 0xA5, 8): the seed alone, a zero-length copy after it + addi a0, sp, 104 + addi a1, sp, 96 + addi a2, zero, 0 + addi a7, zero, -31 + ecall + + # memcpy(sp+110, sp+1, 7) + addi a0, sp, 110 + addi a1, sp, 1 + addi a2, zero, 7 + addi a7, zero, -30 + ecall + + # Halt + addi sp, sp, 128 + addi a0, zero, 0 + addi a7, zero, 93 + ecall diff --git a/executor/programs/asm/test_dense_pages.s b/executor/programs/asm/test_dense_pages.s new file mode 100644 index 000000000..44fc8d3f4 --- /dev/null +++ b/executor/programs/asm/test_dense_pages.s @@ -0,0 +1,33 @@ + # Two genesis pages dense enough for the cross-epoch genesis rule to stack + # them: thousands of nonzero bytes each, 2^18 bytes (one page) apart. The + # no-epoch WHIR block's prepared-opening negatives need two dense pages. + .data + .align 12 +page_a: + .fill 12288, 1, 0x5a + .skip 262144 +page_b: + .fill 12288, 1, 0xa5 + + .text + .attribute 5, "rv64i2p1" + .globl main +main: + # Read one byte of each page, commit their sum. + la t0, page_a + lbu t1, 0(t0) + la t0, page_b + lbu t2, 0(t0) + add t1, t1, t2 + addi sp, sp, -16 + sb t1, 0(sp) + li a0, 1 # fd = 1 + mv a1, sp # buf = sp + li a2, 1 # count = 1 + li a7, 64 # syscall = Commit + ecall + + addi sp, sp, 16 + li a0, 0 + li a7, 93 # syscall = Halt + ecall diff --git a/executor/programs/asm/test_ecrec.s b/executor/programs/asm/test_ecrec.s new file mode 100644 index 000000000..14c2133d2 --- /dev/null +++ b/executor/programs/asm/test_ecrec.s @@ -0,0 +1,131 @@ + .attribute 5, "rv64i2p1_m2p0_zmmul1p0" + .globl main +main: + # Three ECREC calls (ECALL -12): two valid signatures and one with r = 0, which bails (status 1). + # Per call c (base 176·c): sig r ‖ s at +0 (64 bytes, big-endian), hash at +64 (32, big-endian), + # output at +96 (72: the key big-endian, then the status doubleword). The signatures are fixed test vectors. + addi sp, sp, -528 + + # --- call 1 --- + li t0, 0xDF93B6A1E4BA8432 + sd t0, 0(sp) + li t0, 0x1F89E40074C68ABE + sd t0, 8(sp) + li t0, 0x15A359511C68533B + sd t0, 16(sp) + li t0, 0x3971C5D2F3511056 + sd t0, 24(sp) + li t0, 0x4EA528F7F2B2D497 + sd t0, 32(sp) + li t0, 0x1DB1F63722A162D5 + sd t0, 40(sp) + li t0, 0x344CC2FCE726D632 + sd t0, 48(sp) + li t0, 0x57D05B099E72B070 + sd t0, 56(sp) + li t0, 0x4433221100EEFFC0 + sd t0, 64(sp) + li t0, 0xBBAA009988776655 + sd t0, 72(sp) + li t0, 0x33221100FFEEDDCC + sd t0, 80(sp) + li t0, 0xBBAA998877665544 + sd t0, 88(sp) + addi a0, sp, 96 + addi a1, sp, 0 + addi a2, sp, 64 + li a3, 0 + li a7, -12 + ecall + + # --- call 2 --- + li t0, 0xF8BCE8385BD46279 + sd t0, 176(sp) + li t0, 0xF2012A43A8EFA82F + sd t0, 184(sp) + li t0, 0x113F7D4CE2539A0C + sd t0, 192(sp) + li t0, 0xDA2609E7B87C19DF + sd t0, 200(sp) + li t0, 0x79042F811168ECEC + sd t0, 208(sp) + li t0, 0x958F9472BE921F4C + sd t0, 216(sp) + li t0, 0x9DA3454487A7CDA6 + sd t0, 224(sp) + li t0, 0xCDF235C831AB0AB4 + sd t0, 232(sp) + li t0, 0x0000000000000000 + sd t0, 240(sp) + li t0, 0x0000000000000000 + sd t0, 248(sp) + li t0, 0x0000000000000000 + sd t0, 256(sp) + li t0, 0x4200000000000000 + sd t0, 264(sp) + addi a0, sp, 272 + addi a1, sp, 176 + addi a2, sp, 240 + li a3, 1 + li a7, -12 + ecall + + # --- call 3 --- + li t0, 0x0000000000000000 + sd t0, 352(sp) + li t0, 0x0000000000000000 + sd t0, 360(sp) + li t0, 0x0000000000000000 + sd t0, 368(sp) + li t0, 0x0000000000000000 + sd t0, 376(sp) + li t0, 0x4EA528F7F2B2D497 + sd t0, 384(sp) + li t0, 0x1DB1F63722A162D5 + sd t0, 392(sp) + li t0, 0x344CC2FCE726D632 + sd t0, 400(sp) + li t0, 0x57D05B099E72B070 + sd t0, 408(sp) + li t0, 0x4433221100EEFFC0 + sd t0, 416(sp) + li t0, 0xBBAA009988776655 + sd t0, 424(sp) + li t0, 0x33221100FFEEDDCC + sd t0, 432(sp) + li t0, 0xBBAA998877665544 + sd t0, 440(sp) + addi a0, sp, 448 + addi a1, sp, 352 + addi a2, sp, 416 + li a3, 0 + li a7, -12 + ecall + + # Commit output 1 (72 bytes). + li a0, 1 + addi a1, sp, 96 + li a2, 72 + li a7, 64 + ecall + + # Commit output 2 (72 bytes). + li a0, 1 + addi a1, sp, 272 + li a2, 72 + li a7, 64 + ecall + + # Commit output 3 (72 bytes). + li a0, 1 + addi a1, sp, 448 + li a2, 72 + li a7, 64 + ecall + + addi sp, sp, 528 + li a0, 0 + li a7, 93 + ecall +.Lfunc_end1: + .size main, .Lfunc_end1-main diff --git a/executor/programs/asm/test_keccak256.s b/executor/programs/asm/test_keccak256.s new file mode 100644 index 000000000..e63a505f1 --- /dev/null +++ b/executor/programs/asm/test_keccak256.s @@ -0,0 +1,128 @@ + .attribute 5, "rv64i2p1_m2p0_zmmul1p0" +.Lfunc_end0: + .globl main +main: + # Keccak-256 of a buffer (ecall -5): x10 = input, x11 = length, x12 = digest. + # + # A 1024-byte input buffer at sp + 512 holds byte[i] = (13*i + 5) mod 256, + # written a byte at a time. Eleven digests go to the table at sp (32 bytes + # each) and are committed at the end; the host test recomputes each one. + # + # call | input offset | length | case + # -----+--------------+--------+--------------------------------------- + # 0 | 0 | 0 | empty input: padding alone + # 1 | 1 | 1 | one byte, odd address + # 2 | 3 | 135 | len % 136 == 135: 0x01 and 0x80 meet + # 3 | 0 | 136 | len % 136 == 0: a block of padding alone + # 4 | 5 | 137 | two blocks, one tail byte + # 5 | 2 | 271 | two blocks, 135-byte tail + # 6 | 0 | 272 | three blocks, empty tail + # 7 | 7 | 600 | five blocks, unaligned throughout + # 8 | 8 | 20 | an address-sized input, aligned + # 9 | 6 | 32 | a slot-sized input + # 10 | 4 | 140 | digest written over its own input (sp+512+4+10) + # 11 | 9 | 136 | one full block whose last byte is below 0x80 + addi sp, sp, -1536 + + # Fill the input buffer. + addi t0, sp, 512 + li t1, 0 + li t2, 1024 + li t3, 5 +.Lfill: + sb t3, 0(t0) + addi t3, t3, 13 + andi t3, t3, 255 + addi t0, t0, 1 + addi t1, t1, 1 + bne t1, t2, .Lfill + + addi s0, sp, 512 # input base + mv s1, sp # digest table + li a7, -5 + + addi a0, s0, 0 + li a1, 0 + addi a2, s1, 0 + ecall + + addi a0, s0, 1 + li a1, 1 + addi a2, s1, 32 + ecall + + addi a0, s0, 3 + li a1, 135 + addi a2, s1, 64 + ecall + + addi a0, s0, 0 + li a1, 136 + addi a2, s1, 96 + ecall + + addi a0, s0, 5 + li a1, 137 + addi a2, s1, 128 + ecall + + addi a0, s0, 2 + li a1, 271 + addi a2, s1, 160 + ecall + + addi a0, s0, 0 + li a1, 272 + addi a2, s1, 192 + ecall + + addi a0, s0, 7 + li a1, 600 + addi a2, s1, 224 + ecall + + addi a0, s0, 8 + li a1, 20 + addi a2, s1, 256 + ecall + + addi a0, s0, 6 + li a1, 32 + addi a2, s1, 288 + ecall + + addi a0, s0, 9 + li a1, 136 + addi a2, s1, 352 + ecall + + # The digest overwrites bytes 10..42 of its own 140-byte input. + addi a0, s0, 4 + li a1, 140 + addi a2, s0, 14 + ecall + + # Copy that digest into the table (byte by byte), then commit the table. + addi t0, s0, 14 + addi t1, s1, 320 + li t2, 32 +.Lcopy: + lbu t3, 0(t0) + sb t3, 0(t1) + addi t0, t0, 1 + addi t1, t1, 1 + addi t2, t2, -1 + bnez t2, .Lcopy + + li a0, 1 + mv a1, s1 + li a2, 384 + li a7, 64 + ecall + + addi sp, sp, 1536 + li a0, 0 + li a7, 93 + ecall +.Lfunc_end1: + .size main, .Lfunc_end1-main diff --git a/executor/programs/asm/test_private_input_spin.s b/executor/programs/asm/test_private_input_spin.s new file mode 100644 index 000000000..1282a2264 --- /dev/null +++ b/executor/programs/asm/test_private_input_spin.s @@ -0,0 +1,32 @@ + .attribute 5, "rv64i2p1" + .globl main +main: + # Spins a loop whose count is the private input's first word, then + # commits 8 bytes of the input: a guest whose cycle count, and so its epoch + # count, a test chooses (13 cycles with no spins, 2 more per spin). + # + # Layout: [len:u32 LE] at 0xFF000000, then the input bytes. The spin count is + # the input's first u32 (0xFF000004); the committed bytes are the next eight + # (0xFF000008). The commit reads the private-input page in the last epoch, as + # the reads at the start did in the first, so the page crosses every epoch + # boundary the run has. + + li t0, 0xFF000000 # t0 = 0xFF000000 (private input base) + lw t3, 0(t0) # the length (touches the private-input page) + lw t4, 4(t0) # the spin count + beqz t4, done +spin: + addi t4, t4, -1 + bnez t4, spin +done: + # Commit 8 bytes from 0xFF000008 + addi a1, t0, 8 # buf_addr = 0xFF000008 + li a0, 1 # fd = 1 + li a2, 8 # count = 8 + li a7, 64 # syscall = Commit + ecall + + # Halt + li a0, 0 # exit_code = 0 + li a7, 93 # syscall = Halt + ecall diff --git a/executor/programs/rust/dma_memcpy_cases/Cargo.lock b/executor/programs/rust/dma_memcpy_cases/Cargo.lock index 5f1da6b2b..cdca57903 100644 --- a/executor/programs/rust/dma_memcpy_cases/Cargo.lock +++ b/executor/programs/rust/dma_memcpy_cases/Cargo.lock @@ -8,12 +8,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "const-default" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" - [[package]] name = "critical-section" version = "1.2.0" @@ -27,18 +21,6 @@ dependencies = [ "lambda-vm-syscalls", ] -[[package]] -name = "embedded-alloc" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" -dependencies = [ - "const-default", - "critical-section", - "linked_list_allocator", - "rlsf", -] - [[package]] name = "embedded-hal" version = "1.0.0" @@ -72,7 +54,6 @@ dependencies = [ name = "lambda-vm-syscalls" version = "0.1.0" dependencies = [ - "embedded-alloc", "getrandom 0.2.17", "getrandom 0.3.4", "lazy_static", @@ -93,12 +74,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linked_list_allocator" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" - [[package]] name = "paste" version = "1.0.15" @@ -197,24 +172,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" -[[package]] -name = "rlsf" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07393724337be2ee43a9d86164df4505746874a3fa65913374bc6d6a92314362" -dependencies = [ - "cfg-if", - "const-default", - "libc", - "rustversion", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - [[package]] name = "syn" version = "2.0.119" diff --git a/executor/programs/rust/dma_memcpy_implicit/Cargo.lock b/executor/programs/rust/dma_memcpy_implicit/Cargo.lock index 3b4049770..82c9c4078 100644 --- a/executor/programs/rust/dma_memcpy_implicit/Cargo.lock +++ b/executor/programs/rust/dma_memcpy_implicit/Cargo.lock @@ -8,12 +8,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "const-default" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" - [[package]] name = "critical-section" version = "1.2.0" @@ -27,18 +21,6 @@ dependencies = [ "lambda-vm-syscalls", ] -[[package]] -name = "embedded-alloc" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" -dependencies = [ - "const-default", - "critical-section", - "linked_list_allocator", - "rlsf", -] - [[package]] name = "embedded-hal" version = "1.0.0" @@ -72,7 +54,6 @@ dependencies = [ name = "lambda-vm-syscalls" version = "0.1.0" dependencies = [ - "embedded-alloc", "getrandom 0.2.17", "getrandom 0.3.4", "lazy_static", @@ -93,12 +74,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linked_list_allocator" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" - [[package]] name = "paste" version = "1.0.15" @@ -197,24 +172,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" -[[package]] -name = "rlsf" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07393724337be2ee43a9d86164df4505746874a3fa65913374bc6d6a92314362" -dependencies = [ - "cfg-if", - "const-default", - "libc", - "rustversion", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - [[package]] name = "syn" version = "2.0.119" diff --git a/executor/programs/rust/dma_memcpy_min/Cargo.lock b/executor/programs/rust/dma_memcpy_min/Cargo.lock index 06556e1d2..cccf37412 100644 --- a/executor/programs/rust/dma_memcpy_min/Cargo.lock +++ b/executor/programs/rust/dma_memcpy_min/Cargo.lock @@ -8,12 +8,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "const-default" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" - [[package]] name = "critical-section" version = "1.2.0" @@ -27,18 +21,6 @@ dependencies = [ "lambda-vm-syscalls", ] -[[package]] -name = "embedded-alloc" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" -dependencies = [ - "const-default", - "critical-section", - "linked_list_allocator", - "rlsf", -] - [[package]] name = "embedded-hal" version = "1.0.0" @@ -72,7 +54,6 @@ dependencies = [ name = "lambda-vm-syscalls" version = "0.1.0" dependencies = [ - "embedded-alloc", "getrandom 0.2.17", "getrandom 0.3.4", "lazy_static", @@ -93,12 +74,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linked_list_allocator" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" - [[package]] name = "paste" version = "1.0.15" @@ -197,24 +172,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" -[[package]] -name = "rlsf" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07393724337be2ee43a9d86164df4505746874a3fa65913374bc6d6a92314362" -dependencies = [ - "cfg-if", - "const-default", - "libc", - "rustversion", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - [[package]] name = "syn" version = "2.0.119" diff --git a/executor/programs/rust/ethrex/.cargo/config.toml b/executor/programs/rust/ethrex/.cargo/config.toml index 8ef8239bb..f7a0ef71a 100644 --- a/executor/programs/rust/ethrex/.cargo/config.toml +++ b/executor/programs/rust/ethrex/.cargo/config.toml @@ -1,7 +1,15 @@ [target.riscv64im-lambda-vm-elf] rustflags = [ "--cfg", "getrandom_backend=\"custom\"", - "-C", "passes=lower-atomic" + "-C", "passes=lower-atomic", + # compiler_builtins turns its `mem-unaligned` setting on for x86, aarch64 and bpf only, + # so on RISC-V its `memmove` reassembles every misaligned word with shifts; this VM + # executes an unaligned doubleword access as one instruction. Only compiler_builtins + # reads the name, in its own build script, with no stability guarantee. With the copy + # accelerator's stubs as `memcpy`, `memmove` and `memset`, and the runtime's own + # `memcmp`, it reaches only the library `memmove` of a build with syscalls' + # `fast-memcpy` feature. + "--cfg", "feature=\"mem-unaligned\"" ] [env] diff --git a/executor/programs/rust/ethrex/Cargo.lock b/executor/programs/rust/ethrex/Cargo.lock index 63705e473..89d9bd73d 100644 --- a/executor/programs/rust/ethrex/Cargo.lock +++ b/executor/programs/rust/ethrex/Cargo.lock @@ -669,7 +669,7 @@ dependencies = [ [[package]] name = "ethrex-common" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "bytes", "crc32fast", @@ -701,7 +701,7 @@ dependencies = [ [[package]] name = "ethrex-crypto" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "ark-bn254", "ark-ec", @@ -721,7 +721,7 @@ dependencies = [ [[package]] name = "ethrex-guest-program" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "bytes", "ethereum-types", @@ -740,7 +740,7 @@ dependencies = [ [[package]] name = "ethrex-l2-common" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "bytes", "ethereum-types", @@ -758,7 +758,7 @@ dependencies = [ [[package]] name = "ethrex-levm" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "bytes", "derive_more", @@ -776,7 +776,7 @@ dependencies = [ [[package]] name = "ethrex-rlp" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "bytes", "ethereum-types", @@ -786,7 +786,7 @@ dependencies = [ [[package]] name = "ethrex-trie" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "anyhow", "bytes", @@ -806,7 +806,7 @@ dependencies = [ [[package]] name = "ethrex-vm" version = "23.0.0" -source = "git+https://github.com/lambdaclass/ethrex.git?rev=797df5540c7d35cafd69b6971a74b2a49c67d1dd#797df5540c7d35cafd69b6971a74b2a49c67d1dd" +source = "git+https://github.com/yetanotherco/ethrex.git?rev=03c0fbd6b4266039c002fb08c00a861aa182cd19#03c0fbd6b4266039c002fb08c00a861aa182cd19" dependencies = [ "bytes", "derive_more", diff --git a/executor/programs/rust/ethrex/Cargo.toml b/executor/programs/rust/ethrex/Cargo.toml index b93265450..ad1db5c78 100644 --- a/executor/programs/rust/ethrex/Cargo.toml +++ b/executor/programs/rust/ethrex/Cargo.toml @@ -1,11 +1,13 @@ [workspace] -# Thin LTO measurably lowers executed guest cycles (~2.3% on the committed -# ethrex_bench fixtures) at a small ELF-size cost, which is free in this VM -# (execution is priced per executed instruction, not by ELF size). Cargo's -# release default is lto = false / codegen-units = 16. +# Whole-program optimisation: fat LTO over one codegen unit. On mainnet block +# 25475471 the guest executes 5.5 % fewer instructions than with thin LTO and 16 +# units (one unit alone gives 4.8 %), and the ELF is smaller; the guest build +# takes about three times as long. Execution is priced per executed instruction, +# not by ELF size. Cargo's release default is lto = false / codegen-units = 16. [profile.release] -lto = "thin" +lto = "fat" +codegen-units = 1 [package] name = "ethrex" @@ -13,6 +15,8 @@ version = "0.1.0" edition = "2024" [dependencies] +# The runtime's `memcpy`, `memmove` and `memset` are the copy accelerator's stubs +# (syscalls/src/entrypoint.rs). The `fast-memcpy` feature would keep them in software. lambda-vm-syscalls = { path = "../../../../syscalls" } # LambdaVM crypto provider (keccak + ECSM-accelerated ecrecover), defined in the # lambda_vm repo and injected in src/main.rs — so crypto changes stay in our repo @@ -52,3 +56,54 @@ ethrex-guest-program = { git = "https://github.com/lambdaclass/ethrex.git", rev # Exact pin: must match the fixture writer (tooling/ethrex-fixtures) and the # executor test reader so the rkyv ProgramInput layout stays consistent. rkyv = { version = "=0.8.16", features = ["std", "unaligned"] } + +# Every ethrex crate of this guest is built from yetanotherco/ethrex, branch +# lambda-vm/guest-opt-8: the pinned rev above plus guest-speed commits, none of +# which changes a result or ProgramInput, so the fixture writer and the native test +# reference stay on the pin (each commit carries the tests that compare it with the +# code it replaces). In order: +# - jump destinations as a bitmap, and the traced jump out of line; ethrex main's +# #7119 (MLOAD / MSTORE and the memory accessors without repeated checks); +# - U256 big-endian conversions with a 13-instruction byte swap (no Zbb), and the +# transactions, receipts and withdrawals roots built from pairs in key order; +# - from the round-12 guest work: RLP integers and lengths from their significant +# bytes; grouped nibble packing; witness nodes decoded in place, resident trie +# nodes walked by reference, branch preimages in one pass; the EVM stack +# allocated zeroed with single bounds checks; DIV / MOD specialised by divisor +# width; the jump-destination analysis made progressive and shared by Code +# clones (it replaces the eager bitmap), with a lock-free check inside the +# scanned prefix; memory expansion skipped when covered, and zero limbs of +# memory words not byte-swapped; +# - a branch's hash children decoded straight into the node; committed EVM +# checkpoints merged into the larger set; the RLP encoder's payload buffer +# started at 256 bytes; embedded witness nodes decoded without the shared-pointer +# pool; +# - precompile checks rejected by the address prefix; calldata zero bytes counted +# a word at a time; the witness state's lookup maps hashed instead of ordered; +# warm addresses and storage slots kept in one journaled set for the whole +# transaction; +# - LOG topics converted a word at a time; a cached account found with one +# probe; call frames and checkpoint substates moved instead of swapped; +# - the witness state leaked at the end of a zkVM run instead of dropped; no +# dirty-path bookkeeping in tries over an in-memory backend; receipt blooms ORed +# a word at a time; +# - a failed opcode's error kept in the VM (handlers take one argument); the +# program counter stepped as a pointer into the bytecode. +# On mainnet block 25475471 the guest executes 203.12 M instructions without the +# patch's later commits and 130.58 M with all of them and this repo's guest-side +# changes. Ethrex main has carried an equivalent bitmap since 3ca8bc23 (#7095) +# and #7119 since 007882a8; it has none of the rest. +# +# All eight crates are patched together: patching only the two that changed would +# leave two copies of ethrex-crypto in the graph. scripts/set_ethrex_rev.sh checks +# that the fork's base below is the pinned rev and refuses to move the pin past it. +# ethrex-fork-base: 797df5540c7d35cafd69b6971a74b2a49c67d1dd +[patch."https://github.com/lambdaclass/ethrex.git"] +ethrex-common = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } +ethrex-crypto = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } +ethrex-guest-program = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } +ethrex-l2-common = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } +ethrex-levm = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } +ethrex-rlp = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } +ethrex-trie = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } +ethrex-vm = { git = "https://github.com/yetanotherco/ethrex.git", rev = "03c0fbd6b4266039c002fb08c00a861aa182cd19" } diff --git a/executor/programs/rust/ethrex/src/input.rs b/executor/programs/rust/ethrex/src/input.rs new file mode 100644 index 000000000..ffefe74bf --- /dev/null +++ b/executor/programs/rust/ethrex/src/input.rs @@ -0,0 +1,31 @@ +use ethrex_guest_program::l1::ProgramInput; +use rkyv::rancor::Error; + +use crate::shared_table::OffsetSharedValidator; + +// The same checked rkyv access as from_bytes. Validation tracks every shared +// pointer (one per trie node), in a table keyed by offset into the input with the +// same answers as rkyv's own (`shared_table`). The table is sized at a bounded +// estimate for larger witnesses so it rarely grows; an underestimate still grows. +// The deserializer's pool is not reserved: the trie decodes its nodes, the input's +// only shared pointers, without it. +pub fn decode_input(bytes: &[u8]) -> Result { + use rkyv::de::Pool; + use rkyv::validation::{Validator, archive::ArchiveValidator}; + let capacity = if bytes.len() >= 64 * 1024 { + (bytes.len() / 512).min(65536) + } else { + 0 + }; + let archived = { + let mut validator = Validator::new( + ArchiveValidator::new(bytes), + OffsetSharedValidator::new(bytes, capacity), + ); + rkyv::api::access_with_context::, _, Error>( + bytes, + &mut validator, + )? + }; + rkyv::api::deserialize_using::(archived, &mut Pool::new()) +} diff --git a/executor/programs/rust/ethrex/src/main.rs b/executor/programs/rust/ethrex/src/main.rs index eae31effd..e88661f37 100644 --- a/executor/programs/rust/ethrex/src/main.rs +++ b/executor/programs/rust/ethrex/src/main.rs @@ -1,9 +1,16 @@ +#[cfg(target_arch = "riscv64")] use std::sync::Arc; -use ethrex_guest_program::l1::{ProgramInput, execution_program}; +#[cfg(target_arch = "riscv64")] +use ethrex_guest_program::l1::execution_program; +#[cfg(target_arch = "riscv64")] use lambda_vm_ethrex_crypto::LambdaVmEcsmCrypto; -use rkyv::rancor::Error; +#[cfg_attr(not(target_arch = "riscv64"), allow(dead_code))] +mod input; +#[cfg_attr(not(target_arch = "riscv64"), allow(dead_code))] +mod shared_table; +#[cfg(target_arch = "riscv64")] pub fn main() { // Zero-copy private input: borrow the memory-mapped input region in place // (the host pre-loads it before execution) so rkyv deserializes straight @@ -11,7 +18,7 @@ pub fn main() { // a full extra copy and one large allocation (~50k cycles on a 20-tx // block). let input = lambda_vm_syscalls::syscalls::get_private_input_slice(); - let input = rkyv::from_bytes::(input).unwrap(); + let input = input::decode_input(input).unwrap(); // LambdaVM crypto provider, defined in the lambda_vm repo and injected here // (so crypto changes don't require an ethrex PR — see `crypto/ethrex-crypto`). // It accelerates trait-routed `keccak256` (via the keccak_permute precompile) @@ -23,3 +30,7 @@ pub fn main() { let output = execution_program(input, crypto).unwrap(); lambda_vm_syscalls::syscalls::commit(&output.encode()); } + +// On the host this crate builds only for the unit tests of its input decoding. +#[cfg(not(target_arch = "riscv64"))] +pub fn main() {} diff --git a/executor/programs/rust/ethrex/src/shared_table.rs b/executor/programs/rust/ethrex/src/shared_table.rs new file mode 100644 index 000000000..4a868e666 --- /dev/null +++ b/executor/programs/rust/ethrex/src/shared_table.rs @@ -0,0 +1,374 @@ +//! A shared-pointer validator for rkyv's checked access, keyed by offset into the input. +//! +//! rkyv validates every `ArchivedRc` through a [`SharedContext`]: the first visit of an +//! address starts it (and checks the pointee), a later visit finds it finished, a visit +//! while it is still being checked is a cycle, and the same address claimed as two +//! types is an error. rkyv's [`SharedValidator`] keeps that state in a general hash map +//! keyed by address with the full `TypeId` beside it; for the guest's input, one entry +//! per trie node, those map operations were most of the cost of validating a node. +//! +//! [`OffsetSharedValidator`] keeps the same state, with the same answers, in an +//! open-addressing table of one word a slot: the address's offset into the input, a +//! small index into the list of types seen, and the finished bit. Addresses outside the +//! input (which rkyv's bounds check then rejects) go to an inner [`SharedValidator`], +//! so every address has exactly one place where its state can live. + +use core::any::TypeId; +use core::fmt; + +use rkyv::rancor::Source; +use rkyv::validation::SharedContext; +use rkyv::validation::shared::{SharedValidator, ValidationState}; + +/// See the module documentation. +pub struct OffsetSharedValidator { + base: usize, + len: usize, + /// 0 = empty; otherwise `(offset + 1) << 32 | type_index << 1 | finished`. + slots: Vec, + /// `slots.len() - 1`; `slots.len()` is a power of two. + mask: usize, + /// `64 - log2(slots.len())`. + shift: u32, + used: usize, + /// Every type seen, by index; `last` caches the most recent lookup. + types: Vec, + last: Option<(TypeId, u64)>, + outside: SharedValidator, +} + +impl OffsetSharedValidator { + /// A validator for the input `bytes`, its table sized for about `capacity` shared + /// pointers (it grows past that). + pub fn new(bytes: &[u8], capacity: usize) -> Self { + // Offsets are stored in 32 bits: an input of 4 GiB or more keeps every address + // in the inner validator. + let len = if bytes.len() < u32::MAX as usize { + bytes.len() + } else { + 0 + }; + let slots = (capacity.max(8) * 2).next_power_of_two(); + Self { + base: bytes.as_ptr() as usize, + len, + slots: vec![0; slots], + mask: slots - 1, + shift: 64 - slots.trailing_zeros(), + used: 0, + types: Vec::new(), + last: None, + outside: SharedValidator::new(), + } + } + + /// The offset of `address` into the input, if it is inside it. + #[inline(always)] + fn offset(&self, address: usize) -> Option { + let offset = address.wrapping_sub(self.base); + (offset < self.len).then_some(offset as u64) + } + + /// The index of `type_id`, adding it if new. + #[inline(always)] + fn type_index(&mut self, type_id: TypeId) -> u64 { + if let Some((last, index)) = self.last + && last == type_id + { + return index; + } + let index = match self.types.iter().position(|t| *t == type_id) { + Some(i) => i as u64, + None => { + self.types.push(type_id); + (self.types.len() - 1) as u64 + } + }; + self.last = Some((type_id, index)); + index + } + + #[inline(always)] + fn home(&self, key: u64) -> usize { + // Fibonacci hashing of the offset; the table index is the product's top bits. + (key.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> self.shift) as usize + } + + /// The slot holding `key` (offset + 1), or the empty slot where it would go. + #[inline(always)] + fn find(&self, key: u64) -> usize { + let mut i = self.home(key); + loop { + let slot = self.slots[i]; + if slot == 0 || slot >> 32 == key { + return i; + } + i = (i + 1) & self.mask; + } + } + + #[cold] + fn grow(&mut self) { + let old = core::mem::take(&mut self.slots); + let slots = old.len() * 2; + self.slots = vec![0; slots]; + self.mask = slots - 1; + self.shift = 64 - slots.trailing_zeros(); + for slot in old { + if slot != 0 { + let i = self.find(slot >> 32); + self.slots[i] = slot; + } + } + } + + fn type_of(&self, slot: u64) -> TypeId { + self.types[((slot & 0xFFFF_FFFF) >> 1) as usize] + } +} + +impl SharedContext for OffsetSharedValidator { + fn start_shared(&mut self, address: usize, type_id: TypeId) -> Result { + let Some(offset) = self.offset(address) else { + return self.outside.start_shared(address, type_id); + }; + let key = offset + 1; + let index = self.type_index(type_id); + let i = self.find(key); + let slot = self.slots[i]; + if slot == 0 { + self.slots[i] = key << 32 | index << 1; + self.used += 1; + if self.used * 2 > self.slots.len() { + self.grow(); + } + return Ok(ValidationState::Started); + } + if (slot & 0xFFFF_FFFF) >> 1 != index { + return Err(E::new(TypeMismatch { + previous: self.type_of(slot), + current: type_id, + })); + } + if slot & 1 == 0 { + Ok(ValidationState::Pending) + } else { + Ok(ValidationState::Finished) + } + } + + fn finish_shared(&mut self, address: usize, type_id: TypeId) -> Result<(), E> { + let Some(offset) = self.offset(address) else { + return self.outside.finish_shared(address, type_id); + }; + let index = self.type_index(type_id); + let i = self.find(offset + 1); + let slot = self.slots[i]; + if slot == 0 { + return Err(E::new(NotStarted)); + } + if (slot & 0xFFFF_FFFF) >> 1 != index { + return Err(E::new(TypeMismatch { + previous: self.type_of(slot), + current: type_id, + })); + } + if slot & 1 == 1 { + return Err(E::new(AlreadyFinished)); + } + self.slots[i] = slot | 1; + Ok(()) + } +} + +// rkyv's errors for the same three conditions are private; these carry the same +// messages. + +#[derive(Debug)] +struct TypeMismatch { + previous: TypeId, + current: TypeId, +} + +impl fmt::Display for TypeMismatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "the same memory region has been claimed as two different types: {:?} and {:?}", + self.previous, self.current, + ) + } +} + +impl core::error::Error for TypeMismatch {} + +#[derive(Debug)] +struct NotStarted; + +impl fmt::Display for NotStarted { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "shared pointer was not started validation") + } +} + +impl core::error::Error for NotStarted {} + +#[derive(Debug)] +struct AlreadyFinished; + +impl fmt::Display for AlreadyFinished { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "shared pointer was already finished validation") + } +} + +impl core::error::Error for AlreadyFinished {} + +#[cfg(test)] +mod tests { + use super::*; + + type TestError = rkyv::rancor::Failure; + + /// What a validator answered: the state, or that it failed. + fn start>(c: &mut C, a: usize, t: TypeId) -> Option { + match c.start_shared(a, t) { + Ok(ValidationState::Started) => Some(0), + Ok(ValidationState::Pending) => Some(1), + Ok(ValidationState::Finished) => Some(2), + Err(_) => None, + } + } + + fn finish>(c: &mut C, a: usize, t: TypeId) -> Option<()> { + c.finish_shared(a, t).ok() + } + + /// Step by step, the same answers as rkyv's validator: random starts and finishes + /// over in-input and outside addresses, three types, a table small enough to grow + /// several times and to collide. + #[test] + fn answers_like_rkyvs_shared_validator() { + let types = [ + TypeId::of::(), + TypeId::of::(), + TypeId::of::<[u8; 3]>(), + ]; + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + state.wrapping_mul(0x2545_F491_4F6C_DD1D) + }; + for round in 0..200 { + let input = vec![0u8; 4096]; + let base = input.as_ptr() as usize; + let mut ours = OffsetSharedValidator::new(&input, (round % 4) as usize); + let mut theirs = SharedValidator::new(); + let span = [8u64, 64, 600, 4096][round % 4]; + for _ in 0..2000 { + let r = next(); + // Mostly inside the input (first and last bytes included), some just + // outside either end, some far away. + let address = match r % 16 { + 0 => base.wrapping_sub(1 + (r >> 8) as usize % 64), + 1 => base + 4096 + (r >> 8) as usize % 64, + 2 => (r >> 4) as usize, + 3 => base + 4095, + _ => base + ((r >> 8) % span) as usize % 4096, + }; + let t = types[((r >> 40) % if r % 7 == 0 { 3 } else { 1 }) as usize]; + if (r >> 50) % 2 == 0 { + assert_eq!(start(&mut ours, address, t), start(&mut theirs, address, t)); + } else { + assert_eq!( + finish(&mut ours, address, t), + finish(&mut theirs, address, t) + ); + } + } + } + } + + /// The four answers for one address, in order, and a type mismatch both before and + /// after the finish. + #[test] + fn one_address_through_its_states() { + let input = vec![0u8; 64]; + let a = input.as_ptr() as usize + 17; + let (t, u) = (TypeId::of::(), TypeId::of::()); + let mut v = OffsetSharedValidator::new(&input, 0); + assert_eq!(finish(&mut v, a, t), None, "finish before start"); + assert_eq!(start(&mut v, a, t), Some(0), "started"); + assert_eq!(start(&mut v, a, t), Some(1), "pending: a cycle"); + assert_eq!(start(&mut v, a, u), None, "other type while pending"); + assert_eq!(finish(&mut v, a, u), None, "finish as the other type"); + assert_eq!(finish(&mut v, a, t), Some(()), "finished"); + assert_eq!(finish(&mut v, a, t), None, "finished twice"); + assert_eq!(start(&mut v, a, t), Some(2), "finished: seen before"); + assert_eq!(start(&mut v, a, u), None, "other type once finished"); + assert_eq!( + start(&mut v, a + 1, t), + Some(0), + "the next byte is another address" + ); + } + + /// Checked access of the committed ethrex inputs: both validators accept them, and + /// with bytes flipped they accept and reject the same ones. + #[test] + fn checked_access_of_ethrex_inputs_agrees() { + use ethrex_guest_program::l1::ProgramInput; + use rkyv::rancor::Error; + use rkyv::validation::{Validator, archive::ArchiveValidator}; + + fn ours(bytes: &[u8]) -> bool { + let mut v = Validator::new( + ArchiveValidator::new(bytes), + OffsetSharedValidator::new(bytes, bytes.len() / 512), + ); + rkyv::api::access_with_context::, _, Error>(bytes, &mut v) + .is_ok() + } + fn theirs(bytes: &[u8]) -> bool { + let mut v = Validator::new(ArchiveValidator::new(bytes), SharedValidator::new()); + rkyv::api::access_with_context::, _, Error>(bytes, &mut v) + .is_ok() + } + + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../tests"); + let mut state = 0x2545_F491_4F6C_DD1Du64; + let mut flips = 0; + let mut rejected = 0; + for name in [ + "ethrex_10_transfers.bin", + "ethrex_bench_4.bin", + "ethrex_empty_block.bin", + "ethrex_simple_tx.bin", + ] { + let bytes = std::fs::read(format!("{dir}/{name}")).unwrap(); + // rkyv reads the input from an aligned buffer in the guest; copy it into one. + let mut aligned = rkyv::util::AlignedVec::<16>::new(); + aligned.extend_from_slice(&bytes); + assert!(ours(&aligned) && theirs(&aligned), "{name} accepted"); + for _ in 0..300 { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + let r = state.wrapping_mul(0x2545_F491_4F6C_DD1D); + let mut bad = aligned.clone(); + let i = (r >> 8) as usize % bad.len(); + bad[i] ^= 1 << (r % 8); + let (a, b) = (ours(&bad), theirs(&bad)); + assert_eq!(a, b, "{name}: bit {} of byte {i}", r % 8); + flips += 1; + rejected += usize::from(!a); + } + } + assert!( + flips == 1200 && rejected > 0, + "{rejected} of {flips} flips rejected" + ); + } +} diff --git a/executor/programs/rust/hint_min/Cargo.lock b/executor/programs/rust/hint_min/Cargo.lock index cc02eff98..afe265341 100644 --- a/executor/programs/rust/hint_min/Cargo.lock +++ b/executor/programs/rust/hint_min/Cargo.lock @@ -2,42 +2,18 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "const-default" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" - [[package]] name = "critical-section" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" -[[package]] -name = "embedded-alloc" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" -dependencies = [ - "const-default", - "critical-section", - "linked_list_allocator", - "rlsf", -] - [[package]] name = "embedded-hal" version = "1.0.0" @@ -78,7 +54,6 @@ dependencies = [ name = "lambda-vm-syscalls" version = "0.1.0" dependencies = [ - "embedded-alloc", "getrandom 0.2.17", "getrandom 0.3.4", "lazy_static", @@ -99,12 +74,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linked_list_allocator" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" - [[package]] name = "paste" version = "1.0.15" @@ -194,7 +163,7 @@ checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -203,49 +172,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" -[[package]] -name = "rlsf" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1646a59a9734b8b7a0ac51689388a60fe1625d4b956348e9de07591a1478457a" -dependencies = [ - "cfg-if", - "const-default", - "libc", - "rustversion", - "svgbobdoc", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "svgbobdoc" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c04b93fc15d79b39c63218f15e3fdffaa4c227830686e3b7c5f41244eb3e50" -dependencies = [ - "base64", - "proc-macro2", - "quote", - "syn 1.0.109", - "unicode-width", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.119" @@ -274,7 +200,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -283,12 +209,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -327,5 +247,5 @@ checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] diff --git a/executor/programs/rust/hint_multi/Cargo.lock b/executor/programs/rust/hint_multi/Cargo.lock index 9803c875a..3adb3a1bc 100644 --- a/executor/programs/rust/hint_multi/Cargo.lock +++ b/executor/programs/rust/hint_multi/Cargo.lock @@ -2,42 +2,18 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "const-default" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" - [[package]] name = "critical-section" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" -[[package]] -name = "embedded-alloc" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" -dependencies = [ - "const-default", - "critical-section", - "linked_list_allocator", - "rlsf", -] - [[package]] name = "embedded-hal" version = "1.0.0" @@ -78,7 +54,6 @@ dependencies = [ name = "lambda-vm-syscalls" version = "0.1.0" dependencies = [ - "embedded-alloc", "getrandom 0.2.17", "getrandom 0.3.4", "lazy_static", @@ -99,12 +74,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linked_list_allocator" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" - [[package]] name = "paste" version = "1.0.15" @@ -194,7 +163,7 @@ checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -203,49 +172,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" -[[package]] -name = "rlsf" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1646a59a9734b8b7a0ac51689388a60fe1625d4b956348e9de07591a1478457a" -dependencies = [ - "cfg-if", - "const-default", - "libc", - "rustversion", - "svgbobdoc", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "svgbobdoc" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c04b93fc15d79b39c63218f15e3fdffaa4c227830686e3b7c5f41244eb3e50" -dependencies = [ - "base64", - "proc-macro2", - "quote", - "syn 1.0.109", - "unicode-width", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.119" @@ -274,7 +200,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -283,12 +209,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -327,5 +247,5 @@ checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] diff --git a/executor/programs/rust/mem_sweep/.cargo/config.toml b/executor/programs/rust/mem_sweep/.cargo/config.toml new file mode 100644 index 000000000..450842a7e --- /dev/null +++ b/executor/programs/rust/mem_sweep/.cargo/config.toml @@ -0,0 +1,13 @@ +# The flags the ethrex guest is built with, so the library routines this sweep runs +# are built as that guest's are. +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "--cfg", "getrandom_backend=\"custom\"", + "-C", "passes=lower-atomic", + # compiler_builtins turns its `mem-unaligned` setting on for x86, aarch64 and bpf only, + # so on RISC-V its `memmove` reassembles every misaligned word with shifts; this VM + # executes an unaligned doubleword access as one instruction. With the copy + # accelerator's stubs linked, `memmove` is not the library's, so the setting reaches + # only the library routines this guest still takes (`memcmp`). + "--cfg", "feature=\"mem-unaligned\"" +] diff --git a/executor/programs/rust/mem_sweep/Cargo.lock b/executor/programs/rust/mem_sweep/Cargo.lock new file mode 100644 index 000000000..5e2f645aa --- /dev/null +++ b/executor/programs/rust/mem_sweep/Cargo.lock @@ -0,0 +1,251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "lambda-vm-syscalls" +version = "0.1.0" +dependencies = [ + "getrandom 0.2.17", + "getrandom 0.3.4", + "lazy_static", + "rand", + "riscv", + "thiserror", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "mem_sweep" +version = "0.1.0" +dependencies = [ + "lambda-vm-syscalls", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "riscv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05cfa3f7b30c84536a9025150d44d26b8e1cc20ddf436448d74cd9591eefb25" +dependencies = [ + "critical-section", + "embedded-hal", + "paste", + "riscv-macros", + "riscv-pac", +] + +[[package]] +name = "riscv-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "riscv-pac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/executor/programs/rust/mem_sweep/Cargo.toml b/executor/programs/rust/mem_sweep/Cargo.toml new file mode 100644 index 000000000..624779703 --- /dev/null +++ b/executor/programs/rust/mem_sweep/Cargo.toml @@ -0,0 +1,15 @@ +[workspace] + +# The ethrex guest's release profile, so the routines swept here are compiled as that +# guest's are. +[profile.release] +lto = "fat" +codegen-units = 1 + +[package] +name = "mem_sweep" +version = "0.1.0" +edition = "2024" + +[dependencies] +lambda-vm-syscalls = { path = "../../../../syscalls" } diff --git a/executor/programs/rust/mem_sweep/src/main.rs b/executor/programs/rust/mem_sweep/src/main.rs new file mode 100644 index 000000000..3cd8e4395 --- /dev/null +++ b/executor/programs/rust/mem_sweep/src/main.rs @@ -0,0 +1,417 @@ +//! In-VM sweep of the guest's memory routines against values computed from the index. +//! +//! The routine under test is whatever the linker resolved `memcpy` / `memmove` / +//! `memset` / `memcmp` to in this guest. Nothing here copies bytes to build an +//! expectation: the source holds `pat(index)`, the destination is poisoned with a +//! byte `pat` never produces, and every check recomputes `pat`. So a routine cannot +//! be used to verify itself. +//! +//! Public output: `OK` + seven little-endian u32 case counts (memcpy mod 8, memcpy +//! mod 16, memcpy large, memcpy same/adjacent, memmove, memset, memcmp calls), or `FAIL` + +//! the stage byte, length (u32), source and destination alignment, and the index +//! (u32) of the first wrong byte relative to the destination start (negative = in +//! the guard before it, encoded two's complement). + +use core::hint::black_box; +use lambda_vm_syscalls as syscalls; + +unsafe extern "C" { + fn memcpy(dst: *mut u8, src: *const u8, n: usize) -> *mut u8; + fn memmove(dst: *mut u8, src: *const u8, n: usize) -> *mut u8; + fn memset(dst: *mut u8, c: i32, n: usize) -> *mut u8; + fn memcmp(a: *const u8, b: *const u8, n: usize) -> i32; +} + +/// Bytes kept untouched on each side of a destination. +const GUARD: usize = 24; +/// A byte `pat` never returns (it only returns odd bytes). +const POISON: u8 = 0xA4; +const LARGE: usize = 1 << 20; +/// Lengths of the exhaustive sweeps. +const MAX_LEN: usize = 1100; +const MAX_LEN_MOD16: usize = 160; +const LARGE_LENS: [usize; 6] = [4095, 4096, 4097, 8191, 65536, 65543]; +const LARGE_ALIGN: [usize; 4] = [0, 1, 4, 7]; +/// The two largest lengths, and the lengths between `MAX_LEN` and a page, run at four +/// (source, destination) offsets only. +const HUGE_LENS: [usize; 2] = [262149, LARGE]; +const HUGE_ALIGN: [(usize, usize); 4] = [(0, 0), (1, 0), (0, 1), (7, 4)]; +/// Lengths of the memcmp sweep. +const MEMCMP_MAX_LEN: usize = 300; + +/// Where the memcmp sweep puts a differing byte in an `n`-byte operand: everywhere up +/// to 80 bytes (every remainder mod 8 of the length and of the position, the +/// overlapping last doubleword included), and near the start, the middle and the end +/// above that. `memcmp_model.rs` in the runtime sweeps every position on the host. +fn memcmp_positions(n: usize) -> Vec { + if n <= 80 { + return (0..n).collect(); + } + vec![0, 1, 7, 8, 9, n / 2, n - 9, n - 8, n - 1] +} + +/// Source pattern: an odd pseudo-random byte per index, with no short period. +#[inline(always)] +fn pat(i: usize) -> u8 { + ((i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 | 1 +} + +struct Fail { + stage: u8, + len: usize, + sa: usize, + da: usize, + at: isize, +} + +fn fail(stage: u8, len: usize, sa: usize, da: usize, at: isize) -> Fail { + Fail { + stage, + len, + sa, + da, + at, + } +} + +/// Poison `[d - GUARD, d + n + GUARD)` byte by byte through volatile writes, so the +/// compiler cannot turn the loop into a call to a routine under test. +#[inline(never)] +fn poison(base: *mut u8, d: usize, n: usize) { + for i in d - GUARD..d + n + GUARD { + unsafe { core::ptr::write_volatile(base.add(i), POISON) }; + } +} + +/// Check the guards around `[d, d + n)` and that `dst[d + i] == pat(s + i)`. +#[inline(never)] +fn check_copy(base: *const u8, d: usize, n: usize, s: usize) -> Result<(), isize> { + for i in 0..GUARD { + if unsafe { core::ptr::read_volatile(base.add(d - GUARD + i)) } != POISON { + return Err(i as isize - GUARD as isize); + } + if unsafe { core::ptr::read_volatile(base.add(d + n + i)) } != POISON { + return Err((n + i) as isize); + } + } + for i in 0..n { + if unsafe { core::ptr::read_volatile(base.add(d + i)) } != pat(s + i) { + return Err(i as isize); + } + } + Ok(()) +} + +/// Check that `src[i] == pat(i)` over the whole source (the routines must not write it). +#[inline(never)] +fn check_source(src: &[u8]) -> Result<(), isize> { + for i in 0..src.len() { + if unsafe { core::ptr::read_volatile(src.as_ptr().add(i)) } != pat(i) { + return Err(i as isize); + } + } + Ok(()) +} + +/// One `memcpy` case: poison, copy `n` bytes from `src + s` to `dst + d`, check the +/// return value, the copied bytes and the guards. +#[inline(never)] +fn copy_case( + stage: u8, + src: &[u8], + dst: &mut [u8], + s: usize, + d: usize, + n: usize, +) -> Result<(), Fail> { + let base = dst.as_mut_ptr(); + poison(base, d, n); + let to = unsafe { base.add(d) }; + let ret = unsafe { memcpy(black_box(to), black_box(src.as_ptr().add(s)), black_box(n)) }; + if ret != to { + return Err(fail( + stage, + n, + (src.as_ptr() as usize + s) % 16, + (base as usize + d) % 16, + isize::MIN, + )); + } + check_copy(base, d, n, s).map_err(|at| { + fail( + stage, + n, + (src.as_ptr() as usize + s) % 16, + (base as usize + d) % 16, + at, + ) + }) +} + +fn run() -> Result<[u32; 7], Fail> { + let mut counts = [0u32; 7]; + let mut src = vec![0u8; LARGE + 64]; + for (i, b) in src.iter_mut().enumerate() { + *b = pat(i); + } + let mut dst = vec![POISON; LARGE + 2 * GUARD + 64]; + // Destinations start at an 8-aligned (then 16-aligned) address plus the wanted + // offset, whatever the allocator returned. + let d0 = GUARD + (16 - (dst.as_ptr() as usize + GUARD) % 16) % 16; + let s0 = (16 - src.as_ptr() as usize % 16) % 16; + + // 1. memcpy: every length up to MAX_LEN, every source and destination offset mod 8. + for n in 0..=MAX_LEN { + for sa in 0..8 { + for da in 0..8 { + copy_case(1, &src, &mut dst, s0 + sa, d0 + da, n)?; + counts[0] += 1; + } + } + } + // 2. memcpy: every offset mod 16 for the lengths around the routine's thresholds. + for n in 0..=MAX_LEN_MOD16 { + for sa in 0..16 { + for da in 0..16 { + copy_case(2, &src, &mut dst, s0 + sa, d0 + da, n)?; + counts[1] += 1; + } + } + } + // 3. memcpy: large lengths, and lengths up to a page. + for &n in &LARGE_LENS { + for &sa in &LARGE_ALIGN { + for &da in &LARGE_ALIGN { + copy_case(3, &src, &mut dst, s0 + sa, d0 + da, n)?; + counts[2] += 1; + } + } + } + for &n in &HUGE_LENS { + for &(sa, da) in &HUGE_ALIGN { + copy_case(3, &src, &mut dst, s0 + sa, d0 + da, n)?; + counts[2] += 1; + } + } + // Lengths between the exhaustive sweep and a page, at four offset pairs. Step 7 is + // prime to 32, so they cover every remainder of the 32-byte loop and the 8-byte steps. + for n in (MAX_LEN + 1..=4096).step_by(7) { + for &(sa, da) in &HUGE_ALIGN { + copy_case(3, &src, &mut dst, s0 + sa, d0 + da, n)?; + counts[2] += 1; + } + } + // 3b. the compiler's own copies: a slice copy of run-time length and a fixed + // 767-byte block (the size the block guest copies most). + for da in 0..8 { + let n = black_box(767usize); + poison(dst.as_mut_ptr(), d0 + da, n); + dst[d0 + da..d0 + da + n].copy_from_slice(&src[s0 + 1..s0 + 1 + n]); + check_copy(dst.as_ptr(), d0 + da, n, s0 + 1).map_err(|at| fail(3, n, 1, da, at))?; + poison(dst.as_mut_ptr(), d0 + da, 767); + unsafe { + let from = src.as_ptr().add(s0 + 3) as *const [u8; 767]; + let to = dst.as_mut_ptr().add(d0 + da) as *mut [u8; 767]; + core::ptr::copy_nonoverlapping(black_box(from), black_box(to), 1); + } + check_copy(dst.as_ptr(), d0 + da, 767, s0 + 3).map_err(|at| fail(3, 767, 3, da, at))?; + counts[2] += 2; + } + check_source(&src).map_err(|at| fail(3, 0, 0, 0, at))?; + + // 4. memcpy inside one buffer: source == destination (the one overlap the + // compiler may emit), and regions that touch without overlapping. + let mut buf = vec![0u8; 4096]; + let b0 = 64 + (16 - (buf.as_ptr() as usize + 64) % 16) % 16; + for n in 0..=300usize { + for a in 0..8 { + for which in 0..3 { + for (i, b) in buf.iter_mut().enumerate().take(b0 + a + 2 * n + 64) { + *b = pat(i); + } + let p = b0 + a; + // 0: same region; 1: destination right after the source; 2: right before. + let (s, d) = match which { + 0 => (p, p), + 1 => (p, p + n), + _ => (p + n, p), + }; + let base = buf.as_mut_ptr(); + let ret = unsafe { + memcpy( + black_box(base.add(d)), + black_box(base.add(s) as *const u8), + black_box(n), + ) + }; + if ret != unsafe { base.add(d) } { + return Err(fail(4, n, a, which, isize::MIN)); + } + for j in 0..b0 + a + 2 * n + 64 { + let want = if j >= d && j < d + n { + pat(s + (j - d)) + } else { + pat(j) + }; + if unsafe { core::ptr::read_volatile(base.add(j)) } != want { + return Err(fail(4, n, a, which, j as isize - d as isize)); + } + } + counts[3] += 1; + } + } + } + + // 5. memmove: overlapping forward and backward, adjacent, and disjoint regions. + const DELTAS: [isize; 17] = [ + -33, -32, -17, -16, -9, -8, -7, -1, 0, 1, 7, 8, 9, 16, 17, 32, 33, + ]; + for n in 0..=200usize { + for a in 0..8 { + for k in 0..DELTAS.len() + 2 { + let delta = match k { + k if k < DELTAS.len() => DELTAS[k], + k if k == DELTAS.len() => n as isize, + _ => -(n as isize), + }; + let s = b0 + 256 + a; + let d = (s as isize + delta) as usize; + let lo = s.min(d) - GUARD; + let hi = s.max(d) + n + GUARD; + for j in lo..hi { + unsafe { core::ptr::write_volatile(buf.as_mut_ptr().add(j), pat(j)) }; + } + let base = buf.as_mut_ptr(); + let ret = unsafe { + memmove( + black_box(base.add(d)), + black_box(base.add(s) as *const u8), + black_box(n), + ) + }; + if ret != unsafe { base.add(d) } { + return Err(fail(5, n, a, k, isize::MIN)); + } + for j in lo..hi { + let want = if j >= d && j < d + n { + pat(s + (j - d)) + } else { + pat(j) + }; + if unsafe { core::ptr::read_volatile(base.add(j)) } != want { + return Err(fail(5, n, a, k, j as isize - d as isize)); + } + } + counts[4] += 1; + } + } + } + + // 6. memset. + for n in 0..=300usize { + for a in 0..8 { + for &c in &[0x00u8, 0x3C] { + let d = b0 + a; + poison(buf.as_mut_ptr(), d, n); + let base = buf.as_mut_ptr(); + let ret = + unsafe { memset(black_box(base.add(d)), black_box(c as i32), black_box(n)) }; + if ret != unsafe { base.add(d) } { + return Err(fail(6, n, a, c as usize, isize::MIN)); + } + for j in d - GUARD..d + n + GUARD { + let want = if j >= d && j < d + n { c } else { POISON }; + if unsafe { core::ptr::read_volatile(base.add(j)) } != want { + return Err(fail(6, n, a, c as usize, j as isize - d as isize)); + } + } + counts[5] += 1; + } + } + } + // 7. memcmp: every length up to MEMCMP_MAX_LEN and every pair of offsets mod 8. + // Operand `a` is the source pattern; operand `b` is the same pattern in `other`, + // with POISON (a byte `pat` never returns) right before it and right after its + // `n` bytes, so a load outside the operands makes equal operands compare unequal. + // Then one byte of `b` is flipped at the positions of `memcmp_positions`, by 0x80 + // and by 0x01, alone and with a later difference of the opposite sense at the last + // byte, in both argument orders. The exact result is known: the difference of the + // first differing bytes as unsigned values, which compiler_builtins' and the + // runtime's `memcmp` both return. + let mut other = vec![POISON; MEMCMP_MAX_LEN + 32]; + for a in 0..8 { + for b in 0..8 { + for (i, x) in other.iter_mut().enumerate() { + *x = if i < 8 + b { + POISON + } else { + pat(s0 + a + i - 8 - b) + }; + } + // `memcmp(a, b, n) == want` and `memcmp(b, a, n) == -want`, with `b` taken + // from `other` as it is now + let agrees = |other: &[u8], n: usize, want: i32| { + let pa = unsafe { src.as_ptr().add(s0 + a) }; + let pb = unsafe { other.as_ptr().add(8 + b) }; + let cmp = |x: *const u8, y: *const u8| unsafe { + memcmp(black_box(x), black_box(y), black_box(n)) + }; + cmp(pa, pb) == want && cmp(pb, pa) == -want + }; + for n in 0..=MEMCMP_MAX_LEN { + for x in &mut other[8 + b + n..16 + b + n] { + *x = POISON; + } + if !agrees(&other, n, 0) { + return Err(fail(7, n, a, b, -1)); + } + counts[6] += 2; + for at in memcmp_positions(n) { + let keep = other[8 + b + at]; + let last = other[8 + b + n - 1]; + for (x, later) in [(0x80u8, false), (0x01, false), (0x01, true)] { + if later && at + 1 == n { + continue; + } + other[8 + b + at] = keep ^ x; + if later { + other[8 + b + n - 1] = last ^ 0x80; + } + let want = i32::from(keep) - i32::from(keep ^ x); + if !agrees(&other, n, want) { + return Err(fail(7, n, a, b, at as isize)); + } + counts[6] += 2; + other[8 + b + at] = keep; + other[8 + b + n - 1] = last; + } + } + for (i, x) in other[8 + b + n..16 + b + n].iter_mut().enumerate() { + *x = pat(s0 + a + n + i); + } + } + } + } + Ok(counts) +} + +pub fn main() { + let mut out = Vec::with_capacity(32); + match run() { + Ok(counts) => { + out.extend_from_slice(b"OK"); + for c in counts { + out.extend_from_slice(&c.to_le_bytes()); + } + } + Err(f) => { + out.extend_from_slice(b"FAIL"); + out.push(f.stage); + out.extend_from_slice(&(f.len as u32).to_le_bytes()); + out.push(f.sa as u8); + out.push(f.da as u8); + out.extend_from_slice(&(f.at as i32).to_le_bytes()); + } + } + syscalls::syscalls::commit(&out); +} diff --git a/executor/src/tests/blake3_tests.rs b/executor/src/tests/blake3_tests.rs new file mode 100644 index 000000000..a90695544 --- /dev/null +++ b/executor/src/tests/blake3_tests.rs @@ -0,0 +1,471 @@ +//! Tests for the BLAKE3 6-round compression and its accelerator syscall. +//! +//! Ground truth is the validated oracle (`tooling/blake3-second-source/blake3-oracle/`): +//! the pinned canonical 6-round vectors below were emitted by its harness and +//! checked against the official `blake3` crate at the official test-vector +//! parameters. The `t` values exercise the full 64-bit counter range, pinning +//! the `t_lo → v[12]` / `t_hi → v[13]` split order. + +use crate::vm::instruction::decoding::Instruction; +use crate::vm::instruction::execution::{ + BLAKE3_ABSORB_MAX_BLOCKS, BLAKE3_ABSORB_SYSCALL_NUMBER, BLAKE3_SYSCALL_NUMBER, ExecutionError, + blake3_compress_6round, +}; +use crate::vm::memory::Memory; +use crate::vm::registers::Registers; + +/// One pinned 6-round vector from the validated oracle. +struct Blake3Vector { + h: [u32; 8], + m: [u32; 16], + t: u64, + block_len: u32, + flags: u32, + out: [u32; 16], +} + +/// The 10 canonical 6-round vectors, generated from +/// `tooling/blake3-second-source/blake3-oracle/canonical_6round_vectors.json` (which the +/// oracle harness regenerates and which was validated against the official +/// `blake3` crate). Do not edit by hand. +const CANONICAL_6ROUND_VECTORS: &[Blake3Vector] = &[ + Blake3Vector { + h: [ + 0xd82c07cd, 0x6baa9455, 0x82e2e662, 0x7a024204, 0xe87a1613, 0x81332876, 0x48268673, + 0xc17c6279, + ], + m: [ + 0xe6f4590b, 0x4f65d4d9, 0xbad640fb, 0xaf19922a, 0x19c78df4, 0x6f25e2a2, 0xe9bb17bc, + 0x7a1d5006, 0x42af9fc3, 0x03983ca8, 0xde1b372a, 0xded733e8, 0x9148624f, 0xf7b0b7d2, + 0x72ae2244, 0xeece328b, + ], + t: 0xb4e1357d4a84eb03, + block_len: 42, + flags: 52, + out: [ + 0xced9d1ff, 0xc248eeab, 0xbd109b7f, 0x911b48f6, 0x923d62c0, 0xd804903f, 0x5974223e, + 0xaa4f0c80, 0xad61007f, 0xb50b8ddb, 0xe7372be1, 0x33d3d6c3, 0x42aa284b, 0xc5a25f28, + 0x79ac8370, 0xb75f3915, + ], + }, + Blake3Vector { + h: [ + 0xc386bbc4, 0x414c343c, 0x7311d8a3, 0xa6cecc1b, 0xc9e9c616, 0x18072e8c, 0xd5f4b3b2, + 0x7204e52d, + ], + m: [ + 0xf1fd42a2, 0xe6c3f339, 0x07d4bedc, 0x8a9a021e, 0x3bab6c39, 0x05805975, 0xa46d6753, + 0xdc2574bd, 0xab99254a, 0x4da98f1d, 0xe1ea24c4, 0x815a47c5, 0x08d6af57, 0xcc22af58, + 0x2c4a3698, 0x5fec898f, + ], + t: 0xc74803e31ba16215, + block_len: 50, + flags: 94, + out: [ + 0xf2a972e9, 0x81fdb8ec, 0x40c50ebc, 0x4ba1caf9, 0x9ee9e930, 0x6b1a16b2, 0xe9156f47, + 0xa89fb436, 0xa2f616b3, 0x12874c12, 0x30768035, 0xe01a17d9, 0xbee5c17c, 0xd61c0be0, + 0x3041ff46, 0xdfb91125, + ], + }, + Blake3Vector { + h: [ + 0x0e7a269f, 0x15ba2bdd, 0xd5e34124, 0x4ee207f8, 0x9b1f282e, 0x9b575bd1, 0xf30b94fa, + 0x0706a045, + ], + m: [ + 0x6148a86f, 0x8697bbd0, 0x8f7d9b78, 0x3c729578, 0x061b9030, 0x533c9135, 0x829e07b0, + 0xe4c11ab2, 0xcbf87544, 0xc34c769f, 0x5a91c89b, 0xf63f23d0, 0xc1066932, 0x87c56473, + 0x7d718d73, 0xecc1cb63, + ], + t: 0x7604e4b4e73695c3, + block_len: 58, + flags: 124, + out: [ + 0x5aa6b114, 0xc9d6740c, 0x8738caf4, 0xac5f4b72, 0x9fc6b9de, 0x3f2efb8f, 0x8cb7a912, + 0xf497a285, 0x3d062266, 0x7f22380c, 0xafd468fa, 0x122cba80, 0x446b156d, 0xb239d8c2, + 0xc3eab2cf, 0x775f2f92, + ], + }, + Blake3Vector { + h: [ + 0x8b529b4a, 0x9a9a80fd, 0xd6645fa9, 0x3bfd1d33, 0x79f248b0, 0x268ecc45, 0xa2863a7f, + 0x85ef3430, + ], + m: [ + 0xbdc2ae99, 0x10645d51, 0x97524d6a, 0xdd933160, 0xe0f9e038, 0xebcd1f5e, 0xef829c88, + 0xe0fd67dd, 0x18f2c41c, 0x22cedafb, 0x378c74dc, 0x4d100d8f, 0x95c76ab4, 0x95918694, + 0xe779c470, 0xedcf6109, + ], + t: 0x92d3043afcf249f3, + block_len: 36, + flags: 31, + out: [ + 0xeed92fab, 0x138d9358, 0x915bfe3c, 0x13718b01, 0xb506e277, 0xbe4007cd, 0x35847e06, + 0xce1c6896, 0x52fa01b5, 0x4aa26af8, 0xb1078a61, 0x2c517aed, 0xa08867a0, 0xea6ecfea, + 0x6d33d3b0, 0xdc293166, + ], + }, + Blake3Vector { + h: [ + 0x3c6da5d7, 0x656412a9, 0x27ac435a, 0x11072231, 0xeaff1a09, 0xc3e1b258, 0x8963dc6e, + 0x1b2ed40e, + ], + m: [ + 0xed6f0b09, 0xce80c4b0, 0xccea2645, 0x3184ff27, 0x4f5253a0, 0xe14b0190, 0x9b191bf4, + 0xabf4a07c, 0x81862fc9, 0x2d83a823, 0x793d0e45, 0x4cdce7a6, 0xe8abb93f, 0xe1df8af9, + 0x8224b122, 0x69f85e31, + ], + t: 0x49c7b59b995253fd, + block_len: 57, + flags: 41, + out: [ + 0xca00bda3, 0x84239a3a, 0xe7c88e6d, 0x33a8a3d6, 0x09dcd1ce, 0xa1b10212, 0xf48e1156, + 0x8f039915, 0x8a055eaa, 0xff5b11d5, 0xb725085b, 0x2e1ab267, 0x6ae7323d, 0xb2ff6fa8, + 0x7102c8a1, 0x7561eb37, + ], + }, + Blake3Vector { + h: [ + 0x9f767c45, 0xbde5c099, 0xf17fd374, 0xa6233255, 0xe6a16a3b, 0x1cfb10f6, 0x3f1f65a8, + 0x8b33e968, + ], + m: [ + 0x92edcf45, 0x377b9aa2, 0x478c281d, 0xc4069545, 0xcc11d357, 0x9e115e4b, 0x206f5c66, + 0xdf1461aa, 0xfb7ff337, 0xdf561d80, 0x4a0fe75d, 0xf6236bf2, 0x346c6e2b, 0xb0cde917, + 0xe4cc4132, 0x4c7d6df0, + ], + t: 0x6a3753915c76f18a, + block_len: 18, + flags: 67, + out: [ + 0x14a9f66f, 0x101bdfe8, 0x9b0a50dd, 0xee4bb45b, 0x7a914502, 0x77b3486b, 0x59bfc114, + 0xa1ad2afd, 0xc194dde6, 0x894ec54d, 0xad36c805, 0x9018f3f5, 0x165af5d8, 0x3e85b598, + 0x78e76653, 0xbb7a485d, + ], + }, + Blake3Vector { + h: [ + 0xd26b9496, 0x42f9a039, 0x001d9a88, 0x5f877031, 0xc527e279, 0x45cf8aa4, 0xcd4a5557, + 0xae9af169, + ], + m: [ + 0xaf895f5b, 0xd822e2f9, 0x17d7ab26, 0xccdf540b, 0xce06294d, 0x4a8b0188, 0xf38d2e64, + 0x5c41d5c5, 0xe8d5b9e3, 0x5c832a51, 0x9a0c1b76, 0x4de8344e, 0x96d2f9e0, 0x8677a5f2, + 0xa9a967c1, 0x323bbeaf, + ], + t: 0x390567c27bd6aa42, + block_len: 26, + flags: 3, + out: [ + 0x32a6ff70, 0xc30560bc, 0xd1c777c8, 0xf1871821, 0x7207ab54, 0x9f5b83c7, 0xb6561c5d, + 0x991e738f, 0xb38b62b9, 0x0ef6d156, 0x994becb1, 0x09a85d0e, 0x32221741, 0xada3cc5f, + 0x5b654ed6, 0x2a7a62b2, + ], + }, + Blake3Vector { + h: [ + 0x269e0d37, 0xa6a3a450, 0x892f902b, 0x81e74ef5, 0x099950d8, 0x6f03675a, 0x11e20b8f, + 0x6cad4a26, + ], + m: [ + 0xf29d0da9, 0x658cda14, 0xf9ebdacc, 0xdbc496cb, 0x4a23d596, 0x2e44158b, 0xa38fd547, + 0x5f557203, 0x34b9b5df, 0x506bf2ef, 0x7403e430, 0x4cbd87ad, 0xcb5c7427, 0x3e7d1bfb, + 0x930d6eaf, 0x86734721, + ], + t: 0x12bd4acefaecbd38, + block_len: 53, + flags: 42, + out: [ + 0xa632ad45, 0x12ce41f4, 0xd21b2cbd, 0x76795c62, 0x6bec36c1, 0xdafafcde, 0x53ca87b7, + 0x92e8465b, 0x7b424f5d, 0xe1e6ad7f, 0x753ba387, 0xccc50824, 0x69aedf6d, 0xbbbbf253, + 0x78d04883, 0xf3f33689, + ], + }, + Blake3Vector { + h: [ + 0x3a096533, 0xf658f7a7, 0x205738d1, 0xb46ee1da, 0x15ceb3a1, 0x359b1548, 0xa4517d6c, + 0x7589ca4a, + ], + m: [ + 0x74007cb4, 0xd49d0ac1, 0x16edc5d4, 0x685ca8af, 0x4223aa56, 0x10269470, 0x60908405, + 0xa92d04a3, 0x56a3e957, 0xb0f91306, 0xe6c08269, 0xf2306d4a, 0x31a06a7c, 0x9436d6f6, + 0xe18692e2, 0xe0c99f3e, + ], + t: 0x329911da9fbd8735, + block_len: 19, + flags: 91, + out: [ + 0x913b2ae1, 0xc7f73082, 0x45e1c023, 0x6f1f3f82, 0x20aee6f5, 0xdaf21d94, 0xf2c1e4af, + 0xd4f7d4ac, 0x44a45f87, 0xf4c40ce5, 0x613e9b94, 0x08ce53de, 0x4ff07aa4, 0x456bf2e2, + 0x2066ea7f, 0x3c5a654b, + ], + }, + Blake3Vector { + h: [ + 0x5f915ef0, 0x237751aa, 0x01a5ba50, 0x80b65386, 0x14b044d7, 0x61076dc3, 0xb99de255, + 0x283b73a6, + ], + m: [ + 0x3cee5e2c, 0x1c670ea9, 0x972651da, 0x4a8aa593, 0xac9abb0c, 0x35bb5c11, 0x47fbb3b4, + 0xcf3c17e5, 0xe2eb17c8, 0xe11e99fb, 0x7de0d208, 0x0602fe0c, 0x98cae043, 0x9425b3e2, + 0x33fb4b4f, 0x15607df9, + ], + t: 0xeaeb999b8a2e547e, + block_len: 64, + flags: 21, + out: [ + 0xf5ee9114, 0x856cabb8, 0x29be2cf1, 0x603be91c, 0x94a7dd0e, 0x28fc3e27, 0xb64e2cc8, + 0x2d2c67ff, 0x69fac1ba, 0x0c949090, 0xd68de435, 0xce91a527, 0xe80c1815, 0x6d44efe6, + 0x87c7b175, 0xd18a8b94, + ], + }, +]; + +#[test] +fn test_blake3_6round_canonical_vectors() { + for (i, v) in CANONICAL_6ROUND_VECTORS.iter().enumerate() { + let out = blake3_compress_6round(&v.h, &v.m, v.t, v.block_len, v.flags); + assert_eq!(out, v.out, "canonical 6-round vector {i} mismatch"); + } +} + +#[test] +fn test_blake3_syscall_matches_vectors() { + for (i, v) in CANONICAL_6ROUND_VECTORS.iter().enumerate() { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + let addr = 0x1000u64; + + // Lay out the 176-byte state region: h | m | t | (block_len, flags) | out. + let mut words = [0u32; 28]; + words[0..8].copy_from_slice(&v.h); + words[8..24].copy_from_slice(&v.m); + words[24] = v.t as u32; + words[25] = (v.t >> 32) as u32; + words[26] = v.block_len; + words[27] = v.flags; + for k in 0..14 { + let dw = (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32); + memory.store_doubleword(addr + (k as u64) * 8, dw).unwrap(); + } + // Pre-fill the out region so the test catches a partial write. + for k in 14..22 { + memory + .store_doubleword(addr + (k as u64) * 8, 0xDEAD_BEEF_DEAD_BEEFu64) + .unwrap(); + } + + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + registers.write(10, addr).unwrap(); + Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap(); + + let mut got = [0u32; 16]; + for k in 0..8 { + let dw = memory + .load_doubleword(addr + ((14 + k) as u64) * 8) + .unwrap(); + got[2 * k] = dw as u32; + got[2 * k + 1] = (dw >> 32) as u32; + } + assert_eq!(got, v.out, "syscall output mismatch on vector {i}"); + + // The 112 input bytes must be untouched. + for k in 0..14 { + let dw = memory.load_doubleword(addr + (k as u64) * 8).unwrap(); + let expected = (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32); + assert_eq!(dw, expected, "input dword {k} clobbered on vector {i}"); + } + } +} + +#[test] +fn test_blake3_syscall_rejects_unaligned_state_addr() { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + registers.write(10, 0x1004).unwrap(); + + let err = Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap_err(); + assert!(matches!( + err, + ExecutionError::UnalignedBlake3StateAddress(0x1004) + )); +} + +#[test] +fn test_blake3_syscall_rejects_overflowing_state_range() { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + // 22 dwords = 176 bytes; addr + 175 must not overflow. u64::MAX - 167 is + // 8-aligned and the last byte lands at u64::MAX + 8 → overflow. + registers.write(10, u64::MAX - 167).unwrap(); + + let err = Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap_err(); + assert!(matches!( + err, + ExecutionError::Blake3StateAddressOverflow(addr) if addr == u64::MAX - 167 + )); +} + +// ============================================================================= +// The chained-absorb ecall's argument validation +// ============================================================================= +// +// What the absorb ecall COMPUTES is gated in +// `prover::tables::blake3::executor_absorb_parity`, which drives it against +// `crypto`'s `Blake3Chain` over exhaustive message lengths — this crate cannot, +// having no `crypto` dependency. Checked here is the other half: that every +// argument the chip's constraints assume was validated is in fact rejected when +// it is wrong. A handler that accepted one would hand the prover a trace it +// cannot close, or — worse — one whose memory argument quietly means something +// other than the guest asked for. + +/// Registers for a well-formed absorb. +fn absorb_registers(ctrl: u64, msg: u64, num_blocks: u64, first_flags: u64) -> Registers { + let mut registers = Registers::default(); + registers.write(17, BLAKE3_ABSORB_SYSCALL_NUMBER).unwrap(); + registers.write(10, ctrl).unwrap(); + registers.write(11, msg).unwrap(); + registers.write(12, num_blocks).unwrap(); + registers.write(13, first_flags).unwrap(); + registers +} + +fn run_absorb(registers: &mut Registers) -> Result<(), ExecutionError> { + let mut pc = 0; + let mut memory = Memory::default(); + Instruction::EcallEbreak + .run(&mut pc, registers, &mut memory) + .map(|_| ()) +} + +#[test] +fn test_blake3_absorb_rejects_unaligned_addresses() { + // The accelerator reads both regions as doublewords. + for (ctrl, msg) in [(0x1004u64, 0x2000u64), (0x1000, 0x2004)] { + let mut registers = absorb_registers(ctrl, msg, 1, 0); + let err = run_absorb(&mut registers).unwrap_err(); + assert!( + matches!(err, ExecutionError::UnalignedBlake3AbsorbAddress(_)), + "ctrl={ctrl:#x} msg={msg:#x} gave {err:?}" + ); + } +} + +#[test] +fn test_blake3_absorb_rejects_block_counts_outside_the_range() { + // Zero blocks would put an Ecall tuple on the bus with no compression row + // to answer it. + let mut registers = absorb_registers(0x1000, 0x2000, 0, 0); + assert!(matches!( + run_absorb(&mut registers).unwrap_err(), + ExecutionError::Blake3AbsorbBlockCountOutOfRange(0) + )); + + // Past the cap the chip's row counter would no longer be bounded far below + // the field's modulus, and "the counter reaches 1" would stop implying "the + // group has exactly `num_blocks` rows". + let over = BLAKE3_ABSORB_MAX_BLOCKS + 1; + let mut registers = absorb_registers(0x1000, 0x2000, over, 0); + assert!(matches!( + run_absorb(&mut registers).unwrap_err(), + ExecutionError::Blake3AbsorbBlockCountOutOfRange(n) if n == over + )); +} + +#[test] +fn test_blake3_absorb_rejects_flags_wider_than_the_column() { + // The chip commits `flags` as one 32-bit word; a wider value would be + // truncated into the trace and the proof would attest a different hash. + let mut registers = absorb_registers(0x1000, 0x2000, 1, 1u64 << 32); + assert!(matches!( + run_absorb(&mut registers).unwrap_err(), + ExecutionError::Blake3AbsorbFlagsOutOfRange(f) if f == 1u64 << 32 + )); +} + +#[test] +fn test_blake3_absorb_rejects_overlapping_regions() { + // Every access of one absorb carries the same timestamp, so an address in + // both regions would be touched twice at that timestamp and the MEMW + // consistency argument could not order the pair. + let ctrl = 0x1000u64; + for msg in [ctrl, ctrl - 64, ctrl + 32, ctrl + 56] { + let mut registers = absorb_registers(ctrl, msg, 2, 0); + let err = run_absorb(&mut registers).unwrap_err(); + assert!( + matches!(err, ExecutionError::Blake3AbsorbRegionOverlap), + "msg={msg:#x} gave {err:?}" + ); + } + // CONTROL: regions that merely abut are accepted — the message's 128 bytes + // end exactly where the control region starts — so the check is discriminating + // overlap rather than rejecting every nearby address. + let mut registers = absorb_registers(ctrl, ctrl - 128, 2, 0); + assert!(!matches!( + run_absorb(&mut registers), + Err(ExecutionError::Blake3AbsorbRegionOverlap) + )); +} + +#[test] +fn test_blake3_absorb_rejects_overflowing_message_range() { + // The message's last byte must be addressable. This fires on the address + // check, NOT on the `num_blocks * 64` multiply: the block-count cap already + // bounds that product by 4 MiB, so its `checked_mul` is belt-and-braces + // against a future cap change and is unreachable from here. + let mut registers = absorb_registers(0x1000, u64::MAX - 63, 2, 0); + assert!(matches!( + run_absorb(&mut registers).unwrap_err(), + ExecutionError::Blake3AbsorbAddressOverflow(_) + )); +} + +/// A region ending exactly at the top of the address space must not wrap the +/// overlap test. +/// +/// The overlap check is written on each region's LAST addressable byte rather +/// than on `addr + bytes`, because the latter is `2^64` for a region ending at +/// `u64::MAX` — which panics in a debug build on a guest-chosen argument, and +/// in a release build wraps to `0` and makes the check report "no overlap" for +/// two regions that do overlap. Both are reachable from the guest, so neither +/// is acceptable: the executor must return an error, never panic, and never +/// accept an overlap. +#[test] +fn test_blake3_absorb_handles_a_region_ending_at_the_top_of_memory() { + // The message's last byte is exactly u64::MAX: 64 blocks of 64 bytes ending + // at the ceiling. The control region sits inside it, so this IS an overlap + // and must be reported as one rather than wrapping into acceptance. + let msg_addr = u64::MAX - 64 * 64 + 1; + let ctrl_addr = msg_addr + 128; + let mut registers = absorb_registers(ctrl_addr, msg_addr, 64, 0); + assert!(matches!( + run_absorb(&mut registers).unwrap_err(), + ExecutionError::Blake3AbsorbRegionOverlap + )); + + // And the same top-of-memory message with a control region well clear of it + // must get past the range and overlap checks rather than faulting on them. + let mut registers = absorb_registers(0x1000, msg_addr, 64, 0); + let err = run_absorb(&mut registers); + assert!( + !matches!( + err, + Err(ExecutionError::Blake3AbsorbRegionOverlap) + | Err(ExecutionError::Blake3AbsorbAddressOverflow(_)) + ), + "a region ending exactly at u64::MAX is addressable, got {err:?}" + ); +} diff --git a/executor/src/tests/ecrec_tests.rs b/executor/src/tests/ecrec_tests.rs new file mode 100644 index 000000000..a4675c4a0 --- /dev/null +++ b/executor/src/tests/ecrec_tests.rs @@ -0,0 +1,163 @@ +//! Tests for the ECREC (ECDSA public-key recovery) syscall. + +use k256::elliptic_curve::PrimeField; +use k256::elliptic_curve::ops::Reduce; +use k256::elliptic_curve::sec1::ToEncodedPoint; +use k256::{ProjectivePoint, Scalar, U256}; + +use crate::vm::instruction::decoding::Instruction; +use crate::vm::instruction::execution::{ECREC_SYSCALL_NUMBER, ExecutionError}; +use crate::vm::memory::Memory; +use crate::vm::registers::Registers; + +fn store(memory: &mut Memory, addr: u64, bytes: &[u8]) { + for (i, chunk) in bytes.chunks_exact(8).enumerate() { + let dw = u64::from_le_bytes(chunk.try_into().unwrap()); + memory.store_doubleword(addr + 8 * i as u64, dw).unwrap(); + } +} + +fn load(memory: &Memory, addr: u64, len: usize) -> Vec { + (0..len / 8) + .flat_map(|i| { + memory + .load_doubleword(addr + 8 * i as u64) + .unwrap() + .to_le_bytes() + }) + .collect() +} + +/// Runs the call with `sig` at `addr_sig`, `msg` at `addr_msg`, the output at +/// `addr_out` and `parity` in x13; returns the 72 bytes written. +fn run_at( + sig: &[u8; 64], + msg: &[u8; 32], + parity: u64, + addr_out: u64, + addr_sig: u64, + addr_msg: u64, +) -> Result, ExecutionError> { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + store(&mut memory, addr_sig, sig); + store(&mut memory, addr_msg, msg); + registers.write(17, ECREC_SYSCALL_NUMBER).unwrap(); + registers.write(10, addr_out).unwrap(); + registers.write(11, addr_sig).unwrap(); + registers.write(12, addr_msg).unwrap(); + registers.write(13, parity).unwrap(); + Instruction::EcallEbreak.run(&mut pc, &mut registers, &mut memory)?; + Ok(load(&memory, addr_out, 72)) +} + +fn run(sig: &[u8; 64], msg: &[u8; 32], parity: u64) -> Result, ExecutionError> { + run_at(sig, msg, parity, 0x1000, 0x2000, 0x3000) +} + +/// A k256 signature by `d` with nonce `k` on hash `msg`, its R parity and the key. +fn sign(d: u64, k: u64, msg: &[u8; 32]) -> ([u8; 64], u64, Vec) { + let (d, k) = ( + Scalar::from(d) * Scalar::from(d), + Scalar::from(k) * Scalar::from(k), + ); + let r_pt = (ProjectivePoint::GENERATOR * k) + .to_affine() + .to_encoded_point(false); + let r = Option::::from(Scalar::from_repr(*r_pt.x().unwrap())).unwrap(); + let z = >::reduce_bytes(&(*msg).into()); + let s = k.invert().unwrap() * (z + r * d); + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&r.to_bytes()); + sig[32..].copy_from_slice(&s.to_bytes()); + let q = (ProjectivePoint::GENERATOR * d) + .to_affine() + .to_encoded_point(false); + ( + sig, + u64::from(r_pt.y().unwrap()[31] & 1), + q.as_bytes()[1..65].to_vec(), + ) +} + +#[test] +fn ecrec_syscall_recovers_the_key() { + for (d, k) in [(3u64, 5u64), (0xDEAD_BEEF, 0x1234_5678), (u64::MAX, 77)] { + let msg = [0xA7u8; 32]; + let (sig, parity, key) = sign(d, k, &msg); + let out = run(&sig, &msg, parity).unwrap(); + assert_eq!(&out[..64], &key[..], "d = {d}"); + assert_eq!(&out[64..], &[0u8; 8], "status 0"); + // The other parity is the other R: some other key, still status 0. + let other = run(&sig, &msg, parity ^ 1).unwrap(); + assert_ne!(&other[..64], &key[..]); + assert_eq!(other[64], 0); + } +} + +#[test] +fn ecrec_syscall_bails_on_what_it_cannot_compute() { + let msg = [0x11u8; 32]; + let (sig, parity, _) = sign(9, 13, &msg); + let mut n_be = ecsm::N_BYTES; + n_be.reverse(); + let mut cases = vec![]; + for (lo, hi, value) in [ + (0, 32, [0u8; 32]), + (0, 32, n_be), + (32, 64, [0u8; 32]), + (32, 64, n_be), + ] { + let mut s = sig; + s[lo..hi].copy_from_slice(&value); + cases.push((s, msg)); + } + cases.push((sig, [0u8; 32])); // z = 0: u1 = 0 + for (sig, msg) in cases { + let out = run(&sig, &msg, parity).unwrap(); + assert_eq!(&out[..64], &[0u8; 64]); + assert_eq!(out[64], 1, "status 1"); + assert_eq!(&out[65..], &[0u8; 7]); + } +} + +#[test] +fn ecrec_syscall_rejects_a_parity_above_one() { + let msg = [0x22u8; 32]; + let (sig, _, _) = sign(5, 7, &msg); + for parity in [2u64, 3, u64::MAX] { + assert!(matches!( + run(&sig, &msg, parity), + Err(ExecutionError::EcrecParityOutOfRange(p)) if p == parity + )); + } +} + +#[test] +fn ecrec_syscall_rejects_overlapping_operands_and_address_overflow() { + let msg = [0x33u8; 32]; + let (sig, parity, _) = sign(5, 7, &msg); + // The hash overlapping the signature (64 bytes at 0x2000). + for addr_msg in [0x2000u64, 0x2020, 0x2038, 0x1FE8] { + assert!(matches!( + run_at(&sig, &msg, parity, 0x1000, 0x2000, addr_msg), + Err(ExecutionError::EcrecOperandOverlap) + )); + } + run_at(&sig, &msg, parity, 0x1000, 0x2000, 0x2040).expect("hash right after the signature"); + run_at(&sig, &msg, parity, 0x1000, 0x2000, 0x1FE0).expect("hash right before it"); + run_at(&sig, &msg, parity, 0x2000, 0x2000, 0x3000).expect("output aliasing the signature"); + // Each operand's last byte stays in the low limb: +71 out, +63 sig, +31 msg. + for (out, s, m) in [ + (0xFFFF_FFB9, 0x2000, 0x3000), + (0x1000, 0xFFFF_FFC1, 0x3000), + (0x1000, 0x2000, 0xFFFF_FFE1), + ] { + assert!(matches!( + run_at(&sig, &msg, parity, out, s, m), + Err(ExecutionError::EcrecAddressOverflow) + )); + } + run_at(&sig, &msg, parity, 0xFFFF_FFB8, 0x2000, 0x3000).expect("output ending at the limb top"); +} diff --git a/executor/src/tests/exec_paths_tests.rs b/executor/src/tests/exec_paths_tests.rs new file mode 100644 index 000000000..8e8ca0afc --- /dev/null +++ b/executor/src/tests/exec_paths_tests.rs @@ -0,0 +1,329 @@ +//! The executor's access paths ([`ExecPaths`]): every fast path reads and +//! writes the bytes, and marks the written words, the general path does, and an +//! executor on the fast paths writes the logs the general one writes. +use crate::elf::{Elf, Segment}; +use crate::vm::execution::Executor; +use crate::vm::instruction::decoding::Instruction; +use crate::vm::instruction::execution::{ + DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_GAP, DMA_MEMSET_SYSCALL_NUMBER, ExecutionError, +}; +use crate::vm::logs::Log; +use crate::vm::memory::{ExecPaths, Memory, StoreKind}; +use crate::vm::registers::Registers; +use proptest::prelude::*; + +/// Every path setting: the general one, each fast path alone, both. +const SETTINGS: [ExecPaths; 4] = [ + ExecPaths::GENERAL, + ExecPaths { + fixed_access: true, + dma_bulk: false, + }, + ExecPaths { + fixed_access: false, + dma_bulk: true, + }, + ExecPaths::FAST, +]; + +/// Every written word's bytes, in address order: what a snapshot, the prover's +/// image and `iter_bytes` read. +fn written(memory: &Memory) -> Vec<(u64, u8)> { + let mut bytes: Vec<_> = memory.iter_bytes().collect(); + bytes.sort_unstable(); + bytes +} + +/// Bases near the places an access can go wrong: a page's start and end, the +/// low directory's last page, the stack's end of the space and a page in +/// neither directory. +const BASES: [u64; 6] = [ + 0x1000, + (1 << 16) - 16, + (1 << 33) - 24, + u64::MAX - 0x40, + 1 << 40, + 0xFF00_0000, +]; + +#[derive(Clone, Debug)] +enum Access { + Load { width: u8, at: u64 }, + Store { width: u8, at: u64, value: u64 }, +} + +fn access() -> impl Strategy { + let at = (0..BASES.len(), 0u64..32).prop_map(|(b, off)| BASES[b].wrapping_add(off)); + let width = prop_oneof![Just(1u8), Just(2u8), Just(4u8), Just(8u8)]; + prop_oneof![ + (width.clone(), at.clone()).prop_map(|(width, at)| Access::Load { width, at }), + (width, at, any::()).prop_map(|(width, at, value)| Access::Store { width, at, value }), + ] +} + +/// One access on `memory`: a load's value, or `None` for a store; an access +/// past the end of the space is the error either path gives. +fn apply(memory: &mut Memory, access: &Access) -> Result, String> { + let e = |e: crate::vm::memory::MemoryError| e.to_string(); + Ok(match *access { + Access::Load { width, at } => Some(match width { + 1 => memory.load_byte(at) as u64, + 2 => memory.load_half(at).map_err(e)? as u64, + 4 => memory.load_word(at).map_err(e)? as u64, + _ => memory.load_doubleword(at).map_err(e)?, + }), + Access::Store { width, at, value } => { + match width { + 1 => memory.store_byte(at, value as u8), + 2 => memory.store_half(at, value as u16).map_err(e)?, + 4 => memory.store_word(at, value as u32).map_err(e)?, + _ => memory.store_doubleword(at, value).map_err(e)?, + } + None + } + }) +} + +proptest! { + // Any run of loads and stores, inside a page, across a page boundary, in + // either directory and in neither: every setting loads the same values and + // leaves the same written bytes, on both stores. + #[test] + fn every_access_path_reads_and_writes_the_same_bytes( + accesses in prop::collection::vec(access(), 1..200), + ) { + for kind in [StoreKind::Pages, StoreKind::Words] { + let mut reference = Memory::with_store_and_paths(kind, ExecPaths::GENERAL); + let mut fast = Memory::with_store_and_paths(kind, ExecPaths::FAST); + for a in &accesses { + prop_assert_eq!(apply(&mut reference, a), apply(&mut fast, a), "{:?} on {:?}", a, kind); + } + prop_assert_eq!(written(&reference), written(&fast), "{:?}", kind); + } + } + + // A DMA memcpy or memset of any length, overlapping or not, crossing a page + // or not: every setting writes the same log, the same bytes and the same + // written words, and refuses the same calls. + #[test] + fn every_dma_path_copies_the_same_bytes( + base in 0usize..BASES.len(), + src_off in 0u64..64, + gap in -40i64..300, + count in 0u64..=256, + memset in any::(), + seed in prop::collection::vec(any::(), 600), + ) { + let src = BASES[base].wrapping_sub(256).wrapping_add(src_off); + let dst = if memset { src.wrapping_add(DMA_MEMSET_GAP) } else { src.wrapping_add_signed(gap) }; + let number = if memset { DMA_MEMSET_SYSCALL_NUMBER } else { DMA_MEMCPY_SYSCALL_NUMBER }; + let mut outcomes = Vec::new(); + for paths in SETTINGS { + let mut memory = Memory::with_store_and_paths(StoreKind::Pages, paths); + // Some of the bytes around the call written, some never. + for (i, &b) in seed.iter().enumerate() { + if b % 3 != 0 { + memory.store_byte(src.wrapping_sub(300).wrapping_add(i as u64), b); + } + } + let log = run_ecall(&mut memory, number, dst, src, count) + .map(|log| (log.next_pc, log.src1_val, log.src2_val, log.dst_val)) + .map_err(|e| e.to_string()); + outcomes.push((log, written(&memory))); + } + for (paths, outcome) in SETTINGS.iter().zip(&outcomes).skip(1) { + prop_assert_eq!(outcome, &outcomes[0], "{:?}", paths); + } + } +} + +fn run_ecall( + memory: &mut Memory, + number: u64, + dst: u64, + src: u64, + count: u64, +) -> Result { + let mut registers = Registers::default(); + let mut pc = 0x1000; + registers.write(17, number)?; + registers.write(10, dst)?; + registers.write(11, src)?; + registers.write(12, count)?; + Instruction::EcallEbreak.run(&mut pc, &mut registers, memory) +} + +// RV64 encodings for the little program below. +fn i_type(opcode: u32, funct3: u32, rd: u32, rs1: u32, imm: i32) -> u32 { + ((imm as u32 & 0xfff) << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | opcode +} +fn s_type(funct3: u32, rs1: u32, rs2: u32, imm: i32) -> u32 { + let imm = imm as u32 & 0xfff; + ((imm >> 5) << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | ((imm & 0x1f) << 7) | 0x23 +} +fn b_type(funct3: u32, rs1: u32, rs2: u32, imm: i32) -> u32 { + let imm = imm as u32 & 0x1fff; + (((imm >> 12) & 1) << 31) + | (((imm >> 5) & 0x3f) << 25) + | (rs2 << 20) + | (rs1 << 15) + | (funct3 << 12) + | (((imm >> 1) & 0xf) << 8) + | (((imm >> 11) & 1) << 7) + | 0x63 +} +fn addi(rd: u32, rs1: u32, imm: i32) -> u32 { + i_type(0x13, 0, rd, rs1, imm) +} +const ECALL: u32 = 0x73; + +/// A program that stores and loads bytes, halves, words and doublewords at a +/// moving, misaligned address across a page boundary, and copies with both DMA +/// calls, `rounds` times, then halts. +fn program(rounds: i32) -> Elf { + let base = 0x1_0000u64; + let mut code = vec![ + // x5 = rounds, x6 = a moving address below the page boundary at + // 0x30000 (it passes it on the late rounds), x7 = a value. + addi(5, 0, rounds), + (0x30 << 12) | (6 << 7) | 0x37, // lui x6, 0x30 → 0x30000 + addi(6, 6, -0x200), // x6 = 0x2fe00 + addi(7, 0, -0x123), + ]; + let top = code.len(); + code.extend([ + addi(7, 7, 0x2f1), + s_type(0, 6, 7, 3), // sb x7, 3(x6) + s_type(1, 6, 7, 5), // sh x7, 5(x6) + s_type(2, 6, 7, 9), // sw x7, 9(x6) + s_type(3, 6, 7, 0x1fb), // sd x7, 0x1fb(x6): crosses 0x30000 on some rounds + i_type(0x03, 3, 28, 6, 0x1f9), // ld x28, 0x1f9(x6) + i_type(0x03, 4, 29, 6, 3), // lbu x29, 3(x6) + i_type(0x03, 1, 30, 6, 5), // lh x30, 5(x6) + i_type(0x03, 6, 31, 6, 9), // lwu x31, 9(x6) + // memcpy(dst = x6 + 64, src = x6, n = 37) + addi(10, 6, 64), + addi(11, 6, 0), + addi(12, 0, 37), + addi(17, 0, -30), // DMA_MEMCPY_SYSCALL_NUMBER = -30 + ECALL, + // memset(dst = x6 + 8 + 128, src = x6 + 128, n = 29) + addi(11, 6, 128), + addi(10, 11, 8), + addi(12, 0, 29), + addi(17, 0, -31), // DMA_MEMSET_SYSCALL_NUMBER = -31 + ECALL, + addi(6, 6, 7), + addi(5, 5, -1), + ]); + let back = -(((code.len() - top) * 4) as i32); + code.push(b_type(1, 5, 0, back)); // bne x5, x0, top + code.extend([addi(10, 0, 0), addi(17, 0, 93), ECALL]); + Elf { + entry_point: base, + data: vec![Segment { + base_addr: base, + values: code, + is_executable: true, + }], + } +} + +/// Every log of a run, a window of `window` cycles at a time. +fn logs_of(mut executor: Executor, window: usize, owned: bool) -> Vec { + let mut all = Vec::new(); + if owned { + while let Some(logs) = executor.resume_owned(window).expect("the run") { + all.extend(logs); + } + } else { + while let Some(logs) = executor.resume_with_limit(window).expect("the run") { + all.extend_from_slice(logs); + } + } + all +} + +fn fields(logs: &[Log]) -> Vec<[u64; 5]> { + logs.iter() + .map(|l| [l.current_pc, l.next_pc, l.src1_val, l.src2_val, l.dst_val]) + .collect() +} + +// The program above writes the same logs on every path setting, and the +// fast paths are not idle in it (the copies and the page-crossing doubleword). +#[test] +fn every_path_setting_writes_the_same_logs() { + let elf = program(80); + let reference = fields(&logs_of( + Executor::new_with_paths(&elf, vec![], ExecPaths::GENERAL).unwrap(), + 1 << 10, + false, + )); + assert_eq!(reference.len(), 4 + 80 * 22 + 3, "the program's cycles"); + for paths in SETTINGS { + let run = Executor::new_with_paths(&elf, vec![], paths).unwrap(); + assert_eq!( + fields(&logs_of(run, 1 << 10, false)), + reference, + "{paths:?}" + ); + } +} + +// `resume_owned` hands out, window by window, the logs `resume_with_limit` +// lends, including a last window shorter than the rest. +#[test] +fn resume_owned_hands_out_the_logs_resume_lends() { + let elf = program(30); + for window in [1, 7, 64, 1 << 20] { + let lent = logs_of(Executor::new(&elf, vec![]).unwrap(), window, false); + let owned = logs_of(Executor::new(&elf, vec![]).unwrap(), window, true); + assert_eq!(fields(&owned), fields(&lent), "window {window}"); + } + // Every window handed out holds exactly its logs: the first (the + // executor's own buffer was sized before the window was known), the full + // ones and the short last one, for windows below and above that size. + for window in [7, 8, 1 << 17] { + let mut executor = Executor::new(&elf, vec![]).unwrap(); + let mut windows = 0; + while let Some(logs) = executor.resume_owned(window).unwrap() { + assert_eq!(logs.capacity(), logs.len(), "window {window} #{windows}"); + windows += 1; + } + assert!(windows > 0); + } + let mut executor = Executor::new(&elf, vec![]).unwrap(); + let first = executor.resume_owned(7).unwrap().unwrap(); + assert_eq!(first.len(), 7); + let second = executor.resume_owned(7).unwrap().unwrap(); + assert_eq!( + second[0].current_pc, first[6].next_pc, + "the next window goes on" + ); +} + +// The knobs: each path on unless its knob is `0`. +#[test] +fn each_path_is_on_unless_its_knob_is_zero() { + assert_eq!(ExecPaths::from_values(None, None), ExecPaths::FAST); + assert_eq!(ExecPaths::from_values(Some("1"), Some("")), ExecPaths::FAST); + assert_eq!( + ExecPaths::from_values(Some("0"), Some(" 0 ")), + ExecPaths::GENERAL + ); + assert_eq!( + ExecPaths::from_values(Some("0"), None), + ExecPaths { + fixed_access: false, + dma_bulk: true + } + ); + assert_eq!( + ExecPaths::from_values(None, Some("0")), + ExecPaths { + fixed_access: true, + dma_bulk: false + } + ); +} diff --git a/executor/src/tests/keccak_tests.rs b/executor/src/tests/keccak_tests.rs index 1c592f080..ae9dad301 100644 --- a/executor/src/tests/keccak_tests.rs +++ b/executor/src/tests/keccak_tests.rs @@ -86,3 +86,206 @@ fn test_keccak_syscall_rejects_overflowing_state_range() { ExecutionError::KeccakStateAddressOverflow(addr) if addr == u64::MAX - 191 )); } + +// ============================================================================= +// Keccak-256 of a guest buffer (KECCAK256_SYSCALL_NUMBER) +// ============================================================================= + +mod keccak256_call { + use crate::vm::instruction::decoding::Instruction; + use crate::vm::instruction::execution::{ + ExecutionError, KECCAK256_RATE_BYTES, KECCAK256_SYSCALL_NUMBER, keccak_f1600, + keccak256_addresses_ok, keccak256_blocks, keccak256_digest, keccak256_padded_block, + keccak256_permutations, + }; + use crate::vm::memory::Memory; + use crate::vm::registers::Registers; + use tiny_keccak::{Hasher, Keccak}; + + fn reference(input: &[u8]) -> [u8; 32] { + let mut hasher = Keccak::v256(); + hasher.update(input); + let mut out = [0u8; 32]; + hasher.finalize(&mut out); + out + } + + /// Deterministic, non-repeating bytes. + fn bytes(len: usize, seed: u64) -> Vec { + let mut x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect() + } + + /// Runs the ecall over `input` placed at `ptr`, digest at `out`; returns the + /// digest read back and the log's `(src2_val, dst_val)`. + fn run_call(input: &[u8], ptr: u64, out: u64) -> Result<([u8; 32], u64, u64), ExecutionError> { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + for (i, b) in input.iter().enumerate() { + memory.store_byte(ptr + i as u64, *b); + } + registers.write(17, KECCAK256_SYSCALL_NUMBER).unwrap(); + registers.write(10, ptr).unwrap(); + registers.write(11, input.len() as u64).unwrap(); + registers.write(12, out).unwrap(); + let log = Instruction::EcallEbreak.run(&mut pc, &mut registers, &mut memory)?; + assert_eq!(pc, 4, "the call falls through to the next instruction"); + assert_eq!(registers.read(10).unwrap(), ptr, "x10 unchanged"); + assert_eq!( + registers.read(11).unwrap(), + input.len() as u64, + "x11 unchanged" + ); + assert_eq!(registers.read(12).unwrap(), out, "x12 unchanged"); + let digest: [u8; 32] = memory.load_bytes(out, 32).unwrap().try_into().unwrap(); + Ok((digest, log.src2_val, log.dst_val)) + } + + /// The lengths the padding cases turn on, plus multi-block and multi-KB. + const LENGTHS: &[usize] = &[ + 0, 1, 7, 8, 9, 31, 32, 33, 64, 127, 128, 134, 135, 136, 137, 143, 144, 200, 270, 271, 272, + 273, 407, 408, 409, 1000, 4096, 10_000, + ]; + + #[test] + fn digest_matches_the_reference_at_every_padding_case() { + for &len in LENGTHS { + let input = bytes(len, len as u64 + 1); + assert_eq!(keccak256_digest(&input), reference(&input), "len {len}"); + } + // Every length to three blocks and a bit. + for len in 0..450 { + let input = bytes(len, 7); + assert_eq!(keccak256_digest(&input), reference(&input), "len {len}"); + } + } + + #[test] + fn the_call_writes_the_reference_digest_at_any_alignment() { + for &len in LENGTHS { + for ptr_off in 0..8u64 { + for out_off in [0u64, 3, 5] { + let input = bytes(len, 31 * len as u64 + ptr_off); + let ptr = 0x10_000 + ptr_off; + let out = 0x40_000 + out_off; + let (digest, blocks, logged_ptr) = run_call(&input, ptr, out).unwrap(); + assert_eq!( + digest, + reference(&input), + "len {len} ptr+{ptr_off} out+{out_off}" + ); + assert_eq!(blocks, len as u64 / 136 + 1, "logged permutation count"); + assert_eq!(logged_ptr, ptr); + } + } + } + } + + #[test] + fn the_digest_may_overwrite_its_own_input() { + for len in [0usize, 5, 32, 135, 136, 300] { + let input = bytes(len, 99); + let ptr = 0x2_0003u64; + // Output starting inside the input (or right at it). + let (digest, _, _) = run_call(&input, ptr, ptr + (len as u64) / 2).unwrap(); + assert_eq!( + digest, + reference(&input), + "len {len}: hash of the bytes before the call" + ); + } + } + + #[test] + fn permutations_chain_the_padded_blocks() { + for &len in LENGTHS { + let input = bytes(len, 3 * len as u64); + let perms = keccak256_permutations(&input); + assert_eq!(perms.len() as u64, keccak256_blocks(len as u64)); + let mut state = [0u64; 25]; + for (j, (start, end)) in perms.iter().enumerate() { + let block = keccak256_padded_block(&input, j); + let mut want = state; + for (lane, chunk) in want.iter_mut().zip(block.chunks_exact(8)) { + *lane ^= u64::from_le_bytes(chunk.try_into().unwrap()); + } + assert_eq!(*start, want, "len {len} block {j} input"); + keccak_f1600(&mut want); + assert_eq!(*end, want, "len {len} block {j} output"); + state = *end; + } + } + } + + #[test] + fn padded_last_block_cases() { + let rate = KECCAK256_RATE_BYTES as usize; + // len % 136 == 0: the last block is padding alone. + let b = keccak256_padded_block(&[0xAA; 136], 1); + assert_eq!(b[0], 0x01); + assert_eq!(b[rate - 1], 0x80); + assert!(b[1..rate - 1].iter().all(|&x| x == 0)); + // len % 136 == 135: 0x01 and 0x80 meet in byte 135. + let b = keccak256_padded_block(&[0xAA; 135], 0); + assert!(b[..135].iter().all(|&x| x == 0xAA)); + assert_eq!(b[135], 0x81); + // A full block of a longer input is unpadded. + let b = keccak256_padded_block(&[0x55; 300], 1); + assert!(b.iter().all(|&x| x == 0x55)); + } + + #[test] + fn address_conditions_match_the_chip() { + let top = 1u64 << 32; + // The last block's window ends exactly at the boundary: accepted. + assert!(keccak256_addresses_ok(top - 136, 0, 0)); + assert!(keccak256_addresses_ok(top - 272, 136, 0)); + assert!(keccak256_addresses_ok(top - 272, 271, 0)); + // One byte further: refused, even though the input itself ends earlier. + assert!(!keccak256_addresses_ok(top - 135, 0, 0)); + assert!(!keccak256_addresses_ok(top - 271, 136, 0)); + // Only the low word matters. + assert!(keccak256_addresses_ok(5 * top + 0x1000, 100, 7 * top)); + // Lengths of 2^32 or more never fit. + assert!(!keccak256_addresses_ok(0, top, 0)); + assert!(!keccak256_addresses_ok(0, u64::MAX, 0)); + // Output: the 32 bytes stay below the boundary. + assert!(keccak256_addresses_ok(0, 0, top - 32)); + assert!(!keccak256_addresses_ok(0, 0, top - 31)); + } + + #[test] + fn the_call_refuses_operands_across_a_word_boundary() { + let top = 1u64 << 32; + for (ptr, len, out) in [ + (top - 135, 0u64, 0x1000u64), + (top - 100, 99, 0x1000), + (0x1000, 10, top - 31), + (0x1000, top, 0x2000), + (0x1000, u64::MAX, 0x2000), + ] { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + registers.write(17, KECCAK256_SYSCALL_NUMBER).unwrap(); + registers.write(10, ptr).unwrap(); + registers.write(11, len).unwrap(); + registers.write(12, out).unwrap(); + let err = Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap_err(); + assert!( + matches!(err, ExecutionError::Keccak256AddressOverflow { .. }), + "ptr {ptr:#x} len {len} out {out:#x}: {err:?}" + ); + } + } +} diff --git a/executor/src/tests/memory_tests.rs b/executor/src/tests/memory_tests.rs index 6c3b3f2ed..7de682d67 100644 --- a/executor/src/tests/memory_tests.rs +++ b/executor/src/tests/memory_tests.rs @@ -139,3 +139,101 @@ fn test_misaligned_load_store_overflow_errors() { MemoryError::AddressOverflow )); } + +/// The paged store answers every access as the word map does: loads and +/// stores of every width, aligned or not, across 64 KiB pages, in the low and +/// top directories and between them, the overflow refusals at the top of the +/// address space, `load_bytes`, the private input, and `iter_bytes` (as a set: +/// the two iterate in different orders). +#[test] +fn the_paged_store_is_the_word_map() { + use crate::vm::memory::StoreKind; + + const PAGE: u64 = 1 << 16; + let mut pages = Memory::with_store(StoreKind::Pages); + let mut words = Memory::with_store(StoreKind::Words); + pages.store_private_inputs(vec![7u8; 70_001]).unwrap(); + words.store_private_inputs(vec![7u8; 70_001]).unwrap(); + + let spots = [ + 0, + 3 * PAGE - 3, + 0x1_0000_0000 - 2, + (1 << 33) - 5, + 1 << 33, + 0x8000_0000_0000_0000 - 3, + u64::MAX - (1 << 33) - 2, + u64::MAX - (1 << 33) + 6, + u64::MAX - 9, + u64::MAX - 1, + ]; + let mut x = 0x2545_f491_4f6c_dd1du64; + let mut next = || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + }; + let same = |a: Result, b: Result, what: &str| match (a, b) { + (Ok(a), Ok(b)) => assert_eq!(a, b, "{what}"), + (Err(a), Err(b)) => assert_eq!(format!("{a:?}"), format!("{b:?}"), "{what}"), + (a, b) => panic!("{what}: {a:?} vs {b:?}"), + }; + for _ in 0..200_000 { + let r = next(); + let addr = spots[(r % spots.len() as u64) as usize].wrapping_add((r >> 8) % 24); + let value = next(); + let what = format!("op {} at {addr:#x}", (r >> 16) % 10); + match (r >> 16) % 10 { + 0 => { + pages.store_byte(addr, value as u8); + words.store_byte(addr, value as u8); + } + 1 => same( + pages.store_half(addr, value as u16).map(|_| 0), + words.store_half(addr, value as u16).map(|_| 0), + &what, + ), + 2 => same( + pages.store_word(addr, value as u32).map(|_| 0), + words.store_word(addr, value as u32).map(|_| 0), + &what, + ), + 3 => same( + pages.store_doubleword(addr, value).map(|_| 0), + words.store_doubleword(addr, value).map(|_| 0), + &what, + ), + 4 => assert_eq!(pages.load_byte(addr), words.load_byte(addr), "{what}"), + 5 => same( + pages.load_half(addr).map(u64::from), + words.load_half(addr).map(u64::from), + &what, + ), + 6 => same( + pages.load_word(addr).map(u64::from), + words.load_word(addr).map(u64::from), + &what, + ), + 7 | 8 => same( + pages.load_doubleword(addr), + words.load_doubleword(addr), + &what, + ), + _ => { + let len = value % 40; + match (pages.load_bytes(addr, len), words.load_bytes(addr, len)) { + (Ok(a), Ok(b)) => assert_eq!(a, b, "{what}"), + (Err(a), Err(b)) => assert_eq!(format!("{a:?}"), format!("{b:?}"), "{what}"), + (a, b) => panic!("{what}: {a:?} vs {b:?}"), + } + } + } + } + let mut a: Vec<(u64, u8)> = pages.iter_bytes().collect(); + let mut b: Vec<(u64, u8)> = words.iter_bytes().collect(); + a.sort_unstable(); + b.sort_unstable(); + assert_eq!(a.len(), b.len(), "the words written"); + assert!(a == b, "the bytes written"); +} diff --git a/executor/src/tests/mod.rs b/executor/src/tests/mod.rs index fe2f6abc8..d46036998 100644 --- a/executor/src/tests/mod.rs +++ b/executor/src/tests/mod.rs @@ -1,6 +1,9 @@ +pub mod blake3_tests; pub mod commit_ecall_tests; pub mod dma_tests; +pub mod ecrec_tests; pub mod ecsm_tests; +pub mod exec_paths_tests; pub mod flamegraph_tests; pub mod hint_tests; pub mod keccak_tests; diff --git a/executor/src/vm/execution.rs b/executor/src/vm/execution.rs index dc0660178..64a929689 100644 --- a/executor/src/vm/execution.rs +++ b/executor/src/vm/execution.rs @@ -8,7 +8,7 @@ use crate::{ execution::ExecutionError, }, logs::Log, - memory::{Memory, MemoryError, U64HashMap}, + memory::{ExecPaths, Memory, MemoryError, StoreKind, U64HashMap}, registers::Registers, }, }; @@ -30,6 +30,10 @@ pub struct ExecutionResult { /// Size of each log chunk - balances memory usage vs callback overhead pub(crate) const CHUNK_SIZE: usize = 100_000; +/// The most logs [`Executor::resume_owned`] reserves for its next call +/// (2^22, 160 MiB); past it the buffer grows as logs are written. +const MAX_RESERVED_LOGS: usize = 1 << 22; + /// Result of executing one continuation epoch: the logs produced during the /// epoch and the VM state at the epoch boundary. The boundary state is the /// starting state of the next epoch. @@ -52,7 +56,25 @@ pub struct Executor { impl Executor { pub fn new(program: &Elf, private_inputs: Vec) -> Result { - let mut memory = Memory::default(); + Self::with_memory(program, private_inputs, Memory::default()) + } + + /// [`Executor::new`] on the given access paths (the knobs' defaults aside): + /// what a test compares the paths with. + pub fn new_with_paths( + program: &Elf, + private_inputs: Vec, + paths: ExecPaths, + ) -> Result { + let memory = Memory::with_store_and_paths(StoreKind::from_env(), paths); + Self::with_memory(program, private_inputs, memory) + } + + fn with_memory( + program: &Elf, + private_inputs: Vec, + mut memory: Memory, + ) -> Result { memory.store_private_inputs(private_inputs)?; let instructions = InstructionCache::new(&program.data)?; load_program(&program.data, &mut memory)?; @@ -133,6 +155,35 @@ impl Executor { } } + /// [`Executor::resume_with_limit`], the logs handed out as they lie rather + /// than lent: the executor keeps a new buffer of `limit` logs for the next + /// call. The same logs, without the copy a caller that keeps them would + /// make of the borrowed slice. + /// + /// The buffer handed out holds exactly its logs (capacity = length), as a + /// copied slice would: a caller that counts a window's bytes by its + /// capacity when it takes it and by its length when it lets it go (the + /// block prover's memory log does) counts the same bytes both times. + pub fn resume_owned(&mut self, limit: usize) -> Result>, ExecutorError> { + // A window's worth, no more than a bound: a caller passing + // `usize::MAX` as "the rest" must not reserve it. + let window = limit.min(MAX_RESERVED_LOGS); + if self.logs.capacity() < window { + // Reserved once, exactly: no doubling past the window as it fills. + self.logs.clear(); + self.logs.reserve_exact(window); + } + if self.resume_with_limit(limit)?.is_none() { + return Ok(None); + } + let mut logs = std::mem::replace(&mut self.logs, Vec::with_capacity(window)); + // A full window already fills its buffer; the first (the executor's + // own buffer, sized before the window was known) and the last (the + // run ended early) may not. + logs.shrink_to_fit(); + Ok(Some(logs)) + } + fn get_return_values(&self) -> Result { let memory_return_value = self.memory.read_return_value()?; let registers_return_values = self.registers.read_return_values(); diff --git a/executor/src/vm/instruction/execution.rs b/executor/src/vm/instruction/execution.rs index f8424ffb3..52842ca70 100644 --- a/executor/src/vm/instruction/execution.rs +++ b/executor/src/vm/instruction/execution.rs @@ -29,6 +29,11 @@ syscall_numbers! { KeccakPermute = 0, Print = 1, Panic = 2, + // Placeholder discriminant. The actual syscall value is BLAKE3_SYSCALL_NUMBER. + Blake3Compress = 3, + // Placeholder discriminant. The actual syscall value is + // BLAKE3_ABSORB_SYSCALL_NUMBER. + Blake3Absorb = 4, Commit = 64, Halt = 93, /// Placeholder discriminant. The actual syscall value is `ECSM_SYSCALL_NUMBER`. @@ -45,6 +50,12 @@ syscall_numbers! { /// `DMA_MEMSET_SYSCALL_NUMBER`. `memset` chunks are proven by the same /// COPY table, which derives the inverted timestamp order from this number. DmaMemset = 97, + /// Placeholder discriminant. The actual syscall value is + /// `KECCAK256_SYSCALL_NUMBER`: Keccak-256 of a guest buffer, proven by the + /// KECCAK table's sponge mode. + Keccak256 = 98, + /// Placeholder discriminant. The actual syscall value is `ECREC_SYSCALL_NUMBER`. + Ecrec = 99, } /// Syscall number for KeccakPermute (u64::MAX - 1 = 0xFFFF_FFFF_FFFF_FFFE). @@ -53,6 +64,210 @@ syscall_numbers! { pub const KECCAK_SYSCALL_NUMBER: u64 = u64::MAX - 1; const KECCAK_STATE_BYTES: u64 = 25 * 8; +/// Syscall number for Keccak-256 of a guest buffer (`u64::MAX - 4`, ECALL `-5`, +/// in the hash group `-1 ..= -10`). +/// +/// ABI: `x10` = input address (any alignment), `x11` = input length in bytes, +/// `x12` = address the 32-byte digest is written to (any alignment). Registers +/// are left unchanged. The digest is Keccak-256 with the original Keccak padding +/// (`0x01 ... 0x80`), the hash Ethereum uses. +/// +/// The input is read at the ecall's timestamp `T` and the digest written at +/// `T + 1`, so the output may overlap the input: the hash is of the bytes as they +/// were before the call. +/// +/// The KECCAK table proves the call one row per permutation: block `j` of the +/// padded input is read at `ptr + 136·j`, absorbed into the state carried from +/// row `j - 1`, and permuted; the last row writes the first four lanes. Its +/// address arithmetic adds offsets to the low 32-bit word of `ptr` and `out` +/// without a carry, so the call is refused unless +/// [`keccak256_addresses_ok`] holds — the chip accepts exactly the calls this +/// accepts. +pub const KECCAK256_SYSCALL_NUMBER: u64 = u64::MAX - 4; + +/// Bytes of one Keccak-256 block (the rate, 1088 bits). +pub const KECCAK256_RATE_BYTES: u64 = 136; + +/// Permutations (KECCAK rows) one Keccak-256 call of `len` bytes makes: one per +/// whole block and one for the padded last block, `⌊len / 136⌋ + 1`. +pub const fn keccak256_blocks(len: u64) -> u64 { + len / KECCAK256_RATE_BYTES + 1 +} + +/// Whether a Keccak-256 call's operands fit the chip's address arithmetic: +/// +/// - every block's 136-byte window stays below one `2^32` boundary of the low +/// address word: `(ptr mod 2^32) + 136·blocks ≤ 2^32` (this also bounds `len` +/// below `2^32`, which the chip needs for its byte count); +/// - the 32-byte digest does too: `(out mod 2^32) + 31 < 2^32`. +pub fn keccak256_addresses_ok(ptr: u64, len: u64, out: u64) -> bool { + let window = keccak256_blocks(len).checked_mul(KECCAK256_RATE_BYTES); + let input_ok = window.is_some_and(|w| { + (ptr % LOW_LIMB) + .checked_add(w) + .is_some_and(|e| e <= LOW_LIMB) + }); + input_ok && addr_limb_ok(out, 31) +} + +/// The permutations of Keccak-256 over `input`, block by block: each item is the +/// state a block's permutation starts from (the previous output with the padded +/// block XORed into its first 17 lanes) and the state it ends in. The digest is +/// the first four lanes of the last item's output, little-endian. +/// +/// One function for the executor and the trace builder, so the rows the prover +/// builds are the permutations the VM ran. +pub fn keccak256_permutations(input: &[u8]) -> Vec<([u64; 25], [u64; 25])> { + const RATE: usize = KECCAK256_RATE_BYTES as usize; + let blocks = input.len() / RATE + 1; + let mut out = Vec::with_capacity(blocks); + let mut state = [0u64; 25]; + for j in 0..blocks { + let block = keccak256_padded_block(input, j); + for (lane, chunk) in state.iter_mut().zip(block.chunks_exact(8)) { + *lane ^= u64::from_le_bytes(chunk.try_into().expect("8-byte chunk")); + } + let start = state; + keccak_f1600(&mut state); + out.push((start, state)); + } + out +} + +/// Block `j` of `input` after Keccak padding: the input's bytes, then on the last +/// block (`j = ⌊len / 136⌋`) `0x01` right after them and `0x80` on byte 135 +/// (`0x81` when the two meet). +pub fn keccak256_padded_block(input: &[u8], j: usize) -> [u8; 136] { + const RATE: usize = KECCAK256_RATE_BYTES as usize; + let mut block = [0u8; RATE]; + let start = j * RATE; + let take = input.len().saturating_sub(start).min(RATE); + block[..take].copy_from_slice(&input[start..start + take]); + if j == input.len() / RATE { + block[take] ^= 0x01; + block[RATE - 1] ^= 0x80; + } + block +} + +/// Keccak-256 of `input`, through [`keccak256_permutations`]. +pub fn keccak256_digest(input: &[u8]) -> [u8; 32] { + let perms = keccak256_permutations(input); + let (_, last) = perms.last().expect("at least one block"); + let mut digest = [0u8; 32]; + for (k, lane) in last.iter().take(4).enumerate() { + digest[8 * k..8 * k + 8].copy_from_slice(&lane.to_le_bytes()); + } + digest +} + +/// Syscall number for the BLAKE3 6-round compression accelerator +/// (u64::MAX - 2 = 0xFFFF_FFFF_FFFF_FFFD). +/// +/// This is the **6-round internal variant** of the BLAKE3 compression function +/// (`thoughts/blake3/blake3-chip/DESIGN.md`), intended for in-house Merkle / +/// Fiat–Shamir use — it is NOT standard 7-round BLAKE3 and its security rests +/// on the named 6-round assumption recorded in the design. +/// +/// ABI: `x10` = 8-byte-aligned pointer to a 176-byte state region laid out as +/// consecutive little-endian dwords at `addr + 8k`: +/// +/// | dword k | contents | +/// |---------|--------------------------------------------| +/// | 0..=3 | `h[0..8]` chaining value (2 u32 words/dword) | +/// | 4..=11 | `m[0..16]` message block | +/// | 12 | `t` counter (`t_lo = low u32 → v[12]`, `t_hi = high u32 → v[13]`) | +/// | 13 | `block_len` (low u32) \| `flags` (high u32) | +/// | 14..=21 | `out[0..16]` — written by the syscall | +pub const BLAKE3_SYSCALL_NUMBER: u64 = u64::MAX - 2; +/// Bytes of the BLAKE3 state region: 112 input + 64 output. +const BLAKE3_STATE_BYTES: u64 = 22 * 8; +/// Dword offset of `out[0..16]` inside the BLAKE3 state region. +const BLAKE3_OUT_DWORDS: u64 = 14; + +/// Syscall number for the BLAKE3 6-round **chained absorb** accelerator +/// (u64::MAX - 3 = 0xFFFF_FFFF_FFFF_FFFC). +/// +/// Same 6-round internal compression as [`BLAKE3_SYSCALL_NUMBER`], but one ecall +/// folds a run of `num_blocks` consecutive 64-byte message blocks into the +/// chaining value instead of one. That is the whole point: a guest hashing a +/// 1 KiB message pays one ecall and one marshaling round-trip rather than +/// sixteen, and the prover receives one `Ecall` bus tuple per absorb rather than +/// one per compression. +/// +/// ABI: +/// +/// | register | contents | +/// |----------|----------| +/// | `x10` | 8-byte-aligned pointer to the 64-byte control region | +/// | `x11` | 8-byte-aligned pointer to `num_blocks * 64` message bytes | +/// | `x12` | `num_blocks`, in `1..=`[`BLAKE3_ABSORB_MAX_BLOCKS`] | +/// | `x13` | `first_flags`, the flag word of the run's FIRST block (`< 2^32`) | +/// +/// Control region, as consecutive little-endian dwords at `ctrl + 8k`: +/// +/// | dword k | contents | +/// |---------|----------| +/// | 0..=3 | `cv_in[0..8]` — the incoming chaining value (read) | +/// | 4..=7 | `cv_out[0..8]` — the outgoing chaining value (written) | +/// +/// `cv_in` and `cv_out` are disjoint on purpose: every memory access of one +/// absorb happens at a single timestamp, and the chip reads `cv_in` and writes +/// `cv_out` through SEPARATE `Memw` interactions (one gated on the group's first +/// row, one on its last). Two separate interactions at one timestamp on one +/// address are a pair the MEMW consistency argument cannot order, so the two +/// halves have to be disjoint — and for the same reason the control and message +/// regions must not overlap ([`ExecutionError::Blake3AbsorbRegionOverlap`]). +/// +/// Note this is a consequence of the chip's interaction layout, NOT a blanket +/// property of the machine: a single COMBINED read+write interaction on one +/// address at one timestamp is fine, and is exactly what keccak does for its 25 +/// lanes and what the single-compression BLAKE3 mode does for its output region +/// (the `old_out` columns). Stated broadly the rule would imply keccak is +/// broken, which it is not. +/// +/// The block schedule this applies — `t = 0`, `block_len = 64` on every block, +/// `first_flags` on block 0 and `0` on every later block — is a SECOND statement +/// of the flag framing that `crypto`'s `Blake3Chain` owns (this crate cannot +/// call it: `executor` does not depend on `crypto/crypto`). It is therefore +/// parity-gated against `Blake3Chain` over exhaustive message lengths rather +/// than trusted; see [`blake3_absorb_chain_6round`]. +pub const BLAKE3_ABSORB_SYSCALL_NUMBER: u64 = u64::MAX - 3; +/// Bytes of one BLAKE3 message block. +pub const BLAKE3_BLOCK_BYTES: u64 = 64; +/// Dwords in the absorb control region: 4 for `cv_in`, 4 for `cv_out`. +pub const BLAKE3_ABSORB_CTRL_DWORDS: u64 = 8; +/// Dword offset of `cv_out` inside the absorb control region. +pub const BLAKE3_ABSORB_CV_OUT_DWORD: u64 = 4; +/// Largest `num_blocks` one absorb ecall accepts — 1 024 blocks, 64 KiB of +/// message. +/// +/// A guest with more to hash calls again; the chain is stateless across calls +/// because the chaining value travels through the control region. +/// +/// Three things the cap has to do, in increasing order of how easy they are to +/// get wrong: +/// +/// 1. **Soundness.** The chip counts a group's rows down from `num_blocks` in a +/// single field element, so the count must stay far below the modulus for +/// "the group has exactly `num_blocks` rows" to follow from "the counter +/// reaches 1". +/// 2. **Address arithmetic.** `num_blocks * 64` must not wrap. +/// 3. ★ **Trace height.** Every other accelerator is one CPU cycle to one chip +/// row, so an epoch's table heights are implicitly bounded by its cycle +/// budget — epochs split on cycles. This ecall is the first to break that +/// ratio: it is ONE cycle and up to `BLAKE3_ABSORB_MAX_BLOCKS` BLAKE3 rows, +/// and the BLAKE3 table has no entry in `prover::tables::max_rows` at all. +/// At 3 219 columns a row is ~26 KiB of trace, so an unbounded absorb near +/// an epoch boundary could demand a table height nothing sized for. +/// +/// 1 024 is chosen for (3), and it is free: the guest cost of an absorb is flat +/// in message length past ~4 blocks (measured 552 cycles at both 256 B and +/// 1 KiB, 558 at 4 KiB), and real absorbs — verifier leaves, transcript +/// segments — are KiB-class. So capping at 64 KiB costs nothing measurable and +/// bounds one ecall's trace to ~26 MiB instead of ~1.7 GiB. +pub const BLAKE3_ABSORB_MAX_BLOCKS: u64 = 1 << 10; + /// Syscall number for the ECSM (elliptic-curve scalar multiply) accelerator. /// /// The spec uses ECALL number `-11`; interpreted as an unsigned 64-bit value that is @@ -60,6 +275,16 @@ const KECCAK_STATE_BYTES: u64 = 25 * 8; /// bus as `[lo32, hi32] = [2^32 - 11, 2^32 - 1]`. pub const ECSM_SYSCALL_NUMBER: u64 = u64::MAX - 10; +/// Syscall number for the ECDSA public-key recovery accelerator (secp256k1). +/// +/// The spec uses ECALL number `-12`, i.e. `u64::MAX - 11 = 0xFFFF_FFFF_FFFF_FFF4`, which +/// the ECREC table puts on the `Ecall` bus as `[lo32, hi32] = [2^32 - 12, 2^32 - 1]`. +pub const ECREC_SYSCALL_NUMBER: u64 = u64::MAX - 11; + +/// Bytes the ECREC call writes at `x10`: the key's x and y (32 bytes each, big-endian) +/// and the status doubleword (0 = key, 1 = bailed). +pub const ECREC_OUT_BYTES: u64 = 72; + /// Syscall number for the copy accelerator, serving `memcpy` and `memmove`. /// /// The spec uses ECALL number `-30`, i.e. `u64::MAX - 29 = 0xFFFF_FFFF_FFFF_FFE2`, @@ -214,7 +439,11 @@ impl TryFrom for SyscallNumbers { 64 => Ok(SyscallNumbers::Commit), 93 => Ok(SyscallNumbers::Halt), v if v == KECCAK_SYSCALL_NUMBER => Ok(SyscallNumbers::KeccakPermute), + v if v == KECCAK256_SYSCALL_NUMBER => Ok(SyscallNumbers::Keccak256), + v if v == BLAKE3_SYSCALL_NUMBER => Ok(SyscallNumbers::Blake3Compress), + v if v == BLAKE3_ABSORB_SYSCALL_NUMBER => Ok(SyscallNumbers::Blake3Absorb), v if v == ECSM_SYSCALL_NUMBER => Ok(SyscallNumbers::Ecsm), + v if v == ECREC_SYSCALL_NUMBER => Ok(SyscallNumbers::Ecrec), v if v == DMA_MEMCPY_SYSCALL_NUMBER => Ok(SyscallNumbers::DmaMemcpy), v if v == DMA_MEMSET_SYSCALL_NUMBER => Ok(SyscallNumbers::DmaMemset), v if v == HINT_SYSCALL_NUMBER => Ok(SyscallNumbers::Hint), @@ -227,7 +456,12 @@ impl TryFrom for SyscallNumbers { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Accelerator { Keccak, + /// Keccak-256 of a buffer: the KECCAK chip again, `keccak256_blocks(len)` + /// rows a call rather than one. + Keccak256, + Blake3, Ecsm, + Ecrec, Dma, } @@ -237,7 +471,11 @@ impl SyscallNumbers { pub fn raw(self) -> u64 { match self { SyscallNumbers::KeccakPermute => KECCAK_SYSCALL_NUMBER, + SyscallNumbers::Keccak256 => KECCAK256_SYSCALL_NUMBER, + SyscallNumbers::Blake3Compress => BLAKE3_SYSCALL_NUMBER, + SyscallNumbers::Blake3Absorb => BLAKE3_ABSORB_SYSCALL_NUMBER, SyscallNumbers::Ecsm => ECSM_SYSCALL_NUMBER, + SyscallNumbers::Ecrec => ECREC_SYSCALL_NUMBER, SyscallNumbers::DmaMemcpy => DMA_MEMCPY_SYSCALL_NUMBER, SyscallNumbers::DmaMemset => DMA_MEMSET_SYSCALL_NUMBER, SyscallNumbers::Hint => HINT_SYSCALL_NUMBER, @@ -254,7 +492,16 @@ impl SyscallNumbers { pub fn accelerator(self) -> Option { match self { SyscallNumbers::KeccakPermute => Some(Accelerator::Keccak), + // The same chip, one row per permutation of the call; counters read + // the permutation count from the log (`src2_val`). + SyscallNumbers::Keccak256 => Some(Accelerator::Keccak256), + SyscallNumbers::Blake3Compress => Some(Accelerator::Blake3), + // Both BLAKE3 syscalls drive the same chip, so they count as one + // accelerator — a counter that split them would report "BLAKE3 + // calls" that no longer track BLAKE3 rows. + SyscallNumbers::Blake3Absorb => Some(Accelerator::Blake3), SyscallNumbers::Ecsm => Some(Accelerator::Ecsm), + SyscallNumbers::Ecrec => Some(Accelerator::Ecrec), SyscallNumbers::DmaMemcpy | SyscallNumbers::DmaMemset => Some(Accelerator::Dma), SyscallNumbers::Print | SyscallNumbers::Panic @@ -672,6 +919,165 @@ impl Instruction { } src2_val = state_addr; } + SyscallNumbers::Keccak256 => { + // Keccak-256 of [x10, x10 + x11) written to [x12, x12 + 32). + // See KECCAK256_SYSCALL_NUMBER for the ABI. + let ptr = registers.read(10)?; + let len = registers.read(11)?; + let out = registers.read(12)?; + if !keccak256_addresses_ok(ptr, len, out) { + return Err(ExecutionError::Keccak256AddressOverflow { ptr, len, out }); + } + let input = memory.load_bytes(ptr, len)?; + let digest = keccak256_digest(&input); + for (k, chunk) in digest.chunks_exact(8).enumerate() { + let lane = u64::from_le_bytes(chunk.try_into().expect("8-byte chunk")); + memory.store_doubleword(out + 8 * k as u64, lane)?; + } + // The permutation count (KECCAK rows) for the counters, and + // the input address (its alignment decides which memory + // table the chip's reads land in). Inert for the CPU row: + // `EcallEbreak` writes no register, and the trace builder + // recovers x10..x12 from the register state, like ECSM and + // HINT. + src2_val = keccak256_blocks(len); + dst_val = ptr; + } + SyscallNumbers::Blake3Compress => { + // BLAKE3 6-round compression on a 176-byte region at the + // address in x10 (layout: see BLAKE3_SYSCALL_NUMBER docs). + let state_addr = registers.read(10)?; + if !state_addr.is_multiple_of(8) { + return Err(ExecutionError::UnalignedBlake3StateAddress(state_addr)); + } + state_addr + .checked_add(BLAKE3_STATE_BYTES - 1) + .ok_or(ExecutionError::Blake3StateAddressOverflow(state_addr))?; + + // Input: 14 dwords = h[8] | m[16] | t | (block_len, flags), + // each dword two little-endian u32 words. + let mut words = [0u32; 28]; + for k in 0..14 { + let dw = memory.load_doubleword(state_addr + (k as u64) * 8)?; + words[2 * k] = dw as u32; + words[2 * k + 1] = (dw >> 32) as u32; + } + let h: [u32; 8] = words[0..8].try_into().unwrap(); + let m: [u32; 16] = words[8..24].try_into().unwrap(); + let t = (words[24] as u64) | ((words[25] as u64) << 32); + let block_len = words[26]; + let flags = words[27]; + + let out = blake3_compress_6round(&h, &m, t, block_len, flags); + for k in 0..8 { + let dw = (out[2 * k] as u64) | ((out[2 * k + 1] as u64) << 32); + memory.store_doubleword( + state_addr + (BLAKE3_OUT_DWORDS + k as u64) * 8, + dw, + )?; + } + src2_val = state_addr; + } + SyscallNumbers::Blake3Absorb => { + // Chained absorb: fold `num_blocks` consecutive 64-byte + // blocks into the chaining value in one ecall. Layout and + // register roles: see BLAKE3_ABSORB_SYSCALL_NUMBER. + let ctrl_addr = registers.read(10)?; + let msg_addr = registers.read(11)?; + let num_blocks = registers.read(12)?; + let first_flags = registers.read(13)?; + + if !ctrl_addr.is_multiple_of(8) { + return Err(ExecutionError::UnalignedBlake3AbsorbAddress(ctrl_addr)); + } + if !msg_addr.is_multiple_of(8) { + return Err(ExecutionError::UnalignedBlake3AbsorbAddress(msg_addr)); + } + if num_blocks == 0 || num_blocks > BLAKE3_ABSORB_MAX_BLOCKS { + return Err(ExecutionError::Blake3AbsorbBlockCountOutOfRange( + num_blocks, + )); + } + // The chip's flags column is one 32-bit word. + if first_flags > u32::MAX as u64 { + return Err(ExecutionError::Blake3AbsorbFlagsOutOfRange(first_flags)); + } + + let ctrl_bytes = BLAKE3_ABSORB_CTRL_DWORDS * 8; + // Unreachable today — the cap above bounds this by + // 2^16 · 64 = 4 MiB — and kept anyway, so that raising + // BLAKE3_ABSORB_MAX_BLOCKS cannot silently turn a + // multiply into a wrap. The tests reach the address + // overflow below, not this arm. + let msg_bytes = num_blocks + .checked_mul(BLAKE3_BLOCK_BYTES) + .ok_or(ExecutionError::Blake3AbsorbAddressOverflow(msg_addr))?; + // The LAST addressable byte of each region, kept rather + // than discarded: the overlap test below is written on + // these, so it cannot overflow on a guest-chosen address. + // (`end = addr + bytes` would: a region ending exactly at + // 2^64 wraps to 0, which panics in a debug build and in + // release silently reports "no overlap" for regions that + // do overlap.) + let ctrl_last = ctrl_addr + .checked_add(ctrl_bytes - 1) + .ok_or(ExecutionError::Blake3AbsorbAddressOverflow(ctrl_addr))?; + let msg_last = msg_addr + .checked_add(msg_bytes - 1) + .ok_or(ExecutionError::Blake3AbsorbAddressOverflow(msg_addr))?; + + // Every access of one absorb carries the same timestamp, + // so an address touched by both regions would be accessed + // twice at that timestamp and the MEMW consistency + // argument could not order the pair. Reject rather than + // emit a trace the prover cannot close. + if ctrl_addr <= msg_last && msg_addr <= ctrl_last { + return Err(ExecutionError::Blake3AbsorbRegionOverlap); + } + + // cv_in: dwords 0..4 of the control region. + let mut cv = [0u32; 8]; + for k in 0..4u64 { + let dw = memory.load_doubleword(ctrl_addr + k * 8)?; + cv[2 * k as usize] = dw as u32; + cv[2 * k as usize + 1] = (dw >> 32) as u32; + } + + for i in 0..num_blocks { + let block_addr = msg_addr + i * BLAKE3_BLOCK_BYTES; + let mut m = [0u32; 16]; + for k in 0..8u64 { + let dw = memory.load_doubleword(block_addr + k * 8)?; + m[2 * k as usize] = dw as u32; + m[2 * k as usize + 1] = (dw >> 32) as u32; + } + cv = blake3_absorb_step_6round( + &cv, + &m, + if i == 0 { first_flags as u32 } else { 0 }, + ); + } + + // cv_out: dwords 4..8 of the control region. + for k in 0..4u64 { + let dw = (cv[2 * k as usize] as u64) + | ((cv[2 * k as usize + 1] as u64) << 32); + memory.store_doubleword( + ctrl_addr + (BLAKE3_ABSORB_CV_OUT_DWORD + k) * 8, + dw, + )?; + } + + // Logged for the CPU row the way the other accelerators + // log theirs. Both are inert today: `EcallEbreak`'s + // decode writes no register, and `CpuOperation::from_log` + // reads `dst_val` only under the Commit classification. + // They are here for the absorb classification the chip + // will add, which — like ECSM and HINT — will recover + // x10..x13 from the register state rather than the log. + src2_val = ctrl_addr; + dst_val = msg_addr; + } SyscallNumbers::Ecsm => { // ECSM(-11): k×G on secp256k1. // x10 = addr to write xR, x11 = addr of xG, x12 = addr of k. @@ -705,6 +1111,64 @@ impl Instruction { src2_val = addr_xg; dst_val = addr_k; } + SyscallNumbers::Ecrec => { + // ECREC(-12): ECDSA public-key recovery on secp256k1. + // x10 = addr to write the key (Q.x, Q.y, 32 bytes big-endian each) + // and the status doubleword; x11 = addr of the signature r ‖ s + // (64 bytes, big-endian each); x12 = addr of the message hash + // (32 bytes big-endian); x13 = the parity of R's y (0 or 1). + // Status 0: the key. Status 1: the call bailed (the key bytes are + // zero) and the guest recovers the key itself; see + // `ecsm::ecrec` for exactly when. + let addr_out = registers.read(10)?; + let addr_sig = registers.read(11)?; + let addr_msg = registers.read(12)?; + let parity = registers.read(13)?; + if !addr_limb_ok(addr_out, ECREC_OUT_BYTES - 1) + || !addr_limb_ok(addr_sig, 63) + || !addr_limb_ok(addr_msg, 31) + { + return Err(ExecutionError::EcrecAddressOverflow); + } + // The signature is read at T and the hash at T + 1, as unaligned + // doubleword MEMW accesses: the two regions must be disjoint, for + // the same trace-provability reason as ECSM's operands. + if addr_sig < addr_msg.wrapping_add(32) + && addr_msg < addr_sig.wrapping_add(64) + { + return Err(ExecutionError::EcrecOperandOverlap); + } + if parity > 1 { + return Err(ExecutionError::EcrecParityOutOfRange(parity)); + } + let mut sig = [0u8; 64]; + for i in 0..8u64 { + let dw = memory.load_doubleword(addr_sig + 8 * i)?; + sig[8 * i as usize..8 * i as usize + 8] + .copy_from_slice(&dw.to_le_bytes()); + } + let mut msg = [0u8; 32]; + for i in 0..4u64 { + let dw = memory.load_doubleword(addr_msg + 8 * i)?; + msg[8 * i as usize..8 * i as usize + 8] + .copy_from_slice(&dw.to_le_bytes()); + } + let mut out = [0u8; ECREC_OUT_BYTES as usize]; + match ecsm::ecrecover(&sig, &msg, parity as u8) { + Some((x, y)) => { + out[..32].copy_from_slice(&x); + out[32..64].copy_from_slice(&y); + } + None => out[64] = ecsm::ECREC_STATUS_BAIL, + } + for i in 0..9u64 { + let mut dw = [0u8; 8]; + dw.copy_from_slice(&out[8 * i as usize..8 * i as usize + 8]); + memory.store_doubleword(addr_out + 8 * i, u64::from_le_bytes(dw))?; + } + src2_val = addr_sig; + dst_val = addr_msg; + } SyscallNumbers::DmaMemcpy => { // memcpy(dst = x10, src = x11, n = x12). Snapshot the input // before writing, which also gives this ecall well-defined @@ -722,11 +1186,20 @@ impl Instruction { // The fixed-size scratch avoids a heap allocation on every // hot-path ecall while preserving snapshot semantics. let mut bytes = [0u8; DMA_MEMCPY_MAX_BYTES as usize]; - for (i, byte) in bytes[..n as usize].iter_mut().enumerate() { - *byte = memory.load_byte(src + i as u64); - } - for (i, &byte) in bytes[..n as usize].iter().enumerate() { - memory.store_byte(dst + i as u64, byte); + let bytes = &mut bytes[..n as usize]; + if memory.paths().dma_bulk { + // The whole source, then the whole destination: the + // bytes and the words marked written are the byte + // loop's below. + memory.read_range(src, bytes); + memory.write_range(dst, bytes); + } else { + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = memory.load_byte(src + i as u64); + } + for (i, &byte) in bytes.iter().enumerate() { + memory.store_byte(dst + i as u64, byte); + } } src2_val = copy_trace_rows(src, dst, n, false); dst_val = n; @@ -758,9 +1231,26 @@ impl Instruction { return Err(ExecutionError::DmaMemsetBadGap { src, dst }); } - for i in 0..n { - let byte = memory.load_byte(src + i); - memory.store_byte(dst + i, byte); + if memory.paths().dma_bulk { + // With `dst = src + 8` the forward byte walk below + // reads, at step `i >= 8`, the byte it wrote at step + // `i - 8`: the destination is the first + // `min(n, 8)` source bytes repeated. Those are read + // before any write, then the range is written once. + let mut seed = [0u8; DMA_MEMSET_GAP as usize]; + let seed_len = n.min(DMA_MEMSET_GAP) as usize; + memory.read_range(src, &mut seed[..seed_len]); + let mut bytes = [0u8; DMA_MEMCPY_MAX_BYTES as usize]; + let bytes = &mut bytes[..n as usize]; + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = seed[i % DMA_MEMSET_GAP as usize]; + } + memory.write_range(dst, bytes); + } else { + for i in 0..n { + let byte = memory.load_byte(src + i); + memory.store_byte(dst + i, byte); + } } src2_val = copy_trace_rows(src, dst, n, false); dst_val = n; @@ -977,10 +1467,34 @@ pub enum ExecutionError { UnalignedKeccakStateAddress(u64), #[error("Keccak state address range overflows: {0:#018x}")] KeccakStateAddressOverflow(u64), + #[error( + "Keccak-256 operands cross a 2^32 address-word boundary: ptr {ptr:#x}, len {len}, out {out:#x}" + )] + Keccak256AddressOverflow { ptr: u64, len: u64, out: u64 }, + #[error("Unaligned BLAKE3 state address: {0:#018x}")] + UnalignedBlake3StateAddress(u64), + #[error("BLAKE3 state address range overflows: {0:#018x}")] + Blake3StateAddressOverflow(u64), + #[error("Unaligned BLAKE3 absorb address: {0:#018x}")] + UnalignedBlake3AbsorbAddress(u64), + #[error("BLAKE3 absorb address range overflows: {0:#018x}")] + Blake3AbsorbAddressOverflow(u64), + #[error("BLAKE3 absorb block count out of range: {0}")] + Blake3AbsorbBlockCountOutOfRange(u64), + #[error("BLAKE3 absorb first_flags exceeds 32 bits: {0:#018x}")] + Blake3AbsorbFlagsOutOfRange(u64), + #[error("BLAKE3 absorb control and message regions overlap")] + Blake3AbsorbRegionOverlap, #[error("ECSM address range overflows the lower 32-bit limb")] EcsmAddressOverflow, #[error("ECSM xG and k operand ranges overlap")] EcsmOperandOverlap, + #[error("ECREC address range overflows the lower 32-bit limb")] + EcrecAddressOverflow, + #[error("ECREC signature and message hash ranges overlap")] + EcrecOperandOverlap, + #[error("ECREC parity must be 0 or 1, got {0}")] + EcrecParityOutOfRange(u64), #[error("DMA chunk has {0} bytes; maximum per ecall is {DMA_MEMCPY_MAX_BYTES}")] DmaChunkTooLarge(u64), #[error("DMA memset needs dst == src + {DMA_MEMSET_GAP}; got src {src:#x}, dst {dst:#x}")] @@ -1074,3 +1588,148 @@ pub fn keccak_f1600(state: &mut [u64; 25]) { state[0] ^= rc; } } + +// ============================================================================= +// BLAKE3 6-round compression (internal variant) +// ============================================================================= +// +// A Rust port of the validated oracle `tooling/blake3-second-source/blake3-oracle/blake3_ref.py` +// with `rounds = 6` fixed. This is the **6-round internal variant** — NOT +// standard BLAKE3 (7 rounds); its security rests on the named 6-round +// assumption recorded in `thoughts/blake3/blake3-chip/DESIGN.md`. Differentially +// tested against the oracle's canonical 6-round vectors (pinned in +// `tooling/blake3-second-source/blake3-oracle/canonical_6round_vectors.json`, themselves +// validated against the official `blake3` crate). + +/// The BLAKE3 IV (identical to SHA-256's initial state). `IV[0..4]` seeds +/// `v[8..12]` of the compression working state. +pub const BLAKE3_IV: [u32; 8] = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +]; + +/// The BLAKE3 message-schedule permutation, applied between rounds +/// (`m'[i] = m[MSG_PERMUTATION[i]]`). +pub const BLAKE3_MSG_PERMUTATION: [usize; 16] = + [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8]; + +/// Rounds of the internal variant. 6, per the design; standard BLAKE3 is 7. +pub const BLAKE3_ROUNDS: usize = 6; + +/// The BLAKE3 quarter-round G (spec §2.1). +#[inline] +fn blake3_g(v: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, mx: u32, my: u32) { + v[a] = v[a].wrapping_add(v[b]).wrapping_add(mx); + v[d] = (v[d] ^ v[a]).rotate_right(16); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(12); + v[a] = v[a].wrapping_add(v[b]).wrapping_add(my); + v[d] = (v[d] ^ v[a]).rotate_right(8); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(7); +} + +/// The BLAKE3 compression function `f` at 6 rounds (spec §2.2, oracle §2.4). +/// +/// State init: `v[0..8] = h`, `v[8..12] = IV[0..4]`, `v[12] = t as u32`, +/// `v[13] = (t >> 32) as u32`, `v[14] = block_len`, `v[15] = flags`. Six +/// rounds of 8 G-calls (4 columns then 4 diagonals), permuting the message +/// schedule between rounds (`r < rounds - 1`, i.e. 5 permutes — the trailing +/// permute is never consumed). Feed-forward: `out[i] = v[i] ^ v[i+8]`, +/// `out[i+8] = v[i+8] ^ h[i]`. The truncated chaining value is `out[0..8]`. +pub fn blake3_compress_6round( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) -> [u32; 16] { + let mut v: [u32; 16] = [ + h[0], + h[1], + h[2], + h[3], + h[4], + h[5], + h[6], + h[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + t as u32, + (t >> 32) as u32, + block_len, + flags, + ]; + + let mut m = *m; + for r in 0..BLAKE3_ROUNDS { + // Mix the columns. + blake3_g(&mut v, 0, 4, 8, 12, m[0], m[1]); + blake3_g(&mut v, 1, 5, 9, 13, m[2], m[3]); + blake3_g(&mut v, 2, 6, 10, 14, m[4], m[5]); + blake3_g(&mut v, 3, 7, 11, 15, m[6], m[7]); + // Mix the diagonals. + blake3_g(&mut v, 0, 5, 10, 15, m[8], m[9]); + blake3_g(&mut v, 1, 6, 11, 12, m[10], m[11]); + blake3_g(&mut v, 2, 7, 8, 13, m[12], m[13]); + blake3_g(&mut v, 3, 4, 9, 14, m[14], m[15]); + // Permute between rounds; the permute after the last round is never + // consumed (oracle: `r < rounds - 1`). + if r < BLAKE3_ROUNDS - 1 { + let prev = m; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + m[i] = prev[p]; + } + } + } + + let mut out = [0u32; 16]; + for i in 0..8 { + out[i] = v[i] ^ v[i + 8]; + out[i + 8] = v[i + 8] ^ h[i]; + } + out +} + +/// One block of a chained absorb: compress `m` into `cv` under the absorb +/// schedule and return the truncated chaining value. +/// +/// The schedule — `t = 0`, `block_len = 64` — is the absorb ABI's, applied to +/// every block of a run. It is the interior shape of `crypto`'s `Blake3Chain` +/// framing, restated here because `executor` does not depend on `crypto`; +/// [`blake3_absorb_chain_6round`] is where that restatement is gated. +pub fn blake3_absorb_step_6round(cv: &[u32; 8], m: &[u32; 16], flags: u32) -> [u32; 8] { + let out = blake3_compress_6round(cv, m, 0, BLAKE3_BLOCK_BYTES as u32, flags); + let mut next = [0u32; 8]; + next.copy_from_slice(&out[..8]); + next +} + +/// ★ The absorb ecall's semantics, as a pure function over whole blocks. +/// +/// `first_flags` lands on block 0 and every later block carries `0`. The syscall +/// handler is this loop reading its blocks out of VM memory, and the prover's +/// trace builder is this loop again over one row per block — so this is the +/// single object all three agree on, and the one a parity test can drive. +/// +/// **This is a second framing and is treated as one.** `crypto`'s `Blake3Chain` +/// owns the flag schedule (`block_flags`: `CHUNK_START` on the first block, +/// `CHUNK_END | ROOT` and the true byte count on the last). This function +/// restates the *interior* of that schedule — full blocks, no flags after the +/// first — because `executor` cannot call `crypto`. What makes the restatement +/// safe is not that it looks right: it is that a guest's `Blake3Chain` drives +/// this for every block but its last, and the parity tests check the composite +/// against `Blake3Chain` over exhaustive message lengths. A drift in either the +/// flags or the `block_len` moves the digest and fails those tests. +pub fn blake3_absorb_chain_6round( + cv_in: &[u32; 8], + blocks: &[[u32; 16]], + first_flags: u32, +) -> [u32; 8] { + let mut cv = *cv_in; + for (i, m) in blocks.iter().enumerate() { + cv = blake3_absorb_step_6round(&cv, m, if i == 0 { first_flags } else { 0 }); + } + cv +} diff --git a/executor/src/vm/logs.rs b/executor/src/vm/logs.rs index 65ee268c0..87a3de91d 100644 --- a/executor/src/vm/logs.rs +++ b/executor/src/vm/logs.rs @@ -21,7 +21,11 @@ /// executor evaluates where `src`, `dst` and `count` are all known. Today a row's /// width follows from the remaining count alone (eight bytes, or one for the last /// seven), so the CLI report just sums the carried counts. +/// +/// `#[repr(C)]`: five `u64`s in this order and no padding, so a window's logs +/// can cross to a card as they lie (the prover's card fill of the CPU table). #[derive(Debug, Clone)] +#[repr(C)] pub struct Log { /// PC before instruction execution (use this to look up the instruction) pub current_pc: u64, @@ -37,3 +41,13 @@ pub struct Log { /// For ECALL: see the per-syscall table above. pub dst_val: u64, } + +const _: () = { + assert!(std::mem::size_of::() == 40); + assert!(std::mem::align_of::() == 8); + assert!(std::mem::offset_of!(Log, current_pc) == 0); + assert!(std::mem::offset_of!(Log, next_pc) == 8); + assert!(std::mem::offset_of!(Log, src1_val) == 16); + assert!(std::mem::offset_of!(Log, src2_val) == 24); + assert!(std::mem::offset_of!(Log, dst_val) == 32); +}; diff --git a/executor/src/vm/memory.rs b/executor/src/vm/memory.rs index b5e6d72cf..c91b7a79c 100644 --- a/executor/src/vm/memory.rs +++ b/executor/src/vm/memory.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::hash::{BuildHasher, Hasher}; +use std::sync::Arc; /// Fast hasher for u64 keys - uses the key directly as the hash value. /// This avoids the overhead of SipHash for integer keys. @@ -56,148 +57,508 @@ pub const PRIVATE_INPUT_START_INDEX: u64 = 0xFF000000; /// for every page-span computation over the private-input region. pub const PRIVATE_INPUT_LENGTH_PREFIX_BYTES: usize = size_of::(); -#[derive(Default, Debug, Clone)] +/// Guest memory: every byte of the 64-bit address space, zero until written. +/// +/// It keeps the bytes in 64 KiB pages ([`Pages`]): an access finds its page by +/// indexing a directory, with no hashing, so its cost does not grow with the +/// memory a run touches. `LAMBDA_VM_EXEC_MEMORY=words` keeps them in the map of +/// 4-byte words the executor used before ([`Store::Words`]), the control the +/// pages are measured and tested against. Both answer every access alike. +#[derive(Debug, Clone)] pub struct Memory { - cells: U64HashMap<[u8; 4]>, + store: Store, /// Bytes committed to public output via `commit_public_output`. The /// COPY chip's commit rows don't write to a fixed memory region (it streams bytes /// onto the Commit bus by `index`), so this buffer is purely the /// executor's view used by `read_return_value` and CLI display. public_output: Vec, + /// Which of the executor's access paths this memory takes. + paths: ExecPaths, +} + +impl Default for Memory { + fn default() -> Self { + Self::with_store(StoreKind::from_env()) + } +} + +/// The executor's access paths, each its own knob. Every setting reads and +/// writes the same bytes and marks the same written words; only the work per +/// access differs, so the executor's logs are the same under every setting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ExecPaths { + /// A load or store of 1, 2, 4 or 8 bytes that stays inside one page reads + /// or writes the page directly, as a fixed-width copy after one page + /// lookup (`LAMBDA_VM_EXEC_FIXED_ACCESS`, on unless `0`). Off, every + /// access goes through the general byte-range loop. + pub fixed_access: bool, + /// The DMA memcpy and memset ecalls read and write their range in one + /// call each (`LAMBDA_VM_EXEC_DMA_BULK`, on unless `0`). Off, they move it + /// a byte at a time. + pub dma_bulk: bool, +} + +impl ExecPaths { + /// Every access through the general path: the executor before the knobs. + pub const GENERAL: Self = Self { + fixed_access: false, + dma_bulk: false, + }; + /// Every fast path on: the default. + pub const FAST: Self = Self { + fixed_access: true, + dma_bulk: true, + }; + + /// The two knobs, read once: each is on unless set to `0`. + pub fn from_env() -> Self { + static PATHS: std::sync::OnceLock = std::sync::OnceLock::new(); + *PATHS.get_or_init(|| { + Self::from_values( + std::env::var("LAMBDA_VM_EXEC_FIXED_ACCESS").ok().as_deref(), + std::env::var("LAMBDA_VM_EXEC_DMA_BULK").ok().as_deref(), + ) + }) + } + + /// [`ExecPaths::from_env`] of the knobs' values. + pub fn from_values(fixed_access: Option<&str>, dma_bulk: Option<&str>) -> Self { + let on = |value: Option<&str>| value.map(str::trim) != Some("0"); + Self { + fixed_access: on(fixed_access), + dma_bulk: on(dma_bulk), + } + } +} + +/// Which store a [`Memory`] keeps its bytes in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoreKind { + /// 64 KiB pages found through a directory ([`Pages`]). + Pages, + /// A map of 4-byte words keyed by their address. + Words, +} + +impl StoreKind { + /// `LAMBDA_VM_EXEC_MEMORY=words` picks [`StoreKind::Words`]; unset or + /// anything else, [`StoreKind::Pages`]. Read once. + pub fn from_env() -> Self { + static KIND: std::sync::OnceLock = std::sync::OnceLock::new(); + *KIND.get_or_init(|| match std::env::var("LAMBDA_VM_EXEC_MEMORY").as_deref() { + Ok("words") => StoreKind::Words, + _ => StoreKind::Pages, + }) + } +} + +#[derive(Debug, Clone)] +enum Store { + Pages(Pages), + Words(U64HashMap<[u8; 4]>), +} + +const PAGE_BITS: u32 = 16; +const PAGE_SIZE: usize = 1 << PAGE_BITS; +const PAGE_MASK: u64 = PAGE_SIZE as u64 - 1; +/// The highest page number. +const LAST_PAGE: u64 = u64::MAX >> PAGE_BITS; +/// Pages each directory indexes: the low 8 GiB (code, data, heap, private +/// input) and the top 8 GiB (the stack, which starts at `STACK_TOP`). +const DIRECTORY_PAGES: u64 = 1 << (33 - PAGE_BITS); + +/// One page's bytes and, per 4-byte word, whether it was written. +#[derive(Clone)] +struct Page { + bytes: Box<[u8]>, + written: Box<[u64]>, +} + +impl Page { + fn new() -> Self { + Self { + bytes: vec![0u8; PAGE_SIZE].into_boxed_slice(), + written: vec![0u64; PAGE_SIZE / 4 / 64].into_boxed_slice(), + } + } + + /// Writes `data` at `offset` (inside the page) and marks its words. + #[inline] + fn write(&mut self, offset: usize, data: &[u8]) { + self.bytes[offset..offset + data.len()].copy_from_slice(data); + for word in offset / 4..=(offset + data.len() - 1) / 4 { + self.written[word / 64] |= 1 << (word % 64); + } + } +} + +/// The pages a run touched. Page `n` sits in `low[n]` when `n` is below +/// [`DIRECTORY_PAGES`], in `high[LAST_PAGE - n]` when it is among the top +/// [`DIRECTORY_PAGES`], and in `other` otherwise; the directories grow to the +/// furthest page touched. +/// +/// A page is shared behind an [`Arc`], so a clone of the directories (a +/// [`MemorySnapshot`]) shares every page, and a write copies a page that a +/// snapshot still holds before changing it ([`Arc::make_mut`]): a snapshot +/// keeps reading the bytes of its moment. +#[derive(Clone, Default)] +struct Pages { + low: Vec>>, + high: Vec>>, + other: std::collections::BTreeMap>, +} + +impl std::fmt::Debug for Pages { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let count = |d: &[Option>]| d.iter().filter(|p| p.is_some()).count(); + f.debug_struct("Pages") + .field("low", &count(&self.low)) + .field("high", &count(&self.high)) + .field("other", &self.other.len()) + .finish() + } +} + +impl Pages { + /// The bytes the pages and the directories take on the heap. + fn heap_bytes(&self) -> usize { + let page = std::mem::size_of::() + PAGE_SIZE + PAGE_SIZE / 4 / 64 * 8; + let pages = self + .low + .iter() + .chain(&self.high) + .filter(|p| p.is_some()) + .count() + + self.other.len(); + (self.low.capacity() + self.high.capacity()) * std::mem::size_of::>>() + + pages * page + } + + #[inline] + fn page(&self, number: u64) -> Option<&Page> { + if number < DIRECTORY_PAGES { + self.low.get(number as usize)?.as_deref() + } else if LAST_PAGE - number < DIRECTORY_PAGES { + self.high.get((LAST_PAGE - number) as usize)?.as_deref() + } else { + self.other.get(&number).map(|page| &**page) + } + } + + /// Page `number` to write, allocated on first touch, and copied first if + /// a snapshot shares it. + #[inline] + fn page_mut(&mut self, number: u64) -> &mut Page { + fn slot(directory: &mut Vec>>, index: usize) -> &mut Page { + if directory.len() <= index { + directory.resize_with(index + 1, || None); + } + Arc::make_mut(directory[index].get_or_insert_with(|| Arc::new(Page::new()))) + } + if number < DIRECTORY_PAGES { + slot(&mut self.low, number as usize) + } else if LAST_PAGE - number < DIRECTORY_PAGES { + slot(&mut self.high, (LAST_PAGE - number) as usize) + } else { + Arc::make_mut( + self.other + .entry(number) + .or_insert_with(|| Arc::new(Page::new())), + ) + } + } + + /// The bytes of the pages this store holds that `now` no longer shares: + /// what keeping this store as a snapshot costs once `now` has written them. + fn bytes_apart_from(&self, now: &Pages) -> usize { + let page = std::mem::size_of::() + PAGE_SIZE + PAGE_SIZE / 4 / 64 * 8; + let apart = |mine: &Arc, theirs: Option<&Arc>| { + theirs.is_none_or(|theirs| !Arc::ptr_eq(mine, theirs)) + }; + let directory = |mine: &[Option>], theirs: &[Option>]| { + mine.iter() + .enumerate() + .filter(|(i, p)| { + p.as_ref() + .is_some_and(|p| apart(p, theirs.get(*i).and_then(Option::as_ref))) + }) + .count() + }; + let other = self + .other + .iter() + .filter(|(n, p)| apart(p, now.other.get(n))) + .count(); + (directory(&self.low, &now.low) + directory(&self.high, &now.high) + other) * page + } + + /// The bytes at `address..` into `out`; the range must not wrap. + #[inline] + fn read(&self, address: u64, out: &mut [u8]) { + let mut done = 0; + while done < out.len() { + let at = address + done as u64; + let offset = (at & PAGE_MASK) as usize; + let n = (PAGE_SIZE - offset).min(out.len() - done); + match self.page(at >> PAGE_BITS) { + Some(page) => out[done..done + n].copy_from_slice(&page.bytes[offset..offset + n]), + None => out[done..done + n].fill(0), + } + done += n; + } + } + + /// `data` at `address..`; the range must not wrap. + #[inline] + fn write(&mut self, address: u64, data: &[u8]) { + let mut done = 0; + while done < data.len() { + let at = address + done as u64; + let offset = (at & PAGE_MASK) as usize; + let n = (PAGE_SIZE - offset).min(data.len() - done); + self.page_mut(at >> PAGE_BITS) + .write(offset, &data[done..done + n]); + done += n; + } + } + + /// Every byte of every written word, page by page. + fn iter_bytes(&self) -> impl Iterator + '_ { + let low = self + .low + .iter() + .enumerate() + .filter_map(|(n, page)| Some((n as u64, page.as_deref()?))); + let other = self.other.iter().map(|(&n, page)| (n, &**page)); + let high = self + .high + .iter() + .enumerate() + .filter_map(|(i, page)| Some((LAST_PAGE - i as u64, page.as_deref()?))); + low.chain(other).chain(high).flat_map(|(number, page)| { + let base = number << PAGE_BITS; + (0..PAGE_SIZE / 4) + .filter(move |&word| page.written[word / 64] >> (word % 64) & 1 == 1) + .flat_map(move |word| { + (0..4).map(move |i| (base + (4 * word + i) as u64, page.bytes[4 * word + i])) + }) + }) + } +} + +impl Store { + /// The bytes at `address..` into `out`; the range must not wrap. + #[inline] + fn read(&self, address: u64, out: &mut [u8]) { + match self { + Store::Pages(pages) => pages.read(address, out), + Store::Words(cells) => { + let mut done = 0; + while done < out.len() { + let at = address + done as u64; + let offset = (at % 4) as usize; + let n = (4 - offset).min(out.len() - done); + let word = cells + .get(&(at - offset as u64)) + .copied() + .unwrap_or_default(); + out[done..done + n].copy_from_slice(&word[offset..offset + n]); + done += n; + } + } + } + } + + /// `data` at `address..`; the range must not wrap. + #[inline] + fn write(&mut self, address: u64, data: &[u8]) { + match self { + Store::Pages(pages) => pages.write(address, data), + Store::Words(cells) => { + let mut done = 0; + while done < data.len() { + let at = address + done as u64; + let offset = (at % 4) as usize; + let n = (4 - offset).min(data.len() - done); + let word = cells.entry(at - offset as u64).or_insert([0; 4]); + word[offset..offset + n].copy_from_slice(&data[done..done + n]); + done += n; + } + } + } + } } impl Memory { + /// An empty memory over the given store, on the paths the knobs set + /// ([`ExecPaths::from_env`]). + pub fn with_store(kind: StoreKind) -> Self { + Self::with_store_and_paths(kind, ExecPaths::from_env()) + } + + /// An empty memory over the given store, on the given paths. + pub fn with_store_and_paths(kind: StoreKind, paths: ExecPaths) -> Self { + Self { + store: match kind { + StoreKind::Pages => Store::Pages(Pages::default()), + StoreKind::Words => Store::Words(U64HashMap::default()), + }, + public_output: Vec::new(), + paths, + } + } + + /// The access paths this memory takes. + pub fn paths(&self) -> ExecPaths { + self.paths + } + + /// `N` bytes at `address..`; the range must not wrap. With + /// [`ExecPaths::fixed_access`] and the pages store, an access inside one + /// page is one lookup and a fixed-width copy; any other access takes the + /// general loop, which reads the same bytes. + #[inline(always)] + fn read_n(&self, address: u64) -> [u8; N] { + if self.paths.fixed_access + && let Store::Pages(pages) = &self.store + { + let offset = (address & PAGE_MASK) as usize; + if offset + N <= PAGE_SIZE { + return match pages.page(address >> PAGE_BITS) { + Some(page) => { + let mut out = [0u8; N]; + out.copy_from_slice(&page.bytes[offset..offset + N]); + out + } + None => [0; N], + }; + } + } + let mut out = [0u8; N]; + self.store.read(address, &mut out); + out + } + + /// `data` at `address..`; the range must not wrap. The fixed-width + /// counterpart of [`Memory::read_n`]: the same bytes written and the same + /// words marked written as the general loop. + #[inline(always)] + fn write_n(&mut self, address: u64, data: [u8; N]) { + if self.paths.fixed_access + && let Store::Pages(pages) = &mut self.store + { + let offset = (address & PAGE_MASK) as usize; + if offset + N <= PAGE_SIZE { + pages.page_mut(address >> PAGE_BITS).write(offset, &data); + return; + } + } + self.store.write(address, &data); + } + + /// The bytes at `address..` into `out`, in one pass over the pages they + /// span; the range must not wrap. + pub(crate) fn read_range(&self, address: u64, out: &mut [u8]) { + self.store.read(address, out); + } + + /// `data` at `address..`, in one pass over the pages it spans; the range + /// must not wrap. + pub(crate) fn write_range(&mut self, address: u64, data: &[u8]) { + self.store.write(address, data); + } + + /// The bytes the memory takes on the heap: its pages (or the word map's + /// slots, an entry and a control byte each) and the public output. + pub fn heap_bytes(&self) -> usize { + let store = match &self.store { + Store::Pages(pages) => pages.heap_bytes(), + Store::Words(cells) => cells.capacity() * (std::mem::size_of::<(u64, [u8; 4])>() + 1), + }; + store + self.public_output.capacity() + } + + /// The memory as it is now, kept while the run goes on: the pages are + /// shared, and a later write copies a page the snapshot holds before + /// changing it, so the snapshot keeps reading this moment's bytes. Cheap + /// for the pages store (one count per page); the words store copies its + /// map. + pub fn snapshot(&self) -> MemorySnapshot { + MemorySnapshot { + store: self.store.clone(), + } + } + pub fn load_byte(&self, address: u64) -> u8 { - let aligned_address = address - address % 4; - let value = self - .cells - .get(&aligned_address) - .cloned() - .unwrap_or_default(); - value[(address % 4) as usize] + self.read_n::<1>(address)[0] } pub fn store_byte(&mut self, address: u64, value: u8) { - let aligned_address = address - address % 4; - let entry = self - .cells - .entry(aligned_address) - .or_insert_with(|| [0, 0, 0, 0]); - entry[(address % 4) as usize] = value; - } - - /// Iterate over all stored bytes as `(address, value)` pairs. Cells are - /// stored as 4-byte words; each word expands into its four byte addresses. - /// Used to snapshot memory at an epoch boundary. - pub fn iter_bytes(&self) -> impl Iterator + '_ { - self.cells.iter().flat_map(|(&addr, bytes)| { - bytes - .iter() - .enumerate() - .map(move |(i, &b)| (addr + i as u64, b)) - }) + self.write_n::<1>(address, [value]); + } + + /// Iterate over all stored bytes as `(address, value)` pairs. Bytes are + /// stored in 4-byte words; each word ever written yields its four byte + /// addresses. Used to snapshot memory at an epoch boundary. + pub fn iter_bytes(&self) -> Box + '_> { + match &self.store { + Store::Pages(pages) => Box::new(pages.iter_bytes()), + Store::Words(cells) => Box::new(cells.iter().flat_map(|(&addr, bytes)| { + bytes + .iter() + .enumerate() + .map(move |(i, &b)| (addr + i as u64, b)) + })), + } } pub fn load_word(&self, address: u64) -> Result { - if address.is_multiple_of(4) { - let bytes = self.cells.get(&address).cloned().unwrap_or_default(); - Ok(u32::from_le_bytes(bytes)) - } else { + // A 4-aligned word cannot wrap. + if !address.is_multiple_of(4) { address.checked_add(3).ok_or(MemoryError::AddressOverflow)?; - Ok(u32::from_le_bytes([ - self.load_byte(address), - self.load_byte(address + 1), - self.load_byte(address + 2), - self.load_byte(address + 3), - ])) } + Ok(u32::from_le_bytes(self.read_n::<4>(address))) } pub fn store_word(&mut self, address: u64, value: u32) -> Result<(), MemoryError> { - let bytes = value.to_le_bytes(); - if address.is_multiple_of(4) { - self.cells.insert(address, bytes); - } else { + if !address.is_multiple_of(4) { address.checked_add(3).ok_or(MemoryError::AddressOverflow)?; - for (i, b) in bytes.iter().enumerate() { - self.store_byte(address + i as u64, *b); - } } + self.write_n::<4>(address, value.to_le_bytes()); Ok(()) } /// Load a doubleword (64-bit) from memory - for LD instruction pub fn load_doubleword(&self, address: u64) -> Result { - if address.is_multiple_of(8) { - // 8-alignment bounds `address` to `u64::MAX - 7`, so `address + 4` can't overflow. - let low_bytes = self.cells.get(&address).cloned().unwrap_or_default(); - let high_bytes = self.cells.get(&(address + 4)).cloned().unwrap_or_default(); - let low = u32::from_le_bytes(low_bytes) as u64; - let high = u32::from_le_bytes(high_bytes) as u64; - Ok(low | (high << 32)) - } else { + // An 8-aligned doubleword cannot wrap. + if !address.is_multiple_of(8) { address.checked_add(7).ok_or(MemoryError::AddressOverflow)?; - let mut bytes = [0u8; 8]; - for (i, b) in bytes.iter_mut().enumerate() { - *b = self.load_byte(address + i as u64); - } - Ok(u64::from_le_bytes(bytes)) } + Ok(u64::from_le_bytes(self.read_n::<8>(address))) } /// Store a doubleword (64-bit) to memory - for SD instruction pub fn store_doubleword(&mut self, address: u64, value: u64) -> Result<(), MemoryError> { - if address.is_multiple_of(8) { - let low = (value & 0xFFFFFFFF) as u32; - let high = (value >> 32) as u32; - // 8-alignment bounds `address` to `u64::MAX - 7`, so `address + 4` can't overflow. - self.cells.insert(address, low.to_le_bytes()); - self.cells.insert(address + 4, high.to_le_bytes()); - } else { + if !address.is_multiple_of(8) { address.checked_add(7).ok_or(MemoryError::AddressOverflow)?; - let bytes = value.to_le_bytes(); - for (i, b) in bytes.iter().enumerate() { - self.store_byte(address + i as u64, *b); - } } + self.write_n::<8>(address, value.to_le_bytes()); Ok(()) } pub fn load_half(&self, address: u64) -> Result { - if address.is_multiple_of(2) { - let aligned_address = address - address % 4; - let bytes = self - .cells - .get(&aligned_address) - .cloned() - .unwrap_or_default(); - let offset = (address % 4) as usize; - Ok(u16::from_le_bytes([bytes[offset], bytes[offset + 1]])) - } else { + if !address.is_multiple_of(2) { address.checked_add(1).ok_or(MemoryError::AddressOverflow)?; - Ok(u16::from_le_bytes([ - self.load_byte(address), - self.load_byte(address + 1), - ])) } + // A 2-aligned half never leaves its word, so it cannot wrap. + Ok(u16::from_le_bytes(self.read_n::<2>(address))) } pub fn store_half(&mut self, address: u64, value: u16) -> Result<(), MemoryError> { - let bytes = value.to_le_bytes(); - if address.is_multiple_of(2) { - let aligned_address = address - address % 4; - let entry = self - .cells - .entry(aligned_address) - .or_insert_with(|| [0, 0, 0, 0]); - let offset = (address % 4) as usize; - entry[offset] = bytes[0]; - entry[offset + 1] = bytes[1]; - } else { + if !address.is_multiple_of(2) { address.checked_add(1).ok_or(MemoryError::AddressOverflow)?; - self.store_byte(address, bytes[0]); - self.store_byte(address + 1, bytes[1]); } + self.write_n::<2>(address, value.to_le_bytes()); Ok(()) } @@ -242,21 +603,15 @@ impl Memory { Ok(()) } - pub fn load_bytes(&self, mut addr: u64, len: u64) -> Result, MemoryError> { - let end = addr.checked_add(len).ok_or(MemoryError::AddressOverflow)?; + pub fn load_bytes(&self, addr: u64, len: u64) -> Result, MemoryError> { + addr.checked_add(len).ok_or(MemoryError::AddressOverflow)?; let len_usize = usize::try_from(len).map_err(|_| MemoryError::AllocationFailed)?; let mut result = Vec::new(); result .try_reserve_exact(len_usize) .map_err(|_| MemoryError::AllocationFailed)?; - while addr < end { - let aligned = addr - (addr % 4); - let bytes = self.cells.get(&aligned).cloned().unwrap_or_default(); - let offset = (addr % 4) as usize; - let take = std::cmp::min(4 - offset, (end - addr) as usize); - result.extend_from_slice(&bytes[offset..offset + take]); - addr += take as u64; - } + result.resize(len_usize, 0); + self.store.read(addr, &mut result); Ok(result) } @@ -264,22 +619,56 @@ impl Memory { /// Should only be used to write to public output and private input where these limitations are not a problem pub(crate) fn set_bytes_aligned( &mut self, - mut addr: u64, + addr: u64, inputs: &[u8], ) -> Result<(), MemoryError> { if !addr.is_multiple_of(4) { return Err(MemoryError::UnalignedAccess); } - for chunk in inputs.chunks(4) { - let mut bytes = [0u8; 4]; - bytes[..chunk.len()].copy_from_slice(chunk); - self.cells.insert(addr, bytes); - addr += 4; + let whole = inputs.len() - inputs.len() % 4; + self.store.write(addr, &inputs[..whole]); + if whole < inputs.len() { + let mut last = [0u8; 4]; + last[..inputs.len() - whole].copy_from_slice(&inputs[whole..]); + self.store.write(addr + whole as u64, &last); } Ok(()) } } +/// The memory at one moment of a run ([`Memory::snapshot`]). +#[derive(Debug, Clone)] +pub struct MemorySnapshot { + store: Store, +} + +impl MemorySnapshot { + /// The bytes at `address..` into `out`, as they were at the snapshot; the + /// range must not wrap. + pub fn read(&self, address: u64, out: &mut [u8]) { + self.store.read(address, out); + } + + /// The byte at `address`, as it was at the snapshot. + pub fn load_byte(&self, address: u64) -> u8 { + let mut byte = [0u8; 1]; + self.store.read(address, &mut byte); + byte[0] + } + + /// The bytes this snapshot keeps alive that `memory` no longer shares (for + /// the words store, its whole map). + pub fn bytes_apart_from(&self, memory: &Memory) -> usize { + match (&self.store, &memory.store) { + (Store::Pages(mine), Store::Pages(now)) => mine.bytes_apart_from(now), + (Store::Words(cells), _) => { + cells.capacity() * (std::mem::size_of::<(u64, [u8; 4])>() + 1) + } + (Store::Pages(mine), Store::Words(_)) => mine.heap_bytes(), + } + } +} + #[derive(thiserror::Error, Debug)] pub enum MemoryError { #[error("Unaligned memory access")] @@ -327,4 +716,75 @@ mod tests { .unwrap(); assert_eq!(data, inputs); } + + /// Addresses in the low directory, across a page boundary, in the high + /// directory (the stack's end of the space) and in neither. + const SPOTS: [u64; 4] = [0x1000, (1 << PAGE_BITS) - 2, u64::MAX - 0x100, 1 << 40]; + + fn read8(read: impl Fn(u64, &mut [u8]), address: u64) -> u64 { + let mut bytes = [0u8; 8]; + read(address, &mut bytes); + u64::from_le_bytes(bytes) + } + + // A snapshot reads the bytes of its moment while the memory is written on: + // over both stores, in both directories and outside them, across a page + // boundary, for bytes overwritten and for pages first written later. + #[test] + fn a_snapshot_reads_the_bytes_of_its_moment() { + for kind in [StoreKind::Pages, StoreKind::Words] { + let mut memory = Memory::with_store(kind); + for (i, &at) in SPOTS.iter().enumerate() { + memory + .store_doubleword(at, 0x1111 * (i as u64 + 1)) + .unwrap(); + } + let first = memory.snapshot(); + for (i, &at) in SPOTS.iter().enumerate() { + memory + .store_doubleword(at, 0xAAAA * (i as u64 + 1)) + .unwrap(); + memory.store_byte(at.wrapping_add(8 << PAGE_BITS), 0x5A); + } + let second = memory.snapshot(); + memory.store_byte(SPOTS[0], 0x77); + for (i, &at) in SPOTS.iter().enumerate() { + let fresh = at.wrapping_add(8 << PAGE_BITS); + assert_eq!( + read8(|a, out| first.read(a, out), at), + 0x1111 * (i as u64 + 1), + "{kind:?} at {at:#x}" + ); + assert_eq!(first.load_byte(fresh), 0, "{kind:?} at {fresh:#x}"); + assert_eq!(second.load_byte(fresh), 0x5A, "{kind:?} at {fresh:#x}"); + assert_eq!( + read8(|a, out| second.read(a, out), at), + 0xAAAA * (i as u64 + 1), + "{kind:?} at {at:#x}" + ); + } + assert_eq!(memory.load_byte(SPOTS[0]), 0x77, "{kind:?}"); + assert_eq!(second.load_byte(SPOTS[0]), 0xAA, "{kind:?}"); + } + } + + // A snapshot costs the pages written after it, and nothing for the pages + // it still shares or the pages first written after it. + #[test] + fn a_snapshot_costs_the_pages_written_after_it() { + let page = std::mem::size_of::() + PAGE_SIZE + PAGE_SIZE / 4 / 64 * 8; + let mut memory = Memory::with_store(StoreKind::Pages); + for &at in &SPOTS { + memory.store_byte(at, 1); + } + let snapshot = memory.snapshot(); + assert_eq!(snapshot.bytes_apart_from(&memory), 0); + memory.store_byte(SPOTS[0] + 1, 2); + memory.store_byte(SPOTS[0] + 2, 3); + memory.store_byte(1 << 50, 4); + assert_eq!(snapshot.bytes_apart_from(&memory), page); + memory.store_byte(SPOTS[2], 5); + memory.store_byte(SPOTS[3], 6); + assert_eq!(snapshot.bytes_apart_from(&memory), 3 * page); + } } diff --git a/executor/tests/README.md b/executor/tests/README.md index 7ae08e85c..233be158a 100644 --- a/executor/tests/README.md +++ b/executor/tests/README.md @@ -20,6 +20,14 @@ https://github.com/lambdaclass/ethrex.git Five manifests carry that pin, not one. `scripts/set_ethrex_rev.sh --show` prints it and fails if they ever disagree. +The guest builds its ethrex crates from a fork of that commit: a `[patch]` in +`executor/programs/rust/ethrex/Cargo.toml` points them at yetanotherco/ethrex, branch +`lambda-vm/jumpmap` (the pinned commit plus two that keep jump destinations as a bitmap). +`ProgramInput` is the same in both, so the fixtures and the native reference stay on the +pin and also check the forked guest against unpatched ethrex. `--show` fails unless the +fork's recorded base (`# ethrex-fork-base:` in that manifest) is the pin, and a re-pin to +another commit refuses until the fork is rebased onto it. + ### Generation These blobs are generated reproducibly by the in-repo tool `tooling/ethrex-fixtures` diff --git a/executor/tests/exec_paths_block.rs b/executor/tests/exec_paths_block.rs new file mode 100644 index 000000000..cbf4fe7fb --- /dev/null +++ b/executor/tests/exec_paths_block.rs @@ -0,0 +1,59 @@ +//! A real block run on the executor's fast access paths against the general +//! path, log by log: the two must write the same logs and the same public +//! output. Ignored: it needs a guest ELF and a block's private input, +//! `EXEC_PATHS_ELF` and `EXEC_PATHS_INPUT`, e.g. +//! +//! ```text +//! EXEC_PATHS_ELF=ethrex.elf EXEC_PATHS_INPUT=ethrex_mainnet_25475471_8140dd6d.bin \ +//! cargo test --release -p executor --test exec_paths_block -- --ignored --nocapture +//! ``` +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::memory::ExecPaths; + +#[test] +#[ignore = "needs EXEC_PATHS_ELF and EXEC_PATHS_INPUT"] +fn a_block_writes_the_same_logs_on_every_path() { + let elf = std::fs::read(std::env::var("EXEC_PATHS_ELF").expect("EXEC_PATHS_ELF")).unwrap(); + let input = + std::fs::read(std::env::var("EXEC_PATHS_INPUT").expect("EXEC_PATHS_INPUT")).unwrap(); + let program = Elf::load(&elf).expect("the ELF"); + let window = 1 << 20; + let mut reference = + Executor::new_with_paths(&program, input.clone(), ExecPaths::GENERAL).unwrap(); + let mut fast = Executor::new_with_paths(&program, input, ExecPaths::FAST).unwrap(); + let (mut cycles, mut mismatched) = (0usize, 0usize); + // The fast run hands its logs out as the block prover takes them. + while let Some(want) = reference + .resume_with_limit(window) + .expect("the reference run") + { + let got = fast + .resume_owned(window) + .expect("the fast run") + .expect("the fast run ended early"); + assert_eq!(got.len(), want.len(), "window at cycle {cycles}"); + for (i, (a, b)) in got.iter().zip(want).enumerate() { + let (a, b) = ( + [a.current_pc, a.next_pc, a.src1_val, a.src2_val, a.dst_val], + [b.current_pc, b.next_pc, b.src1_val, b.src2_val, b.dst_val], + ); + if a != b { + mismatched += 1; + if mismatched <= 10 { + eprintln!("cycle {}: fast {a:x?} general {b:x?}", cycles + i); + } + } + } + cycles += want.len(); + } + assert!( + fast.resume_owned(window).unwrap().is_none(), + "the fast run went on" + ); + let (reference, fast) = (reference.finish().unwrap(), fast.finish().unwrap()); + assert_eq!(fast.memory_values, reference.memory_values, "public output"); + assert_eq!(fast.register_values, reference.register_values, "a0, a1"); + eprintln!("EXEC PATHS: {cycles} cycles, {mismatched} logs differ, public output equal"); + assert_eq!(mismatched, 0); +} diff --git a/executor/tests/rust.rs b/executor/tests/rust.rs index 597cb6b1c..852303f87 100644 --- a/executor/tests/rust.rs +++ b/executor/tests/rust.rs @@ -1,11 +1,12 @@ use executor::{ - elf::Elf, + elf::{Elf, SymbolTable}, vm::execution::{ExecutionResult, Executor, ReturnValues}, vm::instruction::{ decoding::Instruction, execution::{DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER}, }, }; +use std::ops::Range; // NOTE: These tests require 64-bit RISC-V ELF files (RV64IM). // The test binaries need to be recompiled for 64-bit architecture. @@ -270,6 +271,170 @@ fn test_commit() { ); } +const ETHREX_ELF: &str = "./program_artifacts/rust/ethrex.elf"; +const MEM_SWEEP_ELF: &str = "./program_artifacts/rust/mem_sweep.elf"; + +/// `ecall`. The copy accelerator's `memcpy`, `memmove` and `memset` stubs (syscalls' +/// `entrypoint.rs`) each contain one; compiler_builtins' routines and the +/// `fast-memcpy` software `memcpy` contain none. +const ECALL: u32 = 0x0000_0073; + +/// `ld` (opcode and funct3 bits). The runtime's `memcmp` compares doublewords with it; +/// compiler_builtins' `memcmp` loads one byte at a time. +const LD: u32 = 0x0000_3003; + +/// The address range and the instruction words of the function that starts at `pc`. +fn function_at(program: &Elf, symbols: &SymbolTable, pc: u64) -> Option<(Range, Vec)> { + let f = symbols.lookup(pc).filter(|f| f.address == pc)?; + let segment = program.data.iter().find(|s| { + s.is_executable && pc >= s.base_addr && pc < s.base_addr + 4 * s.values.len() as u64 + })?; + let first = ((pc - segment.base_addr) / 4) as usize; + let words = segment.values[first..first + (f.size / 4) as usize].to_vec(); + Some((pc..pc + f.size, words)) +} + +/// The address range and the instruction words of the function `name` in an ELF. +fn function_in(elf_bytes: &[u8], name: &str) -> (Range, Vec) { + let program = Elf::load(elf_bytes).unwrap(); + let symbols = SymbolTable::parse(elf_bytes); + for segment in program.data.iter().filter(|s| s.is_executable) { + for i in 0..segment.values.len() as u64 { + let pc = segment.base_addr + 4 * i; + if symbols + .lookup(pc) + .is_some_and(|f| f.address == pc && f.name == name) + { + return function_at(&program, &symbols, pc).unwrap(); + } + } + } + panic!("no function `{name}` in the ELF"); +} + +/// The address range and the instruction words of what `name` executes: the +/// function itself or, when it is a two-instruction tail jump (`auipc rd, hi` then +/// `jalr x0, lo(rd)`, which is how these guests' `memmove` enters compiler_builtins' +/// implementation), the function it jumps to. The words are compared across ELFs, +/// so they must not depend on where the code sits. +fn body_of(elf_bytes: &[u8], name: &str) -> (Range, Vec) { + let (range, words) = function_in(elf_bytes, name); + let &[auipc, jalr] = words.as_slice() else { + return (range, words); + }; + let rd = (auipc >> 7) & 31; + let is_auipc = auipc & 0x7f == 0x17; + let is_jump = jalr & 0x7f == 0x67 && (jalr >> 7) & 31 == 0 && (jalr >> 15) & 31 == rd; + if !(is_auipc && is_jump) { + return (range, words); + } + let hi = i64::from((auipc & 0xffff_f000) as i32); + let lo = i64::from(jalr as i32 >> 20); + let target = (range.start as i64 + hi + lo) as u64; + let program = Elf::load(elf_bytes).unwrap(); + let symbols = SymbolTable::parse(elf_bytes); + function_at(&program, &symbols, target) + .unwrap_or_else(|| panic!("`{name}` jumps to {target:#x}, where no function starts")) +} + +/// In-VM sweep of the memory routines the guest links (`memcpy`, `memmove` and +/// `memset` are the runtime's copy-accelerator stubs; `memcmp` is the runtime's +/// doubleword routine). The guest compares every result with values +/// recomputed from the index and commits `OK` plus its case counts, or `FAIL` plus +/// the first failing case. Counts, in order: memcpy at every length 0..=1100 and +/// offsets mod 8; memcpy at lengths 0..=160 and offsets mod 16; memcpy large lengths, +/// lengths up to a page and the compiler's own copies; memcpy onto itself and between +/// touching regions; memmove overlapping both ways; memset; memcmp calls. +#[test] +fn test_mem_sweep() { + let elf = std::fs::read(MEM_SWEEP_ELF).unwrap(); + // The stubs' own ECALLs are the only instructions inside them that read a7. + let stubs = ["memcpy", "memmove", "memset"].map(|name| { + let (range, words) = function_in(&elf, name); + assert!( + words.contains(&ECALL), + "the sweep guest does not link the runtime's `{name}` stub" + ); + range + }); + let program = Elf::load(&elf).unwrap(); + let mut executor = Executor::new(&program, vec![]).expect("Failed to run program"); + let (mut copy_calls, mut set_calls) = (0u64, 0u64); + while let Some(logs) = executor.resume().expect("Failed to run program") { + for log in logs.iter() { + if stubs.iter().any(|r| r.contains(&log.current_pc)) { + copy_calls += u64::from(log.src1_val == DMA_MEMCPY_SYSCALL_NUMBER); + set_calls += u64::from(log.src1_val == DMA_MEMSET_SYSCALL_NUMBER); + } + } + } + let out = executor + .finish() + .expect("Failed to run program") + .memory_values; + if !out.starts_with(b"OK") { + assert!( + out.starts_with(b"FAIL") && out.len() == 15, + "unexpected output {out:?}" + ); + panic!( + "mem_sweep failed: stage {} length {} source alignment {} destination alignment {} first wrong byte at destination + {}", + out[4], + u32::from_le_bytes(out[5..9].try_into().unwrap()), + out[9], + out[10], + i32::from_le_bytes(out[11..15].try_into().unwrap()), + ); + } + let counts: Vec = out[2..] + .chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect(); + // 1101 lengths x 64; 161 lengths x 256; 6 x 16 + 2 x 4 + 428 x 4 + 16; 301 x 8 x 3; + // 201 x 8 x 19; 301 x 8 x 2 memset; 64 offset pairs x 31_322 memcmp calls + // (lengths 0..=300, see `memcmp_positions` in the guest). + assert_eq!( + counts, + [70_464, 41_216, 1_832, 7_224, 30_552, 4_816, 2_004_608] + ); + assert!( + function_in(&elf, "memcmp") + .1 + .iter() + .any(|&w| w & 0x707f == LD), + "the sweep guest's `memcmp` has no doubleword load: it is not the runtime's" + ); + assert!( + copy_calls > 0 && set_calls > 0, + "the sweep made {copy_calls} copy and {set_calls} memset ecalls: the copy \ + accelerator's stubs did not run" + ); +} + +/// The guest every block proof is about links the very `memcpy`, `memmove`, `memset` +/// and `memcmp` that `test_mem_sweep` exercises: the same instruction words in both ELFs. +#[test] +fn test_ethrex_links_the_swept_memory_routines() { + let ethrex = std::fs::read(ETHREX_ELF).unwrap(); + let sweep = std::fs::read(MEM_SWEEP_ELF).unwrap(); + for name in ["memcpy", "memmove", "memset"] { + assert!( + function_in(ðrex, name).1.contains(&ECALL), + "the ethrex guest does not link the runtime's `{name}` stub" + ); + assert_eq!( + body_of(ðrex, name).1, + body_of(&sweep, name).1, + "`{name}` differs between the ethrex guest and the sweep guest" + ); + } + assert_eq!( + body_of(ðrex, "memcmp").1, + body_of(&sweep, "memcmp").1, + "`memcmp` differs between the ethrex guest and the sweep guest" + ); +} + #[test] fn test_ef_io_demo_concatenates_writes() { // Demo guest reads its private input via EF `read_input`, then emits it diff --git a/formal_verification/blake3_absorb/README.md b/formal_verification/blake3_absorb/README.md new file mode 100644 index 000000000..597125899 --- /dev/null +++ b/formal_verification/blake3_absorb/README.md @@ -0,0 +1,274 @@ +# Formal verification of the BLAKE3 chained-absorb mode + +Machine-checks the row-local constraints of the absorb mode added to +`prover/src/tables/blake3.rs` by commit `9cf6c352` — the second syscall the +BLAKE3 table answers, in which one ecall folds `num_blocks` 64-byte blocks and +occupies `num_blocks + 1` rows. + +Method, file layout and the Mandatory-discipline checklist come from +`formal_verification/keccak/README.md`, which is the canonical template. The +compression core's own gate is the earlier +`thoughts/blake3/blake3-chip/z3_blake_verify.py` (#903); this directory is its +successor for the absorb mode and re-runs its G-level check so the board here +stands alone. + +``` +python3 test_ref.py # anchor the reference outside this repo +python3 z3_absorb_verify.py # the gate — 60 queries, ~25 s +make test-blake3-absorb-fv # both, from the repo root +``` + +Only dependency is z3's Python bindings (`pip install z3-solver`). No cargo, no +repo build. + +--- + +## ⚠ The binding scope rule: one round at a time + +**The 6 rounds are never composed in one query, because that query does not +close.** This is not a preference, it is measured. On 2026-08-06 the +predecessor gate's `--full` board ran 145 minutes and returned `unknown` — z3's +timeout answer — for all four of its composed queries (one round, rounds=2, +rounds=6, rounds=7). Its verdict line tested `== unsat`, so four timeouts +scored as four failures and the board printed `FAIL`. Commit `89aeeb8c` +recorded the outcome as ATTEMPTED-INCONCLUSIVE. + +**Re-examined here: that was a scope artifact, not a finding.** ✓ VERIFIED by +reading `z3_blake_verify.py:309-344` (the four checks return `check()` directly +after `s.set("timeout", …)`) against `89aeeb8c`'s own message ("all four +monolithic queries returned `unknown`, not `sat`… The four budgets sum to 140 +min against ~145 min wall"). Nothing was disproven; each check simply spent its +whole allowance. `89aeeb8c` itself names the remedy this directory implements: +"restructure the monolithic query as round-by-round induction rather than one +flat bit-vector problem." + +So each round is factored into two queries that close in under a second: + +| level | query | what it decides | +|---|---|---| +| **P1b** | the byte-level G circuit vs the spec G, **free inputs** | that a G computes BLAKE3's G. One query covers all 48 instances — the circuit text of a G is identical for every one, so 48 copies would re-decide the same formula. | +| **P1a** | round *r*'s **wiring**, with G an uninterpreted function | that round *r* hands each G the state slots `G_INDICES` names and the message words `permute^r` names. Six queries, `r = 0..5`, each with that round's concrete schedule indices. | + +### What is a structural argument, and therefore NOT SMT-proved + +Two compositions are stated here and carried as wiring arguments. Both are +checkable by reading; neither is claimed as a solver result. + +1. **G-internal ∘ round-wiring.** The wiring query treats G as a black box; the + G query proves the black box is the spec's G *for arbitrary inputs*. + Substituting the second into the first yields the round. Sound precisely + because the G query quantifies over free inputs, so it holds at every + instantiation. +2. **Round *r* → round *r+1*.** `run_flow` (`blake3.rs:342-373`) is one loop + whose body writes `v[ia] = a2; v[ib] = b2; v[ic] = c2; v[id] = vd2` back + into the array the next iteration reads. Round *r*'s output columns *are* + round *r+1*'s input columns — there is no committed handoff to constrain and + nothing for a prover to choose. ✓ VERIFIED by reading. + +**If a single-round query ever fails to close, report the bound and stop. Do +not weaken the encoding to make it green.** + +--- + +## Two theories, deliberately + +The compression core is boolean/byte algebra, which **QF-BV** models exactly +(P1). Byte columns are 8-bit bitvectors — that *is* the `AreBytes`/`ByteAlu` +range-check contract, expressed as a sort. + +The absorb mode's own constraints are not byte algebra. They are field +arithmetic over selector columns, a countdown, and packed limbs, and every +attack they defend against is field-level: `256`, `2^16` and `2^32` are +**invertible** mod the Goldilocks prime while they are zero divisors mod `2^n`. +A bitvector model of those constraints would report UNSAT for free and bless a +chip that is forgeable in the field. **P2–P5 therefore run in integer +arithmetic mod p**, every congruence linearized by a bounded quotient. This is +the same split the predecessor gate's `WIDTH AUDIT` section made, and +discipline #3 of the keccak template. + +Two modeling steps are supplied to the solver rather than derived, and are +recorded here because they are the model's trust boundary: + +- **`IS_BIT` as a disjunction.** The circuit emits `x·(1−x) = 0`. Since p is + prime and committed values lie in `[0,p)`, that is *equivalent* to + `x ∈ {0,1}`, and the model asserts the disjunction. z3 cannot derive this — + it does not know p is prime. (Proof: `p | x(1−x) ⟹ p|x or p|(1−x)`.) +- **Quotient bounds.** `expr ≡ 0 (mod p)` is encoded as `expr = k·p` with `k` + bounded. ⚠ **A bound computed for the contract-present case will silently + re-impose the contract when that contract is dropped**, turning a negative + control falsely UNSAT. `pointer_add` carries an explicit comment about this; + it is the reason its bound is `±2^17·p` and not `±2·p`. + +--- + +## The board + +60 queries, ~25 s wall on an M-series laptop, z3 4.15.4. `P*-neg` rows are +negative controls and **must** be SAT. + +| # | property | verdict | +|---|---|---| +| **P1a** | round *r* wiring, `r = 0..5` (6 queries) | PROVED | +| **P1b** | G circuit vs spec G, free inputs | PROVED | +| **P1c** | input feed under the absorb framing: `IV \| t=0 \| block_len=64 \| flags` | PROVED | +| **P1d** | feed-forward, and that the chain payload is the low 8 words | PROVED | +| **P2.1-2** | an interior row's counter bytes are zero and `block_len` bytes are exactly `(64,0,0,0)` | PROVED | +| **P2.3** | ★ an interior row cannot carry **any** flag byte — the forged-shorter-message class | PROVED | +| **P2.4** | a FIRST row's flags word is `< 2^32` | PROVED | +| **P3.1** | exactly **five** row shapes exist; there is no sixth | PROVED | +| **P3.2** | every derived multiplicity — `MU_C`, `MU − END`, `MU_A − FIRST`, `MU_S + FIRST` — lies in `{0,1}` | PROVED | +| **P3.3-4** | no cross-mode bleed, in both directions | PROVED | +| **P3.5** | `MU_S·MU_A = 0` is **implied** by `IS_BIT(MU)` + the partition (informational) | PROVED | +| **P4.1** | a FIRST row's `REMAINING` is in `1..=1024` — the cap, in circuit | PROVED | +| **P4.2-3** | END is neither early nor late: `END = 1 ⟺ REMAINING = 0` | PROVED | +| **P4.4-5** | the END row is inert (`MU_C = 0`); a compressing row decrements by one | PROVED | +| **P4.6** | ★ the cap cannot be wrapped mod p | PROVED | +| **P5.1** | the END row's `cv_out` bytes are **unique** given the chain-delivered words | PROVED | +| **P5.2** | `M_BASE_INCR` is a function of `M_BASE` | PROVED | +| **P6.1-2** | a FIRST row's `M_BASE[0] ≡ 0 (mod 8)` — x11 is 8-aligned — with the model shown consistent | PROVED | +| **P6.3** | a NON-final block's successor address cannot leave the address space | PROVED | +| **P6.4** | …but the FINAL block's may, so a message ending at `2^64` stays provable | SAT (intended) | + +No counterexample was found against any shipped constraint. Every negative +control flipped to SAT. + +**P6 was added by the F1 fix pass**, after the adversarial review demonstrated +two absorbs the chip accepted and the executor rejects: an unaligned `x11`, and +a message region wrapping the address space mid-group. + +### ★ The witness that changed the fix + +`P6-neg: the SCALED form IsB20[Q·2^7] admits an odd base`. The review sketched +the alignment check as `M_BASE[0] = 8·Q` bounded by `IsB20[Q · 2^7]`, mirroring +the block cap's scaling trick. It is **vacuous**: with `Q` bounded only through +the scaled product a prover takes `Q = M_BASE[0] · 8⁻¹ mod p`, and then +`Q · 2^7 = M_BASE[0] · 16`, under `2^20` for ANY halfword base, aligned or not. +The shipped chip bounds `Q` directly with `IsHalfword[Q]` instead, which forces +the congruence to be an integer equation and so forces `8 | M_BASE[0]`. + +The block cap survives the identical attack only because `REMAINING`'s domain +bound arrives first from the ZERO lookup — the same reason P4.6 needs +`zero_domain` present. Modelling the alignment equation as an integer equality +rather than a field congruence hides all of this: no integer `Q` divides an odd +base, so the vacuous variant reports UNSAT and looks safe. That trap is why +`msg_addressing` uses `cong0`. + +### Two witnesses worth reading + +**`drop zero_domain` (P4-neg).** Dropping only the *domain* half of the +`Zero[REMAINING]` contract — keeping `END = (REMAINING == 0)` — lets a FIRST +row claim `REMAINING = 18014398505288705` and still satisfy the `IsB20` cap, +because `REM_DECR · 2^10 ≡ 2^20 − 1 (mod p)`. The cap lookup alone does not +bound the block count; it bounds it **only in combination with** the `Zero` +lookup's `< 2^20` domain. The chip has both, and `blake3.rs:1459-1462` says so; +this makes the dependency machine-checked rather than asserted. + +**`drop byte_range_framing` (P5-neg).** With the `block_len` bytes' range check +removed, `block_len_b1 = 13835058052060938241` satisfies +`MU_C·(word − 64) = 0` while the byte fed to the mixing core is nowhere near +`64`. The word-level constraint means what it says *only* because those four +bytes are `ByteAlu` XOR operands. + +--- + +## Width census — "the chip assumes there are bytes" + +Every column an absorb row consumes at a width, and the contract that pins that +width on the row shape where it is consumed. ✓ VERIFIED by reading each cited +construct. + +| columns | consumed on | pinned by | site | +|---|---|---|---| +| `h`/`cv_in` bytes (32) | compressing rows | `ByteAlu[XOR]` operands (mixing core, gated `MU − END`) | `blake3.rs:1196-1209` | +| `h` bytes | **END row** | `AreBytes`, END-gated — the one row whose core is off | `blake3.rs:1245-1251` | +| `m` bytes (64) | compressing rows | `AreBytes` (m is never XORed) | `blake3.rs:1226-1237` | +| `t_lo`,`t_hi`,`block_len`,`flags` (16) | compressing rows | `ByteAlu[XOR]` operands: they are `v[12..16]`, the `d` of round 0's G-calls 0..3 | `blake3.rs:334, 350` | +| `OUT` bytes (64) | compressing rows | `ByteAlu[XOR]` **outputs** (feed-forward) | `blake3.rs:1196-1209` | +| `OLD_OUT` (64), `ADDR` (8) | `MU` | `AreBytes` (+ AND alignment on `ADDR`) | `blake3.rs:1256-1302` | +| `PTR[0..8]` halfwords | `MU` | `IsHalfword` | `blake3.rs:1308-1320` | +| `M_BASE` halfwords | `MU_A`, incl. END (it rides the chain) | `IsHalfword` | `blake3.rs:1486-1492` | +| `M_BASE_INCR`, `msg_ptr[0..8]` halfwords | `MU_C` | `IsHalfword` | `blake3.rs:1493-1508` | +| `REMAINING` | `MU_A` | `Zero` domain `< 2^20`; x12's `hi32 = 0` constant caps the high half | `blake3.rs:1463-1467, 1385` | +| `REM_DECR` | FIRST | `IsB20[REM_DECR·2^10]` ⟹ `< 2^10`; on later rows pinned by the countdown | `blake3.rs:1474-1481` | +| `MU_S`,`MU_A`,`FIRST`,`END` | all | `IS_BIT` | `blake3.rs:1730-1733` | +| `MU_C` | all | **derived**, `MU_C = MU_A − END`; that it is a bit is theorem **P3.2**, not a constraint | `blake3.rs:1755-1759` | + +**Free-but-unconsumed on the END row — audited, not gaps.** The END row leaves +`m`, `t_lo`/`t_hi`/`block_len`/`flags`, and `OUT` unconstrained, because their +gates (`MU_C`, `MU − END`, `MU_C − FIRST`) are all zero there. None rides a bus +on that row: the message reads are `MU_C`-gated, the x13 read is `FIRST`-gated +and `FIRST·END = 0`, the chain send is `MU_C`-gated, and `cv_out` writes the +`h` columns — not `OUT` (`blake3.rs:1414-1424`). A free column that no +interaction reads cannot forge anything. + +--- + +## Typed contract library (assume-guarantee) + +Each helper lookup is modeled by its contract, never its implementation +(`bitwise.rs:756-830`). Each is a separately verified, fully enumerated +preprocessed table. + +| contract | guarantee modeled | +|---|---| +| `ByteAlu(op,a,b,c)` | `a,b ∈ [0,256)` and `c = a op b`. Operands are byte range-checked **by** the lookup — the table has only byte rows. | +| `AreBytes(a,b)` | `a,b ∈ [0,256)`. | +| `Zero(v) -> z` | `v ∈ [0,2^20)` **and** `z = 1 iff v = 0`. ★ Both halves are load-bearing; see the P4 witness. | +| `IsB20(v)` | `v ∈ [0,2^20)`. | +| `IsHalfword(v)` | `v ∈ [0,2^16)`. | +| Memw register read | the value's `lo32` limb is a 32-bit limb; the `hi32` slot is the **constant** `0` in the tuple (`blake3.rs:1385,1394`). | + +--- + +## Out of scope — do not read this gate as covering these + +- **Bus telescoping / multiset balance of the `Blake3Absorb` chain**: that a + group's rows form one chain, that every send has a receiver, that two groups + cannot interleave. Every claim above is **row-local**. Owned by the in-tree + falsification suite (`blake3.rs` tests `a_tampered_chained_cv_unbalances_the_chain`, + `a_group_without_its_end_row_leaves_a_dangling_send`, + `an_early_end_sends_a_zero_tuple_that_does_not_exist`) and by the `rev-absorb` + review lane. +- **MEMW ordering / the memory argument** — that a read returns what was written. +- **Fiat-Shamir, the transcript, and everything above the AIR.** +- **The helper chips themselves** — assumed via the contract table. +- **The 6-round variant's collision resistance**, which is a named assumption + (`thoughts/blake3/blake3-chip/DESIGN.md`), not a theorem. + +## What a green board still does not show + +Carried forward from the keccak template's discipline #1, and true here: + +- **The model is a hand transcription.** z3 never sees the Rust. Faithfulness + is a human obligation; the long-term fix is to generate the model from the + constraint IR (`prover/src/bin/compute_constraint_artifacts.rs`) instead. + Today that binary emits `.bin` artifacts for `production_airs` only, and + **BLAKE3 is not in that set** (it is a chip-group table), so there is nothing + to diff against yet. The transcription is pinned instead by the in-repo shape + test `the_tables_shape_is_pinned` (3,266 columns / 1,473 interactions / 848 + constraints) and by the ledger comment at `blake3.rs:1517-1544`, which is the + authoritative index map the model follows. +- **A constraint carried by a variable's sort is outside the falsifiable set.** + In P1 the byte range checks are the 8-bit BV sort, so they cannot be dropped + from the model — deleting them from the Rust would leave P1 green. That is + why the same bounds are re-checked *removably* in the P5 field model, where + dropping them does flip the board. +- **Fail-open is the dangerous mode.** A gate that is green for the wrong + reason silently blesses an unsound chip. One such bug was found and fixed + during this gate's development: two "independent" models in the uniqueness + queries were built from identically-named z3 variables, which alias to the + same variable and made the query trivially UNSAT. The `distinct_names` guard + now fails the run rather than printing green. + +--- + +## Files + +- `z3_absorb_verify.py` — the gate: the field model of the absorb constraints, + the QF-BV model of the compression core, all five properties, the negative + controls, and the positive controls. +- `blake3_ref.py` — the independent reference, written from the spec over two + backends (concrete ints for anchoring, z3 bitvectors for the gate). +- `test_ref.py` — anchors the reference outside this repo: the IV against + `frac(sqrt(p))` for the first 8 primes, the 6-round compression against the + 10 recorded canonical vectors, and `absorb()` against chained `compress`. diff --git a/formal_verification/blake3_absorb/blake3_ref.py b/formal_verification/blake3_absorb/blake3_ref.py new file mode 100644 index 000000000..09c352d79 --- /dev/null +++ b/formal_verification/blake3_absorb/blake3_ref.py @@ -0,0 +1,197 @@ +""" +Independent BLAKE3 reference, written from the spec, usable over two backends. + +The same round/compression text runs on concrete `int`s (for anchoring against +the recorded oracle vectors) and on z3 32-bit bitvectors (for the QF-BV gate). +Nothing here is derived from `prover/src/tables/blake3.rs` — that is the point: +a reference copied from the circuit proves only that the circuit equals itself. + +Anchors (see `test_ref.py`): + * the constants below against the BLAKE3 spec's IV (= SHA-256's) and the + published message permutation; + * `compress(..., rounds=7)` against the official BLAKE3 test vectors; + * `compress(..., rounds=6)` against `tooling/blake3-second-source/blake3-oracle/`'s + recorded canonical vectors, which the executor's own primitive reproduces. + +⚠ The chip implements the 6-ROUND internal variant, not standard 7-round +BLAKE3. `rounds` is a parameter here so both can be exercised. +""" + +MASK32 = 0xFFFFFFFF + +# BLAKE3 spec §2: IV = SHA-256's initial hash value. +IV = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +] + +# BLAKE3 spec §2.2, the message word permutation applied between rounds. +MSG_PERMUTATION = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + +# BLAKE3 spec §2.1: the 8 G-calls of a round — 4 column mixes, then 4 diagonals. +# (a, b, c, d) state indices; G-call j consumes message positions 2j and 2j+1. +G_INDICES = [ + (0, 4, 8, 12), + (1, 5, 9, 13), + (2, 6, 10, 14), + (3, 7, 11, 15), + (0, 5, 10, 15), + (1, 6, 11, 12), + (2, 7, 8, 13), + (3, 4, 9, 14), +] + +# The 64-byte block length every absorbed block is framed with, and the counter +# value the absorb mode fixes. Both are ABI facts of the absorb ecall, not spec +# facts: `executor::…::blake3_absorb_step_6round` calls the compression with +# `t = 0` and `block_len = BLAKE3_BLOCK_BYTES`. +ABSORB_BLOCK_LEN = 64 +ABSORB_COUNTER = 0 + + +class IntOps: + """Concrete backend: Python ints reduced mod 2^32.""" + + @staticmethod + def add(a, b): + return (a + b) & MASK32 + + @staticmethod + def xor(a, b): + return a ^ b + + @staticmethod + def rotr(x, n): + return ((x >> n) | (x << (32 - n))) & MASK32 + + @staticmethod + def const(v): + return v & MASK32 + + +class BvOps: + """z3 backend: 32-bit bitvector terms.""" + + @staticmethod + def add(a, b): + return a + b + + @staticmethod + def xor(a, b): + return a ^ b + + @staticmethod + def rotr(x, n): + from z3 import RotateRight + + return RotateRight(x, n) + + @staticmethod + def const(v): + from z3 import BitVecVal + + return BitVecVal(v & MASK32, 32) + + +def g(ops, v, a, b, c, d, mx, my): + """The BLAKE3 quarter-round, spec §2.1, in place on `v`.""" + v[a] = ops.add(ops.add(v[a], v[b]), mx) + v[d] = ops.rotr(ops.xor(v[d], v[a]), 16) + v[c] = ops.add(v[c], v[d]) + v[b] = ops.rotr(ops.xor(v[b], v[c]), 12) + v[a] = ops.add(ops.add(v[a], v[b]), my) + v[d] = ops.rotr(ops.xor(v[d], v[a]), 8) + v[c] = ops.add(v[c], v[d]) + v[b] = ops.rotr(ops.xor(v[b], v[c]), 7) + + +def round_fn(ops, v, m): + """One round: the 8 G-calls, consuming `m` in position order.""" + for j, (a, b, c, d) in enumerate(G_INDICES): + g(ops, v, a, b, c, d, m[2 * j], m[2 * j + 1]) + + +def permute(m): + """m'[i] = m[MSG_PERMUTATION[i]].""" + return [m[MSG_PERMUTATION[i]] for i in range(16)] + + +def schedule_indices(r): + """Indices into the ORIGINAL message consumed at each position of round r. + + Round 0 consumes m[i] at position i; each later round permutes, so + sched_{r+1}[i] = sched_r[P[i]]. Returned as a list of 16 indices. + """ + sched = list(range(16)) + for _ in range(r): + prev = sched + sched = [prev[MSG_PERMUTATION[i]] for i in range(16)] + return sched + + +def initial_state(ops, h, tlo, thi, block_len, flags): + """The 16-word compression state before round 0 (spec §2.3).""" + return [ + h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], + ops.const(IV[0]), ops.const(IV[1]), ops.const(IV[2]), ops.const(IV[3]), + tlo, thi, block_len, flags, + ] + + +def feed_forward(ops, v, h): + """The 16-word output: out[i] = v[i]^v[i+8], out[i+8] = v[i+8]^h[i].""" + out = [None] * 16 + for i in range(8): + out[i] = ops.xor(v[i], v[i + 8]) + out[i + 8] = ops.xor(v[i + 8], h[i]) + return out + + +def compress(ops, h, m, tlo, thi, block_len, flags, rounds): + """Full compression: init, `rounds` rounds with the permuting schedule, + then feed-forward. Returns 16 words.""" + v = initial_state(ops, h, tlo, thi, block_len, flags) + sched = list(m) + for r in range(rounds): + round_fn(ops, v, sched) + if r < rounds - 1: + sched = permute(sched) + return feed_forward(ops, v, h) + + +def absorb_step(ops, cv, m, flags, rounds): + """One absorbed block: compress under the absorb framing, keep 8 words. + + This is the pure function `executor::…::blake3_absorb_step_6round` names, + restated from the spec side. + """ + out = compress( + ops, + cv, + m, + ops.const(ABSORB_COUNTER), + ops.const(ABSORB_COUNTER), + ops.const(ABSORB_BLOCK_LEN), + flags, + rounds, + ) + return out[:8] + + +def absorb(ops, cv, blocks, first_flags, rounds): + """A whole absorb ecall: `first_flags` on block 0, zero on every later one.""" + for i, m in enumerate(blocks): + cv = absorb_step(ops, cv, m, first_flags if i == 0 else ops.const(0), rounds) + return cv + + +# --------------------------------------------------------------------------- +# Concrete helpers (int backend only) +# --------------------------------------------------------------------------- + +def compress_int(h, m, t, block_len, flags, rounds): + """Concrete compression with the 64-bit counter split as the chip splits it.""" + return compress( + IntOps, list(h), list(m), + t & MASK32, (t >> 32) & MASK32, block_len, flags, rounds, + ) diff --git a/formal_verification/blake3_absorb/test_ref.py b/formal_verification/blake3_absorb/test_ref.py new file mode 100644 index 000000000..c47cb1b38 --- /dev/null +++ b/formal_verification/blake3_absorb/test_ref.py @@ -0,0 +1,108 @@ +""" +Anchors the gate's reference to something outside this repo's circuit. + +Discipline #4 of `formal_verification/keccak/README.md`: a reference derived +from the circuit proves only that the circuit equals itself. So: + + 1. the constants are checked against their spec definitions (the IV is + SHA-256's, i.e. the fractional parts of the square roots of the first 8 + primes; the permutation is the published one); + 2. `compress(rounds=7)` — standard BLAKE3 — is checked against the OFFICIAL + BLAKE3 test vectors carried in `tooling/blake3-second-source/blake3-oracle/`; + 3. `compress(rounds=6)` — the chip's variant — is checked against the + recorded canonical vectors in the same place, which the executor's own + `blake3_compress_6round` reproduces; + 4. `absorb()` is checked against a chain of `compress` calls, and against the + one-flag-on-block-0 rule the executor implements. + +Run: python3 test_ref.py +""" +import json +import os +import sys + +import blake3_ref as ref + +HERE = os.path.dirname(os.path.abspath(__file__)) +ORACLE = os.path.join(HERE, "..", "..", "tooling", "blake3-second-source", "blake3-oracle") + + +def check(name, cond): + print(f" {'OK ' if cond else '!! '}{name}") + return cond + + +def sqrt_frac_iv(): + """SHA-256's IV: frac(sqrt(p)) for the first 8 primes, top 32 bits.""" + from decimal import Decimal, getcontext + + getcontext().prec = 60 + primes = [2, 3, 5, 7, 11, 13, 17, 19] + out = [] + for p in primes: + frac = Decimal(p).sqrt() % 1 + out.append(int(frac * (1 << 32))) + return out + + +def main(): + ok = True + print("Reference anchoring") + + ok &= check("IV = frac(sqrt(first 8 primes)) — SHA-256's, per BLAKE3 §2", + ref.IV == sqrt_frac_iv()) + ok &= check("MSG_PERMUTATION is a permutation of 0..16", + sorted(ref.MSG_PERMUTATION) == list(range(16))) + ok &= check("schedule_indices(0) is the identity", + ref.schedule_indices(0) == list(range(16))) + ok &= check("schedule_indices(1) = MSG_PERMUTATION", + ref.schedule_indices(1) == ref.MSG_PERMUTATION) + ok &= check("G_INDICES covers each state word twice per round", + sorted(i for t in ref.G_INDICES for i in t) == + sorted(list(range(16)) + list(range(16)))) + + # --- 6-round canonical vectors (the chip's variant) -------------------- + path = os.path.join(ORACLE, "canonical_6round_vectors.json") + if os.path.exists(path): + with open(path) as f: + vecs = json.load(f) + good = 0 + for v in vecs: + got = ref.compress_int(v["h"], v["m"], v["t"], v["block_len"], + v["flags"], 6) + good += (got == v["out"]) + ok &= check(f"6-round compression vs {len(vecs)} recorded canonical vectors", + good == len(vecs)) + else: + print(f" ?? canonical_6round_vectors.json not found at {path} — SKIPPED") + print(" (the oracle lives in `tooling/blake3-second-source/blake3-oracle/`)") + + # --- 7-round official vectors (standard BLAKE3) ----------------------- + path = os.path.join(ORACLE, "official_test_vectors.json") + if os.path.exists(path): + print(" -- official BLAKE3 vectors present; the chip is the 6-round") + print(" variant, so they anchor the reference, not the chip.") + + # --- absorb == chained compression ------------------------------------ + cv = list(ref.IV) + blocks = [[(i * 16 + j) * 2654435761 & 0xFFFFFFFF for j in range(16)] + for i in range(4)] + manual = list(cv) + for i, m in enumerate(blocks): + manual = ref.compress_int(manual, m, ref.ABSORB_COUNTER, + ref.ABSORB_BLOCK_LEN, + 0x0B if i == 0 else 0, 6)[:8] + got = ref.absorb(ref.IntOps, list(cv), blocks, 0x0B, 6) + ok &= check("absorb() == chained compress(t=0, block_len=64), flags on block 0", + got == manual) + + flagged_later = ref.absorb(ref.IntOps, list(cv), blocks, 0, 6) + ok &= check("absorb is first_flags-sensitive (0x0B vs 0 differ)", + got != flagged_later) + + print("\n" + ("PASS" if ok else "FAIL")) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/formal_verification/blake3_absorb/z3_absorb_verify.py b/formal_verification/blake3_absorb/z3_absorb_verify.py new file mode 100644 index 000000000..172665cff --- /dev/null +++ b/formal_verification/blake3_absorb/z3_absorb_verify.py @@ -0,0 +1,993 @@ +""" +Formal (z3) assume-guarantee gate for the BLAKE3 chip's CHAINED-ABSORB mode. + +Target: `prover/src/tables/blake3.rs` on `blake3-absorb-mode`, the mode +introduced by commit `9cf6c352`. Follows the method of +`formal_verification/keccak/` (the canonical template) and of the earlier +BLAKE3 compression gate `thoughts/blake3/blake3-chip/z3_blake_verify.py`. + +=========================================================================== +BINDING SCOPE RULE — READ BEFORE EXTENDING +=========================================================================== +The 6 rounds are NEVER composed in one query. The monolithic full-compression +query does not close: on 2026-08-06 the predecessor gate's `--full` board ran +145 minutes and returned `unknown` (z3 timeout) for all four of its composed +queries — round, rounds=2, rounds=6, rounds=7 — which its verdict line scored +as failures (recorded in commit `89aeeb8c`). That was a RESOURCE limit, not a +finding: nothing was disproven. + +This gate therefore verifies ONE round at a time, and factors each round into +two queries that each close in well under a second: + + * P1b the byte-level G circuit against the spec G, free inputs. The + circuit text of a G is identical for all 48 instances, so ONE query + covers every G of every round — running 48 copies would re-decide the + same formula. + * P1a round r's WIRING, with G abstracted as an uninterpreted function: + does round r hand each G the state slots `G_INDICES` names and the + message words `permute^r` names? Six queries, r = 0..5, each with + that round's concrete schedule indices. + +Composition of the two levels, and of round r into round r+1, is a STRUCTURAL +WIRING ARGUMENT, stated here and NOT claimed as SMT-proved: + + (i) G-internal ∘ round-wiring: the wiring query treats G as a black box; the + G query proves the black box is the spec's G for arbitrary inputs. Since + the wiring query's UF is applied to exactly the operands the circuit + feeds it, substituting the proved G into the proved wiring gives the + round. This is sound because the G circuit is verified under FREE + inputs, so it holds at every instantiation. + (ii) round r → round r+1: `run_flow` (blake3.rs:342-373) is one loop whose + body writes `v[ia] = a2; v[ib] = b2; v[ic] = c2; v[id] = vd2` back into + the same array the next iteration reads. Round r's output COLUMNS are + literally round r+1's input columns — there is no committed handoff to + constrain and nothing for a prover to choose. ✓ VERIFIED by reading. + +=========================================================================== +OUT OF SCOPE (owned elsewhere — do not read this gate as covering them) +=========================================================================== + * Bus telescoping / multiset balance of the `Blake3Absorb` chain — that a + group's rows form ONE chain, that a send has a receiver, that two groups + cannot interleave. Owned by the in-tree falsification suite + (`blake3.rs` tests `a_tampered_chained_cv_unbalances_the_chain`, + `a_group_without_its_end_row_leaves_a_dangling_send`, + `an_early_end_sends_a_zero_tuple_that_does_not_exist`) and by the + `rev-absorb` review lane. Every claim below is ROW-LOCAL. + * MEMW ordering / the memory argument (that a read returns what was written). + * Fiat-Shamir, the transcript, and everything above the AIR. + * The helper chips themselves: BITWISE's ByteAlu/AreBytes/Zero/IsB20/IsHalf + rows are ASSUMED to obey the contracts in `CONTRACTS` below. Each is a + separately verified, fully enumerated preprocessed table. + +=========================================================================== +TWO THEORIES, DELIBERATELY +=========================================================================== +The compression core is boolean/byte algebra → QF-BV models it exactly (P1). + +The absorb mode's own constraints are NOT byte algebra. They are field +arithmetic over selector columns, countdowns and packed limbs, and the +attacks they defend against are field-level: 256, 2^16 and 2^32 are INVERTIBLE +mod the Goldilocks prime while they are zero divisors mod 2^n. A bitvector +model of those constraints would report UNSAT for free and bless a chip that +is forgeable in the field. P2-P5 therefore run in integer arithmetic mod p, +with every congruence linearized by a bounded quotient. This mirrors the +`WIDTH AUDIT` section of the predecessor gate and discipline #3 of +`formal_verification/keccak/README.md`. + +=========================================================================== +MODELING ASSUMPTION, stated once +=========================================================================== +`IS_BIT(x)` is emitted as `x·(1−x) = 0` over the field. Since p is prime and +committed values lie in [0,p), that is EQUIVALENT to `x ∈ {0,1}`, and the +model encodes it as the disjunction. z3 cannot derive this — it does not know +p is prime — so it is supplied. Same convention as the predecessor gate's +`fresh_bit`. (One-line proof: p | x(1−x) ⟹ p|x or p|(1−x) ⟹ x=0 or x=1.) + +Run: python3 z3_absorb_verify.py (full board, ~seconds) + python3 z3_absorb_verify.py --verbose (also print witnesses) +""" +import sys +import time + +from z3 import ( + And, BitVec, BitVecVal, BitVecSort, Concat, Function, If, Implies, Int, Or, + RotateRight, Solver, ZeroExt, sat, unsat, +) + +import blake3_ref as ref + +# Goldilocks. +P = 2**64 - 2**32 + 1 + +# Widths the chip's helper lookups pin. See CONTRACTS. +BYTE = 256 +HALF = 2**16 +B20 = 2**20 +W32 = 2**32 + +# `ABSORB_MAX_BLOCKS` / `ABSORB_CAP_SCALE` (blake3.rs:125-128). +ABSORB_MAX_BLOCKS = 1 << 10 +ABSORB_CAP_SCALE = (1 << 20) // ABSORB_MAX_BLOCKS # = 2^10 + +CONTRACTS = """ +Typed helper-chip contracts ASSUMED by this gate (bitwise.rs:756-830): + ByteAlu(op,a,b,c) : a,b ∈ [0,256) and c = a op b. Operands are byte + range-checked BY the lookup — the table has only byte rows. + AreBytes(a,b) : a,b ∈ [0,256). + Zero(v) -> z : v ∈ [0,2^20) AND z = 1 if v = 0 else 0. ★ the DOMAIN + bound is as load-bearing as the output — see P4.6. + IsB20(v) : v ∈ [0,2^20). + IsHalfword(v) : v ∈ [0,2^16). + Memw register read: the value's lo32 limb ∈ [0,2^32); the hi32 slot here is + the CONSTANT 0 in the tuple (blake3.rs:1385,1394). +""" + +VERBOSE = "--verbose" in sys.argv + + +# =========================================================================== +# Field-model plumbing: congruences as bounded-quotient linear constraints. +# =========================================================================== +_kid = [0] + + +def cong0(cons, expr, emin, emax): + """Assert expr ≡ 0 (mod p), linearly, given integer bounds on expr.""" + _kid[0] += 1 + k = Int(f"_q{_kid[0]}") + cons.append(k >= emin // P - 1) + cons.append(k <= emax // P + 1) + cons.append(expr == k * P) + + +def fe(name, cons): + """A committed column: an arbitrary field element in [0,p).""" + v = Int(name) + cons.append(v >= 0) + cons.append(v < P) + return v + + +def bit(name, cons): + """A column carrying IS_BIT — see MODELING ASSUMPTION.""" + v = Int(name) + cons.append(Or(v == 0, v == 1)) + return v + + +def distinct_names(*groups): + """Guard for uniqueness queries: two z3 variables with the same NAME are + the same variable, so a query that builds 'two independent models' out of + identically-named vars is trivially UNSAT — vacuously green. This asserts + the models really are independent. (A live bug caught by this check during + development; kept as a permanent guard.)""" + seen = set() + for grp in groups: + names = {str(x) for x in grp} + if names & seen: + raise AssertionError( + f"aliased model variables — the query would be vacuous: " + f"{sorted(names & seen)[:4]}" + ) + seen |= names + return True + + +def solve(cons, goal, timeout_ms=120_000): + """UNSAT ⇒ the goal is impossible ⇒ the property holds.""" + s = Solver() + s.set("timeout", timeout_ms) + for c in cons: + s.add(c) + s.add(goal) + t0 = time.perf_counter() + r = s.check() + dt = time.perf_counter() - t0 + model = s.model() if r == sat else None + return r, dt, model + + +# =========================================================================== +# The absorb mode's row-local constraint set, transcribed from blake3.rs. +# +# Every entry names the construct it transcribes. `drop` removes one by name — +# that is how the negative controls are built (discipline #1: a constraint that +# can be removed from the MODEL is falsifiable; one carried by a variable's +# sort is not, and is called out in the README's width census). +# =========================================================================== +def mode_algebra(cons, drop=()): + """blake3.rs:1721-1780 — IS_BIT band, μ = MU_S+MU_A, exclusivity, MU_C, + the boundary lock, and FIRST·END = 0.""" + v = {} + # idx 814 (IS_BIT MU), idx 815..819 (IS_BIT MU_S/MU_A/FIRST/END). + for name in ("MU", "MU_S", "MU_A", "FIRST", "END"): + v[name] = bit(name, cons) if f"is_bit_{name}" not in drop else fe(name, cons) + # MU_C is NOT independently IS_BIT-constrained: it is a free column pinned + # only by `MU_C = MU_A − END`. That it lands in {0,1} is a THEOREM (P3.2), + # and it matters because MU_C is used as a bus MULTIPLICITY. + v["MU_C"] = fe("MU_C", cons) + + # All operands here are in {0,1} (or [0,p) for a dropped IS_BIT), so each + # polynomial's integer value is far below p and ≡0 (mod p) ⟺ = 0. Where an + # IS_BIT is dropped the value can be large, so the congruence form is used. + def rel(expr, lo, hi, name): + if name in drop: + return + cong0(cons, expr, lo, hi) + + rel(v["MU"] - v["MU_S"] - v["MU_A"], -2 * P, P, "mu_partition") # idx 819 + rel(v["MU_S"] * v["MU_A"], 0, P * P, "mode_exclusive") # idx 820 + rel(v["MU_C"] + v["END"] - v["MU_A"], -2 * P, 2 * P, "mu_c_def") # idx 821 + rel((v["FIRST"] + v["END"]) * (1 - v["MU_A"]), -P * P, 2 * P * P, + "boundary_lock") # idx 822 + rel(v["FIRST"] * v["END"], 0, P * P, "no_zero_block_group") # idx 823 + return v + + +def countdown(cons, v, drop=()): + """blake3.rs:1786-1792 (the countdown) + the Zero and IsB20 lookups + (blake3.rs:1463-1481).""" + v["REMAINING"] = fe("REMAINING", cons) + v["REM_DECR"] = fe("REM_DECR", cons) + + # 10f. Zero[REMAINING] -> END, gated MU_A. Both halves of the contract. + if "zero_lookup" not in drop: + if "zero_domain" not in drop: + cons.append(Or(v["MU_A"] == 0, v["REMAINING"] < B20)) + cons.append(Or(v["MU_A"] == 0, + And(v["REMAINING"] == 0, v["END"] == 1), + And(v["REMAINING"] != 0, v["END"] == 0))) + + # 10g. IsB20[REM_DECR · 2^10], gated FIRST. Linearized: the field product + # reduced mod p must land in [0, 2^20). + if "isb20_cap" not in drop: + prod = fe("_capval", cons) + cong0(cons, ABSORB_CAP_SCALE * v["REM_DECR"] - prod, + -P, ABSORB_CAP_SCALE * P) + cons.append(Or(v["FIRST"] == 0, prod < B20)) + + # idx 824. MU_C·(REM_DECR + 1 − REMAINING) = 0. + if "countdown" not in drop: + cons.append(Or(v["MU_C"] == 0, + (v["REM_DECR"] + 1 - v["REMAINING"]) % P == 0)) + return v + + +def framing(cons, v, drop=()): + """blake3.rs:1854-1869 — the interior schedule (t = 0, block_len = 64, + flags only on FIRST) plus the byte range checks that make those word + equations mean what they say. + + ✓ VERIFIED that the 16 bytes of input words 24..27 ARE ByteAlu operands on + every row where the mixing core is live: `run_flow` puts them at v[12..16] + (blake3.rs:334) and round 0's G-calls 0..3 have d = 12,13,14,15, whose first + use is `f.xor(g, 0, vd, a1)` (blake3.rs:350). The mixing core is gated + `MU − END`, which is 1 on exactly the rows these constraints gate. + """ + words = {} + for wname, widx in (("t_lo", 24), ("t_hi", 25), ("block_len", 26), ("flags", 27)): + bytes_ = [] + for b in range(4): + c = fe(f"{wname}_b{b}", cons) + # ByteAlu operand range check, live iff MU − END = 1. + if "byte_range_framing" not in drop: + cons.append(Or(v["MU"] - v["END"] != 1, c < BYTE)) + bytes_.append(c) + words[wname] = bytes_ + v["framing_bytes"] = words + + def word(bs): + return bs[0] + 256 * bs[1] + 65536 * bs[2] + 16777216 * bs[3] + + v["framing_word"] = {k: word(bs) for k, bs in words.items()} + + # idx 844..847: MU_C·(t_lo) = 0, MU_C·(t_hi) = 0, MU_C·(block_len − 64) = 0. + for wname, want in (("t_lo", 0), ("t_hi", 0), ("block_len", 64)): + if f"schedule_{wname}" in drop: + continue + cons.append(Or(v["MU_C"] == 0, (word(words[wname]) - want) % P == 0)) + # idx 847: (MU_C − FIRST)·flags = 0. + if "schedule_flags" not in drop: + cons.append(Or(v["MU_C"] - v["FIRST"] == 0, + word(words["flags"]) % P == 0)) + return v + + +def end_row_cv(cons, drop=()): + """The END row's `h` bytes: pinned as WORDS by the chain receive + (blake3.rs:1453-1457, `word_of_bytes(IN + 4i)`), pinned as BYTES only by the + END-gated AreBytes at blake3.rs:1245-1251. Their bytes are then written to + memory verbatim by the `cv_out` store (blake3.rs:1414-1424). + + This is the sharpest instance of the "assumes there are bytes" class: the + mixing core that range-checks `h` everywhere else is gated OFF on this row. + """ + a, b = [], [] + for i in range(32): + ca = fe(f"cvA_b{i}", cons) + cb = fe(f"cvB_b{i}", cons) + if "arebytes_end_cv" not in drop: + cons.append(ca < BYTE) + cons.append(cb < BYTE) + a.append(ca) + b.append(cb) + # The chain receive pins the 8 WORD expressions — identical for both + # assignments, which is what "the bus delivered one value" means. + for w in range(8): + lo = 4 * w + wa = a[lo] + 256 * a[lo + 1] + 65536 * a[lo + 2] + 16777216 * a[lo + 3] + wb = b[lo] + 256 * b[lo + 1] + 65536 * b[lo + 2] + 16777216 * b[lo + 3] + cong0(cons, wa - wb, -(2**25) * P, (2**25) * P) + return a, b + + +def pointer_add(cons, tag, drop=(), offset=64): + """`emit_add_pair` (templates.rs:334-374) as the chip applies it to + `M_BASE + 64 = M_BASE_INCR` (blake3.rs:1796-1803): two committed carry bits + and the two 32-bit limb identities, with the limbs packed from IsHalfword + halfwords (blake3.rs:1486-1499). + + ⚠ `tag` is not cosmetic. Two z3 variables with the same name ARE the same + variable, so a uniqueness query that builds two models must name them + apart — otherwise it is trivially UNSAT and the check is vacuous. + """ + base, incr = [], [] + for i in range(4): + hb = fe(f"{tag}_mbase_h{i}", cons) + hi_ = fe(f"{tag}_mincr_h{i}", cons) + if "ishalf_mbase" not in drop: + cons.append(hb < HALF) + cons.append(hi_ < HALF) + base.append(hb) + incr.append(hi_) + + def lo(h): + return h[0] + 65536 * h[1] + + def hi(h): + return h[2] + 65536 * h[3] + + c0 = bit(f"{tag}_addpair_c0", cons) + c1 = bit(f"{tag}_addpair_c1", cons) + # ⚠ The quotient bound must hold with IsHalfword DROPPED too, or the model + # silently re-imposes the bound it is meant to remove and the negative + # control cannot flip. A free halfword pair reaches ~2^16·p, so ±2^17·p. + span = (2**17) * P + cong0(cons, lo(base) + offset - lo(incr) - c0 * W32, -span, span) + cong0(cons, hi(base) + c0 - hi(incr) - c1 * W32, -span, span) + return base, incr, (c0, c1) + + +def msg_addressing(cons, tag="a", drop=()): + """blake3.rs — the two ADDRESS conditions the ABI states and the executor + rejects without, added after the adversarial review found the chip accepted + absorbs the executor refuses: + + * x11 is 8-aligned: `FIRST·(M_BASE[0] − 8·Q) = 0` with `IsHalfword[Q]`; + * the message region wraps only on its LAST block: + `(MU_C − NEXT_IS_END)·carry_1(M_BASE + 64 = M_BASE_INCR) = 0`, + with `NEXT_IS_END` the `Zero[REM_DECR]` output. + + Returns the alignment witness, the address limbs and the successor carry. + """ + base, incr, (c0, c1) = pointer_add(cons, tag, drop=drop) + + q = fe(f"{tag}_msg_align_q", cons) + if "ishalf_q" not in drop: + cons.append(q < HALF) + + # ⚠ The alignment equation is a FIELD congruence, not an integer equation, + # and the difference is the entire content of this property. Written as + # `base[0] == 8*q` over the integers the model silently re-imposes the very + # bound the negative control removes: no integer `q` divides an odd base, so + # the vacuous variant would report unsat and look safe. `8·q` reaches ~8p, so + # the quotient window is (-8p, p]. + if "align" not in drop or "align_scaled" in drop: + cong0(cons, base[0] - 8 * q, -8 * P, P) + if "align_scaled" in drop: + # The REJECTED sketch, modelled so its vacuity is machine-checked rather + # than argued: bound Q ONLY through the scaled IsB20 product. + prod = fe(f"{tag}_scaled", cons) + cong0(cons, prod - 128 * q, -128 * P, P) + cons.append(prod < 2**20) + return {"base": base, "incr": incr, "c0": c0, "c1": c1, "q": q} + + +# =========================================================================== +# P1 — compression equivalence, QF-BV. Byte columns are 8-bit bitvectors, +# which IS the AreBytes/ByteAlu range-check contract. +# =========================================================================== +WIDE = 48 # honest intermediates stay < 2^35 + + +class Circuit: + """The chip's mixing core, transcribed from `run_flow` + the eval + constraints (blake3.rs:326-378, 1647-1716). A word is 4 free 8-bit BVs.""" + + def __init__(self, tag): + self.C = [] + self.tag = tag + self.n = 0 + + def _fresh(self, w=8): + v = BitVec(f"{self.tag}_v{self.n}", w) + self.n += 1 + return v + + def fresh_word(self): + return [self._fresh(8) for _ in range(4)] + + def const_word(self, val): + return [BitVecVal((val >> (8 * i)) & 0xFF, 8) for i in range(4)] + + def wval(self, word): + acc = BitVecVal(0, WIDE) + for i in range(4): + acc = acc + ZeroExt(WIDE - 8, word[i]) * BitVecVal(1 << (8 * i), WIDE) + return acc + + def hwval(self, blo, bhi): + return ZeroExt(WIDE - 8, blo) + ZeroExt(WIDE - 8, bhi) * BitVecVal(256, WIDE) + + def word32(self, word): + return Concat(word[3], word[2], word[1], word[0]) + + def fresh_bit(self): + v = self._fresh(8) + self.C.append(Or(v == 0, v == 1)) # μ-gated IS_BIT, μ = 1 on a real row + return v + + def xor(self, A, B): + """ByteAlu[XOR] — output pinned and all three operands byte-ranged.""" + out = self.fresh_word() + for i in range(4): + self.C.append(out[i] == A[i] ^ B[i]) + return out + + def rotr16(self, A): + return [A[2], A[3], A[0], A[1]] # free byte relabel + + def rotr8(self, A): + return [A[1], A[2], A[3], A[0]] # free byte relabel + + def add2(self, A, B): + """Expression carry: a + b − s = 2^32·carry, carry IS_BIT.""" + s = self.fresh_word() + carry = self.fresh_bit() + self.C.append( + self.wval(A) + self.wval(B) + == self.wval(s) + ZeroExt(WIDE - 8, carry) * BitVecVal(1 << 32, WIDE) + ) + return s + + def add3(self, A, B, M): + """Two summed committed carry bits (degree stays ≤ 3 after gating).""" + s = self.fresh_word() + c1 = self.fresh_bit() + c2 = self.fresh_bit() + csum = ZeroExt(WIDE - 8, c1) + ZeroExt(WIDE - 8, c2) + self.C.append( + self.wval(A) + self.wval(B) + self.wval(M) + == self.wval(s) + csum * BitVecVal(1 << 32, WIDE) + ) + return s + + def rot_shift(self, A, n, wrong_amount=False): + """rotr12/rotr7 as inline μ-gated shift identities + halfword swap.""" + r = {12: 4, 7: 9}[n] + (1 if wrong_amount else 0) + xlo = self.hwval(A[0], A[1]) + xhi = self.hwval(A[2], A[3]) + sll_lo, sllc_lo = self.fresh_word()[:2], self.fresh_word()[:2] + sll_hi, sllc_hi = self.fresh_word()[:2], self.fresh_word()[:2] + SLL_lo, SLLC_lo = self.hwval(*sll_lo), self.hwval(*sllc_lo) + SLL_hi, SLLC_hi = self.hwval(*sll_hi), self.hwval(*sllc_hi) + two_r = BitVecVal(1 << r, WIDE) + two16 = BitVecVal(1 << 16, WIDE) + self.C.append(xlo * two_r == SLLC_lo * two16 + SLL_lo) + self.C.append(xhi * two_r == SLLC_hi * two16 + SLL_hi) + Y = self.fresh_word() + self.C.append(self.hwval(Y[0], Y[1]) == SLL_hi + SLLC_lo) + self.C.append(self.hwval(Y[2], Y[3]) == SLL_lo + SLLC_hi) + return Y + + def build_g(self, v, a, b, c, d, mx, my, bug=None): + """One G, exactly the order `run_flow` emits (blake3.rs:349-365).""" + b_op = c if bug == "swap_g_operand" else b + a1 = self.add3(v[a], v[b_op], mx) + x1 = self.xor(v[d], a1) + d1 = self.rotr16(x1) + c1 = self.add2(v[c], d1) + x2 = self.xor(v[b], c1) + b1 = self.rot_shift(x2, 12, wrong_amount=(bug == "rot_wrong_amount")) + a2 = self.add3(a1, b1, my) + x3 = self.xor(d1, a2) + d2 = self.rotr8(x3) + c2 = self.add2(c1, d2) + x4 = self.xor(b1, c2) + b2 = self.rot_shift(x4, 7) + v[a], v[b], v[c], v[d] = a2, b2, c2, d2 + + +# =========================================================================== +# P1a — per-round WIRING, G abstracted. Six queries, one per round. +# =========================================================================== +def check_round_wiring(r, bug=None, timeout_ms=60_000): + """Round r: does the chip hand each G the slots and message words the spec + names for round r? G is an uninterpreted function, identical on both sides, + so this decides the PLUMBING and nothing else.""" + bv32 = BitVecSort(32) + sig = [bv32] * 6 + Gf = [Function(f"G{r}_{o}", *sig, bv32) for o in range(4)] + + state = [BitVec(f"s{r}_{i}", 32) for i in range(16)] + msg = [BitVec(f"m{r}_{i}", 32) for i in range(16)] + + def apply_g(v, a, b, c, d, mx, my): + args = (v[a], v[b], v[c], v[d], mx, my) + outs = [f(*args) for f in Gf] + v[a], v[b], v[c], v[d] = outs + + # --- chip side: `run_flow`'s own schedule recurrence, transcribed --- + sched = list(range(16)) + for _ in range(r): + prev = sched + sched = [prev[p] for p in ref.MSG_PERMUTATION] + if bug == "swap_sched_pair": + sched[0], sched[1] = sched[1], sched[0] + chip = list(state) + for j, (a, b, c, d) in enumerate(ref.G_INDICES): + ia, ib, ic, idd = (a, b, c, d) + if bug == "swap_state_slot" and j == 0: + ib, ic = ic, ib + apply_g(chip, ia, ib, ic, idd, msg[sched[2 * j]], msg[sched[2 * j + 1]]) + + # --- reference side: permute the message r times, then consume in order --- + rmsg = list(msg) + for _ in range(r): + rmsg = ref.permute(rmsg) + want = list(state) + for j, (a, b, c, d) in enumerate(ref.G_INDICES): + apply_g(want, a, b, c, d, rmsg[2 * j], rmsg[2 * j + 1]) + + return solve([], Or(*[chip[i] != want[i] for i in range(16)]), timeout_ms) + + +# =========================================================================== +# P1b — the G circuit's internals against the spec G, free inputs. +# =========================================================================== +def check_g(bug=None, timeout_ms=600_000): + cir = Circuit("g" + (f"_{bug}" if bug else "")) + va, vb, vc, vd = (cir.fresh_word(), cir.fresh_word(), + cir.fresh_word(), cir.fresh_word()) + mx, my = cir.fresh_word(), cir.fresh_word() + v = [None] * 16 + v[0], v[1], v[2], v[3] = va, vb, vc, vd + cir.build_g(v, 0, 1, 2, 3, mx, my, bug=bug) + rv = [cir.word32(va), cir.word32(vb), cir.word32(vc), cir.word32(vd)] + ref.g(ref.BvOps, rv, 0, 1, 2, 3, cir.word32(mx), cir.word32(my)) + goal = Or(*[cir.word32(v[i]) != rv[i] for i in range(4)]) + return solve(cir.C, goal, timeout_ms) + + +# =========================================================================== +# P1c/P1d — the absorb framing's input feed, and the feed-forward that becomes +# the chain payload. Both are round-free, so they are exact and cheap. +# =========================================================================== +def check_input_feed(bug=None, timeout_ms=60_000): + """The 16-word state entering round 0 on an absorbing row, under the values + P2 proves the framing columns hold: v[8..12] = IV, v[12..16] = (0,0,64,fl). + """ + cir = Circuit("feed" + (f"_{bug}" if bug else "")) + h = [cir.fresh_word() for _ in range(8)] + flags = cir.fresh_word() + iv = list(ref.IV) + if bug == "wrong_iv": + iv[0] ^= 1 + bl = 64 ^ (1 if bug == "wrong_block_len" else 0) + chip = ( + [cir.word32(w) for w in h] + + [BitVecVal(x, 32) for x in iv[:4]] + + [BitVecVal(0, 32), BitVecVal(0, 32), BitVecVal(bl, 32), cir.word32(flags)] + ) + want = ref.initial_state( + ref.BvOps, [cir.word32(w) for w in h], + BitVecVal(0, 32), BitVecVal(0, 32), BitVecVal(64, 32), cir.word32(flags), + ) + return solve(cir.C, Or(*[chip[i] != want[i] for i in range(16)]), timeout_ms) + + +def check_feed_forward(bug=None, timeout_ms=60_000): + """out[i] = v[i]^v[i+8], out[i+8] = v[i+8]^h[i]; the chain carries out[0..8] + (blake3.rs:1451, `chain_values(..., cols::OUT)` takes 8 words).""" + cir = Circuit("ff" + (f"_{bug}" if bug else "")) + h = [cir.fresh_word() for _ in range(8)] + v = [cir.fresh_word() for _ in range(16)] + out = [] + for i in range(8): + out.append(cir.xor(v[i], v[i + 8])) + for i in range(8): + out.append(cir.xor(v[i + 8], h[i])) + if bug == "drop_ff_xor": + out[0] = cir.fresh_word() + if bug == "chain_carries_wrong_half": + payload = [cir.word32(out[8 + i]) for i in range(8)] + else: + payload = [cir.word32(out[i]) for i in range(8)] + want = ref.feed_forward(ref.BvOps, + [cir.word32(w) for w in v], + [cir.word32(w) for w in h])[:8] + return solve(cir.C, Or(*[payload[i] != want[i] for i in range(8)]), timeout_ms) + + +# =========================================================================== +# Board +# =========================================================================== +class Board: + def __init__(self): + self.rows = [] + self.ok = True + + def record(self, prop, name, want, res, dt, model=None, note=""): + got = str(res) + good = got == want + self.ok &= good + self.rows.append((prop, name, want, got, dt, good, note)) + mark = "OK " if good else "!! " + print(f" {mark}{name:<46s} {got:<8s} (want {want:<5s}) {dt:7.2f}s {note}") + if model is not None and (VERBOSE or (want == "unsat" and got == "sat")): + print(f" witness: {self.decode(model)}") + return good + + @staticmethod + def decode(model): + keep = {} + for d in model.decls(): + n = d.name() + if n.startswith("_q") or n.startswith("_cap"): + continue + keep[n] = model[d] + items = sorted(keep.items())[:14] + return ", ".join(f"{k}={v}" for k, v in items) + + +def main(): + import z3 + + print("=" * 78) + print("BLAKE3 CHAINED-ABSORB MODE — z3 gate") + print(f"z3 {z3.get_version_string()} | chip: prover/src/tables/blake3.rs") + print("=" * 78) + print(CONTRACTS) + b = Board() + + # ---------------------------------------------------------------- P1 --- + print("\n=== P1 compression equivalence under the absorb framing (QF-BV) ===") + print(" One round at a time. See BINDING SCOPE RULE.") + for r in range(6): + res, dt, m = check_round_wiring(r) + b.record("P1a", f"round {r} wiring (schedule = permute^{r})", "unsat", res, dt, m) + res, dt, m = check_g() + b.record("P1b", "G circuit vs spec G, free inputs", "unsat", res, dt, m, + "covers all 48 G instances") + res, dt, m = check_input_feed() + b.record("P1c", "input feed: IV | t=0 | block_len=64 | flags", "unsat", res, dt, m) + res, dt, m = check_feed_forward() + b.record("P1d", "feed-forward + chain payload = CV (8 words)", "unsat", res, dt, m) + + print("\n negative controls (must be SAT — a green board with a vacuous") + print(" encoding is the fail-open mode discipline #1 exists to close):") + for bug in ("swap_sched_pair", "swap_state_slot"): + res, dt, m = check_round_wiring(3, bug=bug) + b.record("P1a-neg", f"round 3, {bug}", "sat", res, dt) + for bug in ("rot_wrong_amount", "swap_g_operand"): + res, dt, m = check_g(bug=bug) + b.record("P1b-neg", f"G, {bug}", "sat", res, dt) + for bug in ("wrong_iv", "wrong_block_len"): + res, dt, m = check_input_feed(bug=bug) + b.record("P1c-neg", f"feed, {bug}", "sat", res, dt) + for bug in ("drop_ff_xor", "chain_carries_wrong_half"): + res, dt, m = check_feed_forward(bug=bug) + b.record("P1d-neg", f"feed-forward, {bug}", "sat", res, dt) + + # ---------------------------------------------------------------- P3 --- + # (P3 before P2: the mode algebra is what P2's gates are written over.) + print("\n=== P3 mode gating: selectors, Σ modes = μ, no cross-mode bleed (field) ===") + cons = [] + v = mode_algebra(cons) + LEGAL = Or( + And(v["MU"] == 0, v["MU_S"] == 0, v["MU_A"] == 0, # padding + v["FIRST"] == 0, v["END"] == 0, v["MU_C"] == 0), + And(v["MU"] == 1, v["MU_S"] == 1, v["MU_A"] == 0, # single compression + v["FIRST"] == 0, v["END"] == 0, v["MU_C"] == 0), + And(v["MU"] == 1, v["MU_S"] == 0, v["MU_A"] == 1, # absorb, FIRST + v["FIRST"] == 1, v["END"] == 0, v["MU_C"] == 1), + And(v["MU"] == 1, v["MU_S"] == 0, v["MU_A"] == 1, # absorb, interior + v["FIRST"] == 0, v["END"] == 0, v["MU_C"] == 1), + And(v["MU"] == 1, v["MU_S"] == 0, v["MU_A"] == 1, # absorb, END + v["FIRST"] == 0, v["END"] == 1, v["MU_C"] == 0), + ) + res, dt, m = solve(cons, z3.Not(LEGAL)) + b.record("P3.1", "exactly 5 row shapes exist (no sixth)", "unsat", res, dt, m) + + for expr, nm in ( + (v["MU_C"], "MU_C"), + (v["MU"] - v["END"], "MU − END (mixing-core gate)"), + (v["MU_A"] - v["FIRST"], "MU_A − FIRST (chain receive)"), + (v["MU_S"] + v["FIRST"], "MU_S + FIRST (x10 / cv read)"), + ): + res, dt, m = solve(cons, Or(expr < 0, expr > 1)) + b.record("P3.2", f"multiplicity {nm} ∈ {{0,1}}", "unsat", res, dt, m) + + res, dt, m = solve(cons, And(v["MU_S"] == 1, + Or(v["MU_A"] != 0, v["FIRST"] != 0, + v["END"] != 0, v["MU_C"] != 0))) + b.record("P3.3", "single row: every absorb multiplicity is 0", "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["MU_A"] == 1, v["MU_S"] != 0)) + b.record("P3.4", "absorb row: MU_S-gated interactions are off", "unsat", res, dt, m) + + print("\n negative controls:") + for dropped, goal_name, goal in ( + ("boundary_lock", "padding row mints END (a free cv_out write)", + lambda vv: And(vv["MU"] == 0, vv["END"] == 1)), + ("boundary_lock", "single row mints FIRST (a second Ecall receive)", + lambda vv: And(vv["MU_S"] == 1, vv["FIRST"] == 1)), + ("no_zero_block_group", "zero-block group (FIRST = END = 1)", + lambda vv: And(vv["FIRST"] == 1, vv["END"] == 1)), + ("mu_c_def", "MU_C free ⇒ negative multiplicity", + lambda vv: vv["MU_C"] > 1), + ): + c2 = [] + v2 = mode_algebra(c2, drop=(dropped,)) + res, dt, m = solve(c2, goal(v2)) + b.record("P3-neg", f"drop {dropped}: {goal_name}", "sat", res, dt, m) + + # The documented rationale for MU_S·MU_A = 0 is stronger than the algebra + # needs; IS_BIT(MU) + μ = MU_S + MU_A already forbids MU_S = MU_A = 1. + c3 = [] + v3 = mode_algebra(c3, drop=("mode_exclusive",)) + res, dt, m = solve(c3, And(v3["MU_S"] == 1, v3["MU_A"] == 1)) + b.record("P3.5", "MU_S·MU_A = 0 is IMPLIED by IS_BIT(MU) + partition", + "unsat", res, dt, m, "informational — see README") + + # ---------------------------------------------------------------- P4 --- + print("\n=== P4 countdown / END row-local logic + the boundary lock (field) ===") + cons = [] + v = mode_algebra(cons) + countdown(cons, v) + + res, dt, m = solve(cons, And(v["MU_A"] == 1, v["FIRST"] == 1, + Or(v["REMAINING"] < 1, + v["REMAINING"] > ABSORB_MAX_BLOCKS))) + b.record("P4.1", "FIRST row: 1 ≤ REMAINING ≤ 1024 (the cap, in circuit)", + "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["MU_A"] == 1, v["END"] == 1, v["REMAINING"] != 0)) + b.record("P4.2", "no early END (END ⇒ REMAINING = 0)", "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["MU_A"] == 1, v["REMAINING"] == 0, v["END"] != 1)) + b.record("P4.3", "no late END (REMAINING = 0 ⇒ END)", "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["END"] == 1, + Or(v["MU_A"] != 1, v["MU_C"] != 0))) + b.record("P4.4", "END row is inert: MU_A = 1, MU_C = 0 (no send, no core)", + "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["MU_A"] == 1, v["END"] == 0, + Or(v["MU_C"] != 1, + (v["REM_DECR"] + 1 - v["REMAINING"]) % P != 0))) + b.record("P4.5", "compressing row: MU_C = 1 and REM_DECR = REMAINING − 1", + "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["FIRST"] == 1, v["REM_DECR"] >= ABSORB_MAX_BLOCKS)) + b.record("P4.6", "cap cannot be wrapped: REM_DECR < 2^10 on FIRST", + "unsat", res, dt, m) + + print("\n negative controls:") + for dropped, goal_name, goal in ( + ("zero_lookup", "END is free ⇒ end a group early at REMAINING = 7", + lambda vv: And(vv["MU_A"] == 1, vv["END"] == 1, vv["REMAINING"] == 7)), + ("isb20_cap", "no cap ⇒ a group of 2^19 blocks", + lambda vv: And(vv["MU_A"] == 1, vv["FIRST"] == 1, + vv["REMAINING"] == 2**19)), + ("countdown", "no countdown ⇒ REM_DECR unrelated to REMAINING", + lambda vv: And(vv["MU_A"] == 1, vv["END"] == 0, + vv["REMAINING"] == 5, vv["REM_DECR"] == 99)), + ("zero_domain", "★ Zero's DOMAIN bound dropped ⇒ the cap wraps mod p", + lambda vv: And(vv["MU_A"] == 1, vv["FIRST"] == 1, + vv["REM_DECR"] >= ABSORB_MAX_BLOCKS)), + ): + c2 = [] + v2 = mode_algebra(c2) + countdown(c2, v2, drop=(dropped,)) + res, dt, m = solve(c2, goal(v2)) + b.record("P4-neg", f"drop {dropped}: {goal_name}", "sat", res, dt, m) + + # ---------------------------------------------------------------- P6 --- + print("\n=== P6 the ABI's ADDRESS conditions, in circuit (field) ===") + print(" Added after the review found the chip accepted absorbs the") + print(" executor rejects: an unaligned x11, and a wrapping message region.") + + # --- 6a. alignment ----------------------------------------------------- + cons = [] + a = msg_addressing(cons, "p6a") + # An unaligned low halfword is unreachable: 8·Q is even for every Q, and Q + # is halfword-bounded, so the equality is over integers. + res, dt, m = solve(cons, a["base"][0] % 8 != 0) + b.record("P6.1", "FIRST row: M_BASE[0] ≡ 0 (mod 8) — x11 is 8-aligned", + "unsat", res, dt, m) + res, dt, m = solve(cons, a["base"][0] == 8) + b.record("P6.2", "…and that model is consistent (non-vacuity)", "sat", res, dt, m) + + print("\n negative controls:") + c2 = [] + a2 = msg_addressing(c2, "p6b", drop=("align",)) + res, dt, m = solve(c2, a2["base"][0] == 1) + b.record("P6-neg", "drop the alignment constraint: an odd message base", + "sat", res, dt, m) + + # ★ The rejected sketch. `IsB20[Q·2^7]` looks like the block cap's trick and + # is VACUOUS: with Q bounded only through the scaled product a prover takes + # Q = M_BASE[0]·8⁻¹ mod p, and Q·2^7 = M_BASE[0]·16 < 2^20 for ANY halfword. + c3 = [] + a3 = msg_addressing(c3, "p6c", drop=("ishalf_q", "align", "align_scaled")) + res, dt, m = solve(c3, a3["base"][0] == 1) + b.record("P6-neg", "★ the SCALED form IsB20[Q·2^7] admits an odd base " + "(why the shipped chip uses IsHalfword[Q])", + "sat", res, dt, m) + + # --- 6b. the message region wraps only on its last block --------------- + cons = [] + v = mode_algebra(cons) + countdown(cons, v) + a = msg_addressing(cons, "p6d") + nie = bit("p6d_next_is_end", cons) + # Zero[REM_DECR] -> NEXT_IS_END, on compressing rows. + cons.append(Implies(v["MU_C"] == 1, nie == If(v["REM_DECR"] == 0, 1, 0))) + cons.append(nie * (1 - v["MU_C"]) == 0) + # (MU_C − NEXT_IS_END) · carry_1 = 0 + cons.append((v["MU_C"] - nie) * a["c1"] == 0) + + res, dt, m = solve(cons, And(v["MU_C"] == 1, v["REM_DECR"] != 0, a["c1"] == 1)) + b.record("P6.3", "a NON-final block's successor address cannot wrap", + "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["MU_C"] == 1, v["REM_DECR"] == 0, a["c1"] == 1)) + b.record("P6.4", "…but the FINAL block's may (a message may end at 2^64)", + "sat", res, dt, m) + + print("\n negative controls:") + c4 = [] + v4 = mode_algebra(c4) + countdown(c4, v4) + a4 = msg_addressing(c4, "p6e") + nie4 = bit("p6e_next_is_end", c4) + c4.append(Implies(v4["MU_C"] == 1, nie4 == If(v4["REM_DECR"] == 0, 1, 0))) + c4.append(nie4 * (1 - v4["MU_C"]) == 0) + # gate dropped: no no-overflow at all — the pre-review chip. + res, dt, m = solve(c4, And(v4["MU_C"] == 1, v4["REM_DECR"] == 5, a4["c1"] == 1)) + b.record("P6-neg", "drop the wrap gate: a mid-group block wraps to address 0", + "sat", res, dt, m) + + # ---------------------------------------------------------------- P2 --- + print("\n=== P2 flags schedule / interior framing (field) ===") + cons = [] + v = mode_algebra(cons) + countdown(cons, v) + framing(cons, v) + fw = v["framing_word"] + fb = v["framing_bytes"] + + interior = And(v["MU_A"] == 1, v["END"] == 0, v["FIRST"] == 0) + res, dt, m = solve(cons, And(interior, + Or(*[fb["t_lo"][i] != 0 for i in range(4)], + *[fb["t_hi"][i] != 0 for i in range(4)]))) + b.record("P2.1", "interior row: counter bytes are all zero", "unsat", res, dt, m) + res, dt, m = solve(cons, And(interior, + Or(fb["block_len"][0] != 64, + *[fb["block_len"][i] != 0 for i in (1, 2, 3)]))) + b.record("P2.2", "interior row: block_len bytes are exactly (64,0,0,0)", + "unsat", res, dt, m) + res, dt, m = solve(cons, And(interior, + Or(*[fb["flags"][i] != 0 for i in range(4)]))) + b.record("P2.3", "★ interior row cannot carry ANY flag byte " + "(the forged-shorter-message class)", "unsat", res, dt, m) + res, dt, m = solve(cons, And(v["MU_A"] == 1, v["FIRST"] == 1, v["END"] == 0, + fw["flags"] >= W32)) + b.record("P2.4", "FIRST row: flags word < 2^32 (fits the one-word column)", + "unsat", res, dt, m) + + print("\n negative controls:") + c2 = [] + v2 = mode_algebra(c2) + countdown(c2, v2) + framing(c2, v2, drop=("schedule_flags",)) + res, dt, m = solve(c2, And(v2["MU_A"] == 1, v2["END"] == 0, v2["FIRST"] == 0, + v2["framing_bytes"]["flags"][0] == 0x0A)) + b.record("P2-neg", "drop the flags gate: interior block carries CHUNK_END|ROOT", + "sat", res, dt, m, "= a digest for a PREFIX of the message") + c2 = [] + v2 = mode_algebra(c2) + countdown(c2, v2) + framing(c2, v2, drop=("schedule_block_len",)) + res, dt, m = solve(c2, And(v2["MU_A"] == 1, v2["END"] == 0, + v2["framing_bytes"]["block_len"][0] == 7)) + b.record("P2-neg", "drop block_len = 64: absorb a block under a short framing", + "sat", res, dt, m) + + # ---------------------------------------------------------------- P5 --- + print("\n=== P5 byte/dword width assumptions — 'it assumes there are bytes' (field) ===") + # 5a. The END row's cv bytes: pinned as words by the bus, as bytes only by + # the END-gated AreBytes, then written to memory verbatim. + cons = [] + a, bb = end_row_cv(cons) + distinct_names(a, bb) + res, dt, m = solve(cons, Or(*[a[i] != bb[i] for i in range(32)])) + b.record("P5.1", "END-row cv_out bytes are UNIQUE given the chain words", + "unsat", res, dt, m) + # Non-vacuity of the uniqueness model itself: the constraints must admit + # SOME assignment, or the UNSAT above would be meaningless. + res, dt, _ = solve(cons, a[0] == 200) + b.record("P5.1", "…and that model is consistent (non-vacuity)", "sat", res, dt) + c2 = [] + a2, b2 = end_row_cv(c2, drop=("arebytes_end_cv",)) + res, dt, m = solve(c2, Or(*[a2[i] != b2[i] for i in range(32)])) + b.record("P5-neg", "★ drop the END-row AreBytes: a non-canonical cv_out " + "writes a 'byte' ≥ 256 to memory", "sat", res, dt, m) + + # 5b. The framing words mean what they say only because their bytes are + # ByteAlu operands. Same query as P2.2, with the sort-carried bound + # removed — the one gap a full board of green controls can hide. + c2 = [] + v2 = mode_algebra(c2) + countdown(c2, v2) + framing(c2, v2, drop=("byte_range_framing",)) + res, dt, m = solve(c2, And(v2["MU_A"] == 1, v2["END"] == 0, + v2["framing_bytes"]["block_len"][0] != 64)) + b.record("P5-neg", "★ drop the byte range on block_len: 64 has other " + "decompositions mod p", "sat", res, dt, m) + + # 5c. Pointer limbs: IsHalfword is what makes `M_BASE + 64` a function. + for dropped, want, label in ( + ((), "unsat", "M_BASE_INCR is unique given M_BASE (IsHalfword present)"), + (("ishalf_mbase",), "sat", + "drop IsHalfword: the next block's address is ambiguous"), + ): + both = [] + base, incr, _ = pointer_add(both, "A", drop=dropped) + base2, incr2, _ = pointer_add(both, "B", drop=dropped) + distinct_names(base + incr, base2 + incr2) + both += [base[i] == base2[i] for i in range(4)] + res, dt, m = solve(both, Or(*[incr[i] != incr2[i] for i in range(4)])) + b.record("P5.2" if want == "unsat" else "P5-neg", label, want, res, dt, m) + + # ------------------------------------------------ positive control ------ + print("\n=== POSITIVE CONTROL (non-vacuity: an honest row must be admissible) ===") + cons = [] + v = mode_algebra(cons) + countdown(cons, v) + framing(cons, v) + honest = And(v["MU"] == 1, v["MU_S"] == 0, v["MU_A"] == 1, v["FIRST"] == 1, + v["END"] == 0, v["MU_C"] == 1, v["REMAINING"] == 3, + v["REM_DECR"] == 2, + v["framing_bytes"]["block_len"][0] == 64, + v["framing_bytes"]["flags"][0] == 0x01) + res, dt, m = solve(cons, honest) + b.record("POS", "honest FIRST row of a 3-block group is SAT", "sat", res, dt) + honest_end = And(v["MU"] == 1, v["MU_A"] == 1, v["END"] == 1, v["FIRST"] == 0, + v["MU_C"] == 0, v["REMAINING"] == 0) + res, dt, m = solve(cons, honest_end) + b.record("POS", "honest END row is SAT", "sat", res, dt) + + # ------------------------------------------------------------ verdict --- + print("\n" + "=" * 78) + total = sum(r[4] for r in b.rows) + bad = [r for r in b.rows if not r[5]] + print(f"VERDICT: {'PASS' if b.ok else 'FAIL'} " + f"({len(b.rows) - len(bad)}/{len(b.rows)} queries as expected, " + f"{total:.1f}s total)") + if bad: + print("\nUNEXPECTED:") + for prop, name, want, got, dt, _, _ in bad: + print(f" [{prop}] {name}: got {got}, wanted {want}") + print("\n 'unknown' = z3 timeout, NOT a finding. 'sat' where 'unsat'") + print(" was wanted IS a finding — decode the witness above.") + print("=" * 78) + sys.exit(0 if b.ok else 1) + + +if __name__ == "__main__": + main() diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 7401485cf..57e28da36 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -6,38 +6,98 @@ license.workspace = true [features] default = ["parallel"] -parallel = ["stark/parallel", "math/parallel", "crypto/parallel", "ecsm/parallel", "dep:rayon"] -cuda = ["stark/cuda"] +parallel = [ + "stark/parallel", + "math/parallel", + "crypto/parallel", + "multilinear/parallel", + "ecsm/parallel", + "dep:rayon", +] +cuda = ["stark/cuda", "multilinear/cuda"] test-cuda-faults = ["cuda", "stark/test-cuda-faults"] debug-checks = ["stark/debug-checks"] instruments = ["stark/instruments"] # GPU profiling build (Nsight): implies cuda — the toolkit always traces the GPU prover. nvtx = ["cuda", "instruments", "stark/nvtx"] profile-markers = ["stark/profile-markers"] -disk-spill = ["stark/disk-spill"] -# Host-only verify-hash counter for `test_count_recursion_hashes` (see `hash_metrics`). +# The BLAKE3 round count, for the `LFM_HASH` arm and — since the P-a flip — for +# the commitment backends the whole prover commits under. Off = 7 rounds +# (standard BLAKE3, externally anchored); on = the 6-round A6R variant, which is +# the shipping arm and is ON BY DEFAULT via `crypto`'s default feature set. +# It is a compile-time knob rather than a parameter because the chip's column +# layout is `8 · rounds` G-blocks wide and `hash::num_columns` is a `const fn`. +# +# Naming this feature explicitly is therefore redundant on a default build. It +# stays because it is the knob `make lint`'s matrix and any `--no-default-features` +# consumer need in order to name the round count at all. +# +# It FORWARDS to `crypto`, where the primitive now lives, rather than declaring a +# second knob: the chip and the commitment backends must be at the same round +# count or a build commits under one hash and prices another. `crypto`'s is the +# only definition. The remaining way to desync them is enabling `crypto`'s alone, +# which the `SOCKET_ROUNDS == BLAKE3_ROUNDS` assertion in `lfm::blake3_socket` +# catches at compile time. +blake3-6round = ["crypto/blake3-6round"] +# Forwards `crypto`'s guest-side chained-absorb switch. See that feature's note +# for why it stays separate from `blake3-6round`: a guest emitting the absorb +# ecall needs the prover's absorb mode present to receive it. +blake3-absorb = ["crypto/blake3-absorb"] +# Trace-shape profile emission (segment-profile JSONL). Host-only: pulls +# in serde_json, which the RISC-V recursion guest must not carry. +shape-profile = ["dep:serde", "dep:serde_json"] +# Host-only verify-hash counter for `test_count_recursion_hashes` (see `crypto::hash_metrics`). hash-metrics = ["crypto/hash-metrics"] [dependencies] stark = { path = "../crypto/stark" } +# The multilinear argument: the WHIR proving path (`multilinear_prove`). +multilinear = { path = "../crypto/multilinear" } crypto = { path = "../crypto/crypto" } math = { path = "../crypto/math" } executor = { path = "../executor" } ecsm = { path = "../crypto/ecsm" } rayon = { version = "1.8.0", optional = true } -sysinfo = { version = "0.31", default-features = false, features = ["system"] } log = "0.4" digest = "0.10.7" # pointer_width_64: 32-bit rel-ptrs cap an archive at ~2 GiB, which large # continuation proofs exceed. Keep in sync across all proof-format crates. rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } +serde = { version = "1.0", features = ["derive"], optional = true } +serde_json = { version = "1.0", optional = true } + +# Phase B hands a finished group's records pages back to the OS (`madvise`). +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" [dev-dependencies] env_logger = "*" +# Test-only: tests that pin a guest guard on its ELF's sha256, because two +# builds of the same guest from the same sources share a name and differ in +# bytes. +sha2 = { version = "0.10", default-features = false } +# Test-only: `serde_json` is behind `shape-profile` for the guest's sake, and a +# proof round-trip test should not have to turn that on. +serde_json = "1.0" +rkyv = { version = "0.8.10", default-features = false, features = [ + "alloc", + "bytecheck", + "aligned", + "pointer_width_64", +] } criterion = { version = "0.5", default-features = false } tikv-jemallocator = "0.6" tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] } +# `alloc_purge`'s `arena..purge`, a control that takes no value, which +# `tikv-jemalloc-ctl`'s raw writes cannot express. No default features: the +# allocator's build stays the one `tikv-jemallocator` and `-ctl` already select. +tikv-jemalloc-sys = { version = "0.6", default-features = false } tiny-keccak = { version = "2.0", features = ["keccak"] } +# The external anchor for `lfm::blake3` at 7 rounds and for the `LFM_HASH` +# BLAKE3 socket (`blake3::hash(a ‖ b ‖ "LFMC")`). Test-only on purpose: the +# machine never calls it, so it can never become a second implementation that +# the chip is silently checked against instead of the vectors. +blake3 = { version = "1.8.5", default-features = false, features = ["std", "pure"] } # Enable stark's test-utils so cross-crate tests can reach # `compute_precomputed_commitment_for_testing`. Only active under cargo test/bench. stark = { path = "../crypto/stark", features = ["test-utils"] } diff --git a/prover/src/alloc_purge.rs b/prover/src/alloc_purge.rs new file mode 100644 index 000000000..2674ff812 --- /dev/null +++ b/prover/src/alloc_purge.rs @@ -0,0 +1,1177 @@ +//! Returning the allocator's freed pages to the OS at a phase boundary +//! (`LAMBDA_VM_ALLOC_PURGE`; by default only where memory is short). +//! +//! The posture's jemalloc never purges (`dirty_decay_ms:-1`), and a phase +//! rarely reuses the pages the one before it freed: they sit in other threads' +//! arenas or in size classes the next phase does not ask for. So a block's RSS +//! ratchets phase by phase. At the spill-on median (BIG 584) the live peaks are +//! flat, ≈ 36 GiB in the base, the tree and the verifier (decay 0), while +//! VmRSS climbs 52.75 → 56.56 → 60.28 GiB. One +//! `arena..purge` hands every arena's dirty pages back, so +//! the next phase starts near its live set. Decay 0 does the same continuously +//! at +54 s of base. +//! +//! [`purge_point`] is the call a pipeline makes at a named boundary. The lib +//! does not own the allocator, the binary does: it purges through the +//! [`AllocatorHooks`] the binary [`install`]s (the CLI installs jemalloc's), or, +//! in the lib's own test builds, through the jemalloc their `lib.rs` installs. +//! Anywhere else the call does nothing. The same hooks give the allocator's +//! statistics ([`stats`]) to the readers that need them: the block's memory +//! ledger and the tree's late leaf emission. +//! +//! By default (`auto`) a purge runs only once the block's memory is short: at +//! phase A's end and the base's end, after the spill's queue budgets armed +//! ([`note_memory_pressure`], `block.rs`). At the p90 block 25481021 those two +//! purges took the base's phase-B peak 119.79 → 116.60 GiB and level 0's start +//! ≈ 115 → 55.6 GiB, where without them level 0 was OOM-killed (BIG 120, 1215); +//! a block that fits never arms, so it pays nothing. +//! +//! A block that proves with no disk ([`note_no_disk`]) notes memory pressure +//! from a forecast of the disk path's growth, and then never grows that way: +//! on a 118 GiB host the p90 block stayed under 68 GiB and still paid both +//! purges, 4.1 s of their own time (2.4 s at phase A's end, 1.6 s at the +//! base's). There `auto` purges at its two points only when the +//! resident set has come within the block's headroom of its pressure line +//! ([`has_room`], `LAMBDA_VM_ALLOC_PURGE_ROOM`): a 64 GiB host purges at both +//! as before, a host with room at neither. +//! +//! Phase B's time does not follow the purge. It follows the leaf programs the +//! tree proves beside the base. Under the tree budget's resident reading a +//! block that skips the purges keeps fewer of them there, because that reading +//! counts the pages the allocator still holds: at p90 on that host 13 of 26 +//! stay beside the base, phase B is 3.4 s shorter and the tree 2.7 s longer. +//! With all 26 beside the base and no purge, phase B is as long as it is with +//! the purges. The budget's default reading takes those pages off +//! ([`crate::lfm::program_budget::READING_DEFAULT`]). + +/// The allocator's statistics, in bytes (jemalloc's `stats.*`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AllocStats { + /// Live: what the program asked for and has not freed. + pub allocated: usize, + /// In active pages. + pub active: usize, + /// Held in pages: live, freed-and-dirty, and metadata (an upper bound: a + /// fresh extent counts before it is touched). + pub resident: usize, + pub mapped: usize, + pub retained: usize, +} + +/// The binary's allocator, as the lib may ask it: its statistics, and a purge +/// of every arena's dirty pages (`true` when it ran). +#[derive(Clone, Copy, Debug)] +pub struct AllocatorHooks { + pub stats: fn() -> Option, + pub purge_all: fn() -> bool, +} + +static HOOKS: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// The binary installs its allocator's hooks, once, at startup. Returns `false` +/// when hooks were already installed (the first stay). +pub fn install(hooks: AllocatorHooks) -> bool { + HOOKS.set(hooks).is_ok() +} + +/// The installed hooks, else the lib's own test build's jemalloc. +fn hooks() -> Option { + HOOKS.get().copied().or(BUILT_IN) +} + +/// The lib's test build installs jemalloc as its global allocator (`lib.rs`). +#[cfg(test)] +const BUILT_IN: Option = Some(AllocatorHooks { + stats: jemalloc_stats, + purge_all: jemalloc_purge_all, +}); + +/// Outside the lib's tests the binary owns the allocator: nothing built in. +#[cfg(not(test))] +const BUILT_IN: Option = None; + +/// The allocator's statistics, when the binary's hooks (or the lib's test +/// build) can read them. +pub fn stats() -> Option { + hooks().and_then(|h| (h.stats)()) +} + +/// What one purge did. +#[derive(Clone, Copy, Debug)] +pub struct Purge { + /// The purge's own wall time. + pub secs: f64, + /// jemalloc's resident bytes before and after (live, dirty and metadata). + pub resident_before: usize, + pub resident_after: usize, +} + +/// `LAMBDA_VM_ALLOC_PURGE`: `auto` (and unset or empty) purges at +/// [`AUTO_POINTS`] once memory is short ([`note_memory_pressure`]), a no-disk +/// block with room excepted ([`ROOM_ENV`]); `off` +/// purges nowhere; `all`, or a list of the boundary names [`purge_point`] is +/// called with, separated by commas or dots (`base.tree`, for job specs that +/// split their knobs on commas), purges there regardless. The block's points: +/// `phase-a` (phase A done, `block.rs`), `base` (the base proved) and `tree` +/// (the top proved, before the block verifier), the last two in the +/// whole-block harness. +pub const ALLOC_PURGE_ENV: &str = "LAMBDA_VM_ALLOC_PURGE"; + +/// The points `auto` purges at under memory pressure. `tree` is not one: it +/// trims the block verifier's start, not the proof's peak. +const AUTO_POINTS: [&str; 2] = ["phase-a", "base"]; + +/// Whether the block in progress found its memory short. +static PRESSURE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// The block's memory is short (its spill's queue budgets armed): `auto` +/// purges at its next points. +pub fn note_memory_pressure() { + PRESSURE.store(true, std::sync::atomic::Ordering::Relaxed); +} + +/// A new block starts: no pressure seen yet, and no word on its disk. +pub fn clear_memory_pressure() { + PRESSURE.store(false, std::sync::atomic::Ordering::Relaxed); + note_no_disk(None); +} + +/// What `auto` weighs at its points in a block that proves with no disk (with +/// live regeneration, which drops what it can rebuild): the +/// line past which the block's memory is short (the block's pressure share of +/// its host target) and what the block may still add past a boundary (its +/// regeneration window's ceiling). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NoDisk { + pub line: u64, + pub headroom: u64, +} + +/// The block in progress proves with no disk, when it does. +static NO_DISK: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// The block says whether it proves with no disk (`Some`), once it knows: +/// `auto` then purges at its points only without room ([`has_room`]). `None`, +/// a block without live regeneration, purges there whenever memory is short. +pub fn note_no_disk(no_disk: Option) { + *NO_DISK.lock().unwrap_or_else(|e| e.into_inner()) = no_disk; +} + +/// `LAMBDA_VM_ALLOC_PURGE_ROOM`: `auto` (unset or empty, the default) lets a +/// no-disk block with room skip `auto`'s purges ([`has_room`]); `off` purges +/// at them whenever memory is short, as a block without live regeneration +/// does. +pub const ROOM_ENV: &str = "LAMBDA_VM_ALLOC_PURGE_ROOM"; + +/// What [`ROOM_ENV`] says. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RoomSetting { + Auto, + Off, +} + +/// A value of [`ROOM_ENV`]; nonsense is an error. +pub fn parse_room(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "auto") => Ok(RoomSetting::Auto), + Some("off") => Ok(RoomSetting::Off), + Some(v) => Err(format!("{ROOM_ENV} must be auto or off, got `{v}`")), + } +} + +/// Whether a no-disk block has room at a boundary: its resident set and its +/// headroom together stay under its line. A block without live regeneration +/// has none to weigh, and neither has one whose resident set cannot be read: both purge. +/// +/// The p90 block, VmRSS at phase A's end and at the base's end (ZEN5 096): on a +/// 118 GiB host 66.8 and 68.2 GiB against a line of 92.0, room at both; on a +/// 64 GiB host 36.1 and 45.6 GiB (the median 33.1 and 33.2) against 45.9, room +/// at neither. Without the purge at phase A's end that host admitted 4 leaf +/// programs beside the base where it admitted 24, and proved 2.2 s slower +/// (the tree budget's resident reading, the default then). +pub(crate) fn has_room(no_disk: Option, resident: Option) -> bool { + match (no_disk, resident) { + (Some(no_disk), Some(resident)) => resident.saturating_add(no_disk.headroom) < no_disk.line, + _ => false, + } +} + +/// What the knob says at a point. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Decision { + Purge, + /// `auto` at one of its points, with no memory pressure. + Skip, + /// `auto` at one of its points, under memory pressure, in a no-disk block + /// with room. + Room, + /// Not a point the knob purges at. + No, +} + +fn decide(setting: Option<&str>, point: &str, pressure: bool, room: bool) -> Decision { + match setting.map(str::trim) { + None | Some("") | Some("auto") if AUTO_POINTS.contains(&point) => { + if !pressure { + Decision::Skip + } else if room { + Decision::Room + } else { + Decision::Purge + } + } + None | Some("") | Some("auto") | Some("off") => Decision::No, + Some(setting) if names_point(setting, point) => Decision::Purge, + Some(_) => Decision::No, + } +} + +/// Whether `point` is one of the boundaries `setting` names. +fn names_point(setting: &str, point: &str) -> bool { + setting + .split([',', '.']) + .map(str::trim) + .any(|p| p == "all" || p == point) +} + +/// At the boundary `point`, purge every jemalloc arena's dirty pages when +/// [`ALLOC_PURGE_ENV`] says so ([`decide`]), and print one `ALLOC PURGE` line +/// (the wall time and jemalloc's resident bytes before and after, or why +/// `auto` skipped it). Returns what it did, or `None` when it did not purge or +/// the build cannot. +pub fn purge_point(point: &str) -> Option { + let setting = std::env::var(ALLOC_PURGE_ENV).ok(); + let pressure = PRESSURE.load(std::sync::atomic::Ordering::Relaxed); + let no_disk = match parse_room(std::env::var(ROOM_ENV).ok().as_deref()) { + Ok(RoomSetting::Auto) => *NO_DISK.lock().unwrap_or_else(|e| e.into_inner()), + Ok(RoomSetting::Off) => None, + Err(why) => { + eprintln!("ALLOC PURGE {point}: {why}; the room is not weighed"); + None + } + }; + let resident = crate::lfm::program_budget::resident_bytes(); + let room = has_room(no_disk, resident); + let gib = |b: u64| b as f64 / (1u64 << 30) as f64; + match decide(setting.as_deref(), point, pressure, room) { + Decision::No => return None, + Decision::Skip => { + eprintln!("ALLOC PURGE {point}: skipped (auto, no memory pressure)"); + return None; + } + Decision::Room => { + if let (Some(no_disk), Some(resident)) = (no_disk, resident) { + eprintln!( + "ALLOC PURGE {point}: skipped (auto, no disk, the host has room; VmRSS {:.2} + \ + {:.2} of the line {:.2} GiB)", + gib(resident), + gib(no_disk.headroom), + gib(no_disk.line) + ); + } + return None; + } + Decision::Purge => {} + } + let purge = purge_all_arenas()?; + let gib = |b: usize| b as f64 / (1u64 << 30) as f64; + eprintln!( + "ALLOC PURGE {point}: {:.3}s · jemalloc resident {:.2} → {:.2} GiB", + purge.secs, + gib(purge.resident_before), + gib(purge.resident_after) + ); + Some(purge) +} + +/// `LAMBDA_VM_TREE_LEVEL_PURGE`: a purge after each level of the block tree. +/// `auto` (unset or empty, the default) purges after a level only when the +/// block's memory is short ([`note_memory_pressure`]) and the process's +/// resident set has reached [`LEVEL_PURGE_SHARE`] of the host target, so a +/// host with room never purges; `off` never purges there; `always` purges after +/// every level (a measurement arm). A tree level frees what it allocated — +/// its programs, its proofs' working sets — into pages the next level rarely +/// reuses: at the p90 block on a 74 GiB emulated host the recursion's VmRSS +/// peaked 12.9–30 GiB over its live heap (BIG 662, decay 0 against the +/// posture, I-MEMFIT §1.7b). +pub const LEVEL_PURGE_ENV: &str = "LAMBDA_VM_TREE_LEVEL_PURGE"; + +/// `auto` purges after a level once VmRSS reaches this share of the host +/// target. The p90 block's recursion reaches ≈ 94 GiB against a 110.7 GiB +/// target on a 128 GiB host (85 %), so the purges stay off there. +pub const LEVEL_PURGE_SHARE: f64 = 0.9; + +/// What [`LEVEL_PURGE_ENV`] says. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LevelPurge { + Auto, + Off, + Always, +} + +/// A value of [`LEVEL_PURGE_ENV`]; nonsense is an error. +pub fn parse_level_purge(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "auto") => Ok(LevelPurge::Auto), + Some("off") => Ok(LevelPurge::Off), + Some("always") => Ok(LevelPurge::Always), + Some(v) => Err(format!( + "{LEVEL_PURGE_ENV} must be auto, off or always, got `{v}`" + )), + } +} + +/// Whether to purge after a level: the setting, the block's memory pressure, +/// and the resident set against the target (no reading: no purge under +/// `auto`). `Err` says why `auto` skipped it. +fn level_decision( + setting: LevelPurge, + pressure: bool, + resident: Option, + target: u64, +) -> Result<(), &'static str> { + match setting { + LevelPurge::Off => Err("off"), + LevelPurge::Always => Ok(()), + LevelPurge::Auto if !pressure => Err("auto, no memory pressure"), + LevelPurge::Auto => match resident { + Some(r) if r as f64 >= LEVEL_PURGE_SHARE * target as f64 => Ok(()), + Some(_) => Err("auto, the host has room"), + None => Err("auto, no resident reading"), + }, + } +} + +/// After the block tree's level `level` (0 = the leaves): purge every arena +/// when [`LEVEL_PURGE_ENV`] says so against the host `target`, and print one +/// `ALLOC PURGE level-N` line through `say` (what it did, or why `auto` +/// skipped it; nothing for `off`). Returns the purge when it ran. +pub fn purge_after_level(level: usize, target: u64, say: &dyn Fn(&str)) -> Option { + let setting = match parse_level_purge(std::env::var(LEVEL_PURGE_ENV).ok().as_deref()) { + Ok(s) => s, + Err(why) => { + say(&format!("ALLOC PURGE level-{level}: {why}; not purged")); + return None; + } + }; + let pressure = PRESSURE.load(std::sync::atomic::Ordering::Relaxed); + let resident = crate::lfm::program_budget::resident_bytes(); + let gib = |b: u64| b as f64 / (1u64 << 30) as f64; + if let Err(why) = level_decision(setting, pressure, resident, target) { + if setting != LevelPurge::Off { + say(&format!( + "ALLOC PURGE level-{level}: skipped ({why}; VmRSS {} of the target {:.2} GiB)", + resident.map_or("unread".to_string(), |r| format!("{:.2}", gib(r))), + gib(target) + )); + } + return None; + } + let purge = purge_all_arenas()?; + say(&format!( + "ALLOC PURGE level-{level}: {:.3}s · jemalloc resident {:.2} → {:.2} GiB", + purge.secs, + gib(purge.resident_before as u64), + gib(purge.resident_after as u64) + )); + Some(purge) +} + +/// `LAMBDA_VM_ALLOC_WATERMARK`: a monitor beside the whole block that purges +/// every arena when the block's memory is short ([`note_memory_pressure`]), +/// the process's resident set has reached [`WATERMARK_SHARE`] of the host +/// target, and the allocator holds at least [`WATERMARK_MIN_PURGEABLE`] a +/// purge can return ([`purgeable`]), at most once per [`WATERMARK_SPACING`] +/// after a fire that returned [`WATERMARK_MIN_RETURNED`] and once per +/// [`WATERMARK_RETRY`] after one that returned less ([`watermark_spacing`]). +/// `auto` (unset or empty, the default) runs it; `off` does not. It reaches the +/// pages a phase frees and does not reuse *inside* the phase, which no +/// boundary point can: at the p90 block on a 48 GiB emulated host the binding +/// peaks were phase A's end and level 1's end, both retention made after the +/// last purge point (BIG 683, I-MEMFIT §5.1–§6). A host with room never +/// reaches the share, so it never fires there. Each fire prints one +/// `ALLOC WATERMARK fire` line; [`Watermark::finish`] prints the count. +pub const WATERMARK_ENV: &str = "LAMBDA_VM_ALLOC_WATERMARK"; + +/// The watermark: this share of the host target (the level purges' share, +/// [`LEVEL_PURGE_SHARE`]). +pub const WATERMARK_SHARE: f64 = LEVEL_PURGE_SHARE; + +/// The least purgeable bytes ([`purgeable`]) worth a purge. +pub const WATERMARK_MIN_PURGEABLE: u64 = 4 << 30; + +/// How often the monitor reads the host. +const WATERMARK_TICK: std::time::Duration = std::time::Duration::from_millis(500); + +/// The least time between a fire that returned [`WATERMARK_MIN_RETURNED`] and +/// the next. +pub const WATERMARK_SPACING: std::time::Duration = std::time::Duration::from_secs(5); + +/// A fire that returned at least this starts the full [`WATERMARK_SPACING`]. +pub const WATERMARK_MIN_RETURNED: u64 = 1 << 30; + +/// The least time between a fire that returned less than +/// [`WATERMARK_MIN_RETURNED`] and the next: it does not hold the next fire +/// back for the full spacing, but the monitor cannot spin purges that find +/// little, each of which takes every arena's lock. +pub const WATERMARK_RETRY: std::time::Duration = std::time::Duration::from_secs(1); + +/// The highest-VmRSS reading the watermark monitor took since the window was +/// last read ([`peak_window`]): VmRSS, jemalloc's allocated, active and +/// resident bytes, and the seconds since the monitor started. +#[derive(Clone, Copy, Debug, Default)] +struct PeakSample { + rss: u64, + allocated: u64, + active: u64, + resident: u64, + at: f64, +} + +static PEAK: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// The window's highest-VmRSS reading in words, and a new window; `None` +/// without a running monitor (or before its first reading). A phase's +/// `ALLOC PEAK` line: its live heap (allocated) against what the allocator holds +/// (resident) at its highest VmRSS, read every [`WATERMARK_TICK`], and of +/// that, what a purge could return (resident − active, [`purgeable`]). +pub fn peak_window() -> Option { + let sample = PEAK.lock().unwrap_or_else(|e| e.into_inner()).take()?; + let gib = |b: u64| b as f64 / (1u64 << 30) as f64; + Some(format!( + "VmRSS {:.2} GiB at {:.1}s · jemalloc allocated {:.2} · resident {:.2} (resident − allocated {:.2}) · \ + active {:.2} (purgeable {:.2})", + gib(sample.rss), + sample.at, + gib(sample.allocated), + gib(sample.resident), + gib(sample.resident.saturating_sub(sample.allocated)), + gib(sample.active), + gib(sample.resident.saturating_sub(sample.active)) + )) +} + +/// What [`WATERMARK_ENV`] says. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WatermarkSetting { + Auto, + Off, +} + +/// A value of [`WATERMARK_ENV`]; nonsense is an error. +pub fn parse_watermark(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "auto") => Ok(WatermarkSetting::Auto), + Some("off") => Ok(WatermarkSetting::Off), + Some(v) => Err(format!("{WATERMARK_ENV} must be auto or off, got `{v}`")), + } +} + +/// What a purge can return: jemalloc's resident bytes less its active ones, +/// that is its dirty pages (and metadata). The gap to `allocated` also counts +/// the free space inside active pages, which a purge cannot return: at the +/// median on a 32 GiB emulated host a fire read that gap at ≥ 4 GiB and +/// returned nothing (VmRSS 25.52 → 25.50 GiB), and its spacing held the next +/// fire past level 1's peak (RYZEN 058b, I-MEMFIT §6.13). +pub(crate) fn purgeable(stats: &AllocStats) -> u64 { + stats.resident.saturating_sub(stats.active) as u64 +} + +/// What a purge returned: jemalloc's resident bytes before less after +/// (nothing when they grew while it ran). +fn returned(purge: &Purge) -> u64 { + purge.resident_before.saturating_sub(purge.resident_after) as u64 +} + +/// The least time after a fire that `returned` these bytes before the next: +/// [`WATERMARK_SPACING`] from [`WATERMARK_MIN_RETURNED`] up, else +/// [`WATERMARK_RETRY`]. +fn watermark_spacing(returned: u64) -> std::time::Duration { + if returned >= WATERMARK_MIN_RETURNED { + WATERMARK_SPACING + } else { + WATERMARK_RETRY + } +} + +/// Whether the watermark fires now: memory pressure, the resident set at or +/// over the share of `target`, enough purgeable bytes, and the spacing since +/// the last fire (`last`: the time since it and the bytes it returned). No +/// reading: no fire. +fn watermark_fires( + pressure: bool, + resident: Option, + purgeable: Option, + target: u64, + last: Option<(std::time::Duration, u64)>, +) -> bool { + pressure + && resident.is_some_and(|r| r as f64 >= WATERMARK_SHARE * target as f64) + && purgeable.is_some_and(|p| p >= WATERMARK_MIN_PURGEABLE) + && last.is_none_or(|(since, returned)| since >= watermark_spacing(returned)) +} + +/// The running watermark monitor ([`start_watermark`]); stopped and joined by +/// [`Watermark::finish`] or when dropped. +pub struct Watermark { + stop: std::sync::Arc<(std::sync::Mutex, std::sync::Condvar)>, + handle: Option>, + target: u64, + purges: bool, +} + +/// Starts the watermark monitor against the host `target` (bytes), unless the +/// build cannot read the allocator. Under [`WATERMARK_ENV`] `off` (or +/// nonsense, which it prints) it only samples, for the `ALLOC PEAK` lines +/// ([`peak_window`]), and never purges. +pub fn start_watermark(target: u64) -> Option { + let purges = match parse_watermark(std::env::var(WATERMARK_ENV).ok().as_deref()) { + Ok(WatermarkSetting::Auto) => true, + Ok(WatermarkSetting::Off) => false, + Err(why) => { + eprintln!("ALLOC WATERMARK: {why}; sampling only"); + false + } + }; + let hooks = hooks()?; + *PEAK.lock().unwrap_or_else(|e| e.into_inner()) = None; + let stop = std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + let flag = stop.clone(); + let handle = std::thread::Builder::new() + .name("alloc-watermark".to_string()) + .spawn(move || watch(hooks, target, purges, &flag)) + .ok()?; + Some(Watermark { + stop, + handle: Some(handle), + target, + purges, + }) +} + +/// The monitor's loop: a reading every [`WATERMARK_TICK`] until stopped. +/// Returns the fires and their summed seconds. +fn watch( + hooks: AllocatorHooks, + target: u64, + purges: bool, + stop: &(std::sync::Mutex, std::sync::Condvar), +) -> (usize, f64) { + let gib = |b: u64| b as f64 / (1u64 << 30) as f64; + let t0 = std::time::Instant::now(); + // The last fire: when, and the bytes it returned. + let (mut fires, mut secs, mut last) = (0usize, 0.0f64, None::<(std::time::Instant, u64)>); + let mut stopped = stop.0.lock().unwrap_or_else(|e| e.into_inner()); + loop { + stopped = stop + .1 + .wait_timeout(stopped, WATERMARK_TICK) + .unwrap_or_else(|e| e.into_inner()) + .0; + if *stopped { + return (fires, secs); + } + let resident = crate::lfm::program_budget::resident_bytes(); + let stats = (hooks.stats)(); + if let (Some(rss), Some(st)) = (resident, stats) { + let mut peak = PEAK.lock().unwrap_or_else(|e| e.into_inner()); + if peak.is_none_or(|p| rss > p.rss) { + *peak = Some(PeakSample { + rss, + allocated: st.allocated as u64, + active: st.active as u64, + resident: st.resident as u64, + at: t0.elapsed().as_secs_f64(), + }); + } + } + let pressure = PRESSURE.load(std::sync::atomic::Ordering::Relaxed); + if !purges || !pressure { + continue; + } + let can = stats.as_ref().map(purgeable); + let since = last.map(|(at, r)| (at.elapsed(), r)); + if !watermark_fires(pressure, resident, can, target, since) { + continue; + } + let Some(purge) = purge_with(hooks) else { + continue; + }; + let after = crate::lfm::program_budget::resident_bytes(); + let back = returned(&purge); + fires += 1; + secs += purge.secs; + last = Some((std::time::Instant::now(), back)); + eprintln!( + "ALLOC WATERMARK fire #{fires} at {:.1}s: VmRSS {:.2} → {} GiB · jemalloc resident {:.2} \ + → {:.2} GiB · {:.3}s · purgeable {:.2} GiB, returned {:.2} · next ≥ {}s", + t0.elapsed().as_secs_f64(), + gib(resident.unwrap_or(0)), + after.map_or("unread".to_string(), |a| format!("{:.2}", gib(a))), + gib(purge.resident_before as u64), + gib(purge.resident_after as u64), + purge.secs, + gib(can.unwrap_or(0)), + gib(back), + watermark_spacing(back).as_secs() + ); + } +} + +impl Watermark { + fn stop(&mut self) -> Option<(usize, f64)> { + *self.stop.0.lock().unwrap_or_else(|e| e.into_inner()) = true; + self.stop.1.notify_all(); + self.handle.take().and_then(|h| h.join().ok()) + } + + /// Stops the monitor and says what it did: the `ALLOC WATERMARK end` line. + pub fn finish(mut self) -> String { + let (fires, secs) = self.stop().unwrap_or((0, 0.0)); + if !self.purges { + return "ALLOC WATERMARK end: off (sampling only), 0 fire(s)".to_string(); + } + format!( + "ALLOC WATERMARK end: {fires} fire(s), Σ {secs:.2}s (auto: memory pressure, VmRSS ≥ \ + {WATERMARK_SHARE:.2} × {:.2} GiB, ≥ {} GiB purgeable, ≥ {}s after a fire that returned \ + ≥ {} GiB, else ≥ {}s)", + self.target as f64 / (1u64 << 30) as f64, + WATERMARK_MIN_PURGEABLE >> 30, + WATERMARK_SPACING.as_secs(), + WATERMARK_MIN_RETURNED >> 30, + WATERMARK_RETRY.as_secs() + ) + } +} + +impl Drop for Watermark { + fn drop(&mut self) { + let _ = self.stop(); + } +} + +/// Purge every arena now, whatever [`ALLOC_PURGE_ENV`] says: for a caller +/// that has just freed memory it must see leave the process (a block's +/// partial drops). `None` when the build cannot purge. +pub fn purge_now() -> Option { + purge_all_arenas() +} + +/// Purge every arena through the hooks, timed, with the resident bytes +/// before and after. +fn purge_all_arenas() -> Option { + purge_with(hooks()?) +} + +fn purge_with(hooks: AllocatorHooks) -> Option { + let resident = || (hooks.stats)().map(|s| s.resident); + let resident_before = resident()?; + let t = std::time::Instant::now(); + if !(hooks.purge_all)() { + return None; + } + let secs = t.elapsed().as_secs_f64(); + Some(Purge { + secs, + resident_before, + resident_after: resident()?, + }) +} + +/// jemalloc's statistics, the epoch turned first (they are cached until it +/// turns). +#[cfg(test)] +fn jemalloc_stats() -> Option { + use tikv_jemalloc_ctl::{epoch, stats}; + epoch::advance().ok()?; + Some(AllocStats { + allocated: stats::allocated::read().ok()?, + active: stats::active::read().ok()?, + resident: stats::resident::read().ok()?, + mapped: stats::mapped::read().ok()?, + retained: stats::retained::read().ok()?, + }) +} + +/// `arena..purge` (4096, jemalloc's "every arena" index): a +/// control that neither reads nor writes, so every pointer is null. +#[cfg(test)] +fn jemalloc_purge_all() -> bool { + // SAFETY: the name is NUL-terminated, and a control that takes no value is + // called with null old and new pointers and zero lengths, as jemalloc's + // `NEITHER_READ_NOR_WRITE` requires. + let rc = unsafe { + tikv_jemalloc_sys::mallctl( + c"arena.4096.purge".as_ptr(), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + 0, + ) + }; + if rc != 0 { + eprintln!("ALLOC PURGE: arena.4096.purge returned {rc}"); + } + rc == 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The knob names its boundaries: `all`, or a list split on commas or dots, + /// spaces allowed. + #[test] + fn the_purge_knob_names_its_boundaries() { + assert!(names_point("all", "base")); + assert!(names_point("base,tree", "tree")); + assert!(names_point("base.tree", "base")); + assert!(names_point("phase-a.base", "phase-a")); + assert!(names_point(" base , tree ", "base")); + assert!(!names_point("base,tree", "phase-a")); + assert!(!names_point("", "base")); + } + + /// `auto` (the default) purges at phase A's end and the base's end only + /// under memory pressure, and says it skipped otherwise; never at `tree`; + /// `off` purges nowhere; `all` and a list purge regardless of pressure. + #[test] + fn the_level_purge_knob_reads_auto_off_or_always() { + assert_eq!(parse_level_purge(None), Ok(LevelPurge::Auto)); + assert_eq!(parse_level_purge(Some("")), Ok(LevelPurge::Auto)); + assert_eq!(parse_level_purge(Some("off")), Ok(LevelPurge::Off)); + assert_eq!(parse_level_purge(Some(" always ")), Ok(LevelPurge::Always)); + assert!(parse_level_purge(Some("yes")).is_err()); + } + + /// `auto` purges after a level only under memory pressure and with the + /// host near its target; a host with room never purges. + #[test] + fn a_level_purge_needs_pressure_and_a_full_host() { + const G: u64 = 1 << 30; + let target = 100 * G; + use LevelPurge::*; + assert_eq!(level_decision(Auto, true, Some(95 * G), target), Ok(())); + assert_eq!(level_decision(Auto, true, Some(90 * G), target), Ok(())); + assert_eq!( + level_decision(Auto, true, Some(89 * G), target), + Err("auto, the host has room") + ); + assert_eq!( + level_decision(Auto, false, Some(99 * G), target), + Err("auto, no memory pressure") + ); + assert_eq!( + level_decision(Auto, true, None, target), + Err("auto, no resident reading") + ); + assert_eq!(level_decision(Off, true, Some(99 * G), target), Err("off")); + assert_eq!(level_decision(Always, false, Some(G), target), Ok(())); + } + + #[test] + fn the_watermark_knob_reads_auto_or_off() { + assert_eq!(parse_watermark(None), Ok(WatermarkSetting::Auto)); + assert_eq!(parse_watermark(Some(" auto ")), Ok(WatermarkSetting::Auto)); + assert_eq!(parse_watermark(Some("off")), Ok(WatermarkSetting::Off)); + assert!(parse_watermark(Some("on")).is_err()); + } + + /// The watermark fires only under memory pressure, with the host at the + /// share of its target, enough purgeable bytes and the spacing kept. + #[test] + fn the_watermark_fires_only_under_pressure_on_a_full_host() { + const G: u64 = 1 << 30; + let target = 100 * G; + let s = std::time::Duration::from_secs; + assert!(watermark_fires( + true, + Some(90 * G), + Some(4 * G), + target, + None + )); + assert!(watermark_fires( + true, + Some(99 * G), + Some(9 * G), + target, + Some((s(5), 2 * G)) + )); + assert!( + !watermark_fires(false, Some(99 * G), Some(9 * G), target, None), + "no pressure" + ); + assert!( + !watermark_fires(true, Some(89 * G), Some(9 * G), target, None), + "the host has room" + ); + assert!( + !watermark_fires(true, Some(99 * G), Some(3 * G), target, None), + "too little purgeable" + ); + assert!( + !watermark_fires(true, None, Some(9 * G), target, None), + "no reading" + ); + assert!( + !watermark_fires(true, Some(99 * G), None, target, None), + "no stats" + ); + } + + /// The trigger reads what a purge can return, resident − active, not the + /// gap to allocated, which also counts free space inside active pages + /// (RYZEN 058b: a fire on that gap returned nothing). + #[test] + fn the_watermark_reads_purgeable_bytes_not_the_gap_to_allocated() { + const G: usize = 1 << 30; + let fragmented = AllocStats { + allocated: 20 * G, + active: 26 * G, + resident: 27 * G, + ..AllocStats::default() + }; + assert_eq!(purgeable(&fragmented), G as u64); + assert!( + !watermark_fires( + true, + Some(30 << 30), + Some(purgeable(&fragmented)), + 22 << 30, + None + ), + "7 GiB over allocated, but 1 GiB purgeable: no fire" + ); + let dirty = AllocStats { + active: 22 * G, + ..fragmented + }; + assert_eq!(purgeable(&dirty), 5 * G as u64); + assert!(watermark_fires( + true, + Some(30 << 30), + Some(purgeable(&dirty)), + 22 << 30, + None + )); + let grown = Purge { + secs: 0.7, + resident_before: 26 * G, + resident_after: 27 * G, + }; + assert_eq!(returned(&grown), 0, "resident grew while it ran"); + } + + /// A fire that returned at least a GiB holds the next for the full + /// spacing. + #[test] + fn a_fire_that_returned_a_gib_holds_the_next_for_the_spacing() { + const G: u64 = 1 << 30; + let ms = std::time::Duration::from_millis; + let fires = |since, back| { + watermark_fires( + true, + Some(99 * G), + Some(9 * G), + 100 * G, + Some((since, back)), + ) + }; + assert_eq!(watermark_spacing(WATERMARK_MIN_RETURNED), WATERMARK_SPACING); + assert!(!fires(ms(1000), G), "1 s after a fire that returned 1 GiB"); + assert!( + !fires(ms(4900), 3 * G), + "4.9 s after one that returned 3 GiB" + ); + assert!(fires(ms(5000), 3 * G)); + } + + /// A fire that returned less than a GiB does not hold the next for the + /// full spacing: one second on, the next may fire. + #[test] + fn a_fire_that_returned_little_lets_the_next_come_after_a_second() { + const G: u64 = 1 << 30; + let ms = std::time::Duration::from_millis; + let fires = |since, back| { + watermark_fires( + true, + Some(99 * G), + Some(9 * G), + 100 * G, + Some((since, back)), + ) + }; + assert_eq!(watermark_spacing(G - 1), WATERMARK_RETRY); + assert!(fires(ms(1000), 0), "1 s after a fire that returned nothing"); + assert!( + fires(ms(1500), G - 1), + "1.5 s after one that returned just under 1 GiB" + ); + } + + /// A fire that returned nothing cannot fire again within a second: purges + /// that find little do not spin, each taking every arena's lock. + #[test] + fn a_fire_that_returned_nothing_cannot_fire_again_within_a_second() { + const G: u64 = 1 << 30; + let ms = std::time::Duration::from_millis; + let fires = |since, back| { + watermark_fires( + true, + Some(99 * G), + Some(9 * G), + 100 * G, + Some((since, back)), + ) + }; + assert!(WATERMARK_RETRY >= std::time::Duration::from_secs(1)); + for since in [0, 1, 500, 999] { + assert!( + !fires(ms(since), 0), + "{since} ms after a fire that returned nothing" + ); + } + } + + /// A started monitor samples (a peak window to read), stops, and reports + /// its fires. + #[test] + fn the_watermark_monitor_starts_and_stops() { + if std::env::var_os(WATERMARK_ENV).is_some() { + return; + } + let w = start_watermark(u64::MAX).expect("the test build reads its jemalloc"); + std::thread::sleep(WATERMARK_TICK * 3); + // The window needs a VmRSS reading (Linux's /proc); elsewhere it stays empty. + if crate::lfm::program_budget::resident_bytes().is_some() { + let peak = peak_window().expect("a reading after three ticks"); + assert!(peak.contains("jemalloc allocated"), "{peak}"); + } + let line = w.finish(); + assert!(line.starts_with("ALLOC WATERMARK end: 0 fire(s)"), "{line}"); + } + + #[test] + fn auto_purges_only_under_memory_pressure() { + for setting in [None, Some(""), Some("auto"), Some(" auto ")] { + for point in ["phase-a", "base"] { + assert_eq!( + decide(setting, point, true, false), + Decision::Purge, + "{setting:?} {point}" + ); + assert_eq!( + decide(setting, point, false, false), + Decision::Skip, + "{setting:?} {point}" + ); + } + assert_eq!(decide(setting, "tree", true, false), Decision::No); + } + for point in ["phase-a", "base", "tree"] { + assert_eq!(decide(Some("off"), point, true, false), Decision::No); + assert_eq!(decide(Some("all"), point, false, false), Decision::Purge); + } + assert_eq!( + decide(Some("base.tree"), "tree", false, false), + Decision::Purge + ); + assert_eq!( + decide(Some("base.tree"), "phase-a", true, false), + Decision::No + ); + } + + /// Room changes `auto` under memory pressure at its two points and nothing + /// else: without pressure it skips as before, `tree` is no point of its, + /// `off` purges nowhere, and a point the knob names is purged whatever the + /// room. + #[test] + fn room_spares_only_autos_purges_under_pressure() { + for setting in [None, Some(""), Some("auto"), Some(" auto ")] { + for point in ["phase-a", "base"] { + assert_eq!( + decide(setting, point, true, true), + Decision::Room, + "{setting:?} {point}" + ); + assert_eq!( + decide(setting, point, false, true), + Decision::Skip, + "{setting:?} {point}" + ); + } + assert_eq!(decide(setting, "tree", true, true), Decision::No); + } + for point in ["phase-a", "base", "tree"] { + assert_eq!(decide(Some("off"), point, true, true), Decision::No); + assert_eq!(decide(Some("all"), point, true, true), Decision::Purge); + } + assert_eq!( + decide(Some("phase-a.base"), "phase-a", true, true), + Decision::Purge + ); + assert_eq!(decide(Some("base"), "phase-a", true, true), Decision::No); + } + + #[test] + fn the_room_knob_reads_auto_or_off() { + assert_eq!(parse_room(None), Ok(RoomSetting::Auto)); + assert_eq!(parse_room(Some("")), Ok(RoomSetting::Auto)); + assert_eq!(parse_room(Some(" auto ")), Ok(RoomSetting::Auto)); + assert_eq!(parse_room(Some("off")), Ok(RoomSetting::Off)); + assert!(parse_room(Some("on")).is_err()); + } + + /// GiB as bytes, to the hundredth the logs print. + fn gib(x: f64) -> u64 { + (x * (1u64 << 30) as f64) as u64 + } + + /// The p90 block's two boundaries as ZEN5 096 read them. On a 118 GiB host + /// (line 92.03 GiB) the block has room at both and skips both purges; on a + /// 64 GiB host (line 45.90 GiB) it has room at neither, nor has the median, + /// and purges as it did. + #[test] + fn a_host_with_room_skips_and_a_64_gib_host_purges() { + let headroom = 16 << 30; + let large = Some(NoDisk { + line: gib(92.03), + headroom, + }); + for (resident, at) in [(66.75, "phase A's end"), (68.17, "the base's end")] { + assert!(has_room(large, Some(gib(resident))), "118 GiB, {at}"); + } + let small = Some(NoDisk { + line: gib(45.90), + headroom, + }); + for (resident, at) in [ + (36.08, "p90, phase A's end"), + (45.60, "p90, the base's end"), + (33.13, "the median, phase A's end"), + (33.16, "the median, the base's end"), + ] { + assert!(!has_room(small, Some(gib(resident))), "64 GiB, {at}"); + } + } + + /// Room is the resident set and the headroom strictly under the line: at + /// the line the purge runs. + #[test] + fn at_the_line_there_is_no_room() { + const G: u64 = 1 << 30; + let no_disk = Some(NoDisk { + line: 90 * G, + headroom: 16 * G, + }); + assert!(has_room(no_disk, Some(74 * G - 1))); + assert!(!has_room(no_disk, Some(74 * G))); + assert!(!has_room(no_disk, Some(90 * G))); + assert!(!has_room(no_disk, Some(u64::MAX)), "no overflow"); + } + + /// A resident set that cannot be read is no room: the purge runs, as it + /// did before the room was weighed. + #[test] + fn no_resident_reading_is_no_room() { + const G: u64 = 1 << 30; + let no_disk = Some(NoDisk { + line: 90 * G, + headroom: 16 * G, + }); + assert!(!has_room(no_disk, None)); + } + + /// A block without live regeneration has no line: it purges under pressure + /// whatever its resident set. + #[test] + fn a_block_without_live_regeneration_has_no_room() { + assert!(!has_room(None, Some(1 << 30))); + assert!(!has_room(None, None)); + } + + /// A purge goes through the hooks it is given: it reads the resident bytes + /// before and after, and a purge that did not run is no purge. + #[test] + fn a_purge_runs_through_its_hooks() { + use std::sync::atomic::{AtomicUsize, Ordering}; + static RESIDENT: AtomicUsize = AtomicUsize::new(900); + fn stats() -> Option { + Some(AllocStats { + resident: RESIDENT.load(Ordering::SeqCst), + ..AllocStats::default() + }) + } + fn purged() -> bool { + RESIDENT.store(100, Ordering::SeqCst); + true + } + fn refused() -> bool { + false + } + fn blind() -> Option { + None + } + let purge = purge_with(AllocatorHooks { + stats, + purge_all: purged, + }) + .expect("the hooks purge"); + assert_eq!((purge.resident_before, purge.resident_after), (900, 100)); + let refused = AllocatorHooks { + stats, + purge_all: refused, + }; + assert!(purge_with(refused).is_none()); + let blind = AllocatorHooks { + stats: blind, + purge_all: purged, + }; + assert!(purge_with(blind).is_none(), "no reading, no purge line"); + } + + /// The lib's test build reads its own jemalloc with no hook installed. + #[test] + fn the_test_build_reads_its_jemalloc() { + assert!(stats().is_some_and(|s| s.allocated > 0 && s.resident >= s.allocated)); + } + + /// A purge hands back the pages freed buffers left: 512 buffers of 1 MiB, + /// touched and freed on this thread. Under the posture's never-purge + /// allocator they stay resident until the purge, which returns at least + /// half of them (laptop: 525 → 4 MiB in 4 ms). Under jemalloc's default decay + /// they may be gone already, so the test asks for the posture. + #[test] + #[ignore = "run under the posture: _RJEM_MALLOC_CONF=dirty_decay_ms:-1,muzzy_decay_ms:-1"] + fn a_purge_returns_freed_buffers_pages() { + let resident = || { + tikv_jemalloc_ctl::epoch::advance().expect("the epoch turns"); + tikv_jemalloc_ctl::stats::resident::read().expect("stats.resident reads") + }; + let before = resident(); + let buffers: Vec> = (0..512).map(|_| vec![1u8; 1 << 20]).collect(); + drop(buffers); + let retained = resident(); + assert!( + retained >= before + (256 << 20), + "the freed buffers were not retained ({} → {} MiB): run under the posture's allocator", + before >> 20, + retained >> 20 + ); + let purge = purge_all_arenas().expect("the test build purges"); + assert!( + purge.resident_before >= purge.resident_after + (256 << 20), + "resident {} → {} MiB: the purge returned less than half of 512 MiB freed", + purge.resident_before >> 20, + purge.resident_after >> 20 + ); + println!( + "PURGE UNIT: {:.3}s · resident {} → {} MiB", + purge.secs, + purge.resident_before >> 20, + purge.resident_after >> 20 + ); + } +} diff --git a/prover/src/auto_storage.rs b/prover/src/auto_storage.rs deleted file mode 100644 index 2931bfa40..000000000 --- a/prover/src/auto_storage.rs +++ /dev/null @@ -1,330 +0,0 @@ -//! Automatic `StorageMode` selection from an analytical peak-RAM estimate. -//! -//! `FORCE_DISK_SPILL` env var forces `StorageMode::Disk` regardless of the -//! estimate. - -use crate::tables::bitwise::{ - NUM_ROWS as BITWISE_ROWS, bus_interactions as bitwise_buses, cols::NUM_COLUMNS as BITWISE_COLS, -}; -use crate::tables::branch::{bus_interactions as branch_buses, cols::NUM_COLUMNS as BRANCH_COLS}; -use crate::tables::copy::{bus_interactions as copy_buses, cols::NUM_COLUMNS as COPY_COLS}; -use crate::tables::cpu::{bus_interactions as cpu_buses, cols::NUM_COLUMNS as CPU_COLS}; -use crate::tables::decode::{bus_interactions as decode_buses, cols::NUM_COLUMNS as DECODE_COLS}; -use crate::tables::dvrm::{bus_interactions as dvrm_buses, cols::NUM_COLUMNS as DVRM_COLS}; -use crate::tables::halt::{bus_interactions as halt_buses, cols::NUM_COLUMNS as HALT_COLS}; -use crate::tables::load::{bus_interactions as load_buses, cols::NUM_COLUMNS as LOAD_COLS}; -use crate::tables::lt::{bus_interactions as lt_buses, cols::NUM_COLUMNS as LT_COLS}; -use crate::tables::memw::{bus_interactions as memw_buses, cols::NUM_COLUMNS as MEMW_COLS}; -use crate::tables::memw_aligned::{ - bus_interactions as memw_a_buses, cols::NUM_COLUMNS as MEMW_A_COLS, -}; -use crate::tables::memw_register::{ - bus_interactions as memw_r_buses, cols::NUM_COLUMNS as MEMW_R_COLS, -}; -use crate::tables::mul::{bus_interactions as mul_buses, cols::NUM_COLUMNS as MUL_COLS}; -use crate::tables::page::{ - DEFAULT_PAGE_SIZE as PAGE_SIZE, bus_interactions as page_buses, cols::NUM_COLUMNS as PAGE_COLS, -}; -use crate::tables::register::{ - NUM_REGISTER_ADDRESSES, bus_interactions as register_buses, cols::NUM_COLUMNS as REGISTER_COLS, -}; -use crate::tables::shift::{bus_interactions as shift_buses, cols::NUM_COLUMNS as SHIFT_COLS}; -use crate::tables::trace_builder::TableLengths; -use stark::prover::storage_estimate_parallelism; -use stark::storage_mode::StorageMode; -use sysinfo::System; - -pub(crate) const GOLDILOCKS_BYTES: u64 = 8; -pub(crate) const CUBIC_EXT_BYTES: u64 = 24; -pub(crate) const KECCAK_NODE_BYTES: u64 = 32; -const LOG_STRUCT_BYTES: u64 = 40; -const MEMORY_CELL_BYTES: u64 = 32; -const INSTRUCTION_MAP_BYTES_PER_ROW: u64 = 32; - -/// 9/10 budget headroom for OS, other processes, and allocator slack. -pub const SAFETY_FRACTION_NUM: u64 = 9; -pub const SAFETY_FRACTION_DEN: u64 = 10; - -/// `(rows, main_cols, aux_cols, num_main_merkle_trees)` for a single table. -type TableSpec = (u64, u64, u64, u64); - -/// Bytes counted as alive for the whole proof (LDE columns + main/aux Merkle). -/// Deliberately an over-estimate for the aux half — see `peak_bytes`. -fn persistent_per_table(spec: TableSpec, blowup: u64) -> u64 { - let (rows, main_cols, aux_cols, main_trees) = spec; - let main_lde = rows - .saturating_mul(main_cols) - .saturating_mul(GOLDILOCKS_BYTES) - .saturating_mul(1 + blowup); - let aux_lde = rows - .saturating_mul(aux_cols) - .saturating_mul(CUBIC_EXT_BYTES) - .saturating_mul(1 + blowup); - let main_merkle = main_trees - .saturating_mul(2) - .saturating_mul(rows) - .saturating_mul(blowup) - .saturating_mul(KECCAK_NODE_BYTES); - let aux_merkle = if aux_cols > 0 { - 2u64.saturating_mul(rows) - .saturating_mul(blowup) - .saturating_mul(KECCAK_NODE_BYTES) - } else { - 0 - }; - main_lde - .saturating_add(aux_lde) - .saturating_add(main_merkle) - .saturating_add(aux_merkle) -} - -/// Bytes (constraint evals, composition, FRI) alive during rounds 2-4 for one chunk. -fn transient_per_table(spec: TableSpec, blowup: u64) -> u64 { - let (rows, _, _, _) = spec; - let lde_size = rows.saturating_mul(blowup); - let constraint_evals = lde_size.saturating_mul(CUBIC_EXT_BYTES); - let composition_lde = lde_size.saturating_mul(2).saturating_mul(CUBIC_EXT_BYTES); - let composition_merkle = lde_size.saturating_mul(KECCAK_NODE_BYTES); - let fri_evals = lde_size.saturating_mul(CUBIC_EXT_BYTES); - let fri_merkle = lde_size.saturating_mul(KECCAK_NODE_BYTES); - constraint_evals - .saturating_add(composition_lde) - .saturating_add(composition_merkle) - .saturating_add(fri_evals) - .saturating_add(fri_merkle) -} - -/// Bytes for one Domain/LdeTwiddles cache entry. -fn domain_cache_bytes(rows: u64, blowup: u64) -> u64 { - rows.saturating_mul(3 + 2 * blowup) - .saturating_mul(GOLDILOCKS_BYTES) -} - -fn aux_cols(bus_count: usize) -> u64 { - bus_count.div_ceil(2) as u64 -} - -/// Per-table specs for the heap estimate. The order is not meaningful — every -/// consumer sums or sorts — and it does not track `air_trace_pairs`. -/// -/// Note this list models only the tables `TableLengths` carries: there is no -/// entry for eq, bytewise, store, cpu32, keccak, keccak_rnd, ecsm, ecdas, hint -/// or is_wh, so a run dominated by one of those is under-estimated. -fn table_specs(lengths: &TableLengths) -> Vec { - let bitwise_rows = BITWISE_ROWS as u64; - let register_rows = NUM_REGISTER_ADDRESSES.next_power_of_two() as u64; - let halt_rows = 1u64; - let page_rows = PAGE_SIZE as u64; - - let mut specs = vec![ - ( - lengths.cpu_padded_rows, - CPU_COLS as u64, - aux_cols(cpu_buses(0).len()), - 2, // TIMESTAMP is preprocessed: see the note above BITWISE - ), - ( - lengths.memw_padded_rows, - MEMW_COLS as u64, - aux_cols(memw_buses().len()), - 1, - ), - ( - lengths.memw_aligned_padded_rows, - MEMW_A_COLS as u64, - aux_cols(memw_a_buses().len()), - 1, - ), - ( - lengths.memw_register_padded_rows, - MEMW_R_COLS as u64, - aux_cols(memw_r_buses().len()), - 1, - ), - ( - lengths.load_padded_rows, - LOAD_COLS as u64, - aux_cols(load_buses().len()), - 1, - ), - ( - lengths.lt_padded_rows, - LT_COLS as u64, - aux_cols(lt_buses().len()), - 1, - ), - ( - lengths.shift_padded_rows, - SHIFT_COLS as u64, - aux_cols(shift_buses().len()), - 1, - ), - ( - lengths.mul_padded_rows, - MUL_COLS as u64, - aux_cols(mul_buses().len()), - 1, - ), - ( - lengths.dvrm_padded_rows, - DVRM_COLS as u64, - aux_cols(dvrm_buses().len()), - 1, - ), - ( - lengths.branch_padded_rows, - BRANCH_COLS as u64, - aux_cols(branch_buses().len()), - 1, - ), - ( - lengths.copy_padded_rows, - COPY_COLS as u64, - aux_cols(copy_buses().len()), - 1, - ), - // BITWISE / DECODE / PAGE / REGISTER, and CPU above, take the - // preprocessed-trace commit path: it extracts ALL columns into the LDE and - // builds two Merkle trees (precomputed_tree + mult_tree), so main_cols = - // full NUM_COLUMNS and main_trees = 2. - ( - bitwise_rows, - BITWISE_COLS as u64, - aux_cols(bitwise_buses().len()), - 2, - ), - ( - lengths.decode_rows, - DECODE_COLS as u64, - aux_cols(decode_buses().len()), - 2, - ), - (halt_rows, HALT_COLS as u64, aux_cols(halt_buses().len()), 1), - ( - register_rows, - REGISTER_COLS as u64, - aux_cols(register_buses(0).len()), - 2, - ), - ]; - // Each unique 256 KB page → its own PAGE table at PAGE_SIZE rows. - for _ in 0..lengths.unique_page_count { - specs.push(( - page_rows, - PAGE_COLS as u64, - aux_cols(page_buses(0).len()), - 2, - )); - } - specs -} - -/// Estimates heap from `lengths` and `blowup_factor`. Picks `Disk` if the -/// estimate is greater than available RAM, else `Ram`. `FORCE_DISK_SPILL` env -/// var forces `Disk`. -pub fn decide(lengths: &TableLengths, blowup_factor: u8) -> StorageMode { - if std::env::var("FORCE_DISK_SPILL").is_ok() { - log::info!("storage_mode: Disk (forced via FORCE_DISK_SPILL)"); - return StorageMode::Disk; - } - let estimated = peak_bytes(lengths, blowup_factor, storage_estimate_parallelism()); - let mode = select_storage_mode(estimated, available_ram_bytes()); - log::info!("estimated_peak_bytes: {estimated}, storage_mode: {mode:?}"); - mode -} - -/// Peak RAM estimate in bytes for a proof whose trace shape matches `lengths`. -/// -/// `table_parallelism` is how many tables' rounds 2-4 transients this assumes -/// are alive at once. `decide` passes `storage_estimate_parallelism()`, which -/// is deliberately *not* the scheduler's `k` — that one is `num_airs` under -/// `cuda`, and summing every table's transients here inflates the estimate on -/// many-PAGE shapes (up to +44 %) and makes `Disk` more likely than the real -/// heap warrants. See that function for why the honest bound is a byte budget -/// rather than a count. -pub fn peak_bytes(lengths: &TableLengths, blowup_factor: u8, table_parallelism: usize) -> u64 { - let blowup = blowup_factor as u64; - let k = table_parallelism.max(1); - let specs = table_specs(lengths); - - // Persistent: every table's main LDE + Merkle really is alive at once (the - // Round 1 main commit is a phase-wide barrier). The aux LDE is produced and - // consumed inside one table's fused task, so only the scheduler's k coexist - // — exactly all of them on `cuda`, fewer on CPU builds. Counted for every - // table either way, which is exact on `cuda` and an over-estimate on CPU - // rather than an unsound bound. - let persistent_total: u64 = specs - .iter() - .map(|s| persistent_per_table(*s, blowup)) - .fold(0u64, u64::saturating_add); - - // Transient: k tables' fused aux+rounds tasks assumed in flight at once. - // The top-k tables by transient bytes bound that; with the scheduler's - // heaviest-first admission that top-k is also the set admitted first, so - // this is the realistic peak, not a worst case. - let mut transient_per: Vec = specs - .iter() - .map(|s| transient_per_table(*s, blowup)) - .collect(); - transient_per.sort_unstable_by(|a, b| b.cmp(a)); - let transient_total: u64 = transient_per - .iter() - .take(k) - .copied() - .fold(0u64, u64::saturating_add); - - // Domain + LdeTwiddles cache: one entry per unique padded-row count - // (blowup_factor and coset_offset are constant across tables in this - // codebase, so the unique key collapses to `rows`). - let mut unique_rows: Vec = specs.iter().map(|s| s.0).collect(); - unique_rows.sort_unstable(); - unique_rows.dedup(); - let domain_total: u64 = unique_rows - .iter() - .map(|&r| domain_cache_bytes(r, blowup)) - .fold(0u64, u64::saturating_add); - - // State alive across the prove call (memory cells, log Vec, instruction - // map). Independent of trace shape. - let state_total = lengths - .unique_byte_count - .saturating_mul(MEMORY_CELL_BYTES) - .saturating_add(lengths.cycle_count.saturating_mul(LOG_STRUCT_BYTES)) - .saturating_add( - lengths - .decode_rows - .saturating_mul(INSTRUCTION_MAP_BYTES_PER_ROW), - ); - - persistent_total - .saturating_add(transient_total) - .saturating_add(domain_total) - .saturating_add(state_total) -} - -/// `Disk` if `estimated` exceeds `available` minus a safety margin, else -/// `Ram`. Defaults to `Disk` when `available` is `None`. -pub(crate) fn select_storage_mode(estimated: u64, available: Option) -> StorageMode { - let Some(available) = available else { - log::warn!("Auto disk-spill: sysinfo could not read system memory, defaulting to Disk."); - return StorageMode::Disk; - }; - let threshold = available.saturating_mul(SAFETY_FRACTION_NUM) / SAFETY_FRACTION_DEN; - if estimated > threshold { - StorageMode::Disk - } else { - StorageMode::Ram - } -} - -/// OS-available RAM, or `None` if sysinfo can't read it. -fn available_ram_bytes() -> Option { - let mut sys = System::new(); - sys.refresh_memory(); - // total_memory == 0 means sysinfo can't read; otherwise available is real. - if sys.total_memory() == 0 { - None - } else { - Some(sys.available_memory()) - } -} diff --git a/prover/src/bin/compute_constraint_artifacts.rs b/prover/src/bin/compute_constraint_artifacts.rs new file mode 100644 index 000000000..8989d252f --- /dev/null +++ b/prover/src/bin/compute_constraint_artifacts.rs @@ -0,0 +1,110 @@ +//! Serializes every production table's constraint artifact to disk, so the +//! constraints exist as DATA rather than only as compiled code. +//! +//! Run with: +//! cargo run --bin compute_constraint_artifacts --release -- +//! +//! Writes `/

.bin` per table plus a `MANIFEST.txt` recording each +//! artifact's size, and prints the size table (the recursion machine's +//! program-length budget). +//! +//! Capture is a build-time operation: it hash-conses the whole constraint body, +//! which is exactly what a guest must not do. That is the point of writing the +//! result down — see `stark::constraint_ir::artifact`. +//! +//! The artifacts are NOT proof-options or trace-length dependent (pinned by +//! `artifacts_are_invariant_across_proof_options` and +//! `artifacts_are_invariant_across_trace_length`), so a table's file covers every +//! blowup factor and every epoch size. +//! +//! ⚠️ But six tables are PARAMETERIZED, so this emits ONE REPRESENTATIVE, not +//! the complete set: `PAGE` and `GLOBAL_MEMORY` fold a page base into constant +//! bus terms, both `L2G` tables fold an epoch label, `CPU` its table's +//! timestamp base and `REGISTER` the final PC timestamp. Their files are the +//! artifact at `test_utils::PAGE_TEST_BASE` / `EPOCH_TEST_LABEL` / base 0 only. A real +//! continuation proof needs one artifact per distinct page base and per distinct +//! epoch label — see +//! `constraint_artifact_tests::parameterized_airs_vary_per_parameter_value`. +//! Treating this directory as "the constraint artifacts" would be wrong for +//! exactly the tables a continuation proof cares most about. +//! +//! ⚠️ These bytes are not an oracle. Nothing about a serialized artifact proves +//! it matches the compiled folder — only +//! `prover/src/tests/constraint_artifact_tests.rs` does, by evaluating the +//! deserialized artifact against the folders on random frames. Regenerating +//! these files does not bless a constraint change. + +use std::path::PathBuf; + +use lambda_vm_prover::test_utils::production_airs; +use stark::constraint_ir::ConstraintArtifact; +use stark::proof::options::GoldilocksCubicProofOptions; + +fn main() { + let out_dir: PathBuf = std::env::args() + .nth(1) + .unwrap_or_else(|| "constraint_artifacts".to_string()) + .into(); + + let options = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid"); + + std::fs::create_dir_all(&out_dir) + .unwrap_or_else(|e| panic!("cannot create {}: {e}", out_dir.display())); + + let mut manifest = String::from( + "# Per-table constraint artifacts. Sizes in bytes.\n\ + # table constraints nodes base_consts ext_consts bytes\n", + ); + let mut total_bytes = 0usize; + let mut total_nodes = 0usize; + + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + "table", "constr", "nodes", "bconst", "econst", "bytes" + ); + + for (label, air) in production_airs(&options) { + let artifact = ConstraintArtifact::capture(&*air); + artifact + .validate_against(&*air) + .unwrap_or_else(|e| panic!("[{label}] artifact rejected against its own AIR: {e}")); + let bytes = artifact + .to_bytes() + .unwrap_or_else(|e| panic!("[{label}] serialize failed: {e}")); + + let path = out_dir.join(format!("{label}.bin")); + std::fs::write(&path, &bytes) + .unwrap_or_else(|e| panic!("cannot write {}: {e}", path.display())); + + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + label, + artifact.roots.len(), + artifact.nodes.len(), + artifact.base_consts.len(), + artifact.ext_consts.len(), + bytes.len() + ); + manifest.push_str(&format!( + "{label} {} {} {} {} {}\n", + artifact.roots.len(), + artifact.nodes.len(), + artifact.base_consts.len(), + artifact.ext_consts.len(), + bytes.len() + )); + total_bytes += bytes.len(); + total_nodes += artifact.nodes.len(); + } + + manifest.push_str(&format!("TOTAL - {total_nodes} - - {total_bytes}\n")); + let manifest_path = out_dir.join("MANIFEST.txt"); + std::fs::write(&manifest_path, &manifest) + .unwrap_or_else(|e| panic!("cannot write {}: {e}", manifest_path.display())); + + println!( + "\ntotal: {total_nodes} nodes, {total_bytes} bytes ({:.1} KiB)\nwritten to {}", + total_bytes as f64 / 1024.0, + out_dir.display() + ); +} diff --git a/prover/src/bin/compute_lfm_registry.rs b/prover/src/bin/compute_lfm_registry.rs new file mode 100644 index 000000000..dd388f19e --- /dev/null +++ b/prover/src/bin/compute_lfm_registry.rs @@ -0,0 +1,89 @@ +//! Regenerates the `LFM_REGISTRY` constant table. +//! +//! Usage: `cargo run --bin compute_lfm_registry --release`, then paste the +//! output over the generated block in `prover/src/lfm/registry.rs`. Drift +//! tests recompute and compare on every PR; a drift failure is investigated, +//! never re-blessed (the `compute_static_commitments` policy). +//! +//! ⚠ ORDER: on a hash-pin change run this AFTER `compute_static_commitments` +//! has been run and its output pasted. Slots 13 and 14 of every entry are +//! `keccak_rc` and `bitwise`'s `preprocessed_commitment`, which return the +//! BLESSED static constants in the tree rather than recomputing, and +//! `program_id` folds every root — so a table generated before the statics +//! embeds the outgoing hash's constants, and `machine_tests::registry_drift_*` +//! fires at exactly those two slots. + +use lambda_vm_prover::GoldilocksCubicProofOptions; +use lambda_vm_prover::lfm::programs::{ + KECCAK_SPONGE_LEN, fri_toy_program, keccak_chain_program, keccak_sponge_program, + statement_replay_program, transcript_replay_program, trivial_program, +}; +use lambda_vm_prover::lfm::registry::build_artifacts_with_hasher; +use lambda_vm_prover::lfm::validate; + +/// Blowups registered in v0 (extend alongside `STATIC_BLOWUP_FACTORS` when +/// other presets come online). +const REGISTRY_BLOWUP_FACTORS: &[u8] = &[2]; + +// The permutation this table is blessed under is `registry::REGISTRY_HASHER` — +// a property of the TABLE rather than of this generator, and the same constant +// `build_artifacts` defaults to, so the two cannot drift apart. +use lambda_vm_prover::lfm::registry::REGISTRY_HASHER; + +fn fmt_bytes(bytes: &[u8; 32]) -> String { + let inner = bytes + .iter() + .map(|b| format!("{b:#04x}")) + .collect::>() + .join(", "); + format!("[{inner}]") +} + +fn main() { + let programs = [ + ("TrivialV0", trivial_program()), + ("FriToyV0", fri_toy_program()), + ("KeccakChainV0", keccak_chain_program()), + ("KeccakSpongeV0", keccak_sponge_program(KECCAK_SPONGE_LEN)), + ("TranscriptReplayV0", transcript_replay_program()), + ("StatementReplayV0", statement_replay_program()), + ]; + println!("pub static LFM_REGISTRY: &[LfmRegistryEntry] = &["); + for (kind, program) in &programs { + // A program digest enters the registry only after admission passes — + // the gate `validator.rs` declares, wired here rather than left to the + // convention that every kind also has a hand-written admissibility test. + validate(program).unwrap_or_else(|v| panic!("{kind} is not admissible: {v:?}")); + for &blowup in REGISTRY_BLOWUP_FACTORS { + let options = GoldilocksCubicProofOptions::with_blowup(blowup).expect("proof options"); + let artifacts = build_artifacts_with_hasher(program, &options, REGISTRY_HASHER); + println!(" LfmRegistryEntry {{"); + println!(" kind: LfmProgramKind::{kind},"); + println!(" blowup_factor: {blowup},"); + println!(" roots: ["); + for root in &artifacts.roots { + println!(" {},", fmt_bytes(root)); + } + println!(" ],"); + let heights = artifacts + .log_heights + .iter() + .map(u8::to_string) + .collect::>() + .join(", "); + println!(" log_heights: [{heights}],"); + println!( + " keccak_rnd_chunks: {},", + artifacts.keccak_rnd_chunks + ); + println!(" hasher: HasherKind::{:?},", artifacts.hasher); + println!( + " chip_set: ChipSet {{ keccak: {}, blake3: {}, bitwise: {} }},", + artifacts.chip_set.keccak, artifacts.chip_set.blake3, artifacts.chip_set.bitwise + ); + println!(" program_id: {},", fmt_bytes(&artifacts.program_id)); + println!(" }},"); + } + } + println!("];"); +} diff --git a/prover/src/bin/compute_static_commitments.rs b/prover/src/bin/compute_static_commitments.rs index e16628838..b9425dbd9 100644 --- a/prover/src/bin/compute_static_commitments.rs +++ b/prover/src/bin/compute_static_commitments.rs @@ -1,22 +1,44 @@ -//! Prints static `(bitwise, keccak_rc, zero_page)` preprocessed-table commitments -//! for a fixed set of `blowup_factor` values. The output is pasted into the +//! Prints the static preprocessed-table commitments — FOUR families: `bitwise`, +//! `keccak_rc`, and `page`'s zero-page and private-page (OFFSET-only) constants +//! — for a fixed set of `blowup_factor` values. The output is pasted into the //! `static_commitment` match bodies in `prover/src/tables/{bitwise,keccak_rc}.rs` -//! and the `static_zero_page_commitment` match body in `prover/src/tables/page.rs`. -//! It then prints the CPU timestamp table, pasted over -//! `prover/src/tables/cpu_timestamp_commitments.rs`. +//! and the `static_zero_page_commitment` / `static_private_page_commitment` +//! match bodies in `prover/src/tables/page.rs`. //! The `static_commitments_tests` test suite pins the values so any drift in //! the AIR or FFT pipeline is caught at test time. //! //! Run with: //! cargo run --bin compute_static_commitments --release //! +//! `--layout row` prints the ONE-ROW (S2) twins instead — the same columns +//! committed with one LDE row per leaf — for `STATIC_BLOWUP_FACTORS_ONE_ROW`; +//! they are pasted into the `*_one_row` match bodies next to each constant +//! and pinned by the one-row drift tests. `--layout pair` (the default) is +//! the output above, unchanged. +//! +//! `--hash p1` prints the POSEIDON1 twins of `bitwise` and `keccak_rc` instead +//! — the same columns in row pairs, committed by the Poseidon1 configuration +//! of the block's proof family — for `STATIC_BLOWUP_FACTORS_P1`; they are +//! pasted into the `static_commitment_p1` match bodies and pinned by the +//! Poseidon1 drift tests. `--hash legacy` (the default) is the output above. +//! +//! ⚠ On a hash-pin change run this FIRST and paste before `compute_lfm_registry`: +//! the registry embeds these constants (slots 13 and 14 of every entry, and +//! `program_id` folds them), so a registry generated before the paste carries +//! the outgoing hash's statics and the drift gate catches it. +//! //! ⚠️ Do not run this just to silence a failing drift test — see the //! "Regenerating" section on `static_commitment` in `bitwise.rs` / -//! `keccak_rc.rs` and `static_zero_page_commitment` in `page.rs` for when -//! it's actually appropriate to bless new bytes. +//! `keccak_rc.rs` and the two `page.rs` constants for when it's actually +//! appropriate to bless new bytes. A hash-pin change is one such time, and it +//! regenerates all four families together (`prover/src/hash_pin.rs`). -use lambda_vm_prover::tables::{STATIC_BLOWUP_FACTORS, bitwise, cpu, keccak_rc, max_rows, page}; +use lambda_vm_prover::tables::{ + STATIC_BLOWUP_FACTORS, STATIC_BLOWUP_FACTORS_ONE_ROW, STATIC_BLOWUP_FACTORS_P1, bitwise, + keccak_rc, page, +}; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::proof::options::GoldilocksCubicProofOptions; fn format_commitment(commitment: &Commitment) -> String { @@ -35,16 +57,95 @@ fn format_commitment(commitment: &Commitment) -> String { out } +/// `--layout pair|row` (default `pair`) and `--hash legacy|p1` (default +/// `legacy`). Anything else aborts: a typo must not print another family's +/// constants under this one's name. +fn args() -> (LeafLayout, bool) { + let args: Vec = std::env::args().skip(1).collect(); + let (mut layout, mut p1) = (LeafLayout::RowPair, false); + for pair in args.chunks(2) { + match pair { + [flag, value] if flag == "--layout" => { + layout = match value.as_str() { + "pair" => LeafLayout::RowPair, + "row" => LeafLayout::Row, + other => panic!("--layout must be `pair` or `row`, got `{other}`"), + } + } + [flag, value] if flag == "--hash" => { + p1 = match value.as_str() { + "legacy" => false, + "p1" => true, + other => panic!("--hash must be `legacy` or `p1`, got `{other}`"), + } + } + other => panic!( + "usage: compute_static_commitments [--layout pair|row] [--hash legacy|p1], got {other:?}" + ), + } + } + assert!( + !(p1 && layout.is_one_row()), + "the Poseidon1 twins ship in row pairs only" + ); + (layout, p1) +} + +/// The Poseidon1 twins of `bitwise` and `keccak_rc`, in row pairs. +fn print_p1() { + type P1 = lambda_vm_prover::lfm::p1_commit::P1StarkHash; + println!( + "// commitment hash: Poseidon1 · leaf layout: RowPair\n\ + // Paste these match arms into the `static_commitment_p1` match bodies\n\ + // in `prover/src/tables/{{bitwise,keccak_rc}}.rs`.\n" + ); + for &blowup in STATIC_BLOWUP_FACTORS_P1 { + let options = match GoldilocksCubicProofOptions::with_blowup(blowup) { + Ok(o) => o, + Err(e) => { + eprintln!("skipping blowup={blowup}: {e}"); + continue; + } + }; + let bitwise = + bitwise::compute_preprocessed_commitment_under::(&options, LeafLayout::RowPair); + let keccak_rc = + keccak_rc::compute_preprocessed_commitment_under::(&options, LeafLayout::RowPair); + println!( + "// blowup_factor = {blowup}\n\ + // ---- bitwise:\n \ + {blowup} => Some({bitwise_fmt}),\n\ + // ---- keccak_rc:\n \ + {blowup} => Some({keccak_fmt}),\n", + bitwise_fmt = format_commitment(&bitwise), + keccak_fmt = format_commitment(&keccak_rc), + ); + } +} + fn main() { + let (layout, p1) = args(); + if p1 { + print_p1(); + return; + } + let blowups = match layout { + LeafLayout::RowPair => STATIC_BLOWUP_FACTORS, + LeafLayout::Row => STATIC_BLOWUP_FACTORS_ONE_ROW, + }; + println!("// leaf layout: {layout:?}"); + // The one-row twins go into the `*_one_row` functions beside each constant. + let suffix = if layout.is_one_row() { "_one_row" } else { "" }; println!( - "// Paste these match arms into the `static_commitment` match bodies\n\ + "// Paste these match arms into the `static_commitment{suffix}` match bodies\n\ // in `prover/src/tables/{{bitwise,keccak_rc}}.rs` and the\n\ - // `static_zero_page_commitment` match body in `prover/src/tables/page.rs`.\n" + // `static_zero_page_commitment{suffix}` / `static_private_page_commitment{suffix}`\n\ + // match bodies in `prover/src/tables/page.rs`.\n" ); let zero_page_config = page::PageConfig::zero_init(0); - for &blowup in STATIC_BLOWUP_FACTORS { + for &blowup in blowups { let options = match GoldilocksCubicProofOptions::with_blowup(blowup) { Ok(o) => o, Err(e) => { @@ -53,10 +154,11 @@ fn main() { } }; - let bitwise = bitwise::compute_preprocessed_commitment(&options); - let keccak_rc = keccak_rc::compute_preprocessed_commitment(&options); - let zero_page = page::compute_precomputed_commitment(&zero_page_config, &options); - let private_page = page::compute_offset_only_commitment(&options); + let bitwise = bitwise::compute_preprocessed_commitment_with(&options, layout); + let keccak_rc = keccak_rc::compute_preprocessed_commitment_with(&options, layout); + let zero_page = + page::compute_precomputed_commitment_with(&zero_page_config, &options, layout); + let private_page = page::compute_offset_only_commitment_with(&options, layout); println!( "// blowup_factor = {blowup}\n\ @@ -74,28 +176,4 @@ fn main() { private_page_fmt = format_commitment(&private_page), ); } - - // ---- CPU timestamp (one commitment per table height): - println!( - "// Generated by `cargo run --bin compute_static_commitments --release`.\n\ - // Do not edit by hand; see `static_timestamp_commitment`.\n\n\ - /// `(blowup_factor, [commitment for log2(height) = MIN_LOG_HEIGHT..])`.\n\ - const CPU_TIMESTAMP_COMMITMENTS: &[(u8, &[Commitment])] = &[" - ); - for &blowup in STATIC_BLOWUP_FACTORS { - let Ok(options) = GoldilocksCubicProofOptions::with_blowup(blowup) else { - continue; - }; - println!(" (\n {blowup},\n &["); - for log_height in cpu::MIN_LOG_HEIGHT..=max_rows::CPU.trailing_zeros() { - let commitment = cpu::compute_timestamp_commitment(&options, 1 << log_height); - println!(" // 2^{log_height}"); - println!( - " {},", - format_commitment(&commitment).replace("\n ", "\n ") - ); - } - println!(" ],\n ),"); - } - println!("];"); } diff --git a/prover/src/block_whir.rs b/prover/src/block_whir.rs new file mode 100644 index 000000000..06edde6c3 --- /dev/null +++ b/prover/src/block_whir.rs @@ -0,0 +1,9540 @@ +//! The block proved with the multilinear argument in ONE proof, no epochs: +//! prove-and-retire on WHIR. +//! +//! Same executor, traces and AIRs as [`crate::multilinear_prove`], at a table +//! height the card proves today (2^21 by default); every chunk of every table +//! type is a table of the proof — including KECCAK_RND, split at 2^16 rows so +//! no argument is taller than today's — and memory is the monolithic PAGE +//! argument (no local-to-global bookend, no cross-epoch proof). +//! +//! The proof is [`stark::multilinear_block`]'s: the tables split into groups a +//! card holds ([`block_groups`], a function of the shapes and of +//! [`BlockFormat`]); phase A commits each group and lets its codewords go; +//! every root goes into the transcript and `(z, α, β)` are drawn once; phase B +//! argues and opens each group on its own fork of that transcript. The bus +//! balance is checked once, over every table of the block. +//! +//! The columns an in-guest verifier cannot evaluate — DECODE's, and the dense +//! genesis pages' — are settled by PREPARED openings ([`prepared_tables`]): one +//! stack a group, which both sides derive from the program and the partition, +//! absorbed after the groups' roots and opened on the group's fork, as an epoch +//! settles DECODE. +//! +//! ★ #1010's epoch pipeline is untouched: this is a separate entry point with a +//! statement tag of its own ([`statement::MULTILINEAR_BLOCK_TAG`]). + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use executor::elf::Elf; +use executor::vm::execution::Executor; +use math::field::element::FieldElement; +use multilinear::mle::Mle; +use multilinear::whir_chain::{ArgueFormat, ChainConfig, GroupFold}; +use rayon::prelude::*; +use stark::multilinear_block::{self, BlockCommitted, GroupStamps, block_groups}; +use stark::multilinear_table::{ + BatchedArgue, CommittedTable, MultiProof, TableLayout, TableStatement, batched::VerifierChecks, +}; +use stark::table::Table; +use stark::trace::TraceTable; +use std::time::Instant; + +use crate::multilinear_prove::{ + absorb_tagged, chain_config_under, layout_of, preprocessed_mles, shapes_of, stacks, +}; +use crate::statement::{self, MULTILINEAR_BLOCK_TAG}; +use crate::tables::gpack::{self, TraceForm}; +use crate::tables::trace_builder::{ + ChunkJob, CutPadding, RowFamilies, StreamTable, Traces, WindowStamps, WindowedTraceBuilder, + head_log, parallel, +}; +use crate::test_utils::{E, F}; +use crate::zf_format::ZfFormat; +use crate::{ + BlockAirInputs, Error, FIXED_TABLE_COUNT, MaxRowsConfig, ProofOptions, RuntimePageRange, + TableCounts, VmAirs, block_airs, +}; +use stark::config::CommitmentHash; +use stark::proof::options::{BaseFormat, CapPolicy}; + +pub(crate) mod group_queue; +mod memlog; +pub mod regen; + +/// How many stacked polynomials a group may take. A format constant: both sides +/// derive the groups from it ([`block_groups`]). Three is today's heaviest +/// epoch (three 2^27 polynomials, epochs 3–6 and 13 of block 25368371), so no +/// group asks the card for more than an epoch does. +pub const BLOCK_GROUP_POLYS: usize = 3; + +/// The most KECCAK_RND tables a block statement may declare: a bound on what +/// the verifier builds before anything is checked, far above any block. +pub const BLOCK_MAX_KECCAK_RND: usize = 1 << 12; + +/// The most COPY tables a block statement may declare: a bound on what the +/// verifier builds before anything is checked, far above any block (a p90 +/// mainnet block makes 17 at 2^20 rows). +pub const BLOCK_MAX_COPY: usize = 1 << 12; + +/// The rows each COPY table holds at most — a prover's choice, under the +/// format's cap [`BLOCK_COPY_MAX_VARS`]. COPY is one row per eight bytes a +/// copy, memset or `write` moves (a row per byte for the last seven), so a +/// median mainnet block makes millions, which one table would pad to the next +/// power of two. A call's rows may straddle two tables: they chain only +/// through the CopyNext bus, keyed by the call's timestamp and its running +/// `(src, dst, count)`. At 2^20 rows a table is 41 columns of 2^20. +pub const BLOCK_COPY_ROWS_LOG2: usize = 20; + +/// The rows each KECCAK_RND table holds at most — a prover's choice, under +/// the format's cap [`BLOCK_KECCAK_RND_MAX_VARS`]: 2^16 is today's tallest, so +/// its argument's tree (the largest a block has) is today's. +pub const BLOCK_KECCAK_RND_ROWS_LOG2: usize = 16; + +/// The most ECDAS tables a block statement may declare: a bound on what the +/// verifier builds before anything is checked, far above any block (a p99 +/// mainnet block makes ≈ 15 at 2^17 rows). +pub const BLOCK_MAX_ECDAS: usize = 1 << 12; + +/// The rows each ECDAS table holds at most — a prover's choice, under the +/// format's cap [`BLOCK_ECDAS_MAX_VARS`]. ECDAS is one row per double/add step +/// (≈ 382 per ECSM call), so a median mainnet block makes ≈ 420 k rows: one +/// table of 2^19 would argue on a 12 GiB tree and a p90 block's 2^20 would +/// stack into five polynomials. A call's steps may straddle two tables: they +/// chain only through the Ecdas bus, keyed by the call's timestamp and the +/// step's `(round, op)`. +pub const BLOCK_ECDAS_ROWS_LOG2: usize = 17; + +/// The most KECCAK tables a block statement may declare: a bound on what the +/// verifier builds before anything is checked, far above any block (a +/// keccak-heavy block at the gas limit makes ≈ 16 at 2^17 rows). +pub const BLOCK_MAX_KECCAK: usize = 1 << 12; + +/// The rows each KECCAK table holds at most — a prover's choice, under the +/// format's cap [`BLOCK_KECCAK_MAX_VARS`]. KECCAK is one row per permutation, +/// so a cut falls between permutations; the rows of one Keccak-256 call may +/// straddle two tables, chained only through the KeccakSponge bus keyed by the +/// call's timestamp and the block index. A keccak-heavy block at the gas limit +/// makes ≈ 2^21 permutations, whose one table would stack into many +/// polynomials of a 2^27 stack; at 2^17 rows its 907 columns fit one. +pub const BLOCK_KECCAK_ROWS_LOG2: usize = 17; + +/// The most ECSM tables a block statement may declare: a bound on what the +/// verifier builds before anything is checked, far above any block. +pub const BLOCK_MAX_ECSM: usize = 1 << 12; + +/// The rows each ECSM table holds at most — a prover's choice, under the +/// format's cap [`BLOCK_ECSM_MAX_VARS`]. ECSM is one row per scalar +/// multiplication, so a cut falls between calls (a call's double/add steps are +/// ECDAS rows, reached through the Ecdas bus keyed by the call's timestamp). At +/// 2^17 rows its 667 columns fit one polynomial of a 2^27 stack. +pub const BLOCK_ECSM_ROWS_LOG2: usize = 17; + +/// The tallest KECCAK_RND table a block statement may state, in variables: a +/// verifier constant ([`check_chunked_heights`]), so no statement makes the +/// argument's tree taller than a 2^16-row table's. +pub const BLOCK_KECCAK_RND_MAX_VARS: usize = 16; + +/// The tallest ECDAS table a block statement may state, in variables: a +/// verifier constant ([`check_chunked_heights`]). At 2^17 rows ECDAS's argument +/// tree is ≈ 3 GiB and its columns fit one polynomial of a 2^27 stack. +pub const BLOCK_ECDAS_MAX_VARS: usize = 17; + +/// The tallest KECCAK table a block statement may state, in variables: a +/// verifier constant ([`check_chunked_heights`]), so no statement makes one +/// KECCAK table stack into more than one polynomial of a 2^27 stack. +pub const BLOCK_KECCAK_MAX_VARS: usize = 17; + +/// The tallest ECSM table a block statement may state, in variables: a +/// verifier constant ([`check_chunked_heights`]), so no statement makes one +/// ECSM table stack into more than one polynomial of a 2^27 stack. +pub const BLOCK_ECSM_MAX_VARS: usize = 17; + +/// The tallest COPY table a block statement may state, in variables: a +/// verifier constant ([`check_chunked_heights`]), the block's chunk height +/// (`BlockOptions::production`'s 2^21). +pub const BLOCK_COPY_MAX_VARS: usize = 21; + +/// The tallest CPU table a block statement may state, in variables: a +/// verifier constant ([`block_cpu_heights`]), the block's chunk height +/// (`BlockOptions::production`'s 2^21). The CPU tables' total stays under the +/// clock's bound (`tables::cpu::validate_heights`). +pub const BLOCK_CPU_MAX_VARS: usize = 21; + +/// The tallest table a block statement may state, in variables: a verifier +/// constant ([`check_table_heights`]). A group stacks at `want.min(cap).max(tallest)` +/// variables, so one taller table would make its chain taller than the 2^27 +/// stack and take its fold phase below the chains' conjectured (Johnson) +/// 130.393 bits — 128.393 under [`GroupFold::Indexed`] — (127.39 at a 2^30 +/// table). Honest tables are at most 2^21 rows. +pub const BLOCK_MAX_TABLE_VARS: usize = 27; + +const _: () = assert!(BLOCK_KECCAK_RND_ROWS_LOG2 <= BLOCK_KECCAK_RND_MAX_VARS); +const _: () = assert!(BLOCK_ECDAS_ROWS_LOG2 <= BLOCK_ECDAS_MAX_VARS); +const _: () = assert!(BLOCK_KECCAK_ROWS_LOG2 <= BLOCK_KECCAK_MAX_VARS); +// The KECCAK rows a block statement may declare stay under the sponge mode's +// counting bound (`tables::keccak::SPONGE_MAX_ROWS`): raising either cap past it +// fails to compile instead of weakening the argument. +const _: () = assert!( + (BLOCK_MAX_KECCAK as u128) << BLOCK_KECCAK_MAX_VARS <= crate::tables::keccak::SPONGE_MAX_ROWS +); +const _: () = assert!(BLOCK_ECSM_ROWS_LOG2 <= BLOCK_ECSM_MAX_VARS); +const _: () = assert!(BLOCK_COPY_ROWS_LOG2 <= BLOCK_COPY_MAX_VARS); + +/// Tree levels a group keeps OFF the host between its commit and its opening, +/// for a codeword on the host: a query re-hashes `2^4` of its codeword's cosets +/// there to rebuild them. A prover's choice; the proof does not depend on it. +pub const BLOCK_TREE_DROP_LEVELS: usize = 4; + +/// The same for a codeword the card holds, whose queried blocks are re-hashed +/// on the card (`DeviceCodeword::block_subtrees`): `2^8` cosets a query, which +/// costs the card ≈ 1 ms a round and takes the kept tops to 1/16 of +/// [`BLOCK_TREE_DROP_LEVELS`]'s — 0.0059 GiB a group, 1.1 GiB at p90 (I-WFULL +/// §5). Should the card's re-hash fail, the host re-hashes them instead, ≈ 16× +/// its cost at 4, and the block reports it as a WARN line. +pub const BLOCK_TREE_DROP_LEVELS_DEVICE: usize = 8; + +/// `LAMBDA_VM_BLOCK_TREE_DROP_LEVELS`: `` or `,`, the +/// levels a retired tree drops ([`multilinear::whir_commit::TreeDrop`]); +/// unset or unreadable, [`BLOCK_TREE_DROP_LEVELS_DEVICE`] and +/// [`BLOCK_TREE_DROP_LEVELS`]. A knob a RAM-adaptive policy can set; the proof +/// does not depend on it. +pub fn tree_drop_from_env() -> multilinear::whir_commit::TreeDrop { + parse_tree_drop( + std::env::var("LAMBDA_VM_BLOCK_TREE_DROP_LEVELS") + .ok() + .as_deref(), + ) +} + +fn parse_tree_drop(value: Option<&str>) -> multilinear::whir_commit::TreeDrop { + let default = multilinear::whir_commit::TreeDrop { + device: BLOCK_TREE_DROP_LEVELS_DEVICE, + host: BLOCK_TREE_DROP_LEVELS, + }; + let Some(value) = value.map(str::trim).filter(|v| !v.is_empty()) else { + return default; + }; + let mut parts = value.split(',').map(|p| p.trim().parse::().ok()); + match (parts.next().flatten(), parts.next(), parts.next()) { + (Some(device), None, None) => multilinear::whir_commit::TreeDrop { + device, + host: default.host, + }, + (Some(device), Some(Some(host)), None) => { + multilinear::whir_commit::TreeDrop { device, host } + } + _ => default, + } +} + +/// The verifier-side constants of a block proof. [`BlockFormat::production`] +/// is what the block runs at; the tests shrink the stack so a small program +/// spans several groups. +#[derive(Clone, Copy, Debug)] +pub struct BlockFormat { + pub zf: ZfFormat, + /// The most stacked polynomials a group may take: the prover packs to it, + /// and the verifier refuses a group whose stack needs more. + pub group_polys: usize, + /// The most groups a block statement may declare (the verifier refuses + /// more, and the prover refuses as its groups close). The proven bits are + /// quoted at this count (I-NOEPOCH-W.md §8). + pub max_groups: usize, + /// Whether DECODE's and the dense genesis pages' columns are settled by + /// prepared openings ([`prepared_tables`]) — the production format, which + /// the recursion needs — or evaluated by the host verifier. Off exists to + /// measure what the openings cost the base. + pub prepared: bool, + /// How big the recursion's leaves are + /// ([`crate::lfm::whir_block::LeafScale`]): the recursion's alone — the + /// block proof does not read it — and, like the rest, a constant the + /// prover and the verifier each hold. + pub leaf_scale: crate::lfm::whir_block::LeafScale, + /// How the recursion's leaf checks each chain's queries + /// ([`crate::lfm::whir_chain::LeafQuery`]): the recursion's alone, like + /// [`Self::leaf_scale`]. + pub leaf_query: crate::lfm::whir_chain::LeafQuery, + /// How each group's tables argue: one per group — a lockstep GKR per bin + /// of its trees and one constraint sumcheck over its tables + /// ([`ArgueFormat::Batched`], production: [`ArgueFormat::BATCHED`]) — or + /// one argument per table ([`ArgueFormat::PerTable`], a format to measure + /// against). Under the batched format the proof carries each group's argue + /// in [`BlockWhirProof::argues`]. + pub argue: ArgueFormat, + /// How each group's stacked polynomials are opened: one chain per run of + /// at most four ([`GroupFold::Indexed`], production), or + /// one chain each ([`GroupFold::PerPoly`]). A caller constant like the rest + /// of the format, bound into the statement through the fold word. + pub group: GroupFold, +} + +/// The most groups a block statement may declare: a verifier constant, never in +/// the transcript. The 1× block has 9 and the median under 100; a 200 Mgas +/// block makes some 600 to 925. The bound caps what a statement can make the +/// verifier build, and the block-wide LogUp support: its worst case at 1024 +/// groups is pinned by `block_wide_logup_at_the_group_maximum_is_pinned`. +pub const BLOCK_MAX_GROUPS: usize = 1024; + +/// Refuses `groups` over `max_groups` ([`BlockFormat::max_groups`]). The +/// verifier refuses a statement with more ([`block_frame`]), and the prover +/// refuses as its groups close, so a block the verifier would refuse is not +/// proved. +pub(crate) fn check_group_count(groups: usize, max_groups: usize) -> Result<(), Error> { + if groups > max_groups { + return Err(Error::InvalidTableCounts(format!( + "{groups} groups — a block takes at most {max_groups}" + ))); + } + Ok(()) +} + +/// The tallest 4-ary cap a Poseidon1 base may name: the trees clamp a taller +/// one (`multilinear::whir_commit::tree_cap_height`), so [`checked_base`] +/// refuses it rather than let the tag name a height no tree has. +pub const MAX_P1_CAP_HEIGHT: usize = crypto::merkle_tree::cap::MAX_CAP_HEIGHT / 2; + +/// The block's base, or a typed refusal: exactly [`crate::hash_pin::BLOCK_BASE`], +/// the block proof family's, whose configuration [`crate::with_block_hash!`] +/// binds. The block proves and verifies one family end to end (D-P3B §2.1), +/// so any other base — RPX's included — is refused, never dispatched: no base +/// of one hash under another's recursion. +pub fn checked_base(base: &BaseFormat) -> Result<(), Error> { + let family = crate::hash_pin::BLOCK_BASE; + if *base != family { + return Err(Error::Prover(format!( + "a block's base is its proof family's ({:?}, 4-ary cap {}), not {:?} (cap {})", + family.hash, family.arity4_cap, base.hash, base.arity4_cap + ))); + } + Ok(()) +} + +// The block's base is Poseidon1 at a cap its trees run at. +const _: () = { + assert!(matches!( + crate::hash_pin::BLOCK_BASE.hash, + CommitmentHash::Poseidon1 + )); + if let CapPolicy::Fixed(c) = crate::hash_pin::BLOCK_BASE.arity4_cap { + assert!(c as usize <= MAX_P1_CAP_HEIGHT); + } +}; + +/// The block statement's leading tag under `base`'s geometry: +/// `LAMBDAVM_MULTILINEAR_BLOCK_STATEMENT_V2/P1W16/C`, the effective cap: +/// `C0` uncapped (`Off` and `Fixed(0)`), `C` at a fixed height and `Cauto` +/// under `Auto`, so no two geometries share a tag (REV-P1-JUDGE F2). The hash +/// part is the block family's (`hash_pin::BLOCK_BASE`, Poseidon1 at width +/// 16): every site that absorbs a statement admits the base through +/// [`checked_base`] first, which refuses any other. Domain separation only. +pub fn block_statement_tag(base: &BaseFormat) -> Vec { + #[cfg(test)] + if P1_TAG_OMITTED.with(std::cell::Cell::get) { + return MULTILINEAR_BLOCK_TAG.to_vec(); + } + let height = match base.arity4_cap { + CapPolicy::Off | CapPolicy::Fixed(0) => "0".to_string(), + CapPolicy::Fixed(c) => c.to_string(), + CapPolicy::Auto => "auto".to_string(), + }; + let mut tag = MULTILINEAR_BLOCK_TAG.to_vec(); + tag.extend_from_slice(format!("/P1W16/C{height}").as_bytes()); + tag +} + +#[cfg(test)] +thread_local! { + /// Test only: on this thread, a statement takes the untagged + /// `LAMBDAVM_MULTILINEAR_BLOCK_STATEMENT_V2` (the tag's mutation). + /// Thread-local, so a test that sets it moves no other test's statement. + pub(crate) static P1_TAG_OMITTED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +impl BlockFormat { + /// The process's WHIR format (as [`crate::multilinear_prove::chain_config`] + /// reads it), [`BLOCK_GROUP_POLYS`], [`BLOCK_MAX_GROUPS`], each group's + /// tables argued together ([`ArgueFormat::BATCHED`]; Mauro 10-02: "batch the + /// constraints"), and one WHIR chain per run of a group's stacked + /// polynomials ([`GroupFold::Indexed`]). `ArgueFormat::PerTable` and + /// [`GroupFold::PerPoly`] stay formats to measure against. The leaves' + /// scale is the environment's (`LAMBDA_VM_WHIR_LEAF_DOUBLE_FROM`; unset, + /// [`LeafScale::DEFAULT`](crate::lfm::whir_block::LeafScale::DEFAULT), + /// every leaf doubled and one top over at most three of them; a count `T`, + /// doubled from `T` leaves; `off`, every leaf at the single heights). The + /// leaves' query check is the environment's too + /// (`LAMBDA_VM_WHIR_LEAF_QUERY`; unset, + /// [`LeafQuery::Lean`](crate::lfm::whir_chain::LeafQuery::Lean); `v1`, the + /// check before it). + pub fn production() -> Self { + Self { + zf: *ZfFormat::global(), + group_polys: BLOCK_GROUP_POLYS, + max_groups: BLOCK_MAX_GROUPS, + prepared: true, + leaf_scale: crate::lfm::whir_block::process_leaf_scale(), + leaf_query: crate::lfm::whir_block::process_leaf_query(), + argue: ArgueFormat::BATCHED, + group: GroupFold::Indexed, + } + } + + /// [`production`](Self::production) with each group's stacked polynomials + /// opened per `group`: the caller's constant, which the prover and the + /// verifier of one block must share (it is bound into the statement + /// through the fold word, and never read from a proof). + pub fn production_with(group: GroupFold) -> Self { + Self { + group, + ..Self::production() + } + } + + /// ★ The block VERIFIER's production format, every field a constant: + /// [`ZfFormat::DEFAULT`] (blowup, folds, stack, grind placement and bits, + /// the caps, the round-0 leaf order), + /// [`LeafScale::DEFAULT`](crate::lfm::whir_block::LeafScale::DEFAULT), + /// [`LeafQuery::Lean`](crate::lfm::whir_chain::LeafQuery::Lean), and + /// [`production`](Self::production)'s group size, group cap, argue and + /// group opening. Where [`production`](Self::production) is the process's + /// format (its environment knobs, for the prover and its experiments), + /// this reads nothing from the environment: a verifier's environment + /// cannot choose the format, and with it the security, it checks a proof + /// against. + pub fn pinned() -> Self { + Self { + zf: ZfFormat::DEFAULT, + group_polys: BLOCK_GROUP_POLYS, + max_groups: BLOCK_MAX_GROUPS, + prepared: true, + leaf_scale: crate::lfm::whir_block::LeafScale::DEFAULT, + leaf_query: crate::lfm::whir_chain::LeafQuery::Lean, + argue: ArgueFormat::BATCHED, + group: GroupFold::Indexed, + } + } + + /// [`pinned`](Self::pinned) with each group's stacked polynomials opened + /// per `group`, the verifier's named choice + /// ([`production_with`](Self::production_with)). + pub fn pinned_with(group: GroupFold) -> Self { + Self { + group, + ..Self::pinned() + } + } + + /// This format with its batched argue's GKR trees laid out per `pack`: + /// a named constant the prover and the + /// verifier of one block both take, never read from a proof. Production + /// is [`GkrPack::PRODUCTION`](multilinear::whir_chain::GkrPack::PRODUCTION); + /// [`GkrPack::Off`](multilinear::whir_chain::GkrPack::Off) is one tree a + /// table. A per-table argue has no trees to pack and is returned as it is. + pub fn with_pack(self, pack: multilinear::whir_chain::GkrPack) -> Self { + let argue = match self.argue { + ArgueFormat::Batched { bin_log_cells, .. } => ArgueFormat::Batched { + bin_log_cells, + pack, + }, + ArgueFormat::PerTable => ArgueFormat::PerTable, + }; + Self { argue, ..self } + } + + /// The chain's config over `shapes`: the WHIR format's, with this format's + /// argue, group fold and round-0 leaf order (the block's alone: + /// `ZfFormat::chain_format` leaves every other chain natural). Every side + /// of a block builds its config here. + pub fn chain_config(&self, shapes: &[(usize, usize)]) -> ChainConfig { + let mut config = chain_config_under(&self.zf, shapes); + config.format.argue = self.argue; + config.format.group = self.group; + config.format.leaves = self.zf.whir_leaves; + config + } +} + +/// The prover's own choices: none of them is in the proof. +#[derive(Clone, Debug)] +pub struct BlockOptions { + pub max_rows: MaxRowsConfig, + pub keccak_rnd_rows_log2: usize, + /// ECDAS's tables are cut at 2^`ecdas_rows_log2` rows + /// ([`BLOCK_ECDAS_ROWS_LOG2`] in production). + pub ecdas_rows_log2: usize, + /// KECCAK's tables are cut at 2^`keccak_rows_log2` rows + /// ([`BLOCK_KECCAK_ROWS_LOG2`] in production). + pub keccak_rows_log2: usize, + /// ECSM's tables are cut at 2^`ecsm_rows_log2` rows + /// ([`BLOCK_ECSM_ROWS_LOG2`] in production). + pub ecsm_rows_log2: usize, + /// COPY's tables are cut at 2^`copy_rows_log2` rows + /// ([`BLOCK_COPY_ROWS_LOG2`] in production). + pub copy_rows_log2: usize, + /// The bottom levels each committed tree drops until its opening, on the + /// card and on the host. Production: [`tree_drop_from_env`]. + pub drop_levels: multilinear::whir_commit::TreeDrop, + /// `Some(k)`: build the traces in windows of 2^k cycles and commit each + /// table as its chunk completes ([`WindowedTraceBuilder`]); `None`: build + /// the whole run, then commit. + pub window_log2: Option, + /// With windows: the builder drops each streamed chunk's ops as the chunk + /// leaves ([`WindowedTraceBuilder::drop_streamed_ops`]), so the build does + /// not hold the run's op lists to its end; the traces are the same. On in + /// production (D-MEMORY M1, FAST 501: 1× base −0.74 s, peak 47.4 → 36.6 + /// GiB, the 1.20× block proves). + pub drop_streamed_ops: bool, + /// With windows: `0` lays each streamed chunk out on the layout thread as + /// it arrives; `n > 0` lays them out on `n` threads, packed in arrival + /// order all the same (the same groups and proof bytes), while the layout + /// thread lays out the rest of the run as soon as it is built. Production: + /// [`BLOCK_LAYOUT_WORKERS`]. Once phase A uploads ahead, the inline layout + /// closes the middle groups after the card is free; three workers close + /// them in time (FAST 835: phase A −0.93 s, whole block −1.00 s, peak −0.68 + /// GiB at 1×). Unbounded they held more host memory as the block grew (BIG + /// 390); `layout_ahead` bounds them (BIG 392: flat). + pub layout_workers: usize, + /// With `layout_workers > 0`: `Some(k)` lets at most `k + 1` streamed + /// chunks be laid out (or in the making) and not yet packed, where the + /// inline layout holds one; `None` leaves only the channels to bound them, + /// which BIG 390 saw grow with the block. Production + /// `Some(`[`BLOCK_LAYOUT_AHEAD`]`)`. + pub layout_ahead: Option, + /// How phase A holds each group's columns once committed + /// ([`multilinear_block::Narrowing`]): production packs them narrow on the + /// card, which leaves the proof's bytes as they are. + pub narrow: multilinear_block::Narrowing, + /// Phase A takes the next group and puts its columns on the card while the + /// current group commits, when the ledger takes them + /// ([`BlockCommitted::commit_groups`]); the commits and the proof's bytes + /// are the same. Production: on. + pub upload_ahead: bool, + /// With windows: print where the host memory is, term by term, every half + /// second and at each mark ([`memlog`]'s `BLOCK MEM` lines). A + /// measurement; it moves no proof byte. Production reads + /// `LAMBDA_VM_BLOCK_MEMLOG=1` (off by default). + pub memlog: bool, + /// With windows: the finish builds KECCAK_RND + /// as its 2^`keccak_rnd_rows_log2`-row tables + /// ([`WindowedTraceBuilder::keccak_rnd_chunks_at_finish`]) instead of one + /// table that [`split_keccak_rnd`] then copies apart; the tables are the + /// same. Production: on (BIG 561: the split's copy is the finish's last + /// ≈ 6 GiB at 4.13×). A cut under 32 rows (tests) takes the split. + pub finish_keccak_rnd_chunks: bool, + /// With windows: the finish builds KECCAK, ECSM and ECDAS as their + /// 2^`keccak_rows_log2` / 2^`ecsm_rows_log2` / 2^`ecdas_rows_log2`-row + /// tables ([`WindowedTraceBuilder::cuts_at_finish`]), each packed as it is + /// built with [`Self::pack_finished`], instead of one wide table each that + /// [`split_keccak`], [`split_ecsm`] and [`split_ecdas`] then copy apart; + /// the tables are the same. Production: on unless + /// `LAMBDA_VM_BLOCK_FINISH_PLAN=0` (the A arm: built whole and wide, then + /// split). At p90 the three are the rest's only wide tables (5.09 GiB) and + /// the rest's layout transposes them into fresh pages (+4.7 GiB VmRSS, BIG + /// 660). + pub finish_cuts: bool, + /// With windows, the pipelined layout and the finish's cuts: the finish + /// streams the rest ([`WindowedTraceBuilder::finish_streamed`]): its tables + /// built in waves in AIR order under a byte gate of + /// [`Self::rest_layout_bytes`] (generated and not yet placed), each family's + /// ops freed with its last table, the layout taking each table as it comes + /// and the packer placing it, so the rest's first groups commit while the + /// finish still builds the later ones. The tables, their order and the + /// groups are the same. Production: on unless `LAMBDA_VM_BLOCK_FINISH_STREAM=0` + /// (the A arm: every table built at once, then laid out). + pub finish_stream: bool, + /// With windows: `Some(bytes)` lays the rest of the run out in AIR order, + /// in waves of at most `bytes` of rows (a larger table is a wave of its + /// own), each wave in parallel; `None` lays every table out at once. A + /// table's column copy exists before its rows go, so laying out all of + /// them at once holds most of the rest twice (BIG 561: ≈ 20 GiB at 4.13×, + /// the block's high-water). The tables, their order and the groups are the + /// same. Production: [`BLOCK_REST_LAYOUT_BYTES`]. + /// + /// With the streamed finish ([`Self::finish_stream`]) the same bytes are + /// its byte gate's budget, and the sliding finish (the default, + /// `LAMBDA_VM_BLOCK_REST_SLIDE`) has no waves: it starts each table as the + /// gate admits it. + pub rest_layout_bytes: Option, + /// With the streamed finish sliding: at most this many estimated bytes of + /// the rest's tables build at once, apart from the gate's budget + /// ([`Self::rest_layout_bytes`]), which bounds what is held (built and not + /// yet placed, or building). `None`: the budget alone. Production: + /// [`BLOCK_REST_BUILD_BYTES`]. The tables, their order and the groups are + /// the same. + pub rest_build_bytes: Option, + /// With windows: the finish packs each table it builds at the bytes its + /// columns need as the table is generated + /// ([`WindowedTraceBuilder::pack_finished_tables`]), and the table is laid + /// out narrow from that, committed from the card and never held wide on + /// the host. KECCAK, ECSM, ECDAS and an unchunked KECCAK_RND stay wide. + /// The proof's bytes are the same. Production: on. + pub pack_finished: bool, + /// G-pack (`tables::gpack`): the generators write each table packed as + /// they generate it, with no 64-bit copy — every streamed chunk (laid out + /// narrow from that, when the groups are held narrow) and, with + /// [`Self::pack_finished`], every table the finish packs. The proof's bytes + /// are the same. Production: on unless `LAMBDA_VM_BLOCK_GPACK=0` (the A + /// arm: built at 8 bytes a cell, then laid out or packed as before). + pub gpack: bool, + /// What phase B does with the streamed chunks ([`regen::RegenMode`]): + /// `None` reads `LAMBDA_VM_BLOCK_REGEN` when the prove starts (unset: + /// `off`; a value it does not know refuses the prove), `Some` is chosen. + /// Production: [`regen::production_regen`], `auto` when that knob is + /// unset. Without live regeneration every committed table is held. The + /// proof's bytes are the same under every mode. + pub regen: Option, + /// Phase B opens each group on a thread of its own beside the next + /// group's argue, and under `Pinned` the openings' thread reads back + /// through pinned staging ([`multilinear_block::PhaseBOverlap`]); the + /// proof's bytes are the same. Production: [`phase_b_overlap_from_env`], + /// `Pinned` by default on a card build ([`PHASE_B_OVERLAP_DEFAULT`]). + pub phase_b_overlap: multilinear_block::PhaseBOverlap, +} + +/// The host's memory target, against which live regeneration's `auto` rule +/// arms ([`drop_wanted`]): `LAMBDA_VM_BLOCK_MEMORY_TARGET_GIB` (read under its +/// old name, `LAMBDA_VM_BLOCK_SPILL_TARGET_GIB`, while the new one is unset), +/// else the smaller of the cgroup's memory limit (v2 or v1, [`cgroup_memory`]) +/// and `MemTotal`, less 10 GiB ([`memory_target_from`]) (#1013's +/// `spill_target_bytes` @ 035aef5d6). +pub fn memory_target_bytes() -> u64 { + if let Some(bytes) = parse_target_gib(renamed_knob( + |name| std::env::var(name).ok(), + "LAMBDA_VM_BLOCK_MEMORY_TARGET_GIB", + "LAMBDA_VM_BLOCK_SPILL_TARGET_GIB", + )) { + return bytes; + } + let limit = cgroup_memory( + &std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default(), + std::path::Path::new(CGROUP_ROOT), + CgroupValue::File("memory.max"), + CgroupValue::File("memory.limit_in_bytes"), + ); + let mem_total = std::fs::read_to_string("/proc/meminfo").ok().and_then(|m| { + m.lines() + .find_map(|l| l.strip_prefix("MemTotal:")) + .and_then(|v| v.trim().trim_end_matches("kB").trim().parse::().ok()) + .map(|kib| kib << 10) + }); + memory_target_from(limit, mem_total) +} + +/// A knob's value in `env` under `name`, else under `old`, the name it had +/// while the block had a disk store: the old name is still read for one +/// release. +fn renamed_knob(env: impl Fn(&str) -> Option, name: &str, old: &str) -> Option { + env(name).or_else(|| env(old)) +} + +/// The memory target's knob in GiB as bytes; unset, or not a finite number +/// above 0, is `None` (the machine's target). +fn parse_target_gib(value: Option) -> Option { + value + .and_then(|v| v.trim().parse::().ok()) + .filter(|g| g.is_finite() && *g > 0.0) + .map(|gib| (gib * (1u64 << 30) as f64) as u64) +} + +/// The smaller of a cgroup limit and `MemTotal`, less 10 GiB; no target +/// (`u64::MAX`) when neither is known. A v1 cgroup without a limit reads a +/// sentinel far above `MemTotal`, which the minimum discards (#1013's +/// `spill_target_from`). +fn memory_target_from(limit: Option, mem_total: Option) -> u64 { + const MARGIN: u64 = 10 << 30; + match (limit, mem_total) { + (Some(limit), Some(total)) => limit.min(total), + (Some(bytes), None) | (None, Some(bytes)) => bytes, + (None, None) => return u64::MAX, + } + .saturating_sub(MARGIN) +} + +/// Where the cgroup filesystem is mounted. +const CGROUP_ROOT: &str = "/sys/fs/cgroup"; + +/// One number of a cgroup's memory files: a file that holds just the number, +/// or one `key value` line of `memory.stat` (#1013's `CgroupValue`). +#[derive(Clone, Copy)] +enum CgroupValue<'a> { + File(&'a str), + Stat(&'a str), +} + +impl CgroupValue<'_> { + fn read(self, dir: &std::path::Path) -> Option { + let (file, key) = match self { + CgroupValue::File(file) => (file, None), + CgroupValue::Stat(key) => ("memory.stat", Some(key)), + }; + let text = std::fs::read_to_string(dir.join(file)).ok()?; + match key { + None => text.trim().parse::().ok(), + Some(key) => text.lines().find_map(|l| { + let (k, v) = l.split_once(' ')?; + (k == key).then(|| v.trim().parse::().ok()).flatten() + }), + } + } +} + +/// A number from this process's cgroup memory files: `v2` under the unified +/// hierarchy (`memory.max`, `memory.current`, `memory.stat`'s `inactive_file`), +/// else `v1` under the memory controller (`memory.limit_in_bytes`, +/// `memory.usage_in_bytes`, `memory.stat`'s `total_inactive_file`). +/// `proc_cgroup` is `/proc/self/cgroup`, `root` the cgroup mount. Each is read +/// at the process's cgroup path, then at its hierarchy's root, which a +/// container without a cgroup namespace sees as its own cgroup. `None` where +/// neither reads as a number (v2's `max` included) (#1013's +/// `cgroup_memory`). +fn cgroup_memory( + proc_cgroup: &str, + root: &std::path::Path, + v2: CgroupValue, + v1: CgroupValue, +) -> Option { + let at = |base: &std::path::Path, path: &str, value: CgroupValue| { + value + .read(&base.join(path.trim().trim_start_matches('/'))) + .or_else(|| value.read(base)) + }; + let (mut unified, mut memory) = (None, None); + for line in proc_cgroup.lines() { + let mut fields = line.splitn(3, ':'); + let (Some(id), Some(controllers), Some(path)) = + (fields.next(), fields.next(), fields.next()) + else { + continue; + }; + if id == "0" && controllers.is_empty() { + unified = Some(path); + } else if controllers.split(',').any(|c| c == "memory") { + memory = Some(path); + } + } + unified + .and_then(|path| at(root, path, v2)) + .or_else(|| memory.and_then(|path| at(&root.join("memory"), path, v1))) +} + +/// What `auto` reads of the host ([`HostReading::bytes`]) (#1013's +/// `HostReading` @ 035aef5d6, from cf253d235). +#[derive(Clone, Copy, Debug, Default, PartialEq)] +struct HostReading { + /// The process's peak resident set (`VmHWM`). + hwm: u64, + /// The cgroup's charge (`memory.current`, or v1's `memory.usage_in_bytes`), + /// page cache included. + charged: u64, + /// Its inactive file pages (`memory.stat`'s `inactive_file`, or v1's + /// `total_inactive_file`): the page cache the kernel reclaims first. + inactive_file: u64, +} + +impl HostReading { + fn now() -> Self { + let hwm = std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|s| { + s.lines() + .find_map(|l| l.strip_prefix("VmHWM:")) + .and_then(|v| v.trim().trim_end_matches("kB").trim().parse::().ok()) + .map(|kib| kib << 10) + }); + let proc_cgroup = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default(); + let root = std::path::Path::new(CGROUP_ROOT); + let charged = cgroup_memory( + &proc_cgroup, + root, + CgroupValue::File("memory.current"), + CgroupValue::File("memory.usage_in_bytes"), + ); + let inactive_file = cgroup_memory( + &proc_cgroup, + root, + CgroupValue::Stat("inactive_file"), + CgroupValue::Stat("total_inactive_file"), + ); + Self { + hwm: hwm.unwrap_or(0), + charged: charged.unwrap_or(0), + inactive_file: inactive_file.unwrap_or(0), + } + } + + /// The host's bytes: the larger of the peak resident set (the resident set + /// itself is not monotone under the allocator's posture, which drops and + /// re-faults recycled extents) and the cgroup's working set, its charge + /// less its inactive file pages (the kubelet's measure). Page cache the + /// kernel would give back first is not counted; active file pages are, + /// and the target's 10 GiB margin covers them. + fn bytes(&self) -> u64 { + self.hwm + .max(self.charged.saturating_sub(self.inactive_file)) + } + + /// What the host holds now: the cgroup's working set and the allocator's + /// resident bytes (`allocator`), the larger. Unlike [`Self::bytes`] it + /// falls when a purge hands freed pages back. + fn now_bytes(&self, allocator: Option) -> u64 { + self.charged + .saturating_sub(self.inactive_file) + .max(allocator.unwrap_or(0)) + } +} + +/// The host's bytes a drop decision weighs: its peak ([`HostReading::bytes`]), +/// or, under partial drops once they have armed (`now`), what it holds now +/// ([`HostReading::now_bytes`]): a dropped table's pages, once purged, leave +/// the reading, and later tables may stay. +fn decision_bytes(reading: &HostReading, allocator: Option, now: bool) -> u64 { + if now { + reading.now_bytes(allocator) + } else { + reading.bytes() + } +} + +/// `LAMBDA_VM_BLOCK_DROP`: unset, empty or `partial` (the default) drops, +/// once live regeneration arms, each table phase B can rebuild only while the +/// policy wants it out, on what the host holds now, and purges the dropped +/// tables' pages so the reading falls +/// ([`multilinear_block::BlockRegen::partial`], [`DropPurges`]); `all` drops +/// every such table, those already parked with them (the old default, kept +/// as the rollback). Anything else refuses the prove. The proof is the same. +fn block_drop() -> Result { + parse_drop(std::env::var("LAMBDA_VM_BLOCK_DROP").ok().as_deref()) +} + +/// What live regeneration drops once armed ([`block_drop`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DropMode { + All, + Partial, +} + +fn parse_drop(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "partial") => Ok(DropMode::Partial), + Some("all") => Ok(DropMode::All), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_DROP={v}: all or partial" + ))), + } +} + +/// `LAMBDA_VM_BLOCK_KEEP_CELLS_G`: partial drops' keep budget at arming, in G +/// main cells committed ([`multilinear_block::BlockRegen::keep_cells`]). Once +/// live regeneration arms, the tables phase B can rebuild that were committed +/// past it are dropped back, in rank order, so what phase B holds no longer +/// follows how far phase A got when the policy first wanted a table out. +/// Unset, empty or `off` (the default): no budget. It is a constant read +/// against today's forecast (its reserve per G cells and its target); the +/// `BLOCK REGEN keep` line says each run's arming point. Anything else refuses +/// the prove. The proof is the same. +fn block_keep_cells() -> Result, Error> { + parse_keep_cells(std::env::var(KEEP_CELLS_ENV).ok().as_deref()) +} + +const KEEP_CELLS_ENV: &str = "LAMBDA_VM_BLOCK_KEEP_CELLS_G"; + +/// `LAMBDA_VM_BLOCK_KEEP_BACK_G`: partial drops' keep budget relative to the +/// arming point, in G main cells +/// ([`multilinear_block::BlockRegen::keep_below`]): once live regeneration +/// arms, the tables phase B can rebuild that were committed in the last that +/// many cells before it are dropped back, in rank order. The arming point +/// moves between runs (the host's reading when the forecast crosses), and the +/// budget moves with it. Unset or empty: [`BLOCK_KEEP_BELOW`] (the default); a +/// G-cell count above 0: that margin; `off`: none, arming drops back nothing +/// (the rollback). Anything else refuses the prove. The proof is the same. +fn block_keep_below() -> Result, Error> { + parse_keep_below(std::env::var(KEEP_BELOW_ENV).ok().as_deref()) +} + +const KEEP_BELOW_ENV: &str = "LAMBDA_VM_BLOCK_KEEP_BACK_G"; + +/// Partial drops' keep budget relative to the arming point by default, in main +/// cells below those committed at arming ([`block_keep_below`]): about what +/// the streamed layout eight chunks ahead commits before arming beyond what it +/// does six ahead (1.08 and 1.34 G cells in two runs of the large block), so a +/// phase A that commits sooner keeps no more through phase B than it would +/// have, wherever the run arms. A block that never arms (the median) is +/// untouched. +pub const BLOCK_KEEP_BELOW: u64 = 1_300_000_000; + +fn parse_keep_below(value: Option<&str>) -> Result, Error> { + match value.map(str::trim) { + None | Some("") => Ok(Some(BLOCK_KEEP_BELOW)), + Some("off") => Ok(None), + Some(v) => v + .parse::() + .ok() + .filter(|g| g.is_finite() && *g > 0.0) + .map(|g| Some((g * 1e9) as u64)) + .ok_or_else(|| Error::Prover(format!("{KEEP_BELOW_ENV}={v}: off, or G cells above 0"))), + } +} + +fn parse_keep_cells(value: Option<&str>) -> Result, Error> { + match value.map(str::trim) { + None | Some("" | "off") => Ok(None), + Some(v) => v + .parse::() + .ok() + .filter(|g| g.is_finite() && *g > 0.0) + .map(|g| Some((g * 1e9) as u64)) + .ok_or_else(|| Error::Prover(format!("{KEEP_CELLS_ENV}={v}: off, or G cells above 0"))), + } +} + +/// The dropped bytes after which partial drops purge ([`DropPurges`]). +const DROP_PURGE_BYTES: u64 = 4 << 30; +/// The least time between two of their purges. +const DROP_PURGE_SPACING: std::time::Duration = std::time::Duration::from_secs(5); + +/// Partial drops' purges: once [`DROP_PURGE_BYTES`] have been dropped since the +/// last purge, and that one was [`DROP_PURGE_SPACING`] ago, a purge on a thread +/// of its own hands the allocator's freed pages back, so the host's reading +/// falls by the dropped tables. The committer that decides never waits on it. +struct DropPurges { + purge: fn() -> Option, + pending: std::sync::atomic::AtomicU64, + running: std::sync::Arc, + last: std::sync::Mutex>, + count: std::sync::atomic::AtomicUsize, + micros: std::sync::atomic::AtomicU64, + returned: std::sync::atomic::AtomicU64, +} + +impl DropPurges { + fn new(purge: fn() -> Option) -> std::sync::Arc { + std::sync::Arc::new(Self { + purge, + pending: Default::default(), + running: Default::default(), + last: std::sync::Mutex::new(None), + count: Default::default(), + micros: Default::default(), + returned: Default::default(), + }) + } + + /// A table of `bytes` the policy dropped. Returns the purge's thread when + /// this drop started one. + fn dropped(self: &std::sync::Arc, bytes: u64) -> Option> { + use std::sync::atomic::Ordering::{AcqRel, Relaxed, Release}; + let pending = self.pending.fetch_add(bytes, Relaxed).saturating_add(bytes); + if pending < DROP_PURGE_BYTES { + return None; + } + let spaced = self + .last + .lock() + .unwrap_or_else(|e| e.into_inner()) + .is_none_or(|t| t.elapsed() >= DROP_PURGE_SPACING); + if !spaced || self.running.swap(true, AcqRel) { + return None; + } + self.pending.store(0, Relaxed); + let me = std::sync::Arc::clone(self); + let spawned = std::thread::Builder::new() + .name("block-drop-purge".to_string()) + .spawn(move || { + if let Some(purge) = (me.purge)() { + me.count.fetch_add(1, Relaxed); + me.micros.fetch_add((purge.secs * 1e6) as u64, Relaxed); + let returned = purge.resident_before.saturating_sub(purge.resident_after); + me.returned.fetch_add(returned as u64, Relaxed); + } + *me.last.lock().unwrap_or_else(|e| e.into_inner()) = Some(Instant::now()); + me.running.store(false, Release); + }); + match spawned { + Ok(handle) => Some(handle), + Err(_) => { + self.running.store(false, Release); + None + } + } + } + + /// The `BLOCK DROP` line. + fn line(&self) -> String { + use std::sync::atomic::Ordering::Relaxed; + format!( + "BLOCK DROP: partial · {} purges of the dropped tables' pages, {:.2} s, {:.2} GiB returned", + self.count.load(Relaxed), + self.micros.load(Relaxed) as f64 / 1e6, + self.returned.load(Relaxed) as f64 / (1u64 << 30) as f64 + ) + } +} + +/// The `auto` rule's reserve beside the host's bytes: what the block still +/// needs once the tables committed so far (`cells` main cells) are on the host. #1014's +/// own, inferred from the median block (BIG 565, 33.5 G cells): the finish's +/// p5 transient (25.5 GiB at its peak) and the tree's leaf programs, which +/// live through phase B (16.3 GiB at its end), ≈ 1.25 GiB per total G cells, +/// or ≈ 1.65 per G cells committed so far (the streamed share ≈ 0.76); plus +/// 6 GiB for phase B's bump and the read-back window. `per_g` is the GiB per +/// G cells committed ([`MEMORY_RESERVE_PER_G`] unless the knob retunes it, +/// [`memory_reserve_per_g`]). +fn memory_reserve_bytes(cells: u64, per_g: f64) -> u64 { + (cells as f64 / 1e9 * per_g * (1u64 << 30) as f64) as u64 + (6 << 30) +} + +/// [`memory_reserve_bytes`]'s GiB per G cells committed. +const MEMORY_RESERVE_PER_G: f64 = 1.65; + +/// `LAMBDA_VM_BLOCK_MEMORY_RESERVE_PER_G` (read under its old name, +/// `LAMBDA_VM_BLOCK_SPILL_RESERVE_PER_G`, while the new one is unset): the +/// `auto` rule's reserve per G cells committed, in GiB +/// ([`memory_reserve_bytes`]), a measurement knob for its retune (D-FINISH S4: +/// it still budgets the finish's transient S2b removed); unset, or not a finite +/// number ≥ 0, is [`MEMORY_RESERVE_PER_G`]. +fn memory_reserve_per_g() -> f64 { + parse_reserve_per_g( + renamed_knob( + |name| std::env::var(name).ok(), + "LAMBDA_VM_BLOCK_MEMORY_RESERVE_PER_G", + "LAMBDA_VM_BLOCK_SPILL_RESERVE_PER_G", + ) + .as_deref(), + ) +} + +fn parse_reserve_per_g(value: Option<&str>) -> f64 { + value + .and_then(|v| v.trim().parse::().ok()) + .filter(|g| g.is_finite() && *g >= 0.0) + .unwrap_or(MEMORY_RESERVE_PER_G) +} + +/// The share of the memory target ([`memory_target_bytes`]) past which the block's +/// memory is short for the allocator's purge ([`crate::alloc_purge`]): 85 %, a +/// share so it follows the host's RAM. +const PRESSURE_SHARE: (u64, u64) = (17, 20); + +/// Notes memory pressure for [`crate::alloc_purge`] once the host's bytes +/// ([`HostReading::bytes`]) pass [`PRESSURE_SHARE`] of `target`, and says +/// whether they did. +/// +/// ★ THE ONE WAY #1014'S PURGE DIFFERS FROM #1013'S. The module, its knob +/// (`LAMBDA_VM_ALLOC_PURGE`, `auto` by default) and its points are #1013's; +/// #1013 notes the pressure when its spill's queue budgets arm, and #1014 has +/// no budgets, so it reads the host where live regeneration decides (each +/// committed table) and once more at phase A's end. A block whose host stays +/// under the share (the median: at most 86.6 GiB against 94.1 on BIG) never +/// purges. +fn note_pressure_past(host: u64, target: u64) -> bool { + let short = memory_short(host, target); + if short { + crate::alloc_purge::note_memory_pressure(); + } + short +} + +/// The line past which the block's memory is short: [`PRESSURE_SHARE`] of +/// `target`. +fn pressure_line(target: u64) -> u64 { + let (num, den) = PRESSURE_SHARE; + target / den * num +} + +/// Whether `host` bytes are past [`PRESSURE_SHARE`] of `target`. +fn memory_short(host: u64, target: u64) -> bool { + host > pressure_line(target) +} + +/// What the allocator's purges at phase A's end and the base's end weigh in a +/// block that proves with live regeneration ([`crate::alloc_purge::has_room`]): +/// the pressure line of `target`, and the regeneration window's ceiling as +/// what the block may still add. `None` for a block without it: it purges at +/// both whenever its memory is short, as ever. +/// +/// The pressure such a block notes comes from [`hand_off_need`]'s forecast of +/// the finish's growth, which is the growth of tables that stay; a block with +/// live regeneration drops them, and on a host with room its resident set +/// never comes near the line the forecast crossed. +fn no_disk_room(no_disk: bool, target: u64) -> Option { + no_disk.then(|| crate::alloc_purge::NoDisk { + line: pressure_line(target), + headroom: regen::AHEAD_CAP, + }) +} + +/// The finish's growth of the resident set, from the walk's end to phase A's +/// peak, per walked cycle: BIG 660's p90 (D, at defaults) went 85.19 → 115.27 +/// GiB over 612.3 M cycles (I-WFULL §4.1). The median (653) grew 12.41 GiB over +/// 298.4 M and full gas (660 D) 18.06 over 602.9 M, both under it. +const FINISH_GROWTH_PER_CYCLE: f64 = 30.08 * (1u64 << 30) as f64 / 612.3e6; + +/// What a block hands back at the walk's end +/// ([`multilinear_block::BlockSpill::hand_off`]), decided once there: the +/// forecast of phase A's peak, `hwm` (the resident high-water so far) plus +/// [`FINISH_GROWTH_PER_CYCLE`] for each of the `cycles` walked, past +/// [`PRESSURE_SHARE`] of `target`; 0 when the forecast fits under it. The +/// finish is still ahead, so the parked tables are dropped before its hump +/// builds, and the hump reuses their pages. +fn hand_off_need(hwm: u64, cycles: u64, target: u64) -> u64 { + let forecast = hwm.saturating_add((FINISH_GROWTH_PER_CYCLE * cycles as f64) as u64); + forecast.saturating_sub(pressure_line(target)) +} + +/// `LAMBDA_VM_BLOCK_HAND_OFF`: `auto` (and unset) decides the hand-off at the +/// walk's end ([`hand_off_need`]); `off` never hands off (the parked tables +/// stay on the host until the rule arms). Anything else is `auto`. +fn hand_off_from_env() -> bool { + parse_hand_off(std::env::var("LAMBDA_VM_BLOCK_HAND_OFF").ok().as_deref()) +} + +fn parse_hand_off(value: Option<&str>) -> bool { + value.map(str::trim) != Some("off") +} + +/// The `auto` rule for one committed table of `bytes` packed bytes, whether +/// it is to leave the host (which arms live regeneration): the host's bytes +/// (`host`, [`HostReading::bytes`]), the reserve for the `cells` main cells +/// committed so far (this table's included) at `per_g` +/// ([`memory_reserve_bytes`]) and the table past `target` (#1013's +/// `spill_decision`, with #1014's reserve). +fn drop_wanted( + target: u64, + per_g: f64, + cells: u64, + bytes: u64, + host: impl FnOnce() -> u64, +) -> bool { + host() + .saturating_add(memory_reserve_bytes(cells, per_g)) + .saturating_add(bytes) + > target +} + +impl BlockOptions { + /// Whether the finish streams the rest under its byte gate + /// ([`Self::finish_stream`]): it needs the layout thread, the finish's + /// cuts and KECCAK_RND built as its tables. + pub(crate) fn rest_streams(&self) -> bool { + self.finish_stream + && self.layout_workers > 0 + && self.finish_cuts + && self.finish_keccak_rnd_chunks + && self.keccak_rnd_rows_log2 >= 5 + } + + /// Every chunked table at 2^21 rows, KECCAK_RND at 2^16, ECDAS at 2^17, + /// KECCAK at 2^18, ECSM at 2^17, windows of 2^20 cycles. + pub fn production() -> Self { + Self { + max_rows: MaxRowsConfig::uniform(1 << 21), + keccak_rnd_rows_log2: BLOCK_KECCAK_RND_ROWS_LOG2, + ecdas_rows_log2: BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: BLOCK_COPY_ROWS_LOG2, + drop_levels: tree_drop_from_env(), + window_log2: Some(BLOCK_WINDOW_LOG2), + drop_streamed_ops: true, + layout_workers: BLOCK_LAYOUT_WORKERS, + layout_ahead: Some(BLOCK_LAYOUT_AHEAD), + narrow: multilinear_block::Narrowing::CARD, + upload_ahead: true, + memlog: memlog::from_env(), + finish_keccak_rnd_chunks: true, + finish_cuts: finish_plan_from_env(), + finish_stream: finish_stream_from_env(), + rest_layout_bytes: Some(BLOCK_REST_LAYOUT_BYTES), + rest_build_bytes: Some(BLOCK_REST_BUILD_BYTES), + pack_finished: true, + gpack: gpack_from_env(), + regen: regen::production_regen(), + phase_b_overlap: phase_b_overlap_from_env(), + } + } +} + +/// Phase B's overlap when `LAMBDA_VM_PHASE_B_OVERLAP` is unset: on a card +/// build the openings beside the next group's argue, reading back through +/// pinned staging, with the argue's streams ahead of theirs (the card's +/// stream classes default to `priority`), which is the configuration that was +/// timed; a host-only build keeps the openings in line, untimed beside an +/// argue on the same cores. +pub(crate) const PHASE_B_OVERLAP_DEFAULT: multilinear_block::PhaseBOverlap = + if cfg!(feature = "cuda") { + multilinear_block::PhaseBOverlap::Pinned + } else { + multilinear_block::PhaseBOverlap::Off + }; + +/// `LAMBDA_VM_PHASE_B_OVERLAP`: [`BlockOptions::phase_b_overlap`]'s +/// production value: unset [`PHASE_B_OVERLAP_DEFAULT`], `0` off, `1` on, +/// `pinned` on with pinned read-backs. +pub(crate) fn phase_b_overlap_from_env() -> multilinear_block::PhaseBOverlap { + phase_b_overlap_from(std::env::var("LAMBDA_VM_PHASE_B_OVERLAP").ok().as_deref()) +} + +/// [`phase_b_overlap_from_env`]'s reading of its variable: unset is +/// [`PHASE_B_OVERLAP_DEFAULT`], `0` off, `1` on, `pinned` on with the +/// openings' read-backs through pinned staging; another word is read as off, +/// and said once. +pub(crate) fn phase_b_overlap_from(value: Option<&str>) -> multilinear_block::PhaseBOverlap { + use multilinear_block::PhaseBOverlap; + match value.map(str::trim) { + None => PHASE_B_OVERLAP_DEFAULT, + Some("0") => PhaseBOverlap::Off, + Some("1") => PhaseBOverlap::On, + Some("pinned") => PhaseBOverlap::Pinned, + Some(other) => { + static SAID: std::sync::Once = std::sync::Once::new(); + SAID.call_once(|| { + eprintln!("LAMBDA_VM_PHASE_B_OVERLAP={other}: 0, 1 or pinned — read as off"); + }); + PhaseBOverlap::Off + } + } +} + +/// Phase B's overlap as [`BlockOptions::phase_b_overlap`] asks for it. +fn phase_b_overlap(options: &BlockOptions) -> multilinear_block::PhaseBOverlap { + options.phase_b_overlap +} + +/// `LAMBDA_VM_BLOCK_SCHEDSTAT=1`: phase B reads its driver thread's CPU time +/// and run-queue delay at each group turn and the readout prints them by +/// tenth of phase B ([`BlockStamps::phase_b_sched`]). A report, off by default. +pub(crate) fn phase_b_sched_from_env() -> bool { + phase_b_sched_from(std::env::var("LAMBDA_VM_BLOCK_SCHEDSTAT").ok().as_deref()) +} + +/// [`phase_b_sched_from_env`]'s reading: `1` on, anything else off. +pub(crate) fn phase_b_sched_from(value: Option<&str>) -> bool { + value.map(str::trim) == Some("1") +} + +/// `LAMBDA_VM_BLOCK_GPACK`: [`BlockOptions::gpack`]'s production value, on +/// unless `0`. +pub(crate) fn gpack_from_env() -> bool { + !std::env::var("LAMBDA_VM_BLOCK_GPACK").is_ok_and(|v| v.trim() == "0") +} + +/// `LAMBDA_VM_BLOCK_FINISH_PLAN`: [`BlockOptions::finish_cuts`]'s production +/// value, on unless `0`. +pub(crate) fn finish_plan_from_env() -> bool { + !std::env::var("LAMBDA_VM_BLOCK_FINISH_PLAN").is_ok_and(|v| v.trim() == "0") +} + +/// `LAMBDA_VM_BLOCK_FINISH_STREAM`: [`BlockOptions::finish_stream`]'s +/// production value, on unless `0`. +pub(crate) fn finish_stream_from_env() -> bool { + !std::env::var("LAMBDA_VM_BLOCK_FINISH_STREAM").is_ok_and(|v| v.trim() == "0") +} + +/// `LAMBDA_VM_BLOCK_COMPACT_LT=0`: the LT ops derived from the MEMW ops the +/// builder drops are held at 24 bytes each until the finish +/// ([`WindowedTraceBuilder::raw_memw_lt`]), the A arm of keeping them compact; +/// unset or anything else keeps them compact. The tables are the same (#1013's +/// knob). +fn compact_lt() -> bool { + !std::env::var("LAMBDA_VM_BLOCK_COMPACT_LT").is_ok_and(|v| v.trim() == "0") +} + +/// `LAMBDA_VM_BLOCK_P4_ROWS`: unset, empty or `pool` (the default) has the +/// finish count the deduplicating tables' per-row lookups (LT, BRANCH, +/// BYTEWISE: one `ARE_BYTES` a row of a μ-split operation, and the padding +/// rows') on the whole pool, a chunk a task, counts only, each chunk's map +/// reserved for its distinct operations; `parts` counts each chunk by parts of +/// its ops instead, each part's map small enough to stay in a core's cache; +/// `off` builds each chunk's list of lookups among phase 4's other +/// collectors, as before (the rollback). Anything else refuses the prove. The +/// multiplicities and the tables are the same. +fn block_p4_rows() -> Result { + parse_p4_rows(std::env::var("LAMBDA_VM_BLOCK_P4_ROWS").ok().as_deref()) +} + +/// `LAMBDA_VM_BLOCK_CARD_EARLY`: unset, `1` or `on` (the default) sets the +/// card up (its context, its modules and its staging pools) on a thread of its +/// own as the streamed prove starts, so the first group's upload does not wait +/// for it; empty, `0` or `off` sets it up at the card's first use. Anything +/// else refuses the prove. A build without the card's kernels has nothing to +/// set up. The proof is the same. +fn block_card_early() -> Result { + parse_card_early(std::env::var("LAMBDA_VM_BLOCK_CARD_EARLY").ok().as_deref()) +} + +fn parse_card_early(value: Option<&str>) -> Result { + match value.map(str::trim) { + Some("" | "0" | "off") => Ok(false), + None | Some("1" | "on") => Ok(true), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_CARD_EARLY={v}: 1 or on (the card set up as the prove starts, the \ + default), 0 or off (at its first use)" + ))), + } +} + +/// The card's setup on a thread of its own ([`block_card_early`]); nothing +/// without the card's kernels. +fn set_up_card() { + if cfg!(feature = "cuda") { + std::thread::spawn(|| { + multilinear::gpu::set_up_device(); + head_log::card_ready(); + }); + } +} + +/// `LAMBDA_VM_BLOCK_P4_LOOKUPS`: unset, empty or `direct` (the default) has +/// the finish's phase 4 count LT's per-op lookups and KECCAK's straight into +/// its histograms; `lists` builds each slice's list of them first, as before +/// (the rollback). Anything else refuses the prove. The multiplicities and the +/// tables are the same. +fn block_p4_direct() -> Result { + parse_p4_lookups(std::env::var("LAMBDA_VM_BLOCK_P4_LOOKUPS").ok().as_deref()) +} + +fn parse_p4_lookups(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "direct") => Ok(true), + Some("lists") => Ok(false), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_P4_LOOKUPS={v}: direct (counted straight into the histograms) or lists" + ))), + } +} + +/// How the finish counts the per-row lookups ([`block_p4_rows`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum P4Rows { + /// Each chunk's list among phase 4's collectors. + Off, + /// On the pool, one map a chunk. + Pool, + /// On the pool, each chunk by parts. + Parts, +} + +fn parse_p4_rows(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "pool") => Ok(P4Rows::Pool), + Some("parts") => Ok(P4Rows::Parts), + Some("off") => Ok(P4Rows::Off), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_P4_ROWS={v}: pool (the per-row lookups counted on the pool), parts (counted \ + by parts of each chunk) or off" + ))), + } +} + +/// `LAMBDA_VM_BLOCK_LT_CONCAT=1`: the finish concatenates LT's ops into one +/// list before chunking them ([`WindowedTraceBuilder::concat_lt`]), the A arm +/// of keeping them as segments; off by default. With +/// `LAMBDA_VM_BLOCK_COMPACT_LT=0` it is the finish as it was before both. The +/// tables are the same (#1013's knob). +fn lt_concat() -> bool { + std::env::var("LAMBDA_VM_BLOCK_LT_CONCAT").is_ok_and(|v| v.trim() == "1") +} + +/// `LAMBDA_VM_BLOCK_P4_BESIDE=1`: the streamed finish counts the BITWISE +/// lookups in BITWISE's own table's job, so the slide starts the tables after +/// it while it counts ([`WindowedTraceBuilder::p4_beside`]); off by default. +/// The tables are the same. +fn p4_beside() -> bool { + std::env::var("LAMBDA_VM_BLOCK_P4_BESIDE").is_ok_and(|v| v.trim() == "1") +} + +/// `LAMBDA_VM_BLOCK_REST_BITWISE_LAST`: `1` has the streamed finish count +/// BITWISE's multiplicities beside the rest's builds and, when it slides, hand +/// the rest on in AIR order but BITWISE, which goes after every other table +/// ([`WindowedTraceBuilder::rest_bitwise_last`]), so the tables that do not +/// read the count reach the packer while it counts. Unset, empty or `0` (the +/// default): AIR order, BITWISE first. Anything else refuses the prove. The +/// tables are the same; the groups they are packed into, and so the proof's +/// bytes, are not; the verifier takes any partition the statement declares. +fn block_rest_bitwise_last() -> Result { + parse_rest_bitwise_last( + std::env::var("LAMBDA_VM_BLOCK_REST_BITWISE_LAST") + .ok() + .as_deref(), + ) +} + +/// `LAMBDA_VM_BLOCK_BITWISE_FROM_TABLES`: unset, empty or `1` (the default) +/// counts BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables as each +/// is built, from the table's own rows and their multiplicities, and hands +/// BITWISE on after every other table, where the finish slides and keeps only +/// the streamed chunks' tails ([`WindowedTraceBuilder::bitwise_from_tables`]); +/// `0` (the rollback) has phase 4 count them from their ops and BITWISE lead +/// the rest, as before. Anything else refuses the prove. The tables are the +/// same either way; their order, and so the groups and the proof's bytes, are +/// not. +fn block_bitwise_from_tables() -> Result { + parse_bitwise_from_tables( + std::env::var("LAMBDA_VM_BLOCK_BITWISE_FROM_TABLES") + .ok() + .as_deref(), + ) +} + +fn parse_bitwise_from_tables(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "1") => Ok(true), + Some("0") => Ok(false), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_BITWISE_FROM_TABLES={v}: 1 (BITWISE's lookups from the tables' rows, \ + BITWISE last) or 0" + ))), + } +} + +fn parse_rest_bitwise_last(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "0") => Ok(false), + Some("1") => Ok(true), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_REST_BITWISE_LAST={v}: 1 (BITWISE handed on last, beside the count) or 0" + ))), + } +} + +/// The streamed finish builds each of the rest's tables as soon as the byte +/// gate admits it, while earlier ones still build, instead of in waves that +/// each last as long as their slowest table +/// ([`WindowedTraceBuilder::slide_rest`]): the default. +/// `LAMBDA_VM_BLOCK_REST_SLIDE=0` builds in waves. The tables and their order +/// are the same. +pub(crate) fn rest_slide() -> bool { + parse_rest_slide(std::env::var("LAMBDA_VM_BLOCK_REST_SLIDE").ok().as_deref()) +} + +/// How the rest's layout bytes ([`BlockOptions::rest_layout_bytes`]) bound +/// the rest, for the harnesses' `BLOCK REST LAYOUT CONFIG` line: waves of +/// them where the rest is laid out in waves, and the streamed finish's gate +/// where it slides ([`rest_slide`]), which has no waves. +pub(crate) fn rest_layout_label(options: &BlockOptions, slide: bool) -> String { + let mib = |b: usize| format!("{} MiB", b >> 20); + match (options.rest_streams() && slide, options.rest_layout_bytes) { + (true, Some(b)) => format!("a gate of {}, sliding (no waves)", mib(b)), + (true, None) => "no gate, sliding (no waves)".to_string(), + (false, Some(b)) => format!("waves of {}", mib(b)), + (false, None) => "waves of all at once".to_string(), + } +} + +/// [`rest_slide`] of the knob's value: anything but `0` slides. +fn parse_rest_slide(value: Option<&str>) -> bool { + value.map(str::trim) != Some("0") +} + +/// The families whose tables the card fills from the records their rows come +/// from ([`block_card_fill`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct CardFill { + /// Each KECCAK_RND table the finish cuts, from its keccak operations + /// ([`WindowedTraceBuilder::card_fill_keccak_rnd`]). + pub(crate) keccak_rnd: bool, + /// Each full streamed CPU chunk, from the executor's logs of its cycles, + /// and each full MEMW_R chunk, from its register rows, while the host has + /// room ([`WindowedTraceBuilder::card_fill_streamed`]). + pub(crate) cpu: bool, + pub(crate) memw_r: bool, + /// Each full streamed chunk of these families, from its rows (MEMW_A's + /// aligned rows, LOAD's, STORE's, MEMW's and SHIFT's ops), under the same + /// room rule ([`WindowedTraceBuilder::card_fill_streamed_rows`]). + pub(crate) rows: RowFamilies, +} + +impl CardFill { + /// The families the card fills. + const NAMES: [&'static str; 8] = [ + "keccak_rnd", + "cpu", + "memw_r", + "memw_a", + "load", + "store", + "memw", + "shift", + ]; + + /// Every family. + const EVERY: Self = Self { + keccak_rnd: true, + cpu: true, + memw_r: true, + rows: RowFamilies::ALL, + }; + + /// Unset or empty: every family (KECCAK_RND, CPU, MEMW_R and the five row + /// families) in a build with the card's kernels (the `cuda` feature), + /// nothing in one without, where every fill from records would be the + /// host's. + const DEFAULT: Self = Self { + keccak_rnd: cfg!(feature = "cuda"), + cpu: cfg!(feature = "cuda"), + memw_r: cfg!(feature = "cuda"), + rows: if cfg!(feature = "cuda") { + RowFamilies::ALL + } else { + RowFamilies::NONE + }, + }; + + /// Whether any streamed family is asked of the card. + fn streamed(self) -> bool { + self.cpu || self.memw_r || self.rows.any() + } + + /// Each family's flag, in [`Self::NAMES`] order. + fn flags(&mut self) -> [&mut bool; 8] { + let rows = &mut self.rows; + [ + &mut self.keccak_rnd, + &mut self.cpu, + &mut self.memw_r, + &mut rows.memw_a, + &mut rows.load, + &mut rows.store, + &mut rows.memw, + &mut rows.shift, + ] + } + + fn name(mut self) -> String { + let on = self.flags().map(|on| *on); + let names: Vec<&str> = Self::NAMES + .iter() + .zip(on) + .filter_map(|(name, on)| on.then_some(*name)) + .collect(); + if names.is_empty() { + "off".to_string() + } else { + names.join(",") + } + } +} + +/// `LAMBDA_VM_BLOCK_CARD_FILL`: a comma list of `keccak_rnd`, `cpu`, `memw_r`, +/// `memw_a`, `load`, `store`, `memw` and `shift` (each at most once) has the +/// card fill those families' tables from +/// the records their rows come from as their groups upload them ([`CardFill`]); +/// `on` is every family, `off` none (the host builds every table). Unset or +/// empty is [`CardFill::DEFAULT`]: every family in a build with the card's +/// kernels, nothing without; `keccak_rnd,cpu,memw_r` is the earlier default +/// (no row family), the rollback, and `keccak_rnd` alone KECCAK_RND's fill +/// only. Where the card declines the room, a host reader fills them on the +/// host. The tables' words, and so the proof's bytes, are the same. Anything +/// else refuses the prove. +pub(crate) fn block_card_fill() -> Result { + parse_card_fill(std::env::var("LAMBDA_VM_BLOCK_CARD_FILL").ok().as_deref()) +} + +/// [`block_card_fill`]'s reading of its variable. +fn parse_card_fill(value: Option<&str>) -> Result { + let refused = |v: &str| { + Error::Prover(format!( + "LAMBDA_VM_BLOCK_CARD_FILL={v}: off (the host builds every table), on (the card \ + fills every family it can) or a comma list of {}", + CardFill::NAMES.join(", ") + )) + }; + match value.map(str::trim) { + None | Some("") => Ok(CardFill::DEFAULT), + Some("off") => Ok(CardFill::default()), + Some("on") => Ok(CardFill::EVERY), + Some(v) => { + let mut fill = CardFill::default(); + for name in v.split(',').map(str::trim) { + let Some(at) = CardFill::NAMES.iter().position(|n| *n == name) else { + return Err(refused(v)); + }; + if std::mem::replace(fill.flags()[at], true) { + return Err(refused(v)); + } + } + Ok(fill) + } + } +} + +/// `LAMBDA_VM_BLOCK_RECORDS_DROP`: `park` (and unset or empty) lets live +/// regeneration park and drop a streamed chunk held as its records like any +/// chunk, phase B rebuilding its packed columns checked by its records' +/// digest; `keep` holds such a chunk on the host to the prove's end, with no +/// rank. Anything else refuses the prove. The proof's bytes are the same. +pub(crate) fn records_keep() -> Result { + parse_records_drop( + std::env::var("LAMBDA_VM_BLOCK_RECORDS_DROP") + .ok() + .as_deref(), + ) +} + +/// [`records_keep`]'s reading of its variable: whether records are kept. +fn parse_records_drop(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "park") => Ok(false), + Some("keep") => Ok(true), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_RECORDS_DROP={v}: park (records drop like any chunk) or keep" + ))), + } +} + +/// Whether the streamed CPU and MEMW_R chunks may be held as their records, +/// and why not when they may not. They need the card asked for at least one of +/// the two families (`asked`), and G-pack's narrow chunks of a build that drops +/// its streamed ops (`narrow_dropping`). And they need a way off the host: +/// parked records drop only under live regeneration, so without it (a store, +/// or regeneration off) every one admitted would stay resident to the prove's +/// end, which no room rule bounds on a block that arms. Records kept on purpose +/// (`LAMBDA_VM_BLOCK_RECORDS_DROP=keep`, `keep_records`) are undroppable by +/// design: the A/B control. +fn records_admission( + asked: bool, + narrow_dropping: bool, + keep_records: bool, + live_regen: bool, +) -> Result<(), &'static str> { + if !asked { + Err("off") + } else if !narrow_dropping { + Err("off (not a narrow build that drops its streamed ops)") + } else if !keep_records && !live_regen { + Err("off (parked records drop only under live regeneration, and it is off)") + } else { + Ok(()) + } +} + +/// The most window walkers `LAMBDA_VM_BLOCK_WALKERS` takes: each walker holds +/// a window's state and walked lists, and this is the most whose memory was +/// measured on the largest blocks. +const MAX_WALKERS: usize = 4; + +/// `LAMBDA_VM_BLOCK_WALKERS`: unset or empty walks the run's windows on +/// [`MAX_WALKERS`] threads (the default), each window from the bytes it +/// touches as they stood before it ([`parallel::walk_windows`]); `1` to +/// [`MAX_WALKERS`] on that many; `0` on one thread, the walk before the +/// walkers (the rollback). The tables and their order are the same. Anything +/// else refuses the prove, and so does a count with the executor's words +/// store (`LAMBDA_VM_EXEC_MEMORY=words`), whose every window's snapshot copies +/// the whole map; with that store, unset walks on one thread. +fn block_walkers() -> Result { + parse_walkers( + std::env::var("LAMBDA_VM_BLOCK_WALKERS").ok().as_deref(), + executor::vm::memory::StoreKind::from_env(), + ) +} + +/// [`block_walkers`] of the knob's value and the executor's store. +fn parse_walkers( + value: Option<&str>, + store: executor::vm::memory::StoreKind, +) -> Result { + let walkers = match value.map(str::trim) { + None | Some("") if store == executor::vm::memory::StoreKind::Words => return Ok(0), + None | Some("") => return Ok(MAX_WALKERS), + Some("0") => return Ok(0), + Some(v) => v + .parse::() + .ok() + .filter(|n| (1..=MAX_WALKERS).contains(n)) + .ok_or_else(|| { + Error::Prover(format!( + "LAMBDA_VM_BLOCK_WALKERS={v}: 0 (one walker), or 1 to {MAX_WALKERS} walkers" + )) + })?, + }; + if store == executor::vm::memory::StoreKind::Words { + return Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_WALKERS={walkers} needs the executor's pages store, not \ + LAMBDA_VM_EXEC_MEMORY=words" + ))); + } + Ok(walkers) +} + +/// The rest's layout bytes ([`BlockOptions::rest_layout_bytes`]). On the +/// streamed finish that slides (production) they are its byte gate's budget: +/// what the rest holds built and not yet placed, or building. At 2 GiB the +/// large block's gate filled and its builds waited for room (≈ 1.3 s); with +/// at most [`BLOCK_REST_BUILD_BYTES`] building at once, the first tables the +/// packer needs build as they did at 2 GiB. Where the rest is laid out in +/// waves (the finish not streamed, or not sliding) they bound the rows each +/// wave transposes: each transposition is parallel within its table, so a +/// wave of a few tables keeps the cores busy while bounding the copies in +/// flight. +pub const BLOCK_REST_LAYOUT_BYTES: usize = 4 << 30; + +/// The estimated bytes of the rest's tables the sliding finish builds at once +/// by default ([`BlockOptions::rest_build_bytes`]), apart from the gate's +/// [`BLOCK_REST_LAYOUT_BYTES`]: more builds at once on the shared pool slow +/// the first tables the packer needs. +pub const BLOCK_REST_BUILD_BYTES: usize = 2 << 30; + +/// `BLOCK_WHIR_REST_BUILD=`: the real-block tests' choice of +/// [`BlockOptions::rest_build_bytes`]; unset leaves production's +/// ([`BLOCK_REST_BUILD_BYTES`]). Anything but a positive MiB count is an +/// error; the gate's own budget is no cap beyond the gate. +fn parse_rest_build(value: Option<&str>) -> Result, String> { + match value.map(str::trim) { + None => Ok(None), + Some(mib) => match mib.parse::() { + Ok(mib) if mib > 0 => Ok(Some(mib << 20)), + _ => Err(format!("BLOCK_WHIR_REST_BUILD={mib}: a positive MiB count")), + }, + } +} + +/// `BLOCK_WHIR_REST_LAYOUT=all|`, `BLOCK_WHIR_REST_BUILD=`, +/// `BLOCK_WHIR_KR_FINISH_CHUNKS=0|1` and `BLOCK_WHIR_PACK_FINISHED=0|1`: the +/// real-block tests' choice of [`BlockOptions::rest_layout_bytes`], +/// [`BlockOptions::rest_build_bytes`], [`BlockOptions::finish_keccak_rnd_chunks`] +/// and [`BlockOptions::pack_finished`]; unset leaves the production ones. +pub(crate) fn rest_layout_from_env(options: &mut BlockOptions) { + match parse_rest_build(std::env::var("BLOCK_WHIR_REST_BUILD").ok().as_deref()) { + Ok(Some(bytes)) => options.rest_build_bytes = Some(bytes), + Ok(None) => {} + Err(e) => panic!("{e}"), + } + match std::env::var("BLOCK_WHIR_REST_LAYOUT") + .as_deref() + .map(str::trim) + { + Ok("all") => options.rest_layout_bytes = None, + Ok(mib) => match mib.parse::() { + Ok(mib) if mib > 0 => options.rest_layout_bytes = Some(mib << 20), + _ => panic!("BLOCK_WHIR_REST_LAYOUT={mib}: all or a positive MiB count"), + }, + Err(_) => {} + } + match std::env::var("BLOCK_WHIR_KR_FINISH_CHUNKS") + .as_deref() + .map(str::trim) + { + Ok("0") => options.finish_keccak_rnd_chunks = false, + Ok("1") => options.finish_keccak_rnd_chunks = true, + Ok(other) => panic!("BLOCK_WHIR_KR_FINISH_CHUNKS={other}: 0 or 1"), + Err(_) => {} + } + match std::env::var("BLOCK_WHIR_PACK_FINISHED") + .as_deref() + .map(str::trim) + { + Ok("0") => options.pack_finished = false, + Ok("1") => options.pack_finished = true, + Ok(other) => panic!("BLOCK_WHIR_PACK_FINISHED={other}: 0 or 1"), + Err(_) => {} + } +} + +/// `BLOCK_WHIR_NARROW=wide|card|host`: the real-block tests' choice of +/// [`BlockOptions::narrow`]; `None` leaves the production one. +pub(crate) fn narrow_from_env() -> Option { + use multilinear_block::Narrowing; + match std::env::var("BLOCK_WHIR_NARROW").as_deref().map(str::trim) { + Ok("wide") => Some(Narrowing::Wide), + Ok("card") => Some(Narrowing::CARD), + Ok("host") => Some(Narrowing::Host), + Ok(other) => panic!("BLOCK_WHIR_NARROW={other}: wide, card or host"), + Err(_) => None, + } +} + +/// `BLOCK_WHIR_WINDOW_LOG2=20`: [`BlockOptions::window_log2`] as the tree +/// harness and the CLI read it, production's [`BLOCK_WINDOW_LOG2`] and nothing +/// else. Another window (#1013's one CPU chunk is 2^21 cycles) builds the same +/// tables, but the chunks leave in another order across tables, so the groups +/// (statement data) and the proof bytes differ: the knob refuses it. `None` +/// leaves the production one. +pub(crate) fn window_log2_from_env() -> Option { + parse_window_log2(std::env::var("BLOCK_WHIR_WINDOW_LOG2").ok().as_deref()) + .unwrap_or_else(|e| panic!("{e}")) +} + +/// [`window_log2_from_env`]'s reading of the knob's `value`. +fn parse_window_log2(value: Option<&str>) -> Result, String> { + match value.map(str::trim) { + None => Ok(None), + Some("20") => Ok(Some(BLOCK_WINDOW_LOG2)), + Some(other) => Err(format!( + "BLOCK_WHIR_WINDOW_LOG2={other}: only 20 (another window moves the groups)" + )), + } +} + +/// `BLOCK_WHIR_UPLOAD_AHEAD=0|1`: the real-block tests' choice of +/// [`BlockOptions::upload_ahead`]; `None` leaves the production one. +pub(crate) fn upload_ahead_from_env() -> Option { + match std::env::var("BLOCK_WHIR_UPLOAD_AHEAD") + .as_deref() + .map(str::trim) + { + Ok("0") => Some(false), + Ok("1") => Some(true), + Ok(other) => panic!("BLOCK_WHIR_UPLOAD_AHEAD={other}: 0 or 1"), + Err(_) => None, + } +} + +/// `BLOCK_WHIR_KECCAK_LOG2=k` (0..=[`BLOCK_KECCAK_MAX_VARS`]), +/// `BLOCK_WHIR_ECSM_LOG2=k` (0..=[`BLOCK_ECSM_MAX_VARS`]) and +/// `BLOCK_WHIR_COPY_LOG2=k` (0..=[`BLOCK_COPY_MAX_VARS`]): the real-block +/// tests' KECCAK, ECSM and COPY cuts, to force a split on a block whose tables +/// fit one; unset leaves the production ones. +pub(crate) fn chunk_cuts_from_env(options: &mut BlockOptions) { + for (name, cap, rows_log2) in [ + ( + "BLOCK_WHIR_KECCAK_LOG2", + BLOCK_KECCAK_MAX_VARS, + &mut options.keccak_rows_log2, + ), + ( + "BLOCK_WHIR_ECSM_LOG2", + BLOCK_ECSM_MAX_VARS, + &mut options.ecsm_rows_log2, + ), + ( + "BLOCK_WHIR_COPY_LOG2", + BLOCK_COPY_MAX_VARS, + &mut options.copy_rows_log2, + ), + ] { + if let Ok(v) = std::env::var(name) { + match v.trim().parse() { + Ok(k) if k <= cap => *rows_log2 = k, + _ => panic!("{name}={v}: 0..={cap}"), + } + } + } +} + +/// The chunked tables' heights a statement states, for the real-block +/// readouts: `KECCAK 1 × [14] · KECCAK_RND 4 × [16, 16, 16, 16] · …`. +pub(crate) fn chunked_census(counts: &TableCounts, table_num_vars: &[u8]) -> String { + chunked_table_ranges(counts) + .into_iter() + .map(|(name, range, _)| { + let heights = &table_num_vars[range]; + format!("{name} {} × {heights:?}", heights.len()) + }) + .collect::>() + .join(" · ") +} + +/// Cycles a streamed prove collects at a time: half a CPU instance at 2^21, so +/// every chunk is handed to phase A within a window of completing. +pub const BLOCK_WINDOW_LOG2: usize = 20; + +/// Threads a streamed prove lays its chunks out on +/// ([`BlockOptions::layout_workers`]). Three workers with two chunks ahead +/// bound the median block's windows: the layout thread stayed blocked handing +/// chunks on, and the accumulator waited on it. Six with five ahead leave the +/// walker to set the pace (ZEN5 020, P1 median: phase A −2.32 s against +/// three, peak RSS no higher). The groups and the proof bytes are the same at +/// any count. `BLOCK_WHIR_LAYOUT_WORKERS` overrides it in the tree harness +/// and the CLI. +pub const BLOCK_LAYOUT_WORKERS: usize = 6; + +/// Streamed chunks laid out (or in the making) and not yet packed, beyond the +/// one the inline layout holds ([`BlockOptions::layout_ahead`]): one per +/// worker of [`BLOCK_LAYOUT_WORKERS`] and three more, so the workers go on +/// while the in-order packer waits for a slow chunk (one per worker held the +/// walk back once the windows were walked on several threads). What the extra +/// chunks would add to phase B's holding, the keep budget takes back +/// ([`BLOCK_KEEP_BELOW`]). `BLOCK_WHIR_LAYOUT_AHEAD` overrides it. +pub const BLOCK_LAYOUT_AHEAD: usize = 8; + +/// A block proved in one proof. +/// +/// The statement is [`crate::multilinear_prove::MultilinearVmProof`]'s; the +/// proof is read under the block's forked schedule, and only by +/// [`verify_block_whir`]. +#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct BlockWhirProof { + pub proof: MultiProof, + /// Each table's height in variables, in [`VmAirs::air_refs`] order. + pub table_num_vars: Vec, + pub runtime_page_ranges: Vec, + pub table_counts: TableCounts, + pub public_output: Vec, + pub num_private_input_pages: usize, + /// The groups, each its tables' indices in [`VmAirs::air_refs`] order, in + /// the order the group stacks them. The prover's choice — a streamed prove + /// packs tables as they arrive — bound in the statement before any root + /// ([`absorb_block`]); the verifier checks it is a partition and rebuilds + /// each group's stack from it under its own stack cap. The proof's tables + /// are in this order. + pub groups: Vec>, + /// The prepared openings ([`group_stacks`]), one per group that holds a + /// prepared table, in group order. + pub prepared: Vec>, + /// Under [`ArgueFormat::Batched`], each group's batched argue, in group + /// order, and `proof.tables` is empty; under [`ArgueFormat::PerTable`], + /// empty. Which one a verifier reads is its format's, never the proof's. + pub argues: Vec>, + /// Every carried stacked polynomial's commit-time out-of-domain answer, in + /// root order: one per root of `proof.roots`, absorbed after the roots and + /// before the block's first challenge, and each carried into its chain + /// (`stark::multilinear_block::commit_ood_point`). The verifier refuses + /// any other count. + pub commit_ood: Vec>, +} + +/// Where a block's prove spent its time, for the readout. Seconds. +#[derive(Clone, Debug, Default)] +pub struct BlockStamps { + pub execute: f64, + pub build: f64, + /// KECCAK_RND split, AIRs, transposition and layouts. + pub prep: f64, + pub phase_a: f64, + pub phase_b: f64, + pub tables: usize, + pub cells: usize, + pub groups: Vec, + /// The trace build's phase ends and each phase-5 generator's finish, + /// seconds into the build. + pub build_marks: Vec<(String, f64)>, + /// Phase B's first-round trees: `open_many` calls served from the kept + /// tops and the leaves re-hashed for them. A revived commitment never + /// builds a device tree, so nothing re-commits. + pub top_paths: (u64, u64), + /// Of those, for codewords on the card: `(calls, seconds)` re-hashed on + /// the card, and on the host (the fallback). + pub top_routes: ((u64, f64), (u64, f64)), + /// The levels the retired trees dropped (on the card · on the host). + pub drop_levels: Option, + /// Phase B's openings, by host stage (`LAMBDA_VM_BASE_SPLIT=1`, else + /// empty): the WHIR chains' six stages, their query split, and the kept-top + /// paths' gather and re-hash, seconds on the prover's thread. + pub open_split: Vec<(&'static str, f64)>, + /// A streamed build: when its windows were all collected (seconds since + /// the build started) and how many chunks they handed out. + pub streamed: (f64, usize), + /// A streamed build's windows: the walk, the routing and the streamed + /// chunks' generation, summed. + pub windows: WindowStamps, + /// The prepared commitments: how many, and the seconds deriving them. + pub prepared: (usize, f64), + /// A streamed build's layout: its threads, its marks (seconds since the + /// prove started), when each group closed, the seconds the packer waited + /// on phase A and the seconds laying out chunks, summed over threads. + pub layout: LayoutStamps, + /// A streamed build's producer: what each of its threads waited on. + pub producer: ProducerStamps, + /// A streamed build's executor thread: its own work apart from its waits. + pub executor: ExecutorStamps, + /// How phase A held the committed columns, and the narrow tables phase B + /// widened on the host, `(tables, cells)`: a device path widens on the + /// card. + pub narrow: multilinear_block::Narrowing, + pub host_widens: (u64, u64), + /// Tables filled from their records ([`block_card_fill`]) during the + /// prove: `(on the card, on the host)`. + pub records_fills: (u64, u64), + /// Under `LAMBDA_VM_CARD_UPLOAD_TIMES=1`, the group uploads' tables timed + /// on the card during the prove (`multilinear::records::upload_times`): + /// `((packed tables, copy and widen s), (records tables, copy and fill s))`. + pub upload_times: ((u64, f64), (u64, f64)), + /// With [`BlockOptions::memlog`]: every memory term when the proof is + /// done, bytes; empty without. + pub mem_terms: Vec<(&'static str, usize)>, + /// With live regeneration: the rule it decided by, and the largest host + /// reading it decided on (the `BLOCK DROPS` line). + pub drops: Option, + /// With [`BlockOptions::regen`] set to regenerate: what the regenerator + /// did, and its `BLOCK REGEN` lines (printed as phase B ends, not in + /// [`Self::report`]). + pub regen: Option, + /// A streamed build's `BLOCK GROUP QUEUE` line ([`group_queue::line`], + /// printed as phase A ends, not in [`Self::report`]). + pub group_queue: Option, + /// A streamed build's `BLOCK HEAD` lines ([`head_log`], printed as phase A + /// ends, not in [`Self::report`]); empty unless the head was stamped. + pub head: Vec, + /// A streamed prove's phase B in unix seconds, (start, end), and the order + /// it took the groups in: what a sampler's samples and another thread's + /// stamps are placed against. Empty without a streamed prove. + pub phase_b_unix: Option<(f64, f64)>, + pub phase_b_order: Vec, + /// Phase B ran with its overlap on ([`BlockOptions::phase_b_overlap`]). + pub phase_b_overlap: bool, + /// Its openings' thread read back through pinned staging (`Pinned`). + pub phase_b_pinned: bool, + /// The openings' threads' read-backs during phase B, `(through pinned + /// staging, the stream's own way)`, and the bytes the pinned buffers held + /// at its end. + pub background_readbacks: (u64, u64, u64), + /// Phase B stamped its driver thread's schedstat a group turn + /// ([`phase_b_sched_from_env`]; each group's [`GroupStamps::sched_b`]). + pub phase_b_sched: bool, +} + +/// A streamed build's layout, for the readout ([`BlockStamps::layout`]). +#[derive(Clone, Debug, Default)] +pub struct LayoutStamps { + pub workers: usize, + pub marks: Vec<(&'static str, f64)>, + pub closed_at: Vec, + pub blocked: f64, + pub chunks: f64, + /// The rest's five slowest tables to lay out and its first in AIR order. + pub slowest: Vec<(String, f64)>, + /// The most streamed chunks laid out (or in the making) and not yet + /// packed at once, under [`BlockOptions::layout_ahead`]. + pub ahead_most: usize, + /// The rest's tables laid out narrow from the finish's packed columns + /// ([`BlockOptions::pack_finished`]): how many, and their packed bytes. + pub packed_rest: (usize, usize), + /// The rest's tables laid out as the records their rows come from, for + /// the card to fill ([`block_card_fill`]): how many, and the records' + /// bytes. + pub records_rest: (usize, usize), + /// Each table of the rest in AIR order: its name, when its layout ended + /// (seconds since the prove started) and its group. + pub rest_tables: Vec<(String, f64, usize)>, + /// G-pack ([`BlockOptions::gpack`]): whether it was on, and its counts + /// (`tables::gpack::counts`) at phase A's end. + pub gpack: (bool, [usize; 4]), +} + +/// What a streamed build's producer threads waited on, seconds summed over +/// the windows (the workers' summed over the workers), for the readout +/// ([`BlockStamps::producer`]): the executor sending a window to the walk, the +/// walker waiting for a window and sending it on, the accumulator waiting for +/// a walked window and sending a chunk to the layout, the layout handing a +/// chunk to its workers, and the workers waiting for a permit and for a chunk. +/// A measurement; it changes nothing. +#[derive(Clone, Copy, Debug, Default)] +pub struct ProducerStamps { + pub executor_send: f64, + pub walker_recv: f64, + pub walker_send: f64, + pub accumulator_recv: f64, + pub accumulator_send: f64, + pub layout_send: f64, + pub worker_permit: f64, + pub worker_recv: f64, +} + +/// The streamed build's executor thread, its own work apart from its waits +/// ([`BlockStamps::executor`]); with `send` ([`ProducerStamps::executor_send`]) +/// they add up to the thread's span. A measurement; it changes nothing. +#[derive(Clone, Copy, Debug, Default)] +pub struct ExecutorStamps { + /// When the thread started, since the streamed prove started. + pub started: f64, + /// The executor's setup: the program and the private input loaded. + pub setup: f64, + /// Executing, the logs written as it goes. + pub run: f64, + /// Handing each window out: its logs moved or copied and the memory + /// snapshot at the next window's start. + pub handoff: f64, + /// The windows handed out. + pub windows: usize, + /// The logs left as they lay (`LAMBDA_VM_BLOCK_LOGS_MOVE`, on unless `0`) + /// rather than copied. + pub moved: bool, +} + +impl ExecutorStamps { + /// The `BLOCK EXECUTOR:` line, `send` the thread's blocked sends. + fn line(&self, send: f64) -> String { + format!( + "BLOCK EXECUTOR: started at {:.2}s · setup {:.2}s · execute {:.2}s · hand-off {:.2}s ({} windows, logs {}) · send-blocked {:.2}s · own {:.2}s\n", + self.started, + self.setup, + self.run, + self.handoff, + self.windows, + if self.moved { "moved" } else { "copied" }, + send, + self.setup + self.run + self.handoff, + ) + } +} + +/// `LAMBDA_VM_BLOCK_LOGS_MOVE`: the executor hands each window's logs out as +/// they lie ([`Executor::resume_owned`]) unless set to `0`, which copies them +/// out of the executor's buffer as before. The same logs either way. +fn block_logs_move() -> bool { + parse_logs_move(std::env::var("LAMBDA_VM_BLOCK_LOGS_MOVE").ok().as_deref()) +} + +/// [`block_logs_move`] of the knob's value. +fn parse_logs_move(value: Option<&str>) -> bool { + value.map(str::trim) != Some("0") +} + +/// [`ProducerStamps`] as the threads add to them, in nanoseconds. +#[derive(Default)] +struct ProducerWaits { + executor_send: std::sync::atomic::AtomicU64, + walker_recv: std::sync::atomic::AtomicU64, + walker_send: std::sync::atomic::AtomicU64, + accumulator_recv: std::sync::atomic::AtomicU64, + accumulator_send: std::sync::atomic::AtomicU64, + layout_send: std::sync::atomic::AtomicU64, + worker_permit: std::sync::atomic::AtomicU64, + worker_recv: std::sync::atomic::AtomicU64, +} + +impl ProducerWaits { + /// Adds the time since `since` to `wait`. + fn add(wait: &std::sync::atomic::AtomicU64, since: Instant) { + let nanos = u64::try_from(since.elapsed().as_nanos()).unwrap_or(u64::MAX); + wait.fetch_add(nanos, std::sync::atomic::Ordering::Relaxed); + } + + fn stamps(&self) -> ProducerStamps { + let s = |wait: &std::sync::atomic::AtomicU64| { + wait.load(std::sync::atomic::Ordering::Relaxed) as f64 / 1e9 + }; + ProducerStamps { + executor_send: s(&self.executor_send), + walker_recv: s(&self.walker_recv), + walker_send: s(&self.walker_send), + accumulator_recv: s(&self.accumulator_recv), + accumulator_send: s(&self.accumulator_send), + layout_send: s(&self.layout_send), + worker_permit: s(&self.worker_permit), + worker_recv: s(&self.worker_recv), + } + } +} + +impl BlockStamps { + /// One line a group and a total, `BLOCK …` prefixed, for a box log. + pub fn report(&self) -> String { + let sum = |f: fn(&GroupStamps) -> f64| self.groups.iter().map(f).sum::(); + let polys: usize = self.groups.iter().map(|g| g.polys).sum(); + let tree: usize = self.groups.iter().map(|g| g.tree_bytes).sum(); + let mut out = format!( + "BLOCK SHAPE: tables {} · groups {} · chains {} · cells {:.3} G · tree tops {:.2} GiB\n", + self.tables, + self.groups.len(), + polys, + self.cells as f64 / 1e9, + tree as f64 / (1u64 << 30) as f64, + ); + for (g, s) in self.groups.iter().enumerate() { + out.push_str(&format!( + "BLOCK GROUP {g}: tables {} · polys {} · cells {:.1} M || A wait {:.3} upload {:.3} paid {:.3} commit {:.3} retire {:.3} from@{:.2} done@{:.2} || B upload {:.3} argue {:.3} encode {:.3} open {:.3}{}\n", + s.tables, + s.polys, + s.cells as f64 / 1e6, + s.wait_a, + s.upload_a, + s.upload_paid, + s.commit, + s.retire, + s.committed_at - s.commit - s.retire - s.ood, + s.committed_at, + s.upload_b, + s.argue, + s.encode, + s.open, + // Under the overlap: what of the openings phase B waited for. + if self.phase_b_overlap { + format!(" paid {:.3}", s.open_paid) + } else { + String::new() + }, + )); + } + // Phase A's commit-time out-of-domain answers: how many, where their + // columns were evaluated, and what they cost the committer. + let ood_max = self.groups.iter().map(|g| g.ood).fold(0.0, f64::max); + out.push_str(&format!( + "BLOCK COMMIT OOD: answers {} · columns on card {} · on host {} · seconds {:.3} (max group {:.3})\n", + polys, + self.groups.iter().map(|g| g.ood_counts.on_card).sum::(), + self.groups.iter().map(|g| g.ood_counts.on_host).sum::(), + sum(|g| g.ood), + ood_max, + )); + if !self.build_marks.is_empty() { + let marks: Vec = self + .build_marks + .iter() + .map(|(label, at)| format!("{label} {at:.2}")) + .collect(); + out.push_str(&format!("BLOCK BUILD MARKS: {}\n", marks.join(" · "))); + } + if self.streamed.1 > 0 || self.streamed.0 > 0.0 { + out.push_str(&format!( + "BLOCK STREAM: windows collected at {:.2}s · run built at {:.2}s · {} chunks streamed · layout busy {:.2}s · phase A ended at {:.2}s (all since the streamed prove started, execution included)\n", + self.streamed.0, self.build, self.streamed.1, self.prep, self.phase_a, + )); + out.push_str(&format!( + "BLOCK WINDOWS: {} windows · walk {:.2}s (walker thread) · route+append {:.2}s · chunk handout {:.2}s (accumulator thread) · finish {:.2}s\n", + self.windows.windows, + self.windows.walk, + self.windows.route, + self.windows.generate, + self.build - self.streamed.0, + )); + let p = &self.producer; + out.push_str(&format!( + "BLOCK PRODUCER: executor send-blocked {:.2}s · walker recv-wait {:.2}s send-blocked {:.2}s · accumulator recv-wait {:.2}s send-blocked {:.2}s · layout hand-on blocked {:.2}s · workers permit-wait {:.2}s recv-wait {:.2}s (summed)\n", + p.executor_send, + p.walker_recv, + p.walker_send, + p.accumulator_recv, + p.accumulator_send, + p.layout_send, + p.worker_permit, + p.worker_recv, + )); + if self.executor.windows > 0 { + out.push_str(&self.executor.line(p.executor_send)); + } + let h = &self.windows.handout; + let tables: Vec = StreamTable::ALL + .iter() + .filter(|t| h.chunks[t.position()] > 0) + .map(|&t| { + let i = t.position(); + format!( + "{} {:.2} · {:.2} · {} · {:.1} M", + stream_base(t), + h.take[i], + h.count[i], + h.chunks[i], + h.copied[i] as f64 / 1e6, + ) + }) + .collect(); + out.push_str(&format!( + "BLOCK HANDOUT: take {:.2}s · count {:.2}s · copied {:.1} M ops || per table (take s · count s · chunks · copied ops): {}\n", + h.take.iter().sum::(), + h.count.iter().sum::(), + h.copied.iter().sum::() as f64 / 1e6, + tables.join(" | "), + )); + out.push_str(&match &self.windows.count_thread { + Some(c) => { + let tables: Vec = StreamTable::ALL + .iter() + .filter(|t| c.chunks[t.position()] > 0) + .map(|&t| { + let i = t.position(); + format!("{} {:.2} · {}", stream_base(t), c.busy[i], c.chunks[i]) + }) + .collect(); + format!( + "BLOCK HANDOUT COUNT: on a thread of its own · busy {:.2}s · waited for chunks {:.2}s · the windows' end waited {:.2}s for its last chunks and {:.3}s to add its counts || per table (busy s · chunks): {}\n", + c.busy.iter().sum::(), + c.idle, + c.drain, + c.merge, + tables.join(" | "), + ) + } + None => "BLOCK HANDOUT COUNT: on the accumulator's thread (the count above)\n" + .to_string(), + }); + let l = &self.layout; + let marks: Vec = l + .marks + .iter() + .map(|(label, at)| format!("{label} {at:.2}")) + .collect(); + let closed: Vec = l.closed_at.iter().map(|at| format!("{at:.2}")).collect(); + out.push_str(&format!( + "BLOCK LAYOUT: {} worker(s), at most {} chunk(s) unpacked · {} · groups closed at [{}] · packer waited on phase A {:.2}s · chunks laid out {:.2}s (summed)\n", + l.workers, + if l.ahead_most == 0 && l.workers > 0 { + "unbounded".to_string() + } else { + l.ahead_most.to_string() + }, + marks.join(" · "), + closed.join(", "), + l.blocked, + l.chunks, + )); + let slowest: Vec = l + .slowest + .iter() + .map(|(name, secs)| format!("{name} {secs:.3}")) + .collect(); + out.push_str(&format!( + "BLOCK REST LAYOUT: slowest tables (s) {}\n", + slowest.join(" · ") + )); + out.push_str(&format!( + "BLOCK REST PACKED: {} tables laid out narrow from the finish, {:.2} GiB\n", + l.packed_rest.0, + l.packed_rest.1 as f64 / (1u64 << 30) as f64, + )); + let c = &self.windows.card_fill; + let ((widens, widen_s), (fills, fill_s)) = self.upload_times; + out.push_str(&format!( + "BLOCK CARD FILL: KECCAK_RND {} tables held as their records, {:.2} MiB · filled \ + from records on the card {} · on the host {} · CPU chunks {} as records, {} \ + filled on the host · MEMW_R chunks {} as records, {} filled on the host · \ + streamed records {:.2} GiB · room ran out at {} · CPU logs out of step {} · \ + card upload (timed) packed {widens} tables {widen_s:.3} s, records {fills} \ + tables {fill_s:.3} s · MEMW_A chunks {} as records, {} filled on the host · \ + LOAD chunks {} as records, {} filled on the host · STORE chunks {} as records, \ + {} filled on the host · MEMW chunks {} as records, {} filled on the host · \ + SHIFT chunks {} as records, {} filled on the host\n", + l.records_rest.0, + l.records_rest.1 as f64 / (1u64 << 20) as f64, + self.records_fills.0, + self.records_fills.1, + c.cpu[0], + c.cpu[1], + c.memw_r[0], + c.memw_r[1], + c.records_bytes as f64 / (1u64 << 30) as f64, + c.room_out + .map_or("never".to_string(), |(table, index)| format!( + "{table:?}[{index}]" + )), + c.out_of_step, + c.memw_a[0], + c.memw_a[1], + c.load[0], + c.load[1], + c.store[0], + c.store[1], + c.memw[0], + c.memw[1], + c.shift[0], + c.shift[1], + )); + let released = self.groups.iter().filter(|g| g.records_released > 0); + let mut pages = multilinear::records::PagesReturned::default(); + for g in &self.groups { + pages += g.records_pages; + } + out.push_str(&format!( + "BLOCK CARD FILL released: records let go in phase B after their groups' \ + openings {:.3} GiB · {} groups · pages handed back to the OS {:.3} GiB of {} \ + buffers ({} shared, {} refused) in {:.2} s on a thread of their own\n", + released.clone().map(|g| g.records_released).sum::() as f64 + / (1u64 << 30) as f64, + released.count(), + pages.bytes as f64 / (1u64 << 30) as f64, + pages.buffers, + pages.shared, + pages.failed, + self.groups + .iter() + .map(|g| g.records_return_secs) + .sum::(), + )); + let (on, [direct, narrowed, again, wide]) = l.gpack; + out.push_str(&format!( + "BLOCK GPACK: G-pack {}: {direct} written packed, {narrowed} narrowed after, {again} \ + written again, {wide} built wide then packed\n", + if on { "on" } else { "off" }, + )); + let tables: Vec = l + .rest_tables + .iter() + .map(|(name, at, group)| format!("{name}@{at:.2}·g{group}")) + .collect(); + out.push_str(&format!( + "BLOCK REST TABLES (laid out at · group): {}\n", + tables.join(" ") + )); + } + let ((card_calls, card_secs), (host_calls, host_secs)) = self.top_routes; + out.push_str(&format!( + "BLOCK RECOMMIT: 0 (a revived commitment builds no tree) · first-round paths from kept tops: {} calls · {} leaves re-hashed · on the card {card_calls} calls in {card_secs:.3} s · on the host (fallback) {host_calls} calls in {host_secs:.3} s · dropped {} · phase B on one thread + one upload helper{}\n", + self.top_paths.0, + self.top_paths.1, + self.drop_levels + .map_or("-".to_string(), |d| d.to_string()), + if self.phase_b_overlap { + " + one opening helper" + } else { + "" + }, + )); + if host_calls > 0 { + out.push_str(&format!( + "BLOCK WARN: {host_calls} first-round path calls of codewords on the card were re-hashed on the host (the fallback; LAMBDA_VM_TOP_PATHS_HOST or a failed card re-hash): {host_secs:.2} s\n" + )); + } + let reserved: Vec = self + .groups + .iter() + .map(|g| format!("{}", g.argue_reserved >> 20)) + .collect(); + out.push_str(&format!( + "BLOCK ARGUE HW: max {} MiB · per group [{}] (the ledger's peak through each argue, the next group's upload included)\n", + self.groups + .iter() + .map(|g| g.argue_reserved >> 20) + .max() + .unwrap_or(0), + reserved.join(", "), + )); + // Phase B's ledger, for an overlap of group g's openings with group + // g + 1's argue: what the two would hold at once, against the budget. + let mib = |b: u64| b >> 20; + let per_group: Vec = self + .groups + .iter() + .map(|g| { + format!( + "{}/{}/{}/{}", + mib(g.argue_base), + mib(g.argue_reserved), + mib(g.open_room), + mib(g.open_reserved) + ) + }) + .collect(); + // Under the overlap the two ran at once: each transition's own window + // (in group order of the argue), measured. + let overlap: Vec = if self.phase_b_overlap { + self.groups + .iter() + .filter(|g| g.transition_reserved > 0) + .map(|g| format!("{}", mib(g.transition_reserved))) + .collect() + } else { + self.groups + .windows(2) + .map(|w| { + let both = + w[0].open_reserved + w[1].argue_reserved.saturating_sub(w[1].argue_base); + format!("{}", mib(both)) + }) + .collect() + }; + out.push_str(&format!( + "BLOCK PHASE-B LEDGER: budget {} MiB · per group base/argue/room/open [{}] MiB · open(g) beside argue(g+1) [{}] MiB{} · revive room refusals {} · turn refusals {} · GKR tree refusals {} · openings on the host {}\n", + mib(multilinear::gpu::reserve_budget()), + per_group.join(", "), + overlap.join(", "), + if self.phase_b_overlap { " measured" } else { "" }, + multilinear::gpu::room_revive_refusals(), + multilinear::gpu::room_turn_refusals(), + multilinear::gpu::gkr_tree_refusals(), + multilinear::gpu::open_host_fallbacks(), + )); + out.push_str(&self.overlap_line()); + if self.phase_b_sched { + out.push_str(&self.sched_line()); + } + // Phase A's ledger and its upload: what each group's upload took, what + // the committer paid of it, and what the commit promised beside it. + let hidden = sum(|g| g.upload_a - g.upload_paid); + let uploaded = sum(|g| g.upload_a); + let per_group: Vec = self + .groups + .iter() + .map(|g| format!("{}/{}", mib(g.commit_base), mib(g.commit_reserved))) + .collect(); + out.push_str(&format!( + "BLOCK PHASE-A LEDGER: budget {} MiB · per group base/peak [{}] MiB · upload {:.3}s, paid {:.3}s, hidden {:.3}s ({:.0} %) · stores refused beside a commit {} · commit room refusals {}\n", + mib(multilinear::gpu::reserve_budget()), + per_group.join(", "), + uploaded, + sum(|g| g.upload_paid), + hidden, + if uploaded > 0.0 { 100.0 * hidden / uploaded } else { 0.0 }, + self.groups.iter().filter(|g| g.ahead_refused).count(), + multilinear::gpu::room_commit_refusals(), + )); + if !self.open_split.is_empty() { + let parts: Vec = self + .open_split + .iter() + .map(|(name, v)| format!("{name} {v:.3}")) + .collect(); + out.push_str(&format!( + "BLOCK OPEN SPLIT (host s on the prover's thread): {}\n", + parts.join(" · ") + )); + } + let argue = sum(|g| g.argue); + let open = sum(|g| g.open); + let tax = sum(|g| g.upload_b + g.encode); + out.push_str(&format!( + "BLOCK PREPARED: tables {} · derived in {:.3}\n", + self.prepared.0, self.prepared.1 + )); + let packed_cells: usize = self.groups.iter().map(|g| g.packed_cells).sum(); + let packed_bytes: usize = self.groups.iter().map(|g| g.packed_bytes).sum(); + let gib = |bytes: usize| bytes as f64 / (1u64 << 30) as f64; + out.push_str(&format!( + "BLOCK NARROW: {:?} · tables packed {} of {} · cells {:.3} of {:.3} G · {:.2} → {:.2} GiB ({:.2} B/cell) · pack waited {:.3}s, busy {:.3}s (phase A) · host widens {} tables, {:.3} G cells\n", + self.narrow, + self.groups.iter().map(|g| g.packed_tables).sum::(), + self.tables, + packed_cells as f64 / 1e9, + self.cells as f64 / 1e9, + gib(packed_cells * 8), + gib(packed_bytes), + if packed_cells == 0 { + 0.0 + } else { + packed_bytes as f64 / packed_cells as f64 + }, + sum(|g| g.pack), + sum(|g| g.pack_busy), + self.host_widens.0, + self.host_widens.1 as f64 / 1e9, + )); + if let Some((start, end)) = self.phase_b_unix { + let spans: Vec = self + .phase_b_order + .iter() + .filter_map(|&g| { + self.groups + .get(g) + .map(|s| format!("{g}@{:.3}-{:.3}", s.start_b, s.end_b)) + }) + .collect(); + out.push_str(&format!( + "BLOCK PHASE-B SPAN: unix {start:.3} to {end:.3} · groups in phase B's order (group@start-end, seconds into phase B): {}\n", + spans.join(" ") + )); + } + out.push_str(&format!( + "BLOCK PHASES: execute {:.2} · build {:.2} · prep {:.2} · A {:.2} (wait {:.2} upload {:.2} paid {:.2} commit {:.2} retire {:.2}) · B {:.2} (argue {:.2} open {:.2} tax {:.2} = upload {:.2} + encode {:.2})\n", + self.execute, + self.build, + self.prep, + self.phase_a, + sum(|g| g.wait_a), + sum(|g| g.upload_a), + sum(|g| g.upload_paid), + sum(|g| g.commit), + sum(|g| g.retire), + self.phase_b, + argue, + open, + tax, + sum(|g| g.upload_b), + sum(|g| g.encode), + )); + if let Some(drops) = &self.drops { + out.push_str(&format!("BLOCK DROPS: {drops}\n")); + } + out + } + + /// Phase B's overlap ([`BlockOptions::phase_b_overlap`]), on or off: the + /// transitions whose openings ran beside the next argue and those that ran + /// alone after a refusal; the openings' seconds beside an argue, how many + /// of them the driver still waited for (paid) and the rest (hidden), and + /// the seconds of the openings run alone; the argues' seconds; the largest + /// transition's ledger peak; the openings' stream draws by pool, beside an + /// argue (whose foreground count holds the argue's draws too) and alone + /// (theirs only); the card's drain-and-trims. Off, every opening runs + /// alone. + /// Phase B's driver thread over its group turns + /// ([`GroupStamps::sched_b`]): its CPU time and run-queue delay in all and + /// by tenth of phase B (a turn counted in the tenth it started in), and + /// the first three tenths', when the regenerators are busiest; or why + /// there is nothing to print. A delay that rises where the card idles + /// says the thread was runnable and not running: CPU contention. + fn sched_line(&self) -> String { + let turns: Vec<(f64, u64, u64)> = self + .groups + .iter() + .filter_map(|g| g.sched_b.map(|(cpu, wait)| (g.start_b, cpu, wait))) + .collect(); + if turns.is_empty() { + return "BLOCK PHASE-B SCHED: unavailable (/proc/thread-self/schedstat did not read)\n" + .to_string(); + } + if turns.iter().all(|&(_, cpu, wait)| cpu == 0 && wait == 0) { + return "BLOCK PHASE-B SCHED: unavailable (schedstat reads zeros: the kernel keeps no \ + scheduler statistics)\n" + .to_string(); + } + let ns = 1e-9; + let mut cpu = [0.0f64; 10]; + let mut wait = [0.0f64; 10]; + for &(start, c, w) in &turns { + let tenth = if self.phase_b > 0.0 { + ((start / self.phase_b * 10.0) as usize).min(9) + } else { + 0 + }; + cpu[tenth] += c as f64 * ns; + wait[tenth] += w as f64 * ns; + } + let list = |v: &[f64]| { + v.iter() + .map(|x| format!("{x:.2}")) + .collect::>() + .join(", ") + }; + format!( + "BLOCK PHASE-B SCHED: the driver thread over {} group turns · CPU {:.3} s · run-queue delay \ + {:.3} s · first three tenths of phase B: CPU {:.3} s, run-queue delay {:.3} s · by tenth \ + (a turn in the tenth it began) CPU [{}] · run-queue delay [{}]\n", + turns.len(), + cpu.iter().sum::(), + wait.iter().sum::(), + cpu[..3].iter().sum::(), + wait[..3].iter().sum::(), + list(&cpu), + list(&wait), + ) + } + + fn overlap_line(&self) -> String { + let beside = || self.groups.iter().filter(|g| g.overlapped); + let alone = || self.groups.iter().filter(|g| !g.overlapped); + // Summed from +0.0: an empty f64 sum is -0.0, which prints a sign. + let sum = |values: &mut dyn Iterator| values.fold(0.0, |a, b| a + b); + let opened = sum(&mut beside().map(|g| g.encode + g.open)); + let paid = sum(&mut beside().map(|g| g.open_paid)); + let draws = |groups: &mut dyn Iterator| { + groups.fold((0u64, 0u64), |(f, b), g| { + (f + g.open_draws.0, b + g.open_draws.1) + }) + }; + let (beside_f, beside_b) = draws(&mut beside()); + let (alone_f, alone_b) = draws(&mut alone()); + let peak = self + .groups + .iter() + .map(|g| g.transition_reserved) + .max() + .unwrap_or(0); + format!( + "BLOCK PHASE-B OVERLAP: {} · transitions overlapped {} of {} (serial after a refusal {}) · openings beside an argue {opened:.3} s, hidden {:.3} s, paid {paid:.3} s + alone {:.3} s · argue {:.3} s · transition ledger peak max {} MiB (budget {} MiB) · openings' stream draws beside an argue background {beside_b} / foreground {beside_f} (the argue's too), alone background {alone_b} / foreground {alone_f} · trims {}{}\n", + match (self.phase_b_overlap, self.phase_b_pinned) { + (_, true) => "pinned", + (true, false) => "on", + (false, false) => "off", + }, + beside().count(), + self.groups.len().saturating_sub(1), + self.groups + .iter() + .filter(|g| g.serial_after_refusal) + .count(), + opened - paid, + sum(&mut alone().map(|g| g.open_paid)), + self.groups.iter().map(|g| g.argue).sum::(), + peak >> 20, + multilinear::gpu::reserve_budget() >> 20, + multilinear::gpu::device_trims(), + if self.phase_b_overlap { + let (pinned, pageable, bytes) = self.background_readbacks; + format!( + " · the openings' read-backs pinned {pinned} / pageable {pageable} ({:.1} MiB pinned)", + bytes as f64 / (1u64 << 20) as f64 + ) + } else { + String::new() + }, + ) + } +} + +/// The CPU tables' heights a block statement states, at the CPU tables' +/// positions (`crate::cpu_proof_start`), each at most [`BLOCK_CPU_MAX_VARS`] +/// variables. [`BlockAirInputs::new`] then checks them against the clock. +pub(crate) fn block_cpu_heights( + counts: &TableCounts, + table_num_vars: &[u8], +) -> Result, Error> { + let start = crate::cpu_proof_start(true, counts); + let stated = start + .checked_add(counts.cpu) + .and_then(|end| table_num_vars.get(start..end)) + .ok_or_else(|| { + Error::InvalidTableCounts("the statement states fewer heights than CPU tables".into()) + })?; + stated + .iter() + .enumerate() + .map(|(k, &vars)| { + if usize::from(vars) > BLOCK_CPU_MAX_VARS { + return Err(Error::InvalidTableCounts(format!( + "CPU[{k}] states 2^{vars} rows — a block takes at most 2^{BLOCK_CPU_MAX_VARS}" + ))); + } + Ok(1usize << vars) + }) + .collect() +} + +/// The counts a block statement may declare: [`TableCounts::validate`], with +/// COPY, KECCAK, KECCAK_RND, ECSM and ECDAS chunked (at most +/// [`BLOCK_MAX_COPY`], [`BLOCK_MAX_KECCAK`], [`BLOCK_MAX_KECCAK_RND`], +/// [`BLOCK_MAX_ECSM`] and [`BLOCK_MAX_ECDAS`] tables). +pub fn validate_block_counts(counts: &TableCounts) -> Result<(), Error> { + for (name, count, max) in [ + ("copy", counts.copy, BLOCK_MAX_COPY), + ("keccak", counts.keccak, BLOCK_MAX_KECCAK), + ("keccak_rnd", counts.keccak_rnd, BLOCK_MAX_KECCAK_RND), + ("ecsm", counts.ecsm, BLOCK_MAX_ECSM), + ("ecdas", counts.ecdas, BLOCK_MAX_ECDAS), + ] { + if count > max { + return Err(Error::InvalidTableCounts(format!( + "{name} count is {count} — a block takes at most {max}" + ))); + } + } + let mut rest = counts.clone(); + rest.copy = rest.copy.min(1); + rest.keccak = rest.keccak.min(1); + rest.keccak_rnd = rest.keccak_rnd.min(1); + rest.ecsm = rest.ecsm.min(1); + rest.ecdas = rest.ecdas.min(1); + rest.validate() +} + +/// The proof-order index range of each chunked table, with its height cap in +/// variables. The order is [`VmAirs::air_refs`]'s: the [`FIXED_TABLE_COUNT`] +/// fixed tables, then COPY, KECCAK, KECCAK_RND, ECSM, ECDAS, … (pinned +/// against the AIR names by `chunked_table_ranges_name_the_chunked_airs`). +/// Counts whose total passed [`TableCounts::total`] cannot overflow these sums. +pub(crate) fn chunked_table_ranges( + counts: &TableCounts, +) -> [(&'static str, std::ops::Range, usize); 5] { + let copy = FIXED_TABLE_COUNT; + let keccak = copy + counts.copy; + let keccak_rnd = keccak + counts.keccak; + let ecsm = keccak_rnd + counts.keccak_rnd; + let ecdas = ecsm + counts.ecsm; + [ + ("COPY", copy..keccak, BLOCK_COPY_MAX_VARS), + ("KECCAK", keccak..keccak_rnd, BLOCK_KECCAK_MAX_VARS), + ("KECCAK_RND", keccak_rnd..ecsm, BLOCK_KECCAK_RND_MAX_VARS), + ("ECSM", ecsm..ecdas, BLOCK_ECSM_MAX_VARS), + ("ECDAS", ecdas..ecdas + counts.ecdas, BLOCK_ECDAS_MAX_VARS), + ] +} + +/// Every table at most [`BLOCK_MAX_TABLE_VARS`] variables, so no group's chain +/// is taller than the larger of the format's stack and 2^27. +pub(crate) fn check_table_heights(table_num_vars: &[u8]) -> Result<(), Error> { + if let Some((idx, &num_vars)) = table_num_vars + .iter() + .enumerate() + .find(|&(_, &n)| usize::from(n) > BLOCK_MAX_TABLE_VARS) + { + return Err(Error::InvalidTableCounts(format!( + "table {idx} states 2^{num_vars} rows — a block table takes at most \ + 2^{BLOCK_MAX_TABLE_VARS}" + ))); + } + Ok(()) +} + +/// BITWISE's position in a block's tables: the first fixed table +/// ([`crate::VmAirs::air_refs`]'s order). +pub(crate) const BITWISE_TABLE: usize = 0; + +/// BITWISE's stated height, held to its fixed 2^20 rows +/// ([`NUM_VARS`](crate::tables::bitwise::NUM_VARS)), the only height its +/// preprocessed columns have. Refused by name here rather +/// than left to the preprocessed check's dimension mismatch further on: the +/// WHIR query count is derived from the statement's tallest stack, and +/// BITWISE's (21 columns of 2^20 rows, 25 variables) is what holds it at the +/// production count for every block, so the pin is a rule of its own. +pub(crate) fn check_bitwise_height(table_num_vars: &[u8]) -> Result<(), Error> { + match table_num_vars.get(BITWISE_TABLE) { + Some(&n) if usize::from(n) == crate::tables::bitwise::NUM_VARS => Ok(()), + stated => Err(Error::InvalidTableCounts(format!( + "BITWISE states {} — its table is 2^{} rows", + stated.map_or("no height".to_string(), |n| format!("2^{n} rows")), + crate::tables::bitwise::NUM_VARS + ))), + } +} + +/// Every chunked table under its cap, against the statement's heights: COPY +/// at most [`BLOCK_COPY_MAX_VARS`] variables, KECCAK +/// [`BLOCK_KECCAK_MAX_VARS`], KECCAK_RND [`BLOCK_KECCAK_RND_MAX_VARS`], ECSM +/// [`BLOCK_ECSM_MAX_VARS`] and ECDAS [`BLOCK_ECDAS_MAX_VARS`]. Run once the heights are known to be one per +/// table. +pub(crate) fn check_chunked_heights( + counts: &TableCounts, + table_num_vars: &[u8], +) -> Result<(), Error> { + for (name, range, max_vars) in chunked_table_ranges(counts) { + for (k, idx) in range.enumerate() { + let num_vars = usize::from(table_num_vars[idx]); + if num_vars > max_vars { + return Err(Error::InvalidTableCounts(format!( + "{name}[{k}] (table {idx}) states 2^{num_vars} rows — a block takes at most \ + 2^{max_vars}" + ))); + } + } + } + Ok(()) +} + +/// The block statement: the monolithic multilinear one under +/// [`MULTILINEAR_BLOCK_TAG`], then the group partition — the count, then per +/// group its size and its tables' indices, every value a little-endian `u64` +/// (so the roots that follow stay on a field-element boundary). +#[allow(clippy::too_many_arguments)] +pub(crate) fn absorb_block( + t: &mut impl IsTranscript, + tag: &[u8], + elf_bytes: &[u8], + public_output: &[u8], + table_counts: &TableCounts, + num_private_input_pages: usize, + runtime_page_ranges: &[RuntimePageRange], + table_num_vars: &[u8], + config: &multilinear::whir_chain::ChainConfig, + groups: &[Vec], +) { + absorb_tagged( + t, + tag, + "block", + &statement::elf_digest(elf_bytes), + public_output, + table_counts, + num_private_input_pages, + runtime_page_ranges, + table_num_vars, + config, + ); + t.append_bytes(&(groups.len() as u64).to_le_bytes()); + for group in groups { + t.append_bytes(&(group.len() as u64).to_le_bytes()); + for &index in group { + t.append_bytes(&u64::from(index).to_le_bytes()); + } + } +} + +/// A partition of `tables` tables into non-empty groups: every index below +/// `tables`, each exactly once. Returns the indices in proof order. +pub fn validate_groups(groups: &[Vec], tables: usize) -> Result, Error> { + let mut seen = vec![false; tables]; + let mut order = Vec::with_capacity(tables); + for group in groups { + if group.is_empty() { + return Err(Error::InvalidTableCounts("an empty group".to_string())); + } + for &index in group { + let index = index as usize; + match seen.get_mut(index) { + Some(slot) if !*slot => *slot = true, + Some(_) => { + return Err(Error::InvalidTableCounts(format!( + "table {index} is in two groups" + ))); + } + None => { + return Err(Error::InvalidTableCounts(format!( + "group names table {index} of {tables}" + ))); + } + } + order.push(index); + } + } + if order.len() != tables { + return Err(Error::InvalidTableCounts(format!( + "the groups cover {} of {tables} tables", + order.len() + ))); + } + Ok(order) +} + +/// A table cut into tables of `rows` rows each, in row order. Every VM table's +/// constraints read one row (no shifted reads), so each piece is a table of +/// the same AIR, and the bus sums over the pieces are the sum over the whole. +pub(crate) fn split_rows(table: TraceTable, rows: usize) -> Vec> { + let pieces = table.main_table.height / rows.max(1); + first_pieces(table, rows, pieces) +} + +/// [`split_rows`] for a family the block cuts at a fixed number of rows +/// (KECCAK_RND, KECCAK, ECSM, ECDAS), keeping as many pieces as `padding` +/// counts ([`CutPadding::tables`]) of the table's real rows: its rows up to the +/// last one whose `mu` (the family's multiplicity column, zero on a padding +/// row) is set. Under [`CutPadding::LastTable`] the pieces of padding rows +/// alone go, as the finish leaves them unbuilt; under [`CutPadding::WholeRun`] +/// every piece stays. +fn split_cut( + table: TraceTable, + rows: usize, + mu: usize, + padding: CutPadding, +) -> Vec> { + let height = table.main_table.height; + match padding { + CutPadding::LastTable if height > rows => { + let real = (0..height) + .rev() + .find(|&row| *table.main_table.get(row, mu) != FieldElement::zero()) + .map_or(0, |row| row + 1); + let pieces = padding.tables(real, rows).clamp(1, height / rows); + first_pieces(table, rows, pieces) + } + _ => split_rows(table, rows), + } +} + +/// The first `pieces` of [`split_rows`]'s tables; `table` itself when it is no +/// taller than `rows`. +fn first_pieces(table: TraceTable, rows: usize, pieces: usize) -> Vec> { + let height = table.main_table.height; + if height <= rows { + return vec![table]; + } + let step = table.step_size; + let width = table.main_table.width; + // Row-major in, row-major out: each piece is a run of whole rows, copied + // as they lie (a transposition of KECCAK_RND's 1,480 columns is what this + // used to cost). + (0..pieces.min(height / rows)) + .into_par_iter() + .map(|k| { + let mut data = Vec::with_capacity(rows * width); + for row in k * rows..(k + 1) * rows { + data.extend_from_slice(table.main_table.get_row(row)); + } + TraceTable::new_main(data, width, step) + }) + .collect() +} + +/// One table of the proof: its layout and its columns, taken out of the trace +/// (whose row-major copy goes when `release_rows`), checked against the +/// columns the program implies. +fn table_of<'a>( + air: &'a dyn stark::traits::AIR, + trace: &mut TraceTable, + shape: (usize, usize), + release_rows: bool, +) -> Result, Error> { + table_of_checked(air, trace, shape, release_rows, true) +} + +/// [`table_of`], its preprocessed-column check `check`ed or not (a test's +/// [`Deviations::unchecked_preprocessed`]). +fn table_of_checked<'a>( + air: &'a dyn stark::traits::AIR, + trace: &mut TraceTable, + (width, num_vars): (usize, usize), + release_rows: bool, + check: bool, +) -> Result, Error> { + let layout = layout_of(air, width, num_vars) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name())))?; + let mut columns = trace.main_table.columns_blocked(); + // The row-major copy goes as soon as the columns exist: a block's traces + // are tens of GiB, and two copies of them at once is the peak this entry + // point exists to avoid. + if release_rows { + trace.main_table = Table::new(Vec::new(), 0); + } + for (col, expected) in air.precomputed_columns().iter().enumerate() { + if check && columns.get(col) != Some(expected) { + return Err(Error::Prover(format!( + "{}: preprocessed column {col} is not what the program implies", + air.name(), + ))); + } + } + CommittedTable::from_layout(layout, |col| core::mem::take(&mut columns[col as usize])) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name()))) +} + +/// The CPU tables' heights of a rest the finish built after a streamed walk. A +/// CPU chunk the walk streamed is still a placeholder here, with no rows; its +/// height is the streamed chunk's, `cpu_rows` (a streamed CPU chunk is always +/// full, `MaxRowsConfig::cpu` rows), the height [`StreamAirs`] lays it out with. +/// The finish puts the placeholders first; one after a built table is refused. +fn rest_cpu_heights(traces: &Traces, cpu_rows: usize) -> Result, Error> { + let streamed = traces.cpus.iter().take_while(|t| t.num_rows() == 0).count(); + if traces.cpus[streamed..].iter().any(|t| t.num_rows() == 0) { + return Err(Error::Prover( + "a streamed CPU chunk's placeholder after a built CPU table".to_string(), + )); + } + Ok(traces + .cpus + .iter() + .enumerate() + .map(|(k, t)| if k < streamed { cpu_rows } else { t.num_rows() }) + .collect()) +} + +/// [`table_of`] for a trace the finish packed as it built it +/// ([`BlockOptions::pack_finished`]): its packed columns become the table's, +/// held narrow from the start, with no column of field elements made; the +/// preprocessed columns are checked word for word against the program's. +pub(crate) fn table_of_narrow<'a>( + air: &'a dyn stark::traits::AIR, + trace: &mut TraceTable, + (width, num_vars): (usize, usize), +) -> Result, Error> { + let layout = layout_of(air, width, num_vars) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name())))?; + let packed = trace + .take_narrow_main() + .ok_or_else(|| Error::Prover(format!("{}: no packed columns", air.name())))?; + trace.main_table = Table::new(Vec::new(), 0); + if packed.cols() != width || packed.rows() != 1usize << num_vars { + return Err(Error::Prover(format!( + "{}: packed {} × {}, the table is {width} × 2^{num_vars}", + air.name(), + packed.cols(), + packed.rows() + ))); + } + for (col, expected) in air.precomputed_columns().iter().enumerate() { + let same = col < packed.cols() + && packed + .column(col) + .iter() + .eq(expected.iter().map(|value| value.value())); + if !same { + return Err(Error::Prover(format!( + "{}: preprocessed column {col} is not what the program implies", + air.name(), + ))); + } + } + CommittedTable::from_narrow(layout, packed) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name()))) +} + +/// [`table_of_narrow`] for a trace held as the records its rows come from (a +/// finish's KECCAK_RND table, [`WindowedTraceBuilder::card_fill_keccak_rnd`]; +/// a streamed CPU or MEMW_R chunk, +/// [`WindowedTraceBuilder::card_fill_streamed`]): its records become the +/// table's, and the card fills its columns from them as its group uploads it. +/// A table whose AIR's preprocessed columns are not the leading columns its +/// records fill from the row index alone is refused +/// (`multilinear::records::TableRecords::row_index_columns`). +pub(crate) fn table_of_records<'a>( + air: &'a dyn stark::traits::AIR, + trace: &mut TraceTable, + (width, num_vars): (usize, usize), +) -> Result, Error> { + let layout = layout_of(air, width, num_vars) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name())))?; + let records = trace + .take_records_main() + .ok_or_else(|| Error::Prover(format!("{}: no records", air.name())))?; + trace.main_table = Table::new(Vec::new(), 0); + if records.cols() != width || records.rows() != 1usize << num_vars { + return Err(Error::Prover(format!( + "{}: records of {} × {}, the table is {width} × 2^{num_vars}", + air.name(), + records.cols(), + records.rows() + ))); + } + // The AIR's preprocessed columns must be the ones the fill writes from the + // row index alone: the check of them reads the columns, which only the + // fill makes, so a count the records do not declare is refused. + let preprocessed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + if preprocessed != records.row_index_columns() { + return Err(Error::Prover(format!( + "{}: {preprocessed} preprocessed columns, its records fill {} from the row index", + air.name(), + records.row_index_columns(), + ))); + } + CommittedTable::from_records(layout, records) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name()))) +} + +/// A prepared table's index in [`VmAirs::air_refs`] order and the columns its +/// opening settles. +pub(crate) type PreparedColumns = (usize, Vec>>); + +/// The prepared tables' columns' bytes, at their capacities. +fn prepared_bytes(prepared: &[PreparedColumns]) -> usize { + prepared + .iter() + .flat_map(|(_, columns)| columns) + .map(crate::tables::trace_builder::vec_heap_bytes) + .sum() +} + +/// One group's PREPARED commitment ([`multilinear_block::BlockPrepared`]): the +/// leading preprocessed columns of the group's prepared tables, stacked in the +/// group's order and committed under the block's parameters. Both sides derive +/// it from the program and the partition — the prover opens it, the verifier +/// absorbs its roots and checks the opening — and no proof carries it. +pub(crate) struct GroupPrepared { + pub(crate) group: usize, + /// `(position in the proof's table order, prefix length)`, in stack order. + pub(crate) tables: Vec<(usize, usize)>, + pub(crate) columns: Vec>, + pub(crate) roots: Vec, + pub(crate) commitment: multilinear::stacked_eval::StackedCommitment, +} + +/// One group's prepared stack before its commitment: which tables, and their +/// prefixes' columns concatenated in the group's order. +pub(crate) struct GroupStack { + pub(crate) group: usize, + /// `(position in the proof's table order, AIR index, prefix length)`. + pub(crate) tables: Vec<(usize, usize, usize)>, + pub(crate) columns: Vec>>, +} + +impl GroupStack { + /// The stack's shapes, one `(columns, variables)` per table: what its layout + /// is built from. + pub(crate) fn shapes(&self) -> Vec<(usize, usize)> { + let mut at = 0usize; + self.tables + .iter() + .map(|&(_, _, n)| { + let vars = self.columns[at].len().trailing_zeros() as usize; + at += n; + (n, vars) + }) + .collect() + } +} + +/// The groups' prepared stacks, in group order: each group's prepared tables +/// ([`prepared_tables`]) in the group's table order. A pure function of the +/// prepared set and the partition — what both sides commit. +pub(crate) fn group_stacks( + prepared: Vec, + groups: &[Vec], +) -> Result, Error> { + let mut by_table: Vec>>>> = Vec::new(); + for (table, columns) in prepared { + if by_table.len() <= table { + by_table.resize_with(table + 1, || None); + } + by_table[table] = Some(columns); + } + let mut stacks = Vec::new(); + let mut position = 0usize; + for (g, group) in groups.iter().enumerate() { + let mut stack = GroupStack { + group: g, + tables: Vec::new(), + columns: Vec::new(), + }; + for &table in group { + let table = table as usize; + if let Some(columns) = by_table.get_mut(table).and_then(Option::take) { + stack.tables.push((position, table, columns.len())); + stack.columns.extend(columns); + } + position += 1; + } + if !stack.tables.is_empty() { + stacks.push(stack); + } + } + if let Some(table) = by_table.iter().position(Option::is_some) { + return Err(Error::Prover(format!( + "prepared table {table} is in no group" + ))); + } + Ok(stacks) +} + +/// Where DECODE sits among a block's tables, found by NAME. +/// +/// ⚠ Not a constant and not a position. The opening binds the pinned columns to +/// ONE table's reduced point, and a prover able to aim them at a different +/// table's point would be settling them against challenges they were never +/// bound to. So the index is asserted against the AIR that carries the columns, +/// and "exactly one" is part of the assertion: a table set with two DECODEs, or +/// none, is a layout nobody meant to build. +pub(crate) fn decode_table_index( + airs: &[&dyn stark::traits::AIR], +) -> Result { + let found: Vec = airs + .iter() + .enumerate() + .filter(|(_, air)| air.name() == "DECODE") + .map(|(i, _)| i) + .collect(); + match found.as_slice() { + [only] => Ok(*only), + _ => Err(Error::Prover(format!( + "a block's table set must carry exactly one DECODE, it carries {} (at {found:?})", + found.len(), + ))), + } +} + +/// The tables a block opens out of band, in [`VmAirs::air_refs`] order, with +/// the columns each opening settles: +/// - DECODE: its ELF-derived columns, all of them; +/// - every genesis page the cross-epoch rule stacks +/// ([`crate::continuation::genesis_stack_plan`]): both its columns. +/// +/// These are the columns an in-guest verifier cannot evaluate: five columns of +/// 2^20, and a dense page whose sparse form is millions of rows. So the block +/// settles them with an opening, as an epoch settles DECODE. The result is a +/// function of the ELF and the page configs alone. The rule reads no private +/// page's bytes, so the prover's and the verifier's views of the configs give +/// the same set. +pub(crate) fn prepared_tables( + airs: &VmAirs, + page_configs: &[crate::tables::page::PageConfig], + format: &BlockFormat, +) -> Result, Error> { + if !format.prepared { + return Ok(Vec::new()); + } + let refs = airs.air_refs(); + if page_configs.len() != airs.pages.len() { + return Err(Error::Prover(format!( + "{} page configs for {} page tables", + page_configs.len(), + airs.pages.len() + ))); + } + let first_page = first_page_index(airs)?; + let decode = decode_table_index(&refs)?; + let mut tables = vec![(decode, refs[decode].precomputed_columns())]; + let plan = crate::continuation::genesis_stack_plan( + page_configs, + first_page, + crate::continuation::PAGE_NUM_VARS, + ); + for route in plan.routes.iter().filter(|route| route.dense) { + tables.push(( + route.table, + crate::tables::page::preprocessed_columns(&page_configs[route.table - first_page]), + )); + } + tables.sort_by_key(|&(table, _)| table); + Ok(tables) +} + +/// Where the page tables start in [`VmAirs::air_refs`] order (its length when +/// there are none): page `i` of the configs is table `first + i`. +pub(crate) fn first_page_index(airs: &VmAirs) -> Result { + let refs = airs.air_refs(); + match airs.pages.first() { + Some(page) => refs + .iter() + .position(|air| { + std::ptr::eq( + *air as *const _ as *const (), + page.as_ref() as *const _ as *const (), + ) + }) + .ok_or_else(|| Error::Prover("the page tables are not in the AIR set".into())), + None => Ok(refs.len()), + } +} + +/// Commits one group's prepared stack under `config`. +pub(crate) fn commit_group( + stack: GroupStack, + config: &multilinear::whir_chain::ChainConfig, +) -> Result, Error> { + let shapes = stack.shapes(); + let group = stack.group; + let columns: Vec> = stack + .columns + .into_iter() + .map(|values| Mle::new(values).map_err(|e| Error::Prover(format!("group {group}: {e:?}")))) + .collect::>()?; + let layout = stark::multilinear_table::global_layout(&shapes, config.format.stack) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + let commitment = multilinear::stacked_eval::StackedCommitment::::commit( + layout, + &multilinear::stacking::borrow(&columns), + None, + config, + ) + .map_err(|e| Error::Prover(format!("group {group}: {e:?}")))?; + let roots = commitment.roots(); + Ok(GroupPrepared { + group, + tables: stack.tables.iter().map(|&(p, _, n)| (p, n)).collect(), + columns, + roots, + commitment, + }) +} + +/// Every group's prepared commitment, in group order. +pub(crate) fn commit_groups( + stacks: Vec, + config: &multilinear::whir_chain::ChainConfig, +) -> Result>, Error> { + stacks + .into_iter() + .map(|stack| commit_group::(stack, config)) + .collect() +} + +/// The prepared commitments' roots by group, in group order. +fn prepared_roots( + prepared: &[GroupPrepared], +) -> Vec { + prepared + .iter() + .map(|p| (p.group, p.roots.clone())) + .collect() +} + +/// The prover's prepared commitments over `columns` ([`prepared_tables`]) for +/// the partition `groups`, the deviations applied, the deviations in the block +/// prover's terms, and the seconds they took. +#[allow(clippy::type_complexity)] +fn prover_prepared( + columns: Vec, + groups: &[Vec], + config: &multilinear::whir_chain::ChainConfig, + deviations: &Deviations, +) -> Result< + ( + Vec>, + Vec, + f64, + ), + Error, +> { + let t = Instant::now(); + let originals: Vec = if deviations.prepared_stack.is_empty() { + Vec::new() + } else { + columns.clone() + }; + let mut stacks = group_stacks(columns, groups)?; + let mut touched: Vec = Vec::new(); + // Where table `air`'s block sits: its stack, and its columns' range there. + let block_of = |stacks: &[GroupStack], air: usize| -> Option<(usize, std::ops::Range)> { + stacks.iter().enumerate().find_map(|(k, stack)| { + let mut at = 0usize; + for &(_, table, n) in &stack.tables { + if table == air { + return Some((k, at..at + n)); + } + at += n; + } + None + }) + }; + if deviations.other_prepared + && let Some(stack) = stacks.first_mut() + { + stack.columns[0][0] += FieldElement::::one(); + touched.push(stack.group); + } + for tamper in &deviations.prepared_stack { + match *tamper { + StackTamper::Swap(a, b) => { + let (ka, ra) = block_of(&stacks, a) + .ok_or_else(|| Error::Prover(format!("{a} is not prepared")))?; + let (kb, rb) = block_of(&stacks, b) + .ok_or_else(|| Error::Prover(format!("{b} is not prepared")))?; + if ka != kb || ra.len() != rb.len() { + return Err(Error::Prover( + "a swap within one stack, of equal blocks".into(), + )); + } + for (i, j) in ra.zip(rb) { + stacks[ka].columns.swap(i, j); + } + touched.push(stacks[ka].group); + } + StackTamper::ColumnsOf { table, from } => { + let (k, range) = block_of(&stacks, table) + .ok_or_else(|| Error::Prover(format!("{table} is not prepared")))?; + let source = &originals + .iter() + .find(|(t, _)| *t == from) + .ok_or_else(|| Error::Prover(format!("{from} is not prepared")))? + .1; + if source.len() != range.len() { + return Err(Error::Prover("blocks of different widths".into())); + } + for (i, column) in range.zip(source) { + stacks[k].columns[i] = column.clone(); + } + touched.push(stacks[k].group); + } + } + } + let position_of = |air: usize| { + groups + .iter() + .flatten() + .position(|&t| t as usize == air) + .ok_or_else(|| Error::Prover(format!("table {air} is in no group"))) + }; + let mut points: Vec<(usize, (usize, usize))> = Vec::new(); + for &(table, other) in &deviations.prepared_points { + let (k, _) = block_of(&stacks, table) + .ok_or_else(|| Error::Prover(format!("{table} is not prepared")))?; + points.push((stacks[k].group, (position_of(table)?, position_of(other)?))); + touched.push(stacks[k].group); + } + let prepared = commit_groups::(stacks, config)?; + touched.sort_unstable(); + touched.dedup(); + let tampered = touched + .into_iter() + .map(|group| { + let open = prepared + .iter() + .position(|p| p.group == group) + .expect("a touched group holds a stack"); + multilinear_block::PreparedDeviation { + open, + at_point_of: points + .iter() + .filter(|(g, _)| *g == group) + .map(|&(_, pair)| pair) + .collect(), + consistent: true, + } + }) + .collect(); + Ok((prepared, tampered, t.elapsed().as_secs_f64())) +} + +/// Proves a block in one proof. +pub fn prove_block_whir( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, +) -> Result<(BlockWhirProof, BlockStamps), Error> { + prove_block_whir_with( + elf_bytes, + private_inputs, + proof_options, + format, + options, + &Deviations::default(), + ) +} + +/// What a test makes the prover do wrong, for the verifier to refuse. The +/// default is an honest prover. +#[derive(Default)] +pub(crate) struct Deviations { + /// Leave the first KECCAK_RND table out of the proof — its rows, its + /// argument and its count — as a prover hiding an instance would. The + /// first, because it holds real rounds: a table of padding rows alone + /// carries nothing on the bus, and leaving one out is an honest proof. + pub omit_first_keccak_rnd: bool, + /// Prove group `g` on the fork of this index instead of `g`. + pub fork_of: Option usize>, + /// Commit the first prepared stack over columns that differ from the + /// program's in one entry (DECODE's, the first prepared table), as a prover + /// of another program would, and open it consistently. + pub other_prepared: bool, + /// Commit the prepared stacks wrongly ([`StackTamper`]), opening each + /// consistently. + pub prepared_stack: Vec, + /// `(table, other)`, AIR indices: open `table`'s prepared block at `other`'s + /// point (a table of its group and its height), consistently. + pub prepared_points: Vec<(usize, usize)>, + /// Under the batched argue: a group argued with faults, or every group + /// argued on the host ([`multilinear_block::ArgueDeviation`]). + pub argue: multilinear_block::ArgueDeviation, + /// Faults in phase B's overlap ([`multilinear_block::OverlapFaults`]). + pub overlap_faults: multilinear_block::OverlapFaults, + /// Between the phases, the first narrow table's width map made wrong, so + /// phase B widens other words than were committed + /// ([`BlockCommitted::fault_narrow_width_map`]). + pub narrow_width_map: bool, + /// At the walk's end, the hand-off asked for everything, whatever the + /// forecast ([`hand_off_need`]). + pub hand_off_all: bool, + /// The shadow regenerator's slicer drops the first op of this table's + /// list ([`regen::ShadowRun::spawn`]): a regeneration bug, which the shadow + /// must report and the proof must not see. + pub regen_drop_one_op: Option, + /// Phase B takes the groups in this order, given their count, instead of + /// the prove's own ([`multilinear_block::block_prove_in_order`]): every + /// order proves the same bytes. + pub phase_b_order: Option Vec>, + /// Live regeneration's faults ([`regen::LiveFaults`]): a regenerator that + /// dies or deposits wrong columns, which phase B must refuse, never prove + /// over. + pub regen_faults: regen::LiveFaults, + /// The rest's regenerator's faults (N3, [`regen::RestRun`]): as + /// [`Self::regen_faults`], a table named by its place among the rest's + /// dropped tables. + pub rest_faults: regen::LiveFaults, + /// Both regeneration windows fixed, in bytes (the streamed chunks', the + /// rest's), instead of the knobs: a window of one byte admits only its + /// frontier, so a phase-B order that takes a class out of its rank order + /// cannot complete. + pub regen_windows: Option<(u64, u64)>, + /// When live regeneration starts the streamed chunks' regenerator + /// ([`regen::EarlyStart`]), instead of the knob ([`regen::early_start`]). + pub regen_early: Option, + /// The streamed groups phase B takes ahead of the rest's first + /// ([`led_order`]), instead of the knob ([`regen::regen_lead`]). + pub regen_lead: Option, + /// A pause before each drop of a streamed chunk, so the streamed + /// chunks' regenerator started early meets chunks whose fate phase A has + /// not decided yet. + pub streamed_drop_delay: Option, + /// The builder fails right after the finish (and after any early start): + /// phase A ends in an error with the streamed chunks' regenerator + /// running. + pub abort_after_finish: bool, + /// The rest handed on with BITWISE last, beside the count, as + /// `LAMBDA_VM_BLOCK_REST_BITWISE_LAST=1` asks ([`block_rest_bitwise_last`]). + pub rest_bitwise_last: bool, + /// BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables, BITWISE + /// last (`Some(true)`), or from their ops in phase 4, BITWISE first + /// (`Some(false)`), instead of what `LAMBDA_VM_BLOCK_BITWISE_FROM_TABLES` + /// says (`None`; [`block_bitwise_from_tables`]). + pub bitwise_from_tables: Option, + /// The hand-out's counting on a thread of its own (`Some(true)`) or on + /// the accumulator's (`Some(false)`), instead of what + /// `LAMBDA_VM_HANDOUT_COUNT_THREAD` says (`None`; + /// [`WindowedTraceBuilder::count_handout_on_thread`]). + pub handout_count_thread: Option, + /// The host's memory target, in bytes, instead of the machine's + /// ([`memory_target_bytes`]): 0 makes the `auto` rule want every table out. + pub memory_target: Option, + /// How many closed groups may wait for phase A, instead of the knob + /// ([`group_queue::setting`]). + pub group_queue: Option, + /// A pause as phase A takes each closed group, so that at a test's size + /// the packer closes groups faster than phase A takes them. + pub group_take_pause: Option, + /// On the whole-run path, the traces edited after their cuts and before + /// the proof, as a prover of other tables would. + pub tamper_traces: Option, + /// On the whole-run path, the tables committed without the prover's own + /// check of their preprocessed columns against the program's, so a table + /// whose program columns were edited reaches the verifier. + pub unchecked_preprocessed: bool, + /// Whether the card fills KECCAK_RND from its records, instead of the + /// knob ([`block_card_fill`]). + pub card_fill: Option, + /// With the card fill, each KECCAK_RND table's records start this many + /// rows late ([`WindowedTraceBuilder::card_fill_records_fault`]): one + /// wrong table, the same at both uploads and to any host reader, so the + /// prover's own checks cannot tell it from the committed one. + #[cfg(test)] + pub card_fill_records_fault: usize, + /// With the streamed CPU chunks held as records, each log's first source + /// and written values XORed with this, and with row families' chunks held as their rows, + /// each row's timestamp this much later (a SHIFT op's input) + /// ([`WindowedTraceBuilder::card_fill_streamed_fault`]): one wrong table, + /// the same at both uploads and to any host reader. + #[cfg(test)] + pub card_fill_streamed_fault: u64, + /// Whether the card fills the streamed CPU and MEMW_R chunks from their + /// records (`[cpu, memw_r]`), instead of the knob. + pub card_fill_streamed: Option<[bool; 2]>, + /// Which row families' streamed chunks the card fills from their rows, + /// instead of the knob; with both of the above set and this unset, none. + pub card_fill_rows: Option, + /// The room rule the streamed chunks held as records are admitted by, + /// instead of the host's resident set against its pressure line. + pub records_room: Option, + /// Whether streamed chunks held as records stay on the host instead of + /// parking and dropping, instead of the knob ([`records_keep`]). + pub records_keep: Option, + /// Whether phase A's head is stamped, instead of the knob + /// ([`head_log::knob`]). + pub head_log: Option, + /// The trace alone: phase A drains each closed group as it comes instead + /// of committing it, and the prove stops after the trace with + /// [`HOST_ONLY`] ([`trace_block_host_only`]). + pub host_only: bool, +} + +/// What a host-only build ([`Deviations::host_only`]) drained: its groups, +/// tables and cells, and when the last group was drained (seconds since the +/// streamed prove started). +struct Drained { + groups: usize, + tables: usize, + cells: u64, + at: f64, +} + +/// The error a host-only build stops with, once its trace is built. +pub(crate) const HOST_ONLY: &str = "host only"; + +/// A prepared stack committed wrongly, tables by AIR index. +#[derive(Clone, Copy, Debug)] +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) enum StackTamper { + /// Two tables' blocks exchanged in their (common) group's stack. + Swap(usize, usize), + /// `table`'s block holds `from`'s prepared columns instead of its own. + ColumnsOf { table: usize, from: usize }, +} + +pub(crate) fn prove_block_whir_with( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + deviations: &Deviations, +) -> Result<(BlockWhirProof, BlockStamps), Error> { + prove_block_whir_inner( + elf_bytes, + private_inputs, + proof_options, + format, + options, + deviations, + &|_, _| {}, + &|_| {}, + None, + ) +} + +/// What a caller is handed when the statement is final — after phase A, +/// before phase B: everything the recursion's programs are a function of, and +/// the prover's own prepared roots by group, in group order. +pub type StatementObserver<'a> = &'a (dyn Fn(BlockStatement<'_>, &[PreparedRoots]) + Sync); + +/// A group with a prepared stack, and its stack's roots. +pub type PreparedRoots = (usize, Vec); + +/// What a caller is handed as each group's opening ends in phase B: that +/// group's share of the proof ([`multilinear_block::GroupOpened`]), so the +/// recursion's leaves over finished groups can start before phase B ends. +pub type GroupObserver<'a> = &'a (dyn Fn(multilinear_block::GroupOpened<'_, F, E>) + Sync); + +/// [`prove_block_whir`], calling `on_statement` as soon as the statement is +/// final, so a caller can derive the recursion's programs while phase B runs. +pub fn prove_block_whir_observed( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + on_statement: StatementObserver<'_>, +) -> Result<(BlockWhirProof, BlockStamps), Error> { + prove_block_whir_observed_groups( + elf_bytes, + private_inputs, + proof_options, + format, + options, + on_statement, + &|_| {}, + ) +} + +/// [`prove_block_whir_observed`], also calling `on_group` as each group's +/// opening ends ([`GroupObserver`]). The observers read; the proof is the same. +pub fn prove_block_whir_observed_groups( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + on_statement: StatementObserver<'_>, + on_group: GroupObserver<'_>, +) -> Result<(BlockWhirProof, BlockStamps), Error> { + prove_block_whir_observed_led( + elf_bytes, + private_inputs, + proof_options, + format, + options, + on_statement, + on_group, + None, + ) +} + +/// [`prove_block_whir_observed_groups`] with phase B led by `lead`: once the +/// caller offers the recursion's leaves there ([`PhaseBLead::offer`]), phase B +/// takes the groups of the leaf it can finish soonest next. The proof is the +/// same in any order. +#[allow(clippy::too_many_arguments)] +pub fn prove_block_whir_observed_led( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + on_statement: StatementObserver<'_>, + on_group: GroupObserver<'_>, + lead: Option<&PhaseBLead>, +) -> Result<(BlockWhirProof, BlockStamps), Error> { + prove_block_whir_inner( + elf_bytes, + private_inputs, + proof_options, + format, + options, + &Deviations::default(), + on_statement, + on_group, + lead, + ) +} + +/// The recursion's leaves, offered to phase B while it runs (`W3_LEAF_FIRST`), +/// and what phase B did with them. The tree's planner offers each admitted +/// leaf's groups as soon as it has derived the plan; at its next turn phase B +/// moves the groups of the leaf it can finish soonest ([`lead_order`]) ahead of +/// the others still to come, once, so that leaf's proof can start its execute +/// long before the base ends. Prover-only: every order proves the same bytes. +#[derive(Default)] +pub struct PhaseBLead { + leaves: std::sync::OnceLock>>, + taken: std::sync::Mutex>, +} + +/// What phase B did with the offered leaves ([`PhaseBLead`]). +#[derive(Clone, Debug, PartialEq)] +pub struct LeadTaken { + /// Phase B's turn (0-based) at which it read the offer. + pub turn: usize, + /// Seconds into phase B at that turn. + pub after: f64, + /// The leaf whose groups went first, or `None`: no leaf could (each held + /// a group rebuilt in phase B) or its groups were all taken already. + pub leaf: Option, + /// The groups moved ahead of the others, in their new order. + pub moved: Vec, + /// Phase B's order as it was taken. + pub order: Vec, +} + +impl PhaseBLead { + /// Offers the leaves' groups (leaf `k`'s at `leaves[k]`); only the first + /// offer counts. Returns whether this one did. + pub fn offer(&self, leaves: Vec>) -> bool { + self.leaves.set(leaves).is_ok() + } + + /// What phase B did with the offer, once it read it. + pub fn taken(&self) -> Option { + self.taken.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// Phase B's re-ranking at turn `p` of `order` ([`multilinear_block::PhaseBReorder`]): + /// nothing until the leaves are offered, then once [`lead_order`] over the + /// groups still to come, recorded. + fn turn( + &self, + p: usize, + order: &[usize], + classes: &[Option], + cells: &[u64], + after: f64, + ) -> Option> { + let leaves = self.leaves.get()?; + let mut taken = self.taken.lock().unwrap_or_else(|e| e.into_inner()); + if taken.is_some() { + return None; + } + let rest = &order[p + 1..]; + let led = lead_order(leaves, rest, classes, cells); + let (leaf, moved, now) = match &led { + Some((leaf, moved, now)) => (Some(*leaf), moved.clone(), now.clone()), + None => (None, Vec::new(), rest.to_vec()), + }; + *taken = Some(LeadTaken { + turn: p, + after, + leaf, + moved, + order: order[..=p] + .iter() + .copied() + .chain(now.iter().copied()) + .collect(), + }); + led.map(|(_, _, now)| now).filter(|now| now != rest) + } +} + +/// Phase B's groups still to come (`rest`, in its order) led by one leaf's: +/// among `leaves` (each a list of groups), those none of whose groups in +/// `rest` is rebuilt in phase B (`classes[g]` is `Some`: a class keeps its +/// own order, and its regenerator's pace), the one with the fewest cells still +/// to take (ties to the lower leaf) and with at least one group to take. Its +/// groups in `rest` go first, in their order; the others keep theirs. Returns +/// the leaf, the groups moved and the new `rest`, or `None` when no leaf can +/// lead. The result is a permutation of `rest`. +pub(crate) fn lead_order( + leaves: &[Vec], + rest: &[usize], + classes: &[Option], + cells: &[u64], +) -> Option<(usize, Vec, Vec)> { + let to_come: std::collections::BTreeSet = rest.iter().copied().collect(); + let cell = |g: usize| cells.get(g).copied().unwrap_or(0); + let (leaf, _) = leaves + .iter() + .enumerate() + .filter_map(|(k, groups)| { + let left: Vec = groups + .iter() + .copied() + .filter(|g| to_come.contains(g)) + .collect(); + let rebuilt = left + .iter() + .any(|&g| classes.get(g).copied().flatten().is_some()); + (!rebuilt && !left.is_empty()).then(|| (k, left.iter().map(|&g| cell(g)).sum::())) + }) + .min_by_key(|&(k, cost)| (cost, k))?; + let mine: std::collections::BTreeSet = leaves[leaf].iter().copied().collect(); + let moved: Vec = rest.iter().copied().filter(|g| mine.contains(g)).collect(); + let now: Vec = moved + .iter() + .copied() + .chain(rest.iter().copied().filter(|g| !mine.contains(g))) + .collect(); + Some((leaf, moved, now)) +} + +/// The block's trace as the streamed prove builds it, each closed group +/// drained instead of committed ([`Deviations::host_only`]): a box harness's +/// measure of the host's own pace, with no card to wait for. Returns the +/// stamps and the line naming what was drained. +#[cfg(test)] +pub(crate) fn trace_block_host_only( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, +) -> Result<(BlockStamps, String), Error> { + let window_log2 = options + .window_log2 + .ok_or_else(|| Error::Prover("a host-only build is a streamed one".into()))?; + let program = Elf::load(elf_bytes).map_err(|e| Error::ElfLoad(format!("{e}")))?; + let mut stamps = BlockStamps::default(); + let deviations = Deviations { + host_only: true, + ..Default::default() + }; + match prove_streamed( + &program, + elf_bytes, + private_inputs, + 1usize << window_log2, + proof_options, + format, + options, + &deviations, + &|_, _| {}, + &|_| {}, + None, + &mut stamps, + ) { + Err(Error::Prover(line)) if line.starts_with(HOST_ONLY) => Ok((stamps, line)), + Err(e) => Err(e), + Ok(_) => Err(Error::Prover( + "a host-only build committed its groups".into(), + )), + } +} + +/// [`prove_block_whir_observed`] with a test's deviations. +#[cfg(test)] +pub(crate) fn prove_block_whir_observed_with( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + deviations: &Deviations, + on_statement: StatementObserver<'_>, +) -> Result<(BlockWhirProof, BlockStamps), Error> { + prove_block_whir_inner( + elf_bytes, + private_inputs, + proof_options, + format, + options, + deviations, + on_statement, + &|_| {}, + None, + ) +} + +#[allow(clippy::too_many_arguments)] +fn prove_block_whir_inner( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + deviations: &Deviations, + on_statement: StatementObserver<'_>, + on_group: GroupObserver<'_>, + lead: Option<&PhaseBLead>, +) -> Result<(BlockWhirProof, BlockStamps), Error> { + let mut stamps = BlockStamps::default(); + let program = Elf::load(elf_bytes).map_err(|e| Error::ElfLoad(format!("{e}")))?; + + if let Some(window_log2) = options.window_log2 { + let proof = prove_streamed( + &program, + elf_bytes, + private_inputs, + 1usize << window_log2, + proof_options, + format, + options, + deviations, + on_statement, + on_group, + lead, + &mut stamps, + )?; + return Ok((proof, stamps)); + } + + let t = Instant::now(); + let result = Executor::new(&program, private_inputs.to_vec()) + .map_err(|e| Error::Execution(format!("{e}")))? + .run() + .map_err(|e| Error::Execution(format!("{e}")))?; + stamps.execute = t.elapsed().as_secs_f64(); + + crate::tables::trace_builder::build_stamps::start(); + let t = Instant::now(); + let mut traces = + Traces::from_elf_and_logs(&program, &result.logs, &options.max_rows, private_inputs)?; + drop(result); + stamps.build = t.elapsed().as_secs_f64(); + stamps.build_marks = crate::tables::trace_builder::build_stamps::take(); + + let t = Instant::now(); + let padding = CutPadding::from_env()?; + split_keccak_rnd(&mut traces, options.keccak_rnd_rows_log2, padding)?; + split_ecdas(&mut traces, options.ecdas_rows_log2, padding); + split_keccak(&mut traces, options.keccak_rows_log2, padding); + split_ecsm(&mut traces, options.ecsm_rows_log2, padding); + split_copy(&mut traces, options.copy_rows_log2, padding); + if deviations.omit_first_keccak_rnd && !traces.keccak_rnds.is_empty() { + traces.keccak_rnds.remove(0); + } + if let Some(tamper) = deviations.tamper_traces { + tamper(&mut traces); + } + stamps.prep = t.elapsed().as_secs_f64(); + let proof = prove_traces( + &program, + elf_bytes, + &mut traces, + proof_options, + format, + options, + deviations, + true, + on_statement, + &mut stamps, + )?; + Ok((proof, stamps)) +} + +/// Cuts every KECCAK_RND table into tables of at most `2^rows_log2` rows, as +/// many as `padding` keeps ([`split_cut`]). A padding row sends +/// `ARE_BYTES[0, 0]` (`keccak_rnd::ARE_BYTES_PER_ROW` times), and BITWISE +/// counted those of the whole padded table, so the rows of the tables the cut +/// leaves out (padding rows alone) are taken back off its multiplicity. +pub(crate) fn split_keccak_rnd( + traces: &mut Traces, + rows_log2: usize, + padding: CutPadding, +) -> Result<(), Error> { + let rows = 1usize << rows_log2; + let mu = crate::tables::keccak_rnd::cols::MU; + let before: usize = traces.keccak_rnds.iter().map(|t| t.num_rows()).sum(); + traces.keccak_rnds = std::mem::take(&mut traces.keccak_rnds) + .into_iter() + .flat_map(|table| split_cut(table, rows, mu, padding)) + .collect(); + let after: usize = traces.keccak_rnds.iter().map(|t| t.num_rows()).sum(); + let left_out = before - after; + crate::tables::bitwise::take_back_multiplicity( + &mut traces.bitwise, + crate::tables::bitwise::BitwiseOperation::byte_op( + crate::tables::bitwise::BitwiseOperationType::AreBytes, + 0, + 0, + ), + (left_out * crate::tables::keccak_rnd::ARE_BYTES_PER_ROW) as u64, + ) +} + +/// Cuts every ECDAS table into tables of at most `2^rows_log2` rows, as many +/// as `padding` keeps ([`split_cut`]): a cut may fall inside one scalar +/// multiplication, whose steps chain through the Ecdas bus alone +/// ([`BLOCK_ECDAS_ROWS_LOG2`]). +pub(crate) fn split_ecdas(traces: &mut Traces, rows_log2: usize, padding: CutPadding) { + let rows = 1usize << rows_log2; + let mu = crate::tables::ecdas::cols::MU; + traces.ecdases = std::mem::take(&mut traces.ecdases) + .into_iter() + .flat_map(|table| split_cut(table, rows, mu, padding)) + .collect(); +} + +/// Cuts every KECCAK table into tables of at most `2^rows_log2` rows, as many +/// as `padding` keeps ([`split_cut`]): a row is one whole permutation call, so +/// a cut falls between calls ([`BLOCK_KECCAK_ROWS_LOG2`]). +pub(crate) fn split_keccak(traces: &mut Traces, rows_log2: usize, padding: CutPadding) { + let rows = 1usize << rows_log2; + let mu = crate::tables::keccak::cols::MU; + traces.keccaks = std::mem::take(&mut traces.keccaks) + .into_iter() + .flat_map(|table| split_cut(table, rows, mu, padding)) + .collect(); +} + +/// Cuts every COPY table into tables of at most `2^rows_log2` rows, as many as +/// `padding` keeps ([`split_cut`]): a cut may fall inside one call, whose rows +/// chain through the CopyNext bus alone ([`BLOCK_COPY_ROWS_LOG2`]). +pub(crate) fn split_copy(traces: &mut Traces, rows_log2: usize, padding: CutPadding) { + let rows = 1usize << rows_log2; + let mu = crate::tables::copy::cols::MU; + traces.copies = std::mem::take(&mut traces.copies) + .into_iter() + .flat_map(|table| split_cut(table, rows, mu, padding)) + .collect(); +} + +/// Cuts every ECSM table into tables of at most `2^rows_log2` rows, as many as +/// `padding` keeps ([`split_cut`]): a row is one whole scalar multiplication, +/// so a cut falls between calls ([`BLOCK_ECSM_ROWS_LOG2`]). +pub(crate) fn split_ecsm(traces: &mut Traces, rows_log2: usize, padding: CutPadding) { + let rows = 1usize << rows_log2; + let mu = crate::tables::ecsm::cols::MU; + traces.ecsms = std::mem::take(&mut traces.ecsms) + .into_iter() + .flat_map(|table| split_cut(table, rows, mu, padding)) + .collect(); +} + +/// The block's proof over traces already built (and the chunked tables already +/// split). `release_rows` lets each table's row-major copy go as its columns +/// are taken — what a prove does; a test proving the same traces twice keeps +/// them. Adds `prep`, `phase_a`, `phase_b` and the groups to `stamps`. +#[allow(clippy::too_many_arguments)] +pub(crate) fn prove_traces( + program: &Elf, + elf_bytes: &[u8], + traces: &mut Traces, + proof_options: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + deviations: &Deviations, + release_rows: bool, + on_statement: StatementObserver<'_>, + stamps: &mut BlockStamps, +) -> Result { + let t = Instant::now(); + let table_counts = traces.table_counts(); + validate_block_counts(&table_counts)?; + let runtime_page_ranges = traces.runtime_page_ranges(); + let num_private_input_pages = traces + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(); + let public_output = traces.public_output_bytes.clone(); + + let inputs = BlockAirInputs::new( + proof_options, + program, + crate::cpu_heights_from_traces(traces), + &public_output, + )?; + let airs = block_airs( + program, + proof_options, + &traces.page_configs, + &table_counts, + &inputs, + ); + + let prepared_columns = prepared_tables(&airs, &traces.page_configs, format)?; + let pairs = airs.air_trace_pairs(traces); + let shapes = shapes_of(&pairs)?; + let table_num_vars: Vec = shapes.iter().map(|&(_, n)| n as u8).collect(); + let config = format.chain_config(&shapes); + let sizes = block_groups(&shapes, config.format.stack, format.group_polys) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + check_group_count(sizes.len(), format.max_groups)?; + + stamps.tables = pairs.len(); + stamps.cells = shapes.iter().map(|&(w, n)| w << n).sum(); + stamps.prep += t.elapsed().as_secs_f64(); + + // The pairs, cut into the groups phase A commits: a producer lays each + // group's tables out (its tables in parallel) while the card commits the + // group before it. + let mut groups_of_pairs: Vec, (usize, usize))>> = + Vec::with_capacity(sizes.len()); + { + let mut pairs = pairs.into_iter().zip(shapes.iter().copied()); + for &size in &sizes { + groups_of_pairs.push(pairs.by_ref().take(size).collect()); + } + } + // The groups are contiguous runs in table order here. + let groups: Vec> = { + let mut at = 0u32; + sizes + .iter() + .map(|&size| { + let group = (at..at + size as u32).collect(); + at += size as u32; + group + }) + .collect() + }; + let tag = block_statement_tag(&format.zf.base); + let proof = crate::with_block_hash!(format.zf.base, |H| { + let mut transcript = ::sponge(); + absorb_block( + &mut transcript, + &tag, + elf_bytes, + &public_output, + &table_counts, + num_private_input_pages, + &runtime_page_ranges, + &table_num_vars, + &config, + &groups, + ); + let t = Instant::now(); + let check_preprocessed = !deviations.unchecked_preprocessed; + let (block, produced) = std::thread::scope(|scope| { + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let producer = scope.spawn(move || -> Result { + let mut busy = 0.0; + for group in groups_of_pairs { + let t = Instant::now(); + let tables = group + .into_par_iter() + .map(|((air, trace, _), shape)| { + table_of_checked(air, trace, shape, release_rows, check_preprocessed) + }) + .collect::, Error>>()?; + busy += t.elapsed().as_secs_f64(); + if tx.send(tables).is_err() { + break; + } + } + Ok(busy) + }); + let block = BlockCommitted::commit_streamed::< + H, + ::Sponge, + >( + rx.iter(), + &sizes, + &config, + options.drop_levels, + options.narrow, + options.upload_ahead, + ); + (block, producer.join()) + }); + // The producer's error names the cause; a commit that ran out of + // groups only says that it did. + let produced = + produced.map_err(|_| Error::Prover("the block's table producer panicked".into()))??; + let mut block = block.map_err(|e| Error::Prover(format!("{e:?}")))?; + if deviations.narrow_width_map && !block.fault_narrow_width_map() { + return Err(Error::Prover("no narrow table to break".into())); + } + stamps.prep += produced; + stamps.phase_a = t.elapsed().as_secs_f64(); + let (prepared, tampered, derive) = + prover_prepared::(prepared_columns, &groups, &config, deviations)?; + stamps.prepared = (prepared.len(), derive); + on_statement( + BlockStatement { + table_num_vars: &table_num_vars, + runtime_page_ranges: &runtime_page_ranges, + table_counts: &table_counts, + public_output: &public_output, + num_private_input_pages, + groups: &groups, + }, + &prepared_roots(&prepared), + ); + let borrowed: Vec>> = prepared + .iter() + .map(|p| multilinear::stacking::borrow(&p.columns)) + .collect(); + let openings: Vec> = prepared + .iter() + .zip(&borrowed) + .map(|(p, columns)| multilinear_block::BlockPrepared { + group: p.group, + tables: p.tables.clone(), + commitment: &p.commitment, + columns, + }) + .collect(); + let t = Instant::now(); + let paths_before = multilinear::whir_commit::top_path_counts(); + let routes_before = multilinear::whir_commit::top_path_routes(); + let widens_before = multilinear::narrow::host_widens(); + let fills_before = multilinear::records::fills(); + let times_before = multilinear::records::upload_times(); + let identity = |g: usize| g; + let fork_of: &dyn Fn(usize) -> usize = match &deviations.fork_of { + Some(f) => f, + None => &identity, + }; + if probe_first_groups()?.is_some() { + return Err(Error::Prover( + "W3_PROBE_FIRST_GROUPS orders a streamed prove's phase B only".into(), + )); + } + stamps.phase_b_overlap = options.phase_b_overlap != multilinear_block::PhaseBOverlap::Off; + stamps.phase_b_pinned = options.phase_b_overlap == multilinear_block::PhaseBOverlap::Pinned; + let phase_b_sched = phase_b_sched_from_env(); + stamps.phase_b_sched = phase_b_sched; + let readbacks_before = multilinear::gpu::background_readbacks(); + let (proof, argues, prepared_openings, groups, commit_ood) = + multilinear_block::block_prove_on_forks::<_, _, _, H>( + block + .with_phase_b_overlap(phase_b_overlap(options)) + .with_overlap_faults(deviations.overlap_faults) + .with_sched_stamps(phase_b_sched), + &config, + &mut transcript, + &openings, + &tampered, + fork_of, + &deviations.argue, + ) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + stamps.phase_b = t.elapsed().as_secs_f64(); + let readbacks_after = multilinear::gpu::background_readbacks(); + stamps.background_readbacks = ( + readbacks_after.0 - readbacks_before.0, + readbacks_after.1 - readbacks_before.1, + readbacks_after.2, + ); + let paths_after = multilinear::whir_commit::top_path_counts(); + stamps.top_paths = ( + paths_after.0 - paths_before.0, + paths_after.1 - paths_before.1, + ); + let routes_after = multilinear::whir_commit::top_path_routes(); + stamps.top_routes = ( + ( + routes_after.0.0 - routes_before.0.0, + routes_after.0.1 - routes_before.0.1, + ), + ( + routes_after.1.0 - routes_before.1.0, + routes_after.1.1 - routes_before.1.1, + ), + ); + stamps.drop_levels = Some(options.drop_levels); + let widens_after = multilinear::narrow::host_widens(); + stamps.narrow = options.narrow; + stamps.host_widens = ( + widens_after.0 - widens_before.0, + widens_after.1 - widens_before.1, + ); + let fills_after = multilinear::records::fills(); + stamps.records_fills = ( + fills_after.0 - fills_before.0, + fills_after.1 - fills_before.1, + ); + let ((widens, widen_s), (fills, fill_s)) = multilinear::records::upload_times(); + stamps.upload_times = ( + (widens - times_before.0.0, widen_s - times_before.0.1), + (fills - times_before.1.0, fill_s - times_before.1.1), + ); + stamps.groups = groups; + (proof, argues, prepared_openings, commit_ood) + }); + + Ok(BlockWhirProof { + proof: proof.0, + table_num_vars, + runtime_page_ranges, + table_counts, + public_output, + num_private_input_pages, + groups, + prepared: proof.2, + argues: proof.1, + commit_ood: proof.3, + }) +} + +/// One AIR per streamed table type, alive for the whole prove: a streamed +/// chunk is laid out (and its layout borrows its AIR) before the run's AIR set +/// exists. Every chunk of a type has the same AIR as the run's `TYPE[i]` — the +/// set builds them all with the same constructor — and the verifier checks the +/// result against its own set. +struct StreamAirs { + /// The form the streamed chunks are written in (G-pack, `tables::gpack`). + form: TraceForm, + /// One CPU AIR a streamed CPU chunk, built on first use: chunk `k` is the + /// block's CPU table `k`, every one before it full, so its timestamps start + /// at `4·k·cpu_rows` (`tables::cpu::timestamp_bases`) and its preprocessed + /// TIMESTAMP is the column of its height. A CPU AIR is per table since main's + /// preprocessed timestamp, and the argue reads the AIR a chunk was laid out + /// with. In pages of [`CPU_AIR_PAGE`], each allocated on first use, so a + /// short chunk height does not allocate a slot for every table the row + /// bound allows up front. + cpus: Vec]>>>, + /// The CPU chunk height (`MaxRowsConfig::cpu`). + cpu_rows: usize, + opts: ProofOptions, + memw_register: crate::VmAir, + memw_aligned: crate::VmAir, + memw: crate::VmAir, + load: crate::VmAir, + lt: crate::VmAir, + shift: crate::VmAir, + store: crate::VmAir, + keccak_rnd: crate::VmAir, +} + +type DynAir = dyn stark::traits::AIR; + +/// CPU AIR slots per page of [`StreamAirs`]'s CPU tables. +const CPU_AIR_PAGE: usize = 256; + +impl StreamAirs { + fn new(opts: &ProofOptions, form: TraceForm, cpu_rows: usize) -> Self { + use crate::test_utils::*; + // As many CPU tables as a run's rows allow (`tables::cpu::MAX_CPU_ROWS`). + let most = (crate::tables::cpu::MAX_CPU_ROWS as usize).div_ceil(cpu_rows.max(1)); + Self { + form, + cpus: (0..most.div_ceil(CPU_AIR_PAGE)) + .map(|_| std::sync::OnceLock::new()) + .collect(), + cpu_rows, + opts: opts.clone(), + memw_register: Box::new(create_memw_register_air(opts)), + memw_aligned: Box::new(create_memw_aligned_air(opts)), + memw: Box::new(create_memw_air(opts)), + load: Box::new(create_load_air(opts)), + lt: Box::new(create_lt_air(opts)), + shift: Box::new(create_shift_air(opts)), + store: Box::new(create_store_air(opts)), + keccak_rnd: Box::new(create_keccak_rnd_air(opts)), + } + } + + /// The AIR streamed chunk `index` of `table`, `height` rows, is laid out + /// with: the family's, or CPU table `index`'s own. + fn of_chunk(&self, table: StreamTable, index: usize, height: usize) -> Result<&DynAir, Error> { + if let Some(air) = self.of(table) { + return Ok(air); + } + let page = self.cpus.get(index / CPU_AIR_PAGE).ok_or_else(|| { + Error::Prover(format!("CPU chunk {index} is past the run's row bound")) + })?; + let page = page.get_or_init(|| { + (0..CPU_AIR_PAGE) + .map(|_| std::sync::OnceLock::new()) + .collect() + }); + Ok(page[index % CPU_AIR_PAGE] + .get_or_init(|| { + let ts_base = crate::tables::cpu::TIMESTAMP_STRIDE * (index * self.cpu_rows) as u64; + Box::new( + crate::test_utils::create_cpu_air(&self.opts, ts_base) + .with_name(&format!("CPU[{index}]")) + .with_lazy_preprocessed_columns( + crate::tables::cpu::lazy_timestamp_commitment(&self.opts, height), + crate::tables::cpu::NUM_PRECOMPUTED_COLS, + std::sync::Arc::new(move || { + vec![crate::tables::cpu::timestamp_column(height)] + }), + ), + ) + }) + .as_ref()) + } + + /// The family's AIR, for every streamed family but CPU, whose AIR is a + /// table's own ([`Self::of_chunk`]). + fn of(&self, table: StreamTable) -> Option<&DynAir> { + Some(match table { + StreamTable::Cpu => return None, + StreamTable::MemwRegister => self.memw_register.as_ref(), + StreamTable::MemwAligned => self.memw_aligned.as_ref(), + StreamTable::Memw => self.memw.as_ref(), + StreamTable::Load => self.load.as_ref(), + StreamTable::Lt => self.lt.as_ref(), + StreamTable::Shift => self.shift.as_ref(), + StreamTable::Store => self.store.as_ref(), + StreamTable::KeccakRnd => self.keccak_rnd.as_ref(), + }) + } +} + +/// The name the run's AIR set gives chunk `index` of a streamed table. +fn stream_name(table: StreamTable, index: usize) -> String { + format!("{}[{index}]", stream_base(table)) +} + +/// A streamed table's AIR name, without the chunk's index. +fn stream_base(table: StreamTable) -> &'static str { + match table { + StreamTable::Cpu => "CPU", + StreamTable::MemwRegister => "MEMW_R", + StreamTable::MemwAligned => "MEMW_A", + StreamTable::Memw => "MEMW", + StreamTable::Load => "LOAD", + StreamTable::Lt => "LT", + StreamTable::Shift => "SHIFT", + StreamTable::Store => "STORE", + StreamTable::KeccakRnd => "KECCAK_RND", + } +} + +/// What the builder thread reports when the run is built: when its windows +/// were all collected and when the run was built (seconds since the prove +/// started), how many chunks it handed out, the windows' stamps and `finish`'s +/// phase marks. +type BuilderReport = ( + f64, + f64, + usize, + WindowStamps, + Vec<(String, f64)>, + Option, +); + +/// What the builder thread hands the layout thread. +enum Built { + Job(Box), + Rest(Box), + /// The rest streamed ([`BlockOptions::finish_stream`]). + Stream(Box), +} + +/// The rest as the finish streams it: the header (what the statement reads, +/// and the run's AIRs) and, in AIR order, each table or streamed chunk's slot. +struct RestStream { + header: crate::tables::trace_builder::RestHeader, + tables: std::sync::mpsc::Receiver, +} + +/// The rest as the layout thread receives it. +enum RestIn { + Whole(Box), + Stream(Box), +} + +/// The rest's tables as they stream in, checked against the run's AIRs: each +/// the next AIR in the order the finish hands them on (AIR order, or BITWISE +/// last: [`crate::tables::trace_builder::RestHeader::bitwise_last`]), a +/// streamed chunk's slot or a table of that AIR's width, and every AIR +/// reached. A table's height is checked where it is laid +/// out ([`layout_of`]). +struct RestOrder { + widths: Vec, + /// The positions in the order the finish hands them on + /// ([`crate::tables::trace_builder::hand_on_order`]). + due: Vec, + next: usize, +} + +impl RestOrder { + /// The rest's tables of `widths` columns, due in AIR order, or with the + /// first (BITWISE) after every other when `bitwise_last`. + fn new(widths: Vec, bitwise_last: bool) -> Self { + let due = + crate::tables::trace_builder::hand_on_order(widths.len(), bitwise_last.then_some(0)); + Self { + widths, + due, + next: 0, + } + } + + fn admit(&mut self, position: usize, width: Option) -> Result<(), Error> { + let want = *self.widths.get(position).ok_or_else(|| { + Error::Prover(format!( + "the rest's table {position} is past the run's {} tables", + self.widths.len() + )) + })?; + let due = self.due.get(self.next).copied(); + if Some(position) != due { + return Err(Error::Prover(format!( + "the rest's table {position} came where table {} was due", + due.map_or("none".to_string(), |d| d.to_string()) + ))); + } + if let Some(width) = width + && width != want + { + return Err(Error::Prover(format!( + "the rest's table {position}: {width} columns, its AIR declares {want}" + ))); + } + self.next += 1; + Ok(()) + } + + fn finish(&self) -> Result<(), Error> { + if self.next != self.widths.len() { + return Err(Error::Prover(format!( + "table {} of the rest was never laid out", + self.next + ))); + } + Ok(()) + } +} + +/// A table before the run's AIR order exists: a streamed chunk, or a table of +/// the final build by its AIR index. +#[derive(Clone, Copy)] +enum Key { + Streamed(StreamTable, usize), + Air(usize), +} + +/// A table of the final build, laid out: its AIR index, shape and table. +type Placed<'a> = (usize, (usize, usize), CommittedTable<'a, F, E>); + +/// Packs tables, in the order they come, into groups of at most `max_polys` +/// stacked polynomials, and sends each group to phase A as it closes. +struct Packer<'a> { + open: Vec>, + open_shapes: Vec<(usize, usize)>, + open_keys: Vec, + group_keys: Vec>, + out: group_queue::Out>>, + cap: multilinear::whir_chain::StackVars, + max_polys: usize, + /// The most groups the statement may declare: the group past it is + /// refused as it closes. + max_groups: usize, + /// The prove's clock, when each group closed, and the seconds spent + /// waiting for phase A to take one. + start: Instant, + closed_at: Vec, + blocked: f64, + /// A memory log, and the open group's bytes in it. + ledger: Option<&'a memlog::Ledger>, + open_bytes: usize, +} + +impl<'a> Packer<'a> { + /// A group closes, and goes to the card, when the next table would take + /// it past its polynomial budget. + fn place( + &mut self, + key: Key, + shape: (usize, usize), + table: CommittedTable<'a, F, E>, + ) -> Result<(), Error> { + self.open_shapes.push(shape); + let polys = stark::multilinear_table::global_layout(&self.open_shapes, self.cap) + .map_err(|e| Error::Prover(format!("{e:?}")))? + .num_polys(); + if polys > self.max_polys && !self.open.is_empty() { + self.open_shapes.clear(); + self.open_shapes.push(shape); + self.close()?; + } + if let Some(ledger) = self.ledger { + let bytes = memlog::table_bytes(&table); + // A table of the run's AIR order comes from the rest's layout. + if let Key::Air(_) = key { + ledger + .rest_laid + .fetch_sub(bytes, std::sync::atomic::Ordering::Relaxed); + } + ledger + .open + .fetch_add(bytes, std::sync::atomic::Ordering::Relaxed); + self.open_bytes += bytes; + } + self.open.push(table); + self.open_keys.push(key); + Ok(()) + } + + fn close(&mut self) -> Result<(), Error> { + // As its groups close the prover knows only how far the block got, not + // its total: the refusal says so, so the index is not read as a count. + let reached = self.group_keys.len() + 1; + if reached > self.max_groups { + return Err(Error::InvalidTableCounts(format!( + "a block takes at most {} groups; this one reached {reached} as its groups close", + self.max_groups + ))); + } + if let Some(ledger) = self.ledger { + use std::sync::atomic::Ordering::Relaxed; + let bytes = std::mem::take(&mut self.open_bytes); + ledger.open.fetch_sub(bytes, Relaxed); + ledger.sent.fetch_add(bytes, Relaxed); + } + self.group_keys.push(std::mem::take(&mut self.open_keys)); + self.closed_at.push(self.start.elapsed().as_secs_f64()); + let t = Instant::now(); + let sent = self + .out + .send(std::mem::take(&mut self.open), |group| { + group_queue::Weight { + bytes: memlog::group_bytes(group) as u64, + cells: group + .iter() + .map(|table| (table.num_committed_columns() as u64) << table.num_vars()) + .sum(), + } + }) + .map_err(|_| Error::Prover("phase A stopped".into())); + self.blocked += t.elapsed().as_secs_f64(); + sent + } + + /// Sends the last group; hands back every group's tables, when each + /// closed and the seconds spent waiting on phase A. + fn finish(mut self) -> Result { + if !self.open.is_empty() { + self.close()?; + } + Ok((self.group_keys, self.closed_at, self.blocked)) + } +} + +/// Every group's tables, when each group closed, and the seconds the packer +/// waited on phase A ([`Packer::finish`]). +type Packed = (Vec>, Vec, f64); + +/// A streamed chunk laid out: its table, its index, its shape and the table. +type LaidChunk<'a> = (StreamTable, usize, (usize, usize), CommittedTable<'a, F, E>); + +/// One streamed chunk generated and laid out; a regeneration recorder takes +/// the digest of its packed columns, as committed. +fn lay_out_chunk<'a>( + airs: &'a StreamAirs, + job: ChunkJob, + ledger: Option<&memlog::Ledger>, + recorder: Option<®en::Recorder>, +) -> Result, Error> { + use std::sync::atomic::Ordering::Relaxed; + // A memory log: the job leaves the queue as its ops, then its trace. + let ops = ledger.map_or(0, |ledger| { + let ops = job.op_bytes(); + ledger.jobs.fetch_sub(ops, Relaxed); + ledger.laying.fetch_add(ops, Relaxed); + ops + }); + let mut chunk = job.generate_as(airs.form); + // A chunk held as its records is never dropped: it has no recipe to + // digest for. + if let (Some(recorder), Some(packed)) = + (recorder.filter(|r| r.digests()), chunk.trace.narrow_main()) + { + recorder.packed(chunk.table, chunk.index, packed); + } + let rows = ledger.map_or(0, |ledger| { + let rows = memlog::rows_bytes(&chunk.trace); + ledger.laying.fetch_add(rows, Relaxed); + ledger.laying.fetch_sub(ops, Relaxed); + rows + }); + let height = chunk.trace.main_table.height; + let shape = ( + chunk.trace.main_table.width, + height.trailing_zeros() as usize, + ); + // Held as its records: the card fills its columns as its group uploads + // it. Written packed (G-pack): laid out narrow from its packed columns, + // with no transposition. + let air = airs.of_chunk(chunk.table, chunk.index, height)?; + let table = if chunk.trace.records_main().is_some() { + table_of_records(air, &mut chunk.trace, shape) + } else if chunk.trace.narrow_main().is_some() { + table_of_narrow(air, &mut chunk.trace, shape) + } else { + table_of(air, &mut chunk.trace, shape, true) + }; + if let Some(ledger) = ledger { + ledger.laying.fetch_sub(rows, Relaxed); + } + Ok((chunk.table, chunk.index, shape, table?)) +} + +/// The streamed chunks, packed in arrival order: the packer, each chunk's +/// shape, the seconds laying them out (summed over threads) and when the last +/// was placed; and the rest's tables it packed as they were laid out, by AIR +/// index (a streamed rest's). +struct StreamLaid<'a> { + packer: Packer<'a>, + shapes: Vec<(StreamTable, usize, (usize, usize))>, + chunks: f64, + placed_at: f64, + rest: Vec<(usize, (usize, usize))>, + /// The most chunks laid out (or in the making) and not yet packed at once; + /// 0 when nothing bounded them. + ahead_most: usize, +} + +/// A table of the rest laid out, sent to the packer: its position in AIR order +/// among the rest, the seconds laying it out, and the table. +type RestDone<'a> = (usize, f64, Result, Error>); + +/// Each chunk laid out and packed on this thread as it arrives; hands back +/// the run once it is built. +fn stream_inline<'a>( + brx: std::sync::mpsc::Receiver, + airs: &'a StreamAirs, + mut packer: Packer<'a>, + recorder: Option<®en::Recorder>, +) -> Result<(StreamLaid<'a>, Box), Error> { + let mut shapes = Vec::new(); + let mut chunks = 0.0; + let ledger = packer.ledger; + for item in brx { + match item { + Built::Job(job) => { + let (seq, kind) = (shapes.len(), job.table); + let t = Instant::now(); + let (table, index, shape, laid) = lay_out_chunk(airs, *job, ledger, recorder)?; + head_log::chunk_laid(seq, stream_base(kind), t); + chunks += t.elapsed().as_secs_f64(); + shapes.push((table, index, shape)); + packer.place(Key::Streamed(table, index), shape, laid)?; + head_log::chunk_placed(seq); + } + Built::Stream(_) => { + return Err(Error::Prover( + "a streamed rest needs the pipelined layout".into(), + )); + } + Built::Rest(traces) => { + let placed_at = packer.start.elapsed().as_secs_f64(); + let ahead_most = usize::from(!shapes.is_empty()); + let streamed = StreamLaid { + packer, + shapes, + chunks, + placed_at, + rest: Vec::new(), + ahead_most, + }; + return Ok((streamed, traces)); + } + } + } + Err(Error::Prover( + "the builder stopped before the run was built".into(), + )) +} + +/// Streamed chunks laid out (or in the making) and not yet packed, at most so +/// many ([`BlockOptions::layout_ahead`]): a worker takes a permit before it +/// takes a job, so every job taken holds one and the next chunk to pack is +/// never starved; the packer gives it back once the chunk is packed. +struct Permits { + state: std::sync::Mutex, + freed: std::sync::Condvar, +} + +#[derive(Default)] +struct PermitState { + free: usize, + out: usize, + most: usize, + closed: bool, +} + +impl Permits { + fn new(n: usize) -> Self { + Self { + state: std::sync::Mutex::new(PermitState { + free: n, + ..PermitState::default() + }), + freed: std::sync::Condvar::new(), + } + } + + /// Waits for a permit; `false` once the packer has stopped. + fn take(&self) -> bool { + let Ok(mut state) = self.state.lock() else { + return false; + }; + loop { + if state.closed { + return false; + } + if state.free > 0 { + state.free -= 1; + state.out += 1; + state.most = state.most.max(state.out); + return true; + } + state = match self.freed.wait(state) { + Ok(state) => state, + Err(_) => return false, + }; + } + } + + fn give(&self) { + if let Ok(mut state) = self.state.lock() { + state.free += 1; + state.out = state.out.saturating_sub(1); + } + self.freed.notify_one(); + } + + /// Wakes every waiting worker for good. + fn close(&self) { + if let Ok(mut state) = self.state.lock() { + state.closed = true; + } + self.freed.notify_all(); + } + + fn most(&self) -> usize { + self.state.lock().map_or(0, |state| state.most) + } +} + +/// Closes the permits when the packer stops, however it stops. +struct ClosePermits<'p>(Option<&'p Permits>); + +impl Drop for ClosePermits<'_> { + fn drop(&mut self) { + if let Some(permits) = self.0 { + permits.close(); + } + } +} + +/// The chunks laid out on `workers` threads and packed on one more, in +/// arrival order all the same, so a waiting phase A holds back only the +/// packing; this thread hands them out and, once the run is built, lays out +/// the rest of it (`rest`) while the last chunks are packed. With a streamed +/// rest, `rest` sends the rest's tables to the packer as they are laid out, +/// and it packs them after the chunks, in AIR +/// order, giving each table's bytes back to `gate` as it places it (a +/// streamed rest's byte gate, stopped when the packer stops). Their channel is +/// unbounded: the rest is already in memory, or held under the gate, and a +/// full channel would park the rayon threads phase A's commits need. +#[allow(clippy::too_many_arguments)] +fn stream_pipelined<'a, R>( + brx: std::sync::mpsc::Receiver, + airs: &'a StreamAirs, + mut packer: Packer<'a>, + workers: usize, + ahead: Option, + gate: Option<&'a crate::tables::trace_builder::gate::ByteGate>, + recorder: Option<®en::Recorder>, + waits: Option<&ProducerWaits>, + rest: impl FnOnce(RestIn, Option>>) -> Result, +) -> Result<(StreamLaid<'a>, R), Error> { + type Done<'a> = (usize, f64, Result, Error>); + let permits = ahead.map(|k| Permits::new(k + 1)); + let permits = permits.as_ref(); + let ledger = packer.ledger; + std::thread::scope(|scope| { + let (jtx, jrx) = std::sync::mpsc::sync_channel::<(usize, Box)>(workers); + let jrx = std::sync::Arc::new(std::sync::Mutex::new(jrx)); + let (dtx, drx) = std::sync::mpsc::sync_channel::>(workers); + for _ in 0..workers { + let jrx = std::sync::Arc::clone(&jrx); + let dtx = dtx.clone(); + scope.spawn(move || { + loop { + let t = Instant::now(); + if let Some(permits) = permits + && !permits.take() + { + return; + } + if let Some(waits) = waits { + ProducerWaits::add(&waits.worker_permit, t); + } + // The lock is held for the receive alone. A poisoned lock + // means a sibling panicked, which the scope reports. + let t = Instant::now(); + let next = match jrx.lock() { + Ok(jobs) => jobs.recv(), + Err(_) => Err(std::sync::mpsc::RecvError), + }; + if let Some(waits) = waits { + ProducerWaits::add(&waits.worker_recv, t); + } + let Ok((seq, job)) = next else { + if let Some(permits) = permits { + permits.give(); + } + return; + }; + let (kind, t) = (job.table, Instant::now()); + let laid = lay_out_chunk(airs, *job, ledger, recorder); + head_log::chunk_laid(seq, stream_base(kind), t); + if dtx.send((seq, t.elapsed().as_secs_f64(), laid)).is_err() { + return; + } + } + }); + } + // The workers hold the only receivers and senders: once they stop, + // handing out fails and the packer's loop ends. + drop(jrx); + drop(dtx); + let (rtx, rrx) = std::sync::mpsc::channel::>(); + let placer = scope.spawn(move || -> Result, Error> { + let _close = ClosePermits(permits); + let _stop = gate.map(|gate| { + crate::tables::trace_builder::gate::StopGate(gate, "the packer stopped") + }); + let mut pending = std::collections::BTreeMap::new(); + let mut next = 0usize; + let mut shapes = Vec::new(); + let mut chunks = 0.0; + for (seq, secs, laid) in drx { + chunks += secs; + pending.insert(seq, laid); + while let Some(laid) = pending.remove(&next) { + let (table, index, shape, laid) = laid?; + shapes.push((table, index, shape)); + packer.place(Key::Streamed(table, index), shape, laid)?; + head_log::chunk_placed(next); + if let Some(permits) = permits { + permits.give(); + } + next += 1; + } + } + if !pending.is_empty() { + return Err(Error::Prover(format!( + "streamed chunk {next} was never laid out" + ))); + } + let placed_at = packer.start.elapsed().as_secs_f64(); + // The rest's tables, after every chunk, in AIR order: the first at + // the block index after the last chunk's. + if let Some(recorder) = recorder { + recorder.set_rest_base(shapes.len()); + } + let mut placed = Vec::new(); + let mut waiting = std::collections::BTreeMap::new(); + let mut next = 0usize; + for (k, _, laid) in rrx { + waiting.insert(k, laid); + while let Some(laid) = waiting.remove(&next) { + let (i, shape, table) = laid?; + placed.push((i, shape)); + packer.place(Key::Air(i), shape, table)?; + if let Some(gate) = gate { + gate.release(next); + } + next += 1; + } + } + if !waiting.is_empty() { + return Err(Error::Prover(format!( + "table {next} of the rest was never laid out" + ))); + } + Ok(StreamLaid { + packer, + shapes, + chunks, + placed_at, + rest: placed, + ahead_most: 0, + }) + }); + let mut handed = 0usize; + let mut traces = None; + for item in brx { + match item { + Built::Job(job) => { + let t = Instant::now(); + if jtx.send((handed, job)).is_err() { + break; + } + if let Some(waits) = waits { + ProducerWaits::add(&waits.layout_send, t); + } + handed += 1; + } + Built::Rest(built) => { + traces = Some(RestIn::Whole(built)); + break; + } + Built::Stream(stream) => { + traces = Some(RestIn::Stream(stream)); + break; + } + } + } + drop(jtx); + let rest = match traces { + // A streamed rest is always placed as it is laid out. + Some(traces @ RestIn::Stream(_)) => Some(rest(traces, Some(rtx))), + Some(traces) => { + drop(rtx); + Some(rest(traces, None)) + } + None => { + drop(rtx); + None + } + }; + let mut streamed = placer + .join() + .map_err(|_| Error::Prover("the block's packer panicked".into()))??; + streamed.ahead_most = permits.map_or(0, Permits::most); + if streamed.shapes.len() != handed { + return Err(Error::Prover(format!( + "{} of {handed} streamed chunks packed", + streamed.shapes.len() + ))); + } + let rest = rest.ok_or_else(|| { + Error::Prover("the builder stopped before the run was built".into()) + })??; + Ok((streamed, rest)) + }) +} + +/// The run's tables the windows did not stream, laid out in parallel, and what +/// the statement reads off the run; `marks` are seconds since the prove +/// started. `prepared` is `None` when it is left for after the packing. +struct RestLaid<'a> { + table_counts: TableCounts, + runtime_page_ranges: Vec, + num_private_input_pages: usize, + public_output: Vec, + airs: &'a VmAirs, + page_configs: Vec, + refs: Vec<&'a dyn stark::traits::AIR>, + prepared: Option>, + tables: Vec>, + marks: Vec<(&'static str, f64)>, + /// The five slowest tables to lay out and the first in AIR order, seconds. + slowest: Vec<(String, f64)>, + busy: f64, + /// [`LayoutStamps::packed_rest`]. + packed_rest: (usize, usize), + /// [`LayoutStamps::records_rest`]. + records_rest: (usize, usize), + /// Each table's AIR index and when its layout ended (seconds since the + /// prove started), in AIR order. + laid_at: Vec<(usize, f64)>, +} + +/// `prepared_now`: derive the prepared columns here, before the tables are +/// laid out; otherwise the caller derives them once the groups are sent (phase +/// A reads none of them). `sink`: send each table to the packer as it is laid +/// out, the first in AIR order first, and hand back none. +#[allow(clippy::too_many_arguments)] +fn lay_out_rest<'a>( + mut traces: Box, + program: &Elf, + opts: &ProofOptions, + format: &BlockFormat, + run_airs: &'a std::sync::OnceLock, + start: Instant, + prepared_now: bool, + sink: Option>>, + budget: Option, + ledger: Option<&memlog::Ledger>, + cpu_rows: usize, +) -> Result, Error> { + let t = Instant::now(); + let at = || start.elapsed().as_secs_f64(); + let mut marks = vec![("rest received", at())]; + let table_counts = traces.table_counts(); + validate_block_counts(&table_counts)?; + let runtime_page_ranges = traces.runtime_page_ranges(); + let num_private_input_pages = traces + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(); + let public_output = traces.public_output_bytes.clone(); + let inputs = BlockAirInputs::new( + opts, + program, + rest_cpu_heights(&traces, cpu_rows)?, + &public_output, + )?; + let airs = run_airs + .get_or_init(|| block_airs(program, opts, &traces.page_configs, &table_counts, &inputs)); + marks.push(("airs", at())); + let prepared = if prepared_now { + let prepared = prepared_tables(airs, &traces.page_configs, format)?; + marks.push(("prepared", at())); + if let Some(ledger) = ledger { + ledger.prepared.store( + prepared_bytes(&prepared), + std::sync::atomic::Ordering::Relaxed, + ); + } + Some(prepared) + } else { + None + }; + let page_configs = traces.page_configs.clone(); + let refs = airs.air_refs(); + let pairs = airs.air_trace_pairs(&mut traces); + let built: Vec<_> = pairs + .into_iter() + .zip(refs.iter().copied()) + .enumerate() + .filter(|(_, ((_, trace, _), _))| trace.main_table.width != 0) + .collect(); + // Each table's seconds and when its layout ended, for the readout. + let timed = std::sync::Mutex::new(Vec::with_capacity(built.len())); + let laid_at = std::sync::Mutex::new(Vec::with_capacity(built.len())); + let narrow_tables = std::sync::atomic::AtomicUsize::new(0); + let narrow_bytes = std::sync::atomic::AtomicUsize::new(0); + let records_tables = std::sync::atomic::AtomicUsize::new(0); + let records_bytes = std::sync::atomic::AtomicUsize::new(0); + let lay_out = |(i, ((_, trace, _), air)): (usize, ((_, &mut TraceTable, _), _))| { + let t = Instant::now(); + let shape = ( + trace.main_table.width, + trace.main_table.height.trailing_zeros() as usize, + ); + let rows = ledger.map_or(0, |_| memlog::rows_bytes(trace)); + let laid = if let Some(records) = trace.records_main() { + use std::sync::atomic::Ordering::Relaxed; + records_tables.fetch_add(1, Relaxed); + records_bytes.fetch_add(records.bytes(), Relaxed); + table_of_records(air, trace, shape) + } else if let Some(packed) = trace.narrow_main() { + use std::sync::atomic::Ordering::Relaxed; + narrow_tables.fetch_add(1, Relaxed); + narrow_bytes.fetch_add(packed.data().len(), Relaxed); + table_of_narrow(air, trace, shape) + } else { + table_of(air, trace, shape, true) + } + .map(|table| (i, shape, table)); + if let Ok(mut laid_at) = laid_at.lock() { + laid_at.push((i, start.elapsed().as_secs_f64())); + } + if let (Some(ledger), Ok((_, _, table))) = (ledger, &laid) { + use std::sync::atomic::Ordering::Relaxed; + ledger.rest.fetch_sub(rows, Relaxed); + ledger + .rest_laid + .fetch_add(memlog::table_bytes(table), Relaxed); + } + if let Ok(mut timed) = timed.lock() { + timed.push((i, t.elapsed().as_secs_f64())); + } + laid + }; + let rows_of = |(_, ((_, trace, _), _)): &(usize, ((_, &mut TraceTable, _), _))| { + memlog::rows_bytes(trace) + }; + let tables: Vec> = match (sink, budget) { + (None, None) => built + .into_par_iter() + .map(&lay_out) + .collect::>()?, + (None, Some(budget)) => { + let mut tables = Vec::with_capacity(built.len()); + for wave in waves(built, budget, rows_of) { + tables.extend( + wave.into_par_iter() + .map(&lay_out) + .collect::, Error>>()?, + ); + } + tables + } + (Some(sink), Some(budget)) => { + // Each wave's tables to the packer in AIR order once it is laid + // out. + let mut k = 0usize; + for wave in waves(built, budget, rows_of) { + let laid: Vec<(f64, Result, Error>)> = wave + .into_par_iter() + .map(|table| { + let t = Instant::now(); + let laid = lay_out(table); + (t.elapsed().as_secs_f64(), laid) + }) + .collect(); + for (secs, laid) in laid { + // A packer that stopped has its own error to report. + let _ = sink.send((k, secs, laid)); + k += 1; + } + } + Vec::new() + } + (Some(sink), None) => { + // FIFO: the tables that close the next group are laid out first. + rayon::scope_fifo(|scope| { + for (k, table) in built.into_iter().enumerate() { + let sink = sink.clone(); + let lay_out = &lay_out; + scope.spawn_fifo(move |_| { + let t = Instant::now(); + let laid = lay_out(table); + // A packer that stopped has its own error to report. + let _ = sink.send((k, t.elapsed().as_secs_f64(), laid)); + }); + } + }); + Vec::new() + } + }; + marks.push(("rest laid out", at())); + if let Some(ledger) = ledger { + // What is left of the run's tables (those no AIR proves: a padded + // table of an unused chip) goes with `traces` as this returns. + ledger.rest.store(0, std::sync::atomic::Ordering::Relaxed); + ledger.line("rest laid out"); + } + let mut timed = timed.into_inner().unwrap_or_default(); + let first = timed.iter().map(|&(i, _)| i).min(); + timed.sort_by(|a, b| b.1.total_cmp(&a.1)); + let mut slowest: Vec<(String, f64)> = timed + .iter() + .take(5) + .map(|&(i, secs)| (refs[i].name().to_string(), secs)) + .collect(); + if let Some(&(i, secs)) = first.and_then(|f| timed.iter().find(|&&(i, _)| i == f)) { + slowest.push((format!("first in AIR order: {}", refs[i].name()), secs)); + } + Ok(RestLaid { + table_counts, + runtime_page_ranges, + num_private_input_pages, + public_output, + airs, + page_configs, + refs, + prepared, + tables, + marks, + slowest, + busy: t.elapsed().as_secs_f64(), + packed_rest: (narrow_tables.into_inner(), narrow_bytes.into_inner()), + records_rest: (records_tables.into_inner(), records_bytes.into_inner()), + laid_at: { + let mut laid_at = laid_at.into_inner().unwrap_or_default(); + laid_at.sort_unstable_by_key(|&(i, _)| i); + laid_at + }, + }) +} + +/// The rest as the finish streams it ([`RestStream`]): each table laid out +/// narrow as it comes (wide ones through [`table_of`]) and sent to the packer +/// in AIR order (`sink`), every table checked against the run's AIRs built +/// from the header ([`RestOrder`]). The prepared columns are left for after +/// the packing, as on the pipelined path. +#[allow(clippy::too_many_arguments)] +fn lay_out_rest_streamed<'a>( + stream: Box, + program: &Elf, + opts: &ProofOptions, + run_airs: &'a std::sync::OnceLock, + start: Instant, + sink: Option>>, + ledger: Option<&memlog::Ledger>, + recorder: Option<®en::Recorder>, +) -> Result, Error> { + use std::sync::atomic::Ordering::Relaxed; + let sink = + sink.ok_or_else(|| Error::Prover("a streamed rest needs the packer's sink".into()))?; + let at = || start.elapsed().as_secs_f64(); + let mut marks = vec![("rest received", at())]; + let RestStream { header, tables } = *stream; + validate_block_counts(&header.table_counts)?; + let runtime_page_ranges = header.runtime_page_ranges(); + let num_private_input_pages = header + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(); + let inputs = BlockAirInputs::new( + opts, + program, + header.cpu_heights.clone(), + &header.public_output_bytes, + )?; + let airs = run_airs.get_or_init(|| { + block_airs( + program, + opts, + &header.page_configs, + &header.table_counts, + &inputs, + ) + }); + marks.push(("airs", at())); + let refs = airs.air_refs(); + let mut order = RestOrder::new( + refs.iter().map(|air| air.trace_layout().0).collect(), + header.bitwise_last, + ); + if header.bitwise_last && refs.first().map(|air| air.name()) != Some("BITWISE") { + return Err(Error::Prover( + "the rest's BITWISE goes last, but BITWISE is not the rest's first table".into(), + )); + } + let mut timed: Vec<(usize, f64)> = Vec::new(); + let mut laid_at: Vec<(usize, f64)> = Vec::new(); + let (mut narrow_tables, mut narrow_bytes) = (0usize, 0usize); + let (mut records_tables, mut records_bytes) = (0usize, 0usize); + let mut k = 0usize; + for crate::tables::trace_builder::Emitted { position, table } in tables { + order.admit(position, table.as_ref().map(|t| t.main_table.width))?; + let Some(mut trace) = table else { + continue; + }; + if k == 0 { + marks.push(("rest first received", at())); + } + let t = Instant::now(); + let shape = ( + trace.main_table.width, + trace.main_table.height.trailing_zeros() as usize, + ); + let rows = memlog::rows_bytes(&trace); + let laid = if let Some(records) = trace.records_main() { + // Never dropped, so never one phase B builds again. + records_tables += 1; + records_bytes += records.bytes(); + table_of_records(refs[position], &mut trace, shape) + } else if let Some(packed) = trace.narrow_main() { + narrow_tables += 1; + narrow_bytes += packed.data().len(); + // A table phase B can build again from the kept lists (N3): its + // place among the rest's, before the packer places it. + if let (Some(recorder), Some((family, j))) = ( + recorder, + crate::tables::trace_builder::RegenFamily::of_air(refs[position].name()), + ) { + recorder.rest_tag(k, family, j); + } + table_of_narrow(refs[position], &mut trace, shape) + } else { + table_of(refs[position], &mut trace, shape, true) + }; + let secs = t.elapsed().as_secs_f64(); + timed.push((position, secs)); + laid_at.push((position, at())); + if let (Some(ledger), Ok(table)) = (ledger, &laid) { + ledger.rest.fetch_sub(rows, Relaxed); + ledger + .rest_laid + .fetch_add(memlog::table_bytes(table), Relaxed); + } + // A packer that stopped has its own error to report. + let _ = sink.send((k, secs, laid.map(|table| (position, shape, table)))); + k += 1; + } + order.finish()?; + marks.push(("rest laid out", at())); + if let Some(ledger) = ledger { + ledger.line("rest laid out"); + } + let busy = timed.iter().map(|&(_, secs)| secs).sum(); + let first = timed.first().map(|&(i, _)| i); + let mut slowest_of = timed.clone(); + slowest_of.sort_by(|a, b| b.1.total_cmp(&a.1)); + let mut slowest: Vec<(String, f64)> = slowest_of + .iter() + .take(5) + .map(|&(i, secs)| (refs[i].name().to_string(), secs)) + .collect(); + if let Some(&(i, secs)) = first.and_then(|f| timed.iter().find(|&&(i, _)| i == f)) { + slowest.push((format!("first in AIR order: {}", refs[i].name()), secs)); + } + let crate::tables::trace_builder::RestHeader { + table_counts, + page_configs, + public_output_bytes, + .. + } = header; + Ok(RestLaid { + table_counts, + runtime_page_ranges, + num_private_input_pages, + public_output: public_output_bytes, + airs, + page_configs, + refs, + prepared: None, + tables: Vec::new(), + marks, + slowest, + busy, + packed_rest: (narrow_tables, narrow_bytes), + records_rest: (records_tables, records_bytes), + laid_at, + }) +} + +/// `items` in order, cut into consecutive waves of at most `budget` bytes +/// (`bytes` of an item); a wave holds at least one item. +pub(crate) fn waves(items: Vec, budget: usize, bytes: impl Fn(&T) -> usize) -> Vec> { + let mut waves: Vec> = Vec::new(); + let mut held = 0usize; + for item in items { + let b = bytes(&item); + match waves.last_mut() { + Some(wave) if !wave.is_empty() && held + b <= budget => { + held += b; + wave.push(item); + } + _ => { + held = b; + waves.push(vec![item]); + } + } + } + waves +} + +/// What the layout thread knows once the run is built: the statement, the AIR +/// order's shapes, and each group's tables as AIR indices. +struct Laid { + table_counts: TableCounts, + runtime_page_ranges: Vec, + num_private_input_pages: usize, + public_output: Vec, + shapes: Vec<(usize, usize)>, + groups: Vec>, + /// The prepared tables' columns ([`prepared_tables`]). + prepared: Vec, + busy: f64, + layout: LayoutStamps, + /// Each streamed chunk's table and index, and its AIR index. + streamed_air: Vec<(StreamTable, usize, usize)>, +} + +/// The block's proof with the build streamed into phase A: a builder thread +/// collects the run in windows of `window` cycles ([`WindowedTraceBuilder`]) +/// and hands each full chunk over as it completes; a layout thread lays each +/// one out and packs the tables, in arrival order, into groups of at most +/// `format.group_polys` stacked polynomials; this thread commits each group as +/// it closes. When the run is built the rest of its tables join the packing. +/// The groups are the statement's ([`BlockWhirProof::groups`]). +#[allow(clippy::too_many_arguments)] +fn prove_streamed( + program: &Elf, + elf_bytes: &[u8], + private_inputs: &[u8], + window: usize, + opts: &ProofOptions, + format: &BlockFormat, + options: &BlockOptions, + deviations: &Deviations, + on_statement: StatementObserver<'_>, + on_group: GroupObserver<'_>, + lead: Option<&PhaseBLead>, + stamps: &mut BlockStamps, +) -> Result { + // G-pack writes the streamed chunks packed when the groups are held + // narrow; they are then laid out narrow, as the finish's packed tables. + let stream_form = if options.gpack && options.narrow != multilinear_block::Narrowing::Wide { + TraceForm::Narrow + } else { + TraceForm::Wide + }; + gpack::reset_counts(); + // A new block: no memory pressure seen yet (`alloc_purge`). + crate::alloc_purge::clear_memory_pressure(); + // The tables filled from their records over the whole prove: phase A's + // commits and phase B's arguments alike. + let fills_before = multilinear::records::fills(); + let times_before = multilinear::records::upload_times(); + let stream_airs = StreamAirs::new(opts, stream_form, options.max_rows.cpu); + // Regeneration (`BlockOptions::regen`): its recorder, when phase A can feed + // it — its chunks generated packed. The disk store's knob, which used to + // turn it off, refuses the prove when set. + regen::refuse_spill_knob(std::env::var("LAMBDA_VM_BLOCK_SPILL").ok().as_deref())?; + let regen_mode = match options.regen { + Some(mode) => mode, + None => regen::regen_mode()?, + }; + // A streamed chunk held as its records parks and drops like any chunk, + // unless records are kept: then it has no recipe and no rank. + let keep_records = deviations.records_keep.map_or_else(records_keep, Ok)?; + let regen_recorder = regen::recorder(regen_mode, stream_form == TraceForm::Narrow) + .map(|recorder| { + if keep_records { + recorder.keep_records() + } else { + recorder + } + }) + .map(std::sync::Arc::new); + let recorder = regen_recorder.as_deref(); + // Live regeneration's recorder: the rest's tables it can build again are + // tagged as they are laid out (N3). + let live_recorder = recorder.filter(|_| regen_mode.live()); + // Live regeneration: phase A drops the streamed chunks the policy moves off + // the host (`always`: every one). The streamed chunks are placed first, in + // hand-out order, so the table at block index `t` is the `t`-th chunk + // handed out when there was one. + let drop_mode = block_drop()?; + let keep_cells = block_keep_cells()?; + let keep_below = block_keep_below()?; + let live_regen = regen_recorder + .as_ref() + .filter(|_| regen_mode.live()) + .map(|recorder| { + let recorder = std::sync::Arc::clone(recorder); + // Class 0: the streamed chunks (the replay); class 1: KECCAK_RND's + // and LT's tables of the rest (the kept lists, N3). + multilinear_block::BlockRegen::new( + std::sync::Arc::new(move |t| { + recorder + .chunk_at(t) + .map(|_| (0, t as u64)) + .or_else(|| recorder.rest_at(t).map(|_| (1, t as u64))) + }), + &match deviations.regen_windows { + Some((streamed, rest)) => [streamed, rest], + None => [regen::ahead_policy().initial(), regen::rest_ahead_bytes()], + }, + regen_mode == regen::RegenMode::Always, + ) + }) + .map(|regen| match drop_mode { + DropMode::Partial => regen + .partial() + .keep_cells(keep_cells) + .keep_below(keep_below), + DropMode::All => regen, + }); + if let (Some(regen), Some(pause)) = (&live_regen, deviations.streamed_drop_delay) { + regen.delay_drops(0, pause); + } + // The streamed chunks' regenerator started before phase B, when it is + // (N4c); phase B takes it, and an error before then closes its window and + // joins. + let early_spawn = deviations.regen_early.unwrap_or_else(regen::early_start); + let regen_lead = deviations.regen_lead.unwrap_or_else(regen::regen_lead); + let early_owned: std::sync::Mutex> = std::sync::Mutex::new(None); + let early_slot = &early_owned; + let (early_feed, early_recorder) = (live_regen.clone(), regen_recorder.clone()); + // Opened when phase B starts: until then an early start's windows hold + // only what the host's room allows (N4c). + let early_gate = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let early_gate_in = std::sync::Arc::clone(&early_gate); + // Set at the finish's end: until then an early start's window holds only + // its frontier, so the regenerator does not rebuild beside the finish. + let early_paced = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let run_airs: std::sync::OnceLock = std::sync::OnceLock::new(); + // Phase A's commits read the blowup, the fold schedule and the format of + // the config and nothing else; the full config (its query count reads every + // shape) is built when the shapes are known, and must agree on those. + let commit_config = format.chain_config(&[]); + let cap = commit_config.format.stack; + let start = Instant::now(); + head_log::begin(start, deviations.head_log.map_or_else(head_log::knob, Ok)?); + let card_early = block_card_early()? && cfg!(feature = "cuda"); + eprintln!( + "BLOCK CARD EARLY: {}", + if card_early { + "on (the default: the card set up on a thread of its own as the prove starts)" + } else { + "off (the card set up at its first use)" + } + ); + if card_early { + set_up_card(); + } + // A memory log (`BlockOptions::memlog`): its sampler runs until the + // function returns. + let logged = options.memlog.then(|| memlog::Ledger::new(start)); + let _sampler = logged.clone().map(memlog::Sampler::start); + let ledger = logged.as_deref(); + if let Some(ledger) = ledger { + ledger.thread("prove"); + ledger.pool(); + ledger.line("start"); + } + // Live regeneration decides at each committed table whether it leaves the + // host: the `auto` rule against the host's memory target, and a table it + // would move off the host is dropped if phase B can rebuild it, else + // stays. Without live regeneration every table is held. + eprintln!( + "{}", + regen::regen_mode_line( + regen_mode, + live_regen.is_some(), + std::env::var("LAMBDA_VM_BLOCK_REGEN").ok().as_deref(), + std::env::var("LAMBDA_VM_BLOCK_SPILL").ok().as_deref(), + ) + ); + let target = deviations.memory_target.unwrap_or_else(memory_target_bytes); + // The largest host reading the rule decided on, for the BLOCK DROPS line + // (#1013's `Spill::host_most`). + let host_most: std::sync::Arc>> = + std::sync::Arc::new(std::sync::Mutex::new(None)); + // With live regeneration the allocator's two boundary purges wait for the + // resident set to come within the regeneration window of the pressure line. + let room = no_disk_room(live_regen.is_some(), target); + crate::alloc_purge::note_no_disk(room); + // And closed groups may wait for phase A behind one another while the + // host has that room, at most a regeneration window of them + // ([`group_queue`]); by default one waits. + let queue_setting = match deviations.group_queue { + Some(setting) => setting, + None => group_queue::setting()?, + }; + let queue_rule = group_queue::Rule::of(queue_setting, room, regen::AHEAD_CAP); + let queue_stats = std::sync::Arc::new(group_queue::Stats::default()); + // The host is read at each decision: the `auto` rule decides on it and the + // purge's pressure is noted from it (`note_pressure_past`). + // Once live regeneration has armed, its phase-B reserve counts. While + // closed groups may wait behind one another, their cells count as + // committed ([`group_queue::arming_cells`]). Partial drops, once armed, + // weigh what the host holds now, and purge what they dropped. + let drop_purges = (drop_mode == DropMode::Partial && live_regen.is_some()) + .then(|| DropPurges::new(crate::alloc_purge::purge_now)); + let wanted: std::sync::Arc = { + let host_most = std::sync::Arc::clone(&host_most); + let armed = live_regen.clone(); + let purges = drop_purges.clone(); + let reserve = regen::regen_reserve_bytes(); + let per_g = memory_reserve_per_g(); + let waiting = std::sync::Arc::clone(&queue_stats); + std::sync::Arc::new(move |_kept, cells, bytes| { + let cells = group_queue::arming_cells(queue_rule, &waiting, cells); + let reading = HostReading::now(); + { + let mut most = host_most.lock().unwrap_or_else(|e| e.into_inner()); + if most.is_none_or(|m| reading.bytes() > m.bytes()) { + *most = Some(reading); + } + } + note_pressure_past(reading.bytes(), target); + let is_armed = armed.as_ref().is_some_and(|r| r.is_armed()); + let extra = if is_armed { reserve } else { 0 }; + let now = purges.is_some() && is_armed; + let allocator = now + .then(crate::alloc_purge::stats) + .flatten() + .map(|s| s.resident as u64); + let host = decision_bytes(&reading, allocator, now); + let out = drop_wanted(target, per_g, cells, bytes, || host.saturating_add(extra)); + if out && let Some(purges) = &purges { + purges.dropped(bytes); + } + out + }) + }; + let spill = live_regen + .as_ref() + .map(|regen| multilinear_block::BlockSpill::new(wanted).with_regen(regen.clone())); + + // The streamed finish's byte gate: the rest's tables generated and not yet + // placed ([`BlockOptions::finish_stream`]). + let rest_gate = crate::tables::trace_builder::gate::ByteGate::new( + options.rest_layout_bytes.unwrap_or(usize::MAX), + ) + .with_build(options.rest_build_bytes); + let rest_gate = &rest_gate; + let waits = ProducerWaits::default(); + let waits = &waits; + let tag = block_statement_tag(&format.zf.base); + crate::with_block_hash!(format.zf.base, |H| { + let walkers = block_walkers()?; + let p4_rows = block_p4_rows()?; + let p4_direct = block_p4_direct()?; + let bitwise_last = deviations.rest_bitwise_last || block_rest_bitwise_last()?; + let from_tables = match deviations.bitwise_from_tables { + Some(from_tables) => from_tables, + None => block_bitwise_from_tables()?, + }; + let card_fill = match ( + deviations.card_fill, + deviations.card_fill_streamed, + deviations.card_fill_rows, + ) { + (Some(keccak_rnd), Some([cpu, memw_r]), rows) => CardFill { + keccak_rnd, + cpu, + memw_r, + rows: rows.unwrap_or(RowFamilies::NONE), + }, + (keccak_rnd, streamed, rows) => { + let knob = block_card_fill()?; + let [cpu, memw_r] = streamed.unwrap_or([knob.cpu, knob.memw_r]); + CardFill { + keccak_rnd: keccak_rnd.unwrap_or(knob.keccak_rnd), + cpu, + memw_r, + rows: rows.unwrap_or(knob.rows), + } + } + }; + // The streamed chunks held as records ([`records_admission`]): + // G-pack's narrow chunks only, from a build that drops its streamed + // ops, with live regeneration to drop them (or kept on purpose), + // admitted while the host has room for them by the purge rule's own + // test (its resident set, the chunk and the regeneration window's + // ceiling under the pressure line) and live regeneration has not + // armed. Once a chunk is refused, every later one is filled on the + // host: what is never dropped is a prefix admitted with room. + let admission = records_admission( + card_fill.streamed(), + options.drop_streamed_ops && stream_form == TraceForm::Narrow, + keep_records, + live_regen.is_some(), + ); + let records_room = admission.is_ok().then(|| match &deviations.records_room { + Some(room) => std::sync::Arc::clone(room), + None => { + let line = no_disk_room(true, target); + let armed = live_regen.clone(); + std::sync::Arc::new(move |bytes: usize| { + !armed.as_ref().is_some_and(|regen| regen.is_armed()) + && crate::alloc_purge::has_room( + line, + crate::lfm::program_budget::resident_bytes() + .map(|resident| resident.saturating_add(bytes as u64)), + ) + }) as crate::tables::trace_builder::RecordsRoom + } + }); + eprintln!( + "BLOCK CARD FILL: families {} · streamed chunks as records: {}", + card_fill.name(), + admission.err().unwrap_or("while the host has room") + ); + let (block, built, laid, executed) = std::thread::scope(|scope| { + use std::sync::atomic::Ordering::Relaxed; + // The executor, a window at a time, two windows ahead of the walk; + // with walkers, each window with the memory at its start. + let (ltx, lrx) = std::sync::mpsc::sync_channel::<( + Vec, + Option, + )>(2); + let executor = scope.spawn(move || -> Result<(f64, ExecutorStamps), Error> { + if let Some(ledger) = ledger { + ledger.thread("executor"); + } + let moved = block_logs_move(); + let mut own = ExecutorStamps { + started: start.elapsed().as_secs_f64(), + moved, + ..ExecutorStamps::default() + }; + let t = Instant::now(); + let mut executor = Executor::new(program, private_inputs.to_vec()) + .map_err(|e| Error::Execution(format!("{e}")))?; + own.setup = t.elapsed().as_secs_f64(); + let mut at_start = (walkers > 0).then(|| executor.memory().snapshot()); + loop { + let t = Instant::now(); + let failed = |e: executor::vm::execution::ExecutorError| { + Error::Execution(format!("{e}")) + }; + // `ran`: the window executed; the copy, when the logs are + // not moved, is the hand-off's. + let (logs, ran) = if moved { + ( + executor.resume_owned(window).map_err(failed)?, + Instant::now(), + ) + } else { + match executor.resume_with_limit(window).map_err(failed)? { + Some(logs) => { + let ran = Instant::now(); + (Some(logs.to_vec()), ran) + } + None => (None, Instant::now()), + } + }; + own.run += (ran - t).as_secs_f64(); + let Some(logs) = logs else { + break; + }; + let next = (walkers > 0).then(|| executor.memory().snapshot()); + let start = std::mem::replace(&mut at_start, next); + own.windows += 1; + own.handoff += ran.elapsed().as_secs_f64(); + if let Some(ledger) = ledger { + // The executor keeps a window of logs of its own. + let bytes = memlog::logs_bytes(&logs); + ledger.logs.fetch_add(bytes, Relaxed); + ledger + .exec + .store(executor.memory().heap_bytes() + bytes, Relaxed); + } + let t = Instant::now(); + if ltx.send((logs, start)).is_err() { + break; + } + ProducerWaits::add(&waits.executor_send, t); + } + let executed = start.elapsed().as_secs_f64(); + drop(executor); + if let Some(ledger) = ledger { + ledger.exec.store(0, Relaxed); + ledger.line("executor done"); + } + Ok((executed, own)) + }); + let (btx, brx) = std::sync::mpsc::sync_channel::(64); + let finish_ledger = logged.clone(); + let walk_spill = spill.clone(); + let builder = scope.spawn(move || -> Result { + if let Some(ledger) = ledger { + ledger.thread("builder"); + } + let mut builder = + WindowedTraceBuilder::new(program, private_inputs, &options.max_rows)?; + if options.finish_keccak_rnd_chunks && options.keccak_rnd_rows_log2 >= 5 { + builder = builder + .keccak_rnd_chunks_at_finish(1usize << options.keccak_rnd_rows_log2)?; + } + if options.finish_cuts { + builder = builder.cuts_at_finish( + 1usize << options.keccak_rows_log2, + 1usize << options.ecsm_rows_log2, + 1usize << options.ecdas_rows_log2, + 1usize << options.copy_rows_log2, + )?; + } + if options.pack_finished { + builder = builder.pack_finished_tables(); + if options.gpack { + builder = builder.generate_packed(); + } + } + if let Some(on) = deviations.handout_count_thread { + builder = builder.count_handout_on_thread(on); + } + if options.drop_streamed_ops { + builder = builder.drop_streamed_ops()?; + } + if !compact_lt() { + builder = builder.raw_memw_lt()?; + } + if lt_concat() { + builder = builder.concat_lt(); + } + match p4_rows { + P4Rows::Off => {} + P4Rows::Pool => builder = builder.p4_rows_on_pool(), + P4Rows::Parts => builder = builder.p4_rows_by_parts(), + } + if p4_direct { + builder = builder.p4_lookups_direct(); + } + if p4_beside() { + builder = builder.p4_beside(); + } + if bitwise_last { + builder = builder.rest_bitwise_last(); + } + if from_tables { + builder = builder.bitwise_from_tables(); + } + if rest_slide() { + builder = builder.slide_rest(); + } + if card_fill.keccak_rnd { + builder = builder.card_fill_keccak_rnd(); + #[cfg(test)] + if deviations.card_fill_records_fault > 0 { + builder = + builder.card_fill_records_fault(deviations.card_fill_records_fault); + } + } + if let Some(room) = records_room.clone() { + builder = builder.card_fill_streamed(room, card_fill.cpu, card_fill.memw_r)?; + if card_fill.rows.any() { + builder = builder.card_fill_streamed_rows(card_fill.rows)?; + } + #[cfg(test)] + if deviations.card_fill_streamed_fault > 0 { + builder = + builder.card_fill_streamed_fault(deviations.card_fill_streamed_fault); + } + } + if let Some(ledger) = ledger { + ledger.image.store(builder.image_bytes(), Relaxed); + ledger.line("builder created"); + } + let setup = builder.setup_stamps(); + eprintln!( + "BLOCK BUILDER: ready at {:.2}s · image and memory state {:.2}s · decode table {:.2}s · DECODE's trace generated on a thread of its own", + start.elapsed().as_secs_f64(), + setup.image, + setup.decode, + ); + let mut streamed = 0usize; + // The walk on its own thread, doing nothing but walk; this + // thread appends each walked window, routes it and hands its + // chunks out as jobs (the layout thread generates them). + let last = if walkers > 0 { + // The windows walked from their entries on `walkers` + // threads, absorbed here in order as below. + // This thread's wait for the next window in order: from + // the walk's start to the first window, then between + // windows (the `accumulator recv-wait` term). + let mut idle_from = Instant::now(); + let report = parallel::walk_windows( + &mut builder, + lrx, + walkers, + |accumulator, walked, states| { + ProducerWaits::add(&waits.accumulator_recv, idle_from); + // The walked windows wait in the walk's reorder + // buffer, not on a channel: no `walked` term. + if let Some(ledger) = ledger { + ledger.walk.store(states, Relaxed); + ledger.logs.fetch_sub( + walked.cycles() * size_of::(), + Relaxed, + ); + } + let jobs = match recorder { + Some(recorder) => { + let (jobs, added) = accumulator.absorb_counted(walked); + recorder.window(added, &jobs); + jobs + } + None => accumulator.absorb(walked), + }; + for job in jobs { + streamed += 1; + if let Some(ledger) = ledger { + ledger.jobs.fetch_add(job.op_bytes(), Relaxed); + } + head_log::chunk_handed( + streamed - 1, + stream_base(job.table), + job.index, + job.op_count(), + ); + let t = Instant::now(); + if btx.send(Built::Job(Box::new(job))).is_err() { + return Err(Error::Prover("the layout thread stopped".into())); + } + ProducerWaits::add(&waits.accumulator_send, t); + } + if let Some(ledger) = ledger { + ledger.builder.store(accumulator.held_bytes(), Relaxed); + } + idle_from = Instant::now(); + Ok(()) + }, + )?; + if let Some(ledger) = ledger { + // The window states are gone; the walk's state is the + // finish's, as on one walker. + ledger.walk.store(builder.state_bytes(), Relaxed); + } + eprintln!("{}", report.line()); + report.last + } else { + let (mut walker, mut accumulator) = builder.split(); + std::thread::scope(|inner| -> Result, Error> { + let (wtx, wrx) = std::sync::mpsc::sync_channel(2); + let walking = inner.spawn(move || -> Result<_, Error> { + if let Some(ledger) = ledger { + ledger.thread("walker"); + } + // One window held back: only the run's last window + // is `finish`'s, and it is the last only once the + // executor stops. + let mut held: Option> = None; + loop { + let t = Instant::now(); + let Ok((logs, _)) = lrx.recv() else { + break; + }; + ProducerWaits::add(&waits.walker_recv, t); + if let Some(w) = held.replace(logs) { + let bytes = memlog::logs_bytes(&w); + // The logs go with the window when the + // card fills its CPU chunks from them. + let walked = walker.walk_owned(w)?; + if let Some(ledger) = ledger { + ledger.walk.store(walker.state_bytes(), Relaxed); + ledger.logs.fetch_sub(bytes, Relaxed); + ledger.walked.fetch_add(walked.heap_bytes(), Relaxed); + } + let t = Instant::now(); + if wtx.send(walked).is_err() { + break; + } + ProducerWaits::add(&waits.walker_send, t); + } + } + held.ok_or_else(|| { + Error::Execution("the run executed no cycle".to_string()) + }) + }); + loop { + let t = Instant::now(); + let Ok(walked) = wrx.recv() else { + break; + }; + ProducerWaits::add(&waits.accumulator_recv, t); + if let Some(ledger) = ledger { + ledger.walked.fetch_sub(walked.heap_bytes(), Relaxed); + } + // A recorder notes each chunk before it goes, so the + // layout's digest finds its recipe. + let jobs = match recorder { + Some(recorder) => { + let (jobs, added) = accumulator.absorb_counted(walked); + recorder.window(added, &jobs); + jobs + } + None => accumulator.absorb(walked), + }; + for job in jobs { + streamed += 1; + if let Some(ledger) = ledger { + ledger.jobs.fetch_add(job.op_bytes(), Relaxed); + } + head_log::chunk_handed( + streamed - 1, + stream_base(job.table), + job.index, + job.op_count(), + ); + let t = Instant::now(); + if btx.send(Built::Job(Box::new(job))).is_err() { + return Err(Error::Prover("the layout thread stopped".into())); + } + ProducerWaits::add(&waits.accumulator_send, t); + } + if let Some(ledger) = ledger { + ledger.builder.store(accumulator.held_bytes(), Relaxed); + } + } + walking + .join() + .map_err(|_| Error::Prover("the block's walker panicked".into()))? + })? + }; + // The hand-out's counting thread, if it ran, done counting and + // its counts the builder's: the windows end with it, as they + // end with the count on the accumulator's thread. + builder.join_handout_count()?; + let windows_done = start.elapsed().as_secs_f64(); + let window_stamps = builder.stamps(); + if let Some(ledger) = ledger { + ledger.line("windows walked"); + ledger.parts("builder", builder.heap_parts()); + ledger.largest("builder", builder.largest_parts()); + } + // The walk is done and the block's size known: when the + // finish's forecast passes the pressure share, live + // regeneration arms now, before its hump builds. + if let Some(spill) = walk_spill.as_ref().filter(|_| { + deviations.hand_off_all || hand_off_from_env() + }) { + let hwm = HostReading::now().hwm; + let cycles = (window_stamps.windows as u64).saturating_mul(window as u64); + let need = if deviations.hand_off_all { + u64::MAX + } else { + hand_off_need(hwm, cycles, target) + }; + if need > 0 { + crate::alloc_purge::note_memory_pressure(); + let armed = spill.hand_off(need); + let g = |b: u64| b as f64 / (1u64 << 30) as f64; + eprintln!( + "BLOCK HAND-OFF: at the walk's end ({windows_done:.2} s), VmHWM {:.2} GiB + \ + {:.2} for {:.1} M cycles walked is past {:.2} GiB ({}/{} of the target) by {} · \ + live regeneration armed: {}", + g(hwm), + FINISH_GROWTH_PER_CYCLE * cycles as f64 / (1u64 << 30) as f64, + cycles as f64 / 1e6, + g(pressure_line(target)), + PRESSURE_SHARE.0, + PRESSURE_SHARE.1, + if need == u64::MAX { + "everything (a test's deviation)".to_string() + } else { + format!("{:.2} GiB", g(need)) + }, + if armed { "yes" } else { "no" }, + ); + } + } + // Partial drops: the last drop-back is past, so every table's + // fate is known as phase A commits it from here, and the early + // regenerator may start. + if let Some(regen) = walk_spill.as_ref().and_then(|spill| spill.regen.as_ref()) { + regen.settle(); + } + // The table phase's marks, for `finish` alone. + crate::tables::trace_builder::build_stamps::start(); + if let Some(ledger) = finish_ledger { + crate::tables::trace_builder::build_stamps::set_hook(Some( + std::sync::Arc::new(move |label: &str| { + ledger.line(&format!("finish {label}")) + }), + )); + } + // The rest streamed: the header to the layout thread first, + // then each table as its wave is built, under the byte gate. + let streams = options.rest_streams() && !deviations.omit_first_keccak_rnd; + if streams { + let keep = live_recorder.is_some(); + // The streamed chunks' regenerator: its thread waits + // for every streamed chunk's fate and starts it then; the + // rest's starts with phase B, from its final drops. + let start_early = || { + if let (Some(feed), Some(recorder)) = (&early_feed, &early_recorder) { + let early = regen::EarlyStart::spawn( + feed.clone(), + std::sync::Arc::clone(recorder), + regen::LiveArgs { + elf_bytes: elf_bytes.to_vec(), + private_input: private_inputs.to_vec(), + max_rows: options.max_rows.clone(), + window_cycles: window, + form: stream_form, + faults: deviations.regen_faults, + policy: deviations.regen_windows.map_or_else( + regen::ahead_policy, + |(streamed, _)| regen::AheadPolicy::Fixed(streamed), + ), + }, + std::sync::Arc::clone(&early_gate_in), + std::sync::Arc::clone(&early_paced), + ); + *early_slot.lock().unwrap_or_else(|e| e.into_inner()) = Some(early); + } + }; + // Every streamed chunk is handed out and the finish begins: + // the thread goes up now, whether or not regeneration has + // armed, and takes each chunk's fate as phase A decides it. + if early_spawn == regen::EarlySpawn::Fates { + start_early(); + } + let emitted = builder.finish_streamed_keeping(&last, rest_gate, keep, |header| { + let (tx, rx) = std::sync::mpsc::channel(); + btx.send(Built::Stream(Box::new(RestStream { + header, + tables: rx, + }))) + .map_err(|_| Error::Prover("the layout thread stopped".into()))?; + Ok(Box::new( + move |emitted: crate::tables::trace_builder::Emitted| { + if let (Some(ledger), Some(table)) = (ledger, &emitted.table) { + ledger.rest.fetch_add(memlog::rows_bytes(table), Relaxed); + } + tx.send(emitted).map_err(|_| { + Error::Prover( + "the layout thread stopped taking the rest".into(), + ) + }) + }, + )) + }); + if let Some(ledger) = ledger { + crate::tables::trace_builder::build_stamps::set_hook(None); + ledger.builder.store(0, Relaxed); + ledger.walk.store(0, Relaxed); + ledger.image.store(0, Relaxed); + } + let rest_regen = emitted?; + // The finish's end. `finish`: the regenerator starts here, + // when live regeneration has armed (or drops every table). + // Either way its window may now take the host's room. + if early_spawn == regen::EarlySpawn::Finish + && early_feed.as_ref().is_some_and(|feed| feed.drops_all_from_now()) + { + start_early(); + } + early_paced.store(true, std::sync::atomic::Ordering::Release); + if deviations.abort_after_finish { + return Err(Error::Prover( + "a test's abort right after the finish".into(), + )); + } + let g = |b: usize| b as f64 / (1u64 << 30) as f64; + let (waited, waits) = rest_gate.waited(); + eprintln!( + "BLOCK REST STREAM: the rest built {} under a {:.2} GiB gate, held at most \ + {:.2} GiB (built and not yet placed) · {waits} acquisitions waited {waited:.2}s \ + for room (summed) · building at most {} at once, {:.2} GiB the most", + if rest_slide() { + "as the gate admits each table" + } else { + "in waves" + }, + g(rest_gate.budget()), + g(rest_gate.most()), + if rest_gate.build() == rest_gate.budget() { + "the gate".to_string() + } else { + format!("{:.2} GiB", g(rest_gate.build())) + }, + g(rest_gate.building_most()), + ); + if let Some(line) = crate::tables::trace_builder::take_row_scan_line() { + eprintln!("{line}"); + } + if let Some(line) = crate::tables::trace_builder::take_bitwise_line() { + eprintln!("{line}"); + } + let finish_marks = crate::tables::trace_builder::build_stamps::take(); + let finished = start.elapsed().as_secs_f64(); + if let Some(ledger) = ledger { + ledger.line("finish done"); + ledger.logs.fetch_sub(memlog::logs_bytes(&last), Relaxed); + } + return Ok(( + windows_done, + finished, + streamed, + window_stamps, + finish_marks, + rest_regen, + )); + } + let built_rest = builder.finish(&last); + if let Some(ledger) = ledger { + crate::tables::trace_builder::build_stamps::set_hook(None); + ledger.builder.store(0, Relaxed); + ledger.walk.store(0, Relaxed); + ledger.image.store(0, Relaxed); + } + let mut rest = built_rest?; + let finish_marks = crate::tables::trace_builder::build_stamps::take(); + // The builder's own rule (it read the same variable). + let padding = CutPadding::from_env()?; + split_keccak_rnd(&mut rest, options.keccak_rnd_rows_log2, padding)?; + // Cut at the finish already, and held packed: a packed table + // has no rows for `split_rows` to copy. + if !options.finish_cuts { + split_ecdas(&mut rest, options.ecdas_rows_log2, padding); + split_keccak(&mut rest, options.keccak_rows_log2, padding); + split_ecsm(&mut rest, options.ecsm_rows_log2, padding); + split_copy(&mut rest, options.copy_rows_log2, padding); + } + if deviations.omit_first_keccak_rnd && !rest.keccak_rnds.is_empty() { + rest.keccak_rnds.remove(0); + } + let finished = start.elapsed().as_secs_f64(); + if let Some(ledger) = ledger { + ledger.rest.store(rest.main_bytes(), Relaxed); + ledger.line("finish done"); + } + let _ = btx.send(Built::Rest(Box::new(rest))); + if let Some(ledger) = ledger { + // The run's last window goes as this thread ends. + ledger.logs.fetch_sub(memlog::logs_bytes(&last), Relaxed); + } + Ok(( + windows_done, + finished, + streamed, + window_stamps, + finish_marks, + None, + )) + }); + + let (gtx, grx) = std::sync::mpsc::sync_channel::>>(1); + let (group_out, queued) = match queue_rule { + None => (group_queue::Out::Channel(gtx), None), + Some(rule) => { + let (out, groups) = + group_queue::queue(rule, std::sync::Arc::clone(&queue_stats)); + (group_queue::Out::Queue(out), Some(groups)) + } + }; + let stream_airs = &stream_airs; + let run_airs = &run_airs; + let layout = scope.spawn(move || -> Result { + if let Some(ledger) = ledger { + ledger.thread("layout"); + } + let packer = Packer { + open: Vec::new(), + open_shapes: Vec::new(), + open_keys: Vec::new(), + group_keys: Vec::new(), + out: group_out, + cap, + max_polys: format.group_polys, + max_groups: format.max_groups, + start, + closed_at: Vec::new(), + blocked: 0.0, + ledger, + open_bytes: 0, + }; + // Off the inline path, the prepared columns wait until the + // groups are sent. + let inline = options.layout_workers == 0; + let rest_of = |rest: RestIn, sink| match rest { + RestIn::Whole(traces) => lay_out_rest( + traces, + program, + opts, + format, + run_airs, + start, + inline, + sink, + options.rest_layout_bytes, + ledger, + options.max_rows.cpu, + ), + RestIn::Stream(stream) => lay_out_rest_streamed( + stream, + program, + opts, + run_airs, + start, + sink, + ledger, + live_recorder, + ), + }; + let (streamed, rest) = if inline { + let (streamed, traces) = stream_inline(brx, stream_airs, packer, recorder)?; + (streamed, rest_of(RestIn::Whole(traces), None)?) + } else { + stream_pipelined( + brx, + stream_airs, + packer, + options.layout_workers, + options.layout_ahead, + Some(rest_gate), + recorder, + Some(waits), + rest_of, + )? + }; + let t = Instant::now(); + let StreamLaid { + mut packer, + shapes: streamed_shapes, + chunks, + placed_at, + rest: rest_packed, + ahead_most, + } = streamed; + let RestLaid { + table_counts, + runtime_page_ranges, + num_private_input_pages, + public_output, + airs, + page_configs, + refs, + prepared, + tables: rest_tables, + marks: rest_marks, + slowest, + busy: rest_busy, + packed_rest, + records_rest, + laid_at, + } = rest; + let names: std::collections::HashMap = refs + .iter() + .enumerate() + .map(|(i, air)| (air.name().to_string(), i)) + .collect(); + let mut shapes: Vec> = vec![None; refs.len()]; + let mut streamed_air = Vec::with_capacity(streamed_shapes.len()); + for &(table, index, shape) in &streamed_shapes { + let at = *names.get(&stream_name(table, index)).ok_or_else(|| { + Error::Prover(format!("no AIR for {}", stream_name(table, index))) + })?; + shapes[at] = Some(shape); + streamed_air.push((table, index, at)); + } + // The tables the windows did not stream, packed in AIR order + // after every streamed chunk (or already, as they were laid + // out). + for &(i, shape) in &rest_packed { + if shapes[i].replace(shape).is_some() { + return Err(Error::Prover(format!("table {i} built twice"))); + } + } + for (i, shape, table) in rest_tables { + if shapes[i].replace(shape).is_some() { + return Err(Error::Prover(format!("table {i} built twice"))); + } + packer.place(Key::Air(i), shape, table)?; + } + let (group_keys, closed_at, blocked) = packer.finish()?; + let mut marks = vec![("streamed placed", placed_at)]; + marks.extend(rest_marks); + marks.push(("groups closed", start.elapsed().as_secs_f64())); + let prepared = match prepared { + Some(prepared) => prepared, + None => { + let prepared = prepared_tables(airs, &page_configs, format)?; + marks.push(("prepared", start.elapsed().as_secs_f64())); + if let Some(ledger) = ledger { + ledger.prepared.store(prepared_bytes(&prepared), Relaxed); + } + prepared + } + }; + let shapes: Vec<(usize, usize)> = shapes + .into_iter() + .enumerate() + .map(|(i, s)| s.ok_or_else(|| Error::Prover(format!("table {i} never built")))) + .collect::>()?; + for (air, &(width, _)) in refs.iter().zip(&shapes) { + if air.trace_layout().0 != width { + return Err(Error::Prover(format!( + "{}: {width} columns, the AIR declares {}", + air.name(), + air.trace_layout().0 + ))); + } + } + let groups = group_keys + .into_iter() + .map(|keys| { + keys.into_iter() + .map(|key| match key { + Key::Air(i) => Ok(i as u32), + Key::Streamed(table, index) => names + .get(&stream_name(table, index)) + .map(|&i| i as u32) + .ok_or_else(|| { + Error::Prover(format!( + "no AIR for {}", + stream_name(table, index) + )) + }), + }) + .collect::, Error>>() + }) + .collect::, Error>>()?; + let busy = chunks + rest_busy + t.elapsed().as_secs_f64(); + let group_of = |table: usize| { + groups + .iter() + .position(|g| g.contains(&(table as u32))) + .unwrap_or(usize::MAX) + }; + let rest_tables = laid_at + .iter() + .map(|&(i, at)| (refs[i].name().to_string(), at, group_of(i))) + .collect(); + Ok(Laid { + table_counts, + runtime_page_ranges, + num_private_input_pages, + public_output, + shapes, + groups, + prepared, + busy, + layout: LayoutStamps { + workers: options.layout_workers, + marks, + closed_at, + blocked, + chunks, + slowest, + ahead_most, + packed_rest, + records_rest, + rest_tables, + gpack: (false, [0; 4]), + }, + streamed_air, + }) + }); + + // A memory log: a group leaves the channel as the committer takes + // it. + let groups = match queued { + Some(groups) => group_queue::In::Queue(groups), + None => group_queue::In::Channel(grx.iter()), + }; + let block = if deviations.host_only { + // The trace alone: each closed group counted and dropped as + // it comes, nothing committed. + let mut drained = Drained { + groups: 0, + tables: 0, + cells: 0, + at: 0.0, + }; + for group in groups { + drained.groups += 1; + drained.tables += group.len(); + drained.cells += group + .iter() + .map(|table| (table.num_committed_columns() as u64) << table.num_vars()) + .sum::(); + } + drained.at = start.elapsed().as_secs_f64(); + Err(drained) + } else { + Ok(BlockCommitted::commit_groups_logged::< + H, + ::Sponge, + >( + groups.inspect(|group| { + if let Some(ledger) = ledger { + ledger.sent.fetch_sub(memlog::group_bytes(group), Relaxed); + } + if let Some(pause) = deviations.group_take_pause { + std::thread::sleep(pause); + } + }), + &commit_config, + options.drop_levels, + options.narrow, + options.upload_ahead, + ledger.map(|ledger| ledger.block.clone()), + spill.clone(), + )) + }; + (block, builder.join(), layout.join(), executor.join()) + }); + let queue_line = group_queue::line(queue_setting, queue_rule, &queue_stats); + eprintln!("{queue_line}"); + stamps.group_queue = Some(queue_line); + stamps.head = head_log::take_lines(); + for line in &stamps.head { + eprintln!("{line}"); + } + (stamps.execute, stamps.executor) = + executed.map_err(|_| Error::Prover("the block's executor panicked".into()))??; + // A thread's error names the cause; a commit that ran out of groups only + // says that it did. + let (windows_done, finished, streamed, window_stamps, finish_marks, rest_regen) = + built.map_err(|_| Error::Prover("the block's builder panicked".into()))??; + stamps.windows = window_stamps; + stamps.build_marks = finish_marks; + stamps.producer = waits.stamps(); + let laid = + laid.map_err(|_| Error::Prover("the block's layout thread panicked".into()))??; + let block = match block { + Ok(block) => block, + Err(drained) => { + stamps.build = finished; + stamps.streamed = (windows_done, streamed); + stamps.prep = laid.busy; + stamps.layout = laid.layout.clone(); + stamps.phase_a = start.elapsed().as_secs_f64(); + return Err(Error::Prover(format!( + "{HOST_ONLY}: the trace built and drained at {:.2}s · {} groups · {} tables · {} \ + cells · nothing committed", + drained.at, drained.groups, drained.tables, drained.cells + ))); + } + }; + let mut block = block.map_err(|e| Error::Prover(format!("{e:?}")))?; + if deviations.narrow_width_map && !block.fault_narrow_width_map() { + return Err(Error::Prover("no narrow table to break".into())); + } + stamps.build = finished; + stamps.streamed = (windows_done, streamed); + stamps.prep = laid.busy; + stamps.layout = laid.layout.clone(); + stamps.layout.gpack = (options.gpack, gpack::counts()); + stamps.phase_a = start.elapsed().as_secs_f64(); + if let Some(ledger) = ledger { + ledger.line("phase A end"); + } + // Phase A's freed pages back to the OS before phase B allocates: when + // the block's memory is short (`note_pressure_past`, read once more + // here), or as `LAMBDA_VM_ALLOC_PURGE` names it. + note_pressure_past(HostReading::now().bytes(), target); + crate::alloc_purge::purge_point("phase-a"); + // The shadow regenerator starts here, beside the rest of the prove; it + // drops nothing, and any exit of the prove stops and joins it. + let shadow_recipes = regen_recorder + .as_ref() + .filter(|r| r.digests()) + .map(|r| r.recipes()); + let shadow = shadow_recipes.as_ref().map(|(recipes, _)| { + regen::ShadowRun::spawn( + elf_bytes.to_vec(), + private_inputs.to_vec(), + options.max_rows.clone(), + window, + recipes.clone(), + stream_form, + deviations.regen_drop_one_op, + ) + }); + // Live regeneration: what phase A dropped, rebuilt beside phase B in + // rank order. A dropped table is the chunk its rank names (the chunks + // are placed first, in hand-out order), or the prove stops here. + // The streamed chunks' regenerator started at the finish's end, when it + // did (N4c). + let early = early_owned.lock().unwrap_or_else(|e| e.into_inner()).take(); + let early_spawned = early.as_ref().map(regen::EarlyStart::started); + let (early_runs, early_pacer) = match early.map(regen::EarlyStart::join) { + Some((runs, pacer)) => (Some(runs), pacer), + None => (None, None), + }; + // Its head start is its regenerator's, started once the fates were + // known; a thread that started none (nothing dropped) gave none. + let early_started = early_runs.as_ref().and_then(|e| e.live_started); + let early_streamed_waited = early_runs.as_ref().map(|e| e.streamed_waited); + let mut rest_regen = rest_regen; + let live = match (block.regen().cloned(), ®en_recorder) { + (Some(live_regen), Some(recorder)) => { + let drops = live_regen.report(); + let windows = live_regen.windows().to_vec(); + let window_of = std::sync::Arc::clone(&windows[0]); + let mut plans = live_regen.into_plan().into_iter(); + let streamed_plan = plans.next(); + let rest_plan = plans.next(); + // The rest's dropped tables (N3), built again from the lists the + // finish kept; with none dropped, the lists go now. + let rest_run = match rest_plan { + Some(plan) if !plan.dropped.is_empty() => { + let regen = rest_regen.take().ok_or_else(|| { + Error::Prover( + "rest tables were dropped but the finish kept no lists".into(), + ) + })?; + let producer = plan.producer.ok_or_else(|| { + Error::Prover("the rest's window has no producer".into()) + })?; + Some(regen::RestRun::spawn( + regen, + recorder.rest_blocks(), + std::sync::Arc::clone(&plan.window), + producer, + plan.dropped, + regen::rest_generators(), + deviations.rest_faults, + )) + } + _ => None, + }; + drop(rest_regen.take()); + let run = match early_runs { + Some(early) => { + if let Some(why) = early.error { + return Err(Error::Prover(format!("early regeneration: {why}"))); + } + // It rebuilds the streamed chunks phase A dropped, each + // the chunk its rank names. + let planned: Vec = + streamed_plan.as_ref().map_or_else(Vec::new, |plan| { + plan.dropped.iter().map(|(r, _)| *r).collect() + }); + let started: Vec = early.keys.iter().map(|(r, _)| *r).collect(); + if planned != started { + return Err(Error::Prover(format!( + "the early regenerator took {} streamed chunks, phase A dropped {}", + started.len(), + planned.len() + ))); + } + for (rank, key) in &early.keys { + if !laid + .streamed_air + .get(*rank as usize) + .is_some_and(|&(t, i, _)| (t, i) == *key) + { + return Err(Error::Prover(format!( + "dropped table {rank} is not the chunk its rank names" + ))); + } + } + early.live + } + None => match streamed_plan + .filter(|plan| !plan.dropped.is_empty()) + .and_then(|plan| plan.producer.map(|producer| (plan.dropped, producer))) + { + Some((dropped, producer)) => { + let mut keyed = Vec::with_capacity(dropped.len()); + for (rank, slot) in dropped { + let key = recorder + .chunk_at(rank as usize) + .filter(|key| { + laid.streamed_air + .get(rank as usize) + .is_some_and(|&(t, i, _)| (t, i) == *key) + }) + .ok_or_else(|| { + Error::Prover(format!( + "dropped table {rank} is not the chunk its rank names" + )) + })?; + keyed.push((key, slot)); + } + Some(regen::LiveRun::spawn( + elf_bytes.to_vec(), + private_inputs.to_vec(), + options.max_rows.clone(), + window, + std::sync::Arc::clone(&window_of), + producer, + keyed, + stream_form, + deviations.regen_faults, + deviations + .regen_windows + .map_or_else(regen::ahead_policy, |(streamed, _)| { + regen::AheadPolicy::Fixed(streamed) + }), + None, + )) + } + None => None, + }, + }; + Some((drops, window_of, run, windows.get(1).cloned(), rest_run)) + } + _ => None, + }; + drop(rest_regen); + // Each group's rebuilt class, which phase B's order follows (below); + // all `None`, and the group order, when nothing was dropped. + let classes = block.rebuilt_classes().map_err(|g| { + Error::Prover(format!( + "group {g} dropped tables of two classes for phase B to rebuild" + )) + })?; + stamps.tables = laid.shapes.len(); + stamps.cells = laid.shapes.iter().map(|&(w, n)| w << n).sum(); + + let config = format.chain_config(&laid.shapes); + if config.log_blowup != commit_config.log_blowup + || config.log_folding != commit_config.log_folding + || config.format != commit_config.format + { + return Err(Error::Prover( + "phase A committed under a config the block's shapes do not give".into(), + )); + } + let table_num_vars: Vec = laid.shapes.iter().map(|&(_, n)| n as u8).collect(); + let mut transcript = ::sponge(); + absorb_block( + &mut transcript, + &tag, + elf_bytes, + &laid.public_output, + &laid.table_counts, + laid.num_private_input_pages, + &laid.runtime_page_ranges, + &table_num_vars, + &config, + &laid.groups, + ); + let (prepared, tampered, derive) = + prover_prepared::(laid.prepared, &laid.groups, &config, deviations)?; + stamps.prepared = (prepared.len(), derive); + if let Some(ledger) = ledger { + ledger.line("prepared committed"); + } + on_statement( + BlockStatement { + table_num_vars: &table_num_vars, + runtime_page_ranges: &laid.runtime_page_ranges, + table_counts: &laid.table_counts, + public_output: &laid.public_output, + num_private_input_pages: laid.num_private_input_pages, + groups: &laid.groups, + }, + &prepared_roots(&prepared), + ); + let borrowed: Vec>> = prepared + .iter() + .map(|p| multilinear::stacking::borrow(&p.columns)) + .collect(); + let openings: Vec> = prepared + .iter() + .zip(&borrowed) + .map(|(p, columns)| multilinear_block::BlockPrepared { + group: p.group, + tables: p.tables.clone(), + commitment: &p.commitment, + columns, + }) + .collect(); + let _ = multilinear::whir_split::take_chain(); + let top_secs_before = multilinear::whir_commit::top_path_secs(); + let t = Instant::now(); + let paths_before = multilinear::whir_commit::top_path_counts(); + let routes_before = multilinear::whir_commit::top_path_routes(); + let widens_before = multilinear::narrow::host_widens(); + let identity = |g: usize| g; + let fork_of: &dyn Fn(usize) -> usize = match &deviations.fork_of { + Some(f) => f, + None => &identity, + }; + // Phase B's order (N3): the groups with nothing dropped first, then the + // rebuilt streamed and rest groups interleaved by their cells, so each + // regenerator keeps up with only its share of phase B's pace. + let group_cells = group_cells(&laid.shapes, &laid.groups); + let order: Vec = match deviations.phase_b_order { + Some(order) => order(laid.groups.len()), + None => led_order(&classes, &group_cells, regen_lead), + }; + // `W3_PROBE_FIRST_GROUPS`, a measurement's: these groups first. + let probe_first = probe_first_groups()?; + // A measurement's or a test's order wins over the recursion's lead. + let lead = lead.filter(|_| deviations.phase_b_order.is_none() && probe_first.is_none()); + let order = match probe_first { + None => order, + Some(_) if classes.iter().any(Option::is_some) => { + return Err(Error::Prover( + "W3_PROBE_FIRST_GROUPS: a group is rebuilt in phase B, whose classes go in \ + group order" + .into(), + )); + } + Some(first) => put_first(&first, order).map_err(Error::Prover)?, + }; + // Each class's regenerator deposits in its rank order, its groups' + // order: an order that takes a class's groups otherwise could wait on a + // deposit its window does not admit yet, so it is refused rather than + // run. + for class in 0..2usize { + let taken: Vec = order + .iter() + .copied() + .filter(|&g| classes.get(g).copied().flatten() == Some(class)) + .collect(); + if taken.windows(2).any(|w| w[0] > w[1]) { + return Err(Error::Prover( + "phase B must take each class's rebuilt groups in group order".into(), + )); + } + } + // Phase B starts: an early start's windows go to their phase-B sizes. + early_gate.store(true, std::sync::atomic::Ordering::Release); + let phase_b_start = Instant::now(); + let phase_b_unix = stark::prove_split::epoch_secs(); + stamps.phase_b_order = order.clone(); + stamps.phase_b_overlap = options.phase_b_overlap != multilinear_block::PhaseBOverlap::Off; + stamps.phase_b_pinned = options.phase_b_overlap == multilinear_block::PhaseBOverlap::Pinned; + let phase_b_sched = phase_b_sched_from_env(); + stamps.phase_b_sched = phase_b_sched; + let readbacks_before = multilinear::gpu::background_readbacks(); + // The recursion's lead (`W3_LEAF_FIRST`): once the tree offers its + // leaves, the groups of the one phase B can finish soonest go next. + let lead_turn = |p: usize, order: &[usize]| { + lead.and_then(|lead| { + lead.turn( + p, + order, + &classes, + &group_cells, + phase_b_start.elapsed().as_secs_f64(), + ) + }) + }; + let reorder: Option<&multilinear_block::PhaseBReorder<'_>> = + lead.is_some().then_some(&lead_turn); + let (proof, argues, prepared_openings, groups, commit_ood) = + multilinear_block::block_prove_in_order::<_, _, _, H>( + block + .with_phase_b_overlap(phase_b_overlap(options)) + .with_overlap_faults(deviations.overlap_faults) + .with_sched_stamps(phase_b_sched), + &config, + &mut transcript, + &openings, + &tampered, + fork_of, + &deviations.argue, + on_group, + &order, + reorder, + ) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + stamps.phase_b = t.elapsed().as_secs_f64(); + // The order as phase B took it, when the lead moved groups. + let order = match lead.and_then(PhaseBLead::taken) { + Some(taken) if taken.order.len() == order.len() => taken.order, + _ => order, + }; + stamps.phase_b_order = order.clone(); + stamps.phase_b_unix = Some((phase_b_unix, stark::prove_split::epoch_secs())); + let readbacks_after = multilinear::gpu::background_readbacks(); + stamps.background_readbacks = ( + readbacks_after.0 - readbacks_before.0, + readbacks_after.1 - readbacks_before.1, + readbacks_after.2, + ); + if let Some((drops, window_of, run, rest_window, rest_run)) = live { + let recipes = recorder.map_or(0, |r| r.recipes().0.len()); + let mut lines = vec![regen::dropped_line(regen_mode, &drops, recipes)]; + if let Some(purges) = &drop_purges { + lines.push(regen::keep_line(&drops)); + lines.push(purges.line()); + } + let rest_report = rest_run.map(regen::RestRun::join); + let offset = run.as_ref().map(|run| { + phase_b_start + .saturating_duration_since(run.started()) + .as_secs_f64() + }); + let report = run.map(regen::LiveRun::join); + if let Some(report) = &report { + lines.push(report.line()); + lines.extend(report.pacer.clone()); + for failure in report.failures.iter().take(20) { + lines.push(format!("BLOCK REGEN FAILED {failure}")); + } + } + let head_start = + early_started.map(|t| phase_b_start.saturating_duration_since(t).as_secs_f64()); + let early_pace = early_pacer.and_then(|pacer| pacer.join().ok()); + lines.push(regen::head_start_line( + head_start, + early_spawned.map(|t| phase_b_start.saturating_duration_since(t).as_secs_f64()), + early_streamed_waited, + early_pace, + )); + lines.push(format!("BLOCK REGEN window: {}", window_of.report())); + if let Some(report) = &rest_report { + lines.push(report.line()); + for failure in report.failures.iter().take(20) { + lines.push(format!("BLOCK REGEN REST FAILED {failure}")); + } + } + if let (Some(window), Some((tables, bytes))) = (&rest_window, drops.by_class.get(1)) + && *tables > 0 + { + lines.push(format!( + "BLOCK REGEN rest window: {} · {tables} tables {:.2} GiB dropped", + window.report(), + *bytes as f64 / (1u64 << 30) as f64 + )); + } + let in_class = |c: usize| classes.iter().filter(|&&k| k == Some(c)).count(); + lines.push(format!( + "BLOCK REGEN phase B order: {} of {} groups rebuilt ({} streamed, {} rest), \ + interleaved after the others · phase B began {} after the regenerator · {} \ + streamed groups ahead of the rest's first", + classes.iter().filter(|c| c.is_some()).count(), + classes.len(), + in_class(0), + in_class(1), + offset.map_or("n/a".to_string(), |s| format!("{s:.2} s")), + order + .iter() + .filter(|&&g| classes.get(g).copied().flatten().is_some()) + .take_while(|&&g| classes[g] == Some(0)) + .count(), + )); + let spans: Vec<(f64, f64)> = groups.iter().map(|g| (g.start_b, g.end_b)).collect(); + lines.push(regen::order_taken_line( + &order, + &classes, + &group_cells, + &spans, + )); + for line in &lines { + eprintln!("{line}"); + } + stamps.regen = Some(regen::RegenStamps { + recipes, + dropped: drops.tables, + dropped_bytes: drops.bytes, + armed_kept: drops.armed_kept.0, + regenerated: report.as_ref().map_or(0, |r| r.deposited) + + rest_report.as_ref().map_or(0, |r| r.deposited), + mismatches: report.as_ref().map_or(0, |r| r.mismatches) + + rest_report.as_ref().map_or(0, |r| r.mismatches), + failed: report.as_ref().map_or(0, |r| r.failures.len() + r.skipped) + + rest_report.as_ref().map_or(0, |r| r.failures.len()), + error: report + .as_ref() + .and_then(|r| r.error.clone()) + .or_else(|| rest_report.as_ref().and_then(|r| r.error.clone())), + rest_dropped: drops.by_class.get(1).map_or(0, |c| c.0), + rest_regenerated: rest_report.as_ref().map_or(0, |r| r.deposited), + held: drops.held, + phase_b_order: order.clone(), + head_start, + lines, + ..regen::RegenStamps::default() + }); + } + if let (Some(shadow), Some((recipes, stray))) = (shadow, &shadow_recipes) { + let prove_offset = phase_b_start + .saturating_duration_since(shadow.started()) + .as_secs_f64(); + let waited = Instant::now(); + let report = shadow.join(); + let joined = waited.elapsed().as_secs_f64(); + let group_of_air: std::collections::HashMap = laid + .groups + .iter() + .enumerate() + .flat_map(|(g, tables)| tables.iter().map(move |&t| (t, g))) + .collect(); + let air_of: std::collections::HashMap<(StreamTable, usize), usize> = laid + .streamed_air + .iter() + .map(|&(table, index, air)| ((table, index), air)) + .collect(); + let streamed_cells: u64 = laid + .streamed_air + .iter() + .map(|&(_, _, air)| { + let (w, n) = laid.shapes[air]; + (w as u64) << n + }) + .sum(); + let spans: Vec<(f64, f64)> = groups.iter().map(|g| (g.start_b, g.end_b)).collect(); + let readout = regen::shadow_readout( + recipes, + *stray, + &report, + &|table, index| { + air_of + .get(&(table, index)) + .and_then(|&air| group_of_air.get(&(air as u32)).copied()) + }, + laid.streamed_air.len(), + streamed_cells, + &spans, + stamps.phase_b, + prove_offset, + joined, + ); + for line in &readout.lines { + eprintln!("{line}"); + } + stamps.regen = Some(readout); + } + if spill.is_some() { + let g = |b: u64| b as f64 / (1u64 << 30) as f64; + let host = match *host_most.lock().unwrap_or_else(|e| e.into_inner()) { + Some(h) => format!(" · host at most {:.2} GiB", g(h.bytes())), + None => String::new(), + }; + stamps.drops = Some(format!( + "live regeneration decides as `auto` (target {:.1} GiB){host}", + g(target) + )); + } + if multilinear::whir_split::enabled() { + let chain = multilinear::whir_split::take_chain(); + let top_secs = multilinear::whir_commit::top_path_secs(); + let names = [ + "grind", + "sumcheck", + "fold", + "commit_folded", + "ood", + "queries", + ]; + stamps.open_split = names.into_iter().zip(chain.six).collect(); + stamps.open_split.push(("round_wall", chain.round_wall)); + let q = [ + "query_sample", + "tree_rebuild", + "coset_gather", + "open_assemble", + ]; + stamps.open_split.extend(q.into_iter().zip(chain.queries)); + stamps + .open_split + .push(("top_gather", top_secs.0 - top_secs_before.0)); + stamps + .open_split + .push(("top_rehash", top_secs.1 - top_secs_before.1)); + stamps.open_split.push(("chains", chain.chain_count as f64)); + stamps.open_split.push(("rounds", chain.round_count as f64)); + } + let paths_after = multilinear::whir_commit::top_path_counts(); + stamps.top_paths = ( + paths_after.0 - paths_before.0, + paths_after.1 - paths_before.1, + ); + let routes_after = multilinear::whir_commit::top_path_routes(); + stamps.top_routes = ( + ( + routes_after.0.0 - routes_before.0.0, + routes_after.0.1 - routes_before.0.1, + ), + ( + routes_after.1.0 - routes_before.1.0, + routes_after.1.1 - routes_before.1.1, + ), + ); + stamps.drop_levels = Some(options.drop_levels); + let widens_after = multilinear::narrow::host_widens(); + stamps.narrow = options.narrow; + stamps.host_widens = ( + widens_after.0 - widens_before.0, + widens_after.1 - widens_before.1, + ); + let fills_after = multilinear::records::fills(); + stamps.records_fills = ( + fills_after.0 - fills_before.0, + fills_after.1 - fills_before.1, + ); + let ((widens, widen_s), (fills, fill_s)) = multilinear::records::upload_times(); + stamps.upload_times = ( + (widens - times_before.0.0, widen_s - times_before.0.1), + (fills - times_before.1.0, fill_s - times_before.1.1), + ); + stamps.groups = groups; + if let Some(ledger) = ledger { + ledger.line("proof done"); + ledger.report(); + stamps.mem_terms = ledger.terms(); + } + Ok(BlockWhirProof { + proof, + table_num_vars, + runtime_page_ranges: laid.runtime_page_ranges, + table_counts: laid.table_counts, + public_output: laid.public_output, + num_private_input_pages: laid.num_private_input_pages, + groups: laid.groups, + prepared: prepared_openings, + argues, + commit_ood, + }) + }) +} + +/// The bytes [`absorb_block`] appends, in its order and with its pad: what an +/// in-guest verifier absorbs as one run of program constants. One stream, so a +/// test can hold the two to the same transcript. +pub(crate) fn block_statement_bytes( + statement: BlockStatement<'_>, + tag: &[u8], + elf_digest: &[u8; 32], + config: &multilinear::whir_chain::ChainConfig, +) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(tag); + bytes.extend_from_slice(elf_digest); + bytes.extend_from_slice(&(statement.public_output.len() as u64).to_le_bytes()); + bytes.extend_from_slice(statement.public_output); + for count in statement::table_count_values(statement.table_counts) { + bytes.extend_from_slice(&count.to_le_bytes()); + } + bytes.extend_from_slice(&(statement.num_private_input_pages as u64).to_le_bytes()); + bytes.extend_from_slice(&(statement.runtime_page_ranges.len() as u64).to_le_bytes()); + for range in statement.runtime_page_ranges { + bytes.extend_from_slice(&range.base.to_le_bytes()); + bytes.extend_from_slice(&range.count.to_le_bytes()); + } + bytes.extend_from_slice(&(statement.table_num_vars.len() as u64).to_le_bytes()); + bytes.extend_from_slice(statement.table_num_vars); + for value in [ + config.log_blowup as u64, + config.fold_word(), + config.num_queries as u64, + ] { + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes.extend_from_slice(&[config.grind.folding, config.grind.ood, config.grind.query]); + bytes.resize(bytes.len() + statement::statement_padding(bytes.len()), 0); + bytes.extend_from_slice(&(statement.groups.len() as u64).to_le_bytes()); + for group in statement.groups { + bytes.extend_from_slice(&(group.len() as u64).to_le_bytes()); + for &index in group { + bytes.extend_from_slice(&u64::from(index).to_le_bytes()); + } + } + bytes +} + +/// The `append_bytes` calls [`absorb_block`] makes, in its order: what an +/// in-guest verifier of a Poseidon1 base absorbs call by call, because that +/// sponge binds each call's length (`P1Transcript::append_bytes`). Recorded +/// from [`absorb_block`] itself, so the two cannot drift; their concatenation +/// is [`block_statement_bytes`]. Refuses a statement whose absorb is not only +/// byte strings, which the replay could not follow. +pub(crate) fn block_statement_calls( + statement: BlockStatement<'_>, + tag: &[u8], + elf_bytes: &[u8], + config: &multilinear::whir_chain::ChainConfig, +) -> Result>, String> { + let mut calls = StatementCalls::default(); + absorb_block( + &mut calls, + tag, + elf_bytes, + statement.public_output, + statement.table_counts, + statement.num_private_input_pages, + statement.runtime_page_ranges, + statement.table_num_vars, + config, + statement.groups, + ); + if calls.foreign { + return Err("the block statement absorbs more than byte strings".to_string()); + } + Ok(calls.bytes) +} + +/// A transcript that records `append_bytes` calls and nothing else +/// ([`block_statement_calls`]); any other call marks the record unusable. +#[derive(Default)] +struct StatementCalls { + bytes: Vec>, + foreign: bool, +} + +impl IsTranscript for StatementCalls { + fn append_field_element(&mut self, _element: &FieldElement) { + self.foreign = true; + } + + fn append_bytes(&mut self, new_bytes: &[u8]) { + self.bytes.push(new_bytes.to_vec()); + } + + fn state(&self) -> [u8; 32] { + [0; 32] + } + + fn sample_field_element(&mut self) -> FieldElement { + self.foreign = true; + FieldElement::zero() + } + + fn sample_u64(&mut self, _upper_bound: u64) -> u64 { + self.foreign = true; + 0 + } +} + +/// The statement's fields: everything of a [`BlockWhirProof`] the verifier +/// reads before any root, which is everything but its argument. +#[derive(Clone, Copy, Debug)] +pub struct BlockStatement<'a> { + pub table_num_vars: &'a [u8], + pub runtime_page_ranges: &'a [RuntimePageRange], + pub table_counts: &'a TableCounts, + pub public_output: &'a [u8], + pub num_private_input_pages: usize, + pub groups: &'a [Vec], +} + +/// A [`BlockStatement`] owned: what a statement observer keeps +/// ([`prove_block_whir_observed`]). +#[derive(Clone, Debug)] +pub struct OwnedBlockStatement { + pub table_num_vars: Vec, + pub runtime_page_ranges: Vec, + pub table_counts: TableCounts, + pub public_output: Vec, + pub num_private_input_pages: usize, + pub groups: Vec>, +} + +impl OwnedBlockStatement { + pub fn view(&self) -> BlockStatement<'_> { + BlockStatement { + table_num_vars: &self.table_num_vars, + runtime_page_ranges: &self.runtime_page_ranges, + table_counts: &self.table_counts, + public_output: &self.public_output, + num_private_input_pages: self.num_private_input_pages, + groups: &self.groups, + } + } +} + +impl BlockStatement<'_> { + pub fn to_owned(&self) -> OwnedBlockStatement { + OwnedBlockStatement { + table_num_vars: self.table_num_vars.to_vec(), + runtime_page_ranges: self.runtime_page_ranges.to_vec(), + table_counts: self.table_counts.clone(), + public_output: self.public_output.to_vec(), + num_private_input_pages: self.num_private_input_pages, + groups: self.groups.to_vec(), + } + } +} + +impl BlockWhirProof { + /// The proof's statement. + pub fn statement(&self) -> BlockStatement<'_> { + BlockStatement { + table_num_vars: &self.table_num_vars, + runtime_page_ranges: &self.runtime_page_ranges, + table_counts: &self.table_counts, + public_output: &self.public_output, + num_private_input_pages: self.num_private_input_pages, + groups: &self.groups, + } + } +} + +/// What a verifier derives from the program and a statement before any root, +/// once the statement's checks pass. The host verifier and the recursion's tree +/// plan both start from it, so they check one statement the same way. +pub(crate) struct BlockFrame { + pub(crate) page_configs: Vec, + pub(crate) airs: VmAirs, + /// Per table in [`VmAirs::air_refs`] order: the AIR's width and the stated + /// height in variables. + pub(crate) shapes: Vec<(usize, usize)>, + pub(crate) config: multilinear::whir_chain::ChainConfig, + /// The proof's table order: AIR indices, the groups concatenated. + pub(crate) order: Vec, + /// Tables per group. + pub(crate) sizes: Vec, + /// Each group's stack and domain, from the shapes and the verifier's stack + /// cap. + pub(crate) stack_layouts: Vec, + pub(crate) domains: Vec>, +} + +/// The statement's checks, and what they leave: the counts within bounds, the +/// page layout the ELF and the ranges imply, one height per table the counts +/// imply, each height under its cap, the AIR set, the groups an exact partition within the group maximum +/// and each within the stack budget. +pub(crate) fn block_frame( + statement: BlockStatement<'_>, + elf_bytes: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, +) -> Result { + let program = Elf::load(elf_bytes).map_err(|e| Error::ElfLoad(format!("{e}")))?; + + validate_block_counts(statement.table_counts)?; + let max_pages = crate::tables::page::max_private_input_pages(); + if statement.num_private_input_pages > max_pages { + return Err(Error::InvalidTableCounts(format!( + "num_private_input_pages ({}) exceeds max ({max_pages})", + statement.num_private_input_pages, + ))); + } + let num_tables = statement.table_num_vars.len(); + let page_configs = Traces::page_configs_from_elf_and_runtime( + &program, + statement.runtime_page_ranges, + statement.num_private_input_pages, + num_tables, + )?; + let Some(expected) = statement + .table_counts + .total() + .and_then(|t| t.checked_add(FIXED_TABLE_COUNT)) + .and_then(|t| t.checked_add(page_configs.len())) + else { + return Err(Error::InvalidTableCounts( + "the table counts do not sum to a usize".to_string(), + )); + }; + if expected != num_tables { + return Err(Error::InvalidTableCounts(format!( + "the statement implies {expected} tables and states {num_tables} heights", + ))); + } + check_table_heights(statement.table_num_vars)?; + check_bitwise_height(statement.table_num_vars)?; + check_chunked_heights(statement.table_counts, statement.table_num_vars)?; + + let cpu_heights = block_cpu_heights(statement.table_counts, statement.table_num_vars)?; + let inputs = BlockAirInputs::new( + proof_options, + &program, + cpu_heights, + statement.public_output, + )?; + let airs = block_airs( + &program, + proof_options, + &page_configs, + statement.table_counts, + &inputs, + ); + // The CPU heights were read at `cpu_proof_start`: the tables must be there. + airs.check_cpu_position(statement.table_counts)?; + let air_refs = airs.air_refs(); + if air_refs.len() != num_tables { + return Err(Error::InvalidTableCounts(format!( + "the layout has {} tables, the statement {num_tables}", + air_refs.len(), + ))); + } + // The width is the AIR's, never the proof's; only the height is stated. + let shapes: Vec<(usize, usize)> = air_refs + .iter() + .zip(statement.table_num_vars) + .map(|(air, &num_vars)| (air.trace_layout().0, num_vars as usize)) + .collect(); + let config = format.chain_config(&shapes); + drop(air_refs); + + // The groups are the statement's; their stacks are built here, from the + // shapes and the verifier's stack cap. + check_group_count(statement.groups.len(), format.max_groups)?; + let order = validate_groups(statement.groups, shapes.len())?; + let sizes: Vec = statement.groups.iter().map(Vec::len).collect(); + let group_shapes: Vec<(usize, usize)> = order.iter().map(|&i| shapes[i]).collect(); + let (stack_layouts, domains) = stacks(&group_shapes, &sizes, &config)?; + // Every group within the stack budget — the verifier's constant, not the + // prover's packing — except a table that alone needs more: it cannot be + // split, so it is a group of its own (the packing's rule too). + if let Some((g, layout)) = stack_layouts + .iter() + .enumerate() + .find(|(g, layout)| layout.num_polys() > format.group_polys && sizes[*g] > 1) + { + return Err(Error::InvalidTableCounts(format!( + "group {g} stacks into {} polynomials — a group takes at most {}", + layout.num_polys(), + format.group_polys + ))); + } + Ok(BlockFrame { + page_configs, + airs, + shapes, + config, + order, + sizes, + stack_layouts, + domains, + }) +} + +/// Verifies a proof from [`prove_block_whir`] under `format` — the verifier's +/// own constants, never the proof's. +pub fn verify_block_whir( + proof: &BlockWhirProof, + elf_bytes: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, +) -> Result { + verify_block_whir_with( + proof, + elf_bytes, + proof_options, + format, + false, + VerifierChecks::ALL, + ) +} + +/// [`verify_block_whir`] with the prepared openings left unchecked when +/// `skip_prepared`, and the batched argue's checks and the bus balance switched +/// by `argue_checks` — mutations, for the tests that show each check is what refuses +/// its forgery. +pub(crate) fn verify_block_whir_with( + proof: &BlockWhirProof, + elf_bytes: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + skip_prepared: bool, + argue_checks: VerifierChecks, +) -> Result { + verify_block_whir_traced( + proof, + elf_bytes, + proof_options, + format, + skip_prepared, + argue_checks, + ) + .map(|verdict| verdict.is_ok()) +} + +/// Why [`verify_block_whir_traced`] refused a proof whose statement frames: the +/// derived prepared stacks did not commit, the COMMIT bus's target does not +/// exist, or the multilinear proof's own check refused (with its error). +/// +/// The error is read by the host-build tests only (a card build compiles none). +#[derive(Debug)] +#[allow(dead_code)] +pub(crate) enum BlockRefusal { + Prepared, + Owed, + Proof(multilinear::Error), +} + +/// [`verify_block_whir_with`], a refusal carrying the check that refused: what +/// a negative test asserts, so it cannot pass on another error. +pub(crate) fn verify_block_whir_traced( + proof: &BlockWhirProof, + elf_bytes: &[u8], + proof_options: &ProofOptions, + format: &BlockFormat, + skip_prepared: bool, + argue_checks: VerifierChecks, +) -> Result, Error> { + // One argument per table, or one per group, by the verifier's format; the + // other format's messages absent. + let (tables, argues) = match format.argue { + ArgueFormat::PerTable => (proof.table_num_vars.len(), 0), + ArgueFormat::Batched { .. } => (0, proof.groups.len()), + }; + if proof.proof.tables.len() != tables || proof.argues.len() != argues { + return Err(Error::InvalidTableCounts(format!( + "the proof carries {} table arguments and {} batched argues; its format \ + ({:?}) takes {tables} and {argues}", + proof.proof.tables.len(), + proof.argues.len(), + format.argue, + ))); + } + let frame = block_frame(proof.statement(), elf_bytes, proof_options, format)?; + let BlockFrame { + airs, + page_configs, + shapes, + config, + order, + sizes, + stack_layouts, + domains, + .. + } = &frame; + let air_refs = airs.air_refs(); + let layouts: Vec> = air_refs + .iter() + .zip(shapes) + .map(|(air, &(width, num_vars))| { + layout_of(*air, width, num_vars) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name()))) + }) + .collect::>()?; + let preprocessed: Vec>> = air_refs + .iter() + .map(|air| preprocessed_mles(*air)) + .collect::>()?; + let statements: Vec> = layouts + .iter() + .zip(&preprocessed) + .map(|(layout, cols)| layout.statement_with_preprocessed(cols)) + .collect(); + // The proof's tables are in group order, so the statements are taken in + // that order too. + let statements: Vec> = order.iter().map(|&i| statements[i]).collect(); + let prepared_columns = prepared_tables(airs, page_configs, format)?; + + let tag = block_statement_tag(&format.zf.base); + Ok(crate::with_block_hash!(format.zf.base, |H| { + let mut transcript = ::sponge(); + absorb_block( + &mut transcript, + &tag, + elf_bytes, + &proof.public_output, + &proof.table_counts, + proof.num_private_input_pages, + &proof.runtime_page_ranges, + &proof.table_num_vars, + config, + &proof.groups, + ); + // The prepared commitments, derived here from the program: their roots + // are the verifier's, never the proof's. + let prepared: Vec> = match group_stacks(prepared_columns, &proof.groups) + .and_then(|stacks| commit_groups::(stacks, config)) + { + Ok(prepared) => prepared, + Err(_) => return Ok(Err(BlockRefusal::Prepared)), + }; + let checks: Vec> = prepared + .iter() + .map(|p| multilinear_block::BlockPreparedCheck { + group: p.group, + tables: p.tables.clone(), + roots: &p.roots, + layout: p.commitment.layout(), + domain: p.commitment.domain(), + }) + .collect(); + let derived: Vec = checks + .iter() + .flat_map(|c| c.roots.iter().copied()) + .collect(); + // What the tables owe: the COMMIT bus's counterparty, at the block's + // challenges — replayed on a fork through the roots block itself, + // answers and all. + let mut probe = transcript.clone(); + multilinear_block::absorb_roots_and_answers::( + &mut probe, + &proof.proof.roots, + &derived, + &proof.commit_ood, + ); + let z: FieldElement = probe.sample_field_element(); + let alpha: FieldElement = probe.sample_field_element(); + let Some(owed) = crate::compute_commit_bus_offset(&proof.public_output, 0, &z, &alpha) + else { + return Ok(Err(BlockRefusal::Owed)); + }; + multilinear_block::block_verify_with::<_, _, _, H>( + &proof.proof, + &proof.commit_ood, + &proof.argues, + &proof.prepared, + &checks, + &statements, + stack_layouts, + domains, + sizes, + &owed, + config, + &mut transcript, + skip_prepared, + argue_checks, + ) + .map_err(BlockRefusal::Proof) + })) +} + +#[cfg(test)] +mod memory_target_tests { + use super::{ + CgroupValue, HostReading, MEMORY_RESERVE_PER_G, cgroup_memory, drop_wanted, hand_off_need, + memory_reserve_bytes, memory_short, memory_target_from, no_disk_room, parse_hand_off, + parse_reserve_per_g, parse_target_gib, parse_tree_drop, pressure_line, renamed_knob, + }; + use crate::alloc_purge::{NoDisk, has_room}; + + const GIB: u64 = 1 << 30; + + /// The memory target's knob and the reserve's are read under their new + /// names, and under the names they had with the disk store while the new + /// ones are unset; the target's value is GiB above 0, else the machine's. + #[test] + fn the_renamed_knobs_still_read_their_old_names() { + let env = |pairs: &'static [(&'static str, &'static str)]| { + move |name: &str| { + pairs + .iter() + .find(|(n, _)| *n == name) + .map(|(_, v)| v.to_string()) + } + }; + let (new, old) = ( + "LAMBDA_VM_BLOCK_MEMORY_TARGET_GIB", + "LAMBDA_VM_BLOCK_SPILL_TARGET_GIB", + ); + assert_eq!(renamed_knob(env(&[]), new, old), None); + assert_eq!( + renamed_knob( + env(&[("LAMBDA_VM_BLOCK_SPILL_TARGET_GIB", "108.28")]), + new, + old + ), + Some("108.28".to_string()) + ); + assert_eq!( + renamed_knob( + env(&[ + ("LAMBDA_VM_BLOCK_MEMORY_TARGET_GIB", "54"), + ("LAMBDA_VM_BLOCK_SPILL_TARGET_GIB", "108.28"), + ]), + new, + old + ), + Some("54".to_string()), + "the new name wins" + ); + assert_eq!( + parse_target_gib(Some(" 2.5 ".to_string())), + Some(5 * GIB / 2) + ); + for bad in [ + None, + Some("0"), + Some("-1"), + Some("nan"), + Some("inf"), + Some("x"), + ] { + assert_eq!(parse_target_gib(bad.map(str::to_string)), None, "{bad:?}"); + } + } + + /// The cgroup memory files, v2 and v1, from fake `/proc/self/cgroup` texts + /// and cgroup trees: v2 at the process's path, v2's `max` falling through + /// to v1, v1 at its path or (a container without a cgroup namespace) at + /// the controller's root, a v1 controller sharing its hierarchy, and none + /// (#1013's test @ 035aef5d6, in a directory of its own). + #[test] + fn the_cgroup_memory_files_read_v2_then_v1() { + let root = std::env::temp_dir().join(format!("im4b-whir-cgroup-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let put = |rel: &str, file: &str, value: &str| { + let dir = root.join(rel); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(file), value).unwrap(); + }; + let read = |proc_cgroup: &str, v2, v1| cgroup_memory(proc_cgroup, &root, v2, v1); + let limit = |proc_cgroup: &str| { + read( + proc_cgroup, + CgroupValue::File("memory.max"), + CgroupValue::File("memory.limit_in_bytes"), + ) + }; + let charge = |proc_cgroup: &str| { + read( + proc_cgroup, + CgroupValue::File("memory.current"), + CgroupValue::File("memory.usage_in_bytes"), + ) + }; + let inactive = |proc_cgroup: &str| { + read( + proc_cgroup, + CgroupValue::Stat("inactive_file"), + CgroupValue::Stat("total_inactive_file"), + ) + }; + // v2 at the process's path, its charge and its inactive file pages, + // read by their exact key (a decoy ending in the same name comes + // first). + put("a/b", "memory.max", "129584070656\n"); + put("a/b", "memory.current", "4096\n"); + put( + "a/b", + "memory.stat", + "anon 100\nfile 9000\nx_inactive_file 1\nactive_file 10\ninactive_file 8192\n", + ); + assert_eq!(limit("0::/a/b\n"), Some(129_584_070_656)); + assert_eq!(charge("0::/a/b\n"), Some(4096)); + assert_eq!(inactive("0::/a/b\n"), Some(8192)); + // v2 unlimited and no v1: no limit. + put("c", "memory.max", "max\n"); + assert_eq!(limit("0::/c\n"), None); + // A hybrid host, as FAST: the unified line names a path with no + // memory files, and the memory controller's own cgroup is its root. + // v1's inactive file pages are its hierarchical key, not the local + // one that ends in the same name. + put("memory", "memory.limit_in_bytes", "61774757888\n"); + put("memory", "memory.usage_in_bytes", "17855025152\n"); + put( + "memory", + "memory.stat", + "cache 16000\ninactive_file 5\ntotal_cache 16000\ntotal_inactive_file 15180000000\n", + ); + let fast = "12:memory:/docker/d005\n9:cpu,cpuacct:/docker/d005\n0::/docker/d005\n"; + assert_eq!(limit(fast), Some(61_774_757_888)); + assert_eq!(charge(fast), Some(17_855_025_152)); + assert_eq!(inactive(fast), Some(15_180_000_000)); + // v1 at the process's path wins over the controller's root, and a + // controller sharing its hierarchy is found. + put("memory/docker/d005", "memory.limit_in_bytes", "777\n"); + assert_eq!(limit(fast), Some(777)); + put("memory/p", "memory.limit_in_bytes", "888\n"); + assert_eq!(limit("4:cpu,memory:/p\n"), Some(888)); + // v2's `max` falls through to v1. + assert_eq!(limit("4:memory:/p\n0::/c\n"), Some(888)); + // Nothing readable. + assert_eq!(limit(""), None); + assert_eq!(limit("5:pids:/x\n"), None); + assert_eq!(inactive("0::/c\n"), None); + let _ = std::fs::remove_dir_all(&root); + } + + /// `auto`'s host bytes: the larger of the peak resident set and the + /// cgroup's working set (its charge less its inactive file pages). FAST at + /// 17:28Z: 40.58 GB charged with 15.18 GB inactive file reads 25.40 GB, not + /// 40.58; inactive pages past the charge read zero; the peak resident set + /// wins when it is larger (#1013's test @ 035aef5d6). + #[test] + fn auto_reads_the_hosts_working_set() { + let reading = |hwm, charged, inactive_file| HostReading { + hwm, + charged, + inactive_file, + }; + assert_eq!( + reading(24_680_000_000, 40_580_000_000, 15_180_000_000).bytes(), + 25_400_000_000 + ); + assert_eq!(reading(0, 4 * GIB, 6 * GIB).bytes(), 0); + assert_eq!(reading(30 * GIB, 40 * GIB, 15 * GIB).bytes(), 30 * GIB); + assert_eq!(reading(0, 120 * GIB, 0).bytes(), 120 * GIB); + } + + /// The target is the smaller of the cgroup limit and `MemTotal`, less + /// 10 GiB: v1's unlimited sentinel gives `MemTotal`'s, a limit above + /// `MemTotal` is capped, and with neither there is no target. + #[test] + fn the_memory_target_is_the_smaller_limit_less_ten_gib() { + let total = 62_840_956u64 << 10; + assert_eq!( + memory_target_from(Some(61_774_757_888), Some(total)), + 61_774_757_888 - 10 * GIB + ); + assert_eq!( + memory_target_from(Some(9_223_372_036_854_771_712), Some(total)), + total - 10 * GIB + ); + assert_eq!( + memory_target_from(Some(200 * GIB), Some(125 * GIB)), + 115 * GIB + ); + assert_eq!(memory_target_from(None, Some(64 * GIB)), 54 * GIB); + assert_eq!( + memory_target_from(Some(129_584_070_656), None), + 129_584_070_656 - 10 * GIB + ); + assert_eq!(memory_target_from(Some(8 * GIB), Some(16 * GIB)), 0); + assert_eq!(memory_target_from(None, None), u64::MAX); + } + + /// The `auto` rule wants a table out only once the host, its reserve and + /// the table would pass the target. + #[test] + fn the_auto_rule_wants_a_table_out_only_past_the_target() { + let target = 110 * GIB; + let cells = 10_000_000_000; // 10 G cells: a reserve of 6 + 16.5 GiB + assert_eq!( + memory_reserve_bytes(cells, MEMORY_RESERVE_PER_G), + (22.5 * GIB as f64) as u64 + ); + // The knob's retune: 0.5 GiB per G cells is 6 + 5 GiB at 10 G cells; + // unset or not a number ≥ 0 is today's 1.65. + assert_eq!(memory_reserve_bytes(cells, 0.5), 11 * GIB); + assert_eq!(parse_reserve_per_g(Some(" 0.5 ")), 0.5); + assert_eq!(parse_reserve_per_g(Some("0")), 0.0); + for bad in [None, Some("-1"), Some("nan"), Some("inf"), Some("x")] { + assert_eq!(parse_reserve_per_g(bad), MEMORY_RESERVE_PER_G, "{bad:?}"); + } + let wanted = + |target: u64, per_g: f64, host: u64| drop_wanted(target, per_g, cells, GIB, || host); + assert!(!wanted(target, MEMORY_RESERVE_PER_G, 86 * GIB)); + assert!(wanted(target, MEMORY_RESERVE_PER_G, 87 * GIB)); + assert!(!wanted(u64::MAX, MEMORY_RESERVE_PER_G, u64::MAX)); + // At 0.5 GiB per G cells the same table waits until 98 GiB is passed. + assert!(!wanted(target, 0.5, 98 * GIB)); + assert!(wanted(target, 0.5, 99 * GIB)); + } + + /// The purge's memory pressure on #1014: the host past 85 % of the memory + /// target, a share of whatever host it runs on. On BIG (target 110.69 GiB, + /// share 94.09) the median's highest reading (BIG 653, 86.64 GiB) is not + /// short and p90's and full gas's (BIG 660, 120.05 and 105.55) are; on a + /// 64 GiB host (target 54) the share is 45.9; with no limit known, never. + #[test] + fn memory_is_short_past_85_percent_of_the_target() { + let big = 129_584_070_656 - 10 * GIB; + let share = big / 20 * 17; + assert!(!memory_short(share, big)); + assert!(memory_short(share + 1, big)); + let gib = |g: f64| (g * GIB as f64) as u64; + assert!(!memory_short(gib(86.64), big), "the median never purges"); + assert!(memory_short(gib(120.05), big), "p90 purges"); + assert!(memory_short(gib(105.55), big), "full gas purges"); + let small = memory_target_from(None, Some(64 * GIB)); + assert!(!memory_short(gib(45.8), small)); + assert!(memory_short(gib(46.0), small)); + assert!(!memory_short(1024 * GIB, u64::MAX)); + } + + /// What a block with live regeneration gives the allocator's two boundary + /// purges to weigh: + /// the pressure line of its target, 85 % of it and not the target, and the + /// regeneration window's 16 GiB. On ZEN5 (target 108.28 GiB) the line is + /// 92.03 and the p90 block has room at both boundaries (VmRSS 66.75 and + /// 68.17 GiB, ZEN5 096); on a 64 GiB host (target 54) the line is 45.90 and + /// neither p90 (36.08, 45.60) nor the median (33.13, 33.16) has room: under + /// the target itself the median would have. + #[test] + fn a_no_disk_block_weighs_its_window_against_the_pressure_line() { + let gib = |g: f64| (g * GIB as f64) as u64; + let zen5 = 126_997_233_664 - 10 * GIB; + let large = no_disk_room(true, zen5); + assert_eq!( + large, + Some(NoDisk { + line: pressure_line(zen5), + headroom: 16 * GIB + }) + ); + assert_eq!(pressure_line(zen5), zen5 / 20 * 17); + assert!(memory_short(pressure_line(zen5) + 1, zen5)); + assert!(!memory_short(pressure_line(zen5), zen5)); + for resident in [66.75, 68.17] { + assert!(has_room(large, Some(gib(resident))), "ZEN5 {resident}"); + } + let small = no_disk_room(true, 54 * GIB); + assert_eq!(small.map(|n| n.line), Some(54 * GIB / 20 * 17)); + assert!(pressure_line(54 * GIB).abs_diff(gib(45.9)) < 1 << 20); + for resident in [36.08, 45.60, 33.13, 33.16] { + assert!(!has_room(small, Some(gib(resident))), "64 GiB {resident}"); + } + } + + /// A block without live regeneration gives the purges nothing to weigh: + /// they run whenever its memory is short, whatever its resident set. + #[test] + fn a_block_without_live_regeneration_gives_the_purges_no_line() { + assert_eq!(no_disk_room(false, 54 * GIB), None); + assert!(!has_room(no_disk_room(false, 54 * GIB), Some(GIB))); + } + + /// The hand-off at the walk's end, against the rows its coefficient comes + /// from (I-WFULL §4.1; BIG, target 110.69 GiB, line 94.09): p90 (660 D, + /// VmHWM 85.19 at the walk's end, 584 windows of 2^20 cycles) hands back + /// ≈ 21.2 GiB, which its forecast puts at the line; full gas (86.81, 575) + /// ≈ 22.3; the median (653: 66.74, 285), p90 with nothing kept (660 S: + /// 48.57) and 1× hand back nothing; with no limit known, never. On a + /// 64 GiB host (target 54, line 45.9) a 40 GiB median walk does. + #[test] + fn the_hand_off_forecast_reads_660s_rows() { + let big = 129_584_070_656 - 10 * GIB; + let gib = |g: f64| (g * GIB as f64) as u64; + let as_gib = |b: u64| b as f64 / GIB as f64; + let windows = |n: u64| n << 20; + let p90 = as_gib(hand_off_need(gib(85.19), windows(584), big)); + assert!((20.7..21.7).contains(&p90), "p90 hands back {p90:.2} GiB"); + let full = as_gib(hand_off_need(gib(86.81), windows(575), big)); + assert!( + (21.8..22.8).contains(&full), + "full gas hands back {full:.2} GiB" + ); + assert_eq!( + hand_off_need(gib(66.74), windows(285), big), + 0, + "the median" + ); + assert_eq!( + hand_off_need(gib(48.57), windows(584), big), + 0, + "p90, nothing kept" + ); + assert_eq!(hand_off_need(gib(30.0), windows(30), big), 0, "1×"); + assert_eq!(hand_off_need(gib(85.19), windows(584), u64::MAX), 0); + let small = memory_target_from(None, Some(64 * GIB)); + assert!(hand_off_need(gib(40.0), windows(285), small) > 0); + } + + /// `LAMBDA_VM_BLOCK_TREE_DROP_LEVELS`: unset is 8 on the card and 4 on the + /// host; `` or `,` sets them; anything else is the + /// default. + #[test] + fn the_tree_drop_knob_reads_card_then_host() { + use multilinear::whir_commit::TreeDrop; + let default = TreeDrop { device: 8, host: 4 }; + assert_eq!(parse_tree_drop(None), default); + assert_eq!(parse_tree_drop(Some("")), default); + assert_eq!(parse_tree_drop(Some("6")), TreeDrop { device: 6, host: 4 }); + assert_eq!( + parse_tree_drop(Some(" 6 , 5 ")), + TreeDrop { device: 6, host: 5 } + ); + assert_eq!(parse_tree_drop(Some("4,4")), TreeDrop::uniform(4)); + assert_eq!(parse_tree_drop(Some("x")), default); + assert_eq!(parse_tree_drop(Some("6,5,1")), default); + } + + /// `LAMBDA_VM_BLOCK_HAND_OFF`: unset, `auto` or anything else decides at + /// the walk's end; `off` never hands off. + #[test] + fn the_hand_off_knob_reads_off_only() { + assert!(parse_hand_off(None)); + assert!(parse_hand_off(Some("auto"))); + assert!(parse_hand_off(Some(" on "))); + assert!(!parse_hand_off(Some("off"))); + assert!(!parse_hand_off(Some(" off "))); + } +} + +/// `W3_PROBE_FIRST_GROUPS=g,g,…`, a measurement knob unset in production: +/// phase B takes these groups first, in this order, then the others in its own. +/// Every order proves the same bytes ([`multilinear_block::block_prove_in_order`]). +/// A word that is not a list of group indices is an error. +fn probe_first_groups() -> Result>, Error> { + let Ok(value) = std::env::var("W3_PROBE_FIRST_GROUPS") else { + return Ok(None); + }; + value + .split(',') + .map(|g| g.trim().parse::()) + .collect::, _>>() + .map(Some) + .map_err(|_| { + Error::Prover(format!( + "W3_PROBE_FIRST_GROUPS={value}: group indices, comma-separated" + )) + }) +} + +/// `first`, then the rest of `order` in its order. Refuses a group `order` +/// does not hold or one named twice, so the result is a permutation of `order`. +fn put_first(first: &[usize], order: Vec) -> Result, String> { + let mut taken = vec![false; order.len()]; + for &g in first { + match taken.get_mut(g) { + None => { + return Err(format!( + "W3_PROBE_FIRST_GROUPS: no group {g} of {}", + order.len() + )); + } + Some(t) if *t => return Err(format!("W3_PROBE_FIRST_GROUPS: group {g} named twice")), + Some(t) => *t = true, + } + } + Ok(first + .iter() + .copied() + .chain(order.into_iter().filter(|&g| !taken[g])) + .collect()) +} + +/// Phase B's order when live regeneration rebuilds tables: the groups it +/// rebuilds nothing of first, then the first `lead` groups of the streamed +/// chunks' class (class 0), then the others of both classes interleaved by +/// their cells ([`multilinear_block::interleaved_order`]). Each class's groups +/// stay in group order. The rest's regenerator (class 1) starts with phase B +/// and its first group's tables take it about a second and a half, while the +/// streamed chunks' has started before; taken second, the rest's first group +/// waits for them ([`regen::regen_lead`]). With `lead` 0, or no group of the +/// rest's class, it is the interleave's order. +fn led_order(classes: &[Option], cells: &[u64], lead: usize) -> Vec { + let rest = classes.contains(&Some(1)); + let led: Vec = (0..classes.len()) + .filter(|&g| classes[g] == Some(0)) + .take(if rest { lead } else { 0 }) + .collect(); + if led.is_empty() { + return multilinear_block::interleaved_order(classes, cells); + } + // The interleave puts every group with no class first, in group order: + // with the led groups' class masked, those are the groups phase B rebuilds + // nothing of and the led ones. The first kind goes first. + let mut masked = classes.to_vec(); + for &g in &led { + masked[g] = None; + } + let order = multilinear_block::interleaved_order(&masked, cells); + let front = masked + .iter() + .filter(|c| c.is_none()) + .count() + .min(order.len()); + let (unclassed, interleaved) = order.split_at(front); + unclassed + .iter() + .copied() + .filter(|&g| classes[g].is_none()) + .chain(led.iter().copied()) + .chain(interleaved.iter().copied()) + .collect() +} + +#[cfg(test)] +mod lead_order_tests { + use super::{PhaseBLead, lead_order}; + + /// Each class's groups in `order`, in the order taken. + fn by_class(order: &[usize], classes: &[Option], class: usize) -> Vec { + order + .iter() + .copied() + .filter(|&g| classes[g] == Some(class)) + .collect() + } + + /// The leaf with the fewest cells still to take leads (ties to the lower + /// leaf); only the groups still to come count; its groups go first in + /// their order, the others keep theirs; the result is a permutation. + #[test] + fn the_leaf_with_the_fewest_cells_left_leads() { + let cells = [10, 10, 10, 10, 10, 10, 1, 1]; + let none = [None; 8]; + let leaves = vec![vec![0, 3, 6], vec![1, 4, 7], vec![2, 5]]; + // All to come: leaf 2 holds 20 cells, leaves 0 and 1 hold 21 each. + let rest: Vec = (0..8).collect(); + assert_eq!( + lead_order(&leaves, &rest, &none, &cells), + Some((2, vec![2, 5], vec![2, 5, 0, 1, 3, 4, 6, 7])) + ); + // Groups 0 and 3 taken: leaf 0 has one cell left. + let rest = [1, 2, 4, 5, 6, 7]; + assert_eq!( + lead_order(&leaves, &rest, &none, &cells), + Some((0, vec![6], vec![6, 1, 2, 4, 5, 7])) + ); + // A tie goes to the lower leaf; the moved groups keep `rest`'s order. + let even = [5u64; 8]; + let rest = [7, 6, 5, 4, 3, 2, 1, 0]; + assert_eq!( + lead_order(&leaves, &rest, &none, &even), + Some((2, vec![5, 2], vec![5, 2, 7, 6, 4, 3, 1, 0])) + ); + // Groups the plan holds but phase B has not (out of range) are ignored. + let leaves_wide = vec![vec![1, 40, 41]]; + assert_eq!( + lead_order(&leaves_wide, &[0, 1, 2], &none, &cells), + Some((0, vec![1], vec![1, 0, 2])) + ); + } + + /// A leaf with a group still to come that phase B rebuilds cannot lead + /// (one whose rebuilt group is already taken can); every class keeps its + /// own order; with no leaf able to lead, nothing moves. + #[test] + fn a_leaf_with_a_rebuilt_group_to_come_never_leads() { + let cells = [1, 1, 1, 1, 9, 9, 9, 9]; + let classes = [None, None, None, None, Some(1), Some(0), Some(1), Some(0)]; + // Leaf 0 is the lightest but holds group 6, rebuilt and to come. + let leaves = vec![vec![0, 6], vec![1, 2, 4], vec![3, 5, 7]]; + let rest: Vec = (0..8).collect(); + assert_eq!(lead_order(&leaves, &rest, &classes, &cells), None); + let leaves = vec![vec![0, 6], vec![1, 2], vec![3, 5, 7]]; + let (leaf, moved, now) = + lead_order(&leaves, &rest, &classes, &cells).expect("leaf 1 leads"); + assert_eq!((leaf, moved), (1, vec![1, 2])); + for class in 0..2 { + assert_eq!( + by_class(&now, &classes, class), + by_class(&rest, &classes, class), + "class {class}" + ); + } + // Group 6 taken already: leaf 0 (one cell left) leads. + let rest = [0, 1, 2, 3, 4, 5, 7]; + assert_eq!( + lead_order(&leaves, &rest, &classes, &cells), + Some((0, vec![0], vec![0, 1, 2, 3, 4, 5, 7])) + ); + // A leaf whose groups are all taken cannot lead. + let rest = [1, 2, 3, 4, 5, 7]; + assert_eq!( + lead_order(&leaves, &rest, &classes, &cells), + Some((1, vec![1, 2], vec![1, 2, 3, 4, 5, 7])) + ); + // Every leaf's groups taken, or none offered: nothing leads. + assert_eq!(lead_order(&leaves, &[], &classes, &cells), None); + assert_eq!(lead_order(&[], &[0, 1], &classes, &cells), None); + } + + /// The lead reads the offer once: nothing before it, the re-ranking at + /// the first turn after it (recorded with the order taken), nothing after; + /// a lead whose order is phase B's own is recorded but changes nothing. + #[test] + fn the_lead_reads_the_offer_once() { + let cells = [3, 3, 3, 3, 1, 1]; + let none = [None; 6]; + let order: Vec = (0..6).collect(); + let lead = PhaseBLead::default(); + assert_eq!(lead.turn(0, &order, &none, &cells, 0.0), None); + assert_eq!(lead.taken(), None); + assert!(lead.offer(vec![vec![0, 2], vec![1, 4, 5]])); + assert!(!lead.offer(vec![vec![3]]), "only the first offer counts"); + // At turn 1 (groups 0 and 1 taken): leaf 0 has 3 cells left, leaf 1 two. + assert_eq!( + lead.turn(1, &order, &none, &cells, 0.5), + Some(vec![4, 5, 2, 3]) + ); + let taken = lead.taken().expect("read"); + assert_eq!(taken.turn, 1); + assert_eq!(taken.leaf, Some(1)); + assert_eq!(taken.moved, vec![4, 5]); + assert_eq!(taken.order, vec![0, 1, 4, 5, 2, 3]); + assert_eq!(taken.after, 0.5); + let now = [0, 1, 4, 5, 2, 3]; + assert_eq!(lead.turn(2, &now, &none, &cells, 0.6), None); + assert_eq!(lead.taken(), Some(taken)); + // Phase B's own order already leads with the soonest leaf: no change. + let cells = [1, 1, 1, 9, 9, 9]; + let lead = PhaseBLead::default(); + assert!(lead.offer(vec![vec![1, 2], vec![3, 4, 5]])); + assert_eq!(lead.turn(0, &order, &none, &cells, 0.0), None); + let taken = lead.taken().expect("read"); + assert_eq!((taken.leaf, taken.order), (Some(0), order.clone())); + } +} + +#[cfg(test)] +mod led_order_tests { + use super::{led_order, multilinear_block, put_first}; + + /// The probe's groups go first in the order named, the rest keep phase B's + /// order; a group out of range or named twice is refused. + #[test] + fn named_groups_go_first_and_the_rest_keep_their_order() { + assert_eq!( + put_first(&[3, 1], vec![0, 1, 2, 3, 4]), + Ok(vec![3, 1, 0, 2, 4]) + ); + assert_eq!(put_first(&[], vec![2, 0, 1]), Ok(vec![2, 0, 1])); + assert_eq!(put_first(&[0, 2], vec![2, 1, 0]), Ok(vec![0, 2, 1])); + assert!(put_first(&[5], vec![0, 1, 2]).is_err()); + assert!(put_first(&[1, 1], vec![0, 1, 2]).is_err()); + } + + const S: Option = Some(0); + const R: Option = Some(1); + + /// The groups phase B rebuilds nothing of, then `lead` streamed groups, + /// then the interleave of what is left: the rest's first group comes after + /// `lead` + 1 streamed ones (the interleave's tie goes to the streamed + /// class). + #[test] + fn the_rests_first_group_follows_the_led_streamed_groups() { + let classes = [None, S, S, S, S, S, S, R, R, None]; + let ones = [1u64; 10]; + assert_eq!( + led_order(&classes, &ones, 2), + vec![0, 9, 1, 2, 3, 7, 4, 5, 8, 6] + ); + // A lead of 0, and a block with no rest group, are the interleave's. + assert_eq!( + led_order(&classes, &ones, 0), + multilinear_block::interleaved_order(&classes, &ones) + ); + let no_rest = [None, S, S, S]; + assert_eq!( + led_order(&no_rest, &[1; 4], 3), + multilinear_block::interleaved_order(&no_rest, &[1; 4]) + ); + // A lead past the class takes all of it, then the rest's. + assert_eq!( + led_order(&[R, S, None, S, R], &[1; 5], 9), + vec![2, 1, 3, 0, 4] + ); + // Nothing rebuilt: the group order. + assert_eq!(led_order(&[None; 3], &[], 4), vec![0, 1, 2]); + } + + /// Whatever the lead and the weights: every group once, and each class's + /// groups in group order (their regenerator deposits in that order). + #[test] + fn a_led_order_takes_every_group_once_and_each_class_in_order() { + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for _ in 0..200 { + let n = 1 + (next() % 40) as usize; + let classes: Vec> = (0..n) + .map(|_| match next() % 5 { + 0 => None, + 1..=3 => S, + _ => R, + }) + .collect(); + let cells: Vec = (0..n).map(|_| 1 + next() % 1000).collect(); + let lead = (next() % 8) as usize; + let order = led_order(&classes, &cells, lead); + let mut seen = order.clone(); + seen.sort_unstable(); + assert_eq!(seen, (0..n).collect::>(), "{classes:?} lead {lead}"); + for class in [S, R] { + let taken: Vec = order + .iter() + .copied() + .filter(|&g| classes[g] == class) + .collect(); + assert!( + taken.windows(2).all(|w| w[0] < w[1]), + "{classes:?} lead {lead}" + ); + } + // The groups phase B rebuilds nothing of come first. + let plain = classes.iter().filter(|c| c.is_none()).count(); + assert!(order[..plain].iter().all(|&g| classes[g].is_none())); + // With a rest group, the streamed ones ahead of it are the lead's + // (or all there are), and one more by the interleave's tie. + if classes.contains(&R) { + let ahead = order[plain..] + .iter() + .take_while(|&&g| classes[g] == S) + .count(); + let streamed = classes.iter().filter(|c| **c == S).count(); + assert!( + ahead >= lead.min(streamed), + "{classes:?} lead {lead}: {order:?}" + ); + } + } + } +} + +/// Each group's cells, from its own tables: `shapes` is in AIR order and a +/// group lists its tables' AIR indices, which phase A's packer does not keep +/// contiguous (a group of streamed CPU and MEMW_R chunks sits between rest +/// families in AIR order). Phase B's interleave and its readout weigh the +/// groups by these. +fn group_cells(shapes: &[(usize, usize)], groups: &[Vec]) -> Vec { + groups + .iter() + .map(|group| { + group + .iter() + .filter_map(|&i| shapes.get(i as usize)) + .map(|&(w, n)| (w as u64) << n) + .sum() + }) + .collect() +} + +#[cfg(test)] +mod group_cells_tests { + use super::group_cells; + + /// A group weighs its own tables, wherever they sit in AIR order: here + /// group 0 holds AIRs 3 and 1, group 1 AIRs 0, 4 and 2. + #[test] + fn each_group_weighs_its_own_tables_in_any_air_order() { + let shapes = [(1, 4), (2, 10), (3, 0), (5, 20), (7, 1)]; + let groups = vec![vec![3, 1], vec![0, 4, 2]]; + assert_eq!( + group_cells(&shapes, &groups), + vec![(5 << 20) + (2 << 10), (1 << 4) + (7 << 1) + 3] + ); + } + + /// Groups in AIR order weigh the same as before, and an index past the + /// shapes adds nothing (no panic). + #[test] + fn contiguous_groups_and_a_stray_index() { + let shapes = [(4, 2), (1, 3), (2, 1)]; + assert_eq!(group_cells(&shapes, &[vec![0, 1], vec![2]]), vec![24, 4]); + assert_eq!(group_cells(&shapes, &[vec![2, 9]]), vec![4]); + } +} + +#[cfg(test)] +mod rest_order_tests { + use super::RestOrder; + + fn order() -> RestOrder { + RestOrder::new(vec![21, 6, 38, 38, 17], false) + } + + /// Every AIR in order, streamed slots among them: accepted. + #[test] + fn the_rest_in_air_order_is_taken() { + let mut o = order(); + for (position, width) in [ + (0, Some(21)), + (1, Some(6)), + (2, None), + (3, Some(38)), + (4, Some(17)), + ] { + o.admit(position, width).expect("in order"); + } + o.finish().expect("every AIR reached"); + } + + /// A table out of order, of another AIR's width, past the AIRs, or a + /// stream that stops short: refused, as an error. + #[test] + fn a_table_out_of_order_wrong_or_missing_is_refused() { + let mut o = order(); + o.admit(0, Some(21)).expect("0"); + let err = o.admit(2, Some(38)).expect_err("1 is due"); + assert!( + format!("{err:?}").contains("came where table 1 was due"), + "{err:?}" + ); + let mut o = order(); + let err = o.admit(0, Some(6)).expect_err("BITWISE is 21 wide"); + assert!( + format!("{err:?}").contains("6 columns, its AIR declares 21"), + "{err:?}" + ); + let mut o = order(); + for (position, width) in [(0, Some(21)), (1, Some(6)), (2, None)] { + o.admit(position, width).expect("in order"); + } + let err = o.finish().expect_err("two AIRs never came"); + assert!( + format!("{err:?}").contains("table 3 of the rest was never laid out"), + "{err:?}" + ); + let mut o = RestOrder::new(vec![21], false); + o.admit(0, Some(21)).expect("0"); + let err = o.admit(1, Some(6)).expect_err("past the AIRs"); + assert!( + format!("{err:?}").contains("past the run's 1 tables"), + "{err:?}" + ); + } + + /// With BITWISE handed on last, the rest is due in AIR order but its first + /// table (BITWISE), which comes after every other: that order is taken, AIR + /// order is refused at BITWISE's turn, and without the flag BITWISE late is + /// refused. + #[test] + fn with_bitwise_last_the_rest_is_due_in_air_order_but_bitwise() { + let mut o = RestOrder::new(vec![21, 6, 38, 38, 17], true); + for (position, width) in [ + (1, Some(6)), + (2, None), + (3, Some(38)), + (4, Some(17)), + (0, Some(21)), + ] { + o.admit(position, width).expect("BITWISE last"); + } + o.finish().expect("every AIR reached"); + let mut o = RestOrder::new(vec![21, 6, 38, 38, 17], true); + let err = o.admit(0, Some(21)).expect_err("BITWISE is due last"); + assert!( + format!("{err:?}").contains("came where table 1 was due"), + "{err:?}" + ); + let mut o = order(); + let err = o.admit(1, Some(6)).expect_err("BITWISE is due first"); + assert!( + format!("{err:?}").contains("came where table 0 was due"), + "{err:?}" + ); + } +} + +#[cfg(test)] +mod producer_report_tests { + use super::{ + BLOCK_LAYOUT_AHEAD, BLOCK_LAYOUT_WORKERS, BLOCK_WINDOW_LOG2, BlockOptions, BlockStamps, + ExecutorStamps, ProducerStamps, StreamTable, parse_logs_move, parse_window_log2, + }; + + /// Production lays the streamed chunks out on six workers with eight + /// chunks ahead (nine unpacked at most): a permit per worker and three + /// more, so a worker goes on while the packer waits for a slow chunk. + #[test] + fn production_lays_out_on_six_workers_eight_ahead() { + assert_eq!(BLOCK_LAYOUT_WORKERS, 6); + assert_eq!(BLOCK_LAYOUT_AHEAD, 8); + let options = BlockOptions::production(); + assert_eq!(options.layout_workers, BLOCK_LAYOUT_WORKERS); + assert_eq!(options.layout_ahead, Some(BLOCK_LAYOUT_AHEAD)); + assert_eq!(BLOCK_LAYOUT_AHEAD + 1, BLOCK_LAYOUT_WORKERS + 3); + } + + /// The window knob takes production's 2^20 only, unset is production's, + /// and anything else (#1013's 2^21 included) is refused. + #[test] + fn the_window_knob_takes_20_only() { + assert_eq!(BLOCK_WINDOW_LOG2, 20); + assert_eq!(parse_window_log2(None), Ok(None)); + assert_eq!(parse_window_log2(Some(" 20 ")), Ok(Some(20))); + for bad in ["", "19", "21", " 21 ", "22", "2^20", "x"] { + assert!(parse_window_log2(Some(bad)).is_err(), "{bad:?}"); + } + } + + /// The executor's line: its own work apart from its blocked sends, and + /// whether its logs were moved or copied; no line without a streamed run. + #[test] + fn the_report_says_what_the_executor_did_apart_from_waiting() { + let mut stamps = BlockStamps { + streamed: (8.62, 269), + producer: ProducerStamps { + executor_send: 3.5, + ..ProducerStamps::default() + }, + executor: ExecutorStamps { + started: 0.02, + setup: 0.01, + run: 2.5, + handoff: 0.04, + windows: 134, + moved: true, + }, + ..BlockStamps::default() + }; + let line = |stamps: &BlockStamps| { + stamps + .report() + .lines() + .find(|l| l.starts_with("BLOCK EXECUTOR:")) + .map(str::to_string) + }; + assert_eq!( + line(&stamps).as_deref(), + Some( + "BLOCK EXECUTOR: started at 0.02s · setup 0.01s · execute 2.50s · hand-off 0.04s \ + (134 windows, logs moved) · send-blocked 3.50s · own 2.55s" + ) + ); + stamps.executor.moved = false; + assert!(line(&stamps).unwrap().contains("logs copied")); + stamps.executor = ExecutorStamps::default(); + assert_eq!(line(&stamps), None); + } + + /// `LAMBDA_VM_BLOCK_LOGS_MOVE`: on unless `0`. + #[test] + fn the_logs_are_moved_unless_the_knob_is_zero() { + assert!(parse_logs_move(None)); + assert!(parse_logs_move(Some("1"))); + assert!(parse_logs_move(Some(""))); + assert!(!parse_logs_move(Some("0"))); + assert!(!parse_logs_move(Some(" 0 "))); + } + + /// The readout's two producer lines: what each thread waited on, and the + /// hand-out table by table, only the tables that handed a chunk out (the + /// box gates parse these lines). + #[test] + fn the_report_says_what_the_producer_waited_on() { + let mut stamps = BlockStamps { + streamed: (29.23, 558), + producer: ProducerStamps { + executor_send: 24.1, + walker_recv: 0.4, + walker_send: 11.9, + accumulator_recv: 1.2, + accumulator_send: 1.55, + layout_send: 20.3, + worker_permit: 40.2, + worker_recv: 15.7, + }, + ..BlockStamps::default() + }; + let h = &mut stamps.windows.handout; + let (cpu, store) = (StreamTable::Cpu.position(), StreamTable::Store.position()); + (h.take[cpu], h.count[cpu], h.chunks[cpu], h.copied[cpu]) = (6.5, 1.86, 142, 297_795_584); + (h.take[store], h.count[store], h.chunks[store]) = (0.05, 0.35, 15); + let report = stamps.report(); + let line = |prefix: &str| { + report + .lines() + .find(|l| l.starts_with(prefix)) + .unwrap_or_else(|| panic!("no {prefix} line in\n{report}")) + .to_string() + }; + assert_eq!( + line("BLOCK PRODUCER:"), + "BLOCK PRODUCER: executor send-blocked 24.10s · walker recv-wait 0.40s send-blocked \ + 11.90s · accumulator recv-wait 1.20s send-blocked 1.55s · layout hand-on blocked \ + 20.30s · workers permit-wait 40.20s recv-wait 15.70s (summed)" + ); + assert_eq!( + line("BLOCK HANDOUT:"), + "BLOCK HANDOUT: take 6.55s · count 2.21s · copied 297.8 M ops || per table (take s · \ + count s · chunks · copied ops): CPU 6.50 · 1.86 · 142 · 297.8 M | STORE 0.05 · 0.35 · \ + 15 · 0.0 M" + ); + assert_eq!( + line("BLOCK HANDOUT COUNT:"), + "BLOCK HANDOUT COUNT: on the accumulator's thread (the count above)" + ); + let mut c = crate::tables::trace_builder::CountThreadStamps::default(); + let lt = StreamTable::Lt.position(); + (c.busy[cpu], c.chunks[cpu], c.busy[lt], c.chunks[lt]) = (1.25, 142, 0.5, 3); + (c.idle, c.drain, c.merge) = (6.0, 0.04, 0.021); + stamps.windows.count_thread = Some(c); + let report = stamps.report(); + assert_eq!( + report + .lines() + .find(|l| l.starts_with("BLOCK HANDOUT COUNT:")) + .expect("the count's line"), + "BLOCK HANDOUT COUNT: on a thread of its own · busy 1.75s · waited for chunks 6.00s · \ + the windows' end waited 0.04s for its last chunks and 0.021s to add its counts || \ + per table (busy s · chunks): CPU 1.25 · 142 | LT 0.50 · 3" + ); + } +} + +#[cfg(test)] +mod packer_tests { + use std::sync::Arc; + use std::time::Instant; + + use super::{Key, Packer, StreamAirs, group_queue, table_of}; + use crate::tables::gpack::TraceForm; + use crate::tables::lt::{self, LtOperation}; + use crate::tables::trace_builder::StreamTable; + use multilinear::whir_chain::StackVars; + use stark::proof::options::ProofOptions; + + /// The packer closes the same groups from the same tables, and phase A is + /// handed them in the same order, whatever lets a closed group wait: the + /// channel of one, or the queue with no bytes, one byte or all the bytes + /// to wait in, taken slowly. + #[test] + fn the_packer_closes_the_same_groups_whatever_waits() { + group_queue::within_a_minute("the packer", packs_alike); + } + + fn packs_alike() { + let opts = ProofOptions::default_test_options(); + let airs = StreamAirs::new(&opts, TraceForm::Wide, 1 << 21); + let air = airs.of(StreamTable::Lt).expect("LT's AIR is the family's"); + // Tables of 4, 8 and 16 rows, in an order that makes uneven groups. + let heights: Vec = (0..60).map(|k| [3, 5, 12, 2][k * 7 % 4]).collect(); + let pack = |out: group_queue::Out<_>| -> Vec> { + let mut packer = Packer { + open: Vec::new(), + open_shapes: Vec::new(), + open_keys: Vec::new(), + group_keys: Vec::new(), + out, + cap: StackVars::new(10).expect("a valid stack"), + max_polys: 2, + max_groups: 64, + start: Instant::now(), + closed_at: Vec::new(), + blocked: 0.0, + ledger: None, + open_bytes: 0, + }; + for (index, &ops) in heights.iter().enumerate() { + let ops: Vec = (0..ops as u64) + .map(|k| LtOperation::new(k, k + 1, false)) + .collect(); + let mut trace = lt::generate_lt_trace_as(&ops, TraceForm::Wide); + let shape = ( + trace.main_table.width, + trace.main_table.height.trailing_zeros() as usize, + ); + let table = table_of(air, &mut trace, shape, true).expect("a table"); + packer + .place(Key::Streamed(StreamTable::Lt, index), shape, table) + .expect("phase A takes it"); + } + let (groups, _, _) = packer.finish().expect("phase A takes the last"); + groups + .into_iter() + .map(|keys| { + keys.into_iter() + .map(|key| match key { + Key::Streamed(_, index) => index, + Key::Air(index) => index, + }) + .collect() + }) + .collect() + }; + // What phase A is handed: each group's tables by their heights. + let take = |groups: group_queue::In<'_, Vec<_>>| -> Vec> { + groups + .map( + |group: Vec>| { + std::thread::sleep(std::time::Duration::from_millis(2)); + group.iter().map(|table| table.num_vars()).collect() + }, + ) + .collect() + }; + let (packed, taken) = std::thread::scope(|scope| { + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let taker = scope.spawn(move || take(group_queue::In::Channel(rx.iter()))); + ( + pack(group_queue::Out::Channel(tx)), + taker.join().expect("the taker"), + ) + }); + assert!(packed.len() >= 4, "{} groups", packed.len()); + assert_eq!(packed.concat(), (0..heights.len()).collect::>()); + assert_eq!( + taken.iter().map(Vec::len).collect::>(), + packed.iter().map(Vec::len).collect::>() + ); + for budget in [0, 1, u64::MAX] { + let stats = Arc::new(group_queue::Stats::default()); + let (queued, handed) = std::thread::scope(|scope| { + let (tx, rx) = + group_queue::queue(group_queue::Rule::Bytes(budget), Arc::clone(&stats)); + let taker = scope.spawn(move || take(group_queue::In::Queue(rx))); + ( + pack(group_queue::Out::Queue(tx)), + taker.join().expect("the taker"), + ) + }); + assert_eq!(queued, packed, "budget {budget}: the groups"); + assert_eq!(handed, taken, "budget {budget}: what phase A is handed"); + } + // With every byte to wait in and nobody taking, every group waits at + // once: the drops then count the main cells of all but the first, each + // committed column's rows, as phase A's commit counts a table's. + let stats = Arc::new(group_queue::Stats::default()); + let rule = group_queue::Rule::Bytes(u64::MAX); + let (tx, rx) = group_queue::queue(rule, Arc::clone(&stats)); + assert_eq!(pack(group_queue::Out::Queue(tx)), packed); + let counted = group_queue::arming_cells(Some(rule), &stats, 0); + let cells: Vec = rx + .map( + |group: Vec>| { + group + .iter() + .map(|table| (table.num_committed_columns() as u64) << table.num_vars()) + .sum() + }, + ) + .collect(); + assert_eq!(cells.len(), packed.len()); + assert!(counted > 0); + assert_eq!(counted, cells[1..].iter().sum::()); + assert_eq!(group_queue::arming_cells(Some(rule), &stats, 0), 0); + } +} + +#[cfg(test)] +mod walkers_knob_tests { + use executor::vm::memory::StoreKind; + + use super::{ + CardFill, Error, RowFamilies, parse_card_fill, parse_records_drop, parse_walkers, + records_admission, + }; + + /// Streamed chunks are held as records only with live regeneration to drop + /// them, or kept on purpose: never with a store or with regeneration off + /// and parking, where none would ever leave the host. And never unless the + /// card was asked and the build is narrow and drops its streamed ops. + #[test] + fn records_need_live_regeneration_unless_kept() { + for (asked, narrow_dropping, keep, live, admitted) in [ + (true, true, false, true, true), + (true, true, true, true, true), + (true, true, true, false, true), + (true, true, false, false, false), + (true, false, false, true, false), + (true, false, true, true, false), + (false, true, false, true, false), + (false, true, true, true, false), + ] { + assert_eq!( + records_admission(asked, narrow_dropping, keep, live).is_ok(), + admitted, + "asked {asked}, narrow and dropping {narrow_dropping}, keep {keep}, live {live}" + ); + } + assert!( + records_admission(true, true, false, false) + .is_err_and(|why| why.contains("live regeneration")), + "the line says why" + ); + } + + /// `LAMBDA_VM_BLOCK_RECORDS_DROP`: unset, empty or `park` parks and drops + /// records chunks like any chunk; `keep` keeps them; any other word + /// refuses the prove. + #[test] + fn the_records_drop_knob_takes_park_and_keep_and_refuses_anything_else() { + for (value, keep) in [ + (None, false), + (Some(""), false), + (Some(" park "), false), + (Some("keep"), true), + ] { + assert_eq!(parse_records_drop(value).ok(), Some(keep), "{value:?}"); + } + for value in ["on", "off", "Keep", "drop", "park,keep"] { + let refused = parse_records_drop(Some(value)); + assert!( + matches!(&refused, Err(Error::Prover(m)) if m.contains("LAMBDA_VM_BLOCK_RECORDS_DROP")), + "{value}: {refused:?}" + ); + } + } + + /// `LAMBDA_VM_BLOCK_WALKERS`: unset or empty is four walkers (the + /// default); 0 one walker (the rollback); 1 to 4 that many; any other + /// word refuses the prove, and so does a count with the executor's words + /// store, with which unset is one walker. + #[test] + fn the_walkers_knob_defaults_to_4_takes_0_to_4_and_refuses_anything_else() { + for (value, walkers) in [ + (None, 4), + (Some(""), 4), + (Some(" "), 4), + (Some("0"), 0), + (Some("1"), 1), + (Some(" 3 "), 3), + (Some("4"), 4), + ] { + assert_eq!( + parse_walkers(value, StoreKind::Pages).ok(), + Some(walkers), + "{value:?}" + ); + } + for value in ["5", "8", "16", "17", "-1", "auto", "4x", "1.5"] { + assert!( + parse_walkers(Some(value), StoreKind::Pages).is_err(), + "{value}" + ); + } + for value in [None, Some(""), Some("0")] { + assert_eq!(parse_walkers(value, StoreKind::Words).ok(), Some(0)); + } + for value in ["1", "4"] { + let refused = parse_walkers(Some(value), StoreKind::Words); + assert!( + matches!(&refused, Err(Error::Prover(m)) if m.contains("pages store")), + "{value}: {refused:?}" + ); + } + } + + /// `LAMBDA_VM_BLOCK_CARD_FILL`: a comma list of `keccak_rnd`, `cpu`, + /// `memw_r` and the row families has the card fill those families, `on` + /// every one, `off` none; + /// unset or empty is every family with the card's kernels and nothing + /// without; any other word, a family twice or an empty item refuses the + /// prove. + #[test] + fn the_card_fill_knob_takes_off_on_and_families_and_refuses_anything_else() { + let fill = |keccak_rnd, cpu, memw_r| CardFill { + keccak_rnd, + cpu, + memw_r, + rows: RowFamilies::NONE, + }; + let every = CardFill::EVERY; + let rows = |rows| CardFill { + rows, + ..fill(false, false, false) + }; + let some = |memw_a, load, store, memw, shift| RowFamilies { + memw_a, + load, + store, + memw, + shift, + }; + let card = cfg!(feature = "cuda"); + let default = if card { + every + } else { + fill(false, false, false) + }; + for (value, want) in [ + (None, default), + (Some(""), default), + (Some(" "), default), + (Some("off"), fill(false, false, false)), + (Some(" off "), fill(false, false, false)), + (Some("on"), every), + (Some(" on\n"), every), + (Some("memw_a"), rows(some(true, false, false, false, false))), + (Some("load"), rows(some(false, true, false, false, false))), + (Some("store"), rows(some(false, false, true, false, false))), + (Some("memw"), rows(some(false, false, false, true, false))), + (Some("shift"), rows(some(false, false, false, false, true))), + ( + Some("load,memw_a"), + rows(some(true, true, false, false, false)), + ), + (Some("shift,store,memw,load,memw_a"), rows(RowFamilies::ALL)), + (Some("keccak_rnd"), fill(true, false, false)), + (Some("cpu"), fill(false, true, false)), + (Some("memw_r"), fill(false, false, true)), + (Some("cpu,memw_r"), fill(false, true, true)), + (Some("keccak_rnd,cpu,memw_r"), fill(true, true, true)), + (Some(" memw_r , keccak_rnd "), fill(true, false, true)), + ] { + assert_eq!(parse_card_fill(value).ok(), Some(want), "{value:?}"); + } + assert_eq!(fill(false, false, false).name(), "off"); + // ★ Unset is exactly the gated configuration's words with a card + // (every family: KECCAK_RND, CPU, MEMW_R and the five row families), + // nothing without; the earlier default's words (no row family) are + // the rollback, and `keccak_rnd` alone KECCAK_RND's fill only. + let gated = + parse_card_fill(Some("keccak_rnd,cpu,memw_r,memw_a,load,store,memw,shift")).ok(); + let rollback = parse_card_fill(Some("keccak_rnd,cpu,memw_r")).ok(); + let keccak_rnd = parse_card_fill(Some("keccak_rnd")).ok(); + if card { + assert_eq!(Some(CardFill::DEFAULT), gated, "unset with a card"); + assert_eq!(CardFill::DEFAULT.name(), every.name()); + } else { + assert_eq!( + CardFill::DEFAULT, + fill(false, false, false), + "unset without a card" + ); + } + assert_eq!(rollback, Some(fill(true, true, true)), "the rollback"); + assert!( + !fill(true, true, true).rows.any(), + "the rollback: no row family" + ); + assert_eq!( + keccak_rnd, + Some(fill(true, false, false)), + "KECCAK_RND alone" + ); + assert_eq!( + CardFill::DEFAULT.rows.any(), + card, + "the row families by default with the card's kernels" + ); + assert_eq!(fill(true, false, true).name(), "keccak_rnd,memw_r"); + assert_eq!(fill(true, true, true).name(), "keccak_rnd,cpu,memw_r"); + assert_eq!( + every.name(), + "keccak_rnd,cpu,memw_r,memw_a,load,store,memw,shift" + ); + assert_eq!( + parse_card_fill(Some(&every.name())).ok(), + Some(every), + "the name reads back" + ); + assert_eq!( + rows(some(false, true, false, false, true)).name(), + "load,shift" + ); + for value in [ + "1", + "0", + "ON", + "Off", + "yes", + "true", + "on,off", + "kr", + "cpu,cpu", + "cpu,", + ",memw_r", + "CPU", + "keccak_rnd,on", + "cpu memw_r", + "memw_a,memw_a", + "MEMW_A", + "loads", + "store,shift,store", + "stores", + "memw_w", + ] { + let refused = parse_card_fill(Some(value)); + assert!( + matches!(&refused, Err(Error::Prover(m)) if m.contains("LAMBDA_VM_BLOCK_CARD_FILL")), + "{value}: {refused:?}" + ); + } + } +} + +#[cfg(test)] +mod air_input_tests { + use super::*; + + /// A block statement's CPU heights are read at the CPU tables' positions, + /// each at most 2^21 rows. + #[test] + fn a_block_statement_states_cpu_tables_of_at_most_2_21_rows() { + let mut counts = crate::TableCounts { + cpu: 2, + lt: 0, + memw: 0, + memw_aligned: 0, + load: 0, + mul: 0, + dvrm: 0, + shift: 0, + branch: 0, + memw_register: 0, + eq: 0, + bytewise: 0, + store: 0, + cpu32: 0, + keccak: 0, + keccak_rnd: 0, + ecsm: 0, + ecdas: 0, + hint: 0, + copy: 3, + blake3: 1, + is_wh: 1, + ecrec: 0, + }; + let start = crate::cpu_proof_start(true, &counts); + let mut vars = vec![5u8; start + counts.cpu + 4]; + vars[start] = 21; + vars[start + 1] = 2; + assert_eq!( + block_cpu_heights(&counts, &vars).expect("in bounds"), + vec![1 << 21, 4] + ); + vars[start + 1] = 22; + assert!(matches!( + block_cpu_heights(&counts, &vars), + Err(Error::InvalidTableCounts(_)) + )); + counts.cpu = vars.len(); + assert!(matches!( + block_cpu_heights(&counts, &vars), + Err(Error::InvalidTableCounts(_)) + )); + } + + /// A block's AIR inputs refuse CPU heights the clock cannot take, and a + /// public output whose byte count leaves x254's word, before either sizes + /// anything; the block's own 2^21-row tables are taken. + #[test] + fn block_air_inputs_refuse_what_the_clock_cannot_take() { + let opts = ProofOptions::default_test_options(); + let program = Elf::load(&crate::test_utils::asm_elf_bytes("all_instructions_64")) + .expect("the ELF loads"); + let take = |heights: Vec| BlockAirInputs::new(&opts, &program, heights, &[]); + assert!(take(vec![1 << 21, 1 << 21, 4]).is_ok()); + assert!(take(vec![crate::tables::cpu::MAX_TABLE_ROWS]).is_ok()); + let past_the_clock = vec![1 << 21; 1 << 9]; + for bad in [ + vec![3], + vec![6], + vec![1 << 21, 12], + vec![crate::tables::cpu::MAX_TABLE_ROWS * 2], + past_the_clock, + ] { + assert!( + matches!(take(bad.clone()), Err(Error::InvalidTableCounts(_))), + "heights {:?}", + &bad[..bad.len().min(4)] + ); + } + } +} + +#[cfg(test)] +mod p4_rows_knob_tests { + use super::{ + P4Rows, parse_bitwise_from_tables, parse_card_early, parse_p4_lookups, parse_p4_rows, + parse_rest_bitwise_last, + }; + + /// `LAMBDA_VM_BLOCK_P4_ROWS`: unset, empty or `pool` counts the per-row + /// lookups on the pool (the default); `off` does not; any other word + /// refuses. + #[test] + fn the_p4_rows_knob_defaults_to_pool_takes_off_and_refuses_anything_else() { + for (value, rows) in [ + (None, P4Rows::Pool), + (Some(""), P4Rows::Pool), + (Some(" off "), P4Rows::Off), + (Some("pool"), P4Rows::Pool), + (Some(" pool\n"), P4Rows::Pool), + (Some("parts"), P4Rows::Parts), + (Some(" parts "), P4Rows::Parts), + ] { + assert_eq!(parse_p4_rows(value).ok(), Some(rows), "{value:?}"); + } + for value in ["1", "on", "POOL", "pools", "rows", "part", "PARTS"] { + assert!(parse_p4_rows(Some(value)).is_err(), "{value}"); + } + } + + /// `LAMBDA_VM_BLOCK_P4_LOOKUPS`: unset, empty or `direct` counts straight + /// into the histograms (the default); `lists` builds each slice's lists + /// (the rollback); anything else refuses. + #[test] + fn the_p4_lookups_knob_defaults_to_direct_and_takes_lists() { + for (value, direct) in [ + (None, true), + (Some(""), true), + (Some(" lists "), false), + (Some("direct"), true), + (Some(" direct\n"), true), + ] { + assert_eq!(parse_p4_lookups(value).ok(), Some(direct), "{value:?}"); + } + for value in ["1", "on", "DIRECT", "list", "vec"] { + assert!(parse_p4_lookups(Some(value)).is_err(), "{value}"); + } + } + + /// The finish's phase-4 defaults: the per-row lookups counted on the pool + /// with one map a chunk (not by parts), LT's and KECCAK's lookups counted + /// straight into the histograms, the card set up as the prove starts, and + /// BITWISE's LT, BRANCH and BYTEWISE lookups from their tables, BITWISE + /// last; BITWISE last alone stays opt-in. + #[test] + fn the_finish_counts_on_one_map_a_chunk_and_its_lookups_directly_by_default() { + assert_eq!(parse_p4_rows(None).ok(), Some(P4Rows::Pool)); + assert_eq!(parse_p4_lookups(None).ok(), Some(true)); + assert_eq!(parse_card_early(None).ok(), Some(true)); + assert_eq!(parse_bitwise_from_tables(None).ok(), Some(true)); + assert_eq!(parse_rest_bitwise_last(None).ok(), Some(false)); + } + + /// `LAMBDA_VM_BLOCK_CARD_EARLY`: on unless empty, `0` or `off`; any other + /// word refuses. + #[test] + fn the_card_early_knob_is_on_unless_zero_or_off() { + for (value, on) in [ + (None, true), + (Some("1"), true), + (Some(" on "), true), + (Some(""), false), + (Some("0"), false), + (Some(" off"), false), + ] { + assert_eq!(parse_card_early(value).ok(), Some(on), "{value:?}"); + } + for value in ["ON", "true", "2", "yes"] { + assert!(parse_card_early(Some(value)).is_err(), "{value}"); + } + } + + /// `LAMBDA_VM_BLOCK_REST_BITWISE_LAST`: off (AIR order, BITWISE first) + /// unless `1`; any other word refuses. + #[test] + fn the_bitwise_last_knob_is_off_by_default_and_on_at_1() { + for (value, on) in [ + (None, false), + (Some(""), false), + (Some("0"), false), + (Some(" 1 "), true), + ] { + assert_eq!(parse_rest_bitwise_last(value).ok(), Some(on), "{value:?}"); + } + for value in ["on", "true", "2", "last", "BITWISE"] { + assert!(parse_rest_bitwise_last(Some(value)).is_err(), "{value}"); + } + } + + /// `LAMBDA_VM_BLOCK_BITWISE_FROM_TABLES`: on (BITWISE's three families' + /// lookups from their tables, BITWISE last) unless `0`, the rollback; any + /// other word refuses. + #[test] + fn the_bitwise_from_tables_knob_is_on_by_default_and_off_at_0() { + for (value, on) in [ + (None, true), + (Some(""), true), + (Some(" 1 "), true), + (Some("0"), false), + (Some(" 0\n"), false), + ] { + assert_eq!(parse_bitwise_from_tables(value).ok(), Some(on), "{value:?}"); + } + for value in ["on", "true", "2", "tables", "rows"] { + assert!(parse_bitwise_from_tables(Some(value)).is_err(), "{value}"); + } + } +} + +#[cfg(test)] +mod rest_slide_knob_tests { + use super::{ + BLOCK_REST_BUILD_BYTES, BLOCK_REST_LAYOUT_BYTES, BlockOptions, parse_rest_build, + parse_rest_slide, rest_layout_label, + }; + + /// `BLOCK_WHIR_REST_BUILD`: unset is no cap beyond the gate (production); + /// a positive MiB count is the cap in bytes; anything else is an error. + #[test] + fn the_build_cap_word_takes_a_positive_mib_count() { + assert_eq!(parse_rest_build(None), Ok(None)); + for (value, bytes) in [("2048", 2usize << 30), (" 1 ", 1 << 20)] { + assert_eq!(parse_rest_build(Some(value)), Ok(Some(bytes)), "{value}"); + } + for value in ["", "0", "-1", "2 GiB", "all"] { + assert!(parse_rest_build(Some(value)).is_err(), "{value}"); + } + } + + /// `LAMBDA_VM_BLOCK_REST_SLIDE`: unset (or anything but `0`) slides; `0` + /// builds the rest in waves. + #[test] + fn the_slide_is_the_default_and_0_builds_in_waves() { + for value in [None, Some("1"), Some(""), Some(" 1 ")] { + assert!(parse_rest_slide(value), "{value:?}"); + } + for value in ["0", " 0 "] { + assert!(!parse_rest_slide(Some(value)), "{value}"); + } + } + + /// The layout line says waves only where the rest is laid out in waves: + /// the streamed finish that slides has a gate and no waves. + #[test] + fn the_layout_line_says_waves_only_where_the_rest_builds_in_waves() { + let mut options = BlockOptions::production(); + options.finish_stream = true; + options.finish_cuts = true; + assert!(options.rest_streams()); + assert_eq!( + rest_layout_label(&options, true), + "a gate of 4096 MiB, sliding (no waves)" + ); + assert_eq!(rest_layout_label(&options, false), "waves of 4096 MiB"); + options.rest_layout_bytes = None; + assert_eq!( + rest_layout_label(&options, true), + "no gate, sliding (no waves)" + ); + assert_eq!(rest_layout_label(&options, false), "waves of all at once"); + options.rest_layout_bytes = Some(2 << 30); + options.finish_stream = false; + assert!(!options.rest_streams()); + assert_eq!(rest_layout_label(&options, true), "waves of 2048 MiB"); + options.finish_stream = true; + options.layout_workers = 0; + assert_eq!(rest_layout_label(&options, true), "waves of 2048 MiB"); + } + + /// The rest holds up to 4 GiB with at most 2 GiB building by default; the + /// harness's words 2048 / 2048 are the earlier default (a cap equal to the + /// gate's budget is the gate alone). + #[test] + fn the_rest_holds_4_gib_with_at_most_2_gib_building_by_default() { + assert_eq!(BLOCK_REST_LAYOUT_BYTES, 4 << 30); + assert_eq!(BLOCK_REST_BUILD_BYTES, 2 << 30); + let options = BlockOptions::production(); + assert_eq!(options.rest_layout_bytes, Some(BLOCK_REST_LAYOUT_BYTES)); + assert_eq!(options.rest_build_bytes, Some(BLOCK_REST_BUILD_BYTES)); + assert_eq!(parse_rest_build(Some("2048")), Ok(Some(2 << 30))); + let earlier = crate::tables::trace_builder::gate::ByteGate::new(2 << 30); + let rolled_back = + crate::tables::trace_builder::gate::ByteGate::new(2 << 30).with_build(Some(2 << 30)); + assert_eq!(rolled_back.build(), earlier.build()); + assert_eq!(rolled_back.budget(), earlier.budget()); + } +} + +#[cfg(test)] +mod drop_tests { + use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + + use super::{ + BLOCK_KEEP_BELOW, DROP_PURGE_BYTES, DropMode, DropPurges, HostReading, decision_bytes, + parse_drop, parse_keep_below, parse_keep_cells, + }; + use crate::alloc_purge::Purge; + + const G: u64 = 1 << 30; + + /// `LAMBDA_VM_BLOCK_DROP`: unset, empty or `partial` drops only what the + /// policy wants out once armed (the default); `all` everything; any other + /// word refuses. + #[test] + fn the_drop_knob_defaults_to_partial_takes_all_and_refuses_anything_else() { + for (value, mode) in [ + (None, DropMode::Partial), + (Some(""), DropMode::Partial), + (Some(" partial "), DropMode::Partial), + (Some("all"), DropMode::All), + (Some(" all "), DropMode::All), + ] { + assert_eq!(parse_drop(value).ok(), Some(mode), "{value:?}"); + } + for value in ["some", "1", "on", "partly"] { + assert!(parse_drop(Some(value)).is_err(), "{value}"); + } + } + + /// `LAMBDA_VM_BLOCK_KEEP_CELLS_G`: unset, empty or `off` is no keep budget + /// (the default); a G-cell count above 0 is the budget in main cells; any + /// other word refuses. + #[test] + fn the_keep_budget_is_off_by_default_and_takes_g_cells_above_0() { + for value in [None, Some(""), Some(" off ")] { + assert_eq!(parse_keep_cells(value).ok(), Some(None), "{value:?}"); + } + for (value, cells) in [ + ("20.9", 20_900_000_000u64), + (" 21 ", 21_000_000_000), + ("0.5", 500_000_000), + ] { + assert_eq!( + parse_keep_cells(Some(value)).ok(), + Some(Some(cells)), + "{value}" + ); + } + for value in ["0", "-1", "on", "nan", "inf", "20.9G"] { + assert!(parse_keep_cells(Some(value)).is_err(), "{value}"); + } + } + + /// `LAMBDA_VM_BLOCK_KEEP_BACK_G`: unset or empty is the default margin, + /// 1.3 G cells below the arming point; `off` is none (the rollback); a + /// G-cell count above 0 is that margin in main cells; any other word + /// refuses. + #[test] + fn the_relative_keep_budget_defaults_to_1_3_g_cells_and_off_rolls_back() { + assert_eq!(BLOCK_KEEP_BELOW, 1_300_000_000); + for value in [None, Some(""), Some(" ")] { + assert_eq!( + parse_keep_below(value).ok(), + Some(Some(BLOCK_KEEP_BELOW)), + "{value:?}" + ); + } + assert_eq!(parse_keep_below(Some(" off ")).ok(), Some(None)); + for (value, cells) in [("1.3", 1_300_000_000u64), (" 2 ", 2_000_000_000)] { + assert_eq!( + parse_keep_below(Some(value)).ok(), + Some(Some(cells)), + "{value}" + ); + } + for value in ["0", "-1", "on", "nan", "1.3G"] { + assert!(parse_keep_below(Some(value)).is_err(), "{value}"); + } + } + + /// Partial drops, once armed, weigh what the host holds now: after a purge + /// hands the dropped tables' pages back, the reading falls (the working + /// set and the allocator's resident bytes both did), while the peak the + /// rule weighs otherwise stays where it was. + #[test] + fn after_a_purge_the_partial_reading_falls_and_the_peak_does_not() { + let before = HostReading { + hwm: 66 * G, + charged: 65 * G, + inactive_file: G, + }; + let after = HostReading { + hwm: 66 * G, + charged: 49 * G, + inactive_file: G, + }; + assert_eq!(decision_bytes(&before, Some(60 * G), true), 64 * G); + assert_eq!(decision_bytes(&after, Some(44 * G), true), 48 * G); + // The allocator's resident bytes count when the cgroup reads lower. + assert_eq!(decision_bytes(&after, Some(55 * G), true), 55 * G); + // Not partial, or not armed: the peak, purge or not. + for (reading, allocator) in [(&before, Some(60 * G)), (&after, Some(44 * G))] { + assert_eq!(decision_bytes(reading, allocator, false), 66 * G); + } + } + + /// A jemalloc control of type `T` at `name`: written with `new` when + /// given, the old value returned. + fn mallctl(name: &std::ffi::CStr, new: Option) -> Option { + let mut old = T::default(); + let mut old_len = std::mem::size_of::(); + let mut new = new; + let (new_ptr, new_len) = match new.as_mut() { + Some(v) => ( + v as *mut T as *mut std::ffi::c_void, + std::mem::size_of::(), + ), + None => (std::ptr::null_mut(), 0), + }; + // SAFETY: `name` is NUL-terminated; the old and new pointers point at a + // `T` of the control's type, with their lengths. + let rc = unsafe { + tikv_jemalloc_sys::mallctl( + name.as_ptr(), + &mut old as *mut T as *mut std::ffi::c_void, + &mut old_len, + new_ptr, + new_len, + ) + }; + (rc == 0).then_some(old) + } + + /// The allocator's resident bytes, the reading's other half, fall when a + /// purge hands back a freed gigabyte. The lib's test build runs jemalloc; + /// the calling thread's arena is set to keep its dirty pages until purged, + /// as the block's posture keeps them (`dirty_decay_ms:-1`), for the length + /// of the test. + #[test] + fn a_purge_hands_freed_pages_back_to_the_reading() { + let resident = || { + crate::alloc_purge::stats() + .expect("jemalloc's stats") + .resident as u64 + }; + let arena: u32 = mallctl(c"thread.arena", None).expect("the thread's arena"); + let decay = std::ffi::CString::new(format!("arena.{arena}.dirty_decay_ms")) + .expect("a control name"); + let was: isize = mallctl(&decay, Some(-1isize)).expect("the arena's decay"); + let mut blocks: Vec> = (0..256).map(|_| vec![0u8; 4 << 20]).collect(); + for block in &mut blocks { + for page in block.chunks_mut(4096) { + page[0] = 1; + } + } + std::hint::black_box(&blocks); + let held = resident(); + drop(blocks); + let freed = resident(); + let purge = crate::alloc_purge::purge_now().expect("the test build purges"); + let purged = resident(); + mallctl(&decay, Some(was)); + assert!( + freed + G / 2 > held, + "the freed pages left at once: {held} → {freed} bytes" + ); + assert!( + purged + G / 2 < freed, + "resident {freed} → {purged} bytes ({purge:?})" + ); + } + + static PURGES: AtomicUsize = AtomicUsize::new(0); + + fn counted() -> Option { + PURGES.fetch_add(1, Relaxed); + Some(Purge { + secs: 0.25, + resident_before: (10 * G) as usize, + resident_after: (6 * G) as usize, + }) + } + + /// A purge starts once the dropped bytes reach the threshold, on a thread + /// of its own, and no second one within the spacing; its line counts it. + #[test] + fn partial_drops_purge_past_their_bytes_and_their_spacing() { + let purges = DropPurges::new(counted); + let start = PURGES.load(Relaxed); + assert!(purges.dropped(DROP_PURGE_BYTES / 2).is_none()); + assert!(purges.dropped(DROP_PURGE_BYTES / 2 - 1).is_none()); + purges + .dropped(1) + .expect("the threshold starts a purge") + .join() + .expect("the purge"); + assert_eq!(PURGES.load(Relaxed) - start, 1); + // Past the bytes again, but within the spacing: none. + assert!(purges.dropped(DROP_PURGE_BYTES).is_none()); + assert_eq!(PURGES.load(Relaxed) - start, 1); + assert_eq!( + purges.line(), + "BLOCK DROP: partial · 1 purges of the dropped tables' pages, 0.25 s, 4.00 GiB returned" + ); + } +} + +#[cfg(test)] +mod records_layout_tests { + use std::sync::Arc; + + use multilinear::narrow::NarrowColumns; + use multilinear::records::{KeccakRndOp, Records, RecordsFill, TableRecords}; + use stark::proof::options::ProofOptions; + + use super::{E, F, TraceTable, table_of_records}; + + /// A table's shape, held as records that fill `prefix` leading columns + /// from the row index. + #[derive(Debug)] + struct Shaped { + rows: usize, + cols: usize, + prefix: usize, + } + + impl TableRecords for Shaped { + fn rows(&self) -> usize { + self.rows + } + + fn cols(&self) -> usize { + self.cols + } + + fn fill(&self) -> RecordsFill<'_> { + const NONE: &[KeccakRndOp] = &[]; + RecordsFill::KeccakRnd { ops: NONE, skip: 0 } + } + + fn row_index_columns(&self) -> usize { + self.prefix + } + + fn host_fill(&self) -> Option { + None + } + } + + fn laid_out( + air: &super::DynAir, + (rows, cols): (usize, usize), + prefix: usize, + ) -> Result<(), crate::Error> { + let records = Records(Arc::new(Shaped { rows, cols, prefix })); + let mut trace = + TraceTable::::try_from_records_main(records, 1).expect("a records trace"); + table_of_records(air, &mut trace, (cols, rows.trailing_zeros() as usize)).map(|_| ()) + } + + /// ★ A table held as records is laid out only for an AIR whose + /// preprocessed columns are the ones its records fill from the row index: + /// CPU's TIMESTAMP, none for MEMW_R, all nine of KECCAK_RC's for a records + /// table that declares them. Any other count is refused, either way. + #[test] + fn records_are_laid_out_only_for_their_row_index_columns() { + use crate::tables::{cpu, keccak_rc, memw_register}; + use crate::test_utils::{create_cpu_air, create_keccak_rc_air, create_memw_register_air}; + let opts = ProofOptions::default_test_options(); + // A streamed CPU chunk's AIR, as the block builds it: its TIMESTAMP + // column preprocessed. + let height = 1usize << 4; + let cpu = create_cpu_air(&opts, 0).with_lazy_preprocessed_columns( + cpu::lazy_timestamp_commitment(&opts, height), + cpu::NUM_PRECOMPUTED_COLS, + Arc::new(move || vec![cpu::timestamp_column(height)]), + ); + let memw_r = create_memw_register_air(&opts); + let rc = + create_keccak_rc_air(&opts).with_preprocessed([0; 32], keccak_rc::NUM_PRECOMPUTED_COLS); + let shape = |cols| (height, cols); + assert!( + laid_out( + &cpu, + shape(cpu::cols::NUM_COLUMNS), + cpu::NUM_PRECOMPUTED_COLS + ) + .is_ok() + ); + assert!(laid_out(&memw_r, shape(memw_register::cols::NUM_COLUMNS), 0).is_ok()); + for wrong in [0, 2] { + let refused = laid_out(&cpu, shape(cpu::cols::NUM_COLUMNS), wrong); + assert!( + matches!(&refused, Err(crate::Error::Prover(m)) if m.contains("preprocessed")), + "CPU, {wrong} declared: {refused:?}" + ); + } + let rc_shape = (keccak_rc::NUM_ROWS, keccak_rc::cols::NUM_COLUMNS); + assert!(laid_out(&rc, rc_shape, keccak_rc::NUM_PRECOMPUTED_COLS).is_ok()); + for wrong in [0, keccak_rc::NUM_PRECOMPUTED_COLS - 1] { + let refused = laid_out(&rc, rc_shape, wrong); + assert!( + matches!(&refused, Err(crate::Error::Prover(m)) if m.contains("preprocessed")), + "KECCAK_RC, {wrong} declared: {refused:?}" + ); + } + } +} diff --git a/prover/src/block_whir/group_queue.rs b/prover/src/block_whir/group_queue.rs new file mode 100644 index 000000000..5623bb6dd --- /dev/null +++ b/prover/src/block_whir/group_queue.rs @@ -0,0 +1,1085 @@ +//! The closed groups waiting for phase A (`LAMBDA_VM_BLOCK_GROUP_QUEUE`). +//! +//! The packer closes groups in arrival order and phase A commits them in that +//! order. By default one closed group waits, in a channel of one, and the +//! packer holds the next: a card that is behind holds back the layout, the +//! walk and the executor, and the finish behind its byte gate. +//! +//! With `room`, a block that proves with live regeneration lets closed groups wait +//! behind one another while the host has room for them ([`Rule::Room`]): the +//! host then builds ahead of the card. A group the rule refuses waits in the +//! packer's hands, as every group past the first does by default, until the +//! rule admits it or nothing waits. The groups, their order and the proof's +//! bytes are the same: the queue is first in, first out, and a group is closed +//! before it comes here. +//! +//! While a rule runs, the rule that arms the drops counts the main cells of +//! the groups waiting behind the first as committed ([`arming_cells`]): a +//! table is on the host whether it waits or is committed, so the block arms at +//! the number of tables built it arms at by default. + +use std::collections::VecDeque; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering::Relaxed}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Instant; + +use crate::Error; +use crate::alloc_purge::NoDisk; + +const GIB: f64 = (1u64 << 30) as f64; + +/// `LAMBDA_VM_BLOCK_GROUP_QUEUE`: `none` (and unset or empty) | `room` | +/// ``. Anything else refuses the prove. +pub(crate) const GROUP_QUEUE_ENV: &str = "LAMBDA_VM_BLOCK_GROUP_QUEUE"; + +/// What [`GROUP_QUEUE_ENV`] says. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Setting { + /// One closed group waits: the channel of one. + None, + /// Closed groups wait while the host has room ([`Rule::Room`]). + Room, + /// Closed groups wait up to this many bytes, the host not read + /// ([`Rule::Bytes`]): a measurement arm. + Bytes(u64), +} + +/// [`Setting`] from [`GROUP_QUEUE_ENV`]. +pub(crate) fn setting() -> Result { + parse_setting(std::env::var(GROUP_QUEUE_ENV).ok().as_deref()) +} + +pub(crate) fn parse_setting(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "none") => Ok(Setting::None), + Some("room") => Ok(Setting::Room), + Some(v) => v + .parse::() + .ok() + .filter(|gib| gib.is_finite() && *gib > 0.0) + .map(|gib| Setting::Bytes((gib * GIB) as u64)) + .ok_or_else(|| { + Error::Prover(format!( + "{GROUP_QUEUE_ENV}={v}: none, room or a number of GiB" + )) + }), + } +} + +/// When a closed group may wait behind another. With nothing waiting a group +/// always goes, whatever the rule. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Rule { + /// While the host has room by the purge rule's own test + /// ([`crate::alloc_purge::has_room`]): the resident set, the group's + /// bytes and the block's headroom together stay under its pressure line; + /// and never more than `cap` bytes waiting in all. The bytes waiting are + /// in the resident set already, and count once, there. + Room { room: NoDisk, cap: u64 }, + /// While at most this many bytes wait. + Bytes(u64), +} + +impl Rule { + /// The rule a block runs under `setting`. `no_disk` is what the purge + /// rule weighs for a block that proves with live regeneration; any other + /// block (`None`: one with nothing to rebuild its tables) keeps the + /// channel of one under every setting. + pub(crate) fn of(setting: Setting, no_disk: Option, cap: u64) -> Option { + let room = no_disk?; + match setting { + Setting::None => None, + Setting::Room => Some(Self::Room { room, cap }), + Setting::Bytes(bytes) => Some(Self::Bytes(bytes)), + } + } + + /// Whether a group of `bytes` may wait behind `waiting` bytes. No reading + /// of the resident set is no room. + fn admits(self, waiting: u64, bytes: u64, resident: impl FnOnce() -> Option) -> bool { + let after = waiting.saturating_add(bytes); + match self { + Self::Bytes(budget) => after <= budget, + Self::Room { room, cap } => { + after <= cap + && crate::alloc_purge::has_room( + Some(room), + resident().map(|resident| resident.saturating_add(bytes)), + ) + } + } + } +} + +/// The main cells the rule that arms the drops weighs as committed +/// ([`super::drop_wanted`]): `committed`, and while a rule runs, the cells of +/// the groups waiting behind the first. The first is where the channel of one +/// holds a group, which that rule does not count either; a group's cells leave +/// the count as phase A takes it, before its commit counts them. With no rule +/// the queue is not read: the cells are `committed`. +pub(crate) fn arming_cells(rule: Option, stats: &Stats, committed: u64) -> u64 { + match rule { + None => committed, + Some(_) => committed.saturating_add(stats.behind_cells.load(Relaxed)), + } +} + +/// What a block's queue did, for its `BLOCK GROUP QUEUE` line. +#[derive(Debug, Default)] +pub(crate) struct Stats { + /// Groups sent, and those that went in behind another. + groups: AtomicUsize, + behind: AtomicUsize, + /// Groups the rule refused at least once. + refused: AtomicUsize, + /// The most groups and the most bytes waiting at once. + most_groups: AtomicUsize, + most_bytes: AtomicU64, + /// When the last group went in behind another, in milliseconds since the + /// queue was made. + last_behind_ms: AtomicU64, + /// The main cells of the groups waiting behind the first, as the rule + /// that arms the drops counts them ([`arming_cells`]), and the most at + /// once. + behind_cells: AtomicU64, + most_behind_cells: AtomicU64, +} + +/// The `BLOCK GROUP QUEUE` line of a block: the setting, the rule it ran and +/// what waited. +pub(crate) fn line(setting: Setting, rule: Option, stats: &Stats) -> String { + let gib = |bytes: u64| bytes as f64 / GIB; + let knob = match setting { + Setting::None => "none".to_string(), + Setting::Room => "room".to_string(), + Setting::Bytes(bytes) => format!("{:.2} GiB", gib(bytes)), + }; + let ran = match rule { + None if setting == Setting::None => { + return format!("BLOCK GROUP QUEUE: {knob} · one closed group waits (the default)"); + } + None => { + return format!( + "BLOCK GROUP QUEUE: {knob} · the block proves without live regeneration: one \ + closed group waits" + ); + } + Some(Rule::Room { room, cap }) => format!( + "at most {:.2} GiB waiting while VmRSS + the group + {:.2} stays under the line {:.2} \ + GiB", + gib(cap), + gib(room.headroom), + gib(room.line) + ), + Some(Rule::Bytes(_)) => "fixed, the host not read".to_string(), + }; + let (groups, behind) = (stats.groups.load(Relaxed), stats.behind.load(Relaxed)); + let waited = if behind == 0 { + "none waited behind another".to_string() + } else { + format!( + "{behind} of {groups} groups waited behind another, the last at {:.2}s", + stats.last_behind_ms.load(Relaxed) as f64 / 1e3 + ) + }; + format!( + "BLOCK GROUP QUEUE: {knob} · {ran} · most at once {} groups {:.2} GiB · {waited} · the rule \ + refused {} groups · the drops counted at most {:.2} G cells waiting", + stats.most_groups.load(Relaxed), + gib(stats.most_bytes.load(Relaxed)), + stats.refused.load(Relaxed), + stats.most_behind_cells.load(Relaxed) as f64 / 1e9, + ) +} + +/// A closed group as the queue weighs it: its bytes as held, and its main +/// cells, each committed column's rows. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Weight { + pub(crate) bytes: u64, + pub(crate) cells: u64, +} + +/// The process's resident bytes, as the rule reads them. +type Resident = Arc Option + Send + Sync>; + +struct Queue { + rule: Rule, + resident: Resident, + stats: Arc, + made: Instant, + state: Mutex>, + changed: Condvar, +} + +struct State { + /// The groups waiting, oldest first, each with its weight, and their + /// bytes and cells. + waiting: VecDeque<(T, Weight)>, + bytes: u64, + cells: u64, + sender_gone: bool, + receiver_gone: bool, +} + +impl State { + /// The cells waiting behind the first group. + fn behind_cells(&self) -> u64 { + self.cells - self.waiting.front().map_or(0, |(_, weight)| weight.cells) + } +} + +impl Queue { + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Publishes the cells waiting behind the first, under the state's lock. + fn publish_cells(&self, state: &State) { + let behind = state.behind_cells(); + self.stats.behind_cells.store(behind, Relaxed); + self.stats.most_behind_cells.fetch_max(behind, Relaxed); + } +} + +/// A queue under `rule`: its two ends. The rule reads the process's resident +/// set ([`crate::lfm::program_budget::resident_bytes`]). +pub(crate) fn queue(rule: Rule, stats: Arc) -> (Sender, Receiver) { + queue_reading( + rule, + stats, + Arc::new(crate::lfm::program_budget::resident_bytes), + ) +} + +fn queue_reading(rule: Rule, stats: Arc, resident: Resident) -> (Sender, Receiver) { + let queue = Arc::new(Queue { + rule, + resident, + stats, + made: Instant::now(), + state: Mutex::new(State { + waiting: VecDeque::new(), + bytes: 0, + cells: 0, + sender_gone: false, + receiver_gone: false, + }), + changed: Condvar::new(), + }); + (Sender(Arc::clone(&queue)), Receiver(queue)) +} + +/// The receiver is gone: nobody takes the groups. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Stopped; + +/// The packer's end. +pub(crate) struct Sender(Arc>); + +impl Sender { + /// Hands a closed group of `weight` on, behind those waiting. While + /// another waits and the rule refuses this one, it waits here: each + /// group taken is another look at the rule. + fn send(&self, group: T, weight: Weight) -> Result<(), Stopped> { + let queue = &*self.0; + let mut state = queue.state(); + let mut refused = false; + let behind = loop { + if state.receiver_gone { + return Err(Stopped); + } + let behind = !state.waiting.is_empty(); + if !behind + || queue + .rule + .admits(state.bytes, weight.bytes, || (queue.resident)()) + { + break behind; + } + if !refused { + refused = true; + queue.stats.refused.fetch_add(1, Relaxed); + } + state = queue.changed.wait(state).unwrap_or_else(|e| e.into_inner()); + }; + state.waiting.push_back((group, weight)); + state.bytes += weight.bytes; + state.cells += weight.cells; + queue.publish_cells(&state); + let stats = &queue.stats; + stats.groups.fetch_add(1, Relaxed); + stats.most_groups.fetch_max(state.waiting.len(), Relaxed); + stats.most_bytes.fetch_max(state.bytes, Relaxed); + if behind { + stats.behind.fetch_add(1, Relaxed); + let ms = u64::try_from(queue.made.elapsed().as_millis()).unwrap_or(u64::MAX); + stats.last_behind_ms.store(ms, Relaxed); + } + drop(state); + queue.changed.notify_all(); + Ok(()) + } +} + +impl Drop for Sender { + fn drop(&mut self) { + self.0.state().sender_gone = true; + self.0.changed.notify_all(); + } +} + +/// Phase A's end: the groups in the order they were sent, until the sender is +/// gone and nothing waits. A group's cells leave the count of those waiting +/// before it is handed on. +pub(crate) struct Receiver(Arc>); + +impl Iterator for Receiver { + type Item = T; + + fn next(&mut self) -> Option { + let queue = &*self.0; + let mut state = queue.state(); + loop { + if let Some((group, weight)) = state.waiting.pop_front() { + state.bytes -= weight.bytes; + state.cells -= weight.cells; + queue.publish_cells(&state); + drop(state); + queue.changed.notify_all(); + return Some(group); + } + if state.sender_gone { + return None; + } + state = queue.changed.wait(state).unwrap_or_else(|e| e.into_inner()); + } + } +} + +impl Drop for Receiver { + fn drop(&mut self) { + self.0.state().receiver_gone = true; + self.0.changed.notify_all(); + } +} + +/// A closed group's way to phase A: the channel of one, or the queue. +pub(crate) enum Out { + Channel(std::sync::mpsc::SyncSender), + Queue(Sender), +} + +impl Out { + /// Sends `group`, waiting as its way makes it wait; only the queue weighs + /// the group. + pub(crate) fn send(&self, group: T, weigh: impl FnOnce(&T) -> Weight) -> Result<(), Stopped> { + match self { + Self::Channel(out) => out.send(group).map_err(|_| Stopped), + Self::Queue(out) => { + let weight = weigh(&group); + out.send(group, weight) + } + } + } +} + +/// The closed groups as phase A takes them, from either way. +pub(crate) enum In<'c, T> { + Channel(std::sync::mpsc::Iter<'c, T>), + Queue(Receiver), +} + +impl Iterator for In<'_, T> { + type Item = T; + + fn next(&mut self) -> Option { + match self { + Self::Channel(groups) => groups.next(), + Self::Queue(groups) => groups.next(), + } + } +} + +/// Runs `test` on a thread of its own and fails if it has not ended in a +/// minute: a queue that stops handing groups on hangs, it does not fail. +#[cfg(test)] +pub(crate) fn within_a_minute(what: &str, test: impl FnOnce() + Send + 'static) { + use std::sync::mpsc::RecvTimeoutError; + let (done, ended) = std::sync::mpsc::channel(); + let run = std::thread::spawn(move || { + test(); + let _ = done.send(()); + }); + match ended.recv_timeout(std::time::Duration::from_secs(60)) { + Ok(()) => run.join().expect("the test"), + // The test panicked: its own message. + Err(RecvTimeoutError::Disconnected) => { + std::panic::resume_unwind(run.join().expect_err("a panic")) + } + Err(RecvTimeoutError::Timeout) => panic!("{what}: the queue hung"), + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicBool; + use std::time::Duration; + + use super::*; + + const G: u64 = 1 << 30; + + fn fixed(budget: u64) -> (Sender, Receiver, Arc) { + let stats = Arc::new(Stats::default()); + let (tx, rx) = queue_reading(Rule::Bytes(budget), Arc::clone(&stats), Arc::new(|| None)); + (tx, rx, stats) + } + + /// A group of `bytes`, of twice as many main cells. + fn weight(bytes: u64) -> Weight { + Weight { + bytes, + cells: 2 * bytes, + } + } + + /// Waits for `done`, at most ten seconds. + fn until(what: &str, done: impl Fn() -> bool) { + let t = Instant::now(); + while !done() { + assert!(t.elapsed() < Duration::from_secs(10), "{what}: never"); + std::thread::sleep(Duration::from_millis(1)); + } + } + + /// A small deterministic stream of numbers. + fn stream(seed: u64) -> impl FnMut() -> u64 { + let mut x = seed; + move || { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + x >> 33 + } + } + + /// Groups leave in the order they entered, whatever waits and however + /// slowly they are taken. + #[test] + fn groups_leave_in_the_order_they_entered() { + within_a_minute("the order", in_order); + } + + fn in_order() { + let (tx, rx, stats) = fixed::(64); + let sender = std::thread::spawn(move || { + let mut bytes = stream(1); + for group in 0..2000 { + tx.send(group, weight(1 + bytes() % 8)) + .expect("the receiver is there"); + } + }); + let mut pause = stream(2); + let mut taken = Vec::new(); + for group in rx { + taken.push(group); + if pause().is_multiple_of(16) { + std::thread::sleep(Duration::from_micros(200)); + } + } + sender.join().expect("the sender"); + assert_eq!(taken, (0..2000).collect::>()); + assert!( + stats.most_groups.load(Relaxed) >= 3, + "at most {} groups waited at once: the order was never at stake", + stats.most_groups.load(Relaxed) + ); + } + + /// With no bytes to wait in, one group waits and the next stays in the + /// sender's hands until that one is taken: the channel of one. + #[test] + fn no_budget_is_one_group_waiting() { + within_a_minute("no budget", one_at_a_time); + } + + fn one_at_a_time() { + let (tx, mut rx, stats) = fixed::(0); + let sent = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&sent); + let sender = std::thread::spawn(move || { + for group in 0..4 { + tx.send(group, weight(100)).expect("the receiver is there"); + counted.fetch_add(1, Relaxed); + } + }); + for group in 0..4 { + until("the next group is sent", || sent.load(Relaxed) == group + 1); + if group < 3 { + until("the one after it is refused", || { + stats.refused.load(Relaxed) == group + 1 + }); + } + std::thread::sleep(Duration::from_millis(30)); + assert_eq!(sent.load(Relaxed), group + 1, "a second group went in"); + assert_eq!(rx.next(), Some(group)); + } + sender.join().expect("the sender"); + assert_eq!(rx.next(), None); + assert_eq!(stats.most_groups.load(Relaxed), 1); + assert_eq!(stats.behind.load(Relaxed), 0); + assert_eq!(stats.refused.load(Relaxed), 3); + // The one waiting is where the channel holds one: the drops never + // counted a cell more than by default. + assert_eq!(stats.most_behind_cells.load(Relaxed), 0); + let text = line(Setting::Bytes(0), Some(Rule::Bytes(0)), &stats); + assert!(text.contains("none waited behind another"), "{text}"); + } + + /// Never more than the budget waits, bar a group alone: one larger than + /// the budget goes when nothing waits, so no budget can stop the block. + #[test] + fn at_most_the_budget_waits_and_a_larger_group_still_goes() { + within_a_minute("the budget", under_a_budget); + } + + fn under_a_budget() { + let budget = 1000; + let (tx, rx, stats) = fixed::(budget); + let sizes: Vec = { + let mut next = stream(3); + (0..600) + .map(|k| if k % 97 == 50 { 5000 } else { 1 + next() % 400 }) + .collect() + }; + let sending = sizes.clone(); + let sender = std::thread::spawn(move || { + for bytes in sending { + tx.send(bytes, weight(bytes)) + .expect("the receiver is there"); + } + }); + // Each group as it is taken, with what still waits behind it. + let queue = Arc::clone(&rx.0); + let mut taken = Vec::new(); + let mut pause = stream(4); + for bytes in rx { + let (waiting, groups) = { + let state = queue.state(); + // The cells published are those behind the first, as read + // under the lock they are published under. + let behind: u64 = state.waiting.iter().skip(1).map(|(_, w)| w.cells).sum(); + assert_eq!(stats.behind_cells.load(Relaxed), behind, "the cells behind"); + (state.bytes, state.waiting.len()) + }; + assert!( + waiting <= budget || groups == 1, + "{groups} groups of {waiting} bytes wait, the budget is {budget}" + ); + taken.push(bytes); + if pause().is_multiple_of(4) { + std::thread::sleep(Duration::from_micros(300)); + } + } + sender.join().expect("the sender"); + assert_eq!(taken, sizes); + let most = stats.most_bytes.load(Relaxed); + assert_eq!( + most, 5000, + "the most bytes at once is the large group alone" + ); + assert!( + stats.behind.load(Relaxed) > 0, + "nothing waited behind another" + ); + assert!(stats.refused.load(Relaxed) > 0, "the budget never refused"); + } + + /// A receiver that stops wakes a sender waiting on the rule, with an + /// error; a sender that stops ends the receiver once nothing waits. + #[test] + fn either_end_stopping_ends_the_other() { + within_a_minute("an end stopping", ends_stopping); + } + + fn ends_stopping() { + let (tx, rx, _) = fixed::(0); + tx.send(0, weight(1)).expect("the first goes"); + let stopped = Arc::new(AtomicBool::new(false)); + let seen = Arc::clone(&stopped); + let sender = std::thread::spawn(move || { + let sent = tx.send(1, weight(1)); + seen.store(true, Relaxed); + sent + }); + std::thread::sleep(Duration::from_millis(30)); + assert!( + !stopped.load(Relaxed), + "the second went in behind the first" + ); + drop(rx); + until("the sender wakes", || stopped.load(Relaxed)); + assert_eq!(sender.join().expect("the sender"), Err(Stopped)); + + let (tx, rx, _) = fixed::(u64::MAX); + tx.send(7, weight(1)).expect("the receiver is there"); + tx.send(8, weight(1)).expect("the receiver is there"); + drop(tx); + assert_eq!(rx.collect::>(), [7, 8]); + } + + fn room(line_gib: f64, cap_gib: u64) -> Rule { + Rule::Room { + room: NoDisk { + line: (line_gib * GIB) as u64, + headroom: 16 * G, + }, + cap: cap_gib * G, + } + } + + /// The room rule on the readings of the large block (a group of 0.65 + /// GiB): a 118 GiB host, line 92.03 GiB, has room while the resident set + /// stays under 75.38 GiB (its highest of the commit phase was 66.34), and + /// then the cap's 16 GiB may wait; a 64 GiB host, line 45.90 GiB, has + /// none from 29.25 GiB of resident set on, which it passes nine seconds + /// in. The bytes waiting count once, in the resident set, and the cap + /// holds whatever the host reads. + #[test] + fn the_room_rule_at_the_measured_readings() { + let gib = |g: f64| (g * GIB) as u64; + let group = gib(0.65); + let admits = |rule: Rule, waiting: f64, resident: f64| { + rule.admits(gib(waiting), group, || Some(gib(resident))) + }; + let large = room(92.03, 16); + assert!(admits(large, 0.65, 66.34)); + assert!(admits(large, 9.1, 66.34)); + assert!(admits(large, 0.65, 75.3)); + assert!(!admits(large, 0.65, 75.4)); + // The bytes waiting are not added to the resident set again: the + // answer under the cap is the resident set's alone. + for resident in [66.34, 75.3, 75.4] { + assert_eq!( + admits(large, 0.0, resident), + admits(large, 15.3, resident), + "{resident}" + ); + } + // The cap, on a host with all the room. + assert!(admits(large, 15.3, 10.0)); + assert!(!admits(large, 15.4, 10.0)); + // Room is strictly under the line. + let line = Rule::Room { + room: NoDisk { + line: 40 * G, + headroom: 16 * G, + }, + cap: 16 * G, + }; + assert!(!line.admits(4 * G, G, || Some(23 * G))); + assert!(line.admits(4 * G, G, || Some(23 * G - 1))); + let small = room(45.90, 16); + assert!(admits(small, 0.65, 25.62)); + assert!(admits(small, 3.0, 29.2)); + assert!(!admits(small, 0.65, 29.34)); + assert!(!admits(small, 0.0, 36.08)); + // No reading is no room. + assert!(!large.admits(0, group, || None)); + // A fixed budget does not read the host. + assert!(Rule::Bytes(G).admits(0, G, || unreachable!("the host is not read"))); + assert!(!Rule::Bytes(G).admits(1, G, || unreachable!("the host is not read"))); + } + + /// The room rule in the queue: groups wait behind one another while the + /// resident set leaves room, and one at a time once it does not. + #[test] + fn the_queue_follows_the_resident_set() { + within_a_minute("the resident set", following_the_resident_set); + } + + fn following_the_resident_set() { + let resident = Arc::new(AtomicU64::new(10 * G)); + let reading = Arc::clone(&resident); + let stats = Arc::new(Stats::default()); + let (tx, mut rx) = queue_reading::( + room(45.90, 16), + Arc::clone(&stats), + Arc::new(move || Some(reading.load(Relaxed))), + ); + let sent = Arc::new(AtomicUsize::new(0)); + let go = Arc::new(AtomicBool::new(false)); + let (counted, gone) = (Arc::clone(&sent), Arc::clone(&go)); + let sender = std::thread::spawn(move || { + for group in 0..6 { + if group == 3 { + while !gone.load(Relaxed) { + std::thread::sleep(Duration::from_millis(1)); + } + } + tx.send(group, weight(G / 2)) + .expect("the receiver is there"); + counted.fetch_add(1, Relaxed); + } + }); + // Room: three go in behind one another, none refused. + until("three are sent", || sent.load(Relaxed) == 3); + assert_eq!(stats.behind.load(Relaxed), 2); + assert_eq!(stats.refused.load(Relaxed), 0); + // No room from here: the fourth waits in the sender's hands until + // nothing waits, as with the channel of one, and so does each after. + resident.store(36 * G, Relaxed); + go.store(true, Relaxed); + until("the fourth is refused", || stats.refused.load(Relaxed) == 1); + for group in 0..3 { + std::thread::sleep(Duration::from_millis(10)); + assert_eq!(sent.load(Relaxed), 3, "a group went in with no room"); + assert_eq!(rx.next(), Some(group)); + } + for group in 3..6 { + until("the next is sent", || sent.load(Relaxed) == group + 1); + if group < 5 { + until("the one after it is refused", || { + stats.refused.load(Relaxed) == group - 1 + }); + } + std::thread::sleep(Duration::from_millis(10)); + assert_eq!( + sent.load(Relaxed), + group + 1, + "a second went in with no room" + ); + assert_eq!(rx.next(), Some(group)); + } + sender.join().expect("the sender"); + assert_eq!(rx.next(), None); + assert_eq!(stats.behind.load(Relaxed), 2); + assert_eq!(stats.refused.load(Relaxed), 3); + assert_eq!(stats.most_groups.load(Relaxed), 3); + } + + /// The knob: unset, empty and `none` are the channel of one; `room` and a + /// number of GiB are read; anything else refuses the prove. + #[test] + fn the_knob_is_none_by_default_and_refuses_what_it_does_not_know() { + assert_eq!(parse_setting(None).unwrap(), Setting::None); + assert_eq!(parse_setting(Some("")).unwrap(), Setting::None); + assert_eq!(parse_setting(Some("none")).unwrap(), Setting::None); + assert_eq!(parse_setting(Some(" room ")).unwrap(), Setting::Room); + assert_eq!(parse_setting(Some("16")).unwrap(), Setting::Bytes(16 * G)); + assert_eq!(parse_setting(Some("0.5")).unwrap(), Setting::Bytes(G / 2)); + for bad in ["on", "auto", "0", "-1", "inf", "nan", "room,16", "1e400"] { + assert!(parse_setting(Some(bad)).is_err(), "{bad} was taken"); + } + } + + /// The rule a block runs: none by default, none under any setting for a + /// block that proves without live regeneration, the purge rule's line and + /// headroom under `room`. + #[test] + fn only_a_no_disk_block_runs_a_rule() { + let no_disk = NoDisk { + line: 92 * G, + headroom: 16 * G, + }; + for setting in [Setting::None, Setting::Room, Setting::Bytes(G)] { + assert_eq!(Rule::of(setting, None, 16 * G), None, "{setting:?}"); + } + assert_eq!(Rule::of(Setting::None, Some(no_disk), 16 * G), None); + assert_eq!( + Rule::of(Setting::Room, Some(no_disk), 16 * G), + Some(Rule::Room { + room: no_disk, + cap: 16 * G + }) + ); + assert_eq!( + Rule::of(Setting::Bytes(G), Some(no_disk), 16 * G), + Some(Rule::Bytes(G)) + ); + } + + /// The line a block prints. + #[test] + fn the_line_says_what_ran_and_what_waited() { + let stats = Stats::default(); + assert_eq!( + line(Setting::None, None, &stats), + "BLOCK GROUP QUEUE: none · one closed group waits (the default)" + ); + assert_eq!( + line(Setting::Room, None, &stats), + "BLOCK GROUP QUEUE: room · the block proves without live regeneration: one closed \ + group waits" + ); + stats.groups.store(182, Relaxed); + stats.most_groups.store(1, Relaxed); + stats.most_bytes.store(G / 2, Relaxed); + stats.refused.store(181, Relaxed); + assert_eq!( + line(Setting::Room, Some(room(45.90, 16)), &stats), + "BLOCK GROUP QUEUE: room · at most 16.00 GiB waiting while VmRSS + the group + 16.00 \ + stays under the line 45.90 GiB · most at once 1 groups 0.50 GiB · none waited behind \ + another · the rule refused 181 groups · the drops counted at most 0.00 G cells waiting" + ); + stats.behind.store(96, Relaxed); + stats.most_groups.store(13, Relaxed); + stats.most_bytes.store(8 * G, Relaxed); + stats.refused.store(41, Relaxed); + stats.last_behind_ms.store(58_420, Relaxed); + stats.most_behind_cells.store(4_496_000_000, Relaxed); + assert_eq!( + line(Setting::Bytes(8 * G), Some(Rule::Bytes(8 * G)), &stats), + "BLOCK GROUP QUEUE: 8.00 GiB · fixed, the host not read · most at once 13 groups 8.00 GiB · \ + 96 of 182 groups waited behind another, the last at 58.42s · the rule refused 41 groups · \ + the drops counted at most 4.50 G cells waiting" + ); + } + + /// The channel of one and the queue hand the same groups on in the same + /// order through [`Out`] and [`In`]; only the queue weighs them. + #[test] + fn both_ways_hand_the_same_groups_on() { + within_a_minute("both ways", both_ways); + } + + fn both_ways() { + let groups: Vec> = (0..50u8).map(|k| vec![k; usize::from(k % 7)]).collect(); + let through = |out: Out>, weigh: bool| { + let sending = groups.clone(); + std::thread::spawn(move || { + for group in sending { + out.send(group, |group| { + assert!(weigh, "the channel weighed a group"); + weight(group.len() as u64) + }) + .expect("the receiver is there"); + } + }) + }; + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let sender = through(Out::Channel(tx), false); + assert_eq!(In::Channel(rx.iter()).collect::>(), groups); + sender.join().expect("the sender"); + for budget in [0, 1, u64::MAX] { + let (tx, rx, _) = fixed(budget); + let sender = through(Out::Queue(tx), true); + assert_eq!(In::Queue(rx).collect::>(), groups, "budget {budget}"); + sender.join().expect("the sender"); + } + } + + /// By default the rule that arms the drops weighs the committed cells + /// alone, exactly: the knob unset, empty or `none` runs no rule, and with + /// no rule the queue is not read, whatever it would say. So does a block + /// with a store, under every setting. With a rule, the cells behind the + /// first count. + #[test] + fn by_default_the_drops_count_the_committed_cells_alone() { + let stats = Stats::default(); + stats.behind_cells.store(4_123_000_000, Relaxed); + let no_disk = NoDisk { + line: 92 * G, + headroom: 16 * G, + }; + for value in [None, Some(""), Some("none"), Some(" none ")] { + let rule = Rule::of(parse_setting(value).unwrap(), Some(no_disk), 16 * G); + assert_eq!(rule, None, "{value:?}"); + for committed in [0, 1, 22_790_000_000, u64::MAX] { + assert_eq!( + arming_cells(rule, &stats, committed), + committed, + "{value:?}" + ); + } + } + for setting in [Setting::Room, Setting::Bytes(G)] { + let store = Rule::of(setting, None, 16 * G); + assert_eq!(arming_cells(store, &stats, 22_790_000_000), 22_790_000_000); + let no_disk = Rule::of(setting, Some(no_disk), 16 * G); + assert_eq!( + arming_cells(no_disk, &stats, 22_790_000_000), + 26_913_000_000 + ); + } + } + + /// A group's cells are counted once: behind the first while it waits + /// there, then by phase A's commit once taken, never both. Phase A + /// commits each group as it takes it here, so the drops count every cell + /// sent but the first waiting group's, whatever was sent and taken. + #[test] + fn a_groups_cells_are_counted_once() { + within_a_minute("counted once", counted_once); + } + + fn counted_once() { + let (tx, mut rx, stats) = fixed::(u64::MAX); + let rule = Some(Rule::Bytes(u64::MAX)); + let cells: Vec = (0..40u64).map(|k| 300_000_000 + k * 7_000_000).collect(); + let mut next = stream(5); + let (mut sent, mut taken, mut committed, mut most) = (0, 0, 0u64, 0u64); + while taken < cells.len() { + if sent < cells.len() && (sent == taken || !next().is_multiple_of(3)) { + let weight = Weight { + bytes: 1, + cells: cells[sent], + }; + tx.send(sent, weight).expect("the receiver is there"); + sent += 1; + } else { + assert_eq!(rx.next(), Some(taken)); + committed += cells[taken]; + taken += 1; + } + let first = if taken < sent { cells[taken] } else { 0 }; + let all: u64 = cells[..sent].iter().sum(); + assert_eq!( + arming_cells(rule, &stats, committed) + first, + all, + "{sent} sent, {taken} taken" + ); + most = most.max(all - committed - first); + } + assert_eq!(stats.behind_cells.load(Relaxed), 0); + assert_eq!(stats.most_behind_cells.load(Relaxed), most); + assert!( + most >= 3 * cells[0], + "at most {most} cells waited behind the first" + ); + drop(tx); + assert_eq!(rx.next(), None); + } + + /// The rule that arms the drops on the large block's readings (a 118 GiB + /// host, target 108.28 GiB, a group of 374.8 M cells): by default it + /// armed with 61 groups committed, at a host of about 64.5 GiB, with one + /// group in the channel and one in the packer's hands. With the queue the + /// same tables built can be 51 committed, one first in the queue and + /// eleven behind it. Counted, the eleven make it arm within a group's + /// reserve of the default's host (0.62 GiB: the packer's group now counts); + /// not counted, about 6 GiB higher. + #[test] + fn the_drops_arm_at_the_tables_built_they_arm_at_by_default() { + use crate::block_whir::{MEMORY_RESERVE_PER_G, drop_wanted}; + let gib = |g: f64| (g * GIB) as u64; + let target = gib(108.28); + let group = 374_800_000u64; + // The least host, in steps of 0.01 GiB, at which the rule arms with + // `cells` counted as committed, for a table of 0.1 GiB. + let arms_from = |cells: u64| -> f64 { + (0..12_000u32) + .map(|k| f64::from(k) / 100.0) + .find(|&host| { + drop_wanted(target, MEMORY_RESERVE_PER_G, cells, gib(0.1), || gib(host)) + }) + .expect("it arms under the target") + }; + let (tx, _rx, stats) = fixed::(u64::MAX); + for g in 0..12 { + let weight = Weight { + bytes: gib(0.65), + cells: group, + }; + tx.send(g, weight).expect("the receiver is there"); + } + let committed = 51 * group; + let counted = arming_cells(Some(Rule::Bytes(u64::MAX)), &stats, committed); + assert_eq!(counted, 62 * group); + let default = arms_from(61 * group); + let (with, without) = (arms_from(counted), arms_from(committed)); + assert!( + (default - 64.5).abs() < 0.2, + "the default arms from {default}" + ); + assert!( + with <= default && default - with <= 1.0, + "counted, it arms from {with} GiB against {default}" + ); + assert!( + without - default > 1.0, + "not counted, it arms from {without} GiB against {default}" + ); + } + + /// On a 64 GiB host (target 54 GiB, the pressure line 45.90): groups wait + /// behind one another early in the block, but the room rule refuses any + /// from a resident set of 29.25 GiB on, which the large block passes + /// about nine seconds in, and the drops arm near 33.2 GiB, at ten. By + /// then one group waits at most, and the drops count the committed cells + /// alone: the default's arming, exactly. + #[test] + fn on_a_64_gib_host_the_drops_arm_as_by_default() { + within_a_minute("a 64 GiB host", on_a_64_gib_host); + } + + fn on_a_64_gib_host() { + use crate::block_whir::{MEMORY_RESERVE_PER_G, drop_wanted}; + let gib = |g: f64| (g * GIB) as u64; + let group = Weight { + bytes: gib(0.65), + cells: 374_800_000, + }; + let resident = Arc::new(AtomicU64::new(gib(25.62))); + let reading = Arc::clone(&resident); + let stats = Arc::new(Stats::default()); + let rule = room(45.90, 16); + let (tx, mut rx) = queue_reading::( + rule, + Arc::clone(&stats), + Arc::new(move || Some(reading.load(Relaxed))), + ); + let sent = Arc::new(AtomicUsize::new(0)); + let go = Arc::new(AtomicBool::new(false)); + let (counted, gone) = (Arc::clone(&sent), Arc::clone(&go)); + let sender = std::thread::spawn(move || { + for g in 0..5 { + if g == 3 { + while !gone.load(Relaxed) { + std::thread::sleep(Duration::from_millis(1)); + } + } + tx.send(g, group).expect("the receiver is there"); + counted.fetch_add(1, Relaxed); + } + }); + let rule = Some(rule); + // Early in the block: three wait, the two behind the first count. + until("three are sent", || sent.load(Relaxed) == 3); + let mut committed = 23 * group.cells; + assert_eq!( + arming_cells(rule, &stats, committed), + committed + 2 * group.cells + ); + // Past the room: the fourth is refused until nothing waits. + resident.store(gib(33.3), Relaxed); + go.store(true, Relaxed); + until("the fourth is refused", || stats.refused.load(Relaxed) == 1); + let decide = |cells: u64| { + drop_wanted(gib(54.0), MEMORY_RESERVE_PER_G, cells, gib(0.1), || { + gib(33.3) + }) + }; + for g in 0..5 { + assert_eq!(rx.next(), Some(g)); + committed += group.cells; + let behind = if g == 0 { group.cells } else { 0 }; + assert_eq!( + arming_cells(rule, &stats, committed), + committed + behind, + "after group {g}" + ); + } + // The readings arm by default, and the queue's count is the same. + assert!(decide(committed)); + assert!(!decide(23 * group.cells)); + assert_eq!( + decide(arming_cells(rule, &stats, committed)), + decide(committed) + ); + sender.join().expect("the sender"); + assert_eq!(rx.next(), None); + assert_eq!(stats.most_behind_cells.load(Relaxed), 2 * group.cells); + } +} diff --git a/prover/src/block_whir/memlog.rs b/prover/src/block_whir/memlog.rs new file mode 100644 index 000000000..79b73e00f --- /dev/null +++ b/prover/src/block_whir/memlog.rs @@ -0,0 +1,613 @@ +//! `LAMBDA_VM_BLOCK_MEMLOG=1` ([`super::BlockOptions::memlog`]): where a +//! streamed block's host memory is, by term, as the data moves. +//! +//! A sampler prints a `BLOCK MEM` line every half second, and one at each +//! mark (the executor's end, the windows walked, each phase of the finish, the +//! rest laid out, each group committed and installed, phase A's end, each +//! phase-B group's end, the proof). A line carries the process's resident set +//! and minor faults, jemalloc's allocated / active / resident / mapped / +//! retained bytes (in the lib's tests, which run jemalloc, as the shipped +//! binary does), `rss − resident`, the pinned staging buffers, every term, +//! their sum (`named`) and `unnamed` = allocated − named. `rss − resident` is +//! often below zero: jemalloc counts a huge extent's pages as resident before +//! they are touched, so neither it nor `active` is a lower bound on the +//! resident set; the retention is `rss most − peak active`. Every 0.1 s the sampler reads the +//! allocator; the line at the highest `active` it saw is printed again at the +//! end with each arena's active and dirty bytes and the threads bound to it: +//! under a never-purge posture the resident set only grows, so its peak names +//! no instant, and `active`'s does. +//! +//! It reads sizes only: no trace, no transcript, no proof byte depends on it. + +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Instant; + +use stark::multilinear_block::BlockMem; + +const GIB: f64 = (1u64 << 30) as f64; + +fn gib(bytes: usize) -> f64 { + bytes as f64 / GIB +} + +/// What `LAMBDA_VM_BLOCK_MEMLOG=1` turns on. Read where the options are made. +pub(crate) fn from_env() -> bool { + std::env::var("LAMBDA_VM_BLOCK_MEMLOG").is_ok_and(|v| v.trim() == "1") +} + +/// The block's host bytes by where they are; every list at its capacity. +pub(crate) struct Ledger { + start: Instant, + /// The committer's terms: the group in its commit, the groups being + /// packed, the committed tables as held, the kept tree tops. + pub(crate) block: Arc, + /// The executor's memory and its own window of logs. + pub(crate) exec: AtomicUsize, + /// Windows of logs on their way to the walker or held by it. + pub(crate) logs: AtomicUsize, + /// The walk's carried memory state. + pub(crate) walk: AtomicUsize, + /// Walked windows on their way to the accumulator. + pub(crate) walked: AtomicUsize, + /// The builder's run so far (kept tails or windows, routed segments, + /// what was counted ahead), as of the last window absorbed. + pub(crate) builder: AtomicUsize, + /// The initial memory image the builder keeps. + pub(crate) image: AtomicUsize, + /// Chunk jobs' ops queued for the layout thread. + pub(crate) jobs: AtomicUsize, + /// The chunk the layout thread is generating and laying out: its ops, + /// then its trace. + pub(crate) laying: AtomicUsize, + /// The packer's open group, laid out (column-major), eight bytes a cell. + pub(crate) open: AtomicUsize, + /// Closed groups not yet taken by phase A's committer. + pub(crate) sent: AtomicUsize, + /// The finish's tables, row-major, until each is laid out. + pub(crate) rest: AtomicUsize, + /// The rest's tables laid out and not yet placed in a group. + pub(crate) rest_laid: AtomicUsize, + /// The prepared tables' columns (DECODE's, the dense genesis pages'). + pub(crate) prepared: AtomicUsize, + peak: Mutex>, + /// The highest resident set a sample saw, and when. + rss_most: Mutex<(usize, f64)>, + threads: Mutex>, +} + +/// The sample at the highest `active`: its line and every arena then. +struct Peak { + active: usize, + at: f64, + line: String, + terms: Vec<(&'static str, usize)>, + unnamed: i64, + arenas: Vec, +} + +/// One jemalloc arena's active, dirty and muzzy bytes. +#[derive(Clone, Copy)] +#[cfg_attr(not(test), allow(dead_code))] +struct ArenaBytes { + arena: u32, + active: usize, + dirty: usize, + muzzy: usize, +} + +impl Ledger { + /// A ledger on the prove's clock `start`; the committer's terms mark + /// their events with a line of this ledger. + pub(crate) fn new(start: Instant) -> Arc { + Arc::new_cyclic(|this: &Weak| { + let this = this.clone(); + Ledger { + start, + block: Arc::new(BlockMem::new(move |label| { + if let Some(ledger) = this.upgrade() { + ledger.line(label); + } + })), + exec: AtomicUsize::new(0), + logs: AtomicUsize::new(0), + walk: AtomicUsize::new(0), + walked: AtomicUsize::new(0), + builder: AtomicUsize::new(0), + image: AtomicUsize::new(0), + jobs: AtomicUsize::new(0), + laying: AtomicUsize::new(0), + open: AtomicUsize::new(0), + sent: AtomicUsize::new(0), + rest: AtomicUsize::new(0), + rest_laid: AtomicUsize::new(0), + prepared: AtomicUsize::new(0), + peak: Mutex::new(None), + rss_most: Mutex::new((0, 0.0)), + threads: Mutex::new(Vec::new()), + } + }) + } + + /// Every term, in a fixed order. + pub(crate) fn terms(&self) -> Vec<(&'static str, usize)> { + let b = &self.block; + [ + ("exec", &self.exec), + ("logs", &self.logs), + ("walk", &self.walk), + ("walked", &self.walked), + ("builder", &self.builder), + ("image", &self.image), + ("jobs", &self.jobs), + ("laying", &self.laying), + ("open", &self.open), + ("sent", &self.sent), + ("rest", &self.rest), + ("rest_laid", &self.rest_laid), + ("prepared", &self.prepared), + ("committing", &b.committing), + ("ahead", &b.ahead), + ("pack_wide", &b.packing_wide), + ("pack_ready", &b.packed_ready), + ("held_narrow", &b.held_narrow), + ("held_wide", &b.held_wide), + ("tops", &b.tree_tops), + ] + .into_iter() + .map(|(name, term)| (name, term.load(Relaxed))) + .collect() + } + + /// The line for `label` now, and the allocator's active bytes (0 when + /// they are not read). + fn sample(&self, label: &str) -> (String, usize, Vec<(&'static str, usize)>, i64) { + let t = self.start.elapsed().as_secs_f64(); + let rss = proc_rss_bytes(); + if let Some(rss) = rss + && let Ok(mut most) = self.rss_most.lock() + && rss > most.0 + { + *most = (rss, t); + } + let terms = self.terms(); + let named: usize = terms.iter().map(|&(_, b)| b).sum(); + let heap = heap_stats(); + let unnamed = heap.map_or(0, |[allocated, ..]| allocated as i64 - named as i64); + let signed = |b: i64| b as f64 / GIB; + let mut line = format!( + "BLOCK MEM {label} t={t:.2} · rss {} · minflt {} M", + rss.map_or("n/a".to_string(), |b| format!("{:.2}", gib(b))), + proc_minor_faults().map_or("n/a".to_string(), |f| format!("{:.2}", f as f64 / 1e6)), + ); + match heap { + Some([allocated, active, resident, mapped, retained]) => { + line.push_str(&format!( + " · alloc {:.2} active {:.2} resident {:.2} mapped {:.2} retained {:.2} · \ + rss − resident {:+.2} (pinned {:.2}) · named {:.2} · unnamed {:+.2}", + gib(allocated), + gib(active), + gib(resident), + gib(mapped), + gib(retained), + rss.map_or(0.0, |r| signed(r as i64 - resident as i64)), + gib(multilinear::gpu::pinned_host_bytes()), + gib(named), + signed(unnamed), + )); + } + None => line.push_str(&format!( + " · heap n/a · pinned {:.2} · named {:.2}", + gib(multilinear::gpu::pinned_host_bytes()), + gib(named), + )), + } + let listed: Vec = terms + .iter() + .map(|(name, b)| format!("{name} {:.2}", gib(*b))) + .collect(); + line.push_str(&format!(" || {} (GiB)", listed.join(" · "))); + let active = heap.map_or(0, |[_, active, ..]| active); + (line, active, terms, unnamed) + } + + /// Keeps the sample if its `active` is the highest so far. + fn consider( + &self, + line: String, + active: usize, + terms: Vec<(&'static str, usize)>, + unnamed: i64, + ) { + if active == 0 { + return; + } + let Ok(mut peak) = self.peak.lock() else { + return; + }; + if peak.as_ref().is_some_and(|p| p.active >= active) { + return; + } + *peak = Some(Peak { + active, + at: self.start.elapsed().as_secs_f64(), + line, + terms, + unnamed, + arenas: arena_bytes(), + }); + } + + /// A line now, labelled. + pub(crate) fn line(&self, label: &str) { + let (line, active, terms, unnamed) = self.sample(label); + eprintln!("{line}"); + self.consider(line, active, terms, unnamed); + } + + /// A sampler's tick: the allocator read, the line printed when `print`. + fn tick(&self, print: bool) { + let (line, active, terms, unnamed) = self.sample("tick"); + if print { + eprintln!("{line}"); + } + self.consider(line, active, terms, unnamed); + } + + /// One line of `what`'s parts, largest first, those of 0.01 GiB or more. + pub(crate) fn parts(&self, what: &str, mut parts: Vec<(String, usize)>) { + let total: usize = parts.iter().map(|(_, b)| b).sum(); + parts.sort_by(|a, b| b.1.cmp(&a.1)); + let listed: Vec = parts + .iter() + .filter(|(_, b)| gib(*b) >= 0.01) + .map(|(name, b)| format!("{name} {:.2}", gib(*b))) + .collect(); + eprintln!( + "BLOCK MEM {what} parts: {:.2} GiB · {}", + gib(total), + listed.join(" · ") + ); + } + + /// One line of `what`'s parts' largest single allocations, the six + /// largest first, in MiB. + pub(crate) fn largest(&self, what: &str, mut parts: Vec<(String, usize)>) { + parts.sort_by(|a, b| b.1.cmp(&a.1)); + let listed: Vec = parts + .iter() + .take(6) + .map(|(name, b)| format!("{name} {:.1}", *b as f64 / (1u64 << 20) as f64)) + .collect(); + eprintln!("BLOCK MEM {what} largest (MiB): {}", listed.join(" · ")); + } + + /// Records the arena the calling thread allocates from, as `name`. + pub(crate) fn thread(&self, name: &str) { + if let (Some(arena), Ok(mut threads)) = (thread_arena(), self.threads.lock()) { + threads.push((name.to_string(), arena)); + } + } + + /// Records the arenas the global rayon pool's workers allocate from. + pub(crate) fn pool(&self) { + let arenas = rayon::broadcast(|_| thread_arena()); + if let Ok(mut threads) = self.threads.lock() { + threads.extend( + arenas + .into_iter() + .flatten() + .map(|a| ("rayon".to_string(), a)), + ); + } + } + + /// The peak sample again, its terms largest first, each arena's bytes at + /// the peak and now, the threads bound to each, and the highest resident + /// set a sample saw. + pub(crate) fn report(&self) { + let threads = self.threads.lock().map(|t| t.clone()).unwrap_or_default(); + let who = |arena: u32| -> String { + let mut names: Vec<(String, usize)> = Vec::new(); + for (name, a) in &threads { + if *a != arena { + continue; + } + match names.iter_mut().find(|(n, _)| n == name) { + Some((_, count)) => *count += 1, + None => names.push((name.clone(), 1)), + } + } + let names: Vec = names + .into_iter() + .map(|(n, c)| if c > 1 { format!("{n}×{c}") } else { n }) + .collect(); + names.join(",") + }; + let arenas_line = |arenas: &[ArenaBytes]| -> String { + let mut arenas = arenas.to_vec(); + arenas.sort_by(|a, b| (b.active + b.dirty).cmp(&(a.active + a.dirty))); + let total = |f: fn(&ArenaBytes) -> usize| gib(arenas.iter().map(f).sum()); + let top: Vec = arenas + .iter() + .take(12) + .map(|a| { + format!( + "a{} [{}] {:.2} · {:.2} · {:.2}", + a.arena, + who(a.arena), + gib(a.active), + gib(a.dirty), + gib(a.muzzy) + ) + }) + .collect(); + format!( + "{} arenas, Σ active {:.2} · dirty {:.2} · muzzy {:.2} | {}", + arenas.len(), + total(|a| a.active), + total(|a| a.dirty), + total(|a| a.muzzy), + top.join(" | ") + ) + }; + if let Ok(peak) = self.peak.lock() + && let Some(peak) = peak.as_ref() + { + eprintln!( + "BLOCK MEM PEAK active {:.2} GiB at t={:.2}: {}", + gib(peak.active), + peak.at, + peak.line + ); + let mut terms = peak.terms.clone(); + terms.sort_by(|a, b| b.1.cmp(&a.1)); + let listed: Vec = terms + .iter() + .filter(|(_, b)| gib(*b) >= 0.01) + .map(|(name, b)| format!("{name} {:.2}", gib(*b))) + .collect(); + eprintln!( + "BLOCK MEM PEAK TERMS (GiB): {} · unnamed {:+.2}", + listed.join(" · "), + peak.unnamed as f64 / GIB + ); + eprintln!( + "BLOCK MEM PEAK ARENAS (active · dirty · muzzy GiB): {}", + arenas_line(&peak.arenas) + ); + } + let _ = heap_stats(); + eprintln!( + "BLOCK MEM END ARENAS (active · dirty · muzzy GiB): {}", + arenas_line(&arena_bytes()) + ); + let mut bound: Vec = Vec::new(); + for (name, arena) in &threads { + bound.push(format!("{name} a{arena}")); + } + eprintln!("BLOCK MEM THREADS: {}", bound.join(" · ")); + if let Ok(most) = self.rss_most.lock() { + eprintln!( + "BLOCK MEM RSS MOST: {:.2} GiB, first at t={:.2}", + gib(most.0), + most.1 + ); + if let Ok(peak) = self.peak.lock() + && let Some(peak) = peak.as_ref() + && most.0 > 0 + { + eprintln!( + "BLOCK MEM RETENTION: rss most − peak active {:+.2} GiB (the allocator's freed \ + pages kept beyond the live peak)", + (most.0 as f64 - peak.active as f64) / GIB + ); + } + } + } +} + +/// The bytes a window of logs takes, at its capacity. +pub(crate) fn logs_bytes(logs: &Vec) -> usize { + crate::tables::trace_builder::vec_heap_bytes(logs) +} + +/// A committed table's columns as held on the host: packed, as records, or +/// eight bytes a cell. +pub(crate) fn table_bytes( + table: &stark::multilinear_table::CommittedTable<'_, super::F, super::E>, +) -> usize { + table.compact_bytes().unwrap_or_else(|| { + (table.num_committed_columns() << table.num_vars()) * std::mem::size_of::() + }) +} + +/// A group's tables as held. +pub(crate) fn group_bytes( + group: &[stark::multilinear_table::CommittedTable<'_, super::F, super::E>], +) -> usize { + group.iter().map(table_bytes).sum() +} + +/// A trace table's main columns as held: eight bytes a cell, packed, or as +/// records. +pub(crate) fn rows_bytes(trace: &stark::trace::TraceTable) -> usize { + if let Some(records) = trace.records_main() { + return records.bytes(); + } + trace.narrow_main().map_or_else( + || trace.main_table.width * trace.main_table.height * std::mem::size_of::(), + |packed| packed.data().len(), + ) +} + +/// The ledger's tick every 0.1 s, and its line every half second, on a +/// thread of its own until dropped. +pub(crate) struct Sampler { + stop: Arc, + handle: Option>, +} + +impl Sampler { + pub(crate) fn start(ledger: Arc) -> Self { + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = stop.clone(); + let handle = std::thread::Builder::new() + .name("block-memlog".to_string()) + .spawn(move || { + let mut n = 0usize; + while !flag.load(Relaxed) { + std::thread::park_timeout(std::time::Duration::from_millis(100)); + if !flag.load(Relaxed) { + n += 1; + ledger.tick(n.is_multiple_of(5)); + } + } + }) + .ok(); + Self { stop, handle } + } +} + +impl Drop for Sampler { + fn drop(&mut self) { + self.stop.store(true, Relaxed); + if let Some(handle) = self.handle.take() { + handle.thread().unpark(); + let _ = handle.join(); + } + } +} + +/// This process's minor page faults so far (`/proc/self/stat`, field 10), on +/// Linux: each a first touch of a page the allocator had not kept. +fn proc_minor_faults() -> Option { + let stat = std::fs::read_to_string("/proc/self/stat").ok()?; + // The command name may hold spaces; the fields after it do not. + let after = stat.rsplit_once(')')?.1; + after.split_whitespace().nth(7)?.parse().ok() +} + +/// This process's resident set (`VmRSS`), on Linux. +fn proc_rss_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let kib: usize = status + .lines() + .find(|l| l.starts_with("VmRSS:"))? + .split_whitespace() + .nth(1)? + .parse() + .ok()?; + Some(kib * 1024) +} + +/// jemalloc's allocated, active, resident, mapped and retained bytes, with +/// the statistics refreshed: in the lib's tests, which install jemalloc as +/// the allocator (`lib.rs`). +#[cfg(test)] +fn heap_stats() -> Option<[usize; 5]> { + use tikv_jemalloc_ctl::{epoch, stats}; + // The statistics are cached until the epoch turns. + epoch::advance().ok()?; + Some([ + stats::allocated::read().ok()?, + stats::active::read().ok()?, + stats::resident::read().ok()?, + stats::mapped::read().ok()?, + stats::retained::read().ok()?, + ]) +} + +/// Outside the lib's tests the allocator's statistics are not read. +#[cfg(not(test))] +fn heap_stats() -> Option<[usize; 5]> { + None +} + +/// Each initialised arena's active, dirty and muzzy bytes, as of the last +/// epoch ([`heap_stats`] turns it). +#[cfg(test)] +fn arena_bytes() -> Vec { + use tikv_jemalloc_ctl::raw; + // SAFETY: each key's type is jemalloc's documented one (`unsigned` for + // `arenas.narenas`, `size_t` for the page size and the page counts). + let (narenas, page) = unsafe { + ( + raw::read::(b"arenas.narenas\0").unwrap_or(0), + raw::read::(b"arenas.page\0").unwrap_or(4096), + ) + }; + (0..narenas) + .filter_map(|arena| { + let pages = |what: &str| -> Option { + let key = format!("stats.arenas.{arena}.{what}\0"); + // SAFETY: the page counts are `size_t`. + unsafe { raw::read::(key.as_bytes()).ok() } + }; + Some(ArenaBytes { + arena, + active: pages("pactive")? * page, + dirty: pages("pdirty")? * page, + muzzy: pages("pmuzzy")? * page, + }) + }) + .filter(|a| a.active + a.dirty + a.muzzy > 0) + .collect() +} + +#[cfg(not(test))] +fn arena_bytes() -> Vec { + Vec::new() +} + +/// The arena the calling thread allocates from. +#[cfg(test)] +fn thread_arena() -> Option { + // SAFETY: `thread.arena` is an `unsigned`. + unsafe { tikv_jemalloc_ctl::raw::read::(b"thread.arena\0").ok() } +} + +#[cfg(not(test))] +fn thread_arena() -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A line carries the allocator's statistics and every term, and the + /// peak keeps the highest `active` seen. + #[test] + fn a_line_names_every_term_and_the_peak_is_kept() { + let ledger = Ledger::new(Instant::now()); + ledger.thread("test"); + ledger.open.store(3 << 30, Relaxed); + let (line, active, terms, unnamed) = ledger.sample("test"); + assert!( + active > 0, + "jemalloc's statistics are read in the lib's tests" + ); + assert!( + line.contains("alloc ") && line.contains("unnamed "), + "{line}" + ); + assert!(line.contains("open 3.00"), "{line}"); + assert_eq!(terms.len(), 20); + assert!( + unnamed < 0, + "a term larger than the heap leaves unnamed below zero" + ); + ledger.consider(line, active, terms.clone(), unnamed); + ledger.consider("lower".to_string(), 1, terms, unnamed); + { + let peak = ledger.peak.lock().expect("the peak"); + let peak = peak.as_ref().expect("a peak"); + assert_eq!(peak.active, active); + assert!(peak.line.contains("open 3.00")); + assert!(!peak.arenas.is_empty(), "the arenas are read"); + } + assert!(!ledger.threads.lock().expect("threads").is_empty()); + ledger.report(); + } +} diff --git a/prover/src/block_whir/regen.rs b/prover/src/block_whir/regen.rs new file mode 100644 index 000000000..912679498 --- /dev/null +++ b/prover/src/block_whir/regen.rs @@ -0,0 +1,3426 @@ +//! Regeneration on the WHIR block (D-WHIR-NODISK, #1013's D-REGEN): the shadow +//! regenerator, and live regeneration. +//! +//! `LAMBDA_VM_BLOCK_REGEN=shadow` runs the sequential regenerator beside phase +//! B and drops nothing. Phase A records, per streamed chunk, a [`Recipe`]: +//! which chunk it is, the windows it spans, its place in the hand-out (its +//! rank, which is also its place in the packer's order, so in the groups), and +//! the digest of its packed columns taken on the layout thread that generated +//! them. Phase B re-executes the run from the start on threads of its own, +//! walks it with the regeneration walk ([`RegenBuilder`]), generates every +//! chunk that has a recipe as phase A did, checks its digest, and throws it +//! away. It reports the mismatches, its CPU, and when each chunk would have +//! been ready against phase B's groups as they ran, in the order they ran and +//! with the groups that hold no streamed chunk first ([`Plan`]). The proof is +//! the one the knob's absence makes. +//! +//! `auto` and `always` are live regeneration (#1013's R2, its no-disk P1): +//! phase A drops a streamed chunk instead of keeping it once the `auto` rule +//! would move a table off the host (`always`: every one), and +//! phase B's regenerator ([`run_live`]) deposits each into its slot, paced by +//! the groups that take them ([`stark::multilinear_block::BlockRegen`]). + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use stark::regen::{RegenError, RegenProducer, RegenSlot, RegenWindow}; + +use crate::Error; +use crate::tables::gpack::TraceForm; +use crate::tables::trace_builder::{ChunkJob, RegenBuilder, RegenFamily, RestRegen, StreamTable}; + +const GIB: f64 = (1u64 << 30) as f64; + +/// What phase B does with the streamed chunks (`LAMBDA_VM_BLOCK_REGEN`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RegenMode { + /// `off` (and unset): nothing is recorded or regenerated. + Off, + /// `shadow`: phase A records every streamed chunk's recipe, and phase B + /// regenerates each beside the prove, checks it and throws it away. + Shadow, + /// `auto`: once the rule against the host's memory target + /// ([`super::drop_wanted`]) would move a table off the host, a streamed + /// chunk is dropped (and the parked ones are dropped back), and phase B + /// rebuilds the dropped ones. A block that never arms drops nothing. + Auto, + /// `always`: every streamed chunk is dropped and rebuilt — the + /// byte-identity test mode, not a policy. + Always, +} + +impl RegenMode { + /// Whether phase A drops chunks for phase B to rebuild. + pub fn live(self) -> bool { + matches!(self, Self::Auto | Self::Always) + } +} + +/// [`RegenMode`] from `LAMBDA_VM_BLOCK_REGEN` alone (unset: `off`); any other +/// value is refused (an error, never a panic). The production default is +/// `auto` when the knob is unset ([`production_regen`]). +pub(crate) fn regen_mode() -> Result { + parse_regen_mode(std::env::var("LAMBDA_VM_BLOCK_REGEN").ok().as_deref()) +} + +/// [`crate::block_whir::BlockOptions::production`]'s regeneration +/// ([`production_regen_from`] of the knob). +pub(crate) fn production_regen() -> Option { + production_regen_from(std::env::var("LAMBDA_VM_BLOCK_REGEN").ok().as_deref()) +} + +/// The production default (#1013's 1a9853790): unset, regeneration is `auto`: +/// whenever it arms, the tables phase B can build again (the streamed chunks, +/// KECCAK_RND's and LT's) are dropped and rebuilt in phase B, and the rest +/// stays. A set knob is read when the prove starts (`None`, so a value it does +/// not know refuses the prove). +pub(crate) fn production_regen_from(regen: Option<&str>) -> Option { + match regen { + Some(_) => None, + None => Some(RegenMode::Auto), + } +} + +/// `LAMBDA_VM_BLOCK_SPILL`, the knob of the block's disk store, which is gone: +/// set, it refuses the prove. It used to switch regeneration off when +/// `LAMBDA_VM_BLOCK_REGEN` was unset, so a run that sets it asked for an arm +/// that no longer exists; `LAMBDA_VM_BLOCK_REGEN=off` holds every table. +pub(crate) fn refuse_spill_knob(value: Option<&str>) -> Result<(), Error> { + match value { + None => Ok(()), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_SPILL={} is no longer read: the block has no disk store; unset it \ + (LAMBDA_VM_BLOCK_REGEN=off holds every table in memory)", + v.trim() + ))), + } +} + +/// The `BLOCK REGEN mode` line: the mode a prove runs, the knobs it reads at +/// its start as set (`LAMBDA_VM_BLOCK_SPILL` only to refuse it, +/// [`refuse_spill_knob`]; the production default is `auto` when +/// `LAMBDA_VM_BLOCK_REGEN` is unset), and `no disk` when live regeneration +/// runs. The line is what it was while the block had a disk store, so the +/// read-outs that parse it read it as before. +pub(crate) fn regen_mode_line( + mode: RegenMode, + live: bool, + regen: Option<&str>, + spill: Option<&str>, +) -> String { + let knob = |name: &str, value: Option<&str>| match value { + Some(v) => format!("{name}={}", v.trim()), + None => format!("{name} unset"), + }; + format!( + "BLOCK REGEN mode: {mode:?} · {} · {}{}", + knob("LAMBDA_VM_BLOCK_REGEN", regen), + knob("LAMBDA_VM_BLOCK_SPILL", spill), + if live { " · no disk" } else { "" } + ) +} + +/// [`regen_mode`] from the knob's `value`. +pub(crate) fn parse_regen_mode(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("off") => Ok(RegenMode::Off), + Some("shadow") => Ok(RegenMode::Shadow), + Some("auto") => Ok(RegenMode::Auto), + Some("always") => Ok(RegenMode::Always), + Some(other) => Err(Error::Prover(format!( + "LAMBDA_VM_BLOCK_REGEN must be `off`, `shadow`, `auto` or `always`, got `{other}`" + ))), + } +} + +/// The regenerator's window: the bytes it may hold deposited ahead of the +/// groups that take them, counted from the window's frontier. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) enum AheadPolicy { + /// `LAMBDA_VM_BLOCK_REGEN_AHEAD_GIB`: these bytes, the whole prove. + Fixed(u64), + /// Unset: [`adaptive_ahead`], read by a pacer beside the regenerator. + Adaptive, +} + +impl AheadPolicy { + /// The window before the pacer's first reading. + pub(crate) fn initial(self) -> u64 { + match self { + AheadPolicy::Fixed(bytes) => bytes, + AheadPolicy::Adaptive => AHEAD_FLOOR, + } + } +} + +/// The adaptive window's floor: the window it replaced (the spill read-back's). +pub(crate) const AHEAD_FLOOR: u64 = 4 << 30; +/// Its cap: the smallest window with which phase B's takers did not wait at +/// T200 (RYZEN 052: 0.0 s at 16 and 32 GiB, 3.02 s at 4). +pub(crate) const AHEAD_CAP: u64 = 16 << 30; +/// What it leaves below the run's peak. +pub(crate) const AHEAD_MARGIN: u64 = 4 << 30; + +/// [`AheadPolicy`] from `LAMBDA_VM_BLOCK_REGEN_AHEAD_GIB`. +pub(crate) fn ahead_policy() -> AheadPolicy { + ahead_policy_from( + std::env::var("LAMBDA_VM_BLOCK_REGEN_AHEAD_GIB") + .ok() + .as_deref(), + ) +} + +/// [`ahead_policy`] from the knob's `value`: a size in GiB fixes the window; +/// unset or unreadable, it is adaptive. +pub(crate) fn ahead_policy_from(value: Option<&str>) -> AheadPolicy { + value + .and_then(|v| v.trim().parse::().ok()) + .filter(|g| g.is_finite() && *g >= 0.0) + .map_or(AheadPolicy::Adaptive, |g| { + AheadPolicy::Fixed((g * GIB) as u64) + }) +} + +/// The adaptive window: what the run's peak resident set `hwm` leaves above +/// its resident set now (`rss`, the window's own `parked` deposits left out), +/// less [`AHEAD_MARGIN`], clamped to [[`AHEAD_FLOOR`], [`AHEAD_CAP`]]. Above +/// the floor the window holds only memory the run has already reached, so it +/// never raises the run's peak; phase A sets that peak, and phase B holds far +/// less (RYZEN 049 at T200: 72.5 GiB against 15–21). +pub(crate) fn adaptive_ahead(hwm: u64, rss: u64, parked: u64) -> u64 { + let base = rss.saturating_sub(parked); + hwm.saturating_sub(base) + .saturating_sub(AHEAD_MARGIN) + .clamp(AHEAD_FLOOR, AHEAD_CAP) +} + +/// The process's peak and current resident sets (`VmHWM`, `VmRSS`), where +/// `/proc/self/status` gives them. +fn resident() -> Option<(u64, u64)> { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let kib = |key: &str| { + status + .lines() + .find_map(|l| l.strip_prefix(key)) + .and_then(|v| v.trim().trim_end_matches("kB").trim().parse::().ok()) + .map(|k| k << 10) + }; + Some((kib("VmHWM:")?, kib("VmRSS:")?)) +} + +/// How often the pacer reads the host. +const PACE: std::time::Duration = std::time::Duration::from_millis(250); + +/// The least the pacer moves the window by. A smaller move is the window's own +/// churn, not the host's: a chunk taken or deposited (≈ 0.1 GiB at p90), whose +/// pages the allocator keeps (RYZEN 057 at p90 with no step: 109 shrinks and 28 +/// grows in 455 readings). +pub(crate) const PACE_STEP: u64 = 1 << 30; + +/// The pacer's next window from the current one (`now`, whole steps) and the +/// reading (`reading`, [`adaptive_ahead`]): the reading rounded down to whole +/// steps, taken only once the reading is a step or more away from the window +/// either way. So the window stays under a step above the reading (the run's +/// peak less [`AHEAD_MARGIN`] is still never passed, a step to spare), and a +/// reading that churns inside a step of it moves nothing. +pub(crate) fn paced_ahead(now: u64, reading: u64) -> u64 { + if now >= reading.saturating_add(PACE_STEP) || now.saturating_add(PACE_STEP) <= reading { + reading / PACE_STEP * PACE_STEP + } else { + now + } +} + +/// What the pacer set the window to, and how often it moved it each way (an +/// oscillating window is a finding even when phase B never waits). +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct PacerReport { + pub(crate) first: Option, + pub(crate) most: u64, + pub(crate) last: u64, + pub(crate) readings: usize, + pub(crate) grew: usize, + pub(crate) shrank: usize, +} + +/// The adaptive window's pacer: until `window` closes, every [`PACE`] it +/// moves the window to [`adaptive_ahead`] of the process's resident sets, in +/// [`paced_ahead`]'s steps. +fn pace(window: &RegenWindow, gate: Option<&AtomicBool>) -> PacerReport { + let mut report = PacerReport::default(); + // Started before phase A's end (N4c): the early pacer sizes the window + // until phase B starts; this one paces from then. + while gate.is_some_and(|gate| !gate.load(Ordering::Acquire)) && !window.is_closed() { + std::thread::sleep(PACE); + } + while !window.is_closed() { + if let Some((hwm, rss)) = resident() { + let now = window.ahead(); + let ahead = paced_ahead(now, adaptive_ahead(hwm, rss, window.parked())); + if ahead != now { + if ahead > now { + report.grew += 1; + } else { + report.shrank += 1; + } + window.set_ahead(ahead); + } + report.first.get_or_insert(ahead); + report.most = report.most.max(ahead); + report.last = ahead; + report.readings += 1; + } + std::thread::sleep(PACE); + } + report +} + +/// The `BLOCK REGEN window pacer` line. +pub(crate) fn pacer_line(policy: AheadPolicy, report: Option<&PacerReport>) -> String { + match (policy, report) { + (AheadPolicy::Fixed(bytes), _) => format!( + "BLOCK REGEN window pacer: fixed {:.2} GiB (LAMBDA_VM_BLOCK_REGEN_AHEAD_GIB)", + bytes as f64 / GIB + ), + (AheadPolicy::Adaptive, Some(r)) if r.readings > 0 => format!( + "BLOCK REGEN window pacer: adaptive (VmHWM − (VmRSS − parked) − {:.0} GiB, in [{:.0}, \ + {:.0}] GiB, steps of {:.0} GiB) · first {:.2} GiB · most {:.2} GiB · last {:.2} GiB · {} readings · grew {} \ + times · shrank {} times", + AHEAD_MARGIN as f64 / GIB, + AHEAD_FLOOR as f64 / GIB, + AHEAD_CAP as f64 / GIB, + PACE_STEP as f64 / GIB, + r.first.unwrap_or(0) as f64 / GIB, + r.most as f64 / GIB, + r.last as f64 / GIB, + r.readings, + r.grew, + r.shrank + ), + (AheadPolicy::Adaptive, _) => format!( + "BLOCK REGEN window pacer: adaptive, no readings (no /proc/self/status) · {:.2} GiB", + AHEAD_FLOOR as f64 / GIB + ), + } +} + +/// `LAMBDA_VM_BLOCK_REGEN_REST_AHEAD_GIB` (default 4): the bytes the rest's +/// regenerator (N3) may hold deposited ahead of the groups that take them. Under +/// phase B's interleaved order it supplies only its share of phase B's pace, so +/// a small window suffices (D-WHIR-NODISK § N3.2). +pub(crate) fn rest_ahead_bytes() -> u64 { + gib_knob("LAMBDA_VM_BLOCK_REGEN_REST_AHEAD_GIB", 4.0) +} + +/// When live regeneration starts the streamed chunks' regenerator +/// (`LAMBDA_VM_BLOCK_REGEN_EARLY`). The rest's starts with phase B either way. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EarlySpawn { + /// `0`: with phase B, from phase A's final drops. + Off, + /// `finish`: at the finish's end, when regeneration has armed by then. + /// Its first chunk needs about a second, so a phase B that starts sooner + /// than that after the finish waits for it (the median block's first + /// rebuilt groups waited 0.28-0.64 s, measured on a 118 GiB host with one + /// RTX 5090). + Finish, + /// The default: as soon as every streamed chunk's fate is known. Its + /// thread is there from the walk's end and waits for the fates + /// ([`stark::multilinear_block::BlockRegen::await_rank`]), which the arm + /// and the streamed groups' commits decide while the finish still runs; + /// until the finish's end its window holds only its frontier. + Fates, +} + +/// `LAMBDA_VM_BLOCK_REGEN_EARLY`: [`EarlySpawn::Fates`] unless `0` or +/// `finish`. +pub(crate) fn early_start() -> EarlySpawn { + parse_early(std::env::var("LAMBDA_VM_BLOCK_REGEN_EARLY").ok().as_deref()) +} + +fn parse_early(value: Option<&str>) -> EarlySpawn { + match value.map(str::trim) { + Some("0") => EarlySpawn::Off, + Some("finish") => EarlySpawn::Finish, + _ => EarlySpawn::Fates, + } +} + +/// The streamed groups phase B takes ahead of the rest's first when it +/// rebuilds both classes ([`regen_lead`]). +pub(crate) const REGEN_LEAD: usize = 4; + +/// `LAMBDA_VM_BLOCK_REGEN_LEAD` (default [`REGEN_LEAD`]; `0` is the plain +/// interleave): how many groups of the streamed chunks' class phase B takes +/// before it starts interleaving the two rebuilt classes. The rest's +/// regenerator starts with phase B, from phase A's final drops, and its first +/// group's tables take it about a second and a half; the streamed chunks' has +/// started before. With one streamed group ahead of it, the rest's first group +/// waited 0.31-0.45 s at the median (measured on a 118 GiB host with one RTX +/// 5090). +pub(crate) fn regen_lead() -> usize { + parse_lead(std::env::var("LAMBDA_VM_BLOCK_REGEN_LEAD").ok().as_deref()) +} + +fn parse_lead(value: Option<&str>) -> usize { + value + .and_then(|v| v.trim().parse::().ok()) + .filter(|n| *n <= 64) + .unwrap_or(REGEN_LEAD) +} + +/// `LAMBDA_VM_BLOCK_REGEN_RESERVE_GIB` (default 10, #1013's): the host bytes +/// `auto` reserves for the regenerator in phase B once regeneration is armed — +/// its walk state, windows, jobs and generators' outputs, and the columns +/// deposited ahead of their groups. Never added unarmed. +pub(crate) fn regen_reserve_bytes() -> u64 { + gib_knob("LAMBDA_VM_BLOCK_REGEN_RESERVE_GIB", 10.0) +} + +fn gib_knob(var: &str, default: f64) -> u64 { + let gib = std::env::var(var) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|g| g.is_finite() && *g >= 0.0) + .unwrap_or(default); + (gib * GIB) as u64 +} + +/// The streamed chunks' regenerator's generator threads when nothing is set. +/// It runs beside phase A's finish, where more threads slow the finish. +pub(crate) const REGEN_GENERATORS: usize = 3; + +/// The rest's regenerator's generator threads when nothing is set. It runs +/// only beside phase B. +/// +/// On a 118 GiB host the large block drops 92.36 GiB in phase A. With both +/// regenerators on 3 threads, phase B waited 1.4-2.2 s for the rest's tables; +/// with both on 6 it never waited, and the whole block took 1.44 s less with +/// the argue's round slot on. With the streamed one on 3 and the rest's on 4 +/// it never waited either, and took a further 0.62 s less than both on 6: +/// phase A 0.33 s and the argue 0.28 s, with fewer threads beside them; 5 and +/// 6 for the rest gained less (RTX 5090, Ryzen 9950X). The median block drops +/// nothing. +pub(crate) const REGEN_REST_GENERATORS: usize = 4; + +/// Generator threads of the streamed chunks' regenerator: +/// `LAMBDA_VM_BLOCK_REGEN_GENERATORS` (1..=16), unset [`REGEN_GENERATORS`]. +pub(crate) fn regen_generators() -> usize { + parse_generators( + std::env::var("LAMBDA_VM_BLOCK_REGEN_GENERATORS") + .ok() + .as_deref(), + ) +} + +fn parse_generators(value: Option<&str>) -> usize { + parse_count(value).unwrap_or(REGEN_GENERATORS) +} + +/// Generator threads of the rest's regenerator: +/// `LAMBDA_VM_BLOCK_REGEN_REST_GENERATORS` (1..=16); unset, the count +/// `LAMBDA_VM_BLOCK_REGEN_GENERATORS` sets, so that knob alone still sets both; +/// neither set, [`REGEN_REST_GENERATORS`]. +pub(crate) fn rest_generators() -> usize { + parse_rest_generators( + std::env::var("LAMBDA_VM_BLOCK_REGEN_REST_GENERATORS") + .ok() + .as_deref(), + std::env::var("LAMBDA_VM_BLOCK_REGEN_GENERATORS") + .ok() + .as_deref(), + ) +} + +fn parse_rest_generators(rest: Option<&str>, shared: Option<&str>) -> usize { + parse_count(rest) + .or_else(|| parse_count(shared)) + .unwrap_or(REGEN_REST_GENERATORS) +} + +fn parse_count(value: Option<&str>) -> Option { + value + .and_then(|v| v.trim().parse::().ok()) + .filter(|n| (1..=16).contains(n)) +} + +/// Packed columns' shape and digest ([`stark::regen::digest_of`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Packed { + pub(crate) digest: [u64; 2], + pub(crate) rows: usize, + pub(crate) cols: usize, + pub(crate) bytes: usize, +} + +impl Packed { + pub(crate) fn of(packed: &multilinear::narrow::NarrowColumns) -> Self { + Self { + digest: stark::regen::digest_of(packed), + rows: packed.rows(), + cols: packed.cols(), + bytes: packed.data().len(), + } + } +} + +/// A regenerated chunk that is not the one phase A committed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct RegenMismatch { + pub(crate) table: StreamTable, + pub(crate) index: usize, + pub(crate) want: Packed, + pub(crate) got: Packed, +} + +impl std::fmt::Display for RegenMismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (w, g) = (&self.want, &self.got); + write!( + f, + "{:?}[{}]: regenerated {} rows × {} columns, {} B, digest {:016x}{:016x}; phase A \ + committed {} × {}, {} B, digest {:016x}{:016x}", + self.table, + self.index, + g.rows, + g.cols, + g.bytes, + g.digest[0], + g.digest[1], + w.rows, + w.cols, + w.bytes, + w.digest[0], + w.digest[1], + ) + } +} + +/// The check every regenerated chunk passes before anything reads it: its +/// packed shape and digest are the ones phase A recorded. +pub(crate) fn check_regenerated( + table: StreamTable, + index: usize, + want: &Packed, + got: &Packed, +) -> Result<(), RegenMismatch> { + if want == got { + Ok(()) + } else { + Err(RegenMismatch { + table, + index, + want: *want, + got: *got, + }) + } +} + +/// A streamed chunk as phase A made it (D-REGEN §2.1): which chunk it is, +/// where its ops lie in the run, and its packed columns' digest. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Recipe { + pub(crate) table: StreamTable, + pub(crate) index: usize, + /// Its ops (all streamed chunks are full). + pub(crate) ops: usize, + /// The window its first op lies in and its offset in that window's + /// list, when phase A's windows were counted. + pub(crate) first: Option<(usize, usize)>, + /// The window whose hand-out completed it. + pub(crate) last_window: usize, + /// Its place in phase A's hand-out: its rank, and its place in the + /// packer's order. + pub(crate) order: usize, + /// Its packed columns, once the thread that generated them digested them. + pub(crate) packed: Option, +} + +impl Recipe { + /// Its main cells, once packed. + pub(crate) fn cells(&self) -> Option { + self.packed.map(|p| (p.rows * p.cols) as u64) + } +} + +/// A streamed table's place in the recorder's lists: [`StreamTable::ALL`]'s, +/// KECCAK_RND aside (a build that streams it is never recorded). +fn slot(table: StreamTable) -> Option { + match table { + StreamTable::Cpu => Some(0), + StreamTable::MemwRegister => Some(1), + StreamTable::MemwAligned => Some(2), + StreamTable::Memw => Some(3), + StreamTable::Load => Some(4), + StreamTable::Lt => Some(5), + StreamTable::Shift => Some(6), + StreamTable::Store => Some(7), + StreamTable::KeccakRnd => None, + } +} + +/// Phase A's recipes: fed by the accumulator (each window's list lengths, +/// then the chunks it handed out) and by the threads that generate the +/// streamed chunks (their digests). +#[derive(Default)] +pub(crate) struct Recorder { + inner: Mutex, + /// Whether the layout's threads digest each chunk as they generate it + /// (the shadow; live regeneration digests what it drops, at the drop). + digests: bool, + /// Whether a chunk held as records stays on the host, with no rank + /// (`LAMBDA_VM_BLOCK_RECORDS_DROP=keep`), instead of parking and dropping + /// as its records like any chunk. + keep_records: bool, +} + +#[derive(Default)] +struct Recorded { + /// Per streamed table, its list's length at the end of each window. + ends: [Vec; 8], + windows: usize, + recipes: BTreeMap<(usize, usize), Recipe>, + /// Each recipe's chunk, by its place in the hand-out; `None` for a chunk + /// held as the records its rows come from (`ChunkJob::is_records`) when + /// those are kept ([`Recorder::keep_records`]): such a chunk is never + /// dropped and has no recipe. + by_order: Vec>, + /// Digests for a chunk with no recipe, and chunks of a table no recipe + /// can hold. + stray: usize, + /// The rest's tables phase B can build again (N3), by their place among + /// the rest's laid-out tables, and the block index of the rest's first. + rest: std::collections::HashMap, + rest_base: Option, +} + +impl Recorder { + /// A recorder whose layout threads digest each chunk (`digests`, the + /// shadow's) or not (live regeneration's). + pub(crate) fn new(digests: bool) -> Self { + Self { + digests, + ..Self::default() + } + } + + /// Chunks held as records keep their place in the hand-out but get no + /// recipe and no rank: they are never dropped, and no regenerator waits on + /// their fate. + pub(crate) fn keep_records(mut self) -> Self { + self.keep_records = true; + self + } + + /// The `k`-th table of the rest laid out is `family`'s `j`-th + /// (`KECCAK_RND[j]`, `LT[j]`): phase B can build it again. + pub(crate) fn rest_tag(&self, k: usize, family: RegenFamily, j: usize) { + self.lock().rest.insert(k, (family, j)); + } + + /// The block index of the rest's first table, once the streamed chunks + /// are all placed. + pub(crate) fn rest_base(&self) -> Option { + self.lock().rest_base + } + + /// The rest's tables start at block index `base` (every streamed chunk + /// placed before them). + pub(crate) fn set_rest_base(&self, base: usize) { + self.lock().rest_base = Some(base); + } + + /// The rest table phase B can build again at block index `t`. + pub(crate) fn rest_at(&self, t: usize) -> Option<(RegenFamily, usize)> { + let recorded = self.lock(); + let k = t.checked_sub(recorded.rest_base?)?; + recorded.rest.get(&k).copied() + } + + /// Each rest table phase B can build again, by family and index: its + /// block index. + pub(crate) fn rest_blocks(&self) -> std::collections::HashMap<(RegenFamily, usize), usize> { + let recorded = self.lock(); + let Some(base) = recorded.rest_base else { + return std::collections::HashMap::new(); + }; + recorded + .rest + .iter() + .map(|(&k, &key)| (key, base + k)) + .collect() + } + + /// Whether the layout's threads digest each chunk ([`Self::packed`]). + pub(crate) fn digests(&self) -> bool { + self.digests + } + + /// The chunk handed out `order`-th, when one was and phase B can rebuild + /// it: phase A places the streamed chunks first, in hand-out order, so it + /// is the table at block index `order`. A chunk held as its records when + /// those are kept ([`Self::keep_records`]) is never dropped: `None`, so it + /// has no rank and no regenerator waits on its fate. + pub(crate) fn chunk_at(&self, order: usize) -> Option<(StreamTable, usize)> { + self.lock().by_order.get(order).copied().flatten() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Recorded> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// The next window, with the ops it added to each streamed table's list + /// ([`StreamTable::ALL`] order, KECCAK_RND aside), and the chunks its + /// hand-out completed, in hand-out order. + pub(crate) fn window(&self, added: [usize; 8], jobs: &[ChunkJob]) { + let mut r = self.lock(); + let w = r.windows; + r.windows += 1; + for (t, n) in added.into_iter().enumerate() { + let end = r.ends[t].last().copied().unwrap_or(0) + n; + r.ends[t].push(end); + } + for job in jobs { + // Held as its records and kept: in its place in the hand-out. + if job.is_records() && self.keep_records { + r.by_order.push(None); + continue; + } + let Some(t) = slot(job.table) else { + r.stray += 1; + continue; + }; + let ops = job.op_count(); + let start = job.index * ops; + let first = first_window(&r.ends[t], start); + // Its place in the hand-out, the chunks held as records counted. + let order = r.by_order.len(); + r.by_order.push(Some((job.table, job.index))); + r.recipes.insert( + (t, job.index), + Recipe { + table: job.table, + index: job.index, + ops, + first, + last_window: w, + order, + packed: None, + }, + ); + } + } + + /// The packed columns of chunk `index` of `table`, as the thread that + /// generated them holds them. + pub(crate) fn packed( + &self, + table: StreamTable, + index: usize, + packed: &multilinear::narrow::NarrowColumns, + ) { + let packed = Packed::of(packed); + let mut r = self.lock(); + match slot(table).and_then(|t| r.recipes.get_mut(&(t, index))) { + Some(recipe) => recipe.packed = Some(packed), + None => r.stray += 1, + } + } + + /// The recipes in hand-out order, and the digests that found no recipe. + pub(crate) fn recipes(&self) -> (Vec, usize) { + let r = self.lock(); + let mut recipes: Vec = r.recipes.values().cloned().collect(); + recipes.sort_by_key(|recipe| recipe.order); + (recipes, r.stray) + } +} + +/// The window op `start` of a list lies in, and its offset in that window's +/// part, from the list's length at the end of each window (`ends`). +fn first_window(ends: &[usize], start: usize) -> Option<(usize, usize)> { + let w = ends.iter().position(|&end| end > start)?; + let before = if w == 0 { 0 } else { ends[w - 1] }; + Some((w, start - before)) +} + +/// Phase A's recorder under `mode`, when phase A can feed it: `streams` (it +/// streams its windows and generates each streamed chunk packed, G-pack). +/// None otherwise, saying why. +pub(crate) fn recorder(mode: RegenMode, streams: bool) -> Option { + match (mode, streams) { + (RegenMode::Off, _) => None, + (_, true) => Some(Recorder::new(mode == RegenMode::Shadow)), + (_, false) => { + eprintln!( + "BLOCK REGEN: {mode:?} wanted, but phase A does not stream its chunks packed (windows, \ + G-pack, narrow groups): off" + ); + None + } + } +} + +/// This thread's CPU time so far, in seconds (Linux, +/// `/proc/thread-self/schedstat`). +fn thread_cpu_secs() -> Option { + let s = std::fs::read_to_string("/proc/thread-self/schedstat").ok()?; + let ns: f64 = s.split_whitespace().next()?.parse().ok()?; + Some(ns / 1e9) +} + +/// The CPU a thread used between two readings. +fn cpu_since(start: Option) -> Option { + Some(thread_cpu_secs()? - start?) +} + +/// A chunk the shadow regenerated and checked. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Regenerated { + pub(crate) table: StreamTable, + pub(crate) index: usize, + /// Seconds from the regenerator's start to the chunk checked. + pub(crate) ready: f64, + pub(crate) matched: bool, + pub(crate) bytes: usize, +} + +/// What the shadow regenerator did. +#[derive(Debug)] +pub(crate) struct ShadowReport { + /// Recipes it was given, and of those with no digest (not regenerated). + pub(crate) recipes: usize, + pub(crate) undigested: usize, + pub(crate) regenerated: Vec, + pub(crate) mismatches: Vec, + /// Chunks whose generation failed (a panic caught, columns that would not + /// pack). + pub(crate) failures: Vec, + /// Chunks cut that phase A did not stream (the finish's), not generated. + pub(crate) beyond: usize, + pub(crate) windows: usize, + pub(crate) cycles: usize, + /// Seconds from its start to its last chunk checked, or to its stop. + pub(crate) wall: f64, + /// Thread CPU seconds: the executor, the walker (with the slicer and its + /// start state), the generators summed (Linux only). + pub(crate) cpu: [Option; 3], + pub(crate) generators: usize, + /// Why it stopped before the end, if it did. + pub(crate) error: Option, + /// The walker's carried memory state, in bytes, at its end. + pub(crate) state_bytes: usize, +} + +impl ShadowReport { + fn new(recipes: usize, undigested: usize, generators: usize) -> Self { + Self { + recipes, + undigested, + regenerated: Vec::new(), + mismatches: Vec::new(), + failures: Vec::new(), + beyond: 0, + windows: 0, + cycles: 0, + wall: 0.0, + cpu: [None; 3], + generators, + error: None, + state_bytes: 0, + } + } + + /// The regenerator's CPU seconds, its three parts summed (Linux only). + pub(crate) fn cpu_total(&self) -> Option { + Some(self.cpu[0]? + self.cpu[1]? + self.cpu[2]?) + } + + /// Recipes with a digest the shadow neither checked nor failed. + pub(crate) fn missing(&self) -> usize { + (self.recipes - self.undigested) + .saturating_sub(self.regenerated.len() + self.failures.len()) + } + + /// The `BLOCK REGEN shadow` line. + pub(crate) fn line(&self) -> String { + let n = self.regenerated.len(); + let bytes: usize = self.regenerated.iter().map(|r| r.bytes).sum(); + let secs = |s: Option| s.map_or("n/a".to_string(), |s| format!("{s:.1}")); + let (first, last) = self + .regenerated + .iter() + .fold((f64::INFINITY, 0.0f64), |(lo, hi), r| { + (lo.min(r.ready), hi.max(r.ready)) + }); + format!( + "BLOCK REGEN shadow: {n} of {} chunks regenerated · {} mismatches · {} failed · {} \ + missing · {} undigested · {} not streamed (skipped) · {} windows walked ({:.2} M \ + cycles) · {:.2} GiB packed · ready from {:.2} s to {:.2} s · wall {:.2} s · CPU exec {} \ + · walk {} · {} generators {} = {} s · walk state {:.2} GiB{}", + self.recipes, + self.mismatches.len(), + self.failures.len(), + self.missing(), + self.undigested, + self.beyond, + self.windows, + self.cycles as f64 / 1e6, + bytes as f64 / GIB, + if first.is_finite() { first } else { 0.0 }, + last, + self.wall, + secs(self.cpu[0]), + secs(self.cpu[1]), + self.generators, + secs(self.cpu[2]), + secs(self.cpu_total()), + self.state_bytes as f64 / GIB, + self.error + .as_ref() + .map_or(String::new(), |e| format!(" · stopped: {e}")), + ) + } +} + +/// The shadow regenerator (D-REGEN §2.3, sequential form): `builder` walks +/// the run of `program` on `private_input`, executed `window` cycles at a +/// time on a thread of its own, and every chunk it cuts that has a digested +/// recipe is generated in `form`, packed if it is not, and checked on one of +/// `generators` threads, then dropped. Every time is from `started`. Stops +/// when every recipe is checked, on an error (reported), or when `stop` is +/// set. Never panics: a panic in generation counts as that chunk's failure. +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_shadow( + program: &Elf, + private_input: &[u8], + mut builder: RegenBuilder, + window: usize, + recipes: &[Recipe], + generators: usize, + form: TraceForm, + started: Instant, + stop: &AtomicBool, +) -> ShadowReport { + use std::panic::{AssertUnwindSafe, catch_unwind}; + use std::sync::mpsc; + let wanted: BTreeMap<(usize, usize), Packed> = recipes + .iter() + .filter_map(|r| Some(((slot(r.table)?, r.index), r.packed?))) + .collect(); + let mut report = ShadowReport::new(recipes.len(), recipes.len() - wanted.len(), generators); + if wanted.is_empty() { + return report; + } + let generators = generators.max(1); + let left = AtomicUsize::new(wanted.len()); + let done = AtomicBool::new(false); + let regenerated = Mutex::new(Vec::with_capacity(wanted.len())); + let mismatches = Mutex::new(Vec::new()); + let failures = Mutex::new(Vec::new()); + let last_ready = Mutex::new(0.0f64); + let quit = || stop.load(Ordering::Relaxed) || done.load(Ordering::Relaxed); + let (job_tx, job_rx) = mpsc::sync_channel::<(ChunkJob, Packed)>(2 * generators); + let job_rx = Mutex::new(job_rx); + std::thread::scope(|s| { + let (log_tx, log_rx) = mpsc::sync_channel::>(2); + let quit = &quit; + let exec = std::thread::Builder::new() + .name("regen-exec".to_string()) + .spawn_scoped(s, move || { + let cpu0 = thread_cpu_secs(); + let run = catch_unwind(AssertUnwindSafe(|| -> Result<(), String> { + let mut executor = Executor::new(program, private_input.to_vec()) + .map_err(|e| format!("the executor: {e}"))?; + while let Some(logs) = executor + .resume_with_limit(window) + .map_err(|e| format!("the execution: {e}"))? + { + if quit() || log_tx.send(logs.to_vec()).is_err() { + break; + } + } + Ok(()) + })) + .unwrap_or_else(|_| Err("the executor panicked".to_string())); + (run, cpu_since(cpu0)) + }); + let exec = match exec { + Ok(exec) => exec, + Err(e) => { + report.error = Some(format!("the executor thread did not start: {e}")); + return; + } + }; + let mut gens = Vec::with_capacity(generators); + for g in 0..generators { + let (job_rx, regenerated, mismatches, failures, left, last_ready, done) = ( + &job_rx, + ®enerated, + &mismatches, + &failures, + &left, + &last_ready, + &done, + ); + let spawned = + std::thread::Builder::new() + .name(format!("regen-gen-{g}")) + .spawn_scoped(s, move || { + let cpu0 = thread_cpu_secs(); + loop { + let next = job_rx.lock().unwrap_or_else(|e| e.into_inner()).recv(); + let Ok((job, want)) = next else { + break; + }; + if stop.load(Ordering::Relaxed) { + continue; + } + let (table, index) = (job.table, job.index); + let got = catch_unwind(AssertUnwindSafe(|| { + let mut trace = job.generate_as(form).trace; + trace + .pack_main_narrow() + .then(|| trace.narrow_main().map(Packed::of)) + })); + let ready = started.elapsed().as_secs_f64(); + let got = match got { + Ok(Some(Some(got))) => got, + Ok(_) => { + failures.lock().unwrap_or_else(|e| e.into_inner()).push( + format!("{table:?}[{index}]: the columns did not pack"), + ); + continue; + } + Err(_) => { + failures + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(format!("{table:?}[{index}]: generation panicked")); + continue; + } + }; + let matched = match check_regenerated(table, index, &want, &got) { + Ok(()) => true, + Err(mismatch) => { + mismatches + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(mismatch); + false + } + }; + regenerated.lock().unwrap_or_else(|e| e.into_inner()).push( + Regenerated { + table, + index, + ready, + matched, + bytes: got.bytes, + }, + ); + { + let mut last = last_ready.lock().unwrap_or_else(|e| e.into_inner()); + *last = last.max(ready); + } + if left.fetch_sub(1, Ordering::Relaxed) == 1 { + done.store(true, Ordering::Relaxed); + } + } + cpu_since(cpu0) + }); + match spawned { + Ok(handle) => gens.push(handle), + Err(e) => report + .failures + .push(format!("generator {g} did not start: {e}")), + } + } + if gens.is_empty() { + report.error = Some("no generator thread started".to_string()); + } + // This thread walks and cuts the chunks, and hands out those with a + // recipe. + let cpu0 = thread_cpu_secs(); + let mut dispatched = 0usize; + for logs in log_rx.iter() { + if quit() || gens.is_empty() { + break; + } + report.windows += 1; + let jobs = match catch_unwind(AssertUnwindSafe(|| builder.push(&logs))) { + Ok(Ok(jobs)) => jobs, + Ok(Err(e)) => { + report.error = Some(format!("the regeneration walk: {e}")); + break; + } + Err(_) => { + report.error = Some("the regeneration walk panicked".to_string()); + break; + } + }; + for job in jobs { + match slot(job.table).and_then(|t| wanted.get(&(t, job.index))) { + Some(&want) => { + if job_tx.send((job, want)).is_err() { + break; + } + dispatched += 1; + } + None => report.beyond += 1, + } + } + if dispatched == wanted.len() { + break; + } + } + drop(log_rx); + drop(job_tx); + report.cycles = builder.cycles(); + report.state_bytes = builder.state_bytes(); + report.cpu[1] = cpu_since(cpu0); + let mut gen_cpu = Some(0.0); + for g in gens { + let cpu = g.join().unwrap_or(None); + gen_cpu = gen_cpu.zip(cpu).map(|(a, b)| a + b); + } + report.cpu[2] = gen_cpu; + match exec.join() { + Ok((run, cpu)) => { + report.cpu[0] = cpu; + if let Err(e) = run + && report.error.is_none() + { + report.error = Some(e); + } + } + Err(_) => report.error = Some("the executor thread panicked".to_string()), + } + }); + if stop.load(Ordering::Relaxed) && report.error.is_none() { + report.error = Some("stopped by the prove".to_string()); + } + report.regenerated = regenerated.into_inner().unwrap_or_else(|e| e.into_inner()); + report.mismatches = mismatches.into_inner().unwrap_or_else(|e| e.into_inner()); + report + .failures + .extend(failures.into_inner().unwrap_or_else(|e| e.into_inner())); + report.wall = if report.regenerated.len() == wanted.len() { + *last_ready.lock().unwrap_or_else(|e| e.into_inner()) + } else { + started.elapsed().as_secs_f64() + }; + report +} + +/// The shadow regenerator on a thread of its own (`regen-walk`), for a prove +/// that goes on beside it. Dropped, it stops the regenerator and joins it, so +/// no regenerator outlives its prove on any exit. +pub(crate) struct ShadowRun { + stop: Arc, + handle: Option>, + started: Instant, + recipes: usize, + /// Why the thread did not start, if it did not. + spawn_error: Option, +} + +impl ShadowRun { + /// [`run_shadow`] over the run of `elf_bytes` on `private_input`, executed + /// `window` cycles at a time (phase A's windows), its chunks cut at + /// `max_rows` and generated in `form` (phase A's), on + /// [`regen_generators`] generators; its times are from now. `drop_one_op` + /// is a test's slicer bug ([`RegenBuilder`]'s), `None` outside tests. + pub(crate) fn spawn( + elf_bytes: Vec, + private_input: Vec, + max_rows: crate::tables::MaxRowsConfig, + window: usize, + recipes: Vec, + form: TraceForm, + drop_one_op: Option, + ) -> Self { + let started = Instant::now(); + let stop = Arc::new(AtomicBool::new(false)); + let count = recipes.len(); + let stop_in = Arc::clone(&stop); + let spawned = std::thread::Builder::new() + .name("regen-walk".to_string()) + .spawn(move || { + // The walker's CPU includes building its start state (the + // image, the decode table). + let cpu0 = thread_cpu_secs(); + let program = match Elf::load(&elf_bytes) { + Ok(program) => program, + Err(e) => { + let mut report = ShadowReport::new(recipes.len(), 0, 0); + report.error = Some(format!("the regenerator's ELF: {e}")); + return report; + } + }; + match RegenBuilder::new(&program, &private_input, &max_rows) { + Ok(builder) => { + #[cfg(test)] + let builder = match drop_one_op { + Some(table) => builder.drop_one_op(table), + None => builder, + }; + #[cfg(not(test))] + let _ = drop_one_op; + let setup = cpu_since(cpu0); + let mut report = run_shadow( + &program, + &private_input, + builder, + window, + &recipes, + regen_generators(), + form, + started, + &stop_in, + ); + report.cpu[1] = report.cpu[1].zip(setup).map(|(walk, setup)| walk + setup); + report + } + Err(e) => { + let mut report = ShadowReport::new(recipes.len(), 0, 0); + report.error = Some(format!("the regenerator: {e}")); + report + } + } + }); + let (handle, spawn_error) = match spawned { + Ok(handle) => (Some(handle), None), + Err(e) => (None, Some(format!("the regenerator did not start: {e}"))), + }; + Self { + stop, + handle, + started, + recipes: count, + spawn_error, + } + } + + /// When it started: every time its report gives is from here. + pub(crate) fn started(&self) -> Instant { + self.started + } + + /// Its report once done; a thread that did not start or that panicked is + /// a report with the error. + pub(crate) fn join(mut self) -> ShadowReport { + let failed = |error: String| { + let mut report = ShadowReport::new(self.recipes, 0, 0); + report.error = Some(error); + report + }; + match self.handle.take() { + Some(handle) => handle + .join() + .unwrap_or_else(|_| failed("the regenerator panicked".to_string())), + None => failed( + self.spawn_error + .take() + .unwrap_or_else(|| "the regenerator did not start".to_string()), + ), + } + } +} + +impl Drop for ShadowRun { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// When the regenerated chunks would have been needed by phase B's groups as +/// they ran: each group takes what it took (its measured span), in `order`, +/// and a group's chunks are needed when the group before it in that order +/// starts — phase B brings a group's columns back with the group before it, +/// whose argue uploads them. Lateness is a chunk's ready time (on phase B's +/// clock) less that need; with a serial consumer the stall the order would pay +/// is the largest lateness, at least 0. A chunk not regenerated, or +/// regenerated wrong, is missing, and with any missing there is no +/// would-wait. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Plan { + pub(crate) would_wait: f64, + /// The latest chunk, and how late it was (negative: early). + pub(crate) worst: Option<(StreamTable, usize, f64)>, + /// Chunks ready after they were needed. + pub(crate) late: usize, + /// Chunks in the plan, those never regenerated, and those whose group + /// is not known. + pub(crate) planned: usize, + pub(crate) missing: usize, + pub(crate) ungrouped: usize, + /// Groups holding a streamed chunk, and the others. + pub(crate) streamed_groups: usize, + pub(crate) other_groups: usize, + /// When the first streamed group's chunks are needed (seconds of phase B). + pub(crate) head: f64, +} + +/// The order of phase B's `groups` a [`Plan`] reads: as they ran, or with +/// every group that holds no streamed chunk first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PlanOrder { + GroupOrder, + RestFirst, +} + +/// [`Plan`] for `recipes` regenerated as `report` says, under `order`: +/// `group_of` gives a recipe's group, `groups` each group's phase-B span +/// (start, end) in seconds of phase B, and phase B began `prove_offset` +/// seconds after the regenerator. +pub(crate) fn plan( + recipes: &[Recipe], + group_of: &dyn Fn(&Recipe) -> Option, + report: &ShadowReport, + groups: &[(f64, f64)], + prove_offset: f64, + order: PlanOrder, +) -> Plan { + // A mismatched chunk would be refused, so it is never ready. + let ready: std::collections::HashMap<(StreamTable, usize), f64> = report + .regenerated + .iter() + .filter(|r| r.matched) + .map(|r| ((r.table, r.index), r.ready - prove_offset)) + .collect(); + let mut streamed = vec![false; groups.len()]; + for recipe in recipes { + if let Some(g) = group_of(recipe).filter(|&g| g < groups.len()) { + streamed[g] = true; + } + } + let sequence: Vec = match order { + PlanOrder::GroupOrder => (0..groups.len()).collect(), + PlanOrder::RestFirst => (0..groups.len()) + .filter(|&g| !streamed[g]) + .chain((0..groups.len()).filter(|&g| streamed[g])) + .collect(), + }; + // Each group's need: the start of the group before it in `sequence`. + let mut need = vec![0.0f64; groups.len()]; + let mut at = 0.0f64; + let mut previous_start = 0.0f64; + for (p, &g) in sequence.iter().enumerate() { + need[g] = if p == 0 { 0.0 } else { previous_start }; + previous_start = at; + let (start, end) = groups[g]; + at += (end - start).max(0.0); + } + let head = sequence + .iter() + .find(|&&g| streamed[g]) + .map_or(0.0, |&g| need[g]); + let (mut worst, mut late, mut missing, mut ungrouped, mut planned) = + (None::<(StreamTable, usize, f64)>, 0, 0, 0, 0); + for recipe in recipes.iter().filter(|r| r.packed.is_some()) { + planned += 1; + let Some(g) = group_of(recipe).filter(|&g| g < groups.len()) else { + ungrouped += 1; + continue; + }; + let Some(&ready) = ready.get(&(recipe.table, recipe.index)) else { + missing += 1; + continue; + }; + let lateness = ready - need[g]; + if lateness > 0.0 { + late += 1; + } + if worst.is_none_or(|(_, _, w)| lateness > w) { + worst = Some((recipe.table, recipe.index, lateness)); + } + } + Plan { + would_wait: worst.map_or(0.0, |(_, _, w)| w.max(0.0)), + worst, + late, + planned, + missing, + ungrouped, + streamed_groups: streamed.iter().filter(|&&s| s).count(), + other_groups: streamed.iter().filter(|&&s| !s).count(), + head, + } +} + +impl Plan { + /// The `BLOCK REGEN plan` line, with the regenerator's CPU over phase B. + pub(crate) fn line( + &self, + order: PlanOrder, + report: &ShadowReport, + prove: f64, + prove_offset: f64, + ) -> String { + let cores = report + .cpu_total() + .filter(|_| prove > 0.0) + .map_or("n/a".to_string(), |cpu| format!("{:.2}", cpu / prove)); + format!( + "BLOCK REGEN plan ({}): {} streamed chunks in {} groups after {} other groups (first \ + needed at {:.2} s of phase B) · would-wait {}{} · late {} · missing {} · ungrouped {} \ + · regen CPU {} s = {cores} cores over phase B ({prove:.2} s, starting {prove_offset:.2} \ + s after the regenerator)", + match order { + PlanOrder::GroupOrder => "group order", + PlanOrder::RestFirst => "rest first", + }, + self.planned, + self.streamed_groups, + match order { + PlanOrder::GroupOrder => "or between the".to_string(), + PlanOrder::RestFirst => format!("{}", self.other_groups), + }, + self.head, + if self.missing > 0 || self.ungrouped > 0 { + "n/a".to_string() + } else { + format!("{:.3} s", self.would_wait) + }, + self.worst.map_or(String::new(), |(t, i, w)| format!( + " (latest {t:?}[{i}] at {w:+.3} s)" + )), + self.late, + self.missing, + self.ungrouped, + report + .cpu_total() + .map_or("n/a".to_string(), |c| format!("{c:.1}")), + ) + } +} + +/// The `BLOCK REGEN recorded` line: what phase A recorded against the +/// streamed class (`streamed` chunks of `streamed_cells` cells). +pub(crate) fn recorded_line( + recipes: &[Recipe], + stray: usize, + streamed: usize, + streamed_cells: u64, +) -> String { + let digested = recipes.iter().filter(|r| r.packed.is_some()).count(); + let cells: u64 = recipes.iter().filter_map(Recipe::cells).sum(); + let spans: Vec = recipes + .iter() + .filter_map(|r| Some(r.last_window + 1 - r.first?.0)) + .collect(); + format!( + "BLOCK REGEN recorded: {} streamed chunks of {streamed} · {digested} digested · {stray} \ + stray digests · {:.3} G cells (the streamed class {:.3} G) · windows spanned mean {:.2} \ + max {}", + recipes.len(), + cells as f64 / 1e9, + streamed_cells as f64 / 1e9, + spans.iter().sum::() as f64 / spans.len().max(1) as f64, + spans.iter().max().copied().unwrap_or(0), + ) +} + +/// Which streamed chunk a table is. +pub(crate) type StreamKey = (StreamTable, usize); + +/// A generator alive while held (unwinding included). +struct Alive<'a>(&'a AtomicUsize); + +impl Drop for Alive<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +/// A job and the slot it fills: dropped undeposited (a generator that +/// panicked outside its catch, a channel torn down), it fails its slot (R10). +struct JobGuard { + job: Option, + slot: RegenSlot, + settled: bool, +} + +impl Drop for JobGuard { + fn drop(&mut self) { + if !self.settled { + self.slot.fail("its generator stopped before depositing it"); + } + } +} + +/// What the live regenerator did. +#[derive(Debug, Default)] +pub(crate) struct LiveReport { + pub(crate) dropped: usize, + pub(crate) deposited: usize, + pub(crate) mismatches: usize, + /// Deposits refused because the window had closed. + pub(crate) closed: usize, + pub(crate) failures: Vec, + /// Dropped chunks the slicer did not cut before a higher rank, failed at + /// once (R10). + pub(crate) skipped: usize, + /// Chunks cut that were not dropped, not generated. + pub(crate) beyond: usize, + pub(crate) windows: usize, + pub(crate) cycles: usize, + pub(crate) bytes: u64, + pub(crate) first_ready: Option, + pub(crate) last_ready: Option, + pub(crate) wall: f64, + /// Thread CPU: the executor, the walker (with its start state), the + /// generators summed (Linux only). + pub(crate) cpu: [Option; 3], + pub(crate) generators: usize, + pub(crate) error: Option, + pub(crate) state_bytes: usize, + /// The `BLOCK REGEN window pacer` line, once the run is joined. + pub(crate) pacer: Option, +} + +impl LiveReport { + pub(crate) fn cpu_total(&self) -> Option { + Some(self.cpu[0]? + self.cpu[1]? + self.cpu[2]?) + } + + /// The `BLOCK REGEN live` line. + pub(crate) fn line(&self) -> String { + let secs = |s: Option| s.map_or("n/a".to_string(), |s| format!("{s:.1}")); + format!( + "BLOCK REGEN live: {} of {} dropped chunks deposited · {} mismatches · {} failed · {} \ + skipped (not cut in rank order) · {} refused (window closed) · {} not dropped \ + (skipped) · {} windows walked ({:.2} M cycles) · {:.2} GiB packed · ready from {} s \ + to {} s · wall {:.2} s · CPU exec {} · walk {} · {} generators {} = {} s · walk state \ + {:.2} GiB{}", + self.deposited, + self.dropped, + self.mismatches, + self.failures.len(), + self.skipped, + self.closed, + self.beyond, + self.windows, + self.cycles as f64 / 1e6, + self.bytes as f64 / GIB, + secs(self.first_ready), + secs(self.last_ready), + self.wall, + secs(self.cpu[0]), + secs(self.cpu[1]), + self.generators, + secs(self.cpu[2]), + secs(self.cpu_total()), + self.state_bytes as f64 / GIB, + self.error + .as_ref() + .map_or(String::new(), |e| format!(" · stopped: {e}")), + ) + } +} + +/// A test's faults in the live regenerator's exits and deposits (R2/R10): +/// each makes one of them happen, so a test can check that no slot is left +/// waiting and nothing wrong is proved. All off outside tests. +#[derive(Clone, Copy, Debug, Default)] +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) struct LiveFaults { + /// The walk fails at this window. + pub(crate) walk_error_at: Option, + /// The generator that takes the job of this rank panics outside its + /// catch. + pub(crate) generator_panics_at_rank: Option, + /// The slicer never hands out this rank's chunk. + pub(crate) skip_rank: Option, + /// The chunk of this rank is deposited with one packed bit flipped. + pub(crate) bend_rank: Option, + /// The window checks deposits on their shape alone (the digest off), so + /// a bent deposit reaches the kept tree's check. + pub(crate) digest_off: bool, +} + +/// The live regenerator (D-REGEN §2.3, sequential form): `builder` walks the +/// run of `program` on `private_input`, executed `window_cycles` cycles at a +/// time on a thread of its own; every chunk it cuts that was dropped is +/// generated in `form`, packed if it is not, and deposited into its slot on +/// one of `generators` threads, in rank order of dispatch. Never panics; every +/// exit leaves no slot waiting. +/// +/// The ranks are phase A's hand-out order, window by window, so the windows +/// must be phase A's: a rank the slicer cuts after a higher one is failed — its +/// table refused, never waited on (R10). +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_live( + program: &Elf, + private_input: &[u8], + builder: RegenBuilder, + window_cycles: usize, + window: Arc, + producer: RegenProducer, + dropped: Vec<(StreamKey, RegenSlot)>, + generators: usize, + form: TraceForm, + started: Instant, + faults: LiveFaults, +) -> LiveReport { + use std::panic::{AssertUnwindSafe, catch_unwind}; + use std::sync::mpsc; + let mut builder = builder; + let mut report = LiveReport { + dropped: dropped.len(), + generators, + ..LiveReport::default() + }; + #[cfg(test)] + if faults.digest_off { + window.set_verify(false); + } + // Position of each dropped chunk in rank order. + let position: BTreeMap<(usize, usize), usize> = dropped + .iter() + .enumerate() + .filter_map(|(i, ((table, index), _))| Some(((slot(*table)?, *index), i))) + .collect(); + let mut settled = vec![false; dropped.len()]; + let fail_from = |settled: &mut [bool], from: usize, upto: usize, why: &str| -> usize { + let mut n = 0; + for i in from..upto { + if !settled[i] { + dropped[i].1.fail(why); + settled[i] = true; + n += 1; + } + } + n + }; + let deposited = AtomicUsize::new(0); + let mismatches = AtomicUsize::new(0); + let closed = AtomicUsize::new(0); + let failures = Mutex::new(Vec::new()); + let ready = Mutex::new((None::, None::, 0u64)); + let generators = generators.max(1); + let (job_tx, job_rx) = mpsc::sync_channel::(2 * generators); + let job_rx = Mutex::new(job_rx); + // Generators alive: when none is, the walker stops handing out (a full + // queue would otherwise hold it forever). + let alive = AtomicUsize::new(0); + std::thread::scope(|s| { + let (log_tx, log_rx) = mpsc::sync_channel::>(2); + let window_ref = &window; + let exec = std::thread::Builder::new() + .name("regen-exec".to_string()) + .spawn_scoped(s, move || { + let cpu0 = thread_cpu_secs(); + let run = catch_unwind(AssertUnwindSafe(|| -> Result<(), String> { + let mut executor = Executor::new(program, private_input.to_vec()) + .map_err(|e| format!("the executor: {e}"))?; + while let Some(logs) = executor + .resume_with_limit(window_cycles) + .map_err(|e| format!("the execution: {e}"))? + { + if window_ref.is_closed() || log_tx.send(logs.to_vec()).is_err() { + break; + } + } + Ok(()) + })) + .unwrap_or_else(|_| Err("the executor panicked".to_string())); + (run, cpu_since(cpu0)) + }); + let exec = match exec { + Ok(exec) => Some(exec), + Err(e) => { + report.error = Some(format!("the executor thread did not start: {e}")); + None + } + }; + let mut gens = Vec::with_capacity(generators); + if exec.is_some() { + for g in 0..generators { + let Some(token) = window.producer() else { + break; + }; + let (job_rx, deposited, mismatches, closed, failures, ready, alive) = ( + &job_rx, + &deposited, + &mismatches, + &closed, + &failures, + &ready, + &alive, + ); + alive.fetch_add(1, Ordering::SeqCst); + let spawned = std::thread::Builder::new() + .name(format!("regen-gen-{g}")) + .spawn_scoped(s, move || { + let _token = token; + let _alive = Alive(alive); + let cpu0 = thread_cpu_secs(); + loop { + let next = job_rx.lock().unwrap_or_else(|e| e.into_inner()).recv(); + let Ok(mut guard) = next else { + break; + }; + let Some(job) = guard.job.take() else { + continue; + }; + if faults.generator_panics_at_rank == Some(guard.slot.rank()) { + panic!("a generator dies outside its catch (a test's fault)"); + } + let (table, index) = (job.table, job.index); + // A chunk dropped as its records is rebuilt as + // them too: their digest checks its deposit. + let as_records = guard.slot.is_records(); + let built = catch_unwind(AssertUnwindSafe(|| { + let (records, chunk) = job.regenerate(form, as_records); + let mut trace = chunk.trace; + if trace.narrow_main().is_none() { + trace.pack_main_narrow(); + } + // Moved out, not copied. + trace.take_narrow_main().map(|packed| (records, packed)) + })); + let (records, packed) = match built { + Ok(Some(built)) => built, + Ok(None) => { + let why = + format!("{table:?}[{index}]: the columns did not pack"); + guard.slot.fail(&why); + guard.settled = true; + failures.lock().unwrap_or_else(|e| e.into_inner()).push(why); + continue; + } + Err(_) => { + let why = format!("{table:?}[{index}]: generation panicked"); + guard.slot.fail(&why); + guard.settled = true; + failures.lock().unwrap_or_else(|e| e.into_inner()).push(why); + continue; + } + }; + let packed = if faults.bend_rank == Some(guard.slot.rank()) { + bend(packed) + } else { + packed + }; + let bytes = packed.data().len() as u64; + let out = match (guard.slot.is_records(), records) { + (false, _) => guard.slot.deposit(packed), + (true, Some(digest)) => guard.slot.deposit_checked(packed, digest), + (true, None) => { + let why = format!( + "{table:?}[{index}]: dropped as records, not rebuilt as them" + ); + guard.slot.fail(&why); + Err(RegenError::Failed(why)) + } + }; + // Deposited, refused or closed: the slot is settled + // either way (a closed window settles every waiter). + guard.settled = true; + match out { + Ok(()) => { + deposited.fetch_add(1, Ordering::Relaxed); + let at = started.elapsed().as_secs_f64(); + let mut r = ready.lock().unwrap_or_else(|e| e.into_inner()); + r.0 = Some(r.0.map_or(at, |f: f64| f.min(at))); + r.1 = Some(r.1.map_or(at, |l: f64| l.max(at))); + r.2 += bytes; + } + Err(RegenError::Mismatch) => { + mismatches.fetch_add(1, Ordering::Relaxed); + } + // Keep draining (R3): the walker may be waiting + // on a full queue. + Err(RegenError::Closed(_)) => { + closed.fetch_add(1, Ordering::Relaxed); + } + Err(RegenError::Failed(why)) => { + failures + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(format!("{table:?}[{index}]: {why}")); + } + } + } + cpu_since(cpu0) + }); + match spawned { + Ok(handle) => gens.push(handle), + Err(e) => { + alive.fetch_sub(1, Ordering::SeqCst); + report + .failures + .push(format!("generator {g} did not start: {e}")); + } + } + } + } + // This thread walks and cuts, and hands out the dropped chunks in rank + // order; a dropped chunk not cut before a higher rank is failed at once + // (R10). + let cpu0 = thread_cpu_secs(); + let mut next = 0usize; + let mut stop: Option = None; + if gens.is_empty() { + stop = Some( + report + .error + .clone() + .unwrap_or_else(|| "no generator started".to_string()), + ); + } else { + for logs in log_rx.iter() { + if window.is_closed() { + stop = Some("the window closed".to_string()); + break; + } + if faults.walk_error_at == Some(report.windows) { + stop = Some("the regeneration walk failed (a test's fault)".to_string()); + break; + } + report.windows += 1; + let jobs = match catch_unwind(AssertUnwindSafe(|| builder.push_owned(logs))) { + Ok(Ok(jobs)) => jobs, + Ok(Err(e)) => { + stop = Some(format!("the regeneration walk: {e}")); + break; + } + Err(_) => { + stop = Some("the regeneration walk panicked".to_string()); + break; + } + }; + // The window's dropped chunks in rank order: within a window + // the order is the slicer's own, so only the windows must be + // phase A's. + let mut cut: Vec<(usize, ChunkJob)> = Vec::with_capacity(jobs.len()); + for job in jobs { + match slot(job.table).and_then(|t| position.get(&(t, job.index))) { + Some(&pos) => cut.push((pos, job)), + None => report.beyond += 1, + } + } + cut.sort_by_key(|&(pos, _)| pos); + for (pos, job) in cut { + if faults.skip_rank == Some(dropped[pos].1.rank()) { + continue; + } + if pos < next || settled[pos] { + continue; + } + report.skipped += fail_from( + &mut settled, + next, + pos, + "the slicer did not cut it before a higher rank", + ); + settled[pos] = true; + next = pos + 1; + let mut guard = JobGuard { + job: Some(job), + slot: dropped[pos].1.clone(), + settled: false, + }; + // A job that cannot be handed out is dropped, and its + // guard fails its slot. + loop { + match job_tx.try_send(guard) { + Ok(()) => break, + Err(mpsc::TrySendError::Full(back)) => { + if alive.load(Ordering::SeqCst) == 0 { + drop(back); + stop = Some("every generator stopped".to_string()); + break; + } + guard = back; + std::thread::sleep(std::time::Duration::from_millis(2)); + } + Err(mpsc::TrySendError::Disconnected(back)) => { + drop(back); + stop = Some("the generators stopped".to_string()); + break; + } + } + } + if stop.is_some() { + break; + } + } + if stop.is_some() || next == dropped.len() { + break; + } + } + } + // Every exit: what was not handed out fails, with why (R2). + let why = stop + .clone() + .unwrap_or_else(|| "the run ended before its chunk was cut".to_string()); + let unsent = fail_from(&mut settled, next, dropped.len(), &why); + report.skipped += unsent; + match stop { + Some(stop) if stop != "the window closed" => { + report.error.get_or_insert(stop); + } + None if unsent > 0 => { + report.error.get_or_insert(why); + } + _ => {} + } + drop(log_rx); + drop(job_tx); + drop(producer); + report.cycles = builder.cycles(); + report.state_bytes = builder.state_bytes(); + report.cpu[1] = cpu_since(cpu0); + let mut gen_cpu = Some(0.0); + for g in gens { + let cpu = g.join().unwrap_or_else(|_| { + failures + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("a generator panicked".to_string()); + None + }); + gen_cpu = gen_cpu.zip(cpu).map(|(a, b)| a + b); + } + // Jobs no generator took (every one stopped): their guards fail their + // slots now, not when this function returns. + while let Ok(guard) = job_rx.lock().unwrap_or_else(|e| e.into_inner()).try_recv() { + drop(guard); + } + report.cpu[2] = gen_cpu; + if let Some(exec) = exec { + match exec.join() { + Ok((run, cpu)) => { + report.cpu[0] = cpu; + if let Err(e) = run { + report.error.get_or_insert(e); + } + } + Err(_) => { + report + .error + .get_or_insert("the executor thread panicked".to_string()); + } + } + } + }); + report.deposited = deposited.into_inner(); + report.mismatches = mismatches.into_inner(); + report.closed = closed.into_inner(); + report + .failures + .extend(failures.into_inner().unwrap_or_else(|e| e.into_inner())); + let (first, last, bytes) = ready.into_inner().unwrap_or_else(|e| e.into_inner()); + report.first_ready = first; + report.last_ready = last; + report.bytes = bytes; + report.wall = started.elapsed().as_secs_f64(); + report +} + +/// `packed` with the low bit of its first byte flipped: a wrong word of the +/// same shape (a test's fault). +fn bend(packed: multilinear::narrow::NarrowColumns) -> multilinear::narrow::NarrowColumns { + let (rows, widths, mut data) = packed.into_parts(); + if let Some(b) = data.first_mut() { + *b ^= 1; + } + multilinear::narrow::NarrowColumns::from_parts(rows, widths, data) + .expect("one bit flipped keeps the shape") +} + +/// The live regenerator on a thread of its own (`regen-walk`), for the phase +/// B beside it. Dropped — the prove returned, refused or unwound — it closes +/// the window, so a regenerator waiting to deposit for takers that are gone +/// returns, and joins it (R4): no regenerator outlives its prove. +pub(crate) struct LiveRun { + window: Arc, + handle: Option>, + /// The adaptive window's pacer ([`AheadPolicy::Adaptive`]). + pacer: Option>, + policy: AheadPolicy, + started: Instant, + dropped: usize, + spawn_error: Option, +} + +impl LiveRun { + /// [`run_live`] over the run of `elf_bytes` on `private_input`, executed + /// `window_cycles` at a time (phase A's windows), its chunks cut at + /// `max_rows` and generated in `form` (phase A's), on [`regen_generators`] + /// generators, depositing `dropped` into `window`, which `policy` sizes; + /// its times are from now. A regenerator that cannot start drops + /// `producer`, so every slot fails at once. + #[allow(clippy::too_many_arguments)] + pub(crate) fn spawn( + elf_bytes: Vec, + private_input: Vec, + max_rows: crate::tables::MaxRowsConfig, + window_cycles: usize, + window: Arc, + producer: RegenProducer, + dropped: Vec<(StreamKey, RegenSlot)>, + form: TraceForm, + faults: LiveFaults, + policy: AheadPolicy, + gate: Option>, + ) -> Self { + let started = Instant::now(); + // Before phase B (a `gate` not yet open) the early pacer sizes the + // window ([`EarlyStart`]). + if gate.is_none() { + window.set_ahead(policy.initial()); + } + let pacer = (policy == AheadPolicy::Adaptive) + .then(|| { + let window = Arc::clone(&window); + std::thread::Builder::new() + .name("regen-pacer".to_string()) + .spawn(move || pace(&window, gate.as_deref())) + .ok() + }) + .flatten(); + let count = dropped.len(); + let window_in = Arc::clone(&window); + let spawned = std::thread::Builder::new() + .name("regen-walk".to_string()) + .spawn(move || { + let cpu0 = thread_cpu_secs(); + let failed = |why: String, dropped: &[(StreamKey, RegenSlot)]| { + for (_, slot) in dropped { + slot.fail(&why); + } + LiveReport { + dropped: dropped.len(), + skipped: dropped.len(), + error: Some(why), + ..LiveReport::default() + } + }; + let program = match Elf::load(&elf_bytes) { + Ok(program) => program, + Err(e) => return failed(format!("the regenerator's ELF: {e}"), &dropped), + }; + // A CPU chunk dropped as its logs is checked by them: the + // walk keeps each window's logs for its chunks. + let cpu_logs = dropped + .iter() + .any(|((table, _), slot)| *table == StreamTable::Cpu && slot.is_records()); + match RegenBuilder::new(&program, &private_input, &max_rows) + .and_then(|b| if cpu_logs { b.keep_cpu_logs() } else { Ok(b) }) + { + Ok(builder) => { + let setup = cpu_since(cpu0); + let mut report = run_live( + &program, + &private_input, + builder, + window_cycles, + window_in, + producer, + dropped, + regen_generators(), + form, + started, + faults, + ); + report.cpu[1] = report.cpu[1].zip(setup).map(|(walk, setup)| walk + setup); + report + } + Err(e) => failed(format!("the regenerator: {e}"), &dropped), + } + }); + let (handle, spawn_error) = match spawned { + Ok(handle) => (Some(handle), None), + // The closure went with the failed spawn, its producer with it: + // every slot has failed. + Err(e) => (None, Some(format!("the regenerator did not start: {e}"))), + }; + Self { + window, + handle, + pacer, + policy, + started, + dropped: count, + spawn_error, + } + } + + /// When it started: every time its report gives is from here. + pub(crate) fn started(&self) -> Instant { + self.started + } + + /// Its report once the prove is done (the window closed first, so it + /// returns); a thread that did not start or that panicked is a report + /// with the error. + pub(crate) fn join(mut self) -> LiveReport { + self.window.close("the block's prove ended"); + let failed = |error: String| LiveReport { + dropped: self.dropped, + error: Some(error), + ..LiveReport::default() + }; + let mut report = match self.handle.take() { + Some(handle) => handle + .join() + .unwrap_or_else(|_| failed("the regenerator panicked".to_string())), + None => failed( + self.spawn_error + .take() + .unwrap_or_else(|| "the regenerator did not start".to_string()), + ), + }; + let paced = self.pacer.take().and_then(|pacer| pacer.join().ok()); + report.pacer = Some(pacer_line(self.policy, paced.as_ref())); + report + } +} + +impl Drop for LiveRun { + fn drop(&mut self) { + self.window.close("the block's prove ended"); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + if let Some(pacer) = self.pacer.take() { + let _ = pacer.join(); + } + } +} + +/// What the rest's regenerator did ([`RestRun`]). +#[derive(Clone, Debug, Default)] +pub(crate) struct RestReport { + pub(crate) dropped: usize, + pub(crate) deposited: usize, + pub(crate) mismatches: usize, + /// Deposits refused because the window had closed. + pub(crate) closed: usize, + pub(crate) failures: Vec, + /// Tables built again, by family (KECCAK_RND, LT). + pub(crate) built: [usize; 2], + pub(crate) bytes: u64, + pub(crate) first_ready: Option, + pub(crate) last_ready: Option, + pub(crate) wall: f64, + /// The generators' thread CPU summed (Linux only). + pub(crate) cpu: Option, + pub(crate) generators: usize, + pub(crate) error: Option, +} + +impl RestReport { + /// The `BLOCK REGEN rest` line. + pub(crate) fn line(&self) -> String { + let secs = |s: Option| s.map_or("n/a".to_string(), |s| format!("{s:.1}")); + format!( + "BLOCK REGEN rest: {} of {} dropped tables deposited · {} mismatches · {} failed · {} \ + refused (window closed) · KECCAK_RND {} · LT {} built again · {:.2} GiB packed · \ + ready from {} s to {} s · wall {:.2} s · {} generators CPU {} s{}", + self.deposited, + self.dropped, + self.mismatches, + self.failures.len(), + self.closed, + self.built[0], + self.built[1], + self.bytes as f64 / GIB, + secs(self.first_ready), + secs(self.last_ready), + self.wall, + self.generators, + secs(self.cpu), + self.error + .as_ref() + .map_or(String::new(), |e| format!(" · stopped: {e}")), + ) + } +} + +/// One table the rest's regenerator builds: its place among the rest's +/// dropped tables, its rank (block index), its slot and its job. +type RestWork = (u64, u64, RegenSlot, crate::tables::trace_builder::RegenJob); + +/// The rest's regenerator (D-WHIR-NODISK N3) on threads of its own, for the +/// phase B beside it: KECCAK_RND's and LT's dropped tables built again from the +/// lists the finish kept ([`RestRegen`]), on `generators` threads taking them in +/// rank order, each deposited into its slot (digest-checked: another table +/// fails its slot). A family's list goes once its jobs have run. Dropped — the +/// prove returned, refused or unwound — it closes the window and joins. +pub(crate) struct RestRun { + window: Arc, + handle: Option>, + dropped: usize, + spawn_error: Option, +} + +impl RestRun { + /// The regenerator over `regen`, depositing `dropped` (each a rank, the + /// block index `blocks` gives a family's table, and its slot) into + /// `window`; `producer` holds the window's slots open until it ends. + /// `faults` name a table by its place among the dropped ones, in rank + /// order. + pub(crate) fn spawn( + mut regen: RestRegen, + blocks: std::collections::HashMap<(RegenFamily, usize), usize>, + window: Arc, + producer: RegenProducer, + dropped: Vec<(u64, RegenSlot)>, + generators: usize, + faults: LiveFaults, + ) -> Self { + use std::panic::{AssertUnwindSafe, catch_unwind}; + let count = dropped.len(); + let started = Instant::now(); + let spawned = std::thread::Builder::new() + .name("regen-rest".to_string()) + .spawn(move || { + let mut slots: std::collections::HashMap = + dropped.into_iter().collect(); + let mut report = RestReport { + dropped: count, + generators, + ..RestReport::default() + }; + // The work, in rank order: each family in AIR order, each table + // in the family's order (the rest's block indices ascend so). + let mut work: Vec = Vec::new(); + let mut built = [0usize; 2]; + for (f, family) in RegenFamily::ALL.into_iter().enumerate() { + let jobs = match regen.take(family) { + Ok(jobs) => jobs, + Err(e) => { + report.error = Some(format!("{e:?}")); + break; + } + }; + for (j, job) in jobs.into_iter().enumerate() { + let Some(job) = job else { continue }; + let Some(slot) = blocks + .get(&(family, j)) + .and_then(|&t| slots.remove(&(t as u64)).map(|slot| (t as u64, slot))) + else { + continue; + }; + built[f] += 1; + work.push((0, slot.0, slot.1, job)); + } + } + work.sort_by_key(|(_, _, slot, _)| slot.order_key()); + for (k, item) in work.iter_mut().enumerate() { + item.0 = k as u64; + } + // A dropped table no job builds is failed now, as is every one + // after a stop. + for (rank, slot) in slots.drain() { + let why = format!("rest table {rank} has no job to build it again"); + slot.fail(&why); + report.failures.push(why); + } + if let Some(why) = report.error.clone() { + for (_, _, slot, _) in &work { + slot.fail(&why); + } + drop(producer); + report.wall = started.elapsed().as_secs_f64(); + return report; + } + report.built = built; + let queue = Mutex::new(std::collections::VecDeque::from(work)); + let deposited = AtomicUsize::new(0); + let mismatches = AtomicUsize::new(0); + let closed = AtomicUsize::new(0); + let failures: Mutex> = Mutex::new(Vec::new()); + let ready: Mutex<(Option, Option, u64)> = Mutex::new((None, None, 0)); + let cpu: Mutex> = Mutex::new(Some(0.0)); + std::thread::scope(|scope| { + for _ in 0..generators.max(1) { + scope.spawn(|| { + let cpu0 = thread_cpu_secs(); + loop { + // FIFO: ranks go out in order, so the window's + // frontier is always a table some thread builds. + let next = + queue.lock().unwrap_or_else(|e| e.into_inner()).pop_front(); + let Some((k, rank, slot, job)) = next else { + break; + }; + let fail = |why: String| { + slot.fail(&why); + failures.lock().unwrap_or_else(|e| e.into_inner()).push(why); + }; + if faults.skip_rank == Some(k) { + fail(format!("rest table {rank}: skipped (a test's fault)")); + continue; + } + let built = catch_unwind(AssertUnwindSafe(|| { + if faults.generator_panics_at_rank == Some(k) { + panic!( + "a test's fault: the rest's generator panics at {rank}" + ); + } + job() + })); + let mut table = match built { + Ok(Ok(table)) => table, + Ok(Err(e)) => { + fail(format!("rest table {rank}: {e:?}")); + continue; + } + Err(_) => { + fail(format!("rest table {rank}: its generator panicked")); + continue; + } + }; + let Some(packed) = table.take_narrow_main() else { + fail(format!( + "rest table {rank}: built without packed columns" + )); + continue; + }; + let packed = if faults.bend_rank == Some(k) { + bend(packed) + } else { + packed + }; + let bytes = packed.data().len() as u64; + match slot.deposit(packed) { + Ok(()) => { + deposited.fetch_add(1, Ordering::Relaxed); + let at = started.elapsed().as_secs_f64(); + let mut r = ready.lock().unwrap_or_else(|e| e.into_inner()); + r.0 = Some(r.0.map_or(at, |a| a.min(at))); + r.1 = Some(r.1.map_or(at, |a| a.max(at))); + r.2 += bytes; + } + Err(RegenError::Mismatch) => { + mismatches.fetch_add(1, Ordering::Relaxed); + } + Err(RegenError::Closed(_)) => { + closed.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + failures + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(format!("rest table {rank}: {e}")); + } + } + } + let spent = cpu_since(cpu0); + let mut total = cpu.lock().unwrap_or_else(|e| e.into_inner()); + *total = total.zip(spent).map(|(a, b)| a + b); + }); + } + }); + drop(producer); + report.deposited = deposited.into_inner(); + report.mismatches = mismatches.into_inner(); + report.closed = closed.into_inner(); + report + .failures + .extend(failures.into_inner().unwrap_or_else(|e| e.into_inner())); + let (first, last, bytes) = ready.into_inner().unwrap_or_else(|e| e.into_inner()); + report.first_ready = first; + report.last_ready = last; + report.bytes = bytes; + report.cpu = cpu.into_inner().unwrap_or_else(|e| e.into_inner()); + report.wall = started.elapsed().as_secs_f64(); + report + }); + let (handle, spawn_error) = match spawned { + Ok(handle) => (Some(handle), None), + // The closure went with the failed spawn, its producer with it: + // every slot has failed. + Err(e) => ( + None, + Some(format!("the rest's regenerator did not start: {e}")), + ), + }; + Self { + window, + handle, + dropped: count, + spawn_error, + } + } + + /// Its report once the prove is done (the window closed first, so it + /// returns). + pub(crate) fn join(mut self) -> RestReport { + self.window.close("the block's prove ended"); + match self.handle.take() { + Some(handle) => handle.join().unwrap_or_else(|_| RestReport { + dropped: self.dropped, + error: Some("the rest's regenerator panicked".to_string()), + ..RestReport::default() + }), + None => RestReport { + dropped: self.dropped, + error: Some( + self.spawn_error + .take() + .unwrap_or_else(|| "the rest's regenerator did not start".to_string()), + ), + ..RestReport::default() + }, + } + } +} + +impl Drop for RestRun { + fn drop(&mut self) { + self.window.close("the block's prove ended"); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// The `BLOCK REGEN head start` line (N4c): how long before phase B the +/// streamed chunks' regenerator started (once their fates were known), how +/// long before phase B its thread was there (`spawned`), how long the fates +/// kept it waiting, and what its window held before phase B; or none. +pub(crate) fn head_start_line( + head_start: Option, + spawned: Option, + streamed_waited: Option, + pace: Option, +) -> String { + match head_start { + Some(secs) => format!( + "BLOCK REGEN head start: {secs:.2} s before phase B (the streamed chunks' regenerator \ + started when their fates were known, its thread {:.2} s before phase B; the rest's \ + with phase B) · streamed chunks' fates awaited {:.2} s · its window sized by the \ + host's room until phase B: most {:.2} GiB, {} readings", + spawned.unwrap_or(secs), + streamed_waited.unwrap_or(0.0), + pace.map_or(0, |p| p.most) as f64 / GIB, + pace.map_or(0, |p| p.readings), + ), + None => "BLOCK REGEN head start: none (the regenerators started with phase B)".to_string(), + } +} + +/// The streamed chunks' window size before phase B (N4c), from the process's +/// resident sets and the bytes it holds deposited (`parked`): the room under +/// the run's peak less [`AHEAD_MARGIN`], as N2b's pacer reads it but with no +/// floor, at most its size in phase B (`cap`). So filling it ahead of phase B +/// never lifts the run's peak; with no room it admits only its frontier. +pub(crate) fn early_ahead(hwm: u64, rss: u64, parked: u64, cap: u64) -> u64 { + hwm.saturating_sub(rss.saturating_sub(parked)) + .saturating_sub(AHEAD_MARGIN) + .min(cap) +} + +/// What the early pacer saw ([`EarlyStart`]): its readings and the most the +/// window was given before phase B. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct EarlyPace { + pub(crate) readings: usize, + pub(crate) most: u64, +} + +/// One early reading's window size: nothing to set while the finish still +/// runs (`paced` unset: the window keeps its frontier, so the regenerator does +/// not rebuild beside the finish) or with no reading (no `/proc`); after the +/// finish, [`early_ahead`]'s size (at most `cap`), moved from `now` in whole +/// [`PACE_STEP`]s ([`paced_ahead`]). +pub(crate) fn early_step( + paced: bool, + now: u64, + reading: Option<(u64, u64)>, + parked: u64, + cap: u64, +) -> Option { + let (hwm, rss) = reading.filter(|_| paced)?; + Some(paced_ahead(now, early_ahead(hwm, rss, parked, cap)).min(cap)) +} + +/// Until phase B starts (`gate`) or the window closes: every [`PACE`] the +/// window takes [`early_step`]'s size, its frontier until the finish has ended +/// (`paced`). When phase B starts it takes the size it starts phase B with +/// (`open`), and its own pacer. +fn early_pace( + window: &RegenWindow, + gate: &AtomicBool, + paced: &AtomicBool, + (cap, open): (u64, u64), +) -> EarlyPace { + let mut report = EarlyPace::default(); + loop { + if gate.load(Ordering::Acquire) { + window.set_ahead(open); + return report; + } + if window.is_closed() { + return report; + } + let reading = paced.load(Ordering::Acquire).then(resident).flatten(); + if let Some(ahead) = early_step(true, window.ahead(), reading, window.parked(), cap) { + window.set_ahead(ahead); + report.readings += 1; + report.most = report.most.max(ahead); + } + std::thread::sleep(PACE); + } +} + +/// What [`EarlyStart`] started: the streamed chunks' regenerator over the +/// class's final drops (with each dropped rank's chunk, for phase B to check +/// against the layout), and how long it waited for their fates. +#[derive(Default)] +pub(crate) struct EarlyRuns { + pub(crate) live: Option, + /// When `live` started: every streamed chunk's fate was known. + pub(crate) live_started: Option, + pub(crate) keys: Vec<(u64, StreamKey)>, + pub(crate) streamed_waited: f64, + pub(crate) error: Option, +} + +/// The arguments [`LiveRun::spawn`] takes besides its window, producer and +/// dropped chunks. +pub(crate) struct LiveArgs { + pub(crate) elf_bytes: Vec, + pub(crate) private_input: Vec, + pub(crate) max_rows: crate::tables::MaxRowsConfig, + pub(crate) window_cycles: usize, + pub(crate) form: TraceForm, + pub(crate) faults: LiveFaults, + pub(crate) policy: AheadPolicy, +} + +/// The streamed chunks' regenerator started before phase B, on a thread of +/// its own outside phase A's, once every streamed chunk's +/// fate is known (their groups commit first; each waited for through +/// [`stark::multilinear_block::BlockRegen::await_rank`], which answers nothing +/// for a table the arm may still drop). The thread may be there before +/// regeneration arms ([`EarlySpawn::Fates`]): it then waits for the arm's +/// drop-back, or for phase A's end, where every undropped table is kept and it +/// starts nothing. The rest's regenerator starts with phase B, from phase A's +/// final drops. Dropped before phase B takes it (the prove failed), it closes +/// the streamed window, so nothing waits on it, and joins. +pub(crate) struct EarlyStart { + thread: Option>, + window: Option>, + started: Instant, + /// The early pacer: the window at its frontier until the finish has + /// ended, then sized by the host's room until phase B opens `gate` + /// ([`early_pace`]). + pacer: Option>, +} + +impl EarlyStart { + /// `paced` is set once the finish has ended: until then the window holds + /// only its frontier. `gate` opens when phase B starts. + pub(crate) fn spawn( + feed: stark::multilinear_block::BlockRegen, + recorder: Arc, + live: LiveArgs, + gate: Arc, + paced: Arc, + ) -> Self { + let started = Instant::now(); + let window = feed.windows().first().cloned(); + // Before phase B the window holds only what the host's room allows: + // nothing ahead of its frontier while the finish runs and until the + // early pacer's first reading after it, at most its phase-B size after. + let cap = match live.policy { + AheadPolicy::Fixed(bytes) => bytes, + AheadPolicy::Adaptive => AHEAD_CAP, + }; + let open = live.policy.initial(); + let pacer = window.as_ref().and_then(|window| { + window.set_ahead(0); + let (window, gate) = (Arc::clone(window), Arc::clone(&gate)); + std::thread::Builder::new() + .name("regen-early-pacer".to_string()) + .spawn(move || early_pace(&window, &gate, &paced, (cap, open))) + .ok() + }); + let window_in = window.clone(); + let thread = std::thread::Builder::new() + .name("regen-early".to_string()) + .spawn(move || { + let mut runs = EarlyRuns::default(); + let Some(window) = window_in else { + return runs; + }; + // Every streamed chunk's fate: phase A placed them all before + // the rest, and their groups commit first. + let base = loop { + if let Some(base) = recorder.rest_base() { + break base; + } + if window.is_closed() { + runs.error = Some("the streamed chunks were never all placed".to_string()); + return runs; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + }; + let mut keyed = Vec::new(); + for t in 0..base { + let Some(key) = recorder.chunk_at(t) else { + continue; + }; + let (decision, waited) = feed.await_rank(0, t as u64); + runs.streamed_waited += waited.as_secs_f64(); + match decision { + stark::multilinear_block::RankDecision::Dropped(slot) => { + keyed.push((key, slot)); + runs.keys.push((t as u64, key)); + } + stark::multilinear_block::RankDecision::Kept => {} + stark::multilinear_block::RankDecision::Stopped => { + runs.error = Some(format!( + "the prove ended before streamed chunk {t}'s fate was known" + )); + return runs; + } + } + } + if !keyed.is_empty() { + match feed.take_producer(0) { + Some(producer) => { + runs.live_started = Some(Instant::now()); + runs.live = Some(LiveRun::spawn( + live.elf_bytes, + live.private_input, + live.max_rows, + live.window_cycles, + window, + producer, + keyed, + live.form, + live.faults, + live.policy, + Some(gate), + )); + } + None => { + runs.error = + Some("the streamed chunks' window has no producer".to_string()); + } + } + } + runs + }); + Self { + // No thread: the closure, its runs and the producer it would have + // taken went with it; phase B finds no producer and refuses. + thread: thread.ok(), + window, + started, + pacer, + } + } + + /// When its thread was spawned (its regenerator starts later, once the + /// fates are known: [`EarlyRuns::live_started`]). + pub(crate) fn started(&self) -> Instant { + self.started + } + + /// The runs it started, once it has (phase B's start: every streamed + /// chunk's fate is known by phase A's end), and the early pacer, which + /// ends once phase B opens the gate. + pub(crate) fn join(mut self) -> (EarlyRuns, Option>) { + let pacer = self.pacer.take(); + let runs = match self.thread.take() { + Some(thread) => thread.join().unwrap_or_else(|_| EarlyRuns { + error: Some("the early regeneration thread panicked".to_string()), + ..EarlyRuns::default() + }), + None => EarlyRuns { + error: Some("the early regeneration thread did not start".to_string()), + ..EarlyRuns::default() + }, + }; + (runs, pacer) + } +} + +impl Drop for EarlyStart { + fn drop(&mut self) { + if let Some(window) = &self.window + && (self.thread.is_some() || self.pacer.is_some()) + { + window.close("the block's prove ended before phase B"); + } + // Its run closes the window and joins as it drops; the early pacer + // ends once the window is closed. + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + if let Some(pacer) = self.pacer.take() { + let _ = pacer.join(); + } + } +} + +/// The `BLOCK REGEN dropped` line: what phase A dropped, of `recipes` streamed +/// chunks (always with no disk: the line reads as it did beside a store). +pub(crate) fn dropped_line( + mode: RegenMode, + report: &stark::multilinear_block::DropReport, + recipes: usize, +) -> String { + let armed = match report.armed { + Some(at) => format!("armed at {at:.1} s"), + None if mode == RegenMode::Always => "always".to_string(), + None => "never armed".to_string(), + }; + format!( + "BLOCK REGEN dropped: {mode:?} (no disk) · {} chunks {:.2} GiB (drop-back {} chunks {:.2} GiB) · \ + {armed} · {recipes} recipes · {} refused (no longer on the host) · {} held (their group \ + dropped another class) · {} dropped as their records, {:.2} GiB of records", + report.tables, + report.bytes as f64 / GIB, + report.back_tables, + report.back_bytes as f64 / GIB, + report.refused_late, + report.held, + report.records.0, + report.records.1 as f64 / GIB, + ) +} + +/// The `BLOCK REGEN keep` line under partial drops: the keep budget +/// ([`stark::multilinear_block::BlockRegen::keep_cells`]) and the arming point +/// it is read against (the main cells committed, the candidates kept by then), +/// and what the budget dropped back. +pub(crate) fn keep_line(report: &stark::multilinear_block::DropReport) -> String { + let g = |cells: u64| cells as f64 / 1e9; + let absolute = report + .keep_cells + .map(|cells| format!("{:.2} G cells (LAMBDA_VM_BLOCK_KEEP_CELLS_G)", g(cells))); + let below = report.keep_below.map(|cells| { + format!( + "{:.2} G cells below arming (LAMBDA_VM_BLOCK_KEEP_BACK_G)", + g(cells) + ) + }); + let budget = match (absolute, below) { + (Some(a), Some(b)) => format!("budget {a} or {b}, the lower"), + (Some(one), None) | (None, Some(one)) => format!("budget {one}"), + (None, None) => "budget off".to_string(), + }; + let Some(at) = report.armed else { + return format!("BLOCK REGEN keep: {budget} · never armed"); + }; + let cells = report + .armed_cells + .map_or_else(|| "-".to_string(), |cells| format!("{:.2}", g(cells))); + let used = report.budget.map_or_else(String::new, |cells| { + format!(" · the budget used {:.2} G cells", g(cells)) + }); + format!( + "BLOCK REGEN keep: {budget} · armed at {at:.1} s with {cells} G cells committed and {} \ + candidates kept, {:.2} GiB · dropped back by the budget {} chunks {:.2} GiB{used}", + report.armed_kept.0, + report.armed_kept.1 as f64 / GIB, + report.keep_back.0, + report.keep_back.1 as f64 / GIB, + ) +} + +/// The shadow's readout once phase B is done: the `BLOCK REGEN` lines (what +/// phase A recorded against the `streamed` chunks of `streamed_cells` cells, +/// what the shadow did and each mismatch and failure, the first 20; the plan +/// in group order and rest first over phase B's group `spans`; when the shadow +/// was joined) and the counts. `group_of` gives a streamed chunk's group; +/// phase B took `prove` seconds from `prove_offset` seconds after the +/// regenerator started, and its join waited `joined` seconds after phase B. +#[allow(clippy::too_many_arguments)] +pub(crate) fn shadow_readout( + recipes: &[Recipe], + stray: usize, + report: &ShadowReport, + group_of: &dyn Fn(StreamTable, usize) -> Option, + streamed: usize, + streamed_cells: u64, + spans: &[(f64, f64)], + prove: f64, + prove_offset: f64, + joined: f64, +) -> RegenStamps { + let mut lines = vec![ + recorded_line(recipes, stray, streamed, streamed_cells), + report.line(), + ]; + lines.extend( + report + .mismatches + .iter() + .take(20) + .map(|m| format!("BLOCK REGEN MISMATCH {m}")), + ); + lines.extend( + report + .failures + .iter() + .take(20) + .map(|f| format!("BLOCK REGEN FAILED {f}")), + ); + let of = |r: &Recipe| group_of(r.table, r.index); + let mut would_wait_rest_first = None; + for order in [PlanOrder::GroupOrder, PlanOrder::RestFirst] { + let p = plan(recipes, &of, report, spans, prove_offset, order); + if order == PlanOrder::RestFirst && p.missing == 0 && p.ungrouped == 0 { + would_wait_rest_first = Some(p.would_wait); + } + lines.push(p.line(order, report, prove, prove_offset)); + } + lines.push(format!( + "BLOCK REGEN shadow joined {joined:.2} s after phase B" + )); + RegenStamps { + recipes: recipes.len(), + digested: recipes.iter().filter(|r| r.packed.is_some()).count(), + stray, + regenerated: report.regenerated.len(), + mismatches: report.mismatches.len(), + failed: report.failures.len(), + missing: report.missing(), + error: report.error.clone(), + would_wait_rest_first, + lines, + ..RegenStamps::default() + } +} + +/// What a shadow run left for the readout ([`super::BlockStamps::regen`]). +#[derive(Clone, Debug, Default)] +pub struct RegenStamps { + pub recipes: usize, + /// Live regeneration: the chunks phase A dropped, and their packed bytes. + pub dropped: usize, + pub dropped_bytes: u64, + /// The chunks parked as droppable when live regeneration armed, before + /// any drop-back: what a hand-off at that moment could drop. + pub armed_kept: usize, + pub digested: usize, + pub stray: usize, + pub regenerated: usize, + pub mismatches: usize, + pub failed: usize, + pub missing: usize, + pub error: Option, + /// Would-wait under the rest-first plan, when it is known. + pub would_wait_rest_first: Option, + /// Of `dropped` and `regenerated`, the rest's tables (N3: KECCAK_RND's and + /// LT's, built again from the kept lists). + pub rest_dropped: usize, + pub rest_regenerated: usize, + /// Tables phase B could rebuild kept on the host because their group + /// dropped the other class (N3's one class a group). + pub held: usize, + /// The groups in the order phase B took them. + pub phase_b_order: Vec, + /// Seconds before phase B the regenerators started (N4c), when they + /// started at the finish's end. + pub head_start: Option, + /// The `BLOCK REGEN` lines. + pub lines: Vec, +} + +/// The `BLOCK REGEN phase B order taken` line (N3): the order phase B took, +/// run-length by each group's rebuilt class (`P` nothing rebuilt, `S` the +/// streamed chunks, `R` the rest), and the rest's share of the rebuilt groups' +/// `cells` started by each tenth of phase B (`spans`: each group's start and +/// end, seconds), against its share of them all, the plan's. +pub(crate) fn order_taken_line( + order: &[usize], + classes: &[Option], + cells: &[u64], + spans: &[(f64, f64)], +) -> String { + let class = |g: usize| classes.get(g).copied().flatten(); + let mut runs: Vec<(char, usize)> = Vec::new(); + for &g in order { + let c = match class(g) { + None => 'P', + Some(0) => 'S', + Some(_) => 'R', + }; + match runs.last_mut() { + Some((last, n)) if *last == c => *n += 1, + _ => runs.push((c, 1)), + } + } + let cells_of = |g: usize| cells.get(g).copied().unwrap_or(0) as f64; + let (mut rest_all, mut rebuilt_all) = (0.0, 0.0); + for &g in order { + if let Some(c) = class(g) { + rebuilt_all += cells_of(g); + if c > 0 { + rest_all += cells_of(g); + } + } + } + let share = |rest: f64, all: f64| { + if all > 0.0 { + format!("{:.2}", rest / all) + } else { + "-".to_string() + } + }; + let begin = spans.iter().map(|s| s.0).fold(f64::INFINITY, f64::min); + let end = spans.iter().map(|s| s.1).fold(f64::NEG_INFINITY, f64::max); + let tenths: Vec = (1..=10) + .map(|k| { + let by = begin + (end - begin) * k as f64 / 10.0; + let (mut rest, mut all) = (0.0, 0.0); + for &g in order { + if class(g).is_some() && spans.get(g).is_some_and(|s| s.0 <= by) { + all += cells_of(g); + if class(g) > Some(0) { + rest += cells_of(g); + } + } + } + share(rest, all) + }) + .collect(); + format!( + "BLOCK REGEN phase B order taken: rest share of the rebuilt cells started by each tenth \ + of phase B [{}] against {} overall · {}", + tenths.join(", "), + share(rest_all, rebuilt_all), + runs.iter() + .map(|(c, n)| format!("{c}{n}")) + .collect::>() + .join(" ") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The keep line says the budget, the arming point (time, G cells, the + /// candidates kept) and what the budget dropped back; a run that never + /// armed says so. + #[test] + fn the_keep_line_says_the_budget_and_the_arming_point() { + let report = stark::multilinear_block::DropReport { + armed: Some(16.04), + armed_cells: Some(20_724_000_000), + armed_kept: (477, 41 << 30), + keep_cells: Some(20_900_000_000), + keep_back: (3, 1 << 29), + ..Default::default() + }; + assert_eq!( + keep_line(&report), + "BLOCK REGEN keep: budget 20.90 G cells (LAMBDA_VM_BLOCK_KEEP_CELLS_G) · armed at 16.0 s \ + with 20.72 G cells committed and 477 candidates kept, 41.00 GiB · dropped back by the \ + budget 3 chunks 0.50 GiB" + ); + let off = stark::multilinear_block::DropReport::default(); + assert_eq!( + keep_line(&off), + "BLOCK REGEN keep: budget off · never armed" + ); + // Relative to the arming point: the margin, and the budget it used. + let relative = stark::multilinear_block::DropReport { + armed: Some(15.0), + armed_cells: Some(21_100_000_000), + armed_kept: (488, 40 << 30), + keep_below: Some(1_300_000_000), + budget: Some(19_800_000_000), + keep_back: (31, 5 << 29), + ..Default::default() + }; + assert_eq!( + keep_line(&relative), + "BLOCK REGEN keep: budget 1.30 G cells below arming (LAMBDA_VM_BLOCK_KEEP_BACK_G) · armed \ + at 15.0 s with 21.10 G cells committed and 488 candidates kept, 40.00 GiB · dropped back by \ + the budget 31 chunks 2.50 GiB · the budget used 19.80 G cells" + ); + let both = stark::multilinear_block::DropReport { + keep_cells: Some(20_900_000_000), + keep_below: Some(1_300_000_000), + ..Default::default() + }; + assert_eq!( + keep_line(&both), + "BLOCK REGEN keep: budget 20.90 G cells (LAMBDA_VM_BLOCK_KEEP_CELLS_G) or 1.30 G cells \ + below arming (LAMBDA_VM_BLOCK_KEEP_BACK_G), the lower · never armed" + ); + } + + /// The order line (N3): the classes in the order taken, run-length, and + /// the rest's share of the rebuilt cells started by each tenth of phase B. + #[test] + fn the_order_line_reads_the_rest_share_over_phase_b() { + // Groups 0 and 1 streamed, 2 and 3 rest, 4 plain; phase B takes the + // plain one, then S R S R, each a second long. + let classes = [Some(0), Some(0), Some(1), Some(1), None]; + let cells = [10, 10, 30, 30, 5]; + let order = [4, 0, 2, 1, 3]; + let mut spans = [(0.0, 0.0); 5]; + for (p, &g) in order.iter().enumerate() { + spans[g] = (p as f64, p as f64 + 1.0); + } + assert_eq!( + order_taken_line(&order, &classes, &cells, &spans), + "BLOCK REGEN phase B order taken: rest share of the rebuilt cells started by each tenth \ + of phase B [-, 0.00, 0.00, 0.75, 0.75, 0.60, 0.60, 0.75, 0.75, 0.75] against 0.75 overall \ + · P1 S1 R1 S1 R1" + ); + assert!( + order_taken_line(&[0, 1], &[None, None], &[1, 1], &[(0.0, 1.0), (1.0, 2.0)]) + .ends_with("[-, -, -, -, -, -, -, -, -, -] against - overall · P2") + ); + } + + /// ★ The production default: unset → `auto`; a set regen is read when + /// the prove starts. + #[test] + fn the_production_regen_is_auto_unless_its_knob_is_set() { + assert_eq!(production_regen_from(None), Some(RegenMode::Auto)); + for regen in ["off", "shadow", "auto", "always", "bogus"] { + assert_eq!(production_regen_from(Some(regen)), None, "{regen}"); + } + // A set regen parses as it did; unset, alone, it is off. + assert_eq!(parse_regen_mode(None).unwrap(), RegenMode::Off); + assert!(parse_regen_mode(Some("bogus")).is_err()); + assert_eq!( + regen_mode_line(RegenMode::Auto, true, None, None), + "BLOCK REGEN mode: Auto · LAMBDA_VM_BLOCK_REGEN unset · LAMBDA_VM_BLOCK_SPILL unset \ + · no disk" + ); + assert_eq!( + regen_mode_line(RegenMode::Off, false, Some(" off "), None), + "BLOCK REGEN mode: Off · LAMBDA_VM_BLOCK_REGEN=off · LAMBDA_VM_BLOCK_SPILL unset" + ); + } + + /// The disk store's knob refuses the prove when it is set, whatever its + /// value, and names the arm that replaces its `off`; unset, nothing. + #[test] + fn the_disk_stores_knob_refuses_the_prove_when_set() { + assert!(refuse_spill_knob(None).is_ok()); + for value in ["off", "auto", "always", "2.5", ""] { + let Err(Error::Prover(why)) = refuse_spill_knob(Some(value)) else { + panic!("LAMBDA_VM_BLOCK_SPILL={value} must refuse the prove"); + }; + assert!(why.contains("LAMBDA_VM_BLOCK_REGEN=off"), "{why}"); + } + } + + /// `LAMBDA_VM_BLOCK_REGEN_EARLY`: at the fates unless `0` (with + /// phase B) or `finish` (at the finish's end); and the head-start line, + /// with a start and without. + #[test] + fn the_early_start_reads_its_knob_and_says_its_head_start() { + assert_eq!(parse_early(None), EarlySpawn::Fates); + assert_eq!(parse_early(Some("1")), EarlySpawn::Fates); + assert_eq!(parse_early(Some("yes")), EarlySpawn::Fates); + assert_eq!(parse_early(Some("0")), EarlySpawn::Off); + assert_eq!(parse_early(Some(" 0 ")), EarlySpawn::Off); + assert_eq!(parse_early(Some("finish")), EarlySpawn::Finish); + assert_eq!(parse_early(Some(" finish ")), EarlySpawn::Finish); + assert_eq!(parse_early(Some("Finish")), EarlySpawn::Fates); + let pace = EarlyPace { + readings: 14, + most: 2 << 30, + }; + assert_eq!( + head_start_line(Some(3.4), Some(9.75), Some(0.02), Some(pace)), + "BLOCK REGEN head start: 3.40 s before phase B (the streamed chunks' regenerator \ + started when their fates were known, its thread 9.75 s before phase B; the rest's \ + with phase B) · streamed chunks' fates awaited 0.02 s · its window sized by the \ + host's room until phase B: most 2.00 GiB, 14 readings" + ); + assert_eq!( + head_start_line(None, Some(9.75), None, None), + "BLOCK REGEN head start: none (the regenerators started with phase B)" + ); + // Before phase B the window holds only the room under the peak less + // the margin, capped at its phase-B size. + const G: u64 = 1 << 30; + assert_eq!(early_ahead(70 * G, 60 * G, 0, 16 * G), 6 * G, "70 − 60 − 4"); + assert_eq!(early_ahead(70 * G, 66 * G, 0, 16 * G), 0, "no room"); + assert_eq!( + early_ahead(70 * G, 69 * G, 3 * G, 16 * G), + 0, + "room 0 with 3 parked" + ); + assert_eq!(early_ahead(100 * G, 50 * G, 0, 16 * G), 16 * G, "capped"); + } + + /// `LAMBDA_VM_BLOCK_REGEN_LEAD`: a count up to 64, else the default. + #[test] + fn the_lead_reads_its_knob() { + assert_eq!(parse_lead(None), REGEN_LEAD); + assert_eq!(parse_lead(Some("0")), 0); + assert_eq!(parse_lead(Some(" 6 ")), 6); + assert_eq!(parse_lead(Some("64")), 64); + assert_eq!(parse_lead(Some("65")), REGEN_LEAD); + assert_eq!(parse_lead(Some("-1")), REGEN_LEAD); + assert_eq!(parse_lead(Some("many")), REGEN_LEAD); + } + + /// `LAMBDA_VM_BLOCK_REGEN_GENERATORS`: a count from 1 to 16, else the + /// streamed regenerator's default, 3. + #[test] + fn the_generators_read_their_knob() { + assert_eq!(REGEN_GENERATORS, 3); + assert_eq!(parse_generators(None), REGEN_GENERATORS); + assert_eq!(parse_generators(Some("1")), 1); + assert_eq!(parse_generators(Some(" 3 ")), 3); + assert_eq!(parse_generators(Some("16")), 16); + for bad in ["", "0", "17", "-1", "many"] { + assert_eq!(parse_generators(Some(bad)), REGEN_GENERATORS, "{bad:?}"); + } + } + + /// `LAMBDA_VM_BLOCK_REGEN_REST_GENERATORS`: a count from 1 to 16 for the + /// rest's regenerator alone; unset or unreadable, the count + /// `LAMBDA_VM_BLOCK_REGEN_GENERATORS` sets; neither, the default, 4. + #[test] + fn the_rest_generators_read_their_knob_then_the_shared_one() { + assert_eq!(REGEN_REST_GENERATORS, 4); + assert_eq!(parse_rest_generators(None, None), REGEN_REST_GENERATORS); + for bad in ["", "0", "17", "-1", "many"] { + assert_eq!( + parse_rest_generators(Some(bad), None), + REGEN_REST_GENERATORS, + "{bad:?}" + ); + assert_eq!( + parse_rest_generators(None, Some(bad)), + REGEN_REST_GENERATORS, + "{bad:?}" + ); + assert_eq!(parse_rest_generators(Some(bad), Some("6")), 6, "{bad:?}"); + } + assert_eq!( + parse_rest_generators(None, Some("6")), + 6, + "the shared knob alone sets both" + ); + assert_eq!(parse_rest_generators(Some("5"), Some("3")), 5); + assert_eq!(parse_rest_generators(Some(" 16 "), None), 16); + } + + /// While the finish runs the early window keeps its frontier whatever the + /// host's room (the regenerator must not rebuild beside the finish); after + /// it, it takes the room in whole steps; with no reading it sets nothing. + #[test] + fn the_early_window_keeps_its_frontier_until_the_finish_ends() { + const G: u64 = 1 << 30; + let room = Some((100 * G, 50 * G)); + assert_eq!( + early_step(false, 0, room, 0, 16 * G), + None, + "the finish runs" + ); + assert_eq!(early_step(false, 3 * G, room, 0, 16 * G), None); + assert_eq!( + early_step(true, 0, room, 0, 16 * G), + Some(16 * G), + "after it" + ); + assert_eq!( + early_step(true, 0, Some((70 * G, 60 * G)), 0, 16 * G), + Some(6 * G) + ); + assert_eq!( + early_step(true, 0, Some((70 * G, 66 * G)), 0, 16 * G), + Some(0), + "no room" + ); + assert_eq!(early_step(true, 0, None, 0, 16 * G), None, "no reading"); + // The pacer itself: its frontier while `paced` is unset, and phase B's + // size once the gate opens. + let (window, _producer) = RegenWindow::new(4 * G); + window.set_ahead(0); + let (gate, paced) = (AtomicBool::new(false), AtomicBool::new(false)); + std::thread::scope(|s| { + let pacer = s.spawn(|| early_pace(&window, &gate, &paced, (16 * G, 4 * G))); + std::thread::sleep(PACE * 2); + assert_eq!(window.ahead(), 0, "the frontier while the finish runs"); + gate.store(true, Ordering::Release); + let report = pacer.join().expect("the pacer"); + assert_eq!((report.readings, report.most), (0, 0)); + }); + assert_eq!(window.ahead(), 4 * G, "phase B's size at the gate"); + } + + /// `LAMBDA_VM_BLOCK_REGEN`: `off` (and unset), `shadow`, `auto`, `always`; + /// anything else is refused (an error, not a panic). + #[test] + fn the_regen_mode_reads_its_knob() { + assert_eq!(parse_regen_mode(None).unwrap(), RegenMode::Off); + assert_eq!(parse_regen_mode(Some(" off ")).unwrap(), RegenMode::Off); + assert_eq!(parse_regen_mode(Some("shadow")).unwrap(), RegenMode::Shadow); + assert_eq!(parse_regen_mode(Some("auto")).unwrap(), RegenMode::Auto); + assert_eq!(parse_regen_mode(Some("always")).unwrap(), RegenMode::Always); + for bad in ["", "on", "Shadow", "1", "Auto", "ALWAYS"] { + assert!(parse_regen_mode(Some(bad)).is_err(), "{bad:?}"); + } + assert!(!RegenMode::Off.live() && !RegenMode::Shadow.live()); + assert!(RegenMode::Auto.live() && RegenMode::Always.live()); + } + + /// ★ The adaptive window: what the peak leaves above the resident set (the + /// window's own deposits left out), less 4 GiB, in [4, 16] GiB. + #[test] + fn the_adaptive_window_is_clamped_to_its_bounds() { + const G: u64 = 1 << 30; + // Phase B at T200 (RYZEN 049): a 72.5 GiB peak over ≈ 15 GiB: the cap. + assert_eq!(adaptive_ahead(72 * G + G / 2, 15 * G, 0), AHEAD_CAP); + // p90 with no disk (RYZEN 044): a 66 GiB peak over ≈ 56: in between. + assert_eq!(adaptive_ahead(66 * G, 56 * G, 0), 6 * G); + // The window's own deposits are left out: the same window. + assert_eq!(adaptive_ahead(66 * G, 59 * G, 3 * G), 6 * G); + // No room above the resident set, or above the peak: the floor. + assert_eq!(adaptive_ahead(60 * G, 58 * G, 0), AHEAD_FLOOR); + assert_eq!(adaptive_ahead(60 * G, 61 * G, 0), AHEAD_FLOOR); + assert_eq!(adaptive_ahead(0, 0, 0), AHEAD_FLOOR); + // Exactly at either bound. + assert_eq!(adaptive_ahead(30 * G, 22 * G, 0), AHEAD_FLOOR); + assert_eq!(adaptive_ahead(30 * G, 10 * G, 0), AHEAD_CAP); + assert_eq!(adaptive_ahead(u64::MAX, 0, u64::MAX), AHEAD_CAP); + } + + /// Above the floor, the window and the resident set without it never + /// pass the peak less the margin: the window holds only memory the run has + /// already reached. + #[test] + fn the_adaptive_window_never_passes_the_peak() { + const G: u64 = 1 << 30; + for hwm in (0..=120).step_by(3).map(|g| g * G) { + for rss in (0..=120).step_by(5).map(|g| g * G) { + for parked in [0, G, 3 * G, 16 * G] { + let ahead = adaptive_ahead(hwm, rss, parked); + assert!((AHEAD_FLOOR..=AHEAD_CAP).contains(&ahead)); + if ahead > AHEAD_FLOOR { + let base = rss.saturating_sub(parked); + assert!(base + ahead + AHEAD_MARGIN <= hwm, "{hwm} {rss} {parked}"); + } + } + } + } + } + + /// The pacer moves the window in whole steps, and only once the reading is + /// a step away: a reading that churns inside a step (a chunk taken or + /// deposited) moves nothing, even across a whole GiB; a drift moves it a + /// step at a time; the window never stands a step or more above the reading. + #[test] + fn the_pacer_steps_past_the_window_churn() { + const G: u64 = 1 << 30; + assert_eq!(paced_ahead(4 * G, 16 * G), 16 * G); + assert_eq!(paced_ahead(16 * G, 4 * G), 4 * G); + assert_eq!(paced_ahead(6 * G, 6 * G + G / 2), 6 * G, "inside a step up"); + assert_eq!( + paced_ahead(6 * G, 6 * G - G / 2), + 6 * G, + "inside a step down" + ); + assert_eq!(paced_ahead(6 * G, 5 * G), 5 * G, "a step down"); + assert_eq!(paced_ahead(6 * G, 7 * G + G / 3), 7 * G, "a step up, whole"); + // Churn of ± 0.3 GiB around 6.5 and around 6.0 GiB (across a whole + // GiB): one or two moves up from 4 GiB, then none. + for centre in [6 * G + G / 2, 6 * G] { + let mut now = 4 * G; + let mut moves = 0; + for k in 0..400u64 { + let reading = centre + (k % 7) * G / 10 - 3 * G / 10; + let next = paced_ahead(now, reading); + moves += usize::from(next != now); + now = next; + assert!(now < reading + PACE_STEP, "{now} over {reading}"); + } + assert!((1..=2).contains(&moves), "{moves} moves around {centre}"); + } + // A drift from 16 to 4 GiB in steps of 0.1 GiB: twelve moves down. + let (mut now, mut shrank) = (16 * G, 0); + for k in 0..=120u64 { + let reading = (16 * G).saturating_sub(k * G / 10).max(4 * G); + let next = paced_ahead(now, reading); + shrank += usize::from(next < now); + assert!(next <= now); + now = next; + assert!(now < reading + PACE_STEP); + } + assert_eq!((now, shrank), (4 * G, 12)); + } + + /// `LAMBDA_VM_BLOCK_REGEN_AHEAD_GIB` fixes the window (RYZEN 052's arms); + /// unset or unreadable, it is adaptive, from the floor. + #[test] + fn the_window_knob_overrides_the_adaptive_window() { + const G: u64 = 1 << 30; + assert_eq!(ahead_policy_from(Some("04")), AheadPolicy::Fixed(4 * G)); + assert_eq!(ahead_policy_from(Some(" 32 ")), AheadPolicy::Fixed(32 * G)); + assert_eq!(ahead_policy_from(Some("0.5")), AheadPolicy::Fixed(G / 2)); + assert_eq!(ahead_policy_from(None), AheadPolicy::Adaptive); + for bad in ["", "x", "-1", "nan", "inf"] { + assert_eq!( + ahead_policy_from(Some(bad)), + AheadPolicy::Adaptive, + "{bad:?}" + ); + } + assert_eq!(AheadPolicy::Fixed(16 * G).initial(), 16 * G); + assert_eq!(AheadPolicy::Adaptive.initial(), AHEAD_FLOOR); + assert!(pacer_line(AheadPolicy::Fixed(16 * G), None).contains("fixed 16.00 GiB")); + let r = PacerReport { + first: Some(4 * G), + most: 16 * G, + last: 16 * G, + readings: 9, + grew: 3, + shrank: 1, + }; + let line = pacer_line(AheadPolicy::Adaptive, Some(&r)); + assert!( + line.contains( + "first 4.00 GiB · most 16.00 GiB · last 16.00 GiB · 9 readings · grew 3 times · \ + shrank 1 times" + ), + "{line}" + ); + } + + /// A chunk's first op in the windows: lists of 3, 3, 9 and 14 ops after + /// windows 0..4 (window 1 added none), chunks of 4 ops. + #[test] + fn a_chunks_first_op_is_found_in_its_window() { + let ends = [3, 3, 9, 14]; + assert_eq!(first_window(&ends, 0), Some((0, 0))); + assert_eq!(first_window(&ends, 4), Some((2, 1))); + assert_eq!(first_window(&ends, 8), Some((2, 5))); + assert_eq!(first_window(&ends, 12), Some((3, 3))); + assert_eq!(first_window(&ends, 14), None); + assert_eq!(first_window(&[], 0), None); + } + + fn recipe(table: StreamTable, index: usize, order: usize) -> Recipe { + Recipe { + table, + index, + ops: 4, + first: Some((order, 0)), + last_window: order, + order, + packed: Some(Packed { + digest: [0, 0], + rows: 10, + cols: 10, + bytes: 100, + }), + } + } + + fn ready(table: StreamTable, index: usize, at: f64, matched: bool) -> Regenerated { + Regenerated { + table, + index, + ready: at, + matched, + bytes: 100, + } + } + + /// ★ The plan over five groups of phase B, each 2 s: groups 0 and 1 hold + /// the streamed chunks (CPU[0], MEMW[0] in 0; CPU[1] in 1), groups 2–4 the + /// rest. In group order group 0 is needed at once and group 1 when group 0 + /// starts (0 s); rest first, the groups run 2 3 4 0 1, so group 0 is needed + /// when group 4 starts (4 s) and group 1 when group 0 starts (6 s). + /// Lateness is ready − need on phase B's clock; the would-wait the largest, + /// at least 0; a mismatched chunk is missing, and the line says n/a. + #[test] + fn the_plan_reads_phase_b_in_group_order_and_rest_first() { + use StreamTable::{Cpu, Memw}; + let recipes = [recipe(Cpu, 0, 0), recipe(Memw, 0, 1), recipe(Cpu, 1, 2)]; + let group_of = |r: &Recipe| Some(usize::from(r.table == Cpu && r.index == 1)); + let groups: Vec<(f64, f64)> = (0..5) + .map(|g| (2.0 * g as f64, 2.0 * g as f64 + 2.0)) + .collect(); + let mut report = ShadowReport::new(3, 0, 2); + report.regenerated = vec![ + ready(Cpu, 0, 3.0, true), + ready(Memw, 0, 5.0, true), + ready(Cpu, 1, 8.0, true), + ]; + let p = plan( + &recipes, + &group_of, + &report, + &groups, + 0.0, + PlanOrder::GroupOrder, + ); + assert_eq!((p.planned, p.missing, p.ungrouped, p.late), (3, 0, 0, 3)); + assert_eq!((p.streamed_groups, p.other_groups), (2, 3)); + assert_eq!(p.worst, Some((Cpu, 1, 8.0))); + assert!((p.would_wait - 8.0).abs() < 1e-9); + let p = plan( + &recipes, + &group_of, + &report, + &groups, + 0.0, + PlanOrder::RestFirst, + ); + assert!((p.head - 4.0).abs() < 1e-9, "{p:?}"); + // CPU[0] 3 − 4, MEMW[0] 5 − 4, CPU[1] 8 − 6. + assert_eq!(p.late, 2); + assert_eq!(p.worst, Some((Cpu, 1, 2.0))); + assert!((p.would_wait - 2.0).abs() < 1e-9); + // The regenerator 3 s ahead of phase B: everything early. + let p = plan( + &recipes, + &group_of, + &report, + &groups, + 3.0, + PlanOrder::RestFirst, + ); + assert_eq!((p.late, p.would_wait), (0, 0.0)); + // A mismatched chunk is missing, and the line says n/a. + report.regenerated[1].matched = false; + let p = plan( + &recipes, + &group_of, + &report, + &groups, + 0.0, + PlanOrder::RestFirst, + ); + assert_eq!(p.missing, 1); + assert!( + p.line(PlanOrder::RestFirst, &report, 10.0, 0.0) + .contains("would-wait n/a") + ); + // A chunk whose group is unknown is not planned against anything. + let p = plan( + &recipes, + &|_| None, + &report, + &groups, + 0.0, + PlanOrder::RestFirst, + ); + assert_eq!(p.ungrouped, 3); + } +} diff --git a/prover/src/continuation.rs b/prover/src/continuation.rs index ee4a03b01..644fadb47 100644 --- a/prover/src/continuation.rs +++ b/prover/src/continuation.rs @@ -50,20 +50,22 @@ use std::collections::HashMap; use std::sync::Arc; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use executor::elf::Elf; use executor::vm::execution::Executor; use math::field::element::FieldElement; use stark::config::Commitment; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet, EmptyConstraints}; -use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, LazyCommitment, NullBoundaryConstraintBuilder, +}; +use stark::multilinear_table::PreparedColumn; use stark::proof::options::ProofOptions; use stark::proof::stark::MultiProof; use stark::proof::view::MultiProofView; -use stark::prover::{IsStarkProver, Prover}; +use stark::prover::IsStarkProver; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::statement::{StatementKind, absorb_continuation_global_statement, absorb_statement}; use crate::tables::local_to_global::{self, CellBoundary}; @@ -96,8 +98,8 @@ fn epoch_transcript( epoch_label: u64, is_final: bool, fri_final_poly_log_degree: u8, -) -> DefaultTranscript { - let mut transcript = DefaultTranscript::::new(&[]); +) -> crate::hash_pin::BlockTranscript { + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::ContinuationEpoch { @@ -124,8 +126,8 @@ fn global_transcript( num_private_input_pages: usize, fri_final_poly_log_degree: u8, touched_page_bases: &[u64], -) -> DefaultTranscript { - let mut transcript = DefaultTranscript::::new(&[]); +) -> crate::hash_pin::BlockTranscript { + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_continuation_global_statement( &mut transcript, elf_bytes, @@ -148,7 +150,7 @@ fn global_transcript( /// The L2G epoch-local table's single transition constraint: `MU ∈ {0,1}` /// (`MU·(1−MU) = 0`) at constraint index 0. #[derive(Clone, Copy)] -struct L2gMemoryConstraints; +pub(crate) struct L2gMemoryConstraints; impl ConstraintSet for L2gMemoryConstraints { fn eval>(&self, b: &mut B) { @@ -168,7 +170,7 @@ impl ConstraintSet for L2gMemoryConstraints { /// committed trace (equal Merkle roots). So under collision resistance the trace the /// global bus runs over already satisfies all those constraints — do not add them /// here (it would be redundant, not a missing check). -fn l2g_global_air( +pub(crate) fn l2g_global_air( opts: &ProofOptions, epoch_label: u64, ) -> AirWithBuses { @@ -189,7 +191,7 @@ fn l2g_global_air( /// check too: this proof has the BITWISE provider, and the global proof commits /// the identical trace (the commitment binding compares roots), so checking here /// covers both. `epoch_label` is the `fini_epoch` constant used by both. -fn l2g_memory_air( +pub(crate) fn l2g_memory_air( opts: &ProofOptions, epoch_label: u64, ) -> AirWithBuses { @@ -228,7 +230,7 @@ fn l2g_memory_air( /// genesis commitment from `config.init_values` — the recursion guest's /// supplied roots skip the in-VM FFT + Merkle build (see `verify_global`). /// `None` recomputes from `config` as before. -fn global_memory_air( +pub(crate) fn global_memory_air( opts: &ProofOptions, config: &PageConfig, preprocessed: Option, @@ -248,19 +250,46 @@ fn global_memory_air( // `address_lo = page_base_lo + OFFSET` is prover-chosen and the genesis // token can name an arbitrary address. GLOBAL_MEMORY's OFFSET column is // identical to PAGE's, so the same commitment serves both. - return air.with_preprocessed( - page::private_page_preprocessed_commitment(opts), + return air.with_lazy_preprocessed_columns( + page::private_page_lazy_commitment(opts), page::NUM_PREPROCESSED_COLS_PRIVATE, + Arc::new(|| vec![page::offset_column()]), ); } - let commitment = preprocessed.unwrap_or_else(|| { - if config.init_values.is_some() { - page::compute_precomputed_commitment(config, opts) - } else { - page::zero_init_preprocessed_commitment(opts) + // The columns as well as the root: the univariate path compares the root + // the verifier recomputes, the multilinear one has no second root and + // compares these instead. They are PAGE's — GLOBAL_MEMORY's preprocessed + // prefix is the same OFFSET and INIT, which is why the same commitment + // serves both. + // Both leaf layouts (S2): a zero-init page's one-row root is the static + // twin, a data page's is computed on first use; a supplied root is a + // row-pair root and never stands in for the other layout. + let commitment = if config.init_values.is_some() { + page::data_page_lazy_commitment(config, opts, preprocessed) + } else { + match preprocessed { + Some(c) => page::zero_init_lazy_commitment_from(c, opts), + None => { + let options = opts.clone(); + LazyCommitment::deferred(move || page::zero_init_preprocessed_commitment(&options)) + .with_one_row({ + let options = opts.clone(); + move || { + page::zero_init_preprocessed_commitment_for( + &options, + stark::leaf_layout::LeafLayout::Row, + ) + } + }) + } } - }); - air.with_preprocessed(commitment, global_memory::NUM_PREPROCESSED_COLS) + }; + let config = config.clone(); + air.with_lazy_preprocessed_columns( + commitment, + global_memory::NUM_PREPROCESSED_COLS, + Arc::new(move || page::preprocessed_columns(&config)), + ) } /// The sorted, deduped set of page bases the touched cells fall on — the SINGLE source @@ -270,7 +299,7 @@ fn global_memory_air( /// and verifier iterate the identical sequence — `multi_verify` matches AIRs to sub-proofs /// positionally. Carries page bases ONLY: no cell values, so private-input bytes never /// enter the bundle (unlike the full `CellBoundary`, whose `init.value` is a private byte). -fn touched_page_bases(boundaries: &[Arc>]) -> Vec { +pub(crate) fn touched_page_bases(boundaries: &[Arc>]) -> Vec { boundaries .iter() .flat_map(|epoch| epoch.iter()) @@ -280,6 +309,110 @@ fn touched_page_bases(boundaries: &[Arc>]) -> Vec { .collect() } +/// The cross-epoch page census of an execution, WITHOUT proving anything: how +/// many epochs the guest runs, which page bases the epoch-crossing cells fall +/// on, and how many cells each page contributes. This is the shape data the +/// aggregation layer's census consumes — the global memory proof commits one +/// GLOBAL_MEMORY table per touched page, sized by that page's crossing cells, +/// and the aggregator's cost per page follows the table height — so pricing +/// the aggregation program requires exactly this and nothing heavier. +#[cfg(test)] +pub(crate) struct BlockPageCensus { + pub num_epochs: usize, + /// Sorted, deduped — [`touched_page_bases`]' own order. + pub touched_page_bases: Vec, + pub num_private_input_pages: usize, + /// Epoch-crossing cells per touched page, in `touched_page_bases` order — + /// the GLOBAL_MEMORY table height driver (rows before padding). + pub page_cells: Vec, + /// Cells in each epoch's L2G table (rows before padding), epoch order. + pub l2g_cells: Vec, +} + +/// Runs the guest to completion and collects [`BlockPageCensus`] — the +/// producer loop's sequential-critical half (execute, collect, boundary, +/// image carry) with every prove and trace build omitted. +#[cfg(test)] +pub(crate) fn block_page_census( + elf_bytes: &[u8], + private_inputs: &[u8], + epoch_size_log2: u32, +) -> Result { + let epoch_size = 1usize + .checked_shl(epoch_size_log2) + .ok_or_else(|| Error::InvalidContinuationEpochSize("epoch size overflow".to_string()))?; + let elf = Elf::load(elf_bytes).map_err(|e| Error::ElfLoad(format!("{e}")))?; + let mut executor = Executor::new(&elf, private_inputs.to_vec()) + .map_err(|e| Error::Execution(format!("{e}")))?; + let decode_artifacts = DecodeArtifacts::from_elf(&elf)?; + let mut image = build_initial_image_paged(&elf, private_inputs); + let mut provenance = + local_to_global::genesis_provenance(image.iter().map(|(a, v)| (a, v as u64))); + + let mut boundaries: Vec>> = Vec::new(); + let mut prev_fini: Option> = None; + let mut index: u64 = 0; + loop { + if executor.pc() == 0 { + break; + } + if index >= local_to_global::MAX_EPOCHS { + return Err(Error::InvalidContinuationEpochSize( + "execution exceeds the IsB20 cross-epoch ordering range".to_string(), + )); + } + let register_init: Vec = match (index, prev_fini.take()) { + (0, _) => register::register_init_from_entry_point(elf.entry_point), + (_, Some(fini)) => fini, + (_, None) => { + return Err(Error::ContinuationInvariant( + "previous epoch final registers are missing after the first epoch".to_string(), + )); + } + }; + let logs = match executor + .resume_with_limit(epoch_size) + .map_err(|e| Error::Execution(format!("{e}")))? + { + Some(logs) => logs.to_vec(), + None => break, + }; + let is_final = executor.pc() == 0; + let collected = + Traces::collect_epoch(&decode_artifacts, &image, ®ister_init, &logs, is_final)?; + let boundary = Arc::new(local_to_global::epoch_boundary( + &mut provenance, + local_to_global::epoch_label(index), + &collected.touched_memory_cells(), + )); + prev_fini = Some(collected.register_fini(®ister_init)); + for cell in boundary.iter() { + image.set(cell.address, (cell.fini.value & 0xFF) as u8); + } + boundaries.push(boundary); + if is_final { + break; + } + index += 1; + } + + let bases = touched_page_bases(&boundaries); + let mut per_page: std::collections::BTreeMap = + bases.iter().map(|&b| (b, 0)).collect(); + for b in boundaries.iter().flat_map(|epoch| epoch.iter()) { + *per_page + .get_mut(&page::page_base_for_address(b.address)) + .expect("every crossing cell's page is in the census") += 1; + } + Ok(BlockPageCensus { + num_epochs: boundaries.len(), + page_cells: bases.iter().map(|b| per_page[b]).collect(), + touched_page_bases: bases, + num_private_input_pages: page::private_input_page_count(private_inputs), + l2g_cells: boundaries.iter().map(|b| b.len()).collect(), + }) +} + /// Canonicalize a possibly-untrusted, out-of-order page-base list to the same sorted, /// deduped form the prover produces via [`touched_page_bases`], so the verifier rebuilds /// tables in the committed order regardless of the wire order (a shuffled-but-same-set @@ -301,7 +434,7 @@ fn canonical_page_bases(page_bases: &[u64]) -> Vec { /// genesis from the ELF and never needs the raw private bytes. They are identified EXACTLY /// as the monolithic verifier does — the first `num_private_input_pages` pages from /// `PRIVATE_INPUT_START_INDEX` (see [`page::is_private_input_page`]). -fn global_memory_configs( +pub(crate) fn global_memory_configs( page_bases: &[u64], elf: &Elf, num_private_input_pages: usize, @@ -367,7 +500,7 @@ fn elf_page_has_data(elf: &Elf, page_base: u64) -> bool { /// non-preprocessed and never consults `init_values` (and its `init_page_data` is built /// from the ELF alone, so there is nothing to load) — the config carries an explicitly /// empty vec so no code path can silently start depending on verifier-side private data. -fn global_memory_configs_from_init_page_data( +pub(crate) fn global_memory_configs_from_init_page_data( page_bases: &[u64], init_page_data: &HashMap>, num_private_input_pages: usize, @@ -393,6 +526,588 @@ fn global_memory_configs_from_init_page_data( .collect() } +// =========================================================================== +// THE GENESIS-STACK SPLIT: which cross-epoch pages a PREPARED OPENING carries, +// and which keep the sparse closed form. +// +// ⛔ IT LIVES HERE AND NOT IN `crate::lfm`, AND THAT IS A LAYERING FACT, NOT A +// PREFERENCE. Three parties evaluate this rule — the cross-epoch PROVER, its +// VERIFIER, and the in-guest EMITTER — and they must reach the SAME answer from +// data all three hold, because the stack's root is absorbed in the roots block: +// two sides that disagree about the set commit different stacks, absorb +// different roots, and diverge at `z` with nothing in the failure naming the +// cause. `crate::lfm` depends on this module and not the reverse, so a rule the +// prover and verifier evaluate cannot live there or call anything that does. +// +// # Why there is a split at all +// +// A page's INIT column is its genesis bytes, zero past `init_values.len()`, and +// the in-guest verifier discharges it by the sparse form — one term per NONZERO +// entry, so [`sparse_leg_rows`]. Free for a zero-init page, cheap for a page +// holding a few constants, and ruinous for a page that is genuinely full. +// +// Measured on block 25368371 (ELF sha256 +// `3e8f1ce152d4d3cd60809ee7fc537e5c4f12145efec1881052c28119ff6a5482`, 2,711,800 +// B; input `573004e62e3680a00d3cdbae19dc4897e2ec60d6ec0c1d05d9ef118cb8aef17f`, +// 1,110,183 B; epoch 2^21), the run touches 33 pages — 28 genesis and 5 private +// — and the genesis bytes are CONCENTRATED: `0x0` carries 127,574 nonzero of +// 262,144, `0x200000` 61,514, and the other 26 are all-zero. Total 3,404,088 +// rows for INIT alone, against a 2-4 M band for the WHOLE cross-epoch program. +// Two pages of twenty-eight carry all of it. (The guest as built before fat LTO +// and the runtime memcpy, ELF `8f826601…`, touched 35 pages and charged +// 10,249,056 rows to three dense pages, `0x0`, `0x40000` and `0x280000`.) +// +// ⛔ THE PRIVATE-INPUT PAGES ARE EXCLUDED FIRST, AND NOT AS AN OPTIMISATION. +// [`global_memory_configs_from_init_page_data`] is called with +// `include_private_genesis = true` by the PROVER and `false` by the VERIFIER +// (through [`global_memory_configs`]), so a private-input page carries +// `init_values = Some()` on one side and `Some(vec![])` on +// the other. A threshold evaluated over every config would read a different +// nonzero count for those pages on the two sides and select a different set. +// The filter is on `is_private_input`, which both derive from `page_base` and +// `num_private_input_pages` — two values the cross-epoch statement already +// binds — and the reason is not "private pages are not worth stacking" but that +// a private page presents no INIT column to settle at all. +// +// # The rule, in two parts +// +// 1. **Is this page a CANDIDATE?** Its sparse leg [`sparse_leg_rows`] against +// what carrying it would ADD to the prepared leg, +// [`marginal_stacked_rows`], evaluated at the fixed height +// [`fixed_stack_vars`]. On the block that is `18 + 18*S > 111`, so +// `S >= 6`: a page with six nonzero genesis bytes is already worth more +// carried than kept, and the 26 all-zero pages are not. ⚠ BOTH SIDES MOVE +// WITH THE RUN'S BRACKET — the marginal is a form evaluated at the height +// this run's genesis page count puts the stack at, not a number quoted +// once, and `τ` is 5 below nine genesis pages and 6 from there. +// 2. **Is the CHAIN worth paying for?** The candidates' total savings against +// [`PREPARED_LEG_ROWS`], evaluated ONCE on the whole set. If it refuses, +// every candidate stays sparse — there is no partial answer, because a +// subset still pays the whole chain. +// +// ⛔ WHY IT TAKES TWO PARTS AND NOT ONE. The leg's cost is a fixed chain plus a +// per-page marginal, and no per-page rule can charge a fixed cost correctly: +// charge each page the whole chain (the rule this replaces) and two pages that +// would gladly share one are both refused; charge none of them and a lone page +// worth five rows drags in a chain costing 175,066. Splitting the question is +// what lets each term be charged to the thing that actually causes it. +// +// ⛔⛔ AND ALL THREE INPUTS ARE DATA ALL THREE PARTIES HOLD. Every `S` is a +// nonzero count over a genesis page's `init_values`, and `P_touched` is the +// genesis page count — both read off the very page configs the prover and the +// verifier each build from the ELF and the touched page list, and the emitter +// reads through the plan. Nothing in the rule reads a proof, a private byte, or +// the order the pages were considered in. `the_prover_and_verifier_views_of_a_ +// private_page_plan_alike` is the assertion of it. +// =========================================================================== + +/// A page table's height in variables. +/// +/// ⚠ DERIVED FROM [`page::DEFAULT_PAGE_SIZE`] AND NOT FROM A PROOF. Every +/// GLOBAL_MEMORY table is one page tall and [`page::preprocessed_columns`] +/// builds columns of exactly that many rows, so this is a property of the page +/// and not a claim a bundle gets to make. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) const PAGE_NUM_VARS: usize = page::DEFAULT_PAGE_SIZE.trailing_zeros() as usize; + +/// How many preprocessed columns a genesis page presents: `[OFFSET, INIT]`. +/// +/// ⚠ THE STACK TAKES BOTH, AND THAT IS A PRICE PAID TO A CONTRACT. Only INIT is +/// worth an opening — OFFSET is the identity ramp, whose extension is +/// `sum_k 2^k * r_k` and costs `num_vars - 1` rows to check outright. But +/// `check_preprocessed` skips a PREFIX of a table's preprocessed columns, and +/// `{1}` is not one, so settling INIT alone is inexpressible. Taking `{0, 1}` +/// keeps that contract unchanged at a cost of about 150 rows on the block: 34 +/// lost ramps against 188 for two more stacked columns (51 against 282 on the +/// old guest's three dense pages), on a hybrid costing order `10^5`. +pub(crate) const PAGE_PREPROCESSED_COLUMNS: usize = 2; + +/// The rows the in-guest verifier spends discharging one page's INIT by the +/// sparse closed form: `MLE(r) = sum_{v_i != 0} v_i * eq(r, i)`, which is +/// `num_vars` complements hoisted plus `num_vars` rows per nonzero entry. +/// +/// An all-zero page costs `num_vars` — the interned zero — and not nothing, +/// which is why the block's 26 zero pages are 468 rows and not 0. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn sparse_leg_rows(num_vars: usize, nonzero: usize) -> usize { + num_vars + num_vars * nonzero +} + +/// PART 2's term: the rows the prepared leg costs ONCE, however many pages it +/// carries — the stacked chain itself. +/// +/// ⛔⛔ THIS IS AN UPPER BOUND, NOT THE STACK'S COST, and calling it the cost +/// would be a number answering a retired question. It is V1i's card-free sizing +/// of a stacked family polynomial at **24 variables over 35 columns**; the stack +/// this code actually builds is `2 * n_dense` columns at +/// `PAGE_NUM_VARS + ceil(log2(2 * n_dense))` variables — six columns at 21 on +/// the block. Fewer variables means fewer rounds means fewer rows, so the real +/// chain is CHEAPER than this and the rule is conservative in the direction that +/// matters: the carried set must be worth more than the chain could possibly +/// cost before anything joins it. +/// +/// ⚠ IT PAYS FOR THE CHAIN AND FOR NOTHING ELSE. What a page adds to the leg by +/// being carried is [`marginal_stacked_rows`], and that is charged to the page +/// in part 1. Charging this constant per page — the rule this replaces — asked +/// each page to pay for a chain all of them share. +/// +/// ⚠ CONFIGURATION IS PART OF THE NUMBER. Chain rows move with blowup, folding +/// and the query count. It is a constant here rather than a call because of the +/// layering note above, and `lfm::whir_chain_tests` is where it is PINNED +/// against `chain_shape_rows` at the block's shape — the assertion that keeps +/// the word "bound" honest. +pub(crate) const PREPARED_LEG_ROWS: usize = 175_066; + +/// The stack height the rule CHARGES AT, which is deliberately not the height +/// the stack will have: `num_vars + ceil(log2(2 * genesis_pages))`. +/// +/// ⛔⛔ WHY A FIXED HEIGHT AT ALL. The real stack stands at +/// `num_vars + ceil(log2(2 * n_dense))`, and `n_dense` is the ANSWER — a +/// marginal evaluated there would be defined in terms of its own output, and +/// prover, verifier and emitter would each have to guess where the recursion +/// settled. Evaluating at every touched genesis page instead is a FIXED +/// quantity all three read off the same page configs, before any routing +/// decision is made. `n_dense <= genesis_pages`, so it is at or above the real +/// height. +/// +/// ★ AND THIS IS THE BRACKET THE MARGINAL IS EVALUATED AT, per run. +/// [`marginal_stacked_rows`] reads it, so a run standing taller is charged MORE +/// rather than charged too little — which is the whole gain over the literal +/// this replaces. The block's twenty-eight pages give 24 ([`BLOCK_STACK_VARS`]); a +/// lone page gives 19 and is charged 101 rather than the block's 111. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) const fn fixed_stack_vars(num_vars: usize, genesis_pages: usize) -> usize { + num_vars + (2 * genesis_pages).next_power_of_two().trailing_zeros() as usize +} + +/// The bracket the block's twenty-eight genesis pages put the stack at, and the +/// bracket every consequence quoted in these docs is quoted AT. +/// +/// ⛔⛔ A MARGINAL HAS NO MEANING WITHOUT ITS BRACKET, which is exactly what +/// the literal this replaces could not carry. [`marginal_stacked_rows`] is 101 +/// at a one-page stack, 111 here and 113 one bracket up; `τ` is 5 below 23 +/// variables and 6 from there. A number quoted for "the rule" is quoted at this +/// bracket unless it names another, and a run standing elsewhere is charged its +/// own. +/// +/// ⚠ IT IS A READING OF [`fixed_stack_vars`], NOT AN INPUT TO ANYTHING. No +/// production path reads it — `genesis_stack_plan` computes the bracket from +/// the configs it was handed — and `the_form_and_the_bracket_it_is_quoted_at` +/// asserts the two agree. +pub(crate) const BLOCK_STACK_VARS: usize = 24; + +/// The rows a stacked column costs OUTSIDE `weight_at_rows`: its +/// `absorb_unpack_rows()`, its share of `challenge_powers_rows(columns)` and its +/// entry in `columns_of(..)` — one row each. +/// +/// ⚠ A LITERAL RATHER THAN THREE CALLS, for the layering reason +/// [`PREPARED_LEG_ROWS`] gives: this module does not depend on `lfm`'s cost +/// forms, and the pin is what ties this number to them. +pub(crate) const STACKED_ROWS_PER_COLUMN: usize = 3; + +/// The most rows one more carried page can add to the THREADED sponge — the one +/// term of the marginal with no closed form, and a PROVEN BOUND rather than a +/// measurement. +/// +/// `stacked_verify_cost` absorbs `COORDINATES_PER_EXT` = 3 coordinates per +/// column into ONE buffer and squeezes once, at +/// `squeeze_rows(f) = f.div_ceil(4) + f.div_ceil(8) + 1`. A page is +/// [`PAGE_PREPROCESSED_COLUMNS`] columns, so six more felts, and the `+ 1` is +/// per-squeeze and differences away: +/// +/// ```text +/// Δ ceil(f/4) over f -> f + 6 ∈ {1, 2} +/// Δ ceil(f/8) over f -> f + 6 ∈ {0, 1} +/// ⇒ s ∈ {1, 2, 3}, and the bound is 3 +/// ``` +/// +/// ⚠ THE BOUND IS EXACT AND THE VALUE IS NOT. `s` cycles `2, 3, 1, 3` with +/// period 4 as pages are added, so the marginal at ONE bracket is a spread of +/// three values and the form charges the dearest. Six felts divide neither 4 +/// nor 8, which is the whole of why there is a spread to bound. +pub(crate) const MAX_SPONGE_MARGINAL: usize = 3; + +/// PART 1's term: the rows ONE MORE carried page adds to the prepared leg, at a +/// stack standing at `n_fixed` variables. +/// +/// ⛔⛔ THREE SHAPES WERE TRIED AND THE ARITHMETIC WAS NEVER THE PROBLEM: +/// +/// 1. a FORMULA, wrong because it omitted terms — two careful readings of it +/// each found a term the other's lacked, one the three rows per column paid +/// outside `weight_at_rows`, the other a shared `Sub` that differences to +/// ZERO; +/// 2. a LITERAL, wrong because the quantity is not constant — it took three +/// values at one bracket and a different three one bracket up, and a literal +/// carries its bracket only in prose, where prose goes stale; +/// 3. a FORMULA again, whose one irreducible term is a proven BOUND. +/// +/// ★ THE LESSON, which is the part worth carrying to the next cost like this: +/// when a cost splits into a deterministic part and a bounded nondeterministic +/// one, WRITE BOTH. A literal hides the split; a form that omits the bound +/// looks complete while being wrong. +/// +/// **The terms**, each by its shape (`lfm::whir_stacked::weight_at_rows`, +/// `lfm::whir_stacked::stacked_verify_cost`): +/// +/// 1. `5 * num_vars` — one `eq` over the page's own variables plus the `MulAdd` +/// that joins its group: `4n + (n - 1) + 1`. A page's two preprocessed +/// columns settle at ONE point, its own table's, so they are one group and +/// pay one `eq` between them. +/// 2. [`PAGE_PREPROCESSED_COLUMNS`] prefix indicators at +/// `max(n_fixed - num_vars, 1)` rows apiece. ⚠ THIS is the term that makes +/// the marginal a function of the bracket rather than a number: it grows two +/// rows per prefix bit, so the same page costs 101 in a one-page stack, 111 +/// in the block's and 113 one bracket above it. +/// 3. [`STACKED_ROWS_PER_COLUMN`] per column, the rows paid outside +/// `weight_at_rows`. +/// 4. [`MAX_SPONGE_MARGINAL`], the threaded sponge's contribution — the term +/// with no closed form, bounded rather than computed. +/// +/// ⛔ A FIFTH TERM IS NOT IN THE MARGINAL AT ALL: the shared `Sub` +/// `weight_at_rows` emits once per prefix POSITION any column of the polynomial +/// reads as a zero bit. It is per-POLYNOMIAL, so differenced across one more +/// page it contributes ZERO. It was carried here for a while as "≤ 1, +/// amortised"; differencing is what exposed that as a fudge, and it is why the +/// `+ 6` sits beside a 102 and not a 103. +/// +/// ★ SET-INDEPENDENCE SURVIVES THE CHANGE, which is the property the single +/// literal was protecting. `n_fixed` is [`fixed_stack_vars`] of the GENESIS +/// PAGE COUNT, a quantity prover, verifier and emitter each derive from the ELF +/// and the touched page list before any routing decision exists. It is NOT +/// `n_dense`, which is this rule's own output — see [`fixed_stack_vars`]. +/// +/// ⚠ IT IS AN UPPER BOUND AT EVERY BRACKET, deliberately: part 1 asks a page to +/// beat the dearest position it could occupy, and part 2 understates the set's +/// savings, so both parts are conservative. The block is charged above its own +/// stack twice over — its two dense pages are four columns standing at +/// `n_stack` 20 with a prefix of 2, so its true weight marginal is 94 against +/// the 102 charged here, before `s`. +/// +/// The pin +/// `lfm::whir_stacked_tests::the_marginal_the_routing_rule_charges_is_the_one_the_stack_bills` +/// is where this form is read off `stacked_verify_cost` rather than asserted +/// against itself. +pub(crate) const fn marginal_stacked_rows(num_vars: usize, n_fixed: usize) -> usize { + // `max(n_fixed - num_vars, 1)` without `Ord::max`, which is not const: a + // column filling the whole stack has no indicator and pays only its fold. + let prefix = if n_fixed > num_vars + 1 { + n_fixed - num_vars + } else { + 1 + }; + 5 * num_vars + + PAGE_PREPROCESSED_COLUMNS * prefix + + PAGE_PREPROCESSED_COLUMNS * STACKED_ROWS_PER_COLUMN + + MAX_SPONGE_MARGINAL +} +/// PART 1: whether a page is even a CANDIDATE — whether keeping it sparse costs +/// more than carrying it would add to the leg. +/// +/// ★ SET-INDEPENDENT, AND THE `n_fixed` ARGUMENT IS WHY IT STAYS SO. Both +/// sides of the comparison are this page's own nonzero count and a form +/// evaluated at the GENESIS PAGE COUNT's bracket — never at `n_dense`, which is +/// the answer. Nothing here depends on which other pages qualified or on the +/// order they were considered in, and prover, verifier and emitter reach the +/// same bracket from the same page configs. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn is_candidate(num_vars: usize, n_fixed: usize, nonzero: usize) -> bool { + nonzero >= candidate_threshold_entries(num_vars, n_fixed) +} + +/// The fewest nonzero entries that make a page a candidate — `τ`, the number +/// part 1 is quoted by, and the form [`is_candidate`] is decided on. +/// +/// The least `S` with +/// `num_vars + num_vars * S > marginal_stacked_rows(num_vars, n_fixed)`, +/// which is floor plus one and NOT `div_ceil`: were the division exact, +/// `div_ceil` would hand back an `S` whose leg EQUALS the marginal rather than +/// exceeding it. +/// +/// ⚠ `τ` IS DERIVED FROM THE MARGINAL AND IS NOT ITSELF A RULED QUANTITY. It +/// is 5 at this literal, holding at 5 for a marginal in `[90, 107]`, at 6 on +/// `[108, 125]` and at 7 on `[126, 143]` — so the form's 111 will make it 6. +/// The only page that difference reclassifies is one carrying five nonzero +/// genesis bytes, +/// and neither the block (0, or 61,514 and up) nor any fixture (112, 65,652) +/// sits there; `the_ruled_boundaries_hold_for_every_marginal_in_band` is the +/// reading of that rather than the assurance. +/// +/// ⚠ ONE FORM, NOT TWO. Part 1 could as easily be the comparison +/// `sparse_leg_rows(..) > marginal_stacked_rows(..)`, and the two agree for +/// every `S` — `the_threshold_and_its_entry_count_are_one_rule` is the reading +/// of that rather than the algebra taken on trust. Deciding through `τ` keeps +/// the number a reader is quoted and the number the code branches on one +/// object. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn candidate_threshold_entries(num_vars: usize, n_fixed: usize) -> usize { + if num_vars == 0 { + // A page of no rows has no sparse leg to save, so nothing qualifies. + return usize::MAX; + } + marginal_stacked_rows(num_vars, n_fixed).saturating_sub(num_vars) / num_vars + 1 +} + +/// What carrying one candidate SAVES: its sparse leg, less what it adds to the +/// prepared one. Zero for a page that is not a candidate. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn page_savings(num_vars: usize, n_fixed: usize, nonzero: usize) -> usize { + sparse_leg_rows(num_vars, nonzero).saturating_sub(marginal_stacked_rows(num_vars, n_fixed)) +} + +/// PART 2: whether the candidate set is worth paying the chain for at all. +/// +/// ⛔ EVALUATED ONCE, ON THE WHOLE SET, AND THAT IS WHY IT IS A SEPARATE PART. +/// The chain is paid once however many pages ride it, so no per-page rule can +/// see it: a per-page test with a fixed-cost term in it charges every page for +/// the same chain and refuses two pages that would gladly share one. That is +/// the case the single-page rule got wrong — two pages of 5,000 entries apiece +/// were each worth less than the chain and jointly worth more, and both stayed +/// sparse at 180,036 rows. +/// +/// If this refuses, EVERY candidate stays sparse. There is no partial answer: +/// carrying a subset still pays the whole chain. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn chain_is_paid(total_savings: usize) -> bool { + total_savings > PREPARED_LEG_ROWS +} + +/// The most nonzero entries a page this rule leaves SPARSE can carry. +/// +/// ⛔⛔ THE QUANTITY THE SPARSE-LEG CAP MUST CLEAR, and it is a property of the +/// two-part rule rather than of one page. A page is left sparse only when part +/// 2 refused, so the whole candidate set saved at most [`PREPARED_LEG_ROWS`]; +/// the total is at least this page's own savings, so this page saved at most +/// that too. Invert [`page_savings`]: +/// +/// ```text +/// num_vars + num_vars * S - MARGINAL <= PREPARED_LEG_ROWS +/// S <= (PREPARED_LEG_ROWS + MARGINAL - num_vars) / num_vars +/// ``` +/// +/// ⚠ IT BOUNDS A NON-CANDIDATE TOO, and by much more room: a page failing part +/// 1 has a sparse leg no larger than the marginal, which is three orders under +/// this. So the one number covers both ways a page can end up sparse. +/// +/// ⛔ AND IT MOVES WITH THE BRACKET — one entry per `num_vars` rows of +/// marginal. At the block's 24 variables it is **9,731**, so the pair is +/// `(9,731 sparse, 9,732 dense)`, holding for a marginal in `[110, 127]`; at a +/// LONE page's 19 the marginal is 101 and the pair is `(9,730, 9,731)`, on +/// `[92, 109]`. ⚠ SO THE PAIR IS A PROPERTY OF THE RUN AND NOT OF THE RULE, and +/// no test hard-codes one: they assert this function and `+ 1` at a named +/// bracket, and the band assertion names the pair when it moves. The +/// cap-overlap conclusion is untouched at any of these values — the margin to +/// `MAX_SPARSE_INIT_ENTRIES` is six-fold. +pub(crate) fn densest_sparse_entries(num_vars: usize, n_fixed: usize) -> usize { + if num_vars == 0 { + return 0; + } + (PREPARED_LEG_ROWS + marginal_stacked_rows(num_vars, n_fixed)).saturating_sub(num_vars) + / num_vars +} + +/// How many of the pages are GENESIS pages — the count `n_fixed` is taken at. +/// +/// ⚠ THE PRIVATE FILTER IS APPLIED HERE TOO, and for the same reason it is +/// applied in the plan: the prover's and the verifier's views of a private page +/// differ in their bytes, and a count that included them would still agree, but +/// a rule that read their bytes would not. Counting only what presents an INIT +/// column keeps every input to the routing decision one both sides derive from +/// the ELF. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn genesis_page_count(configs: &[PageConfig]) -> usize { + configs.iter().filter(|c| !c.is_private_input).count() +} + +/// How many of a page's genesis bytes are nonzero — the only quantity the +/// threshold reads. +/// +/// `init_values` is not padded to the page, so every offset at or past its +/// length is zero and costs nothing; a `None` page is zero to the last byte. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn nonzero_entries(config: &PageConfig) -> usize { + config + .init_values + .as_ref() + .map(|values| values.iter().filter(|&&b| b != 0).count()) + .unwrap_or(0) +} + +/// One page of the cross-epoch page family, as the routing decision sees it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PageRoute { + /// Its index in the AIR set, bookends included — the index a + /// `PreparedColumn` names. + pub table: usize, + pub page_base: u64, + pub nonzero: usize, + /// `false` for a private-input page, which presents no INIT column at all. + pub has_init: bool, + /// PART 1's answer: keeping this page sparse costs more than carrying it + /// would add to the leg. + /// + /// ⚠ RECORDED SEPARATELY FROM `dense` ON PURPOSE. A candidate that is not + /// dense is a page the chain could not be paid for, and that is a different + /// state from a page nobody wanted — the one the fixtures are actually in. + /// Without this field a reader meets an absence and has to infer which rule + /// produced it. + pub candidate: bool, + /// Both parts' answer: the opening carries this page. + pub dense: bool, +} + +/// Which genesis pages the prepared opening carries, and what it costs the pages +/// it leaves behind. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct GenesisStackPlan { + /// Every touched page, in the AIR set's own page order. + pub routes: Vec, + /// Where each stacked column is settled, in stack order: both preprocessed + /// columns of each dense page, dense pages in canonical page-base order. + /// + /// ⚠ THE ORDER IS A CONTRACT AND IS ASSERTED, NOT AGREED. Run `k` of the + /// commitment settles table `k`'s claims, so a silent reorder would settle + /// one page's values against another page's commitment while every value + /// gate stayed green. `multilinear_table::prepared_runs` refuses a list that + /// does not visit each table once in stack order. + pub at: Vec, + /// The rows the pages NOT carried still cost by the sparse form. + pub sparse_rows: usize, + /// The height THIS run's stack would stand at, [`fixed_stack_vars`] of the + /// genesis page count. + /// + /// ★ AND IT DECIDES THE MARGINAL. [`marginal_stacked_rows`] is evaluated + /// here, so this field is the bracket every number in this plan is charged + /// at — the block's 24, a lone page's 19. It is recorded rather than + /// recomputed by readers for the reason the order in `at` is asserted: two + /// spellings of one quantity is how they come to differ. + pub n_fixed: usize, + /// What the candidate set saves, the quantity part 2 weighs against + /// [`PREPARED_LEG_ROWS`]. Nonzero even when nothing was carried: that is + /// the reading that says the chain was refused rather than unwanted. + pub savings: usize, +} + +impl GenesisStackPlan { + /// Whether anything is stacked at all. A run whose genesis is entirely + /// sparse carries no prepared opening, and its cross-epoch proof is then + /// byte-for-byte the one it was before this route existed. + #[cfg(test)] + pub fn is_empty(&self) -> bool { + self.at.is_empty() + } + + /// The page-family indices the opening carries, in stack order. + #[cfg(test)] + pub fn dense_pages(&self) -> Vec { + self.routes + .iter() + .enumerate() + .filter(|(_, r)| r.dense) + .map(|(page, _)| page) + .collect() + } +} + +/// The routing decision, from the page configs and where the page family starts +/// in the AIR set. +/// +/// `num_bookends` is how many local-to-global tables precede the pages — the +/// cross-epoch AIR set is every bookend then every page, and `WhirGlobalAirs` +/// is the one place that order is written. The `PreparedColumn::table` indices +/// this produces are indices into THAT order, because that is the order +/// `multi_prove` and `multi_verify` match positionally. +/// +/// ⛔ TWO PASSES, BECAUSE PART 2 IS A PROPERTY OF THE SET. The first decides +/// candidacy and totals the savings; only then can the second know whether the +/// chain is paid for, and therefore whether any candidate is carried. Doing it +/// in one pass would mean deciding a page's fate before the quantity that +/// decides it has been computed — which is the set-dependence this rule is +/// built to avoid, wearing the other hat. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn genesis_stack_plan( + configs: &[PageConfig], + num_bookends: usize, + num_vars: usize, +) -> GenesisStackPlan { + // ⚠ COMPUTED ONCE AND CONSUMED BY BOTH PASSES. This is the height THIS + // run's stack would stand at, and [`marginal_stacked_rows`] is evaluated at + // it, so a run standing taller is charged MORE rather than charged too + // little. It is derived from the genesis page COUNT — never from how many + // pages turn out to be dense, which is this function's own output. + let n_fixed = fixed_stack_vars(num_vars, genesis_page_count(configs)); + + // PASS 1 — candidacy, and what the candidates would save between them. + let mut routes: Vec = Vec::with_capacity(configs.len()); + let mut savings = 0usize; + for (page, config) in configs.iter().enumerate() { + // ⛔ The private filter comes FIRST and is not the threshold's business; + // see the section header for what a threshold reading those bytes does. + let has_init = !config.is_private_input; + let nonzero = if has_init { nonzero_entries(config) } else { 0 }; + let candidate = has_init && is_candidate(num_vars, n_fixed, nonzero); + if candidate { + savings += page_savings(num_vars, n_fixed, nonzero); + } + routes.push(PageRoute { + table: num_bookends + page, + page_base: config.page_base, + nonzero, + has_init, + candidate, + dense: false, + }); + } + + // PASS 2 — the chain, paid for the whole set or for none of it. + let paid = chain_is_paid(savings); + let mut at = Vec::new(); + let mut sparse_rows = 0usize; + for route in &mut routes { + route.dense = route.candidate && paid; + if route.dense { + // BOTH preprocessed columns, in index order: the prefix + // `check_preprocessed` skips. See `PAGE_PREPROCESSED_COLUMNS`. + at.extend(stark::multilinear_table::leading_columns( + route.table, + PAGE_PREPROCESSED_COLUMNS, + )); + } else if route.has_init { + sparse_rows += sparse_leg_rows(num_vars, route.nonzero); + } + } + + GenesisStackPlan { + routes, + at, + sparse_rows, + n_fixed, + savings, + } +} + +/// The stacked columns themselves, in stack order: each dense page's OFFSET then +/// its INIT. +/// +/// ⚠ TAKEN FROM [`page::preprocessed_columns`] AND NOT REBUILT. The columns the +/// opening commits must be the columns the page table's own argument claims, and +/// the page AIR's preprocessed columns come from that function; a second +/// spelling of "the genesis bytes as a column" is how two objects with the same +/// name come to hold different values. +/// +/// ⚠ AND THE ORDER MATCHES [`GenesisStackPlan::at`] BY CONSTRUCTION — both walk +/// `dense_pages()` and then the page's own preprocessed order. The assertion +/// that they agree is in `genesis_prepared_for`, where the two meet. +#[cfg(test)] +pub(crate) fn genesis_stack_columns( + configs: &[PageConfig], + plan: &GenesisStackPlan, +) -> Vec> { + plan.dense_pages() + .into_iter() + .flat_map(|page| page::preprocessed_columns(&configs[page])) + .collect() +} + /// Per-epoch register state and label. struct EpochStart<'a> { register_init: &'a [u32], @@ -407,15 +1122,18 @@ struct EpochStart<'a> { /// `boundary` is shared (`Arc`): the same per-epoch boundary feeds both this /// epoch's prove and the cross-epoch global prove, which starts as soon as the /// producer has prepared the last epoch (see `prove_continuation`). -struct PreparedEpoch { - index: u64, - register_init: Vec, - label: u64, - traces: Traces, - boundary: Arc>, - is_final: bool, +pub(crate) struct PreparedEpoch { + pub index: u64, + pub register_init: Vec, + pub label: u64, + pub traces: Traces, + pub boundary: Arc>, + pub is_final: bool, } +/// Every epoch's boundary, in epoch order. +pub(crate) type EpochBoundaries = Vec>>; + /// A collected-but-not-yet-built epoch, handed from the producer to the trace /// builder pool. Everything sequential (execution, op collection over the /// advancing memory image, boundary + register-fini derivation) already @@ -431,6 +1149,28 @@ struct BuildJob { is_final: bool, } +/// What a trace builder hands the STARK base's prover: a built epoch, or — +/// when the preparation runs ahead of the prover ([`base_prep_ahead`], the +/// default) — one whose host preparation ([`prep_epoch`]) is done too. +/// +/// Unboxed `Built` on purpose: it is the channel item of the schedule that +/// prepares on the prover thread, moved once per epoch, and that schedule +/// keeps exactly its allocations. +#[allow(clippy::large_enum_variant)] +enum ReadyEpoch { + Built(PreparedEpoch), + Prepped { index: u64, prep: Box }, +} + +impl ReadyEpoch { + fn index(&self) -> u64 { + match self { + ReadyEpoch::Built(p) => p.index, + ReadyEpoch::Prepped { index, .. } => *index, + } + } +} + /// One epoch's proof plus everything a standalone verifier needs to re-check it /// using ONLY the bundle (never the prover's in-memory traces). Each field is a /// public value the verifier re-binds: a wrong value either makes the proof's @@ -440,9 +1180,11 @@ struct BuildJob { /// Note: continuation epochs use the L2G memory bookend, so PAGE is skipped and the /// per-epoch page config set is empty — the verifier builds the AIRs with no PAGE /// tables rather than trusting any prover-supplied page config. -#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] -struct EpochProof { - /// The epoch's STARK proof (its tables + the epoch-local L2G sub-table last). +#[derive(Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub(crate) struct EpochProof { + /// The epoch's STARK proof: one `StarkProof` per table, the epoch-local L2G + /// sub-table last. Its main root is the commitment + /// `verify_l2g_commitment_binding_view` ties to the global proof. proof: MultiProof, /// Bytes this epoch committed — the COMMIT-bus receiver reference. public_output: Vec, @@ -496,6 +1238,12 @@ impl ContinuationProof { self.epochs.len() } + /// Every epoch's proof, in execution order. + #[cfg(test)] + pub(crate) fn epochs(&self) -> &[EpochProof] { + &self.epochs + } + /// What each epoch declared it carries, for tests that have to show the /// epochs disagree. `epochs` itself stays private. #[cfg(test)] @@ -504,38 +1252,159 @@ impl ContinuationProof { } } +#[cfg(test)] +impl ContinuationProof { + /// Test-only: flip one bit of epoch `i`'s bound `reg_fini` — a + /// prover-supplied bundle field the verifier re-binds through the REGISTER + /// preprocessed commitment, so any consumer that rebuilds the epoch's AIRs + /// from the bundle must reject the flip. + pub(crate) fn corrupt_epoch_reg_fini_for_tests(&mut self, i: usize) { + self.epochs[i].reg_fini[0] ^= 1; + } + + /// Test-only: flip one bit of epoch `i`'s claimed L2G root — the value + /// [`verify_epoch`] checks against the proof's own committed root. + pub(crate) fn corrupt_epoch_l2g_root_for_tests(&mut self, i: usize) { + self.epochs[i].l2g_root[0] ^= 1; + } +} + +/// The chain-derived inputs epoch `index` of a bundle verifies under, by the +/// same rule as [`verify_continuation_view`]'s loop: `register_init` starts +/// from the ELF entry point, each epoch's bound `reg_fini` becomes the next +/// epoch's INIT, the label is the epoch's position, and the last epoch is +/// final. +#[cfg(test)] +pub(crate) struct EpochChainPosition { + pub(crate) register_init: Vec, + pub(crate) is_final: bool, + pub(crate) label: u64, +} + +/// Derive [`EpochChainPosition`] for epoch `index` of a run of `num_epochs` +/// from its leading `epochs` (a bundle's all, or — before the bundle exists — +/// at least `0..=index`). `Ok(None)` if `index` is out of range, the epoch is +/// not in `epochs`, or an earlier epoch's `reg_fini` has the wrong length (a +/// malformed bundle the verifier rejects up front); `Err` iff a metadata field +/// fails to materialize. +#[cfg(test)] +pub(crate) fn epoch_chain_position( + epochs: &[EpochProof], + num_epochs: usize, + elf: &Elf, + index: usize, +) -> Result, Error> { + if index >= num_epochs || index >= epochs.len() { + return Ok(None); + } + let mut register_init = register::register_init_from_entry_point(elf.entry_point); + for prior in &epochs[..index] { + let view = EpochProofView::Owned(prior); + if view.reg_fini_len() != register::NUM_REGISTER_ADDRESSES { + return Ok(None); + } + register_init = view.reg_fini()?; + } + Ok(Some(EpochChainPosition { + register_init, + is_final: index == num_epochs - 1, + label: local_to_global::epoch_label(index as u64), + })) +} + +/// Zero-copy readers over an ARCHIVED bundle, for the LFM arena filler. +/// +/// Deliberately on the archived type only. The recursion guest never holds a +/// `ContinuationProof` — it reads a blob from private input and verifies in +/// place ([`verify_continuation_archived`]) — so these expose a path production +/// actually traverses. The equivalent on the owned type would expose a structure +/// the real recursion path never sees, which is a weaker proposition. +/// +/// Methods rather than relaxed field visibility because rkyv mirrors the source +/// field's visibility onto the archived struct: opening `epochs` would open the +/// owned type at the same time. +impl ArchivedContinuationProof { + pub(crate) fn num_epochs(&self) -> usize { + self.epochs.len() + } + + /// Epoch `i`'s STARK proof (its tables, epoch-local L2G sub-table last), + /// as the same view the verifier reads in place. + pub(crate) fn epoch_proof(&self, i: usize) -> MultiProofView<'_, F, E, ()> { + MultiProofView::Archived(&self.epochs[i].proof) + } + + /// Bytes epoch `i` committed. + pub(crate) fn epoch_public_output(&self, i: usize) -> &[u8] { + self.epochs[i].public_output.as_slice() + } + + /// Epoch `i`'s own committed L2G table root — the left-hand side of the + /// cross-epoch binding [`crate::verify_l2g_commitment_binding_view`] checks + /// against the global proof's `i`-th sub-proof. + pub(crate) fn epoch_l2g_root(&self, i: usize) -> Commitment { + self.epochs[i].l2g_root + } + + /// Epoch `i`'s final register file `R_{i+1}`, the vector + /// [`build_epoch_airs`] preprocesses as FINI and the chaining loop carries + /// forward as epoch `i+1`'s INIT. + pub(crate) fn epoch_reg_fini(&self, i: usize) -> Result, Error> { + EpochProofView::Archived(&self.epochs[i]).reg_fini() + } + + /// The one cross-epoch global-memory proof, as the same view the verifier + /// reads in place. Its first `num_epochs()` sub-proofs are the per-epoch L2G + /// tables the binding ties to. + pub(crate) fn global_proof(&self) -> MultiProofView<'_, F, E, ()> { + MultiProofView::Archived(&self.global) + } +} + /// Borrowed view over an [`EpochProof`] (owned or archived-in-place). Lets /// `verify_epoch` take a single argument again instead of the field-by-field /// parameter list the owned/archived split used to force on every caller: /// each accessor reads straight off whichever representation is behind it, a /// plain field copy on the owned side and (for the small metadata fields) an -/// `rkyv::deserialize` on the archived side. +/// `rkyv::deserialize` on the archived side. The wrap flow's from-proof +/// constructor reads the same view (via [`ContinuationProof::epoch_view`] + +/// [`reconstruct_epoch_airs`]), so an epoch's verifier-side reconstruction has +/// exactly one reader-facing surface. #[derive(Clone, Copy)] -enum EpochProofView<'a> { +pub(crate) enum EpochProofView<'a> { Owned(&'a EpochProof), Archived(&'a ArchivedEpochProof), } impl<'a> EpochProofView<'a> { - /// The epoch's STARK proof (its tables + the epoch-local L2G sub-table - /// last), as a [`MultiProofView`] — never materialized into an owned - /// `MultiProof` on the archived side. - fn proof(&self) -> MultiProofView<'a, F, E, ()> { + /// The epoch's proof (its tables + the epoch-local L2G sub-table last), as + /// a [`MultiProofView`] — never materialized into an owned `MultiProof` on + /// the archived side. + pub(crate) fn per_table_proof(&self) -> MultiProofView<'a, F, E, ()> { match self { Self::Owned(e) => MultiProofView::Owned(&e.proof), Self::Archived(e) => MultiProofView::Archived(&e.proof), } } + /// Sub-proof count — one per table, which is what + /// [`reconstruct_epoch_airs`]'s structural check needs. + pub(crate) fn num_sub_proofs(&self) -> usize { + match self { + Self::Owned(e) => e.proof.proofs.len(), + Self::Archived(e) => e.proof.proofs.len(), + } + } + /// Bytes this epoch committed (zero-copy borrow either way). - fn public_output(&self) -> &'a [u8] { + pub(crate) fn public_output(&self) -> &'a [u8] { match self { Self::Owned(e) => &e.public_output, Self::Archived(e) => e.public_output.as_slice(), } } - fn table_counts(&self) -> Result { + pub(crate) fn table_counts(&self) -> Result { match self { Self::Owned(e) => Ok(e.table_counts.clone()), Self::Archived(e) => { @@ -549,7 +1418,7 @@ impl<'a> EpochProofView<'a> { /// Always empty for continuation epochs (PAGE is skipped); still routed /// through the archive rather than assumed, so a malformed non-empty /// bundle value surfaces instead of being silently ignored. - fn runtime_page_ranges(&self) -> Result, Error> { + pub(crate) fn runtime_page_ranges(&self) -> Result, Error> { match self { Self::Owned(e) => Ok(e.runtime_page_ranges.clone()), Self::Archived(e) => rkyv::deserialize::, rkyv::rancor::Error>( @@ -568,7 +1437,7 @@ impl<'a> EpochProofView<'a> { } } - fn reg_fini(&self) -> Result, Error> { + pub(crate) fn reg_fini(&self) -> Result, Error> { match self { Self::Owned(e) => Ok(e.reg_fini.clone()), Self::Archived(e) => rkyv::deserialize::, rkyv::rancor::Error>(&e.reg_fini) @@ -578,7 +1447,7 @@ impl<'a> EpochProofView<'a> { } } - fn l2g_root(&self) -> Commitment { + pub(crate) fn l2g_root(&self) -> Commitment { match self { Self::Owned(e) => e.l2g_root, Self::Archived(e) => e.l2g_root, @@ -647,7 +1516,7 @@ impl<'a> ContinuationProofView<'a> { /// use the L2G bookend, so PAGE is skipped and `page_configs` is empty. The /// epoch-local L2G air is built separately by the caller (it needs the `label`). #[allow(clippy::too_many_arguments)] -fn build_epoch_airs( +pub(crate) fn build_epoch_airs( elf: &Elf, opts: &ProofOptions, page_configs: &[PageConfig], @@ -662,6 +1531,11 @@ fn build_epoch_airs( // final register file is a verifier-known public value bound by the REG-C2 // Memory-bus token; reusing the same R_{i+1} as the next epoch's INIT binds // init(epoch i+1) == fini(epoch i). + let register_preprocessed = crate::RegisterPreprocessed { + commitment: register::compute_precomputed_commitment(opts, register_init, reg_fini), + init: register_init, + fini: reg_fini, + }; VmAirs::new( elf, opts, @@ -672,25 +1546,40 @@ fn build_epoch_airs( decode_commitment, is_final, None, - register::compute_precomputed_commitment(opts, register_init, reg_fini), + register_preprocessed, ) } -/// Prove one epoch (prove half only). Commits its local-to-global table (built from -/// `boundary`) on the epoch-local Memory bus and its REGISTER table with FINI -/// preprocessed to the epoch's final register file. Returns the [`EpochProof`] the -/// standalone verifier later re-checks; does NOT verify here. +/// One epoch's host-side preparation for its prove, done: the finished trace +/// set, its AIRs, the epoch-local L2G table and the statement values the +/// transcript absorbs. [`prep_epoch`] builds it and [`prove_prepped_epoch`] +/// proves it, so the two can run on different threads (by default the prep on +/// a trace builder, ahead of the prover: [`base_prep_ahead`]). +pub(crate) struct EpochPrep { + traces: Traces, + airs: VmAirs, + l2g_air: AirWithBuses, + l2g_trace: TraceTable, + reg_fini: Vec, + table_counts: TableCounts, + public_output: Vec, + runtime_page_ranges: Vec, + label: u64, + is_final: bool, +} + +/// [`prove_epoch`]'s host half: everything before the prove. #[allow(clippy::too_many_arguments)] -fn prove_epoch( +fn prep_epoch( elf: &Elf, - elf_bytes: &[u8], - start: &EpochStart, + register_init: &[u32], + label: u64, mut traces: Traces, is_final: bool, boundary: &[CellBoundary], opts: &ProofOptions, decode_commitment: Commitment, -) -> Result { +) -> Result { // Count this L2G table's range-check lookups into the BITWISE table so its // AreBytes/IsHalfword multiplicities balance the range-check senders. crate::tables::bitwise::update_multiplicities( @@ -722,7 +1611,7 @@ fn prove_epoch( &[], &table_counts, &cpu_heights, - start.register_init, + register_init, ®_fini, is_final, // Computed once per prove_continuation — the DECODE commitment is a @@ -731,7 +1620,44 @@ fn prove_epoch( Some(decode_commitment), ); - let label = start.label; + let l2g_air = l2g_memory_air(opts, label); + // Build this epoch's L2G table from the cross-epoch boundary so it is identical + // to the one the global proof commits (the commitment binding compares their + // roots). It is appended to the proof below, not through `air_trace_pairs`. + let l2g_trace = local_to_global::generate_local_to_global_trace(boundary); + + Ok(EpochPrep { + traces, + airs, + l2g_air, + l2g_trace, + reg_fini, + table_counts, + public_output, + runtime_page_ranges, + label, + is_final, + }) +} + +/// [`prove_epoch`]'s prove half, over a finished [`EpochPrep`]. +fn prove_prepped_epoch( + prep: EpochPrep, + elf_bytes: &[u8], + opts: &ProofOptions, +) -> Result { + let EpochPrep { + mut traces, + airs, + l2g_air, + mut l2g_trace, + reg_fini, + table_counts, + public_output, + runtime_page_ranges, + label, + is_final, + } = prep; let seed = || { epoch_transcript( elf_bytes, @@ -744,19 +1670,15 @@ fn prove_epoch( ) }; - let l2g_air = l2g_memory_air(opts, label); - // Build this epoch's L2G table from the cross-epoch boundary so it is identical - // to the one the global proof commits (the commitment binding compares their - // roots). It is appended to the proof below, not through `air_trace_pairs`. - let mut l2g_trace = local_to_global::generate_local_to_global_trace(boundary); - let mut pairs = airs.air_trace_pairs(&mut traces); pairs.push((&l2g_air, &mut l2g_trace, &())); - let proof = Prover::multi_prove( + + #[cfg(feature = "shape-profile")] + crate::shape_profile::capture(pairs.iter().map(|(air, trace, _)| (*air, trace.num_rows()))); + let proof = crate::hash_pin::BlockProver::::multi_prove( pairs, &mut seed(), - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| Error::Prover(format!("{e:?}")))?; @@ -778,33 +1700,68 @@ fn prove_epoch( }) } -/// Verify one epoch using ONLY the epoch's public statement fields (via -/// [`EpochProofView`]) plus the verifier-derived `register_init` (epoch 0: -/// from the ELF; epoch i>0: from the previous epoch's `reg_fini`), `is_final`, -/// and `label`. Rebuilds the AIRs and transcript from the bundle's statement -/// values and indexes commits from the carried x254 -/// (`register_init[X254_INDEX]`), never from the prover's memory. PAGE is -/// skipped for continuation epochs, so the AIRs are built with no page configs -/// (the bundle does not get to supply any). Returns `Ok(true)` iff the proof -/// verifies and its committed L2G root matches the claimed one; `Err` iff a -/// small metadata field failed to materialize off an archived bundle. -/// -/// `epoch` is zero-copy either way: owned or archived (see the two callers). +/// Prove one epoch (prove half only). Commits its local-to-global table (built from +/// `boundary`) on the epoch-local Memory bus and its REGISTER table with FINI +/// preprocessed to the epoch's final register file. Returns the [`EpochProof`] the +/// standalone verifier later re-checks; does NOT verify here. #[allow(clippy::too_many_arguments)] -fn verify_epoch( +fn prove_epoch( elf: &Elf, elf_bytes: &[u8], + start: &EpochStart, + traces: Traces, + is_final: bool, + boundary: &[CellBoundary], + opts: &ProofOptions, + decode_commitment: Commitment, +) -> Result { + let prep = prep_epoch( + elf, + start.register_init, + start.label, + traces, + is_final, + boundary, + opts, + decode_commitment, + )?; + prove_prepped_epoch(prep, elf_bytes, opts) +} + +/// An epoch's verifier-side reconstruction: the AIR set (VM tables + the +/// epoch-local L2G air) and the statement values, all rebuilt from the proof +/// bundle plus the chain-derived inputs (`register_init`, `is_final`, +/// `label`). Shared by [`verify_epoch`] and the wrap flow's +/// `RealEpoch`-from-proof constructor so the two reconstructions cannot +/// diverge. +pub(crate) struct EpochReconstruction { + pub(crate) airs: VmAirs, + pub(crate) l2g_air: Box>, + pub(crate) table_counts: TableCounts, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) reg_fini: Vec, + pub(crate) runtime_page_ranges: Vec, +} + +/// Rebuild epoch `epoch`'s AIR set from the bundle alone. `Ok(None)` = a +/// well-formed bundle that is structurally wrong (degenerate table counts, +/// sub-proof count mismatch) — the cases [`verify_epoch`] rejects with +/// `Ok(false)`; `Err` iff a small metadata field failed to materialize off an +/// archived bundle. +#[allow(clippy::too_many_arguments)] +pub(crate) fn reconstruct_epoch_airs( + elf: &Elf, epoch: EpochProofView<'_>, register_init: &[u32], is_final: bool, label: u64, opts: &ProofOptions, decode_commitment: Option, -) -> Result { +) -> Result, Error> { let table_counts = epoch.table_counts()?; // Reject degenerate table counts (mirrors the monolithic verifier). if table_counts.validate().is_err() { - return Ok(false); + return Ok(None); } // Cross-check table_counts before building AIRs from bundle data. Continuation @@ -815,32 +1772,35 @@ fn verify_epoch( } else { FIXED_TABLE_COUNT - 1 }; - let proof = epoch.proof(); - // Checked: the counts are prover-supplied and a wrapped sum would let one - // field stay huge and still match `proof.len()`. + // Checked: the counts are prover-supplied and release builds wrap, so a + // plain sum would let one astronomically large field still match + // `num_sub_proofs()` and reach `VmAirs::new`, which sizes a `Vec` from that + // field directly. let Some(expected_proof_count) = table_counts .total() .and_then(|t| t.checked_add(fixed_tables)) .and_then(|t| t.checked_add(1)) else { - return Ok(false); + return Ok(None); }; - if expected_proof_count != proof.len() { - return Ok(false); + if expected_proof_count != epoch.num_sub_proofs() { + return Ok(None); } // Every CPU timestamp of the epoch, and the final PC timestamp, follow from - // the CPU heights; reject an epoch over the per-epoch row bound. - let Some(cpu_heights) = crate::cpu_heights_from_proofs(proof, &table_counts, is_final) else { - return Ok(false); + // the CPU heights; reject an epoch over the per-epoch row bound before they + // size anything. + let Some(cpu_heights) = + crate::cpu_heights_from_proofs(epoch.per_table_proof(), &table_counts, is_final) + else { + return Ok(None); }; if crate::tables::cpu::validate_heights(&cpu_heights).is_err() { - return Ok(false); + return Ok(None); } let reg_fini = epoch.reg_fini()?; let runtime_page_ranges = epoch.runtime_page_ranges()?; - let public_output = epoch.public_output(); let airs = build_epoch_airs( elf, @@ -853,9 +1813,67 @@ fn verify_epoch( is_final, decode_commitment, ); - let l2g_air = l2g_memory_air(opts, label); + // The CPU heights were read at `cpu_proof_start`: the tables must be there. + if airs.check_cpu_position(&table_counts).is_err() { + return Ok(None); + } + let l2g_air = Box::new(l2g_memory_air(opts, label)) + as Box>; + Ok(Some(EpochReconstruction { + airs, + l2g_air, + table_counts, + reg_fini, + runtime_page_ranges, + })) +} + +/// Verify one epoch using ONLY the epoch's public statement fields (via +/// [`EpochProofView`]) plus the verifier-derived `register_init` (epoch 0: +/// from the ELF; epoch i>0: from the previous epoch's `reg_fini`), `is_final`, +/// and `label`. Rebuilds the AIRs and transcript from the bundle's statement +/// values ([`reconstruct_epoch_airs`]) and indexes commits from the carried +/// x254 (`register_init[X254_INDEX]`), never from the prover's memory. PAGE is +/// skipped for continuation epochs, so the AIRs are built with no page configs +/// (the bundle does not get to supply any). Returns `Ok(true)` iff the proof +/// verifies and its committed L2G root matches the claimed one; `Err` iff a +/// small metadata field failed to materialize off an archived bundle. +/// +/// `epoch` is zero-copy either way: owned or archived (see the two callers). +#[allow(clippy::too_many_arguments)] +fn verify_epoch( + elf: &Elf, + elf_bytes: &[u8], + epoch: EpochProofView<'_>, + register_init: &[u32], + is_final: bool, + label: u64, + opts: &ProofOptions, + decode_commitment: Option, +) -> Result { + let Some(recon) = reconstruct_epoch_airs( + elf, + epoch, + register_init, + is_final, + label, + opts, + decode_commitment, + )? + else { + return Ok(false); + }; + let EpochReconstruction { + airs, + l2g_air, + table_counts, + reg_fini: _, + runtime_page_ranges, + } = recon; + + let public_output = epoch.public_output(); let mut refs = airs.air_refs(); - refs.push(&l2g_air); + refs.push(&*l2g_air); let seed = || { epoch_transcript( @@ -876,6 +1894,7 @@ fn verify_epoch( .copied() .unwrap_or(0) as u64; + let proof = epoch.per_table_proof(); let expected = match compute_expected_commit_bus_balance_view( &refs, proof, @@ -890,12 +1909,17 @@ fn verify_epoch( stark::profile_markers::step_marker::<{ stark::profile_markers::STEP_AIRS_AND_BUS_BALANCE_DONE }>( ); - if !Verifier::multi_verify_views(&refs, proof, &mut seed(), &expected) { + if !crate::hash_pin::BlockVerifier::::multi_verify_views( + &refs, + proof, + &mut seed(), + &expected, + ) { return Ok(false); } - // The claimed L2G root must be the one this proof actually committed (it is what - // verify_l2g_commitment_binding_view later ties to the global proof). + // The claimed L2G root must be the one this proof actually committed (it is + // what verify_l2g_commitment_binding_view later ties to the global proof). Ok(proof.last().map(|p| *p.lde_trace_main_merkle_root()) == Some(epoch.l2g_root())) } @@ -914,6 +1938,36 @@ fn prove_global( num_private_input_pages: usize, opts: &ProofOptions, ) -> Result, Error> { + let prep = prep_global( + boundaries, + init_page_data, + page_bases, + num_private_input_pages, + opts, + ); + prove_prepped_global(prep, elf_bytes, page_bases, num_private_input_pages, opts) +} + +/// The global proof's host-side preparation, done: every epoch's L2G table +/// and every touched page's GLOBAL_MEMORY table, with their AIRs. Built by +/// [`prep_global`], proved by [`prove_prepped_global`] — split so the prep can +/// run while the last epochs prove ([`base_prep_ahead`]). +pub(crate) struct GlobalPrep { + num_epochs: usize, + l2g_airs: Vec>, + gm_airs: Vec>, + l2g_traces: Vec>, + gm_traces: Vec>, +} + +/// [`prove_global`]'s host half: everything before the prove. +fn prep_global( + boundaries: &[Arc>], + init_page_data: &HashMap>, + page_bases: &[u64], + num_private_input_pages: usize, + opts: &ProofOptions, +) -> GlobalPrep { // Each cell's final state (boundaries are in epoch order, so the last fini wins). let mut final_state: global_memory::FiniStateMap = HashMap::new(); for epoch in boundaries { @@ -935,11 +1989,11 @@ fn prove_global( true, ); - let mut l2g_traces: Vec> = boundaries + let l2g_traces: Vec> = boundaries .iter() .map(|epoch| local_to_global::generate_local_to_global_trace(epoch.as_slice())) .collect(); - let mut gm_traces: Vec> = gm_configs + let gm_traces: Vec> = gm_configs .iter() .map(|config| global_memory::generate_global_trace(config, &final_state)) .collect(); @@ -953,6 +2007,30 @@ fn prove_global( .map(|config| global_memory_air(opts, config, None)) .collect(); + GlobalPrep { + num_epochs: boundaries.len(), + l2g_airs, + gm_airs, + l2g_traces, + gm_traces, + } +} + +/// [`prove_global`]'s prove half, over a finished [`GlobalPrep`]. +fn prove_prepped_global( + prep: GlobalPrep, + elf_bytes: &[u8], + page_bases: &[u64], + num_private_input_pages: usize, + opts: &ProofOptions, +) -> Result, Error> { + let GlobalPrep { + num_epochs, + l2g_airs, + gm_airs, + mut l2g_traces, + mut gm_traces, + } = prep; let mut pairs: Vec<(AirRef, &mut TraceTable, &())> = l2g_airs .iter() .zip(l2g_traces.iter_mut()) @@ -962,17 +2040,19 @@ fn prove_global( pairs.push((air as AirRef, trace, &())); } - Prover::multi_prove( + #[cfg(feature = "shape-profile")] + crate::shape_profile::capture(pairs.iter().map(|(air, trace, _)| (*air, trace.num_rows()))); + + crate::hash_pin::BlockProver::::multi_prove( pairs, &mut global_transcript( elf_bytes, - boundaries.len(), + num_epochs, num_private_input_pages, opts.fri_final_poly_log_degree, page_bases, ), - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| Error::Prover(format!("{e:?}"))) } @@ -1052,7 +2132,7 @@ fn verify_global( stark::profile_markers::step_marker::<{ stark::profile_markers::STEP_AIRS_AND_BUS_BALANCE_DONE }>( ); - Verifier::multi_verify_views( + crate::hash_pin::BlockVerifier::::multi_verify_views( &refs, proof, &mut global_transcript( @@ -1066,6 +2146,19 @@ fn verify_global( ) } +/// The text of a panic payload, for the pipeline error that reports it in +/// place of the panic (`panic!` with a string literal or a formatted message +/// covers every abort the device layer raises). +fn panic_message(payload: &(dyn std::any::Any + Send)) -> String { + if let Some(s) = payload.downcast_ref::<&str>() { + (*s).to_string() + } else if let Some(s) = payload.downcast_ref::() { + s.clone() + } else { + "non-string panic payload".to_string() + } +} + /// Prove a full continuation and return a self-contained [`ContinuationProof`] /// (prove half only — no verification). Splits the execution into `2^epoch_size_log2` /// cycle epochs, proves each, and proves the one cross-epoch global-memory linkage. @@ -1079,23 +2172,223 @@ fn verify_global( /// final epoch keeps its remainder and its HALT, so its padding chain is anchored as /// usual. A program that fits in one epoch runs as a single final (monolithic-style) /// epoch. +/// `LAMBDA_VM_BASE_SPLIT=1` turns on one `BASE EPOCH` line per pipeline stage. +/// +/// ★ WHY. The base prints ONE number for nineteen epochs — `base: 19 epochs in +/// 67.1s` — and behind it is a three-stage pipeline: a single-threaded producer +/// (execute + collect), a small pool of trace builders, and one prover. Those +/// are three different machines with three different levers, and an aggregate +/// cannot say which of them set the wall. The `recv` stage is the one the +/// aggregate hides hardest: it is the prover thread — the only stage that +/// reaches the card — sitting idle waiting for a builder. +/// +/// Runtime-gated rather than `--features instruments` for the reason +/// [`stark::prove_split`] gives: the record is produced by a binary that does +/// not enable the feature, and a split taken from a different binary describes +/// a different run. +pub(crate) fn base_split_enabled() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| match std::env::var("LAMBDA_VM_BASE_SPLIT") { + Ok(v) => !v.is_empty() && v != "0", + Err(_) => false, + }) +} + +/// Open a timed pipeline stage. `None`, and no clock read, when the knob is off. +fn base_stage() -> Option<(std::time::Instant, f64)> { + base_split_enabled().then(|| (std::time::Instant::now(), stark::prove_split::epoch_secs())) +} + +/// Close a stage opened by [`base_stage`] and print its line. +/// +/// The two wall-clock stamps are what let an external GPU sampler be sliced by +/// stage; the duration alone cannot place the stage on the sampler's timeline. +fn base_stage_done(index: u64, stage: &str, open: Option<(std::time::Instant, f64)>) { + if let Some((start, t0)) = open { + let secs = start.elapsed().as_secs_f64(); + println!( + "BASE EPOCH {index}: {stage} {secs:.2}s t=[{t0:.3},{:.3}]", + stark::prove_split::epoch_secs() + ); + } +} + +/// The switch that keeps each base epoch's host preparation, and the global +/// proof's, on the prover thread just before its prove, which is where they ran +/// before they moved ahead of it. Unset or `0`: ahead (the default). +pub const BASE_PREP_ON_PROVER_ENV: &str = "LAMBDA_VM_BASE_PREP_ON_PROVER"; + +/// Whether the preparation runs ahead of the prover thread, for a raw value of +/// [`BASE_PREP_ON_PROVER_ENV`]: unset, empty or `0` is ahead, `1` is on the +/// prover thread, and anything else stops the run — a typo read as the default +/// would measure one schedule under the other's name. +pub fn base_prep_ahead_setting(raw: Option<&str>) -> bool { + match raw.map(str::trim) { + None | Some("") | Some("0") => true, + Some("1") => false, + Some(other) => panic!("{BASE_PREP_ON_PROVER_ENV} must be 0 or 1, got {other:?}"), + } +} + +/// Whether this process prepares each base epoch, and the global proof, ahead +/// of the prover thread: the epochs' on the STARK trace builders or the WHIR +/// producer, the global proof's on the producer once the last epoch is out. +/// The proofs are the same either way; only which thread does the host work, +/// and when, changes. Read once, and named on stderr in one write either way, +/// so a log states the schedule its base ran. +pub(crate) fn base_prep_ahead() -> bool { + static AHEAD: std::sync::OnceLock = std::sync::OnceLock::new(); + *AHEAD.get_or_init(|| { + let ahead = base_prep_ahead_setting(std::env::var(BASE_PREP_ON_PROVER_ENV).ok().as_deref()); + let line = if ahead { + "BASE PREP: ahead of the prover thread (the default)\n".to_string() + } else { + format!("BASE PREP: on the prover thread ({BASE_PREP_ON_PROVER_ENV}=1)\n") + }; + use std::io::Write; + let _ = std::io::stderr().write_all(line.as_bytes()); + ahead + }) +} + +/// Hooks on a base's epoch pipeline, for a caller that starts host-only work on +/// the epoch proofs before the bundle exists: the recursion tree's level-0 +/// lead-in, which builds the first wrap prologues in the base's tail. +/// +/// Called from the pipeline's own threads, so an implementation must return +/// quickly and must never wait on the pipeline. `P` is the base's epoch-proof +/// type (the STARK and the WHIR bases have their own). +pub(crate) trait EpochObserver

: Sync { + /// The run's last epoch has been executed: it has `num_epochs` epochs. + fn on_epoch_count(&self, num_epochs: usize); + /// Epoch `index` is proved. Once per epoch, in completion order, which can + /// differ from index order. + fn on_epoch_proved(&self, index: usize, proof: &P); + /// What the base derived once and an observer would otherwise derive again, + /// shared: the STARK base's DECODE commitment (a `Commitment`), the WHIR + /// base's DECODE root and prepared commitment (a + /// `multilinear_continuation::SharedDecode` under the base's hash — whose + /// derivation is a device commit, which the lead-in must not make). Called + /// once, before any epoch is proved: before the pipeline starts, or — with + /// the WHIR base's head ahead (the default; `LAMBDA_VM_WHIR_HEAD_AHEAD=0` is + /// the serial head), whose DECODE root arrives from a helper — from the + /// consumer, before the first epoch it proves. + fn on_base_shared(&self, _shared: Arc) {} +} + +/// Test-only: an [`EpochObserver`] that records what it is told, for a test +/// asserting what a base reports. +#[cfg(test)] +pub(crate) struct RecordingObserver

{ + pub(crate) counts: std::sync::Mutex>, + pub(crate) epochs: std::sync::Mutex>, + pub(crate) shared: std::sync::Mutex>>, +} + +#[cfg(test)] +impl

Default for RecordingObserver

{ + fn default() -> Self { + Self { + counts: std::sync::Mutex::new(Vec::new()), + epochs: std::sync::Mutex::new(Vec::new()), + shared: std::sync::Mutex::new(Vec::new()), + } + } +} + +#[cfg(test)] +impl

RecordingObserver

{ + /// The epochs it saw, in index order, asserting each of `0..n` exactly once. + pub(crate) fn epochs_in_order(&self, n: usize) -> Vec

{ + let mut seen = std::mem::take(&mut *self.epochs.lock().unwrap()); + seen.sort_by_key(|(index, _)| *index); + assert_eq!( + seen.iter().map(|(index, _)| *index).collect::>(), + (0..n).collect::>(), + "the observer must see every epoch exactly once" + ); + seen.into_iter().map(|(_, proof)| proof).collect() + } +} + +#[cfg(test)] +impl EpochObserver

for RecordingObserver

{ + fn on_epoch_count(&self, num_epochs: usize) { + self.counts.lock().unwrap().push(num_epochs); + } + + fn on_epoch_proved(&self, index: usize, proof: &P) { + self.epochs.lock().unwrap().push((index, proof.clone())); + } + + fn on_base_shared(&self, shared: Arc) { + self.shared.lock().unwrap().push(shared); + } +} + +/// An [`EpochObserver`] for the STARK base, as the thread-local holds it. +pub(crate) type SharedEpochObserver = Arc + Send + Sync>; + +thread_local! { + /// The observer the base proved on THIS thread reports to: installed by + /// [`with_epoch_observer`] for the duration of a call, `None` otherwise — + /// and with `None` the pipeline runs as it does with no observer at all. + /// + /// ⓘ A thread-local rather than a parameter: the base's entry points keep + /// their signatures, and only the tree drivers, which install one, see it. + static BASE_OBSERVER: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +/// Runs `f` with `observer` installed for every base proved on this thread +/// inside it, restoring what was there before on return or unwind. +#[cfg(test)] +pub(crate) fn with_epoch_observer(observer: SharedEpochObserver, f: impl FnOnce() -> R) -> R { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + let previous = self.0.take(); + BASE_OBSERVER.with(|o| *o.borrow_mut() = previous); + } + } + let _restore = Restore(BASE_OBSERVER.with(|o| o.borrow_mut().replace(observer))); + f() +} + pub fn prove_continuation( elf_bytes: &[u8], private_inputs: &[u8], epoch_size_log2: u32, opts: &ProofOptions, ) -> Result { - prove_continuation_with_max_rows( + prove_continuation_keeping_decode(elf_bytes, private_inputs, epoch_size_log2, opts) + .map(|(bundle, _)| bundle) +} + +/// [`prove_continuation`], also handing back the DECODE commitment it derived +/// from the ELF, so a caller that goes on to reconstruct every epoch (the +/// recursion tree's level 0) takes it instead of deriving it a second time. +/// The commitment is a pure function of `(ELF, opts)`: the value is the one +/// `commitment_from_elf(&Elf::load(elf_bytes), opts)` returns. +pub fn prove_continuation_keeping_decode( + elf_bytes: &[u8], + private_inputs: &[u8], + epoch_size_log2: u32, + opts: &ProofOptions, +) -> Result<(ContinuationProof, Commitment), Error> { + prove_continuation_scheduled( elf_bytes, private_inputs, epoch_size_log2, opts, + base_prep_ahead(), &MaxRowsConfig::default(), ) } /// [`prove_continuation`] with explicit per-table row limits, so an epoch can be /// split over several CPU tables (tests use [`MaxRowsConfig::small`]). +#[cfg(test)] pub(crate) fn prove_continuation_with_max_rows( elf_bytes: &[u8], private_inputs: &[u8], @@ -1103,6 +2396,37 @@ pub(crate) fn prove_continuation_with_max_rows( opts: &ProofOptions, max_rows: &MaxRowsConfig, ) -> Result { + prove_continuation_scheduled( + elf_bytes, + private_inputs, + epoch_size_log2, + opts, + base_prep_ahead(), + max_rows, + ) + .map(|(bundle, _)| bundle) +} + +/// [`prove_continuation_keeping_decode`] with the preparation's schedule named +/// by the caller rather than read from [`BASE_PREP_ON_PROVER_ENV`], so one +/// process can run both: `prep_ahead` moves each epoch's host preparation onto +/// the trace builders and the global proof's onto the producer, ahead of the +/// prover thread. The proofs are the same either way; only who does the host +/// work, and when, changes. +pub(crate) fn prove_continuation_scheduled( + elf_bytes: &[u8], + private_inputs: &[u8], + epoch_size_log2: u32, + opts: &ProofOptions, + prep_ahead: bool, + max_rows: &MaxRowsConfig, +) -> Result<(ContinuationProof, Commitment), Error> { + // The observer a caller installed for this thread, if any. It sees the + // pipeline and changes nothing in it. + let installed = BASE_OBSERVER.with(|o| o.borrow().clone()); + let observer: Option<&dyn EpochObserver> = installed + .as_deref() + .map(|o| o as &dyn EpochObserver); if epoch_size_log2 < 2 { return Err(Error::InvalidContinuationEpochSize( "epoch_size_log2 must be at least 2 (4 cycles)".to_string(), @@ -1137,6 +2461,9 @@ pub(crate) fn prove_continuation_with_max_rows( // it once here instead of once per epoch inside `build_epoch_airs`. let decode_commitment = crate::tables::decode::commitment_from_elf(&elf, opts) .map_err(|e| Error::Recursion(format!("DECODE commitment from ELF: {e}")))?; + if let Some(observer) = observer { + observer.on_base_shared(Arc::new(decode_commitment)); + } // Same for the DECODE trace artifacts (instruction map + pristine trace): // a pure function of the ELF, built once and shared by every epoch's trace // build instead of re-parsed/regenerated inside the serial producer chain. @@ -1156,11 +2483,13 @@ pub(crate) fn prove_continuation_with_max_rows( // page-base set is shipped (see `touched_page_bases`). // // The producer publishes each epoch's boundary (an `Arc` share of the one it - // sends to the epoch prover) on this dedicated channel, in epoch order. The - // global-prove thread drains it until the producer hangs up (last epoch - // prepared) — the global proof depends only on these execution artifacts, - // never on an epoch *proof*, so it overlaps the epoch proves' tail instead - // of serializing after them. Proof bytes are unchanged — only the schedule. + // sends to the epoch prover) on this dedicated channel, in epoch order. It + // is unbounded, so the producer never blocks on it and the epoch pipeline's + // own bounded channels stay the only backpressure; it is drained once, after + // the epoch scope joins, and the global proof is proven from it THERE — see + // the ★ note at the drain site for why that is deliberately not overlapped. + // (With `prep_ahead` the producer prepares the global proof from its own + // shares instead, and only the PROVE waits for the drain site.) let (boundary_tx, boundary_rx) = std::sync::mpsc::channel::>>(); // Three-stage epoch pipeline: a producer thread runs the @@ -1176,8 +2505,12 @@ pub(crate) fn prove_continuation_with_max_rows( // // The bounded channels cap peak memory: at most one collected epoch // queued, `builders` building, one built epoch queued, one proving. - let (tx, rx) = std::sync::mpsc::sync_channel::>(1); + let (tx, rx) = std::sync::mpsc::sync_channel::>(1); let (build_tx, build_rx) = std::sync::mpsc::sync_channel::>(1); + // `prep_ahead` ([`base_prep_ahead`], the default): the builders also run + // each epoch's host preparation, and the producer prepares the global proof + // once it has handed over the last epoch, both ahead of the prover thread + // instead of on it. // Trace builders: each turns one collected epoch into full trace tables // (the bulk of the old per-epoch producer latency). 2 is enough to keep // the prove pipeline fed on the measured workloads; builds compete with @@ -1196,10 +2529,11 @@ pub(crate) fn prove_continuation_with_max_rows( // returning: the senders are bounded and can only unblock via a recv, so // an early return would leave a builder parked in `send` forever and the // scope would never join. Draining ends when every sender is dropped. - let prove_worker = |rx: std::sync::mpsc::Receiver>| { + let prove_worker = |rx: std::sync::mpsc::Receiver>| { let mut proved: Vec = Vec::new(); loop { - let prepared = match rx.recv() { + let __bs_recv = base_stage(); + let ready = match rx.recv() { Ok(Ok(p)) => p, Ok(Err(e)) => { first_err.lock().unwrap().get_or_insert(e); @@ -1207,6 +2541,8 @@ pub(crate) fn prove_continuation_with_max_rows( } Err(_) => return proved, // channel closed: no more epochs }; + let index = ready.index(); + base_stage_done(index, "recv", __bs_recv); if first_err.lock().unwrap().is_some() { continue; // an earlier failure is propagating; drain and discard } @@ -1214,29 +2550,65 @@ pub(crate) fn prove_continuation_with_max_rows( // instruments span carries a static label and instances are told // apart by order (phase_table.py reports them per instance). #[cfg(feature = "nvtx")] - let __nvtx = - stark::instruments::nvtx_range_fmt(|| format!("epoch_prove[i={}]", prepared.index)); + let __nvtx = stark::instruments::nvtx_range_fmt(|| format!("epoch_prove[i={index}]")); #[cfg(feature = "instruments")] let __sp = stark::instruments::span("epoch_prove"); - let start = EpochStart { - register_init: &prepared.register_init, - label: prepared.label, - }; - match prove_epoch( - &elf, - elf_bytes, - &start, - prepared.traces, - prepared.is_final, - &prepared.boundary, - opts, - decode_commitment, - ) { - Ok(epoch) => proved.push((prepared.index, epoch)), - Err(e) => { + // A PANIC in the prove — a loud device abort, or any bug — must take + // the same drain path as an `Err`. If this thread simply died, `rx` + // would close, every builder would stop on its dead sender, and the + // producer (which watches `first_err`, never written by a panic) + // would block forever in `build_tx.send` on the capacity-1 build + // channel, whose receiver lives outside the scope: the prove would + // hang instead of failing. Seen once: an aux-build abort slept for + // 21 minutes under the CLI. + let __bs_prove = base_stage(); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + // Inside the guard on purpose: a panic raised on this thread + // OUTSIDE it reproduces the original wedge (which is what the + // test's first placement did, and the test caught it). + #[cfg(test)] + if index == test_fault::FAIL_INDEX && private_inputs == test_fault::PANIC_MAGIC { + panic!("injected prover panic (test)"); + } + match ready { + ReadyEpoch::Built(prepared) => prove_epoch( + &elf, + elf_bytes, + &EpochStart { + register_init: &prepared.register_init, + label: prepared.label, + }, + prepared.traces, + prepared.is_final, + &prepared.boundary, + opts, + decode_commitment, + ), + ReadyEpoch::Prepped { prep, .. } => prove_prepped_epoch(*prep, elf_bytes, opts), + } + })); + base_stage_done(index, "prove", __bs_prove); + match outcome { + Ok(Ok(epoch)) => { + if let Some(observer) = observer { + observer.on_epoch_proved(index as usize, &epoch); + } + proved.push((index, epoch)) + } + Ok(Err(e)) => { first_err.lock().unwrap().get_or_insert(e); continue; // drain mode (see loop comment) } + Err(payload) => { + first_err + .lock() + .unwrap() + .get_or_insert(Error::ContinuationInvariant(format!( + "epoch {index} prover panicked: {}", + panic_message(&*payload) + ))); + continue; // drain mode + } } } }; @@ -1248,7 +2620,7 @@ pub(crate) fn prove_continuation_with_max_rows( // passes (stateless, so concurrent tests can never trip it): exercises the // mid-pipeline error path, which must return `Err` instead of wedging the // bounded channels (see `test_fault`). - let build_worker = |tx: std::sync::mpsc::SyncSender>| { + let build_worker = |tx: std::sync::mpsc::SyncSender>| { loop { let msg = { build_rx.lock().unwrap().recv() }; let job = match msg { @@ -1277,20 +2649,31 @@ pub(crate) fn prove_continuation_with_max_rows( }); #[cfg(feature = "instruments")] let __sp = stark::instruments::span("epoch_trace_build"); - let traces = Traces::build_from_collected( - decode_artifacts_ref, - job.collected, - // Continuation epochs use the L2G bookend: PAGE tables (the - // only image consumers in the build) are skipped. - None::<&std::collections::HashMap>, - &job.register_init, - max_rows, - private_inputs, - job.is_final, - true, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ); + // Same reason as the prover's guard: a builder that dies leaves the + // producer blocked on the build channel once every builder is gone. + let build_index = job.index; + let __bs_build = base_stage(); + let traces = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Traces::build_from_collected( + decode_artifacts_ref, + job.collected, + // Continuation epochs use the L2G bookend: PAGE tables (the + // only image consumers in the build) are skipped. + None::<&std::collections::HashMap>, + &job.register_init, + max_rows, + private_inputs, + job.is_final, + true, + ) + })) { + Ok(traces) => traces, + Err(payload) => Err(Error::ContinuationInvariant(format!( + "epoch {build_index} trace build panicked: {}", + panic_message(&*payload) + ))), + }; + base_stage_done(build_index, "build", __bs_build); // Close the build span BEFORE forwarding: the send below blocks // on prove-channel backpressure, which is waiting, not building. #[cfg(feature = "instruments")] @@ -1307,22 +2690,59 @@ pub(crate) fn prove_continuation_with_max_rows( let mut traces = traces; #[cfg(feature = "instruments")] let __sp = stark::instruments::span("p6_trace_preupload"); + let __bs_pre = base_stage(); traces.preupload_main_traces(); + base_stage_done(job.index, "preupload", __bs_pre); traces }; - let prepared = PreparedEpoch { - index: job.index, - register_init: job.register_init, - label: job.label, - traces, - boundary: job.boundary, - is_final: job.is_final, + let ready = if prep_ahead { + // This epoch's host preparation, here on the builder + // ahead of the prover, under the same panic guard as + // the build. + let __bs_prep = base_stage(); + let prep = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + prep_epoch( + &elf, + &job.register_init, + job.label, + traces, + job.is_final, + &job.boundary, + opts, + decode_commitment, + ) + })); + base_stage_done(job.index, "prep", __bs_prep); + match prep { + Ok(Ok(prep)) => Ok(ReadyEpoch::Prepped { + index: job.index, + prep: Box::new(prep), + }), + Ok(Err(e)) => Err(e), + Err(payload) => Err(Error::ContinuationInvariant(format!( + "epoch {build_index} preparation panicked: {}", + panic_message(&*payload) + ))), + } + } else { + Ok(ReadyEpoch::Built(PreparedEpoch { + index: job.index, + register_init: job.register_init, + label: job.label, + traces, + boundary: job.boundary, + is_final: job.is_final, + })) }; + let failed = ready.is_err(); // A send error means the prover side hung up (its error is // already propagating) — stop quietly. - if tx.send(Ok(prepared)).is_err() { + if tx.send(ready).is_err() { return; } + if failed { + continue; // drain mode + } } Err(e) => { let _ = tx.send(Err(e)); @@ -1332,14 +2752,18 @@ pub(crate) fn prove_continuation_with_max_rows( } }; - // The global prove's result, produced by its own scoped thread. `None` only - // if that thread never ran to completion (a panic — surfaced by the scope). - type GlobalResult = (MultiProof, Vec, usize); - let global_result: std::sync::Mutex>> = - std::sync::Mutex::new(None); - let mut results = std::thread::scope(|scope| -> Result, Error> { + // `prep_ahead`: once it has handed over the last epoch, the PRODUCER + // prepares the global proof, from its own shares of the boundaries, while + // the last epochs still build and prove. The preparation is host work only; + // the global PROVE stays after this scope (the ★ note below), so the scope + // still spawns exactly the producer, the builders and the prover. + type GlobalAhead = (EpochBoundaries, GlobalPrep); + type ScopeOut = Result<(Vec, Option), Error>; + let init_page_data_ref = &init_page_data; + let (mut results, global_ahead) = std::thread::scope(|scope| -> ScopeOut { let elf_ref = &elf; - let producer = scope.spawn(move || { + let producer = scope.spawn(move || -> Option { + let mut kept: EpochBoundaries = Vec::new(); let mut prepare_all = || -> Result<(), Error> { let mut prev_fini: Option> = None; let mut index: u64 = 0; @@ -1382,6 +2806,7 @@ pub(crate) fn prove_continuation_with_max_rows( // clears it). #[cfg(feature = "instruments")] let __sp = stark::instruments::span("epoch_execute"); + let __bs_exec = base_stage(); let logs = match executor .resume_with_limit(epoch_size) .map_err(|e| Error::Execution(format!("{e}")))? @@ -1389,9 +2814,13 @@ pub(crate) fn prove_continuation_with_max_rows( Some(logs) => logs.to_vec(), None => return Ok(()), }; + base_stage_done(index, "execute", __bs_exec); #[cfg(feature = "instruments")] drop(__sp); let is_final = executor.pc() == 0; + if is_final && let Some(observer) = observer { + observer.on_epoch_count(index as usize + 1); + } // Invariant: a non-final epoch ran the full `epoch_size` (a power // of two), so its CPU table has no padding rows. @@ -1411,6 +2840,7 @@ pub(crate) fn prove_continuation_with_max_rows( stark::instruments::nvtx_range_fmt(|| format!("epoch_collect[i={index}]")); #[cfg(feature = "instruments")] let __sp = stark::instruments::span("epoch_collect"); + let __bs_collect = base_stage(); let collected = Traces::collect_epoch( decode_artifacts_ref, &image, @@ -1426,10 +2856,14 @@ pub(crate) fn prove_continuation_with_max_rows( // Publish this epoch's boundary for the global prove (in // epoch order; the channel closes when the producer ends). let _ = boundary_tx.send(Arc::clone(&boundary)); + if prep_ahead { + kept.push(Arc::clone(&boundary)); + } // R_{i+1} from the collected register end state — the exact // value the generated REGISTER trace binds (`fini_from_trace` // equivalence pinned by `fini_from_final_state_matches_trace`). + base_stage_done(index, "collect", __bs_collect); prev_fini = Some(collected.register_fini(®ister_init)); // Carry the image forward: this epoch's fini is the next @@ -1454,7 +2888,10 @@ pub(crate) fn prove_continuation_with_max_rows( }; // A send error means the builder side hung up (its error is // already propagating) — stop preparing quietly. - if build_tx.send(Ok(job)).is_err() || is_final { + let __bs_send = base_stage(); + let send_failed = build_tx.send(Ok(job)).is_err(); + base_stage_done(index, "handoff", __bs_send); + if send_failed || is_final { return Ok(()); } index += 1; @@ -1465,7 +2902,30 @@ pub(crate) fn prove_continuation_with_max_rows( // builder forwards them to the prover); if the downstream side // is already gone the error there wins. let _ = build_tx.send(Err(e)); + return None; } + // `prep_ahead` only, and only once every epoch was produced: a + // downstream failure stops the producer early, and its error is the + // answer. + if !prep_ahead || first_err_ref.lock().unwrap().is_some() { + return None; + } + let open = base_stage(); + let prep = prep_global( + &kept, + init_page_data_ref, + &touched_page_bases(&kept), + page::private_input_page_count(private_inputs), + opts, + ); + if let Some((start, t0)) = open { + println!( + "BASE EPOCH global: prep {:.2}s t=[{t0:.3},{:.3}]", + start.elapsed().as_secs_f64(), + stark::prove_split::epoch_secs() + ); + } + Some((kept, prep)) }); // Trace-builder pool: collected epochs → trace tables → prove channel. @@ -1478,44 +2938,6 @@ pub(crate) fn prove_continuation_with_max_rows( } drop(tx); - // Global prove, overlapped: drain the boundary channel until the - // producer hangs up (last epoch prepared), then prove the cross-epoch - // global memory argument WHILE the tail epochs are still proving. The - // global proof consumes only execution artifacts (boundaries, ELF, - // genesis pages) — never an epoch proof — so this is pure schedule. - let global_result_ref = &global_result; - let init_page_data_ref = &init_page_data; - scope.spawn(move || { - let mut all: Vec>> = Vec::new(); - while let Ok(b) = boundary_rx.recv() { - all.push(b); - } - // An epoch already failed: its error wins and the bundle is never - // assembled — skip the (whole-prove-sized) global prove. - if first_err_ref.lock().unwrap().is_some() { - return; - } - let run = || -> Result { - #[cfg(feature = "instruments")] - let __sp = stark::instruments::span("prove_global"); - let num_private_input_pages = page::private_input_page_count(private_inputs); - // SINGLE source of truth: the same page-base list drives the - // committed GLOBAL_MEMORY tables and is shipped in the bundle, - // so the two can never diverge in set or order. - let touched = touched_page_bases(&all); - let global = prove_global( - &all, - elf_bytes, - init_page_data_ref, - &touched, - num_private_input_pages, - opts, - )?; - Ok((global, touched, num_private_input_pages)) - }; - *global_result_ref.lock().unwrap() = Some(run()); - }); - // Prove epochs as the builders hand them over. Builders can finish // out of index order, so results are re-ordered by epoch index before // the bundle is assembled — proof bytes are identical to the @@ -1526,10 +2948,10 @@ pub(crate) fn prove_continuation_with_max_rows( let proved = prover.join().map_err(|_| { Error::ContinuationInvariant("epoch prover thread panicked".to_string()) })?; - producer.join().map_err(|_| { + let global_ahead = producer.join().map_err(|_| { Error::ContinuationInvariant("epoch preparation thread panicked".to_string()) })?; - Ok(proved) + Ok((proved, global_ahead)) })?; if let Some(e) = first_err.into_inner().unwrap() { return Err(e); @@ -1539,13 +2961,56 @@ pub(crate) fn prove_continuation_with_max_rows( epochs.push(epoch); } - // One global LogUp over all the (kept) local-to-global tables — proven - // concurrently by the scoped thread above; collect its result here. The - // scope guarantees the thread finished, so `None` is unreachable. - let (global, touched_page_bases, num_private_input_pages) = - global_result.into_inner().unwrap().ok_or_else(|| { - Error::ContinuationInvariant("global prove thread produced no result".to_string()) - })??; + // One global LogUp over all the (kept) local-to-global tables. The scope + // above has joined, so every epoch prove has finished and released its + // device memory before this one starts. + // + // ★ Deliberately NOT overlapped with the epoch proves, though nothing in the + // ARGUMENT forbids it: this is a second `stark::prover::multi_prove`, and + // each `multi_prove` builds its OWN `VramGate` from the whole device budget + // (80% of total). Two running at once therefore put 2x the card's admission + // budget on one GPU with nothing summing them, and each additionally holds + // its round-1 working set — every table's LDE, trace snapshot and Merkle + // tree stay device-resident from its commit until its own rounds task ends — + // which no gate counts at all. Measured on an RTX 5090 (31.40 GiB): the + // overlap window opened about `builders + 2` epochs before the end, and + // inside it the card ran out with tables from BOTH proves aborting in the + // same instant (`gpu_lde::refuse_host_recovery`). + // + // The cost of serialising is this proof's own wall time, which the overlap + // used to hide behind the tail epochs. It changes no proof bytes: the global + // proof consumes only execution artifacts (boundaries, ELF, genesis pages), + // never an epoch proof, so the schedule was always free to choose. + let (all, global_prep) = match global_ahead { + Some((all, prep)) => (all, Some(prep)), + None => (boundary_rx.try_iter().collect::(), None), + }; + let num_private_input_pages = page::private_input_page_count(private_inputs); + // SINGLE source of truth: the same page-base list drives the committed + // GLOBAL_MEMORY tables and is shipped in the bundle, so the two can never + // diverge in set or order. + let touched_page_bases = touched_page_bases(&all); + let global = { + #[cfg(feature = "instruments")] + let __sp = stark::instruments::span("prove_global"); + match global_prep { + Some(prep) => prove_prepped_global( + prep, + elf_bytes, + &touched_page_bases, + num_private_input_pages, + opts, + )?, + None => prove_global( + &all, + elf_bytes, + &init_page_data, + &touched_page_bases, + num_private_input_pages, + opts, + )?, + } + }; // Same timeline output as the monolithic path (prover/src/lib.rs): print // the wall-clock span tree and honor LAMBDA_VM_TIMELINE_JSON. Without this, @@ -1562,12 +3027,15 @@ pub(crate) fn prove_continuation_with_max_rows( } } - Ok(ContinuationProof { - epochs, - global, - num_private_input_pages, - touched_page_bases, - }) + Ok(( + ContinuationProof { + epochs, + global, + num_private_input_pages, + touched_page_bases, + }, + decode_commitment, + )) } /// Verify a [`ContinuationProof`] using ONLY the bundle and the ELF — nothing from @@ -1849,6 +3317,9 @@ pub fn prove_and_verify_continuation( #[cfg(test)] pub(crate) mod test_fault { pub(crate) const MAGIC: &[u8] = b"__inject_pipeline_fault__"; + /// Same index, but the PROVER thread panics instead of returning `Err`: + /// the shutdown path a loud device abort takes. + pub(crate) const PANIC_MAGIC: &[u8] = b"__inject_pipeline_panic__"; pub(crate) const FAIL_INDEX: u64 = 3; } @@ -2004,6 +3475,35 @@ mod tests { ); } + // The pipeline's PANIC path: a loud device abort is a panic on the prover + // thread, not an `Err`. Before the guard it wedged the pipeline — the + // builders stopped on their dead sender and the producer blocked forever + // on the build channel — and a real prove slept for 21 minutes. It must + // surface as `Err` carrying the panic's own message. + #[test] + fn test_prover_panic_mid_pipeline_returns_err() { + let _ = env_logger::builder().is_test(true).try_init(); + let elf_bytes = asm_elf_bytes("all_loadstore_32"); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let r = prove_continuation( + &elf_bytes, + test_fault::PANIC_MAGIC, + 2, + &ProofOptions::default_test_options(), + ); + let _ = done_tx.send(r.map(|_| ())); + }); + let result = done_rx + .recv_timeout(std::time::Duration::from_secs(300)) + .expect("prove_continuation wedged: a prover panic did not shut the pipeline down"); + let err = result.expect_err("the injected panic must surface as Err"); + assert!( + format!("{err:?}").contains("injected prover panic"), + "unexpected error: {err:?}" + ); + } + // The pipeline's error path: a mid-run failure with several epochs still // pending (past the bounded channels' slack) must surface as `Err` — the // regression this guards wedged every channel and hung `prove_continuation` @@ -2025,7 +3525,15 @@ mod tests { let _ = done_tx.send(r.map(|_| ())); }); let result = done_rx - .recv_timeout(std::time::Duration::from_secs(300)) + // 1800 s of headroom rather than 300. This is a liveness guard — the + // regression it catches wedges the pipeline FOREVER, so any finite + // bound still catches it — and the bound has to clear an honest run + // under load: an algebraic hash pin doubles this test's own proving + // work (alone, three runs each: 7.3-7.6 s at the BLAKE3 default, + // 15.7-15.9 s under RPX), and inside the full `--lib` suite's + // parallel load the old 300 s fired while the test was still making + // progress. + .recv_timeout(std::time::Duration::from_secs(1800)) .expect("prove_continuation wedged: the pipeline did not shut down on error"); let err = result.expect_err("the injected fault must surface as Err"); assert!( @@ -2141,6 +3649,7 @@ mod tests { ("ecdas", c.ecdas), ("hint", c.hint), ("is_wh", c.is_wh), + ("ecrec", c.ecrec), ] }) .collect(); @@ -2422,7 +3931,7 @@ mod tests { /// A CPU chunk size the verifier would refuse fails before execution. #[test] fn test_continuation_rejects_invalid_cpu_max_rows() { - for cpu in [0, 6, crate::tables::max_rows::CPU * 2] { + for cpu in [0, 6, crate::tables::cpu::MAX_TABLE_ROWS * 2] { let max_rows = MaxRowsConfig { cpu, ..MaxRowsConfig::small() @@ -2491,7 +4000,7 @@ mod tests { bundle.epochs[0].proof.proofs[cpu].trace_length, max_rows.cpu ); - for height in [6, 2 * crate::tables::max_rows::CPU, 2 * max_rows.cpu] { + for height in [6, 2 * crate::tables::cpu::MAX_TABLE_ROWS, 2 * max_rows.cpu] { bundle.epochs[0].proof.proofs[cpu].trace_length = height; assert!( !matches!(verify_continuation(&elf_bytes, &bundle, &opts), Ok(Some(_))), @@ -2508,6 +4017,118 @@ mod tests { // ---- Standalone (split) prover/verifier ---- + /// Preparing ahead of the prover moves each epoch's host preparation onto + /// the trace builders and the global proof's onto the producer. That changes + /// who does the work and when, never what is proved: the committed roots, the + /// L2G roots and the statement values equal those of the schedule that + /// prepares on the prover thread, the DECODE commitment is the same, and the + /// bundle verifies to the same output. + /// + /// ⚠ Six tables are compared by everything BUT their root: BRANCH, DVRM, + /// BYTEWISE, EQ, LT and MUL lay their rows out in `HashMap` iteration + /// order (`op_map.into_iter()`), so their roots differ between two proves + /// of the same run under the SAME schedule (measured: LT[0] in two of three + /// epochs of this program, one schedule twice in one process). + #[test] + fn prep_ahead_proves_what_the_prover_thread_schedule_proves() { + let _ = env_logger::builder().is_test(true).try_init(); + let elf_bytes = asm_elf_bytes("test_commit_split"); + let opts = ProofOptions::default_test_options(); + let (on_prover, decode_on_prover) = prove_continuation_scheduled( + &elf_bytes, + &[], + 4, + &opts, + false, + &MaxRowsConfig::default(), + ) + .unwrap(); + let (ahead, decode_ahead) = prove_continuation_scheduled( + &elf_bytes, + &[], + 4, + &opts, + true, + &MaxRowsConfig::default(), + ) + .unwrap(); + assert!( + ahead.num_epochs() > 1, + "16-cycle epochs must split the run, or the pipeline is not exercised" + ); + assert_eq!(decode_ahead, decode_on_prover, "the DECODE commitment"); + assert_eq!(ahead.num_epochs(), on_prover.num_epochs()); + let roots = |p: &MultiProof| -> Vec { + p.proofs + .iter() + .map(|t| t.lde_trace_main_merkle_root) + .collect() + }; + let hash_ordered = |name: &str| { + ["BRANCH", "DVRM", "BYTEWISE", "EQ", "LT", "MUL"] + .iter() + .any(|t| name.starts_with(&format!("{t}[")) || name == *t) + }; + let elf = Elf::load(&elf_bytes).unwrap(); + for (k, (a, d)) in ahead.epochs.iter().zip(&on_prover.epochs).enumerate() { + let position = epoch_chain_position(&on_prover.epochs, on_prover.num_epochs(), &elf, k) + .unwrap() + .unwrap(); + let recon = reconstruct_epoch_airs( + &elf, + EpochProofView::Owned(d), + &position.register_init, + position.is_final, + position.label, + &opts, + None, + ) + .unwrap() + .unwrap(); + let mut names: Vec<&str> = recon.airs.air_refs().iter().map(|a| a.name()).collect(); + names.push(recon.l2g_air.name()); + let (ra, rd) = (roots(&a.proof), roots(&d.proof)); + assert_eq!(ra.len(), names.len(), "epoch {k}: one root per table"); + let mut compared = 0; + for ((name, x), y) in names.iter().zip(&ra).zip(&rd) { + if !hash_ordered(name) { + assert_eq!(x, y, "epoch {k}: {name}'s trace root"); + compared += 1; + } + } + assert!(compared >= 5, "epoch {k}: only {compared} roots compared"); + assert_eq!(a.l2g_root, d.l2g_root, "epoch {k}: L2G root"); + assert_eq!(a.public_output, d.public_output, "epoch {k}: output"); + assert_eq!(a.reg_fini, d.reg_fini, "epoch {k}: register fini"); + } + assert_eq!( + roots(&ahead.global), + roots(&on_prover.global), + "global trace roots" + ); + assert_eq!(ahead.touched_page_bases, on_prover.touched_page_bases); + let out = verify_continuation(&elf_bytes, &ahead, &opts).unwrap(); + assert_eq!(out.as_deref(), Some(&[0xAA, 0xBB, 0xCC, 0xDD][..])); + } + + /// The schedule switch: unset, empty or `0` prepares ahead of the prover + /// thread, `1` keeps the preparation on it. + #[test] + fn the_base_prep_switch_is_ahead_unless_exactly_one() { + assert!(base_prep_ahead_setting(None)); + assert!(base_prep_ahead_setting(Some(""))); + assert!(base_prep_ahead_setting(Some("0"))); + assert!(!base_prep_ahead_setting(Some("1"))); + assert!(!base_prep_ahead_setting(Some(" 1 "))); + } + + /// Anything else stops the run instead of reading as the default. + #[test] + #[should_panic(expected = "LAMBDA_VM_BASE_PREP_ON_PROVER must be 0 or 1")] + fn the_base_prep_switch_refuses_anything_else() { + base_prep_ahead_setting(Some("yes")); + } + // Round-trip: a bundle from prove_continuation verifies on its own (only the // bundle + ELF) and reconstructs the exact run-wide output. #[test] @@ -3236,4 +4857,927 @@ mod tests { "a global proof spliced in from a different run must be rejected over the archived blob path too" ); } + + // ======================================================================= + // THE GENESIS-STACK SPLIT + // ======================================================================= + + /// The block's measured census, reproduced by the forms — an F1 on the + /// census rather than a restatement of it. + /// + /// Read on a box with the guest of 90e9dde5b (ELF sha256 `3e8f1ce1…`, + /// 2,711,800 B; one execution of the guest, no proving, no card), after a + /// control run on the previous guest (`8f826601…`) reproduced the census + /// these constants held before: `GENESIS CENSUS: 33 pages = 28 genesis + 5 + /// private; 189088 nonzero entries, worst page 127574; 3404088 leg rows`. + const BLOCK_DENSE: [(u64, usize, usize); 2] = + [(0x0, 127_574, 2_296_350), (0x200000, 61_514, 1_107_270)]; + const BLOCK_ZERO_PAGES: usize = 26; + const BLOCK_PAGE_VARS: usize = 18; + const BLOCK_CENSUS_LEG_ROWS: usize = 3_404_088; + /// The census's genesis pages: 33 touched less the 5 private. + const BLOCK_GENESIS_PAGES: usize = 28; + /// The census's epochs at 2^21. + const BLOCK_EPOCHS: usize = 12; + /// The bracket the block stands at and what the form charges there — both + /// named here so a reader of these numbers sees what they rest on. + const BLOCK_N_FIXED: usize = BLOCK_STACK_VARS; + const BLOCK_MARGINAL: usize = marginal_stacked_rows(BLOCK_PAGE_VARS, BLOCK_STACK_VARS); + /// The bracket a LONE genesis page stands at — `18 + ceil(log2(2))` — and + /// what the rule charges there. ⚠ IT IS NOT THE BLOCK'S, and the ten rows + /// between them are the whole reason the marginal stopped being one number. + const LONE_N_FIXED: usize = fixed_stack_vars(BLOCK_PAGE_VARS, 1); + const LONE_MARGINAL: usize = marginal_stacked_rows(BLOCK_PAGE_VARS, LONE_N_FIXED); + + /// The census was read at one page size; every number below is evaluated at + /// that height only while this holds. + #[test] + fn the_blocks_page_height_is_the_page_size_the_code_uses() { + assert_eq!(PAGE_NUM_VARS, BLOCK_PAGE_VARS); + } + + /// A genesis page presents exactly the columns the stack takes. + #[test] + fn a_genesis_pages_preprocessed_columns_are_the_pair_the_stack_carries() { + let config = PageConfig::with_data(0x40000, vec![1u8; 8]); + assert_eq!( + page::preprocessed_columns(&config).len(), + PAGE_PREPROCESSED_COLUMNS, + "the stack takes a prefix of this list, so its length is the contract" + ); + } + + #[test] + fn the_sparse_form_reproduces_the_blocks_census_to_the_row() { + let mut total = 0usize; + for (base, nonzero, rows) in BLOCK_DENSE { + let got = sparse_leg_rows(BLOCK_PAGE_VARS, nonzero); + assert_eq!(got, rows, "page {base:#x}: {nonzero} nonzero entries"); + total += got; + } + // The 26 all-zero pages are not free: each costs the interned zero. + total += BLOCK_ZERO_PAGES * sparse_leg_rows(BLOCK_PAGE_VARS, 0); + assert_eq!( + total, BLOCK_CENSUS_LEG_ROWS, + "the closed form and the box census disagree about what INIT costs" + ); + } + + /// A page set of [`BLOCK_GENESIS_PAGES`] genesis pages whose first ones + /// carry the given nonzero counts and whose rest are all-zero. + /// + /// ⚠ THESE TESTS BUILD A SET AND RUN THE PLANNER, rather than calling the + /// forms. Part 2 is a property of the whole set, so a two-page answer and a + /// twenty-eight-page answer are different questions, and only a real plan + /// exercises the two passes that decide them. + fn genesis_pages_with(counts: &[usize]) -> Vec { + let page = 1u64 << BLOCK_PAGE_VARS; + let mut configs = Vec::with_capacity(BLOCK_GENESIS_PAGES); + let mut base = 0u64; + for &nonzero in counts { + configs.push(data_page(base, vec![1u8; nonzero])); + base += page; + } + while configs.len() < BLOCK_GENESIS_PAGES { + configs.push(PageConfig::zero_init(base)); + base += page; + } + configs + } + + /// The block's own twenty-eight genesis pages: the two dense ones at their + /// census bases and counts, and twenty-six all-zero. + fn block_genesis_configs() -> Vec { + assert_eq!( + BLOCK_DENSE.len() + BLOCK_ZERO_PAGES, + BLOCK_GENESIS_PAGES, + "the census's page counts must add up, or this set is not the block's" + ); + let page = 1u64 << BLOCK_PAGE_VARS; + let mut configs: Vec = BLOCK_DENSE + .iter() + .map(|&(base, nonzero, _)| data_page(base, vec![1u8; nonzero])) + .collect(); + let mut base = 0x1000000u64; + for _ in 0..BLOCK_ZERO_PAGES { + configs.push(PageConfig::zero_init(base)); + base += page; + } + configs.sort_by_key(|config| config.page_base); + configs + } + + /// THE FORM, THE BRACKET IT IS QUOTED AT, AND WHAT IT DERIVES. + /// + /// ⚠ NEITHER `τ` NOR THE MARGINAL IS A RULED QUANTITY, and both move with + /// the RUN's bracket rather than with a decision. `τ` is 5 below `n_fixed` + /// 23 and 6 from there, so a run of nine or more genesis pages sits at 6 + /// and the block does; the only page that difference reclassifies carries + /// five nonzero genesis bytes, and nothing in the block or the fixtures + /// sits there. The band test below walks the brackets and is the reading of + /// that. + #[test] + fn the_form_and_the_bracket_it_is_quoted_at() { + // The bracket is the one the block's page count gives, which is what + // makes 111 the right charge for a block-shaped run and the wrong one + // for a lone page. + assert_eq!( + fixed_stack_vars(BLOCK_PAGE_VARS, BLOCK_GENESIS_PAGES), + BLOCK_STACK_VARS, + "18 + ceil(log2(56))" + ); + assert_eq!(BLOCK_MARGINAL, 111); + assert_eq!(LONE_N_FIXED, 19, "18 + ceil(log2(2))"); + assert_eq!(LONE_MARGINAL, 101); + + // ⚠ THIS IS THE FORM'S ARITHMETIC, NOT THE STACK'S BILL. The pin + // `lfm::whir_stacked_tests::the_marginal_the_routing_rule_charges_is_the_one_the_stack_bills` + // is the only place the form is compared against what + // `stacked_verify_cost` actually charges; here the four terms are added + // a second way so a mistyped one is caught where it is written. + assert_eq!( + BLOCK_MARGINAL, + 5 * BLOCK_PAGE_VARS + + PAGE_PREPROCESSED_COLUMNS * (BLOCK_STACK_VARS - BLOCK_PAGE_VARS) + + PAGE_PREPROCESSED_COLUMNS * STACKED_ROWS_PER_COLUMN + + MAX_SPONGE_MARGINAL, + "eq 90 + indicators 12 + per-column 6 + the sponge bound 3" + ); + // Two rows a page per prefix bit, which is the term that makes the + // marginal a function of the bracket at all. + assert_eq!( + marginal_stacked_rows(BLOCK_PAGE_VARS, BLOCK_STACK_VARS + 1) - BLOCK_MARGINAL, + PAGE_PREPROCESSED_COLUMNS + ); + assert_eq!( + marginal_stacked_rows(BLOCK_PAGE_VARS, BLOCK_STACK_VARS + 1), + 113 + ); + + // τ from the form at the block's bracket: `18 + 18*S > 111` ⇒ S >= 6. + let tau = candidate_threshold_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS); + assert_eq!(tau, 6); + assert!(!is_candidate(BLOCK_PAGE_VARS, BLOCK_STACK_VARS, 5)); + assert!(is_candidate(BLOCK_PAGE_VARS, BLOCK_STACK_VARS, 6)); + // ⚠ FLOOR PLUS ONE AND NOT `div_ceil`: τ's leg must EXCEED the + // marginal, never meet it. + assert!(sparse_leg_rows(BLOCK_PAGE_VARS, tau - 1) <= BLOCK_MARGINAL); + assert!(sparse_leg_rows(BLOCK_PAGE_VARS, tau) > BLOCK_MARGINAL); + // ⛔ AND τ IS 5 AT A LONE PAGE'S BRACKET — the same page, a different + // run, a different answer. A reader quoting "τ = 6" without its bracket + // is quoting the block's number for everyone. + assert_eq!( + candidate_threshold_entries(BLOCK_PAGE_VARS, LONE_N_FIXED), + 5 + ); + + // ⛔ THE ONE BOUND THAT HOLDS AT EVERY BRACKET: the marginal must exceed + // what `weight_at_rows` alone bills for a page at the charging height, + // which is 102 at the block's (eq 90 + two indicators of six). The form + // clears it by construction — the `+ 6` and the sponge bound are the + // terms that closure cannot see — and a form that stopped would charge + // less than a term it provably contains. + // + // ⚠ A `const` ASSERT, NOT A RUNTIME ONE: breaking it stops the tree + // COMPILING rather than failing a test nobody ran. + const { + assert!( + marginal_stacked_rows(BLOCK_PAGE_VARS, BLOCK_STACK_VARS) > 102, + "the marginal must exceed the 102 rows the weight closure alone bills \ + a page at the charging height; `whir_chain_tests` reads that 102 off \ + the emitter rather than restating it" + ) + }; + const { + assert!( + MAX_SPONGE_MARGINAL >= 1, + "every carried page absorbs six more felts into the threaded sponge, so \ + its marginal contribution is at least one row; a zero bound would \ + undercharge every position" + ) + }; + } + + /// ⚠ PART 1's TWO SPELLINGS ARE ONE RULE — read, not taken on trust. + /// + /// [`is_candidate`] branches on `nonzero >= τ`; the rule is stated as + /// `sparse_leg_rows > marginal_stacked_rows(..)`. The equality of the two + /// is a floor-versus-strict-inequality argument, which is exactly the kind + /// that is right until the division comes out exact. So it is executed, + /// over every `S` around the boundary, at every height AND at every + /// bracket — including the pairs where `(marginal - num_vars)` divides + /// evenly by `num_vars`. + #[test] + fn the_threshold_and_its_entry_count_are_one_rule() { + let mut exact_divisions = 0usize; + let mut div_ceil_would_differ = 0usize; + // ⚠ THE HEIGHT AND THE BRACKET ARE BOTH SWEPT, AND BOTH HAVE TO BE. + // `(marginal - n) % n` is zero only when `n` divides `2p + 9`, `p` + // being the prefix width, and at the production `num_vars = 18` it + // never does — a sweep held at 18 could not tell floor-plus-one from + // `div_ceil` at all and would have been the check that cannot fail. + for num_vars in 1..=40usize { + for prefix in 1..=8usize { + let n_fixed = num_vars + prefix; + let marginal = marginal_stacked_rows(num_vars, n_fixed); + if marginal > num_vars && (marginal - num_vars).is_multiple_of(num_vars) { + exact_divisions += 1; + if (marginal - num_vars).div_ceil(num_vars) + != candidate_threshold_entries(num_vars, n_fixed) + { + div_ceil_would_differ += 1; + } + } + for nonzero in 0..200 { + assert_eq!( + is_candidate(num_vars, n_fixed, nonzero), + sparse_leg_rows(num_vars, nonzero) > marginal, + "num_vars {num_vars}, bracket {n_fixed}, {nonzero} entries: the \ + entry count and the row comparison disagree" + ); + } + } + } + // The arms that make this a check rather than a restatement. + // + // ★ AND THE COVERAGE IS NOW A PROPERTY OF THE FORM RATHER THAN OF A + // CHOSEN NUMBER, which is worth saying because it used to be the + // opposite. `marginal - n = 4n + 2p + 9`, so an exact division is `n` + // dividing `2p + 9`: over `p = 1..=8` that is the divisor count of + // {11, 13, 15, 17, 19, 21, 23, 25} at heights under 41, which is 21 + // pairs. A literal gave three heights when it happened to be composite + // and one when it happened to be prime — coverage that moved with a + // number chosen for other reasons. + println!("EXACT DIVISIONS in the sweep: {exact_divisions}"); + assert_eq!( + exact_divisions, 21, + "the divisor count of 2p + 9 over p = 1..=8 at heights under 41; if this \ + moved, a term of the form moved with it and the sweep is no longer \ + separating floor-plus-one from div_ceil where it thinks it is" + ); + assert_eq!( + div_ceil_would_differ, exact_divisions, + "at every exact division `div_ceil` must give a DIFFERENT answer, or the \ + two forms were never actually separated" + ); + } + + /// ★ CONSEQUENCE (a) — THE PRE-REGISTRATION, on a real plan over the + /// block's twenty-eight pages: exactly those two bases, in that order. + #[test] + fn the_threshold_selects_exactly_the_blocks_dense_pages() { + let configs = block_genesis_configs(); + let plan = genesis_stack_plan(&configs, BLOCK_EPOCHS, BLOCK_PAGE_VARS); + assert_eq!(plan.n_fixed, BLOCK_N_FIXED); + + // Part 1: the two, and only the two. The 26 zero pages cost 18 rows + // sparse against a 103-row marginal, so carrying one would COST rows. + assert_eq!(sparse_leg_rows(BLOCK_PAGE_VARS, 0), 18); + assert!(!is_candidate(BLOCK_PAGE_VARS, plan.n_fixed, 0)); + assert_eq!( + plan.routes.iter().filter(|route| route.candidate).count(), + BLOCK_DENSE.len() + ); + + // Part 2: paid, by nearly two orders of magnitude. + let expected: usize = BLOCK_DENSE + .iter() + .map(|&(_, _, rows)| rows - BLOCK_MARGINAL) + .sum(); + assert_eq!(plan.savings, expected); + assert_eq!(plan.savings, 3_403_398); + assert!(chain_is_paid(plan.savings)); + + // ⚠ THE SET AND ITS ORDER, not a count: the stack's column order IS + // page-base order, so two of the wrong pages would pass a count. + let carried: Vec = plan + .routes + .iter() + .filter(|route| route.dense) + .map(|route| route.page_base) + .collect(); + assert_eq!(carried, vec![0x0, 0x200000]); + assert_eq!(plan.sparse_rows, BLOCK_ZERO_PAGES * 18); + } + + /// The decision does not sit near either constant, which is what makes it a + /// routing rule rather than a tuning knob. + #[test] + fn nothing_on_the_block_sits_near_the_threshold() { + let least_dense = BLOCK_DENSE.iter().map(|&(_, _, r)| r).min().expect("two"); + // Part 1: the cheapest carried page clears the marginal nearly + // 10,000-fold (it was 19,000-fold on the previous guest's census). + assert!(least_dense > 9_000 * BLOCK_MARGINAL); + + // ⛔ AND THE ZERO PAGES' MARGIN IS A RATIO WITH A FLOOR, NOT A FIXED + // FACTOR. This read `* 6 <`, which was true at 109, FALSE at 103 and + // true again at the form's 111 — a margin stated as a multiple of a + // number that moves is an assertion about a draft, and it reddened the + // gate for exactly that reason. It is now the ratio it IS, printed, + // floored at the weakest value any bracket gives: a LONE page's 101 + // still clears an all-zero page by five. + let zero_page = sparse_leg_rows(BLOCK_PAGE_VARS, 0); + let margin = BLOCK_MARGINAL / zero_page; + println!( + "ZERO-PAGE MARGIN: an all-zero page costs {zero_page} rows against a \ + {BLOCK_MARGINAL}-row marginal at the block's bracket ({LONE_MARGINAL} at a \ + lone page's) — a factor of {margin}" + ); + assert!( + margin >= 5, + "an all-zero page costs {zero_page} rows against a marginal of \ + {BLOCK_MARGINAL}, within a factor of {margin} of qualifying — the 26 zero \ + pages are supposed to fail part 1 by a wide margin" + ); + assert!( + LONE_MARGINAL / zero_page >= 5, + "the weakest bracket's marginal ({LONE_MARGINAL}) brings an all-zero page \ + within a factor of {} of qualifying", + LONE_MARGINAL / zero_page + ); + // Part 2: the CHEAPEST carried page is worth six chains on its own, + // and the two together nineteen, so no plausible re-sizing of the + // budget changes the block's answer. + assert!( + least_dense > 6 * PREPARED_LEG_ROWS, + "the cheapest stacked page is {least_dense} rows against a \ + {PREPARED_LEG_ROWS} budget" + ); + } + + /// ★ CONSEQUENCE (c) — A LONE PAGE, AT THE BOUNDARY BOTH WAYS. + /// + /// At one genesis page the two parts collapse to a single condition on that + /// page: it is the only candidate, so the set's savings ARE its savings. + /// That is the SHAPE of the rule this replaces — and not its value. + /// + /// ⛔⛔ THE BOUNDARY IS DERIVED FROM THE RUN'S OWN BRACKET AND IS NOT + /// HARD-CODED HERE, and that is deliberate. It is + /// `densest_sparse_entries()` and one more, and it MOVES one entry per + /// `num_vars` rows of marginal. A LONE page stands at 19 variables and is + /// charged 101, which puts it at (9,730 sparse, 9,731 dense) across a + /// marginal of `[92, 109]`; a page inside the BLOCK's twenty-eight stands at 24, + /// is charged 111, and faces (9,731, 9,732) across `[110, 127]`. ⚠ BOTH + /// PAIRS ARE REAL AND NEITHER IS "the" boundary — which is the reading the + /// literal could not produce, since it charged a lone page the block's + /// bracket. `the_ruled_boundaries_hold_for_every_marginal_in_band` states + /// the bands. + /// + /// ⛔ THE RETIRED BREAK-EVEN WAS 9,725, and the entries between it and this + /// boundary are why every bound derived from the old rule has to be + /// re-derived rather than re-read: a test left at 9,724 stays green under + /// both and means something under neither. + #[test] + fn a_lone_page_is_at_the_boundary_its_own_bracket_puts_it_at() { + let sparse_at = densest_sparse_entries(BLOCK_PAGE_VARS, LONE_N_FIXED); + let dense_at = sparse_at + 1; + println!( + "LONE BOUNDARY at bracket {LONE_N_FIXED} (marginal {LONE_MARGINAL}): \ + {sparse_at} sparse, {dense_at} dense; this pair holds across a marginal \ + of [92, 109], so there are {} rows of headroom above this bracket's \ + charge. At the block's bracket ({BLOCK_STACK_VARS}, marginal \ + {BLOCK_MARGINAL}) the same page would face ({}, {})", + 109 - LONE_MARGINAL, + densest_sparse_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS), + densest_sparse_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS) + 1 + ); + assert_eq!( + (sparse_at, dense_at), + (9_730, 9_731), + "the pair at a LONE page's bracket. If it moved, read the marginal: this \ + pair holds across [92, 109], and the block's bracket puts it at \ + (9,731, 9,732) across [110, 127]" + ); + // ⛔ THE CONTRAST, ASSERTED RATHER THAN DESCRIBED: the same entry count, + // a different run, a different answer. A reader quoting one pair without + // its bracket is quoting it for the wrong runs. + assert_eq!( + densest_sparse_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS), + 9_731 + ); + + for (nonzero, carried) in [(sparse_at, false), (dense_at, true)] { + let configs = vec![data_page(0x40000, vec![1u8; nonzero])]; + let plan = genesis_stack_plan(&configs, 3, BLOCK_PAGE_VARS); + // ★ At one page the plan stands at 19 variables and the marginal + // is charged THERE — 101, not the block's 111. The form following + // the run's shape is the whole of what it buys over a literal. + assert_eq!(plan.n_fixed, LONE_N_FIXED); + assert_eq!(plan.n_fixed, 19, "18 + ceil(log2(2))"); + assert_eq!(marginal_stacked_rows(BLOCK_PAGE_VARS, plan.n_fixed), 101); + assert!( + plan.routes[0].candidate, + "a page of {nonzero} entries clears part 1 either way; only part 2 \ + separates these two" + ); + assert_eq!( + plan.savings, + page_savings(BLOCK_PAGE_VARS, plan.n_fixed, nonzero) + ); + assert_eq!( + !plan.is_empty(), + carried, + "a lone page of {nonzero} nonzero entries saves {} against a \ + {PREPARED_LEG_ROWS}-row chain", + plan.savings + ); + } + + // ⛔ HOW MUCH MARGIN THE PAIR HAS, ASSERTED SO NOBODY REDISCOVERS IT. + // The page just over the boundary clears the chain by NINE rows at this + // bracket — so nine rows of marginal, half a prefix bit, would move it. + assert_eq!( + page_savings(BLOCK_PAGE_VARS, LONE_N_FIXED, dense_at), + PREPARED_LEG_ROWS + 9 + ); + + // The retired rule's break-even, kept here as the contrast and nowhere + // else: `18 + 18*S > PREPARED_LEG_ROWS`, floor plus one. + let retired = (PREPARED_LEG_ROWS - BLOCK_PAGE_VARS) / BLOCK_PAGE_VARS + 1; + assert_eq!(retired, 9_725); + assert!( + retired < dense_at, + "the two-part rule charges a lone page its own marginal on top of the \ + chain, so its boundary must sit ABOVE the retired one ({retired} vs \ + {dense_at})" + ); + } + + /// ★ CONSEQUENCE (b) — A CANDIDATE THE CHAIN REFUSES TO PAY FOR, which is + /// where every fixture in the tree sits. + /// + /// `data_page_touch`'s one data page carries 112 nonzero genesis bytes. It + /// passes part 1 comfortably — 2,034 rows against a 111-row marginal — and + /// part 2 refuses it, because 1,923 saved rows do not buy a 175,066-row + /// chain. ⚠ THAT IS THE INTERESTING BRANCH AND IT IS WHY THE FIELD + /// `candidate` EXISTS: under the retired rule this page failed the only + /// test there was, and "no opening" meant "nobody wanted it". + #[test] + fn a_candidate_the_chain_cannot_be_paid_for_stays_sparse() { + let configs = genesis_pages_with(&[112]); + let plan = genesis_stack_plan(&configs, 3, BLOCK_PAGE_VARS); + assert_eq!(plan.n_fixed, BLOCK_N_FIXED); + assert_eq!(sparse_leg_rows(BLOCK_PAGE_VARS, 112), 2_034); + assert!(plan.routes[0].candidate, "2,034 rows against the marginal"); + // ⛔ ONE DERIVATION, AND THE VALUE IN THE MESSAGE. This line used to be + // followed by `assert_eq!(plan.savings, 1_931)` — the same quantity a + // second time, as a bare literal computed under the retired 103. It + // named no symbol, so the sweep that re-pointed every reference to the + // marginal could not see it, and the form moved the savings to 1,923 + // underneath it. ⇒ a number a reader wants is a MESSAGE, never a second + // assertion: the second one drifts, and it drifts silently until the + // day the first one moves. + assert_eq!( + plan.savings, + 2_034 - BLOCK_MARGINAL, + "the sparse leg pinned just above (2,034) less what the block's bracket \ + charges a carried page ({BLOCK_MARGINAL}), which is {} rows", + plan.savings + ); + assert!(!chain_is_paid(plan.savings)); + assert!( + plan.is_empty(), + "the chain is not paid, so nothing is carried" + ); + assert!(!plan.routes[0].dense); + assert_eq!( + plan.sparse_rows, + 2_034 + (BLOCK_GENESIS_PAGES - 1) * 18, + "a refused candidate still pays its sparse leg, and so do the zero pages" + ); + + // And the dense guest written for this route clears part 2 outright: + // `dense_data_page_touch` surrounds its cell with 32 KiB either side. + let dense = genesis_stack_plan(&genesis_pages_with(&[65_652]), 3, BLOCK_PAGE_VARS); + assert_eq!(dense.savings, 18 + 18 * 65_652 - BLOCK_MARGINAL); + assert!(chain_is_paid(dense.savings)); + assert_eq!(dense.dense_pages(), vec![0]); + } + + /// ★ CONSEQUENCE (d) — TWO PAGES THAT SHARE ONE CHAIN, the case the retired + /// rule got wrong. + /// + /// Neither page is worth a chain of its own; between them they are worth + /// two. A rule that charges the chain per page refuses both and spends + /// 180,036 rows keeping them sparse — more than the chain it declined to + /// buy. That is not a tuning miss, it is a fixed cost charged to the wrong + /// thing, and part 2 is where it moves. + #[test] + fn two_pages_worth_less_than_the_chain_apiece_share_one() { + let configs = genesis_pages_with(&[5_000, 5_000]); + let plan = genesis_stack_plan(&configs, 3, BLOCK_PAGE_VARS); + assert_eq!(plan.n_fixed, BLOCK_N_FIXED); + assert_eq!(sparse_leg_rows(BLOCK_PAGE_VARS, 5_000), 90_018); + + // Neither one pays for the chain alone. + let alone = page_savings(BLOCK_PAGE_VARS, plan.n_fixed, 5_000); + assert_eq!(alone, 89_907); + assert!(!chain_is_paid(alone)); + // Together they do, and BOTH are carried — the set is paid for or none + // of it is. + // + // ⛔ ONE DERIVATION, AND THE SUM IS THE SECOND PLACE THIS BIT. A bare + // `assert_eq!(plan.savings, 179_830)` stood here — `2 × 89,915`, the + // retired constant's `alone` DOUBLED. The by-value sweep that caught + // the same shape at the refused candidate looked for the BASE + // quantities and not their MULTIPLES, so re-pointing `alone` from + // 89,915 to 89,907 left its double untouched one line below. ⇒ a sweep + // over a moved form must cover SUMS AND PRODUCTS of what moved, not + // only the terms themselves. + assert_eq!( + plan.savings, + 2 * alone, + "two candidates of {} rows each between them save {} against a \ + {PREPARED_LEG_ROWS}-row chain", + alone, + plan.savings + ); + assert!(chain_is_paid(plan.savings)); + assert_eq!(plan.dense_pages(), vec![0, 1]); + + // What the retired rule spent instead. + assert_eq!(2 * sparse_leg_rows(BLOCK_PAGE_VARS, 5_000), 180_036); + } + + fn data_page(base: u64, bytes: Vec) -> PageConfig { + PageConfig::with_data(base, bytes) + } + + /// ⛔ THE PRIVATE FILTER, adversarially: a private-input page whose genesis + /// bytes WOULD cross the threshold is still not stacked. + /// + /// This is the arm that would have caught the prover and the verifier + /// selecting different sets. On the prover a private page carries the + /// private input; on the verifier it carries an empty vec. + #[test] + fn a_private_page_dense_enough_to_qualify_is_still_not_stacked() { + let dense_bytes = vec![0xABu8; 20_000]; + // ⚠ THE PRECONDITION: these bytes must be carried when they are PUBLIC, + // or the test passes on a page nobody would have stacked anyway. + // ⚠ AT THIS RUN'S OWN BRACKET, which is a ONE-genesis-page stack: the + // private page is filtered out before the count is taken, so the + // precondition has to be evaluated where the plan evaluates it. + assert!(is_candidate( + BLOCK_PAGE_VARS, + LONE_N_FIXED, + dense_bytes.len() + )); + assert!(chain_is_paid(page_savings( + BLOCK_PAGE_VARS, + LONE_N_FIXED, + dense_bytes.len() + ))); + let mut private = data_page(0xff000000, dense_bytes.clone()); + private.is_private_input = true; + let public = data_page(0x40000, dense_bytes); + + let plan = genesis_stack_plan(&[private, public], 3, BLOCK_PAGE_VARS); + assert_eq!( + plan.at, + stark::multilinear_table::leading_columns(4, PAGE_PREPROCESSED_COLUMNS), + "only the non-private page may be stacked, at its own AIR-set index, \ + and with BOTH of its preprocessed columns" + ); + assert!(!plan.routes[0].dense); + assert!(!plan.routes[0].has_init); + assert!(plan.routes[1].dense); + } + + /// The same page set with the private page's bytes REMOVED — the verifier's + /// view — plans identically. That is the property the two sides need, and + /// the reason the filter is on `is_private_input` and not on the bytes. + #[test] + fn the_prover_and_verifier_views_of_a_private_page_plan_alike() { + let dense_bytes = vec![0xABu8; 20_000]; + let mut prover_side = data_page(0xff000000, dense_bytes.clone()); + prover_side.is_private_input = true; + let mut verifier_side = data_page(0xff000000, Vec::new()); + verifier_side.is_private_input = true; + let public = data_page(0x40000, dense_bytes); + + let from_prover = genesis_stack_plan(&[prover_side, public.clone()], 3, BLOCK_PAGE_VARS); + let from_verifier = genesis_stack_plan(&[verifier_side, public], 3, BLOCK_PAGE_VARS); + assert_eq!( + from_prover.at, from_verifier.at, + "prover and verifier must commit the same stack or the roots block diverges" + ); + assert_eq!(from_prover.sparse_rows, from_verifier.sparse_rows); + // ⛔ EVERY INPUT TO THE RULE, NOT JUST ITS ANSWER. The two-part rule + // reads a page count and a savings total as well as each page's bytes, + // and two sides that agreed on the set while disagreeing on either + // would be agreeing by luck. `n_fixed` is the page count's fingerprint + // and `savings` the candidate set's. + assert_eq!( + from_prover.n_fixed, from_verifier.n_fixed, + "the fixed height is taken at the genesis page count, which the private \ + filter must make identical on both sides" + ); + assert_eq!(from_prover.savings, from_verifier.savings); + assert_eq!( + from_prover.routes, from_verifier.routes, + "including each page's candidacy, which is part 1's answer" + ); + } + + /// The table index a stacked column names is the AIR set's, bookends + /// included — not the page's index in its own family. And the columns come + /// out in stack order: page by page, each page's prefix in index order. + #[test] + fn the_stacked_columns_name_their_index_in_the_whole_air_set_in_stack_order() { + let dense = vec![0x01u8; 20_000]; + let configs = vec![ + PageConfig::zero_init(0x0), + data_page(0x40000, dense.clone()), + PageConfig::zero_init(0x80000), + data_page(0x280000, dense), + ]; + let plan = genesis_stack_plan(&configs, 15, BLOCK_PAGE_VARS); + let mut want = stark::multilinear_table::leading_columns(16, PAGE_PREPROCESSED_COLUMNS); + want.extend(stark::multilinear_table::leading_columns( + 18, + PAGE_PREPROCESSED_COLUMNS, + )); + assert_eq!( + plan.at, want, + "fifteen bookends precede the pages, so pages 1 and 3 are tables 16 and 18" + ); + assert_eq!(plan.dense_pages(), vec![1, 3]); + // The two zero pages still cost the interned zero apiece. + assert_eq!(plan.sparse_rows, 2 * sparse_leg_rows(BLOCK_PAGE_VARS, 0)); + + // ⚠ AND THE COLUMNS AGREE WITH `at` IN LENGTH AND IN ORDER. A stack + // whose columns and whose destinations disagreed would settle one page's + // values against another's commitment. + let columns = genesis_stack_columns(&configs, &plan); + assert_eq!(columns.len(), plan.at.len()); + assert_eq!( + columns[0], + page::preprocessed_columns(&configs[1])[0], + "stack column 0 is dense page 1's OFFSET" + ); + assert_eq!( + columns[1], + page::preprocessed_columns(&configs[1])[1], + "stack column 1 is dense page 1's INIT" + ); + assert_eq!( + columns[2], + page::preprocessed_columns(&configs[3])[0], + "stack column 2 is dense page 3's OFFSET" + ); + } + + /// A run whose genesis is entirely sparse carries no opening at all, and its + /// cross-epoch proof is the one it was before this route existed. + /// + /// ⚠ AND IT SAYS WHICH PART REFUSED. The data page IS a candidate here — + /// part 2 is what leaves the set empty — so a green would otherwise be + /// consistent with a part 1 that had stopped working. + #[test] + fn an_all_sparse_page_set_stacks_nothing() { + let configs = vec![ + PageConfig::zero_init(0x0), + data_page(0x40000, vec![1u8; 112]), + ]; + let plan = genesis_stack_plan(&configs, 3, BLOCK_PAGE_VARS); + assert!(plan.is_empty()); + assert!(plan.dense_pages().is_empty()); + assert!(genesis_stack_columns(&configs, &plan).is_empty()); + assert!(!plan.routes[0].candidate, "an all-zero page fails part 1"); + assert!(plan.routes[1].candidate, "112 entries clear part 1"); + assert!( + !chain_is_paid(plan.savings), + "part 2 is what empties this set, and its savings are {}", + plan.savings + ); + } + + /// ⛔⛔ HOW MUCH OF THIS RULE DEPENDS ON THE MARGINAL'S EXACT VALUE — + /// EXECUTED, not asserted to be small. + /// + /// [`marginal_stacked_rows`] is a form now, so the question is no longer + /// "what will the pin say" but "which bracket is this run in": the same + /// form charges 101 at one genesis page, 111 at the block's twenty-eight and 125 + /// at a run of 4,096. So "the exact value is immaterial" is a claim that + /// has to be READ over the whole range a run can put it in, one consequence + /// at a time. + /// + /// ★ AND IT IS HOW THE MOVING QUANTITIES ANNOUNCE THEMSELVES. Two of the + /// five do move, and the bands are asserted rather than the values: + /// + /// | consequence | holds for a marginal in | + /// |---|---| + /// | the block's two dense pages, paid | `[19, 1_107_270]` | + /// | the fixture (S = 112) refused by part 2 | every marginal | + /// | two pages of 5,000 share one chain | `[1, 2_484]` | + /// | the lone-page pair (9,730, 9,731) | `[92, 109]` ⚠ | + /// | `τ = 5` | `[90, 107]` ⚠ — the block's bracket has 6 | + /// + /// ⚠ AND THE BRACKETS ARE WALKED THROUGH THOSE BANDS, which is the arm a + /// literal could not have: `τ` is 5 up to `n_fixed` 22 and 6 from 23, so + /// the rule's threshold depends on the RUN'S PAGE COUNT — nine genesis + /// pages or more and a page needs six nonzero bytes, eight or fewer and it + /// needs five. The day a bracket takes a quantity out of its band, this + /// goes red naming the band and the quantity, and the doc at + /// [`densest_sparse_entries`] says what to write instead. + #[test] + fn the_ruled_boundaries_hold_for_every_marginal_in_band() { + // The forms with the marginal as a PARAMETER. ⚠ They are restated here + // and nowhere else, and the assertions below tie them back to the live + // functions at the shipped literal, so a restatement that drifted from + // the code would be caught rather than swept. + let savings = |m: usize, s: usize| sparse_leg_rows(BLOCK_PAGE_VARS, s).saturating_sub(m); + let tau = |m: usize| m.saturating_sub(BLOCK_PAGE_VARS) / BLOCK_PAGE_VARS + 1; + let densest = |m: usize| (PREPARED_LEG_ROWS + m - BLOCK_PAGE_VARS) / BLOCK_PAGE_VARS; + let m0 = BLOCK_MARGINAL; + assert_eq!( + tau(m0), + candidate_threshold_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS) + ); + assert_eq!( + densest(m0), + densest_sparse_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS) + ); + assert_eq!( + savings(m0, 112), + page_savings(BLOCK_PAGE_VARS, BLOCK_STACK_VARS, 112) + ); + + // THE TWO THAT DO NOT MOVE ANYWHERE NEAR HERE. + for m in 90..=130usize { + assert!( + !chain_is_paid(savings(m, 112)), + "at marginal {m} the fixture's 112-entry page would buy a chain" + ); + assert!( + chain_is_paid(2 * savings(m, 5_000)) && !chain_is_paid(savings(m, 5_000)), + "at marginal {m} the two 5,000-entry pages no longer share exactly one chain" + ); + let block: usize = BLOCK_DENSE.iter().map(|&(_, _, r)| r - m).sum(); + assert!( + chain_is_paid(block), + "at marginal {m} the block would not pay" + ); + assert!( + sparse_leg_rows(BLOCK_PAGE_VARS, 0) <= m, + "at marginal {m} an all-zero page would become a candidate" + ); + } + + // ★ AND THE BLOCK'S SET IS STABLE ACROSS NEARLY FIVE ORDERS OF MAGNITUDE, + // which is the argument that this is a routing rule and not a knob: any + // marginal from the cost of one zero page up to the cheapest dense + // page's own leg selects exactly those two. + for m in [19usize, 109, 1_000, 100_000, 1_107_269] { + let block: usize = BLOCK_DENSE.iter().map(|&(_, _, r)| r - m).sum(); + assert!(chain_is_paid(block) && sparse_leg_rows(BLOCK_PAGE_VARS, 0) <= m); + } + + // ⚠ THE TWO THAT MOVE, asserted as BANDS so the move is legible. + let band = |f: &dyn Fn(usize) -> bool| { + let hits: Vec = (1..=3_000usize).filter(|&m| f(m)).collect(); + let (lo, hi) = (hits[0], hits[hits.len() - 1]); + assert_eq!( + hits, + (lo..=hi).collect::>(), + "a band that is not contiguous is not a band" + ); + (lo, hi) + }; + let tau_is_five = band(&|m| tau(m) == 5); + let lone_pair = + band(&|m| !chain_is_paid(savings(m, 9_730)) && chain_is_paid(savings(m, 9_731))); + println!( + "MARGINAL BANDS at the block's bracket ({BLOCK_STACK_VARS}, marginal {m0}): \ + τ = 5 for {tau_is_five:?} (τ here is {}), the (9730, 9731) pair for \ + {lone_pair:?}, densest sparse {}", + tau(m0), + densest(m0) + ); + // ⛔ THE BANDS ARE COMPLETE AT THE TOP, WHICH THEY WERE NOT. τ was + // tabled as "5 on [90, 107], 6 from 108" and then asserted at m = 130, + // where it is 7 — an open-ended "from" is not a band, and the sweep ran + // past the end of the one that had been written down. All three + // sub-bands are asserted here, with both of their edges. + assert_eq!(tau_is_five, (90, 107)); + assert_eq!(band(&|m| tau(m) == 6), (108, 125)); + assert_eq!(band(&|m| tau(m) == 7), (126, 143)); + assert_eq!(lone_pair, (92, 109)); + assert_eq!(band(&|m| densest(m) == 9_731), (110, 127)); + assert_eq!(band(&|m| densest(m) == 9_732), (128, 145)); + + // ⛔ AND WHERE THE LITERAL FALLS IS COMPUTED, NOT HARD-CODED. Every + // earlier version named the band it expected, so each time the literal + // moved this reddened on the naming rather than on the finding. It now + // asserts only that the literal lies inside the band for ITS OWN value + // — true at the current 103, at the coming form's 111, and at the 113 a + // taller bracket costs — and PRINTS which bands those are. + let my_tau = tau(m0); + let my_pair = densest(m0); + assert_eq!( + my_tau, + candidate_threshold_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS) + ); + assert_eq!( + my_pair, + densest_sparse_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS) + ); + let tau_band = band(&|m| tau(m) == my_tau); + let pair_band = band(&|m| densest(m) == my_pair); + println!( + "THE FORM AT {BLOCK_STACK_VARS} CHARGES {m0}: τ = {my_tau} over \ + {tau_band:?}, pair ({my_pair}, {}) over {pair_band:?}", + my_pair + 1 + ); + assert!(tau_band.0 <= m0 && m0 <= tau_band.1); + assert!(pair_band.0 <= m0 && m0 <= pair_band.1); + + // ⛔⛔ THE WHOLE SPREAD AT ONE BRACKET, EXECUTED. The per-page marginal + // is not one number even there: the threaded sponge makes it 109, 110 + // or 111 by position, and the form charges the dearest — which is why + // `m0` is the TOP of that spread rather than a reading of one position. + // Walking every member means the table in the form's doc is a reading + // and not a claim. + assert_eq!( + m0, 111, + "the form charges the dearest position in the block bracket's spread" + ); + for m in [109usize, 110, 111] { + assert_eq!(tau(m), 6, "τ is 6 across the whole spread"); + } + assert_eq!( + densest(109), + 9_730, + "the cheapest position keeps today's pair" + ); + for m in [110usize, 111] { + assert_eq!(densest(m), 9_731); + } + // ⇒ AND THAT IS WHY THE FORM TAKES THE MAXIMUM RATHER THAN A MIDPOINT: + // τ is invariant across the spread, so the only quantity the choice + // decides is the pair, and charging the dearest position is + // conservative everywhere for the price of one entry. + + // ★★ THE BRACKETS THEMSELVES, WALKED — the arm a literal could not + // have. Every genesis page count from one page to four thousand, the + // marginal the form charges there, and the two derived quantities read + // at each. ⚠ τ IS NOT INVARIANT ACROSS THEM, and that is the finding + // rather than a defect: it is 5 while the marginal is under 108 and 6 + // from there, so the crossing is a property of the RUN's page count. + // (It reaches 7 at 126, which no reachable page count produces: 2^393 + // thousand pages.) + let mut crossings = 0usize; + let mut previous: Option = None; + for pages in [1usize, 2, 4, 8, 9, 16, 30, 64, 256, 4_096] { + let n_fixed = fixed_stack_vars(BLOCK_PAGE_VARS, pages); + let m = marginal_stacked_rows(BLOCK_PAGE_VARS, n_fixed); + let t = candidate_threshold_entries(BLOCK_PAGE_VARS, n_fixed); + println!( + "BRACKET {pages} pages -> n_fixed {n_fixed}, marginal {m}, τ {t}, \ + densest sparse {}", + densest_sparse_entries(BLOCK_PAGE_VARS, n_fixed) + ); + assert_eq!(t, tau(m), "the closure and the live function must agree"); + assert!( + (5..=6).contains(&t), + "τ left the two values every bracket up to 4,096 genesis pages produces" + ); + if previous.is_some_and(|p| p != t) { + crossings += 1; + } + previous = Some(t); + } + assert_eq!( + crossings, 1, + "τ crosses once over this range, between eight and nine genesis pages; a \ + second crossing means a term of the form is not monotone in the bracket" + ); + // The crossing, named: eight pages charge 107 and nine charge 109. + assert_eq!( + candidate_threshold_entries(BLOCK_PAGE_VARS, fixed_stack_vars(BLOCK_PAGE_VARS, 8)), + 5 + ); + assert_eq!( + candidate_threshold_entries(BLOCK_PAGE_VARS, fixed_stack_vars(BLOCK_PAGE_VARS, 9)), + 6 + ); + // ⚠ AND THE PAIR MOVES WITH THE BRACKET TOO, one entry at a time: a + // lone page's 9,730 against the block's 9,731. Both are inside the + // bands above, which is why neither moves a ruled consequence. + assert_eq!(densest_sparse_entries(BLOCK_PAGE_VARS, LONE_N_FIXED), 9_730); + assert_eq!( + densest_sparse_entries(BLOCK_PAGE_VARS, BLOCK_STACK_VARS), + 9_731 + ); + + // ⚠ THE 113 A TALLER RUN COSTS, which the form charges rather than + // understating: a stack at `n_stack` 25 reaches 113 a page. It moves + // NEITHER quantity, both bands reaching past it. + assert_eq!( + marginal_stacked_rows(BLOCK_PAGE_VARS, BLOCK_STACK_VARS + 1), + 113 + ); + assert_eq!(tau(113), 6); + assert_eq!(densest(113), 9_731); + } } diff --git a/prover/src/hash_pin.rs b/prover/src/hash_pin.rs new file mode 100644 index 000000000..398185809 --- /dev/null +++ b/prover/src/hash_pin.rs @@ -0,0 +1,439 @@ +//! ★★ **THE HASH PIN** — the one place a build says which hash the BLOCK path +//! proves under. +//! +//! # Why this is a module and not a line in `crypto/stark` +//! +//! `crypto/stark`'s [`stark::config::DefaultStarkHash`] is the *workspace's* +//! default: it names the hash behind `Commitment`, `BatchedMerkleTree` and every +//! blessed constant in the repo, and a `const` assertion there makes re-pointing +//! it a compile error precisely so those artifacts cannot drift. +//! +//! The hash-comparison branches need something different — to change what the +//! block path commits under **without** touching that default, so BLAKE3's +//! enforcement stays intact while a sibling branch proves under RPO. ✓ VERIFIED +//! that is expressible: `IsStarkProver` +//! is generic over the configuration, and `prover` was already naming +//! `DefaultStarkHash` *explicitly* at each of its prove and verify call sites. +//! Those are type parameters, not a global. Collecting them behind these two +//! names turns "which hash does the block path use" from a property spread over +//! six files into a property of this one. +//! +//! # ⚠ The pin is TWO things, and the second is easy to miss +//! +//! [`StarkHash::Transcript`] names a `TranscriptHash` — a **digest +//! configuration**, which is what GRINDING computes over. The Fiat–Shamir +//! transcript **object** is built by the caller and handed to `multi_prove`, so +//! the type system does not force it to match. +//! +//! For the byte hashes the two coincide: the object is +//! `DefaultTranscript`, a sponge over that digest. **For an +//! algebraic hash they do not.** `AlgebraicTranscript` is a compress chain over +//! cells, not a byte sponge over `AlgebraicDigest`, and a branch that pinned only +//! [`BlockStarkHash`] would commit under RPO while sponging Fiat–Shamir through +//! bytes — self-consistent between prover and verifier, and therefore **silent**. +//! That is the same half-flip `stark::config::DefaultStarkTranscript`'s own doc +//! warns about, and [`block_transcript`] is why it cannot happen here. +//! +//! # What a branch changes +//! +//! Exactly the three items below, and nothing else in the workspace. On an +//! algebraic branch they become, for example: +//! +//! ```ignore +//! pub type BlockStarkHash = crate::lfm::algebraic_commit::RpoStarkHash; +//! pub type BlockTranscript = crate::lfm::algebraic_transcript::AlgebraicTranscript; +//! pub fn block_transcript(seed: &[u8]) -> BlockTranscript { +//! BlockTranscript::with_seed(crate::lfm::hash::HasherKind::Rpo, seed) +//! } +//! ``` +//! +//! ✓ VERIFIED that flip compiles and runs end to end — it was performed, built, +//! and executed against this crate's own prove/verify tests before this module +//! was written, which is how [`BlockProver`], the generic transcript parameter +//! on `compute_expected_commit_bus_balance_view`, and the +//! `IsStreamingLeafBackend` import in `proof_arena` were found. None of those +//! three shows up on a build that only ever pins BLAKE3. +//! +//! # `cuda` on an algebraic pin +//! +//! Compiles, and cannot prove under the wrong hash. The algebraic backends are +//! `DeviceTreeBackend`s carrying their own `CommitmentHash` as the device +//! dispatch key, so a device tree is built by the kernels of the hash it is +//! named for or not built at all. RPX256 has those kernels (`math_cuda::rpx`), +//! so a GPU run under this pin commits on the device; RPO256 and Poseidon do +//! not yet, and a GPU run under one of them aborts at its first device commit +//! with `unimplemented!` naming the hash. ⛔ Neither a `compile_error!` nor a +//! byte-hash fallback belongs here: the first hides the cuda lint arm from the +//! branch, the second is exactly the silent wrong-hash build this module exists +//! to make impossible. +//! +//! # ⚠ TWO regenerations, not one — in THIS order, plus one stray constant +//! +//! A pin change is **not** complete until every root blessed under the old hash +//! is regenerated. There are two families of them and the order is load-bearing: +//! +//! 1. **The static preprocessed commitments — FIRST.** FOUR families: `bitwise`, +//! `keccak_rc`, and `page`'s zero-page AND private-page constants, at blowup +//! 2/4/8. Each returns a BLESSED CONSTANT from `preprocessed_commitment` +//! rather than recomputing, so under a new pin the prover recomputes an +//! algebraic root, compares it against a BLAKE3 constant, and fails with +//! `ProvingError::PrecomputedCommitmentMismatch`. +//! `cargo run --bin compute_static_commitments --release`, then paste. +//! `bitwise` and `keccak_rc` also ship POSEIDON1 twins for the block's proof +//! family (`static_commitment_p1`, `-- --hash p1`): regenerate them with the +//! rest whenever the tables or the commit pipeline change. +//! 2. **`LFM_REGISTRY` — SECOND, only once the statics are in the tree.** +//! `registry.rs` fills slots 13 and 14 of every entry from `keccak_rc` and +//! `bitwise`'s `preprocessed_commitment` — the blessed constants above, not a +//! recomputation — and `lfm_program_id` folds every root. A registry generated +//! before the statics were pasted therefore embeds the OUTGOING hash's +//! constants, and `machine_tests::registry_drift_*` fires at exactly those two +//! slots. The control-first re-run under the outgoing pin cannot see this: +//! both tables are self-consistent there. +//! `cargo run --bin compute_lfm_registry --release`. +//! 3. **`SUB_DECODE_COMMITMENT_BLOWUP_2`** in `tests/decode_tests.rs` — a +//! test-local blessed constant outside both generators, regenerated by the +//! `#[ignore]` test `print_decode_commitment_for_sub`. +//! +//! ✓ VERIFIED (1) empirically: it is exactly how the trial flip failed, and it +//! is the correct failure — loud, at prove time, naming the cause. ✓ VERIFIED +//! (2) empirically too: the first RPX regeneration ran the registry before the +//! statics and all six drift tests fired at slots 13 and 14. `registry.rs` +//! governs both: a drift failure is investigated, never re-blessed to silence +//! the test, and neither table is ever hand-edited. + +/// The commitment configuration the block path proves and verifies under. +/// +/// Every `multi_prove` / `multi_verify` instantiation in this crate names this +/// rather than `stark::config::DefaultStarkHash`, so the two can differ on a +/// branch without the workspace default moving. +pub type BlockStarkHash = crate::lfm::algebraic_commit::RpxStarkHash; + +/// The Fiat–Shamir transcript OBJECT the block path builds. +/// +/// See the module header for why this is pinned separately from +/// [`BlockStarkHash`] rather than derived from it. +pub type BlockTranscript = crate::lfm::algebraic_transcript::AlgebraicTranscript; + +/// A fresh block-path transcript over `seed`. +/// +/// A function rather than a bare `::new`, because the two arms construct +/// differently: a byte transcript takes the seed in its constructor, an +/// algebraic one absorbs it as its first `append_bytes` call. Callers should not +/// have to know which. +pub fn block_transcript(seed: &[u8]) -> BlockTranscript { + BlockTranscript::with_seed(BLOCK_HASHER, seed) +} + +/// The prover the block path drives, at [`BlockStarkHash`]. +/// +/// ⚠ **Not `stark::prover::Prover`.** That alias is `GenericProver` at +/// `DefaultStarkHash`, so it is BLAKE3-fixed regardless of what `H` a call site +/// passes alongside it — and the two disagreeing is a type error rather than a +/// silent wrong hash, which is how this was found. The `IsStarkProver` impl +/// itself is fully generic over `H`; only the alias is pinned, so the fix is an +/// alias at the pin rather than anything in `crypto/stark`. +pub type BlockProver = + stark::prover::GenericProver; + +/// The verifier the block path drives, at [`BlockStarkHash`]. See +/// [`BlockProver`] for why the `stark::verifier::Verifier` alias is not it. +pub type BlockVerifier = + stark::verifier::GenericVerifier; + +/// The `LFM_HASH` socket permutation the block path's programs are EXECUTED and +/// proved under — the machine's own hash chip. +/// +/// ⚠ **A third axis, and it is orthogonal to [`BlockStarkHash`].** That one says +/// which hash the HOST commits under; this says which permutation the MACHINE's +/// `Instr::Hash` rows compute. They have to agree, and nothing in the type +/// system makes them: the socket hasher is passed per call to `execute` and +/// `lfm_prove_with_hasher`. +/// +/// ★ **Why it went unpinned until it bit.** Under a byte hash the emitter's +/// Merkle work goes through `ByteWrapHash::hash_bytes`, which lowers to the +/// dedicated KECCAK / `LFM_BLAKE3` chips and emits **no `Instr::Hash` at all** — +/// so the socket hasher handed to `execute` is never consulted, and passing +/// `TestPermutation` is free and correct. The algebraic arm goes through +/// `b.compress` / `b.permute`, which ARE `Instr::Hash`, executed by whatever is +/// passed. The same distinction the byte hash made irrelevant, needed back. +/// +/// Every `execute` and prove call on the block path names this rather than a +/// literal, so the two axes cannot drift apart in a test harness while +/// production stays correct. +pub const BLOCK_HASHER: crate::lfm::hash::HasherKind = crate::lfm::hash::HasherKind::Rpx; + +// The host block transcript (`block_transcript`) hashes twelve-felt steps with +// `BLOCK_HASHER`; the width-16 socket has no twelve-felt hash (its contract +// refuses), so it can never be the block hasher. A width-16 program takes the +// socket from its own instructions (`LfmProgram::hasher`), never from here. +const _: () = assert!( + !matches!(BLOCK_HASHER, crate::lfm::hash::HasherKind::Poseidon1W16), + "the block hasher hashes twelve-felt transcript steps; the width-16 socket has none" +); + +/// The [`CommitmentHash`] the block path's roots may be called by. +/// +/// ★ Read this rather than `stark::config::COMMITMENT_HASH`. That const names +/// the hash of the workspace ALIASES and says so in its own doc — a prover can +/// run under a configuration whose `COMMITMENT_HASH` differs and the const will +/// not know. The block path IS such a configuration on three of the four +/// branches, so anything describing a block proof's roots must read the pin. +pub const BLOCK_COMMITMENT_HASH: stark::config::CommitmentHash = + ::COMMITMENT_HASH; + +// ============================================================================ +// ★★★ THE BLOCK'S PROOF FAMILY (D-P3B §2.1) +// ============================================================================ + +/// ★★★ Everything a block's recursion proves and verifies with, named ONCE: the +/// LFM proofs' commitment configuration and their transcript, the tag every +/// LFM statement names them by, the wrap hash a node's in-guest legs replay +/// them with, and the socket the tree programs' hash rows run on. The block's +/// WHIR base is pinned beside the family ([`BlockWhirHash`], [`BLOCK_BASE`]), +/// not in it: the family serves the LFM prover's two users (the block path and +/// the legacy users off it), the base only the block path. +/// +/// A compile-time choice, not a run-time one: there is ONE family per build +/// ([`BlockFamily`]), so no proof of the block path is ever made or checked +/// under a mix, and the move to BLAKE3 is a `Blake3Family` and a flip of that +/// alias, which the compiler follows to every site (D-P3B §2.9). Nothing here +/// reads the environment, a format field or a proof. +pub trait ProofFamily: Send + Sync + 'static { + /// The LFM proofs' STARK commitment configuration. + type Stark: stark::config::StarkHash; + /// The LFM proofs' Fiat–Shamir transcript OBJECT (see the module header + /// for why it is named apart from [`Self::Stark`]). + type Transcript: crypto::fiat_shamir::is_transcript::IsStarkTranscript< + crate::tables::types::GoldilocksExtension, + crate::tables::types::GoldilocksField, + > + Clone + + Send; + /// The family's name in every LFM statement: the statement's first + /// `append_bytes` call (`statement::absorb_lfm_statement_tagged`), so a + /// statement names the proof family it was made under. The legacy pin's + /// is `LAMBDAVM_LFM_STATEMENT_V1` itself, so its statements stay today's + /// bytes; another family's replaces it. + const STATEMENT_TAG: &'static [u8]; + /// The in-guest wrap hash a program verifying this family's proofs runs. + const WRAP: crate::lfm::edsl::WrapHash; + /// The machine's hash chip under that wrap hash. + const SOCKET: crate::lfm::hash::HasherKind; + /// A fresh transcript over `seed` (absorbed as its first `append_bytes`). + fn transcript(seed: &[u8]) -> Self::Transcript; +} + +/// The legacy pin as a family: [`BlockStarkHash`] on [`BlockTranscript`], no +/// statement tag — today's LFM proofs byte for byte. The LFM users off the +/// WHIR block path (continuation, the epoch tree, the registry and the probes) +/// stay here until the cleanup commit (D-P3B §2.8). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LegacyFamily; + +impl ProofFamily for LegacyFamily { + type Stark = BlockStarkHash; + type Transcript = BlockTranscript; + const STATEMENT_TAG: &'static [u8] = crate::lfm::statement::LFM_STATEMENT_TAG; + const WRAP: crate::lfm::edsl::WrapHash = crate::lfm::edsl::WrapHash::production(); + const SOCKET: crate::lfm::hash::HasherKind = BLOCK_HASHER; + fn transcript(seed: &[u8]) -> Self::Transcript { + block_transcript(seed) + } +} + +/// ZisK's Poseidon1 end to end: 4-ary trees capped at +/// [`crate::lfm::p1_commit::P1_STARK_CAP`], ZisK's sponge, width-8 grinding, +/// and the width-16 socket in the guest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct P1Family; + +impl ProofFamily for P1Family { + type Stark = crate::lfm::p1_commit::P1StarkHash; + type Transcript = crate::lfm::p1_commit::P1Transcript; + const STATEMENT_TAG: &'static [u8] = P1_LFM_STATEMENT_TAG; + const WRAP: crate::lfm::edsl::WrapHash = crate::lfm::edsl::WrapHash::Poseidon1; + const SOCKET: crate::lfm::hash::HasherKind = crate::lfm::hash::HasherKind::Poseidon1W16; + fn transcript(seed: &[u8]) -> Self::Transcript { + crate::lfm::p1_commit::P1Transcript::with_seed(seed) + } +} + +/// [`P1Family`]'s statement tag (D-P3B §9, shared with #1013): today's +/// `LAMBDAVM_LFM_STATEMENT_V1`, the family and its cap height, `C` + +/// [`crate::lfm::p1_commit::P1_STARK_CAP`] — absorbed in place of today's tag. +pub const P1_LFM_STATEMENT_TAG: &[u8] = b"LAMBDAVM_LFM_STATEMENT_V1/P1W16/C2"; + +// The tag's cap digit is the configuration's. +const _: () = assert!( + P1_LFM_STATEMENT_TAG[P1_LFM_STATEMENT_TAG.len() - 1] + == b'0' + crate::lfm::p1_commit::P1_STARK_CAP +); + +/// ★★★ THE SWITCH: the proof family of the no-epoch WHIR block's recursion. +/// Fixed at compile time: nothing selects it at run time, and nothing but the +/// LFM prover's off-path users (continuation, the epoch tree, the registry, +/// the probes: [`LegacyFamily`]) names another family until the cleanup +/// commit collapses the two. +pub type BlockFamily = P1Family; + +/// ★★★ The no-epoch block's WHIR base configuration: ZisK's Poseidon1 WHIR, +/// the hash [`BlockFamily`]'s recursion is made in — one hash end to end, so +/// no base of one hash ever sits under another's recursion. The block path +/// binds it (`with_block_hash!`) after `block_whir::checked_base` has +/// admitted the format's base, which must be [`BLOCK_BASE`]. +pub type BlockWhirHash = multilinear::whir_hash::P1Whir; + +/// The base format of every no-epoch block: [`BlockWhirHash`]'s hash at its +/// 4-ary cap (a WHIR geometry, the base's own, separate from the LFM trees' +/// `P1_STARK_CAP`). `ZfFormat`'s default names it, and the block path admits +/// no other. +pub const BLOCK_BASE: stark::proof::options::BaseFormat = + stark::proof::options::BaseFormat::P1_WHIR; + +// The base and the recursion are one hash, and the base's configuration is the +// base format's hash: the device key `BlockWhirHash` dispatches on names the +// same hash as `BLOCK_BASE`. +const _: () = { + assert!( + BLOCK_BASE.hash as u8 + == <::Stark as stark::config::StarkHash>::COMMITMENT_HASH + as u8 + ); + let device = match ::DEVICE { + multilinear::whir_hash::DeviceHashKey::Keccak256 => { + stark::config::CommitmentHash::Keccak256 + } + multilinear::whir_hash::DeviceHashKey::Rpx256 => stark::config::CommitmentHash::Rpx256, + multilinear::whir_hash::DeviceHashKey::Poseidon1 => { + stark::config::CommitmentHash::Poseidon1 + } + }; + assert!(device as u8 == BLOCK_BASE.hash as u8); +}; + +/// The root of a static preprocessed table (`table`: `bitwise`, `keccak_rc`) +/// at a key with no blessed constant — the legacy pin's roots and the +/// Poseidon1 twins the block's family asks for are constants (the module +/// header's regenerations; the callers return those first): computed by +/// `compute` once per process for each (hash, blowup, coset, leaf layout) and +/// reused. The cap of an arity-4 configuration does not enter: it moves path +/// nodes, never a root. +pub fn static_root_under( + table: &'static str, + options: &stark::proof::options::ProofOptions, + layout: stark::leaf_layout::LeafLayout, + compute: impl FnOnce() -> stark::config::Commitment, +) -> stark::config::Commitment { + type Key = (&'static str, u8, u8, u64, usize); + static ROOTS: std::sync::Mutex> = + std::sync::Mutex::new(Vec::new()); + let key: Key = ( + table, + S::COMMITMENT_HASH as u8, + options.blowup_factor, + options.coset_offset, + layout.rows_per_leaf(), + ); + let cached = |roots: &[(Key, stark::config::Commitment)]| { + roots.iter().find(|(k, _)| *k == key).map(|(_, r)| *r) + }; + if let Some(root) = cached(&ROOTS.lock().unwrap_or_else(|e| e.into_inner())) { + return root; + } + // Computed outside the lock: a second thread may compute the same root + // meanwhile, and both get the same value. + let root = compute(); + let mut roots = ROOTS.lock().unwrap_or_else(|e| e.into_inner()); + if cached(&roots).is_none() { + roots.push((key, root)); + } + root +} + +#[cfg(test)] +mod tests { + use super::*; + // Named here rather than at module scope: the byte arm's `BlockTranscript` + // mentions the extension field and an algebraic arm's does not, so a + // module-scope import would be unused on one of the two. + use crate::tables::types::GoldilocksExtension as E; + use stark::config::StarkHash; + + /// ✓ The pin is COHERENT: the transcript object the block path builds sponges + /// on the same hash the commitment configuration names. + /// + /// ⚠ This is the half-flip guard, and it is a real one rather than a + /// tautology only because [`BlockTranscript`] is pinned separately — the two + /// names can disagree, which is exactly the failure this catches. It is + /// stated over `NAME` because that is the one thing both sides expose. + #[test] + fn the_transcript_and_the_commitment_configuration_name_one_hash() { + use crypto::fiat_shamir::transcript_hash::TranscriptHash; + + // The byte arm's object IS `DefaultTranscript`, so the + // agreement is by construction here and this test says so cheaply. On an + // algebraic branch the two are independent types and this becomes the + // check that matters. + let named = <::Transcript as TranscriptHash>::NAME; + assert!( + !named.is_empty(), + "a commitment configuration must name its Fiat-Shamir hash" + ); + } + + /// The families are coherent: each one's transcript object hashes with + /// its configuration's Fiat–Shamir hash, the legacy family is the pin, and + /// only a non-legacy family tags its statements. + #[test] + fn each_family_names_one_hash() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + use crypto::fiat_shamir::transcript_hash::TranscriptHash; + + assert_eq!( + <::Stark as StarkHash>::COMMITMENT_HASH, + BLOCK_COMMITMENT_HASH + ); + assert_eq!(LegacyFamily::STATEMENT_TAG, b"LAMBDAVM_LFM_STATEMENT_V1"); + assert_eq!( + <::Stark as StarkHash>::COMMITMENT_HASH, + stark::config::CommitmentHash::Poseidon1 + ); + assert_eq!( + <<::Stark as StarkHash>::Transcript as TranscriptHash>::NAME, + "poseidon1-w16" + ); + assert_eq!( + P1Family::STATEMENT_TAG, + b"LAMBDAVM_LFM_STATEMENT_V1/P1W16/C2" + ); + assert_eq!(P1Family::WRAP, crate::lfm::edsl::WrapHash::Poseidon1); + // The block's base is the family's hash: P1's WHIR at the base's cap. + assert_eq!( + ::NAME, + ::NAME + ); + assert_eq!(BLOCK_BASE, stark::proof::options::BaseFormat::P1_WHIR); + assert_eq!(P1Family::SOCKET, crate::lfm::hash::HasherKind::Poseidon1W16); + // The P1 transcript object is ZisK's sponge seeded like the legacy + // one (the seed is its first `append_bytes`). + let mut t = crate::lfm::p1_commit::P1Transcript::new(); + <_ as IsTranscript>::append_bytes(&mut t, b"seed"); + assert_eq!( + <_ as IsTranscript>::state(&P1Family::transcript(b"seed")), + <_ as IsTranscript>::state(&t) + ); + } + + /// ✓ A fresh transcript is deterministic in its seed — the property every + /// prove/verify pair depends on, and the one a mis-wired constructor breaks. + #[test] + fn a_seeded_transcript_is_a_function_of_its_seed() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let a = >::state(&block_transcript(b"seed-one")); + let b = >::state(&block_transcript(b"seed-one")); + let c = >::state(&block_transcript(b"seed-two")); + assert_eq!(a, b, "the same seed must give the same state"); + assert_ne!(a, c, "a different seed must give a different state"); + } +} diff --git a/prover/src/lfm/SOUNDNESS.md b/prover/src/lfm/SOUNDNESS.md new file mode 100644 index 000000000..600b01739 --- /dev/null +++ b/prover/src/lfm/SOUNDNESS.md @@ -0,0 +1,497 @@ +# LFM write-once memory: the soundness argument (Phase 0b) + +Status: **for review**. This is the document a reviewer is invited to reject. If the argument +below does not convince, the fallback is the VM's timestamped memory argument (known-sound, +priced: 37% of MEMW's columns are timestamps plus 8 `<`-lookups per row) or in-circuit +well-formedness checks (uniqueness, per-selector booleanity, mult bounds — a material repricing). + +## 1. Setting + +An LFM **program** is a set of *instruction column groups*: per-chip preprocessed matrices holding +addresses, opcode selectors and multiplicities. They are committed once (interpolate → LDE → +row-pair Merkle) and their roots are pinned in `LFM_REGISTRY`, a drift-tested Rust constant table. +At verify time the roots are resolved from the registry and the framework rejects any proof whose +preprocessed commitment differs (`verifier.rs` equality check; on the prover side a mismatching +trace fails with `PrecomputedCommitmentMismatch`). The **main** (witness) columns carry values +only. + +Memory is not a table. A cell is a word `(v0..v3)` at an address `a`; the producing instruction's +chip **sends** the token `(a, v0, v1, v2, v3)` on the `LfmMem` bus with multiplicity `mult(a)` — a +*preprocessed* column — and each consuming instruction's chip **receives** the same token once, +with multiplicity gated by its (also preprocessed) `is_real`/selector columns. The LogUp argument +the whole prover already runs enforces, per bus, with soundness error `O(D/|E|)` over the +challenges `z, α` (`E` = the degree-3 extension, `D` = total interaction count): + +> **(B) Balance.** The multiset of sent tokens with multiplicity equals the multiset of received +> tokens with multiplicity. + +### 1.1 The framework premises the machine inherits + +(B) is not a fact about the LFM prover; it is an assumption about the *outer* verifier that checks +the LFM proof (`lfm/proof.rs` → `Verifier::multi_verify_views`), which is ordinary, unmodified +framework code. Four of its checks are load-bearing here and the machine defends none of them +independently: **per-column opening-width pinning** (`trace_opening_widths_well_formed`), which +pins each query opening's precomputed/main/aux split against the AIR rather than only their sum; +**`ood_blocks_well_formed`**, which derives the OOD table's shape from the AIR; **the +precomputed-root equality check** cited above, which is what delivers §2's registry premise; and +**the composition part-count check**, which fixes `num_composition_parts` from the AIR's degree +bound instead of reading it off the proof. + +The width pin is the one worth spelling out, because it is what makes (B) hold with error +`O(D/|E|)` rather than not at all. Every LFM chip is `with_preprocessed` with a non-empty aux trace +whose root is absorbed *after* the LogUp challenges `z, α`. If only the sum of the opening widths is +pinned, a prover may declare one value column into the aux group and choose it with the challenges +in hand; the bus check is a single aggregate scalar equality per proof, so one free extension +element solves it for an arbitrary perturbation — and (B) is gone while §2's whole custody chain +still passes, because that chain binds the *addressing* columns and the break is in the value +columns' commitment timing. **The machine's soundness is therefore only as good as the base tree's +version of these checks**: on a base predating one of them the theorem of §3 has no (B) to consume +and a wrap proof certifies nothing, no matter how much of §2–§6 holds. Item 8 of §7 is the +reviewer's version of this. + +One more check on the same path binds the *timing* of every main trace's codeword. A committed main +trace is close to a list of codewords in the Johnson regime, and the LogUp pair `z, α` is drawn once +for all of a proof's tables while each table's DEEP point comes later, on its own fork; with only +roots absorbed before `z, α`, a prover could pick each table's codeword after seeing them, and the +one bus equation would range over the product of the lists. Every LFM proof is therefore made by +`multi_prove_bound` and verified by `multi_verify_views_bound` (`stark::commit_ood`): after the last +root one point `z0` is drawn outside every table's domains, every non-precomputed main column +answers at it, every answer is absorbed before `z, α`, and each table's DEEP polynomial carries its +answers. The proof file carries the answers beside the multi-table proof (`proof::LfmStarkProof`), +and the node's leg absorbs them in its Phase A and folds the same cells into each table's DEEP. +Item 12 of §7 is the reviewer's version of this. + +## 2. What is vouched, and by whom + +| obligation | enforced by | mechanism | +|---|---|---| +| per-op algebra (`out = a·b + c`, …) | AIR | transition constraints on value columns | +| bit booleanity where a witness bit exists | AIR | degree-2 constraint | +| selector sum-boolean per ALU row | AIR | degree-2 constraint (belt over suspenders) | +| token balance (B) | AIR | LogUp, framework-emitted | +| **(U) uniqueness** — no address written twice | registrar | admission validator check 1 | +| **(A) acyclicity** — operand addr < destination addr | registrar | check 2 (dense emission order gives it by construction; re-checked) | +| **(M) mult-equality** — `mult(a)` = number of emitted reads of `a` | registrar | check 3 | +| **(S) selector one-hot-ness** | registrar | check 4 | +| **(P) padding rows all-zero** (mult = 0, is_real = 0) | registrar | check 5 | +| arena discipline | registrar | check 6 | + +"Registrar" means: the release-mode validator (`lfm/validator.rs`) ran on this exact program +before its digest entered `LFM_REGISTRY`, and the proof's preprocessed roots equal the registry's. +The chain of custody is: validator ⇒ digest ⇒ registry constant ⇒ drift tests on every PR ⇒ +root-equality check at prove and verify time. **There is no runtime off-switch and there must +never be one; the registry check *is* the soundness argument's first premise.** + +This is the industry-standard trust shape for this machine class: SP1 v4 relies on the same +premise but checks it only in dev builds (its validator omits the double-write check entirely); +Risc0 makes it structural (write destinations are program text, so uniqueness is syntactic). We +run the full checklist, in release, at admission — strictly more than either reference. + +## 3. The claim + +> **Theorem.** Assume (U), (A), (M), (S), (P) hold for the program (registrar) and (B) holds for +> the proof (LogUp). Then in any accepted execution, every read of address `a` observes the unique +> value written at `a`. + +**Argument.** By (U) each address has at most one producing instruction, so "the write at `a`" is +well-defined; let `W(a) = (a, w0..w3)` be its token, sent with multiplicity `mult(a)` (the +preprocessed column — the prover cannot vary it). By (M), `mult(a)` equals the number of program +reads of `a`. By (P), padding rows contribute no tokens (their multiplicities are preprocessed +zeros). + +Consider the multiset equation (B) on `LfmMem`. Every receive token is generated by some real +instruction row whose address operand columns are preprocessed, so the *addresses and counts* of +all receives are program text; only the value lanes are witness. Fix an address `a`. The sends at +address `a` are exactly `mult(a)` copies of `W(a)` (one writer, (U)). The receives at address `a` +are exactly the program's reads of `a` — `mult(a)` of them, by (M) — each carrying the value lanes +the reading chip's row exhibits. Balance of the full multiset then forces the sub-multisets at +each address to match (tokens include the address, and the fingerprint separates distinct tuples +except with the LogUp soundness error), so each of the `mult(a)` receive tokens equals `W(a)`: +every read observes the written value. + +Two degenerate cases are closed by the remaining premises. If a value could feed its own +producing row (`a := f(a)`), the send and receive would cancel *within* the row for **any** value +— balance holds vacuously and the value is unconstrained. (A) excludes this: every operand +address is strictly below its destination, so the read-token's address refers to an +earlier-produced cell, and the dataflow relation is a DAG; induction over addresses in ascending +order grounds every value in constants, hints, or hash outputs. If selectors were not one-hot, one +row could emit tokens under two op semantics at once; (S) excludes it beyond the in-AIR +sum-boolean. + +## 4. What the AIR must still get right (per-chip obligations) + +The argument above reduces chip soundness to: *each chip's constraints must force the value lanes +of every token it sends to be the correct function of the value lanes of the tokens it receives, +on every row where its (preprocessed) multiplicities are nonzero.* Concretely: base ops constrain +lane 0 and send `(a, out, 0, 0, 0)` with constant-zero high lanes in the tuple (a base cell +cannot smuggle extension lanes); ext ops likewise pin lane 3 ≡ 0; `MulBase` additionally +constrains the shared B-columns to zero on its rows so the received token matches a base writer's; +`BitDec`'s canonicity gadget (`G/Z/GINV`) forces the 64 bit columns to recompose to the *canonical* +representative — without it, bits summing to `v + p` would satisfy the linear recomposition and +two distinct bit-vectors could both "be" `v`. + +## 5. Arenas + +`Hint` rows send unconstrained words into memory (their chip has no constraints by design). The +**arena rule** restores soundness at the program level: every arena-sourced value must be +transitively authenticated by a hash the machine itself performs (Merkle openings are absorbed +into hashed paths; anything transcript-derived is never hinted). This is a *program-review* +obligation, enforced at emitter review, exactly like the reference systems' hint discipline — the +machine-level theorem above is indifferent to hint values; it only guarantees reads see what was +hinted. + +## 6. Transcript replay (R1d) + +`edsl::TranscriptReplay` reproduces the production `DefaultTranscript` inside the machine. Three +things about it are worth a reviewer's attention. + +### 6.1 Absorbed data is hinted, and that is correct + +The replay absorbs arena-supplied words. That is not a breach of the arena rule (§5). The rule bans +*hinting a challenge*; it does not ban hinting the data a challenge is derived FROM — in +Fiat–Shamir that data is precisely the untrusted proof material, and binding it is the entire +point. Every challenge the machine uses is computed by `LFM_KECCAK` rows from the absorbed +segment, never read from an arena. The obligation that remains is the ordinary one: whatever is +absorbed must also be the thing the rest of the program checks against. + +### 6.2 The canonicity guard is a constraint, not a witness + +`sample_field_element` must reject candidates ≥ `p`. Since `p = (2^32 − 1)·2^32 + 1`, a candidate +`hi·2^32 + lo` with canonical `u32` halves is out of range **iff** `hi = 2^32 − 1 ∧ lo ≠ 0` — the +same predicate `BitDec` already uses for 64-bit canonicity (§4), over the same split. + +The guard emits one instruction: `div(lo, (2^32 − 1) − hi)`. `LFM_BALU` constrains division as +`SEL_DIV·(B·OUT − A) = 0`, so with `B = 0` it reads `A = 0` and leaves `OUT` free. The division is +therefore provable exactly when `hi ≠ 2^32 − 1` or `lo = 0`. Nothing is hinted and nothing needs +verifying — it is the same assert-via-division mechanism `assert_eq` is built from. (An earlier +plan used an `is_zero` gadget with a hinted-and-verified inverse; the division subsumes it.) + +`machine_tests::canonicity_guard_rejects_an_out_of_range_candidate_in_the_proof` exhibits a +coherent forgery at candidate `p` — every bus balances and the mul-add's own constraint is +satisfied — and confirms it is rejected; neutralising `emit_base(3, …)` makes that forgery +ACCEPTED, which is what pins the guard on this one constraint. + +### 6.3 Zero rejection: a completeness restriction, and why it is not a parameter + +The production sampler *loops* on an out-of-range candidate. The number of candidates a draw +consumes is therefore data-dependent, and so is every later draw's position in the output buffer. +A straight-line machine has exactly one shape, so it cannot follow that. The emitted program +encodes the **no-rejection schedule** and is unprovable for any transcript that ever rejects. + +This costs completeness only, never soundness. The emitted relation is a strict subset of the real +one: challenge values are pinned by constraints to the no-rejection schedule, so a transcript that +would have rejected yields *no* LFM proof rather than a wrong one. An honest prover sees it as a +loud `LfmExecError::DivByZero`, not a silent divergence. + +The bound. A candidate is uniform over `2^64` values and `2^64 − p = 2^32 − 1` of them are out of +range, so `q = (2^32 − 1)/2^64 ≈ 2^−32` per candidate. Only `sample_field_element` draws are +exposed: `sample_u64` at a power-of-two bound has `threshold = 0`, so it accepts its first +candidate unconditionally and contributes nothing. Every verifier challenge is a cubic-extension +element, i.e. three independent base draws, so with `E` extension draws the union bound gives + +> `P[the program cannot prove this proof] ≤ 3E · (2^32 − 1)/2^64` + +(`reject_probability_per_proof` in `transcript_replay.rs`, which takes the BASE draw count `3E`). + +The verified per-proof draw schedule, for a multi-proof over `T` tables, is + +> `E = 2` (LogUp `z, α` — shared transcript, drawn before the per-table forks) +> ` + 2` (bus-balance replay, on a forked transcript) +> ` + Σ_t (3 + L_t)` — per table `β`, `z_OOD`, `γ`, then `L_t` FRI fold challenges, +> with `L_t = max(log2(trace_length_t) − 7, 0)`, **independent of the blowup factor**. + +`β` and `γ` are one draw each no matter how many terms they batch (both expand to powers), which +is what keeps `E` small. At `T = 24` with tables at their row cap (`L_t = 12`), `E = 364`, so +`3E = 1,092` base candidates and `P ≈ 2.5·10^−7`. At a larger `T ≈ 60`, `E = 904` and +`P ≈ 6.3·10^−7`. + +`T = 24` was the epoch shape when this section was written: a real two-epoch continuation proof +gave 24 sub-proofs for an intermediate epoch and 25 for the final one, the extra being HALT. + +⚠ TWO CORRECTIONS, and the bound survives both. + +**`T` is now SMALLER, and the bound is monotone in `T`.** #977 (`892c7d1bc` on this lineage) took +`FIXED_TABLE_COUNT` 11 → 5 and made COMMIT, KECCAK, KECCAK_RND, ECSM, ECDAS and HINT counted, so +an epoch that never reaches one carries no sub-proof for it — `epoch_verify_tests`' +`SUB_PROOFS` measures 16 on the fibonacci fixture, down from 25. `E = 4 + Σ_t (3 + L_t)` is +increasing in `T`, and `P ≤ 3E · 2^−32` is increasing in `E`, so a smaller `T` only lowers `P`: +the `P ≈ 2.5·10^−7` quoted at `T = 24` remains a valid upper bound, now a conservative one. The +`T ≈ 60` figure is the side that matters for a real block and is untouched. + +**The cited test does not pin the number.** `machine_tests::arena_filler_reads_real_committed_roots` +asserts `tables > 0`, one main root per sub-proof, and that no root is all-zero — never a count. +So "measured, not assumed" was true of the run that was done and false of the suite: nothing +would have failed when `T` moved, and nothing did. The count that IS asserted is +`epoch_verify_tests`' `SUB_PROOFS`, on a different fixture (`epoch_tests::real_epoch`) than the +two-epoch continuation this paragraph describes. `T` for THAT fixture is unmeasured since #977; +it is bounded above by the 25/26 ceiling in +`tests::constraint_artifact_tests::continuation_epoch_constraint_leg`, which is what the bound +above should be read against until someone runs it. + +**State it as `< 10^−6` per proof at production shapes**, growing by `≈ 1.05·10^−8` per additional +table — each table contributes `3 + L_t ≈ 15` extension draws, so the per-table increment is 15× +the `≈ 7·10^−10` an individual extension draw costs. (Do not quote the per-draw figure as the +per-table one; the two differ by that factor of 15.) + +Headroom is large: 1% failure needs `≈ 4.3·10^7` base candidates (`≈ 2^25.4`) and 50% needs `2^31`, +four-plus orders of magnitude beyond any realistic verifier. Every figure above is pinned by +`machine_tests::zero_rejection_completeness_bound` rather than merely asserted here. + +**One host/machine divergence worth recording.** The verifier does not check that the +prover-supplied `trace_length` is a power of two. A malicious proof could therefore hand +`sample_u64` a non-power-of-two bound, making `threshold` nonzero and putting even a query draw on +the rejection path — the one circumstance in which a `u64` draw could matter to this bound. Such a +proof is simply unprovable in the machine, which is the safe direction (unprovable = rejected), but +it is a case where the host transcript and the emitted program diverge rather than agree. + +**Do not record this as "k-rejection is an emitter parameter later".** It is not. Supporting even +one rejection requires the downstream schedule to branch, which in a straight-line machine means +either a program per rejection pattern (`2^draws` of them, and program identity would become +proof-dependent, breaking the registry premise in §2) or a production transcript change to +constant-consumption sampling. The realistic route, if the bound ever stops being acceptable, is +the latter — make the production sampler consume a fixed number of candidates per draw — and that +is a change to `crypto`, not to this emitter. + +Timing note for whoever picks that up: the ecosystem hash migration already has to rebuild the +transcript (a field-native sponge replaces the keccak chain), and constant-consumption sampling is +a design constraint to carry into that rebuild rather than a separate migration. Fixing it there +costs nothing extra and removes this restriction for every future machine; retrofitting it onto the +current transcript would be a second proof-breaking change for no other benefit. + +### 6.4 The algebraic transcript: the rebuild §6.3 anticipated, and it removes that restriction + +§6.3's closing note said the ecosystem hash migration would have to rebuild the transcript as a +field-native sponge, and that constant-consumption sampling was a design constraint to carry into +that rebuild rather than a separate migration. That rebuild is +`lfm::algebraic_transcript::AlgebraicTranscript`, and the constraint was carried. + +For an algebraic configuration, `CANDIDATES_PER_COORDINATE = Some(1)` — **and it is guaranteed +rather than probabilistic.** A squeeze returns four felts that are canonical *by construction* +(they are field elements, not bytes reinterpreted), so the `u64`s carved out of their 32 canonical +bytes are always in range. There is no rejection to schedule around: not "rejects with probability +2^−32", but *cannot reject*. + +This is strictly stronger than the BLAKE3 configuration, which needs two candidates because its +digest bytes are arbitrary and a single miss has nowhere to go (`transcript_hash.rs`), and it is +stronger than keccak's `None`, which is the data-dependent loop §6.3 exists to talk about. + +**So §6.3's completeness restriction does not apply to an algebraic configuration.** The emitted +program's no-rejection schedule is not a subset of the real relation there — it *is* the real +relation, because the production sampler cannot take the other branch. The `q ≈ 2^−32` per-candidate +bound and the `2^draws` program-explosion argument are both about the byte configurations and +should not be quoted against an algebraic one. + +`algebraic_transcript::tests::the_host_transcript_and_the_machine_replay_derive_the_same_challenges` +is the differential that pins host and machine to the same challenge stream, under every tenant. +`sampling_is_constant_consumption` pins the one-cell-per-draw property directly. + +### 6.5 Grinding reuses the LEAF domain — a recorded weakening, and why it does not bite + +**What the socket separates.** `LFM_HASH` pins a per-mode capacity for `MODE_C` (Merkle parent), +`MODE_T` (transcript step) and `MODE_L` (leaf), so those three are different functions and a row +cannot claim a domain it does not carry (`chips::hash::emit_socket_prefix`; the AIR rejects a row +carrying another mode's capacity). + +**What grinding does.** An algebraic configuration's `StarkHash::Transcript` — the hash the +proof-of-work runs on — reuses the **leaf** construction and the leaf domain +(`algebraic_commit::AlgebraicDigest`), rather than taking a fourth domain of its own. + +**Why the machine cannot do otherwise, which is the actual reason.** The per-mode capacities are +pinned by the *preprocessed* mode selectors, so the only capacities an emitted program can produce +are the three the chip pins (plus whatever a `MODE_P` row carries as program data). A +grinding-specific domain would need a new mode, i.e. a change to the frozen `LFM_HASH` tuple +contract. Grinding hashes a byte string — `state ‖ nonce`, 40 bytes, five felts — which *is* data, +exactly what a leaf is, so the leaf construction is both the natural reading and the only one the +verifier can emit with an existing mode. This is a constraint, not a preference. + +**Why the reuse is not exploitable.** Domain separation matters where a verifier ACCEPTS a digest +the prover supplies, because there the prover chooses which domain's output to present. Grinding +is not that: **the verifier RECOMPUTES the grinding digest itself, over a preimage the protocol +fixes** — the transcript state at that point, concatenated with the nonce — and then tests its +leading zeros. The prover supplies only the nonce, and every other input is already bound. There +is therefore **no substitution surface**: a leaf digest cannot be presented in place of a grinding +digest, whatever it might collide with, because no digest is presented at all. + +⚠ Recorded as a weakening nonetheless, because it *is* one relative to the socket's design intent +— three domains carry four uses — and because the argument above depends on grinding staying a +recompute-and-compare check. **If a future change ever has a verifier accept a grinding digest +rather than recompute it, this subsection is the one that has to be revisited**, and at that point +the fix is a fourth mode rather than a fourth constant. + +### 6.6 The APPEND CALL BOUNDARY is part of the message + +**The rule.** A machine-side transcript replay must reproduce the host's `append_bytes` calls +**one for one** — same number of calls, same split — not merely the concatenation of their bytes. + +**Why the byte arm hides it.** A byte transcript absorbs into one flat segment: the sponge sees a +byte stream, so splitting one 64-byte absorb into two 32-byte ones, or coalescing two into one, is +invisible. Emitters written against a byte hash therefore coalesce freely, and nothing in the code +records that the boundary ever mattered. An **algebraic** transcript length-prefixes every +`append_bytes` call, so the boundary is *in* the message: a coalesced replay absorbs a different +message and derives different challenges. + +**How it fails.** Not as a wrong answer. The replay's challenge diverges from the host's, the leg +then inverts a difference that should have been non-zero, and the executor reports `DivByZero` at +some address — thousands of instructions from the cause, naming nothing. ✓ VERIFIED: six live +instances were found this way on the migration, in the LFM statement, the global statement and the +per-leg absorbs; each was a coalescing that was correct under BLAKE3. + +⚖ ASSESSMENT — **what class this is.** The length prefix is a soundness feature *of the algebraic +transcript*: it makes the absorbed stream unambiguous, so a prover cannot re-split a message to +land on another message's state. A replay that coalesces does not break that — it absorbs a +different, still-unambiguous message — so the defect presents as **completeness** (the wrap program +cannot prove) rather than as a forgery. It is recorded here because the *rule* is what keeps it +that way: the moment a replay is allowed to differ from the host by "the same bytes, different +calls", the transcript's injectivity stops being the thing the two sides agree on. + +**The general form, which is what to carry forward.** *When the byte hash makes a distinction +irrelevant, the code stops expressing it, and the algebraic hash needs it back.* ✓ VERIFIED three +instances of that shape: this one; the Fiat–Shamir transcript OBJECT being separable from the +commitment configuration (§6.7, axis 2); and the `LFM_HASH` socket permutation being consulted at +all (§6.7, axis 3). + +### 6.7 THE HASH IS NAMED, NEVER IMPLIED — three axes, and two deliberate carve-outs + +**The rule.** Every prove, verify, execute and commitment-building call on the block path names the +pin (`hash_pin::BlockStarkHash`, `hash_pin::BlockTranscript` / `block_transcript`, +`hash_pin::BLOCK_HASHER`). None reaches a workspace default alias. + +**Three orthogonal axes.** They must agree and nothing in the type system makes them: + +1. **The commitment configuration** — what the HOST commits under (`BlockStarkHash`). Reaching + `stark::config::DefaultStarkHash`, or the `stark::prover::Prover` / `stark::verifier::Verifier` + aliases (which are `GenericProver`/`GenericVerifier` *at* that default), pins BLAKE3 whatever + `H` the surrounding code passes. +2. **The Fiat–Shamir transcript OBJECT** — built by the caller and handed to `multi_prove`, so its + type is not forced to match (1). ⚠ **This is the dangerous axis**: a half-flip is + self-consistent between prover and verifier and therefore **silent**, where (1) and (3) fail + loudly. +3. **The `LFM_HASH` socket permutation** — which permutation the MACHINE's `Instr::Hash` rows + compute, passed per call to `execute` / `lfm_prove_with_hasher`. ★ Under a byte hash the + emitter's Merkle work lowers to the dedicated KECCAK / `LFM_BLAKE3` chips and emits no + `Instr::Hash` at all, so this argument is never consulted and a toy permutation is free and + correct — which is exactly why it stayed unnamed until an algebraic pin made it load-bearing. + +✓ VERIFIED: eleven sites were carrying an implied hash and were closed by enumerating every call +site rather than by search; two of them were on axis 3. + +⛔ **TWO DELIBERATE CARVE-OUTS. Do not "fix" these — they are pinned KECCAK on every arm.** + +- `programs::emit_program_id` and its host counterpart `recursion::program_id_from_digest`. A + `program_id` identifies a program to CONSUMERS; it is not part of the proof system's commitment + layer. Following the configured hash would make the attestation join disagree with every host + consumer of a `program_id`, and the disagreement would surface as a consumer-side compare + failing rather than as an unprovable program. ✓ VERIFIED both name their hash explicitly. +- `statement::elf_digest` (`prover/src/statement.rs:30`), same class: it names `Keccak256` + directly. ✓ VERIFIED. + +⚠ A naive sweep — "replace every `edsl::keccak256`" — produces a wrong proof at exactly these two +sites, because grinding (`epoch::emit_grinding_check`) shares the same `ByteString` type and DOES +follow the configuration. + +**A consequence for widths.** Because the id stays a two-cell keccak digest while a commitment root +becomes one algebraic cell, a schema holding both carries two different root widths. Anything that +counts published words must read `proof_arena::lanes_per_root` for a COMMITMENT and keep the +literal two for the ID. `epoch_tests::the_arena_writer_and_the_machine_reader_agree_on_a_roots_width` +gates the first; the second is stated at `aggregator_tests::WrapPublicLayout`. + +### 6.8 A FALSE DOC CLAIM RECRUITS THE NEXT CALLER — treat it as a defect, not a typo + +**The rule.** A helper whose doc asserts a capability its body does not have is a defect of the +same class as the code errors above, and is worse than no doc at all: no doc makes a caller read +the body, a wrong doc makes them skip it. + +✓ VERIFIED two instances on this migration, both structurally unreachable at the time and both +therefore invisible to every test: + +- `edsl::digest_halves` — "Hash-agnostic: it is two `Unpack`s." It indexes `d[1]`, and an + algebraic `WrapDigest` is ONE cell whose second slot REPEATS the first + (`WrapDigest::from_cell`), so on that arm it would have returned `lo ‖ lo`: eight plausible + halves, four of them a duplicate, with nothing to notice. Now carries an emit-time width assert. +- `edsl::parent_stream` — "Hash-agnostic … under either hash", on the one function that calls + `digest_halves`. Two false claims on a helper pair, which is the ordinary way this spreads. + +⚖ ASSESSMENT of severity: neither was reachable, so neither was a live bug — and that is exactly +what makes the class worth naming. Unreachable-but-wrong survives every test suite indefinitely +and is discharged only when a future caller takes the claim at face value, at which point it is a +live bug in someone else's change. + +**The cheap instrument.** Grep the hash-adjacent helpers for generality phrases — "hash-agnostic", +"either arm", "both hashes", "on both arms" — and check each against its body. That sweep found +the two above and confirmed six others sound. + +★ **What the sound ones have in common is the fix.** `ByteWrapHash::hash_bytes` says "Both hashes +take the SAME packing" and is correct, because it hangs on `ByteWrapHash`, a type whose only two +inhabitants ARE the byte hashes — the claim cannot over-reach because the type will not let it. +`epoch::RootCells::byte_halves` says "correct on both arms" and is correct because its body has an +explicit algebraic branch. `transcript_replay::sample` names the algebraic arm and explains why its +behaviour there is a DEFINITION rather than a fidelity claim. **So the durable remedy is the same +one the width defects want: put the distinction in a TYPE, and the doc cannot lie about it.** + +## 7. Reviewer checklist (reject if any fails) + +0. For an algebraic configuration: is grinding still a RECOMPUTE-and-compare check (§6.5)? If a + verifier ever accepts a prover-supplied grinding digest, the leaf-domain reuse stops being + safe and needs a fourth socket mode. +1. Is the validator actually on the only path into `LFM_REGISTRY`, in release builds, with no + env-var or feature bypass? +2. Do the drift tests pin the registry on every PR (not merge-queue-only)? +3. Does every chip keep the one sign convention (writes = senders `Column(mult)`, reads = + receivers `Column(is_real)`, no `Negated` forms)? +4. Are all address/selector/mult columns actually in the preprocessed group of every chip + (no witness-supplied addressing anywhere)? +5. Does the BitDec canonicity constraint cover the full 64-bit range for `p = 2^64 − 2^32 + 1` + (top-32-all-ones ⇒ bottom-32-zero)? +6. Is the LogUp soundness error budget (`O(D/|E|)`, `|E| ≈ 2^192`) acceptable at the machine's + interaction counts (≤ 2^25 per epoch)? +7. Does every `TranscriptReplay` challenge reach the program through machine keccak rows rather + than an arena, and is the zero-rejection completeness bound (§6.3) acceptable at this + program's actual draw count? +8. Is the base tree at or past every framework verifier fix the inherited premises of §1.1 name — + currently per-column opening-width pinning (`trace_opening_widths_well_formed`, #909 / + `6949ceb9`)? On an older base (B) is not delivered, and nothing below §1.1 can recover it. +9. For an algebraic configuration: does every machine-side transcript replay match the host's + `append_bytes` calls ONE FOR ONE, not just byte for byte (§6.6)? A coalesced absorb is correct + under a byte hash and derives a different challenge under an algebraic one. +10. Does every prove / verify / execute call on the block path NAME the pin rather than reach a + default alias, on all three axes (§6.7)? And are the two keccak carve-outs — `program_id` and + `elf_digest` — still pinned rather than following the configuration? +11. Does every hash-adjacent helper's doc match its body (§6.8)? A "hash-agnostic" claim on a + function that indexes a second digest cell is a defect even when nothing reaches it. +12. Does every LFM prove go through `multi_prove_bound`, and does every LFM verify — the host + verifier, its separate Phase-A replay for the public balance, the tree's harvest, and the + node's leg — draw `z0` after the last root and absorb every commitment-time answer before + `z, α` (§1.1)? Is the answer count each verifier checks the AIR's, never the proof's, and does + the leg's DEEP read the cells its Phase A absorbed rather than a second hint? + +## 8. Proven bits at the aggregation-era presets (the η re-tune, applied) + +Every security claim in this stack is quoted in PROVEN bits under the +Johnson/proximity-gaps bound (1 − √ρ regime) — never the capacity conjecture. +Two consequences of the adopted η re-tune, both derivation-level (no runtime +parameter moved): + +1. **Per-table / per-proof base.** At blowup 4 / 110 queries the presets' + proven soundness re-derives from ≈ 95.5 bits to ≈ 114–117 bits at the SAME + query count: the slack parameter η in the Johnson-bound proximity term was + set conservatively, and re-optimizing it against the actual domain sizes + tightens bits-per-query with zero prover or verifier cost. The security + audit of 2026-08-15 carries the derivation; the numbers here are its + published range. + +2. **The batched class-split floor tracks the re-tune 1:1.** The batched + format's additional loss (≈ 3.3 bits of class split inside the measured + ≈ 92.2-bit floor against the ≈ 95.5 per-table base) comes from the |D₀|² + union lift in ε_C — a term in DOMAIN SIZE, independent of η. An additive + bits penalty that does not contain η moves with the base unchanged: + post-re-tune the batched floor is ≈ (114–117) − 3.3 ≈ 111–114 proven bits. + This settles the design review's unresolved point ("does the batched floor + move with η") in the affirmative, from the ε_C derivation's own structure. + +Query headroom above the presets is priced in the aggregation census's query +sweep (blowup 4, terminal 2^8): every program boundary — the wrap's 2^18, the +global wrap's 2^20, the aggregator's 2^21 — holds through q = 121 +(≈ 125–128 proven bits post-re-tune); q ≥ 124 additionally requires the +global proof's terminal at 2^8 to keep the global wrap under its boundary. +Raising q is a preset decision with registry consequences (new program +identities), not part of this note. diff --git a/prover/src/lfm/airs.rs b/prover/src/lfm/airs.rs new file mode 100644 index 000000000..dd7814432 --- /dev/null +++ b/prover/src/lfm/airs.rs @@ -0,0 +1,1478 @@ +//! `LfmAirs` — the machine's fixed 15-chip AIR set, a sibling of `VmAirs`. +//! +//! The chip set never varies; only heights do (per program). Programs are +//! supplied preprocessed roots (resolved from `LFM_REGISTRY` at verify time), +//! so constructing the verify-side AIR set costs nothing — there is no keygen +//! in this framework. Proved and verified by the same generic +//! `multi_prove` / `multi_verify_views` machinery as the RV64 VM; **zero +//! `VmAirs` edits** — the sibling-AIR-set property, preserved deliberately. + +use stark::config::Commitment; +use stark::constraints::builder::{ConstraintSet, EmptyConstraints}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, NullBoundaryConstraintBuilder, +}; +use stark::proof::options::ProofOptions; +use stark::trace::TraceTable; +use stark::traits::AIR; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +use crate::tables::{bitwise, keccak_rc, keccak_rnd}; + +use super::blake3_chip; +use super::chips::{balu, bitdec, const_, hash, hint, keccak, lanes, public, range, select, xalu}; +use super::hash::HasherKind; +use super::layout; +use super::merge::{LfmMerge, MergePlan, MergedConstraints, MergedLayout, SlotRole}; +use super::trace::LfmTraces; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +pub type LfmAir = AirWithBuses; +pub type DynLfmAir<'a> = &'a dyn AIR; + +/// The frozen chip order — everywhere: roots, digests, traces, proofs. +/// +/// Slots 12–14 are the production keccak family, hosted unchanged. They belong +/// to the *fixed* machine, so **every** LFM proof carries them — including the +/// 2^20-row BITWISE table, which costs a few seconds of prove time even for a +/// program containing no keccak at all. That is the deliberate price of the +/// fixed-machine principle: the chip set never varies with the program, only +/// heights do, so a program stays nothing but a vector of preprocessed roots +/// plus a registry entry. Making the set program-dependent would move shape +/// negotiation onto the verify path, which this design refuses. +/// +/// This is the count of chip *classes*, and the width of the roots and +/// log-heights arrays. `KECCAK_RND` (slot 12) and `LFM_BLAKE3` (slot 11) may +/// each be instantiated more than once — see [`num_lfm_airs`] — but their chunk +/// counts are program shape read from the registry, not shape negotiated on the +/// verify path, so the principle above holds. +pub const NUM_LFM_CHIPS: usize = 15; +pub const LFM_CHIP_NAMES: [&str; NUM_LFM_CHIPS] = [ + "LFM_CONST", + "LFM_BALU", + "LFM_XALU", + "LFM_SELECT", + "LFM_BITDEC", + "LFM_HASH", + "LFM_KECCAK", + "LFM_LANES", + "LFM_HINT", + "LFM_PUBLIC", + "LFM_RANGE", + "LFM_BLAKE3", + "KECCAK_RND", + "KECCAK_RC", + "BITWISE", +]; + +/// Slot of `KECCAK_RND`, the one AIR in the set with **no** preprocessed +/// columns — it has no root to supply, pin, or bind into the program digest. +/// It is also the one slot that expands into several AIR instances; the +/// chunks sit contiguously at 12.., so `KECCAK_RC` and `BITWISE` follow them +/// in the AIR list while keeping chip-class indices 13 and 14 in the roots +/// and log-heights arrays. +/// +/// `LFM_BLAKE3` was placed at 11 — last of the LFM chips, before the hosted +/// family — rather than appended at 14, so this constant stays the boundary +/// between "chips this machine owns" and "tables it hosts". Appending would +/// have left the count arithmetic below untouched at the cost of putting a +/// program-dependent group after two fixed tables, which is the shape every +/// index expression here is written against. +pub const KECCAK_RND_SLOT: usize = 12; + +/// The hash-family slots [`ChipSet`] gates. `BITWISE` (14) is deliberately not +/// among them: both families send to its `ByteAlu`/`AreBytes` buses, so it is +/// shared infrastructure rather than either family's chip. +/// +/// `BLAKE3_SLOT` is the machine's SECOND splittable slot +/// ([`super::chunking::Blake3Chunking`]): its chunks sit contiguously at 11.., +/// before `KECCAK_RND`'s at 12+chunks... , so the frozen order is the class +/// order with each splittable class expanded in place. Unlike `KECCAK_RND`, the +/// chip has preprocessed columns, so chunk 0's root is the roots array's slot-11 +/// entry and the rest ride +/// [`LfmArtifacts`](super::registry::LfmArtifacts::blake3_chunk_roots). +/// Slot of `LFM_HASH`, the class the hash split may divide in two +/// ([`super::chunking::HashChunking`]): chunk 0 is this slot's AIR, root and +/// height, and chunk 1 follows it in the AIR order. +pub const HASH_SLOT: usize = 5; +pub const KECCAK_SLOT: usize = 6; +pub const BLAKE3_SLOT: usize = 11; +pub const KECCAK_RC_SLOT: usize = 13; +/// Slot of `BITWISE`, the one class [`ChipSet::bitwise`] gates. +pub const BITWISE_SLOT: usize = 14; + +/// AIR instances (and sub-proofs) in a proof whose `KECCAK_RND` is split into +/// `keccak_rnd_chunks` instances and whose `LFM_BLAKE3` is split into +/// `blake3_chunks`. +pub const fn num_lfm_airs(keccak_rnd_chunks: usize, blake3_chunks: usize) -> usize { + ChipSet::FULL.num_airs(keccak_rnd_chunks, blake3_chunks) +} + +/// ★ Which hash-family chip groups a program instantiates. +/// +/// The machine hosts two hash families, and a program uses at most one of them. +/// Carrying the other is a whole AIR per chip — commitment, Merkle tree, +/// opening, FRI — for zero rows. On a real block that bill was measured: the +/// always-on BLAKE3 RV64 table cost the recursion wrap +31.2% of its trace +/// cells and +44% of its peak memory while carrying 0.035% of the epoch's own +/// cells (`thoughts/shared/block-compression/WRAP-GROWTH-BISECT.md`). Whichever +/// hash is not in use should not be paying for a chip group — in **both** +/// directions, which is why this is one mask and not two special cases. +/// +/// ## Scope: the hash families only +/// +/// Deliberately NOT "drop any chip with no rows". Most programs leave several +/// chips at the four-row floor (`TrivialV0` leaves eight), and dropping those +/// would dissolve the fixed-machine property for every program instead of +/// retiring two mutually-exclusive families. `BITWISE` is excluded for a +/// second reason: both families send to its `ByteAlu`/`AreBytes` buses, so it +/// is shared infrastructure, not part of either family. +/// +/// ## Why this does not negotiate shape on the verify path +/// +/// `airs.rs`' header refuses a program-dependent chip set *negotiated from the +/// proof*, and names the exception: shape "read from the registry, not +/// negotiated on the verify path" is legitimate, which is how `KECCAK_RND`'s +/// chunk count already works. This is that. The mask is computed from the +/// compiled program at bless time, stored in the registry entry, and folded +/// into `program_id` via [`Self::as_tag`] — so a verifier takes it from the +/// entry it resolved, never from the prover. There is no prover assertion here +/// to attack, which makes this a strictly stronger position than the RV64 +/// side's conditional BLAKE3 table. +/// +/// ## What decides it +/// +/// A program's own compiled groups. `WrapHash::production()` is the upstream +/// cause — it is what makes an authenticating program's keccak group empty +/// once the commitment hash is BLAKE3 — but it is not the predicate: programs +/// that are ABOUT a hash (`KeccakChainV0`, `KeccakSpongeV0`) name keccak +/// directly and keep the keccak family no matter what the production hash is. +/// Keying off the global would have silently broken exactly those. +/// +/// ## `BITWISE` +/// +/// [`Self::bitwise`] is the third bit, and it is NOT a family: it is on exactly +/// when some chip this mask instantiates has an interaction on a bus `BITWISE` +/// receives ([`Self::bitwise_required`]), or for every program under +/// [`KEEP_BITWISE_ENV`]. Derived from the chips' own interaction lists rather +/// than from the two family bits, because the hash chip is a `BITWISE` sender +/// under one hasher (`HasherKind::Blake3`) and not under the others. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ChipSet { + /// `LFM_KECCAK`, every `KECCAK_RND` chunk, and `KECCAK_RC`. + pub keccak: bool, + /// `LFM_BLAKE3`. + pub blake3: bool, + /// `BITWISE`, the fixed `2^20`-row byte-lookup table. Off for a program + /// whose chips send it nothing, unless [`KEEP_BITWISE_ENV`] keeps it. + pub bitwise: bool, +} + +/// ★ `LAMBDA_VM_LFM_KEEP_BITWISE=1` keeps `BITWISE` in every program this +/// process builds: the machine as it was before the table became conditional, +/// every tag, AIR list, trace and `program_id` byte for byte — the A/B arm and +/// the rollback switch. +/// +/// By default a program carries `BITWISE` only when one of its instantiated +/// chips sends it a lookup ([`ChipSet::bitwise_required`]). Every recursion +/// program of the WHIR pipeline sends none, and so do the nodes, the global +/// parent and the root of the STARK pipeline (its wraps keep the table: their +/// `program_id` fold is a keccak permutation). Each of those proves one +/// sub-proof fewer and its parent re-verifies one fewer — a `2^20`-row table, +/// 26.2 M census cells a proof. Measured on block 25368371 (one RTX 5090, +/// ABBA): WHIR −4.70 s, STARK −5.30 s. +/// +/// Read ONCE per process, at program-emission time, like the chunking +/// policies: the mask is stored in the artifacts and folded into `program_id`, +/// and every verifier takes it from there, never from a proof. Registry rows +/// carry the mask they were blessed with (the default), whatever this process +/// reads. Proving does not depend on the setting either: a trace set whose +/// builder dropped the table gets it back, all zero, when the artifacts keep +/// it ([`LfmAirs::air_trace_pairs`]). +/// +/// Why dropping is sound: `BITWISE` only RECEIVES, with prover-chosen +/// multiplicity columns. With no sender in the proof, the honest multiplicities +/// are all zero and the table contributes nothing to the LogUp sum, so its +/// presence constrains nothing its absence leaves open. What must never happen +/// is the other direction — a sender left without its receiver — and +/// [`ChipSet::bitwise_required`] is the predicate that keeps it unreachable: it +/// reads every instantiated chip's interactions, and the verifier re-checks it +/// against the mask it was handed (`proof::verify_against_chunked_with`). +pub const KEEP_BITWISE_ENV: &str = "LAMBDA_VM_LFM_KEEP_BITWISE"; + +/// [`KEEP_BITWISE_ENV`]'s reading of a raw value: unset keeps the default +/// (drop the table where nothing sends to it). +pub fn keep_bitwise_setting(raw: Option<&str>) -> bool { + env_switch(KEEP_BITWISE_ENV, raw).unwrap_or(false) +} + +/// Whether this process keeps `BITWISE` in every program it builds +/// ([`KEEP_BITWISE_ENV`]). Read once; prints the setting either way — to stderr, +/// so a generator's stdout stays its output — and a log states which programs +/// it proved. +pub fn keep_bitwise() -> bool { + static KEEP: std::sync::OnceLock = std::sync::OnceLock::new(); + *KEEP.get_or_init(|| { + let keep = keep_bitwise_setting(std::env::var(KEEP_BITWISE_ENV).ok().as_deref()); + announce(&if keep { + format!("LFM BITWISE: kept in every program ({KEEP_BITWISE_ENV}=1)") + } else { + "LFM BITWISE: only where a chip sends it a lookup (the default)".to_string() + }); + keep + }) +} + +/// Writes a setting's one-line banner to stderr in ONE write. +/// +/// `eprintln!` writes each piece of its format separately to the unbuffered +/// stderr, so in a log that merges stdout and stderr a whole stdout line can +/// land between two pieces and gain the banner's first half as a prefix. A log +/// reader that finds a census line by its start would miss it. One write of the +/// finished line cannot be split. +pub fn announce(line: &str) { + use std::io::Write; + let _ = std::io::stderr().write_all(format!("{line}\n").as_bytes()); +} + +/// A `LAMBDA_VM_*` on/off switch's value: unset or empty is `None` (the +/// switch's default), `0` is off, `1` is on, and anything else stops the run — +/// a typo read as the default would prove one arm under the other's name. +pub fn env_switch(name: &str, raw: Option<&str>) -> Option { + match raw.map(str::trim) { + None | Some("") => None, + Some("0") => Some(false), + Some("1") => Some(true), + Some(other) => panic!("{name} must be 0 or 1, got {other:?}"), + } +} + +impl ChipSet { + /// Both families — the shape every program had before this was conditional. + pub const FULL: Self = Self { + keccak: true, + blake3: true, + bitwise: true, + }; + + /// The families a compiled program actually uses, for a program proved + /// under the default hasher. See [`Self::for_program_with_hasher`]. + pub fn for_program(program: &super::compiler::LfmProgram) -> Self { + Self::for_program_with_hasher(program, HasherKind::default()) + } + + /// The mask a compiled program proved under `hasher` gets: the families it + /// uses and whether it keeps `BITWISE` (see [`KEEP_BITWISE_ENV`]). + pub fn for_program_with_hasher( + program: &super::compiler::LfmProgram, + hasher: HasherKind, + ) -> Self { + Self::for_program_under(program, hasher, !keep_bitwise()) + } + + /// [`Self::for_program_with_hasher`] with the `BITWISE` setting supplied + /// rather than read from the environment, so a test can build both masks in + /// one process. + pub fn for_program_under( + program: &super::compiler::LfmProgram, + hasher: HasherKind, + drop_unused_bitwise: bool, + ) -> Self { + let families = Self { + keccak: program.groups.keccak.real_rows > 0, + blake3: program.groups.blake3.real_rows > 0, + bitwise: true, + }; + Self { + bitwise: !drop_unused_bitwise || families.bitwise_required(hasher), + ..families + } + } + + /// Whether a chip this mask instantiates, other than `BITWISE` itself, has + /// an interaction on a bus `BITWISE` receives — i.e. whether a proof + /// without `BITWISE` would leave a lookup with no receiver. + /// + /// Read off the interaction lists, not off the family bits: a chip that + /// gains a byte lookup tomorrow makes this true without anyone editing it. + pub fn bitwise_required(self, hasher: HasherKind) -> bool { + let received: Vec = bitwise::bus_interactions() + .iter() + .map(|i| i.bus_id) + .collect(); + self.instantiated_interactions(hasher) + .iter() + .any(|i| received.contains(&i.bus_id)) + } + + /// Every interaction of every chip class this mask instantiates, `BITWISE` + /// excluded. One `KECCAK_RND` instance stands for all its chunks: they are + /// the identical AIR. + fn instantiated_interactions(self, hasher: HasherKind) -> Vec { + let mut all = Vec::new(); + all.extend(const_::bus_interactions()); + all.extend(balu::bus_interactions()); + all.extend(xalu::bus_interactions()); + all.extend(select::bus_interactions()); + all.extend(bitdec::bus_interactions()); + all.extend(hash::bus_interactions(hasher)); + all.extend(lanes::bus_interactions()); + all.extend(hint::bus_interactions()); + all.extend(public::bus_interactions()); + all.extend(range::bus_interactions()); + if self.keccak { + all.extend(keccak::bus_interactions()); + all.extend(keccak_rnd::bus_interactions()); + all.extend(keccak_rc::bus_interactions()); + } + if self.blake3 { + all.extend(blake3_chip::bus_interactions()); + } + all + } + + /// Sub-proofs a proof under this mask carries. + pub const fn num_airs(self, keccak_rnd_chunks: usize, blake3_chunks: usize) -> usize { + // The classes no family owns: all 15 less KECCAK_RND (counted per + // chunk below), less LFM_KECCAK and KECCAK_RC (keccak's), less + // LFM_BLAKE3 (blake3's, counted per chunk below). BITWISE is among + // them and leaves when no instantiated chip sends it a lookup. + let mut n = NUM_LFM_CHIPS - 4; + if self.keccak { + n += 2 + keccak_rnd_chunks; + } + if self.blake3 { + n += blake3_chunks; + } + if !self.bitwise { + n -= 1; + } + n + } + + /// `KECCAK_RND` instances under this mask. Zero when the family is absent — + /// the chunking policy's own floor of one exists to keep an unused chip + /// present, which is the decision this reverses. + pub fn keccak_rnd_chunks(self, policy_chunks: usize) -> usize { + if self.keccak { policy_chunks } else { 0 } + } + + /// `LFM_BLAKE3` instances under this mask, on exactly the same terms. + pub fn blake3_chunks(self, policy_chunks: usize) -> usize { + if self.blake3 { policy_chunks } else { 0 } + } + + /// One byte, folded into `program_id`. The mask is program shape, so a + /// build under a different mask is a different program identity by name — + /// not merely by a root that happens to differ. + /// + /// `BITWISE` rides bit 2 INVERTED — set when the table is dropped — so every + /// mask that keeps it has the tag it had before the table became + /// conditional: a program that sends `BITWISE` lookups, and every program + /// under [`KEEP_BITWISE_ENV`], keeps its digest. + pub const fn as_tag(self) -> u8 { + (self.keccak as u8) | ((self.blake3 as u8) << 1) | ((!self.bitwise as u8) << 2) + } +} + +/// Permutations in each `KECCAK_RND` chunk, in chunk order. +pub fn keccak_rnd_chunk_permutations(program: &super::compiler::LfmProgram) -> Vec { + let total = program.groups.keccak.real_rows; + let per = program.chunking.permutations_per_chunk(); + (0..program.chunking.chunk_count(total)) + .map(|i| total.saturating_sub(i * per).min(per)) + .collect() +} + +/// Each `KECCAK_RND` chunk's trace height: 24 rows per permutation, padded — +/// the same `.next_power_of_two().max(4)` rule `generate_keccak_rnd_trace` +/// applies, now once per chunk. +pub fn keccak_rnd_chunk_rows(program: &super::compiler::LfmProgram) -> Vec { + keccak_rnd_chunk_permutations(program) + .into_iter() + .map(|perms| { + (perms * super::chunking::KECCAK_RND_ROWS_PER_PERMUTATION) + .next_power_of_two() + .max(4) + }) + .collect() +} + +/// Each `LFM_BLAKE3` chunk's trace height: one row per compression, padded to +/// the `padded_rows` rule the compiler applies to every group — now once per +/// chunk, which is exactly what +/// [`LfmProgram::blake3_chunk_group`](super::compiler::LfmProgram::blake3_chunk_group) +/// materializes. Arithmetic rather than a walk of the groups, so a census costs +/// nothing on a program whose chunks are millions of rows; +/// `the_blake3_chunk_arithmetic_is_the_group_split` pins the agreement. +pub fn blake3_chunk_rows(program: &super::compiler::LfmProgram) -> Vec { + program + .blake3_chunk_real_rows() + .into_iter() + .map(super::layout::padded_rows) + .collect() +} + +/// What sets a chip's trace height, and therefore whether the padding below its +/// next power-of-two step is a margin the workload can consume. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HeightRule { + /// `rows` is `real_rows.next_power_of_two()` and `real_rows` moves with the + /// workload. The only rule under which headroom is a cliff warning: this + /// chip's whole contribution doubles the moment the mix outgrows the gap. + Workload, + /// A lookup table at a compile-time constant height. Every row of it is + /// real, so it reports zero headroom — because it is full, not because it + /// is about to double. It cannot move with the workload at all. + Fixed, + /// One chunk of a split table. The chunking policy caps a chunk at or just + /// under a power of two (`KECCAK_RND`: 21,845 permutations = 524,280 of + /// 524,288 rows; `LFM_BLAKE3` at a `2^k` row cap: exactly full), so a full + /// chunk permanently reads ~0% headroom and yet can never cross one — the + /// policy emits another chunk instead. Watching these would be watching a + /// false alarm that is always on. + Chunked, +} + +/// One chip instance's trace geometry, as the cell instrument sees it. +/// +/// The per-chip decomposition of [`lfm_cell_counts`] — that function sums +/// exactly these rows, so a census and a total can never disagree. `name` is not +/// unique: every `KECCAK_RND` chunk reports under the same chip name, which is +/// the point (they are the same AIR at different heights). +#[derive(Clone, Copy, Debug)] +pub struct LfmChipCells { + pub name: &'static str, + /// Padded trace rows — the height the prover commits. + pub rows: u64, + /// Rows the workload actually occupies, before padding to `rows`. Only the + /// gap between the two is available to grow into; see + /// [`LfmChipCells::headroom`]. + pub real_rows: u64, + /// What sets this chip's height, and so whether its headroom is a margin + /// worth watching. See [`HeightRule`]. + pub height_rule: HeightRule, + /// Value columns: the AIR's width less its preprocessed prefix. + pub main_cols: usize, + /// Aux (LogUp) columns, one per pair of bus interactions. + pub aux_cols: usize, +} + +impl LfmChipCells { + pub fn main_cells(&self) -> u64 { + self.rows * self.main_cols as u64 + } + + pub fn aux_cells(&self) -> u64 { + self.rows * self.aux_cols as u64 + } + + /// Fraction of the committed height that is padding, i.e. how far this chip + /// is below the next power-of-two step. + /// + /// A chip's cells are a STEP function of its workload: `rows` is + /// `real_rows.next_power_of_two()`, so a chip sitting at 1% headroom doubles + /// its whole contribution the moment the workload grows 1%, while one at 45% + /// absorbs a near-doubling for free. That asymmetry is invisible in the row + /// count alone, and it is what made #903's four near-empty BLAKE3 rows cost + /// the wrap five simultaneous chip doublings — the campaign was standing + /// 1.2% under a step on `LFM_LANES` and nobody could see it + /// (`thoughts/shared/block-compression/WRAP-GROWTH-BISECT.md`). + /// + /// Only a cliff warning under [`HeightRule::Workload`]; see [`Self::at_risk`]. + pub fn headroom(&self) -> f64 { + if self.rows == 0 { + return 0.0; + } + (self.rows - self.real_rows) as f64 / self.rows as f64 + } + + /// Whether this chip's [`Self::headroom`] is a margin the workload can + /// actually consume, rather than a constant of the machine. + pub fn at_risk(&self) -> bool { + self.height_rule == HeightRule::Workload + } + + /// Base-field equivalents this chip would ADD by crossing its next step — + /// its height doubles, so it adds exactly what it already contributes. + /// An extension element is three base felts, matching the census total. + pub fn cliff_cost(&self) -> u64 { + self.main_cells() + 3 * self.aux_cells() + } +} + +/// Per-chip trace geometry for a compiled program, in the frozen AIR order +/// (`KECCAK_RND`'s chunks expanded, so the vector has one entry per sub-proof). +/// +/// Extracted from [`lfm_cell_counts`] rather than written beside it: a second +/// copy of this table is how a census would come to describe a different machine +/// than the one the totals describe. +pub fn lfm_chip_census(program: &super::compiler::LfmProgram) -> Vec { + lfm_chip_census_with_hasher(program, HasherKind::default()) +} + +/// One entry of [`lfm_chip_census_with_hasher`]'s per-chip table: a chip class's +/// two heights plus its width decomposition. +/// +/// Named rather than a positional tuple because `real_rows` and `padded_rows` +/// are interchangeable at a glance, and swapping them would invert every +/// headroom the panel prints while leaving the cell totals — the numbers under +/// test — correct. +struct ChipShape { + real_rows: u64, + padded_rows: u64, + height_rule: HeightRule, + num_cols: usize, + prep: usize, + interactions: usize, +} + +/// [`lfm_chip_census`] for a program proved under `hasher`. +/// +/// Only `LFM_HASH`'s width moves with the hasher; every other chip is +/// hash-independent, and the preprocessed prefix is the hasher-independent +/// instruction group, so the row counts and the roots do not move either. +pub fn lfm_chip_census_with_hasher( + program: &super::compiler::LfmProgram, + hasher: HasherKind, +) -> Vec { + lfm_chip_census_masked( + program, + hasher, + ChipSet::for_program_with_hasher(program, hasher), + ) +} + +/// [`lfm_chip_census_with_hasher`] under a supplied mask rather than the one +/// the process derives, so a test can take the census of both masks at once. +pub fn lfm_chip_census_masked( + program: &super::compiler::LfmProgram, + hasher: HasherKind, + chip_set: ChipSet, +) -> Vec { + let g = &program.groups; + // A workload-sized chip: the compiler already computed both heights, and + // `padded_rows` is `real_rows.next_power_of_two()`, so the gap between them + // is this chip's distance to its next doubling. + let sized = |group: &super::compiler::ColumnGroup, + num_cols: usize, + prep: usize, + interactions: usize| ChipShape { + real_rows: group.real_rows as u64, + padded_rows: group.padded_rows as u64, + height_rule: HeightRule::Workload, + num_cols, + prep, + interactions, + }; + // A lookup table: its height is a compile-time constant and every row of it + // is real, so it reports no headroom and is flagged as unable to move. + let lookup = |rows: u64, num_cols: usize, prep: usize, interactions: usize| ChipShape { + real_rows: rows, + padded_rows: rows, + height_rule: HeightRule::Fixed, + num_cols, + prep, + interactions, + }; + // Every chip class except `KECCAK_RND`, which is counted per chunk below. + let per_chip: [ChipShape; NUM_LFM_CHIPS - 1] = [ + sized( + &g.const_, + const_::cols::NUM_COLUMNS, + layout::const_::PREP_WIDTH, + const_::bus_interactions().len(), + ), + sized( + &g.balu, + balu::cols::NUM_COLUMNS, + layout::balu::PREP_WIDTH, + balu::bus_interactions().len(), + ), + sized( + &g.xalu, + xalu::cols::NUM_COLUMNS, + layout::xalu::PREP_WIDTH, + xalu::bus_interactions().len(), + ), + sized( + &g.select, + select::cols::NUM_COLUMNS, + layout::select::PREP_WIDTH, + select::bus_interactions().len(), + ), + sized( + &g.bitdec, + bitdec::cols::NUM_COLUMNS, + layout::bitdec::PREP_WIDTH, + bitdec::bus_interactions().len(), + ), + sized( + &g.hash, + hash::num_columns(hasher), + layout::hash::PREP_WIDTH, + hash::bus_interactions(hasher).len(), + ), + sized( + &g.keccak, + keccak::cols::NUM_COLUMNS, + layout::keccak::PREP_WIDTH, + keccak::bus_interactions().len(), + ), + sized( + &g.lanes, + lanes::cols::NUM_COLUMNS, + layout::lanes::PREP_WIDTH, + lanes::bus_interactions().len(), + ), + sized( + &g.hint, + hint::cols::NUM_COLUMNS, + layout::hint::PREP_WIDTH, + hint::bus_interactions().len(), + ), + sized( + &g.public, + public::cols::NUM_COLUMNS, + layout::public::PREP_WIDTH, + public::bus_interactions().len(), + ), + lookup( + layout::range::NUM_ROWS as u64, + range::cols::NUM_COLUMNS, + layout::range::PREP_WIDTH, + range::bus_interactions().len(), + ), + sized( + &g.blake3, + blake3_chip::cols::NUM_COLUMNS, + layout::blake3::PREP_WIDTH, + blake3_chip::bus_interactions().len(), + ), + // The keccak family's two fixed tables. `KECCAK_RND`'s chunks follow. + lookup( + keccak_rc::NUM_ROWS as u64, + keccak_rc::cols::NUM_COLUMNS, + keccak_rc::NUM_PRECOMPUTED_COLS, + keccak_rc::bus_interactions().len(), + ), + lookup( + bitwise::NUM_ROWS as u64, + bitwise::cols::NUM_COLUMNS, + bitwise::NUM_PRECOMPUTED_COLS, + bitwise::bus_interactions().len(), + ), + ]; + // The frozen AIR order is `air_refs`': chip classes 0..=11 (`LFM_BLAKE3` is + // the last of them), then every `KECCAK_RND` chunk, then `KECCAK_RC` and + // `BITWISE`. `per_chip` above lists the classes with the last two at the + // end, so the chunks are spliced in before them rather than appended. + let rnd_interactions = keccak_rnd::bus_interactions().len(); + // The census reports the chips the machine actually instantiates, so it + // gates on the same mask `air_refs` does — a census that counted an absent + // family would describe a different machine than the one being proved, + // which is precisely what this function's doc promises it cannot. + let mut census = Vec::with_capacity( + per_chip.len() + + program.chunking.chunk_count(g.keccak.real_rows) + + program.blake3_chunk_count(), + ); + for (slot, shape) in per_chip.into_iter().enumerate() { + if slot == KECCAK_RND_SLOT && chip_set.keccak { + for (perms, rows) in keccak_rnd_chunk_permutations(program) + .into_iter() + .zip(keccak_rnd_chunk_rows(program)) + { + census.push(LfmChipCells { + name: LFM_CHIP_NAMES[KECCAK_RND_SLOT], + rows: rows as u64, + // `keccak_rnd_chunk_rows` pads exactly this product, so the + // pair is the chunk's real-vs-committed height. + real_rows: (perms * super::chunking::KECCAK_RND_ROWS_PER_PERMUTATION) as u64, + height_rule: HeightRule::Chunked, + main_cols: keccak_rnd::cols::NUM_COLUMNS, + aux_cols: rnd_interactions.div_ceil(2), + }); + } + } + // `per_chip`'s last two entries are chip classes 13 and 14, which sit + // at indices 12 and 13 of that array — hence the shift past the + // `KECCAK_RND` slot rather than a plain index. + let class = if slot >= KECCAK_RND_SLOT { + slot + 1 + } else { + slot + }; + let present = match class { + KECCAK_SLOT | KECCAK_RC_SLOT => chip_set.keccak, + BLAKE3_SLOT => chip_set.blake3, + BITWISE_SLOT => chip_set.bitwise, + _ => true, + }; + if !present { + continue; + } + let ChipShape { + real_rows, + padded_rows, + height_rule, + num_cols, + prep, + interactions, + } = shape; + // `LFM_BLAKE3` expands in place, like `KECCAK_RND` above but at its own + // slot: the class is one entry per chunk, at the chunk's own height. + // A single-chunk program keeps the `Workload` rule — it IS the + // workload-sized table, and its headroom is a real cliff — while a split + // one reads `Chunked` for the reason [`HeightRule::Chunked`] gives. + if class == BLAKE3_SLOT { + let chunk_rows = program.blake3_chunk_real_rows(); + let rule = if chunk_rows.len() > 1 { + HeightRule::Chunked + } else { + height_rule + }; + for real in chunk_rows { + census.push(LfmChipCells { + name: LFM_CHIP_NAMES[BLAKE3_SLOT], + rows: super::layout::padded_rows(real) as u64, + real_rows: real as u64, + height_rule: rule, + main_cols: num_cols - prep, + aux_cols: interactions.div_ceil(2), + }); + } + continue; + } + // A split `LFM_HASH` is two entries, each at its own height. An + // unsplit one is the single workload-sized entry below, unchanged. + // Chunk 0 is exactly full by construction (`Chunked`); chunk 1 holds the + // remainder and its headroom is a real cliff (`Workload`). + if class == HASH_SLOT && program.hash_chunk_count() > 1 { + for (i, real) in program.hash_chunk_real_rows().into_iter().enumerate() { + census.push(LfmChipCells { + name: LFM_CHIP_NAMES[HASH_SLOT], + rows: super::layout::padded_rows(real) as u64, + real_rows: real as u64, + height_rule: if i == 0 { + HeightRule::Chunked + } else { + HeightRule::Workload + }, + main_cols: num_cols - prep, + aux_cols: interactions.div_ceil(2), + }); + } + continue; + } + census.push(LfmChipCells { + name: LFM_CHIP_NAMES[class], + rows: padded_rows, + real_rows, + height_rule, + main_cols: num_cols - prep, + aux_cols: interactions.div_ceil(2), + }); + } + census +} + +/// Trace-cell counts for a compiled program, the LFM analogue of the VM's +/// `total_field_elements` / `total_auxiliary_field_elements` (same +/// semantics: main counts base-field value cells excluding preprocessed +/// columns; aux counts extension-field elements, one per aux column per +/// row). This is the kill-risk-3 instrument: machine cells per verification +/// vs the verified proof's own cells. +pub fn lfm_cell_counts(program: &super::compiler::LfmProgram) -> (u64, u64) { + lfm_cell_counts_with_hasher(program, HasherKind::default()) +} + +/// [`lfm_cell_counts`] for a program proved under `hasher` — the hash matrix's +/// instrument. +pub fn lfm_cell_counts_with_hasher( + program: &super::compiler::LfmProgram, + hasher: HasherKind, +) -> (u64, u64) { + lfm_chip_census_with_hasher(program, hasher) + .iter() + .fold((0u64, 0u64), |(main, aux), c| { + (main + c.main_cells(), aux + c.aux_cells()) + }) +} + +pub struct LfmAirs { + const_: LfmAir, + balu: LfmAir, + xalu: LfmAir, + select: LfmAir, + bitdec: LfmAir, + hash: LfmAir, + /// `LFM_HASH` chunks 1.., each with its own preprocessed root — empty + /// for an unsplit program. Placed right after [`Self::hash`] in the frozen + /// order, as the `LFM_BLAKE3` chunks are expanded in place. + hash_tail: Vec>, + keccak: LfmAir, + lanes: LfmAir, + hint: LfmAir, + public: LfmAir, + range: LfmAir, + /// The BLAKE3 compression chip — a real constrained chip, unlike + /// `LFM_KECCAK`, which is an adapter that delegates its permutation to the + /// hosted family. Its AIR and its trace filler live in + /// [`super::blake3_chip`] rather than in `chips.rs` for that reason: there + /// is nothing here to adapt, only a chip to name. + /// + /// One instance per chunk (see [`super::chunking::Blake3Chunking`]). Unlike + /// `KECCAK_RND`'s vector below, the instances are NOT identical: each carries + /// its own chunk's preprocessed root, so they are built from the supplied + /// root slice rather than in a loop over one argument set. + blake3: Vec>, + /// One instance per `KECCAK_RND` chunk. Every instance is the identical + /// AIR — chunking changes only how many rows each one carries — so they + /// are built in a loop rather than named individually. + keccak_rnd: Vec>, + keccak_rc: LfmAir, + bitwise: LfmAir, + /// Which hash families this set instantiates. An unused family's AIRs may + /// still be BUILT (construction is free — there is no keygen here) but are + /// not offered to the prover or the verifier, so a proof never carries them. + chip_set: ChipSet, + /// The merged AIRs (`LAMBDA_VM_ZF_LFM_MERGE`), one per group of + /// [`Self::merge_plan`], in its order. Empty until [`Self::with_merge`] + /// adds them: one AIR a chip. + merged: Vec>, + /// Which chips each merged AIR holds. A merged chip's own AIR is still + /// built, like an absent family's, and never offered: the merged AIR stands + /// at its first chip's place in the frozen order and the rest leave it. + merge_plan: MergePlan, +} + +/// Builds an AIR with **no** preprocessed columns — `KECCAK_RND` only. +fn build_air_no_prep + 'static>( + num_columns: usize, + interactions: Vec, + options: &ProofOptions, + constraint_set: CS, + name: &'static str, +) -> LfmAir { + AirWithBuses::new( + num_columns, + AuxiliaryTraceBuildData { interactions }, + options, + 1, + constraint_set, + ) + .with_name(name) +} + +#[allow(clippy::too_many_arguments)] +fn build_air + 'static>( + num_columns: usize, + interactions: Vec, + options: &ProofOptions, + constraint_set: CS, + name: &'static str, + root: Commitment, + num_prep: usize, +) -> LfmAir { + AirWithBuses::new( + num_columns, + AuxiliaryTraceBuildData { interactions }, + options, + 1, + constraint_set, + ) + .with_name(name) + .with_preprocessed(root, num_prep) +} + +impl LfmAirs { + /// Builds the chip set against the supplied (registry-resolved or + /// freshly built) instruction-column-group roots, in the frozen order, + /// with `KECCAK_RND` instantiated `keccak_rnd_chunks` times and `LFM_BLAKE3` + /// once. + /// + /// A zero chunk count builds no `KECCAK_RND` at all; callers on the verify + /// path must reject that shape before getting here rather than relying on + /// the resulting AIR-count mismatch (`verify_against` does). + pub fn new( + roots: &[Commitment; NUM_LFM_CHIPS], + options: &ProofOptions, + keccak_rnd_chunks: usize, + chip_set: ChipSet, + ) -> Self { + Self::new_with_hasher( + roots, + options, + keccak_rnd_chunks, + HasherKind::default(), + chip_set, + ) + } + + /// [`LfmAirs::new`] with the `LFM_HASH` permutation chosen explicitly. + /// + /// The hasher is a construction-time property of the AIR set because the + /// chip bakes its round constants into its constraints: the same `hasher` + /// must reach execution and trace generation, which is what + /// `proof::lfm_prove_with_hasher` guarantees. Nothing else in the set moves + /// — the preprocessed prefix is the instruction column group, which no + /// hasher changes, so the preprocessed roots and the program digest are + /// hasher-independent. + pub fn new_with_hasher( + roots: &[Commitment; NUM_LFM_CHIPS], + options: &ProofOptions, + keccak_rnd_chunks: usize, + hasher: HasherKind, + chip_set: ChipSet, + ) -> Self { + // The unchunked `LFM_BLAKE3`: slot 11's root IS chunk 0's root, so a + // one-element window over the array is the whole of what a single-table + // program supplies. See [`Self::new_chunked`]. + Self::new_chunked( + roots, + &roots[BLAKE3_SLOT..=BLAKE3_SLOT], + options, + keccak_rnd_chunks, + hasher, + chip_set, + ) + } + + /// [`LfmAirs::new_with_hasher`] with `LFM_BLAKE3` instantiated once per + /// supplied chunk root. + /// + /// The general constructor. `blake3_roots` is a slice rather than a count + /// because the chunks are not interchangeable: each commits its own slice of + /// the instruction column group, so each AIR needs its own preprocessed + /// root. `blake3_roots[0]` must be `roots[BLAKE3_SLOT]` — the digest binds + /// chunk 0 through the roots array and the rest through + /// [`LfmArtifacts::blake3_chunk_roots`](super::registry::LfmArtifacts::blake3_chunk_roots) + /// — and an empty slice builds no instance at all, which is the shape an + /// absent BLAKE3 family has. + #[allow(clippy::too_many_arguments)] + pub fn new_chunked( + roots: &[Commitment; NUM_LFM_CHIPS], + blake3_roots: &[Commitment], + options: &ProofOptions, + keccak_rnd_chunks: usize, + hasher: HasherKind, + chip_set: ChipSet, + ) -> Self { + debug_assert!( + blake3_roots + .first() + .is_none_or(|first| *first == roots[BLAKE3_SLOT]), + "LFM_BLAKE3 chunk 0's root must be the roots array's slot-11 entry — \ + the digest binds chunk 0 there and the rest as a tail, so a pair that \ + disagrees describes two different programs" + ); + LfmAirs { + const_: build_air( + const_::cols::NUM_COLUMNS, + const_::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[0], + roots[0], + layout::const_::PREP_WIDTH, + ), + balu: build_air( + balu::cols::NUM_COLUMNS, + balu::bus_interactions(), + options, + balu::BaluConstraints, + LFM_CHIP_NAMES[1], + roots[1], + layout::balu::PREP_WIDTH, + ), + xalu: build_air( + xalu::cols::NUM_COLUMNS, + xalu::bus_interactions(), + options, + xalu::XaluConstraints, + LFM_CHIP_NAMES[2], + roots[2], + layout::xalu::PREP_WIDTH, + ), + select: build_air( + select::cols::NUM_COLUMNS, + select::bus_interactions(), + options, + select::SelectConstraints, + LFM_CHIP_NAMES[3], + roots[3], + layout::select::PREP_WIDTH, + ), + bitdec: build_air( + bitdec::cols::NUM_COLUMNS, + bitdec::bus_interactions(), + options, + bitdec::BitDecConstraints, + LFM_CHIP_NAMES[4], + roots[4], + layout::bitdec::PREP_WIDTH, + ), + hash: build_air( + hash::num_columns(hasher), + hash::bus_interactions(hasher), + options, + hash::HashConstraints { kind: hasher }, + LFM_CHIP_NAMES[5], + roots[5], + layout::hash::PREP_WIDTH, + ), + hash_tail: Vec::new(), + keccak: build_air( + keccak::cols::NUM_COLUMNS, + keccak::bus_interactions(), + options, + keccak::KeccakAdapterConstraints, + LFM_CHIP_NAMES[6], + roots[6], + layout::keccak::PREP_WIDTH, + ), + lanes: build_air( + lanes::cols::NUM_COLUMNS, + lanes::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[7], + roots[7], + layout::lanes::PREP_WIDTH, + ), + hint: build_air( + hint::cols::NUM_COLUMNS, + hint::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[8], + roots[8], + layout::hint::PREP_WIDTH, + ), + public: build_air( + public::cols::NUM_COLUMNS, + public::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[9], + roots[9], + layout::public::PREP_WIDTH, + ), + range: build_air( + range::cols::NUM_COLUMNS, + range::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[10], + roots[10], + layout::range::PREP_WIDTH, + ), + blake3: blake3_roots + .iter() + .map(|root| { + build_air( + blake3_chip::cols::NUM_COLUMNS, + blake3_chip::bus_interactions(), + options, + blake3_chip::Blake3LfmConstraints, + LFM_CHIP_NAMES[BLAKE3_SLOT], + *root, + layout::blake3::PREP_WIDTH, + ) + }) + .collect(), + // KECCAK_RND has no preprocessed columns: `roots[KECCAK_RND_SLOT]` + // is the all-zero sentinel and is never consulted. Its correctness + // is entirely its own constraints plus bus balance, both + // program-independent, so there is nothing for a root to pin — + // and nothing that differs between chunks either, which is why + // every instance is built from the same arguments. + keccak_rnd: (0..keccak_rnd_chunks) + .map(|_| { + build_air_no_prep( + keccak_rnd::cols::NUM_COLUMNS, + keccak_rnd::bus_interactions(), + options, + keccak_rnd::KeccakRndConstraints, + LFM_CHIP_NAMES[KECCAK_RND_SLOT], + ) + }) + .collect(), + keccak_rc: build_air( + keccak_rc::cols::NUM_COLUMNS, + keccak_rc::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[13], + roots[13], + keccak_rc::NUM_PRECOMPUTED_COLS, + ), + bitwise: build_air( + bitwise::cols::NUM_COLUMNS, + bitwise::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[14], + roots[14], + bitwise::NUM_PRECOMPUTED_COLS, + ), + chip_set, + merged: Vec::new(), + merge_plan: Vec::new(), + } + } + + /// This set with the merged AIRs `merge` describes: each group's chips + /// side by side ([`super::merge`]), against the group's own preprocessed + /// root. An empty `merge` leaves the set as it is. + /// + /// ★ The ONE door every merged AIR set goes through — the prover, the host + /// verifier, the leg's derivation (`DerivedChild`) and the harvest — so the + /// checks live here: the merge must be well formed against the program's + /// per-slot `log_heights` ([`LfmMerge::check`]: two or more chips, slot + /// order, no chip twice, every chip at the merged height), every part body + /// must be one the remap translates ([`super::merge::audit_body`]) and every + /// bus packing must stay contiguous ([`super::merge::remap_interaction`]). + /// `Err` otherwise; nothing is built. + /// + /// A merged AIR has no one-row root: the format refuses the merge beside + /// one-row openings (`ZfFormat::check_lfm_merge`). + pub fn with_merge( + mut self, + merge: &LfmMerge, + log_heights: &[u8; NUM_LFM_CHIPS], + options: &ProofOptions, + ) -> Result { + merge.check(log_heights)?; + let mut merged = Vec::with_capacity(merge.tables.len()); + for t in &merge.tables { + let layout = MergedLayout::of(&t.chips); + let name = super::merge::merged_name(&t.chips); + merged.push( + AirWithBuses::new( + layout.num_columns, + AuxiliaryTraceBuildData { + interactions: super::merge::merged_interactions(&layout)?, + }, + options, + 1, + MergedConstraints::new(&layout)?, + ) + .with_name(&name) + .with_preprocessed(t.root, layout.num_prep), + ); + } + self.merged = merged; + self.merge_plan = merge.plan(); + Ok(self) + } + + /// The chips each merged AIR holds, in AIR order. + pub fn merge_plan(&self) -> &MergePlan { + &self.merge_plan + } + + /// Where chip slot `slot` stands under this set's merge. + fn role(&self, slot: usize) -> SlotRole { + super::merge::role(&self.merge_plan, slot) + } + + /// Pushes chip `slot`'s AIR in the frozen order: its own, its merged AIR + /// (at the merged AIR's first chip), or nothing (a later merged chip). + fn push_ref<'a>(&'a self, refs: &mut Vec>, slot: usize, own: DynLfmAir<'a>) { + match self.role(slot) { + SlotRole::Own => refs.push(own), + SlotRole::First(g) => refs.push(&self.merged[g]), + SlotRole::Member => {} + } + } + + /// This set with every preprocessed chip's ONE-ROW (S2) root attached: + /// what `precomputed_commitment_for(Row)` returns when the STARK prover or + /// verifier resolves that chip to one row. Without it a one-row chip is a + /// hard miss, never a recompute. `KECCAK_RND` has no preprocessed columns. + pub fn with_one_row_roots(mut self, one_row: &super::registry::LfmOneRowRoots) -> Self { + let r = &one_row.roots; + self.const_ = self.const_.with_one_row_commitment(r[0]); + self.balu = self.balu.with_one_row_commitment(r[1]); + self.xalu = self.xalu.with_one_row_commitment(r[2]); + self.select = self.select.with_one_row_commitment(r[3]); + self.bitdec = self.bitdec.with_one_row_commitment(r[4]); + self.hash = self.hash.with_one_row_commitment(r[5]); + self.hash_tail = std::mem::take(&mut self.hash_tail) + .into_iter() + .enumerate() + .map(|(i, air)| { + air.with_one_row_commitment(one_row.hash_chunk_roots.get(i + 1).copied()) + }) + .collect(); + self.keccak = self.keccak.with_one_row_commitment(r[6]); + self.lanes = self.lanes.with_one_row_commitment(r[7]); + self.hint = self.hint.with_one_row_commitment(r[8]); + self.public = self.public.with_one_row_commitment(r[9]); + self.range = self.range.with_one_row_commitment(r[10]); + self.blake3 = std::mem::take(&mut self.blake3) + .into_iter() + .enumerate() + .map(|(i, air)| air.with_one_row_commitment(one_row.blake3_chunk_roots.get(i).copied())) + .collect(); + self.keccak_rc = self.keccak_rc.with_one_row_commitment(r[13]); + self.bitwise = self.bitwise.with_one_row_commitment(r[14]); + self + } + + /// Number of `KECCAK_RND` instances this set was built with. + pub fn keccak_rnd_chunks(&self) -> usize { + self.keccak_rnd.len() + } + + /// Number of `LFM_BLAKE3` instances this set was built with. + pub fn blake3_chunks(&self) -> usize { + self.blake3.len() + } + + /// This set with `LFM_HASH` split over `hash_roots`: chunk 0 stays + /// the slot-5 AIR, and chunks 1.. are built from `hash_roots[1..]` under the + /// same hasher. A one-element slice adds nothing. + /// + /// Call before [`Self::with_one_row_roots`], which attaches each chunk's + /// one-row root to the instances present. + pub fn with_hash_tail( + mut self, + hash_roots: &[Commitment], + options: &ProofOptions, + hasher: HasherKind, + ) -> Self { + self.hash_tail = hash_roots + .iter() + .skip(1) + .map(|root| { + build_air( + hash::num_columns(hasher), + hash::bus_interactions(hasher), + options, + hash::HashConstraints { kind: hasher }, + LFM_CHIP_NAMES[HASH_SLOT], + *root, + layout::hash::PREP_WIDTH, + ) + }) + .collect(); + self + } + + /// The AIR set a program's artifacts describe: roots, `KECCAK_RND` and + /// `LFM_BLAKE3` chunks, the `LFM_HASH` chunks, the hasher, the mask, the + /// one-row roots and the merged AIRs — every piece from the artifacts, none + /// from a proof. The same set `proof::verify_against_artifacts` builds piece + /// by piece; a caller holding artifacts (the node's child harvest) builds it + /// here. `Err` when the artifacts' merge is refused ([`Self::with_merge`]). + pub fn for_artifacts( + artifacts: &super::registry::LfmArtifacts, + options: &ProofOptions, + ) -> Result { + let mut airs = Self::new_chunked( + &artifacts.roots, + &artifacts.blake3_chunk_roots, + options, + artifacts.keccak_rnd_chunks, + artifacts.hasher, + artifacts.chip_set, + ) + .with_hash_tail(&artifacts.hash_chunk_roots, options, artifacts.hasher); + if let Some(one_row) = &artifacts.one_row_roots { + airs = airs.with_one_row_roots(one_row); + } + airs.with_merge(&artifacts.merge, &artifacts.log_heights, options) + } + + /// Number of `LFM_HASH` instances this set was built with — one unless the + /// hash split divided the table. + pub fn hash_chunks(&self) -> usize { + 1 + self.hash_tail.len() + } + + /// Verify-side projection, frozen order (must match `air_trace_pairs`). + pub fn air_refs(&self) -> Vec> { + let mut refs: Vec> = vec![&self.const_]; + // A merged chip gives its place to its merged AIR, or leaves the + // order; with no merge every chip pushes its own AIR, as always. + self.push_ref(&mut refs, 1, &self.balu); + refs.push(&self.xalu); + self.push_ref(&mut refs, 3, &self.select); + refs.push(&self.bitdec); + refs.push(&self.hash); + refs.extend(self.hash_tail.iter().map(|a| a as DynLfmAir<'_>)); + // The frozen order is unchanged; an absent family leaves a hole in it + // rather than moving anything after it. + if self.chip_set.keccak { + refs.push(&self.keccak); + } + self.push_ref(&mut refs, 7, &self.lanes); + self.push_ref(&mut refs, 8, &self.hint); + refs.push(&self.public); + refs.push(&self.range); + if self.chip_set.blake3 { + refs.extend(self.blake3.iter().map(|a| a as DynLfmAir<'_>)); + } + if self.chip_set.keccak { + refs.extend(self.keccak_rnd.iter().map(|a| a as DynLfmAir<'_>)); + refs.push(&self.keccak_rc); + } + if self.chip_set.bitwise { + refs.push(&self.bitwise); + } + refs + } + + /// Each sub-proof's trace log-height, in [`Self::air_refs`]' frozen order, + /// from `artifacts` alone — what a parent's emitter needs to derive its + /// child's shapes without the child's proof. `None` when the keccak family + /// is present: `KECCAK_RND`'s chunk heights are its workload's, which the + /// artifacts do not carry. + pub fn log_heights_in_air_order(artifacts: &super::registry::LfmArtifacts) -> Option> { + let set = artifacts.chip_set; + if set.keccak { + return None; + } + let h = &artifacts.log_heights; + // A merged AIR stands at its first chip and shares its chips' height; + // its later chips leave the order (`air_refs`). + let plan = artifacts.merge.plan(); + let in_order = |slots: std::ops::Range| -> Vec { + slots + .filter(|&s| super::merge::role(&plan, s) != SlotRole::Member) + .map(|s| h[s]) + .collect() + }; + let mut out: Vec = in_order(0..HASH_SLOT); + out.extend(artifacts.hash_chunk_log_heights.iter().copied()); + out.extend(in_order(KECCAK_SLOT + 1..BLAKE3_SLOT)); + if set.blake3 { + out.extend(artifacts.blake3_chunk_log_heights.iter().copied()); + } + if set.bitwise { + out.push(h[BITWISE_SLOT]); + } + Some(out) + } + + /// Prove-side projection, frozen order (must match `air_refs`). + /// + /// When the keccak family is present, `traces.keccak_rnd` must have exactly + /// one trace per chunk; a mismatch would silently shorten the pair list + /// under `zip`, so it is asserted. When the family is ABSENT the counts + /// legitimately differ: trace building is mask-blind (the chunk split + /// always yields at least one, empty, trace), and the mask simply never + /// pairs it — so the assert is gated, or every keccak-less program would + /// panic any debug-profile prove while release proved fine. + /// + /// `traces.blake3` is asserted the same way and gated for the same reason: + /// slot 11's group is committed for every program, so the split always + /// yields at least one trace, and a BLAKE3-less program simply never pairs + /// it. + #[allow(clippy::type_complexity)] + pub fn air_trace_pairs<'a>( + &'a self, + traces: &'a mut LfmTraces, + ) -> Vec<(DynLfmAir<'a>, &'a mut TraceTable, &'a ())> { + debug_assert!( + !self.chip_set.keccak || self.keccak_rnd.len() == traces.keccak_rnd.len(), + "KECCAK_RND chunk count differs between the AIR set and the traces \ + — artifacts and traces were built from different chunking policies" + ); + debug_assert!( + !self.chip_set.blake3 || self.blake3.len() == traces.blake3.len(), + "LFM_BLAKE3 chunk count differs between the AIR set and the traces \ + — artifacts and traces were built from different chunking policies" + ); + // The trace builder skips BITWISE's `2^20`-row fill when the mask IT + // derives drops the table, and it drops the table only for a program + // whose chips send it nothing — so every multiplicity would be zero. A + // set whose artifacts keep BITWISE for such a program (built under + // `KEEP_BITWISE_ENV`, or under a different setting than this process) + // gets exactly that all-zero table here: which programs keep BITWISE is + // the artifacts' decision, and no proof depends on the process setting. + if self.chip_set.bitwise && traces.bitwise.num_rows() == 0 { + traces.bitwise = crate::tables::bitwise::generate_bitwise_trace(); + } + // The merged tables, one per merged AIR, in its order: a trace set + // filled under another merge is the caller's to refuse + // (`proof::prove_traces_under`), so only a debug build checks here. + debug_assert_eq!( + traces.merge_plan, self.merge_plan, + "the traces were filled under another merge than the AIR set's" + ); + let mut merged: Vec>> = + traces.merged.iter_mut().map(Some).collect(); + let mut pairs: Vec<(DynLfmAir<'a>, &'a mut TraceTable, &'a ())> = + vec![(&self.const_, &mut traces.const_, &())]; + // A merged chip gives its place to its merged AIR and table, or leaves + // the order — `air_refs`' rule, pair by pair. + let mut push = |pairs: &mut Vec<(DynLfmAir<'a>, &'a mut TraceTable, &'a ())>, + slot: usize, + air: DynLfmAir<'a>, + trace: &'a mut TraceTable| { + match self.role(slot) { + SlotRole::Own => pairs.push((air, trace, &())), + SlotRole::First(g) => { + if let Some(t) = merged.get_mut(g).and_then(Option::take) { + pairs.push((&self.merged[g], t, &())); + } + } + SlotRole::Member => {} + } + }; + push(&mut pairs, 1, &self.balu, &mut traces.balu); + pairs.push((&self.xalu, &mut traces.xalu, &())); + push(&mut pairs, 3, &self.select, &mut traces.select); + pairs.push((&self.bitdec, &mut traces.bitdec, &())); + pairs.push((&self.hash, &mut traces.hash, &())); + assert_eq!( + self.hash_tail.len(), + traces.hash_tail.len(), + "LFM_HASH chunk count differs between the AIR set and the traces \ + — artifacts and traces were built from different hash-split policies" + ); + pairs.extend( + self.hash_tail + .iter() + .zip(traces.hash_tail.iter_mut()) + .map(|(air, trace)| (air as DynLfmAir<'a>, trace, &())), + ); + // Same gating as `air_refs`, in the same order — these two ARE the + // proof's layout and must move together. + if self.chip_set.keccak { + pairs.push((&self.keccak, &mut traces.keccak, &())); + } + push(&mut pairs, 7, &self.lanes, &mut traces.lanes); + push(&mut pairs, 8, &self.hint, &mut traces.hint); + pairs.push((&self.public, &mut traces.public, &())); + pairs.push((&self.range, &mut traces.range, &())); + if self.chip_set.blake3 { + pairs.extend( + self.blake3 + .iter() + .zip(traces.blake3.iter_mut()) + .map(|(air, trace)| (air as DynLfmAir<'a>, trace, &())), + ); + } + if self.chip_set.keccak { + pairs.extend( + self.keccak_rnd + .iter() + .zip(traces.keccak_rnd.iter_mut()) + .map(|(air, trace)| (air as DynLfmAir<'a>, trace, &())), + ); + pairs.push((&self.keccak_rc, &mut traces.keccak_rc, &())); + } + if self.chip_set.bitwise { + pairs.push((&self.bitwise, &mut traces.bitwise, &())); + } + pairs + } +} diff --git a/prover/src/lfm/algebraic_commit.rs b/prover/src/lfm/algebraic_commit.rs new file mode 100644 index 000000000..9e52afcf6 --- /dev/null +++ b/prover/src/lfm/algebraic_commit.rs @@ -0,0 +1,1421 @@ +//! Merkle commitment backends for an ALGEBRAIC hash — item A. +//! +//! One implementation, generic over the permutation: a permutation is a **type +//! tag**, not a code path, so another candidate costs a permutation plus a +//! const. RPX is the one tag today. +//! +//! # What made this small +//! +//! ✓ VERIFIED, and each of these was checked rather than assumed: +//! +//! - `IsMerkleTreeBackend::Node` is bound only to +//! `PartialEq + Eq + Clone + Sync + Send` — **the trait never became +//! byte-only**, and the algebraic backends that shipped in January 2026 +//! (`BatchPoseidonTree`, `TreePoseidon`) are still in-tree as proof. +//! - `StarkHash` pins `Node = Commitment` (32 bytes) and **a four-felt +//! Goldilocks digest is exactly 32 canonical bytes**, so the node type does +//! not fight us. +//! - `IsStreamingLeafBackend::Data` is already `Vec>` — felts, +//! not bytes. +//! +//! **BLAKE3's path is untouched.** These are sibling types beside +//! `FieldElementVectorBackend`, not a reparameterisation of it, which is what +//! keeps the measured BLAKE3 record safe from this work. +//! +//! # The conventions, and what pins them +//! +//! **Felt↔byte is canonical BIG-endian**, one rule shared with +//! [`super::algebraic_transcript`] and with `ByteConversion::write_bytes_be`. +//! +//! **A parent is `compress(left, right)`** — one permutation of +//! `[left ‖ right ‖ capacity]` with the compress domain, which is zero, so a +//! parent is literally `Rpo256::merge` and externally checkable. +//! +//! **A leaf is the rate-8 OVERWRITE DUPLEX** (RPO spec §2.6), the convention +//! this lane priced and adopted: capacity lane 0 carries the padding flag +//! `len mod 8`, capacity lane 1 the LEAF domain tag, and each block overwrites +//! the eight rate lanes with no field arithmetic at all. It absorbs eight fresh +//! felts per permutation where the socket's as-built leaf chain absorbs four, +//! which is worth 25% of the aggregation program. +//! +//! ⚠ **`hash_bytes` must equal `hash_data` on the elements those bytes +//! encode** — that is the `IsStreamingLeafBackend` contract, and it is the one +//! place an algebraic backend could silently disagree with itself, because the +//! byte route has to rebuild the felts the felt route was handed. +//! [`tests::hash_bytes_agrees_with_hash_data`] is the gate. +//! +//! # A1 — the incremental leaf hasher buffers +//! +//! `IsStreamingLeafBackend` requires an incremental `LeafHasher`, but **every +//! capacity-flag padding rule needs the total length before the first +//! permutation**. So [`AlgebraicLeafHasher`] buffers the leaf's felts and +//! sponges at `finalize`. The buffer is bounded by the leaf's row width — no +//! worse than the `hash_data(Vec)` route that already exists — and the +//! alternative would be inventing a padding rule, which is not this lane's to +//! invent. + +use core::marker::PhantomData; +use core::num::NonZeroUsize; + +use math::field::element::FieldElement; +use math::field::traits::{IsField, IsPrimeField}; +use math::traits::AsBytes; + +use crypto::fiat_shamir::transcript_hash::TranscriptHash; +use crypto::merkle_tree::traits::{IsLeafHasher, IsMerkleTreeBackend, IsStreamingLeafBackend}; +use stark::config::{Commitment, CommitmentHash, DeviceTreeBackend, StarkHash}; + +use super::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use super::rpo::{DOMAIN_LEAF, RATE_FELTS, domain_iv}; +use super::word::LfmWord; +use crate::tables::types::{FE, GoldilocksField}; + +/// Bytes one Goldilocks felt serialises to — `ByteConversion::BYTE_LEN`. +pub const BYTES_PER_FELT: usize = 8; +/// Felts in a digest, hence `Commitment`'s 32 bytes. +pub const DIGEST_FELTS: usize = 4; + +/// A type-level name for one algebraic permutation. +/// +/// The whole reason the backends below are one implementation: a candidate +/// joins by adding a unit struct, a `KIND` and a `COMMITMENT_HASH`, and +/// nothing else here moves. +pub trait AlgebraicHasher: Clone + Copy + Default + Send + Sync + 'static { + /// The permutation the `LFM_HASH` socket proves for this commitment. + const KIND: HasherKind; + /// The name a root built by this permutation goes by — what both Merkle + /// families report as their device dispatch key and what the `StarkHash` + /// built on them reports as its `COMMITMENT_HASH`, stated once here so the + /// three cannot disagree. + const COMMITMENT_HASH: CommitmentHash; +} + +/// Rescue-Prime eXtended (XHash12). +#[derive(Clone, Copy, Default, Debug, PartialEq, Eq)] +pub struct RpxCommit; +impl AlgebraicHasher for RpxCommit { + const KIND: HasherKind = HasherKind::Rpx; + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Rpx256; +} + +/// Four felts as a 32-byte `Commitment`, canonical big-endian. +pub fn digest_to_commitment(d: &LfmWord) -> Commitment { + let mut out = [0u8; 32]; + for (i, f) in d.iter().enumerate() { + let v = GoldilocksField::canonical(f.value()); + out[i * BYTES_PER_FELT..(i + 1) * BYTES_PER_FELT].copy_from_slice(&v.to_be_bytes()); + } + out +} + +/// A 32-byte `Commitment` back to four felts. +pub fn commitment_to_digest(c: &Commitment) -> LfmWord { + core::array::from_fn(|i| { + let mut b = [0u8; BYTES_PER_FELT]; + b.copy_from_slice(&c[i * BYTES_PER_FELT..(i + 1) * BYTES_PER_FELT]); + FE::from(u64::from_be_bytes(b)) + }) +} + +/// ★ The rate-8 overwrite duplex over a felt stream — the leaf construction. +/// +/// Capacity lane 0 is the padding flag `len mod 8`, lane 1 the LEAF domain. +/// Each block OVERWRITES the eight rate lanes (spec §2.6), so absorption costs +/// no field arithmetic outside the permutation; the tail block is zero-padded. +/// ★ **THE LEAF CAPACITY RULE, stated once.** Lane 0 is the padding flag +/// `len mod 8` — zero when the length divides the rate, which is why no +/// trailing block is spent on an exact multiple — and lane 1 the LEAF domain. +/// +/// ⚠ Exported so the MACHINE side derives it rather than restating it. Under +/// `MODE_P` the capacity is program data, so an emitter must supply exactly +/// this word; a second definition of it is a root nobody can reproduce. +pub fn leaf_capacity(num_felts: usize) -> LfmWord { + let iv = domain_iv(DOMAIN_LEAF); + let mut cap: LfmWord = core::array::from_fn(|k| FE::from(iv[k])); + cap[0] = FE::from((num_felts % RATE_FELTS) as u64); + cap +} + +/// ★ **The three `MODE_P` input cells for a SINGLE-BLOCK leaf sponge** — the +/// rule, exported for the same reason [`leaf_capacity`] is. +/// +/// Two rate cells carrying up to eight felts zero-padded, then the capacity. +/// This is the shape grinding takes (`state ‖ nonce` is 40 bytes, five felts, +/// one block), and the shape the duplex emitter's first block takes. +pub fn single_block_leaf_cells(felts: &[FE]) -> [LfmWord; 3] { + debug_assert!( + felts.len() <= RATE_FELTS, + "a single block carries at most the rate ({RATE_FELTS} felts), got {}", + felts.len() + ); + let lane = |i: usize| felts.get(i).copied().unwrap_or_else(FE::zero); + [ + [lane(0), lane(1), lane(2), lane(3)], + [lane(4), lane(5), lane(6), lane(7)], + leaf_capacity(felts.len()), + ] +} + +pub fn sponge_leaf(kind: HasherKind, felts: &[FE]) -> LfmWord { + let mut state = [FE::zero(); HASH_STATE_FELTS]; + let cap = leaf_capacity(felts.len()); + state[RATE_FELTS..].copy_from_slice(&cap); + + if felts.is_empty() { + return [state[0], state[1], state[2], state[3]]; + } + for block in felts.chunks(RATE_FELTS) { + for (lane, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + *slot = block.get(lane).copied().unwrap_or_else(FE::zero); + } + state = kind.permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// `sponge_leaf(kind, &felts_from_bytes(bytes))`, without materialising the +/// felts. +/// +/// The leaf construction over a byte string is what every [`AlgebraicDigest`] +/// finalize performs, and the intermediate `Vec` it built was pure +/// allocation: the sponge consumes the felts one rate block at a time and never +/// looks back, and the felt count the capacity needs — `ceil(len / 8)` — is +/// known from the length alone. Under the algebraic pin that `Vec` was one of +/// the two heap allocations every proof-of-work grinding trial paid. +/// +/// ⚠ Equivalent to the two-step form BY TEST (`sponge_leaf_bytes_matches_the_felt_form`), +/// not by construction: the trailing partial group is zero-extended on the LOW +/// side here, which is what `felts_from_bytes` does and is easy to get backwards. +pub fn sponge_leaf_bytes(kind: HasherKind, bytes: &[u8]) -> LfmWord { + let num_felts = bytes.len().div_ceil(BYTES_PER_FELT); + let mut state = [FE::zero(); HASH_STATE_FELTS]; + let cap = leaf_capacity(num_felts); + state[RATE_FELTS..].copy_from_slice(&cap); + + if bytes.is_empty() { + return [state[0], state[1], state[2], state[3]]; + } + // One rate block is eight felts, i.e. 64 bytes. + for block in bytes.chunks(RATE_FELTS * BYTES_PER_FELT) { + for (lane, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + let start = lane * BYTES_PER_FELT; + *slot = if start >= block.len() { + FE::zero() + } else { + let end = (start + BYTES_PER_FELT).min(block.len()); + let mut b = [0u8; BYTES_PER_FELT]; + b[..end - start].copy_from_slice(&block[start..end]); + FE::from(u64::from_be_bytes(b)) + }; + } + state = kind.permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// Every 8-byte big-endian group of `bytes` as a felt. +/// +/// The inverse of the serialisation `ByteConversion::write_bytes_be` performs, +/// which is how leaves reach a backend. A trailing partial group is +/// zero-extended on the LOW side, matching how a short write would land. +pub fn felts_from_bytes(bytes: &[u8]) -> Vec { + bytes + .chunks(BYTES_PER_FELT) + .map(|c| { + let mut b = [0u8; BYTES_PER_FELT]; + b[..c.len()].copy_from_slice(c); + FE::from(u64::from_be_bytes(b)) + }) + .collect() +} + +/// Decompose a field element — base or extension — into its base felts, by the +/// same serialisation the STARK uses. +/// +/// ★ Through `AsBytes::stream_bytes` rather than `ByteConversion::write_bytes_be`, +/// and the two are the SAME bytes: ✓ VERIFIED both Goldilocks impls — +/// `FieldElement` streams `canonical_u64().to_be_bytes()` and +/// its `as_bytes` IS `to_bytes_be`, while `FieldElement` +/// streams by calling `write_bytes_be` into a stack buffer. +/// +/// ⚠ The reason for going through `AsBytes` is not style: `StarkHash`'s +/// associated types carry the bound `FieldElement: AsBytes + Sync + Send` and +/// nothing more, so a backend that additionally required `ByteConversion` could +/// not BE a `StarkHash::Batched` — and the algebraic configurations could not be +/// expressed at all. Taking the identical bytes through the weaker bound is what +/// makes them expressible without widening a trait the whole workspace shares. +pub(crate) fn element_felts(e: &FieldElement, out: &mut Vec) +where + F: IsField, + FieldElement: AsBytes, +{ + let mut buf = [0u8; 64]; + let mut len = 0usize; + e.stream_bytes(&mut |bytes| { + debug_assert!( + len + bytes.len() <= buf.len(), + "a field element must fit the scratch" + ); + buf[len..len + bytes.len()].copy_from_slice(bytes); + len += bytes.len(); + }); + out.extend(felts_from_bytes(&buf[..len])); +} + +/// The batched leaf backend — one leaf per row group. +#[derive(Clone, Debug, Default)] +pub struct AlgebraicBatchBackend { + _marker: PhantomData (F, H)>, +} + +/// The FRI-layer backend — one leaf per fixed pair, no `Vec` per leaf. +#[derive(Clone, Debug, Default)] +pub struct AlgebraicPairBackend { + _marker: PhantomData (F, H)>, +} + +/// A parent: `compress(left, right)`, one permutation, the compress domain. +fn parent(left: &Commitment, right: &Commitment) -> Commitment { + let l = commitment_to_digest(left); + let r = commitment_to_digest(right); + digest_to_commitment(&H::KIND.compress(&l, &r)) +} + +impl IsMerkleTreeBackend for AlgebraicBatchBackend +where + F: IsField + 'static, + H: AlgebraicHasher, + FieldElement: AsBytes + Sync + Send, + Vec>: Sync + Send, +{ + type Node = Commitment; + type Data = Vec>; + + fn hash_data(input: &Vec>) -> Commitment { + >::hash_data_from_slices(input, &[]) + } + + fn hash_new_parent(left: &Commitment, right: &Commitment) -> Commitment { + parent::(left, right) + } +} + +impl IsStreamingLeafBackend for AlgebraicBatchBackend +where + F: IsField + 'static, + H: AlgebraicHasher, + FieldElement: AsBytes + Sync + Send, + Vec>: Sync + Send, +{ + /// ⚠ Must equal [`IsMerkleTreeBackend::hash_data`] on the elements `data` + /// encodes — the trait's contract, and the gate is + /// `tests::hash_bytes_agrees_with_hash_data`. + fn hash_bytes(data: &[u8]) -> Commitment { + digest_to_commitment(&sponge_leaf(H::KIND, &felts_from_bytes(data))) + } + + fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> Commitment { + let mut felts = Vec::with_capacity((a.len() + b.len()) * DIGEST_FELTS); + for e in a.iter().chain(b.iter()) { + element_felts(e, &mut felts); + } + digest_to_commitment(&sponge_leaf(H::KIND, &felts)) + } + + type LeafHasher = AlgebraicLeafHasher; + + fn leaf_hasher() -> Self::LeafHasher { + AlgebraicLeafHasher { + felts: Vec::new(), + _marker: PhantomData, + } + } +} + +impl IsMerkleTreeBackend for AlgebraicPairBackend +where + F: IsField + 'static, + H: AlgebraicHasher, + FieldElement: AsBytes + Sync + Send, +{ + type Node = Commitment; + type Data = [FieldElement; 2]; + + fn hash_data(input: &[FieldElement; 2]) -> Commitment { + let mut felts = Vec::with_capacity(2 * DIGEST_FELTS); + element_felts(&input[0], &mut felts); + element_felts(&input[1], &mut felts); + digest_to_commitment(&sponge_leaf(H::KIND, &felts)) + } + + fn hash_new_parent(left: &Commitment, right: &Commitment) -> Commitment { + parent::(left, right) + } +} + +/// ★ The device marker — what makes an algebraic configuration expressible +/// under `cuda`. +/// +/// `StarkHash`'s cuda bounds require both families to be [`DeviceTreeBackend`]s, +/// and the constant is the dispatch key `gpu_lde` hands to `math-cuda`. No +/// device kernels exist for these permutations yet, so every `math-cuda` +/// dispatch site aborts with `unimplemented!` naming the hash the moment a +/// device commit is attempted under one of them — never a byte-hash kernel in +/// its place. That is the discipline HASH-PINNING.md demands: a build under an +/// algebraic pin exists and lints on both feature arms, and it still cannot +/// prove under a hash it is not named for. +impl DeviceTreeBackend for AlgebraicBatchBackend +where + Self: IsMerkleTreeBackend, + H: AlgebraicHasher, +{ + const COMMITMENT_HASH: CommitmentHash = H::COMMITMENT_HASH; +} +impl DeviceTreeBackend for AlgebraicPairBackend +where + Self: IsMerkleTreeBackend, + H: AlgebraicHasher, +{ + const COMMITMENT_HASH: CommitmentHash = H::COMMITMENT_HASH; +} + +/// ★ A1 — the incremental leaf hasher, which BUFFERS. +/// +/// The padding flag is `len mod 8` and the sponge needs it in the capacity +/// before the first permutation, so an incremental sponge cannot start until +/// the length is known. Buffering the leaf's felts is bounded by the row width +/// and is no worse than the `hash_data(Vec)` route; inventing a length-free +/// padding rule instead would be a cryptographic decision this lane does not +/// get to make. +pub struct AlgebraicLeafHasher { + felts: Vec, + /// `fn() -> T` rather than `T`, so the marker is unconditionally `Send` and + /// `Sync` without an `unsafe impl`. The trait requires `Send` because there + /// is one hasher per leaf and a real epoch's base layer has millions of + /// them, absorbed in parallel. + _marker: PhantomData (F, H)>, +} + +impl IsLeafHasher for AlgebraicLeafHasher +where + F: IsField, + H: AlgebraicHasher, + FieldElement: AsBytes, +{ + type Node = Commitment; + + fn update(&mut self, data: &[FieldElement]) { + for e in data { + element_felts(e, &mut self.felts); + } + } + + fn finalize(self) -> Commitment { + digest_to_commitment(&sponge_leaf(H::KIND, &self.felts)) + } +} + +// ========================================================================= +// The GRINDING hash — `StarkHash::Transcript` +// ========================================================================= + +/// A `digest::Digest` over an algebraic permutation, for the ONE thing a +/// configuration's `Transcript` associated type still decides: **grinding**. +/// +/// # Why this exists at all +/// +/// The CHALLENGE stream does not flow through here. +/// [`super::algebraic_transcript::AlgebraicTranscript`] carries that, because +/// `prove`/`verify` take the transcript as a parameter. What +/// `StarkHash::Transcript` forces is what the prover and verifier derive +/// *internally from the configuration* — grinding, whose seed is +/// `transcript.state()` and whose proof-of-work is `H(seed ‖ nonce)`. +/// +/// ⚠ **And grinding is verified IN-VM** (`transcript_replay.rs` carries it; the +/// verifier absorbs `nonce_value.to_be_bytes()`). So whatever hash grinds, the +/// machine must be able to compute it. Leaving BLAKE3 here would keep the +/// 3,056-column slot-11 chip in an algebraic branch's AIR set — which would +/// make the four-branch comparison measure something other than the hash it +/// names. That is why this type exists rather than a `Blake3TranscriptHash` +/// alias. +/// +/// # The construction is the LEAF one, deliberately +/// +/// Grinding hashes a byte string — `state ‖ nonce`, 40 bytes, five felts — which +/// is DATA, exactly what a leaf is. It therefore reuses [`sponge_leaf`] and the +/// LEAF domain rather than inventing a fourth one. +/// +/// ⚖ That is not only simplicity. The socket pins per-mode capacities for +/// `MODE_C`/`MODE_T`/`MODE_L`, so **a grinding-specific domain is not something +/// the in-VM verifier could emit with an existing mode.** Using the leaf +/// construction is what keeps the machine side expressible. The domain reuse is +/// not exploitable: the grinding check tests leading zeros of a hash whose +/// preimage is transcript-bound, so colliding it with some leaf digest buys an +/// adversary nothing. +/// +/// # Buffering, for the third time and the same reason +/// +/// `sponge_leaf`'s padding flag is `len mod 8`, needed in the capacity before +/// the first permutation, so this accumulates bytes and sponges at finalize — +/// the same resolution A1 and the transcript reached. +pub struct AlgebraicDigest { + buf: Vec, + _marker: PhantomData H>, +} + +impl Default for AlgebraicDigest { + fn default() -> Self { + Self { + buf: Vec::new(), + _marker: PhantomData, + } + } +} + +impl Clone for AlgebraicDigest { + fn clone(&self) -> Self { + Self { + buf: self.buf.clone(), + _marker: PhantomData, + } + } +} + +impl AlgebraicDigest { + /// The digest of everything absorbed so far — the leaf construction over + /// the buffered bytes. + pub fn finalize_digest(&self) -> Commitment { + digest_to_commitment(&sponge_leaf_bytes(H::KIND, &self.buf)) + } +} + +/// ★ The device grind arm, derived from the name the configuration already +/// declares — so the two cannot disagree and there is nothing extra to forget. +/// +/// This impl is why the seam is an associated constant rather than a list in +/// `crypto`: `AlgebraicDigest` lives here, `crypto` depends on nothing in this +/// crate, and a `TypeId` list down there could never name it. Before this, +/// every RPX grind on the block path fell to the host rayon search — ~2^20 RPX +/// permutations per table per epoch — with valid proofs and no failure. +impl stark::grinding::GrindDigest for AlgebraicDigest { + const DEVICE_GRIND: Option = match H::COMMITMENT_HASH { + CommitmentHash::Rpx256 => Some(stark::grinding::DeviceGrindKey::Rpx256), + _ => None, + }; +} + +impl digest::HashMarker for AlgebraicDigest {} + +impl digest::OutputSizeUser for AlgebraicDigest { + type OutputSize = digest::typenum::U32; +} + +impl digest::Update for AlgebraicDigest { + fn update(&mut self, data: &[u8]) { + self.buf.extend_from_slice(data); + } +} + +impl digest::FixedOutput for AlgebraicDigest { + fn finalize_into(self, out: &mut digest::Output) { + out.copy_from_slice(&self.finalize_digest()); + } +} + +impl digest::Reset for AlgebraicDigest { + fn reset(&mut self) { + self.buf.clear(); + } +} + +impl digest::FixedOutputReset for AlgebraicDigest { + fn finalize_into_reset(&mut self, out: &mut digest::Output) { + out.copy_from_slice(&self.finalize_digest()); + self.buf.clear(); + } +} + +/// The Fiat–Shamir configuration for an algebraic hash. +/// +/// `CANDIDATES_PER_COORDINATE` is **`Some(1)`, and guaranteed rather than +/// probabilistic** — the strongest schedule any configuration here has. +/// A squeeze yields four felts that are canonical BY CONSTRUCTION, so the +/// `u64`s carved out of their 32 canonical bytes are *always* in the field's +/// range. BLAKE3 needs two candidates because its bytes are arbitrary and a +/// single miss has nowhere to go; an algebraic squeeze cannot miss. +macro_rules! algebraic_transcript_hash { + ($name:ident, $tag:ty, $label:literal, $doc:literal) => { + #[doc = $doc] + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct $name; + + impl TranscriptHash for $name { + type Digest = AlgebraicDigest<$tag>; + /// ⚠ **`true`, and it is load-bearing rather than inherited.** The + /// reversal became a per-configuration choice so the WHIR RPX + /// transcript could drop it, on the argument that an unreversed + /// squeeze hands out canonical felts. That argument applies to + /// these sponges too — but every proof this pipeline has produced + /// came from a `sample()` that reversed, so dropping it here would + /// move the challenges of the STARK pipeline, its wraps and its + /// block root. The saving is real and it is a separate change with + /// its own re-baseline, not a consequence of the WHIR one. + const REVERSES_SQUEEZE: bool = true; + const CANDIDATES_PER_COORDINATE: Option = NonZeroUsize::new(1); + const NAME: &'static str = $label; + } + }; +} + +algebraic_transcript_hash!( + AlgebraicRpxTranscriptHash, + RpxCommit, + "rpx256", + "The RPX256 (XHash12) Fiat–Shamir configuration." +); + +/// ★★ **THE COMMITMENT CONFIGURATIONS** — the piece that makes an algebraic hash +/// nameable by the prover, rather than merely implemented. +/// +/// One `StarkHash` per member: the two Merkle families, the Fiat–Shamir +/// configuration they are paired with, and the [`CommitmentHash`] all of them +/// are. `IsStarkProver` and `IsStarkVerifier` are generic over this parameter +/// already, so naming one of these at a call site is the whole flip — there is +/// no global to re-point and nothing in `crypto/stark` moves. +/// +/// ★ **They live HERE and not in `crypto/stark/src/config.rs`, and that is not a +/// compromise.** `crypto/stark` does not depend on `prover`, so a configuration +/// built from these backends cannot be written there; `impl StarkHash for +/// RpoStarkHash` is legal here because the type is local. The alternative — +/// moving the backends and the three permutations down into `crypto` so the +/// workspace-wide `DefaultStarkHash` alias could name them — would re-point a +/// default the whole workspace shares in order to reach three branches, and +/// would put every blessed BLAKE3 artifact's enforcement (`config.rs`'s +/// `COMMITMENT_HASH` assertion) in the blast radius of a comparison experiment. +/// The pin belongs at the layer the block path lives in. +/// +/// ⚠ **`Batched` and `Pair` are the same hash by construction**, not by +/// convention: both are the generic algebraic backends at the same `H`, so a +/// configuration mixing two permutations is not something to assert against — +/// it is unspellable. +/// ★ **Expressible under `cuda`, and still unable to prove under the wrong +/// hash.** `StarkHash`'s cuda bounds require both families to be +/// [`DeviceTreeBackend`]s; the algebraic backends are, carrying the tag's own +/// [`CommitmentHash`] as the device dispatch key. `math-cuda` has no kernels for +/// these permutations yet, so a device commit under one of them aborts at the +/// first launch with `unimplemented!` naming the hash. The algebraic path is +/// CPU-only in fact, and a GPU run under it fails loudly rather than producing +/// byte-hash trees *labelled* RPO — the property HASH-PINNING.md demands, held +/// by the dispatch key rather than by forking the build. +macro_rules! algebraic_stark_hash { + ($name:ident, $tag:ty, $transcript:ty, $doc:literal) => { + #[doc = $doc] + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct $name; + + impl StarkHash for $name { + type Batched + = AlgebraicBatchBackend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Pair + = AlgebraicPairBackend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Transcript = $transcript; + + // Read from the tag rather than restated: it is the same constant + // the two families report as their device dispatch key, so the + // configuration's name and its kernels' name cannot be edited + // apart. + const COMMITMENT_HASH: CommitmentHash = <$tag as AlgebraicHasher>::COMMITMENT_HASH; + } + }; +} + +algebraic_stark_hash!( + RpxStarkHash, + RpxCommit, + AlgebraicRpxTranscriptHash, + "The RPX256 (XHash12) commitment configuration." +); + +/// ✓ The configuration is INHABITED at the fields the prover actually +/// commits over — the base field for main traces, the cubic extension for aux, +/// composition and FRI, within one proof. +/// +/// A `StarkHash` impl that type-checks in isolation can still be unusable: the +/// associated types are generic over `F`, and the bound that matters is the one +/// the prover instantiates them at. This is that instantiation, as a compile-time +/// check rather than as a comment claiming it holds. +const _: fn() = || { + fn assert_usable() + where + H::Batched: IsMerkleTreeBackend, + H::Pair: IsMerkleTreeBackend, + H::Batched: + IsMerkleTreeBackend, + H::Pair: IsMerkleTreeBackend, + { + } + + assert_usable::(); +}; + +/// ✓ Each configuration's members carry the configuration's own hash as their +/// device dispatch key — the tie that makes the [`DeviceTreeBackend`] impls +/// above true statements: `gpu_lde` dispatches kernels on that constant, so a +/// mismatch here would be a GPU run hashing under a name the roots do not +/// deserve. The same check `crypto/stark/src/config.rs` runs for the byte +/// configurations; discriminants are compared because `PartialEq` is not +/// `const`. +const _: () = { + const fn assert_device_hash(expect: CommitmentHash) { + assert!( + B::COMMITMENT_HASH as u8 == expect as u8, + "a configuration's member must name the configuration's own hash as its device key" + ); + } + + assert_device_hash::<::Batched>( + CommitmentHash::Rpx256, + ); + assert_device_hash::<::Pair>( + CommitmentHash::Rpx256, + ); +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tables::types::{FEE, GoldilocksExtension}; + // ★ The tests build their expected felts with `write_bytes_be` deliberately, + // while `element_felts` now goes through `AsBytes::stream_bytes`. That the + // two must agree is the whole reason the switch was safe, so the differential + // is checked here rather than asserted in a comment. + use math::traits::ByteConversion; + + type Base = GoldilocksField; + type Ext = GoldilocksExtension; + + /// Every tenant, so a break in the CONSTRUCTION shows as all-three-fail + /// rather than being mistaken for something about one permutation. + macro_rules! for_each_tenant { + ($body:ident) => { + $body::("Rpx"); + }; + } + + fn base_leaf(n: usize) -> Vec> { + (0..n as u64).map(|i| FE::from(i * 7 + 1)).collect() + } + + fn ext_leaf(n: usize) -> Vec { + (0..n as u64) + .map(|i| FEE::new([FE::from(i + 1), FE::from(i + 2), FE::from(i + 3)])) + .collect() + } + + /// ★★ **The `IsStreamingLeafBackend` contract**: `hash_bytes` must equal + /// `hash_data` on the elements those bytes encode. + /// + /// This is the one place an algebraic backend can silently disagree with + /// itself, because the byte route has to rebuild exactly the felts the felt + /// route was handed. Checked over BASE and EXTENSION leaves — the extension + /// is the interesting case, since one element is three felts. + #[test] + fn hash_bytes_agrees_with_hash_data() { + fn check(name: &str) { + for n in [1usize, 2, 7, 8, 9, 16, 17, 33] { + let leaf = base_leaf(n); + let mut buf = Vec::new(); + for e in &leaf { + let mut b = [0u8; 8]; + e.write_bytes_be(&mut b); + buf.extend_from_slice(&b); + } + assert_eq!( + as IsStreamingLeafBackend>::hash_bytes( + &buf + ), + as IsMerkleTreeBackend>::hash_data(&leaf), + "{name}: base leaf of {n}" + ); + + let leaf = ext_leaf(n); + let mut buf = Vec::new(); + for e in &leaf { + let mut b = [0u8; 24]; + e.write_bytes_be(&mut b); + buf.extend_from_slice(&b); + } + assert_eq!( + as IsStreamingLeafBackend>::hash_bytes( + &buf + ), + as IsMerkleTreeBackend>::hash_data(&leaf), + "{name}: extension leaf of {n}" + ); + } + } + for_each_tenant!(check); + } + + /// ★ **A1's contract: splitting is free.** For any partition of a leaf's + /// elements into consecutive chunks, updating with each in order and + /// finalizing must equal `hash_data` over the whole — a backend whose + /// framing depended on where the updates fell would produce leaves no + /// verifier could re-derive. + #[test] + fn the_incremental_leaf_hasher_agrees_with_hash_data_under_every_split() { + fn check(name: &str) { + let leaf = base_leaf(19); + let want = as IsMerkleTreeBackend>::hash_data(&leaf); + for cut in 0..=leaf.len() { + let mut h = + as IsStreamingLeafBackend>::leaf_hasher(); + h.update(&leaf[..cut]); + h.update(&leaf[cut..]); + assert_eq!(h.finalize(), want, "{name}: split at {cut}"); + } + // And a three-way split, so the property is not an artifact of two. + let mut h = + as IsStreamingLeafBackend>::leaf_hasher(); + h.update(&leaf[..3]); + h.update(&leaf[3..11]); + h.update(&leaf[11..]); + assert_eq!(h.finalize(), want, "{name}: three-way split"); + } + for_each_tenant!(check); + } + + /// A parent is the socket's `compress` — one permutation, the compress + /// domain, which is zero. Under RPO that makes it literally + /// `Rpo256::merge`, checkable against miden by someone who has never seen + /// this codebase. + #[test] + fn a_parent_is_the_socket_compress() { + fn check(name: &str) { + let l = digest_to_commitment(&[FE::from(1u64), FE::from(2), FE::from(3), FE::from(4)]); + let r = digest_to_commitment(&[FE::from(5u64), FE::from(6), FE::from(7), FE::from(8)]); + let got = + as IsMerkleTreeBackend>::hash_new_parent(&l, &r); + let want = digest_to_commitment( + &H::KIND.compress(&commitment_to_digest(&l), &commitment_to_digest(&r)), + ); + assert_eq!(got, want, "{name}"); + // The pair backend must agree with the batched one on parents, or + // one tree's internal nodes are not the other's. + assert_eq!( + as IsMerkleTreeBackend>::hash_new_parent(&l, &r), + got, + "{name}: pair and batched parents must be one function" + ); + } + for_each_tenant!(check); + } + + /// The HOST proof-of-work search still finds a nonce the predicate accepts, + /// under the RPX configuration and through the streaming leaf. + /// + /// This is the fallback arm: whenever there is no GPU, no CUDA build, a + /// device error, or `LAMBDA_VM_NO_GPU_GRIND`, every table's nonce comes from + /// here. `sponge_leaf_bytes` changed the code path both the search and the + /// verifier's check run on, and a change that broke them *together* would + /// leave a self-consistent prover producing proofs nothing else accepts — + /// so the cross-hash control below, where BLAKE3 work must not satisfy the + /// RPX predicate, is the part that makes this more than a tautology. + /// + /// Factor 16: ~65k trials, a fraction of a second, and a nonce passes by + /// chance with probability 2⁻¹⁶. + #[test] + fn the_host_search_finds_a_valid_nonce_under_rpx() { + type RpxGrind = stark::config::GrindingDigest; + + let seed = [0x3cu8; 32]; + let factor = 16u8; + let nonce = stark::grinding::generate_nonce::(&seed, factor) + .expect("a nonce exists at this factor"); + assert!( + stark::grinding::is_valid_nonce::(&seed, nonce, factor), + "the host search must return a nonce its own predicate accepts" + ); + + // CONTROL: work done against another hash is not work against this one. + // Without it, `generate_nonce::` could be hashing anything at + // all and every assertion above would still hold. + type Blake3Grind = stark::config::GrindingDigest; + let other = stark::grinding::generate_nonce::(&seed, factor) + .expect("a nonce exists at this factor"); + assert!( + !stark::grinding::is_valid_nonce::(&seed, other, factor), + "a BLAKE3-ground nonce must not satisfy the RPX predicate" + ); + } + + /// [`sponge_leaf_bytes`] IS `sponge_leaf(kind, &felts_from_bytes(bytes))`. + /// + /// The streaming form exists to drop the intermediate `Vec` every + /// `AlgebraicDigest` finalize allocated, so the only thing that matters is + /// that it hashes the same message. It is checked at every length across two + /// rate blocks — the empty leaf (no permutation at all), every partial + /// group, both exact-block boundaries where the padding flag returns to zero + /// — and at a length whose last group is partial, which is where a low-side + /// versus high-side zero-extension would diverge. + #[test] + fn sponge_leaf_bytes_matches_the_felt_form() { + fn check(name: &str) { + for len in 0..=132usize { + let bytes: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect(); + assert_eq!( + sponge_leaf_bytes(H::KIND, &bytes), + sponge_leaf(H::KIND, &felts_from_bytes(&bytes)), + "{name}: the streaming leaf must equal the felt form at {len} bytes" + ); + } + } + for_each_tenant!(check); + } + + /// A digest round-trips through its 32 canonical big-endian bytes, or + /// `Commitment` does not name the digest. + #[test] + fn a_digest_round_trips_through_its_commitment_bytes() { + let d: LfmWord = [ + FE::from(1u64), + FE::from(0xFFFF_FFFF_0000_0000u64), + FE::zero(), + FE::from(0x0123_4567_89AB_CDEFu64), + ]; + assert_eq!(commitment_to_digest(&digest_to_commitment(&d)), d); + } + + /// ⚠ **The permutation count is `⌈felts/8⌉`, and an exact multiple spends + /// NO trailing block.** The rate-8 census invariance rests on exactly this, + /// so it is asserted against the same closed form the census uses rather + /// than left to the reader. + #[test] + fn the_block_count_is_the_census_closed_form() { + use crate::lfm::edsl::WrapHash; + use crate::lfm::epoch_verify::blocks_for; + for felts in 1..=512usize { + assert_eq!( + felts.div_ceil(RATE_FELTS), + blocks_for(felts, WrapHash::Blake3), + "{felts} felts must cost the census's block count" + ); + } + // The boundary the padding flag exists for: a rate multiple carries a + // zero flag, a partial block carries its remainder. + assert_eq!(16 % RATE_FELTS, 0); + assert_eq!(17 % RATE_FELTS, 1); + } + + /// Each configuration NAMES its own hash, and the statement's program-identity + /// tag agrees with that name. + /// + /// ⚠ The tag is what makes a proof's roots describable, and a configuration + /// whose `COMMITMENT_HASH` disagreed with the tag its programs carry would + /// produce proofs that verify against the wrong identity rather than failing. + /// Distinctness is asserted too: two configurations sharing a tag is the same + /// failure with an extra step. + #[test] + fn each_configuration_names_its_own_hash_and_tag() { + use crate::lfm::statement::commitment_hash_tag; + + let named = [( + ::COMMITMENT_HASH, + CommitmentHash::Rpx256, + )]; + let mut tags = Vec::new(); + for (got, want) in named { + assert_eq!(got, want, "a configuration must name its own hash"); + tags.push(commitment_hash_tag(got)); + } + // Against the incumbents too — the tags share one space. + tags.push(commitment_hash_tag(CommitmentHash::Keccak256)); + tags.push(commitment_hash_tag(CommitmentHash::Blake3)); + let unique: std::collections::BTreeSet<_> = tags.iter().copied().collect(); + assert_eq!(unique.len(), tags.len(), "every commitment tag is distinct"); + } + + // ---- the grinding leg: shared fixtures ---- + + /// The factor the leg tests grind at: small enough to grind in a unit + /// test, big enough that a wrong digest fails with overwhelming + /// probability. + const GRINDING_FACTOR: u8 = 8; + + /// The leg tests' seed — four felts, the shape `AlgebraicTranscript::state()` + /// has in the machine. + const GRINDING_SEED_CELL: LfmWord = [ + FE::const_from_raw(0x0123_4567_89ab_cdef), + FE::const_from_raw(0x1111_2222_3333_4444), + FE::const_from_raw(0x0fed_cba9_8765_4321), + FE::const_from_raw(0x00ff_00ff_00ff_00ff), + ]; + + /// The seed as the HOST sees it: `AlgebraicTranscript::state()` serialises + /// the state cell canonically big-endian, which is `digest_to_commitment`. + fn grinding_seed_bytes() -> Commitment { + digest_to_commitment(&GRINDING_SEED_CELL) + } + + /// The grinding leg as a program. Arena word 0 is the seed cell; word 1 + /// carries the nonce as `FE::from(nonce)` in lane 0 — the packing the + /// production wrap arena uses — and the production emitter itself, + /// `epoch::emit_grinding_check`, lays down the assertions. The seed is + /// published so the program has an output and the proof is about + /// something; the CHECK is the assertions the emitter laid down. + fn grinding_leg_program() -> crate::lfm::compiler::LfmProgram { + use crate::lfm::builder::{Felt, LfmBuilder}; + use crate::lfm::compiler::compile; + use crate::lfm::edsl::{WrapDigest, WrapHash}; + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let arena = b.declare_arena(2); + let seed = b.hint_word(arena, 0); + let nonce_word = b.hint_word(arena, 1); + let [nonce, _, _, _] = b.unpack(nonce_word); + crate::lfm::epoch::emit_grinding_check( + &mut b, + WrapDigest::from_cell(seed), + Felt(nonce.as_cell().addr()), + GRINDING_FACTOR, + ); + b.public(seed); + compile(b.finish()) + } + + /// `grinding::generate_nonce` under one configuration's `AlgebraicDigest`. + type HostGrind = fn(&Commitment, u8) -> Option; + /// `grinding::is_valid_nonce` under one configuration's `AlgebraicDigest` — + /// the SPECIFICATION of which nonces the leg must accept. + type HostVerdict = fn(&Commitment, u64, u8) -> bool; + + /// The members that HAVE a commitment type — `HasherKind::Test` has none, + /// so there is no host digest to grind against — each with the host's + /// grinder and the host's verdict. + fn grinding_members() -> [(HasherKind, HostGrind, HostVerdict); 1] { + use stark::grinding::{generate_nonce, is_valid_nonce}; + [( + HasherKind::Rpx, + |s, f| generate_nonce::>(s, f), + |s, n, f| is_valid_nonce::>(s, n, f), + )] + } + + /// The MACHINE's verdict: whether the emitted leg proves for `nonce`. + fn leg_proves( + program: &crate::lfm::compiler::LfmProgram, + artifacts: &crate::lfm::registry::LfmArtifacts, + opts: &stark::proof::options::ProofOptions, + hasher: HasherKind, + nonce: u64, + ) -> bool { + use crate::lfm::proof::lfm_prove_with_hasher; + + let arena = vec![vec![ + GRINDING_SEED_CELL, + [FE::from(nonce), FE::zero(), FE::zero(), FE::zero()], + ]]; + lfm_prove_with_hasher(program, artifacts, &arena, opts, hasher).is_ok() + } + + /// ★★ **THE GRINDING LEG GATE.** The production emitter's grinding check — + /// `epoch::emit_grinding_check`, the thing every wrap program actually calls + /// — must accept exactly the nonces `grinding::is_valid_nonce` accepts under + /// the algebraic configuration. + /// + /// The gate above checks the DIGEST; this checks the LEG, which is a + /// different claim: the leg builds two preimages, and getting either one's + /// felt layout wrong gives a digest that is individually well-formed and + /// collectively wrong. + /// + /// It is a provability gate rather than a value comparison, because that is + /// what the leg is — it emits assertions and publishes nothing. A real + /// ground-out nonce must PROVE; the negative control is what makes that + /// mean something. + /// + /// ⚠ **The control is chosen by the host's verdict, never as `nonce + 1`.** + /// At factor 8 one nonce in 256 is valid, so the successor of a valid nonce + /// is itself valid with probability 2⁻⁸ — and `generate_nonce`'s `find_any` + /// re-samples the same few chunk-start winners run after run, so once a + /// recurring winner has a valid successor the "control" recurs as a failure + /// that looks like a value-specific emitter defect: a winner with a valid + /// successor was once asked to REJECT that successor, which was asking the + /// leg to disagree with the specification. The control is therefore the + /// first nonce after the winner that `is_valid_nonce` REJECTS, asserted as + /// such before the leg is asked to agree. + #[test] + fn the_emitted_grinding_check_accepts_exactly_the_hosts_nonces() { + use crate::lfm::registry::build_artifacts_with_hasher; + use stark::proof::options::GoldilocksCubicProofOptions; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("options"); + let program = grinding_leg_program(); + let seed = grinding_seed_bytes(); + + for (hasher, grind, host_accepts) in grinding_members() { + // HOST: a real ground-out nonce for this hasher. WHICH one is + // scheduling-dependent (`find_any`), and that is the point of + // grinding here rather than pinning: every run samples the leg at + // a fresh point of the winners' set. + let nonce = grind(&seed, GRINDING_FACTOR).expect("a nonce must exist at factor 8"); + assert!( + host_accepts(&seed, nonce, GRINDING_FACTOR), + "{hasher:?}: generate_nonce must return a nonce is_valid_nonce accepts" + ); + // The control: the first host-REJECTED nonce after the winner. + let control = (1u64..=1 << 16) + .map(|k| nonce.wrapping_add(k)) + .find(|&n| !host_accepts(&seed, n, GRINDING_FACTOR)) + .expect("a rejected nonce exists within 2^16 of any nonce at factor 8"); + println!("{hasher:?}: ground nonce {nonce:#018x}, control {control:#018x}"); + + let artifacts = build_artifacts_with_hasher(&program, &opts, hasher); + assert!( + leg_proves(&program, &artifacts, &opts, hasher, nonce), + "{hasher:?}: the host's own ground-out nonce {nonce:#018x} must prove" + ); + // ⚠ THE CONTROL. Without it "it proved" says nothing: an emitter + // that asserted nothing would pass the line above. + assert!( + !leg_proves(&program, &artifacts, &opts, hasher, control), + "{hasher:?}: a nonce the host rejects ({control:#018x}) must be unprovable" + ); + } + } + + /// ★ **THE INCIDENT NONCES, WITHOUT GRINDING.** The leg's job is to agree + /// with the host on every nonce, valid or not. This starts at the nonce + /// pair 0x1400000000000009 / 0x140000000000000A (both valid under the + /// retired RPO tenant at the leg tests' seed, which is what once made a + /// `nonce + 1` control fire), then walks forward SEQUENTIALLY — no + /// `find_any`, no scheduling — until each tenant has shown both a + /// host-accepted and a host-rejected nonce, and asserts the leg's verdict + /// equals the host's on every one of them. The arena is constructed + /// directly; nothing here grinds, so the run is deterministic. + #[test] + fn the_emitted_grinding_check_agrees_with_the_host_on_the_incident_nonces() { + use crate::lfm::registry::build_artifacts_with_hasher; + use stark::proof::options::GoldilocksCubicProofOptions; + + const INCIDENT: u64 = 0x1400_0000_0000_0009; + + let seed = grinding_seed_bytes(); + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("options"); + let program = grinding_leg_program(); + + for (hasher, _, host_accepts) in grinding_members() { + // The pinned pair, then forward until both verdicts are present: + // `seen[0]` = a rejected nonce is in the list, `seen[1]` = an + // accepted one is. + let mut nonces = vec![INCIDENT, INCIDENT + 1]; + let mut seen = [false; 2]; + for n in &nonces { + seen[usize::from(host_accepts(&seed, *n, GRINDING_FACTOR))] = true; + } + for k in 2..=(1u64 << 16) { + if seen == [true, true] { + break; + } + let n = INCIDENT + k; + let v = usize::from(host_accepts(&seed, n, GRINDING_FACTOR)); + if !seen[v] { + seen[v] = true; + nonces.push(n); + } + } + assert_eq!( + seen, + [true, true], + "{hasher:?}: both verdicts must appear within 2^16 of the incident" + ); + + let artifacts = build_artifacts_with_hasher(&program, &opts, hasher); + for n in nonces { + let host = host_accepts(&seed, n, GRINDING_FACTOR); + let machine = leg_proves(&program, &artifacts, &opts, hasher, n); + println!("{hasher:?}: nonce {n:#018x} host={host} machine={machine}"); + assert_eq!( + machine, host, + "{hasher:?}: nonce {n:#018x}: the emitted check must agree with is_valid_nonce" + ); + } + } + } + + /// ★★ **THE GRINDING GATE** — the host grinding digest and the in-VM + /// computation of it must agree, for every tenant. + /// + /// ⚠ Grinding is verified INSIDE the machine (`transcript_replay.rs`), so + /// this is not a host-only concern: if the two sides disagree, a proof-of- + /// work the prover found is one the verifier cannot confirm, and that fails + /// as an unprovable program rather than as a wrong answer. + /// + /// The preimage is `state ‖ nonce` — 32 + 8 bytes, five felts, **one rate + /// block** — so the machine side is a single `MODE_P` row. Every cell it + /// feeds is DERIVED from the exported rules ([`single_block_leaf_cells`]), + /// never restated: this lane has now written a host↔machine encoding three + /// times and twice a differential caught a machine side that had + /// hand-written a constant agreeing with the rule only until the rule moved. + #[test] + fn the_host_grinding_digest_and_the_machine_agree() { + use crate::lfm::builder::LfmBuilder; + use crate::lfm::compiler::compile; + use crate::lfm::proof::{lfm_prove_with_hasher, verify_against}; + use crate::lfm::registry::build_artifacts_with_hasher; + use stark::proof::options::GoldilocksCubicProofOptions; + + const SEED: [u8; 32] = [ + 0x9e, 0x37, 0x79, 0xb9, 0x7f, 0x4a, 0x7c, 0x00, 0xd1, 0xb5, 0x4a, 0x32, 0xd1, 0x92, + 0xed, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x11, 0x22, 0x33, 0x44, + 0x55, 0x66, 0x77, 0x00, + ]; + const NONCE: u64 = 0x0123_4567_89AB_CD00; + + fn check(name: &str) { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("options"); + + // HOST: the grinding digest through the public `digest` interface. + let mut d = AlgebraicDigest::::default(); + digest::Digest::update(&mut d, SEED); + digest::Digest::update(&mut d, NONCE.to_be_bytes()); + let want = d.finalize_digest(); + // ⚠ The trait path must agree with the inherent one, or grinding — + // which reaches this through `digest::Digest` — computes something + // this test never checked. + let via_trait: Commitment = digest::Digest::finalize(d.clone()).into(); + assert_eq!( + via_trait, want, + "{name}: Digest::finalize must be finalize_digest" + ); + + // The preimage's felts, and the MODE_P cells, both from the rules. + let mut preimage = SEED.to_vec(); + preimage.extend_from_slice(&NONCE.to_be_bytes()); + let felts = felts_from_bytes(&preimage); + assert_eq!(felts.len(), 5, "state ‖ nonce is five felts, one block"); + let cells = single_block_leaf_cells(&felts); + + // MACHINE: one permutation row, publishing the digest cell. + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let rate0 = b.hint_word(arena, 0); + let rate1 = b.hint_word(arena, 1); + let cap = b.digest_const(cells[2]); + let out = b.permute([rate0, rate1, cap.as_cell()]); + b.public(out[0]); + let program = compile(b.finish()); + + let artifacts = build_artifacts_with_hasher(&program, &opts, H::KIND); + let proved = lfm_prove_with_hasher( + &program, + &artifacts, + &[vec![cells[0], cells[1]]], + &opts, + H::KIND, + ) + .expect("the grinding row must prove"); + + assert_eq!( + digest_to_commitment(&proved.public_words[0].1), + want, + "{name}: the machine's grinding digest must be the host's" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{name}: the grinding proof must verify" + ); + } + for_each_tenant!(check); + } + + /// The `digest` interface must agree with the direct construction — i.e. + /// `update`-then-finalize is the leaf sponge over the concatenation, so a + /// caller that splits its updates gets the same grinding digest. + #[test] + fn the_grinding_digest_is_split_invariant() { + fn check(name: &str) { + let msg: Vec = (0..40u8).collect(); + let want = digest_to_commitment(&sponge_leaf(H::KIND, &felts_from_bytes(&msg))); + for cut in [0usize, 1, 8, 17, 32, 40] { + let mut d = AlgebraicDigest::::default(); + digest::Digest::update(&mut d, &msg[..cut]); + digest::Digest::update(&mut d, &msg[cut..]); + assert_eq!(d.finalize_digest(), want, "{name}: split at {cut}"); + } + } + for_each_tenant!(check); + } + + /// ★★★ **THE EMITTER GATE** — the in-VM leaf and parent must equal this + /// module's host constructions, for every tenant. + /// + /// This is the pair the whole migration turns on: the host commits with + /// `hash_data` / `hash_new_parent`, and the wrap program re-derives them + /// with `WrapHash::Algebraic`'s `leaf_hash` / `hash_pair`. If they disagree + /// the walk reconstructs nothing and the failure surfaces as a `DivByZero` + /// deep in a query walk, naming neither the hash nor the site — so it is + /// gated here, where a divergence is a failing test. + /// + /// The leaf is exercised at lengths that cross the rate boundary in both + /// directions, because the padding flag (`len mod 8`) is the one part of + /// the construction that is not the same on every block. + #[test] + fn the_emitted_leaf_and_parent_equal_the_host_backend() { + use crate::lfm::builder::LfmBuilder; + use crate::lfm::compiler::compile; + use crate::lfm::edsl::{WrapDigest, WrapHash}; + use crate::lfm::proof::lfm_prove_with_hasher; + use crate::lfm::registry::build_artifacts_with_hasher; + use stark::proof::options::GoldilocksCubicProofOptions; + + fn check(name: &str) { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("options"); + + for n in [1usize, 4, 8, 9, 12, 16] { + let leaf = base_leaf(n); + let want = + as IsMerkleTreeBackend>::hash_data(&leaf); + + // The program reads the leaf's felts from an arena, four per + // word, and hashes them the way a wrap program would. + let words = n.div_ceil(4); + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let arena = b.declare_arena(words as u32); + let felts: Vec<_> = (0..words) + .flat_map(|w| { + let c = b.hint_word(arena, w as u32); + b.unpack(c).to_vec() + }) + .take(n) + .collect(); + let d = WrapHash::Algebraic.leaf_hash(&mut b, &felts); + assert_eq!(d.len(), 1, "{name}: an algebraic digest is ONE cell"); + b.public(d[0]); + let program = compile(b.finish()); + + let arena_words: Vec = (0..words) + .map(|w| { + core::array::from_fn(|i| { + leaf.get(4 * w + i).copied().unwrap_or_else(FE::zero) + }) + }) + .collect(); + let artifacts = build_artifacts_with_hasher(&program, &opts, H::KIND); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &[arena_words], &opts, H::KIND) + .expect("the leaf program must prove"); + assert_eq!( + digest_to_commitment(&proved.public_words[0].1), + want, + "{name}: emitted leaf of {n} felts must equal the host's" + ); + } + + // ★ EXTENSION leaves — the decomposition the call sites rely on. + // + // `sub_proof::emit_leaf_hash` absorbs an Fp3 value as + // `unpack(cell)[..3]`, deleting the byte serialisation. + // That is correct only if the host's own decomposition agrees: + // `write_bytes_be` for an Fp3 element writes components 0, 1, 2 in + // order. Verified by reading, and gated here so it stays true. + for n in [1usize, 2, 3, 5] { + let leaf = ext_leaf(n); + let want = as IsMerkleTreeBackend>::hash_data(&leaf); + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let arena = b.declare_arena(n as u32); + let felts: Vec<_> = (0..n) + .flat_map(|i| { + let c = b.hint_word(arena, i as u32); + b.unpack(c)[..3].to_vec() + }) + .collect(); + let d = WrapHash::Algebraic.leaf_hash(&mut b, &felts); + b.public(d[0]); + let program = compile(b.finish()); + + let words: Vec = leaf + .iter() + .map(|e| { + let v = e.value(); + [v[0], v[1], v[2], FE::zero()] + }) + .collect(); + let artifacts = build_artifacts_with_hasher(&program, &opts, H::KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &[words], &opts, H::KIND) + .expect("the extension leaf program must prove"); + assert_eq!( + digest_to_commitment(&proved.public_words[0].1), + want, + "{name}: emitted EXTENSION leaf of {n} elements must equal the host's" + ); + } + + // And the parent. + let l = digest_to_commitment(&[FE::from(3u64), FE::from(5), FE::from(7), FE::from(11)]); + let r = + digest_to_commitment(&[FE::from(13u64), FE::from(17), FE::from(19), FE::from(23)]); + let want = + as IsMerkleTreeBackend>::hash_new_parent(&l, &r); + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let arena = b.declare_arena(2); + let lc = WrapDigest::from_cell(b.hint_word(arena, 0)); + let rc = WrapDigest::from_cell(b.hint_word(arena, 1)); + let d = WrapHash::Algebraic.hash_pair(&mut b, lc, rc); + b.public(d[0]); + let program = compile(b.finish()); + let artifacts = build_artifacts_with_hasher(&program, &opts, H::KIND); + let proved = lfm_prove_with_hasher( + &program, + &artifacts, + &[vec![commitment_to_digest(&l), commitment_to_digest(&r)]], + &opts, + H::KIND, + ) + .expect("the parent program must prove"); + assert_eq!( + digest_to_commitment(&proved.public_words[0].1), + want, + "{name}: emitted parent must equal the host's" + ); + } + for_each_tenant!(check); + } + + /// The leaf is DOMAIN-SEPARATED from a parent: hashing the eight felts of + /// two digests as a LEAF must not equal compressing them as a PARENT. + #[test] + fn a_leaf_over_eight_felts_is_not_the_parent_of_those_two_digests() { + fn check(name: &str) { + let felts: Vec> = (1..=8u64).map(FE::from).collect(); + let leaf = as IsMerkleTreeBackend>::hash_data(&felts); + let l = digest_to_commitment(&[felts[0], felts[1], felts[2], felts[3]]); + let r = digest_to_commitment(&[felts[4], felts[5], felts[6], felts[7]]); + let par = + as IsMerkleTreeBackend>::hash_new_parent(&l, &r); + assert_ne!(leaf, par, "{name}: leaf and parent domains must differ"); + } + for_each_tenant!(check); + } +} diff --git a/prover/src/lfm/algebraic_transcript.rs b/prover/src/lfm/algebraic_transcript.rs new file mode 100644 index 000000000..594edeb59 --- /dev/null +++ b/prover/src/lfm/algebraic_transcript.rs @@ -0,0 +1,1016 @@ +//! The Fiat–Shamir transcript for an ALGEBRAIC commitment hash — the host side +//! of the seam whose other side is [`super::edsl::SpongeVar`]. +//! +//! # The problem this module exists to solve (item A2) +//! +//! A wrap program verifies a proof the HOST produced, which means the in-VM +//! transcript replay must re-derive exactly the challenges the host derived. Get +//! that wrong and Fiat–Shamir does not fail loudly — the walk reconstructs +//! nothing, some difference that should have been non-zero is inverted, and the +//! executor reports `DivByZero` at an address that names neither the hash nor +//! the site (the diagnostic signature `edsl::WrapHash::production`'s header +//! warns about). +//! +//! Under the byte hashes the two sides agree because both are byte sponges. An +//! algebraic hash absorbs FIELD ELEMENTS, while `IsTranscript` — the trait the +//! STARK prover and verifier are generic over — offers `append_bytes`. **The +//! encoding between those two is the convention this module pins.** +//! +//! # ✓ What made this tractable +//! +//! Three facts, each verified rather than assumed: +//! +//! 1. **The transcript is a caller-supplied parameter, not a fixed type.** +//! `prove`/`verify` take `transcript: &mut impl IsStarkTranscript<..>`, so an +//! algebraic transcript is a new type, not a modification of +//! `DefaultTranscript`. Nothing the byte path uses is edited. +//! 2. **`IsTranscript` is already felt-native where it matters** — +//! `append_field_element` and `sample_field_element` speak +//! `FieldElement`, not bytes. Only `append_bytes` and `state()` are byte-typed. +//! 3. **`append_bytes` has a tiny, regular call surface.** Across the whole +//! STARK core it is called with exactly two things: **32-byte Merkle roots** +//! and **8-byte integers** (query indices, heights, widths, the grinding +//! nonce). A 32-byte root under an algebraic hash *is* four felts, and the +//! in-VM `SpongeVar::absorb` consumes exactly one four-felt cell. +//! +//! # ★ THE CONVENTION +//! +//! The state is ONE cell, zero-initialised, and every step is one `LFM_HASH` +//! transcript-domain step — identical to [`super::edsl::SpongeVar`], because +//! matching it is the entire point. +//! +//! | operation | absorbs | +//! |---|---| +//! | `append_bytes(b)` | `[len(b), 0, 0, 0]` as a DIGEST cell, then `⌈len/32⌉` payload cells, each 32 bytes read as four canonical BIG-endian `u64` felts, the last zero-padded | +//! | `append_field_element(x)` | `x`'s three Fp3 coefficients as one DATA cell `[x0, x1, x2, 0]`, through the LEAF encoding | +//! | `state()` | the four state felts, canonical big-endian — the grinding seed | +//! | `sample_field_element()` | squeeze one cell, read lanes 0–2 as Fp3 | +//! | `sample_u64(n)` | squeeze one cell, lane 0 canonical, masked to `n − 1` | +//! +//! ## Why the length prefix — it is the injectivity argument +//! +//! ⚠ Without it the absorb is **not injective** and Fiat–Shamir is not binding: +//! a 32-byte root and an 8-byte integer would both become one cell, so a root +//! whose last 24 bytes are zero would be indistinguishable from the integer in +//! its first 8. Prefixing the byte length separates every call by construction, +//! at a cost of one extra cell per call — and `append_bytes` runs on the order +//! of 10² times per verify, so the cost is noise. +//! +//! The rule is deliberately UNIFORM rather than special-cased on 32 and 8. A +//! conditional encoding is how a collision gets introduced later by someone +//! adding a third call shape. +//! +//! ## ⚠ Byte order: ONE rule, and integers come in byte-swapped +//! +//! Felt↔byte is **canonical BIG-endian** everywhere on the algebraic path — +//! the transcript, the Merkle nodes and the leaf buffers — because that is +//! what `ByteConversion::write_bytes_be` already does for a Goldilocks felt +//! (`canonical_u64().to_be_bytes()`), and that is how every leaf the STARK +//! serialises reaches a commitment backend. A little-endian island here and a +//! big-endian one there is how a root gets produced that nobody can reproduce. +//! +//! The consequence, stated because it looks wrong at first glance: the STARK +//! core hands `append_bytes` **little-endian** integers +//! (`(idx as u64).to_le_bytes()`), so the resulting felt is the byte-SWAP of +//! that integer. This is harmless — those call sites carry compile-time +//! constants, so an emitter materialises whatever felt the rule produces and +//! no runtime swap exists anywhere — but it must be DERIVED from the rule at +//! both ends rather than assumed. The differential gate caught precisely this: +//! a machine side that hand-wrote `FE::from(idx)` disagreed with the host. +//! +//! ## Why DIGEST cells for bytes and a DATA cell for field elements +//! +//! It mirrors `SpongeVar` exactly: `absorb` is for digests, `absorb_felts` for +//! data, and the two are different hash domains. Roots and lengths are digests +//! and small integers — opaque, already committed. A field element is data that +//! an adversary chooses, so it enters through the LEAF encoding, the same way +//! data enters a Merkle tree. Using one domain for both would give a program +//! that absorbs a root the ability to claim it absorbed a field element. +//! +//! ## `sample_u64` is CONSTANT-CONSUMPTION, and it has to be +//! +//! ✓ VERIFIED every call site in the STARK core passes a power of two +//! (`domain_size >> 1`, `1 << (h_max − 1)`, `1 << 9`). The incumbent +//! `DefaultTranscript` rejection-samples, but at a power-of-two bound its +//! threshold is zero and the loop never rejects — which is why the in-VM replay +//! can encode one draw. This implementation masks, so it consumes exactly one +//! cell per draw unconditionally: a straight-line machine cannot emit a loop +//! whose trip count depends on a sampled value (`SOUNDNESS.md` §6.3). +//! +//! # What pins it +//! +//! [`tests::the_host_transcript_and_the_machine_replay_derive_the_same_challenges`] +//! is the gate: it emits a program that drives `SpongeVar` through the same +//! sequence, proves it, and compares the machine's published challenges against +//! this type's. A divergence fails there rather than inside a query walk. + +use math::field::traits::IsPrimeField; + +use crypto::fiat_shamir::is_transcript::{IsStarkTranscript, IsTranscript}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::edsl::SQUEEZE_MARK; +use super::hash::{HasherKind, LfmHasher}; +use super::word::LfmWord; + +/// Bytes one cell carries: four Goldilocks felts, canonical little-endian. +pub const BYTES_PER_CELL: usize = 4 * 8; + +/// The Fiat–Shamir transcript over an algebraic `LFM_HASH` tenant. +/// +/// Parameterised by [`HasherKind`] rather than fixed, so RPO, RPX and the +/// Poseidon reference share one implementation and a new candidate costs +/// nothing here — the objective this whole lane is organised around. +#[derive(Clone, Debug)] +pub struct AlgebraicTranscript { + state: LfmWord, + squeeze_index: u32, + hasher: HasherKind, +} + +impl AlgebraicTranscript { + /// A fresh transcript under `hasher`, state zero. + pub fn new(hasher: HasherKind) -> Self { + Self { + state: [FE::zero(); 4], + squeeze_index: 0, + hasher, + } + } + + /// A fresh transcript under `hasher` with `seed` absorbed — the counterpart + /// of `DefaultTranscript::new(seed)`. + /// + /// ★ **The seed is the transcript's FIRST `append_bytes` call and nothing + /// more special than that.** That is what lets the machine's + /// `TranscriptReplay::new` mirror it with a single `append_const_bytes`, and + /// saying it as a constructor rather than leaving every caller to remember + /// the first absorb is what stops the two sides disagreeing about whether + /// the seed is INSIDE the transcript or beside it. The byte transcript makes + /// that choice in its constructor; so does this. + pub fn with_seed(hasher: HasherKind, seed: &[u8]) -> Self { + let mut t = Self::new(hasher); + t.append_bytes(seed); + t + } + + /// `SQ(i) = [SQUEEZE_MARK, i, 0, 0]` — the advance operand, identical to + /// `SpongeVar`'s. + pub fn squeeze_operand(i: u32) -> LfmWord { + [ + FE::from(u64::from(SQUEEZE_MARK)), + FE::from(u64::from(i)), + FE::zero(), + FE::zero(), + ] + } + + /// The state as it stands — what the KATs pin and what `state()` serialises. + pub fn state_word(&self) -> LfmWord { + self.state + } + + /// One transcript step against a DIGEST cell. + pub fn absorb_cell(&mut self, c: &LfmWord) { + self.state = self.hasher.transcript(&self.state, c); + } + + /// Absorb a cell of four arbitrary FIELD ELEMENTS, through the leaf + /// encoding — the DATA domain. + pub fn absorb_felts(&mut self, c: &LfmWord) { + let d = self.hasher.leaf(&[FE::zero(); 4], c); + self.absorb_cell(&d); + } + + /// Output the current state, then advance past it with `SQ(i)`. + /// + /// Output-then-advance, so no squeezed value is ever the state a later step + /// absorbs into — the ordering `SpongeVar::squeeze_cell` documents. + pub fn squeeze_cell(&mut self) -> LfmWord { + let out = self.state; + let sq = Self::squeeze_operand(self.squeeze_index); + self.state = self.hasher.transcript(&self.state, &sq); + self.squeeze_index += 1; + out + } + + /// ★ **THE `append_bytes` RULE, stated once.** The cells that call absorbs, + /// in order: the length, then the payload in 32-byte groups. + /// + /// ⚠ **Exported so the MACHINE side derives from it rather than restating + /// it.** Every constant an emitter needs for a byte absorb comes from here. + /// This is the third time this lane has written a host↔machine encoding, + /// and twice the differential caught a machine side that had hand-written a + /// constant which agreed with the rule until the rule moved. A restated + /// convention is a convention with two definitions, and the second one is + /// always the one that rots. + pub fn append_bytes_cells(bytes: &[u8]) -> Vec { + let mut cells = Vec::with_capacity(1 + bytes.len().div_ceil(BYTES_PER_CELL)); + cells.push([ + FE::from(bytes.len() as u64), + FE::zero(), + FE::zero(), + FE::zero(), + ]); + cells.extend(bytes.chunks(BYTES_PER_CELL).map(Self::bytes_to_cell)); + cells + } + + /// ★ **THE `append_field_element` RULE, stated once** — an Fp3 element as + /// one DATA cell. Exported for the same reason as + /// [`Self::append_bytes_cells`]. + pub fn field_element_cell(element: &FEE) -> LfmWord { + let v = element.value(); + [v[0], v[1], v[2], FE::zero()] + } + + /// Read 32 bytes as four canonical BIG-endian felts. The inverse of + /// [`Self::cell_to_bytes`]. + /// + /// ★ **Big-endian, and that is not arbitrary.** It is the convention + /// `ByteConversion::write_bytes_be` already uses for a Goldilocks felt + /// (`canonical_u64().to_be_bytes()`), which is how every leaf the STARK + /// serialises reaches a commitment backend. One rule for felt↔byte across + /// the whole algebraic path — the transcript, the Merkle nodes and the leaf + /// buffers — rather than a little-endian island here and a big-endian one + /// there, which is the kind of asymmetry that produces a root nobody can + /// reproduce. + /// + /// ⚠ Bytes reaching here are Merkle roots produced by an algebraic backend, + /// i.e. already canonical felts. A non-canonical eight-byte group would + /// reduce, and reduction is what would make two different roots absorb + /// identically — so the backend's serialisation being canonical is a + /// PRECONDITION of this convention, not a detail. + pub fn bytes_to_cell(chunk: &[u8]) -> LfmWord { + core::array::from_fn(|i| { + let mut b = [0u8; 8]; + let start = i * 8; + if start < chunk.len() { + let end = (start + 8).min(chunk.len()); + b[..end - start].copy_from_slice(&chunk[start..end]); + } + FE::from(u64::from_be_bytes(b)) + }) + } + + /// Four felts as 32 canonical big-endian bytes — the inverse of + /// [`Self::bytes_to_cell`], and the same rule `write_bytes_be` uses. + pub fn cell_to_bytes(c: &LfmWord) -> [u8; BYTES_PER_CELL] { + let mut out = [0u8; BYTES_PER_CELL]; + for (i, f) in c.iter().enumerate() { + let v = GoldilocksField::canonical(f.value()); + out[i * 8..(i + 1) * 8].copy_from_slice(&v.to_be_bytes()); + } + out + } +} + +impl IsTranscript for AlgebraicTranscript { + /// An Fp3 element as one DATA cell `[x0, x1, x2, 0]`. + fn append_field_element(&mut self, element: &FEE) { + self.absorb_felts(&Self::field_element_cell(element)); + } + + /// The length prefix, then the payload in 32-byte cells. See the module + /// header for why the prefix is the injectivity argument. + fn append_bytes(&mut self, new_bytes: &[u8]) { + for cell in Self::append_bytes_cells(new_bytes) { + self.absorb_cell(&cell); + } + } + + /// The four state felts, canonical big-endian — the grinding seed. + fn state(&self) -> [u8; 32] { + Self::cell_to_bytes(&self.state) + } + + fn sample_field_element(&mut self) -> FEE { + let c = self.squeeze_cell(); + FEE::new([c[0], c[1], c[2]]) + } + + /// ⚠ Constant-consumption: exactly one cell, always. See the module header. + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + debug_assert!(upper_bound > 0, "upper_bound must be greater than 0"); + debug_assert!( + upper_bound.is_power_of_two(), + "sample_u64 is masked, so a non-power-of-two bound ({upper_bound}) would be \ + biased; every STARK call site passes a power of two" + ); + let c = self.squeeze_cell(); + GoldilocksField::canonical(c[0].value()) & (upper_bound - 1) + } +} + +impl IsStarkTranscript for AlgebraicTranscript {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lfm::builder::{LfmBuilder, LfmProgramSource}; + use crate::lfm::compiler::compile; + use crate::lfm::edsl::{self, SpongeVar}; + use crate::lfm::proof::{lfm_prove_with_hasher, verify_against}; + use crate::lfm::registry::build_artifacts_with_hasher; + use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + + /// The hashers this convention is defined for. `Test` is included on + /// purpose: it is not cryptographic, but it exercises the same wiring, so a + /// break in the ENCODING shows up under both rather than being mistaken + /// for something about one permutation. + const ALGEBRAIC: [HasherKind; 2] = [HasherKind::Test, HasherKind::Rpx]; + + fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") + } + + /// A 32-byte "root" and an 8-byte integer — the only two shapes the STARK + /// core ever hands `append_bytes`. + const ROOT: [u8; 32] = [ + 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x00, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0x00, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, + 0x32, 0x00, + ]; + const SMALL: u64 = 0x0123_4567; + const QUERY_BITS: usize = 5; + + /// The three Fp3 coefficients the field-element absorb uses. + fn field_element() -> FEE { + FEE::new([FE::from(7u64), FE::from(11u64), FE::from(13u64)]) + } + + /// The HOST side of the sequence, through the public `IsTranscript` API. + fn host_challenges(hasher: HasherKind) -> (FEE, FEE, u64) { + let mut t = AlgebraicTranscript::new(hasher); + t.append_bytes(&ROOT); + let a = t.sample_field_element(); + t.append_bytes(&SMALL.to_le_bytes()); + let b = t.sample_field_element(); + t.append_field_element(&field_element()); + let q = t.sample_u64(1 << QUERY_BITS); + (a, b, q) + } + + /// The MACHINE side: the same sequence expressed in `SpongeVar` ops, with + /// every challenge published so a proof carries them where a verifier can + /// check them. + /// + /// ⚠ This is written from the CONVENTION, not from the host implementation + /// — the length cell, the payload cell, the leaf-encoded field element and + /// the masked draw are each spelled out here. That is what makes the test a + /// differential rather than a tautology: if the host and this disagree + /// about the encoding, they disagree about the challenges. + fn replay_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let root = b.hint_word(arena, 0); + let felts = b.hint_word(arena, 1); + + let mut sponge = SpongeVar::new(&mut b); + + // append_bytes(ROOT) — every cell DERIVED from the rule. The payload + // arrives as an arena word so the program reads real data; the length + // cell is a program constant, and which constant is the rule's to say. + let root_cells = AlgebraicTranscript::append_bytes_cells(&ROOT); + let len32 = b.digest_const(root_cells[0]); + sponge.absorb(&mut b, len32.as_cell()); + sponge.absorb(&mut b, root); + let a = sponge.squeeze_ext(&mut b); + + // append_bytes(SMALL.to_le_bytes()): length cell, then the value cell. + let small_cells = AlgebraicTranscript::append_bytes_cells(&SMALL.to_le_bytes()); + let len8 = b.digest_const(small_cells[0]); + sponge.absorb(&mut b, len8.as_cell()); + // append_bytes(SMALL.to_le_bytes()) — again every cell from the rule. + // ⚠ The payload felt is the byte-SWAP of the integer, because the STARK + // hands `append_bytes` LITTLE-endian integers and the one felt↔byte rule + // reads BIG-endian. Harmless — these are compile-time constants, so an + // emitter materialises whatever the rule produces and no runtime swap + // exists — but a machine side that hand-wrote `FE::from(SMALL)` is what + // this gate caught, which is why nothing here is hand-written. + let small = b.digest_const(small_cells[1]); + sponge.absorb(&mut b, small.as_cell()); + let bb = sponge.squeeze_ext(&mut b); + + // append_field_element: DATA, so the leaf encoding. + sponge.absorb_felts(&mut b, felts); + + b.public(a.as_cell()); + b.public(bb.as_cell()); + let bits = sponge.squeeze_bits(&mut b, QUERY_BITS); + let q = edsl::bits_to_felt(&mut b, &bits); + b.public(q.as_cell()); + b.finish() + } + + fn replay_arena() -> Vec> { + // Both arena words DERIVED from the rule, not restated. + let root_cell = AlgebraicTranscript::append_bytes_cells(&ROOT)[1]; + vec![vec![ + root_cell, + AlgebraicTranscript::field_element_cell(&field_element()), + ]] + } + + /// ★★ **THE A2 GATE.** The host transcript and the in-VM replay must derive + /// the SAME challenges, for every algebraic tenant. + /// + /// This is the one open correctness question in the algebraic swap: get the + /// byte↔felt encoding wrong and Fiat–Shamir does not fail loudly, it fails + /// as a `DivByZero` deep inside a query walk that names neither the hash nor + /// the site. Gating it here means a divergence is a failing test. + #[test] + fn the_host_transcript_and_the_machine_replay_derive_the_same_challenges() { + let opts = options(); + let program = compile(replay_program_source()); + for hasher in ALGEBRAIC { + let (a, b, q) = host_challenges(hasher); + let artifacts = build_artifacts_with_hasher(&program, &opts, hasher); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &replay_arena(), &opts, hasher) + .expect("the replay program must prove"); + + let pub_a = proved.public_words[0].1; + let pub_b = proved.public_words[1].1; + let pub_q = proved.public_words[2].1; + + assert_eq!( + [pub_a[0], pub_a[1], pub_a[2]], + *a.value(), + "{hasher:?}: first sampled field element must agree" + ); + assert_eq!( + [pub_b[0], pub_b[1], pub_b[2]], + *b.value(), + "{hasher:?}: second sampled field element must agree" + ); + assert_eq!( + GoldilocksField::canonical(pub_q[0].value()), + q, + "{hasher:?}: sampled query index must agree" + ); + + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{hasher:?}: the replay proof must verify" + ); + } + } + + /// The seed both sides start from — the shape `DefaultTranscript::new` + /// takes, so the replay's own constructor is under test and not assumed. + const SEED: &[u8] = b"lfm-algebraic-replay-gate-v0"; + + /// The HOST side of the sequence the EMITTER gate mirrors. + /// + /// Deliberately the same four absorb shapes production uses and no others — + /// ✓ VERIFIED against `crypto/stark/src/verifier.rs`, whose entire absorb + /// vocabulary is `append_bytes(root)`, `append_bytes()`, `append_field_element()` and `append_bytes()`. + fn host_replay_challenges(hasher: HasherKind) -> (FEE, FEE, u64, LfmWord) { + let mut t = AlgebraicTranscript::with_seed(hasher, SEED); + t.append_bytes(&ROOT); + let a = t.sample_field_element(); + t.append_bytes(&SMALL.to_le_bytes()); + t.append_field_element(&field_element()); + let b = t.sample_field_element(); + let q = t.sample_u64(1 << QUERY_BITS); + (a, b, q, AlgebraicTranscript::bytes_to_cell(&t.state())) + } + + /// The MACHINE side through `TranscriptReplay` — the emitter production + /// actually uses, not a hand-rolled `SpongeVar` sequence. + /// + /// ★ That is what makes this a strictly stronger gate than the A2 one above. + /// A2 checks that the CONVENTION agrees on both sides; this checks that the + /// replay every wrap program is emitted through IMPLEMENTS that convention — + /// the length prefixes, the one-call-per-host-call boundary, the halves→BE + /// felt regrouping, the single-squeeze ext draw and the state read. + fn replay_emitter_program_source() -> LfmProgramSource { + use crate::lfm::edsl::WrapHash; + use crate::lfm::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let arena = b.declare_arena(3); + // The root exactly as production carries one: two arena words of four + // `u32` halves each, which is `epoch::RootCells::hint`'s shape. So the + // regrouping gadget runs on the real shape rather than a convenient one. + let root_words = [b.hint_word(arena, 0), b.hint_word(arena, 1)]; + let ext_word = b.hint_word(arena, 2); + + let mut t = TranscriptReplay::new(SEED); + t.append_digest(&mut b, &root_words); + let a = t.sample_ext(&mut b); + t.append_const_bytes(&SMALL.to_le_bytes()); + let lanes = b.unpack(ext_word); + t.append_ext(&mut b, [lanes[0], lanes[1], lanes[2]]); + let bb = t.sample_ext(&mut b); + let bits = t.sample_u64_pow2(&mut b, QUERY_BITS); + let q = edsl::bits_to_felt(&mut b, &bits); + let state = t.state(&mut b); + assert_eq!( + state.len(), + 1, + "an algebraic transcript state is ONE cell, not two" + ); + + b.public(a.as_cell()); + b.public(bb.as_cell()); + b.public(q.as_cell()); + b.public(state[0]); + b.finish() + } + + fn replay_emitter_arena() -> Vec> { + // The root as the proof arena lays a commitment out: half `h` is bytes + // `4h..4h+4` LITTLE-endian, four halves to a word. Nothing here is the + // algebraic encoding — that is the point, the machine has to build it. + let mut words: Vec = ROOT + .chunks(16) + .map(|w| { + core::array::from_fn(|j| { + let mut le = [0u8; 4]; + le.copy_from_slice(&w[4 * j..4 * j + 4]); + FE::from(u64::from(u32::from_le_bytes(le))) + }) + }) + .collect(); + words.push(AlgebraicTranscript::field_element_cell(&field_element())); + vec![words] + } + + /// ★★★ **THE PHASE A GATE** — the absorbs between the statement and the + /// first challenge, which is where the spine's `z` diverges. + /// + /// The shared LogUp `z` is drawn after exactly two things: the statement + /// absorb, which its own gate covers, and Phase A. This is Phase A, beside + /// the spine rather than instrumented inside it — an instrument inside it + /// perturbs the program and produced a `DivByZero` of its own when tried. + /// + /// The sequence is every preprocessed root from the AIR set, then the main + /// root, then the pair. Synthetic roots, because what is under test is the + /// SEQUENCE and the ENCODING, and controlling both sides is what makes a + /// disagreement attributable. + #[test] + fn phase_a_absorbs_derive_the_hosts_shared_pair() { + use crate::lfm::builder::LfmBuilder; + use crate::lfm::compiler::compile; + use crate::lfm::edsl::WrapHash; + use crate::lfm::epoch::RootCells; + use crate::lfm::proof::lfm_prove_with_hasher; + use crate::lfm::registry::build_artifacts_with_hasher; + use crate::lfm::transcript_replay::TranscriptReplay; + + // Two preprocessed roots and one main root, distinct and non-canonical + // in their high bytes so a reduction would show. + let root_at = + |k: u8| -> [u8; 32] { core::array::from_fn(|i| k ^ (i as u8).wrapping_mul(29)) }; + let preps = [root_at(0x11), root_at(0x22)]; + let main = root_at(0x33); + + for hasher in ALGEBRAIC { + // HOST: the roots, through the production transcript object. + let mut host = AlgebraicTranscript::with_seed(hasher, SEED); + for p in &preps { + host.append_bytes(p); + } + host.append_bytes(&main); + let want = host.sample_field_element(); + + // MACHINE: the roots as program constants — + // `RootCells::constant`'s provenance, which is what a preprocessed + // root from the AIR set is. + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let mut t = TranscriptReplay::new(SEED); + // ⚠ BOTH production constructions, on the same host call. A + // preprocessed root reaches the transcript one of two ways: program + // TEXT goes through + // `append_const_bytes` as literal bytes, proof-carried cells through + // `RootCells::absorb`. Under a byte hash those are the same 32 + // bytes; under an algebraic one they are a byte cellification and a + // digest cell, which is only the same thing if `bytes_to_cell` and + // `commitment_to_digest` agree — the property this asserts rather + // than assumes. An earlier version of this gate drove only the + // second arm and claimed to cover the phase. + t.append_const_bytes(&preps[0][..]); + let cells = RootCells::constant(&mut b, &preps[1]); + cells.absorb_misaligned(&mut b, &mut t); + let main_cells = RootCells::constant(&mut b, &main); + main_cells.absorb_misaligned(&mut b, &mut t); + let z = t.sample_ext(&mut b); + b.public(z.as_cell()); + let program = compile(b.finish()); + + let artifacts = build_artifacts_with_hasher(&program, &options(), hasher); + let proved = lfm_prove_with_hasher(&program, &artifacts, &[], &options(), hasher) + .expect("the Phase A program must prove"); + let got = proved.public_words[0].1; + assert_eq!( + [got[0], got[1], got[2]], + *want.value(), + "{hasher:?}: Phase A must derive the host's shared pair — one call per \ + root, then the pair" + ); + } + } + + /// ★★★ **THE STATEMENT GATE** — the call SEQUENCE, which was fixed by + /// reading and never checked by running. + /// + /// `absorb_epoch_statement` claims to mirror + /// `statement::absorb_statement_with_digest`'s `append_bytes` calls one for + /// one. That claim was made by comparing the two functions by eye when the + /// append call boundary turned out to be load-bearing, and reading is + /// exactly what has been wrong three times in this migration. + /// + /// ⚠ **A byte transcript cannot check this claim.** It concatenates, so a + /// miscounted or coalesced call is invisible there — which is why the + /// existing statement drift test passes either way and why this gate has to + /// be algebraic. Under length prefixing, one call too many or too few makes + /// every challenge downstream different, and the statement is absorbed + /// FIRST, so "different" means the whole leg. + /// + /// It drives BOTH sides from the same shape data, so what it compares is the + /// two absorb SEQUENCES and nothing else. + #[test] + fn the_statement_replay_absorbs_the_hosts_calls_one_for_one() { + use crate::lfm::builder::{Felt, LfmBuilder}; + use crate::lfm::compiler::compile; + use crate::lfm::edsl::WrapHash; + use crate::lfm::proof::lfm_prove_with_hasher; + use crate::lfm::registry::build_artifacts_with_hasher; + use crate::lfm::statement_replay::{ + EpochStatementShape, EpochStatementVars, NUM_TABLE_COUNTS, absorb_epoch_statement, + }; + use crate::lfm::transcript_replay::TranscriptReplay; + use crate::lfm::word::base_word; + use crate::statement::{StatementKind, absorb_statement_with_digest}; + use crate::{RuntimePageRange, TableCounts}; + + // A public output whose length is neither a multiple of four nor of 32, + // and a page-range list with more than one entry — the two places the + // sequence has a LOOP, which is where a call-count error would live. + const OUTPUT_LEN: usize = 37; + const EPOCH_LABEL: u64 = 0x0000_002A_0000_0007; + let elf_digest: [u8; 32] = core::array::from_fn(|i| 0x40 ^ (i as u8).wrapping_mul(11)); + let public_output: Vec = (0..OUTPUT_LEN) + .map(|i| 0x90 ^ (i as u8).wrapping_mul(23)) + .collect(); + let page_ranges = [ + RuntimePageRange { + base: 0x1000, + count: 3, + }, + RuntimePageRange { + base: 0x9000, + count: 1, + }, + ]; + // Distinct per field, in the host's declaration order, so a transposed + // pair moves the transcript. + let counts = TableCounts { + cpu: 11, + lt: 12, + memw: 13, + memw_aligned: 14, + load: 15, + mul: 16, + dvrm: 17, + shift: 18, + branch: 19, + memw_register: 20, + eq: 21, + bytewise: 22, + store: 23, + cpu32: 24, + keccak: 1, + keccak_rnd: 1, + ecsm: 1, + ecdas: 1, + hint: 1, + copy: 1, + blake3: 1, + is_wh: 1, + ecrec: 1, + }; + // ⚠ Written out rather than taken from `table_count_values`, on purpose: + // the point of this test is that the MACHINE side reproduces the host + // encoding, so the machine's array must be an independent statement of + // it. Sharing the helper would compare the host with itself. + let count_array: [u64; NUM_TABLE_COUNTS] = [ + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ]; + const PAGES: usize = 6; + const FPLD: u8 = 3; + // The statement's last byte. `false` is the role fourteen of a block's + // fifteen epochs carry, so it is the one worth fixing here; the final + // epoch's `true` differs by one byte and by nothing structural. + const IS_FINAL: bool = false; + + // Machine-side halves: four bytes each, LITTLE-endian, the arena layout. + let to_halves = |bytes: &[u8]| -> Vec { + bytes + .chunks(4) + .map(|c| { + let mut le = [0u8; 4]; + le[..c.len()].copy_from_slice(c); + FE::from(u64::from(u32::from_le_bytes(le))) + }) + .collect() + }; + let digest_halves = to_halves(&elf_digest); + let output_halves = to_halves(&public_output); + // A u64 carried as [low32, high32] — `to_le_bytes` needs no manipulation. + let label_halves = to_halves(&EPOCH_LABEL.to_le_bytes()); + + let shape = EpochStatementShape { + public_output_len: OUTPUT_LEN, + table_counts: count_array, + num_private_input_pages: PAGES as u64, + fri_final_poly_log_degree: FPLD, + page_ranges: page_ranges.iter().map(|r| (r.base, r.count)).collect(), + is_final: IS_FINAL, + }; + + for hasher in ALGEBRAIC { + // HOST: production's own statement absorb, into an algebraic + // transcript, then a challenge. + let mut host = AlgebraicTranscript::with_seed(hasher, SEED); + absorb_statement_with_digest( + &mut host, + StatementKind::ContinuationEpoch { + epoch_label: EPOCH_LABEL, + is_final: IS_FINAL, + }, + &elf_digest, + &public_output, + &counts, + PAGES, + &page_ranges, + FPLD, + ); + let want = host.sample_field_element(); + + // MACHINE: the replay, over the same shape, from arena halves. + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let total = digest_halves.len() + output_halves.len() + label_halves.len(); + let arena = b.declare_arena(total as u32); + let all: Vec = (0..total).map(|i| b.hint_felt(arena, i as u32)).collect(); + let (d_cells, rest) = all.split_at(digest_halves.len()); + let (o_cells, l_cells) = rest.split_at(output_halves.len()); + + let mut t = TranscriptReplay::new(SEED); + absorb_epoch_statement( + &mut t, + &shape, + &EpochStatementVars { + elf_digest: d_cells, + public_output: o_cells, + epoch_label: l_cells, + }, + ); + let a = t.sample_ext(&mut b); + b.public(a.as_cell()); + let program = compile(b.finish()); + + let arena_words: Vec = digest_halves + .iter() + .chain(output_halves.iter()) + .chain(label_halves.iter()) + .map(|h| base_word(*h)) + .collect(); + let artifacts = build_artifacts_with_hasher(&program, &options(), hasher); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &[arena_words], &options(), hasher) + .expect("the statement replay program must prove"); + + let got = proved.public_words[0].1; + assert_eq!( + [got[0], got[1], got[2]], + *want.value(), + "{hasher:?}: the statement replay must absorb the host's calls one for one — \ + a coalesced or miscounted call is invisible under a byte transcript and \ + diverges every challenge under this one" + ); + } + } + + /// ★★ **THE MACHINE-BYTES GATE** — every absorb shape the statement leg + /// produces, including the ones the emitter gate above does not reach. + /// + /// That gate drives a 32-byte root through `cells_from_halves_be`, so the + /// ALIGNED whole-half path is covered. The statement leg also produces two + /// shapes it never exercises, and both come from real data rather than from + /// a corner case anyone chose: + /// + /// - a byte length that is **not a multiple of four** — `public_output` is + /// collected one byte per COMMIT, so its length is whatever the workload + /// produced, and the trailing half is masked to its live bytes; + /// - a length that is not a multiple of **32**, so the final payload cell is + /// short and `bytes_to_cell` reads a zero-filled buffer LEFT-justified. + /// + /// ⚠ Those two paddings are independent and both are silent when wrong: the + /// statement is absorbed FIRST, so a single wrong felt here diverges every + /// challenge downstream and surfaces as a `DivByZero` deep in a query walk, + /// naming neither the encoding nor the site. + /// + /// The expectation comes from `AlgebraicTranscript` itself — the same host + /// object production would absorb through — so this compares the machine's + /// regrouping against the rule rather than against a second copy of it. + #[test] + fn the_machine_regroups_bytes_exactly_as_the_host_cellifies_them() { + use crate::lfm::builder::{Felt, LfmBuilder}; + use crate::lfm::compiler::compile; + use crate::lfm::edsl::WrapHash; + use crate::lfm::proof::lfm_prove_with_hasher; + use crate::lfm::registry::build_artifacts_with_hasher; + use crate::lfm::transcript_replay::TranscriptReplay; + use crate::lfm::word::base_word; + + // Lengths chosen on the axes that can hide a break: the 4-byte half + // boundary (the mask) and the 32-byte cell boundary (the left-justified + // tail), each hit exactly and missed in both directions. + const LENGTHS: [usize; 11] = [1, 3, 4, 5, 7, 8, 31, 32, 33, 40, 67]; + + for hasher in ALGEBRAIC { + for len in LENGTHS { + // Distinct, high-bit-set bytes: a dropped or misplaced byte + // moves the felt rather than colliding with a zero. + let bytes: Vec = (0..len) + .map(|i| 0x80 ^ (i as u8).wrapping_mul(37)) + .collect(); + + // HOST: the transcript absorbs those bytes and squeezes. + let mut host = AlgebraicTranscript::with_seed(hasher, SEED); + host.append_bytes(&bytes); + let want = host.sample_field_element(); + + // MACHINE: the same bytes arrive as u32 halves, four bytes each + // LITTLE-endian — `proof_arena`'s layout — through the misaligned + // path, which is the one the statement leg uses. + let halves: Vec = bytes + .chunks(4) + .map(|c| { + let mut le = [0u8; 4]; + le[..c.len()].copy_from_slice(c); + FE::from(u64::from(u32::from_le_bytes(le))) + }) + .collect(); + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let arena = b.declare_arena(halves.len() as u32); + let half_cells: Vec = (0..halves.len()) + .map(|i| b.hint_felt(arena, i as u32)) + .collect(); + let mut t = TranscriptReplay::new(SEED); + t.append_bytes_misaligned(&half_cells, len); + let a = t.sample_ext(&mut b); + b.public(a.as_cell()); + let program = compile(b.finish()); + + let arena_words: Vec = halves.iter().map(|h| base_word(*h)).collect(); + let artifacts = build_artifacts_with_hasher(&program, &options(), hasher); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &[arena_words], &options(), hasher) + .expect("the regrouping program must prove"); + + let got = proved.public_words[0].1; + assert_eq!( + [got[0], got[1], got[2]], + *want.value(), + "{hasher:?} / {len} bytes: the machine's regrouping must be the host's \ + cellification — the trailing half's mask and the short payload cell's \ + left-justification are the two places this differs" + ); + } + } + } + + /// ★★ **THE EMITTER GATE.** `TranscriptReplay` on the algebraic arm must + /// derive the host transcript's challenges — and its STATE — for every + /// algebraic tenant. + /// + /// The state is compared as well as the challenges because grinding seeds + /// from it: a replay that agreed on every challenge and disagreed on the + /// state would pass a challenge-only gate and then grind against the wrong + /// seed, which surfaces as an unprovable nonce check rather than as anything + /// that names the transcript. + #[test] + fn the_emitter_replay_derives_the_host_transcripts_challenges_and_state() { + let opts = options(); + let program = compile(replay_emitter_program_source()); + for hasher in ALGEBRAIC { + let (a, b, q, state) = host_replay_challenges(hasher); + let artifacts = build_artifacts_with_hasher(&program, &opts, hasher); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &replay_emitter_arena(), &opts, hasher) + .expect("the emitter replay program must prove"); + + let pub_a = proved.public_words[0].1; + let pub_b = proved.public_words[1].1; + let pub_q = proved.public_words[2].1; + let pub_state = proved.public_words[3].1; + + assert_eq!( + [pub_a[0], pub_a[1], pub_a[2]], + *a.value(), + "{hasher:?}: the first challenge must agree" + ); + assert_eq!( + [pub_b[0], pub_b[1], pub_b[2]], + *b.value(), + "{hasher:?}: the second challenge must agree — it follows a \ + CONSTANT absorb and a field-element absorb, so a disagreement \ + here and not above is about those two encodings" + ); + assert_eq!( + GoldilocksField::canonical(pub_q[0].value()), + q, + "{hasher:?}: the sampled query index must agree" + ); + assert_eq!( + pub_state, state, + "{hasher:?}: the transcript STATE must agree — this is grinding's seed" + ); + + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{hasher:?}: the emitter replay proof must verify" + ); + } + } + + /// ⚠ The length prefix is what makes the absorb INJECTIVE, and this is the + /// collision it prevents: a 32-byte root whose tail is zero and an 8-byte + /// integer holding the same leading bytes would otherwise absorb + /// identically. + #[test] + fn the_length_prefix_separates_a_root_from_an_integer() { + let mut padded = [0u8; 32]; + padded[..8].copy_from_slice(&SMALL.to_le_bytes()); + + let mut with_root = AlgebraicTranscript::new(HasherKind::Rpx); + with_root.append_bytes(&padded); + + let mut with_int = AlgebraicTranscript::new(HasherKind::Rpx); + with_int.append_bytes(&SMALL.to_le_bytes()); + + assert_ne!( + with_root.state_word(), + with_int.state_word(), + "a 32-byte zero-padded value and the 8-byte value must not collide" + ); + + // And the control: without the length they WOULD be the same cell. + assert_eq!( + AlgebraicTranscript::bytes_to_cell(&padded), + AlgebraicTranscript::bytes_to_cell(&SMALL.to_le_bytes()), + "the payload cells are identical — the prefix is the only separator" + ); + } + + /// The byte round-trip must be exact, or `state()` does not name the state. + #[test] + fn a_cell_round_trips_through_its_canonical_bytes() { + let c: LfmWord = [ + FE::from(1u64), + FE::from(0xFFFF_FFFF_0000_0000u64), + FE::zero(), + FE::from(12345u64), + ]; + assert_eq!( + AlgebraicTranscript::bytes_to_cell(&AlgebraicTranscript::cell_to_bytes(&c)), + c + ); + } + + /// `sample_u64` consumes exactly one cell per draw regardless of the value + /// drawn — the property a straight-line replay depends on. + #[test] + fn sampling_is_constant_consumption() { + let mut t = AlgebraicTranscript::new(HasherKind::Rpx); + for expected in 1..=8u32 { + let _ = t.sample_u64(1 << QUERY_BITS); + assert_eq!(t.squeeze_index, expected, "one squeeze per draw"); + } + } +} diff --git a/prover/src/lfm/blake3.rs b/prover/src/lfm/blake3.rs new file mode 100644 index 000000000..dbf1bab75 --- /dev/null +++ b/prover/src/lfm/blake3.rs @@ -0,0 +1,366 @@ +//! The BLAKE3 compression function the LFM chips are built from. +//! +//! **This module is a re-export.** The implementation lives at +//! [`crypto::hash::blake3`], which is the only crate the three callers that must +//! not disagree can all reach: the Merkle commitment backends (in `crypto`), the +//! `LFM_BLAKE3` chip and the `LFM_HASH` socket (here), and the CUDA kernels' +//! parity reference (in `math-cuda`, which `prover` depends on). A chip and a +//! commitment that hash identically because they call one function is a +//! different claim from two that agree today. +//! +//! Everything the chips used before is still reachable under this path and means +//! the same thing: [`blake3_compress_rounds`], [`blake3_compress_6round`], +//! [`BLAKE3_IV`], [`BLAKE3_MSG_PERMUTATION`], [`BLAKE3_ROUNDS`], +//! [`CANONICAL_VECTORS`] and [`canonical_expected_out`]. +//! +//! The round count travels with it: this crate's `blake3-6round` feature now +//! forwards to `crypto`'s, so [`BLAKE3_ROUNDS`] — and therefore +//! `blake3_socket::SOCKET_ROUNDS`, which is an alias of it — is one symbol for +//! the whole tree. Enabling `crypto/blake3-6round` alone would leave the chip at +//! 6 rounds and is caught by the `SOCKET_ROUNDS == BLAKE3_ROUNDS` assertion, +//! which is why that assertion stays. +//! +//! # Why the tests stayed here +//! +//! The falsification suite below — the negative controls that break one +//! convention at a time, and the `blake3` crate anchor — tests the primitive +//! *through this path*, which is the path the chips use. Keeping it here means +//! the re-export itself is covered: a shim that resolved to the wrong thing +//! would fail these, and moving them down would have made the chips' view of the +//! primitive untested. `crypto` has its own tests for the construction layer +//! ([`crypto::hash::blake3::chain`]) that this module does not use. + +pub use crypto::hash::blake3::*; + +#[cfg(test)] +mod tests { + use super::*; + + /// The conventions a wrong port could get wrong, as data. + /// + /// [`CANONICAL_VECTORS`] is supposed to pin every one of these. Naming them + /// in a struct is what lets the negative control break exactly one at a time. + #[derive(Clone, Copy)] + struct Conventions { + /// The four rotation amounts of `G`, in application order. + rot: [u32; 4], + /// The message-schedule permutation applied between rounds. + perm: [usize; 16], + rounds: usize, + } + + /// The conventions [`CANONICAL_VECTORS`] were generated under. `rounds` is + /// [`BLAKE3_SIX_ROUNDS`], not [`BLAKE3_ROUNDS`]: that table pins the 6-round + /// variant whatever the build is compiled for, and reading the knob here + /// would make the "7 rounds" control below silently stop discriminating at + /// the default. + const CANONICAL: Conventions = Conventions { + rot: [16, 12, 8, 7], + perm: BLAKE3_MSG_PERMUTATION, + rounds: BLAKE3_SIX_ROUNDS, + }; + + /// A deliberately *parameterised* compression, used only to build negative + /// controls: the same dataflow with [`Conventions`] as an input. + /// + /// It is NOT what [`blake3_compress_6round`] calls. Keeping the two apart + /// costs a duplicated loop and buys the thing rule 7 is about: the control + /// tests below compare this function's output against [`CANONICAL_VECTORS`] + /// — a constant that came from outside this file — so they stay meaningful + /// no matter how the real function is later refactored. + fn compress_variant(v: &Vector, c: Conventions) -> [u32; 16] { + let g = |s: &mut [u32; 16], a: usize, b: usize, cc: usize, d: usize, mx: u32, my: u32| { + s[a] = s[a].wrapping_add(s[b]).wrapping_add(mx); + s[d] = (s[d] ^ s[a]).rotate_right(c.rot[0]); + s[cc] = s[cc].wrapping_add(s[d]); + s[b] = (s[b] ^ s[cc]).rotate_right(c.rot[1]); + s[a] = s[a].wrapping_add(s[b]).wrapping_add(my); + s[d] = (s[d] ^ s[a]).rotate_right(c.rot[2]); + s[cc] = s[cc].wrapping_add(s[d]); + s[b] = (s[b] ^ s[cc]).rotate_right(c.rot[3]); + }; + let h = v.h; + let mut s: [u32; 16] = [ + h[0], + h[1], + h[2], + h[3], + h[4], + h[5], + h[6], + h[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + v.t as u32, + (v.t >> 32) as u32, + v.block_len, + v.flags, + ]; + let mut m = v.m; + for r in 0..c.rounds { + g(&mut s, 0, 4, 8, 12, m[0], m[1]); + g(&mut s, 1, 5, 9, 13, m[2], m[3]); + g(&mut s, 2, 6, 10, 14, m[4], m[5]); + g(&mut s, 3, 7, 11, 15, m[6], m[7]); + g(&mut s, 0, 5, 10, 15, m[8], m[9]); + g(&mut s, 1, 6, 11, 12, m[10], m[11]); + g(&mut s, 2, 7, 8, 13, m[12], m[13]); + g(&mut s, 3, 4, 9, 14, m[14], m[15]); + if r < c.rounds - 1 { + let prev = m; + for (i, &p) in c.perm.iter().enumerate() { + m[i] = prev[p]; + } + } + } + let mut out = [0u32; 16]; + for i in 0..8 { + out[i] = s[i] ^ s[i + 8]; + out[i + 8] = s[i + 8] ^ h[i]; + } + out + } + + /// The port reproduces all ten canonical vectors. + #[test] + fn the_compression_matches_the_canonical_six_round_vectors() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + assert_eq!( + blake3_compress_6round(&v.h, &v.m, v.t, v.block_len, v.flags), + v.out, + "canonical 6-round vector {i}" + ); + } + } + + /// The parameterised control, at canonical parameters, IS the port — so a + /// negative control below differs from the real thing in exactly the one + /// convention it names, and nothing else. + #[test] + fn the_variant_at_canonical_parameters_is_the_port() { + for v in CANONICAL_VECTORS.iter() { + assert_eq!( + compress_variant(v, CANONICAL), + v.out, + "the control must reproduce the vectors at canonical parameters" + ); + } + } + + /// NEGATIVE CONTROL (rule 9): each convention the vectors are supposed to + /// pin, broken one at a time, must stop reproducing them. + /// + /// Without this, "the vectors pass" would be evidence only that the vectors + /// are *reachable*, not that they discriminate. Each case names what would + /// silently be unpinned if it ever started passing. + #[test] + fn breaking_one_convention_at_a_time_breaks_the_vectors() { + // The message permutation transposed (its own inverse composition): + // same multiset of indices, same round count, different schedule. + let mut transposed = [0usize; 16]; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + transposed[p] = i; + } + let cases: [(&str, Conventions); 4] = [ + // rotr12 -> rotr13: the one rotation amount that is NOT a byte + // relabel in the chip, so a wrong value here is the wrong-rotation + // bug in its most consequential place. + ( + "rotr12 -> rotr13", + Conventions { + rot: [16, 13, 8, 7], + ..CANONICAL + }, + ), + // rotr16 and rotr8 swapped: both ARE free byte relabels in the + // chip, so transposing them costs no columns and no constraints — + // the cheapest possible way to be wrong. + ( + "rotr16 <-> rotr8", + Conventions { + rot: [8, 12, 16, 7], + ..CANONICAL + }, + ), + ( + "message schedule transposed", + Conventions { + perm: transposed, + ..CANONICAL + }, + ), + ( + "7 rounds (standard BLAKE3)", + Conventions { + rounds: 7, + ..CANONICAL + }, + ), + ]; + for (what, c) in cases { + let v = &CANONICAL_VECTORS[0]; + assert_ne!( + compress_variant(v, c), + v.out, + "{what} still reproduces the canonical vector — the vector does not pin it" + ); + } + } + + /// The port reproduces all ten canonical inputs at **7 rounds** too. + /// + /// [`CANONICAL_OUT_7ROUND`] came from two independently-written Python + /// references that agree on all ten, and whose 7-round paths are pinned by + /// the official BLAKE3 vectors. So this is the same shape of check as the + /// 6-round one above but with a stronger source, and together they are what + /// let `BLAKE3_ROUNDS` be flipped without the chip losing its vector pin. + #[test] + fn the_compression_matches_the_canonical_vectors_at_seven_rounds() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + assert_eq!( + blake3_compress_rounds( + &v.h, + &v.m, + v.t, + v.block_len, + v.flags, + BLAKE3_STANDARD_ROUNDS + ), + CANONICAL_OUT_7ROUND[i], + "7-round canonical vector {i}" + ); + } + } + + /// NEGATIVE CONTROL: the two tables really are different data. Without this, + /// a generation bug that emitted the 6-round outputs twice would leave the + /// test above passing and pinning nothing new. + #[test] + fn the_six_and_seven_round_vector_tables_differ_everywhere() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + assert_ne!(v.out, CANONICAL_OUT_7ROUND[i], "vector {i}"); + } + } + + /// `canonical_expected_out` selects the table matching the compiled knob. + /// This is the accessor `blake3_probe` asserts the chip's `OUT` columns + /// against, so a wrong branch here would silently unpin the chip. + #[test] + fn canonical_expected_out_follows_the_round_knob() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + let want = if BLAKE3_ROUNDS == BLAKE3_STANDARD_ROUNDS { + CANONICAL_OUT_7ROUND[i] + } else { + v.out + }; + assert_eq!(canonical_expected_out(i), want, "vector {i}"); + assert_eq!( + canonical_expected_out(i), + blake3_compress_rounds(&v.h, &v.m, v.t, v.block_len, v.flags, BLAKE3_ROUNDS), + "the accessor must agree with the primitive at the compiled round count" + ); + } + } + + /// ★ **The external anchor, direct.** At 7 rounds this module's compression + /// function IS standard BLAKE3, checked against the `blake3` crate with no + /// oracle, no JSON and no transcription in between. + /// + /// PLAN §2.2 step 4 asked for exactly this and Phase 1 deferred it for want + /// of a cargo dependency; this is that check, discharged. A message of at + /// most 64 bytes is one chunk and one block, so the whole tree hasher + /// collapses to a single `f` invocation: `h = IV`, the block zero-padded to + /// 64 bytes and read as 16 little-endian words, `t = 0`, `block_len` the + /// true length, `flags = CHUNK_START|CHUNK_END|ROOT`. The 32-byte digest is + /// `out[0..8]` in little-endian order. + /// + /// It runs over 65 lengths (0..=64) rather than one, because the length is + /// what `block_len` and the padding both key off, and a port that ignored + /// `block_len` would still pass at a single length. + #[test] + fn seven_rounds_is_the_blake3_crate() { + const CHUNK_START: u32 = 1; + const CHUNK_END: u32 = 2; + const ROOT: u32 = 8; + + for len in 0..=64usize { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect(); + let mut block = [0u8; 64]; + block[..len].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + + let out = blake3_compress_rounds( + &BLAKE3_IV, + &words, + 0, + len as u32, + CHUNK_START | CHUNK_END | ROOT, + BLAKE3_STANDARD_ROUNDS, + ); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + + assert_eq!( + ours, + *blake3::hash(&msg).as_bytes(), + "7-round compression must equal the blake3 crate at length {len}" + ); + } + } + + /// NEGATIVE CONTROL for the anchor above: at 6 rounds it must NOT match. + /// + /// Without this, `seven_rounds_is_the_blake3_crate` would pass just as well + /// if `rounds` were being ignored — which is the one bug that would make the + /// whole external-anchor argument vacuous, since the 6-round variant's only + /// defence is "the same code path with the loop bound changed". + #[test] + fn six_rounds_is_not_the_blake3_crate() { + let msg: [u8; 36] = core::array::from_fn(|i| i as u8); + let mut block = [0u8; 64]; + block[..36].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + // BLAKE3_SIX_ROUNDS, not BLAKE3_ROUNDS: the knob defaults to 7, and + // reading it here would turn this control into a copy of the anchor. + let out = blake3_compress_rounds(&BLAKE3_IV, &words, 0, 36, 1 | 2 | 8, BLAKE3_SIX_ROUNDS); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_ne!(ours, *blake3::hash(&msg).as_bytes()); + } + + /// The counter split is load-bearing and full-width: `t` reaches the state + /// as two 32-bit halves in low-then-high order, so swapping them must move + /// the output. Six of the ten canonical vectors have distinct halves. + #[test] + fn the_counter_halves_are_not_interchangeable() { + let mut checked = 0; + for v in CANONICAL_VECTORS.iter() { + let swapped = v.t.rotate_left(32); + if swapped == v.t { + continue; + } + checked += 1; + assert_ne!( + blake3_compress_6round(&v.h, &v.m, swapped, v.block_len, v.flags), + v.out, + "swapping the counter halves must change the output" + ); + } + assert!( + checked >= 8, + "expected most vectors to have distinct halves, got {checked}" + ); + } +} diff --git a/prover/src/lfm/blake3_chip.rs b/prover/src/lfm/blake3_chip.rs new file mode 100644 index 000000000..5cd5c3fbd --- /dev/null +++ b/prover/src/lfm/blake3_chip.rs @@ -0,0 +1,1213 @@ +//! `LFM_BLAKE3` — the BLAKE3 compression chip, hosted on the LFM bus. +//! +//! Ported from PR #903's `prover/src/tables/blake3.rs` (`yetanotherco/lambda_vm`, +//! head `89aeeb8c2b0389e9d21a861c9e3a10a7b1b5704e`), which is the syscall +//! variant: it takes its inputs and returns its outputs through the VM's memory, +//! so its I/O side is an `Ecall` receiver, an x10 register read and 22 `Memw` +//! dword ops over a 176-byte state region. **The mixing core is unchanged.** +//! What this module replaces is the I/O side, with `LfmMem` word tokens in the +//! discipline `chips::keccak` (`LFM_KECCAK`) established: addresses and +//! multiplicities are preprocessed program data, and a machine word carries +//! four `u32` lanes. +//! +//! # What the swap costs and buys, send for send +//! +//! | | #903 (syscall) | here (LFM) | +//! |---|---|---| +//! | `Ecall` receiver | 1 | — | +//! | `Memw` x10 register read | 1 | — | +//! | `Memw` per state dword | 22 | — | +//! | `LfmMem` word tokens | — | 7 reads + 4 writes + 2 reversed-digest = 13 | +//! | `ByteAlu[XOR]` mixing + feed-forward | 832 | 832 | +//! | `AreBytes` shift halfwords | 384 | 384 | +//! | `AreBytes` message bytes | 32 | 32 | +//! | `AreBytes` OLD_OUT bytes | 32 | — | +//! | `AreBytes` addr bytes + alignment `AND` | 5 | — | +//! | `IsHalfword` pointer halfwords | 88 | — | +//! | **total interactions** | **1,397** | **1,261** | +//! | value columns | 3,219 | 3,056 | +//! +//! (The two reversed-digest sends arrived with registration; the standalone +//! chip measured 1,259. They cost no value column — see +//! `layout::blake3::REV_ADDR0`.) +//! +//! The dropped columns are `TIMESTAMP` (2), `ADDR` (8), `PTR` (88) and +//! `OLD_OUT` (64) — 162 — and `MU` moves into the preprocessed prefix, which +//! the census excludes, for 163 in total. +//! +//! # Why dropping those range checks is sound, not just cheaper +//! +//! Each dropped lookup guarded something that no longer exists: +//! +//! - **`OLD_OUT`'s 32 `AreBytes`.** #903 needs them because the previous memory +//! content of the out region appears only in the `Memw` write ops' `old` +//! field — never XOR-consumed, so its packed linear combinations could alias. +//! An `LfmMem` write carries no `old` field; there are no such columns here. +//! - **The address bytes, the alignment `AND` and the 88 pointer `IsHalfword`s.** +//! #903's state address is prover witness read out of x10 and must be +//! range-checked and shown 8-aligned before 22 pointers are derived from it. +//! Here every address is a *preprocessed* column supplied by the program and +//! vouched by the admission validator, exactly as for every other LFM chip — +//! a prover cannot choose it at all. +//! +//! What is NOT dropped is the byte-range coverage of the data columns, and it +//! carries over intact: +//! +//! - all 64 `m` bytes keep their explicit `AreBytes` (they are never XORed); +//! - `h`'s 32 bytes are XOR operands of the feed-forward (`out[i+8] = v[i+8] ^ h[i]`); +//! - `t_lo`, `t_hi`, `block_len`, `flags` are `v[12..16]`, each the `vd` operand +//! of a round-0 `G`, hence an operand of that `G`'s first XOR; +//! - all 64 `OUT` bytes are *results* of feed-forward XOR lookups. +//! +//! So every byte column reaching an `LfmMem` token is range-checked before the +//! token recomposes it, and a `u32` lane — four values below 2^8 with +//! coefficients 1, 2^8, 2^16, 2^24 — cannot reach 2^32. This is the same +//! transitive argument `chips::keccak` records for its 400 state bytes. +//! +//! # The single-dataflow rule, inherited +//! +//! The compression dataflow is written ONCE, in [`run_flow`], and interpreted +//! twice: [`WireFlow`] (columns — drives constraints and senders) and +//! [`ValueFlow`] (u32 witness — drives the trace and the BITWISE multiplicities). +//! The two cannot diverge on wiring, only on interpretation, which the probe's +//! bus-balance gate checks. That property is #903's and is worth preserving on +//! sight: it is why the sender list and the witness cannot drift apart. +//! +//! # Status +//! +//! This chip is **registered**: it is slot 11 of the fixed AIR set +//! (`airs::LFM_CHIP_NAMES`), reached from the eDSL through +//! [`super::builder::LfmBuilder::blake3_compress`]. `blake3_probe` still proves +//! it standalone against a synthetic `LfmMem` mirror, which is what prices the +//! chip on its own; `blake3_chip_tests` covers it as a member of the machine. +//! +//! ⚠ Round count follows [`super::blake3::BLAKE3_ROUNDS`]: 7 (standard BLAKE3) +//! by default, 6 under the `blake3-6round` feature. The 6-round instantiation +//! rests on the unratified security assumption **A6R**; the 7-round one carries +//! no assumption. Every column count in the table above is the 6-round one and +//! is quoted for continuity with #903 — `blake3_probe` pins both. + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use crate::constraints::templates::{INV_SHIFT_32, emit_is_bit}; +use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; +use crate::tables::types::{ + BusId, FE, GoldilocksExtension, GoldilocksField, alu_op, zeroed_fe_vec, +}; + +use super::blake3::{BLAKE3_IV, BLAKE3_MSG_PERMUTATION, BLAKE3_ROUNDS, blake3_compress_rounds}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// G-instances per compression: 8 per round, at the compiled round count. +pub const NUM_G: usize = BLAKE3_ROUNDS * 8; + +pub use super::layout::blake3::{IN_U32, IN_WORDS, OUT_U32, OUT_WORDS}; + +/// The (a, b, c, d) state indices of the 8 G-calls of one round: +/// 4 column mixes then 4 diagonal mixes (BLAKE3 spec §2.1). +const G_INDICES: [(usize, usize, usize, usize); 8] = [ + (0, 4, 8, 12), + (1, 5, 9, 13), + (2, 6, 10, 14), + (3, 7, 11, 15), + (0, 5, 10, 15), + (1, 6, 11, 12), + (2, 7, 8, 13), + (3, 4, 9, 14), +]; + +/// Shift amounts of the two non-free rotations, as `rotl` inner shifts: +/// rotr12 = rotl20 = rotl16∘rotl4 (r=4); rotr7 = rotl25 = rotl16∘rotl9 (r=9). +pub(crate) const ROT_SHIFT_R: [u32; 2] = [4, 9]; + +// ========================================================================= +// Column layout +// ========================================================================= + +/// The chip's columns: a preprocessed instruction group, then value columns. +/// +/// The prefix is [`crate::lfm::layout::blake3`], re-exported rather than +/// restated — it is the chip's *instruction* column group and belongs beside +/// every other chip's, which is what the compiler and the admission validator +/// read. What stays here is the mixing core's value layout, which nothing +/// outside this chip and `blake3_socket`'s shared G-block offsets shares. +pub mod cols { + pub use crate::lfm::layout::blake3::*; + + use super::NUM_G; + + // --- value columns --- + // + // ⚠ The trailing literals are ROUND-DEPENDENT: everything from `OUT` on is a + // function of `NUM_G = 8 · BLAKE3_ROUNDS`, so each is given at both counts + // rather than at whichever one the reader's build happens to compile. + // `the_registry_blessing_is_round_count_invariant` pins both, and pins that + // the PREPROCESSED prefix — which is what the registry commits — moves with + // neither. + /// Input bytes: `h[32] | m[64] | t_lo[4] | t_hi[4] | block_len[4] | flags[4]`. + pub const IN: usize = PREP_WIDTH; // 20 + /// `NUM_G` G-blocks × 60 cells (56 bytes + 4 carry bits). + pub const G: usize = IN + 4 * IN_U32; // 132 + pub const G_SIZE: usize = 60; + /// Feed-forward output bytes `out[0..16]` (64 bytes). + pub const OUT: usize = G + NUM_G * G_SIZE; // 3012 at 6 rounds, 3492 at 7 + + pub const NUM_COLUMNS: usize = OUT + 4 * OUT_U32; // 3076 at 6 rounds, 3556 at 7 + + /// Input word `i` (0..28: `h[0..8]`, `m[8..24]`, `t_lo=24`, `t_hi=25`, + /// `block_len=26`, `flags=27`), byte `b`. + #[inline] + pub const fn in_word(i: usize, b: usize) -> usize { + IN + i * 4 + b + } + + /// Feed-forward output word `i` (0..16), byte `b`. + #[inline] + pub const fn out_word(i: usize, b: usize) -> usize { + OUT + i * 4 + b + } + + /// Base column of G-block `g`. + #[inline] + pub const fn g_base(g: usize) -> usize { + G + g * G_SIZE + } + + // Offsets inside one G block (56 byte cells + 4 carry bits = 60): + /// add3 #1 output word (4 bytes). + pub const G_A1: usize = 0; + /// add3 #1 carry bits c1, c2. + pub const G_A1_C: usize = 4; + /// X1 = vd ^ A1 (4 bytes). + pub const G_X1: usize = 6; + /// add2 #1 output word (4 bytes). + pub const G_C1: usize = 10; + /// X2 = vb ^ C1 (4 bytes). + pub const G_X2: usize = 14; + /// rotr12 block: SLL_lo(2) SLLC_lo(2) SLL_hi(2) SLLC_hi(2) Y(4). + pub const G_R1: usize = 18; + /// add3 #2 output word (4 bytes). + pub const G_A2: usize = 30; + /// add3 #2 carry bits. + pub const G_A2_C: usize = 34; + /// X3 = vd ^ A2 (4 bytes). + pub const G_X3: usize = 36; + /// add2 #2 output word (4 bytes). + pub const G_C2: usize = 40; + /// X4 = B1 ^ C2 (4 bytes). + pub const G_X4: usize = 44; + /// rotr7 block: same layout as `G_R1`. + pub const G_R2: usize = 48; +} + +/// Value columns the census counts: everything past the preprocessed prefix. +pub const MAIN_COLUMNS: usize = cols::NUM_COLUMNS - cols::PREP_WIDTH; + +// ========================================================================= +// The single dataflow, interpreted twice (verbatim from #903) +// ========================================================================= + +/// The framing degrees of freedom [`run_flow`] itself decides, as opposed to +/// the ones an interpretation decides. +/// +/// These two live here, and not in each `Blake3Flow` impl, for one reason: they +/// change *which calls happen*, so an impl that got them wrong would silently +/// desynchronise the wire interpretation from the value interpretation and the +/// bus sends would stop matching the multiplicities. Deciding them in the single +/// dataflow is what keeps the single-dataflow rule true when there is more than +/// one framing (`blake3_chip`'s syscall shape and `blake3_socket`'s 2-to-1 +/// compress). +#[derive(Clone, Copy, Debug)] +pub(crate) struct FlowConfig { + /// Rounds of 8 G-calls. 6 for this chip; [`super::blake3_socket`] sweeps. + pub rounds: usize, + /// How many of the eight `out[i] = v[i] ^ v[i+8]` words to produce — the + /// truncation window. 8 here; 4 for the socket, whose digest is one cell. + pub out_window: usize, + /// Whether to produce `out[i+8] = v[i+8] ^ h[i]` as well. The socket does + /// not: those words are not part of a truncated 128-bit digest, and never + /// building them is where most of its saving over this chip comes from. + pub full_output: bool, +} + +impl FlowConfig { + /// The syscall-shaped chip's framing: the full 16-word output. + pub(crate) const fn full(rounds: usize) -> Self { + Self { + rounds, + out_window: 8, + full_output: true, + } + } +} + +/// The BLAKE3 compression dataflow, abstracted over its word representation. +pub(crate) trait Blake3Flow { + type Word: Copy; + + /// `h[i]` input word. + fn input_h(&mut self, i: usize) -> Self::Word; + /// `v[12..16]` init words: t_lo, t_hi, block_len, flags. + fn input_v12(&mut self, j: usize) -> Self::Word; + /// `IV[i]` constant (`v[8..12]`). + fn iv_const(&mut self, i: usize) -> Self::Word; + + /// 3-operand add `s = a + b + m[m_idx] mod 2^32` (half 0/1 = which add3 of G g). + fn add3( + &mut self, + g: usize, + half: usize, + a: Self::Word, + b: Self::Word, + m_idx: usize, + ) -> Self::Word; + /// 2-operand add `s = a + b mod 2^32`. + fn add2(&mut self, g: usize, half: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// XOR (slot 0..4 = X1..X4 of G g). Operand order is part of the wire format. + fn xor(&mut self, g: usize, slot: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// rotr16: free byte relabel `[b2,b3,b0,b1]`. + fn rotr16(&mut self, w: Self::Word) -> Self::Word; + /// rotr8: free byte relabel `[b1,b2,b3,b0]`. + fn rotr8(&mut self, w: Self::Word) -> Self::Word; + /// rotr12 (half=0) / rotr7 (half=1) via the inline shift identity. + fn rot_shift(&mut self, g: usize, half: usize, w: Self::Word) -> Self::Word; + /// Feed-forward, low half: `out[i] = v[i] ^ v[i+8]`. + fn feed_forward_low(&mut self, i: usize, vi: Self::Word, vi8: Self::Word); + /// Feed-forward, high half: `out[i+8] = v[i+8] ^ h[i]`. Called only under + /// [`FlowConfig::full_output`]. + fn feed_forward_high(&mut self, i: usize, vi8: Self::Word, hi: Self::Word); +} + +/// Drive the compression through `f`. The message schedule is tracked as +/// indices into the ORIGINAL m (permute^r composition), so both interpretations +/// reference original message words — never copies. +pub(crate) fn run_flow(f: &mut T, cfg: FlowConfig) { + let h: [T::Word; 8] = core::array::from_fn(|i| f.input_h(i)); + let mut v: [T::Word; 16] = core::array::from_fn(|i| { + if i < 8 { + h[i] + } else if i < 12 { + f.iv_const(i - 8) + } else { + f.input_v12(i - 12) + } + }); + + // sched[i] = index into the original m of the word consumed at position i + // this round. permute: m'[i] = m[P[i]] ⇒ sched'[i] = sched[P[i]]. + let mut sched: [usize; 16] = core::array::from_fn(|i| i); + + for r in 0..cfg.rounds { + for (j, &(ia, ib, ic, id)) in G_INDICES.iter().enumerate() { + let g = r * 8 + j; + let (va, vb, vc, vd) = (v[ia], v[ib], v[ic], v[id]); + let mx = sched[2 * j]; + let my = sched[2 * j + 1]; + + let a1 = f.add3(g, 0, va, vb, mx); + let x1 = f.xor(g, 0, vd, a1); + let vd1 = f.rotr16(x1); + let c1 = f.add2(g, 0, vc, vd1); + let x2 = f.xor(g, 1, vb, c1); + let b1 = f.rot_shift(g, 0, x2); // rotr12 + let a2 = f.add3(g, 1, a1, b1, my); + let x3 = f.xor(g, 2, vd1, a2); + let vd2 = f.rotr8(x3); + let c2 = f.add2(g, 1, c1, vd2); + let x4 = f.xor(g, 3, b1, c2); + let b2 = f.rot_shift(g, 1, x4); // rotr7 + + v[ia] = a2; + v[ib] = b2; + v[ic] = c2; + v[id] = vd2; + } + if r < cfg.rounds - 1 { + let prev = sched; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + sched[i] = prev[p]; + } + } + } + + for i in 0..cfg.out_window { + f.feed_forward_low(i, v[i], v[i + 8]); + if cfg.full_output { + f.feed_forward_high(i, v[i + 8], h[i]); + } + } +} + +// ========================================================================= +// Wire interpretation (columns) +// ========================================================================= + +/// A 32-bit word as wiring: four byte columns (LSB first), a constant, or a +/// constant selected by preprocessed mode columns. +/// Constants only ever appear as the IV `v[c]` operands of round-0 add2s. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WordRef { + Cols([usize; 4]), + Const(u32), + /// `Σ_k col_k · tag_k` — a constant selected by PREPROCESSED mode columns. + /// + /// The socket's domain tag, for when one tag is no longer enough. It is a + /// linear form over columns the prover cannot choose, so it is as unchosen + /// as the plain `Const` it replaces, and it costs the same: zero witness + /// columns, zero range checks, degree 1 where the constant was degree 0. + /// `add3`'s body is degree 1 in its operands either way, so the arm's max + /// degree does not move. + /// + /// It is a WHOLE-WORD form only — see [`WordRef::byte`]. + ModeSelected(&'static [(usize, u32)]), +} + +impl WordRef { + /// This word's byte `b`, for a byte-granular consumer (`ByteAlu[XOR]`). + /// + /// # Panics + /// + /// On [`WordRef::ModeSelected`]. A mode-selected word has no byte + /// decomposition without witnessing one, and the whole reason the tag lives + /// in a message word is that message words reach `add3` and nothing else. Panicking + /// says so out loud rather than letting a future byte consumer quietly + /// acquire four columns nobody committed. + pub(crate) fn byte(self, b: usize) -> ByteRef { + match self { + WordRef::Cols(c) => ByteRef::Col(c[b]), + WordRef::Const(w) => ByteRef::Const(((w >> (8 * b)) & 0xFF) as u8), + WordRef::ModeSelected(_) => unreachable!( + "a mode-selected word is a whole-word value: it reaches add3 and \ + nothing byte-granular, so it has no byte columns to name" + ), + } + } + + /// This word rotated right by `bytes` bytes — free wiring, no columns and + /// no constraint: `Cols` permutes its byte columns and `Const` rotates its + /// value. It is BLAKE3's `rotr16`/`rotr8`, whose shifts are byte-aligned. + /// + /// # Panics + /// + /// On [`WordRef::ModeSelected`], for the reason [`WordRef::byte`] gives: + /// the rotation is a byte permutation, and a mode-selected word has no + /// bytes to permute. + pub(crate) fn rotr_bytes(self, bytes: usize) -> WordRef { + match self { + WordRef::Cols(c) => WordRef::Cols(core::array::from_fn(|j| c[(j + bytes) % 4])), + WordRef::Const(v) => WordRef::Const(v.rotate_right(8 * bytes as u32)), + WordRef::ModeSelected(_) => unreachable!( + "a mode-selected word is a whole-word value: it has no byte \ + columns to rotate" + ), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ByteRef { + Col(usize), + Const(u8), +} + +/// One recorded 3-op add: operands (a, b, m), output columns, carries. +/// +/// `m` is a [`WordRef`] rather than four columns because the socket framing +/// makes every message word above the input lanes a compile-time constant — the +/// domain tag and the zero padding of a 52-byte message. Constant message words +/// cost no columns and no +/// range checks, which is the whole reason the tag is free there. +pub(crate) struct Add3Wire { + pub a: WordRef, + pub b: WordRef, + pub m: WordRef, + pub s: [usize; 4], + pub c1: usize, + pub c2: usize, +} + +/// One recorded 2-op add: operands, output columns (carry is an expression). +pub(crate) struct Add2Wire { + pub a: WordRef, + pub b: WordRef, + pub s: [usize; 4], +} + +/// One recorded XOR: per-byte operands and output columns. +pub(crate) struct XorWire { + pub a: WordRef, + pub b: WordRef, + pub out: [usize; 4], +} + +/// One recorded shift rotation: input word, the 8 shift-halfword byte columns +/// (SLL_lo, SLLC_lo, SLL_hi, SLLC_hi — 2 bytes each), output columns, r. +pub(crate) struct RotWire { + pub input: WordRef, + pub sll_lo: [usize; 2], + pub sllc_lo: [usize; 2], + pub sll_hi: [usize; 2], + pub sllc_hi: [usize; 2], + pub y: [usize; 4], + pub r: u32, +} + +/// The full wiring of one compression row, recorded in canonical order. +pub(crate) struct WireFlow { + pub add3s: Vec, + pub add2s: Vec, + pub xors: Vec, + pub rots: Vec, +} + +impl WireFlow { + pub(crate) fn build() -> Self { + let mut w = WireFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + 16), + rots: Vec::with_capacity(NUM_G * 2), + }; + run_flow(&mut w, FlowConfig::full(BLAKE3_ROUNDS)); + w + } +} + +#[inline] +pub(crate) fn word_cols(start: usize) -> [usize; 4] { + [start, start + 1, start + 2, start + 3] +} + +impl Blake3Flow for WireFlow { + type Word = WordRef; + + fn input_h(&mut self, i: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(i, 0))) + } + fn input_v12(&mut self, j: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(24 + j, 0))) + } + fn iv_const(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + fn add3(&mut self, g: usize, half: usize, a: WordRef, b: WordRef, m_idx: usize) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_A1 } else { cols::G_A2 }; + let cbase = cols::g_base(g) + + if half == 0 { + cols::G_A1_C + } else { + cols::G_A2_C + }; + let s = word_cols(base); + self.add3s.push(Add3Wire { + a, + b, + m: WordRef::Cols(word_cols(cols::in_word(8 + m_idx, 0))), + s, + c1: cbase, + c2: cbase + 1, + }); + WordRef::Cols(s) + } + + fn add2(&mut self, g: usize, half: usize, a: WordRef, b: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_C1 } else { cols::G_C2 }; + let s = word_cols(base); + self.add2s.push(Add2Wire { a, b, s }); + WordRef::Cols(s) + } + + fn xor(&mut self, g: usize, slot: usize, a: WordRef, b: WordRef) -> WordRef { + let off = match slot { + 0 => cols::G_X1, + 1 => cols::G_X2, + 2 => cols::G_X3, + _ => cols::G_X4, + }; + let out = word_cols(cols::g_base(g) + off); + self.xors.push(XorWire { a, b, out }); + WordRef::Cols(out) + } + + fn rotr16(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(2) + } + fn rotr8(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(1) + } + + fn rot_shift(&mut self, g: usize, half: usize, w: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_R1 } else { cols::G_R2 }; + let y = word_cols(base + 8); + self.rots.push(RotWire { + input: w, + sll_lo: [base, base + 1], + sllc_lo: [base + 2, base + 3], + sll_hi: [base + 4, base + 5], + sllc_hi: [base + 6, base + 7], + y, + r: ROT_SHIFT_R[half], + }); + WordRef::Cols(y) + } + + fn feed_forward_low(&mut self, i: usize, vi: WordRef, vi8: WordRef) { + self.xors.push(XorWire { + a: vi, + b: vi8, + out: word_cols(cols::out_word(i, 0)), + }); + } + + fn feed_forward_high(&mut self, i: usize, vi8: WordRef, hi: WordRef) { + self.xors.push(XorWire { + a: vi8, + b: hi, + out: word_cols(cols::out_word(i + 8, 0)), + }); + } +} + +// ========================================================================= +// Value interpretation (u32 witness) +// ========================================================================= + +/// Everything the trace filler and the BITWISE collector need for one +/// compression, recorded cell-exactly in the same canonical order as +/// [`WireFlow`]. +pub struct ValueFlow { + /// (s, c1, c2) per add3, canonical order. + pub add3s: Vec<(u32, u8, u8)>, + /// s per add2 (the carry is an expression, not a cell). + pub add2s: Vec, + /// (a, b, out) per XOR word, canonical order (Gs then feed-forward). + pub xors: Vec<(u32, u32, u32)>, + /// (sll_lo, sllc_lo, sll_hi, sllc_hi, y) per shift rotation. + pub rots: Vec<(u16, u16, u16, u16, u32)>, + /// The output words. Entries outside the framing's truncation window are + /// never computed and stay zero — reading one is a caller bug. + pub out: [u32; 16], + + h: [u32; 8], + m: [u32; 16], + v12: [u32; 4], +} + +impl ValueFlow { + /// The syscall-shaped chip's full 16-word compression. + pub fn compute(h: &[u32; 8], m: &[u32; 16], t: u64, block_len: u32, flags: u32) -> Self { + Self::compute_with(h, m, t, block_len, flags, FlowConfig::full(BLAKE3_ROUNDS)) + } + + /// [`ValueFlow::compute`] under an explicit framing. + pub(crate) fn compute_with( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, + cfg: FlowConfig, + ) -> Self { + let g = cfg.rounds * 8; + let mut f = ValueFlow { + add3s: Vec::with_capacity(g * 2), + add2s: Vec::with_capacity(g * 2), + xors: Vec::with_capacity(g * 4 + 16), + rots: Vec::with_capacity(g * 2), + out: [0; 16], + h: *h, + m: *m, + v12: [t as u32, (t >> 32) as u32, block_len, flags], + }; + run_flow(&mut f, cfg); + f + } +} + +impl Blake3Flow for ValueFlow { + type Word = u32; + + fn input_h(&mut self, i: usize) -> u32 { + self.h[i] + } + fn input_v12(&mut self, j: usize) -> u32 { + self.v12[j] + } + fn iv_const(&mut self, i: usize) -> u32 { + BLAKE3_IV[i] + } + + fn add3(&mut self, _g: usize, _half: usize, a: u32, b: u32, m_idx: usize) -> u32 { + let m = self.m[m_idx]; + let wide = a as u64 + b as u64 + m as u64; + let s = wide as u32; + let carry = (wide >> 32) as u8; // 0, 1 or 2 + // Two summed carry bits: c1 + c2 = carry. + let (c1, c2) = match carry { + 0 => (0, 0), + 1 => (1, 0), + _ => (1, 1), + }; + self.add3s.push((s, c1, c2)); + s + } + + fn add2(&mut self, _g: usize, _half: usize, a: u32, b: u32) -> u32 { + let s = a.wrapping_add(b); + self.add2s.push(s); + s + } + + fn xor(&mut self, _g: usize, _slot: usize, a: u32, b: u32) -> u32 { + let out = a ^ b; + self.xors.push((a, b, out)); + out + } + + fn rotr16(&mut self, w: u32) -> u32 { + w.rotate_right(16) + } + fn rotr8(&mut self, w: u32) -> u32 { + w.rotate_right(8) + } + + fn rot_shift(&mut self, _g: usize, half: usize, w: u32) -> u32 { + let r = ROT_SHIFT_R[half]; + let xlo = w & 0xFFFF; + let xhi = w >> 16; + // xlo·2^r = SLLC_lo·2^16 + SLL_lo (and same for hi): Euclidean split. + let sll_lo = ((xlo << r) & 0xFFFF) as u16; + let sllc_lo = ((xlo << r) >> 16) as u16; + let sll_hi = ((xhi << r) & 0xFFFF) as u16; + let sllc_hi = ((xhi << r) >> 16) as u16; + // Recombine + halfword swap: Ylo = SLL_hi + SLLC_lo, Yhi = SLL_lo + SLLC_hi. + let ylo = sll_hi as u32 + sllc_lo as u32; + let yhi = sll_lo as u32 + sllc_hi as u32; + let y = ylo | (yhi << 16); + debug_assert_eq!(y, w.rotate_right(if r == 4 { 12 } else { 7 })); + self.rots.push((sll_lo, sllc_lo, sll_hi, sllc_hi, y)); + y + } + + fn feed_forward_low(&mut self, i: usize, vi: u32, vi8: u32) { + let lo = vi ^ vi8; + self.xors.push((vi, vi8, lo)); + self.out[i] = lo; + } + + fn feed_forward_high(&mut self, i: usize, vi8: u32, hi: u32) { + let w = vi8 ^ hi; + self.xors.push((vi8, hi, w)); + self.out[i + 8] = w; + } +} + +// ========================================================================= +// Operation struct + trace generation +// ========================================================================= + +/// The VALUE half of one compression — everything the mixing core, the trace's +/// value columns and the BITWISE multiplicities depend on, and nothing else. +/// +/// This is what the machine's executor records, because the other half — +/// addresses and multiplicities — is *program* data living in the preprocessed +/// instruction group, exactly as for every other LFM chip. Keeping the split in +/// the type is what stops a witness from describing a compression at addresses +/// the committed program does not name. +#[derive(Debug, Clone, Copy)] +pub struct Blake3Values { + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, +} + +impl Blake3Values { + /// The 28 input `u32` words in machine order: `h | m | t_lo | t_hi | len | flags`. + pub fn input_words(&self) -> [u32; IN_U32] { + let mut w = [0u32; IN_U32]; + w[0..8].copy_from_slice(&self.h); + w[8..24].copy_from_slice(&self.m); + w[24] = self.t as u32; + w[25] = (self.t >> 32) as u32; + w[26] = self.block_len; + w[27] = self.flags; + w + } + + /// The compression output. + pub fn output_words(&self) -> [u32; OUT_U32] { + blake3_compress_rounds( + &self.h, + &self.m, + self.t, + self.block_len, + self.flags, + BLAKE3_ROUNDS, + ) + } +} + +/// One compression, as the standalone probe issues it: [`Blake3Values`] plus +/// the addresses and multiplicities a program would have supplied. +/// +/// The machine does not use this — it reads the prefix out of the committed +/// column group. It exists so `blake3_probe` can drive the chip against a +/// synthetic `LfmMem` mirror with no program behind it. +#[derive(Debug, Clone)] +pub struct Blake3Operation { + pub in_addr: [u64; IN_WORDS], + pub out_addr: [u64; OUT_WORDS], + pub read_counts: [u64; OUT_WORDS], + pub values: Blake3Values, +} + +/// Write a 32-bit word as 4 byte cells at `row[col..col+4]`. +#[inline] +fn set_word_bytes(row: &mut [FE], col: usize, w: u32) { + for b in 0..4 { + row[col + b] = FE::from(u64::from((w >> (8 * b)) & 0xFF)); + } +} + +/// ★ The chip's value columns for one compression, written into a row slice. +/// +/// The single definition, shared by the standalone probe's trace and by the +/// machine's ([`super::trace::build_traces_with_hasher`]). Registration made two +/// callers out of one, and a second transcription of this — 3,056 columns laid +/// out in a fixed order — is exactly the drift the single-dataflow rule above +/// exists to prevent: the constraints and the senders are generated from +/// [`WireFlow`], so a filler that disagreed with them would fail only as an +/// unbalanced bus at prove time, with no signature saying why. +/// +/// Writes nothing before `cols::IN`: the preprocessed prefix is the caller's, +/// and in the machine it is copied verbatim from the committed group. +pub fn fill_blake3_witness(row: &mut [FE], v: &Blake3Values) { + for (i, &w) in v.input_words().iter().enumerate() { + set_word_bytes(row, cols::in_word(i, 0), w); + } + + // The mixing core, cell-exactly in canonical order. + let flow = ValueFlow::compute(&v.h, &v.m, v.t, v.block_len, v.flags); + let mut a3 = flow.add3s.iter(); + let mut a2 = flow.add2s.iter(); + let mut xo = flow.xors.iter(); + let mut ro = flow.rots.iter(); + for g in 0..NUM_G { + let base = cols::g_base(g); + for half in 0..2 { + let (s_off, c_off, x_off, c2_off, x2_off, r_off) = if half == 0 { + ( + cols::G_A1, + cols::G_A1_C, + cols::G_X1, + cols::G_C1, + cols::G_X2, + cols::G_R1, + ) + } else { + ( + cols::G_A2, + cols::G_A2_C, + cols::G_X3, + cols::G_C2, + cols::G_X4, + cols::G_R2, + ) + }; + let &(s, c1, c2) = a3.next().expect("add3 count"); + set_word_bytes(row, base + s_off, s); + row[base + c_off] = FE::from(u64::from(c1)); + row[base + c_off + 1] = FE::from(u64::from(c2)); + + let &(_, _, x) = xo.next().expect("xor count"); + set_word_bytes(row, base + x_off, x); + + let &c = a2.next().expect("add2 count"); + set_word_bytes(row, base + c2_off, c); + + let &(_, _, x2) = xo.next().expect("xor count"); + set_word_bytes(row, base + x2_off, x2); + + let &(sll_lo, sllc_lo, sll_hi, sllc_hi, y) = ro.next().expect("rot count"); + row[base + r_off] = FE::from(u64::from(sll_lo & 0xFF)); + row[base + r_off + 1] = FE::from(u64::from(sll_lo >> 8)); + row[base + r_off + 2] = FE::from(u64::from(sllc_lo & 0xFF)); + row[base + r_off + 3] = FE::from(u64::from(sllc_lo >> 8)); + row[base + r_off + 4] = FE::from(u64::from(sll_hi & 0xFF)); + row[base + r_off + 5] = FE::from(u64::from(sll_hi >> 8)); + row[base + r_off + 6] = FE::from(u64::from(sllc_hi & 0xFF)); + row[base + r_off + 7] = FE::from(u64::from(sllc_hi >> 8)); + set_word_bytes(row, base + r_off + 8, y); + } + } + for i in 0..OUT_U32 { + set_word_bytes(row, cols::out_word(i, 0), flow.out[i]); + } +} + +/// One row per compression; padding rows are ALL ZERO. +/// +/// #903 needs a nonzero pad (`ptr[k] = 8k`) because its pointer columns carry an +/// ungated `addr + 8k` identity. Nothing here is ungated except `IS_BIT(MU)`, +/// which a zero row satisfies, so the pad is genuinely empty — and +/// `padding_rows_are_all_zero` in `blake3_probe` pins that rather than assuming +/// it. +/// +/// The probe's trace only: the machine builds the same rows through +/// `trace::chip_trace`, which copies the preprocessed prefix out of the +/// committed group instead of re-deriving it from an op. +pub fn generate_blake3_trace(ops: &[Blake3Operation]) -> TraceTable { + let num_rows = ops.len().next_power_of_two().max(4); + let mut data = zeroed_fe_vec(num_rows * cols::NUM_COLUMNS); + + for (row, op) in ops.iter().enumerate() { + let r = &mut data[row * cols::NUM_COLUMNS..(row + 1) * cols::NUM_COLUMNS]; + for j in 0..IN_WORDS { + r[cols::in_addr(j)] = FE::from(op.in_addr[j]); + } + for j in 0..OUT_WORDS { + r[cols::out_addr(j)] = FE::from(op.out_addr[j]); + r[cols::mult(j)] = FE::from(op.read_counts[j]); + } + r[cols::MU] = FE::one(); + fill_blake3_witness(r, &op.values); + } + + TraceTable::new_main(data, cols::NUM_COLUMNS, 1) +} + +// ========================================================================= +// Bus interactions +// ========================================================================= + +fn direct(col: usize) -> BusValue { + BusValue::Packed { + start_column: col, + packing: Packing::Direct, + } +} + +/// `u32` word `index` of the byte family at `bytes_start`, recomposed from its +/// four byte columns as `Σ byte_k · 256^k`. +/// +/// The same trick `chips::keccak::half_value` uses: the machine-side `u32` +/// never gets its own column, so there is nothing extra to keep consistent, and +/// the bytes are already range-checked by the lookups that consume them. +fn lane_value(bytes_start: usize, index: usize) -> BusValue { + BusValue::Linear( + (0..4) + .map(|k| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (8 * k), + column: bytes_start + index * 4 + k, + }) + .collect(), + ) +} + +/// An `LfmMem` token `(addr, v0..v3)` for machine word `word` of a byte family. +fn word_token(addr_col: usize, bytes_start: usize, word: usize) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend((0..4).map(|l| lane_value(bytes_start, 4 * word + l))); + v +} + +/// Lane `l` of the byte-REVERSED digest: reversed byte `j` is digest byte +/// `31 − j`, so this lane's bytes are `OUT[31 − 4l − k]` for `k = 0..3` with the +/// usual little-endian coefficients. +/// +/// The digest is `out[0..8]` little-endian, so digest byte `j` is column +/// `cols::OUT + j` — the same identity `chips::keccak::reversed_half_value` +/// relies on, over a different chip's OUT block. Both the byte order WITHIN a +/// lane and the order OF the lanes come out reversed, which is exactly what +/// reversing all 32 bytes means. +fn reversed_lane_value(lane: usize) -> BusValue { + BusValue::Linear( + (0..4) + .map(|k| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (8 * k), + column: cols::OUT + 31 - 4 * lane - k, + }) + .collect(), + ) +} + +/// An `LfmMem` token for word `w` of the reversed digest. +fn reversed_digest_token(addr_col: usize, w: usize) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend((0..4).map(|l| reversed_lane_value(4 * w + l))); + v +} + +/// Order groups: the `LfmMem` reads and writes, then the mixing core's ByteAlu +/// XORs (canonical `WireFlow` order), the shift `AreBytes`, and the message +/// `AreBytes`. +pub fn bus_interactions() -> Vec { + let wires = WireFlow::build(); + let mut interactions = Vec::with_capacity(1_261); + + let byte_bus_value = |b: ByteRef| -> BusValue { + match b { + ByteRef::Col(c) => direct(c), + ByteRef::Const(v) => BusValue::constant(v as u64), + } + }; + + // 1. Reads: the 7 input machine words. + for j in 0..IN_WORDS { + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MU), + word_token(cols::in_addr(j), cols::IN, j), + )); + } + // 2. Writes: the 4 output machine words, each with its own read count. + for j in 0..OUT_WORDS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::mult(j)), + word_token(cols::out_addr(j), cols::OUT, j), + )); + } + // 2b. The reversed digest: the 32 digest bytes read back-to-front, as two + // more words. Free — a different `Linear` over the OUT columns already + // present. See `layout::blake3::REV_ADDR0` for why the transcript needs it. + for w in 0..cols::DIGEST_WORDS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::rev_mult(w)), + reversed_digest_token(cols::rev_addr(w), w), + )); + } + + // 3. Mixing core + feed-forward: ByteAlu[XOR] per byte, canonical order. + for xw in &wires.xors { + for b in 0..4 { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(cols::MU), + vec![ + BusValue::constant(alu_op::XOR as u64), + byte_bus_value(xw.a.byte(b)), + byte_bus_value(xw.b.byte(b)), + direct(xw.out[b]), + ], + )); + } + } + + // 4. Shift-halfword AreBytes: 4 pairs per rotation. + for rw in &wires.rots { + for pair in [rw.sll_lo, rw.sllc_lo, rw.sll_hi, rw.sllc_hi] { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![direct(pair[0]), direct(pair[1])], + )); + } + } + + // 5. Message AreBytes: m is never XORed, so its 64 bytes get no transitive + // range check (#903 DESIGN §4.7/§7.5). 32 pairs. + for i in 0..16 { + for p in 0..2 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + direct(cols::in_word(8 + i, 2 * p)), + direct(cols::in_word(8 + i, 2 * p + 1)), + ], + )); + } + } + + interactions +} + +/// The BITWISE lookups `bus_interactions` sends, mirrored send for send. +/// +/// Forked from #903's `collect_bitwise_from_blake3` with the address-shaped +/// lookups (the alignment `AND`, 4 addr `AreBytes`, 88 pointer `IsHalf`) and +/// the 32 `OLD_OUT` `AreBytes` dropped — the columns they guarded do not exist +/// here. Enumeration order is the senders' own, via the shared `ValueFlow`. +pub fn bitwise_ops_for(ops: &[Blake3Values]) -> Vec { + let mut out = Vec::with_capacity(ops.len() * 1_248); + + for op in ops { + let flow = ValueFlow::compute(&op.h, &op.m, op.t, op.block_len, op.flags); + for &(a, b, _out) in &flow.xors { + for byte in 0..4 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + ((a >> (8 * byte)) & 0xFF) as u8, + ((b >> (8 * byte)) & 0xFF) as u8, + )); + } + } + for &(sll_lo, sllc_lo, sll_hi, sllc_hi, _y) in &flow.rots { + for hw in [sll_lo, sllc_lo, sll_hi, sllc_hi] { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (hw & 0xFF) as u8, + (hw >> 8) as u8, + )); + } + } + for &word in &op.m { + for p in 0..2 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((word >> (16 * p)) & 0xFF) as u8, + ((word >> (16 * p + 8)) & 0xFF) as u8, + )); + } + } + } + + out +} + +// ========================================================================= +// Constraints +// ========================================================================= + +/// Word expression from a [`WordRef`]: `b0 + 256·b1 + 2^16·b2 + 2^24·b3`. +pub(crate) fn word_expr>(b: &B, w: &WordRef) -> B::Expr { + match w { + WordRef::Cols(c) => { + b.main(0, c[0]) + + b.main(0, c[1]) * b.const_base(256) + + b.main(0, c[2]) * b.const_base(65536) + + b.main(0, c[3]) * b.const_base(16777216) + } + WordRef::Const(v) => b.const_base(*v as u64), + WordRef::ModeSelected(terms) => { + let mut iter = terms.iter(); + let term = |b: &B, (col, tag): (usize, u32)| b.main(0, col) * b.const_base(tag as u64); + let &first = iter.next().expect("a mode-selected word selects something"); + iter.fold(term(b, first), |acc, &t| acc + term(b, t)) + } + } +} + +/// Halfword expression from 2 byte columns: `b0 + 256·b1`. +pub(crate) fn half_expr>(b: &B, c: &[usize; 2]) -> B::Expr { + b.main(0, c[0]) + b.main(0, c[1]) * b.const_base(256) +} + +/// The hosted chip's 769 transition constraints: +/// - idx 0..288: 96 add3 groups (sum identity + 2 carry booleanities); +/// - idx 288..384: 96 add2 expression-carry booleanities; +/// - idx 384..768: 96 rotations (2 shift identities + 2 recombine each); +/// - idx 768: `IS_BIT(MU)`, ungated. +/// +/// #903's first 45 constraints — the 22 `ptr[k] = addr + 8k` carry pairs and +/// the top-dword no-overflow check — have no counterpart: addresses here are +/// preprocessed, so there is nothing to derive and nothing a prover chooses. +/// +/// All μ-gated, max degree 3 (the booleanities; identities are degree 2). +#[derive(Clone, Copy)] +pub struct Blake3LfmConstraints; + +impl ConstraintSet for Blake3LfmConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let wires = WireFlow::build(); + let mu = |b: &B| b.main(0, cols::MU); + let mut idx = 0usize; + + let two_32 = b.const_base(1u64 << 32); + let inv_2_32 = b.const_base(INV_SHIFT_32); + + // add3: μ·(a + b + m − s − 2^32·(c1+c2)) = 0; μ·ci·(1−ci) = 0. + for aw in &wires.add3s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let m_w = word_expr(b, &aw.m); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let c1 = b.main(0, aw.c1); + let c2 = b.main(0, aw.c2); + let sum_id = a + bb + m_w - s - (c1.clone() + c2.clone()) * two_32.clone(); + let m = mu(b); + b.emit_base(idx, m * sum_id); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c1.clone() * (one - c1)); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c2.clone() * (one - c2)); + idx += 1; + } + + // add2: carry = (a + b − s)·2^−32; μ·carry·(1−carry) = 0. + for aw in &wires.add2s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let carry = (a + bb - s) * inv_2_32.clone(); + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * carry.clone() * (one - carry)); + idx += 1; + } + + // Rotations: 2 shift identities + 2 recombine identities each. + for rw in &wires.rots { + let (xlo, xhi) = match &rw.input { + WordRef::Cols(c) => (half_expr(b, &[c[0], c[1]]), half_expr(b, &[c[2], c[3]])), + WordRef::Const(_) | WordRef::ModeSelected(_) => { + unreachable!("shift inputs are always committed XOR outputs") + } + }; + let sll_lo = half_expr(b, &rw.sll_lo); + let sllc_lo = half_expr(b, &rw.sllc_lo); + let sll_hi = half_expr(b, &rw.sll_hi); + let sllc_hi = half_expr(b, &rw.sllc_hi); + let ylo = half_expr(b, &[rw.y[0], rw.y[1]]); + let yhi = half_expr(b, &[rw.y[2], rw.y[3]]); + let two_r = b.const_base(1u64 << rw.r); + let two_16 = b.const_base(65536); + + // μ·(xlo·2^r − SLLC_lo·2^16 − SLL_lo) = 0 (and hi). + let m = mu(b); + b.emit_base( + idx, + m * (xlo * two_r.clone() - sllc_lo.clone() * two_16.clone() - sll_lo.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base( + idx, + m * (xhi * two_r - sllc_hi.clone() * two_16 - sll_hi.clone()), + ); + idx += 1; + // μ·(Ylo − SLL_hi − SLLC_lo) = 0; μ·(Yhi − SLL_lo − SLLC_hi) = 0. + let m = mu(b); + b.emit_base(idx, m * (ylo - sll_hi - sllc_lo)); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (yhi - sll_lo - sllc_hi)); + idx += 1; + } + + // Ungated booleanity of the is-real flag. Preprocessed, so the + // registrar already vouches for it; kept because `chips::keccak` keeps + // its mode-sum booleanity for the same belt-over-suspenders reason. + emit_is_bit(b, idx, cols::MU, None); + } +} + +/// Constraints the chip emits — the number the degree/count tests pin. +pub const NUM_CONSTRAINTS: usize = 3 * (NUM_G * 2) + (NUM_G * 2) + 4 * (NUM_G * 2) + 1; diff --git a/prover/src/lfm/blake3_chip_tests.rs b/prover/src/lfm/blake3_chip_tests.rs new file mode 100644 index 000000000..0d5e2c2de --- /dev/null +++ b/prover/src/lfm/blake3_chip_tests.rs @@ -0,0 +1,1946 @@ +//! `LFM_BLAKE3` as a MEMBER of the machine — the chip group, its emitters and +//! the framing above them. +//! +//! [`super::blake3_probe`] proves the chip standalone against a synthetic +//! `LfmMem` mirror, which is what prices it. This suite is the other half: the +//! chip reached through [`super::builder::LfmBuilder::blake3_compress`], the +//! `Blake3Chain` framing emitted over it, and the constructions the wrap builds +//! from that — against the host hash, with a control for every framing choice +//! the emitter makes. +//! +//! ## What the oracle is, and why it is worth more than a KAT table +//! +//! `crypto::hash::blake3::chain::blake3_chain` is the host hash the RV64 prover +//! commits with, and at **seven** rounds it is the `blake3` crate bit for bit +//! over every message up to one chunk (`seven_round_chain_is_the_blake3_crate`). +//! So comparing the in-machine digest against it anchors this emitter to the +//! published hash through one hop, not to a table this repository generated. +//! The 6-round arm differs from that by a loop bound alone. +//! +//! ## What this suite does NOT establish +//! +//! Nothing here ratifies A6R (the 6-round assumption) or the `Blake3Chain` +//! DRAFT forks F1–F3 — `t = 0` throughout, the flag schedule, and no leaf/parent +//! domain separation. The emitter ENCODES all three; a reversal is an emitter +//! rewrite plus a re-bless, and these tests would then pin the new schedule +//! exactly as they pin this one. + +use stark::constraints::builder::{ConstraintSet, check_dense_index_set}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField, VmTable}; + +use super::blake3::chain::{BLOCK_LEN, blake3_chain, block_flags, block_len_of, num_blocks}; +use super::blake3::{BLAKE3_IV, BLAKE3_ROUNDS, blake3_compress_rounds}; +use super::blake3_chip::{self, Blake3LfmConstraints, NUM_CONSTRAINTS, cols}; +use super::builder::{Cell, LfmBuilder}; +use super::compiler::compile; +use super::edsl::{self, WrapHash}; +use super::executor::{LfmExecError, execute}; +use super::hash::TestPermutation; +use super::keccak_host::{num_stream_halves, pack_stream}; +use super::programs::{blake3_sponge_program, keccak_sponge_program}; +use super::proof::{lfm_prove, verify_against}; +use super::registry::build_artifacts; +use super::word::{LfmWord, base_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// The message the framing tests are taken over: byte `i` is `37i + 11`, the +/// generator `crypto`'s own chain KATs use, so a length here and a length there +/// are the same bytes. +fn message(len: usize) -> Vec { + (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect() +} + +fn sponge_arenas(msg: &[u8]) -> Vec> { + vec![pack_stream(msg).into_iter().map(base_word).collect()] +} + +/// The 32 digest bytes out of two published machine words. +fn digest_bytes(public: &[(u32, LfmWord)]) -> [u8; 32] { + use math::field::traits::IsPrimeField; + let mut out = [0u8; 32]; + for (w, (_, word)) in public.iter().enumerate().take(2) { + for (l, lane) in word.iter().enumerate() { + let v = GoldilocksField::canonical(lane.value()) as u32; + out[4 * (4 * w + l)..4 * (4 * w + l) + 4].copy_from_slice(&v.to_le_bytes()); + } + } + out +} + +/// The lengths every framing boundary lives at, and what each discriminates: +/// the empty message is ONE block (0); `block_len` is the true length and the +/// tail is zero-padded (1, 31, 63); 64 bytes is the parent form; 65 moves +/// `CHUNK_END | ROOT` off block 0; an exact multiple emits no spurious final +/// block (128); interior blocks carry no flags at all (192, 256). +/// +/// ★ **Through the chunk boundary, deliberately.** The list stopped at 256, +/// which left the construction's most consequential seam untested by any digest +/// comparison: at 1024 this chain is still standard BLAKE3, and past it the +/// construction knowingly leaves the standard (the standard would start chunk 1 +/// with `t = 1` and a reset chaining value; this keeps one unbounded chunk). +/// 1023/1024/1025 and 1087/1088 bracket that seam from both sides, 511/512/513 +/// bracket an interior multiple, and 2048 is two chunks past it. A framing bug +/// that only bites after the first chunk would have passed the old list. +const FRAMING_LENS: [usize; 19] = [ + 0, 1, 31, 63, 64, 65, 127, 128, 192, 256, 511, 512, 513, 1023, 1024, 1025, 1087, 1088, 2048, +]; + +// ========================================================================= +// The framing, against the host hash +// ========================================================================= + +/// ★ THE ANCHOR: the emitted `Blake3Chain` IS the host's, at every length a +/// block boundary can be wrong at. +/// +/// Execution only — what it establishes is that the EMITTER computes the right +/// function; that the CHIP computes what the executor mirrors is the +/// prove-and-verify test below (standing method rule 2). +#[test] +fn the_emitted_chain_is_the_host_blake3_chain() { + for len in FRAMING_LENS { + let msg = message(len); + let program = blake3_sponge_program(len); + let exec = execute(&program, &sponge_arenas(&msg), &TestPermutation) + .unwrap_or_else(|e| panic!("len {len}: the emitted chain must execute: {e:?}")); + assert_eq!( + digest_bytes(&exec.public_words), + blake3_chain(&msg), + "len {len}: the machine's digest must be the host chain's" + ); + } +} + +/// NON-VACUITY for the anchor: distinct lengths must give distinct digests, or +/// the test above would pass for an emitter that ignored the message. +/// +/// It is also **P4** in its cheapest observable form: lengths sharing a padded +/// block (31 vs 32, 64 vs 65's first block) must not collide, which is what +/// says `block_len` and the flag schedule are consumed. +#[test] +fn distinct_lengths_give_distinct_machine_digests() { + let mut seen: Vec<(usize, [u8; 32])> = Vec::new(); + for len in FRAMING_LENS { + let msg = message(len); + let program = blake3_sponge_program(len); + let exec = execute(&program, &sponge_arenas(&msg), &TestPermutation).expect("execute"); + let digest = digest_bytes(&exec.public_words); + for (other, d) in &seen { + assert_ne!(*d, digest, "lengths {other} and {len} collide"); + } + seen.push((len, digest)); + } +} + +/// ★ The highest-value falsification: break ONE framing choice at a time and +/// watch the digest move. +/// +/// The emitter reads its schedule from `crypto` (`num_blocks`, `block_flags`, +/// `block_len_of`) so it cannot restate it wrongly — which means the thing left +/// to check is that the CHIP consumes each field at all. A chip that ignored +/// `flags`, or `block_len`, or the counter would pass every KAT above at a +/// single length and produce a valid proof of a different hash the moment the +/// message crossed a block. +/// +/// Driven through the raw builder rather than the framing, because breaking a +/// framing choice is precisely what the framing will not do. +#[test] +fn breaking_one_framing_choice_at_a_time_breaks_the_digest() { + let h: [u32; 8] = BLAKE3_IV; + let m: [u32; 16] = core::array::from_fn(|i| 0x0123_4567u32.wrapping_mul(i as u32 + 1)); + + let honest = blake3_compress_rounds( + &h, + &m, + 0, + BLOCK_LEN as u32, + block_flags(0, 1), + BLAKE3_ROUNDS, + ); + + // Each perturbation names the framing decision it breaks. + let broken: [(&str, u64, u32, u32); 4] = [ + ( + "the counter is not zero (F1)", + 1, + BLOCK_LEN as u32, + block_flags(0, 1), + ), + ( + "the counter's HIGH half is not zero", + 1u64 << 32, + BLOCK_LEN as u32, + block_flags(0, 1), + ), + ( + "block_len is the shape, not the true byte count", + 0, + (BLOCK_LEN - 1) as u32, + block_flags(0, 1), + ), + ( + "the flag schedule is an interior block, not a lone one", + 0, + BLOCK_LEN as u32, + block_flags(1, 3), + ), + ]; + for (what, t, block_len, flags) in broken { + assert_ne!( + blake3_compress_rounds(&h, &m, t, block_len, flags, BLAKE3_ROUNDS), + honest, + "the compression must depend on every framing field — {what}" + ); + } + + // And the machine agrees with the primitive on the honest framing, through + // a real `Instr::Blake3` rather than through the framing that produced it. + let program = raw_compress_program(&h, &m, 0, BLOCK_LEN as u32, block_flags(0, 1)); + let exec = execute(&program, &[], &TestPermutation).expect("a raw compression executes"); + assert_eq!( + published_words(&exec.public_words), + honest, + "the chip's output must be the primitive's, field for field" + ); +} + +/// A program that issues ONE raw compression over compile-time constants and +/// publishes all sixteen output words. +fn raw_compress_program( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) -> super::compiler::LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let word = |b: &mut LfmBuilder, lanes: [u32; 4]| -> Cell { + b.digest_const(core::array::from_fn(|i| FE::from(u64::from(lanes[i])))) + .as_cell() + }; + let h_cells = [ + word(&mut b, [h[0], h[1], h[2], h[3]]), + word(&mut b, [h[4], h[5], h[6], h[7]]), + ]; + let m_cells: [Cell; 4] = + core::array::from_fn(|w| word(&mut b, core::array::from_fn(|l| m[4 * w + l]))); + let params = word(&mut b, [t as u32, (t >> 32) as u32, block_len, flags]); + let out = b.blake3_compress(h_cells, m_cells, params); + for cell in out { + b.public(cell); + } + compile(b.finish()) +} + +/// The sixteen `u32`s behind four published machine words. +fn published_words(public: &[(u32, LfmWord)]) -> [u32; 16] { + use math::field::traits::IsPrimeField; + core::array::from_fn(|i| { + let (w, l) = (i / 4, i % 4); + GoldilocksField::canonical(public[w].1[l].value()) as u32 + }) +} + +/// The closed-form schedule the emitter reads is the one it emits — the +/// emitter's own accounting, checked against `crypto`'s. +/// +/// A cheap statement with a real job: it is what says the eDSL and the host +/// hasher agree about how many compressions a message of a given length costs, +/// which is the number the census projects from. +#[test] +fn the_emitted_block_count_is_the_chains() { + for len in FRAMING_LENS.iter().chain(&[1024usize, 1088]) { + let program = blake3_sponge_program(*len); + let compressions = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Blake3(_))) + .count(); + assert_eq!( + compressions, + num_blocks(*len), + "len {len}: one compression per block, and the empty message is ONE" + ); + // The last block carries the true remaining byte count, every earlier + // one a full block. + assert_eq!( + block_len_of(num_blocks(*len) - 1, *len) as usize, + len - BLOCK_LEN * (num_blocks(*len) - 1), + "len {len}: the final block_len is the remainder" + ); + } +} + +// ========================================================================= +// The chip as a member of the machine +// ========================================================================= + +/// ★ **MANDATORY, and release-visible.** Every constraint index is emitted +/// exactly once. +/// +/// `EmitTracker`'s duplicate assert is `#[cfg(debug_assertions)]` and this +/// workspace has no `[profile.release]` override, so under the house +/// `cargo test --release` convention a second `emit_base(idx, …)` **silently +/// overwrites the first** — counts still fill `0..N`, so `NUM_CONSTRAINTS`, the +/// predicted-count tests and `assert_complete` all pass with a constraint +/// deleted. `check_dense_index_set` runs with no cfg, which is what makes this +/// test see the release build's actual emission. +/// +/// The RATE-4 near-miss is the precedent: a hardcoded index there would have +/// overwritten four output pins with nothing failing. +#[test] +fn every_constraint_index_is_emitted_exactly_once() { + let set = Blake3LfmConstraints; + let meta = >::meta(&set); + check_dense_index_set(&meta, NUM_CONSTRAINTS) + .unwrap_or_else(|e| panic!("LFM_BLAKE3 constraint body: {e}")); +} + +/// The chip is a registered member of the fixed set, at the slot the roots and +/// the digest are built against. +/// +/// Pinned by NAME and by INDEX together: the census maps array slots onto +/// `LFM_CHIP_NAMES` across the `KECCAK_RND` slot, and a placement that moved +/// one without the other would leave every root one position out with nothing +/// in the arithmetic to notice. +#[test] +fn the_chip_occupies_its_registered_slot() { + use super::airs::{KECCAK_RND_SLOT, LFM_CHIP_NAMES, NUM_LFM_CHIPS}; + use super::registry::LfmProgramKind; + assert_eq!(NUM_LFM_CHIPS, 15, "the promotion is 14 -> 15"); + assert_eq!(LFM_CHIP_NAMES[11], "LFM_BLAKE3"); + assert_eq!( + KECCAK_RND_SLOT, 12, + "LFM_BLAKE3 sits before the hosted keccak family, so the family moved up" + ); + assert_eq!(LFM_CHIP_NAMES[KECCAK_RND_SLOT], "KECCAK_RND"); + assert_eq!(LFM_CHIP_NAMES[13], "KECCAK_RC"); + assert_eq!(LFM_CHIP_NAMES[14], "BITWISE"); + // Every registry entry carries a root and a height for the new slot, and + // since the flip the heights SPLIT — which is the fixed-machine principle + // seen from both sides. + // + // A program that never compresses still commits the chip's empty group, + // padded to the 4-row minimum: the slot is machine shape, not program + // shape. A program that does compress is above that minimum, and the two + // that are above it are exactly the two the flip moved. + for entry in super::registry::LFM_REGISTRY { + let expected = match entry.kind { + LfmProgramKind::TranscriptReplayV0 => 3, + LfmProgramKind::StatementReplayV0 => 4, + _ => 2, + }; + assert_eq!( + entry.log_heights[11], expected, + "{:?}: LFM_BLAKE3 slot height", + entry.kind + ); + } +} + +/// ★ The registry blessing is INVARIANT under `BLAKE3_ROUNDS`, and the chip's +/// witness is NOT — the two halves of one claim. +/// +/// Without this, one registry entry describes two machines: `blake3-6round` +/// moves `NUM_COLUMNS` 3556 → 3076, `NUM_CONSTRAINTS` 897 → 769 and the +/// interaction count 1453 → 1261. A proof built under one round count and +/// verified under the other fails CLOSED — the OOD width check rejects it +/// before a constraint is evaluated — but it fails on a width mismatch rather +/// than on the axis being NAMED, and named is exactly what `hasher` gets a +/// `program_id` fold for and what `CommitmentHash` gets a compile-time guard +/// for. The round count is the third such axis and has neither. +/// +/// What makes blessing once nonetheless correct is that the round count lives +/// **entirely in the value columns**: the preprocessed instruction group is +/// addresses, multiplicities, the reversed-digest pair and `MU`, and no term of +/// it mentions `NUM_G`. So no root and no log-height can move with the feature, +/// and the one table is readable under either build. +/// +/// Both directions are asserted, because the prefix not moving means nothing +/// unless something else does. Confirmed empirically as well: regenerating +/// `LFM_REGISTRY` under `--features blake3-6round` reproduces the committed +/// table exactly — all 3,072 root and digest bytes, all six `log_heights`, all +/// six `program_id`s. +#[test] +fn the_registry_blessing_is_round_count_invariant() { + // ---- the prefix is a function of the I/O shape alone. + assert_eq!( + cols::PREP_WIDTH, + blake3_chip::IN_WORDS + 2 * blake3_chip::OUT_WORDS + 2 * cols::DIGEST_WORDS + 1, + "the instruction group is 7 input addresses, 4 output addresses and their \ + multiplicities, 2 reversed-digest addresses and theirs, and MU — not one \ + term of it is a function of the round count" + ); + // And the committed group IS that prefix, so no root can move with it. + let program = blake3_sponge_program(65); + assert_eq!( + program.groups.blake3.width, + cols::PREP_WIDTH, + "the committed group is the preprocessed prefix and nothing else" + ); + + // ---- NON-VACUITY: the witness DOES move, so the separation is real. + assert_eq!(blake3_chip::NUM_G, BLAKE3_ROUNDS * 8); + let (value_columns, constraints, interactions, other_value_columns) = if BLAKE3_ROUNDS == 6 { + (3_056usize, 769usize, 1_261usize, 3_536usize) + } else { + (3_536, 897, 1_453, 3_056) + }; + assert_eq!( + cols::NUM_COLUMNS - cols::PREP_WIDTH, + value_columns, + "the value columns are round-dependent" + ); + assert_eq!(NUM_CONSTRAINTS, constraints); + assert_eq!(blake3_chip::bus_interactions().len(), interactions); + assert_ne!( + value_columns, other_value_columns, + "the two round counts must be two different machines, or this test \ + asserts an invariance with nothing to be invariant under" + ); +} + +/// An input lane at or above `2^32` is REJECTED, not reduced. +/// +/// The chip recomposes each `u32` from four BITWISE-range-checked byte columns, +/// so no such value exists on the AIR side and the program would be unprovable. +/// Failing in the executor, with a reason, is what turns that into a diagnosable +/// error instead of an unbalanced bus. +#[test] +fn an_out_of_range_lane_is_rejected_rather_than_reduced() { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let ok = b + .digest_const([FE::zero(), FE::zero(), FE::zero(), FE::zero()]) + .as_cell(); + // Lane 0 of the chaining value is 2^32 — one past the largest `u32`. + let bad = b + .digest_const([FE::from(1u64 << 32), FE::zero(), FE::zero(), FE::zero()]) + .as_cell(); + let out = b.blake3_compress([bad, ok], [ok, ok, ok, ok], ok); + b.public(out[0]); + let program = compile(b.finish()); + + match execute(&program, &[], &TestPermutation) { + Err(LfmExecError::NotU32Half { lane, .. }) => assert_eq!(lane, 0), + other => panic!("a non-u32 lane must be rejected, got {other:?}"), + } + + // HONEST CONTROL: the same shape with every lane a legal `u32` executes. + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let ok = b + .digest_const([FE::zero(), FE::zero(), FE::zero(), FE::zero()]) + .as_cell(); + let good = b + .digest_const([ + FE::from(u64::from(u32::MAX)), + FE::zero(), + FE::zero(), + FE::zero(), + ]) + .as_cell(); + let out = b.blake3_compress([good, ok], [ok, ok, ok, ok], ok); + b.public(out[0]); + let program = compile(b.finish()); + assert!( + execute(&program, &[], &TestPermutation).is_ok(), + "u32::MAX is a legal lane and must execute" + ); +} + +/// ★ The chip PROVES and VERIFIES as a member of the set — the only kind of +/// test that says anything about the constraints (method rule 2). +#[test] +fn the_blake3_chip_proves_and_verifies() { + let opts = options(); + // 65 bytes: two blocks, so the flag schedule and the chaining value are + // both exercised rather than collapsing into the single-block case. + for len in [0usize, 64, 65, 200] { + let msg = message(len); + let program = blake3_sponge_program(len); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts) + .unwrap_or_else(|e| panic!("len {len}: prove failed: {e:?}")); + assert_eq!( + digest_bytes(&proved.public_words), + blake3_chain(&msg), + "len {len}: the PROVED digest must be the host chain's" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "len {len}: the machine proof of Blake3Chain must verify" + ); + } +} + +/// Tampering with the chip's witness is not accepted. +/// +/// The output columns are corrupted after trace generation, so the trace is +/// internally inconsistent in exactly the way a prover trying to claim a +/// different digest would make it. +#[test] +fn tampering_with_the_blake3_witness_is_not_accepted() { + let opts = options(); + let msg = message(65); + let program = blake3_sponge_program(65); + let artifacts = build_artifacts(&program, &opts); + let exec = execute(&program, &sponge_arenas(&msg), &TestPermutation).expect("execute"); + let mut traces = + super::trace::build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + + // One output byte of the first compression. + let col = cols::out_word(0, 0); + let old = traces.blake3[0].main_table.get_row(0)[col]; + traces.blake3[0].main_table.set_fe(0, col, old + FE::one()); + + let proved = super::proof::prove_traces(&artifacts, &mut traces, &exec.public_words, &opts); + match proved { + Err(_) => {} + Ok(proof) => assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a tampered LFM_BLAKE3 witness must not verify" + ), + } +} + +/// The reversed digest is the digest backwards, and a row that does not ask for +/// one emits nothing. +/// +/// Both halves matter. The first is the value; the second is that the two extra +/// sends really are inert when unused, which is what makes the feature free +/// rather than a cost every compression pays. +#[test] +fn the_reversed_digest_is_the_digest_backwards() { + use super::layout::blake3 as l; + + let len = 100usize; + let msg = message(len); + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let num_halves = num_stream_halves(len) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec<_> = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let (plain, rev) = edsl::blake3_256_with_rev(&mut b, &stream, len); + b.public(plain[0]); + b.public(plain[1]); + b.public(rev[0]); + b.public(rev[1]); + let program = compile(b.finish()); + + let exec = execute(&program, &sponge_arenas(&msg), &TestPermutation).expect("execute"); + let plain_bytes = digest_bytes(&exec.public_words[..2]); + let rev_bytes = digest_bytes(&exec.public_words[2..]); + assert_eq!(plain_bytes, blake3_chain(&msg), "the plain digest"); + let mut expected = plain_bytes; + expected.reverse(); + assert_eq!( + rev_bytes, expected, + "the reversed digest is the plain one, backwards" + ); + // And it is not vacuous: the message is chosen so the digest is not a + // palindrome. + assert_ne!( + rev_bytes, plain_bytes, + "a palindromic digest proves nothing" + ); + + // The rows that did NOT request a reversal carry zero rev multiplicities, + // so their two extra sends contribute nothing to the bus. + let group = &program.groups.blake3; + let last = group.real_rows - 1; + for row in 0..last { + for w in 0..l::DIGEST_WORDS { + assert_eq!( + *group.at(row, l::rev_mult(w)), + FE::zero(), + "row {row} did not request a reversed digest" + ); + } + } + assert!( + (0..l::DIGEST_WORDS).any(|w| *group.at(last, l::rev_mult(w)) != FE::zero()), + "the final row DID request one, or the control above is vacuous" + ); +} + +// ========================================================================= +// Cross-hash: the two configurations are two machines +// ========================================================================= + +/// ★ **Required in BOTH directions.** A program under one wrap hash is not a +/// program under the other, and neither one's proof verifies against the +/// other's artifacts. +/// +/// This is the property that says a flip is a flip rather than a relabelling. +/// The two programs have the same SHAPE — same length, same public output — so +/// what separates them is the emitted hash and nothing else. +#[test] +fn the_two_wrap_hashes_produce_mutually_unverifiable_proofs() { + let opts = options(); + let len = 200usize; + let msg = message(len); + + let keccak_program = keccak_sponge_program(len); + let blake3_program = blake3_sponge_program(len); + let keccak_artifacts = build_artifacts(&keccak_program, &opts); + let blake3_artifacts = build_artifacts(&blake3_program, &opts); + + assert_ne!( + keccak_artifacts.program_id, blake3_artifacts.program_id, + "the two hashes must be two program identities" + ); + + let keccak_proof = lfm_prove( + &keccak_program, + &keccak_artifacts, + &sponge_arenas(&msg), + &opts, + ) + .expect("keccak prove"); + let blake3_proof = lfm_prove( + &blake3_program, + &blake3_artifacts, + &sponge_arenas(&msg), + &opts, + ) + .expect("blake3 prove"); + + // The digests differ, which is the cheapest statement that the emitters + // really are different functions. + assert_ne!( + digest_bytes(&keccak_proof.public_words), + digest_bytes(&blake3_proof.public_words), + "the two hashes of the same message must differ" + ); + + let verifies = |artifacts: &super::registry::LfmArtifacts, proved: &super::proof::LfmProof| { + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ) + }; + // The honest control comes first: each proof verifies under its own. + assert!( + verifies(&keccak_artifacts, &keccak_proof), + "keccak honest path" + ); + assert!( + verifies(&blake3_artifacts, &blake3_proof), + "blake3 honest path" + ); + // And neither crosses. + assert!( + !verifies(&blake3_artifacts, &keccak_proof), + "a keccak proof must not verify against the BLAKE3 program's artifacts" + ); + assert!( + !verifies(&keccak_artifacts, &blake3_proof), + "a BLAKE3 proof must not verify against the keccak program's artifacts" + ); +} + +/// The Merkle constructions agree with the host tree under BOTH hashes, through +/// ONE emitter. +/// +/// `WrapHash::merkle_walk` and `merkle_tree_root` are written once and dispatch +/// only at the parent hash, so this is what says the parameterization did not +/// silently specialize to one of them. +#[test] +fn the_merkle_constructions_agree_with_the_host_under_both_hashes() { + use super::keccak_host; + + // A four-leaf tree over 32-byte nodes, built the way `hash_new_parent` + // does: `hash(left ‖ right)`, no domain separation, no ordering flag. + let leaves: [[u8; 32]; 4] = + core::array::from_fn(|i| core::array::from_fn(|j| (i * 32 + j) as u8)); + + for hash in [WrapHash::Keccak, WrapHash::Blake3] { + let host_parent = |l: &[u8; 32], r: &[u8; 32]| -> [u8; 32] { + let mut bytes = Vec::with_capacity(64); + bytes.extend_from_slice(l); + bytes.extend_from_slice(r); + match hash { + WrapHash::Keccak => keccak_host::keccak256(&bytes), + WrapHash::Blake3 => blake3_chain(&bytes), + // This helper is a BYTE-hash host mirror; the algebraic arm's + // parent is `compress` over two four-felt digests and is + // mirrored in `algebraic_commit` instead. + WrapHash::Algebraic | WrapHash::Poseidon1 => unreachable!("byte-hash host mirror"), + } + }; + let host_root = host_parent( + &host_parent(&leaves[0], &leaves[1]), + &host_parent(&leaves[2], &leaves[3]), + ); + + // The same tree, in the machine, from hinted leaf digests. + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let arena = b.declare_arena(8); + let cells: Vec = (0..4) + .map(|i| { + edsl::WrapDigest::from_pair( + b.hint_word(arena, 2 * i), + b.hint_word(arena, 2 * i + 1), + ) + }) + .collect(); + let root = edsl::wrap_merkle_tree_root(&mut b, &cells); + b.public(root[0]); + b.public(root[1]); + let program = compile(b.finish()); + + // At THIS program's width (a byte hash, chosen on its builder above), + // not the configuration's: under an algebraic pin the two differ. + let arena_words: Vec = super::proof_arena::commitments_to_arena_for(&leaves, hash); + let exec = execute(&program, &[arena_words], &TestPermutation) + .unwrap_or_else(|e| panic!("{hash:?}: the tree build must execute: {e:?}")); + assert_eq!( + digest_bytes(&exec.public_words), + host_root, + "{hash:?}: the machine's root must be the host tree's" + ); + } +} + +// ========================================================================= +// Rider 1 — the draw schedule, not just the hash +// ========================================================================= + +/// ★ The BLAKE3 configuration draws TWO candidates per base coordinate, and the +/// keccak one draws one. +/// +/// Emit-time bookkeeping, which is where the schedule lives: `out_pos` is what +/// decides when a refill lands, so a machine that drew a different number would +/// squeeze at different points and reproduce different challenges. +#[test] +fn the_configurations_draw_different_numbers_of_candidates() { + use super::transcript_replay::TranscriptReplay; + + for (hash, per_coordinate) in [(WrapHash::Keccak, 8usize), (WrapHash::Blake3, 16usize)] { + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let mut t = TranscriptReplay::new(b"seed"); + let before = t.out_pos(); + let _ = t.sample_felt(&mut b); + let after = t.out_pos(); + // The first draw forces a refill, so the buffer position after one + // coordinate IS the bytes that coordinate consumed. + assert_eq!( + (after + super::keccak_host::SQUEEZE_LEN - before) % super::keccak_host::SQUEEZE_LEN, + per_coordinate, + "{hash:?}: one base coordinate must consume {per_coordinate} bytes" + ); + } +} + +/// ★ The in-range predicate is the one `assert_canonical` encodes, as a BIT — +/// including the two boundary candidates that decide it. +/// +/// `candidate ≥ p ⟺ hi = 2^32 − 1 ∧ lo ≠ 0`. The vectors are the exact corners: +/// `p − 1` (in range, and the largest that is), `p` (the smallest that is not), +/// and a candidate with the maximal `hi` but a zero `lo` (in range, and the case +/// a naive `hi = 2^32 − 1 ⇒ reject` would get wrong). +#[test] +fn the_in_range_predicate_is_canonicity_as_a_bit() { + use super::transcript_replay::{Candidate, candidate_in_range}; + use math::field::traits::IsPrimeField; + + const HI_MAX: u64 = 0xFFFF_FFFF; + // (hi, lo, expected in_range) + let vectors: [(u64, u64, bool); 6] = [ + (0, 0, true), + (0, 1, true), + (HI_MAX - 1, HI_MAX, true), + (HI_MAX, 0, true), // p − 1 exactly + (HI_MAX, 1, false), // p exactly + (HI_MAX, HI_MAX, false), + ]; + + for (hi, lo, want) in vectors { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let arena = b.declare_arena(2); + let c = Candidate { + lo: b.hint_felt(arena, 0), + hi: b.hint_felt(arena, 1), + }; + let bit = candidate_in_range(&mut b, c); + b.public(bit.as_cell()); + let program = compile(b.finish()); + let arenas = vec![vec![base_word(FE::from(lo)), base_word(FE::from(hi))]]; + let exec = execute(&program, &arenas, &TestPermutation) + .unwrap_or_else(|e| panic!("({hi:#x}, {lo:#x}): must execute: {e:?}")); + let got = GoldilocksField::canonical(exec.public_words[0].1[0].value()); + assert_eq!( + got, + u64::from(want), + "candidate hi={hi:#x} lo={lo:#x}: in_range must be {want}" + ); + } +} + +/// HONEST-PATH CONTROL for the whole stage: keccak still proves and verifies +/// through the rewritten emitters. +/// +/// Without this, an over-broad emitter change reads as a pass — every BLAKE3 +/// test above would go green on a tree where the keccak path had been broken or +/// silently switched. +#[test] +fn keccak_still_proves_and_verifies_through_the_switched_emitters() { + let opts = options(); + let len = 202usize; + let msg = message(len); + let program = keccak_sponge_program(len); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + assert_eq!( + digest_bytes(&proved.public_words), + super::keccak_host::keccak256(&msg), + "the default path must still be keccak256, byte for byte" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the keccak proof must still verify" + ); +} + +/// `WrapHash::default()` is the UNSET value, not the production one. +/// +/// Before the flip these two lines said "nothing selects BLAKE3 by omission". +/// They still hold and they still matter, but the reason inverted: production +/// is BLAKE3 now, and the default stayed keccak deliberately so that no +/// constructor can acquire the production hash by saying nothing. Every +/// `programs.rs` constructor that hashes names its hash; see that module's +/// header for the two kinds of exception. +#[test] +fn the_default_wrap_hash_is_keccak() { + assert_eq!(WrapHash::default(), WrapHash::Keccak); + assert_eq!(LfmBuilder::new().wrap_hash(), WrapHash::Keccak); +} + +/// The chip's layout assigns every column exactly once, prefix included. +/// +/// A layout with a hole is a column no constraint and no send reads — free +/// witness. A layout with an overlap is two things sharing a cell, which the +/// single-dataflow rule cannot see because both interpretations agree. +#[test] +fn the_layout_assigns_every_column_exactly_once() { + let mut seen = vec![0usize; cols::NUM_COLUMNS]; + let mut mark = |col: usize, what: &str| { + assert!(col < cols::NUM_COLUMNS, "{what} at {col} is past the width"); + seen[col] += 1; + }; + + for j in 0..blake3_chip::IN_WORDS { + mark(cols::in_addr(j), "in_addr"); + } + for j in 0..blake3_chip::OUT_WORDS { + mark(cols::out_addr(j), "out_addr"); + mark(cols::mult(j), "mult"); + } + for w in 0..cols::DIGEST_WORDS { + mark(cols::rev_addr(w), "rev_addr"); + mark(cols::rev_mult(w), "rev_mult"); + } + mark(cols::MU, "MU"); + for i in 0..blake3_chip::IN_U32 { + for b in 0..4 { + mark(cols::in_word(i, b), "in byte"); + } + } + for g in 0..blake3_chip::NUM_G { + for k in 0..cols::G_SIZE { + mark(cols::g_base(g) + k, "G cell"); + } + } + for i in 0..blake3_chip::OUT_U32 { + for b in 0..4 { + mark(cols::out_word(i, b), "out byte"); + } + } + + let unassigned: Vec = (0..cols::NUM_COLUMNS).filter(|c| seen[*c] == 0).collect(); + let doubled: Vec = (0..cols::NUM_COLUMNS).filter(|c| seen[*c] > 1).collect(); + assert!( + unassigned.is_empty(), + "columns assigned to nothing: {unassigned:?}" + ); + assert!(doubled.is_empty(), "columns assigned twice: {doubled:?}"); +} + +// ========================================================================= +// ★ The transcript replay, against the REAL host transcript +// ========================================================================= +// +// ## Why the single-coordinate tests above are not enough +// +// `the_configurations_draw_different_numbers_of_candidates` checks that ONE +// base coordinate consumes 16 bytes under BLAKE3 and 8 under keccak, and +// `the_in_range_predicate_is_canonicity_as_a_bit` checks the select's predicate +// at its corners. Both are structurally blind to the bug that matters, and the +// blindness is demonstrable rather than suspected: mutate the replay's schedule +// from n = 2 back to n = 1 and the FIRST challenge is still correct — candidate +// 0 is in range with probability 1 − 2⁻³², so it is the answer either way. The +// divergence appears at the SECOND challenge, because a schedule that consumed +// 8 bytes instead of 16 reads it from the wrong buffer offset and refills at +// the wrong point. +// +// ✓ EXECUTED, not argued: forcing `candidates_per_coordinate` to 1 for the +// BLAKE3 arm leaves `the_in_range_predicate_is_canonicity_as_a_bit` PASSING and +// fails the oracle below on `blake3: second base challenge` — the first is +// still right. That is the whole case for this test existing. +// +// So a consumption-schedule bug is invisible to any test that draws once. What +// sees it is a SCRIPT — several draws of different kinds, an absorb in the +// middle to invalidate the buffer, a raw `sample()` whose reversed digest is +// re-absorbed — with every published value compared against the real +// `DefaultTranscript` under the same configuration. That is what follows, and +// it is the BLAKE3 counterpart of the keccak oracle in +// `machine_tests::transcript_replay_matches_the_host`. + +/// The script's seed and absorb shapes. `ABSORB_A` is a digest-sized absorb and +/// `ABSORB_B` a rate-sized one, so the segment crosses a block boundary between +/// the two draw runs under both hashes. +const ORACLE_SEED: &[u8] = b"lfm-transcript-replay-v0"; +const ORACLE_ABSORB_A: usize = 32; +const ORACLE_ABSORB_B: usize = 136; +const ORACLE_QUERY_BITS: usize = 20; + +fn oracle_absorbs() -> (Vec, Vec) { + let a = (0..ORACLE_ABSORB_A) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let b = (0..ORACLE_ABSORB_B) + .map(|i| (i as u8).wrapping_mul(17).wrapping_add(3)) + .collect(); + (a, b) +} + +fn oracle_arenas() -> Vec> { + let (a, b) = oracle_absorbs(); + let mut bytes = a; + bytes.extend_from_slice(&b); + vec![pack_stream(&bytes).into_iter().map(base_word).collect()] +} + +/// The script, emitted under a chosen wrap hash. +/// +/// Deliberately mixed: two base draws, an extension draw (three coordinates, so +/// the schedule is exercised across a refill), an absorb that invalidates the +/// output buffer, a `sample_u64` draw (which takes the RAW candidate stream in +/// both configurations and must therefore NOT change), a further base draw, a +/// raw `sample()` whose reversed digest becomes the next segment, and one last +/// base draw on the far side of it. +fn oracle_replay_program(hash: WrapHash) -> super::compiler::LfmProgram { + use super::builder::Felt; + use super::edsl::bits_to_felt; + use super::keccak_host::BYTES_PER_HALF; + use super::transcript_replay::TranscriptReplay; + + let total_halves = ((ORACLE_ABSORB_A + ORACLE_ABSORB_B) / BYTES_PER_HALF) as u32; + let halves_a = ORACLE_ABSORB_A / BYTES_PER_HALF; + + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let arena = b.declare_arena(total_halves); + let halves: Vec = (0..total_halves).map(|i| b.hint_felt(arena, i)).collect(); + let (absorb_a, absorb_b) = halves.split_at(halves_a); + + let mut t = TranscriptReplay::new(ORACLE_SEED); + t.append_halves(absorb_a); + let f0 = t.sample_felt(&mut b); + let f1 = t.sample_felt(&mut b); + let e = t.sample_ext(&mut b); + t.append_halves(absorb_b); + let q = t.sample_u64_pow2(&mut b, ORACLE_QUERY_BITS); + let qf = bits_to_felt(&mut b, &q); + let f2 = t.sample_felt(&mut b); + let s = t.sample(&mut b); + let f3 = t.sample_felt(&mut b); + + b.public(f0.as_cell()); + b.public(f1.as_cell()); + b.public(e.as_cell()); + b.public(qf.as_cell()); + b.public(f2.as_cell()); + b.public(s[0]); + b.public(s[1]); + b.public(f3.as_cell()); + compile(b.finish()) +} + +struct OracleExpectation { + f0: FE, + f1: FE, + e: [FE; 3], + q: u64, + f2: FE, + s: [u8; 32], + f3: FE, +} + +/// The oracle: the REAL `DefaultTranscript` under configuration `T`, driven +/// through the same script. +/// +/// Instantiated over the BASE field so `sample_field_element` is one coordinate, +/// matching the machine's `sample_felt`; the extension draw in the middle is +/// three consecutive base draws, which is what the host's ext sampler does +/// (`core::array::from_fn` evaluates in index order). +fn oracle_expectation() -> OracleExpectation +{ + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let (a, b) = oracle_absorbs(); + let mut h = DefaultTranscript::::new(ORACLE_SEED); + h.append_bytes(&a); + let f0 = h.sample_field_element(); + let f1 = h.sample_field_element(); + let e: [FE; 3] = core::array::from_fn(|_| h.sample_field_element()); + h.append_bytes(&b); + let q = h.sample_u64(1 << ORACLE_QUERY_BITS); + let f2 = h.sample_field_element(); + let s = h.sample(); + let f3 = h.sample_field_element(); + OracleExpectation { + f0, + f1, + e, + q, + f2, + s, + f3, + } +} + +fn check_against_oracle(public: &[(u32, LfmWord)], x: &OracleExpectation, what: &str) { + assert_eq!(public.len(), 8, "{what}: published word count"); + assert_eq!(public[0].1[0], x.f0, "{what}: first base challenge"); + assert_eq!(public[1].1[0], x.f1, "{what}: second base challenge"); + for i in 0..3 { + assert_eq!(public[2].1[i], x.e[i], "{what}: ext coordinate {i}"); + } + assert_eq!(public[3].1[0], FE::from(x.q), "{what}: sample_u64 draw"); + assert_eq!(public[4].1[0], x.f2, "{what}: post-absorb challenge"); + assert_eq!(digest_bytes(&public[5..7]), x.s, "{what}: raw sample()"); + assert_eq!(public[7].1[0], x.f3, "{what}: post-sample challenge"); +} + +/// HONEST CONTROL: the keccak arm of the same harness reproduces the host. +/// +/// Without it a BLAKE3 failure below says nothing — it could be the harness, +/// the script, or the comparison. This is also the statement that threading the +/// wrap hash through `TranscriptReplay` did not move the default configuration. +#[test] +fn the_keccak_replay_matches_the_host_transcript() { + use crypto::fiat_shamir::transcript_hash::KeccakTranscriptHash; + let exec = execute( + &oracle_replay_program(WrapHash::Keccak), + &oracle_arenas(), + &TestPermutation, + ) + .expect("the keccak replay must execute"); + check_against_oracle( + &exec.public_words, + &oracle_expectation::(), + "keccak", + ); +} + +/// ★ THE ORACLE: the BLAKE3 configuration's in-machine replay reproduces the +/// host `Blake3TranscriptHash`, value for value. +/// +/// This is the one test that exercises the three Stage-5 transcript changes +/// TOGETHER — the n = 2 consumption schedule across refill boundaries, the +/// select chain that picks the first in-range candidate, and the BLAKE3 squeeze +/// with its reversed digest re-absorbed as the next segment. Any one of them +/// wrong by a byte moves a published challenge, and the module note above says +/// why no single-coordinate test can see it. +#[test] +fn the_blake3_replay_matches_the_host_transcript() { + use crypto::fiat_shamir::transcript_hash::Blake3TranscriptHash; + let exec = execute( + &oracle_replay_program(WrapHash::Blake3), + &oracle_arenas(), + &TestPermutation, + ) + .expect("the blake3 replay must execute"); + check_against_oracle( + &exec.public_words, + &oracle_expectation::(), + "blake3", + ); +} + +/// NON-VACUITY for the pair above: the two configurations must publish +/// DIFFERENT values, or both tests would pass against one oracle. +#[test] +fn the_two_replays_publish_different_challenges() { + let keccak = execute( + &oracle_replay_program(WrapHash::Keccak), + &oracle_arenas(), + &TestPermutation, + ) + .expect("keccak replay"); + let blake3 = execute( + &oracle_replay_program(WrapHash::Blake3), + &oracle_arenas(), + &TestPermutation, + ) + .expect("blake3 replay"); + assert_ne!( + keccak.public_words, blake3.public_words, + "the two transcript configurations must not agree on this script" + ); +} + +/// The select chain's RULE is the host's fallback rule, at every pattern of two +/// candidates including both-miss. +/// +/// ⚠ What this pins and what it does not. It compares the host's +/// `candidate_under_fixed_schedule` against a transcription of the rule +/// `sample_felt` emits — it is a statement about the SPEC the emitter follows, +/// not about the instructions it emits. That the emitted instructions really +/// implement this rule is the oracle test above, end to end. Both are needed: +/// the oracle cannot reach the both-miss corner (probability ≈ 2⁻⁶⁴), and this +/// cannot see an emitter that computes the right rule over the wrong operands. +#[test] +fn the_select_chain_rule_is_the_hosts_fallback() { + use math::field::traits::HasDefaultTranscript; + const P: u64 = 0xFFFF_FFFF_0000_0001; + + // Verbatim from `default_transcript::candidate_under_fixed_schedule`, which + // is `pub(crate)` in `crypto` and so cannot be called from here. + fn candidate_under_fixed_schedule( + n: usize, + mut next: impl FnMut() -> u64, + ) -> u64 { + let mut chosen: Option = None; + let mut last = 0u64; + for _ in 0..n { + let candidate = next(); + last = candidate; + if chosen.is_none() && Fld::candidate_in_range(candidate) { + chosen = Some(candidate); + } + } + chosen.unwrap_or(last) + } + + let vectors: [(u64, u64, &str); 4] = [ + (7, 9, "both in range -> the first"), + (P + 5, 9, "the first misses -> the second"), + (7, P + 5, "the second misses -> still the first"), + ( + P + 5, + P + 11, + "both miss -> the LAST, which is then rejected", + ), + ]; + for (c0, c1, what) in vectors { + let mut it = [c0, c1].into_iter(); + let host = candidate_under_fixed_schedule::(2, || { + it.next().expect("two candidates") + }); + // `sample_felt`'s rule, transcribed: the first in range, else the last. + let machine = if c0 < P { c0 } else { c1 }; + assert_eq!(host, machine, "{what}: c0={c0:#x} c1={c1:#x}"); + } +} + +/// ★ THE FLIP INVENTORY, pinned — which registered programs the Stage-6 flip +/// moved, and which it must NOT have. +/// +/// Review finding rev-emit E3: every `LfmProgramSource` constructor in +/// `programs.rs` builds its own `LfmBuilder`, and before the flip only two of +/// the twenty passed a hash. Each production constructor was edited +/// individually, and this is what catches one left behind — a constructor still +/// on keccak inside a machine everything else believes is BLAKE3 is a valid +/// proof of the wrong digest (R-3), not a failure. +/// +/// **The polarity is now post-flip**, which is the point: before the flip this +/// test asserted `blake3 == 0` for every program, and that assertion passing +/// today would mean the flip did not happen. The counts are MEASURED — a +/// BLAKE3 count is not the keccak count it replaced, because a 136-byte sponge +/// rate and a 64-byte compression block do not divide a message the same way. +/// +/// The classification stays measured rather than asserted: a program that emits +/// no hash instruction is flip-inert whatever its constructor says. The +/// categories are named per entry so adding a registered program forces the +/// question rather than inheriting an answer. +#[test] +fn the_flip_inventory_of_registered_programs_is_pinned() { + /// Compressions the flipped registered programs emit, MEASURED on the + /// flipped tree. + /// + /// These are not the keccak counts they replaced (6 and 5). `Blake3Chain` + /// takes a 64-byte block where the keccak sponge takes a 136-byte rate, so + /// the same message divides into more compressions — which is the in-machine + /// half of the cost the campaign priced, visible here as a row count. + /// + /// ★ StatementReplayV0 moved 9 → 10 on the main-sync port, and the mover is + /// NAMED rather than absorbed. + /// + /// MOVER: `892c7d1bc` (main's `c2ac5d546`, #977) — arm (ii), the STATEMENT + /// ENCODING. That merge's two arms are (i) the table set shrank, empty + /// tables now being elided rather than padded, and (ii) the absorbed epoch + /// statement grew. This pin is on (ii); `epoch_verify_tests`' SUB_PROOFS is + /// on (i). + /// + /// #977 took the six accelerator chips out of + /// `FIXED_TABLE_COUNT` (11 → 5) and made them counted, so the epoch + /// statement absorbs six more `u64` counts, and the same PR appended + /// `is_final` as the statement's last byte. `NUM_TABLE_COUNTS` went 15 → 21 + /// and `EpochStatementShape::byte_len` gained a trailing `+ 1`: the + /// statement is **+49 bytes**, which is under one 64-byte `Blake3Chain` + /// block and therefore worth exactly one more compression. + /// + /// TranscriptReplayV0 stays at 8, and that is what keeps this a pair rather + /// than two literals: it replays no statement, so a change that moved BOTH + /// counts would not be this one and would have to name itself. + const TRANSCRIPT_REPLAY_BLAKE3_ROWS: usize = 8; + const STATEMENT_REPLAY_BLAKE3_ROWS: usize = 10; + + use super::instr::Instr; + use super::programs::{ + KECCAK_SPONGE_LEN, fri_toy_program, keccak_chain_program, keccak_sponge_program, + statement_replay_program, transcript_replay_program, trivial_program, + }; + + /// What the Stage-6 flip did to each registered program. + #[derive(Debug, PartialEq, Eq)] + enum Fate { + /// Emits no hash at all — the flip could not move it. + Inert, + /// Emits the wrap hash, and its constructor took the flip. Emits BLAKE3 + /// and no keccak. + Flipped, + /// Emits keccak deliberately: an instrument that is ABOUT keccak, whose + /// identity is pinned in `LFM_REGISTRY`. A BLAKE3 twin would be a new + /// program and a new row, never a re-blessing of this one. + PinnedKeccak, + } + + // (name, program, fate, keccak rows, blake3 rows) + let cases: [(&str, super::compiler::LfmProgram, Fate, usize, usize); 6] = [ + ("TrivialV0", trivial_program(), Fate::Inert, 0, 0), + ("FriToyV0", fri_toy_program(), Fate::Inert, 0, 0), + ( + "KeccakChainV0", + keccak_chain_program(), + Fate::PinnedKeccak, + 2, + 0, + ), + ( + "KeccakSpongeV0", + keccak_sponge_program(KECCAK_SPONGE_LEN), + Fate::PinnedKeccak, + 2, + 0, + ), + ( + "TranscriptReplayV0", + transcript_replay_program(), + Fate::Flipped, + 0, + TRANSCRIPT_REPLAY_BLAKE3_ROWS, + ), + ( + "StatementReplayV0", + statement_replay_program(), + Fate::Flipped, + 0, + STATEMENT_REPLAY_BLAKE3_ROWS, + ), + ]; + + assert_eq!( + cases.len(), + super::registry::LFM_REGISTRY.len(), + "every registered program must have a stated fate — a new row without \ + one is a program the flip would move or miss by accident" + ); + + for (name, program, fate, keccak_rows, blake3_rows) in &cases { + let keccak = program + .instrs + .iter() + .filter(|i| matches!(i, Instr::KeccakF(_))) + .count(); + let blake3 = program + .instrs + .iter() + .filter(|i| matches!(i, Instr::Blake3(_))) + .count(); + assert_eq!( + keccak, *keccak_rows, + "{name}: the emitted keccak count is what the fate below is a \ + judgement about" + ); + assert_eq!(blake3, *blake3_rows, "{name}: emitted BLAKE3 count moved"); + match fate { + Fate::Inert => assert_eq!( + keccak + blake3, + 0, + "{name} is classified flip-inert but emits hash rows" + ), + // The two halves are what make this an oracle for a missed + // constructor rather than a count: a site left behind shows up as + // keccak > 0, and a site that flipped but emits nothing shows up as + // blake3 == 0. + Fate::Flipped => { + assert_eq!( + keccak, 0, + "{name} must not emit keccak after the flip — its \ + constructor was left on the default" + ); + assert!( + blake3 > 0, + "{name} is classified flipped but emits no BLAKE3" + ); + } + Fate::PinnedKeccak => { + assert!( + keccak > 0, + "{name} is classified as hashing keccak but emits none" + ); + assert_eq!( + blake3, 0, + "{name} is a keccak instrument and must not follow the flip" + ); + } + } + } + + // NON-VACUITY: the classification must actually split the set, or "every + // program has a fate" is satisfied by giving them all the same one. + assert!(cases.iter().any(|c| c.2 == Fate::Inert)); + assert!(cases.iter().any(|c| c.2 == Fate::Flipped)); + assert!(cases.iter().any(|c| c.2 == Fate::PinnedKeccak)); +} + +/// ★ The REVERSED-DIGEST send, PROVED — the chip's last surface that execution +/// alone does not reach. +/// +/// `the_reversed_digest_is_the_digest_backwards` executes the send and checks +/// its value; `the_blake3_chip_proves_and_verifies` proves the chip but through +/// `blake3_256`, which sets no reversed-digest multiplicity, so the two extra +/// `LfmMem` sends are inert in every proof either of them builds. That leaves +/// the flipped-coefficient `Linear` — the one piece of column arithmetic +/// transcribed from `chips::keccak` onto a different chip's OUT block — +/// exercised by the executor's mirror and by nothing on the AIR side. +/// +/// This proves it, at several lengths so the digest being reversed is a +/// different 32 bytes each time. A transcription error in +/// `reversed_lane_value`'s `OUT + 31 − 4l − k` would leave the executor and the +/// bus disagreeing about what was written, which is an unbalanced `LfmMem` +/// multiset: the proof fails to build, or fails to verify. +#[test] +fn the_reversed_digest_send_proves_and_verifies() { + let opts = options(); + // 0 exercises the single-block case, 65 the chain, 200 an interior block. + for len in [0usize, 65, 200] { + let msg = message(len); + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let num_halves = num_stream_halves(len) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec<_> = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let (plain, rev) = edsl::blake3_256_with_rev(&mut b, &stream, len); + b.public(plain[0]); + b.public(plain[1]); + b.public(rev[0]); + b.public(rev[1]); + let program = compile(b.finish()); + + // The multiplicities really are nonzero, or the send under test is the + // inert one again and this proves nothing new. + let group = &program.groups.blake3; + let last = group.real_rows - 1; + assert!( + (0..cols::DIGEST_WORDS).any(|w| *group.at(last, cols::rev_mult(w)) != FE::zero()), + "len {len}: the reversed digest must be READ, or its send is inert" + ); + + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts) + .unwrap_or_else(|e| panic!("len {len}: prove failed: {e:?}")); + + let plain_bytes = digest_bytes(&proved.public_words[..2]); + let rev_bytes = digest_bytes(&proved.public_words[2..]); + assert_eq!(plain_bytes, blake3_chain(&msg), "len {len}: plain digest"); + let mut expected = plain_bytes; + expected.reverse(); + assert_eq!(rev_bytes, expected, "len {len}: reversed digest"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "len {len}: a proof carrying the reversed-digest send must verify" + ); + } +} + +/// ★ R-8: BOTH BLAKE3 surfaces active in one machine, and the shared BITWISE +/// table balances for the right reason. +/// +/// Registering `LFM_BLAKE3` while D0 keeps the socket arm means one program can +/// carry two BLAKE3 AIRs — the socket (`LFM_HASH` under `HasherKind::Blake3`, +/// a 52-byte fixed message, 128-bit digest) and the chip (a raw compression, +/// 256-bit) — and both feed lookups into the SAME 2^20-row BITWISE table. They +/// run the same `run_flow` under different `FlowConfig`s, which is exactly the +/// shape where two producers could balance the shared table between them rather +/// than each against the table: a socket row's missing lookup absorbed by a chip +/// row's spare one would leave the bus balanced and one of the two computations +/// unconstrained. +/// +/// The control is the honest-path one done twice over: each surface alone +/// proves and verifies, and the two TOGETHER prove and verify. A cross-balance +/// would show up as the combined program proving while one of the singles does +/// not, or as the combined proof failing to verify — the histogram is built from +/// the senders' own enumeration, so a mismatch is an unbalanced multiset either +/// way. +#[test] +fn both_blake3_surfaces_in_one_machine_balance_bitwise() { + use super::hash::HasherKind; + use super::proof::lfm_prove_with_hasher; + + // Three programs: socket only, chip only, and both — the same builder + // shapes, so what differs between them is which surface is present. + let socket_only = |with_chip: bool, with_socket: bool| { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let zero = b + .digest_const([FE::zero(), FE::zero(), FE::zero(), FE::zero()]) + .as_cell(); + if with_socket { + // The socket: an `LFM_HASH` compress, whose lanes must be u32. + let a = b + .digest_const(core::array::from_fn(|i| FE::from(1_000u64 + i as u64))) + .as_cell(); + let d = b.compress(a.as_digest(), zero.as_digest()); + b.public(d.as_cell()); + } + if with_chip { + // The chip: one raw compression at the parent framing. + let m: [Cell; 4] = core::array::from_fn(|w| { + b.digest_const(core::array::from_fn(|l| { + FE::from(7_000u64 + (4 * w + l) as u64) + })) + .as_cell() + }); + let h = [ + b.digest_const(core::array::from_fn(|i| FE::from(u64::from(BLAKE3_IV[i])))) + .as_cell(), + b.digest_const(core::array::from_fn(|i| { + FE::from(u64::from(BLAKE3_IV[4 + i])) + })) + .as_cell(), + ]; + let params = b + .digest_const([ + FE::zero(), + FE::zero(), + FE::from(BLOCK_LEN as u64), + FE::from(u64::from(block_flags(0, 1))), + ]) + .as_cell(); + let out = b.blake3_compress(h, m, params); + b.public(out[0]); + } + compile(b.finish()) + }; + + let opts = options(); + for (what, program, socket_rows, chip_rows) in [ + ("socket alone", socket_only(false, true), 1, 0), + ("chip alone", socket_only(true, false), 0, 1), + ("★ both surfaces", socket_only(true, true), 1, 1), + ] { + // NON-VACUITY: the three programs must really differ in which surface + // they carry, or "both together verify" is one surface tested thrice. + assert_eq!( + program.groups.hash.real_rows, socket_rows, + "{what}: LFM_HASH (socket) rows" + ); + assert_eq!( + program.groups.blake3.real_rows, chip_rows, + "{what}: LFM_BLAKE3 (chip) rows" + ); + let artifacts = + super::registry::build_artifacts_with_hasher(&program, &opts, HasherKind::Blake3); + let proved = lfm_prove_with_hasher(&program, &artifacts, &[], &opts, HasherKind::Blake3) + .unwrap_or_else(|e| panic!("{what}: must prove: {e:?}")); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + HasherKind::Blake3, + artifacts.chip_set, + ), + "{what}: must verify — a shared-BITWISE cross-balance between the \ + socket and the chip would show up here" + ); + } +} + +// ========================================================================= +// `LFM_BLAKE3` chunking — the chip past one table +// ========================================================================= +// +// One row per compression at 3,056 value columns, so the chip's matrix is WIDE: +// the aggregation program's ~1.39M compressions are a 2^21 x 3,056 table whose +// blowup-2 LDE is a single ~102 GB allocation. These tests cover the split — the +// shape it produces, that a multi-chunk program proves and verifies, that it +// proves the SAME thing, and the two ways the split itself can be wrong (a +// corrupted non-first chunk, a chunk count that does not match the proof). + +use super::chunking::Blake3Chunking; + +/// 12 compressions: 768 bytes is exactly 12 BLAKE3 blocks. +const CHUNKED_CHAIN_LEN: usize = 768; + +/// Five compressions per chunk. 12 splits 5 + 5 + 2, so the fixture is three +/// chunks with a partial final one — the case a uniform split would miss — and +/// two distinct trace heights (8, 8, 4), so a height read off the wrong chunk +/// cannot pass unnoticed. +fn test_blake3_chunking() -> Blake3Chunking { + Blake3Chunking::from_compressions(5) +} + +fn chunked_chain_program() -> super::compiler::LfmProgram { + blake3_sponge_program(CHUNKED_CHAIN_LEN).with_blake3_chunking(test_blake3_chunking()) +} + +/// The split's shape: chunk count, per-chunk compression counts, per-chunk trace +/// heights, per-chunk roots, AIR count and trace count all agree. +#[test] +fn blake3_chunking_splits_the_chain_into_uneven_chunks() { + use super::airs::{ChipSet, blake3_chunk_rows, num_lfm_airs}; + + let program = chunked_chain_program(); + assert_eq!( + program.groups.blake3.real_rows, 12, + "{CHUNKED_CHAIN_LEN} bytes must be 12 BLAKE3 blocks" + ); + + assert_eq!(program.blake3_chunk_count(), 3); + assert_eq!(program.blake3_chunk_real_rows(), vec![5, 5, 2]); + // 5 rows -> 8; the 2-row tail -> the 4-row group floor. + assert_eq!(blake3_chunk_rows(&program), vec![8, 8, 4]); + + let opts = options(); + let artifacts = build_artifacts(&program, &opts); + assert_eq!(artifacts.blake3_chunks(), 3); + assert_eq!(artifacts.blake3_chunk_log_heights, vec![3, 3, 2]); + assert_eq!( + artifacts.roots[super::airs::BLAKE3_SLOT], + artifacts.blake3_chunk_roots[0], + "slot 11's root IS chunk 0's — that is what makes one chunk the \ + unchunked machine" + ); + assert_eq!( + artifacts.log_heights[super::airs::BLAKE3_SLOT], + artifacts.blake3_chunk_log_heights[0] + ); + // The chunks commit different rows, so no two share a root. Without this the + // per-chunk commitment could be committing the same matrix three times. + for i in 0..3 { + for j in (i + 1)..3 { + assert_ne!( + artifacts.blake3_chunk_roots[i], artifacts.blake3_chunk_roots[j], + "chunks {i} and {j} must commit different matrices" + ); + } + } + // A BLAKE3-only program: the keccak family is masked out, so the AIR count is + // the FULL-mask count less that family. + assert_eq!( + (artifacts.chip_set.keccak, artifacts.chip_set.blake3), + (false, true) + ); + assert_eq!(artifacts.chip_set.num_airs(0, 3), num_lfm_airs(0, 3) - 2); + + let exec = execute( + &program, + &sponge_arenas(&message(CHUNKED_CHAIN_LEN)), + &TestPermutation, + ) + .expect("honest execution"); + let traces = super::trace::build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + assert_eq!(traces.blake3.len(), 3, "one LFM_BLAKE3 trace per chunk"); + assert_eq!( + traces + .blake3 + .iter() + .map(|t| t.num_rows()) + .collect::>(), + vec![8, 8, 4], + "chunk traces must match the heights the artifacts predict" + ); + let _ = ChipSet::FULL; +} + +/// The arithmetic the census and the round shape read must be the split the +/// prover actually materializes. Two derivations of one boundary is how a chunk +/// comes to commit a different matrix than the one it proves. +#[test] +fn the_blake3_chunk_arithmetic_is_the_group_split() { + use super::airs::blake3_chunk_rows; + + for per in [1usize, 2, 5, 7, 12, 13] { + let program = blake3_sponge_program(CHUNKED_CHAIN_LEN) + .with_blake3_chunking(Blake3Chunking::from_compressions(per)); + let real = program.blake3_chunk_real_rows(); + let padded = blake3_chunk_rows(&program); + assert_eq!(real.len(), program.blake3_chunk_count()); + assert_eq!(real.iter().sum::(), 12, "per={per}: rows were lost"); + for (c, (&r, &p)) in real.iter().zip(&padded).enumerate() { + let group = program.blake3_chunk_group(c); + assert_eq!(group.real_rows, r, "per={per} chunk {c}: real rows"); + assert_eq!(group.padded_rows, p, "per={per} chunk {c}: padded rows"); + assert_eq!(group.width, program.groups.blake3.width); + assert_eq!(group.data.len(), p * group.width); + // And the rows really are this chunk's slice of the whole group. + let base = c * per; + for row in 0..r { + for col in 0..group.width { + assert_eq!( + group.at(row, col), + program.groups.blake3.at(base + row, col), + "per={per} chunk {c} row {row} col {col}: wrong source row" + ); + } + } + } + } +} + +/// ★ The acceptance test: a program needing three `LFM_BLAKE3` chunks proves and +/// verifies COMPLETELY, and its digest is still the host chain's. +#[test] +fn chunked_blake3_proves_and_verifies() { + use super::proof::verify_against_artifacts; + + let opts = options(); + let msg = message(CHUNKED_CHAIN_LEN); + let program = chunked_chain_program(); + let artifacts = build_artifacts(&program, &opts); + assert_eq!(artifacts.blake3_chunks(), 3, "this test needs 3 chunks"); + + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + assert_eq!( + digest_bytes(&proved.public_words), + blake3_chain(&msg), + "a chunked proof must hash the same as the host chain" + ); + assert_eq!( + stark::proof::view::MultiProofView::Owned(&proved.proof).len(), + artifacts.chip_set.num_airs(0, 3), + "the proof must carry one sub-proof per AIR instance" + ); + assert!( + verify_against_artifacts(&artifacts, &proved.proof, &proved.public_words, &opts), + "a three-chunk LFM_BLAKE3 proof must verify" + ); +} + +/// ★ Chunking is a prover-side layout choice, not a semantic one: the same +/// message proved at 1 and at 3 chunks publishes byte-identical words. +/// +/// The program *identity* does differ — and here, unlike `KECCAK_RND` chunking, +/// so does slot 11's ROOT, because this chip's chunks each commit their own +/// slice of the instruction group. Every other root is untouched, which is what +/// says the split is confined to the chip it names. +#[test] +fn blake3_chunking_does_not_change_what_is_proved() { + use super::airs::BLAKE3_SLOT; + use super::proof::verify_against_artifacts; + + let opts = options(); + let msg = message(CHUNKED_CHAIN_LEN); + + let one = blake3_sponge_program(CHUNKED_CHAIN_LEN); + let one_artifacts = build_artifacts(&one, &opts); + assert_eq!(one_artifacts.blake3_chunks(), 1); + let one_proof = lfm_prove(&one, &one_artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + let three = chunked_chain_program(); + let three_artifacts = build_artifacts(&three, &opts); + let three_proof = + lfm_prove(&three, &three_artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + assert_eq!( + one_proof.public_words, three_proof.public_words, + "chunking must not change the program's output" + ); + for slot in 0..super::airs::NUM_LFM_CHIPS { + if slot == BLAKE3_SLOT { + continue; + } + assert_eq!( + one_artifacts.roots[slot], three_artifacts.roots[slot], + "slot {slot}: chunking LFM_BLAKE3 must move no other chip's root" + ); + } + assert_ne!( + one_artifacts.roots[BLAKE3_SLOT], three_artifacts.roots[BLAKE3_SLOT], + "chunk 0 is a different matrix than the whole group, so its root differs" + ); + assert_ne!( + one_artifacts.program_id, three_artifacts.program_id, + "the chunk shape is program shape and must be bound into the digest" + ); + for (artifacts, proof) in [ + (&one_artifacts, &one_proof), + (&three_artifacts, &three_proof), + ] { + assert!( + verify_against_artifacts(artifacts, &proof.proof, &proof.public_words, &opts), + "both chunkings must verify against their own artifacts" + ); + } + // And neither crosses: the digests bind the shape, so a proof of one split + // is not a proof under the other's identity. + assert!( + !verify_against_artifacts( + &one_artifacts, + &three_proof.proof, + &three_proof.public_words, + &opts + ), + "a three-chunk proof must not verify against the single-table identity" + ); + assert!( + !verify_against_artifacts( + &three_artifacts, + &one_proof.proof, + &one_proof.public_words, + &opts + ), + "a single-table proof must not verify against the three-chunk identity" + ); +} + +/// ★ Tamper: corrupting a compression that lives in the LAST chunk must reject. +/// The first two chunks are untouched, so this only rejects if chunk 2's rows are +/// really part of the proof's bus balance. +#[test] +fn a_tampered_non_first_blake3_chunk_rejects() { + use super::proof::verify_against_artifacts; + + let opts = options(); + let msg = message(CHUNKED_CHAIN_LEN); + let program = chunked_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let exec = execute(&program, &sponge_arenas(&msg), &TestPermutation).expect("execute"); + + let mut traces = + super::trace::build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + assert_eq!(traces.blake3.len(), 3); + // One output byte of the LAST chunk's first compression — the eleventh of + // the twelve, which no other chunk carries. + let col = cols::out_word(0, 0); + let old = traces.blake3[2].main_table.get_row(0)[col]; + traces.blake3[2].main_table.set_fe(0, col, old + FE::one()); + + match super::proof::prove_traces(&artifacts, &mut traces, &exec.public_words, &opts) { + Err(_) => {} + Ok(proof) => assert!( + !verify_against_artifacts(&artifacts, &proof, &exec.public_words, &opts), + "a corrupted compression in the third chunk must reject" + ), + } +} + +/// The verifier builds its AIR set from the supplied chunk roots, so a chunk list +/// that disagrees with the proof's shape must be rejected — including the +/// single-root list `verify_against` supplies, which is why a chunked caller has +/// to go through the artifacts door. +#[test] +fn verify_rejects_a_blake3_chunk_list_that_does_not_match_the_proof() { + use super::proof::verify_against_chunked; + + let opts = options(); + let msg = message(CHUNKED_CHAIN_LEN); + let program = chunked_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + // The single-table door: one instance against a three-instance proof. + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the single-LFM_BLAKE3 door must not verify a three-chunk proof" + ); + // Every wrong length, and the right length with the TAIL roots swapped. + // Chunk 0 stays put deliberately: it is the roots array's slot-11 entry, and + // a list that disagreed with it would be a caller bug the AIR set asserts on, + // not a proof outcome. Swapping chunks 1 and 2 — which commit different + // matrices at different heights — is the same falsification without that. + let mut roots = artifacts.blake3_chunk_roots.clone(); + roots.swap(1, 2); + for wrong in [ + &artifacts.blake3_chunk_roots[..1], + &artifacts.blake3_chunk_roots[..2], + &roots[..], + ] { + assert!( + !verify_against_chunked( + &artifacts.roots, + wrong, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a {}-root chunk list must not verify this proof", + wrong.len() + ); + } +} + +/// ★ The census counts every chunk — one entry per sub-proof, at the chunk's own +/// height, and the cell totals are that decomposition summed. +#[test] +fn the_census_counts_every_blake3_chunk() { + use super::airs::{HeightRule, blake3_chunk_rows, lfm_cell_counts, lfm_chip_census}; + + let one = blake3_sponge_program(CHUNKED_CHAIN_LEN); + let three = chunked_chain_program(); + + for (label, program, chunks) in [("1 chunk", &one, 1usize), ("3 chunks", &three, 3)] { + let census = lfm_chip_census(program); + let artifacts = build_artifacts(program, &options()); + assert_eq!( + census.len(), + artifacts + .chip_set + .num_airs(artifacts.keccak_rnd_chunks, artifacts.blake3_chunks()), + "{label}: the census must have one entry per sub-proof" + ); + let entries: Vec<_> = census.iter().filter(|c| c.name == "LFM_BLAKE3").collect(); + assert_eq!(entries.len(), chunks, "{label}: one census entry per chunk"); + assert_eq!( + entries.iter().map(|c| c.rows).collect::>(), + blake3_chunk_rows(program) + .into_iter() + .map(|r| r as u64) + .collect::>(), + "{label}: census heights must be the chunk heights" + ); + assert_eq!( + entries.iter().map(|c| c.real_rows).sum::(), + 12, + "{label}: the chunks' real rows must be the program's compressions" + ); + // A split table's headroom is a policy artefact, not a cliff; a single + // table's is the workload's own. + let want = if chunks > 1 { + HeightRule::Chunked + } else { + HeightRule::Workload + }; + assert!( + entries.iter().all(|c| c.height_rule == want), + "{label}: wrong height rule" + ); + // The totals ARE this decomposition, so they cannot disagree with it. + let (main, aux) = lfm_cell_counts(program); + assert_eq!( + (main, aux), + census.iter().fold((0u64, 0u64), |(m, a), c| ( + m + c.main_cells(), + a + c.aux_cells() + )), + "{label}: cell totals must be the census summed" + ); + } +} + +/// The knob's whole path: a variable VALUE becomes a policy, the policy becomes +/// chunks, and the chunked program proves and verifies. The parse itself is +/// tested in [`super::chunking`]; this is what says the value reaches the +/// machine rather than stopping at a struct nobody reads. +#[test] +fn the_env_knob_value_chunks_a_real_prove() { + use super::proof::verify_against_artifacts; + + let opts = options(); + let msg = message(CHUNKED_CHAIN_LEN); + let chunking = Blake3Chunking::from_env_value(Some("2")).expect("a set knob is a policy"); + let program = blake3_sponge_program(CHUNKED_CHAIN_LEN).with_blake3_chunking(chunking); + assert_eq!( + program.blake3_chunk_count(), + 3, + "2^2 rows per chunk over 12 compressions is three chunks" + ); + + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + assert_eq!(digest_bytes(&proved.public_words), blake3_chain(&msg)); + assert!( + verify_against_artifacts(&artifacts, &proved.proof, &proved.public_words, &opts), + "the knob's own shape must prove and verify" + ); +} diff --git a/prover/src/lfm/blake3_probe.rs b/prover/src/lfm/blake3_probe.rs new file mode 100644 index 000000000..f3a2db57f --- /dev/null +++ b/prover/src/lfm/blake3_probe.rs @@ -0,0 +1,615 @@ +//! Prove + verify the LFM-hosted BLAKE3 compression chip standalone. +//! +//! The `keccak_probe` pattern, one hash later: [`super::blake3_chip`] carries +//! the chip's real bus interactions and its real constraints, and this module +//! closes both of its buses — `BusId::ByteAlu` / `BusId::AreBytes` against the +//! UNCHANGED production `BITWISE` table, and `BusId::LfmMem` against a mirror +//! AIR standing in for the machine's memory. The preprocessed prefix is +//! committed for real, so the addresses and multiplicities the chip reads are +//! program data here exactly as they would be in the machine. +//! +//! Standing-decisions rule 2 is why this exists: an execute-only test would +//! prove nothing about the chip, because [`super::blake3::blake3_compress_6round`] +//! and the chip's `ValueFlow` would simply agree with each other. Only a +//! prove+verify makes the chip's constraints and interactions load +//! bearing, which is what turns the measured width into a *column*. +//! +//! # What this probe cannot see +//! +//! - **Whether the machine can drive the chip.** The mirror AIR is a synthetic +//! memory: it sends whatever the ops say the inputs are. Nothing here checks +//! that an LFM program can produce those words at those addresses, that the +//! admission validator would accept the address assignment, or that the +//! multiplicities match real read counts. Those are registrar obligations and +//! they are exactly what registering the chip would exercise. +//! - **The epoch verifier's blake bill.** This measures cells per compression. +//! The permutation count comes from wave 8's rate-parameterised closed form +//! and is inherited, not re-established here. +//! - **Anything cryptographic about the 6-round variant** (assumption A6R). +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::config::Commitment; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, EmptyConstraints, RootKind, num_base_from_meta, +}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, +}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::MultiProofView; +use stark::prover::IsStarkProver; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::IsStarkVerifier; + +use crate::tables::bitwise; +use crate::tables::types::{BusId, FE, FEE, GoldilocksExtension, GoldilocksField, VmTable}; +use crate::test_utils::create_bitwise_air; + +use super::blake3::{BLAKE3_ROUNDS, CANONICAL_VECTORS, canonical_expected_out}; +use super::blake3_chip::{ + self, Blake3LfmConstraints, Blake3Operation, Blake3Values, IN_WORDS, MAIN_COLUMNS, + NUM_CONSTRAINTS, NUM_G, OUT_WORDS, cols, +}; +use super::commit::commit_columns; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type DynAir<'a> = &'a dyn AIR; +type ChipAir = AirWithBuses; +type MirrorAir = AirWithBuses; + +const PROBE_TAG: &[u8] = b"LFM_BLAKE3_PROBE_V1"; +/// Compressions in the probe. Three real rows in a height-4 table leaves one +/// padding row, which `padding_row_turned_real_rejects` needs. +const NUM_OPS: usize = 3; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("probe options") +} + +fn transcript() -> crate::hash_pin::BlockTranscript { + let mut t = crate::hash_pin::block_transcript(&[]); + t.append_bytes(PROBE_TAG); + t +} + +// ========================================================================= +// The ops +// ========================================================================= + +/// Three compressions taken from the canonical 6-round vectors, at disjoint +/// addresses. +/// +/// Using the pinned vectors rather than fresh randomness means the trace's own +/// OUT columns are checkable against a constant that came from outside this +/// repository's Rust (see [`super::blake3`]'s provenance note). +fn probe_ops() -> Vec { + (0..NUM_OPS) + .map(|i| { + let v = &CANONICAL_VECTORS[i]; + let base = 1_000 + (i as u64) * 100; + Blake3Operation { + in_addr: core::array::from_fn(|j| base + j as u64), + out_addr: core::array::from_fn(|j| base + 50 + j as u64), + // Distinct nonzero read counts: a uniform 1 would not notice a + // multiplicity mixed up between output words. + read_counts: core::array::from_fn(|j| 1 + j as u64), + values: Blake3Values { + h: v.h, + m: v.m, + t: v.t, + block_len: v.block_len, + flags: v.flags, + }, + } + }) + .collect() +} + +/// The value halves of a probe op list — what the BITWISE feed is computed from. +fn probe_values(ops: &[Blake3Operation]) -> Vec { + ops.iter().map(|op| op.values).collect() +} + +// ========================================================================= +// The AIRs +// ========================================================================= + +/// The preprocessed prefix, column-major and padded — what the program would +/// supply and what the chip's addresses and multiplicities are read from. +fn prep_columns(ops: &[Blake3Operation], num_rows: usize) -> Vec> { + let mut columns = vec![vec![FE::zero(); num_rows]; cols::PREP_WIDTH]; + for (row, op) in ops.iter().enumerate() { + for j in 0..IN_WORDS { + columns[cols::in_addr(j)][row] = FE::from(op.in_addr[j]); + } + for j in 0..OUT_WORDS { + columns[cols::out_addr(j)][row] = FE::from(op.out_addr[j]); + columns[cols::mult(j)][row] = FE::from(op.read_counts[j]); + } + columns[cols::MU][row] = FE::one(); + } + columns +} + +fn chip_air(prep_root: Commitment, opts: &ProofOptions) -> ChipAir { + AirWithBuses::new( + cols::NUM_COLUMNS, + AuxiliaryTraceBuildData { + interactions: blake3_chip::bus_interactions(), + }, + opts, + 1, + Blake3LfmConstraints, + ) + .with_name("LFM_BLAKE3") + .with_preprocessed(prep_root, cols::PREP_WIDTH) +} + +/// A synthetic `LfmMem` counterparty: `[ADDR, V0..V3, SEND_MULT, RECV_MULT]`. +/// +/// One row per word the chip touches. Input words are SENT here (the chip +/// receives them); output words are RECEIVED here `read_counts` times (the chip +/// sends them once with that multiplicity). Nothing constrains the values — the +/// mirror is memory, and in the machine the `LfmMem` multiset IS the semantics. +mod mirror { + pub const ADDR: usize = 0; + pub const V0: usize = 1; // ..V3 + pub const SEND_MULT: usize = 5; + pub const RECV_MULT: usize = 6; + pub const NUM_COLUMNS: usize = 7; +} + +fn mirror_token() -> Vec { + let mut v = vec![BusValue::Packed { + start_column: mirror::ADDR, + packing: Packing::Direct, + }]; + v.extend((0..4).map(|l| BusValue::Packed { + start_column: mirror::V0 + l, + packing: Packing::Direct, + })); + v +} + +fn mirror_air(opts: &ProofOptions) -> MirrorAir { + let interactions = vec![ + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(mirror::SEND_MULT), + mirror_token(), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(mirror::RECV_MULT), + mirror_token(), + ), + ]; + AirWithBuses::new( + mirror::NUM_COLUMNS, + AuxiliaryTraceBuildData { interactions }, + opts, + 1, + EmptyConstraints, + ) + .with_name("LFM_MEM_MIRROR") +} + +/// Four `u32` lanes of machine word `word` out of a flat `u32` array. +fn lanes(words: &[u32], word: usize) -> [u64; 4] { + core::array::from_fn(|l| words[4 * word + l] as u64) +} + +fn mirror_trace(ops: &[Blake3Operation]) -> TraceTable { + let rows = (ops.len() * (IN_WORDS + OUT_WORDS)) + .next_power_of_two() + .max(4); + let mut trace = TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(rows * mirror::NUM_COLUMNS), + mirror::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + let mut row = 0usize; + for op in ops { + let inputs = op.values.input_words(); + for j in 0..IN_WORDS { + table.set_u64(row, mirror::ADDR, op.in_addr[j]); + for (l, v) in lanes(&inputs, j).into_iter().enumerate() { + table.set_u64(row, mirror::V0 + l, v); + } + table.set_fe(row, mirror::SEND_MULT, FE::one()); + row += 1; + } + let outputs = op.values.output_words(); + for j in 0..OUT_WORDS { + table.set_u64(row, mirror::ADDR, op.out_addr[j]); + for (l, v) in lanes(&outputs, j).into_iter().enumerate() { + table.set_u64(row, mirror::V0 + l, v); + } + table.set_u64(row, mirror::RECV_MULT, op.read_counts[j]); + row += 1; + } + } + trace +} + +fn bitwise_trace(ops: &[Blake3Operation]) -> TraceTable { + let mut hist = bitwise::BitwiseHistogram::new(); + hist.add_ops(&blake3_chip::bitwise_ops_for(&probe_values(ops))); + let mut bw = bitwise::generate_bitwise_trace(); + hist.fill_multiplicities(&mut bw); + bw +} + +/// The three traces, in AIR order: chip, mirror, BITWISE. +fn build_traces(ops: &[Blake3Operation]) -> [TraceTable; 3] { + [ + blake3_chip::generate_blake3_trace(ops), + mirror_trace(ops), + bitwise_trace(ops), + ] +} + +fn prove_traces( + opts: &ProofOptions, + chip: &ChipAir, + traces: &mut [TraceTable; 3], +) -> Result, stark::prover::ProvingError> { + let mirror = mirror_air(opts); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + let [t0, t1, t2] = traces; + let pairs: Vec<(DynAir, &mut TraceTable, &())> = + vec![(chip, t0, &()), (&mirror, t1, &()), (&bw_air, t2, &())]; + let mut t = transcript(); + crate::hash_pin::BlockProver::multi_prove( + pairs, + &mut t, + stark::residency_mode::ResidencyMode::Retain, + ) +} + +fn verify_proof( + opts: &ProofOptions, + chip: &ChipAir, + proof: &stark::proof::stark::MultiProof, +) -> bool { + let mirror = mirror_air(opts); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + let refs: Vec = vec![chip, &mirror, &bw_air]; + let mut vt = transcript(); + crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(proof), + &mut vt, + &FEE::zero(), + ) +} + +/// Prove + verify, optionally corrupting the chip trace in between. +/// +/// `Err` means the prover refused — which for this chip is the *expected* +/// outcome of most tampering, because unlike the keccak adapter it carries 769 +/// polynomial constraints that a wrong cell violates locally. +fn round_trip(mutate: impl FnOnce(&mut TraceTable)) -> Result { + let opts = options(); + let ops = probe_ops(); + let num_rows = ops.len().next_power_of_two().max(4); + let root = commit_columns(&prep_columns(&ops, num_rows), &opts); + let chip = chip_air(root, &opts); + let mut traces = build_traces(&ops); + mutate(&mut traces[0]); + match prove_traces(&opts, &chip, &mut traces) { + Ok(proof) => Ok(verify_proof(&opts, &chip, &proof)), + Err(e) => Err(format!("{e:?}")), + } +} + +/// Assert a mutation does not end in an accepted proof. +/// +/// A refusal by the prover and a rejection by the verifier are both real +/// rejections and this chip produces both: a cell that violates one of its 769 +/// constraints is caught locally, while a cell that only breaks a bus reaches +/// the verifier. Each caller records which it observed in its own doc comment. +fn assert_not_accepted(what: &str, mutate: impl FnOnce(&mut TraceTable)) { + if let Ok(true) = round_trip(mutate) { + panic!("{what} must not produce an accepted proof, but the proof verified"); + } +} + +// ========================================================================= +// The measurement +// ========================================================================= + +/// The blake column's per-compression cell law, on our stack, at BOTH round +/// counts. +/// +/// `main + 3·aux` with `aux = ceil(interactions / 2)` is `airs.rs`'s census +/// formula — the same instrument that produced the keccak and Poseidon columns, +/// so the three are comparable by construction rather than by argument. +/// +/// The closed forms are written out as functions of the round count and the +/// literals for both are pinned, so the A6R price stays visible whichever way +/// the build is compiled; the built layout is then asserted to equal the +/// prediction at the compiled count. Two statements that can disagree. +#[test] +fn the_hosted_chip_cell_budget_at_both_round_counts() { + // 112 input bytes + `8·rounds` G-blocks of 60 + 64 feed-forward bytes. + const fn predicted_main(rounds: usize) -> usize { + 112 + 60 * (8 * rounds) + 64 + } + // 13 `LfmMem` tokens (7 read + 4 written + 2 reversed-digest); + // `ByteAlu[XOR]` over `4·8·rounds` mixing words and 16 feed-forward words; + // `AreBytes` over `2·8·rounds` rotations; 32 message. + const fn predicted_interactions(rounds: usize) -> usize { + 13 + 4 * (4 * (8 * rounds) + 16) + 4 * (2 * (8 * rounds)) + 32 + } + const fn predicted_cells(rounds: usize) -> usize { + predicted_main(rounds) + 3 * predicted_interactions(rounds).div_ceil(2) + } + + // 6 rounds — the A6R variant. These four literals are #903's and were the + // measured figures before the round count became a knob. + assert_eq!(predicted_main(6), 3_056); + assert_eq!(predicted_interactions(6), 1_261); + assert_eq!(predicted_interactions(6).div_ceil(2), 631); + assert_eq!(predicted_cells(6), 4_949); + // Group by group at 6 rounds, so a layout change cannot move the total + // silently: 13 LfmMem + 832 ByteAlu + 384 shift AreBytes + 32 message. + assert_eq!(predicted_interactions(6), 13 + 832 + 384 + 32); + + // 7 rounds — standard BLAKE3, the default. PLAN §7 predicted exactly these + // on paper; this is the same arithmetic against the built layout. + assert_eq!(predicted_main(7), 3_536); + assert_eq!(predicted_interactions(7), 1_453); + assert_eq!(predicted_interactions(7).div_ceil(2), 727); + assert_eq!(predicted_cells(7), 5_717); + + // The built layout IS the prediction at the compiled round count. + let interactions = blake3_chip::bus_interactions().len(); + let aux = interactions.div_ceil(2); + assert_eq!(cols::PREP_WIDTH, 20, "preprocessed prefix"); + assert_eq!(cols::G - cols::IN, 112, "input bytes"); + assert_eq!(cols::OUT - cols::G, 60 * NUM_G, "G-blocks × 60 cells"); + assert_eq!( + cols::NUM_COLUMNS - cols::OUT, + 64, + "feed-forward output bytes" + ); + assert_eq!(MAIN_COLUMNS, predicted_main(BLAKE3_ROUNDS)); + assert_eq!(cols::NUM_COLUMNS, MAIN_COLUMNS + cols::PREP_WIDTH); + assert_eq!( + IN_WORDS + OUT_WORDS + cols::DIGEST_WORDS, + 13, + "LfmMem tokens" + ); + assert_eq!(interactions, predicted_interactions(BLAKE3_ROUNDS)); + assert_eq!( + MAIN_COLUMNS + 3 * aux, + predicted_cells(BLAKE3_ROUNDS), + "base-field-equivalent cells" + ); + + // #903's syscall variant at 6 rounds, for the delta the hosting buys: 3,219 + // main and 1,397 interactions (699 aux) = 5,316. The difference is all I/O. + assert_eq!(3_219 + 3 * 1_397usize.div_ceil(2), 5_316); + + // ★ For the comparison this chip exists to support: the `LFM_HASH` BLAKE3 + // socket arm is cheaper at BOTH round counts — a constant initial state, a + // constant message tail, and twelve of the sixteen output words never built. + // + // Asserted against `blake3_socket_tests`' own census formula rather than + // against a transcription of its output. What stood here was "4,741 at 6 + // rounds and 5,509 at 7": those predated the leaf mode's canonicity block, + // nothing recomputed them, and they were wrong by 8 for as long as they + // stood — then wrong by 36 once the leaf RATE widened the socket. A cost + // figure no test derives is a comment, not a claim. + for rounds in [6, 7] { + let socket = super::blake3_socket_tests::predicted_cells(rounds); + assert!( + socket < predicted_cells(rounds), + "hosting must stay cheaper than the standalone chip at {rounds} \ + rounds: socket {socket}, standalone {}", + predicted_cells(rounds) + ); + } + assert_eq!(super::blake3_socket_tests::predicted_cells(6), 4_777); + assert_eq!(super::blake3_socket_tests::predicted_cells(7), 5_545); +} + +/// Every constraint index is emitted exactly once, and the count is the one the +/// module documents. The syscall chip's single-compression core emits 814; the +/// 45 address-derivation constraints have no counterpart here. +#[test] +fn the_chip_emits_its_constraints_at_degree_3() { + // 16 per G-instance — two add3s (a sum identity and two carry booleanities + // each), two add2 carry booleanities, two rotations of four — plus the + // ungated `IS_BIT(MU)`. 769 at 6 rounds, 897 at 7; both written out. + assert_eq!(16 * (8 * 6) + 1, 769); + assert_eq!(16 * (8 * 7) + 1, 897); + assert_eq!(NUM_CONSTRAINTS, 16 * NUM_G + 1); + assert_eq!( + NUM_CONSTRAINTS, + 3 * (NUM_G * 2) + NUM_G * 2 + 4 * (NUM_G * 2) + 1 + ); + // The syscall variant's single-compression core emits 814 at 6 rounds (its + // absorb mode adds 34 more, none of them per-G); the 45 address-derivation + // constraints have no counterpart here. + if BLAKE3_ROUNDS == 6 { + assert_eq!(814 - 45, NUM_CONSTRAINTS, "vs the syscall variant's core"); + } + + let set = Blake3LfmConstraints; + let meta = ConstraintSet::::meta(&set); + assert_eq!(meta.len(), NUM_CONSTRAINTS, "constraints emitted"); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "meta must be dense and idx-ordered"); + assert_eq!(m.kind, RootKind::Base, "every blake constraint is base"); + } + + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert_eq!(degrees.len(), NUM_CONSTRAINTS, "one emit per constraint"); + + let declared = ConstraintSet::::max_degree(&set); + assert_eq!(declared, 3, "the wrap's blowup 2 depends on this staying 3"); + for &(idx, measured) in °rees { + assert!( + measured <= declared, + "constraint {idx}: measured degree {measured} EXCEEDS declared {declared}" + ); + } + // Not merely `<=`: the μ-gated carry booleanities really are cubic, so a + // set that quietly topped out at 2 would mean the carries had stopped being + // constrained. + assert_eq!( + degrees.iter().map(|&(_, d)| d).max(), + Some(3), + "some constraint must actually reach degree 3" + ); +} + +// ========================================================================= +// The round trip +// ========================================================================= + +#[test] +fn the_hosted_chip_proves_and_verifies() { + let ops = probe_ops(); + let traces = build_traces(&ops); + + // The chip's OUT columns are the real compression, byte for byte, against + // the canonical vectors. + for (row, op) in ops.iter().enumerate() { + let expected = canonical_expected_out(row); + assert_eq!( + expected, + op.values.output_words(), + "op {row}'s output must be the primitive's at the compiled round count" + ); + for (i, &word) in expected.iter().enumerate() { + for b in 0..4 { + assert_eq!( + traces[0].main_table.get_row(row)[cols::out_word(i, b)], + FE::from(u64::from((word >> (8 * b)) as u8)), + "OUT byte ({i}, {b}) of row {row}" + ); + } + } + } + + // The BITWISE feed is exactly the senders' count, with no address-shaped + // lookups: 1,248 per compression at 6 rounds and 1,440 at 7. Both literals + // are written out so the flip cannot quietly move the feed. + const fn predicted_bitwise(rounds: usize) -> usize { + 4 * (4 * (8 * rounds) + 16) + 4 * (2 * (8 * rounds)) + 32 + } + assert_eq!(predicted_bitwise(6), 1_248); + assert_eq!(predicted_bitwise(7), 1_440); + assert_eq!( + blake3_chip::bitwise_ops_for(&probe_values(&ops)).len(), + ops.len() * predicted_bitwise(BLAKE3_ROUNDS), + "per-compression BITWISE lookup count" + ); + // And it is the interaction list less the 13 `LfmMem` tokens — the mirror + // property, which is what stops the feed and the senders from drifting. + // 13, not 11: registration added the two reversed-digest sends, which carry + // no BITWISE lookup because they are a second `Linear` over columns the + // plain digest send already covers. + assert_eq!( + predicted_bitwise(BLAKE3_ROUNDS) + IN_WORDS + OUT_WORDS + cols::DIGEST_WORDS, + blake3_chip::bus_interactions().len() + ); + + assert_eq!(round_trip(|_| {}), Ok(true), "honest proof must verify"); +} + +/// The padding rows carry nothing at all — no pointer pad, no witness. +#[test] +fn padding_rows_are_all_zero() { + let ops = probe_ops(); + let traces = build_traces(&ops); + let row = traces[0].main_table.get_row(NUM_OPS); + assert!( + row.iter().all(|c| *c == FE::zero()), + "the padding row must be entirely zero, so that μ = 0 is the only thing \ + standing between it and the constraint set" + ); +} + +// ========================================================================= +// Falsification (rule 1) +// ========================================================================= + +/// CONTROL. If this ever fails, every mutation below is reporting on a broken +/// harness rather than on the chip — check it first (rule 7's corollary). +#[test] +fn falsification_control_the_untampered_proof_verifies() { + assert_eq!(round_trip(|_| {}), Ok(true), "control must be green"); +} + +/// A flipped OUT byte: it is a feed-forward XOR *result*, so the XOR lookup +/// finds no BITWISE row, and the `LfmMem` word the mirror receives no longer +/// matches either. +#[test] +fn a_tampered_output_byte_rejects() { + assert_not_accepted("a flipped OUT byte", |t| { + let old = t.main_table.get_row(1)[cols::out_word(5, 2)]; + t.main_table + .set_fe(1, cols::out_word(5, 2), old + FE::one()); + }); +} + +/// A flipped message byte. `m` is never XORed, so the only things that see this +/// are its explicit `AreBytes` send, the add3 sum identity and the `LfmMem` +/// read. This is the test that would go green if the 32 message range checks +/// were ever dropped as "redundant" *and* the sum identity were loosened. +#[test] +fn a_tampered_message_byte_rejects() { + assert_not_accepted("a flipped message byte", |t| { + let old = t.main_table.get_row(0)[cols::in_word(8 + 3, 1)]; + t.main_table + .set_fe(0, cols::in_word(8 + 3, 1), old + FE::one()); + }); +} + +/// A padding row turned real. `MU` is preprocessed, so the prover recommits the +/// prefix and refuses before the constraint set is ever consulted — which IS +/// the point: an is-real flag a prover can choose is exactly what preprocessing +/// prevents, and the keccak adapter's `padding_row_multiplicity_rejects` +/// documents the weaker witness-side version. +#[test] +fn a_padding_row_turned_real_rejects() { + assert_not_accepted("an is-real padding row", |t| { + t.main_table.set_fe(NUM_OPS, cols::MU, FE::one()) + }); +} + +/// A bumped output-word read count, in the witness. Same shape: program data. +#[test] +fn a_tampered_read_multiplicity_rejects() { + assert_not_accepted("a bumped output-word read count", |t| { + let old = t.main_table.get_row(0)[cols::mult(2)]; + t.main_table.set_fe(0, cols::mult(2), old + FE::one()); + }); +} + +/// A carry bit flipped on an add3. The sum identity `a + b + m − s − 2^32·(c1+c2)` +/// is the only thing that sees it, and it is exactly the constraint #903's +/// "two summed committed carry bits" decision exists to keep at degree 3. +#[test] +fn a_tampered_add3_carry_bit_rejects() { + assert_not_accepted("a flipped add3 carry bit", |t| { + let col = cols::g_base(7) + cols::G_A1_C; + let old = t.main_table.get_row(2)[col]; + t.main_table.set_fe(2, col, old + FE::one()); + }); +} diff --git a/prover/src/lfm/blake3_socket.rs b/prover/src/lfm/blake3_socket.rs new file mode 100644 index 000000000..7ab2156e1 --- /dev/null +++ b/prover/src/lfm/blake3_socket.rs @@ -0,0 +1,1672 @@ +//! The BLAKE3 arm of `LFM_HASH` — the Option-A 2-to-1 compress socket. +//! +//! This is Route A of `thoughts/shared/lfm-real-hash/PLAN.md` §3: BLAKE3 hosted +//! *behind* the frozen `LFM_HASH` socket, exactly the way Poseidon is. The chip +//! count stays 14 and the 28-column shared value prefix keeps its offsets, so +//! the `LFM_HASH` tuple contract is untouched and everything BLAKE3 witnesses is +//! appended after the prefix. `PREP_WIDTH` is 13 — the transcript selector +//! (option B1) took it from 11 to 12 and the leaf selector (option C) to 13, +//! each moving every preprocessed root and every registered program's digest +//! once, in one re-bless. +//! +//! # What one row proves +//! +//! One row = one compression, in one of THREE domains, specified byte-level in +//! `thoughts/blake3/socket-kats/SOCKET.md` §2.1 and word-level in §2.2, at the +//! leaf RATE of `block-compression/commit-spec/COMMIT.md` §1.2: +//! +//! ```text +//! msg = LE32(lane0..lane11) ‖ tag (52 bytes) +//! digest = BLAKE3(msg)[0..16] (128 bits, 1 cell) +//! ``` +//! +//! | tag | row | the twelve lanes are | +//! |---|---|---| +//! | `"LFMC"` | Merkle parent / 2-to-1 compress | two digest cells, then four zeros | +//! | `"LFMT"` | a Fiat–Shamir transcript step | state ‖ operand ‖ four zeros | +//! | `"LFML"` | a **leaf** over four field elements | the chaining accumulator, then the felts' `lo`/`hi` halves | +//! +//! 52 bytes being one block, that is exactly one compression with `h = IV` (all +//! eight words), `m[0..12] = the lanes`, `m[12] = tag` as a little-endian `u32`, +//! `m[13..16] = 0`, `t = 0`, `block_len = 52`, +//! `flags = CHUNK_START|CHUNK_END|ROOT`, and the digest the LOW four output +//! words. **The three domains differ in `m[12]` and in nothing else**, so one +//! mixing core and one column layout serve all three. +//! +//! # The leaf RATE — why twelve lanes and not eight +//! +//! A leaf row absorbs **four felts and chains an accumulator in ONE +//! compression** (COMMIT.md §1.2): the accumulator cell rides in the message +//! rather than in `h`, so there is no separate fold. That is 4 felts per +//! compression against the 2 the accumulator-free row reached once its digest +//! had to be folded into a chain by an `"LFMC"` parent, and leaf absorption is +//! ~70% of a recursion tower node's bill. +//! +//! The lanes it costs are free of witness columns on the digest modes: lanes +//! 8–11 read `IN8..IN12`, the THIRD input cell, which +//! `chips::hash::emit_unread_input_pins` already pins to zero on every row that +//! does not read it. So a compress row's four new message words are forced to +//! zero by constraints that were already there — see the lane block in +//! [`eval`], and note that twelve is the last lane count for which this holds +//! (at thirteen, `IN0 + 12` is `S8` and the identity would start reading the +//! capacity state as an input felt). +//! +//! **At [`SOCKET_ROUNDS`] = 7 that is literally `blake3::hash(lanes ‖ tag)`,** +//! so the socket has a direct external anchor and needs no oracle in the chain — +//! and the transcript and leaf domains inherit that anchor unchanged, because +//! the tag is the only thing that moved. That is the whole reason the domain tag lives in the +//! *message* rather than in `flags`, `t` or `h`: a tag anywhere else would make +//! even the 7-round socket a nonstandard invocation of `f` that no library +//! computes, throwing the anchor away for nothing (SOCKET.md §2.3). +//! +//! The tag word is a linear form over the three PREPROCESSED mode columns rather than +//! a compile-time constant, which keeps it prover-unchosen and free — see +//! [`TAG_SELECTOR`]. +//! +//! # The LEAF mode, and the one thing not to conclude from it +//! +//! A leaf row reads TWO cells: a chaining accumulator, which is an ordinary +//! digest cell and fills lanes 0–3, and four arbitrary Goldilocks elements, each +//! split into a `lo`/`hi` `u32` pair so that eight halves fill lanes 4–11. +//! `p − 1 = 0xFFFFFFFF_00000000`, so for halves already known to be `u32`: +//! +//! ```text +//! v < p <==> NOT( hi = 2^32−1 AND lo >= 1 ) +//! ``` +//! +//! — "if `hi` is maximal then `lo` is zero", which is two witness columns and +//! four constraints per felt rather than a 64-bit decomposition. Without it one +//! field element would have TWO half-encodings and therefore two leaf digests, +//! which is a collision in the felt→digest map and exactly what a Merkle tree +//! must not have. +//! +//! ⚠ **The canonicity block ASSUMES the `u32` bound; it does not ESTABLISH it.** +//! `lo` and `hi` are ordinary input lanes, so the bound comes from the same O1 +//! machinery as every other lane: byte columns plus the `AreBytes` sends. That +//! is the whole reason this mode is cheap, and it is stated here because the +//! shape invites two opposite mistakes — adding a redundant range check on the +//! halves, or (far worse) **removing the lane identity or the `AreBytes` sends +//! on the theory that canonicity subsumes them. It does not.** With unbounded +//! halves, `hi = 2^32−1` stops being a reachable-and-detectable case and the +//! predicate above stops meaning `v < p` at all. +//! +//! # Why the socket is so much cheaper than the standalone chip +//! +//! [`super::blake3_chip`] is the syscall-shaped chip: 28 input `u32` words and +//! all 16 output words are committed columns. Here `h`, `t`, `block_len`, +//! `flags` and every message word above the lanes are **compile-time +//! constants**, and the truncation +//! window means only 4 of the 16 output words are ever built. What is left as +//! witness is 8 input lanes, the mixing core, and 4 output words. +//! +//! # The two soundness obligations this module discharges +//! +//! - **O1 — input lanes carry a committed byte decomposition.** A digest cell's +//! lane is a Goldilocks felt over `[0, p)` with `p ≈ 2^64`, and +//! `edsl::merkle_walk` feeds `compress` *arena-hinted* — that is, +//! prover-chosen — sibling cells. The contract has two halves and **they buy +//! different things**; conflating them is easy and is why this is spelled out. +//! +//! *The mu-gated linear identity* `IN_lane = Σ MB[k]·2^{8k}` ties the felt to +//! the bytes. Note what follows from it: the mixing core reads **the same +//! linear form** as the message word (`message_word_ref`), so `IN_lane` and +//! `m[lane]` are the same field element by construction. A lane therefore +//! cannot be hashed as anything but itself, and the textbook alias — `v` and +//! `v + 2^32` hashing alike — is **unconstructible here**, not merely +//! prevented. (It is real for a chip that derives the message bytes by +//! reduction mod 2^32 instead of by a checked decomposition, which is why the +//! identity is the right shape; it is not what the `AreBytes` sends buy.) +//! +//! *The `AreBytes` sends* are the message words' **only** range check — +//! `m` reaches `add3` and nothing else, never an XOR, so unlike almost every +//! other word in this design it gets no free byte bound from a lookup that +//! consumes it. And `add3`'s exactness needs `m < 2^32`: in round 0 the `a` +//! and `b` operands are compile-time constants and the output `s` is +//! byte-bounded by the XOR that consumes it, so an unbounded `m` lets a +//! prover solve `m ≡ s + 2^32·k − a − b (mod p)` for any chosen `s`, put the +//! whole value in `MB[0]` with the other three bytes zero — satisfying the +//! identity, since nothing bounds them — and hint the sibling cell to match. +//! The first `add3`'s output, and hence the entire compression, would be +//! prover-chosen. +//! +//! Stated as the one mechanism, since this is the spot the argument keeps +//! drifting: what the sends do is **transfer a bound onto the lane**. Without +//! them the identity is satisfiable for *every* felt `IN_lane` — put the whole +//! value in `MB[0]` — so it bounds nothing. With them the four bytes sum to +//! less than `2^32`, so it is satisfiable exactly when `IN_lane < 2^32`, and +//! then the decomposition is unique. +//! +//! So: neither half alone suffices, and `blake3_socket_tests:: +//! the_lane_range_check_is_load_bearing_on_its_own` pins the separation by +//! exhibiting a witness the eval set cannot see at all. +//! - **O3 — `compress_iv()` does not participate.** The IV enters through `h`, +//! all eight words, not through the state's capacity lanes, so this arm +//! overrides `compress` (and [`LfmHasher::compress_out`]) rather than +//! inheriting the trait's permute-and-truncate default. +//! +//! # ✓ O5 — RETIRED, and enforced by the tag rather than by review +//! +//! The obligation was: leaves and parents must be domain-separated, or a +//! variable-depth tree admits the classic Merkle second-preimage confusion — an +//! internal node replayed as a leaf. It is now discharged **mechanically**. A +//! leaf digest is `BLAKE3(…‖"LFML")` and a parent is `BLAKE3(…‖"LFMC")`, so an +//! internal node cannot be replayed as a leaf whatever the tree's shape, and the +//! tag is selected by a preprocessed column the prover does not choose. +//! +//! What this replaced is worth recording, because it was weaker than it looked. +//! Programs formed leaf digests by compressing raw data rows under the SAME +//! `"LFMC"` tag as parents, so leaves and parents were not separated at all; +//! that was sound only because every eDSL circuit is fixed-shape at build time, +//! so a node at one level could not be replayed at another. Fixed depth remains +//! true of every current program and remains worth having, but **it is no longer +//! load-bearing for second-preimage resistance.** +//! +//! The reviewer's job shrinks accordingly: from "is this a leaf path, and does +//! the tree have fixed depth?" to *"is this row's mode right?"* — which the +//! registrar's one-hot check and controls M9/M10 answer. +//! +//! (BLAKE3's own `PARENT` flag was rejected for the split: it cannot be reused +//! without leaving the standard-hash framing that makes the crate a direct KAT.) +//! +//! Equally on the record: the digest is 128 bits, so this socket offers +//! **64-bit collision resistance** by the birthday bound. That follows from +//! `HASH_DIGEST_FELTS = 4` and the machine's declared 128-bit target — it is +//! not introduced by BLAKE3 or by the truncation window. +//! +//! # ✗ There is no `permute` socket, and there never will be +//! +//! `LFM_HASH` has four modes and this arm implements **three**. The `permute` +//! socket — 12 felts in, 12 out — is unspecified: it has no mapping decision, +//! no KATs, and its security argument is not the same argument as `compress`'s +//! (SOCKET.md §7). Rather than invent one, the AIR forces `MODE_P = 0`, so a +//! program containing a `permute` is *unprovable* under BLAKE3, and +//! [`Blake3Permutation`] rejects one at execution with a message saying why. +//! +//! Option B1 (ratified 2026-08-11) made that permanent by removing the only +//! reason to want one: the Fiat–Shamir sponge is a **compress chain**, not a +//! permutation duplex, so `edsl::SpongeVar` runs on this socket like everything +//! else and `MODE_P` stays pinned forever. Option C then gave leaves their own +//! mode on the same socket rather than a second one. The tag `"LFMP"` that was +//! reserved for the permute socket is retired unused. + +use stark::constraints::builder::ConstraintBuilder; +use stark::lookup::{BusInteraction, BusValue, Multiplicity}; + +use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; +use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField, alu_op}; + +use super::blake3::{BLAKE3_IV, BLAKE3_ROUNDS, blake3_compress_rounds}; +use super::blake3_chip::{ + Add2Wire, Add3Wire, Blake3Flow, ByteRef, FlowConfig, ROT_SHIFT_R, RotWire, ValueFlow, WireFlow, + WordRef, XorWire, half_expr, run_flow, word_cols, word_expr, +}; +use super::chips::hash::NUM_UNREAD_INPUT_PINS; +use super::hash::{HASH_DIGEST_FELTS, HASH_STATE_FELTS, LfmHasher}; +use super::instr::HashMode; +use super::word::LfmWord; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +// ========================================================================= +// The framing (SOCKET.md §2.2) — every one of these is a way to be wrong +// ========================================================================= + +/// Rounds the `LFM_HASH` BLAKE3 arm is compiled for. +/// +/// An alias for [`BLAKE3_ROUNDS`] — 7 (standard BLAKE3) by default, 6 under the +/// `blake3-6round` feature. The socket and the standalone `LFM_BLAKE3` probe +/// share ONE knob on purpose: two would let a sweep leave the machine's hash and +/// the chip it is priced against describing different functions, and the whole +/// value of the probe is that the two are comparable. +pub const SOCKET_ROUNDS: usize = BLAKE3_ROUNDS; + +/// Tripwire for the single-knob invariant. +/// +/// Trivially true while [`SOCKET_ROUNDS`] is an alias — which is the point. The +/// invariant is enforced by that one `=` and by nothing else: `NUM_G == 8 * +/// SOCKET_ROUNDS` and `cols::OUT - cols::G == 60 * NUM_G` are each internally +/// consistent and would pass happily with the socket and the standalone probe +/// compiled at different round counts. That is not hypothetical — it is the +/// shape this tree had before the A6R flip, and re-introducing a second `cfg` +/// pair here is a silent pricing lie: the probe would measure one hash and the +/// machine would use another. This assertion is what fails instead. +const _: () = assert!(SOCKET_ROUNDS == BLAKE3_ROUNDS); + +/// G-instances per compression: 8 per round. +pub const NUM_G: usize = SOCKET_ROUNDS * 8; + +/// The domain tag `"LFMC"`, read as one little-endian `u32` — the message word +/// straight after the lanes, `m[NUM_LANES]`. +/// +/// A tag is never reused for a second purpose, for the same reason +/// `HasherKind::as_tag` never reuses a discriminant. `"LFMT"` is the transcript +/// domain, `"LFML"` is the LEAF domain (live since option C), and `"LFMP"` is +/// RETIRED UNUSED — it was reserved for a permute socket that option B1 decided +/// never to build. Retired rather than deleted: freeing the value would let a +/// later allocation reuse it and create a domain nobody analysed. +pub const TAG_LFMC: u32 = u32::from_le_bytes(*b"LFMC"); + +/// The domain tag `"LFML"` — a Merkle LEAF over four field elements. +/// +/// The third live domain, and the one that retires obligation O5: a leaf digest +/// and a parent digest are different functions of the same bits, so an internal +/// node cannot be replayed as a leaf regardless of tree depth. That previously +/// rested on every eDSL circuit being fixed-shape — true, but enforced by +/// nothing. +/// +/// The message is the felts' checked `u32` halves, so its byte layout is +/// identical to a digest-mode compress and the crate anchor survives: at +/// [`SOCKET_ROUNDS`] = 7 a leaf is +/// `blake3::hash(LE32(lo0)‖LE32(hi0)‖…‖LE32(hi3)‖"LFML")` truncated. +pub const TAG_LFML: u32 = u32::from_le_bytes(*b"LFML"); + +/// The domain tag `"LFMT"` — one step of the Fiat–Shamir transcript chain. +/// +/// The transcript step is this socket in every respect except this word: same +/// `h = IV`, same `m[0..4] = state`, `m[4..8] = operand`, same `t`, +/// `block_len` and `flags`, same four-word truncation. So at +/// [`SOCKET_ROUNDS`] = 7 a transcript step is literally +/// `blake3::hash(state ‖ operand ‖ "LFMT")` truncated to 128 bits, and it +/// inherits the compress socket's external anchor unchanged — which is the +/// whole point of building the transcript out of this socket rather than out of +/// a second one. +pub const TAG_LFMT: u32 = u32::from_le_bytes(*b"LFMT"); + +/// `CHUNK_START | CHUNK_END | ROOT` — the flags a one-block, one-chunk, +/// root-position BLAKE3 hash uses. Matching the tree hasher exactly is what +/// keeps §2.1's byte-level form a plain library call. +pub const FLAGS_LFMC: u32 = 0x0B; + +/// The message length in bytes: one 4-byte word per lane, plus the tag. +/// +/// 52 at [`cols::NUM_LANES`] = 12. **Derived, never written as a literal**: it +/// is `v[14]`, hence the `vd` operand of round-0 G #2 and from there an XOR +/// operand, so it cannot be mode-dependent (`WordRef::byte` panics on a +/// `ModeSelected`) — all three domains move together and a hand-written 52 that +/// disagreed with the lane count would desynchronise the wire interpretation +/// from the host reference (COMMIT.md §1.4.4 **H9**). +/// +/// 52 < 64 keeps a row ONE BLAKE3 block, which is what keeps the crate-KAT +/// anchor: at [`SOCKET_ROUNDS`] = 7 a row is still a plain `blake3::hash` call. +pub const BLOCK_LEN_LFMC: u32 = 4 * (cols::NUM_LANES as u32 + 1); + +/// The counter. Zero: one block, one chunk, chunk index 0. +pub const COUNTER_LFMC: u64 = 0; + +/// The truncation window: the digest is the LOW four of the 16 output words. +pub const OUT_WINDOW: usize = HASH_DIGEST_FELTS; + +/// The dataflow framing [`run_flow`] itself decides — the round count and the +/// truncation window. One value, used by the wire interpretation, the value +/// interpretation and the trace filler alike, so they cannot desynchronise. +pub(crate) const FLOW: FlowConfig = FlowConfig { + rounds: SOCKET_ROUNDS, + out_window: OUT_WINDOW, + full_output: false, +}; + +/// The 16 message words of the socket's 52-byte block, under domain `tag`. +/// +/// The lanes first, the tag straight after them, zeros above — so the byte +/// string is `LE32(lanes) ‖ tag` whatever the mode, and the tag stays LAST as +/// COMMIT.md §1.2 specifies it. +pub fn socket_message(lanes: &[u32; cols::NUM_LANES], tag: u32) -> [u32; 16] { + let mut m = [0u32; 16]; + m[..cols::NUM_LANES].copy_from_slice(lanes); + m[cols::NUM_LANES] = tag; + m +} + +/// **The reference the chip is checked against**: the socket's 2-to-1 step, +/// word-level, at an explicit round count and in an explicit domain. +/// +/// `rounds` is an argument rather than [`SOCKET_ROUNDS`] so the KATs can pin +/// both variants in one test run; the chip itself is compiled for exactly one. +/// `tag` is an argument for the same reason it is a column on the chip: two +/// domains, one function. +pub fn socket_digest_rounds_tagged( + a: &[u32; 4], + b: &[u32; 4], + rounds: usize, + tag: u32, +) -> [u32; 4] { + socket_digest_lanes(&digest_row_lanes(a, b), rounds, tag) +} + +/// A digest row's twelve lanes: the two cells it reads, then the four zeros the +/// third input cell's pins force. +/// +/// Written once, so the host reference cannot disagree with the AIR about what a +/// compress row's new lanes hold. +pub fn digest_row_lanes(a: &[u32; 4], b: &[u32; 4]) -> [u32; cols::NUM_LANES] { + let mut lanes = [0u32; cols::NUM_LANES]; + lanes[0..4].copy_from_slice(a); + lanes[4..8].copy_from_slice(b); + lanes +} + +/// The socket over twelve explicit lanes — the one place the framing is applied. +pub fn socket_digest_lanes(lanes: &[u32; cols::NUM_LANES], rounds: usize, tag: u32) -> [u32; 4] { + let out = blake3_compress_rounds( + &BLAKE3_IV, + &socket_message(lanes, tag), + COUNTER_LFMC, + BLOCK_LEN_LFMC, + FLAGS_LFMC, + rounds, + ); + [out[0], out[1], out[2], out[3]] +} + +/// [`socket_digest_rounds_tagged`] in the MERKLE domain. +pub fn socket_digest_rounds(a: &[u32; 4], b: &[u32; 4], rounds: usize) -> [u32; 4] { + socket_digest_rounds_tagged(a, b, rounds, TAG_LFMC) +} + +/// [`socket_digest_rounds`] at the compiled-in round count — what a `Compress` +/// row proves, and what [`Blake3Permutation::compress`] computes. +pub fn socket_digest(a: &[u32; 4], b: &[u32; 4]) -> [u32; 4] { + socket_digest_rounds(a, b, SOCKET_ROUNDS) +} + +/// One transcript step at an explicit round count — the `"LFMT"` domain. +pub fn transcript_digest_rounds(state: &[u32; 4], operand: &[u32; 4], rounds: usize) -> [u32; 4] { + socket_digest_rounds_tagged(state, operand, rounds, TAG_LFMT) +} + +/// [`transcript_digest_rounds`] at the compiled-in round count — what a +/// `Transcript` row proves, and what [`Blake3Permutation::transcript`] +/// computes. +pub fn transcript_digest(state: &[u32; 4], operand: &[u32; 4]) -> [u32; 4] { + transcript_digest_rounds(state, operand, SOCKET_ROUNDS) +} + +/// The domain tag a row in `mode` hashes under, or `None` for a mode this arm +/// has no socket for. +/// +/// One function, so the executor, the trace filler, the multiplicity histogram +/// and the KATs cannot disagree about which tag a row carries. The AIR gets the +/// same mapping through [`TAG_SELECTOR`], written the one other way it has to +/// be written — as a linear form over the mode columns. +pub const fn tag_for_mode(mode: HashMode) -> Option { + match mode { + HashMode::Compress => Some(TAG_LFMC), + HashMode::Transcript => Some(TAG_LFMT), + HashMode::Leaf => Some(TAG_LFML), + HashMode::Permute => None, + } +} + +// ========================================================================= +// The felt boundary (the LEAF mode) — host side +// ========================================================================= + +/// Field elements one leaf row hashes — the leaf **RATE** (COMMIT.md §1.4.1). +/// +/// Four felts = eight halves, which with the four accumulator lanes fill the +/// socket's twelve message lanes. It is one whole machine cell, which is the +/// property the rate was chosen for: the leaf program reads its felt stream in +/// the natural 4-per-cell layout with no re-packing pass. +pub const FELTS_PER_LEAF: usize = 4; + +/// Goldilocks `p = 2^64 − 2^32 + 1`, as the halves see it: `p − 1` is +/// `hi = 2^32−1`, `lo = 0`. +const MAX_HALF: u32 = u32::MAX; + +/// The chip's canonicity predicate, stated exactly as its constraints do. +/// +/// For halves already known to be `u32`, `v = lo + 2^32·hi < p` **iff** NOT +/// (`hi` maximal AND `lo ≥ 1`) — because `p − 1 = 0xFFFFFFFF_00000000`. That one +/// line is the whole reason this mode is cheap: it costs two witness columns per +/// felt instead of a 64-bit decomposition. +/// +/// ⚠ It **assumes** the `u32` bound rather than establishing it — see the +/// module docs. +pub const fn is_canonical(lo: u32, hi: u32) -> bool { + !(hi == MAX_HALF && lo >= 1) +} + +/// `v → (lo, hi)`, or `None` when `v` is not a canonical Goldilocks element. +/// +/// **REJECTS, never reduces.** A non-canonical value has no satisfying witness, +/// so its row is unprovable; a host that wrapped instead would claim a digest no +/// proof can produce. Same shape as obligation O1's own reject-don't-reduce +/// rule, and the reason is the same. +pub fn felt_halves(v: u64) -> Option<(u32, u32)> { + let (lo, hi) = (v as u32, (v >> 32) as u32); + is_canonical(lo, hi).then_some((lo, hi)) +} + +/// Four felts → the eight message lanes ABOVE the accumulator, +/// `[lo0, hi0, …, lo3, hi3]` — the row's lanes 4–11. +/// +/// A felt's halves are ADJACENT, which is load-bearing: it lets the canonicity +/// gate read one pair of neighbouring lanes instead of reaching across the row. +pub fn leaf_lanes(felts: &LfmWord) -> Option<[u32; 2 * FELTS_PER_LEAF]> { + use math::field::traits::IsPrimeField; + let mut lanes = [0u32; 2 * FELTS_PER_LEAF]; + for (i, f) in felts.iter().enumerate() { + let (lo, hi) = felt_halves(GoldilocksField::canonical(f.value()))?; + lanes[2 * i] = lo; + lanes[2 * i + 1] = hi; + } + Some(lanes) +} + +/// A leaf row's twelve lanes: the accumulator cell, then the felts' halves. +/// +/// `None` if the accumulator is not four `u32` lanes (it is a previous digest, +/// so it is by construction) or if a felt is not canonical. +/// +/// The split is what makes the row a HYBRID and it is the reason this exists as +/// one function: the accumulator is read as digest lanes and the felts as +/// halves, on the same row, and the trace filler, the BITWISE histogram and the +/// host reference must all split it identically (COMMIT.md §1.4.4 **H5**). +pub fn leaf_row_lanes(acc: &LfmWord, felts: &LfmWord) -> Option<[u32; cols::NUM_LANES]> { + let mut lanes = [0u32; cols::NUM_LANES]; + lanes[..cols::NUM_ACC_LANES].copy_from_slice(&lanes_of(acc)?); + lanes[cols::NUM_ACC_LANES..].copy_from_slice(&leaf_lanes(felts)?); + Some(lanes) +} + +/// One leaf row at an explicit round count — the `"LFML"` domain over the +/// accumulator and the felts' halves. +/// +/// **The accumulator rides in the message, so there is no separate fold**: this +/// one compression both absorbs `felts` and chains `acc` (COMMIT.md §1.2). +pub fn leaf_digest_rounds(acc: &LfmWord, felts: &LfmWord, rounds: usize) -> Option<[u32; 4]> { + Some(socket_digest_lanes( + &leaf_row_lanes(acc, felts)?, + rounds, + TAG_LFML, + )) +} + +/// [`leaf_digest_rounds`] at the compiled-in round count — what a `Leaf` row +/// proves, and what [`Blake3Permutation::leaf`] computes. +pub fn leaf_digest(acc: &LfmWord, felts: &LfmWord) -> Option<[u32; 4]> { + leaf_digest_rounds(acc, felts, SOCKET_ROUNDS) +} + +// ========================================================================= +// The lane boundary (obligation O1), host side +// ========================================================================= + +/// A digest cell's four lanes as `u32`s, or `None` if any lane is out of range. +/// +/// **`None` must never be turned into a reduction.** The host and the chip have +/// to agree about what was hashed; a felt outside `[0, 2^32)` has no byte +/// decomposition the chip can commit, so reducing it here would make a +/// host-side assertion pass while the chip proved something else. Rejecting is +/// also what keeps the two consistent in the other direction: the chip refuses +/// such a lane (no byte string satisfies both the identity and `AreBytes`), so +/// a host that reduced would claim a digest no proof can produce. +pub fn lanes_of(word: &LfmWord) -> Option<[u32; 4]> { + use math::field::traits::IsPrimeField; + let mut out = [0u32; 4]; + for (o, felt) in out.iter_mut().zip(word.iter()) { + *o = u32::try_from(GoldilocksField::canonical(felt.value())).ok()?; + } + Some(out) +} + +/// A digest cell built from four `u32` lanes — the inverse of [`lanes_of`], and +/// the `keccak_host` convention (one felt = one `u32` = four little-endian +/// bytes), NOT `word::pack_digest`'s eight-bytes-per-lane serialisation. +pub fn word_of(lanes: &[u32; 4]) -> LfmWord { + core::array::from_fn(|i| FE::from(u64::from(lanes[i]))) +} + +// ========================================================================= +// The host-side hasher +// ========================================================================= + +/// BLAKE3 behind `LFM_HASH`, `compress` only. +/// +/// The trait's `permute` is **partial** for lane-restricted hashers, and this is +/// one: it has no permute socket at all. [`LfmHasher::admits`] is what makes the +/// partiality a rejection rather than a wrong answer. +pub struct Blake3Permutation; + +impl LfmHasher for Blake3Permutation { + /// ✗ Unreachable by construction: [`LfmHasher::admits`] rejects a `Permute` + /// row before the executor gets here, and the AIR forces `MODE_P = 0`. + /// + /// It panics rather than returning something, because every value it could + /// return would be a hash the chip does not prove. + fn permute(&self, _state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + panic!( + "BLAKE3 has no LFM_HASH permute socket: 12-felt permute is unspecified \ + (thoughts/blake3/socket-kats/SOCKET.md §7). Use compress, or select \ + another hasher." + ) + } + + /// `BLAKE3_IV[0..4]`, so the capacity columns carry something meaningful if + /// read — but it is **not** part of the compress framing (obligation O3). + /// The IV enters through `h`, all eight words, and this arm overrides + /// `compress`/`compress_out` rather than inheriting the trait's + /// permute-a‖b‖IV default. + fn compress_iv(&self) -> LfmWord { + core::array::from_fn(|i| FE::from(u64::from(BLAKE3_IV[i]))) + } + + fn compress(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + self.step(a, b, TAG_LFMC) + } + + /// The digest in lanes 0–3 and zeros above, which is exactly what the chip's + /// `OUT` columns carry: `MULT1`/`MULT2` are zero on a Compress row, so the + /// upper eight are sent nowhere, and the AIR pins them to zero. + fn compress_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + Self::widen(self.compress(a, b)) + } + + /// The same socket under the TRANSCRIPT tag — this is where BLAKE3 stops + /// inheriting the trait's single-domain default, and it is the only thing + /// that makes a transcript step un-replayable as a Merkle parent. + fn transcript(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + self.step(a, b, TAG_LFMT) + } + + fn transcript_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + Self::widen(self.transcript(a, b)) + } + + /// The LEAF domain, and the one override that is an ENCODING rather than a + /// tag: the four felts become eight checked `u32` halves before they reach + /// the socket. This is what lets arbitrary Goldilocks data be hashed at all + /// — obligation O1 restricts the *lanes*, and a leaf row satisfies it by + /// construction rather than by luck. + fn leaf(&self, acc: &LfmWord, felts: &LfmWord) -> LfmWord { + word_of(&leaf_digest(acc, felts).expect( + "leaf accumulator lane is not a u32, or a leaf felt is not canonical — admits() \ + should have rejected it (reject, never reduce)", + )) + } + + fn leaf_out(&self, acc: &LfmWord, felts: &LfmWord) -> [FE; HASH_STATE_FELTS] { + Self::widen(self.leaf(acc, felts)) + } + + fn admits(&self, mode: HashMode, state: &[FE; HASH_STATE_FELTS]) -> Result<(), &'static str> { + match mode { + HashMode::Permute => Err( + "BLAKE3 has no LFM_HASH permute socket (SOCKET.md §7); its AIR forces MODE_P = 0", + ), + // ★ A leaf row is a HYBRID and both halves have to be checked, in + // the cells the AIR actually reads them from: the ACCUMULATOR in + // cell 0 is an ordinary digest and carries the full O1 `u32` + // restriction, while the FELTS in cell 1 have none — that is the + // entire point of the mode, since they are split into checked halves + // inside the socket. + // + // ⚠ Checking the wrong cell is a prover PANIC rather than a clean + // rejection: a non-canonical felt would pass here and blow up later + // in the witness filler (COMMIT.md §1.4.4 **H7**). The house rule is + // reject, never reduce — and never panic where a rejection is + // available. + HashMode::Leaf => { + let acc: LfmWord = core::array::from_fn(|i| state[i]); + let felts: LfmWord = core::array::from_fn(|i| state[4 + i]); + if lanes_of(&acc).is_none() { + return Err( + "BLAKE3 leaf accumulator lane is not a u32 (SOCKET.md obligation O1)", + ); + } + if leaf_lanes(&felts).is_none() { + return Err( + "BLAKE3 leaf felt is not a canonical Goldilocks element (LEAF.md §1.1)", + ); + } + Ok(()) + } + HashMode::Compress | HashMode::Transcript => { + let (a, b): (LfmWord, LfmWord) = ( + core::array::from_fn(|i| state[i]), + core::array::from_fn(|i| state[4 + i]), + ); + if lanes_of(&a).is_none() || lanes_of(&b).is_none() { + // Obligation O1, host side, and it binds both two-to-one + // modes: a transcript step is the same socket over the same + // lane columns, so it inherits the same restriction. + // Rejecting rather than reducing is the point: reduction is + // the collision. Data that cannot satisfy this belongs in a + // LEAF row, which is what that mode exists for. + return Err( + "BLAKE3 compress input lane is not a u32 (SOCKET.md obligation O1)", + ); + } + Ok(()) + } + } + } +} + +impl Blake3Permutation { + /// The socket, once, in the named domain — the one place the host computes + /// it, so `compress` and `transcript` cannot drift into different framings. + fn step(&self, a: &LfmWord, b: &LfmWord, tag: u32) -> LfmWord { + let (a, b) = ( + lanes_of(a).expect("socket lane is not a u32 — admits() should have rejected it"), + lanes_of(b).expect("socket lane is not a u32 — admits() should have rejected it"), + ); + word_of(&socket_digest_rounds_tagged(&a, &b, SOCKET_ROUNDS, tag)) + } + + fn widen(digest: LfmWord) -> [FE; HASH_STATE_FELTS] { + let mut out = [FE::zero(); HASH_STATE_FELTS]; + out[0..HASH_DIGEST_FELTS].clone_from_slice(&digest); + out + } +} + +// ========================================================================= +// Column layout +// ========================================================================= + +/// The BLAKE3 arm's columns. +/// +/// The frozen prefix (`IN0..12`, `S8..12`, `OUT0..12`) keeps the offsets +/// `chips::hash::cols` gives it, so the `LFM_HASH` tuple contract stays +/// literally frozen and every existing `edsl::merkle_walk` caller works +/// unchanged. Everything BLAKE3 additionally witnesses is appended from +/// [`LANES`] on. +/// +/// Width, in blocks: 28 shared + 32 lane bytes + `NUM_G · 60` mixing + 16 +/// output bytes. +pub mod cols { + pub use crate::lfm::chips::hash::cols::{ + IN_ADDR0, IN_ADDR1, IN_ADDR2, IN0, MODE_C, MODE_L, MODE_P, MODE_T, MULT0, MULT1, MULT2, + OUT_ADDR0, OUT_ADDR1, OUT_ADDR2, OUT0, PREP_WIDTH, S8, SHARED_VALUE_COLUMNS, + }; + + use super::{FELTS_PER_LEAF, HASH_DIGEST_FELTS, NUM_G, OUT_WINDOW}; + + /// The is-real flag every constraint is gated by and every send's + /// multiplicity: `MODE_C + MODE_T + MODE_L`, the three modes this arm has a + /// socket for. `MODE_P` is pinned to zero, so the sum is a bit on every row + /// and zero on padding. + /// + /// All three are *preprocessed* columns, so a prover chooses neither the + /// gate nor — through the same columns — the domain tag it selects. + pub const MU_COLUMNS: [usize; 3] = [MODE_C, MODE_T, MODE_L]; + + /// The modes whose message lanes above the accumulator ARE the `IN` lanes — + /// the digest modes. A leaf row's lanes 4–11 are its felts' halves instead, + /// so the lane identity is gated on this above [`NUM_ACC_LANES`] rather than + /// on the full mu. Lanes 0–3 are a digest cell in EVERY mode and take the + /// full mu; see [`super::eval`] and COMMIT.md §1.4.4 **H6**. + pub const DIGEST_MODE_COLUMNS: [usize; 2] = [MODE_C, MODE_T]; + + /// First appended witness column: the byte decomposition of the input lanes, + /// 4 bytes each, little-endian (`lane_byte`). + pub const LANES: usize = PREP_WIDTH + SHARED_VALUE_COLUMNS; + + /// Lanes carrying a cell that is a DIGEST under every mode: `a` on a digest + /// row, the chaining accumulator on a leaf row. One identity serves both + /// readings, which is why they can share a gate. + pub const NUM_ACC_LANES: usize = HASH_DIGEST_FELTS; + + /// Input lanes that carry message words. + /// + /// The accumulator cell plus one felt cell's halves — the leaf RATE decides + /// this number, and the digest modes inherit it: `a ‖ b` fills the first + /// eight and the last four are the third input cell, which the unread-`IN` + /// pins force to zero (COMMIT.md §1.4.1). + pub const NUM_LANES: usize = NUM_ACC_LANES + 2 * FELTS_PER_LEAF; + + /// The mixing core: one 60-cell block per G-instance, laid out exactly as + /// `blake3_chip::cols` lays one out (56 byte cells + 4 carry bits). + pub const G: usize = LANES + 4 * NUM_LANES; + pub const G_SIZE: usize = 60; + + /// Feed-forward output bytes — only the truncation window's four words. + pub const OUTW: usize = G + NUM_G * G_SIZE; + + /// The LEAF mode's canonicity witnesses: `Z_i` and `GINV_i` per felt. + /// + /// `LFM_BITDEC`'s own `Z`/`GINV` idiom, applied to two halves instead of 64 + /// bits — the machine's established canonicity shape, not a new invention. + /// Two columns and four constraints per felt, **zero extra sends**. + /// + /// They exist on EVERY row, leaf or not, because a chip has one width. That + /// is the mode's whole marginal cost: +8 value cells per compress row. + pub const CANON: usize = OUTW + 4 * OUT_WINDOW; + + pub const NUM_COLUMNS: usize = CANON + 2 * FELTS_PER_LEAF; + + // Offsets inside one G block, shared verbatim with `blake3_chip::cols` so + // the two chips' blocks are the same shape and the wire interpretation + // below is the same code with different bases. + pub use crate::lfm::blake3_chip::cols::{ + G_A1, G_A1_C, G_A2, G_A2_C, G_C1, G_C2, G_R1, G_R2, G_X1, G_X2, G_X3, G_X4, + }; + + /// Byte `b` of input lane `lane` (0..[`NUM_LANES`]). + #[inline] + pub const fn lane_byte(lane: usize, b: usize) -> usize { + LANES + 4 * lane + b + } + + /// Base column of G-block `g`. + #[inline] + pub const fn g_base(g: usize) -> usize { + G + g * G_SIZE + } + + /// Byte `b` of digest word `i` (0..4). + #[inline] + pub const fn out_byte(i: usize, b: usize) -> usize { + OUTW + 4 * i + b + } + + /// Felt `i`'s canonicity flag: 1 exactly when its high half is maximal. + #[inline] + pub const fn canon_z(i: usize) -> usize { + CANON + 2 * i + } + + /// Felt `i`'s inverse witness for `(2^32 − 1) − hi`, zero when that is zero. + #[inline] + pub const fn canon_ginv(i: usize) -> usize { + CANON + 2 * i + 1 + } + + /// The `IN` column of leaf felt `i` — the SECOND input cell. + /// + /// The felts sit above the accumulator, which is what the halves binding and + /// [`super::lanes_from_cells`] must both read (COMMIT.md §1.4.4 **H4**). + #[inline] + pub const fn leaf_felt(i: usize) -> usize { + IN0 + NUM_ACC_LANES + i + } + + /// Message lane carrying felt `i`'s LOW half. Halves are adjacent, so the + /// canonicity gate reads neighbours rather than reaching across the row; + /// they start above the accumulator lanes. + #[inline] + pub const fn leaf_lo_lane(i: usize) -> usize { + NUM_ACC_LANES + 2 * i + } + + /// Message lane carrying felt `i`'s HIGH half. + #[inline] + pub const fn leaf_hi_lane(i: usize) -> usize { + NUM_ACC_LANES + 2 * i + 1 + } +} + +/// Value columns the census counts: everything past the preprocessed prefix. +pub const MAIN_COLUMNS: usize = cols::NUM_COLUMNS - cols::PREP_WIDTH; + +// ========================================================================= +// Wire interpretation — the socket's framing over the shared dataflow +// ========================================================================= + +/// `m[NUM_LANES] = MODE_C·"LFMC" + MODE_T·"LFMT" + MODE_L·"LFML"` — the row's +/// domain tag. +/// +/// Two corrections this comment used to get wrong, both of them things a reader +/// would act on. The tag sits at `m[NUM_LANES]`, i.e. **`m[12]` at the RATE-4 +/// lane count**, not at a fixed `m[8]` — see [`message_word_ref`], which is the +/// point of use and says so. And the selector is **three** terms, not two: +/// `MODE_L` joined it when the leaf mode got its own domain, which is what +/// makes an internal node un-replayable as a leaf whatever the tree's depth +/// (obligation O5). +/// +/// **Why this is not prover-chosen.** The mode columns are preprocessed: +/// a row's mode is fixed by its position in the preprocessed trace, +/// that trace is fixed by its commitment, and the commitment is folded into +/// `lfm_program_id`. The prover chooses neither, which is the same argument +/// that already makes the mu gate trustworthy. Two constraints make it bite — +/// the mode-sum booleanity (idx 4) forces at most one tag to be selected, and +/// the mode columns being preprocessed is what stops the selector itself being +/// chosen. +/// Controls M5 and M6 in `blake3_socket_tests` are what make each of those +/// dependencies a checked claim rather than an assertion. +const TAG_SELECTOR: &[(usize, u32)] = &[ + (cols::MODE_C, TAG_LFMC), + (cols::MODE_T, TAG_LFMT), + (cols::MODE_L, TAG_LFML), + // ✗ `MODE_P` is deliberately absent, not forgotten: there is no permute + // socket and idx 5 pins the column to zero, so a term for it would be + // identically zero and would suggest a domain that does not exist. +]; + +/// The message word at schedule index `i`, as wiring. +/// +/// `i < NUM_LANES` are the input lanes' byte columns; `m[NUM_LANES]` is the +/// domain tag and everything above it is zero. None of them is a witness column, +/// which is what makes the domain separation free (no cells, no range checks, +/// SOCKET.md §2.3) — the tag went from a constant to a linear form over +/// preprocessed columns and kept that property, because a preprocessed column is +/// not a witness. +/// +/// **The tag sits immediately after the lanes, not at a fixed `m[8]`.** That is +/// what makes the message the byte string `LE32(lanes) ‖ tag` at any lane count, +/// which is the form COMMIT.md §1.2 specifies and the form the KATs pin. A +/// `ModeSelected` word is legal at any index for the same reason a `Const` one +/// is: message words reach `add3` and nothing else. +fn message_word_ref(i: usize) -> WordRef { + match i { + i if i < cols::NUM_LANES => WordRef::Cols(word_cols(cols::lane_byte(i, 0))), + i if i == cols::NUM_LANES => WordRef::ModeSelected(TAG_SELECTOR), + _ => WordRef::Const(0), + } +} + +/// The socket's wire interpretation: same [`run_flow`], different framing and +/// different column bases. +struct SocketWire(WireFlow); + +impl Blake3Flow for SocketWire { + type Word = WordRef; + + /// `h = IV`, all eight words — so the entire initial state is constant and + /// the socket costs zero input-state columns. + fn input_h(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + /// `v[12..16] = t_lo, t_hi, block_len, flags` — all constants here. + fn input_v12(&mut self, j: usize) -> WordRef { + WordRef::Const( + [ + COUNTER_LFMC as u32, + (COUNTER_LFMC >> 32) as u32, + BLOCK_LEN_LFMC, + FLAGS_LFMC, + ][j], + ) + } + + fn iv_const(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + fn add3(&mut self, g: usize, half: usize, a: WordRef, b: WordRef, m_idx: usize) -> WordRef { + let base = cols::g_base(g); + let s = word_cols(base + if half == 0 { cols::G_A1 } else { cols::G_A2 }); + let cbase = base + + if half == 0 { + cols::G_A1_C + } else { + cols::G_A2_C + }; + self.0.add3s.push(Add3Wire { + a, + b, + m: message_word_ref(m_idx), + s, + c1: cbase, + c2: cbase + 1, + }); + WordRef::Cols(s) + } + + fn add2(&mut self, g: usize, half: usize, a: WordRef, b: WordRef) -> WordRef { + let s = word_cols(cols::g_base(g) + if half == 0 { cols::G_C1 } else { cols::G_C2 }); + self.0.add2s.push(Add2Wire { a, b, s }); + WordRef::Cols(s) + } + + fn xor(&mut self, g: usize, slot: usize, a: WordRef, b: WordRef) -> WordRef { + let off = match slot { + 0 => cols::G_X1, + 1 => cols::G_X2, + 2 => cols::G_X3, + _ => cols::G_X4, + }; + let out = word_cols(cols::g_base(g) + off); + self.0.xors.push(XorWire { a, b, out }); + WordRef::Cols(out) + } + + fn rotr16(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(2) + } + + fn rotr8(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(1) + } + + fn rot_shift(&mut self, g: usize, half: usize, w: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_R1 } else { cols::G_R2 }; + let y = word_cols(base + 8); + self.0.rots.push(RotWire { + input: w, + sll_lo: [base, base + 1], + sllc_lo: [base + 2, base + 3], + sll_hi: [base + 4, base + 5], + sllc_hi: [base + 6, base + 7], + y, + r: ROT_SHIFT_R[half], + }); + WordRef::Cols(y) + } + + fn feed_forward_low(&mut self, i: usize, vi: WordRef, vi8: WordRef) { + self.0.xors.push(XorWire { + a: vi, + b: vi8, + out: word_cols(cols::out_byte(i, 0)), + }); + } + + /// ✗ Never called: [`FLOW`] has `full_output = false`. `out[i+8]` is not + /// part of a truncated 128-bit digest, and never building those twelve + /// words is where most of the socket's saving comes from. + fn feed_forward_high(&mut self, _i: usize, _vi8: WordRef, _hi: WordRef) { + unreachable!("the socket's truncation window produces no high output words") + } +} + +/// The socket's full wiring, in canonical order. Built from the single +/// dataflow, so the senders below and the witness written by +/// [`fill_socket_witness`] cannot drift apart. +fn socket_wires() -> WireFlow { + let mut w = SocketWire(WireFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + OUT_WINDOW), + rots: Vec::with_capacity(NUM_G * 2), + }); + run_flow(&mut w, FLOW); + w.0 +} + +/// The value interpretation of the same dataflow, for one row's twelve message +/// lanes in one domain. +/// +/// Lanes rather than cells because a LEAF row's lanes are not cells throughout — +/// lanes 0–3 are its accumulator and lanes 4–11 are four felts' halves. The +/// mixing core does not care which; it sees twelve `u32`s either way, and that is +/// exactly why the leaf mode needs no new layout. +/// +/// The tag is an input because it is a message word: it enters the very first +/// round's `add3` and every value downstream of it, so a row's witness and its +/// BITWISE lookups both depend on which domain the row hashes in. +fn socket_values(lanes: &[u32; cols::NUM_LANES], tag: u32) -> ValueFlow { + ValueFlow::compute_with( + &BLAKE3_IV, + &socket_message(lanes, tag), + COUNTER_LFMC, + BLOCK_LEN_LFMC, + FLAGS_LFMC, + FLOW, + ) +} + +// ========================================================================= +// Bus interactions — the BITWISE half of `chips::hash::bus_interactions` +// ========================================================================= + +fn direct(col: usize) -> BusValue { + BusValue::Packed { + start_column: col, + packing: stark::lookup::Packing::Direct, + } +} + +fn byte_bus_value(b: ByteRef) -> BusValue { + match b { + ByteRef::Col(c) => direct(c), + ByteRef::Const(v) => BusValue::constant(u64::from(v)), + } +} + +/// The BITWISE lookups the BLAKE3 arm adds to `LFM_HASH`'s six `LfmMem` tuples. +/// +/// Three groups, in canonical [`socket_wires`] order: +/// +/// 1. `ByteAlu[XOR]` per XOR byte — the mixing core and the feed-forward. The +/// lookup pins the output *and* byte-range-checks both operands, which is +/// why nearly every word in this design needs no explicit `AreBytes`. +/// 2. `AreBytes` on the four shift halfwords of each rotation. The `SLL` bound +/// is tight and load-bearing: with `2^16` invertible mod `p` it is what pins +/// `SLL = (x · 2^r) mod 2^16` uniquely. +/// 3. `AreBytes` on the input lanes' bytes — obligation O1. These are the only +/// bytes with no XOR consumer, exactly as `m`'s are in `blake3_chip`. +pub fn bitwise_interactions() -> Vec { + let wires = socket_wires(); + let mut interactions = + Vec::with_capacity(4 * wires.xors.len() + 4 * wires.rots.len() + 2 * cols::NUM_LANES); + let mu = || { + Multiplicity::Sum3( + cols::MU_COLUMNS[0], + cols::MU_COLUMNS[1], + cols::MU_COLUMNS[2], + ) + }; + + for xw in &wires.xors { + for b in 0..4 { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + mu(), + vec![ + BusValue::constant(alu_op::XOR as u64), + byte_bus_value(xw.a.byte(b)), + byte_bus_value(xw.b.byte(b)), + direct(xw.out[b]), + ], + )); + } + } + + for rw in &wires.rots { + for pair in [rw.sll_lo, rw.sllc_lo, rw.sll_hi, rw.sllc_hi] { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + mu(), + vec![direct(pair[0]), direct(pair[1])], + )); + } + } + + for lane in 0..cols::NUM_LANES { + for p in 0..2 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + mu(), + vec![ + direct(cols::lane_byte(lane, 2 * p)), + direct(cols::lane_byte(lane, 2 * p + 1)), + ], + )); + } + } + + interactions +} + +/// The BITWISE lookups [`bitwise_interactions`] sends, mirrored send for send, +/// for the multiplicity histogram. Enumeration order is the senders' own, via +/// the shared [`ValueFlow`]. +/// +/// Each row is `(lanes, tag)`: the domain reaches the histogram because it +/// reaches the tag word, and every XOR byte downstream of round 0 differs between the +/// domains. A histogram built with the wrong tag balances against nothing. +pub fn bitwise_ops_for(rows: &[([u32; cols::NUM_LANES], u32)]) -> Vec { + let mut out = Vec::with_capacity( + rows.len() * (4 * (NUM_G * 4 + OUT_WINDOW) + 4 * NUM_G * 2 + 2 * cols::NUM_LANES), + ); + + for (lanes, tag) in rows { + let flow = socket_values(lanes, *tag); + for &(x, y, _out) in &flow.xors { + for byte in 0..4 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + ((x >> (8 * byte)) & 0xFF) as u8, + ((y >> (8 * byte)) & 0xFF) as u8, + )); + } + } + for &(sll_lo, sllc_lo, sll_hi, sllc_hi, _y) in &flow.rots { + for hw in [sll_lo, sllc_lo, sll_hi, sllc_hi] { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (hw & 0xFF) as u8, + (hw >> 8) as u8, + )); + } + } + for &lane in lanes.iter() { + for p in 0..2 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((lane >> (16 * p)) & 0xFF) as u8, + ((lane >> (16 * p + 8)) & 0xFF) as u8, + )); + } + } + } + + out +} + +// ========================================================================= +// Trace +// ========================================================================= + +#[inline] +fn set_word_bytes(row: &mut [FE], col: usize, w: u32) { + for b in 0..4 { + row[col + b] = FE::from(u64::from((w >> (8 * b)) as u8)); + } +} + +/// Writes the BLAKE3 witness into a hash row whose `IN`/`S`/`OUT` columns are +/// already filled. +/// +/// The two input cells are read back out of the row's own `IN0..8` columns — +/// the exact cells the lane-decomposition constraints read — rather than from +/// the executor record, so the witness cannot describe a different input than +/// the one the AIR constrains. That is the discipline `fill_poseidon_witness` +/// established and it matters more here, because the lane boundary is where the +/// only new soundness surface lives. +/// +/// The DOMAIN is read back out of the row's own mode columns for the same +/// reason, and it is the half that matters most: the tag word is a linear form over +/// exactly those columns, so a witness built from them cannot describe a +/// different domain than the one the AIR evaluates. Taking the tag as an +/// argument — as this did at first — left a filler that could be handed the +/// wrong domain for a row whose selectors said otherwise. +/// +/// # Panics +/// +/// If a lane is not a `u32`, or if the row selects no domain this arm has a +/// socket for. `LfmHasher::admits` rejects both at execution, so reaching here +/// means the executor and the trace filler disagree. +pub fn fill_socket_witness(row: &mut [FE]) { + let tag = tag_from_row(row); + fill_socket_witness_tagged(row, tag); +} + +/// The row's domain tag, read off its preprocessed mode columns — the machine +/// side of [`TAG_SELECTOR`], and the same value the tag word evaluates to. +/// +/// # Panics +/// +/// If the row selects neither two-to-one domain. A padding row never reaches +/// the filler (`chip_trace` fills only real rows) and a permute row is +/// unprovable here, so either is a caller bug rather than a case to handle. +fn tag_from_row(row: &[FE]) -> u32 { + let one = FE::one(); + let set: Vec = TAG_SELECTOR + .iter() + .filter(|(col, _)| row[*col] == one) + .map(|(_, tag)| *tag) + .collect(); + match set[..] { + [tag] => tag, + _ => panic!( + "a BLAKE3 hash row must select EXACTLY ONE of the domains this arm \ + has a socket for (MODE_C, MODE_T, MODE_L). None set means a permute \ + or padding row reached the socket witness filler; more than one is a \ + row the registrar's one-hot check should already have refused. \ + Either way its AIR cannot prove the row." + ), + } +} + +/// The twelve message lanes of a row that reads `cells`, split the way its mode +/// reads them. +/// +/// ★ **The row is a HYBRID and this is the only place the split lives.** Lanes +/// 0–3 are `cells[0]` read as digest lanes under EVERY mode — `a` on a digest +/// row, the chaining accumulator on a leaf row. Above that the readings differ: +/// +/// - **digest modes** — lanes 4–11 are `cells[1]` and `cells[2]`, two more cells +/// of four `u32` lanes. The third is unread by these modes and the AIR pins it +/// to zero, so those four lanes are zero on every honest digest row. +/// - **leaf mode** — lanes 4–11 are `cells[1]` read as four FELTS and split into +/// `lo`/`hi` halves. +/// +/// The trace filler and the BITWISE histogram both come through here, so a +/// witness and the multiplicities it must balance against cannot split a row +/// differently (COMMIT.md §1.4.4 **H5**). +pub fn lanes_from_cells(is_leaf: bool, cells: &[LfmWord; 3]) -> [u32; cols::NUM_LANES] { + let mut lanes = [0u32; cols::NUM_LANES]; + lanes[..cols::NUM_ACC_LANES] + .copy_from_slice(&lanes_of(&cells[0]).expect("socket lane is not a u32 (O1)")); + if is_leaf { + lanes[cols::NUM_ACC_LANES..].copy_from_slice( + &leaf_lanes(&cells[1]).expect("leaf felt is not canonical (LEAF.md §1.1)"), + ); + } else { + for (k, cell) in cells[1..].iter().enumerate() { + let base = cols::NUM_ACC_LANES + 4 * k; + lanes[base..base + 4] + .copy_from_slice(&lanes_of(cell).expect("compress lane is not a u32 (O1)")); + } + } + lanes +} + +/// [`lanes_from_cells`] for a trace row, reading the input cells and the mode +/// off the row itself. +/// +/// Keyed on `MODE_L` rather than on a tag, so it is total for any row a control +/// can build — including one whose mode columns are fractional. +fn lanes_from_row(row: &[FE]) -> [u32; cols::NUM_LANES] { + let cell = |base: usize| -> LfmWord { core::array::from_fn(|i| row[base + i]) }; + lanes_from_cells( + row[cols::MODE_L] == FE::one(), + &[cell(cols::IN0), cell(cols::IN0 + 4), cell(cols::IN0 + 8)], + ) +} + +/// The canonicity witnesses for one leaf row's four felts. +/// +/// `Z_i = 1` exactly when felt `i`'s high half is maximal; `GINV_i` inverts +/// `G_i = (2^32 − 1) − hi_i` when that is nonzero and is zero when it is not. +/// The same `Z`/`GINV` pair `LFM_BITDEC` uses for its own canonicity check. +fn fill_canonicity_witness(row: &mut [FE], lanes: &[u32; cols::NUM_LANES]) { + for i in 0..FELTS_PER_LEAF { + let hi = lanes[cols::leaf_hi_lane(i)]; + let g = u64::from(MAX_HALF - hi); + let (z, ginv) = if g == 0 { + (FE::one(), FE::zero()) + } else { + ( + FE::zero(), + FE::from(g).inv().expect("a nonzero field element inverts"), + ) + }; + row[cols::canon_z(i)] = z; + row[cols::canon_ginv(i)] = ginv; + } +} + +/// [`fill_socket_witness`] under an EXPLICIT domain. +/// +/// Exists for the negative controls (M1/M2), which have to build a row whose +/// witness and whose mode columns deliberately disagree — the forgery the +/// domain separation is supposed to reject. Production goes through +/// [`fill_socket_witness`], which cannot construct that. +pub(crate) fn fill_socket_witness_tagged(row: &mut [FE], tag: u32) { + // The lane READING is a property of the row's mode; the `tag` argument is + // only the hash DOMAIN. Keeping them separate is what lets the mode- + // confusion controls build a row that reads its input correctly and hashes + // it in the wrong domain — which is the forgery, and it would be + // unconstructible if one argument decided both. + let lanes = lanes_from_row(row); + + for (lane, &v) in lanes.iter().enumerate() { + set_word_bytes(row, cols::lane_byte(lane, 0), v); + } + // The canonicity witnesses are filled on EVERY row, not only leaf rows: the + // columns exist chip-wide, and a digest row's felts are its lanes, whose + // high halves are never maximal-and-nonzero-low in a way that matters + // because the constraints are `MODE_L`-gated. Filling them uniformly keeps + // the filler branch-free and leaves no uninitialised witness anywhere. + fill_canonicity_witness(row, &lanes); + + let flow = socket_values(&lanes, tag); + let mut a3 = flow.add3s.iter(); + let mut a2 = flow.add2s.iter(); + let mut xo = flow.xors.iter(); + let mut ro = flow.rots.iter(); + for g in 0..NUM_G { + let base = cols::g_base(g); + for half in 0..2 { + let (s_off, c_off, x_off, c2_off, x2_off, r_off) = if half == 0 { + ( + cols::G_A1, + cols::G_A1_C, + cols::G_X1, + cols::G_C1, + cols::G_X2, + cols::G_R1, + ) + } else { + ( + cols::G_A2, + cols::G_A2_C, + cols::G_X3, + cols::G_C2, + cols::G_X4, + cols::G_R2, + ) + }; + let &(s, c1, c2) = a3.next().expect("add3 count"); + set_word_bytes(row, base + s_off, s); + row[base + c_off] = FE::from(u64::from(c1)); + row[base + c_off + 1] = FE::from(u64::from(c2)); + + let &(_, _, x) = xo.next().expect("xor count"); + set_word_bytes(row, base + x_off, x); + + let &c = a2.next().expect("add2 count"); + set_word_bytes(row, base + c2_off, c); + + let &(_, _, x2) = xo.next().expect("xor count"); + set_word_bytes(row, base + x2_off, x2); + + let &(sll_lo, sllc_lo, sll_hi, sllc_hi, y) = ro.next().expect("rot count"); + let hw = |v: u16, k: usize| FE::from(u64::from((v >> (8 * k)) as u8)); + for (k, v) in [sll_lo, sllc_lo, sll_hi, sllc_hi].into_iter().enumerate() { + row[base + r_off + 2 * k] = hw(v, 0); + row[base + r_off + 2 * k + 1] = hw(v, 1); + } + set_word_bytes(row, base + r_off + 8, y); + } + } + + for i in 0..OUT_WINDOW { + set_word_bytes(row, cols::out_byte(i, 0), flow.out[i]); + debug_assert_eq!( + row[cols::OUT0 + i], + FE::from(u64::from(flow.out[i])), + "the digest lane the executor wrote is the one the mixing core produced" + ); + } +} + +// ========================================================================= +// Constraints +// ========================================================================= + +/// Constraints the BLAKE3 arm emits. +/// +/// The framing block — 4 capacity copies, the mode-sum booleanity, the +/// `MODE_P = 0` pin, [`cols::NUM_LANES`] lane decompositions, 8 unused-output +/// pins, 4 digest recompositions, [`NUM_UNREAD_INPUT_PINS`] unread-`IN` pins and +/// 16 leaf felt/canonicity constraints — plus 16 per G-instance: per G, two +/// add3s (a sum identity and two carry booleanities each), two add2 carry +/// booleanities, and two rotations (two shift identities and two recombines +/// each). +pub const NUM_CONSTRAINTS: usize = CORE_IDX + 16 * NUM_G; + +/// First mixing-core constraint index — everything below it is framing. +const CORE_IDX: usize = LEAF_IDX + LEAF_CONSTRAINTS_PER_FELT * FELTS_PER_LEAF; + +/// First lane-decomposition index: after the capacity copies, the mode-sum +/// booleanity and the `MODE_P` pin. +/// +/// Public so the gate suite can name the identity for a specific lane rather +/// than locate it by a literal — the point of the H6 controls is that the +/// violated constraint IS the lane's own identity. +pub const LANE_IDX: usize = 6; + +/// First unused-output pin index. +/// +/// ★ **DERIVED FROM [`cols::NUM_LANES`], never written as a literal.** The lane +/// block grew from 8 to 12 with the leaf RATE, and the constraint COUNT did not +/// move — the unread-`IN` pins lost exactly the four the lanes gained — so a +/// hardcoded `14` here would have silently overwritten the first four output +/// pins with lane identities, leaving lanes 8–11 with no identity at all. +/// `EmitTracker`'s duplicate assert is `#[cfg(debug_assertions)]` and the house +/// convention runs the suite in release, so nothing would have failed +/// (COMMIT.md §1.4.4 **H1**). `blake3_socket_tests:: +/// every_hash_candidate_emits_each_constraint_index_exactly_once` is the +/// release-visible guard that would catch it if these ever go back to literals. +const OUT_PIN_IDX: usize = LANE_IDX + cols::NUM_LANES; + +/// First digest-recomposition index. +const DIGEST_IDX: usize = OUT_PIN_IDX + 8; + +/// First unread-`IN` pin index. +/// +/// Public so the controls can name the pins rather than locate them by a +/// literal — the point of those tests is that the violated set IS the pins. +pub const UNREAD_IDX: usize = DIGEST_IDX + OUT_WINDOW; + +/// First LEAF constraint index: four per felt (the halves binding and the three +/// canonicity constraints), after the shared unread-`IN` pins. +/// +/// Public so the controls can locate a specific canonicity constraint by name +/// rather than by a literal that silently rots when the framing grows. +pub const LEAF_IDX: usize = UNREAD_IDX + NUM_UNREAD_INPUT_PINS; + +/// Constraints per leaf felt: the halves binding, then `canon-a/b/c`. +pub const LEAF_CONSTRAINTS_PER_FELT: usize = 4; + +/// The BLAKE3 arm of `HashConstraints::eval`. +/// +/// Every constraint is mu-gated on `MU = MODE_C + MODE_T + MODE_L` and every +/// bus send carries the same sum, so an all-zero padding row satisfies the set +/// vacuously and emits nothing. Max degree is 3, reached by the mu-gated carry +/// booleanities and by the leaf canonicity block — the wrap's blowup 2 depends +/// on that staying 3, which is why the 3-operand add uses two summed carry BITS +/// rather than one ternary carry (`k(k−1)(k−2) = 0` is already degree 3, and +/// mu-gating would push it to 4). +pub fn eval>(b: &mut B) { + let mu = |b: &B| { + let [c, t, l] = cols::MU_COLUMNS; + b.main(0, c) + b.main(0, t) + b.main(0, l) + }; + let digest_mu = |b: &B| { + let [c, t] = cols::DIGEST_MODE_COLUMNS; + b.main(0, c) + b.main(0, t) + }; + let mode_c = b.main(0, cols::MODE_C); + let mode_t = b.main(0, cols::MODE_T); + let mode_l = b.main(0, cols::MODE_L); + let mode_p = b.main(0, cols::MODE_P); + + // idx 0–3: capacity-state copy, in the same shape every other arm uses — + // `S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i`. A transcript row + // and a leaf row are both compresses in framing, so their capacity prefix is + // still the IV; only the selector widens. With MODE_P pinned to zero below + // this reduces to `S_i = MU·IV_i`; it is written in the general form so the + // shared prefix means the same thing under every hasher. + for (k, iv) in BLAKE3_IV.iter().take(4).enumerate() { + let s = b.main(0, cols::S8 + k); + let in_i = b.main(0, cols::IN0 + 8 + k); + let iv_i = b.const_base(u64::from(*iv)); + let m = mu(b); + b.emit_base(k, s - (mode_p.clone() * in_i + m * iv_i)); + } + + // idx 4: mode sum-boolean (exactly-one-of is the registrar's). This is what + // excludes two selectors both being 1 — which would sum BOTH domain tags + // into the tag word — since the sum would be 2 and 2·(1−2) ≠ 0. ⚠ It does NOT + // force each selector to a bit: a fractional split still satisfies it and + // blends the tags, which is what control M5/M6 demonstrates and what the + // registrar's one-hot check is the actual answer to. + let mode_sum = mode_c + mode_t + mode_l.clone() + mode_p.clone(); + let one = b.one(); + b.emit_base(4, mode_sum.clone() * (one - mode_sum)); + + // idx 5: ✗ no permute socket, PERMANENTLY. Pinning the preprocessed mode + // selector makes a program containing a `permute` unprovable under BLAKE3 + // rather than silently proved against a framing nobody specified. Option B1 + // decided no permute socket is ever built, so this pin is not a placeholder + // waiting to be deleted — it is the decision, written down as a constraint. + b.emit_base(5, mode_p); + + // THE LANE BOUNDARY (obligation O1). One linear identity per input lane; the + // matching `AreBytes` sends are in `bitwise_interactions`. NEITHER ALONE + // SUFFICES, and the two buy DIFFERENT things — see the module docs. This + // identity makes `IN_lane` and `m[lane]` the same field element, because the + // core reads the same linear form; the sends bound the bytes, and are the + // message words' ONLY range check, which is what `add3`'s exactness needs. + // With both, the sum of four bytes weighted by 2^{8k} is < 2^32 ≪ p, so it + // cannot wrap and the lane is forced below 2^32. + // + // ★ THE GATE IS PER LANE RANGE, and getting it wrong in the permissive + // direction is a soundness break (COMMIT.md §1.4.4 **H6**): + // + // - **lanes 0–3 → the full mu.** They are `a` on a digest row and the + // chaining ACCUMULATOR on a leaf row, and the same identity is correct for + // both readings, so one gate serves. This is also the only thing that + // range-checks the accumulator: identity + `AreBytes` is what makes "the + // accumulator is a previous digest, hence `u32`" a constraint rather than + // a hope. Gate these on `digest_mu` and a leaf row's accumulator carries + // no identity at all — the prover picks the chain's message words freely + // and the whole leaf chain unbinds. + // - **lanes 4–11 → the digest modes only.** On a LEAF row they are four + // felts' halves, so `IN_lane` and `m[lane]` are deliberately NOT the same + // field element — the leaf block below states the relation those rows do + // satisfy. Gating these on mu instead would make every leaf row + // unprovable. + // + // ★ On a digest row lanes 8–11 read the THIRD input cell, which the unread- + // `IN` pins force to zero, so the identity reads `0 = Σ bytes·2^{8k}` and + // with the `AreBytes` bound in hand forces all sixteen bytes to zero. That + // is what keeps the four message words the leaf RATE added out of the + // prover's hands on a Merkle parent — free, but only because these + // identities exist. + for lane in 0..cols::NUM_LANES { + let felt = b.main(0, cols::IN0 + lane); + let bytes = word_expr(b, &WordRef::Cols(word_cols(cols::lane_byte(lane, 0)))); + let m = if lane < cols::NUM_ACC_LANES { + mu(b) + } else { + digest_mu(b) + }; + b.emit_base(LANE_IDX + lane, m * (felt - bytes)); + } + + // The digest is ONE cell, so the upper eight `OUT` lanes carry nothing. + // `MULT1`/`MULT2` are zero on a Compress row so they reach no bus, but + // pinning them costs eight degree-1 constraints and removes the question + // entirely. Ungated: they are zero on padding rows too. + for j in 0..8 { + let out = b.main(0, cols::OUT0 + HASH_DIGEST_FELTS + j); + b.emit_base(OUT_PIN_IDX + j, out); + } + + // The digest lanes. No range check is needed on `OUTW`'s bytes — they are + // `ByteAlu[XOR]` outputs, hence already bytes — and the sum is < 2^32 ≪ p, + // so `OUT_i` is forced to the honest u32. That is why the socket's OUTPUT + // always satisfies O1 (obligation O2) and only leaf digests and + // prover-hinted siblings need the input check. + for i in 0..OUT_WINDOW { + let felt = b.main(0, cols::OUT0 + i); + let bytes = word_expr(b, &WordRef::Cols(word_cols(cols::out_byte(i, 0)))); + let m = mu(b); + b.emit_base(DIGEST_IDX + i, m * (felt - bytes)); + } + + // The input cells this row's mode does not read. + // + // ⚠ **LOAD-BEARING, and not only here.** On THIS arm the unread columns + // reach no constraint, so the pin is what keeps them from being an open + // question. On an arm whose constraints read `IN` — `Test` and `Poseidon` + // both do, `A_i = IN_i` for `i < 8` — the same pin is the difference between + // a leaf digest that is a function of its input and one carrying four free + // prover-chosen felts. It shipped missing there once. That is why this is + // `chips::hash`'s single derivation from `HashMode::num_input_cells` and not + // four lines written out per arm. + let next = crate::lfm::chips::hash::emit_unread_input_pins(b, UNREAD_IDX); + debug_assert_eq!(next, LEAF_IDX); + + // ★ THE LEAF MODE. Per felt: the halves binding, then the three canonicity + // constraints. The felts are the SECOND input cell — the first is the + // chaining accumulator, whose lanes the identity block above binds. + // + // `v = lo + 2^32·hi` with `lo, hi < 2^32` is a decomposition, not yet a + // canonical one: `p − 1 = 0xFFFFFFFF_00000000`, so the pairs with `hi` + // maximal and `lo ≥ 1` encode field elements that ALSO have an ordinary + // encoding. Without the canonicity block one felt would have two half-pairs, + // hence two leaf digests — a collision in the felt→digest map, which is + // exactly what a Merkle tree must not have. + // + // `Z`/`GINV` is `LFM_BITDEC`'s own idiom over two halves instead of 64 bits: + // `canon_a` gives `G ≠ 0 ⇒ Z = 0`, `canon_b` gives `G = 0 ⇒ Z = 1`, and + // `canon_c` then reads "hi maximal ⇒ lo zero". + let two_32_leaf = b.const_base(1u64 << 32); + let max_half = b.const_base(u64::from(MAX_HALF)); + for i in 0..FELTS_PER_LEAF { + let lo = word_expr( + b, + &WordRef::Cols(word_cols(cols::lane_byte(cols::leaf_lo_lane(i), 0))), + ); + let hi = word_expr( + b, + &WordRef::Cols(word_cols(cols::lane_byte(cols::leaf_hi_lane(i), 0))), + ); + let v = b.main(0, cols::leaf_felt(i)); + let z = b.main(0, cols::canon_z(i)); + let ginv = b.main(0, cols::canon_ginv(i)); + let g = max_half.clone() - hi.clone(); + let base = LEAF_IDX + LEAF_CONSTRAINTS_PER_FELT * i; + + // binding: the felt IS its two halves. + b.emit_base( + base, + mode_l.clone() * (v - lo.clone() - hi * two_32_leaf.clone()), + ); + // canon-a: G ≠ 0 ⇒ Z = 0. + b.emit_base(base + 1, mode_l.clone() * z.clone() * g.clone()); + // canon-b: G = 0 ⇒ Z = 1. + let one = b.one(); + b.emit_base(base + 2, mode_l.clone() * (one - z.clone() - g * ginv)); + // canon-c: hi maximal ⇒ lo zero. THE constraint; the two above exist to + // make `Z` mean what this one needs it to mean. + b.emit_base(base + 3, mode_l.clone() * z * lo); + } + + // The mixing core, from the single dataflow. + let wires = socket_wires(); + let mut idx = CORE_IDX; + + let two_32 = b.const_base(1u64 << 32); + let inv_2_32 = b.const_base(crate::constraints::templates::INV_SHIFT_32); + + // add3: μ·(a + b + m − s − 2^32·(c1+c2)) = 0; μ·ci·(1−ci) = 0. + for aw in &wires.add3s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let m_w = word_expr(b, &aw.m); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let c1 = b.main(0, aw.c1); + let c2 = b.main(0, aw.c2); + let sum_id = a + bb + m_w - s - (c1.clone() + c2.clone()) * two_32.clone(); + let m = mu(b); + b.emit_base(idx, m * sum_id); + idx += 1; + for c in [c1, c2] { + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c.clone() * (one - c)); + idx += 1; + } + } + + // add2: the carry is an EXPRESSION, `(a + b − s)·2^−32`, not a column — + // `μ·carry·(1−carry) = 0` says `a + b − s ∈ {0, 2^32}`, which is the sum + // identity and the carry's booleanity in one degree-3 constraint. One + // column and one constraint per add2 cheaper than witnessing the carry, and + // exactly as strong. + for aw in &wires.add2s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let carry = (a + bb - s) * inv_2_32.clone(); + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * carry.clone() * (one - carry)); + idx += 1; + } + + // Rotations: two shift identities and two recombines each. Soundness needs + // 2^16 invertible mod p, which is a FIELD fact a bitvector model cannot + // see — it is what makes the tight `AreBytes` bound on `SLL` load-bearing. + for rw in &wires.rots { + let (xlo, xhi) = match &rw.input { + WordRef::Cols(c) => (half_expr(b, &[c[0], c[1]]), half_expr(b, &[c[2], c[3]])), + WordRef::Const(_) | WordRef::ModeSelected(_) => { + unreachable!("shift inputs are always committed XOR outputs") + } + }; + let sll_lo = half_expr(b, &rw.sll_lo); + let sllc_lo = half_expr(b, &rw.sllc_lo); + let sll_hi = half_expr(b, &rw.sll_hi); + let sllc_hi = half_expr(b, &rw.sllc_hi); + let ylo = half_expr(b, &[rw.y[0], rw.y[1]]); + let yhi = half_expr(b, &[rw.y[2], rw.y[3]]); + let two_r = b.const_base(1u64 << rw.r); + let two_16 = b.const_base(65536); + + let m = mu(b); + b.emit_base( + idx, + m * (xlo * two_r.clone() - sllc_lo.clone() * two_16.clone() - sll_lo.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base( + idx, + m * (xhi * two_r - sllc_hi.clone() * two_16 - sll_hi.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (ylo - sll_hi - sllc_lo)); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (yhi - sll_lo - sllc_hi)); + idx += 1; + } + + debug_assert_eq!( + idx, NUM_CONSTRAINTS, + "every declared constraint index must be emitted exactly once" + ); +} diff --git a/prover/src/lfm/blake3_socket_kats.rs b/prover/src/lfm/blake3_socket_kats.rs new file mode 100644 index 000000000..a188300a3 --- /dev/null +++ b/prover/src/lfm/blake3_socket_kats.rs @@ -0,0 +1,144 @@ +//! Socket KATs for the LFM 2-to-1 BLAKE3 compress, at 6 and 7 rounds. +//! +//! GENERATED — do not hand-edit. The INPUT pairs are the union of two +//! independently produced vector tables: +//! `thoughts/blake3/socket-kats/socket_kats.json` (Phase 1, upstream BLAKE3's C +//! at word level + its whole tree hasher at byte level) and the gate-oracle's +//! `socket_kats.json` (a separately written Python oracle). The two share 5 of +//! the 15 input pairs and agreed on every one of them at both round counts, +//! which is what makes this table two sources rather than one transcribed +//! twice. +//! +//! ⚠ **The DIGESTS were re-pinned when the socket widened to twelve lanes**, by +//! `leaf-spec/rate4_kat_gen.py` out of that same gate-oracle BLAKE3. All 30 +//! moved and no input did: `block_len` is `v[14]` and cannot be made +//! mode-dependent, so the Merkle domain re-blesses alongside the leaf domain +//! that needed the width (COMMIT.md §1.4.4 H9). The upstream-C leg of the +//! provenance is therefore historical for the digests and current for the +//! inputs. What still anchors the new values outside this tree is that a +//! 52-byte message is one BLAKE3 block, so at 7 rounds each vector remains a +//! plain `blake3::hash` call. +//! +//! Framing (`SOCKET.md` §2.2 at the COMMIT.md §1.2 width): h = BLAKE3_IV, +//! m[0..4] = a, m[4..8] = b, m[8..12] = 0 — the third input cell, which the +//! unread-`IN` pins force to zero — m[12] = "LFMC", m[13..16] = 0, t = 0, +//! block_len = 52, flags = 0x0B, digest = out[0..4]. + +/// One socket vector: the two input cells and the digest at each round count. +pub struct SocketVector { + pub name: &'static str, + pub a: [u32; 4], + pub b: [u32; 4], + /// Digest at 6 rounds (the A6R variant; no library computes it). + pub digest_6: [u32; 4], + /// Digest at 7 rounds — `blake3::hash(a ‖ b ‖ "LFMC")[..16]`. + pub digest_7: [u32; 4], +} + +pub const SOCKET_VECTORS: [SocketVector; 15] = [ + SocketVector { + name: "a_one/unit_a", // socket-kats+gate-oracle + a: [0x00000001, 0x00000000, 0x00000000, 0x00000000], + b: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + digest_6: [0x34064AA0, 0xD7155685, 0x37B1522B, 0x17147454], + digest_7: [0xFD482C6D, 0xF3D43A7D, 0xECE55DC3, 0xA2594A53], + }, + SocketVector { + name: "all_ones/all_ones", // socket-kats+gate-oracle + a: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + b: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + digest_6: [0xCCEF3565, 0x52BAE2BB, 0x4C0E5777, 0x2896C5CB], + digest_7: [0x6C73358D, 0x7AC15CE6, 0xDB7BFA9C, 0x65CD3364], + }, + SocketVector { + name: "b_one/unit_b", // socket-kats+gate-oracle + a: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + b: [0x00000001, 0x00000000, 0x00000000, 0x00000000], + digest_6: [0x3B0F1AEB, 0x7FBAC531, 0xF7576FDB, 0xC4054075], + digest_7: [0x43B017F6, 0x35DA5810, 0x1C2F8BF8, 0xB16F9B88], + }, + SocketVector { + name: "boundary", // gate-oracle + a: [0x00000000, 0x00000001, 0xFFFFFFFE, 0xFFFFFFFF], + b: [0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF], + digest_6: [0x03FA80ED, 0x79B929E7, 0x5FAF60C4, 0xB1F6E5C2], + digest_7: [0x518B26CE, 0xDD289FC7, 0x5E623AAC, 0xD189075A], + }, + SocketVector { + name: "formula_0", // socket-kats + a: [0x9E3779B9, 0x3C6EF372, 0xDAA66D2B, 0x78DDE6E4], + b: [0x8FF34781, 0x2E2AC13A, 0xCC623AF3, 0x6A99B4AC], + digest_6: [0x937642E5, 0x03E78AFE, 0x85C6D9CD, 0x42824862], + digest_7: [0xAB67E603, 0x5D251992, 0xCC7CE527, 0xD4EF91ED], + }, + SocketVector { + name: "formula_1", // socket-kats + a: [0x81AF1549, 0x1FE68F02, 0xBE1E08BB, 0x5C558274], + b: [0x736AE311, 0x11A25CCA, 0xAFD9D683, 0x4E11503C], + digest_6: [0xACE60EA5, 0x8135DE83, 0xEAE0B6DA, 0x9934A2F0], + digest_7: [0x653C2C04, 0x78EF5846, 0x6A736E8A, 0x914D4DB9], + }, + SocketVector { + name: "formula_1", // gate-oracle + a: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + b: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + digest_6: [0xF227539A, 0x2585193F, 0x16FB766E, 0x4B577722], + digest_7: [0xADED3052, 0x697F82FA, 0x7AEA4B01, 0x04BFCDBC], + }, + SocketVector { + name: "formula_2", // socket-kats + a: [0x6526B0D9, 0x035E2A92, 0xA195A44B, 0x3FCD1E04], + b: [0x56E27EA1, 0xF519F85A, 0x93517213, 0x3188EBCC], + digest_6: [0xC594FDD3, 0xBB338F05, 0x33C5A455, 0x2E9D6C14], + digest_7: [0xE87B6708, 0xCE220AD3, 0x5B64ED4F, 0xD8D574C2], + }, + SocketVector { + name: "formula_2", // gate-oracle + a: [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE], + b: [0x0BADC0DE, 0xD15EA5E5, 0xC0FFEE00, 0xBAAAAAAD], + digest_6: [0x7636E806, 0x8AAB225F, 0x7F947CE5, 0xA73023AF], + digest_7: [0xDBD49162, 0x7A22E380, 0x81E2ECB6, 0xF29F4F76], + }, + SocketVector { + name: "formula_3", // socket-kats + a: [0x489E4C69, 0xE6D5C622, 0x850D3FDB, 0x2344B994], + b: [0x3A5A1A31, 0xD89193EA, 0x76C90DA3, 0x1500875C], + digest_6: [0x4C93E057, 0xA31827EB, 0xFDE7CB52, 0x027F1933], + digest_7: [0x5754C2B6, 0x35C665F2, 0xFFB72630, 0xEC470985], + }, + SocketVector { + name: "formula_3", // gate-oracle + a: [0x7F800001, 0x00000002, 0x80000000, 0x7FFFFFFF], + b: [0x00FF00FF, 0xFF00FF00, 0x0F0F0F0F, 0xF0F0F0F0], + digest_6: [0xFEB2EB59, 0xD8D6F7E1, 0xDC2774D3, 0x09A984FE], + digest_7: [0x5213902B, 0xC0A84BF1, 0xA070DC22, 0x531B944C], + }, + SocketVector { + name: "formula_4", // socket-kats + a: [0x2C15E7F9, 0xCA4D61B2, 0x6884DB6B, 0x06BC5524], + b: [0x1DD1B5C1, 0xBC092F7A, 0x5A40A933, 0xF87822EC], + digest_6: [0x312C20F4, 0x077F08FF, 0x0608FFAF, 0x70423FD2], + digest_7: [0xF910DA3B, 0x5CCD211F, 0xB6D1E097, 0xA7304D4D], + }, + SocketVector { + name: "max_min", // gate-oracle + a: [0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000], + b: [0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF], + digest_6: [0xDCF8F70F, 0x18645178, 0xE0842849, 0x97FEC771], + digest_7: [0x5B047362, 0xE1662BCF, 0x385D410A, 0xFD185A9E], + }, + SocketVector { + name: "nibble_ramp/nibble_ramp", // socket-kats+gate-oracle + a: [0x00000000, 0x11111111, 0x22222222, 0x33333333], + b: [0x44444444, 0x55555555, 0x66666666, 0x77777777], + digest_6: [0xA218F925, 0x7819A69C, 0xC14B82EC, 0x60E8A949], + digest_7: [0x2E24E9F5, 0xA0D24A4B, 0x53909030, 0xB64285F3], + }, + SocketVector { + name: "zeros/zeros", // socket-kats+gate-oracle + a: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + b: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + digest_6: [0x9E1DF680, 0x17A425B2, 0x890775EA, 0xE2C6E09F], + digest_7: [0x9484C177, 0xBA11AECD, 0x45BB9F21, 0x031F727D], + }, +]; diff --git a/prover/src/lfm/blake3_socket_tests.rs b/prover/src/lfm/blake3_socket_tests.rs new file mode 100644 index 000000000..9e8c46b7c --- /dev/null +++ b/prover/src/lfm/blake3_socket_tests.rs @@ -0,0 +1,1641 @@ +//! The BLAKE3 arm of `LFM_HASH`: its framing, its layout, its degree bound, +//! what it accepts, what it rejects, and the prove+verify that turns a +//! predicted cell count into a measured one. +//! +//! ## What pins what +//! +//! Three layers, and they are deliberately not the same evidence: +//! +//! 1. **The primitive** is pinned elsewhere, to the `blake3` crate: +//! `blake3::tests::seven_rounds_is_the_blake3_crate`. Nothing here re-checks +//! the G function or the message schedule. +//! 2. **The framing** — the six choices between "a correct `f`" and "a correct +//! 2-to-1 compress" — is pinned here by [`SOCKET_VECTORS`], which came from +//! two independent generators, plus one negative control per choice. A right +//! constant inside a wrong framing is the normal way this goes wrong, and +//! every primitive test stays green while it happens. +//! 3. **The chip** is pinned here too: that `NUM_CONSTRAINTS` constraints over +//! `MAIN_COLUMNS` value columns say exactly what that framing says, and that +//! they say it inside a real proof produced by the production prover. +//! +//! ## What this suite cannot see +//! +//! It says nothing about the machine's DEFAULT hash, which is still +//! `TestPermutation`; every test constructs the BLAKE3 configuration +//! explicitly. It covers the two two-to-one modes — `compress` and the +//! `transcript` step, which are the same socket under different domain tags — +//! and no `permute`, because option B1 settled that no permute socket is ever +//! built. See `blake3_socket`'s module docs. +//! +//! The transcript's own vectors and its end-to-end behaviour live in +//! `transcript_tests`; what is here is the CHIP side of it — the mode-selected +//! tag, and the M1–M7 controls the transcript spec pre-committed. + +use math::field::element::FieldElement; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, ProverEvalFolder, RootKind, check_dense_index_set, + num_base_from_meta, +}; +use stark::frame::Frame; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::table::TableView; +use stark::trace::TraceTable; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField, VmTable}; + +use super::airs::lfm_chip_census_with_hasher; +use super::blake3::{BLAKE3_IV, BLAKE3_MSG_PERMUTATION}; +use super::blake3_socket::{ + self, BLOCK_LEN_LFMC, Blake3Permutation, COUNTER_LFMC, FLAGS_LFMC, MAIN_COLUMNS, + NUM_CONSTRAINTS, NUM_G, SOCKET_ROUNDS, TAG_LFMC, cols, lanes_of, socket_digest, + socket_digest_rounds, word_of, +}; +use super::blake3_socket_kats::SOCKET_VECTORS; +use super::builder::{Cell, LfmBuilder, LfmProgramSource}; +use super::chips::hash::{self, HashConstraints}; +use super::compiler::{LfmProgram, compile}; +use super::executor::{LfmExecError, execute}; +use super::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use super::instr::HashMode; +use super::programs::{permute_coverage_program, trivial_program}; +use super::proof::{lfm_prove_with_hasher, prove_traces_with_hasher, verify_against}; +use super::registry::{build_artifacts, build_artifacts_with_hasher}; +use super::trace::build_traces_with_hasher; +use super::word::LfmWord; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +const KIND: HasherKind = HasherKind::Blake3; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +// ========================================================================= +// The closed-form budget, written out for BOTH round counts +// ========================================================================= + +/// Value columns per compression, as a function of the round count. +/// +/// Written as a formula over named blocks rather than taken from the layout, +/// because a closed form taken from the code under test would agree with any +/// layout, including a wrong one. 28 shared prefix + `4·12` lane bytes + +/// `8·rounds` G-blocks of 60 + 16 digest bytes + 8 leaf-canonicity witnesses +/// (`Z`/`GINV` per felt — present on EVERY row, since a chip has one width). +/// +/// ★ The lane count is spelled `4 + 2·4` — the accumulator cell plus one felt +/// cell's halves — rather than read from `cols::NUM_LANES`, for exactly the +/// reason the rest is spelled out: taking it from the layout would make this +/// agree with ANY widening instead of with the one the RATE specifies. Note +/// what does NOT move with it: the canonicity witnesses stay at 8, because the +/// accumulator is a previous digest and needs byte decomposition but no +/// canonicity gate. That is the claim that made the RATE cost +16 columns +/// rather than +24. +const fn predicted_main(rounds: usize) -> usize { + 28 + 4 * (4 + 2 * 4) + 60 * (8 * rounds) + 16 + 8 +} + +/// Bus interactions per compression: the frozen six `LfmMem` tuples, four +/// `ByteAlu[XOR]` per XOR word (`4·8·rounds` mixing words + 4 feed-forward), +/// four `AreBytes` per rotation (`2·8·rounds` of them), and two lane `AreBytes` +/// per lane. +const fn predicted_interactions(rounds: usize) -> usize { + 6 + 4 * (4 * (8 * rounds) + 4) + 4 * (2 * (8 * rounds)) + 2 * (4 + 2 * 4) +} + +/// `main + 3·aux` with `aux = ceil(interactions / 2)` — `airs.rs`'s census +/// formula, the same instrument that produced the keccak, Poseidon and +/// standalone-blake columns, so all four are comparable by construction. +/// +/// `pub(super)` so [`super::blake3_probe`] can state the socket-vs-standalone +/// comparison against THIS number instead of a transcription of it. The copy it +/// carried had drifted by 8 — it predated the leaf mode's canonicity block — and +/// a hand-copied cost figure is exactly the kind that rots in silence, because +/// nothing recomputes it. +pub(super) const fn predicted_cells(rounds: usize) -> usize { + predicted_main(rounds) + 3 * predicted_interactions(rounds).div_ceil(2) +} + +const fn predicted_constraints(rounds: usize) -> usize { + 50 + 16 * (8 * rounds) +} + +/// The whole budget, at both round counts, as literals. +/// +/// These are the numbers the report carries and the A6R decision is priced +/// against, so they are written out rather than left as an expression: the +/// arithmetic and the layout are two statements, and a test is only worth +/// having if they can disagree. +#[test] +fn the_socket_budget_is_the_predicted_one_at_both_round_counts() { + // 6 rounds — the A6R variant. + assert_eq!(predicted_main(6), 2_980); + assert_eq!(predicted_interactions(6), 1_198); + assert_eq!(predicted_interactions(6).div_ceil(2), 599); + assert_eq!(predicted_cells(6), 4_777); + assert_eq!(predicted_constraints(6), 818); + + // 7 rounds — standard BLAKE3, the default. + assert_eq!(predicted_main(7), 3_460); + assert_eq!(predicted_interactions(7), 1_390); + assert_eq!(predicted_interactions(7).div_ceil(2), 695); + assert_eq!(predicted_cells(7), 5_545); + assert_eq!(predicted_constraints(7), 946); + + // ★★ WHAT THE LEAF RATE COST, priced here rather than asserted anywhere + // else: +16 main columns (four lanes × four bytes), +8 bus interactions + // (two `AreBytes` per new lane) and +28 census cells at 7 rounds, against a + // 2.0× cut in leaf absorption — which is ~70% of a recursion tower node's + // bill (COMMIT.md §1.4.1). The pre-RATE figures were 3,444 / 1,382 / 5,517. + // The CONSTRAINT count did not move at all, and that is not luck: the lane + // identities gained four while the unread-`IN` pins lost four, which is + // precisely why a hand-numbered framing block could have overwritten the + // pins in silence (§1.4.4 H1). + assert_eq!(predicted_main(7) - (28 + 32 + 60 * 56 + 16 + 8), 16); + assert_eq!(predicted_cells(7) - 5_517, 28); + assert_eq!(predicted_constraints(7), 946, "unchanged by the widening"); + + // ★ The A6R price, on this socket: going 6 → 7 rounds costs +16.07% per + // compression. The plan's paper estimate for the syscall-shaped chip was + // +15.5%; the socket pays slightly more because its constant framing makes + // the round-INDEPENDENT part smaller, so the rounds are a larger share. + assert_eq!( + (predicted_cells(7) - predicted_cells(6)) * 10_000 / predicted_cells(6), + 1_607, + "hundredths of a percent" + ); + + // Both are BELOW the standalone chip's measured 4,946, which is the point of + // hosting: constant `h`/`t`/`block_len`/`flags`, a constant message tail, and a + // truncation window that never builds twelve of the sixteen output words. + assert!(predicted_cells(6) < 4_946); +} + +/// The compiled arm IS the prediction at the round count it was compiled for. +#[test] +fn the_built_layout_matches_the_prediction() { + // The single-knob invariant, asserted where it can actually fail: the + // socket and the standalone probe must be priced at the SAME round count. + // `NUM_G == 8 * SOCKET_ROUNDS` below is internally consistent either way, + // so it cannot see the two chips drifting apart; this can. + assert_eq!(SOCKET_ROUNDS, super::blake3::BLAKE3_ROUNDS); + assert_eq!( + NUM_G, + super::blake3_chip::NUM_G, + "the socket arm and the standalone LFM_BLAKE3 probe must be compiled \ + for the same round count, or the probe prices a hash the machine does \ + not use" + ); + assert_eq!(NUM_G, 8 * SOCKET_ROUNDS); + assert_eq!(MAIN_COLUMNS, predicted_main(SOCKET_ROUNDS)); + assert_eq!( + hash::num_columns(KIND) - cols::PREP_WIDTH, + predicted_main(SOCKET_ROUNDS) + ); + assert_eq!( + hash::bus_interactions(KIND).len(), + predicted_interactions(SOCKET_ROUNDS) + ); + assert_eq!(NUM_CONSTRAINTS, predicted_constraints(SOCKET_ROUNDS)); + // 12 since option B1 added `MODE_T` (was 11). The prefix is the hasher- + // independent instruction group, so this number is the same under every + // candidate — `poseidon_chip_tests` pins the identical value, and the two + // together are what would catch one arm's layout drifting from the other's. + assert_eq!( + cols::PREP_WIDTH, + 13, + "the preprocessed prefix does not move" + ); + assert_eq!(cols::LANES, 41, "the shared value prefix is not reflowed"); +} + +/// The layout is injective and gapless — no column written twice, none unread. +/// +/// The width alone cannot see an off-by-one inside `lane_byte`/`g_base`/ +/// `out_byte`: two blocks could overlap and the total still come out right. +#[test] +fn the_layout_assigns_every_column_exactly_once() { + let mut seen = vec![0usize; cols::NUM_COLUMNS]; + let mut claim = |c: usize| seen[c] += 1; + for i in 0..HASH_STATE_FELTS { + claim(cols::IN0 + i); + claim(cols::OUT0 + i); + } + for k in 0..4 { + claim(cols::S8 + k); + } + for lane in 0..cols::NUM_LANES { + for b in 0..4 { + claim(cols::lane_byte(lane, b)); + } + } + for g in 0..NUM_G { + for off in 0..cols::G_SIZE { + claim(cols::g_base(g) + off); + } + } + for i in 0..4 { + for b in 0..4 { + claim(cols::out_byte(i, b)); + } + } + for i in 0..blake3_socket::FELTS_PER_LEAF { + claim(cols::canon_z(i)); + claim(cols::canon_ginv(i)); + } + for (c, &n) in seen.iter().enumerate().skip(cols::PREP_WIDTH) { + assert_eq!( + n, 1, + "value column {c} is claimed {n} times, want exactly 1" + ); + } + for (c, &n) in seen.iter().enumerate().take(cols::PREP_WIDTH) { + assert_eq!(n, 0, "preprocessed column {c} must not be claimed"); + } +} + +/// The census reports the arm at its real width and its real interaction count, +/// so the hash-matrix instrument prices BLAKE3 rather than a stale Test column. +#[test] +fn the_census_prices_the_blake3_arm() { + let opts = options(); + let program = compress_program(); + let census = lfm_chip_census_with_hasher(&program, KIND); + let hash_chip = census + .iter() + .find(|c| c.name == "LFM_HASH") + .expect("LFM_HASH is in the census"); + assert_eq!(hash_chip.main_cols, predicted_main(SOCKET_ROUNDS)); + assert_eq!( + hash_chip.aux_cols, + predicted_interactions(SOCKET_ROUNDS).div_ceil(2) + ); + assert_eq!( + hash_chip.main_cols + 3 * hash_chip.aux_cols, + predicted_cells(SOCKET_ROUNDS), + "base-field-equivalent cells per compression row" + ); + let _ = opts; +} + +// ========================================================================= +// The framing — SOCKET.md §2, and the controls that make it discriminating +// ========================================================================= + +/// Every framing degree of freedom, in one object, so a negative control can +/// break exactly one at a time and nothing else. +#[derive(Clone, Copy)] +struct Framing { + rounds: usize, + cv: [u32; 8], + tag_word: u32, + counter: u64, + block_len: u32, + flags: u32, + a_slot: usize, + b_slot: usize, + tag_slot: usize, + out_window: usize, + lane_le: bool, + msg_permutation: [usize; 16], +} + +const HONEST: Framing = Framing { + rounds: SOCKET_ROUNDS, + cv: BLAKE3_IV, + tag_word: TAG_LFMC, + counter: COUNTER_LFMC, + block_len: BLOCK_LEN_LFMC, + flags: FLAGS_LFMC, + a_slot: 0, + b_slot: 4, + // Straight after the twelve lanes, not at a fixed 8: the tag is the last + // word of the message under every lane count, which is what keeps the byte + // string `LE32(lanes) ‖ tag` (COMMIT.md §1.2). + tag_slot: 12, + out_window: 0, + lane_le: true, + msg_permutation: BLAKE3_MSG_PERMUTATION, +}; + +/// The message words under a framing. A big-endian lane serialisation changes +/// the message WORDS, because a word is read little-endian from the bytes. +fn framed_message(a: &[u32; 4], b: &[u32; 4], fr: Framing) -> [u32; 16] { + let lane = |v: u32| if fr.lane_le { v } else { v.swap_bytes() }; + let mut m = [0u32; 16]; + for i in 0..4 { + m[fr.a_slot + i] = lane(a[i]); + m[fr.b_slot + i] = lane(b[i]); + } + m[fr.tag_slot] = fr.tag_word; + m +} + +/// A deliberately *parameterised* socket compress, used only to build negative +/// controls: the same dataflow with [`Framing`] as an input. +/// +/// It is NOT what `socket_digest_rounds` calls. Keeping the two apart costs a +/// duplicated loop and buys the thing the controls are for — they compare +/// against [`SOCKET_VECTORS`], constants that came from outside this file, so +/// they stay meaningful no matter how the real function is later refactored. +fn framed_digest(a: &[u32; 4], b: &[u32; 4], fr: Framing) -> [u32; 4] { + let g = |s: &mut [u32; 16], ia: usize, ib: usize, ic: usize, id: usize, mx: u32, my: u32| { + s[ia] = s[ia].wrapping_add(s[ib]).wrapping_add(mx); + s[id] = (s[id] ^ s[ia]).rotate_right(16); + s[ic] = s[ic].wrapping_add(s[id]); + s[ib] = (s[ib] ^ s[ic]).rotate_right(12); + s[ia] = s[ia].wrapping_add(s[ib]).wrapping_add(my); + s[id] = (s[id] ^ s[ia]).rotate_right(8); + s[ic] = s[ic].wrapping_add(s[id]); + s[ib] = (s[ib] ^ s[ic]).rotate_right(7); + }; + let mut v: [u32; 16] = [ + fr.cv[0], + fr.cv[1], + fr.cv[2], + fr.cv[3], + fr.cv[4], + fr.cv[5], + fr.cv[6], + fr.cv[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + fr.counter as u32, + (fr.counter >> 32) as u32, + fr.block_len, + fr.flags, + ]; + let mut m = framed_message(a, b, fr); + for r in 0..fr.rounds { + g(&mut v, 0, 4, 8, 12, m[0], m[1]); + g(&mut v, 1, 5, 9, 13, m[2], m[3]); + g(&mut v, 2, 6, 10, 14, m[4], m[5]); + g(&mut v, 3, 7, 11, 15, m[6], m[7]); + g(&mut v, 0, 5, 10, 15, m[8], m[9]); + g(&mut v, 1, 6, 11, 12, m[10], m[11]); + g(&mut v, 2, 7, 8, 13, m[12], m[13]); + g(&mut v, 3, 4, 9, 14, m[14], m[15]); + if r < fr.rounds - 1 { + let prev = m; + for (i, &p) in fr.msg_permutation.iter().enumerate() { + m[i] = prev[p]; + } + } + } + let w = fr.out_window; + core::array::from_fn(|i| v[w + i] ^ v[w + i + 8]) +} + +/// Everything `f` actually sees under a framing: the initial state, the message +/// schedule at *every* round, and the output window. +/// +/// Two framings with equal traces compute equal digests *necessarily*, so a +/// control whose trace equals the honest one on some input is genuinely +/// INAPPLICABLE there rather than undetected — which is what lets the control +/// suite assert "changes the digest" unconditionally everywhere else. Deriving +/// applicability this way rather than hand-listing it is deliberate: a +/// hand-list goes stale as controls are added, and a stale entry is a control +/// that looks covered and is not. +/// +/// The schedules, not the permutation, are what belong here. `a_one` is the +/// case that proves it: its message has `m[2] = m[6] = 0`, so transposing the +/// first two entries of the permutation produces the identical schedule and the +/// control cannot possibly fire. +fn effective_trace( + a: &[u32; 4], + b: &[u32; 4], + fr: Framing, +) -> (usize, [u32; 8], u64, u32, u32, usize, Vec<[u32; 16]>) { + let mut sched = framed_message(a, b, fr); + let mut scheds = Vec::with_capacity(fr.rounds); + for r in 0..fr.rounds { + scheds.push(sched); + if r < fr.rounds - 1 { + let prev = sched; + for (i, &p) in fr.msg_permutation.iter().enumerate() { + sched[i] = prev[p]; + } + } + } + ( + fr.rounds, + fr.cv, + fr.counter, + fr.block_len, + fr.flags, + fr.out_window, + scheds, + ) +} + +/// ★ **The socket KATs.** The real function reproduces every vector at both +/// round counts. +#[test] +fn the_socket_matches_the_vectors_at_both_round_counts() { + for v in SOCKET_VECTORS.iter() { + assert_eq!( + socket_digest_rounds(&v.a, &v.b, 6), + v.digest_6, + "6-round socket vector {}", + v.name + ); + assert_eq!( + socket_digest_rounds(&v.a, &v.b, 7), + v.digest_7, + "7-round socket vector {}", + v.name + ); + } + // And the compiled-in round count is one of the two, reaching the vectors + // through the entry point the chip and the host actually call. + for v in SOCKET_VECTORS.iter() { + let expected = if SOCKET_ROUNDS == 7 { + v.digest_7 + } else { + v.digest_6 + }; + assert_eq!(socket_digest(&v.a, &v.b), expected, "vector {}", v.name); + } +} + +/// ★ **The external anchor, direct.** At 7 rounds the socket is literally +/// `blake3::hash(a ‖ b ‖ "LFMC")` truncated to 16 bytes — a library call, no +/// oracle, no JSON. +/// +/// This is what SOCKET.md §6 lists as ✗ DEFERRED ("the same equality against +/// the Rust `blake3` crate — needs cargo"). It also re-derives the 52-byte +/// message from the byte-level specification rather than from +/// `socket_message`, so the word-level and byte-level forms are two statements +/// that can disagree. +/// +/// ★ **The anchor is what the leaf RATE had to keep.** The message grew by the +/// third input cell's four zero lanes — 36 bytes to 52 — and 52 is still under +/// 64, so a row is still ONE block and still a plain library call. Carrying the +/// leaf's accumulator in the chaining value `h` instead would have made the row +/// a chunk CONTINUATION and thrown this test away for the same rate. +#[test] +fn seven_rounds_is_blake3_of_the_domain_separated_message() { + for v in SOCKET_VECTORS.iter() { + let mut msg = Vec::with_capacity(52); + for lane in v.a.iter().chain(v.b.iter()) { + msg.extend_from_slice(&lane.to_le_bytes()); + } + // The third input cell: unread by a digest mode, and pinned to zero by + // the unread-`IN` pins, so its four lanes are zero on every honest row. + msg.extend_from_slice(&[0u8; 16]); + msg.extend_from_slice(b"LFMC"); + assert_eq!(msg.len(), 52, "the socket message is one 52-byte block"); + assert!(msg.len() < 64, "and one block is what the anchor needs"); + + let full = blake3::hash(&msg); + let want: [u32; 4] = core::array::from_fn(|i| { + u32::from_le_bytes(full.as_bytes()[4 * i..4 * i + 4].try_into().unwrap()) + }); + assert_eq!( + socket_digest_rounds(&v.a, &v.b, 7), + want, + "7-round socket vector {} must be blake3::hash of its message", + v.name + ); + assert_eq!(want, v.digest_7, "the table itself agrees with the crate"); + } +} + +/// The parameterised control, at canonical parameters, IS the real function — +/// so every control below differs in exactly the one choice it names. +#[test] +fn the_framing_variant_at_canonical_parameters_is_the_socket() { + for v in SOCKET_VECTORS.iter() { + assert_eq!( + framed_digest(&v.a, &v.b, HONEST), + socket_digest(&v.a, &v.b), + "control harness must reproduce the socket at canonical parameters" + ); + } +} + +/// NEGATIVE CONTROL, one per framing degree of freedom. +/// +/// Without this, "the vectors pass" would be evidence only that the vectors are +/// *reachable*, not that they discriminate — and framing is precisely where a +/// correct `f` still gives a wrong hash. Each control must change the digest on +/// every vector where its effective trace differs from the honest one, and must +/// discriminate on at least one vector overall. +#[test] +fn breaking_one_framing_choice_at_a_time_breaks_the_digest() { + let mut transposed = [0usize; 16]; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + transposed[p] = i; + } + let controls: [(&str, Framing); 14] = [ + ( + "swap_a_b", + Framing { + a_slot: 4, + b_slot: 0, + ..HONEST + }, + ), + ( + "tag_changed", + Framing { + tag_word: u32::from_le_bytes(*b"LFMP"), + ..HONEST + }, + ), + ( + "tag_omitted", + Framing { + tag_word: 0, + ..HONEST + }, + ), + ( + // 8 — where the tag sat before the socket widened to twelve lanes, + // so this control is also the discrimination between the two + // framings: a chip that widened the lanes and left the tag behind + // would compute this, and it is a different hash. + "tag_slot_moved", + Framing { + tag_slot: 8, + ..HONEST + }, + ), + ( + "truncate_high_half", + Framing { + out_window: 4, + ..HONEST + }, + ), + ("flags_parent", Framing { flags: 4, ..HONEST }), + ("flags_no_root", Framing { flags: 3, ..HONEST }), + ( + "block_len_64", + Framing { + block_len: 64, + ..HONEST + }, + ), + ( + "block_len_32", + Framing { + block_len: 32, + ..HONEST + }, + ), + ( + "counter_one", + Framing { + counter: 1, + ..HONEST + }, + ), + ( + "cv_zero", + Framing { + cv: [0; 8], + ..HONEST + }, + ), + ( + "lanes_big_endian", + Framing { + lane_le: false, + ..HONEST + }, + ), + ( + "msg_perm_swapped", + Framing { + msg_permutation: { + let mut p = BLAKE3_MSG_PERMUTATION; + p.swap(0, 1); + p + }, + ..HONEST + }, + ), + ( + "other_round_count", + Framing { + rounds: if SOCKET_ROUNDS == 7 { 6 } else { 7 }, + ..HONEST + }, + ), + ]; + + for (what, fr) in controls { + let mut discriminated = 0; + for v in SOCKET_VECTORS.iter() { + let honest = socket_digest(&v.a, &v.b); + if effective_trace(&v.a, &v.b, fr) == effective_trace(&v.a, &v.b, HONEST) { + // Provably inapplicable on this input — `swap_a_b` when a == b, + // `lanes_big_endian` when every lane is a byte-palindrome, + // `msg_perm_swapped` when the two transposed slots hold equal + // words. Asserted as an equality, not skipped: an inapplicable + // control must produce the SAME digest, which is a check in its + // own right on the applicability derivation. + assert_eq!(framed_digest(&v.a, &v.b, fr), honest); + continue; + } + assert_ne!( + framed_digest(&v.a, &v.b, fr), + honest, + "{what} still reproduces the digest on vector {} — the vectors do not pin it", + v.name + ); + discriminated += 1; + } + assert!( + discriminated > 0, + "{what} is discriminated by no vector at all" + ); + } +} + +/// The transposed message permutation is a real permutation and a different +/// one — otherwise `msg_perm_swapped` above would be testing nothing. +#[test] +fn the_message_permutation_control_is_a_different_permutation() { + let mut p = BLAKE3_MSG_PERMUTATION; + p.swap(0, 1); + assert_ne!(p, BLAKE3_MSG_PERMUTATION); + let mut sorted = p; + sorted.sort_unstable(); + assert_eq!(sorted, core::array::from_fn::(|i| i)); +} + +// ========================================================================= +// The lane boundary — obligation O1, host side +// ========================================================================= + +/// ★ **O1, host side.** An out-of-range lane is REJECTED, never reduced. +/// +/// `edsl::merkle_walk` feeds `compress` arena-hinted — prover-chosen — sibling +/// cells, and a lane is a Goldilocks felt over `[0, p)`. The chip can only +/// commit a byte decomposition for a lane below `2^32`, so a host that reduced +/// instead of rejecting would claim a digest no proof can produce. +#[test] +fn an_out_of_range_lane_is_rejected_rather_than_reduced() { + let ok: LfmWord = word_of(&[1, 2, 3, 4]); + assert_eq!(lanes_of(&ok), Some([1, 2, 3, 4])); + + // The alias that would exist under silent reduction. + let aliased: LfmWord = [ + FE::from(1u64 + (1u64 << 32)), + FE::from(2u64), + FE::from(3u64), + FE::from(4u64), + ]; + assert_eq!(lanes_of(&aliased), None, "2^32 + 1 is not a u32 lane"); + + let mut state = [FE::zero(); HASH_STATE_FELTS]; + state[0..4].copy_from_slice(&aliased); + state[4..8].copy_from_slice(&ok); + assert!( + Blake3Permutation + .admits(HashMode::Compress, &state) + .is_err(), + "a non-u32 lane must be refused by admits" + ); + + // HONEST CONTROL: the in-range pair is still accepted. Without it, this + // test would pass equally if `admits` rejected everything. + let mut good = [FE::zero(); HASH_STATE_FELTS]; + good[0..4].copy_from_slice(&ok); + good[4..8].copy_from_slice(&ok); + assert!(Blake3Permutation.admits(HashMode::Compress, &good).is_ok()); +} + +/// The whole-machine version of the same thing: an arena word with a non-`u32` +/// lane makes the program fail to execute, with a reason. +#[test] +fn a_non_u32_arena_word_fails_execution_under_blake3() { + let program = compress_program(); + let mut bad = arenas(); + bad[0][0][0] = FE::from(1u64 << 32); + assert!( + matches!( + execute(&program, &bad, &KIND), + Err(LfmExecError::HasherRejected(_)) + ), + "a non-u32 hinted lane must be rejected at execution" + ); + // HONEST CONTROL. + assert!(execute(&program, &arenas(), &KIND).is_ok()); +} + +/// Obligation O2: the socket is closed on its own output, so a digest fed back +/// in as a sibling always satisfies O1. That is why only leaf digests and +/// prover-hinted siblings need the input check. +#[test] +fn the_socket_output_is_always_a_valid_input() { + for v in SOCKET_VECTORS.iter() { + let d = socket_digest(&v.a, &v.b); + assert_eq!( + lanes_of(&word_of(&d)), + Some(d), + "a socket digest must round-trip as a u32-lane cell" + ); + } +} + +/// Obligation O3: the IV enters through `h`, not through the capacity lanes, so +/// this arm overrides `compress` rather than inheriting permute-and-truncate. +/// +/// Asserted through `HasherKind`'s dispatch, because that is the path the +/// executor takes and a candidate whose override was not honoured there would +/// prove one thing and record another. +#[test] +fn compress_is_overridden_and_the_upper_out_lanes_are_empty() { + let a = word_of(&[0x0102_0304, 0, 0, 0]); + let b = word_of(&[0, 0, 0, 0x0506_0708]); + let via_kind = LfmHasher::compress(&KIND, &a, &b); + assert_eq!( + via_kind, + word_of(&socket_digest( + &[0x0102_0304, 0, 0, 0], + &[0, 0, 0, 0x0506_0708] + )) + ); + + let out = LfmHasher::compress_out(&KIND, &a, &b); + assert_eq!(&out[0..4], &via_kind[..]); + for (j, felt) in out.iter().enumerate().skip(4) { + assert_eq!( + *felt, + FE::zero(), + "OUT lane {j} carries nothing on a compress row" + ); + } + + // `compress_iv` is meaningful if read, and is NOT what the framing uses. + assert_eq!( + LfmHasher::compress_iv(&KIND), + word_of(&[BLAKE3_IV[0], BLAKE3_IV[1], BLAKE3_IV[2], BLAKE3_IV[3]]) + ); +} + +/// ✗ There is no permute socket, and the refusal is explicit rather than a +/// wrong answer. `permute_coverage_program` contains one, so it is unprovable +/// under BLAKE3 — which is the settled state of option B1, not a defect. +/// +/// The program under test used to be `trivial_program`, which no longer has a +/// permute in it: B1 gave the registry's entries up to the real hash, and this +/// test moved to the unregistered fixture that took over permute coverage. +#[test] +fn a_permute_row_is_refused_under_blake3() { + assert!( + Blake3Permutation + .admits(HashMode::Permute, &[FE::zero(); HASH_STATE_FELTS]) + .is_err() + ); + assert!( + matches!( + execute(&permute_coverage_program(), &permute_arenas(), &KIND), + Err(LfmExecError::HasherRejected(_)) + ), + "a program containing a permute must be refused under BLAKE3" + ); + // HONEST CONTROL: the same program executes fine under the hashers that do + // have a permute socket, so the refusal is BLAKE3's domain and not a break. + assert!( + execute( + &permute_coverage_program(), + &permute_arenas(), + &HasherKind::Test + ) + .is_ok() + ); +} + +/// ★ The F3.4-retirement milestone at the registry level: `TrivialV0` — a +/// REGISTERED program — now executes under BLAKE3, which it could not while it +/// held a raw permute. +/// +/// Its arena has to be `u32`-laned (obligation O1); that is the socket's domain, +/// not a property of this program. +#[test] +fn the_trivial_program_runs_under_blake3_now_that_it_has_no_permute() { + assert!( + !trivial_program().instrs.iter().any( + |i| matches!(i, super::instr::Instr::Hash { mode, .. } if *mode == HashMode::Permute) + ), + "a registered program must not contain a permute" + ); + let arenas = vec![ + (0..4u32) + .map(|i| word_of(&[0x1000_0000 * (i + 1), 0x0BAD_F00D ^ i, i, 0xFFFF_FFFF - i])) + .collect(), + ]; + execute(&trivial_program(), &arenas, &KIND).expect("TrivialV0 executes under BLAKE3"); + // HONEST CONTROL: still fine under the default hasher, on its own arenas. + execute(&trivial_program(), &trivial_arenas(), &HasherKind::Test).expect("and under Test"); +} + +// ========================================================================= +// The constraints +// ========================================================================= + +/// Every constraint index is emitted exactly once, the count is the one the +/// module documents, and the degree really reaches — and does not exceed — 3. +#[test] +fn the_arm_emits_its_constraints_at_degree_3() { + let set = HashConstraints::BLAKE3; + assert_eq!(HashConstraints::num_constraints(KIND), NUM_CONSTRAINTS); + + let meta = ConstraintSet::::meta(&set); + assert_eq!(meta.len(), NUM_CONSTRAINTS, "constraints emitted"); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "meta must be dense and idx-ordered"); + assert_eq!(m.kind, RootKind::Base, "every hash constraint is base"); + } + + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert_eq!(degrees.len(), NUM_CONSTRAINTS, "one emit per constraint"); + + let declared = ConstraintSet::::max_degree(&set); + assert_eq!(declared, 3, "the wrap's blowup 2 depends on this staying 3"); + for &(idx, measured) in °rees { + assert!( + measured <= declared, + "constraint {idx}: measured degree {measured} EXCEEDS declared {declared}" + ); + } + // Not merely `<=`: the mu-gated carry booleanities really are cubic, so a + // set that quietly topped out at 2 would mean the carries had stopped being + // constrained. + assert_eq!(degrees.iter().map(|&(_, d)| d).max(), Some(3)); +} + +/// The mode column a row in `mode` sets. +pub(super) fn mode_col(mode: HashMode) -> usize { + match mode { + HashMode::Compress => cols::MODE_C, + HashMode::Transcript => cols::MODE_T, + HashMode::Leaf => cols::MODE_L, + HashMode::Permute => cols::MODE_P, + } +} + +/// A hash row in `mode`, exactly as `trace::build_traces_with_hasher` fills +/// one. +fn hash_row_mode(mode: HashMode, a: [u32; 4], b: [u32; 4]) -> Vec { + let tag = blake3_socket::tag_for_mode(mode).expect("BLAKE3 has a socket for this mode"); + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[mode_col(mode)] = FE::one(); + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&word_of(&a)); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(&word_of(&b)); + for (k, iv) in BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = blake3_socket::socket_digest_rounds_tagged(&a, &b, SOCKET_ROUNDS, tag); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, tag); + row +} + +/// A `Compress` row — the shape most of these tests are about. +fn hash_row(a: [u32; 4], b: [u32; 4]) -> Vec { + hash_row_mode(HashMode::Compress, a, b) +} + +fn evaluate(row: &[FE]) -> Vec { + let set = HashConstraints::BLAKE3; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out +} + +pub(super) fn violations(row: &[FE]) -> Vec { + evaluate(row) + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +/// An honest row satisfies every constraint, and the digest it carries is the +/// KAT's — so the constraint set and the vectors agree about the same row. +#[test] +fn an_honest_row_satisfies_every_constraint() { + for v in SOCKET_VECTORS.iter() { + let row = hash_row(v.a, v.b); + assert_eq!(violations(&row), Vec::::new(), "vector {}", v.name); + let want = if SOCKET_ROUNDS == 7 { + v.digest_7 + } else { + v.digest_6 + }; + for (i, lane) in want.iter().enumerate() { + assert_eq!(row[cols::OUT0 + i], FE::from(u64::from(*lane))); + for byte in 0..4 { + assert_eq!( + row[cols::out_byte(i, byte)], + FE::from(u64::from((lane >> (8 * byte)) as u8)), + "digest byte ({i}, {byte}) of vector {}", + v.name + ); + } + } + } +} + +/// An all-zero padding row satisfies the set, and a row claiming to be real +/// with no witness does not. +/// +/// The second half is what stops the first from being vacuous: a constraint set +/// that accepted anything would pass the padding check just as well. +#[test] +fn padding_is_satisfied_and_a_real_marked_empty_row_is_not() { + assert_eq!( + violations(&vec![FE::zero(); cols::NUM_COLUMNS]), + Vec::::new(), + "an all-zero padding row must satisfy every constraint" + ); + + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_C] = FE::one(); + assert!( + !violations(&row).is_empty(), + "a real-marked row with an all-zero witness must be rejected" + ); +} + +/// The `MODE_P = 0` pin: a permute-marked row is rejected by the AIR itself, +/// independently of the executor's refusal. +#[test] +fn a_permute_marked_row_violates_the_air() { + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_P] = FE::one(); + assert!( + !violations(&row).is_empty(), + "MODE_P = 1 must be unsatisfiable under the BLAKE3 arm" + ); +} + +/// ★ **The lane-decomposition constraint bites.** Retagging a lane's bytes to a +/// different value, or the lane felt to `v + 2^32`, must violate the AIR. +/// +/// The second case is the identity's own job: a lane moved without its bytes. +/// See `the_lane_range_check_is_load_bearing_on_its_own` for the other half — +/// the witness this identity cannot see, which is what the `AreBytes` sends +/// are for. +#[test] +fn the_lane_decomposition_binds_the_felt_to_its_bytes() { + let base = hash_row([0x1234_5678, 1, 2, 3], [4, 5, 6, 7]); + + let mut tampered = base.clone(); + tampered[cols::lane_byte(0, 0)] += FE::one(); + assert!( + !violations(&tampered).is_empty(), + "moving a lane byte must violate the decomposition" + ); + + let mut aliased = base.clone(); + aliased[cols::IN0] += FE::from(1u64 << 32); + assert!( + !violations(&aliased).is_empty(), + "lane + 2^32 must violate the decomposition — this IS obligation O1" + ); + + // HONEST CONTROL. + assert_eq!(violations(&base), Vec::::new()); +} + +/// ★ **O1's OTHER half — the one the rest of this suite never exercises.** +/// +/// The lane contract is an eval constraint AND two `AreBytes` sends, and the +/// module comment says "NEITHER ALONE SUFFICES". Every other control here +/// breaks the linear identity, which the identity alone catches — so until this +/// test existed, the `AreBytes` half was asserted in prose and exercised +/// nowhere. +/// +/// The witness that separates them moves `2^8` from one byte column into the +/// next (`MB[0] += 256`, `MB[1] -= 1`). The weighted sum is unchanged +/// **exactly**, over the field, with no borrow — `256·(b1 − 1) + (b0 + 256) = +/// 256·b1 + b0` — so the lane identity passes, the message word the mixing core +/// reads is the same linear form and therefore also unchanged, and the honest +/// digest still comes out. Nothing in the eval set is wrong with the row. The +/// only defect is that `MB[0]` is no longer a byte, and only the range check +/// can see that. +/// +/// Recorded because it is not what I expected and it sharpens the argument: +/// **a carry-absorbing witness cannot be silent, because the lane bytes ARE the +/// message bytes.** Trying to alias a lane to `v + 2^32` and letting `MB[3]` +/// absorb the carry does satisfy the lane identity — and then breaks the mixing +/// core instead, because the word the core hashes moved by `2^32` too. So the +/// alias is caught either way; what the range check uniquely buys is the case +/// where the *sum* is preserved. +/// +/// And that case is not a curiosity — it is the whole attack surface. The +/// message words reach `add3` and nothing else (never an XOR), so these sends +/// are their ONLY bound. A sum-preserving witness is exactly the door to an +/// unbounded `m`, and `add3`'s exactness in round 0 — constant `a` and `b`, a +/// byte-bounded `s` — is what an unbounded `m` breaks: the prover solves for +/// any `s` it likes and owns the compression. +#[test] +fn the_lane_range_check_is_load_bearing_on_its_own() { + let base = hash_row([0x1234_5678, 1, 2, 3], [4, 5, 6, 7]); + /// Constraint index of input lane 0's decomposition (idx 6–13 are the eight + /// lanes); `CORE_IDX` is 26, so anything below it is framing. + const LANE0: usize = 6; + + // (a) THE ONE ONLY `AreBytes` CATCHES. Identity preserved, core preserved, + // eval set entirely silent. If this half ever starts failing, the proof + // below has stopped testing the range check and has become a duplicate of + // `the_lane_decomposition_binds_the_felt_to_its_bytes`. + let mut shifted = base.clone(); + shifted[cols::lane_byte(0, 0)] += FE::from(256u64); + shifted[cols::lane_byte(0, 1)] = shifted[cols::lane_byte(0, 1)] - FE::one(); + assert_eq!( + violations(&shifted), + Vec::::new(), + "the linear identity alone cannot see a byte column carrying 2^8 — \ + which is exactly why the AreBytes sends are not optional" + ); + + // (b) The naive alias: claim `v + 2^32` and leave the bytes alone. The + // IDENTITY catches this one, at lane 0's own index. + let mut naive = base.clone(); + naive[cols::IN0] += FE::from(1u64 << 32); + assert!( + violations(&naive).contains(&LANE0), + "a lane moved without its bytes must violate its own decomposition" + ); + + // (c) The alias with the carry absorbed, which is the interesting one: the + // lane identity is satisfied — `LANE0` is NOT among the violations — and the + // MIXING CORE rejects instead, because `MB[3]` is a message byte and the + // word being hashed moved by 2^32 as well. + let mut absorbed = base.clone(); + absorbed[cols::IN0] += FE::from(1u64 << 32); + absorbed[cols::lane_byte(0, 3)] += FE::from(256u64); + let v = violations(&absorbed); + assert!( + !v.contains(&LANE0), + "absorbing the carry must satisfy the lane identity — otherwise this \ + case is not demonstrating what it claims" + ); + assert!( + v.iter().all(|&i| i >= 26) && !v.is_empty(), + "and the mixing core must reject it instead, got {v:?}" + ); + + // (d) In a real proof, the `AreBytes` send catches (a). Only the byte + // shuffle is used: it leaves `IN0` untouched, so the `LfmMem` receive token + // is unchanged and the rejection can only come from the range check, not + // from a memory-bus mismatch. + assert_not_accepted("a byte column carrying 2^8, identity preserved", |t| { + let b0 = t.main_table.get_row(0)[cols::lane_byte(0, 0)]; + let b1 = t.main_table.get_row(0)[cols::lane_byte(0, 1)]; + t.main_table + .set_fe(0, cols::lane_byte(0, 0), b0 + FE::from(256u64)); + t.main_table + .set_fe(0, cols::lane_byte(0, 1), b1 - FE::one()); + }); +} + +/// The digest recomposition binds `OUT` to the mixing core's output bytes. +#[test] +fn the_digest_recomposition_binds_out_to_the_core() { + let base = hash_row([9, 8, 7, 6], [5, 4, 3, 2]); + let mut tampered = base.clone(); + tampered[cols::OUT0] += FE::one(); + assert!(!violations(&tampered).is_empty()); + + let mut upper = base.clone(); + upper[cols::OUT0 + 4] = FE::one(); + assert!( + !violations(&upper).is_empty(), + "the unused upper OUT lanes are pinned to zero" + ); + assert_eq!(violations(&base), Vec::::new()); +} + +// ========================================================================= +// M1–M7 — the mode-selected tag, PRE-COMMITTED controls +// +// Named in the transcript spec §5.3 before this chip existed, so they are +// inherited obligations rather than tests written to fit what got built. Each +// one is paired with an honest-path assertion: "the bad row is rejected" passes +// just as well when every row is rejected. +// ========================================================================= + +/// **M1 — a transcript row that hashed under the MERKLE tag is rejected.** +/// +/// Spec form: "`m[8]` pinned to `TAG_LFMC` while `MODE_T = 1` — SAT", i.e. in a +/// model where the tag is free, a transcript row can compute the Merkle +/// function. On the real chip the tag is NOT free, so the same statement is a +/// rejection, and that is what is asserted here. +#[test] +fn m1_a_transcript_row_computing_the_merkle_tag_is_rejected() { + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + let mut row = hash_row_mode(HashMode::Transcript, a, b); + // Recompute the whole witness under the WRONG domain, leaving MODE_T set. + let digest = socket_digest(&a, &b); // "LFMC" + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, TAG_LFMC); + assert!( + !violations(&row).is_empty(), + "a MODE_T row carrying the Merkle computation must be rejected" + ); + + // HONEST CONTROL: the same row under its own domain satisfies everything. + assert_eq!( + violations(&hash_row_mode(HashMode::Transcript, a, b)), + Vec::::new() + ); +} + +/// **M2 — the mirror: a compress row that hashed under the TRANSCRIPT tag is +/// rejected.** Both directions, because a one-directional separation is not one. +#[test] +fn m2_a_compress_row_computing_the_transcript_tag_is_rejected() { + let (a, b) = ([1u32, 2, 3, 4], [5u32, 6, 7, 8]); + let mut row = hash_row_mode(HashMode::Compress, a, b); + let digest = blake3_socket::transcript_digest(&a, &b); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, blake3_socket::TAG_LFMT); + assert!( + !violations(&row).is_empty(), + "a MODE_C row carrying the transcript computation must be rejected" + ); + + assert_eq!( + violations(&hash_row_mode(HashMode::Compress, a, b)), + Vec::::new() + ); +} + +/// **M3 — both mode bits set on one row is unsatisfiable**, and it is the +/// mode-sum booleanity (idx 4) that says so. +/// +/// This is the constraint that stops `m[8]` being `TAG_LFMC + TAG_LFMT`, a tag +/// in neither domain. +#[test] +fn m3_both_two_to_one_modes_on_one_row_is_unsatisfiable() { + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + let mut row = hash_row_mode(HashMode::Compress, a, b); + row[cols::MODE_T] = FE::one(); + assert!( + violations(&row).contains(&4), + "idx 4 — the mode-sum booleanity — must be the constraint that fires" + ); + + // HONEST CONTROL: clearing it again restores an accepted row. + row[cols::MODE_T] = FE::zero(); + assert_eq!(violations(&row), Vec::::new()); +} + +/// **M4 — the mu gate IS the sum of the two two-to-one selectors**, so it +/// cannot be 1 while both are 0. +/// +/// Structural rather than algebraic: on this chip `MU` is not a column a row +/// could set independently, it is the expression `MODE_C + MODE_T`. The test +/// pins that, and pins the consequence — with both zero the row is padding, it +/// satisfies the set vacuously and its bus sends carry multiplicity zero. +#[test] +fn m4_the_mu_gate_is_exactly_the_two_to_one_selector_sum() { + assert_eq!(cols::MU_COLUMNS, [cols::MODE_C, cols::MODE_T, cols::MODE_L]); + + // A row with garbage in every witness column but no mode set is padding. + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + let mut row = hash_row_mode(HashMode::Compress, a, b); + row[cols::MODE_C] = FE::zero(); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&[0, 0, 0, 0])); + for k in 0..4 { + row[cols::S8 + k] = FE::zero(); + } + assert_eq!( + violations(&row), + Vec::::new(), + "with no mode set the row is padding and every mu-gated constraint is vacuous" + ); + + // HONEST CONTROL: it is vacuous because it is UNGATED-satisfiable, not + // because the set accepts anything — restoring the mode makes the same + // garbage row fail. + row[cols::MODE_C] = FE::one(); + assert!(!violations(&row).is_empty()); +} + +/// **M5/M6 — ⚠ the AIR alone does not pin the tag; the PREPROCESSED binding +/// does.** This is the control that turns §3.3 from an assertion into a checked +/// claim, and it fires. +/// +/// Constraint idx 4 pins the mode SUM to a bit, not each selector to a bit. So +/// a row with `MODE_C = x`, `MODE_T = 1 − x` satisfies it for every field +/// element `x`, and `m[8]` becomes `x·"LFMC" + (1−x)·"LFMT"` — which, solving +/// for `x`, is **any 32-bit value the prover likes**. This test picks the tag +/// `"XXXX"`, derives the `x` that produces it, and shows the constraint set +/// ACCEPTS the resulting row. +/// +/// Two mechanisms stop a real prover doing this, and neither is in this file's +/// constraint set: +/// +/// - the mode columns are **preprocessed**, fixed by the row's position in a +/// trace whose commitment is folded into `lfm_program_id`; and +/// - the admission validator's one-hot check rejects any program whose +/// `LFM_HASH` group carries a non-boolean selector. +/// +/// Both are asserted below, so this is a live demonstration of *why* they are +/// load-bearing rather than a latent hole. +#[test] +fn m5_m6_the_mode_columns_must_be_preprocessed_or_the_tag_is_prover_chosen() { + const FORGED_TAG: u32 = u32::from_le_bytes(*b"XXXX"); + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + + // x such that x·LFMC + (1−x)·LFMT = FORGED_TAG. + let lfmc = FE::from(u64::from(TAG_LFMC)); + let lfmt = FE::from(u64::from(blake3_socket::TAG_LFMT)); + let x = (FE::from(u64::from(FORGED_TAG)) - &lfmt) + * (&lfmc - &lfmt).inv().expect("the two tags differ"); + + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_C] = x; + row[cols::MODE_T] = FE::one() - x; + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&word_of(&a)); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(&word_of(&b)); + for (k, iv) in BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = blake3_socket::socket_digest_rounds_tagged(&a, &b, SOCKET_ROUNDS, FORGED_TAG); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, FORGED_TAG); + + assert_eq!( + violations(&row), + Vec::::new(), + "⚠ the constraint set alone accepts a prover-chosen domain tag — the \ + mode columns being preprocessed is what stops this" + ); + assert_ne!( + digest, + socket_digest(&a, &b), + "the forged domain really is a different function" + ); + + // MECHANISM 1: the mode columns are inside the preprocessed prefix, so a + // prover supplies neither. + const { assert!(cols::MODE_C < cols::PREP_WIDTH) }; + const { assert!(cols::MODE_T < cols::PREP_WIDTH) }; + const { assert!(cols::MODE_P < cols::PREP_WIDTH) }; + + // MECHANISM 2: the admission validator rejects a non-one-hot selector, so + // the program above cannot be registered even if a prover could write it. + let mut program = compress_program(); + let g = &mut program.groups.hash; + let row0 = 0; + g.data[row0 * g.width + super::layout::hash::MODE_C] = x; + g.data[row0 * g.width + super::layout::hash::MODE_T] = FE::one() - x; + assert!( + matches!( + super::validator::validate(&program), + Err(super::validator::LfmViolation::NonOneHotSelector { + chip: "LFM_HASH", + .. + }) + ), + "the registrar must reject a fractional mode selector" + ); + + // HONEST CONTROL: the untouched program is admissible, so the rejection + // above is about the tampering and not about the program. + assert!(super::validator::validate(&compress_program()).is_ok()); +} + +/// **M7 — the capacity constraints (idx 0–3) bite on a transcript row.** +/// +/// A transcript row is still a compress, so its capacity prefix is still the +/// IV — the selector widened to `MODE_C + MODE_T` and nothing else did. If the +/// widening had been forgotten, a transcript row's `S` would be pinned to zero +/// instead of to the IV, and this is the test that would have said so. +#[test] +fn m7_the_capacity_constraints_bite_on_a_transcript_row() { + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + for k in 0..4 { + let mut row = hash_row_mode(HashMode::Transcript, a, b); + row[cols::S8 + k] += FE::one(); + assert_eq!( + violations(&row), + vec![k], + "a wrong capacity lane must violate exactly constraint {k}" + ); + } + + // A transcript row's capacity is the IV, same as a compress row's — the two + // differ in `m[8]` and in nothing else. + let transcript = hash_row_mode(HashMode::Transcript, a, b); + let compress = hash_row_mode(HashMode::Compress, a, b); + for k in 0..4 { + assert_eq!(transcript[cols::S8 + k], compress[cols::S8 + k]); + assert_eq!(transcript[cols::S8 + k], FE::from(u64::from(BLAKE3_IV[k]))); + } + assert_eq!(violations(&transcript), Vec::::new()); +} + +/// The degree bound survives the mode-selected tag: `m[8]` went from degree 0 +/// to degree 1, and the wrap's blowup 2 depends on the maximum staying 3. +#[test] +fn the_mode_selected_tag_does_not_raise_the_degree() { + let set = HashConstraints::BLAKE3; + assert_eq!(ConstraintSet::::max_degree(&set), 3); + // Measured, not declared — `the_declared_degree_bound_is_respected` walks + // the captured IR; this asserts the declaration it checks against. +} + +// ========================================================================= +// Prove and verify — rule 2: this is what makes the numbers measurements +// ========================================================================= + +/// A compress-only program: two leaf merges and a parent merge. +/// +/// This is the shape the socket exists for — `edsl::merkle_walk`'s parent +/// compression — and it exercises obligation O2 as well, since `d0` and `d1` are +/// socket outputs fed straight back in as inputs. `trivial_program` cannot be +/// used: it contains a `permute`, which BLAKE3 has no socket for. +fn compress_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(4); + let h: Vec = (0..4).map(|i| b.hint_word(arena, i)).collect(); + let d0 = b.compress(h[0].as_digest(), h[1].as_digest()); + let d1 = b.compress(h[2].as_digest(), h[3].as_digest()); + let root = b.compress(d0, d1); + b.public(root.as_cell()); + b.finish() +} + +fn compress_program() -> LfmProgram { + compile(compress_program_source()) +} + +/// Four arena words whose lanes are `u32`s — the socket's domain (O1). +fn arenas() -> Vec> { + vec![ + (0..4u32) + .map(|i| word_of(&[0x1000_0000 * (i + 1), 0x0BAD_F00D ^ i, i, 0xFFFF_FFFF - i])) + .collect(), + ] +} + +/// `permute_coverage_program`'s arenas — three state cells. +fn permute_arenas() -> Vec> { + vec![ + (0..3u64) + .map(|i| core::array::from_fn(|j| FE::from(500 * (i + 1) + j as u64))) + .collect(), + ] +} + +/// `trivial_program`'s arenas — arbitrary felts, which is exactly why BLAKE3 +/// cannot take them. +fn trivial_arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +/// ★ The production prover builds this AIR, proves a program through it, and +/// the production verifier accepts. +#[test] +fn the_blake3_socket_proves_and_verifies() { + let opts = options(); + let program = compress_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &arenas(), &opts, KIND) + .expect("proving under BLAKE3 must succeed"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "an honest BLAKE3-configured proof must verify" + ); + + // The public output is the Merkle root the socket computed, recomputed here + // from the vectors' own reference function — so the proof's public words + // are checked against the specification, not against the executor. + let a = arenas(); + let lanes = |i: usize| lanes_of(&a[0][i]).expect("u32 lanes"); + let d0 = socket_digest(&lanes(0), &lanes(1)); + let d1 = socket_digest(&lanes(2), &lanes(3)); + let root = socket_digest(&d0, &d1); + assert_eq!(proved.public_words, vec![(0u32, word_of(&root))]); +} + +/// A proof is bound to the hasher it was produced under, in both directions. +#[test] +fn a_blake3_proof_does_not_verify_under_another_hasher() { + let opts = options(); + let program = compress_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &arenas(), &opts, KIND).expect("prove"); + + for other in [HasherKind::Test, HasherKind::Rpx] { + // The digest stays the proved-under one: this isolates the AIR-set + // mismatch rather than passing because the statement also moved. + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + other, + artifacts.chip_set, + ), + "a BLAKE3 proof must not verify under {other:?}" + ); + } +} + +/// The hasher tag moves the program digest and no preprocessed root — the +/// Phase-3 binding, now with a third candidate in it. +/// +/// The third candidate is the point: with only two, a width coincidence was +/// enough to separate them by accident. The tag is what separates them on +/// purpose. +#[test] +fn the_blake3_choice_moves_the_program_digest_and_no_root() { + let opts = options(); + let program = compress_program(); + let test = build_artifacts_with_hasher(&program, &opts, HasherKind::Test); + let blake = build_artifacts_with_hasher(&program, &opts, KIND); + let rpx = build_artifacts_with_hasher(&program, &opts, HasherKind::Rpx); + + assert_eq!(build_artifacts(&program, &opts).program_id, test.program_id); + assert_eq!(test.roots, blake.roots, "no root may move with the hasher"); + assert_eq!(test.log_heights, blake.log_heights); + assert_eq!(test.keccak_rnd_chunks, blake.keccak_rnd_chunks); + assert_ne!(test.program_id, blake.program_id); + assert_ne!(rpx.program_id, blake.program_id); + assert_eq!(KIND.as_tag(), 2, "the wire tag is written out, not derived"); +} + +/// Prove the program with `mutate` applied to the hash trace, and report +/// whether the proof was ACCEPTED. A prover refusal and a verifier rejection +/// are both real rejections and this chip produces both. +fn round_trip(mutate: impl FnOnce(&mut TraceTable)) -> Result { + let opts = options(); + let program = compress_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let exec = execute(&program, &arenas(), &KIND).expect("execute"); + let mut traces = build_traces_with_hasher(&program, &exec.records, KIND); + mutate(&mut traces.hash); + match prove_traces_with_hasher( + &artifacts, + &mut traces, + &exec.public_words, + &opts, + KIND, + stark::residency_mode::ResidencyMode::Retain, + ) { + Ok(proof) => Ok(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + KIND, + artifacts.chip_set, + )), + Err(e) => Err(format!("{e:?}")), + } +} + +fn assert_not_accepted(what: &str, mutate: impl FnOnce(&mut TraceTable)) { + if let Ok(true) = round_trip(mutate) { + panic!("{what} must not produce an accepted proof, but the proof verified"); + } +} + +/// Tamper rejection, one cell at a time, across the three column families the +/// arm adds: a lane byte, a mixing-core carry, and a digest byte. +/// +/// The honest control is `the_blake3_socket_proves_and_verifies` above: without +/// it these would pass just as well if the AIR rejected everything. +#[test] +fn tampering_with_the_witness_is_not_accepted() { + assert_not_accepted("a moved input lane byte", |t| { + let v = t.main_table.get_row(0)[cols::lane_byte(0, 0)]; + t.main_table.set_fe(0, cols::lane_byte(0, 0), v + FE::one()); + }); + assert_not_accepted("a flipped add3 carry bit", |t| { + let c = cols::g_base(0) + cols::G_A1_C; + let v = t.main_table.get_row(0)[c]; + t.main_table.set_fe(0, c, v + FE::one()); + }); + assert_not_accepted("a moved digest byte", |t| { + let v = t.main_table.get_row(0)[cols::out_byte(0, 0)]; + t.main_table.set_fe(0, cols::out_byte(0, 0), v + FE::one()); + }); + assert_not_accepted("a real flag on a padding row", |t| { + t.main_table.set_fe(3, cols::MODE_C, FE::one()); + }); +} + +// ========================================================================= +// F3.4 — what B1 retired, and the ONE thing it did not +// ========================================================================= + +/// ★ `TrivialV0` — a REGISTERED program — proves and verifies under BLAKE3. +/// +/// It could not while it ended on a raw `permute`; option B1 replaced that with +/// a third `compress`, and this is the milestone that states. +#[test] +fn the_trivial_program_proves_and_verifies_under_blake3() { + let opts = options(); + let program = trivial_program(); + let arenas = vec![ + (0..4u32) + .map(|i| word_of(&[0x1000_0000 * (i + 1), 0x0BAD_F00D ^ i, i, 0xFFFF_FFFF - i])) + .collect(), + ]; + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &arenas, &opts, KIND) + .expect("TrivialV0 must prove under BLAKE3"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "an honest BLAKE3 proof of TrivialV0 must verify" + ); +} + +// The O1 tripwire that used to live here is GONE, and that is the deliverable. +// +// It asserted that `FriToyV0` was refused under BLAKE3 for obligation O1, and +// its own doc said it must be replaced by a prove+verify when O1 closed. Option +// C closed it: `leaf_tests::fri_toy_proves_and_verifies_under_blake3` is the +// replacement, and it carries the negative leg the tripwire's criteria asked +// for. + +/// ★ H1 guard — every `LFM_HASH` candidate emits each constraint index exactly +/// once, checked in RELEASE. +/// +/// `EmitTracker`'s duplicate assert is `#[cfg(debug_assertions)]` and this +/// workspace declares no `[profile.release]` override, so under the house +/// convention `cargo test --release` it is a no-op: a second +/// `emit_base(idx, ..)` silently overwrites the first. Nothing else notices, +/// because a body that emits one index twice and another never still fills the +/// declared number of slots — `num_constraints`, `predicted_constraints` and +/// `assert_complete` all still pass while a constraint has been deleted. +/// +/// This runs the real body through `ConstraintSet::meta` (no `cfg`) and demands +/// the emitted index multiset be exactly `0..num_constraints`. The checker's own +/// ability to fail — on this exact shape, a widened lane block overrunning the +/// pins after it — is established in +/// `stark::tests::constraint_index_tests::the_widened_lane_block_collides_and_the_checker_says_so`. +/// +/// Required by COMMIT.md §1.4.4 H1. It guards the chip as it stands today, and +/// it is what would catch the `NUM_LANES` widening if that lands before the +/// framing indices stop being written as literals. +#[test] +fn every_hash_candidate_emits_each_constraint_index_exactly_once() { + for (set, kind) in [ + (HashConstraints::TEST, HasherKind::Test), + (HashConstraints::RPX, HasherKind::Rpx), + (HashConstraints::BLAKE3, HasherKind::Blake3), + ] { + let declared = HashConstraints::num_constraints(kind); + let meta = >::meta(&set); + check_dense_index_set(&meta, declared) + .unwrap_or_else(|e| panic!("{kind:?} LFM_HASH constraint body: {e}")); + } +} diff --git a/prover/src/lfm/block_node.rs b/prover/src/lfm/block_node.rs new file mode 100644 index 000000000..ad9b263f2 --- /dev/null +++ b/prover/src/lfm/block_node.rs @@ -0,0 +1,290 @@ +//! A no-epoch BLOCK NODE — verifies block leaves (or other block nodes) and +//! binds them into one statement about one block; at the top of the tree it +//! closes the block's bus. +//! +//! # The checks (D-NOEPOCH §12.3) +//! +//! Each child is verified exactly as an epoch node verifies its children +//! ([`super::per_table_aggregator::emit_leg`]: the child's LFM statement over its +//! claimed words, its `program_id` absorbed as an emit-time constant, every +//! sub-proof verified). Then, on the children's published words: +//! +//! 1. **One challenge set** — the transcript state digest equal across children, +//! so every leaf forked every instance from the same statement and the same +//! roots (and drew the same `z, α`). +//! 2. **The same block** — the attestation id and the public output halves equal +//! across children. +//! 3. **The bus** — the children's sums added; a non-top node publishes the +//! total, the TOP node asserts it is zero: Σ over every instance of its bus +//! contribution equals the COMMIT-bus target, which exactly one leaf +//! subtracted (the plan's carrier, `block_plan::BlockTreePlan::carrier`). +//! This is the monolithic verifier's bus check (`verifier.rs`, "Σ +//! table_contribution = expected_bus_balance"), moved to where every +//! contribution is in scope. +//! +//! Coverage is not a check here: the children's ids are constants of this +//! program, a leaf's id commits to its instance list, and the lists were +//! validated when the leaves were emitted (`block_leaf::BlockPartition`). + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, LfmBuilder}; +use super::per_table_aggregator::{ + ChildShape, HintedPublicWord, LegArenas, LegCells, assert_words_equal, declare_leg_arenas, + emit_leg, +}; + +/// Where each field sits in a block leaf's or block node's published words. +/// +/// `[id₀, id₁, state…, out halves…, sum]` — a leaf and a non-top node publish +/// the same layout, so a node reads either kind of child alike. The top node +/// publishes the block's claim without the sum it has just closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BlockLayout { + /// Words the transcript state digest occupies: one on an algebraic hash, two + /// on a byte hash (`edsl::digest_words`). + pub state_words: usize, + pub out_halves: usize, + /// Whether the last word is the bus sum. + pub has_sum: bool, +} + +impl BlockLayout { + /// What a leaf and a non-top node publish. + pub fn child(state_words: usize, out_halves: usize) -> Self { + Self { + state_words, + out_halves, + has_sum: true, + } + } + + /// What the top node publishes. + pub fn top(state_words: usize, out_halves: usize) -> Self { + Self { + state_words, + out_halves, + has_sum: false, + } + } + + pub fn id(&self, half: usize) -> usize { + half + } + pub fn state(&self, w: usize) -> usize { + 2 + w + } + pub fn out_half(&self, i: usize) -> usize { + 2 + self.state_words + i + } + pub fn sum(&self) -> usize { + assert!(self.has_sum, "the top node publishes no sum"); + 2 + self.state_words + self.out_halves + } + pub fn total(&self) -> usize { + 2 + self.state_words + self.out_halves + usize::from(self.has_sum) + } +} + +/// Which of the cross-child checks a node emits. Production emits all of them +/// ([`BlockBindings::ALL`]) and no production signature takes another set; the +/// weakened sets exist so each negative test can show its check is the one doing +/// the refusing ([`emit_block_node_with`], [`bind_and_publish_with`], test-only). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct BlockBindings { + id: bool, + state: bool, + out: bool, + /// The top node's zero assert on the bus sum. + bus: bool, +} + +impl BlockBindings { + pub(crate) const ALL: Self = Self { + id: true, + state: true, + out: true, + bus: true, + }; + + /// [`Self::ALL`] minus the named check — a mutation, test-only. + #[cfg(test)] + pub(crate) fn without(check: &str) -> Self { + let mut b = Self::ALL; + match check { + "id" => b.id = false, + "state" => b.state = false, + "out" => b.out = false, + "bus" => b.bus = false, + other => panic!("no block binding named `{other}`"), + } + b + } +} + +/// The sum a child published, as an extension cell. Lane 3 of an extension +/// publish is zero by construction; asserted so a word that is not one cannot be +/// summed as if it were. +fn hinted_ext(b: &mut LfmBuilder, w: &HintedPublicWord) -> Ext { + let zero = b.felt_const(FE::from(0u64)); + b.assert_eq(w.lanes[3], zero); + b.pack_ext(w.lanes[0], w.lanes[1], w.lanes[2]) +} + +/// The cross-child checks on verified children's published words, and the sum of +/// their bus shares. +fn emit_block_bindings( + b: &mut LfmBuilder, + legs: &[LegCells], + layout: &BlockLayout, + checks: BlockBindings, +) -> Ext { + assert!(!legs.is_empty(), "a node has children"); + assert!(layout.has_sum, "children publish their bus share"); + let first = &legs[0]; + for leg in &legs[1..] { + if checks.id { + for half in 0..2 { + assert_words_equal( + b, + &first.publics[layout.id(half)], + &leg.publics[layout.id(half)], + ); + } + } + if checks.state { + for w in 0..layout.state_words { + assert_words_equal( + b, + &first.publics[layout.state(w)], + &leg.publics[layout.state(w)], + ); + } + } + if checks.out { + for i in 0..layout.out_halves { + assert_words_equal( + b, + &first.publics[layout.out_half(i)], + &leg.publics[layout.out_half(i)], + ); + } + } + } + let shares: Vec = legs + .iter() + .map(|leg| hinted_ext(b, &leg.publics[layout.sum()])) + .collect(); + let (head, rest) = shares.split_first().expect("nonempty"); + rest.iter().fold(*head, |acc, s| b.eadd(acc, *s)) +} + +/// Everything a block node's program is a function of. +pub struct BlockNodeInputs<'a> { + pub children: &'a [ChildShape<'a>], + /// The children's layout — one, since leaves and non-top nodes share it. + pub layout: BlockLayout, + /// Whether this node closes the bus and publishes the block's claim. + pub top: bool, +} + +/// Declare every child's arenas (in child order), verify every child, bind them, +/// publish. +pub fn emit_block_node(b: &mut LfmBuilder, inputs: &BlockNodeInputs<'_>) { + emit_node(b, inputs, BlockBindings::ALL); +} + +/// [`emit_block_node`] under a weakened binding set — a mutation, test-only. +#[cfg(test)] +pub(crate) fn emit_block_node_with( + b: &mut LfmBuilder, + inputs: &BlockNodeInputs<'_>, + checks: BlockBindings, +) { + emit_node(b, inputs, checks); +} + +fn emit_node(b: &mut LfmBuilder, inputs: &BlockNodeInputs<'_>, checks: BlockBindings) { + let BlockNodeInputs { + children, + layout, + top, + } = *inputs; + assert!(!children.is_empty(), "a node verifies at least one child"); + for child in children { + assert_eq!( + child.num_public_words, + layout.total(), + "every child publishes the block layout" + ); + } + let arenas: Vec = children + .iter() + .map(|child| declare_leg_arenas(b, child)) + .collect(); + let legs: Vec = children + .iter() + .zip(&arenas) + .map(|(child, a)| emit_leg(b, child, a)) + .collect(); + bind(b, &legs, &layout, top, checks); +} + +/// Everything a node does after verifying its children: the cross-child checks, +/// the top node's bus close, and the publishes. +pub fn bind_and_publish(b: &mut LfmBuilder, legs: &[LegCells], layout: &BlockLayout, top: bool) { + bind(b, legs, layout, top, BlockBindings::ALL); +} + +/// [`bind_and_publish`] under a weakened binding set — a mutation, test-only. +#[cfg(test)] +pub(crate) fn bind_and_publish_with( + b: &mut LfmBuilder, + legs: &[LegCells], + layout: &BlockLayout, + top: bool, + checks: BlockBindings, +) { + bind(b, legs, layout, top, checks); +} + +fn bind( + b: &mut LfmBuilder, + legs: &[LegCells], + layout: &BlockLayout, + top: bool, + checks: BlockBindings, +) { + let sum = emit_block_bindings(b, legs, layout, checks); + if top && checks.bus { + let zero = b.ext_const(&FEE::zero()); + b.assert_eq_ext(sum, zero); + } + emit_block_publishes(b, &legs[0], layout, (!top).then_some(sum)); +} + +/// Republish the first child's id, state and output halves in the form a child +/// published them, then the node's sum unless this is the top. +fn emit_block_publishes( + b: &mut LfmBuilder, + first: &LegCells, + layout: &BlockLayout, + sum: Option, +) { + for half in 0..2 { + let lanes = &first.publics[layout.id(half)].lanes; + let word = b.pack_word([lanes[0], lanes[1], lanes[2], lanes[3]]); + b.public(word); + } + for w in 0..layout.state_words { + let lanes = &first.publics[layout.state(w)].lanes; + let word = b.pack_word([lanes[0], lanes[1], lanes[2], lanes[3]]); + b.public(word); + } + for i in 0..layout.out_halves { + b.public(first.publics[layout.out_half(i)].lanes[0].as_cell()); + } + if let Some(s) = sum { + b.public(s.as_cell()); + } +} diff --git a/prover/src/lfm/builder.rs b/prover/src/lfm/builder.rs new file mode 100644 index 000000000..6db86a7c0 --- /dev/null +++ b/prover/src/lfm/builder.rs @@ -0,0 +1,733 @@ +//! The LFM eDSL: typed SSA handles over write-once cells. +//! +//! The builder is an ordinary Rust API that *emits instructions*; host-side +//! `for` loops unroll and nothing loop-shaped reaches the machine. Every +//! emitted destination gets the next dense address (SSA — uniqueness and +//! acyclicity by construction), every operand use bumps that cell's read +//! count, and the compiler later backfills the counts as the static +//! multiplicities the write-once memory argument needs. + +use std::collections::HashMap; + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::edsl::WrapHash; +use super::instr::{Addr, ArenaId, BaseOp, ExtOp, HashMode, Instr, KeccakMode}; +use super::layout; +use super::word::{LfmWord, base_word, ext_word}; + +/// A cell holding a base field value `(v, 0, 0, 0)`. +#[derive(Debug, Clone, Copy)] +pub struct Felt(pub(crate) Addr); +/// A cell holding an Fp3 value `(a0, a1, a2, 0)`. +#[derive(Debug, Clone, Copy)] +pub struct Ext(pub(crate) Addr); +/// A cell holding a digest (all four lanes). +#[derive(Debug, Clone, Copy)] +pub struct DigestVal(pub(crate) Addr); +/// An untyped word cell. +#[derive(Debug, Clone, Copy)] +pub struct Cell(pub(crate) Addr); +/// A cell holding a boolean `(b, 0, 0, 0)`, `b ∈ {0, 1}`. +#[derive(Debug, Clone, Copy)] +pub struct Bit(pub(crate) Addr); + +macro_rules! handle_addr { + ($($t:ty),*) => {$( + impl $t { + /// The underlying cell address. + pub fn addr(&self) -> Addr { self.0 } + /// Erase the type: any handle is a word cell. + pub fn as_cell(&self) -> Cell { Cell(self.0) } + } + )*}; +} +handle_addr!(Felt, Ext, DigestVal, Cell, Bit); + +impl Bit { + /// A bit is a valid base felt. + pub fn as_felt(&self) -> Felt { + Felt(self.0) + } +} + +impl Cell { + /// Reinterpret as a digest cell (e.g. hint words feeding `compress`). + pub fn as_digest(&self) -> DigestVal { + DigestVal(self.0) + } + + /// Reinterpret as an ext value. Sound by construction: every ext-typed + /// bus receive carries a constant zero in lane 3, so a word whose lane 3 + /// is nonzero makes the program unprovable (and the executor errors). + pub fn as_ext(&self) -> Ext { + Ext(self.0) + } +} + +impl Felt { + /// A base cell `(v, 0, 0, 0)` is a valid ext cell `(v, 0, 0)`. + pub fn as_ext(&self) -> Ext { + Ext(self.0) + } +} + +/// Declared arena lengths (in words), fixed at build time. The executor +/// checks supplied arenas against this schema; the admission validator checks +/// every `Hint` lands inside it. +#[derive(Debug, Clone, Default)] +pub struct ArenaSchema { + pub lens: Vec, +} + +/// Everything the compiler needs: the emitted instructions plus the builder's +/// bookkeeping. Fields are public so tests can hand-build malformed sources +/// to exercise the compiler's invariant panics. +#[derive(Debug)] +pub struct LfmProgramSource { + pub instrs: Vec, + pub num_addrs: u64, + /// Reads per address, indexed BY address. + /// + /// Dense rather than a map because [`LfmBuilder::alloc`] hands out + /// addresses sequentially from zero, so the key space is exactly + /// `0..num_addrs` with no holes — a `HashMap` was paying ~16 bytes plus + /// control per entry, and its bucket array is the emitter's second-largest + /// allocation at production query counts. + pub read_counts: Vec, + pub arena_schema: ArenaSchema, + pub public_len: u32, +} + +#[derive(Default)] +pub struct LfmBuilder { + instrs: Vec, + next_addr: u64, + const_pool: HashMap<[u64; 4], Addr>, + /// Parallel to the address space; see [`LfmProgramSource::read_counts`]. + read_counts: Vec, + arena_schema: ArenaSchema, + public_len: u32, + /// The byte hash this program's commitment layer runs. + /// + /// ★ A property of the PROGRAM, held here rather than threaded through + /// every construction, and that placement is the safety argument. The + /// campaign's standing risk is that a missed emitter site keeps hashing + /// keccak after the flip and produces a valid proof of the wrong digest; + /// with the choice on the builder there is no site to miss, because every + /// `edsl::wrap_*` construction reads this one value. What remains visible + /// is the opposite and much smaller list: the emitters that must NOT follow + /// it still name `keccak256` explicitly (`programs::emit_program_id`, the + /// R1c instruments), so a grep for the pinned hash in `lfm/` returns + /// exactly the deliberate exceptions. + wrap_hash: WrapHash, + /// D-HASH stage-0 census geometry (tests only; `None` everywhere else): + /// the arena the width-16 walk's third sibling is hinted from. See + /// [`LfmBuilder::with_p1w16_census`]. + p1w16_census: Option, +} + +impl LfmBuilder { + pub fn new() -> Self { + Self::default() + } + + /// Build under an explicitly chosen wrap hash. + /// + /// Program-level and set once, before anything is emitted: the hash decides + /// what the program's Merkle and transcript constructions compute, so a + /// program half-emitted under one and half under the other is not a + /// program. It is taken by value for that reason — there is no setter that + /// can run mid-emission. + pub fn with_wrap_hash(mut self, wrap_hash: WrapHash) -> Self { + assert!( + self.instrs.is_empty(), + "the wrap hash is program shape and must be chosen before emission" + ); + self.wrap_hash = wrap_hash; + self + } + + /// The byte hash this program's commitment layer runs. + pub fn wrap_hash(&self) -> WrapHash { + self.wrap_hash + } + + /// The instructions emitted so far, in program order: what a part of the + /// program spends by chip is the slice it emitted. + pub fn instrs(&self) -> &[Instr] { + &self.instrs + } + + /// ⚠ CENSUS ONLY — the emitted program is NOT executable. Emit the + /// algebraic constructions in a Poseidon1 width-16 geometry (rate-12 + /// leaves and transcript, 4-ary Merkle walks and cap roots) so the + /// program's per-chip rows are the ones that geometry would cost + /// (D-HASH stage 0). The socket is still 12 felts wide, so each width-16 + /// hash is emitted as ONE stand-in `permute` row over three of its cells: + /// the row counts are exact, the values are not. Test-only by + /// construction: nothing outside `cfg(test)` can set it. + #[cfg(test)] + pub fn with_p1w16_census(mut self) -> Self { + assert!( + self.instrs.is_empty(), + "the census geometry is program shape and must be chosen before emission" + ); + let arena = self.declare_arena(1); + self.p1w16_census = Some(arena); + self + } + + /// The census geometry's sibling arena, when the census geometry is on. + pub fn p1w16_census(&self) -> Option { + self.p1w16_census + } + + fn alloc(&mut self) -> Addr { + let addr = Addr(self.next_addr); + self.next_addr += 1; + // Keeps `read_counts` exactly as long as the address space, which is + // what lets `read` index instead of hash. + self.read_counts.push(0); + addr + } + + fn read(&mut self, addr: Addr) { + self.read_counts[addr.0 as usize] += 1; + } + + // ---- constants (interned; one LFM_CONST row per distinct word) ---- + + fn word_const(&mut self, value: LfmWord) -> Addr { + let key: [u64; 4] = core::array::from_fn(|i| GoldilocksField::canonical(value[i].value())); + if let Some(&addr) = self.const_pool.get(&key) { + return addr; + } + let out = self.alloc(); + self.instrs.push(Instr::Const { + out, + value, + mult: 0, + }); + self.const_pool.insert(key, out); + out + } + + pub fn felt_const(&mut self, v: FE) -> Felt { + Felt(self.word_const(base_word(v))) + } + + pub fn ext_const(&mut self, v: &FEE) -> Ext { + Ext(self.word_const(ext_word(v))) + } + + pub fn digest_const(&mut self, v: LfmWord) -> DigestVal { + DigestVal(self.word_const(v)) + } + + pub fn bit_const(&mut self, b: bool) -> Bit { + Bit(self.word_const(base_word(if b { FE::one() } else { FE::zero() }))) + } + + // ---- base ALU ---- + + fn balu(&mut self, op: BaseOp, a: Felt, b: Felt, c: Option) -> Felt { + self.read(a.0); + self.read(b.0); + if let Some(c) = c { + self.read(c.0); + } + let out = self.alloc(); + self.instrs.push(Instr::BaseAlu { + op, + out, + a: a.0, + b: b.0, + c: c.map_or(Addr(0), |c| c.0), + mult: 0, + }); + Felt(out) + } + + pub fn add(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Add, a, b, None) + } + pub fn sub(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Sub, a, b, None) + } + pub fn mul(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Mul, a, b, None) + } + /// `a / b` under the machine convention `0/0 = 1`, `x/0 = error`. + pub fn div(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Div, a, b, None) + } + /// `a·b + c` — the Horner step, first-class. + pub fn mul_add(&mut self, a: Felt, b: Felt, c: Felt) -> Felt { + self.balu(BaseOp::MulAdd, a, b, Some(c)) + } + + // ---- Fp3 ALU (lanes 0–2, w³ = 2) ---- + + fn xalu(&mut self, op: ExtOp, a: Addr, b: Addr, c: Option) -> Ext { + self.read(a); + self.read(b); + if let Some(c) = c { + self.read(c); + } + let out = self.alloc(); + self.instrs.push(Instr::ExtAlu { + op, + out, + a, + b, + c: c.unwrap_or(Addr(0)), + mult: 0, + }); + Ext(out) + } + + pub fn eadd(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Add, a.0, b.0, None) + } + pub fn esub(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Sub, a.0, b.0, None) + } + pub fn emul(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Mul, a.0, b.0, None) + } + /// `a / b` under `0/0 = (1, 0, 0)`, `x/0 = error`. + pub fn ediv(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Div, a.0, b.0, None) + } + pub fn emul_add(&mut self, a: Ext, b: Ext, c: Ext) -> Ext { + self.xalu(ExtOp::MulAdd, a.0, b.0, Some(c.0)) + } + /// Extension × base — 3 base multiplies instead of 9. + pub fn emul_base(&mut self, a: Ext, b: Felt) -> Ext { + self.xalu(ExtOp::MulBase, a.0, b.0, None) + } + + // ---- assertions (lowered, no chip) ---- + + /// `assert_eq` lowers to `diff = a − b; _ = diff / ZERO`: provable (and + /// executable) iff `diff = 0` under the `0/0 = 1` convention. + pub fn assert_eq(&mut self, a: Felt, b: Felt) { + let diff = self.sub(a, b); + let zero = self.felt_const(FE::zero()); + let _ = self.div(diff, zero); + } + + pub fn assert_eq_ext(&mut self, a: Ext, b: Ext) { + let diff = self.esub(a, b); + let zero = self.ext_const(&FEE::zero()); + let _ = self.ediv(diff, zero); + } + + // ---- select / bitdec ---- + + /// Conditional swap: `bit = 0 ⇒ (l, r)`; `bit = 1 ⇒ (r, l)`. + pub fn select(&mut self, bit: Bit, l: Cell, r: Cell) -> (Cell, Cell) { + self.read(bit.0); + self.read(l.0); + self.read(r.0); + let out_l = self.alloc(); + let out_r = self.alloc(); + self.instrs.push(Instr::Select { + bit: bit.0, + out_l, + out_r, + in_l: l.0, + in_r: r.0, + mult_l: 0, + mult_r: 0, + }); + (Cell(out_l), Cell(out_r)) + } + + /// Canonical 64-bit decomposition; returns the low `nbits` bits as cells + /// (low-to-high). Only these become memory cells; all 64 bits exist as + /// constrained witness columns either way. + pub fn bit_dec(&mut self, x: Felt, nbits: usize) -> Vec { + assert!(nbits <= 64, "bit_dec: at most 64 bits"); + self.read(x.0); + let bits: Vec<(Addr, u64)> = (0..nbits).map(|_| (self.alloc(), 0)).collect(); + let handles = bits.iter().map(|(a, _)| Bit(*a)).collect(); + self.instrs.push(Instr::BitDec { + input: x.0, + bits, + halves: None, + }); + handles + } + + /// The two BIG-ENDIAN `u32` halves of `x`'s 8-byte rendering — + /// `[byteswap32(high word), byteswap32(low word)]` — as one `BitDec` row + /// and NOTHING else: the halves are bus sends that are linear forms over + /// the row's own bit columns, so the 64 base-ALU rows the in-program + /// recomposition used to cost are gone. Canonicity (`< p`) is enforced by + /// the row, exactly as production renders `canonical_u64()`. + pub fn bit_dec_be_halves(&mut self, x: Felt) -> [Felt; 2] { + self.read(x.0); + let halves = [(self.alloc(), 0), (self.alloc(), 0)]; + self.instrs.push(Instr::BitDec { + input: x.0, + bits: Vec::new(), + halves: Some(halves), + }); + [Felt(halves[0].0), Felt(halves[1].0)] + } + + // ---- hash ---- + + /// Two digest cells → one digest cell. + pub fn compress(&mut self, a: DigestVal, b: DigestVal) -> DigestVal { + self.two_to_one(HashMode::Compress, a, b) + } + + /// One step of a Merkle LEAF chain: absorb one cell, read as four FIELD + /// ELEMENTS, into the running accumulator `acc`. + /// + /// The only mode whose second input is not a digest: each felt is split into + /// a checked `lo`/`hi` `u32` pair inside the chip, so arbitrary Goldilocks + /// data can be hashed by a socket whose lanes must be `u32`. The `"LFML"` + /// domain keeps a leaf un-replayable as a parent whatever the tree's shape. + /// + /// **One call absorbs four felts AND chains**, because the accumulator rides + /// in the message rather than being folded in afterwards: a wide leaf over + /// `k` cells costs `k` hashes against the `2k − 1` a felts-only leaf plus a + /// fold of the results costs (COMMIT.md §1.2). The chain binds cell ORDER + /// for free; what it does not bind is the leaf's SHAPE, which is the header + /// cell's job in the commitment layer above this. + pub fn leaf(&mut self, acc: DigestVal, felts: Cell) -> DigestVal { + self.read(acc.0); + self.read(felts.0); + let out = self.alloc(); + self.instrs.push(Instr::Hash { + mode: HashMode::Leaf, + ins: [acc.0, felts.0, Addr(0)], + outs: [out, Addr(0), Addr(0)], + mults: [0, 0, 0], + }); + DigestVal(out) + } + + /// One step of the Fiat–Shamir transcript chain: two cells → one cell, in + /// the TRANSCRIPT hash domain. + /// + /// The same socket and the same columns as [`LfmBuilder::compress`]; the + /// row's preprocessed mode selects the domain tag, so a transcript step and + /// a Merkle parent over the same two cells are different digests. Callers + /// go through [`super::edsl::SpongeVar`] rather than here — the chain's + /// operand sequence is what its security argument rests on, and a raw step + /// is an easy way to break it. + pub fn transcript_step(&mut self, a: DigestVal, b: DigestVal) -> DigestVal { + self.two_to_one(HashMode::Transcript, a, b) + } + + fn two_to_one(&mut self, mode: HashMode, a: DigestVal, b: DigestVal) -> DigestVal { + debug_assert!(mode.is_two_to_one()); + self.read(a.0); + self.read(b.0); + let out = self.alloc(); + self.instrs.push(Instr::Hash { + mode, + ins: [a.0, b.0, Addr(0)], + outs: [out, Addr(0), Addr(0)], + mults: [0, 0, 0], + }); + DigestVal(out) + } + + /// Full three-cell state permutation. + pub fn permute(&mut self, state: [Cell; 3]) -> [Cell; 3] { + for c in &state { + self.read(c.0); + } + let outs = [self.alloc(), self.alloc(), self.alloc()]; + self.instrs.push(Instr::Hash { + mode: HashMode::Permute, + ins: [state[0].0, state[1].0, state[2].0], + outs, + mults: [0, 0, 0], + }); + outs.map(Cell) + } + + /// One width-16 Poseidon1 permutation (`Instr::Hash16`): four cells in, + /// all sixteen lanes permuted, four cells out. A program that emits one + /// emits no twelve-felt hash row (the admission validator), and proves + /// under `HasherKind::Poseidon1W16`. + pub fn hash16(&mut self, ins: [Cell; 4]) -> [Cell; 4] { + for c in &ins { + self.read(c.0); + } + let outs = [self.alloc(), self.alloc(), self.alloc(), self.alloc()]; + self.instrs + .push(Instr::Hash16(Box::new(super::instr::Hash16Operands { + ins: ins.map(|c| c.0), + outs, + mults: [0; 4], + }))); + outs.map(Cell) + } + + // ---- lane conversion (LFM_LANES) ---- + + /// Split a word into its four lanes as base cells — the only route from + /// a hash-state/digest cell into the ALU. + pub fn unpack(&mut self, c: Cell) -> [Felt; 4] { + self.read(c.0); + let outs = [self.alloc(), self.alloc(), self.alloc(), self.alloc()]; + self.instrs.push(Instr::Unpack { + input: c.0, + outs, + mults: [0; 4], + }); + outs.map(Felt) + } + + /// Assemble a word from four base cells. + pub fn pack_word(&mut self, lanes: [Felt; 4]) -> Cell { + for l in &lanes { + self.read(l.0); + } + let out = self.alloc(); + self.instrs.push(Instr::Pack { + lanes: lanes.map(|f| f.0), + out, + mult: 0, + }); + Cell(out) + } + + /// Assemble an ext cell `(a0, a1, a2, 0)` from three base cells (lane 3 + /// is the shared zero constant). + pub fn pack_ext(&mut self, a0: Felt, a1: Felt, a2: Felt) -> Ext { + let zero = self.felt_const(FE::zero()); + Ext(self.pack_word([a0, a1, a2, zero]).0) + } + + // ---- keccak-f[1600] (LFM_KECCAK) ---- + + /// One `keccak-f[1600]` permutation over 13 state words. + /// + /// The 25 `u64` lanes travel as 50 `u32` halves packed four to a word: + /// word `j` carries halves `4j..4j+3`, half `h` is the low (`h` even) or + /// high (`h` odd) 32 bits of lane `h / 2`. The last word's top two lanes + /// are unused and must be zero — the bus pins them as tuple constants, and + /// the executor errors on a nonzero one. Every lane of every input word + /// must be a canonical value below `2^32`. + pub fn keccak_f(&mut self, state: [Cell; layout::keccak::NUM_WORDS]) -> [Cell; 13] { + self.emit_keccak(KeccakMode::Permute, state, [Cell(Addr(0)); 9], false) + .0 + } + + /// One sponge absorb step: XOR a 136-byte rate block (9 words of `u32` + /// halves, the top two half slots unused and zero) into the state's rate + /// region, then permute. + pub fn keccak_absorb( + &mut self, + state: [Cell; layout::keccak::NUM_WORDS], + block: [Cell; layout::keccak::BLOCK_WORDS], + ) -> [Cell; 13] { + self.emit_keccak(KeccakMode::Absorb, state, block, false).0 + } + + /// Absorb, and additionally materialize the byte-REVERSED digest of the + /// resulting state as two words — the production transcript's `sample()`, + /// which both returns those bytes and re-absorbs them as the next segment's + /// prefix. Free on the bus (see `layout::keccak::REV_ADDR0`). + pub fn keccak_absorb_rev( + &mut self, + state: [Cell; layout::keccak::NUM_WORDS], + block: [Cell; layout::keccak::BLOCK_WORDS], + ) -> ([Cell; 13], [Cell; 2]) { + let (outs, rev) = self.emit_keccak(KeccakMode::Absorb, state, block, true); + (outs, rev.expect("requested")) + } + + fn emit_keccak( + &mut self, + mode: KeccakMode, + state: [Cell; layout::keccak::NUM_WORDS], + block: [Cell; layout::keccak::BLOCK_WORDS], + want_rev: bool, + ) -> ([Cell; 13], Option<[Cell; 2]>) { + for c in &state { + self.read(c.0); + } + if mode == KeccakMode::Absorb { + for c in &block { + self.read(c.0); + } + } + let outs: [Addr; 13] = core::array::from_fn(|_| self.alloc()); + let rev_outs: Option<[Addr; 2]> = want_rev.then(|| core::array::from_fn(|_| self.alloc())); + self.instrs + .push(Instr::KeccakF(Box::new(super::instr::KeccakOperands { + mode, + ins: state.map(|c| c.0), + block: block.map(|c| c.0), + outs, + mults: [0; 13], + rev: rev_outs.map(|outs| super::instr::KeccakReversedDigest { + outs, + mults: [0; 2], + }), + }))); + (outs.map(Cell), rev_outs.map(|r| r.map(Cell))) + } + + // ---- BLAKE3 compression (LFM_BLAKE3) ---- + + /// One BLAKE3 compression: `compress(h, m, t, block_len, flags)`, returning + /// the full 16-word output as four machine words. + /// + /// `h` is the chaining value as two words of four `u32` lanes; `m` is the + /// 64-byte message block as four; `params` is one word carrying + /// `(t_lo, t_hi, block_len, flags)`. Every lane of every input must be a + /// canonical value below `2^32` — the chip decomposes each into four + /// BITWISE-range-checked byte columns, so no larger value exists on the AIR + /// side and the executor errors rather than producing an unprovable witness. + /// + /// The digest of the compression is output words 0 and 1 (`out[0..8]` + /// little-endian), so callers wanting a `Blake3Digest` take the first two of + /// the four. The chaining value of the *next* block is the same two words, + /// which is what makes [`super::edsl::blake3`]'s multi-block framing free of + /// any repacking between blocks. + /// + /// Callers should go through [`super::edsl`]'s framing rather than here: the + /// flag schedule and `block_len` convention are what `Blake3Chain` is, and a + /// raw compression is an easy way to emit a different hash. + pub fn blake3_compress( + &mut self, + h: [Cell; 2], + m: [Cell; 4], + params: Cell, + ) -> [Cell; layout::blake3::OUT_WORDS] { + self.emit_blake3(h, m, params, false).0 + } + + /// [`LfmBuilder::blake3_compress`], additionally materializing the + /// byte-REVERSED digest as two more words — the production transcript's + /// `sample()`, which both returns those bytes and re-absorbs them as the + /// next segment's prefix. Free on the bus (see `layout::blake3::REV_ADDR0`). + pub fn blake3_compress_rev( + &mut self, + h: [Cell; 2], + m: [Cell; 4], + params: Cell, + ) -> ([Cell; layout::blake3::OUT_WORDS], [Cell; 2]) { + let (outs, rev) = self.emit_blake3(h, m, params, true); + (outs, rev.expect("requested")) + } + + fn emit_blake3( + &mut self, + h: [Cell; 2], + m: [Cell; 4], + params: Cell, + want_rev: bool, + ) -> ( + [Cell; layout::blake3::OUT_WORDS], + Option<[Cell; layout::blake3::DIGEST_WORDS]>, + ) { + // Machine order: `h` is `u32` words 0..8, `m` is 8..24, and `params` is + // 24..28 — the layout `Blake3Operands` documents and `blake3_chip`'s + // `input_words` builds. + let ins: [Addr; layout::blake3::IN_WORDS] = + [h[0].0, h[1].0, m[0].0, m[1].0, m[2].0, m[3].0, params.0]; + for a in &ins { + self.read(*a); + } + let outs: [Addr; layout::blake3::OUT_WORDS] = core::array::from_fn(|_| self.alloc()); + let rev_outs: Option<[Addr; layout::blake3::DIGEST_WORDS]> = + want_rev.then(|| core::array::from_fn(|_| self.alloc())); + self.instrs + .push(Instr::Blake3(Box::new(super::instr::Blake3Operands { + ins, + outs, + mults: [0; layout::blake3::OUT_WORDS], + rev: rev_outs.map(|outs| super::instr::Blake3ReversedDigest { + outs, + mults: [0; layout::blake3::DIGEST_WORDS], + }), + }))); + (outs.map(Cell), rev_outs.map(|r| r.map(Cell))) + } + + // ---- hints / public ---- + + pub fn declare_arena(&mut self, len: u32) -> ArenaId { + self.arena_schema.lens.push(len); + (self.arena_schema.lens.len() - 1) as ArenaId + } + + /// How many arenas the program has declared so far. + pub fn num_arenas(&self) -> usize { + self.arena_schema.lens.len() + } + + /// Sets a declared arena's length: for an emitter whose hint count is known + /// only once it has hinted everything from its shapes. + pub fn set_arena_len(&mut self, arena: ArenaId, len: u32) { + self.arena_schema.lens[arena as usize] = len; + } + + /// One arena word → one memory cell. Arena values are unconstrained by + /// the reading chip; the arena rule (transitively hash-authenticate + /// everything hinted; never derive challenges from arenas) is what makes + /// this sound. + pub fn hint_word(&mut self, arena: ArenaId, index: u32) -> Cell { + let out = self.alloc(); + self.instrs.push(Instr::Hint { + arena, + index, + out, + mult: 0, + }); + Cell(out) + } + + /// [`Self::hint_word`] reading arena word `alt` instead when `bit` is 1 + /// ([`Instr::HintOn`]). The bit is not read: it costs no row and adds + /// nothing to its multiplicity, and it must be written before this hint + /// (the validator checks). + pub fn hint_word_on(&mut self, arena: ArenaId, index: u32, bit: Bit, alt: u32) -> Cell { + let out = self.alloc(); + self.instrs.push(Instr::HintOn { + arena, + index, + alt, + bit: bit.0, + out, + mult: 0, + }); + Cell(out) + } + + pub fn hint_felt(&mut self, arena: ArenaId, index: u32) -> Felt { + Felt(self.hint_word(arena, index).0) + } + + /// Expose a cell on the public-output bus (auto-incrementing index). + pub fn public(&mut self, c: Cell) { + self.read(c.0); + let index = self.public_len; + self.public_len += 1; + self.instrs.push(Instr::Public { addr: c.0, index }); + } + + pub fn finish(self) -> LfmProgramSource { + LfmProgramSource { + instrs: self.instrs, + num_addrs: self.next_addr, + read_counts: self.read_counts, + arena_schema: self.arena_schema, + public_len: self.public_len, + } + } +} diff --git a/prover/src/lfm/chips.rs b/prover/src/lfm/chips.rs new file mode 100644 index 000000000..711514abd --- /dev/null +++ b/prover/src/lfm/chips.rs @@ -0,0 +1,1578 @@ +//! The LFM chips: bus interactions and constraint sets. +//! +//! Eleven chips live here. The other three slots of the fixed AIR set are the +//! production keccak family (`KECCAK_RND` / `KECCAK_RC` / `BITWISE`), hosted +//! unchanged from `tables/` and driven by `LFM_KECCAK` below. +//! +//! Shared conventions (see `SOUNDNESS.md` and the design doc): +//! - each chip's trace = its instruction column group (preprocessed, leading +//! columns, layout in [`super::layout`]) followed by the value columns +//! defined here; +//! - one sign convention machine-wide: writes are senders with +//! `Multiplicity::Column(mult)` (a preprocessed column), reads are +//! receivers gated by selectors / `is_real` (also preprocessed); no +//! `Negated` forms anywhere; +//! - the `LfmMem` token is `(addr, v0, v1, v2, v3)`; base values carry +//! constant-zero high lanes *in the tuple*, so a base cell cannot smuggle +//! extension lanes; +//! - in-AIR checks are per-op algebra plus belt-over-suspenders booleanity; +//! uniqueness/acyclicity/mult-equality/one-hot-ness are the registrar's +//! (admission validator), per the soundness split. + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; + +use crate::tables::types::{BusId, GoldilocksExtension, GoldilocksField}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +fn direct(col: usize) -> BusValue { + BusValue::Packed { + start_column: col, + packing: Packing::Direct, + } +} + +fn zero() -> BusValue { + BusValue::constant(0) +} + +/// A word value spread over four adjacent columns. +fn word(cols_start: usize) -> [BusValue; 4] { + [ + direct(cols_start), + direct(cols_start + 1), + direct(cols_start + 2), + direct(cols_start + 3), + ] +} + +/// `Σ` of a run of selector columns, as a LogUp multiplicity (= is_real). +fn selector_sum(first: usize, count: usize) -> Multiplicity { + Multiplicity::Linear( + (0..count) + .map(|i| LinearTerm::ColumnUnsigned { + coefficient: 1, + column: first + i, + }) + .collect(), + ) +} + +/// A base-value memory token: `(addr, v, 0, 0, 0)`. +fn base_token(addr_col: usize, val_col: usize) -> Vec { + vec![direct(addr_col), direct(val_col), zero(), zero(), zero()] +} + +/// An ext-value memory token: `(addr, v0, v1, v2, 0)` — lane 3 is a tuple +/// constant, which is exactly what pins ext cells to lane-3-zero. +fn ext_token(addr_col: usize, lanes_start: usize) -> Vec { + vec![ + direct(addr_col), + direct(lanes_start), + direct(lanes_start + 1), + direct(lanes_start + 2), + zero(), + ] +} + +/// A full-word memory token: `(addr, v0..v3)`. +fn word_token(addr_col: usize, lanes_start: usize) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend(word(lanes_start)); + v +} + +// ========================================================================= +// LFM_CONST — pooled constants (all instruction data preprocessed) +// ========================================================================= + +pub mod const_ { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::const_::*; + /// All-zero main column: the commit path expects a non-empty + /// non-preprocessed subset (KECCAK_RC precedent). + pub const PAD: usize = PREP_WIDTH; + pub const NUM_COLUMNS: usize = PREP_WIDTH + 1; + } + + pub fn bus_interactions() -> Vec { + vec![BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + word_token(cols::ADDR, cols::V0), + )] + } +} + +// ========================================================================= +// LFM_BALU — Goldilocks ALU +// ========================================================================= + +pub mod balu { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::balu::*; + pub const A: usize = PREP_WIDTH; + pub const B: usize = PREP_WIDTH + 1; + pub const C: usize = PREP_WIDTH + 2; + pub const OUT: usize = PREP_WIDTH + 3; + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + vec![ + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + base_token(cols::A_ADDR, cols::A), + ), + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + base_token(cols::B_ADDR, cols::B), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::SEL_MULADD), + base_token(cols::C_ADDR, cols::C), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + base_token(cols::OUT_ADDR, cols::OUT), + ), + ] + } + + pub struct BaluConstraints; + + impl ConstraintSet for BaluConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, cols::A); + let bb = b.main(0, cols::B); + let c = b.main(0, cols::C); + let out = b.main(0, cols::OUT); + let sel = |b: &B, i: usize| b.main(0, cols::SEL_ADD + i); + + // idx 0: add — sel·(a + b − out) + b.emit_base(0, sel(b, 0) * (a.clone() + bb.clone() - out.clone())); + // idx 1: sub — sel·(a − b − out) + b.emit_base(1, sel(b, 1) * (a.clone() - bb.clone() - out.clone())); + // idx 2: mul — sel·(a·b − out) + b.emit_base(2, sel(b, 2) * (a.clone() * bb.clone() - out.clone())); + // idx 3: div as reversed mul — sel·(b·out − a). With b = 0 this + // forces a = 0 and leaves out free (the executor pins 0/0 = 1): + // the assert-via-division mechanism. + b.emit_base(3, sel(b, 3) * (bb.clone() * out.clone() - a.clone())); + // idx 4: mul-add — sel·(a·b + c − out) (the Horner step) + b.emit_base(4, sel(b, 4) * (a * bb + c - out)); + // idx 5: selector sum-boolean (belt; one-hot is the registrar's) + let sum = (1..cols::NUM_SELECTORS).fold(sel(b, 0), |acc, i| acc + sel(b, i)); + let one = b.one(); + b.emit_base(5, sum.clone() * (one - sum)); + } + } +} + +// ========================================================================= +// LFM_XALU — Fp3 ALU on word lanes 0–2 (w³ = 2) +// ========================================================================= + +pub mod xalu { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::xalu::*; + pub const A0: usize = PREP_WIDTH; // ..A2 + pub const B0: usize = PREP_WIDTH + 3; // ..B2 + pub const C0: usize = PREP_WIDTH + 6; // ..C2 + pub const OUT0: usize = PREP_WIDTH + 9; // ..OUT2 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 12; + } + + pub fn bus_interactions() -> Vec { + vec![ + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + ext_token(cols::A_ADDR, cols::A0), + ), + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + ext_token(cols::B_ADDR, cols::B0), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::SEL_MULADD), + ext_token(cols::C_ADDR, cols::C0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + ext_token(cols::OUT_ADDR, cols::OUT0), + ), + ] + } + + pub struct XaluConstraints; + + impl XaluConstraints { + /// The three product lanes of `X·Y` in `Fp[w]/(w³ − 2)`: + /// `p0 = x0y0 + 2(x1y2 + x2y1)`, `p1 = x0y1 + x1y0 + 2·x2y2`, + /// `p2 = x0y2 + x1y1 + x2y0` (matches `Degree3GoldilocksExtensionField::mul`). + fn product>(b: &B, x0: usize, y0: usize) -> [B::Expr; 3] { + let x = |i: usize| b.main(0, x0 + i); + let y = |i: usize| b.main(0, y0 + i); + let two = b.const_base(2); + [ + x(0) * y(0) + two.clone() * (x(1) * y(2) + x(2) * y(1)), + x(0) * y(1) + x(1) * y(0) + two * x(2) * y(2), + x(0) * y(2) + x(1) * y(1) + x(2) * y(0), + ] + } + } + + impl ConstraintSet for XaluConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let sel = |b: &B, i: usize| b.main(0, cols::SEL_ADD + i); + let lane = |b: &B, base: usize, j: usize| b.main(0, base + j); + + // idx 0–2 add / 3–5 sub: componentwise. + for j in 0..3 { + let a = lane(b, cols::A0, j); + let bb = lane(b, cols::B0, j); + let out = lane(b, cols::OUT0, j); + b.emit_base(j, sel(b, 0) * (a.clone() + bb.clone() - out.clone())); + b.emit_base(3 + j, sel(b, 1) * (a - bb - out)); + } + // idx 6–8 mul: P(A, B) = OUT. + let p_ab = Self::product(b, cols::A0, cols::B0); + for (j, p) in p_ab.into_iter().enumerate() { + b.emit_base(6 + j, sel(b, 2) * (p - lane(b, cols::OUT0, j))); + } + // idx 9–11 div as reversed mul: P(B, OUT) = A (0/0 = (1,0,0) in + // the executor; x/0 unprovable — the ext assert mechanism). + let p_bout = Self::product(b, cols::B0, cols::OUT0); + for (j, p) in p_bout.into_iter().enumerate() { + b.emit_base(9 + j, sel(b, 3) * (p - lane(b, cols::A0, j))); + } + // idx 12–14 mul-add: P(A, B) + C = OUT. + let p_ab = Self::product(b, cols::A0, cols::B0); + for (j, p) in p_ab.into_iter().enumerate() { + b.emit_base( + 12 + j, + sel(b, 4) * (p + lane(b, cols::C0, j) - lane(b, cols::OUT0, j)), + ); + } + // idx 15–17 mul-base: OUT_j = A_j · B0. + for j in 0..3 { + b.emit_base( + 15 + j, + sel(b, 5) + * (lane(b, cols::A0, j) * lane(b, cols::B0, 0) - lane(b, cols::OUT0, j)), + ); + } + // idx 18–19: MulBase's operand is a base word — the shared B + // lanes 1–2 must vanish on its rows or the received token would + // not match any base writer's. + b.emit_base(18, sel(b, 5) * lane(b, cols::B0, 1)); + b.emit_base(19, sel(b, 5) * lane(b, cols::B0, 2)); + // idx 20: selector sum-boolean. + let sum = (1..cols::NUM_SELECTORS).fold(sel(b, 0), |acc, i| acc + sel(b, i)); + let one = b.one(); + b.emit_base(20, sum.clone() * (one - sum)); + } + } +} + +// ========================================================================= +// LFM_SELECT — conditional cell swap +// ========================================================================= + +pub mod select { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::select::*; + pub const BIT: usize = PREP_WIDTH; + pub const INL0: usize = PREP_WIDTH + 1; // ..+4 + pub const INR0: usize = PREP_WIDTH + 5; // ..+8 + pub const OUTL0: usize = PREP_WIDTH + 9; // ..+12 + pub const OUTR0: usize = PREP_WIDTH + 13; // ..+16 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 17; + } + + pub fn bus_interactions() -> Vec { + vec![ + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + base_token(cols::BIT_ADDR, cols::BIT), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + word_token(cols::INL_ADDR, cols::INL0), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + word_token(cols::INR_ADDR, cols::INR0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT_L), + word_token(cols::OUTL_ADDR, cols::OUTL0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT_R), + word_token(cols::OUTR_ADDR, cols::OUTR0), + ), + ] + } + + pub struct SelectConstraints; + + impl ConstraintSet for SelectConstraints { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + // idx 0: bit booleanity (belt over suspenders — a witness bit + // exists, so it is constrained here and not only vouched). + let bit = b.main(0, cols::BIT); + let one = b.one(); + b.emit_base(0, bit.clone() * (one - bit)); + // idx 1–4 / 5–8: out_l = in_l + bit·(in_r − in_l); out_r mirrored. + // Trivially satisfied on zero-filled padding rows. + for j in 0..4 { + let bit = b.main(0, cols::BIT); + let inl = b.main(0, cols::INL0 + j); + let inr = b.main(0, cols::INR0 + j); + let outl = b.main(0, cols::OUTL0 + j); + b.emit_base( + 1 + j, + outl - (inl.clone() + bit.clone() * (inr.clone() - inl.clone())), + ); + let outr = b.main(0, cols::OUTR0 + j); + b.emit_base(5 + j, outr - (inr.clone() + bit * (inl - inr))); + } + } + } +} + +// ========================================================================= +// LFM_BITDEC — canonical 64-bit decomposition over p = 2^64 − 2^32 + 1 +// ========================================================================= + +pub mod bitdec { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::bitdec::*; + pub const BITS0: usize = PREP_WIDTH; // 64 bit columns, low-to-high + pub const Z: usize = PREP_WIDTH + NUM_BITS; + pub const GINV: usize = PREP_WIDTH + NUM_BITS + 1; + pub const NUM_COLUMNS: usize = PREP_WIDTH + NUM_BITS + 2; + } + + pub fn bus_interactions() -> Vec { + // The received value is the recomposition Σ 2^i·B_i, expressed as a + // linear bus value over the bit columns — no input column needed. + let recomposition = BusValue::Linear( + (0..cols::NUM_BITS) + .map(|i| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << i, + column: cols::BITS0 + i, + }) + .collect(), + ); + let mut interactions = vec![BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + vec![direct(cols::IN_ADDR), recomposition, zero(), zero(), zero()], + )]; + for i in 0..cols::NUM_BITS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::bit_mult(i)), + base_token(cols::bit_addr(i), cols::BITS0 + i), + )); + } + // The two BIG-ENDIAN halves, as linear forms over the SAME bit + // columns booleanity and canonicity already pin: bit `8k + j` of + // half-word `h` (h = 0 is the value's HIGH word — it leads in + // big-endian order) lands at byte `3 − k`, so its weight is + // `2^(j + 8(3 − k))`. No value column, no constraint: the senders + // are functions of already-constrained columns. + for (h, first) in [(0usize, 32usize), (1, 0)] { + let half = BusValue::Linear( + (0..4) + .flat_map(|k| (0..8).map(move |j| (k, j))) + .map(|(k, j)| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (j + 8 * (3 - k)), + column: cols::BITS0 + first + 8 * k + j, + }) + .collect(), + ); + let (addr, mult) = if h == 0 { + (cols::HALF0_ADDR, cols::HALF0_MULT) + } else { + (cols::HALF1_ADDR, cols::HALF1_MULT) + }; + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(mult), + vec![direct(addr), half, zero(), zero(), zero()], + )); + } + interactions + } + + pub struct BitDecConstraints; + + impl ConstraintSet for BitDecConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + // idx 0–63: booleanity. + for i in 0..cols::NUM_BITS { + let bit = b.main(0, cols::BITS0 + i); + let one = b.one(); + b.emit_base(i, bit.clone() * (one - bit)); + } + // Canonicity: p − 1 = (2^32 − 1)·2^32, i.e. 32 ones ‖ 32 zeros, + // so value < p ⟺ (top 32 bits all ones ⇒ bottom 32 bits zero). + // G = (2^32 − 1) − Σ_{i=32..63} 2^{i−32}·B_i; witnesses Z ("top + // all ones"), GINV (= G⁻¹ when G ≠ 0). + let top = (0..32).fold(None::, |acc, k| { + let term = b.const_base(1u64 << k) * b.main(0, cols::BITS0 + 32 + k); + Some(match acc { + None => term, + Some(a) => a + term, + }) + }); + let g = b.const_base(0xFFFF_FFFF) - top.expect("nonempty"); + let z = b.main(0, cols::Z); + let ginv = b.main(0, cols::GINV); + // idx 64: Z·G = 0 — G ≠ 0 forces Z = 0. + b.emit_base(64, z.clone() * g.clone()); + // idx 65: IS_REAL·(1 − Z − G·GINV) = 0 — G = 0 forces Z = 1. + // Gated by IS_REAL so zero-filled padding rows satisfy it. + let is_real = b.main(0, cols::IS_REAL); + let one = b.one(); + b.emit_base(65, is_real * (one - z.clone() - g * ginv)); + // idx 66: Z·(Σ_{i<32} 2^i·B_i) = 0 — top all ones ⇒ bottom zero. + let low = (0..32).fold(None::, |acc, k| { + let term = b.const_base(1u64 << k) * b.main(0, cols::BITS0 + k); + Some(match acc { + None => term, + Some(a) => a + term, + }) + }); + b.emit_base(66, z * low.expect("nonempty")); + } + } +} + +// ========================================================================= +// LFM_HASH — the chiplet (frozen tuple contract; TestPermutation behind it) +// ========================================================================= + +pub mod hash { + use super::*; + use crate::lfm::hash::{HASH_STATE_FELTS, HasherKind, TestPermutation}; + use crate::lfm::instr::HashMode; + use crate::tables::types::FE; + use math::field::traits::IsPrimeField; + + pub mod cols { + pub use crate::lfm::layout::hash::*; + pub const IN0: usize = PREP_WIDTH; // ..IN11 + /// Materialized capacity-state columns for lanes 8–11: + /// `S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i` (degree-2 + /// copy), so the permutation constraint stays at degree 3. Transcript + /// and leaf rows are compresses in every structural respect, so they + /// take the IV too. + pub const S8: usize = PREP_WIDTH + 12; // ..S11 + pub const OUT0: usize = PREP_WIDTH + 16; // ..OUT11 + /// Value columns every hasher's layout shares: `IN`, `S`, `OUT`. The + /// bus tuples read only these (`bus_interactions`), which is why they + /// keep their offsets in EVERY layout — a candidate appends its + /// witness columns after them rather than reflowing the prefix. + pub const SHARED_VALUE_COLUMNS: usize = 28; + /// Width of the [`HasherKind::Test`] layout. Use [`super::num_columns`] + /// unless you specifically mean `TestPermutation`. + pub const TEST_NUM_COLUMNS: usize = PREP_WIDTH + SHARED_VALUE_COLUMNS; + } + + /// Column layout for the [`HasherKind::Rpx`] configuration — Layout W with + /// a MIXED round schedule. + /// + /// The frozen prefix keeps its offsets, so [`bus_interactions`] is + /// hasher-INDEPENDENT here exactly as it is for RPO and Poseidon. What + /// changes is that RPX's seven rounds are not alike, so the appended block + /// width depends on the round KIND: + /// + /// | round | kind | appended columns | + /// |---|---|---| + /// | 0, 2, 4 | FB | `u²`, `u³`, `y²`, `y³` (48) + the round output (12) = **60** | + /// | 1, 3, 5 | E | `t²`, `t³` in the extension (24) + the round output (12) = **36** | + /// | 6 | M | **0** — `MDS(state) + ARK1[6]` is degree 1 and lands straight in `OUT` | + /// + /// Width: `28 + 3·60 + 3·36 = 316` value columns, against the 436 an + /// all-FB (RPO) layout takes — **27% narrower**, because half the rounds + /// trade a twelve-lane double ladder for four triples of extension + /// intermediates, and the last round commits nothing at all. + pub mod rpx_cols { + use crate::lfm::hash::HASH_STATE_FELTS; + use crate::lfm::rpx::{EXT_DEGREE, EXT_ELEMENTS, NUM_ROUNDS, is_fb_round, is_final_round}; + + pub use super::cols::{ + IN0, MODE_C, MODE_L, MODE_P, MODE_T, OUT0, PREP_WIDTH, S8, SHARED_VALUE_COLUMNS, + }; + + /// First appended witness column. + pub const ROUNDS: usize = PREP_WIDTH + SHARED_VALUE_COLUMNS; + + /// Committed columns per lane in an FB round: `u²`, `u³`, `y²`, `y³`. + pub const FB_LADDER_COLUMNS: usize = 4; + /// Committed extension intermediates per E round lane: `t²`, `t³`. + pub const EXT_LADDER_COLUMNS: usize = 2; + + pub const fn block_width(r: usize) -> usize { + if is_final_round(r) { + 0 + } else if is_fb_round(r) { + FB_LADDER_COLUMNS * HASH_STATE_FELTS + HASH_STATE_FELTS + } else { + EXT_LADDER_COLUMNS * HASH_STATE_FELTS + HASH_STATE_FELTS + } + } + + pub const fn block(r: usize) -> usize { + let mut off = ROUNDS; + let mut i = 0; + while i < r { + off += block_width(i); + i += 1; + } + off + } + + /// `u_lane²` — FB rounds only. + pub const fn u2(r: usize, lane: usize) -> usize { + block(r) + lane + } + /// `u_lane³` — FB rounds only. + pub const fn u3(r: usize, lane: usize) -> usize { + block(r) + HASH_STATE_FELTS + lane + } + /// `y_lane²` — FB rounds only. + pub const fn y2(r: usize, lane: usize) -> usize { + block(r) + 2 * HASH_STATE_FELTS + lane + } + /// `y_lane³` — FB rounds only. + pub const fn y3(r: usize, lane: usize) -> usize { + block(r) + 3 * HASH_STATE_FELTS + lane + } + + /// `x²` in the extension, coefficient `k` of element `e` — E rounds only. + pub const fn t2(r: usize, e: usize, k: usize) -> usize { + block(r) + e * EXT_DEGREE + k + } + /// `x³` in the extension — E rounds only. + pub const fn t3(r: usize, e: usize, k: usize) -> usize { + block(r) + HASH_STATE_FELTS + e * EXT_DEGREE + k + } + + /// Round `r`'s output lane `j`. The M round writes `OUT` directly. + pub const fn out(r: usize, j: usize) -> usize { + if is_final_round(r) { + OUT0 + j + } else if is_fb_round(r) { + block(r) + FB_LADDER_COLUMNS * HASH_STATE_FELTS + j + } else { + block(r) + EXT_LADDER_COLUMNS * HASH_STATE_FELTS + j + } + } + + pub const NUM_COLUMNS: usize = block(NUM_ROUNDS); + + /// 4 capacity copies + 1 mode boolean + the unread-`IN` pins, plus per + /// round: FB five per lane, E three per extension operation on each of + /// four triples, M one per output lane. + pub const NUM_CONSTRAINTS: usize = { + let mut n = 5 + super::NUM_UNREAD_INPUT_PINS; + let mut r = 0; + while r < NUM_ROUNDS { + if is_final_round(r) { + n += HASH_STATE_FELTS; + } else if is_fb_round(r) { + n += 5 * HASH_STATE_FELTS; + } else { + n += 3 * EXT_DEGREE * EXT_ELEMENTS; + } + r += 1; + } + n + }; + } + + /// The chip's total width under `kind` — the number the AIR is built with, + /// the census reads, and the trace filler allocates. + pub const fn num_columns(kind: HasherKind) -> usize { + match kind { + HasherKind::Test => cols::TEST_NUM_COLUMNS, + HasherKind::Blake3 => crate::lfm::blake3_socket::cols::NUM_COLUMNS, + HasherKind::Rpx => rpx_cols::NUM_COLUMNS, + HasherKind::Poseidon1W16 => crate::lfm::p1w16_socket::SOCKET_FORM.num_columns(), + } + } + + /// The chip's bus interactions under `kind`. + /// + /// **Hasher-DEPENDENT, and BLAKE3 is why.** The six `LfmMem` tuples below + /// are the frozen `LFM_HASH` contract and are the same under every + /// candidate; they read and write only the shared value prefix, whose + /// offsets no layout moves. But a candidate built out of byte operations + /// needs a lookup table, and BLAKE3 needs one per XOR byte and one per + /// range-checked byte pair — over a thousand of them, none of which + /// `TestPermutation` or Poseidon has, both being pure field arithmetic. + /// + /// Callers must thread the same `kind` they build the AIR's width and + /// constraints with; `LfmAirs::new_with_hasher` is the one place that does. + pub fn bus_interactions(kind: HasherKind) -> Vec { + // The width-16 socket has its own tuple contract: four cells in, four + // out (`p1w16_socket::bus_interactions`). + if kind == HasherKind::Poseidon1W16 { + return crate::lfm::p1w16_socket::bus_interactions( + crate::lfm::p1w16_socket::SOCKET_FORM, + ); + } + let mut interactions = lfm_mem_interactions(); + if kind == HasherKind::Blake3 { + interactions.extend(crate::lfm::blake3_socket::bitwise_interactions()); + } + interactions + } + + /// The frozen `LFM_HASH` tuple contract: 2 (or 3) cells in, 1 (or 3) out. + /// + /// The FIRST TWO input cells are read in every mode, so their multiplicity + /// is the row's is-real flag: the sum of all four mode selectors, which the + /// AIR pins to a bit. The third is read only by a permutation. + /// + /// ⚠ The second cell's multiplicity used to EXCLUDE `MODE_L`, because a leaf + /// row read one cell of four felts and receiving a second would have claimed + /// a memory read it never made. Under the leaf RATE a leaf row reads two — + /// a chaining accumulator and a felt cell — so that exclusion became the + /// opposite bug: the felts would never be read from memory at all (COMMIT.md + /// §1.4.4 **H3**). The bus ARITY does not move; this multiplicity is the one + /// part of the frozen contract that the RATE does. + fn lfm_mem_interactions() -> Vec { + let is_real = || selector_sum(cols::MODE_C, cols::NUM_SELECTORS); + vec![ + BusInteraction::receiver( + BusId::LfmMem, + is_real(), + word_token(cols::IN_ADDR0, cols::IN0), + ), + BusInteraction::receiver( + BusId::LfmMem, + is_real(), + word_token(cols::IN_ADDR1, cols::IN0 + 4), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_P), + word_token(cols::IN_ADDR2, cols::IN0 + 8), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT0), + word_token(cols::OUT_ADDR0, cols::OUT0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT1), + word_token(cols::OUT_ADDR1, cols::OUT0 + 4), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT2), + word_token(cols::OUT_ADDR2, cols::OUT0 + 8), + ), + ] + } + + fn canonical_u64(fe: &FE) -> u64 { + GoldilocksField::canonical(fe.value()) + } + + /// Every mode selector paired with the mode it selects. + /// + /// One table, so the input pins below and anything else that reasons per + /// mode read the same mapping rather than each carrying its own copy. + pub(crate) const MODE_SELECTORS: [(usize, HashMode); 4] = [ + (cols::MODE_C, HashMode::Compress), + (cols::MODE_T, HashMode::Transcript), + (cols::MODE_L, HashMode::Leaf), + (cols::MODE_P, HashMode::Permute), + ]; + + /// Input cell slots that SOME mode does not read, and which therefore need + /// pinning. Cell 0 is read by every mode and is never a candidate. + /// + /// Derived rather than written down: the leaf RATE took `Leaf` from one + /// input cell to two, which emptied slot 1's set. Left as a literal, the + /// emitter's `.expect("some mode reads fewer than three input cells")` would + /// have fired and AIR construction would have panicked (COMMIT.md §1.4.4 + /// **H2**). + const fn unread_input_slots() -> usize { + let mut slots = 0; + let mut slot = 1; + while slot < 3 { + let mut i = 0; + while i < MODE_SELECTORS.len() { + if MODE_SELECTORS[i].1.num_input_cells() <= slot { + slots += 1; + break; + } + i += 1; + } + slot += 1; + } + slots + } + + /// Constraints [`emit_unread_input_pins`] emits: four per input cell that + /// some mode does not read. + pub(crate) const NUM_UNREAD_INPUT_PINS: usize = 4 * unread_input_slots(); + + /// The first constraint index each arm places the unread-`IN` pins at. + /// + /// Each arm chooses where in its own numbering they land, so the one place + /// that knows all three is here, next to the emitter. The controls read it + /// to assert that a forged row's violated set IS the pins. + #[cfg(test)] + pub(crate) const fn unread_input_pin_base(kind: HasherKind) -> usize { + match kind { + HasherKind::Test => 17, + HasherKind::Blake3 => crate::lfm::blake3_socket::UNREAD_IDX, + HasherKind::Rpx => rpx_cols::NUM_CONSTRAINTS - NUM_UNREAD_INPUT_PINS, + // The width-16 socket reads every input cell on every row, so it + // has no unread-input pins; this index is past its constraints. + HasherKind::Poseidon1W16 => crate::lfm::p1w16_socket::SOCKET_FORM.num_constraints(), + } + } + + /// ★ **Pins the `IN` columns of every input cell a mode does not read.** + /// + /// **This is load-bearing on any arm whose constraints READ `IN`, and that + /// is not something to decide per arm.** A mode that reads fewer cells than + /// the layout provides leaves the rest receiving nothing from `LfmMem` — + /// their multiplicity excludes it — so if anything then reads those columns + /// they are four free felts of prover choice and the row's output stops + /// being a function of its input. + /// + /// That is not hypothetical: it shipped. `MODE_L` reads one cell, the bus + /// and the validator were both taught so, the BLAKE3 arm pinned the unread + /// columns — and the `Test` and `Poseidon` arms, whose round 0 reads + /// `A_i = IN_i` for `i < 8`, were not. Under those two a leaf row carried + /// four unconstrained felts that the permutation consumed, which is a + /// Fiat–Shamir break for any program that absorbs data. Deriving the pins + /// from [`HashMode::num_input_cells`] here, once, is what stops the next + /// mode repeating it: an arm cannot forget a pin it does not write. + /// + /// Degree 2 (a selector sum times a column), so no arm's bound moves. + /// + /// Returns the next free constraint index. + pub(crate) fn emit_unread_input_pins>( + b: &mut B, + first_idx: usize, + ) -> usize { + let mut idx = first_idx; + // Cell 0 is read by every mode, so it is never pinned. A slot EVERY mode + // reads is skipped rather than pinned to nothing — which is the shape + // slot 1 took when the leaf RATE gave `Leaf` a second input cell. + for slot in 1..3usize { + let Some(sel) = MODE_SELECTORS + .iter() + .filter(|(_, mode)| mode.num_input_cells() <= slot) + .fold(None::, |acc, (col, _)| { + let term = b.main(0, *col); + Some(match acc { + None => term, + Some(a) => a + term, + }) + }) + else { + continue; + }; + for j in 0..4 { + let in_col = b.main(0, cols::IN0 + 4 * slot + j); + b.emit_base(idx, sel.clone() * in_col); + idx += 1; + } + } + debug_assert_eq!(idx - first_idx, NUM_UNREAD_INPUT_PINS); + idx + } + + /// The permutation the chip proves, chosen at construction. + /// + /// One struct with a runtime discriminant rather than one type per hasher: + /// `LfmAirs` holds `LfmAir` as a single field, so a + /// per-hasher type would force a trait object or an enum there instead. + pub struct HashConstraints { + pub kind: HasherKind, + } + + impl HashConstraints { + /// The `TestPermutation` configuration — the machine's pre-decision + /// default. `HashConstraints::default()` is the same thing. + pub const TEST: Self = Self { + kind: HasherKind::Test, + }; + + /// The BLAKE3 2-to-1 compress configuration. + pub const BLAKE3: Self = Self { + kind: HasherKind::Blake3, + }; + + /// The Rescue-Prime eXtended configuration. + pub const RPX: Self = Self { + kind: HasherKind::Rpx, + }; + + /// Constraints emitted under `kind` — the count the framework's + /// dense-index invariant requires `eval` to fill exactly. + pub const fn num_constraints(kind: HasherKind) -> usize { + match kind { + HasherKind::Test => 17 + NUM_UNREAD_INPUT_PINS, + HasherKind::Blake3 => crate::lfm::blake3_socket::NUM_CONSTRAINTS, + HasherKind::Rpx => rpx_cols::NUM_CONSTRAINTS, + HasherKind::Poseidon1W16 => crate::lfm::p1w16_socket::SOCKET_FORM.num_constraints(), + } + } + } + + impl Default for HashConstraints { + fn default() -> Self { + Self::TEST + } + } + + impl ConstraintSet for HashConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + match self.kind { + HasherKind::Test => Self::eval_test(b), + // The BLAKE3 arm lives in its own module: it shares the mixing + // dataflow with `blake3_chip` rather than with anything here, + // and putting it beside its column layout, its senders and its + // trace filler is what keeps the four in step. + HasherKind::Blake3 => crate::lfm::blake3_socket::eval(b), + HasherKind::Rpx => Self::eval_rpx(b), + // The width-16 socket lives in its own module, beside its + // layout, its bus contract and its row filler. + HasherKind::Poseidon1W16 => crate::lfm::p1w16_socket::P1W16SocketConstraints { + form: crate::lfm::p1w16_socket::SOCKET_FORM, + } + .eval(b), + } + } + } + + impl HashConstraints { + fn eval_test>(b: &mut B) { + let mode_c = b.main(0, cols::MODE_C); + let mode_t = b.main(0, cols::MODE_T); + let mode_l = b.main(0, cols::MODE_L); + let mode_p = b.main(0, cols::MODE_P); + + // idx 0–3: capacity-state copy — + // S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i. Transcript + // and leaf rows are one-cell-out steps like a compress row, so they + // take the same capacity; `TestPermutation` has one hash domain and + // is field-native, so all three compute the same function (see + // `LfmHasher::transcript_out` / `leaf_out` and their recorded + // weakening). + for (k, iv_raw) in TestPermutation::compress_iv_raw().into_iter().enumerate() { + let s = b.main(0, cols::S8 + k); + let in_i = b.main(0, cols::IN0 + 8 + k); + let iv_i = b.const_base(iv_raw); + let m = mode_c.clone() + mode_t.clone() + mode_l.clone(); + b.emit_base(k, s - (mode_p.clone() * in_i + m * iv_i)); + } + + // idx 4–15: the TestPermutation round — t_i = (A_i + rc_i·m)³ + // with A_i = IN_i (i < 8) or S_i (i ≥ 8) and m the mode sum; + // OUT_j = t_j + Σ_i t_i (mixing matrix M = I + J). The round + // constant is scaled by the mode sum so zero-filled padding rows + // satisfy the constraint (0 = 0) without a degree-4 gate: on real + // rows m = 1 and the permutation is unchanged. + // NON-CRYPTOGRAPHIC — this block behind the bus contract above is + // the hash-swap surface. + let t: Vec = (0..12) + .map(|i| { + let a = if i < 8 { + b.main(0, cols::IN0 + i) + } else { + b.main(0, cols::S8 + (i - 8)) + }; + let rc = b.const_base(canonical_u64(&TestPermutation::round_constant(i))); + let m = b.main(0, cols::MODE_C) + + b.main(0, cols::MODE_T) + + b.main(0, cols::MODE_L) + + b.main(0, cols::MODE_P); + let x = a + rc * m; + x.clone() * x.clone() * x + }) + .collect(); + let sum = t[1..].iter().fold(t[0].clone(), |acc, ti| acc + ti.clone()); + for (j, tj) in t.into_iter().enumerate() { + let out = b.main(0, cols::OUT0 + j); + b.emit_base(4 + j, out - (tj + sum.clone())); + } + + // idx 16: mode sum-boolean (exactly-one-of is the registrar's). + let mode_sum = mode_c + mode_t + mode_l + mode_p; + let one = b.one(); + b.emit_base(16, mode_sum.clone() * (one - mode_sum)); + + // idx 17–24: the unread input cells. ★ REQUIRED HERE, because the + // round above reads `IN_i` for every `i < 8` — including the four a + // leaf row does not read. See `emit_unread_input_pins`. + emit_unread_input_pins(b, 17); + } + + /// ★ Constraints 0–4 of every CAPACITY-SEPARATED algebraic tenant: the + /// per-mode capacity copy and the mode-sum booleanity. + /// + /// Shared rather than restated per arm, and that is the point. RPO and + /// RPX have identical socket geometry — same state width, same rate and + /// capacity, same four-felt digest, same three domain tags — so the one + /// place their AIRs could silently disagree about DOMAIN SEPARATION is + /// this prefix. One emitter means they cannot. + /// + /// `S_k = MODE_P·IN_{8+k} + MODE_C·IVC_k + MODE_T·IVT_k + MODE_L·IVL_k`. + /// A permutation row carries its own third input cell; every other mode + /// takes its domain's capacity. `DOMAIN_COMPRESS` is zero, so that arm + /// contributes nothing and a Merkle parent is a plain `merge`. Degree 2 + /// — a selector column times a constant. + /// + /// Returns the mode sum `m` — the row's is-real flag, which scales every + /// round constant so padding rows need no `IS_REAL` gate — and the + /// `MODE_P` selector. + fn emit_socket_prefix>(b: &mut B) -> (B::Expr, B::Expr) { + use crate::lfm::rpo::{DOMAIN_COMPRESS, DOMAIN_LEAF, DOMAIN_TRANSCRIPT, domain_iv}; + + let mode_c = b.main(0, cols::MODE_C); + let mode_t = b.main(0, cols::MODE_T); + let mode_l = b.main(0, cols::MODE_L); + let mode_p = b.main(0, cols::MODE_P); + let m = mode_c + mode_t + mode_l + mode_p.clone(); + + const MODE_IVS: [(usize, u64); 3] = [ + (cols::MODE_C, DOMAIN_COMPRESS), + (cols::MODE_T, DOMAIN_TRANSCRIPT), + (cols::MODE_L, DOMAIN_LEAF), + ]; + for k in 0..4 { + let s = b.main(0, cols::S8 + k); + let in_k = b.main(0, cols::IN0 + 8 + k); + let mut rhs = mode_p.clone() * in_k; + for (sel_col, domain) in MODE_IVS { + let iv_k = domain_iv(domain)[k]; + if iv_k != 0 { + let sel = b.main(0, sel_col); + rhs = rhs + sel * b.const_base(iv_k); + } + } + b.emit_base(k, s - rhs); + } + + let one = b.one(); + b.emit_base(4, m.clone() * (one - m.clone())); + (m, mode_p) + } + + /// Rescue-Prime eXtended at width 12: `FB E FB E FB E M`, one row per + /// permutation. + /// + /// **Degree is exactly 3, in all three round kinds.** The FB rounds are + /// RPO's round verbatim — forward `u^7 = (u³)²·u`, and + /// the inverse S-box verified as the FORWARD power via `(y³)²·y = v`. + /// The E rounds lower the EXTENSION seventh power the same way: + /// `t² = x·x` and `t³ = t²·x` are committed (degree 2 each), and the + /// output rides as `(t³)²·x`, degree 3. The M round is degree 1. + /// + /// **Each extension operation is THREE constraints, not one** — the + /// product of two `GF(p³)` elements is three base-field coefficients, so + /// the AIR states them coefficient by coefficient. That is the whole + /// reason an E round costs 36 columns where an FB round costs 60. + /// + /// **Padding by zero survives all three kinds.** With `m = 0`: an FB + /// round is RPO's argument unchanged; an E round has `x = 0`, so every + /// extension product is zero and `(0)²·0 = 0` holds; the M round is + /// `MDS·0 + 0 = 0`. No `IS_REAL` gate anywhere, so blowup 2 survives. + fn eval_rpx>(b: &mut B) { + use crate::lfm::rpo::{ARK1, ARK2, MDS_CIRC_ROW}; + use crate::lfm::rpx::{ + EXT_DEGREE, EXT_ELEMENTS, NUM_ROUNDS, is_fb_round, is_final_round, + }; + use rpx_cols as rc; + + let (m, mode_p) = Self::emit_socket_prefix::(b); + + fn mds>(b: &mut B, f: &[B::Expr], o: usize) -> B::Expr { + f.iter() + .enumerate() + .fold(None::, |acc, (i, fi)| { + let c = b.const_base( + MDS_CIRC_ROW[(i + HASH_STATE_FELTS - o) % HASH_STATE_FELTS], + ); + let term = c * fi.clone(); + Some(match acc { + None => term, + Some(x) => x + term, + }) + }) + .expect("twelve lanes") + } + + /// The cubic-extension product `a·b` reduced by `φ³ = φ + 1`, as + /// three coefficient expressions — the same closed form + /// `rpx::cubic_ext::mul` computes, so the chip is a transcription of + /// the host function rather than a second derivation of it. + fn ext_mul>( + a: &[B::Expr; EXT_DEGREE], + bb: &[B::Expr; EXT_DEGREE], + ) -> [B::Expr; EXT_DEGREE] { + let p = |i: usize, j: usize| a[i].clone() * bb[j].clone(); + [ + p(0, 0) + p(1, 2) + p(2, 1), + p(0, 1) + p(1, 0) + p(1, 2) + p(2, 1) + p(2, 2), + p(0, 2) + p(1, 1) + p(2, 0) + p(2, 2), + ] + } + + let mut state: Vec = (0..HASH_STATE_FELTS) + .map(|i| { + if i < 8 { + b.main(0, rc::IN0 + i) + } else { + b.main(0, rc::S8 + (i - 8)) + } + }) + .collect(); + + let mut idx = 5; + for r in 0..NUM_ROUNDS { + if is_final_round(r) { + // (M) — MDS then constants, straight into OUT. Degree 1. + for (o, ark) in ARK1[r].iter().enumerate() { + let mixed = mds(b, &state, o); + let rc_o = b.const_base(*ark); + let out = b.main(0, rc::out(r, o)); + b.emit_base(idx, out - (mixed + rc_o * m.clone())); + idx += 1; + } + } else if is_fb_round(r) { + // (FB) — RPO's round, verbatim. + let u: Vec = (0..HASH_STATE_FELTS) + .map(|o| { + let mixed = mds(b, &state, o); + let rc_o = b.const_base(ARK1[r][o]); + mixed + rc_o * m.clone() + }) + .collect(); + for (lane, u_lane) in u.iter().enumerate() { + let u2 = b.main(0, rc::u2(r, lane)); + let u3 = b.main(0, rc::u3(r, lane)); + b.emit_base(idx, u2.clone() - u_lane.clone() * u_lane.clone()); + b.emit_base(idx + 1, u3 - u2 * u_lane.clone()); + idx += 2; + } + let x: Vec = (0..HASH_STATE_FELTS) + .map(|lane| { + let u3 = b.main(0, rc::u3(r, lane)); + u3.clone() * u3 * u[lane].clone() + }) + .collect(); + let v: Vec = (0..HASH_STATE_FELTS) + .map(|o| { + let mixed = mds(b, &x, o); + let rc_o = b.const_base(ARK2[r][o]); + mixed + rc_o * m.clone() + }) + .collect(); + for (lane, v_lane) in v.into_iter().enumerate() { + let y = b.main(0, rc::out(r, lane)); + let y2 = b.main(0, rc::y2(r, lane)); + let y3 = b.main(0, rc::y3(r, lane)); + b.emit_base(idx, y2.clone() - y.clone() * y.clone()); + b.emit_base(idx + 1, y3.clone() - y2 * y.clone()); + b.emit_base(idx + 2, y3.clone() * y3 * y - v_lane); + idx += 3; + } + } else { + // (E) — constants then x^7 in GF(p³) on four triples. No MDS. + for e in 0..EXT_ELEMENTS { + let base = e * EXT_DEGREE; + let x: [B::Expr; EXT_DEGREE] = core::array::from_fn(|k| { + let rc_k = b.const_base(ARK1[r][base + k]); + state[base + k].clone() + rc_k * m.clone() + }); + let t2_cols: [B::Expr; EXT_DEGREE] = + core::array::from_fn(|k| b.main(0, rc::t2(r, e, k))); + let t3_cols: [B::Expr; EXT_DEGREE] = + core::array::from_fn(|k| b.main(0, rc::t3(r, e, k))); + + // t² = x·x, three coefficients, degree 2. + let sq = ext_mul::(&x, &x); + for (k, want) in sq.into_iter().enumerate() { + b.emit_base(idx, t2_cols[k].clone() - want); + idx += 1; + } + // t³ = t²·x, degree 2. + let cube = ext_mul::(&t2_cols, &x); + for (k, want) in cube.into_iter().enumerate() { + b.emit_base(idx, t3_cols[k].clone() - want); + idx += 1; + } + // out = (t³)²·x — degree 3, the extension fold. + let t6 = ext_mul::(&t3_cols, &t3_cols); + let seventh = ext_mul::(&t6, &x); + for (k, want) in seventh.into_iter().enumerate() { + let out = b.main(0, rc::out(r, base + k)); + b.emit_base(idx, out - want); + idx += 1; + } + } + } + + if !is_final_round(r) { + state = (0..HASH_STATE_FELTS) + .map(|j| b.main(0, rc::out(r, j))) + .collect(); + } + } + + let _ = mode_p; + idx = emit_unread_input_pins(b, idx); + debug_assert_eq!( + idx, + rpx_cols::NUM_CONSTRAINTS, + "every declared constraint index must be emitted exactly once" + ); + } + } +} + +// ========================================================================= +// LFM_KECCAK — the keccak-f[1600] adapter +// ========================================================================= +// +// Replaces the production `KECCAK` core chip, which is VM-coupled (it moves +// the state through timestamped `MEMW` tokens) and therefore unusable here. +// This chip owns exactly the core's two `Keccak` bus tokens and binds them to +// `LfmMem` words instead of memory. The permutation itself is proved by the +// UNCHANGED production `KECCAK_RND` / `KECCAK_RC` / `BITWISE` AIRs — see +// `keccak_adapter` and `keccak_probe`, which pin that contract standalone. +// +// CONSTRAINTS: none, and none are needed for the 400 state byte columns. +// Byte-ness is transitive: every IN byte is an operand of a `BYTE_ALU[XOR]` +// lookup in the round chip's θ column-parity chain (which covers all 25 lanes) +// and again in θ-final, and every OUT byte is the *result* of a `BYTE_ALU[XOR]` +// lookup (χ, or ι for lane 0). BYTE_ALU tokens carry the result as a tuple +// element, so a non-byte value finds no row in the 2^20 BITWISE table and the +// bus cannot balance. That in turn makes each `u32` half — a fixed linear +// combination of four such bytes — free of any separate range check: four +// values below 2^8 with coefficients 1, 2^8, 2^16, 2^24 cannot reach 2^32. + +pub mod keccak { + use super::*; + use crate::lfm::layout::keccak::{ + BLOCK_HALVES, BLOCK_WORDS, NUM_HALVES, NUM_WORDS, RATE_BYTES, RATE_LANES, + }; + use crate::tables::types::alu_op; + + pub mod cols { + pub use crate::lfm::layout::keccak::*; + /// The state as received from memory, 200 byte columns, lane-major: + /// `STATE + lane * 8 + b`. + pub const STATE: usize = PREP_WIDTH; // 56 + /// The rate block as received, 136 byte columns. Block byte `k` is byte + /// `k % 8` of lane `k / 8` — rate bytes are lane-major and + /// little-endian within a lane, exactly like the state columns, so + /// block byte `k` pairs with state byte `k`. (The column-major traversal + /// that bites elsewhere is a property of the *token element order*, not + /// of this column layout — see `keccak_token`.) + pub const BLOCK: usize = STATE + 200; // 256 + /// What enters the permutation: `STATE ⊕ BLOCK` over the rate region on + /// absorb rows, `STATE` everywhere else. + pub const PERM_IN: usize = BLOCK + RATE_BYTES; // 392 + /// The permuted state, 200 byte columns. + pub const OUT: usize = PERM_IN + 200; // 592 + pub const NUM_COLUMNS: usize = OUT + 200; // 792 + + pub const fn state_byte(lane: usize, b: usize) -> usize { + STATE + lane * 8 + b + } + pub const fn perm_in_byte(lane: usize, b: usize) -> usize { + PERM_IN + lane * 8 + b + } + pub const fn out_byte(lane: usize, b: usize) -> usize { + OUT + lane * 8 + b + } + } + + /// The row's is-real flag: exactly one mode on a real row, neither on + /// padding. + fn is_real() -> Multiplicity { + Multiplicity::Sum(cols::MODE_PERM, cols::MODE_ABSORB) + } + + /// Half `h` of the byte family at `bytes_start`, recomposed from its four + /// byte columns as `Σ byte_k · 256^k`. + /// + /// This is the trick the dropped core chip used to rebuild addresses from + /// byte columns (`tables/keccak.rs`): the machine-side value never gets its + /// own column, so there is nothing extra to keep consistent. Half slots at + /// or above `num_halves` are the family's unused top lanes and become tuple + /// constants — a nonzero value there cannot balance. + /// + /// Note `(h / 2) * 8 + 4 * (h % 2) == 4 * h`; the long form is kept because + /// it names why: half `h` is the low or high 4 bytes of lane `h / 2`. + fn half_value(bytes_start: usize, h: usize, num_halves: usize) -> BusValue { + if h >= num_halves { + return zero(); + } + let byte0 = bytes_start + (h / 2) * 8 + 4 * (h % 2); + BusValue::Linear( + (0..4) + .map(|k| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (8 * k), + column: byte0 + k, + }) + .collect(), + ) + } + + /// An `LfmMem` token for word `word` of a byte family: `(addr, h0..h3)`. + fn word_token_from_bytes( + addr_col: usize, + bytes_start: usize, + word: usize, + num_halves: usize, + ) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend((0..4).map(|l| half_value(bytes_start, 4 * word + l, num_halves))); + v + } + + /// Half `h` of the byte-reversed digest: reversed byte `j` is digest byte + /// `31 − j`, so this half's bytes are `OUT[31 − 4h − k]` for `k = 0..3` with + /// the usual little-endian coefficients. Both the byte order WITHIN a half + /// and the order OF the halves come out reversed, which is exactly what + /// reversing all 32 bytes means. + fn reversed_half_value(h: usize) -> BusValue { + BusValue::Linear( + (0..4) + .map(|k| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (8 * k), + column: cols::OUT + 31 - 4 * h - k, + }) + .collect(), + ) + } + + /// An `LfmMem` token for word `w` of the reversed digest. + fn reversed_digest_token(addr_col: usize, w: usize) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend((0..4).map(|l| reversed_half_value(4 * w + l))); + v + } + + /// A `Keccak` bus token: `(tag_lo, tag_hi, round, state[200])`. + /// + /// The 200 state elements are traversed **column-major over lanes** — + /// element `3 + 8·(5x + y) + b` is byte `b` of lane `x + 5y`, so lanes come + /// in the order 0, 5, 10, 15, 20, 1, 6, … That asymmetry is inherited from + /// the production sender's `for x { for y { … } }` loop over a + /// `(x + 5y)·8 + b` column formula; emitting them in natural order instead + /// leaves the bus unbalanced (falsification-verified in R1a). + #[allow(clippy::needless_range_loop)] + fn keccak_token(round: u64, bytes_start: usize) -> Vec { + let mut values = vec![ + direct(cols::TAG_LO), + direct(cols::TAG_HI), + BusValue::constant(round), + ]; + for x in 0..5 { + for y in 0..5 { + for b in 0..8 { + values.push(direct(bytes_start + (x + 5 * y) * 8 + b)); + } + } + } + values + } + + pub fn bus_interactions() -> Vec { + let mut interactions = Vec::with_capacity(2 * NUM_WORDS + BLOCK_WORDS + RATE_BYTES + 2); + // Reads: the 13 state words. + for j in 0..NUM_WORDS { + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + is_real(), + word_token_from_bytes(cols::in_addr(j), cols::STATE, j, NUM_HALVES), + )); + } + // Reads: the 9 rate-block words — absorb rows only, so on a permute row + // the BLOCK columns are read by nothing (no token, no lookup) and are + // simply dead witness. + for j in 0..BLOCK_WORDS { + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_ABSORB), + word_token_from_bytes(cols::block_addr(j), cols::BLOCK, j, BLOCK_HALVES), + )); + } + // Writes: the 13 output words, each with its own read count. + for j in 0..NUM_WORDS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::mult(j)), + word_token_from_bytes(cols::out_addr(j), cols::OUT, j, NUM_HALVES), + )); + } + // The absorb XOR, one BITWISE lookup per rate byte: + // `PERM_IN[k] = STATE[k] ⊕ BLOCK[k]`. + for k in 0..RATE_BYTES { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(cols::MODE_ABSORB), + vec![ + BusValue::constant(alu_op::XOR as u64), + direct(cols::STATE + k), + direct(cols::BLOCK + k), + direct(cols::PERM_IN + k), + ], + )); + } + // The reversed digest: the first 32 output bytes read back-to-front, + // as two words. This is the production transcript's `sample()` — it + // finalizes, reverses the digest in place, absorbs the reversed bytes + // and returns them, so one value serves as both the challenge and the + // next segment's prefix. + // + // Reversal is FREE at the recomposition boundary: the bus already + // rebuilds each `u32` half as a linear combination of four byte + // columns, so flipping the coefficient order (and the half order) is a + // different Linear over the SAME columns — no new value columns, no + // BitDec, no extra permutation. Rows that need no reversed digest leave + // `REV_MULT` at zero and these two sends are inert. + for w in 0..cols::DIGEST_WORDS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::rev_mult(w)), + reversed_digest_token(cols::rev_addr(w), w), + )); + } + // The request/reply pair that drives the production keccak family. + interactions.push(BusInteraction::sender( + BusId::Keccak, + is_real(), + keccak_token(0, cols::PERM_IN), + )); + interactions.push(BusInteraction::receiver( + BusId::Keccak, + is_real(), + keccak_token(24, cols::OUT), + )); + interactions + } + + pub struct KeccakAdapterConstraints; + + impl ConstraintSet for KeccakAdapterConstraints { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + // idx 0..63: the capacity region never absorbs, in either mode — + // `PERM_IN = STATE` for lanes 17..24, ungated (and trivially true on + // zero-filled padding rows). + for i in 0..(25 - RATE_LANES) * 8 { + let k = RATE_BYTES + i; + let s = b.main(0, cols::STATE + k); + let p = b.main(0, cols::PERM_IN + k); + b.emit_base(i, p - s); + } + // idx 64..199: on a permute row nothing is absorbed, so the rate + // region passes through too. On an absorb row this is gated off and + // the BYTE_ALU[XOR] lookups above pin PERM_IN instead. Without this, + // a permute row could feed the family a state unrelated to the one + // it read from memory. + let base = (25 - RATE_LANES) * 8; + for k in 0..RATE_BYTES { + let mode_perm = b.main(0, cols::MODE_PERM); + let s = b.main(0, cols::STATE + k); + let p = b.main(0, cols::PERM_IN + k); + b.emit_base(base + k, mode_perm * (p - s)); + } + // idx 200: mode sum-boolean (exactly-one-of is the registrar's). + let sum = b.main(0, cols::MODE_PERM) + b.main(0, cols::MODE_ABSORB); + let one = b.one(); + b.emit_base(base + RATE_BYTES, sum.clone() * (one - sum)); + } + } +} + +// ========================================================================= +// LFM_LANES — word ↔ lane conversion (Pack / Unpack) +// ========================================================================= +// +// Discovered as a real ISA gap in Milestone C: challenges are squeezed from +// the sponge as *cells*, but the ALU consumes base/ext operands, and no +// composition of the original eight ops can cross that boundary. The chip +// has NO constraints — the shared value columns appearing in both the word +// token and the four lane tokens IS the semantics. + +pub mod lanes { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::lanes::*; + pub const V0: usize = PREP_WIDTH; // ..V3 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + let mut interactions = vec![ + // Pack rows write the assembled word; Unpack rows read one. + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::WORD_MULT), + word_token(cols::WORD_ADDR, cols::V0), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_UNPACK), + word_token(cols::WORD_ADDR, cols::V0), + ), + ]; + for i in 0..4 { + // Unpack rows write the four lanes; Pack rows read them. + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::LANE_MULT0 + i), + base_token(cols::LANE_ADDR0 + i, cols::V0 + i), + )); + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_PACK), + base_token(cols::LANE_ADDR0 + i, cols::V0 + i), + )); + } + interactions + } +} + +// ========================================================================= +// LFM_HINT — arena ingestion (values unconstrained BY DESIGN; arena rule) +// ========================================================================= + +pub mod hint { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::hint::*; + pub const V0: usize = PREP_WIDTH; // ..V3 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + vec![BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + word_token(cols::OUT_ADDR, cols::V0), + )] + } +} + +// ========================================================================= +// LFM_PUBLIC — attestation output (COMMIT-bus closure pattern) +// ========================================================================= + +pub mod public { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::public::*; + pub const V0: usize = PREP_WIDTH; // ..V3 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + let mut send = vec![direct(cols::INDEX)]; + send.extend(word(cols::V0)); + vec![ + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + word_token(cols::IN_ADDR, cols::V0), + ), + BusInteraction::sender(BusId::LfmPublic, Multiplicity::Column(cols::IS_REAL), send), + ] + } +} + +// ========================================================================= +// LFM_RANGE — fixed 2^16 lookup table (idle in v0; the future hash chip's +// byte/limb tables land here) +// ========================================================================= + +pub mod range { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::range::*; + pub const MU: usize = PREP_WIDTH; + pub const NUM_COLUMNS: usize = PREP_WIDTH + 1; + } + + pub fn bus_interactions() -> Vec { + vec![BusInteraction::receiver( + BusId::LfmRange, + Multiplicity::Column(cols::MU), + vec![direct(cols::VALUE)], + )] + } +} diff --git a/prover/src/lfm/chunking.rs b/prover/src/lfm/chunking.rs new file mode 100644 index 000000000..e5d412e51 --- /dev/null +++ b/prover/src/lfm/chunking.rs @@ -0,0 +1,911 @@ +//! Row chunking — how the machine's splittable tables scale past one instance. +//! +//! Three chips are chunked, by the same mechanism and for the same reason: +//! [`KeccakChunking`] splits `KECCAK_RND`, [`Blake3Chunking`] splits +//! `LFM_BLAKE3`, and [`BaluChunking`] splits `LFM_BALU` — the first two because +//! their matrices are WIDE, the third because its matrix is TALL. Everything +//! the next paragraphs say about the first holds for the others; the +//! differences are collected under [`Blake3Chunking`] and [`BaluChunking`]. +//! +//! `KECCAK_RND` costs 24 rows per permutation at 1480 columns, so a single +//! instance saturates a 2^19-row table at ~21.8k permutations while a real +//! proof wrap needs ~460k. The RV64 VM solves the same problem for its own +//! tables by splitting them into chunk-AIRs; LFM does the same, with one +//! simplification: **the chunk count is static program shape**, fixed by +//! [`KeccakChunking`] at compile time, pinned in the registry and bound into +//! the program digest — never derived at prove time and never read off the +//! proof. +//! +//! # Why splitting the rows is free +//! +//! `KECCAK_RND` has no row-to-row transition constraints at all (its +//! [`ConstraintSet`](crate::tables::keccak_rnd::KeccakRndConstraints) is 20 +//! per-row `IS_BIT` checks). The 24-round chain is carried entirely by the +//! `Keccak` bus: row *r* receives `(tag, r, state)` and sends `(tag, r+1, +//! out)`, so consecutive rounds are linked by token *matching*, not by row +//! adjacency. LogUp balances the multiset over every AIR in the proof, so it +//! cannot tell which instance a row lived in. That is what makes chunking need +//! zero pairing logic — the same property the VM's chunked tables rely on. +//! +//! # What is *not* chunked +//! +//! `KECCAK_RC` and `BITWISE` stay single shared instances. Both are receivers +//! whose multiplicity columns count lookups from the whole proof: +//! `keccak_rc::update_multiplicities` writes the total permutation count into +//! every round row, and `bitwise::BitwiseHistogram` accumulates every operation +//! before the trace is filled. Per-chunk copies would each have to carry the +//! full histogram and would then over-receive. Their sizes are fixed anyway +//! (32 and 2^20 rows), so they never needed splitting. + +/// Trace rows one permutation occupies in `KECCAK_RND` — one per round. +pub const KECCAK_RND_ROWS_PER_PERMUTATION: usize = 24; + +/// Rows per `KECCAK_RND` chunk in the default policy. +/// +/// Retuning knob: this trades sub-proof count against per-chunk prover memory, +/// exactly like `max_rows` does for the VM's split tables. 2^19 rows is 21,845 +/// permutations per chunk. +pub const KECCAK_RND_MAX_CHUNK_ROWS: usize = 1 << 19; + +/// How a program's permutations are distributed over `KECCAK_RND` instances. +/// +/// Carried on [`LfmProgram`](super::compiler::LfmProgram), so trace generation +/// and artifact building read the same policy and cannot disagree. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeccakChunking { + permutations_per_chunk: usize, +} + +impl KeccakChunking { + /// The policy that fills chunks to at most `max_rows` trace rows. + /// + /// Panics at compile time (it is `const`) if `max_rows` cannot hold a + /// single permutation. + pub const fn from_max_rows(max_rows: usize) -> Self { + let permutations_per_chunk = max_rows / KECCAK_RND_ROWS_PER_PERMUTATION; + assert!( + permutations_per_chunk > 0, + "a KECCAK_RND chunk must hold at least one permutation (24 rows)" + ); + Self { + permutations_per_chunk, + } + } + + /// The policy that puts at most `permutations_per_chunk` permutations in + /// each chunk. The small-limit constructor tests use to force several + /// chunks out of a tiny program. + pub const fn from_permutations(permutations_per_chunk: usize) -> Self { + assert!( + permutations_per_chunk > 0, + "a KECCAK_RND chunk must hold at least one permutation" + ); + Self { + permutations_per_chunk, + } + } + + pub const fn permutations_per_chunk(self) -> usize { + self.permutations_per_chunk + } + + /// Number of `KECCAK_RND` instances a program with `num_permutations` + /// permutations gets — never zero, so the chip is present (and its + /// constraints verified) even for a program containing no keccak at all. + pub fn chunk_count(self, num_permutations: usize) -> usize { + num_permutations + .div_ceil(self.permutations_per_chunk) + .max(1) + } + + /// Splits per-permutation records into exactly [`Self::chunk_count`] + /// slices. The single rule both trace generation and the artifact/AIR + /// shape derive from; `split_agrees_with_chunk_count` pins the agreement. + pub fn split(self, permutations: &[T]) -> Vec<&[T]> { + if permutations.is_empty() { + vec![&permutations[..0]] + } else { + permutations.chunks(self.permutations_per_chunk).collect() + } + } +} + +impl Default for KeccakChunking { + fn default() -> Self { + Self::from_max_rows(KECCAK_RND_MAX_CHUNK_ROWS) + } +} + +/// Chunks `total` one-row records need at `per` records per chunk — never +/// zero, so a chip stays present for an empty program. The one rule +/// [`Blake3Chunking`] and [`BaluChunking`] share. +fn row_chunk_count(per: usize, total: usize) -> usize { + total.div_ceil(per).max(1) +} + +/// The half-open record range chunk `chunk` covers at `per` records per +/// chunk, clamped to `total`. +fn row_chunk_range(per: usize, total: usize, chunk: usize) -> core::ops::Range { + let start = per.saturating_mul(chunk).min(total); + let end = start.saturating_add(per).min(total); + start..end +} + +/// The environment knob that turns `LFM_BLAKE3` chunking on, read at program +/// EMISSION time by the driver that emits the program. +/// +/// Unset means one table — today's machine, byte for byte. Set to `k` means +/// chunks of at most `2^k` rows, i.e. `2^k` compressions. +pub const BLAKE3_MAX_CHUNK_ROWS_LOG2_ENV: &str = "LFM_BLAKE3_MAX_CHUNK_ROWS_LOG2"; + +/// Trace rows one compression occupies in `LFM_BLAKE3` — exactly one. +/// +/// Named rather than inlined so the row arithmetic below reads the same as +/// [`KECCAK_RND_ROWS_PER_PERMUTATION`]'s, and so a chip that ever compressed +/// over several rows would be a one-line change here instead of a hunt. +pub const BLAKE3_ROWS_PER_COMPRESSION: usize = 1; + +/// How a program's compressions are distributed over `LFM_BLAKE3` instances. +/// +/// Carried on [`LfmProgram`](super::compiler::LfmProgram) beside +/// [`KeccakChunking`], read by trace generation and artifact building alike, and +/// bound into the program digest — the same discipline, for the same reason. +/// +/// # Why this chip needs it +/// +/// `LFM_BLAKE3` is one row per compression at 3,056 value columns, so its +/// matrix is WIDE rather than tall: the aggregation program's ~1.39M +/// compressions land in a 2^21 x 3,056 table whose blowup-2 LDE is a single +/// ~102 GB allocation. Splitting the rows turns that one transient into `n` +/// independent ones without changing a byte of what is proved. +/// +/// # Why splitting the rows is free +/// +/// Same property [`KeccakChunking`] rests on, checked on this chip: every +/// constraint `Blake3LfmConstraints` emits reads `main(0, ..)` — there is no +/// row-to-row coupling at all — and every bus interaction is a within-row token +/// gated by `MU` or by a per-word multiplicity column. A compression's inputs +/// and outputs travel on the `LfmMem` bus by address matching, and the addresses +/// are PREPROCESSED program data, so which instance a row lives in is invisible +/// to the balance. Unlike `LFM_KECCAK`, the chip carries no row-ordinal tag, so +/// there is not even a positional value to preserve. +/// +/// # ★ Where this differs from `KECCAK_RND`, and it matters +/// +/// `KECCAK_RND` has NO preprocessed columns, so its chunk count moves no root: +/// every instance is the identical AIR. `LFM_BLAKE3` carries an instruction +/// column group (addresses, multiplicities, `MU` — 20 columns), so **each chunk +/// is its own committed matrix with its own Merkle root and its own height**. +/// Chunk 0's root is slot 11's entry in the roots array — which is why a +/// single-chunk program is bit-identical to an unchunked one — and the roots of +/// chunks 1.. ride [`LfmArtifacts`](super::registry::LfmArtifacts) and are +/// folded into `program_id`. A chunked program is therefore a different program +/// identity by name, not merely a different layout. +/// +/// # What is *not* chunked +/// +/// `BITWISE`. The chip is a `ByteAlu`/`AreBytes` sender ~1,248 times per row, +/// and `BITWISE` is the shared receiver whose multiplicity columns count the +/// lookups of the WHOLE proof — `bitwise_ops_for` is handed the complete record +/// list regardless of how the rows were split. Per-chunk copies would each carry +/// the full histogram and over-receive. This is the same exclusion +/// [`KeccakChunking`] records for `KECCAK_RC` and `BITWISE`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Blake3Chunking { + compressions_per_chunk: usize, +} + +impl Blake3Chunking { + /// One table, whatever the compression count — **the default**, and the + /// machine as it stood before chunking existed. + /// + /// `usize::MAX` rather than an `Option` so there is ONE code path: every + /// derivation below is the same arithmetic whether chunking is on or off, + /// which is what makes "knob unset" mean "one chunk" rather than "skip the + /// chunking code". + pub const fn unbounded() -> Self { + Self { + compressions_per_chunk: usize::MAX, + } + } + + /// The policy that fills chunks to at most `max_rows` trace rows. + pub const fn from_max_rows(max_rows: usize) -> Self { + let compressions_per_chunk = max_rows / BLAKE3_ROWS_PER_COMPRESSION; + assert!( + compressions_per_chunk > 0, + "an LFM_BLAKE3 chunk must hold at least one compression" + ); + Self { + compressions_per_chunk, + } + } + + /// The policy that puts at most `compressions_per_chunk` compressions in + /// each chunk. The small-limit constructor tests use to force several chunks + /// out of a tiny program. + pub const fn from_compressions(compressions_per_chunk: usize) -> Self { + assert!( + compressions_per_chunk > 0, + "an LFM_BLAKE3 chunk must hold at least one compression" + ); + Self { + compressions_per_chunk, + } + } + + /// The policy [`BLAKE3_MAX_CHUNK_ROWS_LOG2_ENV`] names, or `None` when it is + /// unset. + /// + /// `None` rather than [`Self::unbounded`] so a caller can tell "the operator + /// chose one table" from "the operator said nothing" and print accordingly; + /// both produce the same shape. + /// + /// # Panics + /// + /// On a value that is not a `u32`, or one at or above `usize::BITS`. This is + /// read once, at program-emission time, by a driver an operator launched — + /// a typo there must stop the run, not silently prove a different shape than + /// the one asked for. + pub fn from_env() -> Option { + Self::from_env_value( + std::env::var(BLAKE3_MAX_CHUNK_ROWS_LOG2_ENV) + .ok() + .as_deref(), + ) + } + + /// [`Self::from_env`] with the variable's value supplied. + /// + /// Split out so the parse is testable without mutating process-global state: + /// `std::env::set_var` races every other thread of a parallel test binary, + /// and a knob whose parsing is untested is a knob that silently proves the + /// wrong shape. + pub fn from_env_value(raw: Option<&str>) -> Option { + let raw = raw?; + let log2: u32 = raw.parse().unwrap_or_else(|_| { + panic!("{BLAKE3_MAX_CHUNK_ROWS_LOG2_ENV} must be a base-2 row exponent, got {raw:?}") + }); + assert!( + log2 < usize::BITS, + "{BLAKE3_MAX_CHUNK_ROWS_LOG2_ENV}={log2} is not a representable row count" + ); + Some(Self::from_max_rows(1usize << log2)) + } + + pub const fn compressions_per_chunk(self) -> usize { + self.compressions_per_chunk + } + + /// Number of `LFM_BLAKE3` instances a program with `num_compressions` + /// compressions gets — never zero, so the chip is present (and its + /// constraints verified) even for a program containing no BLAKE3 at all. + /// The chip MASK, not this, is what drops an unused family; see + /// [`ChipSet::blake3_chunks`](super::airs::ChipSet::blake3_chunks). + pub fn chunk_count(self, num_compressions: usize) -> usize { + row_chunk_count(self.compressions_per_chunk, num_compressions) + } + + /// The half-open row range chunk `chunk` covers, clamped to + /// `num_compressions`. The single rule the group split, the record split and + /// the census heights all read, so they cannot disagree about a boundary. + pub fn chunk_range(self, num_compressions: usize, chunk: usize) -> core::ops::Range { + row_chunk_range(self.compressions_per_chunk, num_compressions, chunk) + } + + /// Splits per-compression records into exactly [`Self::chunk_count`] + /// slices — the analogue of [`KeccakChunking::split`]. + pub fn split(self, compressions: &[T]) -> Vec<&[T]> { + (0..self.chunk_count(compressions.len())) + .map(|c| &compressions[self.chunk_range(compressions.len(), c)]) + .collect() + } +} + +impl Default for Blake3Chunking { + fn default() -> Self { + Self::unbounded() + } +} + +/// The environment knob that turns `LFM_BALU` chunking on, read at program +/// EMISSION time like [`BLAKE3_MAX_CHUNK_ROWS_LOG2_ENV`]. +/// +/// Unset means one table — today's machine, byte for byte. Set to `k` means +/// chunks of at most `2^k` rows, i.e. `2^k` ALU operations. +pub const BALU_MAX_CHUNK_ROWS_LOG2_ENV: &str = "LFM_BALU_MAX_CHUNK_ROWS_LOG2"; + +/// Trace rows one ALU operation occupies in `LFM_BALU` — exactly one +/// (`emit_column_groups` opens one row per `Balu` instruction). +pub const BALU_ROWS_PER_OP: usize = 1; + +/// The chunk height the sizing under [`BaluChunking`] arrives at: `2^22` rows. +pub const BALU_TARGET_CHUNK_ROWS_LOG2: u32 = 22; + +/// How a program's ALU operations are distributed over `LFM_BALU` instances — +/// the row-chunking arm for the machine's TALL-NARROW chips. +/// +/// # Why this chip needs it +/// +/// `LFM_BALU` is one row per operation at 4 value + 10 preprocessed columns: +/// narrow, and on the aggregator very tall — the program pads to `2^27` rows +/// at 110 queries and `2^28` at 219, a census contribution that is trivial in +/// cells and enormous in rows. At blowup 2 the ONE-table device set of the R1 +/// commit alone is +/// +/// | rows | LDE `2n·14·8` | snapshot `n·14·8` | tree `(2n−1)·32` | R1 set | +/// |------|---------------|-------------------|------------------|--------| +/// | 2^27 | 28.0 GiB | 14.0 GiB | 8.0 GiB | 50 GiB | +/// | 2^22 | 0.875 GiB | 0.44 GiB | 0.25 GiB | 1.6 GiB| +/// +/// and rounds 2–4 add the aux LDE and its tree, the resident aux trace, `H` +/// and the two composition parts with their tree, the inverted denominators, +/// the DEEP codeword and the FRI chain — the scheduler's throttle model, +/// `stark::device_set::table_device_set`, which +/// `the_balu_chunk_sizing_is_the_doc` reads rather than restates: ~150 GiB at +/// `2^27` against a 32 GiB card, ~4.9 GiB per chunk at `2^22`, so five chunks +/// prove concurrently inside a 25.6 GiB admission budget. `2^22` is the +/// target because `2^24` chunks (~19 GiB each) would serialise the chunks +/// outright, and `2^20` chunks would quadruple the per-chunk FRI and query +/// overhead the verifier pays for nothing. That overhead — one FRI commit and +/// one set of openings PER CHUNK — is the counter-pressure against smaller +/// chunks, and the reason the default stays one table until the aggregator is +/// emitted with the knob set. +/// +/// # Why row chunking, not column streaming +/// +/// Streaming the commit column group by column group lowers only the commit's +/// own peak; rounds 2–4 read every column of every row from the RESIDENT LDE, +/// so the `2n · cols · 8` buffer has to be on the card for the whole table +/// however the leaves were hashed. For a tall-narrow chip that buffer IS the +/// problem (28 GiB at `2^27`), and nothing short of splitting the rows shrinks +/// it. Column streaming is the shape for SHORT-WIDE chips (`LFM_HASH` at +/// `2^21 × 449`), and since the fused commit transposes its one LDE buffer in +/// place it buys little even there: the LDE stays resident for rounds 2–4 +/// either way. +/// +/// # Why splitting the rows is free +/// +/// The property [`KeccakChunking`] and [`Blake3Chunking`] rest on, checked on +/// this chip: every constraint `BaluConstraints` emits reads `main(0, ..)` — +/// no row-to-row coupling — and every bus interaction is a within-row `LfmMem` +/// token gated by the row's own selector or multiplicity column. Operands and +/// results travel by address matching, and the addresses are PREPROCESSED +/// program data, so which instance a row lives in is invisible to the balance. +/// +/// # What it costs +/// +/// Like `LFM_BLAKE3` and unlike `KECCAK_RND`, this chip carries a preprocessed +/// instruction group, so **each chunk is its own committed matrix with its +/// own root and its own height**: a chunked program is a different program +/// identity by name, and the chunk roots ride the artifacts and fold into +/// `program_id` exactly as the BLAKE3 chunk roots do. Wiring — the program +/// field, the per-chunk group, the AIR instances, the artifact roots and the +/// slot map — follows the BLAKE3 template one for one and is not in this +/// module. +/// +/// `LFM_LANES` (4 value + 12 preprocessed, `2^24` rows in the 110-query wrap) +/// is the next chip of this shape; its device set at `2^24` is ~23 GiB under +/// the same model — it proves alone — one doubling from needing the same arm. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BaluChunking { + ops_per_chunk: usize, +} + +impl BaluChunking { + /// One table, whatever the operation count — **the default**, and the + /// machine as it stands before the aggregator is emitted chunked. + /// `usize::MAX` rather than an `Option` for [`Blake3Chunking::unbounded`]'s + /// reason: one code path. + pub const fn unbounded() -> Self { + Self { + ops_per_chunk: usize::MAX, + } + } + + /// The policy that fills chunks to at most `max_rows` trace rows. + pub const fn from_max_rows(max_rows: usize) -> Self { + let ops_per_chunk = max_rows / BALU_ROWS_PER_OP; + assert!( + ops_per_chunk > 0, + "an LFM_BALU chunk must hold at least one operation" + ); + Self { ops_per_chunk } + } + + /// The policy that puts at most `ops_per_chunk` operations in each chunk. + /// The small-limit constructor tests use to force several chunks out of a + /// tiny program. + pub const fn from_ops(ops_per_chunk: usize) -> Self { + assert!( + ops_per_chunk > 0, + "an LFM_BALU chunk must hold at least one operation" + ); + Self { ops_per_chunk } + } + + /// The sizing's target: chunks of `2^22` rows + /// ([`BALU_TARGET_CHUNK_ROWS_LOG2`]). + pub const fn target() -> Self { + Self::from_max_rows(1usize << BALU_TARGET_CHUNK_ROWS_LOG2) + } + + /// The policy [`BALU_MAX_CHUNK_ROWS_LOG2_ENV`] names, or `None` when it is + /// unset — on [`Blake3Chunking::from_env`]'s terms, including the panic on + /// a value that is not a row exponent. + pub fn from_env() -> Option { + Self::from_env_value(std::env::var(BALU_MAX_CHUNK_ROWS_LOG2_ENV).ok().as_deref()) + } + + /// [`Self::from_env`] with the variable's value supplied, so the parse is + /// testable without mutating process-global state. + pub fn from_env_value(raw: Option<&str>) -> Option { + let raw = raw?; + let log2: u32 = raw.parse().unwrap_or_else(|_| { + panic!("{BALU_MAX_CHUNK_ROWS_LOG2_ENV} must be a base-2 row exponent, got {raw:?}") + }); + assert!( + log2 < usize::BITS, + "{BALU_MAX_CHUNK_ROWS_LOG2_ENV}={log2} is not a representable row count" + ); + Some(Self::from_max_rows(1usize << log2)) + } + + pub const fn ops_per_chunk(self) -> usize { + self.ops_per_chunk + } + + /// Number of `LFM_BALU` instances a program with `num_ops` operations gets + /// — never zero, so the chip is present (and its constraints verified) + /// even for a program containing no ALU operation at all. + pub fn chunk_count(self, num_ops: usize) -> usize { + row_chunk_count(self.ops_per_chunk, num_ops) + } + + /// The half-open row range chunk `chunk` covers, clamped to `num_ops` — + /// the single rule a group split, a record split and a census height + /// would all read. + pub fn chunk_range(self, num_ops: usize, chunk: usize) -> core::ops::Range { + row_chunk_range(self.ops_per_chunk, num_ops, chunk) + } + + /// Splits per-operation records into exactly [`Self::chunk_count`] slices. + pub fn split(self, ops: &[T]) -> Vec<&[T]> { + (0..self.chunk_count(ops.len())) + .map(|c| &ops[self.chunk_range(ops.len(), c)]) + .collect() + } +} + +impl Default for BaluChunking { + fn default() -> Self { + Self::unbounded() + } +} + +/// ★ Whether `LFM_HASH` splits into two power-of-two instances where one table +/// would be mostly padding ([`HashChunking::for_rows`]): a PER-PIPELINE default, +/// each pipeline's measured best — off on the WHIR pipeline +/// (`whir/recursion-rpx`), on for the STARK pipeline (`per-table-gpu`). +/// [`HASH_SPLIT_ENV`] overrides it either way. +/// +/// Why the two differ, measured on block 25368371 (one RTX 5090, ABBA, on top of +/// the `BITWISE` drop): the STARK wraps carry 276–299k hash rows in a `2^19` +/// table and split to `2^18 + 2^15`, removing 74.5 M census cells each, and the +/// STARK block ran 1.90 s faster (level 0 −1.80 s). The WHIR wraps carry +/// 172–195k rows in `2^18` and a split removes only `2^16` rows, 21.3 M cells — +/// less than the extra sub-proof costs the wrap, and each split child also +/// costs its parent one more sub-proof to re-verify, which took four of the +/// five fan-in-3 level-1 nodes across a power of two in `LFM_XALU`: the WHIR +/// block ran 2.80 s slower. +/// +/// Read once per process by [`super::compiler::compile`], so every program the +/// process compiles carries the same policy, and the split it produces is +/// program shape: committed per chunk, bound into `program_id`, never read off +/// a proof. Off, every program is one table — the machine before the split +/// existed, byte for byte. +pub const HASH_SPLIT_DEFAULT: bool = false; + +/// `LAMBDA_VM_LFM_HASH_SPLIT`: `0` keeps one table, `1` splits by the rule, +/// unset is [`HASH_SPLIT_DEFAULT`]. Anything else stops the run. +pub const HASH_SPLIT_ENV: &str = "LAMBDA_VM_LFM_HASH_SPLIT"; + +/// [`HASH_SPLIT_ENV`]'s reading of a raw value. +pub fn hash_split_setting(raw: Option<&str>) -> bool { + super::airs::env_switch(HASH_SPLIT_ENV, raw).unwrap_or(HASH_SPLIT_DEFAULT) +} + +/// Whether this process splits `LFM_HASH` by the rule. Read once; the setting is +/// named on stderr either way, so a log states which programs it proved. +pub fn hash_split_enabled() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + let raw = std::env::var(HASH_SPLIT_ENV).ok(); + let on = hash_split_setting(raw.as_deref()); + let source = match raw.as_deref().map(str::trim) { + Some(v) if !v.is_empty() => HASH_SPLIT_ENV, + _ => "the pipeline default", + }; + super::airs::announce(&format!( + "LFM HASH SPLIT: {} ({source})", + if on { "on" } else { "off" } + )); + on + }) +} + +/// The smallest padding (as a power of two of rows) a split must remove to be +/// taken: `2^15` rows, about ten million committed cells at `LFM_HASH`'s width. +/// +/// The counter-pressure is the instance itself and the level above: every extra +/// instance is one more sub-proof to prove and one more for its parent to +/// re-verify — on the block, about 15–20k more hash rows and 185–271k more +/// instructions in the parent per split child, enough to take a parent across a +/// power of two (the WHIR level-1 nodes' `LFM_XALU`). Below this floor the split +/// moves cells up a level instead of removing them; above it the pipeline's +/// measurement decides ([`HASH_SPLIT_DEFAULT`]). +pub const HASH_SPLIT_MIN_SAVING_LOG2: u32 = 15; + +/// How a program's hash rows are spread over `LFM_HASH` instances — at most +/// two, split at a power of two. +/// +/// # Why this chip, and why this rule +/// +/// `LFM_HASH` is the widest chip every recursion proof carries (329 value +/// columns under RPX) and its height is a power of two like every other, so a +/// program with `R` hash rows pays for `R.next_power_of_two()` of them. On the +/// block that is 56 % fill in the typical STARK wrap — 293,778 rows committed as +/// 524,288, 75.8 M cells of padding. Two +/// instances of `2^a` and `(R − 2^a).next_power_of_two()` rows, `2^a` the largest +/// power of two below `R`, commit 294,912 rows for the same program. +/// +/// The split is taken only when it removes at least +/// `2^`[`HASH_SPLIT_MIN_SAVING_LOG2`] rows, so a table that is already mostly +/// full (`R` above three quarters of its padded height) stays one instance. +/// +/// # Why splitting the rows is free +/// +/// The property [`Blake3Chunking`] rests on, checked on this chip: every +/// constraint `HashConstraints` emits reads the current row only — one +/// permutation (or leaf absorb) per row, no row-to-row coupling — and every bus +/// interaction is a within-row `LfmMem` token gated by the row's own mode and +/// multiplicity columns, whose addresses are PREPROCESSED program data. Which +/// instance a row lives in is invisible to the balance. +/// +/// # What it costs +/// +/// Like `LFM_BLAKE3`, the chip carries a preprocessed instruction group, so each +/// chunk is its own committed matrix with its own root and height: chunk 0 is +/// slot 5's entry, chunk 1 rides the artifacts and folds into `program_id` as a +/// tail. A split program is a different program identity by name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HashChunking { + /// Rows of chunk 0 — the split point — or `None` for one table. + split: Option, +} + +impl HashChunking { + /// One table, whatever the row count — **the default**. + pub const fn unbounded() -> Self { + Self { split: None } + } + + /// Chunk 0 holds exactly `rows` rows and chunk 1 the rest. The constructor + /// tests use to force a split of a tiny program. + pub const fn split_at(rows: usize) -> Self { + assert!(rows > 0, "an LFM_HASH chunk must hold at least one row"); + Self { split: Some(rows) } + } + + /// The split rule for a program with `real_rows` hash rows: split at the + /// largest power of two below `real_rows` when that removes at least + /// `2^HASH_SPLIT_MIN_SAVING_LOG2` padded rows, else one table. + pub fn for_rows(real_rows: usize) -> Self { + if real_rows <= 1 || real_rows.is_power_of_two() { + return Self::unbounded(); + } + let first = 1usize << (usize::BITS - 1 - real_rows.leading_zeros()); + let one_table = super::layout::padded_rows(real_rows); + let split = first + super::layout::padded_rows(real_rows - first); + if one_table.saturating_sub(split) >= 1usize << HASH_SPLIT_MIN_SAVING_LOG2 { + Self::split_at(first) + } else { + Self::unbounded() + } + } + + /// The policy this process compiles under: [`Self::for_rows`] when it splits + /// ([`hash_split_enabled`]), one table otherwise. + pub fn for_process(real_rows: usize) -> Self { + if hash_split_enabled() { + Self::for_rows(real_rows) + } else { + Self::unbounded() + } + } + + /// `LFM_HASH` instances for `num_rows` hash rows — one or two, never zero. + pub fn chunk_count(self, num_rows: usize) -> usize { + match self.split { + Some(first) if num_rows > first => 2, + _ => 1, + } + } + + /// The half-open row range chunk `chunk` covers, clamped to `num_rows` — the + /// single rule the group split, the record split and the census read. + pub fn chunk_range(self, num_rows: usize, chunk: usize) -> core::ops::Range { + match (self.split, chunk) { + (Some(first), 0) => 0..first.min(num_rows), + (Some(first), 1) => first.min(num_rows)..num_rows, + (None, 0) => 0..num_rows, + _ => num_rows..num_rows, + } + } +} + +impl Default for HashChunking { + fn default() -> Self { + Self::unbounded() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_policy_is_the_documented_geometry() { + let c = KeccakChunking::default(); + assert_eq!(c.permutations_per_chunk(), 21845); + assert!( + c.permutations_per_chunk() * KECCAK_RND_ROWS_PER_PERMUTATION + <= KECCAK_RND_MAX_CHUNK_ROWS + ); + // One chunk up to the limit, two past it. + assert_eq!(c.chunk_count(21845), 1); + assert_eq!(c.chunk_count(21846), 2); + // The ~460k-permutation wrap the design targets. + assert_eq!(c.chunk_count(460_000), 22); + } + + #[test] + fn empty_programs_still_get_one_chunk() { + for per in [1usize, 2, 7, 21845] { + let c = KeccakChunking::from_permutations(per); + assert_eq!(c.chunk_count(0), 1); + assert_eq!(c.split::(&[]).len(), 1); + assert!(c.split::(&[])[0].is_empty()); + } + } + + /// `split` and `chunk_count` are the same rule seen twice; if they ever + /// disagree the prover builds a different number of traces than the + /// verifier builds AIRs. + #[test] + fn split_agrees_with_chunk_count() { + for per in [1usize, 2, 3, 5, 24] { + let c = KeccakChunking::from_permutations(per); + for n in 0..40usize { + let ops: Vec = (0..n).collect(); + let split = c.split(&ops); + assert_eq!( + split.len(), + c.chunk_count(n), + "per={per} n={n}: split and chunk_count disagree" + ); + assert_eq!( + split.iter().map(|s| s.len()).sum::(), + n, + "per={per} n={n}: split lost or duplicated permutations" + ); + assert!( + split.iter().all(|s| s.len() <= per), + "per={per} n={n}: a chunk exceeded the limit" + ); + } + } + } + + /// The default is ONE table at any scale — the property that makes the + /// knob-unset machine the machine that was there before. + #[test] + fn the_blake3_default_is_a_single_table() { + let c = Blake3Chunking::default(); + assert_eq!(c, Blake3Chunking::unbounded()); + for n in [0usize, 1, 1_000, 1_390_000, usize::MAX - 1] { + assert_eq!(c.chunk_count(n), 1, "n={n} must stay one table"); + } + } + + /// `2^k` rows is `2^k` compressions: the chip is one row per compression, so + /// the row knob and the compression count are the same number. + #[test] + fn the_blake3_row_cap_is_a_compression_cap() { + for log2 in [3usize, 10, 18] { + let c = Blake3Chunking::from_max_rows(1 << log2); + assert_eq!(c.compressions_per_chunk(), 1 << log2); + assert_eq!(c.chunk_count(1 << log2), 1); + assert_eq!(c.chunk_count((1 << log2) + 1), 2); + } + // The aggregation program's shape at the 2^18 target. + assert_eq!( + Blake3Chunking::from_max_rows(1 << 18).chunk_count(1_390_000), + 6 + ); + } + + /// `split`, `chunk_count` and `chunk_range` are one rule seen three times; + /// if they ever disagree the prover builds a different number of traces than + /// the verifier builds AIRs, or a chunk's rows and its records come from + /// different boundaries. + #[test] + fn blake3_split_agrees_with_chunk_count_and_range() { + for per in [1usize, 2, 3, 5, 8] { + let c = Blake3Chunking::from_compressions(per); + for n in 0..40usize { + let ops: Vec = (0..n).collect(); + let split = c.split(&ops); + assert_eq!( + split.len(), + c.chunk_count(n), + "per={per} n={n}: split and chunk_count disagree" + ); + assert_eq!( + split.iter().map(|s| s.len()).sum::(), + n, + "per={per} n={n}: split lost or duplicated compressions" + ); + assert!( + split.iter().all(|s| s.len() <= per), + "per={per} n={n}: a chunk exceeded the limit" + ); + for (i, s) in split.iter().enumerate() { + let r = c.chunk_range(n, i); + assert_eq!( + &ops[r], *s, + "per={per} n={n} chunk {i}: chunk_range and split disagree" + ); + } + } + } + } + + /// The knob's own parse: unset is one table, and a value is a row exponent. + #[test] + fn the_blake3_env_knob_parses_a_row_exponent() { + assert_eq!(Blake3Chunking::from_env_value(None), None); + for log2 in [0usize, 3, 18, 21] { + assert_eq!( + Blake3Chunking::from_env_value(Some(&log2.to_string())), + Some(Blake3Chunking::from_max_rows(1 << log2)), + "{log2} must name 2^{log2} rows per chunk" + ); + } + // The aggregation program's target: 1.39M compressions at 2^18. + assert_eq!( + Blake3Chunking::from_env_value(Some("18")) + .expect("set") + .chunk_count(1_390_000), + 6 + ); + } + + /// A typo stops the run rather than silently proving a different shape. + #[test] + #[should_panic(expected = "must be a base-2 row exponent")] + fn a_malformed_blake3_knob_panics() { + let _ = Blake3Chunking::from_env_value(Some("2^18")); + } + + /// An empty program still gets one chunk, so the chip stays in the set. + #[test] + fn empty_blake3_programs_still_get_one_chunk() { + for per in [1usize, 2, 7] { + let c = Blake3Chunking::from_compressions(per); + assert_eq!(c.chunk_count(0), 1); + assert_eq!(c.split::(&[]).len(), 1); + assert!(c.split::(&[])[0].is_empty()); + assert_eq!(c.chunk_range(0, 0), 0..0); + } + } + + /// The default is ONE `LFM_BALU` table at any scale — the machine as it + /// stands. + #[test] + fn the_balu_default_is_a_single_table() { + let c = BaluChunking::default(); + assert_eq!(c, BaluChunking::unbounded()); + for n in [0usize, 1, 1 << 27, 1 << 28, usize::MAX - 1] { + assert_eq!(c.chunk_count(n), 1, "n={n} must stay one table"); + } + assert_eq!(BaluChunking::from_env_value(None), None); + } + + /// The target policy splits the aggregator's `2^27` (110 q) and `2^28` + /// (219 q) rows into 32 and 64 chunks of `2^22`; the knob names the same + /// policy. + #[test] + fn the_balu_target_sizes_the_aggregator() { + let c = BaluChunking::target(); + assert_eq!(c.ops_per_chunk(), 1 << 22); + assert_eq!(c.chunk_count(1 << 27), 32); + assert_eq!(c.chunk_count(1 << 28), 64); + assert_eq!(c.chunk_count((1 << 27) + 1), 33); + assert_eq!(BaluChunking::from_env_value(Some("22")), Some(c)); + for log2 in [0usize, 3, 18, 22] { + assert_eq!( + BaluChunking::from_env_value(Some(&log2.to_string())), + Some(BaluChunking::from_max_rows(1 << log2)), + "{log2} must name 2^{log2} rows per chunk" + ); + } + } + + /// The device-set arithmetic the `BaluChunking` doc tabulates, read from + /// the scheduler's own model (`stark::device_set`) so the doc cannot drift + /// from what the throttle admits: one table at `2^27` does not fit a + /// 32 GiB card; a `2^22` chunk is ~4.9 GiB, so five prove concurrently + /// inside the 25.6 GiB budget and a `2^24` chunk proves alone. Columns: + /// 14 base (4 value + 10 preprocessed), 2 ext3 aux, 2 composition parts, + /// 2 OOD points, blowup 2. + #[test] + fn the_balu_chunk_sizing_is_the_doc() { + use stark::device_set::{TableShape, table_device_set}; + const GIB: u64 = 1 << 30; + const BUDGET: u64 = 32 * GIB / 5 * 4; + let balu = |n: usize| { + table_device_set(TableShape { + n, + blowup: 2, + main_cols: 14, + aux_cols: 2, + num_parts: 2, + num_eval_points: 2, + }) + }; + let one_table = balu(1 << 27); + assert!(one_table.main.total() > 50 * GIB && one_table.main.total() < 52 * GIB); + assert!( + one_table.total() > 148 * GIB && one_table.total() < 152 * GIB, + "{one_table:?}" + ); + let chunk = balu(1 << 22).total(); + assert!( + chunk > 4 * GIB + GIB / 2 && chunk < 5 * GIB + GIB / 2, + "{chunk}" + ); + assert!(5 * chunk <= BUDGET && 6 * chunk > BUDGET); + assert_eq!(BaluChunking::target().chunk_count(1 << 27), 32); + let big_chunk = balu(1 << 24).total(); + assert!(big_chunk <= BUDGET && 2 * big_chunk > BUDGET, "{big_chunk}"); + } + + /// `split`, `chunk_count` and `chunk_range` are one rule seen three times + /// for this chip too. + #[test] + fn balu_split_agrees_with_chunk_count_and_range() { + for per in [1usize, 2, 3, 5, 8] { + let c = BaluChunking::from_ops(per); + for n in 0..40usize { + let ops: Vec = (0..n).collect(); + let split = c.split(&ops); + assert_eq!(split.len(), c.chunk_count(n), "per={per} n={n}"); + assert_eq!(split.iter().map(|s| s.len()).sum::(), n); + assert!(split.iter().all(|s| s.len() <= per)); + for (i, s) in split.iter().enumerate() { + assert_eq!(&ops[c.chunk_range(n, i)], *s, "per={per} n={n} chunk {i}"); + } + } + assert_eq!(c.chunk_count(0), 1); + assert_eq!(c.chunk_range(0, 0), 0..0); + } + } + + /// A typo stops the run rather than silently proving a different shape. + #[test] + #[should_panic(expected = "must be a base-2 row exponent")] + fn a_malformed_balu_knob_panics() { + let _ = BaluChunking::from_env_value(Some("2^22")); + } +} diff --git a/prover/src/lfm/commit.rs b/prover/src/lfm/commit.rs new file mode 100644 index 000000000..ab5f349ed --- /dev/null +++ b/prover/src/lfm/commit.rs @@ -0,0 +1,430 @@ +//! Instruction-column-group commitment: interpolate → LDE → row-pair Merkle. +//! +//! The same pipeline the static preprocessed tables use (see +//! `tables/bitwise.rs::compute_preprocessed_commitment`), generalized over an +//! arbitrary column matrix so every LFM chip's group — and the registry +//! builder — shares one implementation. Host-side only; runs at program-build +//! and registry-regeneration time (seconds, not a ceremony — there is no +//! keygen in this framework). + +use math::polynomial::Polynomial; +use stark::commitment::commit_bit_reversed_with; +use stark::config::{Commitment, StarkHash}; +use stark::leaf_layout::LeafLayout; +use stark::proof::options::ProofOptions; +use stark::prover::evaluate_polynomial_on_lde_domain; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::compiler::ColumnGroup; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +/// Whether the instruction-group commit pass runs its column loops in parallel. +/// +/// ⛔ THE A/B CONTROL FOR THE PARALLEL BUILD, and it is a separate knob from +/// [`groups_in_flight`](super::registry::groups_in_flight) on purpose. +/// `LFM_ARTIFACT_GROUPS_IN_FLIGHT=1` bounds RESIDENCY and leaves the columns +/// parallel; `LFM_ARTIFACT_PARALLEL=0` restores the pre-change walk exactly — +/// serial columns, serial groups. +/// +/// ⚠ `RAYON_NUM_THREADS=1` is NOT this control. It also serializes work that was +/// already parallel before this change (the Merkle leaf hashing in +/// `stark::commitment`, and the fixed tables' own commitments), so an A/B taken +/// that way attributes their speedup to this commit. +pub fn parallel_build() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init( + // An EMPTY value reads as unset: `FOO= cmd` is the shell's way of + // clearing a variable, and panicking on it would fail a run for a + // spelling of "default". + || match std::env::var("LFM_ARTIFACT_PARALLEL").ok().as_deref() { + Some("0") => false, + None | Some("") | Some("1") => true, + Some(other) => panic!("LFM_ARTIFACT_PARALLEL must be `0` or `1`, got `{other}`"), + }, + ) +} + +/// The coset LDE of a column matrix, column-major and in NATURAL order. +/// +/// Split out of [`commit_columns`] so a caller that needs the evaluations for +/// something else can expand once and commit from the same copy, rather than +/// building a second, independently expanded one. +/// +/// # One pass per column, and it is both faster and smaller +/// +/// This used to interpolate EVERY column into a `Vec>` and then +/// walk that vector expanding each — so the whole coefficient set, one full copy +/// of the group, was alive at once beside the input and the growing LDE. Each +/// column's interpolate → expand is independent of every other's, so the two +/// loops are now one: a column's coefficients are born and consumed inside the +/// same closure and only the workers' in-flight polynomials are ever live. +/// +/// That is what makes the pass parallelizable, and the parallelism is where lane +/// P's 22.0 s per node goes. ★ The output is UNCHANGED — same operations on the +/// same column in the same order, and `collect` on an indexed parallel iterator +/// preserves position — which the registry drift tests pin against blessed +/// roots. +pub fn lde_columns(columns: &[Vec], options: &ProofOptions) -> Vec> { + let num_rows = columns.first().map_or(0, Vec::len); + let coset_offset = FE::from(options.coset_offset); + let expand = |col: &Vec| { + let poly = Polynomial::interpolate_fft::(col) + .expect("FFT interpolation failed for LFM column group"); + evaluate_polynomial_on_lde_domain( + &poly, + options.blowup_factor as usize, + num_rows, + &coset_offset, + ) + .expect("LDE evaluation failed for LFM column group") + }; + #[cfg(feature = "parallel")] + if parallel_build() { + return columns.par_iter().map(expand).collect(); + } + columns.iter().map(expand).collect() +} + +/// Commits an already-expanded LDE column matrix with today's row-pair +/// leaves. +pub fn commit_lde_columns(lde_columns: &[Vec]) -> Commitment { + commit_lde_columns_with(lde_columns, LeafLayout::RowPair) +} + +/// [`commit_lde_columns`] under an explicit trace-tree leaf layout (S2: a +/// one-row table's preprocessed root is this at [`LeafLayout::Row`]). +pub fn commit_lde_columns_with(lde_columns: &[Vec], layout: LeafLayout) -> Commitment { + // ★ Under the block path's PIN, not `stark`'s default aliases. These commit + // the production tables whose roots `lfm_program_id` names, so the hash that + // BUILDS them and the hash the program identity CLAIMS have to be the same + // one — `registry.rs` records that as the condition under which this read + // moves, and the pin is what moved it. + commit_lde_columns_under::(lde_columns, layout) +} + +/// [`commit_lde_columns_with`] under the commitment configuration `S` (a proof +/// family's, `registry::build_artifacts_under`): the root a program's +/// preprocessed column group has when its proofs commit under `S`. +pub fn commit_lde_columns_under( + lde_columns: &[Vec], + layout: LeafLayout, +) -> Commitment { + let (_, root) = commit_bit_reversed_with::>( + lde_columns, + layout.rows_per_leaf(), + ) + .expect("Merkle build failed for LFM column group"); + root +} + +/// Commits a column matrix (each inner `Vec` one column, power-of-two height). +pub fn commit_columns(columns: &[Vec], options: &ProofOptions) -> Commitment { + commit_lde_columns(&lde_columns(columns, options)) +} + +/// [`commit_columns`] under an explicit leaf layout. +pub fn commit_columns_with( + columns: &[Vec], + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + commit_lde_columns_with(&lde_columns(columns, options), layout) +} + +/// A [`ColumnGroup`]'s data, column-major (the commit pipeline's input shape). +/// +/// A strided gather: the group is row-major, so column `c` is read with stride +/// `width`. Parallel over columns because each output column is written by +/// exactly one closure from a shared `&group` — and because on a production node +/// this moves hundreds of megabytes before any FFT starts. +pub fn group_columns(group: &ColumnGroup) -> Vec> { + let column = |c: usize| (0..group.padded_rows).map(|r| *group.at(r, c)).collect(); + #[cfg(feature = "parallel")] + if parallel_build() { + return (0..group.width).into_par_iter().map(column).collect(); + } + (0..group.width).map(column).collect() +} + +/// Whether an instruction column group is committed on the DEVICE when one is +/// present. `LFM_DEVICE_ARTIFACTS=0` forces the host pass and is the A/B control +/// for O1. +pub fn device_artifacts() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init( + || match std::env::var("LFM_DEVICE_ARTIFACTS").ok().as_deref() { + Some("0") => false, + None | Some("") | Some("1") => true, + Some(other) => panic!("LFM_DEVICE_ARTIFACTS must be `0` or `1`, got `{other}`"), + }, + ) +} + +/// The largest device working set any artifact commit has asked for in this +/// process, in bytes. Zero when nothing has gone to the card. +/// +/// ⓘ This is the ADMISSION's own accounting (`stark::device_set`), term by term: +/// one LDE buffer, the trace-domain snapshot, one full Merkle node buffer and +/// the small scratch — not a sampler reading. It is what the artifact build +/// asked the card for, which is the number that decides whether it is admitted; +/// an external sampler is what says what the process actually held. +pub fn device_artifact_peak_bytes() -> u64 { + DEVICE_PEAK_BYTES.load(std::sync::atomic::Ordering::Relaxed) +} + +static DEVICE_PEAK_BYTES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static DEVICE_GROUPS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static HOST_GROUPS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// How many committed groups took the device path and how many the host, over +/// the life of the process. +/// +/// ⛔ THE NUMBER THAT ATTRIBUTES O1's SAVING. `gpu_lde` admits on +/// `padded_rows · blowup >= 2^14`, so a program's SHORT groups stay on the host +/// whatever the card is doing. Without this split, "the artifact build got +/// faster" cannot distinguish "the big groups moved to the device" from "the +/// device took everything" — and the two imply different residual host time. +pub fn device_host_group_counts() -> (u64, u64) { + use std::sync::atomic::Ordering::Relaxed; + (DEVICE_GROUPS.load(Relaxed), HOST_GROUPS.load(Relaxed)) +} + +/// Commit one instruction column group — on the device where there is one, on +/// the host otherwise. +/// +/// # Why the device can do this at all +/// +/// The group is ALREADY row-major (`ColumnGroup.data` is `padded_rows × width`), +/// which is exactly the layout `gpu_lde`'s fused commit takes. So the device path +/// skips `group_columns` entirely — the strided transpose exists only to feed the +/// host's column-major pipeline — and does interpolate, coset-evaluate, leaf-hash +/// and Merkle in one call, returning the root. +/// +/// # The two paths must produce the same root, and that is gated +/// +/// The host commits with `commit_bit_reversed_with(.., ROWS_PER_LEAF)`; the +/// device builds its leaves from the row-major LDE. They are already required to +/// agree elsewhere: `multi_prove` rebuilds the precomputed tree on the device and +/// REFUSES the proof when its root differs from the one the AIR declares, and for +/// an LFM proof that declared root is what this function produced. The six +/// `registry_drift_*` tests are this caller's gate — they rebuild every +/// registered program's artifacts and compare roots against blessed constants, +/// so a device path that hashed differently fails them at once. +/// +/// ⚠ A `None` from the device is a legitimate fallback (no card, a field the +/// kernels do not take, or admission declining the shape), NOT an error swallowed +/// quietly: once admission has ADMITTED a shape, `gpu_lde`'s own contract is that +/// the device is the only path and a failure aborts with a diagnostic rather than +/// returning here. +pub fn commit_group_device_or_host( + label: &str, + group: &ColumnGroup, + options: &ProofOptions, +) -> Commitment { + commit_group_device_or_host_with(label, group, options, LeafLayout::RowPair) +} + +/// [`commit_group_device_or_host`] under an explicit leaf layout. The device +/// commit (`gpu_lde::try_commit_row_major_with`) builds the tree with +/// `layout.rows_per_leaf()` rows per leaf, so a one-row root (S2) takes the +/// device like a row-pair one. +pub fn commit_group_device_or_host_with( + label: &str, + group: &ColumnGroup, + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + commit_group_device_or_host_under::( + label, group, options, layout, + ) +} + +/// [`commit_group_device_or_host_with`] under the commitment configuration +/// `S`: the card's kernels for `S`'s hash (`DeviceTreeBackend`), or the host +/// commit under `S`. +pub fn commit_group_device_or_host_under( + label: &str, + group: &ColumnGroup, + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + #[cfg(feature = "cuda")] + if device_artifacts() && group.padded_rows > 0 && group.width > 0 { + let set = stark::device_set::commit_device_set_rpl( + group.padded_rows, + group.width, + options.blowup_factor as usize, + true, + layout.rows_per_leaf(), + ); + DEVICE_PEAK_BYTES.fetch_max(set.total(), std::sync::atomic::Ordering::Relaxed); + // ⛔ ROUND-3 TREE PROBE (diagnostic, OFF by default). The card permit is + // held across the WHOLE of `build_artifacts_with_hasher`, and THIS call + // is the only device work inside it. So `Σ dispatch` against that hold + // says how much of an exclusive card hold is actually spent on the card: + // far below it and the permit is held over HOST work, which is a lever + // with a mechanism rather than a floor. + // + // ⚠ A HOST STOPWATCH, NOT A CUDA EVENT, and that is the point: + // `try_commit_row_major` blocks until it has a root, so the wall around + // it brackets the dispatch without adding a synchronize the device path + // would otherwise not have. The measurement cannot perturb what it + // measures. + let probe_t = super::tree_probe::enabled().then(std::time::Instant::now); + let committed = stark::gpu_lde::try_commit_row_major_with::< + GoldilocksField, + S::Batched, + >( + label, + &group.data, + group.padded_rows, + group.width, + options.blowup_factor as usize, + &FE::from(options.coset_offset), + layout.rows_per_leaf(), + ); + if let Some(t) = probe_t { + super::tree_probe::note_device_commit(t.elapsed().as_nanos() as u64); + } + if let Some(root) = committed { + DEVICE_GROUPS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return root; + } + } + let _ = label; + HOST_GROUPS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + commit_lde_columns_under::(&lde_columns(&group_columns(group), options), layout) +} + +/// Commits one instruction column group. +pub fn commit_group(group: &ColumnGroup, options: &ProofOptions) -> Commitment { + commit_columns(&group_columns(group), options) +} + +/// ★★★ THE GATE THAT ACTUALLY REACHES THE DEVICE. +/// +/// `#[cfg(feature = "cuda")]` because without it both sides of the comparison +/// are the host pass and the test is a tautology — a check that cannot fail is +/// worse than no check, because it reads like coverage. +/// +/// # Why the registry drift tests are not this gate +/// +/// `gpu_lde` admits on `lde_size = padded_rows · blowup >= 2^14`, a ROW count. +/// ✓ MEASURED over the registered fixtures: EXACTLY ONE group clears it, and it +/// is slot 10 — `LFM_RANGE`, 65,536 rows × 1 column — which is +/// program-INDEPENDENT and identical in all six. Every program-dependent group +/// is far below: the largest is `statement_replay`'s slot 1 at 4,096 rows +/// (lde 8,192 at blowup 2). So the six drift pins are ONE device observation +/// repeated six times, at the narrowest shape the machine has. +/// +/// ⇒ This test commits a group ABOVE the floor at production-like widths, both +/// ways, and compares the roots. It is the smallest thing that can catch a +/// device leaf convention that differs from +/// `commit_bit_reversed_with(.., ROWS_PER_LEAF)` at a shape the recursion +/// actually emits. +#[cfg(all(test, feature = "cuda"))] +mod device_parity { + use super::*; + use crate::lfm::compiler::ColumnGroup; + use stark::proof::options::GoldilocksCubicProofOptions; + + /// A deterministic group of the given shape. Values are position-dependent + /// so a transposed or mis-strided read cannot land on the same root. + fn group(rows: usize, width: usize) -> ColumnGroup { + let data = (0..rows * width) + .map(|i| FE::from((i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ 0xA5A5)) + .collect(); + ColumnGroup { + width, + real_rows: rows, + padded_rows: rows, + data, + } + } + + #[test] + fn the_device_commit_matches_the_host_commit_above_the_floor() { + let options = GoldilocksCubicProofOptions::with_blowup(4).expect("options"); + // 4,096 · 4 = 16,384 is exactly the floor; 8,192 clears it with margin. + // The widths are the production extremes: 1 (LFM_RANGE), 20 + // (LFM_BLAKE3), 134 (LFM_BITDEC, the widest prep group there is). + for (rows, width) in [(4_096usize, 1usize), (8_192, 20), (4_096, 134)] { + let g = group(rows, width); + let lde = rows * options.blowup_factor as usize; + assert!( + lde >= 1 << 14, + "{rows}x{width} gives lde {lde}, BELOW the 2^14 device floor — \ + this case would compare a host root with a host root" + ); + let host = commit_lde_columns(&lde_columns(&group_columns(&g), &options)); + let device = commit_group_device_or_host("device_parity", &g, &options); + assert_eq!( + device, host, + "{rows}x{width}: the device root differs from the host root. The \ + leaf convention diverged — see commitment.rs's leaf definition \ + and gpu_lde's single-tree path" + ); + } + } + + /// S2: the one-row artifact root on the device equals the + /// host one-row root at the same production shapes, and differs from the + /// row-pair root (a device that ignored the layout would equal it). The + /// device one-row tree counter must move once per group, so a host + /// fallback fails this test instead of comparing host with host. + #[test] + fn the_one_row_device_commit_matches_the_host_commit_above_the_floor() { + let options = GoldilocksCubicProofOptions::with_blowup(4).expect("options"); + assert!( + device_artifacts(), + "LFM_DEVICE_ARTIFACTS=0: this test would compare a host root with a host root" + ); + let before = stark::gpu_lde::gpu_one_row_trees(); + let shapes = [(4_096usize, 1usize), (8_192, 20), (4_096, 134)]; + for (rows, width) in shapes { + let g = group(rows, width); + let lde = lde_columns(&group_columns(&g), &options); + let host = commit_lde_columns_with(&lde, LeafLayout::Row); + let pair = commit_lde_columns_with(&lde, LeafLayout::RowPair); + let device = commit_group_device_or_host_with( + "device_parity_one_row", + &g, + &options, + LeafLayout::Row, + ); + assert_eq!( + device, host, + "{rows}x{width}: the one-row device root differs from the host one-row root" + ); + assert_ne!( + device, pair, + "{rows}x{width}: the one-row root equals the row-pair root" + ); + } + let moved = stark::gpu_lde::gpu_one_row_trees() - before; + assert!( + moved >= shapes.len() as u64, + "only {moved} one-row device trees for {} groups: the device declined (host fallback)", + shapes.len() + ); + } + + /// And the control: `LFM_DEVICE_ARTIFACTS=0` must reach the host pass. Read + /// once per process, so this asserts the knob's VALUE agrees with the branch + /// rather than flipping it mid-run. + #[test] + fn the_opt_out_and_the_branch_agree() { + let options = GoldilocksCubicProofOptions::with_blowup(4).expect("options"); + let g = group(4_096, 20); + let host = commit_lde_columns(&lde_columns(&group_columns(&g), &options)); + assert_eq!( + commit_group_device_or_host("device_parity_optout", &g, &options), + host, + "with LFM_DEVICE_ARTIFACTS={}, the commit must still equal the host root", + if device_artifacts() { "1" } else { "0" } + ); + } +} diff --git a/prover/src/lfm/commit_ood_leg_tests.rs b/prover/src/lfm/commit_ood_leg_tests.rs new file mode 100644 index 000000000..7086b7784 --- /dev/null +++ b/prover/src/lfm/commit_ood_leg_tests.rs @@ -0,0 +1,364 @@ +//! An LFM proof's commitment-time answers (`stark::commit_ood`), on the host +//! verifier and in the node's leg (`per_table_aggregator::emit_leg`), over the +//! Poseidon1 child of [`super::p1_node_tests`]: every sub-proof answers its +//! non-precomputed main columns; the leg draws `z0` and the shared LogUp pair +//! where the host does, after the answers; an answer altered, dropped or extra +//! is refused by both; a false answer the prover carries into its DEEP +//! polynomial is refused by both; and with the answers' DEEP quotients left +//! out, that false answer passes the host and a leg without the quotients but +//! not the leg with them. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::commit_ood::faults::{self, Faults}; +use stark::proof::view::MultiProofView; + +use crate::hash_pin::{P1Family, ProofFamily}; +use crate::tables::types::FEE; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::{LfmExecError, execute}; +use super::harvest::try_child_arena_words; +use super::hash::HasherKind; +use super::p1_node_tests::{ + P1Child, arenas, harvest, node, options, p1_child, p1_child_with, refused_by_a_check, runs, +}; +use super::per_table_aggregator::{DerivedChild, declare_leg_arenas, emit_leg_answered}; +use super::proof::{LfmProof, lfm_prove_under, verify_against_artifacts_under}; +use super::word::ext_word; + +fn host_verifies(c: &P1Child, proof: &LfmProof) -> bool { + verify_against_artifacts_under::( + &c.artifacts, + &proof.proof, + &proof.public_words, + &c.opts, + ) +} + +/// The child at enough queries that a false answer's DEEP quotient — far from +/// every codeword — is caught by FRI with overwhelming probability, whatever +/// the grinding nonce makes the queries. +fn child_for_false_answers() -> P1Child { + let mut opts = options(); + opts.fri_number_of_queries = 24; + p1_child_with(opts) +} + +/// The child's program proved again under `faults` (the prover's fault set, +/// read on this thread). +fn prove_with(c: &P1Child, faults: Faults) -> LfmProof { + let program = super::programs::trivial_program(); + faults::with(faults, || { + lfm_prove_under::(&program, &c.artifacts, &arenas(), &c.opts) + }) + .expect("the child proves") +} + +/// The child with `proof` in place of its honest one. +fn with_proof(c: &P1Child, proof: LfmProof) -> P1Child { + P1Child { + artifacts: c.artifacts.clone(), + proof, + opts: c.opts.clone(), + } +} + +/// One leg over the child that publishes its `z`, `α` and `z0`. +fn publishing_node(c: &P1Child) -> LfmProgram { + let derived = DerivedChild::from_artifacts_under::( + &c.artifacts, + &c.opts, + c.proof.public_words.len(), + ) + .unwrap_or_else(|e| panic!("{e}")); + let shape = derived.shape(); + let mut b = LfmBuilder::new().with_wrap_hash(P1Family::WRAP); + let a = declare_leg_arenas(&mut b, &shape); + let (cells, z0) = emit_leg_answered(&mut b, &shape, &a); + for e in [cells.z_alpha.0, cells.z_alpha.1, z0] { + b.public(e.as_cell()); + } + compile(b.finish()) +} + +/// The host's `z0`: the statement, every root, then the draw over the +/// largest domain — the verifier's Phase A up to the answers. +fn host_z0(c: &P1Child) -> FEE { + let airs = super::airs::LfmAirs::for_artifacts(&c.artifacts, &c.opts).expect("the AIR set"); + let refs = airs.air_refs(); + let view = MultiProofView::Owned(&c.proof.proof.multi); + let mut t = P1Family::transcript(&[]); + super::statement::absorb_lfm_statement_tagged( + &mut t, + P1Family::STATEMENT_TAG, + &c.artifacts.program_id, + &c.proof.public_words, + c.opts.fri_final_poly_log_degree, + ); + for (idx, air) in refs.iter().enumerate() { + let v = view.get(idx); + if air.is_preprocessed() { + t.append_bytes( + &super::epoch_verify::layout_precomputed_commitment(*air, v.trace_length()) + .expect("a preprocessed root"), + ); + } + t.append_bytes(v.lde_trace_main_merkle_root()); + } + let domain = stark::commit_ood::point_domain(refs.iter().enumerate().map(|(idx, air)| { + ( + view.get(idx).trace_length(), + air.options().blowup_factor as usize, + air.options().coset_offset, + ) + })) + .expect("one coset offset"); + stark::commit_ood::draw_point(&mut t, domain) +} + +/// ★ Every sub-proof answers exactly its main columns past the precomputed +/// ones (the LFM program itself is precomputed and answers nothing), and the +/// proof verifies with them. +#[test] +fn every_lfm_sub_proof_answers_its_non_precomputed_main_columns() { + let c = p1_child(); + assert!(host_verifies(&c, &c.proof)); + let airs = super::airs::LfmAirs::for_artifacts(&c.artifacts, &c.opts).expect("the AIR set"); + let refs = airs.air_refs(); + assert_eq!( + c.proof.proof.commit_ood.len(), + refs.len(), + "one list per sub-proof" + ); + let mut total = 0; + for (air, answers) in refs.iter().zip(&c.proof.proof.commit_ood) { + let (main, _) = air.trace_layout(); + let precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + assert_eq!(answers.len(), main - precomputed, "{}", air.name()); + assert!(!answers.is_empty(), "{} answers something", air.name()); + total += answers.len(); + println!( + "COMMIT OOD ANSWERS {}: {} (main {main}, precomputed {precomputed})", + air.name(), + answers.len() + ); + } + println!("COMMIT OOD ANSWERS total: {total} ({} bytes)", 24 * total); +} + +/// ★ The leg draws `z0` where the host does — after every root — and the +/// shared LogUp pair where the host does — after every answer: the honest +/// child executes and publishes the host's three values. A leg that absorbed +/// the answers after the pair would publish another `z`. +#[test] +fn the_leg_draws_z0_and_the_shared_pair_where_the_host_does() { + let c = p1_child(); + let honest = harvest(&c); + let program = publishing_node(&c); + let arenas = try_child_arena_words(&honest).unwrap_or_else(|e| panic!("{e}")); + let exec = execute(&program, &arenas, &HasherKind::Poseidon1W16).expect("the honest child"); + let published: Vec<_> = exec.public_words.iter().map(|(_, w)| *w).collect(); + let (z, alpha) = honest.z_alpha; + assert_eq!( + published, + vec![ext_word(&z), ext_word(&alpha), ext_word(&host_z0(&c))], + "z, α and z0 are the host's" + ); +} + +/// An altered answer is refused by the host verifier and has no execution in +/// the leg, at every sub-proof. +#[test] +fn an_altered_answer_is_refused_on_both_sides() { + let c = p1_child(); + let program = node(&c, P1Family::STATEMENT_TAG); + assert!(runs(&program, &harvest(&c)), "the honest child executes"); + for t in 0..c.proof.proof.commit_ood.len() { + let last = c.proof.proof.commit_ood[t].len() - 1; + for i in [0, last] { + let mut proof = LfmProof { + proof: c.proof.proof.clone(), + public_words: c.proof.public_words.clone(), + }; + proof.proof.commit_ood[t][i] += FEE::one(); + assert!(!host_verifies(&c, &proof), "host: table {t} answer {i}"); + let mut bad = harvest(&c); + bad.commit_ood[t][i] += FEE::one(); + assert!( + refused_by_a_check(&program, &bad), + "leg: table {t} answer {i}" + ); + } + } +} + +/// A dropped or extra answer: the host refuses the count before absorbing +/// anything; the leg's answers arena is shape, so the arenas do not fit it. +#[test] +fn a_dropped_or_extra_answer_is_refused() { + let c = p1_child(); + let program = node(&c, P1Family::STATEMENT_TAG); + type Edit<'a> = (&'a str, &'a dyn Fn(&mut Vec>)); + let edits: [Edit<'_>; 4] = [ + ("an answer dropped", &|a| { + a[0].pop(); + }), + ("an answer extra", &|a| a[1].push(FEE::zero())), + ("a sub-proof's answers dropped", &|a| { + a.pop(); + }), + ("no answers", &|a| a.clear()), + ]; + for (what, edit) in edits { + let mut proof = LfmProof { + proof: c.proof.proof.clone(), + public_words: c.proof.public_words.clone(), + }; + edit(&mut proof.proof.commit_ood); + assert!(!host_verifies(&c, &proof), "host: {what}"); + let mut bad = harvest(&c); + edit(&mut bad.commit_ood); + let arenas = try_child_arena_words(&bad).unwrap_or_else(|e| panic!("{e}")); + assert!( + matches!( + execute(&program, &arenas, &HasherKind::Poseidon1W16), + Err(LfmExecError::ArenaLenMismatch { .. }) + ), + "leg: {what}" + ); + } +} + +/// A false answer the prover absorbs and carries into its DEEP polynomial (a +/// consistent proof of the false value) is refused by the host and has no +/// execution in the leg. +#[test] +fn a_false_answer_carried_into_deep_is_refused_on_both_sides() { + let c = child_for_false_answers(); + let program = node(&c, P1Family::STATEMENT_TAG); + let last = c.proof.proof.commit_ood.len() - 1; + for at in [(0, 0), (last, 0)] { + let false_child = with_proof( + &c, + prove_with( + &c, + Faults { + false_answer: Some(at), + without_deep_term: false, + z0_meets_z: false, + }, + ), + ); + assert!( + !host_verifies(&false_child, &false_child.proof), + "host: {at:?}" + ); + assert!( + refused_by_a_check(&program, &harvest(&false_child)), + "leg: {at:?}" + ); + } +} + +/// ★ Load-bearing, both sides: a false answer proved with the answers' DEEP +/// quotients left out passes the host verifier that leaves them out too, and +/// a leg emitted without them; the leg with them refuses it. So the quotients +/// are what refuse the false answer above, on the host and in the leg. +#[test] +fn the_answers_quotients_are_what_refuse_a_false_answer() { + let c = child_for_false_answers(); + let fault = Faults { + false_answer: Some((0, 0)), + without_deep_term: true, + z0_meets_z: false, + }; + let false_child = with_proof(&c, prove_with(&c, fault)); + assert!( + faults::with(fault, || host_verifies(&false_child, &false_child.proof)), + "host without the quotients accepts it" + ); + assert!( + !host_verifies(&false_child, &false_child.proof), + "host with them refuses it" + ); + let child = harvest(&false_child); + let with_term = node(&c, P1Family::STATEMENT_TAG); + assert!(refused_by_a_check(&with_term, &child), "the leg refuses it"); + super::deep::WITHOUT_ANSWERS_TERM.with(|w| w.set(true)); + let without_term = node(&c, P1Family::STATEMENT_TAG); + super::deep::WITHOUT_ANSWERS_TERM.with(|w| w.set(false)); + assert!( + runs(&without_term, &child), + "a leg without the quotients accepts it" + ); +} + +/// The leg refuses a `z0` at a DEEP point: over a small bound shape, the +/// invariants execute at an unrelated `z0` and fail at `z`, `g·z` and `z^P`. +#[test] +fn the_leg_refuses_a_z0_at_a_deep_point() { + use math::field::traits::IsFFTField; + + use super::deep::{CommitOodCols, DeepShape, emit_deep_invariants_bound}; + use crate::tables::types::{FE, GoldilocksField}; + + let shape = DeepShape { + step_size: 1, + num_eval_points: 2, + num_total_cols: 3, + next_row_cols: vec![0], + num_composition_parts: 2, + log2_trace_length: 3, + commit_ood: Some(CommitOodCols { first: 1, count: 2 }), + }; + let ext = |v: u64| FEE::from(v); + let zeta = FEE::new([FE::from(5u64), FE::from(7u64), FE::from(2u64)]); + let run = |z0: FEE| { + // γ, ζ, the 2×3 OOD grid, the two parts, z0, the two answers. + let values = [ + ext(3), + zeta, + ext(1), + ext(2), + ext(4), + ext(6), + ext(8), + ext(9), + ext(10), + ext(13), + z0, + ext(11), + ext(12), + ]; + let words: Vec<_> = values.iter().map(ext_word).collect(); + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(words.len() as u32); + let cells: Vec<_> = (0..words.len() as u32) + .map(|i| b.hint_word(arena, i).as_ext()) + .collect(); + let ood = vec![cells[2..5].to_vec(), cells[5..8].to_vec()]; + let _ = emit_deep_invariants_bound( + &mut b, + &shape, + cells[0], + cells[1], + &ood, + &cells[8..10], + Some((cells[10], &cells[11..13])), + ); + execute(&compile(b.finish()), &[words], &HasherKind::Poseidon1W16) + }; + let g = GoldilocksField::get_primitive_root_of_unity(3).expect("a root of order 8"); + assert!(run(zeta + ext(1)).is_ok(), "an unrelated z0 executes"); + for (what, z0) in [("z", zeta), ("g·z", &g * &zeta), ("z^P", zeta.pow(2usize))] { + assert!( + matches!(run(z0), Err(LfmExecError::DivByZero { .. })), + "z0 = {what} must not execute" + ); + } +} diff --git a/prover/src/lfm/compiler.rs b/prover/src/lfm/compiler.rs new file mode 100644 index 000000000..e45508900 --- /dev/null +++ b/prover/src/lfm/compiler.rs @@ -0,0 +1,720 @@ +//! The LFM straight-line compiler. +//! +//! Pass 1 backfills static multiplicities from the builder's read counters, +//! guarded by the two invariant panics (tripwires — the release-mode +//! admission validator is the gate, the registry is the record): +//! - **panic #1**: an address assigned twice (write-once violated in the +//! builder itself); +//! - **panic #2**: the read-count map is not drained after backfill (a +//! read of an address no instruction writes). +//! +//! Pass 2 emits the per-chip **instruction column groups** — the preprocessed +//! matrices whose Merkle roots become the program's identity. Layouts live in +//! [`super::layout`]; group commitment (interpolate → LDE → Merkle) is wired +//! at registry-build time (Milestone B) through the same pipeline the static +//! tables use. + +use crate::tables::types::FE; + +use super::builder::{ArenaSchema, LfmProgramSource}; +use super::chunking::{Blake3Chunking, HashChunking, KeccakChunking}; +use super::instr::{Addr, BaseOp, ExtOp, HashMode, Instr, KeccakMode}; +use super::layout::{self, padded_rows}; + +/// One chip's instruction column group: a row-major matrix, zero-padded to a +/// power-of-two height (min 4). +#[derive(Debug, Clone)] +pub struct ColumnGroup { + pub width: usize, + pub real_rows: usize, + pub padded_rows: usize, + /// Row-major, `padded_rows × width`. + pub data: Vec, +} + +/// Accumulates one chip's rows directly into the flat row-major buffer the +/// finished [`ColumnGroup`] holds. +/// +/// The emitter used to collect `Vec>` and copy it row by row. That kept +/// two full materializations of every group alive at once and paid a heap +/// allocation per instruction — and the per-row `Vec`s over-allocate badly, +/// because they are grown by `extend`/`push` rather than sized: a 10-wide BALU +/// row lands at capacity 18. Appending into one buffer removes the second +/// materialization, the headers, the rounding waste and the ~271M malloc/free +/// pairs. +/// +/// Rows come out bit-identical: the same values are written at the same +/// row-major offsets, and the tail is zero-padded to the same height. +struct ColumnGroupBuilder { + width: usize, + real_rows: usize, + data: Vec, +} + +impl ColumnGroupBuilder { + fn new(width: usize) -> Self { + ColumnGroupBuilder { + width, + real_rows: 0, + data: Vec::new(), + } + } + + /// The ordinal the next row will take. `LFM_KECCAK` binds it into the row + /// as a structural tag, so it has to be read before [`Self::open_row`]. + fn next_row(&self) -> usize { + self.real_rows + } + + /// Append a zero-filled row, returning its base offset for [`Self::set`]. + fn open_row(&mut self) -> usize { + let base = self.data.len(); + self.data.resize(base + self.width, FE::zero()); + self.real_rows += 1; + base + } + + fn set(&mut self, base: usize, col: usize, v: FE) { + debug_assert!( + col < self.width, + "column {col} outside width {}", + self.width + ); + self.data[base + col] = v; + } + + fn finish(mut self) -> ColumnGroup { + let padded = padded_rows(self.real_rows); + self.data.resize(padded * self.width, FE::zero()); + ColumnGroup { + width: self.width, + real_rows: self.real_rows, + padded_rows: padded, + data: self.data, + } + } +} + +impl ColumnGroup { + pub fn at(&self, row: usize, col: usize) -> &FE { + &self.data[row * self.width + col] + } + + pub fn set(&mut self, row: usize, col: usize, v: FE) { + self.data[row * self.width + col] = v; + } +} + +/// The program-dependent instruction column groups, in the frozen chip order. +/// (`LFM_RANGE`'s group is program-independent and materialized at commitment +/// time; the three hosted keccak-family tables carry no LFM group at all.) +#[derive(Debug, Clone)] +pub struct LfmColumnGroups { + pub const_: ColumnGroup, + pub balu: ColumnGroup, + pub xalu: ColumnGroup, + pub select: ColumnGroup, + pub bitdec: ColumnGroup, + pub hash: ColumnGroup, + pub keccak: ColumnGroup, + pub blake3: ColumnGroup, + pub lanes: ColumnGroup, + pub hint: ColumnGroup, + pub public: ColumnGroup, +} + +/// A compiled LFM program: multiplicity-backfilled instructions plus the +/// emitted instruction column groups. +#[derive(Debug)] +pub struct LfmProgram { + pub instrs: Vec, + pub num_addrs: u64, + pub arena_schema: ArenaSchema, + pub public_len: u32, + pub groups: LfmColumnGroups, + /// How this program's permutations are spread over `KECCAK_RND` + /// instances. Program shape, not a runtime knob: it is fixed here, bound + /// into the program digest and pinned in the registry. + pub chunking: KeccakChunking, + /// How this program's compressions are spread over `LFM_BLAKE3` instances. + /// Program shape on the same terms as [`Self::chunking`] — and with one + /// consequence that policy does not have: `LFM_BLAKE3` carries a + /// preprocessed instruction group, so each chunk commits its own root. + /// Defaults to a single table, which is the machine as it stood before + /// chunking existed. + pub blake3_chunking: Blake3Chunking, + /// How this program's hash rows are spread over `LFM_HASH` instances — one + /// table unless the hash split divides it ([`HashChunking`]). Program shape on + /// [`Self::blake3_chunking`]'s terms: each chunk commits its own + /// instruction group, and the split is bound into `program_id`. + pub hash_chunking: HashChunking, + /// Whether this program's `LFM_HASH` rows are width-16 (`Instr::Hash16`): + /// it then proves under `HasherKind::Poseidon1W16` + /// ([`LfmProgram::hasher`]). Program shape, read off the instructions at + /// compile time; the admission validator refuses a program that mixes the + /// two widths. Private: only [`compile`] sets it, from the instructions + /// (the socket review's A4); read it through [`LfmProgram::hash16`]. + hash16: bool, +} + +impl LfmProgram { + /// Whether this program's `LFM_HASH` rows are width-16 (`Instr::Hash16`). + pub fn hash16(&self) -> bool { + self.hash16 + } + + /// The `LFM_HASH` permutation this program proves under: the width-16 + /// socket's for a program with `Hash16` rows, else `default` (the block + /// hasher every other program uses). + pub fn hasher(&self, default: super::hash::HasherKind) -> super::hash::HasherKind { + if self.hash16 { + super::hash::HasherKind::Poseidon1W16 + } else { + default + } + } + + /// Replaces the `KECCAK_RND` chunking policy. + /// + /// Chunking affects only how the round-chip rows are distributed over AIR + /// instances — never what is compiled — so it is safe to set after + /// compilation. Tests use it to force several chunks out of a program with + /// a handful of permutations; retuning uses it to size chunks per preset. + pub fn with_keccak_chunking(mut self, chunking: KeccakChunking) -> Self { + self.chunking = chunking; + self + } + + /// Replaces the `LFM_BLAKE3` chunking policy. + /// + /// Safe after compilation for [`Self::with_keccak_chunking`]'s reason — + /// chunking redistributes rows over AIR instances and never changes what is + /// compiled — but note the extra consequence: this chip's chunks each commit + /// their own instruction group, so a program re-chunked after + /// `build_artifacts` has been called needs FRESH artifacts, and its + /// `program_id` differs from the unchunked one's. + pub fn with_blake3_chunking(mut self, chunking: Blake3Chunking) -> Self { + self.blake3_chunking = chunking; + self + } + + /// `LFM_BLAKE3` instances this program's policy asks for — at least one, + /// even with no compressions at all. The chip MASK is what drops an unused + /// family; see [`ChipSet::blake3_chunks`](super::airs::ChipSet::blake3_chunks). + pub fn blake3_chunk_count(&self) -> usize { + self.blake3_chunking + .chunk_count(self.groups.blake3.real_rows) + } + + /// Replaces the `LFM_HASH` chunking policy — on + /// [`Self::with_blake3_chunking`]'s terms, fresh artifacts included. + pub fn with_hash_chunking(mut self, chunking: HashChunking) -> Self { + self.hash_chunking = chunking; + self + } + + /// `LFM_HASH` instances this program's policy asks for — one or two. + pub fn hash_chunk_count(&self) -> usize { + self.hash_chunking.chunk_count(self.groups.hash.real_rows) + } + + /// Hash rows in each `LFM_HASH` chunk, in chunk order. + pub fn hash_chunk_real_rows(&self) -> Vec { + let total = self.groups.hash.real_rows; + (0..self.hash_chunk_count()) + .map(|c| self.hash_chunking.chunk_range(total, c).len()) + .collect() + } + + /// One `LFM_HASH` chunk's instruction column group, re-padded to its own + /// power-of-two height — [`Self::blake3_chunk_group`] for the hash chip. + /// + /// # Panics + /// + /// On a chunk index at or past [`Self::hash_chunk_count`]. + pub fn hash_chunk_group(&self, chunk: usize) -> ColumnGroup { + let count = self.hash_chunk_count(); + assert!( + chunk < count, + "LFM_HASH chunk {chunk} requested of a {count}-chunk program" + ); + let group = &self.groups.hash; + if count == 1 { + return group.clone(); + } + let rows = self.hash_chunking.chunk_range(group.real_rows, chunk); + let real_rows = rows.len(); + let padded_rows = padded_rows(real_rows); + let mut data = group.data[rows.start * group.width..rows.end * group.width].to_vec(); + data.resize(padded_rows * group.width, FE::zero()); + ColumnGroup { + width: group.width, + real_rows, + padded_rows, + data, + } + } + + /// Compressions in each `LFM_BLAKE3` chunk, in chunk order. + pub fn blake3_chunk_real_rows(&self) -> Vec { + let total = self.groups.blake3.real_rows; + (0..self.blake3_chunk_count()) + .map(|c| self.blake3_chunking.chunk_range(total, c).len()) + .collect() + } + + /// One `LFM_BLAKE3` chunk's instruction column group: the rows the policy + /// assigns it, re-padded to its own power-of-two height. + /// + /// Built one chunk at a time rather than returned as a `Vec` because the + /// aggregation program's group is ~1.39M rows x 20 columns; the artifact + /// builder and the trace builder each commit or copy a chunk and drop it, so + /// peak residency stays one chunk instead of a second whole group. + /// + /// # Panics + /// + /// On a chunk index at or past [`Self::blake3_chunk_count`] — a caller bug, + /// and this runs at program-build time, never on a verify path. + pub fn blake3_chunk_group(&self, chunk: usize) -> ColumnGroup { + let count = self.blake3_chunk_count(); + assert!( + chunk < count, + "LFM_BLAKE3 chunk {chunk} requested of a {count}-chunk program" + ); + let group = &self.groups.blake3; + let rows = self.blake3_chunking.chunk_range(group.real_rows, chunk); + let real_rows = rows.len(); + let padded_rows = padded_rows(real_rows); + let mut data = group.data[rows.start * group.width..rows.end * group.width].to_vec(); + data.resize(padded_rows * group.width, FE::zero()); + ColumnGroup { + width: group.width, + real_rows, + padded_rows, + data, + } + } +} + +/// Emission backends. Backend 1 (column groups) is the machine; backend 2 is +/// the future circuit specialization, a stub by design so the option stays an +/// edit instead of a rewrite. +pub trait LfmBackend { + type Artifacts; + fn emit(&self, program: &LfmProgram) -> Self::Artifacts; +} + +pub struct ColumnGroupBackend; +impl LfmBackend for ColumnGroupBackend { + type Artifacts = LfmColumnGroups; + fn emit(&self, program: &LfmProgram) -> LfmColumnGroups { + program.groups.clone() + } +} + +/// The circuit backend does not exist yet; it panics so nothing can silently +/// depend on it. +pub struct CircuitBackend; +impl LfmBackend for CircuitBackend { + type Artifacts = (); + fn emit(&self, _program: &LfmProgram) -> () { + unimplemented!( + "LFM circuit backend is a v1+ specialization; only the column-group backend exists" + ) + } +} + +pub fn compile(source: LfmProgramSource) -> LfmProgram { + let LfmProgramSource { + mut instrs, + num_addrs, + mut read_counts, + arena_schema, + public_len, + } = source; + + // Pass 1: occupancy + multiplicity backfill. + let mut written = vec![false; num_addrs as usize]; + let take = |addr: Addr, written: &mut Vec, counts: &mut [u64]| -> u64 { + let slot = written + .get_mut(addr.0 as usize) + .unwrap_or_else(|| panic!("LFM compiler invariant: address {} out of range", addr.0)); + if *slot { + panic!("LFM compiler invariant: address {} written twice", addr.0); + } + *slot = true; + // Taking (not reading) is what drains the counter, so the emptiness + // check below still means "every read had a writer". + core::mem::take(&mut counts[addr.0 as usize]) + }; + for instr in &mut instrs { + match instr { + Instr::Const { out, mult, .. } + | Instr::BaseAlu { out, mult, .. } + | Instr::ExtAlu { out, mult, .. } + | Instr::Hint { out, mult, .. } + | Instr::HintOn { out, mult, .. } + | Instr::Pack { out, mult, .. } => { + *mult = take(*out, &mut written, &mut read_counts); + } + Instr::Unpack { outs, mults, .. } => { + for i in 0..4 { + mults[i] = take(outs[i], &mut written, &mut read_counts); + } + } + Instr::KeccakF(k) => { + for i in 0..layout::keccak::NUM_WORDS { + k.mults[i] = take(k.outs[i], &mut written, &mut read_counts); + } + if let Some(rev) = &mut k.rev { + for i in 0..layout::keccak::DIGEST_WORDS { + rev.mults[i] = take(rev.outs[i], &mut written, &mut read_counts); + } + } + } + Instr::Blake3(k) => { + for i in 0..layout::blake3::OUT_WORDS { + k.mults[i] = take(k.outs[i], &mut written, &mut read_counts); + } + if let Some(rev) = &mut k.rev { + for i in 0..layout::blake3::DIGEST_WORDS { + rev.mults[i] = take(rev.outs[i], &mut written, &mut read_counts); + } + } + } + Instr::Select { + out_l, + out_r, + mult_l, + mult_r, + .. + } => { + *mult_l = take(*out_l, &mut written, &mut read_counts); + *mult_r = take(*out_r, &mut written, &mut read_counts); + } + Instr::BitDec { bits, halves, .. } => { + for (addr, mult) in bits.iter_mut() { + *mult = take(*addr, &mut written, &mut read_counts); + } + if let Some(hs) = halves { + for (addr, mult) in hs.iter_mut() { + *mult = take(*addr, &mut written, &mut read_counts); + } + } + } + Instr::Hash { + mode, outs, mults, .. + } => { + let num_outs = mode.num_output_cells(); + for i in 0..num_outs { + mults[i] = take(outs[i], &mut written, &mut read_counts); + } + } + Instr::Hash16(h) => { + for i in 0..4 { + h.mults[i] = take(h.outs[i], &mut written, &mut read_counts); + } + } + Instr::Public { .. } => {} + } + } + assert!( + read_counts.iter().all(|&c| c == 0), + "LFM compiler invariant: read-count map not drained after backfill — reads of never-written addresses: {:?}", + read_counts + .iter() + .enumerate() + .filter(|(_, c)| **c != 0) + .map(|(a, _)| Addr(a as u64)) + .collect::>() + ); + + // Both are dead from here on and together outweigh the groups being built. + // Dropping them explicitly keeps the emitter's peak off the sum of the two + // materializations — the scope would otherwise hold them to the end. + drop(read_counts); + drop(written); + + let groups = emit_column_groups(&instrs, public_len); + let hash16 = instrs.iter().any(|i| matches!(i, Instr::Hash16(_))); + + LfmProgram { + hash16, + instrs, + num_addrs, + arena_schema, + public_len, + chunking: KeccakChunking::default(), + blake3_chunking: Blake3Chunking::default(), + // The hash split: read once at compile time, so every program this + // process compiles carries the same policy. + hash_chunking: HashChunking::for_process(groups.hash.real_rows), + groups, + } +} + +fn fe(v: u64) -> FE { + FE::from(v) +} + +/// Pass 2: partition instructions per chip (program order preserved) and lay +/// out each chip's instruction fields per [`super::layout`]. +fn emit_column_groups(instrs: &[Instr], _public_len: u32) -> LfmColumnGroups { + let mut const_ = ColumnGroupBuilder::new(layout::const_::PREP_WIDTH); + let mut balu = ColumnGroupBuilder::new(layout::balu::PREP_WIDTH); + let mut xalu = ColumnGroupBuilder::new(layout::xalu::PREP_WIDTH); + let mut select = ColumnGroupBuilder::new(layout::select::PREP_WIDTH); + let mut bitdec = ColumnGroupBuilder::new(layout::bitdec::PREP_WIDTH); + let mut hash = ColumnGroupBuilder::new(layout::hash::PREP_WIDTH); + let mut keccak = ColumnGroupBuilder::new(layout::keccak::PREP_WIDTH); + let mut blake3 = ColumnGroupBuilder::new(layout::blake3::PREP_WIDTH); + let mut lanes = ColumnGroupBuilder::new(layout::lanes::PREP_WIDTH); + let mut hint = ColumnGroupBuilder::new(layout::hint::PREP_WIDTH); + let mut public = ColumnGroupBuilder::new(layout::public::PREP_WIDTH); + + for instr in instrs { + match instr { + Instr::Const { out, value, mult } => { + use layout::const_ as c; + let r = const_.open_row(); + const_.set(r, c::ADDR, fe(out.0)); + for (i, v) in value.iter().enumerate() { + const_.set(r, c::V0 + i, *v); + } + const_.set(r, c::MULT, fe(*mult)); + } + Instr::BaseAlu { + op, + out, + a, + b, + c, + mult, + } => { + use layout::balu as l; + let r = balu.open_row(); + balu.set(r, l::A_ADDR, fe(a.0)); + balu.set(r, l::B_ADDR, fe(b.0)); + balu.set(r, l::C_ADDR, fe(c.0)); + balu.set(r, l::OUT_ADDR, fe(out.0)); + let sel = match op { + BaseOp::Add => l::SEL_ADD, + BaseOp::Sub => l::SEL_SUB, + BaseOp::Mul => l::SEL_MUL, + BaseOp::Div => l::SEL_DIV, + BaseOp::MulAdd => l::SEL_MULADD, + }; + balu.set(r, sel, FE::one()); + balu.set(r, l::MULT, fe(*mult)); + } + Instr::ExtAlu { + op, + out, + a, + b, + c, + mult, + } => { + use layout::xalu as l; + let r = xalu.open_row(); + xalu.set(r, l::A_ADDR, fe(a.0)); + xalu.set(r, l::B_ADDR, fe(b.0)); + xalu.set(r, l::C_ADDR, fe(c.0)); + xalu.set(r, l::OUT_ADDR, fe(out.0)); + let sel = match op { + ExtOp::Add => l::SEL_ADD, + ExtOp::Sub => l::SEL_SUB, + ExtOp::Mul => l::SEL_MUL, + ExtOp::Div => l::SEL_DIV, + ExtOp::MulAdd => l::SEL_MULADD, + ExtOp::MulBase => l::SEL_MULBASE, + }; + xalu.set(r, sel, FE::one()); + xalu.set(r, l::MULT, fe(*mult)); + } + Instr::Select { + bit, + out_l, + out_r, + in_l, + in_r, + mult_l, + mult_r, + } => { + use layout::select as l; + let r = select.open_row(); + select.set(r, l::BIT_ADDR, fe(bit.0)); + select.set(r, l::INL_ADDR, fe(in_l.0)); + select.set(r, l::INR_ADDR, fe(in_r.0)); + select.set(r, l::OUTL_ADDR, fe(out_l.0)); + select.set(r, l::OUTR_ADDR, fe(out_r.0)); + select.set(r, l::MULT_L, fe(*mult_l)); + select.set(r, l::MULT_R, fe(*mult_r)); + select.set(r, l::IS_REAL, FE::one()); + } + Instr::BitDec { + input, + bits, + halves, + } => { + use layout::bitdec as l; + let r = bitdec.open_row(); + bitdec.set(r, l::IN_ADDR, fe(input.0)); + bitdec.set(r, l::IS_REAL, FE::one()); + for (i, (addr, mult)) in bits.iter().enumerate() { + bitdec.set(r, l::bit_addr(i), fe(addr.0)); + bitdec.set(r, l::bit_mult(i), fe(*mult)); + } + if let Some([h0, h1]) = halves { + bitdec.set(r, l::HALF0_ADDR, fe(h0.0.0)); + bitdec.set(r, l::HALF0_MULT, fe(h0.1)); + bitdec.set(r, l::HALF1_ADDR, fe(h1.0.0)); + bitdec.set(r, l::HALF1_MULT, fe(h1.1)); + } + } + Instr::Hash { + mode, + ins, + outs, + mults, + } => { + // One-hot over the three modes. The AIR pins only the SUM to a + // bit; exactly-one-of is this emitter's job, re-checked by the + // admission validator. + use layout::hash as l; + let r = hash.open_row(); + hash.set(r, l::IN_ADDR0, fe(ins[0].0)); + hash.set(r, l::IN_ADDR1, fe(ins[1].0)); + hash.set(r, l::IN_ADDR2, fe(ins[2].0)); + hash.set(r, l::OUT_ADDR0, fe(outs[0].0)); + hash.set(r, l::OUT_ADDR1, fe(outs[1].0)); + hash.set(r, l::OUT_ADDR2, fe(outs[2].0)); + let mode_col = match mode { + HashMode::Compress => l::MODE_C, + HashMode::Transcript => l::MODE_T, + HashMode::Leaf => l::MODE_L, + HashMode::Permute => l::MODE_P, + }; + hash.set(r, mode_col, FE::one()); + hash.set(r, l::MULT0, fe(mults[0])); + hash.set(r, l::MULT1, fe(mults[1])); + hash.set(r, l::MULT2, fe(mults[2])); + } + Instr::Hash16(h) => { + // The width-16 socket's instruction group: the same 13 columns + // as the twelve-felt socket's, in its own order + // (`p1w16_socket::cols`). A program carries one or the other, + // so the group is never mixed (the admission validator). + use super::p1w16_socket::cols as l; + let r = hash.open_row(); + for k in 0..4 { + hash.set(r, l::IN_ADDR0 + k, fe(h.ins[k].0)); + hash.set(r, l::OUT_ADDR0 + k, fe(h.outs[k].0)); + hash.set(r, l::MULT0 + k, fe(h.mults[k])); + } + hash.set(r, l::IS_REAL, FE::one()); + } + Instr::KeccakF(op) => { + use layout::keccak as k; + // The tag is the row ordinal, so uniqueness is structural and + // the prover has no say (it is preprocessed data). See + // `layout::keccak::tag_for_row`. + let tag = k::tag_for_row(keccak.next_row()); + let r = keccak.open_row(); + keccak.set(r, k::TAG_LO, fe(tag & 0xFFFF_FFFF)); + keccak.set(r, k::TAG_HI, fe(tag >> 32)); + for j in 0..k::NUM_WORDS { + keccak.set(r, k::in_addr(j), fe(op.ins[j].0)); + keccak.set(r, k::out_addr(j), fe(op.outs[j].0)); + keccak.set(r, k::mult(j), fe(op.mults[j])); + } + if let Some(rev) = &op.rev { + for w in 0..k::DIGEST_WORDS { + keccak.set(r, k::rev_addr(w), fe(rev.outs[w].0)); + keccak.set(r, k::rev_mult(w), fe(rev.mults[w])); + } + } + match op.mode { + KeccakMode::Permute => keccak.set(r, k::MODE_PERM, FE::one()), + KeccakMode::Absorb => { + keccak.set(r, k::MODE_ABSORB, FE::one()); + for j in 0..k::BLOCK_WORDS { + keccak.set(r, k::block_addr(j), fe(op.block[j].0)); + } + } + } + } + Instr::Blake3(op) => { + use layout::blake3 as l; + let r = blake3.open_row(); + for j in 0..l::IN_WORDS { + blake3.set(r, l::in_addr(j), fe(op.ins[j].0)); + } + for j in 0..l::OUT_WORDS { + blake3.set(r, l::out_addr(j), fe(op.outs[j].0)); + blake3.set(r, l::mult(j), fe(op.mults[j])); + } + if let Some(rev) = &op.rev { + for w in 0..l::DIGEST_WORDS { + blake3.set(r, l::rev_addr(w), fe(rev.outs[w].0)); + blake3.set(r, l::rev_mult(w), fe(rev.mults[w])); + } + } + blake3.set(r, l::MU, FE::one()); + } + // A `HintOn` is a hint row: which arena word it read is the + // executor's, not the row's. + Instr::Hint { out, mult, .. } | Instr::HintOn { out, mult, .. } => { + use layout::hint as l; + let r = hint.open_row(); + hint.set(r, l::OUT_ADDR, fe(out.0)); + hint.set(r, l::MULT, fe(*mult)); + } + Instr::Pack { + lanes: ls, + out, + mult, + } => { + use layout::lanes as l; + let r = lanes.open_row(); + lanes.set(r, l::WORD_ADDR, fe(out.0)); + for (i, lane) in ls.iter().enumerate() { + lanes.set(r, l::LANE_ADDR0 + i, fe(lane.0)); + } + lanes.set(r, l::MODE_PACK, FE::one()); + lanes.set(r, l::WORD_MULT, fe(*mult)); + } + Instr::Unpack { input, outs, mults } => { + use layout::lanes as l; + let r = lanes.open_row(); + lanes.set(r, l::WORD_ADDR, fe(input.0)); + for i in 0..4 { + lanes.set(r, l::LANE_ADDR0 + i, fe(outs[i].0)); + lanes.set(r, l::LANE_MULT0 + i, fe(mults[i])); + } + lanes.set(r, l::MODE_UNPACK, FE::one()); + } + Instr::Public { addr, index } => { + use layout::public as l; + let r = public.open_row(); + public.set(r, l::IN_ADDR, fe(addr.0)); + public.set(r, l::INDEX, fe(*index as u64)); + public.set(r, l::IS_REAL, FE::one()); + } + } + } + + LfmColumnGroups { + const_: const_.finish(), + balu: balu.finish(), + xalu: xalu.finish(), + select: select.finish(), + bitdec: bitdec.finish(), + hash: hash.finish(), + keccak: keccak.finish(), + blake3: blake3.finish(), + lanes: lanes.finish(), + hint: hint.finish(), + public: public.finish(), + } +} diff --git a/prover/src/lfm/constraint_tests.rs b/prover/src/lfm/constraint_tests.rs new file mode 100644 index 000000000..f849d62bf --- /dev/null +++ b/prover/src/lfm/constraint_tests.rs @@ -0,0 +1,1989 @@ +//! The constraint-evaluation leg: lowering differential, cost census, and the +//! falsifications for each mechanism the lowering relies on. +//! +//! ## The oracle +//! +//! `eval_program_verifier` — the production CPU interpreter, on the OOD shape — +//! run over the DESERIALIZED artifact, never a local reimplementation of the +//! algebra. It is in turn pinned against the compiled folders by +//! `tests::constraint_artifact_tests`, so the chain from an AIR's Rust +//! constraints to the number this machine computes has no unpinned link. +//! +//! ## What this suite cannot see +//! +//! It executes; it does not prove. Per method rule 2, execution says nothing +//! about whether the CHIPS agree with the executor — the executor mirrors the +//! ALU it is checking. `constraint_leg_proves_and_verifies` is the test that +//! sees the chips, and it is deliberately on a small AIR: the differential's job +//! is coverage across all 28 tables, the proof's job is to close the +//! executor-vs-chip gap once. + +use stark::constraint_ir::{ConstraintArtifact, eval_program_verifier}; +use stark::frame::Frame; +use stark::proof::options::GoldilocksCubicProofOptions; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; +use crate::test_utils::{NUM_PRODUCTION_AIRS, production_airs}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::constraints::{ + Analysis, OodOperands, analyze, emit_analyzed, hint_ood_frame, ood_frame_words, +}; +use super::executor::execute; +use super::hash::TestPermutation; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +fn options() -> stark::proof::options::ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// Deterministic SplitMix64, matching the artifact suite's generator so the two +/// sweep the same kind of input. +struct SplitMix64(u64); + +impl SplitMix64 { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn fp3(&mut self) -> FEE { + FEE::new([ + FE::from(self.next_u64()), + FE::from(self.next_u64()), + FE::from(self.next_u64()), + ]) + } +} + +/// One AIR's OOD inputs: an all-extension frame with the verifier's next-row +/// PRUNING already applied, plus the per-proof uniforms. +/// +/// The pruning matters to the differential, not just to the cost: the verifier +/// reconstructs an undeclared next-row column as ZERO, so a host frame that put +/// a random value there would be comparing the machine against a frame no +/// verifier can produce. +struct OodFixture { + /// `steps[offset]` = `[main | aux]`, aux starting at `main_width`. + steps: Vec>, + main_width: usize, + aux_width: usize, + rap_challenges: Vec, + alpha_powers: Vec, + table_offset: FEE, +} + +impl OodFixture { + fn sample(artifact: &ConstraintArtifact, rng: &mut SplitMix64) -> Self { + let shape = &artifact.shape; + let main_width = shape.main_width as usize; + let aux_width = shape.aux_width as usize; + let width = main_width + aux_width; + let num_steps = shape.transition_offsets.len().max(1); + + let steps = (0..num_steps) + .map(|offset| { + (0..width) + .map(|col| { + let opened = offset == 0 || shape.next_row_columns.contains(&(col as u32)); + if opened { rng.fp3() } else { FEE::zero() } + }) + .collect() + }) + .collect(); + + Self { + steps, + main_width, + aux_width, + // [z, alpha] — the LogUp RAP challenges, in the verifier's order. + rap_challenges: vec![rng.fp3(), rng.fp3()], + alpha_powers: (0..shape.max_bus_elements as usize + 2) + .map(|_| rng.fp3()) + .collect(), + table_offset: rng.fp3(), + } + } + + /// The oracle's frame: the same values, in the verifier's own container. + fn frame(&self) -> Frame { + Frame::new( + self.steps + .iter() + .map(|s| { + TableView::new( + vec![s[..self.main_width].to_vec()], + vec![s[self.main_width..].to_vec()], + ) + }) + .collect(), + ) + } + + /// The arena the machine reads, in [`hint_ood_frame`]'s order: every opened + /// entry, step by step, column by column — pruned entries omitted because + /// the program supplies its own zero for them. + fn arena(&self, artifact: &ConstraintArtifact) -> Vec { + let shape = &artifact.shape; + let mut out = Vec::new(); + for (offset, step) in self.steps.iter().enumerate() { + for (col, v) in step.iter().enumerate() { + if offset == 0 || shape.next_row_columns.contains(&(col as u32)) { + out.push(ext_word(v)); + } + } + } + out + } + + fn uniform_arena(&self) -> Vec { + self.rap_challenges + .iter() + .chain(&self.alpha_powers) + .chain(std::iter::once(&self.table_offset)) + .map(ext_word) + .collect() + } +} + +/// Builds the differential program for one artifact: hint the frame and the +/// uniforms, lower the constraints, publish nothing. +/// +/// The uniforms are hinted HERE and only here. In the assembled verifier they +/// come from `TranscriptReplay` — an arena would let a prover choose its own +/// challenges — so this shortcut is a property of the isolated slice. The +/// guard asserting it (`challenges_are_not_an_arena_in_the_assembled_verifier`) +/// cannot exist until the assembled verifier does; it is owed as an OPEN entry +/// in `others/lfm-assembly-obligations.md`, not by this file. +fn differential_program( + artifact: &ConstraintArtifact, + an: &Analysis, +) -> (super::compiler::LfmProgram, Vec) { + let mut b = LfmBuilder::new(); + + let frame_arena = b.declare_arena(ood_frame_words(artifact)); + let (steps, words) = hint_ood_frame(&mut b, artifact, frame_arena, 0); + assert_eq!( + words, + ood_frame_words(artifact), + "ood_frame_words must predict what hint_ood_frame consumes" + ); + + let shape = &artifact.shape; + let num_uniforms = 2 + (shape.max_bus_elements + 2) + 1; + let uniform_arena = b.declare_arena(num_uniforms); + let mut next = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(uniform_arena, next).as_ext(); + next += 1; + c + }; + let rap_challenges = vec![take(&mut b), take(&mut b)]; + let alpha_powers: Vec<_> = (0..shape.max_bus_elements + 2) + .map(|_| take(&mut b)) + .collect(); + let table_offset = take(&mut b); + + let ood = OodOperands { + steps, + main_width: shape.main_width as usize, + rap_challenges, + alpha_powers, + table_offset, + }; + let evals = emit_analyzed(&mut b, an, &ood); + for e in &evals { + b.public(e.as_cell()); + } + (compile(b.finish()), evals) +} + +// ============================================================================= +// (a) + (b) — the lowering differential, every production AIR +// ============================================================================= + +/// ★ Every production AIR's lowered constraint program computes exactly what +/// the production interpreter computes, on random all-extension OOD frames. +/// +/// This is the acceptance criterion for the lowering pass. It runs over the +/// DESERIALIZED artifact, so the wire hop is inside the loop, and it compares +/// every constraint of every one of the 28 tables — including the three +/// continuation-only tables that no monolithic proof contains. +#[test] +fn lowered_constraints_match_the_verifier_interpreter() { + const TRIALS: usize = 4; + + let opts = options(); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let bytes = artifact.to_bytes().expect("serialize"); + let artifact = ConstraintArtifact::from_bytes(&bytes).expect("deserialize"); + let prog = artifact.program(); + let n = prog.roots.len(); + + let an = analyze(&artifact); + let (program, evals) = differential_program(&artifact, &an); + validate(&program).unwrap_or_else(|e| panic!("[{label}] lowered program invalid: {e:?}")); + + let mut rng = SplitMix64(0xC0FF_EE00 ^ label.len() as u64); + for trial in 0..TRIALS { + let fixture = OodFixture::sample(&artifact, &mut rng); + + let exec = execute( + &program, + &[fixture.arena(&artifact), fixture.uniform_arena()], + &TestPermutation, + ) + .unwrap_or_else(|e| panic!("[{label}] trial {trial}: execution failed: {e:?}")); + + // --- oracle: the production interpreter, verifier shape --- + let frame = fixture.frame(); + let ctx = TransitionEvaluationContext::::new_verifier( + &frame, + &fixture.rap_challenges, + &fixture.alpha_powers, + &fixture.table_offset, + ); + let mut expected = vec![FEE::zero(); n]; + eval_program_verifier(&prog, &ctx, &mut expected); + + for (c, want) in expected.iter().enumerate() { + let cell = exec + .memory + .get(evals[c].addr()) + .unwrap_or_else(|| panic!("[{label}] constraint {c} cell unwritten")); + let got = word_as_ext(&cell).expect("an ext value has lane 3 zero"); + assert_eq!( + got, *want, + "[{label}] trial {trial}: constraint {c} disagrees with the interpreter" + ); + } + assert_eq!( + fixture.aux_width, artifact.shape.aux_width as usize, + "[{label}] fixture and artifact disagree on aux width" + ); + } + } +} + +/// The differential's own falsification: a lowering that drops the extension +/// arithmetic must be CAUGHT. Perturbing one constraint value by one and +/// re-checking proves the comparison above is load-bearing rather than +/// comparing two zeros. +#[test] +fn the_differential_rejects_a_perturbed_constraint_value() { + let opts = options(); + let airs = production_airs(&opts); + let (label, air) = airs + .iter() + .find(|(l, _)| *l == "L2G_GLOBAL") + .expect("L2G_GLOBAL is a production AIR"); + + let artifact = ConstraintArtifact::capture(&**air); + let prog = artifact.program(); + let an = analyze(&artifact); + let (program, evals) = differential_program(&artifact, &an); + + let mut rng = SplitMix64(1); + let fixture = OodFixture::sample(&artifact, &mut rng); + let exec = execute( + &program, + &[fixture.arena(&artifact), fixture.uniform_arena()], + &TestPermutation, + ) + .expect("execution"); + + let frame = fixture.frame(); + let ctx = TransitionEvaluationContext::::new_verifier( + &frame, + &fixture.rap_challenges, + &fixture.alpha_powers, + &fixture.table_offset, + ); + let mut expected = vec![FEE::zero(); prog.roots.len()]; + eval_program_verifier(&prog, &ctx, &mut expected); + + let got = word_as_ext(&exec.memory.get(evals[0].addr()).expect("written")).expect("ext"); + assert_eq!(got, expected[0], "[{label}] baseline must agree"); + assert_ne!( + got, + &expected[0] + FEE::one(), + "[{label}] a one-off value must NOT compare equal — otherwise the \ + differential above proves nothing" + ); +} + +// ============================================================================= +// (b) — cost, against the design document's measured table +// ============================================================================= + +/// The per-AIR `instr` column of `others/lfm-constraint-lowering-design.md` §8.1, +/// as written there. Copied deliberately rather than recomputed: the point of +/// this test is to compare the emitter against the DESIGN's prediction and +/// report where they differ, which a self-consistent recomputation cannot do. +const DESIGN_INSTR: &[(&str, usize)] = &[ + // 489 → 483: main's preprocessed CPU timestamp (#1025) replaced the + // timestamp's step constraints with a fixed column and a per-table base. + ("CPU", 483), + ("BITWISE", 112), + // LT, SHIFT, EQ, BYTEWISE, LOAD, MUL, DVRM, KECCAK and KECCAK_RND: main's + // multiplicity fix (#1016) bounds each deduplicating table's μ with a byte + // check of multiplicity 1, makes the others' μ a bit, and gives SHIFT's and + // KECCAK_RND's byte checks multiplicity 1: the emitter's counts, as measured + // after it (LT 116 → 125, SHIFT 321 → 318, EQ 88 → 90, BYTEWISE 138 → 147, + // LOAD 162 → 163, MUL 320 → 330, DVRM 423 → 433, KECCAK 3_146 → 3_148, + // KECCAK_RND 12_998 → 12_980). + ("LT", 125), + ("SHIFT", 318), + ("EQ", 90), + ("BYTEWISE", 147), + ("STORE", 149), + ("CPU32", 414), + ("MEMW", 448), + ("MEMW_A", 311), + ("MEMW_R", 153), + ("LOAD", 163), + ("DECODE", 18), + ("MUL", 330), + ("DVRM", 433), + ("BRANCH", 108), + // 701 → 700: main's halt pin (#1020). + ("HALT", 700), + // COPY replaced COMMIT (main's #984) after the design table was written: the + // emitter's own count, as measured, not the design's prediction. + ("COPY", 660), + ("PAGE", 41), + // 29 → 38: main's halt pin (#1020) added the IS_PC column and the final PC + // token at the verifier's timestamp. + ("REGISTER", 38), + // 3_148 → 2_874: main's IS_WH (#1022) moved KECCAK's state pointers to + // `DWordWHH`, their top 48 bits to the IS_WH table. 2_874 → 11_996: the + // sponge mode for Keccak-256 of a buffer (739 constraints and 286 + // interactions, the 136 XOR tuples' padded-byte linear forms among them), + // the emitter's count as measured. 11_996 → 11_997: the round chip's two + // tokens weighted by the mode bits `MU_P + MU_S` instead of `μ`. + ("KECCAK", 11_997), + // 14_016 → 12_998: main replaced the θ/ρ HWSL lookups with inline μ-gated + // linear identities in `KeccakRndConstraints`, which nets −1018 constraint + // arithmetic rows (the same change whose receiver-side multiplicity drop is + // reconciled in `keccak_adapter::bitwise_ops_for`). + ("KECCAK_RND", 12_980), + ("KECCAK_RC", 26), + // Main's IS_WH table (#1022): the emitter's count, as measured. + ("IS_WH", 42), + ("ECSM", 19_264), + ("ECDAS", 22_718), + // The ECDSA-recovery table: the emitter's count, as measured. + ("ECREC", 43_231), + // main's new receiver AIR for the `hint` ecall; 418 → 419 when its syscall + // number moved from `-31` to `-32` (#984), a constant in its Ecall receive. + ("HINT", 419), + ("L2G_GLOBAL", 27), + ("L2G_MEMORY", 65), + ("GLOBAL_MEMORY", 25), +]; + +/// ★ What the emitter actually costs per AIR, against the design's table. +/// +/// Two numbers per table. `unfused` is the design's own column — arithmetic +/// nodes that survive the verify-time-base fold, one row each — and must match +/// it exactly, because a mismatch means the design measured a different program +/// than the one being lowered. `emitted` is what the pass really writes, after +/// `MulAdd` fusion and dead-code elimination. +/// +/// ### What this instrument cannot see +/// +/// Nothing about how an EPOCH is assembled: it is per distinct AIR, and the +/// sub-proof count per epoch comes from `tests::constraint_artifact_tests`, not +/// from here. It also says nothing about padded CELL cost, which depends on how +/// these rows interleave with the rest of a program's. +#[test] +fn constraint_leg_instruction_census() { + let opts = options(); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + println!("\nconstraint-leg lowering cost, per AIR"); + println!( + "{:<14} {:>7} {:>7} {:>6} {:>6} {:>5} {:>6} {:>6} {:>8} {:>8} {:>8} {:>7}", + "table", + "nodes", + "leaves", + "fold", + "foldX", + "dead", + "unrK", + "fused", + "ext", + "mulbase", + "emitted", + "unfused" + ); + + let (mut t_unfused, mut t_emitted, mut t_fused, mut t_dead, mut t_foldx) = (0, 0, 0, 0, 0); + let (mut t_dead_const, mut t_cands, mut t_orphans) = (0, 0, 0); + let mut t_orphans_all = 0; + let mut mismatches: Vec = Vec::new(); + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let r = analyze(&artifact).report().clone(); + + println!( + "{:<14} {:>7} {:>7} {:>6} {:>6} {:>5} {:>6} {:>6} {:>8} {:>8} {:>8} {:>7}", + label, + r.nodes, + r.leaves, + r.fold_base, + r.fold_ext, + r.dead, + r.unreached_const, + r.fused, + r.ext_alu, + r.mul_base, + r.alu_rows(), + r.unfused_alu_rows() + ); + + t_unfused += r.unfused_alu_rows(); + t_emitted += r.alu_rows(); + t_fused += r.fused; + t_dead += r.dead; + t_dead_const += r.unreached_const; + t_cands += r.fuse_candidates; + t_orphans += r.orphans; + t_orphans_all += r.orphans_all_kinds; + t_foldx += r.fold_ext; + + let design = DESIGN_INSTR + .iter() + .find(|(l, _)| l == label) + .map(|(_, n)| *n) + .unwrap_or_else(|| panic!("no design entry for {label}")); + // The design's column counts every surviving arithmetic node once, with + // no DCE and no extension-valued folding, so add back what this pass + // removes on top of that. + let comparable = r.unfused_alu_rows() + r.dead + r.fold_ext + r.aliased; + if comparable != design { + mismatches.push(format!( + "{label}: design {design}, emitter {comparable} (delta {})", + comparable as i64 - design as i64 + )); + } + } + + println!( + "\nTOTALS unfused {t_unfused} emitted {t_emitted} (fusion saves {t_fused})\n\ + beyond the design's rule: {t_dead} dead ROWS eliminated, \ + {t_dead_const} root-unreachable constant nodes (free either way, \ + counted under the design's `fold`), {t_foldx} extension-valued constant \ + subtrees folded\n\ + fusion: {t_cands} candidate (Add, Mul) operand pairs, {t_fused} taken \ + — the gap is sums with TWO single-consumer products, which can absorb \ + only one\n\ + locally-orphaned nodes (the design's fanout-0 measure): {t_orphans} \ + arithmetic, {t_orphans_all} over all node kinds" + ); + + assert!( + mismatches.is_empty(), + "the emitter's per-AIR cost no longer matches the design's §8.1 table \ + (this is a real finding either way — the design is measured, not \ + guessed):\n {}", + mismatches.join("\n ") + ); +} + +// ============================================================================= +// Falsifications — one per mechanism the lowering relies on +// ============================================================================= + +/// Emitting a program for one artifact, purely to count what lands in it. +fn emitted_instrs(artifact: &ConstraintArtifact) -> Vec { + let an = analyze(artifact); + let (program, _) = differential_program(artifact, &an); + program.instrs +} + +fn count_ext(instrs: &[super::instr::Instr], want: super::instr::ExtOp) -> usize { + instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::ExtAlu { op, .. } if *op == want)) + .count() +} + +/// ★ `MulAdd` fusion happens, and the single-consumer guard is what stops it. +/// +/// Two constraint sets differing only in whether the shared product is read +/// twice. Hash-consing collapses the repeated `m0·m1` into ONE node, so the +/// second set is exactly the hazard `ConstraintArtifact`'s doc comment warns +/// about: fusing a shared `Mul` into each consumer would recompute it. +/// +/// The falsification is the second half. A test that only showed fusion +/// happening would pass just as well against an emitter that fused +/// unconditionally — which is the unsound one. +#[test] +fn muladd_fusion_requires_a_single_consumer() { + let single = fusion_air(false); + let shared = fusion_air(true); + + let a_single = ConstraintArtifact::capture(&single); + let a_shared = ConstraintArtifact::capture(&shared); + + let r_single = analyze(&a_single).report().clone(); + let r_shared = analyze(&a_shared).report().clone(); + + assert!( + r_single.fused >= 1, + "a single-consumer Mul under an Add must fuse; report {r_single:?}" + ); + assert_eq!( + r_shared.fused, 0, + "a Mul read by two Adds must NOT fuse — hash-consing makes that a \ + recomputation, not a saving; report {r_shared:?}" + ); + + let i_single = emitted_instrs(&a_single); + assert_eq!( + count_ext(&i_single, super::instr::ExtOp::MulAdd), + r_single.fused, + "every fusion the report claims must be a MulAdd row in the program" + ); + assert_eq!( + count_ext(&emitted_instrs(&a_shared), super::instr::ExtOp::MulAdd), + 0, + "the shared-Mul program must contain no MulAdd row" + ); +} + +/// ★ A constraint root is a consumer: fusing it away would delete the value the +/// quotient recombination reads. +#[test] +fn a_rooted_mul_is_never_fused_away() { + let air = rooted_mul_air(); + let artifact = ConstraintArtifact::capture(&air); + let an = analyze(&artifact); + + assert_eq!( + an.report().fused, + 0, + "the only Add's product operand is also a constraint root, so it must \ + survive as its own row" + ); + + // And the root still evaluates: emit, run, compare against the interpreter. + let prog = artifact.program(); + let (program, evals) = differential_program(&artifact, &an); + let mut rng = SplitMix64(7); + let fixture = OodFixture::sample(&artifact, &mut rng); + let exec = execute( + &program, + &[fixture.arena(&artifact), fixture.uniform_arena()], + &TestPermutation, + ) + .expect("execution"); + + let frame = fixture.frame(); + let ctx = TransitionEvaluationContext::::new_verifier( + &frame, + &fixture.rap_challenges, + &fixture.alpha_powers, + &fixture.table_offset, + ); + let mut expected = vec![FEE::zero(); prog.roots.len()]; + eval_program_verifier(&prog, &ctx, &mut expected); + for (c, want) in expected.iter().enumerate() { + let got = word_as_ext(&exec.memory.get(evals[c].addr()).expect("written")).expect("ext"); + assert_eq!(got, *want, "constraint {c}"); + } +} + +/// ★ `Op::Neg` lowers to a subtract from the pooled zero, and that is the only +/// thing it can lower to — the ISA has no unary negate. +#[test] +fn neg_lowers_to_a_subtract_from_zero() { + let air = neg_air(); + let artifact = ConstraintArtifact::capture(&air); + let prog = artifact.program(); + let negs = prog + .nodes + .iter() + .filter(|n| matches!(n, stark::constraint_ir::Op::Neg(_))) + .count(); + assert!( + negs >= 1, + "the fixture AIR must actually capture a Neg node" + ); + + let instrs = emitted_instrs(&artifact); + let subs = count_ext(&instrs, super::instr::ExtOp::Sub); + let ir_subs = prog + .nodes + .iter() + .filter(|n| matches!(n, stark::constraint_ir::Op::Sub(_, _))) + .count(); + assert_eq!( + subs, + ir_subs + negs, + "each Neg must add exactly one Sub row on top of the IR's own Subs" + ); + + // One pooled zero for all of them: the constant pool interns by value. + let zeros = instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Const { value, .. } if *value == ext_word(&FEE::zero()))) + .count(); + assert!( + zeros <= 1, + "the zero constant must be interned once, found {zeros}" + ); +} + +/// ★ Dead-code elimination drops an unreachable arithmetic node instead of +/// writing it with multiplicity zero. +/// +/// No production artifact exercises this: `constraint_leg_instruction_census` +/// measures ZERO root-unreachable arithmetic nodes across all 28 tables, and +/// only three orphaned nodes of any kind (which is the design §4.3 number, +/// reproduced — they are leaves, not arithmetic, so they never cost a row). +/// A defensive path nothing reaches is a path nothing has tested, so the +/// unreachable node is INJECTED here: one extra `Mul` appended past every root. +/// +/// The falsification is the first assertion. Without it the test would pass +/// against an emitter that lowered the injected node too, since an unread write +/// is legal — merely wasteful — and the differential would not notice. +#[test] +fn dead_nodes_are_eliminated() { + use stark::constraint_ir::ArtifactNode; + + let opts = options(); + let airs = production_airs(&opts); + let (_, air) = airs + .iter() + .find(|(l, _)| *l == "L2G_GLOBAL") + .expect("L2G_GLOBAL is a production AIR"); + + let clean = ConstraintArtifact::capture(&**air); + let baseline = analyze(&clean).report().clone(); + assert_eq!( + baseline.dead, 0, + "the unmodified artifact has no dead nodes" + ); + + // A product of the last two nodes, appended past every root. Operands are + // strictly earlier, so `validate_self` still accepts it. + let mut injected = clean.clone(); + let n = injected.nodes.len() as u32; + injected.nodes.push(ArtifactNode { + op: stark::constraint_ir::device::OP_MUL, + a: n - 2, + b: n - 1, + dim: stark::constraint_ir::artifact::DIM_EXT, + }); + injected + .validate_self() + .expect("an appended node keeps the artifact well-formed"); + + let report = analyze(&injected).report().clone(); + assert_eq!( + report.dead, 1, + "the injected node is reachable from no root and must be counted dead" + ); + assert_eq!( + report.alu_rows(), + baseline.alu_rows(), + "an unreachable node must cost no rows at all" + ); + + let an = analyze(&injected); + let (program, _) = differential_program(&injected, &an); + validate(&program).expect("a program with DCE applied is valid"); + for instr in &program.instrs { + if let super::instr::Instr::ExtAlu { op, mult, .. } = instr { + assert_ne!( + *mult, 0, + "an emitted {op:?} row is never read; DCE should have removed it" + ); + } + } +} + +/// ★ The next-row PRUNING is in the program text, not in the supplied arena. +/// +/// A column the AIR does not declare is reconstructed as ZERO by the verifier. +/// If the machine hinted a value there instead, a prover could supply a next-row +/// opening the real verifier never reads — so this is a soundness property, not +/// a size one. +#[test] +fn pruned_next_row_columns_are_program_zeros() { + let opts = options(); + let airs = production_airs(&opts); + let (label, air) = airs + .iter() + .find(|(l, _)| *l == "CPU") + .expect("CPU is a production AIR"); + let artifact = ConstraintArtifact::capture(&**air); + let shape = &artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let steps = shape.transition_offsets.len(); + assert!(steps >= 2, "[{label}] needs a next-row step to prune"); + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(ood_frame_words(&artifact)); + let (frame, words) = hint_ood_frame(&mut b, &artifact, arena, 0); + + assert_eq!( + words as usize, + width + (steps - 1) * shape.next_row_columns.len(), + "[{label}] only the opened entries may consume arena words" + ); + assert!( + (words as usize) < steps * width, + "[{label}] the pruning must actually save arena words" + ); + + let zero_addr = b.felt_const(FE::zero()).addr(); + for (col, cell) in frame[1].iter().enumerate().take(width) { + let declared = shape.next_row_columns.contains(&(col as u32)); + let is_zero = cell.addr() == zero_addr; + assert_eq!( + is_zero, !declared, + "[{label}] next-row column {col}: declared={declared} but \ + pruned={is_zero}" + ); + } +} + +// ============================================================================= +// Fixture AIRs — capture paths the production tables cannot reach +// ============================================================================= + +use math::field::traits::IsField; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; + +type FixtureAir = AirWithBuses; + +fn fixture_air>( + cols: usize, + set: C, + name: &'static str, +) -> FixtureAir { + AirWithBuses::new( + cols, + AuxiliaryTraceBuildData { + interactions: vec![], + }, + &options(), + 1, + set, + ) + .with_name(name) +} + +/// `shared = false`: one product under one sum — fusable. +/// `shared = true`: the SAME product under two sums — hash-consed to one node +/// with two consumers, so not fusable. +struct FusionConstraints { + shared: bool, +} + +impl ConstraintSet for FusionConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + let m2 = b.main(0, 2); + let m3 = b.main(0, 3); + b.emit_base(0, m0.clone() * m1.clone() + m2.clone()); + if self.shared { + // Structurally identical product: capture hash-conses it. + b.emit_base(1, m0 * m1 + m3); + } else { + b.emit_base(1, m2 * m3 + m0); + } + } +} + +fn fusion_air(shared: bool) -> FixtureAir { + fixture_air( + 4, + FusionConstraints { shared }, + if shared { "SHARED" } else { "SINGLE" }, + ) +} + +/// A product that is BOTH a constraint root and an operand of a sum. +struct RootedMulConstraints; + +impl ConstraintSet for RootedMulConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + let m2 = b.main(0, 2); + b.emit_base(0, m0.clone() * m1.clone()); + b.emit_base(1, m0 * m1 + m2); + } +} + +fn rooted_mul_air() -> FixtureAir { + fixture_air(3, RootedMulConstraints, "ROOTED_MUL") +} + +/// Negation, which no production table's captured IR happens to hold in +/// isolation. +struct NegConstraints; + +impl ConstraintSet for NegConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + b.emit_base(0, -m0.clone() + m1.clone()); + b.emit_base(1, m0 - m1); + } +} + +fn neg_air() -> FixtureAir { + fixture_air(2, NegConstraints, "NEG") +} + +// ============================================================================= +// (c) + (d) — the quotient recombination, against a REAL proof +// ============================================================================= + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::domain::new_verifier_domain; +use stark::lookup::{BusPublicInputs, LOGUP_CHALLENGE_ALPHA, LOGUP_NUM_CHALLENGES}; +use stark::proof::stark::MultiProof; +use stark::proof::view::StarkProofView; +use stark::table::Table; +use stark::traits::AIR; +use stark::verifier::{Challenges, IsStarkVerifier, Verifier}; + +use super::constraints::{BoundaryTerm, QuotientShape, emit_quotient}; +use super::proof::{lfm_prove, verify_against}; +use super::registry::build_artifacts; + +/// A genuine STARK proof of a production AIR, opened up far enough that the +/// machine can be asked to redo the verifier's composition check on it. +/// +/// Everything here is READ OFF a real proof or replayed from a real transcript. +/// Nothing is synthesized: the OOD frame is the prover's, the composition parts +/// are the prover's, and the challenges come out of the production verifier's +/// own `replay_rounds_after_round_1` rather than a local Fiat-Shamir model. +pub(super) struct RealSubProof { + pub(super) artifact: ConstraintArtifact, + pub(super) ood_full: Table, + pub(super) main_width: usize, + pub(super) num_steps: usize, + pub(super) rap_challenges: Vec, + pub(super) alpha_powers: Vec, + pub(super) table_offset: FEE, + /// The table's total bus contribution `L`, undivided. The machine derives + /// `L/N` from THIS cell rather than reading a second arena word — see + /// `constraints::emit_table_offset` for why that is a soundness + /// requirement rather than a saving. + pub(super) contribution: FEE, + pub(super) zeta: FEE, + pub(super) beta: FEE, + pub(super) challenges: Challenges, + pub(super) claimed_parts: Vec, + pub(super) quotient: QuotientShape, +} + +/// Proves L2G_MEMORY — a real continuation table, and the only continuation AIR +/// with genuine constraints — over a real boundary-claim trace. +/// +/// Returns the AIR alongside the proof because the DEEP differential needs both: +/// its oracle is the production reconstruction, which takes the AIR's layout and +/// the proof's own openings. +pub(super) fn real_fixture() -> (BoxedAir, MultiProof) { + use crate::tables::local_to_global::{ + CellBoundary, FiniClaim, InitClaim, generate_local_to_global_trace, + }; + use crate::test_utils::{EPOCH_TEST_LABEL, multi_prove_ram}; + + let opts = options(); + let air = crate::continuation::l2g_memory_air(&opts, EPOCH_TEST_LABEL); + + let boundaries: Vec = (0..4u64) + .map(|i| CellBoundary { + address: 0x1000 + 8 * i, + init: InitClaim { + value: i + 1, + timestamp: 0, + originating_epoch: 0, + }, + fini: FiniClaim { + value: 2 * i + 3, + epoch: EPOCH_TEST_LABEL, + timestamp: 17 + i, + }, + }) + .collect(); + let mut trace = generate_local_to_global_trace(&boundaries); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .expect("the L2G_MEMORY fixture must prove"); + + (Box::new(air), proof) +} + +pub(super) type BoxedAir = Box>; + +pub(super) fn real_sub_proof() -> RealSubProof { + let (air, proof) = real_fixture(); + open_sub_proof(&*air, &proof) +} + +/// Replays the production verifier's rounds over a real single-table proof and +/// packages everything the constraint leg needs. +pub(super) fn open_sub_proof( + air: &dyn AIR, + proof: &MultiProof, +) -> RealSubProof { + let view = StarkProofView::Owned(&proof.proofs[0]); + + // ---- Round 1, Phase A/B/C, transcribed from `multi_verify_views` for the + // single-table case (no per-table domain separator). + let mut transcript = crate::hash_pin::block_transcript(&[]); + if air.is_preprocessed() { + transcript.append_bytes(&super::epoch_verify_tests::layout_precomputed_commitment( + air, + view.trace_length(), + )); + } + transcript.append_bytes(view.lde_trace_main_merkle_root()); + let rap_challenges: Vec = if air.has_aux_trace() { + (0..LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect() + } else { + Vec::new() + }; + if let Some(root) = view.lde_trace_aux_merkle_root() { + transcript.append_bytes(root); + } + if let Some(contribution) = view.bus_table_contribution() { + transcript.append_field_element(&contribution); + } + + let trace_length = view.trace_length(); + let domain = new_verifier_domain(air, trace_length); + let layout = Verifier::ood_layout(air); + let challenges: Challenges = Verifier::replay_rounds_after_round_1( + air, + view, + &(), + &domain, + &mut transcript, + rap_challenges.clone(), + &layout, + ); + + // ---- β, recovered from the verifier's own coefficient run and CHECKED. + // + // `replay_rounds_after_round_1` expands one geometric run of β and splits it + // into the transition coefficients then the boundary ones. That split is + // exactly the term ordering `emit_quotient` folds, so asserting it here — + // against the verifier's values, not a model — is what pins the Horner. + let nt = challenges.transition_coeffs.len(); + assert_eq!( + challenges.transition_coeffs[0], + FEE::one(), + "the coefficient run starts at beta^0" + ); + let beta = challenges.transition_coeffs[1]; + for (c, coeff) in challenges.transition_coeffs.iter().enumerate() { + assert_eq!(*coeff, beta.pow(c as u64), "transition coefficient {c}"); + } + for (k, coeff) in challenges.boundary_coeffs.iter().enumerate() { + assert_eq!( + *coeff, + beta.pow((nt + k) as u64), + "boundary coefficient {k} must continue the same run past the \ + transition constraints" + ); + } + + // ---- the OOD grid, reconstructed by the verifier's own layout so the + // pruning is not modelled here. + let ood_current = view.trace_ood_evaluations(); + let ood_next = view.trace_ood_next_evaluations(); + let ood_full = layout.reconstruct_full( + ood_current.row_major_data(), + ood_current.width(), + ood_next.row_major_data(), + ); + + let (main_width, _) = air.trace_layout(); + let bus_public_inputs = view + .bus_table_contribution() + .map(BusPublicInputs::from_contribution); + let logup_alpha_powers: Vec = if rap_challenges.len() > LOGUP_CHALLENGE_ALPHA { + let alpha = rap_challenges[LOGUP_CHALLENGE_ALPHA]; + (0..air.max_bus_elements()) + .map(|i| alpha.pow(i as u64)) + .collect() + } else { + Vec::new() + }; + let contribution = view.bus_table_contribution().unwrap_or_else(FEE::zero); + let table_offset = FE::from(trace_length as u64) + .inv() + .expect("a nonzero trace length") + * contribution; + + let boundary_constraints = air.boundary_constraints( + &(), + &rap_challenges, + bus_public_inputs.as_ref(), + trace_length, + ); + // `VerifierDomain::trace_primitive_root` is crate-private, so the generator + // is rederived the same way `new_verifier_domain` does: the root of unity of + // order `trace_length`. + let generator = ::get_primitive_root_of_unity( + trace_length.trailing_zeros() as u64, + ) + .expect("a power-of-two trace length has a root of unity"); + let boundary: Vec = boundary_constraints + .constraints + .iter() + .map(|c| BoundaryTerm { + col: if c.is_aux { main_width + c.col } else { c.col }, + point: generator.pow(c.step as u64), + value: c.value, + }) + .collect(); + + let claimed_parts: Vec = view.composition_poly_parts_ood_evaluation().to_vec(); + let artifact = ConstraintArtifact::capture(air); + + RealSubProof { + num_steps: artifact.shape.transition_offsets.len(), + quotient: QuotientShape { + log2_trace_length: trace_length.trailing_zeros(), + num_composition_parts: claimed_parts.len(), + boundary, + }, + artifact, + ood_full, + main_width, + rap_challenges, + alpha_powers: logup_alpha_powers, + table_offset, + contribution, + zeta: challenges.z, + beta, + challenges, + claimed_parts, + } +} + +impl RealSubProof { + /// The OOD frame as the machine's arena sees it: opened entries only, in + /// [`hint_ood_frame`]'s order. + fn frame_arena(&self) -> Vec { + let shape = &self.artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let mut out = Vec::new(); + for offset in 0..self.num_steps { + let row = self.ood_full.get_row(offset); + for (col, v) in row.iter().enumerate().take(width) { + if offset == 0 || shape.next_row_columns.contains(&(col as u32)) { + out.push(ext_word(v)); + } + } + } + out + } + + fn uniform_arena(&self) -> Vec { + // `alpha_powers` are DERIVED in-machine from `rap_challenges[ALPHA]` + // (`constraints::emit_alpha_powers`), so they are deliberately absent + // here — a hinted power is a claim about alpha that nothing checks. + self.rap_challenges + .iter() + .chain([&self.contribution, &self.zeta, &self.beta]) + .map(ext_word) + .collect() + } + + fn parts_arena(&self) -> Vec { + self.claimed_parts.iter().map(ext_word).collect() + } + + pub(super) fn arenas(&self) -> Vec> { + vec![self.frame_arena(), self.uniform_arena(), self.parts_arena()] + } +} + +/// The full composition-check program for one sub-proof: lower the AIR's +/// transition constraints at ζ, recombine them against the shared zerofier and +/// the boundary quotient, and ASSERT the result equals the composition value the +/// proof claims. +/// +/// The assert is the point. A program that merely computed the composition would +/// be a calculator; asserting it against the claimed parts is what makes the +/// machine's acceptance mean something, and it is what the tamper vectors below +/// have to break. +fn composition_program_source(sp: &RealSubProof) -> super::builder::LfmProgramSource { + let mut b = LfmBuilder::new(); + + let frame_arena = b.declare_arena(ood_frame_words(&sp.artifact)); + let (steps, _) = hint_ood_frame(&mut b, &sp.artifact, frame_arena, 0); + + // The alpha POWERS are no longer hinted: they are derived from the one + // alpha challenge, so the uniform arena is that much shorter. + let num_uniforms = (sp.rap_challenges.len() + 3) as u32; + let uniform_arena = b.declare_arena(num_uniforms); + let mut next = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(uniform_arena, next).as_ext(); + next += 1; + c + }; + let rap_challenges: Vec<_> = (0..sp.rap_challenges.len()).map(|_| take(&mut b)).collect(); + let alpha_powers = super::constraints::emit_alpha_powers( + &mut b, + rap_challenges[stark::lookup::LOGUP_CHALLENGE_ALPHA], + sp.alpha_powers.len(), + ); + // `L`, undivided. The per-row offset is DERIVED from it so the constraint + // leg and the LogUp closure consume one cell rather than two independently + // hinted ones (`constraints::emit_table_offset`). + let contribution = take(&mut b); + let table_offset = + super::constraints::emit_table_offset(&mut b, contribution, sp.quotient.log2_trace_length); + let zeta = take(&mut b); + let beta = take(&mut b); + + let parts_arena = b.declare_arena(sp.claimed_parts.len() as u32); + let claimed_parts: Vec<_> = (0..sp.claimed_parts.len() as u32) + .map(|i| b.hint_word(parts_arena, i).as_ext()) + .collect(); + + let ood = OodOperands { + steps, + main_width: sp.main_width, + rap_challenges, + alpha_powers, + table_offset, + }; + let (evals, _) = super::constraints::emit_constraint_evals(&mut b, &sp.artifact, &ood); + let q = emit_quotient( + &mut b, + &sp.quotient, + &ood, + zeta, + beta, + &evals, + &claimed_parts, + ); + + b.assert_eq_ext(q.claimed, q.composition); + b.public(q.composition.as_cell()); + b.finish() +} + +/// ★ (c) The machine reproduces the verifier's composition check on a REAL +/// proof of a REAL production table. +/// +/// The oracle is the proof itself: an honestly generated proof satisfies +/// `Σ_j part_j·ζ^j = boundary_quotient + Σ_c β^c·C_c/Z`, so a machine that +/// computes either side differently cannot execute the in-machine assert. That +/// makes this a differential against the production prover and verifier +/// together, not against a transcription of one formula. +#[test] +fn composition_check_matches_a_real_proof() { + let sp = real_sub_proof(); + + // Make the coverage legible, and fail rather than silently degrade if the + // fixture ever stops exercising a term. + assert_eq!( + sp.quotient.boundary.len(), + 1, + "L2G_MEMORY has bus interactions, so it carries the framework's \ + acc[0] = 0 boundary constraint — without it the boundary half of the \ + recombination would be untested" + ); + assert!( + sp.quotient.log2_trace_length >= 2, + "the zerofier must cost more than a squaring or two" + ); + assert!( + !sp.alpha_powers.is_empty(), + "the LogUp uniforms must be live" + ); + + let program = compile(composition_program_source(&sp)); + validate(&program).expect("the composition program is admissible"); + + let leg = analyze(&sp.artifact).report().clone(); + println!( + "L2G_MEMORY composition check: {} instructions total, of which {} are \ + the constraint leg ({} constraints, {} parts, log2(N) = {})", + program.instrs.len(), + leg.alu_rows(), + sp.artifact.roots.len(), + sp.quotient.num_composition_parts, + sp.quotient.log2_trace_length, + ); + + let exec = execute(&program, &sp.arenas(), &TestPermutation) + .expect("an honest proof's composition check must execute"); + + // The published value is the recomputed composition; it must equal the + // Horner fold of the parts the proof carries. + let expected = sp + .claimed_parts + .iter() + .rev() + .fold(FEE::zero(), |acc, part| acc * sp.zeta + part); + let (_, word) = exec.public_words[0]; + assert_eq!( + word_as_ext(&word).expect("ext"), + expected, + "the machine's composition must equal the claimed composition" + ); + assert!( + expected != FEE::zero(), + "a zero composition would make the assert vacuous" + ); +} + +/// ★ (c) falsification: every input the check depends on, broken one at a time. +/// +/// Each vector leaves a genuine proof's data in place and changes exactly one +/// word. The in-machine `assert_eq_ext` lowers to `diff / ZERO`, which under the +/// machine's `x/0 = error` convention makes a mismatching run UNEXECUTABLE — the +/// earliest and loudest failure, and the one that shows the assert is carrying +/// the check rather than decorating it. +#[test] +fn a_tampered_composition_input_cannot_execute() { + let sp = real_sub_proof(); + let program = compile(composition_program_source(&sp)); + execute(&program, &sp.arenas(), &TestPermutation).expect("baseline honest run"); + + /// One tamper: a name and the single word it corrupts. + type Vector = (&'static str, Box>)>); + + let vectors: Vec = vec![ + ( + "a wrong OOD frame value", + Box::new(|a: &mut Vec>| a[0][0][0] += FE::one()), + ), + ( + "a wrong LogUp challenge", + Box::new(|a: &mut Vec>| a[1][0][0] += FE::one()), + ), + ( + "a wrong out-of-domain point zeta", + Box::new(|a: &mut Vec>| { + let i = a[1].len() - 2; + a[1][i][0] += FE::one(); + }), + ), + ( + "a wrong composition challenge beta", + Box::new(|a: &mut Vec>| { + let i = a[1].len() - 1; + a[1][i][0] += FE::one(); + }), + ), + ( + "a wrong claimed composition part", + Box::new(|a: &mut Vec>| a[2][0][0] += FE::one()), + ), + ]; + + // A zeta ON the trace domain, which makes the zerofier vanish. The + // out-of-domain sampler cannot produce it, but the constraint leg does not + // contain the sampler, so the reciprocal guard rather than an argument about + // a component elsewhere is what rules it out. `g` is chosen over `1` on + // purpose: at zeta = 1 the BOUNDARY denominator vanishes too, and the run + // would fail without saying which guard caught it. + { + let generator = ::get_primitive_root_of_unity( + sp.quotient.log2_trace_length as u64, + ) + .expect("root of unity"); + assert_eq!( + generator.pow(1u64 << sp.quotient.log2_trace_length), + FE::one(), + "g^N = 1, so the zerofier vanishes at zeta = g" + ); + assert_ne!( + generator, + FE::one(), + "but the boundary denominator does not" + ); + let mut arenas = sp.arenas(); + let i = arenas[1].len() - 2; + arenas[1][i] = ext_word(&generator.to_extension::()); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "a zeta on the trace domain must be rejected by the zerofier's \ + reciprocal guard, not silently return 0/0 = 1" + ); + } + + for (what, tamper) in vectors { + let mut arenas = sp.arenas(); + tamper(&mut arenas); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "{what} must make the composition check unexecutable" + ); + } +} + +/// ★ (d) The composition check PROVES and VERIFIES. +/// +/// Per method rule 2 this is the only test in this file that says anything about +/// the chips: everything above runs the executor, which mirrors the very ALU it +/// is checking. Here the emitted rows are proved by `LFM_XALU` and friends and +/// the proof is verified against the program's own committed artifacts. +/// +/// It uses `verify_against` rather than the registry. That is deliberate and is +/// the sanctioned path for a shape that is not registered: this program is one +/// AIR's leg, not the epoch verifier, and pinning its digest would pin a shape +/// that has to move once the DEEP and opening legs land. +#[test] +fn constraint_leg_proves_and_verifies() { + let opts = options(); + let sp = real_sub_proof(); + let program = compile(composition_program_source(&sp)); + let artifacts = build_artifacts(&program, &opts); + + let proved = lfm_prove(&program, &artifacts, &sp.arenas(), &opts) + .expect("the honest composition check must execute and prove"); + + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the proved composition check must verify" + ); + + // A verifier that claims a different composition value must reject, even + // though the proof itself is untouched: the claimed public words are what + // bind the machine's output to the statement. + let mut wrong = proved.public_words.clone(); + wrong[0].1[0] += FE::one(); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &wrong, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a mismatched claimed composition must be rejected" + ); +} + +/// The emitted program is deterministic — same builder calls, same instructions, +/// same digest. That is the property registration would pin, asserted here for a +/// shape that is deliberately not in `LFM_REGISTRY`. +#[test] +fn composition_program_is_deterministic() { + let sp = real_sub_proof(); + let a = compile(composition_program_source(&sp)); + let b = compile(composition_program_source(&sp)); + assert_eq!(a.instrs.len(), b.instrs.len()); + assert_eq!(a.num_addrs, b.num_addrs); + let opts = options(); + assert_eq!( + build_artifacts(&a, &opts).program_id, + build_artifacts(&b, &opts).program_id, + "the same source must produce the same program identity" + ); +} + +// ============================================================================= +// (b) — the per-epoch budget +// ============================================================================= + +/// Rows the recombination costs for one sub-proof, MEASURED by emitting it into +/// a throwaway builder rather than counted off the source by eye. +/// +/// The operand plumbing (hints for the frame, the challenges, the constraint +/// values and the claimed parts) is built twice into two independent builders — +/// once alone and once followed by the quotient — and the difference is the +/// quotient's own rows. Emission is deterministic, so the two plumbings are +/// identical by construction. +fn quotient_rows(artifact: &ConstraintArtifact, log2_trace_length: u32) -> usize { + let shape = &artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let num_steps = shape.transition_offsets.len().max(1); + let num_parts = shape.composition_degree_multiplier as usize; + let num_constraints = artifact.roots.len(); + + let quotient = QuotientShape { + log2_trace_length, + num_composition_parts: num_parts, + // Every table with bus interactions carries the framework's single + // acc[0] = 0 constraint on the last aux column, and no production table + // declares any other boundary constraint. + boundary: if shape.has_trace_interaction { + vec![BoundaryTerm { + col: width - 1, + point: FE::one(), + value: FEE::zero(), + }] + } else { + Vec::new() + }, + }; + + let plumbing = |b: &mut LfmBuilder| { + let total = num_steps * width + 2 + num_constraints + num_parts + 1; + let arena = b.declare_arena(total as u32); + let mut idx = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(arena, idx).as_ext(); + idx += 1; + c + }; + let steps: Vec> = (0..num_steps) + .map(|_| (0..width).map(|_| take(b)).collect()) + .collect(); + let ood = OodOperands { + steps, + main_width: shape.main_width as usize, + rap_challenges: Vec::new(), + alpha_powers: Vec::new(), + table_offset: take(b), + }; + let zeta = take(b); + let beta = take(b); + let evals: Vec<_> = (0..num_constraints).map(|_| take(b)).collect(); + let parts: Vec<_> = (0..num_parts).map(|_| take(b)).collect(); + (ood, zeta, beta, evals, parts) + }; + + let mut bare = LfmBuilder::new(); + let _ = plumbing(&mut bare); + let baseline = bare.finish().instrs.len(); + + let mut full = LfmBuilder::new(); + let (ood, zeta, beta, evals, parts) = plumbing(&mut full); + let q = emit_quotient(&mut full, "ient, &ood, zeta, beta, &evals, &parts); + full.assert_eq_ext(q.claimed, q.composition); + full.finish().instrs.len() - baseline +} + +/// ★ The constraint leg for a CONTINUATION EPOCH, against the design's budget. +/// +/// The composition is `others/lfm-constraint-lowering-design.md` §8.2.2's, which +/// `tests::constraint_artifact_tests::continuation_epoch_constraint_leg` derives +/// from the real epoch shape and pins against a measured 25/26 sub-proof +/// CEILING: 14 split-table families at one chunk each, plus the nine +/// always-on-or-accelerator tables an intermediate epoch can carry (all ten on +/// the final one), plus one L2G_MEMORY. PAGE does not appear — epochs pass +/// `page_configs = &[]`. +/// +/// ⚠ THE FIXED TERM IS A CEILING SINCE #977, not a floor. That PR took +/// `FIXED_TABLE_COUNT` 11 → 5, moving COMMIT, KECCAK, KECCAK_RND, ECSM, ECDAS +/// and HINT into `TableCounts` — a run that never reaches one carries no +/// sub-proof for it. So the sums below are what a workload that calls every +/// accelerator pays, which is the right figure for a budget and the wrong one +/// for a specific epoch: `epoch_verify_tests` measures 16 on the fibonacci +/// fixture, which reaches none of the six. +/// +/// ### What this instrument cannot see +/// +/// It assumes the MINIMUM epoch, one chunk per family. A larger epoch adds +/// chunks of the cheap tables, which the design measures at +642 instructions +/// per doubling past 2^19 cycles. It also fixes one trace length for the +/// zerofier across every sub-proof, so the recombination term is a +/// representative figure rather than a per-chunk one. +#[test] +fn continuation_epoch_constraint_leg_cost() { + /// Trace length assumed for the zerofier's squaring chain. + const LOG2_TRACE_LENGTH: u32 = 20; + + /// The 14 chunked split-table families. + const SPLIT_FAMILIES: &[&str] = &[ + "CPU", "LT", "SHIFT", "EQ", "BYTEWISE", "STORE", "CPU32", "MEMW", "MEMW_A", "MEMW_R", + "LOAD", "MUL", "DVRM", "BRANCH", + ]; + /// The ten an epoch can carry outside the split families: `FIXED_TABLE_COUNT`'s + /// five (BITWISE, DECODE, KECCAK_RC, REGISTER, HALT), which contribute one + /// sub-proof each regardless of `TableCounts`, plus five of the six + /// accelerators #977 moved into `TableCounts`. HALT is last: an intermediate + /// epoch drops it. + /// + /// ⚠ HINT is the sixth accelerator and is absent from this list, which is + /// why the enumeration is ten where the pre-#977 constant said eleven. The + /// omission is inherited, not deliberate, and it makes these sums a LOWER + /// bound on the accelerator-inclusive ceiling. + const FIXED: &[&str] = &[ + "BITWISE", + "DECODE", + "COPY", + "KECCAK", + "KECCAK_RND", + "KECCAK_RC", + "REGISTER", + "ECSM", + "ECDAS", + "HALT", + ]; + + let opts = options(); + let airs = production_airs(&opts); + let cost: std::collections::BTreeMap<&str, (usize, usize, usize)> = airs + .iter() + .map(|(label, air)| { + let artifact = ConstraintArtifact::capture(&**air); + let r = analyze(&artifact).report().clone(); + ( + *label, + ( + r.alu_rows(), + r.unfused_alu_rows(), + quotient_rows(&artifact, LOG2_TRACE_LENGTH), + ), + ) + }) + .collect(); + + let sum = |labels: &[&str], pick: fn(&(usize, usize, usize)) -> usize| -> usize { + labels.iter().map(|l| pick(&cost[l])).sum() + }; + + let families = sum(SPLIT_FAMILIES, |c| c.0); + let fixed_no_halt = sum(&FIXED[..9], |c| c.0); + let halt = cost["HALT"].0; + let l2g = cost["L2G_MEMORY"].0; + + let families_unfused = sum(SPLIT_FAMILIES, |c| c.1); + let fixed_unfused = sum(&FIXED[..9], |c| c.1); + let l2g_unfused = cost["L2G_MEMORY"].1; + + let recombination = + sum(SPLIT_FAMILIES, |c| c.2) + sum(&FIXED[..9], |c| c.2) + cost["L2G_MEMORY"].2; + + let intermediate = families + fixed_no_halt + l2g; + let final_leg = intermediate + halt; + let final_total = final_leg + recombination + cost["HALT"].2; + let design_intermediate = families_unfused + fixed_unfused + l2g_unfused; + + println!( + "\ncontinuation epoch, constraint leg (minimum shape, 25-sub-proof ceiling)\n\ + \x20 14 split families {families:>7} (unfused {families_unfused})\n\ + \x20 9 fixed, no HALT {fixed_no_halt:>7} (unfused {fixed_unfused})\n\ + \x20 1 L2G_MEMORY {l2g:>7} (unfused {l2g_unfused})\n\ + \x20 INTERMEDIATE leg {intermediate:>7} vs the design's {design_intermediate}\n\ + \x20 + recombination @ log2(N) = {LOG2_TRACE_LENGTH} {recombination:>7} \ + (zerofier, beta-fold, one division, claimed-parts Horner, assert)\n\ + \x20 INTERMEDIATE total {:>7} over <=25 sub-proofs\n\ + \x20 FINAL epoch (+HALT) {final_leg:>7} leg, {final_total} total, \ + over <=26 sub-proofs", + intermediate + recombination + ); + + // The design's §8.2.2 arithmetic, reproduced from the emitter's own unfused + // counts. A mismatch means the epoch composition changed, which is a finding + // about the epoch, not about this pass. + // + // 63_393 → 62_375 (−1018): attributed in full to KECCAK_RND, which is in + // `FIXED`. Main replaced its θ/ρ HWSL lookups with inline μ-gated linear + // identities in `KeccakRndConstraints`, netting −1018 constraint arithmetic + // rows — the exact same delta the per-AIR census records for KECCAK_RND + // (14_016 → 12_998). No other table moved; this is not a blind re-bless. + // + // 62_375 → 62_676 (+301): COPY took COMMIT's place in `FIXED` (#984), at 660 + // unfused rows against COMMIT's 359. No other table in the sum moved. + // + // 62_676 → 62_698 (+22): main's multiplicity fix (#1016), the per-AIR census's + // deltas summed (LT +9, SHIFT −3, EQ +2, BYTEWISE +9, LOAD +1, MUL +10, + // DVRM +10, KECCAK +2, KECCAK_RND −18). + // + // 62_698 → 62_424 (−274): main's IS_WH (#1022), KECCAK 3_148 → 2_874 (IS_WH + // itself is not in the sum). + // + // 62_424 → 62_427 (+3): main's halt pin and preprocessed CPU timestamp + // (#1020, #1025), CPU −6 and REGISTER +9 (HALT is not in the sum). + // + // 62_427 → 71_550 (+9_123): KECCAK's sponge mode for Keccak-256 of a buffer + // (2_874 → 11_996) and its round-chip tokens on the mode bits (→ 11_997), + // the per-AIR census's delta; no other table in the sum moved. + assert_eq!( + design_intermediate, 71_550, + "the design's intermediate-epoch budget no longer reproduces" + ); + assert!( + intermediate < design_intermediate, + "fusion must not make the leg more expensive" + ); +} + +// ============================================================================= +// The DEEP leg — differential against the production reconstruction +// ============================================================================= + +use super::deep::{DeepOpening, DeepShape, emit_deep_invariants, emit_deep_point}; + +/// The DEEP shape and the γ challenge, read off a real proof's replayed +/// challenges rather than modelled. +pub(super) fn deep_shape( + sp: &RealSubProof, + air: &dyn AIR, +) -> (DeepShape, FEE) { + let layout = Verifier::ood_layout(air); + let (main_width, aux_width) = air.trace_layout(); + let num_total_cols = main_width + aux_width; + + let shape = DeepShape { + step_size: layout.step_size(), + num_eval_points: sp.num_steps * layout.step_size(), + num_total_cols, + next_row_cols: layout.next_row_cols().to_vec(), + num_composition_parts: sp.claimed_parts.len(), + log2_trace_length: sp.quotient.log2_trace_length, + commit_ood: None, + }; + + // γ, recovered from the coefficient run and CHECKED against every entry the + // verifier built: coeff[c][r] is γ raised to a position-determined exponent, + // so if the emitter's exponent formula is wrong this assertion is what says + // so — not the differential, which would only say the answer differs. + let coeffs = &sp.challenges.trace_term_coeffs; + let gamma = coeffs[1][0]; + #[allow(clippy::needless_range_loop)] // `row` is a column-index, not a row-index, into `coeffs` + for row in 0..shape.num_eval_points { + let (cols, start, stride) = shape.block_for_test(row); + for (k, &c) in cols.iter().enumerate() { + assert_eq!( + coeffs[c][row], + gamma.pow((start + k * stride) as u64), + "trace_term_coeffs[{c}][{row}] disagrees with the emitter's \ + exponent formula" + ); + } + // Every column OUTSIDE the block must carry a zero coefficient — that is + // the pruning, and it is what makes folding the window alone exact. + if row >= shape.step_size { + for c in 0..num_total_cols { + if !cols.contains(&c) { + assert_eq!( + coeffs[c][row], + FEE::zero(), + "column {c} is pruned at row {row}" + ); + } + } + } + } + for (j, g) in sp.challenges.gammas.iter().enumerate() { + assert_eq!( + *g, + gamma.pow((shape.num_surviving() + j) as u64), + "composition gamma {j} must continue the same geometric run" + ); + } + + (shape, gamma) +} + +/// ★ The machine's DEEP reconstruction equals the production verifier's, on a +/// real proof's real query openings. +/// +/// The oracle is `reconstruct_deep_composition_poly_evaluation_pair` itself, +/// fed through `compute_query_invariant_deep_terms` — the exact pair of +/// functions `verify_rounds_2_to_4` calls, with the exact values a real proof +/// carries. Nothing about the algebra is transcribed into the test. +#[test] +fn deep_reconstruction_matches_the_production_verifier() { + let (air, proof) = real_fixture(); + let sp = open_sub_proof(&*air, &proof); + let (shape, gamma) = deep_shape(&sp, &*air); + + let view = StarkProofView::Owned(&proof.proofs[0]); + let layout = Verifier::ood_layout(&*air); + let invariants = Verifier::::compute_query_invariant_deep_terms( + &sp.challenges, + view, + &sp.ood_full, + layout.next_row_cols(), + layout.step_size(), + ) + .expect("a real proof's invariant terms"); + + let domain = new_verifier_domain(&*air, view.trace_length()); + let generator = ::get_primitive_root_of_unity( + sp.quotient.log2_trace_length as u64, + ) + .expect("root of unity"); + + let mut checked = 0usize; + for (q, iota) in sp.challenges.iotas.iter().enumerate() { + let opening = view.deep_poly_opening(q); + let precomputed: &[FE] = opening + .precomputed_trace_polys() + .map(|p| p.evaluations()) + .unwrap_or(&[]); + let main = opening.main_trace_polys().evaluations(); + let aux: &[FEE] = opening + .aux_trace_polys() + .map(|a| a.evaluations()) + .unwrap_or(&[]); + let precomputed_sym: &[FE] = opening + .precomputed_trace_polys() + .map(|p| p.evaluations_sym()) + .unwrap_or(&[]); + let main_sym = opening.main_trace_polys().evaluations_sym(); + let aux_sym: &[FEE] = opening + .aux_trace_polys() + .map(|a| a.evaluations_sym()) + .unwrap_or(&[]); + + type V = Verifier; + let point = V::query_challenge_to_evaluation_point(*iota, false, &domain); + let point_sym = V::query_challenge_to_evaluation_point(*iota, true, &domain); + + // --- oracle --- + let (want, want_sym) = V::reconstruct_deep_composition_poly_evaluation_pair( + &point, + &point_sym, + &generator, + &sp.challenges, + &invariants, + layout.next_row_cols(), + layout.step_size(), + precomputed, + main, + aux, + opening.composition_poly().evaluations(), + precomputed_sym, + main_sym, + aux_sym, + opening.composition_poly().evaluations_sym(), + ) + .expect("a real proof reconstructs"); + + // --- the machine --- + let trace: Vec = precomputed + .iter() + .chain(main.iter()) + .map(|v| v.to_extension::()) + .chain(aux.iter().copied()) + .collect(); + let trace_sym: Vec = precomputed_sym + .iter() + .chain(main_sym.iter()) + .map(|v| v.to_extension::()) + .chain(aux_sym.iter().copied()) + .collect(); + assert_eq!(trace.len(), shape.num_total_cols); + + let ood_words: Vec = (0..shape.num_eval_points) + .flat_map(|r| { + let row = sp.ood_full.get_row(r); + (0..shape.num_total_cols) + .map(|c| ext_word(&row[c])) + .collect::>() + }) + .collect(); + + let mut b = LfmBuilder::new(); + let words: Vec = std::iter::once(ext_word(&gamma)) + .chain(std::iter::once(ext_word(&sp.zeta))) + .chain(ood_words.iter().copied()) + .chain(sp.claimed_parts.iter().map(ext_word)) + .chain(std::iter::once(base_word(point))) + .chain(trace.iter().map(ext_word)) + .chain( + opening + .composition_poly() + .evaluations() + .iter() + .map(ext_word), + ) + .chain(std::iter::once(base_word(point_sym))) + .chain(trace_sym.iter().map(ext_word)) + .chain( + opening + .composition_poly() + .evaluations_sym() + .iter() + .map(ext_word), + ) + .collect(); + let arena = b.declare_arena(words.len() as u32); + let mut idx = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(arena, idx).as_ext(); + idx += 1; + c + }; + let g_cell = take(&mut b); + let z_cell = take(&mut b); + let ood_steps: Vec> = (0..shape.num_eval_points) + .map(|_| (0..shape.num_total_cols).map(|_| take(&mut b)).collect()) + .collect(); + let parts: Vec<_> = (0..shape.num_composition_parts) + .map(|_| take(&mut b)) + .collect(); + let inv = emit_deep_invariants(&mut b, &shape, g_cell, z_cell, &ood_steps, &parts); + + let read_opening = |b: &mut LfmBuilder, idx: &mut u32| { + let p = super::builder::Felt(b.hint_word(arena, *idx).addr()); + *idx += 1; + let mut cells = Vec::with_capacity(shape.num_total_cols); + for _ in 0..shape.num_total_cols { + cells.push(b.hint_word(arena, *idx).as_ext()); + *idx += 1; + } + let mut ps = Vec::with_capacity(shape.num_composition_parts); + for _ in 0..shape.num_composition_parts { + ps.push(b.hint_word(arena, *idx).as_ext()); + *idx += 1; + } + DeepOpening { + point: p, + trace: cells, + parts: ps, + } + }; + let regular = read_opening(&mut b, &mut idx); + let symmetric = read_opening(&mut b, &mut idx); + let got = emit_deep_point(&mut b, &shape, g_cell, &inv, ®ular); + let got_sym = emit_deep_point(&mut b, &shape, g_cell, &inv, &symmetric); + b.public(got.as_cell()); + b.public(got_sym.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the DEEP program is admissible"); + let exec = execute(&program, &[words], &TestPermutation).expect("DEEP executes"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + want, + "query {q}: DEEP at the regular point" + ); + assert_eq!( + word_as_ext(&exec.public_words[1].1).expect("ext"), + want_sym, + "query {q}: DEEP at the symmetric point" + ); + assert_ne!(want, FEE::zero(), "query {q} must not be vacuously zero"); + + checked += 1; + if checked == 3 { + break; + } + } + assert!(checked > 0, "the fixture must carry at least one query"); + println!("DEEP differential: {checked} queries, both points each"); +} + +/// ★ The coefficient-exponent formula holds where no production AIR reaches: +/// `step_size = 2` with two next rows. +/// +/// The DEEP differential above runs on L2G_MEMORY, and every production AIR has +/// `step_size = 1` and a single next row — which collapses both strides to one. +/// A plain Horner in γ would therefore pass every test we have. This one builds +/// the verifier's own coefficient table at a wider step through +/// `build_pruned_trace_term_coeffs` and checks the emitter against it, then +/// shows the stride-1 reading DISAGREES. Without that second half the test would +/// pass against the wrong emitter. +#[test] +fn the_coefficient_exponent_formula_holds_at_a_wider_step() { + use stark::ood::build_pruned_trace_term_coeffs; + + const COLS: usize = 5; + const STEP: usize = 2; + const EVAL_POINTS: usize = 4; // two offsets x step 2 + let next_row_cols = vec![1usize, 3]; + + let shape = DeepShape { + step_size: STEP, + num_eval_points: EVAL_POINTS, + num_total_cols: COLS, + next_row_cols: next_row_cols.clone(), + num_composition_parts: 2, + log2_trace_length: 4, + commit_ood: None, + }; + let surviving = shape.num_surviving(); + assert_eq!( + surviving, + COLS * STEP + next_row_cols.len() * (EVAL_POINTS - STEP) + ); + + let gamma = FEE::new([FE::from(7u64), FE::from(11u64), FE::from(13u64)]); + let powers: Vec = (0..surviving).map(|p| gamma.pow(p as u64)).collect(); + let coeffs = build_pruned_trace_term_coeffs(&powers, COLS, EVAL_POINTS, STEP, &next_row_cols); + + let mut stride_ever_exceeds_one = false; + let mut plain_horner_would_differ = false; + + #[allow(clippy::needless_range_loop)] // `row` is a column-index, not a row-index, into `coeffs` + for row in 0..EVAL_POINTS { + let (cols, start, stride) = shape.block_for_test(row); + if stride > 1 { + stride_ever_exceeds_one = true; + } + for (k, &c) in cols.iter().enumerate() { + assert_eq!( + coeffs[c][row], + gamma.pow((start + k * stride) as u64), + "coeffs[{c}][{row}] disagrees with the emitter's (start {start}, \ + stride {stride}) formula" + ); + // The falsification: what a stride-1 fold would have used. + if coeffs[c][row] != gamma.pow((start + k) as u64) { + plain_horner_would_differ = true; + } + } + // Pruned columns carry a zero coefficient on next rows. + if row >= STEP { + for c in 0..COLS { + if !cols.contains(&c) { + assert_eq!(coeffs[c][row], FEE::zero(), "column {c} at row {row}"); + } + } + } + } + + assert!( + stride_ever_exceeds_one, + "the fixture must actually produce a stride above one" + ); + assert!( + plain_horner_would_differ, + "a plain Horner in gamma must give a DIFFERENT coefficient here, or this \ + test does not show the stride is load-bearing" + ); +} diff --git a/prover/src/lfm/constraints.rs b/prover/src/lfm/constraints.rs new file mode 100644 index 000000000..f76eea7b8 --- /dev/null +++ b/prover/src/lfm/constraints.rs @@ -0,0 +1,888 @@ +//! Lowering a captured [`ConstraintArtifact`] into LFM instructions — the +//! constraint-evaluation leg of the epoch verifier. +//! +//! The pass runs on the HOST at registry-build time, so constant folding, +//! dead-code elimination, fanout analysis and peephole fusion are free: what +//! reaches the machine is fixed program text whose digest the registry pins. +//! +//! # The IR's `dim` tags describe the PROVER; the machine runs the VERIFIER +//! +//! [`Dim`] records what the prover computes over a base-field trace frame. The +//! machine evaluates at the out-of-domain point, where the frame holds only +//! extension elements — `eval_program_verifier` resolves every [`Op::Var`] to an +//! extension value regardless of `main`. Propagating that through the +//! interpreter's rule (base only when both operands are base *values* and the +//! declared dim is base), a node is base at verify time **only if its entire +//! subtree is constants**. Sizing this leg off the declared dims understates +//! extension traffic by roughly 14×. +//! +//! Those constant-only subtrees are exactly the nodes this pass folds, so they +//! cost no rows at all rather than costing base-ALU rows. +//! +//! # What costs a row and what does not +//! +//! | IR op | lowering | rows | +//! |---|---|---| +//! | [`Op::Var`] / [`Op::RapChallenge`] / [`Op::AlphaPow`] / [`Op::TableOffset`] | an address supplied by [`OodOperands`] | 0 | +//! | [`Op::ConstBase`] / [`Op::ConstExt`] | an interned `Instr::Const` word | 1 per distinct word, program-wide | +//! | [`Op::Embed`] | **nothing** — a base word already IS its extension embedding | 0 | +//! | [`Op::Neg`] | `ExtAlu{Sub}` against the pooled zero — the ISA has no unary negate | 1 | +//! | [`Op::Add`] | `ExtAlu{Add}`, or absorbed into a producer `Mul` as `MulAdd` | 1 or 0 | +//! | [`Op::Sub`] | `ExtAlu{Sub}` | 1 | +//! | [`Op::Mul`] | `ExtAlu{Mul}`, or `MulBase` when one operand is a base word | 1 | +//! +//! `MulAdd` costs the same single row as `Mul`, so fusing is not an optimization +//! — an unfused emitter simply pays two rows where one would do. It is sound +//! only under a single-consumer guard: the IR is HASH-CONSED, so a shared `Mul` +//! feeds several parents and fusing it into each would recompute it per parent. +//! The hazard is documented on [`ConstraintArtifact`] itself. +//! +//! # `MulBase` is cost-neutral here, not a saving +//! +//! `LFM_XALU` charges one row for `Mul` and one for `MulBase`, on the same chip +//! at the same width, and its `B` operand is received as an extension token +//! either way (`chips::xalu`). A base-valued word `(c, 0, 0, 0)` is therefore a +//! legal `Mul` operand and yields the same product. This pass routes the case +//! through `MulBase` because that states the intent and pins lanes 1–2 to zero +//! by constraint, but nothing breaks — and no row is added — if it does not. +//! See `others/lfm-constraint-lowering-design.md` §3, which overstates this as a +//! 4× obligation by comparing against a hand-lowering nobody would write. + +use std::collections::HashSet; + +use stark::constraint_ir::{ConstraintArtifact, ConstraintProgram, Dim, Op}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::{Ext, Felt, LfmBuilder}; + +type Prog = ConstraintProgram; + +/// Where a lowered constraint program reads its per-proof operands. +/// +/// The distinction between these four sources is a soundness boundary, not a +/// packaging convenience (`SOUNDNESS.md` §5): OOD frame values are arena-fed and +/// must be authenticated transitively by the DEEP/opening leg, whereas +/// challenges and alpha powers are computed in-machine by the transcript replay +/// and must NEVER come from an arena. This struct takes them as already-resolved +/// cells precisely so the lowering pass cannot invent either one. +pub struct OodOperands { + /// `steps[offset][col]` — the full-width `[main | aux]` OOD frame at each + /// transition offset, aux columns starting at `main_width`. This is the same + /// concatenated indexing the verifier's reconstructed grid uses. + /// + /// Next-row entries outside the AIR's declared `next_row_columns` are + /// reconstructed as ZERO by the verifier, so the caller supplies the pooled + /// zero cell there — see [`hint_ood_frame`]. + pub steps: Vec>, + /// Where the aux columns start inside each step. + pub main_width: usize, + /// The LogUp RAP challenges, transcript-derived. + pub rap_challenges: Vec, + /// Precomputed LogUp alpha powers. + pub alpha_powers: Vec, + /// The LogUp table offset `L/N`. + /// + /// ⚠ In any program that ALSO runs the LogUp closure, this must be the cell + /// [`emit_table_offset`] returns, not a separate hint. See that function for + /// why — the two legs consuming `L` independently makes the bus-balance + /// check vacuous. The synthetic IR-lowering differential is exempt: it has + /// no trace length and no closure, so there is no second consumer to agree + /// with. + pub table_offset: Ext, +} + +/// `L/N` — the LogUp per-row offset — DERIVED from the table's total bus +/// contribution `L` rather than hinted alongside it. +/// +/// # Why this is a soundness requirement and not a convenience +/// +/// `L` has two consumers that never meet. The circular accumulator constraint +/// (`lookup.rs`'s `emit_logup_accumulated`) enforces +/// `acc_next − acc_curr − Σterms + L/N = 0`, which together with the `acc[0] = 0` +/// boundary pins `L` to the aux trace: the accumulator wraps to zero after `N` +/// rows only if `L` really is that table's total. The bus-balance closure +/// separately checks `Σ_tables L = expected`. Production computes the offset +/// from the one `L` the proof carries (`verifier.rs`'s `logup_table_offset`), so +/// the two agree by construction. +/// +/// A machine that hinted `L/N` for the constraint leg and `L` for the closure +/// would hand the prover two independent arena words. Supply a truthful `L₁/N` +/// so every accumulator wraps, and an arbitrary `L₂` so the sum hits the target: +/// both legs pass in isolation and the bus balance is a statement about numbers +/// attached to nothing. Deriving one from the other is what denies that, and it +/// costs a single `MulBase` against a program constant, since `N` is shape. +/// +/// This is the same failure the DEEP/authentication join closes one leg over — +/// two consumers of one value, agreeing only because the host filling the arena +/// agreed with itself. +pub fn emit_table_offset(b: &mut LfmBuilder, contribution: Ext, log2_trace_length: u32) -> Ext { + let n = FE::from(1u64 << log2_trace_length); + let n_inv = n + .inv() + .expect("a power-of-two trace length is nonzero, so invertible"); + let c = b.felt_const(n_inv); + b.emul_base(contribution, c) +} + +/// `[α⁰, α¹, …, α^{n−1}]` — the LogUp alpha powers, DERIVED from the one α the +/// transcript produced rather than hinted one word each. +/// +/// # Why these cannot be arena words +/// +/// Same class as [`emit_table_offset`], and worse in degree. `Op::AlphaPow{idx}` +/// resolves to `alpha_powers[idx]`, and those powers are what build every LogUp +/// FINGERPRINT: a row's tuple `(v₀, v₁, …)` enters the constraint as +/// `z − Σ vⱼ·αʲ`. A prover who supplies the powers independently of α chooses +/// the fingerprints, and with the fingerprints goes the entire lookup argument — +/// any tuple can be made to match any other. The powers are not merely a second +/// consumer of α, they are a CLAIM about α that nothing else checks. +/// +/// Deriving them costs one `ExtAlu{Mul}` per power beyond the first two (α⁰ is +/// the interned one and α¹ is α itself), and `n = AIR::max_bus_elements()` is +/// shape, so the chain length is program text. +/// +/// The alpha this consumes must itself come from the transcript replay, never +/// from an arena — that part is an assembly obligation this function cannot +/// enforce, since it takes α as a cell and cannot see where the cell came from. +pub fn emit_alpha_powers(b: &mut LfmBuilder, alpha: Ext, n: usize) -> Vec { + let mut powers = Vec::with_capacity(n); + for i in 0..n { + powers.push(match i { + 0 => b.ext_const(&FEE::one()), + 1 => alpha, + _ => { + let prev = powers[i - 1]; + b.emul(prev, alpha) + } + }); + } + powers +} + +/// What one AIR's lowering cost, measured by the pass that emitted it. +/// +/// Every field is a count of what the pass DID, not a prediction: [`analyze`] +/// and [`emit_constraint_evals`] share one analysis, so a report can never drift +/// from the program it describes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LoweringReport { + /// IR nodes in the artifact. + pub nodes: usize, + /// Leaf nodes (frame reads and uniforms) — addresses, never rows. + pub leaves: usize, + /// Arithmetic nodes unreachable from any root that would otherwise have + /// cost a ROW — the rows dead-code elimination actually saves. + pub dead: usize, + /// Constant nodes no root reaches — overwhelmingly the INTERIOR of a folded + /// subtree, whose value is absorbed into the fold rather than read. They + /// cost no row either way, and the design's census reports them under + /// `fold`, so a reader comparing the two totals must not add them again. + pub unreached_const: usize, + /// Constant-only subtrees folded to a BASE value at build time — the + /// "verify-time base" set, and the column the design doc calls `fold`. + pub fold_base: usize, + /// Constant-only subtrees whose declared dim widens them to the extension. + /// Folding these too is strictly cheaper and the design's census does not + /// count them; reported separately so the two remain comparable. + pub fold_ext: usize, + /// [`Op::Embed`] nodes lowered to a pure address alias. + pub aliased: usize, + /// `Mul`/`Add` pairs collapsed into one `MulAdd`. + pub fused: usize, + /// Fusable `(Add, Mul)` OPERAND pairs, before the one-multiply-per-row + /// limit is applied. An `Add` whose two operands are both single-consumer + /// `Mul`s offers two candidates and can absorb only one, so this exceeds + /// [`Self::fused`] by exactly the number of such sums — which is the whole + /// gap between a candidate count and a saving. + pub fuse_candidates: usize, + /// Arithmetic nodes no other node references and no root names — "fanout 0" + /// measured LOCALLY, which is the measure the design's §4.3 reports. + pub orphans: usize, + /// The same count over EVERY node kind, leaves and constants included. + pub orphans_all_kinds: usize, + /// Extension ALU rows: `Add`, `Sub`, `Mul`, `Neg` and the fused `MulAdd`s. + pub ext_alu: usize, + /// `MulBase`-routed multiplies (one base-valued operand). + pub mul_base: usize, + /// Distinct constant WORDS this AIR asks the builder for. The builder + /// interns program-wide, so a program covering several AIRs emits fewer + /// `Const` rows than the sum of these. + pub constants: usize, +} + +impl LoweringReport { + /// ALU rows the pass emits for this AIR: `ext_alu + mul_base`. Constants are + /// excluded because interning makes them a program-wide, not per-AIR, cost. + pub fn alu_rows(&self) -> usize { + self.ext_alu + self.mul_base + } + + /// ALU rows this AIR would cost with fusion switched off — the design doc's + /// per-AIR `instr` column, and an upper bound on [`Self::alu_rows`]. + pub fn unfused_alu_rows(&self) -> usize { + self.alu_rows() + self.fused + } +} + +/// A node's compile-time value, when it has one. +#[derive(Clone, Copy, Debug)] +enum Konst { + Base(FE), + Ext(FEE), +} + +impl Konst { + fn to_ext(self) -> FEE { + match self { + Konst::Base(x) => x.to_extension::(), + Konst::Ext(x) => x, + } + } + + fn is_base(self) -> bool { + matches!(self, Konst::Base(_)) + } +} + +/// Host-side analysis of one artifact: which nodes fold, which are dead, which +/// `Mul`s are absorbed by a consumer `Add`. +/// +/// Kept separate from emission so instruction counts can be measured without +/// building a program, and so the numbers reported are by construction the +/// numbers emitted. +pub struct Analysis { + prog: Prog, + konst: Vec>, + live: Vec, + /// On an `Add` that absorbs a producer `Mul`: that `Mul`'s node id. + fuse_src: Vec>, + /// Set on a `Mul` absorbed by its single consumer. + fused_away: Vec, + report: LoweringReport, +} + +impl Analysis { + /// The measured cost of this lowering. + pub fn report(&self) -> &LoweringReport { + &self.report + } + + /// The lifted program the analysis ran over. + pub fn program(&self) -> &Prog { + &self.prog + } +} + +/// Analyse an artifact without emitting anything. +/// +/// # Panics +/// +/// On a malformed artifact (out-of-range operand or constant index): the +/// artifact's own `validate_self` runs first and reports precisely what is +/// wrong. This is a build-time entry point, so failing loudly here is correct — +/// a silently mis-lowered constraint would be a wrong program with a valid +/// digest. +pub fn analyze(artifact: &ConstraintArtifact) -> Analysis { + artifact + .validate_self() + .expect("constraint artifact failed its own consistency check"); + let prog = artifact.program(); + let n = prog.nodes.len(); + + let mut report = LoweringReport { + nodes: n, + ..Default::default() + }; + + // ---- forward pass: compile-time values ---- + // + // Mirrors `interp::run` exactly, including the dim-driven widening in + // `binop`, so a folded value is bit-identical to what the interpreter would + // have computed for that node. + let mut konst: Vec> = vec![None; n]; + for i in 0..n { + let k = match prog.nodes[i] { + Op::ConstBase(idx) => Some(Konst::Base(prog.base_consts[idx as usize])), + Op::ConstExt(idx) => Some(Konst::Ext(prog.ext_consts[idx as usize])), + Op::Var { row, .. } => { + assert_eq!(row, 0, "node {i}: the capture path only reads row 0"); + report.leaves += 1; + None + } + Op::RapChallenge { .. } | Op::AlphaPow { .. } | Op::TableOffset => { + report.leaves += 1; + None + } + Op::Add(a, b) => fold_binop(&konst, a, b, prog.dims[i], |x, y| x + y, |x, y| x + y), + Op::Sub(a, b) => fold_binop(&konst, a, b, prog.dims[i], |x, y| x - y, |x, y| x - y), + Op::Mul(a, b) => fold_binop(&konst, a, b, prog.dims[i], |x, y| x * y, |x, y| x * y), + Op::Neg(a) => match (konst[a as usize], prog.dims[i]) { + (Some(Konst::Base(x)), Dim::Base) => Some(Konst::Base(-x)), + (Some(v), _) => Some(Konst::Ext(-v.to_ext())), + (None, _) => None, + }, + Op::Embed(a) => konst[a as usize].map(|v| Konst::Ext(v.to_ext())), + }; + konst[i] = k; + } + + // ---- backward pass: liveness from the roots ---- + // + // A folded node reads nothing at run time, so it does not keep its operands + // alive; that is what lets a whole constant subtree disappear rather than + // just its top node. + let mut live = vec![false; n]; + for &r in &prog.roots { + live[r as usize] = true; + } + for i in (0..n).rev() { + if !live[i] || konst[i].is_some() { + continue; + } + for a in operands(&prog.nodes[i]) { + live[a as usize] = true; + } + } + + // ---- fanout over EMITTED consumers, then fusion selection ---- + let mut fanout = vec![0u32; n]; + for i in 0..n { + if !live[i] || konst[i].is_some() { + continue; + } + for a in operands(&prog.nodes[i]) { + if konst[a as usize].is_none() { + fanout[a as usize] += 1; + } + } + } + // A root is a consumer: the quotient recombination reads it. + for &r in &prog.roots { + if konst[r as usize].is_none() { + fanout[r as usize] += 1; + } + } + + // Local fanout, the design's own measure: every reference from any node, + // folded or not, plus roots. Distinct from `fanout` above, which counts only + // the consumers that survive to read a cell. + { + let mut refs = vec![0u32; n]; + for i in 0..n { + for a in operands(&prog.nodes[i]) { + refs[a as usize] += 1; + } + } + for &r in &prog.roots { + refs[r as usize] += 1; + } + report.orphans = (0..n) + .filter(|&i| refs[i] == 0 && is_arith(&prog.nodes[i])) + .count(); + report.orphans_all_kinds = (0..n).filter(|&i| refs[i] == 0).count(); + } + + let mut fuse_src: Vec> = vec![None; n]; + let mut fused_away = vec![false; n]; + for i in 0..n { + if !live[i] || konst[i].is_some() { + continue; + } + let Op::Add(a, b) = prog.nodes[i] else { + continue; + }; + // `a` first, then `b` — `Add` is commutative and `MulAdd` computes + // `a·b + c`, so either side may supply the product. Only one can: the + // instruction carries a single multiply. + for cand in [a, b] { + if fusable(&prog, &konst, &fanout, cand) { + report.fuse_candidates += 1; + if fuse_src[i].is_none() { + fuse_src[i] = Some(cand); + fused_away[cand as usize] = true; + } + } + } + } + + // ---- cost accounting ---- + let mut constants: HashSet<[u64; 4]> = HashSet::new(); + let want_const = |k: Konst, set: &mut HashSet<[u64; 4]>| { + let w = match k { + Konst::Base(v) => super::word::base_word(v), + Konst::Ext(v) => super::word::ext_word(&v), + }; + set.insert(core::array::from_fn(|l| { + ::canonical(w[l].value()) + })); + }; + + for i in 0..n { + // Constants are classified BEFORE liveness, because a constant-only + // subtree costs no row whether or not a root reaches it — and because + // that is the split the design's census reports, so the two stay + // comparable. `dead` is then the DCE that actually saves rows. + if let Some(k) = konst[i] { + if is_arith(&prog.nodes[i]) { + if k.is_base() { + report.fold_base += 1; + } else { + report.fold_ext += 1; + } + if !live[i] { + report.unreached_const += 1; + } + } + continue; + } + if !live[i] { + if is_arith(&prog.nodes[i]) { + report.dead += 1; + } + continue; + } + if fused_away[i] { + report.fused += 1; + continue; + } + match prog.nodes[i] { + Op::Embed(_) => report.aliased += 1, + Op::Add(_, _) | Op::Sub(_, _) => report.ext_alu += 1, + Op::Neg(_) => report.ext_alu += 1, + Op::Mul(a, b) => { + if is_base_konst(&konst, a) != is_base_konst(&konst, b) { + report.mul_base += 1; + } else { + report.ext_alu += 1; + } + } + _ => {} + } + } + + // Constants actually referenced by an emitted node or a root, plus the + // pooled zero every `Neg` subtracts from. + let mut needs_zero = false; + for i in 0..n { + if !live[i] || konst[i].is_some() || fused_away[i] { + continue; + } + if matches!(prog.nodes[i], Op::Neg(_)) { + needs_zero = true; + } + for a in operands(&prog.nodes[i]) { + if let Some(k) = konst[a as usize] { + want_const(k, &mut constants); + } + } + } + for &r in &prog.roots { + if let Some(k) = konst[r as usize] { + want_const(k, &mut constants); + } + } + if needs_zero { + want_const(Konst::Base(FE::zero()), &mut constants); + } + report.constants = constants.len(); + + Analysis { + prog, + konst, + live, + fuse_src, + fused_away, + report, + } +} + +/// Lower an artifact's constraint program, returning one cell per constraint +/// root in `constraint_idx` order. +/// +/// The returned values are the AIR's transition-constraint evaluations at the +/// OOD point — the input to the zerofier/quotient recombination, not the +/// quotient itself. +pub fn emit_constraint_evals( + b: &mut LfmBuilder, + artifact: &ConstraintArtifact, + ood: &OodOperands, +) -> (Vec, LoweringReport) { + let analysis = analyze(artifact); + let evals = emit_analyzed(b, &analysis, ood); + (evals, analysis.report) +} + +/// [`emit_constraint_evals`] over an analysis the caller already has. +pub fn emit_analyzed(b: &mut LfmBuilder, an: &Analysis, ood: &OodOperands) -> Vec { + let prog = &an.prog; + let n = prog.nodes.len(); + let mut addr: Vec> = vec![None; n]; + + for i in 0..n { + if !an.live[i] || an.konst[i].is_some() || an.fused_away[i] { + continue; + } + let out = match prog.nodes[i] { + Op::Var { + main, offset, col, .. + } => { + let step = ood + .steps + .get(offset as usize) + .unwrap_or_else(|| panic!("node {i}: frame has no offset {offset}")); + let idx = if main { + col as usize + } else { + ood.main_width + col as usize + }; + *step + .get(idx) + .unwrap_or_else(|| panic!("node {i}: frame step {offset} has no column {idx}")) + } + Op::RapChallenge { idx } => ood.rap_challenges[idx as usize], + Op::AlphaPow { idx } => ood.alpha_powers[idx as usize], + Op::TableOffset => ood.table_offset, + // A base word IS its own extension embedding, so this is an address + // alias and not an instruction. + Op::Embed(a) => operand(b, an, &addr, a), + Op::Add(x, y) => match an.fuse_src[i] { + Some(m) => { + let (p, q) = match prog.nodes[m as usize] { + Op::Mul(p, q) => (p, q), + _ => unreachable!("fusion source is always a Mul"), + }; + let other = if m == x { y } else { x }; + let (p, q, c) = ( + operand(b, an, &addr, p), + operand(b, an, &addr, q), + operand(b, an, &addr, other), + ); + b.emul_add(p, q, c) + } + None => { + let (x, y) = (operand(b, an, &addr, x), operand(b, an, &addr, y)); + b.eadd(x, y) + } + }, + Op::Sub(x, y) => { + let (x, y) = (operand(b, an, &addr, x), operand(b, an, &addr, y)); + b.esub(x, y) + } + Op::Mul(x, y) => { + match (is_base_konst(&an.konst, x), is_base_konst(&an.konst, y)) { + (false, true) => { + let (a, s) = (operand(b, an, &addr, x), base_operand(b, an, y)); + b.emul_base(a, s) + } + (true, false) => { + let (a, s) = (operand(b, an, &addr, y), base_operand(b, an, x)); + b.emul_base(a, s) + } + // Both extension, or both base — a base word is a legal + // extension operand, so the plain product is correct. + _ => { + let (x, y) = (operand(b, an, &addr, x), operand(b, an, &addr, y)); + b.emul(x, y) + } + } + } + // The ISA has no unary negate: subtract from the pooled zero. + Op::Neg(x) => { + let zero = b.felt_const(FE::zero()).as_ext(); + let x = operand(b, an, &addr, x); + b.esub(zero, x) + } + Op::ConstBase(_) | Op::ConstExt(_) => unreachable!("constants fold"), + }; + addr[i] = Some(out); + } + + prog.roots + .iter() + .map(|&r| operand(b, an, &addr, r)) + .collect() +} + +// ============================================================================= +// helpers +// ============================================================================= + +fn operands(op: &Op) -> Vec { + match *op { + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => vec![a, b], + Op::Neg(a) | Op::Embed(a) => vec![a], + _ => Vec::new(), + } +} + +fn is_arith(op: &Op) -> bool { + matches!( + op, + Op::Add(_, _) | Op::Sub(_, _) | Op::Mul(_, _) | Op::Neg(_) | Op::Embed(_) + ) +} + +fn is_base_konst(konst: &[Option], i: u32) -> bool { + konst[i as usize].is_some_and(Konst::is_base) +} + +fn fold_binop( + konst: &[Option], + a: u32, + b: u32, + dim: Dim, + base_op: impl Fn(FE, FE) -> FE, + ext_op: impl Fn(FEE, FEE) -> FEE, +) -> Option { + let (ka, kb) = (konst[a as usize]?, konst[b as usize]?); + Some(match (ka, kb, dim) { + (Konst::Base(x), Konst::Base(y), Dim::Base) => Konst::Base(base_op(x, y)), + _ => Konst::Ext(ext_op(ka.to_ext(), kb.to_ext())), + }) +} + +/// Whether `cand` may be absorbed into a consumer `Add` as a `MulAdd`. +/// +/// The single-consumer guard is what makes this sound: the IR is hash-consed, so +/// a shared `Mul` feeds several parents and fusing it into each would recompute +/// it per parent — a loss, not a saving. `fanout` counts roots as consumers, so +/// a constraint root is never fused away. +fn fusable(prog: &Prog, konst: &[Option], fanout: &[u32], cand: u32) -> bool { + let i = cand as usize; + konst[i].is_none() && matches!(prog.nodes[i], Op::Mul(_, _)) && fanout[i] == 1 +} + +fn operand(b: &mut LfmBuilder, an: &Analysis, addr: &[Option], i: u32) -> Ext { + match an.konst[i as usize] { + Some(Konst::Base(v)) => b.felt_const(v).as_ext(), + Some(Konst::Ext(v)) => b.ext_const(&v), + None => addr[i as usize].unwrap_or_else(|| { + panic!("node {i} is read before it is emitted; the IR claims topological order") + }), + } +} + +fn base_operand(b: &mut LfmBuilder, an: &Analysis, i: u32) -> Felt { + match an.konst[i as usize] { + Some(Konst::Base(v)) => b.felt_const(v), + _ => unreachable!("base_operand is only called on a base-valued constant"), + } +} + +// ============================================================================= +// frame supply +// ============================================================================= + +/// Hint one AIR's OOD frame into the machine, honouring the verifier's next-row +/// PRUNING: at frame offsets past the first, only the columns the AIR declares +/// in `next_row_columns` are opened, and every other column is reconstructed as +/// ZERO. +/// +/// Getting that wrong in the permissive direction is a soundness bug rather than +/// a cost one — a column the AIR omits from its declaration is read as zero by +/// the real verifier, so a machine that hinted a value there would accept frames +/// the verifier rejects. Emitting the pooled zero constant makes the pruning +/// part of the program text instead of a property of the supplied arena. +/// +/// Returns the frame and the number of arena words consumed. +pub fn hint_ood_frame( + b: &mut LfmBuilder, + artifact: &ConstraintArtifact, + arena: super::instr::ArenaId, + first_index: u32, +) -> (Vec>, u32) { + let shape = &artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let steps = shape.transition_offsets.len().max(1); + let next_row: HashSet = shape.next_row_columns.iter().copied().collect(); + + let zero = b.felt_const(FE::zero()).as_ext(); + let mut index = first_index; + let mut out = Vec::with_capacity(steps); + for offset in 0..steps { + let mut step = Vec::with_capacity(width); + for col in 0..width { + let opened = offset == 0 || next_row.contains(&(col as u32)); + if opened { + step.push(b.hint_word(arena, index).as_ext()); + index += 1; + } else { + step.push(zero); + } + } + out.push(step); + } + (out, index - first_index) +} + +/// Arena words [`hint_ood_frame`] consumes for this AIR — the frame's opened +/// entries, which is `width + (steps − 1) · |next_row_columns|`, not `steps · +/// width`. +pub fn ood_frame_words(artifact: &ConstraintArtifact) -> u32 { + let shape = &artifact.shape; + let width = shape.main_width + shape.aux_width; + let steps = shape.transition_offsets.len().max(1) as u32; + width + (steps - 1) * shape.next_row_columns.len() as u32 +} + +// ============================================================================= +// zerofier and quotient recombination +// ============================================================================= + +/// One boundary constraint, as program SHAPE. +/// +/// Boundary constraints are deliberately NOT part of a [`ConstraintArtifact`] — +/// `AIR::boundary_constraints` is a function of the public inputs, so it is not +/// a static property of the AIR and serializing it is a separate problem. Every +/// production VM table uses `NullBoundaryConstraintBuilder`, whose only output is +/// the framework's `acc[0] = 0` on the last aux column, so in practice this is a +/// zero- or one-element list whose `point` is `g^0 = 1` and whose `value` is +/// zero. The general form is carried anyway: an emitter that silently assumed +/// the degenerate case would be wrong the first time an AIR grew a real one. +#[derive(Clone, Debug)] +pub struct BoundaryTerm { + /// Full-width `[main | aux]` column index of the value opened at ζ. + pub col: usize, + /// The trace-domain point `g^step` the constraint is anchored at. + pub point: FE, + /// The value that column must take there. + pub value: FEE, +} + +/// Everything about the recombination that is compile-time for one sub-proof. +#[derive(Clone, Debug)] +pub struct QuotientShape { + /// `log2(N)`. The zerofier costs exactly this many squarings, so the trace + /// length is program SHAPE — a machine that read it from an arena would be + /// letting the prover pick the domain it is checked against. + pub log2_trace_length: u32, + /// `composition_poly_degree_bound(N) / N`, i.e. how many parts the claimed + /// composition evaluation is split into. Also shape: the verifier rejects a + /// proof whose part count disagrees with the AIR. + pub num_composition_parts: usize, + /// The AIR's boundary constraints. + pub boundary: Vec, +} + +/// What the recombination computed. +pub struct QuotientEval { + /// `ζ^N − 1`. + pub zerofier: Ext, + /// `Σ_c β^c·C_c / Z + Σ_k β^{n+k}·(t_k(ζ) − v_k)/(ζ − p_k)`. + pub composition: Ext, + /// `Σ_j part_j·ζ^j`, the claimed value the proof carries. + pub claimed: Ext, +} + +/// Emit the zerofier, the β-power fold and the claimed-composition Horner for +/// one sub-proof. +/// +/// # One division, not one per constraint +/// +/// Every production constraint applies to every row (`RowDomain::ALL`, measured +/// across all 28 tables), so `end_exemptions` is zero everywhere and all of an +/// AIR's constraints share the zerofier `Z = ζ^N − 1`. That lets the division +/// factor out of the β-power sum: +/// +/// ```text +/// Σ_c β^c·C_c/Z = (Σ_c β^c·C_c)/Z +/// ``` +/// +/// one division per AIR rather than one per constraint. The boundary terms do +/// NOT share `Z` — they have their own denominators — so they are pre-scaled by +/// `Z` before entering the same fold, which keeps the single division while +/// still giving each boundary term its own `β` power. Naively recomputing `ζ^N` +/// and a full extension inversion per constraint, as the verifier does today, +/// would cost about 24 rows per constraint instead of per AIR. +/// +/// # Why the reciprocal rather than a direct divide +/// +/// `Z` and each `ζ − p_k` are inverted against the interned one, and the +/// quotient is then a multiply. That costs one extra row apiece and closes a +/// hole: the machine's convention is `0/0 = 1`, so a direct `Div` would silently +/// return 1 for a zero denominator with a zero numerator, whereas `1/0` has no +/// satisfying assignment (`B·OUT = A` becomes `0 = 1`) and is therefore +/// unprovable. `z` is sampled outside the trace domain so neither denominator +/// can vanish in an honest proof — but "the sampler prevents it" is a property +/// of a component this leg does not contain, and the guard costs two rows. +/// +/// `constraint_evals` are in `constraint_idx` order, `claimed_parts` in the +/// order the proof carries them (part `j` multiplying `ζ^j`). +pub fn emit_quotient( + b: &mut LfmBuilder, + shape: &QuotientShape, + ood: &OodOperands, + zeta: Ext, + beta: Ext, + constraint_evals: &[Ext], + claimed_parts: &[Ext], +) -> QuotientEval { + assert!( + !constraint_evals.is_empty() || !shape.boundary.is_empty(), + "a sub-proof with neither transition nor boundary constraints has no \ + composition to check" + ); + assert!( + !claimed_parts.is_empty(), + "the composition is claimed in at least one part" + ); + assert_eq!( + claimed_parts.len(), + shape.num_composition_parts, + "the supplied parts must match the AIR's part count, which is shape and \ + never read off the proof" + ); + + let one = b.ext_const(&FEE::one()); + + // Z = ζ^N − 1, by repeated squaring. log2(N) rows, not N. + let mut power = zeta; + for _ in 0..shape.log2_trace_length { + power = b.emul(power, power); + } + let zerofier = b.esub(power, one); + let z_inv = b.ediv(one, zerofier); + + // Terms of Σ_k β^k·X_k, highest power first: the boundary terms occupy the + // indices past the transition constraints, exactly as `replay_rounds_after_ + // round_1` splits one geometric run of β into transition then boundary + // coefficients. + let mut terms: Vec = Vec::with_capacity(constraint_evals.len() + shape.boundary.len()); + for term in shape.boundary.iter().rev() { + let opened = *ood.steps[0] + .get(term.col) + .unwrap_or_else(|| panic!("boundary column {} is outside the frame", term.col)); + let numerator = if term.value == FEE::zero() { + opened + } else { + let v = b.ext_const(&term.value); + b.esub(opened, v) + }; + let point = b.felt_const(term.point).as_ext(); + let denominator = b.esub(zeta, point); + let den_inv = b.ediv(one, denominator); + let quotient = b.emul(numerator, den_inv); + terms.push(b.emul(quotient, zerofier)); + } + terms.extend(constraint_evals.iter().rev().copied()); + + let mut acc = terms[0]; + for t in &terms[1..] { + acc = b.emul_add(acc, beta, *t); + } + let composition = b.emul(acc, z_inv); + + // claimed = Σ_j part_j·ζ^j, the same Horner the verifier folds. + let mut iter = claimed_parts.iter().rev(); + let mut claimed = *iter.next().expect("checked non-empty"); + for p in iter { + claimed = b.emul_add(claimed, zeta, *p); + } + + QuotientEval { + zerofier, + composition, + claimed, + } +} diff --git a/prover/src/lfm/deep.rs b/prover/src/lfm/deep.rs new file mode 100644 index 000000000..915a5579c --- /dev/null +++ b/prover/src/lfm/deep.rs @@ -0,0 +1,495 @@ +//! The DEEP leg: reconstructing the deep-composition polynomial at one query +//! point, in LFM instructions. +//! +//! This is where the values opened by a FRI query meet the out-of-domain frame +//! the [constraint leg](super::constraints) evaluates. For a query point `υ` the +//! verifier computes +//! +//! ```text +//! DEEP(υ) = Σ_r (Σ_c coeff[c][r]·opened[c] − oodRowSum[r]) / (υ − g^r·z) +//! + (Σ_j γ^{T+j}·H_j(υ) − hSumZpow) / (υ − z^P) +//! ``` +//! +//! and the same at the symmetric point `−υ`, which shares every query-invariant +//! term. `crypto/stark/src/verifier.rs`'s +//! `reconstruct_deep_composition_poly_evaluation_pair` is the definition and the +//! oracle; nothing here re-derives it. +//! +//! # The coefficients are powers, so nothing has to be stored +//! +//! `replay_rounds_after_round_1` samples ONE challenge γ and expands a single +//! geometric run of `num_surviving() + num_parts` powers, handing the leading +//! `num_surviving()` to `build_trace_term_coeffs` and the rest to `gammas`. So +//! `coeff[c][r]` is `γ^p` for a position-determined `p`, and every sum here is a +//! **Horner fold** — one `MulAdd` per opened value, with no coefficient table to +//! materialize, hint or authenticate. That is the single biggest structural +//! saving in this leg, and it comes from a property of the transcript rather +//! than from anything the machine does. +//! +//! The power index runs COLUMN-MAJOR within each block: for every column, each +//! row of the block. So along a fixed row the exponent advances by the block's +//! row count, and a row folds as a Horner in `γ^stride` scaled by `γ^start`. +//! Both strides are one for every production AIR, which is exactly why the +//! stride is carried explicitly rather than assumed — see [`DeepShape::block`]. +//! +//! The query POINT is an input here, not something this leg derives. Production +//! reaches it through a bit-reversal of the query index into the LDE coset +//! (`query_challenge_to_evaluation_point`), which belongs to the FRI/query leg. +//! +//! # Base openings need no conversion +//! +//! Precomputed and main-trace openings are BASE field elements; aux openings are +//! extension. Both enter the same `MulAdd`: a base word `(v, 0, 0, 0)` already +//! IS its extension embedding, so a base column costs exactly what an aux column +//! costs and no `MulBase` routing or repacking appears anywhere in this leg. +//! +//! # Reciprocals, not divisions +//! +//! Production batch-inverts the denominators and REJECTS the proof if any is +//! zero (`inplace_batch_inverse(...).ok()?`). Under the machine's `0/0 = 1` +//! convention a direct divide would instead return 1 whenever the numerator +//! vanished too — accepting exactly the malformed proof the production guard +//! exists to reject. So each denominator is inverted against the interned one, +//! which is unprovable at zero, and the quotient is a multiply. + +use math::field::traits::IsFFTField; + +use crate::tables::types::{FEE, GoldilocksField}; + +use super::builder::{Ext, Felt, LfmBuilder}; + +/// The compile-time shape of one sub-proof's DEEP reconstruction. +/// +/// Every field is program SHAPE: it fixes how many `MulAdd` rows a query costs +/// and which columns are folded. A machine that read any of it from an arena +/// would let the prover choose the sum it is checked against. +#[derive(Clone, Debug)] +pub struct DeepShape { + /// `AIR::step_size`. Rows below this open every column; rows at or above it + /// open only the transition window. + pub step_size: usize, + /// `num_transition_offsets · step_size` — rows in the full OOD grid. + pub num_eval_points: usize, + /// Full `[main | aux]` trace width, precomputed columns included. + pub num_total_cols: usize, + /// The transition-window columns, sorted — the only ones a next row opens. + pub next_row_cols: Vec, + /// Composition-polynomial parts. + pub num_composition_parts: usize, + /// `log2` of the trace length, for `g^r` and `z^P`. + pub log2_trace_length: u32, + /// The columns that answered at the commitment-time point `z0` + /// (`stark::commit_ood`), when the sub-proof is bound: each column's + /// quotient at `z` becomes its two-point quotient, by the term + /// `−c_c·(t_c(υ) − v_i)/(υ − z0)` under its current-row coefficient `c_c`. + pub commit_ood: Option, +} + +/// The answered columns of a bound sub-proof, in `[precomputed | main | aux]` +/// order: `count` columns from `first` (the main columns past the precomputed +/// ones, `stark::commit_ood::answered_columns`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CommitOodCols { + pub first: usize, + pub count: usize, +} + +impl DeepShape { + /// Terms the trace-term coefficient run covers — `OodLayout::num_surviving`. + /// Also the exponent γ is raised to for the first composition gamma. + pub fn num_surviving(&self) -> usize { + let next_rows = self.num_eval_points - self.step_size; + self.num_total_cols * self.step_size + self.next_row_cols.len() * next_rows + } + + /// Row `r`'s coefficient run: which columns it opens, the γ exponent of its + /// first term, and the STRIDE between consecutive terms' exponents. + /// + /// The stride is not always one. `build_pruned_trace_term_coeffs` walks + /// column-major within each block — for each column, every row of the block + /// — so along a fixed row the exponent advances by the block's row count, + /// not by one. Every production AIR has `step_size = 1` and a single next + /// row, which collapses both strides to one; folding a row as a plain Horner + /// in γ would therefore pass every test we have and be wrong for the first + /// AIR that widened a step. Carrying the stride costs one extra power per + /// distinct value and removes the assumption. + pub(crate) fn block(&self, row: usize) -> (Vec, usize, usize) { + let next_rows = self.num_eval_points - self.step_size; + if row < self.step_size { + ((0..self.num_total_cols).collect(), row, self.step_size) + } else { + let start = self.num_total_cols * self.step_size + (row - self.step_size); + (self.next_row_cols.clone(), start, next_rows) + } + } + + /// [`Self::block`], exposed so the differential can check the emitter's + /// exponent formula against the verifier's own coefficient table. + #[cfg(test)] + pub fn block_for_test(&self, row: usize) -> (Vec, usize, usize) { + self.block(row) + } +} + +/// Terms shared by every query of one sub-proof, and by a query's two points. +/// +/// Hoisting these is not an optimization the machine invents: production hoists +/// exactly this set into `QueryInvariantDeepTerms`, for the same reason (they do +/// not depend on the query index). Emitting them once per sub-proof rather than +/// once per query is what keeps a 219-query proof affordable. +pub struct DeepInvariants { + /// `Σ_c coeff[c][r]·oodFull[r][c]`, one per OOD row. + pub ood_row_sum: Vec, + /// `Σ_j γ^{T+j}·H_j(z^P)` over the claimed composition parts. + pub h_sum_zpow: Ext, + /// `z^P`. + pub z_pow: Ext, + /// `g^r·z`, one per OOD row. + pub row_points: Vec, + /// `γ^T`, the exponent the composition gammas start at. + pub gamma_pow_surviving: Ext, + /// `γ^{start}` for each OOD row's block, so a block folds relative to its + /// own start and is scaled once. + pub gamma_pow_block: Vec, + /// `γ^{stride}` for each OOD row's block — the base its Horner runs in. + pub gamma_stride: Vec, + /// The answers' query-invariant terms, on a bound sub-proof. + pub commit_ood: Option, +} + +/// What every query point of a bound sub-proof reuses for its answers' terms: +/// `−(γ^s)^{first}·(H(υ) − Σ_i (γ^s)^i·v_i)/(υ − z0)`, `H(υ)` the answered +/// columns' run of row 0's own Horner fold — the same coefficients row 0 gives +/// those columns at `z`, negated. +pub struct CommitOodInvariants { + /// `z0`. + pub point: Ext, + /// `Σ_i (γ^s)^i·v_i`. + pub answers_sum: Ext, + /// `−(γ^s)^{first}`: minus the answered run's scale inside row 0. + pub scale: Ext, + /// `(γ^s)^{first}`: where the answered run starts inside row 0. + pub gamma_pow_first: Ext, + /// `(γ^s)^{count}`: where the run after it starts, relative to it. + pub gamma_pow_count: Ext, +} + +/// `Σ_k terms[k]·γ^k`, terms low-to-high — one `MulAdd` per term after the +/// first, and no power table. +fn horner(b: &mut LfmBuilder, gamma: Ext, terms: &[Ext]) -> Ext { + let mut iter = terms.iter().rev(); + let mut acc = *iter.next().expect("a fold needs at least one term"); + for t in iter { + acc = b.emul_add(acc, gamma, *t); + } + acc +} + +/// `x^n` for a compile-time `n`, square-and-multiply. `n` is shape, so the row +/// count is program text rather than data-dependent. +fn pow_const(b: &mut LfmBuilder, x: Ext, n: usize) -> Ext { + assert!(n > 0, "pow_const is not defined at zero here"); + let mut result: Option = None; + let mut base = x; + let mut bits = n; + while bits > 0 { + if bits & 1 == 1 { + result = Some(match result { + None => base, + Some(acc) => b.emul(acc, base), + }); + } + bits >>= 1; + if bits > 0 { + base = b.emul(base, base); + } + } + result.expect("n > 0 sets at least one bit") +} + +/// Emit the per-sub-proof terms every query reuses. +/// +/// `ood_steps[r]` is the reconstructed OOD grid's row `r` — the same full-width +/// `[main | aux]` row the constraint leg reads, with pruned next-row entries +/// already the pooled zero. Those zeros are load-bearing here too: the verifier +/// pairs them with zero COEFFICIENTS, so folding the window alone is exact, and +/// a machine that hinted values into pruned slots would compute a different sum. +pub fn emit_deep_invariants( + b: &mut LfmBuilder, + shape: &DeepShape, + gamma: Ext, + zeta: Ext, + ood_steps: &[Vec], + claimed_parts: &[Ext], +) -> DeepInvariants { + emit_deep_invariants_bound(b, shape, gamma, zeta, ood_steps, claimed_parts, None) +} + +/// [`emit_deep_invariants`] for a sub-proof whose shape carries +/// commitment-time answers: `commit_ood` is `(z0, answers)`, the cells Phase A +/// absorbed. Its presence must match the shape's. +pub fn emit_deep_invariants_bound( + b: &mut LfmBuilder, + shape: &DeepShape, + gamma: Ext, + zeta: Ext, + ood_steps: &[Vec], + claimed_parts: &[Ext], + commit_ood: Option<(Ext, &[Ext])>, +) -> DeepInvariants { + assert_eq!( + ood_steps.len(), + shape.num_eval_points, + "the OOD grid must have one row per evaluation point" + ); + assert_eq!( + claimed_parts.len(), + shape.num_composition_parts, + "the part count is shape and is never read off the proof" + ); + + let generator = ::get_primitive_root_of_unity( + shape.log2_trace_length as u64, + ) + .expect("a power-of-two trace length has a root of unity"); + + // Block scalars: γ^start for each row's coefficient run. + let mut gamma_pow_block = Vec::with_capacity(shape.num_eval_points); + let mut gamma_stride = Vec::with_capacity(shape.num_eval_points); + let mut ood_row_sum = Vec::with_capacity(shape.num_eval_points); + let mut row_points = Vec::with_capacity(shape.num_eval_points); + + #[allow(clippy::needless_range_loop)] // `row` indexes three parallel vectors, not one + for row in 0..shape.num_eval_points { + let (cols, start, stride) = shape.block(row); + let scale = if start == 0 { + None + } else { + Some(pow_const(b, gamma, start)) + }; + let base = if stride == 1 { + gamma + } else { + pow_const(b, gamma, stride) + }; + let terms: Vec = cols.iter().map(|&c| ood_steps[row][c]).collect(); + let folded = horner(b, base, &terms); + ood_row_sum.push(match scale { + None => folded, + Some(s) => b.emul(folded, s), + }); + gamma_pow_block.push(scale.unwrap_or_else(|| b.ext_const(&FEE::one()))); + gamma_stride.push(base); + + // g^r·z. `g^r` is a program constant, so this is one MulBase per row + // rather than a chain of multiplies whose length is data. + let g_r = generator.pow(row as u64); + row_points.push(if row == 0 { + zeta + } else { + let c = b.felt_const(g_r); + b.emul_base(zeta, c) + }); + } + + let gamma_pow_surviving = pow_const(b, gamma, shape.num_surviving()); + let folded_parts = horner(b, gamma, claimed_parts); + let h_sum_zpow = b.emul(folded_parts, gamma_pow_surviving); + let z_pow = pow_const(b, zeta, shape.num_composition_parts); + + assert_eq!( + commit_ood.is_some(), + shape.commit_ood.is_some(), + "a sub-proof's answers are present exactly when its shape answers" + ); + let commit_ood = shape + .commit_ood + .zip(commit_ood) + .map(|(cols, (point, answers))| { + assert_eq!(answers.len(), cols.count, "one answer per answered column"); + assert!( + cols.count > 0 && cols.first + cols.count <= shape.num_total_cols, + "the answered columns lie inside the row" + ); + let stride = gamma_stride[0]; + let answers_sum = horner(b, stride, answers); + let one = b.ext_const(&FEE::one()); + // `z0` must avoid every point DEEP already has a quotient at (each + // `g^r·z` and `z^P`), as the host verifier checks: the product of + // the differences, inverted against one, is unprovable when any + // difference is zero. + let mut gap = b.esub(z_pow, point); + for &row_point in &row_points { + let d = b.esub(row_point, point); + gap = b.emul(gap, d); + } + let _ = b.ediv(one, gap); + let gamma_pow_first = match cols.first { + 0 => one, + first => pow_const(b, stride, first), + }; + let zero = b.ext_const(&FEE::zero()); + let scale = b.esub(zero, gamma_pow_first); + let gamma_pow_count = pow_const(b, stride, cols.count); + CommitOodInvariants { + point, + answers_sum, + scale, + gamma_pow_first, + gamma_pow_count, + } + }); + + DeepInvariants { + ood_row_sum, + h_sum_zpow, + z_pow, + row_points, + gamma_pow_surviving, + gamma_pow_block, + gamma_stride, + commit_ood, + } +} + +/// One query point's opened values, in the order the proof carries them. +/// +/// `trace` is the concatenation `precomputed ‖ main ‖ aux` — the same order +/// `reconstruct_deep_composition_poly_evaluation_pair`'s `base_at` walks, with +/// base and aux openings alike presented as extension cells because a base word +/// is already one. +pub struct DeepOpening { + /// The query's domain point, a base-field element. + pub point: Felt, + /// `precomputed ‖ main ‖ aux`, one cell per full-width column. + pub trace: Vec, + /// The composition parts opened at this point. + pub parts: Vec, +} + +/// Emit `DEEP(υ)` for one query point. +/// +/// Returns the reconstructed value; the caller feeds it to the FRI leg, which +/// is where it is finally checked. Cost is `num_surviving() + num_parts` +/// `MulAdd` rows plus a small constant per row, so this is the leg that scales +/// with query count and trace width. +pub fn emit_deep_point( + b: &mut LfmBuilder, + shape: &DeepShape, + gamma: Ext, + inv: &DeepInvariants, + opening: &DeepOpening, +) -> Ext { + assert_eq!( + opening.trace.len(), + shape.num_total_cols, + "an opening covers every trace column" + ); + assert_eq!(opening.parts.len(), shape.num_composition_parts); + + let one = b.ext_const(&FEE::one()); + let point = opening.point.as_ext(); + + let mut trace_term: Option = None; + // The answered columns' run of row 0's fold, kept for their quotients. + let mut answered_run: Option = None; + for row in 0..shape.num_eval_points { + let (cols, start, _) = shape.block(row); + let terms: Vec = cols.iter().map(|&c| opening.trace[c]).collect(); + let folded = match (row, shape.commit_ood, &inv.commit_ood) { + // Row 0 of a bound sub-proof folds as three runs — before, the + // answered columns, after — `before + γ^{first}·(run + + // γ^{count}·after)`: the same MulAdds as one Horner, and the + // middle run is the answers' quotients' fold. + (0, Some(c), Some(ci)) => { + let base = inv.gamma_stride[0]; + let run = horner(b, base, &terms[c.first..c.first + c.count]); + answered_run = Some(run); + let after = &terms[c.first + c.count..]; + let mut acc = if after.is_empty() { + run + } else { + let tail = horner(b, base, after); + b.emul_add(tail, ci.gamma_pow_count, run) + }; + if c.first > 0 { + let head = horner(b, base, &terms[..c.first]); + acc = b.emul_add(acc, ci.gamma_pow_first, head); + } + acc + } + _ => horner(b, inv.gamma_stride[row], &terms), + }; + let scaled = if start == 0 { + folded + } else { + b.emul(folded, inv.gamma_pow_block[row]) + }; + let numerator = b.esub(scaled, inv.ood_row_sum[row]); + // υ − g^r·z, inverted against one so a vanishing denominator is + // unprovable rather than silently 0/0 = 1. + let denominator = b.esub(point, inv.row_points[row]); + let den_inv = b.ediv(one, denominator); + trace_term = Some(match trace_term { + None => b.emul(numerator, den_inv), + Some(acc) => b.emul_add(numerator, den_inv, acc), + }); + } + + let folded_parts = horner(b, gamma, &opening.parts); + let h_sum = b.emul(folded_parts, inv.gamma_pow_surviving); + let h_numerator = b.esub(h_sum, inv.h_sum_zpow); + let h_denominator = b.esub(point, inv.z_pow); + let h_den_inv = b.ediv(one, h_denominator); + + let mut trace_term = trace_term.expect("at least one evaluation point"); + if let Some(ci) = inv.commit_ood.as_ref().filter(|_| answers_term_on()) { + let run = answered_run.expect("row 0 folded the answered run"); + let numerator = b.esub(run, ci.answers_sum); + let scaled = b.emul(numerator, ci.scale); + // υ − z0, inverted against one like every other denominator here. + let denominator = b.esub(point, ci.point); + let den_inv = b.ediv(one, denominator); + trace_term = b.emul_add(scaled, den_inv, trace_term); + } + b.emul_add(h_numerator, h_den_inv, trace_term) +} + +#[cfg(test)] +thread_local! { + /// A mutation for the leg's negative tests: emit DEEP without the answers' + /// quotients, on this thread. + pub(crate) static WITHOUT_ANSWERS_TERM: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Whether DEEP folds the answers' quotients: always, outside a test mutation. +fn answers_term_on() -> bool { + #[cfg(test)] + { + !WITHOUT_ANSWERS_TERM.with(|c| c.get()) + } + #[cfg(not(test))] + { + true + } +} + +/// The two points of one query — the domain point and its symmetric partner — +/// sharing every invariant. +/// +/// `υ_sym = −υ`, which is why the pair costs no extra invariants: production +/// splits the term as `denom·(coeff·base − coeff·ood)` precisely so the OOD walk +/// and the coefficient run are done once for both. +pub fn emit_deep_query( + b: &mut LfmBuilder, + shape: &DeepShape, + gamma: Ext, + inv: &DeepInvariants, + regular: &DeepOpening, + symmetric: &DeepOpening, +) -> (Ext, Ext) { + ( + emit_deep_point(b, shape, gamma, inv, regular), + emit_deep_point(b, shape, gamma, inv, symmetric), + ) +} diff --git a/prover/src/lfm/device_permit.rs b/prover/src/lfm/device_permit.rs new file mode 100644 index 000000000..0ea1be858 --- /dev/null +++ b/prover/src/lfm/device_permit.rs @@ -0,0 +1,913 @@ +//! At most one proof inside its device phases. +//! +//! # Why a gate above the prover's own +//! +//! There are TWO places an LFM proof reaches the card, and neither keeps an +//! account the other can see. +//! +//! ✓ `commit_group_device_or_host` calls `stark::gpu_lde::try_commit_row_major` +//! directly, inside `build_artifacts` — outside `multi_prove` and so outside +//! every `VramGate`. Its only check is `gpu_lde::admit` → `device_set:: +//! admit_bytes`, a `const fn` comparing ONE dispatch's bytes against the WHOLE +//! card budget. No state, no running total. +//! +//! ✓ `multi_prove` builds a fresh `VramGate::new(vram_budget)` per call, with +//! the full budget. It keeps a running total — over the tables of its OWN prove +//! and nothing else. +//! +//! ⇒ **Two proofs in flight can ask the card for twice its budget from two +//! directions at once.** That is the two-VramGates condition that caused the +//! production base aborts, one level up; the answer there was to serialise the +//! callers, and nothing since has given either mechanism a cross-proof total. +//! +//! ⛔ **So this is MUTUAL EXCLUSION, and it cannot be a byte budget.** A +//! byte-budget permit would admit a second holder "because its bytes fit" +//! against a budget the first holder is already spending — there is no +//! cross-proof total for it to compose with. +//! +//! # Why it is acquired TWICE per proof, not held across both phases +//! +//! A node's phases run `host → build_artifacts → host → multi_prove → host`. +//! The two device phases are NOT adjacent: the LFM executor and the trace fill +//! sit between them, and on a measured arity-2 level-1 node that is 2.53 s of a +//! 7.66 s node. Held across both, the permit covers 6.33 s and a level's floor +//! is 83% of its serial wall; released between them it covers 3.80 s and the +//! floor is half. Overlapping the executor IS the lever. +//! +//! Mutual exclusion is unaffected: at every instant at most one proof is inside +//! a device phase. What changes is only that a proof stops occupying the card +//! while it walks the host. +//! +//! ✓ Releasing there is safe because the artifact commit keeps nothing: +//! `try_commit_row_major` binds `(tree, _handle, _lde)` and returns the root, +//! and `_handle` — the `GpuLdeBase` holding the device tree and buffers — +//! drops at function exit. At the release point the proof holds no device +//! allocation of its own. +//! +//! # Inert until armed +//! +//! Unarmed, and at one worker, [`hold`] takes no lock and touches no counter on +//! the contended path: it reads one relaxed `usize` plus the round-3 tree +//! probe's cached `OnceLock` and returns. Nothing that ships arms it, and +//! nothing that ships sets `LAMBDA_VM_TREE_BUSY_PROBE`. + +use std::cell::{Cell, RefCell}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +/// How many sibling proofs the driver intends to run at once. 1 = the serial +/// driver, and the permit is inert. +static WORKERS: AtomicUsize = AtomicUsize::new(1); + +/// The card. One holder. +static CARD: Mutex<()> = Mutex::new(()); + +/// Holders right now. ★ The evidence for the falsifier, not a statistic: the +/// acquire asserts this is 1, so "two holders at once" fails loudly at the +/// moment it happens rather than being inferred afterwards from a VRAM abort. +static IN_FLIGHT: AtomicUsize = AtomicUsize::new(0); +/// The largest value `IN_FLIGHT` has ever taken. Printed, so a green run says +/// so rather than merely not failing. +static PEAK_IN_FLIGHT: AtomicUsize = AtomicUsize::new(0); +static ACQUISITIONS: AtomicUsize = AtomicUsize::new(0); +static HELD_NANOS: AtomicU64 = AtomicU64::new(0); +/// Holds since the process started, armed or not — the number the trace line +/// carries. Separate from `ACQUISITIONS`, which a level clears. +static TRACE_SEQ: AtomicUsize = AtomicUsize::new(0); + +/// `LFM_CARD_TRACE=1` prints one line per hold: the phase, what it waited, what +/// it held, and the two wall-clock stamps that bracket the window. +/// +/// ★ The stamps are what make an external sampler attributable. A 10 Hz +/// `nvidia-smi` log says how busy the card was; it cannot say whose window that +/// was. Slicing the sample by these two numbers answers "how idle is the card +/// INSIDE the held windows", which is the question a mutual-exclusion gate +/// raises and no aggregate can answer. +pub fn trace_enabled() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| match std::env::var("LFM_CARD_TRACE") { + Ok(v) => !v.is_empty() && v != "0", + Err(_) => false, + }) +} + +thread_local! { + /// ★ HOW LONG THIS THREAD HAS BLOCKED ON THE CARD, cumulative and never + /// cleared. + /// + /// A waiting worker's time lands inside whatever phase was running when it + /// asked — so at two siblings a node's `prove` field silently absorbs the + /// wait for `multi_prove` and stops being comparable to the serial arm's. + /// Measured at level 2: `prove` read 5.0 and 7.1 concurrently against 4.8 + /// and 4.1 serial, entirely from this. + /// + /// ⇒ MONOTONE AND READ-ONLY, not a take-and-clear. A counter that clears + /// couples every reader to every other one: whoever samples first silently + /// steals the wait from whoever samples next. Callers bracket the span they + /// care about and subtract, which composes. + static WAITED_NANOS: Cell = const { Cell::new(0) }; + /// ⛔ Re-entry detection. `Mutex` is not reentrant, so a second `hold()` on + /// a thread that already holds one parks it forever — and a silent park is + /// exactly how the census memo cost twelve minutes of a test binary sitting + /// at 0% CPU with nothing printed. A panic names it instead. + static HELD_HERE: Cell = const { Cell::new(false) }; + /// The latch this thread's `multi_prove` holds wait behind — see + /// [`defer_multi_prove_until`]. + static DEFER_PROVE: RefCell>> = const { RefCell::new(None) }; + /// The proof this thread's holds belong to, for the trace line — see + /// [`name_holder`]. + static HOLDER: RefCell> = const { RefCell::new(None) }; +} + +/// Name the proof this thread's card holds belong to until the guard drops, +/// so each `CARD HOLD` trace line ends `· who=`. A level's tasks run one +/// to a thread, so the name is what tells the global child's prove from a wide +/// node's in a log where both are just holds. Trace-only: nothing else reads it. +pub fn name_holder(name: impl Into) -> HolderGuard { + let previous = HOLDER.with(|h| h.borrow_mut().replace(name.into())); + HolderGuard { + previous, + _this_thread: std::marker::PhantomData, + } +} + +/// Puts the thread's previous holder name back when it drops. +pub struct HolderGuard { + previous: Option, + _this_thread: std::marker::PhantomData<*const ()>, +} + +impl Drop for HolderGuard { + fn drop(&mut self) { + let previous = self.previous.take(); + HOLDER.with(|h| *h.borrow_mut() = previous); + } +} + +/// ` · who=` for a trace line, or nothing when no name is set. +fn holder_suffix() -> String { + HOLDER.with(|h| { + h.borrow() + .as_ref() + .map(|name| format!(" · who={name}")) + .unwrap_or_default() + }) +} + +/// A one-way gate a `multi_prove` can be told to wait behind: shut until +/// [`open`](Self::open), then open for good. +/// +/// ★ What it orders is level 1's card. There, the WHIR global child's prove and +/// the last wide node's artifact build ask for the card within a few hundred +/// milliseconds of each other. When the global asks first, the last node's +/// artifacts wait out its whole prove, and the last node's host prep then runs +/// while the card idles; asked the other way round, the global's prove covers +/// that prep. The driver's `LFM_TREE_GLOBAL_AFTER_LAST` puts the global behind +/// this latch and has the last node open it. +pub struct CardLatch { + open: Mutex, + opened: Condvar, +} + +impl CardLatch { + pub fn new() -> Arc { + Arc::new(Self { + open: Mutex::new(false), + opened: Condvar::new(), + }) + } + + /// Open it, for every waiter now and later. + pub fn open(&self) { + *self.open.lock().unwrap_or_else(|e| e.into_inner()) = true; + self.opened.notify_all(); + } + + pub fn is_open(&self) -> bool { + *self.open.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Wait until it opens or `bound` passes: the time waited, and whether the + /// bound ended the wait. + fn wait(&self, bound: Duration) -> (Duration, bool) { + let since = Instant::now(); + let open = self.open.lock().unwrap_or_else(|e| e.into_inner()); + let (open, _) = self + .opened + .wait_timeout_while(open, bound, |open| !*open) + .unwrap_or_else(|e| e.into_inner()); + (since.elapsed(), !*open) + } +} + +/// Opens its latch when it drops: where it is dropped on purpose, and as a +/// panic unwinds, so a waiter is never left behind a task that died. +pub struct OpenOnDrop(Arc); + +impl OpenOnDrop { + pub fn new(latch: Arc) -> Self { + Self(latch) + } +} + +impl Drop for OpenOnDrop { + fn drop(&mut self) { + self.0.open(); + } +} + +/// The longest a deferred `multi_prove` waits for its latch before it queues +/// for the card anyway, and says so. The latch orders the card; it must never +/// be able to stop a proof. On the block the wait is at most about a second. +const DEFER_BOUND: Duration = Duration::from_secs(30); + +/// Deferred `multi_prove` holds, and those the bound released. +static DEFERRED: AtomicUsize = AtomicUsize::new(0); +static DEFER_TIMEOUTS: AtomicUsize = AtomicUsize::new(0); + +/// (deferred `multi_prove` holds, those the bound released) this process. +pub fn defer_stats() -> (usize, usize) { + ( + DEFERRED.load(Ordering::Relaxed), + DEFER_TIMEOUTS.load(Ordering::Relaxed), + ) +} + +/// While the guard lives, this thread's `multi_prove` holds wait for `latch` +/// to open before they queue for the card — for at most [`DEFER_BOUND`]. Other +/// phases do not wait, and neither does anything while the permit is unarmed: +/// the serial driver takes no card at all, so there is nothing to order. +pub fn defer_multi_prove_until(latch: Arc) -> DeferGuard { + DEFER_PROVE.with(|d| *d.borrow_mut() = Some(latch)); + DeferGuard { + _this_thread: std::marker::PhantomData, + } +} + +/// Clears this thread's deferral when it drops. Not `Send`: the deferral +/// belongs to the thread that installed it. +pub struct DeferGuard { + _this_thread: std::marker::PhantomData<*const ()>, +} + +impl Drop for DeferGuard { + fn drop(&mut self) { + DEFER_PROVE.with(|d| *d.borrow_mut() = None); + } +} + +/// Wait out this thread's deferral before a `multi_prove` queues for the card, +/// if it has one, and print what it cost. +fn wait_deferral(phase: &'static str) { + if phase != "multi_prove" { + return; + } + let Some(latch) = DEFER_PROVE.with(|d| d.borrow().clone()) else { + return; + }; + let t0 = stark::prove_split::epoch_secs(); + let (waited, timed_out) = latch.wait(DEFER_BOUND); + DEFERRED.fetch_add(1, Ordering::Relaxed); + if timed_out { + DEFER_TIMEOUTS.fetch_add(1, Ordering::Relaxed); + } + println!( + "CARD DEFER #{} {phase}: waited {:.3}s for its latch{} · t=[{t0:.3},{:.3}]{}", + DEFERRED.load(Ordering::Relaxed), + waited.as_secs_f64(), + if timed_out { + " — THE BOUND RAN OUT, queued anyway" + } else { + "" + }, + stark::prove_split::epoch_secs(), + holder_suffix(), + ); +} + +/// Arm the permit for `workers` concurrent proofs. `workers <= 1` leaves it +/// inert, which is the control arm on the same binary. +pub fn arm(workers: usize) { + WORKERS.store(workers.max(1), Ordering::Relaxed); +} + +/// How many sibling proofs the driver is running at once. +pub fn workers() -> usize { + WORKERS.load(Ordering::Relaxed) +} + +/// Seconds THIS THREAD has spent blocked on the card, cumulative since the +/// process started. Bracket a span and subtract to price its wait. +pub fn waited_secs() -> f64 { + WAITED_NANOS.with(|w| w.get()) as f64 / 1e9 +} + +/// What the permit did, for the line a level prints. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PermitStats { + pub acquisitions: usize, + pub peak_holders: usize, + pub held_nanos: u64, +} + +/// Read and CLEAR the counters, so a level reports its own. +pub fn take_stats() -> PermitStats { + PermitStats { + acquisitions: ACQUISITIONS.swap(0, Ordering::Relaxed), + peak_holders: PEAK_IN_FLIGHT.swap(0, Ordering::Relaxed), + held_nanos: HELD_NANOS.swap(0, Ordering::Relaxed), + } +} + +impl PermitStats { + /// The line a level prints beside its wall. `held` against the level's own + /// wall is the reading that says which resource bound the level: near 100% + /// and the card is the wall, well under and the host is. + pub fn describe(&self, level_wall_secs: f64) -> String { + let held = self.held_nanos as f64 / 1e9; + format!( + "card permit: {} acquisitions · max holders {} · held {held:.1}s of {level_wall_secs:.1}s ({:.0}%)", + self.acquisitions, + self.peak_holders, + if level_wall_secs > 0.0 { + 100.0 * held / level_wall_secs + } else { + 0.0 + }, + ) + } +} + +/// Exclusive use of the card, for as long as it is held. +pub struct CardPermit { + guard: Option>, + since: Instant, + /// ⛔ ROUND-3 TREE PROBE ONLY, `None` unless `LAMBDA_VM_TREE_BUSY_PROBE` is + /// set: which device phase this hold is, and the nanoseconds its holder + /// queued for the card. + /// + /// ⚠ Carried on EVERY permit the probe sees, not only the armed ones. An + /// unarmed stage (`arm(1)` — the WHIR global child, the block-artifact + /// root) returns before a guard exists and accumulates nothing into + /// `HELD_NANOS`, so without this its device-phase time appears in no + /// summary at all — only on the per-hold `LFM_CARD_TRACE` lines. + probe: Option<(&'static str, u64)>, + /// Trace-only, `None` unless `LFM_CARD_TRACE` is set: which device phase + /// this hold is, the seconds it queued, its sequence number, and the epoch + /// second it was acquired. + trace: Option<(&'static str, f64, usize, f64)>, + /// Trace-only: the holder's name suffix, read when the hold began. + who: String, +} + +impl Drop for CardPermit { + fn drop(&mut self) { + if self.guard.is_some() { + HELD_NANOS.fetch_add(self.since.elapsed().as_nanos() as u64, Ordering::Relaxed); + IN_FLIGHT.fetch_sub(1, Ordering::SeqCst); + HELD_HERE.with(|h| h.set(false)); + } + // ⛔ DELIBERATELY OUTSIDE the `guard` branch above. `HELD_NANOS` is the + // ARMED accounting a level reports through `take_stats`; this is a + // SECOND, separate accounting that also sees the `K = 1` stages. It + // never touches `HELD_NANOS`, `ACQUISITIONS` or `PEAK_IN_FLIGHT`, so + // every line the drivers already print keeps its exact meaning and the + // probe cannot move an existing number. + if let Some((phase, waited_nanos)) = self.probe { + super::tree_probe::note_hold( + phase, + self.since.elapsed().as_nanos() as u64, + waited_nanos, + ); + } + if let Some((phase, waited, seq, t0)) = self.trace { + println!( + "CARD HOLD #{seq} {phase}: waited {waited:.3}s · held {:.3}s · t=[{t0:.3},{:.3}]{}", + self.since.elapsed().as_secs_f64(), + stark::prove_split::epoch_secs(), + self.who, + ); + } + } +} + +/// Take the card. Blocks until the current holder releases it. +/// +/// Inert — no lock, no counter, two relaxed reads (the worker count and the +/// round-3 tree probe's `OnceLock`) — while the driver is serial, so every call +/// site can take it unconditionally. +/// +/// # Panics +/// +/// If this thread already holds it (a self-deadlock, named rather than parked), +/// or if two holders are ever observed (the falsifier, caught at the instant it +/// happens rather than inferred later from a VRAM abort). +pub fn hold() -> CardPermit { + hold_labeled("card") +} + +/// [`hold`] with the device phase named, for the trace line. The name is the +/// only thing that tells an artifact commit's window from a `multi_prove`'s in +/// a log where both are just holds. +pub fn hold_labeled(phase: &'static str) -> CardPermit { + let traced = trace_enabled(); + // One `OnceLock` read per hold — tens of them in a whole block run, never in + // a loop. Off, this is the only thing the probe costs anywhere. + let probed = super::tree_probe::enabled(); + if workers() <= 1 && !traced && !probed { + return CardPermit { + guard: None, + since: Instant::now(), + probe: None, + trace: None, + who: String::new(), + }; + } + // ★ Unarmed BUT traced or probed: there is no card to take (the serial + // driver holds it by construction), and the window is still exactly the + // device phase — which is the window the sampler has to be sliced by in the + // K=1 control too, or the two arms are compared on different definitions. + if workers() <= 1 { + let seq = TRACE_SEQ.fetch_add(1, Ordering::Relaxed); + return CardPermit { + guard: None, + since: Instant::now(), + probe: probed.then_some((phase, 0)), + trace: traced.then(|| (phase, 0.0, seq, stark::prove_split::epoch_secs())), + who: if traced { + holder_suffix() + } else { + String::new() + }, + }; + } + // ⛔ Never from a rayon worker. A holder that waits inside rayon — a nested + // `join` whose other half was stolen, a parallel iterator, another pool's + // `install` — runs its pool's queued jobs on its own thread meanwhile, and + // one of them that takes the card is a second hold on this thread (BIG 569's + // arm N parked there; which run trips it is the scheduler's choice). Taken + // from a plain thread, the card is held while rayon builds under it, and + // that thread blocks instead of running anyone's jobs. + #[cfg(feature = "parallel")] + assert!( + rayon::current_thread_index().is_none(), + "the card permit is taken on a rayon worker ({phase}): a job it runs while it \ + waits inside rayon could take it again; take it from a plain thread" + ); + assert!( + !HELD_HERE.with(|h| h.get()), + "the card permit is not reentrant and this thread already holds it; \ + taking it twice without releasing parks the thread forever" + ); + // ★ Before the queue, not in it: a deferred prove must not hold a place in + // the line while it waits for its latch, or it would still go first. + wait_deferral(phase); + // A poisoned card is a panic already being reported: take it through the + // poison rather than turning one failure into two. + let blocked_from = Instant::now(); + let guard = CARD.lock().unwrap_or_else(|e| e.into_inner()); + // Read ONCE. The trace line and `WAITED_NANOS` must carry the same wait, or + // a level's accounting and its per-hold log disagree by the bookkeeping + // below them. + let waited = blocked_from.elapsed(); + WAITED_NANOS.with(|w| w.set(w.get() + waited.as_nanos() as u64)); + HELD_HERE.with(|h| h.set(true)); + let now = IN_FLIGHT.fetch_add(1, Ordering::SeqCst) + 1; + PEAK_IN_FLIGHT.fetch_max(now, Ordering::Relaxed); + ACQUISITIONS.fetch_add(1, Ordering::Relaxed); + assert_eq!( + now, 1, + "★ TWO HOLDERS ON THE CARD. The permit is not a single gate, and two \ + proofs in their device phases can ask for twice the budget: the \ + artifact commit's admission is a per-dispatch bound with no running \ + total, and each `multi_prove` builds its own full-budget VramGate" + ); + CardPermit { + guard: Some(guard), + since: Instant::now(), + // ⚠ The SAME `waited` the trace line and `WAITED_NANOS` carry, not a + // second reading of the clock — three accountings of one wait that + // disagreed would be worse than two that do not exist. + probe: probed.then_some((phase, waited.as_nanos() as u64)), + trace: traced.then(|| { + ( + phase, + waited.as_secs_f64(), + TRACE_SEQ.fetch_add(1, Ordering::Relaxed), + stark::prove_split::epoch_secs(), + ) + }), + who: if traced { + holder_suffix() + } else { + String::new() + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `arm` is process-global, so the tests that change it run one at a time. + static ARM: Mutex<()> = Mutex::new(()); + + fn disarm(_g: &std::sync::MutexGuard<'_, ()>) { + arm(1); + let _ = take_stats(); + } + + /// ★ THE CONTROL ARM IS FREE. Unarmed, `hold` takes no lock — so the K=1 + /// control on the same binary is the serial driver, not the serial driver + /// plus a mutex per phase. + #[test] + fn unarmed_the_permit_takes_no_lock_and_counts_nothing() { + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + { + let _a = hold(); + // ⛔ The check that makes this a test rather than a smoke run: a + // second hold on this thread would DEADLOCK if the permit were + // live, so reaching the line after it proves it is not. + let _b = hold(); + } + // ⓘ Read while nothing else is armed: `disarm` ran above and the `ARM` + // guard is held, so `workers()` is 1 for every thread and no build + // anywhere in the process can reach a counter. + let stats = take_stats(); + assert_eq!(stats.acquisitions, 0, "an inert permit counts nothing"); + assert_eq!(stats.peak_holders, 0); + assert_eq!(stats.held_nanos, 0); + disarm(&g); + } + + /// ★★★ THE FALSIFIER, ASSERTED — with the overlap FORCED. + /// + /// ⛔ An earlier version of this test spun two workers through twenty fast + /// acquisitions each and asserted `peak_holders == 1`. It passed with the + /// exclusion REMOVED: the critical section was nanoseconds, so two threads + /// that were free to overlap simply never did. A check that cannot fail is + /// worse than no check, because it reads as evidence. + /// + /// ⇒ The overlap is now forced rather than hoped for. The first worker + /// takes the card and KEEPS it; the second is released only once the first + /// is inside, and then times its own acquire. With exclusion the second + /// waits out the hold; without it, it walks straight in — and both the + /// wait and `peak_holders` say so. + #[test] + fn armed_a_second_worker_waits_for_the_first() { + const HOLD: std::time::Duration = std::time::Duration::from_millis(200); + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + arm(2); + + let first_is_in = std::sync::Barrier::new(2); + let waited = Mutex::new(std::time::Duration::ZERO); + // ★ The counter the TIMING lines subtract with, read on the waiting + // thread itself. Without it a queued worker's wait stays inside + // whatever phase was running and the per-node numbers stop being + // comparable between arms. + let self_reported = Mutex::new(0.0f64); + std::thread::scope(|s| { + s.spawn(|| { + let _card = hold(); + first_is_in.wait(); + std::thread::sleep(HOLD); + }); + s.spawn(|| { + first_is_in.wait(); + let before = waited_secs(); + let t = Instant::now(); + let _card = hold(); + *waited.lock().expect("the timing lock") = t.elapsed(); + *self_reported.lock().expect("the timing lock") = waited_secs() - before; + }); + }); + + let self_reported = *self_reported.lock().expect("the timing lock"); + let waited = *waited.lock().expect("the timing lock"); + assert!( + self_reported >= HOLD.as_secs_f64() / 2.0, + "the thread must report its OWN wait, got {self_reported}s" + ); + assert!( + (self_reported - waited.as_secs_f64()).abs() < 0.05, + "the reported wait must be the observed one: {self_reported} vs {waited:?}" + ); + assert!( + waited >= HOLD / 2, + "the second worker took the card after {waited:?}, so it did not \ + wait for the first: the permit is not excluding" + ); + let stats = take_stats(); + assert_eq!( + stats.peak_holders, 1, + "★ THE FALSIFIER: two holders on the card at once" + ); + // ⚠ NOT an exact count, and the reason is worth keeping. `arm` is + // process-global by design — a driver arms it for a level, not for a + // call — so while this test holds `ARM`, any OTHER test thread building + // artifacts takes the card too and lands in these counters. An exact 2 + // failed for exactly that reason. The three assertions that remain are + // each true whatever else the process is doing, which is what makes + // them the right ones: exclusion bounds the peak at 1 no matter how + // many threads want the card. + assert!( + stats.acquisitions >= 2, + "both workers must have taken it: {stats:?}" + ); + assert!(stats.held_nanos > 0, "and the held time was accumulated"); + disarm(&g); + } + + /// ⛔ A self-deadlock is NAMED, not parked. Without the re-entry check this + /// test would hang rather than fail, which is the whole point of it. + #[test] + fn armed_a_second_hold_on_one_thread_panics_rather_than_parking() { + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + arm(2); + let panicked = std::thread::spawn(|| { + let _a = hold(); + let _b = hold(); + }) + .join(); + assert!(panicked.is_err(), "a reentrant hold must panic"); + disarm(&g); + } + + /// ★ THE RE-ENTRY BIG 569 HIT, FORCED — and now refused at its first hold. + /// A holder on a rayon worker that waits on rayon while it holds the card — + /// a nested `join` whose other half was stolen, a parallel iterator, another + /// pool's `install` — runs queued jobs of its own pool on its own thread + /// meanwhile, and one of them that takes the card is a second hold on one + /// thread. The W3 tree built its leaves' artifacts as rayon jobs that each + /// held the card around a build that uses rayon, so which run tripped it was + /// down to the scheduler (568 clean, 569's arm N parked). Here `yield_now` is + /// the wait and the sibling the only queued job: before the guard the + /// sibling's hold re-entered every time (i-m4b's repro, m4b/permit-reentry); + /// with it, the first hold on the worker is refused, before the card is + /// taken, so nothing can re-enter. + #[cfg(feature = "parallel")] + #[test] + fn armed_a_hold_on_a_rayon_worker_is_refused_before_a_sibling_can_take_it_again() { + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + arm(2); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .expect("a one-thread pool"); + let ran = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + pool.install(|| { + rayon::join( + || { + let _card = hold_labeled("build_artifacts"); + rayon::yield_now() + }, + || { + let _card = hold_labeled("build_artifacts"); + }, + ) + }) + })); + let payload = ran.expect_err("a hold on a rayon worker must be refused"); + let message = payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_default(); + assert!( + message.contains("the card permit is taken on a rayon worker"), + "another panic: {message}" + ); + // Refused before the card was taken: it is free. + drop(hold()); + disarm(&g); + } + + /// The other half: armed, a PLAIN thread holds the card while rayon works + /// under it (a parallel build, as an artifact build does), and a second + /// plain thread's hold waits for it rather than entering beside it. + #[cfg(feature = "parallel")] + #[test] + fn armed_a_plain_thread_holds_the_card_while_rayon_builds_under_it() { + use rayon::prelude::*; + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + arm(2); + let sum = std::thread::scope(|scope| { + let card = hold_labeled("build_artifacts"); + let other = scope.spawn(|| { + let _card = hold_labeled("build_artifacts"); + IN_FLIGHT.load(Ordering::SeqCst) + }); + let sum: u64 = (0..10_000u64).into_par_iter().map(|x| x * x).sum(); + std::thread::sleep(Duration::from_millis(20)); + drop(card); + assert_eq!(other.join().expect("the second holder"), 1); + sum + }); + assert_eq!(sum, (0..10_000u64).map(|x| x * x).sum::()); + disarm(&g); + } + + /// ★ A DEFERRED `multi_prove` WAITS FOR ITS LATCH, AND NOTHING ELSE DOES. + /// + /// The worker's artifact hold goes straight to the card; its prove hold + /// takes the card only after the latch opens, which the main thread does + /// after a pause the prove cannot have waited out by accident. A permit + /// that ignored the deferral takes the prove's card within milliseconds, + /// before the latch opens, and fails the order assertion. + #[test] + fn armed_a_deferred_prove_waits_for_its_latch_and_an_artifact_build_does_not() { + const PAUSE: Duration = Duration::from_millis(200); + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + arm(2); + let latch = CardLatch::new(); + let (deferred_before, _) = defer_stats(); + let artifacts_at = Mutex::new(None); + let prove_at = Mutex::new(None); + let opened_at = Mutex::new(None); + let t0 = Instant::now(); + std::thread::scope(|s| { + s.spawn(|| { + let _deferred = defer_multi_prove_until(latch.clone()); + { + let _card = hold_labeled("build_artifacts"); + } + *artifacts_at.lock().expect("the timing lock") = Some(t0.elapsed()); + let _card = hold_labeled("multi_prove"); + *prove_at.lock().expect("the timing lock") = Some(t0.elapsed()); + }); + std::thread::sleep(PAUSE); + *opened_at.lock().expect("the timing lock") = Some(t0.elapsed()); + latch.open(); + }); + let artifacts = artifacts_at + .lock() + .expect("the timing lock") + .expect("built"); + let prove = prove_at.lock().expect("the timing lock").expect("proved"); + let opened = opened_at.lock().expect("the timing lock").expect("opened"); + assert!( + artifacts < PAUSE / 2, + "an artifact build must not wait for the latch: it took the card at {artifacts:?}" + ); + assert!( + prove >= opened, + "the deferred prove took the card at {prove:?}, before its latch opened at {opened:?}" + ); + assert!( + defer_stats().0 > deferred_before, + "the deferral must be counted" + ); + disarm(&g); + } + + /// An open latch costs nothing, and a dropped guard leaves no deferral + /// behind. + #[test] + fn armed_an_open_latch_or_a_dropped_guard_does_not_delay_a_prove() { + const QUICK: Duration = Duration::from_millis(100); + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + arm(2); + let open = CardLatch::new(); + open.open(); + { + let _deferred = defer_multi_prove_until(open); + let t = Instant::now(); + let _card = hold_labeled("multi_prove"); + assert!( + t.elapsed() < QUICK, + "an open latch must not delay the prove" + ); + } + let shut = CardLatch::new(); + drop(defer_multi_prove_until(shut)); + let t = Instant::now(); + { + let _card = hold_labeled("multi_prove"); + } + assert!( + t.elapsed() < QUICK, + "a dropped guard must clear the deferral, or the prove waits for a shut latch" + ); + disarm(&g); + } + + /// Unarmed, the permit takes no card, so there is nothing to order: a + /// deferral is ignored rather than parking the serial driver behind a + /// latch only a later task would open. + #[test] + fn unarmed_a_deferral_is_ignored() { + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + let shut = CardLatch::new(); + let _deferred = defer_multi_prove_until(shut); + let t = Instant::now(); + { + let _card = hold_labeled("multi_prove"); + } + assert!( + t.elapsed() < Duration::from_millis(100), + "the serial driver must never wait for a latch" + ); + disarm(&g); + } + + /// The opener opens as a panic unwinds, so a waiter never outlives the + /// task that was to open its latch; and the wait itself is bounded. + #[test] + fn an_opener_opens_as_its_task_unwinds_and_the_wait_is_bounded() { + let latch = CardLatch::new(); + let (waited, bound_ran_out) = latch.wait(Duration::from_millis(50)); + assert!( + bound_ran_out && waited >= Duration::from_millis(40), + "a shut latch must hold a waiter until the bound: {waited:?}" + ); + let opener = OpenOnDrop::new(latch.clone()); + let died = std::thread::spawn(move || { + let _opener = opener; + panic!("the task that was to open the latch died"); + }) + .join(); + assert!(died.is_err(), "the task must have panicked"); + assert!(latch.is_open(), "its unwind must have opened the latch"); + let (waited, bound_ran_out) = latch.wait(Duration::from_secs(5)); + assert!( + !bound_ran_out && waited < Duration::from_millis(100), + "an open latch must not hold a waiter: {waited:?}" + ); + } + + /// With no global child in the level's pool the driver makes no latch, so + /// nothing defers: a node's opener would open a latch nobody waits on, and + /// a prove on a thread with no deferral takes the card at once. Both are + /// asserted, so the no-global tree cannot depend on a latch being made. + #[test] + fn armed_with_no_deferral_nothing_waits_and_an_unwaited_opener_is_harmless() { + let g = ARM.lock().expect("the arm guard is never poisoned"); + disarm(&g); + arm(2); + let unwaited = CardLatch::new(); + drop(OpenOnDrop::new(unwaited.clone())); + assert!(unwaited.is_open(), "the opener opened its latch"); + let (deferred_before, _) = defer_stats(); + let t = Instant::now(); + { + let _card = hold_labeled("multi_prove"); + } + assert!( + t.elapsed() < Duration::from_millis(100), + "a prove with no deferral must take the card at once" + ); + assert_eq!( + defer_stats().0, + deferred_before, + "no deferral may be counted on a thread that installed none" + ); + disarm(&g); + } + + /// The holder name is the thread's, restored when its guard drops. + #[test] + fn a_holder_name_tags_this_threads_holds_and_is_restored() { + assert_eq!(holder_suffix(), ""); + { + let _outer = name_holder("global"); + assert_eq!(holder_suffix(), " · who=global"); + { + let _inner = name_holder("L1N2"); + assert_eq!(holder_suffix(), " · who=L1N2"); + } + assert_eq!( + holder_suffix(), + " · who=global", + "the inner name must be undone" + ); + let elsewhere = std::thread::spawn(holder_suffix) + .join() + .expect("the other thread"); + assert_eq!(elsewhere, "", "the name is this thread's alone"); + } + assert_eq!(holder_suffix(), ""); + } + + /// The level line says which resource bound the level, so a green run + /// reports the permit's occupancy rather than only its safety. + #[test] + fn the_permit_line_reports_occupancy_against_the_level_wall() { + let stats = PermitStats { + acquisitions: 20, + peak_holders: 1, + held_nanos: 38_000_000_000, + }; + let line = stats.describe(43.2); + assert!(line.contains("max holders 1"), "{line}"); + assert!(line.contains("held 38.0s of 43.2s (88%)"), "{line}"); + } +} diff --git a/prover/src/lfm/domain_warm.rs b/prover/src/lfm/domain_warm.rs new file mode 100644 index 000000000..6b1319084 --- /dev/null +++ b/prover/src/lfm/domain_warm.rs @@ -0,0 +1,186 @@ +//! The recursion proofs' domains and twiddles built ahead of their card holds +//! (on by default; `LFM_WARM_DOMAINS=0` or `off` turns it off). +//! +//! A recursion proof builds each table length's domain and twiddles on first +//! use, inside `multi_prove`, under the exclusive card permit: the first proof +//! of a process holds the card for about 0.3 s of host work while the next +//! proof waits for it. On, the early leaf's thread warms its tables' lengths +//! beside phase B, and every proof warms its own lengths before it takes the +//! card; `multi_prove` then finds them in the cache. The cached values are the +//! ones a proof builds for itself, so no proof byte moves. + +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, Ordering}; + +use stark::proof::options::ProofOptions; + +use super::trace::LfmTraces; +use crate::tables::types::GoldilocksField; + +static BUILT: AtomicU64 = AtomicU64::new(0); +static BUILD_NS: AtomicU64 = AtomicU64::new(0); +/// The bytes the built entries hold, and how many of them were built by the +/// time the early leaf's lengths were warm (the ones a base peak can carry). +static BYTES: AtomicU64 = AtomicU64::new(0); +static EARLY_BYTES: AtomicU64 = AtomicU64::new(0); +/// When the early leaf's lengths were warmed, in seconds on the tree's clock +/// (the bits of an `f64`; `u64::MAX` for never). +static EARLY_AT: AtomicU64 = AtomicU64::new(u64::MAX); + +/// `LFM_WARM_DOMAINS` as given: on unless `0` or `off`. On, the median and the +/// large block's recursion were 0.17 s and 0.62 s shorter, with no proof byte +/// moved and ≈ 0.2 GiB more host memory at the base's peak. +fn setting(value: Option<&str>) -> bool { + !matches!(value.map(str::trim), Some("0" | "off")) +} + +/// Whether the warm-up is on, read once. +pub(crate) fn on() -> bool { + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| setting(std::env::var("LFM_WARM_DOMAINS").ok().as_deref())) +} + +/// The shortest table length warmed: the LFM's padded heights start at 4 rows, +/// and a shorter table (a one-row chip) costs nothing to build at its prove. +const MIN_ROWS: usize = 4; + +/// Every table's row count of at least [`MIN_ROWS`], chunks included, each +/// length once, in order. +fn lengths(traces: &LfmTraces) -> Vec { + let mut rows: Vec = [ + &traces.const_, + &traces.balu, + &traces.xalu, + &traces.select, + &traces.bitdec, + &traces.hash, + &traces.keccak, + &traces.lanes, + &traces.hint, + &traces.public, + &traces.range, + &traces.keccak_rc, + &traces.bitwise, + ] + .into_iter() + .chain(&traces.hash_tail) + .chain(&traces.merged) + .chain(&traces.blake3) + .chain(&traces.keccak_rnd) + .map(|t| t.num_rows()) + .filter(|&n| n >= MIN_ROWS) + .collect(); + rows.sort_unstable(); + rows.dedup(); + rows +} + +/// Builds the domain and twiddles of every table length of `traces` under +/// `options` that the cache does not hold yet; returns how many it built. +/// Call it from a plain thread, never a rayon worker. +pub(crate) fn warm(traces: &LfmTraces, options: &ProofOptions) -> usize { + let t = std::time::Instant::now(); + let built: Vec = lengths(traces) + .into_iter() + .filter_map(|n| { + stark::prover::warm_domain_twiddles::( + options.blowup_factor as usize, + options.coset_offset, + n, + ) + }) + .collect(); + BYTES.fetch_add(built.iter().sum::(), Ordering::Relaxed); + let built = built.len(); + BUILT.fetch_add(built as u64, Ordering::Relaxed); + BUILD_NS.fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed); + built +} + +/// The early leaf's lengths are warm, `at` seconds into the tree's clock. +pub(crate) fn note_early(at: f64) { + EARLY_BYTES.store(BYTES.load(Ordering::Relaxed), Ordering::Relaxed); + EARLY_AT.store(at.to_bits(), Ordering::Relaxed); +} + +/// What the warm-up did this process, for a harness's line. +pub fn line() -> String { + if !on() { + return "off (LFM_WARM_DOMAINS=0; the default builds the recursion proofs' domains and twiddles before their card holds)" + .to_string(); + } + let early = EARLY_AT.load(Ordering::Relaxed); + let gib = |b: &AtomicU64| b.load(Ordering::Relaxed) as f64 / (1u64 << 30) as f64; + format!( + "on · {} lengths built in {:.3} s · the early leaf's by {} · {:.3} GiB built, {:.3} GiB by the early leaf's", + BUILT.load(Ordering::Relaxed), + BUILD_NS.load(Ordering::Relaxed) as f64 / 1e9, + if early == u64::MAX { + "never".to_string() + } else { + format!("{:.2}s", f64::from_bits(early)) + }, + gib(&BYTES), + gib(&EARLY_BYTES), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// On by default: unset, empty or `1` warms; `0` or `off` does not. + #[test] + fn the_knob_is_on_unless_zero_or_off() { + for on in [None, Some(""), Some("1"), Some(" 1 ")] { + assert!(setting(on), "{on:?}"); + } + for off in [Some("0"), Some("off"), Some(" 0 ")] { + assert!(!setting(off), "{off:?}"); + } + } + + /// ★ A proof's table lengths are built once and found after: the second + /// warm builds nothing (the proof's own `multi_prove` lookups then hit, + /// which the stark crate's test shows value for value). + #[test] + fn a_proofs_lengths_are_built_once_then_found() { + use crate::tables::types::FE; + let program = super::super::programs::trivial_program(); + // A blowup and coset offset no other test proves under: the + // process-wide cache holds none of these keys yet. + let mut opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(8) + .expect("blowup 8 is valid"); + opts.coset_offset = 11; + let arenas = vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ]; + let hasher = super::super::registry::build_artifacts(&program, &opts).hasher; + let filled = + super::super::proof::lfm_execute_and_fill(&program, &arenas, hasher).expect("filled"); + let warmed: Vec = lengths(filled.traces()) + .into_iter() + .filter(|l| l.is_power_of_two()) + .collect(); + let n = warmed.len(); + assert!(n > 0, "a proof has tables"); + // Each entry holds n·(2·blowup + 3) field elements (the LDE coset and + // trace roots, the two-half sets, the coset weights), 8 bytes each. + let want: u64 = warmed.iter().map(|&l| (l * (2 * 8 + 3) * 8) as u64).sum(); + let before = BYTES.load(Ordering::Relaxed); + assert_eq!(warm(filled.traces(), &opts), n, "every length built once"); + assert_eq!( + BYTES.load(Ordering::Relaxed) - before, + want, + "their bytes counted" + ); + assert_eq!(warm(filled.traces(), &opts), 0, "then found"); + assert_eq!( + BYTES.load(Ordering::Relaxed) - before, + want, + "found ones count nothing" + ); + } +} diff --git a/prover/src/lfm/edsl.rs b/prover/src/lfm/edsl.rs new file mode 100644 index 000000000..73c2ea9ad --- /dev/null +++ b/prover/src/lfm/edsl.rs @@ -0,0 +1,1451 @@ +//! eDSL libraries: transcript, Merkle and FRI expressed as ordinary Rust +//! that *emits instructions*. Host-side `for` loops unroll — nothing +//! loop-shaped reaches the machine; shapes (path depths, query counts, +//! domain parameters) are compile-time constants of the emitted program. +//! +//! The Fiat–Shamir transcript here is the machine side of the protocol loop and +//! is mirrored bit-exactly by `fixture::HostSponge`. Since option B1 it is a +//! **compress chain**, specified in +//! `thoughts/shared/lfm-real-hash/transcript-spec/`; see [`SpongeVar`]. + +use crate::tables::types::FE; + +use super::builder::{Bit, Cell, DigestVal, Ext, Felt, LfmBuilder}; + +/// The advance marker `"SQZ0"`, read as one little-endian `u32`. +/// +/// Lane 0 of the constant cell a squeeze advances with. It distinguishes an +/// advance operand from an absorbed digest as **defence in depth only** — the +/// load-bearing absorb/squeeze separation is that the operation sequence is a +/// compile-time constant of the program (see [`SpongeVar`]). +pub const SQUEEZE_MARK: u32 = u32::from_le_bytes(*b"SQZ0"); + +/// The Fiat–Shamir transcript over `LFM_HASH`: a **compress chain**, state = 1 +/// cell. +/// +/// ```text +/// absorb(c) state ← T(state, c) 1 step +/// absorb2(c0, c1) state ← T(T(state, c0), c1) 2 steps +/// squeeze() out = state ; state ← T(state, SQ(i)) 1 step +/// ``` +/// +/// where `T` is one `LFM_HASH` two-to-one step in the TRANSCRIPT domain +/// ([`LfmBuilder::transcript_step`]) and `SQ(i) = [SQUEEZE_MARK, i, 0, 0]`. +/// Squeeze outputs BEFORE advancing. +/// +/// # Why a chain and not a sponge +/// +/// This replaced an overwrite-rate duplex over a 3-cell permutation (option B1, +/// ratified 2026-08-11). A chain over a collision-resistant compression is the +/// textbook Fiat–Shamir transcript and needs no assumption beyond the one the +/// hash already carries — no T-sponge theorem, no capacity argument. It needs no +/// *permutation* either, which is what lets the machine's real hash have a +/// compress socket and no permute socket at all. Being public-coin is what makes +/// this legitimate: every absorbed value is a public commitment and every +/// squeezed value a public challenge, so there is no secret for a capacity to +/// protect. +/// +/// The state is one cell = 128 bits, so ~**64-bit collision resistance** by the +/// birthday bound. That is `HASH_DIGEST_FELTS = 4` speaking, not this +/// construction: the digest already had that bound. +/// +/// # Why the squeeze counter, and why it is free +/// +/// The eDSL fully unrolls, so `i` is a compile-time constant and `SQ(i)` is a +/// program constant pinned by `program_id` — a constant cell was going to be +/// emitted either way, and this one carries a counter. What it buys: without it +/// a run of consecutive squeezes iterates ONE fixed public non-injective map, +/// whose functional graph an adversary can precompute — the structure the +/// FSE-2014 T-sponge attacks on GLUON-64 exploit. With it every step is a +/// different map and no single functional graph exists. +/// +/// ⚠ **Squeeze runs still lose entropy, and the bound scales with the query +/// count.** A run of `k` consecutive squeezes shrinks the reachable state by +/// `−log₂ α_k` bits, `α_k ~ 2/k`: 1.7 bits at `k = 4`, 7 at `k = 256`, 15 at +/// `k = 2^16`. The counter does not change those numbers (composing distinct +/// random maps obeys the same recursion) — it removes the attack structure. The +/// FRI query loop squeezes once per query with no absorb between, so **its run +/// length IS the query count**. A program whose runs exceed `k = 2^16` must +/// revisit the analysis in the transcript spec §4.2; below that the 64-bit +/// collision bound above dominates and this changes nothing. +#[derive(Clone, Copy, Debug)] +pub struct SpongeVar { + state: Cell, + /// The next squeeze's index — host-side bookkeeping, so it appears in the + /// program only as the constant it selects. + squeeze_index: u32, +} + +impl SpongeVar { + pub fn new(b: &mut LfmBuilder) -> Self { + SpongeVar { + state: b.felt_const(FE::zero()).as_cell(), + squeeze_index: 0, + } + } + + /// Absorb one cell: one transcript step against the current state. + pub fn absorb(&mut self, b: &mut LfmBuilder, c: Cell) { + self.state = b + .transcript_step(self.state.as_digest(), c.as_digest()) + .as_cell(); + } + + /// Absorb a cell of four arbitrary FIELD ELEMENTS. + /// + /// Data enters the transcript the same way it enters a Merkle tree: through + /// the LEAF encoding. The cell is hashed to a digest in the `"LFML"` domain + /// and that digest is absorbed, so the chain binds the data up to the leaf + /// hash's collision resistance. + /// + /// ⚠ **Use this for DATA and [`SpongeVar::absorb`] for DIGESTS.** Absorbing + /// raw field elements would hand the socket lanes that are not `u32`, which + /// under the machine's real hash is not a preference but an unprovable row + /// (obligation O1). A transcript that absorbs commitments needs `absorb`; one + /// that absorbs polynomial coefficients, evaluations or any other field data + /// needs this. + pub fn absorb_felts(&mut self, b: &mut LfmBuilder, c: Cell) { + let acc = leaf_chain_start(b); + let d = b.leaf(acc, c); + self.absorb(b, d.as_cell()); + } + + /// Absorb two cells, in order. Two steps, not one: the chain takes one + /// operand per step, and the ORDER is what the transcript binds. + pub fn absorb2(&mut self, b: &mut LfmBuilder, c0: Cell, c1: Cell) { + self.absorb(b, c0); + self.absorb(b, c1); + } + + /// The chain's state cell as it stands, WITHOUT advancing — what + /// `AlgebraicTranscript::state` serialises, and what grinding seeds from. + /// + /// Observation only: nothing here absorbs or squeezes, so a later step sees + /// the state this returned. That is the same contract + /// `DefaultTranscript::state` has on the byte side, where production + /// finalizes a CLONE of the hasher rather than the hasher. + pub fn state(&self) -> Cell { + self.state + } + + /// Squeeze one cell: the current state, then advance past it with `SQ(i)`. + /// + /// Output-then-advance rather than advance-then-output, so no squeezed + /// value is ever the state a later step absorbs into. + pub fn squeeze_cell(&mut self, b: &mut LfmBuilder) -> Cell { + let out = self.state; + // `SQ(i)`, interned like every other program constant — one `LFM_CONST` + // row per distinct squeeze index, and nothing else. + let sq = b.digest_const([ + FE::from(u64::from(SQUEEZE_MARK)), + FE::from(u64::from(self.squeeze_index)), + FE::zero(), + FE::zero(), + ]); + self.state = b.transcript_step(self.state.as_digest(), sq).as_cell(); + self.squeeze_index += 1; + out + } + + /// Squeeze an ext challenge: lanes 0–2 of a squeezed cell. + pub fn squeeze_ext(&mut self, b: &mut LfmBuilder) -> Ext { + let c = self.squeeze_cell(b); + let [l0, l1, l2, _] = b.unpack(c); + b.pack_ext(l0, l1, l2) + } + + /// Squeeze `nbits` index bits: the canonical bit decomposition of lane 0 + /// of a squeezed cell (masking to a power-of-two bound, so no rejection + /// loop — the convention the RV64 verifier's query sampling already uses). + pub fn squeeze_bits(&mut self, b: &mut LfmBuilder, nbits: usize) -> Vec { + let c = self.squeeze_cell(b); + let [l0, _, _, _] = b.unpack(c); + b.bit_dec(l0, nbits) + } +} + +/// Arena words one commitment DIGEST occupies under this builder's +/// configuration — two on a byte hash, ONE on an algebraic one. +/// +/// ★ The single definition of that stride. Six sites used to spell it as the +/// literal `2`, three of them carrying a comment saying "two arena words per +/// sibling IS the digest's width — when the algebraic path lands this stride +/// follows the width rather than the literal". This is that. +pub fn digest_words(b: &LfmBuilder) -> u32 { + if b.wrap_hash().one_cell_digest() { + 1 + } else { + 2 + } +} + +/// Read one commitment digest out of an arena at `base`, consuming +/// [`digest_words`] words. +pub fn hint_digest(b: &mut LfmBuilder, arena: super::instr::ArenaId, base: u32) -> WrapDigest { + let cells: Vec = (0..digest_words(b)) + .map(|i| b.hint_word(arena, base + i)) + .collect(); + WrapDigest::from_cells(&cells) +} + +/// Where a leaf chain starts: the zero cell. +/// +/// ⚠ **This is a chain START, not a shape HEADER.** COMMIT.md §1.3 opens the +/// chain at `[LEAF_MARK, num_cols, kind, rows_per_leaf]` so that the leaf's +/// width and element kind are bound *inside* the hash — the whole point of that +/// construction. Nothing here has a width to bind: these leaves are fixed-shape +/// by the program that builds them, exactly as they were before the chain +/// existed. The commitment layer that hashes arbitrary-width openings supplies +/// the header instead of this, and it must, or its leaves bind no shape. +/// +/// Interned like every other program constant, so a program's whole leaf traffic +/// costs one `LFM_CONST` row for this. +pub fn leaf_chain_start(b: &mut LfmBuilder) -> DigestVal { + b.digest_const([FE::zero(); 4]) +} + +/// The Merkle LEAF digest of a pair of data cells — eight field elements. +/// +/// **Two compressions, and the shape is the point:** the cells are absorbed in +/// order into one `"LFML"` chain, four felts per hash, each step chaining the +/// last. So a leaf's *data* never enters a compress as a digest, and a parent +/// never enters as data — which is what makes an internal node un-replayable as +/// a leaf regardless of the tree's depth (obligation O5, discharged by the tag +/// rather than by fixed depth). +/// +/// It cost THREE while the accumulator was not in the message: each cell hashed +/// to its own leaf digest and an `"LFMC"` parent folded the two. Absorbing and +/// chaining in the same compression is what took leaf absorption from 2 felts +/// per hash to 4 (COMMIT.md §1.4.1), and leaf absorption is ~70% of a recursion +/// tower node's bill. The chain binds the cells' ORDER for free, where the fold +/// bound it through the parent's operand order. +/// +/// Before either, this was `compress(cell0, cell1)` — one compression that +/// treated arbitrary field elements as if they were `u32` digest lanes. Under a +/// hash whose lanes must BE `u32` that is not merely undesirable, it is +/// unprovable, which is why FRI data could not be hashed at all before the leaf +/// mode existed. +pub fn leaf_hash_pair(b: &mut LfmBuilder, c0: Cell, c1: Cell) -> DigestVal { + let acc = leaf_chain_start(b); + let d0 = b.leaf(acc, c0); + b.leaf(d0, c1) +} + +/// Walk one Merkle authentication path. `bits` are the leaf-index bits +/// low-to-high (level 0 first): bit = 0 ⇒ the current node is the LEFT +/// child. Sibling digests come as (arena-hinted) cells; every hinted value +/// ends up inside a `compress`, which is what authenticates it. +pub fn merkle_walk( + b: &mut LfmBuilder, + leaf: DigestVal, + bits: &[Bit], + siblings: &[Cell], +) -> DigestVal { + assert_eq!(bits.len(), siblings.len(), "one sibling per level"); + let mut current = leaf; + for (bit, sibling) in bits.iter().zip(siblings) { + let (left, right) = b.select(*bit, current.as_cell(), *sibling); + current = b.compress(left.as_digest(), right.as_digest()); + } + current +} + +// ===================== production keccak Merkle ===================== + +/// A 32-byte digest as it lives in the machine: two words of four `u32` halves +/// each, half `h` carrying digest bytes `4h..4h+4`. +/// +/// ★ **One type for both production hashes, and that is load-bearing rather +/// than a coincidence.** `keccak256`'s digest is the state's first 32 bytes and +/// `Blake3Chain`'s is `out[0..8]` little-endian, so under both hashes digest +/// byte `j` is byte `j % 4` of half `j / 4`. Every arena stride, every +/// `siblings: Vec<...>` width and every proof shape is therefore identical +/// across the two, which is what lets the constructions below be parameterized +/// on the hash rather than duplicated per hash. The `LFM_HASH` socket would +/// NOT have this property — its digest is 128 bits, one cell. +/// The largest number of cells any wrap digest occupies — two, for the byte +/// hashes' 32 bytes. +pub const MAX_DIGEST_CELLS: usize = 2; + +/// A commitment digest as it lives in the machine, carrying its own WIDTH. +/// +/// ★ **The width is data, not a constant, and that is the whole point.** A byte +/// hash's digest is 32 bytes — two words of four `u32` halves. An **algebraic** +/// hash's digest is four field elements: **ONE cell**. Both flow through the +/// same walks, comparisons and openings, so the shape travels with the value +/// rather than being written into every call site. +/// +/// Before this, the type was `[Cell; 2]` and five root comparisons plus two +/// Merkle walks each spelled the count out as `d[0]`, `d[1]`. Those are the +/// sites a one-cell digest would have silently half-worked at. +/// +/// Derefs to `[Cell]`, so `d[0]`, `d.len()` and `d.iter()` keep working and the +/// forty-odd sites that only carry a digest around are untouched. +#[derive(Debug, Clone, Copy)] +pub struct WrapDigest { + cells: [Cell; MAX_DIGEST_CELLS], + len: u8, +} + +impl WrapDigest { + /// A two-cell digest — the byte hashes' 32 bytes. + pub fn from_pair(a: Cell, b: Cell) -> Self { + WrapDigest { + cells: [a, b], + len: 2, + } + } + + /// A ONE-cell digest — an algebraic hash's four felts. + /// + /// The unused slot repeats the cell rather than holding a sentinel: nothing + /// reads past `len`, and a repeated handle cannot be mistaken for a real + /// second word the way a zero cell could. + pub fn from_cell(c: Cell) -> Self { + WrapDigest { + cells: [c, c], + len: 1, + } + } + + /// A digest of the same width as `cells`, from a slice. + /// + /// Used by the Merkle walks, which rebuild a digest cell by cell after + /// selecting each against its sibling on the level's bit. + pub fn from_cells(cells: &[Cell]) -> Self { + assert!( + (1..=MAX_DIGEST_CELLS).contains(&cells.len()), + "a digest is one or two cells, got {}", + cells.len() + ); + let mut out = [cells[0]; MAX_DIGEST_CELLS]; + out[..cells.len()].copy_from_slice(cells); + WrapDigest { + cells: out, + len: cells.len() as u8, + } + } + + /// The cells this digest actually occupies. + pub fn cells(&self) -> &[Cell] { + &self.cells[..self.len as usize] + } +} + +impl core::ops::Deref for WrapDigest { + type Target = [Cell]; + fn deref(&self) -> &[Cell] { + self.cells() + } +} + +/// A 32-byte keccak digest. See [`WrapDigest`]. +pub type KeccakDigest = WrapDigest; + +/// A 32-byte BLAKE3 digest. See [`WrapDigest`]. +pub type Blake3Digest = WrapDigest; + +/// Halves in a 32-byte digest. +pub const DIGEST_HALVES: usize = 8; + +/// The eight halves of a BYTE digest, ready to be streamed into another hash. +/// +/// ⛔ **NOT hash-agnostic, whatever an earlier version of this comment said.** +/// It indexes `d[1]`, and an algebraic `WrapDigest` is ONE cell whose second +/// slot REPEATS the first ([`WrapDigest::from_cell`]) — so on that arm this +/// would return `lo ‖ lo`: eight plausible halves, four of them a duplicate, +/// with nothing to notice. The assert below is what makes that loud instead. +/// +/// ✓ VERIFIED unreachable on the algebraic arm today: the only caller is +/// `parent_stream`, inside [`WrapHash::hash_pair`]'s `Some(byte_hash)` arm. The +/// assert is for the NEXT caller, since the stale claim is exactly what would +/// invite one. The felt view of an algebraic digest is +/// `epoch::RootCells::byte_halves`, which renders each felt big-endian rather +/// than pretending the cells are `u32` lanes. +pub fn digest_halves(b: &mut LfmBuilder, d: WrapDigest) -> [Felt; DIGEST_HALVES] { + assert_eq!( + d.cells().len(), + DIGEST_HALVES / 4, + "digest_halves is the BYTE digest's view; an algebraic digest's felts \ + are not `u32` halves and go through RootCells::byte_halves" + ); + let lo = b.unpack(d[0]); + let hi = b.unpack(d[1]); + core::array::from_fn(|h| if h < 4 { lo[h] } else { hi[h - 4] }) +} + +/// The Merkle LEAF hash of a row pair, in the production commitment layout. +/// +/// `values` is `evaluations ‖ evaluations_sym` — the two bit-reversed rows the +/// leaf covers, each written column by column. Every element is a base field +/// element rendered as its canonical `u64` in BIG-endian bytes +/// (`FieldElement::stream_bytes`), so each costs one +/// [`super::transcript_replay::felt_be_halves`]: one `LFM_BITDEC` row and 64 +/// `LFM_BALU` rows. The hash itself is `keccak256` over `8 · values.len()` +/// bytes. +/// +/// ## The byteswapping is NOT what this costs — measured, against expectation +/// +/// A `c`-column table gives `2c` elements, so `2c` decompositions and `128c` +/// ALU rows against only `⌈(16c + 1) / 136⌉` permutations. On row counts the +/// byteswapping looks overwhelming, which is what the R1f handoff predicted. +/// That reading is wrong: rows of different chips are not comparable units. An +/// `LFM_BALU` row carries 4 non-preprocessed columns, while one permutation +/// expands into 24 `KECCAK_RND` rounds of 1480 columns — so in main-trace cells +/// a permutation costs 113 byteswaps, and the hash term dominates at every +/// table width. `machine_tests::keccak_merkle_opening_cost` measures it and +/// asserts the inequality holds. +/// +/// The swap is real work regardless, and it is not avoidable by pre-swapping in +/// the arena: the same opened values are consumed as FIELD ELEMENTS by the FRI +/// algebra and as BYTES by this hash, so something has to connect the two +/// representations, and only the machine can do it in a way the proof binds. +pub fn keccak_leaf_hash(b: &mut LfmBuilder, values: &[Felt]) -> KeccakDigest { + WrapHash::Keccak.leaf_hash(b, values) +} + +/// The byte stream a Merkle LEAF hashes, and the shape it covers. +/// +/// Split out of [`WrapHash::leaf_hash`] so the group's boundary is explicit +/// data rather than a length the hash call happens to receive: the batched +/// (mixed-height, per-matrix) leaf the MMCS lane needs adds a header to +/// exactly this, and can do so without re-cutting the emitter. +/// +/// Every element is a base field element rendered as its canonical `u64` in +/// BIG-endian bytes (`FieldElement::stream_bytes`), so each +/// costs one [`super::transcript_replay::felt_be_halves`]: one `LFM_BITDEC` row +/// and 64 `LFM_BALU` rows. The byte content is the hash's input and is +/// identical under both wrap hashes — only the framing above it differs. +pub fn leaf_stream(b: &mut LfmBuilder, values: &[Felt]) -> (Vec, usize) { + use super::keccak_host::BYTES_PER_HALF; + use super::transcript_replay::felt_be_halves; + + assert!(!values.is_empty(), "a leaf covers at least one column"); + let mut stream = Vec::with_capacity(2 * values.len()); + for v in values { + stream.extend(felt_be_halves(b, *v)); + } + let len_bytes = BYTES_PER_HALF * stream.len(); + (stream, len_bytes) +} + +/// Walk one Merkle authentication path under the PRODUCTION hash. +/// +/// This is the keccak counterpart of [`merkle_walk`], and the two are not +/// interchangeable: `merkle_walk` compresses with `LFM_HASH`/`TestPermutation`, +/// the deliberately non-cryptographic Milestone-C placeholder, so it can only +/// ever authenticate the Milestone-C fixture tree. Production trees are keccak +/// throughout, and this is the walk that authenticates them. +/// +/// `bits` are the leaf index low-to-high, level 0 first; `bit = 0` means the +/// current node is the LEFT child, matching `verify_merkle_path_from_leaf_hash` +/// (`index % 2 == 0 ⇒ hash(current, sibling)`). +/// +/// ## The parent step +/// +/// `hash_new_parent(l, r) = keccak(l ‖ r)` — 64 bytes, no domain separation and +/// no ordering flag, so the ordering is carried entirely by the index bit. 64 +/// bytes sits inside one 136-byte rate block, so a level is exactly ONE +/// permutation. Per level the machine pays two `Select`s (a digest is two words +/// and both must swap on the same bit), four `Unpack`s and that permutation. +pub fn keccak_merkle_walk( + b: &mut LfmBuilder, + leaf: KeccakDigest, + bits: &[Bit], + siblings: &[KeccakDigest], +) -> KeccakDigest { + WrapHash::Keccak.merkle_walk(b, leaf, bits, siblings) +} + +/// The production Merkle PARENT hash: `keccak(left ‖ right)`. +/// +/// `hash_new_parent` streams the two 32-byte nodes into one digest with no +/// domain separation and no ordering flag, so 64 bytes sit inside a single +/// 136-byte rate block and a parent is exactly ONE permutation. +/// +/// This is the step [`keccak_merkle_walk`] performs once per level after its +/// `Select`, and the step a whole-tree build performs once per internal node +/// with no `Select` at all — a tree's child ORDER is known when the program is +/// emitted, so there is no bit to swap on. Keeping the two callers on one +/// primitive is what makes "the walk and the build hash the same way" a +/// property of the code rather than of a comment. +pub fn keccak_hash_pair( + b: &mut LfmBuilder, + left: KeccakDigest, + right: KeccakDigest, +) -> KeccakDigest { + WrapHash::Keccak.hash_pair(b, left, right) +} + +/// The 16 halves a BYTE-hash Merkle PARENT hashes: `left ‖ right`, 64 bytes. +/// +/// ⛔ **Byte arms only, and this doc used to say "hash-agnostic".** It is +/// reached solely from [`WrapHash::hash_pair`]'s `Some(byte_hash)` arm, and it +/// goes through [`digest_halves`], which is the byte digest's view — an +/// algebraic digest has no `u32` halves to stream. The algebraic parent is +/// `algebraic_hash_pair`, a compress over cells, and it builds no stream at all. +/// +/// Across the TWO byte hashes it genuinely is uniform: `hash_new_parent` +/// streams the two 32-byte nodes with no domain separation and no ordering +/// flag under either. What differs is what 64 bytes COSTS — one keccak +/// permutation (inside the 136-byte rate) and one BLAKE3 compression (exactly +/// one 64-byte block). Both are one invocation, which is why the parent layer +/// is 1:1 across that switch and the whole win there is per-compression cost. +/// ⚖ One invocation is also what an algebraic parent costs (8 felts fills the +/// rate exactly), so the LEVEL count is uniform across all three — it is only +/// the byte STREAM that is not. +fn parent_stream(b: &mut LfmBuilder, left: WrapDigest, right: WrapDigest) -> Vec { + let left_halves = digest_halves(b, left); + let right_halves = digest_halves(b, right); + let mut stream = Vec::with_capacity(2 * DIGEST_HALVES); + stream.extend(left_halves); + stream.extend(right_halves); + stream +} + +/// Build a whole Merkle TREE bottom-up and return its root. +/// +/// The counterpart of [`keccak_merkle_walk`]: the walk authenticates ONE leaf +/// against a root it is given, this CONSTRUCTS the root from every leaf. A +/// derivation needs the second — there is no root to authenticate against, +/// producing it is the point. +/// +/// Cost is `leaves − 1` permutations on top of the leaves' own, so a tree over +/// `L` leaves is `2L − 1` permutations in total. +/// +/// ## Power-of-two leaves +/// +/// `MerkleTree::build_from_hashed_leaves` runs `complete_until_power_of_two` +/// first, which pads by REPEATING the last leaf. This asserts a power of two +/// instead of emitting that padding: leaf counts here are shape (an LDE row +/// count over `ROWS_PER_LEAF`), so a non-power-of-two is a caller bug rather +/// than a case to handle, and emitting duplicate-leaf padding no production +/// commitment can reach would be dead program text. +pub fn keccak_merkle_tree_root(b: &mut LfmBuilder, leaves: &[KeccakDigest]) -> KeccakDigest { + WrapHash::Keccak.merkle_tree_root(b, leaves) +} + +// ===================== the wrap's commitment hash ===================== + +/// Which byte hash the wrap's commitment layer runs. +/// +/// ★ **A parameter, not a switch.** The whole construction layer below — +/// leaves, parents, path walks, whole-tree builds — is written ONCE and +/// dispatches only at the two places a hash actually appears: hashing a byte +/// stream, and hashing a 64-byte parent. That is possible because +/// [`WrapDigest`] is the same shape under both, so no call site's types move. +/// +/// Two consequences worth stating because they are the reason for the shape: +/// +/// - A batched (mixed-height, per-matrix) leaf adds ONE construction, not one +/// per hash — which is what the MMCS lane's emitter step needs. +/// - Switching the wrap's hash is a value flowing through the emitters, so an +/// honest-path control (keccak still proves and verifies through the same +/// code) is a real control rather than a different code path. +/// +/// [`Keccak`](WrapHash::Keccak) is the **unset** value, not the production one +/// — see [`WrapHash::production`] and the header of `programs.rs`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WrapHash { + #[default] + Keccak, + Blake3, + /// ★ **One variant for the whole ALGEBRAIC family**, and that is the point. + /// + /// RPO, RPX and Poseidon share this arm because they share everything the + /// EMITTER cares about: state 12, rate 8, capacity 4, and a **one-cell + /// four-felt digest** against the byte hashes' two-cell 32-byte one. Which + /// permutation the emitted socket rows actually prove is `HasherKind`, + /// chosen when the AIR set is built — an orthogonal axis. So a fourth + /// algebraic candidate needs no emitter work at all. + /// + /// ⚠ The byte-stream constructions (`hash_bytes`, `hash_bytes_with_rev`) + /// have **no meaning here**: what they hash IS felts, serialised only + /// because the incumbent hashes are byte-oriented. On this arm the + /// serialisation is deleted at the call site rather than reimplemented. + Algebraic, + /// ZisK's Poseidon1 at width 16 (`super::p1w16_emit`): a one-cell digest + /// like [`WrapHash::Algebraic`]'s, but its own leaf (rate 12), 4-ary trees, + /// sponge transcript and width-8 grinding. The configuration of a level-0 + /// program that verifies a Poseidon1 base proof; its hash rows are + /// `Instr::Hash16`. + Poseidon1, +} + +impl WrapHash { + /// Whether a digest is ONE cell of four field elements (every algebraic + /// configuration) rather than a byte digest's two words. + pub const fn one_cell_digest(self) -> bool { + matches!(self, WrapHash::Algebraic | WrapHash::Poseidon1) + } + + /// Children per Merkle node. + pub const fn arity(self) -> usize { + match self { + WrapHash::Poseidon1 => 4, + _ => 2, + } + } + + /// Path hints one opening of a tree with `nbits` index bits walks: one per + /// binary level, or [`super::p1w16_emit::path_hints`] at arity 4. + pub const fn path_hints(self, nbits: usize) -> usize { + match self { + WrapHash::Poseidon1 => super::p1w16_emit::path_hints(nbits), + _ => nbits, + } + } + + /// ★ The wrap hash that matches the host's commitment configuration. + /// + /// **Any wrap program that AUTHENTICATES a host proof must hash the way the + /// host committed.** Its Merkle walks re-derive roots the host built and its + /// leaf hashes reproduce leaves the host hashed; under the wrong hash none + /// of that reconstructs, and the leg is verifying nothing. + /// + /// ⚠ **It does not fail as a wrong digest.** The walk produces a root + /// matching nothing, the leg then inverts a difference that should have been + /// non-zero, and the executor reports `DivByZero` at some address. That is + /// the diagnostic signature of a leg left on the default — and it is why + /// this is a named function rather than a constant each site spells out, + /// because the failure names neither the hash nor the site. + /// + /// The match is exhaustive on purpose: a third commitment hash cannot be + /// added without deciding what the in-machine verifier does about it, the + /// same tripwire `commitment_hash_tag` carries for program identity. + /// + /// **Not for programs that are ABOUT a hash.** The R1b/R1c/R1d keccak + /// instruments and `program_id_program_source` name their hash directly and + /// must keep doing so. + pub const fn production() -> Self { + match crate::hash_pin::BLOCK_COMMITMENT_HASH { + stark::config::CommitmentHash::Keccak256 => WrapHash::Keccak, + stark::config::CommitmentHash::Blake3 => WrapHash::Blake3, + // ★ The permutation is `HasherKind`, an orthogonal axis chosen + // when the AIR set is built. + stark::config::CommitmentHash::Rpx256 => WrapHash::Algebraic, + // The legacy pin is never Poseidon1: the block's programs, which + // verify Poseidon1 proofs, take the block family's wrap hash + // (`hash_pin::ProofFamily::WRAP`). + stark::config::CommitmentHash::Poseidon1 => WrapHash::Algebraic, + } + } + + /// This configuration's BYTE-stream hash, when it has one. + /// + /// ★ `None` on the algebraic arm is a FACT about that arm, not an error to + /// report. A byte-stream hash returns a 32-byte digest — two cells — and an + /// algebraic digest is four felts, one cell; there is no value of the return + /// type that would be correct. The "rev" half is worse still: a BYTE + /// reversal is not an operation on a field element at all. + /// + /// ⚠ **This replaces an emit-time panic, and the replacement is the point.** + /// While the byte constructions lived on [`WrapHash`] itself they needed an + /// algebraic arm, and that arm could only panic. The panic was recorded as an + /// INTERIM whose exit was "the algebraic path becomes UNABLE to reach a + /// byte-digest API, and then the panic is DELETED rather than kept as a + /// belt" — a retained defensive panic behind a type-level impossibility being + /// exactly the dead production panic this repo's no-production-panic policy + /// exists to prevent. Moving the constructions to [`ByteWrapHash`] is that + /// exit: an algebraic configuration cannot name them, so there is nothing + /// left to defend against. + /// + /// Every caller matches on this, and the `None` arm IS that caller's + /// algebraic implementation — so the exhaustiveness that used to be an + /// assertion is now the control flow. + pub const fn byte_hash(self) -> Option { + match self { + WrapHash::Keccak => Some(ByteWrapHash::Keccak), + WrapHash::Blake3 => Some(ByteWrapHash::Blake3), + WrapHash::Algebraic | WrapHash::Poseidon1 => None, + } + } + + /// The Merkle LEAF hash of a row pair, in the production commitment layout. + /// + /// See [`leaf_stream`] for what the bytes are and what they cost; this adds + /// only the hash. The rate penalty the campaign prices lands here and + /// nowhere else: a leaf absorbs 136 bytes per keccak permutation against 64 + /// per BLAKE3 compression. + pub fn leaf_hash(self, b: &mut LfmBuilder, values: &[Felt]) -> WrapDigest { + if self == WrapHash::Poseidon1 { + return super::p1w16_emit::leaf_hash(b, values); + } + match self.byte_hash() { + None => Self::algebraic_leaf_hash(b, values), + Some(h) => { + let (stream, len_bytes) = leaf_stream(b, values); + h.hash_bytes(b, &stream, len_bytes) + } + } + } + + /// ★ The ALGEBRAIC parent: one `compress` row, one cell out. + /// + /// The socket primitive `b.compress` already exists and is gated; this is + /// the wrap-world name for it. Under RPO the compress domain is zero, so + /// this is literally `Rpo256::merge` and matches + /// `algebraic_commit`'s `parent` on the host. + fn algebraic_hash_pair(b: &mut LfmBuilder, left: WrapDigest, right: WrapDigest) -> WrapDigest { + debug_assert_eq!(left.len(), 1, "an algebraic digest is one cell"); + debug_assert_eq!(right.len(), 1, "an algebraic digest is one cell"); + WrapDigest::from_cell( + b.compress(left[0].as_digest(), right[0].as_digest()) + .as_cell(), + ) + } + + /// ★ The ALGEBRAIC leaf: the rate-8 OVERWRITE DUPLEX. + /// + /// Eight fresh felts per permutation, against the socket leaf chain's four — + /// the convention this lane priced at 25% of the aggregation program. Each + /// block overwrites the two rate cells and carries the capacity cell from + /// the previous permutation, which is exactly `MODE_P`: three cells in, + /// three out, already in the frozen bus contract. + /// + /// ⚠ Every constant comes from the ONE rule + /// (`algebraic_commit::leaf_capacity`), never restated here — the capacity + /// is program data under `MODE_P`, so a second definition of it is a root + /// the host cannot reproduce. This mirrors + /// `algebraic_commit::sponge_leaf` step for step. + fn algebraic_leaf_hash(b: &mut LfmBuilder, values: &[Felt]) -> WrapDigest { + use super::algebraic_commit::leaf_capacity; + + let zero = b.felt_const(FE::zero()); + let zero_cell = b.digest_const([FE::zero(); 4]).as_cell(); + + // Four felts per cell, the tail zero-padded. + let cells: Vec = values + .chunks(4) + .map(|c| { + let lanes: [Felt; 4] = core::array::from_fn(|i| c.get(i).copied().unwrap_or(zero)); + b.pack_word(lanes) + }) + .collect(); + + // The capacity carries the padding flag and the leaf domain. + let mut cap = b.digest_const(leaf_capacity(values.len())).as_cell(); + if cells.is_empty() { + // An empty leaf never permutes — the digest is the initial rate, + // which is zero. `sponge_leaf` returns the same. + return WrapDigest::from_cell(zero_cell); + } + + if b.p1w16_census().is_some() { + // Census geometry: a rate-12 block is three words, one width-16 + // permutation (stand-in row; the capacity chaining is not emitted). + let mut digest = zero_cell; + for block in cells.chunks(3) { + let w = |i: usize| block.get(i).copied().unwrap_or(zero_cell); + digest = b.permute([w(0), w(1), w(2)])[0]; + } + let _ = cap; + return WrapDigest::from_cell(digest); + } + + let mut digest = zero_cell; + for block in cells.chunks(2) { + let rate0 = block[0]; + let rate1 = block.get(1).copied().unwrap_or(zero_cell); + let out = b.permute([rate0, rate1, cap]); + digest = out[0]; + cap = out[2]; + } + WrapDigest::from_cell(digest) + } + + /// The production Merkle PARENT hash: `hash(left ‖ right)`. + /// + /// One invocation under every hash — 64 bytes fits inside keccak's 136-byte + /// rate, IS exactly one BLAKE3 block, and an algebraic parent's two digest + /// cells fill the rate-8 sponge exactly. ⚠ Three arms, not two: the byte + /// ones build a stream through `parent_stream`, the algebraic one + /// compresses cells and builds no stream. This is the step + /// [`WrapHash::merkle_walk`] performs once per level after its `Select`, and + /// the step a whole-tree build performs once per internal node with no + /// `Select` at all: a tree's child ORDER is known when the program is + /// emitted. Keeping the two callers on one primitive is what makes "the walk + /// and the build hash the same way" a property of the code. + pub fn hash_pair(self, b: &mut LfmBuilder, left: WrapDigest, right: WrapDigest) -> WrapDigest { + match self.byte_hash() { + None => Self::algebraic_hash_pair(b, left, right), + Some(h) => { + let stream = parent_stream(b, left, right); + h.hash_bytes(b, &stream, 2 * COMMITMENT_BYTES) + } + } + } + + /// Walk one Merkle authentication path under the production hash. + /// + /// `bits` are the leaf index low-to-high, level 0 first; `bit = 0` means the + /// current node is the LEFT child, matching + /// `verify_merkle_path_from_leaf_hash` (`index % 2 == 0 ⇒ hash(current, + /// sibling)`). Per level: two `Select`s (a digest is two words and both must + /// swap on the same bit), four `Unpack`s and one hash invocation. + /// + /// A loop over the siblings it is given, with no arity or height assumption + /// beyond the one-sibling-per-level assert — deliberately, so a batched path + /// that injects at mixed heights extends this rather than replacing it. + /// + /// Not interchangeable with [`merkle_walk`], which compresses with + /// `LFM_HASH`/`TestPermutation`, the deliberately non-cryptographic + /// Milestone-C placeholder. + pub fn merkle_walk( + self, + b: &mut LfmBuilder, + leaf: WrapDigest, + bits: &[Bit], + siblings: &[WrapDigest], + ) -> WrapDigest { + if self == WrapHash::Poseidon1 { + return super::p1w16_emit::walk4(b, leaf, bits, siblings); + } + assert_eq!(bits.len(), siblings.len(), "one sibling per level"); + if let (Some(arena), None) = (b.p1w16_census(), self.byte_hash()) { + return p1w16_census_walk(b, arena, leaf, bits, siblings); + } + let mut current = leaf; + for (bit, sibling) in bits.iter().zip(siblings) { + // ★ EVERY cell of the digest swaps on the SAME bit — a loop rather + // than two hard-coded halves, so a ONE-cell algebraic digest costs + // ONE select per level where a byte digest costs two. + debug_assert_eq!( + current.len(), + sibling.len(), + "a node and its sibling must be the same width" + ); + let mut left = [current[0]; MAX_DIGEST_CELLS]; + let mut right = [current[0]; MAX_DIGEST_CELLS]; + for k in 0..current.len() { + let (l, r) = b.select(*bit, current[k], sibling[k]); + left[k] = l; + right[k] = r; + } + let n = current.len(); + current = self.hash_pair( + b, + WrapDigest::from_cells(&left[..n]), + WrapDigest::from_cells(&right[..n]), + ); + } + current + } + + /// Build a whole Merkle TREE bottom-up and return its root. + /// + /// The counterpart of [`WrapHash::merkle_walk`]: the walk authenticates ONE + /// leaf against a root it is given, this CONSTRUCTS the root from every + /// leaf. A derivation needs the second — there is no root to authenticate + /// against, producing it is the point. Cost is `leaves − 1` parent hashes. + /// + /// `MerkleTree::build_from_hashed_leaves` runs `complete_until_power_of_two` + /// first, which pads by REPEATING the last leaf. This asserts a power of two + /// instead of emitting that padding: leaf counts here are shape (an LDE row + /// count over `ROWS_PER_LEAF`), so a non-power-of-two is a caller bug, and + /// emitting duplicate-leaf padding no production commitment can reach would + /// be dead program text. + pub fn merkle_tree_root(self, b: &mut LfmBuilder, leaves: &[WrapDigest]) -> WrapDigest { + assert!(!leaves.is_empty(), "a tree has at least one leaf"); + assert!( + leaves.len().is_power_of_two(), + "leaf counts are shape and must be a power of two; production would \ + pad by repeating the last leaf and no caller here needs that" + ); + if self == WrapHash::Poseidon1 { + return super::p1w16_emit::tree_root4(b, leaves); + } + let mut level = leaves.to_vec(); + if b.p1w16_census().is_some() && self.byte_hash().is_none() { + // Census geometry: 4-ary nodes (one stand-in row each); an odd + // height leaves one binary level at the top. + while level.len() > 1 { + level = if level.len().is_multiple_of(4) { + level + .chunks_exact(4) + .map(|q| WrapDigest::from_cell(b.permute([q[0][0], q[1][0], q[2][0]])[0])) + .collect() + } else { + level + .chunks_exact(2) + .map(|pair| self.hash_pair(b, pair[0], pair[1])) + .collect() + }; + } + return level[0]; + } + while level.len() > 1 { + level = level + .chunks_exact(2) + .map(|pair| self.hash_pair(b, pair[0], pair[1])) + .collect(); + } + level[0] + } +} + +/// ⚠ CENSUS ONLY ([`LfmBuilder::with_p1w16_census`]): a 4-ary Merkle walk. +/// +/// Per two index bits `(b0, b1)`: the current node and the level-0 sibling are +/// placed by one `Select` on `b0`; that pair and the two level-1 siblings are +/// placed by two `Select`s on `b1`; one width-16 node hashes the four +/// (stand-in `permute` row over three of the cells). A 4-ary level carries +/// THREE sibling digests where two binary levels carry two: the given +/// `siblings[2k]`, `siblings[2k+1]` and one more hinted word. An odd walk +/// ends with one binary level. +fn p1w16_census_walk( + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + leaf: WrapDigest, + bits: &[Bit], + siblings: &[WrapDigest], +) -> WrapDigest { + let mut current = leaf[0]; + let mut k = 0; + while k + 1 < bits.len() { + let extra = b.hint_word(arena, 0); + let (x0, x1) = b.select(bits[k], current, siblings[k][0]); + let (y0, _y2) = b.select(bits[k + 1], x0, siblings[k + 1][0]); + let (y1, _y3) = b.select(bits[k + 1], x1, extra); + current = b.permute([y0, y1, _y2])[0]; + k += 2; + } + if k < bits.len() { + let (l, r) = b.select(bits[k], current, siblings[k][0]); + current = b.compress(l.as_digest(), r.as_digest()).as_cell(); + } + WrapDigest::from_cell(current) +} + +/// Bytes in a commitment / Merkle node. +pub const COMMITMENT_BYTES: usize = 32; + +/// Assert two words are equal, lane by lane (2 unpacks + 4 lowered asserts). +pub fn assert_word_eq(b: &mut LfmBuilder, x: Cell, y: Cell) { + let yl = b.unpack(y); + assert_word_eq_lanes(b, x, &yl); +} + +/// Assert a word equals four already-unpacked lanes (hoist the reference +/// word's unpack out of a loop — e.g. one root compared per query). +/// ★ Assert a whole DIGEST equals a root's lanes — the digest-shaped form of +/// [`assert_word_eq_lanes`]. +/// +/// Five sites used to spell this as two indexed calls, `root[0]`/`root[1]`, +/// which writes the digest's cell COUNT into every one of them. Looping here +/// concentrates that count in one place, so a digest of a different width +/// (an algebraic root is ONE cell of four felts, not two of 32 bytes) changes +/// this function rather than five call sites. +/// +/// Behaviour is identical for a two-cell digest; this is a refactor, not a +/// change, and the gates that covered those five sites still cover it. +pub fn assert_digest_eq_lanes(b: &mut LfmBuilder, d: WrapDigest, lanes: &[[Felt; 4]]) { + assert_eq!( + d.len(), + lanes.len(), + "a digest and the lanes it is compared against must have the same width" + ); + for (cell, word) in d.iter().zip(lanes.iter()) { + assert_word_eq_lanes(b, *cell, word); + } +} + +pub fn assert_word_eq_lanes(b: &mut LfmBuilder, x: Cell, y_lanes: &[Felt; 4]) { + let xl = b.unpack(x); + for i in 0..4 { + b.assert_eq(xl[i], y_lanes[i]); + } +} + +/// `scale · Π factors[i]^{bits[i]}` — one Select + one Mul per bit. Used to +/// derive domain points (and their inverses) from query-index bits; the +/// factors are program constants, so nothing here touches an arena. +pub fn pow_bits(b: &mut LfmBuilder, bits: &[Bit], factors: &[FE], scale: FE) -> Felt { + assert_eq!(bits.len(), factors.len()); + let mut acc = b.felt_const(scale); + for (bit, factor) in bits.iter().zip(factors) { + let one = b.felt_const(FE::one()); + let f = b.felt_const(*factor); + let (chosen, _) = b.select(*bit, one.as_cell(), f.as_cell()); + acc = b.mul(acc, Felt(chosen.0)); + } + acc +} + +/// [`pow_bits`] at a scale of one with no `Select`: each bit's factor is +/// `1 + b·(f − 1)` (one `MulAdd`, the bit a boolean cell of the index's +/// `BitDec`), multiplied in (one `Mul`; the first factor starts the +/// accumulator). Two base-ALU rows a bit, less one, against one `Select` and +/// one `Mul`: the form for a program whose `Select` table binds and whose base +/// ALU does not (the Poseidon1 leaf's, I-WHIR-P1 §S5.4). The same value: the +/// bit is 0 or 1. +pub fn pow_bits_alu(b: &mut LfmBuilder, bits: &[Bit], factors: &[FE]) -> Felt { + assert_eq!(bits.len(), factors.len()); + let mut acc: Option = None; + for (bit, factor) in bits.iter().zip(factors) { + let one = b.felt_const(FE::one()); + let step = b.felt_const(*factor - FE::one()); + let chosen = b.mul_add(Felt(bit.addr()), step, one); + acc = Some(match acc { + None => chosen, + Some(acc) => b.mul(acc, chosen), + }); + } + acc.unwrap_or_else(|| b.felt_const(FE::one())) +} + +/// `Σ_i coeffs[i]·α^i` over ext, coeffs given low-to-high (base cells are +/// valid ext operands). One `MulAdd` per coefficient — the Horner shape. +pub fn horner_ext(b: &mut LfmBuilder, alpha: Ext, coeffs_low_to_high: &[Ext]) -> Ext { + let mut iter = coeffs_low_to_high.iter().rev(); + let mut acc = *iter.next().expect("at least one coefficient"); + for c in iter { + acc = b.emul_add(acc, alpha, *c); + } + acc +} + +/// One unnormalized FRI fold — our production convention exactly: +/// `(lo + hi) + inv_x·ζ·(lo − hi)` (the missing ½ is absorbed into the +/// terminal polynomial). +pub fn fri_fold(b: &mut LfmBuilder, lo: Ext, hi: Ext, zeta: Ext, inv_x: Felt) -> Ext { + let sum = b.eadd(lo, hi); + let diff = b.esub(lo, hi); + let zd = b.emul(zeta, diff); + let scaled = b.emul_base(zd, inv_x); + b.eadd(sum, scaled) +} + +/// A configuration that hashes a BYTE STREAM — the two incumbents, and only +/// them. +/// +/// ★ It exists so that "this hash has a 32-byte digest" is a TYPE rather than a +/// runtime check. [`WrapHash::byte_hash`] is the only way in, and it cannot +/// produce one from an algebraic configuration — so the byte-stream +/// constructions below are unreachable from the algebraic path by construction, +/// which is what let the emit-time panic that used to guard them be deleted +/// rather than retained. +/// +/// The Merkle constructions (`leaf_hash`, `hash_pair`, `merkle_walk`) stay on +/// [`WrapHash`] and are NOT split, because they are shape-generic: they take and +/// return [`WrapDigest`], which carries its own width. Only the byte-STREAM +/// entries have no algebraic meaning. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ByteWrapHash { + Keccak, + Blake3, +} + +impl ByteWrapHash { + /// The hash of a byte stream supplied as `u32`-half felts. + /// + /// Both hashes take the SAME packing — four bytes per felt, little-endian + /// ([`super::keccak_host::pack_stream`]) — because a BLAKE3 message word is + /// itself a little-endian `u32` of four consecutive message bytes. So a + /// stream built for one is a stream for the other, and only the framing + /// above it changes: 136-byte rate blocks with `pad10*1` against 64-byte + /// blocks with zero padding and an explicit `block_len`. + pub fn hash_bytes(self, b: &mut LfmBuilder, stream: &[Felt], len_bytes: usize) -> WrapDigest { + match self { + ByteWrapHash::Keccak => { + let d = keccak256(b, stream, len_bytes); + WrapDigest::from_pair(d[0], d[1]) + } + ByteWrapHash::Blake3 => { + let d = blake3_256(b, stream, len_bytes); + WrapDigest::from_pair(d[0], d[1]) + } + } + } + + /// [`ByteWrapHash::hash_bytes`] returning BOTH the digest and its byte-REVERSED + /// form — what `DefaultTranscript::sample()` returns and re-absorbs. + /// + /// Free under both hashes, for the same reason: the bus recomposes each + /// `u32` lane from four byte columns, so the reversal is a second `Linear` + /// over the same columns (`layout::keccak::REV_ADDR0`, + /// `layout::blake3::REV_ADDR0`). + pub fn hash_bytes_with_rev( + self, + b: &mut LfmBuilder, + stream: &[Felt], + len_bytes: usize, + ) -> (WrapDigest, [Cell; 2]) { + match self { + ByteWrapHash::Keccak => { + let (d, rev) = keccak256_with_rev(b, stream, len_bytes); + (WrapDigest::from_pair(d[0], d[1]), rev) + } + ByteWrapHash::Blake3 => { + let (d, rev) = blake3_256_with_rev(b, stream, len_bytes); + (WrapDigest::from_pair(d[0], d[1]), rev) + } + } + } +} + +// ============ the configured hash, as free functions ============ +// +// Every construction below reads `b.wrap_hash()`. Call sites take no hash +// parameter, so there is no site to forget at the flip — and the emitters that +// must NOT follow the configuration (`programs::emit_program_id`, the R1c +// instruments) keep naming `keccak256` and friends directly, which makes a grep +// for the pinned hash in `lfm/` return exactly the deliberate exceptions. + +/// [`ByteWrapHash::hash_bytes`] under a hash the CALLER has already established +/// is a byte hash. +/// +/// ⚠ It takes the hash rather than reading `b.wrap_hash()`, and that asymmetry +/// with the other free functions here is deliberate: the rest are total over +/// every configuration, this one is not. A caller reaches it by matching on +/// [`WrapHash::byte_hash`], which is where its algebraic case is handled — so +/// the parameter is the proof that the case was handled. +pub fn wrap_hash_bytes( + b: &mut LfmBuilder, + h: ByteWrapHash, + stream: &[Felt], + len_bytes: usize, +) -> WrapDigest { + h.hash_bytes(b, stream, len_bytes) +} + +/// [`ByteWrapHash::hash_bytes_with_rev`], on the same terms as +/// [`wrap_hash_bytes`]. +pub fn wrap_hash_bytes_with_rev( + b: &mut LfmBuilder, + h: ByteWrapHash, + stream: &[Felt], + len_bytes: usize, +) -> (WrapDigest, [Cell; 2]) { + h.hash_bytes_with_rev(b, stream, len_bytes) +} + +/// [`WrapHash::leaf_hash`] under the builder's configured hash. +pub fn wrap_leaf_hash(b: &mut LfmBuilder, values: &[Felt]) -> WrapDigest { + let h = b.wrap_hash(); + h.leaf_hash(b, values) +} + +/// [`WrapHash::hash_pair`] under the builder's configured hash. +pub fn wrap_hash_pair(b: &mut LfmBuilder, left: WrapDigest, right: WrapDigest) -> WrapDigest { + let h = b.wrap_hash(); + h.hash_pair(b, left, right) +} + +/// [`WrapHash::merkle_walk`] under the builder's configured hash. +pub fn wrap_merkle_walk( + b: &mut LfmBuilder, + leaf: WrapDigest, + bits: &[Bit], + siblings: &[WrapDigest], +) -> WrapDigest { + let h = b.wrap_hash(); + h.merkle_walk(b, leaf, bits, siblings) +} + +/// [`WrapHash::merkle_tree_root`] under the builder's configured hash. +pub fn wrap_merkle_tree_root(b: &mut LfmBuilder, leaves: &[WrapDigest]) -> WrapDigest { + let h = b.wrap_hash(); + h.merkle_tree_root(b, leaves) +} + +// ===================== small arithmetic predicates ===================== + +/// `1` if `x == 0`, else `0`, for an `x` the caller knows is below `2^nbits`. +/// +/// Hint-free by necessity. The textbook `1 − x·x⁻¹` needs an inverse witness, +/// and the only witness channel this machine has is an arena — whose standing +/// rule is that everything hinted is transitively hash-authenticated, which an +/// inverse is not. So the DECOMPOSITION is the witness: `LFM_BITDEC` proves the +/// bits really are `x`'s, and `Π (1 − bᵢ)` is one exactly when every bit is +/// zero. The result is a product of booleans, so it is boolean by construction +/// and legal as a `Select` bit. +/// +/// Cost: one `LFM_BITDEC` row plus `2·nbits − 1` `LFM_BALU` rows. The caller's +/// bound is load-bearing — bits above `nbits` are not decomposed into cells, so +/// a larger `x` would be reported zero on its low bits alone. +pub fn is_zero_bounded(b: &mut LfmBuilder, x: Felt, nbits: usize) -> Bit { + let bits = b.bit_dec(x, nbits); + let one = b.felt_const(FE::one()); + let mut acc = one; + for bit in bits { + let complement = b.sub(one, bit.as_felt()); + acc = b.mul(acc, complement); + } + Bit(acc.0) +} + +/// `1` if `x == 2^nbits − 1`, else `0`, for an `x` below `2^nbits`. +/// +/// The dual of [`is_zero_bounded`] and cheaper by the complements: `Π bᵢ`. +pub fn is_all_ones_bounded(b: &mut LfmBuilder, x: Felt, nbits: usize) -> Bit { + let bits = b.bit_dec(x, nbits); + let one = b.felt_const(FE::one()); + let mut acc = one; + for bit in bits { + acc = b.mul(acc, bit.as_felt()); + } + Bit(acc.0) +} + +// ============================== keccak256 ============================== + +/// `keccak256` over a byte stream supplied as `u32`-half felts (four bytes +/// each, little-endian — see [`super::keccak_host::pack_stream`]). Returns the +/// 32-byte digest as two machine words of halves. +/// +/// Shapes are compile-time, as everywhere in this machine: `len_bytes` fixes +/// the block count and the padding positions, so `pad10*1` is emitted as +/// interned program CONSTANTS rather than computed. A different length is a +/// different program with a different digest — which is the straight-line +/// discipline working as intended, not a limitation to route around. +/// +/// The digest is the state's first 32 bytes = halves 0..7 = words 0 and 1 of +/// the state's word representation, which is exactly `PlatformKeccak256`'s +/// output byte order (byte `j` = byte `j % 4` of half `j / 4`). +pub fn keccak256(b: &mut LfmBuilder, stream: &[Felt], len_bytes: usize) -> [Cell; 2] { + let (state, _) = keccak256_absorb_all(b, stream, len_bytes, false); + [state[0], state[1]] +} + +/// `keccak256`, additionally returning the byte-REVERSED digest — the value the +/// production `DefaultTranscript::sample()` both returns as the challenge and +/// re-absorbs as the next segment's prefix. +/// +/// `sample()` is byte-for-byte identical before and after #841, so this is +/// independent of which transcript revision the caller targets. +pub fn keccak256_rev(b: &mut LfmBuilder, stream: &[Felt], len_bytes: usize) -> [Cell; 2] { + let (_, rev) = keccak256_absorb_all(b, stream, len_bytes, true); + rev.expect("requested") +} + +/// `keccak256` returning BOTH digests — plain and byte-reversed — off the one +/// keccak row that produces them. +/// +/// The transcript replay needs both at once and they are not interchangeable: +/// the reversed digest is what `sample()` returns and re-absorbs, while +/// candidates are read off the PLAIN digest (the reversal and the big-endian +/// candidate read cancel — see [`super::keccak_host::candidate_from_state`]). +pub fn keccak256_with_rev( + b: &mut LfmBuilder, + stream: &[Felt], + len_bytes: usize, +) -> ([Cell; 2], [Cell; 2]) { + let (state, rev) = keccak256_absorb_all(b, stream, len_bytes, true); + ([state[0], state[1]], rev.expect("requested")) +} + +/// `Σ_i 2^i · bits[i]`, bits low-to-high — the value a bit decomposition stands +/// for. Horner from the top: one `MulAdd` per bit after the first. +pub fn bits_to_felt(b: &mut LfmBuilder, bits: &[Bit]) -> Felt { + let two = b.felt_const(FE::from(2u64)); + let mut iter = bits.iter().rev(); + let mut acc = iter.next().expect("at least one bit").as_felt(); + for bit in iter { + acc = b.mul_add(acc, two, bit.as_felt()); + } + acc +} + +// ============================== Blake3Chain ============================== + +/// `Blake3Chain` over a byte stream supplied as `u32`-half felts — the same +/// packing [`keccak256`] takes, because a BLAKE3 message word IS a +/// little-endian `u32` of four consecutive message bytes. +/// +/// Returns the 32-byte digest as two machine words. The digest is the +/// compression's output words 0 and 1 (`out[0..8]` little-endian), so reading it +/// costs nothing — and the CHAINING VALUE of the next block is those same two +/// words, which is why multi-block messages need no repacking between blocks. +/// +/// Shapes are compile-time, as everywhere in this machine: `len_bytes` fixes the +/// block count, the `block_len` of the final block and the whole flag schedule, +/// all of which are emitted as interned program CONSTANTS. The schedule itself +/// is read from [`crypto::hash::blake3::chain`] rather than restated here — see +/// [`blake3_absorb_all`]. +pub fn blake3_256(b: &mut LfmBuilder, stream: &[Felt], len_bytes: usize) -> Blake3Digest { + blake3_absorb_all(b, stream, len_bytes, false).0 +} + +/// [`blake3_256`], returning the byte-REVERSED digest instead — the value the +/// production `DefaultTranscript::sample()` both returns as the challenge and +/// re-absorbs as the next segment's prefix. +pub fn blake3_256_rev(b: &mut LfmBuilder, stream: &[Felt], len_bytes: usize) -> [Cell; 2] { + blake3_absorb_all(b, stream, len_bytes, true) + .1 + .expect("requested") +} + +/// [`blake3_256`] returning BOTH digests — plain and byte-reversed — off the one +/// compression that produces them. +pub fn blake3_256_with_rev( + b: &mut LfmBuilder, + stream: &[Felt], + len_bytes: usize, +) -> (Blake3Digest, [Cell; 2]) { + let (digest, rev) = blake3_absorb_all(b, stream, len_bytes, true); + (digest, rev.expect("requested")) +} + +/// The `Blake3Chain` framing, emitted. +/// +/// Three differences from [`keccak256_absorb_all`], each a place a port goes +/// wrong silently, so each is named: +/// +/// - **No padding constants.** BLAKE3 zero-pads. There is no `pad10*1`, no +/// `pad_half`, and nothing analogous to keccak's `stream[g] + pad` merge — +/// a half past the message is the shared zero cell, full stop. +/// - **`block_len` is data, not shape-implied.** The final block carries the +/// true byte count where keccak encoded the same information positionally in +/// its pad. Compile-time known, so it is an interned constant — but it must be +/// *emitted*. +/// - **Flags are a three-value schedule, not a constant.** First / interior / +/// last, with a single-block message carrying both ends at once +/// (`CHUNK_START | CHUNK_END | ROOT`), which is exactly the parent form. +/// +/// ★ The schedule is [`chain::block_flags`] / [`chain::block_len_of`] / +/// [`chain::num_blocks`] — the host hasher's own, hoisted into `crypto` for +/// this caller. A second statement of "which block carries `CHUNK_START`" is the +/// single most likely way for an in-machine hash to differ from the host's by +/// one compression's flags, and that difference is a valid proof of the wrong +/// digest. +fn blake3_absorb_all( + b: &mut LfmBuilder, + stream: &[Felt], + len_bytes: usize, + want_rev: bool, +) -> (Blake3Digest, Option<[Cell; 2]>) { + use super::blake3::BLAKE3_IV; + use super::blake3::chain::{BLOCK_LEN, block_flags, block_len_of, num_blocks}; + use super::keccak_host::{BYTES_PER_HALF, num_stream_halves}; + + assert_eq!( + stream.len(), + num_stream_halves(len_bytes), + "stream must hold exactly ceil(len_bytes / 4) halves" + ); + + /// `u32` halves in one 64-byte BLAKE3 block: the message words `m[0..16]`. + const HALVES_PER_BLOCK: usize = BLOCK_LEN / BYTES_PER_HALF; // 16 + /// Machine words the message occupies, four `u32` lanes each. + const MESSAGE_WORDS: usize = HALVES_PER_BLOCK / 4; // 4 + + let zero = b.felt_const(FE::zero()); + let blocks = num_blocks(len_bytes); + + // The initial chaining value: `BLAKE3_IV` as two interned constants. `h` is + // `u32` words 0..8, so word 0 is `IV[0..4]` and word 1 is `IV[4..8]`. + let iv_word = |b: &mut LfmBuilder, w: usize| -> Cell { + b.digest_const(core::array::from_fn(|l| { + FE::from(u64::from(BLAKE3_IV[4 * w + l])) + })) + .as_cell() + }; + let mut h: [Cell; 2] = [iv_word(b, 0), iv_word(b, 1)]; + let mut rev: Option<[Cell; 2]> = None; + + for block in 0..blocks { + // Half `h` of this block is half `block * 16 + h` of the message; both a + // block (64 bytes) and a half (4 bytes) divide evenly, so the two + // indexings line up with no straddling. Halves past the message are the + // zero cell — the zero padding, with nothing to splice. + let message: [Cell; MESSAGE_WORDS] = core::array::from_fn(|w| { + let lane = |l: usize| { + let g = block * HALVES_PER_BLOCK + 4 * w + l; + if g < stream.len() { stream[g] } else { zero } + }; + b.pack_word([lane(0), lane(1), lane(2), lane(3)]) + }); + + // `(t_lo, t_hi, block_len, flags)`. `t = 0` at every block: the + // construction is a single chunk that never ends (PA-PLAN §1.7, F1). + let params = b + .digest_const([ + FE::zero(), + FE::zero(), + FE::from(u64::from(block_len_of(block, len_bytes))), + FE::from(u64::from(block_flags(block, blocks))), + ]) + .as_cell(); + + let out = if block + 1 == blocks && want_rev { + let (out, rev_words) = b.blake3_compress_rev(h, message, params); + rev = Some(rev_words); + out + } else { + b.blake3_compress(h, message, params) + }; + // The next block's chaining value is `out[0..8]` — output words 0 and 1 + // — and so is the digest when this was the last block. One and the same, + // which is what the truncation to `[out[0], out[1]]` says. + h = [out[0], out[1]]; + } + + (WrapDigest::from_pair(h[0], h[1]), rev) +} + +fn keccak256_absorb_all( + b: &mut LfmBuilder, + stream: &[Felt], + len_bytes: usize, + want_rev: bool, +) -> ([Cell; 13], Option<[Cell; 2]>) { + use super::keccak_host::{num_blocks, num_stream_halves, pad_half}; + use super::layout::keccak::{BLOCK_HALVES, BLOCK_WORDS, NUM_WORDS}; + + assert_eq!( + stream.len(), + num_stream_halves(len_bytes), + "stream must hold exactly ceil(len_bytes / 4) halves" + ); + + let zero = b.felt_const(FE::zero()); + let mut state: [Cell; NUM_WORDS] = [zero.as_cell(); NUM_WORDS]; + let mut rev: Option<[Cell; 2]> = None; + + for block in 0..num_blocks(len_bytes) { + // Half `h` of this block is half `block * BLOCK_HALVES + h` of the + // padded message; both the rate (136 bytes) and a half (4 bytes) divide + // evenly, so the two indexings line up with no straddling across blocks. + let halves: Vec = (0..BLOCK_HALVES) + .map(|h| { + let g = block * BLOCK_HALVES + h; + let pad = pad_half(len_bytes, g); + match (g < num_stream_halves(len_bytes), pad) { + // Entirely inside the message. + (true, 0) => stream[g], + // Straddles the end: the stream half's high bytes are zero + // by the packing convention, so adding merges the padding in + // without carrying. + (true, p) => { + let c = b.felt_const(FE::from(p)); + b.add(stream[g], c) + } + // Entirely padding (possibly all-zero). + (false, p) => b.felt_const(FE::from(p)), + } + }) + .collect(); + + // 34 halves into 9 words; the last word's top two slots are the unused + // half slots the chip pins to zero. + let block_words: [Cell; BLOCK_WORDS] = core::array::from_fn(|w| { + let lane = |l: usize| { + let h = 4 * w + l; + if h < BLOCK_HALVES { halves[h] } else { zero } + }; + b.pack_word([lane(0), lane(1), lane(2), lane(3)]) + }); + + let last = block + 1 == num_blocks(len_bytes); + if last && want_rev { + let (next, rev_words) = b.keccak_absorb_rev(state, block_words); + state = next; + rev = Some(rev_words); + } else { + state = b.keccak_absorb(state, block_words); + } + } + + (state, rev) +} diff --git a/prover/src/lfm/epoch.rs b/prover/src/lfm/epoch.rs new file mode 100644 index 000000000..87d1f8b12 --- /dev/null +++ b/prover/src/lfm/epoch.rs @@ -0,0 +1,1060 @@ +//! Assembly — the per-table challenge replay that turns the legs into a +//! verifier. +//! +//! Every leg so far took its challenges as ARENA WORDS: `emit_sub_proof` hints +//! `γ` and `ζ`, `declare_fri` hints the folding challenges, and the query index +//! arrives as a hinted felt that the walk decomposes. That is fine for a +//! differential against production, which supplies the true values, and it is +//! fatal in a verifier: a prover who chooses `γ` chooses the DEEP fold, one who +//! chooses `ζ_k` chooses the FRI fold, and one who chooses `ι` chooses which +//! rows are ever opened. This module is where those words stop being data. +//! +//! ## What it replays +//! +//! `verifier.rs`'s per-table body, in production's order and nothing else: +//! +//! - the FORK (`:1263-1266`) — clone the shared post-Phase-A transcript, then a +//! domain separator `idx.to_le_bytes()` when the epoch has more than one +//! table; +//! - the aux root (`:1269-1271`) and the bus contribution `L` (`:1274-1276`); +//! - Round 2 (`:1380-1404`): sample `β`, then absorb the composition root; +//! - Round 3 (`:1411-1434`): sample `z`, then absorb the two pruned OOD blocks +//! COLUMN-major, then the claimed composition parts; +//! - Round 4 (`:1445-1504`): sample `γ`; then per committed FRI layer sample +//! `ζ_k` and absorb root `k`; then `ζ_C` if and only if the codeword folds; +//! then every terminal coefficient; then grinding; then the query indices. +//! +//! The interleaving in Round 4 is the part no leg-side test could catch, and it +//! is load-bearing in both directions: a `ζ_k` sampled after its own layer root +//! is a challenge the prover can answer, and a layer root that is never absorbed +//! leaves the query indices independent of the codeword they index. +//! +//! ## Why the values are returned as CELLS +//! +//! The point of the module is that there is exactly ONE cell per value and both +//! consumers read it. `L` is absorbed here and summed by the LogUp closure; the +//! OOD block cells are absorbed here and folded by the constraint leg and by +//! DEEP; the composition parts likewise; the layer roots are absorbed here and +//! compared against the FRI walk. Nothing is hinted twice, which is the +//! assembly obligation (`others/lfm-assembly-obligations.md`, OPEN 3) stated as +//! a construction rather than as a rule to remember. +//! +//! ## Zero-rejection, one level up +//! +//! `sample_z_ood_with_domain_params` REJECTS a `z` that lands in the trace +//! domain or on the LDE coset and draws again. A straight-line program cannot, +//! so [`emit_z_ood`] draws once and CONSTRAINS both rejection predicates to be +//! false — the same disposition as the sampler's canonicity guard, and the same +//! completeness-only cost (`SOUNDNESS.md` §6.3). + +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +use super::builder::{Bit, Ext, Felt, LfmBuilder}; +use super::fri::FriShape; +use super::layout::keccak::DIGEST_WORDS; +use super::transcript_replay::{ByteString, TranscriptReplay}; + +/// A sub-proof's AIR as the shape derivations read it: the block's VM AIRs and +/// the LFM chips alike. +pub type DynVmAir<'a> = dyn stark::traits::AIR< + Field = crate::tables::types::GoldilocksField, + FieldExtension = GoldilocksExtension, + PublicInputs = (), + > + 'a; + +/// The grinding prefix, `crypto/stark/src/grinding.rs`'s `PREFIX`. +const GRINDING_PREFIX: [u8; 8] = 0x0123_4567_89ab_cded_u64.to_be_bytes(); + +/// A commitment root as the machine holds it: eight `u32` lanes, produced ONCE. +/// +/// Both consumers of a root — the transcript absorb and the Merkle comparison — +/// want a different view of the same 32 bytes, and a root that was hinted twice +/// (or unpacked twice) would let those views drift. So there is one unpack per +/// root and every consumer reads its lanes. +/// +/// The three constructors are the three SOURCES a root can have, which is +/// assembly ledger entry 7's whole content: program text ([`Self::constant`]), +/// an in-machine derivation ([`Self::from_digest`]) or the proof's arena +/// ([`Self::hint`]). Which one a given commitment may use is a property of what +/// the commitment is a function of, not a convenience. +#[derive(Clone)] +pub struct RootCells { + /// The root as the configuration's digest cells — TWO on a byte hash, ONE on + /// an algebraic one. + digest: super::edsl::WrapDigest, + /// Those cells unpacked, ONE ENTRY PER CELL, hoisted so every consumer reads + /// the same lanes. + /// + /// ⚠ A `Vec` rather than `[_; DIGEST_WORDS]`, and that is the whole shape + /// change: the array wrote the byte digest's cell COUNT into the type, so an + /// algebraic root — one cell of four felts, not two of 32 bytes — could not + /// be one of these. `edsl::assert_digest_eq_lanes` already zips a digest + /// against these and asserts the widths match, so it works at either width + /// unchanged once this stops being fixed at two. + /// + /// ⚠ On the algebraic arm these lanes are FULL FELTS, not `u32` halves. + /// Nothing may hand them to a byte-stream absorb; [`RootCells::absorb`] + /// exists so no caller has to know which it is holding. + pub lanes: Vec<[Felt; 4]>, +} + +impl RootCells { + /// Arena words one root occupies — the configuration's digest cell count. + /// + /// Read from the emitter's own `WrapDigest` shape rather than restated, so + /// the machine side and `proof_arena`'s writer cannot disagree about it. + pub fn words_per_root(b: &LfmBuilder) -> u32 { + if b.wrap_hash().one_cell_digest() { + 1 + } else { + DIGEST_WORDS as u32 + } + } + + /// Read a root out of an arena at `base` and hoist its unpack. + /// + /// ⚠ Consumes [`RootCells::words_per_root`] words, not always two: an + /// algebraic root is ONE arena word of four felts. Callers that advance a + /// cursor across several roots must step by the same function. + pub fn hint(b: &mut LfmBuilder, arena: super::instr::ArenaId, base: u32) -> Self { + let n = Self::words_per_root(b); + let words: Vec<_> = (0..n).map(|i| b.hint_word(arena, base + i)).collect(); + Self::from_cells(b, &words) + } + + /// A root already in hand as the configuration's digest cells. + fn from_cells(b: &mut LfmBuilder, cells: &[super::builder::Cell]) -> Self { + RootCells { + digest: super::edsl::WrapDigest::from_cells(cells), + lanes: cells.iter().map(|c| b.unpack(*c)).collect(), + } + } + + /// The root as the configuration's digest cells — what a comparison against + /// a computed digest takes. + pub fn digest(&self) -> super::edsl::WrapDigest { + self.digest + } + + /// Absorb this root into a transcript. + /// + /// ★ The reason `halves()` is gone: nine call sites spelled + /// `t.append_halves(&root.halves())`, which writes "a root is eight `u32` + /// halves" into every one of them. It is one cell of four felts on an + /// algebraic hash, and the absorb is FREE there. + pub fn absorb(&self, b: &mut LfmBuilder, t: &mut TranscriptReplay) { + t.append_root_cells(b, self.digest.cells()); + } + + /// A BYTE-hash root from its eight `u32` halves — the instruments' shape. + /// + /// ⚠ Byte arm only by construction: eight halves IS the byte digest's + /// layout, and a caller holding them is by that fact not on the algebraic + /// arm. Kept for the hash-pinned instruments and the tests that build a root + /// half by half; production reads roots through [`RootCells::hint`]. + pub fn from_halves(b: &mut LfmBuilder, halves: &[Felt]) -> Self { + assert_eq!( + halves.len(), + 4 * DIGEST_WORDS, + "a byte root is eight halves" + ); + let cells: Vec<_> = halves + .chunks(4) + .map(|c| b.pack_word([c[0], c[1], c[2], c[3]])) + .collect(); + Self::from_cells(b, &cells) + } + + /// The root's 32 BYTES as the eight `u32` halves a byte-stream fold takes — + /// correct on both arms. + /// + /// ★ Needed because `program_id` is **deliberately** keccak over bytes + /// whatever the configuration commits under: its host counterpart + /// `recursion::program_id_from_digest` names `PlatformKeccak256` explicitly, + /// since it identifies a program to CONSUMERS rather than being part of the + /// proof system's commitment layer. So one fold on this path still wants + /// bytes after everything around it became felts. + /// + /// On a byte hash the lanes ARE those halves and this is free. On an + /// algebraic one each of the four felts renders as its eight big-endian + /// bytes — `felt_be_halves`, one `LFM_BITDEC` row each — which is exactly + /// the serialisation `digest_to_commitment` performs host-side, so the two + /// agree by construction rather than by coincidence. + /// + /// ⚠ Not [`RootCells::lanes_flat`]: that yields FELTS on the algebraic arm, + /// and handing them to a byte fold would hash four values as if they were + /// eight — silently, since the count would still look plausible. + pub fn byte_halves(&self, b: &mut LfmBuilder) -> Vec { + if b.wrap_hash().one_cell_digest() { + let mut out = Vec::with_capacity(4 * DIGEST_WORDS); + for lanes in &self.lanes { + for lane in lanes { + out.extend_from_slice(&super::transcript_replay::felt_be_halves(b, *lane)); + } + } + return out; + } + self.lanes_flat() + } + + /// The lanes flattened, in order. + /// + /// ⚠ These are `u32` HALVES on a byte hash and FULL FELTS on an algebraic + /// one, so this is meaningful only to a caller that already knows which it + /// is holding — which is why the transcript absorbs go through + /// [`RootCells::absorb`] instead. It exists for the hash-pinned instruments + /// and for tests that publish or compare a root's cells directly. + pub fn lanes_flat(&self) -> Vec { + self.lanes.iter().flatten().copied().collect() + } + + /// [`RootCells::absorb`] where the segment cursor is not 4-byte aligned. + pub fn absorb_misaligned(&self, b: &mut LfmBuilder, t: &mut TranscriptReplay) { + t.append_root_cells_misaligned(b, self.digest.cells()); + } + + /// A root that is PROGRAM TEXT — its eight halves interned as constants. + /// + /// Admissible only for a commitment that is a function of the proof OPTIONS + /// and nothing else, because a program constant is part of program identity: + /// interning a root derived from per-proof data (an ELF, a register file) + /// would give the machine one program per proof instead of one per epoch + /// SHAPE. The two that qualify are BITWISE and KECCAK_RC + /// (`bitwise::preprocessed_commitment(options)`, + /// `tables::keccak_rc::preprocessed_commitment(options)`), plus PAGE's + /// zero-init root, which every zero-initialised page shares. + /// + /// Half `h` is bytes `4h..4h+4` little-endian — + /// `proof_arena::commitment_words`' layout, which is how a keccak digest + /// reaches the chip. + pub fn constant(b: &mut LfmBuilder, root: &[u8; 4 * 4 * DIGEST_WORDS]) -> Self { + // A Poseidon1 digest serialises the same way (`p1_commit`'s node + // encoding is `digest_to_commitment`), so one conversion serves both. + if b.wrap_hash().one_cell_digest() { + // ★ The same 32 bytes read as the FOUR CANONICAL FELTS an algebraic + // backend serialised them from — one interned cell, not eight + // interned halves. The conversion is the backend's own, not a + // second spelling of it. + let cell = super::algebraic_commit::commitment_to_digest(root); + let d = b.digest_const(cell); + return Self::from_cells(b, &[d.as_cell()]); + } + let halves: Vec = root + .chunks(4) + .map(|c| { + let mut bytes = [0u8; 4]; + bytes.copy_from_slice(c); + b.felt_const(FE::from(u64::from(u32::from_le_bytes(bytes)))) + }) + .collect(); + let cells: Vec<_> = (0..DIGEST_WORDS) + .map(|w| { + let lane = |j: usize| halves[4 * w + j]; + b.pack_word([lane(0), lane(1), lane(2), lane(3)]) + }) + .collect(); + Self::from_cells(b, &cells) + } + + /// A root the machine COMPUTED — the derivation's two digest words, unpacked + /// once so every consumer reads the same lanes. + /// + /// This is REGISTER's source: the commitment is a function of the previous + /// epoch's `reg_fini`, and computing it from those cells is what binds them + /// (`programs::emit_register_commitment`). A hinted REGISTER root would leave + /// the register boundary — the carried commit index among it — a free arena + /// word. + pub fn from_digest(b: &mut LfmBuilder, digest: super::edsl::WrapDigest) -> Self { + Self::from_cells(b, digest.cells()) + } +} + +/// The shape of one sub-proof's challenge replay. Every field is a program +/// constant: shape, never proof data. +#[derive(Clone, Debug)] +pub struct TableChallengeShape { + /// Position in the epoch's table list — the fork's domain separator. + pub index: usize, + /// How many sub-proofs the epoch has. Production skips the separator + /// entirely at one table (`verifier.rs:1264`), so this changes the bytes. + pub num_tables: usize, + /// Whether the sub-proof carries an aux (LogUp) trace root. + pub has_aux_root: bool, + /// Whether the sub-proof carries a bus contribution `L`. + pub has_contribution: bool, + /// `log2` of the trace length. + pub log2_trace_length: u32, + /// `log2` of the blowup factor. + pub log2_blowup: u32, + /// `ProofOptions::coset_offset`. + pub coset_offset: FE, + /// `(width, height)` of the current-row OOD block, as the proof carries it. + pub ood_current_dims: (usize, usize), + /// `(width, height)` of the pruned next-row OOD block. + pub ood_next_dims: (usize, usize), + /// Composition-poly parts — `air.composition_poly_degree_bound / N`. + pub num_parts: usize, + /// The FRI shape, which fixes how many `ζ`s are drawn and in what order the + /// layer roots are absorbed. + pub fri: FriShape, + /// `ProofOptions::grinding_factor`. Zero means no nonce at all. + pub grinding_factor: u8, + /// `ProofOptions::fri_number_of_queries`. + pub num_queries: usize, +} + +impl TableChallengeShape { + /// The challenge replay's shape for `air`'s sub-proof over `trace_length` + /// rows, at position `index` of `num_tables` — from the AIR, the trace + /// length and the AIR's options, never from a proof. Every field is the + /// value the host verifier pins the proof's copy to: the part count to the + /// AIR's degree bound (`verifier.rs:1730`), the OOD blocks to + /// `ood_blocks_well_formed`, the `L` presence to `has_trace_interaction`. + /// + /// `trace_length` must be a power of two; the caller bounds it. + pub fn derive( + air: &DynVmAir<'_>, + index: usize, + num_tables: usize, + trace_length: usize, + ) -> Self { + use stark::verifier::IsStarkVerifier; + debug_assert!(trace_length.is_power_of_two(), "a power-of-two trace"); + let opts = air.options(); + let log2_trace_length = trace_length.trailing_zeros(); + let log2_blowup = (opts.blowup_factor as usize).trailing_zeros(); + let layout = stark::verifier::Verifier::< + crate::tables::types::GoldilocksField, + GoldilocksExtension, + (), + >::ood_layout(air); + Self { + index, + num_tables, + has_aux_root: air.has_aux_trace(), + has_contribution: air.has_trace_interaction(), + log2_trace_length, + log2_blowup, + coset_offset: FE::from(opts.coset_offset), + ood_current_dims: ( + air.trace_layout().0 + air.num_auxiliary_rap_columns(), + layout.step_size(), + ), + ood_next_dims: (layout.expected_next_width(), layout.expected_next_height()), + num_parts: air.composition_poly_degree_bound(trace_length) / trace_length, + fri: FriShape::for_layout( + opts, + log2_trace_length + log2_blowup, + stark::leaf_layout::table_leaf_layout(air, trace_length), + ), + grinding_factor: opts.grinding_factor, + num_queries: opts.fri_number_of_queries, + } + } + + /// `log2` of the LDE domain. + pub fn log2_lde_length(&self) -> u32 { + self.log2_trace_length + self.log2_blowup + } + + /// Bits one query index carries — `sample_u64(lde_length >> 1)` + /// (`verifier.rs:138-141`), so one bit narrower than the domain, for row + /// pairs; `sample_u64(lde_length)`, the whole domain, under one-row leaves + /// (S2, `LeafLayout::query_bound`). Either way exactly the Merkle depth the + /// walk consumes — the FRI shape's [`FriShape::index_bits`], one definition. + pub fn index_bits(&self) -> usize { + self.fri.index_bits() + } + + fn check(&self) { + assert!( + self.index < self.num_tables, + "the table index must be in range" + ); + assert_eq!( + self.fri.log2_lde_length, + self.log2_lde_length(), + "the FRI shape and the trace shape must describe one domain" + ); + assert_eq!( + self.fri.num_queries, self.num_queries, + "the query count is one shape, declared once" + ); + assert!(self.num_parts > 0, "a composition polynomial has parts"); + } +} + +/// The proof-carried cells one table's replay absorbs. +/// +/// These are the caller's cells, hinted once and handed here — never re-hinted. +/// The struct is the assembly join surface: the same values go on to the +/// constraint leg, the DEEP fold, the FRI walk and the LogUp closure. +pub struct TableAbsorbs<'a> { + /// The aux trace root, present exactly when the AIR has an aux trace. + pub aux_root: Option<&'a RootCells>, + /// The bus contribution `L`. The LogUp closure sums THIS cell. + pub contribution: Option, + /// The composition polynomial's committed root. + pub composition_root: &'a RootCells, + /// The current-row OOD block, ROW-major as the proof carries it + /// (`width · height` cells). + pub ood_current: &'a [Ext], + /// The pruned next-row OOD block, row-major. + pub ood_next: &'a [Ext], + /// The claimed composition parts at `z^P`. + pub parts: &'a [Ext], + /// The committed FRI layer roots, in fold order. + pub fri_roots: &'a [RootCells], + /// The terminal polynomial's coefficients, low-to-high. + pub fri_coeffs: &'a [Ext], + /// The grinding nonce, present exactly when `grinding_factor > 0`. + /// + /// Carried as a FELT, so a nonce at or above `p` cannot be expressed. That + /// is a completeness restriction and not a soundness one — such a nonce + /// yields no LFM proof, never a wrong verdict — and it is unreachable in + /// practice: the prover searches nonces upward from zero, so reaching `p` + /// would mean grinding 64 bits. + pub nonce: Option, +} + +/// One table's challenges, as the cells the verification legs consume. +pub struct TableChallenges { + /// The constraint-coefficient base. Production expands `β⁰ .. β^{n−1}` and + /// splits the run into transition then boundary coefficients. + pub beta: Ext, + /// The OOD point. + pub z: Ext, + /// The DEEP batching challenge. + pub gamma: Ext, + /// `ζ₀ .. ζ_C`, or empty when the codeword never folds. + pub zetas: Vec, + /// Per query, the index bits low-to-high — `index_bits()` of them. + /// + /// Bits, never a felt: production draws `sample_u64(lde >> 1)`, whose + /// output is `nbits` bits by construction, and the walk consumes bits. A + /// felt would readmit the standalone driver's aliasing (ledger entry 5), + /// where `ι` and `ι + 2^(n−1)` are the same query. + pub iota_bits: Vec>, +} + +/// Fork the shared transcript for table `index` — `verifier.rs:1263-1266`. +/// +/// The clone emits nothing: the shared prefix's keccak rows were emitted once, +/// when Phase A ran, and every fork carries the same cells for them. The +/// separator is a program constant because the table index is shape. +pub fn fork_table(shared: &TranscriptReplay, index: usize, num_tables: usize) -> TranscriptReplay { + assert!(index < num_tables, "the table index must be in range"); + let mut fork = shared.clone(); + if num_tables > 1 { + fork.append_const_bytes(&(index as u64).to_le_bytes()); + } + fork +} + +/// `z ∉ trace domain ∪ LDE coset`, drawn once and constrained. +/// +/// Production loops until both predicates fail +/// (`is_transcript.rs:61-74`). This machine draws one `z` and proves the two +/// non-memberships, which is the same accepted set — a `z` production would +/// have rejected makes the program unprovable rather than making it accept. +/// +/// Both predicates are equalities over `z^N`, so one `N`-power chain +/// (`log2_trace_length` extension squarings) serves both, and each +/// non-equality is one extension division: `1/(a − b)` is provable exactly when +/// `a ≠ b`, the idiom `assert_canonical` uses on the candidate halves. +pub fn emit_z_ood( + b: &mut LfmBuilder, + t: &mut TranscriptReplay, + shape: &TableChallengeShape, +) -> Ext { + let z = t.sample_ext(b); + assert_z_outside_domains(b, z, shape); + z +} + +/// The two non-memberships alone, so they can be driven with a chosen `z`. +/// +/// A transcript-derived `z` is generic with overwhelming probability, so the +/// guard is unreachable from [`emit_z_ood`] — the reason it lives in its own +/// function is that `the_z_guard_rejects_a_point_in_either_domain` can then +/// feed it the points production would have rejected. +pub fn assert_z_outside_domains(b: &mut LfmBuilder, z: Ext, shape: &TableChallengeShape) { + assert_z_outside_domains_raw( + b, + z, + shape.log2_trace_length, + shape.log2_blowup, + shape.coset_offset, + ); +} + +/// [`assert_z_outside_domains`] from the three domain parameters directly — +/// the batched spine draws per-table `z`s without a per-table +/// [`TableChallengeShape`] to hand over. +pub fn assert_z_outside_domains_raw( + b: &mut LfmBuilder, + z: Ext, + log2_trace_length: u32, + log2_blowup: u32, + coset_offset: FE, +) { + // z^N by repeated squaring; N = 2^log2_trace_length. + let mut z_pow_trace = z; + for _ in 0..log2_trace_length { + z_pow_trace = b.emul(z_pow_trace, z_pow_trace); + } + let one = b.ext_const(&FEE::one()); + assert_ne_ext(b, z_pow_trace, one); + + // (z^N)^blowup against coset_offset^lde — the offset power is a program + // constant because the domain is shape. + let mut z_pow_lde = z_pow_trace; + for _ in 0..log2_blowup { + z_pow_lde = b.emul(z_pow_lde, z_pow_lde); + } + let offset_pow = coset_offset + .pow(1u64 << (log2_trace_length + log2_blowup)) + .to_extension::(); + let offset_pow = b.ext_const(&offset_pow); + assert_ne_ext(b, z_pow_lde, offset_pow); +} + +/// Constrain `a ≠ b` by exhibiting `(a − b)⁻¹`. +/// +/// `Div` is constrained as `OUT · B = A`, which for `A = 1` has no witness at +/// `B = 0`: the program is unprovable exactly when the two are equal. +fn assert_ne_ext(b: &mut LfmBuilder, x: Ext, y: Ext) { + let d = b.esub(x, y); + let one = b.ext_const(&FEE::one()); + let _ = b.ediv(one, d); +} + +/// Verify the grinding nonce — `grinding::is_valid_nonce`. +/// +/// Two keccaks: the inner hash over `PREFIX ‖ state ‖ factor` (41 bytes) and +/// the outer over `inner ‖ nonce_be` (40 bytes). +/// +/// The predicate is `u64::from_be_bytes(digest[..8]) < 2^(64 − g)` — "the top +/// `g` bits of the digest's first eight bytes, read big-endian, are zero". Those +/// eight bytes are lanes 0 and 1 of the digest's first WORD, and a lane is four +/// bytes LITTLE-endian, so byte `i` of the big-endian run is bit-range +/// `[8·(i mod 4), 8·(i mod 4) + 8)` of lane `i / 4`. The check is therefore a +/// bit decomposition of at most two lanes plus a run of zero assertions — no +/// comparison and no 64-bit arithmetic, because the bound is a power of two. +/// +/// Skipping the check would not merely be untidy: the nonce is absorbed, so +/// the query indices depend on it, and an unchecked nonce is a free re-roll of +/// every query index at zero cost. +pub(super) fn emit_grinding_check( + b: &mut LfmBuilder, + // `seed` carries its own width, so an algebraic seed (ONE cell) needs no + // change here. + seed: super::edsl::WrapDigest, + // The nonce as a FELT rather than as its two big-endian halves: the halves + // are the byte hash's rendering of it, and the algebraic arm needs the value. + nonce: Felt, + factor: u8, +) { + assert!( + (1..=64).contains(&factor), + "a grinding factor is in 1..=64 (grinding.rs:22-25), got {factor}" + ); + + if b.wrap_hash() == super::edsl::WrapHash::Poseidon1 { + emit_p1_grinding_check(b, seed, nonce, factor); + return; + } + let Some(byte_hash) = b.wrap_hash().byte_hash() else { + emit_algebraic_grinding_check(b, seed, nonce, factor); + return; + }; + + let nonce_halves = nonce_halves(b, nonce); + let mut inner = ByteString::new(); + inner.push_const(&GRINDING_PREFIX); + let mut seed_halves = Vec::with_capacity(8); + for w in seed.iter() { + seed_halves.extend_from_slice(&b.unpack(*w)); + } + inner.push_halves(&seed_halves); + inner.push_const(&[factor]); + let inner_hash = inner.wrap_hash(b, byte_hash); + + let mut outer = ByteString::new(); + let mut inner_halves = Vec::with_capacity(8); + for w in inner_hash.iter() { + inner_halves.extend_from_slice(&b.unpack(*w)); + } + outer.push_halves(&inner_halves); + outer.push_halves(&nonce_halves); + let digest = outer.wrap_hash(b, byte_hash); + + // The zero bits, as `(byte, bit-within-byte)` pairs of the big-endian run: + // `factor / 8` whole leading bytes, then the top `factor % 8` bits of the + // next one. + let whole = factor as usize / 8; + let rest = factor as usize % 8; + let mut wanted: Vec<(usize, usize)> = Vec::with_capacity(factor as usize); + for byte in 0..whole { + wanted.extend((0..8).map(|bit| (byte, bit))); + } + wanted.extend((8 - rest..8).map(|bit| (whole, bit))); + + let lanes = b.unpack(digest[0]); + let zero = b.felt_const(FE::zero()); + let mut decomposed: [Option>; 2] = [None, None]; + for (byte, bit) in wanted { + let lane = byte / 4; + let bits = match &decomposed[lane] { + Some(bits) => bits, + None => { + decomposed[lane] = Some(b.bit_dec(lanes[lane], 32)); + decomposed[lane].as_ref().expect("just decomposed") + } + }; + let v = Felt(bits[8 * (byte % 4) + bit].addr()); + b.assert_eq(v, zero); + } +} + +/// The POSEIDON1 arm of [`emit_grinding_check`]: `p1_commit::P1GrindDigest`'s +/// two width-8 permutations, emulated in their sparse form with the constant +/// input lanes folded and only the lanes read computed +/// ([`super::p1w16_emit::w8_permute_lanes`]: four for the inner digest, lane 0 +/// of the outer), then the top `factor` bits of lane 0's canonical value. +/// +/// The inner preimage is `PREFIX ‖ state ‖ factor` (41 bytes) read as 8-byte +/// big-endian felts: the prefix felt, the state's four lanes (a Poseidon1 +/// digest serialises as its canonical lanes, so the 32 bytes ARE them), and +/// `factor` in the top byte of the sixth. The outer one is the inner digest's +/// four lanes and the nonce. The digest's first eight bytes are lane 0's +/// canonical value big-endian, so "below `2^(64 − g)`" is its top `g` bits zero. +fn emit_p1_grinding_check( + b: &mut LfmBuilder, + seed: super::edsl::WrapDigest, + nonce: Felt, + factor: u8, +) { + use super::p1w16_emit::{Lane, w8_permute_lanes}; + + assert_eq!(seed.len(), 1, "a Poseidon1 transcript state is ONE cell"); + let [s0, s1, s2, s3] = b.unpack(seed[0]); + let prefix = FE::from(u64::from_be_bytes(GRINDING_PREFIX)); + let fac = FE::from(u64::from(factor) << 56); + let (c, v) = (Lane::Const, Lane::Var); + let z = FE::zero(); + let inner = w8_permute_lanes( + b, + [c(prefix), v(s0), v(s1), v(s2), v(s3), c(fac), c(z), c(z)], + 4, + ); + let outer = w8_permute_lanes( + b, + [ + v(inner[0]), + v(inner[1]), + v(inner[2]), + v(inner[3]), + v(nonce), + c(z), + c(z), + c(z), + ], + 1, + ); + let zero = b.felt_const(FE::zero()); + let bits = b.bit_dec(outer[0], 64); + for bit in &bits[64 - factor as usize..] { + b.assert_eq(Felt(bit.addr()), zero); + } +} + +fn emit_algebraic_grinding_check( + b: &mut LfmBuilder, + seed: super::edsl::WrapDigest, + nonce: Felt, + factor: u8, +) { + use super::algebraic_commit::{felts_from_bytes, single_block_leaf_cells}; + + assert_eq!(seed.len(), 1, "an algebraic transcript state is ONE cell"); + + // ---- inner: H(PREFIX ‖ seed ‖ factor), 41 bytes → six felts, one block. + // The seed's 32 bytes are left ZERO here: those four felts are machine + // cells, and leaving them zero is what lets the asserts below prove the + // constants occupy the other two slots and nothing else. + let mut inner_bytes = [0u8; 41]; + inner_bytes[..GRINDING_PREFIX.len()].copy_from_slice(&GRINDING_PREFIX); + inner_bytes[40] = factor; + let inner_felts = felts_from_bytes(&inner_bytes); + assert_eq!(inner_felts.len(), 6, "41 bytes is six big-endian groups"); + assert_eq!( + &inner_felts[1..5], + &[FE::zero(); 4], + "the seed must occupy felts 1..5 exactly, or the packs below are wrong" + ); + let inner_cells = single_block_leaf_cells(&inner_felts); + assert_eq!( + inner_cells[0], + [inner_felts[0], FE::zero(), FE::zero(), FE::zero()], + "rate cell 0 must carry the prefix felt and three seed felts" + ); + assert_eq!( + inner_cells[1], + [FE::zero(), inner_felts[5], FE::zero(), FE::zero()], + "rate cell 1 must carry the fourth seed felt then the factor felt" + ); + + let s = b.unpack(seed[0]); + let zero = b.felt_const(FE::zero()); + let prefix = b.felt_const(inner_felts[0]); + let factor_felt = b.felt_const(inner_felts[5]); + let inner_rate0 = b.pack_word([prefix, s[0], s[1], s[2]]); + let inner_rate1 = b.pack_word([s[3], factor_felt, zero, zero]); + let inner_cap = b.digest_const(inner_cells[2]); + let inner = b.permute([inner_rate0, inner_rate1, inner_cap.as_cell()])[0]; + + // ---- outer: H(inner ‖ nonce_be), 40 bytes → five felts, one block. + // ★ The inner digest cell IS rate cell 0 — its four lanes are felts 0..4 of + // this preimage — so nothing repacks it. + let outer_felts = felts_from_bytes(&[0u8; 40]); + assert_eq!(outer_felts.len(), 5, "40 bytes is five big-endian groups"); + let outer_cells = single_block_leaf_cells(&outer_felts); + assert_eq!( + outer_cells[1], + [FE::zero(); 4], + "the nonce must be felt 4, i.e. lane 0 of rate cell 1" + ); + let outer_rate1 = b.pack_word([nonce, zero, zero, zero]); + let outer_cap = b.digest_const(outer_cells[2]); + let digest = b.permute([inner, outer_rate1, outer_cap.as_cell()])[0]; + + // ---- the check: the first eight BIG-endian bytes of the digest, read as a + // `u64`, must be below `2^(64 − factor)`. Those eight bytes are lane 0's + // canonical `u64` (`cell_to_bytes` writes lane 0 first), so this is "the top + // `factor` bits of lane 0 are zero" — and `bit_dec` enforces canonicity as + // part of the row, which is exactly the `canonical()` the host applies. + let [lane0, _, _, _] = b.unpack(digest); + let bits = b.bit_dec(lane0, 64); + for bit in bits.iter().rev().take(factor as usize) { + b.assert_eq(Felt(bit.addr()), zero); + } +} + +/// `nonce.to_be_bytes()` as the two `u32` halves the transcript absorbs. +/// +/// The transcript reads halves as four LITTLE-endian bytes, so the big-endian +/// rendering is the felt's two halves in reversed ORDER, each byte-swapped — +/// which is exactly what `felt_be_halves` produces. +pub(super) fn nonce_halves(b: &mut LfmBuilder, nonce: Felt) -> [Felt; 2] { + super::transcript_replay::felt_be_halves(b, nonce) +} + +// The transcript-order mutation (tamper T6, in-guest): a test build can +// replay a one-row table with a ζ drawn BEFORE the input root and watch the +// challenge differential go red. Production has no switch. +#[cfg(test)] +thread_local! { + pub(super) static ZETA_BEFORE_INPUT_ROOT: core::cell::Cell = + const { core::cell::Cell::new(false) }; +} + +#[inline] +fn zeta_before_input_root() -> bool { + #[cfg(test)] + { + ZETA_BEFORE_INPUT_ROOT.with(|c| c.get()) + } + #[cfg(not(test))] + { + false + } +} + +/// Replay one table's rounds 2 to 4 against a FORKED transcript. +/// +/// `t` must be the fork ([`fork_table`]), not the shared transcript. Returns +/// the challenges as cells; every absorbed value came from the caller. +pub fn emit_table_challenges( + b: &mut LfmBuilder, + t: &mut TranscriptReplay, + shape: &TableChallengeShape, + absorbs: &TableAbsorbs<'_>, +) -> TableChallenges { + shape.check(); + assert_eq!( + absorbs.aux_root.is_some(), + shape.has_aux_root, + "the aux root's presence is shape" + ); + assert_eq!( + absorbs.contribution.is_some(), + shape.has_contribution, + "the contribution's presence is shape" + ); + assert_eq!( + absorbs.ood_current.len(), + shape.ood_current_dims.0 * shape.ood_current_dims.1, + "the current-row OOD block must match its declared dimensions" + ); + assert_eq!( + absorbs.ood_next.len(), + shape.ood_next_dims.0 * shape.ood_next_dims.1, + "the next-row OOD block must match its declared dimensions" + ); + assert_eq!(absorbs.parts.len(), shape.num_parts, "one cell per part"); + assert_eq!( + absorbs.fri_roots.len(), + shape.fri.num_committed(), + "one root per committed FRI layer" + ); + assert_eq!( + absorbs.fri_coeffs.len(), + shape.fri.num_terminal_coeffs(), + "the terminal polynomial's coefficient count is shape" + ); + assert_eq!( + absorbs.nonce.is_some(), + shape.grinding_factor > 0, + "a nonce exists exactly when grinding is on" + ); + + // ---- Phase C and the contribution bind, inside the fork. + if let Some(root) = absorbs.aux_root { + root.absorb(b, t); + } + if let Some(l) = absorbs.contribution { + append_ext_cell(b, t, l); + } + + // ---- Round 2: β, then the composition root. + let beta = t.sample_ext(b); + absorbs.composition_root.absorb(b, t); + + // ---- Round 3: z, then both OOD blocks COLUMN-major, then the parts. + let z = emit_z_ood(b, t, shape); + for (dims, block) in [ + (shape.ood_current_dims, absorbs.ood_current), + (shape.ood_next_dims, absorbs.ood_next), + ] { + let (width, height) = dims; + for col in 0..width { + for row in 0..height { + append_ext_cell(b, t, block[row * width + col]); + } + } + } + for part in absorbs.parts { + append_ext_cell(b, t, *part); + } + + // ---- Round 4: γ, the interleaved FRI commit phase, then the queries. + let gamma = t.sample_ext(b); + + let mut zetas = Vec::with_capacity(shape.fri.num_zetas()); + for (j, root) in absorbs.fri_roots.iter().enumerate() { + // Sample FIRST, absorb SECOND — a ζ drawn after its own layer root is a + // challenge the prover answers rather than one that binds them. + // + // ★ Except the one-row INPUT tree (S2): root 0 is + // the DEEP codeword itself, committed BEFORE any folding challenge — + // absorbed right after γ, with no ζ ahead of it. A ζ drawn before it + // would let the prover pick the codeword after seeing λ₁; the host + // replay (`verifier.rs`, `replay_rounds_after_round_1`) + // is the same loop. + if !(shape.fri.one_row() && j == 0) || zeta_before_input_root() { + zetas.push(t.sample_ext(b)); + } + root.absorb(b, t); + } + if shape.fri.total_folds() > 0 { + zetas.push(t.sample_ext(b)); + } + for c in absorbs.fri_coeffs { + append_ext_cell(b, t, *c); + } + + if let Some(nonce) = absorbs.nonce { + let seed = t.state(b); + emit_grinding_check(b, seed, nonce, shape.grinding_factor); + // `append_felt` IS `append_halves(&felt_be_halves(nonce))` on the byte + // arm — the same two halves this used to build by hand — and on the + // algebraic arm it is the free lane-0 cell. + t.append_felt(b, nonce); + } + + let iota_bits = (0..shape.num_queries) + .map(|_| t.sample_u64_pow2(b, shape.index_bits())) + .collect(); + + TableChallenges { + beta, + z, + gamma, + zetas, + iota_bits, + } +} + +/// Rebuild the full OOD grid from the two pruned blocks the proof carries. +/// +/// The in-machine analogue of `ood::reconstruct_ood_full` +/// (`crypto/stark/src/ood.rs`), and the seam between the spine and the two legs +/// that fold the grid: the same cells [`emit_table_challenges`] absorbed +/// column-major come back here as `num_eval_points × num_total_cols` rows, which +/// is the shape `constraints::emit_analyzed` reads `Op::Var{offset, col}` out of +/// and the shape `deep::emit_deep_invariants` sums. +/// +/// ## The zeros are program text, not arena data +/// +/// A pruned next-row entry is reconstructed as ZERO by the real verifier — no +/// transition constraint reads a pruned column at the next row, and DEEP pairs +/// those positions with zero coefficients. Emitting the pooled zero constant +/// makes the pruning part of the program rather than a property of the supplied +/// arena, which is the standing decision ("next-row pruning likewise, because the +/// verifier reconstructs an undeclared column as ZERO"). The permissive +/// direction is the dangerous one: a machine that hinted a value into a pruned +/// slot would fold a frame the real verifier cannot see. +/// +/// This emits no instruction beyond interning that zero — it is cell plumbing, +/// which is the point. The blocks are the transcript's own cells, so there is no +/// second copy of the grid for a prover to disagree with. +pub fn emit_reconstruct_ood( + b: &mut LfmBuilder, + deep: &super::deep::DeepShape, + current: &[Ext], + next: &[Ext], +) -> Vec> { + let width = deep.num_total_cols; + let mask_width = deep.next_row_cols.len(); + let next_rows = deep + .num_eval_points + .checked_sub(deep.step_size) + .expect("the OOD grid is at least the current-row block"); + assert_eq!( + current.len(), + deep.step_size * width, + "the current-row block is step_size × num_total_cols" + ); + assert_eq!( + next.len(), + next_rows * mask_width, + "the next-row block is (num_eval_points − step_size) × |next_row_cols|" + ); + + let zero = b.felt_const(FE::zero()).as_ext(); + let mut rows = Vec::with_capacity(deep.num_eval_points); + for r in 0..deep.step_size { + rows.push(current[r * width..(r + 1) * width].to_vec()); + } + for r in 0..next_rows { + let mut row = vec![zero; width]; + for (m, &col) in deep.next_row_cols.iter().enumerate() { + assert!( + col < width, + "a next-row column must index into the trace: {col} against {width}" + ); + row[col] = next[r * mask_width + m]; + } + rows.push(row); + } + rows +} + +/// Absorb an extension cell the way `append_field_element` streams it: three +/// coordinates, each eight big-endian bytes. +pub(super) fn append_ext_cell(b: &mut LfmBuilder, t: &mut TranscriptReplay, v: Ext) { + let coords = b.unpack(v.as_cell()); + t.append_ext(b, [coords[0], coords[1], coords[2]]); +} + +/// Constrain `v < 2^32` — the felt-width guard assembly ledger entry 1 owes. +/// +/// An LFM arena is untyped felts; production's `reg_fini` and `register_init` are +/// `Vec`, and that TYPE is the whole enforcement on their side. So without +/// this the machine's accepted set is strictly wider than production's: a prover +/// could derive the REGISTER preprocessed commitment over a boundary value no +/// production epoch can hold. +/// +/// One `BitDec` plus one recomposition: `bit_dec` exposes the low 32 bits, and +/// asserting `v` equals their weighted sum has no witness above `2^32 − 1`. The +/// same idiom `emit_output_bytes` uses on the public-output halves, and for the +/// same reason. +/// +/// Deliberately at the ASSEMBLY call site rather than inside +/// `programs::emit_register_commitment`: the isolated derivation's width gap is +/// pinned by a guard test that asserts the hazard still exists +/// (`machine_tests::the_derivation_extends_a_non_u32_register_value_demonstrating_ +/// hazard`), which is correct — an isolated derivation binds nothing, and closing +/// the gap is what assembly is for. +pub fn assert_u32(b: &mut LfmBuilder, v: Felt) { + let bits = b.bit_dec(v, 32); + let two = b.felt_const(FE::from(2u64)); + let mut acc = Felt(bits[31].addr()); + for k in (0..31).rev() { + let bit = Felt(bits[k].addr()); + acc = b.mul_add(acc, two, bit); + } + b.assert_eq(v, acc); +} + +/// `β⁰ .. β^{n−1}` — production's `compute_alpha_powers(&beta, n)`, which the +/// quotient fold consumes as transition coefficients then boundary ones. +/// +/// Derived in-machine from the ONE `β` the transcript produced, for the reason +/// `constraints::emit_alpha_powers` exists: a hinted power run is a prover's +/// free choice of every constraint coefficient. +pub fn emit_beta_powers(b: &mut LfmBuilder, beta: Ext, n: usize) -> Vec { + super::constraints::emit_alpha_powers(b, beta, n) +} + +/// The public output as one cell per BYTE, derived from the `u32` halves the +/// statement absorbed. +/// +/// The output has two consumers with incompatible shapes: the statement absorb +/// wants four-byte halves, and the COMMIT-bus target folds one term per byte +/// (`logup::emit_commit_bus_target`). Two arenas would be two claims about the +/// same string — the assembly obligation's fifth instance. So the halves are +/// the arena, and the bytes are DERIVED here. +/// +/// Each half costs one `BitDec` and one `MulAdd`, and the recomposition assert +/// is what makes it a range check as well: a hinted half at or above `2^32` +/// cannot equal the sum of the four bytes read out of its low 32 bits, so the +/// program is unprovable rather than absorbing one string and folding another. +/// +/// A trailing partial half is masked the same way the transcript masks it: only +/// `len_bytes` bytes are returned, and the unused high bytes of the last half +/// are pinned to zero — without that a prover could absorb one string while the +/// length prefix claimed another. +pub fn emit_output_bytes(b: &mut LfmBuilder, halves: &[Felt], len_bytes: usize) -> Vec { + assert_eq!( + halves.len(), + len_bytes.div_ceil(4), + "one half per four output bytes, the last one partial" + ); + let mut out = Vec::with_capacity(len_bytes); + let zero = b.felt_const(FE::zero()); + for (h, half) in halves.iter().enumerate() { + let bits = b.bit_dec(*half, 32); + let bytes: Vec = (0..4) + .map(|k| super::edsl::bits_to_felt(b, &bits[8 * k..8 * k + 8])) + .collect(); + // half = Σ byteₖ·2^{8k}, which pins the half to its four bytes AND to + // the range `[0, 2^32)`. + let mut acc = bytes[3]; + for k in (0..3).rev() { + let shift = b.felt_const(FE::from(256u64)); + acc = b.mul_add(acc, shift, bytes[k]); + } + b.assert_eq(*half, acc); + + for (k, byte) in bytes.into_iter().enumerate() { + if 4 * h + k < len_bytes { + out.push(byte); + } else { + b.assert_eq(byte, zero); + } + } + } + out +} diff --git a/prover/src/lfm/epoch_tests.rs b/prover/src/lfm/epoch_tests.rs new file mode 100644 index 000000000..1b9b76a7a --- /dev/null +++ b/prover/src/lfm/epoch_tests.rs @@ -0,0 +1,3045 @@ +//! The assembled verifier's challenge replay, differentialled against +//! production's own. +//! +//! ## The oracle +//! +//! `Verifier::replay_rounds_after_round_1` — the function `multi_verify_views` +//! itself calls. Nothing here models Fiat-Shamir; the expected `β`, `z`, `γ`, +//! `ζ_k` and `ι_s` are the values the production verifier computed for a real +//! proof of a real AIR, and the machine is asked to reproduce them from the +//! proof's own bytes. +//! +//! ## What this suite can see that no leg-side suite could +//! +//! The FRI commit phase's INTERLEAVING (ledger entry 4). `fri_tests` supplies +//! `ζ_k` from the same replay it checks against, so absorbing the layer roots +//! in the wrong order — or not at all — moves nothing there. Here every +//! challenge is derived from the absorbed bytes, so a misordered absorb changes +//! `ζ`, and a `ζ` change moves the fold. The four fixtures span +//! `num_committed = 0, 1, 2, 3`, so the loop is exercised at zero, one and +//! several layers. +//! +//! ## What it cannot see +//! +//! It stops at the challenges. That the legs then CONSUME these cells is +//! [`the_legs_consume_the_replayed_challenges`]'s job, and the whole-epoch +//! composition (16 sub-proofs behind one statement on this fixture, 25 at the +//! post-#977 ceiling) is not built here. + +use stark::config::Commitment; +use stark::proof::stark::MultiProof; +use stark::proof::view::StarkProofView; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::edsl; +use super::epoch::{ + RootCells, TableAbsorbs, TableChallengeShape, emit_table_challenges, fork_table, +}; +use super::executor::execute; +use super::fri::FriShape; +use super::instr::ArenaId; +use super::transcript_replay::TranscriptReplay; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// Everything one real sub-proof supplies to the replay (the reader lives in +/// [`super::harvest`], where the block tree's driver uses it too). +pub(super) use super::harvest::HostTable; + +/// Read a real single-table proof into [`HostTable`], taking the challenges +/// from the production verifier rather than recomputing them. +fn host_table( + air: &dyn AIR, + proof: &MultiProof, +) -> HostTable { + let sp = super::constraint_tests::open_sub_proof(air, proof); + let view = StarkProofView::Owned(&proof.proofs[0]); + let opts = air.options(); + + let trace_length = view.trace_length(); + let log2_trace_length = trace_length.trailing_zeros(); + let log2_blowup = (opts.blowup_factor as usize).trailing_zeros(); + let fri = FriShape::for_layout( + opts, + log2_trace_length + log2_blowup, + stark::leaf_layout::table_leaf_layout(air, trace_length), + ); + + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + + // `γ` comes back from the DEEP shape derivation, which recovers it from the + // verifier's own coefficient run — the same route `join_tests` uses. + let (_deep, gamma) = super::constraint_tests::deep_shape(&sp, air); + + let shape = TableChallengeShape { + index: 0, + num_tables: 1, + has_aux_root: view.lde_trace_aux_merkle_root().is_some(), + has_contribution: view.bus_table_contribution().is_some(), + log2_trace_length, + log2_blowup, + coset_offset: FE::from(opts.coset_offset), + ood_current_dims: (ood_c.width(), ood_c.height()), + ood_next_dims: (ood_n.width(), ood_n.height()), + num_parts: view.composition_poly_parts_ood_evaluation().len(), + fri, + grinding_factor: opts.grinding_factor, + num_queries: opts.fri_number_of_queries, + }; + + HostTable { + shape, + precomputed_root: air.is_preprocessed().then(|| { + super::epoch_verify_tests::layout_precomputed_commitment(air, view.trace_length()) + }), + main_root: *view.lde_trace_main_merkle_root(), + aux_root: view.lde_trace_aux_merkle_root().copied(), + contribution: view.bus_table_contribution(), + composition_root: *view.composition_poly_root(), + ood_current: ood_c.row_major_data().to_vec(), + ood_next: ood_n.row_major_data().to_vec(), + parts: view.composition_poly_parts_ood_evaluation().to_vec(), + fri_roots: view.fri_layers_merkle_roots().to_vec(), + fri_coeffs: view.fri_final_poly_coeffs().to_vec(), + nonce: view.nonce(), + needs_lookup_challenges: air.has_aux_trace(), + beta: sp.beta, + z: sp.challenges.z, + gamma, + zetas: sp.challenges.zetas.clone(), + iotas: sp.challenges.iotas.clone(), + } +} + +/// Arena identifiers of the challenge program, in declaration order. +struct Arenas { + main_root: ArenaId, + aux_root: Option, + contribution: Option, + composition_root: ArenaId, + ood_current: ArenaId, + ood_next: ArenaId, + parts: ArenaId, + fri_roots: ArenaId, + fri_coeffs: ArenaId, + nonce: Option, + /// The verification legs' two arenas, present only in the ASSEMBLED + /// verifier — the trace openings and the FRI layer openings. `None` in the + /// spine-only program, which verifies nothing and so opens nothing. + legs: Option, +} + +/// A program that replays ONE table's challenges and publishes them. +/// +/// The transcript prefix is `multi_verify_views`' single-table Phase A: the +/// hardcoded precomputed commitment when the AIR is preprocessed, the main +/// root, then the shared LogUp challenges. The fork follows, then rounds 2-4. +fn challenge_program(h: &HostTable) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let shape = &h.shape; + + let a = Arenas { + main_root: b.declare_arena(super::proof_arena::words_per_root() as u32), + aux_root: shape + .has_aux_root + .then(|| b.declare_arena(super::proof_arena::words_per_root() as u32)), + contribution: shape.has_contribution.then(|| b.declare_arena(1)), + composition_root: b.declare_arena(super::proof_arena::words_per_root() as u32), + ood_current: b.declare_arena((shape.ood_current_dims.0 * shape.ood_current_dims.1) as u32), + ood_next: b.declare_arena((shape.ood_next_dims.0 * shape.ood_next_dims.1) as u32), + parts: b.declare_arena(shape.num_parts as u32), + fri_roots: b.declare_arena( + super::proof_arena::words_per_root() as u32 * shape.fri.num_committed() as u32, + ), + fri_coeffs: b.declare_arena(shape.fri.num_terminal_coeffs() as u32), + nonce: (shape.grinding_factor > 0).then(|| b.declare_arena(1)), + legs: None, + }; + + let mut t = TranscriptReplay::new(&[]); + if let Some(prep) = h.precomputed_root { + t.append_const_bytes(&prep); + } + let main = RootCells::hint(&mut b, a.main_root, 0); + main.absorb(&mut b, &mut t); + if h.needs_lookup_challenges { + for _ in 0..stark::lookup::LOGUP_NUM_CHALLENGES { + t.sample_ext(&mut b); + } + } + + let aux = a.aux_root.map(|id| RootCells::hint(&mut b, id, 0)); + let contribution = a.contribution.map(|id| b.hint_word(id, 0).as_ext()); + let composition = RootCells::hint(&mut b, a.composition_root, 0); + let ood_current: Vec<_> = (0..(shape.ood_current_dims.0 * shape.ood_current_dims.1) as u32) + .map(|i| b.hint_word(a.ood_current, i).as_ext()) + .collect(); + let ood_next: Vec<_> = (0..(shape.ood_next_dims.0 * shape.ood_next_dims.1) as u32) + .map(|i| b.hint_word(a.ood_next, i).as_ext()) + .collect(); + let parts: Vec<_> = (0..shape.num_parts as u32) + .map(|i| b.hint_word(a.parts, i).as_ext()) + .collect(); + let fri_roots: Vec<_> = (0..shape.fri.num_committed()) + .map(|i| { + RootCells::hint( + &mut b, + a.fri_roots, + super::proof_arena::words_per_root() as u32 * i as u32, + ) + }) + .collect(); + let fri_coeffs: Vec<_> = (0..shape.fri.num_terminal_coeffs() as u32) + .map(|i| b.hint_word(a.fri_coeffs, i).as_ext()) + .collect(); + let nonce = a.nonce.map(|id| b.hint_felt(id, 0)); + + let mut fork = fork_table(&t, shape.index, shape.num_tables); + let ch = emit_table_challenges( + &mut b, + &mut fork, + shape, + &TableAbsorbs { + aux_root: aux.as_ref(), + contribution, + composition_root: &composition, + ood_current: &ood_current, + ood_next: &ood_next, + parts: &parts, + fri_roots: &fri_roots, + fri_coeffs: &fri_coeffs, + nonce, + }, + ); + + b.public(ch.beta.as_cell()); + b.public(ch.z.as_cell()); + b.public(ch.gamma.as_cell()); + for zeta in &ch.zetas { + b.public(zeta.as_cell()); + } + for bits in &ch.iota_bits { + let felt = edsl::bits_to_felt(&mut b, bits); + b.public(felt.as_cell()); + } + + let program = compile(b.finish()); + validate(&program).expect("the challenge program must be admissible"); + program +} + +/// The arenas [`challenge_program`] declares, in the same order. +fn challenge_arenas(h: &HostTable) -> Vec> { + let mut out = vec![super::proof_arena::commitments_to_arena(&[h.main_root])]; + if let Some(r) = h.aux_root { + out.push(super::proof_arena::commitments_to_arena(&[r])); + } + if let Some(c) = h.contribution { + out.push(vec![ext_word(&c)]); + } + out.push(super::proof_arena::commitments_to_arena(&[ + h.composition_root + ])); + out.push(h.ood_current.iter().map(ext_word).collect()); + out.push(h.ood_next.iter().map(ext_word).collect()); + out.push(h.parts.iter().map(ext_word).collect()); + out.push(super::proof_arena::commitments_to_arena(&h.fri_roots)); + out.push(h.fri_coeffs.iter().map(ext_word).collect()); + if let Some(n) = h.nonce { + out.push(vec![base_word(FE::from(n))]); + } + out +} + +/// Run the program and read the published challenges back. +fn run(h: &HostTable) -> (FEE, FEE, FEE, Vec, Vec) { + let program = challenge_program(h); + let arenas = challenge_arenas(h); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the replay must execute"); + + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + let beta = pub_ext(0); + let z = pub_ext(1); + let gamma = pub_ext(2); + let zetas: Vec = (0..h.zetas.len()).map(|k| pub_ext(3 + k)).collect(); + let base = 3 + h.zetas.len(); + let iotas: Vec = (0..h.shape.num_queries) + .map(|q| { + let w = exec.public_words[base + q].1; + let felt = super::word::word_as_base(&w).expect("an index is a base felt"); + felt.to_raw() + }) + .collect(); + (beta, z, gamma, zetas, iotas) +} + +/// ★ The whole point of the leg: every challenge the legs consume is the one +/// production derived, and it came out of the transcript rather than an arena. +/// +/// Swept over four real proofs whose committed FRI layer counts are 0, 1, 2 and +/// 3, because the Round-4 interleaving only has anything to get wrong once a +/// layer exists. +#[test] +fn the_challenge_replay_matches_production() { + for (boundaries, committed) in [(4usize, 0usize), (512, 1), (1024, 2), (2048, 3)] { + let (air, proof) = super::fri_tests::folding_fixture(boundaries, 2); + let h = host_table(&*air, &proof); + assert_eq!( + h.shape.fri.num_committed(), + committed, + "fixture of {boundaries} boundaries must commit {committed} layers" + ); + assert_eq!( + h.zetas.len(), + if h.shape.fri.total_folds() > 0 { + committed + 1 + } else { + 0 + }, + "folds exceed committed layers by one, and vanish when nothing folds" + ); + + let (beta, z, gamma, zetas, iotas) = run(&h); + assert_eq!(beta, h.beta, "beta at {boundaries} boundaries"); + assert_eq!(z, h.z, "z at {boundaries} boundaries"); + assert_eq!(gamma, h.gamma, "gamma at {boundaries} boundaries"); + assert_eq!(zetas, h.zetas, "the FRI zetas at {boundaries} boundaries"); + let want: Vec = h.iotas.iter().map(|i| *i as u64).collect(); + assert_eq!(iotas, want, "the query indices at {boundaries} boundaries"); + } +} + +/// ★ S2 (one-row leaves): the in-machine replay of a +/// one-row table reproduces production's challenges — the input root absorbed +/// right after `γ` with NO challenge ahead of it, one `ζ` per committed layer +/// (layer `j` folds with `ζ_j`), and the query indices sampled over the WHOLE +/// LDE (`log2(lde)` bits, the upper half reached). Swept over folding counts 0 +/// (the zero-fold case: no input tree at all), 1+ layers. The transcript order +/// is load-bearing: the replay with a `ζ` drawn BEFORE the input root (the test +/// mutation) diverges from production wherever an input tree exists. +#[test] +fn the_one_row_challenge_replay_matches_production() { + for (boundaries, fri) in [ + (4usize, stark::proof::options::FriMode::Pair), + (512, stark::proof::options::FriMode::Pair), + (2048, stark::proof::options::FriMode::Pair), + (2048, stark::proof::options::FriMode::Dp), + ] { + let mut opts = + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup 2"); + opts.format.one_row = stark::proof::options::OneRowMode::On; + opts.format.fri_mode = fri; + let (air, proof) = super::fri_tests::folding_fixture_with(boundaries, opts); + let h = host_table(&*air, &proof); + let label = format!("{boundaries} boundaries, fri={fri:?}"); + assert!(h.shape.fri.one_row(), "{label}"); + assert_eq!(h.shape.index_bits(), h.shape.log2_lde_length() as usize); + assert_eq!( + h.zetas.len(), + h.shape.fri.num_committed(), + "{label}: one challenge per committed layer (the input tree has none)" + ); + assert_eq!(h.zetas.len(), h.shape.fri.num_zetas()); + + let (beta, z, gamma, zetas, iotas) = run(&h); + assert_eq!(beta, h.beta, "{label}: beta"); + assert_eq!(z, h.z, "{label}: z"); + assert_eq!(gamma, h.gamma, "{label}: gamma"); + assert_eq!(zetas, h.zetas, "{label}: the FRI zetas"); + let want: Vec = h.iotas.iter().map(|i| *i as u64).collect(); + assert_eq!(iotas, want, "{label}: the query indices"); + let lde = 1u64 << h.shape.log2_lde_length(); + if h.iotas.len() >= 8 { + assert!( + want.iter().any(|&r| r >= lde / 2), + "{label}: one-row indices range over the whole LDE" + ); + } + + if h.shape.fri.num_committed() > 0 { + super::epoch::ZETA_BEFORE_INPUT_ROOT.with(|c| c.set(true)); + let mutated = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let program = challenge_program(&h); + let arenas = challenge_arenas(&h); + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .ok() + .map(|exec| { + (0..h.zetas.len()) + .map(|k| word_as_ext(&exec.public_words[3 + k].1).expect("ext")) + .collect::>() + }) + })); + super::epoch::ZETA_BEFORE_INPUT_ROOT.with(|c| c.set(false)); + let mutated = mutated.expect("the mutated replay still emits"); + assert_ne!( + mutated.as_ref(), + Some(&h.zetas), + "{label}: a ζ drawn before the input root must move the challenges" + ); + } + println!( + "{label}: {} layers, {} zetas, {} queries over 2^{} — replay == production", + h.shape.fri.num_committed(), + h.zetas.len(), + h.iotas.len(), + h.shape.log2_lde_length() + ); + } +} + +/// ★ Two defects the differential above CANNOT see, pinned so they are not +/// mistaken for coverage. +/// +/// Both are degenerate parameters of the single-table L2G fixture, and the +/// falsification runs found them rather than reasoning predicting them: +/// injecting a ROW-major OOD absorb and deleting the fork's domain separator +/// both left `the_challenge_replay_matches_production` green. +#[test] +fn the_single_table_fixture_is_blind_to_two_defects() { + let (air, proof) = super::fri_tests::folding_fixture(2048, 2); + let h = host_table(&*air, &proof); + println!( + "ood_current {:?} ood_next {:?} num_tables {}", + h.shape.ood_current_dims, h.shape.ood_next_dims, h.shape.num_tables + ); + assert_eq!( + h.shape.ood_current_dims.1, 1, + "a one-ROW OOD block reads the same column-major as row-major, so this \ + fixture cannot witness the absorb order" + ); + assert_eq!( + h.shape.ood_next_dims.1, 1, + "likewise for the next-row block" + ); + assert_eq!( + h.shape.num_tables, 1, + "production emits no domain separator at one table (verifier.rs:1264), \ + so this fixture cannot witness the fork's separator" + ); +} + +/// ★ The `z` guard, driven with the points production rejects. +/// +/// `sample_z_ood_with_domain_params` loops until `z` is outside both the trace +/// domain and the LDE coset. The machine cannot loop, so it constrains the +/// first draw — and that constraint is unreachable from a real transcript, +/// which is why the guard is emitted against a HINTED `z` here. Both rejection +/// branches are exercised separately, with a generic `z` as the positive +/// control: without it, a guard that rejected everything would look identical. +#[test] +fn the_z_guard_rejects_a_point_in_either_domain() { + use math::field::traits::IsFFTField; + + let shape = TableChallengeShape { + index: 0, + num_tables: 1, + has_aux_root: false, + has_contribution: false, + log2_trace_length: 4, + log2_blowup: 1, + coset_offset: FE::from(3u64), + ood_current_dims: (1, 1), + ood_next_dims: (0, 0), + num_parts: 1, + fri: FriShape::from_options(&super::proof_fixture::fixture_options(), 5), + grinding_factor: 0, + num_queries: 1, + }; + + let program = { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let a = b.declare_arena(1); + let z = b.hint_word(a, 0).as_ext(); + super::epoch::assert_z_outside_domains(&mut b, z, &shape); + let program = compile(b.finish()); + validate(&program).expect("the guard program must be admissible"); + program + }; + let runs = |z: FEE| { + execute( + &program, + &[vec![ext_word(&z)]], + &crate::hash_pin::BLOCK_HASHER, + ) + .is_ok() + }; + + // Positive control: a generic point passes, so a guard that rejected + // everything would not be mistaken for a working one. + assert!( + runs(FEE::new([FE::from(7u64), FE::from(11u64), FE::from(13u64)])), + "a generic z must pass both non-memberships" + ); + + // In the trace domain: a 16th root of unity, so z^16 = 1. + let g = ::get_primitive_root_of_unity(4).expect("root of unity"); + for k in [0u64, 1, 5] { + let z = g.pow(k).to_extension::(); + assert!( + !runs(z), + "z = g^{k} is in the trace domain and production would have redrawn" + ); + } + + // On the LDE coset: z = offset · ω^k with ω the 32nd root of unity, so + // z^32 = offset^32. + let w = ::get_primitive_root_of_unity(5).expect("root of unity"); + for k in [0u64, 3, 17] { + let z = (FE::from(3u64) * w.pow(k)).to_extension::(); + assert!( + !runs(z), + "z = 3·ω^{k} is on the LDE coset and production would have redrawn" + ); + } +} + +/// ★ The grinding check, which the challenge differential is structurally +/// blind to. +/// +/// A wrong nonce changes the query indices, so the differential above would +/// simply compare different-but-consistent values; deleting +/// `emit_grinding_check` entirely left it green (falsification run +/// `grinding_check`). What makes a wrong nonce REJECT is the proof-of-work +/// predicate, and this drives it: the proof's own nonce runs, and eight +/// neighbouring nonces — which absorb just as happily — do not. +/// +/// This matters beyond tidiness. The nonce is absorbed before the query +/// indices are drawn, so an unchecked nonce is a free re-roll of every query +/// index: a prover with a bad codeword re-grinds until the indices miss it, +/// at the cost production charges 2^20 hashes for. +#[test] +fn a_nonce_that_did_not_grind_is_rejected() { + let (air, proof) = super::fri_tests::folding_fixture(4, 2); + let h = host_table(&*air, &proof); + assert!( + h.shape.grinding_factor > 0, + "the fixture must actually grind, or this test proves nothing" + ); + let real = h.nonce.expect("a grinding proof carries a nonce"); + + let program = challenge_program(&h); + let runs = |nonce: u64| { + let mut h2 = h.clone(); + h2.nonce = Some(nonce); + execute( + &program, + &challenge_arenas(&h2), + &crate::hash_pin::BLOCK_HASHER, + ) + .is_ok() + }; + + assert!(runs(real), "the proof's own nonce must satisfy the check"); + let mut rejected = 0; + for delta in 1..=8u64 { + if !runs(real.wrapping_add(delta)) { + rejected += 1; + } + } + assert_eq!( + rejected, 8, + "at a grinding factor of {} a neighbouring nonce passes with \ + probability 2^-{}, so all eight must be rejected", + h.shape.grinding_factor, h.shape.grinding_factor + ); +} + +// ============================================================================= +// The whole epoch: one statement, Phase A over every sub-proof, then a fork per +// table. +// ============================================================================= + +/// A real continuation epoch, proved over the production epoch AIR set, with +/// production's own per-table challenges extracted for every sub-proof. +/// +/// Built the way `prove_continuation` builds epoch 0 — the same construction +/// `logup_tests::a_zero_row_fixed_table_carries_some_zero_not_none` proves and +/// production ACCEPTS. Nothing here is synthetic: the statement is the real +/// one, the forks carry the real domain separators, and the challenges come +/// from `replay_rounds_after_round_1` on each fork. +/// ★ Where a preprocessed commitment COMES FROM — assembly ledger entry 7, +/// as a type. +/// +/// Production absorbs every preprocessed root from the AIR and never from the +/// proof, so the machine owes each one a provenance of the same strength. The +/// three variants are the three that exist, and which one a commitment gets is +/// decided by what the commitment is a function of: +/// +/// - options only ⇒ [`Self::Constant`], interned as program text. Safe because +/// the proof options are already program SHAPE. +/// - the previous epoch's register boundary ⇒ [`Self::Register`], DERIVED +/// in-machine. Interning it would pin one LFM program per register file; +/// hinting it would leave the boundary — the carried commit index among it — a +/// free arena word (ledger entry 2). +/// - the inner ELF ⇒ [`Self::ElfDependent`], an arena cell bound one level up by +/// the attestation's `program_id` fold. Interning it would make LFM program +/// identity a function of the guest ELF, which is an always-stop item. +/// +/// [`prep_source`] decides the variant by MATCHING the AIR's own commitment +/// against production's candidate functions, so an epoch that grew a preprocessed +/// table with no known provenance panics instead of quietly hinting an unbound +/// root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum PrepSource { + /// A function of `ProofOptions` alone: BITWISE, KECCAK_RC, or PAGE's + /// shared zero-init root; or of the options and the table's height, which + /// the statement carries: a CPU table's TIMESTAMP. + Constant(Commitment), + /// REGISTER: `compute_precomputed_commitment(options, INIT, FINI)`. + Register(Commitment), + /// A function of the inner ELF: DECODE, or an ELF-data page's root. + ElfDependent(Commitment), +} + +impl PrepSource { + /// Whether this root occupies arena words — true for exactly the + /// ELF-dependent family. + fn is_arena(self) -> bool { + matches!(self, PrepSource::ElfDependent(_)) + } +} + +/// How many of an epoch's preprocessed roots come from each source — +/// `(options-only, derived, ELF-dependent)`. +pub(super) fn prep_source_census(e: &RealEpoch) -> (usize, usize, usize) { + let mut census = (0, 0, 0); + for source in e.phase_a.iter().filter_map(|(p, _)| *p) { + match source { + PrepSource::Constant(_) => census.0 += 1, + PrepSource::Register(_) => census.1 += 1, + PrepSource::ElfDependent(_) => census.2 += 1, + } + } + census +} + +/// Classify one preprocessed commitment by recomputing every candidate +/// production has and seeing which one it IS. +/// +/// A match is not a heuristic: these are keccak Merkle roots over different +/// tables, so two candidates agreeing would be a collision. What the function +/// really buys is the failure mode — a preprocessed AIR whose root matches +/// nothing known is a root the machine has no binding for, and this panics +/// rather than hinting it. +/// +/// `layout` is the table's resolved trace-tree leaf layout (S2): every +/// candidate is recomputed AT that layout, so a one-row table's root is matched +/// against the one-row candidates only (a row-pair root never stands in). +fn prep_source( + root: Commitment, + opts: &crate::ProofOptions, + elf: &executor::elf::Elf, + register_init: &[u32], + reg_fini: &[u32], + height: usize, + layout: stark::leaf_layout::LeafLayout, +) -> PrepSource { + use crate::tables::{bitwise, cpu, decode, keccak_rc, page, register}; + + if Some(root) == bitwise::preprocessed_commitment_for(opts, layout) + || Some(root) == keccak_rc::preprocessed_commitment_for(opts, layout) + || Some(root) == page::zero_init_preprocessed_commitment_for(opts, layout) + || root == cpu::compute_timestamp_commitment_with(opts, height, layout) + { + return PrepSource::Constant(root); + } + if root == register::compute_precomputed_commitment_with(opts, register_init, reg_fini, layout) + { + return PrepSource::Register(root); + } + let instructions = + decode::instructions_from_elf(elf).expect("the DECODE commitment must compute"); + if root == decode::compute_precomputed_commitment_with(&instructions, opts, layout) { + return PrepSource::ElfDependent(root); + } + panic!( + "a preprocessed sub-proof carries a root matching none of production's \ + candidate sources (BITWISE, KECCAK_RC, PAGE zero-init, CPU TIMESTAMP, \ + REGISTER-with-FINI, DECODE-from-ELF). The machine has no binding for it, \ + so it must not be hinted: extend the taxonomy (assembly ledger entry 7) \ + rather than this list" + ); +} + +pub(super) struct RealEpoch { + pub(super) statement: super::statement_replay::EpochStatementShape, + elf_digest: [u8; 32], + pub(super) public_output: Vec, + epoch_label: u64, + /// Per table, in sub-proof order: the preprocessed commitment and WHERE IT + /// COMES FROM (when the AIR is preprocessed), and the proof's main trace root. + phase_a: Vec<(Option, Commitment)>, + /// The epoch's INIT register file — production's `register_init`. The whole + /// vector, not just the carried commit index: the REGISTER preprocessed + /// commitment is derived from it. + register_init: Vec, + /// The epoch's FINAL register file, the other half of that derivation. + reg_fini: Vec, + /// The inner ELF's entry point — `program_id`'s `pc_start`. + pc_start: u64, + /// The ELF-data page genesis roots the attestation folds. EMPTY for a + /// continuation epoch's own verification: continuation epochs carry no PAGE + /// sub-proof at all (`continuation.rs:695-702`), so these belong to the + /// GLOBAL proof and reach the fold from outside. + page_commitments: Vec<(u64, Commitment)>, + /// The inner proof's LDE domain, for the REGISTER derivation. Both fields are + /// proof OPTIONS, hence program shape. + reg_shape: super::programs::RegisterDerivationShape, + /// `recursion::program_id_from_digest` over this epoch's own inputs — the + /// oracle for the attestation fold. + pub(super) expected_program_id: [u8; 32], + /// Per table, everything the fork absorbs plus the oracle challenges. + pub(super) tables: Vec, + /// Per table, everything the VERIFICATION LEGS read — the shapes, the + /// constraint analysis and the per-query openings. Built in the same pass as + /// `tables` because it needs the AIRs and the proof view, which do not + /// outlive this function. + pub(super) legs: Vec, + /// The shared LogUp challenges Phase A ends on. + pub(super) z_alpha: (FEE, FEE), + /// The carried commit index — `reg_init[X254_INDEX]` of this epoch, which + /// is the PREVIOUS epoch's `reg_fini[64]`. + pub(super) start_index: u64, + /// The COMMIT-bus target production computed, and therefore the value the + /// closure must reach. + pub(super) expected_bus_balance: FEE, +} + +pub(super) fn real_epoch() -> RealEpoch { + real_epoch_with(super::proof_fixture::fixture_options()) +} + +/// What an epoch is built FROM: the inner guest, its private input, and the +/// epoch size. Separated from the proof options because the two axes are +/// independent — options change what verifying the epoch costs, these change +/// what the epoch IS. +/// +/// [`EpochInputs::fixture`] is the continuation fixture every existing test +/// builds; [`EpochInputs::from_env`] is that with the three overrides a +/// measurement run needs, and it is what [`real_epoch_with`] uses, so with +/// nothing set every caller keeps the exact path it had. +pub(super) struct EpochInputs { + pub(super) elf_bytes: Vec, + pub(super) private_input: Vec, + pub(super) epoch_log2: u32, + /// Names the guest in printed measurements, since a real-block run and the + /// fixture otherwise report identically shaped numbers. + pub(super) label: String, +} + +impl EpochInputs { + /// The continuation fixture: the ELF the recursion suite builds, no private + /// input, [`FIXTURE_EPOCH_LOG2`](super::proof_fixture::FIXTURE_EPOCH_LOG2). + /// + /// ★ The guest is [`FIXTURE_INNER_ELF`](super::proof_fixture::FIXTURE_INNER_ELF), + /// which commits and then keeps running, so the epoch this builds — the + /// INTERMEDIATE one [`EpochFront::build`] insists on — actually carries + /// public output. Under a guest that commits immediately before halting it + /// does not, and every test that tampers `public_output` is vacuous. + pub(super) fn fixture() -> Self { + Self { + elf_bytes: super::proof_fixture::read_inner_elf(), + private_input: Vec::new(), + epoch_log2: super::proof_fixture::FIXTURE_EPOCH_LOG2, + label: format!("{} fixture", super::proof_fixture::FIXTURE_INNER_ELF), + } + } + + /// [`EpochInputs::fixture`] with the measurement overrides applied: + /// + /// - `LFM_CENSUS_ELF` — path to the inner guest ELF. + /// - `LFM_CENSUS_INPUT` — path to a file holding its private input. + /// - `LFM_CENSUS_EPOCH_LOG2` — epoch size, log2. + /// + /// The input override exists because a guest's epoch count is a property of + /// its INPUT, not just its ELF: the fixture guest reads its iteration + /// count from private input, so a run that needs a multi-epoch execution has + /// to be able to ask for one without a recompile. The ELF and epoch-size + /// overrides are what let the same harness build a real Ethereum-block + /// epoch, which is far too large to be a checked-in fixture. + /// + /// With none set this is byte-for-byte [`EpochInputs::fixture`]. + pub(super) fn from_env() -> Self { + let mut inputs = Self::fixture(); + if let Ok(p) = std::env::var("LFM_CENSUS_ELF") { + inputs.elf_bytes = + std::fs::read(&p).unwrap_or_else(|e| panic!("LFM_CENSUS_ELF {p}: {e}")); + inputs.label = p; + } + if let Ok(p) = std::env::var("LFM_CENSUS_INPUT") { + inputs.private_input = + std::fs::read(&p).unwrap_or_else(|e| panic!("LFM_CENSUS_INPUT {p}: {e}")); + } + if let Ok(v) = std::env::var("LFM_CENSUS_EPOCH_LOG2") { + inputs.epoch_log2 = v + .parse() + .unwrap_or_else(|e| panic!("LFM_CENSUS_EPOCH_LOG2 {v}: {e}")); + } + inputs + } +} + +/// [`real_epoch`] under supplied proof options — the wrap run's blowup axis. +/// +/// The options are the INNER proof's, so they change what the verifier has to do: +/// the query count, the LDE depth every Merkle walk climbs, and how many FRI +/// layers commit. What the epoch IS comes from [`EpochInputs::from_env`], which +/// is the continuation fixture unless a measurement run overrode it — so two runs +/// at different options stay comparable, and assembly ledger entry 10 still +/// holds: the trace-length profile travels with every number. +pub(super) fn real_epoch_with(opts: crate::ProofOptions) -> RealEpoch { + real_epoch_from(opts, EpochInputs::from_env()) +} + +/// The statement-seeded transcript every prover and every verifier of one +/// epoch starts from. One function rather than per-harness closures so no two +/// harnesses can drift on the absorb — a drift here would fail neither +/// harness's own gate; it would just make their proofs answer different +/// statements, which is a failure no harness can detect from inside. +pub(super) fn epoch_seed( + epoch_label: u64, + elf_bytes: &[u8], + public_output: &[u8], + table_counts: &crate::TableCounts, + runtime_page_ranges: &[crate::RuntimePageRange], + fri_final_poly_log_degree: u8, + is_final: bool, +) -> crate::hash_pin::BlockTranscript { + let mut t = crate::hash_pin::block_transcript(&[]); + crate::statement::absorb_statement( + &mut t, + crate::statement::StatementKind::ContinuationEpoch { + epoch_label, + is_final, + }, + elf_bytes, + public_output, + table_counts, + 0, + runtime_page_ranges, + fri_final_poly_log_degree, + ); + t +} + +/// The census-env epoch-0 construction the per-table harness +/// ([`real_epoch_from`]) is built on: same executor slice, same traces, same +/// L2G bookend, same statement-seeded transcript. Sharing this front is what +/// makes "same epoch, different prove" structural rather than by-inspection +/// for any two runs this file hosts. +/// +/// Only epoch 0 is built: the boundary starts from genesis provenance and the +/// label is `epoch_label(0)`, so a later epoch would need the previous one's +/// provenance carried in. That is a real limit of this harness and not an +/// oversight — the first epoch is what the compression work needs. +pub(super) struct EpochFront { + pub(super) opts: crate::ProofOptions, + pub(super) elf_bytes: Vec, + pub(super) elf: executor::elf::Elf, + pub(super) guest_label: String, + pub(super) epoch_log2: u32, + /// How many cycles the executor actually ran — reported alongside the + /// prove, since a duration without its workload identifies nothing. + pub(super) cycles_executed: usize, + pub(super) traces: crate::tables::trace_builder::Traces, + pub(super) airs: crate::VmAirs, + pub(super) l2g_air: Box>, + pub(super) l2g_trace: stark::trace::TraceTable, + pub(super) register_init: Vec, + pub(super) reg_fini: Vec, + pub(super) table_counts: crate::TableCounts, + pub(super) public_output: Vec, + pub(super) runtime_page_ranges: Vec, + /// `epoch_label(0)`. + pub(super) label: u64, + /// The attestation fold's DECODE input, from PRODUCTION's own function — + /// the same value `VmAirs::new` puts on the DECODE AIR, and the same one + /// `recursion::check_attestation` recomputes from a trusted ELF. + pub(super) decode_root: Commitment, +} + +impl EpochFront { + pub(super) fn build(opts: crate::ProofOptions, inputs: EpochInputs) -> Self { + use crate::tables::trace_builder::{Traces, build_initial_image_paged}; + use crate::tables::{MaxRowsConfig, bitwise, local_to_global, register}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let EpochInputs { + elf_bytes, + private_input, + epoch_log2, + label: guest_label, + } = inputs; + let elf = Elf::load(&elf_bytes).expect("the inner ELF must load"); + let epoch_size = 1usize << epoch_log2; + + let mut executor = Executor::new(&elf, private_input.clone()).expect("executor"); + let image = build_initial_image_paged(&elf, &private_input); + let register_init = register::register_init_from_entry_point(elf.entry_point); + let logs = executor + .resume_with_limit(epoch_size) + .expect("resume") + .expect("the guest runs at least one epoch") + .to_vec(); + let is_final = executor.pc() == 0; + assert!(!is_final, "wanted an INTERMEDIATE epoch"); + + let mut traces = Traces::from_image_and_logs( + &elf, + &image, + ®ister_init, + &logs, + &MaxRowsConfig::default(), + &private_input, + is_final, + true, + ) + .expect("the epoch trace must build"); + + let label = local_to_global::epoch_label(0); + let mut provenance = + local_to_global::genesis_provenance(image.iter().map(|(a, v)| (a, v as u64))); + let boundary = + local_to_global::epoch_boundary(&mut provenance, label, &traces.touched_memory_cells); + bitwise::update_multiplicities( + &mut traces.bitwise, + &local_to_global::collect_bitwise_from_l2g(&boundary), + ); + + let reg_fini = register::fini_from_trace(&traces.register); + let table_counts = traces.table_counts(); + let public_output = traces.public_output_bytes.clone(); + let runtime_page_ranges = traces.runtime_page_ranges(); + + let airs = crate::VmAirs::new( + &elf, + &opts, + false, + &[], + &table_counts, + &crate::cpu_heights_from_traces(&traces), + None, + is_final, + None, + crate::RegisterPreprocessed { + commitment: register::compute_precomputed_commitment( + &opts, + ®ister_init, + ®_fini, + ), + init: ®ister_init, + fini: ®_fini, + }, + ); + let decode_root = crate::tables::decode::commitment_from_elf(&elf, &opts) + .expect("the DECODE commitment must compute"); + let l2g_air = Box::new(crate::continuation::l2g_memory_air(&opts, label)); + let l2g_trace = local_to_global::generate_local_to_global_trace(&boundary); + + Self { + opts, + elf_bytes, + elf, + guest_label, + epoch_log2, + cycles_executed: logs.len(), + traces, + airs, + l2g_air, + l2g_trace, + register_init, + reg_fini, + table_counts, + public_output, + runtime_page_ranges, + label, + decode_root, + } + } + + /// [`epoch_seed`] over this epoch's own statement. + pub(super) fn seed(&self) -> crate::hash_pin::BlockTranscript { + epoch_seed( + self.label, + &self.elf_bytes, + &self.public_output, + &self.table_counts, + &self.runtime_page_ranges, + self.opts.fri_final_poly_log_degree, + // HALT's presence IS the role: one source, so the seed cannot state + // a role the AIR set contradicts. + self.airs.include_halt, + ) + } + + /// Every table's `(air, trace, public inputs)` in sub-proof order — the + /// VM tables then the L2G bookend — the one argument both prove calls + /// take. + #[allow(clippy::type_complexity)] + pub(super) fn pairs( + &mut self, + ) -> Vec<( + &dyn AIR, + &mut stark::trace::TraceTable, + &(), + )> { + let mut pairs = self.airs.air_trace_pairs(&mut self.traces); + pairs.push((&*self.l2g_air, &mut self.l2g_trace, &())); + pairs + } +} + +/// [`real_epoch_with`] with the guest, its input and the epoch size supplied +/// rather than read from the environment. Construction is [`EpochFront`]; +/// this adds the per-table prove, production's acceptance, and the replay +/// harvest. +pub(super) fn real_epoch_from(opts: crate::ProofOptions, inputs: EpochInputs) -> RealEpoch { + use stark::proof::view::MultiProofView; + + let mut front = EpochFront::build(opts, inputs); + + let proof = { + let mut transcript = front.seed(); + let t = std::time::Instant::now(); + let proof = crate::test_utils::multi_prove_ram(front.pairs(), &mut transcript) + .expect("the epoch must prove"); + // The inner prove is the expensive half of a real-block run and is + // otherwise invisible inside the wrap's own timings, so it reports + // itself — with the guest and epoch size, since a number without them + // does not identify a workload. + eprintln!( + "inner epoch: {}, 2^{} cycles, {} cycles executed, \ + {} sub-proofs, proved in {:.1}s", + front.guest_label, + front.epoch_log2, + front.cycles_executed, + proof.proofs.len(), + t.elapsed().as_secs_f64() + ); + proof + }; + + // The traces fed the prove; drop them here — what follows reads the PROOF. + let EpochFront { + opts, + elf_bytes, + elf, + airs, + l2g_air, + register_init, + reg_fini, + table_counts, + public_output, + runtime_page_ranges, + label, + decode_root, + .. + } = front; + harvest_real_epoch( + &opts, + elf_bytes, + &elf, + &airs, + &*l2g_air, + register_init, + reg_fini, + table_counts, + public_output, + runtime_page_ranges, + label, + decode_root, + MultiProofView::Owned(&proof), + ) + .expect("production must accept the epoch this suite differentials against") +} + +/// The replay harvest shared by [`real_epoch_from`] (session path — the epoch +/// proved inline moments before) and [`real_epoch_from_continuation`] +/// (from-proof path — the epoch read off an existing continuation bundle): +/// everything a [`RealEpoch`] holds, read from the AIR set, the statement +/// values and the PROOF view. Checks production accepts the (AIRs, proof, +/// statement) triple first — `Err` otherwise, and nothing below the check +/// describes a real epoch. +#[allow(clippy::too_many_arguments)] +fn harvest_real_epoch( + opts: &crate::ProofOptions, + elf_bytes: Vec, + elf: &executor::elf::Elf, + airs: &crate::VmAirs, + l2g_air: &dyn AIR, + register_init: Vec, + reg_fini: Vec, + table_counts: crate::TableCounts, + public_output: Vec, + runtime_page_ranges: Vec, + label: u64, + decode_root: Commitment, + view: stark::proof::view::MultiProofView<'_, Gl, Ext3, ()>, +) -> Result { + use crate::tables::register; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use stark::verifier::IsStarkVerifier; + + let seed = || { + epoch_seed( + label, + &elf_bytes, + &public_output, + &table_counts, + &runtime_page_ranges, + opts.fri_final_poly_log_degree, + airs.include_halt, + ) + }; + let refs = { + let mut r = airs.air_refs(); + r.push(l2g_air); + r + }; + assert_eq!(refs.len(), view.len(), "one AIR per sub-proof"); + + // ---- production must ACCEPT it, or nothing below describes a real epoch. + // + // ★ TIMED, AND ON ITS OWN LINE, because this half and the replay below it + // have different owners. This is a complete host STARK verify of the base + // epoch's sub-proofs whose only use is to refuse — the harness will not + // read an epoch production rejects. The replay that follows is the Phase A + // walk and the per-table forks a driver genuinely needs. Reported as one + // `reconstruct` figure the two are indistinguishable, and the block's + // production-equivalent time was a BAND for exactly that reason. + // + // ⛔ Timed, never skipped and never gated: an epoch nothing verified is not + // a faster epoch, it is a different experiment. + let t_verify = std::time::Instant::now(); + let start_index = register_init[register::X254_INDEX] as u64; + let expected = crate::compute_expected_commit_bus_balance_view( + &refs, + view, + &public_output, + start_index, + &mut seed(), + ) + .ok_or("the COMMIT bus target must compute")?; + if !crate::hash_pin::BlockVerifier::::multi_verify_views( + &refs, + view, + &mut seed(), + &expected, + ) { + return Err("production's verifier rejects the epoch".to_string()); + } + let verify_secs = t_verify.elapsed().as_secs_f64(); + let t_replay = std::time::Instant::now(); + + // ---- Phase A, transcribed from `multi_verify_views:1160-1227`. + let mut transcript = seed(); + let mut phase_a = Vec::new(); + // S2: the REGISTER table's leaf layout (the in-circuit register commitment + // is emitted at its `rows_per_leaf`) and the DECODE root Phase A absorbs + // (the attestation folds that very root) — both at the table's resolved + // layout, which is today's row pair at the default format. + let mut register_layout = stark::leaf_layout::LeafLayout::RowPair; + let mut absorbed_decode_root = decode_root; + for (idx, air) in refs.iter().enumerate() { + let v = view.get(idx); + if air.is_preprocessed() { + let layout = stark::leaf_layout::table_leaf_layout(*air, v.trace_length()); + let prep = air + .precomputed_commitment_for(layout) + .unwrap_or_else(|| panic!("table {idx}: no precomputed root at {layout:?}")); + transcript.append_bytes(&prep); + transcript.append_bytes(v.lde_trace_main_merkle_root()); + let source = prep_source( + prep, + opts, + elf, + ®ister_init, + ®_fini, + v.trace_length(), + layout, + ); + match source { + PrepSource::Register(_) => register_layout = layout, + PrepSource::ElfDependent(root) => absorbed_decode_root = root, + PrepSource::Constant(_) => {} + } + phase_a.push((Some(source), *v.lde_trace_main_merkle_root())); + } else { + transcript.append_bytes(v.lde_trace_main_merkle_root()); + phase_a.push((None, *v.lde_trace_main_merkle_root())); + } + } + if opts.format.one_row == stark::proof::options::OneRowMode::Off { + assert_eq!( + absorbed_decode_root, decode_root, + "at the default format Phase A absorbs today's DECODE root" + ); + assert_eq!(register_layout, stark::leaf_layout::LeafLayout::RowPair); + } + let needs_lookup_challenges = refs.iter().any(|a| a.has_aux_trace()); + assert!(needs_lookup_challenges, "an epoch uses LogUp"); + let lookup_challenges: Vec = (0..stark::lookup::LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect(); + let z_alpha = (lookup_challenges[0], lookup_challenges[1]); + + // ---- one fork per table, and the rounds replayed on it. + let num_tables = refs.len(); + let tables = refs + .iter() + .enumerate() + .map(|(idx, air)| { + let v = view.get(idx); + let mut fork = transcript.clone(); + if num_tables > 1 { + fork.append_bytes(&(idx as u64).to_le_bytes()); + } + if let Some(root) = v.lde_trace_aux_merkle_root() { + fork.append_bytes(root); + } + if let Some(c) = v.bus_table_contribution() { + fork.append_field_element(&c); + } + host_table_forked(*air, v, idx, num_tables, &mut fork, &lookup_challenges) + }) + .collect(); + + // The legs' own reading of the same sub-proofs. Separate pass rather than + // part of `host_table_forked` because the challenge replay must run against + // a fork positioned exactly as production leaves it, and this reads nothing + // from the transcript at all. + let legs = refs + .iter() + .enumerate() + .map(|(idx, air)| { + super::epoch_verify_tests::build_table_legs(*air, view.get(idx), &lookup_challenges) + }) + .collect(); + + println!( + " epoch {label}: production verify {verify_secs:.2}s (harness-only) \ + · replay {:.2}s", + t_replay.elapsed().as_secs_f64() + ); + + Ok(RealEpoch { + statement: super::statement_replay::EpochStatementShape { + public_output_len: public_output.len(), + // ★ The single source, not a hand-copied order. This array WAS a + // third transcription of `TableCounts`' declaration order, and the + // main-sync port is what showed the cost: the host list grew by six + // and a copy like this would have gone on absorbing fourteen with + // nothing to say so but a diverged challenge. + table_counts: crate::statement::table_count_values(&table_counts), + num_private_input_pages: 0, + fri_final_poly_log_degree: opts.fri_final_poly_log_degree, + page_ranges: runtime_page_ranges + .iter() + .map(|r| (r.base, r.count)) + .collect(), + // From the AIR set rather than a parameter: `is_final` and HALT's + // presence are the same fact, and taking both would let them differ. + is_final: airs.include_halt, + }, + elf_digest: crate::statement::elf_digest(&elf_bytes), + public_output, + epoch_label: label, + phase_a, + register_init, + reg_fini, + pc_start: elf.entry_point, + // ★ EMPTY, and it is a claim about PRODUCTION rather than about this + // fixture: `prove_epoch` REJECTS an epoch with any PAGE config + // ("continuation epoch must have no PAGE configs (L2G bookend replaces + // PAGE)", `continuation.rs:695-702`) and both `build_epoch_airs` call + // sites pass `&[]`. The ELF-data page genesis roots the attestation folds + // are the GLOBAL proof's GlobalMemory AIRs' preprocessed commitments + // (`continuation.rs:997-1010`), never an epoch's. + page_commitments: Vec::new(), + reg_shape: super::programs::RegisterDerivationShape { + blowup: opts.blowup_factor as usize, + coset_offset: opts.coset_offset, + // The REGISTER table's own leaf layout: 2 at the default format. + rows_per_leaf: register_layout.rows_per_leaf(), + }, + // The attestation folds the DECODE root Phase A absorbed — the + // row-pair `decode_root` at the default format (asserted below), the + // DECODE table's one-row root when S2 resolves it to one row (the + // attestation id moves with the knob). + expected_program_id: crate::recursion::program_id_from_digest( + &crate::statement::elf_digest(&elf_bytes), + elf.entry_point, + &absorbed_decode_root, + &[], + ), + tables, + legs, + z_alpha, + start_index, + expected_bus_balance: expected, + }) +} + +/// [`RealEpoch`] for epoch `epoch_index` of an EXISTING continuation bundle — +/// the from-proof path. The AIR set and statement values come from +/// [`crate::continuation::reconstruct_epoch_airs`], the SAME reconstruction +/// `verify_epoch` runs; the chain position (INIT chained from the prior +/// epoch's bound fini, final = last, label = position) from +/// [`crate::continuation::epoch_chain_position`]; the replay harvest is +/// [`harvest_real_epoch`], shared with the session path. So a wrap can be +/// emitted for ANY epoch of a continuation — including the FINAL one, which +/// the session harness cannot build — from the proofs alone, with no live +/// proving session. +/// +/// `decode_commitment`: `Some` reuses a DECODE root computed once per bundle +/// (it is a function of (ELF, opts) only); `None` recomputes it here. +/// `Err` = malformed bundle, out-of-range index, or a proof production's +/// verifier rejects. +pub(super) fn real_epoch_from_continuation( + opts: &crate::ProofOptions, + elf_bytes: &[u8], + bundle: &crate::continuation::ContinuationProof, + epoch_index: usize, + decode_commitment: Option, +) -> Result { + let konsts = EpochConstants::load(elf_bytes, opts, decode_commitment)?; + real_epoch_from_constants(opts, &konsts, bundle, epoch_index) +} + +/// The two things an epoch harvest needs that are the SAME for every epoch of a +/// run: the parsed guest ELF and its DECODE preprocessed commitment. +/// +/// # Why this is a type and not two arguments +/// +/// Both are pure functions of the guest binary, and `commitment_from_elf` also +/// takes `opts` — so the commitment belongs to one (ELF, options) pair and to no +/// other. Passed as two arguments they could be handed in from different +/// sources; constructed together they cannot. `prove_continuation` hoists the +/// same pair and says so in a comment (`continuation.rs`, *"compute it once here +/// instead of once per epoch inside `build_epoch_airs`"*); the recursion driver +/// did not inherit the fix, and lane P measured the cost: with +/// `decode_commitment: None` a 19-epoch walk builds the DECODE commitment +/// **38 times** — once inside `reconstruct_epoch_airs` and once again in the +/// fallback below it — and re-parses the 3,448,712-byte ELF 19 times, for one +/// distinct value each. +pub(super) struct EpochConstants<'a> { + elf_bytes: &'a [u8], + elf: executor::elf::Elf, + decode_commitment: Commitment, +} + +impl<'a> EpochConstants<'a> { + /// Parse the ELF once and take its DECODE commitment once. + /// + /// `supplied` lets a caller that already has the commitment skip the build; + /// `None` builds it here, which is the one place a run should. + pub(super) fn load( + elf_bytes: &'a [u8], + opts: &crate::ProofOptions, + supplied: Option, + ) -> Result { + use executor::elf::Elf; + + let elf = Elf::load(elf_bytes).map_err(|e| format!("the inner ELF must load: {e}"))?; + let decode_commitment = match supplied { + Some(c) => c, + None => crate::tables::decode::commitment_from_elf(&elf, opts) + .map_err(|e| format!("DECODE commitment from ELF: {e}"))?, + }; + Ok(EpochConstants { + elf_bytes, + elf, + decode_commitment, + }) + } +} + +/// [`real_epoch_from_continuation`] for a caller walking EVERY epoch of one +/// bundle, which is every production caller. +/// +/// ⛔ The commitment reaches `reconstruct_epoch_airs` as `Some`, which is what +/// makes this a hoist rather than a rename: handed `None` that function builds +/// the commitment itself and the `match` below builds a second one, so the +/// saving is two builds per epoch, not one. +pub(super) fn real_epoch_from_constants( + opts: &crate::ProofOptions, + konsts: &EpochConstants<'_>, + bundle: &crate::continuation::ContinuationProof, + epoch_index: usize, +) -> Result { + real_epoch_from_epochs( + opts, + konsts, + bundle.epochs(), + bundle.num_epochs(), + epoch_index, + ) +} + +/// [`real_epoch_from_constants`] from the leading epochs of a run of +/// `num_epochs`, before its bundle exists — the same function over the same +/// proofs, which is what lets the level-0 lead-in build a wrap's prologue in the base's +/// tail. `epochs` must hold at least epochs `0..=epoch_index`. +pub(super) fn real_epoch_from_epochs( + opts: &crate::ProofOptions, + konsts: &EpochConstants<'_>, + epochs: &[crate::continuation::EpochProof], + num_epochs: usize, + epoch_index: usize, +) -> Result { + let elf = &konsts.elf; + let position = crate::continuation::epoch_chain_position(epochs, num_epochs, elf, epoch_index) + .map_err(|e| format!("chain position for epoch {epoch_index}: {e:?}"))? + .ok_or_else(|| format!("epoch {epoch_index} is out of range or the bundle is malformed"))?; + let view = crate::continuation::EpochProofView::Owned(&epochs[epoch_index]); + let recon = crate::continuation::reconstruct_epoch_airs( + elf, + view, + &position.register_init, + position.is_final, + position.label, + opts, + Some(konsts.decode_commitment), + ) + .map_err(|e| format!("reconstructing epoch {epoch_index}: {e:?}"))? + .ok_or_else(|| format!("epoch {epoch_index} is structurally invalid"))?; + harvest_real_epoch( + opts, + konsts.elf_bytes.to_vec(), + elf, + &recon.airs, + &*recon.l2g_air, + position.register_init, + recon.reg_fini, + recon.table_counts, + view.public_output().to_vec(), + recon.runtime_page_ranges, + position.label, + konsts.decode_commitment, + view.per_table_proof(), + ) +} + +/// Inner options for the P1 from-proof gates: MIN's cost profile but at +/// blowup 4, because the fixture epoch carries height-1 tables and the +/// per-table replay indexes each table's own LDE pairs +/// (`TableChallengeShape::index_bits` = `log2(h·blowup) − 1`, which must be +/// ≥ 1). Diagnostics-grade, not a security parameter set. +pub(super) fn from_proof_gate_options() -> crate::ProofOptions { + crate::ProofOptions { + blowup_factor: 4, + fri_number_of_queries: 2, + coset_offset: 3, + grinding_factor: 1, + fri_final_poly_log_degree: 7, + format: stark::proof::options::ProofFormat::DEFAULT, + } +} + +/// ★ GATE A (P1) — the from-proof constructor IS the session harness. +/// +/// Two constructions of the same fixture epoch: [`real_epoch_from`] proves +/// epoch 0 inline; [`real_epoch_from_continuation`] reads epoch 0 of the +/// fixture CONTINUATION bundle (same guest, same epoch size, same options). +/// The proof bytes differ between the two proves (grinding nonces move the +/// query openings), but everything shape- and statement-derived must agree — +/// and the emitted wrap programs must be byte-identical, checked through +/// their registry artifacts (the roots ARE commitments over the program + +/// preprocessed text). +/// +/// Epoch 1 — the bundle's FINAL epoch, the shape the real block's last epoch +/// has and the session harness cannot build (it wants an intermediate +/// epoch) — must also reconstruct, pass production's verifier inside the +/// constructor, and emit a program. +/// ★ THE LEAD-IN'S IDENTITY, at fixture scale. The base's observer is told the +/// epoch count exactly once and handed every epoch the bundle carries, byte for +/// byte; and a wrap prologue built from those copies — before any bundle +/// existed, from the leading epochs alone — is the one built from the bundle: +/// the same program and the same arenas. That is what lets level 0 take a +/// lead-in's prologue in place of its own (the block run's IDENTITY lines are +/// the same gate at scale). +#[test] +fn a_lead_in_prologue_is_the_one_built_from_the_bundle() { + let elf_bytes = super::proof_fixture::read_inner_elf(); + let opts = from_proof_gate_options(); + let recorder = std::sync::Arc::new(crate::continuation::RecordingObserver::< + crate::continuation::EpochProof, + >::default()); + let bundle = crate::continuation::with_epoch_observer(recorder.clone(), || { + crate::continuation::prove_continuation( + &elf_bytes, + &[], + super::proof_fixture::FIXTURE_EPOCH_LOG2, + &opts, + ) + }) + .expect("the fixture continuation must prove"); + let n = bundle.num_epochs(); + assert!(n >= 2, "the fixture continuation must have a final epoch"); + assert_eq!( + *recorder.counts.lock().unwrap(), + vec![n], + "the epoch count, exactly once" + ); + let shared = std::mem::take(&mut *recorder.shared.lock().unwrap()); + assert_eq!(shared.len(), 1, "the DECODE commitment is shared once"); + let shared = *shared[0] + .clone() + .downcast::() + .unwrap_or_else(|_| panic!("the STARK base shares its DECODE commitment")); + let elf = executor::elf::Elf::load(&elf_bytes).expect("the fixture ELF"); + assert_eq!( + shared, + crate::tables::decode::commitment_from_elf(&elf, &opts).expect("the DECODE commitment"), + "the shared commitment is the ELF's" + ); + let copies = recorder.epochs_in_order(n); + for (i, (copy, kept)) in copies.iter().zip(bundle.epochs()).enumerate() { + assert_eq!( + rkyv::to_bytes::(copy) + .expect("serialize") + .as_slice(), + rkyv::to_bytes::(kept) + .expect("serialize") + .as_slice(), + "the observer's copy of epoch {i} is the bundle's" + ); + } + + // The lead-in's constants take the shared commitment; the pool's derive it. + let konsts = EpochConstants::load(&elf_bytes, &opts, None).expect("the fixture ELF"); + let lead_konsts = + EpochConstants::load(&elf_bytes, &opts, Some(shared)).expect("the fixture ELF"); + for k in 0..n { + let from_bundle = real_epoch_from_constants(&opts, &konsts, &bundle, k) + .unwrap_or_else(|e| panic!("epoch {k} from the bundle: {e}")); + let from_lead = real_epoch_from_epochs(&opts, &lead_konsts, &copies[..=k], n, k) + .unwrap_or_else(|e| panic!("epoch {k} from its leading epochs: {e}")); + assert_eq!( + format!( + "{:?}", + epoch_program_publishing(&from_bundle, true, Publishes::Aggregation) + ), + format!( + "{:?}", + epoch_program_publishing(&from_lead, true, Publishes::Aggregation) + ), + "wrap {k}'s program" + ); + assert_eq!( + epoch_arena_words(&from_bundle, true), + epoch_arena_words(&from_lead, true), + "wrap {k}'s arenas" + ); + } + // ⛔ And the leading epochs must reach the one asked for: a list that stops + // short is refused, never read as a shorter run. + assert!( + real_epoch_from_epochs(&opts, &lead_konsts, &copies[..1], n, 1).is_err(), + "epoch 1 cannot be built from epoch 0 alone" + ); +} + +#[test] +fn the_from_proof_constructor_matches_the_session_harness() { + let elf_bytes = super::proof_fixture::read_inner_elf(); + let opts = from_proof_gate_options(); + + let session = real_epoch_from(from_proof_gate_options(), EpochInputs::fixture()); + let bundle = crate::continuation::prove_continuation( + &elf_bytes, + &[], + super::proof_fixture::FIXTURE_EPOCH_LOG2, + &opts, + ) + .expect("the fixture continuation must prove"); + assert!( + bundle.num_epochs() >= 2, + "the fixture continuation must have a second (final) epoch — \ + re-measure FIXTURE_EPOCH_LOG2 (see proof_fixture)" + ); + + let from_proof = real_epoch_from_continuation(&opts, &elf_bytes, &bundle, 0, None) + .expect("epoch 0 must reconstruct from the bundle alone"); + + // Statement-level scalars: two paths, one epoch. + assert_eq!(session.register_init, from_proof.register_init); + assert_eq!(session.reg_fini, from_proof.reg_fini); + assert_eq!(session.public_output, from_proof.public_output); + assert_eq!(session.epoch_label, from_proof.epoch_label); + assert_eq!(session.start_index, from_proof.start_index); + assert_eq!(session.expected_program_id, from_proof.expected_program_id); + + // Preprocessed PROVENANCE is shape/statement-derived and must agree per + // slot. The committed MAIN roots are allowed to differ: the session + // harness builds traces through `Traces::from_image_and_logs` and + // `prove_continuation` through `Traces::build_from_collected`, and the + // two builders may make different (equally valid, production-accepted) + // byte choices inside one table — both proofs pass `multi_verify_views` + // above. A root difference therefore also moves the transcript-derived + // oracles (`z_alpha`, the bus target), so those are per-proof values, + // not cross-path invariants. What MUST be invariant is the program. + assert_eq!(session.phase_a.len(), from_proof.phase_a.len()); + for (i, (s, f)) in session.phase_a.iter().zip(&from_proof.phase_a).enumerate() { + assert_eq!(s.0, f.0, "preprocessed provenance differs at sub-proof {i}"); + if s.1 != f.1 { + println!( + " note: sub-proof {i}'s committed main root differs between \ + the two trace builders (both production-accepted)" + ); + } + } + + // The emitted wrap programs, byte-identical through the registry artifacts. + let p_session = epoch_program(&session, true); + let p_proof = epoch_program(&from_proof, true); + assert_eq!(p_session.instrs.len(), p_proof.instrs.len()); + let a_session = super::registry::build_artifacts(&p_session, &opts); + let a_proof = super::registry::build_artifacts(&p_proof, &opts); + assert_eq!( + a_session.roots, a_proof.roots, + "the two paths must emit byte-identical wrap programs" + ); + assert_eq!(a_session.program_id, a_proof.program_id); + assert_eq!(a_session.log_heights, a_proof.log_heights); + + // Epoch 1: the FINAL shape, from proofs alone. The constructor's internal + // production-verify is the acceptance gate; emission must handle the + // final-epoch table set (HALT on board). + let final_epoch = real_epoch_from_continuation(&opts, &elf_bytes, &bundle, 1, None) + .expect("the FINAL epoch must reconstruct from the bundle alone"); + let p_final = epoch_program(&final_epoch, true); + let a_final = super::registry::build_artifacts(&p_final, &opts); + println!( + "★ P1 GATE A: epoch 0 program identical across both paths ({} instrs); \ + FINAL epoch reconstructed and emitted ({} instrs, {} chips)", + p_proof.instrs.len(), + p_final.instrs.len(), + a_final.log_heights.len(), + ); +} + +/// ★ GATE C (P1) — one corrupted byte of the bundle, rejected, and WHERE. +/// +/// `reg_fini` is a prover-supplied bundle field the verifier re-binds through +/// the REGISTER preprocessed commitment: corrupting it makes the +/// constructor's own production-verify reject (the constructor layer). The +/// claimed `l2g_root` is not an AIR input — the constructor does not read +/// it — so its corruption is caught by `verify_continuation`'s root re-check +/// instead (the bundle-verify layer the block driver runs first). +#[test] +fn the_from_proof_constructor_rejects_a_tampered_bundle() { + let elf_bytes = super::proof_fixture::read_inner_elf(); + let opts = super::proof_fixture::fixture_options(); + let mut bundle = crate::continuation::prove_continuation( + &elf_bytes, + &[], + super::proof_fixture::FIXTURE_EPOCH_LOG2, + &opts, + ) + .expect("the fixture continuation must prove"); + assert!(bundle.num_epochs() >= 2); + + // Layer 1: the constructor's production-verify. + bundle.corrupt_epoch_reg_fini_for_tests(1); + let err = match real_epoch_from_continuation(&opts, &elf_bytes, &bundle, 1, None) { + Err(e) => e, + Ok(_) => panic!("a corrupted reg_fini must not reconstruct"), + }; + assert!( + err.contains("rejects"), + "the corruption is caught by the production verify inside the constructor: {err}" + ); + bundle.corrupt_epoch_reg_fini_for_tests(1); + + // Layer 2: the bundle verifier's claimed-root re-check. + bundle.corrupt_epoch_l2g_root_for_tests(1); + assert!( + crate::continuation::verify_continuation(&elf_bytes, &bundle, &opts) + .expect("the tampered bundle is still well-formed") + .is_none(), + "a corrupted claimed L2G root must fail bundle verification" + ); + println!( + "★ P1 GATE C: reg_fini corruption rejected by the constructor; \ + l2g_root corruption rejected by the bundle verifier" + ); +} + +/// [`host_table`] for a sub-proof inside a multi-table epoch +/// ([`super::harvest::host_table_forked`]), panicking on a proof that +/// disagrees with its AIR. +pub(super) fn host_table_forked( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + index: usize, + num_tables: usize, + fork: &mut crate::hash_pin::BlockTranscript, + lookup_challenges: &[FEE], +) -> HostTable { + super::harvest::host_table_forked(air, view, index, num_tables, fork, lookup_challenges) + .unwrap_or_else(|e| panic!("{e}")) +} + +/// The whole epoch's Fiat-Shamir spine, as one program. +/// +/// Statement, then Phase A over every sub-proof, then a fork per table and its +/// rounds 2-4. This is the assembled verifier's skeleton: what hangs off each +/// fork (constraint evaluation, the query legs, the closure) consumes the cells +/// this returns. +/// +/// ## ⚠ The preprocessed commitments are HINTED here, and they must not stay so +/// +/// Production takes each preprocessed root from the AIR +/// (`verifier.rs:1187`), never from the proof, and rejects the sub-proof unless +/// the proof's copy matches. Only one of those roots has an in-machine +/// derivation today — REGISTER's, from the previous epoch's `reg_fini` +/// (`programs::register_derivation_program`). The others (BITWISE, DECODE, +/// KECCAK_RC, PAGE) are hinted, and PAGE's in particular CANNOT become a +/// program constant: it is a function of the inner ELF, which is per-proof arena +/// data. Baking it would make program identity proof-dependent. So each is a +/// derivation the assembly still owes; see the ledger entry this leg added. +fn epoch_challenge_program(e: &RealEpoch) -> LfmProgram { + epoch_program(e, false) +} + +/// The epoch program, with or without the verification LEGS hung off the spine. +/// +/// One emitter for both, deliberately. A second copy of the spine would be a +/// place for the assembled verifier's Fiat-Shamir to drift from the one +/// `the_epoch_challenge_spine_matches_production` checks against production — +/// and drift is exactly what the leg wiring must not introduce, since every leg +/// consumes the cells this spine bound. `with_legs = false` declares no leg +/// arenas and emits no verification, so the spine test's own arena-word count is +/// untouched. +pub(super) fn epoch_program(e: &RealEpoch, with_legs: bool) -> LfmProgram { + epoch_program_with(e, with_legs, false, Publishes::Diagnostic) +} + +/// Which words [`epoch_program`] publishes. +/// +/// The wrap's published words are the ONLY thing an aggregation node can read +/// about it, and a node pays for every one of them: eight hinted halves, four +/// canonicity guards, four recombinations, thirty-six bytes of statement absorb +/// and one extension-field inverse in the `LfmPublic` balance — per word, per +/// child. So what a wrap publishes is the size of the layer above it. +/// +/// At the production posture the diagnostic set is 10,507 words, of which the +/// binding set is ≈80. The other ~10,400 have no consumer above: `composition` +/// is already asserted equal to the claimed Horner INSIDE +/// `epoch_verify::emit_table_verification`, the DEEP invariants are documented +/// as exposed for tests, and β/z/γ/ζ/ι are challenges the machine derives for +/// itself. They are an oracle for this crate's differentials, not a binding, and +/// a node that hinted them would be paying to re-read numbers it could recompute. +/// +/// ⚠ [`Publishes::Diagnostic`] is the DEFAULT and every existing gate keeps +/// exactly the words it had. Dropping the differential surface is a choice made +/// per emission by the aggregator, never a global one: the differentials against +/// production's own replay are how this machine is known to derive production's +/// challenges, and a preset that quietly removed them everywhere would trade the +/// evidence for the saving. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum Publishes { + /// The shared pair, the attestation id, the block-binding schema, then every + /// per-sub-proof diagnostic, then the bus total. + Diagnostic, + /// The shared pair, the attestation id, the block-binding schema and the bus + /// total — nothing per sub-proof. What a wrap feeding an aggregator emits. + Aggregation, +} + +/// [`epoch_program`] with the publish set named — the aggregation lever. +pub(super) fn epoch_program_publishing( + e: &RealEpoch, + with_legs: bool, + publishes: Publishes, +) -> LfmProgram { + epoch_program_with(e, with_legs, false, publishes) +} + +/// The epoch program, optionally with the DECODE cell SPLIT — a deliberately +/// broken control, and the falsification the entry-7 ruling asked for. +/// +/// `split_decode = true` gives the attestation fold its own arena copy of the +/// DECODE root instead of the cell Phase A absorbed. Nothing about the program +/// then looks wrong: every assert still passes, the challenges are still +/// production's, and an honest host that fills both copies with the same 32 bytes +/// gets the same published `program_id`. That is exactly why the join has to be +/// denied STRUCTURALLY rather than by a differential — +/// [`a_split_decode_cell_forges_the_attestation`] runs the coherent forgery this +/// admits, and +/// [`the_assembled_verifier_declares_exactly_the_shape_words`] is what refuses it. +/// +/// The extra arena is declared LAST so no existing arena index moves. +fn epoch_program_with( + e: &RealEpoch, + with_legs: bool, + split_decode: bool, + publishes: Publishes, +) -> LfmProgram { + use super::statement_replay::{EpochStatementVars, PhaseATable, absorb_epoch_statement}; + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let n = e.tables.len(); + assert_eq!(e.legs.len(), n, "one leg reading per sub-proof"); + + // ---- arenas, in declaration order ---- + let stmt_halves = 8 + e.statement.public_output_len.div_ceil(4) + 2; + let a_stmt = b.declare_arena(stmt_halves as u32); + // ★ Only the ELF-DEPENDENT preprocessed roots are arena data (ledger entry + // 7). The options-only ones are interned as program text and the REGISTER one + // is derived in-machine, so neither takes a word here. + let num_arena_prep = e + .phase_a + .iter() + .filter(|(p, _)| p.is_some_and(PrepSource::is_arena)) + .count(); + let a_prep_roots = + b.declare_arena(super::proof_arena::words_per_root() as u32 * num_arena_prep as u32); + let a_main_roots = b.declare_arena(super::proof_arena::words_per_root() as u32 * n as u32); + // The register boundary vectors, at production's width. `start_index` is slot + // 64 of INIT, and the REGISTER preprocessed root is COMPUTED from both — which + // is what ties the index to the chain (ledger entry 2): production has no + // arithmetic `start + len` check anywhere, it rebuilds the commitment from + // these vectors and rejects unless the absorbed root matches. + let num_reg = crate::tables::register::NUM_REGISTER_ADDRESSES as u32; + let a_reg_init = b.declare_arena(num_reg); + let a_reg_fini = b.declare_arena(num_reg); + // The attestation fold's own inputs. `elf_digest` is NOT here — it is the + // statement's, which is the join. `pc_start` has one consumer in an epoch + // verifier, and the page roots have none at all (a continuation epoch carries + // no PAGE sub-proof), so both are plain proof data the fold hashes. + let a_pc_start = b.declare_arena(2); + let a_page_roots = (!e.page_commitments.is_empty()) + .then(|| b.declare_arena(10 * e.page_commitments.len() as u32)); + let per_table: Vec = e + .tables + .iter() + .zip(&e.legs) + .map(|(h, leg)| Arenas { + main_root: a_main_roots, + aux_root: h + .shape + .has_aux_root + .then(|| b.declare_arena(super::proof_arena::words_per_root() as u32)), + contribution: h.shape.has_contribution.then(|| b.declare_arena(1)), + composition_root: b.declare_arena(super::proof_arena::words_per_root() as u32), + ood_current: b + .declare_arena((h.shape.ood_current_dims.0 * h.shape.ood_current_dims.1) as u32), + ood_next: b.declare_arena((h.shape.ood_next_dims.0 * h.shape.ood_next_dims.1) as u32), + parts: b.declare_arena(h.shape.num_parts as u32), + fri_roots: b.declare_arena( + super::proof_arena::words_per_root() as u32 * h.shape.fri.num_committed() as u32, + ), + fri_coeffs: b.declare_arena(h.shape.fri.num_terminal_coeffs() as u32), + nonce: (h.shape.grinding_factor > 0).then(|| b.declare_arena(1)), + legs: with_legs.then(|| super::epoch_verify::declare_table_arenas(&mut b, &leg.verify)), + }) + .collect(); + // Last in declaration order, so turning the control on shifts no other arena. + // One root, at THIS builder's digest width — the width `RootCells::hint` + // reads it back at. + let root_words = RootCells::words_per_root(&b); + let a_split_decode = split_decode.then(|| b.declare_arena(root_words)); + + // ---- the statement ---- + let stmt: Vec<_> = (0..stmt_halves as u32) + .map(|i| b.hint_felt(a_stmt, i)) + .collect(); + let out_halves = e.statement.public_output_len.div_ceil(4); + let (elf_digest, rest) = stmt.split_at(8); + let (public_output, epoch_label) = rest.split_at(out_halves); + + let mut t = TranscriptReplay::new(&[]); + absorb_epoch_statement( + &mut t, + &e.statement, + &EpochStatementVars { + elf_digest, + public_output, + epoch_label, + }, + ); + + // ---- ★ the preprocessed roots, each from the source its provenance admits + // + // Ledger entry 7, and entry 2 closes with it. `PrepSource` was decided + // host-side by recomputing production's candidate functions, so the split here + // is not a hardcoded sub-proof index: a preprocessed table with an unknown + // provenance would already have panicked. + let reg_init: Vec<_> = (0..num_reg).map(|r| b.hint_felt(a_reg_init, r)).collect(); + let reg_fini: Vec<_> = (0..num_reg).map(|r| b.hint_felt(a_reg_fini, r)).collect(); + // ★ LEDGER ENTRY 1. Production's boundary vectors are `Vec` and the TYPE + // is the whole enforcement; an arena is untyped felts, so without this the + // machine would derive a commitment over a value no production epoch can hold. + // The entry's stated default was "emit the check if the no->u32 argument is + // still unverified when assembly arrives" — it is, and assembly has arrived. + for cell in reg_init.iter().chain(®_fini) { + super::epoch::assert_u32(&mut b, *cell); + } + let reg_shape = e.reg_shape; + + let mut next_arena_prep = 0usize; + let mut decode_cells: Option = None; + let prep_cells: Vec> = e + .phase_a + .iter() + .map(|(prep, _)| match prep { + None => None, + Some(PrepSource::Constant(c)) => Some(RootCells::constant(&mut b, c)), + Some(PrepSource::Register(_)) => { + let digest = super::programs::emit_register_commitment( + &mut b, reg_shape, ®_init, ®_fini, + ); + Some(RootCells::from_digest(&mut b, digest)) + } + Some(PrepSource::ElfDependent(_)) => { + let cells = RootCells::hint( + &mut b, + a_prep_roots, + super::proof_arena::words_per_root() as u32 * next_arena_prep as u32, + ); + next_arena_prep += 1; + // Every ELF-dependent root of a continuation EPOCH is DECODE (the + // page family lives in the global proof), and the attestation + // folds exactly one DECODE root — so a second one here would mean + // the fold's input is ambiguous, not that the fold needs a loop. + assert!( + decode_cells.is_none(), + "a continuation epoch has one ELF-dependent preprocessed root \ + (DECODE); a second one has no place in the program_id fold" + ); + decode_cells = Some(cells.clone()); + Some(cells) + } + }) + .collect(); + assert_eq!( + next_arena_prep, num_arena_prep, + "every declared preprocessed arena word must be read" + ); + + // ---- Phase A ---- + let main_cells: Vec = (0..n) + .map(|i| { + RootCells::hint( + &mut b, + a_main_roots, + super::proof_arena::words_per_root() as u32 * i as u32, + ) + }) + .collect(); + // ⚠ `byte_halves`, not `lanes_flat`: Phase A absorbs a root through + // `append_halves_misaligned`, whose byte length is `4 · halves.len()`, and + // the host absorbs the root's THIRTY-TWO bytes in one `append_bytes`. On an + // algebraic arm `lanes_flat` is four FULL FELTS, so that call would declare + // sixteen bytes where the host declared thirty-two — a different length + // ⚠ The DIGEST's felts, not the root's bytes. `replay_phase_a` absorbs + // through `absorb_root_felts`, which declares the host's 32 bytes on both + // arms and packs the algebraic arm's four felts into the one digest cell + // they already are — so the root absorb CANCELS there rather than paying a + // byte regrouping. `byte_halves` is for `program_id`, which is deliberately + // keccak-over-bytes; handing it here would regroup felts the host never + // serialised. + let prep_halves: Vec>> = prep_cells + .iter() + .map(|c| c.as_ref().map(RootCells::lanes_flat)) + .collect(); + let main_halves: Vec> = main_cells.iter().map(RootCells::lanes_flat).collect(); + // The interned bytes, hoisted so Phase A can borrow them for the whole replay. + let prep_constants: Vec> = e + .phase_a + .iter() + .map(|(p, _)| match p { + Some(PrepSource::Constant(c)) => Some(*c), + _ => None, + }) + .collect(); + let tables: Vec = (0..n) + .map(|i| PhaseATable { + // A program-text root absorbs as literal BYTES — no splice arithmetic + // at all, which is the whole economy of interning it. A derived or + // supplied one absorbs as the cells its consumers share. + preprocessed_root: match (&prep_constants[i], &prep_halves[i]) { + (Some(bytes), _) => { + Some(super::statement_replay::PhaseAPreprocessed::Constant(bytes)) + } + (None, Some(halves)) => Some(super::statement_replay::PhaseAPreprocessed::Cells( + &halves[..], + )), + (None, None) => None, + }, + main_root: &main_halves[i][..], + }) + .collect(); + let (z, alpha) = super::statement_replay::replay_phase_a(&mut t, &mut b, &tables); + b.public(z.as_cell()); + b.public(alpha.as_cell()); + + // ---- ★ the attestation join: the DECODE cell Phase A absorbed, folded + // + // One cell, two consumers. Without this the DECODE root would be a free arena + // word — the machine would absorb whatever the prover offered and publish + // nothing that depended on it. + { + let pc_start: Vec<_> = (0..2).map(|i| b.hint_felt(a_pc_start, i)).collect(); + let page_cells: Vec<(Vec<_>, RootCells)> = e + .page_commitments + .iter() + .enumerate() + .map(|(k, _)| { + let base = 10 * k as u32; + let arena = a_page_roots.expect("a page arena exists when pages do"); + let base_halves: Vec<_> = (0..2).map(|j| b.hint_felt(arena, base + j)).collect(); + let root_halves: Vec<_> = + (0..8).map(|j| b.hint_felt(arena, base + 2 + j)).collect(); + (base_halves, RootCells::from_halves(&mut b, &root_halves)) + }) + .collect(); + let page_halves: Vec<(Vec<_>, Vec<_>)> = page_cells + .iter() + .map(|(base, root)| (base.clone(), root.byte_halves(&mut b))) + .collect(); + let page_refs: Vec<(&[_], &[_])> = page_halves + .iter() + .map(|(base, root)| (&base[..], &root[..])) + .collect(); + let decode = match a_split_decode { + // ★ THE BROKEN CONTROL: a second, independent reading of the DECODE + // root. The fold now attests to a value Phase A never absorbed. + Some(arena) => RootCells::hint(&mut b, arena, 0).byte_halves(&mut b), + None => decode_cells + .as_ref() + .expect("a continuation epoch has a DECODE sub-proof") + .byte_halves(&mut b), + }; + let id = super::programs::emit_program_id( + &mut b, + super::programs::ProgramIdShape { + num_pages: e.page_commitments.len(), + }, + elf_digest, + &pc_start, + &decode, + &page_refs, + ); + b.public(id[0]); + b.public(id[1]); + } + + // ---- ★ THE BLOCK-BINDING SCHEMA, published right after the attestation id + // + // An aggregation node over these wraps sees exactly two things about a child: + // its `program_id`, which is an emit-time CONSTANT of the parent, and its + // PUBLISHED WORDS. Everything below is arena data that the epoch statement or + // Phase A already bound, and every one of them is material a node must CHECK + // rather than trust — so without these publishes the cross-wrap bindings are + // not weak, they are unbuildable. + // + // In the batched format these were the CARVED schema, published by the carve + // rather than by the program. The carve went with the format, so the program + // publishes them. + // + // Publishing costs no soundness, because each is already bound: the register + // vectors through the REGISTER preprocessed commitment DERIVED from them + // (`PrepSource::Register`), the label and the output halves through + // `absorb_epoch_statement`, and the L2G root through Phase A's absorb. + // + // ⚠ This order IS [`SchemaLayout`]'s, which the aggregator indexes by. A + // field inserted in the middle silently re-binds every field below it, so + // append here and extend `SchemaLayout` in the same commit. + for cell in reg_init.iter().chain(®_fini) { + b.public(cell.as_cell()); + } + for half in epoch_label { + b.public(half.as_cell()); + } + for half in public_output { + b.public(half.as_cell()); + } + // ★ The L2G bookend is the LAST sub-proof — `EpochSession::pairs` proves the + // VM tables and then pushes `l2g_air`, and the from-proof reconstruction + // rebuilds the AIR list the same way. Asserted rather than assumed: publishing + // some other sub-proof's root would bind the aggregator's L2G compare to a + // table with nothing to do with the global memory argument, and would still + // look like a passing gate. + assert_eq!( + e.tables[n - 1].shape.index, + n - 1, + "the L2G bookend is the epoch's last sub-proof" + ); + for half in main_cells[n - 1].lanes_flat() { + b.public(half.as_cell()); + } + + // ---- one fork per table ---- + let mut contributions: Vec = Vec::new(); + for (i, h) in e.tables.iter().enumerate() { + let a = &per_table[i]; + let aux = a.aux_root.map(|id| RootCells::hint(&mut b, id, 0)); + let contribution = a.contribution.map(|id| b.hint_word(id, 0).as_ext()); + let composition = RootCells::hint(&mut b, a.composition_root, 0); + let ood_current: Vec<_> = (0..(h.shape.ood_current_dims.0 * h.shape.ood_current_dims.1) + as u32) + .map(|k| b.hint_word(a.ood_current, k).as_ext()) + .collect(); + let ood_next: Vec<_> = (0..(h.shape.ood_next_dims.0 * h.shape.ood_next_dims.1) as u32) + .map(|k| b.hint_word(a.ood_next, k).as_ext()) + .collect(); + let parts: Vec<_> = (0..h.shape.num_parts as u32) + .map(|k| b.hint_word(a.parts, k).as_ext()) + .collect(); + let fri_roots: Vec<_> = (0..h.shape.fri.num_committed()) + .map(|k| { + RootCells::hint( + &mut b, + a.fri_roots, + super::proof_arena::words_per_root() as u32 * k as u32, + ) + }) + .collect(); + let fri_coeffs: Vec<_> = (0..h.shape.fri.num_terminal_coeffs() as u32) + .map(|k| b.hint_word(a.fri_coeffs, k).as_ext()) + .collect(); + let nonce = a.nonce.map(|id| b.hint_felt(id, 0)); + + if let Some(c) = contribution { + contributions.push(c); + } + let mut fork = fork_table(&t, h.shape.index, h.shape.num_tables); + let absorbs = TableAbsorbs { + aux_root: aux.as_ref(), + contribution, + composition_root: &composition, + ood_current: &ood_current, + ood_next: &ood_next, + parts: &parts, + fri_roots: &fri_roots, + fri_coeffs: &fri_coeffs, + nonce, + }; + let ch = emit_table_challenges(&mut b, &mut fork, &h.shape, &absorbs); + + // ---- ★ THE SEAM: the verification legs, on the cells just absorbed and + // the challenges just derived. `absorbs` is passed on by REFERENCE rather + // than rebuilt, so there is no second reading of the proof for a leg to + // disagree with the transcript about. + if let Some(leg_arenas) = &a.legs { + let leg = &e.legs[i]; + let out = super::epoch_verify::emit_table_verification( + &mut b, + &leg.verify, + &leg.analysis, + &ch, + &absorbs, + &super::epoch_verify::TableInputs { + // The precomputed root Phase A absorbed — the SAME cells, + // which is what makes production's explicit + // proof-copy-equals-AIR-copy check the absence of a second + // value here rather than a comparison. + precomputed_root: prep_cells[i].as_ref(), + main_root: &main_cells[i], + rap_challenges: &[z, alpha], + commit_ood: None, + }, + leg_arenas, + ); + if publishes == Publishes::Diagnostic { + b.public(out.composition.as_cell()); + for v in &out.fri_terminal { + b.public(v.as_cell()); + } + } + } + if publishes == Publishes::Diagnostic { + b.public(ch.beta.as_cell()); + b.public(ch.z.as_cell()); + b.public(ch.gamma.as_cell()); + for zeta in &ch.zetas { + b.public(zeta.as_cell()); + } + for bits in &ch.iota_bits { + // The recombination is emitted only to be published; under + // `Aggregation` it is dead work, so it goes with the publish. + let felt = edsl::bits_to_felt(&mut b, bits); + b.public(felt.as_cell()); + } + } + } + + // ---- the LogUp closure, on the cells the forks already absorbed ---- + // + // Every `L` here is the cell its own fork bound into the transcript, and + // the output bytes are derived from the halves the statement absorbed — so + // the closure cannot be summing a different `L`, or folding a different + // output, from the one the challenges were drawn against. + let shape = super::logup::LogUpShape { + num_contributing_tables: contributions.len(), + num_output_bytes: e.statement.public_output_len, + }; + // ★ LEDGER ENTRY 2 CLOSES HERE. The carried commit index is not a word of its + // own and not even a second READ of one: it is the very cell the REGISTER + // preprocessed derivation consumed as INIT slot 64, so the COMMIT-bus target + // and the root Phase A absorbed are functions of one value. Production binds + // `start_index` exactly this way — it has no arithmetic `start + len` check + // anywhere, it rebuilds the commitment from the boundary vectors and rejects + // unless the absorbed root matches. + let start = reg_init[crate::tables::register::X254_INDEX]; + let bytes = super::epoch::emit_output_bytes(&mut b, public_output, shape.num_output_bytes); + let target = super::logup::emit_commit_bus_target(&mut b, &shape, z, alpha, start, &bytes); + let total = super::logup::emit_bus_closure(&mut b, &shape, &contributions, target); + b.public(total.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the epoch challenge program must be admissible"); + program +} + +/// The arenas [`epoch_challenge_program`] declares, in the same order. +fn epoch_arenas(e: &RealEpoch) -> Vec> { + epoch_arena_words(e, false) +} + +/// How many EPOCH-WIDE arenas [`epoch_program`] declares before the first +/// table's — statement, ELF-dependent preprocessed roots, main roots, the two +/// register boundary vectors, `pc_start`, and the page roots when there are any. +/// +/// Exposed rather than hardcoded because a test that walks to a per-table arena +/// by index silently tampers the WRONG arena when this changes, and reports a +/// pass: wiring ledger entry 7 moved it from 4 to 6. +pub(super) fn num_epoch_wide_arenas(e: &RealEpoch) -> usize { + 6 + usize::from(!e.page_commitments.is_empty()) +} + +/// How many words [`epoch_program`] publishes for the BLOCK-BINDING SCHEMA — the +/// run that sits between the attestation id and the first sub-proof's block. +/// +/// Exposed rather than recomputed at each reader for the reason +/// [`num_epoch_wide_arenas`] is: a gate that walks the published words by index +/// starts checking the WRONG field when this changes, and reports a pass. The +/// aggregator's `SchemaLayout` indexes the same run and must agree with this +/// number field for field. +/// +/// The order is the emitter's: register INIT, register FINI, the two epoch-label +/// halves, the public-output halves, then the L2G re-commit root's lanes. +pub(super) fn schema_words(e: &RealEpoch) -> usize { + 2 * crate::tables::register::NUM_REGISTER_ADDRESSES + + 2 + + e.statement.public_output_len.div_ceil(4) + + super::proof_arena::lanes_per_root() +} + +/// The arenas [`epoch_program`] declares, in the same order. +pub(super) fn epoch_arena_words(e: &RealEpoch, with_legs: bool) -> Vec> { + let mut stmt: Vec = Vec::new(); + let halves = |bytes: &[u8]| -> Vec { + bytes + .chunks(4) + .map(|c| { + let mut w = [0u8; 4]; + w[..c.len()].copy_from_slice(c); + FE::from(u32::from_le_bytes(w) as u64) + }) + .collect() + }; + stmt.extend(halves(&e.elf_digest)); + stmt.extend(halves(&e.public_output)); + stmt.extend(halves(&e.epoch_label.to_le_bytes())); + + // Only the ELF-DEPENDENT roots take arena words; the rest are program text or + // derived in-machine. + let prep: Vec = e + .phase_a + .iter() + .filter_map(|(p, _)| match p { + Some(PrepSource::ElfDependent(c)) => Some(*c), + _ => None, + }) + .collect(); + let main: Vec = e.phase_a.iter().map(|(_, m)| *m).collect(); + + // The register boundary, at production's width. The carried commit index sits + // in slot 64 of INIT, and the REGISTER preprocessed root is derived from both + // vectors — so this arena is not padding around one word any more. + let reg = |v: &[u32]| -> Vec { + assert_eq!( + v.len(), + crate::tables::register::NUM_REGISTER_ADDRESSES, + "a register boundary vector is one word per register word address" + ); + v.iter() + .map(|w| base_word(FE::from(u64::from(*w)))) + .collect() + }; + assert_eq!( + e.register_init[crate::tables::register::X254_INDEX] as u64, + e.start_index, + "the carried commit index must BE slot 64 of the INIT vector, or the \ + COMMIT-bus target and the REGISTER derivation are reading two values" + ); + let mut out = vec![ + stmt.iter().map(|h| base_word(*h)).collect(), + super::proof_arena::commitments_to_arena(&prep), + super::proof_arena::commitments_to_arena(&main), + reg(&e.register_init), + reg(&e.reg_fini), + super::keccak_host::pack_stream(&e.pc_start.to_le_bytes()) + .into_iter() + .map(base_word) + .collect(), + ]; + if !e.page_commitments.is_empty() { + let mut pages: Vec = Vec::new(); + for (base, c) in &e.page_commitments { + pages.extend( + super::keccak_host::pack_stream(&base.to_le_bytes()) + .into_iter() + .map(base_word), + ); + pages.extend( + super::keccak_host::pack_stream(c) + .into_iter() + .map(base_word), + ); + } + out.push(pages); + } + for (h, leg) in e.tables.iter().zip(&e.legs) { + if let Some(r) = h.aux_root { + out.push(super::proof_arena::commitments_to_arena(&[r])); + } + if let Some(c) = h.contribution { + out.push(vec![ext_word(&c)]); + } + out.push(super::proof_arena::commitments_to_arena(&[ + h.composition_root + ])); + out.push(h.ood_current.iter().map(ext_word).collect()); + out.push(h.ood_next.iter().map(ext_word).collect()); + out.push(h.parts.iter().map(ext_word).collect()); + out.push(super::proof_arena::commitments_to_arena(&h.fri_roots)); + out.push(h.fri_coeffs.iter().map(ext_word).collect()); + if let Some(nc) = h.nonce { + out.push(vec![base_word(FE::from(nc))]); + } + if with_legs { + out.push(leg.opening_arena()); + out.push(leg.fri_arena()); + out.extend(leg.caps_arena()); + } + } + out +} + +/// A keccak digest published as two words starting at `at` — eight `u32` halves, +/// four per word, each four bytes little-endian. +fn published_digest(public: &[(u32, LfmWord)], at: usize) -> [u8; 32] { + use math::field::traits::IsPrimeField; + let mut out = [0u8; 32]; + for h in 0..8 { + let lane = public[at + h / 4].1[h % 4]; + let half = GoldilocksField::canonical(lane.value()) as u32; + out[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + out +} + +/// ★ THE RUN: the assembled verifier's Fiat-Shamir spine, executed against a +/// real continuation epoch proof that production accepts. +/// +/// This is what the single-table differential could not reach. Both defects +/// `the_single_table_fixture_is_blind_to_two_defects` pins are live here — the +/// epoch has many tables, so the fork's domain separator matters, and the +/// production AIRs have multi-row OOD blocks, so the column-major absorb +/// matters. +#[test] +fn the_epoch_challenge_spine_matches_production() { + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let arenas = epoch_arenas(&e); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the epoch spine must execute"); + + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the shared LogUp challenge z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the shared LogUp challenge alpha"); + + // ★ The attestation fold, published right after Phase A. Its DECODE input is + // the very cell Phase A absorbed, so this differential is simultaneously a + // check of the fold and of the join: had the fold read a second copy, this + // would still pass — which is why the split is denied STRUCTURALLY by + // `the_assembled_verifier_declares_exactly_the_shape_words` and demonstrated + // by `a_split_decode_cell_forges_the_attestation`. + assert_eq!( + published_digest(&exec.public_words, 2), + e.expected_program_id, + "the attestation program_id must equal production's \ + `program_id_from_digest` over the same inputs" + ); + + // The pair, the two attestation-id words, then the BLOCK-BINDING SCHEMA — + // skipped by NAME, never by a literal. + // + // ⚠ `epoch_challenge_program` is `epoch_program(e, false)`, the SAME emitter + // with the legs off — the doc on `epoch_program` says so, and says why: a + // second copy of the spine would be a place for the assembled verifier's + // Fiat-Shamir to drift from the one this test checks. So every publish the + // assembled verifier gained, this spine gained too. A literal 4 here read + // register INIT slot 0 as `beta of table 0`, and since x0 is hard-wired zero + // the failure printed an all-zero challenge — which reads like a challenge + // that was never derived rather than like a cursor pointing at the wrong + // field. + let mut cursor = 4 + schema_words(&e); + let mut multi_row_ood = 0; + for (i, h) in e.tables.iter().enumerate() { + assert_eq!(pub_ext(cursor), h.beta, "beta of table {i}"); + assert_eq!(pub_ext(cursor + 1), h.z, "z of table {i}"); + assert_eq!(pub_ext(cursor + 2), h.gamma, "gamma of table {i}"); + cursor += 3; + for (k, want) in h.zetas.iter().enumerate() { + assert_eq!(pub_ext(cursor + k), *want, "zeta {k} of table {i}"); + } + cursor += h.zetas.len(); + for q in 0..h.shape.num_queries { + let w = exec.public_words[cursor + q].1; + let got = super::word::word_as_base(&w).expect("an index is a base felt"); + assert_eq!(got, FE::from(h.iotas[q] as u64), "iota {q} of table {i}"); + } + cursor += h.shape.num_queries; + if h.shape.ood_current_dims.1 > 1 || h.shape.ood_next_dims.1 > 1 { + multi_row_ood += 1; + } + println!( + " table {i:2}: ood_current {:?} ood_next {:?} parts {} fri_layers {} log2_trace {}", + h.shape.ood_current_dims, + h.shape.ood_next_dims, + h.shape.num_parts, + h.shape.fri.num_committed(), + h.shape.log2_trace_length + ); + } + // The closure's total, published last. Reaching it at all means the + // in-machine `assert_eq_ext` against the COMMIT-bus target already held. + assert_eq!( + word_as_ext(&exec.public_words[cursor].1).expect("the bus total is ext"), + e.expected_bus_balance, + "the LogUp closure must reach production's own COMMIT-bus target" + ); + cursor += 1; + assert_eq!( + cursor, + exec.public_words.len(), + "every published word must be checked" + ); + + // The blindness this fixture removes, asserted rather than hoped for. + assert!( + e.tables.len() > 1, + "the fork's domain separator needs more than one table to matter" + ); + // ★ MEASURED, not assumed: every one of the epoch's OOD blocks is ONE row + // tall, so column-major and row-major absorbs coincide on all of them. The + // current block's height IS `step_size` (`ood.rs:110-114`), and the phase + // already knows `step_size = 1` collapses production; the next block's is + // `num_eval_points − step_size`, which is 1 whenever an AIR has two + // transition offsets. So the absorb ORDER has no production witness at all, + // and closing it needs a synthetic AIR — see the ledger entry this leg added. + assert_eq!( + multi_row_ood, 0, + "an OOD block taller than one row appeared: the absorb-order blindness \ + recorded here is over, and the differential now covers it" + ); + // ---- THE MEASUREMENT ---- + // + // What this is and is NOT: the spine is the Fiat-Shamir half of the + // verifier — statement, Phase A, 24 forks, rounds 2-4 and the LogUp + // closure. The opening/DEEP/FRI-walk and constraint legs are NOT in this + // program, so these numbers say nothing about the composed per-epoch + // predictions (213,744 opening permutations at blowup 8, ~460k total). + // Those remain unconfirmed. This is the first per-epoch figure that is a + // RUN rather than a composition, and it is the cost of the part that had + // no per-epoch number at all. + let perms = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::KeccakF(_))) + .count(); + let hints = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Hint { .. })) + .count(); + let arena_words: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + let bit_decs = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::BitDec { .. })) + .count(); + // Attribution, not a guess: every EXTENSION value the transcript absorbs is + // three base felts, and each base felt is streamed BIG-endian, which costs + // one `felt_be_halves` — a `BitDec` plus its recomposition. So the absorbed + // ext count times three should account for nearly every `BitDec` here. + let ext_absorbs: usize = e + .tables + .iter() + .map(|h| { + h.ood_current.len() + + h.ood_next.len() + + h.parts.len() + + h.fri_coeffs.len() + + usize::from(h.contribution.is_some()) + }) + .sum(); + println!( + "\nepoch spine (min preset: blowup 2, {} quer{}/table, grinding {}):\n\ + \x20 sub-proofs {}\n\ + \x20 instructions {}\n\ + \x20 keccak perms {}\n\ + \x20 arena words {} ({} hinted)\n\ + \x20 published words {}\n\ + \x20 multi-row OOD {}\n\ + \x20 BitDec rows {}\n\ + \x20 ext values absorbed {} (x3 felts = {} big-endian streams, \ + {:.1}% of the BitDecs)", + e.tables[0].shape.num_queries, + if e.tables[0].shape.num_queries == 1 { + "y" + } else { + "ies" + }, + e.tables[0].shape.grinding_factor, + e.tables.len(), + program.instrs.len(), + perms, + arena_words, + hints, + exec.public_words.len(), + multi_row_ood, + bit_decs, + ext_absorbs, + 3 * ext_absorbs, + 100.0 * (3 * ext_absorbs) as f64 / bit_decs as f64 + ); +} + +/// ★ An ABSOLUTE structural guard (standing-decisions rule 7): no proof value +/// in the assembled spine is hinted twice. +/// +/// The two-consumer class hides exactly where a differential cannot look — a +/// value hinted once per consumer, with the host packing the same number into +/// both, passes every comparison against production and still lets a real +/// prover supply two different numbers. So this is a count over the emitted +/// program, not a comparison of two runs: every arena word is read by at most +/// one `Hint`, and the arenas whose values have two consumers (the roots, the +/// contributions, the statement's public output) are read exactly once. +/// +/// The exception this test used to carry is GONE: the register-boundary arena +/// had only its commit index read while the REGISTER derivation was unbuilt, and +/// wiring the derivation (ledger entries 7 and 2) makes every declared word live. +/// So the positive control is now exact — `declared` words, `declared` reads — +/// which is a strictly stronger statement than the one it replaces. +#[test] +fn the_spine_hints_each_proof_value_once() { + use std::collections::HashMap; + + let e = real_epoch(); + let program = epoch_challenge_program(&e); + + let mut hints: HashMap<(super::instr::ArenaId, u32), usize> = HashMap::new(); + for instr in &program.instrs { + if let super::instr::Instr::Hint { arena, index, .. } = instr { + *hints.entry((*arena, *index)).or_default() += 1; + } + } + let doubled: Vec<_> = hints.iter().filter(|(_, n)| **n > 1).collect(); + assert!( + doubled.is_empty(), + "these arena words are hinted more than once, which is the two-consumer \ + hazard the assembly exists to remove: {doubled:?}" + ); + + // Positive control: the count is nonzero and covers the whole proof, so a + // guard that simply found no hints would not pass for the wrong reason. + let declared: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + hints.len(), + declared, + "every declared arena word must be read exactly once" + ); +} + +/// Arena words the epoch program MUST declare, as arithmetic over the epoch's +/// shapes. +/// +/// Deliberately not derived from the emitter (standing-decisions rule 7's +/// refinement: a count taken from our own emitter is still a relative test). Every +/// term here comes from the production proof view `real_epoch` read, so the +/// comparison against the compiled program is absolute. +fn expected_arena_words(e: &RealEpoch, with_legs: bool) -> usize { + let num_reg = crate::tables::register::NUM_REGISTER_ADDRESSES; + // A root's width in arena words — the host's counterpart of the emitter's + // `digest_words`: two on a byte hash, one on an algebraic one. The register + // vectors, `pc_start` and the page bases are NOT roots and keep their own + // widths below. + let dw = super::proof_arena::words_per_root(); + let mut total = 8 + e.statement.public_output_len.div_ceil(4) + 2; + // ★ One root's width per ELF-DEPENDENT preprocessed root and NOT ONE MORE. + // The options-only roots are program text and the REGISTER root is derived, + // so a program that hinted any of them — or that kept a second copy of + // DECODE for the attestation fold — declares more words than this. + total += dw + * e.phase_a + .iter() + .filter(|(p, _)| p.is_some_and(PrepSource::is_arena)) + .count(); + total += dw * e.tables.len(); + total += 2 * num_reg; + total += 2; + total += 10 * e.page_commitments.len(); + for (h, leg) in e.tables.iter().zip(&e.legs) { + let s = &h.shape; + total += dw * usize::from(s.has_aux_root); + total += usize::from(s.has_contribution); + total += dw; + total += s.ood_current_dims.0 * s.ood_current_dims.1; + total += s.ood_next_dims.0 * s.ood_next_dims.1; + total += s.num_parts; + total += dw * s.fri.num_committed(); + total += s.fri.num_terminal_coeffs(); + total += usize::from(s.grinding_factor > 0); + if with_legs { + total += leg.verify.opening_words(dw) + leg.verify.fri_words(dw); + } + } + total +} + +/// ★ An ABSOLUTE guard on the arena SCHEMA — the structural half of the +/// attestation join (entry-7 ruling, condition (a)). +/// +/// The hinted-once guard denies a value being read twice from ONE word. It cannot +/// deny a value being supplied twice in TWO words, which is the whole two-consumer +/// hazard: an honest host fills both copies alike, every differential passes, and a +/// real prover supplies two different roots. What denies that is the schema itself +/// — the program declares exactly the words the epoch's shapes prescribe, so there +/// is nowhere for a second copy to live. +/// +/// Together the two guards are complete for this class: a second copy must either +/// re-read an existing word (hinted-once fails) or add one (this fails). A fold +/// that instead read some OTHER existing value would publish a `program_id` that is +/// not production's, which the spine differential catches. +#[test] +fn the_assembled_verifier_declares_exactly_the_shape_words() { + let e = real_epoch(); + for with_legs in [false, true] { + let program = epoch_program(&e, with_legs); + let declared: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + declared, + expected_arena_words(&e, with_legs), + "with_legs = {with_legs}: the arena schema must be exactly the epoch's \ + shapes and nothing more — a surplus word is where a second copy of a \ + joined value hides" + ); + } + + // Positive control on the guard itself: the split-cell control program DOES + // declare a surplus word, and this is the comparison that sees it. + let split = epoch_program_with(&e, false, true, Publishes::Diagnostic); + let split_declared: usize = split.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + split_declared, + expected_arena_words(&e, false) + super::proof_arena::words_per_root(), + "the split-cell control must declare exactly one root's width of surplus \ + words, or it is not the forgery this guard claims to deny" + ); +} + +/// ★ FALSIFICATION of the attestation join, as a COHERENT FORGERY rather than a +/// count (standing-decisions method rule 4). +/// +/// The attack: verify a real epoch proof of ELF X while attesting to the +/// `program_id` of a different ELF Y. A consumer who trusts Y's id accepts the +/// proof, and X is whatever the prover likes. +/// +/// On the SPLIT program this succeeds completely — every assert passes, the run +/// finishes, and the published id is the one computed from the substituted root, +/// not from the root the proof was made against. On the JOINED program the attack +/// is not merely rejected, it cannot be EXPRESSED: there is one cell, so changing +/// the fold's input changes what Phase A absorbed, which moves every challenge and +/// the run dies. Both halves are asserted, because "the joined program rejects it" +/// alone would be satisfied by a program that rejects everything. +#[test] +fn a_split_decode_cell_forges_the_attestation() { + let e = real_epoch(); + let honest = epoch_arena_words(&e, false); + + // A DECODE root for some other program. Any 32 bytes the honest arena does not + // carry will do; what matters is the id it produces. + let real_decode = e + .phase_a + .iter() + .find_map(|(p, _)| match p { + Some(PrepSource::ElfDependent(c)) => Some(*c), + _ => None, + }) + .expect("the epoch has a DECODE sub-proof"); + let mut substituted = real_decode; + substituted[0] ^= 0xa5; + substituted[31] ^= 0x5a; + let forged_id = + crate::recursion::program_id_from_digest(&e.elf_digest, e.pc_start, &substituted, &[]); + assert_ne!( + forged_id, e.expected_program_id, + "the substituted root must produce a different id, or this proves nothing" + ); + + // ---- (a) the SPLIT program: the forgery runs and publishes the forged id. + let split = epoch_program_with(&e, false, true, Publishes::Diagnostic); + let mut split_arenas = honest.clone(); + split_arenas.push(super::proof_arena::commitments_to_arena(&[substituted])); + let exec = execute(&split, &split_arenas, &crate::hash_pin::BLOCK_HASHER).expect( + "the split-cell program must RUN on the forgery — that is the hazard, and \ + a rejection here would mean this control does not demonstrate it", + ); + assert_eq!( + published_digest(&exec.public_words, 2), + forged_id, + "the split program must attest to the SUBSTITUTED root while verifying a \ + proof made against the real one" + ); + // And it is genuinely a proof of the real epoch: the same program, given the + // honest root in the surplus arena, publishes the honest id. + let mut split_honest = honest.clone(); + split_honest.push(super::proof_arena::commitments_to_arena(&[real_decode])); + let exec_honest = execute(&split, &split_honest, &crate::hash_pin::BLOCK_HASHER) + .expect("the split program must also run honestly"); + assert_eq!( + published_digest(&exec_honest.public_words, 2), + e.expected_program_id, + "the split program's two runs differ only in the surplus arena, so the \ + forgery is a free choice and not a broken proof" + ); + + // ---- (b) the JOINED program: the same substitution is inexpressible. + // + // There is no surplus arena to put it in, so the only way to move the fold's + // input is to move the cell Phase A absorbed — which moves every challenge + // derived after it. + let joined = epoch_program(&e, false); + let mut joined_arenas = honest.clone(); + joined_arenas[1] = super::proof_arena::commitments_to_arena(&[substituted]); + assert!( + execute(&joined, &joined_arenas, &crate::hash_pin::BLOCK_HASHER).is_err(), + "with one cell, substituting the DECODE root must break the run: the \ + transcript absorbed it, so the challenges cannot survive it" + ); +} + +/// ★ LEDGER ENTRY 2, closed and falsified: the whole register boundary is bound, +/// not just the commit index. +/// +/// Production ties epoch N's carried commit index to the chain by REBUILDING the +/// REGISTER preprocessed commitment from epoch N−1's FINI vector and rejecting +/// unless the absorbed root matches — there is no arithmetic `start + len` check +/// anywhere (`lfm-team-lead-start-index-research.md`). So the machine's binding is +/// the derivation, and what must be true is that moving ANY word of either vector +/// makes the epoch unverifiable. +/// +/// Before the derivation was wired, 66 of the 67 INIT words were declared and never +/// read: moving them changed nothing at all. The positive control for that is +/// structural rather than historical — `the_spine_hints_each_proof_value_once` now +/// requires every declared word to be read, and it did not before. +/// +/// Slot 64 is the commit index and is tested separately by +/// [`the_closure_rejects_a_moved_index_or_output`]; the slots here are deliberately +/// elsewhere, including the first and last of each vector, because a derivation that +/// only really consumed a prefix would pass a test that only moved slot 64. +#[test] +fn the_derivation_binds_every_register_boundary_word() { + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let good = epoch_arenas(&e); + assert!( + execute(&program, &good, &crate::hash_pin::BLOCK_HASHER).is_ok(), + "the untampered epoch must run" + ); + + let last = crate::tables::register::NUM_REGISTER_ADDRESSES - 1; + let x254 = crate::tables::register::X254_INDEX; + let mut moved = 0; + for (arena, what) in [(3usize, "INIT"), (4, "FINI")] { + for slot in [0usize, 1, 33, x254 + 1, last] { + let mut arenas = good.clone(); + let bumped = arenas[arena][slot][0] + FE::one(); + arenas[arena][slot] = base_word(bumped); + assert!( + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER).is_err(), + "{what} slot {slot} moved by one must not verify: the REGISTER \ + preprocessed root is derived from it, and the transcript absorbed \ + that root" + ); + moved += 1; + } + } + assert_eq!(moved, 10, "every planned vector must have been run"); +} + +/// ★ LEDGER ENTRY 1, in two halves — and the obvious formulation of this test is +/// VACUOUS, which is worth stating because I wrote it first. +/// +/// The tempting test is "set a boundary word to `2^32` and watch the assembled +/// epoch fail". It does fail — and it fails with the check REMOVED too, because a +/// wide value moves the derived REGISTER root, which moves every challenge drawn +/// after Phase A absorbs it. So that test says nothing about the width check at +/// all; it is the same rejection +/// `the_derivation_binds_every_register_boundary_word` already gets from moving a +/// word by one. +/// +/// What is not vacuous is the pair below, and together they are complete: +/// +/// 1. **What [`super::epoch::assert_u32`] does**, in isolation: the whole `u32` +/// range runs and everything at or above `2^32` is unprovable. Absolute — it is +/// a property of the check's own output, with no epoch involved. +/// 2. **That it is applied to every one of the 134 boundary cells**, structurally: +/// each register-arena `Hint` output is the INPUT of a 32-bit `BitDec`. A check +/// emitted over a prefix — the failure mode a value-tamper test cannot see, +/// since any single moved word rejects anyway — fails this. +#[test] +fn the_register_boundary_is_width_checked() { + // ---- (1) the check itself. + let drive = |v: u64| { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(1); + let cell = b.hint_felt(arena, 0); + super::epoch::assert_u32(&mut b, cell); + let program = compile(b.finish()); + validate(&program).expect("the width check must be admissible"); + execute( + &program, + &[vec![base_word(FE::from(v))]], + &crate::hash_pin::BLOCK_HASHER, + ) + .is_ok() + }; + // ⚠ The bad values are CANONICAL felts, and that is not pedantry — it is the + // exact size of the gap. An arena word is a field element, so `FE::from(v)` + // reduces: `u64::MAX − 1` is the felt `2^32 − 3`, a perfectly good `u32`, and a + // test that used it would report the check broken when it is not (it did). The + // widening entry 1 names is therefore the interval `[2^32, p)` and nothing + // beyond — there is no felt at or above `p` to worry about. + const P_MINUS_1: u64 = 0xFFFF_FFFF_0000_0000; // Goldilocks p − 1 = 2^64 − 2^32 + for ok in [0u64, 1, 255, 1 << 31, (1u64 << 32) - 1] { + assert!(drive(ok), "{ok} is a u32 and must be admitted"); + } + for bad in [1u64 << 32, (1u64 << 32) + 1, 1 << 40, P_MINUS_1] { + assert!( + !drive(bad), + "{bad} is not a u32 and must be unprovable: production's boundary \ + vectors are Vec and the TYPE is their only enforcement" + ); + } + + // ---- (2) every boundary cell reaches it, in the assembled program. + use std::collections::HashSet; + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let num_reg = crate::tables::register::NUM_REGISTER_ADDRESSES; + + // The two register arenas are the ones whose declared length is + // NUM_REGISTER_ADDRESSES; identified by length rather than by index so that + // adding an epoch-wide arena cannot silently point this test at the wrong one. + let reg_arenas: Vec = program + .arena_schema + .lens + .iter() + .enumerate() + .filter(|(_, l)| **l as usize == num_reg) + .map(|(i, _)| i as super::instr::ArenaId) + .collect(); + assert_eq!( + reg_arenas.len(), + 2, + "expected exactly the INIT and FINI arenas to have the register width" + ); + + let mut boundary_cells: HashSet = HashSet::new(); + for instr in &program.instrs { + if let super::instr::Instr::Hint { arena, out, .. } = instr + && reg_arenas.contains(arena) + { + boundary_cells.insert(*out); + } + } + assert_eq!( + boundary_cells.len(), + 2 * num_reg, + "every declared boundary word must be read exactly once" + ); + + let decomposed: HashSet = program + .instrs + .iter() + .filter_map(|i| match i { + super::instr::Instr::BitDec { input, bits, .. } if bits.len() == 32 => Some(*input), + _ => None, + }) + .collect(); + let unchecked: Vec<_> = boundary_cells.difference(&decomposed).collect(); + assert!( + unchecked.is_empty(), + "these register-boundary cells are never bit-decomposed, so their width \ + is unconstrained: {unchecked:?}" + ); +} + +/// ★ The closure's two joins, falsified by tampering. +/// +/// The COMMIT-bus target is a function of the carried commit index and of the +/// public output, and both reach it through cells another consumer already +/// used — `start_index` from the register-boundary arena the REGISTER +/// derivation will bind, the output bytes from the halves the STATEMENT +/// absorbed. Moving either must break the run. +#[test] +fn the_closure_rejects_a_moved_index_or_output() { + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let good = epoch_arenas(&e); + assert!( + execute(&program, &good, &crate::hash_pin::BLOCK_HASHER).is_ok(), + "the untampered epoch must run" + ); + + // The carried commit index. Production derives it from the previous epoch's + // FINI vector; a machine that let the prover pick it would let them + // renumber the whole output stream. + for delta in [1u64, 2, 7] { + let mut arenas = good.clone(); + arenas[3][crate::tables::register::X254_INDEX] = base_word(FE::from(e.start_index + delta)); + assert!( + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER).is_err(), + "start_index + {delta} must not close the bus" + ); + } + + // The public output. Moving a half moves both the statement the challenges + // were drawn against and the bytes the target folds, so this rejects + // whichever check notices first — but reject it must. + assert!( + !e.public_output.is_empty(), + "the fixture epoch must actually commit output, or this proves nothing" + ); + for half in 0..e.statement.public_output_len.div_ceil(4) { + let mut arenas = good.clone(); + let idx = 8 + half; + let bumped = arenas[0][idx][0] + FE::one(); + arenas[0][idx] = base_word(bumped); + assert!( + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER).is_err(), + "moving output half {half} must not verify" + ); + } +} + +/// ★ **THE ROOT-WIDTH GATE** — the arena WRITER and the machine READER must +/// agree on how many words a commitment occupies, and on what its published +/// lanes are. +/// +/// Three counts have to line up for a root to survive the trip: +/// `proof_arena::words_per_root` (host — what the arena writer strides by), +/// `epoch::RootCells::words_per_root` (machine — what `RootCells::hint` +/// consumes) and `proof_arena::lanes_per_root` (what a PUBLISHED root occupies +/// in the schema). All three read the configuration's digest width and none +/// restates it — but "none restates it" is an argument, and this is the check. +/// +/// ⚠ **The failure it exists for is not a shape error.** A reader striding by +/// two where the writer strode by one reads root k+1's first word as root k's +/// second: a well-formed root that authenticates nothing, surfacing as +/// `DivByZero` thousands of instructions downstream with nothing named. Two +/// roots are hinted BACK TO BACK for exactly that reason — a stride that is +/// wrong only *between* roots reads correctly at root 0 and cannot hide here. +/// +/// ⚖ ASSESSMENT: this gate is about WIDTH, not about the felt↔byte rendering. +/// `lanes_flat` yields `u32` halves on a byte hash and full felts on an +/// algebraic one, and a caller that hands the wrong one to a byte fold gets a +/// plausible count with wrong content — `RootCells::byte_halves` exists for +/// that distinction and is gated where it is used, not here. +#[test] +fn the_arena_writer_and_the_machine_reader_agree_on_a_roots_width() { + use super::proof_arena::{commitment_lanes, commitment_words, lanes_per_root, words_per_root}; + + // Two DISTINCT roots. Any 32 bytes are a valid commitment on either arm: + // the byte arm reads them as eight little-endian `u32` halves, the + // algebraic one as four big-endian felts reduced into the field, and both + // conversions are the backend's own. + let roots: [Commitment; 2] = [ + core::array::from_fn(|i| (0x03u8.wrapping_mul(i as u8)).wrapping_add(0x11)), + core::array::from_fn(|i| (0x05u8.wrapping_mul(i as u8)).wrapping_add(0xA3)), + ]; + + // ---- the three counts agree ---- + let per_root = words_per_root(); + for root in &roots { + assert_eq!( + commitment_words(root).len(), + per_root, + "the arena writer must emit exactly `words_per_root` words" + ); + assert_eq!( + commitment_lanes(root).len(), + lanes_per_root(), + "a published root's lanes are its words' lanes" + ); + } + assert_eq!( + lanes_per_root(), + super::word::WORD_LANES * per_root, + "a lane count is a word count times the word's lanes" + ); + + // ---- the machine reads them back at the same stride ---- + let mut b = LfmBuilder::new().with_wrap_hash(edsl::WrapHash::production()); + assert_eq!( + RootCells::words_per_root(&b) as usize, + per_root, + "the machine reader and the arena writer must stride alike" + ); + let arena = b.declare_arena(2 * per_root as u32); + for k in 0..2u32 { + let cells = RootCells::hint(&mut b, arena, k * per_root as u32); + for lane in cells.lanes_flat() { + b.public(lane.as_cell()); + } + } + let program = compile(b.finish()); + let words: Vec = roots.iter().flat_map(commitment_words).collect(); + let exec = execute(&program, &[words], &crate::hash_pin::BLOCK_HASHER) + .expect("the root-width program must execute"); + + let want: Vec = roots.iter().flat_map(commitment_lanes).collect(); + assert_eq!( + exec.public_words.len(), + want.len(), + "two roots publish exactly `2 * lanes_per_root` lanes" + ); + for (i, want) in want.into_iter().enumerate() { + let got = super::word::word_as_base(&exec.public_words[i].1).expect("a root lane"); + assert_eq!( + got, + want, + "root {} lane {}", + i / lanes_per_root(), + i % lanes_per_root() + ); + } +} diff --git a/prover/src/lfm/epoch_verify.rs b/prover/src/lfm/epoch_verify.rs new file mode 100644 index 000000000..b0d3fb0ea --- /dev/null +++ b/prover/src/lfm/epoch_verify.rs @@ -0,0 +1,927 @@ +//! One sub-proof VERIFIED — the legs hung off the Fiat-Shamir spine. +//! +//! [`super::epoch`] replays production's challenge derivation and hands back +//! [`TableChallenges`]; every leg built so far took those same challenges as +//! arena words instead. This module is where the two meet: it takes the cells +//! the spine absorbed and the challenges the spine derived, and emits the four +//! checks a real verifier performs on one sub-proof. +//! +//! ```text +//! spine gives leg consumes +//! ----------- ------------ +//! ood_current / ood_next ──────► the reconstructed grid: constraints AND DEEP +//! parts ──────► the quotient's claimed value AND DEEP's h_sum +//! z ──────► the zerofier's ζ AND DEEP's row points +//! beta ──────► the β-power fold +//! gamma ──────► the DEEP batching challenge +//! zetas ──────► the FRI fold chain +//! iota_bits ──────► the Merkle walk, the query point, the FRI walk +//! contribution (L) ──────► the table offset AND the LogUp closure +//! every ROOT ──────► the authentication compare +//! ``` +//! +//! Nothing in that table is hinted twice. The roots arrive as +//! [`super::epoch::RootCells`] and become [`GroupCommitment`]s through +//! `from_lanes`, the OOD blocks become one grid through +//! [`super::epoch::emit_reconstruct_ood`], and the query index never exists as a +//! felt. What the arenas still carry, per sub-proof, is exactly the data a real +//! proof carries and a verifier cannot derive: the opened row pairs, the Merkle +//! paths, and the FRI layers' symmetric evaluations. +//! +//! ## What this module cannot see +//! +//! It verifies ONE sub-proof. It says nothing about the epoch's statement, about +//! Phase A, or about the LogUp closure across tables — those are the spine's and +//! [`super::logup`]'s. It also does not check the preprocessed commitments +//! against anything: production takes them from the AIR, and where they come +//! from in the assembled machine is [`TableInputs`]' caller's problem (assembly +//! ledger entry 7). + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, LfmBuilder}; +use super::constraints::{ + Analysis, BoundaryTerm, OodOperands, QuotientShape, emit_alpha_powers, emit_analyzed, + emit_quotient, emit_table_offset, +}; +use super::deep::{DeepInvariants, emit_deep_invariants_bound}; +use super::edsl::WrapHash; +use super::epoch::{RootCells, TableAbsorbs, TableChallenges, emit_reconstruct_ood}; +use super::fri::{ + FriCommitments, FriQuery, FriShape, LayerCommitment, emit_query_fri, hint_layer_openings_from, +}; +use super::instr::ArenaId; +use super::sub_proof::{GroupCommitment, GroupOpening, SubProofShape, emit_query_from_bits}; + +/// The compile-time shape of one sub-proof's full verification. +/// +/// Every field is program SHAPE, in the sense `others/lfm-target-shape.md` fixes: +/// a value the AIR set and the proof options determine, never a value the proof +/// carries. The one exception worth naming is [`Self::quotient`]'s boundary list, +/// which production computes from the public inputs — see [`boundary_terms`] for +/// the rule it is built from and the premise that rule rests on. +#[derive(Clone, Debug)] +pub struct TableVerifyShape { + /// The trace/opening shape: DEEP columns, the committed groups, the tree + /// depth and the LDE domain. + pub sub: SubProofShape, + /// The FRI shape, which must describe the same LDE domain. + pub fri: FriShape, + /// The zerofier, the part count and the boundary constraints. + pub quotient: QuotientShape, + /// Where the aux columns start in a full-width `[main | aux]` row. + pub main_width: usize, + /// `AIR::max_bus_elements()` — how long the α-power chain is. Zero for an + /// AIR with no aux trace, which has no `Op::AlphaPow` to resolve. + pub num_alpha_powers: usize, + /// Queries the sub-proof carries. + pub num_queries: usize, +} + +impl TableVerifyShape { + /// Frame STEPS the constraint program indexes — `Op::Var{offset}` runs over + /// these, not over the OOD grid's rows. + /// + /// A frame step is `step_size` grid rows and production's own interpreter + /// reads only row 0 of each (`constraint_ir/interp.rs:240-242` asserts + /// `row == 0`), so the constraint leg's view of the grid is every + /// `step_size`-th row while DEEP's is all of it. The two coincide at + /// `step_size = 1`, which every production AIR has — carrying the stride + /// anyway is the same discipline `DeepShape::block` applies to the + /// coefficient run, and for the same reason. + pub fn num_frame_steps(&self) -> usize { + self.sub.deep.num_eval_points / self.sub.deep.step_size + } + + fn check(&self) { + assert_eq!( + self.sub.log2_lde_length, self.fri.log2_lde_length, + "both legs verify one sub-proof over one LDE domain" + ); + assert_eq!( + self.sub.merkle_depth, + self.fri.index_bits(), + "the FRI layers consume suffixes of the trace walk's decomposition" + ); + assert_eq!( + self.sub.layout, + self.fri.leaf_layout(), + "the trace trees and the FRI chain verify one table at one leaf layout" + ); + assert_eq!( + self.fri.num_queries, self.num_queries, + "the query count is one shape, declared once" + ); + assert_eq!( + self.sub.deep.num_composition_parts, self.quotient.num_composition_parts, + "the part count is one shape: DEEP folds the same parts the quotient \ + Horner claims" + ); + assert_eq!( + self.sub.deep.log2_trace_length, self.quotient.log2_trace_length, + "the trace length is one shape" + ); + assert_eq!( + self.num_frame_steps() * self.sub.deep.step_size, + self.sub.deep.num_eval_points, + "the OOD grid is a whole number of frame steps" + ); + assert!( + self.main_width <= self.sub.deep.num_total_cols, + "the aux columns start inside the row" + ); + self.check_caps(); + } + + /// Arena words this sub-proof's trace openings occupy, at `digest_words` + /// per sibling digest — the builder's width on the machine side, the host's + /// on the host side (see `SubProofShape::query_words`). + pub fn opening_words(&self, digest_words: usize) -> usize { + self.num_queries * self.sub.opening_words(digest_words) + } + + /// Arena words this sub-proof's FRI openings occupy, at `digest_words` per + /// sibling digest. + pub fn fri_words(&self, digest_words: usize) -> usize { + self.num_queries * self.fri.query_words(digest_words) + } + + /// Arena words this sub-proof's Merkle caps occupy, once per sub-proof: + /// the committed matrices' caps (group order), then the committed FRI + /// layers' (layer order). Zero at the default format. + pub fn cap_words(&self, digest_words: usize) -> usize { + self.sub.cap_words(digest_words) + self.fri.cap_words(digest_words) + } + + fn check_caps(&self) { + assert_eq!( + self.sub.arity, + self.fri.arity(), + "the trace trees and the FRI layer trees are one configuration's" + ); + assert_eq!( + self.sub.trace_cap, + stark::merkle_caps::StarkCaps::tree_cap_height( + self.fri.trees.cap_policy(self.fri.format.merkle_cap), + self.num_queries, + self.sub.merkle_depth, + self.sub.arity, + ), + "the trace trees' cap is the configuration's, at their depth and query count" + ); + } + + /// The verification legs' shape for `air`'s sub-proof over `trace_length` + /// rows, and the AIR's constraint analysis — from the AIR, the trace length + /// and the AIR's options, never from a proof. The part count is the AIR's + /// degree bound over `trace_length` (the host verifier pins the proof's copy + /// to it, `verifier.rs:1730`). + /// + /// Refuses a trace length that is not a power of two and a format the host + /// would lay out with other tree heights than these shapes (`StarkCaps`, the + /// prover's and verifier's own derivation). + pub fn derive( + air: &super::epoch::DynVmAir<'_>, + trace_length: usize, + ) -> Result<(Self, Analysis), String> { + Self::derive_with(air, trace_length, stark::config::TreeGeometry::BINARY) + } + + /// [`Self::derive`] for a sub-proof committed under a configuration of + /// `trees` (its arity and arity-4 cap, `TreeGeometry::of::()`): the + /// tree heights and path shapes that configuration's prover lays out, and + /// the host's `StarkCaps` under it as the cross-check. + pub fn derive_with( + air: &super::epoch::DynVmAir<'_>, + trace_length: usize, + trees: stark::config::TreeGeometry, + ) -> Result<(Self, Analysis), String> { + use stark::verifier::IsStarkVerifier; + if !trace_length.is_power_of_two() { + return Err(format!("trace length {trace_length} is not a power of two")); + } + let opts = air.options(); + let layout = stark::verifier::Verifier::< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + (), + >::ood_layout(air); + let artifact = stark::constraint_ir::ConstraintArtifact::capture(air); + + let (main_width, aux_width) = air.trace_layout(); + let num_total_cols = main_width + aux_width; + let num_precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + let log2_trace_length = trace_length.trailing_zeros(); + let log2_lde_length = log2_trace_length + (opts.blowup_factor as usize).trailing_zeros(); + let num_parts = air.composition_poly_degree_bound(trace_length) / trace_length; + + // The trace matrices in DEEP column order — precomputed, main, aux. + let mut trace_groups = Vec::new(); + if num_precomputed > 0 { + trace_groups.push(super::sub_proof::GroupShape { + num_columns: num_precomputed, + is_ext: false, + }); + } + trace_groups.push(super::sub_proof::GroupShape { + num_columns: main_width - num_precomputed, + is_ext: false, + }); + if aux_width > 0 { + trace_groups.push(super::sub_proof::GroupShape { + num_columns: aux_width, + is_ext: true, + }); + } + let deep = super::deep::DeepShape { + step_size: layout.step_size(), + num_eval_points: artifact.shape.transition_offsets.len() * layout.step_size(), + num_total_cols, + next_row_cols: layout.next_row_cols().to_vec(), + num_composition_parts: num_parts, + log2_trace_length, + commit_ood: None, + }; + + // The table's leaf layout (S2): the host prover's and verifier's own + // per-table resolution. + let leaf_layout = stark::leaf_layout::table_leaf_layout(air, trace_length); + let merkle_depth = leaf_layout.tree_depth(log2_lde_length as usize); + let sub = SubProofShape { + deep, + trace_groups, + merkle_depth, + log2_lde_length, + coset_offset: FE::from(opts.coset_offset), + trace_cap: stark::merkle_caps::StarkCaps::tree_cap_height( + trees.cap_policy(opts.format.merkle_cap), + opts.fri_number_of_queries, + merkle_depth, + trees.arity, + ), + arity: trees.arity, + layout: leaf_layout, + }; + let has_aux_trace = air.has_aux_trace(); + let shape = Self { + quotient: QuotientShape { + log2_trace_length, + num_composition_parts: num_parts, + boundary: boundary_terms(has_aux_trace, num_total_cols), + }, + fri: FriShape::for_layout(opts, log2_lde_length, leaf_layout).with_trees(trees), + main_width, + num_alpha_powers: if has_aux_trace { + artifact.shape.max_bus_elements as usize + } else { + 0 + }, + num_queries: opts.fri_number_of_queries, + sub, + }; + + // The cap heights: these shapes' against the host's own `StarkCaps`, so + // the two sides derive every tree's height and depth from one function. + let host = stark::merkle_caps::StarkCaps::for_options_with( + opts, + log2_lde_length as usize, + leaf_layout.is_one_row(), + trees, + ) + .map_err(|e| format!("a format the host does not lay out: {e:?}"))?; + let fri_agrees = host.fri.len() == shape.fri.num_committed() + && host + .fri_depths + .iter() + .zip(&host.fri) + .enumerate() + .all(|(i, (&d, &c))| d == shape.fri.layer_depth(i) && c == shape.fri.layer_cap(i)); + if host.trace_depth != shape.sub.merkle_depth + || host.trace != shape.sub.trace_cap + || !fri_agrees + { + return Err("the legs' tree heights differ from the host's StarkCaps".to_string()); + } + Ok((shape, super::constraints::analyze(&artifact))) + } +} + +impl TableVerifyShape { + /// This shape for a sub-proof of a bound proof (`stark::commit_ood`): its + /// DEEP carries the answers of `air`'s answered columns. Every LFM proof + /// is bound; a VM epoch's sub-proofs are not. + pub fn with_commit_ood(mut self, air: &super::epoch::DynVmAir<'_>) -> Self { + let cols = stark::commit_ood::answered_columns(air); + self.sub.deep.commit_ood = (!cols.is_empty()).then_some(super::deep::CommitOodCols { + first: cols.start, + count: cols.len(), + }); + self + } + + /// The answers this sub-proof's Phase A absorbs: zero when unbound. + pub fn num_answers(&self) -> usize { + self.sub.deep.commit_ood.map_or(0, |c| c.count) + } +} + +/// The preprocessed root the host verifier takes for `air` over `trace_length` +/// rows: `precomputed_commitment_for` the table's resolved leaf layout (S2 — a +/// layout with no root is `None`, never the other layout's root). A verifier +/// constant: never the proof's copy. +pub fn layout_precomputed_commitment( + air: &dyn stark::traits::AIR< + Field = crate::tables::types::GoldilocksField, + FieldExtension = crate::tables::types::GoldilocksExtension, + PublicInputs = PI, + >, + trace_length: usize, +) -> Option { + air.precomputed_commitment_for(stark::leaf_layout::table_leaf_layout(air, trace_length)) +} + +/// The two arenas one sub-proof's query verification reads, in declaration +/// order. +/// +/// Deliberately only two. Everything else a leg used to hint — the roots, the +/// challenges, the OOD grid, the claimed parts, the FRI layer roots and terminal +/// coefficients — reaches the legs as cells the spine already bound. +#[derive(Clone, Copy, Debug)] +pub struct TableQueryArenas { + /// Per query, per group: the row-pair values then the sibling digests + /// (`edsl::digest_words` per level). NO index word — the index is the + /// transcript's. + pub openings: ArenaId, + /// Per query, per committed FRI layer: the symmetric evaluation then the + /// sibling digests. + pub fri: ArenaId, + /// The sub-proof's Merkle caps ([`TableVerifyShape::cap_words`]), declared + /// only when the format caps some tree — so the default format's arena + /// schema, program and program id are today's. + pub caps: Option, +} + +/// Declare the query arenas for one sub-proof. +pub fn declare_table_arenas(b: &mut LfmBuilder, shape: &TableVerifyShape) -> TableQueryArenas { + let digest_words = super::edsl::digest_words(b) as usize; + let openings = b.declare_arena(shape.opening_words(digest_words) as u32); + let fri = b.declare_arena(shape.fri_words(digest_words) as u32); + let cap_words = shape.cap_words(digest_words); + TableQueryArenas { + openings, + fri, + caps: (cap_words > 0).then(|| b.declare_arena(cap_words as u32)), + } +} + +/// The cells one sub-proof's verification takes from OUTSIDE its own arenas. +pub struct TableInputs<'a> { + /// The precomputed-columns root, when the AIR is preprocessed. + /// + /// Production never reads this from the proof: it takes + /// `air.precomputed_commitment_for(layout)` (the root of the table's leaf + /// layout), absorbs THAT, and rejects a proof whose + /// copy disagrees (`verifier.rs:1184-1209`). So the cells here are the ones + /// Phase A absorbed, and the equality production checks explicitly is, in + /// this machine, the absence of a second value. + pub precomputed_root: Option<&'a RootCells>, + /// The main trace root — the cells Phase A absorbed. + pub main_root: &'a RootCells, + /// The shared LogUp challenges, sampled once in Phase A and passed to every + /// table (`verifier.rs:1216-1227`). Never per-table. + pub rap_challenges: &'a [Ext], + /// A bound sub-proof's commitment-time point `z0` and its answers — the + /// cells Phase A drew and absorbed, never a second hint. Present exactly + /// when the shape answers. + pub commit_ood: Option<(Ext, &'a [Ext])>, +} + +/// What one sub-proof's verification produced, for the epoch to compose. +pub struct TableVerifyOutput { + /// The recomputed composition at `z`, asserted equal to the claimed Horner + /// inside the program. + pub composition: Ext, + /// Per query, the FRI terminal value the chain arrived at. + pub fri_terminal: Vec, + /// The per-sub-proof DEEP invariants, exposed so a test can publish them. + pub deep: DeepInvariants, +} + +/// Emit one sub-proof's verification onto the spine's cells. +/// +/// `challenges` must be the output of [`super::epoch::emit_table_challenges`] on +/// THIS table's fork, and `absorbs` the very struct that call was given. Passing +/// a different one would be the two-consumer hazard reintroduced by hand, which +/// is why both are borrowed rather than rebuilt. +pub fn emit_table_verification( + b: &mut LfmBuilder, + shape: &TableVerifyShape, + analysis: &Analysis, + challenges: &TableChallenges, + absorbs: &TableAbsorbs<'_>, + inputs: &TableInputs<'_>, + arenas: &TableQueryArenas, +) -> TableVerifyOutput { + shape.check(); + assert_eq!( + challenges.iota_bits.len(), + shape.num_queries, + "one index per query" + ); + + // ---- the OOD grid, from the two blocks the transcript absorbed. + let grid = emit_reconstruct_ood(b, &shape.sub.deep, absorbs.ood_current, absorbs.ood_next); + + // ---- the LogUp uniforms, DERIVED (never hinted): the α powers from the one + // α Phase A sampled, and the per-row offset from the one `L` this table's + // fork absorbed and the closure sums. + let alpha_powers = if shape.num_alpha_powers > 0 { + let alpha = inputs + .rap_challenges + .get(stark::lookup::LOGUP_CHALLENGE_ALPHA) + .copied() + .expect("an AIR with bus elements has the shared LogUp challenges"); + emit_alpha_powers(b, alpha, shape.num_alpha_powers) + } else { + Vec::new() + }; + let table_offset = match absorbs.contribution { + Some(l) => emit_table_offset(b, l, shape.quotient.log2_trace_length), + // An AIR with no bus contribution has no `Op::TableOffset` to resolve; + // the pooled zero is a placeholder the lowering never reads. It is a + // program constant, so a prover cannot reach it either way. + None => b.felt_const(FE::zero()).as_ext(), + }; + + // The constraint program indexes FRAME STEPS; DEEP folds every grid row. + // Both views are of the one grid above, which is what makes the two legs + // agree by construction rather than by the host filling two arenas alike. + let steps = frame_step_view(&grid, shape.sub.deep.step_size); + assert_eq!( + steps.len(), + shape.num_frame_steps(), + "the strided view must have one entry per frame step" + ); + let ood = OodOperands { + steps, + main_width: shape.main_width, + rap_challenges: inputs.rap_challenges.to_vec(), + alpha_powers, + table_offset, + }; + + // ---- (1) the constraint evaluation and (2) the quotient check. + let evals = emit_analyzed(b, analysis, &ood); + let q = emit_quotient( + b, + &shape.quotient, + &ood, + challenges.z, + challenges.beta, + &evals, + absorbs.parts, + ); + b.assert_eq_ext(q.claimed, q.composition); + + // ---- (3) DEEP, over the same grid and the same parts. + let inv = emit_deep_invariants_bound( + b, + &shape.sub.deep, + challenges.gamma, + challenges.z, + &grid, + absorbs.parts, + inputs.commit_ood, + ); + + // ---- the committed matrices, in DEEP column order then the parts. + let groups = shape.sub.groups(); + let mut commitments: Vec = Vec::with_capacity(groups.len()); + let push = |root: &RootCells, out: &mut Vec| { + let g = groups[out.len()]; + out.push(GroupCommitment::from_lanes(root.lanes.clone(), g)); + }; + if let Some(prep) = inputs.precomputed_root { + push(prep, &mut commitments); + } + push(inputs.main_root, &mut commitments); + if let Some(aux) = absorbs.aux_root { + push(aux, &mut commitments); + } + push(absorbs.composition_root, &mut commitments); + assert_eq!( + commitments.len(), + groups.len(), + "one commitment per committed matrix: the sub-proof shape and the \ + supplied roots must describe the same proof" + ); + + // ---- the FRI commitments, likewise from the transcript's own cells. + let layers = absorbs + .fri_roots + .iter() + .map(|r| LayerCommitment::from_lanes(r.lanes.clone())) + .collect(); + let mut fri = FriCommitments::new( + b, + shape.fri, + layers, + challenges.zetas.clone(), + absorbs.fri_coeffs.to_vec(), + ); + + // ---- the Merkle caps, once per tree, against the SAME root cells the + // transcript absorbed: the matrices in group order, + // then the FRI layers. Every opening below is checked against these cells. + let digest_words = super::edsl::digest_words(b) as usize; + assert_eq!( + arenas.caps.is_some(), + shape.cap_words(digest_words) > 0, + "a caps arena exists exactly when the format caps some tree" + ); + if let Some(caps) = arenas.caps { + let mut at = 0u32; + for c in &mut commitments { + at = c.hint_cap(b, caps, at, shape.sub.trace_cap, shape.sub.merkle_depth); + } + assert_eq!(at as usize, shape.sub.cap_words(digest_words)); + let mut fri_at = at; + for (i, layer) in fri.layers.iter_mut().enumerate() { + fri_at = layer.hint_cap( + b, + caps, + fri_at, + shape.fri.layer_cap(i), + shape.fri.layer_depth(i), + ); + } + assert_eq!( + fri_at as usize, + shape.cap_words(digest_words), + "the caps arena is filled exactly" + ); + } + + // ---- (4) per query: authenticate, fold DEEP, then fold FRI. + let stride = shape + .sub + .opening_words(super::edsl::digest_words(b) as usize); + let mut fri_terminal = Vec::with_capacity(shape.num_queries); + for (qi, bits) in challenges.iota_bits.iter().enumerate() { + let mut cursor = (qi * stride) as u32; + let openings: Vec = groups + .iter() + .map(|g| { + let values = (0..shape.sub.group_values(g)) + .map(|_| { + let c = b.hint_word(arenas.openings, cursor); + cursor += 1; + c + }) + .collect(); + let siblings = (0..shape.sub.path_len()) + .map(|_| { + // The stride follows the DIGEST's width, not a literal. + let d = super::edsl::hint_digest(b, arenas.openings, cursor); + cursor += super::edsl::digest_words(b); + d + }) + .collect(); + GroupOpening { values, siblings } + }) + .collect(); + assert_eq!( + cursor as usize, + (qi + 1) * stride, + "the emitter's cursor must agree with the declared query stride" + ); + + let out = emit_query_from_bits( + b, + &shape.sub, + challenges.gamma, + &inv, + &commitments, + bits.clone(), + &openings, + ); + let layers = hint_layer_openings_from(b, shape.fri, arenas.fri, qi); + fri_terminal.push(emit_query_fri( + b, + shape.fri, + &fri, + &FriQuery { + p0: out.deep, + p0_sym: out.deep_sym, + point: out.point, + point_sym: out.point_sym, + bits: &out.bits, + }, + &layers, + )); + } + + TableVerifyOutput { + composition: q.composition, + fri_terminal, + deep: inv, + } +} + +/// The constraint frame's view of the reconstructed OOD grid: row 0 of each +/// evaluation STEP, which is every `step_size`-th grid row. +/// +/// This is assembly ledger entry 9, extracted so it can be differentialled. The +/// rule is production's, not ours: the verifier builds its frame with +/// `StarkTableView::into_frame(main_cols, step_size)` +/// (`verifier.rs:320-321`), which groups the `num_eval_points`-row grid into +/// `step_size`-row steps, and `Op::Var{offset, row}` resolves to +/// `frame.get_evaluation_step(offset).get_main_evaluation_element(0, col)` with +/// `row == 0` asserted (`constraint_ir/interp.rs:240-242`). So step `o`'s value is +/// grid row `o · step_size`, and the rows between are read by DEEP alone. +/// +/// A generic function rather than the two lines it replaces, because those two +/// lines had no witness: at `step_size = 1` the strided view and the whole grid +/// are the same vector, so an emitter that passed the whole grid to the +/// constraint fold — which is what the wave-5 sketch did — was indistinguishable. +/// `step_size_tests::the_frame_step_view_matches_productions_own_frame_assembly` +/// compares THIS function against `into_frame` at `step_size = 2`, where they +/// differ. +pub fn frame_step_view(grid: &[Vec], step_size: usize) -> Vec> { + assert!(step_size > 0, "a frame step is at least one row"); + assert!( + grid.len().is_multiple_of(step_size), + "the OOD grid is a whole number of frame steps: {} rows at step_size {step_size}", + grid.len() + ); + grid.iter().step_by(step_size).cloned().collect() +} + +/// Keccak permutations one sub-proof's committed leaves cost, per query. +/// +/// A leaf is NOT one permutation. It covers `ROWS_PER_LEAF · num_columns` +/// elements at 8 or 24 bytes each, and the sponge absorbs `⌊bytes/136⌋ + 1` rate +/// blocks — so the epoch's widest table (2,056 OOD columns) has a leaf worth +/// hundreds of permutations while a FRI layer's one-column leaf is worth one. +/// Predicting the leg's bill as "one leaf plus one per level" undercounts it by +/// the whole width of the trace, which is exactly the mistake this function +/// exists to not make. +pub fn leaf_permutations(shape: &SubProofShape) -> usize { + shape + .groups() + .iter() + .map(|g| super::keccak_host::num_blocks(g.leaf_bytes_at(shape.rows_per_leaf()))) + .sum() +} + +/// Felts a keccak permutation absorbs — the 136-byte rate at 8 bytes per felt. +/// +/// `⌊bytes/136⌋` and `⌊felts/17⌋` are the same function because every element the +/// leaf hasher streams is 8 bytes wide (a base felt) or 24 (an extension element, +/// three felts), so `bytes = 8 · felts` with no remainder either way. +pub const KECCAK_RATE_FELTS: usize = 17; + +/// Felts an `LFM_HASH` invocation absorbs — **one digest cell**. +/// +/// ⚠ **This was 8 and is now 4, because the construction it was derived from no +/// longer exists.** The old value was "2 of 3 state cells" — the rate of the +/// overwrite-duplex `edsl::SpongeVar` used to be. Option B1 replaced that with a +/// **compress chain** that absorbs exactly one cell per step, so the rate is +/// [`super::hash::HASH_DIGEST_FELTS`]. It is written as that constant rather +/// than as a literal so it cannot outlive its own derivation a second time. +/// +/// **This is 4.25× worse than keccak's 17**, not the 2.125× the duplex gave, +/// and it is the one axis on which a field-native candidate loses: it pays more +/// permutations, each far cheaper. +/// +/// **The lever moved with the construction, and it is now a worse one.** Under +/// the duplex the rate followed from `HASH_STATE_FELTS = 12`, so widening the +/// state bought throughput and nothing else. Under the chain it follows from +/// `HASH_DIGEST_FELTS = 4` — the absorbed operand IS a digest — so the only way +/// to raise it is to widen the digest, which is the same constant the socket's +/// 64-bit collision bound rests on. Throughput and collision resistance are no +/// longer independent knobs. +/// +/// ✗ **What this models, and its remaining assumption.** A rate is the right +/// shape for a CHAIN — `state ← T(state, cell)`, one fresh cell per invocation — +/// which is what the machine now has. It is *not* the right shape for a 2-to-1 +/// tree over leaf data, where both inputs are fresh and the count is a tree size +/// rather than `felts / rate`. Which of the two a candidate uses for LEAVES is +/// the still-open O1 leaf-convention question, so a future answer there could +/// move this model's FORMULA and not merely this constant. +pub const LFM_HASH_RATE_FELTS: usize = super::hash::HASH_DIGEST_FELTS; + +/// Felts in one FRI-layer leaf: the symmetric evaluation PAIR, two extension +/// elements of three felts each. +/// +/// Named because it is the number that decides whether the FRI leaf term is +/// rate-sensitive: six felts fit one keccak block (rate 17) and do NOT fit one +/// block at the candidate's rate 4. +pub const FRI_LEAF_FELTS: usize = 6; + +/// Felts one query's row-pair opening of a group covers, the felt-side +/// counterpart of [`super::sub_proof::GroupShape::leaf_bytes`]. +pub fn group_leaf_felts(g: &super::sub_proof::GroupShape) -> usize { + group_leaf_felts_at(g, super::sub_proof::ROWS_PER_LEAF) +} + +/// [`group_leaf_felts`] at `rows_per_leaf` rows per leaf (1 under S2's +/// one-row leaves) — what the closed forms price, at the sub-proof's layout. +pub fn group_leaf_felts_at(g: &super::sub_proof::GroupShape, rows_per_leaf: usize) -> usize { + g.values_at(rows_per_leaf) * if g.is_ext { 3 } else { 1 } +} + +/// Permutations a sponge of `rate_felts` spends absorbing `felts`, under keccak's +/// own padding convention — `⌊n/rate⌋ + 1`, i.e. always at least one block and +/// always a padding block even when the length divides the rate. +/// +/// Carrying keccak's convention over to a candidate is deliberately +/// CONSERVATIVE: a candidate needs no trailing block, because it +/// domain-separates outside the absorbed data — in the capacity for a sponge, +/// and in the message tag for the B1 compress chain — so its true count lies +/// between `felts.div_ceil(rate)` and this. Using the same convention on both +/// sides is +/// what makes the rate-17 case reproduce [`leaf_permutations`] exactly, which is +/// the check that this felt-side reformulation is right at all. +pub fn blocks_at_rate(felts: usize, rate_felts: usize) -> usize { + felts / rate_felts + 1 +} + +/// [`leaf_permutations`] at an arbitrary sponge rate. +/// +/// Computed independently of [`leaf_permutations`] — through felts and a rate +/// rather than through bytes and `keccak_host::num_blocks` — precisely so that +/// asserting the two agree at [`KECCAK_RATE_FELTS`] is a real differential and +/// not two spellings of one function. Making either delegate to the other would +/// kill that test silently (standing-decisions rule 7). +pub fn leaf_permutations_at_rate(shape: &SubProofShape, rate_felts: usize) -> usize { + shape + .groups() + .iter() + .map(|g| blocks_at_rate(group_leaf_felts_at(g, shape.rows_per_leaf()), rate_felts)) + .sum() +} + +/// FRI-layer LEAF permutations one query costs, at an arbitrary rate. +/// +/// Split out of [`super::fri::FriShape::permutations_per_query`] because it is +/// the half of that number which MOVES with the rate: a layer leaf is +/// [`FRI_LEAF_FELTS`] felts, and how many blocks that takes depends on the rate +/// like any other absorption. At keccak's 17 it is one block and this reduces to +/// `num_committed()`, which is what keeps the felt-side and byte-side closed +/// forms agreeing there. +/// +/// ⚠ It used to be folded into the rate-INVARIANT remainder, on the premise +/// that "a FRI layer leaf fits any rate at or above six". That premise was true +/// while the candidate's rate was 8 and is false now that it is 4 — six felts +/// take two blocks. The premise is gone rather than re-asserted; this function +/// is what replaced it. +pub fn fri_leaf_permutations_at_rate(fri: &FriShape, rate_felts: usize) -> usize { + // Per layer: the pair's six felts under `pair`, a `2^d`-value group's + // `3·2^d` under a fold schedule (`FriShape::layer_leaf_felts`). + (0..fri.num_committed()) + .map(|j| blocks_at_rate(fri.layer_leaf_felts(j), rate_felts)) + .sum() +} + +/// [`query_permutations`] at an arbitrary sponge rate. +/// +/// **ABSORPTION is rate-sensitive; COMPRESSION is not.** That is the whole +/// decomposition, and it is a property of what each step does rather than of +/// which leg it belongs to: +/// +/// - **Leaves absorb**, so both kinds move with the rate — the trace groups' +/// ([`leaf_permutations_at_rate`]) and the FRI layers' +/// ([`fri_leaf_permutations_at_rate`]). +/// - A **Merkle parent compresses**, so it is one permutation at any rate: +/// `LfmHasher::compress` is "a single permutation of `[a ‖ b ‖ IV]` truncated +/// to the first cell" (`hash.rs:23-26`), and keccak's 64-byte parent likewise +/// sits inside one 136-byte block (`edsl.rs:151-155`). Both the trace trees' +/// parents and the FRI layers' path steps are of this kind. +pub fn query_permutations_at_rate(shape: &TableVerifyShape, rate_felts: usize) -> usize { + let groups = shape.sub.groups().len(); + let per_query = leaf_permutations_at_rate(&shape.sub, rate_felts) + + fri_leaf_permutations_at_rate(&shape.fri, rate_felts) + + groups * shape.sub.path_permutations() + + shape.fri.path_steps_per_query(); + shape.num_queries * per_query +} + +/// Felts a BLAKE3 block holds: 64 bytes at 8 bytes per felt. +pub const BLAKE3_BLOCK_FELTS: usize = 8; + +/// Compressions absorbing `felts` costs under `hash`, with each hash's OWN +/// block rule. +/// +/// The two rules genuinely differ and the difference is not a rate change. +/// Keccak's `⌊n/rate⌋ + 1` always spends a trailing block, because `pad10*1` +/// appends one even when the length divides the rate. `Blake3Chain` does not: +/// an exact multiple of 64 emits no spurious final block — that is KAT K5, and +/// the P2 parent identity depends on it. So BLAKE3 is `max(1, ⌈n/8⌉)`, with the +/// `max` carrying the other half of the same fact: the empty message is ONE +/// block, not zero. +pub fn blocks_for(felts: usize, hash: WrapHash) -> usize { + match hash { + WrapHash::Keccak => blocks_at_rate(felts, KECCAK_RATE_FELTS), + WrapHash::Blake3 => felts.div_ceil(BLAKE3_BLOCK_FELTS).max(1), + // ✓ PROVEN identical to the BLAKE3 arm for every leaf of one felt or + // more (`rpo_chip_tests::the_rate_eight_census_is_hash_invariant`): + // the rate is the same eight felts and neither spends a trailing block + // on an exact multiple. Written as its own arm rather than merged, so + // the equality stays a checked property instead of an assumption. + WrapHash::Algebraic => felts.div_ceil(super::rpo::RATE_FELTS).max(1), + // ZisK's leaf absorbs twelve felts a permutation, and an empty leaf + // never permutes (`super::p1w16_emit::leaf_hash`). + WrapHash::Poseidon1 => felts.div_ceil(super::p1w16_emit::RATE_FELTS), + } +} + +/// [`query_permutations`] under an explicit wrap hash. +/// +/// Same decomposition as [`query_permutations_at_rate`] — **absorption moves +/// with the block, compression does not** — but it takes the hash rather than a +/// rate, because the block COUNT rule is part of the hash and not a parameter +/// of it (see [`blocks_for`]). A Merkle parent is 64 bytes and is exactly one +/// compression under both, so every parent term is unchanged; only the leaf +/// absorptions move. +pub fn query_permutations_for(shape: &TableVerifyShape, hash: WrapHash) -> usize { + let groups = shape.sub.groups().len(); + let rows = shape.sub.rows_per_leaf(); + let leaves: usize = shape + .sub + .groups() + .iter() + .map(|g| blocks_for(group_leaf_felts_at(g, rows), hash)) + .sum(); + // Per committed layer: a pair leaf (six felts), or a `2^d`-value group. + let fri_leaves = shape.fri.leaf_permutations_per_query(hash); + let per_query = leaves + + fri_leaves + + groups * shape.sub.path_permutations() + + shape.fri.path_steps_per_query(); + shape.num_queries * per_query +} + +/// Permutations one sub-proof's Merkle cap checks cost, ONCE per sub-proof +/// (not per query): every capped tree hashes its `2^c` cap up to its root, +/// `2^c − 1` parents. Zero at the +/// default format. +pub fn cap_permutations(shape: &TableVerifyShape) -> usize { + shape.sub.cap_permutations() + shape.fri.cap_permutations() +} + +/// Every permutation one sub-proof's verification legs cost: +/// [`query_permutations_for`] (per query, with the capped path lengths) plus +/// [`cap_permutations`] (once). This is the closed form the emitted legs are +/// pinned against at every format; at the default it IS +/// [`query_permutations_for`]. +pub fn table_permutations_for(shape: &TableVerifyShape, hash: WrapHash) -> usize { + query_permutations_for(shape, hash) + cap_permutations(shape) +} + +/// Keccak permutations one sub-proof's whole query verification costs, from +/// shape alone. +/// +/// Per query: every group's leaf ([`leaf_permutations`]), one permutation per +/// Merkle level per group (a parent hashes 64 bytes, one rate block), and the +/// FRI leg's own [`FriShape::permutations_per_query`]. A closed form over the +/// shapes, so comparing it against the emitted count is an absolute check and +/// not a difference of two of our own emitter passes. +pub fn query_permutations(shape: &TableVerifyShape) -> usize { + let groups = shape.sub.groups().len(); + let per_query = leaf_permutations(&shape.sub) + + groups * shape.sub.path_permutations() + + shape.fri.permutations_per_query(); + shape.num_queries * per_query +} + +/// The boundary constraints every production VM table carries, as program shape. +/// +/// `AIR::boundary_constraints` is a function of the public inputs, so it is not a +/// static property of the AIR and cannot be captured into a +/// `ConstraintArtifact`. What IS static — and is checked against every AIR of a +/// real epoch by `epoch_verify_tests::the_boundary_terms_are_program_shape` — is +/// that a table with an aux trace carries exactly the framework's `acc[0] = 0` on +/// its last aux column and nothing else: a zero VALUE at the trace generator's +/// zeroth power, neither of which depends on a challenge or on the proof. +/// +/// Building the list from that rule rather than from the call is what keeps the +/// emitted program independent of the proof it verifies; the test is what stops +/// the rule from silently ceasing to hold. Note which direction the risk runs: a +/// boundary term this rule MISSED would be a constraint the machine never +/// checks, so the test asserts set equality and not containment. +pub fn boundary_terms(has_aux_trace: bool, num_total_cols: usize) -> Vec { + if has_aux_trace { + vec![BoundaryTerm { + col: num_total_cols - 1, + point: FE::one(), + value: FEE::zero(), + }] + } else { + Vec::new() + } +} diff --git a/prover/src/lfm/epoch_verify_tests.rs b/prover/src/lfm/epoch_verify_tests.rs new file mode 100644 index 000000000..de8e467f0 --- /dev/null +++ b/prover/src/lfm/epoch_verify_tests.rs @@ -0,0 +1,1358 @@ +//! ★ The assembled epoch verifier — spine plus legs — run on a real +//! continuation epoch proof. +//! +//! [`super::epoch_tests`] built the Fiat-Shamir spine and checked all 79 of a +//! real 16-sub-proof epoch's challenges against production's own replay. Every +//! verification leg, meanwhile, was driven by its own isolation program with +//! HINTED challenges. This module hangs the legs off the spine: per sub-proof the +//! OOD grid is rebuilt from the two pruned blocks the transcript absorbed, the +//! constraint evaluation and quotient check run at the spine's `z` and `β`, and +//! each query's index bits go straight from `TableChallenges::iota_bits` into the +//! Merkle walk, the DEEP fold and the FRI chain. +//! +//! ## The oracle, and what is left of it +//! +//! There is deliberately LESS oracle here than in any leg suite, and that is the +//! point. A leg suite checks a computed value against production's own answer for +//! the same inputs. Here the checks are INSIDE the program: the quotient check is +//! `assert_eq_ext(claimed, composition)`, every Merkle walk ends in +//! `assert_word_eq_lanes` against a root the transcript absorbed, and the FRI +//! chain ends in `assert_eq_ext` against the terminal polynomial. A program that +//! executes at all has passed them. So the differential that remains is the +//! spine's — the 79 challenges, still checked — plus the fact of execution, and +//! the falsification tests below are what turn "it executed" into evidence, by +//! showing what does NOT execute. +//! +//! ## What this suite cannot see +//! +//! The preset. The fixture epoch is proved at the MIN preset (blowup 2, one +//! query per table, grinding factor 1), because that is what +//! `proof_fixture::fixture_options` gives and what keeps a 16-sub-proof epoch +//! provable in a unit test. Every per-query cost here is therefore ONE query's, +//! and the blowup-8 predictions the phase pinned (73 queries, 14,454 FRI +//! permutations per sub-proof) are reached by scaling, not by measurement — the +//! scaling factors are stated in [`the_assembled_epoch_verifier_runs`]'s output +//! rather than hidden in a comment. It also cannot see PAGE's preprocessed +//! commitment problem (ledger entry 7), which is about where a root COMES from +//! and not about what is done with it. + +use stark::config::Commitment; +use stark::proof::view::StarkProofView; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::epoch_verify::{TableVerifyShape, boundary_terms}; +use super::executor::execute; +use super::fri::FriShape; +use super::word::word_as_ext; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// Everything the verification legs read about one real sub-proof (the reader +/// lives in [`super::harvest`], where the block tree's driver uses it too). +pub(super) use super::harvest::TableLegs; + +/// [`super::harvest::build_table_legs`], panicking on a proof that disagrees +/// with its AIR. +pub(super) fn build_table_legs( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + rap_challenges: &[FEE], +) -> TableLegs { + super::harvest::build_table_legs(air, view, rap_challenges).unwrap_or_else(|e| panic!("{e}")) +} + +/// [`super::epoch_verify::layout_precomputed_commitment`], where a layout +/// with no root is a hard error. At row pairs it IS +/// `air.precomputed_commitment()`. +pub(super) fn layout_precomputed_commitment( + air: &dyn AIR, + trace_length: usize, +) -> Commitment { + super::harvest::layout_precomputed_commitment(air, trace_length) + .unwrap_or_else(|e| panic!("{e}")) +} + +/// [`super::harvest::fri_layer_openings`], panicking on a proof that disagrees +/// with its FRI shape. +#[allow(clippy::type_complexity)] +pub(super) fn fri_layer_openings( + view: StarkProofView<'_, Gl, Ext3, PI>, + fri: FriShape, +) -> (Vec, Vec)>>, Vec) +where + PI: rkyv::Archive, + ::Archived: rkyv::Deserialize, +{ + super::harvest::fri_layer_openings(view, fri).unwrap_or_else(|e| panic!("{e}")) +} + +/// ★ THE RUN: the whole epoch verifier — spine AND legs — on a real +/// continuation epoch proof that production accepts. +/// +/// What executing proves, stated precisely. Every check is an assert inside the +/// program, so reaching the end means: all 16 quotient identities held at the +/// spine's own `z` and `β`; every one of the 16 sub-proofs' opened row pairs +/// hashed to a leaf that walked to the root the transcript absorbed, at the index +/// the transcript sampled; every DEEP reconstruction fed a FRI chain that folded +/// to the terminal polynomial the transcript absorbed; and the LogUp closure +/// reached production's COMMIT-bus target. The 79 published challenges are +/// checked against production's replay on top, so the Fiat-Shamir the whole thing +/// hangs from is still differentialled. +#[test] +fn the_assembled_epoch_verifier_runs() { + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + // ★ The PINNED socket permutation, not a literal. `epoch_program` builds at + // `WrapHash::production()`, and the classification rule is that such a + // program emits `Instr::Hash` and must run under `BLOCK_HASHER`; only a + // program pinning a byte hash on its own builder may take the default. + // + // ⚠ Under a BYTE pin this is inert — `ByteWrapHash` lowers to the KECCAK / + // `LFM_BLAKE3` chips and emits no `Instr::Hash`, so the socket is never + // consulted and a toy permutation was free and correct. Under an ALGEBRAIC + // pin the walks ARE `Instr::Hash`: a toy would rebuild roots the host never + // committed, and this test would fail on its HONEST path, naming nothing. + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the assembled verifier must execute"); + + // ---- the spine's differential, unchanged: production's own challenges. + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the shared LogUp challenge z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the shared LogUp challenge alpha"); + + // The attestation fold is published right after Phase A (two digest words), + // and its DECODE input is the cell Phase A absorbed — the join ledger entry 7 + // rests on. Its value is differentialled in the spine test; here it only has to + // be skipped, and skipped by NAME rather than by a literal. + let program_id_words = 2usize; + // ★ Then the BLOCK-BINDING SCHEMA — the register boundary vectors, the epoch + // label, the public-output halves and the L2G re-commit root. Skipped by NAME + // (`epoch_tests::schema_words`) rather than by a literal, for the same reason + // the id is: a literal here would start checking `beta of table 0` against a + // register slot the moment the schema moves, and would report a pass while + // doing it. Its VALUES are the aggregator's subject and are differentialled + // there; this gate only has to walk past them and still account for every word. + let schema_words = super::epoch_tests::schema_words(&e); + let mut cursor = 2 + program_id_words + schema_words; + let mut checked = 2usize; + for (i, (h, leg)) in e.tables.iter().zip(&e.legs).enumerate() { + // The legs publish first: the recomputed composition, then a terminal + // value per query. + cursor += 1 + leg.verify.num_queries; + assert_eq!(pub_ext(cursor), h.beta, "beta of table {i}"); + assert_eq!(pub_ext(cursor + 1), h.z, "z of table {i}"); + assert_eq!(pub_ext(cursor + 2), h.gamma, "gamma of table {i}"); + cursor += 3; + checked += 3; + for (k, want) in h.zetas.iter().enumerate() { + assert_eq!(pub_ext(cursor + k), *want, "zeta {k} of table {i}"); + } + cursor += h.zetas.len(); + checked += h.zetas.len(); + for q in 0..h.shape.num_queries { + let w = exec.public_words[cursor + q].1; + let got = super::word::word_as_base(&w).expect("an index is a base felt"); + assert_eq!(got, FE::from(h.iotas[q] as u64), "iota {q} of table {i}"); + } + cursor += h.shape.num_queries; + checked += h.shape.num_queries; + } + // ★ A literal on purpose — deriving the expectation from `e.tables` would + // restate the loop above and check nothing. What moved it from 111 is + // named rather than absorbed: an intermediate epoch was 24 sub-proofs when + // 111 was written, and each always-on RV64 table (the campaign's HINT, + // then #903's BLAKE3) adds one sub-proof and, at the MIN preset, its four + // challenges — a (β, z, γ) triple and one query index, with no DEEP zeta + // because a near-empty fixed table opens nothing. The accounting is + // asserted, so the next always-on table fails here saying which input + // moved instead of leaving a bare literal to re-bless. + // + // It has now moved the other way, by the same accounting: BLAKE3 stopped + // being always-on (it is `TableCounts::blake3`, and this epoch does not use + // it), so 26 → 25 and 119 → 115. This epoch is exactly the workload the + // bisect priced — a near-empty always-on table is four challenges here and + // +31.2% of the wrap's cells at the secure preset. + // + // ★ And then FURTHER the same way, for the same reason at six times the + // size. + // + // MOVER: `892c7d1bc` (main's `c2ac5d546`, #977) — arm (i), the TABLE SET. + // That merge's two arms are (i) the table set shrank, empty tables now + // being elided rather than padded, and (ii) the absorbed epoch statement + // grew 49 bytes. This pin is on (i); `blake3_chip_tests`' + // STATEMENT_REPLAY_BLAKE3_ROWS and `machine_tests`' statement byte length + // are on (ii). + // + // #977 did to the six accelerator + // chips what #903's revert did to BLAKE3: `FIXED_TABLE_COUNT` 11 → 5, with + // `commit`, `keccak`, `keccak_rnd`, `ecsm`, `ecdas` and `hint` becoming + // `TableCounts` fields that a run which never reaches them reports as zero. + // This fixture epoch reaches none of the six, so 25 → 16 and 115 → 79. + // Six of the nine are those accelerators: the pre-#977 25 was 14 split + // families + 10 intermediate fixed (11 less HALT) + 1 L2G_MEMORY, and the + // fixed term is now 4, which lands at 19. The remaining three are split + // families this fixture leaves empty, and they are NOT named here because + // the measurement does not name them — only `e.tables.len()` is read. + // + // The accounting below is what says the whole move is empty tables at four + // challenges each: every one of them opens nothing, so it contributes a + // (β, z, γ) triple and one query index and no DEEP zeta, exactly as an + // always-on table did. If that model is wrong for any of the nine, the + // `checked` assertion two below fails and says so. + // + // The identity is spelled with both sides positive because `SUB_PROOFS` is + // now BELOW the 24 it used to be measured against, and `SUB_PROOFS - 24` on + // a `usize` is an underflow rather than a failed assertion. Moved this way + // it still fails — in either direction — if the four-challenge model stops + // describing the move. + // + // `LFM_BLAKE3` (P-a Stage 5) does NOT appear in either number: it is a chip + // of the LFM machine, counted by `NUM_LFM_CHIPS`, and this is the RV64 + // epoch the LFM machine verifies. + const SUB_PROOFS: usize = 16; + const CHALLENGES_AT_MIN_PRESET: usize = 79; + const CHALLENGES_PER_ALWAYS_ON_TABLE: usize = 4; + assert_eq!( + e.tables.len(), + SUB_PROOFS, + "the epoch's sub-proof count is what the challenge count below is a \ + function of" + ); + assert_eq!( + CHALLENGES_AT_MIN_PRESET + CHALLENGES_PER_ALWAYS_ON_TABLE * 24, + 111 + CHALLENGES_PER_ALWAYS_ON_TABLE * SUB_PROOFS, + "the tables that left the proof account for the whole move from the \ + original 111" + ); + assert_eq!( + checked, CHALLENGES_AT_MIN_PRESET, + "the same challenges the spine test checks must still be checked" + ); + assert_eq!( + word_as_ext(&exec.public_words[cursor].1).expect("the bus total is ext"), + e.expected_bus_balance, + "the LogUp closure must reach production's own COMMIT-bus target" + ); + assert_eq!( + cursor + 1, + exec.public_words.len(), + "every published word must be accounted for" + ); + + // ---- THE MEASUREMENT ---- + let spine = super::epoch_tests::epoch_program(&e, false); + // The CONFIGURED hash's compressions — the closed form counts Merkle + // levels and query paths, which is hash-independent, so counting keccak + // specifically read zero under BLAKE3. + let perms = |p: &_| super::machine_tests::wrap_hash_instrs(p); + let words = |p: &super::compiler::LfmProgram| -> usize { + p.arena_schema.lens.iter().map(|l| *l as usize).sum() + }; + + let queries = e.legs[0].verify.num_queries; + let opening_perms = perms(&program) - perms(&spine); + let legs_published: usize = e.legs.iter().map(|l| 1 + l.verify.num_queries).sum(); + println!( + "\n★ ASSEMBLED EPOCH VERIFIER (min preset: blowup 2, {queries} quer\ + {}/table, grinding {}):\n\ + \x20 spine +legs legs alone\n\ + \x20 instructions {:>10} {:>10} {:>10}\n\ + \x20 keccak perms {:>10} {:>10} {:>10}\n\ + \x20 arena words {:>10} {:>10} {:>10}\n\ + \x20 published {:>10} {:>10} {:>10}", + if queries == 1 { "y" } else { "ies" }, + e.tables[0].shape.grinding_factor, + spine.instrs.len(), + program.instrs.len(), + program.instrs.len() - spine.instrs.len(), + perms(&spine), + perms(&program), + opening_perms, + words(&spine), + words(&program), + words(&program) - words(&spine), + // The spine's own published count. Was `len - (x - x)` — a leftover that + // printed the assembled figure in the spine column. + exec.public_words.len() - legs_published, + exec.public_words.len(), + legs_published, + ); + + // ---- the constraint leg's share, from the analyses themselves. + // + // `Analysis::report` is the count of what the lowering pass DID, and + // `emit_analyzed` runs over the very analysis reported here — the module's own + // doc comment makes that a construction, not a coincidence — so summing the + // reports attributes the constraint evaluation inside the assembled program + // without a second emitter pass. `alu_rows` excludes constants because the + // builder interns them program-wide, so the sum is a lower bound on the + // constraint leg's instructions and not the whole of it. + let constraint_alu: usize = e.legs.iter().map(|l| l.analysis.report().alu_rows()).sum(); + let constraint_unfused: usize = e + .legs + .iter() + .map(|l| l.analysis.report().unfused_alu_rows()) + .sum(); + // The recombination half, measured in ISOLATION against its own plumbing + // baseline and compared against a number that did not come from this emitter + // (`others/lfm-constraint-lowering-design.md:604` splits the pinned 57,252 + // into 54,358 lowering + 2,894 recombination). That is what makes a + // two-pass difference admissible here — the comparison target is external. + let recombination: usize = e + .legs + .iter() + .map(|l| { + let plumb = |b: &mut super::builder::LfmBuilder| { + let n = 2 + + l.verify.sub.deep.num_composition_parts + + l.verify.num_frame_steps() * l.verify.sub.deep.num_total_cols + + l.analysis.report().nodes; + let a = b.declare_arena(n as u32); + let mut i = 0u32; + let mut take = |b: &mut super::builder::LfmBuilder| { + let c = b.hint_word(a, i).as_ext(); + i += 1; + c + }; + let z = take(b); + let beta = take(b); + let parts: Vec<_> = (0..l.verify.sub.deep.num_composition_parts) + .map(|_| take(b)) + .collect(); + let steps: Vec> = (0..l.verify.num_frame_steps()) + .map(|_| { + (0..l.verify.sub.deep.num_total_cols) + .map(|_| take(b)) + .collect() + }) + .collect(); + // One evaluation cell per constraint root, which is what + // `emit_analyzed` returns and `emit_quotient` folds. + let evals: Vec<_> = (0..l.analysis.program().roots.len()) + .map(|_| take(b)) + .collect(); + (z, beta, parts, steps, evals) + }; + let mut bare = super::builder::LfmBuilder::new() + .with_wrap_hash(super::edsl::WrapHash::production()); + let _ = plumb(&mut bare); + let baseline = bare.finish().instrs.len(); + + let mut full = super::builder::LfmBuilder::new() + .with_wrap_hash(super::edsl::WrapHash::production()); + let (z, beta, parts, steps, evals) = plumb(&mut full); + let ood = super::constraints::OodOperands { + steps, + main_width: l.verify.main_width, + rap_challenges: Vec::new(), + alpha_powers: Vec::new(), + table_offset: z, + }; + super::constraints::emit_quotient( + &mut full, + &l.verify.quotient, + &ood, + z, + beta, + &evals, + &parts, + ); + full.finish().instrs.len() - baseline + }) + .sum(); + println!( + "\x20 constraint leg inside the assembled verifier: {constraint_alu} ALU \ + rows lowering ({constraint_unfused} unfused) + {recombination} \ + recombination = {} over 16 sub-proofs [pinned: see the run output]\ + \n\x20 that is {:.1}% of the legs' {} instructions", + constraint_alu + recombination, + 100.0 * (constraint_alu + recombination) as f64 + / (program.instrs.len() - spine.instrs.len()) as f64, + program.instrs.len() - spine.instrs.len(), + ); + + // ---- the permutation bill, against a CLOSED FORM over the shapes. + // + // Not a difference of two emitter passes (which rule 7's refinement rules + // out) but arithmetic over byte widths: every group's leaf costs the + // configured hash's block count, every Merkle level is ONE compression + // under EVERY hash — a byte parent is 64 bytes, inside keccak's rate and + // exactly one BLAKE3 block, and an algebraic parent's two digest cells fill + // the rate-8 sponge exactly — and FRI splits the same way. + // Asserted, not printed, so a leg that silently stopped hashing a group + // would fail here. + // + // The leaf and FRI-leaf halves are the ones that move with the hash — + // absorption is block-sensitive, compression is not — so they go through + // `blocks_for` while the walk terms stay plain counts. + use super::epoch_verify::blocks_for; + let hash = super::edsl::WrapHash::production(); + let mut fri_perms = 0usize; + let mut leaf_perms = 0usize; + let mut walk_perms = 0usize; + for leg in &e.legs { + let groups = leg.verify.sub.groups().len(); + let fri_leaves = + leg.verify.fri.num_committed() * blocks_for(super::epoch_verify::FRI_LEAF_FELTS, hash); + fri_perms += leg.verify.num_queries * (fri_leaves + leg.verify.fri.path_steps_per_query()); + leaf_perms += leg.verify.num_queries + * leg + .verify + .sub + .groups() + .iter() + .map(|g| blocks_for(super::epoch_verify::group_leaf_felts(g), hash)) + .sum::(); + walk_perms += leg.verify.num_queries * groups * leg.verify.sub.merkle_depth; + } + let predicted: usize = e + .legs + .iter() + .map(|l| { + super::epoch_verify::query_permutations_for( + &l.verify, + super::edsl::WrapHash::production(), + ) + }) + .sum(); + assert_eq!( + predicted, + leaf_perms + walk_perms + fri_perms, + "the closed form must decompose into exactly its three parts" + ); + assert_eq!( + opening_perms, predicted, + "the emitted permutation count must equal the closed form over the shapes" + ); + println!( + "\x20 leg permutations = {leaf_perms} leaves + {walk_perms} Merkle levels \ + + {fri_perms} FRI = {predicted} (closed form) = {opening_perms} (emitted)" + ); + println!( + "\x20 FRI layers committed across the epoch: {} | widest leaf: {} bytes", + e.legs + .iter() + .map(|l| l.verify.fri.num_committed()) + .sum::(), + e.legs + .iter() + .flat_map(|l| l.verify.sub.groups()) + .map(|g| g.leaf_bytes()) + .max() + .expect("the epoch has groups") + ); + + // ---- RECONCILIATION with the phase's pinned blowup-8 predictions. + // + // The pinned 213,744 came from `join_tests::join_leg_cost`, whose stated + // assumptions are: all 28 PRODUCTION AIRs, every trace at a UNIFORM + // 2^20, blowup 8, 73 queries, and NO FRI (the joined leg has none). The + // measurement above is: this epoch's 16 sub-proofs, at their REAL trace + // lengths, blowup 2, one query, FRI included. Three parameters differ, so + // the two numbers cannot be compared directly — they are projected onto each + // other one parameter at a time instead, which is also what says which + // assumption carries the difference. + let at_blowup_8 = |leg: &TableLegs, uniform_log2_trace: Option| -> TableVerifyShape { + let log2_trace = uniform_log2_trace.unwrap_or(leg.verify.sub.deep.log2_trace_length); + let log2_lde = log2_trace + 3; + let mut out = leg.verify.clone(); + out.sub.log2_lde_length = log2_lde; + out.sub.merkle_depth = log2_lde as usize - 1; + out.sub.deep.log2_trace_length = log2_trace; + out.quotient.log2_trace_length = log2_trace; + out.fri = FriShape { + log2_lde_length: log2_lde, + blowup_log: 3, + num_queries: 73, + ..leg.verify.fri + }; + out.num_queries = 73; + out + }; + let openings_only = |s: &TableVerifyShape| -> usize { + s.num_queries + * (super::epoch_verify::leaf_permutations(&s.sub) + + s.sub.groups().len() * s.sub.merkle_depth) + }; + + let real_lengths: Vec = e.legs.iter().map(|l| at_blowup_8(l, None)).collect(); + let uniform: Vec = e.legs.iter().map(|l| at_blowup_8(l, Some(20))).collect(); + let sum = |v: &[TableVerifyShape], f: &dyn Fn(&TableVerifyShape) -> usize| -> usize { + v.iter().map(f).sum() + }; + + // ---- THE HASH MATRIX'S PERMUTATION AXIS, at the production shape. + // + // A candidate hash moves two independent things: cells per permutation (its + // AIR's shape, which needs the AIR) and permutations per verify (the sponge's + // rate, which needs only arithmetic over these shapes). This block pins the + // second WITHOUT any candidate permutation existing, so the remaining unknown + // in a candidate's predicted column is one factor and not two. + // + // The differential: `query_permutations_at_rate` is written through felts and a + // rate, `query_permutations` through bytes and `keccak_host::num_blocks`. + // Neither delegates to the other, so their agreement at rate 17 is a real check + // on the felt-side reformulation — and the existing assert above already ties + // `query_permutations` to the EMITTED count, so the chain reaches the emitter. + use super::epoch_verify::{ + FRI_LEAF_FELTS, KECCAK_RATE_FELTS, LFM_HASH_RATE_FELTS, blocks_at_rate, + fri_leaf_permutations_at_rate, group_leaf_felts, query_permutations_at_rate, + }; + for s in &real_lengths { + assert_eq!( + query_permutations_at_rate(s, KECCAK_RATE_FELTS), + super::epoch_verify::query_permutations(s), + "the felt-side closed form must reproduce the byte-side one at keccak's rate" + ); + } + // ⚠ A FRI layer leaf is six felts. It fits ONE keccak block and it does NOT + // fit one block at the candidate's rate — which was 8 under the deleted + // three-cell duplex and is 4 under the B1 compress chain. So the FRI leaf + // term is rate-SENSITIVE and is no longer part of the invariant remainder. + // The old premise assertion here (`6 <= LFM_HASH_RATE_FELTS`) is gone: it + // was true at 8, is false at 4, and re-asserting it would have pinned the + // model to a construction that no longer exists. + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, KECCAK_RATE_FELTS), 1); + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, LFM_HASH_RATE_FELTS), 2); + + let keccak_p = sum(&real_lengths, &|s| { + query_permutations_at_rate(s, KECCAK_RATE_FELTS) + }); + let cand_p = sum(&real_lengths, &|s| { + query_permutations_at_rate(s, LFM_HASH_RATE_FELTS) + }); + // Decompose so the penalty is attributed rather than asserted in aggregate. + // The split is ABSORPTION vs COMPRESSION, not leaf vs rest: leaves of both + // kinds absorb and move with the rate, Merkle parents of both kinds + // compress and do not. + let absorb_at = |rate: usize| { + sum(&real_lengths, &|s| { + s.num_queries + * (super::epoch_verify::leaf_permutations_at_rate(&s.sub, rate) + + fri_leaf_permutations_at_rate(&s.fri, rate)) + }) + }; + let leaf_k = absorb_at(KECCAK_RATE_FELTS); + let leaf_c = absorb_at(LFM_HASH_RATE_FELTS); + let paths = keccak_p - leaf_k; + assert_eq!( + cand_p, + leaf_c + paths, + "only ABSORPTION may move with the rate; compression (Merkle parents, \ + trace trees and FRI path steps alike) must not" + ); + assert!( + cand_p > keccak_p, + "the candidate's smaller rate must COST permutations — if this ever fails, \ + the rate penalty reasoning in others/lfm-hash-matrix-scope.md is wrong" + ); + let widest = real_lengths + .iter() + .map(|s| { + s.sub + .groups() + .iter() + .map(group_leaf_felts) + .max() + .unwrap_or(0) + }) + .max() + .expect("the epoch has groups"); + println!( + "\n ★ HASH MATRIX — the PERMUTATION axis at blowup 8 / 73 queries, real \ + trace lengths (no candidate permutation exists yet; this is shape \ + arithmetic only):\n\ + \x20 keccak rate {KECCAK_RATE_FELTS:>2} felts/perm: {keccak_p:>9} permutations \ + ({leaf_k} absorbed + {paths} compressed)\n\ + \x20 LFM_HASH rate {LFM_HASH_RATE_FELTS:>2} felts/perm: {cand_p:>9} permutations \ + ({leaf_c} absorbed + {paths} compressed)\n\ + \x20 candidate/keccak = {:.4}x (absorption term alone {:.4}x; the \ + ceiling is 17/{LFM_HASH_RATE_FELTS} = {:.3}x and only absorption pays it)\n\ + \x20 absorption-bound share of the keccak bill: {:.1}% widest leaf: \ + {widest} felts\n\ + \x20 ⚠ the candidate rate is 4, not the 8 this model carried before \ + option B1 — the compress chain absorbs ONE cell per step, so both the \ + trace-group leaves and the 6-felt FRI-layer leaves take two blocks", + cand_p as f64 / keccak_p as f64, + leaf_c as f64 / leaf_k as f64, + KECCAK_RATE_FELTS as f64 / LFM_HASH_RATE_FELTS as f64, + 100.0 * leaf_k as f64 / keccak_p as f64, + ); + + println!( + "\n RECONCILIATION against the pinned blowup-8 predictions (projections \ + from shapes — this run is at the min preset and measures none of them):\n\ + \x20 openings only, 73 queries, UNIFORM 2^20 (deep-join's own \ + assumption, over this epoch's 16 sub-proofs): {} [pinned: see the run output \ + over all 28 production AIRs]\n\ + \x20 openings only, 73 queries, this epoch's REAL trace lengths: {}\n\ + \x20 openings + FRI, 73 queries, real lengths: {}\n\ + \x20 FRI alone, 73 queries, real lengths: {} [pinned: 14,454 per \ + sub-proof at blowup 8, i.e. for a 2^20 table]", + sum(&uniform, &openings_only), + sum(&real_lengths, &openings_only), + sum(&real_lengths, &|s| super::epoch_verify::query_permutations( + s + )), + sum(&real_lengths, &|s: &TableVerifyShape| s.num_queries + * s.fri.permutations_per_query()), + ); + // The one sub-proof that IS a 2^20 table, so the per-sub-proof FRI figure the + // FRI leg pinned has something to be checked against. + let biggest = e + .legs + .iter() + .max_by_key(|l| l.verify.sub.deep.log2_trace_length) + .expect("the epoch has sub-proofs"); + let big8 = at_blowup_8(biggest, None); + println!( + "\x20 the epoch's 2^{} sub-proof at blowup 8: FRI {} permutations \ + ({} committed layers), openings {}", + big8.sub.deep.log2_trace_length, + big8.num_queries * big8.fri.permutations_per_query(), + big8.fri.num_committed(), + openings_only(&big8), + ); + println!("\x20 trace lengths in this epoch (log2): {:?}", { + let mut v: Vec = e + .legs + .iter() + .map(|l| l.verify.sub.deep.log2_trace_length) + .collect(); + v.sort_unstable(); + v + }); +} + +/// Where each per-table arena sits in the declaration order +/// `epoch_tests::epoch_arena_words` produces. +/// +/// Computed from the presence flags rather than hardcoded, because a table +/// without an aux root or without grinding shifts every arena behind it — which +/// is precisely the failure mode the per-field arena packing exists to prevent +/// and a hardcoded index would reintroduce in the TEST. +pub(super) struct ArenaIndex { + pub(super) openings: usize, + fri: usize, + parts: usize, + ood_current: usize, +} + +pub(super) fn arena_index(e: &super::epoch_tests::RealEpoch, table: usize) -> ArenaIndex { + // The epoch-wide arenas come first, and their COUNT comes from the emitter's + // own side rather than from a literal here: wiring ledger entry 7 added the + // second register vector, `pc_start` and (when non-empty) the page roots, and a + // literal `4` would have left every vector below tampering the wrong arena. + let mut at = super::epoch_tests::num_epoch_wide_arenas(e); + for (i, h) in e.tables.iter().enumerate() { + let aux = usize::from(h.shape.has_aux_root); + let contribution = usize::from(h.shape.has_contribution); + let nonce = usize::from(h.shape.grinding_factor > 0); + let composition = at + aux + contribution; + if i == table { + return ArenaIndex { + ood_current: composition + 1, + parts: composition + 3, + openings: composition + 6 + nonce, + fri: composition + 7 + nonce, + }; + } + // The table's last arena is `fri` at `composition + 7 + nonce`, so the + // next table starts one past it. Getting this stride wrong is how the + // first version of this test came to tamper an EMPTY arena two tables + // later — which is why the loop below checks every computed index + // against the arena lengths the shapes fix. + at = composition + 8 + nonce; + } + unreachable!("table index out of range"); +} + +/// ★ FALSIFICATION: run the attacks the wiring denies, and watch each fail. +/// +/// Every check the legs add is an `assert` inside the program, so "it executed" +/// is the whole positive result — which makes this test the entire negative half. +/// Each vector is a single arena word moved by one, and each must make the +/// program unexecutable. What each one proves is different, so they are labelled +/// rather than swept anonymously: +/// +/// - an OPENED VALUE: the leaf hash changes, so the walk reaches a root the +/// transcript never absorbed. This is also the two-consumer join — the same +/// cell is what DEEP folds, so there is no way to move one without the other. +/// - a MERKLE SIBLING, both words: a path that authenticates nothing. Both words +/// are hit deliberately; a past tamper suite in this phase touched only byte 0 +/// of every digest, so a digest's second word was never checked. +/// - a FRI SYMMETRIC EVALUATION and a FRI SIBLING: the layer walk, on the one +/// sub-proof of this epoch that actually folds (12 committed layers). +/// - a CLAIMED COMPOSITION PART: this one is absorbed, so it moves the +/// challenges as well — it must reject, and the interesting part is that it +/// cannot reject "only" the quotient check. +/// - an OOD CELL: likewise absorbed, and read by both the constraint fold and +/// DEEP. +#[test] +fn the_assembled_verifier_rejects_tampered_leg_data() { + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + let good = super::epoch_tests::epoch_arena_words(&e, true); + // The pin, for the same reason as `the_assembled_epoch_verifier_runs`: this + // is the same `WrapHash::production()` program. It matters most on THIS + // arm — the honest control is what a wrong socket permutation breaks first, + // and a tamper suite whose control is broken rejects everything and passes. + assert!( + execute(&program, &good, &crate::hash_pin::BLOCK_HASHER).is_ok(), + "the untampered assembled verifier must run" + ); + + // The sub-proof that folds, so the FRI vectors reach the layer walk. + let folding = e + .legs + .iter() + .position(|l| l.verify.fri.num_committed() > 0) + .expect("this epoch has a sub-proof with committed FRI layers"); + + // ★ The index arithmetic above is a claim about the declaration order, and a + // WRONG index would make this whole test lie — it would tamper some other + // arena, still get a rejection, and report a pass. So the claim is checked + // against the arena LENGTHS, which the shapes fix independently. + for (t, leg) in e.legs.iter().enumerate() { + let ix = arena_index(&e, t); + assert_eq!( + good[ix.openings].len(), + leg.verify + .opening_words(super::proof_arena::words_per_root()), + "table {t}: the arena at the computed openings index is not the \ + openings arena" + ); + assert_eq!( + good[ix.fri].len(), + leg.verify.fri_words(super::proof_arena::words_per_root()), + "table {t}: the arena at the computed FRI index is not the FRI arena" + ); + assert_eq!( + good[ix.parts].len(), + e.tables[t].parts.len(), + "table {t}: the arena at the computed parts index is not the parts arena" + ); + assert_eq!( + good[ix.ood_current].len(), + e.tables[t].ood_current.len(), + "table {t}: the arena at the computed OOD index is not the OOD arena" + ); + } + + let mut vectors: Vec<(String, usize, usize)> = Vec::new(); + // Trace openings: the first value and both words of the first sibling + // digest, on three tables including the folding one. + for &t in &[0usize, 1, folding] { + let ix = arena_index(&e, t); + let leg = &e.legs[t]; + let values = leg.verify.sub.groups()[0].num_values(); + vectors.push((format!("table {t}: opened value 0"), ix.openings, 0)); + vectors.push(( + format!("table {t}: last opened value of group 0"), + ix.openings, + values - 1, + )); + vectors.push((format!("table {t}: sibling lo"), ix.openings, values)); + vectors.push((format!("table {t}: sibling hi"), ix.openings, values + 1)); + vectors.push((format!("table {t}: claimed part 0"), ix.parts, 0)); + vectors.push((format!("table {t}: OOD cell 0"), ix.ood_current, 0)); + } + // FRI: the first layer's symmetric evaluation, then both words of its first + // sibling. + let fri_ix = arena_index(&e, folding).fri; + vectors.push(("FRI layer 0 sym".to_string(), fri_ix, 0)); + vectors.push(("FRI layer 0 sibling lo".to_string(), fri_ix, 1)); + vectors.push(("FRI layer 0 sibling hi".to_string(), fri_ix, 2)); + + for (label, arena, word) in &vectors { + let mut arenas = good.clone(); + let before = arenas[*arena][*word]; + arenas[*arena][*word][0] = before[0] + FE::one(); + assert!( + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER).is_err(), + "tampering {label} must make the assembled verifier unexecutable, \ + and did not" + ); + } + println!(" {} tamper vectors, all rejected", vectors.len()); +} + +/// ★ The boundary list the emitted program carries is a PROGRAM CONSTANT, and +/// this is the premise that makes it one. +/// +/// `AIR::boundary_constraints` takes the public inputs and the bus public inputs +/// — both proof data — so building the emitted list from that call would make the +/// program depend on the proof it verifies. `epoch_verify::boundary_terms` builds +/// it from a rule instead. The rule is only safe while it agrees with the call on +/// every AIR of a real epoch, so this compares them as SETS: a term the rule +/// missed would be a constraint the machine silently never checks. +#[test] +fn the_boundary_terms_are_program_shape() { + let e = super::epoch_tests::real_epoch(); + let mut with_boundary = 0usize; + for (i, leg) in e.legs.iter().enumerate() { + let rule = boundary_terms(leg.has_aux_trace, leg.verify.sub.deep.num_total_cols); + let want = &leg.production_boundary; + assert_eq!( + rule.len(), + want.len(), + "table {i}: the rule and production disagree about how many boundary \ + constraints the AIR has" + ); + for (r, w) in rule.iter().zip(want) { + assert_eq!(r.col, w.col, "table {i}: boundary column"); + assert_eq!(r.point, w.point, "table {i}: boundary point"); + assert_eq!(r.value, w.value, "table {i}: boundary value"); + } + if !want.is_empty() { + with_boundary += 1; + } + } + // Positive control: a suite where every AIR had an empty list would pass + // vacuously, and the rule's interesting branch would be untested. + assert!( + with_boundary > 0, + "no sub-proof carries a boundary constraint, so this proves nothing about \ + the rule's non-empty branch" + ); + println!( + " boundary premise: {with_boundary} of {} sub-proofs carry the \ + framework's acc[0] = 0 and nothing else", + e.legs.len() + ); +} + +/// ★ An ABSOLUTE structural guard over the ASSEMBLED verifier: no proof value is +/// hinted twice, legs included. +/// +/// This is the count that closes assembly obligation 3. The spine's own version +/// (`epoch_tests::the_spine_hints_each_proof_value_once`) could only say the +/// spine hinted nothing twice — the legs were not in the program, so their second +/// consumers had nothing to disagree with. Now they are, and the same absolute +/// property must hold over the whole thing: the OOD grid, the claimed parts, +/// every root and every challenge reach the legs as cells, never as a second +/// read. +#[test] +fn the_assembled_verifier_hints_each_proof_value_once() { + use std::collections::HashMap; + + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + + let mut hints: HashMap<(super::instr::ArenaId, u32), usize> = HashMap::new(); + for instr in &program.instrs { + if let super::instr::Instr::Hint { arena, index, .. } = instr { + *hints.entry((*arena, *index)).or_default() += 1; + } + } + let doubled: Vec<_> = hints.iter().filter(|(_, n)| **n > 1).collect(); + assert!( + doubled.is_empty(), + "these arena words are hinted more than once, which is the two-consumer \ + hazard the assembly exists to remove: {doubled:?}" + ); + + let declared: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + hints.len(), + declared, + "every declared arena word must be read exactly once" + ); + // The legs are actually IN this program — without this the guard would pass + // just as happily over the spine alone. + let spine = super::epoch_tests::epoch_program(&e, false); + assert!( + declared + > spine + .arena_schema + .lens + .iter() + .map(|l| *l as usize) + .sum::(), + "the assembled program must declare more arena words than the spine, or \ + the legs are not wired and this guard is vacuous" + ); +} + +/// ★ The preprocessed-commitment inventory of a real epoch — the EVIDENCE +/// assembly ledger entry 7 was opened without. +/// +/// Entry 7 says five AIRs are preprocessed (BITWISE, DECODE, KECCAK_RC, REGISTER, +/// PAGE), that three are compile-time constants, that REGISTER has a derivation +/// and that PAGE cannot become a program constant because it is a function of the +/// inner ELF. That is a claim about the AIR SET. This test asks the real epoch +/// which of its sub-proofs are actually preprocessed, and how many columns each +/// commits, so the proposal that closes the entry is built on a census rather +/// than on a recollection. +/// +/// `VmAirs::air_refs` fixes the order (`lib.rs:610-625`): BITWISE, DECODE, COMMIT, +/// KECCAK, KECCAK_RND, KECCAK_RC, ECSM, ECDAS, REGISTER, then optional HALT, then +/// the chunked tables, then the PAGE tables, and this suite appends L2G_MEMORY. +/// So a preprocessed sub-proof at index 8 is REGISTER and one past the chunked +/// tables is a PAGE — which is what makes "which sub-proof is which AIR" program +/// shape rather than proof data. +#[test] +fn the_preprocessed_commitments_of_a_real_epoch() { + let e = super::epoch_tests::real_epoch(); + let preprocessed: Vec<(usize, usize)> = e + .legs + .iter() + .enumerate() + .filter(|(_, l)| l.num_precomputed_cols > 0) + .map(|(i, l)| (i, l.num_precomputed_cols)) + .collect(); + println!( + " {} of {} sub-proofs are preprocessed: {:?} (index, precomputed columns)", + preprocessed.len(), + e.legs.len(), + preprocessed + ); + // The REGISTER slot, checked by its column count rather than assumed from its + // index: the derivation commits OFFSET ‖ INIT ‖ FINI. + let register = e + .legs + .iter() + .position(|l| l.num_precomputed_cols == crate::tables::register::NUM_PREPROCESSED_COLS); + println!( + " the sub-proof whose preprocessed width is NUM_PREPROCESSED_COLS \ + ({}): index {:?}", + crate::tables::register::NUM_PREPROCESSED_COLS, + register + ); + // Every preprocessed sub-proof's commitment must actually be present, or the + // spine would be absorbing something it did not get from the AIR. + for (i, _) in &preprocessed { + assert!( + e.legs[*i].precomputed_commitment.is_some(), + "sub-proof {i} declares preprocessed columns but has no AIR commitment" + ); + } + assert!( + !preprocessed.is_empty(), + "an epoch with no preprocessed sub-proof cannot witness entry 7 at all" + ); + + // ---- ★ the PROVENANCE census, which is what entry 7 actually turns on. + // + // `epoch_tests::prep_source` decided each root's source by recomputing every + // candidate production has; reaching this line means every preprocessed root of + // a real epoch matched one, so nothing is hinted without a binding. What is + // asserted here is the SHAPE of the taxonomy — that the epoch is not all + // constants (which would make the derivation and the fold untested) and not all + // ELF-dependent (which would mean interning bought nothing). + let sources = super::epoch_tests::prep_source_census(&e); + println!( + " provenance: {} options-only (interned as program text), {} derived \ + in-machine (REGISTER), {} ELF-dependent (arena cell + attestation join)", + sources.0, sources.1, sources.2 + ); + assert_eq!( + sources.0 + sources.1 + sources.2, + preprocessed.len(), + "every preprocessed sub-proof must have exactly one classified source" + ); + assert!( + sources.0 > 0, + "no options-only root: the interning path is unexercised" + ); + assert_eq!( + sources.1, 1, + "exactly one derived root — the REGISTER commitment, from the epoch's own \ + register boundary" + ); + assert_eq!( + sources.2, 1, + "exactly one ELF-dependent root in a continuation epoch — DECODE. A second \ + would mean the attestation fold's input is ambiguous" + ); + + // ★ AND THE PAGE HALF OF ENTRY 7 IS NOT A FIXTURE ARTEFACT. `prove_epoch` + // rejects any epoch carrying a PAGE config ("continuation epoch must have no + // PAGE configs (L2G bookend replaces PAGE)", `continuation.rs:695-702`) and both + // `build_epoch_airs` call sites pass `&[]`. So no continuation epoch of any + // guest has a PAGE sub-proof, and the ELF-data page genesis roots the + // attestation folds are the GLOBAL proof's GlobalMemory AIRs' preprocessed + // commitments (`continuation.rs:997-1010`) — a different proof, out of an epoch + // verifier's scope. + // + // Asserted rather than remembered. The width test is unambiguous only because + // a continuation epoch's REGISTER always uses the WITH_FINI layout + // (`build_epoch_airs` always supplies `register_preprocessed`), and PAGE's + // width coincides with the non-FINI REGISTER one — so that premise is checked + // first. An ELF-data page root would in any case make `prep_source` panic, + // since its provenance is not in the classifier's candidate list. + assert!( + register.is_some(), + "a continuation epoch's REGISTER is preprocessed WITH FINI; without that \ + the width check below cannot tell a PAGE table from a REGISTER one" + ); + assert!( + e.legs + .iter() + .all(|l| l.num_precomputed_cols != crate::tables::page::NUM_PREPROCESSED_COLS), + "a sub-proof with PAGE's preprocessed width appeared: continuation epochs \ + are supposed to carry none, and the entry-7 taxonomy changes if they do" + ); +} + +/// ★ The composition and FRI-terminal CHECKS are in the program, counted. +/// +/// This guard exists because falsification found the hole it closes. Deleting +/// `assert_eq_ext(q.claimed, q.composition)` from the emitter fails NOTHING in +/// this suite: with honest data the two values ARE equal, so no differential and +/// no arena tamper can see the assert's absence. And no arena tamper ever will — +/// every input to the quotient identity (the OOD grid, the claimed parts, `z`, +/// `β`) is absorbed by the transcript, so moving any of them moves the challenges +/// and the run fails at the Merkle walk instead, for the wrong reason. +/// +/// What DOES witness the check is a mutation that makes the identity false while +/// leaving the transcript alone — emptying the boundary-term list does exactly +/// that, and three tests catch it. But "a mutation elsewhere catches it" is not +/// the same as "the check is present", so this counts the checks directly. +/// +/// `assert_eq_ext(a, b)` lowers to `esub` then `ediv(diff, ZERO)` +/// (`builder.rs:243-247`): division by the interned zero has a witness only when +/// the numerator vanishes, since `OUT · 0 = A` forces `A = 0`. So an extension +/// division whose DIVISOR is the pooled zero constant is an equality assertion, +/// and nothing else in the machine produces one — every other `ediv` here +/// inverts against the interned ONE. +/// +/// The expected count is arithmetic over the shapes, not a second emitter pass: +/// one composition check per sub-proof, plus per query one FRI terminal check +/// when the codeword folds and TWO when it does not (the zero-fold shape checks +/// `P` at both `υ` and `−υ`). +#[test] +fn the_assembled_verifier_contains_every_composition_and_terminal_check() { + use super::instr::{ExtOp, Instr}; + + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + let spine = super::epoch_tests::epoch_program(&e, false); + + let asserts = |p: &super::compiler::LfmProgram| -> usize { + // The interned all-zero word. `felt_const(0)` and `ext_const(0)` are the + // same word, and the builder interns program-wide, so there is one. + let zeros: Vec<_> = p + .instrs + .iter() + .filter_map(|i| match i { + Instr::Const { out, value, .. } if value.iter().all(|v| *v == FE::zero()) => { + Some(*out) + } + _ => None, + }) + .collect(); + assert_eq!( + zeros.len(), + 1, + "the zero word must be interned exactly once, or this count is \ + ambiguous" + ); + let zero = zeros[0]; + p.instrs + .iter() + .filter(|i| { + matches!( + i, + Instr::ExtAlu { + op: ExtOp::Div, + b, + .. + } if *b == zero + ) + }) + .count() + }; + + let expected: usize = e + .legs + .iter() + .map(|l| { + let terminal = if l.verify.fri.total_folds() > 0 { 1 } else { 2 }; + 1 + l.verify.num_queries * terminal + }) + .sum(); + assert_eq!( + asserts(&program) - asserts(&spine), + expected, + "the legs must add exactly one composition check per sub-proof plus the \ + FRI terminal checks the shapes call for" + ); + // Positive control: the count must be nonzero and the shapes must actually + // include both FRI branches, or the formula's second case is untested. + assert!(expected > 0); + assert!( + e.legs.iter().any(|l| l.verify.fri.total_folds() > 0) + && e.legs.iter().any(|l| l.verify.fri.total_folds() == 0), + "this epoch must exercise BOTH the folding and the zero-fold terminal \ + shapes, or the expected count is only half checked" + ); + println!( + " {} equality assertions added by the legs (25 composition + FRI \ + terminals)", + expected + ); +} + +/// The rate model's corrected pieces, WITHOUT a real epoch. +/// +/// The hash-matrix permutation-axis block that consumes these lives inside +/// `the_assembled_epoch_verifier_runs`, which needs the fixture guest ELF. That is +/// exactly how `LFM_HASH_RATE_FELTS = 8` outlived the three-cell duplex it was +/// derived from: nothing that ran in a bare checkout touched it. This test does, +/// on shapes built by hand. +/// +/// What it pins is the correction itself — the constant's derivation, and that +/// the FRI-leaf term is rate-sensitive and still reduces to `num_committed()` at +/// keccak's rate, which is the identity that keeps the felt-side and byte-side +/// closed forms agreeing. +#[test] +fn the_candidate_rate_model_is_derived_not_remembered() { + use super::epoch_verify::{ + FRI_LEAF_FELTS, KECCAK_RATE_FELTS, LFM_HASH_RATE_FELTS, blocks_at_rate, + fri_leaf_permutations_at_rate, + }; + use super::fri::FriShape; + use super::hash::HASH_DIGEST_FELTS; + + // The chain absorbs ONE cell per step, so the rate IS the digest width. + // Written as the derivation, not as a literal, because the literal is what + // went stale. + assert_eq!(LFM_HASH_RATE_FELTS, HASH_DIGEST_FELTS); + assert_eq!(LFM_HASH_RATE_FELTS, 4, "was 8 under the deleted duplex"); + + // ⚠ The premise the old model folded the FRI leaf into: "a layer leaf fits + // one block at the candidate's rate". True at 8, FALSE at 4. + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, KECCAK_RATE_FELTS), 1); + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, 8), 1, "the old rate did fit"); + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, LFM_HASH_RATE_FELTS), 2); + + let fri = FriShape { + log2_lde_length: 20, + blowup_log: 3, + final_poly_log_degree: 3, + coset_offset: 3, + num_queries: 73, + format: stark::proof::options::ProofFormat::DEFAULT, + trees: stark::config::TreeGeometry::BINARY, + }; + assert!(fri.num_committed() > 0, "the shape must exercise the term"); + + // At keccak's rate the new term reduces to the old `num_committed()`, which + // is why splitting it out did not move the rate-17 column. + assert_eq!( + fri_leaf_permutations_at_rate(&fri, KECCAK_RATE_FELTS), + fri.num_committed() + ); + // At the candidate's rate it doubles — the cost the old model hid. + assert_eq!( + fri_leaf_permutations_at_rate(&fri, LFM_HASH_RATE_FELTS), + 2 * fri.num_committed() + ); + + // A shape with nothing committed contributes nothing at any rate, so the + // correction cannot invent cost where there is no FRI leg. + let terminal = FriShape { + log2_lde_length: 6, + ..fri + }; + assert_eq!(terminal.num_committed(), 0); + for rate in [KECCAK_RATE_FELTS, LFM_HASH_RATE_FELTS] { + assert_eq!(fri_leaf_permutations_at_rate(&terminal, rate), 0); + } +} + +/// Queries the knob-on twin proves at: enough openings that `auto` caps every +/// tall tree at height 3 (from 20 openings on). +const PROCESS_FORMAT_QUERIES: usize = 24; + +/// ★ The KNOB-ON TWIN of [`the_assembled_epoch_verifier_runs`] (box only): a +/// real continuation epoch proved at the PROCESS format — `ZfFormat::global()`, +/// i.e. `LAMBDA_VM_ZF_CAP` / `LAMBDA_VM_ZF_FRI` — at the MIN preset with +/// [`PROCESS_FORMAT_QUERIES`] queries, verified by the assembled machine. +/// +/// Asserts, per format: the program executes (every cap authenticated once per +/// tree, every opening checked against it, every FRI group folded to the +/// terminal); the legs' emitted permutations equal the closed form +/// `Σ table_permutations_for` (per-query paths cut at each tree's cap plus +/// `2^c − 1` once per capped tree; group leaves and group paths under +/// `fri = dp`); a moved cap word does not execute. Prints the census to +/// compare across arms (instructions, permutations, `Select`s, cells per +/// chip). At the default format it is the MIN-preset run at 24 queries. +#[test] +#[ignore = "a real epoch proof at 24 queries and its assembled verifier: box only"] +fn the_assembled_epoch_verifier_runs_at_the_process_format() { + let mut opts = super::proof_fixture::fixture_options(); + opts.fri_number_of_queries = PROCESS_FORMAT_QUERIES; + assembled_twin_at_the_process_format(opts); +} + +/// ★ [`the_assembled_epoch_verifier_runs_at_the_process_format`] at BLOWUP 4 +/// — the S2 (one-row) twin, box only. One-row static roots exist at blowup 4 +/// only (`STATIC_BLOWUP_FACTORS_ONE_ROW`; a missing twin is a +/// proving error), so the MIN preset's blowup 2 cannot prove a one-row +/// BITWISE; this arm keeps every other MIN-preset option and lifts the blowup +/// to 4 for every format, so its knob-off and knob-on runs are one A/B. Under +/// `one_row = auto` the epoch's tables resolve their layouts one by one +/// (printed per leg), so the assembled machine verifies a MIXED-layout proof; +/// the REGISTER root is derived in-machine at that table's own layout. +#[test] +#[ignore = "a real epoch proof at blowup 4, 24 queries, and its assembled verifier: box only"] +fn the_assembled_epoch_verifier_runs_at_blowup_4_at_the_process_format() { + let mut opts = super::proof_fixture::fixture_options(); + opts.fri_number_of_queries = PROCESS_FORMAT_QUERIES; + opts.blowup_factor = 4; + assembled_twin_at_the_process_format(opts); +} + +/// The body of the assembled-verifier twins: `base` with the process format +/// stamped on, proved, harvested and verified by the assembled machine. +fn assembled_twin_at_the_process_format(base: crate::ProofOptions) { + let format = crate::zf_format::ZfFormat::global(); + let opts = format.options(base); + let e = super::epoch_tests::real_epoch_with(opts.clone()); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the assembled verifier must execute at the process format"); + + let spine = super::epoch_tests::epoch_program(&e, false); + let perms = |p: &_| super::machine_tests::wrap_hash_instrs(p); + let selects = |p: &super::compiler::LfmProgram| { + p.instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Select { .. })) + .count() + }; + let hash = super::edsl::WrapHash::production(); + let emitted = perms(&program) - perms(&spine); + let predicted: usize = e + .legs + .iter() + .map(|l| super::epoch_verify::table_permutations_for(&l.verify, hash)) + .sum(); + let cap_perms: usize = e + .legs + .iter() + .map(|l| super::epoch_verify::cap_permutations(&l.verify)) + .sum(); + println!( + "\n★ ASSEMBLED EPOCH VERIFIER AT THE PROCESS FORMAT\n {}\n opts: blowup {}, \ + {} queries, grinding {}, k {}\n sub-proofs {} | legs: {} instructions, \ + {} permutations ({} of them cap roots), {} selects | whole: {} instructions, \ + {} permutations", + format.banner(), + opts.blowup_factor, + opts.fri_number_of_queries, + opts.grinding_factor, + opts.fri_final_poly_log_degree, + e.legs.len(), + program.instrs.len() - spine.instrs.len(), + emitted, + cap_perms, + selects(&program) - selects(&spine), + program.instrs.len(), + perms(&program), + ); + for (i, l) in e.legs.iter().enumerate() { + let f = l.verify.fri; + println!( + " leg {i:>2}: log2(lde) {:>2} layout {:?} trace cap {} FRI schedule {:?} \ + depths {:?} caps {:?} {} permutations", + l.verify.sub.log2_lde_length, + l.verify.sub.layout, + l.verify.sub.trace_cap, + f.schedule(), + (0..f.num_committed()) + .map(|j| f.layer_depth(j)) + .collect::>(), + (0..f.num_committed()) + .map(|j| f.layer_cap(j)) + .collect::>(), + super::epoch_verify::table_permutations_for(&l.verify, hash), + ); + } + for c in super::airs::lfm_chip_census(&program) { + println!( + " CENSUS {:<14} real {:>10} padded {:>10} cells {:>12}", + c.name, + c.real_rows, + c.rows, + c.main_cells() + ); + } + assert_eq!( + emitted, predicted, + "the legs' emitted permutations must equal the closed form at the process format" + ); + println!(" emitted permutations == closed form: {emitted}"); + // One parseable line for the box wrapper's cross-arm comparison. + println!( + "ZFTWIN legs_permutations={emitted} cap_root_permutations={cap_perms} \ + legs_instructions={} legs_selects={} whole_instructions={}", + program.instrs.len() - spine.instrs.len(), + selects(&program) - selects(&spine), + program.instrs.len(), + ); + // S2: how many legs verify one-row tables (0 at `one_row = 0`, every leg + // at `1`, the AIR widths' choice at `auto`), and the blowup of the arm. + let one_row_legs = e + .legs + .iter() + .filter(|l| l.verify.sub.layout.is_one_row()) + .count(); + println!( + "ZFS2TWIN blowup={} legs={} one_row_legs={one_row_legs} row_pair_legs={}", + opts.blowup_factor, + e.legs.len(), + e.legs.len() - one_row_legs, + ); + match opts.format.one_row { + stark::proof::options::OneRowMode::Off => assert_eq!(one_row_legs, 0), + stark::proof::options::OneRowMode::On => assert_eq!(one_row_legs, e.legs.len()), + stark::proof::options::OneRowMode::Auto => {} + } + // A one-row leg's input-tree group value (query 0, layer 0, value 0) moved + // must not execute — the input group is authenticated and slot-checked. + // The FRI arena is found by content rather than by a hand-counted offset. + if let Some((k, words)) = e + .legs + .iter() + .enumerate() + .find(|(_, l)| l.verify.sub.layout.is_one_row() && l.verify.fri.num_committed() > 0) + .map(|(k, l)| (k, l.fri_arena())) + { + let at = arenas + .iter() + .position(|a| *a == words) + .expect("the one-row leg's FRI arena is among the program's arenas"); + let mut bad = arenas.clone(); + bad[at][0][0] += FE::one(); + execute(&program, &bad, &crate::hash_pin::BLOCK_HASHER) + .expect_err("a moved input-tree value must not execute"); + println!(" leg {k}: a moved one-row input-tree value is refused"); + } + + // A moved cap word must not execute (only when the format caps a tree). + // The caps arena is found by content rather than by a hand-counted offset. + if let Some((k, words)) = e + .legs + .iter() + .enumerate() + .find_map(|(k, l)| l.caps_arena().map(|w| (k, w))) + { + let at = arenas + .iter() + .position(|a| *a == words) + .expect("the caps arena is among the program's arenas"); + let mut bad = arenas.clone(); + bad[at][0][0] += FE::one(); + execute(&program, &bad, &crate::hash_pin::BLOCK_HASHER) + .expect_err("a moved cap word must not execute"); + println!(" leg {k}: a moved cap word is refused"); + } +} diff --git a/prover/src/lfm/exec_identity_tests.rs b/prover/src/lfm/exec_identity_tests.rs new file mode 100644 index 000000000..1b176a82f --- /dev/null +++ b/prover/src/lfm/exec_identity_tests.rs @@ -0,0 +1,700 @@ +//! The gate on `execute`'s schedule: the witness is a pure function of the +//! program and its arenas, so **which order** the executor evaluates the program +//! in may not change one word of it. +//! +//! This is the sibling of [`super::trace_identity_tests`] one stage earlier in +//! the pipeline. That file gates the row walk that turns records into traces; +//! this one gates the interpreter that produces the records in the first place. +//! Together they pin the whole host side of a proof against a schedule change: +//! program → records (here) → traces (there). +//! +//! ## The two arms +//! +//! [`reference`] is the serial `for instr in &program.instrs` loop the file has +//! always had. [`candidate`] is [`Schedule::LevelParallel`]: each depth level's +//! mutually independent hash instructions on rayon's global pool, everything +//! else serial and in program order. +//! +//! ⚠ The knob that selects the schedule in production cannot be the thing this +//! test flips. The precedent (`LFM_ARTIFACT_PARALLEL`, `commit.rs:35`) caches +//! its environment read in a `OnceLock`, so a process reads it once and a test +//! that set it per-case would gate whichever arm happened to run first. The two +//! arms are therefore reached by an explicit argument, the way +//! `build_traces_walked` takes a [`super::trace::Walk`], and the environment +//! chooses only the production default. +//! +//! ## What is compared +//! +//! Everything an [`LfmExecution`] carries: every memory address through the +//! public accessor (including *unwritten*, which must stay unwritten on both +//! sides — the write-once bit is the only thing separating that from a written +//! zero), the const counter, all ten record vectors element for element, and +//! `public_words`. +//! +//! Elements are compared through their derived `Debug`, which prints a +//! `FieldElement`'s **raw** `u64` rather than its canonical residue. That is +//! deliberate and is strictly stronger than `==` on the field, whose `eq` +//! canonicalises both sides (`goldilocks.rs:157`): two executions of the same +//! instruction stream perform the same arithmetic in the same order, so they owe +//! each other bit equality, not just congruence. It also means no field of any +//! record type is transcribed here — a row that grows a column is compared on +//! its new column the day it grows one, with no edit to this file. + +use super::executor::{LfmExecError, LfmExecution, Schedule, execute_scheduled}; +use super::instr::Addr; +use super::trace_identity_tests::{Case, cases}; + +/// The cut-off the candidate arm runs at, and it is not the production one. +/// +/// ⛔ The laptop cases are 15 wide at their widest level (printed by the gate +/// below). Anything at or above 16 — which `PRODUCTION_COALESCE_BELOW` is — +/// would send every level of every case down the serial branch, and this file +/// would then pass whatever the parallel path did, on every run, forever. At 2, +/// every level of width ≥ 2 really forks. +const GATE_COALESCE_BELOW: usize = 2; + +/// The serial reference: the interpreter as it stands. +fn reference(case: &Case) -> LfmExecution { + execute_scheduled(&case.program, &case.arenas, &case.hasher, Schedule::Serial) + .unwrap_or_else(|e| panic!("{}: the reference must execute: {e:?}", case.name)) +} + +/// The schedule under test: levels, on rayon's global pool. +fn candidate(case: &Case) -> LfmExecution { + execute_scheduled( + &case.program, + &case.arenas, + &case.hasher, + Schedule::LevelParallel { + coalesce_below: GATE_COALESCE_BELOW, + }, + ) + .unwrap_or_else(|e| panic!("{}: the candidate must execute: {e:?}", case.name)) +} + +/// Two record vectors, element for element, with the first divergence named. +/// +/// Generic over the row type and asking only for `Debug`, so it covers a chip +/// whose row struct this file has never heard of. +fn assert_rows_identical(case: &str, chip: &str, a: &[T], b: &[T]) { + assert_eq!( + a.len(), + b.len(), + "{case}/{chip}: the two schedules produced different row counts" + ); + for (i, (x, y)) in a.iter().zip(b).enumerate() { + let (dx, dy) = (format!("{x:?}"), format!("{y:?}")); + assert_eq!( + dx, + dy, + "{case}/{chip}: the two schedules disagree at record {i} of {}", + a.len() + ); + } +} + +/// Asserts two executions are the same object, and says where they first differ +/// if they are not. +pub(super) fn assert_executions_identical( + case: &str, + num_addrs: u64, + a: &LfmExecution, + b: &LfmExecution, +) { + // Memory first: it is the object every record is derived from, so a + // divergence here localises the failing instruction by its destination + // address, which is its position in emission order. + for i in 0..num_addrs { + let (x, y) = (a.memory.get(Addr(i)), b.memory.get(Addr(i))); + let (dx, dy) = (format!("{x:?}"), format!("{y:?}")); + assert_eq!( + dx, dy, + "{case}: the two schedules left different values at address {i}" + ); + } + + let (ra, rb) = (&a.records, &b.records); + assert_eq!( + ra.num_consts, rb.num_consts, + "{case}: the const count moved with the schedule" + ); + assert_rows_identical(case, "LFM_BALU", &ra.balu, &rb.balu); + assert_rows_identical(case, "LFM_XALU", &ra.xalu, &rb.xalu); + assert_rows_identical(case, "LFM_SELECT", &ra.select, &rb.select); + assert_rows_identical(case, "LFM_BITDEC", &ra.bitdec, &rb.bitdec); + assert_rows_identical(case, "LFM_HASH", &ra.hash, &rb.hash); + assert_rows_identical(case, "LFM_KECCAK", &ra.keccak, &rb.keccak); + assert_rows_identical(case, "LFM_BLAKE3", &ra.blake3, &rb.blake3); + assert_rows_identical(case, "LFM_LANES", &ra.lanes, &rb.lanes); + assert_rows_identical(case, "LFM_HINT", &ra.hint, &rb.hint); + assert_rows_identical(case, "LFM_PUBLIC", &ra.public, &rb.public); + assert_rows_identical(case, "public_words", &a.public_words, &b.public_words); +} + +/// ★ The gate. Both schedules, every case, word for word. +#[test] +fn the_parallel_executor_is_byte_identical_to_the_serial_reference() { + // Records a schedule change could reorder. A case list that gave some chip + // no rows would satisfy the comparison above on that chip vacuously. + let mut covered: Vec<&str> = Vec::new(); + let (mut widest, mut deepest) = (0u32, 0usize); + + for case in cases() { + let a = reference(&case); + let b = candidate(&case); + assert_executions_identical(case.name, case.program.num_addrs, &a, &b); + + // ★ The candidate must have USED the path under test. Without this the + // whole file is satisfied by a candidate that coalesced every level onto + // the calling thread and ran the serial code twice. + assert_eq!( + a.split.parallel_levels, 0, + "{}: the reference arm must not schedule levels at all", + case.name + ); + if b.split.levels > 1 { + assert!( + b.split.parallel_levels > 0 && b.split.parallel_hashes > 0, + "{}: the candidate coalesced all {} levels onto the calling \ + thread, so nothing this test compares went through rayon", + case.name, + b.split.levels + ); + } + + let r = &a.records; + for (chip, rows) in [ + ("LFM_BALU", r.balu.len()), + ("LFM_XALU", r.xalu.len()), + ("LFM_SELECT", r.select.len()), + ("LFM_BITDEC", r.bitdec.len()), + ("LFM_HASH", r.hash.len()), + ("LFM_KECCAK", r.keccak.len()), + ("LFM_BLAKE3", r.blake3.len()), + ("LFM_LANES", r.lanes.len()), + ("LFM_HINT", r.hint.len()), + ("LFM_PUBLIC", r.public.len()), + ] { + if rows > 0 && !covered.contains(&chip) { + covered.push(chip); + } + } + + // ★ The ONE thing that decides whether this gate says anything about a + // level-parallel schedule: the shape of the DAG it runs over. A case + // whose hash levels are all one wide takes the serial path in both arms + // however the candidate is built, and would pass a broken parallel + // executor without ever forking. + let p = super::reach_profile::profile(&case.program); + widest = widest.max(p.widest_level()); + deepest = deepest.max(p.hash_depth); + println!( + "{:<18} addrs {:>7} instrs {:>6} hash rows {:>6} \ + records {:>6} public {:>4} hash depth {:>4} widest level {:>4}", + case.name, + case.program.num_addrs, + case.program.instrs.len(), + r.hash.len(), + r.balu.len() + r.xalu.len() + r.select.len() + r.bitdec.len() + r.lanes.len(), + a.public_words.len(), + p.hash_depth, + p.widest_level(), + ); + } + + for chip in RECORDED_CHIPS { + assert!( + covered.contains(chip), + "no case gave {chip} a record, so the two schedules were never \ + compared on it and the gate is vacuous there. Add a case that \ + exercises it rather than dropping it from this list." + ); + } + + // ⚠ The SHAPE guard, and it is the one that decides what this test is worth. + // A schedule can only be wrong where there is a choice to get wrong: over a + // program of depth 1, or of width 1 at every level, every schedule is the + // same schedule. These bounds are what the case list happens to provide, so + // a case list that shrank below them would be caught here rather than by a + // green run that gated nothing. + assert!( + widest >= MIN_WIDEST_LEVEL, + "the widest hash level across every case is {widest}, under {MIN_WIDEST_LEVEL}: \ + there is not enough independent work here for two schedules to differ, \ + so a green run would mean nothing" + ); + assert!( + deepest >= MIN_HASH_DEPTH, + "the deepest hash chain across every case is {deepest}, under {MIN_HASH_DEPTH}: \ + a program this shallow has almost no level boundaries to get wrong" + ); + println!( + "shape: widest hash level {widest} · deepest hash chain {deepest} \ + (floors {MIN_WIDEST_LEVEL} / {MIN_HASH_DEPTH})" + ); +} + +/// ⛔ The laptop's case list is a TOY next to a wrap (17,627 wide and 2,237 deep +/// on the 2026-09-12 reading; these cases measure **15 wide and 14 deep**, and +/// only the two `FriToyV0` cases have any `Instr::Hash` at all — the two sponge +/// programs drive the keccak and BLAKE3 chips, which are not hash rows). The +/// floors below are what the list does provide, held against shrinkage; they are +/// not a claim that this gate covers production shapes. The gate that does is +/// the tree-scale `IDENTITY:` diff. +/// +/// ⚠ **A consequence for the schedule, and it is load-bearing.** A level-parallel +/// executor wants to run narrow levels serially — a width-1 level on rayon is a +/// fork/join for one permutation. Any such cut-off at or above 15 would send +/// **every** level of every case here down the serial path, and this test would +/// then pass whatever the parallel path did. The candidate arm must therefore be +/// reached with the cut-off lowered to 2, so each of these levels really forks; +/// the production default is a separate number. +const MIN_WIDEST_LEVEL: u32 = 8; +const MIN_HASH_DEPTH: usize = 8; + +/// The chips the executor pushes a record for. `LFM_CONST` is absent because it +/// keeps a counter rather than a vector, and it is asserted separately. +const RECORDED_CHIPS: &[&str] = &[ + "LFM_BALU", + "LFM_XALU", + "LFM_SELECT", + "LFM_BITDEC", + "LFM_HASH", + "LFM_KECCAK", + "LFM_BLAKE3", + "LFM_LANES", + "LFM_HINT", + "LFM_PUBLIC", +]; + +/// The record-slot plan's unit cost, pinned. +/// +/// A level-parallel executor pre-sizes `records.hash` and has each hash write +/// its own slot, so the vector's footprint is the plan's memory cost and is +/// quoted in the design note. It is twelve `IN` felts and twelve `OUT` felts of +/// eight bytes with nothing else in the struct; pinning it here is what stops +/// that quote from becoming a stale number if a column is ever added. +#[test] +fn a_hash_record_slot_is_192_bytes() { + assert_eq!( + std::mem::size_of::(), + 192, + "a hash record is 24 Goldilocks felts and nothing else" + ); + assert_eq!( + std::mem::size_of::(), + 8, + "a Goldilocks felt is one u64, which is what the 192 above is built on" + ); +} + +/// Every record row's width, pinned. +/// +/// ⛔ These are the multipliers in `LfmRecords::bytes`, and `bytes` is what the +/// split line reports as the size of what `setup` first-touches — the figure the +/// campaign will size the phase against. A row that silently grew a column would +/// move that figure with nothing to say so, and the arithmetic in the handoff +/// would go on quoting the old one. +#[test] +fn the_record_rows_are_the_widths_the_sizing_assumes() { + use super::blake3_chip::Blake3Values; + use super::executor::{BaluRow, BitDecRow, HashRow, KeccakRow, SelectRow, XaluRow}; + use super::word::LfmWord; + use std::mem::size_of; + + for (name, got, want) in [ + ("BaluRow", size_of::(), 32), + ("XaluRow", size_of::(), 96), + ("SelectRow", size_of::(), 136), + ("BitDecRow", size_of::(), 528), + ("HashRow", size_of::(), 192), + ("KeccakRow", size_of::(), 744), + ("Blake3Values", size_of::(), 112), + ("LfmWord", size_of::(), 32), + ] { + assert_eq!( + got, want, + "{name} is {got} bytes, not the {want} the sizing assumes" + ); + } +} + +/// ★★ The property the `unsafe` in `commit_slots` rests on: a walk that fails +/// partway NEVER yields records. +/// +/// The level walk writes rows into spare capacity and leaves every vector at +/// length zero until `commit_slots`, which runs at one place after the last +/// instruction. So a program that errors mid-walk returns `Err` and the +/// half-written buffers are freed without a single slot being read — the +/// uninitialised state is not merely unlikely to be observed, it is not +/// reachable through the API at all. +/// +/// The merged schedule is a walk that fails partway by construction, which is +/// what makes this checkable rather than assertable: it errors with rows already +/// written, and there is still no value for a caller to inspect. +#[test] +fn a_failed_walk_yields_no_records_at_all() { + let mut checked = 0; + for case in cases() { + if super::reach_profile::profile(&case.program).hash_rows == 0 { + continue; + } + let out = super::executor::execute_with_merged_levels( + &case.program, + &case.arenas, + &case.hasher, + GATE_COALESCE_BELOW, + 2, + ); + assert!( + out.is_err(), + "{}: this walk must fail partway, or it is not the case this test needs", + case.name + ); + checked += 1; + } + assert!(checked > 0, "no case exercised a partial walk"); +} + +/// ★★ THE MUTATION. Merge adjacent depth levels and the executor must FAIL — at +/// the same address, on every run. +/// +/// This is what makes the gate above a gate rather than a green light. The level +/// boundary is the only thing separating a hash from the input another hash +/// produces; `merge = 2` removes it, and if the executor still produced a +/// correct witness that would mean the boundary was never doing anything and the +/// identity assertion was passing for some other reason. +/// +/// ⓘ It fails deterministically because the workers hold `&WriteOnceMemory` and +/// nothing holds `&mut` during a level: the premature read finds an unwritten +/// cell, which is [`LfmExecError::ReadBeforeWrite`], never a race. Three runs, +/// asserted to name one address, is how that claim is checked rather than +/// asserted. +#[test] +fn merging_adjacent_levels_fails_at_one_address_every_run() { + let mut checked = 0; + for case in cases() { + // A case with no hashes has no level structure to break. + if super::reach_profile::profile(&case.program).hash_rows == 0 { + continue; + } + let mut seen: Vec = Vec::new(); + for run in 0..3 { + let err = super::executor::execute_with_merged_levels( + &case.program, + &case.arenas, + &case.hasher, + GATE_COALESCE_BELOW, + 2, + ) + .expect_err("a merged schedule launches a hash before its input exists"); + let LfmExecError::ReadBeforeWrite(addr) = err else { + panic!( + "{}: run {run} failed with {err:?}, not ReadBeforeWrite — \ + the merged schedule has to fail for the schedule's reason", + case.name + ); + }; + // ★ And it has to fail for the RIGHT reason: the cell it read too + // early must be one a HASH writes. A merge that collapsed the + // constants into the first hash level would also raise + // ReadBeforeWrite, and would prove nothing about whether a hash can + // be launched before the hash it depends on. + let writer = case + .program + .instrs + .iter() + .find(|i| i.writes().iter().any(|a| a.0 == addr)); + assert!( + matches!(writer, Some(super::instr::Instr::Hash { .. })), + "{}: the premature read was of address {addr}, which is written \ + by {writer:?} rather than by a hash — the mutation broke some \ + other ordering", + case.name + ); + seen.push(format!("{err:?}")); + } + assert!( + seen.windows(2).all(|w| w[0] == w[1]), + "{}: three runs of the merged schedule named different addresses \ + ({seen:?}) — the failure is timing-dependent, which is exactly what \ + this design is supposed to make impossible", + case.name + ); + println!("{:<18} merged levels -> {}", case.name, seen[0]); + checked += 1; + } + assert!( + checked > 0, + "no case had a hash to mis-schedule, so the mutation proved nothing" + ); +} + +// ==================== the streamed execution ==================== + +/// A node-shaped program: three "children", each a BLAKE3 sponge over an arena +/// of its own (independent, as a node's verification of each child is), then a +/// little work over all three (as a node's cross-child checks are). +fn three_children_program(len_bytes: usize) -> super::compiler::LfmProgram { + use super::builder::LfmBuilder; + use super::edsl::{ByteWrapHash, WrapHash, wrap_hash_bytes}; + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let mut firsts = Vec::new(); + for _ in 0..3 { + let arena = b.declare_arena(halves); + let stream: Vec<_> = (0..halves).map(|i| b.hint_felt(arena, i)).collect(); + let digest = wrap_hash_bytes(&mut b, ByteWrapHash::Blake3, &stream, len_bytes); + b.public(digest[0]); + b.public(digest[1]); + firsts.push(stream[0]); + } + let x = b.add(firsts[0], firsts[2]); + let y = b.mul(x, firsts[1]); + let joined = b.pack_word([x, y, x, y]); + b.public(joined); + super::compiler::compile(b.finish()) +} + +/// `case` streamed: each arena its own group, landing in `order`. +fn streamed(case: &Case, order: &[usize]) -> (LfmExecution, Vec) { + use super::executor::StreamedExecution; + let groups: Vec = (0..case.arenas.len()).collect(); + let mut ex = StreamedExecution::new(&case.program, groups, &case.hasher) + .unwrap_or_else(|e| panic!("{}: a streamed execution starts: {e:?}", case.name)); + let mut ran = vec![ex.ran()]; + for &g in order { + ex.land(g, std::slice::from_ref(&case.arenas[g])) + .unwrap_or_else(|e| panic!("{}: group {g} lands: {e:?}", case.name)); + ran.push(ex.ran()); + } + let done = ex + .finish() + .unwrap_or_else(|e| panic!("{}: the streamed execution finishes: {e:?}", case.name)); + (done, ran) +} + +fn three_children_case() -> Case { + let len = 202; + let arena = |seed: u8| -> Vec { + let msg: Vec = (0..len) + .map(|i| (37 * i as u32 + 11 + 13 * u32::from(seed)) as u8) + .collect(); + super::keccak_host::pack_stream(&msg) + .into_iter() + .map(super::word::base_word) + .collect() + }; + Case { + name: "ThreeChildren(202)", + program: three_children_program(len), + arenas: vec![arena(0), arena(1), arena(2)], + hasher: super::hash::HasherKind::Test, + } +} + +/// ★ The streamed execution's gate: whatever order the arena groups land in, +/// the witness is the serial reference's, word for word — on the node-shaped +/// case in all six orders and on every identity case in two — and on the +/// node-shaped case most of the program has run before the last group lands. +#[test] +fn a_streamed_execution_is_byte_identical_in_any_landing_order() { + let node = three_children_case(); + let reference_node = reference(&node); + for order in [ + [0, 1, 2], + [0, 2, 1], + [1, 0, 2], + [1, 2, 0], + [2, 0, 1], + [2, 1, 0], + ] { + let (got, ran) = streamed(&node, &order); + assert_executions_identical(node.name, node.program.num_addrs, &reference_node, &got); + let n = node.program.instrs.len(); + assert_eq!( + *ran.last().expect("waves"), + n, + "{order:?}: every instruction ran once" + ); + assert!( + ran[2] * 3 >= n * 2, + "{order:?}: only {} of {n} instructions ran before the last group landed", + ran[2] + ); + } + for case in cases() { + let a = reference(&case); + let k = case.arenas.len(); + let forward: Vec = (0..k).collect(); + let backward: Vec = (0..k).rev().collect(); + for order in [forward, backward] { + let (b, ran) = streamed(&case, &order); + assert_executions_identical(case.name, case.program.num_addrs, &a, &b); + assert_eq!(*ran.last().expect("waves"), case.program.instrs.len()); + } + } +} + +/// A streamed execution refuses what it cannot run: finishing before every +/// group has landed, a group landing twice, a group with the wrong number of +/// arenas or an arena of the wrong length — each an error, not a witness. +#[test] +fn a_streamed_execution_refuses_an_incomplete_or_malformed_landing() { + use super::executor::StreamedExecution; + let node = three_children_case(); + let start = + || StreamedExecution::new(&node.program, vec![0, 1, 2], &node.hasher).expect("starts"); + + let mut ex = start(); + ex.land(0, std::slice::from_ref(&node.arenas[0])) + .expect("lands"); + assert!(matches!(ex.finish(), Err(LfmExecError::Internal(_)))); + + let mut ex = start(); + ex.land(1, std::slice::from_ref(&node.arenas[1])) + .expect("lands"); + assert!(matches!( + ex.land(1, std::slice::from_ref(&node.arenas[1])), + Err(LfmExecError::Internal(_)) + )); + + let mut ex = start(); + assert!(matches!( + ex.land(2, &node.arenas[1..3]), + Err(LfmExecError::ArenaCountMismatch { .. }) + )); + let short = vec![node.arenas[2][..node.arenas[2].len() - 1].to_vec()]; + assert!(matches!( + ex.land(2, &short), + Err(LfmExecError::ArenaLenMismatch { .. }) + )); +} + +/// A node-shaped program over children that supply different numbers of +/// arenas (`counts[c]` for child `c`, declared in child order): a BLAKE3 sponge +/// over each arena, then a little work across the children — the shape of a +/// node whose children hold different AIR sets, as under the equal-height +/// merge. +fn uneven_children_case(counts: &[usize]) -> Case { + use super::builder::LfmBuilder; + use super::edsl::{ByteWrapHash, WrapHash, wrap_hash_bytes}; + let len = 202; + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let halves = super::keccak_host::num_stream_halves(len) as u32; + let mut firsts = Vec::new(); + for _ in 0..counts.iter().sum::() { + let arena = b.declare_arena(halves); + let stream: Vec<_> = (0..halves).map(|i| b.hint_felt(arena, i)).collect(); + let digest = wrap_hash_bytes(&mut b, ByteWrapHash::Blake3, &stream, len); + b.public(digest[0]); + firsts.push(stream[0]); + } + let x = b.add(firsts[0], firsts[firsts.len() - 1]); + let y = b.mul(x, firsts[1]); + let joined = b.pack_word([x, y, x, y]); + b.public(joined); + let arena = |seed: usize| -> Vec { + let msg: Vec = (0..len).map(|i| (37 * i + 11 + 13 * seed) as u8).collect(); + super::keccak_host::pack_stream(&msg) + .into_iter() + .map(super::word::base_word) + .collect() + }; + Case { + name: "UnevenChildren(202)", + program: super::compiler::compile(b.finish()), + arenas: (0..counts.iter().sum::()).map(arena).collect(), + hasher: super::hash::HasherKind::Test, + } +} + +/// The streamed top ([`super::whir_block_tree::execute_streamed_with`]) of +/// `program` over children supplying `words` (child `c`'s arenas in order), +/// split at `split` (what the top is told each child supplies), the children +/// published in `order`: the first before the streamed execution starts, each +/// later one 10 ms after the one before, so the top lands them in that order. +pub(super) fn streamed_top_over( + program: &super::compiler::LfmProgram, + words: &[Vec>], + split: &[usize], + order: &[usize], + hasher: &(impl super::hash::LfmHasher + Sync), +) -> Result { + use super::whir_block_tree::{Published, execute_streamed_with}; + let kids: Vec>>> = + words.iter().map(|_| Published::new()).collect(); + let _ = kids[order[0]].0.set(Ok(words[order[0]].clone())); + std::thread::scope(|scope| { + let kids = &kids; + scope.spawn(move || { + for &c in &order[1..] { + std::thread::sleep(std::time::Duration::from_millis(10)); + let _ = kids[c].0.set(Ok(words[c].clone())); + } + }); + let refs: Vec<&Published<_>> = kids.iter().collect(); + execute_streamed_with(program, split, &refs, hasher, |w| Ok(w.clone())) + .map(|(execution, _)| execution) + }) +} + +/// [`streamed_top_over`] for `case`, its arenas taken by children of `counts`. +fn streamed_top( + case: &Case, + counts: &[usize], + split: &[usize], + order: &[usize], +) -> Result { + let mut at = 0; + let words: Vec>> = counts + .iter() + .map(|&n| { + at += n; + case.arenas[at - n..at].to_vec() + }) + .collect(); + streamed_top_over(&case.program, &words, split, order, &case.hasher) +} + +/// ★ A streamed top splits its arenas by what each child supplies, not +/// evenly: over a child of one arena and a child of three (a node whose +/// children's AIR sets differ, as the equal-height merge makes them), in +/// either landing order the witness is the serial reference's, word for word. +/// Told an even split (two each, the split the top assumed before), the first +/// child lands one arena where two were expected — the large block's +/// `ArenaCountMismatch { expected: 105, found: 99 }` — and counts that do not +/// cover the program's arenas are refused before anything runs. +#[test] +fn a_streamed_top_splits_its_arenas_by_each_childs_count() { + let counts = [1, 3]; + let case = uneven_children_case(&counts); + let reference = reference(&case); + for order in [[0, 1], [1, 0]] { + let got = streamed_top(&case, &counts, &counts, &order) + .unwrap_or_else(|e| panic!("{order:?}: the streamed top executes: {e}")); + assert_executions_identical(case.name, case.program.num_addrs, &reference, &got); + } + let even = streamed_top(&case, &counts, &[2, 2], &[0, 1]).expect_err("an even split"); + assert!( + even.contains("child 0 lands: ArenaCountMismatch { expected: 2, found: 1 }"), + "the even split fails as the large block did: {even}" + ); + for bad in [&[1, 2][..], &[4][..], &[1, 3, 0][..]] { + let err = streamed_top(&case, &counts, bad, &[0, 1]).expect_err("counts that do not fit"); + assert!(err.contains("arenas over"), "{bad:?}: {err}"); + } +} + +/// An execution refuses to set its public words past their buffer's +/// capacity, rather than asserting it. +#[test] +fn public_words_short_of_the_programs_rows_are_refused() { + use super::executor::public_rows_fit; + let words: Vec = Vec::with_capacity(4); + assert!(public_rows_fit(&words, words.capacity()).is_ok()); + assert!(matches!( + public_rows_fit(&words, words.capacity() + 1), + Err(LfmExecError::Internal(_)) + )); +} diff --git a/prover/src/lfm/exec_schedule.rs b/prover/src/lfm/exec_schedule.rs new file mode 100644 index 000000000..9a0ce24e2 --- /dev/null +++ b/prover/src/lfm/exec_schedule.rs @@ -0,0 +1,570 @@ +//! The execution schedule: which instructions of an LFM program may run at +//! once, computed from the program itself rather than guessed. +//! +//! # Why the levels are exact rather than heuristic +//! +//! Addresses are dense, assigned in emission order, written once, and every +//! operand address is strictly below its destination (`instr.rs`), so the +//! program IS a topologically sorted DAG and one forward pass suffices. +//! +//! Carry, per instruction, `depth` = the length of the longest chain of +//! *dependent [`Instr::Hash`] instructions* ending there. Group by it and the +//! groups have a property no partition heuristic can claim: +//! +//! > **Two hash instructions at the same depth cannot depend on each other.** +//! > If `j` depended on `i` by any path, `depth[j] ≥ depth[i] + 1`, because `i` +//! > is itself a hash. So the levels ARE the available parallelism, with no +//! > segmentation, no guessing, and no soundness argument to make. +//! +//! ★ The same number also orders the work that is NOT a hash, and the argument +//! is one line: if a non-hash `X` has depth `d` and a hash `H` reads `X`, then +//! `H` is a hash, so `depth(H) = d + 1`. **Therefore no hash at depth `d` reads +//! any non-hash at depth `d`** — running a level's hashes first and its non-hash +//! instructions after is safe. Non-hash work at depth `d` may read the hashes at +//! `d` and may read other non-hash work at `d`, and program order (a topological +//! order) covers both. So the whole program is covered by: +//! +//! ```text +//! for d in 0..=D: { all hashes at depth d, in any order } then +//! { all non-hash instructions at depth d, in program order } +//! ``` +//! +//! `Instr::Const` reads nothing, so every constant is depth 0 and the globally +//! interned pool (`builder.rs`) runs first, before anything reads it. +//! +//! # What this module is not +//! +//! It is not [`super::reach_profile`], which measures the same structure for +//! reporting. That one also carries `instr_depth`, a writer array and a +//! histogram, and it allocates a `Vec` per instruction through `Instr::reads`. +//! This one is on the proving path and keeps one array per address and one per +//! instruction, both dropped before the machine is built. + +use super::compiler::LfmProgram; +use super::instr::{Addr, Instr}; + +/// How a program's instructions are grouped for execution. +/// +/// `order`-free by construction: the two index vectors below hold instruction +/// indices (and, for hashes, record row numbers) already grouped by level and +/// ascending inside a level, so the executor never sorts and never searches. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LevelSchedule { + /// Record row numbers of the hash instructions, grouped by level. Ascending + /// within a level, which is program order, which is the order the serial + /// executor appends them in. + hash_rows: Vec, + /// `hash_starts[d] .. hash_starts[d + 1]` is level `d`'s slice of + /// [`Self::hash_rows`]. + hash_starts: Vec, + /// Program index of the hash instruction that owns each record row, indexed + /// BY ROW. `hash_instr[r]` is the `r`-th hash in program order. + hash_instr: Vec, + /// Program indices of the non-hash instructions, grouped by level and + /// ascending (= program order) within one. + other: Vec, + /// `other_starts[d] .. other_starts[d + 1]` is level `d`'s slice of + /// [`Self::other`]. + other_starts: Vec, + /// Each instruction's row in ITS OWN chip's record vector, in PROGRAM order. + /// + /// ⛔ The level order is not program order, so an executor that appended as + /// it went would hand the trace fill ten permuted vectors. Every arm writes + /// its slot instead, and this is the slot. + record_row: Vec, +} + +impl LevelSchedule { + /// Levels in the schedule. Equal to the longest dependent hash chain plus + /// one for the depth-0 level that holds the constants. + pub fn levels(&self) -> usize { + self.hash_starts.len() - 1 + } + + /// Record rows of the hashes at level `d` — mutually independent by the + /// property in this module's header. + pub fn hashes_at(&self, d: usize) -> &[u32] { + let (a, b) = ( + self.hash_starts[d] as usize, + self.hash_starts[d + 1] as usize, + ); + &self.hash_rows[a..b] + } + + /// Program indices of the non-hash instructions at level `d`, in program + /// order. They may depend on each other and on the hashes at `d`. + pub fn others_at(&self, d: usize) -> &[u32] { + let (a, b) = ( + self.other_starts[d] as usize, + self.other_starts[d + 1] as usize, + ); + &self.other[a..b] + } + + /// The program index of the hash instruction that fills record row `row`. + #[inline] + pub fn instr_of_row(&self, row: u32) -> usize { + self.hash_instr[row as usize] as usize + } + + /// Instruction `i`'s row in its own chip's record vector. + #[inline] + pub fn record_row(&self, i: usize) -> u32 { + self.record_row[i] + } + + /// Bytes this schedule holds while the program executes — the figure the + /// design note quotes, computed rather than estimated. + pub fn resident_bytes(&self) -> usize { + size_of::() + * (self.hash_rows.len() + + self.hash_starts.len() + + self.hash_instr.len() + + self.other.len() + + self.other_starts.len() + + self.record_row.len()) + } +} + +/// The forward pass, with a level-merge factor. +/// +/// ⛔ **`merge` is `1` on every production path and the executor passes nothing +/// else.** Any other value is a DELIBERATELY WRONG SCHEDULE and exists for the +/// gate that proves the right one is load-bearing: `merge = 2` puts depths `2k` +/// and `2k+1` in one level, so a hash whose input another hash of the merged +/// pair produces is launched before that input exists. +/// +/// The executor must then fail, on every run and at the same address, rather +/// than sometimes. It does, and the reason is the borrow checker rather than a +/// check: during a level the workers hold `&WriteOnceMemory` and nothing holds +/// `&mut`, so no write can land while the level runs and the premature read is +/// always [`super::executor::LfmExecError::ReadBeforeWrite`] — never a race, and +/// never timing-dependent. A gate that passed with the level boundary removed +/// would not be a gate; this is what shows it does not. +pub fn build_with_merge(program: &LfmProgram, merge: u32) -> LevelSchedule { + assert!(merge >= 1, "a level merge factor is at least 1"); + let n = program.instrs.len(); + assert!( + u32::try_from(n).is_ok(), + "the schedule indexes instructions in a u32; this program has {n}" + ); + + // ⚠ Depth per ADDRESS, not per writing instruction. The reach profile keeps + // a writer index and a per-instruction depth and reads one through the + // other; carrying the depth directly is the same answer in half the memory, + // and it makes an unwritten read fall out correctly rather than needing a + // sentinel: it reads 0, its hash lands in level 0, and the executor's own + // `read_word` raises `ReadBeforeWrite` there. Nothing here has to detect it. + let mut addr_depth = vec![0u32; program.num_addrs as usize]; + let mut depth = vec![0u32; n]; + let (mut hash_counts, mut other_counts) = (vec![0u32; 1], vec![0u32; 1]); + let mut scratch: Vec = Vec::with_capacity(32); + + for (i, instr) in program.instrs.iter().enumerate() { + scratch.clear(); + // A `HintOn`'s bit orders it after the bit's writer, though no + // multiplicity counts it ([`Instr::schedule_reads_into`]). + instr.schedule_reads_into(&mut scratch); + let mut d = 0u32; + for r in &scratch { + // ⚠ `get`, not `[]`, and the reason is a real program rather than + // caution. An address past `num_addrs` is legal to EMIT — the + // executor is what rejects it, a read as `ReadBeforeWrite` and a + // write as `Internal("address out of range")`, and + // `executor_rejects_addresses_outside_the_program` pins both + // directions. A schedule that panicked here would turn that + // rejectable program into a crash before the executor ever saw it. + // Treating the address as depth 0 leaves the rejection exactly + // where it was. + d = d.max(addr_depth.get(r.0 as usize).copied().unwrap_or(0)); + } + let is_hash = matches!(instr, Instr::Hash { .. } | Instr::Hash16(_)); + if is_hash { + d += 1; + } + scratch.clear(); + instr.writes_into(&mut scratch); + for w in &scratch { + // Same rule: nothing to record for an address the machine has no + // cell for, and the executor still errors when it reaches it. + if let Some(slot) = addr_depth.get_mut(w.0 as usize) { + *slot = d; + } + } + + // Rounded UP, so `merge = 2` pairs depths 1+2, 3+4, … and leaves depth 0 + // alone. That matters for the mutation gate: collapsing depth 0 into + // depth 1 would break the schedule at the constants, and the resulting + // failure would say nothing about whether a hash can be launched before + // the HASH it depends on. With the offset, the first thing the merged + // schedule gets wrong is exactly that. + let level = (d.div_ceil(merge)) as usize; + depth[i] = level as u32; + let counts = if is_hash { + &mut hash_counts + } else { + &mut other_counts + }; + if counts.len() <= level { + counts.resize(level + 1, 0); + } + counts[level] += 1; + } + drop(addr_depth); + + let num_levels = hash_counts.len().max(other_counts.len()); + hash_counts.resize(num_levels, 0); + other_counts.resize(num_levels, 0); + + // Prefix sums, then a stable scatter: `counts` becomes the cursor array and + // ends up one level shifted, which is exactly the starts array with a + // leading zero prepended. + let starts = |counts: &[u32]| -> Vec { + let mut s = Vec::with_capacity(counts.len() + 1); + let mut acc = 0u32; + s.push(0); + for &c in counts { + acc += c; + s.push(acc); + } + s + }; + let hash_starts = starts(&hash_counts); + let other_starts = starts(&other_counts); + + let hash_total = *hash_starts.last().expect("starts is never empty") as usize; + let other_total = *other_starts.last().expect("starts is never empty") as usize; + let mut hash_rows = vec![0u32; hash_total]; + let mut hash_instr = vec![0u32; hash_total]; + let mut other = vec![0u32; other_total]; + let mut hash_cursor = hash_starts.clone(); + let mut other_cursor = other_starts.clone(); + + // Program order, so every bucket ends up ascending: the hash rows of a level + // come out in the order the serial executor would have pushed them, and the + // non-hash instructions of a level come out in program order, which is the + // topological order they need. + let mut row = 0u32; + let mut record_row = vec![0u32; n]; + let mut chip_rows = ChipRows::default(); + for (i, instr) in program.instrs.iter().enumerate() { + let level = depth[i] as usize; + record_row[i] = chip_rows.take(instr); + if matches!(instr, Instr::Hash { .. } | Instr::Hash16(_)) { + hash_instr[row as usize] = i as u32; + let at = &mut hash_cursor[level]; + hash_rows[*at as usize] = row; + *at += 1; + row += 1; + } else { + let at = &mut other_cursor[level]; + other[*at as usize] = i as u32; + *at += 1; + } + } + chip_rows.assert_matches_census(program); + + LevelSchedule { + hash_rows, + hash_starts, + hash_instr, + other, + other_starts, + record_row, + } +} + +/// One row counter per chip, advanced in program order. +/// +/// ★ **This is where the length check that `resize` would have made vacuous went +/// instead, and it is a stronger one.** The executor used to `push` and assert +/// at the end that each vector's length equalled its column group's row count — +/// a real check that every instruction produced exactly one record. Slot-writing +/// sets those lengths up front, so that assertion would pass whatever happened. +/// Counting the rows the schedule hands out and comparing THOSE to the census +/// says the same thing, earlier, and about the schedule rather than about the +/// vector. +#[derive(Default)] +struct ChipRows { + const_: u32, + balu: u32, + xalu: u32, + select: u32, + bitdec: u32, + hash: u32, + keccak: u32, + blake3: u32, + lanes: u32, + hint: u32, + public: u32, +} + +impl ChipRows { + /// The next row for `instr`'s chip, post-incrementing that chip's counter. + /// Every arm of the executor pushes exactly one record, so every + /// instruction takes exactly one row. + #[inline] + fn take(&mut self, instr: &Instr) -> u32 { + let c = match instr { + Instr::Const { .. } => &mut self.const_, + Instr::BaseAlu { .. } => &mut self.balu, + Instr::ExtAlu { .. } => &mut self.xalu, + Instr::Select { .. } => &mut self.select, + Instr::BitDec { .. } => &mut self.bitdec, + // One chip: a program's `LFM_HASH` rows are all `Hash` or all + // `Hash16`. + Instr::Hash { .. } | Instr::Hash16(_) => &mut self.hash, + Instr::KeccakF(_) => &mut self.keccak, + Instr::Blake3(_) => &mut self.blake3, + // Pack and Unpack are one chip: both open an `LFM_LANES` row. + Instr::Pack { .. } | Instr::Unpack { .. } => &mut self.lanes, + Instr::Hint { .. } | Instr::HintOn { .. } => &mut self.hint, + Instr::Public { .. } => &mut self.public, + }; + let row = *c; + *c += 1; + row + } + + fn assert_matches_census(&self, program: &LfmProgram) { + let g = &program.groups; + let mine = [ + self.const_, + self.balu, + self.xalu, + self.select, + self.bitdec, + self.hash, + self.keccak, + self.blake3, + self.lanes, + self.hint, + self.public, + ]; + let census = [ + g.const_.real_rows, + g.balu.real_rows, + g.xalu.real_rows, + g.select.real_rows, + g.bitdec.real_rows, + g.hash.real_rows, + g.keccak.real_rows, + g.blake3.real_rows, + g.lanes.real_rows, + g.hint.real_rows, + g.public.real_rows, + ]; + assert_eq!( + mine.map(|v| v as usize), + census, + "the schedule handed out a different number of record rows than the \ + compiler emitted column-group rows (order: const, balu, xalu, \ + select, bitdec, hash, keccak, blake3, lanes, hint, public)" + ); + } +} + +/// The schedule a program executes under. +pub fn build(program: &LfmProgram) -> LevelSchedule { + build_with_merge(program, 1) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lfm::builder::LfmBuilder; + use crate::lfm::compiler::compile; + use crate::lfm::word::LfmWord; + use crate::tables::types::FE; + + fn word(v: u64) -> LfmWord { + core::array::from_fn(|i| FE::from(v + i as u64)) + } + + /// ★ The gate: a program whose parallel structure is known by construction, + /// and the schedule must report exactly it. + /// + /// Four independent chains of three compressions over a shared seed. The + /// right answer is a depth-0 level holding the two constants and no hashes, + /// then three levels of four hashes each. + fn four_chains_of_three() -> crate::lfm::compiler::LfmProgram { + let mut b = LfmBuilder::new(); + let seed = b.digest_const(word(1)); + for chain in 0..4u64 { + let mut acc = b.digest_const(word(100 + chain)); + for _ in 0..3 { + acc = b.compress(acc, seed); + } + } + compile(b.finish()) + } + + #[test] + fn the_schedule_reports_a_known_chain_structure() { + let s = build(&four_chains_of_three()); + assert_eq!(s.levels(), 4, "one const level and three hash levels"); + assert!( + s.hashes_at(0).is_empty(), + "a hash is at least depth 1, so level 0 holds no hashes" + ); + for d in 1..4 { + assert_eq!( + s.hashes_at(d).len(), + 4, + "four independent compressions at depth {d}" + ); + } + // Row numbers partition 0..12 exactly once: the record vector is filled + // by slot, so a repeated or missing row is a silently wrong witness. + let mut seen: Vec = (0..4).flat_map(|d| s.hashes_at(d).to_vec()).collect(); + seen.sort_unstable(); + assert_eq!(seen, (0..12).collect::>(), "every row exactly once"); + // Ascending within a level, which is what lets the apply step walk a + // level's rows in the order the serial executor pushed them. + for d in 0..4 { + let rows = s.hashes_at(d); + assert!(rows.windows(2).all(|w| w[0] < w[1]), "level {d} ascends"); + } + } + + /// The counter-case, so the test above is not passing for a shape-blind + /// reason: ONE chain of twelve is twelve levels of one, and no schedule + /// finds parallelism that is not there. + #[test] + fn a_single_chain_has_no_parallelism_to_find() { + let mut b = LfmBuilder::new(); + let seed = b.digest_const(word(1)); + let mut acc = b.digest_const(word(7)); + for _ in 0..12 { + acc = b.compress(acc, seed); + } + let s = build(&compile(b.finish())); + assert_eq!(s.levels(), 13, "twelve chained hashes plus the const level"); + for d in 1..13 { + assert_eq!(s.hashes_at(d).len(), 1, "a chain is one wide at every step"); + } + } + + /// ★ Every instruction lands in exactly one level, hash or not. A schedule + /// that dropped one would execute a program with a hole in it, and the + /// witness would be wrong in a way only the prover would notice. + #[test] + fn every_instruction_is_scheduled_exactly_once() { + let program = crate::lfm::programs::fri_toy_program(); + let s = build(&program); + let mut seen = vec![0u32; program.instrs.len()]; + for d in 0..s.levels() { + for &row in s.hashes_at(d) { + seen[s.instr_of_row(row)] += 1; + } + for &i in s.others_at(d) { + seen[i as usize] += 1; + } + } + assert!( + seen.iter().all(|&c| c == 1), + "{} of {} instructions were scheduled a number of times other than once", + seen.iter().filter(|&&c| c != 1).count(), + seen.len(), + ); + } + + /// ★★ The ordering property the whole design rests on, checked on a real + /// program rather than argued: **no instruction at level `d` reads an + /// address written at a level above `d`, and no hash at level `d` reads one + /// written by a non-hash at level `d`.** The second half is what makes + /// "hashes first, then the rest" safe inside a level. + #[test] + fn a_level_never_reads_above_itself() { + let program = crate::lfm::programs::fri_toy_program(); + let s = build(&program); + + // level[i], and whether i is a hash — rebuilt from the schedule, not + // from a second depth pass, so this checks the schedule and not a copy + // of the code that made it. + let mut level = vec![u32::MAX; program.instrs.len()]; + let mut is_hash = vec![false; program.instrs.len()]; + for d in 0..s.levels() { + for &row in s.hashes_at(d) { + let i = s.instr_of_row(row); + level[i] = d as u32; + is_hash[i] = true; + } + for &i in s.others_at(d) { + level[i as usize] = d as u32; + } + } + + let mut writer = vec![u32::MAX; program.num_addrs as usize]; + for (i, instr) in program.instrs.iter().enumerate() { + for w in instr.writes() { + writer[w.0 as usize] = i as u32; + } + } + + let (mut checked, mut hash_checked) = (0u64, 0u64); + for (i, instr) in program.instrs.iter().enumerate() { + for r in instr.schedule_reads() { + let w = writer[r.0 as usize]; + if w == u32::MAX { + continue; // unwritten; the executor raises ReadBeforeWrite + } + let (mine, theirs) = (level[i], level[w as usize]); + assert!( + theirs <= mine, + "instruction {i} at level {mine} reads address {} written at \ + level {theirs} — the schedule would run it too early", + r.0 + ); + checked += 1; + if is_hash[i] { + // ★ The half that makes "hashes first, then the rest" safe: + // a hash reads NOTHING at its own level. Not "nothing from a + // non-hash at its own level" — nothing at all, because a + // hash reading another hash would be one level deeper by the + // definition of the depth. + assert!( + theirs < mine, + "hash {i} at level {mine} reads address {} written at \ + the SAME level by {w} — running a level's hashes before \ + its other work would launch this one too early", + r.0 + ); + hash_checked += 1; + } + } + } + assert!( + checked > 500 && hash_checked > 50, + "the program must exercise both halves of the property: {checked} \ + read edges of which {hash_checked} are a hash's" + ); + } + + /// The mutation the gate in `exec_identity_tests` rides on, checked here at + /// the structural level: merging adjacent depths really does put a hash and + /// one of its inputs in one level, so there is something for the executor to + /// fail on. + #[test] + fn merging_levels_puts_a_hash_in_the_same_level_as_its_input() { + let program = four_chains_of_three(); + let merged = build_with_merge(&program, 2); + assert!( + merged.levels() < build(&program).levels(), + "the merge must actually collapse levels" + ); + // Depths 1 and 2 — two HASH depths, one feeding the other — now share + // level 1, while depth 0 (the constants) keeps its own level. + assert_eq!( + merged.hashes_at(0).len(), + 0, + "depth 0 must stay a hash-free level, or the mutation would break at \ + the constants and prove nothing about hash ordering" + ); + assert_eq!( + merged.hashes_at(1).len(), + 8, + "depths 1 and 2 hold four hashes each and must now share a level" + ); + } +} diff --git a/prover/src/lfm/executor.rs b/prover/src/lfm/executor.rs new file mode 100644 index 000000000..950ec6483 --- /dev/null +++ b/prover/src/lfm/executor.rs @@ -0,0 +1,1825 @@ +//! The LFM executor / witness generator. +//! +//! One `for` over the straight-line program, against write-once memory and +//! the host-supplied arenas. Produces per-chip **value-only** records — +//! addresses, selectors and multiplicities come from the program (they are +//! preprocessed data), so records carry values only, and the executor ignores +//! `mult` entirely: execution semantics never depend on it. +//! +//! Defense in depth the reference machine omits: double-writes and +//! read-before-write are checked at runtime here, independently of both the +//! compiler's tripwire panics and the admission validator. + +use std::time::Instant; + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::blake3_chip::Blake3Values; +use super::compiler::{LfmColumnGroups, LfmProgram}; +use super::exec_schedule::{self, LevelSchedule}; +use super::hash::{HASH_STATE_FELTS, LfmHasher}; +use super::instr::{Addr, BaseOp, ExtOp, HashMode, Instr, KeccakMode}; +use super::word::{LfmWord, base_word, ext_word}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LfmExecError { + DoubleWrite(u64), + ReadBeforeWrite(u64), + /// `x / 0` with `x ≠ 0` — this is also how a failed assertion surfaces. + DivByZero { + addr: u64, + }, + NonBooleanBit(u64), + /// A base-typed read found nonzero lanes 1–3 (the bus token would not + /// match any base write, so the AIR-side program would be unprovable). + NotBaseWord(u64), + /// An ext-typed read found a nonzero lane 3. + NotExtWord(u64), + /// A `KeccakF` or `Blake3` input word lane held a value at or above `2^32`, + /// so it is not a `u32` half of a keccak lane (respectively a BLAKE3 input + /// word). Both chips recompose each `u32` from four BITWISE-constrained byte + /// columns, so no such value exists on the AIR side — the program would be + /// unprovable. + NotU32Half { + addr: u64, + lane: usize, + }, + /// A `KeccakF` input word's unused top lane (the state is 50 halves in 52 + /// slots) was nonzero; the bus pins those slots to zero as tuple constants. + KeccakSpareLaneNonZero { + addr: u64, + lane: usize, + }, + ArenaCountMismatch { + expected: usize, + found: usize, + }, + ArenaLenMismatch { + arena: u32, + expected: u32, + found: usize, + }, + ArenaOutOfBounds { + arena: u32, + index: u32, + }, + /// An `Instr::Hash` outside the selected hasher's domain, with the reason + /// the hasher gave (`LfmHasher::admits`). BLAKE3 raises both of its: a + /// `Permute` row, for which it has no socket, and a `Compress` input lane + /// at or above `2^32`, which its chip cannot decompose into bytes. In both + /// cases the program is unprovable under that hasher, so failing here — with + /// a reason — beats failing later inside the prover. + HasherRejected(&'static str), + Internal(&'static str), +} + +// ---- per-chip value records (values only; the program carries the rest) ---- + +#[derive(Debug, Clone)] +pub struct BaluRow { + pub a: FE, + pub b: FE, + pub c: FE, + pub out: FE, +} + +#[derive(Debug, Clone)] +pub struct XaluRow { + pub a: [FE; 3], + pub b: [FE; 3], + pub c: [FE; 3], + pub out: [FE; 3], +} + +#[derive(Debug, Clone)] +pub struct SelectRow { + pub bit: FE, + pub in_l: LfmWord, + pub in_r: LfmWord, + pub out_l: LfmWord, + pub out_r: LfmWord, +} + +#[derive(Debug, Clone)] +pub struct BitDecRow { + /// All 64 bit values, low-to-high (constrained witness columns). + pub bits: [FE; 64], + /// The canonicity gadget's witnesses: `z` = "top 32 bits all ones", + /// `ginv` = inverse of `(2^32 − 1) − top` when that is nonzero. + pub z: FE, + pub ginv: FE, +} + +#[derive(Debug, Clone)] +pub struct HashRow { + /// The 12 input columns: full state for `Permute`; `[a ‖ b ‖ 0⁴]` for the + /// two-to-one modes (lanes 8–11 are unconstrained on those rows — the AIR + /// injects the IV there via the mode selector). + pub ins: [FE; HASH_STATE_FELTS], + /// The full permuted state. + pub outs: [FE; HASH_STATE_FELTS], +} + +/// One width-16 `LFM_HASH` row (`Instr::Hash16`): the sixteen lanes read and +/// the sixteen the permutation returns. +#[derive(Debug, Clone)] +pub struct Hash16Row { + pub ins: [FE; 16], + pub outs: [FE; 16], +} + +/// One `LFM_KECCAK` row. The 400 byte columns are derived from these two +/// states; the tag is the row ordinal (`layout::keccak::tag_for_row`), so it is +/// not recorded here — it is program data, not witness. +#[derive(Debug, Clone)] +pub struct KeccakRow { + pub mode: KeccakMode, + /// The state as received from memory. + pub state: [u64; 25], + /// The 136-byte rate block as received (all zero on `Permute` rows, where + /// the block columns are dead — nothing reads them). + pub block: [u8; 136], + /// What actually enters the permutation: `state` with `block` XORed into + /// its rate region on absorb rows, `state` unchanged on permute rows. + pub perm_in: [u64; 25], + pub output: [u64; 25], +} + +#[derive(Debug, Default)] +pub struct LfmRecords { + pub num_consts: usize, + pub balu: Vec, + pub xalu: Vec, + pub select: Vec, + pub bitdec: Vec, + pub hash: Vec, + /// The `LFM_HASH` rows of a width-16 program (`LfmProgram::hash16`); such a + /// program's `hash` is empty, and every other program's `hash16` is. + pub hash16: Vec, + pub keccak: Vec, + /// One `LFM_BLAKE3` row. Values only, like every other record: the chip's + /// addresses and multiplicities are preprocessed program data. + pub blake3: Vec, + /// One word per Pack/Unpack row (the shared value columns). + pub lanes: Vec, + pub hint: Vec, + pub public: Vec, +} + +impl LfmRecords { + /// Records sized from the census the compiler already holds. + /// + /// Pass 2 opens exactly one column-group row per instruction, and this + /// executor pushes exactly one record per instruction of the same chip — so + /// `ColumnGroup::real_rows` *is* each vector's final length, not an upper + /// bound. The trace fill already depends on that identity: it fills rows + /// `0..group.real_rows` by indexing the record vector + /// ([`super::trace`]), so a vector short of its group's row count is + /// already a panic there. + /// + /// Growing these by `push` instead re-allocated and copied every one of them + /// ~log2(rows) times per proof — a few hundred MB of memcpy and ~20 large + /// `mremap`s under the process allocator, on every concurrent worker at once. + fn with_capacity(groups: &LfmColumnGroups, hash16: bool) -> Self { + let (narrow, wide) = if hash16 { + (0, groups.hash.real_rows) + } else { + (groups.hash.real_rows, 0) + }; + LfmRecords { + num_consts: 0, + balu: Vec::with_capacity(groups.balu.real_rows), + xalu: Vec::with_capacity(groups.xalu.real_rows), + select: Vec::with_capacity(groups.select.real_rows), + bitdec: Vec::with_capacity(groups.bitdec.real_rows), + hash: Vec::with_capacity(narrow), + hash16: Vec::with_capacity(wide), + keccak: Vec::with_capacity(groups.keccak.real_rows), + blake3: Vec::with_capacity(groups.blake3.real_rows), + lanes: Vec::with_capacity(groups.lanes.real_rows), + hint: Vec::with_capacity(groups.hint.real_rows), + public: Vec::with_capacity(groups.public.real_rows), + } + } +} + +impl LfmRecords { + /// Publish every slot the level walk wrote. + /// + /// # Safety + /// + /// Every row `0..real_rows` of every vector must already have been written + /// through [`Sink::Slot`]. Three things establish that together, and the + /// third is what makes it an observation rather than a hope: + /// + /// 1. the schedule hands out exactly one row per instruction, ascending from + /// zero per chip, and `ChipRows::assert_matches_census` fails the run + /// before anything executes unless those per-chip totals equal the + /// `real_rows` this function is about to set; + /// 2. `every_instruction_is_scheduled_exactly_once` pins that the level + /// index lists every instruction once, so every row is reached; + /// 3. `exec_identity_tests` READS every element of every record vector and + /// compares it to the serial reference's, so a slot that was never + /// written is a failing test rather than a quiet zero. + /// + /// The caller must also call this ONLY on the success path. It is invoked at + /// exactly one place for that reason: any `?` before it drops zero-length + /// vectors, which frees the buffer without reading a slot. + unsafe fn commit_slots(&mut self, groups: &LfmColumnGroups, hash16: bool) { + // A `set_len` past the capacity would be the one way to turn the + // reasoning above into unsoundness, so it is checked rather than + // reasoned about, in every build. + macro_rules! commit { + ($($v:ident),+ $(,)?) => {$( + assert!( + self.$v.capacity() >= groups.$v.real_rows, + concat!("record slots for ", stringify!($v), " were not reserved"), + ); + unsafe { self.$v.set_len(groups.$v.real_rows) }; + )+}; + } + commit!( + balu, xalu, select, bitdec, keccak, blake3, lanes, hint, public + ); + // `LFM_HASH`'s rows are one vector or the other, by the program's width. + let rows = groups.hash.real_rows; + if hash16 { + assert!( + self.hash16.capacity() >= rows, + "record slots for hash16 were not reserved" + ); + unsafe { self.hash16.set_len(rows) }; + } else { + assert!( + self.hash.capacity() >= rows, + "record slots for hash were not reserved" + ); + unsafe { self.hash.set_len(rows) }; + } + } + + /// Bytes the record vectors hold once filled — what `setup` first-touches. + fn bytes(groups: &LfmColumnGroups, hash16: bool) -> usize { + let hash_row = if hash16 { + size_of::() + } else { + size_of::() + }; + groups.balu.real_rows * size_of::() + + groups.xalu.real_rows * size_of::() + + groups.select.real_rows * size_of::() + + groups.bitdec.real_rows * size_of::() + + groups.hash.real_rows * hash_row + + groups.keccak.real_rows * size_of::() + + groups.blake3.real_rows * size_of::() + + (groups.lanes.real_rows + groups.hint.real_rows + groups.public.real_rows) + * size_of::() + } +} + +/// Where an executed instruction puts its record. +/// +/// ⛔ **The level schedule does not run instructions in program order, but the +/// records must end up in it** — the trace fill indexes a chip's vector by row +/// and the row is the instruction's ordinal within its chip. So the serial loop +/// appends, as it always has, and the level loop writes the slot the schedule +/// assigned. One set of arms serves both; the only thing that differs is where +/// the row lands. +#[derive(Debug, Clone, Copy)] +enum Sink { + /// Program order is the walk order: append. + Push, + /// Level order is the walk order: write row `n` of this chip. + Slot(u32), +} + +impl Sink { + #[inline] + fn put(self, v: &mut Vec, value: T) { + match self { + Sink::Push => v.push(value), + // ⛔ SAFETY, and the whole of it. `v` was built by + // `LfmRecords::with_capacity`, so it has capacity for exactly + // `real_rows` elements and length ZERO; `n` is a row the schedule + // handed out, and `ChipRows::assert_matches_census` has already + // failed the run if the schedule's per-chip row count is not that + // same `real_rows`. So `n < capacity` and this writes inside the + // allocation. It writes rather than assigns because the slot holds + // no value yet — assigning would drop whatever bytes are there. + // + // The vector's LENGTH stays zero until `commit_slots`, which runs at + // one place: immediately before the success value is built, after + // every instruction has run. Any `?` on the way out therefore drops + // a zero-length vector, and a slot written but never committed is + // freed without ever being read. That is what makes a partially + // executed program safe rather than merely unlikely. + Sink::Slot(n) => { + debug_assert!( + (n as usize) < v.capacity(), + "row {n} is outside the chip's reserved capacity {}", + v.capacity() + ); + unsafe { v.as_mut_ptr().add(n as usize).write(value) } + } + } + } +} + +/// The executor's write-once memory: one value array plus one occupancy bit per +/// address. +/// +/// This used to be a single `Vec>`. `LfmWord` is `[F; 4]` = 32 +/// bytes and `F` has no niche, so `Option` is **40** bytes: the +/// occupancy flag cost 8 bytes per address *and* pushed the stride off the +/// cache line, so half of all cells spanned two lines. Splitting the flag out +/// restores the 32-byte stride (a cell never spans two lines) and puts the whole +/// occupancy map in `num_addrs / 8` bytes, which stays cache-resident where the +/// value array cannot. +/// +/// The write-once rules are unchanged and still checked here, independently of +/// the compiler's tripwire panics and of the admission validator: a second write +/// to an address is [`LfmExecError::DoubleWrite`], and a read of an address no +/// instruction has written yet is [`LfmExecError::ReadBeforeWrite`]. ⚠ The bit +/// is the *only* thing that separates "unwritten" from "written zero" now — the +/// value array is zero-filled, so dropping the check would silently hand out +/// zeros instead of failing. The `executor_rejects_read_before_write` test is +/// what makes that unreachable. +#[derive(Debug)] +pub struct WriteOnceMemory { + words: Vec, + /// Bit `i % 64` of word `i / 64` is "address `i` has been written". + written: Vec, +} + +impl WriteOnceMemory { + fn new(num_addrs: usize) -> Self { + WriteOnceMemory { + words: vec![[FE::zero(); 4]; num_addrs], + written: vec![0u64; num_addrs.div_ceil(64)], + } + } + + /// The final value at `addr`, or `None` if no instruction wrote it (which + /// includes an address outside the program's range). + pub fn get(&self, addr: Addr) -> Option { + self.read(addr).ok() + } + + #[inline] + fn read(&self, addr: Addr) -> Result { + let i = addr.0 as usize; + let Some(&w) = self.words.get(i) else { + return Err(LfmExecError::ReadBeforeWrite(addr.0)); + }; + // `written` is sized `words.len().div_ceil(64)`, so the index is in + // range here; `get` rather than `[]` keeps a would-be panic an error. + let bits = self.written.get(i >> 6).copied().unwrap_or(0); + if (bits >> (i & 63)) & 1 == 0 { + return Err(LfmExecError::ReadBeforeWrite(addr.0)); + } + Ok(w) + } + + #[inline] + fn write(&mut self, addr: Addr, w: LfmWord) -> Result<(), LfmExecError> { + let i = addr.0 as usize; + let slot = self + .words + .get_mut(i) + .ok_or(LfmExecError::Internal("address out of range"))?; + let bits = self + .written + .get_mut(i >> 6) + .ok_or(LfmExecError::Internal("address out of range"))?; + let mask = 1u64 << (i & 63); + if *bits & mask != 0 { + return Err(LfmExecError::DoubleWrite(addr.0)); + } + *bits |= mask; + *slot = w; + Ok(()) + } +} + +#[derive(Debug)] +pub struct LfmExecution { + pub records: LfmRecords, + /// The public output, in emission order: `(index, word)`. + pub public_words: Vec<(u32, LfmWord)>, + /// Final memory, exposed for tests and debugging. + pub memory: WriteOnceMemory, + /// Where the walk's time went. Diagnostic only: no proving decision reads + /// it, and it is deliberately NOT compared by the identity gate, because a + /// clock is the one thing two schedules are expected to disagree about. + pub split: ExecSplit, +} + +struct Machine<'a> { + memory: WriteOnceMemory, + /// The arenas by id. A streamed execution ([`StreamedExecution`]) leaves an + /// arena empty until its group lands, so a read of it before then is + /// `ArenaOutOfBounds` — an error, never a value. + arenas: Vec<&'a [LfmWord]>, +} + +impl Machine<'_> { + #[inline] + fn write(&mut self, addr: Addr, w: LfmWord) -> Result<(), LfmExecError> { + self.memory.write(addr, w) + } + + #[inline] + fn read_word(&self, addr: Addr) -> Result { + self.memory.read(addr) + } + + fn read_base(&self, addr: Addr) -> Result { + let w = self.read_word(addr)?; + super::word::word_as_base(&w).ok_or(LfmExecError::NotBaseWord(addr.0)) + } + + fn read_ext(&self, addr: Addr) -> Result { + let w = self.read_word(addr)?; + super::word::word_as_ext(&w).ok_or(LfmExecError::NotExtWord(addr.0)) + } +} + +/// The READ-ONLY half of a hash instruction: gather its input cells, check the +/// hasher admits the row, run the permutation, and hand back the record. +/// +/// ⛔ **This half takes `&WriteOnceMemory` and that is the whole safety argument +/// of the parallel schedule.** A level's hashes are mutually independent +/// ([`super::exec_schedule`]), so they may run at once — but only because +/// nothing in this function can write. The borrow checker, not a comment and not +/// a runtime check, is what stops a worker from publishing a cell another worker +/// in the same level might read: there is no `&mut` to do it with. A hash +/// launched too early therefore reads a cell nobody has written and gets +/// [`LfmExecError::ReadBeforeWrite`] deterministically, on every run, instead of +/// a race whose outcome depends on the thread schedule. +/// +/// It is also the one transcription of the hash semantics. The serial loop and +/// the level loop both call it, so the two schedules cannot drift apart in what +/// they compute — only in when they compute it. +fn hash_compute( + memory: &WriteOnceMemory, + hasher: &impl LfmHasher, + mode: HashMode, + ins: &[Addr; 3], +) -> Result { + let mut state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|_| FE::zero()); + let mut in_cols: [FE; HASH_STATE_FELTS] = core::array::from_fn(|_| FE::zero()); + if mode.num_input_cells() == 2 { + // Two cells, whatever they MEAN: two digests under Compress and + // Transcript, a chaining accumulator and four field elements under Leaf. + // What each cell is read AS belongs to the hasher and to the chip's lane + // split; what the executor owes is the memory reads the `LfmMem` + // receives claim, and those are the same two under all three. + let a = memory.read(ins[0])?; + let b = memory.read(ins[1])?; + state[0..4].clone_from_slice(&a); + state[4..8].clone_from_slice(&b); + // The capacity is the MODE's, not always the compress one: a hasher that + // domain-separates through the capacity (RPO) makes a transcript step + // and a parent different functions here, and the chip's `S8` copy + // constraint agrees because both read `LfmHasher::mode_iv`. + state[8..12].clone_from_slice(&hasher.mode_iv(mode)); + in_cols[0..4].clone_from_slice(&a); + in_cols[4..8].clone_from_slice(&b); + // lanes 8–11 of the IN columns stay zero on two-cell rows + } else { + for (cell, chunk) in ins.iter().zip(state.chunks_exact_mut(4)) { + chunk.clone_from_slice(&memory.read(*cell)?); + } + in_cols = state; + } + // A hasher whose socket does not cover this row says so here, with a reason, + // rather than producing a witness no AIR accepts. + hasher + .admits(mode, &state) + .map_err(LfmExecError::HasherRejected)?; + let out_state = match mode { + // Through `compress_out`/`transcript_out`, NOT `permute`: a hasher that + // overrides the two-to-one construction — BLAKE3 does, its IV entering + // through `h` rather than the capacity lanes, and its transcript domain + // differing from its Merkle one — must have both overrides reach the + // `OUT` columns. + HashMode::Compress | HashMode::Transcript => { + let a: LfmWord = core::array::from_fn(|i| state[i]); + let b: LfmWord = core::array::from_fn(|i| state[4 + i]); + if mode == HashMode::Compress { + hasher.compress_out(&a, &b) + } else { + hasher.transcript_out(&a, &b) + } + } + HashMode::Leaf => { + let acc: LfmWord = core::array::from_fn(|i| state[i]); + let f: LfmWord = core::array::from_fn(|i| state[4 + i]); + hasher.leaf_out(&acc, &f) + } + HashMode::Permute => hasher.permute(state), + }; + Ok(HashRow { + ins: in_cols, + outs: out_state, + }) +} + +/// The WRITING half: publish a computed row's output cells. +/// +/// Always on the thread that owns the memory, under both schedules, so +/// [`LfmExecError::DoubleWrite`] stays exactly the check it has always been and +/// the occupancy bitset is never touched from two threads at once. That is why +/// the parallel path needs no atomics: making `written` an `AtomicU64` would put +/// a read-modify-write on every one of a wrap's ~6–9 M writes, on the serial +/// path too, to protect a word this design never shares. +fn hash_apply( + memory: &mut WriteOnceMemory, + mode: HashMode, + outs: &[Addr; 3], + row: &HashRow, +) -> Result<(), LfmExecError> { + if mode.num_output_cells() == 1 { + let digest: LfmWord = core::array::from_fn(|i| row.outs[i]); + memory.write(outs[0], digest)?; + } else { + for (cell, chunk) in outs.iter().zip(row.outs.chunks_exact(4)) { + let w: LfmWord = core::array::from_fn(|i| chunk[i]); + memory.write(*cell, w)?; + } + } + Ok(()) +} + +/// A width-16 row's read-only half: the four input words, permuted. +fn hash16_compute( + memory: &WriteOnceMemory, + hasher: &impl LfmHasher, + ins: &[Addr; 4], +) -> Result { + hasher + .admits_width16() + .map_err(LfmExecError::HasherRejected)?; + let mut state = [FE::zero(); 16]; + for (cell, chunk) in ins.iter().zip(state.chunks_exact_mut(4)) { + chunk.copy_from_slice(&memory.read(*cell)?); + } + Ok(Hash16Row { + ins: state, + outs: crypto::hash::poseidon1_w16::permute(state), + }) +} + +/// A width-16 row's write half: all four output words. One nobody reads is +/// written all the same (its multiplicity is zero), as `Instr::writes_into` +/// lists it. +fn hash16_apply( + memory: &mut WriteOnceMemory, + outs: &[Addr; 4], + row: &Hash16Row, +) -> Result<(), LfmExecError> { + for (cell, chunk) in outs.iter().zip(row.outs.chunks_exact(4)) { + let w: LfmWord = core::array::from_fn(|i| chunk[i]); + memory.write(*cell, w)?; + } + Ok(()) +} + +/// One instruction, against the machine — the body the serial loop always had, +/// lifted out so the level schedule runs the SAME arms rather than a second copy +/// of them. +/// +/// The level schedule calls this only for non-hash instructions; its hash rows +/// go through [`hash_compute`] and [`hash_apply`], which this function's own +/// `Hash` arm is itself built from. So both schedules execute one transcription +/// of every arm. +fn step( + m: &mut Machine<'_>, + records: &mut LfmRecords, + public_words: &mut Vec<(u32, LfmWord)>, + instr: &Instr, + hasher: &impl LfmHasher, + sink: Sink, +) -> Result<(), LfmExecError> { + { + match instr { + Instr::Const { out, value, .. } => { + m.write(*out, *value)?; + records.num_consts += 1; + } + Instr::BaseAlu { + op, out, a, b, c, .. + } => { + let av = m.read_base(*a)?; + let bv = m.read_base(*b)?; + let cv = if *op == BaseOp::MulAdd { + m.read_base(*c)? + } else { + FE::zero() + }; + let ov = match op { + BaseOp::Add => &av + &bv, + BaseOp::Sub => &av - &bv, + BaseOp::Mul => &av * &bv, + BaseOp::Div => { + if bv == FE::zero() { + if av == FE::zero() { + FE::one() // the 0/0 = 1 convention + } else { + return Err(LfmExecError::DivByZero { addr: a.0 }); + } + } else { + &av * &bv.inv().map_err(|_| LfmExecError::Internal("base inv"))? + } + } + BaseOp::MulAdd => &av * &bv + &cv, + }; + m.write(*out, base_word(ov))?; + sink.put( + &mut records.balu, + BaluRow { + a: av, + b: bv, + c: cv, + out: ov, + }, + ); + } + Instr::ExtAlu { + op, out, a, b, c, .. + } => { + let ae = m.read_ext(*a)?; + let (be, bv_base) = if *op == ExtOp::MulBase { + let bb = m.read_base(*b)?; + (FEE::zero(), Some(bb)) + } else { + (m.read_ext(*b)?, None) + }; + let ce = if *op == ExtOp::MulAdd { + m.read_ext(*c)? + } else { + FEE::zero() + }; + let oe = match op { + ExtOp::Add => &ae + &be, + ExtOp::Sub => &ae - &be, + ExtOp::Mul => &ae * &be, + ExtOp::Div => { + if be == FEE::zero() { + if ae == FEE::zero() { + FEE::one() // 0/0 = (1, 0, 0) + } else { + return Err(LfmExecError::DivByZero { addr: a.0 }); + } + } else { + &ae * &be.inv().map_err(|_| LfmExecError::Internal("ext inv"))? + } + } + ExtOp::MulAdd => &ae * &be + &ce, + ExtOp::MulBase => { + let bb = bv_base.ok_or(LfmExecError::Internal("mulbase"))?; + let [a0, a1, a2] = *ae.value(); + FEE::new([&a0 * &bb, &a1 * &bb, &a2 * &bb]) + } + }; + m.write(*out, ext_word(&oe))?; + let lanes = |e: &FEE| -> [FE; 3] { *e.value() }; + sink.put( + &mut records.xalu, + XaluRow { + a: lanes(&ae), + b: bv_base.map_or_else(|| lanes(&be), |bb| [bb, FE::zero(), FE::zero()]), + c: lanes(&ce), + out: lanes(&oe), + }, + ); + } + Instr::Select { + bit, + out_l, + out_r, + in_l, + in_r, + .. + } => { + let bv = m.read_base(*bit).map_err(|e| match e { + LfmExecError::NotBaseWord(a) => LfmExecError::NonBooleanBit(a), + other => other, + })?; + let l = m.read_word(*in_l)?; + let r = m.read_word(*in_r)?; + let (ol, or) = if bv == FE::zero() { + (l, r) + } else if bv == FE::one() { + (r, l) + } else { + return Err(LfmExecError::NonBooleanBit(bit.0)); + }; + m.write(*out_l, ol)?; + m.write(*out_r, or)?; + sink.put( + &mut records.select, + SelectRow { + bit: bv, + in_l: l, + in_r: r, + out_l: ol, + out_r: or, + }, + ); + } + Instr::BitDec { + input, + bits, + halves, + } => { + let v = m.read_base(*input)?; + let canon = GoldilocksField::canonical(v.value()); + let bit_vals: [FE; 64] = core::array::from_fn(|i| FE::from((canon >> i) & 1)); + let top = (canon >> 32) as u32; + let g = 0xFFFF_FFFFu64 - top as u64; + let (z, ginv) = if g == 0 { + (FE::one(), FE::zero()) + } else { + ( + FE::zero(), + FE::from(g) + .inv() + .map_err(|_| LfmExecError::Internal("bitdec ginv"))?, + ) + }; + for (i, (addr, _)) in bits.iter().enumerate() { + m.write(*addr, base_word(bit_vals[i]))?; + } + if let Some([h0, h1]) = halves { + // Half 0 is the HIGH word: it leads in big-endian order. + let hi = (canon >> 32) as u32; + let lo = (canon & 0xFFFF_FFFF) as u32; + m.write(h0.0, base_word(FE::from(hi.swap_bytes() as u64)))?; + m.write(h1.0, base_word(FE::from(lo.swap_bytes() as u64)))?; + } + sink.put( + &mut records.bitdec, + BitDecRow { + bits: bit_vals, + z, + ginv, + }, + ); + } + Instr::Hash { + mode, ins, outs, .. + } => { + let row = hash_compute(&m.memory, hasher, *mode, ins)?; + hash_apply(&mut m.memory, *mode, outs, &row)?; + sink.put(&mut records.hash, row); + } + Instr::Hash16(h) => { + let row = hash16_compute(&m.memory, hasher, &h.ins)?; + hash16_apply(&mut m.memory, &h.outs, &row)?; + sink.put(&mut records.hash16, row); + } + Instr::KeccakF(op) => { + use super::layout::keccak as k; + // 13 words × 4 lanes → 50 u32 halves (+ 2 must-be-zero slots). + let mut halves = [0u32; k::NUM_HALVES]; + for (j, cell) in op.ins.iter().enumerate() { + let w = m.read_word(*cell)?; + for (l, lane) in w.iter().enumerate() { + let h = 4 * j + l; + let v = GoldilocksField::canonical(lane.value()); + if h >= k::NUM_HALVES { + if v != 0 { + return Err(LfmExecError::KeccakSpareLaneNonZero { + addr: cell.0, + lane: l, + }); + } + } else if v >= 1u64 << 32 { + return Err(LfmExecError::NotU32Half { + addr: cell.0, + lane: l, + }); + } else { + halves[h] = v as u32; + } + } + } + let state = super::keccak_adapter::halves_to_state(&halves); + + // Absorb: XOR the rate block into the state's first 136 bytes. + // Block byte k is byte k % 8 of lane k / 8, which is exactly + // state byte offset k — rate bytes are lane-major and + // little-endian within a lane, same as the byte columns. + let mut block = [0u8; k::RATE_BYTES]; + let mut perm_in = state; + if op.mode == KeccakMode::Absorb { + let mut bh = [0u32; k::BLOCK_HALVES]; + for (j, cell) in op.block.iter().enumerate() { + let w = m.read_word(*cell)?; + for (l, lane) in w.iter().enumerate() { + let h = 4 * j + l; + let v = GoldilocksField::canonical(lane.value()); + if h >= k::BLOCK_HALVES { + if v != 0 { + return Err(LfmExecError::KeccakSpareLaneNonZero { + addr: cell.0, + lane: l, + }); + } + } else if v >= 1u64 << 32 { + return Err(LfmExecError::NotU32Half { + addr: cell.0, + lane: l, + }); + } else { + bh[h] = v as u32; + } + } + } + for (h, half) in bh.iter().enumerate() { + block[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + for lane in 0..k::RATE_LANES { + let mut chunk = [0u8; 8]; + chunk.copy_from_slice(&block[lane * 8..lane * 8 + 8]); + perm_in[lane] ^= u64::from_le_bytes(chunk); + } + } + + let output = super::keccak_adapter::permute(perm_in); + for (cell, w) in op + .outs + .iter() + .zip(super::keccak_adapter::state_to_words(&output)) + { + m.write(*cell, w)?; + } + if let Some(rev) = &op.rev { + let words = super::keccak_adapter::reversed_digest_words(&output); + for (cell, w) in rev.outs.iter().zip(words) { + m.write(*cell, w)?; + } + } + sink.put( + &mut records.keccak, + KeccakRow { + mode: op.mode, + state, + block, + perm_in, + output, + }, + ); + } + Instr::Blake3(op) => { + use super::layout::blake3 as l; + // 7 words × 4 lanes → the 28 input `u32` words, with no spare + // slots: 28 divides by 4 exactly, so unlike `KeccakF` there is + // no must-be-zero tail lane to police. + let mut words = [0u32; l::IN_U32]; + for (j, cell) in op.ins.iter().enumerate() { + let w = m.read_word(*cell)?; + for (lane, value) in w.iter().enumerate() { + let v = GoldilocksField::canonical(value.value()); + if v >= 1u64 << 32 { + return Err(LfmExecError::NotU32Half { addr: cell.0, lane }); + } + words[4 * j + lane] = v as u32; + } + } + let values = Blake3Values { + h: core::array::from_fn(|i| words[i]), + m: core::array::from_fn(|i| words[8 + i]), + t: u64::from(words[24]) | (u64::from(words[25]) << 32), + block_len: words[26], + flags: words[27], + }; + + let out = values.output_words(); + for (j, cell) in op.outs.iter().enumerate() { + let word: LfmWord = + core::array::from_fn(|lane| FE::from(u64::from(out[4 * j + lane]))); + m.write(*cell, word)?; + } + if let Some(rev) = &op.rev { + // The 32-byte digest is `out[0..8]` little-endian; reversing + // it is reading those bytes back-to-front, which is what the + // chip's flipped-coefficient send computes on the AIR side. + let mut digest = [0u8; 32]; + for i in 0..8 { + digest[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + digest.reverse(); + for (w, cell) in rev.outs.iter().enumerate() { + let word: LfmWord = core::array::from_fn(|lane| { + let h = 4 * w + lane; + let mut b = [0u8; 4]; + b.copy_from_slice(&digest[4 * h..4 * h + 4]); + FE::from(u64::from(u32::from_le_bytes(b))) + }); + m.write(*cell, word)?; + } + } + sink.put(&mut records.blake3, values); + } + Instr::Hint { + arena, index, out, .. + } + | Instr::HintOn { + arena, index, out, .. + } => { + // A `HintOn` reads `alt` when its bit is 1: the bit was written + // before it (a schedule dependency, not a read). + let index = match instr { + Instr::HintOn { alt, bit, .. } => { + let v = m.read_base(*bit)?; + if v == FE::one() { + alt + } else if v == FE::zero() { + index + } else { + return Err(LfmExecError::NonBooleanBit(bit.0)); + } + } + _ => index, + }; + let words = + m.arenas + .get(*arena as usize) + .ok_or(LfmExecError::ArenaOutOfBounds { + arena: *arena, + index: *index, + })?; + let w = *words + .get(*index as usize) + .ok_or(LfmExecError::ArenaOutOfBounds { + arena: *arena, + index: *index, + })?; + m.write(*out, w)?; + sink.put(&mut records.hint, w); + } + Instr::Pack { lanes, out, .. } => { + let mut word = [FE::zero(), FE::zero(), FE::zero(), FE::zero()]; + for (i, lane) in lanes.iter().enumerate() { + word[i] = m.read_base(*lane)?; + } + m.write(*out, word)?; + sink.put(&mut records.lanes, word); + } + Instr::Unpack { input, outs, .. } => { + let word = m.read_word(*input)?; + for (i, out) in outs.iter().enumerate() { + m.write(*out, base_word(word[i]))?; + } + sink.put(&mut records.lanes, word); + } + Instr::Public { addr, index } => { + let w = m.read_word(*addr)?; + sink.put(&mut records.public, w); + sink.put(public_words, (*index, w)); + } + } + } + Ok(()) +} + +/// Which order [`execute_scheduled`] evaluates a program in. +/// +/// ⛔ **A schedule may not change one word of the witness.** The program is a +/// DAG and every instruction is a pure function of the cells it reads, so the +/// result is schedule-independent by construction; `exec_identity_tests` is +/// what shows the construction was implemented, and the tree-scale `IDENTITY:` +/// diff is what shows it at production shapes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Schedule { + /// The plain `for instr in &program.instrs` loop. The reference, kept in the + /// file rather than in history: every A/B, and the byte gate itself, is this + /// against the other arm in ONE binary. + Serial, + /// Level-scheduled: the mutually independent hash instructions of each + /// depth level run on rayon's global pool, everything else stays serial. + LevelParallel { + /// Levels narrower than this run on the calling thread instead of being + /// handed to rayon. + /// + /// ⓘ A parameter rather than a constant because it decides whether a + /// test can see the parallel path at all: the laptop cases are 15 wide + /// at their widest, so anything at or above 16 sends every level of + /// every case down the serial branch and the gate goes quiet. The gate + /// runs at 2; production runs at [`PRODUCTION_COALESCE_BELOW`]. + coalesce_below: usize, + }, +} + +/// Where a level stops being worth a fork/join, on the 09-12 wrap histogram: +/// 1,767 of 2,237 levels are ≤ 16 wide and hold 7,303 hashes (1.1%). Serially +/// those cost 7,303 × 2,356 ns ≈ 17 ms; on rayon they would cost ≈ 4 ms of +/// permutation plus 1,767 fork/joins, which at 2–5 µs is 3.5–8.8 ms. So the two +/// are within ~10 ms of each other on a ~450 ms budget and the serial branch +/// wins on the tie-breaker that matters: it removes 79% of this executor's +/// interactions with a pool that up to four sibling proofs are sharing. +pub const PRODUCTION_COALESCE_BELOW: usize = 16; + +/// Where `execute`'s time went, so a lever aimed at one phase can be read. +/// +/// ⚠ The phases are recorded, not derived: a number obtained by subtracting two +/// others assumes there is nothing else between them, which is how a lever gets +/// credited with a stage it never touched. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct ExecSplit { + /// Levels in the schedule; `0` under [`Schedule::Serial`]. + pub levels: usize, + /// Levels actually handed to rayon — **`0` here with `levels` large means + /// every level was coalesced and nothing ran in parallel**, which is the + /// reading that would otherwise be invisible. + pub parallel_levels: usize, + /// Hash rows inside those levels. + pub parallel_hashes: usize, + /// Building the schedule: one forward pass plus a counting sort. + pub depth_pass: f64, + /// Allocating the machine and the records. + /// + /// ⚠ Its own field, and not because it is interesting. The level walk writes + /// record SLOTS, so its vectors are filled with blanks up front — one extra + /// pass of stores over a few hundred MiB that the serial walk never makes. + /// Left out of the split, that cost would appear only as the gap between + /// `execute` and the sum of the phases below, which is the shape of a number + /// nobody can attribute. + pub setup: f64, + /// Computing the permutations, whether on the pool or coalesced. + pub hash_phase: f64, + /// Publishing each level's output cells and record rows, on this thread. + pub apply: f64, + /// Everything that is not a hash instruction. + pub residue: f64, + /// Bytes the record vectors occupy — the size of what `setup` touches. + pub record_bytes: usize, + /// Hashes run on the calling thread because their level was too narrow to + /// hand to rayon, and the seconds they took. + /// + /// ★ Together these are a per-permutation cost measured IN THIS PROCESS, on + /// THIS box, under THIS load, with no rayon in the path — so + /// `parallel_hashes × (serial_hash_secs / serial_hashes) / hash_phase` is + /// the effective width the pool actually gave, as a reading rather than an + /// argument. A wall alone cannot separate "the box is busy" from "the + /// schedule is wrong", and those want opposite fixes. + pub serial_hashes: usize, + pub serial_hash_secs: f64, +} + +impl ExecSplit { + /// Seconds per permutation, measured off the coalesced levels. + pub fn secs_per_perm(&self) -> f64 { + if self.serial_hashes == 0 { + return 0.0; + } + self.serial_hash_secs / self.serial_hashes as f64 + } + + /// The width the pool actually delivered during the hash phase. + pub fn effective_width(&self) -> f64 { + let per = self.secs_per_perm(); + if per <= 0.0 || self.hash_phase <= 0.0 { + return 0.0; + } + self.parallel_hashes as f64 * per / self.hash_phase + } +} + +/// The witness, under whichever schedule the caller names. +/// +/// `Schedule::Serial` is the reference loop, unchanged. The level schedule +/// evaluates, for each depth level in order, the level's hashes (on rayon when +/// the level is wide enough) and then the level's non-hash instructions in +/// program order — which is sound for the reason spelled out in +/// [`super::exec_schedule`] and safe for the reason spelled out on +/// [`hash_compute`]. +pub fn execute_scheduled( + program: &LfmProgram, + arenas: &[Vec], + hasher: &(impl LfmHasher + Sync), + schedule: Schedule, +) -> Result { + execute_inner(program, arenas, hasher, schedule, 1) +} + +/// The production entry point. Signature unchanged; the schedule comes from the +/// environment so every existing caller keeps working and the box arms select an +/// arm per process. +pub fn execute( + program: &LfmProgram, + arenas: &[Vec], + hasher: &(impl LfmHasher + Sync), +) -> Result { + execute_inner(program, arenas, hasher, default_schedule(), 1) +} + +/// `LFM_EXEC_PARALLEL=0` selects the serial reference; unset or `1` selects the +/// level schedule. +/// +/// ⚠ Cached, so one process sees one value — which is why it is NOT how the +/// identity gate reaches the two arms (a test flipping this would gate whichever +/// arm ran first). The gate calls [`execute_scheduled`]; this exists so a box arm +/// can pick a schedule without a rebuild. +pub fn default_schedule() -> Schedule { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init( + // An EMPTY value reads as unset: `FOO= cmd` is the shell's way of + // clearing a variable, and panicking on it would fail a run for a + // spelling of "default". + || match std::env::var("LFM_EXEC_PARALLEL").ok().as_deref() { + Some("0") => Schedule::Serial, + None | Some("") | Some("1") => Schedule::LevelParallel { + coalesce_below: PRODUCTION_COALESCE_BELOW, + }, + Some(other) => panic!("LFM_EXEC_PARALLEL must be `0` or `1`, got `{other}`"), + }, + ) +} + +/// ⛔ Test-only: run the level schedule over a DELIBERATELY MERGED set of +/// levels, so a hash and one of its own inputs share a level. +/// +/// The executor must then fail with [`LfmExecError::ReadBeforeWrite`], at the +/// same address, on every run — see [`super::exec_schedule::build_merged`]. A +/// gate that still passed with the level boundary removed would not be a gate. +/// [`execute`] on the serial schedule, whatever `LFM_EXEC_PARALLEL` says — the +/// reference the level schedule is compared against. +#[cfg(test)] +pub fn execute_serial( + program: &LfmProgram, + arenas: &[Vec], + hasher: &(impl LfmHasher + Sync), +) -> Result { + execute_inner(program, arenas, hasher, Schedule::Serial, 1) +} + +#[cfg(test)] +pub fn execute_with_merged_levels( + program: &LfmProgram, + arenas: &[Vec], + hasher: &(impl LfmHasher + Sync), + coalesce_below: usize, + merge: u32, +) -> Result { + execute_inner( + program, + arenas, + hasher, + Schedule::LevelParallel { coalesce_below }, + merge, + ) +} + +fn execute_inner( + program: &LfmProgram, + arenas: &[Vec], + hasher: &(impl LfmHasher + Sync), + schedule: Schedule, + merge: u32, +) -> Result { + let schema = &program.arena_schema.lens; + if arenas.len() != schema.len() { + return Err(LfmExecError::ArenaCountMismatch { + expected: schema.len(), + found: arenas.len(), + }); + } + for (i, (arena, &len)) in arenas.iter().zip(schema).enumerate() { + if arena.len() != len as usize { + return Err(LfmExecError::ArenaLenMismatch { + arena: i as u32, + expected: len, + found: arena.len(), + }); + } + } + + // ⚠ Built BEFORE the machine, and its two big scratch arrays are freed + // inside, so the pass's own peak (~66–80 MiB on a wrap) never stacks on top + // of the memory array's 188–275 MiB. What survives into the run is the index + // vectors alone. + let t = Instant::now(); + let levels = match schedule { + Schedule::Serial => None, + Schedule::LevelParallel { .. } => Some(exec_schedule::build_with_merge(program, merge)), + }; + let mut split = ExecSplit { + depth_pass: t.elapsed().as_secs_f64(), + levels: levels.as_ref().map_or(0, |l| l.levels()), + ..ExecSplit::default() + }; + + // The serial walk IS program order, so it appends into empty vectors exactly + // as it always did; the level walk is not, so it writes slots that already + // exist. Same bytes reserved either way. + // ⛔ ONE allocation path for both walks, and the records are NOT pre-filled. + // They used to be, so the level walk could assign into a live element — and + // that cost a full pass of stores over half a gigabyte of rows the walk was + // about to overwrite anyway, plus first-touching every one of those pages + // here instead of where the work is. The slots are written once, into spare + // capacity, and `commit_slots` publishes them. + let t = Instant::now(); + let mut m = Machine { + memory: WriteOnceMemory::new(program.num_addrs as usize), + arenas: arenas.iter().map(Vec::as_slice).collect(), + }; + let mut records = LfmRecords::with_capacity(&program.groups, program.hash16()); + let mut public_words = Vec::with_capacity(program.groups.public.real_rows); + split.setup = t.elapsed().as_secs_f64(); + split.record_bytes = LfmRecords::bytes(&program.groups, program.hash16()); + + match &levels { + None => { + let t = Instant::now(); + for instr in &program.instrs { + step( + &mut m, + &mut records, + &mut public_words, + instr, + hasher, + Sink::Push, + )?; + } + split.residue = t.elapsed().as_secs_f64(); + } + Some(levels) => { + let coalesce_below = match schedule { + Schedule::LevelParallel { coalesce_below } => coalesce_below, + Schedule::Serial => unreachable!("a schedule was built, so it is not Serial"), + }; + run_levels( + program, + levels, + hasher, + coalesce_below, + &mut m, + &mut records, + &mut public_words, + &mut split, + None, + )?; + public_rows_fit(&public_words, program.groups.public.real_rows)?; + // SAFETY: `run_levels` returned `Ok`, so every instruction ran and + // every row the schedule handed out was written — see + // `LfmRecords::commit_slots`. This is the one call site, and it is + // after the `?` so a failed walk never reaches it; the public + // words' capacity covers the rows set (checked above). + unsafe { + records.commit_slots(&program.groups, program.hash16()); + public_words.set_len(program.groups.public.real_rows); + } + } + } + + // The sizing above is exact, not an estimate: assert it on the success path + // so a chip whose emitter and executor arm drift apart says so here, where + // the two are written, instead of as an index panic inside the parallel + // fill. Chip order: const, balu, xalu, select, bitdec, hash, keccak, + // blake3, lanes, hint, public. (A `?` above returns early with short + // vectors by design — that program did not finish.) + // + // ⓘ This is a real check on the SERIAL walk, where the lengths are push + // counts. The level walk sizes its vectors up front, so on that path the + // nine row counts are true by construction and `num_consts` is the only one + // still saying anything; the check that replaces them there is + // `ChipRows::assert_matches_census`, which compares the rows the SCHEDULE + // hands out against the same census before a single instruction runs. + debug_assert_eq!( + [ + records.num_consts, + records.balu.len(), + records.xalu.len(), + records.select.len(), + records.bitdec.len(), + records.hash.len() + records.hash16.len(), + records.keccak.len(), + records.blake3.len(), + records.lanes.len(), + records.hint.len(), + records.public.len(), + ], + [ + program.groups.const_.real_rows, + program.groups.balu.real_rows, + program.groups.xalu.real_rows, + program.groups.select.real_rows, + program.groups.bitdec.real_rows, + program.groups.hash.real_rows, + program.groups.keccak.real_rows, + program.groups.blake3.real_rows, + program.groups.lanes.real_rows, + program.groups.hint.real_rows, + program.groups.public.real_rows, + ], + "record counts must equal the emitted column-group row counts" + ); + + Ok(LfmExecution { + records, + public_words, + memory: m.memory, + split, + }) +} + +/// Whether the public words' buffer can hold the program's `rows` before an +/// execution sets its length to them: a shortfall is refused rather than +/// asserted (it would be a sizing bug, never a property of the input). +pub(super) fn public_rows_fit(words: &Vec, rows: usize) -> Result<(), LfmExecError> { + match words.capacity() >= rows { + true => Ok(()), + false => Err(LfmExecError::Internal( + "the public words hold fewer rows than the program's", + )), + } +} + +/// An execution that runs as its arenas arrive, a group at a time: the +/// instructions that read only landed groups run as soon as the last of those +/// groups lands, the rest wait. A tree node's children land at different times, +/// and most of a node is the verification of one child, so most of it can run +/// before its last child is proved. +/// +/// The witness is [`execute`]'s, word for word (the gate: +/// `exec_identity_tests`): each instruction runs exactly once, in the level +/// schedule's order restricted to a wave, writes the record slot the schedule +/// gives it, and reads memory that write-once [`WriteOnceMemory`] guards — so a +/// wrong `need` (an instruction run before an input exists) is +/// `ReadBeforeWrite` or `ArenaOutOfBounds`, never a different value. +/// +/// `need[i]` is the set of groups instruction `i` reads, through any chain of +/// memory: a [`Instr::Hint`] reads its arena's group, every instruction the +/// union of its inputs' sets (one forward pass: every operand address is below +/// its destination). Instruction `i` runs in the wave in which its last group +/// lands; the ones that read no arena run before any lands. +pub struct StreamedExecution<'p, 'a, H: LfmHasher + Sync> { + program: &'p LfmProgram, + hasher: &'p H, + levels: LevelSchedule, + need: Vec, + /// Each arena's group. + group_of: Vec, + landed: u64, + ran: usize, + m: Machine<'a>, + records: LfmRecords, + public_words: Vec<(u32, LfmWord)>, + split: ExecSplit, +} + +impl<'p, 'a, H: LfmHasher + Sync> StreamedExecution<'p, 'a, H> { + /// A streamed execution of `program`, its arenas in groups (`group_of[a]` is + /// arena `a`'s group, at most 64 groups); runs everything that reads no + /// arena now. + pub fn new( + program: &'p LfmProgram, + group_of: Vec, + hasher: &'p H, + ) -> Result { + let schema = &program.arena_schema.lens; + if group_of.len() != schema.len() { + return Err(LfmExecError::ArenaCountMismatch { + expected: schema.len(), + found: group_of.len(), + }); + } + if group_of.iter().any(|&g| g >= 64) { + return Err(LfmExecError::Internal( + "a streamed execution takes at most 64 groups", + )); + } + let t = Instant::now(); + let levels = exec_schedule::build_with_merge(program, 1); + // The forward pass: each address's group set, then each instruction's. + let mut addr_need = vec![0u64; program.num_addrs as usize]; + let mut need = vec![0u64; program.instrs.len()]; + let mut scratch: Vec = Vec::with_capacity(32); + for (i, instr) in program.instrs.iter().enumerate() { + scratch.clear(); + instr.schedule_reads_into(&mut scratch); + let mut n = scratch + .iter() + .map(|r| addr_need.get(r.0 as usize).copied().unwrap_or(0)) + .fold(0u64, |a, b| a | b); + if let Instr::Hint { arena, .. } | Instr::HintOn { arena, .. } = instr { + // An arena past the schema reads as no group: the executor + // rejects the read itself, as it does unstreamed. + n |= group_of.get(*arena as usize).map_or(0, |&g| 1u64 << g); + } + need[i] = n; + scratch.clear(); + instr.writes_into(&mut scratch); + for w in &scratch { + if let Some(slot) = addr_need.get_mut(w.0 as usize) { + *slot = n; + } + } + } + drop(addr_need); + let mut split = ExecSplit { + depth_pass: t.elapsed().as_secs_f64(), + levels: levels.levels(), + ..ExecSplit::default() + }; + let t = Instant::now(); + let m = Machine { + memory: WriteOnceMemory::new(program.num_addrs as usize), + arenas: vec![&[]; schema.len()], + }; + let records = LfmRecords::with_capacity(&program.groups, program.hash16()); + let public_words = Vec::with_capacity(program.groups.public.real_rows); + split.setup = t.elapsed().as_secs_f64(); + split.record_bytes = LfmRecords::bytes(&program.groups, program.hash16()); + let mut streamed = Self { + program, + hasher, + levels, + need, + group_of, + landed: 0, + ran: 0, + m, + records, + public_words, + split, + }; + streamed.wave(None)?; + Ok(streamed) + } + + /// Group `group` has landed with `arenas` (its arenas, in arena order): runs + /// every instruction whose last group it is. + pub fn land(&mut self, group: usize, arenas: &'a [Vec]) -> Result<(), LfmExecError> { + if group >= 64 || self.landed & (1u64 << group) != 0 { + return Err(LfmExecError::Internal( + "a group landed twice, or is out of range", + )); + } + let ids: Vec = (0..self.group_of.len()) + .filter(|&a| self.group_of[a] == group) + .collect(); + if ids.len() != arenas.len() { + return Err(LfmExecError::ArenaCountMismatch { + expected: ids.len(), + found: arenas.len(), + }); + } + let schema = &self.program.arena_schema.lens; + for (&a, words) in ids.iter().zip(arenas) { + if words.len() != schema[a] as usize { + return Err(LfmExecError::ArenaLenMismatch { + arena: a as u32, + expected: schema[a], + found: words.len(), + }); + } + self.m.arenas[a] = words; + } + let before = self.landed; + self.landed |= 1u64 << group; + self.wave(Some(before)) + } + + /// Runs the instructions whose groups have all landed and had not all + /// landed `before` (every instruction that reads no arena, at `None`). + fn wave(&mut self, before: Option) -> Result<(), LfmExecError> { + let (need, landed) = (&self.need, self.landed); + let keep = |i: usize| { + let n = need[i]; + n & !landed == 0 && before.is_none_or(|b| n & !b != 0) + }; + self.ran += run_levels( + self.program, + &self.levels, + self.hasher, + PRODUCTION_COALESCE_BELOW, + &mut self.m, + &mut self.records, + &mut self.public_words, + &mut self.split, + Some(&keep), + )?; + Ok(()) + } + + /// Instructions run so far. + pub fn ran(&self) -> usize { + self.ran + } + + /// The execution, once every group has landed: [`execute`]'s, word for word. + pub fn finish(mut self) -> Result { + let groups = self.group_of.iter().fold(0u64, |a, &g| a | (1u64 << g)); + if self.landed != groups { + return Err(LfmExecError::Internal( + "a streamed execution finished before every group landed", + )); + } + if self.ran != self.program.instrs.len() { + return Err(LfmExecError::Internal( + "a streamed execution ran a different number of instructions than the program has", + )); + } + public_rows_fit(&self.public_words, self.program.groups.public.real_rows)?; + // SAFETY: every instruction ran exactly once (each in the one wave in + // which its last group landed; `ran` counts them) and every wave + // returned `Ok`, so every record slot the schedule hands out was + // written — `LfmRecords::commit_slots`'s condition, as in `execute`; + // and the public words' capacity covers the rows set (checked above). + unsafe { + self.records + .commit_slots(&self.program.groups, self.program.hash16()); + self.public_words + .set_len(self.program.groups.public.real_rows); + } + Ok(LfmExecution { + records: self.records, + public_words: self.public_words, + memory: self.m.memory, + split: self.split, + }) + } +} + +/// The level schedule: for each depth level, its hashes, then its non-hash work. +/// +/// # The protocol, and why it needs no lock, no atomic and no `unsafe` +/// +/// A level's hashes are mutually independent, so they may be computed in any +/// order. They are computed and NOT applied: the workers see `&WriteOnceMemory` +/// and hand back rows, and this thread then publishes every output cell and +/// every record slot before the next level starts. Three things fall out of +/// that, and each of them is a bug that did not have to be prevented: +/// +/// * **No data race is expressible.** No `&mut` to the memory exists while the +/// level runs, so the borrow checker rejects a worker that tried to write. +/// * **A wrong schedule is a deterministic error, not a race.** A hash launched +/// before its input is applied reads an unwritten cell and gets +/// [`LfmExecError::ReadBeforeWrite`] — the same error, at the same address, on +/// every run. That is what the merged-level mutation gate rides on. +/// * **`DoubleWrite` and the occupancy bitset are untouched.** Both still go +/// through the one `WriteOnceMemory::write` on one thread. +/// +/// # Order +/// +/// Rows are applied in ascending row number, which inside a level is program +/// order, so `records.hash` ends up exactly the vector the serial loop would +/// have pushed. The row numbers come from the schedule, so a row is written once +/// and read once and the vector is never searched. +/// +/// `keep`, when given, runs only the instructions it keeps — a streamed +/// execution's wave ([`StreamedExecution`]) — in the same order; returns how many +/// instructions ran. +#[allow(clippy::too_many_arguments)] +fn run_levels( + program: &LfmProgram, + levels: &LevelSchedule, + hasher: &(impl LfmHasher + Sync), + coalesce_below: usize, + m: &mut Machine<'_>, + records: &mut LfmRecords, + public_words: &mut Vec<(u32, LfmWord)>, + split: &mut ExecSplit, + keep: Option<&dyn Fn(usize) -> bool>, +) -> Result { + // One buffer, reused by every level, so a 2,237-level program allocates once + // rather than 2,237 times. It grows to the widest level and stays there: + // ~3.4 MiB on a wrap, ~4.8 MiB on a node. + let mut computed: Vec> = Vec::new(); + // A wave's share of a level, when `keep` filters: the same order, a subset. + let (mut kept_rows, mut kept_others): (Vec, Vec) = (Vec::new(), Vec::new()); + let mut ran = 0usize; + + for d in 0..levels.levels() { + let rows = match keep { + None => levels.hashes_at(d), + Some(keep) => { + kept_rows.clear(); + kept_rows.extend( + levels + .hashes_at(d) + .iter() + .filter(|&&row| keep(levels.instr_of_row(row))), + ); + &kept_rows + } + }; + ran += rows.len(); + + let t = Instant::now(); + if rows.len() < coalesce_below { + // Narrow: a fork/join for a handful of permutations costs more than + // it saves, and on a box up to four sibling proofs are sharing this + // pool. 79% of a wrap's levels come through here holding 1.1% of its + // hashes. + computed.clear(); + for &row in rows { + let instr = &program.instrs[levels.instr_of_row(row)]; + computed.push(hash_one(&m.memory, hasher, instr)); + } + // ★ The same permutation, on this thread, with no pool in the path — + // so the ratio of this to the parallel phase is the width the pool + // actually gave, measured in the same process on the same box under + // the same load. Free: this work runs either way, and it is 1.1% of + // the hashes. + split.serial_hashes += rows.len(); + split.serial_hash_secs += t.elapsed().as_secs_f64(); + } else { + compute_level(&m.memory, hasher, program, levels, rows, &mut computed); + split.parallel_levels += 1; + split.parallel_hashes += rows.len(); + } + split.hash_phase += t.elapsed().as_secs_f64(); + + // ⓘ The FIRST failure by row number, not whichever worker finished + // first: the same program must report the same address on every run, or + // the mutation gate below it is flaky and the campaign's error messages + // stop being comparable between arms. + let t = Instant::now(); + for (&row, out) in rows.iter().zip(&computed) { + let row_value = match out { + Ok(r) => r, + Err(e) => return Err(e.clone()), + }; + let instr = &program.instrs[levels.instr_of_row(row)]; + // The hash chip's row goes through the same slot write as every + // other chip's: the vector's length is zero until `commit_slots`. + match (instr, row_value) { + (Instr::Hash { mode, outs, .. }, HashOut::Narrow(r)) => { + hash_apply(&mut m.memory, *mode, outs, r)?; + Sink::Slot(row).put(&mut records.hash, r.clone()); + } + (Instr::Hash16(h), HashOut::Wide(r)) => { + hash16_apply(&mut m.memory, &h.outs, r)?; + Sink::Slot(row).put(&mut records.hash16, (**r).clone()); + } + _ => { + return Err(LfmExecError::Internal( + "the schedule listed a non-hash instruction as a hash row", + )); + } + } + } + split.apply += t.elapsed().as_secs_f64(); + + // The level's non-hash work. It may read this level's hashes and each + // other, so it stays serial and in program order — which is a + // topological order, so that is enough. + let t = Instant::now(); + let others = match keep { + None => levels.others_at(d), + Some(keep) => { + kept_others.clear(); + kept_others.extend(levels.others_at(d).iter().filter(|&&i| keep(i as usize))); + &kept_others + } + }; + ran += others.len(); + for &i in others { + let i = i as usize; + step( + m, + records, + public_words, + &program.instrs[i], + hasher, + Sink::Slot(levels.record_row(i)), + )?; + } + split.residue += t.elapsed().as_secs_f64(); + } + Ok(ran) +} + +/// A level's computed hash row: a twelve-felt one, or a width-16 one (boxed, +/// so the twelve-felt level buffer keeps its size). +enum HashOut { + Narrow(HashRow), + Wide(Box), +} + +/// One hash instruction's read-only half, dispatched from a program index. +fn hash_one( + memory: &WriteOnceMemory, + hasher: &impl LfmHasher, + instr: &Instr, +) -> Result { + match instr { + Instr::Hash { mode, ins, .. } => { + hash_compute(memory, hasher, *mode, ins).map(HashOut::Narrow) + } + Instr::Hash16(h) => { + hash16_compute(memory, hasher, &h.ins).map(|r| HashOut::Wide(Box::new(r))) + } + _ => Err(LfmExecError::Internal( + "the schedule listed a non-hash instruction as a hash row", + )), + } +} + +/// A level's permutations, on rayon's GLOBAL pool. +/// +/// ⚠ The global pool on purpose, and never a private one. With four sibling +/// proofs live at level 0 a private pool per executor would be 120 threads on 30 +/// cores; the global pool shares itself between them instead, which is the +/// oversubscription finding turned into a default rather than a knob. +#[cfg(feature = "parallel")] +fn compute_level( + memory: &WriteOnceMemory, + hasher: &(impl LfmHasher + Sync), + program: &LfmProgram, + levels: &LevelSchedule, + rows: &[u32], + out: &mut Vec>, +) { + use rayon::prelude::*; + + rows.par_iter() + .map(|&row| hash_one(memory, hasher, &program.instrs[levels.instr_of_row(row)])) + .collect_into_vec(out); +} + +/// [`compute_level`] without the feature: the same bodies in order. This is the +/// schedule, not the reference — a `--no-default-features` build still has to +/// produce the same witness, and it does, because the level order alone is what +/// makes the result right. +#[cfg(not(feature = "parallel"))] +fn compute_level( + memory: &WriteOnceMemory, + hasher: &(impl LfmHasher + Sync), + program: &LfmProgram, + levels: &LevelSchedule, + rows: &[u32], + out: &mut Vec>, +) { + out.clear(); + out.extend( + rows.iter() + .map(|&row| hash_one(memory, hasher, &program.instrs[levels.instr_of_row(row)])), + ); +} + +/// ★ Which instruction WROTE `addr`, with a window of its neighbours — the map +/// from a [`LfmExecError::DivByZero`] address back to the assert that failed. +/// +/// ⚠ **This is more generally useful than it looks.** `assert_eq` lowers to +/// `diff = a − b; _ = diff / ZERO` (`builder.rs`), and the executor reports the +/// NUMERATOR's address. So a `DivByZero` is never an inversion gone wrong — it +/// is always a failing equality assert, and the address always names the `diff` +/// cell. Given the address, the `Sub` that produced it and the two operands +/// feeding that `Sub` identify the assert exactly, without bisecting the +/// emitter. +/// +/// Returns a human-readable report; for diagnostics, not for proving. +pub fn locate_addr(program: &LfmProgram, addr: u64) -> String { + use core::fmt::Write as _; + + let writes = |i: &Instr| -> Vec { + match i { + Instr::Const { out, .. } => vec![out.0], + Instr::BaseAlu { out, .. } | Instr::ExtAlu { out, .. } => vec![out.0], + Instr::Select { out_l, out_r, .. } => vec![out_l.0, out_r.0], + Instr::BitDec { bits, halves, .. } => bits + .iter() + .map(|(a, _)| a.0) + .chain(halves.iter().flatten().map(|(a, _)| a.0)) + .collect(), + Instr::Hash { outs, .. } => outs.iter().map(|a| a.0).collect(), + Instr::Hash16(h) => h.outs.iter().map(|a| a.0).collect(), + Instr::Unpack { outs, .. } => outs.iter().map(|a| a.0).collect(), + Instr::Pack { out, .. } => vec![out.0], + _ => Vec::new(), + } + }; + + let mut out = String::new(); + let Some(idx) = program + .instrs + .iter() + .position(|i| writes(i).contains(&addr)) + else { + let _ = writeln!( + out, + "addr {addr}: no instruction writes it (an arena word?)" + ); + return out; + }; + let _ = writeln!( + out, + "addr {addr} written by instruction {idx} of {}:", + program.instrs.len() + ); + let lo = idx.saturating_sub(4); + let hi = (idx + 3).min(program.instrs.len()); + for (k, i) in program.instrs[lo..hi].iter().enumerate() { + let marker = if lo + k == idx { "→" } else { " " }; + let _ = writeln!(out, " {marker} [{}] {i:?}", lo + k); + } + out +} diff --git a/prover/src/lfm/first_prove_warm.rs b/prover/src/lfm/first_prove_warm.rs new file mode 100644 index 000000000..119f7eb9b --- /dev/null +++ b/prover/src/lfm/first_prove_warm.rs @@ -0,0 +1,280 @@ +//! The recursion tree's first prove warmed up beside the base +//! (`LFM_WARM_FIRST_PROVE`; on by default, `=0` turns it off). +//! +//! The tree's first prove pays three first-use costs on the tree's critical +//! path, with the card idle or waiting: +//! +//! - its per-domain prove constants (`stark::prover::warm_prove_constants`: +//! LDE-size boundary-zerofier inversions and their card copies), built in its +//! host preparation after the leaves' artifacts are published; +//! - the DEEP's whole LDE coset, uploaded from pageable memory inside its card +//! window (`stark::prover::warm_deep_coset_points`; 192 MiB at a doubled leaf); +//! - the staging pairs' pinned allocations, made inside the first artifact +//! build's and the first prove's card windows. +//! +//! On, the early leaf's thread makes the staging pairs when it starts and, +//! after its fill, warms the first two for its own tables from an AIR set built +//! from shape-only artifacts (`registry::census_artifacts_merged`: every height, +//! chunk and merge the real artifacts have, every root zero). Every prove also +//! warms the DEEP cosets before it takes the card, as it warms its constants. +//! A warmed value is the one the prove caches for itself, keyed by the domain, +//! never by a root, so no proof byte moves. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; + +use stark::proof::options::{OneRowMode, ProofOptions}; + +use super::compiler::LfmProgram; +use super::hash::HasherKind; +use super::trace::LfmTraces; + +/// The staging pairs made ahead: what the tree made on first use at both block +/// sizes, as measured (`W3 STAGING AT END: staging pairs 4 of 8`). +pub const STAGING_PAIRS: usize = 4; + +/// `LFM_WARM_FIRST_PROVE` as given: on when unset, `1` or `on`; off when +/// empty, `0` or `off`; any other value is an error. +fn setting(value: Option<&str>) -> bool { + match value.map(str::trim) { + Some("" | "0" | "off") => false, + None | Some("1" | "on") => true, + Some(other) => { + panic!("LFM_WARM_FIRST_PROVE must be `0`, `1`, `off` or `on`, got `{other}`") + } + } +} + +/// Runtime override of [`on`]: 0 = the environment's choice, 1 = off, 2 = on. +static OVERRIDE: AtomicU8 = AtomicU8::new(0); + +/// Force the warm-up on, off, or back to the environment (tests). +pub fn set(on: Option) { + let v = match on { + None => 0, + Some(false) => 1, + Some(true) => 2, + }; + OVERRIDE.store(v, Ordering::Relaxed); +} + +/// Whether the warm-up is on (the environment read once). +pub fn on() -> bool { + match OVERRIDE.load(Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + static ENV: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENV.get_or_init(|| setting(std::env::var("LFM_WARM_FIRST_PROVE").ok().as_deref())) +} + +static PAIRS_MADE: AtomicU64 = AtomicU64::new(0); +static PAIRS_NS: AtomicU64 = AtomicU64::new(0); +static EARLY_NS: AtomicU64 = AtomicU64::new(0); +static EARLY_TABLES: AtomicU64 = AtomicU64::new(0); +static EARLY_COSETS: AtomicU64 = AtomicU64::new(0); +/// When the early leaf's warm-up ended, in seconds on the tree's clock (the +/// bits of an `f64`; `u64::MAX` for never). +static EARLY_AT: AtomicU64 = AtomicU64::new(u64::MAX); +/// The host preparation of the first proves (AIR set and pre-hold warm-up), +/// seconds, in prove order; both arms record it. +static PREPS: Mutex> = Mutex::new(Vec::new()); +const PREPS_KEPT: usize = 16; + +/// Makes the staging pairs the tree's transfers would make on first use +/// ([`STAGING_PAIRS`]), and counts them. A no-op without a card. +// The tree's early leaf (`whir_block_tree`, a `parallel` build) is the caller. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn prewarm_staging() { + let t = std::time::Instant::now(); + let made = stark::prover::prewarm_staging_pairs(STAGING_PAIRS); + PAIRS_MADE.fetch_add(made as u64, Ordering::Relaxed); + PAIRS_NS.fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed); +} + +/// Warms what `program`'s first prove would build on first use, for its +/// filled `traces` under `options`: the AIR set from shape-only artifacts, +/// then the per-domain prove constants and the DEEP's LDE cosets for its +/// tables (what [`super::proof::warm_before_hold`] warms, whichever of its +/// knobs are on). Returns how many tables it paired. +/// Nothing under a one-row format (its AIRs need the real one-row roots). +/// Call it from a plain thread, not a rayon worker. +// The tree's early leaf (`whir_block_tree`, a `parallel` build) is the caller. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn warm( + program: &LfmProgram, + traces: &mut LfmTraces, + options: &ProofOptions, + hasher: HasherKind, +) -> Result { + if options.format.one_row != OneRowMode::Off { + return Ok(0); + } + let t = std::time::Instant::now(); + let cosets_before = stark::prover::warmed_deep_cosets(); + let shape = super::registry::census_artifacts_merged( + program, + hasher, + options.format.merge_equal_height_tables, + ); + traces.merge_to(&shape.merge.plan())?; + let airs = super::proof::lfm_airs(&shape, options, hasher).map_err(|e| format!("{e:?}"))?; + let pairs = airs.air_trace_pairs(traces); + let tables: Vec<_> = pairs + .iter() + .map(|(air, trace, pi)| (*air, trace.num_rows(), *pi)) + .collect(); + stark::prover::warm_prove_constants(&tables); + stark::prover::warm_deep_coset_points(&tables); + let tables = tables.len(); + EARLY_TABLES.fetch_add(tables as u64, Ordering::Relaxed); + EARLY_COSETS.fetch_add( + stark::prover::warmed_deep_cosets() - cosets_before, + Ordering::Relaxed, + ); + EARLY_NS.fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed); + Ok(tables) +} + +/// The early leaf's warm-up ended `at` seconds into the tree's clock. +// The tree's early leaf (`whir_block_tree`, a `parallel` build) is the caller. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn note_early(at: f64) { + EARLY_AT.store(at.to_bits(), Ordering::Relaxed); +} + +/// A prove's host preparation before it takes the card, in seconds. +pub(crate) fn note_prep(secs: f64) { + let mut preps = PREPS.lock().unwrap_or_else(|e| e.into_inner()); + if preps.len() < PREPS_KEPT { + preps.push(secs); + } +} + +/// What the warm-up did this process, for a harness's line. The proves' +/// preparation times are printed in both arms. +pub fn line() -> String { + let preps = PREPS.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let preps = preps + .iter() + .map(|s| format!("{s:.3}")) + .collect::>() + .join(" "); + if !on() { + return format!("off · prove prep s [{preps}]"); + } + let early = EARLY_AT.load(Ordering::Relaxed); + format!( + "on · staging pairs made {} in {:.3} s · early leaf warmed {} tables, {} DEEP cosets in {:.3} s, by {} · prove prep s [{preps}]", + PAIRS_MADE.load(Ordering::Relaxed), + PAIRS_NS.load(Ordering::Relaxed) as f64 / 1e9, + EARLY_TABLES.load(Ordering::Relaxed), + EARLY_COSETS.load(Ordering::Relaxed), + EARLY_NS.load(Ordering::Relaxed) as f64 / 1e9, + if early == u64::MAX { + "never".to_string() + } else { + format!("{:.2}s", f64::from_bits(early)) + }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// On by default: unset, `1` or `on`; off for empty, `0` or `off`. + #[test] + fn the_knob_is_on_unless_zero_or_off() { + for off in [Some(""), Some("0"), Some("off"), Some(" 0 ")] { + assert!(!setting(off), "{off:?}"); + } + for on in [None, Some("1"), Some("on"), Some(" 1 ")] { + assert!(setting(on), "{on:?}"); + } + } + + #[test] + #[should_panic(expected = "LFM_WARM_FIRST_PROVE must be")] + fn an_unknown_value_is_refused() { + setting(Some("yes")); + } + + /// The shape-only artifacts the warm-up builds its AIR set from carry every + /// height, chunk list, chip set and merge a real build has; only roots and + /// the id differ. So the warmed tables are the prove's tables, row for row. + #[test] + fn the_census_artifacts_have_the_built_shape() { + let opts = super::super::proof::aggregation_wrap_options(); + for program in [ + super::super::programs::trivial_program(), + super::super::programs::fri_toy_program(), + ] { + let built = super::super::registry::build_artifacts(&program, &opts); + let shape = super::super::registry::census_artifacts_merged( + &program, + built.hasher, + opts.format.merge_equal_height_tables, + ); + assert_eq!(shape.log_heights, built.log_heights); + assert_eq!(shape.keccak_rnd_chunks, built.keccak_rnd_chunks); + assert_eq!( + shape.blake3_chunk_log_heights, + built.blake3_chunk_log_heights + ); + assert_eq!(shape.hash_chunk_log_heights, built.hash_chunk_log_heights); + assert_eq!(shape.chip_set, built.chip_set); + assert_eq!(shape.hasher, built.hasher); + assert_eq!(shape.merge.plan(), built.merge.plan()); + let heights = |m: &super::super::merge::LfmMerge| { + m.tables.iter().map(|t| t.log_height).collect::>() + }; + assert_eq!(heights(&shape.merge), heights(&built.merge)); + } + } + + /// ★ The warm-up leaves nothing for the prove's own pre-hold warm-up to + /// build: before it the real AIR set's tables (built from the real + /// artifacts) are cold, after it they find every boundary-zerofier + /// inverse in the cache. + #[test] + fn the_early_warm_leaves_the_prove_nothing_to_build() { + use crate::tables::types::FE; + let program = super::super::programs::trivial_program(); + // The recursion's options (its merge, which the fill follows) under a + // coset offset no other test proves under: the process-wide caches + // hold none of these domains yet. + let mut opts = super::super::proof::aggregation_wrap_options(); + opts.coset_offset = 19; + let arenas = vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(2_000 * (i + 1) + j as u64))) + .collect(), + ]; + let artifacts = super::super::registry::build_artifacts(&program, &opts); + let mut filled = + super::super::proof::lfm_execute_and_fill(&program, &arenas, artifacts.hasher) + .expect("filled"); + let real = super::super::proof::lfm_airs(&artifacts, &opts, artifacts.hasher) + .expect("the AIR set"); + let real_warm = |filled: &mut super::super::proof::LfmFilled| { + let pairs = real.air_trace_pairs(filled.traces_mut()); + let tables: Vec<_> = pairs + .iter() + .map(|(air, trace, pi)| (*air, trace.num_rows(), *pi)) + .collect(); + stark::prover::prove_constants_warm(&tables) + }; + assert!( + !real_warm(&mut filled), + "cold before the warm-up: these domains were in the cache already" + ); + let tables = warm(&program, filled.traces_mut(), &opts, artifacts.hasher).expect("warmed"); + assert!(tables > 0, "a program has tables"); + assert!( + real_warm(&mut filled), + "the real AIR set's tables still have an inverse to build after the warm-up" + ); + } +} diff --git a/prover/src/lfm/fixture.rs b/prover/src/lfm/fixture.rs new file mode 100644 index 000000000..00bf5e167 --- /dev/null +++ b/prover/src/lfm/fixture.rs @@ -0,0 +1,384 @@ +//! The Milestone-C inner-proof fixture: a host-side FRI commitment-opening +//! prover the machine verifies. +//! +//! Structurally real, deliberately small: coset LDE domains (offset 3, the +//! production pin), row-pair Merkle leaves, per-layer commitments, the +//! **unnormalized fold** convention (`(lo+hi) + inv_x·ζ·(lo−hi)`), a compress-chain +//! transcript over the machine's own hash, query indices sampled at a +//! power-of-two bound, and a terminal polynomial checked at the queried +//! points. What it is NOT: the production 25-AIR proof format — that lands +//! when the ecosystem hash decision unblocks the real machine-facing +//! pipeline (`crypto/stark` hardcodes keccak at its Merkle layer; the +//! measured 26-site migration seam is deliberately not touched here). +//! +//! Everything here mirrors `edsl.rs` bit-exactly; the emitted verifier +//! program (`programs::fri_toy_program`) consumes exactly the arena layout +//! `fixture_prove` produces. + +use math::field::traits::{IsFFTField, IsPrimeField}; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::edsl::SQUEEZE_MARK; +use super::hash::{HasherKind, LfmHasher}; +use super::word::{LfmWord, base_word, ext_word}; + +/// The fixed shape — compile-time constants of the emitted program. +pub mod shape { + /// log2 of the LDE domain size. + pub const LOG_LDE: usize = 5; // 32 points + pub const LDE_SIZE: usize = 1 << LOG_LDE; + /// Trace length 8 = LDE/blowup (blowup 4). + pub const TRACE_LEN: usize = 8; + /// Committed base columns (one machine word per row). + pub const NUM_COLS: usize = 4; + /// Two folds: 32 → 16 → 8, terminal degree < 2. + pub const NUM_LAYERS: usize = 2; + pub const TERMINAL_LEN: usize = 2; + pub const NUM_QUERIES: usize = 4; + /// Query index bits (indices sampled in [0, LDE/2)). + pub const QUERY_BITS: usize = LOG_LDE - 1; + /// The production coset offset. + pub const COSET_OFFSET: u64 = 3; + /// Words per query in the openings arena. + pub const WORDS_PER_QUERY: usize = 17; +} + +/// Host mirror of [`super::edsl::SpongeVar`] — the compress chain, state 1 +/// cell. +/// +/// Bit-exact by construction, not by coincidence: every operation here is the +/// same sequence of [`LfmHasher::transcript`] calls the emitted program makes +/// of `LFM_HASH`, in the same order, on the same operands. The two are rewritten +/// together for exactly that reason; a divergence would show up as a fixture +/// proof the machine rejects, which is a slow and confusing way to learn about +/// it. +/// +/// Parameterised by hasher because the transcript is: `Test` and `Poseidon` +/// hash a transcript step with their single domain, BLAKE3 with the `"LFMT"` +/// tag, and the host has to agree with whichever one the proof is under. +pub struct HostSponge { + state: LfmWord, + squeeze_index: u32, + hasher: HasherKind, +} + +impl Default for HostSponge { + fn default() -> Self { + Self::new() + } +} + +impl HostSponge { + /// The chain under the machine's default hasher. + pub fn new() -> Self { + Self::with_hasher(HasherKind::default()) + } + + pub fn with_hasher(hasher: HasherKind) -> Self { + HostSponge { + state: [FE::zero(); 4], + squeeze_index: 0, + hasher, + } + } + + /// `SQ(i) = [SQUEEZE_MARK, i, 0, 0]` — the advance operand. + pub fn squeeze_operand(i: u32) -> LfmWord { + [ + FE::from(u64::from(SQUEEZE_MARK)), + FE::from(u64::from(i)), + FE::zero(), + FE::zero(), + ] + } + + /// The state as it stands — for the KATs, which pin it after every step. + pub fn state(&self) -> LfmWord { + self.state + } + + pub fn absorb(&mut self, c: &LfmWord) { + self.state = self.hasher.transcript(&self.state, c); + } + + pub fn absorb2(&mut self, c0: &LfmWord, c1: &LfmWord) { + self.absorb(c0); + self.absorb(c1); + } + + /// Absorb a cell of four arbitrary FIELD ELEMENTS — the host mirror of + /// [`super::edsl::SpongeVar::absorb_felts`]. Data enters the transcript + /// through the leaf encoding, exactly as it enters a tree. + pub fn absorb_felts(&mut self, c: &LfmWord) { + let d = self.hasher.leaf(&leaf_chain_start(), c); + self.absorb(&d); + } + + /// Output the current state, then advance past it with `SQ(i)`. + pub fn squeeze_cell(&mut self) -> LfmWord { + let out = self.state; + let sq = Self::squeeze_operand(self.squeeze_index); + self.state = self.hasher.transcript(&self.state, &sq); + self.squeeze_index += 1; + out + } + + pub fn squeeze_ext(&mut self) -> FEE { + let c = self.squeeze_cell(); + FEE::new([c[0], c[1], c[2]]) + } + + pub fn squeeze_index(&mut self, nbits: usize) -> u64 { + let c = self.squeeze_cell(); + GoldilocksField::canonical(c[0].value()) & ((1 << nbits) - 1) + } +} + +/// Where a leaf chain starts — the host mirror of +/// [`super::edsl::leaf_chain_start`], and a chain START rather than a shape +/// HEADER for the reason stated there. +pub fn leaf_chain_start() -> LfmWord { + [FE::zero(); 4] +} + +/// The host's Merkle LEAF over a pair of DATA cells — the mirror of +/// [`super::edsl::leaf_hash_pair`]. +/// +/// Two hasher calls, in the machine's order: one `"LFML"` chain absorbing the +/// cells in sequence, four felts and one chaining step per call. Written beside +/// the tree rather than inside it because a tree's *leaves* are data and its +/// *nodes* are digests, and this is the one place that distinction becomes two +/// different hash domains. +pub fn host_leaf_hash_pair(hasher: HasherKind, c0: &LfmWord, c1: &LfmWord) -> LfmWord { + let acc = hasher.leaf(&leaf_chain_start(), c0); + hasher.leaf(&acc, c1) +} + +/// A binary Merkle tree over word digests. +/// +/// ⚠ Parameterised by hasher, and it has to be: the machine authenticates these +/// openings with `edsl::merkle_walk`, which compresses under whichever hasher +/// the proof is built with. A tree that hard-coded one hasher would produce +/// roots the machine cannot reproduce the moment the proof is under another — +/// the failure would surface as an authentication error deep in a query walk, +/// which is a slow way to learn about a fixture bug. +pub struct HostTree { + /// levels[0] = leaves … levels.last() = [root]. + pub levels: Vec>, +} + +impl HostTree { + pub fn build(hasher: HasherKind, leaves: Vec) -> Self { + assert!(leaves.len().is_power_of_two()); + let mut levels = vec![leaves]; + while levels.last().unwrap().len() > 1 { + let prev = levels.last().unwrap(); + let next: Vec = prev + .chunks_exact(2) + .map(|pair| hasher.compress(&pair[0], &pair[1])) + .collect(); + levels.push(next); + } + HostTree { levels } + } + + pub fn root(&self) -> LfmWord { + self.levels.last().unwrap()[0] + } + + /// Sibling digests along the path from leaf `index`, level 0 first. + pub fn open(&self, mut index: usize) -> Vec { + let mut siblings = Vec::new(); + for level in &self.levels[..self.levels.len() - 1] { + siblings.push(level[index ^ 1]); + index >>= 1; + } + siblings + } +} + +/// The fixture proof, already in the machine's arena layout: +/// arena 0 = `[main_root, l1_root, t0, t1]`; arena 1 = per-query openings +/// (`shape::WORDS_PER_QUERY` words each, order pinned by the emitter). +pub struct FriToyProof { + pub commitments: Vec, + pub openings: Vec, +} + +/// The committed columns: fixed low-degree polynomials evaluated over the +/// LDE coset. Deterministic — the honest witness. +pub fn fixture_columns() -> [Vec; shape::NUM_COLS] { + let omega = GoldilocksField::get_primitive_root_of_unity(shape::LOG_LDE as u64) + .expect("32nd root of unity"); + let offset = FE::from(shape::COSET_OFFSET); + core::array::from_fn(|k| { + // degree < TRACE_LEN coefficients, fixed per column. + let coeffs: Vec = (0..shape::TRACE_LEN) + .map(|j| FE::from(1_000 * (k as u64 + 1) + j as u64 + 1)) + .collect(); + (0..shape::LDE_SIZE) + .map(|i| { + let x = &offset * omega.pow(i as u64); + coeffs.iter().rev().fold(FE::zero(), |acc, c| acc * &x + c) + }) + .collect() + }) +} + +fn row_word(cols: &[Vec; shape::NUM_COLS], i: usize) -> LfmWord { + core::array::from_fn(|k| cols[k][i]) +} + +/// Runs the fixture prover over the honest columns, under the machine's +/// default hasher. +pub fn fixture_prove() -> FriToyProof { + fixture_prove_columns(&fixture_columns()) +} + +/// [`fixture_prove`] under an explicitly chosen hasher. +pub fn fixture_prove_with_hasher(hasher: HasherKind) -> FriToyProof { + fixture_prove_columns_with_hasher(&fixture_columns(), hasher) +} + +/// The prover proper, over arbitrary columns (tests tamper these). +pub fn fixture_prove_columns(cols: &[Vec; shape::NUM_COLS]) -> FriToyProof { + fixture_prove_columns_with_hasher(cols, HasherKind::default()) +} + +/// [`fixture_prove_columns`] under an explicitly chosen hasher. +/// +/// Every hash this performs — leaves, tree nodes and the transcript — goes +/// through `hasher`, so the proof it produces is one the machine can +/// authenticate when proved under the same choice, and only then. +pub fn fixture_prove_columns_with_hasher( + cols: &[Vec; shape::NUM_COLS], + hasher: HasherKind, +) -> FriToyProof { + let omega = GoldilocksField::get_primitive_root_of_unity(shape::LOG_LDE as u64) + .expect("32nd root of unity"); + let offset = FE::from(shape::COSET_OFFSET); + let half = shape::LDE_SIZE / 2; // 16 + + // Main tree: row-pair leaves. A leaf is DATA, so it hashes in the leaf + // domain — two LFML rows and an LFMC parent, mirroring the emitter. + let leaves: Vec = (0..shape::LDE_SIZE / 2) + .map(|l| host_leaf_hash_pair(hasher, &row_word(cols, 2 * l), &row_word(cols, 2 * l + 1))) + .collect(); + let main_tree = HostTree::build(hasher, leaves); + + let mut sponge = HostSponge::with_hasher(hasher); + sponge.absorb(&main_tree.root()); + let alpha = sponge.squeeze_ext(); + let zeta0 = sponge.squeeze_ext(); + + // g0 = α-combination of the columns, over the full LDE domain. + let g0: Vec = (0..shape::LDE_SIZE) + .map(|i| { + let row = row_word(cols, i); + row.iter().rev().fold(FEE::zero(), |acc, v| { + acc * &alpha + FEE::new([*v, FE::zero(), FE::zero()]) + }) + }) + .collect(); + + // Fold 0 (unnormalized): g1[j] = (g0[j]+g0[j+16]) + x_j⁻¹·ζ0·(g0[j]−g0[j+16]). + let g1: Vec = (0..half) + .map(|j| { + let x = &offset * omega.pow(j as u64); + let inv_x = x.inv().expect("nonzero domain point"); + let (lo, hi) = (&g0[j], &g0[j + half]); + (lo + hi) + (&zeta0 * (lo - hi)) * FEE::new([inv_x, FE::zero(), FE::zero()]) + }) + .collect(); + + // L1 tree co-locates fold partners: leaf j covers g1[j] and g1[j+8]. These + // are folded EXTENSION elements — arbitrary field data — so they are leaves + // in exactly the same sense the trace rows are. + let quarter = half / 2; // 8 + let l1_leaves: Vec = (0..quarter) + .map(|j| host_leaf_hash_pair(hasher, &ext_word(&g1[j]), &ext_word(&g1[j + quarter]))) + .collect(); + let l1_tree = HostTree::build(hasher, l1_leaves); + + sponge.absorb(&l1_tree.root()); + let zeta1 = sponge.squeeze_ext(); + + // Fold 1 over the size-16 domain c²·⟨ω²⟩: y_j = c²ω^{2j}. + let g2: Vec = (0..quarter) + .map(|j| { + let y = offset.square() * omega.pow(2 * j as u64); + let inv_y = y.inv().expect("nonzero domain point"); + let (lo, hi) = (&g1[j], &g1[j + quarter]); + (lo + hi) + (&zeta1 * (lo - hi)) * FEE::new([inv_y, FE::zero(), FE::zero()]) + }) + .collect(); + + // Terminal polynomial (degree < 2) over c⁴·⟨ω⁴⟩, from two points; the + // remaining points must agree — the honest-witness sanity check. + let y_a = offset.square().square(); + let y_b = &y_a * omega.pow(4u64); + let embed = |x: &FE| FEE::new([*x, FE::zero(), FE::zero()]); + let t1 = (&g2[1] - &g2[0]) * (embed(&(&y_b - &y_a))).inv().expect("distinct points"); + let t0 = &g2[0] - &t1 * embed(&y_a); + for (j, v) in g2.iter().enumerate() { + let y = &y_a * omega.pow(4 * j as u64); + debug_assert_eq!(*v, &t0 + &t1 * embed(&y), "terminal degree bound violated"); + } + + // t0 and t1 are the terminal polynomial's COEFFICIENTS — field data, not + // digests — so they enter the transcript through the leaf encoding. + sponge.absorb_felts(&ext_word(&t0)); + sponge.absorb_felts(&ext_word(&t1)); + + // Queries. + let mut openings = Vec::new(); + for _ in 0..shape::NUM_QUERIES { + let q0 = sponge.squeeze_index(shape::QUERY_BITS) as usize; // [0, 16) + let leaf_a = q0 >> 1; + let leaf_b = leaf_a + shape::LDE_SIZE / 4; // + 8 + + // Main leaf A: its two rows + path. + openings.push(row_word(cols, 2 * leaf_a)); + openings.push(row_word(cols, 2 * leaf_a + 1)); + openings.extend(main_tree.open(leaf_a)); + // Main leaf B. + openings.push(row_word(cols, 2 * leaf_b)); + openings.push(row_word(cols, 2 * leaf_b + 1)); + openings.extend(main_tree.open(leaf_b)); + // L1 leaf pair + path. + let j = q0 % quarter; + openings.push(ext_word(&g1[j])); + openings.push(ext_word(&g1[j + quarter])); + openings.extend(l1_tree.open(j)); + } + debug_assert_eq!(openings.len(), shape::NUM_QUERIES * shape::WORDS_PER_QUERY); + + FriToyProof { + commitments: vec![ + main_tree.root(), + l1_tree.root(), + ext_word(&t0), + ext_word(&t1), + ], + openings, + } +} + +/// A word with lane 0 bumped — the tamper helper. +pub fn bump_lane0(w: &LfmWord) -> LfmWord { + [&w[0] + FE::one(), w[1], w[2], w[3]] +} + +/// Re-exported for the emitter: ω and the coset offset as constants. +pub fn domain_constants() -> (FE, FE) { + let omega = GoldilocksField::get_primitive_root_of_unity(shape::LOG_LDE as u64) + .expect("32nd root of unity"); + (omega, FE::from(shape::COSET_OFFSET)) +} + +// Small helper so `base_word` isn't unused when records are built elsewhere. +#[allow(dead_code)] +fn _base(v: FE) -> LfmWord { + base_word(v) +} diff --git a/prover/src/lfm/framework_probe.rs b/prover/src/lfm/framework_probe.rs new file mode 100644 index 000000000..80580d34b --- /dev/null +++ b/prover/src/lfm/framework_probe.rs @@ -0,0 +1,227 @@ +//! B0 de-risk probe (Milestone B entry gate). +//! +//! Every LFM chip stakes its instruction column group on one framework +//! pattern no in-tree chip exercises today: a **preprocessed column used as a +//! LogUp `Multiplicity`** (plus preprocessed bus values, which KECCAK_RC does +//! exercise). This probe round-trips a minimal sender/receiver pair through +//! the real `multi_prove` / `multi_verify_views`, with the sender's value and +//! multiplicity columns both preprocessed, and pins the tamper behavior: +//! a flipped preprocessed root is rejected by the prover (recommit mismatch) +//! and by the verifier (root equality), and a tampered witness value breaks +//! the bus balance. +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::config::Commitment; +use stark::constraints::builder::EmptyConstraints; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, +}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::MultiProofView; +use stark::prover::IsStarkProver; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::IsStarkVerifier; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::commit::commit_columns; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type ProbeAir = AirWithBuses; +type DynAir<'a> = &'a dyn AIR; + +/// Scratch bus id, far above the live `BusId` range. +const PROBE_BUS: u64 = 63; +const PROBE_TAG: &[u8] = b"LFM_B0_PROBE_V1"; +const NUM_ROWS: usize = 256; + +fn fe(v: u64) -> FE { + FE::from(v) +} + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("probe options") +} + +fn values() -> Vec { + (0..NUM_ROWS as u64).map(|i| fe(1_000 + 3 * i)).collect() +} + +/// Sender: cols `[VAL (prep 0), MULT (prep 1), PAD (main 2)]` — both the bus +/// value AND the multiplicity read preprocessed columns. +fn sender_air(prep_root: Commitment, opts: &ProofOptions) -> ProbeAir { + let interactions = vec![BusInteraction::sender( + PROBE_BUS, + Multiplicity::Column(1), + vec![BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }], + )]; + AirWithBuses::new( + 3, + AuxiliaryTraceBuildData { interactions }, + opts, + 1, + EmptyConstraints, + ) + .with_name("B0_SEND") + .with_preprocessed(prep_root, 2) +} + +/// Receiver: cols `[VAL (main 0), MULT (main 1)]` — plain witness echo. +fn receiver_air(opts: &ProofOptions) -> ProbeAir { + let interactions = vec![BusInteraction::receiver( + PROBE_BUS, + Multiplicity::Column(1), + vec![BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }], + )]; + AirWithBuses::new( + 2, + AuxiliaryTraceBuildData { interactions }, + opts, + 1, + EmptyConstraints, + ) + .with_name("B0_RECV") +} + +fn sender_trace() -> TraceTable { + let mut data = Vec::with_capacity(NUM_ROWS * 3); + for v in values() { + data.extend([v, FE::one(), FE::zero()]); + } + TraceTable::new_main(data, 3, 1) +} + +fn receiver_trace() -> TraceTable { + let mut data = Vec::with_capacity(NUM_ROWS * 2); + for v in values() { + data.extend([v, FE::one()]); + } + TraceTable::new_main(data, 2, 1) +} + +fn prep_root(opts: &ProofOptions) -> Commitment { + commit_columns(&[values(), vec![FE::one(); NUM_ROWS]], opts) +} + +fn transcript() -> crate::hash_pin::BlockTranscript { + let mut t = crate::hash_pin::block_transcript(&[]); + t.append_bytes(PROBE_TAG); + t +} + +fn prove( + sender: &ProbeAir, + receiver: &ProbeAir, +) -> Result, stark::prover::ProvingError> { + let mut st = sender_trace(); + let mut rt = receiver_trace(); + let pairs: Vec<(DynAir, &mut TraceTable, &())> = + vec![(sender, &mut st, &()), (receiver, &mut rt, &())]; + let mut t = transcript(); + crate::hash_pin::BlockProver::multi_prove( + pairs, + &mut t, + stark::residency_mode::ResidencyMode::Retain, + ) +} + +#[test] +fn b0_preprocessed_multiplicity_round_trips() { + let opts = options(); + let root = prep_root(&opts); + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + let proof = prove(&sender, &receiver).expect("prove with preprocessed multiplicity"); + + let refs: Vec = vec![&sender, &receiver]; + let mut vt = transcript(); + assert!( + crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(&proof), + &mut vt, + &FEE::zero(), + ), + "honest proof must verify" + ); +} + +#[test] +fn b0_prover_rejects_mismatched_preprocessed_root() { + let opts = options(); + let mut root = prep_root(&opts); + root[0] ^= 1; + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + assert!( + prove(&sender, &receiver).is_err(), + "prover must reject a trace that does not recommit to the supplied root" + ); +} + +#[test] +fn b0_verifier_rejects_wrong_preprocessed_root() { + let opts = options(); + let root = prep_root(&opts); + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + let proof = prove(&sender, &receiver).expect("honest prove"); + + let mut bad_root = root; + bad_root[0] ^= 1; + let bad_sender = sender_air(bad_root, &opts); + let refs: Vec = vec![&bad_sender, &receiver]; + let mut vt = transcript(); + assert!( + !crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(&proof), + &mut vt, + &FEE::zero(), + ), + "a supplied root differing from the proof's must reject" + ); +} + +#[test] +fn b0_tampered_witness_value_breaks_balance() { + let opts = options(); + let root = prep_root(&opts); + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + + // Receiver echoes one wrong value: prove succeeds locally (no constraint + // relates the two tables directly) but the bus no longer balances to 0. + let mut st = sender_trace(); + let mut rt = receiver_trace(); + rt.set_main(0, 0, fe(999_999)); + let pairs: Vec<(DynAir, &mut TraceTable, &())> = + vec![(&sender, &mut st, &()), (&receiver, &mut rt, &())]; + let mut t = transcript(); + let proof = crate::hash_pin::BlockProver::multi_prove( + pairs, + &mut t, + stark::residency_mode::ResidencyMode::Retain, + ) + .expect("locally consistent"); + + let refs: Vec = vec![&sender, &receiver]; + let mut vt = transcript(); + assert!( + !crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(&proof), + &mut vt, + &FEE::zero(), + ), + "unbalanced bus must reject" + ); +} diff --git a/prover/src/lfm/fri.rs b/prover/src/lfm/fri.rs new file mode 100644 index 000000000..3b1cca886 --- /dev/null +++ b/prover/src/lfm/fri.rs @@ -0,0 +1,1400 @@ +//! FRI: the compile-time shape of the emitted verifier. +//! +//! Slice 1 of the FRI leg — the arithmetic only. `others/lfm-fri-verify-spec.md` +//! is the verified account of the production verify path this mirrors; §2 is +//! the section this file implements. +//! +//! ## Why the shape is a struct and not a runtime computation +//! +//! Production derives the fold layout at verify time from the AIR's options and +//! domain (`FriFoldLayout::new`, `fri/terminal.rs:45`). The machine cannot: it +//! is straight-line, so the layer count fixes how many walks and folds are +//! EMITTED. Every field below is therefore program shape in the sense of +//! `others/lfm-target-shape.md`, and a program that read any of it from an +//! arena would let the prover choose how much FRI to verify — the degenerate +//! case being "none". +//! +//! ## What this module is checked against +//! +//! `FriFoldLayout` is `pub(crate)` inside `crypto/stark`, so this mirror cannot +//! be differentialled against the struct itself. The oracle is production's +//! observable BEHAVIOUR instead — the vector lengths a real proof carries and +//! the verifier structurally enforces before its query loop +//! (`verifier.rs:426-448`): `fri_layers_merkle_roots.len() == num_committed` +//! and `fri_final_poly_coeffs.len() == 1 << effective_k`. That is a stronger +//! check than reading the struct would be, because those are the lengths the +//! verifier actually rejects on. +//! +//! ## What this cannot see +//! +//! It is arithmetic over a shape; it says nothing about whether the emitted +//! walk or fold is correct, only about how many of each there should be. It +//! also mirrors the CPU layout only — `fri/mod.rs` has cuda fast paths that +//! claim the same layout, unverified here and never run by the machine. + +use stark::config::TreeGeometry; +use stark::fri::schedule::FriFormat; +use stark::leaf_layout::LeafLayout; +use stark::proof::options::{FriMode, OneRowMode, ProofFormat, ProofOptions}; + +use crate::tables::types::FE; + +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::edsl::{self, WrapDigest}; +use super::instr::ArenaId; +use super::merkle_cap::CapCells; +use super::sub_proof::{self, GroupShape}; + +/// The compile-time shape of one sub-proof's FRI verification. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FriShape { + /// `log2` of the LDE (deep-composition) codeword length. + pub log2_lde_length: u32, + /// `log2` of the blowup factor. + pub blowup_log: u32, + /// The requested terminal log-degree, `ProofOptions::fri_final_poly_log_degree`. + pub final_poly_log_degree: u32, + /// The LDE coset offset. Carried rather than assumed: the emitter bakes + /// domain constants derived from it, and the standing deferral on + /// `coset_offset != 3` is about test COVERAGE, not about a hardcoded 3 — + /// so the value has to come from the options, and [`Self::from_options`] is + /// the only constructor that reads it. + pub coset_offset: u64, + /// Queries the sub-proof carries. + pub num_queries: usize, + /// The inner proof's FORMAT: its Merkle cap + /// policy caps every committed layer tree. A verifier constant, taken from + /// the inner proof's options — never from the proof. + /// + /// `format.one_row` is the table's RESOLVED leaf layout (S2): `Off` (row + /// pairs) or `On` (one row), never `Auto` — `auto` is resolved per table + /// from the AIR's widths ([`Self::for_layout`]) before a shape exists, and + /// [`Self::check`] refuses an unresolved one. + pub format: ProofFormat, + /// The layer trees' arity and arity-4 cap: the inner proof's commitment + /// configuration's (`TreeGeometry::of::()`), [`TreeGeometry::BINARY`] + /// for every binary one. A verifier constant, never the proof's. + pub trees: TreeGeometry, +} + +impl FriShape { + /// Derive the shape from the inner proof's own options. + /// + /// Every FRI-relevant parameter comes from `options` — including the coset + /// offset, which discharges the plumbing half of the `coset_offset != 3` + /// deferral recorded in `others/lfm-assembly-obligations.md`. + /// + /// # Panics + /// + /// On `one_row = auto`: the layout of an `auto` table is resolved from its + /// AIR's committed widths (`stark::leaf_layout::table_leaf_layout`), which + /// the options alone do not carry — use [`Self::for_layout`] with the + /// table's resolved layout. `Off` and `On` resolve themselves. + pub fn from_options(options: &ProofOptions, log2_lde_length: u32) -> Self { + let layout = match options.format.one_row { + OneRowMode::Off => LeafLayout::RowPair, + OneRowMode::On => LeafLayout::Row, + OneRowMode::Auto => panic!( + "one_row = auto resolves per table from the AIR's widths: build the \ + FRI shape with FriShape::for_layout(options, lde, table_leaf_layout(air, n))" + ), + }; + Self::for_layout(options, log2_lde_length, layout) + } + + /// The shape of a table proved under `options` whose trace trees use the + /// RESOLVED leaf `layout` (the table's `stark::leaf_layout::table_leaf_layout` + /// — what the host prover and verifier lay the proof out with). The + /// resolved layout is stored in `format.one_row` (`Off` / `On`). + pub fn for_layout(options: &ProofOptions, log2_lde_length: u32, layout: LeafLayout) -> Self { + let mut format = options.format; + format.one_row = if layout.is_one_row() { + OneRowMode::On + } else { + OneRowMode::Off + }; + Self { + log2_lde_length, + blowup_log: (options.blowup_factor as u32).trailing_zeros(), + final_poly_log_degree: options.fri_final_poly_log_degree as u32, + coset_offset: options.coset_offset, + num_queries: options.fri_number_of_queries, + format, + trees: TreeGeometry::BINARY, + } + } + + /// This shape with its layer trees of `trees` (the inner proof's + /// configuration's geometry). + pub fn with_trees(self, trees: TreeGeometry) -> Self { + Self { trees, ..self } + } + + /// Children per node of the layer trees. + pub fn arity(self) -> usize { + self.trees.arity + } + + /// Whether the table's trace trees hold one row per leaf (S2): the DEEP + /// codeword is then committed as FRI layer 0 (the input tree), the query + /// index has `log2(lde)` bits and no fold precedes layer 0. + pub fn one_row(self) -> bool { + match self.format.one_row { + OneRowMode::Off => false, + OneRowMode::On => true, + OneRowMode::Auto => { + panic!("a FRI shape carries a RESOLVED layout, never one_row = auto") + } + } + } + + /// The trace trees' leaf layout this shape verifies. + pub fn leaf_layout(self) -> LeafLayout { + LeafLayout::from_one_row(self.one_row()) + } + + /// `log2` of the terminal codeword length, clamped to the full LDE for + /// traces too small to fold that far (`terminal.rs:46`'s `.min(lde_log)`). + pub fn terminal_log(self) -> u32 { + (self.blowup_log + self.final_poly_log_degree).min(self.log2_lde_length) + } + + /// Folds from the LDE codeword down to the terminal codeword. + pub fn total_folds(self) -> u32 { + self.log2_lde_length - self.terminal_log() + } + + /// Whether the proof uses today's FRI encoding: pair layers, one sibling + /// value per committed layer (`fri = pair` with row-pair openings). + /// Decided by the FORMAT, never by the schedule's values: a `dp` schedule + /// of all ones still uses the group encoding, and so does every one-row + /// table (`FriFormat::is_legacy`: its layer 0 is the committed DEEP + /// codeword, opened as a full group). Every non-legacy path below is the + /// group path; the legacy emission is today's, instruction for instruction. + pub fn is_legacy(self) -> bool { + self.format.fri_mode == FriMode::Pair && !self.one_row() + } + + /// The host's own FRI format for this shape (the fold-schedule DP's + /// inputs): the mode, the query count (every FRI tree is opened once per + /// query), the cap policy and the test-only schedule override. + fn fri_format(self) -> FriFormat { + FriFormat { + mode: self.format.fri_mode, + one_row: self.one_row(), + num_queries: self.num_queries as u64, + cap: self.format.merkle_cap, + schedule_override: self.format.fri_schedule_override, + } + } + + /// ★ The committed layers' fold exponents, first committed layer first — + /// the SAME function the host prover and verifier lay out with + /// (`stark::fri::schedule::FriFormat::schedule`: the all-ones schedule + /// under `pair`, the cost-law DP under `dp`). A format + /// constant: nothing here reads a proof. + /// + /// ⚠ `num_queries` is a DP input (and a cap-policy input): a program that + /// verifies a SUBSET of a proof's queries has a different `dp` schedule + /// and `auto` caps than the proof unless the query count is kept. + pub fn schedule(self) -> Vec { + self.fri_format() + .schedule(self.log2_lde_length, self.terminal_log()) + } + + /// Committed (Merkle-rooted) layers — one root, one auth path per query, + /// and one Merkle walk to emit, each: the schedule's length. + /// + /// **`total_folds − 1` under `pair`, not `total_folds`.** The final fold is + /// performed and never committed (`fri/mod.rs:114-118`), so a query folds + /// once more than it authenticates. This off-by-one is the readiest way to + /// build a verifier that looks right and checks one layer too few. Under + /// one-row leaves the chain starts at the DEEP codeword itself (layer 0 = + /// the input tree), so the pair schedule is `total_folds` ones. + pub fn num_committed(self) -> usize { + self.schedule().len() + } + + /// Folding challenges the proof draws (`FriFoldLayout::num_zetas`): + /// one per committed layer plus the final fold's for row pairs (fold 0 + /// consumes the first), one per committed layer under one row (layer 0 is + /// committed before any challenge); none when nothing folds. + pub fn num_zetas(self) -> usize { + if self.total_folds() == 0 { + 0 + } else { + self.num_committed() + usize::from(!self.one_row()) + } + } + + /// Index of committed layer `j`'s challenge in the ζ list: `j + 1` for row + /// pairs (`ζ₀` drove the uncommitted fold 0), `j` under one row. + pub fn layer_zeta_index(self, layer: usize) -> usize { + layer + usize::from(!self.one_row()) + } + + /// Fold exponent `d_j` of committed layer `j`: a leaf groups `2^{d_j}` + /// consecutive values (1 = today's pair). + pub fn layer_fold(self, layer: usize) -> u32 { + u32::from(self.schedule()[layer]) + } + + /// Index bits consumed before committed layer `j`: `G_j = Σ_{i usize { + self.schedule()[..layer].iter().map(|&d| d as usize).sum() + } + + /// Opened values one query's opening of committed layer `j` carries: the + /// sibling alone under `pair`, the whole `2^{d_j}` group otherwise + /// (the query's own value included). + pub fn layer_values(self, layer: usize) -> usize { + if self.is_legacy() { + 1 + } else { + 1usize << self.layer_fold(layer) + } + } + + /// Felts committed layer `j`'s leaf hashes: `2^{d_j}` extension values of + /// three felts — six (the pair) under `pair`. + pub fn layer_leaf_felts(self, layer: usize) -> usize { + 3 << self.layer_fold(layer) + } + + /// Leaf permutations one query costs across every committed layer under + /// `hash`'s own block rule (`epoch_verify::blocks_for`). + pub fn leaf_permutations_per_query(self, hash: super::edsl::WrapHash) -> usize { + (0..self.num_committed()) + .map(|j| super::epoch_verify::blocks_for(self.layer_leaf_felts(j), hash)) + .sum() + } + + /// Folds a query performs: `num_committed + 1` whenever anything folds at + /// all, and 0 when the codeword is already terminal. + pub fn num_folds(self) -> usize { + self.total_folds() as usize + } + + /// Terminal codeword length. + pub fn terminal_len(self) -> usize { + 1usize << self.terminal_log() + } + + /// The terminal log-degree actually used — `min(k, trace_bits)`. Equals + /// `final_poly_log_degree` except under the clamp. + pub fn effective_k(self) -> u32 { + self.terminal_log() - self.blowup_log + } + + /// Coefficients the proof carries for the terminal polynomial. + pub fn num_terminal_coeffs(self) -> usize { + 1usize << self.effective_k() + } + + /// Tree depth of committed layer `j`: the layer is `2^(n − 1 − G_j)` + /// values long and its leaves group `2^{d_j}`, so the tree has + /// `2^(n − 1 − G_j − d_j)` leaves — `n − j − 2` under `pair`. + pub fn layer_depth(self, layer: usize) -> usize { + let schedule = self.schedule(); + assert!( + layer < schedule.len(), + "layer index must be below num_committed" + ); + let consumed: usize = schedule[..=layer].iter().map(|&d| d as usize).sum(); + self.index_bits() - consumed + } + + /// Merkle-cap height of committed layer `i`'s tree under the trees' cap + /// policy, in their arity's levels: every layer tree is opened once per + /// query (`0` = uncapped). The same function the host prover and verifier + /// use (`stark::merkle_caps::StarkCaps`), at the same depth. + pub fn layer_cap(self, layer: usize) -> usize { + stark::merkle_caps::StarkCaps::tree_cap_height( + self.trees.cap_policy(self.format.merkle_cap), + self.num_queries, + self.layer_depth(layer), + self.arity(), + ) + } + + /// Sibling digests a query's opening of committed layer `i` carries: the + /// tree's depth less its cap height at arity 2, three per walked 4-ary + /// level at arity 4 (the owner path's cap is split off into the caps + /// arena; [`super::merkle_cap::path_hints`]). + pub fn layer_path_len(self, layer: usize) -> usize { + super::merkle_cap::path_hints(self.layer_depth(layer), self.layer_cap(layer), self.arity()) + } + + /// Node hashes a query's walk of committed layer `i` costs. + pub fn layer_path_permutations(self, layer: usize) -> usize { + super::merkle_cap::path_permutations( + self.layer_depth(layer), + self.layer_cap(layer), + self.arity(), + ) + } + + /// Digests in committed layer `i`'s cap (1, the root, uncapped). + pub fn layer_cap_nodes(self, layer: usize) -> usize { + super::merkle_cap::cap_nodes(self.layer_depth(layer), self.layer_cap(layer), self.arity()) + } + + /// Arena words the committed layers' caps occupy, once per sub-proof. + pub fn cap_words(self, digest_words: usize) -> usize { + (0..self.num_committed()) + .map(|i| match self.layer_cap(i) { + 0 => 0, + _ => self.layer_cap_nodes(i) * digest_words, + }) + .sum() + } + + /// Permutations the committed layers' cap checks cost, once per + /// sub-proof: each capped layer's cap hashed to its root. + pub fn cap_permutations(self) -> usize { + (0..self.num_committed()) + .map(|i| { + super::merkle_cap::cap_root_permutations(self.layer_cap_nodes(i), self.arity()) + }) + .sum() + } + + /// Merkle path steps (node hashes) one query walks across every committed + /// layer. + pub fn path_steps_per_query(self) -> usize { + (0..self.num_committed()) + .map(|i| self.layer_path_permutations(i)) + .sum() + } + + /// Sibling digests one query's paths carry across every committed layer + /// ([`Self::path_steps_per_query`] at arity 2). + pub fn path_hints_per_query(self) -> usize { + (0..self.num_committed()) + .map(|i| self.layer_path_len(i)) + .sum() + } + + /// Permutations one query costs under the production wrap hash: every + /// committed layer's leaf (a 48-byte pair is one block under every hash; + /// a `2^d` group is `⌈3·2^d / 8⌉` at the rate-8 algebraic sponge) plus one + /// per path step (a parent is one compression under every hash). + pub fn permutations_per_query(self) -> usize { + self.leaf_permutations_per_query(super::edsl::WrapHash::production()) + + self.path_steps_per_query() + } + + /// Index bits a query carries — `log2(lde) − 1` for row pairs (the pair + /// index `iota`), `log2(lde)` under one-row leaves (`r` over the whole + /// LDE) — which is both the TRACE trees' Merkle depth and the bit + /// width of the index. + /// + /// The FRI layers consume SUFFIXES of this one decomposition rather than + /// decompositions of their own, which is what makes the emitted walks + /// address the same query the trace openings did. Layer `i` reads `bits[i]` + /// as its leaf-ordering parity and `bits[i+1..]` as its walk, and + /// `bits[i+1..].len() = n − i − 2 = layer_path_len(i)` exactly — the layer + /// tree's depth is not a separate fact to keep in sync, it is what is left + /// of the index after the folds already performed. (Under a Merkle cap the + /// top `layer_cap(i)` of those bits pick the cap node instead of being + /// walked; the split is the cap's own, [`CapCells::verify_path`].) + pub fn index_bits(self) -> usize { + self.leaf_layout().tree_depth(self.log2_lde_length as usize) + } + + /// Arena words one query's FRI opening occupies: per committed layer its + /// opened values ([`Self::layer_values`]: the symmetric evaluation, or the + /// whole group) and its path (`digest_words` per level). + /// + /// `digest_words` is the BUILDER's digest width on the machine side + /// (`edsl::digest_words(b)`) and `proof_arena::words_per_root()` on the + /// host side — see `SubProofShape::query_words` for why it is an argument. + pub fn query_words(self, digest_words: usize) -> usize { + // The path stride is the DIGEST's width, not a literal two. + let values: usize = (0..self.num_committed()) + .map(|j| self.layer_values(j)) + .sum(); + values + digest_words * self.path_hints_per_query() + } + + /// Keccak permutations the whole sub-proof's FRI costs. + pub fn permutations(self) -> usize { + self.num_queries * self.permutations_per_query() + } + + /// Invariants a caller cannot assemble their way out of. + pub fn check(self) { + assert!( + matches!(self.arity(), 2 | 4), + "a layer tree has 2 or 4 children per node, not {}", + self.arity() + ); + assert!( + self.format.one_row != OneRowMode::Auto, + "a FRI shape carries a RESOLVED layout (FriShape::for_layout), never one_row = auto" + ); + // The schedule covers exactly the committed folds (`FriFoldLayout`'s + // constructor invariant, which refuses a proof otherwise). + let schedule = self.schedule(); + let covered: u32 = schedule.iter().map(|&d| u32::from(d)).sum(); + assert!( + schedule + .iter() + .all(|&d| (1..=stark::fri::schedule::FRI_SCHEDULE_DMAX).contains(&u32::from(d))), + "every fold exponent is in 1..=DMAX: {schedule:?}" + ); + // Row pairs: fold 0 is binary and uncommitted. One row: every fold is + // a committed layer's (layer 0 is the DEEP codeword). + let committed_folds = if self.one_row() { + self.total_folds() + } else { + self.total_folds().saturating_sub(1) + }; + assert_eq!( + covered, committed_folds, + "the schedule {schedule:?} must cover the committed folds" + ); + assert!( + self.blowup_log >= 1, + "a blowup of 1 is not a low-degree extension" + ); + assert!( + self.log2_lde_length > self.blowup_log, + "the LDE must be strictly larger than the blowup: a trace of one \ + row has no FRI to do" + ); + assert!( + self.terminal_log() <= self.log2_lde_length, + "the terminal codeword cannot exceed the LDE" + ); + assert!( + self.effective_k() <= self.final_poly_log_degree, + "the clamp can only lower the terminal degree, never raise it" + ); + assert_eq!( + self.terminal_len(), + 1usize << (self.blowup_log + self.effective_k()), + "terminal_len must equal 2^(blowup_log + effective_k)" + ); + } +} + +// ============================ the emitter ============================ +// +// Slice 2: the per-layer walk, the fold chain, and the terminal check. The +// shape above says how many of each; this says what each one is. +// +// ## What the machine emits, against what production runs +// +// Production's `verify_query_and_sym_openings` (`verifier.rs:660-748`) is a loop +// over committed layers with a running `(v, index)` pair. Here the loop is +// unrolled at build time and `index` never exists as a value: every use of it is +// a use of some suffix of the query's bit decomposition. The three uses map as +// +// ```text +// production machine +// ---------- ------- +// iota % 2 (leaf order) bits[i] +// iota >> 1 (leaf position) bits[i+1..] (the walk) +// index >>= 1 (next layer) i += 1 (a host-side index) +// ``` +// +// so the halving that production performs per layer is, here, reading one bit +// further along a vector that was decomposed once — by the trace leg, for its +// own walk. That is the join: there is no second index in the program to +// disagree with the first. +// +// ## What this cannot see +// +// It emits ONE sub-proof's FRI. Nothing here says the terminal coefficients or +// the folding challenges are the ones the transcript produced — they arrive as +// arena values, exactly as `γ` and `ζ` do in [`super::sub_proof`], and binding +// them to a transcript replay is assembly's obligation, covered by the standing +// clause in `others/lfm-assembly-obligations.md`. It also says nothing about +// whether `p₀` is the DEEP value of the authenticated opening; that is the +// previous leg's join, consumed here as cells. + +/// The group shape of a FRI layer leaf: ONE extension column, so a leaf covers +/// `ROWS_PER_LEAF = 2` values and 48 bytes. +/// +/// Reusing [`super::sub_proof::emit_leaf_hash`] rather than writing a second +/// gadget is deliberate and was checked rather than assumed — see +/// `fri_tests::the_fri_leaf_is_byte_identical_to_productions_own_backends`, +/// which runs the machine's leaf against BOTH production backends on vectors +/// that differ in every one of the 48 bytes. +/// +/// It is worth stating why the shapes coincide at all, because the two +/// commitments are built by different code: a trace leaf applies +/// `reverse_index` INSIDE the leaf builder and concatenates column-by-column +/// across a row pair (`commitment.rs:81-91`), while a FRI layer leaf is +/// `evals.chunks_exact(2)` of an ALREADY bit-reversed single codeword +/// (`fri/mod.rs:96-99`). At one column those two descriptions produce the same +/// byte string from the same pair — the permutation a trace leaf applies is the +/// permutation a FRI codeword already carries — and at more than one column +/// they do not. So this constant is not "the trace shape with a 1 in it"; it is +/// the point where the two layouts happen to meet. +pub const FRI_LEAF_GROUP: GroupShape = GroupShape { + num_columns: 1, + is_ext: true, +}; + +/// One committed FRI layer's root, unpacked once per sub-proof. +/// +/// The hoist matters at production query counts for the same reason +/// [`super::sub_proof::GroupCommitment`]'s does: a root is a per-sub-proof value +/// and a 219-query proof would otherwise pay 219 redundant `Unpack`s per layer. +pub struct LayerCommitment { + /// The root's two words as lanes. + /// ⚠ A `Vec`, ONE ENTRY PER DIGEST CELL, not `[_; 2]`: the array wrote the + /// byte digest's cell COUNT into the type, and an algebraic root is one cell + /// of four felts. `edsl::assert_digest_eq_lanes` zips a digest against these + /// and asserts the widths agree, so it works at either width unchanged. + pub root_lanes: Vec<[Felt; 4]>, + /// The layer tree's authenticated Merkle cap, when the format caps it + /// (see [`super::sub_proof::GroupCommitment::cap`]). `None` = today. + pub cap: Option, +} + +impl LayerCommitment { + /// Read a layer root out of the arena and hoist its unpack. + pub fn hint(b: &mut LfmBuilder, arena: ArenaId, base: u32) -> Self { + // ⚠ The configuration's word count, not two: an algebraic root is ONE + // arena word of four felts. `RootCells::words_per_root` is the single + // definition; a second `+ 1` here would be a second definition. + let n = super::epoch::RootCells::words_per_root(b); + let root_lanes: Vec<[Felt; 4]> = (0..n) + .map(|i| { + let w = b.hint_word(arena, base + i); + b.unpack(w) + }) + .collect(); + LayerCommitment { + root_lanes, + cap: None, + } + } + + /// A layer commitment over lanes the caller already holds. + /// + /// The assembled verifier's route: a FRI layer root is absorbed by the + /// transcript in Round 4 (right after its own `ζ`) and compared against here, + /// and those two consumers must read one cell. See + /// [`super::sub_proof::GroupCommitment::from_lanes`] for the same argument at + /// the trace trees. + pub fn from_lanes(root_lanes: Vec<[Felt; 4]>) -> Self { + LayerCommitment { + root_lanes, + cap: None, + } + } + + /// Hint this layer tree's height-`c` cap out of `arena` at `base` and + /// authenticate it against the root lanes, once per tree (see + /// [`super::sub_proof::GroupCommitment::hint_cap`]). Returns the next free + /// word; `c = 0` hints nothing. + pub fn hint_cap( + &mut self, + b: &mut LfmBuilder, + arena: ArenaId, + base: u32, + c: usize, + depth: usize, + ) -> u32 { + if c == 0 { + return base; + } + let (cap, next) = + super::merkle_cap::hint_and_authenticate(b, arena, base, c, depth, &self.root_lanes); + self.cap = Some(cap); + next + } + + /// Authenticate an opened leaf of this layer at the tree's WHOLE leaf + /// index: against the cap when capped, else against the root lanes (the + /// uncapped emission is today's, instruction for instruction). + fn authenticate( + &self, + b: &mut LfmBuilder, + leaf: WrapDigest, + index_bits: &[Bit], + siblings: &[WrapDigest], + ) { + sub_proof::authenticate_leaf( + b, + leaf, + &self.root_lanes, + self.cap.as_ref(), + index_bits, + siblings, + ); + } + + fn cap_height(&self) -> usize { + self.cap.as_ref().map_or(0, CapCells::height) + } +} + +/// Hint and authenticate every capped committed layer's cap, in layer order, +/// out of `arena` from word 0 — once per sub-proof. The words it reads are +/// exactly [`FriShape::cap_words`]. +pub fn hint_layer_caps( + b: &mut LfmBuilder, + shape: FriShape, + arena: ArenaId, + layers: &mut [LayerCommitment], +) { + assert_eq!( + layers.len(), + shape.num_committed(), + "one commitment per layer" + ); + let mut at = 0u32; + for (i, layer) in layers.iter_mut().enumerate() { + at = layer.hint_cap(b, arena, at, shape.layer_cap(i), shape.layer_depth(i)); + } + assert_eq!( + at as usize, + shape.cap_words(edsl::digest_words(b) as usize), + "the FRI caps fill exactly what the shape declares" + ); +} + +/// A sub-proof's FRI data that does not depend on the query. +pub struct FriCommitments { + /// One per committed layer, in fold order. + pub layers: Vec, + /// The folding challenges `ζ₀ .. ζ_C` — `num_committed + 1` of them, or + /// none when nothing folds. The asymmetry is the whole off-by-one of this + /// leg: the first fold consumes the DEEP pair and is not committed, so + /// folds exceed layers by one (`fri/mod.rs:114-118`). Under one-row + /// leaves there is no such fold: `num_committed` challenges + /// ([`FriShape::num_zetas`]). + pub zetas: Vec, + /// The terminal polynomial's `2^effective_k` coefficients, low-to-high. + pub coeffs: Vec, + /// Under the group encoding (S3, and every one-row table): per committed + /// layer `j`, the challenges its `d_j` binary folds use — `ζ, ζ², …, + /// ζ^{2^{d_j−1}}` for `ζ = ζ_{j+1}` (row pairs) or `ζ_j` (one row, + /// [`FriShape::layer_zeta_index`]) — squared ONCE per + /// sub-proof, not per query. Empty under the legacy encoding, where each + /// layer folds once with `ζ_{j+1}` itself. + pub zeta_powers: Vec>, +} + +impl FriCommitments { + /// The commitments of one sub-proof's FRI, with the group encoding's + /// challenge powers hoisted ([`Self::zeta_powers`]; nothing is emitted + /// under `pair`, so today's program is unchanged). + pub fn new( + b: &mut LfmBuilder, + shape: FriShape, + layers: Vec, + zetas: Vec, + coeffs: Vec, + ) -> Self { + let zeta_powers = if shape.is_legacy() || zetas.is_empty() { + Vec::new() + } else { + (0..shape.num_committed()) + .map(|j| { + let mut z = zetas[shape.layer_zeta_index(j)]; + let mut powers = vec![z]; + for _ in 1..shape.layer_fold(j) { + z = b.emul(z, z); + powers.push(z); + } + powers + }) + .collect() + }; + FriCommitments { + layers, + zetas, + coeffs, + zeta_powers, + } + } +} + +/// One query's opening of one committed layer. +/// +/// There is deliberately no constructor that hints — like +/// [`super::sub_proof::GroupOpening`], the values are the caller's, so what the +/// walk authenticates is what the fold consumes. +pub struct LayerOpening { + /// Under `pair`: ONE value, `pᵢ(−υ^(2ⁱ))` — the conjugate the prover + /// supplies. Its partner `pᵢ(υ^(2ⁱ))` is not in the proof at all: the + /// verifier computed it as the previous fold's output, which is why a pair + /// layer opening is one value and not two. + /// + /// Under the group encoding: the whole group of `2^{d_j}` values in + /// position (bit-reversed) order, the query's own value at its slot + /// included — the leaf is hashed straight from them and the + /// slot check `values[slot] == v` ties them to the previous fold. + pub values: Vec, + /// Sibling digests, LEAF LEVEL FIRST. + pub siblings: Vec, +} + +/// What the FRI leg needs from a query the trace legs already verified. +/// +/// Every field is a CELL the previous leg produced, never a fresh hint or a +/// re-derivation. [`super::sub_proof::QueryOutput`] is exactly this shape's +/// supplier. +pub struct FriQuery<'a> { + /// `p₀(υ)` — the DEEP reconstruction at the query point (`DEEP(x_r)` under + /// one-row leaves). + pub p0: Ext, + /// `p₀(−υ)` for a row-pair shape; `None` under one-row leaves, which open + /// one point. + pub p0_sym: Option, + /// `υ` (or `x_r`). Not Merkle-checked here and not hinted: it is the point + /// the authenticated opening was folded at. + pub point: Felt, + /// `−υ`, needed only by the row-pair zero-fold shape; `None` under one row. + pub point_sym: Option, + /// The query index low-to-high, `shape.index_bits()` of them — the cells + /// the trace walk consumed. + pub bits: &'a [Bit], +} + +impl FriQuery<'_> { + /// `p₀(−υ)` — a row-pair shape's. + fn p0_sym(&self) -> Ext { + self.p0_sym + .expect("a row-pair FRI query carries the symmetric DEEP value") + } +} + +/// The arenas one sub-proof's FRI verification reads, in declaration order. +pub struct FriArenas { + /// Two words per committed layer root, in fold order. + pub roots: ArenaId, + /// `ζ₀ .. ζ_C`, one word each. Empty when nothing folds. + pub zetas: ArenaId, + /// The terminal polynomial's coefficients, low-to-high. + pub coeffs: ArenaId, + /// Per query, per committed layer: the symmetric evaluation, then the + /// sibling digests (two words per level). + pub queries: ArenaId, + /// Per capped committed layer, its `2^c` cap digests — declared only when + /// the format caps some layer ([`FriShape::cap_words`] `> 0`). + pub caps: Option, +} + +/// Declare the FRI arenas and hoist everything a query does not depend on. +pub fn declare_fri( + b: &mut LfmBuilder, + shape: FriShape, + num_queries: usize, +) -> (FriArenas, FriCommitments) { + shape.check(); + assert!(num_queries > 0, "a proof carries at least one query"); + let c = shape.num_committed(); + let num_zetas = shape.num_zetas(); + + let roots = b.declare_arena(edsl::digest_words(b) * c as u32); + let zetas = b.declare_arena(num_zetas as u32); + let coeffs = b.declare_arena(shape.num_terminal_coeffs() as u32); + let queries = + b.declare_arena((num_queries * shape.query_words(edsl::digest_words(b) as usize)) as u32); + let cap_words = shape.cap_words(edsl::digest_words(b) as usize); + let caps = (cap_words > 0).then(|| b.declare_arena(cap_words as u32)); + + let mut layers: Vec = (0..c) + .map(|i| LayerCommitment::hint(b, roots, edsl::digest_words(b) * i as u32)) + .collect(); + if let Some(caps) = caps { + hint_layer_caps(b, shape, caps, &mut layers); + } + let zeta_cells: Vec = (0..num_zetas as u32) + .map(|i| b.hint_word(zetas, i).as_ext()) + .collect(); + let coeff_cells: Vec = (0..shape.num_terminal_coeffs() as u32) + .map(|i| b.hint_word(coeffs, i).as_ext()) + .collect(); + + ( + FriArenas { + roots, + zetas, + coeffs, + queries, + caps, + }, + FriCommitments::new(b, shape, layers, zeta_cells, coeff_cells), + ) +} + +/// Hint one query's layer openings out of the query arena. +pub fn hint_layer_openings( + b: &mut LfmBuilder, + shape: FriShape, + arenas: &FriArenas, + query: usize, +) -> Vec { + hint_layer_openings_from(b, shape, arenas.queries, query) +} + +/// [`hint_layer_openings`] against a query arena the caller declared itself. +/// +/// The assembled verifier declares one arena per sub-proof and takes the roots, +/// the folding challenges and the terminal coefficients from the transcript +/// replay rather than from [`declare_fri`]'s three other arenas — so it needs +/// this one without the other three. +pub fn hint_layer_openings_from( + b: &mut LfmBuilder, + shape: FriShape, + arena: ArenaId, + query: usize, +) -> Vec { + let stride = shape.query_words(edsl::digest_words(b) as usize); + let mut cursor = (query * stride) as u32; + let openings: Vec = (0..shape.num_committed()) + .map(|layer| { + let values: Vec = (0..shape.layer_values(layer)) + .map(|_| { + let v = b.hint_word(arena, cursor).as_ext(); + cursor += 1; + v + }) + .collect(); + let siblings: Vec = (0..shape.layer_path_len(layer)) + .map(|_| { + // The stride follows the DIGEST's width, not a literal. + let d = edsl::hint_digest(b, arena, cursor); + cursor += edsl::digest_words(b); + d + }) + .collect(); + LayerOpening { values, siblings } + }) + .collect(); + assert_eq!( + cursor as usize, + (query + 1) * stride, + "the emitter's cursor must agree with the declared query stride" + ); + openings +} + +/// `P(x)` for the terminal polynomial — Horner over the coefficients the proof +/// carries, low-to-high. +/// +/// See [`emit_query_fri`] for why this is an evaluation and not a lookup into a +/// materialized codeword, which is what production does. +fn emit_terminal_eval(b: &mut LfmBuilder, fri: &FriCommitments, x: Felt) -> Ext { + edsl::horner_ext(b, x.as_ext(), &fri.coeffs) +} + +/// Emit one query's FRI verification: fold, authenticate each committed layer, +/// and check the terminal polynomial. +/// +/// Returns the terminal-layer value `v` — the quantity production compares +/// against its terminal codeword — so a caller can publish it. Nothing depends +/// on the caller doing so: the check is `assert_eq_ext` INSIDE the program, so a +/// query that failed would not execute. +/// +/// # The terminal check is an EVALUATION, not a codeword lookup — a deliberate +/// deviation from the spec +/// +/// `others/lfm-fri-verify-spec.md` §5 says to emit the FFT, on the strength of a +/// measurement (sim/24) that replacing production's terminal FFT with per-point +/// Horner cost +20M cycles in the RV64 guest verifier. That measurement is +/// sound and it does not transfer, because the two machines disagree about the +/// price of an array index. +/// +/// Production materializes the terminal codeword once per proof and then does +/// `terminal_codeword.get(index)` per query — one load. This machine is +/// straight-line with no addressable memory, so the same lookup is a `Select` +/// tree over `terminal_len` cells: `terminal_len − 1` `Select`s per query. At +/// blowup 8 (`terminal_len = 1024`, 73 queries) that is 74,679 `Select`s, +/// against which the FFT itself — `(terminal_len/2)·log₂(terminal_len) = 5,120` +/// butterflies at ~3 rows each — is the smaller half of the bill. +/// +/// Evaluating instead costs `2^effective_k − 1 = 127` ext `MulAdd`s per query +/// plus `total_folds` squarings for the point, and no FFT at all: 140 rows per +/// query, 10,220 at 73 queries, against ~90,000. The direction reverses because +/// the guest amortizes one FFT across queries while paying nothing per lookup, +/// and this machine pays nothing for the FFT it does not run and everything for +/// the lookup it cannot do. +/// +/// The two checks are the same check, and the argument is short. The terminal +/// codeword is `P` evaluated over the terminal coset in bit-reversed order +/// (`terminal.rs:134-155`), so position `index` holds +/// `P(terminal_offset · ω_T^{br(index)})`. With `index = iota >> C`, +/// `terminal_offset = coset_offset^(2^total_folds)` and `ω_T = g^(2^total_folds)`, +/// that point is exactly `υ^(2^total_folds)` — the bits of `iota` that survive +/// the shift are the bits `br` puts inside `ω_T`'s order. So the machine raises +/// the query point to `2^total_folds` and evaluates, which also makes the +/// terminal point BOUND to the query point by construction rather than by a +/// second derivation. `fri_tests::the_terminal_point_is_the_query_point_folded` +/// checks that identity against production's own FFT at every index of several +/// shapes; a wrong exponent, a missing coset offset or a dropped bit reversal +/// all fail it. +/// +/// # The zero-fold shape +/// +/// When `total_folds == 0` no challenge was ever drawn and the terminal codeword +/// IS `p₀` (`verifier.rs:683-690`), so the check becomes `terminal[2·iota] = p₀` +/// and `terminal[2·iota+1] = p₀ˢ`. Under evaluation the two branches unify: +/// `2^total_folds = 1`, the two positions are `υ` and `−υ`, and the shape simply +/// evaluates `P` twice instead of once. This is not a dead branch to pin — it is +/// the real proof fixture's own shape (`min` preset over a 2^4-step epoch), and +/// a real production path for any table small enough that its LDE is already +/// terminal. +pub fn emit_query_fri( + b: &mut LfmBuilder, + shape: FriShape, + fri: &FriCommitments, + q: &FriQuery<'_>, + openings: &[LayerOpening], +) -> Ext { + let c = shape.num_committed(); + assert_eq!( + b.wrap_hash().arity(), + shape.arity(), + "the builder hashes the layer trees the shape describes" + ); + assert_eq!( + q.bits.len(), + shape.index_bits(), + "the FRI leg reads suffixes of the trace walk's own decomposition, so \ + it needs all of its index bits (log2(lde) − 1 for row pairs, log2(lde) \ + for one row)" + ); + assert_eq!(fri.layers.len(), c, "one commitment per committed layer"); + assert_eq!(openings.len(), c, "one opening per committed layer"); + for (i, layer) in fri.layers.iter().enumerate() { + assert_eq!( + layer.cap_height(), + shape.layer_cap(i), + "layer {i} is capped at the shape's height" + ); + } + assert_eq!( + fri.coeffs.len(), + shape.num_terminal_coeffs(), + "the terminal polynomial carries 2^effective_k coefficients" + ); + + assert_eq!( + q.p0_sym.is_none(), + shape.one_row(), + "a one-row query opens ONE point, a row-pair query two" + ); + assert_eq!(q.point_sym.is_none(), shape.one_row()); + + if shape.total_folds() == 0 { + assert!( + fri.zetas.is_empty(), + "a codeword that never folds draws no folding challenge" + ); + // One row: the terminal codeword IS the DEEP codeword and + // `terminal[r] == DEEP(x_r)` is the whole check (host + // `verify_query_groups`); row pairs check both points. + let at = emit_terminal_eval(b, fri, q.point); + b.assert_eq_ext(at, q.p0); + if let (Some(p0_sym), Some(point_sym)) = (q.p0_sym, q.point_sym) { + let at_sym = emit_terminal_eval(b, fri, point_sym); + b.assert_eq_ext(at_sym, p0_sym); + } + return q.p0; + } + assert_eq!( + fri.zetas.len(), + shape.num_zetas(), + "folds exceed committed layers by one (row pairs), equal them (one row)" + ); + + // `υ⁻¹`, once. Production batch-inverts across queries and REJECTS on a + // zero point (`verifier.rs:465`, fails closed on a malformed index); the + // machine's `Div` errors on a zero divisor, which is the same disposition — + // an unprovable program rather than a wrong answer. + let one = b.felt_const(FE::one()); + let inv = b.div(one, q.point); + + if shape.one_row() { + // ★ S2: layer 0 IS the committed DEEP + // codeword, so no fold precedes it. The query's value there is + // `DEEP(x_r)` itself and the point's inverse is `x_r⁻¹`; the layer-0 + // slot check of `emit_group_layer` is then the INPUT-SLOT check + // `group₀[slot] == DEEP(x_r)` — the only thing tying the FRI chain to + // the authenticated trace openings (host `verify_query_groups`, M1). + assert_eq!( + fri.zeta_powers.len(), + c, + "the challenge powers are hoisted once per committed layer" + ); + let mut v = q.p0; + let mut y_inv = inv; + for (j, opening) in openings.iter().enumerate() { + (v, y_inv) = emit_group_layer( + b, + shape, + j, + &fri.layers[j], + &fri.zeta_powers[j], + v, + y_inv, + opening, + q.bits, + ); + } + return emit_terminal_check(b, shape, fri, q.point, v); + } + + // Fold 0 consumes the DEEP pair and authenticates nothing: there is no + // layer under it, which is why `zetas` is one longer than `layers`. + let mut v = edsl::fri_fold(b, q.p0, q.p0_sym(), fri.zetas[0], inv); + + // The point chain is one squaring per layer and nothing else — no bit + // reversal, no domain lookup, no coset offset past the first point + // (spec §6). And no parity branch, because the sign the odd slot introduces + // into `x⁻¹` is the same sign it introduces into `v − sym`, so the two + // cancel (spec §3). Parity is consulted ONLY for the leaf byte order below. + if shape.is_legacy() { + let mut inv_pow = inv; + for (i, opening) in openings.iter().enumerate() { + (v, inv_pow) = emit_pair_layer( + b, + i, + &fri.layers[i], + fri.zetas[i + 1], + v, + inv_pow, + opening, + q.bits, + ); + } + } else { + // The group encoding (S3): committed layer `j` opens a whole coset of + // `2^{d_j}` values. `y⁻¹` at committed layer 0 is `υ^{−2}`, and each + // layer hands the next its own point (`x_g^{2^d}`). + assert_eq!( + fri.zeta_powers.len(), + c, + "the challenge powers are hoisted once per committed layer" + ); + let mut y_inv = b.mul(inv, inv); + for (j, opening) in openings.iter().enumerate() { + (v, y_inv) = emit_group_layer( + b, + shape, + j, + &fri.layers[j], + &fri.zeta_powers[j], + v, + y_inv, + opening, + q.bits, + ); + } + } + + emit_terminal_check(b, shape, fri, q.point, v) +} + +/// `x = υ^(2^total_folds)`: where the fold chain has arrived, and the terminal +/// codeword's point at position `iota >> C` (`r >> total_folds` under one row +/// — the same point, since `x_r` IS the query point at layer 0). See +/// [`emit_query_fri`]'s doc comment. Asserts `P(x) == v` and returns `v`. +fn emit_terminal_check( + b: &mut LfmBuilder, + shape: FriShape, + fri: &FriCommitments, + point: Felt, + v: Ext, +) -> Ext { + let mut x = point; + for _ in 0..shape.total_folds() { + x = b.mul(x, x); + } + let at = emit_terminal_eval(b, fri, x); + b.assert_eq_ext(at, v); + v +} + +/// One committed layer under TODAY's pair encoding (`FriShape::is_legacy`): +/// the opening carries the sibling value only. With the query's value `v` at +/// this layer and `x⁻¹` of its point one layer up (`inv_pow`), order the pair +/// by the parity bit `bits[layer]`, hash it as the leaf, authenticate it at the +/// tree's leaf index `bits[layer + 1..]`, square the point and fold with +/// `zeta`. Returns `(v, inv_pow)` at the next layer. The rows it emits are +/// `stark::fri::schedule::fri_pair_layer_rows` (pinned in `fri_group_tests`). +#[allow(clippy::too_many_arguments)] +pub fn emit_pair_layer( + b: &mut LfmBuilder, + layer: usize, + commitment: &LayerCommitment, + zeta: Ext, + v: Ext, + inv_pow: Felt, + opening: &LayerOpening, + bits: &[Bit], +) -> (Ext, Felt) { + assert_eq!(opening.values.len(), 1, "a pair layer opens its sibling"); + let sym = opening.values[0]; + // `if index % 2 == 1 { [sym, v] } else { [v, sym] }` (`verifier.rs:637`) + // — the even codeword slot leads. `select(bit, l, r)` returns `(l, r)` + // at 0 and `(r, l)` at 1, so this IS that conditional. + let (first, second) = b.select(bits[layer], v.as_cell(), sym.as_cell()); + let leaf = sub_proof::emit_leaf_hash(b, FRI_LEAF_GROUP, &[first, second]); + // `bits[layer+1..]` is this layer tree's whole leaf index; a cap walks its + // low bits and muxes the top ones. + commitment.authenticate(b, leaf, &bits[layer + 1..], &opening.siblings); + + // `evaluation_point_vec[i] = υ^(−2^(i+1))` — `inv.square()` then one + // squaring per layer (`verifier.rs:692-697`). + let inv_pow = b.mul(inv_pow, inv_pow); + (edsl::fri_fold(b, v, sym, zeta, inv_pow), inv_pow) +} + +/// The program constants of one group fold of exponent `d`, in +/// the host verifier's own terms (`fri::group::group_fold`, whose table is +/// `ω_{2^d}^t` for `ω_{2^d} = get_primitive_root_of_unity(d)`): +/// +/// - `slot[ℓ] = ω_{2^d}^{2^{d−1−ℓ}}`, so `x_g⁻¹ = y⁻¹·Π_ℓ slot[ℓ]^{s_ℓ} +/// = y⁻¹·ω_{2^d}^{br_d(s)}` for the slot `s` (bits `s_ℓ`, low first); +/// - `kappa[ℓ][j] = ω_{2^d}^{−2^ℓ·br_{d−ℓ−1}(j)}`: fold level `ℓ`'s pair `j` +/// sits at `(X, −X)` with `X⁻¹ = x_g^{−2^ℓ}·kappa[ℓ][j]` (`kappa[ℓ][0] = 1`). +fn group_fold_constants(d: u32) -> (Vec, Vec>) { + use math::fft::bit_reversing::reverse_index; + use math::field::traits::IsFFTField; + + let n = 1usize << d; + let w = ::get_primitive_root_of_unity( + u64::from(d), + ) + .expect("2^d divides the two-adicity for d <= DMAX"); + let pow = |e: usize| w.pow(e as u64); + let slot = (0..d as usize) + .map(|l| pow(1 << (d as usize - 1 - l))) + .collect(); + let kappa = (0..d as usize) + .map(|l| { + let half = n >> (l + 1); + (0..half) + .map(|j| { + let br = if half > 1 { + reverse_index(j, half as u64) + } else { + 0 + }; + pow((n - (br << l)) % n) + }) + .collect() + }) + .collect(); + (slot, kappa) +} + +// The load-bearing test of the slot check (the in-guest M1): a test build can +// emit without it and watch a moved `p₀` execute. Production has no switch. +#[cfg(test)] +thread_local! { + pub(super) static SKIP_SLOT_CHECK: core::cell::Cell = + const { core::cell::Cell::new(false) }; +} + +#[inline] +fn skip_slot_check() -> bool { + #[cfg(test)] + { + SKIP_SLOT_CHECK.with(|c| c.get()) + } + #[cfg(not(test))] + { + false + } +} + +/// `values[slot]` for the slot's bits, LOW first: a balanced mux of +/// `2^d − 1` `Select`s over ext cells, pairs `(2t, 2t + 1)` level by level. +fn emit_value_mux(b: &mut LfmBuilder, values: &[Ext], slot_bits: &[Bit]) -> Ext { + assert_eq!( + values.len(), + 1usize << slot_bits.len(), + "one mux level per slot bit" + ); + let mut level: Vec = values.iter().map(|v| v.as_cell()).collect(); + for bit in slot_bits { + let mut next = Vec::with_capacity(level.len() / 2); + for pair in level.chunks_exact(2) { + next.push(b.select(*bit, pair[0], pair[1]).0); + } + level = next; + } + level[0].as_ext() +} + +/// ★ One committed layer under the group encoding (S3). +/// +/// With `d = d_j`, `G = G_j`, the query's bits `bits` (low first, all +/// `index_bits`), its value `v` at this layer (the previous fold's output) and +/// the inverse `y⁻¹` of its point here: +/// +/// 1. **slot check** — `values[bits[G..G+d]] == v`, a `2^d − 1`-select mux and +/// an `assert_eq_ext`: the round-consistency check tying the opened group +/// to the value the previous fold produced (M1 on the host); +/// 2. **the group is the leaf** — hashed in full, position order (a +/// `GroupShape` of `2^{d−1}` ext columns covers `2^d` values), and +/// authenticated at the tree's leaf index `bits[G+d..]` against the +/// layer's root or cap; +/// 3. **the group fold** with `ζ, ζ², …, ζ^{2^{d−1}}`: `x_g⁻¹ = y⁻¹·ω_{2^d}^{br_d(s)}` +/// (`d` selects of constants and `d` base muls), then `d` levels of +/// `fri_fold` over the pairs, the level's `x_g^{−2^ℓ}` squared once per +/// level. A level with more than two pairs folds `ζ^{2^ℓ}·x_g^{−2^ℓ}` into +/// the challenge once (one `emul_base`) and multiplies each pair by its +/// constant inside the fold; a level with one or two pairs multiplies the +/// point instead (at most one base mul). After `d` levels the point is +/// `x_g^{−2^d}`, the NEXT layer's `y⁻¹`. +/// +/// Returns `(v, y⁻¹)` at the next layer. +#[allow(clippy::too_many_arguments)] +pub fn emit_group_layer( + b: &mut LfmBuilder, + shape: FriShape, + layer: usize, + commitment: &LayerCommitment, + zeta_powers: &[Ext], + v: Ext, + y_inv: Felt, + opening: &LayerOpening, + bits: &[Bit], +) -> (Ext, Felt) { + let d = shape.layer_fold(layer); + let g = shape.layer_bit_offset(layer); + let n = 1usize << d; + assert_eq!(opening.values.len(), n, "a group layer opens 2^d values"); + assert_eq!( + zeta_powers.len(), + d as usize, + "one challenge power per fold level" + ); + assert_eq!( + bits.len() - (g + d as usize), + shape.layer_depth(layer), + "the tree's leaf index is what is left of the query after the slot" + ); + let slot_bits = &bits[g..g + d as usize]; + + // (1) the slot check. + let v_slot = emit_value_mux(b, &opening.values, slot_bits); + if !skip_slot_check() { + b.assert_eq_ext(v_slot, v); + } + + // (2) the group is the leaf. + let cells: Vec = opening.values.iter().map(|x| x.as_cell()).collect(); + let leaf = sub_proof::emit_leaf_hash( + b, + GroupShape { + num_columns: n / 2, + is_ext: true, + }, + &cells, + ); + commitment.authenticate(b, leaf, &bits[g + d as usize..], &opening.siblings); + + // (3) the group fold. + let (slot_factors, kappa) = group_fold_constants(d); + let mut xinv = y_inv; + for (bit, factor) in slot_bits.iter().zip(&slot_factors) { + let one = b.felt_const(FE::one()); + let f = b.felt_const(*factor); + let (chosen, _) = b.select(*bit, one.as_cell(), f.as_cell()); + xinv = b.mul(xinv, Felt(chosen.0)); + } + let mut vals = opening.values.clone(); + for (l, zeta) in zeta_powers.iter().enumerate() { + let half = vals.len() / 2; + let scaled = (half > 2).then(|| b.emul_base(*zeta, xinv)); + let mut next = Vec::with_capacity(half); + for j in 0..half { + let (lo, hi) = (vals[2 * j], vals[2 * j + 1]); + let folded = match scaled { + Some(zx) => { + let k = b.felt_const(kappa[l][j]); + edsl::fri_fold(b, lo, hi, zx, k) + } + None => { + let x = if j == 0 { + xinv + } else { + let k = b.felt_const(kappa[l][j]); + b.mul(xinv, k) + }; + edsl::fri_fold(b, lo, hi, *zeta, x) + } + }; + next.push(folded); + } + vals = next; + xinv = b.mul(xinv, xinv); + } + (vals[0], xinv) +} + +/// A whole sub-proof, both legs: every query's openings authenticated and folded +/// to `p₀` ([`super::sub_proof::emit_sub_proof_with_bits`]), then that `p₀` +/// folded down FRI's layers to the terminal check. +/// +/// This is where the two legs become one program rather than two. Every seam is +/// a shared CELL, not a shared convention: `p₀`/`p₀ˢ` are the DEEP outputs, `υ` +/// is the point they were evaluated at, and the index bits are the ones the +/// trace walk selected on. Returns the per-query terminal values. +pub fn emit_sub_proof_with_fri( + b: &mut LfmBuilder, + sub: &super::sub_proof::SubProofShape, + shape: FriShape, + num_queries: usize, +) -> (super::sub_proof::SubProofArenas, FriArenas, Vec) { + assert_eq!( + sub.log2_lde_length, shape.log2_lde_length, + "both legs verify the same sub-proof over the same LDE domain" + ); + assert_eq!( + sub.merkle_depth, + shape.index_bits(), + "the FRI layers consume suffixes of the trace walk's decomposition, so \ + the two shapes must agree about how long it is" + ); + assert_eq!( + shape.num_queries, num_queries, + "the query count is one shape, declared once" + ); + assert_eq!( + sub.layout, + shape.leaf_layout(), + "both legs verify one table at one leaf layout" + ); + + let (sub_arenas, queries) = super::sub_proof::emit_sub_proof_with_bits(b, sub, num_queries); + let (fri_arenas, fri) = declare_fri(b, shape, num_queries); + + let terminal = queries + .iter() + .enumerate() + .map(|(i, out)| { + let openings = hint_layer_openings(b, shape, &fri_arenas, i); + emit_query_fri( + b, + shape, + &fri, + &FriQuery { + p0: out.deep, + p0_sym: out.deep_sym, + point: out.point, + point_sym: out.point_sym, + bits: &out.bits, + }, + &openings, + ) + }) + .collect(); + + (sub_arenas, fri_arenas, terminal) +} diff --git a/prover/src/lfm/fri_group_tests.rs b/prover/src/lfm/fri_group_tests.rs new file mode 100644 index 000000000..0a5583813 --- /dev/null +++ b/prover/src/lfm/fri_group_tests.rs @@ -0,0 +1,770 @@ +//! S3 in the in-guest FRI verifier: the shape from the shared schedule (G1) +//! and the group-layer emitter (G2). +//! +//! Checked against the host's own artefacts, never against a second model: +//! - the in-guest shape (schedule, layer depths, caps) against the host's +//! `StarkCaps::for_options` / `FriFormat::schedule` over a sweep of shapes; +//! - the emitted verifier against the host's checked-in RPX vectors +//! (`crypto/stark/tests/vectors/zf_fri/d_proof_rpx_*`: pair, dp, the uneven +//! `[3, 1, 3]` override, and the two capped Q = 20 formats), +//! executed, with its permutation count equal to the closed form; +//! - tampers of every value a group opening carries, and the slot check shown +//! load-bearing (a moved `p₀` executes when, and only when, it is skipped); +//! - the {cap off, auto} × {pair, dp, uneven dp} round-trip matrix on a real +//! laptop-scale proof, both legs as one program; +//! - every row the emitter emits per FRI layer (group and +//! pair, capped and uncapped) equals the DP's model (`stark::fri::schedule`), +//! and a DEEP point's rows equal the S2 `auto` rule's DEEP term. + +use crypto::merkle_tree::cap::CapPolicy; +use serde_json::Value; +use stark::examples::read_only_memory_logup::LogReadOnlyPublicInputs; +use stark::fri::schedule::{ + FRI_COST_WEIGHTS, FriLayerRows, fri_group_layer_rows, fri_layer_cost_q, fri_pair_layer_cost_q, + fri_pair_layer_rows, +}; +use stark::leaf_layout::deep_point_xalu_rows; +use stark::merkle_caps::StarkCaps; +use stark::proof::options::{FriMode, FriScheduleOverride, ProofFormat, ProofOptions}; +use stark::proof::stark::StarkProof; +use stark::proof::view::StarkProofView; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::Felt; +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::deep::{DeepInvariants, DeepOpening, DeepShape, emit_deep_point}; +use super::executor::execute; +use super::fri::{FriShape, LayerCommitment, LayerOpening, emit_group_layer, emit_pair_layer}; +use super::fri_tests::{folding_fixture_with, fri_only_program, host_fri_from, permutations}; +use super::instr::Instr; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type VectorProof = StarkProof>; + +// ============================================================================= +// G1 — the in-guest shape IS the host's layout +// ============================================================================= + +/// ★ One schedule, one depth, one cap per layer, on both sides: the in-guest +/// `FriShape` (the emitter's program shape) against the host's `StarkCaps` — +/// the function the prover embeds caps with and the verifier checks them with, +/// itself built on the host's `FriFoldLayout` — over every LDE size of +/// interest, both terminals in production (T = 9 base legs, T = 10 LFM +/// proofs), both FRI modes and both cap policies. +#[test] +fn the_in_guest_fri_shape_is_the_hosts_layout() { + let mut checked = 0usize; + for (blowup, k) in [(4u8, 7u8), (4, 8), (2, 7)] { + for queries in [3usize, 24, 110] { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + for fri in [FriMode::Pair, FriMode::Dp] { + let blowup_log = (blowup as u32).trailing_zeros(); + for lde_log in (blowup_log + 1)..=25 { + let opts = ProofOptions { + blowup_factor: blowup, + fri_number_of_queries: queries, + coset_offset: 3, + grinding_factor: 0, + fri_final_poly_log_degree: k, + format: ProofFormat { + merkle_cap: cap, + fri_mode: fri, + ..ProofFormat::DEFAULT + }, + }; + let shape = FriShape::from_options(&opts, lde_log); + shape.check(); + let host = StarkCaps::for_options(&opts, lde_log as usize, false) + .expect("a row-pair format lays out"); + let depths: Vec = (0..shape.num_committed()) + .map(|j| shape.layer_depth(j)) + .collect(); + let caps: Vec = (0..shape.num_committed()) + .map(|j| shape.layer_cap(j)) + .collect(); + assert_eq!(depths, host.fri_depths, "{opts:?} lde {lde_log}"); + assert_eq!(caps, host.fri, "{opts:?} lde {lde_log}"); + assert_eq!(shape.index_bits(), host.trace_depth); + assert_eq!(shape.is_legacy(), fri == FriMode::Pair); + checked += 1; + } + } + } + } + } + println!("{checked} shapes: in-guest schedule, depths and caps == the host's"); +} + +// ============================================================================= +// G2 — the emitted verifier on the host's RPX vectors +// ============================================================================= + +fn ext_of(v: &Value) -> FEE { + let limbs: Vec = v + .as_array() + .expect("an ext value is three limbs") + .iter() + .map(|x| x.as_u64().expect("a canonical limb")) + .collect(); + assert_eq!(limbs.len(), 3); + FEE::new([FE::from(limbs[0]), FE::from(limbs[1]), FE::from(limbs[2])]) +} + +/// One checked-in RPX (d) vector: its JSON, its proof, and the in-guest shape +/// the emitter builds for it from the vector's FORMAT (the host generator's +/// own `proof_formats`, never re-spelled here). +struct Vector { + name: &'static str, + json: Value, + proof: VectorProof, + shape: FriShape, +} + +fn rpx_vectors() -> Vec { + stark::fri::vectors::proof_formats() + .into_iter() + .map(|(name, format, queries)| { + let dir = stark::fri::vectors::vectors_dir(); + let stem = format!("d_proof_rpx_{name}"); + let json: Value = serde_json::from_slice( + &std::fs::read(dir.join(format!("{stem}.json"))).expect("the vector JSON"), + ) + .expect("valid JSON"); + let bytes = std::fs::read(dir.join(format!("{stem}.rkyv"))).expect("the vector proof"); + let proof: VectorProof = + rkyv::from_bytes::(&bytes).expect("rkyv"); + let opts = stark::fri::vectors::proof_options(format, queries); + let lde_log = json["lde_log"].as_u64().expect("lde_log") as u32; + let shape = FriShape::from_options(&opts, lde_log); + Vector { + name, + json, + proof, + shape, + } + }) + .collect() +} + +impl Vector { + /// The arenas [`fri_only_program`] declares: `(ι, p₀(υ), p₀(−υ))` per + /// query, then the roots, the ζs, the terminal coefficients, the per-query + /// layer openings and (when capped) the caps. + fn arenas(&self) -> Vec> { + let queries = self.json["queries_detail"].as_array().expect("queries"); + let mut deep = Vec::new(); + for q in queries { + deep.push(base_word(FE::from(q["iota"].as_u64().expect("iota")))); + deep.push(ext_word(&ext_of(&q["deep"]))); + deep.push(ext_word(&ext_of(&q["deep_sym"]))); + } + let view = StarkProofView::Owned(&self.proof); + let (openings, caps) = super::epoch_verify_tests::fri_layer_openings(view, self.shape); + let mut per_query = Vec::new(); + for query in &openings { + for (values, path) in query { + per_query.extend(values.iter().map(ext_word)); + per_query.extend(super::proof_arena::commitments_to_arena(path)); + } + } + let zetas: Vec = self.json["zetas"] + .as_array() + .expect("zetas") + .iter() + .map(|z| ext_word(&ext_of(z))) + .collect(); + let mut out = vec![ + deep, + super::proof_arena::commitments_to_arena(&self.proof.fri_layers_merkle_roots), + zetas, + self.proof + .fri_final_poly_coeffs + .iter() + .map(ext_word) + .collect(), + per_query, + ]; + if self.shape.cap_words(super::proof_arena::words_per_root()) > 0 { + out.push(super::proof_arena::commitments_to_arena(&caps)); + } + out + } + + fn program(&self) -> LfmProgram { + fri_only_program(self.shape, self.shape.num_queries) + } +} + +/// ★ The emitted FRI verifier accepts every RPX (d) vector — today's pair +/// proof, the DP schedule, the uneven `[3, 1, 3]` override (the only shape +/// that catches a fold-count off-by-one) and both capped Q = 20 +/// formats — with the vector's schedule, depths and caps derived by the +/// emitter's own shape, and the permutation count exactly the closed form. +#[test] +fn the_emitted_fri_verifier_accepts_every_rpx_vector() { + for v in rpx_vectors() { + let s = v.shape; + s.check(); + let schedule: Vec = v.json["schedule"] + .as_array() + .expect("schedule") + .iter() + .map(|d| d.as_u64().expect("d") as u8) + .collect(); + assert_eq!(s.schedule(), schedule, "{}: the schedule", v.name); + assert_eq!( + s.is_legacy(), + v.json["legacy_encoding"] + .as_bool() + .expect("legacy_encoding"), + "{}", + v.name + ); + if let Some(caps) = v.json.get("fri_caps") { + let want: Vec = caps + .as_array() + .expect("fri_caps") + .iter() + .map(|c| c.as_u64().expect("c") as usize) + .collect(); + let depths: Vec = v.json["fri_tree_depths"] + .as_array() + .expect("depths") + .iter() + .map(|c| c.as_u64().expect("d") as usize) + .collect(); + assert_eq!( + (0..s.num_committed()) + .map(|j| s.layer_cap(j)) + .collect::>(), + want, + "{}: caps", + v.name + ); + assert_eq!( + (0..s.num_committed()) + .map(|j| s.layer_depth(j)) + .collect::>(), + depths, + "{}: depths", + v.name + ); + } + let program = v.program(); + let exec = execute(&program, &v.arenas(), &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("{}: the honest vector must execute: {e:?}", v.name)); + assert_eq!(exec.public_words.len(), s.num_queries); + let closed = s.num_queries * s.permutations_per_query() + s.cap_permutations(); + assert_eq!( + permutations(&program), + closed, + "{}: emitted permutations against the closed form", + v.name + ); + println!( + "{:<9} Q={:<2} schedule {:?} caps {:?}: {} permutations, {} instructions", + v.name, + s.num_queries, + s.schedule(), + (0..s.num_committed()) + .map(|j| s.layer_cap(j)) + .collect::>(), + closed, + program.instrs.len() + ); + } +} + +/// ★ Every value a group opening carries is bound: the slot value, a non-slot +/// value, the last value of the group, a sibling, a cap word, a folding +/// challenge, a terminal coefficient, and the DEEP value the first slot check +/// compares against. Run on the uneven override and on the capped DP vector. +#[test] +fn no_tampered_group_opening_value_can_pass() { + for v in rpx_vectors() { + if !matches!(v.name, "dp_3_1_3" | "cap_dp") { + continue; + } + let program = v.program(); + let honest = v.arenas(); + execute(&program, &honest, &crate::hash_pin::BLOCK_HASHER).expect("honest"); + let q0 = &v.json["queries_detail"][0]["layers"][0]; + let slot = q0["slot"].as_u64().expect("slot") as usize; + let d0 = 1usize << v.shape.layer_fold(0); + let other = (slot + 1) % d0; + // Arenas: deep, roots, zetas, coeffs, queries[, caps]. + let mut bump: Vec<(String, usize, usize)> = vec![ + ("p0 (the DEEP value)".into(), 0, 1), + ("zeta_1".into(), 2, 1), + ("terminal coefficient 0".into(), 3, 0), + (format!("query 0 layer 0 slot value (slot {slot})"), 4, slot), + (format!("query 0 layer 0 non-slot value {other}"), 4, other), + ("query 0 layer 0 last group value".into(), 4, d0 - 1), + ("query 0 layer 0 first sibling".into(), 4, d0), + ]; + if honest.len() == 6 { + bump.push(("cap word 0".into(), 5, 0)); + bump.push(("last cap word".into(), 5, honest[5].len() - 1)); + } + for (label, arena, word) in bump { + let mut bad = honest.clone(); + bad[arena][word][0] += FE::one(); + execute(&program, &bad, &crate::hash_pin::BLOCK_HASHER).expect_err(&format!( + "{}: moving {label} must make the program unexecutable", + v.name + )); + } + } +} + +/// ★ The slot check is LOAD-BEARING (the in-guest M1). Under the group +/// encoding the value the first fold produces from the DEEP pair meets the +/// committed layers ONLY at the slot check: the leaf hashes the group, the +/// walk authenticates it, the group fold reads it. So a moved `p₀(υ)` is +/// refused with the check and ACCEPTED without it — which is exactly a +/// verifier that would accept FRI for a different codeword than the trace +/// openings commit to. +#[test] +fn the_slot_check_is_load_bearing() { + let v = rpx_vectors() + .into_iter() + .find(|v| v.name == "cap_dp") + .expect("the capped dp vector"); + let honest = v.arenas(); + let mut moved = honest.clone(); + moved[0][1][0] += FE::one(); + + let with = v.program(); + execute(&with, &honest, &crate::hash_pin::BLOCK_HASHER).expect("honest"); + execute(&with, &moved, &crate::hash_pin::BLOCK_HASHER) + .expect_err("a moved p0 must be refused by the slot check"); + + super::fri::SKIP_SLOT_CHECK.with(|c| c.set(true)); + let without = v.program(); + super::fri::SKIP_SLOT_CHECK.with(|c| c.set(false)); + execute(&without, &moved, &crate::hash_pin::BLOCK_HASHER) + .expect("WITHOUT the slot check a moved p0 is accepted — the check is the only binding"); +} + +// ============================================================================= +// The {cap} × {fri} round-trip matrix, both legs, on a real proof +// ============================================================================= + +/// ★ The cap × FRI matrix on a real laptop-scale proof (L2G_MEMORY, 2048 +/// rows, blowup 2, `k = 2` so the committed chain covers 11 → 3, Q = 24): +/// {cap off, auto} × {pair, dp, dp `[3, 1, 4]`}. Per cell the FRI leg alone +/// and both legs as one program execute over every query, reach the terminal +/// codeword production computed, and emit exactly the closed form. +#[test] +fn the_cap_and_fri_matrix_round_trips_in_guest() { + use super::epoch_verify::{blocks_for, group_leaf_felts}; + + let hash = super::edsl::WrapHash::production(); + for cap in [CapPolicy::Off, CapPolicy::Auto] { + for (label, fri, over) in [ + ("pair", FriMode::Pair, None), + ("dp", FriMode::Dp, None), + ("dp [3,1,4]", FriMode::Dp, Some(&[3u8, 1, 4][..])), + ] { + let mut opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2"); + opts.fri_number_of_queries = 24; + opts.grinding_factor = 0; + opts.fri_final_poly_log_degree = 2; + opts.format = ProofFormat { + merkle_cap: cap, + fri_mode: fri, + fri_schedule_override: over.and_then(FriScheduleOverride::new), + ..ProofFormat::DEFAULT + }; + let (air, proof) = folding_fixture_with(2048, opts); + let h = host_fri_from(&*air, &proof); + let s = h.shape; + let all: Vec = (0..h.trace.iotas.len()).collect(); + let codeword = h.terminal_codeword(); + let position = |iota: usize| iota >> (s.total_folds() - 1); + + // The FRI leg alone. + let program = fri_only_program(s, all.len()); + let exec = execute( + &program, + &h.all_arenas(&all), + &crate::hash_pin::BLOCK_HASHER, + ) + .unwrap_or_else(|e| panic!("cap={cap} fri={label}: FRI leg: {e:?}")); + for (k, &q) in all.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!( + v, + codeword[position(h.trace.iotas[q])], + "cap={cap} fri={label}" + ); + } + assert_eq!( + permutations(&program), + all.len() * s.permutations_per_query() + s.cap_permutations(), + "cap={cap} fri={label}: FRI leg closed form" + ); + + // Both legs as one program. + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let (_, _, terminal) = + super::fri::emit_sub_proof_with_fri(&mut b, &h.trace.shape, s, all.len()); + for t in &terminal { + b.public(t.as_cell()); + } + let joined = compile(b.finish()); + let mut arenas = h.trace.arenas(&all); + arenas.extend(h.fri_arenas(&all)); + let exec = execute(&joined, &arenas, &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("cap={cap} fri={label}: joined: {e:?}")); + for (k, &q) in all.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!(v, codeword[position(h.trace.iotas[q])]); + } + let sub = &h.trace.shape; + let leaves: usize = sub + .groups() + .iter() + .map(|g| blocks_for(group_leaf_felts(g), hash)) + .sum(); + let closed = all.len() + * (leaves + sub.groups().len() * sub.path_len() + s.permutations_per_query()) + + sub.cap_permutations() + + s.cap_permutations(); + assert_eq!( + permutations(&joined), + closed, + "cap={cap} fri={label}: both legs' closed form" + ); + println!( + "cap={cap:<4} fri={label:<10} schedule {:?} FRI caps {:?} trace cap {}: FRI leg \ + {} perms, both legs {} perms / {} instructions", + s.schedule(), + (0..s.num_committed()) + .map(|j| s.layer_cap(j)) + .collect::>(), + sub.trace_cap, + permutations(&program), + closed, + joined.instrs.len(), + ); + } + } +} + +// ============================================================================= +// Every emitted row per FRI layer and per DEEP point, against +// the host's cost model (`stark::fri::schedule`, `stark::leaf_layout`) +// ============================================================================= + +/// The kinds of every instruction a program emits: `(selects, XALU, BALU, +/// hashes, unpacks, packs, hints, other)`. +fn count_kinds(instrs: &[Instr]) -> [usize; 8] { + let mut k = [0usize; 8]; + for i in instrs { + let slot = match i { + Instr::Select { .. } => 0, + Instr::ExtAlu { .. } => 1, + Instr::BaseAlu { .. } => 2, + Instr::Hash { .. } => 3, + Instr::Unpack { .. } => 4, + Instr::Pack { .. } => 5, + Instr::Hint { .. } => 6, + _ => 7, + }; + k[slot] += 1; + } + k +} + +/// The rows `emit(times)` adds per repetition: the program is built at +/// `times = 1` and `times = 2` over the same hinted inputs, and the difference +/// is one repetition's rows — interned program constants and one-time setup +/// (the index decomposition, the root's unpack, the cap's hints and root +/// check) fall out of it. Asserts the repetition emits nothing but the priced +/// row kinds. +fn rows_of_one(emit: &dyn Fn(usize) -> LfmProgram) -> FriLayerRows { + let (once, twice) = (emit(1), emit(2)); + let (a, b) = (count_kinds(&once.instrs), count_kinds(&twice.instrs)); + let d: Vec = (0..8).map(|i| (b[i] - a[i]) as u64).collect(); + assert_eq!(d[7], 0, "a repetition emits only priced row kinds"); + assert_eq!( + (twice.instrs.len() - once.instrs.len()) as u64, + d.iter().sum::(), + "every instruction is counted" + ); + FriLayerRows { + selects: d[0], + xalu: d[1], + balu: d[2], + hashes: d[3], + unpacks: d[4], + packs: d[5], + hints: d[6], + } +} + +/// Tree depth of the measured layers. +const MEASURED_DEPTH: usize = 2; + +/// A program emitting `times` openings of one committed FRI layer (group +/// layer of exponent `d`, or today's pair layer when `d == 0`) over a +/// `MEASURED_DEPTH`-level tree capped at `c`, every shared input hinted before +/// the first opening. Every opening's values and siblings are hinted in the +/// loop (as `hint_layer_openings` does), so they count. +fn fri_layer_program(d: u32, c: usize, times: usize) -> LfmProgram { + let pair = d == 0; + let fold = if pair { 1 } else { d }; + let shape = FriShape { + log2_lde_length: fold + MEASURED_DEPTH as u32 + 1, + blowup_log: 1, + final_poly_log_degree: 1, + coset_offset: 3, + num_queries: 1, + format: if pair { + ProofFormat::DEFAULT + } else { + ProofFormat { + fri_mode: FriMode::Dp, + fri_schedule_override: FriScheduleOverride::new(&[d as u8]), + ..ProofFormat::DEFAULT + } + }, + trees: stark::config::TreeGeometry::BINARY, + }; + shape.check(); + assert_eq!(shape.is_legacy(), pair); + assert_eq!(shape.schedule(), vec![fold as u8]); + assert_eq!(shape.layer_depth(0), MEASURED_DEPTH); + + let n = if pair { 1 } else { 1usize << d }; + let num_siblings = MEASURED_DEPTH - c; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + assert_eq!( + super::edsl::digest_words(&b), + 1, + "the model prices the production one-cell digest" + ); + let arena = b.declare_arena((4 + fold as usize + (1 << c) + times * (n + num_siblings)) as u32); + let root = b.hint_word(arena, 0); + let mut commitment = LayerCommitment::from_lanes(vec![b.unpack(root)]); + let v = b.hint_word(arena, 1).as_ext(); + let y_inv = b.hint_felt(arena, 2); + let index = b.hint_felt(arena, 3); + let bits = b.bit_dec(index, shape.index_bits()); + let zetas: Vec<_> = (0..fold) + .map(|i| b.hint_word(arena, 4 + i).as_ext()) + .collect(); + let mut at = commitment.hint_cap(&mut b, arena, 4 + fold, c, shape.layer_depth(0)); + for _ in 0..times { + let values = (0..n) + .map(|_| { + at += 1; + b.hint_word(arena, at - 1).as_ext() + }) + .collect(); + let siblings = (0..num_siblings) + .map(|_| { + at += 1; + super::edsl::WrapDigest::from_cell(b.hint_word(arena, at - 1)) + }) + .collect(); + let opening = LayerOpening { values, siblings }; + if pair { + emit_pair_layer(&mut b, 0, &commitment, zetas[0], v, y_inv, &opening, &bits); + } else { + emit_group_layer( + &mut b, + shape, + 0, + &commitment, + &zetas, + v, + y_inv, + &opening, + &bits, + ); + } + } + compile(b.finish()) +} + +/// ★ The rows one query's opening of a committed FRI layer +/// emits in-guest EQUAL the host model's, kind by kind and in total, for group +/// layers `d = 1..=6` and today's pair layer, uncapped and capped (`c = 1, 2` +/// on a two-level tree): +/// +/// ```text +/// group d, depth D, cap c (stark::fri::schedule::fri_group_layer_rows): +/// selects (2^d − 1) slot mux + (D − c) walk + (2^c − 1) cap mux + d x_g +/// XALU 5·(2^d − 1) folds + 2 slot assert + max(0, d − 2) scaling +/// BALU d twiddles + d x_g + [d ≥ 2] scaling + 8 root compare +/// hashes leaf(d) + (D − c) +/// unpacks 2^d values + 1 walked digest + [c ≥ 1] cap node +/// packs ⌈3·2^d / 4⌉ leaf words +/// hints 2^d values + (D − c) siblings +/// pair, depth D, cap c (fri_pair_layer_rows): +/// selects 1 + (D − c) + (2^c − 1), XALU 5, BALU 1 + 8, hashes 1 + (D − c), +/// unpacks 2 + 1 + [c ≥ 1], packs 2, hints 1 + (D − c) +/// ``` +/// +/// The DP prices exactly these rows: `fri_layer_cost_q` is the uncapped rows +/// minus the cap's gain and the `c` sibling hints it removes, checked here +/// against the capped rows priced directly plus the cap's per-tree cost. A +/// change to the emitter that is not also a change to the model fails here. +#[test] +fn every_emitted_fri_row_is_priced() { + let w = FRI_COST_WEIGHTS; + println!("\n layer c | sel XALU BALU hash unpack pack hint | ns/query"); + for c in 0..=2usize { + for d in 0..=6u32 { + let got = rows_of_one(&|times| fri_layer_program(d, c, times)); + let (label, model) = if d == 0 { + ( + "pair".to_string(), + fri_pair_layer_rows(MEASURED_DEPTH as u32, c as u32), + ) + } else { + ( + format!("d={d}"), + fri_group_layer_rows(d, MEASURED_DEPTH as u32, c as u32), + ) + }; + assert_eq!(got, model, "{label} c={c}: emitted rows == model rows"); + println!( + " {label:>5} {c} | {:>3} {:>4} {:>4} {:>4} {:>6} {:>4} {:>4} | {:>8}", + got.selects, + got.xalu, + got.balu, + got.hashes, + got.unpacks, + got.packs, + got.hints, + got.price(&w) + ); + } + } + + // The DP's per-layer price is these rows: uncapped exactly; capped, the + // capped rows plus the cap's once-per-tree cost (`cap_gain`'s per-tree + // term: 2^c − 1 compressions, 2^c hints, one compare). + let depth = 10u32; + for q in [1u64, 20, 110] { + for cap in [ + CapPolicy::Off, + CapPolicy::Fixed(1), + CapPolicy::Fixed(3), + CapPolicy::Auto, + ] { + let c = cap.height(q as usize, depth as usize) as u32; + let per_tree = if c == 0 { + 0 + } else { + ((1u64 << c) - 1) * w.cap.compress + (1u64 << c) * w.cap.hint + w.cap.compare + }; + for d in 1..=6u32 { + let direct = q * fri_group_layer_rows(d, depth, c).price(&w) + per_tree; + assert_eq!( + fri_layer_cost_q(&w, d, depth, q, cap), + direct, + "group d={d} q={q} cap={cap:?}" + ); + } + let direct = q * fri_pair_layer_rows(depth, c).price(&w) + per_tree; + assert_eq!( + fri_pair_layer_cost_q(&w, depth, q, cap), + direct, + "pair q={q} cap={cap:?}" + ); + } + } +} + +/// A program emitting `times` DEEP points of `shape` over hinted openings and +/// hinted invariants (the invariants hinted before the first point). +fn deep_point_program(shape: &DeepShape, times: usize) -> LfmProgram { + let e = shape.num_eval_points; + let cols = shape.num_total_cols; + let parts = shape.num_composition_parts; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena((4 * e + 4 + times * (1 + cols + parts)) as u32); + let mut at = 0u32; + let mut next = |b: &mut LfmBuilder| { + at += 1; + b.hint_word(arena, at - 1).as_ext() + }; + let gamma = next(&mut b); + let inv = DeepInvariants { + ood_row_sum: (0..e).map(|_| next(&mut b)).collect(), + h_sum_zpow: next(&mut b), + z_pow: next(&mut b), + row_points: (0..e).map(|_| next(&mut b)).collect(), + gamma_pow_surviving: next(&mut b), + gamma_pow_block: (0..e).map(|_| next(&mut b)).collect(), + gamma_stride: (0..e).map(|_| next(&mut b)).collect(), + commit_ood: None, + }; + for _ in 0..times { + let point = Felt(next(&mut b).as_cell().0); + let opening = DeepOpening { + point, + trace: (0..cols).map(|_| next(&mut b)).collect(), + parts: (0..parts).map(|_| next(&mut b)).collect(), + }; + emit_deep_point(&mut b, shape, gamma, &inv, &opening); + } + compile(b.finish()) +} + +/// ★ The XALU rows of ONE in-guest DEEP point EQUAL the S2 `auto` +/// rule's DEEP term (`stark::leaf_layout::deep_point_xalu_rows`: +/// `num_surviving + 4·E + P + 3`), over shapes with and without a next row, +/// a widened step, and one or many composition parts. DEEP emits no other +/// row kind; the point's hinted inputs here stand in for the cells the trace +/// walk already authenticated. +#[test] +fn the_deep_point_rows_are_the_auto_rules_deep_term() { + let shapes = [ + // (step, offsets, cols, next-row cols, parts) + (1usize, 1usize, 7usize, vec![], 1usize), + (1, 2, 5, vec![1, 3], 2), + (1, 2, 40, vec![0, 5, 39], 3), + (2, 2, 6, vec![2], 2), + (1, 3, 9, vec![0, 8], 4), + ]; + for (step, offsets, cols, next_cols, parts) in shapes { + let shape = DeepShape { + step_size: step, + num_eval_points: offsets * step, + num_total_cols: cols, + next_row_cols: next_cols.clone(), + num_composition_parts: parts, + log2_trace_length: 8, + commit_ood: None, + }; + let got = rows_of_one(&|times| deep_point_program(&shape, times)); + let want = deep_point_xalu_rows( + shape.num_surviving() as u64, + shape.num_eval_points as u64, + parts as u64, + ); + let ctx = + format!("step {step} offsets {offsets} cols {cols} next {next_cols:?} parts {parts}"); + assert_eq!(got.xalu, want, "{ctx}: DEEP XALU rows"); + assert_eq!( + (got.selects, got.balu, got.hashes, got.unpacks, got.packs), + (0, 0, 0, 0, 0), + "{ctx}: DEEP emits only XALU rows" + ); + assert_eq!( + got.hints as usize, + 1 + cols + parts, + "{ctx}: the stand-in hints" + ); + } +} diff --git a/prover/src/lfm/fri_tests.rs b/prover/src/lfm/fri_tests.rs new file mode 100644 index 000000000..cdfefe902 --- /dev/null +++ b/prover/src/lfm/fri_tests.rs @@ -0,0 +1,1493 @@ +//! The FRI leg: the per-layer walk, the fold chain, and the terminal check. +//! +//! ## The instrument problem this suite had to solve first +//! +//! `join_tests::the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold` +//! pins the difficulty: the join fixture's sub-proof has `total_folds = 0`, so a +//! differential over it sees no fold, no per-layer walk and no terminal lookup. +//! The retiring FRI agent concluded the only witness was synthetic codewords +//! driven through production's commit and query phases. +//! +//! It is better than that, and the reason is one line of the fixture: the trace +//! is `boundaries.len().next_power_of_two()` rows +//! (`local_to_global.rs:269`). Ask for 512 boundaries instead of 4 and the same +//! production prover, the same AIR and the same verifier replay produce a proof +//! that FOLDS — real committed layer roots, real authentication paths, real +//! terminal coefficients, and the folding challenges out of production's own +//! `replay_rounds_after_round_1`. Nothing in this suite is synthetic. The layer +//! count is swept by asking for more rows. +//! +//! ## What this suite cannot see +//! +//! `k = 7` and `coset_offset = 3` in every configuration the prover can be +//! asked for, so — exactly as spec §7 says — nothing here distinguishes an +//! implementation that reads them from one that hardcodes them. That half is +//! discharged host-side by `join_tests::the_fold_layout_is_right_off_productions_constants`, +//! which sweeps `k ∈ {0, 6, 7, 63}` and the clamp regime over the shape +//! arithmetic. What IS now witnessed on real data is everything the shape feeds: +//! the `num_committed = total_folds − 1` off-by-one, the fold chain, the parity +//! branch, the walk depths and the terminal check. +//! +//! It also sees one sub-proof at a time. Nothing here says an epoch's sub-proofs +//! compose, and nothing here binds the terminal coefficients or the folding +//! challenges to a transcript — they arrive as arena values, and tying them to a +//! replay is assembly's obligation. + +use crypto::merkle_tree::cap::CapPolicy; +use math::field::traits::IsPrimeField; +use math::polynomial::Polynomial; +use stark::config::Commitment; +use stark::proof::stark::MultiProof; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::constraint_tests::BoxedAir; +use super::executor::execute; +use super::fri::{ + FRI_LEAF_GROUP, FriQuery, FriShape, declare_fri, emit_query_fri, hint_layer_openings, +}; +use super::join_tests::{HostSubProof, build_host_sub_proof}; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// A base element as its image in the cubic extension — component 0, the +/// embedding `IsSubFieldOf` uses implicitly wherever production multiplies a +/// base by an ext. +fn embed(x: &FE) -> FEE { + FEE::new([*x, FE::zero(), FE::zero()]) +} + +// ============================================================================= +// A real proof that folds +// ============================================================================= + +/// Proves L2G_MEMORY over `num_boundaries` boundary claims at `blowup`. +/// +/// The same AIR, prover and options path as `constraint_tests::real_fixture` — +/// only the row count differs, and the row count is what decides whether FRI +/// folds. `num_boundaries` must be a power of two so the trace length is exactly +/// it (the generator pads to the next power of two, which would silently change +/// the shape this suite is measuring). +pub(super) fn folding_fixture( + num_boundaries: usize, + blowup: usize, +) -> (BoxedAir, MultiProof) { + let opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(blowup as u8) + .expect("a power-of-two blowup is valid"); + folding_fixture_with(num_boundaries, opts) +} + +/// [`folding_fixture`] under explicit proof options — the format axis (a +/// Merkle cap, a FRI fold schedule) and the query count a cap needs. +pub(super) fn folding_fixture_with( + num_boundaries: usize, + opts: stark::proof::options::ProofOptions, +) -> (BoxedAir, MultiProof) { + use crate::tables::local_to_global::{ + CellBoundary, FiniClaim, InitClaim, generate_local_to_global_trace, + }; + use crate::test_utils::{EPOCH_TEST_LABEL, multi_prove_ram}; + + assert!( + num_boundaries.is_power_of_two(), + "the trace is padded to a power of two, so a non-power-of-two row count \ + would not be the shape asked for" + ); + let air = crate::continuation::l2g_memory_air(&opts, EPOCH_TEST_LABEL); + + let boundaries: Vec = (0..num_boundaries as u64) + .map(|i| CellBoundary { + address: 0x1000 + 8 * i, + init: InitClaim { + value: i + 1, + timestamp: 0, + originating_epoch: 0, + }, + fini: FiniClaim { + value: 2 * i + 3, + epoch: EPOCH_TEST_LABEL, + timestamp: 17 + i, + }, + }) + .collect(); + let mut trace = generate_local_to_global_trace(&boundaries); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .expect("the L2G_MEMORY fixture must prove at any power-of-two row count"); + + (Box::new(air), proof) +} + +/// Everything the FRI leg reads about one real sub-proof. +pub(super) struct HostFri { + pub(super) shape: FriShape, + /// The trace-side host fixture over the SAME proof: the openings, the roots, + /// and production's own DEEP answers, which are this leg's `p₀`. + pub(super) trace: HostSubProof, + /// One root per committed layer, in fold order. + pub(super) layer_roots: Vec, + /// `ζ₀ .. ζ_C` from the verifier's replay. + pub(super) zetas: Vec, + /// The terminal polynomial's coefficients, low-to-high. + pub(super) coeffs: Vec, + /// `[query][layer]` — `(opened values, path)`: the sibling `pᵢ(−υ^(2ⁱ))` + /// under `pair`, the whole group under a fold schedule. Paths are cut at + /// each layer's cap (query 0's cap split off into [`Self::caps`]). + pub(super) openings: Vec, Vec)>>, + /// Every capped layer's cap, in layer order — the caps arena. Empty at + /// the default format. + pub(super) caps: Vec, +} + +/// Build the FRI host fixture for a real proof of `num_boundaries` rows. +fn host_fri(num_boundaries: usize, blowup: usize) -> HostFri { + let (air, proof) = folding_fixture(num_boundaries, blowup); + host_fri_from(&*air, &proof) +} + +/// [`host_fri`] for a proof the caller already holds — needed where the test +/// also wants the AIR's verifier domain. +pub(super) fn host_fri_from( + air: &dyn AIR, + proof: &MultiProof, +) -> HostFri { + use stark::proof::view::StarkProofView; + + let trace = build_host_sub_proof(air, proof); + let view = StarkProofView::Owned(&proof.proofs[0]); + let opts = air.options(); + let shape = FriShape::for_layout(opts, trace.shape.log2_lde_length, trace.shape.layout); + shape.check(); + + // Per layer the opened values (the sibling, or the whole group) and the + // path cut at the layer's cap; query 0's caps go to the caps arena. + let (openings, caps) = super::epoch_verify_tests::fri_layer_openings(view, shape); + + HostFri { + shape, + layer_roots: view.fri_layers_merkle_roots().to_vec(), + zetas: trace.zetas.clone(), + coeffs: view.fri_final_poly_coeffs().to_vec(), + openings, + caps, + trace, + } +} + +impl HostFri { + /// The arenas the FRI-only program declares, for the given queries. + pub(super) fn fri_arenas(&self, queries: &[usize]) -> Vec> { + let mut out = vec![ + super::proof_arena::commitments_to_arena(&self.layer_roots), + self.zetas.iter().map(ext_word).collect(), + self.coeffs.iter().map(ext_word).collect(), + self.query_arena(queries), + ]; + // Declared by `declare_fri` only when the format caps some layer. + if self.shape.cap_words(super::proof_arena::words_per_root()) > 0 { + out.push(super::proof_arena::commitments_to_arena(&self.caps)); + } + out + } + + /// Per query, per layer: the symmetric evaluation then its path. + pub(super) fn query_arena(&self, queries: &[usize]) -> Vec { + let mut out = Vec::new(); + for &q in queries { + for (values, path) in &self.openings[q] { + out.extend(values.iter().map(ext_word)); + out.extend(super::proof_arena::commitments_to_arena(path)); + } + } + out + } + + /// The terminal codeword, rebuilt exactly as `terminal_codeword_from_coeffs` + /// does (`fri/terminal.rs:134-155`) out of production's own FFT and + /// bit-reverse permutation. + /// + /// That module is `pub(crate)` inside `crypto/stark`, so this is a mirror of + /// its three lines rather than a call to it. The mirror is what + /// [`the_terminal_point_is_the_query_point_folded`] tests the emitter's + /// evaluation against — and the mirror itself is checked, because the same + /// codeword must reproduce the values the PROVER folded to, which no reading + /// of these three lines could fake. + pub(super) fn terminal_codeword(&self) -> Vec { + use math::fft::bit_reversing::in_place_bit_reverse_permute; + + let coset_offset = FE::from(self.shape.coset_offset); + let terminal_offset = coset_offset.pow(1u64 << self.shape.total_folds()); + let poly = Polynomial::new(&self.coeffs); + let blowup = self.shape.terminal_len() / self.coeffs.len(); + let mut natural = Polynomial::evaluate_offset_fft::( + &poly, + blowup, + Some(self.coeffs.len()), + &terminal_offset, + ) + .expect("the terminal coset is a power of two inside the two-adicity"); + in_place_bit_reverse_permute(&mut natural); + natural + } +} + +// ============================================================================= +// The owed check: is the leaf gadget reusable? +// ============================================================================= + +/// ★ The check the retiring agent OWED and never ran: the machine's leaf hash at +/// `GroupShape { num_columns: 1, is_ext: true }` is byte-identical to the FRI +/// layer leaf, run against BOTH production backends. +/// +/// The two sides genuinely use different types — the prover commits under +/// `PairKeccak256Backend` (`fri/mod.rs:100`) and the verifier authenticates +/// under `BatchedMerkleTreeBackend` (`verifier.rs:643`) — and the spec's claim is +/// that they are byte-identical. This asserts both, so a divergence shows up as +/// a named failure rather than as a mysterious walk that will not reach its root. +/// +/// ## The vectors, and the lesson they encode +/// +/// A tamper suite whose every vector differed in byte 0 is one of the holes this +/// phase found by falsifying its own guards. A leaf here is 48 bytes: two +/// extension elements, three components each, eight big-endian bytes each. So the +/// vectors are built to make **every one of the 48 byte positions carry a +/// distinct value**, with no component equal to another and none symmetric under +/// byte reversal. A wrong component order, a wrong element order, a +/// little-endian limb or a dropped high byte each move a different subset of the +/// 48, and all of them move at least one. +#[test] +fn the_fri_leaf_is_byte_identical_to_productions_own_backends() { + use super::proof_arena::{BlockBatched, BlockPair}; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + // Six distinct components, each with six distinct nonzero bytes in + // descending positions, so no two of the 48 bytes agree and no component is + // a byte-reversal of itself or of another. + let component = |i: u64| FE::from(0x0102_0304_0506_0708u64 * (i + 1) + 0x11 * (i + 1)); + let ext = |base: u64| FEE::new([component(base), component(base + 1), component(base + 2)]); + let vectors: [(FEE, FEE); 4] = [ + (ext(0), ext(3)), + // Order-sensitivity: the same two elements swapped must hash differently + // (checked below), which is what says the leaf is ordered at all. + (ext(3), ext(0)), + // A zero element beside a maximal one: catches a gadget that skips or + // truncates a zero limb, and a canonicity slip at p−1. + ( + FEE::zero(), + FEE::new([ + FE::from(Gl::modulus_minus_one()), + FE::one(), + FE::from(0xFFFF_FFFF_0000_0000u64), + ]), + ), + // One bit apart in the LAST byte of the LAST component — the position a + // suite that only ever varied byte 0 would never reach. + (ext(9), FEE::new([component(12), component(13), FE::one()])), + ]; + + // The wrap hash production commits under: this leg's whole claim is that + // the machine's leaf IS the verifier's leaf, and the verifier's backend + // follows the aliases. + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(2); + let v0 = b.hint_word(arena, 0); + let v1 = b.hint_word(arena, 1); + let leaf = super::sub_proof::emit_leaf_hash(&mut b, FRI_LEAF_GROUP, &[v0, v1]); + // ⚠ The digest's OWN width. Two publishes assumed a byte digest; an + // algebraic one is a single cell whose second slot repeats the first, so + // the comparison below would have read one lane twice. + for cell in leaf.cells() { + b.public(*cell); + } + let program = compile(b.finish()); + validate(&program).expect("the leaf program is admissible"); + + let mut digests = Vec::new(); + for (i, (a, c)) in vectors.iter().enumerate() { + let arenas = vec![vec![ext_word(a), ext_word(c)]]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the leaf hash executes"); + let got: Vec = exec.public_words.iter().map(|(_, w)| *w).collect(); + + let batched = as IsMerkleTreeBackend>::hash_data(&vec![*a, *c]); + let paired = as IsMerkleTreeBackend>::hash_data(&[*a, *c]); + assert_eq!( + batched, paired, + "vector {i}: the spec's claim is that the prover's pair backend and \ + the verifier's batched backend are byte-identical; they are not" + ); + assert_eq!( + got.as_slice(), + super::proof_arena::commitment_words(&batched).as_slice(), + "vector {i}: the machine's leaf must be the verifier's leaf — this \ + is the byte-level check the FRI leg was handed as unverified" + ); + digests.push(got); + } + assert_ne!( + digests[0], digests[1], + "swapping the two elements must change the leaf, or the emitted order is \ + not carried into the hash and the parity Select is decoration" + ); + println!( + "machine leaf == BatchedMerkleTreeBackend == PairKeccak256Backend on {} \ + vectors covering all 48 bytes", + vectors.len() + ); +} + +// ============================================================================= +// The emitter, against production +// ============================================================================= + +/// The FRI leg alone, driven by a hinted index and a hinted DEEP pair. +/// +/// Used where the point of the test is FRI rather than the join: the trace legs +/// cost ~5,000 instructions per query per group and would dominate a run whose +/// subject is the fold. The index still goes through one `bit_dec` and the point +/// still comes from [`super::sub_proof::emit_points_from_bits`], so the +/// machine's own derivation is under test rather than a supplied point. +/// +/// Arena order: the per-query `(index, p₀, p₀ˢ)` block, then the four +/// [`FriArenas`]. +pub(super) fn fri_only_program(shape: FriShape, num_queries: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + // Per query `(index, p₀, p₀ˢ)`, or `(r, DEEP(x_r))` under one-row leaves. + let per = fri_deep_words(shape) as u32; + let q = b.declare_arena(per * num_queries as u32); + let (arenas, fri) = declare_fri(&mut b, shape, num_queries); + for i in 0..num_queries { + let index = b.hint_felt(q, per * i as u32); + let p0 = b.hint_word(q, per * i as u32 + 1).as_ext(); + let bits = b.bit_dec(index, shape.index_bits()); + let (p0_sym, point, point_sym) = if shape.one_row() { + let point = super::sub_proof::emit_point_from_row_bits( + &mut b, + shape.log2_lde_length, + FE::from(shape.coset_offset), + &bits, + ); + (None, point, None) + } else { + let p0_sym = b.hint_word(q, per * i as u32 + 2).as_ext(); + let (point, point_sym) = super::sub_proof::emit_points_from_bits( + &mut b, + shape.log2_lde_length, + FE::from(shape.coset_offset), + &bits, + ); + (Some(p0_sym), point, Some(point_sym)) + }; + let openings = hint_layer_openings(&mut b, shape, &arenas, i); + let v = emit_query_fri( + &mut b, + shape, + &fri, + &FriQuery { + p0, + p0_sym, + point, + point_sym, + bits: &bits, + }, + &openings, + ); + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the FRI program must be admissible"); + program +} + +/// Words per query of [`fri_only_program`]'s DEEP arena: `(index, p₀, p₀ˢ)` +/// for row pairs, `(r, DEEP(x_r))` under one-row leaves. +pub(super) fn fri_deep_words(shape: FriShape) -> usize { + if shape.one_row() { 2 } else { 3 } +} + +impl HostFri { + /// The DEEP arena [`fri_only_program`] reads: `(index, p₀, p₀ˢ)` per + /// query, or `(r, DEEP(x_r))` under one-row leaves. + pub(super) fn deep_arena(&self, queries: &[usize]) -> Vec { + let mut out = Vec::new(); + for &q in queries { + out.push(base_word(FE::from(self.trace.iotas[q] as u64))); + if self.shape.one_row() { + out.push(ext_word(&self.trace.expected_at_r[q])); + } else { + out.push(ext_word(&self.trace.expected[q].0)); + out.push(ext_word(&self.trace.expected[q].1)); + } + } + out + } + + /// Every arena [`fri_only_program`] declares, in order. + pub(super) fn all_arenas(&self, queries: &[usize]) -> Vec> { + let mut all = vec![self.deep_arena(queries)]; + all.extend(self.fri_arenas(queries)); + all + } +} + +/// ★ The premise of this suite, checked before anything is built on it: the +/// production prover FOLDS when the trace is big enough, and the layer count is +/// steerable by the row count. +/// +/// This is the finding that retires the leg's instrument problem. The FRI leg +/// was handed the conclusion that "the production instance exercises none of the +/// mechanism" and that synthetic codewords were the only witness. That was true +/// of the fixture as written and false of the fixture as available: the row count +/// is `boundaries.len().next_power_of_two()`, and `num_committed = trace_bits − 8`, +/// so 512 boundaries commit one layer and 2048 commit three. Everything below +/// therefore differentials against production data rather than against a +/// synthesized input, and the `saturating_sub(1)` off-by-one that +/// `join_tests::the_fold_layout_is_right_off_productions_constants` could only +/// catch host-side is now caught by an executed walk that fails to reach a real +/// root. +/// +/// The four shapes are the sweep the successor brief asked for — `num_committed` +/// over 0, 1, 2, 3 — and the zero row is the original fixture, unchanged. +#[test] +fn the_real_prover_folds_and_the_layer_count_follows_the_row_count() { + println!("rows n folds committed coeffs zetas queries terminal_len"); + for (rows, committed) in [(4usize, 0usize), (512, 1), (1024, 2), (2048, 3)] { + let h = host_fri(rows, 2); + println!( + "{rows:>5} {:>6} {:>6} {:>10} {:>7} {:>6} {:>8} {:>13}", + h.shape.log2_lde_length, + h.shape.total_folds(), + h.shape.num_committed(), + h.coeffs.len(), + h.zetas.len(), + h.trace.iotas.len(), + h.shape.terminal_len(), + ); + assert_eq!( + h.shape.num_committed(), + committed, + "{rows} rows must commit {committed} FRI layers" + ); + // The three structural lengths the verifier rejects on before its query + // loop (`verifier.rs:426-448`), asked of the real proof. + assert_eq!( + h.layer_roots.len(), + h.shape.num_committed(), + "committed roots" + ); + assert_eq!( + h.coeffs.len(), + h.shape.num_terminal_coeffs(), + "terminal coefficients" + ); + assert_eq!( + h.zetas.len(), + if h.shape.total_folds() > 0 { + h.shape.num_committed() + 1 + } else { + 0 + }, + "folds exceed committed layers by one, and nothing folds at all when \ + the codeword is already terminal" + ); + for (q, per_layer) in h.openings.iter().enumerate() { + assert_eq!(per_layer.len(), h.shape.num_committed(), "query {q} layers"); + for (i, (_, path)) in per_layer.iter().enumerate() { + assert_eq!( + path.len(), + h.shape.layer_path_len(i), + "query {q} layer {i}: the layer tree is one level shallower \ + per fold, so its path length is n − i − 2" + ); + } + } + } +} + +/// ★ The deviation from spec §5, justified numerically against production's own +/// FFT: the terminal codeword's value at position `iota >> C` is the terminal +/// polynomial evaluated at `υ^(2^total_folds)`. +/// +/// The emitter checks `P(υ^(2^total_folds)) = v` where production checks +/// `terminal_codeword[index] = v`. If those are not the same number the emitter +/// is wrong, and the machine's own assertions would not say so — they would +/// simply both be wrong together. So the identity is checked here, host-side, +/// over the real proofs at every one of their 219 indices: the codeword side +/// comes from production's `evaluate_offset_fft` + `in_place_bit_reverse_permute` +/// (`terminal.rs:150-155`) and the point side from production's own +/// `query_challenge_to_evaluation_point` raised by repeated squaring. +/// +/// A missing coset offset, a wrong exponent, a dropped bit reversal or the wrong +/// shift on `iota` each break it. The zero-fold shape is included, where the +/// claim is that the two positions `2·iota` and `2·iota+1` are `υ` and `−υ` — +/// the identity that lets one emitted shape serve production's two branches. +#[test] +fn the_terminal_point_is_the_query_point_folded() { + use stark::domain::new_verifier_domain; + use stark::verifier::{IsStarkVerifier, Verifier}; + type V = Verifier; + + for rows in [4usize, 512, 1024, 2048] { + let (air, proof) = folding_fixture(rows, 2); + let h = host_fri_from(&*air, &proof); + let codeword = h.terminal_codeword(); + assert_eq!(codeword.len(), h.shape.terminal_len()); + let domain = new_verifier_domain(&*air, proof.proofs[0].trace_length); + let c = h.shape.num_committed(); + + let mut distinct = std::collections::HashSet::new(); + for &iota in &h.trace.iotas { + let point = V::query_challenge_to_evaluation_point(iota, false, &domain); + let mut x = point; + for _ in 0..h.shape.total_folds() { + x = x.square(); + } + let at = Polynomial::new(&h.coeffs).evaluate(&embed(&x)); + + // The position production compares at, per branch. The two branches + // index DIFFERENTLY and conflating them is the mistake this test + // made on its first run: the folding branch walks `index` down from + // `iota` (`verifier.rs:735`), so it lands on `iota >> C`, while the + // zero-fold branch never has an `index` at all and reads the pair + // positions `2·iota` and `2·iota+1` directly (`verifier.rs:684-690`) + // — its terminal codeword IS the deep codeword, in which `iota` + // numbers pairs rather than elements. + let position = if h.shape.total_folds() == 0 { + iota * 2 + } else { + iota >> c + }; + assert_eq!( + at, + codeword[position], + "rows {rows} iota {iota}: P(υ^(2^{})) must be the terminal \ + codeword at position {position}", + h.shape.total_folds() + ); + if h.shape.total_folds() == 0 { + assert_eq!(x, point, "nothing folds, so the point is unchanged"); + assert_eq!( + Polynomial::new(&h.coeffs).evaluate(&embed( + &V::query_challenge_to_evaluation_point(iota, true, &domain) + )), + codeword[iota * 2 + 1], + "iota {iota}: the symmetric position must be −υ" + ); + } + distinct.insert(position); + } + println!( + "rows {rows:>5}: identity holds at all {} indices ({} distinct \ + terminal positions of {})", + h.trace.iotas.len(), + distinct.len(), + codeword.len() + ); + assert!( + distinct.len() > 1 || codeword.len() == 1, + "if every query landed on the same terminal position the check would \ + be one equation, not a sweep" + ); + } +} + +/// ★ The parity branch is REACHED, at every layer. +/// +/// The leaf order is `[sym, v]` at odd index and `[v, sym]` at even +/// (`verifier.rs:637-641`), selected on bit `i` of `iota` at layer `i`. An +/// implementation with the two arms swapped, or with no `Select` at all, is +/// invisible to a fixture whose indices all share a parity — the same +/// degenerate-parameter trap as the fold itself, one level down. This asserts the +/// real proof's 219 indices carry both parities at every committed layer, which +/// is what makes `no_tampered_fri_value_can_pass` able to catch the swap. +#[test] +fn the_real_indices_reach_both_leaf_parities_at_every_layer() { + for rows in [512usize, 1024, 2048] { + let h = host_fri(rows, 2); + for layer in 0..h.shape.num_committed() { + let (even, odd): (Vec<_>, Vec<_>) = h + .trace + .iotas + .iter() + .map(|iota| (iota >> layer) & 1) + .partition(|b| *b == 0); + assert!( + !even.is_empty() && !odd.is_empty(), + "rows {rows} layer {layer}: {} even and {} odd indices — a layer \ + reached by only one parity leaves the leaf-order Select \ + unexercised", + even.len(), + odd.len() + ); + } + println!( + "rows {rows:>5}: both parities present at all {} layers", + h.shape.num_committed() + ); + } +} + +/// ★ THE HEADLINE: the emitted FRI leg verifies every query of a real proof that +/// really folds, at three layer counts, and its terminal value is the one +/// production would have looked up. +/// +/// ## Why this is a strong check and not just an endpoint check +/// +/// The published value is only the LAST link. Every intermediate `v` is pinned +/// too, and not by an assertion this test writes — by the proof itself. At layer +/// `i` the machine hashes `{v, sym}` into a leaf and walks it to +/// `fri_layers_merkle_roots[i]`, a root the production prover committed to its +/// own folded codeword. So a `v` that were wrong at any layer could not reach +/// that root, and the run would not execute at all. The fold chain, the point +/// chain, the parity ordering, the walk depths and the layer-to-`ζ` alignment are +/// all inside that. +/// +/// What the published comparison adds is the terminal link, which no Merkle root +/// covers: the final fold is never committed (`fri/mod.rs:114-118`), so `v` at +/// the terminal layer is checked only against the coefficients. That is compared +/// here against production's own codeword at production's own position. +#[test] +fn the_fri_emitter_verifies_every_query_of_a_real_folding_proof() { + for rows in [4usize, 512, 1024, 2048] { + let h = host_fri(rows, 2); + let all: Vec = (0..h.trace.iotas.len()).collect(); + let program = fri_only_program(h.shape, all.len()); + let exec = execute( + &program, + &h.all_arenas(&all), + &crate::hash_pin::BLOCK_HASHER, + ) + .expect( + "an honest FRI decommitment must authenticate every layer and reach \ + the terminal polynomial", + ); + + let codeword = h.terminal_codeword(); + let c = h.shape.num_committed(); + let mut nonzero = 0usize; + for (k, &q) in all.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("the fold output is ext"); + let iota = h.trace.iotas[q]; + let position = if h.shape.total_folds() == 0 { + iota * 2 + } else { + iota >> c + }; + assert_eq!( + v, codeword[position], + "rows {rows} query {q} (iota {iota}): the machine's terminal value \ + must be the terminal codeword at the position production compares \ + at" + ); + if v != FEE::zero() { + nonzero += 1; + } + } + assert_eq!( + nonzero, + all.len(), + "a vacuously zero fold would make the differential empty" + ); + println!( + "rows {rows:>5}: {} queries, {} committed layers, {} instructions, \ + {} permutations — every terminal value matches production's codeword", + all.len(), + c, + program.instrs.len(), + permutations(&program), + ); + } +} + +/// ★ Both legs as one program, on a real folding proof: the openings +/// authenticated, DEEP folded from the authenticated cells, and FRI folded from +/// DEEP's own output at DEEP's own point. +/// +/// This is the seam the leg exists to close. The FRI leg could be correct in +/// isolation and still verify a different query than the trace leg did — folding +/// `p₀` values it was handed while the walks authenticated some other index. Here +/// there is nothing to hand: `emit_sub_proof_with_fri` takes the `QueryOutput` +/// cells, so `p₀`, `υ` and the index bits are the same addresses in both legs by +/// construction. +/// +/// Run over a subset of queries because the trace side is ~50× the FRI side per +/// query at this shape; the coverage of the FRI mechanism itself is +/// [`the_fri_emitter_verifies_every_query_of_a_real_folding_proof`]'s, over all +/// 219. +#[test] +fn the_two_legs_verify_one_real_folding_proof_as_one_program() { + let h = host_fri(1024, 2); + let queries: Vec = (0..6).collect(); + assert_eq!( + h.shape.num_committed(), + 2, + "the 1024-row shape commits two layers" + ); + // The query count is program shape, so a subset run is a different shape and + // has to say so — `emit_sub_proof_with_fri` refuses to emit a query count + // that disagrees with the one in the shape it was handed. + let shape = FriShape { + num_queries: queries.len(), + ..h.shape + }; + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, _, terminal) = + super::fri::emit_sub_proof_with_fri(&mut b, &h.trace.shape, shape, queries.len()); + for v in &terminal { + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the joined program is admissible"); + + let mut arenas = h.trace.arenas(&queries); + arenas.extend(h.fri_arenas(&queries)); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the honest proof must authenticate, fold and reach the terminal"); + + let codeword = h.terminal_codeword(); + for (k, &q) in queries.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!( + v, + codeword[h.trace.iotas[q] >> h.shape.num_committed()], + "query {q}: the joined program's terminal value" + ); + } + println!( + "joined trace+DEEP+FRI over {} queries of a folding proof: {} \ + instructions, {} permutations", + queries.len(), + program.instrs.len(), + permutations(&program), + ); +} + +pub(super) fn permutations(program: &LfmProgram) -> usize { + // The CONFIGURED wrap hash's compressions. Filtering `KeccakF` here read + // zero the moment production moved to BLAKE3, turning a cost measurement + // into a failed assertion about a count nobody had re-derived. + super::machine_tests::wrap_hash_instrs(program) +} + +pub(super) fn count_matching bool>( + program: &LfmProgram, + f: F, +) -> usize { + program.instrs.iter().filter(|i| f(i)).count() +} + +/// The marginal per-query cost of a shape, by emitting one query and two and +/// differencing — so no per-sub-proof plumbing (the hoisted root unpacks, the +/// coefficient hints) lands in the figure. +struct PerQuery { + perms: usize, + swaps: usize, + instrs: usize, +} + +fn marginal_fri(shape: FriShape) -> PerQuery { + let one = fri_only_program( + FriShape { + num_queries: 1, + ..shape + }, + 1, + ); + let two = fri_only_program( + FriShape { + num_queries: 2, + ..shape + }, + 2, + ); + let dec = + |p: &LfmProgram| count_matching(p, |i| matches!(i, super::instr::Instr::BitDec { .. })); + PerQuery { + perms: permutations(&two) - permutations(&one), + swaps: dec(&two) - dec(&one), + instrs: two.instrs.len() - one.instrs.len(), + } +} + +/// ★ MEASURED against the prediction pinned before the emitter existed. +/// +/// `join_tests::the_fri_sizing_prediction` recorded 174/186/198 permutations per +/// query and 38,106/20,460/14,454 per sub-proof at blowup 2/4/8, `trace_bits = +/// 20`, derived from spec §8. This counts the `LFM_KECCAK` rows the emitter +/// actually emits at those shapes and asserts the same numbers. +/// +/// The 2^20-row shape is emitted, not proved — a real proof at that size is a +/// prover run, not a test — but the quantity predicted IS the emitted +/// permutation count, so this is a measurement of the thing predicted rather +/// than a model of it. That the same formula holds on EXECUTED programs is +/// [`the_fri_emitter_verifies_every_query_of_a_real_folding_proof`]'s doing at +/// n = 10/11/12, where 219 queries produced exactly 1,971 / 4,161 / 6,570 +/// permutations against `219 × (C + Σ pathlen)` = 219 × 9 / 19 / 30. +/// +/// ## Two currencies that point opposite ways +/// +/// The other columns are reported because permutations alone hide where the rows +/// go, and because the two honest answers disagree. Rendering the two extension +/// values of a layer leaf into 48 big-endian bytes costs `6C` byteswaps per +/// query, each one `LFM_BITDEC` row plus 64 `LFM_BALU` rows — which makes +/// byteswapping the majority of the leg's INSTRUCTIONS. In main-trace CELLS the +/// same comparison inverts by two orders of magnitude, because a permutation +/// expands into 24 `KECCAK_RND` rounds of 1,480 columns while a byteswap carries +/// 322 cells. `others/lfm-target-shape.md`'s rule that rows of different chips +/// are not comparable is exactly this, so both are printed and neither is called +/// "the" cost. +#[test] +fn the_emitted_permutation_count_meets_the_pinned_prediction() { + const TRACE_BITS: u32 = 20; + let swap_cells = super::machine_tests::byteswap_cells(); + let perm_cells = super::machine_tests::permutation_cells(); + println!( + "blowup C Q perms/q predicted total predicted swaps/q instr/q hash cells/q swap cells/q" + ); + for (blowup_log, queries, predicted_per_query, predicted_total) in [ + (1u32, 219usize, 174usize, 38_106usize), + (2, 110, 186, 20_460), + (3, 73, 198, 14_454), + ] { + let shape = FriShape { + log2_lde_length: TRACE_BITS + blowup_log, + blowup_log, + final_poly_log_degree: 7, + coset_offset: 3, + num_queries: queries, + format: stark::proof::options::ProofFormat::DEFAULT, + trees: stark::config::TreeGeometry::BINARY, + }; + shape.check(); + let per = marginal_fri(shape); + println!( + " 2^{blowup_log} {:>3} {:>3} {:>8} {:>10} {:>8} {:>10} {:>8} {:>8} {:>13} {:>13}", + shape.num_committed(), + queries, + per.perms, + predicted_per_query, + per.perms * queries, + predicted_total, + per.swaps, + per.instrs, + per.perms as u64 * perm_cells, + per.swaps as u64 * swap_cells, + ); + assert_eq!( + per.perms, predicted_per_query, + "blowup 2^{blowup_log}: emitted permutations per query against the \ + pinned prediction" + ); + assert_eq!( + per.perms * queries, + predicted_total, + "blowup 2^{blowup_log}: emitted permutations per sub-proof" + ); + // The same number, from the shape arithmetic rather than from the + // emitted program. Equal counts here mean the emitter walks the depths + // the shape says it should — the one place a wrong `layer_path_len` + // would show up as agreement between two wrongs is if BOTH came from the + // shape, and only one of these does. + assert_eq!( + per.perms, + shape.permutations_per_query(), + "the emitted program and the shape arithmetic must agree" + ); + // One byteswap per extension component per leaf value: two values, three + // components, per committed layer — ON A BYTE HASH. Under the algebraic + // arm the term is zero and the only decomposition left per query is the + // index's, which is why the count is written as a WIDTH times a shape. + let per_value = super::machine_tests::byteswaps_per_value(); + assert_eq!( + per.swaps, + 1 + 6 * shape.num_committed() * per_value, + "the index decomposition plus six component byteswaps per layer" + ); + // The inversion, asserted rather than left to the reader: byteswapping + // is the majority of the instructions and a rounding error in cells. + // + // ⚠ Both halves are statements about a BYTE hash. The algebraic arm + // absorbs felts, so there is no rendering to dominate anything and the + // first half is simply false there — guarded rather than deleted, + // because it still holds and still has to hold under keccak and BLAKE3. + let swap_instrs = per.swaps * 65; + if per_value == 1 { + assert!( + swap_instrs * 2 > per.instrs, + "byteswapping should be the majority of the leg's instructions ({swap_instrs} of {})", + per.instrs + ); + } + assert!( + per.perms as u64 * perm_cells > 100 * per.swaps as u64 * swap_cells, + "and a rounding error in main-trace cells" + ); + } +} + +/// ★ ABSOLUTE (rule 7): the joined program contains ONE point derivation per +/// query and ONE decomposition of the index, and every term of the count comes +/// from a SHAPE rather than from a second emission. +/// +/// ## This test was wrong first, and how it was caught matters more than the fix +/// +/// Its first form measured the FRI leg's marginal `Select` count as +/// `selects(joined) − selects(trace_only)` and asserted the difference was +/// `C + 2 · path_steps`, reasoning that a second point derivation would add +/// `index_bits`. That is vacuous, and injecting the exact defect it denies — a +/// `QueryOutput` handing out a freshly derived point — left it GREEN. The reason +/// is rule 7's failure mode wearing a different hat: the defect lives in +/// `emit_sub_proof_with_bits`, which is what BOTH sides of the subtraction call, +/// so both gained `index_bits` selects and the difference never moved. +/// +/// **A difference of two counts taken from our own emitter is still a relative +/// test, however much it looks like a count.** The marginal-cost idiom this phase +/// uses everywhere is safe only when the RESULT is compared against a number that +/// did not come from the emitter — a pinned prediction, or a closed form over the +/// shapes: +/// +/// ```text +/// selects/query = index_bits (pow_bits, once per query) +/// + w · merkle_depth · num_groups (trace walks) +/// + num_committed (FRI leaf ordering) +/// + w · path_steps_per_query (FRI walks) +/// ``` +/// +/// where `w` is the digest's width in arena words — two on a byte hash, one on +/// an algebraic one. `pow_bits` emits one `Select` per bit (`edsl.rs:257-262`) +/// and each walk level one per digest word, since every word of a digest must +/// swap on the same bit (`edsl.rs:164-169`). A second derivation makes the +/// measured count exceed the closed form by exactly `index_bits`, and nothing +/// cancels it. Re-falsified in that form: the injected defect now fails with "a +/// surplus of 11 index bits". +#[test] +fn the_fri_join_adds_no_second_point_derivation() { + let h = host_fri(2048, 2); + let sub = &h.trace.shape; + let groups = sub.groups(); + let selects = + |p: &LfmProgram| count_matching(p, |i| matches!(i, super::instr::Instr::Select { .. })); + let decs = + |p: &LfmProgram| count_matching(p, |i| matches!(i, super::instr::Instr::BitDec { .. })); + + let emit = |n: usize| { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + super::fri::emit_sub_proof_with_fri( + &mut b, + sub, + FriShape { + num_queries: n, + ..h.shape + }, + n, + ); + compile(b.finish()) + }; + // Marginal, so the per-sub-proof plumbing is out of the figure — but the + // figure is then compared against the shapes, never against another emission. + let one = emit(1); + let two = emit(2); + let per_query_selects = selects(&two) - selects(&one); + let per_query_decs = decs(&two) - decs(&one); + + // The digest's width, as the HOST reads it: `emit` builds at + // `WrapHash::production()`, whose builder width this is the counterpart of. + let dw = super::proof_arena::words_per_root(); + let expected_selects = h.shape.index_bits() + + dw * sub.merkle_depth * groups.len() + + h.shape.num_committed() + + dw * h.shape.path_steps_per_query(); + assert_eq!( + per_query_selects, + expected_selects, + "selects per query: {} index bits for the ONE point derivation, {} for \ + {} trace walks over {} levels, {} FRI leaf orderings, {} for {} FRI path \ + steps. A surplus of {} index bits is a second point derivation or a \ + second index decomposition", + h.shape.index_bits(), + dw * sub.merkle_depth * groups.len(), + groups.len(), + sub.merkle_depth, + h.shape.num_committed(), + dw * h.shape.path_steps_per_query(), + h.shape.path_steps_per_query(), + h.shape.index_bits(), + ); + + // One decomposition of the index, plus one byteswap per field element that + // enters a leaf: a base element is one, an extension element three — and + // that whole term is a BYTE-hash term, so it carries the same width factor + // `dw` carries above. On the algebraic arm it vanishes and the expected + // count is 1, which is the property this test is named for, undiluted. + let leaf_swaps: usize = groups + .iter() + .map(|g| g.num_values() * if g.is_ext { 3 } else { 1 }) + .sum(); + let expected_decs = 1 + + (leaf_swaps + 6 * h.shape.num_committed()) * super::machine_tests::byteswaps_per_value(); + assert_eq!( + per_query_decs, + expected_decs, + "decompositions per query: ONE for the index, {leaf_swaps} for the trace \ + leaves, {} for the FRI layer leaves. A surplus of one is a second index \ + decomposition", + 6 * h.shape.num_committed(), + ); + println!( + "per query: {per_query_selects} selects and {per_query_decs} \ + decompositions, both equal to the closed form over the shapes — one \ + point derivation, one index decomposition" + ); +} + +// ============================================================================= +// Falsification: what the leg denies +// ============================================================================= + +/// ★ No arena value the FRI leg reads can be moved without the run failing. +/// +/// The emitted checks are `assert_eq` inside the program, which lowers to +/// `diff / 0` — provable and executable only when `diff` is zero. So "the tamper +/// is caught" and "the run does not execute" are the same statement, and a +/// tamper that still executed would be a hole. +/// +/// The vectors sweep every KIND of value the leg reads, and the last one is the +/// only interesting attack: a COHERENT forgery in the sense of method rule 4 — +/// every value in it is a genuine value the production prover committed to, just +/// belonging to a different query. Nothing in it is malformed, no hash is +/// invented, and the leaf it builds is a leaf that really exists in the real +/// layer tree. What rejects it is only that the walk climbs at the index bits of +/// THIS query, so a real leaf at the wrong position cannot reach the root. +/// +/// ## What is deliberately absent +/// +/// There is no vector that tampers `p₀`. It is not an arena value here — it is a +/// cell the DEEP leg computed — and moving it is +/// `join_tests::no_tampered_value_can_move_the_fold_without_moving_the_root`'s +/// subject one leg back. A FRI-side tamper of `p₀` would only be possible in the +/// standalone driver, where it is hinted for isolation, and would prove nothing +/// about the joined program. +#[test] +fn no_tampered_fri_value_can_pass() { + const ROWS: usize = 2048; + let h = host_fri(ROWS, 2); + let c = h.shape.num_committed(); + assert_eq!(c, 3, "this suite wants several layers to tamper inside"); + + // Two queries whose layer-0 parities DIFFER, so the splice below moves a + // leaf between positions of opposite parity as well as of different index. + let a = (0..h.trace.iotas.len()) + .find(|&q| h.trace.iotas[q].is_multiple_of(2)) + .expect("an even index"); + let b = (0..h.trace.iotas.len()) + .find(|&q| !h.trace.iotas[q].is_multiple_of(2)) + .expect("an odd index"); + let queries = vec![a, b]; + let shape = FriShape { + num_queries: queries.len(), + ..h.shape + }; + let program = fri_only_program(shape, queries.len()); + let honest = h.all_arenas(&queries); + execute(&program, &honest, &crate::hash_pin::BLOCK_HASHER) + .expect("the honest run must execute"); + + // Host-side offsets at the host's digest width; the program was built at + // `WrapHash::production()`, which this is the counterpart of. + let dw = super::proof_arena::words_per_root(); + let stride = h.shape.query_words(dw); + // (label, arena, word) — arena order is the driver's: deep, roots, zetas, + // coeffs, queries. + let mut bump: Vec<(&str, usize, usize)> = vec![ + ("query index", 0, 0), + ("layer 0 root", 1, 0), + ("layer 2 root", 1, dw * (c - 1)), + ("zeta_0 (the DEEP fold's challenge)", 2, 0), + ("zeta_C (the uncommitted final fold)", 2, c), + ("terminal coefficient 0", 3, 0), + ("terminal coefficient 127", 3, h.coeffs.len() - 1), + ("layer 0 symmetric evaluation", 4, 0), + ("layer 0 sibling, leaf level", 4, 1), + ( + "layer 0 sibling, top level", + 4, + dw * h.shape.layer_path_len(0) - 1, + ), + ("second query's layer 0 evaluation", 4, stride), + ]; + if dw == 2 { + // Only a byte digest has a second word to move; an algebraic root is one. + bump.push(("layer 0 root, second word", 1, 1)); + } + for (label, arena, word) in bump { + let mut tampered = honest.clone(); + tampered[arena][word][0] += FE::one(); + let err = execute(&program, &tampered, &crate::hash_pin::BLOCK_HASHER).expect_err( + &format!("moving the {label} must make the program unexecutable"), + ); + println!(" {label:<40} rejected: {err:?}"); + } + + // The coherent forgery: query `a` presented with query `b`'s layer-0 + // decommitment. Every word is a real prover value. + let mut spliced = honest.clone(); + let (from, to) = (stride, 0usize); + let len = 1 + dw * h.shape.layer_path_len(0); + let borrowed: Vec = spliced[4][from..from + len].to_vec(); + assert_ne!( + borrowed, + spliced[4][to..to + len], + "the two queries must actually have different layer-0 openings, or the \ + splice is a no-op and this vector proves nothing" + ); + spliced[4][to..to + len].copy_from_slice(&borrowed); + let err = execute(&program, &spliced, &crate::hash_pin::BLOCK_HASHER).expect_err( + "a REAL leaf and a REAL path, at the wrong index, must still be rejected \ + — the walk climbs at this query's own bits", + ); + println!( + " {:<40} rejected: {err:?}", + "another query's real layer-0 opening" + ); +} + +/// ★ The three structural length checks production performs at RUNTIME are, in +/// this machine, impossible to fail — and that is worth demonstrating rather +/// than asserting. +/// +/// `verifier.rs:426-448` rejects on three lengths before its query loop, and the +/// comment there is emphatic about why: the per-query auth-path and +/// evaluation-sym vectors are **not** bound into the Fiat-Shamir transcript, so a +/// prover could send them EMPTY — making the fold loop run zero iterations and +/// accept the query vacuously — and that length check is the only thing pinning +/// them. +/// +/// In LFM there is no vector to send. `declare_fri` fixes each arena's length +/// from the shape, and the executor refuses an arena of any other length +/// (`ArenaLenMismatch`) before a single instruction runs. So the attack the +/// production comment describes is not defended against here, it is +/// unrepresentable: there is no encoding of "a proof with no FRI layers" that the +/// program for a 3-layer shape will accept. This test spells out each of the +/// three, including the vacuous-fold one. +#[test] +fn the_shape_pins_the_lengths_production_must_check_at_runtime() { + use super::executor::LfmExecError; + + let h = host_fri(2048, 2); + let queries = vec![0usize]; + let shape = FriShape { + num_queries: 1, + ..h.shape + }; + let program = fri_only_program(shape, 1); + let honest = h.all_arenas(&queries); + execute(&program, &honest, &crate::hash_pin::BLOCK_HASHER) + .expect("the honest run must execute"); + + // (label, arena, what the truncation would buy a prover) + let attacks: [(&str, usize, &str); 3] = [ + ( + "no committed layer roots", + 1, + "production's `fri_layers_merkle_roots().len() != num_committed` check", + ), + ( + "fewer terminal coefficients", + 3, + "production's `fri_final_poly_coeffs().len() != 1 << effective_k` check", + ), + ( + "an EMPTY per-query decommitment — the vacuous fold", + 4, + "production's per-query `layers_auth_paths_len()` check, the one its \ + comment calls the only thing pinning these vecs", + ), + ]; + for (label, arena, mirrors) in attacks { + let mut truncated = honest.clone(); + truncated[arena].clear(); + let err = execute(&program, &truncated, &crate::hash_pin::BLOCK_HASHER) + .expect_err(&format!("{label} must be refused")); + assert!( + matches!(err, LfmExecError::ArenaLenMismatch { .. }), + "{label} must be refused for its LENGTH, before any instruction \ + runs — got {err:?}" + ); + println!(" {label:<52} refused as {err:?}\n mirrors {mirrors}"); + } +} + +/// ★ The FRI leg PROVES and VERIFIES — method rule 2, discharged rather than +/// argued. +/// +/// Every other test here calls `execute`, which runs the executor and the +/// arena/assert semantics but never builds a trace or a proof. Rule 2 is explicit +/// that an execute-only test says nothing about the chips: where the executor +/// mirrors a computation the chip also does, only a prove+verify run sees the +/// chip. +/// +/// It is tempting to argue the coverage away — the FRI leg emits no instruction +/// the trace legs do not already emit, and `join_tests::the_join_proves_and_verifies` +/// proves those. That argument is probably true and is exactly the kind of thing +/// rule 5 says to check instead of assert, so this proves the JOINED program: the +/// openings authenticated, DEEP folded, and FRI folded to the terminal check, all +/// in one proved and verified run over a real folding sub-proof. +/// +/// One query, because the point is the chips rather than the sweep — the fold +/// mechanism's coverage is +/// [`the_fri_emitter_verifies_every_query_of_a_real_folding_proof`]'s, over all +/// 219 of three shapes. +#[test] +fn the_fri_leg_proves_and_verifies() { + use super::proof::{lfm_prove, verify_against}; + use super::registry::build_artifacts_with_hasher; + + let h = host_fri(512, 2); + assert_eq!( + h.shape.num_committed(), + 1, + "one committed layer is enough to put a leaf hash, a walk, a root compare \ + and both folds through the prover" + ); + let queries = [0usize]; + let opts = super::join_tests::prove_options(); + let shape = FriShape { + num_queries: queries.len(), + ..h.shape + }; + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, _, terminal) = + super::fri::emit_sub_proof_with_fri(&mut b, &h.trace.shape, shape, queries.len()); + for v in &terminal { + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the joined program is admissible"); + + let mut arenas = h.trace.arenas(&queries); + arenas.extend(h.fri_arenas(&queries)); + // Built at `WrapHash::production()`, so it emits `Instr::Hash` and the + // artifacts must carry the pin's tenant — the classification rule in + // HASH-PINNING.md. `build_artifacts` defaults to the registry's blessed + // hasher, which is correct for registry programs and wrong for this one. + let artifacts = build_artifacts_with_hasher(&program, &opts, crate::hash_pin::BLOCK_HASHER); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts) + .expect("the joined trace+DEEP+FRI program must prove"); + + // The proved run's published terminal value, against production's own + // codeword — so the proof is not merely valid but computes the right thing. + let codeword = h.terminal_codeword(); + assert_eq!( + word_as_ext(&proved.public_words[0].1).expect("ext"), + codeword[h.trace.iotas[queries[0]] >> h.shape.num_committed()], + "the PROVED run must publish the terminal codeword value production \ + would have looked up" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the joined FRI run must verify" + ); + println!( + "proved and verified: {} instructions, {} permutations, {} committed layer", + program.instrs.len(), + permutations(&program), + h.shape.num_committed(), + ); +} + +// ============================================================================= +// Merkle caps in the FRI leg (S1) +// ============================================================================= + +/// The folding fixture's options under a cap policy: blowup 2, `queries` +/// queries (a cap needs openings: `auto` caps at 3 from 20 on), no grinding. +fn capped_options(policy: CapPolicy, queries: usize) -> stark::proof::options::ProofOptions { + let mut o = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + o.fri_number_of_queries = queries; + o.grinding_factor = 0; + o.format.merkle_cap = policy; + o +} + +/// 2048 rows at blowup 2: LDE 2^12, trace trees 11 deep, three committed FRI +/// layers 10, 9 and 8 deep — every tree tall enough for a height-3 cap. +const CAPPED_ROWS: usize = 2048; + +fn capped_host(policy: CapPolicy, queries: usize) -> HostFri { + let (air, proof) = folding_fixture_with(CAPPED_ROWS, capped_options(policy, queries)); + host_fri_from(&*air, &proof) +} + +/// ★ The FRI leg verifies every query of a real CAPPED folding proof, and its +/// permutation count is the capped closed form exactly: per query one leaf and +/// `depth − c` parents per layer, plus `2^c − 1` parents per capped layer ONCE +/// (the cap hashed up to its root). +#[test] +fn the_fri_emitter_verifies_a_capped_folding_proof() { + for policy in [CapPolicy::Fixed(1), CapPolicy::Fixed(2), CapPolicy::Auto] { + let h = capped_host(policy, 24); + assert_eq!(h.shape.num_committed(), 3); + for i in 0..3 { + let want = if policy == CapPolicy::Fixed(1) { + 1 + } else if policy == CapPolicy::Fixed(2) { + 2 + } else { + 3 + }; + assert_eq!(h.shape.layer_cap(i), want, "{policy}: layer {i}"); + } + let all: Vec = (0..h.trace.iotas.len()).collect(); + let program = fri_only_program(h.shape, all.len()); + let exec = execute( + &program, + &h.all_arenas(&all), + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("an honest capped FRI decommitment must execute"); + + let codeword = h.terminal_codeword(); + let c = h.shape.num_committed(); + for (k, &q) in all.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!(v, codeword[h.trace.iotas[q] >> c], "{policy} query {q}"); + } + let emitted = permutations(&program); + let closed = all.len() * h.shape.permutations_per_query() + h.shape.cap_permutations(); + assert_eq!( + emitted, closed, + "{policy}: emitted permutations against the capped closed form" + ); + // And the saving against the uncapped shape is the cap's own formula: + // per tree `Q·c − (2^c − 1)`. + let uncapped = FriShape { + format: stark::proof::options::ProofFormat::DEFAULT, + ..h.shape + }; + let saved: usize = (0..c) + .map(|i| { + let cap = h.shape.layer_cap(i); + all.len() * cap - ((1usize << cap) - 1) + }) + .sum(); + assert_eq!( + all.len() * uncapped.permutations_per_query() - saved, + emitted, + "{policy}: the cap saves Q·c − (2^c − 1) per layer tree" + ); + println!( + "{policy}: {} queries, caps {:?}: {emitted} permutations (uncapped {})", + all.len(), + (0..c).map(|i| h.shape.layer_cap(i)).collect::>(), + all.len() * uncapped.permutations_per_query(), + ); + } +} + +/// ★ Every cap word of every capped FRI layer is bound — including the ones no +/// query reaches, which only the once-per-tree cap-to-root check can reject +/// (in-guest). One query at a height-3 cap reaches one of +/// eight nodes per layer, so seven words per layer are rejected by that check +/// alone. +#[test] +fn every_fri_cap_word_is_bound_even_the_unreached_ones() { + let h = capped_host(CapPolicy::Fixed(3), 24); + let queries = vec![0usize]; + let shape = FriShape { + num_queries: 1, + ..h.shape + }; + let program = fri_only_program(shape, 1); + let honest = h.all_arenas(&queries); + execute(&program, &honest, &crate::hash_pin::BLOCK_HASHER).expect("honest"); + // Arena order: deep, roots, zetas, coeffs, queries, caps. + let caps = honest.len() - 1; + assert_eq!( + honest[caps].len(), + 3 * 8 * super::proof_arena::words_per_root(), + "three layers, eight cap digests each" + ); + for w in 0..honest[caps].len() { + let mut bad = honest.clone(); + bad[caps][w][0] += FE::one(); + execute(&program, &bad, &crate::hash_pin::BLOCK_HASHER) + .expect_err(&format!("cap word {w} moved must not execute")); + } +} + +/// ★ Both legs as one program over a CAPPED folding proof: the four trace +/// trees' caps and the three FRI layers' caps authenticated once, every opening +/// checked against them, and the permutation count the capped closed form. +#[test] +fn the_two_legs_verify_one_capped_folding_proof_as_one_program() { + use super::epoch_verify::{blocks_for, group_leaf_felts}; + + let h = capped_host(CapPolicy::Fixed(3), 24); + assert_eq!(h.trace.shape.trace_cap, 3); + let queries: Vec = (0..6).collect(); + let shape = FriShape { + num_queries: queries.len(), + ..h.shape + }; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, _, terminal) = + super::fri::emit_sub_proof_with_fri(&mut b, &h.trace.shape, shape, queries.len()); + for v in &terminal { + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the joined capped program is admissible"); + + let mut arenas = h.trace.arenas(&queries); + arenas.extend(h.fri_arenas(&queries)); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the honest capped proof must authenticate, fold and reach the terminal"); + let codeword = h.terminal_codeword(); + for (k, &q) in queries.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!(v, codeword[h.trace.iotas[q] >> h.shape.num_committed()]); + } + + let sub = &h.trace.shape; + let hash = super::edsl::WrapHash::production(); + let leaves: usize = sub + .groups() + .iter() + .map(|g| blocks_for(group_leaf_felts(g), hash)) + .sum(); + let closed = queries.len() + * (leaves + sub.groups().len() * sub.path_len() + shape.permutations_per_query()) + + sub.cap_permutations() + + shape.cap_permutations(); + assert_eq!(permutations(&program), closed, "the capped closed form"); + + // A trace-tree cap word moved: the caps arena of the TRACE leg is the + // sixth arena (uniforms, ood, parts, roots, queries, caps). + let mut bad = arenas.clone(); + bad[5][0][0] += FE::one(); + execute(&program, &bad, &crate::hash_pin::BLOCK_HASHER) + .expect_err("a moved trace-tree cap word must not execute"); +} diff --git a/prover/src/lfm/harvest.rs b/prover/src/lfm/harvest.rs new file mode 100644 index 000000000..9fed87a02 --- /dev/null +++ b/prover/src/lfm/harvest.rs @@ -0,0 +1,838 @@ +//! Reading a proof production accepts into the arenas the recursion's programs +//! verify it from: per sub-proof what the transcript absorbs ([`HostTable`]) +//! and what the legs open ([`TableLegs`]), and a child LFM proof as a whole +//! ([`HarvestedChild`]). The WHIR block tree's driver (`whir_block_tree`) +//! fills each node's arenas from its children with these. +//! +//! Every shape is the AIR's at the proof's trace length; the proof is checked +//! against it, never read into it, and a proof that disagrees is an error. The +//! oracle fields the suites compare against (the challenges production derived, +//! the child's LogUp pair) are read the same way in every build, and read only +//! by the suites. +//! +//! The suites that grew these readers (`epoch_tests`, `epoch_verify_tests`, +//! `per_table_aggregator_tests`) keep their names and their panicking forms +//! through re-exports and shims. + +use stark::config::Commitment; +use stark::proof::view::StarkProofView; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::constraints::{Analysis, BoundaryTerm}; +use super::epoch::TableChallengeShape; +use super::epoch_verify::TableVerifyShape; +use super::fri::FriShape; +use super::word::{LfmWord, base_word, ext_word}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// Everything one real sub-proof supplies to the replay, plus the challenges +/// production derived from it. +#[derive(Clone)] +pub(crate) struct HostTable { + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) shape: TableChallengeShape, + /// The verifier's HARDCODED precomputed commitment, when the AIR is + /// preprocessed. A program constant, not arena data: the verifier does not + /// take this from the proof (`verifier.rs:1187`). + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) precomputed_root: Option, + pub(crate) main_root: Commitment, + pub(crate) aux_root: Option, + pub(crate) contribution: Option, + pub(crate) composition_root: Commitment, + /// Row-major, as `row_major_data` carries it. + pub(crate) ood_current: Vec, + pub(crate) ood_next: Vec, + pub(crate) parts: Vec, + pub(crate) fri_roots: Vec, + pub(crate) fri_coeffs: Vec, + pub(crate) nonce: Option, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) needs_lookup_challenges: bool, + + // ---- the oracle ---- + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) beta: FEE, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) z: FEE, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) gamma: FEE, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) zetas: Vec, + /// Every query's leaf index, the transcript's: an arity-4 path is laid + /// out at it in the walk's hint order ([`build_table_legs_with`]). + pub(crate) iotas: Vec, +} + +/// A sub-proof inside a multi-table proof, read into [`HostTable`]: the fork +/// is already positioned (separator, aux root and `L` absorbed), so the oracle +/// comes from `replay_rounds_after_round_1` on THAT transcript. +#[cfg(test)] +pub(crate) fn host_table_forked( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + index: usize, + num_tables: usize, + fork: &mut crate::hash_pin::BlockTranscript, + lookup_challenges: &[FEE], +) -> Result { + host_table_forked_under::( + air, + view, + index, + num_tables, + fork, + lookup_challenges, + ) +} + +/// [`host_table_forked`] for a sub-proof of a proof made under the family +/// `Fam`: `Fam`'s transcript, its configuration's verifier replay. +pub(crate) fn host_table_forked_under( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + index: usize, + num_tables: usize, + fork: &mut Fam::Transcript, + lookup_challenges: &[FEE], +) -> Result { + use stark::domain::new_verifier_domain; + use stark::verifier::IsStarkVerifier; + type Verifier = + stark::verifier::GenericVerifier::Stark>; + + let trace_length = view.trace_length(); + let domain = new_verifier_domain(air, trace_length); + let layout = Verifier::::ood_layout(air); + let challenges = Verifier::::replay_rounds_after_round_1( + air, + view, + &(), + &domain, + fork, + lookup_challenges.to_vec(), + &layout, + ); + + let nt = challenges.transition_coeffs.len(); + let beta = if nt > 1 { + challenges.transition_coeffs[1] + } else { + challenges.boundary_coeffs[0] + }; + // `γ` is the second term of the DEEP coefficient run, which starts at one — + // the same recovery `constraint_tests::deep_shape` makes. + let gamma = challenges.trace_term_coeffs[1][0]; + + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + // The shape is the AIR's at this trace length; the proof is checked against + // it, never read into it. + let shape = TableChallengeShape::derive(air, index, num_tables, trace_length); + let proved = ( + view.lde_trace_aux_merkle_root().is_some(), + view.bus_table_contribution().is_some(), + (ood_c.width(), ood_c.height()), + (ood_n.width(), ood_n.height()), + view.composition_poly_parts_ood_evaluation().len(), + ); + let derived = ( + shape.has_aux_root, + shape.has_contribution, + shape.ood_current_dims, + shape.ood_next_dims, + shape.num_parts, + ); + if proved != derived { + return Err(format!( + "table {index}: the proof's aux root, L, OOD blocks and part count are the AIR's: \ + {proved:?} != {derived:?}" + )); + } + + Ok(HostTable { + shape, + precomputed_root: if air.is_preprocessed() { + Some(layout_precomputed_commitment(air, view.trace_length())?) + } else { + None + }, + main_root: *view.lde_trace_main_merkle_root(), + aux_root: view.lde_trace_aux_merkle_root().copied(), + contribution: view.bus_table_contribution(), + composition_root: *view.composition_poly_root(), + ood_current: ood_c.row_major_data().to_vec(), + ood_next: ood_n.row_major_data().to_vec(), + parts: view.composition_poly_parts_ood_evaluation().to_vec(), + fri_roots: view.fri_layers_merkle_roots().to_vec(), + fri_coeffs: view.fri_final_poly_coeffs().to_vec(), + nonce: view.nonce(), + needs_lookup_challenges: true, + beta, + z: challenges.z, + gamma, + zetas: challenges.zetas.clone(), + iotas: challenges.iotas.clone(), + }) +} + +/// Everything the verification legs read about one real sub-proof. +/// +/// The split against [`HostTable`] is by CONSUMER, not by convenience: that +/// struct holds what the transcript absorbs, this one holds what the legs open. +/// Nothing appears in both — which is the arena-join obligation showing up in +/// the reader as well as in the emitted program. +pub(crate) struct TableLegs { + pub(crate) verify: TableVerifyShape, + /// The wrap hash the node reading these legs runs (the child's proof + /// family's, `ProofFamily::WRAP`): every digest in the arenas is laid at + /// its width (`proof_arena::words_per_root_for`), the width the node's + /// builder reads (`edsl::digest_words`). + pub(crate) wrap: super::edsl::WrapHash, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) analysis: Analysis, + /// `[query][group]` — the row pair in leaf order, then the path. + pub(crate) openings: Vec, Vec)>>, + /// `[query][layer]` — `(opened values, path)`: the sibling `pᵢ(−υ^(2ⁱ))` + /// under `pair`, the whole `2^{d_j}` group under a fold schedule. + pub(crate) fri_openings: Vec, Vec)>>, + /// Every capped tree's cap, split off query 0's (owner) path, in the caps + /// arena's order: the committed matrices in group order, then the capped + /// FRI layers. Empty at the default format. + pub(crate) caps: Vec, + /// Production's OWN boundary-constraint list for this AIR, kept so + /// `the_boundary_terms_are_program_shape` can compare the program-shape + /// rule against the call rather than against a belief about it. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) production_boundary: Vec, + /// `AIR::has_aux_trace`, the rule's input. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) has_aux_trace: bool, + /// Preprocessed-column count, zero when the AIR is not preprocessed. Which + /// sub-proofs are preprocessed is what assembly ledger entry 7 is about. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) num_precomputed_cols: usize, + /// The commitment production absorbs for this table, when preprocessed — + /// `air.precomputed_commitment()`, taken from the AIR and never from the + /// proof. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) precomputed_commitment: Option, +} + +/// `Err` with `what` unless `got == want`. +fn check_eq(got: T, want: T, what: &str) -> Result<(), String> { + if got == want { + Ok(()) + } else { + Err(format!("{what}: {got:?} != {want:?}")) + } +} + +/// Read one real sub-proof into the shapes and openings the legs consume. +/// +/// Every shape here is derived from the AIR and the proof OPTIONS. The one +/// parameter that is neither is `log2_trace_length` — a table's chunk length is +/// chosen by the prover's row counts — and it is program shape in the assembled +/// verifier for the reason the arena schema makes it one: the program is emitted +/// for a specific epoch shape, and a proof whose trace length disagreed would +/// not match the arenas it declares. +#[cfg(test)] +pub(crate) fn build_table_legs( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + rap_challenges: &[FEE], +) -> Result { + build_table_legs_with( + air, + view, + rap_challenges, + stark::config::TreeGeometry::BINARY, + None, + super::edsl::WrapHash::production(), + ) +} + +/// [`build_table_legs`] for a sub-proof whose trees are `trees` (its +/// configuration's arity and arity-4 cap), with every query's leaf index +/// (`iotas`, the transcript's, from [`HostTable::iotas`]): every path is laid +/// out to the cap in the walk's order — at arity 4 the hint order of +/// [`super::p1w16_emit::walk4`] at the opening's index +/// ([`super::merkle_cap::path_to_cap`]), so an arity-4 proof needs `iotas`; a +/// binary one ignores them. The arenas are laid at `wrap`'s digest width, the +/// reading node's. +pub(crate) fn build_table_legs_with( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + rap_challenges: &[FEE], + trees: stark::config::TreeGeometry, + iotas: Option<&[usize]>, + wrap: super::edsl::WrapHash, +) -> Result { + // The shapes, from the AIR and the trace length alone; the proof's blocks + // are then checked against them, never read into them. + let trace_length = view.trace_length(); + let (verify, analysis) = TableVerifyShape::derive_with(air, trace_length, trees) + .map_err(|e| format!("the legs' shape derives: {e}"))?; + let (main_width, aux_width) = air.trace_layout(); + let num_precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + let log2_trace_length = trace_length.trailing_zeros(); + check_eq( + view.composition_poly_parts_ood_evaluation().len(), + verify.quotient.num_composition_parts, + "the proof's part count is the AIR's degree bound", + )?; + // The grid the machine rebuilds and the blocks the proof carries must + // describe one table. Checked rather than assumed because the machine's + // reconstruction is indexed by the SHAPE and filled from the BLOCKS: a width + // disagreement would silently scatter the next-row values into wrong columns. + let deep = &verify.sub.deep; + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + check_eq( + ood_c.width(), + deep.num_total_cols, + "the current-row OOD block is the full trace width", + )?; + check_eq( + ood_c.height(), + deep.step_size, + "the current-row block's height IS step_size (ood.rs:110-114)", + )?; + check_eq( + ood_n.width(), + deep.next_row_cols.len(), + "the next-row block is as wide as the transition window", + )?; + check_eq( + ood_n.height(), + deep.num_eval_points - deep.step_size, + "the next-row block covers every evaluation point past the first step", + )?; + + // ---- the owner split: query 0 of a capped tree carries the cap at the end + // of its path; the arenas take the path to the cap (in the walk's order), + // the caps arena the cap. + let mut trace_caps: Vec> = Vec::new(); + let arity = verify.sub.arity; + let mut split = |q: usize, + path: &[Commitment], + depth: usize, + c: usize| + -> Result, String> { + let index = leaf_index(iotas, q, 0)?; + let (siblings, cap) = super::merkle_cap::path_to_cap(path, depth, c, arity, q == 0, index) + .map_err(|e| format!("query {q}: {e}"))?; + trace_caps.extend(cap); + Ok(siblings) + }; + let (depth, c_trace) = (verify.sub.merkle_depth, verify.sub.trace_cap); + + // ---- the openings, per query, in the emitter's group order. + let mut openings = Vec::with_capacity(view.deep_poly_openings_len()); + for q in 0..view.deep_poly_openings_len() { + let o = view.deep_poly_opening(q); + let mut groups: Vec<(Vec, Vec)> = Vec::new(); + if num_precomputed > 0 { + let p = o + .precomputed_trace_polys() + .ok_or("a preprocessed air opens its precomputed columns")?; + groups.push(( + p.evaluations() + .iter() + .chain(p.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + split(q, p.merkle_path(), depth, c_trace)?, + )); + } + let m = o.main_trace_polys(); + groups.push(( + m.evaluations() + .iter() + .chain(m.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + split(q, m.merkle_path(), depth, c_trace)?, + )); + if aux_width > 0 { + let a = o.aux_trace_polys().ok_or("an aux opening")?; + groups.push(( + a.evaluations() + .iter() + .chain(a.evaluations_sym()) + .map(ext_word) + .collect(), + split(q, a.merkle_path(), depth, c_trace)?, + )); + } + let c = o.composition_poly(); + groups.push(( + c.evaluations() + .iter() + .chain(c.evaluations_sym()) + .map(ext_word) + .collect(), + split(q, c.merkle_path(), depth, c_trace)?, + )); + openings.push(groups); + } + + let (fri_openings, fri_caps) = fri_layer_openings_at(view, verify.fri, iotas)?; + + // Production's own boundary list, for the premise check only. It takes the + // bus public inputs, which are PROOF data — which is exactly why the emitted + // program must not be built from this call. + let bus_public_inputs = view + .bus_table_contribution() + .map(stark::lookup::BusPublicInputs::from_contribution); + let generator = ::get_primitive_root_of_unity( + log2_trace_length as u64, + ) + .map_err(|_| "a power-of-two trace length has a root of unity")?; + let production_boundary = air + .boundary_constraints( + &(), + rap_challenges, + bus_public_inputs.as_ref(), + trace_length, + ) + .constraints + .iter() + .map(|c| BoundaryTerm { + col: if c.is_aux { main_width + c.col } else { c.col }, + point: generator.pow(c.step as u64), + value: c.value, + }) + .collect(); + + let caps: Vec = trace_caps.into_iter().flatten().chain(fri_caps).collect(); + let root_words = super::proof_arena::words_per_root_for(wrap); + check_eq( + caps.len() * root_words, + verify.cap_words(root_words), + "every capped tree's cap, and nothing else", + )?; + + Ok(TableLegs { + verify, + wrap, + analysis, + openings, + fri_openings, + caps, + production_boundary, + has_aux_trace: air.has_aux_trace(), + num_precomputed_cols: num_precomputed, + precomputed_commitment: if air.is_preprocessed() { + Some(layout_precomputed_commitment(air, trace_length)?) + } else { + None + }, + }) +} + +/// [`super::epoch_verify::layout_precomputed_commitment`], where a layout +/// with no root is an error. At row pairs it IS +/// `air.precomputed_commitment()`. +pub(crate) fn layout_precomputed_commitment( + air: &dyn AIR, + trace_length: usize, +) -> Result { + super::epoch_verify::layout_precomputed_commitment(air, trace_length) + .ok_or_else(|| format!("no precomputed commitment at {trace_length} rows")) +} + +/// One query's FRI layer openings, per layer `(opened values, path)`. +type FriOpenings = Vec, Vec)>>; + +/// Every query's FRI layer openings, per layer `(opened values, path)`, and +/// the capped layers' caps (layer order) split off query 0's owner paths. +/// +/// The proof's flat `layers_evaluations_sym` is one sibling per layer under +/// `pair` and every layer's full group (`2^{d_j}` values, position order) +/// under a fold schedule; `FriShape::layer_values` says which. +/// Each path is cut at its layer's cap: query 0 of a capped layer carries +/// `D − c + 2^c` nodes, every other query `D − c`. +#[cfg(test)] +pub(crate) fn fri_layer_openings( + view: StarkProofView<'_, Gl, Ext3, PI>, + fri: FriShape, +) -> Result<(FriOpenings, Vec), String> +where + PI: rkyv::Archive, + ::Archived: rkyv::Deserialize, +{ + fri_layer_openings_at(view, fri, None) +} + +/// [`fri_layer_openings`] with every query's index (see +/// [`build_table_legs_with`]): layer `j`'s tree leaf is the index past the +/// layer's slot, `iota >> (G_j + d_j)` (`FriShape::layer_bit_offset`). +pub(crate) fn fri_layer_openings_at( + view: StarkProofView<'_, Gl, Ext3, PI>, + fri: FriShape, + iotas: Option<&[usize]>, +) -> Result<(FriOpenings, Vec), String> +where + PI: rkyv::Archive, + ::Archived: rkyv::Deserialize, +{ + let mut caps: Vec = Vec::new(); + let mut openings = Vec::with_capacity(view.query_list_len()); + for q in 0..view.query_list_len() { + let d = view.query(q); + let flat = d.layers_evaluations_sym(); + let per_query: usize = (0..fri.num_committed()).map(|j| fri.layer_values(j)).sum(); + check_eq( + flat.len(), + per_query, + &format!("query {q}: the opened values per query"), + )?; + let mut offset = 0usize; + let mut layers = Vec::with_capacity(fri.num_committed()); + for i in 0..fri.num_committed() { + let values = flat[offset..offset + fri.layer_values(i)].to_vec(); + offset += fri.layer_values(i); + let path = d.layer_auth_path(i); + let (depth, c) = (fri.layer_depth(i), fri.layer_cap(i)); + let shift = fri.layer_bit_offset(i) + fri.layer_fold(i) as usize; + let index = leaf_index(iotas, q, shift)?; + let (siblings, cap) = + super::merkle_cap::path_to_cap(path, depth, c, fri.arity(), q == 0, index) + .map_err(|e| format!("query {q} FRI layer {i}: {e}"))?; + if let Some(cap) = cap { + caps.extend(cap); + } + layers.push((values, siblings)); + } + openings.push(layers); + } + Ok((openings, caps)) +} + +/// Query `q`'s leaf index shifted right by `shift` (a FRI layer's tree), when +/// the caller supplied the indices. +fn leaf_index(iotas: Option<&[usize]>, q: usize, shift: usize) -> Result, String> { + iotas + .map(|all| { + all.get(q) + .map(|iota| iota >> shift) + .ok_or_else(|| format!("no index for query {q}")) + }) + .transpose() +} + +impl TableLegs { + /// Per query, per group: the row-pair values then the sibling digests. + /// + /// NO index word, which is the whole difference from + /// `join_tests::HostSubProof::query_arena`: the assembled verifier's index is + /// the transcript's own bits, so an arena that carried one would be offering + /// the prover a second index. + pub(crate) fn try_opening_arena(&self) -> Result, String> { + let mut out = Vec::new(); + for query in &self.openings { + for (values, siblings) in query { + out.extend(values.iter().copied()); + out.extend(super::proof_arena::commitments_to_arena_for( + siblings, self.wrap, + )); + } + } + check_eq( + out.len(), + self.verify + .opening_words(super::proof_arena::words_per_root_for(self.wrap)), + "the opening arena must fill exactly what the shape declares", + )?; + Ok(out) + } + + /// The sub-proof's Merkle caps, once — `None` at the default format, where + /// the emitter declares no caps arena + /// (`epoch_verify::declare_table_arenas`). + pub(crate) fn try_caps_arena(&self) -> Result>, String> { + let words = self + .verify + .cap_words(super::proof_arena::words_per_root_for(self.wrap)); + if words == 0 { + check_eq(self.caps.len(), 0, "no caps at a format without caps")?; + return Ok(None); + } + let out = super::proof_arena::commitments_to_arena_for(&self.caps, self.wrap); + check_eq( + out.len(), + words, + "the caps arena is what the shape declares", + )?; + Ok(Some(out)) + } + + /// Per query, per committed layer: the symmetric evaluation then its path. + pub(crate) fn try_fri_arena(&self) -> Result, String> { + let mut out = Vec::new(); + for query in &self.fri_openings { + for (values, path) in query { + out.extend(values.iter().map(ext_word)); + out.extend(super::proof_arena::commitments_to_arena_for( + path, self.wrap, + )); + } + } + check_eq( + out.len(), + self.verify + .fri_words(super::proof_arena::words_per_root_for(self.wrap)), + "the FRI arena must fill exactly what the shape declares", + )?; + Ok(out) + } + + /// [`Self::try_opening_arena`], panicking: the suites' form. + #[cfg(test)] + pub(crate) fn opening_arena(&self) -> Vec { + self.try_opening_arena().unwrap_or_else(|e| panic!("{e}")) + } + + /// [`Self::try_caps_arena`], panicking: the suites' form. + #[cfg(test)] + pub(crate) fn caps_arena(&self) -> Option> { + self.try_caps_arena().unwrap_or_else(|e| panic!("{e}")) + } + + /// [`Self::try_fri_arena`], panicking: the suites' form. + #[cfg(test)] + pub(crate) fn fri_arena(&self) -> Vec { + self.try_fri_arena().unwrap_or_else(|e| panic!("{e}")) + } +} + +/// One child LFM proof, production-accepted, harvested for emission. +/// +/// The per-table sibling of `epoch_tests::RealEpoch`, over an LFM machine's +/// proof rather than the VM's. ★ Nothing in the harvest is LFM-specific: +/// [`host_table_forked`] and [`build_table_legs`] take `(&dyn AIR, +/// StarkProofView)`, so the same two functions read a wrap proof, a node proof +/// and a VM epoch proof alike. That is what makes one node emitter serve every +/// level. +pub(crate) struct HarvestedChild { + /// The child's proof family's wrap hash: the node reading the child runs + /// it, and every digest of the child's arenas is laid at its width. + pub(crate) wrap: super::edsl::WrapHash, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) artifacts: super::registry::LfmArtifacts, + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) opts: crate::ProofOptions, + pub(crate) public_words: Vec<(u32, LfmWord)>, + /// Every sub-proof's commitment-time answers, in proof order: Phase A + /// absorbs them between the roots and the shared LogUp pair. + pub(crate) commit_ood: Vec>, + pub(crate) tables: Vec, + pub(crate) legs: Vec, + /// The child's OWN shared LogUp pair, recovered host-side by + /// `verify_against_chunked`'s own Phase A replay — the oracle the node's leg + /// must reproduce in-machine. Consumed by + /// `the_leaf_node_verifies_and_binds_two_wraps` through + /// `NodePublishSet::Diagnostic`. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) z_alpha: (FEE, FEE), +} + +/// [`harvest_child_under`] at the legacy pin, after verifying the proof against +/// its artifacts, and the seconds that verify cost: a child read from a proof +/// production would reject describes nothing, so a refusal is an error. +#[cfg(test)] +pub(crate) fn harvest_child_verified( + artifacts: super::registry::LfmArtifacts, + opts: crate::ProofOptions, + proved: &super::proof::LfmProof, +) -> Result<(HarvestedChild, f64), String> { + harvest_child_verified_under::(artifacts, opts, proved) +} + +/// [`harvest_child_verified`] of a proof made under the family `Fam`: verified +/// under `Fam`, then [`harvest_child_under`]. +#[cfg(test)] +pub(crate) fn harvest_child_verified_under( + artifacts: super::registry::LfmArtifacts, + opts: crate::ProofOptions, + proved: &super::proof::LfmProof, +) -> Result<(HarvestedChild, f64), String> { + let t_verify = std::time::Instant::now(); + if !super::proof::verify_against_artifacts_under::( + &artifacts, + &proved.proof, + &proved.public_words, + &opts, + ) { + return Err("the child proof does not verify against its artifacts".to_string()); + } + let verify_secs = t_verify.elapsed().as_secs_f64(); + Ok(( + harvest_child_under::(artifacts, opts, proved)?, + verify_secs, + )) +} + +/// A child proof of the family `Fam` read without its verify — `Fam`'s +/// statement seed and transcript, its configuration's trees — for a caller +/// that verifies the same proof elsewhere and fails the run on a refusal +/// before it reports anything (the block tree's children, verified beside the +/// timed path). +pub(crate) fn harvest_child_under( + artifacts: super::registry::LfmArtifacts, + opts: crate::ProofOptions, + proved: &super::proof::LfmProof, +) -> Result { + use crypto::fiat_shamir::is_transcript::IsTranscript; + use stark::proof::view::MultiProofView; + use stark::verifier::IsStarkVerifier; + + // The AIR set the artifacts describe — `KECCAK_RND`/`LFM_BLAKE3` chunks, the + // `LFM_HASH` chunks and (S2) the one-row preprocessed roots, exactly + // as `verify_against_artifacts` builds it: a one-row chip's Phase A root and + // leg compare use them. + let airs = super::airs::LfmAirs::for_artifacts(&artifacts, &opts)?; + let refs = airs.air_refs(); + let view = MultiProofView::Owned(&proved.proof); + check_eq(refs.len(), view.len(), "one AIR per sub-proof")?; + + // The seed IS `verify_against_chunked`'s: the LFM statement over the claimed + // words, and nothing before it. + let seed = || { + let mut t = Fam::transcript(&[]); + super::statement::absorb_lfm_statement_tagged( + &mut t, + Fam::STATEMENT_TAG, + &artifacts.program_id, + &proved.public_words, + opts.fri_final_poly_log_degree, + ); + t + }; + + // Phase A — every root, `z0`, every answer, then the shared LogUp pair — + // is the verifier's own replay, so the fork every sub-proof starts from is + // the one the verifier forks. + let mut transcript = seed(); + let lookup: Vec = + stark::verifier::GenericVerifier::::replay_phase_a_bound( + &refs, + view, + &proved.proof.commit_ood, + &mut transcript, + ) + .ok_or("the child's Phase A does not replay (a preprocessed root or an answer count)")?; + + let num_tables = refs.len(); + let tables = refs + .iter() + .enumerate() + .map(|(idx, air)| { + let v = view.get(idx); + let mut fork = transcript.clone(); + if num_tables > 1 { + fork.append_bytes(&(idx as u64).to_le_bytes()); + } + if let Some(root) = v.lde_trace_aux_merkle_root() { + fork.append_bytes(root); + } + if let Some(c) = v.bus_table_contribution() { + fork.append_field_element(&c); + } + host_table_forked_under::(*air, v, idx, num_tables, &mut fork, &lookup) + }) + .collect::, String>>()?; + // Each query's leaf index from the fork's replay: an arity-4 path is laid + // out at it (the walk's hint order); a binary one ignores it. + let legs = refs + .iter() + .enumerate() + .zip(&tables) + .map(|((idx, air), table)| { + // An LFM proof is bound: each table's DEEP carries its answers. + build_table_legs_with( + *air, + view.get(idx), + &lookup, + stark::config::TreeGeometry::of::(), + Some(&table.iotas), + Fam::WRAP, + ) + .map(|mut legs| { + legs.verify = legs.verify.with_commit_ood(*air); + legs + }) + }) + .collect::, String>>()?; + + Ok(HarvestedChild { + wrap: Fam::WRAP, + artifacts, + opts, + public_words: proved.public_words.clone(), + commit_ood: proved.proof.commit_ood.clone(), + tables, + legs, + z_alpha: (lookup[0], lookup[1]), + }) +} + +/// The child's arenas, in `declare_leg_arenas`' declaration order. +pub(crate) fn try_child_arena_words(c: &HarvestedChild) -> Result>, String> { + let mut arenas: Vec> = Vec::new(); + // The published words, eight halves each — the statement's own layout. + let mut publics = Vec::with_capacity(8 * c.public_words.len()); + for (_, word) in &c.public_words { + for lane in word { + let v: u64 = lane.canonical(); + publics.push(base_word(FE::from(v & 0xFFFF_FFFF))); + publics.push(base_word(FE::from(v >> 32))); + } + } + arenas.push(publics); + arenas.push(super::proof_arena::commitments_to_arena_for( + &c.tables.iter().map(|h| h.main_root).collect::>(), + c.wrap, + )); + // Every sub-proof's commitment-time answers, one arena, proof order. + arenas.push(c.commit_ood.iter().flatten().map(ext_word).collect()); + for (h, leg) in c.tables.iter().zip(&c.legs) { + if let Some(root) = &h.aux_root { + arenas.push(super::proof_arena::commitments_to_arena_for( + &[*root], + c.wrap, + )); + } + if let Some(l) = &h.contribution { + arenas.push(vec![ext_word(l)]); + } + arenas.push(super::proof_arena::commitments_to_arena_for( + &[h.composition_root], + c.wrap, + )); + arenas.push(h.ood_current.iter().map(ext_word).collect()); + arenas.push(h.ood_next.iter().map(ext_word).collect()); + arenas.push(h.parts.iter().map(ext_word).collect()); + arenas.push(super::proof_arena::commitments_to_arena_for( + &h.fri_roots, + c.wrap, + )); + arenas.push(h.fri_coeffs.iter().map(ext_word).collect()); + if let Some(nonce) = h.nonce { + arenas.push(vec![base_word(FE::from(nonce))]); + } + arenas.push(leg.try_opening_arena()?); + arenas.push(leg.try_fri_arena()?); + arenas.extend(leg.try_caps_arena()?); + } + Ok(arenas) +} diff --git a/prover/src/lfm/hash.rs b/prover/src/lfm/hash.rs new file mode 100644 index 000000000..91240d413 --- /dev/null +++ b/prover/src/lfm/hash.rs @@ -0,0 +1,479 @@ +//! The LFM hash interface — the machine's swap surface. +//! +//! The ecosystem hash decision is open (Poseidon2 is broken; candidates are +//! Poseidon-original, RPO/XHash, Monolith and reduced-round Blake2s), so the +//! machine freezes only the *contract*: `Compress` and `Transcript` map two +//! digest cells to one — in different hash domains — `Permute` maps the +//! three-cell state to itself, and the `LFM_HASH` +//! bus tuples and opcode numbers are fixed. Whatever sits behind the trait is +//! the only thing a hash migration replaces. +//! +//! `TestPermutation` below is **NOT cryptographic**. It exists so the machine +//! can be built, executed and proved end-to-end before the hash decision +//! lands; it must never appear outside tests and pre-decision experiments. + +use crate::tables::types::FE; + +use super::instr::HashMode; +use super::word::LfmWord; + +/// Felts in the sponge state (three machine cells). +pub const HASH_STATE_FELTS: usize = 12; +/// Felts in a digest (one machine cell). +pub const HASH_DIGEST_FELTS: usize = 4; + +/// The machine's hash contract. `compress` has a default implementation as a +/// single permutation of `[a ‖ b ‖ IV]` truncated to the first cell, which is +/// the construction the chip's `Compress` mode implements; a real hash may +/// override it, but the bus contract (2 cells in, 1 cell out) is frozen. +/// +/// ⚠ **`permute` is not total for every candidate.** It is typed over arbitrary +/// Goldilocks elements, but a hasher built on 32-bit words can only accept +/// lane-restricted state, and a hasher may implement one socket and not the +/// other. [`LfmHasher::admits`] is where such a restriction is *declared* and +/// rejected; silently reducing an out-of-range input instead is the bug that +/// would make a host-side assertion pass while the chip proved something else. +pub trait LfmHasher { + /// The full state permutation (three cells → three cells). + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS]; + + /// The capacity cell injected into lanes 8–11 in `Compress` mode. + fn compress_iv(&self) -> LfmWord; + + /// The capacity cell injected in `Transcript` mode. + /// + /// The default is [`LfmHasher::compress_iv`], which is the single-domain + /// reading and carries the weakening [`LfmHasher::transcript_out`] records. + /// A hasher that separates its domains through the CAPACITY — the natural + /// hook for a sponge, and what RPX uses — overrides this instead of + /// overriding `transcript_out`, because then the chip's `S8` copy + /// constraint separates the domains too rather than the host doing it alone. + fn transcript_iv(&self) -> LfmWord { + self.compress_iv() + } + + /// The capacity cell injected in `Leaf` mode. Same rule as + /// [`LfmHasher::transcript_iv`]. + fn leaf_iv(&self) -> LfmWord { + self.compress_iv() + } + + /// The capacity cell a row of `mode` takes — the ONE rule the executor, the + /// trace filler and the chip's constraints all read. + /// + /// `Permute` rows have no injected capacity (they carry their own third + /// input cell), so this is never consulted for them; it answers with the + /// compress capacity rather than panicking, and the callers gate on the + /// mode before asking. + fn mode_iv(&self, mode: HashMode) -> LfmWord { + match mode { + HashMode::Transcript => self.transcript_iv(), + HashMode::Leaf => self.leaf_iv(), + HashMode::Compress | HashMode::Permute => self.compress_iv(), + } + } + + /// The twelve `OUT` felts the chip writes on a `Compress` row. + /// + /// The default is the permute-and-truncate construction: all twelve lanes + /// of `permute(a ‖ b ‖ IV)`, of which the low four are the digest. The + /// executor records exactly this into the row's `OUT` columns, so a hasher + /// that overrides [`LfmHasher::compress`] must override this too — or the + /// trace would describe a permutation its own AIR does not constrain. + fn compress_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + self.permute_two_cells(a, b, &self.compress_iv()) + } + + /// The two-cells-plus-capacity permutation every two-cell mode's default is + /// built from: `permute(a ‖ b ‖ iv)`. + /// + /// Factored out because the three modes differ ONLY in which capacity they + /// inject, and writing that difference three times is how a mode ends up + /// silently sharing another's domain — which is exactly what the trait's + /// defaults used to do, `transcript_out` and `leaf_out` both routing through + /// `compress_out` and picking up the compress IV on the way. + fn permute_two_cells(&self, a: &LfmWord, b: &LfmWord, iv: &LfmWord) -> [FE; HASH_STATE_FELTS] { + let mut state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|_| FE::zero()); + state[0..4].clone_from_slice(a); + state[4..8].clone_from_slice(b); + state[8..12].clone_from_slice(iv); + self.permute(state) + } + + /// Two digest cells → one digest cell. + fn compress(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + let out = self.compress_out(a, b); + [out[0], out[1], out[2], out[3]] + } + + /// One Fiat–Shamir transcript step: the same two-cells-in, one-cell-out + /// shape as [`LfmHasher::compress`], in the TRANSCRIPT hash domain. + /// + /// The default is `compress_out` — correct for a hasher with a single + /// domain, which is what `TestPermutation` is here. A hasher + /// that *has* domain separation overrides it, and BLAKE3 does: its socket + /// carries the domain tag in a message word, so a transcript step + /// and a Merkle parent over the same two cells are different digests. + /// + /// ⚠ The default is a real weakening for a hasher that ALSO leaves + /// [`LfmHasher::transcript_iv`] at its default, and it is deliberate rather + /// than overlooked: under `Test` a transcript step IS a Merkle parent, so + /// that hasher separates the domains not at all. It is not a production + /// hash — `TestPermutation` is explicitly non-cryptographic. + /// + /// A production candidate must separate them, by ONE of two mechanisms: + /// override this function (BLAKE3 does — its domain rides in a message + /// word), or override `transcript_iv` and let this default carry it (RPX + /// does — its domain rides in the capacity, which has the advantage that the + /// chip's `S8` copy constraint separates the domains too). + fn transcript_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + self.permute_two_cells(a, b, &self.transcript_iv()) + } + + /// [`LfmHasher::transcript_out`] truncated to the digest cell. + fn transcript(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + let out = self.transcript_out(a, b); + [out[0], out[1], out[2], out[3]] + } + + /// A Merkle LEAF: a chaining accumulator and one cell read as four arbitrary + /// FIELD ELEMENTS. + /// + /// **The accumulator is what makes the leaf a chain rather than a tree.** A + /// wide leaf is an arbitrary-width row pair, so its felts arrive four at a + /// time; carrying the running digest as this call's first operand absorbs + /// four felts AND chains in ONE hash, where folding a felts-only leaf digest + /// into the chain with a separate parent cost two (COMMIT.md §1.2). Leaf + /// absorption is the dominant term of a recursion tower node, which is why + /// the shape of this signature is worth the ripple. + /// + /// The default is a compress of the accumulator against the felts, which is + /// the natural reading for a **field-native** hasher: `TestPermutation` + /// takes arbitrary Goldilocks elements directly, so a leaf needs no encoding + /// from it. + /// + /// BLAKE3 overrides it, and the override is the point of the whole mode: its + /// lanes must be `u32`, so each felt becomes a checked `lo`/`hi` pair inside + /// the socket, under the `"LFML"` tag. + /// + /// ⚠ Same weakening as [`LfmHasher::transcript_out`], with the same two + /// escapes: a hasher that overrides neither this nor + /// [`LfmHasher::leaf_iv`] does not separate a leaf from a parent, so under + /// `Test` the O5 second-preimage split is carried by fixed tree depth alone. + /// BLAKE3 escapes by overriding this; RPX escapes by overriding `leaf_iv`. + fn leaf_out(&self, acc: &LfmWord, felts: &LfmWord) -> [FE; HASH_STATE_FELTS] { + self.permute_two_cells(acc, felts, &self.leaf_iv()) + } + + /// [`LfmHasher::leaf_out`] truncated to the digest cell. + fn leaf(&self, acc: &LfmWord, felts: &LfmWord) -> LfmWord { + let out = self.leaf_out(acc, felts); + [out[0], out[1], out[2], out[3]] + } + + /// Rejects a hash instruction this hasher's chip cannot prove, naming why. + /// + /// Total for every candidate whose domain is the whole state under both + /// modes, which is why the default is `Ok`. It exists for the ones whose + /// domain is smaller: `HasherKind::Blake3` uses it for both of its + /// restrictions — it has no `permute` socket at all, and its `compress` + /// lanes must be `u32`. Returning an error here is what turns "the AIR + /// would reject this" into "the executor says so, with a reason". + fn admits(&self, mode: HashMode, state: &[FE; HASH_STATE_FELTS]) -> Result<(), &'static str> { + let _ = (mode, state); + Ok(()) + } + + /// Whether this hasher's socket takes width-16 rows (`Instr::Hash16`). + /// + /// Only [`HasherKind::Poseidon1W16`]'s does; every twelve-felt socket refuses + /// one here, so a width-16 row under the wrong hasher is an executor error + /// with a reason rather than a witness no AIR accepts. + fn admits_width16(&self) -> Result<(), &'static str> { + Err( + "this hasher's LFM_HASH socket is twelve felts wide; a width-16 row needs HasherKind::Poseidon1W16", + ) + } +} + +/// [`HasherKind::Poseidon1W16`]'s answers to the twelve-felt contract: none it +/// can prove. The width-16 socket has no twelve-felt mode, so [`LfmHasher::admits`] +/// refuses every one. +/// +/// ✗ The hashing methods are unreachable by construction, and they panic rather +/// than return: every value they could return would be a hash no AIR proves (the +/// socket review's A2), as `Blake3Permutation::permute` does for its missing +/// mode. The executor checks `admits` before it hashes a row, the twelve-felt +/// trace filler runs only for programs without `Hash16` rows (`trace.rs`), and +/// the host transcripts are built over the block hasher, never over this one. +/// Every derived hashing method (`compress`, `transcript`, `leaf` and their +/// `_out` forms) goes through [`LfmHasher::permute`], so that one refusal +/// covers the contract. The IVs stay the zero constant: they are not hashes, +/// and the executor reads `mode_iv` to build the state `admits` then refuses. +pub struct Poseidon1W16Narrow; + +impl Poseidon1W16Narrow { + #[cold] + fn no_twelve_felt_mode() -> ! { + panic!( + "the width-16 Poseidon1 socket has no twelve-felt hash: emit Instr::Hash16 \ + rows (admits refuses every twelve-felt mode)" + ) + } +} + +impl LfmHasher for Poseidon1W16Narrow { + fn permute(&self, _state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + Self::no_twelve_felt_mode() + } + + fn compress_iv(&self) -> LfmWord { + core::array::from_fn(|_| FE::zero()) + } + + fn admits(&self, _mode: HashMode, _state: &[FE; HASH_STATE_FELTS]) -> Result<(), &'static str> { + Err("the width-16 Poseidon1 socket has no twelve-felt mode; emit Instr::Hash16 rows") + } + + fn admits_width16(&self) -> Result<(), &'static str> { + Ok(()) + } +} + +/// A placeholder permutation: one round of `x ↦ (x + rc)³` followed by the +/// mixing matrix `M = I + J` (identity plus all-ones; eigenvalues 13 and 1, +/// so invertible over Goldilocks). Degree 3, one trace row per invocation. +/// +/// **NOT CRYPTOGRAPHIC — wiring placeholder only.** No diffusion analysis, no +/// round count, nothing: it is a stand-in with the right shape and degree +/// while the ecosystem hash decision is open. +pub struct TestPermutation; + +impl TestPermutation { + /// Fixed round "constants" — an odd multiplier walk; arbitrary, public. + pub fn round_constant(i: usize) -> FE { + FE::from(0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1)) + } + + /// The compress-mode capacity constants as raw u64s (the chip bakes them + /// into its constraints via `const_base`; `FE::from` reduces identically). + pub fn compress_iv_raw() -> [u64; 4] { + core::array::from_fn(|i| 0xC0DE_0000_0000_0001u64.wrapping_add(i as u64)) + } +} + +impl LfmHasher for TestPermutation { + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + // t_i = (s_i + rc_i)^3 ; out_j = t_j + Σ_i t_i (M = I + J) + let t: Vec = state + .iter() + .enumerate() + .map(|(i, s)| { + let x = s + Self::round_constant(i); + &x * &x * x + }) + .collect(); + let sum: FE = t.iter().fold(FE::zero(), |acc, x| acc + x); + core::array::from_fn(|j| &t[j] + &sum) + } + + fn compress_iv(&self) -> LfmWord { + Self::compress_iv_raw().map(FE::from) + } +} + +/// Which permutation the `LFM_HASH` chip proves — a **construction-time** +/// choice, fixed before any trace exists. +/// +/// The chips bake their hasher's round constants into their constraints, so +/// execution, trace generation and the AIR set must all agree (`proof.rs` +/// enforces that by construction: one kind reaches all three). This enum is +/// what carries the agreement, and it is threaded rather than global so a +/// single process can prove under both. +/// +/// ⚠ **`Test` is the default and the machine's real hash is UNDECIDED.** The +/// default exists so every pre-decision call site keeps proving what it always +/// proved; it is not a statement that `TestPermutation` is the machine's hash. +/// The ecosystem hash decision is what the candidate columns feed. +/// +/// The discriminants are written out and `#[repr(u8)]` because [`as_tag`] feeds +/// `lfm_program_id`'s preimage: the wire value must never follow declaration +/// order, or inserting a variant would silently move every program digest. +/// Values 1 and 3 belonged to the retired Poseidon-original and RPO arms and +/// are never reused. +/// +/// [`as_tag`]: HasherKind::as_tag +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[repr(u8)] +pub enum HasherKind { + /// [`TestPermutation`] — NOT cryptographic. One degree-3 round. + #[default] + Test = 0, + /// [`super::blake3_socket::Blake3Permutation`] — BLAKE3 behind the Option-A + /// 2-to-1 compress socket, `compress` only. + /// + /// The one candidate here that is a real, standard, externally anchored + /// hash: at the default `SOCKET_ROUNDS = 7` a compress is literally + /// `blake3::hash(a ‖ b ‖ "LFMC")` truncated to 128 bits. It is also the one + /// with a restricted domain — no `permute` socket, and `u32` lanes — which + /// [`LfmHasher::admits`] enforces. + Blake3 = 2, + /// [`super::rpx::Rpx256`] — Rescue-Prime eXtended (XHash12), width 12, + /// `FB E FB E FB E M`. + /// + /// RPO's geometry with RPO's constants and a different round schedule: + /// three of the seven rounds trade the ~2^63-dense inverse S-box for a + /// seventh power in the degree-3 EXTENSION field, and the last round is + /// linear. Cheaper on the host and narrower in the AIR than RPO, at a + /// weaker provenance — miden publishes no known-answer table for it, so + /// [`super::rpo`]'s known-answer vectors anchor the constants the two + /// share. + Rpx = 4, + /// ZisK's Poseidon1 at width 16 (`crypto::hash::poseidon1_w16`) behind + /// the width-16 socket ([`super::p1w16_socket`]): four cells in, four out, + /// one mode, `Instr::Hash16` rows only. The recursion's verifier of + /// Poseidon1 base proofs (P3) proves under it; nothing else does. + Poseidon1W16 = 5, +} + +impl HasherKind { + /// The stable one-byte tag bound into `lfm_program_id`. + /// + /// A new candidate takes the next unused value and never reuses a retired + /// one: a tag collision would give two different permutations one program + /// identity, which is the whole thing this binding exists to prevent. + pub const fn as_tag(self) -> u8 { + self as u8 + } +} + +impl LfmHasher for HasherKind { + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.permute(state), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.permute(state), + HasherKind::Rpx => super::rpx::Rpx256.permute(state), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.permute(state), + } + } + + fn compress_iv(&self) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.compress_iv(), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.compress_iv(), + HasherKind::Rpx => super::rpx::Rpx256.compress_iv(), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.compress_iv(), + } + } + + /// Delegated explicitly, for the same reason `compress_out` is: RPX's + /// transcript CAPACITY differs from its compress one, + /// and a dispatch that fell through to this enum's own `compress_iv` would + /// hand the executor and the trace filler a capacity the chip does not + /// constrain — a host/chip disagreement, not a wrong answer the chip + /// catches. + fn transcript_iv(&self) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.transcript_iv(), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.transcript_iv(), + HasherKind::Rpx => super::rpx::Rpx256.transcript_iv(), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.transcript_iv(), + } + } + + /// Delegated explicitly, same reason as [`HasherKind::transcript_iv`]. + fn leaf_iv(&self) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.leaf_iv(), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.leaf_iv(), + HasherKind::Rpx => super::rpx::Rpx256.leaf_iv(), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.leaf_iv(), + } + } + + /// Delegated explicitly rather than left to the trait default: a candidate + /// that overrides `compress` must be honoured through this dispatch too. + fn compress(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.compress(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.compress(a, b), + HasherKind::Rpx => super::rpx::Rpx256.compress(a, b), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.compress(a, b), + } + } + + /// Delegated explicitly, for the same reason `compress` is: BLAKE3 + /// overrides it, and a default that quietly permuted instead would write + /// twelve felts its own AIR pins to four. + fn compress_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.compress_out(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.compress_out(a, b), + HasherKind::Rpx => super::rpx::Rpx256.compress_out(a, b), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.compress_out(a, b), + } + } + + /// Delegated explicitly, third time for the same reason: BLAKE3 is the one + /// candidate whose transcript domain differs from its compress domain, and + /// a dispatch that fell through to the trait default would hash a + /// transcript step under the MERKLE tag while its AIR proved the transcript + /// one — a host/chip disagreement, not a wrong answer the chip catches. + fn transcript_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.transcript_out(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.transcript_out(a, b), + HasherKind::Rpx => super::rpx::Rpx256.transcript_out(a, b), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.transcript_out(a, b), + } + } + + fn transcript(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.transcript(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.transcript(a, b), + HasherKind::Rpx => super::rpx::Rpx256.transcript(a, b), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.transcript(a, b), + } + } + + /// Delegated explicitly, fourth time for the same reason: BLAKE3's leaf mode + /// is an ENCODING, not just a tag, so a dispatch that fell through to the + /// trait default would hash four felts as a digest cell — a host answer no + /// chip proves. + fn leaf_out(&self, acc: &LfmWord, felts: &LfmWord) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.leaf_out(acc, felts), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.leaf_out(acc, felts), + HasherKind::Rpx => super::rpx::Rpx256.leaf_out(acc, felts), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.leaf_out(acc, felts), + } + } + + fn leaf(&self, acc: &LfmWord, felts: &LfmWord) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.leaf(acc, felts), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.leaf(acc, felts), + HasherKind::Rpx => super::rpx::Rpx256.leaf(acc, felts), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.leaf(acc, felts), + } + } + + fn admits(&self, mode: HashMode, state: &[FE; HASH_STATE_FELTS]) -> Result<(), &'static str> { + match self { + HasherKind::Test => TestPermutation.admits(mode, state), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.admits(mode, state), + HasherKind::Rpx => super::rpx::Rpx256.admits(mode, state), + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.admits(mode, state), + } + } + + fn admits_width16(&self) -> Result<(), &'static str> { + match self { + HasherKind::Poseidon1W16 => Poseidon1W16Narrow.admits_width16(), + _ => Err( + "this hasher's LFM_HASH socket is twelve felts wide; a width-16 row needs HasherKind::Poseidon1W16", + ), + } + } +} diff --git a/prover/src/lfm/hint_on_tests.rs b/prover/src/lfm/hint_on_tests.rs new file mode 100644 index 000000000..5f65172d3 --- /dev/null +++ b/prover/src/lfm/hint_on_tests.rs @@ -0,0 +1,390 @@ +//! `Instr::HintOn`: a hint whose arena word the executor picks on a bit. +//! +//! The bit is a schedule dependency and never a read: the hint row is a plain +//! `LFM_HINT` row, the bit's multiplicity counts no `HintOn`, and every +//! executor (serial, level, streamed) runs the hint after the bit's writer. + +use crate::tables::types::FE; +use crypto::hash::poseidon1_stark::{self as zisk, Merkle4}; +use stark::proof::options::GoldilocksCubicProofOptions; + +use super::builder::{Bit, Cell, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::WrapDigest; +use super::exec_schedule; +use super::executor::{LfmExecError, StreamedExecution, execute, execute_serial}; +use super::hash::HasherKind; +use super::instr::{Addr, ArenaId, Instr}; +use super::p1w16_emit::{hint_path_rotated, walk4}; +use super::proof::{LfmProof, lfm_prove, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::validator::{LfmViolation, validate}; +use super::word::{LfmWord, base_word}; + +const SOCKET: HasherKind = HasherKind::Poseidon1W16; + +fn word(v: u64) -> LfmWord { + [ + FE::from(v), + FE::from(v + 1), + FE::from(v + 2), + FE::from(v + 3), + ] +} + +/// The published words of `program` over `arenas`, run serially and by +/// levels: the two must agree. +fn run(program: &LfmProgram, arenas: &[Vec]) -> Result, LfmExecError> { + let serial = execute_serial(program, arenas, &SOCKET).map(|e| e.public_words); + let levels = execute(program, arenas, &SOCKET).map(|e| e.public_words); + assert_eq!( + serial.is_ok(), + levels.is_ok(), + "serial {serial:?} against levels {levels:?}" + ); + let words = serial?; + assert_eq!(words, levels?, "both schedules publish the same words"); + Ok(words.into_iter().map(|(_, w)| w).collect()) +} + +/// Two bits of a hinted felt (low first), then a `HintOn` on each over a +/// two-word arena: word 0 on a clear bit, word 1 on a set one. +fn two_bit_program() -> (LfmProgram, [Bit; 2]) { + let mut b = LfmBuilder::new(); + let a_idx = b.declare_arena(1); + let a_words = b.declare_arena(2); + let idx = b.hint_felt(a_idx, 0); + let bits = b.bit_dec(idx, 2); + for bit in &bits { + let c = b.hint_word_on(a_words, 0, *bit, 1); + b.public(c); + } + let program = compile(b.finish()); + (program, [bits[0], bits[1]]) +} + +/// ★ A `HintOn` reads `index` on a clear bit and `alt` on a set one, under +/// both schedules; it is an `LFM_HINT` row, and its bit's multiplicity counts +/// no read from it (the program validates, every multiplicity recounted). +#[test] +fn a_hint_on_reads_its_alternative_on_a_set_bit() { + let (program, bits) = two_bit_program(); + validate(&program).expect("admissible: the bits' multiplicities count no HintOn"); + for bit in bits { + let mult = program.instrs.iter().find_map(|i| match i { + Instr::BitDec { bits, .. } => { + bits.iter().find(|(a, _)| *a == bit.addr()).map(|(_, m)| *m) + } + _ => None, + }); + assert_eq!(mult, Some(0), "no instruction reads the bit"); + } + assert_eq!( + program.groups.hint.real_rows, 3, + "the index and two HintOn rows" + ); + let words = vec![word(10), word(20)]; + for (idx, want) in [ + (0u64, [word(10), word(10)]), + (1, [word(20), word(10)]), + (2, [word(10), word(20)]), + (3, [word(20), word(20)]), + ] { + let got = + run(&program, &[vec![base_word(FE::from(idx))], words.clone()]).expect("executes"); + assert_eq!(got, want.to_vec(), "index {idx}"); + } +} + +/// A `HintOn` on a cell that is not a bit has no execution. +#[test] +fn a_hint_on_a_non_boolean_cell_is_refused() { + let mut b = LfmBuilder::new(); + let a = b.declare_arena(2); + let v = b.hint_felt(a, 0); + let c = b.hint_word_on(a, 0, Bit(v.addr()), 1); + b.public(c); + let program = compile(b.finish()); + let arena = vec![base_word(FE::from(2u64)), word(5)]; + assert!(matches!( + execute_serial(&program, &[arena], &SOCKET), + Err(LfmExecError::NonBooleanBit(_)) + )); +} + +/// The validator holds a `HintOn`'s bit to a read's order (written, and +/// before the hint) and both its indices to the arena. +#[test] +fn the_validator_refuses_a_hint_on_out_of_order_or_out_of_bounds() { + let (program, _) = two_bit_program(); + let at = program + .instrs + .iter() + .position(|i| matches!(i, Instr::HintOn { .. })) + .expect("a HintOn"); + let num_addrs = program.num_addrs; + let mutate = |f: &dyn Fn(&mut Instr)| { + let (mut p, _) = two_bit_program(); + f(&mut p.instrs[at]); + validate(&p) + }; + let own_out = |i: &mut Instr| { + if let Instr::HintOn { bit, out, .. } = i { + *bit = *out; + } + }; + assert!(matches!( + mutate(&own_out), + Err(LfmViolation::CyclicRead { .. }) + )); + let unwritten = |i: &mut Instr| { + if let Instr::HintOn { bit, .. } = i { + *bit = Addr(num_addrs + 7); + } + }; + assert!(matches!( + mutate(&unwritten), + Err(LfmViolation::AddressOutOfRange { .. } | LfmViolation::ReadOfUnwritten { .. }) + )); + let alt_past = |i: &mut Instr| { + if let Instr::HintOn { alt, .. } = i { + *alt = 2; + } + }; + assert!(matches!( + mutate(&alt_past), + Err(LfmViolation::ArenaOutOfBounds { index: 2, .. }) + )); +} + +/// A bit that only exists after a permutation: a hinted 0/1 felt routed +/// through the first lane of a `Hash16` output times zero, so its writer sits +/// a level deep. Then a `HintOn` on it, over its own arena. +fn deep_bit_program() -> LfmProgram { + let mut b = LfmBuilder::new(); + let a_x = b.declare_arena(1); + let a_words = b.declare_arena(2); + let x = b.hint_felt(a_x, 0); + let h = b.hash16([x.as_cell(); 4]); + let lane = b.unpack(h[0])[0]; + let zero = b.felt_const(FE::zero()); + let y = b.mul_add(lane, zero, x); + let bit = b.bit_dec(y, 1)[0]; + let c: Cell = b.hint_word_on(a_words, 0, bit, 1); + b.public(c); + compile(b.finish()) +} + +/// ★ The level schedule puts a `HintOn` at or after its bit's level (the bit +/// is a schedule dependency), so both schedules execute the deep bit's hint. +#[test] +fn a_hint_on_runs_after_a_bit_drawn_from_a_permutation() { + let program = deep_bit_program(); + validate(&program).expect("admissible"); + let levels = exec_schedule::build(&program); + let level_of = |pred: &dyn Fn(&Instr) -> bool| { + (0..levels.levels()) + .find(|&d| { + levels + .others_at(d) + .iter() + .any(|&i| pred(&program.instrs[i as usize])) + }) + .expect("scheduled") + }; + let bit_level = level_of(&|i| matches!(i, Instr::BitDec { .. })); + let hint_level = level_of(&|i| matches!(i, Instr::HintOn { .. })); + assert!(bit_level >= 1, "the bit is a permutation deep"); + assert!(hint_level >= bit_level, "the hint runs after its bit"); + let words = vec![word(30), word(40)]; + for (x, want) in [(0u64, word(30)), (1, word(40))] { + let got = run(&program, &[vec![base_word(FE::from(x))], words.clone()]).expect("executes"); + assert_eq!(got, vec![want], "x = {x}"); + } +} + +/// ★ Streamed, the `HintOn` waits for its bit's group as well as its arena's: +/// landing its arena first runs nothing that reads the bit, and the execution +/// then equals the serial one. +#[test] +fn a_streamed_hint_on_waits_for_its_bits_group() { + let program = deep_bit_program(); + let arenas = [vec![base_word(FE::from(1u64))], vec![word(30), word(40)]]; + let serial = execute_serial(&program, &arenas, &SOCKET).expect("executes"); + for order in [[1usize, 0], [0, 1]] { + let mut ex = StreamedExecution::new(&program, vec![0, 1], &SOCKET).expect("starts"); + for g in order { + ex.land(g, std::slice::from_ref(&arenas[g])) + .unwrap_or_else(|e| panic!("order {order:?}: group {g} lands: {e:?}")); + } + let done = ex.finish().expect("finishes"); + assert_eq!( + done.public_words, serial.public_words, + "order {order:?}: the serial words" + ); + } +} + +fn digest(seed: u64, i: usize) -> [FE; 4] { + core::array::from_fn(|j| { + let mut z = seed + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add((4 * i + j) as u64 + 1); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + FE::from(z ^ (z >> 31)) + }) +} + +/// The path's sibling triples in child order, as `zisk` lists them. +fn child_order(path: &[[FE; zisk::SIBLING_FELTS]]) -> Vec { + path.iter() + .flat_map(|level| (0..3).map(move |k| core::array::from_fn(|j| level[4 * k + j]))) + .collect() +} + +/// How a 4-ary walk's sibling words are hinted. +#[derive(Clone, Copy)] +enum HintMode { + /// Production: `hint_path_rotated`. + Rotated, + /// Each `HintOn` with `index` and `alt` swapped. + Swapped, + /// Each `HintOn` a plain `Hint` of its `index` word. + Plain, +} + +fn walk_hints(b: &mut LfmBuilder, a: ArenaId, bits: &[Bit], mode: HintMode) -> Vec { + if let HintMode::Rotated = mode { + return hint_path_rotated(b, a, 0, bits); + } + let mut out = Vec::new(); + let mut w = 0u32; + for pair in bits.chunks(2) { + let [_, b1] = pair else { + panic!("even depth only") + }; + for k in 0..3 { + let (index, alt) = (w + k, w + (k + 2) % 3); + let c = match mode { + HintMode::Swapped => b.hint_word_on(a, alt, *b1, index), + HintMode::Plain => b.hint_word(a, index), + HintMode::Rotated => unreachable!(), + }; + out.push(WrapDigest::from_cell(c)); + } + w += 3; + } + out +} + +/// A 4-ary walk of `depth` levels from a hinted leaf and index, its sibling +/// words hinted as `mode` says, publishing the walked root's first felt word. +fn walk_program(depth: usize, mode: HintMode) -> LfmProgram { + let mut b = LfmBuilder::new(); + let a_leaf = b.declare_arena(1); + let a_idx = b.declare_arena(1); + let a_path = b.declare_arena(3 * (depth / 2) as u32); + let leaf = WrapDigest::from_cell(b.hint_word(a_leaf, 0)); + let idx = b.hint_felt(a_idx, 0); + let bits = b.bit_dec(idx, depth); + let h = walk_hints(&mut b, a_path, &bits, mode); + let walked = walk4(&mut b, leaf, &bits, &h); + b.public(walked.cells()[0]); + let program = compile(b.finish()); + validate(&program).expect("admissible"); + program +} + +/// ★ `HintOn` at the AIR, not the executor: +/// 1. a rotated 4-ary walk (`hint_path_rotated` + `walk4`) proves and verifies +/// and publishes the tree's root; +/// 2. the program with every `HintOn` a plain `Hint`, or with its `index` and +/// `alt` swapped, is the same circuit (equal program ids), so the AIR binds +/// neither the bit nor the arena choice; +/// 3. so a proof of the swapped program over an arena rearranged to match +/// verifies under the honest program's artifacts and publishes the same +/// root: the freedom over the hint order is real and buys nothing; +/// 4. a wrong leaf walks to another root, and the honest root claimed for +/// that proof is refused. +#[test] +fn a_hint_on_is_a_hint_row_and_a_rotated_walk_proves() { + let depth = 4; + let n = 1usize << depth; + let leaves: Vec<[FE; 4]> = (0..n).map(|i| digest(7, i)).collect(); + let tree = Merkle4::new(&leaves).expect("non-empty"); + let root = tree.root(); + // An index whose b1 bits are both set, so the rotation is live on every level. + let index = 0b1010; + let path = child_order(&tree.path(index).expect("leaf")); + let arenas = vec![ + vec![leaves[index]], + vec![base_word(FE::from(index as u64))], + path.clone(), + ]; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup 2"); + let rotated = walk_program(depth, HintMode::Rotated); + let swapped = walk_program(depth, HintMode::Swapped); + let plain = walk_program(depth, HintMode::Plain); + let art = build_artifacts_with_hasher(&rotated, &opts, SOCKET); + let id = |p: &LfmProgram| build_artifacts_with_hasher(p, &opts, SOCKET).program_id; + assert_eq!( + id(&swapped), + art.program_id, + "swapped index/alt: same circuit" + ); + assert_eq!(id(&plain), art.program_id, "plain Hint: same circuit"); + + let proved = lfm_prove(&rotated, &art, &arenas, &opts).expect("honest proves"); + assert_eq!(proved.public_words[0].1, root, "the walked root"); + let verify = |proof: &LfmProof, public: &[(u32, LfmWord)]| { + verify_against( + &art.roots, + &art.program_id, + art.keccak_rnd_chunks, + &proof.proof, + public, + &opts, + art.hasher, + art.chip_set, + ) + }; + assert!(verify(&proved, &proved.public_words), "honest verifies"); + + // At each level with b1 = 1 the rotated program reads hint k from word + // (k + 2) % 3 and the swapped one from word k; with b1 = 0 the other way + // round. Rearrange each triple so the swapped program reads what the + // rotated one read. + let mut rearranged = path.clone(); + for (level, t) in rearranged.chunks_exact_mut(3).enumerate() { + let b1 = (index >> (2 * level + 1)) & 1; + let orig = [t[0], t[1], t[2]]; + for k in 0..3 { + let want = if b1 == 1 { orig[(k + 2) % 3] } else { orig[k] }; + let slot = if b1 == 1 { k } else { (k + 2) % 3 }; + t[slot] = want; + } + } + let arenas2 = vec![arenas[0].clone(), arenas[1].clone(), rearranged]; + let proved2 = lfm_prove(&swapped, &art, &arenas2, &opts).expect("rearranged proves"); + assert_eq!(proved2.public_words[0].1, root, "same root"); + assert!( + verify(&proved2, &proved2.public_words), + "a proof of the swapped program verifies under the honest program's artifacts" + ); + + let mut bad = arenas.clone(); + bad[0][0][0] += FE::one(); + let proved3 = lfm_prove(&rotated, &art, &bad, &opts).expect("executes: no root compare"); + assert_ne!( + proved3.public_words[0].1, root, + "another leaf, another root" + ); + let mut claim = proved3.public_words.clone(); + claim[0].1 = root; + assert!( + !verify(&proved3, &claim), + "the honest root is refused for it" + ); +} diff --git a/prover/src/lfm/instr.rs b/prover/src/lfm/instr.rs new file mode 100644 index 000000000..d95b1113f --- /dev/null +++ b/prover/src/lfm/instr.rs @@ -0,0 +1,472 @@ +//! The LFM instruction set: eight algebra-shaped operations, no control flow. +//! +//! Addresses are dense indices assigned in emission order, so every operand +//! address is strictly below its destination (acyclicity by construction — +//! the validator re-checks it anyway). Every write carries its statically +//! known read count `mult`, backfilled by the compiler from the builder's +//! read counters. There is no pc, no branch, no computed address, no halt: +//! the program is a straight line and the dataflow is the execution. +//! +//! Assertions are deliberately not an instruction: `assert_eq(a, b)` lowers +//! to `diff = a - b; _ = diff / ZERO` under the division convention +//! `0/0 = 1, x/0 = error` — the AIR's division constraint `in2·out = in1` +//! with `in2 = 0` forces `in1 = 0`, and the executor errors on a nonzero +//! numerator. + +use crate::tables::types::FE; + +use super::layout; + +/// A write-once memory cell address (dense index into the address space). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Addr(pub u64); + +/// Identifies a host-supplied arena (id-addressed word sequence). +pub type ArenaId = u32; + +/// Base-field ALU operations. `MulAdd` computes `a·b + c` (the Horner step). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BaseOp { + Add, + Sub, + Mul, + Div, + MulAdd, +} + +/// Fp3 ALU operations, on word lanes 0–2. `MulAdd` is `a·b + c`; `MulBase` +/// multiplies an extension element by a base element (3 base muls, not 9). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExtOp { + Add, + Sub, + Mul, + Div, + MulAdd, + MulBase, +} + +/// The four hash-chiplet modes. `Compress`: two digest cells → one digest +/// cell. `Transcript`: the same shape in the Fiat–Shamir domain. `Leaf`: one +/// cell of four FIELD ELEMENTS → one digest cell. `Permute`: three state cells +/// → three state cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HashMode { + Compress, + /// One step of the Fiat–Shamir transcript chain. + /// + /// Structurally identical to [`HashMode::Compress`] — two cells in, one + /// cell out, same socket, same columns — and DIFFERENT in exactly one + /// thing: the hash domain. Under BLAKE3 the row's domain tag is the + /// message word immediately after the lanes (`m[12]` at the RATE-4 lane + /// count), selected by the preprocessed mode columns, so a + /// transcript step cannot be replayed as a Merkle parent or the reverse. + /// Hashers with a single domain (`Test`, `Poseidon`) compute the same + /// function in both modes; the separation is a property of the hasher, not + /// of the machine. + Transcript, + /// A Merkle LEAF over four arbitrary field elements. + /// + /// **This mode implies felt-input semantics**, by decision rather than by + /// inference. The other modes read both their input cells as digests — four + /// `u32` lanes each; this one reads its FIRST cell that way, as a chaining + /// accumulator, and its SECOND as four Goldilocks elements, splitting each + /// into a checked `lo`/`hi` `u32` pair so eight halves fill the message + /// lanes above the accumulator. That is what lets FRI data — LDE evaluations + /// and folded extension elements, none of them `u32` — reach a hash whose + /// inputs must be `u32`, and it absorbs four felts per compression because + /// the chaining rides in the message rather than in a separate fold. + /// + /// It is also what retires obligation O5 — **under a hasher that separates + /// the domains.** BLAKE3 does: a leaf is `BLAKE3(…‖"LFML")` and a parent is + /// `BLAKE3(…‖"LFMC")`, so an internal node cannot be replayed as a leaf + /// whatever the tree's shape, where before that rested on every eDSL circuit + /// being fixed-depth — true, but enforced by nothing. + /// + /// ⚠ The mode is a machine-level shape, not a guarantee. A single-domain + /// hasher computes the same function in both modes, so under `Test` and + /// `Poseidon` O5 still rests on fixed depth exactly as it did before. The + /// separation is a property of the HASHER; see `LfmHasher::leaf_out`. + Leaf, + Permute, +} + +impl HashMode { + /// Whether this mode is the two-cells-in, one-cell-out shape — true for + /// `Compress` and `Transcript`, which differ only in hash domain. + /// + /// Every place that used to match `Compress` for a *shape* reason routes + /// through here, so adding a domain cannot silently take the permute arm. + pub const fn is_two_to_one(self) -> bool { + matches!(self, HashMode::Compress | HashMode::Transcript) + } + + /// Input cells this mode reads from memory: 2 or 3. + /// + /// The `LFM_HASH` bus receives are gated by exactly this, so a mode that + /// reads fewer cells must not receive the ones it does not read — a row + /// receiving a cell it never reads would claim a memory read it never makes. + /// + /// A `Leaf` reads TWO: its chaining accumulator, then the four felts it + /// absorbs. It read one until the leaf RATE put the accumulator in the + /// message rather than in a separate `"LFMC"` fold (COMMIT.md §1.2), which + /// is what took leaf absorption from 2 felts per compression to 4. + pub const fn num_input_cells(self) -> usize { + match self { + HashMode::Compress | HashMode::Transcript | HashMode::Leaf => 2, + HashMode::Permute => 3, + } + } + + /// Output cells this mode writes: 1 for every hashing mode, 3 for a + /// permutation. + pub const fn num_output_cells(self) -> usize { + match self { + HashMode::Permute => 3, + _ => 1, + } + } +} + +/// Operands of a [`Instr::KeccakF`]: 13 words of `u32`-half keccak state in, +/// 13 out, plus each output's static read count. +/// +/// The state's 25 `u64` lanes are not felt-representable (values in `[p, 2^64)` +/// exist), so they travel as 50 `u32` halves packed four to a word; the last +/// word's top two lanes are unused and must be zero. The permutation itself is +/// proved by the production `KECCAK_RND` / `KECCAK_RC` / `BITWISE` chips — +/// `LFM_KECCAK` only binds these words to the `Keccak` bus tokens. +#[derive(Debug, Clone)] +pub struct KeccakOperands { + pub mode: KeccakMode, + pub ins: [Addr; 13], + /// Rate-block words, read only in [`KeccakMode::Absorb`]; `Addr(0)` + /// placeholders otherwise (the receives are gated by the mode selector, so + /// the placeholders are never read). + pub block: [Addr; 9], + pub outs: [Addr; 13], + pub mults: [u64; 13], + /// When set, the row ALSO writes the byte-reversed first 32 bytes of the + /// output state as two words — the production transcript's `sample()`. + pub rev: Option, +} + +/// The reversed-digest outputs of a keccak row (see `layout::keccak::REV_ADDR0`). +#[derive(Debug, Clone)] +pub struct KeccakReversedDigest { + pub outs: [Addr; 2], + pub mults: [u64; 2], +} + +/// The adapter's two modes. +/// +/// `Permute` is the bare permutation (R1b). `Absorb` XORs a 136-byte rate block +/// into the state's rate region first — the sponge step, with the XOR done by +/// `BYTE_ALU[XOR]` lookups into the same BITWISE table the round chip uses. +/// Doing the XOR here rather than on the LFM side (bit-decompose, recombine) is +/// orders of magnitude cheaper: the adapter already owns byte-granular columns +/// and already talks to BITWISE. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeccakMode { + Permute, + Absorb, +} + +/// Operands of an [`Instr::Blake3`]: 7 words of `u32` input in, 4 words of +/// compression output out, plus each output's static read count. +/// +/// The 28 input `u32` words are `h[8] ‖ m[16] ‖ t_lo ‖ t_hi ‖ block_len ‖ +/// flags`, four to a machine word — so word 0–1 carry `h`, words 2–5 carry `m`, +/// and word 6 is `(t_lo, t_hi, block_len, flags)`. 28 divides by 4 exactly, +/// so unlike [`KeccakOperands`] there are no unused lane slots. Every lane of +/// every input word must be a canonical value below `2^32`. +/// +/// Unlike `KeccakF`, the compression is proved **here** — `LFM_BLAKE3` carries +/// its own AIR — rather than delegated to a hosted family, so there is no tag +/// binding a request token to a reply token and nothing for the admission +/// validator to check for uniqueness. +#[derive(Debug, Clone)] +pub struct Blake3Operands { + pub ins: [Addr; layout::blake3::IN_WORDS], + pub outs: [Addr; layout::blake3::OUT_WORDS], + pub mults: [u64; layout::blake3::OUT_WORDS], + /// When set, the row ALSO writes the byte-reversed 32-byte digest as two + /// words — the production transcript's `sample()`. Free on the bus (see + /// `layout::blake3::REV_ADDR0`). + pub rev: Option, +} + +/// One width-16 Poseidon1 permutation (`super::p1w16_socket`): four input +/// words read, all sixteen lanes permuted, four output words written. Every +/// output is allocated; one nobody reads takes multiplicity zero. +#[derive(Debug, Clone)] +pub struct Hash16Operands { + pub ins: [Addr; 4], + pub outs: [Addr; 4], + pub mults: [u64; 4], +} + +/// The reversed-digest outputs of a BLAKE3 row (see `layout::blake3::REV_ADDR0`). +#[derive(Debug, Clone)] +pub struct Blake3ReversedDigest { + pub outs: [Addr; layout::blake3::DIGEST_WORDS], + pub mults: [u64; layout::blake3::DIGEST_WORDS], +} + +/// One LFM instruction. Operand-field conventions: +/// +/// - `c` on the ALU ops is meaningful iff the op is `MulAdd` (and is emitted +/// as address 0 otherwise — the corresponding bus receive is gated by the +/// `MulAdd` selector, so the placeholder is never read). +/// - `Hash` uses `ins[..num_input_cells()]` and `outs[..num_output_cells()]` +/// only — 2/1 for `Compress` and `Transcript`, 1/1 for `Leaf`, 3/3 for +/// `Permute`; the remaining slots are `Addr(0)` placeholders with `mults` +/// fixed to 0, and the validator checks both ends. +/// - `BitDec.bits` lists, low-to-high from bit 0, exactly the bit cells the +/// program consumes; bits beyond `bits.len()` exist as constrained witness +/// columns but get no memory cell. +#[derive(Debug, Clone)] +pub enum Instr { + Const { + out: Addr, + value: [FE; 4], + mult: u64, + }, + BaseAlu { + op: BaseOp, + out: Addr, + a: Addr, + b: Addr, + c: Addr, + mult: u64, + }, + ExtAlu { + op: ExtOp, + out: Addr, + a: Addr, + b: Addr, + c: Addr, + mult: u64, + }, + Select { + bit: Addr, + out_l: Addr, + out_r: Addr, + in_l: Addr, + in_r: Addr, + mult_l: u64, + mult_r: u64, + }, + BitDec { + input: Addr, + bits: Vec<(Addr, u64)>, + /// The value's two BIG-ENDIAN `u32` halves as output cells — + /// `[high-word half, low-word half]`, i.e. what + /// `append_field_element` puts on the wire. `None` for a plain + /// decomposition. The halves are linear forms over the bit columns, + /// so they cost no extra row and no ALU work. + halves: Option<[(Addr, u64); 2]>, + }, + Hash { + mode: HashMode, + ins: [Addr; 3], + outs: [Addr; 3], + mults: [u64; 3], + }, + Hint { + arena: ArenaId, + index: u32, + out: Addr, + mult: u64, + }, + /// A `Hint` whose arena word is `index` when the cell at `bit` holds 0 + /// and `alt` when it holds 1: the same `LFM_HINT` row. + /// + /// Which word is executor-only, like a hint's index: the row carries the + /// output address and multiplicity and nothing else, and a hinted value is + /// free witness, so the choice changes no constraint. The bit is therefore + /// a SCHEDULE dependency ([`Instr::schedule_reads_into`]) and never a read + /// ([`Instr::reads_into`], whose recount sets the bit's multiplicity): a + /// bus read the chip does not make would unbalance it. + HintOn { + arena: ArenaId, + index: u32, + alt: u32, + bit: Addr, + out: Addr, + mult: u64, + }, + /// Assemble a word from four base cells (unused lanes take the shared + /// zero-constant cell). The lane↔word coupling is enforced purely by the + /// `LFM_LANES` chip's bus tokens — no constraints. + Pack { + lanes: [Addr; 4], + out: Addr, + mult: u64, + }, + /// Split a word into four base cells — the only way a hash-state or + /// digest lane can reach the ALU (discovered as a real ISA gap in + /// Milestone C: challenges are squeezed as cells but consumed as felts). + Unpack { + input: Addr, + outs: [Addr; 4], + mults: [u64; 4], + }, + /// One `keccak-f[1600]` permutation over 13 words of `u32`-half state. + /// + /// Boxed: the 13-wide operand arrays are 312 bytes, four times the next + /// largest variant, and inlining them would quadruple every instruction in + /// the program vector. + KeccakF(Box), + /// One BLAKE3 compression over 7 input words, writing 4 output words. + /// + /// Boxed for the reason `KeccakF` is: the operand arrays are ~150 bytes, + /// twice the next largest variant, and inlining them would grow every + /// instruction in the program vector. + Blake3(Box), + /// One width-16 Poseidon1 permutation, the `LFM_HASH` row of a program + /// under `HasherKind::Poseidon1W16`. A program carries `Hash` rows or + /// `Hash16` rows, never both. + /// + /// Boxed for the reason `KeccakF` is: the 12-felt `Hash` is the default + /// path, and an inline 4-wide variant would grow every instruction of every + /// program. + Hash16(Box), + Public { + addr: Addr, + index: u32, + }, +} + +impl Instr { + /// The addresses this instruction writes, appended to `out` in ascending + /// order. + /// + /// ⛔ **This, not [`Instr::writes`], is where the operand conventions are + /// written down.** A whole-program dataflow pass visits every instruction + /// once and cannot afford a `Vec` per visit — 5.3–7.4 M allocations per + /// recursion proof — but a private copy of "which operands are live under + /// which selector" is exactly the kind of duplicate that drifts and then + /// reports a dependency structure the machine does not have. So the caller + /// brings its own buffer and there is still only one transcription. + /// + /// Does NOT clear `out`; the caller owns it. + pub fn writes_into(&self, out: &mut Vec) { + match self { + Instr::Const { out: a, .. } + | Instr::BaseAlu { out: a, .. } + | Instr::ExtAlu { out: a, .. } + | Instr::Hint { out: a, .. } + | Instr::HintOn { out: a, .. } + | Instr::Pack { out: a, .. } => out.push(*a), + Instr::Unpack { outs, .. } => out.extend_from_slice(outs), + Instr::KeccakF(k) => { + out.extend_from_slice(&k.outs); + if let Some(rev) = &k.rev { + out.extend_from_slice(&rev.outs); + } + } + Instr::Blake3(k) => { + out.extend_from_slice(&k.outs); + if let Some(rev) = &k.rev { + out.extend_from_slice(&rev.outs); + } + } + Instr::Select { out_l, out_r, .. } => { + out.push(*out_l); + out.push(*out_r); + } + Instr::BitDec { bits, halves, .. } => { + out.extend(bits.iter().map(|(a, _)| *a)); + out.extend(halves.iter().flat_map(|hs| hs.iter().map(|(a, _)| *a))); + } + Instr::Hash { mode, outs, .. } => { + out.extend_from_slice(&outs[..mode.num_output_cells()]) + } + Instr::Hash16(h) => out.extend_from_slice(&h.outs), + Instr::Public { .. } => {} + } + } + + /// The addresses this instruction reads (meaningful operands only, per the + /// field conventions above), appended to `out`. + /// + /// The read half of [`Instr::writes_into`], and the same rule applies: this + /// is the transcription, [`Instr::reads`] is the convenience wrapper. + /// Does NOT clear `out`. + pub fn reads_into(&self, out: &mut Vec) { + match self { + Instr::Const { .. } | Instr::Hint { .. } | Instr::HintOn { .. } => {} + Instr::BaseAlu { op, a, b, c, .. } => { + out.push(*a); + out.push(*b); + if *op == BaseOp::MulAdd { + out.push(*c); + } + } + Instr::ExtAlu { op, a, b, c, .. } => { + out.push(*a); + out.push(*b); + if *op == ExtOp::MulAdd { + out.push(*c); + } + } + Instr::Select { + bit, in_l, in_r, .. + } => { + out.push(*bit); + out.push(*in_l); + out.push(*in_r); + } + Instr::BitDec { input, .. } => out.push(*input), + Instr::Hash { mode, ins, .. } => out.extend_from_slice(&ins[..mode.num_input_cells()]), + Instr::Hash16(h) => out.extend_from_slice(&h.ins), + Instr::Pack { lanes, .. } => out.extend_from_slice(lanes), + Instr::Unpack { input, .. } => out.push(*input), + Instr::KeccakF(k) => { + out.extend_from_slice(&k.ins); + if k.mode == KeccakMode::Absorb { + out.extend_from_slice(&k.block); + } + } + // Every input word is read on every row: the chip has one mode, so + // there is no gated operand and no placeholder to exclude. + Instr::Blake3(k) => out.extend_from_slice(&k.ins), + Instr::Public { addr, .. } => out.push(*addr), + } + } + + /// The addresses this instruction writes, in ascending order. + pub fn writes(&self) -> Vec { + let mut v = Vec::new(); + self.writes_into(&mut v); + v + } + + /// The addresses this instruction reads (meaningful operands only, per + /// the field conventions above). + pub fn reads(&self) -> Vec { + let mut v = Vec::new(); + self.reads_into(&mut v); + v + } + + /// The addresses whose cells must be written before this instruction + /// runs: [`Self::reads_into`], plus a [`Instr::HintOn`]'s bit, which the + /// executor reads to pick the arena word and the machine never reads. For + /// schedules only; a multiplicity counts [`Self::reads_into`]. Does NOT + /// clear `out`. + pub fn schedule_reads_into(&self, out: &mut Vec) { + self.reads_into(out); + if let Instr::HintOn { bit, .. } = self { + out.push(*bit); + } + } + + /// [`Self::schedule_reads_into`] into a fresh vector. + pub fn schedule_reads(&self) -> Vec { + let mut v = Vec::new(); + self.schedule_reads_into(&mut v); + v + } +} diff --git a/prover/src/lfm/join_tests.rs b/prover/src/lfm/join_tests.rs new file mode 100644 index 000000000..d8c59135a --- /dev/null +++ b/prover/src/lfm/join_tests.rs @@ -0,0 +1,2013 @@ +//! The DEEP/Merkle join: DEEP across a full sub-proof, folding the SAME arena +//! cells the Merkle authentication authenticates. +//! +//! ## The oracle +//! +//! Two production functions, neither of them re-derived here: +//! `reconstruct_deep_composition_poly_evaluation_pair` for the fold, and the +//! proof's own committed roots for the authentication. The fixture is a real +//! proof of a real production AIR, produced by the production prover, and its +//! query indices come from a replay of the production verifier's transcript +//! rather than from a search. +//! +//! ## What this suite cannot see +//! +//! The FRI leg that consumes `DEEP(υ)` — nothing here checks that the +//! reconstructed value is the one the folding chain expects, only that it is +//! the value the production verifier would have computed. It also cannot see +//! whether the epoch's OTHER sub-proofs compose, since a sub-proof is verified +//! in isolation here. +//! +//! DEPTH it sees only as far as the fixtures go: six levels on the +//! preprocessed fixture (64 rows at blowup 2), two on the production one. +//! `join_leg_cost` emits at depth 22 but never runs it, and R1f's +//! `keccak_merkle_walk_authenticates_a_real_opening` remains the only executed +//! walk at production depth (20, main trace only). Six levels is enough to +//! distinguish a per-level walk from a two-level one; it is not enough to catch +//! something that only appears past a word boundary in the index. + +use crypto::merkle_tree::traits::IsStreamingLeafBackend; +use math::field::traits::IsFFTField; +use stark::config::Commitment; +use stark::domain::new_verifier_domain; +use stark::proof::view::StarkProofView; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::constraint_tests::{deep_shape, open_sub_proof, real_fixture}; +use super::executor::execute; +use super::sub_proof::{ + GroupShape, ROWS_PER_LEAF, SubProofShape, emit_sub_proof, emit_sub_proof_with_bits, +}; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type V = Verifier; + +/// One committed matrix's data for one query, host side: the row pair in leaf +/// order and the path that authenticates it. +pub(super) struct HostGroupOpening { + /// `evaluations ‖ evaluations_sym`, as arena words. + values: Vec, + siblings: Vec, +} + +/// Everything the machine reads about one sub-proof, read off a real proof. +/// +/// Assembled once and shared, because `open_sub_proof` replays the whole +/// verifier transcript and the fixture proof is regenerated on every call. +pub(super) struct HostSubProof { + pub(super) shape: SubProofShape, + pub(super) gamma: FEE, + pub(super) zeta: FEE, + /// The OOD grid, row-major. + ood: Vec, + claimed_parts: Vec, + /// One root per group, in `SubProofShape::groups` order. + roots: Vec, + /// Every group's Merkle cap in group order, split off query 0's path; empty + /// when the format caps nothing. + trace_caps: Vec, + /// `[query][group]`. + openings: Vec>, + pub(super) iotas: Vec, + /// The FRI folding challenges, from the production verifier's own + /// `replay_rounds_after_round_1` (`verifier.rs:1461-1483`) — one per + /// committed layer plus the final-fold one. The FRI leg reads them; the + /// trace leg does not. + pub(super) zetas: Vec, + /// The production reconstruction's answer per query, `(regular, sym)`. + /// Row-pair shapes only (empty under one-row leaves). + pub(super) expected: Vec<(FEE, FEE)>, + /// Under one-row leaves (S2): production's DEEP at the ONE point `x_r` + /// per query (`reconstruct_deep_composition_poly_evaluation_at`). Empty + /// for row pairs. + pub(super) expected_at_r: Vec, + /// The same, asked of production with the PRECOMPUTED and MAIN slices + /// swapped — the alternative column order a fixture without a precomputed + /// group cannot distinguish. Empty when there is no precomputed group, or + /// when the two base groups are different widths (the swap would not be a + /// well-formed reading). + expected_base_swapped: Vec<(FEE, FEE)>, + /// Production's query points, kept so the machine's derivation can be + /// checked against them rather than against a local formula: `(υ, −υ)` + /// for row pairs, `(x_r, None)` under one-row leaves. + pub(super) points: Vec<(FE, Option)>, +} + +fn host_sub_proof() -> &'static HostSubProof { + use std::sync::OnceLock; + static CELL: OnceLock = OnceLock::new(); + CELL.get_or_init(|| { + let (air, proof) = real_fixture(); + build_host_sub_proof(&*air, &proof) + }) +} + +pub(super) fn build_host_sub_proof( + air: &dyn stark::traits::AIR, + proof: &stark::proof::stark::MultiProof, +) -> HostSubProof { + let sp = open_sub_proof(air, proof); + let (deep, gamma) = deep_shape(&sp, air); + let view = StarkProofView::Owned(&proof.proofs[0]); + + let (main_width, aux_width) = air.trace_layout(); + let num_precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + let mut trace_groups = Vec::new(); + if num_precomputed > 0 { + trace_groups.push(GroupShape { + num_columns: num_precomputed, + is_ext: false, + }); + } + trace_groups.push(GroupShape { + num_columns: main_width - num_precomputed, + is_ext: false, + }); + if aux_width > 0 { + trace_groups.push(GroupShape { + num_columns: aux_width, + is_ext: true, + }); + } + + let blowup = air.options().blowup_factor as usize; + let lde_length = view.trace_length() * blowup; + // The table's leaf layout — the host prover's and verifier's own + // resolution (S2: `auto` per table from the AIR's widths). + let leaf_layout = stark::leaf_layout::table_leaf_layout(air, view.trace_length()); + let merkle_depth = leaf_layout.tree_depth(lde_length.trailing_zeros() as usize); + let opts = air.options(); + let trace_cap = opts + .format + .merkle_cap + .height(opts.fri_number_of_queries, merkle_depth); + let shape = SubProofShape { + deep: deep.clone(), + trace_groups, + merkle_depth, + log2_lde_length: lde_length.trailing_zeros(), + coset_offset: FE::from(air.options().coset_offset), + trace_cap, + layout: leaf_layout, + arity: 2, + }; + // Query 0 of a capped tree is its owner: its path carries the cap after + // the `D − c` siblings. The query arena takes the siblings, the caps + // arena the caps (group order). + let mut trace_caps: Vec = Vec::new(); + let mut split = |q: usize, path: &[Commitment]| -> Vec { + if trace_cap == 0 || q != 0 { + assert_eq!( + path.len(), + merkle_depth - trace_cap, + "query {q}: a path to the cap" + ); + return path.to_vec(); + } + let (siblings, cap) = + crypto::merkle_tree::cap::split_owner_path(path, merkle_depth, trace_cap) + .expect("the owner path is D − c + 2^c long"); + trace_caps.extend_from_slice(cap); + siblings.to_vec() + }; + + let mut roots = vec![]; + if num_precomputed > 0 { + roots.push( + *view + .lde_trace_precomputed_merkle_root() + .expect("a preprocessed air commits its precomputed columns"), + ); + } + roots.push(*view.lde_trace_main_merkle_root()); + if aux_width > 0 { + roots.push(*view.lde_trace_aux_merkle_root().expect("an aux root")); + } + roots.push(*view.composition_poly_root()); + + let domain = new_verifier_domain(air, view.trace_length()); + let layout = V::ood_layout(air); + let invariants = V::compute_query_invariant_deep_terms( + &sp.challenges, + view, + &sp.ood_full, + layout.next_row_cols(), + layout.step_size(), + ) + .expect("a real proof's invariant terms"); + let generator = ::get_primitive_root_of_unity(deep.log2_trace_length as u64) + .expect("root of unity"); + + let swap_is_well_formed = + num_precomputed > 0 && main_width - num_precomputed == num_precomputed; + let mut openings = Vec::new(); + let mut expected = Vec::new(); + let mut expected_at_r = Vec::new(); + let mut expected_base_swapped = Vec::new(); + let mut points = Vec::new(); + for (q, iota) in sp.challenges.iotas.iter().enumerate() { + let o = view.deep_poly_opening(q); + let mut groups: Vec = Vec::new(); + if num_precomputed > 0 { + let p = o.precomputed_trace_polys().expect("precomputed opening"); + groups.push(HostGroupOpening { + values: p + .evaluations() + .iter() + .chain(p.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + siblings: split(q, p.merkle_path()), + }); + } + let m = o.main_trace_polys(); + groups.push(HostGroupOpening { + values: m + .evaluations() + .iter() + .chain(m.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + siblings: split(q, m.merkle_path()), + }); + if aux_width > 0 { + let a = o.aux_trace_polys().expect("aux opening"); + groups.push(HostGroupOpening { + values: a + .evaluations() + .iter() + .chain(a.evaluations_sym()) + .map(ext_word) + .collect(), + siblings: split(q, a.merkle_path()), + }); + } + let c = o.composition_poly(); + groups.push(HostGroupOpening { + values: c + .evaluations() + .iter() + .chain(c.evaluations_sym()) + .map(ext_word) + .collect(), + siblings: split(q, c.merkle_path()), + }); + openings.push(groups); + + if leaf_layout.is_one_row() { + // S2: one point, `x_r`, and DEEP there alone — production's own + // one-row functions (`query_point`, `…_evaluation_at`). + let point = V::query_point(leaf_layout, *iota, &domain); + let empty_base: &[FE] = &[]; + let want = V::reconstruct_deep_composition_poly_evaluation_at( + &point, + &generator, + &sp.challenges, + &invariants, + layout.next_row_cols(), + layout.step_size(), + o.precomputed_trace_polys() + .map(|p| p.evaluations()) + .unwrap_or(empty_base), + m.evaluations(), + o.aux_trace_polys().map(|a| a.evaluations()).unwrap_or(&[]), + c.evaluations(), + ) + .expect("a real one-row proof reconstructs"); + expected_at_r.push(want); + points.push((point, None)); + continue; + } + + let point = V::query_challenge_to_evaluation_point(*iota, false, &domain); + let point_sym = V::query_challenge_to_evaluation_point(*iota, true, &domain); + let empty_base: &[FE] = &[]; + let (want, want_sym) = V::reconstruct_deep_composition_poly_evaluation_pair( + &point, + &point_sym, + &generator, + &sp.challenges, + &invariants, + layout.next_row_cols(), + layout.step_size(), + o.precomputed_trace_polys() + .map(|p| p.evaluations()) + .unwrap_or(empty_base), + m.evaluations(), + o.aux_trace_polys().map(|a| a.evaluations()).unwrap_or(&[]), + c.evaluations(), + o.precomputed_trace_polys() + .map(|p| p.evaluations_sym()) + .unwrap_or(empty_base), + m.evaluations_sym(), + o.aux_trace_polys() + .map(|a| a.evaluations_sym()) + .unwrap_or(&[]), + c.evaluations_sym(), + ) + .expect("a real proof reconstructs"); + expected.push((want, want_sym)); + if swap_is_well_formed { + let p = o.precomputed_trace_polys().expect("precomputed opening"); + let swapped = V::reconstruct_deep_composition_poly_evaluation_pair( + &point, + &point_sym, + &generator, + &sp.challenges, + &invariants, + layout.next_row_cols(), + layout.step_size(), + m.evaluations(), + p.evaluations(), + o.aux_trace_polys().map(|a| a.evaluations()).unwrap_or(&[]), + c.evaluations(), + m.evaluations_sym(), + p.evaluations_sym(), + o.aux_trace_polys() + .map(|a| a.evaluations_sym()) + .unwrap_or(&[]), + c.evaluations_sym(), + ) + .expect("the swapped reading is well formed, so it reconstructs"); + expected_base_swapped.push(swapped); + } + points.push((point, Some(point_sym))); + } + + let ood: Vec = (0..deep.num_eval_points) + .flat_map(|r| sp.ood_full.get_row(r)[..deep.num_total_cols].to_vec()) + .collect(); + + HostSubProof { + shape, + gamma, + zeta: sp.zeta, + ood, + claimed_parts: sp.claimed_parts.clone(), + roots, + trace_caps, + openings, + iotas: sp.challenges.iotas.clone(), + zetas: sp.challenges.zetas.clone(), + expected, + expected_at_r, + expected_base_swapped, + points, + } +} + +impl HostSubProof { + /// The arenas [`emit_sub_proof`] declares, in its declaration order. + pub(super) fn arenas(&self, queries: &[usize]) -> Vec> { + let mut out = vec![ + vec![ext_word(&self.gamma), ext_word(&self.zeta)], + self.ood.iter().map(ext_word).collect(), + self.claimed_parts.iter().map(ext_word).collect(), + super::proof_arena::commitments_to_arena(&self.roots), + self.query_arena(queries), + ]; + // The caps arena, declared by the emitter only when the shape caps. + if self.shape.trace_cap > 0 { + out.push(super::proof_arena::commitments_to_arena(&self.trace_caps)); + } + out + } + + /// Per query: the index, then per group the row-pair values and the + /// sibling digests — the order the emitter's cursor walks. + pub(super) fn query_arena(&self, queries: &[usize]) -> Vec { + let mut out = Vec::new(); + for &q in queries { + out.push(base_word(FE::from(self.iotas[q] as u64))); + for group in &self.openings[q] { + out.extend(group.values.iter().copied()); + out.extend(super::proof_arena::commitments_to_arena(&group.siblings)); + } + } + out + } +} + +/// ★ Scrutinise the oracle before anything is built on it. +/// +/// Four separate premises the join rests on, each checked against the real +/// proof rather than assumed: that every group commits at the SAME depth (one +/// index addresses all four trees), that the depth is one below the LDE domain +/// (a leaf is a row pair), that the machine's point derivation reproduces +/// production's `query_challenge_to_evaluation_point` at every one of the +/// proof's indices, and that the symmetric point really is the negation. +#[test] +fn the_join_premises_hold_on_a_real_proof() { + let h = host_sub_proof(); + let s = &h.shape; + let groups = s.groups(); + + println!( + "sub-proof: {} groups {:?}, depth {}, log2(lde) {}, {} queries", + groups.len(), + groups + .iter() + .map(|g| (g.num_columns, g.is_ext)) + .collect::>(), + s.merkle_depth, + s.log2_lde_length, + h.iotas.len() + ); + + assert_eq!( + s.merkle_depth + 1, + s.log2_lde_length as usize, + "a leaf is a row pair, so the tree has one level fewer than the domain" + ); + assert_eq!( + ROWS_PER_LEAF, + stark::commitment::ROWS_PER_LEAF, + "the machine's leaf shape is a copy of the commitment layer's constant; \ + if that moves, every leaf hash and every DEEP index in this module goes \ + with it, and no differential would say so because both sides would move \ + together" + ); + for (q, per_group) in h.openings.iter().enumerate() { + for (g, opening) in per_group.iter().enumerate() { + assert_eq!( + opening.siblings.len(), + s.merkle_depth, + "query {q} group {g}: every tree must have the same depth, or \ + one index cannot address them all" + ); + assert_eq!( + opening.values.len(), + groups[g].num_values(), + "query {q} group {g}: width" + ); + } + } + + // The point derivation, run IN THE MACHINE at every one of the proof's + // indices and compared against production's own function. Recomputing the + // bit weights here instead would only check a host formula against + // production and leave the emitter unexamined — the same oracle mistake + // the method rules warn about, one level up. + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(1); + let index = b.hint_felt(arena, 0); + let bits = b.bit_dec(index, s.merkle_depth); + let (point, point_sym) = super::sub_proof::emit_query_points(&mut b, s, &bits); + b.public(point.as_cell()); + b.public(point_sym.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the point-derivation program is admissible"); + + for (q, iota) in h.iotas.iter().enumerate() { + let arenas = vec![vec![base_word(FE::from(*iota as u64))]]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the derivation executes"); + assert_eq!( + exec.public_words[0].1[0], h.points[q].0, + "query {q}: the machine's point must be \ + query_challenge_to_evaluation_point(iota, false)" + ); + assert_eq!( + exec.public_words[1].1[0], + h.points[q].1.expect("a row-pair fixture"), + "query {q}: the machine's symmetric point must be \ + query_challenge_to_evaluation_point(iota, true)" + ); + } + println!( + "in-machine point derivation checked against production at all {} indices", + h.iotas.len() + ); +} + +/// ★ The headline differential: the machine's DEEP equals the production +/// verifier's, at every query of a full sub-proof, with every opened value +/// authenticated to the proof's own committed roots in the same run. +/// +/// The authentication is not a separate assertion here — it is `assert_word_eq` +/// inside the program, so a run in which any leaf failed to reach its root +/// would not execute at all. That the run produces DEEP values is already the +/// statement that the values it folded are the committed ones. +#[test] +fn the_join_matches_the_production_verifier_on_every_query() { + let h = host_sub_proof(); + let all: Vec = (0..h.iotas.len()).collect(); + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, all.len()); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the joined sub-proof program is admissible"); + + let exec = execute(&program, &h.arenas(&all), &crate::hash_pin::BLOCK_HASHER) + .expect("an honest sub-proof must authenticate and fold"); + + let mut nonzero = 0usize; + for q in &all { + let (want, want_sym) = h.expected[*q]; + assert_eq!( + word_as_ext(&exec.public_words[2 * q].1).expect("ext"), + want, + "query {q}: DEEP at the regular point" + ); + assert_eq!( + word_as_ext(&exec.public_words[2 * q + 1].1).expect("ext"), + want_sym, + "query {q}: DEEP at the symmetric point" + ); + if want != FEE::zero() { + nonzero += 1; + } + } + assert_eq!( + nonzero, + all.len(), + "a vacuously zero reconstruction would make the differential empty" + ); + println!( + "joined sub-proof: {} queries, {} instructions, {} distinct indices", + all.len(), + program.instrs.len(), + { + let mut d = h.iotas.clone(); + d.sort_unstable(); + d.dedup(); + d.len() + } + ); +} + +// ============================================================================= +// Cost +// ============================================================================= + +/// Precomputed columns a table carries IN PRODUCTION. +/// +/// `test_utils::production_airs` builds the AIR objects without their +/// preprocessed commitments — the commitments need an ELF, a register file or a +/// page config, none of which a shape census has. `is_preprocessed()` is +/// therefore FALSE on five tables that are preprocessed in the real epoch +/// (`lib.rs`'s `VmAirs::new` wires BITWISE, DECODE, KECCAK_RC, REGISTER and +/// PAGE; `continuation.rs` wires GLOBAL_MEMORY), and reading the flag off these +/// objects would drop one opening group — one leaf hash and one path walk — +/// from each of them. +/// +/// The split is what matters here, not the commitment value: a preprocessed +/// table's columns `0..n` are committed in their own tree and the rest in the +/// main tree, so the same columns are hashed as TWO leaves instead of one. +fn production_num_precomputed( + label: &str, + air: &dyn stark::traits::AIR, +) -> usize { + use crate::tables::{bitwise, decode, keccak_rc, page, register}; + + let wired = match label { + "BITWISE" => bitwise::NUM_PRECOMPUTED_COLS, + "DECODE" => decode::NUM_PRECOMPUTED_COLS, + "KECCAK_RC" => keccak_rc::NUM_PRECOMPUTED_COLS, + "REGISTER" => register::NUM_PREPROCESSED_COLS, + "PAGE" => page::NUM_PREPROCESSED_COLS, + _ => 0, + }; + if air.is_preprocessed() { + // Already wired by the constructor (GLOBAL_MEMORY): trust the object. + assert_eq!( + wired, 0, + "{label} is wired preprocessed AND listed above; one of the two is stale" + ); + return air.num_precomputed_columns(); + } + wired +} + +/// The DEEP shape and the opening groups of a production AIR, as a sub-proof of +/// `log2_trace_length` rows at `blowup` would carry them. +fn shape_for( + air: &dyn stark::traits::AIR, + num_precomputed: usize, + log2_trace_length: u32, + log2_blowup: u32, +) -> SubProofShape { + use stark::constraint_ir::ConstraintArtifact; + + let artifact = ConstraintArtifact::capture(air); + let layout = V::ood_layout(air); + let (main_width, aux_width) = air.trace_layout(); + + let mut trace_groups = Vec::new(); + if num_precomputed > 0 { + trace_groups.push(GroupShape { + num_columns: num_precomputed, + is_ext: false, + }); + } + trace_groups.push(GroupShape { + num_columns: main_width - num_precomputed, + is_ext: false, + }); + if aux_width > 0 { + trace_groups.push(GroupShape { + num_columns: aux_width, + is_ext: true, + }); + } + + SubProofShape { + deep: super::deep::DeepShape { + step_size: layout.step_size(), + num_eval_points: artifact.shape.transition_offsets.len() * layout.step_size(), + num_total_cols: main_width + aux_width, + next_row_cols: layout.next_row_cols().to_vec(), + num_composition_parts: artifact.shape.composition_degree_multiplier as usize, + log2_trace_length, + commit_ood: None, + }, + trace_groups, + merkle_depth: (log2_trace_length + log2_blowup) as usize - 1, + log2_lde_length: log2_trace_length + log2_blowup, + coset_offset: FE::from(3u64), + trace_cap: 0, + layout: stark::leaf_layout::LeafLayout::RowPair, + arity: 2, + } +} + +fn count bool>( + program: &super::compiler::LfmProgram, + f: F, +) -> usize { + program.instrs.iter().filter(|i| f(i)).count() +} + +fn permutations(program: &super::compiler::LfmProgram) -> usize { + count(program, |i| matches!(i, super::instr::Instr::KeccakF(_))) +} + +/// Byte swaps — one `LFM_BITDEC` row each. Every field element that enters a +/// leaf hash needs one; nothing else in this leg decomposes, except the one +/// index decomposition per query. +fn bit_decs(program: &super::compiler::LfmProgram) -> usize { + count(program, |i| matches!(i, super::instr::Instr::BitDec { .. })) +} + +/// Marginal per-query cost of one shape, by emitting one query and two and +/// differencing — so no per-sub-proof plumbing (the invariants, the OOD grid, +/// the hoisted root unpacks) leaks into the figure. +struct PerQuery { + instrs: usize, + perms: usize, + swaps: usize, +} + +fn marginal(shape: &SubProofShape) -> PerQuery { + let mut one = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + emit_sub_proof(&mut one, shape, 1); + let one = compile(one.finish()); + let mut two = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + emit_sub_proof(&mut two, shape, 2); + let two = compile(two.finish()); + PerQuery { + instrs: two.instrs.len() - one.instrs.len(), + perms: permutations(&two) - permutations(&one), + swaps: bit_decs(&two) - bit_decs(&one), + } +} + +/// The DEEP fold alone, both points, with no authentication — the same +/// measurement `constraint_tests::deep_leg_cost` reports, repeated here so the +/// two halves of the joined leg can be compared on one line. +fn deep_only_rows(shape: &SubProofShape) -> usize { + use super::deep::{DeepOpening, emit_deep_invariants, emit_deep_point}; + + let d = &shape.deep; + let plumb = |b: &mut LfmBuilder| { + let n = 2 + + d.num_eval_points * d.num_total_cols + + d.num_composition_parts + + 2 * (d.num_total_cols + d.num_composition_parts) + + 2; + let arena = b.declare_arena(n as u32); + let mut i = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(arena, i).as_ext(); + i += 1; + c + }; + let g = take(b); + let z = take(b); + let steps: Vec> = (0..d.num_eval_points) + .map(|_| (0..d.num_total_cols).map(|_| take(b)).collect()) + .collect(); + let parts: Vec<_> = (0..d.num_composition_parts).map(|_| take(b)).collect(); + let openings: Vec<(Vec<_>, Vec<_>)> = (0..2) + .map(|_| { + ( + (0..d.num_total_cols).map(|_| take(b)).collect(), + (0..d.num_composition_parts).map(|_| take(b)).collect(), + ) + }) + .collect(); + let points: Vec<_> = (0..2) + .map(|_| super::builder::Felt(take(b).addr())) + .collect(); + (g, z, steps, parts, openings, points) + }; + + let mut bare = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let _ = plumb(&mut bare); + let baseline = bare.finish().instrs.len(); + + let mut inv_only = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (g, z, steps, parts, _, _) = plumb(&mut inv_only); + let _ = emit_deep_invariants(&mut inv_only, d, g, z, &steps, &parts); + let invariant_rows = inv_only.finish().instrs.len() - baseline; + + let mut full = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (g, z, steps, parts, openings, points) = plumb(&mut full); + let inv = emit_deep_invariants(&mut full, d, g, z, &steps, &parts); + for (k, (trace, qparts)) in openings.into_iter().enumerate() { + emit_deep_point( + &mut full, + d, + g, + &inv, + &DeepOpening { + point: points[k], + trace, + parts: qparts, + }, + ); + } + full.finish().instrs.len() - baseline - invariant_rows +} + +/// ★ What the joined leg costs, per query and per epoch, and how the bill +/// splits between folding the values and authenticating them. +/// +/// Measured by emitting one query and two and differencing, so the marginal +/// figure carries no per-sub-proof plumbing. Three currencies, because the +/// sizing rule in `others/lfm-target-shape.md` says rows of different chips are +/// not comparable: instructions, keccak permutations, and main-trace CELLS — +/// the last being the only one in which a byteswap and a permutation can be +/// added together. +/// +/// ### What this instrument cannot see +/// +/// The trace LENGTH of each table in a real epoch. It is workload-dependent and +/// enters only through the Merkle depth (`log2(N·blowup) − 1`), which the walk +/// is linear in, so the line below is parameterised on one uniform length +/// rather than measured. It also cannot see FRI, whose own layer openings are a +/// separate authentication bill this leg does not carry, nor the query COUNT, +/// which is a proof-options property. +#[test] +fn join_leg_cost() { + /// Queries at blowup 8 — a proof-options property, stated not measured. + const QUERIES: usize = 73; + const LOG2_BLOWUP: u32 = 3; + const LOG2_TRACE: u32 = 20; + + let swap_cells = super::machine_tests::byteswap_cells(); + let perm_cells = super::machine_tests::permutation_cells(); + + let opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(1 << LOG2_BLOWUP) + .expect("a power-of-two blowup is valid"); + let airs = crate::test_utils::production_airs(&opts); + assert_eq!(airs.len(), crate::test_utils::NUM_PRODUCTION_AIRS); + + println!( + "\nJoined DEEP+authentication, per query, at log2(N) = {LOG2_TRACE}, \ + blowup 2^{LOG2_BLOWUP} (Merkle depth {})", + LOG2_TRACE + LOG2_BLOWUP - 1 + ); + println!( + "{:<14} {:>5} {:>4} {:>9} {:>8} {:>7} {:>8} {:>12}", + "table", "cols", "grp", "instr/qry", "of it DEEP", "perm", "swaps", "cells/qry" + ); + + let mut total_instr = 0usize; + let mut total_deep = 0usize; + let mut total_perm = 0usize; + let mut total_swaps = 0usize; + for (label, air) in &airs { + let num_precomputed = production_num_precomputed(label, &**air); + let shape = shape_for(&**air, num_precomputed, LOG2_TRACE, LOG2_BLOWUP); + let per = marginal(&shape); + let deep = deep_only_rows(&shape); + + total_instr += per.instrs; + total_deep += deep; + total_perm += per.perms; + total_swaps += per.swaps; + println!( + "{:<14} {:>5} {:>4} {:>9} {:>8} {:>7} {:>8} {:>12}", + label, + shape.deep.num_total_cols, + shape.groups().len(), + per.instrs, + deep, + per.perms, + per.swaps, + per.perms as u64 * perm_cells + per.swaps as u64 * swap_cells, + ); + } + + let total_cells = total_perm as u64 * perm_cells + total_swaps as u64 * swap_cells; + println!( + "\nOne query, all {} AIRs: {total_instr} instructions ({total_deep} of \ + them the DEEP fold, {:.1}%), {total_perm} permutations, \ + {total_swaps} byteswaps.", + airs.len(), + 100.0 * total_deep as f64 / total_instr as f64, + ); + println!( + "In main-trace CELLS: {} hashing, {} byteswapping — hashing is {:.1}x \ + the swap bill.", + total_perm as u64 * perm_cells, + total_swaps as u64 * swap_cells, + (total_perm as u64 * perm_cells) as f64 / (total_swaps as u64 * swap_cells) as f64, + ); + println!( + "At {QUERIES} queries: {} instructions, {} permutations, {} cells.", + total_instr * QUERIES, + total_perm * QUERIES, + total_cells * QUERIES as u64, + ); + + // ---- what a SHARED commitment would cost, exactly, under one assumption - + // + // `others/lfm-team-lead-shared-commitment-ruling.md` parks the lever and + // pins a prediction of 55-70k permutations, noting that leaf WIDENING under + // a shared tree is unmeasured and could offset the walk saving. It can be + // settled without building anything: a permutation count is a function of + // the shape alone -- `ceil(leaf_bytes / 136)` absorbs plus one per level -- + // so the only thing being assumed is the SHAPE (one tree per sub-proof + // whose leaf is the four matrices' row pairs concatenated in matrix order). + // Nothing about the arithmetic is estimated. + // + // Widening cannot offset much, and the reason is structural: absorbs scale + // with total bytes, which do not change when the matrices share a leaf, + // while walks scale with the number of TREES, which is what collapses. The + // only bytes lost are the per-leaf padding of the groups that disappear. + const RATE_BYTES: usize = 136; + let mut shared_perm = 0usize; + for (label, air) in &airs { + let num_precomputed = production_num_precomputed(label, &**air); + let shape = shape_for(&**air, num_precomputed, LOG2_TRACE, LOG2_BLOWUP); + let leaf_bytes: usize = shape.groups().iter().map(GroupShape::leaf_bytes).sum(); + shared_perm += leaf_bytes.div_ceil(RATE_BYTES) + shape.merkle_depth; + } + println!( + "\nOne shared tree per sub-proof instead of four: {shared_perm} \ + permutations per query against {total_perm} ({:.0}% collapse), \ + {} per epoch at {QUERIES} queries.", + 100.0 * (1.0 - shared_perm as f64 / total_perm as f64), + shared_perm * QUERIES, + ); + println!( + "Absorbs are {} of the shared figure and walks {}; widening costs \ + nothing here because total leaf BYTES do not change when matrices \ + share a leaf -- only the padding of the vanished leaves.", + shared_perm - airs.len() * (LOG2_TRACE + LOG2_BLOWUP - 1) as usize, + airs.len() * (LOG2_TRACE + LOG2_BLOWUP - 1) as usize, + ); +} + +// ============================================================================= +// Falsification: the join, and the two attacks it denies +// ============================================================================= + +use super::builder::{Bit, Cell, Ext, Felt}; +use super::deep::{DeepOpening, emit_deep_invariants, emit_deep_point}; +use super::proof::{lfm_prove, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::sub_proof::{ + GroupCommitment, GroupOpening, emit_group_authentication, emit_query_points, +}; + +pub(super) fn prove_options() -> stark::proof::options::ProofOptions { + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// How a control program differs from the joined one. Each variant is an +/// attack surface the join closes, built so the attack can be RUN rather than +/// argued about. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Control { + /// DEEP folds a second arena instead of the authenticated cells — the + /// "two parallel copies" shape. + SplitValues, + /// The query point is hinted instead of derived from the walk's own index + /// bits, so the leaf may be authenticated at one index and folded at + /// another point. + HintedPoint, +} + +/// A one-query program in one of the control shapes. +/// +/// Deliberately NOT a variant of [`emit_sub_proof`]: the production emitter has +/// no switch that could produce these, and giving it one would be a runtime +/// off-switch on a soundness obligation. This is a test artifact that exists to +/// be attacked. +/// +/// Arenas: the joined program's five, plus one extra carrying whatever the +/// control decouples. +fn control_program_source( + shape: &SubProofShape, + control: Control, +) -> super::builder::LfmProgramSource { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let groups = shape.groups(); + + let uniforms = b.declare_arena(2); + let ood = b.declare_arena((shape.deep.num_eval_points * shape.deep.num_total_cols) as u32); + let parts_arena = b.declare_arena(shape.deep.num_composition_parts as u32); + // The roots and the query stride follow THIS builder's digest width, as the + // production emitter's do — a literal two here is a byte-hash assumption + // that the executor's arena-length check refuses under an algebraic pin. + let dw = super::edsl::digest_words(&b); + let roots = b.declare_arena(dw * groups.len() as u32); + let queries = b.declare_arena(shape.query_words(dw as usize) as u32); + let extra = b.declare_arena(match control { + // A second copy of every folded value, both points. + Control::SplitValues => { + 2 * (shape.deep.num_total_cols + shape.deep.num_composition_parts) as u32 + } + // The two points. + Control::HintedPoint => 2, + }); + + let gamma = b.hint_word(uniforms, 0).as_ext(); + let zeta = b.hint_word(uniforms, 1).as_ext(); + let mut next = 0u32; + let ood_steps: Vec> = (0..shape.deep.num_eval_points) + .map(|_| { + (0..shape.deep.num_total_cols) + .map(|_| { + let c = b.hint_word(ood, next).as_ext(); + next += 1; + c + }) + .collect() + }) + .collect(); + let claimed_parts: Vec = (0..shape.deep.num_composition_parts as u32) + .map(|j| b.hint_word(parts_arena, j).as_ext()) + .collect(); + let commitments: Vec = groups + .iter() + .enumerate() + .map(|(i, g)| GroupCommitment::hint(&mut b, roots, dw * i as u32, *g)) + .collect(); + let inv = emit_deep_invariants(&mut b, &shape.deep, gamma, zeta, &ood_steps, &claimed_parts); + + let mut cursor = 0u32; + let index = b.hint_felt(queries, cursor); + cursor += 1; + let openings: Vec = groups + .iter() + .map(|g| { + let values: Vec = (0..g.num_values()) + .map(|_| { + let c = b.hint_word(queries, cursor); + cursor += 1; + c + }) + .collect(); + let siblings: Vec = (0..shape.merkle_depth) + .map(|_| { + // The stride follows THIS builder's digest width, as the + // production emitter's does — not a literal two. + let d = super::edsl::hint_digest(&mut b, queries, cursor); + cursor += dw; + d + }) + .collect(); + GroupOpening { values, siblings } + }) + .collect(); + + let bits: Vec = b.bit_dec(index, shape.merkle_depth); + for (commitment, opening) in commitments.iter().zip(&openings) { + emit_group_authentication(&mut b, commitment, opening, &bits); + } + + let (point, point_sym) = match control { + Control::HintedPoint => ( + Felt(b.hint_word(extra, 0).addr()), + Felt(b.hint_word(extra, 1).addr()), + ), + Control::SplitValues => emit_query_points(&mut b, shape, &bits), + }; + + let read = |b: &mut LfmBuilder, k: usize, point: Felt| -> DeepOpening { + let width = shape.deep.num_total_cols + shape.deep.num_composition_parts; + let base = (k * width) as u32; + let (trace, parts): (Vec, Vec) = match control { + Control::SplitValues => ( + (0..shape.deep.num_total_cols) + .map(|c| b.hint_word(extra, base + c as u32).as_ext()) + .collect(), + (0..shape.deep.num_composition_parts) + .map(|j| { + b.hint_word(extra, base + (shape.deep.num_total_cols + j) as u32) + .as_ext() + }) + .collect(), + ), + Control::HintedPoint => { + let mut trace = Vec::new(); + for (opening, g) in openings.iter().zip(&groups).take(shape.trace_groups.len()) { + for c in 0..g.num_columns { + trace.push(opening.values[k * g.num_columns + c].as_ext()); + } + } + let parts_opening = openings.last().expect("parts"); + let np = shape.deep.num_composition_parts; + ( + trace, + (0..np) + .map(|j| parts_opening.values[k * np + j].as_ext()) + .collect(), + ) + } + }; + DeepOpening { + point, + trace, + parts, + } + }; + let regular = read(&mut b, 0, point); + let symmetric = read(&mut b, 1, point_sym); + let got = emit_deep_point(&mut b, &shape.deep, gamma, &inv, ®ular); + let got_sym = emit_deep_point(&mut b, &shape.deep, gamma, &inv, &symmetric); + b.public(got.as_cell()); + b.public(got_sym.as_cell()); + b.finish() +} + +impl HostSubProof { + /// The values one query folds, in the order a [`Control::SplitValues`] + /// program reads them: the regular point's trace then parts, then the + /// symmetric point's. + fn split_values(&self, q: usize) -> Vec { + let groups = self.shape.groups(); + let mut out = Vec::new(); + for k in 0..ROWS_PER_LEAF { + for (opening, g) in self.openings[q] + .iter() + .zip(&groups) + .take(self.shape.trace_groups.len()) + { + out.extend(&opening.values[k * g.num_columns..(k + 1) * g.num_columns]); + } + let parts = self.openings[q].last().expect("parts"); + let np = self.shape.deep.num_composition_parts; + out.extend(&parts.values[k * np..(k + 1) * np]); + } + out + } +} + +/// ★ The joined program authenticates and folds under a real PROOF, not just +/// an execution. +/// +/// Method rule 2: the executor mirrors the ALU it is checking, so nothing run +/// so far says the CHIPS agree. One query, because the whole point of this test +/// is the chips and 219 of them would only repeat the same rows. +#[test] +fn the_join_proves_and_verifies() { + let h = host_sub_proof(); + let opts = prove_options(); + let queries = [0usize]; + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, queries.len()); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + // Built at `WrapHash::production()`, so it emits `Instr::Hash` and the + // artifacts must carry the pin's tenant — the classification rule in + // HASH-PINNING.md. `build_artifacts` defaults to the registry's blessed + // hasher, which is correct for registry programs and wrong for this one. + let artifacts = build_artifacts_with_hasher(&program, &opts, crate::hash_pin::BLOCK_HASHER); + let proved = lfm_prove(&program, &artifacts, &h.arenas(&queries), &opts) + .expect("the joined sub-proof must prove"); + + let (want, want_sym) = h.expected[queries[0]]; + assert_eq!( + word_as_ext(&proved.public_words[0].1).expect("ext"), + want, + "the proved run must publish the production reconstruction" + ); + assert_eq!( + word_as_ext(&proved.public_words[1].1).expect("ext"), + want_sym + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the joined run must verify" + ); +} + +/// ★ The join, stated as the property it exists for: there is no arena the +/// prover can move that changes the folded value without breaking the +/// authentication. +/// +/// Every opened value of one query is tampered, one at a time, and each vector +/// is run BOTH ways. Incoherent (claim the real root) must not execute. +/// Coherent (also claim the root the tampered leaf really folds to, so nothing +/// in the run is inconsistent) must execute — and then publish a DEEP value +/// that is not the production one, against a root that is not the committed +/// one. A prover who wants the wrong fold must pay with the wrong root. +/// +/// Run over BOTH fixtures. The three-matrix one is the real production table; +/// the four-matrix one is the only fixture in which the precomputed group's own +/// leaf and path are ever tampered, and its 64-row trace is the only executed +/// walk in this file deeper than two levels. +#[test] +fn no_tampered_value_can_move_the_fold_without_moving_the_root() { + sweep_tampers(host_sub_proof(), "L2G_MEMORY (3 matrices, depth 2)"); + sweep_tampers( + preprocessed_sub_proof(), + "PREPROCESSED_FIXTURE (4 matrices, depth 6)", + ); +} + +fn sweep_tampers(h: &HostSubProof, label: &str) { + use super::proof_arena::{commitments_to_arena, walk_to_root}; + + let q = 0usize; + let groups = h.shape.groups(); + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, 1); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + let honest = + execute(&program, &h.arenas(&[q]), &crate::hash_pin::BLOCK_HASHER).expect("honest"); + + // Sweep every value slot of every group, so no vector class (first group, + // first column, regular point) is silently the only one tested. + let mut vectors = 0usize; + for (g, group) in groups.iter().enumerate() { + for slot in 0..group.num_values() { + let mut arenas = h.arenas(&[q]); + let word_of_slot = { + // Offset of this group's value `slot` inside the query arena. + let mut off = 1usize; + for prior in groups.iter().take(g) { + off += prior.num_values() + + super::proof_arena::words_per_root() * h.shape.merkle_depth; + } + off + slot + }; + arenas[4][word_of_slot][0] += FE::one(); + + // Incoherent: the real roots, a moved leaf. + let err = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .err() + .unwrap_or_else(|| { + panic!("{label}: group {g} slot {slot}: a moved value must not authenticate") + }); + + // Coherent: recompute the leaf the tampered values really give and + // the root that leaf really reaches, using PRODUCTION's hashers. + let leaf = tampered_leaf(h, q, g, slot); + let forged = walk_to_root(leaf, h.iotas[q], &h.openings[q][g].siblings); + assert_ne!( + forged, h.roots[g], + "{label}: group {g} slot {slot}: the tamper must move the root, or the \ + vector is vacuous" + ); + let mut coherent_roots = h.roots.clone(); + coherent_roots[g] = forged; + arenas[3] = commitments_to_arena(&coherent_roots); + let forged_run = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| { + panic!( + "{label}: group {g} slot {slot}: the coherent forgery must execute: {e:?}" + ) + }); + // Which of the two points moves is not incidental: a leaf holds + // the row PAIR, its first half is the regular point and its second + // the symmetric, and folding the halves into the wrong point is a + // mistake no root check would catch. Asserting exactly one moved, + // and which, is what pins that split. + let moved = [ + forged_run.public_words[0].1 != honest.public_words[0].1, + forged_run.public_words[1].1 != honest.public_words[1].1, + ]; + let regular_half = slot < group.num_columns; + assert_eq!( + moved, + [regular_half, !regular_half], + "{label}: group {g} slot {slot}: a value in the leaf's {} half must move \ + DEEP at {} and nothing else", + if regular_half { "first" } else { "second" }, + if regular_half { + "the regular point" + } else { + "-v" + }, + ); + if vectors == 0 { + println!("first incoherent rejection: {err:?}"); + } + vectors += 1; + } + } + println!("{label}: {vectors} tamper vectors, every value slot of every group, both ways round"); + + // ---- the index, which this leg binds to the POINT as well as the leaf -- + // + // R1f authenticated a leaf at an index; here the same bits also derive the + // evaluation point, so moving the index has to move the reconstruction as + // well as the walk. A padding-heavy table can have several indices that + // authenticate (identical rows give identical leaves), which would make the + // walk half of this vector vacuous — so that is asserted, not assumed. + for level in 0..h.shape.merkle_depth { + let bad = h.iotas[q] ^ (1 << level); + let mut arenas = h.arenas(&[q]); + arenas[4][0] = base_word(FE::from(bad as u64)); + + let mut moved_a_root = false; + let mut coherent_roots = h.roots.clone(); + for (g, group) in groups.iter().enumerate() { + let words = &h.openings[q][g].values; + let leaf = if group.is_ext { + type ExtBackend = super::proof_arena::BlockBatched; + let v: Vec = words.iter().map(|w| FEE::new([w[0], w[1], w[2]])).collect(); + >::hash_data_from_slices(&v, &[]) + } else { + type BaseBackend = super::proof_arena::BlockBatched; + let v: Vec = words.iter().map(|w| w[0]).collect(); + >::hash_data_from_slices(&v, &[]) + }; + coherent_roots[g] = walk_to_root(leaf, bad, &h.openings[q][g].siblings); + moved_a_root |= coherent_roots[g] != h.roots[g]; + } + assert!( + moved_a_root, + "{label}: flipping index bit {level} left every root unchanged — the fixture's \ + trees are degenerate at this index and the walk half of this vector \ + tests nothing" + ); + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .err() + .unwrap_or_else(|| { + panic!("{label}: index bit {level}: a moved index must not authenticate") + }); + + arenas[3] = commitments_to_arena(&coherent_roots); + let forged = + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!("{label}: index bit {level}: coherent forgery must execute: {e:?}") + }); + assert_ne!( + forged.public_words[0].1, honest.public_words[0].1, + "{label}: index bit {level}: the index derives the evaluation point, so a \ + forged walk at another index must also fold at another point" + ); + } + println!( + "{label}: {} index vectors, one per level", + h.shape.merkle_depth + ); + + // ---- a sibling, at every level ------------------------------------- + for level in 0..h.shape.merkle_depth { + let mut siblings = h.openings[q][0].siblings.clone(); + siblings[level][0] ^= 1; + let mut arenas = h.arenas(&[q]); + let base = 1 + groups[0].num_values(); + arenas[4][base..base + super::proof_arena::words_per_root() * h.shape.merkle_depth] + .copy_from_slice(&commitments_to_arena(&siblings)); + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .err() + .unwrap_or_else(|| { + panic!("{label}: sibling level {level}: a moved path must not authenticate") + }); + } + println!( + "{label}: {} sibling vectors, one per level", + h.shape.merkle_depth + ); + + /// The leaf hash a tampered opening really produces, under production's own + /// backend rather than a local model. + fn tampered_leaf(h: &HostSubProof, q: usize, g: usize, slot: usize) -> Commitment { + type BaseBackend = super::proof_arena::BlockBatched; + type ExtBackend = super::proof_arena::BlockBatched; + let group = h.shape.groups()[g]; + let words = &h.openings[q][g].values; + if group.is_ext { + let mut v: Vec = words.iter().map(|w| FEE::new([w[0], w[1], w[2]])).collect(); + v[slot] = &v[slot] + FEE::new([FE::one(), FE::zero(), FE::zero()]); + >::hash_data_from_slices(&v, &[]) + } else { + let mut v: Vec = words.iter().map(|w| w[0]).collect(); + v[slot] += FE::one(); + >::hash_data_from_slices(&v, &[]) + } + } +} + +/// ★ The two attacks the join denies, RUN against control programs that permit +/// them. +/// +/// A join is a negative claim — "these cannot disagree" — and a negative claim +/// is only worth what its counterexample is worth. So each control is the +/// joined program with exactly one link cut, fed inputs that are honest +/// everywhere else, and each one accepts a reconstruction the production +/// verifier would not have produced. That is the thing the joined program has +/// to refuse, and the test above shows it does. +#[test] +fn the_controls_show_what_the_join_denies() { + let h = host_sub_proof(); + let q = 0usize; + + // ---- Control 1: DEEP folds a parallel copy. ------------------------- + let program = compile(control_program_source(&h.shape, Control::SplitValues)); + validate(&program).expect("admissible"); + let mut arenas = h.arenas(&[q]); + arenas.push(h.split_values(q)); + let clean = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the control must accept honest inputs"); + assert_eq!( + word_as_ext(&clean.public_words[0].1).expect("ext"), + h.expected[q].0, + "the control must agree with production before it is attacked, or the \ + attack below proves nothing" + ); + + let mut attacked = arenas.clone(); + attacked[5][0][0] += FE::one(); + let forged = execute(&program, &attacked, &crate::hash_pin::BLOCK_HASHER).expect( + "SplitValues: authenticating one set of values and folding another is \ + exactly what this control permits", + ); + assert_ne!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + h.expected[q].0, + "the attack must actually move the reconstruction" + ); + println!("SplitValues control: forged fold accepted against honest roots"); + + // ---- Control 2: the query point is hinted. -------------------------- + // Two queries with DIFFERENT indices: authenticate one, fold at the + // other's point. + let other = (0..h.iotas.len()) + .find(|&i| h.iotas[i] != h.iotas[q]) + .expect("the fixture must carry two distinct query indices"); + let program = compile(control_program_source(&h.shape, Control::HintedPoint)); + validate(&program).expect("admissible"); + let mut arenas = h.arenas(&[q]); + arenas.push(vec![ + base_word(h.points[q].0), + base_word(h.points[q].1.expect("a row-pair fixture")), + ]); + let clean = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER).expect("honest"); + assert_eq!( + word_as_ext(&clean.public_words[0].1).expect("ext"), + h.expected[q].0 + ); + + let mut attacked = arenas.clone(); + attacked[5] = vec![ + base_word(h.points[other].0), + base_word(h.points[other].1.expect("a row-pair fixture")), + ]; + let forged = execute(&program, &attacked, &crate::hash_pin::BLOCK_HASHER).expect( + "HintedPoint: a hinted point is not tied to the authenticated index, \ + which is what this control permits", + ); + assert_ne!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + h.expected[q].0, + "folding query {q}'s values at query {other}'s point must give a \ + different answer" + ); + assert_ne!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + h.expected[other].0, + "and it must not accidentally be the other query's answer either" + ); + println!( + "HintedPoint control: query {q}'s leaf authenticated, folded at query \ + {other}'s point, accepted" + ); +} + +// ============================================================================= +// The degenerate parameter this leg introduced: the precomputed group +// ============================================================================= + +/// A PREPROCESSED sub-proof, so the four-group shape is exercised. +/// +/// L2G_MEMORY — the fixture everything above runs on — is not preprocessed, and +/// neither is any AIR a single-table proof can cheaply be built from: the real +/// preprocessed tables are BITWISE (2^20 rows), DECODE, KECCAK_RC, REGISTER and +/// PAGE. So on that fixture the precomputed group is ABSENT, DEEP's column +/// order `precomputed ‖ main ‖ aux` degenerates to `main ‖ aux`, and an emitter +/// that put main first would pass every test in this file. That is the same +/// hazard as `step_size = 1` and it needs the same answer: a case production +/// does not produce. +/// +/// Built the way `tests::bitwise_tests` builds its preprocessed receiver — a +/// small `AirWithBuses` whose commitment comes from the prover's own +/// `compute_precomputed_commitment_for_testing`, so the precomputed root in the +/// proof and the one the AIR declares are computed by the same code the real +/// tables use. Widths are 2 and 2 so the two base groups can be SWAPPED, which +/// the falsification half needs. +fn preprocessed_fixture() -> ( + super::constraint_tests::BoxedAir, + stark::proof::stark::MultiProof, +) { + use crate::tables::types::{BusId, alu_op}; + use crate::test_utils::multi_prove_ram; + use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, + }; + use stark::prover::IsStarkProver; + use stark::trace::TraceTable; + use stark::traits::AIR; + + /// Columns 0..3 are precomputed (x, y, x&y); 3..6 are the multiplicity + /// block (a copy of x&y, a spare, and the bus multiplicity). The copy is + /// there so the table carries a real TRANSITION constraint: `EmptyConstraints` + /// leaves a single coefficient in the run and `open_sub_proof` recovers + /// `beta` from its second element. + const NUM_COLS: usize = 6; + const NUM_PRECOMPUTED: usize = 3; + /// 64 rows, not the 4 the other fixture uses. Trace length only enters this + /// leg through the Merkle depth, and at 4 rows every executed walk in the + /// suite is two levels deep — enough to hide a level-count error. 64 rows at + /// blowup 2 gives depth 6, which is the only executed multi-level walk over + /// all four committed matrices. + const NUM_ROWS: usize = 64; + + let opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup=2 is valid"); + + let build = |commitment: Option| { + let air = AirWithBuses::::new( + NUM_COLS, + AuxiliaryTraceBuildData { + interactions: vec![BusInteraction::receiver( + BusId::ByteAlu, + // The multiplicity is the LAST column, past the precomputed + // block — the production split (`0..n` precomputed, the + // rest multiplicities). + Multiplicity::Column(5), + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 2, + packing: Packing::Direct, + }, + ], + )], + }, + &opts, + 1, + CopiedColumn, + ) + .with_name("PREPROCESSED_FIXTURE"); + match commitment { + Some(c) => air.with_preprocessed(c, NUM_PRECOMPUTED), + None => air, + } + }; + + // Distinct rows, so the committed leaves are distinct and the tree is not + // the degenerate one R1f warns about. + let make_trace = || { + let mut data = vec![FE::zero(); NUM_ROWS * NUM_COLS]; + for r in 0..NUM_ROWS { + let x = 5u64 + r as u64; + let y = 3u64 + 2 * r as u64; + data[r * NUM_COLS] = FE::from(x); + data[r * NUM_COLS + 1] = FE::from(y); + data[r * NUM_COLS + 2] = FE::from(x & y); + data[r * NUM_COLS + 3] = FE::from(x & y); + data[r * NUM_COLS + 5] = FE::one(); + } + TraceTable::::new_main(data, NUM_COLS, 1) + }; + + let trace = make_trace(); + // `DefaultStarkHash`, not a named hash: this commitment has to be the one + // `multi_prove_ram` below recomputes, and that follows the alias. + let commitment = as IsStarkProver< + Gl, + Ext3, + (), + crate::hash_pin::BlockStarkHash, + >>::compute_precomputed_commitment_for_testing( + &trace, &build(None), NUM_PRECOMPUTED + ) + .expect("the precomputed columns commit"); + + let air = build(Some(commitment)); + let mut trace = make_trace(); + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .expect("the preprocessed fixture must prove"); + (Box::new(air), proof) +} + +/// `main[3] == main[2]` — one transition constraint, satisfied by the fixture +/// trace, spanning the precomputed/multiplicity boundary. +struct CopiedColumn; + +impl + stark::constraints::builder::ConstraintSet for CopiedColumn +{ + fn eval>(&self, b: &mut B) { + let precomputed_and = b.main(0, 2); + let copied_and = b.main(0, 3); + b.emit_base(0, copied_and - precomputed_and); + } +} + +fn preprocessed_sub_proof() -> &'static HostSubProof { + use std::sync::OnceLock; + static CELL: OnceLock = OnceLock::new(); + CELL.get_or_init(|| { + let (air, proof) = preprocessed_fixture(); + build_host_sub_proof(&*air, &proof) + }) +} + +/// ★ The four-group shape, and the witness that the group ORDER is +/// load-bearing. +/// +/// Both halves, as the degenerate-parameter rule requires. The machine +/// reproduces the production reconstruction on a proof that HAS a precomputed +/// group — and production's own reconstruction, handed the same two base +/// groups in the opposite order, gives a DIFFERENT answer. Without the second +/// half this test would pass against a main-first emitter, which is the exact +/// failure mode it exists to prevent. +#[test] +fn the_precomputed_group_comes_first_and_that_is_checkable() { + let h = preprocessed_sub_proof(); + let groups = h.shape.groups(); + + assert_eq!( + groups.len(), + 4, + "the point of this fixture is a sub-proof with all four committed \ + matrices; got {groups:?}" + ); + assert_eq!(h.shape.trace_groups[0].num_columns, 3, "precomputed width"); + assert!(!h.shape.trace_groups[0].is_ext); + assert_eq!(h.shape.trace_groups[1].num_columns, 3, "main width"); + assert_eq!( + h.shape.trace_groups[0].num_columns, h.shape.trace_groups[1].num_columns, + "the two base groups must be the same width, or the swap below is not \ + a well-formed alternative reading" + ); + println!( + "preprocessed fixture: groups {:?}, depth {}, {} queries", + groups + .iter() + .map(|g| (g.num_columns, g.is_ext)) + .collect::>(), + h.shape.merkle_depth, + h.iotas.len() + ); + + // ---- half one: the machine agrees with production. ------------------- + let queries: Vec = (0..h.iotas.len().min(16)).collect(); + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, queries.len()); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("admissible"); + let exec = execute( + &program, + &h.arenas(&queries), + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("the four-group sub-proof must authenticate and fold"); + for (k, q) in queries.iter().enumerate() { + assert_eq!( + word_as_ext(&exec.public_words[2 * k].1).expect("ext"), + h.expected[*q].0, + "query {q}: DEEP at the regular point" + ); + assert_eq!( + word_as_ext(&exec.public_words[2 * k + 1].1).expect("ext"), + h.expected[*q].1, + "query {q}: DEEP at the symmetric point" + ); + } + + // ---- half two: the swapped reading DISAGREES. ------------------------ + // + // Asked of production's own reconstruction, not of a model of it: hand it + // the main slice where the precomputed one belongs and vice versa. If that + // came out equal, the order would be unobservable and this fixture would be + // no witness at all. + let swapped = &h.expected_base_swapped; + assert_eq!( + swapped.len(), + h.expected.len(), + "the swapped reading must have been computed for this fixture" + ); + let mut differs = 0usize; + for q in &queries { + if swapped[*q].0 != h.expected[*q].0 || swapped[*q].1 != h.expected[*q].1 { + differs += 1; + } + } + assert_eq!( + differs, + queries.len(), + "swapping the precomputed and main slices must change the \ + reconstruction at every query, or the column order is not observable \ + on this fixture and it witnesses nothing" + ); + println!( + "column order is load-bearing: the swapped reading differs at all {} \ + checked queries", + queries.len() + ); +} + +/// ⚠ The FRI leg's instrument problem, pinned: the proof fixture carries ZERO +/// committed FRI layers, so a differential over it cannot see the fold loop, +/// the per-layer walks, or the terminal check. +/// +/// `FriFoldLayout::new(lde_log, blowup_log, k)` sets +/// `terminal_log = min(blowup_log + k, lde_log)` and +/// `num_committed = (lde_log - terminal_log) - 1`. The fixture is the `min` +/// preset — blowup 2 (`blowup_log = 1`), `fri_final_poly_log_degree = 7` — over +/// an epoch of 2^4 steps, so its sub-proof has `log2(lde) = 3` and +/// `terminal_log = min(8, 3) = 3`: no folds at all, and `query_phase` returns +/// the empty-decommitment branch. +/// +/// This is the degenerate-parameter rule in its most extreme form. Not "one +/// value hides a difference between two implementations" but "the production +/// instance exercises none of the mechanism". The assertions below are still +/// exactly true of THIS fixture, and this test still earns its place: the day +/// the fixture grows and starts folding, the change is announced rather than +/// silently altering what the FRI tests cover. +/// +/// ## ⚠ CORRECTION — the conclusion drawn from this was wrong +/// +/// This test's original text went on to say that no amount of care with real +/// data could repair the gap, and that the FRI leg's primary instrument had to +/// be SYNTHETIC codewords. That is false, and the counterexample is one line of +/// the fixture: the trace is `boundaries.len().next_power_of_two()` +/// (`local_to_global.rs:269`), and `num_committed = trace_bits − 8`. So the same +/// construction with 512, 1024 or 2048 boundaries yields real production proofs +/// with one, two or three committed layers — real roots, real paths, real +/// terminal coefficients, real folding challenges — in under a second each. +/// `fri_tests::the_real_prover_folds_and_the_layer_count_follows_the_row_count` +/// is that sweep, and the FRI leg is differentialled entirely against real +/// proofs. Nothing in it is synthetic. +/// +/// The lesson is narrower than the one first drawn here. "The production +/// instances all share a degenerate parameter" was a claim about the fixtures on +/// hand, not about the prover, and the two are not the same claim. Worth +/// checking which one is being made before concluding that real data cannot +/// reach a mechanism. +/// +/// The zero-layer case is not merely an artifact to route around, either — it +/// is a real production path (small tables fold no further than their terminal) +/// and the emitted verifier handles it as a first-class shape. +#[test] +fn the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold() { + let (_air, proof) = real_fixture(); + assert_eq!( + proof.proofs.len(), + 1, + "the join fixture is a single sub-proof" + ); + let p = &proof.proofs[0]; + println!( + "fixture sub-proof: fri_layers_merkle_roots = {}, fri_final_poly_coeffs = {}, \ + query decommitments = {}", + p.fri_layers_merkle_roots.len(), + p.fri_final_poly_coeffs.len(), + p.deep_poly_openings.len(), + ); + assert_eq!( + p.fri_layers_merkle_roots.len(), + 0, + "the fixture is expected to carry no committed FRI layers; if it now \ + folds, the FRI leg's coverage story changed and its synthetic sweep \ + should be re-justified against what the real proof now exercises" + ); + // The coefficient count is `2^effective_k` with + // `effective_k = terminal_log - blowup_log = 3 - 1`. Checking it is what + // says the layout arithmetic above is read correctly rather than merely + // asserted: a wrong reading of `FriFoldLayout` would land on a different + // power of two here. + assert_eq!( + p.fri_final_poly_coeffs.len(), + 4, + "terminal codeword encodes a degree-<2^2 polynomial at this shape" + ); +} + +/// ★ The bits handed to a later leg are the cells the WALK ITSELF consumed. +/// +/// The FRI leg reuses a query's index per layer (leaf position `index >> 1`, +/// partner `index ^ 1`, halving each layer). Were it to decompose its own copy +/// it would authenticate at one index and fold at another — the gap this module +/// closes, reopened one level up. +/// +/// ## Why this is not the obvious test +/// +/// The obvious test compares the program `emit_sub_proof` emits against the one +/// `emit_sub_proof_with_bits` emits and asserts they are identical. That test is +/// VACUOUS and I wrote it before catching it: `emit_sub_proof` is implemented by +/// delegating to `emit_sub_proof_with_bits`, so the two sides are the same +/// program by construction and any defect lands on both and cancels. Injecting a +/// second `bit_dec` — the precise failure this is meant to deny — left it green. +/// +/// What discriminates is an ABSOLUTE property rather than a relative one: every +/// returned bit must be consumed by a `Select`. The walk selects sibling order +/// on each bit and `pow_bits` selects the point factors on the same bits, so a +/// bit the emitter actually used is necessarily read by one. A freshly +/// decomposed second copy would be read by nothing. +#[test] +fn the_exposed_bits_are_the_cells_the_walk_consumed() { + let h = host_sub_proof(); + const QUERIES: usize = 3; + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_, out) = emit_sub_proof_with_bits(&mut b, &h.shape, QUERIES); + let src = b.finish(); + assert_eq!(out.len(), QUERIES); + + // Every address any Select reads as its selector. + let selector_bits: std::collections::HashSet = src + .instrs + .iter() + .filter_map(|i| match i { + super::instr::Instr::Select { bit, .. } => Some(bit.0), + _ => None, + }) + .collect(); + assert!( + !selector_bits.is_empty(), + "the walk and the point derivation both select on bits; an empty set \ + means this test is looking at the wrong instruction" + ); + + for (q, output) in out.iter().enumerate() { + assert_eq!( + output.bits.len(), + h.shape.merkle_depth, + "query {q}: one bit per Merkle level" + ); + for (level, bit) in output.bits.iter().enumerate() { + assert!( + selector_bits.contains(&bit.0.0), + "query {q} level {level}: the returned bit is read by no Select, \ + so it is not a cell the walk or the point derivation used — a \ + second decomposition of the index has been handed out" + ); + } + } +} + +// ==================== FRI slice 1: the fold layout ==================== + +use super::fri::FriShape; +use stark::proof::options::ProofFormat; + +/// ★ The shape mirror against production's observable BEHAVIOUR on the real +/// proof — the vector lengths the verifier structurally enforces. +/// +/// `FriFoldLayout` is `pub(crate)` inside `crypto/stark`, so the mirror cannot +/// be compared against the struct. It is compared against what a real proof +/// actually carries instead, which is the better oracle: `verifier.rs:426-448` +/// rejects on exactly these two lengths before its query loop runs, and the +/// spec notes they are the ONLY thing pinning vectors Fiat-Shamir does not bind. +#[test] +fn the_fri_shape_predicts_the_real_proofs_vector_lengths() { + let (_air, proof) = real_fixture(); + let h = host_sub_proof(); + let opts = prove_options(); + let shape = FriShape::from_options(&opts, h.shape.log2_lde_length); + shape.check(); + + let p = &proof.proofs[0]; + println!( + "FRI shape: lde 2^{}, blowup 2^{}, k {}, terminal_log {}, total_folds {}, \ + committed {}, coeffs {}", + shape.log2_lde_length, + shape.blowup_log, + shape.final_poly_log_degree, + shape.terminal_log(), + shape.total_folds(), + shape.num_committed(), + shape.num_terminal_coeffs(), + ); + assert_eq!( + p.fri_layers_merkle_roots.len(), + shape.num_committed(), + "committed layer count must match what the proof carries" + ); + assert_eq!( + p.fri_final_poly_coeffs.len(), + shape.num_terminal_coeffs(), + "terminal coefficient count must match 2^effective_k" + ); + assert_eq!( + shape.coset_offset, opts.coset_offset, + "the shape must take its coset offset from the options, not a literal" + ); +} + +/// ★ The synthetic sweep §7 requires, and the reason it is needed. +/// +/// Production pins `k = 7` and `coset_offset = 3` in every configuration, so no +/// real proof distinguishes an implementation that reads `k` from one that +/// hardcodes 7, and none reaches the clamp (`trace_bits <= 7`) at all. In LFM +/// these are not dead emitted branches — shape is compile-time, so they are +/// host-side arithmetic — which is exactly why they are testable here for free, +/// with no proving. +/// +/// Each row is `(trace_bits, blowup_log, k)` with its expected +/// `(total_folds, num_committed, effective_k, terminal_len)`, derived by hand +/// from `terminal.rs:45-54` rather than from this module. +/// +/// ## ★ Falsified, and the result is the leg's blindness finding made concrete +/// +/// Deleting the `saturating_sub(1)` from `FriShape::num_committed` — the +/// off-by-one that makes a verifier authenticate one layer FEWER than the proof +/// commits — fails this test and +/// [`the_fri_sizing_prediction`], and **passes** +/// [`the_fri_shape_predicts_the_real_proofs_vector_lengths`]. The fixture has +/// `total_folds = 0`, so `0` and `0.saturating_sub(1)` are the same number and +/// the real proof cannot tell the two implementations apart. +/// +/// So the most soundness-relevant constant in this leg is invisible to the only +/// real data available. That is not an argument for a better fixture; it is the +/// reason these synthetic rows are the primary instrument rather than a +/// supplement. +#[test] +fn the_fold_layout_is_right_off_productions_constants() { + /// `(total_folds, num_committed, effective_k, terminal_len)`. + type Layout = (u32, usize, u32, usize); + // (trace_bits, blowup_log, k) -> Layout + let cases: [(u32, u32, u32, Layout); 10] = [ + // The production point, at three blowups. k = 7 throughout. Note + // `total_folds = trace_bits - k` is INDEPENDENT of the blowup: the + // blowup enters `n` and `terminal_log` identically and cancels. That + // cancellation is what makes the scout's `num_committed = trace_bits - 8` + // invariant hold across every preset, and mis-expanding it (subtracting + // the blowup twice) is how the first version of this table was wrong. + (20, 1, 7, (13, 12, 7, 256)), + (20, 2, 7, (13, 12, 7, 512)), + (20, 3, 7, (13, 12, 7, 1024)), + // k = 0: fold all the way down to one coefficient per coset. + (10, 1, 0, (10, 9, 0, 2)), + // k = 6, one below production. + (10, 1, 6, (4, 3, 6, 128)), + // The CLAMP regime, trace_bits <= k: terminal_log pins to lde_log, so + // nothing folds and effective_k drops below the requested k. + (7, 1, 7, (0, 0, 7, 256)), + (4, 1, 7, (0, 0, 4, 32)), + (2, 3, 7, (0, 0, 2, 32)), + // k = 63 — far past any real trace, so the clamp always wins. + (5, 1, 63, (0, 0, 5, 64)), + // A single fold — `trace_bits = k + 1` — commits NOTHING, because the + // last fold is never committed. The row that catches the off-by-one. + (8, 1, 7, (1, 0, 7, 256)), + ]; + for (trace_bits, blowup_log, k, expected) in cases { + let shape = FriShape { + log2_lde_length: trace_bits + blowup_log, + blowup_log, + final_poly_log_degree: k, + coset_offset: 3, + num_queries: 1, + format: ProofFormat::DEFAULT, + trees: stark::config::TreeGeometry::BINARY, + }; + shape.check(); + let got = ( + shape.total_folds(), + shape.num_committed(), + shape.effective_k(), + shape.terminal_len(), + ); + assert_eq!( + got, expected, + "trace_bits {trace_bits} blowup 2^{blowup_log} k {k}: \ + (total_folds, committed, effective_k, terminal_len)" + ); + // Folds exceed committed layers by exactly one whenever anything folds. + if shape.total_folds() > 0 { + assert_eq!( + shape.num_folds(), + shape.num_committed() + 1, + "the final fold is never committed" + ); + } + } +} + +/// ★ The sizing prediction, PINNED BEFORE MEASURING — the leg's +/// measurements-vs-prediction target. +/// +/// Derived from `others/lfm-fri-verify-spec.md` §8: `pathlen(i) = n − i − 2`, +/// so a query walks `Σ pathlen(i)` steps and pays one permutation per step plus +/// one leaf hash per committed layer. The blowup-2 row reproduces the spec's +/// own worked example (162 steps, 174 permutations, 38,106 total), which is +/// what says the formula is being read as written rather than re-derived. +/// +/// Recorded as a test rather than a comment so that the emitter, when it +/// arrives, is measured against a number that was fixed beforehand. +#[test] +fn the_fri_sizing_prediction() { + println!("blowup n C Q steps/q perms/q total"); + // (blowup_log, queries, expected steps/q, perms/q, total) at trace_bits = 20. + for (blowup_log, queries, steps, perms, total) in [ + (1u32, 219usize, 162usize, 174usize, 38_106usize), + (2, 110, 174, 186, 20_460), + (3, 73, 186, 198, 14_454), + ] { + let shape = FriShape { + log2_lde_length: 20 + blowup_log, + blowup_log, + final_poly_log_degree: 7, + coset_offset: 3, + num_queries: queries, + format: ProofFormat::DEFAULT, + trees: stark::config::TreeGeometry::BINARY, + }; + shape.check(); + println!( + " 2^{blowup_log} {:>3} {:>3} {:>4} {:>8} {:>8} {:>9}", + shape.log2_lde_length, + shape.num_committed(), + shape.num_queries, + shape.path_steps_per_query(), + shape.permutations_per_query(), + shape.permutations(), + ); + assert_eq!( + shape.path_steps_per_query(), + steps, + "blowup 2^{blowup_log} steps" + ); + assert_eq!( + shape.permutations_per_query(), + perms, + "blowup 2^{blowup_log} perms/query" + ); + assert_eq!(shape.permutations(), total, "blowup 2^{blowup_log} total"); + } +} diff --git a/prover/src/lfm/keccak_adapter.rs b/prover/src/lfm/keccak_adapter.rs new file mode 100644 index 000000000..4404f4e37 --- /dev/null +++ b/prover/src/lfm/keccak_adapter.rs @@ -0,0 +1,516 @@ +//! R1a probe adapter: host the production keccak table family in a foreign AIR set. +//! +//! The production keccak family is three chips: `KECCAK` (the core, in +//! `tables::keccak`), `KECCAK_RND` (24 rounds of the permutation, in +//! `tables::keccak_rnd`) and `KECCAK_RC` (the round-constant fixed table). Only +//! the core is VM-coupled — it reads and writes the 25 state lanes through +//! timestamped `MEMW` tokens, so it cannot be lifted into the LFM machine as-is. +//! `KECCAK_RND` and `KECCAK_RC` are pure: they speak only the `Keccak`, +//! `KeccakRc` and BITWISE buses. +//! +//! This module is the minimal chip that replaces the core: it opens the +//! `Keccak` bus with a request token and closes it with the reply token, +//! reproducing exactly the two tokens `tables::keccak::bus_interactions` emits +//! (see `keccak.rs:264-325`) and nothing else. No memory, no timestamps, no +//! address range checks. Feeding those two tokens is the entire contract the +//! round chip needs, so the unchanged `KECCAK_RND` + `KECCAK_RC` + `BITWISE` +//! AIRs prove real `keccak-f[1600]` permutations driven by this adapter. +//! +//! # Token layout +//! +//! Both tokens are 203 bus elements: `[tag_lo, tag_hi, round, state[200]]`, +//! `round = 0` on the request (sent) and `round = 24` on the reply (received). +//! The 200 state elements are traversed **column-major over lanes**: element +//! `3 + 8 * (5x + y) + b` is byte `b` (LSB-first) of lane `x + 5y`, so lanes are +//! visited in the order 0, 5, 10, 15, 20, 1, 6, ... That asymmetry is inherited +//! from the production sender's `for x { for y { for b } } }` loop over +//! `cols::input_state(x, y, b) = INPUT_STATE + (x + 5y) * 8 + b`; emitting the +//! lanes in natural order instead would leave the bus unbalanced. +//! +//! # Constraints +//! +//! The adapter carries no polynomial constraints. Byte-ness of the 400 state +//! columns is enforced transitively rather than locally: every one of the 200 +//! IN bytes is an operand of at least one `BYTE_ALU[XOR]` lookup in the round +//! chip's θ column-parity chain (which covers all 25 lanes) and again in its θ +//! final XOR, and every OUT byte is the *result* of a `BYTE_ALU[XOR]` lookup +//! (χ, or ι for lane 0). A non-byte value in any of those columns finds no row +//! in the BITWISE table and breaks the bus balance. The tag columns are pure +//! labels and are deliberately unconstrained here. +//! +//! The production LFM adapter will add what this probe omits: binding the state +//! columns to `LfmMem` words (so the permutation's input and output are the +//! machine's data, not free witness), and sourcing the tag from preprocessed +//! program data. +//! +//! # Tag uniqueness is soundness-critical +//! +//! Nothing but the tag binds a request token to its reply token. Two rows +//! carrying the same tag let a malicious prover swap their output states: the +//! two `(tag, 24, ·)` receives and the two `(tag, 24, ·)` sends still form the +//! same multiset, so the bus balances and the proof verifies. The probe test +//! `duplicate_tag_output_swap_accepts_demonstrating_hazard` pins exactly this. +//! This probe uses distinct per-row constant tags; the production LFM adapter +//! will carry tags as preprocessed program data with registrar-vouched +//! uniqueness, so a prover cannot choose them at all. + +use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; +use crate::tables::keccak_rnd::KeccakRoundOperation; +use crate::tables::types::BusId; +use crate::tables::types::{ + FE, GoldilocksExtension, GoldilocksField, VmTable, dword_wl, zeroed_fe_vec, +}; + +use super::layout; +use super::word::LfmWord; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// Column layout: `[TAG_LO, TAG_HI, IN[200], OUT[200], MU]`. +pub mod cols { + /// Low 32 bits of the tag (matches the core chip's `TIMESTAMP_0`, a `DWordWL`). + pub const TAG_LO: usize = 0; + /// High 32 bits of the tag (matches the core chip's `TIMESTAMP_1`). + pub const TAG_HI: usize = 1; + /// Input state, 200 bytes, lane-major: `IN + (x + 5y) * 8 + b`. + pub const IN: usize = 2; + /// Output state, 200 bytes, same indexing as [`IN`]. + pub const OUT: usize = IN + 200; // 202 + /// Is-real column; the multiplicity of both `Keccak` bus tokens. + pub const MU: usize = OUT + 200; // 402 + + pub const NUM_COLUMNS: usize = MU + 1; // 403 + + /// Column holding byte `b` of input lane `x + 5y`. + #[inline] + pub const fn in_byte(x: usize, y: usize, b: usize) -> usize { + IN + (x + 5 * y) * 8 + b + } + + /// Column holding byte `b` of output lane `x + 5y`. + #[inline] + pub const fn out_byte(x: usize, y: usize, b: usize) -> usize { + OUT + (x + 5 * y) * 8 + b + } +} + +/// One permutation: `output = keccak_f1600(input)`, labelled by `tag`. +#[derive(Debug, Clone, Copy)] +pub struct KeccakAdapterOperation { + /// Binds the request token to its reply token. MUST be unique across rows. + pub tag: u64, + pub input: [u64; 25], +} + +/// The two `Keccak` bus tokens, mirroring `tables::keccak::bus_interactions` +/// interactions 2 and 3 with the memory-coupled columns dropped. +#[allow(clippy::needless_range_loop)] +pub fn bus_interactions() -> Vec { + let mut interactions = Vec::with_capacity(2); + + let tag_values = || { + vec![ + BusValue::Packed { + start_column: cols::TAG_LO, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::TAG_HI, + packing: Packing::Direct, + }, + ] + }; + + // Request: send (tag, 0, input_state[200]). + { + let mut values = tag_values(); + values.push(BusValue::constant(0)); + for x in 0..5 { + for y in 0..5 { + for b in 0..8 { + values.push(BusValue::Packed { + start_column: cols::in_byte(x, y, b), + packing: Packing::Direct, + }); + } + } + } + interactions.push(BusInteraction::sender( + BusId::Keccak, + Multiplicity::Column(cols::MU), + values, + )); + } + + // Reply: receive (tag, 24, output_state[200]). + { + let mut values = tag_values(); + values.push(BusValue::constant(24)); + for x in 0..5 { + for y in 0..5 { + for b in 0..8 { + values.push(BusValue::Packed { + start_column: cols::out_byte(x, y, b), + packing: Packing::Direct, + }); + } + } + } + interactions.push(BusInteraction::receiver( + BusId::Keccak, + Multiplicity::Column(cols::MU), + values, + )); + } + + interactions +} + +/// One row per permutation; padding rows are all-zero (so `MU = 0` and they +/// send nothing). +pub fn generate_adapter_trace(ops: &[KeccakAdapterOperation]) -> TraceTable { + let n_rows = ops.len().next_power_of_two().max(4); + let mut trace = TraceTable::new_main( + zeroed_fe_vec(n_rows * cols::NUM_COLUMNS), + cols::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + + for (row, op) in ops.iter().enumerate() { + let [lo, hi] = dword_wl(op.tag); + table.set_fe(row, cols::TAG_LO, lo); + table.set_fe(row, cols::TAG_HI, hi); + + let output = permute(op.input); + for (lane, (&in_lane, &out_lane)) in op.input.iter().zip(output.iter()).enumerate() { + table.set_dword_bl(row, cols::IN + lane * 8, in_lane); + table.set_dword_bl(row, cols::OUT + lane * 8, out_lane); + } + + table.set_fe(row, cols::MU, FE::one()); + } + + trace +} + +/// `keccak-f[1600]` as the VM defines it — the same primitive the production +/// trace builder replays (`trace_builder.rs:635`). +pub fn permute(input: [u64; 25]) -> [u64; 25] { + let mut state = input; + executor::vm::instruction::execution::keccak_f1600(&mut state); + state +} + +// ========================================================================= +// Machine-word view of the state (the LFM_KECCAK chip's u32-half convention) +// ========================================================================= + +/// Half `h` of a state: the low (`h` even) or high (`h` odd) 32 bits of lane +/// `h / 2`. +/// +/// A keccak lane is a `u64` and so is *not* felt-representable — values in +/// `[p, 2^64)` exist — which is why machine-side keccak state travels as `u32` +/// halves, one per felt lane. +#[inline] +pub fn half_of(state: &[u64; 25], h: usize) -> u32 { + (state[h / 2] >> (32 * (h % 2))) as u32 +} + +/// A state as [`layout::keccak::NUM_WORDS`] machine words, four halves each. +/// +/// The top `WORD_SLOTS − NUM_HALVES` lanes of the last word are unused and set +/// to zero; the `LFM_KECCAK` chip pins them as bus tuple constants, so a +/// nonzero value there cannot balance. +pub fn state_to_words(state: &[u64; 25]) -> [LfmWord; layout::keccak::NUM_WORDS] { + core::array::from_fn(|j| { + core::array::from_fn(|l| { + let h = 4 * j + l; + if h < layout::keccak::NUM_HALVES { + FE::from(u64::from(half_of(state, h))) + } else { + FE::zero() + } + }) + }) +} + +/// The `BYTE_ALU[XOR]` lookups the `LFM_KECCAK` chip's absorb rows send: one +/// per rate byte, `PERM_IN[k] = STATE[k] ⊕ BLOCK[k]`. +/// +/// Permute rows send none — their XOR interactions are gated by `MODE_ABSORB`. +pub fn absorb_bitwise_ops(rows: &[super::executor::KeccakRow]) -> Vec { + use super::instr::KeccakMode; + let mut out = Vec::new(); + for r in rows.iter().filter(|r| r.mode == KeccakMode::Absorb) { + for k in 0..layout::keccak::RATE_BYTES { + let state_byte = (r.state[k / 8] >> (8 * (k % 8))) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + state_byte, + r.block[k], + )); + } + } + out +} + +/// The byte-REVERSED first 32 bytes of a state, as two machine words — the +/// value the production transcript's `sample()` both returns and re-absorbs. +/// +/// Reversed byte `j` is digest byte `31 − j`, so both the byte order within a +/// half and the order of the halves flip. The `LFM_KECCAK` chip produces this +/// with two extra bus sends over the SAME output byte columns; this is the host +/// mirror. +pub fn reversed_digest_words(state: &[u64; 25]) -> [LfmWord; 2] { + let mut digest = [0u8; 32]; + for (lane, chunk) in state[..4].iter().zip(digest.chunks_exact_mut(8)) { + chunk.copy_from_slice(&lane.to_le_bytes()); + } + digest.reverse(); + core::array::from_fn(|w| { + core::array::from_fn(|l| { + let h = 4 * w + l; + let mut half = [0u8; 4]; + half.copy_from_slice(&digest[4 * h..4 * h + 4]); + FE::from(u64::from(u32::from_le_bytes(half))) + }) + }) +} + +/// Reassembles the 25 `u64` lanes from 50 `u32` halves. +pub fn halves_to_state(halves: &[u32; layout::keccak::NUM_HALVES]) -> [u64; 25] { + core::array::from_fn(|lane| { + u64::from(halves[2 * lane]) | (u64::from(halves[2 * lane + 1]) << 32) + }) +} + +/// The `KECCAK_RND` operations matching `ops`: one per permutation, expanding to +/// 24 trace rows each. The round chip keys its rows on `timestamp`, which is our +/// tag. Its `output` field is dead (the trace builder recomputes the state round +/// by round) but is filled with the true value anyway. +pub fn round_operations(ops: &[KeccakAdapterOperation]) -> Vec { + ops.iter() + .map(|op| KeccakRoundOperation { + timestamp: op.tag, + input: op.input, + output: permute(op.input), + }) + .collect() +} + +/// BITWISE lookups the `KECCAK_RND` rows of `ops` send: exactly `24 * 1028` per +/// permutation. +/// +/// This is the per-round half of `trace_builder::collect_bitwise_from_keccak`, +/// forked rather than called: the original also emits the 105 address-shaped +/// lookups (1 `BYTE_ALU[AND]` alignment check, 4 `ARE_BYTES` on the address +/// bytes, 100 `IS_HALF` on the lane pointers) that belong to the dropped core +/// chip. Calling it with a synthetic address and subtracting would depend on +/// those counts staying fixed; forking the loop keeps the coupling explicit. +/// +/// The θ/ρ halfword shifts no longer emit HWSL lookups (120 sends/round removed: +/// 20 in θ, 100 in ρ) — they are enforced by inline μ-gated linear identities on +/// the round chip, matching main's `keccak_rnd::bus_interactions` / +/// `KeccakRndConstraints` and `trace_builder::collect_bitwise_from_keccak`. Per +/// round: 1148 − 120 = 1028. +#[allow(clippy::needless_range_loop)] +pub fn bitwise_ops_for(ops: &[KeccakAdapterOperation]) -> Vec { + use executor::vm::instruction::execution::{KECCAK_RC, KECCAK_RHO}; + + let mut out = Vec::with_capacity(ops.len() * 24 * 1028); + + for op in ops { + let mut state = op.input; + for round in 0..24 { + // --- theta: Cxz chain BYTE_ALU[XOR] (160) --- + let mut cxz = [[[0u8; 8]; 4]; 5]; + for x in 0..5 { + for b in 0..8 { + let v0 = ((state[x] >> (b * 8)) & 0xFF) as u8; + let v1 = ((state[x + 5] >> (b * 8)) & 0xFF) as u8; + cxz[x][0][b] = v0 ^ v1; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + v0, + v1, + )); + } + for stage in 1..4usize { + let y = stage + 1; + for b in 0..8 { + let prev = cxz[x][stage - 1][b]; + let sv = ((state[x + 5 * y] >> (b * 8)) & 0xFF) as u8; + cxz[x][stage][b] = prev ^ sv; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + prev, + sv, + )); + } + } + } + + // theta: ARE_BYTES on Cxz_left (20 pairs). The rotate-C-by-1 halfword + // shift no longer emits an HWSL lookup — it is enforced by an inline + // μ-gated linear identity on the round chip (matching main's + // `KeccakRndConstraints`; the HWSL sends were dropped from + // `keccak_rnd::bus_interactions`). Cxz_right is range-checked by + // IS_BIT polynomial constraints on the round chip (spec d75944ee). + let mut rotated_c = [[0u8; 8]; 5]; + for x in 0..5 { + let c = cxz[x][3]; + for hw in 0..4 { + let halfword = (c[hw * 2] as u16) | ((c[hw * 2 + 1] as u16) << 8); + let shifted = halfword << 1; // u16 wraps + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (shifted & 0xFF) as u8, + ((shifted >> 8) & 0xFF) as u8, + )); + } + let mut left_bytes = [0u8; 8]; + let mut right_bits = [0u8; 4]; + for hw in 0..4 { + let halfword = (c[hw * 2] as u16) | ((c[hw * 2 + 1] as u16) << 8); + let shifted = halfword << 1; + left_bytes[hw * 2] = (shifted & 0xFF) as u8; + left_bytes[hw * 2 + 1] = ((shifted >> 8) & 0xFF) as u8; + right_bits[hw] = (halfword >> 15) as u8; + } + for b in 0usize..8 { + let right_contribution = if b.is_multiple_of(2) { + right_bits[(b / 2 + 3) % 4] + } else { + 0 + }; + rotated_c[x][b] = left_bytes[b].wrapping_add(right_contribution); + } + } + + // theta: Dxz BYTE_ALU[XOR] (40) + let mut d_bytes = [[0u8; 8]; 5]; + for x in 0..5 { + for b in 0..8 { + let a = cxz[(x + 4) % 5][3][b]; + let rb = rotated_c[(x + 1) % 5][b]; + d_bytes[x][b] = a ^ rb; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + a, + rb, + )); + } + } + + // theta final: BYTE_ALU[XOR] (200) + let mut theta_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let lane = state[x + 5 * y]; + let mut d_lane = 0u64; + for b in 0..8 { + d_lane |= (d_bytes[x][b] as u64) << (b * 8); + } + theta_lanes[x + 5 * y] = lane ^ d_lane; + for b in 0..8 { + let s = ((lane >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + s, + d_bytes[x][b], + )); + } + } + } + + // rho: ARE_BYTES (200 pairs). The ρ halfword shifts no longer emit + // HWSL lookups — enforced by inline μ-gated identities on the round + // chip (matching main's `KeccakRndConstraints`). + for x in 0..5 { + for y in 0..5 { + let rho_offset = KECCAK_RHO[x][y] as usize; + let rnc_val = (rho_offset % 16) as u8; + let theta_lane = theta_lanes[x + 5 * y]; + for hw in 0..4 { + let halfword = ((theta_lane >> (hw * 16)) & 0xFFFF) as u16; + let (shifted, carry) = if rnc_val == 0 { + (halfword, 0u16) + } else { + (halfword << rnc_val, halfword >> (16 - rnc_val)) + }; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (shifted & 0xFF) as u8, + (carry & 0xFF) as u8, + )); + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((shifted >> 8) & 0xFF) as u8, + ((carry >> 8) & 0xFF) as u8, + )); + } + } + } + + // pi + let mut pi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let rotated = theta_lanes[x + 5 * y].rotate_left(KECCAK_RHO[x][y]); + let dst_x = y; + let dst_y = (2 * x + 3 * y) % 5; + pi_lanes[dst_x + 5 * dst_y] = rotated; + } + } + + // chi: BYTE_ALU[AND] (200) + BYTE_ALU[XOR] (200) + let mut chi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let not_next = !pi_lanes[(x + 1) % 5 + 5 * y]; + let next2 = pi_lanes[(x + 2) % 5 + 5 * y]; + let and_val = not_next & next2; + chi_lanes[x + 5 * y] = pi_lanes[x + 5 * y] ^ and_val; + for b in 0..8 { + let not_byte = ((not_next >> (b * 8)) & 0xFF) as u8; + let n2_byte = ((next2 >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluAnd, + not_byte, + n2_byte, + )); + let pi_byte = ((pi_lanes[x + 5 * y] >> (b * 8)) & 0xFF) as u8; + let and_byte = ((and_val >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + pi_byte, + and_byte, + )); + } + } + } + + // iota: BYTE_ALU[XOR] (8) + let rc_val = KECCAK_RC[round]; + for b in 0..8 { + let chi_byte = ((chi_lanes[0] >> (b * 8)) & 0xFF) as u8; + let rc_byte = ((rc_val >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + chi_byte, + rc_byte, + )); + } + + chi_lanes[0] ^= rc_val; + state = chi_lanes; + } + } + + out +} diff --git a/prover/src/lfm/keccak_host.rs b/prover/src/lfm/keccak_host.rs new file mode 100644 index 000000000..4d5886c44 --- /dev/null +++ b/prover/src/lfm/keccak_host.rs @@ -0,0 +1,198 @@ +//! Host side of the machine's keccak256: the byte-stream packing convention +//! and the padding the emitter bakes in as program constants. +//! +//! The machine has no bytes — its cells are felts — so a byte stream reaches +//! `edsl::keccak256` pre-packed as `u32` halves, four bytes each, little-endian. +//! That is the same convention the state itself uses (`keccak_adapter`), which +//! is what lets a rate block be assembled with plain `Pack` instructions and no +//! per-byte arithmetic. + +use crate::tables::types::FE; + +use super::layout::keccak::RATE_BYTES; + +/// Bytes carried by one `u32`-half felt. +pub const BYTES_PER_HALF: usize = 4; + +/// Packs a byte stream into `u32`-half felts, four bytes each, little-endian. +/// +/// The final half is zero-padded when `bytes.len()` is not a multiple of four. +/// The emitter relies on exactly that: where a half straddles the end of the +/// message it adds the padding constant to the stream half, and addition equals +/// bitwise-or only because the stream half's high bytes are known zero. +/// [`assert_high_bytes_zero`] is the executable statement of that obligation. +pub fn pack_stream(bytes: &[u8]) -> Vec { + bytes + .chunks(BYTES_PER_HALF) + .map(|chunk| { + let mut half = [0u8; BYTES_PER_HALF]; + half[..chunk.len()].copy_from_slice(chunk); + FE::from(u64::from(u32::from_le_bytes(half))) + }) + .collect() +} + +/// Number of halves [`pack_stream`] produces for `len_bytes`. +pub const fn num_stream_halves(len_bytes: usize) -> usize { + len_bytes.div_ceil(BYTES_PER_HALF) +} + +/// The keccak256 padded length: `pad10*1` always adds at least one byte, so the +/// message grows to the next multiple of the rate even when it already is one. +pub const fn padded_len(len_bytes: usize) -> usize { + (len_bytes / RATE_BYTES + 1) * RATE_BYTES +} + +/// Number of rate blocks the emitter absorbs for `len_bytes`. +pub const fn num_blocks(len_bytes: usize) -> usize { + padded_len(len_bytes) / RATE_BYTES +} + +/// The `pad10*1` byte at padded position `pos` for a message of `len_bytes`: +/// `0x01` at the first padding position, `0x80` at the last of the final block, +/// and `0x81` when they coincide. +pub fn pad_byte(len_bytes: usize, pos: usize) -> u8 { + debug_assert!(pos >= len_bytes && pos < padded_len(len_bytes)); + let mut v = 0u8; + if pos == len_bytes { + v |= 0x01; + } + if pos == padded_len(len_bytes) - 1 { + v |= 0x80; + } + v +} + +/// The padding contribution to half `h` of the padded message — the value the +/// emitter adds to (or uses in place of) the stream half. +/// +/// Returns `0` for halves that lie entirely inside the message. +pub fn pad_half(len_bytes: usize, h: usize) -> u64 { + let mut acc = 0u64; + for j in 0..BYTES_PER_HALF { + let pos = h * BYTES_PER_HALF + j; + if pos >= len_bytes && pos < padded_len(len_bytes) { + acc |= u64::from(pad_byte(len_bytes, pos)) << (8 * j); + } + } + acc +} + +/// Checks the packing obligation for the half that straddles the end of the +/// message: its bytes at or beyond `len_bytes` must be zero, or the emitter's +/// `stream_half + pad_half` would carry instead of merging. +pub fn assert_high_bytes_zero(stream: &[FE], len_bytes: usize) { + use math::field::traits::IsPrimeField; + let tail = len_bytes % BYTES_PER_HALF; + if tail == 0 { + return; + } + let h = len_bytes / BYTES_PER_HALF; + let v = crate::tables::types::GoldilocksField::canonical(stream[h].value()); + assert_eq!( + v >> (8 * tail), + 0, + "stream half {h} must be zero above byte {tail}: pack_stream guarantees it" + ); +} + +/// `keccak256` over `bytes`, as the production hasher computes it. The machine +/// program's public output is compared against this. +pub fn keccak256(bytes: &[u8]) -> [u8; 32] { + use crypto::hash::platform_keccak::PlatformKeccak256 as Keccak256; + use digest::Digest; + let mut h = Keccak256::new(); + h.update(bytes); + h.finalize().into() +} + +// ===================== Host model of DefaultTranscript ===================== + +/// Bytes in one squeeze; the duplex output buffer hands them out 8 at a time. +pub const SQUEEZE_LEN: usize = 32; + +/// Host mirror of the production `DefaultTranscript` (post-#841), tracking the +/// same state the machine emitter tracks at emit time. +/// +/// This exists so the emitter's static consumption schedule — which candidate +/// comes from which squeeze, and where absorbs invalidate the buffer — can be +/// derived and tested without a machine proof. It is checked against the real +/// `DefaultTranscript` in `machine_tests`. +#[derive(Clone)] +pub struct TranscriptModel { + /// Bytes absorbed since the last finalize (the hasher's pending input). + segment: Vec, + buf: [u8; SQUEEZE_LEN], + /// Bytes already handed out of `buf`; `SQUEEZE_LEN` means empty. + pos: usize, +} + +impl TranscriptModel { + pub fn new(data: &[u8]) -> Self { + Self { + segment: data.to_vec(), + buf: [0u8; SQUEEZE_LEN], + pos: SQUEEZE_LEN, + } + } + + /// Absorbing invalidates the buffer: a later challenge must depend on this + /// input, so bytes squeezed before it are dropped. + pub fn append(&mut self, bytes: &[u8]) { + self.pos = SQUEEZE_LEN; + self.segment.extend_from_slice(bytes); + } + + /// Finalize, reverse, re-absorb the reversed bytes, return them. Also + /// invalidates the buffer. + pub fn sample(&mut self) -> [u8; SQUEEZE_LEN] { + let mut digest = keccak256(&self.segment); + digest.reverse(); + self.segment = digest.to_vec(); + self.pos = SQUEEZE_LEN; + digest + } + + /// Next big-endian 64-bit candidate, refilling with one squeeze when fewer + /// than 8 bytes remain. + pub fn next_u64(&mut self) -> u64 { + if self.pos + 8 > SQUEEZE_LEN { + self.buf = self.sample(); + self.pos = 0; + } + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(&self.buf[self.pos..self.pos + 8]); + self.pos += 8; + u64::from_be_bytes(bytes) + } + + /// Whether the next `next_u64` would refill (the emitter needs this to know + /// where to place a keccak row). + pub fn would_refill(&self) -> bool { + self.pos + 8 > SQUEEZE_LEN + } + + pub fn pos(&self) -> usize { + self.pos + } +} + +/// THE IDENTITY THE MACHINE EMITTER RELIES ON. +/// +/// The four big-endian candidates carved out of a reversed digest are exactly +/// the ORIGINAL digest's first four `u64` lanes, in reverse lane order — so the +/// machine never has to reverse anything to read candidates. +/// +/// Candidate `i` is `Σ_{k<8} reversed[8i+k]·2^(8(7−k))`, and `reversed[j]` is +/// `digest[31−j]`, so substituting `m = 7−k` gives +/// `Σ_{m<8} digest[24−8i+m]·2^(8m)` — the LITTLE-endian `u64` at digest byte +/// offset `24−8i`, i.e. keccak state lane `3−i`. The big-endian read and the +/// byte reversal cancel exactly. +/// +/// Consequence: candidates come straight off the plain digest words (state +/// lanes, already `u32` halves on the bus), and the reversed digest is needed +/// only for the RE-ABSORB. Verified by `be_candidates_are_plain_state_lanes`. +pub fn candidate_from_state(state: &[u64; 25], index: usize) -> u64 { + debug_assert!(index < 4); + state[3 - index] +} diff --git a/prover/src/lfm/keccak_probe.rs b/prover/src/lfm/keccak_probe.rs new file mode 100644 index 000000000..bf4963418 --- /dev/null +++ b/prover/src/lfm/keccak_probe.rs @@ -0,0 +1,294 @@ +//! R1a probe: prove real `keccak-f[1600]` permutations through the UNCHANGED +//! production `KECCAK_RND` + `KECCAK_RC` + `BITWISE` AIRs, driven by +//! [`super::keccak_adapter`] instead of the VM-coupled `KECCAK` core chip. +//! +//! This is the entry gate for hosting the keccak table family inside the LFM +//! recursion machine's AIR set: it establishes that the family's only coupling +//! to the VM is the core chip's two `Keccak` bus tokens, and that a chip owning +//! nothing but those tokens is a sufficient driver. +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::constraints::builder::EmptyConstraints; +use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::MultiProofView; +use stark::prover::IsStarkProver; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::IsStarkVerifier; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField, VmTable}; +use crate::tables::{bitwise, keccak_rc, keccak_rnd}; +use crate::test_utils::{create_bitwise_air, create_keccak_rc_air, create_keccak_rnd_air}; + +use super::keccak_adapter::{self, KeccakAdapterOperation, cols}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type AdapterAir = AirWithBuses; +type DynAir<'a> = &'a dyn AIR; + +const PROBE_TAG: &[u8] = b"LFM_R1A_KECCAK_PROBE_V1"; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("probe options") +} + +fn transcript() -> crate::hash_pin::BlockTranscript { + let mut t = crate::hash_pin::block_transcript(&[]); + t.append_bytes(PROBE_TAG); + t +} + +fn adapter_air(opts: &ProofOptions) -> AdapterAir { + AirWithBuses::new( + cols::NUM_COLUMNS, + AuxiliaryTraceBuildData { + interactions: keccak_adapter::bus_interactions(), + }, + opts, + 1, + EmptyConstraints, + ) + .with_name("KECCAK_ADAPTER") +} + +/// Three permutations, distinct nontrivial inputs, distinct tags. +/// +/// Tags are `row + 1` here. The production LFM adapter will source them from +/// preprocessed program data — see the tag-uniqueness note on +/// [`super::keccak_adapter`] and `duplicate_tag_output_swap_accepts_demonstrating_hazard`. +fn probe_ops() -> Vec { + (0..3u64) + .map(|i| { + let mut input = [0u64; 25]; + for (lane, slot) in input.iter_mut().enumerate() { + *slot = (lane as u64) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(i.wrapping_mul(0xD1B5_4A32_D192_ED03)) + ^ 0x0123_4567_89AB_CDEF; + } + KeccakAdapterOperation { tag: i + 1, input } + }) + .collect() +} + +/// The four traces, in AIR order: adapter, KECCAK_RND, KECCAK_RC, BITWISE. +fn build_traces(ops: &[KeccakAdapterOperation]) -> [TraceTable; 4] { + let adapter = keccak_adapter::generate_adapter_trace(ops); + let rnd = keccak_rnd::generate_keccak_rnd_trace(&keccak_adapter::round_operations(ops)); + + let mut rc = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut rc, ops.len()); + + let mut hist = bitwise::BitwiseHistogram::new(); + hist.add_ops(&keccak_adapter::bitwise_ops_for(ops)); + // The round table's padding rows send its byte checks as `ARE_BYTES[0, 0]`. + hist.bump_n( + bitwise::BitwiseOperation::byte_op(bitwise::BitwiseOperationType::AreBytes, 0, 0), + keccak_rnd::padding_are_bytes(ops.len()), + ); + let mut bw = bitwise::generate_bitwise_trace(); + hist.fill_multiplicities(&mut bw); + + [adapter, rnd, rc, bw] +} + +/// Prove the four-AIR set over `traces`, which the caller may have tampered +/// with after generation. +fn prove_traces( + opts: &ProofOptions, + adapter: &AdapterAir, + traces: &mut [TraceTable; 4], +) -> Result, stark::prover::ProvingError> { + let rnd_air = create_keccak_rnd_air(opts); + let rc_air = create_keccak_rc_air(opts).with_preprocessed( + keccak_rc::preprocessed_commitment(opts), + keccak_rc::NUM_PRECOMPUTED_COLS, + ); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + + let [t0, t1, t2, t3] = traces; + let pairs: Vec<(DynAir, &mut TraceTable, &())> = vec![ + (adapter, t0, &()), + (&rnd_air, t1, &()), + (&rc_air, t2, &()), + (&bw_air, t3, &()), + ]; + let mut t = transcript(); + crate::hash_pin::BlockProver::multi_prove(pairs, &mut t, Default::default()) +} + +fn verify_proof( + opts: &ProofOptions, + adapter: &AdapterAir, + proof: &stark::proof::stark::MultiProof, +) -> bool { + let rnd_air = create_keccak_rnd_air(opts); + let rc_air = create_keccak_rc_air(opts).with_preprocessed( + keccak_rc::preprocessed_commitment(opts), + keccak_rc::NUM_PRECOMPUTED_COLS, + ); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + let refs: Vec = vec![adapter, &rnd_air, &rc_air, &bw_air]; + let mut vt = transcript(); + crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(proof), + &mut vt, + &FEE::zero(), + ) +} + +/// Prove + verify, optionally corrupting the adapter trace in between. +/// +/// `Err` means the prover refused. The adapter carries no constraints, so +/// tampering with its values is expected to reach the verifier and be caught +/// there (`Ok(false)`); the reject tests assert that stronger outcome rather +/// than accepting either failure, so a prover-side refusal would show up as a +/// change in behavior instead of hiding behind a passing test. +fn round_trip(mutate: impl FnOnce(&mut TraceTable)) -> Result { + let opts = options(); + let adapter = adapter_air(&opts); + let ops = probe_ops(); + let mut traces = build_traces(&ops); + mutate(&mut traces[0]); + match prove_traces(&opts, &adapter, &mut traces) { + Ok(proof) => Ok(verify_proof(&opts, &adapter, &proof)), + Err(e) => Err(format!("{e:?}")), + } +} + +/// Assert the prover accepted the tampered trace and the verifier rejected it. +fn assert_proves_but_fails_verification(what: &str, mutate: impl FnOnce(&mut TraceTable)) { + match round_trip(mutate) { + Ok(true) => panic!("{what} must break the Keccak bus balance, but the proof verified"), + Ok(false) => {} + Err(e) => panic!("{what} should reach the verifier, but the prover refused first: {e}"), + } +} + +#[test] +fn adapter_probe_proves_real_permutations() { + let ops = probe_ops(); + let traces = build_traces(&ops); + + // The adapter's OUT columns must be the real permutation, byte for byte. + for (row, op) in ops.iter().enumerate() { + let expected = keccak_adapter::permute(op.input); + for (lane, &value) in expected.iter().enumerate() { + for b in 0..8 { + assert_eq!( + traces[0].main_table.get_row(row)[cols::OUT + lane * 8 + b], + FE::from(u64::from((value >> (b * 8)) as u8)), + "OUT byte ({lane}, {b}) of row {row}" + ); + } + } + } + + // Known-answer vector: keccak-f[1600] of the all-zero state. Same constant + // the executor pins in `executor/src/tests/keccak_tests.rs`. + let zero_out = keccak_adapter::permute([0u64; 25]); + assert_eq!( + zero_out[0], 0xF1258F7940E1DDE7, + "keccak_f1600(0) lane 0 must match the published vector" + ); + + // The BITWISE feed is exactly the per-round half of the production + // collector: 1028 lookups per round, no address-shaped lookups and no HWSL + // (the θ/ρ shifts are inline μ-gated identities on the round chip). + assert_eq!( + keccak_adapter::bitwise_ops_for(&ops).len(), + ops.len() * 24 * 1028, + "per-permutation BITWISE lookup count" + ); + + // The round-trip through the AIRs is the real check: KECCAK_RND enforces + // all 24 rounds, and the bus only balances if the adapter's OUT state is + // what those rounds actually produce from its IN state. + assert_eq!( + round_trip(|_| {}), + Ok(true), + "honest keccak adapter proof must verify" + ); +} + +#[test] +fn tampered_output_byte_rejects() { + assert_proves_but_fails_verification("a flipped OUT byte", |t| { + let old = t.main_table.get_row(1)[cols::out_byte(2, 3, 4)]; + t.main_table + .set_fe(1, cols::out_byte(2, 3, 4), old + FE::one()); + }); +} + +#[test] +fn tampered_input_byte_rejects() { + assert_proves_but_fails_verification("a flipped IN byte", |t| { + let old = t.main_table.get_row(0)[cols::in_byte(4, 1, 7)]; + t.main_table + .set_fe(0, cols::in_byte(4, 1, 7), old + FE::one()); + }); +} + +#[test] +fn padding_row_multiplicity_rejects() { + // Row 3 is padding (3 real ops, height 4). Turning it real makes the + // adapter send a (tag=0, round=0, all-zero state) request and receive a + // (tag=0, round=24, all-zero state) reply that no KECCAK_RND row answers. + assert_proves_but_fails_verification("an is-real padding row", |t| { + t.main_table.set_fe(3, cols::MU, FE::one()) + }); +} + +/// DOCUMENTS A HAZARD — this test asserts that a forgery SUCCEEDS. +/// +/// Nothing in the bus contract binds a request token to its reply token except +/// the tag. Given two permutations sharing a tag, a prover can hand back each +/// one's output as the other's: the reply multiset `{(tag, 24, A_out), +/// (tag, 24, B_out)}` is unchanged by the swap, so the `Keccak` bus still +/// balances and the proof verifies even though neither adapter row states a +/// true permutation. +/// +/// This is why [`super::keccak_adapter`] requires unique tags, and why the +/// production LFM adapter must carry them as preprocessed program data with +/// registrar-vouched uniqueness rather than as prover-chosen witness. If this +/// test ever starts FAILING, something began binding request to reply and the +/// tag-uniqueness obligation should be re-derived before it is relaxed. +#[test] +fn duplicate_tag_output_swap_accepts_demonstrating_hazard() { + let opts = options(); + let adapter = adapter_air(&opts); + + let mut ops = probe_ops(); + ops.truncate(2); + ops[1].tag = ops[0].tag; // the whole point: duplicate tag + + let mut traces = build_traces(&ops); + assert_ne!( + keccak_adapter::permute(ops[0].input), + keccak_adapter::permute(ops[1].input), + "the two outputs must differ or the swap is a no-op" + ); + + // Swap the two rows' 200 OUT bytes. + for col in cols::OUT..cols::MU { + let a = traces[0].main_table.get_row(0)[col]; + let b = traces[0].main_table.get_row(1)[col]; + traces[0].main_table.set_fe(0, col, b); + traces[0].main_table.set_fe(1, col, a); + } + + let proof = prove_traces(&opts, &adapter, &mut traces).expect("locally consistent"); + assert!( + verify_proof(&opts, &adapter, &proof), + "documents the tag-uniqueness obligation: with duplicate tags the swapped \ + outputs still balance the bus, so the verifier cannot catch the forgery" + ); +} diff --git a/prover/src/lfm/layout.rs b/prover/src/lfm/layout.rs new file mode 100644 index 000000000..1a8f29af8 --- /dev/null +++ b/prover/src/lfm/layout.rs @@ -0,0 +1,364 @@ +//! Instruction column-group layouts — the single source of truth shared by +//! the compiler's group emission (Milestone A), the admission validator, and +//! the chips' preprocessed column constants (Milestone B). +//! +//! Each chip's instruction fields (addresses, opcode selectors, +//! multiplicities, pooled constants) form its *instruction column group*: +//! the leading, preprocessed columns of that chip's trace, committed once per +//! program and supplied at verify time. Value columns follow after +//! `PREP_WIDTH` and are defined by the chips. + +/// `LFM_CONST` — pooled constants and immediates. +pub mod const_ { + pub const ADDR: usize = 0; + pub const V0: usize = 1; // .. V3 = 4 + pub const MULT: usize = 5; + pub const PREP_WIDTH: usize = 6; +} + +/// `LFM_BALU` — Goldilocks ALU. +pub mod balu { + pub const A_ADDR: usize = 0; + pub const B_ADDR: usize = 1; + pub const C_ADDR: usize = 2; + pub const OUT_ADDR: usize = 3; + pub const SEL_ADD: usize = 4; + pub const SEL_SUB: usize = 5; + pub const SEL_MUL: usize = 6; + pub const SEL_DIV: usize = 7; + pub const SEL_MULADD: usize = 8; + pub const MULT: usize = 9; + pub const PREP_WIDTH: usize = 10; + pub const NUM_SELECTORS: usize = 5; +} + +/// `LFM_XALU` — Fp3 ALU (word lanes 0–2, `w³ = 2`). +pub mod xalu { + pub const A_ADDR: usize = 0; + pub const B_ADDR: usize = 1; + pub const C_ADDR: usize = 2; + pub const OUT_ADDR: usize = 3; + pub const SEL_ADD: usize = 4; + pub const SEL_SUB: usize = 5; + pub const SEL_MUL: usize = 6; + pub const SEL_DIV: usize = 7; + pub const SEL_MULADD: usize = 8; + pub const SEL_MULBASE: usize = 9; + pub const MULT: usize = 10; + pub const PREP_WIDTH: usize = 11; + pub const NUM_SELECTORS: usize = 6; +} + +/// `LFM_SELECT` — conditional cell swap. +pub mod select { + pub const BIT_ADDR: usize = 0; + pub const INL_ADDR: usize = 1; + pub const INR_ADDR: usize = 2; + pub const OUTL_ADDR: usize = 3; + pub const OUTR_ADDR: usize = 4; + pub const MULT_L: usize = 5; + pub const MULT_R: usize = 6; + pub const IS_REAL: usize = 7; + pub const PREP_WIDTH: usize = 8; +} + +/// `LFM_BITDEC` — canonical 64-bit decomposition. Per-bit pairs +/// `(BIT_ADDR_i, MULT_i)` at `2 + 2i` / `3 + 2i`. +pub mod bitdec { + pub const IN_ADDR: usize = 0; + pub const IS_REAL: usize = 1; + pub const NUM_BITS: usize = 64; + pub const fn bit_addr(i: usize) -> usize { + 2 + 2 * i + } + pub const fn bit_mult(i: usize) -> usize { + 3 + 2 * i + } + /// The two BIG-ENDIAN `u32` halves of the decomposed value, sent as + /// LINEAR forms over the bit columns — the keccak `REV_ADDR` pattern: + /// two interactions, four preprocessed columns, NO value columns. This + /// is what lets `felt_be_halves` cost one row instead of one row plus + /// 64 base-ALU rows: the byte swap lives in the bus coefficients. + pub const HALF0_ADDR: usize = 2 + 2 * NUM_BITS; // 130 + pub const HALF0_MULT: usize = HALF0_ADDR + 1; + pub const HALF1_ADDR: usize = HALF0_ADDR + 2; + pub const HALF1_MULT: usize = HALF0_ADDR + 3; + pub const PREP_WIDTH: usize = 2 + 2 * NUM_BITS + 4; // 134 +} + +/// `LFM_HASH` — the hash chiplet (frozen tuple contract). +/// +/// Four mode selectors, all preprocessed, at most one of them set: +/// +/// | selector | shape | domain | +/// |---|---|---| +/// | `MODE_C` | 2 cells → 1 | Merkle parent / 2-to-1 compress | +/// | `MODE_T` | 2 cells → 1 | a Fiat–Shamir transcript step | +/// | `MODE_L` | **1 cell → 1** | a **leaf** over four arbitrary FIELD ELEMENTS† | +/// | `MODE_P` | 3 cells → 3 | the full permutation | +/// +/// Being preprocessed is what makes them trustworthy: a row's mode is fixed by +/// its position in the committed instruction group, so a prover chooses neither +/// which domain a row hashes in nor which input semantics it has. +/// +/// † The mode is a shape the machine offers; whether a leaf and a parent are +/// actually different FUNCTIONS is the hasher's business. BLAKE3 separates them +/// by tag, and a single-domain hasher does not — see `LfmHasher::leaf_out`. +pub mod hash { + pub const IN_ADDR0: usize = 0; + pub const IN_ADDR1: usize = 1; + pub const IN_ADDR2: usize = 2; + pub const OUT_ADDR0: usize = 3; + pub const OUT_ADDR1: usize = 4; + pub const OUT_ADDR2: usize = 5; + pub const MODE_C: usize = 6; + pub const MODE_P: usize = 7; + /// The transcript-domain selector. + /// + /// A FRESH column, not a repurposed `MODE_P`: `MODE_P` is pinned to zero + /// under BLAKE3 but still carries its own meaning under `Test` and + /// `Poseidon`, and one preprocessed column meaning two things under two + /// hashers is worse than the column it saves. + /// + /// It sits INSIDE the selector run rather than after the multiplicities, + /// because the admission validator's one-hot check reads the selectors as a + /// contiguous span (`NUM_SELECTORS` from `MODE_C`). A selector parked past + /// the mults would be outside that span and silently unchecked, which is + /// the sort of gap that only shows up when someone forges a row. + pub const MODE_T: usize = 8; + /// The LEAF-domain selector, and the machine's felt-input mode. + /// + /// **`MODE_L` implies felt-input semantics** — that is a decision, not an + /// inference. A leaf row reads ONE cell of four arbitrary Goldilocks + /// elements and hashes them as eight checked `u32` halves under the `"LFML"` + /// tag, which is what lets FRI data reach a hash whose inputs are `u32` + /// lanes. It is a constraint rather than a convention: a `MODE_L` row that + /// skipped the canonicity block would be unprovable. + /// + /// Placed inside the selector run for the reason [`MODE_T`] gives, which is + /// the same mistake caught once already and spec'd since so it is not made + /// a third time. + pub const MODE_L: usize = 9; + /// Mode selectors, contiguous from [`MODE_C`]: exactly one is set on a real + /// row. + pub const NUM_SELECTORS: usize = 4; + pub const MULT0: usize = 10; + pub const MULT1: usize = 11; + pub const MULT2: usize = 12; + pub const PREP_WIDTH: usize = 13; +} + +/// `LFM_KECCAK` — the keccak-f[1600] adapter: binds 13 machine words of state +/// to the production `KECCAK_RND` family's two `Keccak`-bus tokens. +/// +/// A keccak lane is a `u64`, which is **not** felt-representable (values in +/// `[p, 2^64)` exist), so machine-side keccak state travels as `u32` halves, +/// one half per felt lane: 25 lanes = 50 halves = 13 words of 4 lanes, with the +/// last word's top two lanes unused (tuple constants — see `chips::keccak`). +pub mod keccak { + /// Low 32 bits of the row's tag (matches `KECCAK_RND`'s `DWordWL` timestamp). + pub const TAG_LO: usize = 0; + /// High 32 bits of the row's tag. + pub const TAG_HI: usize = 1; + /// Machine words per keccak state: `ceil(50 / 4)`. + pub const NUM_WORDS: usize = 13; + /// `u32` halves per keccak state: `2 × 25`. + pub const NUM_HALVES: usize = 50; + /// Half slots the words provide: `4 × NUM_WORDS`. The top + /// `WORD_SLOTS − NUM_HALVES = 2` are unused and pinned to zero on the bus. + pub const WORD_SLOTS: usize = 4 * NUM_WORDS; + /// Sponge rate for keccak256: 136 bytes = 17 lanes = 34 halves. + pub const RATE_BYTES: usize = 136; + pub const RATE_LANES: usize = 17; + pub const BLOCK_HALVES: usize = RATE_BYTES / 4; // 34 + /// Machine words per rate block: `ceil(34 / 4)`. The top + /// `4 * BLOCK_WORDS − BLOCK_HALVES = 2` half slots are unused. + pub const BLOCK_WORDS: usize = BLOCK_HALVES.div_ceil(4); // 9 + + pub const IN_ADDR0: usize = 2; // ..IN_ADDR12 = 14 + pub const OUT_ADDR0: usize = IN_ADDR0 + NUM_WORDS; // 15 ..27 + pub const MULT0: usize = OUT_ADDR0 + NUM_WORDS; // 28 ..40 + pub const BLOCK_ADDR0: usize = MULT0 + NUM_WORDS; // 41 ..49 + /// One-hot mode selectors. Their sum is the row's is-real flag, so a + /// padding row (both zero) emits no bus tokens. + pub const MODE_PERM: usize = BLOCK_ADDR0 + BLOCK_WORDS; // 50 + pub const MODE_ABSORB: usize = MODE_PERM + 1; // 51 + /// The production transcript's `sample()` finalizes, REVERSES the 32 digest + /// bytes, absorbs the reversed bytes, and returns them — the returned + /// challenge and the next segment's prefix are the same 32 bytes. Reversal + /// is free here: the bus recomposes each `u32` half from byte columns + /// anyway, so a second send with the coefficients (and half order) flipped + /// costs two interactions and four preprocessed columns, and NO value + /// columns. `sample()` is byte-for-byte identical pre- and post-#841, so + /// this primitive is independent of which transcript revision is targeted. + pub const REV_ADDR0: usize = MODE_ABSORB + 1; // 52 + pub const REV_ADDR1: usize = REV_ADDR0 + 1; // 53 + pub const REV_MULT0: usize = REV_ADDR1 + 1; // 54 + pub const REV_MULT1: usize = REV_MULT0 + 1; // 55 + pub const PREP_WIDTH: usize = REV_MULT1 + 1; // 56 + + /// Machine words in a keccak256 digest: 32 bytes = 8 halves. + pub const DIGEST_WORDS: usize = 2; + + pub const fn rev_addr(word: usize) -> usize { + REV_ADDR0 + word + } + pub const fn rev_mult(word: usize) -> usize { + REV_MULT0 + word + } + + pub const fn in_addr(word: usize) -> usize { + IN_ADDR0 + word + } + pub const fn out_addr(word: usize) -> usize { + OUT_ADDR0 + word + } + /// Write-multiplicity of output word `word`. + pub const fn mult(word: usize) -> usize { + MULT0 + word + } + /// Address of rate-block word `word` (absorb rows only). + pub const fn block_addr(word: usize) -> usize { + BLOCK_ADDR0 + word + } + + /// The tag a keccak row carries, as a function of its row index. + /// + /// SOUNDNESS: the tag is the *only* thing binding a row's request token to + /// its reply token, so tags must be unique across real rows — with a + /// duplicate, a prover can swap two permutations' outputs and the bus still + /// balances (pinned empirically by `keccak_probe`'s + /// `duplicate_tag_output_swap_accepts_demonstrating_hazard`). Making the tag + /// the row ordinal gives uniqueness by construction, and putting it in the + /// *preprocessed* group means the prover cannot choose it at all. The + /// admission validator re-checks uniqueness independently — this function is + /// the compiler's rule, not the guarantee. + pub const fn tag_for_row(row: usize) -> u64 { + row as u64 + 1 + } +} + +/// `LFM_BLAKE3` — the BLAKE3 compression chip: seven machine words of +/// `h[8] ‖ m[16] ‖ (t_lo, t_hi, block_len, flags)` in, four words of +/// `out[0..16]` out. +/// +/// Every input `u32` travels as one felt lane, four lanes to a machine word, +/// and 28 divides by 4 exactly — so unlike [`keccak`] there are **no spare half +/// slots** to pin as tuple constants and no `KeccakSpareLaneNonZero` analogue. +/// A lane at or above `2^32` is still rejected: the chip recomposes each `u32` +/// from four BITWISE-constrained byte columns, so no such value exists on the +/// AIR side. +/// +/// This module holds the *instruction column group* — addresses, +/// multiplicities, the is-real flag — the same split [`keccak`] uses. The +/// mixing core's value columns stay with the chip, in `blake3_chip::cols`, +/// which re-exports everything here so there is one set of constants. +pub mod blake3 { + /// `u32` words the chip reads: `h[8] | m[16] | t_lo | t_hi | block_len | flags`. + pub const IN_U32: usize = 28; + /// `u32` words the chip writes: the full 16-word compression output. + pub const OUT_U32: usize = 16; + /// Machine words read (four `u32` lanes each). 28 / 4 divides exactly. + pub const IN_WORDS: usize = IN_U32 / 4; // 7 + /// Machine words written. 16 / 4 divides exactly. + pub const OUT_WORDS: usize = OUT_U32 / 4; // 4 + + /// Machine words in a 32-byte BLAKE3 digest. + /// + /// `Blake3Chain`'s digest is `out[0..8]` little-endian, i.e. output `u32` + /// words 0..8, i.e. output machine words 0 and 1 — so the digest is a + /// PREFIX of what the chip already writes, and reading it costs nothing. + /// The byte convention is keccak's exactly (digest byte `j` is byte `j % 4` + /// of half `j / 4`), which is what lets the whole construction layer be a + /// 1:1 port rather than a re-derivation. + pub const DIGEST_WORDS: usize = 2; + + pub const IN_ADDR0: usize = 0; // ..IN_ADDR6 = 6 + pub const OUT_ADDR0: usize = IN_ADDR0 + IN_WORDS; // 7 ..10 + /// Read count of each output word (its LogUp send multiplicity). + pub const MULT0: usize = OUT_ADDR0 + OUT_WORDS; // 11 ..14 + /// The byte-REVERSED digest, as two more written words. + /// + /// `DefaultTranscript::sample()` finalizes, reverses the 32 digest bytes, + /// absorbs the reversed bytes and returns them — hash-agnostically, so the + /// BLAKE3 configuration squeezes exactly the same way the keccak one does + /// (`crypto/crypto/src/fiat_shamir/default_transcript.rs`). Reversal is + /// free at the recomposition boundary for the same reason it is free for + /// [`keccak`]: the bus already rebuilds each `u32` lane as a linear + /// combination of four byte columns, so flipping the coefficient order and + /// the lane order is a different `Linear` over the SAME columns. Two + /// interactions and four preprocessed columns, and **no value columns**. + /// + /// Without it every in-machine squeeze would pay eight explicit byteswaps — + /// a `LFM_BITDEC` row and 64 `LFM_BALU` rows each — for a digest the chip is + /// already holding byte by byte. Rows that need no reversed digest leave + /// `REV_MULT` at zero and the two sends are inert. + pub const REV_ADDR0: usize = MULT0 + OUT_WORDS; // 15, 16 + pub const REV_MULT0: usize = REV_ADDR0 + DIGEST_WORDS; // 17, 18 + /// Is-real flag: gates every constraint, every read and every write. + pub const MU: usize = REV_MULT0 + DIGEST_WORDS; // 19 + pub const PREP_WIDTH: usize = MU + 1; // 20 + + pub const fn in_addr(word: usize) -> usize { + IN_ADDR0 + word + } + pub const fn out_addr(word: usize) -> usize { + OUT_ADDR0 + word + } + /// Write-multiplicity of output word `word`. + pub const fn mult(word: usize) -> usize { + MULT0 + word + } + pub const fn rev_addr(word: usize) -> usize { + REV_ADDR0 + word + } + pub const fn rev_mult(word: usize) -> usize { + REV_MULT0 + word + } +} + +/// `LFM_LANES` — word ↔ lane conversion (Pack / Unpack). Pack rows receive +/// four lane cells and send the assembled word; Unpack rows receive a word +/// and send its four lanes as base cells. The shared value columns appear in +/// both tuples, which IS the semantics — the chip has no constraints. +pub mod lanes { + pub const WORD_ADDR: usize = 0; + pub const LANE_ADDR0: usize = 1; // ..LANE_ADDR3 = 4 + pub const MODE_PACK: usize = 5; + pub const MODE_UNPACK: usize = 6; + pub const WORD_MULT: usize = 7; // write-mult of the word (Pack rows only) + pub const LANE_MULT0: usize = 8; // ..LANE_MULT3 = 11 (Unpack rows only) + pub const PREP_WIDTH: usize = 12; +} + +/// `LFM_HINT` — arena ingestion. +pub mod hint { + pub const OUT_ADDR: usize = 0; + pub const MULT: usize = 1; + pub const PREP_WIDTH: usize = 2; +} + +/// `LFM_PUBLIC` — attestation output. +pub mod public { + pub const IN_ADDR: usize = 0; + pub const INDEX: usize = 1; + pub const IS_REAL: usize = 2; + pub const PREP_WIDTH: usize = 3; +} + +/// `LFM_RANGE` — the fixed 2^16 lookup table (program-independent; its group +/// is materialized at commitment time, Milestone B). +pub mod range { + pub const VALUE: usize = 0; + pub const PREP_WIDTH: usize = 1; + pub const NUM_ROWS: usize = 1 << 16; +} + +/// Minimum padded height for any instruction column group (the in-tree +/// `.next_power_of_two().max(4)` convention). +pub const MIN_GROUP_ROWS: usize = 4; + +/// Pads a real row count to its committed height. +pub fn padded_rows(real_rows: usize) -> usize { + real_rows.next_power_of_two().max(MIN_GROUP_ROWS) +} diff --git a/prover/src/lfm/lde.rs b/prover/src/lfm/lde.rs new file mode 100644 index 000000000..b9158a64b --- /dev/null +++ b/prover/src/lfm/lde.rs @@ -0,0 +1,173 @@ +//! Low-degree extension, emitted as machine instructions. +//! +//! A preprocessed column reaches its Merkle commitment as evaluations on the +//! LDE domain, and production gets there in two steps — +//! `Polynomial::interpolate_fft` then `evaluate_polynomial_on_lde_domain` +//! (`tables/register.rs::commit_register_columns` is one caller of exactly this +//! pair). Any leg that must DERIVE a preprocessed commitment rather than read +//! it has to emit that transform, because the values are what the derivation +//! binds; hinting the extended column would hand the prover a degree of freedom +//! the protocol does not give them. +//! +//! Everything here is shape-static. The domain size, the blowup, the coset +//! offset and therefore every twiddle are compile-time constants of the emitted +//! program: host-side `for` loops unroll and nothing loop-shaped reaches the +//! machine, exactly as in [`super::edsl`]. +//! +//! ## Why cosets rather than one big transform +//! +//! Production zero-pads the `n` coefficients to `n·blowup` and runs a single +//! transform of that size. Emitting that shape would cost +//! `(n·blowup)/2 · log₂(n·blowup)` butterflies. Splitting the output domain +//! into its `blowup` cosets of the size-`n` subgroup instead costs +//! `blowup · (n/2 · log₂ n)` butterflies plus `blowup · n` scaling +//! multiplications. At two `LFM_BALU` rows per butterfly and one per scaling +//! (see [`butterfly`]) that is `n·log₂n + blowup·(n + n·log₂n)` rows per column +//! against `n·log₂n + n + n·blowup·log₂(n·blowup)` — **9,088 against 11,264** +//! at the register shape (`n = 128`, blowup 8), and the gap widens with the +//! blowup. +//! +//! Both are arithmetic, not measurement, and both are PER COLUMN; +//! `machine_tests::register_derivation_cost` asserts the emitted total (two +//! columns, 18,176 rows at that shape) against the first formula. The two +//! schemes evaluate the same polynomial on the same points, and the +//! differential tests against production are what says so. +//! +//! ## What this cannot see +//! +//! The emitter is validated by differential tests against production's own +//! `interpolate_fft`/`evaluate_polynomial_on_lde_domain` pair over the field +//! elements those functions produce. It says nothing about domains whose size +//! exceeds Goldilocks' two-adicity (`root_of_unity` panics there rather than +//! emitting a wrong program), and nothing about extension-field columns — +//! preprocessed columns are base-field throughout. + +use math::fft::bit_reversing::reverse_index; +use math::field::traits::IsFFTField; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::builder::{Felt, LfmBuilder}; + +/// The `2^log_n`-th root of unity production's FFT uses. +/// +/// `get_primitive_root_of_unity` is the same entry point `LayerTwiddles` builds +/// its twiddles from, and it is defined by repeated squaring of the field's +/// two-adic generator, so `root_of_unity(k + 1)² == root_of_unity(k)`. That +/// nesting is what lets the coset decomposition below index the big domain with +/// the small domain's root. +pub fn root_of_unity(log_n: u32) -> FE { + GoldilocksField::get_primitive_root_of_unity(log_n as u64) + .expect("the LDE domain must fit Goldilocks' two-adicity") +} + +/// One radix-2 butterfly: `(u + w·x, u − w·x)`. +/// +/// `w` is a program constant, so the subtracting half is `mul_add` against the +/// interned constant `−w` rather than a separate negation — two `LFM_BALU` rows +/// per butterfly, not three. At `w = 1` (every level-1 butterfly, and the first +/// of every later block) there is no multiplication at all. +fn butterfly(b: &mut LfmBuilder, u: Felt, x: Felt, w: FE) -> (Felt, Felt) { + if w == FE::one() { + (b.add(u, x), b.sub(u, x)) + } else { + let pos = b.felt_const(w); + let neg = b.felt_const(-w); + (b.mul_add(pos, x, u), b.mul_add(neg, x, u)) + } +} + +/// Decimation-in-time radix-2 transform: `out[m] = Σᵢ input[i]·root^(i·m)`. +/// +/// `input` is in natural order (the bit-reverse the algorithm needs is a +/// host-side index permutation and costs nothing). Pass `root = ω_n` for the +/// forward direction and `root = ω_n⁻¹` for the inverse — the inverse's `1/n` +/// is NOT applied here, so callers that follow it with a scaling pass fold the +/// factor into their own constants. +fn dit(b: &mut LfmBuilder, input: &[Felt], root: FE) -> Vec { + let n = input.len(); + let log_n = n.trailing_zeros(); + let mut a: Vec = (0..n).map(|i| input[reverse_index(i, n as u64)]).collect(); + for s in 1..=log_n { + let m = 1usize << s; + let step = root.pow(n / m); + for k in (0..n).step_by(m) { + let mut w = FE::one(); + for j in 0..m / 2 { + let (hi, lo) = butterfly(b, a[k + j], a[k + j + m / 2], w); + a[k + j] = hi; + a[k + j + m / 2] = lo; + w *= step; + } + } + } + a +} + +/// Emit the low-degree extension of a column given by its `n` evaluations on +/// the size-`n` subgroup, onto `coset_offset · ⟨ω_{n·blowup}⟩`. +/// +/// The result is in NATURAL domain order — output `j` is the value at +/// `coset_offset · ω_{n·blowup}^j` — which is the layout +/// `stark::commitment::commit_bit_reversed` consumes (it applies the +/// bit-reversal itself). +/// +/// ## The decomposition +/// +/// With `c = iFFT(values)` the polynomial's coefficients and +/// `s_k = coset_offset · ω_{n·blowup}^k`, domain index `j = k + blowup·m` +/// carries the point `s_k · ω_n^m`, so the `k`-th coset is one size-`n` forward +/// transform of `c` scaled by `s_k^i`. The interpolation's `1/n` rides along in +/// those scaling constants, which is why the inverse pass emits no scaling of +/// its own. +pub fn coset_lde( + b: &mut LfmBuilder, + values: &[Felt], + blowup: usize, + coset_offset: FE, +) -> Vec { + let n = values.len(); + assert!( + n.is_power_of_two(), + "the interpolation domain is a subgroup" + ); + assert!( + blowup.is_power_of_two() && blowup > 0, + "blowup is a power of two" + ); + + let log_n = n.trailing_zeros(); + let omega_n = root_of_unity(log_n); + let omega_big = root_of_unity(log_n + blowup.trailing_zeros()); + + // `n · coefficients`: the 1/n is folded into the per-coset scaling below. + let scaled_coeffs = dit( + b, + values, + omega_n.inv().expect("a root of unity is invertible"), + ); + + let n_inv = FE::from(n as u64) + .inv() + .expect("the domain size is nonzero in Goldilocks"); + + let mut out: Vec> = vec![None; n * blowup]; + for k in 0..blowup { + let s = coset_offset * omega_big.pow(k); + let mut weight = n_inv; + let coset_coeffs: Vec = (0..n) + .map(|i| { + let c = b.felt_const(weight); + let scaled = b.mul(scaled_coeffs[i], c); + weight *= s; + scaled + }) + .collect(); + for (m, value) in dit(b, &coset_coeffs, omega_n).into_iter().enumerate() { + out[k + blowup * m] = Some(value); + } + } + out.into_iter() + .map(|v| v.expect("every domain index is written exactly once")) + .collect() +} diff --git a/prover/src/lfm/leaf_kats.rs b/prover/src/lfm/leaf_kats.rs new file mode 100644 index 000000000..165aa89e0 --- /dev/null +++ b/prover/src/lfm/leaf_kats.rs @@ -0,0 +1,211 @@ +//! LEAF-mode KATs for the LFM `"LFML"` domain, at 6 and 7 rounds. +//! +//! GENERATED — do not hand-edit. Rendered by +//! `thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py` from +//! `gate-oracle/blake3_oracle.py`, a Python BLAKE3 written **before any Rust +//! existed**. These vectors are a specification the implementation is checked +//! against, not a recording of what the implementation happened to do. +//! +//! A leaf row hashes FOUR arbitrary Goldilocks elements AND chains an +//! accumulator, in ONE compression (COMMIT.md §1.2). The accumulator is a digest +//! cell and fills lanes 0–3; each felt occupies two lanes above it as checked +//! `u32` halves, `[lo0, hi0, …, lo3, hi3]`. So the message is +//! `LE32(acc ‖ halves) ‖ "LFML"` — 52 bytes, still one BLAKE3 block, so the +//! crate-KAT anchor survives the widening. + +/// One leaf row: the chaining accumulator, four felts, the twelve lanes they +/// become, and the digest at each round count. +pub struct LeafVector { + pub name: &'static str, + pub acc: [u32; 4], + pub felts: [u64; 4], + pub lanes: [u32; 12], + /// Digest at 6 rounds (the A6R variant; no library computes it). + pub digest_6: [u32; 4], + /// Digest at 7 rounds — `blake3::hash(LE32(lanes) ‖ "LFML")[..16]`. + pub digest_7: [u32; 4], +} + +pub const LEAF_VECTORS: [LeafVector; 6] = [ + LeafVector { + name: "zeros", + acc: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + felts: [0u64, 0u64, 0u64, 0u64], + lanes: [ + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + ], + digest_6: [0x9D79DC29, 0xFC6E166E, 0x30387614, 0xF6B51296], + digest_7: [0xB30DB92A, 0xC648E66E, 0x85368146, 0x30A98B38], + }, + LeafVector { + name: "boundary_mix", + acc: [0x00000000, 0x00000001, 0xFFFFFFFE, 0xFFFFFFFF], + felts: [0u64, 1u64, 18446744069414584320u64, 4294967296u64], + lanes: [ + 0x00000000, 0x00000001, 0xFFFFFFFE, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000001, + 0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000001, + ], + digest_6: [0xA101443C, 0xA70F5A93, 0xAD973E8C, 0x17C8F7BA], + digest_7: [0x0E214E2C, 0x5D16CE5C, 0xA4DE74CF, 0x9FA39D59], + }, + LeafVector { + name: "all_p_minus_1", + acc: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + felts: [ + 18446744069414584320u64, + 18446744069414584320u64, + 18446744069414584320u64, + 18446744069414584320u64, + ], + lanes: [ + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, + 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, + ], + digest_6: [0x5FDFAEF4, 0x0F63DEEE, 0xCC4EC296, 0x675289C3], + digest_7: [0xAE3CB971, 0x6F0EDEE7, 0x75BBD078, 0xC07D12D6], + }, + LeafVector { + name: "ramp", + acc: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + felts: [ + 72623859790382856u64, + 1230066625199609624u64, + 2387509390608836392u64, + 3544952156018063160u64, + ], + lanes: [ + 0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10, 0x05060708, 0x01020304, 0x15161718, + 0x11121314, 0x25262728, 0x21222324, 0x35363738, 0x31323334, + ], + digest_6: [0x7E8EC742, 0x478136B7, 0xDC4010C2, 0xA7B85A1F], + digest_7: [0x6F6562C3, 0x6755528E, 0xBD65A6F0, 0xA9B1551D], + }, + LeafVector { + name: "u32_edges", + acc: [0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF], + felts: [4294967295u64, 4294967296u64, 18446744065119617025u64, 1u64], + lanes: [ + 0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF, 0xFFFFFFFF, 0x00000000, 0x00000000, + 0x00000001, 0x00000001, 0xFFFFFFFE, 0x00000001, 0x00000000, + ], + digest_6: [0x0E17BDDF, 0x1E0CA3B6, 0x7F8B414F, 0xDDF551B2], + digest_7: [0x77E4CDFD, 0x92CC8E05, 0x1BBC4BD0, 0x64B4D8D2], + }, + LeafVector { + name: "acc_ignored_control", + acc: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + felts: [0u64, 0u64, 0u64, 0u64], + lanes: [ + 0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + ], + digest_6: [0xE4AE2501, 0x1FCF9DAB, 0x85643F4E, 0xE24B3793], + digest_7: [0xB8094093, 0xA7EBC1A4, 0xB7955183, 0x0BA8929B], + }, +]; + +/// A boundary felt and the halves it must decompose into. +pub struct BoundaryFelt { + pub name: &'static str, + pub felt: u64, + pub lo: u32, + pub hi: u32, +} + +/// The six canonical boundary cases, `p − 1` included — the tight one. +pub const BOUNDARY_FELTS: [BoundaryFelt; 6] = [ + BoundaryFelt { + name: "zero", + felt: 0u64, + lo: 0x00000000, + hi: 0x00000000, + }, + BoundaryFelt { + name: "one", + felt: 1u64, + lo: 0x00000001, + hi: 0x00000000, + }, + BoundaryFelt { + name: "u32_max", + felt: 4294967295u64, + lo: 0xFFFFFFFF, + hi: 0x00000000, + }, + BoundaryFelt { + name: "two_pow_32", + felt: 4294967296u64, + lo: 0x00000000, + hi: 0x00000001, + }, + BoundaryFelt { + name: "p_minus_2_32", + felt: 18446744065119617025u64, + lo: 0x00000001, + hi: 0xFFFFFFFE, + }, + BoundaryFelt { + name: "p_minus_1", + felt: 18446744069414584320u64, + lo: 0x00000000, + hi: 0xFFFFFFFF, + }, +]; + +/// Values the leaf mode must REJECT rather than reduce. Each has `hi` maximal +/// and `lo >= 1`, so each aliases a canonical felt — which is exactly the +/// collision the canonicity block exists to prevent. +pub struct NonCanonical { + pub name: &'static str, + pub value: u64, + pub lo: u32, + pub hi: u32, +} + +pub const NON_CANONICAL: [NonCanonical; 3] = [ + NonCanonical { + name: "p", + value: 18446744069414584321u64, + lo: 0x00000001, + hi: 0xFFFFFFFF, + }, + NonCanonical { + name: "p_plus_1", + value: 18446744069414584322u64, + lo: 0x00000002, + hi: 0xFFFFFFFF, + }, + NonCanonical { + name: "two_pow_64_minus_1", + value: 18446744073709551615u64, + lo: 0xFFFFFFFF, + hi: 0xFFFFFFFF, + }, +]; + +/// The eight-felt `FriToyV0` leaf: ONE `LFML` chain, two rows, no fold. +pub struct FriLeafVector { + pub felts: [u64; 8], + pub digest_6: [u32; 4], + pub digest_7: [u32; 4], + /// Compressions the whole leaf costs — 3 before the accumulator moved into + /// the message, 2 after (COMMIT.md §1.4.1: the RATE, measured). + pub compresses: usize, +} + +pub const FRI_LEAF: FriLeafVector = FriLeafVector { + felts: [ + 18446744069414584320u64, + 0u64, + 1u64, + 4294967296u64, + 12345678901234567u64, + 4294967295u64, + 18446744065119617025u64, + 999u64, + ], + digest_6: [0x8578A6BC, 0x9160F074, 0x3F4C82B9, 0x98C5C775], + digest_7: [0x9C36DE23, 0xCD397230, 0x2013BF3D, 0xD72A0346], + compresses: 2, +}; diff --git a/prover/src/lfm/leaf_tests.rs b/prover/src/lfm/leaf_tests.rs new file mode 100644 index 000000000..69e98a348 --- /dev/null +++ b/prover/src/lfm/leaf_tests.rs @@ -0,0 +1,887 @@ +//! The `"LFML"` leaf mode (option C): its vectors, its canonicity gate, and the +//! milestone it unblocks — `FriToyV0` proving under BLAKE3. +//! +//! ## What pins what +//! +//! 1. **The felt boundary** — halves and canonicity — against +//! [`super::leaf_kats`]'s boundary table, `p − 1` and the non-canonical +//! aliases included. +//! 2. **The step function** against the `blake3` crate: at 7 rounds a leaf is +//! `blake3::hash(LE32(lanes) ‖ "LFML")` truncated, so the leaf domain +//! inherits the socket's external anchor rather than claiming a new one. +//! 3. **The chip**: M9 (mode confusion, six ordered pairs) and M10 (a `MODE_L` +//! row that skips canonicity), which are what make "`MODE_L` implies +//! felt-input semantics" a constraint rather than a convention. +//! 4. **The assembled program**: `FriToyV0` proves and verifies under BLAKE3 — +//! the F3.4 milestone — with a NEGATIVE leg showing the canonicity gate does +//! work in the real proof and not only in a unit test. +//! +//! Every rejection test is paired with an honest-path assertion. + +use math::field::traits::IsPrimeField; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::blake3_socket::{ + self, FELTS_PER_LEAF, SOCKET_ROUNDS, TAG_LFMC, TAG_LFML, TAG_LFMT, cols, felt_halves, + is_canonical, leaf_digest_rounds, leaf_lanes, word_of, +}; +use super::hash::HasherKind; +use super::instr::{HashMode, Instr}; +use super::leaf_kats::{BOUNDARY_FELTS, FRI_LEAF, LEAF_VECTORS, NON_CANONICAL}; +use super::proof::{lfm_prove_with_hasher, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::word::LfmWord; + +const KIND: HasherKind = HasherKind::Blake3; + +/// Goldilocks `p`. +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +fn felts_of(vals: &[u64; 4]) -> LfmWord { + core::array::from_fn(|i| FE::from(vals[i])) +} + +// ========================================================================= +// L2/L3/L4 — the felt boundary +// ========================================================================= + +/// Every boundary felt splits into the halves the spec pins, `p − 1` included — +/// the tight case, where `hi` is maximal and `lo` is exactly zero. +#[test] +fn every_boundary_felt_round_trips_through_its_halves() { + for v in BOUNDARY_FELTS.iter() { + let (lo, hi) = felt_halves(v.felt).unwrap_or_else(|| panic!("{} is canonical", v.name)); + assert_eq!((lo, hi), (v.lo, v.hi), "halves of {}", v.name); + assert!(is_canonical(lo, hi), "{} must pass the predicate", v.name); + assert_eq!( + u64::from(lo) + (u64::from(hi) << 32), + v.felt, + "halves must recompose {}", + v.name + ); + } +} + +/// ★ Non-canonical values are REJECTED, never reduced — and the test says why +/// it matters: each one ALIASES a canonical felt, so reducing would give one +/// field element two leaf digests. +#[test] +fn non_canonical_values_are_rejected_not_reduced() { + for v in NON_CANONICAL.iter() { + assert!(!is_canonical(v.lo, v.hi), "{} is non-canonical", v.name); + assert_eq!(felt_halves(v.value), None, "{} must be refused", v.name); + + // The alias, spelled out: hi maximal makes 2^32·hi = p − 1 ≡ −1, so the + // pair encodes `lo − 1`, which has its own ordinary encoding. THAT is + // the collision the canonicity block prevents. + let aliased = (u128::from(v.lo) + (u128::from(v.hi) << 32)) % u128::from(P); + assert_eq!(aliased, u128::from(v.lo) - 1); + let (clo, chi) = felt_halves(aliased as u64).expect("the alias target is canonical"); + assert_ne!( + (clo, chi), + (v.lo, v.hi), + "{}: two half-pairs for one felt is exactly the hazard", + v.name + ); + } +} + +/// L4 — the predicate IS `v < p`, over every boundary and a dense sweep. +/// +/// The spec ran 300,007 cases; this runs the same boundaries plus a sweep near +/// the wrap, which is where a predicate that is merely *nearly* right fails. +#[test] +fn the_canonicity_predicate_is_exactly_less_than_p() { + let check = |v: u64| { + let (lo, hi) = (v as u32, (v >> 32) as u32); + assert_eq!( + is_canonical(lo, hi), + v < P, + "predicate disagrees with v < p at {v:#x}" + ); + }; + for v in [ + 0u64, + 1, + u64::from(u32::MAX), + 1 << 32, + P - 2, + P - 1, + P, + P + 1, + u64::MAX, + ] { + check(v); + } + // The whole neighbourhood of the wrap, both sides. + for d in 0..2_000u64 { + check(P.wrapping_sub(d)); + check(P.wrapping_add(d)); + } + // A stride across the space, so the sweep is not only local. + for k in 0..20_000u64 { + check(k.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + } +} + +// ========================================================================= +// L1 — the leaf digest, and its crate anchor +// ========================================================================= + +/// Every leaf vector reproduces at BOTH round counts, lanes included. +#[test] +fn every_leaf_vector_reproduces_at_both_round_counts() { + for v in LEAF_VECTORS.iter() { + let (acc, felts) = (word_of(&v.acc), felts_of(&v.felts)); + assert_eq!( + blake3_socket::leaf_row_lanes(&acc, &felts).expect("canonical"), + v.lanes, + "lanes of {}", + v.name + ); + assert_eq!( + leaf_lanes(&felts).expect("canonical"), + v.lanes[4..], + "the halves of {} sit ABOVE the accumulator", + v.name + ); + assert_eq!( + leaf_digest_rounds(&acc, &felts, 6).expect("canonical"), + v.digest_6, + "6-round leaf {}", + v.name + ); + assert_eq!( + leaf_digest_rounds(&acc, &felts, 7).expect("canonical"), + v.digest_7, + "7-round leaf {}", + v.name + ); + } +} + +/// ★ The accumulator REACHES the digest — the property the whole RATE rests on. +/// +/// A row that absorbed its felts and dropped the chaining value would still +/// satisfy every canonicity and halves constraint, and would still be a +/// perfectly good hash of the felts; what it would not be is a CHAIN, and a wide +/// leaf built out of it would bind only its last four felts. The table carries +/// `acc_ignored_control` for exactly this: same felts as `zeros`, different +/// accumulator. +#[test] +fn the_accumulator_changes_the_leaf_digest() { + let find = |name: &str| { + LEAF_VECTORS + .iter() + .find(|v| v.name == name) + .expect("vector present") + }; + let (zeros, control) = (find("zeros"), find("acc_ignored_control")); + assert_eq!( + zeros.felts, control.felts, + "the control varies ONLY the acc" + ); + assert_ne!(zeros.acc, control.acc); + assert_ne!(zeros.digest_6, control.digest_6); + assert_ne!(zeros.digest_7, control.digest_7); +} + +/// ★ **The external anchor, direct.** At 7 rounds a leaf is literally +/// `blake3::hash(LE32(acc)‖LE32(lo0)‖LE32(hi0)‖…‖LE32(hi3)‖"LFML")` truncated to +/// 16 bytes. +/// +/// The message is rebuilt from the byte-level specification rather than from +/// `socket_message`, so the word-level and byte-level forms can disagree. This +/// is the property option C was chosen to preserve and the RATE had to keep: +/// putting the felt encoding INSIDE the socket keeps the message layout +/// byte-identical to a digest-mode compress, and 52 bytes is still ONE block, so +/// the crate stays a direct KAT for the leaf domain. Carrying the accumulator in +/// the chaining value `h` instead would have made the row a chunk continuation +/// and thrown this away for the same rate. +#[test] +fn seven_rounds_is_blake3_of_the_leaf_message() { + for v in LEAF_VECTORS.iter() { + let mut msg = Vec::with_capacity(52); + for lane in v.lanes.iter() { + msg.extend_from_slice(&lane.to_le_bytes()); + } + msg.extend_from_slice(b"LFML"); + assert_eq!(msg.len(), 52, "a leaf row is one 52-byte block"); + assert!(msg.len() < 64, "and one block is what the anchor needs"); + + let full = blake3::hash(&msg); + let want: [u32; 4] = core::array::from_fn(|i| { + u32::from_le_bytes(full.as_bytes()[4 * i..4 * i + 4].try_into().unwrap()) + }); + assert_eq!(want, v.digest_7, "leaf {} must be blake3::hash", v.name); + } +} + +/// The tag word is the ASCII, little-endian, and the three live domains are +/// pairwise distinct as VALUES — the cheap check that a typo cannot pass. +#[test] +fn the_leaf_tag_is_lfml_and_the_three_domains_are_distinct() { + assert_eq!(TAG_LFML, u32::from_le_bytes(*b"LFML")); + assert_eq!(TAG_LFML.to_le_bytes(), *b"LFML"); + assert_ne!(TAG_LFML, TAG_LFMC); + assert_ne!(TAG_LFML, TAG_LFMT); + assert_ne!(TAG_LFMC, TAG_LFMT); +} + +/// L5 — the three domains produce three different digests from the SAME twelve +/// lanes. Distinct tag values are necessary; distinct digests are the property. +#[test] +fn the_three_domains_differ_on_the_same_lanes() { + for v in LEAF_VECTORS.iter() { + for rounds in [6, 7] { + let d: Vec<[u32; 4]> = [TAG_LFMC, TAG_LFMT, TAG_LFML] + .iter() + .map(|t| blake3_socket::socket_digest_lanes(&v.lanes, rounds, *t)) + .collect(); + assert_ne!(d[0], d[1], "{} @{rounds}: LFMC == LFMT", v.name); + assert_ne!(d[0], d[2], "{} @{rounds}: LFMC == LFML", v.name); + assert_ne!(d[1], d[2], "{} @{rounds}: LFMT == LFML", v.name); + } + } +} + +/// L6 — an eight-felt leaf is exactly TWO compressions: one `LFML` chain of two +/// rows, absorbing four felts and chaining in each. +/// +/// ★ **This is the RATE, measured rather than asserted.** It was three — two +/// felts-only leaf rows and an `LFMC` parent folding them — which is 2 felts per +/// compression. Moving the accumulator into the message makes the fold +/// unnecessary and takes it to 4, and leaf absorption is ~70% of a recursion +/// tower node's bill (COMMIT.md §1.4.1). The count is pinned in the vector table +/// so the saving cannot quietly regress. +#[test] +fn an_eight_felt_leaf_is_one_chain_of_two_rows() { + let lo: LfmWord = core::array::from_fn(|i| FE::from(FRI_LEAF.felts[i])); + let hi: LfmWord = core::array::from_fn(|i| FE::from(FRI_LEAF.felts[4 + i])); + let start = super::fixture::leaf_chain_start(); + for (rounds, want) in [(6, FRI_LEAF.digest_6), (7, FRI_LEAF.digest_7)] { + let d0 = word_of(&leaf_digest_rounds(&start, &lo, rounds).expect("canonical")); + let chained = leaf_digest_rounds(&d0, &hi, rounds).expect("canonical"); + assert_eq!(chained, want, "the 8-felt leaf at {rounds} rounds"); + } + assert_eq!(FRI_LEAF.compresses, 2, "8 felts / RATE 4 = 2 compressions"); + + // The HOST path agrees with the reference — `host_leaf_hash_pair` is what + // the fixture builds its trees with, so a divergence here is a fixture that + // the machine cannot authenticate. + let host = super::fixture::host_leaf_hash_pair(KIND, &lo, &hi); + let want = if SOCKET_ROUNDS == 7 { + FRI_LEAF.digest_7 + } else { + FRI_LEAF.digest_6 + }; + assert_eq!(host, word_of(&want)); +} + +// ========================================================================= +// H6 — the lane-identity gate, both directions +// ========================================================================= +// +// The gate is per LANE RANGE, not per mode, and COMMIT.md §1.4.2 asks for a +// control in the WA1/WA2 style because one direction is a soundness break: gate +// lanes 0–3 on `digest_mu` and a leaf row's accumulator carries no identity at +// all, so the prover picks the chain's message words freely and the whole leaf +// chain unbinds. Two tests, one per direction, because a gate that is wrong +// EITHER way passes the other test. + +/// ★★ **H6, the soundness direction: a leaf row's ACCUMULATOR lanes are +/// constrained.** +/// +/// Shaped like WA9 — not "the tampered row is rejected", which would pass for a +/// set that rejected it incidentally, but "the violated set IS the accumulator +/// lane's own identity". That carries both legs: with the identity the row is +/// rejected, and without it every other constraint still evaluates to zero on +/// this row, so it would be accepted. If the gate were `digest_mu` here, this +/// row would satisfy everything. +#[test] +fn h6_a_leaf_rows_accumulator_lanes_carry_the_identity() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let base = leaf_row(&acc, &felts); + assert_eq!( + super::blake3_socket_tests::violations(&base), + Vec::::new(), + "HONEST CONTROL: a leaf row must satisfy every constraint" + ); + + for lane in 0..cols::NUM_ACC_LANES { + // Move the accumulator FELT and leave its byte columns honest. The + // message the row hashes is the bytes, so this is the forgery that + // matters: a prover claiming one accumulator in `IN` while the mixing + // core consumes another. Only the identity ties the two together. + let mut forged = base.clone(); + forged[cols::IN0 + lane] += FE::one(); + assert_eq!( + super::blake3_socket_tests::violations(&forged), + vec![blake3_socket::LANE_IDX + lane], + "lane {lane} of the accumulator must be pinned to its bytes, and by \ + ITS identity — anything else and the dropped-leg claim fails" + ); + } +} + +/// ★ **H6, the other direction: the identity must NOT hold on a leaf row's FELT +/// lanes.** +/// +/// Lanes 4–11 are the felts' `lo`/`hi` halves, so `IN` and the message word are +/// deliberately different field elements there; the halves binding is what +/// relates them. Gating those on the full `mu` would make every leaf row +/// unprovable — the failure the original eight-lane comment warned about, which +/// survives the widening in exactly this narrowed form. +/// +/// Asserted as arithmetic on an honest row rather than by building a broken +/// chip: if `IN(felt i) == lo_i` for every felt, the claim is vacuous and this +/// test says so. +#[test] +fn h6_the_felt_lanes_do_not_satisfy_the_lane_identity() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let row = leaf_row(&acc, &felts); + + let mut discriminated = 0; + for i in 0..FELTS_PER_LEAF { + let lo_lane = cols::leaf_lo_lane(i); + let bytes: u64 = (0..4) + .map(|b| { + GoldilocksField::canonical(row[cols::lane_byte(lo_lane, b)].value()) << (8 * b) + }) + .sum(); + let felt = GoldilocksField::canonical(row[cols::leaf_felt(i)].value()); + // The felt is `lo + 2^32·hi`, so it equals its low half only when the + // high half is zero. The vector is chosen so that is not always true. + if felt != bytes { + discriminated += 1; + } + } + assert!( + discriminated > 0, + "the vector must contain a felt wider than 32 bits, or this test is \ + vacuous and the gate could be `mu` without anyone noticing" + ); +} + +// ========================================================================= +// M9 / M10 — the chip-level controls the leaf spec pre-committed +// ========================================================================= + +/// A hash row in `mode` over `felts`/`lanes`, exactly as the trace filler builds +/// one — for the controls, which need to force a mismatch the filler cannot. +fn leaf_row(acc: &LfmWord, felts: &LfmWord) -> Vec { + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_L] = FE::one(); + row[cols::IN0..cols::IN0 + 4].copy_from_slice(acc); + row[cols::leaf_felt(0)..cols::leaf_felt(0) + FELTS_PER_LEAF].copy_from_slice(felts); + for (k, iv) in super::blake3::BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = leaf_digest_rounds(acc, felts, SOCKET_ROUNDS).expect("canonical"); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness(&mut row); + row +} + +/// **M9 — mode confusion, all six ordered pairs.** A row in one domain whose +/// witness computes another domain's digest must be rejected. +/// +/// Three tags means six ordered confusions, and the leaf domain adds four of +/// them. `L5` above shows the three functions differ; this shows the CHIP +/// notices, which is a different claim. +#[test] +fn m9_no_domain_can_compute_another_domains_digest() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let halves = leaf_lanes(&felts).expect("canonical"); + // Two digest cells for the two-to-one modes. A leaf row's lanes are its own + // (accumulator ‖ halves) and cannot be shared with a digest row's: lanes + // 8–11 read the third input cell, which the unread-`IN` pins hold at zero on + // every digest row. So each mode is built with the lanes it actually has, + // and the digest is taken over THOSE — the confusion under test is the + // TAG's, not the lanes'. + let (a, b) = ( + blake3_socket::lanes_of(&acc).expect("a digest cell is u32 lanes"), + [halves[0], halves[1], halves[2], halves[3]], + ); + + for (mode, own) in [ + (HashMode::Compress, TAG_LFMC), + (HashMode::Transcript, TAG_LFMT), + (HashMode::Leaf, TAG_LFML), + ] { + for other in [TAG_LFMC, TAG_LFMT, TAG_LFML] { + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[super::blake3_socket_tests::mode_col(mode)] = FE::one(); + let lanes = if mode == HashMode::Leaf { + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&acc); + row[cols::leaf_felt(0)..cols::leaf_felt(0) + FELTS_PER_LEAF] + .copy_from_slice(&felts); + blake3_socket::leaf_row_lanes(&acc, &felts).expect("canonical") + } else { + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&word_of(&a)); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(&word_of(&b)); + blake3_socket::digest_row_lanes(&a, &b) + }; + for (k, iv) in super::blake3::BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = blake3_socket::socket_digest_lanes(&lanes, SOCKET_ROUNDS, other); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, other); + + let violated = super::blake3_socket_tests::violations(&row); + if other == own { + assert_eq!( + violated, + Vec::::new(), + "HONEST CONTROL: {mode:?} in its own domain must be accepted" + ); + } else { + assert!( + !violated.is_empty(), + "{mode:?} computing the {other:#010x} domain must be rejected" + ); + } + } + } +} + +/// **M10 — `MODE_L` implies felt-input semantics, as a CONSTRAINT.** +/// +/// A leaf row that skips the canonicity block must be rejected. Two ways to +/// skip it, and both are tried: zero the witnesses, and install the +/// non-canonical alias with a witness that would satisfy every constraint the +/// canonicity block does not impose. +#[test] +fn m10_a_leaf_row_cannot_skip_canonicity() { + let acc = word_of(&LEAF_VECTORS[1].acc); + let felts = felts_of(&LEAF_VECTORS[1].felts); + let base = leaf_row(&acc, &felts); + assert_eq!( + super::blake3_socket_tests::violations(&base), + Vec::::new(), + "HONEST CONTROL: a canonical leaf row satisfies every constraint" + ); + + // (a) blank the canonicity witnesses. `canon_b` pins `Z` from `G`, so a + // zeroed `Z` is only satisfiable when `G` is invertible AND `GINV` matches; + // blanking both breaks it. + let mut blanked = base.clone(); + for i in 0..FELTS_PER_LEAF { + blanked[cols::canon_z(i)] = FE::zero(); + blanked[cols::canon_ginv(i)] = FE::zero(); + } + assert!( + !super::blake3_socket_tests::violations(&blanked).is_empty(), + "a leaf row with no canonicity witness must be rejected" + ); + + // (b) ★ THE ALIAS. Re-encode felt 0 as (lo + 1, hi = 2^32 − 1), which is the + // SAME field element — the binding constraint is satisfied — and let the + // witness be otherwise consistent. Only canonicity can catch this, and the + // test asserts it is `canon-c` that does. + let target = felts_of(&[0, 0, 0, 0]); + let mut alias = leaf_row(&acc, &target); + // felt 0's lo half becomes 1 and its hi half 2^32 − 1. ⚠ Located through + // `leaf_lo_lane`/`leaf_hi_lane` rather than as lanes 0 and 1: the felts sit + // ABOVE the accumulator now, and a literal 0/1 here would silently corrupt + // the accumulator instead and test nothing (COMMIT.md §1.4.4 H4). + for (lane, v) in [ + (cols::leaf_lo_lane(0), 1u32), + (cols::leaf_hi_lane(0), u32::MAX), + ] { + for byte in 0..4 { + alias[cols::lane_byte(lane, byte)] = FE::from(u64::from((v >> (8 * byte)) as u8)); + } + } + // The witness the alias would need: hi is maximal, so G = 0 and Z = 1. + alias[cols::canon_z(0)] = FE::one(); + alias[cols::canon_ginv(0)] = FE::zero(); + + let violated = super::blake3_socket_tests::violations(&alias); + // canon-c for felt 0 — located from the arm's own indices rather than by a + // literal, so growing the framing cannot silently point this at another + // constraint. + const CANON_C_FELT0: usize = + blake3_socket::LEAF_IDX + blake3_socket::LEAF_CONSTRAINTS_PER_FELT - 1; + assert!( + violated.contains(&CANON_C_FELT0), + "the alias must be caught by canon-c (idx {CANON_C_FELT0}), got {violated:?}" + ); + + // And the alias really is the same field element, so nothing ELSE could + // have caught it — that is what makes canonicity load-bearing rather than + // redundant with the binding constraint. + assert_eq!( + (u128::from(1u32) + (u128::from(u32::MAX) << 32)) % u128::from(P), + 0, + "the alias encodes felt 0" + ); +} + +// ========================================================================= +// ★ The F3.4 milestone: FriToyV0 under BLAKE3 +// ========================================================================= + +fn fri_arenas(inner: &super::fixture::FriToyProof) -> Vec> { + vec![inner.commitments.clone(), inner.openings.clone()] +} + +/// ★★ **`FriToyV0` proves and verifies under BLAKE3.** The milestone the whole +/// campaign was for: a real verification program, over real FRI data — LDE +/// evaluations and folded extension elements, none of them `u32` — proved under +/// the machine's real hash. +/// +/// This replaces `blake3_socket_tests::fri_toy_is_still_blocked_by_o1…`, whose +/// own doc required a prove+verify rather than an execute when O1 closed. +#[test] +fn fri_toy_proves_and_verifies_under_blake3() { + let opts = options(); + let program = super::programs::fri_toy_program(); + let inner = super::fixture::fixture_prove_with_hasher(KIND); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&inner), &opts, KIND) + .expect("FriToyV0 must prove under BLAKE3"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "an honest BLAKE3 proof of FriToyV0 must verify" + ); + // The attested output is the inner proof's identity: both roots. + assert_eq!(proved.public_words[0].1, inner.commitments[0]); + assert_eq!(proved.public_words[1].1, inner.commitments[1]); +} + +/// The same program under the other hashers — B1 and option C both changed +/// shared constructions, so all of them must stay green. +#[test] +fn fri_toy_proves_and_verifies_under_every_hasher() { + let opts = options(); + let program = super::programs::fri_toy_program(); + for kind in [HasherKind::Test, HasherKind::Rpx, HasherKind::Blake3] { + let inner = super::fixture::fixture_prove_with_hasher(kind); + let artifacts = build_artifacts_with_hasher(&program, &opts, kind); + let proved = lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&inner), &opts, kind) + .unwrap_or_else(|e| panic!("prove under {kind:?}: {e:?}")); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "an honest proof of FriToyV0 must verify under {kind:?}" + ); + } +} + +/// ⚠ **The NEGATIVE leg — the criterion most likely to be skipped.** +/// +/// A fixture whose tree was built under a DIFFERENT hasher must not +/// authenticate. This is what shows the leaf digests are load-bearing in the +/// assembled program: every opened row is authenticated by re-deriving its leaf, +/// so a leaf computed by another hash breaks the walk. +#[test] +fn fri_toy_rejects_a_fixture_built_under_another_hasher() { + let opts = options(); + let program = super::programs::fri_toy_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let mismatched = super::fixture::fixture_prove_with_hasher(HasherKind::Test); + assert!( + lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&mismatched), &opts, KIND).is_err(), + "a Test-hashed fixture must not authenticate under BLAKE3" + ); + + // HONEST CONTROL: the matching fixture does prove, so the rejection is about + // the hasher and not about the program. + let matching = super::fixture::fixture_prove_with_hasher(KIND); + assert!( + lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&matching), &opts, KIND).is_ok() + ); +} + +/// The leaf mode is what closed O1 for this program: the fixture's committed +/// values are still not `u32`-laned, and it proves anyway. +/// +/// The old tripwire asserted the opposite conclusion from the same premise. It +/// is kept as a positive statement because the premise is what makes the +/// milestone meaningful — proving over `u32`-shaped data would have proved +/// nothing about the leaf mode. +#[test] +fn the_fixture_data_is_still_not_u32_and_that_is_the_point() { + let over = super::fixture::fixture_columns() + .iter() + .flatten() + .filter(|v| GoldilocksField::canonical(v.value()) >= 1u64 << 32) + .count(); + assert!( + over > 0, + "if the fixture became u32-laned the milestone would be vacuous" + ); + + // Every leaf row in the emitted program is a `Leaf`, and every Merkle-walk + // step is a `Compress` — the split the O5 retirement rests on. + let program = super::programs::fri_toy_program(); + let leaves = program + .instrs + .iter() + .filter(|i| matches!(i, Instr::Hash { mode, .. } if *mode == HashMode::Leaf)) + .count(); + assert_eq!( + leaves, 26, + "4 queries × 3 data leaves × 2 LFML rows, plus the two terminal \ + coefficients the transcript absorbs as data" + ); +} + +// ========================================================================= +// D1 — the unread input cells, on EVERY arm +// ========================================================================= + +/// Evaluates a hash row against `kind`'s constraint set and returns the +/// violated indices. +fn violations_under(kind: HasherKind, row: &[FE]) -> Vec { + use math::field::element::FieldElement; + use stark::constraints::builder::{ConstraintSet, ProverEvalFolder}; + use stark::frame::Frame; + use stark::table::TableView; + use stark::traits::TransitionEvaluationContext; + + use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + + let set = super::chips::hash::HashConstraints { kind }; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new( + vec![row.to_vec()], + vec![vec![]], + )]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +/// A `MODE_L` row for `kind` whose THIRD input cell carries `extra`. +/// +/// Everything downstream is derived from the cells the mode reads, by each arm's +/// own rule, so the row is internally consistent whatever `extra` is: `extra = 0` +/// is the honest row a trace filler would write, and any other `extra` is the +/// forgery a prover controlling the whole trace would actually submit. Building +/// both the same way is what makes "only the pins fire" a meaningful assertion — +/// a half-built forgery would trip the round constraints instead and prove +/// nothing about the pins. +fn leaf_row_with_third_cell( + kind: HasherKind, + acc: &LfmWord, + felts: &LfmWord, + extra: &LfmWord, +) -> Vec { + use super::hash::{HASH_STATE_FELTS, LfmHasher}; + + let mut row = vec![FE::zero(); super::chips::hash::num_columns(kind)]; + row[cols::MODE_L] = FE::one(); + row[cols::IN0..cols::IN0 + 4].copy_from_slice(acc); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(felts); + row[cols::IN0 + 8..cols::IN0 + 12].copy_from_slice(extra); + // A LEAF row's capacity, which is not always the compress one: RPO + // domain-separates its modes through the capacity. + let iv = kind.leaf_iv(); + row[cols::S8..cols::S8 + iv.len()].copy_from_slice(&iv); + + match kind { + // BLAKE3 reads an accumulator and four felts and nothing else, so its + // output does not depend on the third cell at all — which is exactly why + // only a pin can notice junk there. + HasherKind::Blake3 => { + let out = kind.leaf_out(acc, felts); + row[cols::OUT0..cols::OUT0 + out.len()].copy_from_slice(&out); + blake3_socket::fill_socket_witness(&mut row); + } + // The field-native arms permute the state — which is the two cells the + // mode reads and the IV, never the third cell. + HasherKind::Test => { + let mut state = [FE::zero(); HASH_STATE_FELTS]; + state[0..4].copy_from_slice(acc); + state[4..8].copy_from_slice(felts); + state[8..12].copy_from_slice(&iv); + let out = kind.permute(state); + row[cols::OUT0..cols::OUT0 + out.len()].copy_from_slice(&out); + } + HasherKind::Rpx => { + // The filler reads `IN`/`S` back out of the row and writes every + // round intermediate, so the junk propagates through every round + // rather than only into `OUT`. + let out = kind.leaf_out(acc, felts); + row[cols::OUT0..cols::OUT0 + out.len()].copy_from_slice(&out); + super::trace::fill_rpx_witness(&mut row); + } + // The width-16 socket has no twelve-felt LEAF mode to forge a row in. + HasherKind::Poseidon1W16 => {} + } + row +} + +/// ★★ **D1 — a leaf row's UNREAD input cell is pinned on every arm.** +/// +/// `MODE_L` reads two cells; the third receives nothing from `LfmMem`, so unless +/// a constraint pins it, it is four free felts. +/// +/// ⚠ **The break this regression-tests was on the SECOND cell**, back when a +/// leaf read one: `Test`'s and `Poseidon`'s round 0 reads `A_i = IN_i` for +/// `i < 8`, so on those arms the four free felts were consumed by the +/// permutation the AIR proves and `leaf(c)` stopped being a function of `c` — a +/// Fiat–Shamir break for any program that absorbs data through `absorb_felts`. +/// It shipped that way and an adversarial review executed it: Poseidon proved +/// AND verified with attacker junk in those columns. The leaf RATE closed that +/// hole structurally by making the second cell a cell the mode READS (it carries +/// the felts now, with the accumulator in the first), which is why this test +/// moved up to the third cell rather than being deleted: what it guards is the +/// derivation, and the derivation is what stops the NEXT mode repeating the +/// defect. +/// +/// It runs on all three arms because the defect was that one arm had the pin and +/// two did not. +/// +/// **Shaped like WA9**: it does not merely show the junk row is rejected, it +/// shows the pins are what rejects it — the violated set is exactly those four +/// constraints, so a set without them accepts the row. Necessary, not just +/// present. +#[test] +fn d1_the_unread_input_pins_are_load_bearing_under_every_hasher() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let zero: LfmWord = [FE::zero(); 4]; + + for kind in [HasherKind::Test, HasherKind::Rpx, HasherKind::Blake3] { + // HONEST CONTROL FIRST: the pin must not reject honest rows. It cannot — + // every arm's `leaf_out` leaves the unread cell zero — but a fix that + // rejected everything would pass the negative leg on its own. + let honest = leaf_row_with_third_cell(kind, &acc, &felts, &zero); + assert_eq!( + violations_under(kind, &honest), + Vec::::new(), + "{kind:?}: an honest leaf row must still satisfy every constraint" + ); + + // ★ The forgery, built the way an attacker would: junk in the cell the + // mode does not read, and the rest of the row made CONSISTENT with it — + // a prover controls the whole trace, so they would never leave a + // detectable inconsistency behind. That is what makes the assertion + // below exact: with the row otherwise honest, the pins are the only + // constraints that can fire, so `== 4` says the PIN caught it rather + // than something downstream noticing the junk by accident. + let junk: LfmWord = core::array::from_fn(|j| FE::from(0x9E37_79B9_u64 + j as u64)); + let forged = leaf_row_with_third_cell(kind, &acc, &felts, &junk); + + // ⚠ The third cell reaches NO arm's output: `S_i = MODE_P·IN_i + …` + // gates it on the permute selector, and no hashing mode's state includes + // it. So on a leaf row this junk is inert on all three arms and the pin + // is hygiene rather than a live soundness fix — which was NOT true of + // the second cell before the RATE made it a read cell, and is why the + // assertion below is about the pins being the only thing that fires. + assert_eq!( + forged[cols::OUT0], + honest[cols::OUT0], + "{kind:?}: the third cell must not reach the digest" + ); + + // ★ THE WA9 SHAPE. Not "the row is rejected" — that would pass for a + // constraint set that rejected it for some incidental reason, and would + // say nothing about whether the pins are needed. What is asserted is + // that the violated set IS EXACTLY the pins for the cell that was + // forged, which carries both legs at once: + // + // - WITH the pins, the row is rejected; + // - WITHOUT them — delete those four constraints and every other + // constraint in the set still evaluates to zero on this row — it is + // ACCEPTED. That is the dropped-leg, and it is what makes the pins + // load-bearing rather than merely present. + // + // On `Test` and `Poseidon` that acceptance was the shipped behaviour and + // an executed Fiat–Shamir break; on BLAKE3 the row is inert either way, + // which is why the same assertion means "hygiene" there and "soundness" + // on the two arms whose round 0 reads `IN4..8`. + let base = super::chips::hash::unread_input_pin_base(kind); + let expected: Vec = (base..base + 4).collect(); + let violated = violations_under(kind, &forged); + assert_eq!( + violated, expected, + "{kind:?}: the violated set must be EXACTLY the four pins for the \ + forged cell — anything else and the dropped-leg claim does not hold" + ); + } +} + +/// The pins are derived from `HashMode::num_input_cells`, not written per arm — +/// so a mode added later cannot acquire free columns by an arm forgetting it. +/// +/// Structural, and deliberately so: the test above shows the pins fire on the +/// three arms that exist, this shows a fourth arm could not miss them. +#[test] +fn d1_the_pins_come_from_one_derivation() { + use super::chips::hash::{MODE_SELECTORS, NUM_UNREAD_INPUT_PINS}; + + // Every selector is in the table exactly once, and the table agrees with the + // layout's contiguous one-hot span. + assert_eq!(MODE_SELECTORS.len(), super::layout::hash::NUM_SELECTORS); + let mut cols_seen: Vec = MODE_SELECTORS.iter().map(|(c, _)| *c).collect(); + cols_seen.sort_unstable(); + let span: Vec = (super::layout::hash::MODE_C + ..super::layout::hash::MODE_C + super::layout::hash::NUM_SELECTORS) + .collect(); + assert_eq!(cols_seen, span, "the selectors are the one-hot span"); + + // Four pins per unread cell, over the two cells some mode does not read. + let unread: usize = (1..3) + .filter(|slot| { + MODE_SELECTORS + .iter() + .any(|(_, m)| m.num_input_cells() <= *slot) + }) + .count(); + assert_eq!(NUM_UNREAD_INPUT_PINS, 4 * unread); + + // ⚠ And the counts the derivation runs over. `Leaf` reads TWO cells under + // the RATE — accumulator and felts — which is what empties slot 1's set and + // takes the pins from 8 to 4. An emitter that assumed some mode always + // under-reads slot 1 panicked on exactly this (COMMIT.md §1.4.4 H2), so the + // count is asserted rather than assumed. + assert_eq!(HashMode::Leaf.num_input_cells(), 2); + assert_eq!(HashMode::Compress.num_input_cells(), 2); + assert_eq!(HashMode::Transcript.num_input_cells(), 2); + assert_eq!(HashMode::Permute.num_input_cells(), 3); + assert_eq!(unread, 1, "only the third cell is under-read now"); + assert_eq!(NUM_UNREAD_INPUT_PINS, 4); +} diff --git a/prover/src/lfm/lean_upload_tests.rs b/prover/src/lfm/lean_upload_tests.rs new file mode 100644 index 000000000..d7953400c --- /dev/null +++ b/prover/src/lfm/lean_upload_tests.rs @@ -0,0 +1,260 @@ +//! The socket's lean upload: the `LFM_HASH` trace crosses to the card as each +//! row's preprocessed prefix and sixteen input lanes, and the card rebuilds the +//! round witness (`stark::trace::MainDevProducer::poseidon1_w16_rows`). +//! +//! - On the laptop (the `cuda` feature with no card): the producer is attached +//! to a `Poseidon1W16` program's socket traces exactly when the knob is on. +//! - On the card (ignored; run on the box with `--ignored --test-threads=1`): +//! the card's trace equals the host fill word for word, raw `u64`s, padding +//! rows included; and a program whose socket table crosses the device +//! threshold proves to the same bytes with the lean upload on and off, the +//! "on" prove having taken its trace from the producer. + +use stark::proof::options::ProofOptions; +use stark::trace::TraceTable; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; + +use super::builder::{Cell, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::hash::HasherKind; +use super::p1w16_socket::set_lean_upload; +use super::trace::build_traces_with_hasher; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +const KIND: HasherKind = HasherKind::Poseidon1W16; + +/// Restores the knob to the environment's choice when a test ends, pass or fail. +struct Restore; +impl Drop for Restore { + fn drop(&mut self) { + set_lean_upload(None); + } +} + +fn word(seed: u64) -> [FE; 4] { + core::array::from_fn(|i| FE::from(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ (i as u64 + 1))) +} + +/// `perms` chained width-16 permutations (each reads the previous output and +/// three constants), the last output published. +fn chained(perms: usize) -> LfmProgram { + let mut b = LfmBuilder::new(); + let ins: [Cell; 4] = core::array::from_fn(|k| b.digest_const(word(k as u64 + 1)).as_cell()); + let mut out = b.hash16(ins); + for _ in 1..perms { + out = b.hash16([out[0], ins[1], out[2], ins[3]]); + } + for c in out { + b.public(c); + } + compile(b.finish()) +} + +fn hash_trace(program: &LfmProgram) -> TraceTable { + let exec = execute(program, &[], &KIND).expect("execute"); + build_traces_with_hasher(program, &exec.records, KIND).hash +} + +#[test] +fn the_producer_is_attached_to_the_socket_trace_exactly_when_the_knob_is_on() { + let _restore = Restore; + let program = chained(3); + assert!(program.hash16(), "a Poseidon1W16 program builds the socket"); + set_lean_upload(Some(true)); + assert!(hash_trace(&program).has_main_dev_producer()); + set_lean_upload(Some(false)); + assert!(!hash_trace(&program).has_main_dev_producer()); +} + +/// The card's trace for a socket table of `2^12` rows, `3000` of them real, is +/// the host fill's, word for word. +#[test] +#[ignore = "card: rebuilds the socket witness on the GPU; run on the box with --ignored"] +fn the_card_rebuilds_the_host_socket_trace_word_for_word() { + use super::p1w16_socket::{SOCKET_FORM, lean_upload_producer}; + + let program = chained(3000); + let trace = hash_trace(&program); + let (rows, cols) = (trace.num_rows(), trace.num_cols()); + let real = program.groups.hash.real_rows; + assert!( + real < rows, + "padding rows are part of the check ({real} of {rows})" + ); + let (data, width) = trace.main_data_row_major(); + assert_eq!(width, cols); + // SAFETY: `FE` is `#[repr(transparent)]` over `u64`. + let words: &[u64] = + unsafe { std::slice::from_raw_parts(data.as_ptr() as *const u64, data.len()) }; + + let producer = lean_upload_producer(SOCKET_FORM, real); + let card = (producer.0)(words, rows, cols).expect("the card builds the trace"); + let stream = math_cuda::device::backend().expect("card").next_stream(); + let back = stream.clone_dtoh(&card).expect("download"); + stream.synchronize().expect("sync"); + assert_eq!(back.len(), words.len()); + let first = back.iter().zip(words).position(|(a, b)| a != b); + assert_eq!( + first.map(|i| (i / cols, i % cols, back[i], words[i])), + None, + "(row, column, card word, host word) of the first difference" + ); +} + +/// A program whose socket table crosses the device threshold, proved with the +/// lean upload off and on (grinding off, so the bytes are reproducible): the +/// same proof, verified, and the "on" prove took its socket trace from the +/// producer. +#[test] +#[ignore = "card: proves on the GPU; run on the box with --ignored --test-threads=1"] +fn a_lean_upload_proves_the_same_bytes() { + use super::proof::{prove_traces_with_hasher, verify_against}; + use super::registry::build_artifacts_with_hasher; + use stark::proof::options::GoldilocksCubicProofOptions; + + let _restore = Restore; + let mut opts: ProofOptions = GoldilocksCubicProofOptions::with_blowup(4).expect("options"); + opts.grinding_factor = 0; + // 2^13 rows at blowup 4 is a 2^15-row LDE, above the device commit floor. + let program = chained(6000); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let exec = execute(&program, &[], &KIND).expect("execute"); + + let prove = |on: bool| { + set_lean_upload(Some(on)); + let mut traces = build_traces_with_hasher(&program, &exec.records, KIND); + assert_eq!(traces.hash.has_main_dev_producer(), on); + let before = stark::trace::main_dev_producer_calls(); + let proof = prove_traces_with_hasher( + &artifacts, + &mut traces, + &exec.public_words, + &opts, + KIND, + stark::residency_mode::ResidencyMode::Retain, + ) + .expect("prove"); + (proof, stark::trace::main_dev_producer_calls() - before) + }; + let (off, off_calls) = prove(false); + let (on, on_calls) = prove(true); + eprintln!("LEAN UPLOAD producer calls: off {off_calls} · on {on_calls}"); + assert_eq!(off_calls, 0, "the knob off attaches no producer"); + assert!( + on_calls > 0, + "the lean prove never took its trace from the producer" + ); + for proof in [&off, &on] { + assert!(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + proof, + &exec.public_words, + &opts, + KIND, + artifacts.chip_set, + )); + } + let bytes = |p: &super::proof::LfmStarkProof| { + rkyv::to_bytes::(p) + .expect("rkyv") + .to_vec() + }; + assert!( + bytes(&on) == bytes(&off), + "the lean upload moved a proof byte" + ); +} + +/// The lean upload (round 1: the socket trace built on the card) and the +/// sub-coset composition evaluation (round 2: the constraints walked on `2n` +/// rows) in the same prove: both off against both on, the same proof bytes, +/// and the "on" prove took both paths. +#[test] +#[ignore = "card: proves on the GPU; run on the box with --ignored --test-threads=1"] +fn the_lean_upload_and_the_subcoset_composition_together_prove_the_same_bytes() { + use super::proof::{prove_traces_with_hasher, verify_against}; + use super::registry::build_artifacts_with_hasher; + use stark::proof::options::GoldilocksCubicProofOptions; + use stark::prover::{composition_subcoset_tables, set_composition_subcoset}; + + struct RestoreBoth; + impl Drop for RestoreBoth { + fn drop(&mut self) { + set_lean_upload(None); + set_composition_subcoset(None); + } + } + let _restore = RestoreBoth; + let mut opts: ProofOptions = GoldilocksCubicProofOptions::with_blowup(4).expect("options"); + opts.grinding_factor = 0; + // 2^13 rows at blowup 4 is a 2^15-row LDE, above the device commit floor. + let program = chained(6000); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let exec = execute(&program, &[], &KIND).expect("execute"); + + let prove = |on: bool| { + set_lean_upload(Some(on)); + set_composition_subcoset(Some(on)); + let mut traces = build_traces_with_hasher(&program, &exec.records, KIND); + assert_eq!(traces.hash.has_main_dev_producer(), on); + let before = ( + stark::trace::main_dev_producer_calls(), + composition_subcoset_tables(), + ); + let proof = prove_traces_with_hasher( + &artifacts, + &mut traces, + &exec.public_words, + &opts, + KIND, + stark::residency_mode::ResidencyMode::Retain, + ) + .expect("prove"); + let calls = ( + stark::trace::main_dev_producer_calls() - before.0, + composition_subcoset_tables() - before.1, + ); + (proof, calls) + }; + let (off, off_calls) = prove(false); + let (on, on_calls) = prove(true); + eprintln!( + "LEAN UPLOAD + SUBCOSET (producer calls, sub-coset tables): off {off_calls:?} · on {on_calls:?}" + ); + assert_eq!(off_calls, (0, 0), "both off takes neither path"); + assert!( + on_calls.0 > 0, + "the prove never took its trace from the producer" + ); + assert!( + on_calls.1 > 0, + "the prove never evaluated a composition on the sub-coset" + ); + for proof in [&off, &on] { + assert!(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + proof, + &exec.public_words, + &opts, + KIND, + artifacts.chip_set, + )); + } + let bytes = |p: &super::proof::LfmStarkProof| { + rkyv::to_bytes::(p) + .expect("rkyv") + .to_vec() + }; + assert!( + bytes(&on) == bytes(&off), + "the lean upload with the sub-coset composition moved a proof byte" + ); +} diff --git a/prover/src/lfm/logup.rs b/prover/src/lfm/logup.rs new file mode 100644 index 000000000..ff06a5e9f --- /dev/null +++ b/prover/src/lfm/logup.rs @@ -0,0 +1,150 @@ +//! The LogUp closure: `Σ_tables L = expected_bus_balance`. +//! +//! Every other leg verifies one sub-proof. This is the only one that is about +//! the epoch as a whole: each table exposes the total of its LogUp terms, `L`, +//! and the bus balances when those totals sum to the target. Production's check +//! is `verifier.rs`'s final block — `Σ L over tables with trace interactions`, +//! compared against an `expected_bus_balance` the caller supplies. +//! +//! # The target is computed, not zero +//! +//! It would be zero if every bus participant were an in-trace table. One is +//! not: the COMMIT output bus has a receiver the verifier computes rather than +//! proves, so the target is that missing positive remainder +//! (`lib.rs`'s `compute_commit_bus_offset`): +//! +//! ```text +//! expected = Σ_i 1 / (z − (BusId::Commit + (start + i)·α + byte_i·α²)) +//! ``` +//! +//! over the public output BYTES. So half this leg is a per-byte gadget over the +//! epoch's public output, not a comparison against a constant. +//! +//! # Reciprocals, and why the machine's division convention matters here +//! +//! Production batch-inverts the fingerprints and REJECTS on a zero divisor — +//! `inplace_batch_inverse(...).ok()?`, a fingerprint collision. The machine's +//! `x/0` is an error and `0/0` is one, so `1/fingerprint` is unprovable at a +//! collision and provable everywhere else: the convention already matches, but +//! only because the numerator is the constant one. The DEEP leg had to invert +//! against an interned one for exactly this reason and a direct divide would +//! have accepted what production rejects; the same care applies here. +//! +//! # What is shape and what is data +//! +//! Which tables carry a bus contribution is `AIR::has_trace_interaction()` — +//! AIR shape, so a program constant. The number of public output bytes is shape +//! too, because it fixes the gadget's length; the byte VALUES are data. `start` +//! is the carried commit index, data. + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, Felt, LfmBuilder}; + +/// The compile-time shape of one epoch's LogUp closure. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LogUpShape { + /// Sub-proofs whose `L` enters the sum — those with trace interactions. + /// SHAPE: a program that read this off the proof would let the prover + /// choose which tables are on the bus. + pub num_contributing_tables: usize, + /// Public output bytes the COMMIT-bus target folds over. + pub num_output_bytes: usize, +} + +/// The `BusId::Commit` discriminant, as the fingerprint's constant term. +/// +/// Mirrored rather than imported so this module does not depend on the VM's bus +/// enum; [`bus_id_matches_production`] pins the two together. +pub const COMMIT_BUS_ID: u64 = crate::tables::types::BusId::Commit as u64; + +/// The COMMIT-bus target: `Σ_i 1/(z − (busId + (start+i)·α + byte_i·α²))`. +/// +/// `bytes` are the public output bytes as base cells, one byte per cell, in +/// order — the same order `compute_commit_bus_offset` enumerates them. `start` +/// is the carried commit index (`x254`): zero for a monolithic proof or a first +/// epoch, nonzero for an epoch continuing a prior one. +/// +/// The index `start + i` is derived by ADDING ONE per byte rather than by +/// hinting each index, so a prover cannot renumber the output: `i` is position, +/// and position is program text. +pub fn emit_commit_bus_target( + b: &mut LfmBuilder, + shape: &LogUpShape, + z: Ext, + alpha: Ext, + start: Felt, + bytes: &[Felt], +) -> Ext { + assert_eq!( + bytes.len(), + shape.num_output_bytes, + "the output length is shape and fixes the gadget's size" + ); + if shape.num_output_bytes == 0 { + // `compute_commit_bus_offset` short-circuits to zero on empty output. + return b.ext_const(&FEE::zero()); + } + + let one = b.ext_const(&FEE::one()); + let bus_id = b.ext_const(&FEE::from(COMMIT_BUS_ID)); + let alpha_sq = b.emul(alpha, alpha); + let one_base = b.felt_const(FE::one()); + + let mut acc: Option = None; + let mut index = start; + for (i, byte) in bytes.iter().enumerate() { + // linear = busId + index·α + byte·α². + let index_term = b.emul_base(alpha, index); + let byte_term = b.emul_base(alpha_sq, *byte); + let linear = b.eadd(bus_id, index_term); + let linear = b.eadd(linear, byte_term); + let fingerprint = b.esub(z, linear); + // Inverted against the interned one: a collision is `1/0`, which is + // unprovable, matching production's rejection. A direct divide of a + // vanishing numerator would instead give `0/0 = 1`. + let term = b.ediv(one, fingerprint); + acc = Some(match acc { + None => term, + Some(a) => b.eadd(a, term), + }); + if i + 1 < bytes.len() { + index = b.add(index, one_base); + } + } + acc.expect("a nonempty output folds at least one term") +} + +/// The closure: sum the per-table contributions and assert the bus balances. +/// +/// `contributions` are the `L` cells — and they must be the SAME cells the +/// constraint leg divided by `N` to get its per-row offset (see +/// [`super::constraints::emit_table_offset`]). A program that hinted `L` here +/// and `L/N` there would let the prover pick both, and this assert would be a +/// statement about numbers bound to no trace. +/// +/// Returns the published sum, so a verifier sees what balanced rather than only +/// that something did. +pub fn emit_bus_closure( + b: &mut LfmBuilder, + shape: &LogUpShape, + contributions: &[Ext], + target: Ext, +) -> Ext { + assert_eq!( + contributions.len(), + shape.num_contributing_tables, + "the contributing-table count is shape and is never read off the proof" + ); + let mut total = match contributions.first() { + Some(first) => *first, + // No table carries a bus interaction: production skips the check + // entirely, so the honest total is zero and the target must be too. + None => b.ext_const(&FEE::zero()), + }; + for c in contributions.iter().skip(1) { + total = b.eadd(total, *c); + } + b.assert_eq_ext(total, target); + total +} diff --git a/prover/src/lfm/logup_tests.rs b/prover/src/lfm/logup_tests.rs new file mode 100644 index 000000000..ab70abdc3 --- /dev/null +++ b/prover/src/lfm/logup_tests.rs @@ -0,0 +1,1458 @@ +//! The LogUp closure, and the join it inherits. +//! +//! ## The oracles +//! +//! Two, both production's own. `compute_commit_bus_offset` (`lib.rs`) for the +//! COMMIT-bus target, and `crate::hash_pin::BlockVerifier::multi_verify` for the balance itself — the +//! fixture is a real sender/receiver pair whose bus genuinely closes, and +//! production accepting it at target zero is what says so. Nothing here asserts +//! a balance this file computed. +//! +//! ## What this suite cannot see +//! +//! Whether `L` is bound to a table's aux TRACE. That binding is the circular +//! accumulator constraint plus the `acc[0] = 0` boundary, and it belongs to the +//! constraint leg; this suite checks only that the closure consumes the same +//! `L` that leg divides by `N`. +//! +//! Most fixtures here are two or three tables rather than the twenty-odd a +//! continuation epoch carries, so they exercise the SUM but not its length. One +//! is not: [`a_zero_row_fixed_table_carries_some_zero_not_none`] proves and +//! verifies a real epoch and runs the closure over all twenty-four of its +//! contributions. +use stark::proof::stark::MultiProof; +use stark::proof::view::StarkProofView; +use stark::traits::AIR; +use stark::verifier::IsStarkVerifier; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::executor::execute; +use super::hash::TestPermutation; +use super::logup::{COMMIT_BUS_ID, LogUpShape, emit_bus_closure, emit_commit_bus_target}; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +fn options() -> stark::proof::options::ProofOptions { + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// The mirrored bus discriminant really is the VM's. +/// +/// A copied protocol constant is the one thing a differential cannot catch, +/// because both sides of it move together — the same reason the join suite pins +/// `ROWS_PER_LEAF` against `crypto/stark`'s. +#[test] +fn bus_id_matches_production() { + assert_eq!( + COMMIT_BUS_ID, + crate::tables::types::BusId::Commit as u64, + "the COMMIT fingerprint's constant term is the VM's own bus id" + ); +} + +// ============================================================================= +// The COMMIT-bus target +// ============================================================================= + +/// Emit the target gadget alone and run it. +fn run_target(bytes: &[u8], start: u64, z: FEE, alpha: FEE) -> Option { + let shape = LogUpShape { + num_contributing_tables: 0, + num_output_bytes: bytes.len(), + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena((3 + bytes.len()) as u32); + let z_cell = b.hint_word(arena, 0).as_ext(); + let alpha_cell = b.hint_word(arena, 1).as_ext(); + let start_cell = b.hint_felt(arena, 2); + let byte_cells: Vec<_> = (0..bytes.len() as u32) + .map(|i| b.hint_felt(arena, 3 + i)) + .collect(); + let target = + emit_commit_bus_target(&mut b, &shape, z_cell, alpha_cell, start_cell, &byte_cells); + b.public(target.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the target program is admissible"); + + let words: Vec = std::iter::once(ext_word(&z)) + .chain(std::iter::once(ext_word(&alpha))) + .chain(std::iter::once(base_word(FE::from(start)))) + .chain(bytes.iter().map(|v| base_word(FE::from(*v as u64)))) + .collect(); + execute(&program, &[words], &TestPermutation) + .ok() + .map(|e| word_as_ext(&e.public_words[0].1).expect("ext")) +} + +/// ★ The machine's COMMIT-bus target equals production's, over a sweep of +/// lengths and carried start indices. +/// +/// The lengths are not decorative. `start` advances by one per byte inside the +/// gadget, so a formula that reset it, or that folded the bytes in reverse, +/// agrees with production only at length one — and the empty case is a separate +/// short-circuit in production that a nonempty-only test would never reach. +#[test] +fn the_commit_bus_target_matches_production() { + let z = FEE::new([FE::from(7u64), FE::from(11u64), FE::from(13u64)]); + let alpha = FEE::new([FE::from(5u64), FE::from(3u64), FE::from(2u64)]); + + let mut checked = 0usize; + for len in [0usize, 1, 2, 3, 7, 8, 33] { + // Distinct byte values, so a gadget that mixed up index and value would + // not accidentally agree. + let bytes: Vec = (0..len).map(|i| (17 * i + 3) as u8).collect(); + for start in [0u64, 1, 254, 1_000_000] { + let want = crate::compute_commit_bus_offset(&bytes, start, &z, &alpha) + .expect("no collision on this fixture"); + let got = run_target(&bytes, start, z, alpha) + .unwrap_or_else(|| panic!("len {len} start {start}: the target must execute")); + assert_eq!(got, want, "len {len}, start {start}"); + if len > 0 { + assert_ne!( + got, + FEE::zero(), + "len {len} start {start}: a zero target would make the \ + comparison vacuous" + ); + } + checked += 1; + } + } + println!("commit-bus target: {checked} (length, start) combinations vs production"); +} + +/// ★ A fingerprint COLLISION is rejected, not silently folded. +/// +/// Production batch-inverts and returns `None` on a zero divisor. The machine +/// divides the interned ONE by the fingerprint, so a collision is `1/0` — an +/// error, hence unprovable. Had the term been written as a direct division with +/// a vanishing numerator instead, the `0/0 = 1` convention would have accepted +/// exactly the proof production rejects, which is the mistake the DEEP leg +/// documents and this test exists to keep from recurring. +#[test] +fn a_fingerprint_collision_is_unprovable() { + // fingerprint_0 = z − (busId + start·α + byte·α²). With α = 1, start = 0 + // and byte = 0 that is z − busId, so z = busId collides exactly. + let alpha = FEE::one(); + let z = FEE::from(COMMIT_BUS_ID); + let bytes = [0u8]; + + assert!( + crate::compute_commit_bus_offset(&bytes, 0, &z, &alpha).is_none(), + "the fixture must be a genuine collision for production too, or this \ + test is checking the machine against nothing" + ); + assert!( + run_target(&bytes, 0, z, alpha).is_none(), + "a colliding fingerprint must make the run unexecutable" + ); + + // And the same shape one step away from the collision still works, so the + // rejection is the collision and not the shape. + let z_ok = z + FEE::one(); + let want = crate::compute_commit_bus_offset(&bytes, 0, &z_ok, &alpha).expect("no collision"); + assert_eq!(run_target(&bytes, 0, z_ok, alpha).expect("executes"), want); + println!("collision rejected; the neighbouring non-colliding z still folds"); +} + +// ============================================================================= +// The closure, over a bus that really balances +// ============================================================================= + +/// A sender/receiver pair over one bus, proved together. +/// +/// Modelled on `tests::bitwise_tests`' pair: the sender emits one AND lookup, +/// the receiver answers it. Their contributions are equal and opposite, so the +/// bus closes at zero — and `multi_verify` accepting at target zero is what +/// establishes that, rather than any arithmetic here. +fn balanced_pair() -> (Vec, MultiProof) { + use crate::tables::types::{BusId, alu_op}; + use crate::test_utils::multi_prove_ram; + use stark::constraints::builder::EmptyConstraints; + use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, + }; + use stark::trace::TraceTable; + + const X: u64 = 5; + const Y: u64 = 3; + const NUM_ROWS: usize = 4; + + type Air = AirWithBuses; + let opts = options(); + + // Columns: 0 = x, 1 = y, 2 = and, 3 = multiplicity/flag. Same layout both + // sides, so one trace builder serves. + // Both sides fingerprint the SAME tuple; only the sender/receiver sign + // differs, which is what makes the two contributions cancel. + let values = || { + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 2, + packing: Packing::Direct, + }, + ] + }; + + let sender = Air::new( + 4, + AuxiliaryTraceBuildData { + interactions: vec![BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(3), + values(), + )], + }, + &opts, + 1, + EmptyConstraints, + ) + .with_name("SENDER"); + let receiver = Air::new( + 4, + AuxiliaryTraceBuildData { + interactions: vec![BusInteraction::receiver( + BusId::ByteAlu, + Multiplicity::Column(3), + values(), + )], + }, + &opts, + 1, + EmptyConstraints, + ) + .with_name("RECEIVER"); + + let make_trace = || { + let mut data = vec![FE::zero(); NUM_ROWS * 4]; + data[0] = FE::from(X); + data[1] = FE::from(Y); + data[2] = FE::from(X & Y); + data[3] = FE::one(); + TraceTable::::new_main(data, 4, 1) + }; + let mut sender_trace = make_trace(); + let mut receiver_trace = make_trace(); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&sender, &mut sender_trace, &()), + (&receiver, &mut receiver_trace, &()), + ]; + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .expect("the balanced pair must prove"); + (vec![Box::new(sender), Box::new(receiver)], proof) +} + +type BoxedAir = Box>; + +/// ★ The machine's closure accepts a bus production says balances, and rejects +/// every single-word move away from it. +/// +/// The oracle is `multi_verify` at target zero. It is checked FIRST: if the +/// fixture's bus did not actually close, the machine agreeing with it would say +/// nothing. +#[test] +fn the_closure_matches_a_bus_that_really_balances() { + let (airs, proof) = balanced_pair(); + let air_refs: Vec<&dyn AIR> = + airs.iter().map(|a| &**a).collect(); + + assert!( + crate::hash_pin::BlockVerifier::multi_verify( + &air_refs, + &proof, + &mut crate::hash_pin::block_transcript(&[]), + &FEE::zero(), + ), + "production must accept this pair at target zero, or the fixture is not \ + a balanced bus and nothing below means anything" + ); + + let contributions: Vec = (0..proof.proofs.len()) + .map(|i| { + StarkProofView::Owned(&proof.proofs[i]) + .bus_table_contribution() + .expect("both tables carry a contribution") + }) + .collect(); + assert_eq!(contributions.len(), 2); + assert!( + contributions.iter().all(|c| *c != FEE::zero()), + "both contributions must be nonzero, else the sum is vacuously zero: {contributions:?}" + ); + assert_eq!( + contributions[0] + contributions[1], + FEE::zero(), + "the pair is equal and opposite" + ); + + let shape = LogUpShape { + num_contributing_tables: 2, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let cells: Vec<_> = (0..2u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + let total = emit_bus_closure(&mut b, &shape, &cells, zero); + b.public(total.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the closure program is admissible"); + + let honest: Vec = contributions.iter().map(ext_word).collect(); + let exec = execute(&program, std::slice::from_ref(&honest), &TestPermutation) + .expect("a balanced bus must close in the machine too"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + FEE::zero() + ); + + // Falsification: move either contribution, in any lane. + let mut vectors = 0usize; + for table in 0..2usize { + for lane in 0..3usize { + let mut arenas = vec![honest.clone()]; + arenas[0][table][lane] += FE::one(); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "table {table} lane {lane}: an unbalanced bus must not close" + ); + vectors += 1; + } + } + println!("closure: balanced pair accepted, {vectors} single-lane moves rejected"); +} + +// ============================================================================= +// The join: one `L`, two consumers +// ============================================================================= + +use super::constraint_tests::{RealSubProof, real_sub_proof}; +use super::constraints::{ + OodOperands, emit_alpha_powers, emit_constraint_evals, emit_quotient, emit_table_offset, + hint_ood_frame, ood_frame_words, +}; + +/// Where the per-row offset comes from. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Offset { + /// `L/N` derived in-machine from the closure's own `L`. What production + /// does, and what [`emit_table_offset`] exists to enforce. + Derived, + /// `L/N` hinted as its own arena word, independent of the `L` the closure + /// sums. The shape this leg exists to forbid — a test artifact, built to be + /// attacked. + HintedSeparately, +} + +/// One sub-proof's composition check AND the LogUp closure, over the same +/// proof, with the offset wired either way. +/// +/// Arenas 0-2 are the constraint leg's (frame, uniforms, parts); arena 3 is the +/// closure's expected balance, plus — in the split shape only — the separately +/// hinted offset. +fn composition_and_closure_source( + sp: &RealSubProof, + offset: Offset, +) -> super::builder::LfmProgramSource { + let mut b = LfmBuilder::new(); + + let frame_arena = b.declare_arena(ood_frame_words(&sp.artifact)); + let (steps, _) = hint_ood_frame(&mut b, &sp.artifact, frame_arena, 0); + + let num_uniforms = (sp.rap_challenges.len() + 3) as u32; + let uniform_arena = b.declare_arena(num_uniforms); + let mut next = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(uniform_arena, next).as_ext(); + next += 1; + c + }; + let rap_challenges: Vec<_> = (0..sp.rap_challenges.len()).map(|_| take(&mut b)).collect(); + let alpha_powers = emit_alpha_powers( + &mut b, + rap_challenges[stark::lookup::LOGUP_CHALLENGE_ALPHA], + sp.alpha_powers.len(), + ); + let contribution = take(&mut b); + let zeta = take(&mut b); + let beta = take(&mut b); + + let parts_arena = b.declare_arena(sp.claimed_parts.len() as u32); + let claimed_parts: Vec<_> = (0..sp.claimed_parts.len() as u32) + .map(|i| b.hint_word(parts_arena, i).as_ext()) + .collect(); + + let closure_arena = b.declare_arena(match offset { + Offset::Derived => 1, + Offset::HintedSeparately => 2, + }); + let target = b.hint_word(closure_arena, 0).as_ext(); + + let table_offset = match offset { + Offset::Derived => emit_table_offset(&mut b, contribution, sp.quotient.log2_trace_length), + Offset::HintedSeparately => b.hint_word(closure_arena, 1).as_ext(), + }; + + let ood = OodOperands { + steps, + main_width: sp.main_width, + rap_challenges, + alpha_powers, + table_offset, + }; + let (evals, _) = emit_constraint_evals(&mut b, &sp.artifact, &ood); + let q = emit_quotient( + &mut b, + &sp.quotient, + &ood, + zeta, + beta, + &evals, + &claimed_parts, + ); + b.assert_eq_ext(q.claimed, q.composition); + + let shape = LogUpShape { + num_contributing_tables: 1, + num_output_bytes: 0, + }; + let total = emit_bus_closure(&mut b, &shape, &[contribution], target); + b.public(total.as_cell()); + b.finish() +} + +/// Arenas for the program above. `delta` moves the `L` the CLOSURE sums (and +/// the target with it, so the closure itself still balances); the offset stays +/// truthful, which is what a forger would want. +fn join_arenas(sp: &RealSubProof, offset: Offset, delta: FEE) -> Vec> { + let mut arenas = sp.arenas(); + let forged = sp.contribution + delta; + // Slot of `contribution` inside the uniform arena. + let slot = sp.rap_challenges.len(); + arenas[1][slot] = ext_word(&forged); + let mut closure = vec![ext_word(&forged)]; + if offset == Offset::HintedSeparately { + closure.push(ext_word(&sp.table_offset)); + } + arenas.push(closure); + arenas +} + +/// ★ The join, stated as the property it exists for: a prover cannot feed the +/// bus a contribution the constraint leg did not accept. +/// +/// The honest run passes both halves. Moving `L` — while keeping the closure +/// self-consistent by moving its target too, which is exactly what a forger +/// would do — must break the CONSTRAINT half, because the offset is derived +/// from the very cell that moved. +/// +/// The control is the same program with the offset hinted separately. It +/// accepts the forgery: the accumulator sees a truthful `L/N` and wraps, the +/// closure sees a fabricated `L` and balances, and the bus statement is about a +/// number attached to no trace. That is what the derivation denies, and it is +/// run here rather than argued. +#[test] +fn the_closure_cannot_sum_a_contribution_the_constraints_rejected() { + let sp = real_sub_proof(); + assert_ne!( + sp.contribution, + FEE::zero(), + "the fixture's table must carry a real bus contribution, or moving it \ + is not a tamper" + ); + + let joined = compile(composition_and_closure_source(&sp, Offset::Derived)); + validate(&joined).expect("the joined program is admissible"); + let split = compile(composition_and_closure_source( + &sp, + Offset::HintedSeparately, + )); + validate(&split).expect("the control is admissible"); + + // Honest, both shapes. + for (label, program, offset) in [ + ("joined", &joined, Offset::Derived), + ("control", &split, Offset::HintedSeparately), + ] { + let exec = execute( + program, + &join_arenas(&sp, offset, FEE::zero()), + &TestPermutation, + ) + .unwrap_or_else(|e| panic!("{label}: the honest run must execute: {e:?}")); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + sp.contribution, + "{label}: the published total is the contribution that was checked" + ); + } + + // Forge, several deltas and several lanes, so the vector class is not one + // value in one coordinate. + let deltas = [ + FEE::one(), + FEE::new([FE::zero(), FE::one(), FE::zero()]), + FEE::new([FE::zero(), FE::zero(), FE::from(7u64)]), + FEE::new([FE::from(3u64), FE::from(5u64), FE::from(9u64)]), + ]; + for delta in deltas { + assert!( + execute( + &joined, + &join_arenas(&sp, Offset::Derived, delta), + &TestPermutation + ) + .is_err(), + "joined: a forged contribution must break the constraint half \ + (delta {delta:?})" + ); + let forged = execute( + &split, + &join_arenas(&sp, Offset::HintedSeparately, delta), + &TestPermutation, + ) + .unwrap_or_else(|e| panic!("control: the split shape is what PERMITS this forgery: {e:?}")); + assert_eq!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + sp.contribution + delta, + "control: the forgery publishes its own fabricated contribution" + ); + } + println!( + "join: {} forged contributions rejected by the derivation, all accepted \ + by the split control", + deltas.len() + ); +} + +/// ★ The two-consumer rule, as an ABSOLUTE property of the emitted program. +/// +/// Method rule 7: a relative test dies the moment its two sides unify, so this +/// asserts nothing about variants and everything about the program itself — +/// which cells are arena hints and which are computed. `L/N` and every alpha +/// power must be COMPUTED, because a hinted one is a claim about `L` or `α` +/// that no other constraint checks; `L` itself and the raw challenges must be +/// hints, or the test would pass vacuously against an emitter that simply +/// dropped them. +/// +/// This survives any refactor of how the offset is produced. It only fails if +/// something starts reading these values out of an arena again, which is +/// exactly the regression it exists to catch. +#[test] +fn the_derived_uniforms_are_not_arena_words() { + use super::instr::Instr; + + let sp = real_sub_proof(); + assert!( + !sp.alpha_powers.is_empty(), + "the fixture must exercise LogUp alpha powers" + ); + + let mut b = LfmBuilder::new(); + let uniform_arena = b.declare_arena((sp.rap_challenges.len() + 1) as u32); + let rap: Vec<_> = (0..sp.rap_challenges.len() as u32) + .map(|i| b.hint_word(uniform_arena, i).as_ext()) + .collect(); + let contribution = b + .hint_word(uniform_arena, sp.rap_challenges.len() as u32) + .as_ext(); + let alpha_powers = emit_alpha_powers( + &mut b, + rap[stark::lookup::LOGUP_CHALLENGE_ALPHA], + sp.alpha_powers.len(), + ); + let table_offset = emit_table_offset(&mut b, contribution, sp.quotient.log2_trace_length); + let source = b.finish(); + + let hinted: std::collections::HashSet = source + .instrs + .iter() + .filter_map(|i| match i { + Instr::Hint { out, .. } => Some(out.0), + _ => None, + }) + .collect(); + + // Positive control: the things that SHOULD be arena words are. + assert!( + hinted.contains(&contribution.addr().0), + "L itself is proof data and must be hinted, or this test is checking \ + an emitter that reads nothing" + ); + for (i, c) in rap.iter().enumerate() { + assert!( + hinted.contains(&c.addr().0), + "rap challenge {i} is hinted in this isolated slice" + ); + } + + // The property: derived values are not arena words. + assert!( + !hinted.contains(&table_offset.addr().0), + "L/N must be COMPUTED from L; a hinted offset lets a prover satisfy \ + every accumulator while the closure sums a different L" + ); + for (i, p) in alpha_powers.iter().enumerate() { + // alpha^1 IS the alpha cell, which is legitimately hinted here; every + // other power must be computed (alpha^0 is an interned constant). + if i == stark::lookup::LOGUP_CHALLENGE_ALPHA { + continue; + } + assert!( + !hinted.contains(&p.addr().0), + "alpha power {i} must be COMPUTED from alpha; a hinted power is a \ + claim about alpha that nothing checks, and the LogUp fingerprints \ + are built out of exactly these" + ); + } + println!( + "absolute check: L/N and {} alpha powers are computed, not hinted", + alpha_powers.len() + ); +} + +// ============================================================================= +// Degenerate parameter: per-CHUNK vs per-FAMILY accumulation +// ============================================================================= + +/// One sender and TWO receiver chunks of the same family, proved together. +/// +/// A continuation epoch splits each table family into chunks, and `VmAirs::new` +/// builds one AIR per chunk (`lib.rs`: `(0..table_counts.cpu).map(|i| … CPU[i])`), +/// so a family of `k` chunks is `k` entries in the AIR vector and `k` +/// sub-proofs. The closure iterates those entries, so it accumulates PER CHUNK. +/// +/// Every fixture up to here had one sub-proof per family, which makes +/// per-chunk and per-family the same sum — the degenerate case. Two chunks of +/// one family is the smallest shape that tells them apart: the sender's two +/// lookups are answered one per chunk, so dropping either chunk leaves a +/// nonzero remainder. +fn chunked_family() -> (Vec, MultiProof) { + use crate::tables::types::{BusId, alu_op}; + use crate::test_utils::multi_prove_ram; + use stark::constraints::builder::EmptyConstraints; + use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, + }; + use stark::trace::TraceTable; + + /// The two lookups, answered by one receiver chunk each. + const LOOKUPS: [(u64, u64); 2] = [(5, 3), (9, 6)]; + const NUM_ROWS: usize = 4; + + type Air = AirWithBuses; + let opts = options(); + + let values = || { + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 2, + packing: Packing::Direct, + }, + ] + }; + let build = |sender: bool, name: &str| { + let interaction = if sender { + BusInteraction::sender(BusId::ByteAlu, Multiplicity::Column(3), values()) + } else { + BusInteraction::receiver(BusId::ByteAlu, Multiplicity::Column(3), values()) + }; + Air::new( + 4, + AuxiliaryTraceBuildData { + interactions: vec![interaction], + }, + &opts, + 1, + EmptyConstraints, + ) + .with_name(name) + }; + + // The two receiver chunks are the SAME construction — one family, two + // instances, exactly as `CPU[0]` and `CPU[1]` are. + let sender = build(true, "SENDER"); + let recv0 = build(false, "RECEIVER[0]"); + let recv1 = build(false, "RECEIVER[1]"); + + let trace_for = |rows: &[(u64, u64)]| { + let mut data = vec![FE::zero(); NUM_ROWS * 4]; + for (r, (x, y)) in rows.iter().enumerate() { + data[r * 4] = FE::from(*x); + data[r * 4 + 1] = FE::from(*y); + data[r * 4 + 2] = FE::from(x & y); + data[r * 4 + 3] = FE::one(); + } + TraceTable::::new_main(data, 4, 1) + }; + let mut sender_trace = trace_for(&LOOKUPS); + let mut recv0_trace = trace_for(&LOOKUPS[..1]); + let mut recv1_trace = trace_for(&LOOKUPS[1..]); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&sender, &mut sender_trace, &()), + (&recv0, &mut recv0_trace, &()), + (&recv1, &mut recv1_trace, &()), + ]; + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .expect("the chunked family must prove"); + ( + vec![Box::new(sender), Box::new(recv0), Box::new(recv1)], + proof, + ) +} + +/// ★ The closure accumulates per CHUNK, and that is observable. +/// +/// Both halves of the degenerate-parameter rule. The machine's three-term sum +/// closes a bus production accepts at target zero; and every two-term sum — a +/// per-family reading, which would collapse the two receiver chunks into one +/// contribution — is NONZERO, so the distinction is load-bearing on this +/// fixture rather than merely stated. +/// +/// Without the second half this test would pass against an emitter that folded +/// a family's chunks into a single term, because on every earlier fixture, and +/// on any workload whose families happen to be one chunk each, the two readings +/// agree. +#[test] +fn the_closure_accumulates_per_chunk_not_per_family() { + let (airs, proof) = chunked_family(); + let air_refs: Vec<&dyn AIR> = + airs.iter().map(|a| &**a).collect(); + assert_eq!(air_refs.len(), 3, "one sender and two chunks of one family"); + + assert!( + crate::hash_pin::BlockVerifier::multi_verify( + &air_refs, + &proof, + &mut crate::hash_pin::block_transcript(&[]), + &FEE::zero(), + ), + "production must accept the chunked family at target zero, or the \ + fixture is not a balanced bus" + ); + + let contributions: Vec = (0..proof.proofs.len()) + .map(|i| { + StarkProofView::Owned(&proof.proofs[i]) + .bus_table_contribution() + .expect("every table here has interactions") + }) + .collect(); + assert!( + contributions.iter().all(|c| *c != FEE::zero()), + "each chunk must carry its OWN nonzero contribution — a zero one would \ + make dropping it invisible: {contributions:?}" + ); + + // Half one: the per-chunk sum closes, in the machine. + let shape = LogUpShape { + num_contributing_tables: 3, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(3); + let cells: Vec<_> = (0..3u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + let total = emit_bus_closure(&mut b, &shape, &cells, zero); + b.public(total.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("admissible"); + + let words: Vec = contributions.iter().map(ext_word).collect(); + let exec = execute(&program, std::slice::from_ref(&words), &TestPermutation) + .expect("the per-chunk sum must close"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + FEE::zero() + ); + + // Half two: every two-term reading DISAGREES. Dropping chunk 1 or chunk 2 + // is exactly what a per-family accumulator would do. + for dropped in 0..3usize { + let partial: FEE = contributions + .iter() + .enumerate() + .filter(|(i, _)| *i != dropped) + .map(|(_, c)| *c) + .fold(FEE::zero(), |a, c| a + c); + assert_ne!( + partial, + FEE::zero(), + "dropping table {dropped} must break the balance, or the per-chunk \ + reading is not observable on this fixture" + ); + } + // …and the same statement in the MACHINE: a closure compiled for two + // contributing tables — what a per-family emitter would build, one term per + // family — fed a well-formed two-word arena, must not close. + let family_shape = LogUpShape { + num_contributing_tables: 2, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let cells: Vec<_> = (0..2u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + emit_bus_closure(&mut b, &family_shape, &cells, zero); + let per_family = compile(b.finish()); + for dropped in 1..3usize { + let words: Vec = contributions + .iter() + .enumerate() + .filter(|(i, _)| *i != dropped) + .map(|(_, c)| ext_word(c)) + .collect(); + assert!( + execute(&per_family, &[words], &TestPermutation).is_err(), + "a per-family closure that folded chunk {dropped} away must not \ + close the bus" + ); + } + + println!( + "per-chunk accumulation witnessed: 3 chunks close, all 3 two-term \ + readings nonzero, and a per-family closure rejects both chunk drops" + ); +} + +/// ★ `has_trace_interaction()` is SHAPE, and production checks the proof's +/// presence against it in BOTH directions. +/// +/// `verifier.rs:1238` rejects an AIR with interactions whose proof carries no +/// bus public inputs, and `:1244` rejects the converse. So the count of +/// contributing tables is fixed by the AIR set, never read off the proof — +/// which is why [`LogUpShape::num_contributing_tables`] is a program constant. +/// A machine that sized its sum from the arena would let a prover drop a +/// table's contribution from the bus by omitting it. +#[test] +fn the_contributing_table_count_is_shape() { + let (airs, proof) = chunked_family(); + for (i, air) in airs.iter().enumerate() { + let view = StarkProofView::Owned(&proof.proofs[i]); + assert!( + air.has_trace_interaction(), + "table {i} of this fixture declares interactions" + ); + assert_eq!( + air.has_trace_interaction(), + view.has_bus_public_inputs(), + "table {i}: production rejects any disagreement between the AIR's \ + declared interactions and the proof's bus public inputs, in both \ + directions — so the two can never disagree in a proof that verifies" + ); + } + + // The shape is the AIR set's property. A program compiled for three + // contributing tables cannot read a two-table arena: the schema mismatches. + let shape = LogUpShape { + num_contributing_tables: 3, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(3); + let cells: Vec<_> = (0..3u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + emit_bus_closure(&mut b, &shape, &cells, zero); + let program = compile(b.finish()); + assert!( + execute( + &program, + &[vec![ext_word(&FEE::zero()); 2]], + &TestPermutation + ) + .is_err(), + "a short arena must not satisfy a program compiled for three tables" + ); + println!("contributing-table count is shape: a short arena is rejected"); +} + +// ============================================================================= +// Degenerate parameter: a fixed table with NO rows, on a real epoch +// ============================================================================= + +/// How a fixed table's "no rows on the bus" claim is witnessed from its TRACE. +/// +/// Needed because a zero-row table is not the same thing as a blank one. Two +/// padding conventions are in play, and taking either for the general case would +/// have mislabelled the other: +/// +/// - `generate_keccak_rnd_trace` and ECSM's write nothing at all when there are +/// no operations, so their traces are literally zero. +/// - `generate_keccak_trace` pads with `state_ptr[lane] = 8·lane` (and KECCAK_RC +/// is a preprocessed constant table, ECDAS pads likewise). Those traces are +/// NOT zero, yet no row of them is on any bus, because every interaction's +/// multiplicity column is zero. +/// +/// So the second form names the multiplicity columns. They come from each +/// table's own `bus_interactions()`, read there rather than through the AIR: +/// `&dyn AIR` does not expose the interaction list. KECCAK's eight interactions +/// and KECCAK_RND's fourteen are all `Multiplicity::Column(cols::MU)`, KECCAK_RC's +/// single one likewise, and ECDAS's three are `MU` twice plus `NEXT_OP` once. +/// ECSM's include `cols::k_bit(i)` as well, which is why the blank witness — the +/// stronger of the two — WAS the one used for it. +/// +/// ⚠ THAT WITNESS IS GONE, because the state it named stopped being reachable. +/// `Blank` meant "the generator wrote nothing, so this table has no rows at +/// all". Since the accelerators became counted chips, a table with no rows is +/// not in the proof — there is nothing left to classify. The two tables it used +/// to describe here, ECSM and KECCAK_RND, are no longer on this list, and none +/// of the four that remain can be blank. Kept as a comment rather than a +/// variant nobody constructs: re-add it the moment an always-on table can +/// legitimately be empty again. +enum RowWitness { + /// Padding carries canonical values, so the trace is not blank. The witness + /// is that every column any interaction uses as MULTIPLICITY is zero on + /// every row, which gates every LogUp term off. + GatedOff(&'static [usize]), + /// This workload populates the table; no zero-row claim is made. Checked to + /// be non-blank, so a misclassification here does not pass silently. + Populated, +} + +/// ★ MEASURED: a fixed table with no rows carries `Some(zero)`, never `None`. +/// +/// ## Why this had to be measured +/// +/// `FIXED_TABLE_COUNT` forces a sub-proof for its five tables whatever the +/// workload — ten before #977 moved the six accelerators into `TableCounts` — +/// so a real epoch always carries tables with no real rows. The +/// closure's [`LogUpShape::num_contributing_tables`] is a program CONSTANT, so +/// if such a table reported `None` the count would be workload-dependent and the +/// constant wrong. The LogUp leg closed with this labelled INFERENCE: production +/// rejects any AIR/proof disagreement (`verifier.rs:1238`), and +/// `has_trace_interaction()` is shape, so `None` would make every real epoch +/// unverifiable — therefore it must be `Some`. True, but an argument, and the +/// experiment is cheap. This is the experiment. +/// +/// Note where the damage would have been: a zero `L` is arithmetically inert, so +/// dropping one would not move the SUM. What `None` would break is the arena +/// SCHEMA — a program compiled for `n` contributions fed `n − 1` words — which is +/// why the answer matters to the count and not to the balance. +/// +/// ## What is measured, and against what +/// +/// One REAL continuation epoch — epoch 0 of the LFM fixture guest, built by +/// `Traces::from_image_and_logs` and proved over the production epoch AIR set +/// (`VmAirs` + the epoch-local L2G table) under the real epoch statement, then +/// ACCEPTED by `crate::hash_pin::BlockVerifier::multi_verify_views` against production's own +/// `compute_expected_commit_bus_balance_view`. The acceptance is load-bearing +/// twice over: it is what makes this "what a verifying epoch proof carries" +/// rather than "what some prover run emitted", and it is what runs +/// `verifier.rs:1238`'s presence check over these very sub-proofs. +/// +/// "No rows" is read off the TRACE, not inferred from the workload, and not read +/// back off the contribution being measured — see [`RowWitness`] for the two +/// forms it takes and why one would not do. `FIXED_TABLE_COUNT` keeps the +/// sub-proof either way, and KECCAK_RC is the example since #977: it is one of +/// the five always-on tables, and its trace is padded with `MU` gated off +/// rather than dropped. KECCAK itself is no longer such a table — it is +/// `TableCounts::keccak`, and an epoch that calls no keccak carries no KECCAK +/// sub-proof to measure. +/// +/// ## Which sub-proof is which table +/// +/// Positional, because `VmAirs::new` builds these nine without `.with_name(…)` — +/// `AIR::name()` answers `"unknown"` for every one of them, so there is no name +/// on the proof side to match. The order is `lib.rs`'s own, and `air_refs()` and +/// `air_trace_pairs()` list it identically; that identity is what makes sub-proof +/// `i` this table's proof. It is not taken on trust: each position's sub-proof +/// must report the trace length that position's TRACE built. +/// +/// ## What this test cannot see +/// +/// The row-count cross-check cannot separate two tables of equal height, so +/// swapping (say) KECCAK and ECSM — both four rows here — would relabel two +/// results without failing. It catches the reorderings that change a height, +/// which is every one that could move a populated table into a zero-row slot. +/// +/// Whether a fixed table whose interactions took a CONSTANT multiplicity would +/// answer differently. None does today — every multiplicity in the five zero-row +/// tables is a column, checked by reading their `bus_interactions()` — but such a +/// table would carry a nonzero `L` with no real rows, and this test would report +/// the changed contribution without explaining it. It measures an INTERMEDIATE +/// epoch, so HALT is out of scope (`VmAirs` omits it unless the epoch is final), and one workload, so it says nothing about +/// which tables are unused in general — only what a table with no rows carries. +#[test] +fn a_zero_row_fixed_table_carries_some_zero_not_none() { + use crate::tables::trace_builder::{Traces, build_initial_image_paged}; + use crate::tables::{MaxRowsConfig, bitwise, local_to_global, register}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + use math::field::traits::IsPrimeField; + use stark::proof::view::MultiProofView; + use stark::trace::TraceTable; + + let opts = super::proof_fixture::fixture_options(); + let elf_bytes = super::proof_fixture::read_inner_elf(); + let elf = Elf::load(&elf_bytes).expect("the fixture ELF must load"); + let epoch_size = 1usize << super::proof_fixture::FIXTURE_EPOCH_LOG2; + + // ---- epoch 0, built exactly as `prove_continuation` builds it ---- + let mut executor = Executor::new(&elf, vec![]).expect("executor"); + let image = build_initial_image_paged(&elf, &[]); + let register_init = register::register_init_from_entry_point(elf.entry_point); + let logs = executor + .resume_with_limit(epoch_size) + .expect("resume") + .expect("the guest runs at least one epoch") + .to_vec(); + let is_final = executor.pc() == 0; + assert!( + !is_final, + "wanted an INTERMEDIATE epoch (every fixed table but HALT), but the \ + guest finished inside one epoch of {epoch_size} cycles" + ); + + let mut traces = Traces::from_image_and_logs( + &elf, + &image, + ®ister_init, + &logs, + &MaxRowsConfig::default(), + &[], + is_final, + true, + ) + .expect("the epoch trace must build"); + + let label = local_to_global::epoch_label(0); + let mut provenance = + local_to_global::genesis_provenance(image.iter().map(|(a, v)| (a, v as u64))); + let boundary = + local_to_global::epoch_boundary(&mut provenance, label, &traces.touched_memory_cells); + // `prove_epoch`'s first act: the L2G table's range-check lookups must be + // counted into BITWISE, or the epoch's own bus does not close. + bitwise::update_multiplicities( + &mut traces.bitwise, + &local_to_global::collect_bitwise_from_l2g(&boundary), + ); + + // ---- the trace-side census, taken before proving borrows the traces ---- + let all_main_zero = |t: &TraceTable| -> bool { + (0..t.main_table.height).all(|r| t.main_table.get_row(r).iter().all(|v| *v == FE::zero())) + }; + let columns_zero = |t: &TraceTable, cols: &[usize]| -> bool { + (0..t.main_table.height).all(|r| { + let row = t.main_table.get_row(r); + cols.iter().all(|c| row[*c] == FE::zero()) + }) + }; + // `(name, rows, has_no_bus_rows)`. + // + // ONE list, name and trace and witness together, deliberately: a version + // that kept the names in a separate constant and zipped them onto the + // traces passed with two names swapped — the swap moved only the label, so + // the row-count cross-check below still compared the right trace against the + // right sub-proof and saw nothing wrong. Merged, a reordering moves the + // TRACE too, which that cross-check does catch. + // + // ★ The list is every always-on table an INTERMEDIATE epoch carries — + // `FIXED_TABLE_COUNT` less HALT, which `VmAirs::air_refs` includes only on a + // final epoch — **in `air_refs`' own order**, which the position-by-position + // trace-length check below depends on. It carried NINE entries while the + // constant was 11 and then 12: HINT and #903's BLAKE3 were never added, and + // the shortfall was invisible because a fixture bug stopped this test + // reaching its own assertion. A hand-written census that does not pin its + // own length against the constant it is a census OF goes stale exactly that + // way again, so the length is now asserted below. + // + // BLAKE3 left this list in the other direction: it is no longer always-on + // but counted in `TableCounts::blake3`, and this epoch does not use it, so + // it contributes no sub-proof to census. + // + // ★ SEVEN MORE LEFT THE SAME WAY, and the list is now four. COMMIT, KECCAK, + // KECCAK_RND, ECSM, ECDAS and HINT became counted chips (`FIXED_TABLE_COUNT` + // 11 → 5), so a chip this epoch does not reach carries no sub-proof at all + // rather than a padded one. What that costs this census is the per-chip + // witness classification above — an ECSM the epoch never called used to be + // asserted all-zero HERE. It is not replaced with a length check against + // `table_counts`, because `table_counts` is built FROM these vectors and + // such a check could not fail. What still binds all seven is the sub-proof + // identity below: `view.len()` must equal this census plus `total()` plus + // one L2G, and `total()` is where every counted chip now appears. + let census: Vec<(&str, usize, bool)> = { + use crate::tables::keccak_rc; + let fixed: [(&str, &TraceTable, RowWitness); crate::FIXED_TABLE_COUNT - 1] = [ + ("BITWISE", &traces.bitwise, RowWitness::Populated), + ("DECODE", &traces.decode, RowWitness::Populated), + ( + "KECCAK_RC", + &traces.keccak_rc, + RowWitness::GatedOff(&[keccak_rc::cols::MU]), + ), + ("REGISTER", &traces.register, RowWitness::Populated), + ]; + fixed + .into_iter() + .map(|(name, t, witness)| { + let no_bus_rows = match witness { + RowWitness::GatedOff(cols) => { + assert!( + columns_zero(t, cols), + "{name} was expected to have no rows on any bus, but \ + one of its multiplicity columns {cols:?} is nonzero" + ); + true + } + RowWitness::Populated => { + assert!( + !all_main_zero(t), + "{name} was classified as populated by this workload \ + but its main trace is entirely zero — the \ + classification, not the measurement, is wrong" + ); + false + } + }; + (name, t.num_rows(), no_bus_rows) + }) + .collect() + }; + + // ---- prove it, over the production epoch AIR set ---- + let reg_fini = register::fini_from_trace(&traces.register); + let table_counts = traces.table_counts(); + let public_output = traces.public_output_bytes.clone(); + let runtime_page_ranges = traces.runtime_page_ranges(); + + let airs = crate::VmAirs::new( + &elf, + &opts, + false, + &[], + &table_counts, + &crate::cpu_heights_from_traces(&traces), + None, + is_final, + None, + crate::RegisterPreprocessed { + commitment: register::compute_precomputed_commitment(&opts, ®ister_init, ®_fini), + init: ®ister_init, + fini: ®_fini, + }, + ); + let l2g_air = crate::continuation::l2g_memory_air(&opts, label); + let mut l2g_trace = local_to_global::generate_local_to_global_trace(&boundary); + + // The real epoch statement, so the challenges are the ones a production + // epoch proof is bound to. + let seed = || { + let mut t = crate::hash_pin::block_transcript(&[]); + crate::statement::absorb_statement( + &mut t, + crate::statement::StatementKind::ContinuationEpoch { + epoch_label: label, + // HALT's presence IS the role; taking it from the AIR set keeps + // the seed and the proof from stating different ones. + is_final: airs.include_halt, + }, + &elf_bytes, + &public_output, + &table_counts, + 0, + &runtime_page_ranges, + opts.fri_final_poly_log_degree, + ); + t + }; + + let proof = { + let mut pairs = airs.air_trace_pairs(&mut traces); + pairs.push((&l2g_air, &mut l2g_trace, &())); + crate::test_utils::multi_prove_ram(pairs, &mut seed()).expect("the epoch must prove") + }; + + let refs = { + let mut r = airs.air_refs(); + r.push(&l2g_air); + r + }; + let view = MultiProofView::Owned(&proof); + assert_eq!( + census.len(), + crate::FIXED_TABLE_COUNT - 1, + "the census must name every always-on table an intermediate epoch \ + carries (all but HALT), or the sub-proof identity below is satisfied \ + by an undercount on both sides" + ); + assert_eq!( + view.len(), + census.len() + table_counts.total().expect("the fixture's counts sum") + 1, + "an intermediate epoch is {} fixed tables, the chunked families, and \ + one L2G_MEMORY", + crate::FIXED_TABLE_COUNT - 1 + ); + assert_eq!(refs.len(), view.len(), "one AIR per sub-proof"); + + // ---- production must ACCEPT it, or nothing below is about a real proof ---- + let expected = crate::compute_expected_commit_bus_balance_view( + &refs, + view, + &public_output, + register_init[register::X254_INDEX] as u64, + &mut seed(), + ) + .expect("the COMMIT-bus target must exist"); + assert!( + crate::hash_pin::BlockVerifier::multi_verify_views(&refs, view, &mut seed(), &expected), + "production must ACCEPT this epoch proof — the measurement is about what \ + a VERIFYING proof carries, and this is also the run of \ + verifier.rs:1238's presence check" + ); + + // ---- THE MEASUREMENT ---- + println!( + "\nreal continuation epoch (intermediate, {} sub-proofs), fixed tables:\n\ + \x20 {:<11} {:>9} {:>10} {:>5} {:>5} {:>4} contribution", + view.len(), + "table", + "rows", + "proof_len", + "iact", + "bpi", + "zero" + ); + let mut zero_row = Vec::new(); + let mut with_rows = Vec::new(); + for (i, (name, rows, no_rows)) in census.iter().enumerate() { + let sp = view.get(i); + let interacts = refs[i].has_trace_interaction(); + let present = sp.has_bus_public_inputs(); + let contribution = sp.bus_table_contribution(); + assert_eq!( + sp.trace_length(), + *rows, + "position {i} was labelled {name} but proved a trace of {} rows, not \ + the {rows} that table built — the census order no longer matches \ + air_refs()/air_trace_pairs()", + sp.trace_length() + ); + assert_eq!( + interacts, present, + "{name}: production rejects any disagreement between the AIR's \ + declared interactions and the proof's bus public inputs, in both \ + directions (verifier.rs:1238 and :1244)" + ); + println!( + "\x20 {:<11} {:>9} {:>10} {:>5} {:>5} {:>4} {}", + name, + rows, + sp.trace_length(), + interacts, + present, + contribution.as_ref().is_some_and(|c| *c == FEE::zero()), + match &contribution { + None => "None".to_string(), + Some(c) => format!( + "Some({:?})", + c.value() + .iter() + .map(|l| Gl::canonical(l.value())) + .collect::>() + ), + } + ); + if *no_rows { + // THE ANSWER. A zero-row fixed table is still a contributing table. + assert!( + present, + "{name} has no rows on any bus and must STILL carry bus public \ + inputs — a None here would make num_contributing_tables \ + workload-dependent, and the closure's program constant wrong" + ); + assert_eq!( + contribution, + Some(FEE::zero()), + "{name} has no rows on any bus, so every LogUp term is gated to \ + zero and its L must be exactly zero" + ); + zero_row.push(*name); + } else { + with_rows.push((*name, contribution)); + } + } + + // Non-vacuity, both directions: there IS a zero-row fixed table in a real + // epoch, and the observation distinguishes it from a populated one. Without + // the second half, "every zero-row table reports Some(zero)" could hold + // because every table reports Some(zero). + assert!( + !zero_row.is_empty(), + "this epoch has no zero-row fixed table, so it cannot settle the \ + question — pick a guest that leaves one unused" + ); + assert!( + census + .iter() + .enumerate() + .all(|(i, _)| view.get(i).has_bus_public_inputs()), + "every fixed table of an epoch is a contributing table, populated or not" + ); + assert!( + with_rows.iter().any(|(_, c)| *c != Some(FEE::zero())), + "no fixed table carries a NONZERO contribution, so Some(zero) is not a \ + distinguishing observation on this epoch: {with_rows:?}" + ); + println!( + "\x20 ANSWER: Some(zero), not None. {} zero-row fixed tables {:?}, each \ + Some(zero); {} populated, {} of them nonzero.", + zero_row.len(), + zero_row, + with_rows.len(), + with_rows + .iter() + .filter(|(_, c)| *c != Some(FEE::zero())) + .count() + ); + + // ---- the converse, also measured: None would be REJECTED ---- + // The inference this experiment replaces ran the other way — a zero-row + // table cannot report None, because production checks presence against + // has_trace_interaction() before anything else (verifier.rs:1238), so a + // None would make every real epoch unverifiable. That is now a run: strip + // the bus public inputs off a zero-row sub-proof and watch this very proof + // stop verifying. Only the `is_some` direction can be tested on an epoch — + // every sub-proof this epoch carries declares interactions, so :1244's + // converse has no subject here. It reads "all 16" since #977 (`892c7d1bc`) + // rather than all 25, and the count is deliberately not spelled: the loop + // walks `census`, so a literal here would be a second, driftable copy of a + // length the loop already has. + for (i, (name, _, no_rows)) in census.iter().enumerate() { + if !no_rows { + continue; + } + let mut tampered = proof.clone(); + tampered.proofs[i].bus_public_inputs = None; + assert!( + !crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(&tampered), + &mut seed(), + &expected, + ), + "{name} has no rows, but dropping its bus public inputs must still \ + be REJECTED — that rejection is why Some(zero) is forced rather \ + than merely observed" + ); + } + + // ---- and the closure itself, over the REAL epoch's whole table set ---- + // The handoff's other open item: every earlier fixture is two or three + // tables, so the SUM was exercised but its LENGTH was not. + let contributions: Vec = (0..view.len()) + .filter(|i| refs[*i].has_trace_interaction()) + .map(|i| { + view.get(i) + .bus_table_contribution() + .expect("presence was just checked against the AIR") + }) + .collect(); + let shape = LogUpShape { + num_contributing_tables: contributions.len(), + num_output_bytes: public_output.len(), + }; + let (z, alpha) = crate::replay_transcript_phase_a_view(&refs, view, &mut seed()) + .expect("every preprocessed table has a root for its layout"); + + let n_tables = contributions.len() as u32; + let n_bytes = public_output.len() as u32; + let mut b = LfmBuilder::new(); + // Only the cells the gadget reads may be declared: an unread arena word is a + // compile error, and an empty output makes the target a constant that reads + // neither z, alpha, start nor any byte. + let head = if n_bytes == 0 { 0 } else { 3 + n_bytes }; + let arena = b.declare_arena(head + n_tables); + let target = if n_bytes == 0 { + b.ext_const(&FEE::zero()) + } else { + let z_cell = b.hint_word(arena, 0).as_ext(); + let alpha_cell = b.hint_word(arena, 1).as_ext(); + let start_cell = b.hint_felt(arena, 2); + let byte_cells: Vec<_> = (0..n_bytes).map(|i| b.hint_felt(arena, 3 + i)).collect(); + emit_commit_bus_target(&mut b, &shape, z_cell, alpha_cell, start_cell, &byte_cells) + }; + let contrib_cells: Vec<_> = (0..n_tables) + .map(|i| b.hint_word(arena, head + i).as_ext()) + .collect(); + let total = emit_bus_closure(&mut b, &shape, &contrib_cells, target); + b.public(total.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the epoch closure program is admissible"); + + let mut words: Vec = Vec::new(); + if n_bytes > 0 { + words.push(ext_word(&z)); + words.push(ext_word(&alpha)); + words.push(base_word(FE::from( + register_init[register::X254_INDEX] as u64, + ))); + words.extend(public_output.iter().map(|v| base_word(FE::from(*v as u64)))); + } + words.extend(contributions.iter().map(ext_word)); + let exec = execute(&program, std::slice::from_ref(&words), &TestPermutation) + .expect("a real epoch's LogUp bus must close in the machine"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + expected, + "the machine's published total must be production's own expected balance" + ); + + // Falsification: move any one contribution, in any lane. Every zero-row + // table is in here too, so this is also the check that a Some(zero) term is + // a real summand and not a no-op the emitter could drop. + let mut vectors = 0usize; + for table in 0..contributions.len() { + for lane in 0..3usize { + let mut arenas = vec![words.clone()]; + arenas[0][head as usize + table][lane] += FE::one(); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "table {table} lane {lane}: a moved contribution must not close" + ); + vectors += 1; + } + } + println!( + "\x20 the closure also runs on the real epoch: {} contributing tables of \ + {} sub-proofs, {} output bytes, {vectors} single-lane moves rejected\n", + contributions.len(), + view.len(), + public_output.len() + ); +} diff --git a/prover/src/lfm/machine_tests.rs b/prover/src/lfm/machine_tests.rs new file mode 100644 index 000000000..708c9ae58 --- /dev/null +++ b/prover/src/lfm/machine_tests.rs @@ -0,0 +1,5119 @@ +//! Milestone B end-to-end: the machine proves a trivial program; valid +//! accepts, tampered variants reject, and the registry drift test pins the +//! program's identity (recompute-and-compare, the static-commitments policy). + +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use crate::tables::types::FE; + +use super::airs::BLAKE3_SLOT; +use super::executor::LfmExecError; +use super::fixture::{self, bump_lane0, fixture_prove}; +use super::programs::{fri_toy_program, trivial_program, trivial_program_source}; +use super::proof::{LfmProveError, lfm_prove, lfm_verify, take_prove_split}; +use super::registry::{LfmProgramKind, LfmRegistryError, build_artifacts, resolve}; +use super::validator::validate; +use super::word::LfmWord; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +#[test] +fn trivial_program_is_admissible() { + let program = trivial_program(); + validate(&program).expect("the registered program must pass admission"); +} + +#[test] +fn trivial_program_proves_and_verifies() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + let ok = lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("registry entry exists"); + assert!(ok, "honest machine proof must verify"); +} + +/// ★ THE GATE ON THE PROVE SPLIT — that it is recorded, that it covers the +/// function rather than a corner of it, and that it is consumed. +/// +/// Every one of the three assertions below can fail, and each fails on a +/// different mistake. Drop the recording, or move it above one of the `?`s, +/// and the `Some` goes. Time two statements instead of three and the coverage +/// floor goes. Leave the cell set and the second take stops being `None` — +/// which is the one that matters in a run with a cache, because a stage that +/// LOADED would then be reported with the previous stage's numbers, and a +/// figure attributed to the wrong stage is worse than no figure. +#[test] +fn the_prove_split_is_recorded_covers_the_prove_and_is_consumed() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + + // Anything an earlier prove on this thread may have left behind. + let _ = take_prove_split(); + + let wall = std::time::Instant::now(); + let _ = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + let wall = wall.elapsed().as_secs_f64(); + + let split = take_prove_split().expect("a prove that returned Ok must leave its split"); + assert!( + split.execute > 0.0 && split.fill > 0.0 && split.multi_prove > 0.0, + "every phase must have a positive span: {split:?}" + ); + // ⓘ The wait belongs in the sum: `multi_prove` is reported NET of it, so + // without this term the coverage floor would start failing the moment a + // sibling held the card — the one regime it most needs to hold in. + let sum = split.execute + split.fill + split.multi_prove + split.permit_wait; + assert!( + sum <= wall, + "three disjoint spans inside the call cannot exceed the call: {sum} > {wall}" + ); + assert!( + sum >= 0.7 * wall, + "the three spans must cover the prove, not a corner of it: {sum} of {wall}" + ); + assert!( + take_prove_split().is_none(), + "the split is taken once, not once per read — a second reader would \ + attribute this prove's numbers to whatever ran next" + ); +} + +#[test] +fn tampered_claimed_public_word_rejects() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + + let mut claimed = proved.public_words.clone(); + claimed[0].1[0] = &claimed[0].1[0] + FE::from(1u64); + let ok = lfm_verify(LfmProgramKind::TrivialV0, &proved.proof, &claimed, &opts) + .expect("registry entry exists"); + assert!(!ok, "a tampered claimed public word must reject"); +} + +#[test] +fn different_arena_values_change_the_public_output_not_the_program() { + // Same program identity, different hints: proves and verifies against its + // own (different) public words. + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + + let other: Vec> = vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(7_777 * (i + 1) + j as u64))) + .collect(), + ]; + let a = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove a"); + let b = lfm_prove(&program, &artifacts, &other, &opts).expect("prove b"); + assert_ne!(a.public_words, b.public_words); + assert!( + lfm_verify(LfmProgramKind::TrivialV0, &b.proof, &b.public_words, &opts).expect("entry") + ); + // Cross-claiming rejects: proof b with proof a's public words. + assert!( + !lfm_verify(LfmProgramKind::TrivialV0, &b.proof, &a.public_words, &opts).expect("entry") + ); +} + +#[test] +fn registry_miss_is_a_hard_error() { + let opts = GoldilocksCubicProofOptions::with_blowup(8).expect("options"); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + let err = lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .unwrap_err(); + assert_eq!( + err, + LfmRegistryError::UnknownProgram { + kind: LfmProgramKind::TrivialV0, + blowup_factor: 8 + }, + "no registry entry ⇒ hard error, never a fallback" + ); +} + +/// The registry drift test — the LFM analogue of +/// `static_commitments_tests.rs`. A failure here means the trivial program, +/// a chip layout, the commit pipeline or the digest changed: investigate, +/// never re-bless. +#[test] +fn registry_drift_trivial_v0_blowup2() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = resolve(LfmProgramKind::TrivialV0, 2).expect("TrivialV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +#[test] +fn trivial_program_source_is_deterministic() { + let a = trivial_program_source(); + let b = trivial_program_source(); + assert_eq!(a.num_addrs, b.num_addrs); + assert_eq!(a.instrs.len(), b.instrs.len()); +} + +// ======================= Milestone C: the FRI verifier ======================= + +fn fri_arenas(proof: &fixture::FriToyProof) -> Vec> { + vec![proof.commitments.clone(), proof.openings.clone()] +} + +#[test] +fn fri_toy_program_is_admissible() { + validate(&fri_toy_program()).expect("the FRI verifier program must pass admission"); +} + +/// The Milestone-C headline: the machine verifies a structurally real FRI +/// commitment-opening proof (sponge transcript, Merkle-authenticated +/// openings, α-combination, two unnormalized folds, terminal check) and the +/// resulting machine proof verifies against the registry. +#[test] +fn machine_verifies_fixture_fri_proof_end_to_end() { + let opts = options(); + let program = fri_toy_program(); + let artifacts = build_artifacts(&program, &opts); + let inner = fixture_prove(); + let proved = + lfm_prove(&program, &artifacts, &fri_arenas(&inner), &opts).expect("machine accepts"); + // The attested public output is the inner proof's identity: both roots. + assert_eq!(proved.public_words[0].1, inner.commitments[0]); + assert_eq!(proved.public_words[1].1, inner.commitments[1]); + assert!( + lfm_verify( + LfmProgramKind::FriToyV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("FriToyV0 is registered"), + "the machine proof of FRI verification must verify" + ); +} + +/// Every tamper vector must make the verification program *unprovable* (the +/// executor hits the same failed assert the AIR's division constraint makes +/// unsatisfiable). +#[test] +fn machine_rejects_tampered_fri_proofs() { + let opts = options(); + let program = fri_toy_program(); + let artifacts = build_artifacts(&program, &opts); + let honest = fixture_prove(); + + let expect_reject = |arenas: Vec>, what: &str| match lfm_prove( + &program, &artifacts, &arenas, &opts, + ) { + Err(LfmProveError::Exec(LfmExecError::DivByZero { .. })) => {} + other => panic!( + "{what}: expected a failed in-machine assert, got {:?}", + other.map(|_| "accepted") + ), + }; + + // (a) a tampered opened row value breaks its Merkle path. + let mut t = fri_arenas(&honest); + t[1][0] = bump_lane0(&t[1][0]); + expect_reject(t, "tampered opened row"); + + // (b) a tampered sibling digest breaks the walk. + let mut t = fri_arenas(&honest); + t[1][2] = bump_lane0(&t[1][2]); + expect_reject(t, "tampered sibling"); + + // (c) a tampered main root diverges the transcript: different queries, + // openings no longer match. + let mut t = fri_arenas(&honest); + t[0][0] = bump_lane0(&t[0][0]); + expect_reject(t, "tampered main root"); + + // (d) a tampered terminal coefficient fails the terminal check. + let mut t = fri_arenas(&honest); + t[0][2] = bump_lane0(&t[0][2]); + expect_reject(t, "tampered terminal coefficient"); + + // (e) a tampered L1 opened value fails fold-consistency or its path. + let mut t = fri_arenas(&honest); + t[1][12] = bump_lane0(&t[1][12]); + expect_reject(t, "tampered layer-1 opening"); +} + +#[test] +fn registry_drift_fri_toy_v0_blowup2() { + let opts = options(); + let program = fri_toy_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = resolve(LfmProgramKind::FriToyV0, 2).expect("FriToyV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +/// The kill-risk-3 instrument on the first real verification program. +#[test] +fn fri_toy_cell_counts() { + let program = fri_toy_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "FriToyV0: {} instructions, {} addresses, {} main value cells, {} aux ext elements", + program.instrs.len(), + program.num_addrs, + main, + aux + ); + assert!(main > 0 && aux > 0); +} + +// ===================== R1b: keccak-f[1600] in the machine ===================== + +use super::compiler::LfmProgram; +use super::keccak_adapter; +use super::layout::keccak as klayout; +use super::programs::{keccak_chain_program, keccak_chain_program_source}; +use super::proof::prove_traces; +use super::registry::LfmArtifacts; +use super::trace::{LfmTraces, build_traces_with_hasher}; +use super::validator::LfmViolation; +use crate::lfm::chips::keccak as kchip; +use crate::tables::types::VmTable; +use stark::prover::ProvingError; + +/// A keccak state derived from `seed`, in the machine's word form. +fn keccak_state(seed: u64) -> [u64; 25] { + core::array::from_fn(|i| { + seed.wrapping_mul(i as u64 + 1) + .wrapping_add(0x9E37_79B9_7F4A_7C15) + ^ 0x0123_4567_89AB_CDEF + }) +} + +fn keccak_arenas(seed: u64) -> Vec> { + vec![keccak_adapter::state_to_words(&keccak_state(seed)).to_vec()] +} + +type KeccakChainProof = super::proof::LfmStarkProof; + +/// Execute + build traces, let the caller corrupt them, then prove. +fn prove_keccak_chain_with_tamper( + program: &LfmProgram, + artifacts: &LfmArtifacts, + seed: u64, + mutate: impl FnOnce(&mut LfmTraces), +) -> Result<(KeccakChainProof, Vec<(u32, LfmWord)>), ProvingError> { + let opts = options(); + let exec = + super::executor::execute(program, &keccak_arenas(seed), &super::hash::TestPermutation) + .expect("honest execution"); + let mut traces = build_traces_with_hasher(program, &exec.records, artifacts.hasher); + mutate(&mut traces); + let proof = prove_traces(artifacts, &mut traces, &exec.public_words, &opts)?; + Ok((proof, exec.public_words)) +} + +#[test] +fn keccak_chain_program_is_admissible() { + validate(&keccak_chain_program()).expect("the keccak-chain program must pass admission"); +} + +#[test] +fn keccak_chain_source_is_deterministic() { + let a = keccak_chain_program_source(); + let b = keccak_chain_program_source(); + assert_eq!(a.num_addrs, b.num_addrs); + assert_eq!(a.instrs.len(), b.instrs.len()); +} + +/// The R1b headline: the machine proves two *chained* real `keccak-f[1600]` +/// permutations, with the state bound to `LfmMem` words and the permutation +/// itself discharged by the unchanged production `KECCAK_RND` / `KECCAK_RC` / +/// `BITWISE` chips. +#[test] +fn keccak_chain_proves_and_verifies() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &keccak_arenas(7), &opts).expect("machine proves"); + + // Host-side reference: the same two permutations, same word convention. + let once = keccak_adapter::permute(keccak_state(7)); + let twice = keccak_adapter::permute(once); + let once_words = keccak_adapter::state_to_words(&once); + let twice_words = keccak_adapter::state_to_words(&twice); + assert_eq!(proved.public_words[0].1, once_words[0], "first permutation"); + assert_eq!(proved.public_words[1].1, twice_words[0], "second, word 0"); + assert_eq!(proved.public_words[2].1, twice_words[1], "second, word 1"); + + assert!( + lfm_verify( + LfmProgramKind::KeccakChainV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("KeccakChainV0 is registered"), + "the machine proof of two chained keccak permutations must verify" + ); +} + +/// Flipping one output byte — i.e. one quarter of one `u32` half — must break +/// the proof. The byte columns feed both the `Keccak` reply token and, through +/// the `Linear` half recomposition, the `LfmMem` word the next instruction +/// reads, so either bus catches it. +#[test] +fn tampered_keccak_output_half_rejects() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let (proof, public) = prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + let col = kchip::cols::out_byte(3, 2); + let old = t.keccak.main_table.get_row(0)[col]; + t.keccak.main_table.set_fe(0, col, old + FE::from(1u64)); + }) + .expect("the adapter has no constraints, so the prover accepts"); + + assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "a flipped output byte must reject" + ); +} + +/// Flipping an input byte likewise rejects: the request token no longer matches +/// the round chip's first receive. +#[test] +fn tampered_keccak_input_half_rejects() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let (proof, public) = prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + let col = kchip::cols::state_byte(11, 5); + let old = t.keccak.main_table.get_row(1)[col]; + t.keccak.main_table.set_fe(1, col, old + FE::from(1u64)); + }) + .expect("locally consistent"); + + assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "a flipped input byte must reject" + ); +} + +/// CLOSES THE R1a HAZARD. +/// +/// `keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` +/// exhibits a live forgery against the raw keccak family: given two rows +/// sharing a tag, swapping their output states leaves the `Keccak` bus +/// balanced, so the verifier accepts two permutations neither of which is +/// genuine. Nothing but the tag binds a request token to its reply. +/// +/// Moving the tag into the preprocessed column group closes it, in three legs +/// asserted below: +/// 1. the compiled program's keccak rows carry *distinct* tags; +/// 2. swapping two rows' output states now REJECTS (it accepted in R1a); +/// 3. the prover cannot repair leg 2 by colliding the tags, because they are +/// preprocessed — editing them fails the recommit before a proof exists. +#[test] +fn preprocessed_tags_close_the_output_swap_hazard() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let group = &program.groups.keccak; + assert_eq!(group.real_rows, 2, "the chain program has two keccak rows"); + + // Leg 1: distinct tags, assigned by the compiler as row ordinals. + let tag = |row: usize| { + ( + *group.at(row, klayout::TAG_LO), + *group.at(row, klayout::TAG_HI), + ) + }; + assert_ne!(tag(0), tag(1), "keccak tags must be distinct"); + + // Leg 2: the R1a forgery, replayed — swap the two rows' 200 output bytes. + let (proof, public) = prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + for col in kchip::cols::OUT..kchip::cols::NUM_COLUMNS { + let a = t.keccak.main_table.get_row(0)[col]; + let b = t.keccak.main_table.get_row(1)[col]; + t.keccak.main_table.set_fe(0, col, b); + t.keccak.main_table.set_fe(1, col, a); + } + }) + .expect("locally consistent"); + assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "with distinct tags the swapped outputs must no longer balance" + ); + + // Leg 3: colliding the tags is not available to the prover. The TAG columns + // are PREPROCESSED — pinned by the group's precomputed commitment — so a + // prover cannot obtain a VERIFYING proof with the tags rewritten. Copying + // row 0's tag over row 1's makes the trace's leading columns disagree with + // the committed group. + // + // Where the failure lands is precomputed-tree-cache-dependent (the cache is + // keyed by the expected root): on a cache MISS the prover recomputes the + // preprocessed tree from the tampered columns and refuses with + // `PrecomputedCommitmentMismatch`; on a warm cache HIT it commits the pinned + // tree without re-checking (so it does NOT refuse at prove time) and the + // tampered proof is caught at VERIFY instead. Either way the forgery fails — + // which is the obligation this leg exists to pin. + match prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + let lo = t.keccak.main_table.get_row(0)[klayout::TAG_LO]; + let hi = t.keccak.main_table.get_row(0)[klayout::TAG_HI]; + t.keccak.main_table.set_fe(1, klayout::TAG_LO, lo); + t.keccak.main_table.set_fe(1, klayout::TAG_HI, hi); + }) { + Err(err) => assert!( + matches!(err, ProvingError::PrecomputedCommitmentMismatch), + "expected a preprocessed recommit failure, got {err:?}" + ), + Ok((proof, public)) => assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "a tag-rewrite tamper must not yield a verifying proof" + ), + } +} + +/// The registrar's independent gate on the same obligation: even if a future +/// compiler change stopped assigning distinct tags, admission would catch it. +#[test] +fn duplicate_keccak_tags_fail_admission() { + let mut program = keccak_chain_program(); + let lo = *program.groups.keccak.at(0, klayout::TAG_LO); + let hi = *program.groups.keccak.at(0, klayout::TAG_HI); + program.groups.keccak.set(1, klayout::TAG_LO, lo); + program.groups.keccak.set(1, klayout::TAG_HI, hi); + assert_eq!( + validate(&program), + Err(LfmViolation::DuplicateKeccakTag { tag: (1, 0) }), + "duplicate keccak tags must fail admission" + ); +} + +/// A keccak lane is a `u64`, but a felt lane carrying a half must be a `u32`. +/// A hinted word above that bound is caught by the executor — on the AIR side +/// no such value exists, since each half is a fixed combination of four +/// BITWISE-constrained bytes. +#[test] +fn keccak_rejects_non_u32_half() { + let program = keccak_chain_program(); + let mut arenas = keccak_arenas(7); + arenas[0][0][0] = FE::from(1u64 << 32); + assert_eq!( + super::executor::execute(&program, &arenas, &super::hash::TestPermutation).unwrap_err(), + LfmExecError::NotU32Half { addr: 0, lane: 0 } + ); +} + +/// The state is 50 halves in 52 word slots; the two spare slots are pinned to +/// zero as bus tuple constants, so a nonzero one is unprovable. +#[test] +fn keccak_rejects_nonzero_spare_lane() { + let program = keccak_chain_program(); + let mut arenas = keccak_arenas(7); + let last = klayout::NUM_WORDS - 1; + arenas[0][last][2] = FE::from(1u64); + assert_eq!( + super::executor::execute(&program, &arenas, &super::hash::TestPermutation).unwrap_err(), + LfmExecError::KeccakSpareLaneNonZero { + addr: last as u64, + lane: 2 + } + ); +} + +#[test] +fn registry_drift_keccak_chain_v0_blowup2() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = + resolve(LfmProgramKind::KeccakChainV0, 2).expect("KeccakChainV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +/// The kill-risk-3 instrument with the keccak family in the set. +#[test] +fn keccak_chain_cell_counts() { + let program = keccak_chain_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "KeccakChainV0: {} instructions, {} main value cells, {} aux ext elements", + program.instrs.len(), + main, + aux + ); + assert!(main > 0 && aux > 0); +} + +// ==================== R1c: keccak256 over byte streams ==================== + +use super::keccak_host; +use super::programs::{KECCAK_SPONGE_LEN, keccak_sponge_program}; +use super::proof::verify_against; + +/// Reference messages. Together they cover: the empty string (padding only), +/// a short message, the exact rate boundary, one byte either side of it, and +/// two lengths whose final `u32` half mixes message bytes with padding. +fn reference_messages() -> Vec> { + let lens = [0usize, 1, 4, 135, 136, 137, KECCAK_SPONGE_LEN, 272]; + lens.iter() + .map(|&n| { + (0..n) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect() + }) + .collect() +} + +fn sponge_arenas(msg: &[u8]) -> Vec> { + let halves = keccak_host::pack_stream(msg); + keccak_host::assert_high_bytes_zero(&halves, msg.len()); + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +/// The 32 bytes a published digest stands for, at whatever width it was +/// published. +/// +/// A BYTE digest is two words read as eight little-endian `u32` halves: byte +/// `j` is byte `j % 4` of half `j / 4`, and half `h` is lane `h % 4` of word +/// `h / 4`. An ALGEBRAIC digest is ONE word of four canonical felts, and its 32 +/// bytes are the backend's own serialisation of them. +/// +/// ⛔ **The discriminator is the slice's own length, NOT +/// `WrapHash::production()`, and that distinction is the whole point.** This +/// helper's callers mix two kinds of program: ones that pin a byte hash on their +/// own builder (`keccak_sponge_program`, `blake3_sponge_program` — always two +/// words, on every branch, because their identity is registry-pinned) and ones +/// that follow the configuration (`merkle_opening_program` — one word on an +/// algebraic arm). Branching on the global configuration would render the first +/// group wrong on an algebraic branch, which is the same scope error as reading +/// a root's width from the configuration instead of from the root. +fn digest_bytes(public: &[(u32, LfmWord)]) -> [u8; 32] { + use math::field::traits::IsPrimeField; + if public.len() == 1 { + return super::algebraic_commit::digest_to_commitment(&public[0].1); + } + assert_eq!( + public.len(), + 2, + "a digest is one algebraic word or two byte words" + ); + let mut out = [0u8; 32]; + for h in 0..8 { + let lane = public[h / 4].1[h % 4]; + let half = crate::tables::types::GoldilocksField::canonical(lane.value()) as u32; + out[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + out +} + +/// Bit-exactness against the production hasher, execute-only (fast): every +/// reference length must reproduce `PlatformKeccak256` byte for byte. +#[test] +fn keccak256_matches_platform_hasher() { + for msg in reference_messages() { + let program = keccak_sponge_program(msg.len()); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("len {}: execution failed: {e:?}", msg.len())); + assert_eq!( + digest_bytes(&exec.public_words), + keccak_host::keccak256(&msg), + "keccak256 mismatch at len {}", + msg.len() + ); + } +} + +#[test] +fn keccak_sponge_program_is_admissible() { + for msg in reference_messages() { + validate(&keccak_sponge_program(msg.len())) + .unwrap_or_else(|e| panic!("len {} must pass admission: {e:?}", msg.len())); + } +} + +/// The R1c headline: the machine PROVES keccak256 of real byte streams and the +/// proofs verify, with the digest matching `PlatformKeccak256` byte for byte. +/// +/// The four lengths cover the shapes that differ structurally: padding-only +/// (empty), a single block whose last half mixes message and padding bytes, +/// a multi-block message crossing the rate boundary, and an exact multiple of +/// the rate — which `pad10*1` grows by a whole extra block. +#[test] +fn keccak_sponge_reference_lengths_prove_and_verify() { + let opts = options(); + for len in [0usize, 135, KECCAK_SPONGE_LEN, 272] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(len); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts) + .unwrap_or_else(|e| panic!("len {len}: prove failed: {e:?}")); + assert_eq!( + digest_bytes(&proved.public_words), + keccak_host::keccak256(&msg), + "len {len}: digest must match the production hasher" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "len {len}: the machine proof of keccak256 must verify" + ); + } +} + +/// The registered length, through the full registry-resolving verify path. +#[test] +fn keccak_sponge_proves_and_verifies() { + let opts = options(); + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + assert_eq!( + digest_bytes(&proved.public_words), + keccak_host::keccak256(&msg) + ); + assert!( + lfm_verify( + LfmProgramKind::KeccakSpongeV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("KeccakSpongeV0 is registered"), + "the registered keccak256 program must verify" + ); +} + +/// Claiming the honest digest for a message whose stream was altered must +/// reject: the absorbed block differs, so the sponge produces a different +/// digest and the claimed public words no longer match the proof. +#[test] +fn tampered_stream_half_rejects() { + let opts = options(); + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + let mut tampered = sponge_arenas(&msg); + tampered[0][3][0] = &tampered[0][3][0] + FE::from(1u64); + let forged = lfm_prove(&program, &artifacts, &tampered, &opts).expect("prove"); + + assert_ne!( + forged.public_words, honest.public_words, + "a changed stream half must change the digest" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "claiming the honest digest for a tampered stream must reject" + ); +} + +/// An absorb row's XOR is pinned by BITWISE lookups, so corrupting the +/// permutation input the family sees — without touching the state read from +/// memory — must reject. +#[test] +fn tampered_absorb_xor_rejects() { + let opts = options(); + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + let mut traces = build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + // Rate byte 5 of the first absorb row: XOR(state, block) no longer holds. + let col = kchip::cols::PERM_IN + 5; + let old = traces.keccak.main_table.get_row(0)[col]; + traces + .keccak + .main_table + .set_fe(0, col, old + FE::from(1u64)); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a broken absorb XOR must reject" + ); +} + +#[test] +fn registry_drift_keccak_sponge_v0_blowup2() { + let opts = options(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let entry = + resolve(LfmProgramKind::KeccakSpongeV0, 2).expect("KeccakSpongeV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +#[test] +fn keccak_sponge_cell_counts() { + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "KeccakSpongeV0 ({} bytes): {} instructions, {} main value cells, {} aux ext elements", + KECCAK_SPONGE_LEN, + program.instrs.len(), + main, + aux + ); + assert!(main > 0 && aux > 0); +} + +/// Isolates the rate-region pass-through constraint +/// `MODE_PERM · (PERM_IN − STATE) = 0`. +/// +/// Absorb rows get `PERM_IN` pinned by the BYTE_ALU[XOR] lookups; permute rows +/// have no lookups, so without this constraint a prover could feed the keccak +/// family a permutation input unrelated to the state it read from memory. Trace +/// tampering alone does not reach that hole — it desynchronises the round chip +/// and the bus catches it first. So this builds the *coordinated* forgery: the +/// last keccak row's `perm_in` is replaced BEFORE trace generation, so the +/// KECCAK_RND rows, the BITWISE multiplicities, the reply token and the output +/// words are all internally consistent with the forged input, and the claimed +/// public words are recomputed to match. Every bus balances. The only thing +/// standing between this and an accepted proof is the constraint. +#[test] +fn permute_row_cannot_substitute_the_permuted_state() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let mut exec = + super::executor::execute(&program, &keccak_arenas(7), &super::hash::TestPermutation) + .expect("honest execution"); + + // Forge the second (last) permutation's input, and make everything + // downstream of it consistent. + let last = exec.records.keccak.len() - 1; + let mut forged = exec.records.keccak[last].perm_in; + forged[0] ^= 1; + let output = keccak_adapter::permute(forged); + exec.records.keccak[last].perm_in = forged; + exec.records.keccak[last].output = output; + + // The chain program publics are once[0], twice[0], twice[1]; the last two + // come from this row, so claim the values the forged run actually produces. + let words = keccak_adapter::state_to_words(&output); + exec.public_words[1].1 = words[0]; + exec.public_words[2].1 = words[1]; + exec.records.public[1] = words[0]; + exec.records.public[2] = words[1]; + + let mut traces = build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + let proof = prove_traces(&artifacts, &mut traces, &exec.public_words, &opts) + .expect("the prover has no constraint checks, so it accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a permute row whose PERM_IN differs from the state it read must reject" + ); +} + +// ============ R1d groundwork: DefaultTranscript::sample() replay ============ + +/// The machine's reversed digest must equal the production transcript's +/// `sample()` byte for byte. +/// +/// This is a REAL bit-exactness check against `DefaultTranscript`, not a +/// reimplementation: `sample()` — finalize, reverse the 32 bytes, absorb the +/// reversed bytes, return them — is identical before and after #841, so it can +/// be verified even though this worktree predates that change. The buffered +/// candidate machinery that #841 introduced is what is blocked, not this. +#[test] +fn machine_reversed_digest_matches_default_transcript_sample() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + for len in [0usize, 1, 135, KECCAK_SPONGE_LEN] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = super::programs::keccak_sample_program(len); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("len {len}: execution failed: {e:?}")); + + let mut host = DefaultTranscript::::new(&msg); + let expected = host.sample(); + assert_eq!( + digest_bytes(&exec.public_words), + expected, + "len {len}: reversed digest must match DefaultTranscript::sample()" + ); + } +} + +/// The reversed-digest send must actually reverse: the machine's own +/// non-reversed digest and its reversed digest are byte-reverses of each other. +#[test] +fn reversed_digest_is_the_reverse_of_the_digest() { + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let plain = super::executor::execute( + &keccak_sponge_program(msg.len()), + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("exec"); + let reversed = super::executor::execute( + &super::programs::keccak_sample_program(msg.len()), + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("exec"); + + let mut want = digest_bytes(&plain.public_words); + want.reverse(); + assert_eq!(digest_bytes(&reversed.public_words), want); +} + +/// PROVES the `sample()` replay, which the execute-only test above does NOT. +/// +/// This distinction bit me: `execute` writes the reversed words from the host +/// mirror (`keccak_adapter::reversed_digest_words`), so an execute-only test +/// passes no matter what the CHIP's reversed-coefficient `Linear` says. The two +/// have to agree, and only a proof checks that — if the bus send recomposes the +/// bytes in any other order, the words it sends differ from the ones the +/// executor wrote to memory and the `LfmMem` bus stops balancing. Neutralising +/// the reversal in the chip leaves the execute-only test green and makes THIS +/// one fail, which is how it should be. +#[test] +fn machine_proves_the_sample_replay() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + let opts = options(); + for len in [0usize, 135, KECCAK_SPONGE_LEN] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = super::programs::keccak_sample_program(len); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts) + .unwrap_or_else(|e| panic!("len {len}: prove failed: {e:?}")); + + let mut host = DefaultTranscript::::new(&msg); + assert_eq!( + digest_bytes(&proved.public_words), + host.sample(), + "len {len}: proved sample() must match DefaultTranscript" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "len {len}: the machine proof of sample() must verify" + ); + } +} + +// ============ R1d: DefaultTranscript model + candidate identity ============ + +/// The host model must track the real post-#841 `DefaultTranscript` exactly, +/// across an interleaving that exercises buffer refill AND absorb invalidation. +#[test] +fn transcript_model_matches_default_transcript() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let mut host = DefaultTranscript::::new(b"seed"); + let mut model = keccak_host::TranscriptModel::new(b"seed"); + + // `sample_u64(2^n)` has threshold 0, so it consumes exactly one candidate + // and returns its low n bits. Compare the model's raw candidate masked the + // same way; the raw 32-byte squeezes are compared exactly further down. + const MASK: u64 = (1u64 << 63) - 1; + // Drain a full squeeze (4 candidates) and force a refill on the 5th. + for i in 0..5 { + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "candidate {i}" + ); + } + // Absorb mid-buffer: both must drop the remaining squeezed bytes. + host.append_bytes(b"abc"); + model.append(b"abc"); + for i in 0..3 { + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "post-absorb {i}" + ); + } + // A raw sample() also invalidates. + assert_eq!(model.sample(), host.sample(), "raw sample"); + for i in 0..2 { + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "post-sample {i}" + ); + } + // Absorbs of several lengths, including one crossing the keccak rate. + for len in [1usize, 135, 136, 200] { + let msg: Vec = (0..len).map(|i| (i as u8).wrapping_mul(7)).collect(); + host.append_bytes(&msg); + model.append(&msg); + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "len {len}" + ); + } +} + +/// THE IDENTITY THE EMITTER RESTS ON: the four big-endian candidates carved out +/// of a reversed digest are the ORIGINAL digest's `u64` lanes 3, 2, 1, 0. +/// +/// If this holds, the machine reads candidates straight off the plain digest +/// words — already `u32` halves on the bus — and never reverses anything to +/// sample. The big-endian read and the byte reversal cancel. +#[test] +fn be_candidates_are_plain_state_lanes() { + for len in [0usize, 1, 135, 202] { + let msg: Vec = (0..len).map(|i| (i as u8).wrapping_mul(13)).collect(); + + // The state whose first 32 bytes are the digest. + let digest = keccak_host::keccak256(&msg); + let mut state = [0u64; 25]; + for (lane, chunk) in state[..4].iter_mut().zip(digest.chunks_exact(8)) { + let mut b = [0u8; 8]; + b.copy_from_slice(chunk); + *lane = u64::from_le_bytes(b); + } + + let mut model = keccak_host::TranscriptModel::new(&msg); + for i in 0..4 { + assert_eq!( + model.next_u64(), + keccak_host::candidate_from_state(&state, i), + "len {len}, candidate {i} must be state lane {}", + 3 - i + ); + } + } +} + +// ================= R1d: the TranscriptReplay emitter ================= + +use super::programs::{ + TRANSCRIPT_ABSORB_A, TRANSCRIPT_ABSORB_B, TRANSCRIPT_ARENA_HALVES, TRANSCRIPT_QUERY_BITS, + TRANSCRIPT_SEED, canonicity_guard_program, transcript_replay_program, + transcript_replay_program_source, +}; + +/// Goldilocks: `p = 2^64 − 2^32 + 1`. +const P: u64 = 0xFFFF_FFFF_0000_0001; + +// ---------------------------- oracle scrutiny ---------------------------- + +/// The assumption that lets a BASE-field `DefaultTranscript` be the oracle for a +/// machine script containing an EXTENSION draw: an ext3 element is three +/// consecutive base draws, in coordinate order 0, 1, 2. +/// +/// Read off `Degree3GoldilocksExtensionField::sample_field_element_from`, which +/// is `from_fn(|_| GoldilocksField::sample_field_element_from(&mut next_u64))`. +/// `from_fn` evaluating in index order is the load-bearing part, so it is pinned +/// here against the real thing rather than trusted. +#[test] +fn ext_draw_is_three_base_draws_in_coordinate_order() { + use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let mut ext = DefaultTranscript::::new(TRANSCRIPT_SEED); + let mut base = DefaultTranscript::::new(TRANSCRIPT_SEED); + // Four draws = twelve candidates, so this spans three refills and cannot be + // satisfied by a coincidence inside one squeeze. + for draw in 0..4 { + let e = ext.sample_field_element(); + let coords: [FE; 3] = core::array::from_fn(|_| base.sample_field_element()); + assert_eq!(*e.value(), coords, "ext draw {draw}"); + } +} + +/// The guard's predicate, host-side: a candidate is out of range exactly when +/// `hi = 2^32 − 1 ∧ lo ≠ 0`. +fn machine_accepts(lo: u64, hi: u64) -> bool { + !(hi == 0xFFFF_FFFF && lo != 0) +} + +/// THE DERIVATION, checked against the production sampler rather than against +/// itself: for every candidate, the machine's one-instruction predicate agrees +/// with `GoldilocksField::sample_field_element_from` on whether the FIRST draw +/// is accepted, and on the value when it is. +/// +/// The production sampler is probed by feeding it the candidate under test and +/// then zeros: it took a second draw exactly when it rejected the first. +#[test] +fn canonicity_predicate_matches_production_sampler() { + use math::field::traits::HasDefaultTranscript; + + let production = |candidate: u64| -> Option { + let mut draws = 0usize; + let v = crate::tables::types::GoldilocksField::sample_field_element_from(|| { + draws += 1; + if draws == 1 { candidate } else { 0 } + }); + (draws == 1).then_some(v) + }; + + let mut candidates: Vec = vec![0, 1, 1 << 32, 0xFFFF_FFFF, u64::MAX]; + // Dense coverage of the boundary itself. + for d in 0..40u64 { + candidates.push(P.wrapping_sub(20).wrapping_add(d)); + } + // The whole of the reject region's shape: hi pinned at 2^32 − 1. + for lo in [0u64, 1, 2, 3, 0x7FFF_FFFF, 0xFFFF_FFFE, 0xFFFF_FFFF] { + candidates.push((0xFFFF_FFFFu64 << 32) | lo); + candidates.push((0xFFFF_FFFEu64 << 32) | lo); + } + // A broad deterministic sweep. + let mut x = 0x1234_5678_9abc_def0u64; + for _ in 0..5000 { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + candidates.push(x); + } + + let mut rejects = 0usize; + for c in candidates { + let (lo, hi) = (c & 0xFFFF_FFFF, c >> 32); + let accepted = production(c); + assert_eq!( + machine_accepts(lo, hi), + accepted.is_some(), + "candidate {c:#018x}: guard and production sampler disagree" + ); + match accepted { + Some(v) => { + let recomposed = &(&FE::from(hi) * &FE::from(1u64 << 32)) + &FE::from(lo); + assert_eq!(recomposed, v, "candidate {c:#018x}: value"); + } + None => rejects += 1, + } + } + assert!( + rejects >= 20, + "the sweep must actually exercise the reject branch, saw {rejects}" + ); +} + +// -------------------------- the canonicity guard -------------------------- + +fn guard_arenas(lo: u64, hi: u64) -> Vec> { + vec![vec![ + super::word::base_word(FE::from(lo)), + super::word::base_word(FE::from(hi)), + ]] +} + +/// The machine's guard at the boundary, which the replay itself cannot reach: +/// producing a digest whose candidate is ≥ p by search costs about 2^32 keccaks. +#[test] +fn machine_canonicity_guard_accepts_and_rejects_at_the_boundary() { + let program = canonicity_guard_program(); + validate(&program).expect("the guard harness must pass admission"); + let run = |c: u64| { + super::executor::execute( + &program, + &guard_arenas(c & 0xFFFF_FFFF, c >> 32), + &super::hash::TestPermutation, + ) + }; + + for c in [0u64, 1, 12345, 1 << 32, P - 2, P - 1] { + let exec = run(c).unwrap_or_else(|e| panic!("{c:#018x} is canonical: {e:?}")); + assert_eq!( + exec.public_words[0].1[0], + FE::from(c), + "{c:#018x}: recomposed value" + ); + } + for c in [P, P + 1, P + 12345, u64::MAX] { + match run(c) { + Err(LfmExecError::DivByZero { .. }) => {} + other => panic!( + "{c:#018x} is ≥ p and must fail the guard, got {:?}", + other.map(|_| "accepted") + ), + } + } +} + +/// The guard has to hold against a prover, not just against the executor. +/// +/// Trace tampering cannot show this — changing the arena makes the executor +/// refuse, and changing one trace cell desynchronises the memory bus, which +/// rejects for the wrong reason. So this is the coherent forgery (§ the +/// permute-row precedent): start from candidate `p − 1`, whose guard row is +/// `div(lo = 0, g = 0)`, and forge `lo = 1` — i.e. candidate `p` — in EVERY row +/// that touches that cell, recomputing the published value to what the forged +/// halves really give ((2^32 − 1)·2^32 + 1 = p ≡ 0). The hint's send, both +/// receives, the mul-add's own constraint and the public output are then all +/// internally consistent and every bus balances. The single division constraint +/// `SEL_DIV·(B·OUT − A) = 0`, which now reads `0·1 − 1 ≠ 0`, is the only thing +/// left standing between this and an accepted proof. +#[test] +fn canonicity_guard_rejects_an_out_of_range_candidate_in_the_proof() { + let opts = options(); + let program = canonicity_guard_program(); + let artifacts = build_artifacts(&program, &opts); + let mut exec = super::executor::execute( + &program, + &guard_arenas(0, 0xFFFF_FFFF), + &super::hash::TestPermutation, + ) + .expect("p − 1 is canonical"); + + let one = FE::one(); + exec.records.hint[0] = super::word::base_word(one); + // BALU rows in emission order: sub (g = 2^32 − 1 − hi), div (the guard), + // mul-add (the value). Only the guard's numerator and the value move. + exec.records.balu[1].a = one; + exec.records.balu[2].c = one; + exec.records.balu[2].out = FE::zero(); + exec.records.public[0] = super::word::base_word(FE::zero()); + exec.public_words[0].1 = super::word::base_word(FE::zero()); + + let mut traces = build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + let proof = prove_traces(&artifacts, &mut traces, &exec.public_words, &opts) + .expect("the prover has no constraint checks, so it accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a candidate at p must fail the canonicity guard" + ); +} + +// ---------------------------- the replay itself ---------------------------- + +/// The two absorbed blobs. Both lengths are multiples of four, so packing their +/// CONCATENATION into halves gives each blob its own whole halves — which is +/// also the property `append_halves` relies on. +fn transcript_absorbs() -> (Vec, Vec) { + let a = (0..TRANSCRIPT_ABSORB_A) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let b = (0..TRANSCRIPT_ABSORB_B) + .map(|i| (i as u8).wrapping_mul(17).wrapping_add(3)) + .collect(); + (a, b) +} + +fn transcript_arenas() -> Vec> { + let (a, b) = transcript_absorbs(); + let mut bytes = a; + bytes.extend_from_slice(&b); + let halves = keccak_host::pack_stream(&bytes); + assert_eq!(halves.len(), TRANSCRIPT_ARENA_HALVES as usize); + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +struct ReplayExpectation { + f0: FE, + f1: FE, + e: [FE; 3], + q: u64, + f2: FE, + s: [u8; 32], + f3: FE, +} + +/// The oracle: the REAL `DefaultTranscript`, driven through the same script. +/// +/// Instantiated over the base field so that `sample_field_element` is one draw, +/// matching the machine's `sample_felt`; the extension draw in the middle is +/// three consecutive base draws, which +/// `ext_draw_is_three_base_draws_in_coordinate_order` pins against the real ext +/// sampler independently. +fn host_expectation() -> ReplayExpectation { + use crate::tables::types::GoldilocksField; + + use crypto::fiat_shamir::is_transcript::IsTranscript; + use stark::config::DefaultStarkTranscript; + + let (a, b) = transcript_absorbs(); + let mut h = DefaultStarkTranscript::::new(TRANSCRIPT_SEED); + h.append_bytes(&a); + let f0 = h.sample_field_element(); + let f1 = h.sample_field_element(); + let e: [FE; 3] = core::array::from_fn(|_| h.sample_field_element()); + h.append_bytes(&b); + let q = h.sample_u64(1 << TRANSCRIPT_QUERY_BITS); + let f2 = h.sample_field_element(); + let s = h.sample(); + let f3 = h.sample_field_element(); + ReplayExpectation { + f0, + f1, + e, + q, + f2, + s, + f3, + } +} + +fn check_replay_publics(public: &[(u32, LfmWord)], what: &str) { + let x = host_expectation(); + assert_eq!(public.len(), 8, "{what}: public word count"); + assert_eq!(public[0].1[0], x.f0, "{what}: first base challenge"); + assert_eq!(public[1].1[0], x.f1, "{what}: second base challenge"); + for i in 0..3 { + assert_eq!(public[2].1[i], x.e[i], "{what}: ext coordinate {i}"); + } + assert_eq!(public[3].1[0], FE::from(x.q), "{what}: sample_u64 draw"); + assert_eq!(public[4].1[0], x.f2, "{what}: post-absorb challenge"); + assert_eq!(digest_bytes(&public[5..7]), x.s, "{what}: raw sample()"); + assert_eq!(public[7].1[0], x.f3, "{what}: post-sample challenge"); +} + +#[test] +fn transcript_replay_program_is_admissible() { + validate(&transcript_replay_program()).expect("the replay must pass admission"); +} + +#[test] +fn transcript_replay_source_is_deterministic() { + let a = transcript_replay_program_source(); + let b = transcript_replay_program_source(); + assert_eq!(a.instrs.len(), b.instrs.len()); + assert_eq!(a.num_addrs, b.num_addrs); + assert_eq!(format!("{:?}", a.instrs), format!("{:?}", b.instrs)); +} + +/// Bit-exactness against the real transcript, execute-only (fast). Validates the +/// EMITTER — the consumption schedule, the invalidation rules, the candidate +/// lane mapping — against `DefaultTranscript` itself. +#[test] +fn transcript_replay_matches_default_transcript() { + let exec = super::executor::execute( + &transcript_replay_program(), + &transcript_arenas(), + &super::hash::TestPermutation, + ) + .expect("the replay must execute"); + check_replay_publics(&exec.public_words, "execute"); +} + +/// The R1d headline: the machine PROVES a scripted `DefaultTranscript` +/// interleaving and the proof verifies through the registry, with every sampled +/// value identical to the real transcript's. +/// +/// The proving half is not redundant with the execute-only test above. Per the +/// R1c lesson, `execute` fills the keccak rows from the host mirror, so an +/// execute-only test says nothing about whether the CHIP agrees — and this +/// program leans on the chip's reversed-digest send (the re-absorb), on `Unpack` +/// of keccak output words (the candidates), and on the BALU division that +/// enforces canonicity. +#[test] +fn transcript_replay_proves_and_verifies() { + let opts = options(); + let program = transcript_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &transcript_arenas(), &opts).expect("prove"); + check_replay_publics(&proved.public_words, "prove"); + assert!( + lfm_verify( + LfmProgramKind::TranscriptReplayV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("TranscriptReplayV0 is registered"), + "the registered transcript replay must verify" + ); +} + +/// Flipping one absorbed half must reject: the absorb feeds a squeeze, so every +/// later challenge moves, and claiming the honest ones no longer matches. +/// +/// Both blobs are covered — the first is absorbed before any squeeze, the second +/// invalidates a buffer mid-flight, and they reach the sponge by different +/// paths. +#[test] +fn tampered_transcript_absorb_half_rejects() { + let opts = options(); + let program = transcript_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove(&program, &artifacts, &transcript_arenas(), &opts).expect("prove"); + check_replay_publics(&honest.public_words, "honest"); + + for (half, what) in [(5u32, "first absorb"), (20, "second absorb")] { + let mut tampered = transcript_arenas(); + tampered[0][half as usize][0] = &tampered[0][half as usize][0] + FE::from(1u64); + let forged = lfm_prove(&program, &artifacts, &tampered, &opts).expect("prove"); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a changed half must change the challenges" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{what}: claiming the honest challenges for a tampered absorb must reject" + ); + } +} + +#[test] +fn registry_drift_transcript_replay_v0_blowup2() { + let opts = options(); + let program = transcript_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = resolve(LfmProgramKind::TranscriptReplayV0, 2) + .expect("TranscriptReplayV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +/// Pins the emitted SHAPE, which the value tests would only catch indirectly: +/// the script's five squeezes span six rate blocks (segment #3 is 168 bytes and +/// takes two), so the program must hold exactly six keccak rows. An extra or +/// missing squeeze — the classic invalidation-rule bug — moves this number. +#[test] +fn transcript_replay_cell_counts() { + /// Compressions the replay emits under the hash it NAMES. It was 6 under + /// keccak — "five squeezes over six rate blocks" — and a 136-byte sponge + /// rate does not divide this script the way a 64-byte BLAKE3 block does, so + /// the number was re-derived rather than carried over. + const TRANSCRIPT_REPLAY_COMPRESSIONS: usize = 8; + + let program = transcript_replay_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "TranscriptReplayV0: {} instructions, {} addresses, {} main value cells, {} aux ext elements", + program.instrs.len(), + program.num_addrs, + main, + aux + ); + assert_eq!( + wrap_hash_rows_at(&program, super::programs::TRANSCRIPT_REPLAY_WRAP_HASH), + TRANSCRIPT_REPLAY_COMPRESSIONS, + "the replay's compression count" + ); + assert!(main > 0 && aux > 0); +} + +// ------------------------- emitter-contract guards ------------------------- + +/// The UPPER bound only. There is no lower bound any more: `nbits = 0` is legal +/// (a one-row trace at blowup 2 has a two-leaf LDE, one query index, and that +/// index is 0), and that the emitter still CONSUMES a draw for it is pinned by +/// `per_table_aggregator_tests::a_zero_bit_query_draw_consumes_what_the_host_does`. +/// +/// ⚠ The expected string names "at most 32" deliberately. A substring loose +/// enough to match any assert in this function would pass on a reintroduced +/// lower bound as readily as on this one, which is the failure mode this test +/// exists to catch. +#[test] +#[should_panic(expected = "nbits must be at most 32")] +fn sample_u64_pow2_rejects_more_than_32_bits() { + use super::transcript_replay::TranscriptReplay; + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + let _ = t.sample_u64_pow2(&mut b, 33); +} + +/// The packing obligation, made unmissable: a constant whose length is not a +/// multiple of four leaves the segment byte-misaligned, and machine data +/// appended after one would straddle a half boundary — which needs the +/// byte-level splice the statement-absorb leg will build, not a silent +/// miscoding here. +#[test] +#[should_panic(expected = "must start on a 4-byte boundary")] +fn machine_data_after_a_misaligned_constant_is_rejected() { + use super::transcript_replay::TranscriptReplay; + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(b"abc"); + let z = b.felt_const(FE::zero()); + t.append_halves(&[z]); +} + +/// Pins the completeness figures `SOUNDNESS.md` §6.3 quotes, so a doc number +/// cannot drift away from the arithmetic behind it. +#[test] +fn zero_rejection_completeness_bound() { + use super::transcript_replay::{ + reject_probability_per_candidate, reject_probability_per_proof, + }; + + // q = (2^32 − 1)/2^64: just under 2^−32, and within a hair of it. + let q = reject_probability_per_candidate(); + assert!(q < 2f64.powi(-32), "q must be strictly below 2^-32"); + assert!(q > 2f64.powi(-32) * (1.0 - 1e-9), "q ≈ 2^-32"); + + // The verified schedule: E = 4 + T·(3 + L_t) extension draws per proof, each + // three base candidates. L_t = 12 with tables at their 2^19 row cap. + let ext_draws = |tables: usize, fold_challenges: usize| 4 + tables * (3 + fold_challenges); + assert_eq!(ext_draws(24, 12), 364, "T = 24 (the structural minimum)"); + assert_eq!(ext_draws(60, 12), 904, "T ≈ 60 (realistic)"); + + let p_min = reject_probability_per_proof(3 * ext_draws(24, 12)); + let p_real = reject_probability_per_proof(3 * ext_draws(60, 12)); + assert!( + (p_min - 2.54e-7).abs() < 0.01e-7, + "the T = 24 bound moved: {p_min:e}" + ); + assert!( + (p_real - 6.31e-7).abs() < 0.01e-7, + "the T = 60 bound moved: {p_real:e}" + ); + assert!( + p_real < 1e-6, + "the headline claim is < 1e-6 at production shapes" + ); + + // Per-table growth is 15 extension draws, NOT one: the per-draw figure is + // ~7e-10 and the per-table figure is 15x that. Conflating them was a real + // error in an earlier draft of §6.3, so both are pinned. + let per_draw = reject_probability_per_proof(3); + let per_table = (p_real - p_min) / 36.0; + assert!((per_draw - 6.98e-10).abs() < 0.01e-10, "per extension draw"); + assert!( + (per_table - 1.048e-8).abs() < 0.01e-8, + "per additional table" + ); + assert!( + (per_table / per_draw - 15.0).abs() < 1e-6, + "a table is 3 + L_t = 15 extension draws" + ); + + // Where it stops being negligible: ~4.3e7 base candidates for 1%, 2^31 for 50%. + assert!(reject_probability_per_proof(43_000_000) > 0.01); + assert!(reject_probability_per_proof(42_000_000) < 0.01); + assert!((reject_probability_per_proof(1 << 31) - 0.5).abs() < 1e-6); +} + +/// Pins `append_digest`'s word-to-halves byte order: a machine-computed keccak +/// digest absorbed into the replay must reach the sponge as the same 32 bytes +/// `DefaultTranscript::append_bytes` sees. +/// +/// This is the absorb path a real verifier uses for every commitment root, and +/// the acceptance script above does not reach it (it absorbs arena halves +/// directly). Reversing the lane order inside `append_word` fails this test and +/// nothing else. +#[test] +fn absorbed_machine_digest_matches_default_transcript() { + use crate::tables::types::GoldilocksField; + + use crypto::fiat_shamir::is_transcript::IsTranscript; + use stark::config::DefaultStarkTranscript; + + for len in [0usize, 135, KECCAK_SPONGE_LEN] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = super::programs::transcript_absorb_digest_program(len); + validate(&program).unwrap_or_else(|e| panic!("len {len}: admission: {e:?}")); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("len {len}: execution failed: {e:?}")); + + let mut h = DefaultStarkTranscript::::new(TRANSCRIPT_SEED); + h.append_bytes(&keccak_host::keccak256(&msg)); + assert_eq!( + exec.public_words[0].1[0], + h.sample_field_element(), + "len {len}: challenge after absorbing a machine-computed digest" + ); + } +} + +/// Makes the buffer-position table in `transcript_replay_program_source`'s doc +/// comment executable, so the documented interleaving cannot drift away from the +/// emitted one. +/// +/// The value tests would catch a schedule change too, but only as "the numbers +/// moved". This says which step moved. +#[test] +fn transcript_replay_schedule_matches_the_documented_table() { + use super::transcript_replay::TranscriptReplay; + + let halves_a = TRANSCRIPT_ABSORB_A / keccak_host::BYTES_PER_HALF; + let mut b = super::builder::LfmBuilder::new(); + let arena = b.declare_arena(TRANSCRIPT_ARENA_HALVES); + let halves: Vec<_> = (0..TRANSCRIPT_ARENA_HALVES) + .map(|i| b.hint_felt(arena, i)) + .collect(); + let (absorb_a, absorb_b) = halves.split_at(halves_a); + + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_halves(absorb_a); + assert_eq!( + (t.out_pos(), t.segment_len()), + (32, TRANSCRIPT_SEED.len() + TRANSCRIPT_ABSORB_A), + "after absorb A: buffer empty, segment is seed ‖ A" + ); + + let _ = t.sample_felt(&mut b); + assert_eq!( + (t.out_pos(), t.segment_len()), + (8, 32), + "squeeze #1, then one candidate; the segment becomes the reversed digest" + ); + let _ = t.sample_felt(&mut b); + assert_eq!(t.out_pos(), 16, "second candidate, no squeeze"); + let _ = t.sample_ext(&mut b); + assert_eq!( + t.out_pos(), + 8, + "three more candidates: squeeze #2 lands INSIDE the extension draw" + ); + + t.append_halves(absorb_b); + assert_eq!( + (t.out_pos(), t.segment_len()), + (32, 32 + TRANSCRIPT_ABSORB_B), + "absorb B invalidates a buffer with 24 live bytes; 168 bytes = two blocks" + ); + let _ = t.sample_u64_pow2(&mut b, TRANSCRIPT_QUERY_BITS); + assert_eq!(t.out_pos(), 8, "squeeze #3"); + let _ = t.sample_felt(&mut b); + assert_eq!(t.out_pos(), 16, "no squeeze"); + let _ = t.sample(&mut b); + assert_eq!( + t.out_pos(), + 32, + "raw sample #4 invalidates a buffer with 16 live bytes" + ); + let _ = t.sample_felt(&mut b); + assert_eq!(t.out_pos(), 8, "squeeze #5"); +} + +/// The segment-level packing rule, made executable: consecutive constant appends +/// are ONE byte run, chunked into halves only at the squeeze. +/// +/// `"abc"` then `"de"` must hash as the five-byte string `"abcde"` — two halves +/// — not as two independently packed pieces (which would give `"abc\0de\0\0"`). +/// Per-append packing cannot pass this. +#[test] +fn constant_appends_concatenate_across_append_boundaries() { + use super::transcript_replay::TranscriptReplay; + use crate::tables::types::GoldilocksField; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + for pieces in [ + vec![&b"abc"[..], &b"de"[..]], + vec![&b"a"[..], &b"b"[..], &b"c"[..], &b"d"[..], &b"e"[..]], + vec![&b""[..], &b"abcde"[..]], + vec![&b"abcde"[..]], + ] { + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(pieces[0]); + for p in &pieces[1..] { + t.append_const_bytes(p); + } + let f = t.sample_felt(&mut b); + b.public(f.as_cell()); + let program = super::compiler::compile(b.finish()); + let exec = + super::executor::execute(&program, &[], &super::hash::TestPermutation).expect("exec"); + + let mut h = DefaultTranscript::::new(b"abcde"); + assert_eq!( + exec.public_words[0].1[0], + h.sample_field_element(), + "{pieces:?} must absorb as the concatenation \"abcde\"" + ); + } +} + +/// The alignment rule is about the SEGMENT's length, not about whether some +/// earlier append happened to be misaligned: a 3-byte constant followed by a +/// 1-byte constant leaves the segment 4-byte aligned, so machine data may follow. +#[test] +fn machine_data_may_follow_constants_that_together_align() { + use super::transcript_replay::TranscriptReplay; + + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(b"abc"); + t.append_const_bytes(b"d"); + let z = b.felt_const(FE::zero()); + t.append_halves(&[z]); + assert_eq!(t.segment_len(), 8, "4 constant bytes plus one machine half"); +} + +/// Cross-check of the emitter's squeeze economics against the verified +/// production draw schedule. +/// +/// Per table the verifier draws β, z_OOD, γ and `L` FRI fold challenges, each +/// preceded by a root absorb that invalidates the buffer — so each extension +/// draw costs one fresh squeeze and uses three of its four candidates. The `Q` +/// query indices are then drawn back to back, costing `⌈Q/4⌉`. +/// +/// Keccak ROWS equal squeezes here because every segment stays inside one +/// 136-byte rate block: 32 reversed-digest bytes plus a 32-byte root is 64. +#[test] +fn squeeze_economics_match_the_verified_draw_schedule() { + use super::transcript_replay::TranscriptReplay; + + const L: usize = 12; // fold challenges: log2(trace) − 7, tables at the 2^19 cap + const Q: usize = 219; // Preset::Blowup2 query count + let ext_draws = 3 + L; // β, z_OOD, γ, then L fold challenges + let halves_per_root = 8; + + let mut b = super::builder::LfmBuilder::new(); + let arena = b.declare_arena(((ext_draws + 1) * halves_per_root) as u32); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + for r in 0..=ext_draws { + let root: Vec<_> = (0..halves_per_root) + .map(|i| b.hint_felt(arena, (r * halves_per_root + i) as u32)) + .collect(); + t.append_halves(&root); + // The last absorb stands for the grinding/final-poly absorb that precedes + // query sampling; it draws nothing. + if r < ext_draws { + let _ = t.sample_ext(&mut b); + } + } + for _ in 0..Q { + let _ = t.sample_u64_pow2(&mut b, 20); + } + let program = super::compiler::compile(b.finish()); + + let expected = ext_draws + Q.div_ceil(4); + assert_eq!( + program.groups.keccak.real_rows, + expected, + "expected {ext_draws} squeezes for the extension draws (one each, the \ + preceding absorb having invalidated the buffer) plus ⌈{Q}/4⌉ = {} for the \ + query draws", + Q.div_ceil(4) + ); + assert_eq!(expected, 70, "15 extension squeezes + 55 query squeezes"); +} + +// ============ R1e slice a: field elements on the wire (big-endian) ============ + +/// Byte patterns that make an endianness or permutation error impossible to +/// miss: every byte of the first value is distinct, and the boundary values pin +/// the canonical range `bit_dec` enforces. +fn be_reference_felts() -> Vec { + vec![ + 0, + 1, + 0x0123_4567_89ab_cdef, + 0xfedc_ba98_7654_3210, + 0xff, + 0xff00_0000, + 1 << 32, + P - 1, + P - 2, + ] +} + +/// A base field element must reach the sponge as the same 8 big-endian bytes +/// `append_field_element` streams. +/// +/// The program publishes the raw 32-byte squeeze rather than a sampled +/// challenge, so a mismatch localises to the absorbed bytes. +#[test] +fn append_felt_matches_default_transcript() { + use crate::tables::types::GoldilocksField; + + use crypto::fiat_shamir::is_transcript::IsTranscript; + use stark::config::DefaultStarkTranscript; + + let program = super::programs::append_felt_program(); + validate(&program).expect("admission"); + for v in be_reference_felts() { + let arenas = vec![vec![super::word::base_word(FE::from(v))]]; + let exec = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .unwrap_or_else(|e| panic!("{v:#018x}: execution failed: {e:?}")); + + let mut h = DefaultStarkTranscript::::new(TRANSCRIPT_SEED); + h.append_field_element(&FE::from(v)); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "{v:#018x}: absorbed bytes must match append_field_element" + ); + } +} + +/// The same for a cubic-extension element — 24 bytes, coordinates 0, 1, 2. +/// +/// The three coordinates are deliberately distinct, so a reversed coordinate +/// order (the other byte order this file offers, which belongs to the raw +/// `[FpE; 3]` type) fails rather than coincidentally passing. +#[test] +fn append_ext_matches_default_transcript() { + use crate::tables::types::GoldilocksExtension; + + use crypto::fiat_shamir::is_transcript::IsTranscript; + use math::field::element::FieldElement; + use stark::config::DefaultStarkTranscript; + + let program = super::programs::append_ext_program(); + validate(&program).expect("admission"); + for coords in [ + [0u64, 1, 2], + [0x0123_4567_89ab_cdef, 0xfedc_ba98_7654_3210, 0xff], + [P - 1, 0, 1 << 32], + ] { + let arenas = vec![ + coords + .iter() + .map(|&c| super::word::base_word(FE::from(c))) + .collect::>(), + ]; + let exec = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .unwrap_or_else(|e| panic!("{coords:?}: execution failed: {e:?}")); + + let e = + FieldElement::::new(core::array::from_fn(|i| FE::from(coords[i]))); + let mut h = DefaultStarkTranscript::::new(TRANSCRIPT_SEED); + h.append_field_element(&e); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "{coords:?}: absorbed bytes must match append_field_element" + ); + } +} + +/// The byteswap gadget PROVED, not just executed. +/// +/// `felt_be_halves` is the first thing in this emitter that leans on `LFM_BITDEC` +/// for a value rather than for index bits, and on a 32-term `MulAdd` chain whose +/// weights carry the byte permutation. Execution alone would not catch a +/// chip-vs-executor disagreement in either. +#[test] +fn append_ext_proves_and_verifies() { + let opts = options(); + let program = super::programs::append_ext_program(); + let artifacts = build_artifacts(&program, &opts); + let coords = [0x0123_4567_89ab_cdefu64, 0xfedc_ba98_7654_3210, P - 1]; + let arenas = vec![ + coords + .iter() + .map(|&c| super::word::base_word(FE::from(c))) + .collect::>(), + ]; + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the big-endian absorb must verify" + ); +} + +/// Pins the gadget's cost: ONE `BitDec` row per felt and NOTHING else. +/// +/// The 64 base-ALU recomposition rows this gadget used to cost are gone — +/// the chip sends the two big-endian halves as linear forms over its own bit +/// columns (the keccak `REV_ADDR` pattern), so the byte permutation lives in +/// the bus coefficients. At the real epoch that recomposition was 99.9% of +/// all BALU rows and 77% of the whole wrap program (measured 2026-08-21), +/// which is why this pin exists in the zero direction too: a regression that +/// reintroduced per-felt ALU work must fail here, not resurface as a moved +/// census. +#[test] +fn felt_be_halves_cost() { + let program = super::programs::append_felt_program(); + println!( + "append_felt: {} instructions, bitdec {}, balu {}", + program.instrs.len(), + program.groups.bitdec.real_rows, + program.groups.balu.real_rows + ); + assert_eq!( + program.groups.bitdec.real_rows, 1, + "one decomposition per felt" + ); + assert_eq!( + program.groups.balu.real_rows, 0, + "the halves are bus sends over the row's own bit columns — no ALU work" + ); +} + +// ==================== R1e slice b: the byte-level splice ==================== + +use super::programs::{ + SPLICE_ALT_DIGEST_HALVES, SPLICE_ALT_FIELD_HALVES, SPLICE_ALT_TAG, splice_alternating_program, + splice_dynamic, splice_prefix, splice_program, +}; + +fn splice_arenas(byte_len: usize) -> Vec> { + vec![ + keccak_host::pack_stream(&splice_dynamic(byte_len)) + .into_iter() + .map(super::word::base_word) + .collect(), + ] +} + +/// The splice at every shift, against the REAL transcript. +/// +/// The oracle is `DefaultTranscript` over the concatenated byte string, which is +/// the definition of what the machine must reproduce: append boundaries leave no +/// trace in the digest input, so the whole segment is one byte string and the +/// machine's job is to hash exactly it. +/// +/// Shift 0 is included as the control — it takes the aligned fast path, so if +/// the splice were silently applied there it would show up here. +#[test] +fn splice_matches_default_transcript_at_every_shift() { + use crate::tables::types::GoldilocksField; + + use stark::config::DefaultStarkTranscript; + + const DYN_BYTES: usize = 32; + let halves = (DYN_BYTES / keccak_host::BYTES_PER_HALF) as u32; + for prefix_len in [0usize, 1, 2, 3, 4, 5, 6, 7, 29, 30, 31, 32] { + let program = splice_program(prefix_len, halves); + validate(&program).unwrap_or_else(|e| panic!("prefix {prefix_len}: admission: {e:?}")); + let exec = super::executor::execute( + &program, + &splice_arenas(DYN_BYTES), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("prefix {prefix_len}: execution failed: {e:?}")); + + let mut bytes = splice_prefix(prefix_len); + bytes.extend_from_slice(&splice_dynamic(DYN_BYTES)); + let mut h = DefaultStarkTranscript::::new(&bytes); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "prefix {prefix_len} (shift {}): spliced bytes must equal the concatenation", + prefix_len % keccak_host::BYTES_PER_HALF + ); + } +} + +/// The statement's real shape: alternating constant and dynamic runs where a +/// one-byte field moves the shift from 2 to 3 partway through. +#[test] +fn splice_alternating_runs_match_default_transcript() { + use crate::tables::types::GoldilocksField; + + use stark::config::DefaultStarkTranscript; + + let d = SPLICE_ALT_DIGEST_HALVES as usize * keccak_host::BYTES_PER_HALF; + let f = SPLICE_ALT_FIELD_HALVES as usize * keccak_host::BYTES_PER_HALF; + let program = splice_alternating_program(); + validate(&program).expect("admission"); + let exec = super::executor::execute( + &program, + &splice_arenas(d + 2 * f), + &super::hash::TestPermutation, + ) + .expect("execution"); + + // The same byte string, built independently in absorb order. + let dynamic = splice_dynamic(d + 2 * f); + let mut bytes = splice_prefix(SPLICE_ALT_TAG); + bytes.extend_from_slice(&dynamic[..d]); + bytes.extend_from_slice(&splice_prefix(8)); + bytes.extend_from_slice(&dynamic[d..d + f]); + bytes.extend_from_slice(&splice_prefix(1)); + bytes.extend_from_slice(&dynamic[d + f..]); + + let mut h = DefaultStarkTranscript::::new(&bytes); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "alternating const/dynamic runs across a shift change must match" + ); +} + +/// The splice PROVED, not just executed: it leans on `BitDec` plus a weighted +/// sum plus the recomposition assert, and only a proof sees the chips. +#[test] +fn splice_proves_and_verifies() { + let opts = options(); + let program = splice_program(30, 8); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &splice_arenas(32), &opts).expect("prove"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the spliced absorb must verify" + ); +} + +/// A half at or above `2^32` has no four-byte rendering, so the splice must +/// refuse it rather than silently absorb the wrong bytes. +/// +/// `bit_dec` alone bounds its input by `p`, not by `2^32`; the recomposition +/// assert inside `split_half` is what closes the gap, and this is the test that +/// fails if it is removed. +#[test] +fn splice_rejects_a_non_u32_half() { + let program = splice_program(2, 8); + let mut arenas = splice_arenas(32); + arenas[0][0][0] = FE::from(1u64 << 32); + match super::executor::execute(&program, &arenas, &super::hash::TestPermutation) { + Err(LfmExecError::DivByZero { .. }) => {} + other => panic!( + "a half at 2^32 must fail the splice's recomposition assert, got {:?}", + other.map(|_| "accepted") + ), + } +} + +/// Pins the splice's cost, and that the ALIGNED path is still free. +#[test] +fn splice_cost() { + let spliced = splice_program(2, 8); + let aligned = splice_program(4, 8); + println!( + "splice 8 halves @shift2: bitdec {}, balu {} | aligned: bitdec {}, balu {}", + spliced.groups.bitdec.real_rows, + spliced.groups.balu.real_rows, + aligned.groups.bitdec.real_rows, + aligned.groups.balu.real_rows, + ); + assert_eq!( + spliced.groups.bitdec.real_rows, 8, + "one decomposition per spliced half" + ); + assert_eq!( + aligned.groups.bitdec.real_rows, 0, + "the aligned path must emit no splice at all" + ); + assert_eq!( + aligned.groups.balu.real_rows, 0, + "the aligned path must stay instruction-free" + ); +} + +// ========== R1e slices c+d: the epoch statement and Phase A ========== + +use super::programs::{ + STMT_PREPROCESSED, STMT_PUBLIC_OUTPUT_LEN, epoch_statement_shape, statement_replay_program, + stmt_arena_halves, +}; + +/// The per-proof statement values and the Phase-A roots, as bytes. The machine +/// arena and the host oracle are both built from this, so they cannot drift. +struct StatementFixture { + elf_digest: [u8; 32], + public_output: Vec, + epoch_label: u64, + /// `(preprocessed_root, main_root)` per sub-proof, in air order. + roots: Vec<(Option<[u8; 32]>, [u8; 32])>, +} + +fn statement_fixture() -> StatementFixture { + let root = |seed: u8| -> [u8; 32] { + core::array::from_fn(|i| (i as u8).wrapping_mul(seed).wrapping_add(seed)) + }; + StatementFixture { + elf_digest: root(7), + public_output: (0..STMT_PUBLIC_OUTPUT_LEN) + .map(|i| (i as u8).wrapping_mul(19).wrapping_add(5)) + .collect(), + epoch_label: 0x0123_4567_89ab_cdef, + roots: STMT_PREPROCESSED + .iter() + .enumerate() + .map(|(i, &prep)| { + let p = prep.then(|| root(11 + 2 * i as u8)); + (p, root(31 + 2 * i as u8)) + }) + .collect(), + } +} + +/// Each field gets its OWN halves. The arena is a vector of `u32` words, not a +/// byte stream, so concatenating first and packing after would let a field whose +/// length is not a multiple of four shift every field behind it — which is +/// exactly what an unaligned `public_output` does. `pack_stream` zeroes the +/// trailing half's unused high bytes, the property the machine's mask pins. +fn statement_arenas(f: &StatementFixture) -> Vec> { + let mut halves = keccak_host::pack_stream(&f.elf_digest); + halves.extend(keccak_host::pack_stream(&f.public_output)); + halves.extend(keccak_host::pack_stream(&f.epoch_label.to_le_bytes())); + for (prep, main) in &f.roots { + if let Some(p) = prep { + halves.extend(keccak_host::pack_stream(p)); + } + halves.extend(keccak_host::pack_stream(main)); + } + assert_eq!(halves.len(), stmt_arena_halves() as usize); + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +/// The host reference: the REAL `absorb_statement_with_digest`, then Phase A. +/// +/// The statement half of this is production code, not a reimplementation — which +/// matters, because that encoding has ten fields and is exactly where a replay +/// would go wrong. The Phase-A half is a four-line transcription of +/// `crate::replay_transcript_phase_a_view` (`lib.rs`: for each air, the +/// precomputed commitment when `is_preprocessed()`, then +/// `lde_trace_main_merkle_root()`, then `z` and `α`); calling the helper itself +/// would mean synthesising `dyn AIR`s and proof views for three fake tables, +/// which would test the fakes rather than the replay. +type ExtFE = math::field::element::FieldElement; + +fn host_statement_challenges(f: &StatementFixture) -> (ExtFE, ExtFE) { + use crate::statement::{StatementKind, absorb_statement_with_digest}; + use crate::tables::types::GoldilocksExtension; + use crate::{RuntimePageRange, TableCounts}; + + use crypto::fiat_shamir::is_transcript::IsTranscript; + use stark::config::DefaultStarkTranscript; + + let shape = epoch_statement_shape(); + let c = shape.table_counts.map(|v| v as usize); + let counts = TableCounts { + cpu: c[0], + lt: c[1], + memw: c[2], + memw_aligned: c[3], + load: c[4], + mul: c[5], + dvrm: c[6], + shift: c[7], + branch: c[8], + memw_register: c[9], + eq: c[10], + bytewise: c[11], + store: c[12], + cpu32: c[13], + keccak: c[14], + keccak_rnd: c[15], + ecsm: c[16], + ecdas: c[17], + hint: c[18], + copy: c[19], + blake3: c[20], + is_wh: c[21], + ecrec: c[22], + }; + let ranges: Vec = shape + .page_ranges + .iter() + .map(|&(base, count)| RuntimePageRange { base, count }) + .collect(); + + let mut t = DefaultStarkTranscript::::new(&[]); + absorb_statement_with_digest( + &mut t, + StatementKind::ContinuationEpoch { + epoch_label: f.epoch_label, + is_final: shape.is_final, + }, + &f.elf_digest, + &f.public_output, + &counts, + shape.num_private_input_pages as usize, + &ranges, + shape.fri_final_poly_log_degree, + ); + for (prep, main) in &f.roots { + if let Some(p) = prep { + t.append_bytes(p); + } + t.append_bytes(main); + } + (t.sample_field_element(), t.sample_field_element()) +} + +fn assert_challenges_match(public: &[(u32, LfmWord)], f: &StatementFixture, what: &str) { + let (z, alpha) = host_statement_challenges(f); + assert_eq!(public.len(), 2, "{what}: z and alpha"); + for (i, (name, want)) in [("z", z), ("alpha", alpha)].iter().enumerate() { + for lane in 0..3 { + assert_eq!( + public[i].1[lane], + want.value()[lane], + "{what}: {name} coordinate {lane}" + ); + } + } +} + +/// Pins where the epoch statement leaves the byte cursor, which decides whether +/// Phase A is spliced and at what shift. +/// +/// CORRECTION to an earlier claim of mine: the statement is NOT unconditionally +/// 3 bytes past a boundary. Its length is `272 + L + 16R`, so the shift Phase A +/// inherits is `L mod 4` — since `L` is one byte per COMMIT op and therefore +/// workload-determined, the Phase-A splice cost is workload-dependent, and free +/// (Phase A entirely unspliced) for about one workload in four. +/// +/// ★ THE CONSTANT TERM'S OWN SHIFT IS GONE, and this test is where that is +/// pinned. It was `215 + L + 16R`, giving `(3 + L) mod 4` — free only at +/// `L ≡ 1` — and `statement_replay`'s module doc said a single pad byte at the +/// end of the encoding would make all of Phase A free. The main-sync port +/// supplied it without aiming at it. +/// +/// MOVER: `892c7d1bc` (main's `c2ac5d546`, #977) — arm (ii), the STATEMENT +/// ENCODING. That merge's two arms are (i) the table set shrank, empty tables +/// now being elided rather than padded, and (ii) this one. `epoch_verify_tests`' +/// SUB_PROOFS is the pin on (i). +/// +/// #977 took the six accelerator chips out of +/// `FIXED_TABLE_COUNT` (11 → 5) and made them counted, so `NUM_TABLE_COUNTS` +/// went 15 → 21 (`+48 ≡ 0 mod 4`, which moves no shift), and the same PR +/// appended `is_final` as the statement's last byte (`+1`). `215 + 49 = 264`, +/// and `264 ≡ 0 (mod 4)`. Main's IS_WH table (#1022) is counted too, so +/// `NUM_TABLE_COUNTS` went 21 → 22 (`+8 ≡ 0 mod 4`, which moves no shift): +/// `264 + 8 = 272`. The ECREC table is counted too (22 → 23, `+8`, no shift): +/// `272 + 8 = 280`. +/// +/// The acceptance shape is still spliced, which is what keeps the splice path +/// under test: `L = 14 ≢ 0 (mod 4)`, so the cursor lands at shift 2 where it +/// used to land at 3. +#[test] +fn epoch_statement_cursor_is_the_output_len_alone() { + let shape = epoch_statement_shape(); + let r = shape.page_ranges.len(); + // 280, not the 272 of the twenty-two-count era, the 264 of the + // twenty-one-count one, the 215 of the fifteen-count one or the 207 of the + // fourteen-count one: six accelerator counts (+48), the trailing `is_final` + // byte (+1), IS_WH's count (+8) and ECREC's (+8). + // The terms are named in the doc above, because 280 is the sum of changes + // that move NO shift and one that moves it by one. + assert_eq!(shape.byte_len(), 280 + STMT_PUBLIC_OUTPUT_LEN + 16 * r); + for l in 0..8usize { + let total = 280 + l + 16 * r; + assert_eq!( + total % keccak_host::BYTES_PER_HALF, + l % keccak_host::BYTES_PER_HALF, + "Phase A inherits shift L mod 4" + ); + } + // The acceptance shape is chosen to exercise BOTH new paths at once: an + // unaligned public output (so the trailing half is masked) and a nonzero + // inherited shift (so Phase A is spliced). + assert_ne!(STMT_PUBLIC_OUTPUT_LEN % keccak_host::BYTES_PER_HALF, 0); + assert_ne!(shape.byte_len() % keccak_host::BYTES_PER_HALF, 0); +} + +#[test] +fn statement_replay_program_is_admissible() { + validate(&statement_replay_program()).expect("admission"); +} + +/// R1e's acceptance: the machine's `(z, α)` must equal what the REAL statement +/// absorb plus Phase A produce. +#[test] +fn statement_replay_matches_the_host_challenges() { + let f = statement_fixture(); + let exec = super::executor::execute( + &statement_replay_program(), + &statement_arenas(&f), + &super::hash::TestPermutation, + ) + .expect("execution"); + assert_challenges_match(&exec.public_words, &f, "execute"); +} + +/// The same, PROVED and verified through the registry. +#[test] +fn statement_replay_proves_and_verifies() { + let opts = options(); + let f = statement_fixture(); + let program = statement_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &statement_arenas(&f), &opts).expect("prove"); + assert_challenges_match(&proved.public_words, &f, "prove"); + assert!( + lfm_verify( + LfmProgramKind::StatementReplayV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("StatementReplayV0 is registered"), + "the registered statement replay must verify" + ); +} + +/// Both tamper vectors: a flipped Phase-A root half and a flipped statement +/// byte. Each must move the challenges, and claiming the honest ones must reject. +#[test] +fn tampered_statement_or_root_rejects() { + let opts = options(); + let f = statement_fixture(); + let program = statement_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove(&program, &artifacts, &statement_arenas(&f), &opts).expect("prove"); + + // Half 0 is the ELF digest (statement); half 14 is inside the first + // sub-proof's preprocessed root (Phase A). + for (half, what) in [(0usize, "statement byte"), (14, "Phase-A root half")] { + let mut arenas = statement_arenas(&f); + arenas[0][half][0] = &arenas[0][half][0] + FE::from(1u64); + let forged = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a flip must move z or alpha" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{what}: claiming the honest challenges must reject" + ); + } +} + +#[test] +fn registry_drift_statement_replay_v0_blowup2() { + let opts = options(); + let artifacts = build_artifacts(&statement_replay_program(), &opts); + let entry = resolve(LfmProgramKind::StatementReplayV0, 2) + .expect("StatementReplayV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +#[test] +fn statement_replay_cell_counts() { + let program = statement_replay_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "StatementReplayV0: {} instructions, keccak {}, bitdec {}, balu {}, {main} main cells, {aux} aux", + program.instrs.len(), + program.groups.keccak.real_rows, + program.groups.bitdec.real_rows, + program.groups.balu.real_rows, + ); + assert!(main > 0 && aux > 0); +} + +/// The masked trailing half's soundness obligation: bytes PAST the encoded +/// length must not reach the sponge. +/// +/// `public_output` is length-prefixed, so its final arena half has live bytes +/// only up to `len % 4`. The high bytes of that felt are arena data and +/// otherwise unconstrained — without the zero-pin in `Packer::push_masked` a +/// prover could put anything there and change the absorbed byte string while the +/// length prefix said otherwise. Here byte 3 of the trailing half is past the +/// 14-byte length, so the program must refuse to execute rather than absorb it. +#[test] +fn statement_rejects_garbage_past_the_public_output_length() { + let f = statement_fixture(); + let mut arenas = statement_arenas(&f); + // Halves: elf 0..8, public_output 8..12 (14 bytes = 3 whole + 2 live), + // so half 11's top two bytes are past the length. + arenas[0][11][0] = &arenas[0][11][0] + FE::from(1u64 << 24); + match super::executor::execute( + &statement_replay_program(), + &arenas, + &super::hash::TestPermutation, + ) { + Err(LfmExecError::DivByZero { .. }) => {} + other => panic!( + "bytes past the public-output length must be pinned to zero, got {:?}", + other.map(|_| "accepted") + ), + } +} +// ======================= KECCAK_RND chunking ======================= +// +// `KECCAK_RND` costs 24 rows per permutation, so one instance cannot hold the +// ~460k permutations a real proof wrap needs. These tests cover the split: the +// shape it produces, that a multi-chunk program proves and verifies, and the +// two ways the split itself can be wrong (a corrupted chunk, a dropped +// permutation). + +use super::airs::{keccak_rnd_chunk_permutations, keccak_rnd_chunk_rows, num_lfm_airs}; +use super::chunking::KeccakChunking; +use crate::tables::keccak_rnd; + +/// A 3-permutation sponge: `pad10*1` grows 280 bytes to 3 rate blocks, so at +/// two permutations per chunk it splits unevenly (2 + 1) — the partial-final +/// chunk is the case a uniform split would miss. +const CHUNKED_SPONGE_LEN: usize = 280; + +/// Two permutations per chunk. Small enough that the multi-chunk tests prove in +/// seconds instead of the 21,845 permutations the default policy would need. +fn test_chunking() -> KeccakChunking { + KeccakChunking::from_permutations(2) +} + +fn chunked_sponge_program() -> LfmProgram { + keccak_sponge_program(CHUNKED_SPONGE_LEN).with_keccak_chunking(test_chunking()) +} + +fn chunked_sponge_msg() -> Vec { + (0..CHUNKED_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect() +} + +/// The permutation-level round operations a program's execution produces — +/// the same list `build_traces` chunks, rebuilt here so the tamper tests can +/// re-chunk it by hand. +fn round_ops_of( + program: &LfmProgram, + arenas: &[Vec], +) -> Vec { + let exec = super::executor::execute(program, arenas, &super::hash::TestPermutation) + .expect("honest execution"); + let ops: Vec<_> = exec + .records + .keccak + .iter() + .enumerate() + .map(|(row, r)| keccak_adapter::KeccakAdapterOperation { + tag: klayout::tag_for_row(row), + input: r.perm_in, + }) + .collect(); + keccak_adapter::round_operations(&ops) +} + +/// Every registered program that HAS the keccak family is single-chunk under +/// the default policy, so the production path is unchanged by this feature — +/// chunking is dormant until a program exceeds 21,845 permutations. +/// +/// A program without the family carries zero instances, not one. The policy's +/// own floor of one exists to keep an unused chip present, and `ChipSet` is +/// what retired that: post-flip the authenticating programs hash with BLAKE3 +/// and have no keccak work at all, so a KECCAK_RND instance would be a whole +/// AIR — commitment, tree, opening, FRI — carrying nothing. +#[test] +fn registered_programs_are_single_chunk() { + for entry in super::registry::LFM_REGISTRY { + let expected = usize::from(entry.chip_set.keccak); + assert_eq!( + entry.keccak_rnd_chunks, expected, + "{:?} is registered with a chunk count its chip set does not imply", + entry.kind + ); + } +} + +/// ★ The registry is six rows and every `(kind, blowup_factor)` is unique. +/// +/// `resolve` keys on that pair alone, so a seventh row for a pair that already +/// exists would be unreachable data — and `compute_lfm_registry`'s doc promises +/// exactly that shape ("a second hasher becomes additional rows"). `resolve` +/// now rejects the ambiguity rather than taking the first match; this is the +/// half that makes ADDING a row a deliberate act instead of a silent widening, +/// since nothing else in the crate pinned the table's length. +#[test] +fn the_registry_is_six_unambiguous_rows() { + use super::registry::{LFM_REGISTRY, LfmRegistryError, resolve}; + + assert_eq!( + LFM_REGISTRY.len(), + 6, + "one entry per LfmProgramKind at the single registered blowup; adding \ + rows is a re-blessing decision, not a regeneration side effect" + ); + + for (i, entry) in LFM_REGISTRY.iter().enumerate() { + for other in LFM_REGISTRY.iter().skip(i + 1) { + assert!( + entry.kind != other.kind || entry.blowup_factor != other.blowup_factor, + "{:?} at blowup {} appears twice — `resolve` keys on this pair", + entry.kind, + entry.blowup_factor + ); + } + // The honest control: every row is reachable through the real lookup. + assert!( + resolve(entry.kind, entry.blowup_factor).is_ok(), + "{:?} must resolve", + entry.kind + ); + } + + // And the miss is still a hard error, not a fallback. + assert!(matches!( + resolve(super::registry::LfmProgramKind::TrivialV0, 8), + Err(LfmRegistryError::UnknownProgram { + blowup_factor: 8, + .. + }) + )); +} + +/// Every `HasherKind` there is. Not derived — a new candidate must be added +/// here by hand, which is the point: the two tests below are what say a new +/// hasher gets its own program identity rather than sharing one. +const ALL_HASHERS: [super::hash::HasherKind; 3] = [ + super::hash::HasherKind::Test, + super::hash::HasherKind::Rpx, + super::hash::HasherKind::Blake3, +]; + +/// ★ Each registered entry's digest is bound to the hasher the entry names, +/// and to no other. +/// +/// The first assertion is the honest control: the stored `program_id` really is +/// what the stored `(roots, heights, chunks, hasher)` derive, so the table is +/// internally consistent. The second is the property — swapping *only* the +/// hasher, with every root held fixed, must move the digest. Held together they +/// say a Test-backed and an RPX-backed machine of the same program are two +/// programs, which is what stops a verifier from pairing one hasher's digest +/// with another hasher's AIR set. +#[test] +fn every_registry_entry_binds_its_hasher_into_its_digest() { + for entry in super::registry::LFM_REGISTRY { + assert_eq!( + super::statement::lfm_program_id( + &entry.roots, + &entry.log_heights, + entry.keccak_rnd_chunks, + entry.hasher, + entry.chip_set, + &[entry.roots[BLAKE3_SLOT]], + &[entry.log_heights[BLAKE3_SLOT]], + ), + entry.program_id, + "{:?}: the stored digest must be what the stored shape derives", + entry.kind + ); + for other in ALL_HASHERS { + if other == entry.hasher { + continue; + } + assert_ne!( + super::statement::lfm_program_id( + &entry.roots, + &entry.log_heights, + entry.keccak_rnd_chunks, + other, + entry.chip_set, + &[entry.roots[BLAKE3_SLOT]], + &[entry.log_heights[BLAKE3_SLOT]], + ), + entry.program_id, + "{:?}: {other:?} must not share {:?}'s program identity", + entry.kind, + entry.hasher + ); + } + } +} + +/// The registry-path consequence: `lfm_verify` builds its AIR set from the +/// entry's hasher, so an honest proof of a registered program verifies, and the +/// same proof against the same entry's roots and digest under any *other* +/// hasher does not. +/// +/// The accept half is not decoration. A verifier that rejected everything would +/// pass the reject half on its own, so the pair is what makes this a binding +/// test rather than a "does it say no" test. +#[test] +fn the_registry_hasher_is_what_verify_builds() { + let opts = options(); + let program = trivial_program(); + let entry = resolve(LfmProgramKind::TrivialV0, opts.blowup_factor).expect("registered"); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + + assert!( + lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts + ) + .expect("registered"), + "the honest proof must verify under the hasher the entry names" + ); + + for other in ALL_HASHERS { + if other == entry.hasher { + continue; + } + assert!( + !verify_against( + &entry.roots, + &entry.program_id, + entry.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + other, + entry.chip_set, + ), + "the entry's own proof must not verify under {other:?}" + ); + } +} + +/// The split's shape: chunk count, per-chunk permutation counts, per-chunk +/// trace heights, AIR count and trace count all agree. +#[test] +fn chunking_splits_the_sponge_into_two_uneven_chunks() { + let program = chunked_sponge_program(); + assert_eq!( + program.groups.keccak.real_rows, 3, + "a {CHUNKED_SPONGE_LEN}-byte message must be 3 rate blocks" + ); + + assert_eq!(keccak_rnd_chunk_permutations(&program), vec![2, 1]); + // 2 permutations = 48 rows → 64; 1 permutation = 24 rows → 32. + assert_eq!(keccak_rnd_chunk_rows(&program), vec![64, 32]); + + let artifacts = build_artifacts(&program, &options()); + assert_eq!(artifacts.keccak_rnd_chunks, 2); + assert_eq!(num_lfm_airs(2, 1), super::NUM_LFM_CHIPS + 1); + + let exec = super::executor::execute( + &program, + &sponge_arenas(&chunked_sponge_msg()), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + let traces = build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + assert_eq!(traces.keccak_rnd.len(), 2, "one KECCAK_RND trace per chunk"); + assert_eq!( + traces + .keccak_rnd + .iter() + .map(|t| t.num_rows()) + .collect::>(), + vec![64, 32], + "chunk traces must match the heights the artifacts predict" + ); +} + +/// ★ The acceptance test: a program needing more than one `KECCAK_RND` chunk +/// proves and verifies end to end, and its digest still matches the production +/// hasher. +#[test] +fn chunked_sponge_proves_and_verifies() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + assert_eq!(artifacts.keccak_rnd_chunks, 2, "this test needs 2 chunks"); + + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + assert_eq!( + digest_bytes(&proved.public_words), + keccak_host::keccak256(&msg), + "a chunked proof must hash the same as the production hasher" + ); + assert_eq!( + stark::proof::view::MultiProofView::Owned(&proved.proof).len(), + artifacts.chip_set.num_airs(2, artifacts.blake3_chunks()), + "the proof must carry one sub-proof per AIR instance" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a two-chunk KECCAK_RND proof must verify" + ); +} + +/// Chunking is a prover-side layout choice, not a semantic one: the same +/// message proved at 1 and at 2 chunks yields the same public output. (The +/// program *identity* does differ — the chunk count is bound into the digest — +/// which is exactly why the two need different artifacts.) +#[test] +fn chunking_does_not_change_what_is_proved() { + let opts = options(); + let msg = chunked_sponge_msg(); + + let one = keccak_sponge_program(CHUNKED_SPONGE_LEN); + let one_artifacts = build_artifacts(&one, &opts); + assert_eq!(one_artifacts.keccak_rnd_chunks, 1); + let one_proof = lfm_prove(&one, &one_artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + let two = chunked_sponge_program(); + let two_artifacts = build_artifacts(&two, &opts); + let two_proof = lfm_prove(&two, &two_artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + assert_eq!( + one_proof.public_words, two_proof.public_words, + "chunking must not change the program's output" + ); + assert_eq!( + one_artifacts.roots, two_artifacts.roots, + "chunking must not move any preprocessed root" + ); + assert_ne!( + one_artifacts.program_id, two_artifacts.program_id, + "the chunk count is program shape and must be bound into the digest" + ); + for (artifacts, proof) in [(&one_artifacts, &one_proof), (&two_artifacts, &two_proof)] { + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof.proof, + &proof.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "both chunkings must verify against their own artifacts" + ); + } +} + +/// ★ Tamper: corrupting a permutation that lives in the *second* chunk must +/// reject. The first chunk is untouched, so this only rejects if chunk 1's +/// rows are really part of the proof's bus balance. +#[test] +fn tampered_second_chunk_permutation_rejects() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + + let mut traces = build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + assert_eq!(traces.keccak_rnd.len(), 2); + // Byte 0 of lane (0,0) on the second chunk's first row: the `Keccak` + // receive token no longer matches the send that fed it. + let col = keccak_rnd::cols::start(0, 0, 0); + let old = traces.keccak_rnd[1].main_table.get_row(0)[col]; + traces.keccak_rnd[1] + .main_table + .set_fe(0, col, old + FE::from(1u64)); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a corrupted permutation in the second chunk must reject" + ); +} + +/// ★ Falsifies the split itself: drop the permutation the second chunk holds. +/// The `LFM_KECCAK` chip still sends its request token, so the `Keccak` bus is +/// left with a send that nothing receives. If this ever accepts, chunks are +/// not actually contributing their rows to the balance. +#[test] +fn dropping_the_second_chunks_permutation_rejects() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + + let mut traces = build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + // Same chunk COUNT — so the AIR set and the digest still match — but the + // last chunk is now empty. + traces.keccak_rnd[1] = keccak_rnd::generate_keccak_rnd_trace(&[]); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a chunk missing its permutation must reject" + ); +} + +/// The mechanism's foundation, stated positively: which chunk a permutation +/// lands in is free. `KECCAK_RND` has no row-to-row constraints and its rounds +/// are linked by `Keccak` bus tokens rather than row adjacency, so LogUp +/// cannot tell a 2+1 split from a 1+2 one. This is why chunking needs no +/// pairing logic — and if it ever fails, the round chip has grown a +/// cross-row dependency that chunking would silently break. +#[test] +fn permutations_may_be_reassigned_across_chunk_boundaries() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + + let round_ops = round_ops_of(&program, &sponge_arenas(&msg)); + assert_eq!(round_ops.len(), 3); + + let mut traces = build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + // Canonical split is 2 + 1; re-split as 1 + 2. + traces.keccak_rnd[0] = keccak_rnd::generate_keccak_rnd_trace(&round_ops[..1]); + traces.keccak_rnd[1] = keccak_rnd::generate_keccak_rnd_trace(&round_ops[1..]); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "chunk assignment is free — a 1+2 split proves the same statement as 2+1" + ); +} + +/// The verifier builds its AIR set from the supplied chunk count, so a count +/// that disagrees with the proof's shape must be rejected — including zero, +/// which would drop `KECCAK_RND` and its constraints from the set. +/// +/// Two layers enforce this and the test does not distinguish them: the +/// explicit length check in `verify_against`, and the framework's own +/// AIR-count handling. Measured: deleting the explicit check leaves this test +/// green, so it pins the *behaviour*, not that particular guard. The guard +/// stays because it makes the shape contract local and legible, not because +/// this test would catch its removal. +#[test] +fn verify_rejects_a_chunk_count_that_does_not_match_the_proof() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + for wrong in [0usize, 1, 3, 14] { + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + wrong, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "chunk count {wrong} must not verify a 2-chunk proof" + ); + } +} + +/// What chunking costs: `KECCAK_RND` pads each chunk to its own power of two, +/// so the only overhead is padding, and splitting can even reduce it. +#[test] +fn chunking_cell_cost() { + let one = keccak_sponge_program(CHUNKED_SPONGE_LEN); + let two = chunked_sponge_program(); + let perms = one.groups.keccak.real_rows as u64; + + let (main_one, aux_one) = super::airs::lfm_cell_counts(&one); + let (main_two, aux_two) = super::airs::lfm_cell_counts(&two); + println!( + "{CHUNKED_SPONGE_LEN}-byte sponge, {perms} permutations:\n \ + 1 chunk rows {:?} main {main_one} aux {aux_one}\n \ + 2 chunks rows {:?} main {main_two} aux {aux_two}\n \ + delta main {} aux {}", + keccak_rnd_chunk_rows(&one), + keccak_rnd_chunk_rows(&two), + main_two as i64 - main_one as i64, + aux_two as i64 - aux_one as i64, + ); + + // KECCAK_RND rows are the only thing chunking moves; here 128 padded rows + // in one chunk versus 64 + 32 in two. + assert_eq!(keccak_rnd_chunk_rows(&one).iter().sum::(), 128); + assert_eq!(keccak_rnd_chunk_rows(&two).iter().sum::(), 96); + assert!( + main_two < main_one, + "this split lands on tighter power-of-two boundaries, so it is cheaper" + ); +} + +/// At the default policy's geometry chunking does not cost rows, it saves +/// them. A single table must pad to one power of two for the whole program; N +/// chunks each pad to their own, and every full chunk is within 8 rows of its +/// power of two by construction. +#[test] +fn default_policy_beats_a_single_table_at_wrap_scale() { + let c = KeccakChunking::default(); + let per = c.permutations_per_chunk(); + let full_chunk_rows = (per * 24).next_power_of_two(); + assert_eq!(full_chunk_rows, 1 << 19); + assert_eq!( + full_chunk_rows - per * 24, + 8, + "a full chunk wastes 8 rows of 524,288" + ); + + // The proof wrap this feature exists for. + const WRAP_PERMUTATIONS: usize = 460_000; + let chunks = c.chunk_count(WRAP_PERMUTATIONS); + assert_eq!(chunks, 22, "21 full chunks plus a partial one"); + + let chunked_rows: usize = (0..chunks) + .map(|i| { + let perms = WRAP_PERMUTATIONS.saturating_sub(i * per).min(per); + (perms * 24).next_power_of_two().max(4) + }) + .sum(); + let single_table_rows = (WRAP_PERMUTATIONS * 24).next_power_of_two(); + + println!( + "{WRAP_PERMUTATIONS} permutations: {chunks} chunks = {chunked_rows} rows, \ + single table = {single_table_rows} rows ({:.1}% saved)", + 100.0 * (1.0 - chunked_rows as f64 / single_table_rows as f64), + ); + assert!( + chunked_rows < single_table_rows, + "chunking must not cost more rows than one table would" + ); + // 2^24 rows at 1480 columns is also far past what one table can hold. + assert_eq!(single_table_rows, 1 << 24); +} + +// ================= R1f slice b: real continuation-proof bytes ================= + +use super::proof_fixture; + +/// Cache path for the fixture blob. Outside the repository on purpose: a +/// checked-in binary can drift from the encoder silently, so the generation path +/// is what a cold run exercises. +/// +/// ★ **Keyed on the configuration, not only on the name.** `load_or_generate` +/// hands back whatever sits at this path if the file exists, so a fixed name +/// meant the first run after any fixture change read the PREVIOUS +/// configuration's proof — four tests below failing on epoch count and passing +/// on a second run, which reads as flakiness rather than as a stale blob. Both +/// inputs that decide the blob's shape are in the name, so a change to either +/// simply misses the cache. +fn fixture_cache() -> std::path::PathBuf { + let elf = proof_fixture::FIXTURE_INNER_ELF.replace(['/', '\\', '.'], "_"); + // The format key keeps branches that encode differently off each other's + // blob — see `proof_fixture::cache_format_key`. + let fmt = proof_fixture::cache_format_key().replace(['/', '\\', '.'], "_"); + std::env::temp_dir().join(format!( + "lfm-r1f-continuation-fixture-{elf}-log2_{}-{fmt}.bin", + proof_fixture::FIXTURE_EPOCH_LOG2 + )) +} + +/// R1f(b): the machine's fixture is a REAL two-epoch continuation proof, encoded +/// by the same function that builds the recursion guest's private input. +#[test] +fn continuation_fixture_generates_two_epochs() { + let (blob, num_epochs) = proof_fixture::generate(); + println!( + "R1f fixture: inner={} epoch_log2={} epochs={} blob={} bytes", + proof_fixture::FIXTURE_INNER_ELF, + proof_fixture::FIXTURE_EPOCH_LOG2, + num_epochs, + blob.len() + ); + assert!( + proof_fixture::has_recursion_prefix(&blob), + "the blob must carry the recursion input wire format's magic prefix" + ); + assert!( + num_epochs >= 2, + "a CONTINUATION fixture needs more than one epoch, got {num_epochs} — \ + lower FIXTURE_EPOCH_LOG2" + ); + // Cache it for the slices that consume it. Atomic: this test runs in + // parallel with readers of the same path, and blobs are not reproducible, + // so a torn write would hand someone a truncated proof. + proof_fixture::write_cache(&fixture_cache(), &blob); +} + +/// R1f(a): the arena filler reads a REAL proof's committed roots out of the +/// guest's wire-format blob, in place, exactly as the recursion guest would. +#[test] +fn arena_filler_reads_real_committed_roots() { + use super::proof_arena; + use super::proof_fixture::FixtureArchive; + + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = FixtureArchive::open(&blob); + + let epochs = proof_arena::num_epochs(&archive); + assert_eq!(epochs, 2, "the fixture is a two-epoch continuation"); + + for epoch in 0..epochs { + let tables = proof_arena::epoch_num_tables(&archive, epoch); + let roots = proof_arena::epoch_main_roots(&archive, epoch); + assert_eq!(roots.len(), tables, "one main root per sub-proof"); + assert!(tables > 0, "epoch {epoch} must have sub-proofs"); + // Real commitments, not defaults: an all-zero root would mean the reader + // is looking at the wrong bytes rather than at the proof. + assert!( + roots.iter().all(|r| *r != [0u8; 32]), + "epoch {epoch}: every committed root must be nonzero" + ); + let halves = proof_arena::roots_to_halves(&roots); + assert_eq!(halves.len(), tables * proof_arena::ROOT_HALVES); + println!( + "R1f arena: epoch {epoch} -> {tables} sub-proofs, {} arena halves, output {} bytes", + halves.len(), + proof_arena::epoch_public_output(&archive, epoch).len() + ); + } +} + +/// Verifies the team lead's ruling premise directly against the blob: the +/// SUPPLIED preprocessed roots really are embedded, so replaying Phase A does +/// not need `build_epoch_airs` reachable. +/// +/// Checked here rather than taken on trust, because the whole leg's shape +/// depends on it. +#[test] +fn supplied_preprocessed_roots_are_embedded_in_the_blob() { + use super::proof_fixture::FixtureArchive; + + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = FixtureArchive::open(&blob); + let gi = archive.guest_input(); + + // DECODE: one commitment, directly in the guest input. + assert_ne!( + gi.decode_commitment, [0u8; 32], + "the DECODE root must be embedded and nonzero" + ); + // Per-page genesis roots: (base, commitment) pairs, also directly embedded. + println!( + "R1f supplied roots: decode present, {} page commitments", + gi.page_commitments.len() + ); + for pair in gi.page_commitments.iter() { + assert_ne!(pair.1, [0u8; 32], "page genesis roots must be nonzero"); + } +} + +// ============ R1f (c)+(d): a REAL Merkle opening, in the machine ============ +// +// Everything up to here ran on data this machine produced. This is the first +// leg that authenticates production-committed data: one FRI query's main-trace +// opening from a real two-epoch continuation proof, walked under the production +// keccak Merkle conventions, against that proof's own committed root. +// +// The oracle is the proof's root. Nothing here recomputes an expected answer +// with a local model and compares the machine against itself. + +use super::programs::{MerkleOpeningShape, keccak_merkle_opening_program, merkle_opening_program}; +use super::proof_arena::MainTraceOpening; + +/// Which opening the leg authenticates. +/// +/// Epoch 0's first sub-proof, chosen on measured grounds. Two things make it +/// the right target, and only the first is stable across blobs. +/// +/// **Depth.** It is one of exactly two depth-20 trees in the fixture (the other +/// is epoch 1's table 0); everything else is depth 7 or less, and half the +/// sub-proofs are depth 2. Depth is SHAPE, so it does not move when the blob +/// does — see `fixture_generation_is_not_reproducible`. +/// +/// **A unique leaf index.** Measured on one blob, 24 of the 49 sub-proofs have +/// exactly one index that verifies and 25 have several: a table whose trace is +/// mostly padding commits identical rows, so identical leaves sit under +/// identical subtrees and every index checks out. On one of those, "flip an +/// index bit" is not a tamper at all and the (d) vector would pass while +/// testing nothing. That split is blob-dependent, so it is NOT pinned as a +/// constant — `real_opening_is_a_usable_tamper_target` asserts uniqueness at +/// run time on whatever blob it is handed. +const R1F_EPOCH: usize = 0; +const R1F_TABLE: usize = 0; +const R1F_QUERY: usize = 0; + +/// The pinned shape, asserted against the real proof rather than read from it — +/// program shape is compile-time by construction, so if the fixture ever moves, +/// this must fail loudly rather than quietly recompile to a new program. +const R1F_SHAPE: MerkleOpeningShape = MerkleOpeningShape { + leaf_values: 20, + depth: 20, +}; + +/// The opening and its recovered leaf index, resolved once per test binary. +/// +/// The index costs a `2^depth` sweep (~4 s at depth 20) because `iota` is a +/// transcript challenge and is not in the proof; see +/// [`MainTraceOpening::indices_that_verify`]. Sharing it across the tests that +/// need it keeps that to one sweep. +fn r1f_opening() -> &'static (MainTraceOpening, usize) { + use std::sync::OnceLock; + static CELL: OnceLock<(MainTraceOpening, usize)> = OnceLock::new(); + CELL.get_or_init(|| { + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + let opening = MainTraceOpening::extract(&archive, R1F_EPOCH, R1F_TABLE, R1F_QUERY); + let hits = opening.indices_that_verify(); + assert_eq!( + hits.len(), + 1, + "the authenticated opening must sit at exactly one index, else the \ + index-tamper vector tests nothing; got {hits:?}" + ); + (opening, hits[0]) + }) +} + +fn merkle_arenas(opening: &MainTraceOpening, index: usize) -> Vec> { + vec![ + opening.leaf_arena(), + opening.sibling_arena(), + vec![super::word::base_word(FE::from(index as u64))], + opening.root_arena(), + ] +} + +/// Scrutinises the oracle before anything is built on it: the opening really is +/// what the leg assumes, and PRODUCTION's own path check accepts it. +#[test] +fn real_opening_is_a_usable_tamper_target() { + let (opening, index) = r1f_opening(); + assert_eq!( + opening.depth(), + R1F_SHAPE.depth, + "the fixture's tree depth moved; R1F_SHAPE is program shape and must be updated deliberately" + ); + assert_eq!( + opening.values.len(), + R1F_SHAPE.leaf_values, + "the fixture's column count moved" + ); + assert_eq!(opening.num_columns, R1F_SHAPE.columns()); + assert!( + opening.verifies_at(*index), + "production's own checker must accept the opening we are about to \ + authenticate in the machine" + ); + assert!( + !opening.verifies_at(index ^ 1), + "flipping the low index bit must break production's check" + ); + println!( + "R1f target: epoch {R1F_EPOCH} table {R1F_TABLE} query {R1F_QUERY} — \ + {} columns, row pair = {} values, depth {}, index {index}", + opening.num_columns, + opening.values.len(), + opening.depth() + ); +} + +/// ★ The headline: the machine walks a real opening to a real committed root, +/// PROVED and verified. +/// +/// Two independent things are checked. The published root equals the root the +/// proof committed to — that is the authentication, and its oracle is the proof +/// itself. And the machine proof verifies against those published words — that +/// is what makes it a proof rather than an execution, which per method rule 2 +/// is the only thing that says anything about the chips. +#[test] +fn the_merkle_walk_authenticates_a_real_opening() { + let opts = options(); + let (opening, index) = r1f_opening(); + // The PRODUCTION twin: the opening is real, so the walk has to re-derive a + // root the host actually built. The keccak instrument cannot, and the name + // moved with the hash rather than outliving it. + let program = merkle_opening_program(R1F_SHAPE); + // Built at `WrapHash::production()`, so it emits `Instr::Hash` and must be + // proved under the pin's tenant — the classification rule in HASH-PINNING.md. + let artifacts = super::registry::build_artifacts_with_hasher( + &program, + &opts, + crate::hash_pin::BLOCK_HASHER, + ); + let proved = lfm_prove(&program, &artifacts, &merkle_arenas(opening, *index), &opts) + .expect("the honest opening must execute and prove"); + + assert_eq!( + digest_bytes(&proved.public_words), + opening.root, + "the walked root must be the root the proof committed to" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the authenticated opening must verify" + ); +} + +/// One tamper vector: corrupted arenas, plus the root those arenas really fold +/// to — which is what lets the same vector be run both incoherently (claiming +/// the real root) and coherently (claiming its own). +struct TamperVector { + what: &'static str, + arenas: Vec>, + root: [u8; 32], +} + +/// ★ (d) Tamper, both ways round, for all three inputs the walk consumes. +/// +/// INCOHERENT: change one input and leave the claimed root alone. The +/// in-machine root assert makes the program unexecutable — the earliest and +/// loudest failure, and the one that shows the assert is load-bearing. +/// +/// COHERENT (method rule 4): change the input AND supply the root that input +/// really folds to, so every value in the run is consistent with every other, +/// nothing asserts, and a proof comes out. The forgery then fails on the one +/// thing it cannot fake — the published root is not the root the proof +/// committed to, so a verifier claiming the real one rejects. +#[test] +fn tampered_merkle_opening_rejects() { + let opts = options(); + let (opening, index) = r1f_opening(); + // Same production twin as the honest-path test above — a tamper control is + // only a control over the walk the honest path uses. + let program = merkle_opening_program(R1F_SHAPE); + // Built at `WrapHash::production()`, so it emits `Instr::Hash` and must be + // proved under the pin's tenant — the classification rule in HASH-PINNING.md. + let artifacts = super::registry::build_artifacts_with_hasher( + &program, + &opts, + crate::hash_pin::BLOCK_HASHER, + ); + let honest = lfm_prove(&program, &artifacts, &merkle_arenas(opening, *index), &opts) + .expect("honest prove"); + + let mut vectors: Vec = Vec::new(); + + // 1. A wrong sibling at the leaf level. + { + let mut siblings = opening.siblings.clone(); + siblings[0][0] ^= 1; + let mut arenas = merkle_arenas(opening, *index); + arenas[1] = siblings + .iter() + .flat_map(super::proof_arena::commitment_words) + .collect(); + let root = super::proof_arena::walk_to_root(opening.leaf_hash(), *index, &siblings); + vectors.push(TamperVector { + what: "wrong sibling", + arenas, + root, + }); + } + + // 2. Wrong index bits: the same leaf and the same path, walked in the other + // order at level 0. + { + let bad = index ^ 1; + let arenas = merkle_arenas(opening, bad); + let root = super::proof_arena::walk_to_root(opening.leaf_hash(), bad, &opening.siblings); + vectors.push(TamperVector { + what: "wrong index bits", + arenas, + root, + }); + } + + // 3. A wrong opened value: one field element of the row pair. + { + let mut tampered = MainTraceOpening { + root: opening.root, + values: opening.values.clone(), + num_columns: opening.num_columns, + siblings: opening.siblings.clone(), + }; + tampered.values[0] = &tampered.values[0] + FE::from(1u64); + let mut arenas = merkle_arenas(opening, *index); + arenas[0] = tampered.leaf_arena(); + let root = + super::proof_arena::walk_to_root(tampered.leaf_hash(), *index, &tampered.siblings); + vectors.push(TamperVector { + what: "wrong leaf value", + arenas, + root, + }); + } + + for TamperVector { + what, + arenas, + root: forged_root, + } in vectors + { + assert_ne!( + forged_root, opening.root, + "{what}: the tamper must actually move the root, or the vector is vacuous" + ); + + // Incoherent: still claiming the real root. + let err = super::executor::execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .err() + .unwrap_or_else(|| panic!("{what}: claiming the real root must not execute")); + println!("R1f tamper {what}: incoherent run rejected with {err:?}"); + + // Coherent: claim the root the tampered inputs really reach. + let mut coherent = arenas; + coherent[3] = super::proof_arena::commitment_words(&forged_root).to_vec(); + let proved = lfm_prove(&program, &artifacts, &coherent, &opts) + .unwrap_or_else(|e| panic!("{what}: the coherent forgery must prove: {e:?}")); + assert_eq!( + digest_bytes(&proved.public_words), + forged_root, + "{what}: the coherent forgery must publish its own root" + ); + assert_ne!( + proved.public_words, honest.public_words, + "{what}: the forgery must not publish the honest root" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &honest.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{what}: claiming the real committed root for a forged walk must reject" + ); + } +} + +/// Main-trace cells one byteswap costs: one `LFM_BITDEC` row and 64 `LFM_BALU` +/// rows, each at its chip's non-preprocessed width — the same accounting +/// [`super::airs::lfm_cell_counts`] uses. +pub(super) fn byteswap_cells() -> u64 { + use super::chips::{balu, bitdec}; + use super::layout; + let bitdec_w = (bitdec::cols::NUM_COLUMNS - layout::bitdec::PREP_WIDTH) as u64; + let balu_w = (balu::cols::NUM_COLUMNS - layout::balu::PREP_WIDTH) as u64; + bitdec_w + 64 * balu_w +} + +/// ★ Rows the CONFIGURED wrap hash occupies in a program. +/// +/// Structural counts — "a Merkle tree over `n` leaves costs `2n − 1` hashes" — +/// are claims about tree shape, not about a hash, so they must be counted +/// against whichever chip group the configured hash fills. Reading +/// `groups.keccak` directly made them silently read ZERO the moment production +/// moved to BLAKE3, which reports a true structural claim as a failed one. +pub(super) fn wrap_hash_rows(program: &super::compiler::LfmProgram) -> usize { + wrap_hash_rows_at(program, super::edsl::WrapHash::production()) +} + +/// [`wrap_hash_rows`] against a NAMED hash rather than the production pin. +/// +/// ★ For the programs that are ABOUT a hash. `WrapHash::production`'s own doc +/// carves them out — the R1b/R1c/R1d instruments name their hash directly and +/// must keep doing so — and a count taken over such a program has to name the +/// same one, or it reads zero the moment the pin moves away from it and reports +/// a true structural claim as a failed one. That is not hypothetical: it is +/// exactly what the RPX pin did to `transcript_replay_cell_counts`, whose script +/// is a BYTE sponge (a 136-byte segment spanning two rate blocks) and has no +/// meaning on the algebraic arm at all. +pub(super) fn wrap_hash_rows_at( + program: &super::compiler::LfmProgram, + hash: super::edsl::WrapHash, +) -> usize { + match hash { + super::edsl::WrapHash::Keccak => program.groups.keccak.real_rows, + super::edsl::WrapHash::Blake3 => program.groups.blake3.real_rows, + // The algebraic wrap hash IS the socket, so its rows are the hash + // chip's — the same table `HasherKind` selects the permutation for. + super::edsl::WrapHash::Algebraic | super::edsl::WrapHash::Poseidon1 => { + program.groups.hash.real_rows + } + } +} + +/// ★ Byteswaps one leaf VALUE costs under the CONFIGURED wrap hash. +/// +/// A byte hash renders every felt big-endian before absorbing it — one +/// `LFM_BITDEC` row plus 64 `LFM_BALU` rows apiece. An algebraic leaf packs +/// felts straight into the rate (`edsl::algebraic_leaf_hash` emits no `BitDec` +/// at all), so the rendering does not exist and the count is ZERO rather than +/// smaller. Closed forms over leaf values multiply by this for the same reason +/// they multiply by `proof_arena::words_per_root`: the shape of the claim is +/// hash-independent, the width of a term is not. +pub(super) fn byteswaps_per_value() -> usize { + usize::from(super::edsl::WrapHash::production().byte_hash().is_some()) +} + +/// [`wrap_hash_rows`]'s instruction-stream twin: emitted compressions of the +/// configured wrap hash. +pub(super) fn wrap_hash_instrs(program: &super::compiler::LfmProgram) -> usize { + use super::instr::Instr; + program + .instrs + .iter() + .filter(|i| match super::edsl::WrapHash::production() { + super::edsl::WrapHash::Keccak => matches!(i, Instr::KeccakF(_)), + super::edsl::WrapHash::Blake3 => matches!(i, Instr::Blake3(_)), + super::edsl::WrapHash::Algebraic => matches!(i, Instr::Hash { .. }), + super::edsl::WrapHash::Poseidon1 => matches!(i, Instr::Hash16(_)), + }) + .count() +} + +/// Main-trace cells one keccak permutation costs: the `LFM_KECCAK` row that +/// requests it, plus the 24 `KECCAK_RND` rounds that carry it. +pub(super) fn permutation_cells() -> u64 { + use super::chips::keccak; + use super::chunking::KECCAK_RND_ROWS_PER_PERMUTATION as ROUNDS; + use super::layout; + use crate::tables::keccak_rnd; + let keccak_w = (keccak::cols::NUM_COLUMNS - layout::keccak::PREP_WIDTH) as u64; + keccak_w + ROUNDS as u64 * keccak_rnd::cols::NUM_COLUMNS as u64 +} + +/// Pins the widths the cost model above is built on. +/// +/// Not ceremony: the inline `// 52 / 252 / 388 / 588 / 788` comments on +/// `chips::keccak::cols` were stale by 4 (R1d widened `PREP_WIDTH` for the +/// reversed-digest columns and they were not updated), and reading them instead +/// of the constants is what produced a wrong per-permutation figure on the first +/// pass through this measurement. A wrong width silently rescales every cell +/// number in `keccak_merkle_opening_cost`, so the widths get an assertion of +/// their own rather than a comment. +#[test] +fn cost_model_widths_are_what_the_chips_declare() { + use super::chips::{balu, bitdec, keccak}; + use super::layout; + use crate::tables::keccak_rnd; + assert_eq!(keccak::cols::NUM_COLUMNS, 792, "LFM_KECCAK total width"); + assert_eq!(layout::keccak::PREP_WIDTH, 56, "LFM_KECCAK preprocessed"); + assert_eq!(balu::cols::NUM_COLUMNS - layout::balu::PREP_WIDTH, 4); + assert_eq!(bitdec::cols::NUM_COLUMNS - layout::bitdec::PREP_WIDTH, 66); + assert_eq!(keccak_rnd::cols::NUM_COLUMNS, 1480); + assert_eq!(byteswap_cells(), 322, "66 + 64 x 4"); + assert_eq!(permutation_cells(), 36_256, "736 + 24 x 1480"); +} + +/// ★ The leg's headline measurement — and it REFUTES the prediction it was set +/// up to confirm. +/// +/// The R1f handoff predicted that byteswapping the opened values would dominate +/// the leaf, "not the hashing", on the strength of the row counts: a 10-column +/// table pays 20 `LFM_BITDEC` + 1280 `LFM_BALU` rows of byteswapping against +/// only 22 permutations. Those row counts are right. The conclusion drawn from +/// them is wrong, because rows of different chips are not comparable units. +/// +/// A byteswap's rows are narrow — `LFM_BALU` carries 4 non-preprocessed columns +/// — while a permutation expands into 24 `KECCAK_RND` rounds at 1480 columns +/// each. Priced in main-trace cells, the unit the proof actually pays in, the +/// measured figures are **322 cells per byteswap against 36,256 per +/// permutation, a factor of 113**. Hashing then dominates at every width in the +/// fixture: 124× at the 10-column table this leg authenticates, 8.9× at 511 +/// columns, 7.4× at 1480. The crossover this test was written to find does not +/// exist. Both terms are linear in the column count — `2c` byteswaps against +/// `≈16c/136` rate blocks — so the ratio flattens near 6.6× rather than +/// inverting. +/// +/// This is why a byteswap chiplet is NOT the lever it looked like, and the +/// measurement rather than the intuition is what says so. The attribution is +/// MARGINAL (real rows, not padded), so it answers "what does one more column +/// cost" and not "what does this proof cost"; the whole-program figure is +/// printed alongside because the fixed floor — `BITWISE` is 2^20 rows whatever +/// the program does — dwarfs both terms at these sizes. +#[test] +fn keccak_merkle_opening_cost() { + let (opening, _) = r1f_opening(); + println!( + "one byteswap = {} main cells; one permutation = {} main cells ({:.0}x)", + byteswap_cells(), + permutation_cells(), + permutation_cells() as f64 / byteswap_cells() as f64, + ); + println!("shape instrs keccak bitdec balu select lanes"); + let mut shapes = vec![R1F_SHAPE]; + // Two wider tables from the same fixture, to show the scaling rather than + // assert a single point. 511 and 1480 columns are real widths in it. + for columns in [511usize, 1480] { + shapes.push(MerkleOpeningShape { + leaf_values: 2 * columns, + depth: R1F_SHAPE.depth, + }); + } + for shape in &shapes { + let program = keccak_merkle_opening_program(*shape); + println!( + "{:>4} cols d={:<3} {:>7} {:>7} {:>7} {:>7} {:>7} {:>7}", + shape.columns(), + shape.depth, + program.instrs.len(), + program.groups.keccak.real_rows, + program.groups.bitdec.real_rows, + program.groups.balu.real_rows, + program.groups.select.real_rows, + program.groups.lanes.real_rows, + ); + } + + // The same three shapes priced in main-trace cells, which is where the + // prediction inverts. `swap` counts only the byteswapping; `hash` counts + // every permutation (leaf blocks and walk levels alike). + println!("shape swap cells hash cells hash/swap whole program"); + for shape in &shapes { + let program = keccak_merkle_opening_program(*shape); + let swap = shape.leaf_values as u64 * byteswap_cells(); + let hash = program.groups.keccak.real_rows as u64 * permutation_cells(); + let (main, _aux) = super::airs::lfm_cell_counts(&program); + println!( + "{:>4} cols d={:<3} {:>12} {:>12} {:>11.1} {:>15}", + shape.columns(), + shape.depth, + swap, + hash, + hash as f64 / swap as f64, + main, + ); + assert!( + hash > swap, + "{} columns: hashing must dominate — if this ever flips, the \ + byteswap-chiplet argument becomes live and the docs above are stale", + shape.columns() + ); + } + + // Pin the real shape's decomposition, so a regression in either half shows. + let program = keccak_merkle_opening_program(R1F_SHAPE); + let leaf_bytes = 8 * R1F_SHAPE.leaf_values; + let leaf_perms = super::keccak_host::num_blocks(leaf_bytes); + assert_eq!( + program.groups.keccak.real_rows, + leaf_perms + R1F_SHAPE.depth, + "one permutation per rate block of the leaf, plus one per level" + ); + assert_eq!( + program.groups.bitdec.real_rows, + R1F_SHAPE.leaf_values + 1, + "one decomposition per opened value, plus one for the index" + ); + assert_eq!( + program.groups.balu.real_rows, + 8 * 2, + "the byteswaps are bus sends off the BitDec rows now, so only the two \ + root asserts remain (4 sub + 4 div each)" + ); + assert_eq!( + program.groups.select.real_rows, + 2 * R1F_SHAPE.depth, + "two selects per level: a digest is two words and both swap together" + ); + println!( + "R1f leaf: {} values -> {leaf_bytes} bytes -> {leaf_perms} permutations, \ + against {} bitdec rows (and ZERO balu rows) of byteswapping", + R1F_SHAPE.leaf_values, R1F_SHAPE.leaf_values, + ); + // The fixed floor, for scale: BITWISE alone is 2^20 rows regardless of what + // the program does, so nothing above is a claim about total proof cost. + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!("R1f whole program: {main} main cells, {aux} aux cells"); + assert_eq!(opening.values.len(), R1F_SHAPE.leaf_values); +} + +/// ⚠ STANDING EVIDENCE that the fixture is not reproducible. +/// +/// Two `generate()` calls on identical inputs — same ELF, same empty input, +/// same epoch size, same options — produce different blobs, and the difference +/// is semantic rather than archive padding: sub-proof roots move, which moves +/// the Fiat-Shamir challenges, which opens different leaves. Measured at ~65k +/// of 587k bytes differing. +/// +/// This is why R1f pins SHAPE (`R1F_SHAPE`) and RECOVERS per-blob values +/// (`r1f_opening`) instead of pinning a query index or a root. A pinned index +/// would pass for exactly as long as the cache survived and then fail on the +/// next cold run, which is the worst possible failure mode for a fixture. +/// +/// `#[ignore]`d because it costs two full continuation proofs (~17 s) and +/// asserts nothing the rest of the suite depends on. Run it with +/// `--ignored` when the question comes up again; it is here so the claim has +/// evidence attached rather than being folklore in a status log. +#[test] +#[ignore] +fn fixture_generation_is_not_reproducible() { + let (a, _) = proof_fixture::generate(); + let (b, _) = proof_fixture::generate(); + let differing = a.iter().zip(b.iter()).filter(|(x, y)| x != y).count(); + println!( + "two fixture generations: {} vs {} bytes, {differing} differing", + a.len(), + b.len() + ); + assert_ne!( + a, b, + "if this ever passes, the prover became reproducible \ + and the no-pinning rule in proof_fixture can be relaxed" + ); + + // The difference reaches the committed data, so it is not archive padding. + let (aa, bb) = ( + super::proof_fixture::FixtureArchive::open(&a), + super::proof_fixture::FixtureArchive::open(&b), + ); + assert_ne!( + super::proof_arena::epoch_main_roots(&aa, 0), + super::proof_arena::epoch_main_roots(&bb, 0), + "the divergence must be semantic — if the roots match, this is padding \ + and the no-pinning rule is too strong" + ); + + // The tree this leg authenticates keeps its SHAPE across runs, which is the + // property R1F_SHAPE relies on, while its opened values do not. + let oa = MainTraceOpening::extract(&aa, 0, 0, 0); + let ob = MainTraceOpening::extract(&bb, 0, 0, 0); + assert_eq!( + oa.depth(), + ob.depth(), + "tree depth is shape and must be stable" + ); + assert_eq!( + oa.num_columns, ob.num_columns, + "column count is shape and must be stable" + ); + println!( + "table 0 across runs: depth {} stable, same root = {}, same opened values = {}", + oa.depth(), + oa.root == ob.root, + oa.values == ob.values + ); +} + +// ============ R1g (ii): the cross-epoch L2G commitment binding ============ +// +// The first obligation of the chaining leg, and the first time the machine +// reads ACROSS structures: each epoch's own committed L2G root against the +// corresponding sub-proof of the one global proof. R1f stayed inside a single +// epoch's own sub-proof. + +use super::programs::l2g_binding_program; + +/// Epochs the binding program is compiled for. SHAPE, not a blob-derived +/// constant: the epoch count follows from the inner ELF and +/// `FIXTURE_EPOCH_LOG2`, not from anything the prover chooses per run. Asserted +/// against the real bundle rather than read from it, so a fixture change is +/// loud — the same discipline `R1F_SHAPE` uses. +const R1G_EPOCHS: usize = 2; + +/// The `i`-th 32-byte root in a program's published words, at the root's own +/// width — the L2G binding follows the configuration, so a root is +/// `words_per_root()` words: two byte words or one algebraic word. +fn published_root(public: &[(u32, LfmWord)], i: usize) -> [u8; 32] { + let w = super::proof_arena::words_per_root(); + digest_bytes(&public[w * i..w * (i + 1)]) +} + +fn l2g_arenas( + epoch: &[stark::config::Commitment], + global: &[stark::config::Commitment], +) -> Vec> { + use super::proof_arena::commitments_to_arena; + vec![commitments_to_arena(epoch), commitments_to_arena(global)] +} + +/// The real bundle's L2G roots, resolved once. +fn r1g_l2g_roots() -> &'static ( + Vec, + Vec, +) { + use std::sync::OnceLock; + static CELL: OnceLock<( + Vec, + Vec, + )> = OnceLock::new(); + CELL.get_or_init(|| { + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + let epoch = super::proof_arena::epoch_l2g_roots(&archive); + let global = super::proof_arena::global_l2g_roots(&archive, epoch.len()); + (epoch, global) + }) +} + +/// Scrutinises the oracle before building on it: production's binding really +/// does hold on the real bundle, and — the part that matters for (d) — the +/// per-epoch roots are DISTINCT, so swapping two of them is a real tamper. +/// +/// Without that second check the position-swap vector would pass while testing +/// nothing, exactly as an index-bit flip would have on a degenerate tree in R1f. +#[test] +fn l2g_binding_holds_on_the_real_bundle() { + let (epoch, global) = r1g_l2g_roots(); + assert_eq!( + epoch.len(), + R1G_EPOCHS, + "the fixture's epoch count moved; R1G_EPOCHS is program shape" + ); + assert_eq!(epoch, global, "production's own L2G binding must hold"); + assert!( + epoch.iter().all(|r| *r != [0u8; 32]), + "every L2G root must be nonzero" + ); + for i in 0..epoch.len() { + for j in (i + 1)..epoch.len() { + assert_ne!( + epoch[i], epoch[j], + "epochs {i} and {j} share an L2G root, so swapping them is not a tamper" + ); + } + } + println!( + "R1g(ii): {} epochs, binding holds, roots pairwise distinct", + epoch.len() + ); +} + +/// ★ The binding, emitted and PROVED against the real bundle. +#[test] +fn l2g_binding_proves_and_verifies() { + let opts = options(); + let (epoch, global) = r1g_l2g_roots(); + let program = l2g_binding_program(R1G_EPOCHS); + // A production() program: proved under the pin's tenant, as above. + let artifacts = super::registry::build_artifacts_with_hasher( + &program, + &opts, + crate::hash_pin::BLOCK_HASHER, + ); + let proved = lfm_prove(&program, &artifacts, &l2g_arenas(epoch, global), &opts) + .expect("the honest binding must execute and prove"); + + for (i, root) in epoch.iter().enumerate().take(R1G_EPOCHS) { + assert_eq!( + published_root(&proved.public_words, i), + *root, + "published root {i} must be epoch {i}'s committed L2G root" + ); + } + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the L2G binding must verify" + ); +} + +/// ★ (d) Tamper. Same two-mode structure as R1f: incoherent trips the assert, +/// coherent satisfies every assert and then fails on the published roots. +/// +/// The SWAP vector is the one that matters — it is what "position-sensitive" +/// means. Epoch `i`'s root must meet global sub-proof `i` and no other, so a +/// bundle whose L2G roots are correct as a SET but wrong in ORDER must reject. +#[test] +fn tampered_l2g_binding_rejects() { + let opts = options(); + let (epoch, global) = r1g_l2g_roots(); + let program = l2g_binding_program(R1G_EPOCHS); + // A production() program: proved under the pin's tenant, as above. + let artifacts = super::registry::build_artifacts_with_hasher( + &program, + &opts, + crate::hash_pin::BLOCK_HASHER, + ); + let honest = + lfm_prove(&program, &artifacts, &l2g_arenas(epoch, global), &opts).expect("honest prove"); + + // Incoherent: one side changed, the other left honest — the assert must fire. + // + // The two byte positions are chosen, not arbitrary. A digest spans TWO + // machine words (bytes 0-15 and 16-31) and each needs its own + // `assert_word_eq`; vectors that all land in byte 0 would leave an emitter + // that compares only the first word completely uncaught. Byte 31 covers the + // second word. Falsification F32 confirmed the gap was real before this. + let mut bad_epoch = epoch.clone(); + bad_epoch[0][0] ^= 1; + let mut bad_global = global.clone(); + bad_global[1][31] ^= 1; + let swapped_one_side = { + let mut s = epoch.clone(); + s.swap(0, 1); + s + }; + for (what, arenas) in [ + ("wrong epoch root", l2g_arenas(&bad_epoch, global)), + ("wrong global root", l2g_arenas(epoch, &bad_global)), + ( + "epoch roots swapped on one side", + l2g_arenas(&swapped_one_side, global), + ), + ] { + let err = super::executor::execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .err() + .unwrap_or_else(|| panic!("{what}: must not execute")); + println!("R1g tamper {what}: rejected with {err:?}"); + } + + // Coherent: BOTH sides swapped consistently. Every assert passes — the + // bundle's roots are the right set — but the order is wrong, so the + // published roots are not the ones the real bundle commits to. + let mut swapped = epoch.clone(); + swapped.swap(0, 1); + let proved = lfm_prove(&program, &artifacts, &l2g_arenas(&swapped, &swapped), &opts) + .expect("the coherent swap must prove — every assert is satisfied"); + assert_eq!( + published_root(&proved.public_words, 0), + epoch[1], + "the coherent forgery publishes the swapped order" + ); + assert_ne!( + proved.public_words, honest.public_words, + "a reordered binding must not publish the honest roots" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &honest.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "claiming the real per-epoch roots for a reordered binding must reject" + ); +} + +// ============ R1g (iii): the attestation's program id ============ + +use super::programs::{ProgramIdShape, program_id_program}; + +/// Arena for a `program_id` fold, each field in its own halves. +fn program_id_arenas( + elf_digest: &[u8; 32], + pc_start: u64, + decode: &stark::config::Commitment, + pages: &[(u64, stark::config::Commitment)], +) -> Vec> { + let mut halves = keccak_host::pack_stream(elf_digest); + halves.extend(keccak_host::pack_stream(&pc_start.to_le_bytes())); + halves.extend(keccak_host::pack_stream(decode)); + for (base, c) in pages { + halves.extend(keccak_host::pack_stream(&base.to_le_bytes())); + halves.extend(keccak_host::pack_stream(c)); + } + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +/// The real fixture's program-id inputs. +fn r1g_program_id_inputs() -> ( + [u8; 32], + u64, + stark::config::Commitment, + Vec<(u64, stark::config::Commitment)>, +) { + use super::proof_arena; + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + let elf_bytes = proof_arena::inner_elf(&archive).to_vec(); + let elf = executor::elf::Elf::load(&elf_bytes).expect("the fixture's inner ELF must load"); + ( + crate::statement::elf_digest(&elf_bytes), + elf.entry_point, + proof_arena::decode_commitment(&archive), + proof_arena::page_commitments(&archive), + ) +} + +/// ★ The fold, PROVED, bit-exact against production's own `program_id_from_digest`. +/// +/// The oracle is the production function, not a local re-implementation. +#[test] +fn program_id_matches_production_on_the_real_fixture() { + let opts = options(); + let (elf_digest, pc_start, decode, pages) = r1g_program_id_inputs(); + assert!( + pages.is_empty(), + "the continuation fixture is expected to touch no data pages; if this \ + changes, the shape below must change with it" + ); + let shape = ProgramIdShape { + num_pages: pages.len(), + }; + let program = program_id_program(shape); + let artifacts = build_artifacts(&program, &opts); + let arenas = program_id_arenas(&elf_digest, pc_start, &decode, &pages); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + + let expected = crate::recursion::program_id_from_digest(&elf_digest, pc_start, &decode, &pages); + assert_eq!( + digest_bytes(&proved.public_words), + expected, + "the machine's program id must equal production's" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the program-id fold must verify" + ); + println!( + "R1g(iii): {} pages, {} bytes hashed, tag {} bytes (shift {})", + shape.num_pages, + shape.byte_len(), + crate::recursion::PROGRAM_ID_TAG.len(), + crate::recursion::PROGRAM_ID_TAG.len() % 4, + ); +} + +/// ★ The page loop, exercised. The fixture has ZERO page commitments, so the +/// sorted-page path is present-but-untested on real data — the caveat the team +/// lead flagged for the supplied roots applies here too. This drives it with a +/// synthetic shape against the same production oracle, so "it compiles" is not +/// mistaken for "it is covered". +/// +/// Proved, not just executed: the fold's byte length changes with the page +/// count, which moves every padding position, and only a proof sees the keccak +/// chip agree with the executor about that. +#[test] +fn program_id_folds_pages_in_the_production_layout() { + let opts = options(); + let (elf_digest, pc_start, decode, _) = r1g_program_id_inputs(); + for num_pages in [1usize, 3] { + let pages: Vec<(u64, stark::config::Commitment)> = (0..num_pages) + .map(|i| { + let base = 0x1000u64 * (i as u64 + 1); + let mut c = [0u8; 32]; + for (j, b) in c.iter_mut().enumerate() { + *b = (17 * i + j) as u8; + } + (base, c) + }) + .collect(); + let shape = ProgramIdShape { num_pages }; + let program = program_id_program(shape); + let artifacts = build_artifacts(&program, &opts); + let arenas = program_id_arenas(&elf_digest, pc_start, &decode, &pages); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + let expected = + crate::recursion::program_id_from_digest(&elf_digest, pc_start, &decode, &pages); + assert_eq!( + digest_bytes(&proved.public_words), + expected, + "{num_pages} pages: the machine's fold must match production's" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{num_pages} pages: the fold must verify" + ); + } +} + +/// ★ (d) Tamper: every folded field must move the id. +/// +/// Coherent by construction — nothing asserts, so each forgery proves cleanly +/// and fails on the published id, which is the whole mechanism: the id IS the +/// claim, and a consumer comparing against its own recompute rejects. +#[test] +fn tampered_program_id_inputs_change_the_id() { + let opts = options(); + let (elf_digest, pc_start, decode, _) = r1g_program_id_inputs(); + let pages = vec![(0x1000u64, [7u8; 32]), (0x2000u64, [9u8; 32])]; + let shape = ProgramIdShape { + num_pages: pages.len(), + }; + let program = program_id_program(shape); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove( + &program, + &artifacts, + &program_id_arenas(&elf_digest, pc_start, &decode, &pages), + &opts, + ) + .expect("honest prove"); + + let mut d2 = elf_digest; + d2[31] ^= 1; + let mut dec2 = decode; + dec2[0] ^= 1; + let mut pages_value = pages.clone(); + pages_value[1].1[31] ^= 1; + let mut pages_base = pages.clone(); + pages_base[0].0 ^= 1; + let mut pages_order = pages.clone(); + pages_order.swap(0, 1); + + for (what, arenas) in [ + ( + "elf digest", + program_id_arenas(&d2, pc_start, &decode, &pages), + ), + ( + "entry point", + program_id_arenas(&elf_digest, pc_start ^ 1, &decode, &pages), + ), + ( + "decode root", + program_id_arenas(&elf_digest, pc_start, &dec2, &pages), + ), + ( + "page commitment", + program_id_arenas(&elf_digest, pc_start, &decode, &pages_value), + ), + ( + "page base", + program_id_arenas(&elf_digest, pc_start, &decode, &pages_base), + ), + ( + "page ORDER", + program_id_arenas(&elf_digest, pc_start, &decode, &pages_order), + ), + ] { + let forged = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a change must move the program id" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{what}: claiming the honest id must reject" + ); + } +} + +// ============ R1g (i): the REGISTER preprocessed derivation ============ + +use super::programs::{RegisterDerivationShape, lde_probe_program, register_derivation_program}; + +/// The inner proof's blowup for the real target (`lfm-RESUME.md`), and the +/// sweep the leg was asked to measure. +const INNER_BLOWUPS: [usize; 3] = [2, 4, 8]; + +/// Every production `ProofOptions` uses this offset. +const PRODUCTION_COSET_OFFSET: u64 = 3; + +fn derivation_shape(blowup: usize) -> RegisterDerivationShape { + RegisterDerivationShape { + blowup, + coset_offset: PRODUCTION_COSET_OFFSET, + rows_per_leaf: stark::commitment::ROWS_PER_LEAF, + } +} + +/// The inner proof's options at a given blowup — the ones whose REGISTER +/// commitment is being derived, which are NOT the LFM proof's own options. +fn inner_options(blowup: usize) -> ProofOptions { + let opts = GoldilocksCubicProofOptions::with_blowup(blowup as u8).expect("inner options"); + assert_eq!( + opts.coset_offset, PRODUCTION_COSET_OFFSET, + "the shape constant must track production's coset offset" + ); + opts +} + +/// Deterministic pseudo-random `u32`s (splitmix64, high word) — a register file +/// whose rows are pairwise distinct and all nonzero, asserted by +/// [`the_fixture_register_boundary_is_mostly_zeros`] rather than assumed. +fn synthetic_register_file(seed: u64) -> Vec { + let mut state = seed; + (0..crate::tables::register::NUM_REGISTER_ADDRESSES) + .map(|_| { + state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + ((z ^ (z >> 31)) >> 32) as u32 + }) + .collect() +} + +fn register_arenas(init: &[u32], fini: &[u32]) -> Vec> { + let column = |v: &[u32]| { + v.iter() + .map(|&x| super::word::base_word(FE::from(x as u64))) + .collect() + }; + vec![column(init), column(fini)] +} + +/// ★ Oracle scrutiny, before anything is built on it: the row order this leg +/// restates really is production's. +/// +/// `register_word_address_list` is private, so the program assembles the same +/// list from the public per-register helper. This pins the result against the +/// layout the table's own docs state (x0–x31 at 0..63, x254 at 508–509 since +/// main made the commit index two words, x255 at 510–511) — a check with teeth +/// only because it is written from the docs rather than from the assembly +/// under test. +#[test] +fn the_register_offset_column_is_productions_row_order() { + use crate::tables::register::{ + NUM_REGISTER_ADDRESSES, PC_HI_INDEX, PC_LO_INDEX, X254_HI_INDEX, X254_INDEX, + register_word_addresses, + }; + let mut expected: Vec = (0..64).collect(); + expected.extend([508, 509, 510, 511]); + assert_eq!(expected.len(), NUM_REGISTER_ADDRESSES); + + let mut derived = Vec::new(); + for reg in 0..32u8 { + derived.extend(register_word_addresses(reg)); + } + derived.extend(register_word_addresses(254)); + derived.extend(register_word_addresses(255)); + assert_eq!( + derived, expected, + "the REGISTER row order moved; the derivation's OFFSET column is built \ + from this list and every derived root depends on it" + ); + // The positional constants the table exports must agree with it too. + assert_eq!(derived[X254_INDEX], 508); + assert_eq!(derived[X254_HI_INDEX], 509); + assert_eq!(derived[PC_LO_INDEX], 510); + assert_eq!(derived[PC_HI_INDEX], 511); +} + +/// ★ The emitted LDE against production's own `interpolate_fft` + +/// `evaluate_polynomial_on_lde_domain`, at sizes and offsets production never +/// takes. +/// +/// The register leg only ever runs the transform at `n = 128`, offset 3. Per +/// `lfm-target-shape.md`'s degenerate-parameter rule, that means the real +/// differential cannot tell a general emitter from one that is accidentally +/// right there, so the synthetic sizes are the only witness. Execute-only: this +/// is pure `LFM_BALU` arithmetic and the proved test below covers the chips. +#[test] +fn the_emitted_lde_matches_productions_transform() { + use math::polynomial::Polynomial; + use stark::prover::evaluate_polynomial_on_lde_domain; + + for (n, blowup, offset) in [ + (2usize, 2usize, 3u64), + (4, 2, 3), + (8, 2, 3), + (8, 4, 3), + (8, 8, 3), + (16, 4, 7), + (32, 2, 1), + (8, 1, 3), + (128, 2, 3), + ] { + let program = lde_probe_program(n, blowup, offset); + validate(&program).expect("admission"); + let source = synthetic_register_file(n as u64 * 31 + offset); + let values: Vec = (0..n) + .map(|i| FE::from(source[i % source.len()] as u64 + 1)) + .collect(); + let arenas = vec![values.iter().copied().map(super::word::base_word).collect()]; + let exec = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .unwrap_or_else(|e| panic!("n={n} blowup={blowup}: execution failed: {e:?}")); + + let poly = Polynomial::interpolate_fft::(&values) + .expect("interpolate"); + let expected = + evaluate_polynomial_on_lde_domain(&poly, blowup, n, &FE::from(offset)).expect("lde"); + let got: Vec = exec.public_words.iter().map(|(_, w)| w[0]).collect(); + assert_eq!( + got.len(), + n * blowup, + "n={n} blowup={blowup}: the extension must cover the whole domain" + ); + assert_eq!( + got, expected, + "n={n} blowup={blowup} offset={offset}: the emitted LDE must equal production's" + ); + } +} + +/// ★ The whole derivation, against production's own +/// `compute_precomputed_commitment`, across the blowup sweep. +/// +/// Two register files per blowup and they do different jobs. The REAL fixture's +/// file is what the target runs on; a SYNTHETIC one where all 67 entries are +/// distinct and nonzero is what makes the test able to fail — a real register +/// file is mostly zeros, so an emitter that dropped or duplicated rows could +/// agree with production on it and disagree everywhere else. +#[test] +fn the_register_derivation_matches_production() { + for blowup in INNER_BLOWUPS { + let shape = derivation_shape(blowup); + let opts = inner_options(blowup); + let program = register_derivation_program(shape); + validate(&program).expect("admission"); + + for (what, init, fini) in register_file_cases() { + let arenas = register_arenas(&init, &fini); + let exec = super::executor::execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("blowup {blowup} / {what}: execution failed: {e:?}")); + let expected = + crate::tables::register::compute_precomputed_commitment(&opts, &init, &fini); + assert_eq!( + digest_bytes(&exec.public_words), + expected, + "blowup {blowup} / {what}: the derived root must equal production's" + ); + } + } +} + +/// The register files the differential runs on: the fixture's real boundary +/// pair, plus synthetics that exercise every row. +fn register_file_cases() -> Vec<(&'static str, Vec, Vec)> { + let (init, fini) = fixture_register_boundary(); + vec![ + ("the fixture's real epoch boundary", init, fini), + ( + "a synthetic file with every row distinct", + synthetic_register_file(1), + synthetic_register_file(2), + ), + ( + "init and fini equal (an epoch that changed nothing)", + synthetic_register_file(3), + synthetic_register_file(3), + ), + ] +} + +/// Epoch 0's real `(register_init, reg_fini)` from the proof fixture — the +/// verifier-derived INIT from the entry point and the epoch's bound FINI. +fn fixture_register_boundary() -> (Vec, Vec) { + use std::sync::OnceLock; + static CELL: OnceLock<(Vec, Vec)> = OnceLock::new(); + CELL.get_or_init(|| { + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + super::proof_arena::register_boundary(&archive, 0) + }) + .clone() +} + +/// ★ Scrutinise the fixture boundary before trusting the differential that +/// runs on it. +/// +/// A register file is mostly zeros and `init` differs from `fini` in only a +/// handful of places, so "the real case passed" is weak evidence on its own. +/// This says exactly how weak, in numbers, which is what justifies the +/// synthetic cases carrying the load in +/// [`the_register_derivation_matches_production`]. +#[test] +fn the_fixture_register_boundary_is_mostly_zeros() { + use crate::tables::register::{NUM_REGISTER_ADDRESSES, PC_LO_INDEX}; + let (init, fini) = fixture_register_boundary(); + assert_eq!(init.len(), NUM_REGISTER_ADDRESSES); + assert_eq!(fini.len(), NUM_REGISTER_ADDRESSES); + + let nonzero = |v: &[u32]| v.iter().filter(|&&x| x != 0).count(); + let differing = init.iter().zip(&fini).filter(|(a, b)| a != b).count(); + println!( + "R1g(i) fixture boundary: init {}/{} nonzero, fini {}/{} nonzero, \ + {differing} rows differ (pc {} -> {})", + nonzero(&init), + NUM_REGISTER_ADDRESSES, + nonzero(&fini), + NUM_REGISTER_ADDRESSES, + init[PC_LO_INDEX], + fini[PC_LO_INDEX], + ); + assert_ne!( + init, fini, + "an epoch that changed no register would be a degenerate case" + ); + assert!( + nonzero(&fini) > 0, + "an all-zero fini would make the differential blind to the FINI column" + ); + // Nonzero is not enough: rows that share a value are still indistinguishable + // to a differential, so the synthetic file has to be pairwise DISTINCT for + // "it exercises every row" to mean anything. + let synthetic = synthetic_register_file(1); + assert_eq!( + nonzero(&synthetic), + NUM_REGISTER_ADDRESSES, + "the synthetic file must exercise every row, which the real one does not" + ); + let distinct: std::collections::HashSet = synthetic.iter().copied().collect(); + assert_eq!( + distinct.len(), + NUM_REGISTER_ADDRESSES, + "the synthetic file's rows must be pairwise distinct, else a dropped or \ + duplicated row could still agree with production" + ); +} + +/// ★ The measurement the leg was asked for: permutations against the predicted +/// 255 / 511 / 1023, and the derivation's share of an epoch verify. +/// +/// The prediction is `2·leaves − 1` with `leaves = 128·blowup / ROWS_PER_LEAF`, +/// i.e. `128·blowup − 1`. A miss is not something to round off: it would mean +/// the leaf grouping, the domain size or the padding is not what the design +/// says, so this asserts rather than prints. +#[test] +fn register_derivation_cost() { + // `lfm-target-shape.md`'s *Scale*: keccak permutations per epoch verify. + let epoch_permutations = |blowup: usize| match blowup { + 2 => Some(1_400_000f64), + 8 => Some(460_000f64), + _ => None, + }; + println!( + "one permutation = {} main cells; one byteswap = {} main cells; \ + KECCAK_RND chunk ceiling {} rows", + permutation_cells(), + byteswap_cells(), + super::chunking::KECCAK_RND_MAX_CHUNK_ROWS, + ); + println!( + "blowup rows leaves perms predicted instrs const balu bitdec keccak \ + main cells % of epoch hashing" + ); + for blowup in INNER_BLOWUPS { + let shape = derivation_shape(blowup); + let predicted = 128 * blowup - 1; + let program = register_derivation_program(shape); + let (main, _aux) = super::airs::lfm_cell_counts(&program); + let share = epoch_permutations(blowup) + .map(|total| format!("{:.4}%", 100.0 * shape.permutations() as f64 / total)) + .unwrap_or_else(|| "-".to_string()); + println!( + "{blowup:>6} {:>6} {:>8} {:>7} {:>10} {:>8} {:>7} {:>7} {:>7} {:>7} {:>12} {:>19}", + shape.lde_rows(), + shape.leaves(), + program.groups.keccak.real_rows, + predicted, + program.instrs.len(), + program.groups.const_.real_rows, + program.groups.balu.real_rows, + program.groups.bitdec.real_rows, + program.groups.keccak.real_rows, + main, + share, + ); + assert_eq!( + shape.permutations(), + predicted, + "blowup {blowup}: the shape's own arithmetic must give the predicted count" + ); + assert_eq!( + wrap_hash_rows(&program), + predicted, + "blowup {blowup}: the EMITTED compression count must be 2·leaves − 1 \ + ({predicted}); a miss means the tree's shape is not what the design \ + says. Counted against the CONFIGURED hash: the claim is about tree \ + shape, and it holds under both — a 48-byte leaf and a 64-byte parent \ + are each one keccak rate block and each one BLAKE3 block" + ); + // Every leaf is 48 bytes and every parent 64 — one rate block each, so + // the permutation count is exactly the node count and nothing else. + assert_eq!( + super::keccak_host::num_blocks(8 * 3 * stark::commitment::ROWS_PER_LEAF), + 1, + "a three-column row pair must fit one keccak rate block" + ); + assert_eq!( + super::keccak_host::num_blocks(2 * super::edsl::COMMITMENT_BYTES), + 1, + "a Merkle parent must fit one keccak rate block" + ); + + // Where the arithmetic goes, to the row. The transform is + // `2 · (n/2·log₂n butterflies + blowup · (n scalings + n/2·log₂n + // butterflies))` at two rows per butterfly and one per scaling, over + // the TWO dynamic columns. The byte swapping costs NO balu rows any + // more — the halves are bus sends off the BitDec rows — so the pin is + // the transform alone, and a reintroduced per-value swap cost fails + // here rather than showing up as one moved total. + let n = shape.num_rows() as u64; + let butterflies = n / 2 * n.trailing_zeros() as u64; + let per_column = 2 * butterflies + blowup as u64 * (n + 2 * butterflies); + let transform = 2 * per_column; + assert_eq!( + program.groups.balu.real_rows as u64, transform, + "blowup {blowup}: LFM_BALU rows must be exactly the {transform} of \ + the transform — byte swapping is free now" + ); + assert_eq!( + program.groups.bitdec.real_rows, + shape.leaves() * 6 * byteswaps_per_value(), + "blowup {blowup}: one bit decomposition per leaf value — the leaf \ + gadget is `keccak_leaf_hash` reused, not a second one. The term is \ + a BYTE-hash term: an algebraic leaf absorbs the six felts directly \ + and decomposes nothing" + ); + // Chunking is not a constraint at this scale and the leg should say so + // rather than leave the next reader to work it out: the whole tree at + // blowup 8 is 1023 permutations against a ceiling of 2^19 ROWS. + assert_eq!( + super::chunking::KeccakChunking::default().chunk_count(shape.permutations()), + 1, + "blowup {blowup}: the register tree must fit one KECCAK_RND chunk" + ); + assert_eq!( + program.groups.select.real_rows, 0, + "blowup {blowup}: a TREE build knows every child's side at emission \ + time, so it must emit no Select at all; routing it through \ + `keccak_merkle_walk` would put one per parent here" + ); + println!( + " blowup {blowup}: transform {transform} balu rows (byteswap is \ + free now); a Select would cost {} cells against a permutation's {}", + select_cells(), + permutation_cells(), + ); + } +} + +/// Main-trace cells one `LFM_SELECT` row costs — the unit the tree build avoids +/// by knowing child order at emission time. +fn select_cells() -> u64 { + use super::chips::select; + use super::layout; + (select::cols::NUM_COLUMNS - layout::select::PREP_WIDTH) as u64 +} + +/// ★ The derivation PROVED, not merely executed — and against the REAL +/// fixture's own options rather than a reconstruction of them. +/// +/// The executor mirrors the keccak the chip also does, so an execute-only +/// differential cannot see the `LFM_KECCAK` adapter, the `KECCAK_RND` chunking +/// or the lane plumbing agree with it. Blowup 2 (255 permutations) is the cheap +/// end of the sweep; the shape's arithmetic is what carries 4 and 8, and +/// `register_derivation_cost` asserts it. +/// +/// `fixture_options()` is `MIN_PROOF_OPTIONS` — blowup 2, coset offset 3, the +/// only two fields that reach the commitment — so this is not merely "the +/// machine agrees with a production function on some inputs". The root proved +/// here IS the preprocessed REGISTER commitment epoch 0 of the fixture's own +/// continuation was built against. +#[test] +fn the_register_derivation_proves_and_verifies() { + let opts = options(); + let inner = proof_fixture::fixture_options(); + let shape = RegisterDerivationShape { + blowup: inner.blowup_factor as usize, + coset_offset: inner.coset_offset, + rows_per_leaf: stark::commitment::ROWS_PER_LEAF, + }; + assert_eq!( + shape, + derivation_shape(2), + "the fixture is proved at blowup 2 / offset 3; if that moves, this test is no longer about the fixture's own commitment" + ); + let program = register_derivation_program(shape); + // ★ The BLOCK pin's permutation, named at the call site — `build_artifacts` + // defaults to `REGISTRY_HASHER = Test` and must, but its own doc licenses + // that default only for programs that "pin a byte hash on their own builders, + // emit no `Instr::Hash`, and never consult the socket". Since the derivation + // follows `WrapHash::production()`, this one IS a socket program, and proving + // it under a toy permutation would derive a root production never computes. + let artifacts = super::registry::build_artifacts_with_hasher( + &program, + &opts, + crate::hash_pin::BLOCK_HASHER, + ); + let (init, fini) = fixture_register_boundary(); + let arenas = register_arenas(&init, &fini); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + + assert_eq!( + digest_bytes(&proved.public_words), + crate::tables::register::compute_precomputed_commitment(&inner, &init, &fini), + "the proved root must equal the fixture epoch's own REGISTER commitment" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the derivation must verify" + ); + println!( + "R1g(i) proved: blowup {}, {} permutations, {} KECCAK_RND chunks", + shape.blowup, program.groups.keccak.real_rows, artifacts.keccak_rnd_chunks, + ); +} + +/// ★ Tamper: every register word must move the derived root, and claiming the +/// honest root for a tampered file must reject. +/// +/// Coherent by construction — the program asserts nothing, so each forgery +/// proves cleanly and fails on the PUBLISHED root. That is the mechanism +/// working: the root is the claim, and in the assembled verifier it is what +/// Phase A absorbs, so a `reg_fini` the prover did not honour produces a root +/// the epoch's own proof was not made against. +#[test] +fn tampering_the_register_files_moves_the_derived_root() { + let opts = options(); + let shape = derivation_shape(2); + let program = register_derivation_program(shape); + let artifacts = build_artifacts(&program, &opts); + let (init, fini) = fixture_register_boundary(); + let honest = lfm_prove(&program, &artifacts, ®ister_arenas(&init, &fini), &opts) + .expect("honest prove"); + + use crate::tables::register::{NUM_REGISTER_ADDRESSES, PC_HI_INDEX, X254_INDEX}; + // A zero row, a row the epoch changed, and the two whose values are not + // plain GPR words — the PC high half and the synthetic commit index, which + // are the rows an implementation is most likely to mislay. + let mut cases: Vec<(String, Vec, Vec)> = Vec::new(); + for row in [ + 0usize, + 5, + X254_INDEX, + PC_HI_INDEX, + NUM_REGISTER_ADDRESSES - 1, + ] { + let mut i2 = init.clone(); + i2[row] ^= 1; + cases.push((format!("init row {row}"), i2, fini.clone())); + let mut f2 = fini.clone(); + f2[row] ^= 1; + cases.push((format!("fini row {row}"), init.clone(), f2)); + } + + for (what, i, f) in cases { + let forged = lfm_prove(&program, &artifacts, ®ister_arenas(&i, &f), &opts) + .unwrap_or_else(|e| panic!("{what}: a tampered file must still prove: {e:?}")); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a changed register must move the derived root" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "{what}: claiming the honest root must reject" + ); + } +} + +/// ⚠ Documents a LIVE gap — **asserts it SUCCEEDS**, per the guard-test map's +/// convention. If this ever starts failing, something began constraining the +/// arena; re-derive before relaxing it. +/// +/// Production's `reg_fini` is a `Vec`, so every value it can commit is +/// below 2^32 and the type is the whole enforcement. An LFM arena is untyped +/// field elements, and nothing in the derivation narrows them: the program +/// happily extends and commits a column production could not have built. +/// +/// ## What is and is not claimed +/// +/// This is NOT a hole in the derivation. A root over a non-`u32` column matches +/// no commitment a production epoch proof was made against, so the epoch fails +/// — the same argument that makes the derivation a binding at all. What it IS +/// is a place where the machine's accepted set is WIDER than the RV64 +/// verifier's, and the assembled verifier owes one of two things: an arena +/// range check (67 extra `bit_dec`s per column, cheap at this scale), or the +/// argument that no epoch proof can exist over such a column in the first +/// place. That second argument is PLAUSIBLE — REG-C2 puts FINI on the Memory +/// bus as a value word and the memory side decomposes values into bytes — but +/// it is unverified here, and per the standing rule a deferral's safety +/// argument is itself a claim needing evidence. Stated, not assumed. +#[test] +fn the_derivation_extends_a_non_u32_register_value_demonstrating_hazard() { + let shape = derivation_shape(2); + let program = register_derivation_program(shape); + let (init, fini) = fixture_register_boundary(); + + let honest = register_arenas(&init, &fini); + let mut wide = honest.clone(); + // 2^32 — one past every value a `Vec` can hold. + wide[1][0] = super::word::base_word(FE::from(1u64 << 32)); + + let run = |arenas: &[Vec]| { + digest_bytes( + &super::executor::execute(&program, arenas, &super::hash::TestPermutation) + .expect("the machine must accept any felt in the arena") + .public_words, + ) + }; + assert_ne!( + run(&wide), + run(&honest), + "the out-of-range value must actually reach the commitment, else this \ + test documents nothing" + ); +} + +/// The honest-path control for the admission gate as a whole: every kind in +/// `LFM_REGISTRY` still passes, together, in one place. +/// +/// Each program has its own `*_is_admissible` test; this one exists because a +/// rejection rule is worth having only if it rejects nothing that ships, and +/// the cheapest way for a new check to be wrong is to be right about the one +/// program its author had in mind. It also fails loudly when a seventh kind is +/// registered without being run past the registrar here. +#[test] +fn every_registered_program_passes_admission() { + let programs: [(&str, LfmProgram); 6] = [ + ("TrivialV0", trivial_program()), + ("FriToyV0", fri_toy_program()), + ("KeccakChainV0", keccak_chain_program()), + ("KeccakSpongeV0", keccak_sponge_program(KECCAK_SPONGE_LEN)), + ("TranscriptReplayV0", transcript_replay_program()), + ("StatementReplayV0", statement_replay_program()), + ]; + for (name, program) in &programs { + validate(program) + .unwrap_or_else(|e| panic!("registered program {name} must pass admission: {e:?}")); + } +} + +/// Check 9 against a *registered* program's committed group — the object the +/// AIR actually reads. The instruction list stays untouched, so checks 1–4 see +/// a pristine program and nothing but check 9 stands between this group and +/// the registry. +#[test] +fn negative_hash_multiplicity_in_a_registered_group_fails_admission() { + let mut program = trivial_program(); + validate(&program).expect("honest control: the registered program is admissible"); + + let negative_one = FE::from(P - 1); // p − 1, i.e. −1 on the bus + program + .groups + .hash + .set(0, super::layout::hash::MULT1, negative_one); + assert!( + matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_HASH", + row: 0, + col, + .. + } if col == super::layout::hash::MULT1 + ), + "a negative multiplicity in the committed group must fail admission" + ); +} + +// ============================================================================= +// ★ R2 — the symmetric conditional hash-family chip group +// ============================================================================= +// +// The machine hosts two hash families and a program uses at most one. Carrying +// the other costs a whole AIR per chip for zero rows. `ChipSet` retires the +// unused one — in BOTH directions from one mask, which is the point: post-flip +// the live direction is dropping the KECCAK family, and pre-flip it was +// dropping LFM_BLAKE3. +// +// Unlike the RV64 side's conditional BLAKE3 table, nothing here is +// prover-asserted. The mask is computed from the compiled program at bless +// time, stored in the registry entry and folded into `program_id`, so a +// verifier reads it from the entry it resolved. There is no forgery direction +// to test because there is no prover claim to forge — the tests that matter are +// that the mask is what the programs actually use, that both directions prove +// and verify, and that a wrong mask cannot be substituted. + +/// ★ PRIMARY — the live post-flip direction: a program that authenticates with +/// BLAKE3 carries no keccak family at all. +/// +/// `StatementReplayV0` is the real one (the continuation-epoch statement bind +/// plus Phase A). Post-flip it hashes with BLAKE3, so LFM_KECCAK, every +/// KECCAK_RND chunk and KECCAK_RC are all absent — four sub-proofs that were +/// pure overhead. +#[test] +fn a_blake3_authenticating_program_carries_no_keccak_family() { + let opts = options(); + let program = statement_replay_program(); + let artifacts = build_artifacts(&program, &opts); + + assert_eq!( + (artifacts.chip_set.keccak, artifacts.chip_set.blake3), + (false, true), + "post-flip the authenticating programs hash with BLAKE3 and use no keccak" + ); + assert_eq!( + artifacts.keccak_rnd_chunks, 0, + "an absent keccak family carries zero KECCAK_RND instances" + ); + + let airs = super::airs::LfmAirs::new_with_hasher( + &artifacts.roots, + &opts, + artifacts.keccak_rnd_chunks, + artifacts.hasher, + artifacts.chip_set, + ); + let names: Vec<&str> = airs.air_refs().iter().map(|a| a.name()).collect(); + for absent in ["LFM_KECCAK", "KECCAK_RND", "KECCAK_RC"] { + assert!( + !names.contains(&absent), + "{absent} must not be instantiated for a BLAKE3-authenticating program" + ); + } + assert!( + names.contains(&"LFM_BLAKE3"), + "the family it DOES use must be present" + ); + assert!( + names.contains(&"BITWISE"), + "BITWISE is shared infrastructure — both families send to its buses — \ + and must survive dropping either one" + ); + + // The honest path: it still proves and verifies without them. + let f = statement_fixture(); + let proved = lfm_prove(&program, &artifacts, &statement_arenas(&f), &opts).expect("prove"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a BLAKE3 program must prove and verify with the keccak family absent" + ); +} + +/// The other direction of the SAME mask: a program that is ABOUT keccak keeps +/// the keccak family and drops LFM_BLAKE3. +/// +/// This is the control that makes the primary test mean something. It also +/// pins why the mask is per-program rather than keyed on +/// `WrapHash::production()`: the production wrap hash here is BLAKE3, and +/// keying off it would have silently stripped the keccak family from the very +/// programs that exist to exercise keccak. +#[test] +fn a_keccak_program_keeps_the_keccak_family_whatever_the_production_hash() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + + assert_eq!( + (artifacts.chip_set.keccak, artifacts.chip_set.blake3), + (true, false), + "a keccak instrument uses keccak and no BLAKE3, regardless of the \ + production commitment hash" + ); + assert_eq!(artifacts.keccak_rnd_chunks, 1); + + let airs = super::airs::LfmAirs::new_with_hasher( + &artifacts.roots, + &opts, + artifacts.keccak_rnd_chunks, + artifacts.hasher, + artifacts.chip_set, + ); + let names: Vec<&str> = airs.air_refs().iter().map(|a| a.name()).collect(); + assert!(names.contains(&"LFM_KECCAK") && names.contains(&"KECCAK_RC")); + assert!( + !names.contains(&"LFM_BLAKE3"), + "the unused BLAKE3 chip must not be instantiated" + ); +} + +/// A program using NEITHER family drops both — the mask is not a two-way +/// switch, it is a per-family presence bit. +#[test] +fn a_program_using_no_hash_family_carries_neither() { + let opts = options(); + let artifacts = build_artifacts(&trivial_program(), &opts); + assert_eq!( + (artifacts.chip_set.keccak, artifacts.chip_set.blake3), + (false, false), + "the trivial program hashes nothing" + ); + let airs = super::airs::LfmAirs::new_with_hasher( + &artifacts.roots, + &opts, + artifacts.keccak_rnd_chunks, + artifacts.hasher, + artifacts.chip_set, + ); + let names: Vec<&str> = airs.air_refs().iter().map(|a| a.name()).collect(); + for absent in ["LFM_KECCAK", "KECCAK_RND", "KECCAK_RC", "LFM_BLAKE3"] { + assert!(!names.contains(&absent), "{absent} must be absent"); + } +} + +/// Every registered program next to the kind it is blessed under — the list the +/// entry-vs-program tests recompute from. One copy, so a seventh row cannot be +/// added to the table while a test keeps checking six. +fn registry_program_cases() -> [(LfmProgramKind, super::compiler::LfmProgram); 6] { + [ + (LfmProgramKind::TrivialV0, trivial_program()), + (LfmProgramKind::FriToyV0, super::programs::fri_toy_program()), + (LfmProgramKind::KeccakChainV0, keccak_chain_program()), + ( + LfmProgramKind::KeccakSpongeV0, + super::programs::keccak_sponge_program(super::programs::KECCAK_SPONGE_LEN), + ), + ( + LfmProgramKind::TranscriptReplayV0, + super::programs::transcript_replay_program(), + ), + ( + LfmProgramKind::StatementReplayV0, + statement_replay_program(), + ), + ] +} + +/// ★ The premise `LfmRegistryEntry::artifacts` rests on: every registered +/// program is a SINGLE `LFM_BLAKE3` table. +/// +/// The entry has no `blake3_chunks` column — the chunk shape is derived as "slot +/// 11's own root and height" — which is only correct while every registered +/// program sits at the policy default. Recomputing each program's chunk count +/// here is what turns that from a convention into a checked premise: blessing a +/// chunked program fails this test, which is the moment to add the column. +#[test] +fn every_registry_entry_is_a_single_blake3_table() { + let opts = options(); + for (kind, program) in registry_program_cases() { + assert_eq!( + program.blake3_chunk_count(), + 1, + "{kind:?}: a registered program must be a single LFM_BLAKE3 table" + ); + let entry = resolve(kind, opts.blowup_factor).expect("registered"); + let artifacts = build_artifacts(&program, &opts); + let derived = entry.artifacts(); + assert_eq!( + derived.blake3_chunk_roots, artifacts.blake3_chunk_roots, + "{kind:?}: the derived chunk roots must be the program's own" + ); + assert_eq!( + derived.blake3_chunk_log_heights, artifacts.blake3_chunk_log_heights, + "{kind:?}: the derived chunk heights must be the program's own" + ); + assert_eq!( + derived.program_id, artifacts.program_id, + "{kind:?}: and the digest the pair derives must be the blessed one" + ); + } +} + +/// The registry's masks are what the programs actually compile to. +/// +/// The mask is the authority the verifier reads, so a registry entry claiming a +/// family the program does not use — or omitting one it does — would build the +/// wrong AIR set. This recomputes every entry's mask from its program. +#[test] +fn every_registry_mask_is_the_programs_own_usage() { + let opts = options(); + let cases = registry_program_cases(); + for (kind, program) in &cases { + let entry = super::registry::resolve(*kind, 2).expect("registered"); + let computed = super::airs::ChipSet::for_program(program); + assert_eq!( + entry.chip_set, computed, + "{kind:?}: the registry mask must be the program's own family usage" + ); + assert_eq!( + entry.keccak_rnd_chunks, + computed.keccak_rnd_chunks( + program + .chunking + .chunk_count(program.groups.keccak.real_rows) + ), + "{kind:?}: chunk count must follow the mask" + ); + // And the mask is bound: a different one is a different identity. + let other = super::airs::ChipSet { + keccak: !computed.keccak, + ..computed + }; + let _ = &opts; + assert_ne!( + super::statement::lfm_program_id( + &entry.roots, + &entry.log_heights, + entry.keccak_rnd_chunks, + entry.hasher, + other, + &[entry.roots[BLAKE3_SLOT]], + &[entry.log_heights[BLAKE3_SLOT]], + ), + entry.program_id, + "{kind:?}: the chip set must be folded into program_id, or a verifier \ + could resolve one mask and build another" + ); + } +} + +/// M-7's entry point must agree with the seven-argument form it delegates to — +/// on an honest proof, and on a tampered digest. Without the negative this +/// would pass for a function that returned `true` unconditionally. +#[test] +fn verify_against_artifacts_agrees_with_the_registry_path() { + use crate::lfm::verify_against_artifacts; + + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = + lfm_prove(&program, &artifacts, &arenas(), &opts).expect("the trivial program must prove"); + + assert!( + verify_against_artifacts(&artifacts, &proved.proof, &proved.public_words, &opts), + "honest-path control: artifacts built from the program must verify its proof" + ); + assert!( + lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts + ) + .expect("TrivialV0@2 is registered"), + "and the registry path must agree" + ); + + let mut wrong = artifacts; + wrong.program_id[0] ^= 0xff; + assert!( + !verify_against_artifacts(&wrong, &proved.proof, &proved.public_words, &opts), + "a program digest the proof was not bound to must be rejected" + ); +} diff --git a/prover/src/lfm/merge.rs b/prover/src/lfm/merge.rs new file mode 100644 index 000000000..544a2649c --- /dev/null +++ b/prover/src/lfm/merge.rs @@ -0,0 +1,793 @@ +//! Equal-height LFM chips proved as ONE AIR (`LAMBDA_VM_ZF_LFM_MERGE`, on in the +//! production format; `=0` is the rollback). +//! +//! An LFM proof carries one sub-proof a chip: its own trees, quotient, FRI, +//! grind and queries, and a parent re-verifies each of them. Four chips — +//! `LFM_BALU`, `LFM_SELECT`, `LFM_LANES` and `LFM_HINT` — sit at the same padded +//! height in every leaf of today's trees, so under this format the chips of a +//! program that share a padded height are proved as one AIR: their columns side +//! by side over the same rows. +//! +//! # Why the merged AIR proves exactly what the parts did +//! +//! It is the direct product of the parts, nothing more: +//! - **columns**: every part's preprocessed prefix first, in part order, then +//! every part's value columns, in part order (the STARK keeps preprocessed +//! columns as a prefix: `AirWithBuses::with_preprocessed`). A part's columns +//! keep their order inside each region; [`PartMap`] is the only translation. +//! - **constraints**: each part's own body, run through [`Remap`], which reads +//! the part's columns where they now sit and shifts its constraint indices +//! past the parts before it. No body reads another part's column. +//! - **buses**: the parts' interactions, concatenated in part order with every +//! column index remapped. The bus sum over the merged rows is the sum of the +//! parts' sums, so the LogUp closure is unchanged. +//! - **rows**: equal padded heights only — nothing is padded to merge — so a +//! part's next-row window wraps at the same row it did alone, and the merged +//! trace is the parts' traces side by side, row for row. +//! +//! # Where the layout comes from +//! +//! A function of the program's padded heights and the format flag +//! (`ProofFormat::merge_equal_height_tables`), computed where the artifacts are +//! built, stored in them ([`LfmMerge`]) and folded into `program_id`. A +//! verifier takes it from the artifacts it derived, never from a proof. + +use stark::config::Commitment; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet, Nil, RowDomain}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity}; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; + +use super::airs::{LFM_CHIP_NAMES, NUM_LFM_CHIPS}; +use super::chips::{balu, hint, lanes, select}; +use super::compiler::{ColumnGroup, LfmProgram}; +use super::layout; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// A chip a merge may take: the four the trees hold at one height. Not +/// `LFM_HASH` (the socket, with its own chunks and card paths), not +/// `LFM_PUBLIC` (it carries the public-word closure), not the fixed tables, and +/// not the small chips, which sit at heights of their own. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MergeChip { + Balu, + Select, + Lanes, + Hint, +} + +/// The candidates in slot order. +pub const MERGE_CANDIDATES: [MergeChip; 4] = [ + MergeChip::Balu, + MergeChip::Select, + MergeChip::Lanes, + MergeChip::Hint, +]; + +impl MergeChip { + /// The chip's slot in the frozen chip order (`airs::LFM_CHIP_NAMES`). + pub const fn slot(self) -> usize { + match self { + Self::Balu => 1, + Self::Select => 3, + Self::Lanes => 7, + Self::Hint => 8, + } + } + + /// The candidate at chip slot `slot`, if it is one. + pub fn at_slot(slot: usize) -> Option { + MERGE_CANDIDATES.into_iter().find(|c| c.slot() == slot) + } +} + +/// One merged AIR of a program: the chips it holds, in slot order, their common +/// log-height and the root of their preprocessed columns side by side. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MergedTable { + pub chips: Vec, + pub log_height: u8, + pub root: Commitment, +} + +/// A program's merged AIRs. Empty — every program built with the flag off (the +/// library's default `ProofFormat`, and the rollback `LAMBDA_VM_ZF_LFM_MERGE=0`) +/// — is the machine of one AIR a chip, byte for byte. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LfmMerge { + pub tables: Vec, +} + +impl LfmMerge { + /// No merged AIR. + pub const fn none() -> Self { + Self { tables: Vec::new() } + } + + pub fn is_empty(&self) -> bool { + self.tables.is_empty() + } + + /// The groups of chips, without the roots. + pub fn plan(&self) -> MergePlan { + self.tables.iter().map(|t| t.chips.clone()).collect() + } + + /// How many AIRs the merge removes from a proof: a table of `k` chips is + /// one AIR where there were `k`. + pub fn airs_saved(&self) -> usize { + self.tables + .iter() + .map(|t| t.chips.len().saturating_sub(1)) + .sum() + } + + /// Whether this merge is one a program can have, against the program's + /// per-slot `log_heights`: every merged AIR holds two or more chips in + /// strictly increasing slot order, no chip sits in two merged AIRs, and + /// every chip's height is its merged AIR's `log_height` (equal heights + /// only, the premise every part's next-row window rests on). + /// + /// [`plan_for_heights`] only ever builds such a merge; this is the check + /// every AIR-set build runs (`LfmAirs::with_merge`), so a hand-made or + /// corrupted one is refused on the prove, the host-verify and the leg's + /// derivation path alike rather than built. + pub fn check(&self, log_heights: &[u8; NUM_LFM_CHIPS]) -> Result<(), String> { + let mut seen: Vec = Vec::new(); + for t in &self.tables { + let slots: Vec = t.chips.iter().map(|c| c.slot()).collect(); + if slots.len() < 2 { + return Err(format!( + "a merged AIR of {} chip(s): {slots:?}", + slots.len() + )); + } + if !slots.windows(2).all(|w| w[0] < w[1]) { + return Err(format!("merged chips out of slot order: {slots:?}")); + } + if let Some(s) = slots.iter().find(|s| seen.contains(s)) { + return Err(format!("chip slot {s} in two merged AIRs")); + } + if let Some(s) = slots.iter().find(|&&s| log_heights[s] != t.log_height) { + return Err(format!( + "chip slot {s} at log-height {} in a merged AIR at {}", + log_heights[*s], t.log_height + )); + } + seen.extend(slots); + } + Ok(()) + } +} + +/// The merged AIRs' chips, in AIR order; each group in slot order. +pub type MergePlan = Vec>; + +/// Which merged AIR, if any, chip slot `slot` belongs to under `plan`. +pub fn group_of(plan: &[Vec], slot: usize) -> Option { + plan.iter().position(|g| g.iter().any(|c| c.slot() == slot)) +} + +/// What a chip slot is in the AIR order under `plan`: its own AIR, the first +/// chip of merged AIR `g` (where that AIR sits), or a later chip of one (no AIR +/// of its own). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SlotRole { + Own, + First(usize), + Member, +} + +pub fn role(plan: &[Vec], slot: usize) -> SlotRole { + match group_of(plan, slot) { + None => SlotRole::Own, + Some(g) if plan[g].first().map(|c| c.slot()) == Some(slot) => SlotRole::First(g), + Some(_) => SlotRole::Member, + } +} + +/// The merge plan for chips at `log_heights` (the artifacts' per-slot +/// log-heights): the candidates grouped by log-height in slot order, every +/// class of two or more one merged AIR, ordered by its first chip. Empty when +/// `on` is false. +pub fn plan_for_heights(log_heights: &[u8; NUM_LFM_CHIPS], on: bool) -> MergePlan { + if !on { + return Vec::new(); + } + let mut groups: MergePlan = Vec::new(); + for chip in MERGE_CANDIDATES { + let h = log_heights[chip.slot()]; + match groups + .iter_mut() + .find(|g| g.first().is_some_and(|c| log_heights[c.slot()] == h)) + { + Some(g) => g.push(chip), + None => groups.push(vec![chip]), + } + } + groups.retain(|g| g.len() > 1); + groups +} + +/// The merge plan for a compiled program, from its groups' padded heights — +/// what [`plan_for_heights`] reads off the artifacts the same program builds. +pub fn plan_for_program(program: &LfmProgram, on: bool) -> MergePlan { + let mut heights = [0u8; NUM_LFM_CHIPS]; + for chip in MERGE_CANDIDATES { + heights[chip.slot()] = group_of_program(program, chip).padded_rows.trailing_zeros() as u8; + } + plan_for_heights(&heights, on) +} + +/// A candidate chip's instruction column group in `program`. +pub fn group_of_program(program: &LfmProgram, chip: MergeChip) -> &ColumnGroup { + let g = &program.groups; + match chip { + MergeChip::Balu => &g.balu, + MergeChip::Select => &g.select, + MergeChip::Lanes => &g.lanes, + MergeChip::Hint => &g.hint, + } +} + +/// A merge candidate's AIR shape: preprocessed prefix, total width, buses and +/// constraint body. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PartBody { + Balu, + Select, + /// LogUp only (`LFM_LANES`, `LFM_HINT`). + Empty, +} + +#[derive(Clone, Copy, Debug)] +pub struct PartShape { + pub chip: MergeChip, + pub prep: usize, + pub width: usize, + pub body: PartBody, +} + +impl PartShape { + pub fn of(chip: MergeChip) -> Self { + let (prep, width, body) = match chip { + MergeChip::Balu => ( + layout::balu::PREP_WIDTH, + balu::cols::NUM_COLUMNS, + PartBody::Balu, + ), + MergeChip::Select => ( + layout::select::PREP_WIDTH, + select::cols::NUM_COLUMNS, + PartBody::Select, + ), + MergeChip::Lanes => ( + layout::lanes::PREP_WIDTH, + lanes::cols::NUM_COLUMNS, + PartBody::Empty, + ), + MergeChip::Hint => ( + layout::hint::PREP_WIDTH, + hint::cols::NUM_COLUMNS, + PartBody::Empty, + ), + }; + Self { + chip, + prep, + width, + body, + } + } + + pub fn interactions(&self) -> Vec { + match self.chip { + MergeChip::Balu => balu::bus_interactions(), + MergeChip::Select => select::bus_interactions(), + MergeChip::Lanes => lanes::bus_interactions(), + MergeChip::Hint => hint::bus_interactions(), + } + } + + /// The base constraints the part's body emits (its whole meta: the bodies + /// are base-only). + pub fn num_base_constraints(&self) -> usize { + match self.body { + PartBody::Balu => ConstraintSet::::meta(&balu::BaluConstraints).len(), + PartBody::Select => ConstraintSet::::meta(&select::SelectConstraints).len(), + PartBody::Empty => 0, + } + } + + pub fn max_degree(&self) -> usize { + match self.body { + PartBody::Balu => ConstraintSet::::max_degree(&balu::BaluConstraints), + PartBody::Select => ConstraintSet::::max_degree(&select::SelectConstraints), + PartBody::Empty => { + ConstraintSet::::max_degree(&stark::constraints::builder::EmptyConstraints) + } + } + } +} + +/// Where one part's columns sit in the merged AIR: `map[c]` is part column +/// `c`'s merged column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartMap { + pub chip: MergeChip, + pub map: Vec, + /// The merged column of the part's first preprocessed cell. + pub prep_at: usize, + /// The merged column of the part's first value cell. + pub main_at: usize, +} + +/// The merged AIR's whole layout: each part's map, the merged widths. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MergedLayout { + pub parts: Vec, + pub num_prep: usize, + pub num_columns: usize, +} + +impl MergedLayout { + /// `slots` side by side: every part's prefix, then every part's values. + pub fn of(chips: &[MergeChip]) -> Self { + let shapes: Vec = chips.iter().map(|&c| PartShape::of(c)).collect(); + let num_prep: usize = shapes.iter().map(|s| s.prep).sum(); + let num_columns: usize = shapes.iter().map(|s| s.width).sum(); + let mut prep_at = 0; + let mut main_at = num_prep; + let parts = shapes + .iter() + .map(|s| { + let map = (0..s.width) + .map(|c| { + if c < s.prep { + prep_at + c + } else { + main_at + (c - s.prep) + } + }) + .collect(); + let part = PartMap { + chip: s.chip, + map, + prep_at, + main_at, + }; + prep_at += s.prep; + main_at += s.width - s.prep; + part + }) + .collect(); + Self { + parts, + num_prep, + num_columns, + } + } +} + +/// The merged AIR's name: the parts' chip names joined, so a log line names +/// the chips it holds. +pub fn merged_name(chips: &[MergeChip]) -> String { + let parts: Vec<&str> = chips + .iter() + .map(|c| LFM_CHIP_NAMES[c.slot()].trim_start_matches("LFM_")) + .collect(); + format!("LFM_MERGED[{}]", parts.join("+")) +} + +// ============================ the constraint body ========================== + +/// One part of a [`MergedConstraints`]: its body, its column map and the index +/// its first constraint takes. +#[derive(Clone, Debug)] +struct MergedPart { + body: PartBody, + map: Vec, + idx_offset: usize, +} + +/// The merged AIR's constraint set: each part's own body, in part order, its +/// columns remapped and its indices shifted ([`Remap`]). +#[derive(Clone, Debug)] +pub struct MergedConstraints { + parts: Vec, + max_degree: usize, +} + +impl MergedConstraints { + /// The merged body of `layout`'s parts. `Err` when a part's body is not + /// one [`Remap`] can translate ([`audit_body`]): the check runs here, once + /// per AIR build, so a candidate that ever reads a leaf the remap does not + /// move is refused when its AIR is built, never proved. + pub fn new(layout: &MergedLayout) -> Result { + let mut idx_offset = 0; + let mut max_degree = 0; + let mut parts = Vec::with_capacity(layout.parts.len()); + for p in &layout.parts { + let shape = PartShape::of(p.chip); + audit_body(shape)?; + parts.push(MergedPart { + body: shape.body, + map: p.map.clone(), + idx_offset, + }); + idx_offset += shape.num_base_constraints(); + max_degree = max_degree.max(shape.max_degree()); + } + Ok(Self { parts, max_degree }) + } +} + +/// Runs a part's body through a recording builder and refuses one [`Remap`] +/// could not translate faithfully: a read of an aux column, a challenge, an +/// alpha power or the table offset (the remap passes those through unmoved, so +/// they would be the MERGED AIR's), an extension constraint, a main column past +/// the part's width or past the next row, or constraint indices that are not +/// exactly `0..n` (the shift assumes a dense base prefix). +pub fn audit_body(shape: PartShape) -> Result<(), String> { + let label = format!("{:?}", shape.chip); + match shape.body { + PartBody::Balu => audit_set(&balu::BaluConstraints, shape.width, &label), + PartBody::Select => audit_set(&select::SelectConstraints, shape.width, &label), + PartBody::Empty => Ok(()), + } +} + +/// [`audit_body`] over any constraint set of a `width`-column table. +pub fn audit_set>( + cs: &CS, + width: usize, + label: &str, +) -> Result<(), String> { + let mut a = Audit { + width, + bad: std::cell::Cell::new(None), + emitted: Vec::new(), + }; + cs.eval(&mut a); + if let Some(why) = a.bad.take() { + return Err(format!("{label}: {why}")); + } + a.emitted.sort_unstable(); + if a.emitted != (0..a.emitted.len()).collect::>() { + return Err(format!( + "{label}: constraint indices {:?} are not dense", + a.emitted + )); + } + Ok(()) +} + +/// [`audit_body`]'s builder: no arithmetic, a record of what was read. +struct Audit { + width: usize, + bad: std::cell::Cell>, + emitted: Vec, +} + +impl Audit { + fn refuse(&self, why: &'static str) -> Nil { + if self.bad.get().is_none() { + self.bad.set(Some(why)); + } + Nil + } +} + +impl ConstraintBuilder for Audit { + type Expr = Nil; + type ExprE = Nil; + fn main(&self, offset: usize, col: usize) -> Nil { + if col >= self.width { + return self.refuse("reads a main column past its width"); + } + if offset > 1 { + return self.refuse("reads past the next row"); + } + Nil + } + fn aux(&self, _: usize, _: usize) -> Nil { + self.refuse("reads an aux column") + } + fn challenge(&self, _: usize) -> Nil { + self.refuse("reads a challenge") + } + fn alpha_pow(&self, _: usize) -> Nil { + self.refuse("reads an alpha power") + } + fn table_offset(&self) -> Nil { + self.refuse("reads the table offset") + } + fn const_base(&self, _: u64) -> Nil { + Nil + } + fn const_signed(&self, _: i64) -> Nil { + Nil + } + fn emit_base_rows(&mut self, idx: usize, _: RowDomain, _: Nil) { + self.emitted.push(idx); + } + fn emit_ext_rows(&mut self, _: usize, _: RowDomain, _: Nil) { + self.refuse("emits an extension constraint"); + } + fn fold_fingerprint_term(&self, _: Nil, _: Nil, _: usize) -> Nil { + self.refuse("folds a fingerprint") + } +} + +impl ConstraintSet for MergedConstraints { + /// The largest of the parts' declared degrees: the merged quotient is as + /// wide as its widest part's. + fn max_degree(&self) -> usize { + self.max_degree + } + + fn eval>(&self, b: &mut B) { + for part in &self.parts { + #[cfg(test)] + if test_hooks::vacuous() == Some(part.body) { + // The mutant: the part's indices kept dense, every constraint + // of its body zero. + let shape = MERGE_CANDIDATES + .into_iter() + .map(PartShape::of) + .find(|s| s.body == part.body); + for i in 0..shape.map_or(0, |s| s.num_base_constraints()) { + let z = b.zero(); + b.emit_base(part.idx_offset + i, z); + } + continue; + } + let mut r = Remap { + inner: &mut *b, + map: &part.map, + idx_offset: part.idx_offset, + }; + match part.body { + PartBody::Balu => balu::BaluConstraints.eval(&mut r), + PartBody::Select => select::SelectConstraints.eval(&mut r), + PartBody::Empty => {} + } + } + } +} + +/// A part's view of the merged builder: its `main(offset, c)` reads merged +/// column `map[c]`, and its constraint `i` is merged constraint +/// `idx_offset + i`. +/// +/// ⚠ Every OTHER leaf — `aux`, `challenge`, `alpha_pow`, `table_offset`, +/// `fold_fingerprint_term`, `emit_ext_rows` — passes through UNMOVED: it would +/// read the merged AIR's aux columns and LogUp state, not the part's own. That is +/// correct only because the candidate bodies are base-only and read none of +/// them, and it is guarded rather than assumed: [`MergedConstraints::new`] runs +/// every part body through [`audit_body`] when the AIR is built and refuses one +/// that reads any of these leaves (`a_part_body_reads_only_its_main_columns` +/// pins the audit). +pub struct Remap<'a, B> { + inner: &'a mut B, + map: &'a [usize], + idx_offset: usize, +} + +impl> ConstraintBuilder for Remap<'_, B> { + type Expr = B::Expr; + type ExprE = B::ExprE; + + fn main(&self, offset: usize, col: usize) -> Self::Expr { + self.inner.main(offset, self.map[col]) + } + fn aux(&self, offset: usize, col: usize) -> Self::ExprE { + self.inner.aux(offset, col) + } + fn challenge(&self, idx: usize) -> Self::ExprE { + self.inner.challenge(idx) + } + fn alpha_pow(&self, idx: usize) -> Self::ExprE { + self.inner.alpha_pow(idx) + } + fn table_offset(&self) -> Self::ExprE { + self.inner.table_offset() + } + fn const_base(&self, v: u64) -> Self::Expr { + self.inner.const_base(v) + } + fn const_signed(&self, v: i64) -> Self::Expr { + self.inner.const_signed(v) + } + fn one(&self) -> Self::Expr { + self.inner.one() + } + fn zero(&self) -> Self::Expr { + self.inner.zero() + } + fn emit_base_rows(&mut self, constraint_idx: usize, rows: RowDomain, e: Self::Expr) { + self.inner + .emit_base_rows(self.idx_offset + constraint_idx, rows, e) + } + fn emit_ext_rows(&mut self, constraint_idx: usize, rows: RowDomain, e: Self::ExprE) { + self.inner + .emit_ext_rows(self.idx_offset + constraint_idx, rows, e) + } + fn fold_fingerprint_term( + &self, + fp: Self::ExprE, + v: Self::Expr, + alpha_idx: usize, + ) -> Self::ExprE { + self.inner.fold_fingerprint_term(fp, v, alpha_idx) + } +} + +// =============================== the buses ================================= + +/// `interaction` with every column it reads moved by `map`. +/// +/// A multi-column packing must stay contiguous under `map`. A part's map moves +/// its prefix and its values as two blocks, so only a packing that crossed the +/// prefix's end could break — and no merge candidate has one: every packing +/// they use is one column (`every_merge_remaps_its_buses` pins it, against the +/// unmapped reading of the same interaction). The check is a real one, in +/// every build: a packing that would land on non-adjacent merged columns is +/// refused (`Err`), never remapped onto the wrong columns. +pub fn remap_interaction( + interaction: &BusInteraction, + map: &[usize], +) -> Result { + let term = |t: &LinearTerm| match *t { + LinearTerm::Column { + coefficient, + column, + } => LinearTerm::Column { + coefficient, + column: map[column], + }, + LinearTerm::ColumnUnsigned { + coefficient, + column, + } => LinearTerm::ColumnUnsigned { + coefficient, + column: map[column], + }, + LinearTerm::Constant(c) => LinearTerm::Constant(c), + }; + let multiplicity = match &interaction.multiplicity { + Multiplicity::One => Multiplicity::One, + Multiplicity::Column(c) => Multiplicity::Column(map[*c]), + Multiplicity::Sum(a, b) => Multiplicity::Sum(map[*a], map[*b]), + Multiplicity::Negated(c) => Multiplicity::Negated(map[*c]), + Multiplicity::Diff(a, b) => Multiplicity::Diff(map[*a], map[*b]), + Multiplicity::Sum3(a, b, c) => Multiplicity::Sum3(map[*a], map[*b], map[*c]), + Multiplicity::Linear(terms) => Multiplicity::Linear(terms.iter().map(term).collect()), + }; + let values = interaction + .values + .iter() + .map(|v| match v { + BusValue::Packed { + start_column, + packing, + } => { + let start = map[*start_column]; + let contiguous = (0..packing.num_columns()) + .all(|k| map.get(start_column + k) == Some(&(start + k))); + if !contiguous { + return Err(format!( + "a {packing:?} packing at column {start_column} is not contiguous in the \ + merged layout" + )); + } + Ok(BusValue::Packed { + start_column: start, + packing: *packing, + }) + } + BusValue::Linear(terms) => Ok(BusValue::Linear(terms.iter().map(term).collect())), + }) + .collect::, String>>()?; + Ok(BusInteraction { + bus_id: interaction.bus_id, + multiplicity, + values, + is_sender: interaction.is_sender, + }) +} + +/// The merged AIR's interactions: each part's, in part order, remapped. `Err` +/// names the first one [`remap_interaction`] refuses. +pub fn merged_interactions(layout: &MergedLayout) -> Result, String> { + let mut out = Vec::new(); + for p in &layout.parts { + for i in PartShape::of(p.chip).interactions() { + out.push(remap_interaction(&i, &p.map).map_err(|e| format!("{:?}: {e}", p.chip))?); + } + } + Ok(out) +} + +// ======================== the preprocessed columns ========================= + +/// The parts' instruction column groups side by side, row for row: the merged +/// AIR's preprocessed prefix, committed once (its root is [`MergedTable::root`]). +/// The groups must share their padded height. +pub fn merged_group(groups: &[&ColumnGroup]) -> ColumnGroup { + let rows = groups.first().map_or(0, |g| g.padded_rows); + debug_assert!(groups.iter().all(|g| g.padded_rows == rows)); + let width: usize = groups.iter().map(|g| g.width).sum(); + let mut data = crate::tables::types::zeroed_fe_vec(rows * width); + let fill_row = |row: usize, out: &mut [FE]| { + let mut at = 0; + for g in groups { + out[at..at + g.width].copy_from_slice(&g.data[row * g.width..(row + 1) * g.width]); + at += g.width; + } + }; + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + data.par_chunks_mut(width) + .enumerate() + .for_each(|(row, out)| fill_row(row, out)); + } + #[cfg(not(feature = "parallel"))] + for (row, out) in data.chunks_mut(width).enumerate() { + fill_row(row, out); + } + ColumnGroup { + width, + real_rows: groups.iter().map(|g| g.real_rows).max().unwrap_or(0), + padded_rows: rows, + data, + } +} + +/// [`merged_group`] for each group of `plan`, from `program`'s groups. +pub fn merged_groups(program: &LfmProgram, plan: &[Vec]) -> Vec { + plan.iter() + .map(|chips| { + let groups: Vec<&ColumnGroup> = chips + .iter() + .map(|&c| group_of_program(program, c)) + .collect(); + merged_group(&groups) + }) + .collect() +} + +/// The mutation hook of `merge_tests`: a merged AIR in which one part's body is +/// VACUOUS (its constraint indices kept, every constraint zero), to show that +/// body is what refuses a bad row of that part. Test builds only; a +/// process-wide switch, so every test that builds a merged AIR holds +/// `merge_tests`' lock. +#[cfg(test)] +pub(crate) mod test_hooks { + use std::sync::atomic::{AtomicU8, Ordering}; + + use super::PartBody; + + /// 0 = none, 1 = `LFM_BALU`'s body, 2 = `LFM_SELECT`'s. + static VACUOUS: AtomicU8 = AtomicU8::new(0); + + pub(crate) fn vacuous() -> Option { + match VACUOUS.load(Ordering::SeqCst) { + 1 => Some(PartBody::Balu), + 2 => Some(PartBody::Select), + _ => None, + } + } + + pub(crate) fn set_vacuous(body: Option) { + let v = match body { + Some(PartBody::Balu) => 1, + Some(PartBody::Select) => 2, + _ => 0, + }; + VACUOUS.store(v, Ordering::SeqCst); + } +} diff --git a/prover/src/lfm/merge_tests.rs b/prover/src/lfm/merge_tests.rs new file mode 100644 index 000000000..ef3ad151e --- /dev/null +++ b/prover/src/lfm/merge_tests.rs @@ -0,0 +1,1205 @@ +//! The merged LFM tables ([`super::merge`], `LAMBDA_VM_ZF_LFM_MERGE`): the plan +//! takes equal heights only; the layout puts every prefix first; a part's body +//! reads its own main columns and nothing else; the buses carry over column by +//! column; the direct fill is the parts side by side; a merged program proves +//! and verifies, under its own artifacts only; a bad row inside a merged table +//! is refused, and the part's body is what refuses it (the mutant); a node's +//! leg verifies a merged child and refuses its tampers; and the format knob +//! parses, prints and refuses as its siblings do. With the flag off nothing +//! moves: the plan is empty, the digest is the unmerged one, the fill is the +//! unmerged fill. + +use std::cell::RefCell; +use std::sync::{Mutex, MutexGuard}; + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet, Nil, RowDomain}; +use stark::proof::options::{GoldilocksCubicProofOptions, OneRowMode, ProofOptions}; +use stark::proof::view::MultiProofView; +use stark::trace::TraceTable; + +use crate::hash_pin::{P1Family, ProofFamily}; +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::airs::{LfmAirs, NUM_LFM_CHIPS}; +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::merge::{ + LfmMerge, MERGE_CANDIDATES, MergeChip, MergedConstraints, MergedLayout, MergedTable, PartBody, + PartShape, merged_interactions, plan_for_heights, plan_for_program, test_hooks, +}; +use super::p1_node_tests::{P1Child, harvest, node, runs}; +use super::proof::{LfmProof, lfm_prove, lfm_prove_under, prove_traces, verify_against_artifacts}; +use super::registry::{LfmArtifacts, build_artifacts, build_artifacts_under}; +use super::trace::{LfmTraces, build_traces_with_hasher, build_traces_with_merge}; +use super::word::{LfmWord, base_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// Every test here that builds a merged AIR holds this: the mutation hook +/// ([`test_hooks`]) is process-wide. +fn lock() -> MutexGuard<'static, ()> { + static LOCK: Mutex<()> = Mutex::new(()); + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +fn fe(v: u64) -> FE { + FE::from(v) +} + +fn ext(a: u64, b: u64, c: u64) -> FEE { + FEE::new([fe(a), fe(b), fe(c)]) +} + +fn options(merge: bool) -> ProofOptions { + let mut o = GoldilocksCubicProofOptions::with_blowup(2).expect("options"); + o.format.merge_equal_height_tables = merge; + o +} + +/// A program with real rows in all four candidates — base ALU (add, sub, mul), +/// one select, an unpack and a pack, one hinted felt — each padded to the +/// four-row floor, so the four share a height and merge. +fn four_chip_program() -> (LfmProgram, Vec>) { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(1); + let h0 = b.hint_felt(arena, 0); + let x = b.felt_const(fe(7)); + let y = b.felt_const(fe(5)); + let s = b.add(x, y); + let d = b.sub(s, h0); + let m = b.mul(d, y); + // An add nobody reads: its output's write multiplicity is zero, so its row + // sends no memory token — only BALU's own body constrains its OUT + // (`a_bad_balu_row_in_a_merged_table_is_refused_by_balus_own_body`). + let _unread = b.add(y, x); + let e0 = b.ext_const(&ext(1, 2, 3)); + let e1 = b.ext_const(&ext(4, 5, 6)); + let ep = b.emul(e0, e1); + let bit = b.bit_const(true); + let (l, r) = b.select(bit, x.as_cell(), y.as_cell()); + let da = b.digest_const([fe(1), fe(2), fe(3), fe(4)]); + let db = b.digest_const([fe(5), fe(6), fe(7), fe(8)]); + let dc = b.compress(da, db); + let lanes = b.unpack(dc.as_cell()); + let packed = b.pack_word(lanes); + b.public(packed); + b.public(l); + b.public(r); + b.public(ep.as_cell()); + b.public(m.as_cell()); + (compile(b.finish()), vec![vec![base_word(fe(3))]]) +} + +fn verifies(artifacts: &LfmArtifacts, proof: &LfmProof, opts: &ProofOptions) -> bool { + verify_against_artifacts(artifacts, &proof.proof, &proof.public_words, opts) +} + +fn sub_proofs(proof: &LfmProof) -> usize { + MultiProofView::Owned(&proof.proof.multi).len() +} + +// ================================ the plan ================================= + +#[test] +fn the_plan_merges_equal_heights_only() { + let mut h = [0u8; NUM_LFM_CHIPS]; + let set = |h: &mut [u8; NUM_LFM_CHIPS], b, s, l, i| { + h[MergeChip::Balu.slot()] = b; + h[MergeChip::Select.slot()] = s; + h[MergeChip::Lanes.slot()] = l; + h[MergeChip::Hint.slot()] = i; + }; + // A leaf: all four at 2^21. + set(&mut h, 21, 21, 21, 21); + assert_eq!(plan_for_heights(&h, true), vec![MERGE_CANDIDATES.to_vec()]); + assert!(plan_for_heights(&h, false).is_empty(), "off merges nothing"); + // A top: SELECT and HINT at 2^20, BALU and LANES at 2^19 — two merged AIRs, + // ordered by their first chip. + set(&mut h, 19, 20, 19, 20); + assert_eq!( + plan_for_heights(&h, true), + vec![ + vec![MergeChip::Balu, MergeChip::Lanes], + vec![MergeChip::Select, MergeChip::Hint] + ] + ); + // A chip alone at its height keeps its own AIR. + set(&mut h, 18, 20, 20, 19); + assert_eq!( + plan_for_heights(&h, true), + vec![vec![MergeChip::Select, MergeChip::Lanes]] + ); + // Four heights: nothing to merge. + set(&mut h, 17, 18, 19, 20); + assert!(plan_for_heights(&h, true).is_empty()); + // Other chips' heights never matter. + let mut other = [21u8; NUM_LFM_CHIPS]; + set(&mut other, 17, 18, 19, 20); + assert!(plan_for_heights(&other, true).is_empty()); +} + +#[test] +fn the_layout_puts_every_prefix_first_and_maps_each_column_once() { + let layout = MergedLayout::of(&MERGE_CANDIDATES); + let shapes: Vec = MERGE_CANDIDATES.iter().map(|&c| PartShape::of(c)).collect(); + assert_eq!( + layout.num_prep, + shapes.iter().map(|s| s.prep).sum::() + ); + assert_eq!( + layout.num_columns, + shapes.iter().map(|s| s.width).sum::() + ); + // The census of the note: 32 preprocessed, 61 in all. + assert_eq!((layout.num_prep, layout.num_columns), (32, 61)); + let mut hit = vec![0usize; layout.num_columns]; + for (part, shape) in layout.parts.iter().zip(&shapes) { + assert_eq!(part.map.len(), shape.width); + for (c, &m) in part.map.iter().enumerate() { + hit[m] += 1; + // A prefix column stays in the merged prefix; a value column after it. + assert_eq!( + c < shape.prep, + m < layout.num_prep, + "{:?} column {c}", + part.chip + ); + } + // Each region keeps the part's order and is contiguous. + assert!( + part.map + .windows(2) + .all(|w| w[1] > w[0] || w[1] < layout.num_prep) + ); + } + assert!( + hit.iter().all(|&n| n == 1), + "every merged column is one part's" + ); +} + +// ============================ the constraint body ========================== + +/// A builder that records which leaves a body reads. +#[derive(Default)] +struct Recorder { + mains: RefCell>, + others: RefCell, + emitted: Vec, +} + +impl ConstraintBuilder for Recorder { + type Expr = Nil; + type ExprE = Nil; + fn main(&self, offset: usize, col: usize) -> Nil { + self.mains.borrow_mut().push((offset, col)); + Nil + } + fn aux(&self, _: usize, _: usize) -> Nil { + *self.others.borrow_mut() += 1; + Nil + } + fn challenge(&self, _: usize) -> Nil { + *self.others.borrow_mut() += 1; + Nil + } + fn alpha_pow(&self, _: usize) -> Nil { + *self.others.borrow_mut() += 1; + Nil + } + fn table_offset(&self) -> Nil { + *self.others.borrow_mut() += 1; + Nil + } + fn const_base(&self, _: u64) -> Nil { + Nil + } + fn const_signed(&self, _: i64) -> Nil { + Nil + } + fn emit_base_rows(&mut self, idx: usize, _: RowDomain, _: Nil) { + self.emitted.push(idx); + } + fn emit_ext_rows(&mut self, _: usize, _: RowDomain, _: Nil) { + *self.others.borrow_mut() += 1; + } +} + +/// ★ The premise [`super::merge::Remap`] rests on: a candidate's body reads +/// only its own main columns, at the current row, and emits base constraints +/// only — so remapping `main` and shifting the indices is the whole +/// translation. Then the merged body emits every part's constraints, densely, +/// each reading exactly its part's columns where the layout put them. +#[test] +fn a_part_body_reads_only_its_main_columns() { + let _g = lock(); + for chip in MERGE_CANDIDATES { + let shape = PartShape::of(chip); + let mut r = Recorder::default(); + match shape.body { + PartBody::Balu => super::chips::balu::BaluConstraints.eval(&mut r), + PartBody::Select => super::chips::select::SelectConstraints.eval(&mut r), + PartBody::Empty => {} + } + assert_eq!(*r.others.borrow(), 0, "{chip:?} reads no aux leaf"); + assert!( + r.mains + .borrow() + .iter() + .all(|&(o, c)| o == 0 && c < shape.width), + "{chip:?} reads its own columns at the current row" + ); + assert_eq!(r.emitted.len(), shape.num_base_constraints()); + } + let layout = MergedLayout::of(&MERGE_CANDIDATES); + let set = MergedConstraints::new(&layout).expect("the candidates pass the audit"); + let mut r = Recorder::default(); + set.eval(&mut r); + let mut emitted = r.emitted.clone(); + emitted.sort_unstable(); + let total: usize = MERGE_CANDIDATES + .iter() + .map(|&c| PartShape::of(c).num_base_constraints()) + .sum(); + assert_eq!(emitted, (0..total).collect::>(), "dense, each once"); + assert_eq!(ConstraintSet::::max_degree(&set), 3, "BALU's degree"); + // BALU reads A, B, C, OUT and its five selectors; under the merge those are + // its map's images. + let balu = &layout.parts[0]; + let mut own = Recorder::default(); + super::chips::balu::BaluConstraints.eval(&mut own); + let want: Vec<(usize, usize)> = own + .mains + .borrow() + .iter() + .map(|&(o, c)| (o, balu.map[c])) + .collect(); + assert_eq!(r.mains.borrow()[..want.len()], want[..]); +} + +/// ★ The buses carry over interaction by interaction: same bus, same side, +/// same element count, and every column read moved by its part's map — and +/// every packing stays contiguous (the premise of `remap_interaction`). +#[test] +fn every_merge_remaps_its_buses() { + let layout = MergedLayout::of(&MERGE_CANDIDATES); + let merged = merged_interactions(&layout).expect("contiguous"); + let mut at = 0; + for part in &layout.parts { + for original in PartShape::of(part.chip).interactions() { + let m = &merged[at]; + at += 1; + assert_eq!(m.bus_id, original.bus_id); + assert_eq!(m.is_sender, original.is_sender); + assert_eq!(m.num_bus_elements(), original.num_bus_elements()); + let mut want: Vec = original + .columns_read() + .iter() + .map(|&c| part.map[c]) + .collect(); + want.sort_unstable(); + assert_eq!(m.columns_read(), want, "{:?}", part.chip); + for v in &original.values { + if let stark::lookup::BusValue::Packed { + start_column, + packing, + } = v + { + let first = part.map[*start_column]; + assert!( + (0..packing.num_columns()).all(|k| part.map[start_column + k] == first + k) + ); + } + } + } + } + assert_eq!(at, merged.len()); + // 4 + 5 + 10 + 1 interactions: ten aux columns where the parts had eleven. + assert_eq!(merged.len(), 20); +} + +// ================================ the traces =============================== + +fn main_cells(t: &TraceTable) -> Vec { + t.main_data_row_major().0.to_vec() +} + +/// ★ The direct fill IS the parts side by side: the merged table built by the +/// fill equals the one laid out from the unmerged tables, cell for cell, and +/// splitting it back gives the unmerged tables. With an empty plan the fill is +/// the unmerged fill. +#[test] +fn the_merged_fill_is_the_parts_side_by_side() { + let (program, arenas) = four_chip_program(); + let hasher = build_artifacts(&program, &options(false)).hasher; + let exec = execute(&program, &arenas, &hasher).expect("executes"); + let plan = plan_for_program(&program, true); + assert_eq!( + plan, + vec![MERGE_CANDIDATES.to_vec()], + "the four share the floor" + ); + + let plain = build_traces_with_hasher(&program, &exec.records, hasher); + let same = build_traces_with_merge(&program, &exec.records, hasher, &Vec::new()); + let chips = |t: &LfmTraces| -> Vec> { + [&t.balu, &t.select, &t.lanes, &t.hint] + .iter() + .map(|x| main_cells(x)) + .collect() + }; + assert_eq!(chips(&plain), chips(&same), "no plan, no change"); + assert!(same.merged.is_empty()); + + let direct = build_traces_with_merge(&program, &exec.records, hasher, &plan); + assert_eq!(direct.merged.len(), 1); + assert!(direct.balu.num_rows() == 0 && direct.hint.num_rows() == 0); + let mut copied = build_traces_with_hasher(&program, &exec.records, hasher); + copied.merge_to(&plan).expect("merges"); + assert_eq!(main_cells(&direct.merged[0]), main_cells(&copied.merged[0])); + // Cell by cell against the parts' own tables. + let layout = MergedLayout::of(&plan[0]); + let parts = chips(&plain); + let merged = main_cells(&direct.merged[0]); + for (part, cells) in layout.parts.iter().zip(&parts) { + let w = part.map.len(); + for row in 0..direct.merged[0].num_rows() { + for (c, &m) in part.map.iter().enumerate() { + assert_eq!(merged[row * layout.num_columns + m], cells[row * w + c]); + } + } + } + copied.merge_to(&Vec::new()).expect("splits back"); + assert_eq!(chips(&copied), chips(&plain)); + assert!(copied.merged.is_empty()); +} + +// ============================ prove and verify ============================= + +/// ★ The round trip, and identity: a merged program has one AIR where its +/// four chips had four, its merged chips' own roots are the zero sentinel and +/// the merged root is bound into a different `program_id`; it proves and +/// verifies under its own artifacts and is refused under the unmerged ones, +/// and the reverse. With the flag off the artifacts carry no merge. +#[test] +fn a_merged_program_proves_and_verifies_under_its_own_artifacts_only() { + let _g = lock(); + let (program, arenas) = four_chip_program(); + let (on, off) = (options(true), options(false)); + let merged = build_artifacts(&program, &on); + let plain = build_artifacts(&program, &off); + assert!(plain.merge.is_empty(), "off: no merge"); + assert_eq!(merged.merge.plan(), vec![MERGE_CANDIDATES.to_vec()]); + assert_ne!( + merged.program_id, plain.program_id, + "the layout is identity" + ); + for c in MERGE_CANDIDATES { + assert_eq!( + merged.roots[c.slot()], + [0u8; 32], + "{c:?}: no root of its own" + ); + assert_ne!(plain.roots[c.slot()], [0u8; 32]); + } + let p_merged = lfm_prove(&program, &merged, &arenas, &on).expect("merged proves"); + let p_plain = lfm_prove(&program, &plain, &arenas, &off).expect("plain proves"); + assert_eq!(sub_proofs(&p_merged) + 3, sub_proofs(&p_plain)); + assert_eq!(p_merged.public_words, p_plain.public_words); + assert!( + verifies(&merged, &p_merged, &on), + "the merged proof verifies" + ); + assert!(verifies(&plain, &p_plain, &off), "the plain proof verifies"); + assert!( + !verifies(&plain, &p_merged, &off), + "a merged proof under plain artifacts" + ); + assert!( + !verifies(&merged, &p_plain, &on), + "a plain proof under merged artifacts" + ); +} + +/// The merged artifacts' AIR order: the merged AIR stands at BALU's place, +/// the other three chips leave, and the heights in AIR order follow +/// (`log_heights_in_air_order`, what a parent derives its child from). +#[test] +fn the_merged_air_stands_at_its_first_chip() { + let _g = lock(); + let (program, _) = four_chip_program(); + let opts = options(true); + let artifacts = build_artifacts(&program, &opts); + let airs = LfmAirs::for_artifacts(&artifacts, &opts).expect("the AIR set"); + let names: Vec = airs + .air_refs() + .iter() + .map(|a| a.name().to_string()) + .collect(); + assert_eq!(names[1], "LFM_MERGED[BALU+SELECT+LANES+HINT]", "{names:?}"); + for gone in ["LFM_BALU", "LFM_SELECT", "LFM_LANES", "LFM_HINT"] { + assert!(!names.iter().any(|n| n == gone), "{gone} left: {names:?}"); + } + let heights = LfmAirs::log_heights_in_air_order(&artifacts).expect("no keccak"); + assert_eq!(heights.len(), names.len()); + let merged = &airs.air_refs()[1]; + assert_eq!(merged.num_precomputed_columns(), 32); + assert_eq!(merged.context().trace_columns, 61 + 10, "61 main, 10 aux"); +} + +/// The traces with SELECT's padding row 2 given a non-boolean bit: no bus +/// token moves (the row's multiplicities are zero), only SELECT's booleanity +/// fails. +fn bad_select_traces( + program: &LfmProgram, + arenas: &[Vec], + artifacts: &LfmArtifacts, +) -> (LfmTraces, Vec<(u32, LfmWord)>) { + let exec = execute(program, arenas, &artifacts.hasher).expect("executes"); + let plan = artifacts.merge.plan(); + let mut traces = build_traces_with_merge(program, &exec.records, artifacts.hasher, &plan); + let layout = MergedLayout::of(&plan[0]); + let select = layout + .parts + .iter() + .find(|p| p.chip == MergeChip::Select) + .expect("SELECT is merged"); + let row = 2; + assert!(row >= program.groups.select.real_rows, "a padding row"); + traces.merged[0].set_main(row, select.map[super::chips::select::cols::BIT], fe(2)); + (traces, exec.public_words) +} + +/// ★ Negative, then the mutant: a merged table with a non-boolean SELECT bit +/// on a padding row is refused by the verifier — and with the merged AIR's +/// SELECT body vacuous (its indices kept, every constraint zero; prover and +/// verifier both), the same table proves and VERIFIES. So the part's body +/// inside the merged AIR is what refuses it. +#[test] +fn a_bad_select_row_in_a_merged_table_is_refused_by_selects_own_body() { + let _g = lock(); + let (program, arenas) = four_chip_program(); + let opts = options(true); + let artifacts = build_artifacts(&program, &opts); + + let (mut traces, words) = bad_select_traces(&program, &arenas, &artifacts); + let refused = match prove_traces(&artifacts, &mut traces, &words, &opts) { + Ok(proof) => !verify_against_artifacts(&artifacts, &proof, &words, &opts), + Err(_) => true, + }; + assert!(refused, "the bad SELECT row must not verify"); + + test_hooks::set_vacuous(Some(PartBody::Select)); + let mutant = std::panic::catch_unwind(|| { + let (mut traces, words) = bad_select_traces(&program, &arenas, &artifacts); + let proof = + prove_traces(&artifacts, &mut traces, &words, &opts).expect("the mutant proves"); + verify_against_artifacts(&artifacts, &proof, &words, &opts) + }); + test_hooks::set_vacuous(None); + assert!( + mutant.expect("the mutant runs"), + "without SELECT's body the bad row verifies: the body is load-bearing" + ); +} + +/// A wrong BALU output inside the merged table (a real row, so it also moves +/// the memory bus) is refused. +#[test] +fn a_wrong_balu_output_in_a_merged_table_is_refused() { + let _g = lock(); + let (program, arenas) = four_chip_program(); + let opts = options(true); + let artifacts = build_artifacts(&program, &opts); + let exec = execute(&program, &arenas, &artifacts.hasher).expect("executes"); + let plan = artifacts.merge.plan(); + let mut traces = build_traces_with_merge(&program, &exec.records, artifacts.hasher, &plan); + let layout = MergedLayout::of(&plan[0]); + let col = layout.parts[0].map[super::chips::balu::cols::OUT]; + let v = *traces.merged[0].get_main(0, col); + traces.merged[0].set_main(0, col, v + fe(1)); + let refused = match prove_traces(&artifacts, &mut traces, &exec.public_words, &opts) { + Ok(proof) => !verify_against_artifacts(&artifacts, &proof, &exec.public_words, &opts), + Err(_) => true, + }; + assert!(refused); +} + +// ============================== the node's leg ============================= + +/// ★ A node's leg verifies a MERGED Poseidon1 child in-guest: the leg is +/// derived from the merged artifacts (no leg-specific code), the honest child +/// executes, and tampers of the merged table's root, an opened value and an +/// OOD value are refused. The node over the merged child takes fewer socket +/// rows than over the plain one (printed for the census). +#[test] +fn a_node_verifies_a_merged_child_and_refuses_its_tampers() { + let _g = lock(); + let program = super::programs::trivial_program(); + let child = |merge: bool| -> P1Child { + let mut opts = super::p1_node_tests::options(); + opts.format.merge_equal_height_tables = merge; + let hasher = build_artifacts(&program, &opts).hasher; + let artifacts = + build_artifacts_under::<::Stark>(&program, &opts, hasher); + let proof = lfm_prove_under::( + &program, + &artifacts, + &super::p1_node_tests::arenas(), + &opts, + ) + .expect("prove"); + assert!(super::proof::verify_against_artifacts_under::( + &artifacts, + &proof.proof, + &proof.public_words, + &opts + )); + P1Child { + artifacts, + proof, + opts, + } + }; + let c = child(true); + assert!(!c.artifacts.merge.is_empty(), "the trivial program merges"); + let honest = harvest(&c); + let node_program = node(&c, P1Family::STATEMENT_TAG); + assert!( + runs(&node_program, &honest), + "the honest merged child executes" + ); + + let airs = LfmAirs::for_artifacts(&c.artifacts, &c.opts).expect("the AIR set"); + let t = airs + .air_refs() + .iter() + .position(|a| a.name().starts_with("LFM_MERGED")) + .expect("a merged AIR"); + let refused = |what: &str, edit: &dyn Fn(&mut super::harvest::HarvestedChild)| { + let mut bad = harvest(&c); + edit(&mut bad); + assert!(!runs(&node_program, &bad), "{what}: the node must refuse"); + }; + refused("merged main root", &|h| h.tables[t].main_root[31] ^= 1); + refused("merged opened value", &|h| { + h.legs[t].openings[0][0].0[0][0] += fe(1) + }); + refused("merged OOD value", &|h| { + h.tables[t].ood_current[0] += FEE::from(1u64) + }); + + let plain = child(false); + let plain_node = node(&plain, P1Family::STATEMENT_TAG); + let (rows_merged, rows_plain) = ( + node_program.groups.hash.real_rows, + plain_node.groups.hash.real_rows, + ); + println!( + "MERGE NODE CENSUS: socket rows for one child: merged {rows_merged} · plain {rows_plain} · AIRs {} vs {}", + sub_proofs(&c.proof), + sub_proofs(&plain.proof) + ); + assert!(rows_merged < rows_plain, "fewer tables, fewer paths"); +} + +/// ★ A node over a merged and a plain child — children of one node whose AIR +/// sets differ, as the large block's two level-1 nodes did (8 and 9 AIRs) — +/// streamed as the tree's top is: each child's arena count as the node's +/// emitter declares it (`leg_arena_count` over the derived shape) is what its +/// harvest supplies, the two differ, and split by those counts the streamed +/// execution is the unstreamed one, word for word, in either landing order. An +/// even split (what the top assumed before) is refused, never a witness. +#[test] +fn a_streamed_node_splits_a_merged_and_a_plain_child_by_their_own_arenas() { + use super::exec_identity_tests::{assert_executions_identical, streamed_top_over}; + use super::harvest::try_child_arena_words; + use super::hash::HasherKind; + use super::per_table_aggregator::{ + DerivedChild, declare_leg_arenas, emit_leg, leg_arena_count, + }; + let _g = lock(); + let child = |merge: bool| { + let mut opts = super::p1_node_tests::options(); + opts.format.merge_equal_height_tables = merge; + super::p1_node_tests::p1_child_with(opts) + }; + let kids = [child(true), child(false)]; + assert!( + !kids[0].artifacts.merge.is_empty(), + "the trivial program merges" + ); + assert!( + kids[1].artifacts.merge.is_empty(), + "and not with the flag off" + ); + let derived: Vec = kids + .iter() + .map(|c| { + DerivedChild::from_artifacts_under::( + &c.artifacts, + &c.opts, + c.proof.public_words.len(), + ) + .expect("the child's shape") + }) + .collect(); + let words: Vec>> = kids + .iter() + .map(|c| try_child_arena_words(&harvest(c)).expect("the child's arenas")) + .collect(); + let counts: Vec = derived + .iter() + .map(|d| leg_arena_count(&d.shape(), P1Family::WRAP)) + .collect(); + assert_eq!( + counts, + words.iter().map(Vec::len).collect::>(), + "each child's declared arenas are what its harvest supplies" + ); + assert!( + counts[0] < counts[1], + "the merged child supplies fewer arenas: {counts:?}" + ); + + let shapes: Vec<_> = derived.iter().map(|d| d.shape()).collect(); + let mut b = LfmBuilder::new().with_wrap_hash(P1Family::WRAP); + let arenas: Vec<_> = shapes + .iter() + .map(|s| declare_leg_arenas(&mut b, s)) + .collect(); + for (shape, a) in shapes.iter().zip(&arenas) { + let _ = emit_leg(&mut b, shape, a); + } + let program = compile(b.finish()); + assert_eq!( + program.arena_schema.lens.len(), + counts.iter().sum::() + ); + let hasher = HasherKind::Poseidon1W16; + let reference = execute(&program, &words.concat(), &hasher).expect("the node executes"); + for order in [[0, 1], [1, 0]] { + let got = streamed_top_over(&program, &words, &counts, &order, &hasher) + .unwrap_or_else(|e| panic!("{order:?}: the streamed node executes: {e}")); + assert_executions_identical("merged + plain", program.num_addrs, &reference, &got); + } + let total: usize = counts.iter().sum(); + let even = [total / 2, total - total / 2]; + let err = streamed_top_over(&program, &words, &even, &[0, 1], &hasher) + .expect_err("an even split is refused"); + assert!( + err.contains("child 0 lands: ArenaCountMismatch"), + "the even split fails where the large block did: {err}" + ); + println!("MERGE STREAMED NODE: arenas a child {counts:?} (an even split {even:?} refused)"); +} + +// ================================ the knob ================================= + +#[test] +fn the_merge_knob_parses_prints_and_refuses_like_its_siblings() { + use crate::zf_format::{ENV_LFM_MERGE, ENV_ONE_ROW, ZfFormat}; + let parse = |pairs: &[(&str, &str)]| { + ZfFormat::from_lookup(|name| { + pairs + .iter() + .find(|(k, _)| *k == name) + .map(|(_, v)| v.to_string()) + }) + }; + // The default merges (the flip); its banner is unchanged, the rollback + // prints its word. + let default = parse(&[]).expect("default"); + assert!(default.lfm_merge && default.proof_format().merge_equal_height_tables); + assert!( + !default.banner().contains("lfm_merge"), + "the default banner is today's" + ); + let on = parse(&[(ENV_LFM_MERGE, "1")]).expect("on"); + assert_eq!(on, default); + let off = parse(&[(ENV_LFM_MERGE, "0")]).expect("off"); + assert!(!off.lfm_merge && !off.proof_format().merge_equal_height_tables); + assert!(off.banner().ends_with(" lfm_merge=0")); + assert!(parse(&[(ENV_LFM_MERGE, "2")]).is_err(), "a typo aborts"); + assert!( + parse(&[(ENV_LFM_MERGE, "1"), (ENV_ONE_ROW, "1")]).is_err(), + "no merged one-row layout" + ); + // One-row alone keeps parsing: the merge is then off. + for one_row in ["1", "auto"] { + let f = parse(&[(ENV_ONE_ROW, one_row)]).expect("one-row alone"); + assert!(!f.lfm_merge, "one_row={one_row}"); + } + const { assert!(!ZfFormat::LEGACY.lfm_merge && ZfFormat::DEFAULT.lfm_merge) }; + // The library treats the pair as off on both sides. + let (program, _) = four_chip_program(); + let mut both = options(true); + both.format.one_row = OneRowMode::On; + assert!(super::registry::merge_plan_for(&program, &both).is_empty()); + assert_eq!(LfmMerge::none().airs_saved(), 0); +} + +// ================================ the census =============================== + +/// One tree program's AIRs under its artifacts: count, and main + aux cells +/// (an aux column is three base cells) over every AIR. +fn airs_and_cells(artifacts: &LfmArtifacts, wrap: &ProofOptions) -> (usize, u64) { + let airs = LfmAirs::for_artifacts(artifacts, wrap).expect("the AIR set"); + let heights = LfmAirs::log_heights_in_air_order(artifacts).expect("no keccak"); + let refs = airs.air_refs(); + let cells = refs + .iter() + .zip(&heights) + .map(|(air, h)| { + let aux = air.context().trace_columns - air.trace_layout().0; + (1u64 << h) * (air.trace_layout().0 + 3 * aux) as u64 + }) + .sum(); + (refs.len(), cells) +} + +/// ★ The merge census (instrument, `--ignored --exact`): the median mix's +/// plan (the production format, the Poseidon1 base) under today's default +/// leaf scale, every leaf emitted once, heights only; per arm (plain, merged): +/// each leaf's AIRs and cells, then the top emitted over the leaves' shapes — +/// its socket rows, its chips' rows and its own AIRs — and the socket rows one +/// node spends on one leaf. No commit, no proof. +/// +/// `CENSUS_ELF` names a real guest (default: the asm test guest). +#[test] +#[ignore = "instrument: run alone with --exact"] +fn merge_census() { + use super::whir_block::WhirBlockPlan; + use super::whir_chain::ChipRows; + use crate::block_whir::BlockFormat; + let elf_bytes = match std::env::var("CENSUS_ELF") { + Ok(path) => std::fs::read(path).expect("read CENSUS_ELF"), + Err(_) => crate::test_utils::asm_elf_bytes("poc_rodata_commit"), + }; + let opts = super::proof::block_base_options(); + let production = BlockFormat::production(); + let format = BlockFormat { + zf: production + .zf + .with_base(stark::proof::options::BaseFormat::P1_WHIR), + ..production + }; + let owned = super::whir_p1_census_tests::median_statement(&elf_bytes, &opts, &format); + let plan = WhirBlockPlan::derive(&elf_bytes, &opts, &format, owned.view(), None) + .expect("the plan derives"); + let words = plan.child_layout().total(); + println!( + "MERGE CENSUS plan: {} groups · {} leaves · levels {:?}", + plan.num_groups(), + plan.partition().num_leaves(), + plan.levels() + .iter() + .map(|l| l.arities.clone()) + .collect::>() + ); + for on in [false, true] { + let label = if on { "merged" } else { "plain" }; + let mut wrap = super::proof::aggregation_wrap_options(); + wrap.format.merge_equal_height_tables = on; + let mut kids = Vec::new(); + for k in 0..plan.partition().num_leaves() { + let program = plan.leaf_program(k).expect("the leaf emits"); + let a = super::registry::census_artifacts_merged( + &program, + program.hasher(::SOCKET), + on, + ); + let (n, cells) = airs_and_cells(&a, &wrap); + println!( + "MERGE CENSUS {label} leaf {k}: {n} AIRs · {:.1} M main+aux cells · merged {:?}", + cells as f64 / 1e6, + a.merge.plan() + ); + kids.push(super::whir_block::derived_of(&a, &wrap, words).expect("derives")); + } + let one = plan + .node_program(&[&kids[0]], false) + .expect("one-child node"); + let one_rows = ChipRows::of(&one); + println!( + "MERGE CENSUS {label} one-child node over leaf 0: socket {} · {one_rows:?}", + one_rows.hash + ); + drop(one); + // The first node above the leaves (the top when the tree has one + // level): its rows, and its own AIRs under the arm. + let level = plan.levels(); + let top_arity = level.first().map_or(1, |l| l.arities[0]); + { + let is_top = level.len() == 1; + let refs: Vec<_> = kids.iter().take(top_arity).collect(); + let top = plan.node_program(&refs, is_top).expect("the node emits"); + let rows = ChipRows::of(&top); + let a = super::registry::census_artifacts_merged( + &top, + top.hasher(::SOCKET), + on, + ); + let (n, cells) = airs_and_cells(&a, &wrap); + println!( + "MERGE CENSUS {label} {} over {top_arity}: socket {} · {rows:?} · {n} AIRs · {:.1} M cells · merged {:?} · heights {:?}", + if is_top { "top" } else { "first node" }, + rows.hash, + cells as f64 / 1e6, + a.merge.plan(), + a.log_heights + ); + } + } +} + +// ============================= review hardening ============================ + +/// ★ The merge's well-formedness is checked on EVERY path that +/// builds a merged AIR set — `LfmAirs::for_artifacts` (the leg's derivation and +/// the harvest), the host verifier and the prover — against the artifacts' +/// heights: a one-chip group, chips out of slot order, a chip in two merged +/// AIRs and a chip at another height than its merged AIR are each refused, and +/// nothing is built. +#[test] +fn a_malformed_merge_is_refused_on_every_path() { + let _g = lock(); + let (program, arenas) = four_chip_program(); + let opts = options(true); + let good = build_artifacts(&program, &opts); + let proof = lfm_prove(&program, &good, &arenas, &opts).expect("proves"); + assert!(verifies(&good, &proof, &opts)); + let t = &good.merge.tables[0]; + let with = |tables: Vec| LfmArtifacts { + merge: LfmMerge { tables }, + ..good.clone() + }; + let lone = MergedTable { + chips: vec![MergeChip::Balu], + ..t.clone() + }; + let rest = MergedTable { + chips: t.chips[1..].to_vec(), + ..t.clone() + }; + let swapped = MergedTable { + chips: vec![ + MergeChip::Select, + MergeChip::Balu, + MergeChip::Lanes, + MergeChip::Hint, + ], + ..t.clone() + }; + let taller = MergedTable { + log_height: t.log_height + 1, + ..t.clone() + }; + let cases = [ + ("a one-chip group", with(vec![lone, rest])), + ("chips out of slot order", with(vec![swapped])), + ( + "a chip in two merged AIRs", + with(vec![t.clone(), t.clone()]), + ), + ("a chip at another height", with(vec![taller])), + ]; + for (what, bad) in &cases { + let err = LfmAirs::for_artifacts(bad, &opts).err(); + assert!(err.is_some(), "{what}: the AIR set must be refused"); + println!("MERGE REFUSED {what}: {}", err.unwrap_or_default()); + assert!( + super::per_table_aggregator::DerivedChild::from_artifacts(bad, &opts, 5).is_err(), + "{what}: the leg's derivation must refuse it" + ); + assert!(!verifies(bad, &proof, &opts), "{what}: the host verifier"); + let exec = execute(&program, &arenas, &bad.hasher).expect("executes"); + let mut traces = build_traces_with_hasher(&program, &exec.records, bad.hasher); + assert!( + prove_traces(bad, &mut traces, &exec.public_words, &opts).is_err(), + "{what}: the prover" + ); + } +} + +/// ★ The bus remap refuses, in every build, a packing that the +/// merged layout would split: a two-column packing over a part's last prefix +/// column and its first value column lands on non-adjacent merged columns. +#[test] +fn a_packing_split_by_the_layout_is_refused() { + use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; + let layout = MergedLayout::of(&[MergeChip::Balu, MergeChip::Select]); + let balu = &layout.parts[0]; + let prep = PartShape::of(MergeChip::Balu).prep; + let straddle = BusInteraction::sender( + 7u64, + Multiplicity::One, + vec![BusValue::Packed { + start_column: prep - 1, + packing: Packing::Word2L, + }], + ); + assert!(super::merge::remap_interaction(&straddle, &balu.map).is_err()); + let inside = BusInteraction::sender( + 7u64, + Multiplicity::One, + vec![BusValue::Packed { + start_column: prep, + packing: Packing::Word2L, + }], + ); + let ok = super::merge::remap_interaction(&inside, &balu.map).expect("contiguous"); + assert_eq!(ok.columns_read(), vec![balu.map[prep], balu.map[prep] + 1]); +} + +/// A body the remap could not translate faithfully. +struct ReadsAux; +impl ConstraintSet for ReadsAux { + fn eval>(&self, b: &mut B) { + let a = b.aux(0, 0); + let m = b.main(0, 0); + b.emit_ext(0, m * a); + } +} +struct ReadsPastWidth; +impl ConstraintSet for ReadsPastWidth { + fn eval>(&self, b: &mut B) { + let m = b.main(0, 99); + b.emit_base(0, m); + } +} +struct Sparse; +impl ConstraintSet for Sparse { + fn eval>(&self, b: &mut B) { + let m = b.main(0, 0); + b.emit_base(1, m); + } +} + +/// ★ The guard behind `Remap`'s pass-through leaves: the audit +/// every merged AIR build runs accepts the four candidates and refuses a body +/// that reads an aux column (or emits an extension constraint), one that reads +/// past its width, and one whose indices are not dense. +#[test] +fn the_body_audit_refuses_what_the_remap_cannot_translate() { + for chip in MERGE_CANDIDATES { + super::merge::audit_body(PartShape::of(chip)).expect("a candidate passes"); + } + assert!(super::merge::audit_set(&ReadsAux, 4, "aux").is_err()); + assert!(super::merge::audit_set(&ReadsPastWidth, 4, "width").is_err()); + assert!(super::merge::audit_set(&Sparse, 4, "sparse").is_err()); +} + +/// The commitment-time answers a merged AIR must carry: every part's value +/// columns, counted from the parts' own shapes (not from the AIR the answer set +/// is computed from). +fn answers_of(chips: &[MergeChip]) -> usize { + chips + .iter() + .map(|&c| { + let s = PartShape::of(c); + s.width - s.prep + }) + .sum() +} + +/// For every merged AIR of `artifacts` (in AIR order), its sub-proof index. +fn merged_positions(artifacts: &LfmArtifacts, opts: &ProofOptions) -> Vec { + let airs = LfmAirs::for_artifacts(artifacts, opts).expect("the AIR set"); + airs.air_refs() + .iter() + .enumerate() + .filter(|(_, a)| a.name().starts_with("LFM_MERGED")) + .map(|(i, _)| i) + .collect() +} + +/// ★ A merged AIR answers, at the commitment-time point, exactly +/// its parts' value columns — 29 for the four candidates — and a false answer +/// at its last answered column is refused. +#[test] +fn a_merged_air_answers_every_parts_value_columns() { + let _g = lock(); + let (program, arenas) = four_chip_program(); + let opts = options(true); + let artifacts = build_artifacts(&program, &opts); + let proof = lfm_prove(&program, &artifacts, &arenas, &opts).expect("proves"); + let at = merged_positions(&artifacts, &opts); + assert_eq!(at.len(), 1); + let answers = &proof.proof.commit_ood[at[0]]; + assert_eq!(answers.len(), answers_of(&MERGE_CANDIDATES)); + assert_eq!(answers.len(), 29); + let mut bad = proof; + let last = bad.proof.commit_ood[at[0]].len() - 1; + bad.proof.commit_ood[at[0]][last] += FEE::from(1u64); + assert!(!verifies(&artifacts, &bad, &opts), "a false last answer"); +} + +/// A program whose SELECT and HINT sit at eight rows and BALU and LANES at the +/// four-row floor: the two-merged-AIR shape of a node or a top. +fn two_group_program() -> (LfmProgram, Vec>) { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(6); + let felts: Vec<_> = (0..6).map(|i| b.hint_felt(arena, i)).collect(); + let s = b.add(felts[0], felts[1]); + b.public(s.as_cell()); + for i in 0..6 { + let bit = b.bit_const(i % 2 == 0); + let (l, r) = b.select(bit, felts[i].as_cell(), felts[(i + 1) % 6].as_cell()); + b.public(l); + b.public(r); + } + let da = b.digest_const([fe(1), fe(2), fe(3), fe(4)]); + let lanes = b.unpack(da.as_cell()); + let packed = b.pack_word(lanes); + b.public(packed); + let words = (0..6u64).map(|i| base_word(fe(100 + i))).collect(); + (compile(b.finish()), vec![words]) +} + +/// ★ Two merged AIRs in one proof — {BALU, LANES} and +/// {SELECT, HINT}, the node and top shape: the plan, the round trip, the +/// cross refusals, two AIRs fewer than plain, and each merged AIR's answers. +#[test] +fn two_merged_airs_in_one_proof_prove_and_verify() { + let _g = lock(); + let (program, arenas) = two_group_program(); + let (on, off) = (options(true), options(false)); + let merged = build_artifacts(&program, &on); + let plain = build_artifacts(&program, &off); + assert_eq!( + merged.merge.plan(), + vec![ + vec![MergeChip::Balu, MergeChip::Lanes], + vec![MergeChip::Select, MergeChip::Hint] + ] + ); + let p_merged = lfm_prove(&program, &merged, &arenas, &on).expect("merged proves"); + let p_plain = lfm_prove(&program, &plain, &arenas, &off).expect("plain proves"); + assert_eq!(sub_proofs(&p_merged) + 2, sub_proofs(&p_plain)); + assert!(verifies(&merged, &p_merged, &on)); + assert!(verifies(&plain, &p_plain, &off)); + assert!(!verifies(&plain, &p_merged, &off)); + assert!(!verifies(&merged, &p_plain, &on)); + let at = merged_positions(&merged, &on); + assert_eq!(at.len(), 2); + for (pos, table) in at.iter().zip(&merged.merge.tables) { + assert_eq!( + p_merged.proof.commit_ood[*pos].len(), + answers_of(&table.chips), + "{:?}", + table.chips + ); + } + assert_eq!( + ( + p_merged.proof.commit_ood[at[0]].len(), + p_merged.proof.commit_ood[at[1]].len() + ), + (8, 21) + ); +} + +/// ★ Negative, then the mutant, for BALU: an add nobody reads +/// (write multiplicity zero, so no memory token moves) given a wrong output +/// inside the merged table is refused — and with BALU's body vacuous in the +/// merged AIR, the same table proves and VERIFIES. BALU's body is what refuses +/// it. +#[test] +fn a_bad_balu_row_in_a_merged_table_is_refused_by_balus_own_body() { + let _g = lock(); + let (program, arenas) = four_chip_program(); + let opts = options(true); + let artifacts = build_artifacts(&program, &opts); + let g = &program.groups.balu; + let row = (0..g.real_rows) + .find(|&r| { + *g.at(r, super::layout::balu::MULT) == fe(0) + && *g.at(r, super::layout::balu::SEL_ADD) == fe(1) + }) + .expect("the unread add"); + let bad = || { + let exec = execute(&program, &arenas, &artifacts.hasher).expect("executes"); + let plan = artifacts.merge.plan(); + let mut traces = build_traces_with_merge(&program, &exec.records, artifacts.hasher, &plan); + let col = MergedLayout::of(&plan[0]).parts[0].map[super::chips::balu::cols::OUT]; + let v = *traces.merged[0].get_main(row, col); + traces.merged[0].set_main(row, col, v + fe(1)); + (traces, exec.public_words) + }; + let (mut traces, words) = bad(); + let refused = match prove_traces(&artifacts, &mut traces, &words, &opts) { + Ok(proof) => !verify_against_artifacts(&artifacts, &proof, &words, &opts), + Err(_) => true, + }; + assert!(refused, "the bad BALU row must not verify"); + + test_hooks::set_vacuous(Some(PartBody::Balu)); + let mutant = std::panic::catch_unwind(|| { + let (mut traces, words) = bad(); + let proof = + prove_traces(&artifacts, &mut traces, &words, &opts).expect("the mutant proves"); + verify_against_artifacts(&artifacts, &proof, &words, &opts) + }); + test_hooks::set_vacuous(None); + assert!( + mutant.expect("the mutant runs"), + "with BALU's body vacuous the bad row verifies: the body is load-bearing" + ); +} + +/// ★ The merged root is the program's identity: two programs +/// that differ only in one merged chip's instruction columns (an add's +/// operands swapped, same heights) get different ids, and so does the same +/// program with its merged root moved. +#[test] +fn the_merged_root_is_folded_into_the_program_id() { + let _g = lock(); + let opts = options(true); + let build = |swap: bool| { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(1); + let h0 = b.hint_felt(arena, 0); + let x = b.felt_const(fe(7)); + let y = b.felt_const(fe(5)); + let s = if swap { b.add(y, x) } else { b.add(x, y) }; + let d = b.sub(s, h0); + let bit = b.bit_const(true); + let (l, _) = b.select(bit, x.as_cell(), d.as_cell()); + let lanes = b.unpack(l); + let packed = b.pack_word(lanes); + b.public(packed); + build_artifacts(&compile(b.finish()), &opts) + }; + let (a, b) = (build(false), build(true)); + assert_eq!(a.log_heights, b.log_heights, "the same heights"); + assert_eq!(a.merge.plan(), b.merge.plan()); + assert_ne!( + a.merge.tables[0].root, b.merge.tables[0].root, + "BALU's columns differ" + ); + for c in MERGE_CANDIDATES { + assert_eq!(a.roots[c.slot()], b.roots[c.slot()], "both zero"); + } + assert_ne!(a.program_id, b.program_id, "the merged root is identity"); + let mut moved = a.merge.clone(); + moved.tables[0].root[0] ^= 1; + let id = |merge: &LfmMerge| { + super::statement::lfm_program_id_merged_under( + crate::hash_pin::BLOCK_COMMITMENT_HASH, + &a.roots, + &a.log_heights, + a.keccak_rnd_chunks, + a.hasher, + a.chip_set, + &a.blake3_chunk_roots, + &a.blake3_chunk_log_heights, + &a.hash_chunk_roots, + &a.hash_chunk_log_heights, + merge, + ) + }; + assert_ne!(id(&a.merge), id(&moved)); +} diff --git a/prover/src/lfm/merkle_cap.rs b/prover/src/lfm/merkle_cap.rs new file mode 100644 index 000000000..8803364a9 --- /dev/null +++ b/prover/src/lfm/merkle_cap.rs @@ -0,0 +1,396 @@ +//! ★ One tree's authenticated Merkle cap, in-guest — the one gadget the WHIR +//! chain verifier (W1) and the STARK sub-proof verifier (S1) share. +//! +//! A tree of depth `D` committed with a height-`c` cap is authenticated in two +//! places, and the in-guest verifier makes the dangerous state of each +//! unconstructible rather than checked: +//! +//! - **Once per tree**, [`CapCells::authenticate`] hashes the `2^c` hinted cap +//! digests up to their root and asserts it equals the tree's root lanes. It +//! is the ONLY constructor, so every [`CapCells`] value is a cap that hashes +//! to its root — and a tree has exactly one: the cells checked against the +//! root and the cells the mux reads are the same cells. +//! - **Per opening**, [`CapCells::verify_path`] is the ONE entry point. It takes +//! the opened leaf, the tree's WHOLE leaf index (low bit first, one bit per +//! level) and the path to the cap, walks the low `D − c` bits, picks +//! `cap[index >> (D − c)]` with the top `c` bits and asserts the two digests +//! equal. The split point is computed here from the index's own length and +//! the cap's height; the mux is private, so no caller can feed it a constant, +//! a hinted bit or a sub-slice of its own choosing. +//! +//! The mux is a balanced tree of `2^c − 1` `Select`s per digest cell: the LFM +//! has no load at a computed address, which is why the cap height is priced by +//! the cost law and stays at most 3. +//! +//! **Arity 4** (a [`edsl::WrapHash::Poseidon1`] builder, the host's 4-ary +//! trees). `D` stays the binary depth, the tree has `h = ⌈D/2⌉` levels and `c` +//! counts them; the cap is the level `c` below the root, real nodes only: +//! [`cap_nodes`] = `4^c`, or `2·4^(c−1)` at odd `D` (the top group is two real +//! nodes and two zero digests). Both are powers of two, so the same mux picks +//! the node with the top [`mux_bits`] = `log2(cap_nodes)` bits; the walk takes +//! the low `D − mux_bits = 2(h − c)` bits, whole 4-ary levels of three hints +//! each, and the cap's root is the 4-ary build with `Merkle4`'s zero padding +//! (`p1w16_emit::tree_root4`, the host's `cap_root`). +//! +//! ⚠ What a caller still owes: `index_bits` must be the tree's own leaf index +//! as the TRANSCRIPT produced it — the query's bits, or a suffix of them for a +//! tree whose leaves cover several positions (a FRI layer). Those bits reach +//! every caller as cells of the one `sample_u64_pow2` decomposition; nothing +//! here can tell a transcript bit from a hinted one. + +use super::builder::{Bit, Felt, LfmBuilder}; +use super::edsl::{self, WrapDigest}; +use super::instr::ArenaId; +use stark::config::Commitment; + +/// Nodes in the height-`c` cap of a tree of `arity` children per node over +/// `2^depth` leaves: `2^c` at arity 2, the real nodes of the level `c` below +/// the root at arity 4 (`crypto`'s `cap_len`, the host's own count); `1` (the +/// root) at `c = 0`. +pub fn cap_nodes(depth: usize, c: usize, arity: usize) -> usize { + crypto::merkle_tree::cap::cap_len(depth, c, arity) + .unwrap_or_else(|| panic!("a height-{c} cap over depth {depth} at arity {arity}")) +} + +/// Index bits the cap's mux reads: `log2(cap_nodes)` (`c` at arity 2, none +/// uncapped). +pub fn mux_bits(depth: usize, c: usize, arity: usize) -> usize { + cap_nodes(depth, c, arity).trailing_zeros() as usize +} + +/// Sibling digests one opening's path to the cap carries: one per walked +/// level at arity 2 (`D − c`), three per walked 4-ary level at arity 4 and one +/// at an uncapped odd-depth top (`p1w16_emit::path_hints` over the walked +/// bits). +pub fn path_hints(depth: usize, c: usize, arity: usize) -> usize { + let walked = depth - mux_bits(depth, c, arity); + if arity == 4 { + super::p1w16_emit::path_hints(walked) + } else { + walked + } +} + +/// Node hashes one opening's walk to the cap costs: one per walked level +/// (`⌈walked bits / 2⌉` at arity 4). +pub fn path_permutations(depth: usize, c: usize, arity: usize) -> usize { + let walked = depth - mux_bits(depth, c, arity); + if arity == 4 { + walked.div_ceil(2) + } else { + walked + } +} + +/// Node hashes a cap of `nodes` costs up to its root: `nodes − 1` at arity 2, +/// `⌈n/4⌉` per level at arity 4 (a short group padded). None for `nodes = 1`. +pub fn cap_root_permutations(nodes: usize, arity: usize) -> usize { + if arity != 4 { + return nodes - 1; + } + let (mut n, mut perms) = (nodes, 0); + while n > 1 { + n = n.div_ceil(4); + perms += n; + } + perms +} + +/// One tree's authenticated Merkle cap. +pub struct CapCells { + cap: Vec, + /// The cap height in the tree's own levels (4-ary under Poseidon1). + height: usize, + /// Index bits the mux reads, `log2(cap.len())`. + mux_bits: usize, +} + +impl CapCells { + /// Authenticate a hinted binary cap against a tree's root lanes, once per + /// tree. + /// + /// `cap` must be `2^c` digests, `c ≥ 1`: a tree at `c = 0` has no cap and + /// is checked against its root. `root_lanes` holds one entry per digest + /// cell (one for an algebraic root, two for a byte digest), as + /// [`edsl::assert_digest_eq_lanes`] takes them. An arity-4 cap's height + /// depends on its tree's depth: [`Self::authenticate_at`]. + pub fn authenticate(b: &mut LfmBuilder, cap: &[WrapDigest], root_lanes: &[[Felt; 4]]) -> Self { + assert_eq!( + b.wrap_hash().arity(), + 2, + "an arity-4 cap names its height (authenticate_at)" + ); + let height = cap.len().trailing_zeros() as usize; + Self::authenticate_at(b, cap, height, root_lanes) + } + + /// [`Self::authenticate`] for a cap of `height` levels of the builder's + /// arity: `2^height` digests at arity 2, `4^height` or `2·4^(height−1)` + /// at arity 4 ([`cap_nodes`]). The root is the arity's own build. + pub fn authenticate_at( + b: &mut LfmBuilder, + cap: &[WrapDigest], + height: usize, + root_lanes: &[[Felt; 4]], + ) -> Self { + assert!( + height >= 1 && cap.len() >= 2 && cap.len().is_power_of_two(), + "a cap is 2^k digests at height c >= 1, got {} at height {height}", + cap.len() + ); + let mux_bits = cap.len().trailing_zeros() as usize; + let fits = match b.wrap_hash().arity() { + 4 => mux_bits == 2 * height || mux_bits + 1 == 2 * height, + _ => mux_bits == height, + }; + assert!( + fits, + "{} cap nodes are not a height-{height} cap at arity {}", + cap.len(), + b.wrap_hash().arity() + ); + let root = edsl::wrap_merkle_tree_root(b, cap); + edsl::assert_digest_eq_lanes(b, root, root_lanes); + Self { + cap: cap.to_vec(), + height, + mux_bits, + } + } + + /// The cap height `c`, in the tree's own levels. + pub fn height(&self) -> usize { + self.height + } + + /// Index bits the cap node's mux reads: an opening walks the others. + pub fn mux_bits(&self) -> usize { + self.mux_bits + } + + /// ★ Authenticate EVERY leaf of a tree that lies wholly in this cap (no + /// level walked: the cap is the leaf level), as a REFUSAL: leaf `j`'s + /// digest must equal cap node `j`, which is what [`Self::verify_path`] + /// muxes for index `j` with no walked bits. The positions are program + /// constants, so no node is muxed. + pub fn verify_every_leaf(&self, b: &mut LfmBuilder, leaves: &[WrapDigest]) { + assert_eq!( + leaves.len(), + self.cap.len(), + "one digest a leaf of a tree lying wholly in its cap" + ); + for (leaf, node) in leaves.iter().zip(&self.cap) { + for (x, y) in leaf.iter().zip(node.iter()) { + edsl::assert_word_eq(b, *x, *y); + } + } + } + + /// ★ Authenticate one opening against this cap, as a REFUSAL: `leaf` is the + /// opened leaf's digest, `index_bits` the tree's WHOLE leaf index (low + /// first, `D` bits) and `siblings` the path to the cap ([`path_hints`] + /// digests, leaf level first). The low `D − mux_bits` bits are walked, the + /// top `mux_bits` pick the cap node, and the walked digest must equal it. + pub fn verify_path( + &self, + b: &mut LfmBuilder, + leaf: WrapDigest, + index_bits: &[Bit], + siblings: &[WrapDigest], + ) { + self.verify_path_by(b, leaf, index_bits, siblings, false); + } + + /// [`Self::verify_path`] with an arity-4 path's siblings in CHILD order, as + /// the proof carries them ([`super::p1w16_emit::walk4_child`]); at arity 2 + /// it is [`Self::verify_path`]. + pub fn verify_path_child( + &self, + b: &mut LfmBuilder, + leaf: WrapDigest, + index_bits: &[Bit], + siblings: &[WrapDigest], + ) { + self.verify_path_by(b, leaf, index_bits, siblings, true); + } + + fn verify_path_by( + &self, + b: &mut LfmBuilder, + leaf: WrapDigest, + index_bits: &[Bit], + siblings: &[WrapDigest], + child_order: bool, + ) { + assert!( + index_bits.len() >= self.mux_bits, + "the index covers the cap: {} bits under a {}-bit mux", + index_bits.len(), + self.mux_bits + ); + let walked = index_bits.len() - self.mux_bits; + let hash = b.wrap_hash(); + assert!( + hash.arity() != 4 || walked.is_multiple_of(2), + "an arity-4 cap sits on a 4-ary level: {walked} walked bits" + ); + assert_eq!( + siblings.len(), + hash.path_hints(walked), + "a path to the cap: its hints for every level below it" + ); + let (walk_bits, top_bits) = index_bits.split_at(walked); + let walked = if child_order && hash.arity() == 4 { + super::p1w16_emit::walk4_child(b, leaf, walk_bits, siblings) + } else { + edsl::wrap_merkle_walk(b, leaf, walk_bits, siblings) + }; + let node = self.select(b, top_bits); + for (x, y) in walked.iter().zip(node.iter()) { + edsl::assert_word_eq(b, *x, *y); + } + } + + /// `cap[index >> (depth − mux_bits)]` from the index's top bits, LOW + /// first: a balanced mux, `cap.len() − 1` `Select` rows a digest cell. + /// Pairs are `(2t, 2t + 1)` because the bits arrive low first (the slot + /// mux's reason, `whir_chain::emit_slot_mux`). + fn select(&self, b: &mut LfmBuilder, top_bits: &[Bit]) -> WrapDigest { + assert_eq!(top_bits.len(), self.mux_bits, "one mux level per index bit"); + let mut level: Vec = self.cap.clone(); + for bit in top_bits { + level = level + .chunks_exact(2) + .map(|pair| { + let cells: Vec<_> = pair[0] + .iter() + .zip(pair[1].iter()) + .map(|(l, r)| b.select(*bit, *l, *r).0) + .collect(); + WrapDigest::from_cells(&cells) + }) + .collect(); + } + level[0] + } +} + +/// Hint the height-`c` cap of a tree over `2^depth` leaves — [`cap_nodes`] +/// digests at the builder's arity, [`edsl::digest_words`] words each — out of +/// `arena` from word `base`, and authenticate it against `root_lanes`. +/// Returns the cells and the next free word. +pub fn hint_and_authenticate( + b: &mut LfmBuilder, + arena: ArenaId, + base: u32, + c: usize, + depth: usize, + root_lanes: &[[Felt; 4]], +) -> (CapCells, u32) { + let dw = edsl::digest_words(b); + let nodes = cap_nodes(depth, c, b.wrap_hash().arity()); + let mut cursor = base; + let cap: Vec = (0..nodes) + .map(|_| { + let d = edsl::hint_digest(b, arena, cursor); + cursor += dw; + d + }) + .collect(); + (CapCells::authenticate_at(b, &cap, c, root_lanes), cursor) +} + +/// One opening's path, cut to the cap, in the arena's order, and the cap +/// itself when this is the tree's owner path (its first opening in proof +/// order, of a capped tree). +/// +/// At arity 2 the path to the cap is the first `D − c` siblings as the proof +/// carries them. At arity 4 the proof carries three siblings per 4-ary level in +/// child order (`3(⌈D/2⌉ − c)` of them, the owner path then the cap's nodes); +/// they are laid out in the walk's hint order for the opening at `index` +/// ([`super::p1w16_emit::hint_order`], which drops an uncapped odd-depth top's +/// two padding siblings), so an arity-4 path needs its leaf index. +pub fn path_to_cap( + path: &[Commitment], + depth: usize, + c: usize, + arity: usize, + owner: bool, + index: Option, +) -> Result<(Vec, Option>), String> { + let check = |got: usize, want: usize, what: &str| { + if got == want { + Ok(()) + } else { + Err(format!("{what}: {got} nodes, the shape takes {want}")) + } + }; + let (siblings, cap) = if c == 0 || !owner { + let kept = if arity == 4 { + 3 * (crypto::merkle_tree::cap::tree_levels(depth, 4) - c) + } else { + depth - c + }; + check(path.len(), kept, "a path to the cap")?; + (path, None) + } else { + let (siblings, cap) = + crypto::merkle_tree::cap::split_owner_path_arity(path, depth, c, arity) + .ok_or("the owner path is the path to the cap, then the cap")?; + (siblings, Some(cap.to_vec())) + }; + if arity != 4 { + return Ok((siblings.to_vec(), cap)); + } + let index = index.ok_or("an arity-4 path is laid out at its leaf index")?; + let walked = depth - mux_bits(depth, c, 4); + check( + siblings.len(), + 3 * walked.div_ceil(2), + "three siblings per walked 4-ary level", + )?; + let triples: Vec<[Commitment; 3]> = siblings + .chunks_exact(3) + .map(|t| [t[0], t[1], t[2]]) + .collect(); + Ok((super::p1w16_emit::hint_order(index, walked, &triples), cap)) +} + +/// [`path_to_cap`] with an arity-4 path left in CHILD order (the proof's own, +/// [`super::p1w16_emit::walk4_child`]'s hints), so it needs no leaf index. An +/// uncapped odd-depth tree's top triple keeps only its first sibling, the +/// partner: the other two are the padding the walk supplies as constants. +pub fn child_path_to_cap( + path: &[Commitment], + depth: usize, + c: usize, + arity: usize, + owner: bool, +) -> Result<(Vec, Option>), String> { + if arity != 4 { + return path_to_cap(path, depth, c, arity, owner, None); + } + let levels = crypto::merkle_tree::cap::tree_levels(depth, 4); + let (siblings, cap) = if c == 0 || !owner { + if path.len() != 3 * (levels - c) { + return Err(format!( + "a path to the cap: {} nodes, the shape takes {}", + path.len(), + 3 * (levels - c) + )); + } + (path, None) + } else { + let (siblings, cap) = crypto::merkle_tree::cap::split_owner_path_arity(path, depth, c, 4) + .ok_or("the owner path is the path to the cap, then the cap")?; + (siblings, Some(cap.to_vec())) + }; + let walked = depth - mux_bits(depth, c, 4); + let mut out = siblings.to_vec(); + if walked % 2 == 1 { + // The uncapped odd top: drop its two padding siblings. + out.truncate(out.len() - 2); + } + debug_assert_eq!(out.len(), super::p1w16_emit::path_hints(walked)); + Ok((out, cap)) +} diff --git a/prover/src/lfm/mod.rs b/prover/src/lfm/mod.rs new file mode 100644 index 000000000..1dd538e8f --- /dev/null +++ b/prover/src/lfm/mod.rs @@ -0,0 +1,255 @@ +//! LFM — the Lambda Field Machine. +//! +//! A fixed, straight-line, field-native recursion machine for verifying +//! Lambda VM STARK proofs: the SP1 v4 mechanism (the program is the machine's +//! preprocessed columns; write-once memory closed by pure LogUp balance; no +//! pc, no branches, no fetch/decode) with on-demand registration instead of +//! exhaustive shape enumeration — our framework has no keygen, so a program +//! is nothing but a vector of supplied preprocessed roots plus a registry +//! entry. +//! +//! Design authority: `others/lfm-design.md` (v0). This module is the +//! software layer (Milestone A): word model, instruction set, eDSL builder, +//! straight-line compiler, executor/witness generator, admission validator, +//! and the hash interface with a placeholder permutation. The chips and prover +//! integration follow (Milestone B); the fixed AIR set is 14 chips, the last +//! three being the production keccak family hosted unchanged (see `airs`). + +pub mod airs; +pub mod algebraic_commit; +pub mod algebraic_transcript; +pub mod blake3; +pub mod blake3_chip; +pub mod blake3_socket; +pub mod block_node; +pub mod builder; +pub mod chips; +pub mod chunking; +pub mod commit; +pub mod compiler; +pub mod constraints; +pub mod deep; +pub mod device_permit; +pub mod domain_warm; +pub mod edsl; +pub mod epoch; +pub mod epoch_verify; +pub mod exec_schedule; +pub mod executor; +pub mod first_prove_warm; +pub mod fixture; +pub mod fri; +pub(crate) mod harvest; +pub mod hash; +pub mod instr; +pub mod keccak_adapter; +pub mod keccak_host; +pub mod layout; +pub mod lde; +pub mod logup; +pub mod merge; +pub mod merkle_cap; +pub mod p1_commit; +pub mod p1w16_chip; +pub mod p1w16_emit; +pub mod p1w16_socket; +pub mod per_table_aggregator; +pub mod preprocessed; +pub mod program_budget; +pub mod program_census; +pub mod programs; +pub mod proof; +pub mod proof_arena; +pub mod proof_fixture; +pub mod registry; +pub mod rpo; +pub mod rpx; +pub mod statement; +pub mod statement_replay; +pub mod sub_proof; +pub mod trace; +pub mod transcript_replay; +/// ⛔ ROUND-3 TREE/WRAP DISCRIMINATOR — a diagnostic, OFF by default. +pub mod tree_probe; +pub(crate) mod tree_run; +pub mod validator; +pub mod whir_air; +/// The batched argue as a machine leg (D-BATCH B-4a). +pub mod whir_batch; +#[cfg(feature = "parallel")] +pub mod whir_block; +#[cfg(feature = "parallel")] +pub mod whir_block_tree; +pub mod whir_bus; +pub mod whir_chain; +pub mod whir_epoch; +pub mod whir_fold; +pub mod whir_gkr; +pub mod whir_open; +pub mod whir_poly; +pub mod whir_program; +pub mod whir_reduce; +pub mod whir_stacked; +pub mod whir_table; +pub mod whir_transcript; +pub mod whir_transcript_kats; +pub mod word; + +pub use airs::{LfmAirs, NUM_LFM_CHIPS, num_lfm_airs}; +pub use builder::{ArenaSchema, LfmBuilder, LfmProgramSource}; +pub use chunking::{KECCAK_RND_MAX_CHUNK_ROWS, KeccakChunking}; +pub use commit::{commit_columns, commit_group, commit_lde_columns, lde_columns}; +pub use compiler::{ColumnGroup, LfmColumnGroups, LfmProgram, compile}; +pub use executor::{LfmExecError, LfmExecution, LfmRecords, execute}; +pub use hash::{HasherKind, LfmHasher, TestPermutation}; +pub use instr::{Addr, ArenaId, BaseOp, ExtOp, HashMode, Instr}; +pub use program_census::build_artifacts_counted; +pub use proof::{LfmProof, LfmProveError, lfm_prove, lfm_verify, verify_against_artifacts}; +pub use registry::{ + LFM_REGISTRY, LfmArtifacts, LfmProgramKind, LfmRegistryEntry, LfmRegistryError, + build_artifacts, build_artifacts_with_hasher, resolve, +}; +pub use statement::{LFM_MACHINE_VERSION, lfm_program_id}; +pub use transcript_replay::{Candidate, TranscriptReplay}; +pub use validator::{LfmViolation, validate}; +pub use word::{LfmWord, base_word, ext_word, pack_digest, unpack_digest}; + +// The algebraic `StarkHash` configurations this differential drives the host +// with are `#[cfg(not(feature = "cuda"))]` — inexpressible under cuda, by +// design — so the gate follows them rather than failing to compile there. +#[cfg(test)] +mod blake3_chip_tests; +#[cfg(test)] +mod blake3_probe; +#[cfg(test)] +mod blake3_socket_kats; +#[cfg(test)] +mod blake3_socket_tests; +#[cfg(test)] +mod commit_ood_leg_tests; +#[cfg(test)] +mod constraint_tests; +#[cfg(test)] +mod epoch_tests; +#[cfg(test)] +mod epoch_verify_tests; +#[cfg(test)] +mod exec_identity_tests; +#[cfg(test)] +mod framework_probe; +#[cfg(test)] +mod fri_group_tests; +#[cfg(test)] +mod fri_tests; +#[cfg(test)] +mod hint_on_tests; +#[cfg(test)] +mod join_tests; +#[cfg(test)] +mod keccak_probe; +#[cfg(test)] +mod leaf_kats; +#[cfg(test)] +mod leaf_tests; +#[cfg(all(test, feature = "cuda"))] +mod lean_upload_tests; +#[cfg(test)] +mod logup_tests; +#[cfg(test)] +mod machine_tests; +#[cfg(test)] +mod merge_tests; +#[cfg(test)] +mod node_emit_timing_tests; +#[cfg(test)] +mod one_row_guest_tests; +#[cfg(test)] +mod one_row_tests; +#[cfg(test)] +mod p1_family_tests; +#[cfg(test)] +mod p1_node_tests; +#[cfg(test)] +mod p1w16_chip_tests; +#[cfg(test)] +mod p1w16_emit_tests; +#[cfg(test)] +mod p1w16_machine_tests; +#[cfg(test)] +mod p1w16_proof_tests; +#[cfg(test)] +mod p1w16_socket_tests; +#[cfg(test)] +mod per_table_aggregator_tests; +#[cfg(test)] +mod preprocessed_tests; +#[cfg(test)] +mod program_shape_tests; +#[cfg(test)] +mod rp1a_review_tests; +#[cfg(test)] +mod rp1wa_review_tests; +#[cfg(test)] +mod rp1wb_review_tests; +#[cfg(test)] +mod tree_plan_property_tests; +#[cfg(test)] +mod whir_air_tests; +#[cfg(test)] +mod whir_batch_tests; +#[cfg(all(test, feature = "parallel"))] +mod whir_block_tests; +#[cfg(test)] +mod whir_bus_tests; +#[cfg(test)] +mod whir_chain_tests; +#[cfg(test)] +mod whir_commit_ood_leg_tests; +#[cfg(test)] +mod whir_fold_tests; +#[cfg(test)] +mod whir_gkr_tests; +#[cfg(test)] +mod whir_lean_tests; +#[cfg(test)] +mod whir_onechain_tests; +#[cfg(test)] +mod whir_open_tests; +#[cfg(test)] +pub(crate) mod whir_p1_census_tests; +#[cfg(test)] +mod whir_p1_leaf_tests; +#[cfg(test)] +mod whir_poly_tests; +#[cfg(test)] +mod whir_program_tests; +#[cfg(test)] +mod whir_reduce_tests; +#[cfg(test)] +mod whir_stacked_tests; +#[cfg(test)] +mod whir_table_tests; +#[cfg(test)] +mod whir_transcript_kat_tests; +#[cfg(test)] +mod whir_transcript_replay_tests; +// ★ Test-only: the dependency-structure measurement that prices a parallel +// executor before one is written (lane E). No production path reaches it. +#[cfg(test)] +mod reach_profile; +#[cfg(test)] +mod rpx_chip_tests; +#[cfg(test)] +mod step_size_tests; +#[cfg(test)] +mod tests; +#[cfg(test)] +mod trace_identity_tests; +#[cfg(all(test, not(feature = "cuda")))] +mod transcript_diff_tests; +#[cfg(test)] +mod transcript_kats; +#[cfg(test)] +mod transcript_tests; +#[cfg(test)] +mod wrap_tests; diff --git a/prover/src/lfm/node_emit_timing_tests.rs b/prover/src/lfm/node_emit_timing_tests.rs new file mode 100644 index 000000000..7ba0207f1 --- /dev/null +++ b/prover/src/lfm/node_emit_timing_tests.rs @@ -0,0 +1,104 @@ +//! Where a block node's program emission spends its time: the builder's walk, +//! the source's finish, the compile and the admission validator, for a level-1 +//! node over three median-shaped leaves and the top over two such nodes. The +//! children are heights only (`registry::census_artifacts` with the leaves' +//! padded heights): a node's instructions depend on its children's shapes, never +//! on a root, so no commit and no proof are needed. +//! +//! Run alone: `cargo test --release -p lambda-vm-prover --lib -- --ignored +//! --exact lfm::node_emit_timing_tests::node_emission_split --nocapture`. +//! Prints one `NODE EMIT …` line per node. + +use std::time::Instant; + +use crate::hash_pin::{BlockFamily, ProofFamily}; + +use super::block_node::{BlockLayout, BlockNodeInputs, emit_block_node}; +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::per_table_aggregator::DerivedChild; +use super::registry::{LfmArtifacts, census_artifacts}; + +/// A median leaf's padded heights by chip slot (a median block's `W3 LEAF CENSUS`): +/// `LFM_CONST` 2^11, base ALU 2^20, extension ALU 2^21, `SELECT` 2^20, +/// bit-dec 2^15, the socket 2^18, lanes and hints 2^20, publics 2^6, +/// `LFM_RANGE` 2^16. The keccak and BLAKE3 groups keep the trivial program's. +const LEAF_LOG_HEIGHTS: [(usize, u8); 10] = [ + (0, 11), + (1, 20), + (2, 21), + (3, 20), + (4, 15), + (5, 18), + (7, 20), + (8, 20), + (9, 6), + (10, 16), +]; + +fn leaf_artifacts() -> LfmArtifacts { + let program = super::programs::trivial_program(); + let mut a = census_artifacts(&program, ::SOCKET); + for (slot, h) in LEAF_LOG_HEIGHTS { + a.log_heights[slot] = h; + } + a.hash_chunk_log_heights = vec![a.log_heights[super::airs::HASH_SLOT]]; + a.hash_chunk_roots = vec![[0u8; 32]]; + a +} + +/// Emit one node over `kids` and print where the time went; returns the +/// compiled program. +fn timed_node(label: &str, kids: &[&DerivedChild], layout: BlockLayout, top: bool) -> LfmProgram { + let shapes: Vec<_> = kids.iter().map(|c| c.shape()).collect(); + let t = Instant::now(); + let mut b = LfmBuilder::new().with_wrap_hash(::WRAP); + emit_block_node( + &mut b, + &BlockNodeInputs { + children: &shapes, + layout, + top, + }, + ); + let walk = t.elapsed().as_secs_f64(); + let t = Instant::now(); + let source = b.finish(); + let finish = t.elapsed().as_secs_f64(); + let t = Instant::now(); + let program = compile(source); + let compiled = t.elapsed().as_secs_f64(); + let t = Instant::now(); + super::validator::validate(&program).expect("the node is admissible"); + let validated = t.elapsed().as_secs_f64(); + let split = super::validator::validate_split(&program) + .iter() + .map(|(n, s)| format!("{n} {s:.3}")) + .collect::>() + .join(" · "); + println!("NODE VALIDATE {label}: {split}"); + let rows = super::whir_chain::ChipRows::of(&program); + println!( + "NODE EMIT {label}: walk {walk:.3}s · finish {finish:.3}s · compile {compiled:.3}s · \ + validate {validated:.3}s · total {:.3}s · {} instructions · rows {rows:?}", + walk + finish + compiled + validated, + program.instrs.len(), + ); + program +} + +#[test] +#[ignore = "instrument: run alone with --exact"] +fn node_emission_split() { + let wrap = super::proof::aggregation_wrap_options(); + let layout = BlockLayout::child(1, 8); + let words = layout.total(); + let leaf = DerivedChild::from_artifacts_under::(&leaf_artifacts(), &wrap, words) + .expect("the leaf derives"); + let level1 = timed_node("L1 (3 leaves)", &[&leaf, &leaf, &leaf], layout, false); + let node = census_artifacts(&level1, ::SOCKET); + drop(level1); + let node = DerivedChild::from_artifacts_under::(&node, &wrap, words) + .expect("the node derives"); + timed_node("top (2 nodes)", &[&node, &node], layout, true); +} diff --git a/prover/src/lfm/one_row_guest_tests.rs b/prover/src/lfm/one_row_guest_tests.rs new file mode 100644 index 000000000..4826044de --- /dev/null +++ b/prover/src/lfm/one_row_guest_tests.rs @@ -0,0 +1,711 @@ +//! S2 in the in-guest (LFM) STARK verifier (G3): one-row trace leaves, DEEP +//! at ONE point, the committed FRI input and +//! its input-slot check, index bits over the whole LDE, no `−υ` point. +//! +//! Checked against the host's own artefacts, never against a second model: +//! - the in-guest one-row shape (index bits, schedule, layer depths, caps, +//! challenge count) against the host's `StarkCaps::for_options(.., true)`; +//! - the emitted FRI verifier against the host's checked-in RPX (e) proofs +//! (`crypto/stark/tests/vectors/zf_fri/e_proof_rpx_*`), executed, with its +//! permutation count equal to the closed form; +//! - the in-guest one-row (and row-pair) trace leaf against the (e) leaf +//! digests; +//! - tampers of every value the input tree's opening carries, and the +//! input-slot check shown load-bearing (a moved `DEEP(x_r)` executes when, +//! and only when, the slot check is skipped); +//! - both legs as one program on real laptop-scale proofs at `one_row` ∈ +//! {1, auto} × cap {off, auto} × fri {pair, dp}, and one program verifying a +//! one-row table and a row-pair table side by side (mixed layouts). + +use crypto::merkle_tree::cap::CapPolicy; +use math::fft::bit_reversing::reverse_index; +use serde_json::Value; +use stark::config::Commitment; +use stark::examples::read_only_memory_logup::LogReadOnlyPublicInputs; +use stark::leaf_layout::LeafLayout; +use stark::merkle_caps::StarkCaps; +use stark::proof::options::{FriMode, FriScheduleOverride, OneRowMode, ProofFormat, ProofOptions}; +use stark::proof::stark::StarkProof; +use stark::proof::view::StarkProofView; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::epoch_verify::{blocks_for, group_leaf_felts_at}; +use super::executor::execute; +use super::fri::FriShape; +use super::fri_tests::{ + HostFri, folding_fixture_with, fri_only_program, host_fri_from, permutations, +}; +use super::sub_proof::{GroupShape, emit_leaf_hash_rows}; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type VectorProof = StarkProof>; + +// ============================================================================= +// The shape — the in-guest one-row layout IS the host's +// ============================================================================= + +/// ★ One index width, one schedule, one depth and one cap per layer on both +/// sides under one-row leaves: the in-guest `FriShape` (resolved to one row) +/// against the host's `StarkCaps` at `one_row = true` — built on the host's +/// `FriFoldLayout` — over every LDE size of interest, both production +/// terminals, both FRI modes and both cap policies. Also: the index is +/// `log2(lde)` bits wide (not `log2(lde) − 1`), the chain covers EVERY fold +/// (layer 0 is the DEEP codeword), and a proof draws one challenge per +/// committed layer (none before the input tree). +#[test] +fn the_in_guest_one_row_shape_is_the_hosts_layout() { + let mut checked = 0usize; + for (blowup, k) in [(4u8, 7u8), (4, 8), (2, 7)] { + for queries in [3usize, 24, 110] { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + for fri in [FriMode::Pair, FriMode::Dp] { + let blowup_log = (blowup as u32).trailing_zeros(); + for lde_log in (blowup_log + 1)..=25 { + let opts = ProofOptions { + blowup_factor: blowup, + fri_number_of_queries: queries, + coset_offset: 3, + grinding_factor: 0, + fri_final_poly_log_degree: k, + format: ProofFormat { + merkle_cap: cap, + fri_mode: fri, + one_row: OneRowMode::On, + ..ProofFormat::DEFAULT + }, + }; + let shape = FriShape::from_options(&opts, lde_log); + shape.check(); + assert!(shape.one_row()); + assert!(!shape.is_legacy(), "a one-row table is never legacy"); + let host = StarkCaps::for_options(&opts, lde_log as usize, true) + .expect("a one-row format lays out"); + assert_eq!(shape.index_bits(), lde_log as usize); + assert_eq!(shape.index_bits(), host.trace_depth); + let depths: Vec = (0..shape.num_committed()) + .map(|j| shape.layer_depth(j)) + .collect(); + let caps: Vec = (0..shape.num_committed()) + .map(|j| shape.layer_cap(j)) + .collect(); + assert_eq!(depths, host.fri_depths, "{opts:?} lde {lde_log}"); + assert_eq!(caps, host.fri, "{opts:?} lde {lde_log}"); + let covered: u32 = shape.schedule().iter().map(|&d| u32::from(d)).sum(); + assert_eq!(covered, shape.total_folds(), "every fold is committed"); + assert_eq!( + shape.num_zetas(), + if shape.total_folds() > 0 { + shape.num_committed() + } else { + 0 + } + ); + checked += 1; + } + } + } + } + } + println!( + "{checked} one-row shapes: in-guest index bits, schedule, depths and caps == the host's" + ); +} + +/// `one_row = auto` has no layout of its own: a shape is built at the table's +/// RESOLVED layout (the AIR's widths decide), and asking the options alone is +/// refused rather than guessed. +#[test] +#[should_panic(expected = "one_row = auto resolves per table")] +fn an_unresolved_auto_layout_is_refused() { + let mut opts = + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(4).expect("blowup 4"); + opts.format.one_row = OneRowMode::Auto; + let _ = FriShape::from_options(&opts, 12); +} + +// ============================================================================= +// (e) — the emitted FRI verifier on the host's one-row RPX proofs +// ============================================================================= + +fn ext_of(v: &Value) -> FEE { + let limbs: Vec = v + .as_array() + .expect("an ext value is three limbs") + .iter() + .map(|x| x.as_u64().expect("a canonical limb")) + .collect(); + assert_eq!(limbs.len(), 3); + FEE::new([FE::from(limbs[0]), FE::from(limbs[1]), FE::from(limbs[2])]) +} + +fn commitment_of_hex(s: &str) -> Commitment { + assert_eq!(s.len(), 64, "a 32-byte digest"); + let mut out = [0u8; 32]; + for (i, byte) in out.iter_mut().enumerate() { + *byte = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).expect("hex"); + } + out +} + +/// One checked-in RPX (e) proof: its JSON, its proof, and the in-guest shape +/// built from the vector's FORMAT (the host generator's own +/// `one_row_proof_formats`, never re-spelled here). +struct OneRowVector { + name: &'static str, + json: Value, + proof: VectorProof, + shape: FriShape, +} + +fn one_row_rpx_vectors() -> Vec { + stark::fri::vectors::one_row_proof_formats() + .into_iter() + .map(|(name, format)| { + let dir = stark::fri::vectors::vectors_dir(); + let stem = format!("e_proof_rpx_{name}"); + let json: Value = serde_json::from_slice( + &std::fs::read(dir.join(format!("{stem}.json"))).expect("the vector JSON"), + ) + .expect("valid JSON"); + let bytes = std::fs::read(dir.join(format!("{stem}.rkyv"))).expect("the vector proof"); + let proof: VectorProof = + rkyv::from_bytes::(&bytes).expect("rkyv"); + let queries = json["queries"].as_u64().expect("queries") as usize; + let opts = stark::fri::vectors::proof_options(format, queries); + let lde_log = json["lde_log"].as_u64().expect("lde_log") as u32; + let shape = FriShape::from_options(&opts, lde_log); + OneRowVector { + name, + json, + proof, + shape, + } + }) + .collect() +} + +impl OneRowVector { + /// The arenas [`fri_only_program`] declares for a one-row shape: + /// `(r, DEEP(x_r))` per query, then the roots (the input tree's first), + /// the ζs (one per layer), the terminal coefficients and the per-query + /// layer openings (every layer a full group — layer 0 the input group). + fn arenas(&self) -> Vec> { + let queries = self.json["queries_detail"].as_array().expect("queries"); + let mut deep = Vec::new(); + for q in queries { + deep.push(base_word(FE::from(q["iota"].as_u64().expect("r")))); + deep.push(ext_word(&ext_of(&q["deep"]))); + } + let view = StarkProofView::Owned(&self.proof); + let (openings, caps) = super::epoch_verify_tests::fri_layer_openings(view, self.shape); + let mut per_query = Vec::new(); + for query in &openings { + for (values, path) in query { + per_query.extend(values.iter().map(ext_word)); + per_query.extend(super::proof_arena::commitments_to_arena(path)); + } + } + let zetas: Vec = self.json["zetas"] + .as_array() + .expect("zetas") + .iter() + .map(|z| ext_word(&ext_of(z))) + .collect(); + let mut out = vec![ + deep, + super::proof_arena::commitments_to_arena(&self.proof.fri_layers_merkle_roots), + zetas, + self.proof + .fri_final_poly_coeffs + .iter() + .map(ext_word) + .collect(), + per_query, + ]; + if self.shape.cap_words(super::proof_arena::words_per_root()) > 0 { + out.push(super::proof_arena::commitments_to_arena(&caps)); + } + out + } + + fn program(&self) -> LfmProgram { + fri_only_program(self.shape, self.shape.num_queries) + } +} + +/// ★ The emitted FRI verifier accepts every one-row RPX (e) proof — the pair +/// schedule from the input tree and the uneven `[3, 2, 1, 2]` override — with +/// the shape's index width, schedule, challenge count and layer depths equal +/// to the vector's, and the permutation count exactly the closed form. +#[test] +fn the_emitted_fri_verifier_accepts_every_one_row_rpx_vector() { + let vectors = one_row_rpx_vectors(); + assert_eq!(vectors.len(), 2, "one_row_pair and one_row_3_2_1_2"); + for v in vectors { + let s = v.shape; + s.check(); + assert!(v.json["one_row"].as_bool().expect("one_row")); + assert!(s.one_row()); + let schedule: Vec = v.json["schedule"] + .as_array() + .expect("schedule") + .iter() + .map(|d| d.as_u64().expect("d") as u8) + .collect(); + assert_eq!(s.schedule(), schedule, "{}: the schedule", v.name); + assert_eq!( + s.is_legacy(), + v.json["legacy_encoding"].as_bool().expect("legacy"), + "{}", + v.name + ); + assert_eq!( + s.index_bits() as u64, + v.json["trace_tree_depth"].as_u64().expect("depth"), + "{}: r has log2(lde) bits", + v.name + ); + assert_eq!( + 1u64 << s.index_bits(), + v.json["query_bound"].as_u64().expect("bound"), + "{}: r ranges over the whole LDE", + v.name + ); + assert_eq!( + s.num_zetas(), + v.json["zetas"].as_array().expect("zetas").len(), + "{}: one challenge per committed layer, none before the input tree", + v.name + ); + assert_eq!(s.num_committed(), v.proof.fri_layers_merkle_roots.len()); + for (qi, q) in v.json["queries_detail"] + .as_array() + .expect("queries") + .iter() + .enumerate() + { + for (j, layer) in q["layers"].as_array().expect("layers").iter().enumerate() { + assert_eq!( + s.layer_path_len(j) as u64, + layer["path_len"].as_u64().expect("path_len"), + "{} query {qi} layer {j}", + v.name + ); + } + } + + let program = v.program(); + let exec = execute(&program, &v.arenas(), &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("{}: the honest vector must execute: {e:?}", v.name)); + assert_eq!(exec.public_words.len(), s.num_queries); + let closed = s.num_queries * s.permutations_per_query() + s.cap_permutations(); + assert_eq!( + permutations(&program), + closed, + "{}: emitted permutations against the closed form", + v.name + ); + println!( + "{:<16} Q={} index bits {} schedule {:?} zetas {}: {} permutations, {} instructions", + v.name, + s.num_queries, + s.index_bits(), + s.schedule(), + s.num_zetas(), + closed, + program.instrs.len() + ); + } +} + +/// ★ Every value the INPUT tree's opening carries is bound, and so is the DEEP +/// value it is checked against: `DEEP(x_r)`, the input root, `ζ₀` (layer 0's +/// challenge under one row), a terminal coefficient, the input group's slot +/// value, a non-slot value, its last value and its first sibling. Run on both +/// (e) formats. +#[test] +fn no_tampered_input_tree_value_can_pass() { + for v in one_row_rpx_vectors() { + let program = v.program(); + let honest = v.arenas(); + execute(&program, &honest, &crate::hash_pin::BLOCK_HASHER).expect("honest"); + let q0 = &v.json["queries_detail"][0]["layers"][0]; + let slot = q0["slot"].as_u64().expect("slot") as usize; + assert_eq!( + ext_of(&q0["values"][slot]), + ext_of(&v.json["queries_detail"][0]["deep"]), + "{}: the input group's slot holds DEEP(x_r) (the host's input-slot check)", + v.name + ); + let d0 = 1usize << v.shape.layer_fold(0); + let other = (slot + 1) % d0; + // Arenas: deep (r, DEEP) per query, roots, zetas, coeffs, queries. + let bump: Vec<(String, usize, usize)> = vec![ + ("DEEP(x_r)".into(), 0, 1), + ("the input root".into(), 1, 0), + ("zeta_0 (layer 0's challenge)".into(), 2, 0), + ("terminal coefficient 0".into(), 3, 0), + (format!("input group slot value (slot {slot})"), 4, slot), + (format!("input group non-slot value {other}"), 4, other), + ("input group last value".into(), 4, d0 - 1), + ("input group first sibling".into(), 4, d0), + ]; + for (label, arena, word) in bump { + let mut bad = honest.clone(); + bad[arena][word][0] += FE::one(); + execute(&program, &bad, &crate::hash_pin::BLOCK_HASHER).expect_err(&format!( + "{}: moving {label} must make the program unexecutable", + v.name + )); + } + } +} + +/// ★ The INPUT-SLOT check is LOAD-BEARING (the in-guest M1 at the input +/// tree). Under one-row leaves `DEEP(x_r)` meets the committed FRI +/// chain ONLY at `group₀[slot] == DEEP(x_r)`: the input leaf hashes the group, +/// the walk authenticates it, the group fold reads it — none reads `DEEP(x_r)`. +/// So a moved `DEEP(x_r)` is refused with the check and ACCEPTED without it, +/// which is exactly a verifier that would run FRI on a codeword the trace +/// openings do not commit to. +#[test] +fn the_input_slot_check_is_load_bearing() { + for v in one_row_rpx_vectors() { + let honest = v.arenas(); + let mut moved = honest.clone(); + moved[0][1][0] += FE::one(); + + let with = v.program(); + execute(&with, &honest, &crate::hash_pin::BLOCK_HASHER).expect("honest"); + execute(&with, &moved, &crate::hash_pin::BLOCK_HASHER) + .expect_err("a moved DEEP(x_r) must be refused by the input-slot check"); + + super::fri::SKIP_SLOT_CHECK.with(|c| c.set(true)); + let without = v.program(); + super::fri::SKIP_SLOT_CHECK.with(|c| c.set(false)); + execute(&without, &moved, &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!( + "{}: WITHOUT the slot check a moved DEEP(x_r) is accepted — the input-slot \ + check is the only binding: {e:?}", + v.name + ) + }); + } +} + +// ============================================================================= +// (e) — the one-row trace leaf +// ============================================================================= + +/// ★ The in-guest trace leaf at `rows_per_leaf = 1` (and at 2, today's) is +/// the host's: every leaf of the (e) KAT matrices (16 rows × 5 base +/// columns, 16 rows × 2 ext3 columns, read as bit-reversed LDE columns) under +/// the production hash. One row: leaf `i` = the row at bit-reversed position +/// `i`, columns in order. Row pair: rows `2i` then `2i + 1`. +#[test] +fn the_one_row_trace_leaf_is_the_hosts() { + let dir = stark::fri::vectors::vectors_dir(); + let json: Value = serde_json::from_slice( + &std::fs::read(dir.join("e_leaf_digests_rpx.json")).expect("the (e) leaf digests"), + ) + .expect("valid JSON"); + let rows = json["rows"].as_u64().expect("rows") as usize; + let base: Vec> = json["base_columns"] + .as_array() + .expect("base") + .iter() + .map(|c| { + c.as_array() + .expect("a column") + .iter() + .map(|x| FE::from(x.as_u64().expect("a felt"))) + .collect() + }) + .collect(); + let ext: Vec> = json["ext_columns"] + .as_array() + .expect("ext") + .iter() + .map(|c| c.as_array().expect("a column").iter().map(ext_of).collect()) + .collect(); + + let mut checked = 0usize; + for layout in json["layouts"].as_array().expect("layouts") { + let rows_per_leaf = layout["rows_per_leaf"].as_u64().expect("rows") as usize; + for (is_ext, key, width) in [ + (false, "base_leaves", base.len()), + (true, "ext_leaves", ext.len()), + ] { + let shape = GroupShape { + num_columns: width, + is_ext, + }; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(shape.values_at(rows_per_leaf) as u32); + let cells: Vec<_> = (0..shape.values_at(rows_per_leaf) as u32) + .map(|i| b.hint_word(arena, i)) + .collect(); + let leaf = emit_leaf_hash_rows(&mut b, shape, rows_per_leaf, &cells); + for cell in leaf.cells() { + b.public(*cell); + } + let program = compile(b.finish()); + + let leaves = layout[key].as_array().expect("leaves"); + assert_eq!(leaves.len(), rows / rows_per_leaf); + for (i, want) in leaves.iter().enumerate() { + let mut words = Vec::new(); + for k in 0..rows_per_leaf { + let row = reverse_index(rows_per_leaf * i + k, rows as u64); + if is_ext { + words.extend(ext.iter().map(|c| ext_word(&c[row]))); + } else { + words.extend(base.iter().map(|c| base_word(c[row]))); + } + } + let exec = execute(&program, &[words], &crate::hash_pin::BLOCK_HASHER) + .expect("the leaf hash executes"); + let got: Vec = exec.public_words.iter().map(|(_, w)| *w).collect(); + assert_eq!( + got, + super::proof_arena::commitment_words(&commitment_of_hex( + want.as_str().expect("hex") + )), + "rows_per_leaf {rows_per_leaf} {key} leaf {i}" + ); + checked += 1; + } + } + } + assert_eq!(checked, 16 * 2 + 8 * 2, "every leaf of both layouts"); + println!("{checked} in-guest trace leaves == the host's (e) digests (rpx)"); +} + +// ============================================================================= +// Round trips on real proofs — both legs as one program +// ============================================================================= + +fn opts_with(one_row: OneRowMode, cap: CapPolicy, fri: FriMode) -> ProofOptions { + let mut opts = + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup 2"); + opts.fri_number_of_queries = 24; + opts.grinding_factor = 0; + opts.fri_final_poly_log_degree = 2; + opts.format = ProofFormat { + merkle_cap: cap, + fri_mode: fri, + one_row, + ..ProofFormat::DEFAULT + }; + opts +} + +/// The terminal-codeword position a query arrives at. +fn terminal_position(s: FriShape, index: usize) -> usize { + if s.one_row() { + index >> s.total_folds() + } else { + index >> (s.total_folds() - 1) + } +} + +/// Both legs of one real sub-proof emitted into `b`, returning the program's +/// arenas for it (the trace leg's then the FRI leg's) and the closed-form +/// permutation count of the two legs. +fn emit_both_legs(b: &mut LfmBuilder, h: &HostFri) -> (Vec>, usize) { + let s = h.shape; + let all: Vec = (0..h.trace.iotas.len()).collect(); + let (_, _, terminal) = super::fri::emit_sub_proof_with_fri(b, &h.trace.shape, s, all.len()); + for t in &terminal { + b.public(t.as_cell()); + } + let mut arenas = h.trace.arenas(&all); + arenas.extend(h.fri_arenas(&all)); + + let hash = super::edsl::WrapHash::production(); + let sub = &h.trace.shape; + let leaves: usize = sub + .groups() + .iter() + .map(|g| blocks_for(group_leaf_felts_at(g, sub.rows_per_leaf()), hash)) + .sum(); + let closed = all.len() + * (leaves + sub.groups().len() * sub.path_len() + s.permutations_per_query()) + + sub.cap_permutations() + + s.cap_permutations(); + (arenas, closed) +} + +/// ★ The in-guest round trip at `one_row` ∈ {1, auto} × cap {off, auto} × fri +/// {pair, dp} on a real laptop-scale proof (L2G_MEMORY, 2048 rows, blowup 2, +/// `k = 2`, Q = 24): the FRI leg alone and both legs as one program execute +/// over every query, reach the terminal codeword production computed, and emit +/// exactly the closed form. Under `auto` the table's layout is the host's own +/// resolution (printed). At `one_row = 1` a moved one-row trace opening value +/// is refused (the join still binds the fold to the leaf). +#[test] +fn one_row_round_trips_in_guest() { + let mut layouts = Vec::new(); + for one_row in [OneRowMode::On, OneRowMode::Auto] { + for cap in [CapPolicy::Off, CapPolicy::Auto] { + for fri in [FriMode::Pair, FriMode::Dp] { + let label = format!("one_row={one_row} cap={cap} fri={fri:?}"); + let (air, proof) = folding_fixture_with(2048, opts_with(one_row, cap, fri)); + let h = host_fri_from(&*air, &proof); + let s = h.shape; + assert_eq!(h.trace.shape.layout, s.leaf_layout()); + if one_row == OneRowMode::On { + assert!(s.one_row(), "{label}"); + } + layouts.push((label.clone(), s.leaf_layout())); + let all: Vec = (0..h.trace.iotas.len()).collect(); + let codeword = h.terminal_codeword(); + + // The FRI leg alone. + let program = fri_only_program(s, all.len()); + let exec = execute( + &program, + &h.all_arenas(&all), + &crate::hash_pin::BLOCK_HASHER, + ) + .unwrap_or_else(|e| panic!("{label}: FRI leg: {e:?}")); + for (k, &q) in all.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!( + v, + codeword[terminal_position(s, h.trace.iotas[q])], + "{label}" + ); + } + assert_eq!( + permutations(&program), + all.len() * s.permutations_per_query() + s.cap_permutations(), + "{label}: FRI leg closed form" + ); + + // Both legs as one program. + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (arenas, closed) = emit_both_legs(&mut b, &h); + let joined = compile(b.finish()); + let exec = execute(&joined, &arenas, &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("{label}: joined: {e:?}")); + for (k, &q) in all.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!( + v, + codeword[terminal_position(s, h.trace.iotas[q])], + "{label}" + ); + } + assert_eq!( + permutations(&joined), + closed, + "{label}: both legs' closed form" + ); + + if s.one_row() { + // A one-row trace opening value (query 0, main column 0 — + // right after the index word) must not execute. + let mut bad = arenas.clone(); + bad[4][1][0] += FE::one(); + execute(&joined, &bad, &crate::hash_pin::BLOCK_HASHER).expect_err(&format!( + "{label}: a moved one-row trace value must be refused" + )); + // The upper half of the LDE is reached: r is not a pair index. + let lde = 1usize << s.log2_lde_length; + assert!( + h.trace.iotas.iter().any(|&r| r >= lde / 2), + "{label}: 24 one-row indices over the whole LDE reach its upper half" + ); + } + println!( + "{label:<34} layout {:?} index bits {} schedule {:?} FRI caps {:?} trace cap \ + {}: FRI leg {} perms, both legs {} perms / {} instructions", + s.leaf_layout(), + s.index_bits(), + s.schedule(), + (0..s.num_committed()) + .map(|j| s.layer_cap(j)) + .collect::>(), + h.trace.shape.trace_cap, + permutations(&program), + closed, + joined.instrs.len(), + ); + } + } + } + assert_eq!(layouts.len(), 8); +} + +/// ★ Mixed layouts in ONE program: a one-row table (L2G_MEMORY at 2048 rows, +/// `one_row = 1`, `fri = dp`, cap auto) and a row-pair table (L2G_MEMORY at +/// 1024 rows, today's format) verified side by side, both legs each — the +/// shape of an `auto` epoch whose tables resolve differently. Each table's +/// layout is its own verifier constant; the program executes, every terminal +/// is production's, and the permutations are the sum of the two closed forms. +#[test] +fn a_one_row_and_a_row_pair_table_verify_in_one_program() { + let (air_a, proof_a) = folding_fixture_with( + 2048, + opts_with(OneRowMode::On, CapPolicy::Auto, FriMode::Dp), + ); + let (air_b, proof_b) = folding_fixture_with( + 1024, + opts_with(OneRowMode::Off, CapPolicy::Off, FriMode::Pair), + ); + let a = host_fri_from(&*air_a, &proof_a); + let b_host = host_fri_from(&*air_b, &proof_b); + assert_eq!(a.shape.leaf_layout(), LeafLayout::Row); + assert_eq!(b_host.shape.leaf_layout(), LeafLayout::RowPair); + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (mut arenas, closed_a) = emit_both_legs(&mut b, &a); + let (arenas_b, closed_b) = emit_both_legs(&mut b, &b_host); + arenas.extend(arenas_b); + let program = compile(b.finish()); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("a one-row and a row-pair table verify in one program"); + let mut k = 0usize; + for h in [&a, &b_host] { + let codeword = h.terminal_codeword(); + for &iota in &h.trace.iotas { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!(v, codeword[terminal_position(h.shape, iota)]); + k += 1; + } + } + assert_eq!(permutations(&program), closed_a + closed_b); + println!( + "mixed program: one-row table {} perms + row-pair table {} perms = {} ({} instructions)", + closed_a, + closed_b, + closed_a + closed_b, + program.instrs.len() + ); +} + +/// The one-row query index needs a schedule override that the DP never +/// picks to exercise unequal neighbouring exponents from the INPUT tree +/// (the fold-count off-by-one check at layer 0): `[3, 1, 3, 2]` over the 9 committed folds of a +/// 2048-row, blowup-2, `k = 2` one-row table (`12 → 3`, every fold committed) +/// — both legs, executed. +#[test] +fn an_uneven_one_row_schedule_round_trips_in_guest() { + let mut opts = opts_with(OneRowMode::On, CapPolicy::Off, FriMode::Dp); + opts.format.fri_schedule_override = FriScheduleOverride::new(&[3, 1, 3, 2]); + let (air, proof) = folding_fixture_with(2048, opts); + let h = host_fri_from(&*air, &proof); + assert_eq!(h.shape.schedule(), vec![3, 1, 3, 2]); + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (arenas, closed) = emit_both_legs(&mut b, &h); + let program = compile(b.finish()); + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the uneven one-row schedule verifies"); + assert_eq!(permutations(&program), closed); +} diff --git a/prover/src/lfm/one_row_tests.rs b/prover/src/lfm/one_row_tests.rs new file mode 100644 index 000000000..9f51258d8 --- /dev/null +++ b/prover/src/lfm/one_row_tests.rs @@ -0,0 +1,211 @@ +//! S2 (one-row openings) on the LFM side, host only: the commit helpers at +//! both leaf layouts, the registry +//! policy (a one-row format never reads `LFM_REGISTRY`), the one-row roots of +//! a program's artifacts, and the in-circuit register commitment against its +//! host twin at BOTH layouts. Execute-only and artifact builds; nothing here +//! proves. + +use stark::leaf_layout::LeafLayout; +use stark::proof::options::{GoldilocksCubicProofOptions, OneRowMode, ProofOptions}; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::commit::{ + commit_columns_with, commit_lde_columns, commit_lde_columns_with, group_columns, +}; +use super::programs::{RegisterDerivationShape, register_derivation_program}; +use super::registry::{ + LfmProgramKind, PROGRAM_GROUP_SLOTS, REGISTRY_READS, build_artifacts, program_groups, resolve, + resolve_artifacts, +}; +use super::validator::validate; +use super::word::LfmWord; + +fn options(blowup: u8, one_row: OneRowMode) -> ProofOptions { + let mut o = GoldilocksCubicProofOptions::with_blowup(blowup).expect("options"); + o.format.one_row = one_row; + o +} + +fn splitmix(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// `lfm::commit` at one row is `stark::commitment` at `rows_per_leaf = 1`, +/// under the block pin; at row pairs it is today's helper. +#[test] +fn the_commit_helpers_follow_the_layout() { + let mut st = 11u64; + let cols: Vec> = (0..3) + .map(|_| (0..64).map(|_| FE::from(splitmix(&mut st))).collect()) + .collect(); + type B = + ::Batched; + let (_, row) = + stark::commitment::commit_bit_reversed_with::(&cols, 1).expect("tree"); + let (_, pair) = + stark::commitment::commit_bit_reversed_with::(&cols, 2).expect("tree"); + assert_eq!(commit_lde_columns_with(&cols, LeafLayout::Row), row); + assert_eq!(commit_lde_columns_with(&cols, LeafLayout::RowPair), pair); + assert_eq!( + commit_lde_columns(&cols), + pair, + "today's helper is the row-pair one" + ); + assert_ne!(row, pair); +} + +/// ★ The registry policy: `LFM_REGISTRY` stays row-pair only. +/// At the default format `resolve_artifacts` IS the registry row; under a +/// one-row format (`On` or `Auto`) it never reads the registry and builds the +/// program's artifacts at run time — with the SAME row-pair roots and program +/// id (so the identity is unchanged) plus the one-row roots. +#[test] +fn a_one_row_format_never_reads_the_registry() { + let kind = LfmProgramKind::TrivialV0; + let reads = || REGISTRY_READS.with(|c| c.get()); + + let before = reads(); + let default = resolve_artifacts(kind, &options(2, OneRowMode::Off)).expect("registered"); + assert_eq!(reads(), before + 1, "the default format reads the registry"); + assert_eq!(default, resolve(kind, 2).expect("row").artifacts()); + assert!(default.one_row_roots.is_none()); + + for mode in [OneRowMode::On, OneRowMode::Auto] { + let before = reads(); + let built = resolve_artifacts(kind, &options(2, mode)).expect("built"); + assert_eq!(reads(), before, "{mode:?}: LFM_REGISTRY must not be read"); + assert_eq!( + built.roots, default.roots, + "{mode:?}: row-pair roots unchanged" + ); + assert_eq!( + built.program_id, default.program_id, + "{mode:?}: identity unchanged" + ); + let one_row = built.one_row_roots.as_ref().expect("one-row roots built"); + for slot in 0..=10 { + let root = one_row.roots[slot].expect("every committed group has a one-row root"); + assert_ne!(root, built.roots[slot], "slot {slot}: layouts differ"); + } + // Blowup 2 has no one-row static twin: the hosted KECCAK_RC and + // BITWISE roots are hard misses, not recomputes. + assert_eq!(one_row.roots[13], None); + assert_eq!(one_row.roots[14], None); + } +} + +/// The one-row roots are each group's own one-row commitment, and the hosted +/// static tables take their shipped twins (blowup 4, the knob's blowup). +#[test] +fn artifacts_carry_each_groups_one_row_root() { + let program = LfmProgramKind::TrivialV0.program(); + let opts = options(4, OneRowMode::On); + let artifacts = build_artifacts(&program, &opts); + let one_row = artifacts.one_row_roots.as_ref().expect("one-row roots"); + let groups = program_groups(&program); + for (slot, group) in groups.iter().enumerate().take(PROGRAM_GROUP_SLOTS) { + assert_eq!( + one_row.roots[slot], + Some(commit_columns_with( + &group_columns(group), + &opts, + LeafLayout::Row + )), + "slot {slot}" + ); + } + assert_eq!( + one_row.roots[13], + crate::tables::keccak_rc::preprocessed_commitment_for(&opts, LeafLayout::Row) + ); + assert!(one_row.roots[13].is_some() && one_row.roots[14].is_some()); + assert_eq!( + one_row.roots[12], None, + "KECCAK_RND has no preprocessed columns" + ); + assert_eq!( + one_row.roots[super::airs::BLAKE3_SLOT], + one_row.blake3_chunk_roots.first().copied() + ); + // The default format builds none, and its artifacts are unchanged. + let default = build_artifacts(&program, &options(4, OneRowMode::Off)); + assert!(default.one_row_roots.is_none()); + assert_eq!(default.roots, artifacts.roots); + assert_eq!(default.program_id, artifacts.program_id); +} + +fn register_file(seed: u64) -> Vec { + let mut st = seed; + (0..crate::tables::register::NUM_REGISTER_ADDRESSES) + .map(|_| (splitmix(&mut st) >> 32) as u32) + .collect() +} + +fn digest_bytes(public: &[(u32, LfmWord)]) -> [u8; 32] { + use math::field::traits::IsPrimeField; + if public.len() == 1 { + return super::algebraic_commit::digest_to_commitment(&public[0].1); + } + assert_eq!( + public.len(), + 2, + "a digest is one algebraic word or two byte words" + ); + let mut out = [0u8; 32]; + for h in 0..8 { + let lane = public[h / 4].1[h % 4]; + let half = GoldilocksField::canonical(lane.value()) as u32; + out[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + out +} + +/// ★ The in-circuit register commitment against its host twin at BOTH leaf +/// layouts. A mismatch would show only as a runtime +/// `DivByZero` deep in a node, so each layout gets its own root equality. One +/// emitter, two constants (`RegisterDerivationShape::rows_per_leaf`). +#[test] +fn the_register_derivation_matches_its_host_twin_at_both_layouts() { + for blowup in [2usize, 4] { + let opts = GoldilocksCubicProofOptions::with_blowup(blowup as u8).expect("options"); + for layout in [LeafLayout::RowPair, LeafLayout::Row] { + let shape = RegisterDerivationShape { + blowup, + coset_offset: opts.coset_offset, + rows_per_leaf: layout.rows_per_leaf(), + }; + assert_eq!(shape.leaves(), 128 * blowup / layout.rows_per_leaf()); + let program = register_derivation_program(shape); + validate(&program).expect("admission"); + let (init, fini) = (register_file(1), register_file(2)); + let column = |v: &[u32]| { + v.iter() + .map(|&x| super::word::base_word(FE::from(x as u64))) + .collect::>() + }; + let arenas = vec![column(&init), column(&fini)]; + let exec = super::executor::execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("blowup {blowup} {layout:?}: {e:?}")); + let host = crate::tables::register::compute_precomputed_commitment_with( + &opts, &init, &fini, layout, + ); + assert_eq!( + digest_bytes(&exec.public_words), + host, + "blowup {blowup} {layout:?}: the emitted root must equal the host twin's" + ); + if layout == LeafLayout::RowPair { + assert_eq!( + host, + crate::tables::register::compute_precomputed_commitment(&opts, &init, &fini), + "row pairs are today's commitment" + ); + } + } + } +} diff --git a/prover/src/lfm/p1_commit.rs b/prover/src/lfm/p1_commit.rs new file mode 100644 index 000000000..51ca56345 --- /dev/null +++ b/prover/src/lfm/p1_commit.rs @@ -0,0 +1,24 @@ +//! ZisK's Poseidon1 host configuration, as the recursion's emitters and tests +//! name it: the Merkle backends and the Fiat–Shamir sponge live in `crypto` +//! (`merkle_tree::backends::p1`, `fiat_shamir::p1_transcript`), where the WHIR +//! crates can reach them; this module re-exports them under the names the +//! in-guest constructions (`super::p1w16_emit`) are checked against, and names +//! the STARK configuration P1 proofs commit under. + +pub use crypto::fiat_shamir::p1_transcript::{P1GrindDigest, P1Transcript, P1TranscriptHash}; +pub use crypto::merkle_tree::backends::p1::{ + P1BatchBackend, P1LeafHasher, P1PairBackend, leaf, node4, +}; +pub use stark::config::P1StarkHashCapped; + +/// The 4-ary cap height of every Poseidon1 STARK tree (trace, precomputed, +/// aux, composition, FRI layers): C2, `c_L` for both PRs (lead, 10-11). #1013's +/// box arms measured C2 1.46 s faster in median recursion than C1 with 2.9 GiB +/// less peak VRAM (ULTRA 131), and the node census under the hint-order walk +/// puts C2 4.5 % under C1 (I-P1REC §P2.5). +pub const P1_STARK_CAP: u8 = 2; + +/// ★ ZisK's Poseidon1 as a STARK commitment configuration: 4-ary trees of +/// ZisK's tagged leaf hash, capped at 4-ary height [`P1_STARK_CAP`], ground +/// with the width-8 permutation. Prove and verify it with [`P1Transcript`]. +pub type P1StarkHash = P1StarkHashCapped; diff --git a/prover/src/lfm/p1_family_tests.rs b/prover/src/lfm/p1_family_tests.rs new file mode 100644 index 000000000..dfcae396e --- /dev/null +++ b/prover/src/lfm/p1_family_tests.rs @@ -0,0 +1,246 @@ +//! An LFM proof under the block's proof family (`hash_pin::P1Family`): the +//! machine's trivial program proved and verified on ZisK's Poseidon1 — +//! artifacts committed under `P1StarkHash`, ZisK's transcript, the family's +//! statement tag — and refused across the families, either way and with +//! either family's artifacts. The tag is load-bearing: a family that differs +//! from P1's only by keeping today's `LAMBDAVM_LFM_STATEMENT_V1` tag accepts +//! nothing P1 proves, and P1 accepts nothing it proves. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::config::StarkHash; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use crate::hash_pin::{LegacyFamily, P1Family, ProofFamily}; +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; + +use super::p1_commit::P1StarkHash; +use super::programs::trivial_program; +use super::proof::{LfmProof, lfm_prove_under, verify_against_artifacts_under}; +use super::registry::{LfmArtifacts, build_artifacts_under}; +use super::statement::{LFM_STATEMENT_TAG, absorb_lfm_statement, absorb_lfm_statement_tagged}; +use super::word::LfmWord; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +/// P1 in everything but the statement tag, which is today's +/// `LAMBDAVM_LFM_STATEMENT_V1`: the mutant the tag must refuse. +struct P1Untagged; + +impl ProofFamily for P1Untagged { + type Stark = ::Stark; + type Transcript = ::Transcript; + const STATEMENT_TAG: &'static [u8] = LFM_STATEMENT_TAG; + const WRAP: super::edsl::WrapHash = P1Family::WRAP; + const SOCKET: super::hash::HasherKind = P1Family::SOCKET; + fn transcript(seed: &[u8]) -> Self::Transcript { + P1Family::transcript(seed) + } +} + +fn artifacts_under(opts: &ProofOptions) -> LfmArtifacts { + let program = trivial_program(); + let hasher = super::registry::build_artifacts(&program, opts).hasher; + build_artifacts_under::(&program, opts, hasher) +} + +fn prove(artifacts: &LfmArtifacts, opts: &ProofOptions) -> LfmProof { + lfm_prove_under::(&trivial_program(), artifacts, &arenas(), opts).expect("prove") +} + +fn verifies( + artifacts: &LfmArtifacts, + proof: &LfmProof, + opts: &ProofOptions, +) -> bool { + verify_against_artifacts_under::(artifacts, &proof.proof, &proof.public_words, opts) +} + +/// The artifacts under P1 are another program identity: every committed root +/// is P1's (4-ary, ZisK's leaf), and the id names Poseidon1. The legacy +/// family's are `build_artifacts`' byte for byte. +#[test] +fn p1_artifacts_are_their_own_identity() { + let opts = options(); + let program = trivial_program(); + let legacy = super::registry::build_artifacts(&program, &opts); + assert_eq!( + artifacts_under::(&opts).program_id, + legacy.program_id, + "the legacy family's artifacts are today's" + ); + let p1 = artifacts_under::(&opts); + assert_ne!(p1.program_id, legacy.program_id); + for slot in [0usize, 10, 13, 14] { + assert_ne!(p1.roots[slot], legacy.roots[slot], "slot {slot}"); + } + assert_eq!(p1.log_heights, legacy.log_heights); + assert_eq!( + artifacts_under::(&opts).program_id, + p1.program_id, + "deterministic" + ); + assert_eq!( + ::COMMITMENT_HASH, + stark::config::CommitmentHash::Poseidon1 + ); +} + +/// The trivial program proves and verifies under P1, and the families do not +/// cross: a legacy verifier refuses the P1 proof with either family's +/// artifacts, a P1 verifier refuses the legacy proof with either, and neither +/// verifier accepts its own family's proof against the other's artifacts. +#[test] +fn a_p1_lfm_proof_verifies_and_the_families_do_not_cross() { + let opts = options(); + let p1_art = artifacts_under::(&opts); + let rpx_art = artifacts_under::(&opts); + let p1 = prove::(&p1_art, &opts); + let rpx = prove::(&rpx_art, &opts); + assert!(verifies::(&p1_art, &p1, &opts), "honest P1 proof"); + assert!( + verifies::(&rpx_art, &rpx, &opts), + "honest legacy proof" + ); + assert_eq!(p1.public_words, rpx.public_words, "one execution"); + for art in [&p1_art, &rpx_art] { + assert!( + !verifies::(art, &p1, &opts), + "P1 proof, legacy verifier" + ); + assert!( + !verifies::(art, &rpx, &opts), + "legacy proof, P1 verifier" + ); + } + assert!( + !verifies::(&rpx_art, &p1, &opts), + "P1 proof against legacy artifacts" + ); + // A claimed word the execution did not publish is refused under P1. + let mut claimed = p1.public_words.clone(); + claimed[0].1[0] = &claimed[0].1[0] + FE::from(1u64); + assert!(!verify_against_artifacts_under::( + &p1_art, &p1.proof, &claimed, &opts + )); +} + +/// The family's statement tag is load-bearing: P1 under today's tag accepts +/// no P1 proof, and P1 accepts none of its proofs; each accepts its own. +#[test] +fn the_p1_statement_tag_is_load_bearing() { + let opts = options(); + let art = artifacts_under::(&opts); + let tagged = prove::(&art, &opts); + let untagged = prove::(&art, &opts); + assert!(verifies::(&art, &tagged, &opts)); + assert!(verifies::(&art, &untagged, &opts)); + assert!( + !verifies::(&art, &tagged, &opts), + "a tagged proof under the untagged verifier" + ); + assert!( + !verifies::(&art, &untagged, &opts), + "an untagged proof under the tagged verifier" + ); +} + +/// The family's tag REPLACES today's: the statement is the tag as its first +/// call, then today's calls after `LAMBDAVM_LFM_STATEMENT_V1` unchanged. The +/// legacy family's tag is `LAMBDAVM_LFM_STATEMENT_V1` itself, so its statement +/// is today's bytes. +#[test] +fn the_tag_replaces_todays_first_call() { + type E = GoldilocksExtension; + let id = [7u8; 32]; + let words: Vec<(u32, LfmWord)> = vec![ + (0, core::array::from_fn(|j| FE::from(j as u64 + 1))), + (3, core::array::from_fn(|j| FE::from(u64::MAX - j as u64))), + ]; + let state = |tag: &[u8]| { + let mut t = P1Family::transcript(&[]); + absorb_lfm_statement_tagged(&mut t, tag, &id, &words, 8); + <_ as IsTranscript>::state(&t) + }; + // Today's statement, call by call, under `first` as its first call. + let by_hand = |first: &[u8]| { + let mut t = P1Family::transcript(&[]); + let mut append = |bytes: &[u8]| <_ as IsTranscript>::append_bytes(&mut t, bytes); + append(first); + append(&id); + append(&super::statement::LFM_MACHINE_VERSION.to_le_bytes()); + append(&(words.len() as u64).to_le_bytes()); + for (index, word) in &words { + append(&index.to_le_bytes()); + for lane in word { + append(&GoldilocksField::canonical(lane.value()).to_le_bytes()); + } + } + append(&[8]); + <_ as IsTranscript>::state(&t) + }; + let mut today = P1Family::transcript(&[]); + absorb_lfm_statement(&mut today, &id, &words, 8); + let today = <_ as IsTranscript>::state(&today); + assert_eq!(state(LegacyFamily::STATEMENT_TAG), today); + assert_eq!(by_hand(b"LAMBDAVM_LFM_STATEMENT_V1"), today); + assert_eq!( + state(P1Family::STATEMENT_TAG), + by_hand(crate::hash_pin::P1_LFM_STATEMENT_TAG) + ); + assert_ne!(state(P1Family::STATEMENT_TAG), today); + // Replaced, not prepended: the P1 tag then today's tag is another state. + let mut prepended = P1Family::transcript(&[]); + <_ as IsTranscript>::append_bytes(&mut prepended, P1Family::STATEMENT_TAG); + absorb_lfm_statement(&mut prepended, &id, &words, 8); + assert_ne!( + state(P1Family::STATEMENT_TAG), + <_ as IsTranscript>::state(&prepended) + ); + assert!(P1Family::STATEMENT_TAG.starts_with(LFM_STATEMENT_TAG)); +} + +/// The host lays a root at the width the reading node's builder reads it +/// (REV-P3B14-A A2): `proof_arena::words_per_root_for` is `edsl::digest_words` +/// of a builder under the same wrap hash, for every wrap hash, and a +/// one-cell digest (RPX, Poseidon1) is the backend's four canonical felts. So +/// the harvest, which lays a child's arenas at its family's `WRAP`, follows a +/// family flip to a byte-digest hash instead of keeping the legacy pin's width. +#[test] +fn each_wrap_hash_lays_a_root_at_its_builders_width() { + use super::edsl::WrapHash; + use super::proof_arena::{commitment_words, commitment_words_for, words_per_root_for}; + let root: [u8; 32] = core::array::from_fn(|i| (7 * i + 3) as u8); + for hash in [ + WrapHash::Keccak, + WrapHash::Blake3, + WrapHash::Algebraic, + WrapHash::Poseidon1, + ] { + let b = super::builder::LfmBuilder::new().with_wrap_hash(hash); + let words = words_per_root_for(hash); + assert_eq!(words as u32, super::edsl::digest_words(&b), "{hash:?}"); + assert_eq!(commitment_words_for(&root, hash).len(), words, "{hash:?}"); + } + assert_eq!( + commitment_words_for(&root, P1Family::WRAP), + vec![super::algebraic_commit::commitment_to_digest(&root)], + "a P1 root is one cell" + ); + assert_eq!( + commitment_words(&root), + commitment_words_for(&root, LegacyFamily::WRAP), + "the legacy pin's width is its family's" + ); +} diff --git a/prover/src/lfm/p1_node_tests.rs b/prover/src/lfm/p1_node_tests.rs new file mode 100644 index 000000000..d85b3fdd9 --- /dev/null +++ b/prover/src/lfm/p1_node_tests.rs @@ -0,0 +1,594 @@ +//! A node verifies a Poseidon1 LFM child proof in-guest (D-P3B P2): the +//! trivial program proved under `hash_pin::P1Family`, harvested into the +//! node's arenas, and one leg (`per_table_aggregator::emit_leg`) emitted under +//! `WrapHash::Poseidon1` — ZisK's transcript replay with the family's +//! statement tag, the width-8 grind check, the tagged leaf hash and the 4-ary +//! hint-order walks (`p1w16_emit::walk4`) to each tree's cap +//! (`P1_STARK_CAP`). The honest arenas execute; every tamper below is refused +//! by the machine, never by the host harvest. The node-level checks that one +//! constraint alone holds — the grind at the node's factor, the cap-to-root +//! of every cap node no query reaches — each have a test of their own, and the +//! legs' opening (hint-order walk, cap mux and root) is checked exhaustively +//! against the host's trees at small depths. + +use crypto::merkle_tree::merkle::MerkleTree; +use stark::config::Commitment; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use crate::hash_pin::{P1Family, ProofFamily}; +use crate::tables::types::{FE, GoldilocksField}; + +use super::algebraic_commit::{commitment_to_digest, digest_to_commitment}; +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::edsl::{WrapDigest, WrapHash}; +use super::executor::{LfmExecError, execute, execute_serial}; +use super::harvest::{HarvestedChild, harvest_child_under, try_child_arena_words}; +use super::hash::HasherKind; +use super::per_table_aggregator::{DerivedChild, declare_leg_arenas, emit_leg}; +use super::programs::trivial_program; +use super::proof::{LfmProof, lfm_prove_under, verify_against_artifacts_under}; +use super::registry::{LfmArtifacts, build_artifacts_under}; +use super::word::{LfmWord, base_word}; + +/// The child's options: the production format's `fri = dp` at blowup 4, with +/// few queries so the node stays small, and a grind so the leg checks one. +/// Its binary cap policy is OFF: an arity-4 configuration caps its trees at +/// its own height (`P1_STARK_CAP`) whatever the format says, so a shape that +/// took the format's policy instead would lay the trees out uncapped and +/// refuse the child (at six queries `auto` could coincide with it and hide +/// that). +pub(super) fn options() -> ProofOptions { + options_at(4) +} + +/// [`options`] at `grinding` bits. +fn options_at(grinding: u8) -> ProofOptions { + let mut o = GoldilocksCubicProofOptions::with_blowup(4).expect("options"); + o.fri_number_of_queries = 6; + o.grinding_factor = grinding; + o.fri_final_poly_log_degree = 2; + let mut o = crate::zf_format::ZfFormat::DEFAULT.options(o); + o.format.merkle_cap = stark::proof::options::CapPolicy::Off; + o +} + +pub(super) fn arenas() -> Vec> { + arenas_salted(0) +} + +/// The trivial program's arena, its words moved by `salt` (another +/// execution, another transcript). +fn arenas_salted(salt: u64) -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64 + 17 * salt))) + .collect(), + ] +} + +pub(super) struct P1Child { + pub(super) artifacts: LfmArtifacts, + pub(super) proof: LfmProof, + pub(super) opts: ProofOptions, +} + +pub(super) fn p1_child() -> P1Child { + p1_child_with(options()) +} + +/// [`p1_child`] proved and checked at `opts`. +pub(super) fn p1_child_with(opts: ProofOptions) -> P1Child { + let program = trivial_program(); + let hasher = super::registry::build_artifacts(&program, &opts).hasher; + let artifacts = + build_artifacts_under::<::Stark>(&program, &opts, hasher); + let proof = lfm_prove_under::(&program, &artifacts, &arenas(), &opts).expect("prove"); + assert!(verify_against_artifacts_under::( + &artifacts, + &proof.proof, + &proof.public_words, + &opts + )); + P1Child { + artifacts, + proof, + opts, + } +} + +pub(super) fn harvest(c: &P1Child) -> HarvestedChild { + harvest_child_under::(c.artifacts.clone(), c.opts.clone(), &c.proof) + .unwrap_or_else(|e| panic!("{e}")) +} + +/// One leg over the child, under `wrap`, with the child's statement tag +/// `tag` (the family's, unless a mutation swaps it). +pub(super) fn node(c: &P1Child, tag: &'static [u8]) -> LfmProgram { + node_at(&c.artifacts, &c.opts, c.proof.public_words.len(), tag) +} + +/// One leg over a child of `artifacts` whose shapes — the grind factor +/// among them — the node derives at `opts`. +fn node_at( + artifacts: &LfmArtifacts, + opts: &ProofOptions, + words: usize, + tag: &'static [u8], +) -> LfmProgram { + let derived = DerivedChild::from_artifacts_under::(artifacts, opts, words) + .unwrap_or_else(|e| panic!("{e}")); + let mut shape = derived.shape(); + shape.statement_tag = tag; + let mut b = LfmBuilder::new().with_wrap_hash(P1Family::WRAP); + let a = declare_leg_arenas(&mut b, &shape); + let _ = emit_leg(&mut b, &shape, &a); + compile(b.finish()) +} + +pub(super) fn runs(program: &LfmProgram, child: &HarvestedChild) -> bool { + let arenas = try_child_arena_words(child).unwrap_or_else(|e| panic!("{e}")); + execute(program, &arenas, &HasherKind::Poseidon1W16).is_ok() +} + +/// Whether the node refuses the child at one of its checks: the execution +/// fails, and not at the arenas' layout or the socket (a refusal of those +/// kinds would say nothing about the check a test isolates). +pub(super) fn refused_by_a_check(program: &LfmProgram, child: &HarvestedChild) -> bool { + let arenas = try_child_arena_words(child).unwrap_or_else(|e| panic!("{e}")); + match execute(program, &arenas, &HasherKind::Poseidon1W16) { + Ok(_) => false, + Err( + e @ (LfmExecError::ArenaCountMismatch { .. } + | LfmExecError::ArenaLenMismatch { .. } + | LfmExecError::ArenaOutOfBounds { .. } + | LfmExecError::HasherRejected(_) + | LfmExecError::Internal(_)), + ) => panic!("refused before any check: {e:?}"), + Err(_) => true, + } +} + +fn flip(c: &mut [u8; 32]) { + c[31] ^= 1; +} + +#[test] +fn a_p1_node_verifies_a_p1_child_and_refuses_its_tampers() { + let c = p1_child(); + let honest = harvest(&c); + let program = node(&c, P1Family::STATEMENT_TAG); + assert!( + program.hasher(crate::hash_pin::BLOCK_HASHER) == HasherKind::Poseidon1W16, + "a P1 node's hash rows are the width-16 socket's" + ); + assert!(runs(&program, &honest), "the honest child executes"); + // The child's arenas are laid at the family's digest width, the node's. + assert_eq!(honest.wrap, P1Family::WRAP); + assert!(honest.legs.iter().all(|l| l.wrap == P1Family::WRAP)); + // The P1 child's trees are capped (`P1_STARK_CAP`): a cap is hinted per + // tree. + assert!( + honest.legs.iter().any(|l| !l.caps.is_empty()), + "some table carries caps" + ); + + let refused = |what: &str, edit: &dyn Fn(&mut HarvestedChild)| { + let mut bad = harvest(&c); + edit(&mut bad); + assert!(!runs(&program, &bad), "{what}: the node must refuse"); + }; + // A table whose trees are capped and whose FRI commits a layer. + let t = honest + .legs + .iter() + .position(|l| !l.caps.is_empty() && !l.fri_openings[0].is_empty()) + .expect("a capped table with a committed FRI layer"); + refused("main root", &|h| flip(&mut h.tables[t].main_root)); + refused("composition root", &|h| { + flip(&mut h.tables[t].composition_root) + }); + refused("trace sibling", &|h| { + flip(&mut h.legs[t].openings[1][0].1[0]) + }); + refused("cap node", &|h| { + let last = h.legs[t].caps.len() - 1; + flip(&mut h.legs[t].caps[last]) + }); + refused("FRI sibling", &|h| { + flip(&mut h.legs[t].fri_openings[2][0].1[0]) + }); + refused("FRI root", &|h| flip(&mut h.tables[t].fri_roots[0])); + refused("opened value", &|h| { + h.legs[t].openings[0][0].0[0][0] += FE::from(1u64) + }); + refused("OOD value (the transcript)", &|h| { + h.tables[t].ood_current[0] += crate::tables::types::FEE::from(1u64) + }); + refused("nonce", &|h| { + h.tables[t].nonce = h.tables[t].nonce.map(|n| n ^ 1) + }); + // ★ The query index is the transcript's: swapping two queries' openings + // (each internally consistent) moves every leaf to another index. + refused("swapped queries", &|h| { + h.legs[t].openings.swap(0, 1); + h.legs[t].fri_openings.swap(0, 1); + }); + refused("a claimed word", &|h| { + h.public_words[0].1[0] += FE::from(1u64) + }); +} + +/// The family's statement tag is load-bearing in the guest: a node that +/// replays the child's statement under today's tag (the legacy family's) +/// refuses the honest child. +#[test] +fn a_node_without_the_family_tag_refuses_the_p1_child() { + let c = p1_child(); + let honest = harvest(&c); + assert!(runs(&node(&c, P1Family::STATEMENT_TAG), &honest)); + assert!(!runs( + &node(&c, crate::hash_pin::LegacyFamily::STATEMENT_TAG), + &honest + )); +} + +/// ★ The leg checks each table's grind at the factor the NODE derives from +/// the child's options, not at one the child chose (REV-P3B14-A A1): a child +/// proved at `g − 1` bits whose nonces do not all meet `g` is accepted by the +/// node derived at `g − 1` and refused by the one derived at `g`, as the host +/// verifier at `g` refuses it. `rp1wa_the_emitted_grind_accepts_exactly_the_ +/// hosts_nonces` drives the grind gadget directly, and the tamper test's nonce +/// flip re-rolls every query, so neither sees the leg's call site +/// (`epoch::emit_table_challenges`) drop the check or skip it on the P1 wrap +/// hash; this does. (A weaker factor at the call site is no stealthy mutant: +/// the factor is part of the grind's hashed preimage, so the honest node +/// tests refuse too.) From the reviewers' short-grind tests (REV-P3B14). +#[test] +fn a_p1_node_refuses_a_child_ground_one_bit_short() { + let (short, full) = (options_at(3), options_at(4)); + let program = trivial_program(); + let hasher = super::registry::build_artifacts(&program, &short).hasher; + let artifacts = + build_artifacts_under::<::Stark>(&program, &short, hasher); + let full_artifacts = + build_artifacts_under::<::Stark>(&program, &full, hasher); + assert_eq!( + artifacts.program_id, full_artifacts.program_id, + "the grind is no part of the program" + ); + // A child at 3 bits that some table's nonce does not carry to 4: the + // factor is part of the hashed preimage, so a 3-bit nonce passes a 4-bit + // check about 1/16 of the time per table, and the first salt nearly + // always gives one. + let (proof, salt) = (0..16u64) + .find_map(|salt| { + let proof = + lfm_prove_under::(&program, &artifacts, &arenas_salted(salt), &short) + .expect("prove"); + let verifies = |opts: &ProofOptions| { + verify_against_artifacts_under::( + &artifacts, + &proof.proof, + &proof.public_words, + opts, + ) + }; + assert!(verifies(&short), "salt {salt}: the honest child at 3 bits"); + (!verifies(&full)).then_some((proof, salt)) + }) + .expect("a child whose nonces do not all meet 4 bits"); + println!("P1 NODE GRIND: salt {salt}: the host at 4 bits refuses the 3-bit child"); + let child = harvest_child_under::(artifacts.clone(), short.clone(), &proof) + .unwrap_or_else(|e| panic!("{e}")); + let words = proof.public_words.len(); + assert!( + runs( + &node_at(&artifacts, &short, words, P1Family::STATEMENT_TAG), + &child + ), + "the node at 3 bits accepts the 3-bit child" + ); + assert!( + refused_by_a_check( + &node_at(&artifacts, &full, words, P1Family::STATEMENT_TAG), + &child + ), + "the node at 4 bits must refuse a child ground at 3" + ); +} + +/// ★ Every cap node of a P1 child that NO query reaches is bound in the node: +/// such a node is read only by `CapCells::authenticate_at`'s cap-to-root (the +/// mux passes it into a branch no opening keeps), so moving it is refused by +/// that check alone. Over every table, every capped trace group and FRI layer: +/// the cap nodes the transcript's indices reach are computed from the +/// harvest's `iotas` (trace leaf `iota`, layer `j` leaf `iota >> (G_j + d_j)`, +/// cap node `leaf >> (D − log2(cap nodes))`), and each unreached one is moved. +/// Which nodes a run reaches follows its grind nonces, so the set is computed, +/// never assumed (the tamper test above flips one node whose reach it does not +/// control). From REV-P3B14-B's +/// `rp3b14b_every_unqueried_cap_node_of_a_p1_child_is_bound_in_the_node`. +#[test] +#[ignore = "proves a P1 child and runs its node once per unreached cap node; box tier"] +fn every_unqueried_cap_node_of_a_p1_child_is_bound_in_the_node() { + let c = p1_child(); + let honest = harvest(&c); + let program = node(&c, P1Family::STATEMENT_TAG); + assert!(runs(&program, &honest), "the honest child executes"); + let mut unreached: Vec<(usize, usize)> = Vec::new(); + let mut reached_one: Option<(usize, usize)> = None; + for (t, leg) in honest.legs.iter().enumerate() { + let v = &leg.verify; + let iotas = &honest.tables[t].iotas; + // (depth, cap height, the leaf index of each query) per capped tree, + // in the caps arena's order: the groups, then the capped layers. + let mut capped: Vec<(usize, usize, Vec)> = Vec::new(); + if v.sub.trace_cap > 0 { + for _ in 0..v.sub.groups().len() { + capped.push((v.sub.merkle_depth, v.sub.trace_cap, iotas.clone())); + } + } + for j in 0..v.fri.num_committed() { + if v.fri.layer_cap(j) > 0 { + let shift = v.fri.layer_bit_offset(j) + v.fri.layer_fold(j) as usize; + let leaves = iotas.iter().map(|i| i >> shift).collect(); + capped.push((v.fri.layer_depth(j), v.fri.layer_cap(j), leaves)); + } + } + let mut at = 0usize; + for (depth, cap, leaves) in capped { + let nodes = super::merkle_cap::cap_nodes(depth, cap, 4); + let walked = depth - super::merkle_cap::mux_bits(depth, cap, 4); + let hit: std::collections::BTreeSet = + leaves.iter().map(|l| l >> walked).collect(); + for k in 0..nodes { + if hit.contains(&k) { + reached_one.get_or_insert((t, at + k)); + } else { + unreached.push((t, at + k)); + } + } + at += nodes; + } + assert_eq!( + at, + leg.caps.len(), + "table {t}: the caps arena is every capped tree's cap" + ); + } + assert!( + !unreached.is_empty(), + "some cap node is reached by no query" + ); + for &(t, k) in &unreached { + let mut bad = harvest(&c); + flip(&mut bad.legs[t].caps[k]); + assert!( + refused_by_a_check(&program, &bad), + "table {t} cap word {k}: an unreached cap node moved" + ); + } + let (t, k) = reached_one.expect("some cap node is reached"); + let mut bad = harvest(&c); + flip(&mut bad.legs[t].caps[k]); + assert!( + refused_by_a_check(&program, &bad), + "a reached cap node moved" + ); + println!( + "P1 NODE CAPS: {} unreached cap nodes moved and refused (of {} cap words)", + unreached.len(), + honest.legs.iter().map(|l| l.caps.len()).sum::() + ); +} + +// ============ the legs' opening against the host's own trees ============ + +type P1Backend = super::p1_commit::P1BatchBackend; + +fn mix(seed: u64, i: u64) -> u64 { + let mut z = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(i + 1); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +fn lanes(seed: u64) -> [FE; 4] { + core::array::from_fn(|k| FE::from(mix(seed, k as u64))) +} + +/// One opening of a host Poseidon1 tree over `2^depth` leaves, checked the +/// way an LFM leg checks a trace group or a FRI layer: the path as the proof +/// carries it (`embed_cap_arity` appends the cap to the owner's and cuts every +/// other), laid out by the harvest's `path_to_cap` at `layout_at` (the index +/// the harvest believes), the cap hinted and authenticated against the root by +/// `hint_and_authenticate`, then `authenticate_leaf` at the bits of `bits_of` +/// (the transcript's index, decomposed by `BitDec`). `index` is the leaf whose +/// path is supplied; `owner` picks query 0's path or another query's; `tamper` +/// moves one arena word (arenas: 0 leaf, 1 index, 2 hints, 3 cap, 4 root). +fn leg_opening( + depth: usize, + c: usize, + index: usize, + layout_at: usize, + bits_of: usize, + owner: bool, + tamper: Option<(usize, usize)>, +) -> Result<(), String> { + let n = 1usize << depth; + let leaves: Vec = (0..n) + .map(|i| digest_to_commitment(&lanes(31_000 + depth as u64 * 257 + i as u64))) + .collect(); + let tree = MerkleTree::::build_from_hashed_leaves(leaves.clone()).expect("a tree"); + let full = tree + .get_proof_by_pos(index) + .expect("a leaf") + .merkle_path + .clone(); + let cap = if c == 0 { + Vec::new() + } else { + tree.cap(c).expect("the cap") + }; + let (mut owner_path, mut other_path) = (full.clone(), full); + if c > 0 { + crypto::merkle_tree::cap::embed_cap_arity( + &mut [&mut owner_path, &mut other_path], + depth, + &cap, + 4, + ) + .expect("the proof's paths"); + } + let path = if owner { &owner_path } else { &other_path }; + let (hints, split) = super::merkle_cap::path_to_cap(path, depth, c, 4, owner, Some(layout_at)) + .map_err(|e| format!("layout: {e}"))?; + if owner { + assert_eq!( + split.unwrap_or_default(), + cap, + "the cap rides on the owner path" + ); + } else { + assert!(split.is_none(), "only the owner carries the cap"); + } + assert_eq!(hints.len(), super::merkle_cap::path_hints(depth, c, 4)); + + let word = |c: &Commitment| -> LfmWord { commitment_to_digest(c) }; + let mut arenas: Vec> = vec![ + vec![word(&leaves[index])], + vec![base_word(FE::from(bits_of as u64))], + hints.iter().map(word).collect(), + cap.iter().map(word).collect(), + vec![word(&tree.root)], + ]; + if let Some((a, w)) = tamper { + arenas[a][w][1] += FE::from(1u64); + } + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let lens: Vec = arenas.iter().map(|a| a.len().max(1) as u32).collect(); + let ids: Vec<_> = lens.iter().map(|&l| b.declare_arena(l)).collect(); + let leaf = WrapDigest::from_cell(b.hint_word(ids[0], 0)); + let idx = b.hint_felt(ids[1], 0); + let bits = b.bit_dec(idx, depth); + let h: Vec = (0..arenas[2].len() as u32) + .map(|i| WrapDigest::from_cell(b.hint_word(ids[2], i))) + .collect(); + let root = b.hint_word(ids[4], 0); + let root_lanes = [b.unpack(root)]; + let capped = (c > 0).then(|| { + let (cells, next) = + super::merkle_cap::hint_and_authenticate(&mut b, ids[3], 0, c, depth, &root_lanes); + assert_eq!(next as usize, cap.len(), "the caps arena is read exactly"); + cells + }); + super::sub_proof::authenticate_leaf(&mut b, leaf, &root_lanes, capped.as_ref(), &bits, &h); + // An arena the check reads nothing from (no hints under a leaf-level cap, + // no cap uncapped) is declared one word long. + for a in arenas.iter_mut().filter(|a| a.is_empty()) { + a.push(base_word(FE::from(0u64))); + } + let program = compile(b.finish()); + super::validator::validate(&program).map_err(|e| format!("{e:?}"))?; + execute_serial(&program, &arenas, &HasherKind::Poseidon1W16) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) +} + +/// ★ The LFM legs' production opening (hint-order `walk4` to the cap, the +/// cap's mux and its root) against the host's own Poseidon1 trees, +/// EXHAUSTIVELY at small depths: at every depth 1–7 (both parities), every cap +/// height up to the tree's levels and EVERY leaf index, the harvest's layout +/// of query 0's path (cap split off) and of any other query's path executes; +/// a leaf's path walked at another index's bits is refused, whether the hints +/// are laid out for the leaf's index or for the other one (the prover's choice +/// of hint order cannot move the leaf: every placement is a `Select` on a +/// transcript bit); and every word the check reads — the leaf, each hint, each +/// cap node (queried or not), the root — moved refuses it. Then the production +/// cap (C2) at depths 8–12 over a spread of indices. From REV-P3B14-B's +/// `rp3b14b_the_hint_order_walk_and_cap_bind_every_index`. +#[test] +fn the_hint_order_walk_and_cap_bind_every_index() { + let mut refused_moves = 0usize; + for depth in 1usize..=7 { + let n = 1usize << depth; + let levels = crypto::merkle_tree::cap::tree_levels(depth, 4); + for c in 0..=levels { + let hints = super::merkle_cap::path_hints(depth, c, 4); + let nodes = if c == 0 { + 0 + } else { + super::merkle_cap::cap_nodes(depth, c, 4) + }; + for index in 0..n { + for owner in [true, false] { + leg_opening(depth, c, index, index, index, owner, None).unwrap_or_else(|e| { + panic!("depth {depth} cap {c} index {index} owner {owner}: {e}") + }); + } + let others: Vec = if depth <= 4 { + (0..n).filter(|&o| o != index).collect() + } else { + [ + index ^ 1, + index ^ 2, + index ^ 3, + index ^ (n / 2), + n - 1 - index, + ] + .into_iter() + .filter(|&o| o != index) + .collect() + }; + for other in others { + assert!( + leg_opening(depth, c, index, index, other, true, None).is_err(), + "depth {depth} cap {c}: leaf {index}'s hints walked at {other}" + ); + assert!( + leg_opening(depth, c, index, other, other, false, None).is_err(), + "depth {depth} cap {c}: leaf {index}'s path laid out and walked at {other}" + ); + } + if depth <= 5 || index % 9 == 0 { + let mut words = vec![(0usize, 0usize), (4, 0)]; + words.extend((0..hints).map(|w| (2, w))); + words.extend((0..nodes).map(|w| (3, w))); + for (a, w) in words { + assert!( + leg_opening(depth, c, index, index, index, true, Some((a, w))).is_err(), + "depth {depth} cap {c} index {index}: arena {a} word {w} moved" + ); + refused_moves += 1; + } + } + } + } + } + // The production cap height on taller trees. + for depth in 8usize..=12 { + let n = 1usize << depth; + for index in [0, 1, 2, 3, n / 3, n / 2 + 5, n - 4, n - 1] { + for owner in [true, false] { + leg_opening(depth, 2, index, index, index, owner, None) + .unwrap_or_else(|e| panic!("depth {depth} C2 index {index}: {e}")); + } + for other in [index ^ 1, index ^ 4, index ^ (n / 2)] { + assert!(leg_opening(depth, 2, index, index, other, false, None).is_err()); + assert!(leg_opening(depth, 2, index, other, other, false, None).is_err()); + } + } + let nodes = super::merkle_cap::cap_nodes(depth, 2, 4); + for w in 0..nodes { + assert!( + leg_opening(depth, 2, 5, 5, 5, true, Some((3, w))).is_err(), + "depth {depth} C2: cap node {w} moved" + ); + } + } + assert!( + refused_moves > 1000, + "the moves were probed: {refused_moves}" + ); +} diff --git a/prover/src/lfm/p1w16_chip.rs b/prover/src/lfm/p1w16_chip.rs new file mode 100644 index 000000000..9a704070e --- /dev/null +++ b/prover/src/lfm/p1w16_chip.rs @@ -0,0 +1,455 @@ +//! A Poseidon1 width-16 `LFM_HASH`-style chip — a MEASUREMENT chip for the +//! D-HASH track, not wired into the machine. +//! +//! The socket today is 12 felts wide (`HASH_STATE_FELTS`), so a width-16 +//! permutation cannot be a `HasherKind` arm without a new tuple contract. What +//! the base-hash decision needs first is the chip's WIDTH (cells per +//! permutation), which decides most of the recursion side of D-HASH's model +//! (§2.3). This module builds the width-16 constraint set in two layouts, +//! fills rows for it, and pins both to the host reference +//! [`crypto::hash::poseidon1_w16`]: +//! +//! - [`Layout::Rule`] — our W12 chip's rule at width 16 (`chips::hash:: +//! poseidon_cols`): per round `x²`, `x³` for each S-boxed lane plus a fresh +//! 16-lane post-MDS output block (the last round's output is `OUT`). +//! - [`Layout::Compact`] — per S-boxed lane `x³ = a³` and `x⁷ = (x³)²·a`, and +//! NO state columns: every round's input is an affine form in the earlier +//! `x⁷` columns and the prefix, carried symbolically. Both constraints are +//! degree 3. Plonky3's `poseidon1-air` commits a post-round state per full +//! round on top of this (444 value columns at width 16); we skip it. +//! +//! # The socket prefix (width 16) +//! +//! Preprocessed ([`PREP_WIDTH`] = 16, not counted as cells): four input and +//! four output addresses, the four mode selectors, four multiplicities. +//! Value prefix (36): `IN0..16`, the capacity copies `S12..16`, `OUT0..16`. +//! The bus would carry four input and four output word tuples (8 interactions, +//! 4 LogUp aux columns); the aux columns are counted, not built. +//! +//! Modes: `MODE_P` (permutation) and `MODE_C` (4-ary node: the four child +//! digests fill all 16 lanes, so it takes its capacity lanes from `IN` like a +//! permutation) copy `S_k = IN_{12+k}`; `MODE_T` and `MODE_L` take a domain +//! capacity `[0, domain, 0, 0]`. The round constants are scaled by the mode +//! sum, so an all-zero padding row satisfies every constraint. + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; +use crypto::hash::poseidon1_w16::{ + self as p1, DOMAIN_LEAF, NUM_ROUNDS, RATE_FELTS, STATE_FELTS, constants::CAUCHY_MDS, + constants::MDS_CIRC_ROW, constants::ROUND_CONSTANTS, is_full_round, +}; +use math::field::traits::IsPrimeField; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// The transcript domain of this measurement instance (`"P1WT"`). +pub const DOMAIN_TRANSCRIPT: u64 = u32::from_le_bytes(*b"P1WT") as u64; + +/// Preprocessed columns. +pub const IN_ADDR0: usize = 0; // ..IN_ADDR3 +pub const OUT_ADDR0: usize = 4; // ..OUT_ADDR3 +pub const MODE_C: usize = 8; +pub const MODE_P: usize = 9; +pub const MODE_T: usize = 10; +pub const MODE_L: usize = 11; +pub const MULT0: usize = 12; // ..MULT3 +pub const PREP_WIDTH: usize = 16; + +/// Value prefix, shared by both layouts. +pub const IN0: usize = PREP_WIDTH; +pub const S12: usize = IN0 + STATE_FELTS; +pub const OUT0: usize = S12 + 4; +pub const PREFIX_VALUE_COLUMNS: usize = 2 * STATE_FELTS + 4; +/// First appended witness column. +pub const ROUNDS: usize = PREP_WIDTH + PREFIX_VALUE_COLUMNS; + +/// Bus interactions the width-16 socket would carry: four word reads, four +/// word writes. +pub const BUS_INTERACTIONS: usize = 8; +/// LogUp aux columns: one per pair of interactions. +pub const AUX_COLUMNS: usize = BUS_INTERACTIONS.div_ceil(2); +/// Base-field cells per aux column (a cubic-extension accumulator), the census +/// convention `621 = 612 + 3·3` of the W12 chip. +pub const CELLS_PER_AUX_COLUMN: usize = 3; + +/// S-boxed lanes in round `r`. +pub const fn sboxed_lanes(r: usize) -> usize { + if is_full_round(r) { STATE_FELTS } else { 1 } +} + +/// The two measured layouts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Layout { + /// Our W12 rule at width 16: `x²`, `x³` per S-box plus a per-round state. + Rule, + /// `x³`, `x⁷` per S-box and no state columns. + Compact, +} + +impl Layout { + /// Width of round `r`'s appended block. + pub const fn block_width(self, r: usize) -> usize { + match self { + Layout::Rule => { + let out = if r + 1 == NUM_ROUNDS { 0 } else { STATE_FELTS }; + 2 * sboxed_lanes(r) + out + } + Layout::Compact => 2 * sboxed_lanes(r), + } + } + + /// First column of round `r`'s block. + pub const fn block(self, r: usize) -> usize { + let mut off = ROUNDS; + let mut i = 0; + while i < r { + off += self.block_width(i); + i += 1; + } + off + } + + /// The first S-box witness of `lane` in round `r`: `x²` (Rule) or `x³` (Compact). + pub const fn w1(self, r: usize, lane: usize) -> usize { + self.block(r) + lane + } + + /// The second S-box witness: `x³` (Rule) or `x⁷` (Compact). + pub const fn w2(self, r: usize, lane: usize) -> usize { + self.block(r) + sboxed_lanes(r) + lane + } + + /// Rule only: round `r`'s post-MDS output lane `j` (`OUT` for the last round). + pub const fn out(self, r: usize, j: usize) -> usize { + if r + 1 == NUM_ROUNDS { + OUT0 + j + } else { + self.block(r) + 2 * sboxed_lanes(r) + j + } + } + + /// The chip's total width, preprocessed prefix included. + pub const fn num_columns(self) -> usize { + self.block(NUM_ROUNDS) + } + + /// Value columns: the census's `main_cols`. + pub const fn value_columns(self) -> usize { + self.num_columns() - PREP_WIDTH + } + + /// Base-field cells per permutation: value columns plus the aux columns + /// at three cells each. + pub const fn cells_per_permutation(self) -> usize { + self.value_columns() + CELLS_PER_AUX_COLUMN * AUX_COLUMNS + } + + /// Constraints: 4 capacity copies, the mode-sum booleanity, then per round + /// two per S-box; Rule adds 16 MDS outputs per round, Compact 16 at the end. + pub const fn num_constraints(self) -> usize { + let mut n = 5; + let mut r = 0; + while r < NUM_ROUNDS { + n += 2 * sboxed_lanes(r); + if let Layout::Rule = self { + n += STATE_FELTS; + } + r += 1; + } + if let Layout::Compact = self { + n += STATE_FELTS; + } + n + } +} + +/// The capacity cell for a domain: `[0, domain, 0, 0]`. +const fn domain_iv(domain: u64) -> [u64; 4] { + [0, domain, 0, 0] +} + +/// The mode a row is filled under. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mode { + /// A 4-ary node: all 16 lanes from the input. + Node, + /// A full permutation. + Permute, + /// A transcript step: rate from the input, the transcript capacity. + Transcript, + /// A leaf block: rate from the input, the leaf capacity. + Leaf, +} + +/// Which MDS the instance uses: Plonky3's small circulant (the measured +/// default) or the paper's Grain Cauchy matrix (the alternative left open for +/// the MDS decision). The layouts are the same — the linear layer costs no +/// columns — so only the constants in the constraints differ. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mds { + Circulant, + Cauchy, +} + +impl Mds { + /// `M[o][i]`. + pub const fn entry(self, o: usize, i: usize) -> u64 { + match self { + Mds::Circulant => MDS_CIRC_ROW[(i + STATE_FELTS - o) % STATE_FELTS], + Mds::Cauchy => CAUCHY_MDS[o][i], + } + } + + /// The host reference's linear layer for this MDS. + fn apply(self, s: &[FE; STATE_FELTS]) -> [FE; STATE_FELTS] { + match self { + Mds::Circulant => p1::mds(s), + Mds::Cauchy => p1::mds_cauchy(s), + } + } +} + +/// A row for `input` under `mode`, witnessed in `layout` for the instance with +/// `mds`. Lanes the mode takes from its capacity are overwritten; the filler +/// walks the rounds itself rather than delegating to [`p1::permute`], and the +/// tests pin `OUT` to it. +pub fn fill_row(layout: Layout, mds: Mds, mode: Mode, input: [FE; STATE_FELTS]) -> Vec { + let mut row = vec![FE::zero(); layout.num_columns()]; + let sel = match mode { + Mode::Node => MODE_C, + Mode::Permute => MODE_P, + Mode::Transcript => MODE_T, + Mode::Leaf => MODE_L, + }; + row[sel] = FE::one(); + let mut state = input; + match mode { + Mode::Node | Mode::Permute => {} + Mode::Transcript | Mode::Leaf => { + let domain = if mode == Mode::Leaf { + DOMAIN_LEAF + } else { + DOMAIN_TRANSCRIPT + }; + for (k, v) in domain_iv(domain).iter().enumerate() { + state[RATE_FELTS + k] = FE::from(*v); + } + } + } + let read_lanes = match mode { + Mode::Node | Mode::Permute => STATE_FELTS, + Mode::Transcript | Mode::Leaf => RATE_FELTS, + }; + row[IN0..IN0 + read_lanes].copy_from_slice(&state[..read_lanes]); + row[S12..S12 + 4].copy_from_slice(&state[RATE_FELTS..]); + + let mut s = state; + for r in 0..NUM_ROUNDS { + let a: [FE; STATE_FELTS] = core::array::from_fn(|i| s[i] + FE::from(ROUND_CONSTANTS[r][i])); + let mut f = a; + for lane in 0..sboxed_lanes(r) { + let x2 = a[lane] * a[lane]; + let x3 = x2 * a[lane]; + let x7 = x3 * x3 * a[lane]; + match layout { + Layout::Rule => { + row[layout.w1(r, lane)] = x2; + row[layout.w2(r, lane)] = x3; + } + Layout::Compact => { + row[layout.w1(r, lane)] = x3; + row[layout.w2(r, lane)] = x7; + } + } + f[lane] = x7; + } + s = mds.apply(&f); + if layout == Layout::Rule && r + 1 < NUM_ROUNDS { + for j in 0..STATE_FELTS { + row[layout.out(r, j)] = s[j]; + } + } + } + row[OUT0..OUT0 + STATE_FELTS].copy_from_slice(&s); + row +} + +/// An affine form over the row: `Σ col[c]·main(c) + m_coeff·m`, with `m` the +/// mode sum. Dense over the columns; the chip is under a thousand wide. +#[derive(Clone)] +struct Affine { + col: Vec, + m: FE, +} + +impl Affine { + fn zero(width: usize) -> Self { + Self { + col: vec![FE::zero(); width], + m: FE::zero(), + } + } + + fn column(width: usize, c: usize) -> Self { + let mut a = Self::zero(width); + a.col[c] = FE::one(); + a + } + + fn add_scaled(&mut self, other: &Affine, k: FE) { + for (x, y) in self.col.iter_mut().zip(&other.col) { + *x += k * y; + } + self.m += k * other.m; + } + + fn expr>(&self, b: &B, m: &B::Expr) -> B::Expr { + let mut acc = b.const_base(canon(&self.m)) * m.clone(); + for (c, k) in self.col.iter().enumerate() { + if *k != FE::zero() { + acc = acc + b.const_base(canon(k)) * b.main(0, c); + } + } + acc + } +} + +fn canon(x: &FE) -> u64 { + GoldilocksField::canonical(x.value()) +} + +/// The MDS over affine forms: `out_o = Σ_i M[o][i]·f_i`. +fn mds_affine(mds: Mds, f: &[Affine]) -> Vec { + let width = f[0].col.len(); + (0..STATE_FELTS) + .map(|o| { + let mut acc = Affine::zero(width); + for (i, fi) in f.iter().enumerate() { + acc.add_scaled(fi, FE::from(mds.entry(o, i))); + } + acc + }) + .collect() +} + +/// The width-16 constraint set under a layout, for the instance with `mds`. +pub struct P1W16Constraints { + pub layout: Layout, + pub mds: Mds, +} + +impl ConstraintSet for P1W16Constraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let layout = self.layout; + let width = layout.num_columns(); + let mode_c = b.main(0, MODE_C); + let mode_p = b.main(0, MODE_P); + let mode_t = b.main(0, MODE_T); + let mode_l = b.main(0, MODE_L); + let m = mode_c.clone() + mode_p.clone() + mode_t.clone() + mode_l.clone(); + + // idx 0–3: S_k = (MODE_C + MODE_P)·IN_{12+k} + MODE_T·IVT_k + MODE_L·IVL_k. + for k in 0..4 { + let s = b.main(0, S12 + k); + let in_k = b.main(0, IN0 + RATE_FELTS + k); + let mut rhs = (mode_c.clone() + mode_p.clone()) * in_k; + for (sel, domain) in [(&mode_t, DOMAIN_TRANSCRIPT), (&mode_l, DOMAIN_LEAF)] { + let iv = domain_iv(domain)[k]; + if iv != 0 { + rhs = rhs + sel.clone() * b.const_base(iv); + } + } + b.emit_base(k, s - rhs); + } + // idx 4: the mode sum is a bit. + let one = b.one(); + b.emit_base(4, m.clone() * (one - m.clone())); + + let mut idx = 5; + // The state entering round 0: IN for the rate, S for the capacity. + let mut state: Vec = (0..STATE_FELTS) + .map(|i| { + let c = if i < RATE_FELTS { + IN0 + i + } else { + S12 + (i - RATE_FELTS) + }; + Affine::column(width, c) + }) + .collect(); + for (r, rc_row) in ROUND_CONSTANTS.iter().enumerate() { + // a_i = state_i + rc[r][i]·m. + let a: Vec = state + .iter() + .zip(rc_row) + .map(|(s, rc)| { + let mut x = s.clone(); + x.m += FE::from(*rc); + x + }) + .collect(); + let mut f: Vec = a.clone(); + for lane in 0..sboxed_lanes(r) { + let a_e = a[lane].expr(b, &m); + let w1 = b.main(0, layout.w1(r, lane)); + let w2 = b.main(0, layout.w2(r, lane)); + match layout { + Layout::Rule => { + // x2 = a², x3 = x2·a; the S-box output (x3)²·a is an + // expression entering the MDS constraint below. + b.emit_base(idx, w1.clone() - a_e.clone() * a_e.clone()); + b.emit_base(idx + 1, w2 - w1 * a_e); + } + Layout::Compact => { + // x3 = a³, x7 = (x3)²·a — both degree 3. + b.emit_base(idx, w1.clone() - a_e.clone() * a_e.clone() * a_e.clone()); + b.emit_base(idx + 1, w2 - w1.clone() * w1 * a_e); + f[lane] = Affine::column(width, layout.w2(r, lane)); + } + } + idx += 2; + } + match layout { + Layout::Rule => { + // out_o = MDS(f)_o, with f_lane = (x3)²·a on S-boxed lanes. + for o in 0..STATE_FELTS { + let mut acc = b.zero(); + for (i, fi) in f.iter().enumerate() { + let c = b.const_base(self.mds.entry(o, i)); + let term = if i < sboxed_lanes(r) { + let x3 = b.main(0, layout.w2(r, i)); + x3.clone() * x3 * a[i].expr(b, &m) + } else { + fi.expr(b, &m) + }; + acc = acc + c * term; + } + let out = b.main(0, layout.out(r, o)); + b.emit_base(idx, out - acc); + idx += 1; + } + state = (0..STATE_FELTS) + .map(|j| Affine::column(width, layout.out(r, j))) + .collect(); + } + Layout::Compact => { + state = mds_affine(self.mds, &f); + } + } + } + if layout == Layout::Compact { + for (j, s) in state.iter().enumerate() { + let out = b.main(0, OUT0 + j); + b.emit_base(idx, out - s.expr(b, &m)); + idx += 1; + } + } + debug_assert_eq!(idx, layout.num_constraints()); + } +} diff --git a/prover/src/lfm/p1w16_chip_tests.rs b/prover/src/lfm/p1w16_chip_tests.rs new file mode 100644 index 000000000..2c3ce9919 --- /dev/null +++ b/prover/src/lfm/p1w16_chip_tests.rs @@ -0,0 +1,408 @@ +//! The width-16 measurement chip: its two widths, its degree bound, what it +//! accepts and what it rejects. +//! +//! The permutation is pinned in `crypto::hash::poseidon1_w16` (vectors agreed +//! by four implementations); here the chip's rows are pinned to that host +//! reference, and every value column is shown load-bearing — under both MDS +//! options (Plonky3's circulant and the Grain Cauchy alternative). + +use math::field::element::FieldElement; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, ProverEvalFolder, RootKind, num_base_from_meta, +}; +use stark::frame::Frame; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; +use crypto::hash::poseidon1_w16 as p1; + +use super::p1w16_chip::*; + +type Gl = GoldilocksField; +type Gl3 = GoldilocksExtension; + +const LAYOUTS: [Layout; 2] = [Layout::Rule, Layout::Compact]; +const MDSS: [Mds; 2] = [Mds::Circulant, Mds::Cauchy]; +const MODES: [Mode; 4] = [Mode::Node, Mode::Permute, Mode::Transcript, Mode::Leaf]; + +/// Pinned as literals, from the closed forms in the module header, so a layout +/// bug cannot agree with itself. +/// Rule: 36 + 7·48 + 32 + 22·18. Compact: 36 + 8·32 + 22·2. +const RULE_VALUE_COLUMNS: usize = 800; +const COMPACT_VALUE_COLUMNS: usize = 336; +/// Plus four LogUp aux columns at three cells each. +const RULE_CELLS: usize = 812; +const COMPACT_CELLS: usize = 348; +/// 5 + 8·32 + 22·2 + 30·16, and 5 + 8·32 + 22·2 + 16. +const RULE_CONSTRAINTS: usize = 785; +const COMPACT_CONSTRAINTS: usize = 321; + +fn evaluate(layout: Layout, mds: Mds, row: &[FE]) -> Vec { + let set = P1W16Constraints { layout, mds }; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out +} + +fn violations(layout: Layout, mds: Mds, row: &[FE]) -> Vec { + evaluate(layout, mds, row) + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +fn sample_input() -> [FE; p1::STATE_FELTS] { + core::array::from_fn(|i| FE::from(0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1))) +} + +#[test] +fn the_widths_are_the_closed_forms() { + assert_eq!(Layout::Rule.value_columns(), RULE_VALUE_COLUMNS); + assert_eq!(Layout::Compact.value_columns(), COMPACT_VALUE_COLUMNS); + assert_eq!(RULE_VALUE_COLUMNS, 36 + 7 * 48 + 32 + 22 * 18); + assert_eq!(COMPACT_VALUE_COLUMNS, 36 + 8 * 32 + 22 * 2); + assert_eq!(Layout::Rule.cells_per_permutation(), RULE_CELLS); + assert_eq!(Layout::Compact.cells_per_permutation(), COMPACT_CELLS); + assert_eq!(Layout::Rule.num_constraints(), RULE_CONSTRAINTS); + assert_eq!(Layout::Compact.num_constraints(), COMPACT_CONSTRAINTS); +} + +/// Every value column is handed out exactly once (the Rule layout's last-round +/// output IS `OUT`, asserted as an alias). +#[test] +fn every_layout_assigns_every_column_exactly_once() { + for layout in LAYOUTS { + let mut seen = vec![0usize; layout.num_columns()]; + let prefix = [(IN0, p1::STATE_FELTS), (S12, 4), (OUT0, p1::STATE_FELTS)]; + for (start, len) in prefix { + for n in &mut seen[start..start + len] { + *n += 1; + } + } + for r in 0..p1::NUM_ROUNDS { + for lane in 0..sboxed_lanes(r) { + seen[layout.w1(r, lane)] += 1; + seen[layout.w2(r, lane)] += 1; + } + if layout == Layout::Rule { + if r + 1 < p1::NUM_ROUNDS { + for j in 0..p1::STATE_FELTS { + seen[layout.out(r, j)] += 1; + } + } else { + assert_eq!(layout.out(r, 0), OUT0); + } + } + } + for (c, &n) in seen.iter().enumerate() { + let want = usize::from(c >= PREP_WIDTH); + assert_eq!(n, want, "{layout:?}: column {c} claimed {n} times"); + } + } +} + +#[test] +fn every_constraint_is_degree_three_or_less_and_some_reach_three() { + for (layout, mds) in LAYOUTS.into_iter().flat_map(|l| MDSS.map(|m| (l, m))) { + let set = P1W16Constraints { layout, mds }; + let meta = ConstraintSet::::meta(&set); + assert_eq!(meta.len(), layout.num_constraints(), "{layout:?}"); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "{layout:?}: meta dense and ordered"); + assert_eq!(m.kind, RootKind::Base); + } + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert_eq!(degrees.len(), meta.len()); + assert!(degrees.iter().all(|&(_, d)| d <= 3), "{layout:?}"); + assert_eq!(degrees.iter().map(|&(_, d)| d).max(), Some(3), "{layout:?}"); + } +} + +/// An honest row satisfies every constraint in every mode, and its `OUT` is +/// the host reference's permutation of the state the mode builds. +#[test] +fn honest_rows_satisfy_every_constraint_and_match_the_host_reference() { + for (layout, mds) in LAYOUTS.into_iter().flat_map(|l| MDSS.map(|m| (l, m))) { + for mode in MODES { + let input = sample_input(); + let row = fill_row(layout, mds, mode, input); + assert_eq!( + violations(layout, mds, &row), + Vec::::new(), + "{layout:?} {mds:?} {mode:?}" + ); + let mut state = input; + if matches!(mode, Mode::Transcript | Mode::Leaf) { + let domain = if mode == Mode::Leaf { + p1::DOMAIN_LEAF + } else { + DOMAIN_TRANSCRIPT + }; + for (k, v) in [0, domain, 0, 0].iter().enumerate() { + state[p1::RATE_FELTS + k] = FE::from(*v); + } + } + let want = match mds { + Mds::Circulant => p1::permute(state), + Mds::Cauchy => p1::permute_cauchy(state), + }; + assert_eq!( + row[OUT0..OUT0 + p1::STATE_FELTS].to_vec(), + want.to_vec(), + "{layout:?} {mds:?} {mode:?}" + ); + } + } +} + +/// The chip reproduces the first known-answer vector end to end. +#[test] +fn a_permutation_row_reproduces_the_first_known_answer() { + const KAT0_OUT_LANE0: u64 = 9_350_316_517_402_464_675; + const CAUCHY_KAT0_OUT_LANE0: u64 = 7_675_469_683_446_723_213; + for layout in LAYOUTS { + let input: [FE; p1::STATE_FELTS] = core::array::from_fn(|i| FE::from(i as u64)); + let row = fill_row(layout, Mds::Circulant, Mode::Permute, input); + assert_eq!(row[OUT0], FE::from(KAT0_OUT_LANE0), "{layout:?}"); + let row = fill_row(layout, Mds::Cauchy, Mode::Permute, input); + assert_eq!( + row[OUT0], + FE::from(CAUCHY_KAT0_OUT_LANE0), + "{layout:?} cauchy" + ); + } +} + +/// A row filled under one MDS does not satisfy the other's constraints — the +/// constants are load-bearing, so the two instances are not interchangeable. +#[test] +fn a_row_under_one_mds_fails_the_other() { + for layout in LAYOUTS { + let row = fill_row(layout, Mds::Circulant, Mode::Permute, sample_input()); + assert!( + !violations(layout, Mds::Cauchy, &row).is_empty(), + "{layout:?}" + ); + let row = fill_row(layout, Mds::Cauchy, Mode::Permute, sample_input()); + assert!( + !violations(layout, Mds::Circulant, &row).is_empty(), + "{layout:?}" + ); + } +} + +#[test] +fn an_all_zero_padding_row_satisfies_every_constraint() { + for (layout, mds) in LAYOUTS.into_iter().flat_map(|l| MDSS.map(|m| (l, m))) { + let row = vec![FE::zero(); layout.num_columns()]; + assert_eq!( + violations(layout, mds, &row), + Vec::::new(), + "{layout:?} {mds:?}" + ); + } +} + +/// Mutation: perturbing ANY value column of an honest permutation row breaks +/// some constraint, so no column is free (a layout that forgot to constrain +/// one would pass every test above). +#[test] +fn every_value_column_is_load_bearing() { + for (layout, mds) in LAYOUTS.into_iter().flat_map(|l| MDSS.map(|m| (l, m))) { + let row = fill_row(layout, mds, Mode::Permute, sample_input()); + for c in PREP_WIDTH..layout.num_columns() { + let mut bad = row.clone(); + bad[c] += FE::one(); + assert!( + !violations(layout, mds, &bad).is_empty(), + "{layout:?} {mds:?}: column {c} is unconstrained" + ); + } + } +} + +/// A row claiming the wrong mode is rejected: a permutation's capacity under +/// the leaf selector violates the capacity copy. +#[test] +fn a_mode_swap_is_rejected() { + for (layout, mds) in LAYOUTS.into_iter().flat_map(|l| MDSS.map(|m| (l, m))) { + let mut row = fill_row(layout, mds, Mode::Permute, sample_input()); + row[MODE_P] = FE::zero(); + row[MODE_L] = FE::one(); + assert!( + violations(layout, mds, &row).iter().any(|&i| i < 4), + "{layout:?} {mds:?}" + ); + } +} + +/// The evaluation cost the census does not see: hash-consed constraint-program +/// nodes (and multiplications) per row, for both layouts against the RPX and +/// W12 Poseidon chips. Informational (`--nocapture`); asserts only that the +/// programs are non-empty. +#[test] +fn constraint_program_sizes() { + use super::chips::hash::HashConstraints; + use stark::constraint_ir::ir::Op; + + fn size>(set: &S) -> (usize, usize) { + let meta = set.meta(); + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (prog, _) = cb.finish(num_base_from_meta(&meta)); + let muls = prog + .nodes + .iter() + .filter(|o| matches!(o, Op::Mul(..))) + .count(); + (prog.len(), muls) + } + let p1 = |layout, mds| size(&P1W16Constraints { layout, mds }); + for (name, (nodes, muls), cols) in [ + ( + "p1w16 rule", + p1(Layout::Rule, Mds::Circulant), + Layout::Rule.value_columns(), + ), + ( + "p1w16 compact", + p1(Layout::Compact, Mds::Circulant), + Layout::Compact.value_columns(), + ), + ( + "p1w16-cauchy rule", + p1(Layout::Rule, Mds::Cauchy), + Layout::Rule.value_columns(), + ), + ( + "p1w16-cauchy compact", + p1(Layout::Compact, Mds::Cauchy), + Layout::Compact.value_columns(), + ), + ("rpx", size(&HashConstraints::RPX), 0), + ] { + assert!(nodes > 0); + println!("PROGRAM {name}: nodes {nodes} muls {muls} value-cols {cols}"); + } +} + +/// What the sub-coset composition evaluation saves on the card, per kernel. +/// +/// Times the fused composition evaluation (`try_eval_composition_gpu`, host +/// mode: the kernel plus the `H` download) of the production hash socket's own +/// constraints — the widest leaf table's program — over a leaf-height LDE at +/// blowup 4 (`2^18` trace rows, `2^20` LDE rows of random columns), at stride 1 +/// (the whole coset) and stride 2 (the `2n`-point sub-coset the degree-2 split +/// needs). Median of 7 after a warm-up each. The ratio is the mechanism row of +/// the sub-coset change: ≈ 0.5 if the walk is per row and nothing else binds. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "card: times the socket's composition evaluation at strides 1 and 2; run on the box with --ignored"] +fn socket_composition_subcoset_time_on_the_card() { + use std::sync::Arc; + use std::time::Instant; + + use math_cuda::lde::{GpuLdeBase, GpuLdeExt3}; + use stark::constraint_ir::gpu_interp::{CompositionInputs, try_eval_composition_gpu}; + + use super::p1w16_socket::{P1W16SocketConstraints, SOCKET_FORM}; + + const BLOWUP: usize = 4; + const LOG_LDE: u32 = 20; + const REPS: usize = 7; + let n = 1usize << LOG_LDE; + + let set = P1W16SocketConstraints { form: SOCKET_FORM }; + let width = SOCKET_FORM.num_columns(); + let meta = set.meta(); + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (prog, _) = cb.finish(num_base_from_meta(&meta)); + let main = GpuLdeBase { + ready: None, + buf: Arc::new(math_cuda::p1w16::random_device_matrix(width * n).expect("card")), + m: width, + lde_size: n, + tree: None, + trace_dev: None, + trace_rows: 0, + }; + let aux = GpuLdeExt3 { + ready: None, + buf: Arc::new(math_cuda::p1w16::random_device_matrix(3).expect("card")), + m: 0, + lde_size: n, + tree: None, + }; + let beta: Vec> = (0..prog.roots.len()) + .map(|i| FieldElement::::from(0x9E37_79B9 + 7 * i as u64)) + .collect(); + let z_inv: Vec = (0..BLOWUP).map(|i| FE::from(3 + i as u64)).collect(); + let zero = FieldElement::::zero(); + let time = |row_stride: usize| -> (f64, Vec) { + let inputs = CompositionInputs:: { + beta_trans: &beta, + z_inv: &z_inv, + b_col: &[], + b_is_aux: &[], + b_value: &[], + b_beta: &[], + b_z_inv: &[], + row_stride, + }; + let run = || match try_eval_composition_gpu( + &prog, + &main, + &aux, + &[], + &[], + &zero, + BLOWUP, + n, + &inputs, + false, + ) { + Some(stark::constraint_ir::gpu_interp::GpuComposition::Host(raw)) => raw, + _ => panic!("the card evaluates the program at stride {row_stride}"), + }; + let h = run(); // warm-up: the lowering cache, the module, clocks + let mut t: Vec = (0..REPS) + .map(|_| { + let start = Instant::now(); + run(); + start.elapsed().as_secs_f64() * 1e3 + }) + .collect(); + t.sort_by(|a, b| a.partial_cmp(b).expect("finite")); + (t[REPS / 2], h) + }; + let (t1, h1) = time(1); + let (t2, h2) = time(2); + // The strided evaluation is the whole one's even rows (a correctness check + // on real program shape, beside the timing). + let even: Vec = (0..n / 2) + .flat_map(|j| h1[j * 6..j * 6 + 3].iter().copied()) + .collect(); + assert_eq!(h2, even, "stride-2 H must be the whole H's even rows"); + println!( + "SUBCOSET KERNEL socket cols {width} nodes {} roots {} · LDE 2^{LOG_LDE} at blowup {BLOWUP} · \ + stride 1 {t1:.2} ms · stride 2 {t2:.2} ms · ratio {:.3}", + prog.len(), + prog.roots.len(), + t2 / t1 + ); +} diff --git a/prover/src/lfm/p1w16_emit.rs b/prover/src/lfm/p1w16_emit.rs new file mode 100644 index 000000000..a18f8feaa --- /dev/null +++ b/prover/src/lfm/p1w16_emit.rs @@ -0,0 +1,771 @@ +//! ZisK's Poseidon1 constructions as emitted programs — what a level-0 program +//! runs to verify a Poseidon1 base proof (P3a). +//! +//! Every construction here mirrors a host function the base prover and its +//! verifier already run, and each is pinned to it by an execution test +//! (`p1w16_emit_tests`): +//! +//! | here | host | +//! |---|---| +//! | [`leaf_hash`] | `poseidon1_stark::linear_hash` (ZisK's chain with the width tag) | +//! | [`node4`], [`walk4`], [`tree_root4`] | `poseidon1_w16::compress4`, `Merkle4` (zero-digest padding) | +//! | [`P1SpongeVar`] | `poseidon1_stark::Transcript` under `p1_commit::P1Transcript`'s encodings | +//! | [`w8_permute`] | `poseidon1_w8::permute` (the grinding permutation) | +//! +//! The width-16 permutation runs on the socket (`Instr::Hash16`); the width-8 +//! one, needed only by the grinding check, is emulated with ALU instructions +//! (two per check, ≈ 2,300 instructions each). +//! +//! # A 4-ary level's hints +//! +//! ZisK's path lists a level's three siblings in child order. The walk takes them +//! in a different order, which the host arranges when it fills the arena +//! ([`hint_order`]): the current node's PARTNER (the other child of its pair), +//! then the OTHER PAIR in child order. Then the bit `b0` orders the current pair +//! with one `Select` per digest cell and `b1` orders the two pairs with two, all +//! outputs used. Hint order is the prover's to choose; the root equality is what +//! binds every sibling. +//! +//! An odd-depth tree's top level holds two real children and two zero digests +//! (`Merkle4`'s padding). There the index's phantom bit is zero, the walk takes +//! ONE hint (the partner), and the two padding children are constant zero cells, +//! not hints, so they cannot be anything else. +//! +//! [`walk4_child`] takes the siblings in ZisK's child order instead, as the +//! proof carries them, and spends two more `Select`s a full level placing +//! them: its arena needs no leaf index (the WHIR leaf's, whose arena is filled +//! from the proof alone). +//! +//! [`hint_path_rotated`] keeps the arena in child order and [`walk4`]'s three +//! `Select`s: the EXECUTOR reads each level's hints in hint order, choosing the +//! arena word on the level's `b1` (`Instr::Hint`'s alternative index). Which +//! word a hint reads is witness, so this binds nothing the walk's `Select`s do +//! not. + +use crate::tables::types::FE; +use crypto::hash::poseidon1_w8 as w8; + +use super::builder::{Bit, Cell, Felt, LfmBuilder}; +use super::edsl::WrapDigest; +use super::instr::ArenaId; + +/// Felts of rate in one width-16 block. +pub const RATE_FELTS: usize = 12; + +fn zero_cell(b: &mut LfmBuilder) -> Cell { + b.digest_const([FE::zero(); 4]).as_cell() +} + +/// A lane the sponge or a leaf packs: a program constant or a machine felt. +#[derive(Clone, Copy, Debug)] +pub enum Lane { + Const(FE), + Var(Felt), +} + +/// Four lanes as one word: an interned constant when every lane is one, else a +/// `Pack` with the constant lanes interned. +fn pack(b: &mut LfmBuilder, lanes: [Lane; 4]) -> Cell { + if let [ + Lane::Const(a), + Lane::Const(c), + Lane::Const(d), + Lane::Const(e), + ] = lanes + { + return b.digest_const([a, c, d, e]).as_cell(); + } + let felts = lanes.map(|l| match l { + Lane::Const(v) => b.felt_const(v), + Lane::Var(f) => f, + }); + b.pack_word(felts) +} + +/// Up to twelve lanes as the three rate words, zero-filled. +fn rate_cells(b: &mut LfmBuilder, lanes: &[Lane]) -> [Cell; 3] { + debug_assert!(lanes.len() <= RATE_FELTS); + core::array::from_fn(|k| { + let word: [Lane; 4] = core::array::from_fn(|j| { + lanes + .get(4 * k + j) + .copied() + .unwrap_or(Lane::Const(FE::zero())) + }); + pack(b, word) + }) +} + +/// The P1 leaf hash over `felts` (`poseidon1_stark::linear_hash`): blocks of +/// twelve (the last zero-filled), the capacity the width tag +/// `[len, LEAF_DOMAIN, 0, 0]` (a program constant: the leaf's width is program +/// shape) for the first block and the previous output's cell 0 for every later +/// one; the digest is the last output's cell 0. No felts hash to the zero +/// digest, as on the host. +pub fn leaf_hash(b: &mut LfmBuilder, felts: &[Felt]) -> WrapDigest { + if felts.is_empty() { + return WrapDigest::from_cell(zero_cell(b)); + } + let tag = crypto::hash::poseidon1_stark::leaf_capacity(felts.len()); + let mut carry = b.digest_const(tag).as_cell(); + for block in felts.chunks(RATE_FELTS) { + let lanes: Vec = block.iter().map(|f| Lane::Var(*f)).collect(); + let [c0, c1, c2] = rate_cells(b, &lanes); + carry = b.hash16([c0, c1, c2, carry])[0]; + } + WrapDigest::from_cell(carry) +} + +/// [`leaf_hash`] over `felts` felts arriving as WORDS, four a word in order, +/// the last zero-filled: each word is the rate word [`leaf_hash`] would pack +/// from the same four felts, so the digest is the same — the same width tag +/// and twelve-felt block schedule, three words a block, constant zero words +/// past the last. A word's lanes past `felts` are hashed as given, so anything +/// but zero there gives a digest no committer computed. +pub fn leaf_hash_words(b: &mut LfmBuilder, words: &[Cell], felts: usize) -> WrapDigest { + assert_eq!( + words.len(), + felts.div_ceil(4), + "four felts a word, the last zero-filled" + ); + if felts == 0 { + return WrapDigest::from_cell(zero_cell(b)); + } + let tag = crypto::hash::poseidon1_stark::leaf_capacity(felts); + let mut carry = b.digest_const(tag).as_cell(); + for block in words.chunks(RATE_FELTS / 4) { + let rate: [Cell; 3] = + core::array::from_fn(|k| block.get(k).copied().unwrap_or_else(|| zero_cell(b))); + carry = b.hash16([rate[0], rate[1], rate[2], carry])[0]; + } + WrapDigest::from_cell(carry) +} + +/// One 4-ary node: the four children, permuted, truncated to cell 0. +pub fn node4(b: &mut LfmBuilder, children: [Cell; 4]) -> Cell { + b.hash16(children)[0] +} + +/// Hints a path of `nbits` index bits takes: three per full 4-ary level, one at +/// an odd-depth top level. +pub const fn path_hints(nbits: usize) -> usize { + 3 * (nbits / 2) + nbits % 2 +} + +/// Walk one 4-ary Merkle path: `bits` the leaf index low first, `hints` in +/// [`hint_order`] (partner, then the other pair, per level; one at an odd top). +pub fn walk4( + b: &mut LfmBuilder, + leaf: WrapDigest, + bits: &[Bit], + hints: &[WrapDigest], +) -> WrapDigest { + assert_eq!( + hints.len(), + path_hints(bits.len()), + "three hints per 4-ary level, one at an odd top" + ); + let mut cur = leaf.cells()[0]; + let mut h = hints.iter().map(|d| d.cells()[0]); + for pair in bits.chunks(2) { + let partner = h.next().expect("counted above"); + let (l, r) = b.select(pair[0], cur, partner); + cur = match pair { + [_, b1] => { + let u0 = h.next().expect("counted above"); + let u1 = h.next().expect("counted above"); + let (c0, c2) = b.select(*b1, l, u0); + let (c1, c3) = b.select(*b1, r, u1); + node4(b, [c0, c1, c2, c3]) + } + _ => { + // The odd top: two real children and two zero digests. + let z = zero_cell(b); + node4(b, [l, r, z, z]) + } + }; + } + WrapDigest::from_cell(cur) +} + +/// Walk one 4-ary Merkle path whose hints are in CHILD order: per full level +/// the three siblings as ZisK's path lists them (the children other than the +/// current one, in child order), at an odd-depth top only the partner (the +/// first sibling; the two padding children are constant zeros, never hints). +/// +/// With the current node at child `p = b0 + 2·b1`, the partner is sibling 0 +/// when `b1 = 0` and sibling 2 when `b1 = 1`, and the other pair is siblings +/// `(1, 2)` or `(0, 1)`: hint order is the siblings rotated by one place when +/// `b1 = 1`, two `Select`s on `b1` using both of their outputs — `(partner, y)` +/// from `(s0, s2)`, then the other pair from `(s1, y)` — and [`walk4`]'s level +/// follows. Every placement is a `Select` on an index bit, so the node hashed +/// is the one the index names, whatever the hints. +pub fn walk4_child( + b: &mut LfmBuilder, + leaf: WrapDigest, + bits: &[Bit], + siblings: &[WrapDigest], +) -> WrapDigest { + assert_eq!( + siblings.len(), + path_hints(bits.len()), + "three siblings per 4-ary level, one at an odd top" + ); + let mut cur = leaf.cells()[0]; + let mut s = siblings.iter().map(|d| d.cells()[0]); + for pair in bits.chunks(2) { + cur = match pair { + [b0, b1] => { + let s0 = s.next().expect("counted above"); + let s1 = s.next().expect("counted above"); + let s2 = s.next().expect("counted above"); + // b1 = 0: (s0, s2), then (s1, s2); b1 = 1: (s2, s0), then (s0, s1). + let (partner, y) = b.select(*b1, s0, s2); + let (u0, u1) = b.select(*b1, s1, y); + let (l, r) = b.select(*b0, cur, partner); + let (c0, c2) = b.select(*b1, l, u0); + let (c1, c3) = b.select(*b1, r, u1); + node4(b, [c0, c1, c2, c3]) + } + _ => { + // The odd top: the partner, then two zero digests. + let partner = s.next().expect("counted above"); + let (l, r) = b.select(pair[0], cur, partner); + let z = zero_cell(b); + node4(b, [l, r, z, z]) + } + }; + } + WrapDigest::from_cell(cur) +} + +/// A 4-ary tree's root over `leaves`, each level padded with zero digests to a +/// multiple of four (`Merkle4`). One leaf is its own root. +pub fn tree_root4(b: &mut LfmBuilder, leaves: &[WrapDigest]) -> WrapDigest { + assert!(!leaves.is_empty(), "a tree has at least one leaf"); + let mut level: Vec = leaves.iter().map(|d| d.cells()[0]).collect(); + while level.len() > 1 { + let z = zero_cell(b); + level.resize(level.len().div_ceil(4) * 4, z); + level = level + .chunks_exact(4) + .map(|q| node4(b, [q[0], q[1], q[2], q[3]])) + .collect(); + } + WrapDigest::from_cell(level[0]) +} + +/// Hints a 4-ary path that `arena` holds in CHILD order from word `at` (as the +/// proof carries it, [`walk4_child`]'s layout) in [`hint_order`], for +/// [`walk4`] over `bits`: per full level the executor reads the siblings +/// `(s0, s1, s2)` when the level's `b1` is 0 and `(s2, s0, s1)` when it is 1 +/// (the partner, then the other pair); at an odd top the one word, the +/// partner either way. The bits must be written before the hints. +pub fn hint_path_rotated( + b: &mut LfmBuilder, + arena: ArenaId, + at: u32, + bits: &[Bit], +) -> Vec { + let mut hints = Vec::with_capacity(path_hints(bits.len())); + let mut word = at; + for pair in bits.chunks(2) { + match pair { + [_, b1] => { + for k in 0..3 { + let cell = b.hint_word_on(arena, word + k, *b1, word + (k + 2) % 3); + hints.push(WrapDigest::from_cell(cell)); + } + word += 3; + } + _ => { + hints.push(WrapDigest::from_cell(b.hint_word(arena, word))); + word += 1; + } + } + } + hints +} + +/// Host side: ZisK's path for leaf `index` (per level, its three siblings in +/// child order) as the walk's hints, leaf level first. At an odd-depth top +/// (`depth_bits` odd) only the partner is kept: the other two are the zero +/// padding the walk supplies itself. +pub fn hint_order(index: usize, depth_bits: usize, path: &[[D; 3]]) -> Vec { + assert_eq!( + path.len(), + depth_bits.div_ceil(2), + "one sibling triple per 4-ary level" + ); + let mut out = Vec::with_capacity(path_hints(depth_bits)); + let mut i = index; + for (level, sib) in path.iter().enumerate() { + let p = i % 4; + // The children other than `p`, in child order, as ZisK lists them. + let child = |c: usize| -> D { + debug_assert_ne!(c, p); + sib[if c < p { c } else { c - 1 }] + }; + out.push(child(p ^ 1)); + let odd_top = depth_bits % 2 == 1 && level + 1 == path.len(); + if !odd_top { + let other = 2 * (1 - p / 2); + out.push(child(other)); + out.push(child(other + 1)); + } + i /= 4; + } + out +} + +/// ZisK's transcript (`poseidon1_stark::Transcript`) as an emitted sponge. +/// +/// The state is the last permutation's four output cells (zero before the +/// first). Absorbed lanes collect in a buffer of up to twelve; a full buffer, +/// or a squeeze after an absorb, permutes `buffer (zero-filled) ‖ state[0]`. A +/// squeeze reads the output's lanes 0, 1, …, 15 in order and permutes again +/// once all sixteen are read. As on the host, every absorb sets the unread +/// count to zero BEFORE a full buffer flushes, so a squeeze right after exactly +/// twelve absorbs reads the fresh output. +#[derive(Clone, Debug)] +pub struct P1SpongeVar { + state: [Cell; 4], + lanes: [Option<[Felt; 4]>; 4], + pending: Vec, + unread: usize, +} + +impl P1SpongeVar { + /// The empty transcript. + pub fn new(b: &mut LfmBuilder) -> Self { + let z = zero_cell(b); + Self { + state: [z; 4], + lanes: [None; 4], + pending: Vec::with_capacity(RATE_FELTS), + unread: 0, + } + } + + /// Absorb one lane. + pub fn put(&mut self, b: &mut LfmBuilder, x: Lane) { + self.pending.push(x); + self.unread = 0; + if self.pending.len() == RATE_FELTS { + self.update(b); + } + } + + /// Absorb a machine felt. + pub fn put_felt(&mut self, b: &mut LfmBuilder, f: Felt) { + self.put(b, Lane::Var(f)); + } + + /// Absorb a program constant. + pub fn put_const(&mut self, b: &mut LfmBuilder, v: FE) { + self.put(b, Lane::Const(v)); + } + + fn permute_pending(&self, b: &mut LfmBuilder) -> [Cell; 4] { + let [c0, c1, c2] = rate_cells(b, &self.pending); + b.hash16([c0, c1, c2, self.state[0]]) + } + + fn update(&mut self, b: &mut LfmBuilder) { + self.state = self.permute_pending(b); + self.lanes = [None; 4]; + self.pending.clear(); + self.unread = 16; + } + + /// Squeeze one felt. + pub fn squeeze(&mut self, b: &mut LfmBuilder) -> Felt { + if self.unread == 0 { + self.update(b); + } + let idx = 16 - self.unread; + self.unread -= 1; + let word = idx / 4; + let state = self.state[word]; + let lanes = *self.lanes[word].get_or_insert_with(|| b.unpack(state)); + lanes[idx % 4] + } + + /// The transcript's state digest without advancing it — `P1Transcript::state`, + /// which flushes a COPY: a non-empty buffer costs one permutation whose + /// output is not adopted. + pub fn digest(&self, b: &mut LfmBuilder) -> Cell { + if self.pending.is_empty() { + self.state[0] + } else { + self.permute_pending(b)[0] + } + } +} + +/// `x⁷` in four multiplications. +fn sbox7(b: &mut LfmBuilder, x: Felt) -> Felt { + let x2 = b.mul(x, x); + let x3 = b.mul(x2, x); + let x6 = b.mul(x3, x3); + b.mul(x6, x) +} + +/// The width-8 permutation (`poseidon1_w8::permute`) in ALU instructions: +/// each round's MDS row is a `MulAdd` chain seeded with the NEXT round's +/// constant, so the round-constant additions after round 0 cost nothing. 8 + +/// 8·(32 + 64) + 22·(4 + 64) = 2,272 instructions. +pub fn w8_permute(b: &mut LfmBuilder, state: [Felt; 8]) -> [Felt; 8] { + use w8::constants::{MDS_CIRC_ROW, ROUND_CONSTANTS}; + const T: usize = w8::STATE_FELTS; + let half = w8::HALF_FULL_ROUNDS; + let partial = half..half + w8::PARTIAL_ROUNDS; + let m = |o: usize, i: usize| FE::from(MDS_CIRC_ROW[(i + T - o) % T]); + + let mut a: [Felt; T] = core::array::from_fn(|i| { + let c = b.felt_const(FE::from(ROUND_CONSTANTS[0][i])); + b.add(state[i], c) + }); + for r in 0..w8::NUM_ROUNDS { + let mut f = a; + let lanes = if partial.contains(&r) { 1 } else { T }; + for x in f.iter_mut().take(lanes) { + *x = sbox7(b, *x); + } + a = core::array::from_fn(|o| { + let mut acc = if r + 1 < w8::NUM_ROUNDS { + b.felt_const(FE::from(ROUND_CONSTANTS[r + 1][o])) + } else { + let k = b.felt_const(m(o, 0)); + b.mul(k, f[0]) + }; + let first = usize::from(r + 1 == w8::NUM_ROUNDS); + for (i, fi) in f.iter().enumerate().skip(first) { + let k = b.felt_const(m(o, i)); + acc = b.mul_add(k, *fi, acc); + } + acc + }); + } + a +} + +// ============================ the sparse width-8 ============================ + +/// The width-8 permutation's partial rounds in sparse form, precomputed from +/// the textbook constants ([`sparse_w8`]); [`w8_sparse_host`] is its host +/// form, held to `poseidon1_w8::permute` by test. +/// +/// Two equivalences, both exact over the field: +/// +/// - **Constants.** A partial round adds `c` before an S-box on lane 0 only, +/// so `c`'s lanes 1..8 commute past the S-box and through the round's MDS +/// into the next round's constant. Pushed forward, every partial round adds +/// one scalar to lane 0, and the last one's residue joins the first +/// terminal full round's constant. +/// - **Matrices.** Any `A = [[a, vᵀ], [w, Â]]` with `Â` invertible is +/// `Sp · D` with `D = diag(1, Â)` and `Sp = [[a, (Â⁻ᵀv)ᵀ], [w, I]]`. `D` +/// touches neither lane 0 nor the S-box, so it moves to the round before, +/// whose matrix becomes `D · M`; from the last partial round back, every +/// partial round keeps a sparse `Sp` (15 multiply-adds against the MDS's +/// 64) and the first-half's last full round takes `D₀ · M`, still dense. +/// `Â` is invertible at every step: `M̂` is (every square submatrix of an +/// MDS matrix is), and each later `Â` is a product of invertible ones. +pub struct SparseW8 { + /// Full rounds' constants, `[full round][lane]`: rounds 0–3, then 26–29 + /// (the first of those carrying the partial block's residue). + full_rc: [[FE; W8]; 8], + /// The first half's last full round's matrix, `D₀ · M`. + first_matrix: [[FE; W8]; W8], + /// Per partial round: its lane-0 constant, and its `Sp = [[a, vᵀ], [w, I]]`. + partial_rc: [FE; w8::PARTIAL_ROUNDS], + sp_a: [FE; w8::PARTIAL_ROUNDS], + sp_v: [[FE; W8 - 1]; w8::PARTIAL_ROUNDS], + sp_w: [[FE; W8 - 1]; w8::PARTIAL_ROUNDS], +} + +const W8: usize = w8::STATE_FELTS; + +fn w8_mds() -> [[FE; W8]; W8] { + core::array::from_fn(|i| { + core::array::from_fn(|j| FE::from(w8::constants::MDS_CIRC_ROW[(j + W8 - i) % W8])) + }) +} + +fn mat_mul(a: &[[FE; W8]; W8], b: &[[FE; W8]; W8]) -> [[FE; W8]; W8] { + core::array::from_fn(|i| { + core::array::from_fn(|j| (0..W8).fold(FE::zero(), |acc, k| acc + a[i][k] * b[k][j])) + }) +} + +fn mat_vec(a: &[[FE; W8]; W8], x: &[FE; W8]) -> [FE; W8] { + core::array::from_fn(|i| (0..W8).fold(FE::zero(), |acc, k| acc + a[i][k] * x[k])) +} + +/// The inverse of a 7×7 matrix by Gauss–Jordan; `None` if singular. +fn inverse7(m: &[[FE; W8 - 1]; W8 - 1]) -> Option<[[FE; W8 - 1]; W8 - 1]> { + const N: usize = W8 - 1; + let mut a = *m; + let mut inv: [[FE; N]; N] = core::array::from_fn(|i| { + core::array::from_fn(|j| if i == j { FE::one() } else { FE::zero() }) + }); + for col in 0..N { + let pivot = (col..N).find(|&r| a[r][col] != FE::zero())?; + a.swap(col, pivot); + inv.swap(col, pivot); + let p = a[col][col].inv().ok()?; + for j in 0..N { + a[col][j] *= p; + inv[col][j] *= p; + } + for r in 0..N { + if r != col && a[r][col] != FE::zero() { + let f = a[r][col]; + for j in 0..N { + let (ac, ic) = (a[col][j], inv[col][j]); + a[r][j] = a[r][j] - f * ac; + inv[r][j] = inv[r][j] - f * ic; + } + } + } + } + Some(inv) +} + +/// [`SparseW8`], computed once from `poseidon1_w8`'s constants. +pub fn sparse_w8() -> &'static SparseW8 { + static TABLE: std::sync::OnceLock = std::sync::OnceLock::new(); + TABLE.get_or_init(|| { + let rc = |r: usize| -> [FE; W8] { + core::array::from_fn(|i| FE::from(w8::constants::ROUND_CONSTANTS[r][i])) + }; + let m = w8_mds(); + let half = w8::HALF_FULL_ROUNDS; + let partial = w8::PARTIAL_ROUNDS; + // Constants: push each partial round's lanes 1..8 forward. + let mut partial_rc = [FE::zero(); w8::PARTIAL_ROUNDS]; + let mut carry = rc(half); + for (k, out) in partial_rc.iter_mut().enumerate() { + *out = carry[0]; + let mut rest = carry; + rest[0] = FE::zero(); + let pushed = mat_vec(&m, &rest); + carry = if k + 1 < partial { + let next = rc(half + k + 1); + core::array::from_fn(|i| next[i] + pushed[i]) + } else { + pushed + }; + } + let mut full_rc: [[FE; W8]; 8] = core::array::from_fn(|f| { + rc(if f < half { + f + } else { + half + partial + f - half + }) + }); + for (i, v) in full_rc[half].iter_mut().enumerate() { + *v += carry[i]; + } + // Matrices: from the last partial round back. + let mut sp_a = [FE::zero(); w8::PARTIAL_ROUNDS]; + let mut sp_v = [[FE::zero(); W8 - 1]; w8::PARTIAL_ROUNDS]; + let mut sp_w = [[FE::zero(); W8 - 1]; w8::PARTIAL_ROUNDS]; + let mut a = m; + for k in (0..partial).rev() { + let hat: [[FE; W8 - 1]; W8 - 1] = + core::array::from_fn(|i| core::array::from_fn(|j| a[i + 1][j + 1])); + let hat_inv = + inverse7(&hat).expect("an MDS matrix's square submatrices are invertible"); + sp_a[k] = a[0][0]; + // ṽᵀ = vᵀ · Â⁻¹. + sp_v[k] = core::array::from_fn(|j| { + (0..W8 - 1).fold(FE::zero(), |acc, i| acc + a[0][i + 1] * hat_inv[i][j]) + }); + sp_w[k] = core::array::from_fn(|i| a[i + 1][0]); + let d: [[FE; W8]; W8] = core::array::from_fn(|i| { + core::array::from_fn(|j| match (i, j) { + (0, 0) => FE::one(), + (0, _) | (_, 0) => FE::zero(), + _ => hat[i - 1][j - 1], + }) + }); + a = mat_mul(&d, &m); + } + SparseW8 { + full_rc, + first_matrix: a, + partial_rc, + sp_a, + sp_v, + sp_w, + } + }) +} + +/// The sparse form on the host: what [`w8_permute_lanes`] emits, round for +/// round. Equal to `poseidon1_w8::permute` (`p1w16_emit_tests`). +pub fn w8_sparse_host(input: [FE; W8]) -> [FE; W8] { + let t = sparse_w8(); + let m = w8_mds(); + let half = w8::HALF_FULL_ROUNDS; + let sbox = |x: FE| x * x * x * x * x * x * x; + let mut s = input; + for r in 0..half { + for (x, c) in s.iter_mut().zip(&t.full_rc[r]) { + *x = sbox(*x + *c); + } + s = mat_vec(if r + 1 == half { &t.first_matrix } else { &m }, &s); + } + for k in 0..w8::PARTIAL_ROUNDS { + s[0] = sbox(s[0] + t.partial_rc[k]); + let s0 = s[0]; + let y0 = s[1..] + .iter() + .zip(&t.sp_v[k]) + .fold(t.sp_a[k] * s0, |acc, (x, v)| acc + *v * *x); + for (x, w) in s[1..].iter_mut().zip(&t.sp_w[k]) { + *x += *w * s0; + } + s[0] = y0; + } + for f in half..2 * half { + for (x, c) in s.iter_mut().zip(&t.full_rc[f]) { + *x = sbox(*x + *c); + } + s = mat_vec(&m, &s); + } + s +} + +/// `seed + Σ coeff·lane` in as few rows as the terms allow: constant terms +/// fold into the seed; a variable term of coefficient one starts the chain +/// when the seed is zero; every other variable term is one `Mul`/`MulAdd`. +fn lin(b: &mut LfmBuilder, seed: FE, terms: &[(FE, Lane)]) -> Lane { + let mut seed = seed; + let mut vars: Vec<(FE, Felt)> = Vec::with_capacity(terms.len()); + for &(k, lane) in terms { + match lane { + Lane::Const(v) => seed += k * v, + Lane::Var(f) if k != FE::zero() => vars.push((k, f)), + Lane::Var(_) => {} + } + } + if vars.is_empty() { + return Lane::Const(seed); + } + let mut acc: Option = None; + if seed == FE::zero() + && let Some(at) = vars.iter().position(|(k, _)| *k == FE::one()) + { + acc = Some(vars.remove(at).1); + } + for (k, f) in vars { + let kc = b.felt_const(k); + acc = Some(match acc { + Some(a) => b.mul_add(kc, f, a), + None if seed == FE::zero() => b.mul(kc, f), + None => { + let c = b.felt_const(seed); + b.mul_add(kc, f, c) + } + }); + } + Lane::Var(acc.expect("at least one variable term")) +} + +fn sbox_lane(b: &mut LfmBuilder, x: Lane) -> Lane { + match x { + Lane::Const(v) => Lane::Const(v * v * v * v * v * v * v), + Lane::Var(f) => Lane::Var(sbox7(b, f)), + } +} + +/// ★ The width-8 permutation in its sparse form ([`SparseW8`]), with the +/// constant input lanes folded at emit time and only the first `keep` output +/// lanes computed. Each round's constant seeds the chains of the matrix before +/// it, as [`w8_permute`] does, so after round 0 the constants cost nothing. +/// +/// Rows: a partial round is its S-box (four `Mul`s) and `Sp` (eight for the +/// new lane 0, seven `MulAdd`s for the rest) — 19 against the dense 68 — and +/// a constant input lane costs no S-box and no matrix term in round 0. +pub fn w8_permute_lanes(b: &mut LfmBuilder, input: [Lane; W8], keep: usize) -> Vec { + assert!((1..=W8).contains(&keep), "keep 1..=8 output lanes"); + let t = sparse_w8(); + let m = w8_mds(); + let half = w8::HALF_FULL_ROUNDS; + let partial = w8::PARTIAL_ROUNDS; + // Round 0's constant: added (a variable lane costs one `Add`). + let mut s: [Lane; W8] = core::array::from_fn(|i| match input[i] { + Lane::Const(v) => Lane::Const(v + t.full_rc[0][i]), + Lane::Var(f) if t.full_rc[0][i] == FE::zero() => Lane::Var(f), + Lane::Var(f) => { + let c = b.felt_const(t.full_rc[0][i]); + Lane::Var(b.add(f, c)) + } + }); + // The first half: full rounds, the last one's matrix `D₀ · M`, each seeded + // with the next round's constant (the first partial round's is lane 0's). + for r in 0..half { + let f: [Lane; W8] = core::array::from_fn(|i| sbox_lane(b, s[i])); + let matrix = if r + 1 == half { &t.first_matrix } else { &m }; + s = core::array::from_fn(|o| { + let seed = if r + 1 < half { + t.full_rc[r + 1][o] + } else if o == 0 { + t.partial_rc[0] + } else { + FE::zero() + }; + let terms: Vec<(FE, Lane)> = (0..W8).map(|i| (matrix[o][i], f[i])).collect(); + lin(b, seed, &terms) + }); + } + // The partial rounds: S-box on lane 0, then `Sp`, seeded with the next + // round's constant (after the last, the terminal full round's). + for k in 0..partial { + let s0 = sbox_lane(b, s[0]); + let next: [FE; W8] = if k + 1 < partial { + core::array::from_fn(|i| { + if i == 0 { + t.partial_rc[k + 1] + } else { + FE::zero() + } + }) + } else { + t.full_rc[half] + }; + let mut terms0: Vec<(FE, Lane)> = vec![(t.sp_a[k], s0)]; + terms0.extend((1..W8).map(|j| (t.sp_v[k][j - 1], s[j]))); + let y0 = lin(b, next[0], &terms0); + for j in 1..W8 { + s[j] = lin(b, next[j], &[(FE::one(), s[j]), (t.sp_w[k][j - 1], s0)]); + } + s[0] = y0; + } + // The terminal full rounds, the last computing only `keep` lanes. + for f in half..2 * half { + let x: [Lane; W8] = core::array::from_fn(|i| sbox_lane(b, s[i])); + let last = f + 1 == 2 * half; + let outputs = if last { keep } else { W8 }; + let mut out = s; + for (o, slot) in out.iter_mut().enumerate().take(outputs) { + let seed = if last { + FE::zero() + } else { + t.full_rc[f + 1][o] + }; + let terms: Vec<(FE, Lane)> = (0..W8).map(|i| (m[o][i], x[i])).collect(); + *slot = lin(b, seed, &terms); + } + s = out; + } + s[..keep] + .iter() + .map(|lane| match *lane { + Lane::Const(v) => b.felt_const(v), + Lane::Var(f) => f, + }) + .collect() +} diff --git a/prover/src/lfm/p1w16_emit_tests.rs b/prover/src/lfm/p1w16_emit_tests.rs new file mode 100644 index 000000000..94c88f94c --- /dev/null +++ b/prover/src/lfm/p1w16_emit_tests.rs @@ -0,0 +1,758 @@ +//! The emitted Poseidon1 constructions against the host's: each program is +//! executed (the width-16 socket on the executor's host permutation) and its +//! published words compared with `poseidon1_stark` / `poseidon1_w8`. + +use crate::tables::types::FE; +use crypto::hash::poseidon1_stark::{self as zisk, Merkle4}; +use crypto::hash::{poseidon1_w8, poseidon1_w16}; + +use super::builder::{Felt, LfmBuilder}; +use super::compiler::compile; +use super::edsl::WrapDigest; +use super::executor::{execute, execute_serial}; +use super::hash::HasherKind; +use super::p1w16_emit::*; +use super::word::{LfmWord, base_word}; + +fn felt(seed: u64, i: usize) -> FE { + let mut z = seed + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(i as u64 + 1); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + FE::from(z ^ (z >> 31)) +} + +/// Run `b`'s program over `arenas`; the published words, flattened. +fn run(b: LfmBuilder, arenas: &[Vec]) -> Vec { + let program = compile(b.finish()); + super::validator::validate(&program).expect("admissible"); + let exec = execute_serial(&program, arenas, &HasherKind::Poseidon1W16).expect("executes"); + exec.public_words.iter().flat_map(|(_, w)| *w).collect() +} + +fn hinted_felts(b: &mut LfmBuilder, n: usize) -> Vec { + let a = b.declare_arena(n as u32); + (0..n as u32).map(|i| b.hint_felt(a, i)).collect() +} + +#[test] +fn the_leaf_hash_is_the_tagged_linear_hash() { + for n in [0usize, 1, 4, 11, 12, 13, 24, 25, 36, 41] { + let values: Vec = (0..n).map(|i| felt(n as u64, i)).collect(); + let mut b = LfmBuilder::new(); + let felts = hinted_felts(&mut b, n); + let d = leaf_hash(&mut b, &felts); + b.public(d.cells()[0]); + let got = run(b, &[values.iter().copied().map(base_word).collect()]); + assert_eq!(got, zisk::linear_hash(&values).to_vec(), "{n} felts"); + } +} + +/// `values` four a word, in order, the last word's lanes past them `fill`. +fn packed_words(values: &[FE], fill: FE) -> Vec { + values + .chunks(4) + .map(|lanes| core::array::from_fn(|j| lanes.get(j).copied().unwrap_or(fill))) + .collect() +} + +/// [`leaf_hash_words`] over `words` hinted, published. +fn run_word_leaf(words: Vec, felts: usize) -> Vec { + let mut b = LfmBuilder::new(); + let a = b.declare_arena(words.len() as u32); + let cells: Vec<_> = (0..words.len() as u32).map(|i| b.hint_word(a, i)).collect(); + let d = leaf_hash_words(&mut b, &cells, felts); + b.public(d.cells()[0]); + run(b, &[words]) +} + +/// ★ The leaf hash over values four a word is the same tagged linear hash, at +/// lengths on and off a word and a block; a lane past the last value that is +/// not zero gives another digest. +#[test] +fn the_word_leaf_hash_is_the_tagged_linear_hash() { + for n in [0usize, 1, 2, 3, 4, 5, 11, 12, 13, 16, 24, 25, 36, 41, 64] { + let values: Vec = (0..n).map(|i| felt(n as u64 + 99, i)).collect(); + let host = zisk::linear_hash(&values).to_vec(); + let got = run_word_leaf(packed_words(&values, FE::zero()), n); + assert_eq!(got, host, "{n} felts"); + if n % 4 != 0 { + let junk = run_word_leaf(packed_words(&values, FE::from(1u64)), n); + assert_ne!(junk, host, "{n} felts: a nonzero lane past the last value"); + } + } +} + +fn digest(seed: u64, i: usize) -> [FE; 4] { + core::array::from_fn(|k| felt(seed, 4 * i + k)) +} + +fn sibling_triples(path: &[[FE; zisk::SIBLING_FELTS]]) -> Vec<[[FE; 4]; 3]> { + path.iter() + .map(|s| core::array::from_fn(|j| core::array::from_fn(|k| s[4 * j + k]))) + .collect() +} + +/// Every leaf of trees of even and odd depth (and the one-leaf tree): the walk +/// over the hinted path reaches `Merkle4`'s root, and `tree_root4` builds it. +#[test] +fn the_walk_and_the_tree_reach_merkle4s_root() { + for depth in 0usize..=5 { + let n = 1usize << depth; + let leaves: Vec<[FE; 4]> = (0..n).map(|i| digest(depth as u64 + 7, i)).collect(); + let tree = Merkle4::new(&leaves).expect("non-empty"); + let root = tree.root(); + for index in [0, n / 2, n - 1] { + let hints = hint_order( + index, + depth, + &sibling_triples(&tree.path(index).expect("leaf")), + ); + assert_eq!(hints.len(), path_hints(depth)); + let mut b = LfmBuilder::new(); + let a_leaf = b.declare_arena(1); + let a_idx = b.declare_arena(1); + let a_hint = b.declare_arena(hints.len() as u32); + let leaf = WrapDigest::from_cell(b.hint_word(a_leaf, 0)); + let idx = b.hint_felt(a_idx, 0); + let bits = b.bit_dec(idx, depth); + let h: Vec = (0..hints.len() as u32) + .map(|i| WrapDigest::from_cell(b.hint_word(a_hint, i))) + .collect(); + let walked = walk4(&mut b, leaf, &bits, &h); + b.public(walked.cells()[0]); + let got = run( + b, + &[ + vec![leaves[index]], + vec![base_word(FE::from(index as u64))], + hints.clone(), + ], + ); + assert_eq!(got, root.to_vec(), "depth {depth} index {index}"); + } + + let mut b = LfmBuilder::new(); + let a = b.declare_arena(n as u32); + let ds: Vec = (0..n as u32) + .map(|i| WrapDigest::from_cell(b.hint_word(a, i))) + .collect(); + let r = tree_root4(&mut b, &ds); + b.public(r.cells()[0]); + assert_eq!( + run(b, std::slice::from_ref(&leaves)), + root.to_vec(), + "tree, depth {depth}" + ); + } +} + +/// ZisK's path for a leaf in the WHIR leaf arena's CHILD order: per level its +/// three siblings as listed, at an odd top only the partner (the first). +fn child_order_path(depth: usize, path: &[[FE; zisk::SIBLING_FELTS]]) -> Vec { + let triples = sibling_triples(path); + let mut words = Vec::new(); + for (level, t) in triples.iter().enumerate() { + if depth % 2 == 1 && level + 1 == triples.len() { + words.push(t[0]); + } else { + words.extend_from_slice(t); + } + } + words +} + +/// A 4-ary walk over a path held in child order and hinted by `hint`: the +/// leaf, the index's bits, then `walk4`; the program and the walked root. +fn rotated_walk( + depth: usize, + index: usize, + leaf: LfmWord, + path: Vec, + hint: impl Fn(&mut LfmBuilder, super::instr::ArenaId, &[super::builder::Bit]) -> Vec, +) -> (super::compiler::LfmProgram, Vec) { + let mut b = LfmBuilder::new(); + let a_leaf = b.declare_arena(1); + let a_idx = b.declare_arena(1); + let a_path = b.declare_arena(path.len() as u32); + let leaf_cell = WrapDigest::from_cell(b.hint_word(a_leaf, 0)); + let idx = b.hint_felt(a_idx, 0); + let bits = b.bit_dec(idx, depth); + let hints = hint(&mut b, a_path, &bits); + let walked = walk4(&mut b, leaf_cell, &bits, &hints); + b.public(walked.cells()[0]); + let program = compile(b.finish()); + super::validator::validate(&program).expect("admissible"); + let arenas = [vec![leaf], vec![base_word(FE::from(index as u64))], path]; + let serial = execute_serial(&program, &arenas, &HasherKind::Poseidon1W16).expect("executes"); + let levels = execute(&program, &arenas, &HasherKind::Poseidon1W16).expect("executes"); + assert_eq!(serial.public_words, levels.public_words, "both schedules"); + let root = serial.public_words.iter().flat_map(|(_, w)| *w).collect(); + (program, root) +} + +/// ★ The executor's rotation turns a child-order path into `walk4`'s hint +/// order: every leaf of trees of even and odd depth reaches `Merkle4`'s root +/// with three `Select`s a full level (one at an odd top), and with the +/// rotation swapped (`alt` on a clear bit) no leaf below a full level does. +#[test] +fn the_rotated_hints_walk_a_child_order_path_to_merkle4s_root() { + let count = |p: &super::compiler::LfmProgram, f: fn(&super::instr::Instr) -> bool| { + p.instrs.iter().filter(|i| f(i)).count() + }; + for depth in 0usize..=5 { + let n = 1usize << depth; + let leaves: Vec<[FE; 4]> = (0..n).map(|i| digest(depth as u64 + 31, i)).collect(); + let tree = Merkle4::new(&leaves).expect("non-empty"); + let root = tree.root().to_vec(); + for (index, &leaf) in leaves.iter().enumerate() { + let path = child_order_path(depth, &tree.path(index).expect("leaf")); + assert_eq!(path.len(), path_hints(depth)); + let (program, got) = rotated_walk(depth, index, leaf, path.clone(), |b, a, bits| { + hint_path_rotated(b, a, 0, bits) + }); + assert_eq!(got, root, "depth {depth} index {index}"); + assert_eq!( + count(&program, |i| matches!( + i, + super::instr::Instr::Select { .. } + )), + path_hints(depth), + "three Selects a full level, one at an odd top" + ); + assert_eq!( + count(&program, |i| matches!( + i, + super::instr::Instr::HintOn { .. } + )), + 3 * (depth / 2) + ); + // Swapped: each hint reads `alt` on a clear bit and `index` on a set + // one. + let (_, swapped) = rotated_walk(depth, index, leaf, path, |b, a, bits| { + let mut hints = Vec::new(); + let mut w = 0u32; + for pair in bits.chunks(2) { + if let [_, b1] = pair { + for k in 0..3 { + let c = b.hint_word_on(a, w + (k + 2) % 3, *b1, w + k); + hints.push(WrapDigest::from_cell(c)); + } + w += 3; + } else { + hints.push(WrapDigest::from_cell(b.hint_word(a, w))); + } + } + hints + }); + if depth >= 2 { + assert_ne!( + swapped, root, + "depth {depth} index {index}: the rotation is read" + ); + } + } + } +} + +/// A wrong sibling moves the walked root: the hints are bound, including the +/// partner at an odd top. +#[test] +fn a_tampered_hint_moves_the_walked_root() { + for depth in [3usize, 4] { + let n = 1usize << depth; + let leaves: Vec<[FE; 4]> = (0..n).map(|i| digest(99, i)).collect(); + let tree = Merkle4::new(&leaves).expect("non-empty"); + let index = n - 2; + let hints = hint_order( + index, + depth, + &sibling_triples(&tree.path(index).expect("leaf")), + ); + for t in 0..hints.len() { + let mut bad = hints.clone(); + bad[t][1] += FE::one(); + let mut b = LfmBuilder::new(); + let a_leaf = b.declare_arena(1); + let a_idx = b.declare_arena(1); + let a_hint = b.declare_arena(bad.len() as u32); + let leaf = WrapDigest::from_cell(b.hint_word(a_leaf, 0)); + let idx = b.hint_felt(a_idx, 0); + let bits = b.bit_dec(idx, depth); + let h: Vec = (0..bad.len() as u32) + .map(|i| WrapDigest::from_cell(b.hint_word(a_hint, i))) + .collect(); + let walked = walk4(&mut b, leaf, &bits, &h); + b.public(walked.cells()[0]); + let got = run( + b, + &[ + vec![leaves[index]], + vec![base_word(FE::from(index as u64))], + bad, + ], + ); + assert_ne!(got, tree.root().to_vec(), "depth {depth}, hint {t}"); + } + } +} + +/// One step of a transcript script, run on both sides. +#[derive(Clone, Copy, Debug)] +enum Op { + /// Absorb a machine felt. + Felt(u64), + /// Absorb a program constant. + Const(u64), + /// Squeeze one felt (published). + Squeeze, + /// The state digest, without advancing (published). + State, +} + +fn run_script(ops: &[Op]) { + // The host. + let mut host = zisk::Transcript::new(); + let mut want: Vec = Vec::new(); + for op in ops { + match *op { + Op::Felt(v) | Op::Const(v) => host.put1(FE::from(v)), + Op::Squeeze => want.push(host.squeeze1()), + Op::State => want.extend_from_slice(&host.clone().state()[..4]), + } + } + // The machine. + let mut b = LfmBuilder::new(); + let vars: Vec = ops + .iter() + .filter_map(|op| match op { + Op::Felt(v) => Some(*v), + _ => None, + }) + .collect(); + let a = b.declare_arena(vars.len() as u32); + let mut s = P1SpongeVar::new(&mut b); + let mut next = 0u32; + let zero = b.felt_const(FE::zero()); + for op in ops { + match *op { + Op::Felt(_) => { + let f = b.hint_felt(a, next); + next += 1; + s.put_felt(&mut b, f); + } + Op::Const(v) => s.put_const(&mut b, FE::from(v)), + Op::Squeeze => { + let f = s.squeeze(&mut b); + let w = b.pack_word([f, zero, zero, zero]); + b.public(w); + } + Op::State => { + let c = s.digest(&mut b); + b.public(c); + } + } + } + let got = run(b, &[vars.iter().map(|v| base_word(FE::from(*v))).collect()]); + // Squeezes publish one lane and three zeros; states publish four lanes. + let mut flat = Vec::new(); + let mut it = got.chunks(4); + for op in ops { + match op { + Op::Squeeze => flat.push(it.next().expect("a word")[0]), + Op::State => flat.extend_from_slice(it.next().expect("a word")), + _ => {} + } + } + assert_eq!(flat, want, "{ops:?}"); +} + +#[test] +fn the_sponge_is_zisks_transcript() { + use Op::*; + // A squeeze before any absorb; then squeezes across the sixteen-lane + // boundary; a state with an empty and a non-empty buffer. + run_script(&[Squeeze, Squeeze, State]); + run_script(&(0..18).map(|_| Squeeze).collect::>()); + run_script(&[ + Felt(1), + State, + Squeeze, + Felt(2), + Const(3), + Squeeze, + State, + Squeeze, + ]); + // Exactly twelve absorbs flush the buffer and the squeeze reads that output. + let mut ops: Vec = (0..12).map(|i| Felt(100 + i)).collect(); + ops.extend([Squeeze, Squeeze]); + run_script(&ops); + // Thirteen, then a mix of constants and machine felts over two buffers. + let mut ops: Vec = (0..13) + .map(|i| if i % 3 == 0 { Const(i) } else { Felt(i) }) + .collect(); + ops.extend([State, Squeeze, Squeeze, Squeeze]); + ops.extend((0..25).map(|i| Felt(1_000 + i))); + ops.extend([Squeeze, State]); + run_script(&ops); +} + +#[test] +fn the_emulated_width8_permutation_is_poseidon1_w8() { + for seed in [0u64, 1, 0xFFFF] { + let input: [FE; 8] = core::array::from_fn(|i| felt(seed, i)); + let mut b = LfmBuilder::new(); + let s: [Felt; 8] = { + let v = hinted_felts(&mut b, 8); + core::array::from_fn(|i| v[i]) + }; + let out = w8_permute(&mut b, s); + for q in out.chunks(4) { + let w = b.pack_word([q[0], q[1], q[2], q[3]]); + b.public(w); + } + let got = run(b, &[input.iter().copied().map(base_word).collect()]); + assert_eq!(got, poseidon1_w8::permute(input).to_vec(), "seed {seed}"); + } + // The socket's width-16 permutation is the same instance's, for the record. + let x: [FE; 16] = core::array::from_fn(|i| FE::from(i as u64)); + assert_eq!( + poseidon1_w16::permute(x)[0], + FE::from(9_350_316_517_402_464_675u64) + ); +} + +/// ★ The sparse form of the width-8 permutation is the textbook one: on the +/// host (the precomputed constants and matrices) and emitted, at every input +/// pattern the grind uses and with every lane variable, and it spends the +/// rows its derivation says. +#[test] +fn the_sparse_width8_permutation_is_poseidon1_w8() { + for seed in [0u64, 1, 2, 0xFFFF] { + let input: [FE; 8] = core::array::from_fn(|i| felt(seed, i)); + assert_eq!( + w8_sparse_host(input), + poseidon1_w8::permute(input), + "host, seed {seed}" + ); + } + // (constant lanes, kept lanes): all variable, the grind's inner and outer. + let patterns: [(&[usize], usize); 3] = [(&[], 8), (&[0, 5, 6, 7], 4), (&[5, 6, 7], 1)]; + let mut rows = Vec::new(); + for (consts, keep) in patterns { + for seed in [3u64, 4] { + let mut input: [FE; 8] = core::array::from_fn(|i| felt(seed, i)); + for &i in consts { + if i >= 6 { + input[i] = FE::zero(); + } + } + let mut b = LfmBuilder::new(); + let vars = hinted_felts(&mut b, 8); + let lanes: [Lane; 8] = core::array::from_fn(|i| { + if consts.contains(&i) { + Lane::Const(input[i]) + } else { + Lane::Var(vars[i]) + } + }); + let out = w8_permute_lanes(&mut b, lanes, keep); + for f in &out { + b.public(f.as_cell()); + } + let program = compile(b.finish()); + // The permutation is the program's only arithmetic. + let spent = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::BaseAlu { .. })) + .count(); + super::validator::validate(&program).expect("admissible"); + let exec = execute_serial( + &program, + &[input.iter().copied().map(base_word).collect()], + &HasherKind::Poseidon1W16, + ) + .expect("executes"); + let got: Vec = exec.public_words.iter().flat_map(|(_, w)| *w).collect(); + let want = poseidon1_w8::permute(input); + let got_lanes: Vec = got.chunks(4).map(|w| w[0]).collect(); + assert_eq!( + got_lanes, + want[..keep].to_vec(), + "consts {consts:?} keep {keep}" + ); + rows.push((consts.len(), keep, spent)); + } + } + println!("SPARSE W8 ROWS (consts, keep, rows): {rows:?}"); + // Dense: 8 + 8·(32 + 64) + 22·(4 + 64) = 2,272. Sparse, all variable: 8 + // adds, 8 full rounds of 32 + 64, 22 partial rounds of 4 + 8 + 7, 7 more + // where the last partial round's seeds meet the terminal constant, and 8 + // fewer in the unseeded last round, whose MDS rows each start from their + // coefficient-one term: 8 + 768 + 418 + 7 − 8 = 1,193. The grind's inner + // hash (four constant lanes, four kept) saves four adds, four S-boxes and + // 32 matrix terms in round 0 and four last-round outputs (32 − 4): 1,113; + // its outer (three constant lanes, lane 0 kept) 1,105 — 2,218 a check + // against the dense 4,544. + let spent: Vec = rows.iter().step_by(2).map(|r| r.2).collect(); + assert_eq!( + spent, + vec![1_193, 1_113, 1_105], + "all variable; inner; outer" + ); +} + +/// The transcript replay's Poseidon1 arm against `P1Transcript`: every append +/// kind a sub-proof replays (constant bytes, a root, a felt, an extension +/// element, machine bytes) and every draw (an extension element, index bits, +/// a felt, the state). +#[test] +fn the_replay_arm_is_p1_transcript() { + use super::edsl::WrapHash; + use super::p1_commit::P1Transcript; + use super::transcript_replay::TranscriptReplay; + use crate::tables::types::FEE; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let seed = b"p3 replay seed"; + let root: [FE; 4] = digest(5, 0); + let felt_v = felt(6, 0); + let ext_v: [FE; 3] = core::array::from_fn(|i| felt(7, i)); + let bytes: [u8; 8] = 0x0123_4567_89ab_cdefu64.to_le_bytes(); + let nbits = 13usize; + + // The host. + let mut host = P1Transcript::with_seed(seed); + host.append_bytes(&super::algebraic_commit::digest_to_commitment(&root)); + host.append_bytes(&felt_v.canonical().to_be_bytes()); + host.append_field_element(&FEE::new(ext_v)); + host.append_bytes(&bytes); + let e = host.sample_field_element(); + let idx = host.sample_u64(1 << nbits); + host.append_bytes(&[1, 2, 3]); + let state = host.state(); + let f = host.sample_field_element(); + + // The machine. + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let a_root = b.declare_arena(1); + let a_felts = b.declare_arena(6); + let r = b.hint_word(a_root, 0); + let fv = b.hint_felt(a_felts, 0); + let ev: [Felt; 3] = core::array::from_fn(|i| b.hint_felt(a_felts, 1 + i as u32)); + let halves = [b.hint_felt(a_felts, 4), b.hint_felt(a_felts, 5)]; + let mut t = TranscriptReplay::new(seed); + t.append_root_cells(&mut b, &[r]); + t.append_felt(&mut b, fv); + t.append_ext(&mut b, ev); + t.append_halves(&halves); + let e_m = t.sample_ext(&mut b); + b.public(e_m.as_cell()); + let bits = t.sample_u64_pow2(&mut b, nbits); + let idx_m = super::edsl::bits_to_felt(&mut b, &bits); + let zero = b.felt_const(FE::zero()); + let w = b.pack_word([idx_m, zero, zero, zero]); + b.public(w); + t.append_const_bytes(&[1, 2, 3]); + let s = t.state(&mut b); + b.public(s.cells()[0]); + let f_m = t.sample_ext(&mut b); + b.public(f_m.as_cell()); + + let half = |k: usize| { + FE::from(u64::from(u32::from_le_bytes( + bytes[4 * k..4 * k + 4].try_into().unwrap(), + ))) + }; + let got = run( + b, + &[ + vec![root], + [felt_v, ext_v[0], ext_v[1], ext_v[2], half(0), half(1)] + .map(base_word) + .to_vec(), + ], + ); + let ext3 = |x: FEE| -> Vec { + let v = x.value(); + vec![v[0], v[1], v[2], FE::zero()] + }; + let mut want = ext3(e); + want.extend([FE::from(idx), FE::zero(), FE::zero(), FE::zero()]); + want.extend(super::algebraic_commit::commitment_to_digest(&state)); + want.extend(ext3(f)); + assert_eq!(got, want); +} + +/// The grinding check accepts the host's nonce and refuses one that misses. +#[test] +fn the_grinding_check_is_p1s() { + use super::edsl::WrapHash; + use super::p1_commit::P1GrindDigest; + + let lanes: [FE; 4] = digest(11, 0); + let seed = super::algebraic_commit::digest_to_commitment(&lanes); + let factor = 6u8; + let valid = |n: u64| crypto::grinding::is_valid_nonce::(&seed, n, factor); + let good = (0u64..).find(|&n| valid(n)).expect("a nonce"); + let bad = (0u64..).find(|&n| !valid(n)).expect("a miss"); + + let check = |nonce: u64| { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let a_seed = b.declare_arena(1); + let a_nonce = b.declare_arena(1); + let s = WrapDigest::from_cell(b.hint_word(a_seed, 0)); + let n = b.hint_felt(a_nonce, 0); + super::epoch::emit_grinding_check(&mut b, s, n, factor); + let program = compile(b.finish()); + execute_serial( + &program, + &[vec![lanes], vec![base_word(FE::from(nonce))]], + &HasherKind::Poseidon1W16, + ) + .map(|_| ()) + }; + assert!(check(good).is_ok(), "nonce {good}"); + assert!(check(bad).is_err(), "nonce {bad}"); +} + +/// Run `b`'s program over `arenas`, `Err` when it does not execute. +fn try_run(b: LfmBuilder, arenas: &[Vec]) -> Result<(), String> { + let program = compile(b.finish()); + super::validator::validate(&program).map_err(|e| format!("{e:?}"))?; + execute_serial(&program, arenas, &HasherKind::Poseidon1W16) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) +} + +/// One opening of a host Poseidon1 tree checked in-guest at cap height `c`: +/// the owner path as the proof carries it (the path to the cap, then the cap, +/// `embed_cap_arity`) laid out by the harvest (`path_to_cap`), the cap +/// authenticated against the root and the opening against the cap — or, at +/// `c = 0`, walked to the root. `tamper` moves one arena word first. +fn cap_opening( + depth: usize, + c: usize, + index: usize, + tamper: Option<(usize, usize)>, +) -> Result<(), String> { + use super::algebraic_commit::{commitment_to_digest, digest_to_commitment}; + use super::edsl::WrapHash; + use super::merkle_cap::CapCells; + use crypto::merkle_tree::merkle::MerkleTree; + use stark::config::Commitment; + type B = super::p1_commit::P1BatchBackend; + + let n = 1usize << depth; + let leaves: Vec = (0..n) + .map(|i| digest_to_commitment(&digest(depth as u64 + 31, i))) + .collect(); + let tree = MerkleTree::::build_from_hashed_leaves(leaves.clone()).expect("a tree"); + let mut path = tree + .get_proof_by_pos(index) + .expect("a leaf") + .merkle_path + .clone(); + let cap = if c == 0 { + Vec::new() + } else { + tree.cap(c).expect("the cap") + }; + if c > 0 { + crypto::merkle_tree::cap::embed_cap_arity(&mut [&mut path], depth, &cap, 4) + .expect("the owner path"); + } + let (hints, split) = + super::merkle_cap::path_to_cap(&path, depth, c, 4, true, Some(index)).expect("laid out"); + assert_eq!( + split.unwrap_or_default(), + cap, + "the cap rides on the owner path" + ); + assert_eq!( + hints.len(), + super::merkle_cap::path_hints(depth, c, 4), + "depth {depth} cap {c}" + ); + + let word = |c: &Commitment| -> LfmWord { commitment_to_digest(c) }; + let mut arenas: Vec> = vec![ + vec![word(&leaves[index])], + vec![base_word(FE::from(index as u64))], + hints.iter().map(word).collect(), + cap.iter().map(word).collect(), + vec![word(&tree.root)], + ]; + if let Some((a, w)) = tamper { + arenas[a][w][2] += FE::one(); + } + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let lens: Vec = arenas.iter().map(|a| a.len() as u32).collect(); + let ids: Vec<_> = lens.iter().map(|&l| b.declare_arena(l)).collect(); + let leaf = WrapDigest::from_cell(b.hint_word(ids[0], 0)); + let idx = b.hint_felt(ids[1], 0); + let bits = b.bit_dec(idx, depth); + let h: Vec = (0..lens[2]) + .map(|i| WrapDigest::from_cell(b.hint_word(ids[2], i))) + .collect(); + let root = b.hint_word(ids[4], 0); + let root_lanes = [b.unpack(root)]; + if c == 0 { + let walked = super::edsl::wrap_merkle_walk(&mut b, leaf, &bits, &h); + super::edsl::assert_digest_eq_lanes(&mut b, walked, &root_lanes); + } else { + let nodes: Vec = (0..lens[3]) + .map(|i| WrapDigest::from_cell(b.hint_word(ids[3], i))) + .collect(); + let capped = CapCells::authenticate_at(&mut b, &nodes, c, &root_lanes); + assert_eq!(capped.height(), c); + capped.verify_path(&mut b, leaf, &bits, &h); + } + try_run(b, &arenas) +} + +/// ★ Arity-4 caps against the host's own trees: at every depth (both +/// parities) and every cap height up to the tree's levels, an opening checked +/// against the host's cap (`MerkleTree::cap`, `4^c` or `2·4^(c−1)` nodes) over +/// the path the proof carries executes; uncapped, the walk reaches the root +/// (the odd top's padding supplied by the walk). +#[test] +fn an_arity4_cap_checks_the_hosts_openings() { + for depth in 1usize..=7 { + let n = 1usize << depth; + let levels = crypto::merkle_tree::cap::tree_levels(depth, 4); + for c in 0..=levels { + for index in [0, n / 2, n - 1, (5 * n) / 7] { + cap_opening(depth, c, index, None).unwrap_or_else(|e| { + panic!("depth {depth} cap {c} index {index}: {e}"); + }); + } + } + } +} + +/// Every word the check reads is bound: a moved hint, cap node or root makes +/// the program unexecutable, at an even and an odd depth, capped and not. +#[test] +fn an_arity4_cap_refuses_a_moved_word() { + for (depth, c) in [(6usize, 2usize), (7, 1), (7, 0), (5, 3)] { + let index = (1usize << depth) - 3; + let hints = super::merkle_cap::path_hints(depth, c, 4); + let nodes = super::merkle_cap::cap_nodes(depth, c, 4); + let mut words = vec![(0usize, 0usize), (4, 0)]; + words.extend((0..hints).map(|w| (2, w))); + if c > 0 { + words.extend((0..nodes).map(|w| (3, w))); + } + for (a, w) in words { + assert!( + cap_opening(depth, c, index, Some((a, w))).is_err(), + "depth {depth} cap {c}: arena {a} word {w} moved and the opening executed" + ); + } + } +} diff --git a/prover/src/lfm/p1w16_machine_tests.rs b/prover/src/lfm/p1w16_machine_tests.rs new file mode 100644 index 000000000..298ef06ec --- /dev/null +++ b/prover/src/lfm/p1w16_machine_tests.rs @@ -0,0 +1,312 @@ +//! The width-16 socket wired into the machine: `Instr::Hash16` from the +//! builder through compilation, admission, both execution schedules and the +//! `LFM_HASH` trace, under `HasherKind::Poseidon1W16`. + +use math::field::element::FieldElement; +use stark::constraints::builder::{ConstraintSet, ProverEvalFolder}; +use stark::frame::Frame; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; +use crypto::hash::poseidon1_w16 as p1; + +use super::builder::{Cell, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::{LfmExecError, execute_serial, execute_with_merged_levels}; +use super::hash::HasherKind; +use super::p1w16_socket::{P1W16SocketConstraints, SOCKET_FORM, cols}; +use super::validator::{LfmViolation, validate}; + +type Gl = GoldilocksField; +type Gl3 = GoldilocksExtension; + +fn word(seed: u64) -> [FE; 4] { + core::array::from_fn(|i| FE::from(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ i as u64)) +} + +/// Two chained width-16 permutations over constant cells, the first's cell 0 +/// read twice and every output cell of the second published, so the +/// multiplicities are 2, 0, 0, 0 and then 1, 1, 1, 1. Returns the program and +/// the host's expected outputs of both rows. +fn chained_program() -> (LfmProgram, [FE; 16], [FE; 16]) { + let mut b = LfmBuilder::new(); + let ins: [Cell; 4] = core::array::from_fn(|k| b.digest_const(word(k as u64 + 1)).as_cell()); + let first = b.hash16(ins); + let zero = b.digest_const([FE::zero(); 4]).as_cell(); + let second = b.hash16([first[0], zero, first[0], ins[3]]); + for c in second { + b.public(c); + } + let program = compile(b.finish()); + + let mut s0 = [FE::zero(); 16]; + for k in 0..4 { + s0[4 * k..4 * k + 4].copy_from_slice(&word(k as u64 + 1)); + } + let o0 = p1::permute(s0); + let mut s1 = [FE::zero(); 16]; + s1[0..4].copy_from_slice(&o0[0..4]); + s1[8..12].copy_from_slice(&o0[0..4]); + s1[12..16].copy_from_slice(&word(4)); + (program, o0, p1::permute(s1)) +} + +fn socket_violations(row: &[FE]) -> Vec { + let set = P1W16SocketConstraints { form: SOCKET_FORM }; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +#[test] +fn a_width16_program_compiles_admits_and_proves_under_the_socket() { + let (program, _, _) = chained_program(); + assert!(program.hash16()); + assert_eq!( + program.hasher(crate::hash_pin::BLOCK_HASHER), + HasherKind::Poseidon1W16 + ); + assert_eq!(program.groups.hash.real_rows, 2); + validate(&program).expect("admissible"); + + // The instruction group is the socket's: addresses, IS_REAL, and the + // backfilled multiplicities 2,0,0,0 then 1,1,1,1. + let g = &program.groups.hash; + assert_eq!(*g.at(0, cols::IS_REAL), FE::one()); + assert_eq!(*g.at(1, cols::IS_REAL), FE::one()); + let mults = |r: usize| -> Vec { (0..4).map(|k| *g.at(r, cols::MULT0 + k)).collect() }; + assert_eq!(mults(0), [2u64, 0, 0, 0].map(FE::from).to_vec()); + assert_eq!(mults(1), [1u64, 1, 1, 1].map(FE::from).to_vec()); +} + +#[test] +fn both_schedules_execute_the_permutations_and_agree() { + let (program, o0, o1) = chained_program(); + let hasher = HasherKind::Poseidon1W16; + let serial = execute_serial(&program, &[], &hasher).expect("serial"); + let level = execute_with_merged_levels(&program, &[], &hasher, 0, 1).expect("levels"); + for exec in [&serial, &level] { + assert!(exec.records.hash.is_empty()); + assert_eq!(exec.records.hash16.len(), 2); + assert_eq!(exec.records.hash16[0].outs, o0); + assert_eq!(exec.records.hash16[1].outs, o1); + let published: Vec = exec.public_words.iter().flat_map(|(_, w)| *w).collect(); + assert_eq!(published, o1.to_vec()); + } + for (a, b) in serial.records.hash16.iter().zip(&level.records.hash16) { + assert_eq!((a.ins, a.outs), (b.ins, b.outs)); + } +} + +/// Every `LFM_HASH` row of the trace — the two real rows and the padding — +/// satisfies the socket's constraints, and a real row's preprocessed prefix is +/// its instruction's. +#[test] +fn the_hash_trace_satisfies_the_socket() { + let (program, _, _) = chained_program(); + let hasher = HasherKind::Poseidon1W16; + let exec = execute_serial(&program, &[], &hasher).expect("executes"); + let traces = super::trace::build_traces_with_hasher(&program, &exec.records, hasher); + let t = &traces.hash; + assert_eq!(t.num_cols(), SOCKET_FORM.num_columns()); + assert!(t.num_rows() >= 2); + for r in 0..t.num_rows() { + let row: Vec = (0..t.num_cols()).map(|c| *t.get_main(r, c)).collect(); + assert_eq!(socket_violations(&row), Vec::::new(), "row {r}"); + let real = row[cols::IS_REAL] == FE::one(); + assert_eq!(real, r < 2, "row {r}"); + } +} + +#[test] +fn a_width16_row_under_a_twelve_felt_hasher_is_refused() { + let (program, _, _) = chained_program(); + let err = execute_serial(&program, &[], &HasherKind::Rpx).map(|_| ()); + assert!( + matches!(err, Err(LfmExecError::HasherRejected(_))), + "{err:?}" + ); +} + +#[test] +fn a_twelve_felt_row_under_the_socket_is_refused() { + let mut b = LfmBuilder::new(); + let a = b.digest_const(word(1)); + let c = b.digest_const(word(2)); + let d = b.compress(a, c); + b.public(d.as_cell()); + let program = compile(b.finish()); + assert!(!program.hash16()); + let err = execute_serial(&program, &[], &HasherKind::Poseidon1W16).map(|_| ()); + assert!( + matches!(err, Err(LfmExecError::HasherRejected(_))), + "{err:?}" + ); +} + +/// The socket review's A2: under the width-16 socket the twelve-felt contract +/// hands out no hash at all — every hashing method refuses, so no host caller +/// can take an unproved twelve-felt Poseidon1 value for a proved one. (The +/// executor never gets that far: `admits` refuses the row first, as +/// [`a_twelve_felt_row_under_the_socket_is_refused`] shows.) +#[test] +fn the_twelve_felt_contract_hands_out_no_hash_under_the_socket() { + use super::hash::LfmHasher; + let h = HasherKind::Poseidon1W16; + let w = word(3); + type Call = fn(HasherKind, [FE; 4]); + let calls: [(&str, Call); 4] = [ + ("permute", |h, _| { + let _ = h.permute([FE::zero(); 12]); + }), + ("compress", |h, w| { + let _ = h.compress(&w, &w); + }), + ("transcript", |h, w| { + let _ = h.transcript(&w, &w); + }), + ("leaf", |h, w| { + let _ = h.leaf(&w, &w); + }), + ]; + for (what, call) in calls { + assert!( + std::panic::catch_unwind(move || call(h, w)).is_err(), + "{what} returned a twelve-felt hash under the socket" + ); + } +} + +/// A2's reachability, driven: every production entry point that hashes a +/// twelve-felt row with a program's hasher, handed a twelve-felt row of each +/// mode under the width-16 socket, refuses it as a TYPED error and never reaches +/// the refusing `permute` (a panic here fails the test). +/// +/// - The executor, under all three schedules: `hash_compute` reads `mode_iv` +/// (the zero IV), then `admits` refuses before any hashing method runs. +/// - The trace filler over twelve-felt records (an RPX execution's) handed the +/// socket's hasher: its socket arm fills no witness and hashes nothing. +/// +/// The other callers of the twelve-felt contract are not production entry +/// points with a program's hasher: the host block transcript is pinned to +/// `BLOCK_HASHER` at compile time (`hash_pin`), and `fixture`'s host sponge and +/// tree serve the fixture programs and the suites only. +#[test] +fn no_production_entry_point_reaches_the_twelve_felt_hash_under_the_socket() { + use super::executor::execute; + use super::trace::build_traces_with_hasher; + use std::panic::{AssertUnwindSafe, catch_unwind}; + + let programs: Vec<(&str, LfmProgram)> = vec![ + ("compress", { + let mut b = LfmBuilder::new(); + let a = b.digest_const(word(1)); + let c = b.digest_const(word(2)); + let d = b.compress(a, c); + b.public(d.as_cell()); + compile(b.finish()) + }), + ("transcript", { + let mut b = LfmBuilder::new(); + let a = b.digest_const(word(3)); + let c = b.digest_const(word(4)); + let d = b.transcript_step(a, c); + b.public(d.as_cell()); + compile(b.finish()) + }), + ("leaf", { + let mut b = LfmBuilder::new(); + let a = b.digest_const(word(5)); + let f = b.digest_const(word(6)).as_cell(); + let d = b.leaf(a, f); + b.public(d.as_cell()); + compile(b.finish()) + }), + ("permute", { + let mut b = LfmBuilder::new(); + let s: [Cell; 3] = + core::array::from_fn(|k| b.digest_const(word(7 + k as u64)).as_cell()); + let out = b.permute(s); + b.public(out[0]); + compile(b.finish()) + }), + ]; + let socket = HasherKind::Poseidon1W16; + for (mode, program) in &programs { + assert!(!program.hash16(), "{mode}: a twelve-felt program"); + type Entry = fn(&LfmProgram, HasherKind) -> Result<(), LfmExecError>; + let entries: [(&str, Entry); 3] = [ + ("execute (the default schedule)", |p, h| { + execute(p, &[], &h).map(|_| ()) + }), + ("execute_serial", |p, h| { + execute_serial(p, &[], &h).map(|_| ()) + }), + ("execute_with_merged_levels", |p, h| { + execute_with_merged_levels(p, &[], &h, 1, 2).map(|_| ()) + }), + ]; + for (entry, run) in entries { + let got = catch_unwind(AssertUnwindSafe(|| run(program, socket))); + assert!( + matches!(got, Ok(Err(LfmExecError::HasherRejected(_)))), + "{entry} on a {mode} row under the socket: {:?}", + got.map_err(|_| "panicked") + ); + } + // The trace filler: twelve-felt records, the socket's hasher. + let exec = execute_serial(program, &[], &HasherKind::Rpx).expect("executes under RPX"); + let filled = catch_unwind(AssertUnwindSafe(|| { + build_traces_with_hasher(program, &exec.records, socket); + })); + assert!( + filled.is_ok(), + "the trace filler hashed a {mode} row under the socket" + ); + } +} + +#[test] +fn a_program_mixing_the_two_widths_is_not_admitted() { + let mut b = LfmBuilder::new(); + let ins: [Cell; 4] = core::array::from_fn(|k| b.digest_const(word(k as u64)).as_cell()); + let out = b.hash16(ins); + let d = b.compress(out[0].as_digest(), out[1].as_digest()); + b.public(d.as_cell()); + let program = compile(b.finish()); + assert!(matches!( + validate(&program), + Err(LfmViolation::MixedHashWidths { .. }) + )); +} + +/// The AIR set under the socket: `LFM_HASH` is the socket's AIR, the same +/// width as RPX's, with its eight interactions. +#[test] +fn the_air_set_under_the_socket_carries_the_socket_chip() { + use super::airs::{ChipSet, LfmAirs, NUM_LFM_CHIPS}; + use super::chips::hash; + let opts = super::proof::aggregation_wrap_options(); + let roots = [[0u8; 32]; NUM_LFM_CHIPS]; + let airs = LfmAirs::new_with_hasher(&roots, &opts, 1, HasherKind::Poseidon1W16, ChipSet::FULL); + let names: Vec<&str> = airs.air_refs().iter().map(|a| a.name()).collect(); + assert!(names.contains(&"LFM_HASH")); + assert_eq!(hash::num_columns(HasherKind::Poseidon1W16), 329); + assert_eq!(hash::bus_interactions(HasherKind::Poseidon1W16).len(), 8); + assert_eq!(HasherKind::Poseidon1W16.as_tag(), 5); +} diff --git a/prover/src/lfm/p1w16_proof_tests.rs b/prover/src/lfm/p1w16_proof_tests.rs new file mode 100644 index 000000000..676622c88 --- /dev/null +++ b/prover/src/lfm/p1w16_proof_tests.rs @@ -0,0 +1,606 @@ +//! The width-16 socket proved and verified, and the necessity of each of its +//! constraints (the socket review R-P1SOCKET's R1–R3): +//! +//! - **R1.** A `Hash16` program proved and verified end to end through the +//! production LogUp path, and rows that satisfy every socket constraint but +//! disagree with memory (or park a real witness on padding) refused by the +//! verifier. +//! - **R2.** Each S-box constraint individually necessary: the other square +//! root of `x³` violates exactly its `x³` constraint, and a moved `x⁷` with +//! every later round recomputed violates exactly its `x⁷` constraint. The +//! value-column perturbations in `p1w16_socket_tests` are caught by the NEXT +//! round's constraints and cannot see either go missing. The bus side: a +//! transposed coefficient matrix misreads the outputs. +//! - **R3.** The socket's `LFM_HASH` AIR lowered in-guest (the leg a level-1 +//! node runs for a `Poseidon1W16` child) agrees with the IR interpreter and the +//! host verifier folder. + +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::trace::TraceTable; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField, VmTable}; +use crypto::hash::poseidon1_w16 as p1; + +use super::builder::{Cell, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::hash::HasherKind; +use super::p1w16_socket::{SOCKET_FORM, cols, fill_row, sboxed_lanes}; +use super::proof::{prove_traces_with_hasher, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::trace::build_traces_with_hasher; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +const KIND: HasherKind = HasherKind::Poseidon1W16; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") +} + +fn word(seed: u64) -> [FE; 4] { + core::array::from_fn(|i| FE::from(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ i as u64)) +} + +/// Two chained permutations; the second reads the first's cell 0 twice and +/// every output of the second is published. +fn program() -> LfmProgram { + let mut b = LfmBuilder::new(); + let ins: [Cell; 4] = core::array::from_fn(|k| b.digest_const(word(k as u64 + 1)).as_cell()); + let first = b.hash16(ins); + let zero = b.digest_const([FE::zero(); 4]).as_cell(); + let second = b.hash16([first[0], zero, first[0], ins[3]]); + for c in second { + b.public(c); + } + compile(b.finish()) +} + +fn row_of(t: &TraceTable, r: usize) -> Vec { + (0..t.num_cols()).map(|c| *t.get_main(r, c)).collect() +} + +fn write_row(t: &mut TraceTable, r: usize, row: &[FE]) { + for (c, v) in row.iter().enumerate().skip(cols::PREP_WIDTH) { + t.main_table.set_fe(r, c, *v); + } +} + +/// `fill_row` without round constants: the witness a padding row +/// (`IS_REAL = 0`) satisfies for any `IN`. +fn fill_padding_row(input: [FE; 16], row: &mut [FE]) { + row[cols::IN0..cols::IN0 + 16].copy_from_slice(&input); + let mut s = input; + for r in 0..p1::NUM_ROUNDS { + let mut f = s; + for (lane, a) in f.iter_mut().enumerate().take(sboxed_lanes(r)) { + let x3 = *a * *a * *a; + let x7 = x3 * x3 * *a; + row[SOCKET_FORM.x3(r, lane)] = x3; + row[SOCKET_FORM.x7(r, lane)] = x7; + *a = x7; + } + s = p1::mds(&f); + } +} + +/// `Ok(true)` = accepted; `Ok(false)` = verifier rejection; `Err` = prover refusal. +fn round_trip(mutate: impl FnOnce(&mut TraceTable)) -> Result { + let opts = options(); + let program = program(); + super::validator::validate(&program).expect("admissible"); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let exec = execute(&program, &[], &KIND).expect("execute"); + let mut traces = build_traces_with_hasher(&program, &exec.records, KIND); + mutate(&mut traces.hash); + // The edit is to the host trace: the card must not rebuild the unedited rows. + #[cfg(feature = "cuda")] + traces.hash.clear_main_dev_producer(); + match prove_traces_with_hasher( + &artifacts, + &mut traces, + &exec.public_words, + &opts, + KIND, + stark::residency_mode::ResidencyMode::Retain, + ) { + Ok(proof) => Ok(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + KIND, + artifacts.chip_set, + )), + Err(e) => Err(format!("{e:?}")), + } +} + +/// Every socket constraint on `row` evaluates to zero (the row-local view). +fn satisfies_socket(row: &[FE]) -> bool { + violated(row).is_empty() +} + +#[test] +fn honest_hash16_program_proves_and_verifies() { + assert_eq!(round_trip(|_| {}), Ok(true)); +} + +/// Row 0 recomputed from a different input: every constraint of the row holds, +/// its reads disagree with memory and its write disagrees with row 1's reads. +#[test] +fn a_consistent_row_over_a_wrong_input_is_not_accepted() { + let r = round_trip(|t| { + let mut row = row_of(t, 0); + let mut input: [FE; 16] = core::array::from_fn(|i| row[cols::IN0 + i]); + input[5] += FE::one(); + fill_row(SOCKET_FORM, input, &mut row); + assert!( + satisfies_socket(&row), + "the forged row is constraint-consistent" + ); + write_row(t, 0, &row); + }); + assert_ne!(r, Ok(true), "{r:?}"); +} + +/// The published row recomputed from a different input: the outputs it sends +/// are a true permutation, but of the wrong cells. +#[test] +fn a_consistent_published_row_over_a_wrong_input_is_not_accepted() { + let r = round_trip(|t| { + let mut row = row_of(t, 1); + let mut input: [FE; 16] = core::array::from_fn(|i| row[cols::IN0 + i]); + input[15] += FE::one(); + fill_row(SOCKET_FORM, input, &mut row); + assert!(satisfies_socket(&row)); + write_row(t, 1, &row); + }); + assert_ne!(r, Ok(true), "{r:?}"); +} + +/// A padding row's `IN` is free and harmless: any input with the +/// constant-free witness is accepted, because the row sends and receives +/// nothing (IS_REAL = 0, MULT_k = 0 in the preprocessed group). +#[test] +fn garbage_on_a_padding_row_is_accepted_and_harmless() { + let r = round_trip(|t| { + assert!(t.num_rows() > 2, "a padding row exists"); + let last = t.num_rows() - 1; + let mut row = row_of(t, last); + assert_eq!(row[cols::IS_REAL], FE::zero()); + let input: [FE; 16] = core::array::from_fn(|i| FE::from(0xDEAD_0000 + i as u64)); + fill_padding_row(input, &mut row); + assert!(satisfies_socket(&row)); + write_row(t, last, &row); + }); + assert_eq!(r, Ok(true), "{r:?}"); +} + +/// The honest real-row witness placed on a padding row is NOT a valid padding +/// row (the constants do not vanish from it), so the prover cannot park a real +/// permutation there either. +#[test] +fn a_real_witness_on_a_padding_row_is_not_accepted() { + let r = round_trip(|t| { + let last = t.num_rows() - 1; + let mut row = row_of(t, last); + let input: [FE; 16] = core::array::from_fn(|i| FE::from(7 + i as u64)); + fill_row(SOCKET_FORM, input, &mut row); + assert!(!satisfies_socket(&row)); + write_row(t, last, &row); + }); + assert_ne!(r, Ok(true), "{r:?}"); +} + +/// The second square root: `x3 -> -x3` keeps `x7 = x3^2 * a`, so ONLY the +/// `x3 = a^3` constraint stands between the prover and a free S-box output. +/// Every single-column `+1` perturbation (the existing load-bearing test) is +/// caught by `x7`'s constraint instead, so it cannot see that constraint go. +#[test] +fn the_other_square_root_of_x3_is_caught_by_the_x3_constraint_alone() { + let input: [FE; 16] = core::array::from_fn(|i| FE::from(0x1234_5678 + 17 * i as u64)); + let mut row = vec![FE::zero(); SOCKET_FORM.num_columns()]; + row[cols::IS_REAL] = FE::one(); + fill_row(SOCKET_FORM, input, &mut row); + assert!(satisfies_socket(&row)); + let mut idx = 1 + 4; + let mut checked = 0; + for r in 0..p1::NUM_ROUNDS { + for lane in 0..sboxed_lanes(r) { + let mut bad = row.clone(); + let c = SOCKET_FORM.x3(r, lane); + bad[c] = -bad[c]; + assert_eq!(violated(&bad), vec![idx], "round {r} lane {lane}"); + idx += 2; + checked += 1; + } + } + assert_eq!(checked, 150); +} + +/// The production host evaluator of a bus value (`BusValue::combine_from`, the +/// i64 coefficient path) reads the OnBus output tokens as the permutation's +/// output cells. +#[test] +fn the_production_bus_evaluator_reads_the_outputs() { + use math::field::element::FieldElement; + let input: [FE; 16] = core::array::from_fn(|i| -FE::from(1 + i as u64)); + let mut row = vec![FE::zero(); SOCKET_FORM.num_columns()]; + row[cols::IS_REAL] = FE::one(); + let out = fill_row(SOCKET_FORM, input, &mut row); + assert_eq!(out, p1::permute(input)); + let ix = super::p1w16_socket::bus_interactions(SOCKET_FORM); + for (k, i) in ix.iter().enumerate().skip(4) { + let cell = k - 4; + let token: Vec> = i + .values + .iter() + .flat_map(|v| v.combine_from(|c| row[c].to_extension::())) + .collect(); + for j in 0..4 { + assert_eq!( + token[1 + j], + out[4 * cell + j].to_extension::(), + "cell {cell}" + ); + } + } +} + +/// The test-side mutation the review could not run in the chip +/// (`mds_entry(lane, i)` → `mds_entry(i, lane)` in `bus_interactions`): the +/// production interactions with their OnBus coefficient matrix transposed, +/// read by the production evaluator, no longer carry the permutation's +/// outputs — so the output check above is load-bearing for the coefficients' +/// orientation (Plonky3's circulant is not symmetric, so the transpose is a +/// different matrix). +#[test] +fn a_transposed_bus_coefficient_matrix_misreads_the_outputs() { + use stark::lookup::{BusValue, LinearTerm}; + let input: [FE; 16] = core::array::from_fn(|i| FE::from(0x0BAD_C0DE + 29 * i as u64)); + let mut row = vec![FE::zero(); SOCKET_FORM.num_columns()]; + row[cols::IS_REAL] = FE::one(); + let out = fill_row(SOCKET_FORM, input, &mut row); + let honest = super::p1w16_socket::bus_interactions(SOCKET_FORM); + // The production matrix, read off the interactions: lane `o`'s term on + // the last round's `x⁷(i)`. + let last = p1::NUM_ROUNDS - 1; + let mut m = [[0i64; 16]; 16]; + for (k, ix) in honest.iter().enumerate().skip(4) { + for (j, v) in ix.values.iter().enumerate().skip(1) { + let BusValue::Linear(terms) = v else { + panic!("an OnBus output is a linear form") + }; + for t in terms { + let LinearTerm::Column { + coefficient, + column, + } = t + else { + panic!("an OnBus term is a column term") + }; + let i = (0..16) + .find(|&i| SOCKET_FORM.x7(last, i) == *column) + .expect("a last-round x7 column"); + m[4 * (k - 4) + j - 1][i] = *coefficient; + } + } + } + assert!( + (0..16).any(|o| (0..16).any(|i| m[o][i] != m[i][o])), + "the MDS matrix is not symmetric, so its transpose is another matrix" + ); + let mut mutant = honest.clone(); + for (k, ix) in mutant.iter_mut().enumerate().skip(4) { + for (j, v) in ix.values.iter_mut().enumerate().skip(1) { + let o = 4 * (k - 4) + j - 1; + *v = BusValue::Linear( + (0..16) + .map(|i| LinearTerm::Column { + coefficient: m[i][o], + column: SOCKET_FORM.x7(last, i), + }) + .collect(), + ); + } + } + let read = |ixs: &[stark::lookup::BusInteraction]| -> Vec { + ixs.iter() + .skip(4) + .flat_map(|ix| { + ix.values + .iter() + .skip(1) + .map(|v| v.combine_from(|c| row[c])[0]) + }) + .collect() + }; + assert_eq!( + read(&honest), + out.to_vec(), + "the production matrix reads P(IN)" + ); + let misread = read(&mutant); + assert_ne!( + misread, + out.to_vec(), + "the transposed matrix reads something else" + ); + assert!( + misread.iter().zip(&out).filter(|(a, b)| a != b).count() > 1, + "the transpose moves more than one output lane" + ); +} + +fn violated(row: &[FE]) -> Vec { + use math::field::element::FieldElement; + use stark::constraints::builder::{ConstraintSet, ProverEvalFolder}; + use stark::frame::Frame; + use stark::table::TableView; + use stark::traits::TransitionEvaluationContext; + let set = super::p1w16_socket::P1W16SocketConstraints { form: SOCKET_FORM }; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + base_out + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +/// A real row whose S-box `(fr, flane)` output is moved by `delta` AFTER its +/// constraints are checked, every later round recomputed from the forged state. +fn forged_x7_row(input: [FE; 16], fr: usize, flane: usize, delta: FE) -> Vec { + use super::p1w16_socket::cols; + let mut row = vec![FE::zero(); SOCKET_FORM.num_columns()]; + row[cols::IS_REAL] = FE::one(); + row[cols::IN0..cols::IN0 + 16].copy_from_slice(&input); + let mut s = input; + for (r, rc) in p1::constants::ROUND_CONSTANTS.iter().enumerate() { + let mut f: [FE; 16] = core::array::from_fn(|i| s[i] + FE::from(rc[i])); + for (lane, a) in f.iter_mut().enumerate().take(sboxed_lanes(r)) { + let x3 = *a * *a * *a; + let mut x7 = x3 * x3 * *a; + if (r, lane) == (fr, flane) { + x7 += delta; + } + row[SOCKET_FORM.x3(r, lane)] = x3; + row[SOCKET_FORM.x7(r, lane)] = x7; + *a = x7; + } + s = p1::mds(&f); + } + row +} + +/// Each `x7 = x3^2 * a` constraint is individually necessary: a row that +/// moves one S-box output and recomputes everything downstream violates +/// exactly that constraint and nothing else. (The existing single-column +/// perturbation tests are caught by the NEXT round's constraints for every +/// round but the last, so they would not notice one of these going missing.) +#[test] +fn every_x7_constraint_is_individually_necessary() { + let input: [FE; 16] = core::array::from_fn(|i| FE::from(0xABCD_0000 + 3 * i as u64)); + let mut idx = 1 + 4; + let mut checked = 0; + for r in 0..p1::NUM_ROUNDS { + for lane in 0..sboxed_lanes(r) { + let row = forged_x7_row(input, r, lane, FE::one()); + assert_eq!(violated(&row), vec![idx + 1], "round {r} lane {lane}"); + idx += 2; + checked += 1; + } + } + assert_eq!(checked, 150); +} + +/// R3, point (a) of the review on the VERIFIER paths: the machine's `LFM_HASH` AIR under +/// `Poseidon1W16` (socket constraints + LogUp with the OnBus i64 terms), +/// captured, serialized, lowered to LFM instructions (the in-guest leg) and +/// executed, agrees with the IR interpreter and with the host verifier folder +/// (`AIR::compute_transition`) on random all-extension OOD frames. +#[test] +fn the_socket_air_lowered_in_guest_matches_the_host_verifier() { + use super::airs::{ChipSet, LfmAirs, NUM_LFM_CHIPS}; + use super::constraints::{ + OodOperands, analyze, emit_analyzed, hint_ood_frame, ood_frame_words, + }; + use super::word::{ext_word, word_as_ext}; + use crate::tables::types::FEE; + use stark::constraint_ir::{ConstraintArtifact, eval_program_verifier}; + use stark::frame::Frame; + use stark::table::TableView; + use stark::traits::TransitionEvaluationContext; + + let opts = options(); + let airs = LfmAirs::new_with_hasher(&[[0u8; 32]; NUM_LFM_CHIPS], &opts, 1, KIND, ChipSet::FULL); + let refs = airs.air_refs(); + let air = refs + .iter() + .find(|a| a.name() == "LFM_HASH") + .expect("LFM_HASH in the set"); + assert_eq!( + air.constraints_meta().len(), + SOCKET_FORM.num_constraints() + 4 + ); + + let artifact = ConstraintArtifact::capture(&**air); + let bytes = artifact.to_bytes().expect("serialize"); + let artifact = ConstraintArtifact::from_bytes(&bytes).expect("deserialize"); + let prog = artifact.program(); + let n = prog.roots.len(); + let an = analyze(&artifact); + + // The differential program (constraint_tests::differential_program). + let mut b = LfmBuilder::new(); + let frame_arena = b.declare_arena(ood_frame_words(&artifact)); + let (steps, words) = hint_ood_frame(&mut b, &artifact, frame_arena, 0); + assert_eq!(words, ood_frame_words(&artifact)); + let shape = &artifact.shape; + let num_uniforms = 2 + (shape.max_bus_elements as usize + 2) + 1; + let uniform_arena = b.declare_arena(num_uniforms as u32); + let mut next = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(uniform_arena, next).as_ext(); + next += 1; + c + }; + let rap_challenges = vec![take(&mut b), take(&mut b)]; + let alpha_powers: Vec<_> = (0..shape.max_bus_elements as usize + 2) + .map(|_| take(&mut b)) + .collect(); + let table_offset = take(&mut b); + let ood = OodOperands { + steps, + main_width: shape.main_width as usize, + rap_challenges, + alpha_powers, + table_offset, + }; + let evals = emit_analyzed(&mut b, &an, &ood); + for e in &evals { + b.public(e.as_cell()); + } + let program = compile(b.finish()); + super::validator::validate(&program).expect("admissible"); + + let main_width = shape.main_width as usize; + let width = main_width + shape.aux_width as usize; + let num_steps = shape.transition_offsets.len().max(1); + let mut seed = 0x5EED_0001u64; + let mut rnd = || { + seed = seed.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = seed; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + }; + for trial in 0..3 { + let mut fp3 = || FEE::new([FE::from(rnd()), FE::from(rnd()), FE::from(rnd())]); + let ostep: Vec> = (0..num_steps) + .map(|offset| { + (0..width) + .map(|col| { + if offset == 0 || shape.next_row_columns.contains(&(col as u32)) { + fp3() + } else { + FEE::zero() + } + }) + .collect() + }) + .collect(); + let rap: Vec = vec![fp3(), fp3()]; + let alphas: Vec = (0..shape.max_bus_elements as usize + 2) + .map(|_| fp3()) + .collect(); + let offset_v = fp3(); + + let mut frame_words = Vec::new(); + for (offset, step) in ostep.iter().enumerate() { + for (col, v) in step.iter().enumerate() { + if offset == 0 || shape.next_row_columns.contains(&(col as u32)) { + frame_words.push(ext_word(v)); + } + } + } + let uniforms: Vec<_> = rap + .iter() + .chain(&alphas) + .chain(std::iter::once(&offset_v)) + .map(ext_word) + .collect(); + let exec = execute(&program, &[frame_words, uniforms], &HasherKind::Rpx).expect("executes"); + + let frame = Frame::::new( + ostep + .iter() + .map(|s| { + TableView::new( + vec![s[..main_width].to_vec()], + vec![s[main_width..].to_vec()], + ) + }) + .collect(), + ); + let ctx = + TransitionEvaluationContext::::new_verifier(&frame, &rap, &alphas, &offset_v); + let mut interp = vec![FEE::zero(); n]; + eval_program_verifier(&prog, &ctx, &mut interp); + let folder = air.compute_transition(&ctx); + assert!( + interp.iter().all(|v| *v != FEE::zero()), + "trial {trial}: a random frame leaves every constraint nonzero" + ); + assert_eq!(folder.len(), n); + for c in 0..n { + let cell = exec.memory.get(evals[c].addr()).expect("written"); + let got = word_as_ext(&cell).expect("ext"); + assert_eq!( + got, interp[c], + "trial {trial}: constraint {c}: guest vs interpreter" + ); + assert_eq!( + folder[c], interp[c], + "trial {trial}: constraint {c}: host folder vs interpreter" + ); + } + } +} + +/// Point (a) of the review on the GPU aux path's encoding: the fingerprint descriptor the +/// device kernel consumes (its host mirror `eval_fingerprint`) gives every +/// socket interaction the fingerprint the host computes from the bus values. +#[cfg(feature = "cuda")] +#[test] +fn the_gpu_fingerprint_descriptor_matches_the_host() { + use math::field::element::FieldElement; + use stark::logup_gpu::{build_fingerprint_descriptor, eval_fingerprint}; + let input: [FE; 16] = core::array::from_fn(|i| FE::from(0xFFFF_FFFF_0000_0000 - i as u64)); + let mut row = vec![FE::zero(); SOCKET_FORM.num_columns()]; + row[cols::IS_REAL] = FE::one(); + for k in 0..4 { + row[cols::IN_ADDR0 + k] = FE::from(10 + k as u64); + row[cols::OUT_ADDR0 + k] = FE::from(20 + k as u64); + } + fill_row(SOCKET_FORM, input, &mut row); + let ix = super::p1w16_socket::bus_interactions(SOCKET_FORM); + let d = build_fingerprint_descriptor(&ix); + let z = FieldElement::::new([FE::from(3u64), FE::from(5u64), FE::from(7u64)]); + let alpha = FieldElement::::new([FE::from(11u64), FE::from(13u64), FE::from(17u64)]); + let powers: Vec> = (0..d.alpha_powers_len.max(6)) + .scan(FieldElement::::one(), |p, _| { + let cur = *p; + *p = &*p * α + Some(cur) + }) + .collect(); + for (k, i) in ix.iter().enumerate() { + let mut lc = FieldElement::::from(i.bus_id); + for (j, v) in i.values.iter().enumerate() { + let val = v.combine_from(|c| row[c].to_extension::()); + lc += &val[0] * &powers[j + 1]; + } + let want = &z - &lc; + let got = eval_fingerprint(&d, k, |c| &row[c], &powers, &z); + assert_eq!(got, want, "interaction {k}"); + } + let max_coef = d.term_coef.iter().copied().max().unwrap(); + assert_eq!( + max_coef, 101, + "the OnBus coefficients reach the descriptor as the MDS entries" + ); +} diff --git a/prover/src/lfm/p1w16_socket.rs b/prover/src/lfm/p1w16_socket.rs new file mode 100644 index 000000000..c4de98f07 --- /dev/null +++ b/prover/src/lfm/p1w16_socket.rs @@ -0,0 +1,430 @@ +//! The width-16 `LFM_HASH` socket: ZisK's Poseidon1 permutation +//! ([`crypto::hash::poseidon1_w16`]) behind a four-cell-in, four-cell-out +//! memory contract. +//! +//! The machine's socket today is twelve felts (three cells), which a width-16 +//! permutation does not fit. ZisK's constructions over this permutation need no +//! more than one shape: +//! +//! | construction | input cells | output cells read | +//! |---|---|---| +//! | 4-ary node (`compress4`) | the four children | cell 0 (the digest) | +//! | leaf block (`linear_hash`) | three rate cells + the carried digest (the width tag `[len, LEAF_DOMAIN, 0, 0]` first) | cell 0 | +//! | transcript update | three rate cells + the previous state's cell 0 | all four (every lane is squeezed) | +//! +//! So the socket has ONE mode: a row reads four cells, permutes all sixteen +//! lanes, and offers each output cell to memory with its own multiplicity +//! (zero for an output nobody reads). No lane is a constant the chip injects — +//! ZisK's instance has no capacity IV and no domain tag — so there are no +//! capacity-copy columns and no unread input slots to pin. +//! +//! # Columns +//! +//! Preprocessed ([`cols::PREP_WIDTH`] = 13, RPX's width): four input and four +//! output addresses, `IS_REAL`, four output multiplicities. +//! Value: `IN0..16`, then per round the compact S-box witness — `x³ = a³` and +//! `x⁷ = (x³)²·a` for every S-boxed lane, both degree 3 — with every round's +//! S-box input carried as an affine form in the earlier `x⁷` columns and `IN`, +//! never committed. Under [`OutForm::Columns`] the sixteen output lanes are +//! committed as `OUT0..16`; under [`OutForm::OnBus`] they are not columns at +//! all: the last round is a full round followed by the MDS, so each output +//! lane is a fixed small-integer combination of that round's sixteen `x⁷` +//! columns, and the output tokens carry exactly that combination. +//! +//! The round constants are scaled by `IS_REAL`, so an all-zero padding row +//! satisfies every constraint and emits no bus token; an output multiplicity +//! on a padding row is rejected. + +use stark::config::Commitment; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, LinearTerm, Multiplicity, + Packing, +}; +use stark::proof::options::ProofOptions; + +use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField}; +use crypto::hash::poseidon1_w16::{ + self as p1, NUM_ROUNDS, STATE_FELTS, constants::MDS_CIRC_ROW, constants::ROUND_CONSTANTS, + is_full_round, +}; +use math::field::traits::IsPrimeField; + +use super::airs::LfmAir; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// Felts per memory cell. +pub const CELL_FELTS: usize = 4; +/// Cells a row reads, and cells it may write. +pub const CELLS: usize = STATE_FELTS / CELL_FELTS; + +/// Column offsets. +pub mod cols { + use super::{CELLS, STATE_FELTS}; + + /// First input address; cell `k` is read at `IN_ADDR0 + k`. + pub const IN_ADDR0: usize = 0; + /// First output address; cell `k` is written at `OUT_ADDR0 + k`. + pub const OUT_ADDR0: usize = IN_ADDR0 + CELLS; + /// 1 on a permutation row, 0 on padding: the inputs' read multiplicity. + pub const IS_REAL: usize = OUT_ADDR0 + CELLS; + /// First output multiplicity: how many reads cell `k`'s write serves. + pub const MULT0: usize = IS_REAL + 1; + /// Preprocessed columns (the instruction column group). + pub const PREP_WIDTH: usize = MULT0 + CELLS; + /// The sixteen input lanes, cell-major. + pub const IN0: usize = PREP_WIDTH; + /// The value prefix every [`super::OutForm`] shares. + pub const PREFIX_END: usize = IN0 + STATE_FELTS; +} + +// The socket shares the twelve-felt socket's instruction column group width, so +// the compiler's one `LFM_HASH` group serves both. +const _: () = assert!(cols::PREP_WIDTH == super::layout::hash::PREP_WIDTH); + +/// Where the sixteen output lanes live. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OutForm { + /// Committed as sixteen `OUT` columns, each constrained to its affine form. + Columns, + /// Not committed: each output token lane is the last round's MDS + /// combination of its sixteen `x⁷` columns. + OnBus, +} + +/// The form the machine wires (`HasherKind::Poseidon1W16`'s `LFM_HASH`): the +/// outputs on the bus. BIG 664: 1.042–1.049× RPX's prove time per +/// permutation, against 1.067–1.094× with the outputs in columns. +pub const SOCKET_FORM: OutForm = OutForm::OnBus; + +/// S-boxed lanes in round `r`. +pub const fn sboxed_lanes(r: usize) -> usize { + if is_full_round(r) { STATE_FELTS } else { 1 } +} + +/// LogUp aux columns: the eight interactions in pairs. +pub const AUX_COLUMNS: usize = (2 * CELLS).div_ceil(2); +/// Base-field cells per aux column (a cubic-extension accumulator). +pub const CELLS_PER_AUX_COLUMN: usize = 3; + +impl OutForm { + /// First `OUT` column ([`OutForm::Columns`] only). + pub const fn out0(self) -> usize { + cols::PREFIX_END + } + + /// First round-witness column. + pub const fn rounds(self) -> usize { + match self { + OutForm::Columns => cols::PREFIX_END + STATE_FELTS, + OutForm::OnBus => cols::PREFIX_END, + } + } + + /// First column of round `r`'s witness block (`2·sboxed_lanes(r)` wide). + pub const fn block(self, r: usize) -> usize { + let mut off = self.rounds(); + let mut i = 0; + while i < r { + off += 2 * sboxed_lanes(i); + i += 1; + } + off + } + + /// `x³` of `lane` in round `r`. + pub const fn x3(self, r: usize, lane: usize) -> usize { + self.block(r) + lane + } + + /// `x⁷` of `lane` in round `r`. + pub const fn x7(self, r: usize, lane: usize) -> usize { + self.block(r) + sboxed_lanes(r) + lane + } + + /// The chip's width, preprocessed prefix included. + pub const fn num_columns(self) -> usize { + self.block(NUM_ROUNDS) + } + + /// Value columns: the census's `main_cols`. + pub const fn value_columns(self) -> usize { + self.num_columns() - cols::PREP_WIDTH + } + + /// Base-field cells per permutation: value columns plus the aux columns. + pub const fn cells_per_permutation(self) -> usize { + self.value_columns() + CELLS_PER_AUX_COLUMN * AUX_COLUMNS + } + + /// Constraints: `IS_REAL` booleanity, one per output multiplicity (zero + /// unless the row is real), two per S-box, and under [`OutForm::Columns`] + /// one per `OUT` lane. + pub const fn num_constraints(self) -> usize { + let mut n = 1 + CELLS; + let mut r = 0; + while r < NUM_ROUNDS { + n += 2 * sboxed_lanes(r); + r += 1; + } + match self { + OutForm::Columns => n + STATE_FELTS, + OutForm::OnBus => n, + } + } +} + +/// The MDS entry `M[o][i]` of Plonky3's circulant. +const fn mds_entry(o: usize, i: usize) -> u64 { + MDS_CIRC_ROW[(i + STATE_FELTS - o) % STATE_FELTS] +} + +/// The socket's eight interactions: the four input cells received with +/// multiplicity `IS_REAL`, the four output cells sent with their own +/// multiplicities. The `LfmMem` token is `(addr, v0, v1, v2, v3)`. +pub fn bus_interactions(form: OutForm) -> Vec { + let direct = |c: usize| BusValue::Packed { + start_column: c, + packing: Packing::Direct, + }; + let mut v = Vec::with_capacity(2 * CELLS); + for k in 0..CELLS { + let mut token = vec![direct(cols::IN_ADDR0 + k)]; + token.extend((0..CELL_FELTS).map(|j| direct(cols::IN0 + CELL_FELTS * k + j))); + v.push(BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + token, + )); + } + for k in 0..CELLS { + let mut token = vec![direct(cols::OUT_ADDR0 + k)]; + for j in 0..CELL_FELTS { + let lane = CELL_FELTS * k + j; + token.push(match form { + OutForm::Columns => direct(form.out0() + lane), + OutForm::OnBus => BusValue::Linear( + (0..STATE_FELTS) + .map(|i| LinearTerm::Column { + coefficient: mds_entry(lane, i) as i64, + column: form.x7(NUM_ROUNDS - 1, i), + }) + .collect(), + ), + }); + } + v.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT0 + k), + token, + )); + } + v +} + +/// The socket's AIR under `options`, its instruction column group committed +/// at `root`. +pub fn air( + form: OutForm, + options: &ProofOptions, + root: Commitment, +) -> LfmAir { + AirWithBuses::new( + form.num_columns(), + AuxiliaryTraceBuildData { + interactions: bus_interactions(form), + }, + options, + 1, + P1W16SocketConstraints { form }, + ) + .with_name("LFM_HASH16") + .with_preprocessed(root, cols::PREP_WIDTH) +} + +/// Runtime override of the lean upload: 0 = the environment's choice, 1 = off, +/// 2 = on. +static LEAN_UPLOAD_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +/// Force the lean upload on (`Some(true)`), off (`Some(false)`) or back to the +/// environment's choice (`None`). It changes which bytes cross to the card, +/// never the trace or the proof. +pub fn set_lean_upload(on: Option) { + let v = match on { + None => 0, + Some(false) => 1, + Some(true) => 2, + }; + LEAN_UPLOAD_OVERRIDE.store(v, std::sync::atomic::Ordering::Relaxed); +} + +/// Whether the socket's trace crosses to the card lean: the preprocessed prefix +/// and the sixteen input lanes only, the round witness rebuilt on the card +/// (`stark::trace::MainDevProducer::poseidon1_w16_rows`) — 29 of the 329 words a +/// row. On by default; `LAMBDA_VM_LFM_LEAN_UPLOAD=0` uploads the whole row as +/// before (an A/B knob: the proof bytes do not depend on it). +pub fn lean_upload_enabled() -> bool { + match LEAN_UPLOAD_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + static ENV: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENV.get_or_init(|| { + std::env::var("LAMBDA_VM_LFM_LEAN_UPLOAD") + .map(|v| v != "0") + .unwrap_or(true) + }) +} + +/// The device producer for a socket trace of `real_rows` permutation rows +/// (the rest padding) under `form`: the R1 main commit uploads each row's +/// prefix and inputs and rebuilds the witness on the card. +#[cfg(feature = "cuda")] +pub fn lean_upload_producer(form: OutForm, real_rows: usize) -> stark::trace::MainDevProducer { + stark::trace::MainDevProducer::poseidon1_w16_rows( + real_rows, + cols::PREP_WIDTH, + form.num_columns(), + form.rounds(), + ) +} + +/// Writes a permutation row's value columns for `input` (the preprocessed +/// prefix is the caller's) and returns the permutation's output, which is the +/// host reference's [`p1::permute`]. +pub fn fill_row(form: OutForm, input: [FE; STATE_FELTS], row: &mut [FE]) -> [FE; STATE_FELTS] { + row[cols::IN0..cols::IN0 + STATE_FELTS].copy_from_slice(&input); + let mut s = input; + for (r, rc) in ROUND_CONSTANTS.iter().enumerate() { + let mut f: [FE; STATE_FELTS] = core::array::from_fn(|i| s[i] + FE::from(rc[i])); + for (lane, a) in f.iter_mut().enumerate().take(sboxed_lanes(r)) { + let x3 = *a * *a * *a; + let x7 = x3 * x3 * *a; + row[form.x3(r, lane)] = x3; + row[form.x7(r, lane)] = x7; + *a = x7; + } + s = p1::mds(&f); + } + if form == OutForm::Columns { + row[form.out0()..form.out0() + STATE_FELTS].copy_from_slice(&s); + } + s +} + +/// An affine form over the row: `Σ col[c]·main(c) + real·IS_REAL`. +#[derive(Clone)] +struct Affine { + col: Vec, + real: FE, +} + +impl Affine { + fn zero(width: usize) -> Self { + Self { + col: vec![FE::zero(); width], + real: FE::zero(), + } + } + + fn column(width: usize, c: usize) -> Self { + let mut a = Self::zero(width); + a.col[c] = FE::one(); + a + } + + fn add_scaled(&mut self, other: &Affine, k: FE) { + for (x, y) in self.col.iter_mut().zip(&other.col) { + *x += k * y; + } + self.real += k * other.real; + } + + fn expr>(&self, b: &B, is_real: &B::Expr) -> B::Expr { + let mut acc = b.const_base(canon(&self.real)) * is_real.clone(); + for (c, k) in self.col.iter().enumerate() { + if *k != FE::zero() { + acc = acc + b.const_base(canon(k)) * b.main(0, c); + } + } + acc + } +} + +fn canon(x: &FE) -> u64 { + GoldilocksField::canonical(x.value()) +} + +/// The socket's constraint set. +pub struct P1W16SocketConstraints { + pub form: OutForm, +} + +impl ConstraintSet for P1W16SocketConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let form = self.form; + let width = form.num_columns(); + let is_real = b.main(0, cols::IS_REAL); + let one = b.one(); + let not_real = one - is_real.clone(); + b.emit_base(0, is_real.clone() * not_real.clone()); + // An output is offered only by a real row. A padding row reads nothing + // (its `IN` is free) and computes the permutation without round + // constants, so a write from one would put a prover-chosen value in + // memory. The program builder never emits one; this makes it + // unprovable rather than merely unemitted. + for k in 0..CELLS { + b.emit_base(1 + k, b.main(0, cols::MULT0 + k) * not_real.clone()); + } + let mut idx = 1 + CELLS; + + let mut state: Vec = (0..STATE_FELTS) + .map(|i| Affine::column(width, cols::IN0 + i)) + .collect(); + for (r, rc) in ROUND_CONSTANTS.iter().enumerate() { + // a_i = state_i + rc[r][i]·IS_REAL; S-boxed lanes become their x⁷. + let mut f: Vec = state + .iter() + .zip(rc) + .map(|(s, c)| { + let mut a = s.clone(); + a.real += FE::from(*c); + a + }) + .collect(); + for (lane, a) in f.iter_mut().enumerate().take(sboxed_lanes(r)) { + let a_e = a.expr(b, &is_real); + let x3 = b.main(0, form.x3(r, lane)); + let x7 = b.main(0, form.x7(r, lane)); + b.emit_base(idx, x3.clone() - a_e.clone() * a_e.clone() * a_e.clone()); + b.emit_base(idx + 1, x7 - x3.clone() * x3 * a_e); + idx += 2; + *a = Affine::column(width, form.x7(r, lane)); + } + state = (0..STATE_FELTS) + .map(|o| { + let mut acc = Affine::zero(width); + for (i, fi) in f.iter().enumerate() { + acc.add_scaled(fi, FE::from(mds_entry(o, i))); + } + acc + }) + .collect(); + } + if form == OutForm::Columns { + for (j, s) in state.iter().enumerate() { + let out = b.main(0, form.out0() + j); + b.emit_base(idx, out - s.expr(b, &is_real)); + idx += 1; + } + } + debug_assert_eq!(idx, form.num_constraints()); + } +} diff --git a/prover/src/lfm/p1w16_socket_tests.rs b/prover/src/lfm/p1w16_socket_tests.rs new file mode 100644 index 000000000..9af874181 --- /dev/null +++ b/prover/src/lfm/p1w16_socket_tests.rs @@ -0,0 +1,342 @@ +//! The width-16 socket: its widths, what it accepts and rejects, its bus +//! tokens, and (on the card) what one permutation costs an LFM prove against +//! RPX's `LFM_HASH`. + +use math::field::element::FieldElement; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, ProverEvalFolder, RootKind, num_base_from_meta, +}; +use stark::frame::Frame; +use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, BusValue, Multiplicity}; +use stark::proof::options::ProofOptions; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; +use crypto::hash::poseidon1_w16 as p1; +use math::field::traits::IsPrimeField; + +use super::airs::LfmAir; +use super::p1w16_socket::*; + +type Gl = GoldilocksField; +type Gl3 = GoldilocksExtension; + +const FORMS: [OutForm; 2] = [OutForm::Columns, OutForm::OnBus]; + +/// Pinned as literals from the closed forms, so a layout bug cannot agree with +/// itself: 16 `IN` + 8·32 + 22·2 S-box witnesses (+ 16 `OUT`). +const ONBUS_VALUE_COLUMNS: usize = 316; +const COLUMNS_VALUE_COLUMNS: usize = 332; +/// Plus four LogUp aux columns at three cells each. +const ONBUS_CELLS: usize = 328; +const COLUMNS_CELLS: usize = 344; +/// 1 + 4 + 300 (+ 16). +const ONBUS_CONSTRAINTS: usize = 305; +const COLUMNS_CONSTRAINTS: usize = 321; + +fn evaluate(form: OutForm, row: &[FE]) -> Vec { + let set = P1W16SocketConstraints { form }; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out +} + +fn violations(form: OutForm, row: &[FE]) -> Vec { + evaluate(form, row) + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +fn sample_input() -> [FE; p1::STATE_FELTS] { + core::array::from_fn(|i| FE::from(0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1))) +} + +/// A real permutation row: the preprocessed prefix says `IS_REAL`, the value +/// columns are the honest witness. +fn real_row(form: OutForm, input: [FE; p1::STATE_FELTS]) -> (Vec, [FE; p1::STATE_FELTS]) { + let mut row = vec![FE::zero(); form.num_columns()]; + row[cols::IS_REAL] = FE::one(); + let out = fill_row(form, input, &mut row); + (row, out) +} + +/// A bus value read off a row by the production evaluator, +/// `BusValue::combine_from` (the LogUp fingerprint's, with its own i64 +/// coefficient conversion): one felt per socket value. +fn bus_value(v: &BusValue, row: &[FE]) -> FE { + let got = v.combine_from(|c| row[c]); + assert_eq!(got.len(), 1, "a socket value is one felt"); + got[0] +} + +#[test] +fn the_widths_are_the_closed_forms() { + assert_eq!(cols::PREP_WIDTH, 13); + assert_eq!( + cols::PREP_WIDTH, + crate::lfm::layout::hash::PREP_WIDTH, + "the same instruction group width as the twelve-felt socket" + ); + assert_eq!(OutForm::OnBus.value_columns(), ONBUS_VALUE_COLUMNS); + assert_eq!(OutForm::Columns.value_columns(), COLUMNS_VALUE_COLUMNS); + assert_eq!(ONBUS_VALUE_COLUMNS, 16 + 8 * 32 + 22 * 2); + assert_eq!(COLUMNS_VALUE_COLUMNS, ONBUS_VALUE_COLUMNS + 16); + assert_eq!(OutForm::OnBus.cells_per_permutation(), ONBUS_CELLS); + assert_eq!(OutForm::Columns.cells_per_permutation(), COLUMNS_CELLS); + assert_eq!(OutForm::OnBus.num_constraints(), ONBUS_CONSTRAINTS); + assert_eq!(OutForm::Columns.num_constraints(), COLUMNS_CONSTRAINTS); + for form in FORMS { + assert_eq!(bus_interactions(form).len(), 8); + } +} + +/// Every value column is handed out exactly once. +#[test] +fn every_value_column_is_assigned_exactly_once() { + for form in FORMS { + let mut seen = vec![0usize; form.num_columns()]; + for n in &mut seen[cols::IN0..cols::IN0 + p1::STATE_FELTS] { + *n += 1; + } + if form == OutForm::Columns { + for n in &mut seen[form.out0()..form.out0() + p1::STATE_FELTS] { + *n += 1; + } + } + for r in 0..p1::NUM_ROUNDS { + for lane in 0..sboxed_lanes(r) { + seen[form.x3(r, lane)] += 1; + seen[form.x7(r, lane)] += 1; + } + } + for (c, &n) in seen.iter().enumerate() { + let want = usize::from(c >= cols::PREP_WIDTH); + assert_eq!(n, want, "{form:?}: column {c} claimed {n} times"); + } + } +} + +#[test] +fn every_constraint_is_degree_three_or_less_and_some_reach_three() { + for form in FORMS { + let set = P1W16SocketConstraints { form }; + let meta = ConstraintSet::::meta(&set); + assert_eq!(meta.len(), form.num_constraints(), "{form:?}"); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "{form:?}: meta dense and ordered"); + assert_eq!(m.kind, RootKind::Base); + } + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert!(degrees.iter().all(|&(_, d)| d <= 3), "{form:?}"); + assert_eq!(degrees.iter().map(|&(_, d)| d).max(), Some(3), "{form:?}"); + } +} + +/// An honest row satisfies every constraint, and the filler's output is the +/// host reference's permutation (and the first known answer). +#[test] +fn honest_rows_satisfy_every_constraint_and_match_the_host_reference() { + const KAT0_OUT_LANE0: u64 = 9_350_316_517_402_464_675; + for form in FORMS { + for input in [ + sample_input(), + core::array::from_fn(|i| FE::from(i as u64)), + [FE::zero(); p1::STATE_FELTS], + [-FE::one(); p1::STATE_FELTS], + ] { + let (row, out) = real_row(form, input); + assert_eq!(violations(form, &row), Vec::::new(), "{form:?}"); + assert_eq!(out, p1::permute(input), "{form:?}"); + } + let (_, out) = real_row(form, core::array::from_fn(|i| FE::from(i as u64))); + assert_eq!(out[0], FE::from(KAT0_OUT_LANE0), "{form:?}"); + } +} + +#[test] +fn an_all_zero_padding_row_satisfies_every_constraint() { + for form in FORMS { + let row = vec![FE::zero(); form.num_columns()]; + assert_eq!(violations(form, &row), Vec::::new(), "{form:?}"); + } +} + +/// A padding row cannot carry a permutation: with `IS_REAL = 0` the round +/// constants drop out, so an honest witness is rejected. +#[test] +fn a_real_witness_under_is_real_zero_is_rejected() { + for form in FORMS { + let (mut row, _) = real_row(form, sample_input()); + row[cols::IS_REAL] = FE::zero(); + assert!(!violations(form, &row).is_empty(), "{form:?}"); + } +} + +/// `IS_REAL` is a bit. +#[test] +fn a_non_boolean_is_real_is_rejected() { + for form in FORMS { + let mut row = vec![FE::zero(); form.num_columns()]; + row[cols::IS_REAL] = FE::from(2u64); + assert!(violations(form, &row).contains(&0), "{form:?}"); + } +} + +/// A padding row cannot write: an output multiplicity under `IS_REAL = 0` +/// is rejected (its `IN` is unread, so the write would be prover-chosen). +#[test] +fn an_output_multiplicity_on_a_padding_row_is_rejected() { + for form in FORMS { + for k in 0..CELLS { + let mut row = vec![FE::zero(); form.num_columns()]; + row[cols::MULT0 + k] = FE::one(); + assert_eq!(violations(form, &row), vec![1 + k], "{form:?} cell {k}"); + // The same multiplicity on a real row is fine. + let (mut real, _) = real_row(form, sample_input()); + real[cols::MULT0 + k] = FE::from(3u64); + assert_eq!( + violations(form, &real), + Vec::::new(), + "{form:?} cell {k}" + ); + } + } +} + +/// Mutation: perturbing ANY value column of an honest row breaks some +/// constraint, so no column is free. +#[test] +fn every_value_column_is_load_bearing() { + for form in FORMS { + let (row, _) = real_row(form, sample_input()); + for c in cols::PREP_WIDTH..form.num_columns() { + let mut bad = row.clone(); + bad[c] += FE::one(); + assert!( + !violations(form, &bad).is_empty(), + "{form:?}: column {c} is unconstrained" + ); + } + } +} + +/// The tokens: four reads of the input cells at `IS_REAL`, four writes of the +/// output cells at their own multiplicities, and each written word IS the +/// permutation's output cell — under `OnBus` that is the MDS combination the +/// sender carries, under `Columns` the `OUT` columns. +#[test] +fn the_tokens_carry_the_input_and_output_cells() { + use stark::lookup::BusInteraction; + for form in FORMS { + let input = sample_input(); + let (mut row, out) = real_row(form, input); + for k in 0..CELLS { + row[cols::IN_ADDR0 + k] = FE::from(100 + k as u64); + row[cols::OUT_ADDR0 + k] = FE::from(200 + k as u64); + } + let ix: Vec = bus_interactions(form); + for (k, i) in ix.iter().enumerate() { + let cell = k % CELLS; + let token: Vec = i.values.iter().map(|v| bus_value(v, &row)).collect(); + assert_eq!(token.len(), 1 + CELL_FELTS, "{form:?} interaction {k}"); + if k < CELLS { + assert!(!i.is_sender, "{form:?}: interaction {k} reads"); + assert!(matches!(i.multiplicity, Multiplicity::Column(c) if c == cols::IS_REAL)); + assert_eq!(token[0], FE::from(100 + cell as u64)); + assert_eq!(&token[1..], &input[4 * cell..4 * cell + 4], "{form:?}"); + } else { + assert!(i.is_sender, "{form:?}: interaction {k} writes"); + assert!( + matches!(i.multiplicity, Multiplicity::Column(c) if c == cols::MULT0 + cell) + ); + assert_eq!(token[0], FE::from(200 + cell as u64)); + assert_eq!(&token[1..], &out[4 * cell..4 * cell + 4], "{form:?}"); + } + } + } +} + +/// A row whose last round is perturbed writes a different output token: the +/// `OnBus` senders read the last round's `x⁷` columns, which the S-box +/// constraints pin, so a forged output must break a constraint. +#[test] +fn a_forged_output_cell_breaks_a_constraint() { + let form = OutForm::OnBus; + let (row, _) = real_row(form, sample_input()); + let last = p1::NUM_ROUNDS - 1; + for lane in 0..p1::STATE_FELTS { + let mut bad = row.clone(); + bad[form.x7(last, lane)] += FE::one(); + assert!(!violations(form, &bad).is_empty(), "x7 lane {lane}"); + } +} + +/// The RPX `LFM_HASH` AIR exactly as the machine builds it. +pub(super) fn rpx_air( + options: &ProofOptions, + root: stark::config::Commitment, +) -> LfmAir { + use super::chips::hash; + use super::hash::HasherKind; + AirWithBuses::new( + hash::num_columns(HasherKind::Rpx), + AuxiliaryTraceBuildData { + interactions: hash::bus_interactions(HasherKind::Rpx), + }, + options, + 1, + hash::HashConstraints::RPX, + ) + .with_name("LFM_HASH") + .with_preprocessed(root, crate::lfm::layout::hash::PREP_WIDTH) +} + +/// The evaluation cost the cell census does not see: each AIR's full +/// composition program (its constraints plus the LogUp terms) as the card +/// lowers it. Informational (`--nocapture`). +#[test] +fn composition_program_sizes() { + use stark::constraint_ir::DeviceProgram; + use stark::traits::AIR; + let opts = super::proof::aggregation_wrap_options(); + let report = |name: &str, prog: &stark::constraint_ir::ConstraintProgram| { + let dev = DeviceProgram::lower(prog); + println!( + "PROGRAM {name:<14} nodes {} lowered {} roots {}", + prog.len(), + dev.nodes.len(), + dev.roots.len(), + ); + dev.nodes.len() + }; + let rpx = report("rpx", rpx_air(&opts, [0u8; 32]).constraint_program()); + assert!(rpx > 0); + for form in FORMS { + let n = report( + &format!("p1w16 {form:?}"), + air(form, &opts, [0u8; 32]).constraint_program(), + ); + assert!(n > 0); + } +} + +/// Canonical `u64` of a felt (test printing). +#[allow(dead_code)] +fn canon(x: &FE) -> u64 { + Gl::canonical(x.value()) +} diff --git a/prover/src/lfm/per_table_aggregator.rs b/prover/src/lfm/per_table_aggregator.rs new file mode 100644 index 000000000..a6f99e569 --- /dev/null +++ b/prover/src/lfm/per_table_aggregator.rs @@ -0,0 +1,1191 @@ +//! The aggregation NODE's building block: a per-table VERIFY LEG — the emitted +//! verifier of one child LFM proof — plus the cross-child binding legs. +//! +//! # What a leg is +//! +//! A leg is the machine twin of [`super::proof::verify_against_chunked`], which +//! is four steps and no more: absorb the LFM statement, fork the statement-bound +//! state and replay Phase A to recover the shared LogUp pair, recompute the +//! `LfmPublic` balance from the CLAIMED public words, and verify every +//! sub-proof against it. [`emit_leg`] emits exactly those four, so the leg and +//! the host verifier are two renderings of one contract rather than two +//! implementations of one idea. +//! +//! Nothing here is new cryptographic arithmetic. The spine is +//! [`super::epoch::emit_table_challenges`], the per-sub-proof verification is +//! [`super::epoch_verify::emit_table_verification`], Phase A is +//! [`super::statement_replay::replay_phase_a`] and the closure is +//! [`super::logup::emit_bus_closure`] — every one already gated by the epoch +//! wrap and by the global-memory leg. This module contributes the LFM-shaped +//! statement, the public-word hinting under a canonicity guard, the balance +//! target, and the binding legs. +//! +//! # Why a node is uniform +//! +//! A child is a plain per-table `MultiProof` whichever level produced it — a +//! wrap of an epoch, or another node. So one emitter serves every level, and a +//! node's statement is its children's published words and roots. What is NOT +//! uniform is the node's IDENTITY: a leg absorbs its child's `program_id` as an +//! emit-time CONSTANT, and `program_id` is derived from the compiled program, so +//! a program that verified its own shape would need its own id inside its own +//! instruction stream. That fixed point is why each level is a distinct program. +//! +//! # What the bindings are for +//! +//! Verifying two children says nothing about their relationship. The chain is a +//! CHECK on published words and never a trust: one shared attestation id across +//! every child, each child's register fini vector equal to the next child's init +//! vector, and each child's epoch label pinned to its chain position as an +//! emit-time constant. Those are [`emit_chain_bindings`], and they are the whole +//! reason a tree of these proves something a bag of them does not. + +use stark::config::Commitment; + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, Felt, LfmBuilder}; +use super::constraints::Analysis; +use super::epoch::{RootCells, TableAbsorbs, TableChallengeShape, fork_table}; +use super::epoch_verify::{TableQueryArenas, TableVerifyShape}; +use super::instr::ArenaId; +use super::statement::{LFM_MACHINE_VERSION, LFM_STATEMENT_TAG}; +use super::statement_replay::{ + CommitOodDomain, PhaseAPreprocessed, PhaseATable, replay_phase_a_answered, +}; +use super::transcript_replay::{Candidate, TranscriptReplay, assert_canonical, candidate_to_felt}; + +/// `u32` halves one lane's canonical `u64` occupies — the unit +/// [`super::statement::absorb_lfm_statement`] appends a lane in. +const HALVES_PER_LANE: usize = 2; + +/// Lanes one published word carries. +const LANES_PER_WORD: usize = super::word::WORD_LANES; + +// ======================= the child's shape, at emit time ================== + +/// One sub-proof of a child, as the leg needs it. +/// +/// Every field is program SHAPE — derived from the AIR and the proof options, +/// never read off the proof at verify time. The one exception is the trace +/// length inside `verify`, which the prover chooses and which is program shape +/// here for the reason the arena schema makes it one: the node is emitted for a +/// specific child shape, and a child whose trace length disagreed would not fill +/// the arenas the node declares. +pub struct ChildTable<'a> { + /// What the fork absorbs and what challenges come out of it. + pub challenge: &'a TableChallengeShape, + /// What the verification legs open. + pub verify: &'a TableVerifyShape, + /// The constraint program, captured from the AIR. + pub analysis: &'a Analysis, + /// The preprocessed-columns commitment, when the AIR is preprocessed. + /// + /// An AIR-SET constant at emit time, exactly as production takes it + /// (`air.precomputed_commitment_for(layout)` at the table's leaf layout, + /// never the proof's copy). Interning it + /// here is what makes production's explicit proof-copy-equals-AIR-copy check + /// the ABSENCE of a second value in this machine rather than a comparison. + pub precomputed_root: Option<&'a Commitment>, +} + +/// One child proof, as the node's emitter needs it. +pub struct ChildShape<'a> { + /// The identity of the program that produced this child — a PROGRAM + /// CONSTANT of the node, which is what pins WHICH program the node accepts a + /// proof of. + pub program_id: &'a Commitment, + /// How many words the child publishes. Publish indices auto-increment from + /// zero (`LfmBuilder::public`), so the count is the whole layout. + pub num_public_words: usize, + /// The child's `ProofOptions::fri_final_poly_log_degree` — the statement's + /// last byte. + pub fri_final_poly_log_degree: u8, + /// The child's proof family's statement tag + /// (`hash_pin::ProofFamily::STATEMENT_TAG`), the statement's first call: + /// [`LFM_STATEMENT_TAG`] for the legacy pin. A program constant, like the + /// id. + pub statement_tag: &'static [u8], + /// The child's sub-proofs, in proof order. + pub tables: Vec>, +} + +impl ChildShape<'_> { + /// The commitment-time answers the child's Phase A absorbs: one per + /// answered column of every sub-proof. + pub fn num_answers(&self) -> usize { + self.tables.iter().map(|t| t.verify.num_answers()).sum() + } +} + +/// One sub-proof's shapes in a [`DerivedChild`]. +struct DerivedTable { + challenge: TableChallengeShape, + verify: TableVerifyShape, + analysis: Analysis, + precomputed_root: Option, +} + +/// A child's shapes derived from its ARTIFACTS — its program's AIR set and +/// heights — with no proof of it: what a verifier needs to derive a node's +/// program from the programs below it. [`Self::shape`] is the emitter's view. +pub struct DerivedChild { + program_id: Commitment, + num_public_words: usize, + fri_final_poly_log_degree: u8, + statement_tag: &'static [u8], + tables: Vec, +} + +impl DerivedChild { + /// The child whose program `artifacts` describes, proved under `opts` and + /// publishing `num_public_words` words (its layout's, a constant of the + /// tree). Every table shape is the AIR's at the artifacts' height + /// ([`TableChallengeShape::derive`], [`TableVerifyShape::derive`]); the + /// preprocessed roots are the AIR set's, as `verify_against_artifacts` + /// builds it. + pub fn from_artifacts( + artifacts: &super::registry::LfmArtifacts, + opts: &crate::ProofOptions, + num_public_words: usize, + ) -> Result { + Self::from_artifacts_under::( + artifacts, + opts, + num_public_words, + ) + } + + /// [`Self::from_artifacts`] for a child proved under the family `Fam`: + /// its trees are `Fam::Stark`'s (arity, cap) and its statement carries + /// `Fam`'s tag. `artifacts` must be built under `Fam::Stark` + /// (`registry::build_artifacts_under`). + pub fn from_artifacts_under( + artifacts: &super::registry::LfmArtifacts, + opts: &crate::ProofOptions, + num_public_words: usize, + ) -> Result { + let trees = stark::config::TreeGeometry::of::(); + let heights = super::airs::LfmAirs::log_heights_in_air_order(artifacts) + .ok_or("a child with KECCAK_RND chunks has workload heights")?; + let airs = super::airs::LfmAirs::for_artifacts(artifacts, opts)?; + let refs = airs.air_refs(); + if heights.len() != refs.len() { + return Err(format!("{} heights for {} AIRs", heights.len(), refs.len())); + } + let n = refs.len(); + let tables = refs + .iter() + .zip(heights) + .enumerate() + .map(|(idx, (air, h))| { + let rows = 1usize << h; + // An LFM proof is bound: every sub-proof's DEEP carries its + // commitment-time answers. + let (verify, analysis) = TableVerifyShape::derive_with(*air, rows, trees)?; + let verify = verify.with_commit_ood(*air); + let precomputed_root = if air.is_preprocessed() { + Some( + super::epoch_verify::layout_precomputed_commitment(*air, rows) + .ok_or_else(|| format!("table {idx}: no preprocessed root"))?, + ) + } else { + None + }; + Ok(DerivedTable { + challenge: TableChallengeShape::derive(*air, idx, n, rows), + verify, + analysis, + precomputed_root, + }) + }) + .collect::, String>>()?; + Ok(Self { + program_id: artifacts.program_id, + num_public_words, + fri_final_poly_log_degree: opts.fri_final_poly_log_degree, + statement_tag: Fam::STATEMENT_TAG, + tables, + }) + } + + /// The child as the node's emitter reads it. + pub fn shape(&self) -> ChildShape<'_> { + ChildShape { + program_id: &self.program_id, + num_public_words: self.num_public_words, + fri_final_poly_log_degree: self.fri_final_poly_log_degree, + statement_tag: self.statement_tag, + tables: self + .tables + .iter() + .map(|t| ChildTable { + challenge: &t.challenge, + verify: &t.verify, + analysis: &t.analysis, + precomputed_root: t.precomputed_root.as_ref(), + }) + .collect(), + } + } + + /// Each sub-proof's `log2` trace length, in proof order. + pub fn log2_trace_lengths(&self) -> Vec { + self.tables + .iter() + .map(|t| t.challenge.log2_trace_length) + .collect() + } +} + +// ==================== the emitted statement + publics ==================== + +/// One hinted public word of a child: the emit-time-constant index, the eight +/// hinted halves (absorbed by the statement), and the four lanes reassembled as +/// CANONICITY-GUARDED felts (consumed by the balance and the binding legs). +pub struct HintedPublicWord { + pub index: u32, + pub halves: Vec, + pub lanes: Vec, +} + +/// Hint a child's published words from `arena` (eight halves per word, the +/// serializer's layout) and reassemble each lane under the canonicity guard. +/// +/// The guard is the same `(lo, hi)` predicate the transcript replay's own +/// digest-to-felt path enforces, so a hinted half pair CANNOT alias a second +/// representation of the same felt while absorbing different bytes. Without it a +/// prover could absorb one byte string into the statement and hand the balance +/// and the binding legs a different value for the same word. +/// +/// ⚠ This is the node's per-word bill — eight hints, four guards, four +/// recombinations — and it is paid per word per child. What a child publishes is +/// therefore the size of the layer above it; see +/// `epoch_tests::Publishes`. +pub fn hint_public_words( + b: &mut LfmBuilder, + arena: ArenaId, + count: usize, +) -> Vec { + let mut cursor = 0u32; + (0..count) + .map(|index| { + let mut halves = Vec::with_capacity(LANES_PER_WORD * HALVES_PER_LANE); + let mut lanes = Vec::with_capacity(LANES_PER_WORD); + for _ in 0..LANES_PER_WORD { + let lo = b.hint_felt(arena, cursor); + let hi = b.hint_felt(arena, cursor + 1); + cursor += HALVES_PER_LANE as u32; + let c = Candidate { lo, hi }; + assert_canonical(b, c); + lanes.push(candidate_to_felt(b, c)); + halves.push(lo); + halves.push(hi); + } + HintedPublicWord { + index: index as u32, + halves, + lanes, + } + }) + .collect() +} + +/// The INVERSE of [`hint_public_words`]: a child's published words as the +/// eight-halves-per-word arena that function reads. +/// +/// ★ ONE COPY, reached by every fixture. `child_arena_words` builds exactly this +/// arena from a real harvest; a gate that drives an emitter over fabricated +/// published words needs the same layout, and a per-gate copy of "lo 32 bits +/// then hi 32 bits, four lanes, in order" is a rule free to drift from the +/// serializer it is supposed to mirror. +#[cfg(test)] +pub(super) fn publics_arena(words: &[super::word::LfmWord]) -> Vec { + use math::field::traits::IsPrimeField; + let mut out = Vec::with_capacity(LANES_PER_WORD * HALVES_PER_LANE * words.len()); + for w in words { + for lane in w { + let v: u64 = crate::tables::types::GoldilocksField::canonical(lane.value()); + out.push(super::word::base_word(FE::from(v & 0xFFFF_FFFF))); + out.push(super::word::base_word(FE::from(v >> 32))); + } + } + out +} + +/// Emits [`super::statement::absorb_lfm_statement`] byte for byte: the tag, the +/// child's program id (a PROGRAM CONSTANT), the machine version, the word count, +/// each word's emit-time-constant index and hinted lane halves, and the FRI +/// terminal byte. +pub fn emit_lfm_statement( + t: &mut TranscriptReplay, + program_id: &Commitment, + words: &[HintedPublicWord], + fri_final_poly_log_degree: u8, +) { + emit_lfm_statement_tagged( + t, + LFM_STATEMENT_TAG, + program_id, + words, + fri_final_poly_log_degree, + ); +} + +/// Emits [`super::statement::absorb_lfm_statement_tagged`]: the child +/// family's tag (a PROGRAM CONSTANT) in place of [`LFM_STATEMENT_TAG`], then +/// [`emit_lfm_statement`]'s other calls. The legacy pin's tag is +/// [`LFM_STATEMENT_TAG`]. +pub fn emit_lfm_statement_tagged( + t: &mut TranscriptReplay, + tag: &[u8], + program_id: &Commitment, + words: &[HintedPublicWord], + fri_final_poly_log_degree: u8, +) { + t.append_const_bytes(tag); + t.append_const_bytes(program_id); + t.append_const_bytes(&LFM_MACHINE_VERSION.to_le_bytes()); + t.append_const_bytes(&(words.len() as u64).to_le_bytes()); + for word in words { + t.append_const_bytes(&word.index.to_le_bytes()); + // ⚠ ONE CALL PER LANE, not one for the word. `absorb_lfm_statement` + // appends each lane's canonical `u64` separately, so a word is FIVE host + // calls — the index and four lanes — not two. A byte transcript + // concatenates and cannot tell the difference; an ALGEBRAIC one + // length-prefixes every call, so absorbing the eight halves in one go is + // a DIFFERENT transcript, and since the statement is absorbed first that + // means every challenge downstream. See `transcript_replay::Append`. + for lane in word.halves.chunks(HALVES_PER_LANE) { + t.append_halves_misaligned(lane); + } + } + t.append_const_bytes(&[fri_final_poly_log_degree]); +} + +/// The `LfmPublic` balance the leg's LogUp closure must reach — the machine twin +/// of `proof::expected_public_balance`: +/// `Σ_i 1/(z − (LfmPublic + index_i·α + Σ_l lane_l·α^{2+l}))`, with each division +/// by an interned one so a fingerprint collision with `z` is unprovable rather +/// than `0/0 = 1`. +pub fn emit_public_balance( + b: &mut LfmBuilder, + words: &[HintedPublicWord], + z: Ext, + alpha: Ext, +) -> Ext { + let bus = b.ext_const(&FEE::from(crate::tables::types::BusId::LfmPublic as u64)); + let one = b.ext_const(&FEE::one()); + // α¹..α⁵ — the index takes α, lane l takes α^{2+l}. + let mut powers = Vec::with_capacity(1 + LANES_PER_WORD); + powers.push(alpha); + for i in 1..=LANES_PER_WORD { + let next = b.emul(powers[i - 1], alpha); + powers.push(next); + } + let mut acc: Option = None; + for word in words { + let idx = b.felt_const(FE::from(word.index as u64)); + let idx_term = b.emul_base(powers[0], idx); + let mut linear = b.eadd(bus, idx_term); + for (l, lane) in word.lanes.iter().enumerate() { + let term = b.emul_base(powers[1 + l], *lane); + linear = b.eadd(linear, term); + } + let fingerprint = b.esub(z, linear); + let term = b.ediv(one, fingerprint); + acc = Some(match acc { + None => term, + Some(a) => b.eadd(a, term), + }); + } + acc.unwrap_or_else(|| b.ext_const(&FEE::zero())) +} + +// ============================== the verify leg ============================ + +/// The arenas one child's leg reads, in DECLARATION ORDER — which is absorb +/// order. The caller declares one set per child, in child order, before emitting +/// any leg, so the node's declaration order is its absorb order end to end. +pub struct LegArenas { + publics: ArenaId, + main_roots: ArenaId, + /// Every sub-proof's commitment-time answers, proof order, one cell each. + answers: ArenaId, + per_table: Vec, +} + +struct TableArenas { + aux_root: Option, + contribution: Option, + composition_root: ArenaId, + ood_current: ArenaId, + ood_next: ArenaId, + parts: ArenaId, + fri_roots: ArenaId, + fri_coeffs: ArenaId, + nonce: Option, + legs: TableQueryArenas, +} + +/// Declare one child's arenas. +pub fn declare_leg_arenas(b: &mut LfmBuilder, child: &ChildShape<'_>) -> LegArenas { + let per_root = RootCells::words_per_root(b); + let publics = + b.declare_arena((LANES_PER_WORD * HALVES_PER_LANE * child.num_public_words) as u32); + let main_roots = b.declare_arena(per_root * child.tables.len() as u32); + let answers = b.declare_arena(child.num_answers() as u32); + let per_table = child + .tables + .iter() + .map(|t| { + let c = t.challenge; + TableArenas { + aux_root: c.has_aux_root.then(|| b.declare_arena(per_root)), + contribution: c.has_contribution.then(|| b.declare_arena(1)), + composition_root: b.declare_arena(per_root), + ood_current: b.declare_arena((c.ood_current_dims.0 * c.ood_current_dims.1) as u32), + ood_next: b.declare_arena((c.ood_next_dims.0 * c.ood_next_dims.1) as u32), + parts: b.declare_arena(c.num_parts as u32), + fri_roots: b.declare_arena(per_root * c.fri.num_committed() as u32), + fri_coeffs: b.declare_arena(c.fri.num_terminal_coeffs() as u32), + nonce: (c.grinding_factor > 0).then(|| b.declare_arena(1)), + legs: super::epoch_verify::declare_table_arenas(b, t.verify), + } + }) + .collect(); + LegArenas { + publics, + main_roots, + answers, + per_table, + } +} + +/// How many arenas [`declare_leg_arenas`] declares for `child` under `wrap`: +/// a node's arenas are its children's legs', in child order, so these counts +/// are where each child's arenas begin and end. Children of one node need not +/// declare the same number: a child has a sub-proof per AIR, and its AIR set +/// follows its own program's heights (under the equal-height merge, a node's +/// children can hold different merged tables). +pub fn leg_arena_count(child: &ChildShape<'_>, wrap: super::edsl::WrapHash) -> usize { + let mut b = LfmBuilder::new().with_wrap_hash(wrap); + declare_leg_arenas(&mut b, child); + b.num_arenas() +} + +/// What a leg hands the node's binding layer. +pub struct LegCells { + /// The child's published words — index plus canonicity-guarded lanes. This + /// is the binding legs' entire input, and the only thing a node learns about + /// a child beyond "its proof verifies". + pub publics: Vec, + /// The child's own shared LogUp pair, exposed for the differential gates. + pub z_alpha: (Ext, Ext), +} + +/// Emit ONE child's complete verification. +/// +/// ⚠ ONE TRANSCRIPT PER CHILD. Each child proof was produced against a +/// transcript seeded by its OWN statement, so a node verifying two children runs +/// two independent replays: two Phase A's, two `(z, α)` pairs, two closures. A +/// shared spine that re-indexed every sub-proof over the union of both children +/// would be verifying one proof with `2n` tables — a different statement, and +/// not one either child ever made. (`per_table_census_tests::tenant_node_program` +/// is shaped that way ON PURPOSE, as a census instrument; it is not a template.) +pub fn emit_leg(b: &mut LfmBuilder, child: &ChildShape<'_>, a: &LegArenas) -> LegCells { + emit_leg_answered(b, child, a).0 +} + +/// [`emit_leg`], also returning the child's commitment-time point `z0` +/// (`stark::commit_ood`) for the differential gates. +pub(crate) fn emit_leg_answered( + b: &mut LfmBuilder, + child: &ChildShape<'_>, + a: &LegArenas, +) -> (LegCells, Ext) { + let n = child.tables.len(); + let per_root = RootCells::words_per_root(b); + + // ---- the statement, over the child's claimed published words ---- + let publics = hint_public_words(b, a.publics, child.num_public_words); + let mut t = TranscriptReplay::new(&[]); + emit_lfm_statement_tagged( + &mut t, + child.statement_tag, + child.program_id, + &publics, + child.fri_final_poly_log_degree, + ); + + // ---- Phase A: preprocessed roots as AIR-set constants, main roots hinted. + let main_cells: Vec = (0..n) + .map(|i| RootCells::hint(b, a.main_roots, per_root * i as u32)) + .collect(); + // ⚠ The DIGEST's felts, not the root's bytes. `replay_phase_a` absorbs + // through `absorb_root_felts`, which declares the host's 32 bytes on both + // arms and packs the algebraic arm's four felts into the one digest cell they + // already are, so the root absorb CANCELS rather than paying a byte + // regrouping. `byte_halves` is for `program_id`, which is deliberately + // keccak-over-bytes. + let main_halves: Vec> = main_cells.iter().map(RootCells::lanes_flat).collect(); + let prep_cells: Vec> = child + .tables + .iter() + .map(|t| t.precomputed_root.map(|c| RootCells::constant(b, c))) + .collect(); + let phase_a: Vec = child + .tables + .iter() + .enumerate() + .map(|(i, t)| PhaseATable { + preprocessed_root: t.precomputed_root.map(PhaseAPreprocessed::Constant), + main_root: &main_halves[i][..], + }) + .collect(); + // The commitment-time answers: hinted once, absorbed by Phase A after + // `z0`, and the same cells handed to every table's DEEP below. + let answers: Vec = (0..child.num_answers() as u32) + .map(|k| b.hint_word(a.answers, k).as_ext()) + .collect(); + let domain = CommitOodDomain::over(child.tables.iter().map(|t| { + ( + t.challenge.log2_trace_length, + t.challenge.log2_blowup, + t.challenge.coset_offset, + ) + })); + let (z, alpha, z0) = replay_phase_a_answered(&mut t, b, &phase_a, &answers, domain); + let mut answer_at = 0usize; + + // ---- one fork per sub-proof, with the full verification legs ---- + let mut contributions: Vec = Vec::new(); + for (i, table) in child.tables.iter().enumerate() { + let c = table.challenge; + let arenas = &a.per_table[i]; + let aux = arenas.aux_root.map(|id| RootCells::hint(b, id, 0)); + let contribution = arenas.contribution.map(|id| b.hint_word(id, 0).as_ext()); + let composition = RootCells::hint(b, arenas.composition_root, 0); + let ood_current: Vec = (0..(c.ood_current_dims.0 * c.ood_current_dims.1) as u32) + .map(|k| b.hint_word(arenas.ood_current, k).as_ext()) + .collect(); + let ood_next: Vec = (0..(c.ood_next_dims.0 * c.ood_next_dims.1) as u32) + .map(|k| b.hint_word(arenas.ood_next, k).as_ext()) + .collect(); + let parts: Vec = (0..c.num_parts as u32) + .map(|k| b.hint_word(arenas.parts, k).as_ext()) + .collect(); + let fri_roots: Vec = (0..c.fri.num_committed()) + .map(|k| RootCells::hint(b, arenas.fri_roots, per_root * k as u32)) + .collect(); + let fri_coeffs: Vec = (0..c.fri.num_terminal_coeffs() as u32) + .map(|k| b.hint_word(arenas.fri_coeffs, k).as_ext()) + .collect(); + let nonce = arenas.nonce.map(|id| b.hint_felt(id, 0)); + if let Some(l) = contribution { + contributions.push(l); + } + + let mut fork = fork_table(&t, c.index, c.num_tables); + let absorbs = TableAbsorbs { + aux_root: aux.as_ref(), + contribution, + composition_root: &composition, + ood_current: &ood_current, + ood_next: &ood_next, + parts: &parts, + fri_roots: &fri_roots, + fri_coeffs: &fri_coeffs, + nonce, + }; + let ch = super::epoch::emit_table_challenges(b, &mut fork, c, &absorbs); + // ★ THE SEAM: `absorbs` is passed on by REFERENCE rather than rebuilt, + // so there is no second reading of the proof for a leg to disagree with + // the transcript about. + super::epoch_verify::emit_table_verification( + b, + table.verify, + table.analysis, + &ch, + &absorbs, + &super::epoch_verify::TableInputs { + precomputed_root: prep_cells[i].as_ref(), + main_root: &main_cells[i], + rap_challenges: &[z, alpha], + commit_ood: (table.verify.num_answers() > 0).then(|| { + ( + z0, + &answers[answer_at..answer_at + table.verify.num_answers()], + ) + }), + }, + &arenas.legs, + ); + answer_at += table.verify.num_answers(); + } + assert_eq!(answer_at, answers.len(), "every answer goes to its table"); + + // ---- the closure, against the PUBLIC balance ---- + // + // Every other LFM bus balances to zero internally; `LfmPublic` is the one + // whose target is the claimed words, which is what makes this the binding + // between "the proof verifies" and "it published THESE words". + let target = emit_public_balance(b, &publics, z, alpha); + let shape = super::logup::LogUpShape { + num_contributing_tables: contributions.len(), + num_output_bytes: 0, + }; + super::logup::emit_bus_closure(b, &shape, &contributions, target); + + ( + LegCells { + publics, + z_alpha: (z, alpha), + }, + z0, + ) +} + +// ============================ the binding legs ============================ + +/// Where each field of the block-binding schema sits in a child's published +/// words — for a WRAP child and for a NODE child alike. +/// +/// The two differ in their head and in two fields, and the difference is real +/// rather than incidental, so it is a constructor apiece rather than a flag: +/// +/// | | wrap child | node child | +/// |---|---|---| +/// | head | `z, α, id₀, id₁` | `id₀, id₁` — a node has FAN_IN Phase A's and no single pair | +/// | labels | one epoch label | the FIRST and LAST label of its subtree | +/// | L2G | the epoch's re-commit root, as lanes | the subtree's FOLDED digest, as cells | +/// | tail | the closure's bus total | nothing | +/// +/// A node has no single `(z, α)` because it replays one transcript per child, +/// and no single epoch label because it covers a RANGE — publishing the range's +/// two ends is what lets a parent pin them to constants and get contiguity +/// across siblings for free. +/// +/// ⚠ `l2g_words` is a WORD count, not a lane count, and the two constructors +/// disagree on purpose: a wrap publishes `RootCells::lanes_flat` (four lanes per +/// root word) while a node publishes the fold's digest CELLS. The codebase has +/// been bitten by lanes-versus-words twice; the field is named for the unit it +/// actually is. +pub struct SchemaLayout { + /// Where the two attestation-id words start. + pub id_at: usize, + /// Words published before the schema run. + pub head: usize, + pub num_reg: usize, + /// Label words: two for a wrap (one label, lo/hi), four for a node. + pub label_words: usize, + pub out_halves: usize, + pub l2g_words: usize, + /// Words published after the schema run. + pub tail: usize, +} + +impl SchemaLayout { + /// The layout of an epoch WRAP published under `Publishes::Aggregation`. + pub fn wrap(out_halves: usize) -> Self { + Self { + id_at: 2, + head: 4, + num_reg: crate::tables::register::NUM_REGISTER_ADDRESSES, + label_words: 2, + out_halves, + l2g_words: super::proof_arena::lanes_per_root(), + tail: 1, + } + } + + /// The layout of an aggregation NODE, at any level. + pub fn node(out_halves: usize) -> Self { + Self { + id_at: 0, + head: 2, + num_reg: crate::tables::register::NUM_REGISTER_ADDRESSES, + label_words: 4, + out_halves, + // ★ THE SAME SHAPE A WRAP PUBLISHES ITS ROOT IN — `lanes_per_root` + // base words, one lane each — not the fold's digest CELLS. + // + // Publishing cells would have been one word instead of four, and it + // would have made a node child and a wrap child structurally + // different to read: `emit_node_publishes` takes `lanes[0]` of each + // published l2g word, which is right for a wrap's lane-per-word + // layout and silently wrong for a four-lane digest word — it would + // hand ONE felt to `digest_from_lanes` where four are required. + // Building the inner-node arm is what surfaced that; the asymmetry + // is removed here rather than parameterised around. + l2g_words: super::proof_arena::lanes_per_root(), + tail: 0, + } + } + + pub fn schema_words(&self) -> usize { + 2 * self.num_reg + self.label_words + self.out_halves + self.l2g_words + } + + /// Every word the child publishes. + pub fn total(&self) -> usize { + self.head + self.schema_words() + self.tail + } + + /// Pin the layout against what the child actually publishes. + /// + /// A level confusion — a wrap layout applied to a node, or a layout built + /// for one block applied to another — is a loud failure here rather than a + /// silent mis-binding fifty asserts later. + pub fn assert_covers(&self, num_public_words: usize) { + assert_eq!( + self.total(), + num_public_words, + "the layout must cover the child's published words exactly \ + (head={}, num_reg={}, label_words={}, out_halves={}, l2g_words={}, tail={})", + self.head, + self.num_reg, + self.label_words, + self.out_halves, + self.l2g_words, + self.tail, + ); + } + + pub fn id(&self, half: usize) -> usize { + self.id_at + half + } + pub fn reg_init(&self, r: usize) -> usize { + self.head + r + } + pub fn reg_fini(&self, r: usize) -> usize { + self.head + self.num_reg + r + } + pub fn label(&self, i: usize) -> usize { + self.head + 2 * self.num_reg + i + } + pub fn out_half(&self, i: usize) -> usize { + self.head + 2 * self.num_reg + self.label_words + i + } + pub fn l2g_word(&self, w: usize) -> usize { + self.head + 2 * self.num_reg + self.label_words + self.out_halves + w + } +} + +/// Assert two hinted public words carry the same value, lane by lane. +pub fn assert_words_equal(b: &mut LfmBuilder, x: &HintedPublicWord, y: &HintedPublicWord) { + for (xl, yl) in x.lanes.iter().zip(&y.lanes) { + let xe = xl.as_ext(); + let ye = yl.as_ext(); + b.assert_eq_ext(xe, ye); + } +} + +/// Assert a hinted public word's base value equals a program constant — lanes +/// `1..4` must be zero, which is what a BASE publish looks like. +pub fn assert_word_is_const(b: &mut LfmBuilder, x: &HintedPublicWord, v: u64) { + let c = b.ext_const(&FEE::from(v)); + let x0 = x.lanes[0].as_ext(); + b.assert_eq_ext(x0, c); + let zero = b.ext_const(&FEE::zero()); + for lane in &x.lanes[1..] { + let le = lane.as_ext(); + b.assert_eq_ext(le, zero); + } +} + +/// The cross-child binding legs: one shared attestation id across every child, +/// each child's register fini vector equal to the next child's init vector, and +/// each child's epoch label pinned to its chain position as an emit-time +/// constant. +/// +/// These are CHECKS on published words, never trusts. Verifying two children +/// says nothing about their relationship; this is what makes a tree of proofs a +/// statement about one execution rather than about several. +pub fn emit_chain_bindings( + b: &mut LfmBuilder, + legs: &[LegCells], + layouts: &[SchemaLayout], + labels: &[&[u64]], +) { + assert_eq!(legs.len(), layouts.len(), "one layout per child"); + assert_eq!(legs.len(), labels.len(), "one chain position per child"); + + // ---- one attestation id answers for every child. + for k in 1..legs.len() { + for half in 0..2 { + assert_words_equal( + b, + &legs[0].publics[layouts[0].id(half)], + &legs[k].publics[layouts[k].id(half)], + ); + } + } + // ---- the register chain, across every seam. + for k in 0..legs.len().saturating_sub(1) { + for r in 0..layouts[k].num_reg { + assert_words_equal( + b, + &legs[k].publics[layouts[k].reg_fini(r)], + &legs[k + 1].publics[layouts[k + 1].reg_init(r)], + ); + } + } + // ---- each label pinned to its position, as a constant of THIS program. + // + // A wrap child carries ONE label; a node child carries the first and last of + // its subtree. Pinning every one of them to a constant is what makes + // contiguity across siblings free: the emitter knows the true label + // sequence, so a child covering the wrong range cannot satisfy the pins. + for (k, child_labels) in labels.iter().enumerate() { + assert_eq!( + 2 * child_labels.len(), + layouts[k].label_words, + "child {k} publishes {} label words but {} labels were given", + layouts[k].label_words, + child_labels.len() + ); + for (i, &label) in child_labels.iter().enumerate() { + assert_word_is_const( + b, + &legs[k].publics[layouts[k].label(2 * i)], + label & 0xFFFF_FFFF, + ); + assert_word_is_const( + b, + &legs[k].publics[layouts[k].label(2 * i + 1)], + label >> 32, + ); + } + } +} + +// ============================ the aggregation node ======================== + +/// Children per node on the STARK tree — that tree's arity. +/// +/// A DEFAULT, not an assumption: every emitter below takes a slice, so the +/// arity is whatever the caller passes and nothing here depends on this value. +/// It exists so the STARK tree builder has one place to change. +/// +/// Two is the brief's working default and three is COORD's tie-break, on the +/// grounds that over ten epochs it is 5 distinct programs / 7 proofs / 3 levels +/// against two's 6 / 11 / 4. The measured host peak decides; until it has, the +/// conservative value stands. +/// +/// ⛔ **AND ON THIS TREE IT STILL STANDS, for a CARD reason rather than a host +/// one.** The STARK arity-3 interior node does not fit the device: measured, +/// ds15/ds16. The host peak that settles the question for a WHIR tree says +/// nothing about a node that aborts on VRAM before its host working set is ever +/// live, so the two trees answer to different constraints and get one constant +/// each. Every STARK tree number on record — the D-S arms, PR #999 — was taken +/// at two, and this constant is what keeps them comparable. +pub const FAN_IN: usize = 2; + +/// A digest rebuilt from the lanes a child PUBLISHED for it. +/// +/// The inverse of `RootCells::lanes_flat`, and correct on both arms — four +/// lanes pack into one word, and a root is `words_per_root` of them. +/// +/// ⚠ Deliberately not `RootCells::from_halves`, which asserts EIGHT halves and +/// is byte-arm-only by construction. Handing it an algebraic root's four felts +/// fails the assert; handing a byte root's eight halves to a caller expecting +/// felts would hash four values as if they were eight, silently. +pub fn digest_from_lanes(b: &mut LfmBuilder, lanes: &[Felt]) -> super::edsl::WrapDigest { + let words = super::proof_arena::words_per_root(); + assert_eq!( + lanes.len(), + LANES_PER_WORD * words, + "a published root is four lanes per root word" + ); + let cells: Vec = lanes + .chunks(LANES_PER_WORD) + .map(|c| b.pack_word([c[0], c[1], c[2], c[3]])) + .collect(); + match cells.len() { + 1 => super::edsl::WrapDigest::from_cell(cells[0]), + 2 => super::edsl::WrapDigest::from_pair(cells[0], cells[1]), + n => unreachable!("a root is one or two words, got {n}"), + } +} + +/// Fold a subtree's L2G re-commit roots into ONE digest — a left fold of the +/// production hash's two-to-one compression, identity on a singleton. +/// +/// # Why a fold and not a list +/// +/// The batched aggregator compared each epoch's published L2G root against the +/// global proof's re-commit root for that epoch, and could do it as a local +/// assert because all six legs were in ONE program. A tree splits them: the +/// epoch wraps sit in leaf nodes and the global wrap rides at the ROOT, so the +/// compare must be deferred to their common ancestor. +/// +/// If each node re-published its subtree's roots as a LIST, the node schema +/// would grow with the subtree and every level would have a different published +/// width — which is what makes the parent's leg shape depend on the block's +/// epoch count. Folding them keeps the node schema FIXED SIZE at every level, +/// and the root recomputes the same fold over the global wrap's published roots +/// and compares one digest. +/// +/// ⛔ **THE FOLD COMPOUNDS: IT IS TREE-SHAPED, NOT FLAT.** An earlier version of +/// this paragraph said only *"left fold, in tree order"*, which is true of ONE +/// node and actively misleading at the ROOT — which is exactly where it is read. +/// +/// A node folds its children's PUBLISHED digests, and a node child's published +/// digest is already a fold. So a level-1 node publishes `H(r0, r1)` and a +/// level-2 node publishes `H(H(r0,r1), H(r2,r3))` — **not** the flat left fold +/// `H(H(H(r0,r1),r2),r3)`. `hash_pair` is a two-to-one compression and is not +/// associative, so those are different digests. +/// +/// ⇒ The global wrap publishes the FLAT list of per-epoch roots. A root that +/// folds them left to right computes a digest an honest prover never produced, +/// and the failure lands on **completeness, not soundness**: correct code, +/// correct inputs, honest prover, wrong answer — the worst kind to diagnose from +/// a failing prove. The root must group them exactly as the interior did, level +/// by level, which is what `block_root::FoldShape` exists to make unavoidable. +/// +/// Within ONE node the rule is: left fold, `hash_pair(acc, next)`, in child +/// order; a single child folds to itself. +pub fn fold_l2g( + b: &mut LfmBuilder, + digests: &[super::edsl::WrapDigest], +) -> super::edsl::WrapDigest { + assert!(!digests.is_empty(), "a subtree covers at least one epoch"); + let hash = b.wrap_hash(); + let mut acc = digests[0]; + for next in &digests[1..] { + acc = hash.hash_pair(b, acc, *next); + } + acc +} + +/// What a node publishes, so that its parent binds it exactly as it binds a +/// wrap. See [`SchemaLayout::node`] for the layout this fills. +pub struct NodePublishes<'a> { + /// The children's legs, in chain order. + pub legs: &'a [LegCells], + /// One layout per child. + pub layouts: &'a [SchemaLayout], + /// The first and last epoch label of the subtree — emit-time constants. + pub label_range: (u64, u64), +} + +/// Emit a node's published words: the shared attestation id, the first child's +/// register INIT vector, the last child's register FINI vector, the subtree's +/// first and last epoch labels, the last child's output halves, and the folded +/// L2G digest. +/// +/// Every value is republished in the SAME form the wrap published it, so a +/// parent's `assert_words_equal` reads a node and a wrap alike: the id as a +/// four-lane digest word, the register / label / output items as base words. +pub fn emit_node_publishes(b: &mut LfmBuilder, p: &NodePublishes<'_>) { + let NodePublishes { + legs, + layouts, + label_range, + } = *p; + assert_eq!(legs.len(), layouts.len(), "one layout per child"); + assert!(!legs.is_empty(), "a node has children"); + let first = &legs[0]; + let last = legs.last().expect("nonempty"); + let l_first = &layouts[0]; + let l_last = layouts.last().expect("nonempty"); + + // ---- the attestation id, as the four-lane word the wrap published. + for half in 0..2 { + let lanes = &first.publics[l_first.id(half)].lanes; + let word = b.pack_word([lanes[0], lanes[1], lanes[2], lanes[3]]); + b.public(word); + } + // ---- the chain's two ends. + for r in 0..l_first.num_reg { + b.public(first.publics[l_first.reg_init(r)].lanes[0].as_cell()); + } + for r in 0..l_last.num_reg { + b.public(last.publics[l_last.reg_fini(r)].lanes[0].as_cell()); + } + // ---- the label RANGE, as constants of this program. Publishing the ends + // rather than the list is what keeps the schema fixed-size; the parent pins + // both to constants and gets contiguity across siblings for free. + for label in [label_range.0, label_range.1] { + let lo = b.felt_const(FE::from(label & 0xFFFF_FFFF)); + b.public(lo.as_cell()); + let hi = b.felt_const(FE::from(label >> 32)); + b.public(hi.as_cell()); + } + // ---- the last child's output halves — the block's output, carried up. + for i in 0..l_last.out_halves { + b.public(last.publics[l_last.out_half(i)].lanes[0].as_cell()); + } + // ---- the folded L2G digest. + let digests: Vec = legs + .iter() + .zip(layouts) + .map(|(leg, layout)| { + let lanes: Vec = (0..layout.l2g_words) + .map(|w| leg.publics[layout.l2g_word(w)].lanes[0]) + .collect(); + digest_from_lanes(b, &lanes) + }) + .collect(); + let folded = fold_l2g(b, &digests); + // Unpacked to lanes, so a node's L2G item reads exactly like a wrap's. + for cell in folded.cells() { + for lane in b.unpack(*cell) { + b.public(lane.as_cell()); + } + } +} + +/// One aggregation node, end to end: verify every child, bind them, publish the +/// node's own schema. +/// +/// The SAME function serves every level — a child is a plain per-table +/// `MultiProof` whether a wrap or another node produced it, and the only thing +/// that changes is the children's shapes and layouts. What does not carry across +/// levels is the node's IDENTITY: the legs absorb their children's `program_id` +/// as emit-time constants, so each level compiles to its own program. +pub struct NodeInputs<'a> { + pub children: &'a [ChildShape<'a>], + pub layouts: &'a [SchemaLayout], + /// Per child, the epoch labels it must carry: one for a wrap child, the two + /// ends of its subtree for a node child. + pub labels: &'a [&'a [u64]], + /// The first and last epoch label this node's subtree covers. + pub label_range: (u64, u64), + /// Which words this node publishes. + pub publishes: NodePublishSet, +} + +/// Which words a node publishes — the node-level counterpart of +/// `epoch_tests::Publishes`, and it exists for a different reason. +/// +/// A node under [`NodePublishSet::Aggregation`] publishes no challenges at all, +/// which leaves its legs with no differential surface: the only evidence that a +/// leg derived its CHILD's challenges is that the node executes, since a leg on +/// different challenges cannot authenticate the child's walks. That is +/// implication rather than a value comparison, and it is weaker than what every +/// other emitted verifier in this crate is held to — the epoch wrap and the +/// global leg both publish their pair and differential it against production's +/// own replay. +/// +/// [`NodePublishSet::Diagnostic`] restores that surface by publishing each +/// child's `(z, α)` AFTER the schema, so the schema's own indices do not move +/// and `SchemaLayout::node` reads both variants' heads identically. It is a gate +/// shape, never a child: nothing verifies a diagnostic node. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum NodePublishSet { + /// The node schema and nothing else — what a node that will be VERIFIED + /// publishes. + Aggregation, + /// The schema, then each child's `(z, α)` in child order — the differential + /// surface. + Diagnostic, +} + +/// Declare every arena the node reads, in absorb order, then emit it. +pub fn emit_node(b: &mut LfmBuilder, inputs: &NodeInputs<'_>) { + let NodeInputs { + children, + layouts, + labels, + label_range, + publishes, + } = *inputs; + assert!(!children.is_empty(), "a node verifies at least one child"); + assert_eq!(children.len(), layouts.len(), "one layout per child"); + assert_eq!(children.len(), labels.len(), "one label list per child"); + for (child, layout) in children.iter().zip(layouts) { + layout.assert_covers(child.num_public_words); + } + + // Declaration order IS absorb order, end to end: every child's arenas are + // declared before any leg is emitted. + let arenas: Vec = children + .iter() + .map(|child| declare_leg_arenas(b, child)) + .collect(); + let legs: Vec = children + .iter() + .zip(&arenas) + .map(|(child, a)| emit_leg(b, child, a)) + .collect(); + emit_chain_bindings(b, &legs, layouts, labels); + emit_node_publishes( + b, + &NodePublishes { + legs: &legs, + layouts, + label_range, + }, + ); + // The differential surface, AFTER the schema so no schema index moves. + if publishes == NodePublishSet::Diagnostic { + for leg in &legs { + b.public(leg.z_alpha.0.as_cell()); + b.public(leg.z_alpha.1.as_cell()); + } + } +} + +// ============================ the tree's shape ============================ + +/// One level of the tree: how many proofs it consumes and how they group. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Level { + /// Children per node, in order. Every entry is `1..=fan_in`. + pub arities: Vec, +} + +impl Level { + /// Proofs this level produces — one per node. + pub fn nodes(&self) -> usize { + self.arities.len() + } +} + +/// The whole tree's shape, derived from the epoch count and an arity. +/// +/// # The leftover rule: WRAP, do not carry +/// +/// A level with an odd count leaves one proof over. Two ways, and the choice +/// changes the PROGRAM SET rather than the code: +/// +/// - **carry** it up unwrapped ⇒ its parent has children from two different +/// levels, so two different `program_id`s and two different shapes. A node +/// embeds each child's id as an emit-time constant, so every such parent is a +/// distinct program — and the VERIFIER must emit each one too. +/// - **wrap** it in an arity-1 node ⇒ one extra proof per odd level, and every +/// parent's children stay homogeneous: one program per `(level, arity)`. +/// +/// ⇒ This wraps. Program-set growth lands on both sides of the protocol and is +/// combinatorial; the extra proofs are bounded and countable (three at 36 +/// epochs, fan-in 2). ⚠ Revisit if a node prove is ever measured to dominate +/// program emission — the trade is real, and it is recorded rather than assumed. +/// +/// ⓘ The ROOT is not described here. It takes `fan_in + 1` children — the global +/// wrap is the extra — performs the L2G compare and the attestation fold, and +/// publishes the block artifact's schema rather than the node schema. This +/// function describes the interior. +pub fn tree_shape(epochs: usize, fan_in: usize) -> Vec { + assert!(epochs >= 1, "a block has at least one epoch"); + assert!(fan_in >= 2, "a tree needs an arity of at least two"); + let mut levels = Vec::new(); + let mut n = epochs; + while n > 1 { + let mut arities = Vec::with_capacity(n.div_ceil(fan_in)); + let mut left = n; + while left > 0 { + let take = left.min(fan_in); + arities.push(take); + left -= take; + } + n = arities.len(); + levels.push(Level { arities }); + } + levels +} + +/// Total aggregator proofs the interior costs — one per node, every level. +pub fn tree_node_count(shape: &[Level]) -> usize { + shape.iter().map(Level::nodes).sum() +} diff --git a/prover/src/lfm/per_table_aggregator_tests.rs b/prover/src/lfm/per_table_aggregator_tests.rs new file mode 100644 index 000000000..29b2d4503 --- /dev/null +++ b/prover/src/lfm/per_table_aggregator_tests.rs @@ -0,0 +1,1602 @@ +//! The aggregation node over real child proofs: the child harvest the block +//! tree's tests share ([`real_child`], [`child_shape`], [`child_arena_words`]), +//! nodes emitted over STARK-proved epoch wraps and proved as the next level's +//! child, and the in-order pool the tree driver runs a level on. + +use crate::tables::types::{FE, FEE}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::transcript_replay::TranscriptReplay; +use super::word::{LfmWord, base_word}; + +#[test] +fn the_aggregation_publish_profile_drops_only_diagnostics() { + use super::epoch_tests::{Publishes, epoch_program_publishing, schema_words}; + + let e = super::epoch_tests::real_epoch(); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + + let run = |publishes| { + let program = epoch_program_publishing(&e, true, publishes); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the assembled verifier must execute under either profile"); + (program.instrs.len(), exec.public_words) + }; + let (diag_instrs, diag) = run(Publishes::Diagnostic); + let (agg_instrs, agg) = run(Publishes::Aggregation); + + // ---- the head: the pair, the id and the schema, identical cells. + let head = 2 + 2 + schema_words(&e); + assert_eq!( + diag[..head], + agg[..head], + "the binding head must not move with the profile" + ); + // ---- the tail: the closure's total, which both profiles publish LAST. + // + // ⚠ Against PRODUCTION'S OWN TARGET, one profile at a time — never by + // comparing the two lists' last entries to each other. A published word is + // an `(index, value)` pair, and the two indices cannot be equal: making the + // lists different lengths is the entire point of R2. That was this + // assertion's first form, and it failed on box A with the four field + // elements matching exactly and only the indices differing (307 against + // 144) — the emitter doing precisely what it should, caught by a test + // asserting something it never meant. + // + // The oracle here is also stronger than the one it replaces: production's + // COMMIT-bus balance, rather than "the other profile agrees with me". + for (label, words) in [("Diagnostic", &diag), ("Aggregation", &agg)] { + let (index, word) = words.last().expect("a profile publishes words"); + assert_eq!( + *index as usize, + words.len() - 1, + "{label}: publish indices auto-increment, so the last word's index \ + is len-1" + ); + assert_eq!( + super::word::word_as_ext(word).expect("the bus total is ext"), + e.expected_bus_balance, + "{label}: the closure's total must end the list and reach \ + production's own COMMIT-bus target" + ); + } + assert_eq!(agg.len(), head + 1, "Aggregation is the head and the total"); + + // ---- what was dropped, from the epoch's own shapes rather than from a + // literal: per sub-proof the composition, one terminal per query, the + // (beta, z, gamma) triple, the DEEP zetas and one index per query. + let dropped: usize = e + .tables + .iter() + .zip(&e.legs) + .map(|(h, leg)| 1 + leg.verify.num_queries + 3 + h.zetas.len() + h.shape.num_queries) + .sum(); + assert_eq!( + diag.len(), + agg.len() + dropped, + "Aggregation must drop exactly the per-sub-proof diagnostics" + ); + + println!( + "★ publish profile over {} sub-proofs: Diagnostic {} words / {diag_instrs} instrs, \\ + Aggregation {} words / {agg_instrs} instrs ({:.1}% of the words, {:.1}% of the \\ + instructions)", + e.tables.len(), + diag.len(), + agg.len(), + 100.0 * agg.len() as f64 / diag.len() as f64, + 100.0 * agg_instrs as f64 / diag_instrs as f64, + ); +} + +// ======================= the child harvest and the node =================== + +/// One child LFM proof, production-accepted, harvested for emission (the +/// reader lives in [`super::harvest`], where the block tree's driver uses it +/// too). +pub(super) use super::harvest::HarvestedChild as RealChild; + +/// Harvest a child from a proof PRODUCTION ACCEPTS. Panics loudly otherwise — +/// nothing downstream may read a proof the verifier would reject. +pub(super) fn real_child( + artifacts: super::registry::LfmArtifacts, + opts: crate::ProofOptions, + proved: &super::proof::LfmProof, +) -> RealChild { + real_child_timed(artifacts, opts, proved).0 +} + +/// [`real_child`], and the seconds its opening assert cost. +/// +/// ⛔ THE ASSERT IS TIMED, NOT SKIPPED, AND THERE IS NO KNOB THAT SKIPS IT. +/// It is a complete host STARK verify of the proof produced moments earlier, +/// and it is what the harness's trustworthiness rests on — a child read from a +/// proof production would reject describes nothing. It belongs on a line of +/// its own rather than inside a `harvest` figure that then gets quoted as the +/// cost of harvesting. +pub(super) fn real_child_timed( + artifacts: super::registry::LfmArtifacts, + opts: crate::ProofOptions, + proved: &super::proof::LfmProof, +) -> (RealChild, f64) { + super::harvest::harvest_child_verified(artifacts, opts, proved) + .unwrap_or_else(|e| panic!("{e}")) +} + +/// The child's shape, as the node's emitter reads it. +pub(super) fn child_shape(c: &RealChild) -> super::per_table_aggregator::ChildShape<'_> { + super::per_table_aggregator::ChildShape { + program_id: &c.artifacts.program_id, + num_public_words: c.public_words.len(), + fri_final_poly_log_degree: c.opts.fri_final_poly_log_degree, + statement_tag: super::statement::LFM_STATEMENT_TAG, + tables: c + .tables + .iter() + .zip(&c.legs) + .map(|(h, leg)| super::per_table_aggregator::ChildTable { + challenge: &h.shape, + verify: &leg.verify, + analysis: &leg.analysis, + precomputed_root: leg.precomputed_commitment.as_ref(), + }) + .collect(), + } +} + +/// The child's arenas, in `declare_leg_arenas`' declaration order. +pub(super) fn child_arena_words(c: &RealChild) -> Vec> { + super::harvest::try_child_arena_words(c).unwrap_or_else(|e| panic!("{e}")) +} + +/// The aggregation node over `children`, at any level and any arity. +pub(super) fn node_program( + children: &[RealChild], + layouts: &[super::per_table_aggregator::SchemaLayout], + labels: &[&[u64]], + label_range: (u64, u64), + publishes: super::per_table_aggregator::NodePublishSet, +) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let shapes: Vec<_> = children.iter().map(child_shape).collect(); + super::per_table_aggregator::emit_node( + &mut b, + &super::per_table_aggregator::NodeInputs { + children: &shapes, + layouts, + labels, + label_range, + publishes, + }, + ); + compile(b.finish()) +} + +/// Name any sub-proof whose query sampler would be handed a zero bit width, +/// BEFORE emission reaches it. +/// +/// `epoch::emit_table_challenges` samples each query index with +/// `sample_u64_pow2(shape.index_bits())`, and `index_bits()` is +/// `log2_trace_length + log2_blowup - 1`. A one-row trace at blowup 2 makes that +/// ZERO, and the sampler's own assert then fires deep inside emission with no +/// idea which table or which side of the tree it came from — which is exactly +/// how it surfaced on box A: a bare "nbits must be in 1..=32, got 0" with +/// nothing to attach it to. +/// +/// A diagnostic, not a fix. If it fires, the shape is real and the question is +/// whether a one-row sub-proof should exist at all at this preset. +fn assert_samplable(label: &str, shapes: &[&super::epoch::TableChallengeShape]) { + for (i, s) in shapes.iter().enumerate() { + let lde = s.log2_trace_length + s.log2_blowup; + assert!( + lde >= 1, + "{label} sub-proof {i}: log2_trace {} + log2_blowup {} = {lde}, so \ + index_bits() would be {} — the query sampler needs at least one bit", + s.log2_trace_length, + s.log2_blowup, + lde as i64 - 1, + ); + } + let worst = shapes + .iter() + .enumerate() + .min_by_key(|(_, s)| s.log2_trace_length + s.log2_blowup) + .expect("a proof has sub-proofs"); + println!( + " {label}: {} sub-proofs, shallowest is #{} at log2_trace {} + log2_blowup {} = {} index bits", + shapes.len(), + worst.0, + worst.1.log2_trace_length, + worst.1.log2_blowup, + worst.1.index_bits(), + ); +} + +/// ★ THE LEAF NODE RUNS — the first aggregation node over real per-table wrap +/// proofs. +/// +/// # What it is +/// +/// `FAN_IN` epochs of a fixture continuation, each wrapped by the assembled +/// per-table epoch verifier under `Publishes::Aggregation`, and ONE emitted +/// program that verifies both wrap proofs and binds them: the shared attestation +/// id, the register fini→init seam, and each epoch's label pinned to its chain +/// position as a constant. The node then publishes the schema its own parent +/// will read. +/// +/// # Why the tamper arms are three and not one +/// +/// Each binding leg can fail on its own and a single arm would not tell them +/// apart. The register seam, the shared id and the label pin are three +/// independent claims about the relationship between two proofs that both +/// verify — and "both children verified" is exactly what a broken binding still +/// looks like. Each arm moves ONE published word in ONE child's arena and +/// nothing else, so what fails is named by which arm failed. +/// +/// # Cost +/// +/// Box tier and `#[ignore]`d: two epoch wraps proved (a wrap proof carries a +/// full LFM chip set), then a node program that verifies both. The suite gates +/// the pieces — `wrap_tests::the_fixture_epoch_wraps` proves one wrap, +/// `the_global_verifier_leg_runs_and_rejects_tampers` runs a per-table leg, and +/// `the_aggregation_publish_profile_drops_only_diagnostics` pins what a wrap +/// publishes — so this is the assembly rather than any of its parts. +#[test] +#[ignore = "box tier: proves FAN_IN epoch wraps and a node over them"] +fn the_leaf_node_verifies_and_binds_two_wraps() { + use super::epoch_tests::Publishes; + use super::per_table_aggregator::{FAN_IN, NodePublishSet, SchemaLayout}; + use super::program_census::build_artifacts_counted; + use super::proof::lfm_prove; + use std::time::Instant; + + let elf_bytes = super::proof_fixture::read_inner_elf(); + let inner = super::proof_fixture::fixture_options(); + let bundle = crate::continuation::prove_continuation( + &elf_bytes, + &[], + super::proof_fixture::FIXTURE_EPOCH_LOG2, + &inner, + ) + .expect("the fixture continuation must prove"); + assert!( + bundle.num_epochs() >= FAN_IN, + "a fan-in-{FAN_IN} leaf needs {FAN_IN} epochs, the fixture has {}", + bundle.num_epochs() + ); + + // ---- the children: one wrap per epoch, at the AGGREGATION publish set. + let t = Instant::now(); + let mut children = Vec::with_capacity(FAN_IN); + let mut layouts = Vec::with_capacity(FAN_IN); + let mut labels = Vec::with_capacity(FAN_IN); + let wrap_opts = super::proof::aggregation_wrap_options(); + let epoch_konsts = super::epoch_tests::EpochConstants::load(&elf_bytes, &inner, None) + .expect("the inner ELF and its DECODE commitment must build once"); + for k in 0..FAN_IN { + let e = super::epoch_tests::real_epoch_from_constants(&inner, &epoch_konsts, &bundle, k) + .expect("every epoch must reconstruct from proofs alone"); + let out_halves = e.statement.public_output_len.div_ceil(4); + // Pre-flight: the wrap program emits one query sampler per INNER + // sub-proof, so a shape it cannot sample must be named here rather than + // deep inside `epoch_program_publishing`. + let inner_shapes: Vec<&super::epoch::TableChallengeShape> = + e.tables.iter().map(|h| &h.shape).collect(); + assert_samplable(&format!("inner epoch {k}"), &inner_shapes); + let program = + super::epoch_tests::epoch_program_publishing(&e, true, Publishes::Aggregation); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let artifacts = + build_artifacts_counted(&program, &wrap_opts, crate::hash_pin::BLOCK_HASHER); + let proved = lfm_prove(&program, &artifacts, &arenas, &wrap_opts) + .expect("the epoch wrap must prove at the aggregation preset"); + let layout = SchemaLayout::wrap(out_halves); + layout.assert_covers(proved.public_words.len()); + layouts.push(layout); + // The label is a pure function of the chain position, exactly as the + // global proof's own AIR reconstruction derives it. + labels.push([crate::tables::local_to_global::epoch_label(k as u64)]); + children.push(real_child(artifacts, wrap_opts.clone(), &proved)); + } + let label_refs: Vec<&[u64]> = labels.iter().map(|l| &l[..]).collect(); + let label_range = (labels[0][0], labels[FAN_IN - 1][0]); + // S2: how many of each wrap's sub-proofs the node verifies at one-row + // leaves (0 at the default format, all at `one_row = 1`, the AIR widths' + // choice at `auto`). One parseable line per child for the box wrapper. + for (k, c) in children.iter().enumerate() { + let one_row = c + .legs + .iter() + .filter(|l| l.verify.sub.layout.is_one_row()) + .count(); + println!( + "ZFS2NODE child={k} {} sub_proofs={} one_row_legs={one_row}", + crate::zf_format::ZfFormat::global().banner(), + c.legs.len() + ); + } + println!( + " {FAN_IN} epoch wraps proved in {:.1}s, {} published words each, \ + {} sub-proofs each\n RSS high-water AFTER the wrap proves: {:?} GiB", + t.elapsed().as_secs_f64(), + children[0].public_words.len(), + children[0].tables.len(), + super::wrap_tests::peak_rss_gib(), + ); + + // ---- the node. Same pre-flight on the CHILD side, so a zero bit width is + // attributed to the wrap's own sub-proofs rather than to the inner epoch's. + for (k, c) in children.iter().enumerate() { + let shapes: Vec<&super::epoch::TableChallengeShape> = + c.tables.iter().map(|h| &h.shape).collect(); + assert_samplable(&format!("child {k} (a wrap proof)"), &shapes); + } + let arenas: Vec> = children.iter().flat_map(child_arena_words).collect(); + let node_layout = SchemaLayout::node(layouts[FAN_IN - 1].out_halves); + + // ---- ★ THE DIFFERENTIAL, on the same shape with the surface restored. + // + // Without this the only evidence that a leg derived its CHILD's challenges + // is that the node executes — a leg on different challenges cannot + // authenticate the child's walks, so execution implies agreement. That is + // implication, and every other emitted verifier in this crate is held to a + // value comparison against production's own replay. This is that comparison: + // the pair each leg reaches, against the pair `verify_against_chunked`'s own + // Phase A recovered host-side from the same proof. + let t = Instant::now(); + let diagnostic = node_program( + &children, + &layouts, + &label_refs, + label_range, + NodePublishSet::Diagnostic, + ); + let exec_diag = execute(&diagnostic, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the diagnostic node must execute"); + let tail = node_layout.head + node_layout.schema_words(); + assert_eq!( + exec_diag.public_words.len(), + tail + 2 * FAN_IN, + "the diagnostic node publishes the schema then one pair per child" + ); + for (k, child) in children.iter().enumerate() { + let got = |i: usize| { + super::word::word_as_ext(&exec_diag.public_words[i].1).expect("an ext challenge") + }; + assert_eq!(got(tail + 2 * k), child.z_alpha.0, "child {k}: the leg's z"); + assert_eq!( + got(tail + 2 * k + 1), + child.z_alpha.1, + "child {k}: the leg's alpha" + ); + } + println!( + " ✓ differential: every leg reaches its child's OWN (z, alpha) \ + ({:.1}s, {} instructions)\n RSS high-water AFTER the diagnostic arm: {:?} GiB", + t.elapsed().as_secs_f64(), + diagnostic.instrs.len(), + super::wrap_tests::peak_rss_gib(), + ); + + // ---- the node a parent would verify. + let t = Instant::now(); + let program = node_program( + &children, + &layouts, + &label_refs, + label_range, + NodePublishSet::Aggregation, + ); + println!( + " leaf node emitted in {:.1}s: {} instructions", + t.elapsed().as_secs_f64(), + program.instrs.len() + ); + + let t = Instant::now(); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("★ the leaf node must execute"); + node_layout.assert_covers(exec.public_words.len()); + println!( + " ★ LEAF NODE EXECUTED in {:.1}s: {} published words (schema {} + head {})", + t.elapsed().as_secs_f64(), + exec.public_words.len(), + node_layout.schema_words(), + node_layout.head, + ); + // ★ The reading the fan-in arithmetic needs. `peak_rss_gib` is `VmHWM`, a + // PROCESS high-water mark that only ever rises, so the run's final figure + // spans the wrap proves, the diagnostic arm and the node alike. Printing it + // at each boundary turns one conflated number into a bound per phase: what + // the node's own prove costs is at most the rise from here. + println!( + " RSS high-water BEFORE the node prove: {:?} GiB", + super::wrap_tests::peak_rss_gib() + ); + + // ---- the node's own proof, so the level above has something to verify. + // + // ⚠ THREE marks, not one. The mark above is taken BEFORE the artifact + // build, which is not a bookkeeping call: it runs `lde_columns` + + // `commit_lde_columns` over every chip group and builds the prep round — a + // full commitment pass over the whole program. So a single "before the + // prove" mark brackets the artifact build and the prove TOGETHER and cannot + // say which of them costs what. These split it. + // ⓘ This measurement runs ONE proof, so it is always a cache MISS and the + // figure below is a real build — which is what it is here to price. + let t = Instant::now(); + let artifacts = build_artifacts_counted(&program, &wrap_opts, crate::hash_pin::BLOCK_HASHER); + println!( + " RSS high-water AFTER build_artifacts ({:.1}s): {:?} GiB", + t.elapsed().as_secs_f64(), + super::wrap_tests::peak_rss_gib() + ); + let t = Instant::now(); + let proved = + lfm_prove(&program, &artifacts, &arenas, &wrap_opts).expect("★ THE LEAF NODE MUST PROVE"); + let prove_secs = t.elapsed().as_secs_f64(); + println!( + " RSS high-water AFTER lfm_prove: {:?} GiB", + super::wrap_tests::peak_rss_gib() + ); + // ★ The census beside the measurement, so the two are never quoted apart. + // The empty LFM machine costs 26,482,828 base-field-equivalent cells — the + // 0-word row of `the_node_cost_model_is_measured` — so at fixture scale most + // of a node's census is the machine's padding FLOOR rather than its + // verification work, and a fixture node sits on a different part of the curve + // from a production one. + { + let (main, aux) = + super::airs::lfm_cell_counts_with_hasher(&program, crate::hash_pin::BLOCK_HASHER); + let cells = main + 3 * aux; + const EMPTY_MACHINE_CELLS: u64 = 26_482_828; + println!( + " node census: {cells} cells, of which {} are the empty machine's \ + floor ({:.0}%) and {} are verification work", + EMPTY_MACHINE_CELLS, + 100.0 * EMPTY_MACHINE_CELLS as f64 / cells as f64, + cells.saturating_sub(EMPTY_MACHINE_CELLS), + ); + } + let t = Instant::now(); + assert!( + super::proof::verify_against_artifacts( + &artifacts, + &proved.proof, + &proved.public_words, + &wrap_opts + ), + "the leaf node's proof must verify" + ); + println!( + "\n★ LEAF NODE PROVED AND VERIFIED\n prove {prove_secs:.1}s\n verify {:.2}s\n \ + {} sub-proofs, {} published words\n peak RSS {:?} GiB", + t.elapsed().as_secs_f64(), + proved.proof.proofs.len(), + proved.public_words.len(), + super::wrap_tests::peak_rss_gib(), + ); + + // ---- ONE TAMPER ARM PER BINDING LEG. + // + // Each moves a single published HALF in one child's publics arena — arena 0 + // of that child, eight halves per word — and nothing else. Both children's + // proofs still verify on their own; what breaks is the relationship. + let arena_of = |child: usize| -> usize { + // Each child contributes `child_arena_words(child).len()` arenas, and its + // publics arena is the first of them. Counted rather than assumed, so a + // declaration-order change tampers the right arena or fails loudly. + children[..child] + .iter() + .map(|c| child_arena_words(c).len()) + .sum() + }; + let bump = |arenas: &mut Vec>, arena: usize, word: usize| { + let half = 8 * word; + arenas[arena][half] = + base_word(super::word::word_as_base(&arenas[arena][half]).expect("a half") + FE::one()); + }; + for (name, child, word) in [ + ("the register chain", 0usize, layouts[0].reg_fini(0)), + ("the shared attestation id", 1usize, layouts[1].id(0)), + ("the epoch label pin", 1usize, layouts[1].label(0)), + ] { + let mut tampered = arenas.clone(); + bump(&mut tampered, arena_of(child), word); + assert!( + execute(&program, &tampered, &crate::hash_pin::BLOCK_HASHER).is_err(), + "moving {name} in child {child} must make the node unprovable" + ); + println!(" ✓ tamper arm: {name} rejected"); + } +} + +/// ★ A ZERO-BIT QUERY DRAW CONSUMES WHAT THE HOST CONSUMES. +/// +/// # The defect this holds shut +/// +/// A one-row trace at blowup 2 has a two-leaf LDE, so production's +/// `sample_query_indexes` calls `sample_u64(domain_size >> 1)` = `sample_u64(1)` +/// for every query of that table. Both host transcripts CONSUME before masking — +/// the byte arm draws one `next_sample_u64()`, the pinned algebraic arm squeezes +/// a cell — and return index 0. The emitted sampler refused `nbits = 0` outright, +/// so the epoch verifier could not be built over such an epoch at all; that is +/// what `the_leaf_node_verifies_and_binds_two_wraps` hit on box A, at inner epoch +/// 1's sub-proof #10. +/// +/// # Why the obvious fix would have been the bug +/// +/// "One index, so skip the sample" is wrong: the host does not skip it. An +/// emitter that skipped would be one squeeze short for that table and every +/// challenge after it in that fork would diverge. +/// +/// # Why this test needs the follow-up draw +/// +/// ⚠ A consumption desync here is invisible to a value differential ON the +/// table: every index is 0 whether the draw happened or not, and `iota_bits` is +/// the LAST thing `epoch::emit_table_challenges` samples, so nothing later in +/// that fork disagrees either. A green leaf test therefore proves nothing about +/// consumption — it proves only that no panic fired. +/// +/// So this samples an extension element AFTER the zero-bit draw on both sides. +/// That element is the single observable that differs if the squeeze is missing, +/// and comparing it against the HOST's is what makes this emitter-host agreement +/// rather than emitter self-consistency. +/// +/// ⓘ It lives here because lane A owns `transcript_replay.rs` only for this fix; +/// its natural home is beside the other transcript differentials. +#[test] +fn a_zero_bit_query_draw_consumes_what_the_host_does() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + + const SEED: &[u8] = b"lane-A zero-bit query draw v0"; + + // ---- the HOST, exactly as `sample_query_indexes` drives it at a one-row + // table: one `sample_u64(1)`, then the next thing the transcript would give. + let mut host = crate::hash_pin::block_transcript(SEED); + let index = host.sample_u64(1); + assert_eq!(index, 0, "a two-leaf domain has exactly one query index"); + let host_after: FEE = host.sample_field_element(); + + // ---- the EMITTER, same seed, same sequence. + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let mut t = TranscriptReplay::new(SEED); + let bits = t.sample_u64_pow2(&mut b, 0); + assert!(bits.is_empty(), "a zero-bit draw yields no bits"); + let after = t.sample_ext(&mut b); + b.public(after.as_cell()); + let program = compile(b.finish()); + let exec = execute(&program, &[], &crate::hash_pin::BLOCK_HASHER) + .expect("a zero-bit query draw must emit and execute"); + + assert_eq!( + super::word::word_as_ext(&exec.public_words[0].1).expect("an ext"), + host_after, + "the draw AFTER a zero-bit query index must equal the host's — if the \ + emitter skipped the squeeze, this is the ONLY place it shows" + ); +} + +// ============================== the stage cache ============================== + +/// The named experiment, applied to every cached stage of a run. +/// +/// ⛔ ONE mode decision, read once, applied everywhere. Two independent +/// cache-if-present decisions is how a harness comes to load one stage and prove +/// another, and the run then answers a question nobody asked: an unconditional +/// `A_BUNDLE` export once made a relaunch load a base an earlier run had saved, +/// so `prove_continuation` never ran and the run answered the CONTROL having been +/// launched as the TEST. `A_BUNDLE_MODE` names the mode; nothing infers it from +/// the filesystem. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum CacheMode { + /// No caching: every stage proves. The default, and what CI runs. + Off, + /// Prove each stage and SAVE it. Refuses to overwrite an existing stage. + Prove, + /// LOAD each stage. Refuses to prove one whose cache is missing. + Load, +} + +impl CacheMode { + /// `A_BUNDLE_MODE`, checked against whether a cache location was given. + pub(super) fn from_env(located: bool) -> Self { + match (located, std::env::var("A_BUNDLE_MODE").ok().as_deref()) { + (false, _) => Self::Off, + (true, None) => panic!( + "a cache location is set but A_BUNDLE_MODE is not. Name the \ + experiment — `prove` (prove each stage and save it) or `load` \ + (load each saved stage) — so the harness cannot pick one from \ + filesystem state" + ), + (true, Some("prove")) => Self::Prove, + (true, Some("load")) => Self::Load, + (true, Some(other)) => { + panic!("A_BUNDLE_MODE must be `prove` or `load`, got `{other}`") + } + } + } +} + +/// ★ THE `prove` FIELD, SPLIT — `execute · fill · multi_prove`. +/// +/// The per-node TIMING line prints the whole of `lfm_prove` as one number, and +/// the three phases inside it are three different machines: a single-threaded +/// interpreter, a parallel trace fill, and the only one that reaches the card. +/// A lever aimed at the wrong one reads zero. +/// +/// ⓘ Printed HERE rather than beside the timers, for the stage's label — a +/// bare line is unattributable the moment two proofs are in flight. Silent on a +/// LOADED stage, because nothing proved: `take_prove_split` returns `None` and +/// there is nothing to attribute. +fn print_prove_split(label: &str) { + if let Some(split) = super::proof::take_prove_split() { + println!( + " {label} LFM PROVE: execute {:.2}s · fill {:.2}s · multi_prove {:.2}s{}", + split.execute, + split.fill, + split.multi_prove, + // ⓘ Absent when there was no wait, so a serial line is byte-identical + // to every one already in the campaign's logs and the two arms diff + // on their numbers rather than on their shape. + if split.permit_wait > 0.0 { + format!(" · permit wait {:.2}s", split.permit_wait) + } else { + String::new() + }, + ); + // ⛔ A SECOND LINE, never extra fields on the first. Every log the + // campaign has compared greps `LFM PROVE:` and reads its three numbers + // positionally; widening that line would re-baseline every one of them. + // + // ⓘ `parallel 0 levels` beside a large `levels` is the reading this line + // exists to make visible: it means every level was coalesced onto the + // calling thread and the parallel path never ran, which a wall alone + // would show only as a lever that did nothing. + let e = split.exec; + println!( + " {label} LFM EXEC: levels {} · parallel {} levels / {} hashes · \ + depth pass {:.2}s · setup {:.2}s · hash phase {:.2}s · apply {:.2}s · \ + residue {:.2}s · sum {:.2}s", + e.levels, + e.parallel_levels, + e.parallel_hashes, + e.depth_pass, + e.setup, + e.hash_phase, + e.apply, + e.residue, + e.depth_pass + e.setup + e.hash_phase + e.apply + e.residue, + ); + // ⓘ A THIRD line, for the two questions a wall cannot answer: how big + // the thing `setup` touches is, and what width the pool actually gave. + // `ns/perm` is measured on the coalesced levels of THIS proof — same + // box, same load, no rayon — so `width` is a reading, not an argument. + if e.serial_hashes > 0 { + println!( + " {label} LFM EXEC WIDTH: records {:.0} MiB · {} serial hashes at \ + {:.0} ns/perm · effective width {:.1}", + e.record_bytes as f64 / (1u64 << 20) as f64, + e.serial_hashes, + e.secs_per_perm() * 1e9, + e.effective_width(), + ); + } + } +} + +/// Run `prove`, or substitute a cached proof, as `mode` says — and say which. +/// +/// ★ WHY ONLY THE PROOF IS CACHED. A level's `LfmProof` is the one thing it +/// produces that cannot be re-derived cheaply: the program is a pure function of +/// its children's shapes and re-emits in seconds, and `build_artifacts` is a pure +/// function of the program. So the cache holds proofs and re-derives everything +/// else — which is also what makes a loaded arm's `L` the right number, because a +/// tree-builder carries the harvested children, not the emitters that made them. +/// +/// ⇒ This is the precondition for pricing a LEVEL. Proving levels 0 and 1 in the +/// measuring process leaves their residue live and their peak already set, and a +/// `VmHWM` that never moves afterwards then reports a whole-process bound rather +/// than the level's cost. One arm per process is the fix, and skipping the +/// children's proves is what makes one arm per process affordable. +fn cached_stage( + mode: CacheMode, + path: Option, + label: &str, + prove: impl FnOnce() -> super::proof::LfmProof, +) -> super::proof::LfmProof { + type CachedProof = (super::proof::LfmStarkProof, Vec<(u32, LfmWord)>); + let Some(p) = path.filter(|_| mode != CacheMode::Off) else { + println!(" {label}: PROVED in-process, NOT cached"); + let proved = prove(); + print_prove_split(label); + return proved; + }; + match mode { + CacheMode::Load => { + assert!( + p.exists(), + "A_BUNDLE_MODE=load but {label}'s cache {} is absent: this would \ + silently become a full prove, i.e. a different experiment", + p.display() + ); + let bytes = std::fs::read(&p).expect("the cached stage must read"); + let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len()); + aligned.extend_from_slice(&bytes); + let (proof, public_words) = + rkyv::from_bytes::(&aligned) + .expect("the cached stage must deserialize"); + println!( + " {label}: LOADED from {} ({} bytes) — NOT proved in this process", + p.display(), + bytes.len() + ); + super::proof::LfmProof { + proof, + public_words, + } + } + CacheMode::Prove => { + assert!( + !p.exists(), + "A_BUNDLE_MODE=prove but {label}'s cache {} already exists: \ + refusing to overwrite it, and refusing to silently load it \ + instead", + p.display() + ); + let proved = prove(); + let cached: CachedProof = (proved.proof.clone(), proved.public_words.clone()); + let bytes = + rkyv::to_bytes::(&cached).expect("the stage must serialize"); + if let Some(dir) = p.parent() { + std::fs::create_dir_all(dir).expect("the cache directory must exist"); + } + std::fs::write(&p, &bytes).expect("the stage must persist"); + println!( + " {label}: PROVED and saved to {} ({} bytes)", + p.display(), + bytes.len() + ); + print_prove_split(label); + proved + } + CacheMode::Off => unreachable!("filtered above"), + } +} + +/// [`cached_stage`] for the base continuation, which is not an `LfmProof`. +/// +/// ★ The pair of numbers a bundle cache buys is the measurement, not a +/// convenience: proving the base in-process leaves its residue live when a node +/// runs, so the node's measured carry-in is `L_children + L_base_residue`, while +/// a run that LOADS the bundle carries in `L_children` alone — which is what any +/// separately-written tree-builder would carry. The difference between the two IS +/// the base residue, and therefore how much of any high peak belongs to this +/// harness rather than to the tree. The deserialise path is production's own +/// (`bin/cli/src/main.rs:877-886`). +/// A stamped wall-clock window for a stretch no `TIMING` line covers. +/// +/// ★★ WHY THESE EXIST. The pass-5 profile found **10.4 s at 3.8% GPU +/// utilisation** between the global L2G prove and the first wrap's device hold — +/// the largest contiguous idle block in the whole run — and NOTHING measured it. +/// It could only be decomposed by subtracting assumed terms, which is the exact +/// move this campaign keeps having to retract. Three stamps end that. +/// +/// Reuses `LFM_CARD_TRACE` rather than adding a knob, because it is the same +/// job: hand an external sampler two wall-clock stamps to slice itself by. +fn stamped(label: &str, f: impl FnOnce() -> T) -> T { + if !super::device_permit::trace_enabled() { + return f(); + } + let t0 = stark::prove_split::epoch_secs(); + let t = std::time::Instant::now(); + let out = f(); + println!( + "STAGE {label}: {:.2}s t=[{t0:.3},{:.3}]", + t.elapsed().as_secs_f64(), + stark::prove_split::epoch_secs(), + ); + out +} + +fn cached_bundle( + mode: CacheMode, + path: Option, + prove: impl FnOnce() -> crate::continuation::ContinuationProof, +) -> crate::continuation::ContinuationProof { + let Some(p) = path.filter(|_| mode != CacheMode::Off) else { + println!( + " base: PROVED in-process, NOT cached — carries in L_children + the base residue" + ); + return prove(); + }; + match mode { + CacheMode::Load => { + assert!( + p.exists(), + "A_BUNDLE_MODE=load but {} does not exist: this would silently \ + become a full base prove, i.e. a different experiment", + p.display() + ); + let bytes = std::fs::read(&p).expect("the cached bundle must read"); + let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len()); + aligned.extend_from_slice(&bytes); + let bundle = rkyv::from_bytes::< + crate::continuation::ContinuationProof, + rkyv::rancor::Error, + >(&aligned) + .expect("the cached bundle must deserialize"); + println!( + " base: LOADED from {} — carries in L_children ALONE, no base residue", + p.display() + ); + bundle + } + CacheMode::Prove => { + assert!( + !p.exists(), + "A_BUNDLE_MODE=prove but {} already exists: refusing to overwrite \ + a saved base, and refusing to silently load it instead", + p.display() + ); + let bundle = prove(); + // ⛔ HARNESS-ONLY, AND IT IS INSIDE THE `base:` NUMBER. Serialising + // and writing a 19-epoch bundle is multi-second disk work that no + // production pipeline does, and until this stamp existed it was + // indistinguishable from proving. + stamped("bundle cache write (HARNESS-ONLY)", || { + let bytes = rkyv::to_bytes::(&bundle) + .expect("the bundle must serialize"); + if let Some(dir) = p.parent() { + std::fs::create_dir_all(dir).expect("the cache directory must exist"); + } + std::fs::write(&p, &bytes).expect("the bundle must persist"); + }); + println!( + " base: PROVED in-process and saved to {} — carries in \ + L_children + the base residue", + p.display() + ); + bundle + } + CacheMode::Off => unreachable!("filtered above"), + } +} + +/// Where a stage of THIS run's tree caches, or `None` when `A_CACHE_DIR` is unset. +/// +/// One directory, one file per stage, named for the stage rather than for the +/// run: a level-2 arm must load exactly the level-1 proofs an earlier arm saved, +/// and a name that encoded anything else would let two different trees share a +/// cache entry. +fn stage_path(dir: Option<&str>, stage: &str) -> Option { + dir.map(|d| std::path::Path::new(d).join(format!("{stage}.rkyv"))) +} + +/// Prove one aggregation node and hand it back as a CHILD of the next level. +/// +/// The whole composition argument in one function: a node's proof is a plain +/// per-table `MultiProof`, so `real_child` reads it exactly as it reads a wrap's, +/// and the layout that describes it is `SchemaLayout::node`. Nothing about the +/// level appears here — which is what "the same emitter serves every level" +/// means operationally. +#[allow(clippy::too_many_arguments)] +fn prove_node_as_child( + label: &str, + children: &[RealChild], + layouts: &[super::per_table_aggregator::SchemaLayout], + labels: &[&[u64]], + label_range: (u64, u64), + out_halves: usize, + opts: &crate::ProofOptions, + mode: CacheMode, + cache: Option, +) -> (RealChild, super::per_table_aggregator::SchemaLayout) { + use super::per_table_aggregator::NodePublishSet; + + let program = node_program( + children, + layouts, + labels, + label_range, + NodePublishSet::Aggregation, + ); + prove_node_program_as_child(label, &program, children, out_halves, opts, mode, cache) +} + +/// [`prove_node_as_child`] for a caller that has ALREADY emitted the program. +/// +/// ⛔ EMITTING IT TWICE IS NOT FREE. The tree driver builds a node's program to +/// take its census and its chip panel, and then called `prove_node_as_child`, +/// which emitted the identical program a second time — 7.4M instructions per +/// node, twice, at every one of 21 nodes. It was invisible because both halves +/// were correct and the only symptom was wall clock, which is exactly the +/// quantity we had agreed to treat as context. +/// +/// ⇒ The census and the prove now share one program, and the split timings below +/// are what would have made the duplication visible in the first place: an +/// unattributed "60 s per node" cannot say whether it is emission, artifacts or +/// the prove. +#[allow(clippy::too_many_arguments)] +fn prove_node_program_as_child( + label: &str, + program: &LfmProgram, + children: &[RealChild], + out_halves: usize, + opts: &crate::ProofOptions, + mode: CacheMode, + cache: Option, +) -> (RealChild, super::per_table_aggregator::SchemaLayout) { + use super::per_table_aggregator::SchemaLayout; + use std::time::Instant; + + let wait_before = super::device_permit::waited_secs(); + let t = Instant::now(); + let arenas: Vec> = children.iter().flat_map(child_arena_words).collect(); + let t_arenas = t.elapsed().as_secs_f64(); + let t = Instant::now(); + let artifacts = super::program_census::build_artifacts_counted( + program, + opts, + crate::hash_pin::BLOCK_HASHER, + ); + let t_artifacts = t.elapsed().as_secs_f64(); + // ★ WHAT THE ARTIFACT BUILD SPENT QUEUEING rather than committing. Bracketed + // and subtracted rather than taken-and-cleared: a counter that clears + // couples every reader to every other one. + let wait_artifacts = (super::device_permit::waited_secs() - wait_before).max(0.0); + let wait_before = super::device_permit::waited_secs(); + let t = Instant::now(); + let proved = cached_stage(mode, cache, label, || { + super::proof::lfm_prove(program, &artifacts, &arenas, opts) + .expect("an aggregation node must prove") + }); + let t_prove = t.elapsed().as_secs_f64(); + // ⛔ A LIVE MARK, NOT A HIGH-WATER — and within-run is not enough on its own. + // + // This line used to print `peak_rss_gib()`, a `VmHWM`. Read across leaf 0, + // leaf 1 and an inner node it never moved, and the flatness was published as + // "the tree does not grow per level". A high-water only rises: with the base + // and four wraps already proved in the same process and no mark before leaf + // 0, the reading bounds the WHOLE process and prices no level inside it — + // level 1 included. Being a within-run comparison does not rescue it. + // ⇒ The live figure plus `t=` is what prices a phase: `L` before, `L` after, + // and an external sampler sliced to the window between the two stamps. One + // arm per process (`A_CACHE_DIR` + `A_BUNDLE_MODE=load`) is what makes the + // live figure mean the level rather than the run. + println!(" {label}: {} instructions", program.instrs.len()); + mark(&format!("AFTER {label}")); + let layout = SchemaLayout::node(out_halves); + layout.assert_covers(proved.public_words.len()); + let t = Instant::now(); + let (child, t_verify) = real_child_timed(artifacts, opts.clone(), &proved); + let t_harvest = t.elapsed().as_secs_f64(); + // ★ THE SPLIT, because "60 s per node" attributes nothing. Which half a + // second cache layer would have to hold — the artifacts or the harvest — + // is a different build depending on this line, and guessing it is how a + // campaign builds the wrong cache. + // + // ★ And the harvest carries its OWN split, because the two halves belong to + // different owners: `verify` is the harness's assert, `replay` is the + // transcript walk a driver genuinely needs. Summed, the field prices a + // phase no real pipeline has. + // ★★ THE WAIT IS CONTAINED IN THE TWO FIELDS ABOVE, NOT ADDITIONAL TO THEM. + // `build_artifacts` and `prove` are wall times and a queued worker's wait + // sits inside whichever was running, which is why the same node reads 4.1 s + // serial and 7.1 s at two siblings. Naming it here keeps the per-node + // numbers comparable across arms AND closes the level's accounting: at K + // workers the level's worker-seconds are K × wall, of which the permit's + // `held` is the card, the waits are the queue, and the rest is host work. + let wait_prove = (super::device_permit::waited_secs() - wait_before).max(0.0); + let wait_total = wait_artifacts + wait_prove; + println!( + " {label} TIMING: arenas {t_arenas:.1}s · build_artifacts {t_artifacts:.1}s \ + · prove {t_prove:.1}s · harvest {t_harvest:.1}s (verify {t_verify:.2} + replay {:.2}){}", + t_harvest - t_verify, + if wait_total > 0.0 { + format!( + " · permit wait {wait_total:.2}s (inside build_artifacts {wait_artifacts:.2} \ + + prove {wait_prove:.2})" + ) + } else { + String::new() + }, + ); + (child, layout) +} + +/// ★ THE INNER NODE — a node whose children are NODE proofs. +/// +/// # What this adds over the leaf gate +/// +/// The leaf verifies wraps; this verifies leaves. Three things differ and each +/// is exercised here for the first time: +/// +/// 1. **`SchemaLayout::node`** rather than `::wrap` — a different head (a node +/// has FAN_IN Phase A's and so no single `(z, α)`), a four-word label run +/// rather than two, and no trailing bus total. +/// 2. **A label RANGE per child**: each leaf carries the first and last label of +/// its subtree, and the inner node pins both ends of both. That is what makes +/// contiguity across sibling subtrees a consequence of the pins rather than a +/// separate check. +/// 3. **The L2G fold composing**: each leaf published a fold over ITS wraps' +/// roots, and the inner node folds those two folds. A single-child node folds +/// to identity and would not exercise `hash_pair` at this level, which is why +/// this needs two real leaves rather than one. +/// +/// ⚠ Building this arm is what found the bug it now covers: a node used to +/// publish its fold as digest CELLS (four lanes in one word) while a wrap +/// publishes its root as lanes (one lane per word), and `emit_node_publishes` +/// reads `lanes[0]` of each published l2g word. A node child would have handed +/// ONE felt to `digest_from_lanes` where four are required. The fix removed the +/// asymmetry rather than parameterising it, so both layouts now read alike. +/// +/// # Cost, and why the epoch requirement is asserted rather than worked around +/// +/// A two-level tree at fan-in N needs N² epochs: N wraps per leaf, N leaves. +/// Padding the shortfall would mean a pad child with no epoch to belong to, +/// which breaks the register chain and the label pin — the same trade rejected +/// when the tree shape was priced. So this asserts the fixture is deep enough +/// and names the number if it is not. +#[test] +#[ignore = "box tier: proves FAN_IN^2 wraps, FAN_IN leaf nodes and one inner node"] +fn the_inner_node_verifies_two_leaf_nodes() { + use super::epoch_tests::Publishes; + use super::per_table_aggregator::{FAN_IN, SchemaLayout}; + use super::program_census::build_artifacts_counted; + use super::proof::lfm_prove; + use std::time::Instant; + + let elf_bytes = super::proof_fixture::read_inner_elf(); + let inner = super::proof_fixture::fixture_options(); + // ★ A LOCAL epoch size, NOT the shared `FIXTURE_EPOCH_LOG2`. + // + // A two-level tree needs FAN_IN^2 epochs and the shared constant selects two. + // `epoch_size_log2` is a PARAMETER of `prove_continuation` (floor 2), so a + // smaller epoch here yields more of them from the same guest without moving + // the ground under the nine gates that already run against the shared + // constant — every one of which would otherwise be re-baselined by a change + // whose only purpose is to give THIS test more epochs. + // + // ⚠ Smaller epochs mean shallower tables, which is where the degenerate + // shapes live. That is a feature: the one-row sub-proof and the one-leaf + // Merkle tree were both found this way, and both are now gated in + // milliseconds. If a third appears, it is a shape the emitter has to handle + // and this is the cheapest place to find it. + let epoch_log2 = super::proof_fixture::FIXTURE_EPOCH_LOG2 - 1; + + // ★★ ONE ARM PER PROCESS, which is what lets a LEVEL be priced. + // + // With `A_CACHE_DIR` set and `A_BUNDLE_MODE=load`, every stage below level 2 + // is read from disk instead of proved: the base, the four wraps and the two + // leaf nodes. The inner node is then the only prove in the process, so `L` + // before it is exactly what a tree-builder would carry — the bundle plus two + // level-1 children — and the live marks around it price the level rather than + // the run. An `A_BUNDLE_MODE=prove` arm writes that cache; the arms are + // otherwise identical, and neither picks itself from filesystem state. + // + // ⚠ Loading a level-1 proof does NOT skip re-emitting its program: a node's + // program is a pure function of its children's shapes and its `program_id` + // must match the loaded proof, so the wrap harvests are still built (cheaply, + // artifacts only) to re-emit the leaf programs. They are dropped before the + // inner node proves, because they are not its children. + let cache_dir = std::env::var("A_CACHE_DIR").ok(); + let mode = CacheMode::from_env(cache_dir.is_some()); + println!( + "★ INNER NODE ARM: cache {mode:?}{}", + match &cache_dir { + Some(d) => format!(" in {d}"), + None => String::new(), + } + ); + + let bundle = cached_bundle(mode, stage_path(cache_dir.as_deref(), "bundle"), || { + crate::continuation::prove_continuation(&elf_bytes, &[], epoch_log2, &inner) + .expect("the fixture continuation must prove") + }); + let needed = FAN_IN * FAN_IN; + assert!( + bundle.num_epochs() >= needed, + "a two-level fan-in-{FAN_IN} tree needs {needed} epochs; at epoch_log2 \ + {epoch_log2} this guest gives {}. Lower `epoch_log2` further (its floor is \ + 2) or use a longer guest — do NOT pad, a pad child belongs to no epoch \ + and breaks the register chain and the label pin", + bundle.num_epochs() + ); + + let wrap_opts = super::proof::aggregation_wrap_options(); + let label_of = |k: usize| crate::tables::local_to_global::epoch_label(k as u64); + + // ---- level 0: one wrap per epoch, then level 1: one leaf per FAN_IN wraps. + let t = Instant::now(); + let mut leaves = Vec::with_capacity(FAN_IN); + let mut leaf_layouts = Vec::with_capacity(FAN_IN); + let mut leaf_labels: Vec<[u64; 2]> = Vec::with_capacity(FAN_IN); + let epoch_konsts = super::epoch_tests::EpochConstants::load(&elf_bytes, &inner, None) + .expect("the inner ELF and its DECODE commitment must build once"); + for leaf in 0..FAN_IN { + let mut wraps = Vec::with_capacity(FAN_IN); + let mut wrap_layouts = Vec::with_capacity(FAN_IN); + let mut wrap_labels: Vec<[u64; 1]> = Vec::with_capacity(FAN_IN); + let mut out_halves = 0usize; + for i in 0..FAN_IN { + let k = leaf * FAN_IN + i; + let e = + super::epoch_tests::real_epoch_from_constants(&inner, &epoch_konsts, &bundle, k) + .expect("every epoch must reconstruct from proofs alone"); + out_halves = e.statement.public_output_len.div_ceil(4); + let program = + super::epoch_tests::epoch_program_publishing(&e, true, Publishes::Aggregation); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let artifacts = + build_artifacts_counted(&program, &wrap_opts, crate::hash_pin::BLOCK_HASHER); + let proved = cached_stage( + mode, + stage_path(cache_dir.as_deref(), &format!("wrap-{leaf}-{i}")), + &format!("wrap {k} (leaf {leaf}, child {i})"), + || { + lfm_prove(&program, &artifacts, &arenas, &wrap_opts) + .expect("the epoch wrap must prove") + }, + ); + let layout = SchemaLayout::wrap(out_halves); + layout.assert_covers(proved.public_words.len()); + wrap_layouts.push(layout); + wrap_labels.push([label_of(k)]); + wraps.push(real_child(artifacts, wrap_opts.clone(), &proved)); + } + let refs: Vec<&[u64]> = wrap_labels.iter().map(|l| &l[..]).collect(); + let range = ( + label_of(leaf * FAN_IN), + label_of(leaf * FAN_IN + FAN_IN - 1), + ); + let (child, layout) = prove_node_as_child( + &format!("leaf {leaf} (level 1)"), + &wraps, + &wrap_layouts, + &refs, + range, + out_halves, + &wrap_opts, + mode, + stage_path(cache_dir.as_deref(), &format!("leaf-{leaf}")), + ); + println!( + " leaf {leaf}: {} published words, {} sub-proofs", + child.public_words.len(), + child.tables.len() + ); + // ★ THE LEVEL-0 HARVESTS GO, and the trough is the point. + // + // A tree-builder proving level 2 holds its level-1 children and nothing + // below them. Keeping the wraps alive here would inflate `L` by a whole + // level that the real thing would have released, and a high-water could + // not have shown the difference — only a live mark on either side of the + // drop can. Emission borrowed their shapes; the leaf `RealChild` owns its + // own artifacts, so nothing here is still borrowed. + drop(wraps); + drop(wrap_layouts); + mark(&format!("AFTER dropping leaf {leaf}'s wrap harvests")); + leaves.push(child); + leaf_layouts.push(layout); + leaf_labels.push([range.0, range.1]); + } + println!( + " {FAN_IN} leaf nodes over {needed} wraps in {:.1}s", + t.elapsed().as_secs_f64() + ); + + // ---- level 2: the inner node, over NODE proofs. + let refs: Vec<&[u64]> = leaf_labels.iter().map(|l| &l[..]).collect(); + let range = (leaf_labels[0][0], leaf_labels[FAN_IN - 1][1]); + let out_halves = leaf_layouts[FAN_IN - 1].out_halves; + let t = Instant::now(); + // ⛔ The inner node is NEVER cached — it is the measurement. `CacheMode::Off` + // here regardless of the arm, so a `load` arm cannot accidentally read back a + // level-2 proof and report the load as the level's cost. + mark("BEFORE the inner node (this live figure is L for level 2)"); + let (inner_node, inner_layout) = prove_node_as_child( + "the INNER node (level 2)", + &leaves, + &leaf_layouts, + &refs, + range, + out_halves, + &wrap_opts, + CacheMode::Off, + None, + ); + println!( + "\n★ INNER NODE PROVED AND VERIFIED (a node over {FAN_IN} NODE proofs)\n \ + prove+harvest {:.1}s\n {} published words, {} sub-proofs\n \ + schema {} + head {}\n peak RSS {:?} GiB", + t.elapsed().as_secs_f64(), + inner_node.public_words.len(), + inner_node.tables.len(), + inner_layout.schema_words(), + inner_layout.head, + super::wrap_tests::peak_rss_gib(), + ); + + // ---- the composition property, asserted rather than implied: an inner + // node's published schema has the SAME shape as its children's, which is + // what lets the level above it use the identical emitter. + assert_eq!( + inner_layout.total(), + leaf_layouts[0].total(), + "a node's published schema must not change with its level — that is what \ + makes the same emitter serve the level above" + ); +} + +/// ★ A DEPTH-ZERO MERKLE WALK STILL BINDS THE LEAF TO THE ROOT. +/// +/// # The shape +/// +/// A one-row trace at blowup 2 has a two-leaf LDE, one row PAIR, and therefore a +/// Merkle tree with a single leaf and no levels — the leaf hash IS the root. +/// `SubProofShape::check` refused it outright (`merkle_depth >= 1`), which is +/// what `the_leaf_node_verifies_and_binds_two_wraps` hit once the query sampler +/// stopped refusing zero index bits. +/// +/// # Why this is not "make the walk a no-op" +/// +/// It is not a no-op and must not become one. Both sides do the same thing at +/// depth 0 and neither needs a special case: +/// +/// - host `verify_merkle_path_from_leaf_hash` loops over an empty path and +/// returns `root_hash == hashed_value`; +/// - `emit_group_authentication` hashes the leaf, walks zero levels, and asserts +/// the result equals the committed root. +/// +/// The compare is the entire binding, and it survives. **The rejection arm below +/// is what proves that** — if relaxing the shape check had let the walk skip its +/// root comparison, the honest arm would still pass and only this one would fail. +/// +/// # Where the host enters +/// +/// The root is not invented here: it is the leaf hash the emitter itself +/// computes, and `emit_leaf_hash`'s agreement with the host's backend is gated +/// separately by `algebraic_commit`'s leaf/parent differential. Composing the two +/// is what makes this emitter-versus-host rather than emitter-versus-itself, and +/// it is stated rather than assumed because the composition is the argument. +#[test] +fn a_depth_zero_walk_still_binds_leaf_to_root() { + use super::sub_proof::{ + GroupCommitment, GroupOpening, GroupShape, emit_group_authentication, emit_leaf_hash, + }; + + const COLS: usize = 3; + let shape = GroupShape { + num_columns: COLS, + is_ext: false, + }; + let values: Vec = (0..shape.num_values() as u64) + .map(|i| FE::from(7 * i + 1)) + .collect(); + + // ---- the root, from the emitter's own leaf hash over those values. + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(shape.num_values() as u32); + let cells: Vec<_> = (0..shape.num_values() as u32) + .map(|i| b.hint_word(arena, i)) + .collect(); + let leaf = emit_leaf_hash(&mut b, shape, &cells); + for cell in leaf.cells() { + b.public(*cell); + } + let leaf_program = compile(b.finish()); + let leaf_arena: Vec = values.iter().map(|v| base_word(*v)).collect(); + let leaf_exec = execute( + &leaf_program, + std::slice::from_ref(&leaf_arena), + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("the leaf hash must execute"); + let root_words: Vec = leaf_exec.public_words.iter().map(|(_, w)| *w).collect(); + + // ---- the authentication at depth ZERO: no bits, no siblings. + let build = || { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let a_vals = b.declare_arena(shape.num_values() as u32); + let a_root = b.declare_arena(root_words.len() as u32); + let vals: Vec<_> = (0..shape.num_values() as u32) + .map(|i| b.hint_word(a_vals, i)) + .collect(); + let commitment = GroupCommitment::hint(&mut b, a_root, 0, shape); + emit_group_authentication( + &mut b, + &commitment, + &GroupOpening { + values: vals, + siblings: Vec::new(), + }, + &[], + ); + compile(b.finish()) + }; + let program = build(); + + // ---- the honest arm: the leaf hash IS the root, so this must execute. + execute( + &program, + &[leaf_arena.clone(), root_words.clone()], + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("★ a one-leaf tree must authenticate against its own leaf hash"); + + // ---- ★ THE REJECTION ARM — the one that proves the compare survives. + let mut wrong = root_words.clone(); + wrong[0][0] += FE::one(); + assert!( + execute( + &program, + &[leaf_arena, wrong], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "a depth-zero walk must still REJECT a root that is not the leaf hash — \ + if this passes, the walk stopped binding and the honest arm proves nothing" + ); + println!( + " ✓ depth-0 walk: {} column pair binds to its root, and a moved root is rejected", + COLS + ); +} + +/// Both memory numbers at once — `(VmRSS, VmHWM)` in GiB, live and high-water. +/// +/// ⚠ `VmHWM` alone cannot see a TROUGH, and the trough is half the model. A +/// high-water mark only rises, so a mark taken after a phase reports the largest +/// the process has EVER been, not what that phase left resident. That difference +/// decides whether the next phase's peak is `live + its own working set` or is +/// hidden under an earlier phase's mark entirely — and on this workload the two +/// diverge hard: box A measured `VmHWM` static at 72.50 GiB while `VmRSS` +/// oscillated between 19.98 and 58.13. +/// +/// A `ps` sample of the LIVE figure once reached this lane as if it were a +/// high-water mark, and every derivation built on it was wrong by the gap +/// between them. Both are printed so that cannot recur. +fn rss_marks() -> (Option, Option) { + let read = |key: &str| -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|l| l.starts_with(key))?; + let kb: f64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb / (1024.0 * 1024.0)) + }; + (read("VmRSS:"), read("VmHWM:")) +} + +/// One labelled mark: `live` is what the next phase carries in, `high-water` is +/// what the process has ever held, `t` is the wall clock the sampler shares. +/// +/// ⚠ `t` is UNIX-epoch seconds, not elapsed. An external VRAM sampler +/// (`nvidia-smi` at 10 Hz) has no view of this process's phases, so a shared +/// clock is the only thing that lets its trace be sliced to the NODE window — +/// without it a whole-run GPU peak gets attributed to whichever phase the reader +/// assumes, which is the same masking that made a process high-water read as one +/// tree level's cost. +fn mark(label: &str) { + let (rss, hwm) = rss_marks(); + let t = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + println!(" MARK {label}: live {rss:?} GiB / high-water {hwm:?} GiB / t={t:.1}"); +} + +/// ★ THE TREE'S SHAPE COMES FROM THE EPOCH COUNT, not a constant. +/// +/// Pure arithmetic — no proving, so it runs on every suite. It exists because +/// the shape was planned more than once against a number that was not the +/// block's. Block 25368371 is **23,935,913** cycles: 6 epochs at 2^22, **12 at +/// 2^21**. The retired **74,819,518** — the July prebuilt guest, ~3x dearer at +/// identical work — gives 18 and 36 instead, i.e. a tree two levels too deep. A +/// tree built to a constant answers whichever question that constant came from. +/// +/// ⚠ **2^21 is the posture of record.** 2^22 does not fit a 32 GiB card at +/// blowup 4 — the resident per-table LDE + snapshot + tree roughly doubles from +/// 2^21 while the card does not, and it fails even at one table in flight. The +/// 2^22 rows below are kept as a CONTROL on the arithmetic, not as a posture. +/// +/// The arities are asserted rather than the level count alone, because the +/// LEFTOVER RULE is the design decision: every node takes `1..=fan_in` children +/// and an odd level ends in a short node rather than carrying a proof upward to +/// a parent with mixed-shape children. Asserting only the depth would pass under +/// either rule. +#[test] +fn the_tree_shape_matches_the_epoch_count() { + use super::per_table_aggregator::{tree_node_count, tree_shape}; + + // The real block, both postures. Epoch counts are `ceil(cycles / 2^k)` for + // block 25368371's **23,935,913** cycles: the guest built with fat LTO over + // one codegen unit, the runtime's own memcpy and the Keccak final block + // absorbed without its buffer. + // + // ⚠ THREE stale figures now, and they went stale the same way. + // + // 74,819,518 is the JULY prebuilt guest; thin-LTO and the accelerators made + // it ~47% cheaper at identical work — same block, same keccak 10,478, same + // ECSM 116. The stale figure still sits in a table in + // `real_block_benchmark_selection.md`, with the correction in an addendum + // BELOW it. Reading the table row and stopping gives 18/36 epochs and a + // tree two levels too deep. + // + // 39,631,559 is the 2026-09-07 guest, and it is what THIS comment said + // until #894. That bump cost another ~23% at identical work, so the 2^21 + // posture went from 19 epochs to 15 and the 2^22 posture from 10 to 8 — + // one whole level off the fan-in-2 tree in both. + // + // 30,498,818 is the #894 guest. The guest build that followed (fat LTO, + // the runtime memcpy, the leaner Keccak final block) cost 21.5% less at + // identical work: 15 epochs became 12 at 2^21 and 8 became 6 at 2^22. + // Nothing failed when either changed: every number below is a literal, so + // the test went on passing while describing a block this repository no + // longer proves. Any epoch count, tree shape or IDENTITY-line count derived + // from an old figure is stale, and that includes saved run logs. + for (epochs, fan_in, levels, nodes) in [ + (6usize, 2usize, 3usize, 6usize), // 2^22: 6 -> 3 -> 2 -> 1 + (6, 3, 2, 3), // 2^22: 6 -> 2 -> 1 + (12, 2, 4, 12), // 2^21: 12 -> 6 -> 3 -> 2 -> 1 + (12, 3, 3, 7), // 2^21: 12 -> 4 -> 2 -> 1 + (12, 4, 2, 4), // 2^21, fan-in 4 (the WHIR tree's opt-out): 12 -> 3 -> 1 + (12, 5, 2, 4), // 2^21, pure WHIR's default (the WHIR tree only): 12 -> 3 -> 1 + ] { + let shape = tree_shape(epochs, fan_in); + assert_eq!( + shape.len(), + levels, + "levels at {epochs} epochs, fan-in {fan_in}" + ); + assert_eq!( + tree_node_count(&shape), + nodes, + "aggregator proofs at {epochs} epochs, fan-in {fan_in}" + ); + // Every node takes 1..=fan_in, and each level consumes exactly what the + // one below produced — the invariant a carried leftover would break. + let mut below = epochs; + for (i, level) in shape.iter().enumerate() { + assert!( + level.arities.iter().all(|a| (1..=fan_in).contains(a)), + "level {i} has an arity outside 1..={fan_in}: {:?}", + level.arities + ); + assert_eq!( + level.arities.iter().sum::(), + below, + "level {i} must consume exactly the {below} proofs beneath it" + ); + below = level.nodes(); + } + assert_eq!(below, 1, "the tree must close to a single root proof"); + } + + // A degenerate block is a legal shape, not an error: one epoch is already + // the root and the interior is empty. + assert!(tree_shape(1, 2).is_empty(), "one epoch needs no interior"); + + // ★ The leftover rule is EXERCISED, not merely permitted: a shape that + // never produced a short node would pass every assertion above while + // testing nothing about wrap-versus-carry. + let short_levels = |epochs: usize, fan_in: usize| -> usize { + tree_shape(epochs, fan_in) + .iter() + .filter(|l| l.arities.iter().any(|a| *a < fan_in)) + .count() + }; + + // The shipped posture. ⚠ It is WEAKER than it was: at 19 epochs this was 3 + // short levels (19, 5 and 3 all left one over), and at 12 it is 1 — the + // 3 -> 2 level — because 12 and 6 are even. So the real block no longer + // covers this rule on its own. + assert_eq!( + short_levels(12, 2), + 1, + "12 at fan-in 2 must still exercise the leftover rule at least once" + ); + assert_eq!( + short_levels(8, 2), + 0, + "8 at fan-in 2 is a perfect binary tree — recorded so the row below is \ + understood as necessary rather than redundant" + ); + + // ⇒ A SYNTHETIC row carries the coverage the guest bump took away. 19 is + // kept as the shape, not because any block has 19 epochs now, but because + // it is the smallest one that makes three separate levels carry a leftover, + // which is what distinguishes wrapping from carrying. + assert_eq!( + short_levels(19, 2), + 3, + "the synthetic shape must exercise the leftover rule on three levels" + ); + assert!( + short_levels(12, 3) >= 2, + "fan-in 3 at the shipped epoch count must leave a short node on at \ + least two levels" + ); + assert_eq!( + short_levels(12, 4), + 1, + "fan-in 4 fills level 1 (4 / 4 / 4); its short node is the root's level (3)" + ); + assert_eq!( + short_levels(12, 5), + 2, + "pure WHIR's default leaves a short node at level 1 (5 / 5 / 2) and at the root's level (3)" + ); +} + +// ======================= the tree's in-order pool ======================= + +pub(super) use super::tree_run::in_index_order; + +/// ★★★ THE ORDERING GATE, with the completion order FORCED to the reverse of +/// the index order. +/// +/// Index `j` sleeps for `(n - j)` ticks, so index 0 finishes LAST and index +/// `n-1` first. A drain in completion order returns the reverse; a drain by +/// index returns what the serial loop would have. Without the forcing this test +/// would pass on a scheduler that happens to finish in order, which is the +/// failure mode the permit's own first test had. +/// +/// This is the assertion behind "scheduling is invisible to the bytes": a +/// level's children, layouts and labels are three parallel vectors and the +/// level above takes contiguous subslices of each, so a permuted drain builds a +/// different tree whose label pins still verify one subtree at a time. +#[test] +fn in_index_order_returns_index_order_when_completion_is_reversed() { + const N: usize = 8; + let finished: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + let out = in_index_order(N, 4, |j| { + std::thread::sleep(std::time::Duration::from_millis(20 * (N - j) as u64)); + finished.lock().expect("the order log").push(j); + j * 10 + }); + assert_eq!( + out, + (0..N).map(|j| j * 10).collect::>(), + "results must arrive in INDEX order" + ); + let finished = finished.into_inner().expect("the order log"); + assert_ne!( + finished, + (0..N).collect::>(), + "the test did not force a reordering, so it proved nothing: {finished:?}" + ); +} + +/// One worker, and the same answer — the control arm runs `task` inline and +/// must not be a different computation from the parallel one. +#[test] +fn in_index_order_with_one_worker_matches_the_parallel_result() { + let serial = in_index_order(6, 1, |j| j * j); + let parallel = in_index_order(6, 3, |j| j * j); + assert_eq!(serial, parallel); +} + +/// ⚠ A worker panic must arrive with its MESSAGE, not as "a scoped thread +/// panicked". Eleven anonymous failures in one box suite run is what the +/// payload capture costs to omit. +#[test] +fn in_index_order_re_raises_a_worker_panic_with_its_message() { + let caught = std::panic::catch_unwind(|| { + in_index_order(4, 2, |j| { + assert_ne!(j, 2, "NODE 2 SAYS SO"); + j + }) + }); + let payload = caught.expect_err("a panicking task must panic the caller"); + let msg = payload + .downcast_ref::() + .map(String::as_str) + .or_else(|| payload.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!( + msg.contains("NODE 2 SAYS SO"), + "the worker's own message must survive, got: {msg}" + ); +} diff --git a/prover/src/lfm/preprocessed.rs b/prover/src/lfm/preprocessed.rs new file mode 100644 index 000000000..ecca014c0 --- /dev/null +++ b/prover/src/lfm/preprocessed.rs @@ -0,0 +1,762 @@ +//! The preprocessed-column checks the WHIR verifier owes, in the machine. +//! +//! On the multilinear path nothing but `check_preprocessed` says a preprocessed +//! table is the one the program implies: the commitment binds a prover to the +//! columns it committed, not to the right ones. The host discharges that by +//! rebuilding each column and evaluating its multilinear extension at the +//! reduced point — `2^n` per column, and a real epoch carries 16,777,888 such +//! fold steps. A verifier that is itself proven cannot pay it: BITWISE alone is +//! eleven columns of 2^20, about 23 M straight-line rows, four times the whole +//! of today's recursion wrap. +//! +//! It does not have to, because BITWISE's columns are **multilinear in the row +//! index**. [`crate::tables::bitwise::preprocessed_mle_at`] is the closed form +//! and carries the argument; this module is the same expression emitted, so the +//! two can be read side by side and a divergence is visible rather than +//! inferred. +//! +//! # What this leg is NOT +//! +//! It is not a substitute for binding the ELF-derived columns. DECODE's five +//! are a function of the program, not of the index, and no closed form exists +//! for them — they need a commitment whose root is pinned in the program text. +//! This leg covers the columns whose values follow from where they sit. + +use crate::tables::bitwise::{NUM_PRECOMPUTED_COLS, NUM_VARS}; +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +use super::builder::{Ext, LfmBuilder}; +use super::word::{LfmWord, ext_word}; + +/// INSTRUCTIONS this leg emits, by construction rather than by measurement. +/// +/// Every term names the shape it comes from, so a change to the emitter that +/// does not change this number is as visible as one that does. Pinned against +/// the emitter by `preprocessed_tests::the_bitwise_leg_emits_its_closed_form`. +/// +/// ⚠ Instructions, not ALU rows — the two differ by the interned constant, and +/// the first draft of this function counted rows and came in one short. A +/// `Const` is an `LFM_CONST` row like any other and the emitter pays for it. +/// +/// Identity operations are not emitted and so are not counted: `2^0` is that +/// same interned one, a running sum starts at its first term rather than at +/// zero, and the final `suffix` advance feeds nothing. +pub const fn bitwise_preprocessed_rows() -> usize { + let interned_one = 1; // `1`, interned once: 2^0, every (1 − b), and each accumulator's identity + let pow2 = 15; // 2^1..2^15 by doubling; 2^0 is that constant + let linear = 7 + 7 + 3; // X, Y, Z: one MulAdd per bit past the first + let logic = 4 + 7 * 7; // AND/OR/XOR; bit 0 is cheaper because its weight is one + let is_zero = 20 + 19; // eq(0, ·): a subtract per bit, a product per bit past the first + let prefix = 15; // the halfword's running linear form + let z_not = 4; // the four (1 − z_bit) factors, hoisted out of the sixteen indicators + let indicators = 16 * 3; // eq(j, Z): four factors, three products + let sll = 1 + 15 * 3; // j = 0 needs no weight and no accumulate + let sllc_and_suffix = 3 + 13 * 4 + 2; // j = 0 is empty, j = 15 advances nothing + interned_one + + pow2 + + linear + + logic + + is_zero + + prefix + + z_not + + indicators + + sll + + sllc_and_suffix +} + +/// ★ The eleven BITWISE preprocessed columns' multilinear extensions at one +/// point, emitted — `O(NUM_VARS)` rows where the fold is `O(2^NUM_VARS)`. +/// +/// `point` is in the order [`multilinear::mle::Mle::evaluate_in`] takes it, so +/// its FIRST coordinate binds the HIGH half of the table and coordinate `i` +/// carries index bit `NUM_VARS - 1 - i`. The two run in opposite directions; +/// the reversal is written once, here and in the host closed form, and nowhere +/// else. +/// +/// Returns the columns in [`crate::tables::bitwise::generate_bitwise_row`]'s +/// order: X, Y, Z, AND, OR, XOR, MSB8, MSB16, ZERO, SLL, SLLC. +pub fn emit_bitwise_preprocessed(b: &mut LfmBuilder, point: &[Ext]) -> [Ext; NUM_PRECOMPUTED_COLS] { + assert_eq!( + point.len(), + NUM_VARS, + "the point must have one coordinate per BITWISE row-index bit" + ); + // The variable carried by index bit k. + let v: Vec = (0..NUM_VARS).map(|k| point[NUM_VARS - 1 - k]).collect(); + let one = b.ext_const(&FEE::one()); + + // Powers of two by doubling. 2^0 is the constant itself, so a weight of one + // multiplies nothing. + let mut pow2 = Vec::with_capacity(16); + pow2.push(one); + for i in 1..16 { + let prev = pow2[i - 1]; + pow2.push(b.eadd(prev, prev)); + } + + // X, Y, Z: linear forms over disjoint bit ranges, accumulated from the + // lowest bit so the first term needs no weight. + let mut linear = [one; 3]; + for (slot, (lo, len)) in [(0usize, 8usize), (8, 8), (16, 4)].into_iter().enumerate() { + let mut acc = v[lo]; + for i in 1..len { + acc = b.emul_add(pow2[i], v[lo + i], acc); + } + linear[slot] = acc; + } + let [x, y, z] = linear; + + // AND, OR, XOR: one product of two DISTINCT variables per bit, so each is + // degree one in each of them. + let mut and = one; + let mut or = one; + let mut xor = one; + for i in 0..8 { + let (a, c) = (v[i], v[8 + i]); + let ab = b.emul(a, c); + let sum = b.eadd(a, c); + let or_bit = b.esub(sum, ab); + let xor_bit = b.esub(or_bit, ab); + if i == 0 { + // Weight one: the accumulators start at the bit itself. + and = ab; + or = or_bit; + xor = xor_bit; + } else { + and = b.emul_add(pow2[i], ab, and); + or = b.emul_add(pow2[i], or_bit, or); + xor = b.emul_add(pow2[i], xor_bit, xor); + } + } + + // Bit 7 of X, and bit 15 of X + 256·Y, which is bit 7 of Y. + let msb8 = v[7]; + let msb16 = v[15]; + + // ZERO is eq(0, ·): the product over every bit of (1 − b). + let mut is_zero = one; + for (k, vk) in v.iter().enumerate().take(NUM_VARS) { + let term = b.esub(one, *vk); + is_zero = if k == 0 { term } else { b.emul(is_zero, term) }; + } + + // Halfword bit i is index bit i for i < 16, so one prefix sweep serves both + // shifts: prefix[m] = Σ_{i ≤ m} 2^i · bit(i). + let mut prefix = Vec::with_capacity(16); + let mut running = v[0]; + prefix.push(running); + for i in 1..16 { + running = b.emul_add(pow2[i], v[i], running); + prefix.push(running); + } + + // The four complements, hoisted: sixteen indicators share them. + let z_not: Vec = (0..4).map(|k| b.esub(one, v[16 + k])).collect(); + + let mut sll = one; + let mut sllc = one; + // `suffix` holds halfword >> (16 − j) on entry to iteration j, advanced by + // suffix_{j+1} = 2·suffix_j + bit(15 − j). It is zero at j = 0, which is + // why SLLC starts one iteration later than SLL. + let mut suffix = one; + for j in 0..16usize { + // eq(j, the four Z bits): four factors, and the first needs no product. + let mut indicator = one; + for k in 0..4 { + let factor = if (j >> k) & 1 == 1 { + v[16 + k] + } else { + z_not[k] + }; + indicator = if k == 0 { + factor + } else { + b.emul(indicator, factor) + }; + } + // (halfword << j) mod 2^16 keeps bits 0 ..= 15 − j at weight 2^(i + j). + let shifted = if j == 0 { + prefix[15] + } else { + b.emul(prefix[15 - j], pow2[j]) + }; + let term = b.emul(indicator, shifted); + sll = if j == 0 { term } else { b.eadd(sll, term) }; + + if j > 0 { + let carried = b.emul(indicator, suffix); + sllc = if j == 1 { + carried + } else { + b.eadd(sllc, carried) + }; + } + // The last advance would feed nothing. + if j < 15 { + suffix = if j == 0 { + v[15] + } else { + let doubled = b.eadd(suffix, suffix); + b.eadd(doubled, v[15 - j]) + }; + } + } + + [x, y, z, and, or, xor, msb8, msb16, is_zero, sll, sllc] +} + +// ============================================================================= +// The preprocessed columns NO closed form covers +// ============================================================================= + +/// ⛔ The largest `num_vars` [`emit_const_mle_at`] will serve — A CAP, NOT A +/// TUNING KNOB. +/// +/// That emitter is the host's own fold with the leaves interned, so it is +/// `O(2^n)`: a table whose unsettled preprocessed columns sat at twenty +/// variables would emit a million rows a column and nothing in the program would +/// say so. The two tables that DO sit at twenty variables are exactly the two +/// that never arrive here — BITWISE has [`emit_bitwise_preprocessed`], and +/// DECODE's five are settled out of band by the prepared opening, which is what +/// `settled_out_of_band` counts. What is left in a real continuation epoch is +/// KECCAK_RC at five variables and REGISTER at seven, so this sits above both +/// and far below twenty. +/// +/// A new preprocessed table above the cap is an EMIT-TIME REFUSAL. That refusal +/// is the whole point: the alternative is a program that silently becomes +/// unprovable, which is a failure nobody reads until a prove does not finish. +pub const MAX_CONST_MLE_VARS: usize = 12; + +/// INSTRUCTIONS the shared `eq(point, ·)` table costs over `num_vars` +/// variables. +/// +/// Level 0 is ONE `Sub`: its two entries are `1 − p_0` and the wire `p_0` +/// itself, because the parent is the interned one and multiplying by it emits +/// nothing. Every later level turns each of its `2^i` entries into two — one +/// `Mul` by `p_i` for the set bit and one `Sub` taking the complement from the +/// parent — so level `i` costs `2^{i + 1}` and the tail sums to `2^n⁺¹ − 4`. +/// +/// The interned `1` is not counted here: it is an `LFM_CONST` shared with every +/// other leg of the program, and this form counts operation rows. +pub const fn eq_table_rows(num_vars: usize) -> usize { + if num_vars == 0 { + // `eq` over no variables is the interned one, and nothing is emitted. + return 0; + } + 1 + (1usize << (num_vars + 1)) - 4 +} + +/// INSTRUCTIONS one constant column's fold against a built `eq` table costs. +/// +/// One row per NONZERO entry — a `Mul` for the first and a `MulAdd` for each +/// one after, which are the same row — and a zero entry emits nothing at all, +/// which is why a sparse column is cheap. An ALL-zero column emits no operation +/// and is the interned zero. +/// +/// ⚠ A coefficient of ONE is NOT special-cased, and the form says so rather +/// than the emitter hiding it: skipping the multiply for a unit coefficient +/// would save a row per such entry and cost a second shape the form has to +/// know about. The saving is a handful of rows on the two tables this serves. +pub fn const_column_rows(column: &[FE]) -> usize { + column.iter().filter(|value| **value != FE::zero()).count() +} + +/// INSTRUCTIONS [`emit_const_mle_at`] emits for one table's columns at one +/// point: the shared `eq` table, then each column's fold. +/// +/// The `eq` table is built ONCE per call and every column of the table folds +/// against it, which is the structure — a table's preprocessed columns are all +/// claimed at that table's own reduced point. +pub fn const_mle_rows(columns: &[&[FE]], num_vars: usize) -> usize { + eq_table_rows(num_vars) + + columns + .iter() + .map(|column| const_column_rows(column)) + .sum::() +} + +/// The `LFM_CONST` words [`emit_const_mle_at`] interns, deduplicated and BY +/// VALUE, so a caller can union them into the one pool its program has. +/// +/// Three kinds: `FEE::one()`, which level 0 subtracts from and which every +/// other leg shares; each DISTINCT nonzero coefficient, embedded into the +/// extension the accumulator lives in; and `FEE::zero()` when some column is +/// entirely zero, because that column IS the interned zero. +pub fn const_mle_constants(columns: &[&[FE]]) -> Vec { + let mut words: Vec = vec![ext_word(&FEE::one())]; + for column in columns { + if column.iter().all(|value| *value == FE::zero()) { + let zero = ext_word(&FEE::zero()); + if !words.contains(&zero) { + words.push(zero); + } + } + for value in column.iter() { + if *value == FE::zero() { + continue; + } + let word = ext_word(&value.to_extension::()); + if !words.contains(&word) { + words.push(word); + } + } + } + words +} + +/// ★ `Mle::evaluate_in` over EMIT-TIME CONSTANT columns at a WIRE point — +/// `check_preprocessed`'s remaining obligation, emitted. +/// +/// `check_preprocessed` (`multilinear_table.rs:959`) evaluates every +/// preprocessed column past `settled_out_of_band` at the reduced point and +/// compares it against the value that table's own argument settled on. The +/// columns are a property of the program the epoch is OF, so they are program +/// text here and only the point is a wire — which is what makes the fold a +/// straight line of interned coefficients rather than a pass over hinted data. +/// +/// `point` is in [`multilinear::mle::Mle::evaluate_in`]'s order: `point[0]` +/// binds the HIGH index bit, so the `eq` table is doubled with `point[0]` most +/// significant and entry `j` is the row whose index is `j`. Getting this +/// backwards is silent on a symmetric column and wrong on every other, which is +/// why it is a named mutation rather than a comment. +/// +/// Returns one value per column, in the order given. +/// +/// # Panics +/// +/// Above [`MAX_CONST_MLE_VARS`], and on a column whose length is not `2^n`. +/// Both are emit-time shape refusals, the `epoch_verify.rs:171-179` idiom: a +/// table of another shape has no way to be supplied to a program that emits a +/// fixed straight line. +pub fn emit_const_mle_at(b: &mut LfmBuilder, columns: &[&[FE]], point: &[Ext]) -> Vec { + let num_vars = point.len(); + assert!( + num_vars <= MAX_CONST_MLE_VARS, + "a preprocessed column at {num_vars} variables costs 2^{num_vars} rows a column; \ + it needs a closed form or a prepared opening, not this route" + ); + let height = 1usize << num_vars; + for column in columns { + assert_eq!( + column.len(), + height, + "a preprocessed column must have one entry per row of its table" + ); + } + + let one = b.ext_const(&FEE::one()); + // `eq[j] = Π_i (p_i if bit i of j else 1 − p_i)`, doubled with `point[0]` + // most significant. Level 0's parent is the interned one, so its set half is + // the wire itself and only its clear half is emitted. + let mut eq: Vec = vec![one]; + for (level, &p) in point.iter().enumerate() { + let mut next: Vec = Vec::with_capacity(eq.len() * 2); + for &parent in &eq { + let set = if level == 0 { p } else { b.emul(parent, p) }; + let clear = if level == 0 { + b.esub(one, p) + } else { + b.esub(parent, set) + }; + next.push(clear); + next.push(set); + } + eq = next; + } + debug_assert_eq!(eq.len(), height); + + columns + .iter() + .map(|column| { + let mut acc: Option = None; + for (row, value) in column.iter().enumerate() { + if *value == FE::zero() { + continue; + } + let coefficient = b.ext_const(&value.to_extension::()); + acc = Some(match acc { + None => b.emul(coefficient, eq[row]), + Some(running) => b.emul_add(coefficient, eq[row], running), + }); + } + acc.unwrap_or_else(|| b.ext_const(&FEE::zero())) + }) + .collect() +} + +// ============================================================================= +// The OFFSET ramp — the cross-epoch proof's page tables +// ============================================================================= + +/// ★ THE IDENTITY RAMP'S MULTILINEAR EXTENSION, IN CLOSED FORM. +/// +/// `page::offset_column()` is `(0..DEFAULT_PAGE_SIZE).map(FE::from)` — the row +/// index itself — and it is the preprocessed column every GLOBAL_MEMORY table +/// carries (`continuation.rs:241-251`: OFFSET alone on a private-input page, +/// OFFSET and INIT on any other). Its extension is the one multilinear +/// polynomial with a one-line answer: +/// +/// ```text +/// MLE(r) = Σ_i i · eq(r, i) = Σ_k 2^k · r_k +/// ``` +/// +/// because `Σ_i bit_k(i) · eq(r, i)` marginalises to `r_k`. So a `2^18` fold per +/// page becomes `num_vars − 1` rows, and the cross-epoch proof's thirty-five +/// page tables cost about six hundred rows between them instead of nine million. +/// +/// ⚠ THE BIT ORDER IS THE WHOLE CORRECTNESS OF THIS, and it is taken from +/// [`emit_bitwise_preprocessed`] rather than assumed: index bit `k` is carried +/// by `point[num_vars − 1 − k]`, so `point[0]` is the HIGH bit. Horner from +/// `point[0]` therefore accumulates the right weights, and the gate against +/// `Mle::evaluate_in` over the real column is what says so — a reversed +/// convention is a different number at every point but the symmetric ones. +pub fn emit_offset_ramp(b: &mut LfmBuilder, point: &[Ext]) -> Ext { + assert!( + !point.is_empty(), + "a ramp over 2^0 rows is the constant zero, and no page is that shape" + ); + let two = b.ext_const(&FEE::from(2u64)); + // `point[0]` carries 2^(n-1); each step doubles what is held and adds the + // next coordinate, so the last one lands at 2^0. + let mut acc = point[0]; + for coordinate in &point[1..] { + acc = b.emul_add(two, acc, *coordinate); + } + acc +} + +/// INSTRUCTIONS [`emit_offset_ramp`] emits, CONST-FREE. +/// +/// One `MulAdd` per coordinate after the first; the leading coordinate is a +/// wire the caller already holds, so it costs nothing. +pub const fn offset_ramp_rows(num_vars: usize) -> usize { + num_vars.saturating_sub(1) +} + +/// The ONE constant [`emit_offset_ramp`] interns. +/// +/// By value, like [`const_mle_constants`], because an epoch's or a global's +/// program has ONE pool and thirty-five ramps share this word between them — +/// a form that charged a constant per page would over-count by thirty-four. +pub fn offset_ramp_constants() -> Vec { + vec![ext_word(&FEE::from(2u64))] +} + +/// CPU's preprocessed TIMESTAMP (main's #1025), `4·(i + 1)` in every CPU table, +/// at `point`: `4·ramp + 4`, the identity ramp ([`emit_offset_ramp`]) scaled +/// and shifted — the extension of a constant column is the constant, so the +/// `+ 4` needs no fold. A table's own base `4·S_k` is a constant of its AIR's +/// bus values, never of the column, so one form serves every CPU table. +pub fn emit_timestamp_ramp(b: &mut LfmBuilder, point: &[Ext]) -> Ext { + let ramp = emit_offset_ramp(b, point); + let stride = b.ext_const(&FEE::from(crate::tables::cpu::TIMESTAMP_STRIDE)); + b.emul_add(stride, ramp, stride) +} + +/// INSTRUCTIONS [`emit_timestamp_ramp`] emits, CONST-FREE: the ramp's and one +/// `MulAdd`. +pub const fn timestamp_ramp_rows(num_vars: usize) -> usize { + offset_ramp_rows(num_vars) + 1 +} + +/// The constants [`emit_timestamp_ramp`] interns: the ramp's `2` and the stride. +pub fn timestamp_ramp_constants() -> Vec { + vec![ + ext_word(&FEE::from(2u64)), + ext_word(&FEE::from(crate::tables::cpu::TIMESTAMP_STRIDE)), + ] +} + +/// The host's closed form of [`emit_timestamp_ramp`], for the differential. +pub fn timestamp_ramp_at(point: &[FEE]) -> FEE { + let stride = FEE::from(crate::tables::cpu::TIMESTAMP_STRIDE); + &offset_ramp_at(point) * &stride + &stride +} + +/// The host's own closed form, for the differential. +/// +/// ⚠ A THIRD DERIVATION, deliberately: the test compares the EMITTED value +/// against `Mle::evaluate_in` over the real column (the fold being replaced) and +/// against this (the arithmetic being claimed). Comparing the emitter against +/// only this one would be two halves that share an author. +pub fn offset_ramp_at(point: &[FEE]) -> FEE { + assert!(!point.is_empty(), "a ramp needs at least one variable"); + let two = FEE::from(2u64); + let mut acc = point[0]; + for coordinate in &point[1..] { + acc = &acc * &two + coordinate; + } + acc +} + +// ============================================================================= +// The SPARSE route — the cross-epoch proof's page INIT columns +// ============================================================================= + +/// ⛔ WHY INIT IS NOT A PREPARED OPENING, WHICH IS THE FIRST THING TO READ HERE. +/// +/// A page's INIT column is its genesis bytes: a function of the program, not of +/// the row index, so [`emit_offset_ramp`]'s kind of closed form does not exist +/// for it. DECODE's answer to exactly that problem is a PREPARED OPENING +/// against a commitment pinned in the program text — and on the cross-epoch +/// path that answer is **unavailable**, for two reasons that are worth stating +/// where somebody would otherwise reach for it: +/// +/// 1. **The transcript.** `multi_prove` absorbs a prepared commitment's roots in +/// the roots block — `absorb_roots_and_challenge(transcript, committed.roots(), +/// &prepared_roots)` — and `prove_global` / `verify_global_bookends` both pass +/// `None`, so those roots are EMPTY in every cross-epoch proof that exists. A +/// program that absorbed an INIT root would absorb a root the honest proof +/// never absorbed, derive a different `z`, and stop executing at the first +/// table. The opening is not a machine-side addition; it is a change to the +/// cross-epoch prover, its verifier and its proof bytes. +/// 2. **The shape of `Prepared`.** It names ONE table and settles its columns +/// with `Claimed::Shared` at that table's single reduced point, because +/// DECODE's five columns share one. The cross-epoch INIT family spans one +/// page table per touched page, each with its OWN reduced point. +/// +/// ⇒ So the leg below is not a cheaper opening. It is the fold itself, emitted +/// only where the column is not zero — which is the whole cost, because a page's +/// genesis is zero everywhere past `init_values.len()` and an all-zero page +/// (stack, heap, BSS) emits nothing at all. +/// +/// ★ THE CLOSED FORM, which is the host's own definition restricted to the +/// support: +/// +/// ```text +/// MLE(r) = Σ_i v_i · eq(r, i) = Σ_{i : v_i ≠ 0} v_i · eq(r, i) +/// ``` +/// +/// `eq(r, i)` is a product of one factor per variable, each `r_k` or `1 − r_k`, +/// so a single entry costs `num_vars` rows and a zero entry costs none. Against +/// [`emit_const_mle_at`], which builds the whole `eq` TABLE first, this trades +/// `2^{n+1} − 3` rows of table for `n − 1` rows per surviving entry: at +/// `n = 18` the table alone is 524,285 rows and the sparse form is 18 per entry, +/// so the two cross at about thirty thousand entries and the host's own fold +/// (`2^n` = 262,144) at about fourteen thousand. +/// +/// ⚠ WHICH IS WHY THE CAP EXISTS AND IS A REFUSAL — see +/// [`MAX_SPARSE_INIT_ENTRIES`]. A dense column has no cheap route on this path at +/// all, and the honest outcome is a build that fails naming both numbers, never +/// a program nobody can prove. +/// +/// ⚠ THE BIT ORDER IS TAKEN FROM [`emit_const_mle_at`], NEVER ASSUMED. That +/// emitter doubles its table with `point[0]` MOST significant, so entry `j` is +/// the row whose index is `j` and index bit `k` (counting from the least +/// significant) is carried by `point[num_vars − 1 − k]`. The same convention is +/// spelled once, below, and a reversed one is a different value at every point +/// but the symmetric ones — which is a named test arm rather than a comment. +/// +/// Returns one value per column, in the order given. +/// +/// # Panics +/// +/// On a column whose length is not `2^num_vars`, and above [`MAX_SPARSE_INIT_ENTRIES`] +/// surviving entries. Both are emit-time shape refusals, the +/// `epoch_verify.rs:171-179` idiom. +pub fn emit_sparse_mle_at(b: &mut LfmBuilder, columns: &[&[FE]], point: &[Ext]) -> Vec { + let num_vars = point.len(); + let height = 1usize << num_vars; + for column in columns { + assert_eq!( + column.len(), + height, + "a preprocessed column must have one entry per row of its table" + ); + } + let entries = sparse_entries(columns); + assert!( + entries <= MAX_SPARSE_INIT_ENTRIES, + "these preprocessed columns carry {entries} nonzero entries, which this leg \ + emits {} rows for; the cap is {MAX_SPARSE_INIT_ENTRIES} entries. A column this \ + dense has no closed form here, and the prepared genesis opening is the route \ + it belongs on. ⛔ THE ROUTING RULE SHOULD ALREADY HAVE TAKEN IT: \ + `continuation::genesis_stack_plan` leaves a genesis page sparse only up to \ + {} entries at this height when the run stands at the block's bracket ({} \ + stacked variables; a run with fewer genesis pages is charged less and leaves \ + a slightly sparser page), so a column arriving here this dense is not a page \ + that needs a bigger cap. \ + It is that rule and this one having drifted apart, and neither can be fixed \ + without the other; \ + `whir_chain_tests::every_page_the_threshold_leaves_sparse_is_one_the_sparse_leg_will_emit` \ + is the assertion that they overlap", + entries * num_vars, + crate::continuation::densest_sparse_entries( + num_vars, + crate::continuation::BLOCK_STACK_VARS + ), + crate::continuation::BLOCK_STACK_VARS, + ); + + let one = b.ext_const(&FEE::one()); + // The complements, hoisted: every entry's `eq` reads from these, and a + // column with a hundred entries would otherwise pay for them a hundred + // times. Emitted for every variable rather than for the ones some entry + // happens to clear, so the count is a function of the SHAPE and not of the + // data — which is what lets `sparse_mle_rows` be evaluated before the + // program is built. + let complements: Vec = point.iter().map(|&p| b.esub(one, p)).collect(); + + columns + .iter() + .map(|column| { + let mut acc: Option = None; + for (row, value) in column.iter().enumerate() { + if *value == FE::zero() { + continue; + } + // `point[level]` carries index bit `num_vars − 1 − level`. + let mut eq: Option = None; + for (level, &p) in point.iter().enumerate() { + let factor = if (row >> (num_vars - 1 - level)) & 1 == 1 { + p + } else { + complements[level] + }; + eq = Some(match eq { + None => factor, + Some(running) => b.emul(running, factor), + }); + } + let eq = eq.expect("a point with at least one variable"); + // ⛔ THE INTERNED GENESIS BYTE, AND THE STATEMENT IT OWES. + // This constant is a byte of the page's genesis image, read + // from the ELF and frozen into the program text — the same + // standing as the interned DECODE root, and bound the same way: + // by the attestation id over the ELF digest, not by anything + // inside one program. So the obligation carried out of band is + // NOT "a root equals `compute_precomputed_commitment`'s + // inputs"; it is **the interned entries are the ELF's genesis + // bytes at those page bases**. The value gate against + // `Mle::evaluate_in` over the real column establishes it at + // fixture scale and the block instrument at block scale. + // Recorded as owed, never described as covered. + let coefficient = b.ext_const(&value.to_extension::()); + acc = Some(match acc { + None => b.emul(coefficient, eq), + Some(running) => b.emul_add(coefficient, eq, running), + }); + } + // ⛔ An all-zero column IS the interned zero, and it is the common + // case: every stack, heap and BSS page's genesis is zero to the last + // byte. The leg emits no operation for it at all. + acc.unwrap_or_else(|| b.ext_const(&FEE::zero())) + }) + .collect() +} + +/// ⛔ The most surviving entries [`emit_sparse_mle_at`] will serve — A CAP, NOT +/// A TUNING KNOB, and the sibling of [`MAX_CONST_MLE_VARS`]. +/// +/// The leg is `O(entries × num_vars)`, so a dense page column at eighteen +/// variables would emit about 4.7 M rows and nothing in the program would say +/// so. +/// +/// ⚠⚠ THIS NUMBER IS A PLACEHOLDER AND IT IS OWED A CENSUS — said here rather +/// than left for a reader to take it for a measurement. The quantity that +/// decides whether a real block fits is how many nonzero genesis bytes its +/// touched non-private pages carry between them, which a census reads +/// prove-free and card-free, off one execution of the guest. Until that reading +/// exists this bound is a round number sized to keep the family's contribution +/// near a million rows — the order the cross-epoch program's other legs cost +/// between them, inside the campaign's pre-registered 2–4 M band — and nothing +/// at fixture scale comes near it, so no gate exercises it. +/// +/// ⇒ WHEN THE CENSUS READS AND THE BLOCK DOES NOT FIT, the answer is NOT a +/// bigger cap. It is the fallback the cross-epoch proof format can still take +/// while nothing in flight depends on it: a host commitment over the page +/// family's INIT columns, absorbed in the roots block, opened through a +/// `Prepared` that names a TABLE LIST rather than one table — a change to the +/// cross-epoch prover, its verifier and its proof bytes. The refusal below +/// names it, so the choice is made against the number rather than against a +/// build that failed. +pub const MAX_SPARSE_INIT_ENTRIES: usize = 60_000; + +/// Surviving entries across a set of columns: what the leg's cost is linear in. +/// +/// Named rather than spelled inline at the three places that need it — the +/// emitter's cap, the row form and the constant pool — because a count that is +/// re-spelled is a count that can disagree with itself. +pub fn sparse_entries(columns: &[&[FE]]) -> usize { + columns.iter().map(|column| const_column_rows(column)).sum() +} + +/// INSTRUCTIONS [`emit_sparse_mle_at`] emits, CONST-FREE. +/// +/// The hoisted complements, one `Sub` per variable, then per surviving entry the +/// `num_vars − 1` products of its `eq` and the one `Mul`/`MulAdd` that weighs it +/// and accumulates it — `num_vars` rows an entry. +/// +/// ⚠ ZERO for an all-zero set, and that is the point of the route rather than an +/// edge case: the complements are still emitted (the shape is a shape), but no +/// entry is. A caller reading a zero here is reading a genuinely free leg. +pub fn sparse_mle_rows(columns: &[&[FE]], num_vars: usize) -> usize { + num_vars + sparse_entries(columns) * num_vars +} + +/// The `LFM_CONST` words [`emit_sparse_mle_at`] interns, deduplicated and BY +/// VALUE, so a caller can union them into the one pool its program has. +/// +/// The same three kinds [`const_mle_constants`] names — the shared `one` the +/// complements subtract from, each DISTINCT surviving coefficient, and +/// `FEE::zero()` for a column that is entirely zero and therefore IS that +/// constant. +/// +/// ⚠ A page's genesis is BYTES, so the distinct coefficients number at most 255 +/// however many entries survive; a form that charged one constant per entry +/// would over-count a real page by orders of magnitude. +pub fn sparse_mle_constants(columns: &[&[FE]]) -> Vec { + let mut words: Vec = vec![ext_word(&FEE::one())]; + for column in columns { + if column.iter().all(|value| *value == FE::zero()) { + let zero = ext_word(&FEE::zero()); + if !words.contains(&zero) { + words.push(zero); + } + } + for value in column.iter() { + if *value == FE::zero() { + continue; + } + let word = ext_word(&value.to_extension::()); + if !words.contains(&word) { + words.push(word); + } + } + } + words +} + +/// The host's own sparse form, for the differential. +/// +/// ⚠ A THIRD DERIVATION, deliberately, exactly as [`offset_ramp_at`] is: the +/// test compares the EMITTED value against `Mle::evaluate_in` over the real +/// column (the fold being replaced) AND against this (the arithmetic being +/// claimed). Comparing the emitter against only this one would be two halves +/// that share an author. +pub fn sparse_mle_at(column: &[FE], point: &[FEE]) -> FEE { + let num_vars = point.len(); + assert_eq!( + column.len(), + 1usize << num_vars, + "a column must have one entry per row of its table" + ); + let one = FEE::one(); + let mut acc = FEE::zero(); + for (row, value) in column.iter().enumerate() { + if *value == FE::zero() { + continue; + } + let mut eq = one; + for (level, p) in point.iter().enumerate() { + let factor = if (row >> (num_vars - 1 - level)) & 1 == 1 { + *p + } else { + &one - p + }; + eq = &eq * &factor; + } + acc = &acc + &(&eq * &value.to_extension::()); + } + acc +} diff --git a/prover/src/lfm/preprocessed_tests.rs b/prover/src/lfm/preprocessed_tests.rs new file mode 100644 index 000000000..3c3f63196 --- /dev/null +++ b/prover/src/lfm/preprocessed_tests.rs @@ -0,0 +1,899 @@ +//! Gates for the preprocessed leg. +//! +//! Two obligations, and they fail for different reasons. The closed form in +//! [`super::preprocessed::bitwise_preprocessed_rows`] is a claim about how much +//! the emitter costs; `emit_bitwise_preprocessed` is a claim about what it +//! computes. A leg can satisfy either alone: one that emits the predicted +//! number of rows and the wrong values, or the right values at a cost nobody +//! predicted. So the count is pinned against the emitter and the values are +//! pinned against the host's own fold over the real 2^20 columns — never +//! against each other. + +use multilinear::mle::Mle; + +use crate::tables::bitwise::{NUM_PRECOMPUTED_COLS, NUM_VARS, preprocessed_mle_at}; +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::preprocessed::{ + MAX_CONST_MLE_VARS, bitwise_preprocessed_rows, const_mle_constants, const_mle_rows, + emit_bitwise_preprocessed, emit_const_mle_at, eq_table_rows, +}; +use super::validator::validate; +use super::whir_chain_tests::const_rows; +use super::word::{ext_word, word_as_ext}; + +/// A fixed-seed xorshift, so a failure names one reproducible point. +fn sample_point(seed: u64) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..NUM_VARS) + .map(|_| FEE::new([next(), next(), next()])) + .collect() +} + +/// The leg alone: the point arrives by hint, the eleven columns are published. +/// +/// Hinting the point is a TEST convenience, not the production shape — there +/// the point is the reduced claim the sumcheck left, and a challenge must never +/// come from an arena. +fn leg_only_program() -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(NUM_VARS as u32); + let point: Vec<_> = (0..NUM_VARS) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let columns = emit_bitwise_preprocessed(&mut b, &point); + for c in columns { + b.public(c.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the preprocessed leg must be admissible"); + program +} + +/// The rows the leg costs, measured as the MARGINAL cost of emitting it. +/// +/// A whole-program count would carry the hints, the publishes and whatever the +/// builder interns; the difference between a program with the leg and the same +/// program without it is the leg and nothing else. +fn marginal_rows() -> usize { + let with = leg_only_program(); + let without = { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(NUM_VARS as u32); + let point: Vec<_> = (0..NUM_VARS) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + // Publish the inputs, so the hints and publishes cancel and only the + // leg's own rows survive the subtraction. + for p in point.iter().take(NUM_PRECOMPUTED_COLS) { + b.public(p.as_cell()); + } + compile(b.finish()) + }; + with.instrs.len() - without.instrs.len() +} + +/// ★ F1. The emitted count equals the closed form. +#[test] +fn the_bitwise_leg_emits_its_closed_form() { + let measured = marginal_rows(); + let predicted = bitwise_preprocessed_rows(); + println!( + "BITWISE preprocessed leg: {measured} rows emitted, {predicted} predicted; \ + the fold it replaces is {} steps", + 11u64 << NUM_VARS + ); + assert_eq!( + measured, predicted, + "the emitted row count must equal the closed form — if the emitter \ + changed, the closed form's named terms say which one" + ); +} + +/// ★ The leg computes what the host's fold computes, over the real columns. +/// +/// The right-hand side is `Mle::evaluate_in` over +/// `bitwise::preprocessed_columns()` — the 2^20-row fold this leg exists to +/// avoid — so the comparison is against the function being replaced rather than +/// against the closed form that replaced it. +#[test] +fn the_bitwise_leg_computes_what_the_host_fold_computes() { + let program = leg_only_program(); + let mles: Vec> = + crate::tables::bitwise::preprocessed_columns() + .into_iter() + .map(|values| Mle::new(values).expect("a power-of-two column")) + .collect(); + assert_eq!(mles.len(), NUM_PRECOMPUTED_COLS); + + for seed in [0x5eed_1001u64, 0x5eed_1002] { + let point = sample_point(seed); + let arenas = vec![point.iter().map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the preprocessed leg executes"); + let got: Vec = exec + .public_words + .iter() + .map(|(_, w)| word_as_ext(w).expect("a published extension value")) + .collect(); + assert_eq!(got.len(), NUM_PRECOMPUTED_COLS); + + // The closed form the emitter mirrors, and the fold it replaces. + let closed = preprocessed_mle_at(&point).expect("the point is NUM_VARS long"); + for (col, mle) in mles.iter().enumerate() { + let folded = mle + .evaluate_in::(&point) + .expect("the column has NUM_VARS variables"); + assert_eq!( + got[col], folded, + "column {col} at seed {seed:#x}: the EMITTED leg disagrees with \ + the host's 2^20 fold" + ); + assert_eq!( + got[col], closed[col], + "column {col} at seed {seed:#x}: the emitted leg disagrees with \ + the host closed form it mirrors" + ); + } + } +} + +// ============================================================================= +// The columns no closed form covers: KECCAK_RC and REGISTER +// ============================================================================= + +/// A fixed-seed point of `n` coordinates, so a failure names one reproducible +/// point. +fn sample_point_n(seed: u64, n: usize) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..n).map(|_| FEE::new([next(), next(), next()])).collect() +} + +/// The two real continuation-epoch tables this route serves, with their columns +/// as the AIR hands them over. +/// +/// REGISTER's `init` and `fini` are a register file, so they are chosen here +/// rather than zeroed: an all-zero column is the one shape whose fold cannot +/// distinguish a right answer from a wrong one. +fn const_mle_fixtures() -> Vec<(&'static str, Vec>)> { + let init: Vec = (0..crate::tables::register::NUM_REGISTER_ADDRESSES) + .map(|i| (i as u32).wrapping_mul(7).wrapping_add(3) % 251) + .collect(); + let fini: Vec = (0..crate::tables::register::NUM_REGISTER_ADDRESSES) + .map(|i| (i as u32).wrapping_mul(11).wrapping_add(5) % 241) + .collect(); + vec![ + ( + "KECCAK_RC", + crate::tables::keccak_rc::preprocessed_columns(), + ), + ( + "REGISTER", + crate::tables::register::preprocessed_columns(&init, &fini), + ), + ] +} + +/// The leg alone: the point arrives by hint, the columns' values are published. +fn const_mle_program(columns: &[Vec], num_vars: usize, leg: bool) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(num_vars as u32); + let point: Vec<_> = (0..num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + if !leg { + // The hints and the publishes cancel out of the subtraction, so what + // survives it is the leg and nothing else. + // + // ⚠ ONE publish per COLUMN, not per coordinate. A table can have more + // columns than variables (KECCAK_RC has nine of each five), and a + // `take(columns.len())` over a shorter point silently publishes fewer + // cells here than the leg does — which the subtraction then charges to + // the leg. That is exactly how this came in four rows long. + for index in 0..columns.len() { + b.public(point[index % num_vars].as_cell()); + } + return compile(b.finish()); + } + let borrowed: Vec<&[FE]> = columns.iter().map(Vec::as_slice).collect(); + for value in emit_const_mle_at(&mut b, &borrowed, &point) { + b.public(value.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the constant-MLE leg must be admissible"); + program +} + +/// The `LFM_CONST` words a program interns, in no particular order. +fn const_words_of(program: &LfmProgram) -> Vec { + program + .instrs + .iter() + .filter_map(|instr| match instr { + super::instr::Instr::Const { value, .. } => Some(*value), + _ => None, + }) + .collect() +} + +/// ★ THE VALUE GATE: the leg computes what `check_preprocessed`'s own fold +/// computes, over the real columns of the two real tables. +/// +/// The right-hand side is `Mle::evaluate_in`, which is the exact call +/// `check_preprocessed` (`multilinear_table.rs:1001`) makes — the host +/// function this leg replaces, not a model of it. +#[test] +fn the_constant_mle_leg_is_the_hosts_fold() { + for (name, columns) in const_mle_fixtures() { + let num_vars = columns[0].len().trailing_zeros() as usize; + let program = const_mle_program(&columns, num_vars, true); + for seed in [1u64, 0x5eed, 0xDEAD_BEEF] { + let point = sample_point_n(seed, num_vars); + let arena: Vec<_> = point.iter().map(ext_word).collect(); + let exec = execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("{name}: the leg must execute: {e:?}")); + assert_eq!( + exec.public_words.len(), + columns.len(), + "{name}: one published value per preprocessed column" + ); + for (index, column) in columns.iter().enumerate() { + let emitted = word_as_ext(&exec.public_words[index].1).expect("a published value"); + let host = Mle::new(column.clone()) + .expect("a preprocessed column is a power of two long") + .evaluate_in::(&point) + .expect("the point has the column's variables"); + assert_eq!( + emitted, host, + "{name} column {index} at seed {seed}: the machine must \ + compute the fold `check_preprocessed` computes" + ); + } + } + } +} + +/// ★ F1. The emitted count equals the closed form, and the pool is NAMED. +/// +/// The second half is the assertion instance 63 earned: the test ASKS which +/// constants the program interns that the form does not name, so the next +/// unnamed constant fails here instead of being absorbed into a fudge factor. +#[test] +fn the_constant_mle_leg_emits_its_closed_form() { + for (name, columns) in const_mle_fixtures() { + let num_vars = columns[0].len().trailing_zeros() as usize; + let with = const_mle_program(&columns, num_vars, true); + let without = const_mle_program(&columns, num_vars, false); + let consts_with = const_words_of(&with); + let consts_without = const_words_of(&without); + let measured = + (with.instrs.len() - without.instrs.len()) - (consts_with.len() - consts_without.len()); + let borrowed: Vec<&[FE]> = columns.iter().map(Vec::as_slice).collect(); + let predicted = const_mle_rows(&borrowed, num_vars); + println!( + "{name} constant-MLE leg at {num_vars} vars, {} columns: \ + {measured} rows emitted, {predicted} predicted \ + (eq table {}, folds {}); the host fold it replaces is {} steps", + columns.len(), + eq_table_rows(num_vars), + predicted - eq_table_rows(num_vars), + columns.len() << num_vars, + ); + assert_eq!( + measured, predicted, + "{name}: the emitted operation count must equal the closed form" + ); + + // `LfmWord` is four field elements and field elements are not ordered, + // so the two sets are compared by containment both ways rather than by + // sorting — which also reports WHICH word is unnamed. + let named = const_mle_constants(&borrowed); + let mut interned: Vec = Vec::new(); + for word in consts_with { + if !consts_without.contains(&word) && !interned.contains(&word) { + interned.push(word); + } + } + for word in &interned { + assert!( + named.contains(word), + "{name}: the program interns a constant the form does not name \ + ({word:?}) — that is a term, not a fudge factor" + ); + } + for word in &named { + assert!( + interned.contains(word), + "{name}: the form names a constant the program does not intern \ + ({word:?})" + ); + } + assert_eq!( + interned.len(), + named.len(), + "{name}: the interned pool and the named pool must be the same set" + ); + } +} + +/// ⛔ The cap refuses rather than emitting a program nobody can prove. +/// +/// The failure this asserts is the one that would otherwise arrive as a prove +/// that does not finish: a preprocessed table at twenty variables costs `2^20` +/// rows a column on this route, and BITWISE and DECODE are precisely the tables +/// that must not take it. +#[test] +#[should_panic(expected = "needs a closed form or a prepared opening")] +fn a_table_above_the_cap_is_refused() { + let num_vars = MAX_CONST_MLE_VARS + 1; + let column: Vec = vec![FE::from(1u64); 1usize << num_vars]; + const_mle_program(std::slice::from_ref(&column), num_vars, true); +} + +/// The cap's CONTROL: one variable below it emits and executes, so the refusal +/// above is the cap and not a leg that cannot serve any table. +#[test] +fn a_table_at_the_cap_still_emits() { + let num_vars = MAX_CONST_MLE_VARS; + let column: Vec = (0..1usize << num_vars) + .map(|i| FE::from(i as u64)) + .collect(); + let columns = vec![column.clone()]; + let program = const_mle_program(&columns, num_vars, true); + let point = sample_point_n(7, num_vars); + let arena: Vec<_> = point.iter().map(ext_word).collect(); + let exec = execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER) + .expect("a table at the cap must still execute"); + let emitted = word_as_ext(&exec.public_words[0].1).expect("a published value"); + let host = Mle::new(column) + .expect("a power of two") + .evaluate_in::(&point) + .expect("the point has the column's variables"); + assert_eq!( + emitted, host, + "the value at the cap is still the host's fold" + ); +} + +// ============================================================================= +// The OFFSET ramp — the cross-epoch proof's page tables +// ============================================================================= + +/// The ramp leg alone: the point arrives by hint, the one value is published. +fn ramp_only_program(num_vars: usize, leg: bool) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(num_vars as u32); + let point: Vec<_> = (0..num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + // Both arms publish ONE word, so the publish cancels out of the delta and + // what is left is the ramp. + let out = if leg { + super::preprocessed::emit_offset_ramp(&mut b, &point) + } else { + point[0] + }; + b.public(out.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the ramp leg must be admissible"); + program +} + +/// ★★ THE GATE: the emitted ramp equals the host's `2^18` FOLD over +/// `page::offset_column()` — the very work this closed form exists to avoid. +/// +/// Three derivations meet here and no two of them share an author: the EMITTED +/// leg, `Mle::evaluate_in` over the real column, and +/// `preprocessed::offset_ramp_at`. The fold is the one that matters; the third +/// is there so a disagreement says WHICH of the two claims broke. +#[test] +fn the_offset_ramp_computes_what_the_hosts_page_fold_computes() { + let column = crate::tables::page::offset_column(); + let num_vars = column.len().trailing_zeros() as usize; + assert_eq!( + 1usize << num_vars, + column.len(), + "the page size must be a power of two for the column to be an MLE" + ); + let mle = Mle::new(column).expect("a power-of-two column"); + let program = ramp_only_program(num_vars, true); + + for seed in [0x0ff5_e701u64, 0x0ff5_e702, 0x0ff5_e703] { + let point = sample_point_n(seed, num_vars); + let arenas = vec![point.iter().map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the ramp leg executes"); + assert_eq!(exec.public_words.len(), 1, "the leg publishes one value"); + let got = word_as_ext(&exec.public_words[0].1).expect("an extension value"); + + let folded = mle + .evaluate_in::(&point) + .expect("the column has num_vars variables"); + let closed = super::preprocessed::offset_ramp_at(&point); + + // ⛔ ANTI-VACUITY ON THE ANSWER, not on the inputs (instance 72). The + // trivial leg — the one a dropped Horner loop leaves behind — returns + // `point[0]`, and at a random point the true value is not that. + assert_ne!( + got, point[0], + "seed {seed:#x}: the ramp returned its leading coordinate, which is \ + what a leg that emitted nothing would return" + ); + assert_eq!( + got, folded, + "seed {seed:#x}: the EMITTED ramp disagrees with the host's 2^{num_vars} fold" + ); + assert_eq!( + got, closed, + "seed {seed:#x}: the emitted ramp disagrees with the host closed form it mirrors" + ); + } +} + +/// ⛔ THE BIT ORDER IS LOAD-BEARING, AND THIS IS WHAT SAYS SO. +/// +/// The convention — index bit `k` carried by `point[num_vars − 1 − k]` — is +/// taken from `emit_bitwise_preprocessed`, not derived here. A test that only +/// compared the emitter against `offset_ramp_at` would agree with itself under +/// EITHER convention, because both are written by the same hand. So the +/// reversed point is evaluated against the REAL fold and must differ: that is +/// the observation which would fail if the convention were the other one. +#[test] +fn the_offset_ramp_is_wrong_under_the_reversed_bit_order() { + let column = crate::tables::page::offset_column(); + let num_vars = column.len().trailing_zeros() as usize; + let mle = Mle::new(column).expect("a power-of-two column"); + let point = sample_point_n(0x0ff5_e7a0, num_vars); + let reversed: Vec = point.iter().rev().cloned().collect(); + + let folded = mle + .evaluate_in::(&point) + .expect("the column has num_vars variables"); + assert_eq!( + super::preprocessed::offset_ramp_at(&point), + folded, + "the convention this crate uses must reproduce the host's fold" + ); + assert_ne!( + super::preprocessed::offset_ramp_at(&reversed), + folded, + "a reversed point must give a DIFFERENT value, or this test cannot see \ + the convention at all" + ); +} + +/// ★ F1: `num_vars − 1` rows and ONE interned constant, whatever the page size. +#[test] +fn the_offset_ramp_emits_its_closed_form() { + for num_vars in [2usize, 5, 18] { + let with = ramp_only_program(num_vars, true); + let without = ramp_only_program(num_vars, false); + let with_consts = const_rows(&with); + let without_consts = const_rows(&without); + let measured = (with.instrs.len() - without.instrs.len()) - (with_consts - without_consts); + let predicted = super::preprocessed::offset_ramp_rows(num_vars); + println!( + "offset ramp at {num_vars} vars: {measured} rows emitted, {predicted} predicted, \ + {} constants interned", + with_consts - without_consts + ); + assert_eq!( + measured, predicted, + "{num_vars} vars: the emitted operation count must equal the closed form" + ); + + // The pool, by value and both ways. + let named = super::preprocessed::offset_ramp_constants(); + let interned: Vec = with + .instrs + .iter() + .filter_map(|instr| match instr { + super::instr::Instr::Const { value, .. } => Some(*value), + _ => None, + }) + .filter(|word| { + !without.instrs.iter().any(|instr| { + matches!(instr, super::instr::Instr::Const { value, .. } if value == word) + }) + }) + .collect(); + for word in &interned { + assert!( + named.contains(word), + "{num_vars} vars: the ramp interns {word:?}, which the form does not name" + ); + } + assert_eq!( + interned.len(), + named.len(), + "{num_vars} vars: the interned pool and the named pool must be the same set" + ); + } +} + +/// ★ THE SIZE OF THE PRIZE, asserted rather than asserted-in-a-comment: the +/// closed form is smaller than the fold it replaces by more than four orders of +/// magnitude at the real page size. +#[test] +fn the_offset_ramp_is_cheaper_than_the_fold_it_replaces() { + let rows = crate::tables::page::DEFAULT_PAGE_SIZE; + let num_vars = rows.trailing_zeros() as usize; + let closed = super::preprocessed::offset_ramp_rows(num_vars); + println!( + "OFFSET at {rows} rows: {closed} rows closed-form vs {rows} for a fold \ + ({}x), and the cross-epoch proof carries one per page", + rows / closed.max(1) + ); + assert!( + closed * 10_000 < rows, + "the ramp costs {closed} rows against a {rows}-row fold, which is not the \ + saving this leg exists for" + ); +} + +// ============================================================================= +// The TIMESTAMP ramp — CPU's preprocessed clock, `4·(row + 1)` +// ============================================================================= + +/// The timestamp leg alone, like [`ramp_only_program`]: the control arm +/// publishes the leading coordinate, so the publish cancels out of the delta. +fn timestamp_only_program(num_vars: usize, leg: bool) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(num_vars as u32); + let point: Vec<_> = (0..num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let out = if leg { + super::preprocessed::emit_timestamp_ramp(&mut b, &point) + } else { + point[0] + }; + b.public(out.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the timestamp leg must be admissible"); + program +} + +/// ★★ The emitted clock equals the host's fold over `cpu::timestamp_column` — +/// the column the univariate verifier commits and the multilinear one opens — +/// at every height a CPU table takes, smallest to a real chunk's. +#[test] +fn the_timestamp_ramp_computes_what_the_hosts_cpu_fold_computes() { + for num_vars in [2usize, 5, 12] { + let column = crate::tables::cpu::timestamp_column(1 << num_vars); + let mle = Mle::new(column).expect("a power-of-two column"); + let program = timestamp_only_program(num_vars, true); + for seed in [0x7135_0001u64, 0x7135_0002, 0x7135_0003] { + let point = sample_point_n(seed ^ num_vars as u64, num_vars); + let arenas = vec![point.iter().map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the timestamp leg executes"); + assert_eq!(exec.public_words.len(), 1, "the leg publishes one value"); + let got = word_as_ext(&exec.public_words[0].1).expect("an extension value"); + + let folded = mle + .evaluate_in::(&point) + .expect("the column has num_vars variables"); + let closed = super::preprocessed::timestamp_ramp_at(&point); + // The bare identity ramp is what a leg that dropped the scale and + // shift would return; at a random point the clock is not that. + assert_ne!( + got, + super::preprocessed::offset_ramp_at(&point), + "{num_vars} vars, seed {seed:#x}: the clock is the bare row ramp" + ); + assert_eq!( + got, folded, + "{num_vars} vars, seed {seed:#x}: the EMITTED clock disagrees with the host's fold" + ); + assert_eq!( + got, closed, + "{num_vars} vars, seed {seed:#x}: the emitted clock disagrees with its closed form" + ); + } + } +} + +/// The clock's bit order is the page ramp's, and a reversed point reads a +/// different value off the real column. +#[test] +fn the_timestamp_ramp_is_wrong_under_the_reversed_bit_order() { + let num_vars = 10; + let mle = Mle::new(crate::tables::cpu::timestamp_column(1 << num_vars)) + .expect("a power-of-two column"); + let point = sample_point_n(0x7135_00a0, num_vars); + let reversed: Vec = point.iter().rev().cloned().collect(); + let folded = mle + .evaluate_in::(&point) + .expect("the column has num_vars variables"); + assert_eq!(super::preprocessed::timestamp_ramp_at(&point), folded); + assert_ne!( + super::preprocessed::timestamp_ramp_at(&reversed), + folded, + "a reversed point must give a different value" + ); +} + +/// ⛔ The check is load-bearing: a CPU column with ONE timestamp moved (a +/// prover's free clock at one row) folds to another value, so the emitted +/// closed form refuses its opening, at every row the move can land on. +#[test] +fn a_cpu_column_with_one_timestamp_moved_is_not_the_clock() { + let num_vars = 6; + let program = timestamp_only_program(num_vars, true); + let point = sample_point_n(0x7135_00b0, num_vars); + let arenas = vec![point.iter().map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the timestamp leg executes"); + let emitted = word_as_ext(&exec.public_words[0].1).expect("an extension value"); + for row in [0, 1, 37, (1 << num_vars) - 1] { + let mut column = crate::tables::cpu::timestamp_column(1 << num_vars); + column[row] += FE::from(crate::tables::cpu::TIMESTAMP_STRIDE); + let moved = Mle::new(column) + .expect("a power-of-two column") + .evaluate_in::(&point) + .expect("the column has num_vars variables"); + assert_ne!( + emitted, moved, + "row {row}: a moved timestamp opens to the clock's value" + ); + } +} + +/// The leg costs the ramp's rows plus one, and interns exactly the constants +/// its form names. +#[test] +fn the_timestamp_ramp_emits_its_closed_form() { + for num_vars in [2usize, 5, 18] { + let with = timestamp_only_program(num_vars, true); + let without = timestamp_only_program(num_vars, false); + let with_consts = const_rows(&with); + let without_consts = const_rows(&without); + let measured = (with.instrs.len() - without.instrs.len()) - (with_consts - without_consts); + assert_eq!( + measured, + super::preprocessed::timestamp_ramp_rows(num_vars), + "{num_vars} vars: the emitted operation count must equal the closed form" + ); + + let named = super::preprocessed::timestamp_ramp_constants(); + let interned: Vec = with + .instrs + .iter() + .filter_map(|instr| match instr { + super::instr::Instr::Const { value, .. } => Some(*value), + _ => None, + }) + .filter(|word| { + !without.instrs.iter().any(|instr| { + matches!(instr, super::instr::Instr::Const { value, .. } if value == word) + }) + }) + .collect(); + for word in &interned { + assert!( + named.contains(word), + "{num_vars} vars: the clock interns {word:?}, which the form does not name" + ); + } + assert_eq!( + interned.len(), + named.len(), + "{num_vars} vars: the interned pool and the named pool must be the same set" + ); + } +} + +// ============================================================================= +// The SPARSE leg — the cross-epoch proof's page INIT columns +// ============================================================================= + +/// The sparse leg alone, over one column, publishing its one value. +/// +/// The control arm publishes the interned ZERO — which is exactly what the +/// emitter returns for a column with no surviving entry, and therefore exactly +/// what a leg whose coefficient loop emitted nothing would return. Both arms +/// publish ONE word, so the publish cancels out of the delta and what is left is +/// the leg. +fn sparse_only_program(column: &[FE], num_vars: usize, leg: bool) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(num_vars as u32); + let point: Vec<_> = (0..num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let out = if leg { + super::preprocessed::emit_sparse_mle_at(&mut b, &[column], &point)[0] + } else { + b.ext_const(&FEE::zero()) + }; + b.public(out.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the sparse leg must be admissible"); + program +} + +/// A real page's genesis column: the bytes an ELF data page loads, zero to the +/// end of the page. +/// +/// ★ THE REAL OBJECT, not a hand-built vector. `page::preprocessed_columns` is +/// what `global_memory_air` hands the verifier, column 1 is INIT, and the whole +/// claim of the sparse route is about THAT column's shape — mostly zero, with a +/// short nonzero prefix. A synthetic column could be sparse for reasons the real +/// one is not. +fn real_genesis_column(bytes: &[u8]) -> Vec { + let config = crate::tables::page::PageConfig::with_data(0, bytes.to_vec()); + let mut columns = crate::tables::page::preprocessed_columns(&config); + assert_eq!( + columns.len(), + 2, + "a non-private page carries OFFSET and INIT" + ); + columns.remove(1) +} + +/// ★★ THE SPARSE LEG AGAINST THE FOLD IT REPLACES, AND AGAINST A THIRD +/// DERIVATION — the ramp's own pattern, over a real page's INIT column. +/// +/// Three derivations and no two share an author: `Mle::evaluate_in` over the +/// real `2^18` column (the host fold this leg exists to avoid), the host's own +/// [`super::preprocessed::sparse_mle_at`] (the arithmetic being claimed), and +/// the EMITTED program's published value. +#[test] +fn the_sparse_leg_computes_what_the_hosts_genesis_fold_computes() { + // Eight nonzero bytes with no repeats, so the distinct-coefficient pool is + // eight and a dropped entry moves the answer. + let bytes = [0xF0u8, 0xDE, 0xBC, 0x9A, 0x78, 0x56, 0x34, 0x12]; + let column = real_genesis_column(&bytes); + let num_vars = column.len().trailing_zeros() as usize; + assert_eq!( + 1usize << num_vars, + column.len(), + "a page is a power of two tall" + ); + let entries = super::preprocessed::sparse_entries(&[column.as_slice()]); + assert_eq!( + entries, + bytes.len(), + "the support this arm is written against: every genesis byte is nonzero \ + and the rest of the page is zero" + ); + + let mle = Mle::new(column.clone()).expect("a power-of-two column"); + let program = sparse_only_program(&column, num_vars, true); + + for seed in [0x5a17_0001u64, 0x5a17_0002, 0x5a17_0003] { + let point = sample_point_n(seed, num_vars); + let arenas = vec![point.iter().map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the sparse leg executes"); + assert_eq!(exec.public_words.len(), 1, "the leg publishes one value"); + let got = word_as_ext(&exec.public_words[0].1).expect("an extension value"); + + let folded = mle + .evaluate_in::(&point) + .expect("the column has num_vars variables"); + let closed = super::preprocessed::sparse_mle_at(&column, &point); + + // ⛔ ANTI-VACUITY ON THE ANSWER (instance 72). A leg whose coefficient + // loop emitted nothing returns the interned ZERO, and at a random point + // over a column with support the true value is not zero. + assert_ne!( + got, + FEE::zero(), + "seed {seed:#x}: the sparse leg returned zero, which is what a leg that \ + emitted nothing over a column with {entries} surviving entries would return" + ); + assert_eq!( + got, folded, + "seed {seed:#x}: the EMITTED sparse leg disagrees with the host's \ + 2^{num_vars} fold over the same column" + ); + assert_eq!( + got, closed, + "seed {seed:#x}: the emitted leg disagrees with the host closed form it mirrors" + ); + } +} + +/// ⛔ THE BIT ORDER IS OBSERVABLE, so the convention is pinned rather than +/// agreed with itself. +/// +/// The emitter's order is taken from `emit_const_mle_at` — `point[0]` binds the +/// HIGH index bit. Reversing the point must give a DIFFERENT value against the +/// real fold; if it did not, no arm of this suite could see the convention +/// reversed and the comment claiming it would be the only thing holding it. +#[test] +fn the_sparse_legs_bit_order_is_observable() { + let column = real_genesis_column(&[0x01u8, 0x02, 0x03, 0x04]); + let num_vars = column.len().trailing_zeros() as usize; + let mle = Mle::new(column.clone()).expect("a power-of-two column"); + let program = sparse_only_program(&column, num_vars, true); + + let point = sample_point_n(0x5a17_0b17, num_vars); + let reversed: Vec = point.iter().rev().cloned().collect(); + + let run = |p: &[FEE]| { + let arenas = vec![p.iter().map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the sparse leg executes"); + word_as_ext(&exec.public_words[0].1).expect("an extension value") + }; + + let forward = run(&point); + let backward = run(&reversed); + assert_eq!( + forward, + mle.evaluate_in::(&point) + .expect("the fold"), + "the emitted leg is the fold at the point as given" + ); + assert_ne!( + forward, backward, + "the two bit orders agree at this point, so a reversed convention would be \ + invisible here — pick another point" + ); + assert_ne!( + backward, + mle.evaluate_in::(&point) + .expect("the fold"), + "the REVERSED order must disagree with the fold, which is what makes the \ + convention a checkable fact rather than a shared assumption" + ); +} + +/// ★ THE ROW FORM, against the emitter, as a DELTA between the two arms. +/// +/// `sparse_mle_rows` is a claim about cost and the value gate above is a claim +/// about arithmetic; a leg can satisfy either alone. The delta cancels the +/// hints, the publish and the compiler's own overhead, so what is left is the +/// leg — and the constant pool is asserted BOTH ways, by count and by value. +#[test] +fn the_sparse_leg_emits_its_row_form() { + for bytes in [vec![0xABu8, 0xCD, 0xEF], Vec::new(), vec![7u8; 5]] { + let column = real_genesis_column(&bytes); + let num_vars = column.len().trailing_zeros() as usize; + let leg = sparse_only_program(&column, num_vars, true); + let control = sparse_only_program(&column, num_vars, false); + + let emitted = + (leg.instrs.len() - const_rows(&leg)) - (control.instrs.len() - const_rows(&control)); + let predicted = super::preprocessed::sparse_mle_rows(&[column.as_slice()], num_vars); + let constants = super::preprocessed::sparse_mle_constants(&[column.as_slice()]); + println!( + "sparse leg over {} genesis bytes at {num_vars} vars: {emitted} rows emitted, \ + {predicted} predicted, {} constants named, {} interned", + bytes.len(), + constants.len(), + const_rows(&leg), + ); + assert_eq!( + emitted, + predicted, + "the sparse leg's row form missed the emitter over {} genesis bytes", + bytes.len() + ); + // ⚠ The control interns the ZERO it publishes, and the leg interns it + // too when the column is empty — so the pools are compared by the form's + // own list, which names that zero exactly when the emitter does. + assert_eq!( + const_rows(&leg), + constants.len(), + "the pool the form names is the pool the emitter interns" + ); + } +} diff --git a/prover/src/lfm/program_budget.rs b/prover/src/lfm/program_budget.rs new file mode 100644 index 000000000..cab275d37 --- /dev/null +++ b/prover/src/lfm/program_budget.rs @@ -0,0 +1,1043 @@ +//! The tree programs' host budget (`LAMBDA_VM_TREE_PROGRAM_BUDGET`) and the +//! ordered streaming emitter that produces the programs it defers. +//! +//! A block tree's program — a leaf's or a node's [`super::compiler::LfmProgram`] +//! — is emitted on the host, waits until a prover takes it, and is dropped when +//! its proof returns. Emitted all ahead, a p90 block's 107 leaf programs (45.76 +//! GiB) were held from phase B to level 0: they bound phase B and the +//! recursion's floor at every host size (BIG 659). Emitted just in time, the +//! same block's phase B fell 78.4 → 40.8 GiB and its recursion 93.7 → 63.3 GiB +//! (BIG 662, I-MEMFIT §1.7b). The budget decides, program by program, which +//! may be emitted ahead and which wait: +//! +//! - **Room.** `auto`, the default, admits a program while the process's +//! resident bytes now (`VmRSS`) plus the program's bytes, and the bytes of the +//! programs admitted but not yet emitted, stay under the host target the +//! caller gives: the pipeline's memory target (`LAMBDA_VM_BLOCK_MEMORY_TARGET_GIB`, +//! else the cgroup's limit or `MemTotal` less 10 GiB). One knob sizes both, +//! and the allocator's retained pages count because `VmRSS` counts them. A +//! roomy host never reaches the target, so every program is emitted where it +//! was before. `off` never refuses: the control. `` caps the bytes of +//! the admitted programs that are still alive, whatever the host. +//! - **The next [`AHEAD`] programs the provers will take are always +//! admitted**, room or not: a program comes in once fewer than `AHEAD` +//! programs below it in prove order are still untaken. The program the next +//! prover needs never waits for room, so a tight budget cannot deadlock the +//! tree; and with room gone the emission still runs `AHEAD` programs ahead +//! of the provers (one at a time would make every emission wait for the +//! take before it: ≈ 6 s a node). The programs alive stay under the room +//! plus `AHEAD` more than the provers in flight hold. +//! - **No program is admitted while a lower one waits**, so programs emitted +//! early (a node's, during level 0) never take the room a leaf needs. +//! +//! Prove order is the leaves 0..n, then level 1's nodes, level 2's, … the top: +//! the order the provers take them (`in_index_order` within a level, the levels +//! one after another). The budget only moves when a program is emitted; a +//! program's bytes are a function of the plan and its children's derived +//! artifacts, never of the time or the thread that emitted it, so the +//! programs, their ids and the top are the same under any setting. +//! +//! `VmRSS` is read from `/proc/self/status` (the kernel's RSS counters, no +//! page walk; one open and read, tens of µs): once per admission decision and +//! once per 200 ms while a program waits, never on a per-op path. Without the +//! file (not Linux) `auto` has no reading and admits by the other two rules +//! only, which is what the line reports. +//! +//! What `auto` reads of the host is [`READING_ENV`]'s choice. `live`, the +//! default ([`READING_DEFAULT`]), takes off what a purge could return +//! ([`live_bytes`]): after phase A frees its tables the resident set is mostly +//! pages the allocator holds and a new program is written into, and counting +//! them made the room shortest on the hosts with the most of it. A large +//! block on a 118 GiB host stated with 66.9 GiB resident, 64.2 GiB of it +//! purgeable and 10.8 GiB live, and 13 of its 26 leaf programs came in beside +//! phase B; the same block on a 64 GiB host, purged at that point, got 24 in. +//! Under `live` a host is short only of what a purge cannot give back, so the +//! budget is a ceiling there and no longer a schedule. Where nothing is +//! purgeable the two readings are one. `resident` is `VmRSS` as it is, the +//! reading before. Against it, on one card (a 5090 held at 400 W, a 9950X) at +//! a 108.28 GiB host target: the large block took all 26 leaf programs beside +//! phase B where `resident` took 11 to 13, and proved 3.3 s faster (the tree +//! 3.7 s), four runs each; the median, all 13 beside phase B either way, +//! +0.2 s. At a 54 GiB target the large block took 19 at the statement under +//! either reading and `live` proved 2.1 s faster, its programs waiting 24 s for +//! room against 55 s (one run each). +//! +//! ⛔ [`ProgramBudget::acquire`] parks its thread on a condvar. Call it only +//! from a plain thread, never from a rayon worker: a worker that parked there +//! could hold the pool's jobs a prover waits for (the derive gate's hazard, +//! [`super::derive_gate`]). [`ProgramBudget::try_acquire`] never waits. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ops::Range; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +const GIB: u64 = 1 << 30; + +/// The programs admitted past the room: the next this many to be taken. Four +/// is the streaming emitter's thread count, and the window BIG 662 measured +/// (a p90 recursion of 63.3 GiB, emitted by chunks of four). +pub const AHEAD: usize = 4; + +/// The budget's knob. +pub const PROGRAM_BUDGET_ENV: &str = "LAMBDA_VM_TREE_PROGRAM_BUDGET"; + +/// What [`PROGRAM_BUDGET_ENV`] says. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BudgetSetting { + /// Unset, empty or `auto` (the default): room is the host, `VmRSS` + /// against the target the caller gives ([`Room::Host`]). + Auto, + /// `off`: never refuse ([`Room::Unbounded`]). + Off, + /// ``: the admitted programs alive may hold this many bytes + /// ([`Room::Bytes`]). + Bytes(u64), +} + +/// [`PROGRAM_BUDGET_ENV`], parsed. +pub fn setting_from_env() -> Result { + parse_setting(std::env::var(PROGRAM_BUDGET_ENV).ok().as_deref()) +} + +/// A value of [`PROGRAM_BUDGET_ENV`]; nonsense is an error, not a default. +pub fn parse_setting(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "auto") => Ok(BudgetSetting::Auto), + Some("off") => Ok(BudgetSetting::Off), + Some(v) => v + .parse::() + .ok() + .filter(|g| g.is_finite() && *g >= 0.0) + .map(|g| BudgetSetting::Bytes((g * GIB as f64) as u64)) + .ok_or_else(|| { + format!("{PROGRAM_BUDGET_ENV} must be auto, off or a size in GiB, got `{v}`") + }), + } +} + +/// `auto`'s room line as a share of the host target: programs come in by room +/// only while the host stays under this share, so the room a purge returns +/// (the allocator's watermark, at [`crate::alloc_purge::WATERMARK_SHARE`] of +/// the target) is not spent again on programs held ahead. Above the line, +/// admissions run by the [`AHEAD`] rule alone. At the p90 block on a 48 GiB +/// emulated host the budget, its room line at the target, turned what the +/// watermark returned into held programs: level 1's live heap 42–46 GiB +/// against 30–32 without the watermark (RYZEN 050, I-MEMFIT §6.5). +pub const ROOM_SHARE: f64 = 0.75; + +/// [`ROOM_SHARE`]'s knob: a share in (0, 1]; `1.0` puts the line at the +/// target (the budget before the share). +pub const ROOM_SHARE_ENV: &str = "LAMBDA_VM_TREE_PROGRAM_BUDGET_SHARE"; + +/// [`ROOM_SHARE_ENV`], parsed. +pub fn share_from_env() -> Result { + parse_share(std::env::var(ROOM_SHARE_ENV).ok().as_deref()) +} + +/// A value of [`ROOM_SHARE_ENV`]: unset or empty is [`ROOM_SHARE`]; nonsense +/// is an error. +pub fn parse_share(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("") => Ok(ROOM_SHARE), + Some(v) => v + .parse::() + .ok() + .filter(|s| s.is_finite() && *s > 0.0 && *s <= 1.0) + .ok_or_else(|| format!("{ROOM_SHARE_ENV} must be a share in (0, 1], got `{v}`")), + } +} + +/// What `auto` reads of the host: [`READING_ENV`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Reading { + /// `resident`: the process's resident bytes ([`resident_bytes`]), the + /// reading before. + Resident, + /// Unset, empty or `live` (the default): the resident bytes less what a + /// purge could return ([`live_bytes`]). + Live, +} + +/// [`READING_ENV`] unset or empty: the live reading. `resident` is the way +/// back. +pub const READING_DEFAULT: Reading = Reading::Live; + +/// [`Reading`]'s knob. +pub const READING_ENV: &str = "LAMBDA_VM_TREE_PROGRAM_BUDGET_READING"; + +/// [`READING_ENV`], parsed. +pub fn reading_from_env() -> Result { + parse_reading(std::env::var(READING_ENV).ok().as_deref()) +} + +/// A value of [`READING_ENV`]: unset or empty is [`READING_DEFAULT`]; +/// nonsense is an error. +pub fn parse_reading(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("") => Ok(READING_DEFAULT), + Some("resident") => Ok(Reading::Resident), + Some("live") => Ok(Reading::Live), + Some(v) => Err(format!("{READING_ENV} must be resident or live, got `{v}`")), + } +} + +/// What counts as room. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Room { + /// Everything fits. + Unbounded, + /// The admitted programs alive hold at most this many bytes. + Bytes(u64), + /// The host's reading ([`Reading`]: the process's resident bytes, or those + /// a purge could not return), plus the programs admitted and not yet + /// emitted, plus the new one, stay at or under `line`: the host target + /// times the room share ([`ROOM_SHARE`]). + Host { line: u64 }, +} + +impl Room { + /// The room a setting gives against the host `target`, `auto`'s line at + /// `share` of it. + pub fn of(setting: BudgetSetting, target: u64, share: f64) -> Self { + match setting { + BudgetSetting::Auto => Room::Host { + line: (target as f64 * share) as u64, + }, + BudgetSetting::Off => Room::Unbounded, + BudgetSetting::Bytes(bytes) => Room::Bytes(bytes), + } + } +} + +/// This process's resident bytes (`VmRSS`), on Linux. +pub fn resident_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let kib: u64 = status + .lines() + .find(|l| l.starts_with("VmRSS:"))? + .split_whitespace() + .nth(1)? + .parse() + .ok()?; + Some(kib << 10) +} + +/// The part of `rss` a purge could not return: `rss` less the allocator's +/// purgeable bytes ([`crate::alloc_purge::purgeable`]: its resident bytes less +/// its active ones), never under its active bytes and never over `rss`. +/// +/// The floor is there because the allocator's resident figure is an upper +/// bound (an extent counts before it is touched) and so is its purgeable one: +/// at the statement of a large block the difference alone read 2.7 GiB where +/// the live heap was 10.8. What the process holds outside the allocator is in +/// `rss` and in neither of the allocator's figures; where the floor decides, +/// the reading is short by it (about 5 GiB at that point), which the room +/// share's quarter of the host target covers. Where nothing is purgeable this +/// is `rss`. Without the allocator's figures (`None`) it is `rss` too: the +/// reading before. +fn live_of(rss: u64, stats: Option<&crate::alloc_purge::AllocStats>) -> u64 { + match stats { + Some(stats) => rss + .saturating_sub(crate::alloc_purge::purgeable(stats)) + .max(stats.active as u64) + .min(rss), + None => rss, + } +} + +/// This process's resident bytes less what a purge could return +/// ([`live_of`]), on Linux. A program written into pages the allocator held +/// moves them from purgeable to live, so this reading counts an emitted +/// program where `VmRSS` does not move. The allocator's figures are its +/// statistics behind one epoch turn, read where `VmRSS` is: per admission +/// decision and per 200 ms of a wait. +pub fn live_bytes() -> Option { + let rss = resident_bytes()?; + Some(live_of(rss, crate::alloc_purge::stats().as_ref())) +} + +/// How a program got in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Admission { + Room, + Ahead, +} + +#[derive(Default)] +struct State { + /// The bytes of the admitted programs still alive. + held: u64, + /// Of them, the estimates of the programs admitted and not yet emitted + /// (the host does not hold them yet). + unemitted: u64, + /// The admitted programs alive: order → taken by a prover. + alive: BTreeMap, + /// Every order ever admitted, and every order ever taken by a prover. + admitted: BTreeSet, + claimed: BTreeSet, + /// The orders waiting in [`ProgramBudget::acquire`]. + waiting: BTreeMap, + failed: bool, + // What the summary reports. + high_water: u64, + by_room: usize, + by_ahead: usize, + refused: usize, + waited_secs: f64, +} + +impl State { + fn admit(&mut self, order: usize, bytes: u64, how: Admission) { + self.held += bytes; + self.unemitted += bytes; + self.high_water = self.high_water.max(self.held); + self.alive.insert(order, false); + self.admitted.insert(order); + match how { + Admission::Room => self.by_room += 1, + Admission::Ahead => self.by_ahead += 1, + } + } +} + +/// The budget of one tree's programs. Every program is admitted once, by its +/// prove order, before it is emitted; its [`Permit`] travels with it and +/// gives its bytes back when it is dropped. +pub struct ProgramBudget { + room: Room, + ahead: usize, + reading: Box Option + Send + Sync>, + /// What `reading` is, for the line ([`Self::describe`]). + reads: Reading, + state: Mutex, + changed: Condvar, +} + +/// A program's admission: claimed when a prover takes it, released when +/// dropped (with the program). +pub struct Permit { + budget: Arc, + order: usize, + bytes: u64, + emitted: bool, +} + +impl ProgramBudget { + /// A budget with `room` and [`AHEAD`], reading the host as `reads` says: + /// [`resident_bytes`] or [`live_bytes`]. + pub fn new(room: Room, reads: Reading) -> Arc { + let reading: Box Option + Send + Sync> = match reads { + Reading::Resident => Box::new(resident_bytes), + Reading::Live => Box::new(live_bytes), + }; + Arc::new(Self { + reads, + ..Self::over(room, AHEAD, reading) + }) + } + + /// A budget with `room`, `ahead` programs always admitted ahead of the + /// provers (at least one), and the host reading `reading` (the tests' + /// fake hosts). + pub fn with_reading( + room: Room, + ahead: usize, + reading: Box Option + Send + Sync>, + ) -> Arc { + Arc::new(Self::over(room, ahead, reading)) + } + + fn over(room: Room, ahead: usize, reading: Box Option + Send + Sync>) -> Self { + Self { + room, + ahead: ahead.max(1), + reading, + reads: Reading::Resident, + state: Mutex::new(State::default()), + changed: Condvar::new(), + } + } + + /// Whether this budget ever refuses. + pub fn bounded(&self) -> bool { + self.room != Room::Unbounded + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Whether `order` may come in now, and how. + fn admissible(&self, st: &State, order: usize, bytes: u64) -> Option { + if st.waiting.range(..order).next().is_some() { + return None; + } + let room = match self.room { + Room::Unbounded => true, + Room::Bytes(limit) => st.held + bytes <= limit, + Room::Host { line } => { + (self.reading)().is_none_or(|rss| rss + st.unemitted + bytes <= line) + } + }; + if room { + return Some(Admission::Room); + } + let untaken_below = order - st.claimed.range(..order).count(); + (untaken_below < self.ahead).then_some(Admission::Ahead) + } + + fn permit(self: &Arc, order: usize, bytes: u64) -> Permit { + Permit { + budget: self.clone(), + order, + bytes, + emitted: false, + } + } + + /// Admits program `order` with `bytes` estimated, waiting until it may + /// come in. An error once the budget has failed ([`Self::fail`]). + pub fn acquire(self: &Arc, order: usize, bytes: u64) -> Result { + let t = Instant::now(); + let mut st = self.lock(); + assert!( + !st.admitted.contains(&order), + "program {order} admitted twice" + ); + *st.waiting.entry(order).or_insert(0) += 1; + let how = loop { + if st.failed { + break None; + } + if let Some(how) = self.admissible(&st, order, bytes) { + break Some(how); + } + st = self + .changed + .wait_timeout(st, Duration::from_millis(200)) + .unwrap_or_else(|e| e.into_inner()) + .0; + }; + match st.waiting.get_mut(&order) { + Some(n) if *n > 1 => *n -= 1, + _ => { + st.waiting.remove(&order); + } + } + st.waited_secs += t.elapsed().as_secs_f64(); + let Some(how) = how else { + drop(st); + self.changed.notify_all(); + return Err(format!("program {order}: the tree's builder failed")); + }; + st.admit(order, bytes, how); + drop(st); + self.changed.notify_all(); + Ok(self.permit(order, bytes)) + } + + /// Admits program `order` only if it may come in now. + pub fn try_acquire(self: &Arc, order: usize, bytes: u64) -> Option { + let mut st = self.lock(); + if st.failed || st.admitted.contains(&order) { + return None; + } + match self.admissible(&st, order, bytes) { + Some(how) => { + st.admit(order, bytes, how); + Some(self.permit(order, bytes)) + } + None => { + st.refused += 1; + None + } + } + } + + /// Every waiter returns an error from now on (the builder failed). + pub fn fail(&self) { + self.lock().failed = true; + self.changed.notify_all(); + } + + /// The summary: the most bytes alive at once, how the programs came in, + /// and how long admissions waited. + pub fn summary(&self) -> String { + let st = self.lock(); + format!( + "{} · held at most {:.2} GiB · {} by room, {} among the next {} to be taken · {} \ + refused beside the base · admissions waited Σ {:.2}s", + self.describe(), + st.high_water as f64 / GIB as f64, + st.by_room, + st.by_ahead, + self.ahead, + st.refused, + st.waited_secs + ) + } + + /// The room in words. + pub fn describe(&self) -> String { + let gib = |b: u64| b as f64 / GIB as f64; + match self.room { + Room::Unbounded => "off (every program where it was)".to_string(), + Room::Bytes(b) => format!("{:.2} GiB of programs", gib(b)), + Room::Host { line } => match (self.reading)() { + Some(rss) => format!( + "auto: {} ({:.2} GiB now) against the room line {:.2} GiB", + match self.reads { + Reading::Resident => "VmRSS", + Reading::Live if crate::alloc_purge::stats().is_some() => + "VmRSS less what a purge could return", + Reading::Live => "VmRSS, the allocator's figures unread", + }, + gib(rss), + gib(line) + ), + None => format!( + "auto: no VmRSS reading (room line {:.2} GiB): room is never short", + gib(line) + ), + }, + } + } +} + +impl Permit { + /// The program is emitted: its estimate becomes its `actual` bytes, now + /// on the host. + pub fn emitted(&mut self, actual: u64) { + let mut st = self.budget.lock(); + if !self.emitted { + st.unemitted -= self.bytes; + self.emitted = true; + } + st.held = st.held - self.bytes + actual; + st.high_water = st.high_water.max(st.held); + self.bytes = actual; + drop(st); + self.budget.changed.notify_all(); + } + + /// A prover took the program. + pub fn claim(&self) { + let mut st = self.budget.lock(); + if let Some(taken) = st.alive.get_mut(&self.order) { + *taken = true; + } + st.claimed.insert(self.order); + drop(st); + self.budget.changed.notify_all(); + } + + /// The program's prove order. + pub fn order(&self) -> usize { + self.order + } +} + +impl Drop for Permit { + fn drop(&mut self) { + let mut st = self.budget.lock(); + st.held -= self.bytes; + if !self.emitted { + st.unemitted -= self.bytes; + } + st.alive.remove(&self.order); + drop(st); + self.budget.changed.notify_all(); + } +} + +/// Emits `items` on `threads` plain threads, each taking the next item as it +/// is free, and hands each result to `deliver` strictly in item order, as soon +/// as it and every item before it are done (a reorder buffer: a slow item +/// holds back the delivery of the faster ones after it, not their emission). +/// Delivery stops after an error is delivered or once `deliver` returns +/// `false`; the workers then stop taking items. The bytes ahead are bounded +/// by what `emit` admits (a [`ProgramBudget`]), not here. +pub fn emit_ordered( + items: Range, + threads: usize, + emit: &(dyn Fn(usize) -> Result + Sync), + deliver: &mut dyn FnMut(usize, Result) -> bool, +) { + let next = AtomicUsize::new(items.start); + let stop = std::sync::atomic::AtomicBool::new(false); + let (tx, rx) = std::sync::mpsc::channel::<(usize, Result)>(); + std::thread::scope(|scope| { + for _ in 0..threads.max(1) { + let (tx, next, stop) = (tx.clone(), &next, &stop); + scope.spawn(move || { + while !stop.load(Ordering::SeqCst) { + let k = next.fetch_add(1, Ordering::SeqCst); + if k >= items.end { + return; + } + if tx.send((k, emit(k))).is_err() { + return; + } + } + }); + } + drop(tx); + let mut pending: BTreeMap> = BTreeMap::new(); + let mut want = items.start; + 'deliver: while want < items.end { + let Ok((k, result)) = rx.recv() else { + break; + }; + pending.insert(k, result); + while let Some(result) = pending.remove(&want) { + let failed = result.is_err(); + if !deliver(want, result) || failed { + break 'deliver; + } + want += 1; + } + } + stop.store(true, Ordering::SeqCst); + // The workers still emitting finish their item; what they send is + // dropped with the channel. + drop(rx); + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicU64; + + const MIB: u64 = 1 << 20; + + #[test] + fn the_knob_reads_auto_off_or_a_size_and_refuses_nonsense() { + assert_eq!(parse_setting(None), Ok(BudgetSetting::Auto)); + assert_eq!(parse_setting(Some("")), Ok(BudgetSetting::Auto)); + assert_eq!(parse_setting(Some(" auto ")), Ok(BudgetSetting::Auto)); + assert_eq!(parse_setting(Some("off")), Ok(BudgetSetting::Off)); + assert_eq!( + parse_setting(Some("1.5")), + Ok(BudgetSetting::Bytes(3 * GIB / 2)) + ); + assert_eq!(parse_setting(Some("0")), Ok(BudgetSetting::Bytes(0))); + assert!(parse_setting(Some("-1")).is_err()); + assert!(parse_setting(Some("lots")).is_err()); + assert_eq!( + Room::of(BudgetSetting::Auto, 54 * GIB, 1.0), + Room::Host { line: 54 * GIB } + ); + assert_eq!( + Room::of(BudgetSetting::Off, 54 * GIB, 0.75), + Room::Unbounded + ); + } + + /// A fake host: its resident bytes are the budget's held bytes plus a + /// fixed rest, as if every emitted program were resident. + fn host(rest: u64) -> (Arc, Box Option + Send + Sync>) { + let rss = Arc::new(AtomicU64::new(rest)); + let r = rss.clone(); + (rss, Box::new(move || Some(r.load(Ordering::SeqCst)))) + } + + #[test] + fn the_share_knob_reads_a_share_and_refuses_nonsense() { + assert_eq!(parse_share(None), Ok(ROOM_SHARE)); + assert_eq!(parse_share(Some(" ")), Ok(ROOM_SHARE)); + assert_eq!(parse_share(Some("1.0")), Ok(1.0)); + assert_eq!(parse_share(Some("0.5")), Ok(0.5)); + for bad in ["0", "1.5", "-0.2", "most"] { + assert!(parse_share(Some(bad)).is_err(), "{bad}"); + } + } + + /// `auto`'s room line sits at the share of the target: a program comes + /// in by room exactly up to the line, and past it only by the AHEAD rule. + /// Share 1.0 is the line at the target, the budget before the share. + #[test] + fn auto_admits_by_room_exactly_up_to_the_share_line() { + let target = 100 * GIB; + let room = Room::of(BudgetSetting::Auto, target, 0.75); + assert_eq!(room, Room::Host { line: 75 * GIB }); + assert_eq!( + Room::of(BudgetSetting::Auto, target, 1.0), + Room::Host { line: target }, + "share 1.0: the line at the target" + ); + let b = ProgramBudget::with_reading(room, 1, Box::new(|| Some(74 * GIB))); + let p0 = b.try_acquire(0, 0).expect("74 ≤ 75"); + let p1 = b + .try_acquire(1, GIB) + .expect("74 + 1 = 75: on the line, by room"); + assert!( + b.try_acquire(2, 1).is_none(), + "74 + 1 (not yet emitted) + 1 byte > 75, and two untaken below: AHEAD-only" + ); + p0.claim(); + p1.claim(); + assert!( + b.try_acquire(2, 1).is_some(), + "past the line, the next to be taken still comes in" + ); + assert!(b.summary().contains("2 by room, 1 among the next 1")); + // The same host and program against the line at the target: room. + let full = ProgramBudget::with_reading( + Room::of(BudgetSetting::Auto, target, 1.0), + 1, + Box::new(|| Some(74 * GIB)), + ); + let _q0 = full.try_acquire(0, 0).expect("room"); + let _q1 = full.try_acquire(1, GIB).expect("room"); + assert!( + full.try_acquire(2, 1).is_some(), + "75 GiB + 1 byte ≤ 100: room" + ); + } + + fn bytes_budget(limit: u64, ahead: usize) -> Arc { + ProgramBudget::with_reading(Room::Bytes(limit), ahead, Box::new(|| None)) + } + + #[test] + fn room_admits_and_a_full_budget_refuses_all_but_the_next_to_be_taken() { + let b = bytes_budget(100 * MIB, 1); + let p0 = b.try_acquire(0, 60 * MIB).expect("room"); + let p1 = b.try_acquire(1, 30 * MIB).expect("room"); + assert!(b.try_acquire(3, 30 * MIB).is_none(), "no room, not next"); + assert!( + b.try_acquire(2, 30 * MIB).is_none(), + "next pending, but 0 and 1 are not taken" + ); + p0.claim(); + p1.claim(); + let p2 = b.try_acquire(2, 30 * MIB).expect("the next to be taken"); + assert!( + b.try_acquire(3, 30 * MIB).is_none(), + "only one over the room while it is not taken (ahead 1)" + ); + drop(p0); + let p3 = b.try_acquire(3, 30 * MIB).expect("room again"); + assert!( + b.summary() + .contains("3 by room, 1 among the next 1 to be taken") + ); + drop((p1, p2, p3)); + assert_eq!(b.lock().held, 0); + } + + /// With room gone, the next `ahead` programs to be taken still come in + /// (the emission keeps that many ahead of the provers), and no more. + #[test] + fn with_no_room_the_next_ahead_programs_still_come_in() { + let b = bytes_budget(0, 3); + let p: Vec = (0..3) + .map(|k| b.try_acquire(k, MIB).expect("one of the next three")) + .collect(); + assert!(b.try_acquire(3, MIB).is_none(), "three untaken below 3"); + p[0].claim(); + let p3 = b.try_acquire(3, MIB).expect("two untaken below 3"); + assert!(b.try_acquire(4, MIB).is_none(), "three untaken below 4"); + drop(p); + assert!( + b.try_acquire(4, MIB).is_none(), + "dropped without a take: 1 and 2 are still untaken" + ); + drop(p3); + } + + #[test] + fn auto_counts_the_resident_set_and_the_programs_not_yet_emitted() { + let (rss, reading) = host(40 * MIB); + let b = ProgramBudget::with_reading(Room::Host { line: 100 * MIB }, 1, reading); + let mut p0 = b.try_acquire(0, 30 * MIB).expect("40 + 30 ≤ 100"); + assert!( + b.try_acquire(1, 31 * MIB).is_none(), + "40 resident + 30 admitted, not emitted + 31 > 100" + ); + // Emitted: the host now holds it, and the reading says so. + rss.fetch_add(30 * MIB, Ordering::SeqCst); + p0.emitted(30 * MIB); + assert!(b.try_acquire(1, 31 * MIB).is_none(), "70 + 31 > 100"); + let p1 = b.try_acquire(1, 30 * MIB).expect("70 + 30 ≤ 100"); + drop((p0, p1)); + let none = ProgramBudget::with_reading(Room::Host { line: 0 }, 1, Box::new(|| None)); + assert!( + none.try_acquire(0, GIB).is_some(), + "no reading: room never short" + ); + } + + /// Unset or empty, the knob reads live; `resident` is the reading before. + #[test] + fn the_reading_knob_reads_resident_or_live_and_refuses_nonsense() { + assert_eq!(READING_DEFAULT, Reading::Live); + assert_eq!(parse_reading(None), Ok(Reading::Live)); + assert_eq!(parse_reading(Some(" ")), Ok(Reading::Live)); + assert_eq!(parse_reading(Some("resident")), Ok(Reading::Resident)); + assert_eq!(parse_reading(Some(" live ")), Ok(Reading::Live)); + for bad in ["rss", "1", "LIVE", "purgeable"] { + assert!(parse_reading(Some(bad)).is_err(), "{bad}"); + } + } + + /// The allocator's figures, in MiB. + fn allocator(resident: u64, active: u64) -> crate::alloc_purge::AllocStats { + crate::alloc_purge::AllocStats { + allocated: (active * MIB) as usize, + active: (active * MIB) as usize, + resident: (resident * MIB) as usize, + ..Default::default() + } + } + + /// The live reading is the resident set less what a purge could return, + /// never under the allocator's active bytes, never over the resident set, + /// and the resident set itself when nothing is purgeable or the allocator + /// cannot be read. + #[test] + fn the_live_reading_takes_off_what_a_purge_could_return_and_no_more() { + // Nothing purgeable: the two readings are one. + assert_eq!(live_of(140 * MIB, Some(&allocator(90, 90))), 140 * MIB); + // 80 purgeable of 490 resident. + assert_eq!(live_of(490 * MIB, Some(&allocator(480, 400))), 410 * MIB); + // The allocator's resident figure runs over the process's (a large + // block's statement, in units of 0.1 GiB): the difference alone would + // read 27, the live heap is 108. + assert_eq!(live_of(669 * MIB, Some(&allocator(750, 108))), 108 * MIB); + // Active pages never touched: not over the resident set. + assert_eq!(live_of(200 * MIB, Some(&allocator(300, 300))), 200 * MIB); + // No figures from the allocator: the resident set, the reading before. + assert_eq!(live_of(669 * MIB, None), 669 * MIB); + } + + /// The count of 1.08-unit programs a budget admits in one burst, as the + /// planner asks at the statement (none emitted in between). + fn admitted_at_once(line: u64, reading: u64) -> usize { + let b = + ProgramBudget::with_reading(Room::Host { line }, 1, Box::new(move || Some(reading))); + let permits: Vec = (0..26).map_while(|k| b.try_acquire(k, 108 * MIB)).collect(); + permits.len() + } + + /// A host with room, in units of 0.01 GiB: 66.9 resident, 64.2 of it + /// purgeable, the room line at 81.21. The resident reading admits 13 of + /// 26 programs; the live reading admits them all. A host whose memory is + /// really short (the same block purged at that point on a 64 GiB host: + /// 14.6 resident, nothing purgeable, the line at 40.5) admits 24 under + /// either reading and refuses the 25th. + #[test] + fn the_live_reading_admits_into_freed_pages_and_a_short_host_still_refuses() { + let (rss, stats) = (6690 * MIB, allocator(7500, 1080)); + assert_eq!(admitted_at_once(8121 * MIB, rss), 13); + assert_eq!(admitted_at_once(8121 * MIB, live_of(rss, Some(&stats))), 26); + let (rss, stats) = (1458 * MIB, allocator(900, 900)); + assert_eq!(live_of(rss, Some(&stats)), rss); + assert_eq!(admitted_at_once(4050 * MIB, rss), 24); + assert_eq!(admitted_at_once(4050 * MIB, live_of(rss, Some(&stats))), 24); + // Short of live memory with pages to purge as well: the live heap + // alone fills the line, and only the next to be taken comes in. + let (rss, stats) = (5000 * MIB, allocator(5400, 4000)); + assert_eq!(live_of(rss, Some(&stats)), 4000 * MIB); + assert_eq!(admitted_at_once(4050 * MIB, live_of(rss, Some(&stats))), 1); + } + + /// The line names the reading: `VmRSS` for the resident one (and for a + /// test's own reading), as it always read; the default's line says what + /// it takes off. + #[test] + fn the_budget_line_names_its_reading() { + let room = Room::Host { line: 3 * GIB }; + let before = ProgramBudget::with_reading(room, 1, Box::new(|| Some(GIB))); + assert_eq!( + before.describe(), + "auto: VmRSS (1.00 GiB now) against the room line 3.00 GiB" + ); + let resident = ProgramBudget::new(room, Reading::Resident).describe(); + assert!( + resident.starts_with("auto: VmRSS (") || resident.starts_with("auto: no VmRSS reading"), + "{resident}" + ); + let live = ProgramBudget::new(room, Reading::Live).describe(); + assert!( + live.starts_with("auto: VmRSS less what a purge could return (") + || live.starts_with("auto: no VmRSS reading"), + "{live}" + ); + let default = ProgramBudget::new(room, parse_reading(None).expect("unset")).describe(); + assert!( + default.starts_with("auto: VmRSS less what a purge could return (") + || default.starts_with("auto: no VmRSS reading"), + "{default}" + ); + } + + #[test] + fn a_program_never_jumps_a_lower_waiter() { + let b = bytes_budget(50 * MIB, 1); + let p0 = b.try_acquire(0, 50 * MIB).expect("room"); + std::thread::scope(|scope| { + let waiter = scope.spawn(|| b.acquire(1, 10 * MIB).map(|p| p.order())); + while b.lock().waiting.is_empty() { + std::thread::yield_now(); + } + drop(p0); + // Room for 2 now, but 1 waits below it and comes first. + let got1 = waiter.join().expect("the waiter").expect("admitted"); + assert_eq!(got1, 1); + }); + let mut st = b.lock(); + st.waiting.insert(3, 1); + drop(st); + assert!(b.try_acquire(4, MIB).is_none(), "3 waits below 4"); + assert!(b.try_acquire(2, MIB).is_some(), "2 is below the waiter"); + } + + /// A one-byte budget and three levels of in-order consumers, as the tree + /// takes its programs: every program is admitted, in order, no consumer + /// waits forever, and at most `ahead` programs are alive beyond those in a + /// prover's hands (ahead 1 and [`AHEAD`]). Without the lowest-pending rule the tree deadlocks: the + /// timeout then fails the budget, which frees every thread, and the test + /// fails by name instead of hanging. + #[test] + fn a_one_byte_budget_cannot_deadlock_the_tree() { + for ahead in [1, AHEAD] { + one_byte_tree(ahead); + } + } + + fn one_byte_tree(ahead: usize) { + let b = bytes_budget(1, ahead); + let levels: [Range; 3] = [0..12, 12..15, 15..16]; + let slots: Vec>> = (0..16).map(|_| Mutex::new(None)).collect(); + let max_alive = AtomicUsize::new(0); + let stop = std::sync::atomic::AtomicBool::new(false); + let (done_tx, done_rx) = std::sync::mpsc::channel::<()>(); + let finished = std::thread::scope(|scope| { + // Two emitters: the leaves in order, the nodes in order. + for orders in [0..12usize, 12..16] { + let (b, slots) = (&b, &slots); + scope.spawn(move || { + for k in orders { + let Ok(p) = b.acquire(k, 10 * MIB) else { + return; + }; + *slots[k].lock().unwrap() = Some(p); + } + }); + } + let (b, slots, max_alive, stop) = (&b, &slots, &max_alive, &stop); + scope.spawn(move || { + for level in levels { + // Three provers a level, each taking the next order. + let next = AtomicUsize::new(level.start); + std::thread::scope(|s| { + for _ in 0..3 { + s.spawn(|| { + loop { + let k = next.fetch_add(1, Ordering::SeqCst); + if k >= level.end { + return; + } + let p = loop { + if let Some(p) = slots[k].lock().unwrap().take() { + break p; + } + if stop.load(Ordering::SeqCst) { + return; + } + std::thread::sleep(Duration::from_millis(1)); + }; + p.claim(); + max_alive.fetch_max(b.lock().alive.len(), Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(2)); + drop(p); + } + }); + } + }); + } + let _ = done_tx.send(()); + }); + let finished = done_rx.recv_timeout(Duration::from_secs(20)).is_ok(); + if !finished { + stop.store(true, Ordering::SeqCst); + b.fail(); + } + finished + }); + assert!( + finished, + "the tree deadlocked under a one-byte budget (ahead {ahead})" + ); + assert!( + max_alive.load(Ordering::SeqCst) <= 3 + ahead, + "{} alive: more than the provers in flight plus {ahead}", + max_alive.load(Ordering::SeqCst) + ); + assert_eq!(b.lock().held, 0); + } + + #[test] + fn a_failed_budget_wakes_every_waiter() { + let b = bytes_budget(0, 1); + let held = b.try_acquire(0, MIB).expect("the next to be taken"); + std::thread::scope(|scope| { + let w = scope.spawn(|| b.acquire(1, MIB).err()); + while b.lock().waiting.is_empty() { + std::thread::yield_now(); + } + b.fail(); + assert!(w.join().unwrap().is_some(), "the waiter returns an error"); + }); + drop(held); + } + + /// The streaming emitter delivers in item order even when later items + /// finish first. Delivering in completion order instead fails here. + #[test] + fn the_streaming_emitter_delivers_in_order_under_adversarial_completion() { + let items = 3..23; + // Earlier items take longer: completion order is the reverse of item + // order within each wave of four. + let emit = |k: usize| -> Result { + std::thread::sleep(Duration::from_millis(((23 - k) * 3) as u64)); + Ok(k * 10) + }; + let mut got = Vec::new(); + emit_ordered(items.clone(), 4, &emit, &mut |k, r| { + got.push((k, r.expect("emitted"))); + true + }); + let want: Vec<(usize, usize)> = items.map(|k| (k, k * 10)).collect(); + assert_eq!(got, want); + } + + /// An error is delivered after every item before it, and nothing after. + #[test] + fn the_streaming_emitter_stops_at_an_error() { + let emit = |k: usize| -> Result { + if k == 7 { + Err(format!("item {k} does not emit")) + } else { + Ok(k) + } + }; + let mut got = Vec::new(); + emit_ordered(5..30, 3, &emit, &mut |k, r| { + got.push((k, r)); + true + }); + assert_eq!( + got, + vec![ + (5, Ok(5)), + (6, Ok(6)), + (7, Err("item 7 does not emit".to_string())) + ] + ); + } +} diff --git a/prover/src/lfm/program_census.rs b/prover/src/lfm/program_census.rs new file mode 100644 index 000000000..5074b9592 --- /dev/null +++ b/prover/src/lfm/program_census.rs @@ -0,0 +1,533 @@ +//! Per level: how many proofs were emitted, and how many DISTINCT PROGRAMS were +//! behind them. +//! +//! # This line exists to document why there is no artifact cache +//! +//! `build_artifacts_with_hasher` is a pure function of `(program, options, +//! hasher)`, and lane P measured it as the largest phase of the recursion +//! pipeline — 22.0 s of an L1 interior node's 34.0 s wall, all host CPU. That +//! shape invites a memo: if a level's proofs shared a program, a level would +//! build its artifacts once instead of once per proof. +//! +//! ⛔ **They do not share one, and it is load-bearing that they do not.** Every +//! child's epoch label and every node's label range are pinned as CONSTANTS of +//! the emitted program — that pinning is exactly what makes contiguity across +//! sibling subtrees a consequence of the pins rather than a check of its own. So +//! sibling programs differ by construction: different constants, different +//! instruction column groups, different preprocessed roots, different +//! `program_id`. A memo keyed on anything sound records zero hits, and an LDE is +//! global, so no incremental Merkle trick survives a changed constant either. +//! +//! ⇒ What is left is the COUNT, and the count is worth printing: it is the +//! evidence for the paragraph above. A level reporting `N proofs, N distinct +//! programs` is the measurement that says a cache was not left on the table. A +//! level that ever reported fewer programs than proofs would mean the premise +//! had changed and the memo was worth revisiting. +//! +//! # The key is `program_id`, and here it can be +//! +//! A memo cannot be keyed on `program_id`: it is the build's OUTPUT, the digest +//! of the roots the build produces, so a lookup meant to AVOID the build cannot +//! have it in hand. A census has no such problem — it counts after the build — +//! so it keys on the identity the parents absorb and the soundness argument +//! already trusts, rather than on a second hash of the build's inputs. + +use std::collections::HashSet; +use std::sync::{Mutex, MutexGuard, OnceLock}; +use std::thread::ThreadId; +use std::time::{Duration, Instant}; + +use stark::config::Commitment; +use stark::proof::options::ProofOptions; + +use super::compiler::LfmProgram; +use super::hash::HasherKind; +use super::registry::LfmArtifacts; +#[cfg(test)] +use super::registry::build_artifacts_with_hasher; + +/// What one level's worth of artifact builds did. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LevelStats { + /// Proofs that built artifacts inside the window. + pub proofs: usize, + /// Distinct `program_id`s among them. Expected to equal `proofs`; see the + /// module note for why, and for what it would mean if it did not. + pub distinct: usize, + /// Time spent inside `build_artifacts_with_hasher`, SUMMED over the + /// proofs above. + /// + /// ⚠ A SUM, not a wall, and the distinction only appears when a level + /// proves its siblings concurrently: two overlapping 9 s builds sum to 18 s + /// inside a 9 s window. Serial, the two are the same number, which is + /// exactly why this needs saying before anything runs concurrently — a + /// figure that has been a wall for the whole campaign would silently start + /// being something else and read as a regression. [`Self::wall_nanos`] is + /// the window's own elapsed time, and [`Self::describe`] says so on the + /// line whenever the two disagree. + pub build_nanos: u128, + /// How long the window was open: `begin_level` to `end_level`. + pub wall_nanos: u128, + /// The largest device working set any artifact commit has ASKED FOR in this + /// process, in bytes — `stark::device_set`'s own term-by-term accounting, + /// which is the number admission decides on. Zero when nothing went to the + /// card, which is every host build, so the line stays quiet there. + /// + /// ⓘ A running process maximum, not this level's own: the commits are + /// sequential and the card is released between them, so the largest single + /// set is what the build ever needed. It is what the artifact build ASKED + /// for; an external sampler is what says what the process held. + pub device_peak_bytes: u64, + /// Committed groups that took the device path, and those that stayed on the + /// host because `padded_rows · blowup` was under `gpu_lde`'s 2^14 floor. + /// ★ THIS LEVEL's own, not the process total — see [`Window::groups_at_open`]. + pub device_groups: u64, + pub host_groups: u64, +} + +impl LevelStats { + /// The line the driver prints per level. + pub fn describe(&self, label: &str) -> String { + format!( + "{label}: {} proofs, {} distinct programs, artifacts {:.1}s{}{}", + self.proofs, + self.distinct, + Duration::from_nanos(self.build_nanos as u64).as_secs_f64(), + match self.device_peak_bytes { + 0 => String::new(), + // ⚠ The two figures have DIFFERENT scopes and the line says so. + // The groups are this level's; the set is the largest any commit + // has asked for since the process started, because it is an + // atomic maximum and a window cannot un-see one. + b => format!( + " · groups {}/{} on device · device set <= {:.2} GiB (process max)", + self.device_groups, + self.device_groups + self.host_groups, + b as f64 / (1024.0 * 1024.0 * 1024.0), + ), + }, + if self.distinct == self.proofs { + " (N/N — sibling programs differ by construction; nothing to cache)" + } else { + " ⚠ FEWER PROGRAMS THAN PROOFS — the label pinning changed, a memo is back on the table" + }, + ) + &self.concurrency_note() + } + + /// Empty while the level is serial, so the line is byte-identical to every + /// one already in the campaign's logs — and present the moment the sum + /// stops being a wall, so nobody reads a concurrent level's `artifacts` + /// figure as the wall it used to be. + /// + /// The 5% margin is a clock margin, not a tolerance for overlap: the two + /// spans are taken by different timers and a serial level lands a hair + /// under, never over. + fn concurrency_note(&self) -> String { + let sum = Duration::from_nanos(self.build_nanos as u64).as_secs_f64(); + let wall = Duration::from_nanos(self.wall_nanos as u64).as_secs_f64(); + if wall > 0.0 && sum > wall * 1.05 { + format!( + " · ⓘ artifacts is a SUM over {} proofs ({sum:.1}s) inside a {wall:.1}s \ + window — this level proved concurrently", + self.proofs + ) + } else { + String::new() + } + } +} + +struct Window { + seen: HashSet, + stats: LevelStats, + /// When the window opened, for [`LevelStats::wall_nanos`]. + opened_at: Instant, + /// ★★★ THE THREADS THIS LEVEL IS MADE OF. A build on any other thread is + /// not this level's work and is not counted. + /// + /// ⛔ THIS IS WHY THE WINDOW CAN SURVIVE CONCURRENCY. It is process-global + /// and it already miscounted once: at `--test-threads=2` a sibling test + /// building on another thread landed inside an open window and the counting + /// test read 5 proofs where it had made 3 — and a first run that happened + /// to serialize them passed, which is the worst way for a counter to be + /// wrong. Serializing the builders was the fix available then. It is not + /// available to a level that proves its siblings concurrently, because + /// concurrent builders are the point. + /// + /// ⇒ So membership is EXPLICIT rather than ambient. A thread that did not + /// enrol cannot inflate the window no matter what it builds or when, which + /// makes the bad state unreachable instead of merely unlikely — and the + /// same guarantee covers the serial path, where it is the case that + /// actually bit. + enrolled: HashSet, + /// The device/host group counters as they stood when the window OPENED. + /// + /// ⛔ The counters in `commit` are process-cumulative, because the device + /// floor is a process constant and a running total is the useful thing to + /// keep. A per-LEVEL line reporting them raw reads as "all groups so far", + /// which is exactly how a reader ends up subtracting two levels by hand to + /// recover the one they wanted. The window keeps its own baseline and + /// reports the DELTA. + groups_at_open: (u64, u64), +} + +#[derive(Default)] +struct Census { + window: Option, +} + +fn census() -> &'static Mutex { + static CENSUS: OnceLock> = OnceLock::new(); + CENSUS.get_or_init(|| Mutex::new(Census::default())) +} + +fn lock() -> MutexGuard<'static, Census> { + census() + .lock() + .expect("the program census mutex is never held across a panic") +} + +/// [`super::registry::build_artifacts_with_hasher`], counted. +/// +/// ⛔ NOT a cache and never becomes one: no map, no key, no reuse. It runs the +/// build every time, exactly as the raw call does, and records which program it +/// built for so a level can report how many distinct ones it saw. +/// +/// ⚠ The lock is taken ONCE, after the build, and the counter is updated inside +/// it. The memo this replaced took it twice — once to look up, once to count — +/// and held the first guard across the second call. `Mutex` is not reentrant, so +/// that was a self-deadlock, and a quiet one: the thread parked, the test binary +/// sat at 0% CPU and 22 MB of RSS, and nothing printed. +pub fn build_artifacts_counted( + program: &LfmProgram, + options: &ProofOptions, + hasher: HasherKind, +) -> LfmArtifacts { + build_artifacts_counted_under::(program, options, hasher) +} + +/// [`build_artifacts_counted`] with the roots built under the commitment +/// configuration `S` ([`super::registry::build_artifacts_under`]). +pub fn build_artifacts_counted_under( + program: &LfmProgram, + options: &ProofOptions, + hasher: HasherKind, +) -> LfmArtifacts { + // ⛔ THE CARD, FOR THE FIRST OF A PROOF'S TWO DEVICE PHASES. + // `commit_group_device_or_host` dispatches to `gpu_lde` from inside this + // build, outside `multi_prove` and so outside every `VramGate`, and its + // admission is a per-dispatch bound with no running total. Inert unless a + // driver has armed it. + // + // ⚠ Inside the timer on purpose: what a build COST a concurrent level + // includes what it waited for the card, and a figure that excluded the wait + // would make a device-bound level look host-bound. + let t = Instant::now(); + // The merged AIRs' preprocessed groups (`LAMBDA_VM_ZF_LFM_MERGE`) are laid + // side by side on the host BEFORE the card is taken: a copy of the parts' + // groups, which has no business inside the hold. Nothing without a merge. + let plan = super::registry::merge_plan_for(program, options); + let merged = super::merge::merged_groups(program, &plan); + let artifacts = { + let _card = super::device_permit::hold_labeled("build_artifacts"); + super::registry::build_artifacts_merged_under::(program, options, hasher, &plan, &merged) + }; + drop(merged); + let build_nanos = t.elapsed().as_nanos(); + lock().record(artifacts.program_id, build_nanos); + artifacts +} + +impl Census { + fn record(&mut self, program_id: Commitment, build_nanos: u128) { + let Some(w) = self.window.as_mut() else { + return; + }; + if !w.enrolled.contains(&std::thread::current().id()) { + return; + } + w.seen.insert(program_id); + w.stats.proofs += 1; + w.stats.distinct = w.seen.len(); + w.stats.build_nanos += build_nanos; + } +} + +/// A worker thread's membership of the open level window, for as long as it is +/// held. +/// +/// Taken by each thread that proves a sibling of the level being counted, and +/// dropped when that thread is done. The driver's own thread is enrolled by +/// [`begin_level`], so a serial level needs none of this. +#[must_use = "an enrolment that is dropped immediately counts nothing"] +pub struct Enrolment { + thread: ThreadId, +} + +impl Drop for Enrolment { + fn drop(&mut self) { + // ⚠ NOT `lock()`. This runs during unwinding when a worker panics, and + // `lock()` panics on a poisoned mutex — a panic inside a `Drop` that is + // already unwinding aborts the process. A counter is not worth that, so + // a poisoned census here simply leaves the enrolment in place: the + // window is about to be discarded by a failing run anyway. + if let Ok(mut census) = census().lock() + && let Some(w) = census.window.as_mut() + { + w.enrolled.remove(&self.thread); + } + } +} + +/// Enrol the CALLING thread in the open level window. +/// +/// ⚠ Call it on the worker itself, not on the thread that spawned it: the +/// enrolment names `std::thread::current()`, and enrolling from the spawner +/// would enrol the spawner twice and the worker never — a mistake that counts +/// zero rather than counting wrong, which is the direction this should fail in. +/// +/// A no-op when no window is open, so a worker need not know whether its level +/// is being counted. +pub fn enrol() -> Enrolment { + let thread = std::thread::current().id(); + if let Some(w) = lock().window.as_mut() { + w.enrolled.insert(thread); + } + Enrolment { thread } +} + +/// Start counting a level. Replaces any window already open — the driver's +/// levels do not nest. +pub fn begin_level() { + let groups_at_open = super::commit::device_host_group_counts(); + lock().window = Some(Window { + seen: HashSet::new(), + stats: LevelStats::default(), + opened_at: Instant::now(), + // The caller's own thread, so a level that never spawns anything counts + // exactly what it did before this existed. + enrolled: HashSet::from([std::thread::current().id()]), + groups_at_open, + }); +} + +/// Close the window and take what it counted. `None` when no window was open. +pub fn end_level() -> Option { + let w = lock().window.take()?; + let mut stats = w.stats; + stats.wall_nanos = w.opened_at.elapsed().as_nanos(); + stats.device_peak_bytes = super::commit::device_artifact_peak_bytes(); + let (dev, host) = super::commit::device_host_group_counts(); + stats.device_groups = dev.saturating_sub(w.groups_at_open.0); + stats.host_groups = host.saturating_sub(w.groups_at_open.1); + Some(stats) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lfm::programs::{fri_toy_program, trivial_program}; + use crate::tables::types::FE; + use stark::proof::options::GoldilocksCubicProofOptions; + + /// ⛔ EVERY TEST HERE THAT BUILDS TAKES THIS, not just the one that counts. + /// + /// It no longer guards the COUNT. A sibling test building on another + /// harness thread used to land inside an open window and inflate it — at + /// `--test-threads=2` the counting test read 5 proofs where it had made 3, + /// and a first run that happened to serialize them passed. Enrolment closed + /// that by construction: an un-enrolled thread's build is not counted, and + /// `a_build_on_an_unenrolled_thread_is_not_counted` is the assertion that it + /// stays closed. + /// + /// What is left for the guard is the WINDOW ITSELF: `begin_level` replaces + /// whatever is open, so two tests opening one concurrently would each see + /// the other's. One window at a time is the invariant, and this is it. + /// + /// Outside this module nothing touches the census in a default `cargo test` + /// run — the driver tests that do are all `#[ignore]`. + static WINDOW: Mutex<()> = Mutex::new(()); + + fn opts() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") + } + + /// ★★★ THE GATE. Counting must not change what is built — field by field, + /// roots included, not `program_id` alone. + #[test] + fn counted_artifacts_are_byte_identical_to_a_raw_build() { + let _guard = WINDOW.lock().expect("the window guard is never poisoned"); + let program = trivial_program(); + let options = opts(); + let raw = build_artifacts_with_hasher(&program, &options, HasherKind::Test); + let counted = build_artifacts_counted(&program, &options, HasherKind::Test); + assert_eq!(raw, counted, "the counted build drifted from the raw one"); + assert_eq!(raw.roots, counted.roots, "group roots"); + assert_eq!(raw.log_heights, counted.log_heights, "group heights"); + assert_eq!( + raw.blake3_chunk_roots, counted.blake3_chunk_roots, + "LFM_BLAKE3 chunk roots" + ); + assert_eq!(raw.program_id, counted.program_id, "program_id"); + } + + /// ★ THE PREMISE THE CENSUS EXISTS TO CHECK, at the smallest scale that has + /// it: two programs that differ only in an emitted CONSTANT are two + /// programs with two identities. That is why sibling wraps and sibling + /// nodes never share artifacts, and it is asserted here rather than in + /// prose. + #[test] + fn one_changed_felt_is_a_different_program() { + let _guard = WINDOW.lock().expect("the window guard is never poisoned"); + let options = opts(); + let base = trivial_program(); + let mut tweaked = trivial_program(); + let g = &mut tweaked.groups.hint; + assert!(!g.data.is_empty(), "the HINT group must carry columns"); + let last = g.data.len() - 1; + g.data[last] += FE::one(); + assert_eq!(g.width, base.groups.hint.width, "the shapes must agree"); + assert_eq!( + g.padded_rows, base.groups.hint.padded_rows, + "the shapes must agree" + ); + + let a = build_artifacts_counted(&base, &options, HasherKind::Test); + let b = build_artifacts_counted(&tweaked, &options, HasherKind::Test); + assert_ne!( + a.program_id, b.program_id, + "one changed field element must move the program identity" + ); + assert_ne!(a.roots, b.roots, "and must move the preprocessed roots"); + } + + /// Both readings the driver can print, including the one that would put a + /// memo back on the table — so the counter is known to be able to produce + /// it rather than only ever printing N/N. + #[test] + fn a_level_window_counts_proofs_and_programs() { + let _guard = WINDOW.lock().expect("the window guard is never poisoned"); + let options = opts(); + + begin_level(); + for _ in 0..3 { + let _ = build_artifacts_counted(&fri_toy_program(), &options, HasherKind::Test); + } + let repeated = end_level().expect("a window was open"); + assert_eq!(repeated.proofs, 3); + assert_eq!(repeated.distinct, 1, "three proofs of ONE program"); + assert!( + !repeated.describe("level x").contains("N/N"), + "1 of 3 is not N/N: {}", + repeated.describe("level x") + ); + + begin_level(); + let _ = build_artifacts_counted(&fri_toy_program(), &options, HasherKind::Test); + let _ = build_artifacts_counted(&trivial_program(), &options, HasherKind::Test); + let distinct = end_level().expect("a window was open"); + assert_eq!(distinct.proofs, 2); + assert_eq!(distinct.distinct, 2, "two proofs of TWO programs"); + assert!( + distinct.describe("level y").contains("N/N"), + "an N/N level must say so: {}", + distinct.describe("level y") + ); + + assert!( + end_level().is_none(), + "the window closes once, not once per read" + ); + } + + /// ★★★ THE CONTAMINATION, MADE UNREACHABLE. A thread that did not enrol + /// builds inside an open window and the window does not move. + /// + /// This is the defect that already shipped once and passed on scheduling + /// luck. It is asserted rather than avoided because avoiding it — keeping + /// the builders serial — is precisely what a level proving its siblings + /// concurrently cannot do. + #[test] + fn a_build_on_an_unenrolled_thread_is_not_counted() { + let _guard = WINDOW.lock().expect("the window guard is never poisoned"); + let options = opts(); + + begin_level(); + std::thread::scope(|s| { + s.spawn(|| { + // No `enrol()`. This thread is not part of the level. + let _ = build_artifacts_counted(&fri_toy_program(), &options, HasherKind::Test); + }); + }); + let stats = end_level().expect("a window was open"); + assert_eq!( + stats.proofs, 0, + "an un-enrolled thread's build inflated the window: {stats:?}" + ); + assert_eq!( + stats.build_nanos, 0, + "and it must contribute no time either" + ); + } + + /// ★★ AND THE OTHER DIRECTION, which is what makes the test above a gate + /// rather than a way of counting nothing: an ENROLLED worker IS counted. + /// + /// Without this, `record` could return early on every call and the + /// unreachability test would still pass. + #[test] + fn an_enrolled_worker_thread_is_counted() { + let _guard = WINDOW.lock().expect("the window guard is never poisoned"); + let options = opts(); + + begin_level(); + std::thread::scope(|s| { + for p in [fri_toy_program(), trivial_program()] { + let options = &options; + s.spawn(move || { + let _enrolled = enrol(); + let _ = build_artifacts_counted(&p, options, HasherKind::Test); + }); + } + }); + let stats = end_level().expect("a window was open"); + assert_eq!( + stats.proofs, 2, + "two enrolled workers, two proofs: {stats:?}" + ); + assert_eq!(stats.distinct, 2, "two different programs"); + assert!( + stats.build_nanos > 0, + "and their build time reached the window" + ); + } + + /// ★ THE LINE SAYS WHEN ITS `artifacts` FIGURE STOPPED BEING A WALL — and + /// says nothing when it has not, so every line already in the campaign's + /// logs still reads the same. + #[test] + fn the_level_line_announces_a_concurrent_sum_and_only_then() { + let serial = LevelStats { + proofs: 2, + distinct: 2, + build_nanos: 9_000_000_000, + wall_nanos: 9_100_000_000, + ..LevelStats::default() + }; + assert!( + !serial.describe("level x").contains("SUM"), + "a serial level must not have acquired a new clause: {}", + serial.describe("level x") + ); + + let concurrent = LevelStats { + build_nanos: 18_000_000_000, + ..serial + }; + assert!( + concurrent.describe("level x").contains("SUM"), + "a sum twice its window must say so: {}", + concurrent.describe("level x") + ); + } +} diff --git a/prover/src/lfm/program_shape_tests.rs b/prover/src/lfm/program_shape_tests.rs new file mode 100644 index 000000000..04775b528 --- /dev/null +++ b/prover/src/lfm/program_shape_tests.rs @@ -0,0 +1,708 @@ +//! Recursion-program shape: `BITWISE` only in the programs whose instantiated +//! chips send it a lookup ([`ChipSet::bitwise`]), and `LFM_HASH` split into two +//! power-of-two instances where one table would be mostly padding +//! ([`HashChunking`]). +//! +//! The premise tests read interaction lists, split rules and digests and prove +//! nothing; the tests that prove (`*_proves_*`, `*_refused`, `an_emitted_leg_*`) +//! belong on the box. + +use stark::proof::options::ProofOptions; +use stark::proof::view::MultiProofView; + +use crate::GoldilocksCubicProofOptions; +use crate::tables::types::FE; + +use stark::config::Commitment; + +use super::airs::{ + BITWISE_SLOT, ChipSet, HASH_SLOT, KEEP_BITWISE_ENV, LFM_CHIP_NAMES, LfmAirs, env_switch, + keep_bitwise, keep_bitwise_setting, lfm_chip_census_masked, +}; +use super::chunking::HashChunking; +use super::hash::HasherKind; +use super::programs::{keccak_chain_program, trivial_program}; +use super::proof::{lfm_prove, lfm_prove_with_hasher, verify_against, verify_against_artifacts}; +use super::registry::{LfmArtifacts, build_artifacts, build_artifacts_with_hasher}; +use super::statement::{lfm_program_id, lfm_program_id_chunked}; +use super::word::LfmWord; + +const HASHERS: [HasherKind; 3] = [HasherKind::Test, HasherKind::Blake3, HasherKind::Rpx]; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") +} + +/// The four family combinations, each keeping `BITWISE`. +fn family_masks() -> [ChipSet; 4] { + [(false, false), (true, false), (false, true), (true, true)].map(|(keccak, blake3)| ChipSet { + keccak, + blake3, + bitwise: true, + }) +} + +#[test] +fn an_on_off_switch_reads_zero_one_and_unset() { + assert_eq!(env_switch("K", None), None, "unset is the switch's default"); + assert_eq!(env_switch("K", Some("")), None, "empty is unset"); + assert_eq!(env_switch("K", Some("0")), Some(false)); + assert_eq!(env_switch("K", Some(" 0 ")), Some(false)); + assert_eq!(env_switch("K", Some("1")), Some(true)); + assert_eq!(env_switch("K", Some(" 1\n")), Some(true)); +} + +/// A typo stops the run rather than proving the default under the other arm's +/// name. +#[test] +#[should_panic(expected = "must be 0 or 1")] +fn a_malformed_switch_stops_the_run() { + let _ = keep_bitwise_setting(Some("yes")); +} + +/// The default drops `BITWISE` where nothing sends to it; the opt-out keeps it. +#[test] +fn the_bitwise_default_drops_the_table_and_the_opt_out_keeps_it() { + assert_eq!(KEEP_BITWISE_ENV, "LAMBDA_VM_LFM_KEEP_BITWISE"); + assert!(!keep_bitwise_setting(None), "unset: the default drops it"); + assert!(!keep_bitwise_setting(Some("0"))); + assert!(keep_bitwise_setting(Some("1")), "the opt-out keeps it"); + let trivial = trivial_program(); + for (raw, kept) in [(None, false), (Some("1"), true)] { + let mask = + ChipSet::for_program_under(&trivial, HasherKind::Rpx, !keep_bitwise_setting(raw)); + assert_eq!(mask.bitwise, kept, "{raw:?}"); + } +} + +/// ★ The opt-out is the machine as it was, digest for digest: keeping `BITWISE` +/// in the two registry programs that send it nothing reproduces the +/// `program_id`s their rows carried before the table became conditional (the +/// rows now carry the default's). +#[test] +fn keeping_bitwise_reproduces_the_legacy_registry_digests() { + use super::programs::fri_toy_program; + let hex = |id: &Commitment| id.iter().map(|b| format!("{b:02x}")).collect::(); + for (name, program, legacy) in [ + ( + "TrivialV0", + trivial_program(), + "ffaff6eef4dc287ff394d191613cda007d2ec76daa6e38966481deed27fd68de", + ), + ( + "FriToyV0", + fri_toy_program(), + "55b64ced570cb199c605d819770701c23e0b93da00c6b46d99d57ec06bb649f4", + ), + ] { + let built = build_artifacts(&program, &options()); + assert_eq!( + built.chip_set.bitwise, + keep_bitwise(), + "{name}: sends BITWISE nothing, so only the opt-out keeps it" + ); + let kept = under_mask( + &built, + ChipSet { + bitwise: true, + ..built.chip_set + }, + ); + assert_eq!(hex(&kept.program_id), legacy, "{name}: the legacy digest"); + } +} + +/// ★ THE PREMISE THE MASK RESTS ON, read off the interaction lists: the only chips +/// that touch a bus `BITWISE` receives are the keccak family, `LFM_BLAKE3`, and +/// the hash chip under the BLAKE3 socket. If any other chip gains a byte lookup, +/// this fails — and so does every mask that would have dropped its receiver. +#[test] +fn only_the_byte_lookup_chips_send_to_bitwise() { + for mask in family_masks() { + for hasher in HASHERS { + assert_eq!( + mask.bitwise_required(hasher), + mask.keccak || mask.blake3 || hasher == HasherKind::Blake3, + "{mask:?} under {hasher:?}" + ); + } + } +} + +/// Keeping `BITWISE` is the machine as it was: the tag every digest folds and +/// the sub-proof count are unchanged. Dropping it sets tag bit 2 and removes +/// exactly one sub-proof. +#[test] +fn keeping_bitwise_keeps_every_tag_and_air_count() { + for kept in family_masks() { + assert_eq!( + kept.as_tag(), + (kept.keccak as u8) | ((kept.blake3 as u8) << 1), + "{kept:?}: the tag of a mask that keeps BITWISE must not move" + ); + let dropped = ChipSet { + bitwise: false, + ..kept + }; + assert_eq!(dropped.as_tag(), kept.as_tag() | 0b100, "{dropped:?}"); + for (rnd, b3) in [(0, 0), (1, 1), (3, 2)] { + let rnd = kept.keccak_rnd_chunks(rnd.max(1)); + let b3 = kept.blake3_chunks(b3.max(1)); + assert_eq!( + dropped.num_airs(rnd, b3) + 1, + kept.num_airs(rnd, b3), + "{kept:?}" + ); + } + } + assert_eq!(ChipSet::FULL.as_tag(), 0b011); + const { assert!(ChipSet::FULL.bitwise) }; +} + +/// The mask follows the program: a program with no byte lookups drops +/// `BITWISE` by default and keeps it under the opt-out; a keccak program keeps +/// it either way; the BLAKE3 socket keeps it for a program with no family at +/// all. +#[test] +fn the_bitwise_mask_follows_the_program() { + let trivial = trivial_program(); + let chain = keccak_chain_program(); + for hasher in [HasherKind::Test, HasherKind::Rpx] { + let off = ChipSet::for_program_under(&trivial, hasher, false); + let on = ChipSet::for_program_under(&trivial, hasher, true); + assert!( + !off.keccak && !off.blake3, + "the trivial program uses no family" + ); + assert!(off.bitwise, "the opt-out keeps BITWISE"); + assert!( + !on.bitwise, + "the default drops it from a program that sends it nothing" + ); + assert_eq!((on.keccak, on.blake3), (off.keccak, off.blake3)); + assert!(ChipSet::for_program_under(&chain, hasher, true).bitwise); + } + assert!(ChipSet::for_program_under(&trivial, HasherKind::Blake3, true).bitwise); + // The process mask is the same rule at the process's own setting. + assert_eq!( + ChipSet::for_program_with_hasher(&trivial, HasherKind::Rpx), + ChipSet::for_program_under(&trivial, HasherKind::Rpx, !keep_bitwise()) + ); +} + +/// The census describes the machine the mask proves: with `BITWISE` dropped it +/// has no `BITWISE` row, one entry fewer, and still one entry per sub-proof. +#[test] +fn the_census_follows_the_mask() { + let program = trivial_program(); + let hasher = HasherKind::Rpx; + let kept = ChipSet::for_program_under(&program, hasher, false); + let dropped = ChipSet::for_program_under(&program, hasher, true); + let with = lfm_chip_census_masked(&program, hasher, kept); + let without = lfm_chip_census_masked(&program, hasher, dropped); + let bitwise = LFM_CHIP_NAMES[BITWISE_SLOT]; + assert_eq!(with.iter().filter(|c| c.name == bitwise).count(), 1); + assert_eq!(without.iter().filter(|c| c.name == bitwise).count(), 0); + assert_eq!(without.len() + 1, with.len()); + let blake3 = |m: ChipSet| m.blake3_chunks(program.blake3_chunk_count()); + assert_eq!(with.len(), kept.num_airs(0, blake3(kept))); + assert_eq!(without.len(), dropped.num_airs(0, blake3(dropped))); + let cells = |census: &[super::airs::LfmChipCells]| { + census + .iter() + .map(|c| c.main_cells() + 3 * c.aux_cells()) + .sum::() + }; + let bitwise_cells = with + .iter() + .filter(|c| c.name == bitwise) + .map(|c| c.main_cells() + 3 * c.aux_cells()) + .sum::(); + assert_eq!(cells(&with) - cells(&without), bitwise_cells); +} + +/// A trace set whose builder dropped `BITWISE` pairs the all-zero table when the +/// artifacts being proved keep it — exactly the builder's own table for a +/// program that sends it nothing — so which programs carry `BITWISE` is decided +/// by the artifacts alone and proving never reads the process setting. +#[test] +fn artifacts_that_keep_bitwise_pair_the_all_zero_table() { + let program = trivial_program(); + let hasher = HasherKind::Rpx; + let exec = + super::executor::execute(&program, &trivial_arenas(), &hasher).expect("the program runs"); + let mut traces = super::trace::build_traces_with_hasher(&program, &exec.records, hasher); + if !keep_bitwise() { + assert_eq!( + traces.bitwise.num_rows(), + 0, + "the builder skips the table for a program that sends it nothing" + ); + } + let built = build_artifacts_with_hasher(&program, &options(), hasher); + let kept = under_mask( + &built, + ChipSet { + bitwise: true, + ..built.chip_set + }, + ); + let airs = LfmAirs::for_artifacts(&kept, &options()).expect("the AIR set"); + let bitwise = LFM_CHIP_NAMES[BITWISE_SLOT]; + let paired: Vec = airs + .air_trace_pairs(&mut traces) + .iter() + .map(|(air, _, _)| air.name().to_string()) + .collect(); + assert_eq!( + paired + .iter() + .filter(|name| name.as_str() == bitwise) + .count(), + 1, + "the artifacts keep BITWISE, so the proof carries it" + ); + assert!( + traces.bitwise.main_table.row_major_data() + == crate::tables::bitwise::generate_bitwise_trace() + .main_table + .row_major_data(), + "the paired table is the all-zero BITWISE" + ); +} + +// ============================ proving (box) ================================ + +fn trivial_arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +fn keccak_arenas() -> Vec> { + let state: [u64; 25] = core::array::from_fn(|i| 7u64.wrapping_mul(i as u64 + 1) ^ 0x5A5A); + vec![super::keccak_adapter::state_to_words(&state).to_vec()] +} + +/// `artifacts` under another mask: the same committed roots and heights, the +/// program id re-derived over the new mask — exactly what the artifact build +/// produces for that mask, since the mask moves no root. +fn under_mask(artifacts: &LfmArtifacts, chip_set: ChipSet) -> LfmArtifacts { + let mut out = artifacts.clone(); + out.chip_set = chip_set; + out.program_id = lfm_program_id_chunked( + &artifacts.roots, + &artifacts.log_heights, + artifacts.keccak_rnd_chunks, + artifacts.hasher, + chip_set, + &artifacts.blake3_chunk_roots, + &artifacts.blake3_chunk_log_heights, + &artifacts.hash_chunk_roots, + &artifacts.hash_chunk_log_heights, + ); + out +} + +/// ★ The drop end to end on a program with no byte lookups: it proves and +/// verifies with `BITWISE` dropped, carries one sub-proof fewer, is a different +/// program identity, and neither proof verifies as the other program. Both arms +/// prove in one process whatever its setting: the arm that keeps `BITWISE` +/// gets the all-zero table at pairing if the trace builder dropped it. +#[test] +fn a_program_without_byte_lookups_proves_and_verifies_without_bitwise() { + let opts = options(); + let program = trivial_program(); + let full = build_artifacts(&program, &opts); + let mask = ChipSet::for_program_under(&program, full.hasher, true); + assert!(!mask.bitwise, "the trivial program sends BITWISE nothing"); + let full = under_mask( + &full, + ChipSet { + bitwise: true, + ..mask + }, + ); + let r1 = under_mask(&full, mask); + assert_ne!(r1.program_id, full.program_id, "the mask is bound"); + + let with = lfm_prove(&program, &full, &trivial_arenas(), &opts).expect("proves with BITWISE"); + let without = lfm_prove(&program, &r1, &trivial_arenas(), &opts).expect("proves without it"); + assert_eq!(with.public_words, without.public_words, "same statement"); + assert_eq!( + MultiProofView::Owned(&without.proof).len() + 1, + MultiProofView::Owned(&with.proof).len(), + "one sub-proof fewer" + ); + assert!(verify_against_artifacts( + &full, + &with.proof, + &with.public_words, + &opts + )); + assert!(verify_against_artifacts( + &r1, + &without.proof, + &without.public_words, + &opts + )); + assert!( + !verify_against_artifacts(&full, &without.proof, &without.public_words, &opts), + "a proof without BITWISE is not a proof of the program that keeps it" + ); + assert!( + !verify_against_artifacts(&r1, &with.proof, &with.public_words, &opts), + "and the program that dropped it accepts no proof carrying it" + ); +} + +/// ★ The other direction, which is the one that matters: a mask that drops +/// `BITWISE` from a program whose chips DO send it lookups is refused, and so is +/// every proof against it — the keccak family's byte lookups would otherwise +/// have no receiver. +#[test] +fn dropping_bitwise_under_a_byte_lookup_family_is_refused() { + let opts = options(); + let program = keccak_chain_program(); + let full = build_artifacts(&program, &opts); + assert!(full.chip_set.keccak && full.chip_set.bitwise); + let forged = under_mask( + &full, + ChipSet { + bitwise: false, + ..full.chip_set + }, + ); + assert!(forged.chip_set.bitwise_required(forged.hasher)); + + let honest = lfm_prove(&program, &full, &keccak_arenas(), &opts).expect("proves"); + assert!(verify_against_artifacts( + &full, + &honest.proof, + &honest.public_words, + &opts + )); + assert!( + !verify_against_artifacts(&forged, &honest.proof, &honest.public_words, &opts), + "an honest proof does not verify against a mask that drops a needed BITWISE" + ); + // A proof made under the forged mask — the keccak lookups sent, their + // receiver absent — is refused too. + if let Ok(dropped) = lfm_prove(&program, &forged, &keccak_arenas(), &opts) { + assert!( + !verify_against_artifacts(&forged, &dropped.proof, &dropped.public_words, &opts), + "a proof whose byte lookups have no receiver must not verify" + ); + } +} + +// ========================== the LFM_HASH split ============================== +// +// `LFM_HASH` split into two power-of-two instances when one table would be +// mostly padding (`chunking::HashChunking`). + +/// The rule on the block's own heights (`thoughts/zf/gap/W2-RECURSION.md`, +/// `CENSUS.md`): split where one table would pad at least `2^15` rows, never +/// where the split saves nothing. +#[test] +fn the_hash_split_rule_on_the_blocks_heights() { + for (rows, want) in [ + (293_778, Some(1usize << 18)), // the typical STARK wrap: 294,912 for 524,288 + (299_131, Some(1 << 18)), // STARK wrap 5 + (358_634, Some(1 << 18)), // STARK wrap 6: 393,216 for 524,288 + (295_573, Some(1 << 18)), // STARK L2 node + (173_628, Some(1 << 17)), // WHIR wrap 0: 196,608 for 262,144 + (412_149, None), // STARK L1 node: 2^18 + 2^18 is one table + (224_938, None), // 2^17 + 2^17 is one table + (1 << 18, None), // already full + (40_000, None), // would save 24,576 < 2^15 rows + (70_000, Some(1 << 16)), // saves 57,344 rows + (0, None), + (1, None), + (3, None), + ] { + let got = HashChunking::for_rows(rows); + let want = want.map_or(HashChunking::unbounded(), HashChunking::split_at); + assert_eq!(got, want, "{rows} hash rows"); + } +} + +/// The split is a per-pipeline default ([`super::chunking::HASH_SPLIT_DEFAULT`]) +/// that `LAMBDA_VM_LFM_HASH_SPLIT` overrides either way; the process policy is +/// the rule at the process's setting. Nothing here states the default's value, +/// which differs between the two pipelines' branches. +#[test] +fn the_hash_split_setting_reads_the_pipeline_default_and_the_override() { + use super::chunking::{ + HASH_SPLIT_DEFAULT, HASH_SPLIT_ENV, hash_split_enabled, hash_split_setting, + }; + assert_eq!(HASH_SPLIT_ENV, "LAMBDA_VM_LFM_HASH_SPLIT"); + assert_eq!( + hash_split_setting(None), + HASH_SPLIT_DEFAULT, + "unset is the default" + ); + assert_eq!( + hash_split_setting(Some("")), + HASH_SPLIT_DEFAULT, + "empty is unset" + ); + assert!(!hash_split_setting(Some("0")), "0 keeps one table"); + assert!(hash_split_setting(Some("1")), "1 splits by the rule"); + let rows = 293_778; + let want = if hash_split_enabled() { + HashChunking::for_rows(rows) + } else { + HashChunking::unbounded() + }; + assert_eq!(HashChunking::for_process(rows), want); +} + +/// A typo stops the run rather than proving the default under the other arm's +/// name. +#[test] +#[should_panic(expected = "must be 0 or 1")] +fn a_malformed_hash_split_setting_stops_the_run() { + let _ = super::chunking::hash_split_setting(Some("on")); +} + +/// `chunk_count` and `chunk_range` are one rule: the ranges are disjoint, in +/// order, and cover every row exactly once. +#[test] +fn the_hash_chunk_ranges_cover_every_row_once() { + for policy in [ + HashChunking::unbounded(), + HashChunking::split_at(1), + HashChunking::split_at(4), + HashChunking::split_at(1 << 18), + ] { + for n in [0usize, 1, 3, 4, 5, 9, (1 << 18) + 31_634] { + let count = policy.chunk_count(n); + assert!(count == 1 || count == 2, "{policy:?} n={n}"); + let mut next = 0; + for c in 0..count { + let r = policy.chunk_range(n, c); + assert_eq!(r.start, next, "{policy:?} n={n} chunk {c}"); + next = r.end; + } + assert_eq!(next, n, "{policy:?} n={n}: every row in one chunk"); + assert!(policy.chunk_range(n, count).is_empty()); + } + } +} + +/// An unsplit `LFM_HASH` keeps the digest it always had; a split one is a +/// different program by name. +#[test] +fn an_unsplit_hash_keeps_the_program_digest() { + let program = trivial_program(); + let artifacts = build_artifacts(&program, &options()); + assert_eq!(artifacts.hash_chunk_roots, vec![artifacts.roots[HASH_SLOT]]); + let a = &artifacts; + let digest = |roots: &[Commitment], heights: &[u8]| { + lfm_program_id_chunked( + &a.roots, + &a.log_heights, + a.keccak_rnd_chunks, + a.hasher, + a.chip_set, + &a.blake3_chunk_roots, + &a.blake3_chunk_log_heights, + roots, + heights, + ) + }; + let legacy = lfm_program_id( + &a.roots, + &a.log_heights, + a.keccak_rnd_chunks, + a.hasher, + a.chip_set, + &a.blake3_chunk_roots, + &a.blake3_chunk_log_heights, + ); + assert_eq!(a.program_id, legacy); + assert_eq!(digest(&[], &[]), legacy); + assert_eq!( + digest(&a.hash_chunk_roots, &a.hash_chunk_log_heights), + legacy + ); + let tail = [a.roots[HASH_SLOT], [7u8; 32]]; + assert_ne!(digest(&tail, &[2, 2]), legacy, "a tail moves the digest"); + assert_ne!( + digest(&tail, &[2, 2]), + digest(&tail, &[2, 3]), + "and the tail's heights are bound" + ); +} + +/// A split program describes two `LFM_HASH` tables everywhere its shape is read: +/// the chunk groups, the census, and one more sub-proof. +#[test] +fn a_split_program_describes_two_hash_tables() { + let whole = trivial_program(); + assert_eq!( + whole.groups.hash.real_rows, 3, + "the fixture's three compressions" + ); + assert_eq!(whole.hash_chunk_count(), 1); + let split = trivial_program().with_hash_chunking(HashChunking::split_at(2)); + assert_eq!(split.hash_chunk_count(), 2); + assert_eq!(split.hash_chunk_real_rows(), vec![2, 1]); + let (g0, g1) = (split.hash_chunk_group(0), split.hash_chunk_group(1)); + let g = &split.groups.hash; + assert_eq!(g0.data[..2 * g.width], g.data[..2 * g.width]); + assert_eq!(g1.data[..g.width], g.data[2 * g.width..3 * g.width]); + assert!(g0.data[2 * g.width..].iter().all(|v| *v == FE::zero())); + assert!(g1.data[g.width..].iter().all(|v| *v == FE::zero())); + + let hasher = HasherKind::Rpx; + let mask = ChipSet::for_program_under(&split, hasher, false); + let one = lfm_chip_census_masked(&whole, hasher, mask); + let two = lfm_chip_census_masked(&split, hasher, mask); + let hash = LFM_CHIP_NAMES[HASH_SLOT]; + assert_eq!(one.iter().filter(|c| c.name == hash).count(), 1); + let rows: Vec<(u64, u64)> = two + .iter() + .filter(|c| c.name == hash) + .map(|c| (c.real_rows, c.rows)) + .collect(); + assert_eq!(rows, vec![(2, 4), (1, 4)]); + assert_eq!(two.len(), one.len() + 1); +} + +// ============================ proving (box) ================================ + +/// ★ The split end to end on the host: a split program proves and verifies, carries +/// one sub-proof more, is a different program identity, and neither proof +/// verifies as the other program. A forged tail root, and the single-table +/// door, reject the split proof. +#[test] +fn a_split_hash_proves_and_verifies_as_its_own_program() { + let opts = options(); + let hasher = HasherKind::Rpx; + let whole = trivial_program(); + let split = trivial_program().with_hash_chunking(HashChunking::split_at(2)); + let a1 = build_artifacts_with_hasher(&whole, &opts, hasher); + let a2 = build_artifacts_with_hasher(&split, &opts, hasher); + assert_eq!(a1.hash_chunk_roots.len(), 1); + assert_eq!(a2.hash_chunk_roots.len(), 2); + assert_eq!(a2.hash_chunk_roots[0], a2.roots[HASH_SLOT]); + assert_ne!(a1.program_id, a2.program_id); + + let p1 = lfm_prove_with_hasher(&whole, &a1, &trivial_arenas(), &opts, hasher).expect("whole"); + let p2 = lfm_prove_with_hasher(&split, &a2, &trivial_arenas(), &opts, hasher).expect("split"); + assert_eq!(p1.public_words, p2.public_words, "same statement"); + assert_eq!( + MultiProofView::Owned(&p2.proof).len(), + MultiProofView::Owned(&p1.proof).len() + 1, + "one sub-proof more" + ); + assert!(verify_against_artifacts( + &a1, + &p1.proof, + &p1.public_words, + &opts + )); + assert!(verify_against_artifacts( + &a2, + &p2.proof, + &p2.public_words, + &opts + )); + assert!(!verify_against_artifacts( + &a1, + &p2.proof, + &p2.public_words, + &opts + )); + assert!(!verify_against_artifacts( + &a2, + &p1.proof, + &p1.public_words, + &opts + )); + + let mut forged = a2.clone(); + forged.hash_chunk_roots[1] = forged.hash_chunk_roots[0]; + assert!( + !verify_against_artifacts(&forged, &p2.proof, &p2.public_words, &opts), + "chunk 1 is checked against its own root" + ); + assert!( + !verify_against( + &a2.roots, + &a2.program_id, + a2.keccak_rnd_chunks, + &p2.proof, + &p2.public_words, + &opts, + a2.hasher, + a2.chip_set, + ), + "the single-table door rejects a split proof on the AIR count" + ); +} + +/// ★ The in-guest side, which is what a node runs: an emitted verify leg +/// accepts a child whose `LFM_HASH` is split AND whose mask drops `BITWISE`, +/// and rejects the same proof when the leg is emitted with chunk 1's root +/// replaced — the root Phase A absorbs is the chunk's own. +#[test] +fn an_emitted_leg_verifies_a_split_child_without_bitwise() { + use super::builder::LfmBuilder; + use super::per_table_aggregator::{declare_leg_arenas, emit_leg}; + use super::per_table_aggregator_tests::{child_arena_words, child_shape, real_child}; + + let opts = super::proof::aggregation_wrap_options(); + let hasher = crate::hash_pin::BLOCK_HASHER; + let program = trivial_program().with_hash_chunking(HashChunking::split_at(2)); + let built = build_artifacts_with_hasher(&program, &opts, hasher); + let artifacts = under_mask( + &built, + ChipSet { + bitwise: false, + ..built.chip_set + }, + ); + assert!(!artifacts.chip_set.bitwise_required(hasher)); + let proved = lfm_prove_with_hasher(&program, &artifacts, &trivial_arenas(), &opts, hasher) + .expect("the split, BITWISE-free child proves"); + let child = real_child(artifacts, opts.clone(), &proved); + let words = child_arena_words(&child); + + let leg_program = |shape: &super::per_table_aggregator::ChildShape<'_>| { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arenas = declare_leg_arenas(&mut b, shape); + let _ = emit_leg(&mut b, shape, &arenas); + super::compiler::compile(b.finish()) + }; + + let shape = child_shape(&child); + let honest = leg_program(&shape); + assert!( + super::executor::execute(&honest, &words, &hasher).is_ok(), + "the leg must accept the split child" + ); + + // The same proof, the leg emitted against chunk 0's root in chunk 1's place. + let hash = LFM_CHIP_NAMES[HASH_SLOT]; + let airs = super::airs::LfmAirs::for_artifacts(&child.artifacts, &opts).expect("the AIR set"); + let tail = airs + .air_refs() + .iter() + .enumerate() + .filter(|(_, air)| air.name() == hash) + .map(|(i, _)| i) + .nth(1) + .expect("a split child has two LFM_HASH sub-proofs"); + let wrong = child.artifacts.hash_chunk_roots[0]; + let mut forged = child_shape(&child); + forged.tables[tail].precomputed_root = Some(&wrong); + assert!( + super::executor::execute(&leg_program(&forged), &words, &hasher).is_err(), + "a leg bound to the wrong chunk root must reject the proof" + ); +} diff --git a/prover/src/lfm/programs.rs b/prover/src/lfm/programs.rs new file mode 100644 index 000000000..2f9c0edb6 --- /dev/null +++ b/prover/src/lfm/programs.rs @@ -0,0 +1,1453 @@ +//! Registered LFM programs. +//! +//! Every program here is deterministic — same builder calls, same +//! instructions, same column groups, same digest — which is what lets the +//! registry pin it and the drift tests recompute it on every PR. Arena +//! *values* vary per proof; the program (and its identity) never does. +//! +//! # The wrap hash is named at every constructor that has one +//! +//! `WrapHash::default()` is `Keccak` and stays that way: it is the *unset* +//! value, not the production one. Letting production be inherited from a +//! `derive` three files away is the shape that made the P-a flip a twenty-site +//! audit (PA-PLAN §6.0, finding rev-emit E3), so every constructor here that +//! emits a hash says which one, and the exceptions read as exceptions rather +//! than as omissions: +//! +//! - [`keccak_chain_program_source`], [`keccak_sponge_program_source`] and +//! [`keccak_sample_program_source`] are instruments ABOUT keccak (R1b/R1c/R1d). +//! A BLAKE3 twin would be a new program and a new registry row, never a +//! re-blessing of one of these. +//! - ★ [`program_id_program_source`] mirrors `recursion::program_id_from_digest`, +//! which names `PlatformKeccak256`. It is the attestation join: it must +//! compute what the host computes, and the host's program-identity digest is +//! keccak whatever the commitment hash is. Staying keccak is the binding, not +//! an oversight to be tidied later. +//! +//! Constructors that emit no hash name nothing, because a hash they never +//! compute is not a property they have: `trivial`, `fri_toy`, `lde_probe`, +//! `l2g_binding` (word equality only), and `permute_coverage` (which drives the +//! `LFM_HASH` socket — a different axis entirely). + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Cell, LfmBuilder, LfmProgramSource}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::WrapHash; + +/// The Milestone-B trivial program: a few hundred instructions exercising +/// every chip — constants, base ALU (incl. the assert lowering), Fp3 ALU, +/// bit decomposition, selects driven by decomposed bits, a chain of hash +/// compressions, hints and public output. +/// +/// ## It contains no `permute`, deliberately +/// +/// It used to end on a raw `b.permute`, which made it unprovable under the +/// machine's real hash — a REGISTERED program whose cryptographic meaning +/// depended on a placeholder permutation, which is the disclosure this whole +/// effort exists to retire. The permutation is now a third `compress`: every +/// registry entry is provable under every hasher, and the swap is marginally +/// cheaper besides. +/// +/// Permute mode did not disappear with it — `Test` and `Poseidon` still +/// implement it, and it still needs coverage or the arms rot. But coverage does +/// not need a registry ENTRY: [`permute_coverage_program_source`] exercises the +/// arms without claiming a program identity. +pub fn trivial_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + + let arena = b.declare_arena(4); + let h: Vec = (0..4).map(|i| b.hint_word(arena, i)).collect(); + + // Base-field leg: s = 16, m = 112, q = m/s = 7; assert q == x. + let x = b.felt_const(FE::from(7u64)); + let y = b.felt_const(FE::from(9u64)); + let s = b.add(x, y); + let m = b.mul(s, x); + let q = b.div(m, s); + b.assert_eq(q, x); + + // Fp3 leg: product, Horner step, base scaling. + let e1 = b.ext_const(&FEE::new([FE::from(1u64), FE::from(2u64), FE::from(3u64)])); + let e2 = b.ext_const(&FEE::new([FE::from(4u64), FE::from(5u64), FE::from(6u64)])); + let p = b.emul(e1, e2); + let pm = b.emul_add(p, e1, e2); + let _pb = b.emul_base(pm, q); + + // Bit-decomposition leg: m = 112 = 0b1110000; bits drive the selects. + let bits = b.bit_dec(m, 8); + let (l, _r) = b.select(bits[4], h[0], h[1]); // bit 4 of 112 = 1 → swap + let (l2, _r2) = b.select(bits[0], l, h[2]); // bit 0 = 0 → pass through + + // Hash leg: three compressions chained through memory. Feeding `d1` back in + // is the point — a socket's own output must be a legal input to the next + // one, which is what a Merkle walk does at every level. + // + // ✓ SWEPT for the leaf-mode migration and deliberately LEFT as `compress`: + // these are the only place in any registered program where raw arena data + // enters a compress, and they form a CHAIN, not a tree. There is no leaf and + // no parent here, so there is no leaf/parent confusion for `MODE_L` to + // separate — what the mode buys elsewhere it would not buy here. The + // consequence to keep in mind is that this program's arena words must be + // `u32`-laned under BLAKE3 (obligation O1), which its tests supply; data + // that cannot be is what `leaf` exists for. + let d0 = b.compress(h[0].as_digest(), h[1].as_digest()); + let d1 = b.compress(d0, l2.as_digest()); + let d2 = b.compress(d1, h[3].as_digest()); + + // Public output: two chained digests and one ALU result. + b.public(d1.as_cell()); + b.public(d2.as_cell()); + b.public(m.as_cell()); + + b.finish() +} + +pub fn trivial_program() -> LfmProgram { + compile(trivial_program_source()) +} + +/// A `permute`-mode fixture — **not a registry entry, and it must not become +/// one.** +/// +/// [`trivial_program_source`] gave up its raw `b.permute` so that every +/// registered program runs under the machine's real hash. Permute mode is still +/// live under `Test` and `Poseidon`, so it still needs a program that exercises +/// the executor arm, the trace filler and the AIR's three-cell tuple contract — +/// this is that program. It is deliberately unregistered: a registry entry is a +/// claim about a program's identity, and this one exists only to keep two +/// hashers' arms honest. +/// +/// It is unprovable under BLAKE3 by design (`MODE_P = 0`), which is itself worth +/// testing. +#[cfg(test)] +pub fn permute_coverage_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(3); + let h: Vec = (0..3).map(|i| b.hint_word(arena, i)).collect(); + + // Two permutations chained, so an output cell is also an input cell. + let s0 = b.permute([h[0], h[1], h[2]]); + let s1 = b.permute([s0[2], s0[0], s0[1]]); + + for c in s1 { + b.public(c); + } + b.finish() +} + +#[cfg(test)] +pub fn permute_coverage_program() -> LfmProgram { + compile(permute_coverage_program_source()) +} + +/// Number of arena words the keccak-chain program ingests: one full state. +pub const KECCAK_CHAIN_ARENA_WORDS: u32 = super::layout::keccak::NUM_WORDS as u32; + +/// The R1b keccak program: a hint-fed state pushed through two *chained* +/// `keccak-f[1600]` permutations, the second consuming the first's output words +/// directly out of memory. +/// +/// Chaining is the point. It proves the `u32`-half word convention round-trips: +/// the output words `LFM_KECCAK` writes are immediately legal input words, so +/// the halves it produces are canonical `u32`s and the state's two unused top +/// lanes come back zero — no repacking instruction in between. +pub fn keccak_chain_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Keccak); + + let arena = b.declare_arena(KECCAK_CHAIN_ARENA_WORDS); + let state: [Cell; 13] = core::array::from_fn(|i| b.hint_word(arena, i as u32)); + + let once = b.keccak_f(state); + let twice = b.keccak_f(once); + + // Expose enough to pin both permutations: the intermediate state's first + // word and the final state's first two. + b.public(once[0]); + b.public(twice[0]); + b.public(twice[1]); + + b.finish() +} + +pub fn keccak_chain_program() -> LfmProgram { + compile(keccak_chain_program_source()) +} + +/// Message length of the registered `KeccakSpongeV0`. +/// +/// 202 bytes is chosen to exercise all three shapes at once: it crosses the +/// 136-byte rate boundary (2 blocks), it is not a multiple of the rate (so the +/// padding is not a whole block), and `202 % 4 == 2` puts the `0x01` pad byte in +/// the same `u32` half as the message's last two bytes — the mixed-half case the +/// emitter handles by adding a padding constant to the stream half. +pub const KECCAK_SPONGE_LEN: usize = 202; + +/// `keccak256` over a hint-supplied byte stream of exactly `len_bytes`, with +/// the 32-byte digest as public output. +/// +/// Length is program shape, not data: a straight-line machine has no loops, so +/// each length compiles to its own program and its own identity. +pub fn keccak_sponge_program_source(len_bytes: usize) -> LfmProgramSource { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Keccak); + let num_halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec<_> = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let digest = super::edsl::keccak256(&mut b, &stream, len_bytes); + b.public(digest[0]); + b.public(digest[1]); + b.finish() +} + +pub fn keccak_sponge_program(len_bytes: usize) -> LfmProgram { + compile(keccak_sponge_program_source(len_bytes)) +} + +/// `Blake3Chain` over a hint-supplied byte stream of exactly `len_bytes`, with +/// the 32-byte digest as public output — the BLAKE3 twin of +/// [`keccak_sponge_program_source`], and the smallest program that exercises +/// `LFM_BLAKE3` end to end. +/// +/// The one place in this module that SELECTS the configured hash rather than +/// inheriting the default. Deliberately **not registered**: it is an +/// instrument, and adding registry rows is its own decision — `resolve` keys on +/// `(kind, blowup_factor)` alone, so rows are not a free-form extension point. +pub fn blake3_sponge_program_source(len_bytes: usize) -> LfmProgramSource { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let num_halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec<_> = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + // The builder is pinned to BLAKE3 above, so this names the byte hash + // directly — which a program that is ABOUT a hash should do anyway. + let digest = super::edsl::wrap_hash_bytes( + &mut b, + super::edsl::ByteWrapHash::Blake3, + &stream, + len_bytes, + ); + b.public(digest[0]); + b.public(digest[1]); + b.finish() +} + +pub fn blake3_sponge_program(len_bytes: usize) -> LfmProgram { + compile(blake3_sponge_program_source(len_bytes)) +} + +/// `DefaultTranscript::sample()` over a hint-supplied stream: keccak256 of the +/// absorbed bytes, then the 32 digest bytes REVERSED — which is both the +/// challenge the transcript returns and the prefix it re-absorbs. +/// +/// This is the R1d groundwork that is independent of the #841 revision: +/// `sample()` itself is unchanged between them. +pub fn keccak_sample_program_source(len_bytes: usize) -> LfmProgramSource { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Keccak); + let num_halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec<_> = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let rev = super::edsl::keccak256_rev(&mut b, &stream, len_bytes); + b.public(rev[0]); + b.public(rev[1]); + b.finish() +} + +pub fn keccak_sample_program(len_bytes: usize) -> LfmProgram { + compile(keccak_sample_program_source(len_bytes)) +} + +// ==================== R1d: the transcript replay ==================== + +/// Seed the registered transcript-replay program starts from — a program +/// constant, exactly as a domain separator would be. 24 bytes, so the segment +/// stays half-aligned for the machine-supplied absorbs that follow. +pub const TRANSCRIPT_SEED: &[u8] = b"lfm-transcript-replay-v0"; + +/// First absorb: 32 bytes, the shape a commitment root arrives in. +pub const TRANSCRIPT_ABSORB_A: usize = 32; + +/// Second absorb: one full keccak rate, chosen so the segment it lands in +/// (32 reversed-digest bytes + 136) needs TWO rate blocks — the multi-block +/// path inside a replay, which no earlier test reaches. +pub const TRANSCRIPT_ABSORB_B: usize = 136; + +/// Arena words the replay program ingests: both absorbs as `u32` halves. +pub const TRANSCRIPT_ARENA_HALVES: u32 = + ((TRANSCRIPT_ABSORB_A + TRANSCRIPT_ABSORB_B) / super::keccak_host::BYTES_PER_HALF) as u32; + +/// Index bits the replay program's `sample_u64` draw asks for. +pub const TRANSCRIPT_QUERY_BITS: usize = 20; + +/// The R1d headline program: a scripted `DefaultTranscript` interleaving, +/// replayed in the machine, with every sampled value published. +/// +/// The script is chosen so the emitter's bookkeeping is load-bearing at every +/// step. Buffer positions, in bytes, as the emitter tracks them: +/// +/// | step | before | after | squeeze | +/// |-----------------------|--------|-------|---------| +/// | `append` A (32 B) | 32 | 32 | — | +/// | `sample_felt` | 32 | 8 | **#1** | +/// | `sample_felt` | 8 | 16 | — | +/// | `sample_ext` (3 draws)| 16 | 8 | **#2** | +/// | `append` B (136 B) | 8 | 32 | — | +/// | `sample_u64_pow2` | 32 | 8 | **#3** | +/// | `sample_felt` | 8 | 16 | — | +/// | `sample()` | 16 | 32 | **#4** | +/// | `sample_felt` | 32 | 8 | **#5** | +/// +/// So it exercises: a refill in the MIDDLE of an extension draw (squeeze #2 +/// lands between coordinates 1 and 2), an absorb that invalidates a buffer with +/// 24 bytes still in it, a raw `sample()` that invalidates with 16 bytes still +/// in it, a two-block segment (squeeze #3), and both draw kinds. Get any of the +/// invalidation rules wrong and the values diverge from the real transcript. +/// ★ The hash the R1d replay is ABOUT, named once so a count over it cannot +/// drift from the program it counts. +/// +/// `WrapHash::production` is deliberately NOT read here — its own doc carves out +/// the instruments that name their hash directly, and this is one of them. The +/// script is a BYTE sponge: the table above is denominated in 32- and 136-byte +/// segments and in which rate block each squeeze lands in, none of which is a +/// statement an algebraic sponge can be right or wrong about. Exporting it is +/// what lets `machine_tests::transcript_replay_cell_counts` count against the +/// same hash the builder used instead of against the pin, which read zero. +pub const TRANSCRIPT_REPLAY_WRAP_HASH: WrapHash = WrapHash::Blake3; + +pub fn transcript_replay_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::edsl::bits_to_felt; + use super::transcript_replay::TranscriptReplay; + + let halves_a = TRANSCRIPT_ABSORB_A / super::keccak_host::BYTES_PER_HALF; + + let mut b = LfmBuilder::new().with_wrap_hash(TRANSCRIPT_REPLAY_WRAP_HASH); + let arena = b.declare_arena(TRANSCRIPT_ARENA_HALVES); + let halves: Vec = (0..TRANSCRIPT_ARENA_HALVES) + .map(|i| b.hint_felt(arena, i)) + .collect(); + let (absorb_a, absorb_b) = halves.split_at(halves_a); + + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_halves(absorb_a); + let f0 = t.sample_felt(&mut b); + let f1 = t.sample_felt(&mut b); + let e = t.sample_ext(&mut b); + t.append_halves(absorb_b); + let q = t.sample_u64_pow2(&mut b, TRANSCRIPT_QUERY_BITS); + let qf = bits_to_felt(&mut b, &q); + let f2 = t.sample_felt(&mut b); + let s = t.sample(&mut b); + let f3 = t.sample_felt(&mut b); + + b.public(f0.as_cell()); + b.public(f1.as_cell()); + b.public(e.as_cell()); + b.public(qf.as_cell()); + b.public(f2.as_cell()); + b.public(s[0]); + b.public(s[1]); + b.public(f3.as_cell()); + b.finish() +} + +pub fn transcript_replay_program() -> LfmProgram { + compile(transcript_replay_program_source()) +} + +/// Absorbs a machine-COMPUTED keccak digest and samples one challenge from it — +/// the shape a commitment root takes in a real verifier, and the only path that +/// exercises `append_digest`'s word-to-halves byte order. +/// +/// Not registered: it exists to pin that byte order against the real transcript, +/// which execution alone establishes (the executor computes the digest FROM the +/// unpacked halves, so a wrong order moves the sampled value). +pub fn transcript_absorb_digest_program_source(len_bytes: usize) -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let num_halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let digest = super::edsl::keccak256(&mut b, &stream, len_bytes); + + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_digest(&mut b, &digest); + let f = t.sample_felt(&mut b); + b.public(f.as_cell()); + b.finish() +} + +pub fn transcript_absorb_digest_program(len_bytes: usize) -> LfmProgram { + compile(transcript_absorb_digest_program_source(len_bytes)) +} + +// ============ R1e slice a: field elements on the wire (big-endian) ============ + +/// Absorbs one hint-supplied BASE field element the way `append_field_element` +/// streams it (canonical `u64`, 8 bytes big-endian) and returns the raw squeeze. +/// +/// Publishing `sample()` rather than a sampled challenge is deliberate: the test +/// then compares the 32 squeezed bytes directly, so a failure means the ABSORBED +/// BYTES are wrong and nothing else. Not registered — proved through +/// `verify_against`, like the per-length keccak programs. +pub fn append_felt_program_source() -> LfmProgramSource { + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let arena = b.declare_arena(1); + let v = b.hint_felt(arena, 0); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_felt(&mut b, v); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn append_felt_program() -> LfmProgram { + compile(append_felt_program_source()) +} + +/// The same for one CUBIC-EXTENSION element: coordinates 0, 1, 2, each 8 bytes +/// big-endian, 24 bytes total. +pub fn append_ext_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let arena = b.declare_arena(3); + let coords: [Felt; 3] = core::array::from_fn(|i| b.hint_felt(arena, i as u32)); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_ext(&mut b, coords); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn append_ext_program() -> LfmProgram { + compile(append_ext_program_source()) +} + +// ==================== R1e slice b: the byte-level splice ==================== + +/// Deterministic constant bytes for the splice programs; the tests build the +/// host reference from the same function, so the two cannot drift apart. +pub fn splice_prefix(len: usize) -> Vec { + (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect() +} + +/// Deterministic machine-supplied bytes for the splice programs. +pub fn splice_dynamic(len: usize) -> Vec { + (0..len) + .map(|i| (i as u8).wrapping_mul(53).wrapping_add(29)) + .collect() +} + +/// A constant prefix of `prefix_len` bytes followed by `num_halves` hinted +/// machine halves, then a raw squeeze. The shift under test is +/// `prefix_len % 4`; at 0 it takes the aligned fast path and serves as control. +pub fn splice_program_source(prefix_len: usize, num_halves: u32) -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let arena = b.declare_arena(num_halves); + let halves: Vec = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let mut t = TranscriptReplay::new(&splice_prefix(prefix_len)); + t.append_halves_misaligned(&halves); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn splice_program(prefix_len: usize, num_halves: u32) -> LfmProgram { + compile(splice_program_source(prefix_len, num_halves)) +} + +/// Tag length of the alternating splice program — the real +/// `LAMBDAVM_CONTINUATION_EPOCH_V5` is exactly this long. (Every suffix this +/// tag has carried is 30 bytes, so the bump does not move this constant.) +pub const SPLICE_ALT_TAG: usize = 30; +pub const SPLICE_ALT_DIGEST_HALVES: u32 = 8; +pub const SPLICE_ALT_FIELD_HALVES: u32 = 2; + +/// The continuation-epoch statement's shape in miniature: alternating constant +/// and dynamic runs, with the shift CHANGING mid-stream. +/// +/// The byte offsets are the whole point. A 30-byte tag leaves shift 2; the +/// 32-byte digest and an 8-byte field keep it there; then a ONE-byte field — +/// standing for the real encoding's `fri_final_poly_log_degree` — moves every +/// later dynamic value to shift 3. A splice that handles only a single fixed +/// shift passes the fixed-prefix test above and fails this one. +pub fn splice_alternating_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let total = SPLICE_ALT_DIGEST_HALVES + 2 * SPLICE_ALT_FIELD_HALVES; + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let arena = b.declare_arena(total); + let h: Vec = (0..total).map(|i| b.hint_felt(arena, i)).collect(); + let d = SPLICE_ALT_DIGEST_HALVES as usize; + let f = SPLICE_ALT_FIELD_HALVES as usize; + + let mut t = TranscriptReplay::new(&splice_prefix(SPLICE_ALT_TAG)); + t.append_halves_misaligned(&h[..d]); + t.append_const_bytes(&splice_prefix(8)); + t.append_halves_misaligned(&h[d..d + f]); + t.append_const_bytes(&splice_prefix(1)); + t.append_halves_misaligned(&h[d + f..]); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn splice_alternating_program() -> LfmProgram { + compile(splice_alternating_program_source()) +} + +// ============ R1e slices c+d: the epoch statement and Phase A ============ + +/// Public-output length of the acceptance shape. Deliberately NOT a multiple of +/// four: an epoch's public output is collected one byte per COMMIT op, so the +/// unaligned case is the general one and the acceptance must exercise it. +pub const STMT_PUBLIC_OUTPUT_LEN: usize = 14; + +/// Whether each of the acceptance shape's sub-proofs is preprocessed. Mixed on +/// purpose: the verifier absorbs a preprocessed commitment only for the airs +/// that have one, so a replay that absorbs unconditionally must diverge. +pub const STMT_PREPROCESSED: [bool; 3] = [true, false, true]; + +/// Halves per 32-byte commitment. +const ROOT_HALVES: u32 = 8; + +/// Arena halves the statement-replay program reads. +pub fn stmt_arena_halves() -> u32 { + let vars = ROOT_HALVES + STMT_PUBLIC_OUTPUT_LEN.div_ceil(4) as u32 + 2; + let roots: u32 = STMT_PREPROCESSED + .iter() + .map(|&p| if p { 2 * ROOT_HALVES } else { ROOT_HALVES }) + .sum(); + vars + roots +} + +/// The acceptance shape's shape-static statement fields. +pub fn epoch_statement_shape() -> super::statement_replay::EpochStatementShape { + super::statement_replay::EpochStatementShape { + public_output_len: STMT_PUBLIC_OUTPUT_LEN, + // Twenty-three: fourteen split families, six accelerators, BLAKE3, IS_WH, + // ECREC. + table_counts: [ + 3, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ], + num_private_input_pages: 2, + fri_final_poly_log_degree: 7, + page_ranges: vec![(0x1000, 4), (0x8000, 1)], + is_final: true, + } +} + +/// The R1e headline program: a continuation-epoch statement bound into the +/// transcript, then Phase A over three sub-proofs, publishing the shared LogUp +/// challenges `z` and `α`. +/// +/// This is the first leg of a real verifier the machine runs end to end — +/// everything a `multi_verify` does before the per-table forks. What `z` and `α` +/// feed into (the bus-balance replay, the chaining obligations) is R1f. +pub fn statement_replay_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::statement_replay::{ + EpochStatementVars, PhaseATable, absorb_epoch_statement, replay_phase_a, + }; + use super::transcript_replay::TranscriptReplay; + + let shape = epoch_statement_shape(); + let total = stmt_arena_halves(); + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let arena = b.declare_arena(total); + let h: Vec = (0..total).map(|i| b.hint_felt(arena, i)).collect(); + + let out_halves = STMT_PUBLIC_OUTPUT_LEN.div_ceil(4); + let (elf, rest) = h.split_at(ROOT_HALVES as usize); + let (public_output, rest) = rest.split_at(out_halves); + let (epoch_label, mut roots) = rest.split_at(2); + + // The verifier seeds an empty transcript and binds the statement first. + let mut t = TranscriptReplay::new(&[]); + absorb_epoch_statement( + &mut t, + &shape, + &EpochStatementVars { + elf_digest: elf, + public_output, + epoch_label, + }, + ); + + let mut tables = Vec::new(); + for &preprocessed in &STMT_PREPROCESSED { + let prep = if preprocessed { + let (p, r) = roots.split_at(ROOT_HALVES as usize); + roots = r; + Some(p) + } else { + None + }; + let (main, r) = roots.split_at(ROOT_HALVES as usize); + roots = r; + tables.push(PhaseATable { + // This driver supplies every root as arena cells on purpose: it is the + // statement/Phase-A differential, and where a root COMES FROM is the + // assembled verifier's decision (ledger entry 7), not this program's. + preprocessed_root: prep.map(super::statement_replay::PhaseAPreprocessed::Cells), + main_root: main, + }); + } + let (z, alpha) = replay_phase_a(&mut t, &mut b, &tables); + + b.public(z.as_cell()); + b.public(alpha.as_cell()); + b.finish() +} + +pub fn statement_replay_program() -> LfmProgram { + compile(statement_replay_program_source()) +} + +/// A harness for the candidate canonicity guard alone: `(lo, hi)` arrive as +/// hinted halves, the guard runs, the recomposed felt is published. +/// +/// Not a sound construction on its own — nothing here range-checks the hinted +/// halves to `u32`, which the derivation in +/// [`super::transcript_replay::assert_canonical`] assumes. In the replay they +/// come from an `Unpack` of a `LFM_KECCAK` output word and the adapter +/// range-checks them. This program exists so the guard's PREDICATE can be +/// exercised at the `p − 2 / p − 1 / p` boundary, which is unreachable through +/// the replay: finding a message whose digest yields an out-of-range candidate +/// means about 2^32 keccaks. +pub fn canonicity_guard_program_source() -> LfmProgramSource { + use super::transcript_replay::{Candidate, assert_canonical, candidate_to_felt}; + + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Blake3); + let arena = b.declare_arena(2); + let c = Candidate { + lo: b.hint_felt(arena, 0), + hi: b.hint_felt(arena, 1), + }; + assert_canonical(&mut b, c); + let v = candidate_to_felt(&mut b, c); + b.public(v.as_cell()); + b.finish() +} + +pub fn canonicity_guard_program() -> LfmProgram { + compile(canonicity_guard_program_source()) +} + +/// The Milestone-C verifier program: verifies the fixture FRI +/// commitment-opening proof (`fixture::fixture_prove`) — sponge transcript +/// replay, two Merkle-authenticated opening sets, the α-combination Horner, +/// two unnormalized folds with index-bit-derived domain-point inverses, and +/// the terminal-polynomial check. Straight-line: every loop below unrolls at +/// emission; the shape is a compile-time constant of the program. +pub fn fri_toy_program_source() -> LfmProgramSource { + use super::builder::Cell; + use super::edsl::{self, SpongeVar}; + use super::fixture::{domain_constants, shape}; + + let (omega, offset) = domain_constants(); + let omega_inv = omega.inv().expect("root of unity is invertible"); + let offset_inv = offset.inv().expect("coset offset is invertible"); + // Fold-0 point inverses over q0's bits: x = c·ω^{q0} ⇒ factors ω^{-2^i}. + let invx_factors: Vec = (0..shape::QUERY_BITS) + .map(|i| omega_inv.pow(1u64 << i)) + .collect(); + // Fold-1 over j = q0 mod 8: y = c²·ω^{2j} ⇒ factors ω^{-2·2^i}, scale c⁻². + let invy_factors: Vec = (0..3).map(|i| omega_inv.pow(2u64 << i)).collect(); + let offset2_inv = offset_inv.square(); + // Terminal point y₂ = c⁴·ω^{4j}. + let y2_factors: Vec = (0..3).map(|i| omega.pow(4u64 << i)).collect(); + let offset4 = offset.square().square(); + + let mut b = LfmBuilder::new(); + let commits = b.declare_arena(4); + let opens = b.declare_arena((shape::NUM_QUERIES * shape::WORDS_PER_QUERY) as u32); + + let mut sponge = SpongeVar::new(&mut b); + let main_root = b.hint_word(commits, 0); + sponge.absorb(&mut b, main_root); + let alpha = sponge.squeeze_ext(&mut b); + let zeta0 = sponge.squeeze_ext(&mut b); + let l1_root = b.hint_word(commits, 1); + sponge.absorb(&mut b, l1_root); + let zeta1 = sponge.squeeze_ext(&mut b); + let t0w = b.hint_word(commits, 2); + let t1w = b.hint_word(commits, 3); + // The terminal coefficients are field DATA, not digests, so they enter the + // transcript through the leaf encoding — the same rule the trees follow. + sponge.absorb_felts(&mut b, t0w); + sponge.absorb_felts(&mut b, t1w); + let t0 = t0w.as_ext(); + let t1 = t1w.as_ext(); + + // Hoisted reference lanes for the per-query root comparisons. + let main_root_lanes = b.unpack(main_root); + let l1_root_lanes = b.unpack(l1_root); + + for q in 0..shape::NUM_QUERIES { + let off = (q * shape::WORDS_PER_QUERY) as u32; + let bits = sponge.squeeze_bits(&mut b, shape::QUERY_BITS); // q0 = b0..b3 + let zero_bit = b.bit_const(false); + let one_bit = b.bit_const(true); + let path_a = [bits[1], bits[2], bits[3], zero_bit]; + let path_b = [bits[1], bits[2], bits[3], one_bit]; + + // Main-tree opening A (rows 2·l_A, 2·l_A+1 with l_A = q0 >> 1). + let row_a_even = b.hint_word(opens, off); + let row_a_odd = b.hint_word(opens, off + 1); + let leaf_a = edsl::leaf_hash_pair(&mut b, row_a_even, row_a_odd); + let sibs_a: Vec = (0..4).map(|i| b.hint_word(opens, off + 2 + i)).collect(); + let root_a = edsl::merkle_walk(&mut b, leaf_a, &path_a, &sibs_a); + edsl::assert_word_eq_lanes(&mut b, root_a.as_cell(), &main_root_lanes); + + // Main-tree opening B (leaf l_A + 8, i.e. rows q0+16's pair). + let row_b_even = b.hint_word(opens, off + 6); + let row_b_odd = b.hint_word(opens, off + 7); + let leaf_b = edsl::leaf_hash_pair(&mut b, row_b_even, row_b_odd); + let sibs_b: Vec = (0..4).map(|i| b.hint_word(opens, off + 8 + i)).collect(); + let root_b = edsl::merkle_walk(&mut b, leaf_b, &path_b, &sibs_b); + edsl::assert_word_eq_lanes(&mut b, root_b.as_cell(), &main_root_lanes); + + // Row parity: q0 and q0+16 share bit 0. + let (row_a, _) = b.select(bits[0], row_a_even, row_a_odd); + let (row_b, _) = b.select(bits[0], row_b_even, row_b_odd); + + // g0 at the two points: α-combination of the opened row columns. + let la = b.unpack(row_a); + let lo = edsl::horner_ext( + &mut b, + alpha, + &[ + la[0].as_ext(), + la[1].as_ext(), + la[2].as_ext(), + la[3].as_ext(), + ], + ); + let lb = b.unpack(row_b); + let hi = edsl::horner_ext( + &mut b, + alpha, + &[ + lb[0].as_ext(), + lb[1].as_ext(), + lb[2].as_ext(), + lb[3].as_ext(), + ], + ); + + // Fold 0 → must equal the opened g1[q0]. + let inv_x = edsl::pow_bits(&mut b, &bits, &invx_factors, offset_inv); + let v1 = edsl::fri_fold(&mut b, lo, hi, zeta0, inv_x); + + let l1_lo = b.hint_word(opens, off + 12); + let l1_hi = b.hint_word(opens, off + 13); + let l1_leaf = edsl::leaf_hash_pair(&mut b, l1_lo, l1_hi); + let l1_sibs: Vec = (0..3).map(|i| b.hint_word(opens, off + 14 + i)).collect(); + let l1_path = [bits[0], bits[1], bits[2]]; + let l1_root_c = edsl::merkle_walk(&mut b, l1_leaf, &l1_path, &l1_sibs); + edsl::assert_word_eq_lanes(&mut b, l1_root_c.as_cell(), &l1_root_lanes); + + let (g1_at_q0, _) = b.select(bits[3], l1_lo, l1_hi); + b.assert_eq_ext(v1, g1_at_q0.as_ext()); + + // Fold 1 → must equal the terminal polynomial at y₂. + let inv_y = edsl::pow_bits(&mut b, &bits[0..3], &invy_factors, offset2_inv); + let v2 = edsl::fri_fold(&mut b, l1_lo.as_ext(), l1_hi.as_ext(), zeta1, inv_y); + + let y2 = edsl::pow_bits(&mut b, &bits[0..3], &y2_factors, offset4); + let t1y = b.emul_base(t1, y2); + let t_eval = b.eadd(t0, t1y); + b.assert_eq_ext(v2, t_eval); + } + + b.public(main_root); + b.public(l1_root); + b.finish() +} + +pub fn fri_toy_program() -> LfmProgram { + compile(fri_toy_program_source()) +} + +// ============ R1f: a real Merkle opening under the production hash ============ + +/// Everything about a Merkle-opening program that is compile-time. +/// +/// Both fields are SHAPE, in the sense of `others/lfm-target-shape.md`: they fix +/// how many arena words the program reads, how many byteswaps it emits and how +/// many permutations the walk costs. A program that read them from an arena +/// would be claiming to authenticate a tree whose geometry the prover chose. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MerkleOpeningShape { + /// Field elements in the leaf. A leaf is a row PAIR (`ROWS_PER_LEAF = 2`), + /// so this is `2 × columns`. + pub leaf_values: usize, + /// Tree depth: index bits consumed, siblings read, permutations walked. + pub depth: usize, +} + +impl MerkleOpeningShape { + pub const fn columns(self) -> usize { + self.leaf_values / 2 + } +} + +/// Authenticates one FRI query's main-trace opening against a committed root, +/// under the PRODUCTION keccak Merkle conventions. +/// +/// Four arenas, each field in its own words (the R1e packing rule): +/// +/// 0. the leaf's field elements, one base word each, in hash order +/// (`evaluations ‖ evaluations_sym`); +/// 1. the sibling digests, two `u32`-half words per level, LEAF LEVEL FIRST; +/// 2. the leaf index, one base word; +/// 3. the committed root, two `u32`-half words. +/// +/// The walked root is asserted equal to arena 3 and then PUBLISHED. Both matter +/// and they do different jobs. The assert is the composition-ready shape — in +/// the assembled verifier the expected root arrives exactly like this, as an +/// arena value that Phase A has already bound into the transcript, and +/// `fri_toy_program` compares its roots the same way. Publishing is what makes +/// the result a claim rather than an internal fact: public words are absorbed +/// into the LFM statement, so a verifier that supplies the real committed root +/// as the claimed output is checking the machine reached THAT root and not some +/// other one the prover found convenient. +/// +/// ## What this program does and does not bind +/// +/// It binds the leaf, the path and the low `depth` bits of the index to the +/// root. It does not bind the index to a transcript — `bit_dec` constrains the +/// hinted index to its own decomposition and the walk uses the low `depth` +/// bits, so a prover may add any multiple of `2^depth` without changing +/// anything. That is correct here and unsound alone: in the assembled verifier +/// the bits come from `TranscriptReplay::sample_u64_pow2`, which produces +/// exactly this `Vec` from a squeezed candidate. This program is the +/// authentication half of that pair, built and measured before the sampler is +/// wired to it. +pub fn keccak_merkle_opening_program_source(shape: MerkleOpeningShape) -> LfmProgramSource { + // Keccak by ARGUMENT, not by omission. This is the R1f instrument and a + // registry fixture: its identity is pinned in `LFM_REGISTRY`, so it stays + // keccak whatever the wrap's configured hash becomes. A BLAKE3 twin is a + // second program (`merkle_opening_program_source_with_hash`) and would be a + // second registry row, never a re-blessing of this one. + merkle_opening_program_source_with_hash(shape, super::edsl::WrapHash::Keccak) +} + +/// [`keccak_merkle_opening_program_source`] under an explicitly chosen wrap +/// hash — the shared body, so the two hashes exercise one emitter rather than +/// two that have to be shown to coincide. +pub fn merkle_opening_program_source_with_hash( + shape: MerkleOpeningShape, + wrap_hash: super::edsl::WrapHash, +) -> LfmProgramSource { + use super::edsl; + + assert!(shape.leaf_values > 0, "a leaf covers at least one column"); + assert!( + shape.leaf_values.is_multiple_of(2), + "a leaf is a row PAIR, so it holds an even number of values" + ); + assert!( + (1..=32).contains(&shape.depth), + "depth must be in 1..=32: below, there is no path; above, the index \ + would outrun a single transcript candidate half" + ); + + let mut b = LfmBuilder::new().with_wrap_hash(wrap_hash); + // ⚠ A root is `digest_words` arena words, not two. This program takes its + // hash BY ARGUMENT and `merkle_opening_program` passes + // `WrapHash::production()`, so under an algebraic pin every stride here is + // ONE word — while `machine_tests::merkle_arenas` writes the siblings with + // `proof_arena::commitment_words`, which already follows the pin. Spelling + // the stride `2` made the writer and the reader disagree exactly as the + // aggregator's published-word schema did. + let words = edsl::digest_words(&b); + let leaf_arena = b.declare_arena(shape.leaf_values as u32); + let sibling_arena = b.declare_arena(words * shape.depth as u32); + let index_arena = b.declare_arena(1); + let root_arena = b.declare_arena(words); + + let values: Vec<_> = (0..shape.leaf_values as u32) + .map(|i| b.hint_felt(leaf_arena, i)) + .collect(); + let leaf = edsl::wrap_leaf_hash(&mut b, &values); + + let index = b.hint_felt(index_arena, 0); + let bits = b.bit_dec(index, shape.depth); + + let siblings: Vec = (0..shape.depth as u32) + .map(|l| edsl::hint_digest(&mut b, sibling_arena, words * l)) + .collect(); + + let root = edsl::wrap_merkle_walk(&mut b, leaf, &bits, &siblings); + + // The claimed root, compared cell by cell at whatever width the digest is. + let expected = edsl::hint_digest(&mut b, root_arena, 0); + for (got, want) in root.cells().iter().zip(expected.cells()) { + edsl::assert_word_eq(&mut b, *got, *want); + } + for cell in root.cells() { + b.public(*cell); + } + b.finish() +} + +pub fn keccak_merkle_opening_program(shape: MerkleOpeningShape) -> LfmProgram { + compile(keccak_merkle_opening_program_source(shape)) +} + +/// The R1f Merkle-opening walk at the hash production commits under — the +/// BLAKE3 twin [`keccak_merkle_opening_program_source`]'s comment anticipates. +/// +/// Use this to authenticate a REAL opening: the walk has to re-derive a root +/// the host built, so it must hash the way the host committed. The keccak +/// program above stays exactly as it is — it is the instrument whose identity +/// the registry pins, and this is a second program, not a re-blessing of it. +pub fn merkle_opening_program(shape: MerkleOpeningShape) -> LfmProgram { + merkle_opening_program_with_hash(shape, WrapHash::production()) +} + +pub fn merkle_opening_program_with_hash( + shape: MerkleOpeningShape, + wrap_hash: super::edsl::WrapHash, +) -> LfmProgram { + compile(merkle_opening_program_source_with_hash(shape, wrap_hash)) +} + +// ============ R1g(ii): the cross-epoch L2G commitment binding ============ + +/// Ties each epoch's own committed L2G root to the corresponding sub-proof of +/// the global proof — `verify_l2g_commitment_binding_view` (`lib.rs:993`), +/// emitted. +/// +/// Two arenas, each root in its own two words: +/// +/// 0. the per-epoch L2G roots, `EpochProof::l2g_root`, epoch order; +/// 1. the global proof's first `num_epochs` sub-proof main-trace roots. +/// +/// Every pair is asserted equal and the epoch side is published. As in +/// [`keccak_merkle_opening_program_source`], the assert is the relation and the +/// publish is what makes it a claim: the equality alone would be satisfied by +/// any two matching arena values, so the published roots are what a verifier +/// pins against the real bundle. +/// +/// ## What binds each side, and what this slice does not do +/// +/// This program asserts the RELATION. What binds each root to its proof is the +/// composition's job: the epoch root is bound by that epoch's own Phase A +/// absorb, the global root by the global proof's. Until those legs exist, both +/// sides are arena values and a prover could satisfy the equality with two +/// matching lies — which is exactly why the roots are published rather than +/// merely compared. +/// +/// ## Why the epoch count is a constant +/// +/// `num_epochs` is shape: it fixes how many roots are read and how many asserts +/// are emitted. Production's `final_proof.len() >= epoch_l2g_roots.len()` guard +/// has no counterpart here because a program compiled for `n` epochs cannot read +/// an `n+1`-epoch bundle — the arena schema would not match. +pub fn l2g_binding_program_source(num_epochs: usize) -> LfmProgramSource { + use super::edsl; + + assert!(num_epochs > 0, "a continuation has at least one epoch"); + + // The L2G roots are the block path's own commitments, so this program + // follows the configuration and reads each root at the DIGEST's width — + // two words on a byte hash, one on an algebraic one — never a literal two. + let mut b = LfmBuilder::new().with_wrap_hash(edsl::WrapHash::production()); + let dw = edsl::digest_words(&b); + let words = dw * num_epochs as u32; + let epoch_arena = b.declare_arena(words); + let global_arena = b.declare_arena(words); + + for i in 0..num_epochs as u32 { + let epoch = edsl::hint_digest(&mut b, epoch_arena, dw * i); + let global = edsl::hint_digest(&mut b, global_arena, dw * i); + for (e, g) in epoch.cells().iter().zip(global.cells()) { + edsl::assert_word_eq(&mut b, *e, *g); + b.public(*e); + } + } + b.finish() +} + +pub fn l2g_binding_program(num_epochs: usize) -> LfmProgram { + compile(l2g_binding_program_source(num_epochs)) +} + +// ============ R1g(iii): the attestation's program id ============ + +/// Halves in a `u64` rendered little-endian. +const U64_HALVES: u32 = 2; + +/// Everything about a `program_id` fold that is compile-time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProgramIdShape { + /// Page genesis commitments folded in. SHAPE: it fixes the byte length of + /// the hashed string, hence the block count and every padding position. + pub num_pages: usize, +} + +impl ProgramIdShape { + /// Bytes the fold hashes — `tag ‖ elf_digest ‖ pc_start ‖ decode ‖ n ‖ + /// (base ‖ commitment)*`. + pub fn byte_len(self) -> usize { + use crate::recursion::PROGRAM_ID_TAG; + PROGRAM_ID_TAG.len() + 32 + 8 + 32 + 8 + 40 * self.num_pages + } +} + +/// Emits `recursion::program_id_from_digest` — the fold the recursion guest +/// commits as the first 32 bytes of its attestation. +/// +/// One arena, each field in its own halves (the R1e packing rule): +/// the 32-byte ELF digest, the `u64` entry point, the 32-byte DECODE root, then +/// per page a `u64` base and a 32-byte commitment. +/// +/// ## Why the tag makes this the splice case +/// +/// `PROGRAM_ID_TAG` is 22 bytes, `≡ 2 (mod 4)`, so the ELF digest immediately +/// after it straddles half boundaries and so does everything behind it — the +/// same shape as R1e's 30-byte epoch tag. [`super::transcript_replay::ByteString`] +/// carries the byte-granular packer that handles it; alignment is a property of +/// the cursor, not of the field. +/// +/// ## What this program does NOT establish +/// +/// The attestation is deliberately **not self-enforcing**, and emitting the fold +/// in the machine does not change that. The guest uses SUPPLIED roots verbatim +/// without binding them to the inner ELF; the binding happens outside, when a +/// consumer recomputes the id from an ELF it trusts and compares +/// (`recursion::check_attestation`, an expensive native FFT + Merkle pass done +/// once at top level, never in-VM). A machine-emitted attestation inherits that +/// model unchanged — the same consumer-side compare closes it. Do not read +/// "the machine folded the roots" as "the machine bound the roots". +/// +/// ## Page ordering +/// +/// `program_id_from_digest` SORTS pages by base before folding. This program +/// folds them in supplied order, so the arena filler owes sortedness. That is +/// not a soundness hole: an unsorted fold yields an id that differs from the +/// consumer's recompute, so the proof is rejected there — the prover only +/// breaks their own attestation. It IS a completeness obligation, so it is +/// stated rather than assumed. +pub fn program_id_program_source(shape: ProgramIdShape) -> LfmProgramSource { + use super::builder::Felt; + + let root_halves = ROOT_HALVES; + let per_page = U64_HALVES + root_halves; + let total = root_halves + U64_HALVES + root_halves + per_page * shape.num_pages as u32; + + // ★ Keccak, and it must stay keccak: this mirrors + // `recursion::program_id_from_digest`, which names `PlatformKeccak256`. The + // program-identity digest is a host-side artifact that does not move when + // the commitment hash does, so following the flip here would break the + // attestation join rather than complete it. + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Keccak); + let arena = b.declare_arena(total); + let h: Vec = (0..total).map(|i| b.hint_felt(arena, i)).collect(); + + let (elf_digest, rest) = h.split_at(root_halves as usize); + let (pc_start, rest) = rest.split_at(U64_HALVES as usize); + let (decode, mut pages) = rest.split_at(root_halves as usize); + + let page_cells: Vec<(&[Felt], &[Felt])> = (0..shape.num_pages) + .map(|_| { + let (base, r) = pages.split_at(U64_HALVES as usize); + let (commitment, r) = r.split_at(root_halves as usize); + pages = r; + (base, commitment) + }) + .collect(); + + let id = emit_program_id(&mut b, shape, elf_digest, pc_start, decode, &page_cells); + b.public(id[0]); + b.public(id[1]); + b.finish() +} + +/// The `program_id` fold over cells the caller already holds — the form the +/// ASSEMBLED verifier needs. +/// +/// This exists for assembly ledger entry 7's DECODE half. DECODE's preprocessed +/// commitment is a function of the inner ELF, so it can be neither interned (that +/// would make LFM program identity ELF-dependent) nor left unbound. The +/// resolution ruled on 2026-08-04 is the **attestation join**: the same arena cell +/// Phase A absorbs is the cell this fold consumes, so a prover who substitutes a +/// DECODE root changes the published `program_id` and the consumer's own recompute +/// rejects it. That makes DECODE exactly as bound as `elf_digest` and `pc_start` +/// already are — and the join is only real if it is STRUCTURAL, one cell with two +/// consumers, which is why this takes cells rather than an arena. +/// +/// `elf_digest` is the same eight halves the epoch STATEMENT absorbs, so that +/// value's join comes free. +/// +/// ⚠ The join's strength is the consumer-side compare +/// (`recursion::check_attestation`), which has zero production call sites. Folding +/// the roots does not bind them by itself; it makes a substitution DETECTABLE by a +/// consumer who performs the ritual. +pub fn emit_program_id( + b: &mut LfmBuilder, + shape: ProgramIdShape, + elf_digest: &[super::builder::Felt], + pc_start: &[super::builder::Felt], + decode: &[super::builder::Felt], + pages: &[(&[super::builder::Felt], &[super::builder::Felt])], +) -> super::edsl::WrapDigest { + use super::transcript_replay::ByteString; + use crate::recursion::PROGRAM_ID_TAG; + + assert_eq!( + elf_digest.len(), + ROOT_HALVES as usize, + "the ELF digest is 32 bytes" + ); + assert_eq!( + pc_start.len(), + U64_HALVES as usize, + "the entry point is one u64" + ); + assert_eq!( + decode.len(), + ROOT_HALVES as usize, + "the DECODE commitment is 32 bytes" + ); + assert_eq!( + pages.len(), + shape.num_pages, + "the page count is SHAPE: it fixes the hashed length and every padding \ + position" + ); + + let mut s = ByteString::new(); + s.push_const(PROGRAM_ID_TAG); + s.push_halves(elf_digest); + s.push_halves(pc_start); + s.push_halves(decode); + s.push_const(&(shape.num_pages as u64).to_le_bytes()); + for (base, commitment) in pages { + assert_eq!(base.len(), U64_HALVES as usize, "a page base is one u64"); + assert_eq!( + commitment.len(), + ROOT_HALVES as usize, + "a page commitment is 32 bytes" + ); + s.push_halves(base); + s.push_halves(commitment); + } + assert_eq!(s.len(), shape.byte_len(), "byte accounting must agree"); + + // ⛔ **PINNED KECCAK — does not follow the configured wrap hash, and must + // not.** The host counterpart `recursion::program_id_from_digest` names + // `PlatformKeccak256` explicitly rather than the configuration's hash, in + // the same sense `statement::elf_digest` does: this is an INDEPENDENT + // keccak that identifies a program to consumers, not part of the proof + // system's commitment layer. Switching it would make the attestation join + // disagree with every host consumer of a `program_id`, and the disagreement + // would surface as a consumer-side compare failing, not as an unprovable + // program. + // + // `ByteString` therefore has two hashing methods and this call selects the + // pinned one deliberately. The naive sweep — "replace every + // `edsl::keccak256`" — produces a wrong proof exactly here, because + // grinding (`epoch::emit_grinding_check`) shares this type and DOES follow + // the configuration. + let d = s.keccak256(b); + super::edsl::WrapDigest::from_pair(d[0], d[1]) +} + +pub fn program_id_program(shape: ProgramIdShape) -> LfmProgram { + compile(program_id_program_source(shape)) +} + +// ======== R1g(i): the next epoch's REGISTER preprocessed commitment ======== + +/// Everything about a REGISTER-derivation program that is compile-time. +/// +/// Both fields belong to the INNER proof's `ProofOptions`, and both are SHAPE +/// in the sense of `others/lfm-target-shape.md`: they fix the LDE domain, hence +/// every twiddle, every leaf's byte layout and the whole tree's permutation +/// count. A program that read them from an arena would let the prover pick the +/// domain its commitment was computed over. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RegisterDerivationShape { + /// The inner proof's blowup factor. + pub blowup: usize, + /// The inner proof's coset offset (`ProofOptions::coset_offset`). + pub coset_offset: u64, + /// Rows per Merkle leaf of the inner REGISTER tree: the inner table's leaf + /// layout (`stark::leaf_layout::LeafLayout::rows_per_leaf`) — 2 today, 1 + /// under one-row openings (S2). One emitter, two constants; the host twin + /// is `register::compute_precomputed_commitment_with`. + pub rows_per_leaf: usize, +} + +impl RegisterDerivationShape { + /// Rows in the interpolation domain — `NUM_REGISTER_ADDRESSES` rounded up. + pub fn num_rows(self) -> usize { + crate::tables::register::NUM_REGISTER_ADDRESSES.next_power_of_two() + } + + /// Rows in the LDE domain. + pub fn lde_rows(self) -> usize { + self.num_rows() * self.blowup + } + + /// Merkle leaves — one per `rows_per_leaf` rows (a row PAIR today). + pub fn leaves(self) -> usize { + self.lde_rows() / self.rows_per_leaf + } + + /// Permutations the tree costs: one per leaf plus one per internal node. + /// Leaves are 48 bytes and parents 64, so each is a single rate block. + pub fn permutations(self) -> usize { + 2 * self.leaves() - 1 + } +} + +/// The REGISTER preprocessed columns' word addresses, in row order. +/// +/// Mirrors the private `register::register_word_address_list`, but assembled +/// from the PUBLIC `register_word_addresses` rather than hand-copied, so only +/// the ORDER is restated here. Nothing pins that order locally and nothing +/// needs to: the derived root is compared against production's own +/// `compute_precomputed_commitment`, and any disagreement about which +/// address sits in which row moves the root. +fn register_offsets() -> Vec { + use crate::tables::register::{NUM_REGISTER_ADDRESSES, register_word_addresses}; + let mut addrs = Vec::with_capacity(NUM_REGISTER_ADDRESSES); + for reg in 0..32u8 { + addrs.extend(register_word_addresses(reg)); + } + addrs.extend(register_word_addresses(254)); + addrs.extend(register_word_addresses(255)); + assert_eq!( + addrs.len(), + NUM_REGISTER_ADDRESSES, + "the register address list must cover every table row" + ); + addrs +} + +/// Derives the next epoch's REGISTER preprocessed commitment from `reg_fini` — +/// `register::compute_precomputed_commitment`, emitted. +/// +/// Two arenas, one base word per register word address (the R1e packing rule): +/// +/// 0. `R_i`, the epoch's INIT register file; +/// 1. `R_{i+1}`, the epoch's `reg_fini`. +/// +/// The derived root is PUBLISHED. There is nothing to assert it against, and +/// that is the mechanism rather than an omission — see below. +/// +/// ## Why this is a derivation and not a comparison +/// +/// The chaining obligation is often written as "check `reg_fini` against the +/// next epoch's supplied REGISTER root". There is no supplied root. +/// `build_epoch_airs` (`continuation.rs:636`) CONSTRUCTS the preprocessed +/// commitment from `register_init` and `reg_fini`, and `VmAirs::new`'s +/// `register_preprocessed` parameter — which every verify caller passes `None` +/// to — must stay unwired: computing the commitment from the values is what +/// ties the values to it. Supply the root instead and `reg_fini` has no +/// remaining role, so a prover could offer a root consistent with a `reg_fini` +/// it never honoured and the cross-epoch chain would go unenforced. +/// +/// ## Three columns, one of them free +/// +/// Production commits OFFSET ‖ INIT ‖ FINI. OFFSET holds the register word +/// addresses, which are fixed, so its LDE is a program CONSTANT — the shape +/// rule applying in the machine's favour for once. Only INIT and FINI carry +/// arena values and only they pay for a transform, which is why the derivation +/// emits two LDEs for three columns. +/// +/// The constant column still costs at leaf-hashing time: its values are +/// byte-swapped into the leaf like any other. That swap is what +/// [`RegisterDerivationShape::permutations`] does NOT count, and the cost test +/// prints both. +/// +/// ## What this program does NOT bind +/// +/// The two arenas are unbound here. In the assembled verifier `R_{i+1}` is the +/// same vector the next epoch reads as its INIT and the published root is what +/// that epoch's Phase A absorbs; until those joins exist a prover may supply +/// any pair and get the honestly-derived root for it. The derivation is +/// correct in isolation and binds nothing in isolation — the same standing +/// caveat as the L2G binding leg. +pub fn register_derivation_program_source(shape: RegisterDerivationShape) -> LfmProgramSource { + use crate::tables::register::NUM_REGISTER_ADDRESSES; + + let supplied = NUM_REGISTER_ADDRESSES as u32; + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::production()); + let init_arena = b.declare_arena(supplied); + let fini_arena = b.declare_arena(supplied); + let init: Vec<_> = (0..supplied).map(|r| b.hint_felt(init_arena, r)).collect(); + let fini: Vec<_> = (0..supplied).map(|r| b.hint_felt(fini_arena, r)).collect(); + + let root = emit_register_commitment(&mut b, shape, &init, &fini); + for cell in root.cells() { + b.public(*cell); + } + b.finish() +} + +/// The REGISTER preprocessed commitment over INIT and FINI cells the caller +/// already holds — [`register_derivation_program_source`] without the arenas. +/// +/// This is the form the ASSEMBLED verifier needs, and the reason it exists is +/// assembly ledger entries 7 and 2, which close together. The spine declares the +/// register boundary vector as one arena and reads `start_index` out of slot 64 +/// for the COMMIT-bus target; passing those very cells here means the root Phase +/// A absorbs is COMPUTED from them. That computation is the binding: production +/// has no arithmetic `start + len` check anywhere, it rebuilds the commitment +/// from the register vectors and rejects unless the absorbed root matches, so +/// `start_index` is tied to the chain exactly when the machine does the same. +/// +/// Hinting the root instead — which the spine did until this existed — leaves +/// `start_index` a free arena word: a prover supplies whatever index makes the +/// COMMIT bus close, and the unrelated hinted root satisfies Phase A. +/// +/// `init` and `fini` are `NUM_REGISTER_ADDRESSES` cells each. Rows past that are +/// the pooled ZERO constant, matching `zeroed_fe_vec`: production writes only the +/// supplied prefix, and making the padding program text rather than arena data is +/// the same discipline the OOD next-row pruning follows. +pub fn emit_register_commitment( + b: &mut LfmBuilder, + shape: RegisterDerivationShape, + init: &[super::builder::Felt], + fini: &[super::builder::Felt], +) -> super::edsl::WrapDigest { + use super::edsl; + use super::lde::coset_lde; + use crate::tables::register::{NUM_PREPROCESSED_COLS, NUM_REGISTER_ADDRESSES}; + use math::fft::bit_reversing::reverse_index; + + assert_eq!( + NUM_PREPROCESSED_COLS, 4, + "the derivation commits OFFSET ‖ INIT ‖ FINI ‖ IS_PC; a fifth preprocessed \ + column changes the leaf layout and the arena schema together" + ); + assert_eq!( + init.len(), + NUM_REGISTER_ADDRESSES, + "one INIT cell per register word address" + ); + assert_eq!( + fini.len(), + NUM_REGISTER_ADDRESSES, + "one FINI cell per register word address" + ); + let num_rows = shape.num_rows(); + let coset_offset = FE::from(shape.coset_offset); + + // Padding rows are zero in all four columns, exactly as `zeroed_fe_vec` + // leaves them: production writes only the first NUM_REGISTER_ADDRESSES. + let zero = b.felt_const(FE::zero()); + let offsets = register_offsets(); + let offset_col: Vec = (0..num_rows) + .map(|r| offsets.get(r).map_or(FE::zero(), |&a| FE::from(a))) + .collect(); + let column = |supplied: &[super::builder::Felt]| { + (0..num_rows) + .map(|r| supplied.get(r).copied().unwrap_or(zero)) + .collect::>() + }; + let init_col = column(init); + let fini_col = column(fini); + // IS_PC (main's final-PC pin): 1 on x255's two words, fixed like OFFSET. + let is_pc_col: Vec = (0..num_rows) + .map(|r| { + offsets.get(r).map_or(FE::zero(), |&a| { + FE::from(u64::from(crate::tables::register::is_pc_address(a))) + }) + }) + .collect(); + + // OFFSET and IS_PC are fixed, so their extensions are interned constants + // rather than an emitted transform — and they are taken from PRODUCTION's own + // transform, not from `lde`'s. That is deliberate: the four columns land in + // one tree, so a root that matches production pins the emitter against the + // very function it is emitting, inside the same hash. + let fixed_lde = |b: &mut LfmBuilder, column: &[FE], what: &str| -> Vec<_> { + use math::polynomial::Polynomial; + use stark::prover::evaluate_polynomial_on_lde_domain; + let poly = Polynomial::interpolate_fft::(column) + .unwrap_or_else(|_| panic!("the {what} column interpolates")); + evaluate_polynomial_on_lde_domain(&poly, shape.blowup, num_rows, &coset_offset) + .unwrap_or_else(|_| panic!("the {what} polynomial extends")) + .into_iter() + .map(|v| b.felt_const(v)) + .collect() + }; + let offset_lde = fixed_lde(b, &offset_col, "OFFSET"); + let is_pc_lde = fixed_lde(b, &is_pc_col, "IS_PC"); + let init_lde = coset_lde(b, &init_col, shape.blowup, coset_offset); + let fini_lde = coset_lde(b, &fini_col, shape.blowup, coset_offset); + + // Leaf `i` hashes the bit-reversed rows `R·i .. R·i + R − 1` (`R` = + // `shape.rows_per_leaf`: the pair `2i`, `2i+1` today), each written column + // by column in big-endian — `keccak_leaves_bit_reversed_grouped`. + let rows_per_leaf = shape.rows_per_leaf; + assert!( + rows_per_leaf == 1 || rows_per_leaf == 2, + "a REGISTER leaf holds one row or a row pair" + ); + let lde_rows = shape.lde_rows(); + let leaves: Vec<_> = (0..shape.leaves()) + .map(|leaf| { + let mut values = Vec::with_capacity(rows_per_leaf * NUM_PREPROCESSED_COLS); + for k in 0..rows_per_leaf { + let row = reverse_index(rows_per_leaf * leaf + k, lde_rows as u64); + values.extend([ + offset_lde[row], + init_lde[row], + fini_lde[row], + is_pc_lde[row], + ]); + } + edsl::wrap_leaf_hash(b, &values) + }) + .collect(); + + edsl::wrap_merkle_tree_root(b, &leaves) +} + +pub fn register_derivation_program(shape: RegisterDerivationShape) -> LfmProgram { + compile(register_derivation_program_source(shape)) +} + +/// A bare [`super::lde::coset_lde`], publishing every extended value. +/// +/// The instrument behind the LDE differential. `register_derivation_program` +/// exercises the transform only at the register shape — `n = 128`, coset offset +/// 3 — and every production REGISTER table has exactly that shape, so a +/// differential over production data cannot distinguish an emitter that is +/// right in general from one that is accidentally right at 128. This drives +/// synthetic sizes and offsets against production's own transform. +pub fn lde_probe_program_source(n: usize, blowup: usize, coset_offset: u64) -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(n as u32); + let values: Vec<_> = (0..n as u32).map(|i| b.hint_felt(arena, i)).collect(); + for v in super::lde::coset_lde(&mut b, &values, blowup, FE::from(coset_offset)) { + b.public(v.as_cell()); + } + b.finish() +} + +pub fn lde_probe_program(n: usize, blowup: usize, coset_offset: u64) -> LfmProgram { + compile(lde_probe_program_source(n, blowup, coset_offset)) +} diff --git a/prover/src/lfm/proof.rs b/prover/src/lfm/proof.rs new file mode 100644 index 000000000..13b662488 --- /dev/null +++ b/prover/src/lfm/proof.rs @@ -0,0 +1,1007 @@ +//! LFM prove / verify entry points. +//! +//! Prove: execute → traces → statement-bound transcript → the same generic +//! `multi_prove` the RV64 VM uses. Verify: registry-resolve the program's +//! roots (hard error on a miss — no fallback), rebuild the AIR set, replay +//! Phase A on a forked transcript to recover the shared LogUp challenges, +//! compute the expected `LfmPublic` balance from the *claimed* public words +//! (the COMMIT-bus pattern), and run `multi_verify_views`. + +use std::cell::Cell; +use std::time::Instant; + +use math::field::element::FieldElement; +use math::field::traits::IsPrimeField; +use stark::config::Commitment; +use stark::proof::options::ProofOptions; +use stark::proof::stark::MultiProof; +use stark::proof::view::MultiProofView; +use stark::prover::{IsStarkProver, ProvingError}; +use stark::residency_mode::ResidencyMode; +use stark::verifier::IsStarkVerifier; + +use crate::tables::types::{BusId, GoldilocksExtension, GoldilocksField}; + +use super::airs::{BLAKE3_SLOT, ChipSet, HASH_SLOT, LfmAirs, NUM_LFM_CHIPS}; +use super::compiler::LfmProgram; +use super::executor::{LfmExecError, LfmExecution, execute}; +use super::hash::HasherKind; +use super::registry::{LfmArtifacts, LfmProgramKind, LfmRegistryError}; +use super::statement::absorb_lfm_statement_tagged; +use super::trace::{LfmTraces, build_traces_with_merge}; +use super::word::LfmWord; +use crate::hash_pin::{LegacyFamily, ProofFamily}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +pub struct LfmProof { + pub proof: LfmStarkProof, + /// The public output the execution produced, in emission order. + pub public_words: Vec<(u32, LfmWord)>, +} + +/// An LFM proof's STARK part: the multi-table proof and every table's +/// commitment-time answers (`stark::commit_ood`), which ride beside it rather +/// than inside it so every other STARK proof keeps its encoding. +/// +/// Every LFM proof carries them and every LFM verifier requires them: there is +/// no LFM format without answers. +/// +/// Derefs to the multi-table proof, so the readers of its sub-proofs read it +/// as before. +#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct LfmStarkProof { + pub multi: MultiProof, + /// Per sub-proof, in proof order: its non-precomputed main columns at the + /// shared point `z0`. + pub commit_ood: Vec>>, +} + +impl std::ops::Deref for LfmStarkProof { + type Target = MultiProof; + fn deref(&self) -> &Self::Target { + &self.multi + } +} + +impl std::ops::DerefMut for LfmStarkProof { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.multi + } +} + +/// The three phases of a prove, in seconds: the LFM interpreter, the chip +/// trace fill, and `multi_prove`. +/// +/// ★ RECORDED, NOT PRINTED. This file is a production path — every shipped +/// prove goes through [`lfm_prove`] — so a `println!` beside the three +/// statements would put a timing line on the stdout of everything that proves, +/// which the rest of the module deliberately does not do (its one diagnostic is +/// a `log::info!`). The driver that wants the split takes it and prints it +/// beside the stage label it already holds. +/// +/// ⇒ Two things fall out of recording instead of printing, and both are why it +/// is worth the extra type. A stage LOADED from cache did not prove, so +/// [`take_prove_split`] returns `None` and the driver prints nothing rather +/// than reprinting the previous stage's numbers. And the cell is PER-THREAD, +/// so when sibling proofs run concurrently each worker fills and reads its own +/// — a shared print would emit unattributable lines the moment a second proof +/// is in flight. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct ProveSplit { + /// `execute` — the LFM interpreter's walk over the program. + pub execute: f64, + /// `build_traces_with_hasher` — the chip trace fill. + pub fill: f64, + /// `prove_traces_with_hasher` — AIR construction, the transcript and + /// `multi_prove` itself, NET of any wait for the card. + /// + /// ⛔ Net on purpose. Gross, this field absorbs however long a sibling held + /// the card, so the same work reads differently at one worker and at two + /// and the arms stop being comparable — which is exactly what a scheduling + /// A/B needs them to be. + pub multi_prove: f64, + /// Seconds spent BLOCKED waiting for the card before `multi_prove` began. + /// Zero whenever the permit is inert, so a serial line is unchanged. + pub permit_wait: f64, + /// The phases INSIDE `execute`. Carried beside it rather than folded into + /// it: a lever that moves one phase and not another cannot be read off their + /// sum, which is the lesson the three fields above were split out for in the + /// first place. + pub exec: super::executor::ExecSplit, +} + +thread_local! { + static LAST_PROVE_SPLIT: Cell> = const { Cell::new(None) }; +} + +/// Take the split of the most recent prove ON THIS THREAD, clearing it. +/// +/// `None` when nothing has proved on this thread since the last take. A caller +/// that prints on `Some` therefore stays silent for a cached stage, which is +/// the reading it wants: a line that appears is a prove that happened. +pub fn take_prove_split() -> Option { + LAST_PROVE_SPLIT.with(|c| c.take()) +} + +#[derive(Debug)] +pub enum LfmProveError { + Exec(LfmExecError), + Prover(ProvingError), + /// Traces filled for one permutation proved against artifacts built for + /// another: a caller's mismatch, refused before the card + /// ([`LfmFilled::prove`]). + HasherMismatch { + artifacts: HasherKind, + filled: HasherKind, + }, +} + +/// Proves under the permutation `artifacts` was built for. +/// +/// The hasher comes from the artifacts rather than from a default, because +/// `artifacts.program_id` is derived from it: taking it from anywhere else +/// would let the statement claim one permutation while the AIRs prove another. +pub fn lfm_prove( + program: &LfmProgram, + artifacts: &LfmArtifacts, + arenas: &[Vec], + options: &ProofOptions, +) -> Result { + lfm_prove_with_hasher(program, artifacts, arenas, options, artifacts.hasher) +} + +/// [`lfm_prove`] with the `LFM_HASH` permutation named explicitly at the call +/// site instead of read off `artifacts`. +/// +/// The chips bake their hasher's constants into their constraints, so execution +/// must use the same hasher — this function is the single place that holds them +/// together, passing one `hasher` to the executor, the trace filler and the AIR +/// set. Verification needs the same value ([`verify_against`]). +/// +/// # Errors +/// +/// [`LfmProveError::HasherMismatch`] if `hasher` is not the one `artifacts` was +/// built for, before anything runs. The two are not independent: +/// `artifacts.program_id` binds the hasher, so a mismatch would produce a proof +/// whose statement names a permutation the trace does not use — unverifiable +/// everywhere, and confusing at exactly the point (registry regeneration) where +/// it would be introduced. The agreement is a caller bug, refused rather than +/// a panic in the prover. +pub fn lfm_prove_with_hasher( + program: &LfmProgram, + artifacts: &LfmArtifacts, + arenas: &[Vec], + options: &ProofOptions, + hasher: HasherKind, +) -> Result { + same_hasher(artifacts.hasher, hasher)?; + lfm_prove_with_residency( + program, + artifacts, + arenas, + options, + hasher, + decide_lfm_residency(), + ) +} + +/// [`lfm_prove`] under the proof family `Fam` (`hash_pin::ProofFamily`): its +/// commitment configuration, transcript and statement tag. `artifacts` must be +/// built under `Fam::Stark` (`registry::build_artifacts_under`); artifacts of +/// another configuration fail at the precomputed-root check. +pub fn lfm_prove_under( + program: &LfmProgram, + artifacts: &LfmArtifacts, + arenas: &[Vec], + options: &ProofOptions, +) -> Result { + lfm_execute_and_fill(program, arenas, artifacts.hasher)?.prove_under::( + artifacts, + options, + decide_lfm_residency(), + ) +} + +/// [`lfm_prove_with_hasher`] with the residency mode supplied instead of read +/// from the environment, so a test can prove the same program under both modes +/// in one process without touching global state. +pub(crate) fn lfm_prove_with_residency( + program: &LfmProgram, + artifacts: &LfmArtifacts, + arenas: &[Vec], + options: &ProofOptions, + hasher: HasherKind, + residency: ResidencyMode, +) -> Result { + lfm_execute_and_fill(program, arenas, hasher)?.prove(artifacts, options, residency) +} + +/// A program executed and its traces filled: everything [`lfm_prove`] does +/// before the card, held for [`LfmFilled::prove`]. +/// +/// Split out so a driver can run the host phase while the program's artifacts +/// are still being built (they hold the card; the execution and the fill do +/// not read them). The two halves are [`lfm_prove_with_residency`] cut in +/// two, in the same order, on whatever thread calls each. +pub(crate) struct LfmFilled { + traces: LfmTraces, + public_words: Vec<(u32, LfmWord)>, + hasher: HasherKind, + execute_secs: f64, + fill_secs: f64, + exec_split: super::executor::ExecSplit, +} + +/// The host half of [`lfm_prove_with_residency`]: execute `program` over +/// `arenas` and fill its traces for `hasher`. +pub(crate) fn lfm_execute_and_fill( + program: &LfmProgram, + arenas: &[Vec], + hasher: HasherKind, +) -> Result { + // ★ THE SPLIT OF THE `prove` FIELD. The driver's per-node TIMING line prints + // this whole function as one number, and the three statements below are + // three different machines: `execute` is a single-threaded interpreter, + // `build_traces_with_hasher` is a parallel fill, and only the third reaches + // the card. Timing them separately is what lets a lever be aimed at one. + // + // ⚠ Recorded on the SUCCESS path only. The `?`s below return to a caller + // that panics, so a partial split would describe a run that produced no + // proof — and the cell would then hand it to the NEXT stage on this thread + // as if it were that stage's own. + // + // ⚠ The two `drop`s in [`lfm_fill_executed`] are the other half of that reasoning: this + // function's PEAK, not its clock, is what fences a third concurrent sibling + // on a 57.5 GiB host. Both are placed AFTER the elapsed-time read, so + // `execute` and `fill` keep measuring exactly what they measured before and + // stay comparable across the change; only the wall absorbs the free, which + // is a handful of `munmap`s. + let t = Instant::now(); + let execution = execute(program, arenas, &hasher).map_err(LfmProveError::Exec)?; + Ok(lfm_fill_executed( + program, + execution, + hasher, + t.elapsed().as_secs_f64(), + )) +} + +/// The fill half of [`lfm_execute_and_fill`], over an execution already run — +/// a streamed one ([`super::executor::StreamedExecution`]), whose witness is +/// [`execute`]'s word for word. `execute_secs` is what the split reports as +/// the execute. +pub(crate) fn lfm_fill_executed( + program: &LfmProgram, + execution: LfmExecution, + hasher: HasherKind, + execute_secs: f64, +) -> LfmFilled { + let LfmExecution { + records, + public_words, + memory, + split: exec_split, + } = execution; + // The final write-once array: 32 bytes per address, a few hundred MB for a + // wrap. It is a diagnostic surface for tests — nothing on the proving path + // reads it — so held to the end of this scope it would stay live through the + // fill AND the whole device phase, on every worker at once, for nothing. + drop(memory); + + let t = Instant::now(); + // The process format's merge (`LAMBDA_VM_ZF_LFM_MERGE`), filled straight + // into the merged tables. Only where the copy happens rides on it: the + // proof takes its layout from the artifacts, and a set filled under + // another one is laid out again before the card (`prove_traces_under`). + let traces = build_traces_with_merge(program, &records, hasher, &process_merge_plan(program)); + let fill_secs = t.elapsed().as_secs_f64(); + // Same reason, larger: the records are what the fill consumes, and they are + // dead the moment it returns. The traces it produced are the live set from + // here on; holding the records as well doubles the values through the card + // phase. + drop(records); + LfmFilled { + traces, + public_words, + hasher, + execute_secs, + fill_secs, + exec_split, + } +} + +impl LfmFilled { + /// The filled traces, for a caller that warms their domains before the + /// prove ([`super::domain_warm`]). + pub(crate) fn traces(&self) -> &LfmTraces { + &self.traces + } + + /// The filled traces, for the first prove's warm-up, which pairs them with + /// an AIR set as the prove does ([`super::first_prove_warm`]). + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn traces_mut(&mut self) -> &mut LfmTraces { + &mut self.traces + } + + /// The card half of [`lfm_prove_with_residency`]: prove the filled traces + /// against `artifacts`, which must be built for the hasher they were + /// filled with — a mismatch is refused ([`LfmProveError::HasherMismatch`]) + /// before anything reaches the card. A caller that fills before its + /// artifacts exist (the W3 tree's leaves and nodes) can name the wrong + /// hasher; the proof it would get names a permutation its traces do not + /// use. + pub(crate) fn prove( + self, + artifacts: &LfmArtifacts, + options: &ProofOptions, + residency: ResidencyMode, + ) -> Result { + self.prove_under::(artifacts, options, residency) + } + + /// [`Self::prove`] under the proof family `Fam` ([`prove_traces_under`]). + pub(crate) fn prove_under( + self, + artifacts: &LfmArtifacts, + options: &ProofOptions, + residency: ResidencyMode, + ) -> Result { + let LfmFilled { + mut traces, + public_words, + hasher, + execute_secs, + fill_secs, + exec_split, + } = self; + same_hasher(artifacts.hasher, hasher)?; + // The domains and twiddles this proof's tables need, built before it + // takes the card rather than under it (`LFM_WARM_DOMAINS`). + if super::domain_warm::on() { + super::domain_warm::warm(&traces, options); + } + let t = Instant::now(); + let waited_before = super::device_permit::waited_secs(); + let proof = prove_traces_under::( + artifacts, + &mut traces, + &public_words, + options, + hasher, + residency, + ) + .map_err(LfmProveError::Prover)?; + // The wait is SUBTRACTED rather than left inside, so `multi_prove` means + // the same thing at one worker and at two. + let permit_wait = (super::device_permit::waited_secs() - waited_before).max(0.0); + let multi_prove_secs = (t.elapsed().as_secs_f64() - permit_wait).max(0.0); + + LAST_PROVE_SPLIT.with(|c| { + c.set(Some(ProveSplit { + execute: execute_secs, + fill: fill_secs, + multi_prove: multi_prove_secs, + permit_wait, + exec: exec_split, + })) + }); + + Ok(LfmProof { + proof, + public_words, + }) + } +} + +/// The traces' hasher against the artifacts': [`LfmFilled::prove`]'s refusal. +fn same_hasher(artifacts: HasherKind, filled: HasherKind) -> Result<(), LfmProveError> { + match artifacts == filled { + true => Ok(()), + false => Err(LfmProveError::HasherMismatch { artifacts, filled }), + } +} + +/// Proves an already-built trace set against `artifacts`. +/// +/// Split out of [`lfm_prove`] so callers that need to inspect or corrupt a +/// trace between generation and proving (the tamper tests) share this +/// transcript setup instead of reimplementing it. `lfm_prove` itself goes +/// through [`prove_traces_with_hasher`], so this artifacts-hasher form has only +/// test callers. +#[cfg(test)] +pub(crate) fn prove_traces( + artifacts: &LfmArtifacts, + traces: &mut LfmTraces, + public_words: &[(u32, LfmWord)], + options: &ProofOptions, +) -> Result { + prove_traces_with_hasher( + artifacts, + traces, + public_words, + options, + artifacts.hasher, + decide_lfm_residency(), + ) +} + +/// [`prove_traces`] against an AIR set built for `hasher`. The traces must have +/// been built with the same one. +/// +/// Residency mode comes from [`decide_lfm_residency`] rather than a parameter: +/// it is a resource decision, invisible to the proof — recompute changes how +/// long an LDE lives, never a byte the transcript absorbs — so threading it +/// through the prove signature would put a knob with no wire meaning in front +/// of every caller. +#[cfg(test)] +pub(crate) fn prove_traces_with_hasher( + artifacts: &LfmArtifacts, + traces: &mut LfmTraces, + public_words: &[(u32, LfmWord)], + options: &ProofOptions, + hasher: HasherKind, + residency: ResidencyMode, +) -> Result { + prove_traces_under::(artifacts, traces, public_words, options, hasher, residency) +} + +/// Proves an already-built trace set against an AIR set built for `hasher` +/// (the traces must have been built with the same one), under the proof +/// family `Fam`: the transcript is `Fam`'s, seeded and statement-bound as the +/// legacy one is but under the family's statement tag, and the proof commits +/// under `Fam::Stark`. +/// +/// Residency mode is the caller's: a resource decision, invisible to the proof. +pub(crate) fn prove_traces_under( + artifacts: &LfmArtifacts, + traces: &mut LfmTraces, + public_words: &[(u32, LfmWord)], + options: &ProofOptions, + hasher: HasherKind, + residency: ResidencyMode, +) -> Result { + // The traces in the artifacts' merge (`LAMBDA_VM_ZF_LFM_MERGE`): a no-op for + // a set the fill already merged as the artifacts ask, which is the + // production path; a copy, before the card, for one filled unmerged. + artifacts + .merge + .check(&artifacts.log_heights) + .map_err(ProvingError::WrongParameter)?; + let prep = Instant::now(); + traces + .merge_to(&artifacts.merge.plan()) + .map_err(ProvingError::WrongParameter)?; + // ⛔ THE CARD, FOR THE SECOND OF A PROOF'S TWO DEVICE PHASES. Inert unless + // a driver has armed it, and then exclusive: `multi_prove` builds its own + // full-budget `VramGate`, so two of them in flight would budget the card + // twice over. Held here rather than around the whole of `lfm_prove` + // deliberately — the executor and the trace fill run BEFORE this call and + // must be free to overlap another proof's device phase, which is the entire + // point of the lever. + let airs = lfm_airs(artifacts, options, hasher)?; + warm_before_hold(&airs, traces); + super::first_prove_warm::note_prep(prep.elapsed().as_secs_f64()); + // ⛔ The permit is taken only now: the AIR set and the warm-up above are + // host work that must not hold the card. + let _card = super::device_permit::hold_labeled("multi_prove"); + let mut transcript = Fam::transcript(&[]); + absorb_lfm_statement_tagged( + &mut transcript, + Fam::STATEMENT_TAG, + &artifacts.program_id, + public_words, + options.fri_final_poly_log_degree, + ); + let (multi, commit_ood) = + stark::prover::GenericProver::::multi_prove_bound( + airs.air_trace_pairs(traces), + &mut transcript, + residency, + )?; + Ok(LfmStarkProof { multi, commit_ood }) +} + +/// The AIR set a proof against `artifacts` is made under: every chip's AIR +/// with its preprocessed root, the chunk and hash tails, the one-row roots +/// when the format has them, and the artifacts' merge. +pub(crate) fn lfm_airs( + artifacts: &LfmArtifacts, + options: &ProofOptions, + hasher: HasherKind, +) -> Result { + let mut airs = LfmAirs::new_chunked( + &artifacts.roots, + &artifacts.blake3_chunk_roots, + options, + artifacts.keccak_rnd_chunks, + hasher, + artifacts.chip_set, + ) + .with_hash_tail(&artifacts.hash_chunk_roots, options, hasher); + // One-row chips (S2) take their roots from the artifacts; without them a + // chip resolved to one row is refused by `multi_prove`, never recomputed. + if let Some(one_row) = &artifacts.one_row_roots { + airs = airs.with_one_row_roots(one_row); + } + airs.with_merge(&artifacts.merge, &artifacts.log_heights, options) + .map_err(ProvingError::WrongParameter) +} + +/// The per-domain constants each table's prove would build on first use +/// inside the card window, built before it instead +/// (`stark::prover::warm_prove_constants`), and under `LFM_WARM_FIRST_PROVE` +/// the DEEP's whole LDE cosets on the card +/// (`stark::prover::warm_deep_coset_points`). All are cached for the process +/// as the prove caches them, so no value moves. +pub(crate) fn warm_before_hold(airs: &LfmAirs, traces: &mut LfmTraces) { + let constants = stark::prover::warm_prove_constants_enabled(); + let cosets = super::first_prove_warm::on(); + if !constants && !cosets { + return; + } + let pairs = airs.air_trace_pairs(traces); + let tables: Vec<_> = pairs + .iter() + .map(|(air, trace, pi)| (*air, trace.num_rows(), *pi)) + .collect(); + if constants { + stark::prover::warm_prove_constants(&tables); + } + if cosets { + stark::prover::warm_deep_coset_points(&tables); + } +} + +/// The merge the process format asks of `program` (`LAMBDA_VM_ZF_LFM_MERGE`), +/// as `registry::merge_plan_for` reads it from the production options. +fn process_merge_plan(program: &LfmProgram) -> super::merge::MergePlan { + let format = crate::zf_format::ZfFormat::global(); + super::merge::plan_for_program( + program, + format.lfm_merge && format.one_row == stark::proof::options::OneRowMode::Off, + ) +} + +/// The LFM wrap's [`ResidencyMode`]: `RecomputeLde` when `LAMBDA_VM_RESIDENCY` +/// is set to `recompute`, else `Retain`. +/// +/// An explicit knob for the same reason the storage mode is one: the wrap has +/// no calibrated peak estimate to decide from, and the trade this mode makes — +/// one extra forward NTT per table against dropping the `O(N)` main-LDE +/// retention — is only worth taking when `N` is large. The fixture wrap has one +/// or two `KECCAK_RND` chunks and would just pay the NTT. +/// +/// `RecomputeLde` also releases each table's aux columns once its proof exists, +/// so callers that read the traces after proving must leave this unset. Nothing +/// on the wrap path does. +pub(crate) fn decide_lfm_residency() -> ResidencyMode { + match std::env::var("LAMBDA_VM_RESIDENCY").as_deref() { + Ok("recompute") => { + log::info!("lfm residency_mode: RecomputeLde (LAMBDA_VM_RESIDENCY=recompute)"); + ResidencyMode::RecomputeLde + } + _ => ResidencyMode::Retain, + } +} + +/// `Err` = registry miss (the hard, no-fallback path). `Ok(false)` = invalid +/// proof or claimed-public mismatch. +pub fn lfm_verify( + kind: LfmProgramKind, + proof: &LfmStarkProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, +) -> Result { + let artifacts = super::registry::resolve_artifacts(kind, options)?; + Ok(verify_against_artifacts( + &artifacts, + proof, + claimed_public, + options, + )) +} + +/// [`verify_against`] driven by a whole [`LfmArtifacts`]. +/// +/// # Why this exists +/// +/// `verify_against` takes seven separate pieces of program shape, so every new +/// thing the registry pins would change its signature and every call site with +/// it. Taking the struct means a field added to `LfmArtifacts` reaches the +/// verifier without moving anyone, and `prover/tests/d0_king_gate.rs` compiles +/// unchanged across such an arrival because of it. +pub fn verify_against_artifacts( + artifacts: &LfmArtifacts, + proof: &LfmStarkProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, +) -> bool { + verify_against_artifacts_under::(artifacts, proof, claimed_public, options) +} + +/// [`verify_against_artifacts`] under the proof family `Fam`: `Fam`'s +/// transcript and statement tag, `Fam::Stark`'s verifier. The family is the +/// verifier's constant: a proof made under any other fails its first Merkle +/// check or transcript draw. +pub fn verify_against_artifacts_under( + artifacts: &LfmArtifacts, + proof: &LfmStarkProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, +) -> bool { + verify_against_chunked_with::( + artifacts.one_row_roots.as_ref(), + (&artifacts.merge, &artifacts.log_heights), + &artifacts.roots, + &artifacts.blake3_chunk_roots, + &artifacts.hash_chunk_roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + proof, + claimed_public, + options, + artifacts.hasher, + artifacts.chip_set, + ) +} + +/// Verifies against a supplied root vector, program digest, `KECCAK_RND` chunk +/// count and hasher instead of a registry entry. +/// +/// The registry lookup in [`lfm_verify`] is the soundness argument's first +/// premise and has no off-switch; this is not one. It exists for callers that +/// legitimately hold freshly built artifacts — the registry regeneration path, +/// and tests covering program shapes that are not (and need not be) registered, +/// such as the per-length keccak256 programs. +/// +/// ⚠ This is the SINGLE-`LFM_BLAKE3` door: it builds one instance, from slot +/// 11's root. A program whose `LFM_BLAKE3` is chunked has extra roots that the +/// fifteen-wide array cannot hold, so its callers go through +/// [`verify_against_artifacts`] (or [`verify_against_chunked`]); handing a +/// chunked proof to this function rejects it on the AIR count rather than +/// verifying it against the wrong shape. +/// +/// Every piece is supplied for the same reason: it is program shape the +/// verifier must know to build the AIR set, and none of it is ever read off the +/// proof. That includes the hasher — which a caller holding artifacts should +/// pass as `artifacts.hasher`, since the digest it is paired with was derived +/// from exactly that value. There is deliberately no defaulting form: a +/// verifier that silently assumed a permutation would be assuming the one thing +/// the roots cannot tell it. +#[allow(clippy::too_many_arguments)] +pub fn verify_against( + roots: &[Commitment; NUM_LFM_CHIPS], + program_id: &Commitment, + keccak_rnd_chunks: usize, + proof: &LfmStarkProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, + hasher: HasherKind, + chip_set: ChipSet, +) -> bool { + // Slot 11's root IS chunk 0's, so a one-element window over the array is the + // whole `LFM_BLAKE3` shape a single-table program has. A CHUNKED program's + // extra roots live only in `LfmArtifacts`, which is why chunked callers go + // through `verify_against_artifacts` — this door would build one instance + // against a proof carrying several and reject on the AIR count. + verify_against_chunked( + roots, + &roots[BLAKE3_SLOT..=BLAKE3_SLOT], + program_id, + keccak_rnd_chunks, + proof, + claimed_public, + options, + hasher, + chip_set, + ) +} + +/// [`verify_against`] with `LFM_BLAKE3` split over `blake3_roots` instances. +/// +/// The general form. Chunk roots are supplied rather than derived for the same +/// reason every other piece here is: they are program shape the verifier must +/// know to build the AIR set, and none of it is ever read off the proof. +#[allow(clippy::too_many_arguments)] +pub fn verify_against_chunked( + roots: &[Commitment; NUM_LFM_CHIPS], + blake3_roots: &[Commitment], + program_id: &Commitment, + keccak_rnd_chunks: usize, + proof: &LfmStarkProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, + hasher: HasherKind, + chip_set: ChipSet, +) -> bool { + verify_against_chunked_with::( + None, + (&super::merge::LfmMerge::none(), &[0u8; NUM_LFM_CHIPS]), + roots, + blake3_roots, + // One `LFM_HASH` table: the hash split rides the artifacts, so a split + // program comes through `verify_against_artifacts`, and this door + // rejects its proof on the AIR count. + &roots[HASH_SLOT..=HASH_SLOT], + program_id, + keccak_rnd_chunks, + proof, + claimed_public, + options, + hasher, + chip_set, + ) +} + +/// [`verify_against_chunked`] with the program's one-row (S2) roots, when it +/// has them (`None` = row-pair roots only: a chip resolved to one row then +/// rejects). +#[allow(clippy::too_many_arguments)] +fn verify_against_chunked_with( + one_row_roots: Option<&super::registry::LfmOneRowRoots>, + (merge, log_heights): (&super::merge::LfmMerge, &[u8; NUM_LFM_CHIPS]), + roots: &[Commitment; NUM_LFM_CHIPS], + blake3_roots: &[Commitment], + hash_roots: &[Commitment], + program_id: &Commitment, + keccak_rnd_chunks: usize, + proof: &LfmStarkProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, + hasher: HasherKind, + chip_set: ChipSet, +) -> bool { + // The chunk count and the mask must agree, and BOTH come from the resolved + // registry entry rather than the proof — so this rejects a malformed entry, + // not a hostile prover. + // + // With the keccak family present a zero chunk count would drop KECCAK_RND — + // and its constraints — from a set that still contains LFM_KECCAK's sends, + // which is the shape the old unconditional guard existed to refuse. With + // the family absent, zero is the only correct count: the chip has no work, + // no sends and no reason to exist. + if chip_set.keccak != (keccak_rnd_chunks > 0) { + return false; + } + // A mask without BITWISE is only well formed for a set that sends + // BITWISE nothing. Dropped from a set with a sender, the sender's lookups + // would have no receiver to be checked against — so the predicate the + // artifacts were built with is re-checked here, on the mask as handed in. + if !chip_set.bitwise && chip_set.bitwise_required(hasher) { + return false; + } + // Chunk 0 of `LFM_HASH` is slot 5's root, so the list is never + // empty, and every chunk past it is one more sub-proof. + if hash_roots.first() != Some(&roots[HASH_SLOT]) { + return false; + } + // The merge is the artifacts' (never the proof's); `LfmAirs::with_merge` + // below refuses one that is not well formed against the artifacts' heights + // (two or more chips in slot order, no chip twice, every chip at the merged + // height), and the proof is then refused. + let view = MultiProofView::Owned(&proof.multi); + if view.len() + != chip_set.num_airs(keccak_rnd_chunks, blake3_roots.len()) + hash_roots.len() + - 1 + - merge.airs_saved() + { + return false; + } + + let mut airs = LfmAirs::new_chunked( + roots, + blake3_roots, + options, + keccak_rnd_chunks, + hasher, + chip_set, + ) + .with_hash_tail(hash_roots, options, hasher); + if let Some(one_row) = one_row_roots { + airs = airs.with_one_row_roots(one_row); + } + let Ok(airs) = airs.with_merge(merge, log_heights, options) else { + return false; + }; + let refs = airs.air_refs(); + + let mut transcript = Fam::transcript(&[]); + absorb_lfm_statement_tagged( + &mut transcript, + Fam::STATEMENT_TAG, + program_id, + claimed_public, + options.fri_final_poly_log_degree, + ); + + // Fork the statement-bound state and replay Phase A — the roots, `z0` and + // every commitment-time answer — to recover the shared LogUp challenges; + // the expected balance is the LfmPublic sum recomputed from the claimed + // words (all other LFM buses balance to zero internally). + type Verifier = stark::verifier::GenericVerifier::Stark>; + let mut replay = transcript.clone(); + let Some(challenges) = + Verifier::::replay_phase_a_bound(&refs, view, &proof.commit_ood, &mut replay) + else { + return false; + }; + let Some(expected) = expected_public_balance(claimed_public, &challenges[0], &challenges[1]) + else { + return false; + }; + + Verifier::::multi_verify_views_bound( + &refs, + view, + &proof.commit_ood, + &mut transcript, + &expected, + ) +} + +/// `Σ_i 1/(z − (LfmPublic + index_i·α + Σ_l v_l·α^{2+l}))` — the fingerprint +/// layout matches the `LFM_PUBLIC` sender token `(index, v0..v3)`. +/// +/// `pub(crate)` because the W-LFM verifier (`whir_proof`) closes its bus +/// against the same target: the `LFM_PUBLIC` chip is the same chip whichever +/// prover proved it. +pub(crate) fn expected_public_balance( + words: &[(u32, LfmWord)], + z: &FieldElement, + alpha: &FieldElement, +) -> Option> { + let bus = FieldElement::::from(BusId::LfmPublic as u64); + let mut powers = [FieldElement::::zero(); 5]; + powers[0] = *alpha; + for i in 1..5 { + powers[i] = &powers[i - 1] * alpha; + } + let mut fingerprints: Vec> = words + .iter() + .map(|(index, word)| { + let mut acc = &bus + FieldElement::::from(*index as u64) * &powers[0]; + for (l, lane) in word.iter().enumerate() { + let v = GoldilocksField::canonical(lane.value()); + acc += FieldElement::::from(v) * &powers[1 + l]; + } + z - acc + }) + .collect(); + // A zero fingerprint (a collision with z) is a failure, like COMMIT's. + FieldElement::inplace_batch_inverse(&mut fingerprints).ok()?; + Some( + fingerprints + .iter() + .fold(FieldElement::::zero(), |acc, t| acc + t), + ) +} + +// =========================================================================== +// Presets +// =========================================================================== + +/// The wrap layer's options when the wrap feeds the AGGREGATOR: blowup 4 with +/// [`LFM_QUERIES`] queries and the FRI terminal at degree 2^8. +/// +/// The choice optimizes the wrap's VERIFIER — the aggregation program pays +/// per query per wrap, and 110 queries against blowup-4 trees nearly halve +/// its Merkle-walk volume versus the 219-query blowup-2 wrap (the priced +/// A-decision point). The terminal at 2^8 trades one committed FRI layer — +/// 110 more openings the aggregator would walk — for 128 more terminal +/// coefficients it merely absorbs. Inner epochs are NOT touched by this +/// choice: the wrap PROGRAM is a function of the inner proof's options, so +/// this constructor moves no program identity. +/// +/// ★ A PRODUCTION FORMAT SITE: the process's [`ZfFormat`](crate::zf_format::ZfFormat) +/// is stamped on here (`LAMBDA_VM_ZF_CAP`, `_FRI`, `_ONE_ROW`), so every LFM +/// proof — wraps, nodes, the root — and every emitter that derives its shape +/// from these options sees one format. Unset knobs give +/// [`ZfFormat::DEFAULT`](crate::zf_format::ZfFormat::DEFAULT), the measured +/// configuration; every knob at its off spelling gives the legacy options. +pub fn aggregation_wrap_options() -> ProofOptions { + aggregation_wrap_options_under(crate::zf_format::ZfFormat::global()) +} + +/// [`aggregation_wrap_options`] under the format `zf` rather than the +/// process's: what a verifier that pins its format checks the tree under. +pub fn aggregation_wrap_options_under(zf: &crate::zf_format::ZfFormat) -> ProofOptions { + let mut opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(4) + .expect("blowup=4 is valid"); + opts.fri_number_of_queries = LFM_QUERIES; + opts.fri_final_poly_log_degree = 8; + zf.options(opts) +} + +/// The LFM proofs' query count: two above the blowup-4 preset's 110, which +/// the preset derives for 128 bits at the Johnson radius with a 20-bit grind +/// (`with_params`: ⌈108 / −log2(1/2 + 1/300)⌉). Each query adds ≈ 0.99 bits +/// there, so the two keep the query term ≈ 2 bits above 128 (≈ 130.9) — a +/// margin bought with queries rather than grinding, which costs the LFM +/// prover four times the nonce search for the same two bits. +pub const LFM_QUERIES: usize = 112; + +/// The STARK block's base-epoch options: the blowup-4 preset the production +/// tree proves its epochs under, with the process's +/// [`ZfFormat`](crate::zf_format::ZfFormat) stamped on — a PRODUCTION FORMAT +/// SITE, like [`aggregation_wrap_options`]. +/// +/// Not [`crate::recursion::Preset::options`] itself: that value also fixes +/// the RV64 recursion guest's verifier, which stays on the LEGACY format +/// (its presets name it). +pub fn block_base_options() -> ProofOptions { + block_base_options_under(crate::zf_format::ZfFormat::global()) +} + +/// [`block_base_options`] under the format `zf` rather than the process's. +pub fn block_base_options_under(zf: &crate::zf_format::ZfFormat) -> ProofOptions { + zf.options(crate::recursion::Preset::Blowup4.options()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Filled traces prove only against artifacts built for their hasher; any + /// other pair is refused with both hashers named, not a panic. + #[test] + fn a_fill_and_artifacts_for_different_hashers_are_refused() { + assert!(same_hasher(HasherKind::Rpx, HasherKind::Rpx).is_ok()); + match same_hasher(HasherKind::Rpx, HasherKind::Poseidon1W16) { + Err(LfmProveError::HasherMismatch { artifacts, filled }) => { + assert_eq!( + (artifacts, filled), + (HasherKind::Rpx, HasherKind::Poseidon1W16) + ) + } + other => panic!("a mismatch must be refused, got {other:?}"), + } + } + + /// Proving under a hasher other than the artifacts' is refused before + /// anything runs, naming both, not a panic. + #[test] + fn proving_under_another_hasher_than_the_artifacts_is_refused() { + let artifacts = LfmArtifacts { + roots: [[0u8; 32]; NUM_LFM_CHIPS], + log_heights: [0; NUM_LFM_CHIPS], + keccak_rnd_chunks: 0, + blake3_chunk_roots: Vec::new(), + blake3_chunk_log_heights: Vec::new(), + hash_chunk_roots: Vec::new(), + hash_chunk_log_heights: Vec::new(), + hasher: HasherKind::Rpx, + chip_set: ChipSet { + keccak: false, + blake3: false, + bitwise: false, + }, + program_id: [0u8; 32], + one_row_roots: None, + merge: super::super::merge::LfmMerge::none(), + }; + // One arena too many: an execution would refuse it, so the refusal + // below can only come from the check that runs first. + let program = super::super::programs::trivial_program(); + let arenas = vec![Vec::new(); program.arena_schema.lens.len() + 1]; + let got = lfm_prove_with_hasher( + &program, + &artifacts, + &arenas, + &ProofOptions::default_test_options(), + HasherKind::Poseidon1W16, + ); + match got { + Err(LfmProveError::HasherMismatch { artifacts, filled }) => { + assert_eq!( + (artifacts, filled), + (HasherKind::Rpx, HasherKind::Poseidon1W16) + ) + } + Err(other) => panic!("proving under another hasher must be refused, got {other:?}"), + Ok(_) => panic!("proving under another hasher must be refused, but it proved"), + } + } +} diff --git a/prover/src/lfm/proof_arena.rs b/prover/src/lfm/proof_arena.rs new file mode 100644 index 000000000..f795465ce --- /dev/null +++ b/prover/src/lfm/proof_arena.rs @@ -0,0 +1,454 @@ +//! Host-side arena filler: real proof BYTES → LFM arena words. +//! +//! Input is the guest's wire-format blob, not an in-memory bundle — see +//! [`super::proof_fixture`] for why that fidelity matters. Everything here reads +//! the archived view in place, exactly as the recursion guest does. +//! +//! ## The packing rule this module exists to enforce +//! +//! An arena is a vector of `u32` words, NOT a byte stream. Every field must be +//! packed into its OWN halves; concatenating fields and packing afterwards lets +//! a field whose length is not a multiple of four shift every field behind it. +//! That bug cost real debugging time in R1e and it is silent — the halves count +//! still comes out right, only the values are wrong. + +use crypto::merkle_tree::proof::verify_merkle_path_from_leaf_hash; +// Through the TRAIT rather than the byte backend's inherent method: the pinned +// backend is whichever `hash_pin` names, and only the trait form is common to +// all of them. +use crypto::merkle_tree::traits::IsStreamingLeafBackend; +use math::field::element::FieldElement; +use stark::config::Commitment; + +use crate::tables::types::GoldilocksField; + +use super::keccak_host::pack_stream; +use super::proof_fixture::FixtureArchive; +use super::word::{LfmWord, base_word}; + +type FE = FieldElement; + +/// The BATCHED Merkle backend the block path commits under, over any field. +/// +/// ⛔ Use this rather than `stark::config::BatchedMerkleTreeBackend`, which is +/// `BatchBlake3Backend` by definition — a workspace-default ALIAS, and therefore +/// the same silent spelling of the default that `Prover` and `Verifier` are. A +/// test comparing a machine leaf against that alias compares against BLAKE3 +/// whatever the branch pins. +pub type BlockBatched = + ::Batched; + +/// The PAIR backend FRI layers commit under. See [`BlockBatched`]; the alias it +/// replaces is `stark::config::FriLayerMerkleTreeBackend` = `PairBlake3Backend`. +pub type BlockPair = ::Pair; + +/// The Merkle backend the main trace is committed under — the BLOCK PATH's pin, +/// not a locally chosen equivalent and no longer `stark`'s default alias, so a +/// branch that pins a different hash reaches this module too. +type MainBackend = BlockBatched; + +/// Halves in one 32-byte commitment. +pub const ROOT_HALVES: usize = 8; + +/// The main-trace Merkle roots an epoch's sub-proofs commit to, in air order. +/// +/// These are the roots Phase A absorbs and, more importantly for R1f, the roots +/// a Merkle opening is authenticated AGAINST. They come straight off the proof. +/// +/// NOTE for the Phase-A leg: the verifier also absorbs each air's PREPROCESSED +/// commitment, and that one does NOT live in the proof — it comes from the AIR +/// set (`air.precomputed_commitment()`), which means replaying Phase A over a +/// real proof needs the epoch's AIRs rebuilt, not just its bytes. Out of scope +/// here and flagged rather than papered over. +pub fn epoch_main_roots(archive: &FixtureArchive, epoch: usize) -> Vec { + let bundle = &archive.guest_input().bundle; + assert!( + epoch < bundle.num_epochs(), + "epoch {epoch} out of range ({} epochs)", + bundle.num_epochs() + ); + let proofs = bundle.epoch_proof(epoch); + (0..proofs.len()) + .map(|i| *proofs.get(i).lde_trace_main_merkle_root()) + .collect() +} + +/// Number of sub-proofs (tables) in an epoch. +pub fn epoch_num_tables(archive: &FixtureArchive, epoch: usize) -> usize { + archive.guest_input().bundle.epoch_proof(epoch).len() +} + +pub fn num_epochs(archive: &FixtureArchive) -> usize { + archive.guest_input().bundle.num_epochs() +} + +/// Bytes epoch `epoch` committed — the statement's `public_output` field. +pub fn epoch_public_output(archive: &FixtureArchive, epoch: usize) -> &[u8] { + archive.guest_input().bundle.epoch_public_output(epoch) +} + +/// Packs commitments into arena halves, each root into its OWN eight halves. +pub fn roots_to_halves(roots: &[Commitment]) -> Vec { + let mut out = Vec::with_capacity(roots.len() * ROOT_HALVES); + for root in roots { + let halves = pack_stream(root); + debug_assert_eq!(halves.len(), ROOT_HALVES); + out.extend(halves); + } + out +} + +/// Wraps packed halves as arena words. +pub fn halves_to_arena(halves: Vec) -> Vec { + halves.into_iter().map(base_word).collect() +} + +/// Arena words one commitment occupies, HOST side — the counterpart of +/// `edsl::digest_words`, which is the machine side's reader. +/// +/// ⚠ These two must agree or every root in the arena is off by a word. They do +/// because both are functions of the configuration's digest width and neither +/// restates it: this reads `WrapHash::production()`, that reads the builder's. +/// A program built under another wrap hash (a proof family's) is fed at +/// [`words_per_root_for`] that hash. +pub fn words_per_root() -> usize { + words_per_root_for(super::edsl::WrapHash::production()) +} + +/// [`words_per_root`] under the wrap hash `hash`: one word for a one-cell +/// digest (every algebraic configuration, Poseidon1 included), two for a byte +/// digest — `edsl::digest_words` of a builder under `hash`. +pub fn words_per_root_for(hash: super::edsl::WrapHash) -> usize { + if hash.one_cell_digest() { 1 } else { 2 } +} + +/// LANES one commitment occupies once its arena words are unpacked — four per +/// word, so EIGHT on a byte hash and FOUR on an algebraic one. +/// +/// This is the count a PUBLISHED root has: everything that publishes a root +/// publishes `epoch::RootCells::lanes_flat`, which is the unpack of exactly +/// [`commitment_words`]. +/// +/// ⚠ Equal to [`ROOT_HALVES`] on the byte arm and NOT the same quantity. A byte +/// lane is a `u32` half of the root's 32 bytes; an algebraic lane is a full +/// canonical felt. A reader that spells the count `8` therefore reads the +/// algebraic arm at twice the stride and walks off the schema — the failure is +/// an index into the wrong field, not an out-of-bounds, so it surfaces as a +/// value comparison rather than a shape error. +pub fn lanes_per_root() -> usize { + super::word::WORD_LANES * words_per_root() +} + +/// A commitment as the LANES the machine publishes for it. +/// +/// The flattened [`commitment_words`], which is by construction what +/// `RootCells::lanes_flat` yields for a root hinted out of those same words — +/// so a test comparing published roots against host material has one function +/// to call instead of a rendering to re-spell per arm. +pub fn commitment_lanes(c: &Commitment) -> Vec { + commitment_words(c).into_iter().flatten().collect() +} + +/// A 32-byte commitment as the arena words the machine reads it from — TWO on a +/// byte hash, ONE on an algebraic one. +/// +/// A byte digest lives on the bus as eight `u32` halves, four per word, half `h` +/// = bytes `4h..4h+4` little-endian, and must be handed to the chip that way. +/// +/// ★ The algebraic arm is the layout this function's own doc used to name as the +/// thing it was NOT: [`super::word::pack_digest`]'s four FULL felts, the +/// `LFM_HASH` digest. An algebraic backend serialises its digest as four +/// canonical big-endian felts, so those 32 bytes ARE that word — the conversion +/// is the backend's own (`algebraic_commit::commitment_to_digest`), not a second +/// spelling of it. +/// +/// ⚠ The machine side reads the same count through +/// `epoch::RootCells::words_per_root`, and the two must agree or every root in +/// the arena is off by a word. They agree because both are functions of the +/// configuration's `WrapDigest` width and neither restates it. +pub fn commitment_words(c: &Commitment) -> Vec { + commitment_words_for(c, super::edsl::WrapHash::production()) +} + +// ==================== one query's main-trace opening ==================== + +/// One FRI query's MAIN-trace opening, in the form the machine consumes it. +/// +/// This is the input to [`crate::lfm::edsl::wrap_merkle_walk`] and the thing +/// R1f authenticates: a real row pair from a real continuation-epoch proof, +/// against that proof's own committed root. +/// +/// ## What the verifier does with these fields +/// +/// `Verifier::verify_opening_pair` hashes `evaluations ‖ evaluations_sym` into +/// one leaf and folds it up `merkle_path` at index `iota`. The pair is one leaf +/// because `ROWS_PER_LEAF = 2`: a query opens a value and its symmetric +/// counterpart, which are the two bit-reversed rows `2·iota` and `2·iota+1`, so +/// a single path authenticates both. +pub struct MainTraceOpening { + /// The committed root, read off the proof — the oracle for the whole leg. + pub root: Commitment, + /// `evaluations ‖ evaluations_sym` in hash order: the row pair written + /// column by column, each element rendered big-endian by the leaf hasher. + pub values: Vec, + /// Where `evaluations_sym` starts — i.e. the table's column count. + pub num_columns: usize, + /// Sibling digests, LEAF LEVEL FIRST. That is the order + /// `verify_merkle_path_from_leaf_hash` consumes them in: it walks the vector + /// forwards while shifting the index right, so element 0 pairs with the + /// index's least significant bit. (`Proof`'s doc comment describes the + /// reverse; the code is what this mirrors.) + pub siblings: Vec, +} + +impl MainTraceOpening { + /// Reads query `query` of sub-proof `table` in epoch `epoch`. + pub fn extract( + archive: &FixtureArchive, + epoch: usize, + table: usize, + query: usize, + ) -> MainTraceOpening { + let bundle = &archive.guest_input().bundle; + assert!(epoch < bundle.num_epochs(), "epoch {epoch} out of range"); + let proofs = bundle.epoch_proof(epoch); + assert!(table < proofs.len(), "table {table} out of range"); + let proof = proofs.get(table); + assert!( + query < proof.deep_poly_openings_len(), + "query {query} out of range ({} openings)", + proof.deep_poly_openings_len() + ); + let opening = proof.deep_poly_opening(query).main_trace_polys(); + let evaluations = opening.evaluations(); + let sym = opening.evaluations_sym(); + assert_eq!( + evaluations.len(), + sym.len(), + "a row pair's two rows must have the same width" + ); + MainTraceOpening { + root: *proof.lde_trace_main_merkle_root(), + num_columns: evaluations.len(), + values: evaluations.iter().chain(sym.iter()).cloned().collect(), + siblings: opening.merkle_path().to_vec(), + } + } + + /// Path length = tree depth = the number of index bits the walk consumes. + pub fn depth(&self) -> usize { + self.siblings.len() + } + + /// The leaf hash, computed by the PRODUCTION hasher on the production + /// split — literally the call `verify_opening_pair` makes. + pub fn leaf_hash(&self) -> Commitment { + // ⚠ Named through the TRAIT, not left to inherent-method resolution. The + // byte backend happens to carry an inherent `hash_data_from_slices` and + // an algebraic one does not, so the unqualified call silently resolves to + // the byte backend's own method and then fails to exist under any other + // pin. The trait form is the one every backend has. + >::hash_data_from_slices( + &self.values[..self.num_columns], + &self.values[self.num_columns..], + ) + } + + /// Whether production's own path check accepts this opening at `index`. + pub fn verifies_at(&self, index: usize) -> bool { + verify_merkle_path_from_leaf_hash::( + &self.siblings, + &self.root, + index, + self.leaf_hash(), + ) + } + + /// Every leaf index at which this opening authenticates. + /// + /// ## Why a search, and why that is honest + /// + /// The index is the FRI query challenge `iota`, and it is NOT in the proof — + /// the verifier derives it from the transcript, which needs the epoch's + /// statement and its AIR set, neither of which a byte blob carries (the + /// preprocessed commitments come from `air.precomputed_commitment()`). Since + /// the path, the leaf and the root are all fixed by the proof, the index is + /// nonetheless determined by them, so recovering it by exhaustion asks the + /// proof rather than inventing an answer — and the oracle doing the asking + /// is production's `verify_merkle_path_from_leaf_hash`, not a local model. + /// + /// The result is a LIST because a degenerate tree has several: a table + /// whose trace is mostly padding commits identical rows, so identical + /// leaves sit under identical subtrees and many indices verify. Any opening + /// used for an index-tamper vector must have exactly one — otherwise + /// "flip an index bit" is not a tamper at all. Callers assert that. + /// + /// Costs `2^depth` path walks; fine at the fixture's depths, not a + /// mechanism anything but a fixture should use. + pub fn indices_that_verify(&self) -> Vec { + (0..(1usize << self.depth())) + .filter(|i| self.verifies_at(*i)) + .collect() + } + + /// The leaf's field elements as arena words: one base word each, since the + /// machine byteswaps them itself (they are full felts, not `u32` halves). + pub fn leaf_arena(&self) -> Vec { + self.values.iter().copied().map(base_word).collect() + } + + /// The sibling digests as arena words, [`words_per_root`] per level, leaf + /// level first — TWO on a byte hash, ONE on an algebraic one. + pub fn sibling_arena(&self) -> Vec { + self.siblings.iter().flat_map(commitment_words).collect() + } + + /// The committed root as arena words. + pub fn root_arena(&self) -> Vec { + commitment_words(&self.root) + } +} + +/// Host mirror of the machine's walk, returning the root it reaches. +/// +/// Production's checker returns a bool, so it cannot supply the root a TAMPERED +/// input folds to — which a coherent forgery needs (the forged run must claim a +/// root consistent with its own inputs, or it fails in-machine before the +/// interesting check). Built from the production parent hash, so the only thing +/// local about it is the loop. +pub fn walk_to_root(leaf: Commitment, index: usize, siblings: &[Commitment]) -> Commitment { + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + let mut node = leaf; + let mut index = index; + for sibling in siblings { + node = if index.is_multiple_of(2) { + MainBackend::hash_new_parent(&node, sibling) + } else { + MainBackend::hash_new_parent(sibling, &node) + }; + index >>= 1; + } + node +} + +// ==================== the cross-epoch L2G binding ==================== + +/// Each epoch's own committed L2G table root, in epoch order. +/// +/// The left-hand side of `verify_l2g_commitment_binding_view`: epoch `i`'s +/// `EpochProof::l2g_root`, which that epoch's own proof commits to. +pub fn epoch_l2g_roots(archive: &FixtureArchive) -> Vec { + let bundle = &archive.guest_input().bundle; + (0..bundle.num_epochs()) + .map(|i| bundle.epoch_l2g_root(i)) + .collect() +} + +/// The global proof's first `count` sub-proof main-trace roots — the right-hand +/// side of the same binding. +/// +/// The global proof carries one L2G sub-proof per epoch FIRST, then +/// GLOBAL_MEMORY, so sub-proof `i` is epoch `i`'s L2G table. Production also +/// checks `final_proof.len() >= epoch_l2g_roots.len()`; here that is structural, +/// since a machine program compiled for `n` epochs reads exactly `n` roots and +/// this function panics rather than short-reading. +pub fn global_l2g_roots(archive: &FixtureArchive, count: usize) -> Vec { + let global = archive.guest_input().bundle.global_proof(); + assert!( + global.len() >= count, + "the global proof has {} sub-proofs, need {count}", + global.len() + ); + (0..count) + .map(|i| *global.get(i).lde_trace_main_merkle_root()) + .collect() +} + +/// Commitments as arena words, two per root, in order. +pub fn commitments_to_arena(roots: &[Commitment]) -> Vec { + roots.iter().flat_map(commitment_words).collect() +} + +/// [`commitments_to_arena`] at the width of an EXPLICIT wrap hash rather than +/// the configuration's — the host half of the rule that the arena stride is the +/// BUILDER's digest width. +/// +/// A program that pins a byte hash on its own builder reads two words per root +/// whatever the pin says (`edsl::digest_words` of that builder), so the host +/// feeding it must serialise at that width too; under an algebraic pin the +/// configuration-following [`commitments_to_arena`] would hand it one word per +/// root and the executor's arena-length check refuses the program outright. +pub fn commitments_to_arena_for(roots: &[Commitment], hash: super::edsl::WrapHash) -> Vec { + roots + .iter() + .flat_map(|c| commitment_words_for(c, hash)) + .collect() +} + +/// [`commitment_words`] at the width of an explicit wrap hash. See +/// [`commitments_to_arena_for`]. A one-cell digest (RPX, Poseidon1: four +/// canonical big-endian felts) is one word; a byte digest is two. +pub fn commitment_words_for(c: &Commitment, hash: super::edsl::WrapHash) -> Vec { + if hash.one_cell_digest() { + return vec![super::algebraic_commit::commitment_to_digest(c)]; + } + let halves = pack_stream(c); + debug_assert_eq!(halves.len(), ROOT_HALVES); + vec![ + [halves[0], halves[1], halves[2], halves[3]], + [halves[4], halves[5], halves[6], halves[7]], + ] +} + +// ==================== the attestation's program id ==================== + +/// The inner ELF bytes the guest input carries. +pub fn inner_elf(archive: &FixtureArchive) -> &[u8] { + archive.guest_input().inner_elf.as_slice() +} + +/// The supplied DECODE preprocessed root. +pub fn decode_commitment(archive: &FixtureArchive) -> Commitment { + archive.guest_input().decode_commitment +} + +/// The supplied per-page genesis roots, `(base, commitment)`. +/// +/// ⚠ EMPTY for the `continuation-fixture` guest — it touches no data pages — so +/// any test that only uses the fixture leaves the page path unexercised. Drive +/// it with a synthetic shape rather than treating it as covered. +pub fn page_commitments(archive: &FixtureArchive) -> Vec<(u64, Commitment)> { + archive + .guest_input() + .page_commitments + .iter() + .map(|p| (p.0.to_native(), p.1)) + .collect() +} + +// ============ the cross-epoch REGISTER boundary ============ + +/// Epoch `i`'s `(register_init, reg_fini)` — the pair +/// `register::compute_precomputed_commitment` turns into that epoch's +/// preprocessed REGISTER commitment. +/// +/// INIT is the VERIFIER's derivation, never a bundled value: epoch 0's comes +/// from the inner ELF's entry point and every later epoch's is the previous +/// epoch's `reg_fini`. That is the whole point of the chaining obligation, so +/// reading INIT off the proof here would quietly test a different mechanism — +/// the walk below is the same one `verify_continuation_archived` performs. +pub fn register_boundary(archive: &FixtureArchive, epoch: usize) -> (Vec, Vec) { + let bundle = &archive.guest_input().bundle; + assert!( + epoch < bundle.num_epochs(), + "epoch {epoch} out of range ({} epochs)", + bundle.num_epochs() + ); + let elf = executor::elf::Elf::load(inner_elf(archive)).expect("the inner ELF must load"); + let mut init = crate::tables::register::register_init_from_entry_point(elf.entry_point); + for i in 0..epoch { + init = bundle.epoch_reg_fini(i).expect("reg_fini deserializes"); + } + let fini = bundle.epoch_reg_fini(epoch).expect("reg_fini deserializes"); + (init, fini) +} diff --git a/prover/src/lfm/proof_fixture.rs b/prover/src/lfm/proof_fixture.rs new file mode 100644 index 000000000..5ca3f1d0e --- /dev/null +++ b/prover/src/lfm/proof_fixture.rs @@ -0,0 +1,326 @@ +//! Real continuation-proof BYTES for the machine to consume. +//! +//! R1f's premise: everything before this point ran on synthetic or +//! self-generated data. This module produces an actual two-epoch continuation +//! proof in exactly the encoding the RV64 recursion guest receives. +//! +//! ## Why bytes, and why THESE bytes +//! +//! The guest never sees a `ContinuationProof`. It gets a blob in private input +//! and reads it zero-copy through rkyv. So a machine-side reader whose input is +//! a byte blob is the direct analogue of the guest's reader, and a disagreement +//! between the two is a meaningful signal; a reader that consumed an in-memory +//! `ContinuationProof` would be exercising a path production does not have. +//! +//! The encoding is therefore NOT invented here. It is +//! [`crate::recursion::encode_continuation_guest_input`] — the same encoder the +//! guest's blob comes from — so the fixture cannot drift from production without +//! the encoder itself changing. +//! +//! ## Why not the existing dump test +//! +//! `tests::recursion_smoke_test::test_dump_recursion_input` produces exactly +//! these bytes, but it is `#[ignore]`d as a diagnostic, is driven by five +//! environment variables, and writes to a fixed `/tmp` path. None of that is +//! usable from a deterministic unit test. This module calls the same two public +//! functions it calls — `prove_continuation` then +//! `encode_continuation_guest_input` — and nothing else, so the ENCODER (the +//! part that must not drift) is shared while the harness around it is not. + +use std::path::{Path, PathBuf}; + +use stark::proof::options::ProofOptions; + +use crate::recursion::MIN_PROOF_OPTIONS; + +/// Inner guest whose execution the fixture proves. +/// +/// ★ **Not `fibonacci`, and the difference is a property the fixture has to +/// have.** Every guest in this suite before this one committed its public output +/// immediately before `halt()`, which puts the output in the FINAL epoch. But +/// `epoch_tests::EpochFront::build` asserts `!is_final` — it harvests an +/// INTERMEDIATE epoch on purpose — so under such a guest `RealEpoch::public_output` +/// is empty for every epoch that harness can reach, and the output half of +/// `the_closure_rejects_a_moved_index_or_output` has nothing to tamper. That test +/// spent four weeks red on its own anti-vacuity guard saying exactly that. +/// +/// `continuation-fixture` commits and then does bounded work, so the committing +/// epoch is intermediate by CONSTRUCTION. `fibonacci` could not simply grow a +/// tail: `bench_vs/run.sh` builds it as the Lambda VM arm of a cross-prover +/// benchmark against `bench_vs/sp1/fibonacci`, and the two arms have to run the +/// same program. +pub const FIXTURE_INNER_ELF: &str = "continuation-fixture"; + +/// Epoch size, as `log2(cycles)`. +/// +/// Measured, not guessed: the guest runs **48 cycles**, committing at cycle 13. +/// A 32-cycle epoch therefore splits it into two, with the commit 19 cycles +/// inside the first and the halt 16 cycles past the boundary. A single-epoch +/// fixture would defeat the point, since the whole target is a CONTINUATION, +/// and `continuation_fixture_generates_two_epochs` is the canary for it. +/// +/// ★ The three margins are the design, not a happy result — see the guest's own +/// `TAIL_STEPS`, which is the dial that sets them at two cycles per step — and +/// [`tests::the_fixture_guest_commits_in_an_intermediate_epoch`] asserts them. +/// This constant used to be a four-cycle question, and the answer moved under it +/// when the toolchain rebuilt the ELF two instructions shorter; the epoch COUNT +/// did not move when that happened, which is why the count alone is not the +/// canary it was taken for. +/// +/// ⚠ **The cycle count is a property of the compiled ELF, not of the guest +/// source.** `bench_vs/lambda/continuation-fixture` has no dependencies, so +/// nothing in this workspace moves it — but the pinned nightly and the sysroot +/// do. If the canary reports anything but two epochs, re-measure rather than +/// guess: run the ELF to completion under `Executor::resume_with_limit`, count +/// the logs (one per cycle), and note which one carries syscall 64 — this must +/// stay strictly between that cycle and the count. +/// +/// Blob size for the record: **596,828 bytes** at the two epochs this selects. +pub const FIXTURE_EPOCH_LOG2: u32 = 5; + +/// Proof options the fixture is proved under: the `min` preset, which is the +/// cheapest to generate. It is explicitly NOT a secure parameter set — this +/// fixture exists to exercise byte layout and Merkle structure, not to stand in +/// for a production proof's security. +pub fn fixture_options() -> ProofOptions { + MIN_PROOF_OPTIONS +} + +/// Repository root, derived from this crate's manifest directory. +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("prover/ has a parent") + .to_path_buf() +} + +/// Reads a recursion-suite guest ELF (built by `make compile-recursion-elfs`). +pub fn read_inner_elf() -> Vec { + let path = workspace_root().join(format!( + "executor/program_artifacts/recursion/{FIXTURE_INNER_ELF}.elf" + )); + std::fs::read(&path).unwrap_or_else(|e| { + panic!( + "failed to read {} — run `make compile-recursion-elfs`: {e}", + path.display() + ) + }) +} + +/// Proves the fixture continuation and encodes the guest blob. +/// +/// Returns `(blob, num_epochs)`. The epoch count is read before encoding +/// because the encoder consumes the bundle. +pub fn generate() -> (Vec, usize) { + let elf = read_inner_elf(); + let opts = fixture_options(); + let bundle = crate::continuation::prove_continuation(&elf, &[], FIXTURE_EPOCH_LOG2, &opts) + .expect("fixture continuation must prove"); + let num_epochs = bundle.num_epochs(); + let blob = crate::recursion::encode_continuation_guest_input(bundle, &elf, &opts) + .expect("fixture blob must encode"); + (blob, num_epochs) +} + +/// ★ The cache key for everything that changes these bytes INCOMPATIBLY. +/// +/// The cache lives in the shared temp directory and is keyed on the inner ELF +/// and the epoch size — neither of which says anything about the proof format. +/// Every format-moving branch on one machine therefore wrote its blob over +/// everyone else's, and the next branch to read it got bytes its own +/// `rkyv::access` could not validate. That surfaces as +/// `"fixture blob must validate"` in whichever test happened to read first +/// (`arena_filler_reads_real_committed_roots`, `keccak_merkle_opening_cost`) — +/// a phantom failure with no relationship to the branch under test, which cost +/// a baseline reconciliation to diagnose. +/// +/// It is worse than a wasted debugging hour: the stash-and-rerun control used +/// to separate "pre-existing failure" from "my change broke it" SHARES this +/// cache, so a poisoned blob makes both arms fail and the control cannot tell +/// them apart. An exoneration taken against a stale blob proves nothing. +/// +/// So the key names the two axes that actually move the bytes: the statement +/// encoding version and the commitment hash. A branch that changes either gets +/// its own file instead of corrupting the shared one. +pub fn cache_format_key() -> String { + let tag = std::str::from_utf8(crate::statement::DOMAIN_TAG).unwrap_or("stmt"); + format!("{tag}-{:?}", crate::hash_pin::BLOCK_COMMITMENT_HASH) +} + +/// Whether `bytes` still look like a blob this build can read. +/// +/// Only the wire prefix — the magic and the version [`crate::encode_recursion_input`] +/// writes. It is deliberately cheap: this is the last line of defence for a +/// truncated or foreign file, not the format check. Separating +/// format-incompatible builds is [`cache_format_key`]'s job, because two +/// branches can share this prefix and still disagree about everything after it. +fn prefix_is_readable(bytes: &[u8]) -> bool { + bytes.len() > crate::RECURSION_INPUT_PREFIX_LEN + && bytes[0..4] == crate::RECURSION_INPUT_MAGIC + && u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]) + == crate::RECURSION_INPUT_VERSION +} + +/// Loads the cached blob, generating and caching it when absent. +/// +/// Proving is slow enough that regenerating per test is not viable, but a +/// checked-in binary is worse: it can drift from the encoder silently. So the +/// cache lives outside the repository and the GENERATION path is what tests +/// exercise on a cold cache. +/// +/// ## ⚠ The blob is NOT reproducible — measured, and it constrains callers +/// +/// Two `generate()` calls on identical inputs produce blobs that differ in +/// ~65k of 587k bytes, and the difference is SEMANTIC, not archive padding: +/// some sub-proofs commit to different roots, which moves the Fiat-Shamir +/// challenges, which opens different leaves. (`machine_tests:: +/// fixture_generation_is_not_reproducible` is the standing evidence.) +/// +/// So **nothing derived from a specific blob may be pinned as a constant** — +/// not a query index, not a leaf value, not a root. Pin SHAPE (column counts, +/// tree depths), which is stable, and recover per-blob values from the blob. +/// R1f's `R1F_SHAPE` and its recovered leaf index are built that way; a pinned +/// index would have broken on the very next cold cache. +/// +/// The write ([`write_cache`]) is atomic (temp file then rename) because the +/// cache is shared by tests that run in parallel and one of them regenerates +/// it: without the rename a reader can observe a half-written blob, and since +/// blobs differ, "it was fine last time" proves nothing. +pub fn load_or_generate(cache: &Path) -> Vec { + if let Ok(bytes) = std::fs::read(cache) { + // A stale or foreign blob REGENERATES rather than erroring downstream: + // the failure it used to cause named a validation site, never the cache. + if prefix_is_readable(&bytes) { + return bytes; + } + eprintln!( + "fixture cache at {} is not readable by this build — regenerating", + cache.display() + ); + } + let (blob, _) = generate(); + write_cache(cache, &blob); + blob +} + +/// Publishes `blob` at `cache` atomically, so a concurrent reader sees either +/// the old complete blob or the new one, never a torn prefix. +pub fn write_cache(cache: &Path, blob: &[u8]) { + if let Some(dir) = cache.parent() { + let _ = std::fs::create_dir_all(dir); + } + let staging = cache.with_extension(format!("tmp{}", std::process::id())); + if std::fs::write(&staging, blob).is_ok() { + let _ = std::fs::rename(&staging, cache); + } +} + +/// Checks the blob carries the recursion input's magic prefix — i.e. that it is +/// the guest's wire format and not some other encoding. +pub fn has_recursion_prefix(blob: &[u8]) -> bool { + blob.len() > crate::RECURSION_INPUT_PREFIX_LEN + && blob.starts_with(&crate::RECURSION_INPUT_MAGIC) +} + +/// An opened fixture blob, holding the aligned bytes the archived view borrows +/// from. +/// +/// Mirrors `recursion::verify_continuation_and_attest`'s decode exactly: strip +/// the magic/version prefix, re-align if the host `Vec` is not on rkyv's +/// alignment (guest slices are aligned by construction; host ones carry no such +/// guarantee), then `rkyv::access` with validation. The owning struct exists +/// because the archived view borrows from the aligned buffer. +pub struct FixtureArchive { + aligned: rkyv::util::AlignedVec<{ crate::RECURSION_INPUT_ALIGN }>, +} + +impl FixtureArchive { + pub fn open(blob: &[u8]) -> Self { + let archive_bytes = crate::recursion_archive_bytes(blob) + .expect("fixture blob must carry the recursion magic and version"); + let mut aligned = rkyv::util::AlignedVec::new(); + aligned.extend_from_slice(archive_bytes); + Self { aligned } + } + + /// The validated archived guest input. + pub fn guest_input(&self) -> &crate::recursion::ArchivedContinuationGuestInput { + rkyv::access::( + &self.aligned, + ) + .expect("fixture blob must validate") + } +} + +#[cfg(test)] +mod tests { + use super::{FIXTURE_EPOCH_LOG2, read_inner_elf}; + + /// `SYSCALL_COMMIT` as the guest spells it — the syscall number an `ECALL` + /// log carries in `src1_val`. + const SYSCALL_COMMIT: u64 = 64; + + /// ★ The fixture's THREE MARGINS, asserted rather than recorded in a doc. + /// + /// `machine_tests::continuation_fixture_generates_two_epochs` checks the + /// epoch COUNT, which is necessary and was never sufficient. A run can split + /// in two with its commit on either side of the boundary, and only one of + /// those is a fixture the output-tamper tests can use: `EpochFront::build` + /// harvests epoch 0 and asserts it is INTERMEDIATE, so the public output has + /// to be committed before the first boundary and the guest has to still be + /// running after it. + /// + /// That is the property [`the_closure_rejects_a_moved_index_or_output`]'s + /// second half needs, it is the one that silently went away when the + /// toolchain rebuilt the old guest two instructions shorter, and the epoch + /// count did not move when it did. So it is asserted here, in the same + /// currency the guest's `TAIL_STEPS` is denominated in. + #[test] + fn the_fixture_guest_commits_in_an_intermediate_epoch() { + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let bytes = read_inner_elf(); + let elf = Elf::load(&bytes).expect("the fixture ELF must load"); + let mut executor = Executor::new(&elf, Vec::new()).expect("executor"); + let mut logs = Vec::new(); + while let Some(batch) = executor.resume_with_limit(1 << 20).expect("resume") { + logs.extend_from_slice(batch); + } + + let commit = logs + .iter() + .position(|l| l.src1_val == SYSCALL_COMMIT) + .expect("the fixture guest must COMMIT — that is the whole point of it"); + let total = logs.len(); + let boundary = 1usize << FIXTURE_EPOCH_LOG2; + + println!( + "fixture guest: {total} cycles, commit at {commit}, boundary {boundary} — \ + margins {}/{}/{}", + boundary - commit, + total - boundary, + 2 * boundary - total, + ); + + assert!( + commit < boundary, + "the commit is at cycle {commit} and the epoch boundary at {boundary}: \ + epoch 0 would carry no public output, and every test that tampers it \ + is vacuous. Lower FIXTURE_EPOCH_LOG2 or commit earlier" + ); + assert!( + total > boundary, + "the guest halts at cycle {total}, inside the first {boundary}-cycle \ + epoch: epoch 0 is FINAL and `EpochFront::build`'s `!is_final` fires. \ + Give the guest more tail" + ); + assert!( + total <= 2 * boundary, + "the guest runs {total} cycles over {boundary}-cycle epochs, which is \ + more than two: harmless for the closure test but it makes every \ + fixture prove dearer, and `continuation_fixture_generates_two_epochs` \ + is written for two" + ); + } +} diff --git a/prover/src/lfm/reach_profile.rs b/prover/src/lfm/reach_profile.rs new file mode 100644 index 000000000..642fb0372 --- /dev/null +++ b/prover/src/lfm/reach_profile.rs @@ -0,0 +1,354 @@ +//! ★ How much of an LFM program could run at once — the dependency structure, +//! measured rather than argued. +//! +//! # Why this exists +//! +//! Lane E measured the executor at **83% one software RPX permutation per +//! `Instr::Hash`** (644,250 of a wrap's 5,388,182 instructions, at 2,356 ns a +//! permutation on Zen 5). That makes "run the permutations in parallel" the +//! largest remaining lever on the host — and the case for it so far rests on +//! *reading the emitters*: the per-table legs are transcript-forked, the +//! per-query blocks are contiguous, so the hash DAG "should be" ~10⁴ wide and +//! ~60 deep. +//! +//! ⛔ **That is an argument, not a number, and the campaign's own miss list is +//! mostly arguments that had no row for something the code does.** This module +//! replaces it with a measurement taken from the program itself, before anyone +//! writes a parallel executor. +//! +//! # What it computes, and why the answer is exact rather than heuristic +//! +//! Addresses are dense, assigned in emission order, written once, and every +//! operand address is strictly below its destination (`instr.rs`), so the +//! program IS a topologically sorted DAG and one forward pass suffices. +//! +//! For each instruction the pass carries `hash_depth` = the length of the +//! longest chain of *dependent `Instr::Hash` instructions* ending there. Group +//! the hash instructions by that number and the groups have a property no +//! partition heuristic can claim: +//! +//! > **Two hash instructions at the same level cannot depend on each other.** +//! > If `j` depended on `i` by any path, `hash_depth[j] ≥ hash_depth[i] + 1`, +//! > because `i` is itself a hash. So the levels ARE the available parallelism, +//! > with no segmentation, no guessing, and no soundness argument to make. +//! +//! From the level sizes, the ideal wall at `W` parallel workers is +//! `Σ_level ceil(size / W)` permutation-steps — the list-scheduling bound for +//! unit-cost tasks. [`ReachProfile::describe`] prints that ladder, which is the +//! reading that prices the lever at the worker counts a box can actually run. +//! +//! ⚠ **What the bound is NOT.** It counts permutation STEPS and charges nothing +//! for the memory traffic, the record appends, or the scheduling itself; it +//! assumes a level's work can be handed out freely. It is therefore a FLOOR on +//! a parallel executor's hash phase and an upper bound on the speedup — the +//! honest use is "this is the most the lever could ever be worth", and a +//! measured arm still has to earn it. + +use std::collections::BTreeMap; + +use super::compiler::LfmProgram; +use super::instr::Instr; + +/// The dependency structure of one program, and the const-pool hazard's size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReachProfile { + pub instrs: usize, + pub num_addrs: u64, + /// `Instr::Hash` count — the permutations the executor runs. + pub hash_rows: usize, + /// Longest chain of DEPENDENT hash instructions. The serial floor. + pub hash_depth: usize, + /// Longest chain of dependent instructions of any kind. + pub instr_depth: usize, + /// `hash_levels[k]` = hash instructions at hash-depth `k + 1`. Every entry + /// is a set of mutually independent permutations. + pub hash_levels: Vec, + /// `Instr::Const` count — the globally interned pool, and the one thing + /// that makes a contiguous partition unsafe (`builder.rs` interns across + /// the whole program, so a constant first emitted inside query 5 is read by + /// query 7). + pub consts: usize, + /// Reads whose writer is an `Instr::Const`: the cross-segment edges a + /// const-hoisting pre-pass removes. + pub const_reads: u64, + /// Reads whose address no instruction writes. The compiler's pass-1 + /// tripwire says this is zero; counting it here checks the same invariant + /// from the other side, on a real program, for free. + pub unwritten_reads: u64, +} + +/// Whether the level structure was computed over a program that has any hashes +/// at all — a program with none is legal and its ladder is empty. +impl ReachProfile { + /// Permutation STEPS at `workers` parallel workers: `Σ ceil(level / W)`. + /// + /// At `workers = 1` this is exactly [`Self::hash_rows`] (what the executor + /// does today); as `workers → ∞` it falls to [`Self::hash_depth`]. + pub fn steps_at(&self, workers: usize) -> u64 { + assert!(workers >= 1, "a schedule has at least one worker"); + self.hash_levels + .iter() + .map(|&n| u64::from(n).div_ceil(workers as u64)) + .sum() + } + + /// The widest level — the most permutations that are ever simultaneously + /// available. + pub fn widest_level(&self) -> u32 { + self.hash_levels.iter().copied().max().unwrap_or(0) + } + + /// Mean available width, `hash_rows / hash_depth`. + pub fn mean_width(&self) -> f64 { + if self.hash_depth == 0 { + return 0.0; + } + self.hash_rows as f64 / self.hash_depth as f64 + } + + /// The report. `ns_per_perm` is the measured host cost of one permutation + /// (2,356 on Zen 5, 3,000 on Zen 4) so the ladder reads in seconds rather + /// than in steps. + pub fn describe(&self, label: &str, ns_per_perm: f64) -> String { + use std::fmt::Write as _; + let mut s = String::new(); + let secs = |steps: u64| steps as f64 * ns_per_perm / 1e9; + let _ = writeln!( + s, + " ★ REACH {label}: {} instrs · {} addrs · {} hash rows ({:.1}%)", + self.instrs, + self.num_addrs, + self.hash_rows, + 100.0 * self.hash_rows as f64 / self.instrs.max(1) as f64, + ); + let _ = writeln!( + s, + " depth: hash {} · any-instr {} · mean width {:.0} · widest level {}", + self.hash_depth, + self.instr_depth, + self.mean_width(), + self.widest_level(), + ); + let _ = write!(s, " ideal hash wall:"); + for w in [1usize, 2, 4, 8, 16, 30, 64] { + let _ = write!(s, " W={w} {:.2}s", secs(self.steps_at(w))); + } + let _ = writeln!( + s, + "\n floor (W=∞) {:.3}s = {} steps · speedup ceiling {:.1}×", + secs(self.hash_depth as u64), + self.hash_depth, + if self.hash_depth == 0 { + 0.0 + } else { + self.hash_rows as f64 / self.hash_depth as f64 + }, + ); + let _ = writeln!( + s, + " const pool: {} consts, {} reads of them (the cross-block edges a \ + hoist removes) · unwritten reads {}", + self.consts, self.const_reads, self.unwritten_reads, + ); + // A coarse shape of the ladder: where the work sits, not every level. + let mut buckets: BTreeMap = BTreeMap::new(); + for &n in &self.hash_levels { + let bucket = if n == 0 { 0 } else { n.next_power_of_two() }; + let e = buckets.entry(bucket).or_default(); + e.0 += 1; + e.1 += u64::from(n); + } + let _ = write!(s, " level widths (≤2^k → levels, hashes):"); + for (b, (levels, hashes)) in buckets { + let _ = write!(s, " {b}→{levels}/{hashes}"); + } + s.push('\n'); + s + } +} + +/// One forward pass over the program. `O(instructions + addresses)` time and +/// `4 · addresses + 8 · instructions` bytes of scratch. +/// +/// ⚠ Uses [`Instr::reads`] and [`Instr::writes`] rather than a second +/// transcription of the operand conventions. They allocate a small `Vec` per +/// call, which costs this analysis a few seconds on a 7M-instruction program — +/// deliberately paid, because a private copy of "which operands are live under +/// which selector" is exactly the kind of duplicate that drifts and then +/// reports a dependency structure the machine does not have. +pub fn profile(program: &LfmProgram) -> ReachProfile { + const NO_WRITER: u32 = u32::MAX; + let n = program.instrs.len(); + assert!( + n < NO_WRITER as usize, + "the profile indexes instructions in a u32; this program has {n}" + ); + + let mut writer = vec![NO_WRITER; program.num_addrs as usize]; + let mut hash_depth = vec![0u32; n]; + let mut instr_depth = vec![0u32; n]; + let mut levels: Vec = Vec::new(); + let (mut consts, mut hash_rows) = (0usize, 0usize); + let (mut const_reads, mut unwritten_reads) = (0u64, 0u64); + + for (i, instr) in program.instrs.iter().enumerate() { + let (mut hd, mut id) = (0u32, 0u32); + for r in instr.schedule_reads() { + let w = writer[r.0 as usize]; + if w == NO_WRITER { + unwritten_reads += 1; + continue; + } + if matches!(program.instrs[w as usize], Instr::Const { .. }) { + const_reads += 1; + } + hd = hd.max(hash_depth[w as usize]); + id = id.max(instr_depth[w as usize]); + } + let is_hash = matches!(instr, Instr::Hash { .. } | Instr::Hash16(_)); + if is_hash { + hash_rows += 1; + hd += 1; + let k = hd as usize - 1; + if levels.len() <= k { + levels.resize(k + 1, 0); + } + levels[k] += 1; + } + if matches!(instr, Instr::Const { .. }) { + consts += 1; + } + hash_depth[i] = hd; + instr_depth[i] = id + 1; + for a in instr.writes() { + writer[a.0 as usize] = i as u32; + } + } + + ReachProfile { + instrs: n, + num_addrs: program.num_addrs, + hash_rows, + hash_depth: levels.len(), + instr_depth: instr_depth.iter().copied().max().unwrap_or(0) as usize, + hash_levels: levels, + consts, + const_reads, + unwritten_reads, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lfm::builder::LfmBuilder; + use crate::lfm::compiler::compile; + use crate::tables::types::FE; + + fn word(v: u64) -> crate::lfm::word::LfmWord { + core::array::from_fn(|i| FE::from(v + i as u64)) + } + + /// ★★ The gate: a program whose parallel structure is known by + /// construction, and the profile must report exactly it. + /// + /// Four independent chains of three compressions. The right answer is three + /// levels of four, a speedup ceiling of 4×, and a W=4 schedule that takes + /// three steps where W=1 takes twelve. + #[test] + fn the_profile_reports_a_known_chain_structure() { + let mut b = LfmBuilder::new(); + let seed = b.digest_const(word(1)); + for chain in 0..4u64 { + let mut acc = b.digest_const(word(100 + chain)); + for _ in 0..3 { + acc = b.compress(acc, seed); + } + } + let p = profile(&compile(b.finish())); + + assert_eq!(p.hash_rows, 12, "twelve compressions"); + assert_eq!(p.hash_depth, 3, "three deep"); + assert_eq!( + p.hash_levels, + vec![4, 4, 4], + "four independent at each depth" + ); + assert_eq!(p.widest_level(), 4); + assert_eq!(p.steps_at(1), 12, "one worker does every permutation"); + assert_eq!(p.steps_at(2), 6); + assert_eq!(p.steps_at(4), 3, "four workers reach the critical path"); + assert_eq!(p.steps_at(30), 3, "and cannot beat it"); + assert_eq!(p.unwritten_reads, 0, "every read has a writer"); + } + + /// The counter-case, so the test above is not passing for a shape-blind + /// reason: ONE chain of twelve is depth twelve and width one, and no worker + /// count helps. + #[test] + fn a_single_chain_has_no_parallelism_to_find() { + let mut b = LfmBuilder::new(); + let seed = b.digest_const(word(1)); + let mut acc = b.digest_const(word(7)); + for _ in 0..12 { + acc = b.compress(acc, seed); + } + let p = profile(&compile(b.finish())); + + assert_eq!(p.hash_rows, 12); + assert_eq!(p.hash_depth, 12, "a chain is as deep as it is long"); + assert_eq!(p.hash_levels, vec![1; 12]); + assert_eq!(p.steps_at(30), 12, "thirty workers buy nothing"); + } + + /// ★ The const-pool hazard, resolved as a property rather than a hope. + /// + /// `builder.rs` interns constants across the WHOLE program, so a contiguous + /// partition has cross-block read edges into `Instr::Const` cells. Hoisting + /// every `Const` to a pre-pass removes those edges — and is only sound + /// because a `Const` reads nothing, which is what this asserts. If a future + /// `Const` variant ever gained an operand, a parallel executor built on the + /// hoist would race, and this test is what would say so first. + #[test] + fn a_const_depends_on_nothing_so_hoisting_it_is_sound() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(FE::from(7)); + let y = b.felt_const(FE::from(5)); + let s = b.add(x, y); + let _ = b.mul(s, y); + let d = b.digest_const(word(3)); + let _ = b.compress(d, d); + let program = compile(b.finish()); + + let mut consts = 0; + for i in &program.instrs { + if matches!(i, Instr::Const { .. }) { + consts += 1; + assert!( + i.reads().is_empty(), + "an Instr::Const must read nothing, or the const hoist that \ + makes a contiguous partition safe is unsound: {i:?}" + ); + } + } + assert!(consts > 0, "the program must contain constants to check"); + assert_eq!(profile(&program).consts, consts, "the profile counts them"); + } + + /// The describe() line renders, carries the ladder, and does not panic on a + /// program with no hashes at all. + #[test] + fn a_program_without_hashes_profiles_to_an_empty_ladder() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(FE::from(7)); + let y = b.felt_const(FE::from(5)); + let _ = b.add(x, y); + let p = profile(&compile(b.finish())); + + assert_eq!(p.hash_rows, 0); + assert_eq!(p.hash_depth, 0); + assert_eq!(p.steps_at(4), 0); + assert_eq!(p.mean_width(), 0.0); + assert!(p.describe("empty", 2356.0).contains("0 hash rows")); + } +} diff --git a/prover/src/lfm/registry.rs b/prover/src/lfm/registry.rs new file mode 100644 index 000000000..ed25b0e6c --- /dev/null +++ b/prover/src/lfm/registry.rs @@ -0,0 +1,1551 @@ +//! `LFM_REGISTRY` — the machine's program registry. +//! +//! A program is nothing but a vector of supplied preprocessed roots plus this +//! table's entry. Maintained exactly like `compute_static_commitments`: +//! regenerated by `cargo run --bin compute_lfm_registry --release`, pinned by +//! drift tests that run on every PR, and governed by the same standing policy +//! — **a drift failure is investigated, never re-blessed to silence the +//! test.** +//! +//! Verify-side resolution returns a hard `Err` on a missing entry. There is +//! **no runtime off-switch**: no environment variable, no feature flag, no +//! fallback that recomputes or skips. The registry check is the soundness +//! argument's first premise (see `SOUNDNESS.md`). + +use stark::config::Commitment; +use stark::proof::options::ProofOptions; + +use crate::tables::{bitwise, keccak_rc}; + +use super::airs::{BLAKE3_SLOT, ChipSet, HASH_SLOT, NUM_LFM_CHIPS, blake3_chunk_rows}; + +use super::commit::commit_group_device_or_host_under; +use super::compiler::{ColumnGroup, LfmProgram}; +use super::hash::HasherKind; +use super::merge::{LfmMerge, MergePlan, MergedTable}; +use super::statement::lfm_program_id_merged_under; +use super::trace::range_group; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LfmProgramKind { + /// The Milestone-B trivial program (`programs::trivial_program`). + TrivialV0, + /// The Milestone-C FRI-opening verifier (`programs::fri_toy_program`). + FriToyV0, + /// The R1b two-permutation keccak chain (`programs::keccak_chain_program`). + KeccakChainV0, + /// The R1c keccak256 sponge at `programs::KECCAK_SPONGE_LEN` bytes. + KeccakSpongeV0, + /// The R1d scripted `DefaultTranscript` replay + /// (`programs::transcript_replay_program`). + TranscriptReplayV0, + /// The R1e continuation-epoch statement bind plus Phase A + /// (`programs::statement_replay_program`). + StatementReplayV0, +} + +impl LfmProgramKind { + /// The fixture program this kind names, built from code — what + /// `compute_lfm_registry` blesses into the row. + pub fn program(self) -> LfmProgram { + use super::programs::{ + KECCAK_SPONGE_LEN, fri_toy_program, keccak_chain_program, keccak_sponge_program, + statement_replay_program, transcript_replay_program, trivial_program, + }; + match self { + Self::TrivialV0 => trivial_program(), + Self::FriToyV0 => fri_toy_program(), + Self::KeccakChainV0 => keccak_chain_program(), + Self::KeccakSpongeV0 => keccak_sponge_program(KECCAK_SPONGE_LEN), + Self::TranscriptReplayV0 => transcript_replay_program(), + Self::StatementReplayV0 => statement_replay_program(), + } + } +} + +/// ★ The artifacts a fixture program is verified against under `options`. +/// +/// The registry policy: `LFM_REGISTRY` is blessed at +/// today's leaf layout and STAYS row-pair only. At the default format this is +/// [`resolve`] — the registry row, no fallback. Under a one-row format (`On` +/// or `Auto`) the registry is NOT read: the program is rebuilt from code and +/// its artifacts computed at run time (row-pair AND one-row roots, as +/// `compute_lfm_registry` would), which is what the registry pins anyway — +/// `registry_drift_*` hold the two equal at the default. +pub fn resolve_artifacts( + kind: LfmProgramKind, + options: &ProofOptions, +) -> Result { + if options.format.one_row == stark::proof::options::OneRowMode::Off + && !options.format.merge_equal_height_tables + { + return Ok(resolve(kind, options.blowup_factor)?.artifacts()); + } + Ok(build_artifacts(&kind.program(), options)) +} + +#[cfg(test)] +thread_local! { + /// Reads of `LFM_REGISTRY` on this thread (test builds only), for the + /// registry-policy test. + pub(crate) static REGISTRY_READS: core::cell::Cell = + const { core::cell::Cell::new(0) }; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LfmRegistryError { + UnknownProgram { + kind: LfmProgramKind, + blowup_factor: u8, + }, + /// Two or more rows claim the same `(kind, blowup_factor)`. + /// + /// The table's key is that pair and nothing else — not the hasher, not the + /// chunk count — so a second row for a pair that already exists is not an + /// alternative a caller can select. It is unreachable data, and the + /// reachable row is whichever the generator happened to print first. See + /// [`resolve`]. + AmbiguousProgram { + kind: LfmProgramKind, + blowup_factor: u8, + rows: usize, + }, +} + +/// One blessed program: its committed roots, its heights and the pieces of +/// proof shape a verifier must not read off the proof. +/// +/// ★ **There is deliberately no `blake3_chunks` column**, unlike +/// [`Self::keccak_rnd_chunks`]. Every registered program is a fixture that +/// compresses a handful of blocks at most, so all six sit at the policy default +/// of ONE `LFM_BLAKE3` table, and a column that can only ever read 1 would be a +/// shape a verifier could get wrong for no reason. [`Self::artifacts`] derives +/// it, and `every_registry_entry_is_a_single_blake3_table` pins the premise. +/// Blessing a chunked program means adding the column then, alongside the row. +pub struct LfmRegistryEntry { + pub kind: LfmProgramKind, + pub blowup_factor: u8, + pub roots: [Commitment; NUM_LFM_CHIPS], + pub log_heights: [u8; NUM_LFM_CHIPS], + /// `KECCAK_RND` instances this program is proved and verified with. + pub keccak_rnd_chunks: usize, + /// The `LFM_HASH` permutation this program is proved and verified under. + /// + /// Program shape, like the roots and the chunk count — never read off the + /// proof. `lfm_verify` builds the AIR set with this value, and it is folded + /// into `program_id`, so the digest and the permutation it names are + /// computed together and cannot be paired up wrongly at verify time. + pub hasher: HasherKind, + /// Which hash-family chip groups this program instantiates. Program shape + /// for the same reason and read the same way — from this entry, never from + /// the proof. See [`ChipSet`]. + pub chip_set: ChipSet, + pub program_id: Commitment, +} + +impl LfmRegistryEntry { + /// This entry as [`LfmArtifacts`] — the shape both verify paths read. + /// + /// The `LFM_BLAKE3` chunk list is DERIVED rather than stored: every + /// registered program sits at the policy default of one table, so the + /// committed list is exactly slot 11's own root and height. See the note on + /// the struct for why there is no column, and + /// `every_registry_entry_is_a_single_blake3_table` for what keeps the + /// premise honest. + pub fn artifacts(&self) -> LfmArtifacts { + LfmArtifacts { + roots: self.roots, + log_heights: self.log_heights, + keccak_rnd_chunks: self.keccak_rnd_chunks, + blake3_chunk_roots: vec![self.roots[BLAKE3_SLOT]], + blake3_chunk_log_heights: vec![self.log_heights[BLAKE3_SLOT]], + // One `LFM_HASH` table, like one `LFM_BLAKE3` table: the hash split divides + // nothing a registry entry names. + hash_chunk_roots: vec![self.roots[HASH_SLOT]], + hash_chunk_log_heights: vec![self.log_heights[HASH_SLOT]], + hasher: self.hasher, + chip_set: self.chip_set, + program_id: self.program_id, + // The registry is ROW-PAIR ONLY: a one-row + // format never reads it — `resolve_artifacts` builds at run time. + one_row_roots: None, + // And one AIR a chip: a merging format never reads it either. + merge: LfmMerge::none(), + } + } +} + +/// A program's committed artifacts (what a registry entry pins). +/// +/// `PartialEq` + `Debug` because the gates compare two builds FIELD BY FIELD. +/// Comparing `program_id` alone would pass on a value whose roots had drifted, +/// since a digest is only as binding as the fields a test actually reads. +/// `Clone` because the value is a few hundred bytes — fifteen roots, fifteen +/// heights and two short chunk lists — and callers that hold one while handing +/// another onward should not have to thread a borrow to do it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LfmArtifacts { + pub roots: [Commitment; NUM_LFM_CHIPS], + pub log_heights: [u8; NUM_LFM_CHIPS], + pub keccak_rnd_chunks: usize, + /// One preprocessed root per `LFM_BLAKE3` chunk, in chunk order — empty when + /// the BLAKE3 family is absent. + /// + /// The chip's chunks are NOT interchangeable the way `KECCAK_RND`'s are: + /// each commits its own slice of the instruction column group, so each has + /// its own root and its own height. `blake3_chunk_roots[0]` IS + /// `roots[BLAKE3_SLOT]` and `blake3_chunk_log_heights[0]` IS + /// `log_heights[BLAKE3_SLOT]`, which is what makes a single-chunk program + /// bit-identical to an unchunked one; chunks 1.. live only here and are + /// folded into `program_id` as a tail (see + /// [`lfm_program_id`](super::statement::lfm_program_id)). + pub blake3_chunk_roots: Vec, + /// Trace log-height of each `LFM_BLAKE3` chunk, in chunk order. + pub blake3_chunk_log_heights: Vec, + /// One preprocessed root per `LFM_HASH` chunk, in chunk order — one entry + /// unless the hash split divided the table (`chunking::HashChunking`). + /// `hash_chunk_roots[0]` IS `roots[HASH_SLOT]`, so an unsplit program is + /// bit-identical to the machine before the split existed; chunk 1 is folded + /// into `program_id` as a tail, like the `LFM_BLAKE3` chunks. + pub hash_chunk_roots: Vec, + /// Trace log-height of each `LFM_HASH` chunk, in chunk order. + pub hash_chunk_log_heights: Vec, + /// The hasher `program_id` was derived under; the prove and verify paths + /// both take it from here rather than defaulting. + pub hasher: HasherKind, + /// The hash families this program instantiates, derived from its own + /// compiled groups at bless time. See [`ChipSet`]. + pub chip_set: ChipSet, + pub program_id: Commitment, + /// The ONE-ROW (S2) preprocessed roots of the same groups, built only when + /// the options' format has one-row openings on (`On` or `Auto`: which chips + /// `Auto` resolves to one row is decided later, per chip, by the STARK + /// prover and verifier, so every chip gets one). `None` at the default. + /// + /// NOT folded into `program_id`: the identity stays the row-pair roots' + /// (the one-row roots are a deterministic function of the same columns), + /// so a program keeps one id across layouts on the LFM side. + pub one_row_roots: Option, + /// The merged AIRs (`LAMBDA_VM_ZF_LFM_MERGE`, [`super::merge`]): which + /// chips of equal padded height this program proves as one AIR, and each + /// one's preprocessed root. Empty — the default — is one AIR a chip. Built + /// from the program's heights under the options' format, folded into + /// `program_id`; the prove and verify paths take it from here. + pub merge: LfmMerge, +} + +/// The one-row preprocessed roots of an [`LfmArtifacts`] (see its field). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LfmOneRowRoots { + /// Per chip slot, as `LfmArtifacts::roots`; `None` = no one-row root (a + /// static table with no one-row twin at this blowup — a hard miss if the + /// chip resolves to one row). Slot 12 (`KECCAK_RND`) has no + /// preprocessed columns and stays `None`. + pub roots: [Option; NUM_LFM_CHIPS], + /// One per `LFM_BLAKE3` chunk, as `LfmArtifacts::blake3_chunk_roots`. + pub blake3_chunk_roots: Vec, + /// One per `LFM_HASH` chunk, as `LfmArtifacts::hash_chunk_roots`. + pub hash_chunk_roots: Vec, +} + +impl LfmArtifacts { + /// `LFM_BLAKE3` instances this program COMMITS — never zero, since slot 11's + /// group is committed even for a program that never compresses. + /// + /// Whether a PROOF carries them is the chip mask's decision, taken where it + /// always was: [`ChipSet::num_airs`](super::airs::ChipSet::num_airs) gates on + /// `blake3`, and [`LfmAirs::air_refs`](super::airs::LfmAirs::air_refs) omits + /// the instances. That is why this reports the committed count rather than + /// the gated one, unlike [`Self::keccak_rnd_chunks`] — `KECCAK_RND` commits + /// nothing, so it can legitimately drop to zero instances. + pub fn blake3_chunks(&self) -> usize { + self.blake3_chunk_roots.len() + } +} + +/// Commits every instruction column group (plus the fixed tables) at the given +/// options and derives the program digest. Host-side, seconds — there is no +/// keygen in this framework. +/// +/// What the digest binds, per slot class: +/// +/// - **slots 0–9** — the program-dependent instruction column groups. Root and +/// height both vary with the program; this is the program's identity. +/// - **slot 10 (`LFM_RANGE`)** — a fixed table whose group is +/// program-independent but still committed and bound, so a change to the +/// table or to the commit pipeline moves every program digest. +/// - **slot 11 (`LFM_BLAKE3`)** — a program-dependent instruction column group +/// like slots 0–9, placed after `LFM_RANGE` so the hosted keccak family stays +/// contiguous at the end (`airs::KECCAK_RND_SLOT`). ★ It is also SPLITTABLE +/// (`chunking::Blake3Chunking`), and unlike `KECCAK_RND` it commits: a chunked +/// program's chunk 0 is what this slot holds, and chunks 1.. ride +/// [`LfmArtifacts::blake3_chunk_roots`], bound into the digest as a tail. A +/// single-chunk program — every registered one — is bit-identical to the +/// unchunked machine, which is why no blessed digest moved when chunking +/// arrived. +/// - **slots 13–14 (`KECCAK_RC`, `BITWISE`)** — same treatment as `LFM_RANGE`, +/// except their preprocessed columns are owned by `tables/`, so the roots come +/// from those modules' own `preprocessed_commitment` — which is both what the +/// AIRs are built with and what the prover recommits against. Binding them +/// means a change to either production table moves every LFM program digest; +/// that is deliberate, since those tables are now part of the machine. +/// - **slot 12 (`KECCAK_RND`)** — has no preprocessed columns at all, so there +/// is nothing to commit. Its entry stays the all-zero sentinel at height 0 and +/// binds nothing. Sound because the chip is program-independent in both +/// directions: its constraints are fixed, and its trace height is free (extra +/// rows are padding with `MU = 0`, which emits no bus tokens and satisfies +/// every constraint) — the same freedom the production VM's chips have. +/// What the entry *does* pin for this slot is `keccak_rnd_chunks`: how many +/// instances of it the proof carries. That freedom is the same freedom as the +/// height — a prover who used a different count could not forge anything, +/// only fail to balance the `Keccak` bus (too few chunks) or waste rows (too +/// many) — so pinning it is a shape decision, not a soundness gate. Pinning +/// it here keeps the verifier's AIR set derivable from the registry alone, +/// with nothing about proof shape read off the proof. +/// +/// The `LFM_HASH` permutation is bound too, but not through a root: it selects +/// which chip fills the `LFM_HASH` slot, and every candidate's preprocessed +/// width is the same (the instruction group is hasher-independent), so no root +/// moves with the choice. [`build_artifacts_with_hasher`] folds the kind's tag +/// into the digest instead — measured by +/// `the_blake3_choice_moves_the_program_digest_and_no_root`. +/// +/// That is a statement about the machine's *own* hash, not about the hash these +/// roots are built with. The two are separate axes and both are now named in +/// `program_id` — see [`build_artifacts_with_hasher`]. If the machine's hash +/// ever also selects the commitment scheme the roots are committed under, every +/// root above moves with it and the tag on its own stops being the whole +/// binding. +/// ★ The `LFM_HASH` permutation `LFM_REGISTRY` is blessed under. +/// +/// Bound into every digest in that table — `HasherKind::as_tag` is folded into +/// `lfm_program_id` — so changing it here is a re-blessing of the whole table, +/// not a re-run. **A second hasher becomes additional ROWS, never a silent +/// replacement of these.** +/// +/// ⚠ Lives here rather than in `compute_lfm_registry` because it is a property +/// of the TABLE, not of the generator, and because [`build_artifacts`] has to +/// name it: a default spelled `HasherKind::default()` is a silent global, while +/// one spelled `REGISTRY_HASHER` is a statement about registry identity that a +/// reader can check against the table. +pub const REGISTRY_HASHER: HasherKind = HasherKind::Test; + +pub fn build_artifacts(program: &LfmProgram, options: &ProofOptions) -> LfmArtifacts { + // ⛔ **DEFAULTS TO `Test`, AND MUST.** The hasher is part of program + // IDENTITY — `HasherKind::as_tag` is folded into `lfm_program_id` — and + // `LFM_REGISTRY` is blessed under `compute_lfm_registry`'s + // `REGISTRY_HASHER = Test`, whose own doc calls changing it a re-blessing of + // the whole table rather than a re-run. + // + // ⚠ This entry point was briefly changed to name `hash_pin::BLOCK_HASHER`, + // to fix a real defect on the AGGREGATOR path where `lfm_prove_batched` + // inherited a toy permutation from a defaulted build. That fix was correct + // about the defect and wrong about its scope: it moved every registry + // program's identity away from the blessed table, and + // `rpo_chip_tests::the_rpo_choice_moves_the_program_digest_and_no_root` + // asserts this function defaults to `Test` in as many words. + // + // ★ The block path names its hasher AT THE CALL SITE + // ([`build_artifacts_with_hasher`]) instead. Registry programs pin a byte + // hash on their own builders, emit no `Instr::Hash`, and never consult the + // socket — they are correct at `Test` under every pin. + build_artifacts_with_hasher(program, options, REGISTRY_HASHER) +} + +/// How many committed groups the artifact build expands at once. +/// +/// Four by default, and the number is a RESIDENCY bound rather than a +/// parallelism target. +/// +/// ⚠ The pass was never STRICTLY one-core — `dispatch_fft` routes any buffer of +/// 2^14 elements or more to the parallel Bowers FFT — but it is nowhere near +/// full either, and the difference is the point. `bowers_fft.rs`'s per-layer +/// `adaptive_parallel_threshold` is `max(num_threads * 4, 16)` BLOCKS, and the +/// early layers of a single column have far fewer blocks than that, so they run +/// serially over the whole array. ✓ MEASURED by lane P on a production node: +/// `emit_merkle` 2,652% CPU (26.5 of 30.7 cores), `emit_lde` **445% (4.5 +/// cores)**. The column-level `par_iter` in [`lde_columns`] is what overlaps one +/// column's serial early layers with another's, and that is where the headroom +/// is. This window is the second-order half — it covers the SHORT groups, under +/// 2^14 rows, where each column's FFT stays sequential and a width of 1, 2 or 3 +/// cannot fill anything alone — while multiplying the build's peak host +/// residency by at most four rather than by the group count. +/// +/// `LFM_ARTIFACT_GROUPS_IN_FLIGHT=1` walks the groups one at a time and leaves +/// the column loops parallel; [`parallel_build`](super::commit::parallel_build) +/// is the knob that restores the pre-change pass entirely. +pub fn groups_in_flight() -> usize { + static N: std::sync::OnceLock = std::sync::OnceLock::new(); + *N.get_or_init(|| { + // An empty value reads as unset — see `commit::parallel_build`. + match std::env::var("LFM_ARTIFACT_GROUPS_IN_FLIGHT") { + Ok(v) if v.is_empty() => 4, + Ok(v) => v + .parse::() + .ok() + .filter(|n| *n > 0) + .unwrap_or_else(|| { + panic!("LFM_ARTIFACT_GROUPS_IN_FLIGHT must be a positive integer, got `{v}`") + }), + Err(_) => 4, + } + }) +} + +/// `items.iter().map(f).collect()`, on the global rayon pool where there is one. +/// +/// ⛔ Indexed and ordered on both arms. `par_iter().map().collect()` over a slice +/// preserves position, so slot `k` of a window is slot `k` of the result and the +/// roots land where the serial walk put them. A `for_each` writing through a +/// shared handle would not have that property for free. +fn map_maybe_parallel(items: &[T], f: impl Fn(&T) -> R + Sync + Send) -> Vec +where + T: Sync, + R: Send, +{ + #[cfg(feature = "parallel")] + if super::commit::parallel_build() { + use rayon::prelude::*; + return items.par_iter().map(f).collect(); + } + items.iter().map(f).collect() +} + +/// The dispatch label an artifact-prep commit carries into `gpu_lde`'s +/// diagnostics. It names the PHASE rather than a chip, because the abort a +/// reader would be holding says "which commit was this" and every group in this +/// walk answers that the same way; the group's own shape is in the same message. +const PREP_GROUP_LABEL: &str = "LFM_PREP_GROUP"; + +/// The same, for the chunked `LFM_BLAKE3` groups — held apart so an abort says +/// which of the two walks it came from. +const BLAKE3_CHUNK_LABEL: &str = "LFM_PREP_BLAKE3_CHUNK"; + +/// The same, for the `LFM_HASH` tail chunk. +const HASH_CHUNK_LABEL: &str = "LFM_PREP_HASH_CHUNK"; + +/// Slots 0..=9 — the instruction column groups that belong to the PROGRAM. +/// +/// Slot 10 (`LFM_RANGE`) is committed with them and is not one of them: its +/// group comes from [`range_group`], which takes no arguments and so cannot +/// vary with the program. +pub const PROGRAM_GROUP_SLOTS: usize = 10; + +/// The program's own committed instruction column groups, in slot order. +/// +/// Named rather than written inline in [`build_artifacts_with_hasher`] so the +/// build's group list is one readable thing, and so the windowed walk below can +/// index it. `LFM_RANGE` is appended by the build at slot 10; `LFM_BLAKE3` is +/// slot 11 and is committed per chunk, which is why neither is here. +pub fn program_groups(program: &LfmProgram) -> [&ColumnGroup; PROGRAM_GROUP_SLOTS] { + [ + &program.groups.const_, + &program.groups.balu, + &program.groups.xalu, + &program.groups.select, + &program.groups.bitdec, + &program.groups.hash, + &program.groups.keccak, + &program.groups.lanes, + &program.groups.hint, + &program.groups.public, + ] +} + +/// [`build_artifacts`] for a program proved under an explicitly chosen +/// `LFM_HASH` permutation. +/// +/// The returned artifacts carry `hasher`, and `program_id` is derived from it — +/// so the same program under two hashers is two program identities, and the +/// prove/verify paths that read `LfmArtifacts` cannot pair one hasher's digest +/// with another hasher's AIR set. +/// +/// # `hasher` and the commitment hash are two axes, and both are named +/// +/// `hasher` names the `LFM_HASH` chip the machine RUNS. The hash the roots below +/// are BUILT with is a different question: `commit_group` and the two +/// `preprocessed_commitment` helpers all commit through `stark`'s Merkle layer +/// under whatever [`crate::hash_pin::BLOCK_COMMITMENT_HASH`] names — BLAKE3 on +/// this branch, keccak on a `cuda` build, an algebraic hash on a branch that +/// pins one. +/// +/// Both are folded into `program_id` (see +/// [`lfm_program_id`](super::statement::lfm_program_id)), which is what +/// discharges the compile-time guard this function used to carry. That guard +/// refused to compile once `stark` gained a second commitment hash, on the +/// grounds that artifacts naming one hash over roots built with another is a +/// claim nothing downstream would check. Naming the second axis is the decision +/// it was holding out for: the artifacts now say which hash built the roots, so +/// there is no unstated claim left to be wrong about. +/// +/// ★ **The read has MOVED, on exactly the condition this note set.** The commit +/// helpers were hard-wired to `stark`'s default aliases, so the global const was +/// then the truthful name for their output. They are now threaded through +/// [`crate::hash_pin::BlockStarkHash`] — the block path's own configuration, +/// which a hash-comparison branch re-points without touching the workspace +/// default — and `lfm_program_id` reads `BLOCK_COMMITMENT_HASH` with them. The +/// two axes still move together, which is the property that matters; what +/// changed is that the axis they follow is the pin rather than the alias. +/// +/// ⚠ Still deliberately NOT done: an `H` PARAMETER on this function. Threading +/// the helpers to a build-wide pin keeps one answer per build; a parameter would +/// reintroduce the failure this note was guarding against, artifacts naming an +/// `H` the helpers did not use. +pub fn build_artifacts_with_hasher( + program: &LfmProgram, + options: &ProofOptions, + hasher: HasherKind, +) -> LfmArtifacts { + build_artifacts_under::(program, options, hasher) +} + +/// [`build_artifacts_with_hasher`] with every root built under the commitment +/// configuration `S` — a proof family's (`hash_pin::ProofFamily::Stark`) — +/// and `program_id` naming `S`'s hash: the artifacts a program's proofs under +/// that family are made and checked against. The parameter is the one hash +/// both the commits and the identity read, so they cannot name two hashes. +/// At the legacy pin it is [`build_artifacts_with_hasher`] byte for byte. +pub fn build_artifacts_under( + program: &LfmProgram, + options: &ProofOptions, + hasher: HasherKind, +) -> LfmArtifacts { + let plan = merge_plan_for(program, options); + let merged = super::merge::merged_groups(program, &plan); + build_artifacts_merged_under::(program, options, hasher, &plan, &merged) +} + +/// The merge `options` asks of `program`: its equal-height candidates under +/// `merge_equal_height_tables` ([`super::merge::plan_for_program`]), none +/// otherwise. Also none beside one-row openings, which the format refuses by +/// name (`ZfFormat::check_lfm_merge`): a merged AIR has no one-row root, and a +/// caller that sets both directly gets one AIR a chip on the prove and the +/// verify side alike, since both read this. +pub fn merge_plan_for(program: &LfmProgram, options: &ProofOptions) -> MergePlan { + let on = options.format.merge_equal_height_tables + && options.format.one_row == stark::proof::options::OneRowMode::Off; + super::merge::plan_for_program(program, on) +} + +/// [`build_artifacts_under`] with the merged groups already laid side by side +/// (`merged[k]` is `plan[k]`'s, [`super::merge::merged_groups`]), so a caller +/// can build them before it takes the card ([`super::program_census`]). +pub(crate) fn build_artifacts_merged_under( + program: &LfmProgram, + options: &ProofOptions, + hasher: HasherKind, + plan: &MergePlan, + merged: &[ColumnGroup], +) -> LfmArtifacts { + use stark::leaf_layout::LeafLayout::RowPair; + debug_assert_eq!(plan.len(), merged.len(), "one merged group per merged AIR"); + let range = range_group(); + // Slots 0..=10 in slot order: the program's own nine groups, then + // `LFM_RANGE`. Slot 11 (`LFM_BLAKE3`) is not here because it is the CHUNKED + // one: it contributes one committed matrix per chunk, built and absorbed + // after this list, which is where slot order puts it anyway. + let program_slots = program_groups(program); + // A split `LFM_HASH` commits chunk 0 at slot 5 and chunk 1 as a + // tail below. An unsplit program materializes nothing here — slot 5 is the + // compiled group itself, exactly as before the split existed. + let hash_chunks: Vec = if program.hash_chunk_count() > 1 { + (0..program.hash_chunk_count()) + .map(|c| program.hash_chunk_group(c)) + .collect() + } else { + Vec::new() + }; + let groups: [&ColumnGroup; 11] = std::array::from_fn(|i| { + if i == HASH_SLOT && !hash_chunks.is_empty() { + &hash_chunks[0] + } else if i < PROGRAM_GROUP_SLOTS { + program_slots[i] + } else { + &range + } + }); + let mut roots = [[0u8; 32]; NUM_LFM_CHIPS]; + let mut log_heights = [0u8; NUM_LFM_CHIPS]; + + // Heights first, from the compiled groups: the LDE walk below commits them + // and the digest binds them, so both read one derivation. + for (i, g) in groups.iter().enumerate() { + log_heights[i] = g.padded_rows.trailing_zeros() as u8; + } + // The chunk heights are arithmetic (`blake3_chunk_rows`), so they are known + // without materializing a single chunk group. Slot 11's own entry is chunk + // 0's — the array stays the shape a single-table program has. + let blake3_chunk_log_heights: Vec = blake3_chunk_rows(program) + .into_iter() + .map(|rows| rows.trailing_zeros() as u8) + .collect(); + log_heights[BLAKE3_SLOT] = blake3_chunk_log_heights[0]; + + // ★ N GROUPS IN FLIGHT, NOT ONE AND NOT ELEVEN. Each pass is an independent + // expand-and-commit of its own matrix, so the loop parallelizes on its face; + // what stopped it was the residency trade the old comment named — "peak + // residency is one group's LDE" — taken when host memory was the binding + // constraint on this path. It is not any more: lane P measured a production + // L1 node's host peak at 14.4 GiB of 57.53. + // + // ⚠ So the trade is RE-PRICED, not discarded. Running all eleven at once + // would multiply the peak by the group count for a program whose group sizes + // we do not control; a fixed window multiplies it by at most + // `groups_in_flight()` and says so. + // + // ⛔ AND THE WINDOW IS THE SECOND-ORDER HALF. `lde_columns` → `dispatch_fft` + // already sends any buffer of 2^14 elements or more to the parallel Bowers + // FFT, so the work inside this loop was never strictly serial — but its + // per-layer block threshold leaves the early layers sequential, and lane P + // measures the result at 445% CPU (4.5 of 30.7 cores) on a production node + // against `emit_merkle`'s 2,652%. The COLUMN-level `par_iter` is what closes + // that gap; this window covers only the short groups, where each column's + // FFT stays sequential. + // + // ⚠ My own laptop A/B read both as a wash (serial 1.547 s, windowed 1.447 s + // on a 27 MiB program). That fixture concentrates its felts in ONE NARROW + // group, which is close to the worst case for a per-column spread — it is a + // statement about the fixture, not about the change. Lane P's production + // split is the number to plan against. + // + // ⓘ ON A CUDA BUILD THE WINDOW IS A HOST-FALLBACK BOUND, not the live one. + // `commit_group_device_or_host` sends each group to the card, where the + // residency that matters is `stark::device_set`'s and admission enforces it + // per call. The window still bounds the host path exactly as before, which + // is the path a machine with no card takes. + // + // ★ A MERGED chip is not committed on its own: its columns are committed + // inside its merged AIR's group, and its slot keeps the all-zero root (the + // merged table's root is bound into `program_id` by the merge tail). The + // merged groups follow the slots in the same windows, so with no merge the + // walk is the eleven slots in slot order, as it always was. + enum Job<'a> { + Slot(usize, &'a ColumnGroup), + Merged(usize, &'a ColumnGroup), + } + let jobs: Vec> = groups + .iter() + .enumerate() + .filter(|(i, _)| super::merge::group_of(plan, *i).is_none()) + .map(|(i, g)| Job::Slot(i, g)) + .chain(merged.iter().enumerate().map(|(k, g)| Job::Merged(k, g))) + .collect(); + let in_flight = groups_in_flight(); + let mut merged_roots: Vec = vec![[0u8; 32]; merged.len()]; + for window in jobs.chunks(in_flight) { + let commits = map_maybe_parallel(window, |job| { + let g = match job { + Job::Slot(_, g) | Job::Merged(_, g) => g, + }; + commit_group_device_or_host_under::(PREP_GROUP_LABEL, g, options, RowPair) + }); + for (job, root) in window.iter().zip(commits) { + match job { + Job::Slot(i, _) => roots[*i] = root, + Job::Merged(k, _) => merged_roots[*k] = root, + } + } + } + let merge = LfmMerge { + tables: plan + .iter() + .zip(merged_roots) + .map(|(chips, root)| MergedTable { + chips: chips.clone(), + log_height: chips.first().map_or(0, |c| log_heights[c.slot()]), + root, + }) + .collect(), + }; + // Then `LFM_BLAKE3`, one window of chunks at a time: materialize each + // chunk's group, expand it, commit it, drop both. Peak residency is the + // window's chunks — which is still the point of chunking this chip, at a + // bound that names itself. + let chunks: Vec = (0..blake3_chunk_log_heights.len()).collect(); + let mut blake3_chunk_roots: Vec = Vec::with_capacity(chunks.len()); + for window in chunks.chunks(in_flight) { + blake3_chunk_roots.extend(map_maybe_parallel(window, |c| { + let group = program.blake3_chunk_group(*c); + commit_group_device_or_host_under::(BLAKE3_CHUNK_LABEL, &group, options, RowPair) + })); + } + roots[BLAKE3_SLOT] = blake3_chunk_roots[0]; + // The hash tail: `LFM_HASH` chunks 1.., committed like the BLAKE3 chunks. + // Chunk 0 is slot 5's root, committed in the window above. + let mut hash_chunk_roots: Vec = vec![roots[HASH_SLOT]]; + let mut hash_chunk_log_heights: Vec = vec![log_heights[HASH_SLOT]]; + let tail: Vec<&ColumnGroup> = hash_chunks.iter().skip(1).collect(); + for window in tail.chunks(in_flight) { + hash_chunk_roots.extend(map_maybe_parallel(window, |g| { + commit_group_device_or_host_under::(HASH_CHUNK_LABEL, g, options, RowPair) + })); + } + hash_chunk_log_heights.extend(tail.iter().map(|g| g.padded_rows.trailing_zeros() as u8)); + // Slot 12 (KECCAK_RND) keeps the all-zero sentinel installed above. + // The static tables: the legacy pin's blessed roots, the Poseidon1 + // family's where they ship, or `S`'s computed once per process. + roots[13] = keccak_rc::preprocessed_root_under::(options); + log_heights[13] = keccak_rc::NUM_ROWS.trailing_zeros() as u8; + roots[14] = bitwise::preprocessed_root_under::(options); + log_heights[14] = bitwise::NUM_ROWS.trailing_zeros() as u8; + + // The families this program uses, and the chunk count that follows from + // them: zero KECCAK_RND instances when the keccak family is absent, since + // the chunking policy's floor of one exists only to keep an unused chip + // present. + let chip_set = ChipSet::for_program_with_hasher(program, hasher); + let keccak_rnd_chunks = chip_set.keccak_rnd_chunks( + program + .chunking + .chunk_count(program.groups.keccak.real_rows), + ); + // ★ `blake3_chunk_roots` is NOT mask-gated the way `keccak_rnd_chunks` is, + // and the asymmetry is real: `KECCAK_RND` commits nothing, so an absent + // family can drop to zero instances for free, while `LFM_BLAKE3`'s + // instruction group is COMMITTED whether the family is used or not — slot 11 + // has always carried a root and a height for a program that never + // compresses. So the chunk lists describe what was committed (never empty) + // and the mask decides what a proof carries, exactly where it always did: + // `ChipSet::num_airs` and `LfmAirs::air_refs`. + + let program_id = lfm_program_id_merged_under( + S::COMMITMENT_HASH, + &roots, + &log_heights, + keccak_rnd_chunks, + hasher, + chip_set, + &blake3_chunk_roots, + &blake3_chunk_log_heights, + &hash_chunk_roots, + &hash_chunk_log_heights, + &merge, + ); + let one_row_roots = (options.format.one_row != stark::proof::options::OneRowMode::Off) + .then(|| build_one_row_roots::(program, options, &groups, &tail)); + LfmArtifacts { + roots, + log_heights, + keccak_rnd_chunks, + blake3_chunk_roots, + blake3_chunk_log_heights, + hash_chunk_roots, + hash_chunk_log_heights, + hasher, + chip_set, + program_id, + one_row_roots, + merge, + } +} + +/// [`build_artifacts_under`]'s heights, chunk lists and chip set with NO +/// commit (every root zero, the id zero): what a node's shape over this +/// program depends on (`DerivedChild`), for the census, which must not pay +/// the commits, and the AIR set the first prove's warm-up reads its tables' +/// shapes from (`first_prove_warm`), before the artifacts exist. +/// `first_prove_warm::tests::the_census_artifacts_have_the_built_shape` pins +/// it against a real build. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn census_artifacts(program: &LfmProgram, hasher: HasherKind) -> LfmArtifacts { + let range = range_group(); + let program_slots = program_groups(program); + let hash_chunks: Vec = if program.hash_chunk_count() > 1 { + (0..program.hash_chunk_count()) + .map(|c| program.hash_chunk_group(c)) + .collect() + } else { + Vec::new() + }; + let groups: [&ColumnGroup; 11] = std::array::from_fn(|i| { + if i == HASH_SLOT && !hash_chunks.is_empty() { + &hash_chunks[0] + } else if i < PROGRAM_GROUP_SLOTS { + program_slots[i] + } else { + &range + } + }); + let mut log_heights = [0u8; NUM_LFM_CHIPS]; + for (i, g) in groups.iter().enumerate() { + log_heights[i] = g.padded_rows.trailing_zeros() as u8; + } + let blake3_chunk_log_heights: Vec = blake3_chunk_rows(program) + .into_iter() + .map(|rows| rows.trailing_zeros() as u8) + .collect(); + log_heights[BLAKE3_SLOT] = blake3_chunk_log_heights[0]; + let mut hash_chunk_log_heights: Vec = vec![log_heights[HASH_SLOT]]; + hash_chunk_log_heights.extend( + hash_chunks + .iter() + .skip(1) + .map(|g| g.padded_rows.trailing_zeros() as u8), + ); + log_heights[13] = keccak_rc::NUM_ROWS.trailing_zeros() as u8; + log_heights[14] = bitwise::NUM_ROWS.trailing_zeros() as u8; + let chip_set = ChipSet::for_program_with_hasher(program, hasher); + let keccak_rnd_chunks = chip_set.keccak_rnd_chunks( + program + .chunking + .chunk_count(program.groups.keccak.real_rows), + ); + LfmArtifacts { + roots: [[0u8; 32]; NUM_LFM_CHIPS], + log_heights, + keccak_rnd_chunks, + blake3_chunk_roots: vec![[0u8; 32]; blake3_chunk_log_heights.len()], + blake3_chunk_log_heights, + hash_chunk_roots: vec![[0u8; 32]; hash_chunk_log_heights.len()], + hash_chunk_log_heights, + hasher, + chip_set, + program_id: [0u8; 32], + one_row_roots: None, + merge: LfmMerge::none(), + } +} + +/// [`census_artifacts`] under the merge `on` asks of `program`, every merged +/// root zero: the shape a node over a merged program derives (the merge +/// census's), and the merged AIR set the first prove's warm-up builds. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn census_artifacts_merged( + program: &LfmProgram, + hasher: HasherKind, + on: bool, +) -> LfmArtifacts { + let mut artifacts = census_artifacts(program, hasher); + let plan = super::merge::plan_for_program(program, on); + artifacts.merge = LfmMerge { + tables: plan + .into_iter() + .map(|chips| MergedTable { + log_height: chips.first().map_or(0, |c| artifacts.log_heights[c.slot()]), + chips, + root: [0u8; 32], + }) + .collect(), + }; + artifacts +} + +/// The one-row roots of every committed group (host pass: the device commit +/// builds row-pair leaves only), plus the static tables' one-row twins. +fn build_one_row_roots( + program: &LfmProgram, + options: &ProofOptions, + groups: &[&ColumnGroup; 11], + hash_tail: &[&ColumnGroup], +) -> LfmOneRowRoots { + use stark::leaf_layout::LeafLayout::Row; + let mut roots: [Option; NUM_LFM_CHIPS] = [None; NUM_LFM_CHIPS]; + let commits = map_maybe_parallel(groups, |g| { + commit_group_device_or_host_under::(PREP_GROUP_LABEL, g, options, Row) + }); + for (slot, root) in commits.into_iter().enumerate() { + roots[slot] = Some(root); + } + let chunks: Vec = (0..blake3_chunk_rows(program).len()).collect(); + let blake3_chunk_roots = map_maybe_parallel(&chunks, |c| { + let group = program.blake3_chunk_group(*c); + commit_group_device_or_host_under::(BLAKE3_CHUNK_LABEL, &group, options, Row) + }); + roots[BLAKE3_SLOT] = blake3_chunk_roots.first().copied(); + roots[13] = keccak_rc::preprocessed_commitment_under::(options, Row); + roots[14] = bitwise::preprocessed_commitment_under::(options, Row); + // Chunk 0 is slot 5's one-row root; the hash tail follows. + let mut hash_chunk_roots: Vec = roots[HASH_SLOT].into_iter().collect(); + hash_chunk_roots.extend(map_maybe_parallel(hash_tail, |g| { + commit_group_device_or_host_under::(HASH_CHUNK_LABEL, g, options, Row) + })); + LfmOneRowRoots { + roots, + blake3_chunk_roots, + hash_chunk_roots, + } +} + +/// Resolves a registry entry or fails hard. No fallback path exists or may +/// ever be added. +/// ★ **A duplicate `(kind, blowup_factor)` is an error, not a first-match win.** +/// +/// `compute_lfm_registry`'s own doc promises that a second `LFM_HASH` hasher +/// "becomes additional rows, never a silent replacement of these" — but the key +/// here carries no hasher, so `.find()` returned whichever row the generator +/// printed first and the other was unreachable, unnoticed. Since this function +/// is the soundness argument's first premise, the failure mode it left open was +/// a verifier checking a proof against a *different* program's roots than the +/// one whose entry an operator believes is live. +/// +/// Rejecting the shape is the cheap half of the fix; the other half is that +/// adding rows stays a deliberate act, which `the_registry_is_six_unambiguous_rows` +/// pins. It returns an `Err` rather than panicking because this runs on the +/// verify path, where an unusable input is a rejected proof. +pub fn resolve( + kind: LfmProgramKind, + blowup_factor: u8, +) -> Result<&'static LfmRegistryEntry, LfmRegistryError> { + #[cfg(test)] + REGISTRY_READS.with(|c| c.set(c.get() + 1)); + let mut matches = LFM_REGISTRY + .iter() + .filter(|e| e.kind == kind && e.blowup_factor == blowup_factor); + let first = matches.next().ok_or(LfmRegistryError::UnknownProgram { + kind, + blowup_factor, + })?; + let extra = matches.count(); + if extra > 0 { + return Err(LfmRegistryError::AmbiguousProgram { + kind, + blowup_factor, + rows: extra + 1, + }); + } + Ok(first) +} + +// ========================================================================= +// GENERATED — do not edit by hand. Regenerate with: +// cargo run --bin compute_lfm_registry --release +// and paste the output below. Drift tests recompute and compare on every PR. +// +// ★ LAST REGENERATION, and what forced it: the ECREC table's count joined the +// absorbed continuation-epoch statement (`NUM_TABLE_COUNTS` 22 -> 23, statement +// V7, 8 bytes more). Only StatementReplayV0 replays that statement: its roots[0] +// and roots[7] and its `program_id` moved; its log-heights, `keccak_rnd_chunks`, +// `hasher` and `chip_set`, and all five other entries, came back identical +// (`registry_drift_*`: five pass unchanged, StatementReplayV0's passes after the +// paste). +// +// ★ THE REGENERATION BEFORE, and what forced it: the main-sync merge `892c7d1bc` +// (main's `c2ac5d546`, #977) grew the absorbed continuation-epoch statement by +// 49 bytes — six accelerator counts joined it when `FIXED_TABLE_COUNT` went +// 11 -> 5 (`NUM_TABLE_COUNTS` 15 -> 21, +48) and `is_final` was appended as its +// last byte (+1). StatementReplayV0 is the one registered program that replays +// that statement, so its script gained one `Blake3Chain` compression and seven +// of its fifteen roots moved with `program_id`. +// +// ⚠ EXACTLY ONE ENTRY MOVED, and that was checked rather than assumed: the +// other five programs build no epoch statement, and all fifteen roots, the +// log-heights, `keccak_rnd_chunks`, `hasher` and `chip_set` of each came back +// byte-identical. `log_heights[11]` did NOT move even for StatementReplayV0 — +// 9 and 10 compressions both pad to the 16-row group. The regenerated +// `roots[11]` also equals the value `machine_tests:: +// every_registry_entry_is_a_single_blake3_table` computes from the program +// (`7a 3b 86 1b ...`), which is what says the paste closes the gap the pin +// found rather than moving the pin to meet the paste. +// +// `compute_static_commitments` was NOT re-run and did not need to be: this is +// not a hash-pin change, so slots 13 and 14 (KECCAK_RC's and BITWISE's static +// preprocessed commitments) are untouched — and they came back identical. +// +// ★ WHICH CONFIGURATION THIS TABLE ASSUMES, and what would move it. +// +// Three axes are BOUND into every digest below, and the fourth stopped being +// free at the P-a flip: +// +// * `hasher` (the `LFM_HASH` socket permutation) is folded into `program_id`, +// so two hashers are two program identities. Blessed under +// `HasherKind::Test`; a second hasher becomes additional ROWS, and `resolve` +// rejects a duplicate `(kind, blowup_factor)` rather than shadowing one. +// * `blowup_factor` is a column of the table and half of `resolve`'s key. +// Blessed at 2, the only registered preset. +// * ★ The COMMITMENT hash — what `stark`'s Merkle layer builds these roots +// with, as opposed to what the machine runs — is folded into `program_id` +// too, since the flip. Blessed under `CommitmentHash::Blake3`, which is the +// non-`cuda` default. Every root below moves with it, and now so does the +// name. +// +// ⚠ **This table therefore describes ONE arm of the `cuda` fork.** A `cuda` +// build's default is keccak (`stark::config::DefaultStarkHash`), so it +// commits different roots and folds a different tag, and the six +// `machine_tests::registry_drift_*` tests fail there — correctly, because a +// `cuda` build really is a different machine with different program +// identities. The tests are NOT cfg'd off: the incompatibility is the point, +// and hiding it would leave a build silently claiming identities it does not +// produce. It resolves when the fork retires (see `DefaultStarkHash`), not +// before, and blessing a second table is a decision nobody has taken. +// * ⚠ `BLAKE3_ROUNDS` (the `blake3-6round` feature) — **the previous note here +// said this table is round-count-invariant, and the flip ENDED that.** It +// was true while the roots were keccak: the round count moves `LFM_BLAKE3`'s +// VALUE columns only (width 3556 → 3076, constraints 897 → 769, +// interactions 1453 → 1261), never the preprocessed instruction group this +// table commits for that slot. That separation still holds, and +// `blake3_chip_tests::the_registry_blessing_is_round_count_invariant` still +// pins it — read it for what it now asserts, which is about COLUMN COUNTS, +// not about these bytes. +// +// What changed is underneath it: these roots are now built BY BLAKE3, so the +// round count moves the Merkle hash itself and therefore every root here. +// One table no longer serves two builds. That is why the round count is a +// compile error rather than a convention — see `stark::config`'s assertion — +// and it is blessed at six, the shipping arm. +// ========================================================================= +pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ + LfmRegistryEntry { + kind: LfmProgramKind::TrivialV0, + blowup_factor: 2, + roots: [ + [ + 0x82, 0x54, 0x4b, 0xc4, 0x61, 0x8d, 0x49, 0x29, 0x43, 0x97, 0x57, 0x20, 0xca, 0x82, + 0xd6, 0x74, 0xfe, 0x72, 0x82, 0x09, 0x39, 0x3c, 0xe6, 0x45, 0x42, 0xab, 0xbf, 0x15, + 0x9f, 0xec, 0xad, 0xfd, + ], + [ + 0xe7, 0x02, 0x4b, 0x94, 0x35, 0x37, 0xe8, 0x74, 0x72, 0x94, 0xc0, 0x66, 0xf7, 0xa7, + 0x40, 0x08, 0xbb, 0x59, 0x87, 0xa4, 0xe2, 0xf6, 0x50, 0x80, 0x4b, 0xe0, 0xb3, 0x67, + 0x01, 0xbc, 0x82, 0x91, + ], + [ + 0x05, 0x9b, 0xa3, 0x44, 0xe5, 0x9c, 0x68, 0xbd, 0xa5, 0x82, 0xee, 0xec, 0xca, 0xee, + 0x30, 0xbb, 0x3f, 0x0e, 0x73, 0x19, 0x07, 0x82, 0x52, 0x8d, 0x39, 0x43, 0x78, 0x04, + 0x61, 0x09, 0xc5, 0x26, + ], + [ + 0x87, 0xfb, 0xc4, 0x76, 0x2e, 0x30, 0xa3, 0xe6, 0x6e, 0x84, 0x5a, 0xd8, 0xec, 0x2b, + 0x40, 0xe7, 0x45, 0x95, 0xb1, 0xa4, 0x59, 0xbd, 0x1c, 0x4b, 0xed, 0xcc, 0x9d, 0xb1, + 0xa6, 0xdd, 0x7b, 0x52, + ], + [ + 0x0d, 0x28, 0x93, 0xa3, 0xb5, 0xa3, 0x06, 0x0c, 0xab, 0x08, 0x2f, 0x56, 0x8f, 0x9a, + 0x4b, 0x02, 0x6b, 0x58, 0x55, 0x45, 0xf5, 0xc3, 0xfd, 0x0d, 0x12, 0x39, 0x4d, 0x99, + 0x48, 0x41, 0x4b, 0xf7, + ], + [ + 0xee, 0x9a, 0x81, 0xfb, 0xc8, 0xbd, 0x72, 0x5f, 0x89, 0x27, 0x3f, 0xd4, 0xe8, 0xcb, + 0x20, 0x07, 0xf8, 0x01, 0x7f, 0x7e, 0x78, 0x34, 0x38, 0x9a, 0xef, 0xce, 0xfd, 0x39, + 0xaf, 0xe0, 0x9c, 0x6d, + ], + [ + 0x8d, 0xa4, 0x49, 0x54, 0x80, 0xa3, 0xfe, 0x4d, 0x2f, 0xb1, 0xe0, 0x1d, 0xd9, 0x6b, + 0x82, 0xa6, 0x65, 0x36, 0x1f, 0x21, 0xf9, 0x92, 0x67, 0x4f, 0x88, 0x90, 0x70, 0xab, + 0xff, 0x31, 0xec, 0xb2, + ], + [ + 0x9b, 0xd2, 0x23, 0x26, 0x3c, 0xa7, 0x25, 0xb3, 0xc1, 0x3c, 0x18, 0xe9, 0x9c, 0xb2, + 0x28, 0xdb, 0x4f, 0x29, 0x58, 0xac, 0xdb, 0x52, 0x58, 0xe8, 0x23, 0x2e, 0xb0, 0x8b, + 0x0d, 0xd5, 0xed, 0x41, + ], + [ + 0xd7, 0x1c, 0x38, 0x03, 0x8f, 0x76, 0x18, 0x6f, 0xde, 0xf9, 0x59, 0x89, 0x9e, 0xc4, + 0x6d, 0xa2, 0x5a, 0x5b, 0x94, 0xd0, 0x5f, 0x34, 0x32, 0x0a, 0xd8, 0xbc, 0xf3, 0x54, + 0xda, 0xf0, 0x81, 0xcb, + ], + [ + 0xca, 0x9c, 0x81, 0x3f, 0xdc, 0xd4, 0x49, 0x28, 0x06, 0xd2, 0x9b, 0x22, 0x09, 0x77, + 0x0f, 0x27, 0x87, 0x6e, 0xb1, 0x28, 0xf4, 0x3c, 0x4a, 0x38, 0xf7, 0x57, 0x42, 0x92, + 0x29, 0xeb, 0x86, 0x8c, + ], + [ + 0xa3, 0xd2, 0x1d, 0x58, 0xbf, 0x0c, 0x09, 0xb2, 0x14, 0xce, 0xe6, 0x4f, 0x94, 0xf8, + 0x10, 0xc2, 0xc2, 0xf0, 0xcb, 0x13, 0xd8, 0x8a, 0x66, 0xbc, 0x8f, 0x4c, 0x76, 0x72, + 0xe8, 0xc4, 0x17, 0x99, + ], + [ + 0x4b, 0xd7, 0x50, 0x10, 0xec, 0x2d, 0xfb, 0x77, 0xbd, 0xdd, 0x72, 0x55, 0xec, 0xb4, + 0x35, 0x03, 0xb1, 0x99, 0x66, 0xac, 0xe9, 0x33, 0xe6, 0xcc, 0x08, 0xae, 0xba, 0x7c, + 0x54, 0xdb, 0x18, 0x9c, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0x0a, 0x3c, 0x86, 0xe2, 0xd3, 0x37, 0x98, 0xf2, 0x68, 0x45, 0x42, 0xbf, 0x0b, 0x3d, + 0x52, 0x02, 0x4b, 0x08, 0x8b, 0x7a, 0x4b, 0xc8, 0xb0, 0x2a, 0xe4, 0x14, 0x86, 0x09, + 0x35, 0xdc, 0xa6, 0x7c, + ], + [ + 0x46, 0x8d, 0xfe, 0x65, 0xed, 0x50, 0x25, 0xbf, 0x7e, 0xfa, 0x04, 0x11, 0x28, 0x36, + 0x29, 0x06, 0xbd, 0x97, 0x29, 0xeb, 0x5b, 0x3a, 0xae, 0xbf, 0x66, 0xea, 0x5e, 0x59, + 0xe4, 0xd7, 0x17, 0x6c, + ], + ], + log_heights: [3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 16, 2, 0, 5, 20], + keccak_rnd_chunks: 0, + hasher: HasherKind::Test, + chip_set: ChipSet { + keccak: false, + blake3: false, + bitwise: false, + }, + program_id: [ + 0x5e, 0xbb, 0x83, 0x47, 0xc0, 0x5f, 0x77, 0xfd, 0x00, 0x41, 0x09, 0x1c, 0x11, 0x20, + 0xd4, 0x3a, 0x49, 0x2f, 0x40, 0x83, 0x56, 0xa5, 0x3b, 0x35, 0x78, 0xe0, 0x0f, 0x46, + 0x81, 0xcc, 0x6d, 0x80, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::FriToyV0, + blowup_factor: 2, + roots: [ + [ + 0x9a, 0x59, 0x5c, 0x71, 0xd1, 0xa7, 0x6a, 0xf8, 0x76, 0x0a, 0x56, 0x5f, 0x3d, 0xf5, + 0x87, 0x9f, 0x71, 0x48, 0xf0, 0xf0, 0xd3, 0x49, 0x38, 0x5c, 0x2b, 0x9a, 0x6d, 0x05, + 0x91, 0x18, 0x93, 0x0d, + ], + [ + 0xed, 0x75, 0x5c, 0x0a, 0x8b, 0x67, 0x41, 0x15, 0x9c, 0x13, 0xdd, 0xcc, 0xbb, 0x80, + 0x34, 0x20, 0xf9, 0x11, 0x8e, 0x74, 0x4b, 0xe2, 0x4c, 0xc7, 0x79, 0xe0, 0x4f, 0x4d, + 0x76, 0x31, 0x09, 0xd0, + ], + [ + 0x20, 0x8b, 0x3c, 0x06, 0x1e, 0x1c, 0xf4, 0x7a, 0xe3, 0x6b, 0xb6, 0xf3, 0x62, 0x1e, + 0xda, 0x7e, 0x0e, 0x5a, 0x1c, 0x7c, 0x77, 0x4d, 0xf5, 0x1e, 0xb9, 0xd1, 0xad, 0x27, + 0xed, 0xac, 0xba, 0x44, + ], + [ + 0x62, 0x88, 0x6d, 0xac, 0x61, 0xce, 0x6f, 0x89, 0x64, 0x26, 0x67, 0x3d, 0xcb, 0x77, + 0x0a, 0xbb, 0x85, 0x53, 0x99, 0xf4, 0x09, 0xc0, 0x33, 0x1b, 0xe6, 0xf7, 0x93, 0x65, + 0x17, 0x7f, 0x15, 0x5c, + ], + [ + 0xd9, 0xe6, 0xaf, 0xf7, 0x07, 0x25, 0xaa, 0x8c, 0x15, 0x0f, 0x41, 0x3e, 0xd5, 0x46, + 0x13, 0x06, 0x70, 0x75, 0x63, 0x1b, 0xc0, 0x65, 0x98, 0x0f, 0xa9, 0x25, 0x2e, 0x5f, + 0xdc, 0x62, 0xeb, 0x65, + ], + [ + 0x7f, 0x87, 0x55, 0xc5, 0x81, 0x3b, 0xeb, 0x94, 0x20, 0x1d, 0xf1, 0xe5, 0x17, 0x93, + 0x1a, 0xe2, 0xe9, 0x46, 0x58, 0xb6, 0x00, 0x04, 0x99, 0xb1, 0x54, 0xdc, 0xa2, 0x7a, + 0xa8, 0xf4, 0xe3, 0xa4, + ], + [ + 0x8d, 0xa4, 0x49, 0x54, 0x80, 0xa3, 0xfe, 0x4d, 0x2f, 0xb1, 0xe0, 0x1d, 0xd9, 0x6b, + 0x82, 0xa6, 0x65, 0x36, 0x1f, 0x21, 0xf9, 0x92, 0x67, 0x4f, 0x88, 0x90, 0x70, 0xab, + 0xff, 0x31, 0xec, 0xb2, + ], + [ + 0x96, 0x15, 0x02, 0xab, 0x70, 0xc5, 0x29, 0x42, 0xe0, 0xa6, 0xd7, 0xe6, 0x8c, 0x4e, + 0x0e, 0xf6, 0x1f, 0x28, 0x9b, 0xc5, 0xfa, 0x61, 0xa5, 0xc6, 0x96, 0x3a, 0xf1, 0xd4, + 0xe6, 0x91, 0xe5, 0x2b, + ], + [ + 0xd2, 0x8e, 0xe0, 0xb7, 0x30, 0x2f, 0xc0, 0xf3, 0x1c, 0x75, 0xe3, 0x19, 0x08, 0x9c, + 0xd3, 0x00, 0x7b, 0x8c, 0xbe, 0x40, 0x85, 0xad, 0xf6, 0xbe, 0xb8, 0xc6, 0x2f, 0x46, + 0x8f, 0xfd, 0x03, 0xcd, + ], + [ + 0x51, 0x61, 0x61, 0x29, 0x2f, 0x64, 0x86, 0xf3, 0x77, 0x19, 0x29, 0xae, 0x32, 0x5d, + 0x7e, 0xbb, 0xe2, 0x8a, 0x8d, 0x56, 0x27, 0x43, 0x1e, 0xf3, 0xd2, 0x3b, 0x82, 0x79, + 0xec, 0x71, 0x01, 0x6f, + ], + [ + 0xa3, 0xd2, 0x1d, 0x58, 0xbf, 0x0c, 0x09, 0xb2, 0x14, 0xce, 0xe6, 0x4f, 0x94, 0xf8, + 0x10, 0xc2, 0xc2, 0xf0, 0xcb, 0x13, 0xd8, 0x8a, 0x66, 0xbc, 0x8f, 0x4c, 0x76, 0x72, + 0xe8, 0xc4, 0x17, 0x99, + ], + [ + 0x4b, 0xd7, 0x50, 0x10, 0xec, 0x2d, 0xfb, 0x77, 0xbd, 0xdd, 0x72, 0x55, 0xec, 0xb4, + 0x35, 0x03, 0xb1, 0x99, 0x66, 0xac, 0xe9, 0x33, 0xe6, 0xcc, 0x08, 0xae, 0xba, 0x7c, + 0x54, 0xdb, 0x18, 0x9c, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0x0a, 0x3c, 0x86, 0xe2, 0xd3, 0x37, 0x98, 0xf2, 0x68, 0x45, 0x42, 0xbf, 0x0b, 0x3d, + 0x52, 0x02, 0x4b, 0x08, 0x8b, 0x7a, 0x4b, 0xc8, 0xb0, 0x2a, 0xe4, 0x14, 0x86, 0x09, + 0x35, 0xdc, 0xa6, 0x7c, + ], + [ + 0x46, 0x8d, 0xfe, 0x65, 0xed, 0x50, 0x25, 0xbf, 0x7e, 0xfa, 0x04, 0x11, 0x28, 0x36, + 0x29, 0x06, 0xbd, 0x97, 0x29, 0xeb, 0x5b, 0x3a, 0xae, 0xbf, 0x66, 0xea, 0x5e, 0x59, + 0xe4, 0xd7, 0x17, 0x6c, + ], + ], + log_heights: [5, 8, 7, 7, 2, 7, 2, 5, 7, 2, 16, 2, 0, 5, 20], + keccak_rnd_chunks: 0, + hasher: HasherKind::Test, + chip_set: ChipSet { + keccak: false, + blake3: false, + bitwise: false, + }, + program_id: [ + 0x2e, 0x0b, 0x68, 0x93, 0x07, 0x3b, 0x29, 0xa4, 0x55, 0x65, 0x67, 0xb7, 0x5e, 0x09, + 0xb5, 0xa9, 0x7d, 0xd1, 0xd9, 0x4a, 0x92, 0x9e, 0x34, 0xae, 0xe5, 0xe5, 0x3d, 0xdf, + 0x76, 0xe2, 0x64, 0xb7, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::KeccakChainV0, + blowup_factor: 2, + roots: [ + [ + 0xe1, 0xc3, 0x27, 0x68, 0xac, 0x08, 0x46, 0xd6, 0x86, 0xeb, 0xae, 0x4c, 0xff, 0x10, + 0x92, 0x9e, 0x5c, 0xcc, 0x6a, 0xdd, 0xd5, 0xcd, 0xf2, 0xce, 0x5c, 0x1e, 0x60, 0x7d, + 0x6f, 0xd1, 0x77, 0x33, + ], + [ + 0x31, 0xf9, 0x22, 0xf3, 0x9d, 0xf1, 0xcb, 0xd0, 0xeb, 0x62, 0x25, 0x93, 0xea, 0x75, + 0x43, 0x30, 0x8b, 0xb1, 0x02, 0x45, 0x20, 0xde, 0xf9, 0x17, 0x78, 0x17, 0x27, 0xc6, + 0xc4, 0x9a, 0x99, 0x28, + ], + [ + 0x07, 0xdb, 0x70, 0x37, 0x6d, 0xff, 0x1c, 0x50, 0x7f, 0x82, 0xf7, 0x83, 0x73, 0x7c, + 0x2e, 0xad, 0xab, 0x48, 0xbc, 0x68, 0x82, 0xf7, 0xff, 0x66, 0x80, 0x88, 0xe9, 0x2c, + 0xae, 0x17, 0x71, 0x37, + ], + [ + 0x54, 0xb6, 0x8d, 0xc0, 0x91, 0x2b, 0x77, 0xc9, 0xeb, 0x5e, 0xe5, 0x1a, 0x1d, 0x13, + 0x91, 0x0a, 0x95, 0xdb, 0xec, 0x96, 0x87, 0x73, 0x43, 0x0c, 0x3d, 0xf9, 0xe6, 0x5d, + 0x80, 0xee, 0x1d, 0xd5, + ], + [ + 0x16, 0x7c, 0xec, 0xa5, 0x7c, 0x2b, 0x9b, 0x02, 0x25, 0x0b, 0x93, 0x86, 0xdc, 0xe0, + 0x05, 0x81, 0xa3, 0x67, 0xf4, 0xbe, 0xd3, 0x4e, 0x06, 0x78, 0xae, 0xb6, 0x52, 0x05, + 0xff, 0xdd, 0xfb, 0xd5, + ], + [ + 0x7d, 0x25, 0xc1, 0xee, 0x40, 0x2b, 0x03, 0x6b, 0xf9, 0x14, 0x9a, 0xa3, 0x50, 0x04, + 0xb8, 0x62, 0x5f, 0x24, 0x5b, 0x6d, 0x11, 0x4b, 0x36, 0xa9, 0xea, 0x0b, 0xd0, 0x85, + 0x58, 0xd8, 0x02, 0xf5, + ], + [ + 0xfc, 0x62, 0x00, 0xa7, 0x23, 0x66, 0x4f, 0x48, 0xcf, 0xc1, 0x4b, 0xd8, 0xf7, 0x3a, + 0x4c, 0x77, 0x05, 0xe4, 0x5c, 0x49, 0x3c, 0x9f, 0x95, 0x3d, 0x5e, 0xbe, 0x60, 0x95, + 0x38, 0x8e, 0x26, 0xfc, + ], + [ + 0x9b, 0xd2, 0x23, 0x26, 0x3c, 0xa7, 0x25, 0xb3, 0xc1, 0x3c, 0x18, 0xe9, 0x9c, 0xb2, + 0x28, 0xdb, 0x4f, 0x29, 0x58, 0xac, 0xdb, 0x52, 0x58, 0xe8, 0x23, 0x2e, 0xb0, 0x8b, + 0x0d, 0xd5, 0xed, 0x41, + ], + [ + 0x2f, 0x2d, 0xfa, 0xe3, 0x1c, 0xbe, 0x00, 0xe3, 0xdc, 0x4a, 0xa5, 0x48, 0xc6, 0x72, + 0x28, 0x7b, 0x26, 0x05, 0x9a, 0x42, 0xf3, 0x3c, 0x72, 0xb0, 0xbe, 0x5f, 0x73, 0x47, + 0x00, 0x0e, 0xf3, 0xeb, + ], + [ + 0x9a, 0x39, 0xe1, 0x95, 0x0c, 0x1b, 0x7a, 0x36, 0x16, 0xfc, 0xb5, 0xde, 0xf0, 0x2f, + 0xac, 0x61, 0x3b, 0x32, 0xa3, 0xe9, 0x4a, 0x63, 0xfe, 0x66, 0x59, 0x08, 0x5b, 0x22, + 0xe3, 0x87, 0xab, 0x5c, + ], + [ + 0xa3, 0xd2, 0x1d, 0x58, 0xbf, 0x0c, 0x09, 0xb2, 0x14, 0xce, 0xe6, 0x4f, 0x94, 0xf8, + 0x10, 0xc2, 0xc2, 0xf0, 0xcb, 0x13, 0xd8, 0x8a, 0x66, 0xbc, 0x8f, 0x4c, 0x76, 0x72, + 0xe8, 0xc4, 0x17, 0x99, + ], + [ + 0x4b, 0xd7, 0x50, 0x10, 0xec, 0x2d, 0xfb, 0x77, 0xbd, 0xdd, 0x72, 0x55, 0xec, 0xb4, + 0x35, 0x03, 0xb1, 0x99, 0x66, 0xac, 0xe9, 0x33, 0xe6, 0xcc, 0x08, 0xae, 0xba, 0x7c, + 0x54, 0xdb, 0x18, 0x9c, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0x0a, 0x3c, 0x86, 0xe2, 0xd3, 0x37, 0x98, 0xf2, 0x68, 0x45, 0x42, 0xbf, 0x0b, 0x3d, + 0x52, 0x02, 0x4b, 0x08, 0x8b, 0x7a, 0x4b, 0xc8, 0xb0, 0x2a, 0xe4, 0x14, 0x86, 0x09, + 0x35, 0xdc, 0xa6, 0x7c, + ], + [ + 0x46, 0x8d, 0xfe, 0x65, 0xed, 0x50, 0x25, 0xbf, 0x7e, 0xfa, 0x04, 0x11, 0x28, 0x36, + 0x29, 0x06, 0xbd, 0x97, 0x29, 0xeb, 0x5b, 0x3a, 0xae, 0xbf, 0x66, 0xea, 0x5e, 0x59, + 0xe4, 0xd7, 0x17, 0x6c, + ], + ], + log_heights: [2, 2, 2, 2, 2, 2, 2, 2, 4, 2, 16, 2, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + chip_set: ChipSet { + keccak: true, + blake3: false, + bitwise: true, + }, + program_id: [ + 0x55, 0x79, 0xf4, 0xc0, 0x15, 0xb3, 0xd9, 0x73, 0xc1, 0x7d, 0xe0, 0x41, 0x58, 0xaa, + 0x06, 0xff, 0x4e, 0x56, 0x57, 0x50, 0x3c, 0x9e, 0x51, 0xc9, 0xe0, 0x40, 0x3c, 0xc8, + 0x58, 0xd4, 0x2a, 0x33, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::KeccakSpongeV0, + blowup_factor: 2, + roots: [ + [ + 0xc7, 0xa2, 0x00, 0xa8, 0x75, 0xaa, 0x69, 0x79, 0xe0, 0x0a, 0x38, 0x87, 0x0a, 0xb4, + 0x1b, 0x04, 0x7a, 0x8c, 0xa0, 0x57, 0x10, 0x3f, 0xbc, 0xb8, 0x28, 0xa8, 0x2c, 0xc8, + 0x53, 0x41, 0x3b, 0xf5, + ], + [ + 0x13, 0xf7, 0xa3, 0xbe, 0x73, 0x3b, 0xec, 0x61, 0xcf, 0xd6, 0xc1, 0x69, 0x8c, 0xef, + 0x90, 0x3a, 0x89, 0x8b, 0x76, 0xeb, 0x9f, 0x0d, 0x1d, 0x87, 0x7d, 0xe5, 0xac, 0x22, + 0x82, 0xef, 0x4f, 0xd8, + ], + [ + 0x07, 0xdb, 0x70, 0x37, 0x6d, 0xff, 0x1c, 0x50, 0x7f, 0x82, 0xf7, 0x83, 0x73, 0x7c, + 0x2e, 0xad, 0xab, 0x48, 0xbc, 0x68, 0x82, 0xf7, 0xff, 0x66, 0x80, 0x88, 0xe9, 0x2c, + 0xae, 0x17, 0x71, 0x37, + ], + [ + 0x54, 0xb6, 0x8d, 0xc0, 0x91, 0x2b, 0x77, 0xc9, 0xeb, 0x5e, 0xe5, 0x1a, 0x1d, 0x13, + 0x91, 0x0a, 0x95, 0xdb, 0xec, 0x96, 0x87, 0x73, 0x43, 0x0c, 0x3d, 0xf9, 0xe6, 0x5d, + 0x80, 0xee, 0x1d, 0xd5, + ], + [ + 0x16, 0x7c, 0xec, 0xa5, 0x7c, 0x2b, 0x9b, 0x02, 0x25, 0x0b, 0x93, 0x86, 0xdc, 0xe0, + 0x05, 0x81, 0xa3, 0x67, 0xf4, 0xbe, 0xd3, 0x4e, 0x06, 0x78, 0xae, 0xb6, 0x52, 0x05, + 0xff, 0xdd, 0xfb, 0xd5, + ], + [ + 0x7d, 0x25, 0xc1, 0xee, 0x40, 0x2b, 0x03, 0x6b, 0xf9, 0x14, 0x9a, 0xa3, 0x50, 0x04, + 0xb8, 0x62, 0x5f, 0x24, 0x5b, 0x6d, 0x11, 0x4b, 0x36, 0xa9, 0xea, 0x0b, 0xd0, 0x85, + 0x58, 0xd8, 0x02, 0xf5, + ], + [ + 0xde, 0x04, 0x16, 0x1b, 0x9a, 0x9a, 0x84, 0x02, 0xd9, 0x21, 0x50, 0xa3, 0x6b, 0x62, + 0xb6, 0x0a, 0x0e, 0x69, 0x9d, 0x6f, 0xe3, 0xea, 0x80, 0x0d, 0x0a, 0x9b, 0x76, 0xee, + 0x82, 0x89, 0x46, 0xf2, + ], + [ + 0x5e, 0x14, 0xfe, 0xa2, 0x64, 0x4f, 0x4a, 0x70, 0xbf, 0x0e, 0x0d, 0x93, 0x09, 0xd8, + 0xb4, 0xbb, 0xba, 0xd1, 0xe6, 0x91, 0x4a, 0x71, 0x7b, 0xec, 0x11, 0xf5, 0x37, 0x3b, + 0x3f, 0xe9, 0xe8, 0x5a, + ], + [ + 0x5a, 0xa5, 0xed, 0xac, 0xa3, 0x06, 0xbb, 0x97, 0x08, 0xc3, 0xe3, 0x1e, 0x0d, 0x8c, + 0x00, 0x1d, 0xed, 0xd2, 0x6e, 0x49, 0x6d, 0xd9, 0xc7, 0x85, 0xd6, 0xd4, 0xd6, 0xbf, + 0x83, 0x2d, 0x6d, 0xb9, + ], + [ + 0xec, 0x03, 0x54, 0x70, 0x9b, 0x7f, 0x21, 0x5b, 0xf9, 0x48, 0x5c, 0xd8, 0x64, 0xa8, + 0xb5, 0x2b, 0x6e, 0x22, 0x79, 0x74, 0x50, 0x93, 0xbb, 0x21, 0x04, 0x53, 0x80, 0xd7, + 0x99, 0x26, 0x5d, 0xd3, + ], + [ + 0xa3, 0xd2, 0x1d, 0x58, 0xbf, 0x0c, 0x09, 0xb2, 0x14, 0xce, 0xe6, 0x4f, 0x94, 0xf8, + 0x10, 0xc2, 0xc2, 0xf0, 0xcb, 0x13, 0xd8, 0x8a, 0x66, 0xbc, 0x8f, 0x4c, 0x76, 0x72, + 0xe8, 0xc4, 0x17, 0x99, + ], + [ + 0x4b, 0xd7, 0x50, 0x10, 0xec, 0x2d, 0xfb, 0x77, 0xbd, 0xdd, 0x72, 0x55, 0xec, 0xb4, + 0x35, 0x03, 0xb1, 0x99, 0x66, 0xac, 0xe9, 0x33, 0xe6, 0xcc, 0x08, 0xae, 0xba, 0x7c, + 0x54, 0xdb, 0x18, 0x9c, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0x0a, 0x3c, 0x86, 0xe2, 0xd3, 0x37, 0x98, 0xf2, 0x68, 0x45, 0x42, 0xbf, 0x0b, 0x3d, + 0x52, 0x02, 0x4b, 0x08, 0x8b, 0x7a, 0x4b, 0xc8, 0xb0, 0x2a, 0xe4, 0x14, 0x86, 0x09, + 0x35, 0xdc, 0xa6, 0x7c, + ], + [ + 0x46, 0x8d, 0xfe, 0x65, 0xed, 0x50, 0x25, 0xbf, 0x7e, 0xfa, 0x04, 0x11, 0x28, 0x36, + 0x29, 0x06, 0xbd, 0x97, 0x29, 0xeb, 0x5b, 0x3a, 0xae, 0xbf, 0x66, 0xea, 0x5e, 0x59, + 0xe4, 0xd7, 0x17, 0x6c, + ], + ], + log_heights: [2, 2, 2, 2, 2, 2, 2, 5, 6, 2, 16, 2, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + chip_set: ChipSet { + keccak: true, + blake3: false, + bitwise: true, + }, + program_id: [ + 0x0a, 0x94, 0x02, 0xe9, 0x21, 0x34, 0xae, 0xbc, 0x2a, 0xcb, 0xbb, 0x94, 0x2b, 0xdf, + 0xa9, 0x91, 0xaf, 0x2c, 0xd1, 0xc7, 0xb2, 0xff, 0xbb, 0xf9, 0x14, 0xed, 0x40, 0xde, + 0xf6, 0x54, 0x80, 0xf0, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::TranscriptReplayV0, + blowup_factor: 2, + roots: [ + [ + 0x05, 0xd2, 0x17, 0x2e, 0x98, 0xf4, 0x8e, 0xc9, 0xb2, 0x7e, 0xd4, 0xbc, 0xd9, 0x4a, + 0xba, 0x0d, 0x16, 0x43, 0x99, 0xcf, 0x6e, 0x13, 0xe6, 0xa5, 0x44, 0x4e, 0xda, 0x9d, + 0xfe, 0xb3, 0x6e, 0xed, + ], + [ + 0x89, 0x96, 0xd2, 0xab, 0x4d, 0xee, 0xd9, 0x5d, 0x70, 0x0b, 0xae, 0x67, 0xe9, 0xca, + 0xb1, 0xf4, 0xbf, 0x1b, 0x70, 0x6a, 0x58, 0x4c, 0xc0, 0xfe, 0x29, 0x05, 0xb8, 0xea, + 0x74, 0xd6, 0xb5, 0xfe, + ], + [ + 0x07, 0xdb, 0x70, 0x37, 0x6d, 0xff, 0x1c, 0x50, 0x7f, 0x82, 0xf7, 0x83, 0x73, 0x7c, + 0x2e, 0xad, 0xab, 0x48, 0xbc, 0x68, 0x82, 0xf7, 0xff, 0x66, 0x80, 0x88, 0xe9, 0x2c, + 0xae, 0x17, 0x71, 0x37, + ], + [ + 0xc4, 0xed, 0x3f, 0xac, 0x6e, 0x05, 0x97, 0xc7, 0x01, 0x3c, 0xba, 0x47, 0x1a, 0x9e, + 0x0d, 0xfc, 0x71, 0x40, 0x55, 0x2d, 0xa8, 0xec, 0xa4, 0xee, 0xd2, 0x13, 0x17, 0xd9, + 0x7f, 0xb8, 0x62, 0x61, + ], + [ + 0x2a, 0x1f, 0xcd, 0x6c, 0x67, 0x77, 0xdc, 0x14, 0x3c, 0x7b, 0xb8, 0x48, 0x24, 0xdb, + 0xde, 0x29, 0x86, 0xaa, 0x0a, 0x0b, 0xe4, 0x82, 0xcd, 0x83, 0x67, 0xbe, 0xff, 0xbe, + 0xda, 0x77, 0x03, 0x1c, + ], + [ + 0x7d, 0x25, 0xc1, 0xee, 0x40, 0x2b, 0x03, 0x6b, 0xf9, 0x14, 0x9a, 0xa3, 0x50, 0x04, + 0xb8, 0x62, 0x5f, 0x24, 0x5b, 0x6d, 0x11, 0x4b, 0x36, 0xa9, 0xea, 0x0b, 0xd0, 0x85, + 0x58, 0xd8, 0x02, 0xf5, + ], + [ + 0x8d, 0xa4, 0x49, 0x54, 0x80, 0xa3, 0xfe, 0x4d, 0x2f, 0xb1, 0xe0, 0x1d, 0xd9, 0x6b, + 0x82, 0xa6, 0x65, 0x36, 0x1f, 0x21, 0xf9, 0x92, 0x67, 0x4f, 0x88, 0x90, 0x70, 0xab, + 0xff, 0x31, 0xec, 0xb2, + ], + [ + 0xef, 0x55, 0xdd, 0x07, 0x83, 0xed, 0x34, 0xf5, 0xd0, 0xed, 0xd3, 0x97, 0x85, 0xf8, + 0x0c, 0x84, 0x2e, 0x58, 0xd2, 0x5b, 0xfc, 0x7f, 0x13, 0x5f, 0x80, 0x2f, 0xca, 0xd9, + 0x33, 0xb3, 0x4a, 0x79, + ], + [ + 0xf5, 0x03, 0xc2, 0x09, 0xbd, 0xdd, 0x4a, 0x2c, 0x01, 0x55, 0xba, 0x0a, 0x7a, 0x8d, + 0xe6, 0x7c, 0xf7, 0xeb, 0x3a, 0x3e, 0x66, 0xb9, 0x55, 0x99, 0x8c, 0xab, 0x9b, 0x6f, + 0xde, 0x6a, 0xa9, 0x80, + ], + [ + 0x58, 0x9a, 0x27, 0xbe, 0x91, 0xff, 0x77, 0x5d, 0x0c, 0xba, 0x66, 0x5d, 0x35, 0x7b, + 0xa0, 0x91, 0x86, 0x81, 0x5a, 0xf6, 0x84, 0x3b, 0xd1, 0x35, 0x66, 0x51, 0x91, 0xd6, + 0x63, 0x4e, 0xbd, 0xec, + ], + [ + 0xa3, 0xd2, 0x1d, 0x58, 0xbf, 0x0c, 0x09, 0xb2, 0x14, 0xce, 0xe6, 0x4f, 0x94, 0xf8, + 0x10, 0xc2, 0xc2, 0xf0, 0xcb, 0x13, 0xd8, 0x8a, 0x66, 0xbc, 0x8f, 0x4c, 0x76, 0x72, + 0xe8, 0xc4, 0x17, 0x99, + ], + [ + 0x42, 0xd6, 0x1f, 0x87, 0x71, 0x67, 0xbf, 0xcf, 0x68, 0x1b, 0xb3, 0x7d, 0x5f, 0xfb, + 0x2e, 0x0f, 0x03, 0x3f, 0x1b, 0x4b, 0xbe, 0x5b, 0xb4, 0x01, 0x3e, 0x47, 0x88, 0x43, + 0x6e, 0xd9, 0xfc, 0xdd, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0x0a, 0x3c, 0x86, 0xe2, 0xd3, 0x37, 0x98, 0xf2, 0x68, 0x45, 0x42, 0xbf, 0x0b, 0x3d, + 0x52, 0x02, 0x4b, 0x08, 0x8b, 0x7a, 0x4b, 0xc8, 0xb0, 0x2a, 0xe4, 0x14, 0x86, 0x09, + 0x35, 0xdc, 0xa6, 0x7c, + ], + [ + 0x46, 0x8d, 0xfe, 0x65, 0xed, 0x50, 0x25, 0xbf, 0x7e, 0xfa, 0x04, 0x11, 0x28, 0x36, + 0x29, 0x06, 0xbd, 0x97, 0x29, 0xeb, 0x5b, 0x3a, 0xae, 0xbf, 0x66, 0xea, 0x5e, 0x59, + 0xe4, 0xd7, 0x17, 0x6c, + ], + ], + log_heights: [5, 10, 2, 4, 4, 2, 2, 6, 6, 3, 16, 3, 0, 5, 20], + keccak_rnd_chunks: 0, + hasher: HasherKind::Test, + chip_set: ChipSet { + keccak: false, + blake3: true, + bitwise: true, + }, + program_id: [ + 0x4d, 0xf6, 0x91, 0xc0, 0x40, 0xd6, 0x21, 0x39, 0xb8, 0x4a, 0x72, 0xe0, 0xcc, 0x1c, + 0x08, 0x77, 0xa5, 0x67, 0x4e, 0x53, 0x1e, 0x36, 0xdd, 0x56, 0x89, 0xc9, 0x4c, 0xc9, + 0x88, 0x15, 0x64, 0x4f, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::StatementReplayV0, + blowup_factor: 2, + roots: [ + [ + 0x3f, 0x68, 0x06, 0x26, 0x22, 0x68, 0x29, 0x05, 0x94, 0x60, 0xfc, 0xf2, 0x52, 0x2e, + 0x2c, 0x75, 0x72, 0xc3, 0x2e, 0xa3, 0x7d, 0xa4, 0xb8, 0x82, 0xbe, 0x69, 0x77, 0x97, + 0xc3, 0x56, 0x60, 0x88, + ], + [ + 0xfb, 0xca, 0xf1, 0x74, 0xa1, 0xad, 0xf6, 0x3d, 0x24, 0x92, 0x44, 0x80, 0xc8, 0x64, + 0x4d, 0xb2, 0xdc, 0x8a, 0x33, 0xc0, 0x98, 0x7a, 0x9f, 0xba, 0xbd, 0xe4, 0x8e, 0xa6, + 0xa9, 0x43, 0x5c, 0x5a, + ], + [ + 0x07, 0xdb, 0x70, 0x37, 0x6d, 0xff, 0x1c, 0x50, 0x7f, 0x82, 0xf7, 0x83, 0x73, 0x7c, + 0x2e, 0xad, 0xab, 0x48, 0xbc, 0x68, 0x82, 0xf7, 0xff, 0x66, 0x80, 0x88, 0xe9, 0x2c, + 0xae, 0x17, 0x71, 0x37, + ], + [ + 0x36, 0xb7, 0x63, 0x4d, 0x83, 0x5b, 0xfc, 0x7e, 0x23, 0x62, 0xfe, 0x2c, 0xf1, 0xc4, + 0xf6, 0x2f, 0x2f, 0x8d, 0x45, 0x5c, 0x21, 0xaf, 0x24, 0xd8, 0x01, 0xfe, 0x3a, 0x68, + 0x64, 0x9e, 0xdd, 0x38, + ], + [ + 0x5c, 0x51, 0x64, 0x8c, 0xad, 0x71, 0x1f, 0x07, 0x17, 0x09, 0x71, 0x74, 0x03, 0x60, + 0x90, 0x87, 0x6e, 0x7b, 0x9d, 0xe6, 0xd0, 0x59, 0x6e, 0xf4, 0x2c, 0x1b, 0xb5, 0x83, + 0x58, 0xf2, 0x98, 0x1f, + ], + [ + 0x7d, 0x25, 0xc1, 0xee, 0x40, 0x2b, 0x03, 0x6b, 0xf9, 0x14, 0x9a, 0xa3, 0x50, 0x04, + 0xb8, 0x62, 0x5f, 0x24, 0x5b, 0x6d, 0x11, 0x4b, 0x36, 0xa9, 0xea, 0x0b, 0xd0, 0x85, + 0x58, 0xd8, 0x02, 0xf5, + ], + [ + 0x8d, 0xa4, 0x49, 0x54, 0x80, 0xa3, 0xfe, 0x4d, 0x2f, 0xb1, 0xe0, 0x1d, 0xd9, 0x6b, + 0x82, 0xa6, 0x65, 0x36, 0x1f, 0x21, 0xf9, 0x92, 0x67, 0x4f, 0x88, 0x90, 0x70, 0xab, + 0xff, 0x31, 0xec, 0xb2, + ], + [ + 0x61, 0x9a, 0x18, 0xe0, 0xad, 0x49, 0xfb, 0x41, 0x27, 0x81, 0x1a, 0xe3, 0x68, 0xb7, + 0x13, 0xcc, 0xf2, 0x95, 0xa5, 0xcc, 0xc9, 0xa7, 0xc4, 0x7b, 0x56, 0xaf, 0x44, 0x95, + 0xc4, 0xc6, 0xff, 0x8b, + ], + [ + 0x8a, 0xc9, 0x0a, 0xc6, 0x8d, 0x5c, 0x71, 0xf7, 0x0d, 0x60, 0x13, 0x14, 0x21, 0xcc, + 0x03, 0x06, 0x79, 0x30, 0x77, 0x81, 0xd9, 0xc1, 0xc4, 0x1c, 0x9f, 0x55, 0xa8, 0x02, + 0xc5, 0xb2, 0x5e, 0x21, + ], + [ + 0x99, 0x86, 0x5e, 0xbf, 0x36, 0xe0, 0xaa, 0xf0, 0x3a, 0xed, 0x45, 0x21, 0xb9, 0xca, + 0x8c, 0x4f, 0x9e, 0x5a, 0x16, 0x37, 0x1d, 0xea, 0xd9, 0x84, 0xae, 0x56, 0x93, 0xf8, + 0x7f, 0xcb, 0x4a, 0x18, + ], + [ + 0xa3, 0xd2, 0x1d, 0x58, 0xbf, 0x0c, 0x09, 0xb2, 0x14, 0xce, 0xe6, 0x4f, 0x94, 0xf8, + 0x10, 0xc2, 0xc2, 0xf0, 0xcb, 0x13, 0xd8, 0x8a, 0x66, 0xbc, 0x8f, 0x4c, 0x76, 0x72, + 0xe8, 0xc4, 0x17, 0x99, + ], + [ + 0x7a, 0x3b, 0x86, 0x1b, 0xc0, 0x56, 0x74, 0x90, 0x30, 0x0f, 0x37, 0x0b, 0x20, 0x41, + 0x27, 0x22, 0x29, 0x0d, 0xa4, 0x4d, 0xa1, 0xeb, 0x68, 0xed, 0x82, 0x13, 0x1e, 0xa6, + 0xfd, 0x6f, 0x02, 0x41, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0x0a, 0x3c, 0x86, 0xe2, 0xd3, 0x37, 0x98, 0xf2, 0x68, 0x45, 0x42, 0xbf, 0x0b, 0x3d, + 0x52, 0x02, 0x4b, 0x08, 0x8b, 0x7a, 0x4b, 0xc8, 0xb0, 0x2a, 0xe4, 0x14, 0x86, 0x09, + 0x35, 0xdc, 0xa6, 0x7c, + ], + [ + 0x46, 0x8d, 0xfe, 0x65, 0xed, 0x50, 0x25, 0xbf, 0x7e, 0xfa, 0x04, 0x11, 0x28, 0x36, + 0x29, 0x06, 0xbd, 0x97, 0x29, 0xeb, 0x5b, 0x3a, 0xae, 0xbf, 0x66, 0xea, 0x5e, 0x59, + 0xe4, 0xd7, 0x17, 0x6c, + ], + ], + log_heights: [5, 12, 2, 4, 7, 2, 2, 6, 6, 2, 16, 4, 0, 5, 20], + keccak_rnd_chunks: 0, + hasher: HasherKind::Test, + chip_set: ChipSet { + keccak: false, + blake3: true, + bitwise: true, + }, + program_id: [ + 0x52, 0x12, 0x44, 0xef, 0x80, 0x0b, 0x23, 0x73, 0x84, 0x32, 0xa5, 0x3e, 0xa8, 0x48, + 0xc9, 0x78, 0x2e, 0x31, 0x9b, 0xf6, 0x2a, 0xcf, 0x0e, 0xd4, 0x3a, 0xbf, 0x01, 0x20, + 0xe3, 0x20, 0x2b, 0x34, + ], + }, +]; diff --git a/prover/src/lfm/rp1a_review_tests.rs b/prover/src/lfm/rp1a_review_tests.rs new file mode 100644 index 000000000..a909ce0fe --- /dev/null +++ b/prover/src/lfm/rp1a_review_tests.rs @@ -0,0 +1,118 @@ +//! The adversarial review REV-P1-A's prover tests (the subset REV-P1-JUDGE §7 +//! keeps), flipped where the judge's fixes change their expectation: the +//! uncapped odd-depth top's padding is bound on the host as in-guest (F4), and +//! a raw sample under Poseidon1 is refused loudly. The statement tag's test +//! (F2) is the WHIR block's own (`tests::block_whir_p1_tests`). + +use crate::tables::types::{FE, GoldilocksField}; +use crypto::merkle_tree::cap::CappedRoot; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use stark::config::Commitment; + +use super::algebraic_commit::{commitment_to_digest, digest_to_commitment}; +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::edsl::{WrapDigest, WrapHash}; +use super::executor::execute_serial; +use super::hash::HasherKind; +use super::p1w16_emit::hint_order; +use super::word::{LfmWord, base_word}; + +type B = super::p1_commit::P1BatchBackend; + +fn d(seed: u64) -> Commitment { + digest_to_commitment(&core::array::from_fn(|k| { + FE::from(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ (k as u64 + 1)) + })) +} + +/// ★ Host and guest agree at an UNCAPPED odd-depth tree (`arity4_cap = Off`): +/// the top level's two padding siblings must be the padding node on the host +/// (REV-P1-JUDGE F4), as the guest's walk supplies them as constants. A root +/// over non-zero padding opens on neither, at every real index. +#[test] +fn rp1a_uncapped_odd_top_host_and_guest_refuse() { + let depth = 3usize; + let leaves: Vec = (0..8).map(|i| d(100 + i)).collect(); + let n0 = B::hash_four(&[leaves[0], leaves[1], leaves[2], leaves[3]]); + let n1 = B::hash_four(&[leaves[4], leaves[5], leaves[6], leaves[7]]); + let junk = d(7); + let forged = B::hash_four(&[n0, n1, junk, junk]); + + for index in [0usize, 5] { + let base = index / 4 * 4; + let mut path: Vec = (base..base + 4) + .filter(|&c| c != index) + .map(|c| leaves[c]) + .collect(); + path.extend([if index < 4 { n1 } else { n0 }, junk, junk]); + + // Host (`stark::merkle_caps::TreeCheck` at c = 0 is this check). + assert!( + !CappedRoot::uncapped(&forged, depth).verify::(&path, index, leaves[index]), + "host refuses index {index}" + ); + + // Guest: the harvest's layout (`path_to_cap` → `hint_order`, which + // drops the two padding siblings), walked and compared with the root. + let triples: Vec<[Commitment; 3]> = + path.chunks_exact(3).map(|t| [t[0], t[1], t[2]]).collect(); + let hints = hint_order(index, depth, &triples); + let word = |c: &Commitment| -> LfmWord { commitment_to_digest(c) }; + let arenas: Vec> = vec![ + vec![word(&leaves[index])], + vec![base_word(FE::from(index as u64))], + hints.iter().map(word).collect(), + vec![word(&forged)], + ]; + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let ids: Vec<_> = arenas + .iter() + .map(|a| b.declare_arena(a.len() as u32)) + .collect(); + let leaf = WrapDigest::from_cell(b.hint_word(ids[0], 0)); + let idx = b.hint_felt(ids[1], 0); + let bits = b.bit_dec(idx, depth); + let h: Vec = (0..arenas[2].len() as u32) + .map(|i| WrapDigest::from_cell(b.hint_word(ids[2], i))) + .collect(); + let root = b.hint_word(ids[3], 0); + let root_lanes = [b.unpack(root)]; + let walked = super::edsl::wrap_merkle_walk(&mut b, leaf, &bits, &h); + super::edsl::assert_digest_eq_lanes(&mut b, walked, &root_lanes); + let program = compile(b.finish()); + super::validator::validate(&program).expect("admissible"); + assert!( + execute_serial(&program, &arenas, &HasherKind::Poseidon1W16).is_err(), + "guest refuses index {index}" + ); + } +} + +/// `TranscriptReplay::sample` (the keccak duplex's raw sample) has NO +/// Poseidon1 arm: under a Poseidon1 builder it falls to the RPX chain, which +/// would drain the pending segment away from ZisK's sponge. It is caught +/// LOUDLY, not silently: the chain's rows are twelve-felt `Hash` rows, and a +/// program that also has the socket's `Hash16` rows is refused by the +/// validator (`MixedHashWidths`). +#[test] +fn rp1a_a_raw_sample_under_poseidon1_is_refused_not_silent() { + use super::transcript_replay::TranscriptReplay; + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let a = b.declare_arena(1); + let w = b.hint_word(a, 0); + let mut t = TranscriptReplay::new(b"seed"); + let f = t.sample_felt(&mut b); // drives ZisK's sponge: Hash16 rows + b.public(f.as_cell()); + t.append_word(&mut b, w); // lazily pending + let s = t.sample(&mut b); // the missing arm: the RPX chain takes it + b.public(s.cells()[0]); + let f2 = t.sample_felt(&mut b); + b.public(f2.as_cell()); + let program = compile(b.finish()); + let err = super::validator::validate(&program).expect_err("mixed widths"); + assert!( + matches!(err, super::validator::LfmViolation::MixedHashWidths { .. }), + "{err:?}" + ); +} diff --git a/prover/src/lfm/rp1wa_review_tests.rs b/prover/src/lfm/rp1wa_review_tests.rs new file mode 100644 index 000000000..c6fc7eb77 --- /dev/null +++ b/prover/src/lfm/rp1wa_review_tests.rs @@ -0,0 +1,232 @@ +//! REV-P1W-A (break reviewer) probes of the Poseidon1 WHIR leaf: the width-8 +//! grind check against the host's accept set, bit for bit, and the +//! child-order 4-ary walk + cap mux against the host's trees at every index. + +use crate::tables::types::{FE, GoldilocksField}; +use crypto::merkle_tree::merkle::MerkleTree; +use stark::config::Commitment; + +use super::algebraic_commit::{commitment_to_digest, digest_to_commitment}; +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::edsl::{WrapDigest, WrapHash}; +use super::executor::execute_serial; +use super::hash::HasherKind; +use super::merkle_cap::CapCells; +use super::p1_commit::P1GrindDigest; +use super::word::{LfmWord, base_word}; + +type B = super::p1_commit::P1BatchBackend; + +fn mix(seed: u64, i: u64) -> u64 { + let mut z = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(i + 1); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +fn lanes(seed: u64) -> [FE; 4] { + core::array::from_fn(|k| FE::from(mix(seed, k as u64))) +} + +/// The emitted Poseidon1 grind check over `(seed, nonce, factor)`: Ok iff it +/// executes. +fn guest_grind(seed: [FE; 4], nonce: u64, factor: u8) -> Result<(), String> { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let a_seed = b.declare_arena(1); + let a_nonce = b.declare_arena(1); + let s = WrapDigest::from_cell(b.hint_word(a_seed, 0)); + let n = b.hint_felt(a_nonce, 0); + super::epoch::emit_grinding_check(&mut b, s, n, factor); + let program = compile(b.finish()); + super::validator::validate(&program).map_err(|e| format!("{e:?}"))?; + execute_serial( + &program, + &[vec![seed], vec![base_word(FE::from(nonce))]], + &HasherKind::Poseidon1W16, + ) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) +} + +/// ★ The sparse width-8 grind check accepts EXACTLY the nonces the host's +/// `is_valid_nonce::` accepts: over many nonces at several +/// factors and seeds (lanes near `p` included), and, sharper than one good +/// and one bad nonce, at the boundary — a nonce valid at `g − 1` bits and not +/// at `g` is refused at `g` and accepted at `g − 1` (an off-by-one in the bit +/// count or a low-bit check would pass the existing pair). Nonces past `p` +/// reduce on both sides alike. +#[test] +fn rp1wa_the_emitted_grind_accepts_exactly_the_hosts_nonces() { + let p = 0xFFFF_FFFF_0000_0001u64; + let mut seeds: Vec<[FE; 4]> = (0..3).map(|s| lanes(900 + s)).collect(); + seeds.push([ + FE::from(p - 1), + FE::from(p - 2), + FE::zero(), + FE::from(1u64 << 63), + ]); + let mut boundary_hits = 0; + for seed in &seeds { + let bytes = digest_to_commitment(seed); + for factor in [1u8, 2, 3, 5, 7] { + let host = + |n: u64| crypto::grinding::is_valid_nonce::(&bytes, n, factor); + let mut nonces: Vec = (0..48).collect(); + nonces.extend([p, p + 1, p + 5, u64::MAX, u64::MAX - 3]); + for n in nonces { + assert_eq!( + guest_grind(*seed, n, factor).is_ok(), + host(n), + "factor {factor} nonce {n}: guest and host disagree" + ); + } + if factor >= 2 { + let edge = (0u64..1 << 16).find(|&n| { + crypto::grinding::is_valid_nonce::(&bytes, n, factor - 1) + && !host(n) + }); + if let Some(n) = edge { + boundary_hits += 1; + assert!( + guest_grind(*seed, n, factor).is_err(), + "nonce {n} has exactly {} leading zero bits and passed at {factor}", + factor - 1 + ); + assert!( + guest_grind(*seed, n, factor - 1).is_ok(), + "nonce {n} at {}", + factor - 1 + ); + } + } + } + } + assert!( + boundary_hits >= 12, + "the boundary was probed: {boundary_hits}" + ); +} + +/// One opening of a host Poseidon1 tree over `2^depth` leaves checked in-guest +/// the way the WHIR leaf checks it: the owner path as the proof carries it +/// (`embed_cap_arity`), laid out in CHILD order by `child_path_to_cap`, the cap +/// authenticated and the opening checked through `verify_path_child` (or, at +/// `c = 0`, `walk4_child` to the root). `bits_of` is the index whose bits the +/// guest walks (the transcript's), `index` the leaf whose path is supplied. +/// `tamper` moves one arena word. +fn child_opening( + depth: usize, + c: usize, + index: usize, + bits_of: usize, + tamper: Option<(usize, usize)>, +) -> Result<(), String> { + let n = 1usize << depth; + let leaves: Vec = (0..n) + .map(|i| digest_to_commitment(&lanes(7_000 + depth as u64 * 131 + i as u64))) + .collect(); + let tree = MerkleTree::::build_from_hashed_leaves(leaves.clone()).expect("a tree"); + let mut path = tree + .get_proof_by_pos(index) + .expect("a leaf") + .merkle_path + .clone(); + let cap = if c == 0 { + Vec::new() + } else { + tree.cap(c).expect("the cap") + }; + if c > 0 { + crypto::merkle_tree::cap::embed_cap_arity(&mut [&mut path], depth, &cap, 4) + .expect("the owner path"); + } + let (hints, split) = + super::merkle_cap::child_path_to_cap(&path, depth, c, 4, true).expect("laid out"); + assert_eq!(split.unwrap_or_default(), cap); + assert_eq!(hints.len(), super::merkle_cap::path_hints(depth, c, 4)); + + let word = |c: &Commitment| -> LfmWord { commitment_to_digest(c) }; + let mut arenas: Vec> = vec![ + vec![word(&leaves[index])], + vec![base_word(FE::from(bits_of as u64))], + hints.iter().map(word).collect(), + cap.iter().map(word).collect(), + vec![word(&tree.root)], + ]; + if let Some((a, w)) = tamper { + arenas[a][w][1] += FE::one(); + } + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let lens: Vec = arenas.iter().map(|a| a.len() as u32).collect(); + let ids: Vec<_> = lens.iter().map(|&l| b.declare_arena(l)).collect(); + let leaf = WrapDigest::from_cell(b.hint_word(ids[0], 0)); + let idx = b.hint_felt(ids[1], 0); + let bits = b.bit_dec(idx, depth); + let h: Vec = (0..lens[2]) + .map(|i| WrapDigest::from_cell(b.hint_word(ids[2], i))) + .collect(); + let root = b.hint_word(ids[4], 0); + let root_lanes = [b.unpack(root)]; + if c == 0 { + let walked = super::p1w16_emit::walk4_child(&mut b, leaf, &bits, &h); + super::edsl::assert_digest_eq_lanes(&mut b, walked, &root_lanes); + } else { + let nodes: Vec = (0..lens[3]) + .map(|i| WrapDigest::from_cell(b.hint_word(ids[3], i))) + .collect(); + let capped = CapCells::authenticate_at(&mut b, &nodes, c, &root_lanes); + capped.verify_path_child(&mut b, leaf, &bits, &h); + } + let program = compile(b.finish()); + super::validator::validate(&program).map_err(|e| format!("{e:?}"))?; + execute_serial(&program, &arenas, &HasherKind::Poseidon1W16) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) +} + +/// ★ The production walk (child order, two placing `Select`s a level) and the +/// 4-ary cap mux against the host's trees, EXHAUSTIVELY at small depths: at +/// every depth 1–7 (both parities), every cap height up to the tree's levels, +/// and EVERY leaf index, the honest opening executes; the same path walked +/// with another index's bits does not (the index places the node); and every +/// arena word the check reads — leaf, each sibling, each cap node (queried or +/// not), the root — moved refuses it. +#[test] +fn rp1wa_the_child_order_walk_and_cap_bind_every_index() { + for depth in 1usize..=7 { + let n = 1usize << depth; + let levels = crypto::merkle_tree::cap::tree_levels(depth, 4); + for c in 0..=levels { + let hints = super::merkle_cap::path_hints(depth, c, 4); + let nodes = if c == 0 { + 0 + } else { + super::merkle_cap::cap_nodes(depth, c, 4) + }; + for index in 0..n { + child_opening(depth, c, index, index, None) + .unwrap_or_else(|e| panic!("depth {depth} cap {c} index {index}: {e}")); + let other = (index + 1 + index % 3) % n; + if other != index { + assert!( + child_opening(depth, c, index, other, None).is_err(), + "depth {depth} cap {c}: leaf {index}'s path walked at index {other}" + ); + } + // Every word, at a spread of indices (all of them at depth <= 5). + if depth <= 5 || index % 9 == 0 { + let mut words = vec![(0usize, 0usize), (4, 0)]; + words.extend((0..hints).map(|w| (2, w))); + words.extend((0..nodes).map(|w| (3, w))); + for (a, w) in words { + assert!( + child_opening(depth, c, index, index, Some((a, w))).is_err(), + "depth {depth} cap {c} index {index}: arena {a} word {w} moved" + ); + } + } + } + } + } +} diff --git a/prover/src/lfm/rp1wb_review_tests.rs b/prover/src/lfm/rp1wb_review_tests.rs new file mode 100644 index 000000000..e532e8f1f --- /dev/null +++ b/prover/src/lfm/rp1wb_review_tests.rs @@ -0,0 +1,278 @@ +//! Reviewer r-p1w-b's tests on the Poseidon1 WHIR leaf (REV-P1W-B): the +//! child-order walk is bound to the transcript's query index (no sidecar), and +//! the sparse width-8 permutation is the dense one. (The grind's accept set is +//! pinned by `rp1wa_review_tests`.) + +use crypto::fiat_shamir::p1_transcript::P1Transcript; +use crypto::hash::poseidon1_w8; +use multilinear::mle::Mle; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, ChainProof, GrindBits, RoundOpenings, commit, prove, + verify, +}; +use multilinear::whir_hash::P1Whir; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::commitment_to_digest; +use super::builder::{Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::WrapHash; +use super::executor::execute; +use super::hash::HasherKind; +use super::p1w16_emit::{Lane, w8_permute, w8_permute_lanes}; +use super::validator::validate; +use super::whir_chain::{ + ChainShape, RoundStorage, emit_verify_weighted, push_round_words, round_words, +}; +use super::whir_poly::emit_eq_eval; +use super::whir_transcript::WhirTranscript; +use super::word::{LfmWord, base_word, ext_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +const SOCKET: HasherKind = HasherKind::Poseidon1W16; + +fn p1_builder() -> LfmBuilder { + LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1) +} + +fn felt(seed: u64, i: usize) -> FE { + let mut z = seed + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(i as u64 + 1); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + FE::from(z ^ (z >> 31)) +} + +fn fee(seed: u64, i: usize) -> FEE { + FEE::new([ + felt(seed, 3 * i), + felt(seed, 3 * i + 1), + felt(seed, 3 * i + 2), + ]) +} + +fn run(b: LfmBuilder, arenas: &[Vec]) -> Result, String> { + let program = compile(b.finish()); + validate(&program).map_err(|e| format!("not admissible: {e:?}"))?; + let exec = execute(&program, arenas, &SOCKET).map_err(|e| format!("{e:?}"))?; + Ok(exec.public_words.iter().flat_map(|(_, w)| *w).collect()) +} + +// ======================= the walk is bound to the index ===================== + +fn p1_config(num_queries: usize, grind: u8, cap4: CapPolicy) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries, + grind: GrindBits::uniform(grind), + format: ChainFormat { + arity4_cap: cap4, + ..ChainFormat::DEFAULT + }, + } +} + +struct Fixture { + cfg: ChainConfig, + proof: ChainProof, + z: Vec, + y: FEE, + root_bytes: [u8; 32], + domain: Domain, + shape: ChainShape, +} + +fn fixture(num_vars: usize, cfg: ChainConfig) -> Fixture { + let vals = (0..1usize << num_vars) + .map(|i| FE::from((i as u64).wrapping_mul(0x2545_F491_4F6C_DD1D) >> 13)) + .collect(); + let f = Mle::new(vals).expect("a cube"); + let z: Vec = (0..num_vars).map(|i| fee(77, i)).collect(); + let y = f.evaluate_in::(&z).expect("f takes its own point"); + let (commitment, domain) = commit::(&f, &cfg, true).expect("commits"); + let mut proving = P1Transcript::new(); + let proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut proving).expect("proves"); + let root_bytes = commitment.root(); + let shape = ChainShape::new_at(&cfg, num_vars, 4); + let fx = Fixture { + cfg, + proof, + z, + y, + root_bytes, + domain, + shape, + }; + assert!( + host_accepts(&fx, &fx.proof), + "the host accepts its own proof" + ); + fx +} + +fn host_accepts(f: &Fixture, proof: &ChainProof) -> bool { + let mut t = P1Transcript::new(); + verify::(proof, &f.root_bytes, &f.z, f.y, &f.domain, &f.cfg, &mut t).is_ok() +} + +fn chain_program(shape: &ChainShape) -> LfmProgram { + let mut b = p1_builder(); + let mut total = (shape.num_vars + 3) as u32; + let first_round = total; + for r in 0..shape.rounds() { + total += round_words(shape, r); + } + let arena = b.declare_arena(total); + let mut transcript = WhirTranscript::for_builder(&mut b); + let z: Vec = (0..shape.num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let y = b.hint_word(arena, shape.num_vars as u32).as_ext(); + let root = b.hint_word(arena, shape.num_vars as u32 + 1); + let root_lanes = b.unpack(root); + let final_value = b.hint_word(arena, shape.num_vars as u32 + 2).as_ext(); + let storage = RoundStorage::hint(&mut b, arena, first_round, shape); + let (current, next) = storage.openings(); + let wires = storage.wires(¤t, &next); + let domain = Domain::::new(shape.domain_log[0]).expect("the first domain"); + emit_verify_weighted( + &mut b, + &mut transcript, + &wires, + &root_lanes, + final_value, + y, + shape, + &domain, + |b, alphas| emit_eq_eval(b, &z, alphas), + ); + let program = compile(b.finish()); + validate(&program).expect("admissible"); + program +} + +fn chain_arena(f: &Fixture, proof: &ChainProof) -> Vec { + let mut words: Vec = f.z.iter().map(ext_word).collect(); + words.push(ext_word(&f.y)); + words.push(commitment_to_digest(&f.root_bytes)); + words.push(ext_word(&proof.final_value)); + push_round_words(&mut words, &f.shape, proof); + words +} + +/// Swaps two non-owner openings of one side of round `r` (current or +/// successor) when they open different blocks: each is a valid opening of +/// the committed tree, at the OTHER query's index. `false` when the side has +/// fewer than three openings or the two open the same block. +fn swap_openings(proof: &mut ChainProof, r: usize, current: bool) -> bool { + fn side( + v: &mut [multilinear::whir_commit::CosetOpening], + ) -> bool { + if v.len() < 3 || v[1].values == v[2].values { + return false; + } + v.swap(1, 2); + true + } + match (&mut proof.rounds[r].openings, current) { + (RoundOpenings::Base(p), true) => side(&mut p.current), + (RoundOpenings::Base(p), false) => side(&mut p.next), + (RoundOpenings::Extension(p), true) => side(&mut p.current), + (RoundOpenings::Extension(p), false) => side(&mut p.next), + (RoundOpenings::BaseGrouped(_), _) => unreachable!("a per-polynomial chain"), + } +} + +/// ★ The leaf's arena is filled from the proof alone (no index sidecar), and +/// the walk takes the proof's siblings in child order: the opening a query +/// checks must still be the one at the TRANSCRIPT's index. Two openings of the +/// committed tree, each valid at its own index, swapped between two queries, +/// are refused by the host and by the machine — on round 0's base tree, on +/// a later round's extension tree, and on a successor tree, uncapped and at +/// the production 4-ary cap (C2). +#[test] +fn rp1wb_an_opening_moved_to_another_query_index_is_refused() { + for cap4 in [CapPolicy::Off, CapPolicy::Fixed(2)] { + let f = fixture(10, p1_config(4, 0, cap4)); + let program = chain_program(&f.shape); + assert!( + execute(&program, &[chain_arena(&f, &f.proof)], &SOCKET).is_ok(), + "the untouched proof executes (caps {:?})", + f.shape.caps + ); + let mut tried = 0; + for (r, current) in [(0usize, true), (0, false), (1, true)] { + let mut forged = f.proof.clone(); + if !swap_openings(&mut forged, r, current) { + continue; + } + tried += 1; + assert!( + !host_accepts(&f, &forged), + "round {r} current={current} caps {:?}: the host must refuse", + f.shape.caps + ); + assert!( + execute(&program, &[chain_arena(&f, &forged)], &SOCKET).is_err(), + "round {r} current={current} caps {:?}: the machine must refuse an opening \ + at another query's index", + f.shape.caps + ); + } + assert!( + tried >= 2, + "at least two sides were swapped (caps {cap4:?})" + ); + } +} + +/// The sparse width-8 permutation equals the dense one (and the host's) on +/// random inputs, every lane variable and under random constant/variable +/// patterns, at every `keep`. +#[test] +fn rp1wb_the_sparse_w8_is_the_dense_w8() { + for trial in 0..6u64 { + let input: [FE; 8] = core::array::from_fn(|i| felt(100 + trial, i)); + let want = poseidon1_w8::permute(input); + let mask = (trial * 0x5b) as u8; // which lanes are constants + for keep in [1usize, 4, 8] { + let mut b = p1_builder(); + let a = b.declare_arena(8); + let vars: [_; 8] = core::array::from_fn(|i| b.hint_felt(a, i as u32)); + let lanes: [Lane; 8] = core::array::from_fn(|i| { + if mask >> i & 1 == 1 { + Lane::Const(input[i]) + } else { + Lane::Var(vars[i]) + } + }); + let sparse = w8_permute_lanes(&mut b, lanes, keep); + let dense = w8_permute(&mut b, vars); + for (s, d) in sparse.iter().zip(&dense) { + b.public(s.as_cell()); + b.public(d.as_cell()); + } + let arena: Vec = input.iter().map(|v| base_word(*v)).collect(); + let got = run(b, &[arena]).expect("executes"); + for k in 0..keep { + assert_eq!( + got[8 * k], + want[k], + "trial {trial} keep {keep} sparse lane {k}" + ); + assert_eq!( + got[8 * k + 4], + want[k], + "trial {trial} keep {keep} dense lane {k}" + ); + } + } + } +} diff --git a/prover/src/lfm/rpo.rs b/prover/src/lfm/rpo.rs new file mode 100644 index 000000000..eee155ed9 --- /dev/null +++ b/prover/src/lfm/rpo.rs @@ -0,0 +1,914 @@ +//! Rescue-Prime Optimized (RPO256) over Goldilocks at width 12: the round +//! primitives and constants [`super::rpx`] runs on, and the external +//! known-answer anchor for them. RPO itself no longer has an `LFM_HASH` socket +//! arm; RPX is the tenant. +//! +//! # Why this hash +//! +//! The Poseidon family is off the table (see [`super::poseidon`]'s header and +//! the break record: eprint 2026/306 for Poseidon2's linear layers, eprint +//! 2026/1692 for Poseidon-original's partial layer). RPO is the strongest +//! algebraic candidate that the break class does not structurally reach: it has +//! **no partial rounds** — all twelve lanes pass a nonlinear map in every +//! half-round, so the subspace-restriction gadget has no affine complement to +//! live in — and **no cheap algebraic direction**, because composing forward +//! costs `x^{1/7}` layers and composing backward costs `x^7` layers. That is +//! Rescue's entire design thesis and it is why the Poseidon literature keeps not +//! porting to it. +//! +//! ⚠ This is a risk decision, not an arithmetic one: RPO is a 2022 design and +//! carries a "young design" discount BLAKE3 does not. It is unbroken since the +//! Marvellous line began in 2019, and the spec defends its N = 7 (12.5% under +//! its own formula's 8) with a 1.5× margin argument, but the Gröbner frontier +//! moves every year. +//! +//! # Parameter provenance — READ THIS BEFORE TRUSTING A MEASUREMENT +//! +//! Every constant below has **two independent sources that were checked against +//! each other**, which is the whole reason to trust them: +//! +//! 1. The spec's own generator ([eprint 2022/1577](https://eprint.iacr.org/2022/1577), +//! reference implementation at `github.com/ASDiscreteMathematics/rpo`, +//! `rescue_prime_optimized.sage::get_round_constants`): round constants are +//! `SHAKE256("RPO(18446744069414584321,12,4,128)", 9·2·12·7)` cut into +//! nine-byte little-endian chunks reduced mod `p`. +//! 2. `miden-crypto`'s shipped `ARK1`/`ARK2` tables +//! (`src/hash/algebraic_sponge/rescue/mod.rs`), production code since 2022. +//! +//! The SHAKE256 derivation was re-run outside this repository and reproduces +//! miden's 168 constants exactly. The MDS row is likewise the spec's +//! `get_mds(12)` and miden's `MDS` first row, identically. +//! +//! # Lane convention — a gift, not a choice +//! +//! The RPO **permutation** is lane-agnostic; a deployment picks which lanes are +//! rate, which are capacity and where the digest is read. Two conventions exist: +//! the 2022 paper's (capacity `0..4`, rate `4..12`, digest `4..8`) and miden's +//! (rate `0..8`, capacity `8..12`, digest `0..4`). +//! +//! **This module follows miden's, because it is exactly the `LFM_HASH` socket's +//! own layout**: the socket materializes the capacity cell at lanes 8–11 +//! (`S8..S11`) and reads the digest from `OUT0..3`. No lane permutation is +//! needed anywhere — the chip's frozen prefix already sits where RPO256 wants +//! it, and a `Compress` row with the zero IV is *literally* `Rpo256::merge`. +//! +//! # Domain separation — the capacity slot, per miden's blessed construction +//! +//! [`LfmHasher`]'s trait defaults let a single-domain hasher ship a transcript +//! that is indistinguishable from a Merkle parent; `hash.rs` records that +//! weakening deliberately and says a production candidate must override it. +//! RPO does, by the mechanism miden ships as `merge_in_domain`: **capacity lane +//! 9 carries a domain identifier**, while capacity lane 8 stays reserved for the +//! sponge's padding flag. The security argument is the RPX spec's Appendix C +//! ([eprint 2023/1045](https://eprint.iacr.org/2023/1045)): setting a capacity +//! element to a domain tag degrades only pre-image resistance, by at most the +//! log₂ of the domain space, and pre-image is not the sponge's binding term +//! until it falls under 2^128 — which three one-word tags do not approach. +//! +//! The tags reuse the BLAKE3 socket's ASCII names ([`super::blake3_socket`]'s +//! `TAG_LFMT` / `TAG_LFML`) so one domain has one name across both tenants. +//! `Compress` takes domain ZERO on purpose: that makes a Merkle parent under +//! this machine bit-identical to a standard `Rpo256::merge`, externally +//! checkable against miden. +//! +//! # Oracle +//! +//! [`tests::the_sponge_matches_the_miden_known_answer_vectors`] replays all +//! nineteen of miden-crypto's `hash_elements` test vectors through this +//! permutation. Nothing in this repository produced those numbers, and they +//! pin the constants, the round order, the MDS orientation, both S-box +//! exponents and the lane convention at once. + +use crate::tables::types::FE; + +use super::hash::{HASH_STATE_FELTS, LfmHasher}; +use super::word::LfmWord; + +/// The forward S-box exponent. Like Poseidon's, 7 is forced by Goldilocks: +/// `p - 1 = 2^32 · 3 · 5 · 17 · 257 · 65537`, so neither 3 nor 5 is coprime to +/// it and neither `x³` nor `x⁵` is a permutation. +pub const ALPHA: u32 = 7; + +/// The inverse S-box exponent, `ALPHA⁻¹ mod (p − 1)`. +/// +/// ≈ 2^63, and that is the point: the map is cheap in one direction and +/// astronomically dense in the other, in BOTH directions of the round function. +/// [`tests::the_inverse_exponent_inverts_alpha`] re-derives it rather than +/// trusting the literal. +pub const INV_ALPHA: u64 = 10540996611094048183; + +/// Rounds. The spec's own formula gives 8; RPO ships 7 and defends the 12.5% +/// shave in §4.2 with a 1.5× margin argument and Gröbner estimates above twice +/// the security level. This is the number the AIR's column count is linear in — +/// raising it to 8 costs 48 value columns. +pub const NUM_ROUNDS: usize = 7; + +/// Half-rounds per round: forward `x^7`, then inverse `x^{1/7}`. +pub const HALVES_PER_ROUND: usize = 2; + +/// First ROW of the circulant MDS matrix, so `M[i][j] = MDS_CIRC_ROW[(j - i) mod 12]`. +/// +/// The spec's `get_mds(12)` and miden's `MDS[0]`, identically. RPO's security +/// argument is MDS-AGNOSTIC (spec §4.1: "Rescue-Prime is secure when +/// instantiated with any MDS matrix"), so this row is a speed choice — it is +/// NTT-friendly — and not a security parameter. That matters: it is exactly the +/// property Poseidon2 lacked. +pub const MDS_CIRC_ROW: [u64; HASH_STATE_FELTS] = [7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8]; + +/// [`MDS_CIRC_ROW`] expanded to the whole circulant at compile time, +/// `MDS_MATRIX[i][j] = MDS_CIRC_ROW[(j − i) mod 12]`, so [`Rpo256::mds`] reads +/// every row as constants instead of rotating an index at run time. +const MDS_MATRIX: [[u64; HASH_STATE_FELTS]; HASH_STATE_FELTS] = { + let mut m = [[0; HASH_STATE_FELTS]; HASH_STATE_FELTS]; + let mut i = 0; + while i < HASH_STATE_FELTS { + let mut j = 0; + while j < HASH_STATE_FELTS { + m[i][j] = MDS_CIRC_ROW[(j + HASH_STATE_FELTS - i) % HASH_STATE_FELTS]; + j += 1; + } + i += 1; + } + m +}; + +/// Round constants for the FIRST half of each round — added after the first MDS +/// and before the forward `x^7` layer. See the module header for the two-source +/// provenance. +pub const ARK1: [[u64; HASH_STATE_FELTS]; NUM_ROUNDS] = [ + [ + 5789762306288267392, + 6522564764413701783, + 17809893479458208203, + 107145243989736508, + 6388978042437517382, + 15844067734406016715, + 9975000513555218239, + 3344984123768313364, + 9959189626657347191, + 12960773468763563665, + 9602914297752488475, + 16657542370200465908, + ], + [ + 12987190162843096997, + 653957632802705281, + 4441654670647621225, + 4038207883745915761, + 5613464648874830118, + 13222989726778338773, + 3037761201230264149, + 16683759727265180203, + 8337364536491240715, + 3227397518293416448, + 8110510111539674682, + 2872078294163232137, + ], + [ + 18072785500942327487, + 6200974112677013481, + 17682092219085884187, + 10599526828986756440, + 975003873302957338, + 8264241093196931281, + 10065763900435475170, + 2181131744534710197, + 6317303992309418647, + 1401440938888741532, + 8884468225181997494, + 13066900325715521532, + ], + [ + 5674685213610121970, + 5759084860419474071, + 13943282657648897737, + 1352748651966375394, + 17110913224029905221, + 1003883795902368422, + 4141870621881018291, + 8121410972417424656, + 14300518605864919529, + 13712227150607670181, + 17021852944633065291, + 6252096473787587650, + ], + [ + 4887609836208846458, + 3027115137917284492, + 9595098600469470675, + 10528569829048484079, + 7864689113198939815, + 17533723827845969040, + 5781638039037710951, + 17024078752430719006, + 109659393484013511, + 7158933660534805869, + 2955076958026921730, + 7433723648458773977, + ], + [ + 16308865189192447297, + 11977192855656444890, + 12532242556065780287, + 14594890931430968898, + 7291784239689209784, + 5514718540551361949, + 10025733853830934803, + 7293794580341021693, + 6728552937464861756, + 6332385040983343262, + 13277683694236792804, + 2600778905124452676, + ], + [ + 7123075680859040534, + 1034205548717903090, + 7717824418247931797, + 3019070937878604058, + 11403792746066867460, + 10280580802233112374, + 337153209462421218, + 13333398568519923717, + 3596153696935337464, + 8104208463525993784, + 14345062289456085693, + 17036731477169661256, + ], +]; + +/// Round constants for the SECOND half of each round — added after the second +/// MDS and before the inverse `x^{1/7}` layer. +pub const ARK2: [[u64; HASH_STATE_FELTS]; NUM_ROUNDS] = [ + [ + 6077062762357204287, + 15277620170502011191, + 5358738125714196705, + 14233283787297595718, + 13792579614346651365, + 11614812331536767105, + 14871063686742261166, + 10148237148793043499, + 4457428952329675767, + 15590786458219172475, + 10063319113072092615, + 14200078843431360086, + ], + [ + 6202948458916099932, + 17690140365333231091, + 3595001575307484651, + 373995945117666487, + 1235734395091296013, + 14172757457833931602, + 707573103686350224, + 15453217512188187135, + 219777875004506018, + 17876696346199469008, + 17731621626449383378, + 2897136237748376248, + ], + [ + 8023374565629191455, + 15013690343205953430, + 4485500052507912973, + 12489737547229155153, + 9500452585969030576, + 2054001340201038870, + 12420704059284934186, + 355990932618543755, + 9071225051243523860, + 12766199826003448536, + 9045979173463556963, + 12934431667190679898, + ], + [ + 18389244934624494276, + 16731736864863925227, + 4440209734760478192, + 17208448209698888938, + 8739495587021565984, + 17000774922218161967, + 13533282547195532087, + 525402848358706231, + 16987541523062161972, + 5466806524462797102, + 14512769585918244983, + 10973956031244051118, + ], + [ + 6982293561042362913, + 14065426295947720331, + 16451845770444974180, + 7139138592091306727, + 9012006439959783127, + 14619614108529063361, + 1394813199588124371, + 4635111139507788575, + 16217473952264203365, + 10782018226466330683, + 6844229992533662050, + 7446486531695178711, + ], + [ + 3736792340494631448, + 577852220195055341, + 6689998335515779805, + 13886063479078013492, + 14358505101923202168, + 7744142531772274164, + 16135070735728404443, + 12290902521256031137, + 12059913662657709804, + 16456018495793751911, + 4571485474751953524, + 17200392109565783176, + ], + [ + 17130398059294018733, + 519782857322261988, + 9625384390925085478, + 1664893052631119222, + 7629576092524553570, + 3485239601103661425, + 9755891797164033838, + 15218148195153269027, + 16460604813734957368, + 9643968136937729763, + 3611348709641382851, + 18256379591337759196, + ], +]; + +/// Felts the sponge absorbs per permutation — the RATE. +/// +/// ★ **This is why the census needs no RPO arm.** The census counts a "block" +/// as an 8-felt absorb (`epoch_verify::blocks_for`, BLAKE3's `⌈felts/8⌉`), and +/// RPO's rate is also 8, with the same no-spurious-final-block rule: an exact +/// multiple of the rate emits no extra permutation under either. So for every +/// leaf of one felt or more the two counts are EQUAL, and a Merkle parent is one +/// invocation under both (four-felt digests on both sides, eight felts in). +/// `rpo_chip_tests::the_rate_eight_census_is_hash_invariant` is the proof. +pub const RATE_FELTS: usize = 8; + +/// Capacity lane carrying the sponge padding flag — reserved, never a domain. +/// +/// Miden's `hash_elements` writes `total_len % RATE` here; the socket's modes +/// are all exactly one full rate block, for which that flag is zero, so the +/// socket leaves it zero. Naming it stops the domain from being put here. +pub const CAPACITY_PAD_LANE: usize = 0; + +/// Capacity lane carrying the DOMAIN identifier — miden's `merge_in_domain` +/// slot. See the module header for why this is sound. +pub const CAPACITY_DOMAIN_LANE: usize = 1; + +/// The `Compress` (Merkle parent) domain: ZERO, deliberately. +/// +/// A compress row is then bit-identical to `Rpo256::merge`, so a parent this +/// machine proves is checkable against miden's shipped implementation without +/// knowing anything about this codebase. +pub const DOMAIN_COMPRESS: u64 = 0; + +/// The Fiat–Shamir transcript domain — `"LFMT"` as a little-endian `u32`, the +/// same name [`super::blake3_socket::TAG_LFMT`] carries. +pub const DOMAIN_TRANSCRIPT: u64 = u32::from_le_bytes(*b"LFMT") as u64; + +/// The Merkle LEAF domain — `"LFML"`, matching +/// [`super::blake3_socket::TAG_LFML`]. +pub const DOMAIN_LEAF: u64 = u32::from_le_bytes(*b"LFML") as u64; + +/// The capacity cell for a domain: `[0, domain, 0, 0]`. +/// +/// One rule, so the chip's constraints, the executor's state and the trace +/// filler cannot disagree about which lane the tag lives in. +pub const fn domain_iv(domain: u64) -> [u64; 4] { + let mut iv = [0u64; 4]; + iv[CAPACITY_DOMAIN_LANE] = domain; + iv +} + +/// Rescue-Prime Optimized at width 12, rate 8, capacity 4, 7 rounds — RPO256. +pub struct Rpo256; + +impl Rpo256 { + /// `x^7` by square-and-multiply, in exactly the association the AIR's + /// degree-3 lowering uses (`x²`, `x³ = x²·x`, `x^7 = (x³)²·x`), so the + /// executor and the chip agree by construction rather than by luck. + pub(crate) fn sbox(x: &FE) -> FE { + let x2 = x * x; + let x3 = &x2 * x; + let x6 = &x3 * &x3; + &x6 * x + } + + /// `x^{1/7}` over the WHOLE STATE — the inverse S-box layer, by + /// miden-crypto's documented addition chain (72 multiplications for a + /// ~2^63 exponent, against ~93 for naive square-and-multiply). + /// + /// ★ **Whole-state rather than per-element, and that is a measurement, not + /// a style.** The chain is 72 multiplications each depending on the last, + /// so a single lane is LATENCY-bound: the Goldilocks multiply's ~2.6 ns of + /// latency times 72 is the entire cost, and the multiplier pipeline sits + /// idle between them. The twelve lanes are independent, so running them in + /// lockstep interleaves twelve chains and fills that pipeline — the same + /// shape miden's scalar fallback has, and the shape its SVE/AVX2/AVX512 + /// kernels vectorize from. + /// + /// This layer is 87% of the permutation and the permutation is the host's + /// commitment cost, so this loop is the hot one in the whole hash. + /// [`tests::the_inverse_sbox_chain_agrees_with_the_exponent`] pins it + /// against `pow(INV_ALPHA)` — a different algorithm for the same number — + /// and [`tests::the_inverse_sbox_inverts_the_forward_sbox`] pins both + /// against the property that actually matters. + pub fn inv_sbox_layer(state: &mut [FE; HASH_STATE_FELTS]) { + // `base^(2^m) · tail`, lane-wise — the chain's one building block. + fn exp_acc( + base: &[FE; HASH_STATE_FELTS], + tail: &[FE; HASH_STATE_FELTS], + m: usize, + ) -> [FE; HASH_STATE_FELTS] { + let mut acc = *base; + for _ in 0..m { + for a in acc.iter_mut() { + *a = a.square(); + } + } + core::array::from_fn(|i| &acc[i] * &tail[i]) + } + + let t1: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| state[i].square()); + let t2: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| t1[i].square()); + let t3 = exp_acc(&t2, &t2, 3); + let t4 = exp_acc(&t3, &t3, 6); + let t5 = exp_acc(&t4, &t4, 12); + let t6 = exp_acc(&t5, &t3, 6); + let t7 = exp_acc(&t6, &t6, 31); + for (i, s) in state.iter_mut().enumerate() { + let a = (&t7[i].square() * &t6[i]).square().square(); + let b = &(&t1[i] * &t2[i]) * &*s; + *s = &a * &b; + } + } + + /// [`Rpo256::inv_sbox_layer`] for a single element. + /// + /// The trace filler and the permutation both go through the layer; this + /// exists for the tests and for the witness recorder's per-lane reading, and + /// it is deliberately the SAME chain rather than a second transcription. + pub fn inv_sbox(x: &FE) -> FE { + let mut state = [*x; HASH_STATE_FELTS]; + Self::inv_sbox_layer(&mut state); + state[0] + } + + /// The circulant MDS product, `out_i = Σ_j MDS_CIRC_ROW[(j − i) mod 12]·s_j` + /// — the orientation the external KAT pins, and the same one + /// `poseidon::PoseidonGoldilocks::mds` uses. + /// + /// ★ **One `u128` accumulation and one reduction per lane, not twelve field + /// multiplications.** The MDS constants are all ≤ 26, so every term + /// `c·s_j` fits in 70 bits and the whole twelve-term row sum fits in 73 — + /// comfortably inside a `u128` (the bound is asserted in + /// [`tests::the_mds_row_sum_cannot_overflow_a_u128`]). So the row is + /// accumulated with no reduction at all and reduced once at the end, using + /// `2^64 ≡ EPSILON (mod p)`: `hi·2^64 + lo ≡ lo + hi·EPSILON`, and with + /// `hi < 2^9` the correction term `hi·EPSILON < 2^41` needs no reduction of + /// its own. + /// + /// This matters because the MDS runs FOURTEEN times per permutation and, + /// once the inverse S-box layer stops dominating, it is the largest + /// remaining share of the host's commitment cost. + /// + /// ⚠ **Plain loops over [`MDS_MATRIX`], not a `core::array::from_fn` + /// closure.** The closure's `from_fn` wrapper is a generic instance that + /// rustc places in a codegen unit of its own choosing, and it is inlined here + /// only when that unit happens to be this function's. When it is not, every + /// lane becomes an out-of-line call that re-derives `(j − i) mod 12` with a + /// 64-bit multiply per term — about a fifth more instructions per RPX + /// permutation, decided by unrelated edits elsewhere in the crate. Loops over + /// a compile-time matrix compile the same way in every build. + pub(crate) fn mds(state: &[FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + /// `2^32 − 1`, and `2^64 ≡ EPSILON (mod p)` for the Goldilocks prime. + /// Written here rather than imported because the field crate keeps its + /// own copy private; [`tests::the_epsilon_identity_holds`] re-derives it. + const EPSILON: u64 = 0xFFFF_FFFF; + + let mut out = [FE::zero(); HASH_STATE_FELTS]; + for (row, o) in MDS_MATRIX.iter().zip(out.iter_mut()) { + let mut acc: u128 = 0; + for (c, s) in row.iter().zip(state) { + acc += (*s.value() as u128) * (*c as u128); + } + let lo = acc as u64; + let hi = (acc >> 64) as u64; + // hi < 2^9, so hi·EPSILON < 2^41 and neither `from` reduces twice. + *o = FE::from(lo) + FE::from(hi * EPSILON); + } + out + } +} + +impl LfmHasher for Rpo256 { + /// Seven rounds of `MDS → +ARK1 → x^7 → MDS → +ARK2 → x^{1/7}`. + /// + /// Note the round STARTS with the linear layer, which is RPO's reordering + /// of Rescue-Prime (spec §2.4) and is what lets an AIR fold a round into + /// one row: nothing precedes the first MDS, so the row's input columns feed + /// it directly. + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + let mut s = state; + for r in 0..NUM_ROUNDS { + s = Self::mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += FE::from(ARK1[r][lane]); + } + for v in s.iter_mut() { + *v = Self::sbox(v); + } + s = Self::mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += FE::from(ARK2[r][lane]); + } + Self::inv_sbox_layer(&mut s); + } + s + } + + /// The Merkle-parent capacity: all zeros, so a compress row IS + /// `Rpo256::merge`. + fn compress_iv(&self) -> LfmWord { + domain_iv(DOMAIN_COMPRESS).map(FE::from) + } + + /// The transcript capacity — `merge_in_domain(·, "LFMT")`. + fn transcript_iv(&self) -> LfmWord { + domain_iv(DOMAIN_TRANSCRIPT).map(FE::from) + } + + /// The leaf capacity — `merge_in_domain(·, "LFML")`. + /// + /// This is what retires the weakening `LfmHasher::leaf_out` records for the + /// single-domain hashers: under RPO a leaf and a parent over the same two + /// cells are different functions, so the O5 second-preimage split no longer + /// rests on fixed tree depth alone. + fn leaf_iv(&self) -> LfmWord { + domain_iv(DOMAIN_LEAF).map(FE::from) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// miden-crypto's own `hash_elements` known-answer table — an EXTERNAL + /// oracle. + /// + /// Source: `miden-crypto/src/hash/algebraic_sponge/rescue/rpo/tests.rs`, + /// `EXPECTED` / `hash_test_vectors`. Entry `n` is the digest of the field + /// elements `[0, 1, …, n]`. Nothing in this repository produced these + /// seventy-six numbers. + /// + /// Entries 1–7 and 9–19 exercise the padding path (`total_len % 8 ≠ 0`), + /// entries 8 and 16 the exact-block path, and everything above 8 chains two + /// permutations through the capacity — so the table pins the sponge's + /// carry, not only one permutation. + const MIDEN_HASH_ELEMENTS: [[u64; 4]; 19] = [ + [ + 8563248028282119176, + 14757918088501470722, + 14042820149444308297, + 7607140247535155355, + ], + [ + 8762449007102993687, + 4386081033660325954, + 5000814629424193749, + 8171580292230495897, + ], + [ + 16710087681096729759, + 10808706421914121430, + 14661356949236585983, + 5683478730832134441, + ], + [ + 5309818427047650994, + 17172251659920546244, + 8288476618870804357, + 18080473279382182941, + ], + [ + 3647545403045515695, + 3358383208908083302, + 8797161010298072910, + 2412100201132087248, + ], + [ + 8409780526028662686, + 214479528340808320, + 13626616722984122219, + 13991752159726061594, + ], + [ + 4800410126693035096, + 8293686005479024958, + 16849389505608627981, + 12129312715917897796, + ], + [ + 5421234586123900205, + 9738602082989433872, + 7017816005734536787, + 8635896173743411073, + ], + [ + 11707446879505873182, + 7588005580730590001, + 4664404372972250366, + 17613162115550587316, + ], + [ + 6991094187713033844, + 10140064581418506488, + 1235093741254112241, + 16755357411831959519, + ], + [ + 18007834547781860956, + 5262789089508245576, + 4752286606024269423, + 15626544383301396533, + ], + [ + 5419895278045886802, + 10747737918518643252, + 14861255521757514163, + 3291029997369465426, + ], + [ + 16916426112258580265, + 8714377345140065340, + 14207246102129706649, + 6226142825442954311, + ], + [ + 7320977330193495928, + 15630435616748408136, + 10194509925259146809, + 15938750299626487367, + ], + [ + 9872217233988117092, + 5336302253150565952, + 9650742686075483437, + 8725445618118634861, + ], + [ + 12539853708112793207, + 10831674032088582545, + 11090804155187202889, + 105068293543772992, + ], + [ + 7287113073032114129, + 6373434548664566745, + 8097061424355177769, + 14780666619112596652, + ], + [ + 17147873541222871127, + 17350918081193545524, + 5785390176806607444, + 12480094913955467088, + ], + [ + 17273934282489765074, + 8007352780590012415, + 16690624932024962846, + 8137543572359747206, + ], + ]; + + /// miden's `hash_elements`, in this module's lane convention: capacity lane + /// 8 takes `len % 8`, the rate is OVERWRITTEN (spec §2.6 — absorption costs + /// no field operations), the tail is zero-padded, the digest is lanes 0–3. + /// + /// Test-only: the machine's sponge lives in the eDSL, not here. This exists + /// to drive the external vectors through [`Rpo256::permute`]. + fn hash_elements(elements: &[u64]) -> [FE; 4] { + let mut state = [FE::zero(); HASH_STATE_FELTS]; + state[8] = FE::from((elements.len() % 8) as u64); + let mut i = 0; + for e in elements { + state[i] = FE::from(*e); + i += 1; + if i == 8 { + state = Rpo256.permute(state); + i = 0; + } + } + if i > 0 { + while i < 8 { + state[i] = FE::zero(); + i += 1; + } + state = Rpo256.permute(state); + } + [state[0], state[1], state[2], state[3]] + } + + /// ★ The differential this whole module rests on. + #[test] + fn the_sponge_matches_the_miden_known_answer_vectors() { + for (n, want) in MIDEN_HASH_ELEMENTS.iter().enumerate() { + let input: Vec = (0..=n as u64).collect(); + let got = hash_elements(&input); + let want: [FE; 4] = core::array::from_fn(|i| FE::from(want[i])); + assert_eq!(got, want, "hash_elements of 0..={n} must match miden"); + } + } + + /// A `Compress` row is a standard `Rpo256::merge`, pinned to the external + /// table rather than to our own permutation: merging two digest cells is + /// the same thing as hashing the eight felts they hold, and the vector for + /// eight elements is `MIDEN_HASH_ELEMENTS[7]`. + /// + /// This is what makes the zero compress IV a checkable claim instead of a + /// convention: any implementation of RPO256 anywhere computes this digest + /// for this Merkle parent. + #[test] + fn a_compress_row_is_a_standard_rpo256_merge() { + let a: LfmWord = core::array::from_fn(|i| FE::from(i as u64)); + let b: LfmWord = core::array::from_fn(|i| FE::from(i as u64 + 4)); + let got = Rpo256.compress(&a, &b); + let want: [FE; 4] = core::array::from_fn(|i| FE::from(MIDEN_HASH_ELEMENTS[7][i])); + assert_eq!(got, want, "compress(a, b) must be Rpo256::merge([a, b])"); + } + + /// `ALPHA · INV_ALPHA ≡ 1 (mod p − 1)`, re-derived rather than trusted. + #[test] + fn the_inverse_exponent_inverts_alpha() { + const P_MINUS_ONE: u128 = (1u128 << 64) - (1u128 << 32); + assert_eq!( + (ALPHA as u128 * INV_ALPHA as u128) % P_MINUS_ONE, + 1, + "INV_ALPHA must be ALPHA's inverse in the exponent group" + ); + } + + /// The S-box exponent must be coprime to `p − 1`, or it is not a + /// permutation — the same trap `poseidon` asserts against, checked here so + /// this module stands on its own. + #[test] + fn the_sbox_exponent_is_coprime_to_the_group_order() { + const P_MINUS_ONE: u128 = (1u128 << 64) - (1u128 << 32); + fn gcd(a: u128, b: u128) -> u128 { + if b == 0 { a } else { gcd(b, a % b) } + } + assert_eq!(gcd(ALPHA as u128, P_MINUS_ONE), 1); + } + + /// The addition chain must compute the exponent it claims. Checked against + /// `pow`, which is a different algorithm for the same number. + #[test] + fn the_inverse_sbox_chain_agrees_with_the_exponent() { + for seed in 0..16u64 { + let x = FE::from(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(7)); + assert_eq!( + Rpo256::inv_sbox(&x), + x.pow(INV_ALPHA), + "the chain must equal x^INV_ALPHA at seed {seed}" + ); + } + } + + /// The property that actually matters, and the one the AIR verifies in the + /// forward direction: `(x^{1/7})^7 = x`. + #[test] + fn the_inverse_sbox_inverts_the_forward_sbox() { + for seed in 0..16u64 { + let x = FE::from(seed.wrapping_mul(0xD1B5_4A32_D192_ED03).wrapping_add(3)); + let y = Rpo256::inv_sbox(&x); + assert_eq!(Rpo256::sbox(&y), x, "sbox(inv_sbox(x)) must be x at {seed}"); + assert_eq!(Rpo256::inv_sbox(&Rpo256::sbox(&x)), x); + } + // Zero is the fixed point the all-zero padding row rides on. + assert_eq!(Rpo256::inv_sbox(&FE::zero()), FE::zero()); + } + + /// The `u128` accumulation in [`Rpo256::mds`] must not be able to overflow, + /// and the margin must be large rather than lucky. + #[test] + fn the_mds_row_sum_cannot_overflow_a_u128() { + let max_c = MDS_CIRC_ROW.iter().copied().max().expect("twelve entries"); + // Every stored value is < 2^64 (the field allows non-canonical storage). + let max_row_sum = (HASH_STATE_FELTS as u128) * (max_c as u128) * ((1u128 << 64) - 1); + assert!( + max_row_sum.checked_add(1).is_some(), + "the twelve-term row sum must fit a u128" + ); + // The carry the reduction multiplies must stay small enough that + // `hi * EPSILON` cannot itself overflow a u64. + let max_hi = max_row_sum >> 64; + assert!( + max_hi * 0xFFFF_FFFF < (1u128 << 64), + "hi·EPSILON must fit a u64 without its own reduction" + ); + } + + /// [`Rpo256::mds`] reads a precomputed matrix; this pins it to the + /// definition it replaces, `out_i = Σ_j MDS_CIRC_ROW[(j − i) mod 12]·s_j` in + /// field arithmetic, on states that reach the top of the field. + #[test] + fn the_mds_is_the_circulant_of_its_first_row() { + for seed in 0..64u64 { + let state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|j| { + let x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) + ^ (j as u64).wrapping_mul(0xD1B5_4A32_D192_ED03); + if seed % 4 == 0 { + FE::from(u64::MAX - j as u64) + } else { + FE::from(x) + } + }); + let want: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| { + (0..HASH_STATE_FELTS).fold(FE::zero(), |acc, j| { + acc + FE::from(MDS_CIRC_ROW[(j + HASH_STATE_FELTS - i) % HASH_STATE_FELTS]) + * state[j] + }) + }); + assert_eq!(Rpo256::mds(&state), want, "seed {seed}"); + } + } + + /// `2^64 ≡ EPSILON (mod p)` — the identity [`Rpo256::mds`]'s single + /// reduction rests on, re-derived rather than trusted to a comment. + #[test] + fn the_epsilon_identity_holds() { + const P: u128 = (1u128 << 64) - (1u128 << 32) + 1; + assert_eq!((1u128 << 64) % P, 0xFFFF_FFFF); + } + + #[test] + fn the_round_constant_tables_have_one_row_per_round() { + assert_eq!(ARK1.len(), NUM_ROUNDS); + assert_eq!(ARK2.len(), NUM_ROUNDS); + assert_eq!(NUM_ROUNDS, 7); + assert_eq!(HALVES_PER_ROUND, 2); + } + + /// The three socket domains must be three different capacities, or the + /// separation the module header claims does not exist. + #[test] + fn the_three_socket_domains_are_distinct() { + let c = Rpo256.compress_iv(); + let t = Rpo256.transcript_iv(); + let l = Rpo256.leaf_iv(); + assert_ne!(c, t); + assert_ne!(c, l); + assert_ne!(t, l); + // The padding lane stays zero in every domain — the tag goes in lane 9. + for iv in [&c, &t, &l] { + assert_eq!(iv[CAPACITY_PAD_LANE], FE::zero()); + assert_eq!(iv[2], FE::zero()); + assert_eq!(iv[3], FE::zero()); + } + assert_eq!(c[CAPACITY_DOMAIN_LANE], FE::zero()); + } + + /// A transcript step and a Merkle parent over the SAME two cells must be + /// different digests. This is the weakening `LfmHasher::transcript_out` + /// records for the single-domain hashers, asserted as retired here. + #[test] + fn a_transcript_step_is_not_a_merkle_parent() { + let a: LfmWord = core::array::from_fn(|i| FE::from(11 * i as u64 + 1)); + let b: LfmWord = core::array::from_fn(|i| FE::from(7 * i as u64 + 2)); + let parent = Rpo256.compress(&a, &b); + let step = Rpo256.transcript(&a, &b); + let leaf = Rpo256.leaf(&a, &b); + assert_ne!(parent, step); + assert_ne!(parent, leaf); + assert_ne!(step, leaf); + } + + /// ★ The padding row. An all-zero row must satisfy every constraint the + /// chip emits, which is what lets the AIR carry no `IS_REAL` gate — and + /// the property rests on `0^7 = 0` and `0^{1/7} = 0` at every lane of every + /// round. + #[test] + fn an_all_zero_state_permutes_to_zero_when_the_constants_are_gated_off() { + // With the mode sum `m = 0` the chip scales every round constant to + // zero, so the host analogue is the constant-free permutation. + let mut s = [FE::zero(); HASH_STATE_FELTS]; + for _ in 0..NUM_ROUNDS { + s = Rpo256::mds(&s); + for v in s.iter_mut() { + *v = Rpo256::sbox(v); + } + s = Rpo256::mds(&s); + for v in s.iter_mut() { + *v = Rpo256::inv_sbox(v); + } + } + assert_eq!(s, [FE::zero(); HASH_STATE_FELTS]); + } +} diff --git a/prover/src/lfm/rpx.rs b/prover/src/lfm/rpx.rs new file mode 100644 index 000000000..3dfd93de1 --- /dev/null +++ b/prover/src/lfm/rpx.rs @@ -0,0 +1,596 @@ +//! Rescue-Prime eXtended (RPX256 / XHash12) over Goldilocks at width 12 — the +//! `LFM_HASH` socket's second production-candidate tenant, and the fast one. +//! +//! # What it is, and what it shares with RPO +//! +//! RPX is a **round-function swap on RPO's geometry**, not a redesign +//! ([eprint 2023/1045](https://eprint.iacr.org/2023/1045); family paper in +//! Designs, Codes and Cryptography, 2026). ✓ VERIFIED against miden-crypto's +//! `Rpx256`: it takes **the same state width 12, the same rate 8 / capacity 4, +//! the same 4-felt digest, the same MDS and — literally — the same `ARK1`/`ARK2` +//! constant tables** as [`super::rpo`]. This module imports them rather than +//! re-deriving them, so the two tenants cannot drift apart. +//! +//! What changes is the seven-round schedule: +//! +//! | round | kind | content | +//! |---|---|---| +//! | 0, 2, 4 | **FB** | MDS → +ARK1 → `x^7` → MDS → +ARK2 → `x^{1/7}` — RPO's round exactly | +//! | 1, 3, 5 | **E** | +ARK1 → `x^7` in the **degree-3 EXTENSION** field, on four lane-triples. **No MDS.** | +//! | 6 | **M** | MDS → +ARK1. A linear finish; no S-box at all. | +//! +//! # Why this lane cares +//! +//! The inverse S-box is ~2^63-dense and this codebase measured it at **60% of +//! an RPO permutation on the box** (`rpo::throughput`). RPX runs **three** +//! inverse layers where RPO runs seven, and replaces the rest with an extension +//! `x^7` that is four multiplications on a triple. That is a direct attack on +//! the one term that dominates, which is why the design is reported ~2× faster. +//! +//! It is also narrower in the AIR: **316 value columns against RPO's 436**, +//! because an E round commits two extension intermediates per triple where an FB +//! round commits two ladders per lane, and the M round commits nothing at all. +//! +//! # ⚠ PROVENANCE — WEAKER THAN RPO'S, AND THAT MUST BE SAID +//! +//! [`super::rpo`] rests on nineteen EXTERNAL known-answer vectors published by +//! miden-crypto. **miden publishes no RPX known-answer table** — ✓ VERIFIED, its +//! `rpx/tests.rs` carries only structural tests (consistency, determinism, +//! padding, no-panic), no oracle. So RPX cannot be anchored the way RPO is, and +//! this module does not pretend otherwise. What it anchors instead: +//! +//! 1. **The shared half is externally anchored through RPO.** The constants, the +//! MDS and the whole FB round are RPO's, pinned by RPO's nineteen vectors. +//! 2. **The new half is pinned to an INDEPENDENT algorithm.** The cubic +//! extension's product is checked against naive polynomial multiplication +//! reduced mod `x³ − x − 1`, and `power7` against generic square-and-multiply +//! exponentiation in that extension — different algorithms for the same +//! functions, not a second transcription of the same one. +//! 3. **The schedule** is the one miden's `Rpx256::apply_permutation` runs. +//! +//! ⚖ Net: strong on arithmetic, weaker on end-to-end identity than RPO. A +//! deployment decision should treat "no published KAT" as a real cost. +//! +//! # ⚠ NOT XHash8 +//! +//! XHash8 is the faster sibling and it is **deliberately not built here**. Its +//! extra speed comes from a PARTIAL S-box layer (8 lanes of 12), and a partial +//! layer is one of the three structural footholds this project's own break +//! analysis identified in the 2026 Poseidon collapse — eprint 2026/1692's +//! S-box-skipping gadget restricts into the affine complement of the unS-boxed +//! lanes and works independent of round constants and MDS choice. XHash8's +//! S-boxes are not Poseidon's and eprint 2024/605 analyses XHASH8/12 directly, +//! so this is a flag rather than a verdict — but it is not a thing to adopt +//! quietly for the speed. + +use crate::tables::types::FE; + +use super::hash::{HASH_STATE_FELTS, LfmHasher}; +use super::rpo::{ARK1, ARK2, DOMAIN_COMPRESS, DOMAIN_LEAF, DOMAIN_TRANSCRIPT, Rpo256, domain_iv}; +use super::word::LfmWord; + +/// Rounds — the same seven RPO has, differently shaped. +pub const NUM_ROUNDS: usize = 7; + +/// Lanes per extension element: the extension is degree 3, so an E round reads +/// the twelve-lane state as FOUR triples. +pub const EXT_DEGREE: usize = 3; + +/// Extension elements per E round. +pub const EXT_ELEMENTS: usize = HASH_STATE_FELTS / EXT_DEGREE; + +/// Is round `r` an **FB** round — MDS, forward S-box, MDS, inverse S-box? +pub const fn is_fb_round(r: usize) -> bool { + r.is_multiple_of(2) && r + 1 < NUM_ROUNDS +} + +/// Is round `r` an **E** round — constants then `x^7` in the cubic extension, +/// with NO linear layer? +pub const fn is_ext_round(r: usize) -> bool { + !r.is_multiple_of(2) +} + +/// Is round `r` the **M** round — MDS then constants, and nothing else? +pub const fn is_final_round(r: usize) -> bool { + r + 1 == NUM_ROUNDS +} + +/// Arithmetic in `GF(p³) = GF(p)[φ] / (φ³ − φ − 1)`. +/// +/// ⚠ **This is NOT the VM's own extension.** `crate::tables::types::FEE` is +/// built on `w³ = 2` (see `layout::xalu`); RPX's is `φ³ = φ + 1`. Mixing them +/// would be a wrong hash that still type-checks, so this module carries its own +/// arithmetic explicitly and never reaches for `FEE`. +pub mod cubic_ext { + use super::FE; + + /// An extension element `a0 + a1·φ + a2·φ²`. + pub type Ext = [FE; super::EXT_DEGREE]; + + /// The product, reduced by `φ³ = φ + 1` and `φ⁴ = φ² + φ`. + /// + /// Written as the closed form rather than miden's Karatsuba arrangement: + /// the AIR has to state these three coefficients as constraints, so the + /// host computing them the same way is what makes the chip a transcription + /// of this function instead of a second derivation. + /// [`super::tests::the_extension_product_matches_naive_polynomial_arithmetic`] + /// pins it against an independent algorithm. + pub fn mul(a: &Ext, b: &Ext) -> Ext { + [ + &(&a[0] * &b[0]) + &(&(&a[1] * &b[2]) + &(&a[2] * &b[1])), + &(&(&a[0] * &b[1]) + &(&a[1] * &b[0])) + + &(&(&(&a[1] * &b[2]) + &(&a[2] * &b[1])) + &(&a[2] * &b[2])), + &(&(&a[0] * &b[2]) + &(&a[1] * &b[1])) + &(&(&a[2] * &b[0]) + &(&a[2] * &b[2])), + ] + } + + /// The square. One function, so a squaring and a product can never disagree. + pub fn square(a: &Ext) -> Ext { + mul(a, a) + } + + /// `a^7` by the chain `a² → a³ → a⁶ → a⁷` — two squarings and two products, + /// in exactly the association the degree-3 AIR lowering uses + /// (`a⁷ = (a³)²·a` over the witnessed `a²`/`a³`). + pub fn power7(a: &Ext) -> Ext { + let a2 = square(a); + let a3 = mul(&a2, a); + let a6 = square(&a3); + mul(&a6, a) + } +} + +/// One **FB** round's recorded intermediates — RPO's round, in the same +/// shape. +#[derive(Clone, Copy, Debug)] +pub struct FbRound { + /// `u = MDS(state) + ARK1[r]` — the forward S-box input. + pub u: [FE; HASH_STATE_FELTS], + /// `u²`, committed. + pub u2: [FE; HASH_STATE_FELTS], + /// `u³`, committed. `u^7 = (u³)²·u` is then degree 3. + pub u3: [FE; HASH_STATE_FELTS], + /// `v = MDS(u^7) + ARK2[r]` — the inverse S-box input. + pub v: [FE; HASH_STATE_FELTS], + /// `y = v^{1/7}` — the round output, verified as the FORWARD power. + pub y: [FE; HASH_STATE_FELTS], + /// `y²`, committed. + pub y2: [FE; HASH_STATE_FELTS], + /// `y³`, committed. + pub y3: [FE; HASH_STATE_FELTS], +} + +/// One **E** round's recorded intermediates, per lane-triple. +#[derive(Clone, Copy, Debug)] +pub struct ExtRound { + /// `x = state + ARK1[r]` — the extension input. Recorded for cross-checking; + /// the AIR recomputes it as a degree-1 expression. + pub x: [FE; HASH_STATE_FELTS], + /// `x²` in the extension, committed — four triples laid out flat. + pub t2: [FE; HASH_STATE_FELTS], + /// `x³` in the extension, committed. + pub t3: [FE; HASH_STATE_FELTS], + /// `x⁷ = (x³)²·x` — the round output. + pub out: [FE; HASH_STATE_FELTS], +} + +/// Every intermediate the AIR witnesses. Indexed by round; the entry a round +/// does not use stays zero, the same convention `poseidon::PoseidonWitness` +/// follows for its partial rounds. +#[derive(Clone, Copy, Debug)] +pub struct RpxWitness { + /// Rounds 0, 2, 4. + pub fb: [FbRound; 3], + /// Rounds 1, 3, 5. + pub ext: [ExtRound; 3], + /// The M round's output — `MDS(state) + ARK1[6]`, which IS `OUT`. + pub final_out: [FE; HASH_STATE_FELTS], +} + +/// The index of round `r` within its own kind's array. +pub const fn kind_index(r: usize) -> usize { + r / 2 +} + +/// Records the permutation's intermediates for the trace generator. +/// +/// ⚠ Written independently of [`Rpx256::permute`] rather than factored out of +/// it, the same discipline `rpo::permutation_witness` follows (standing-decisions +/// rule 7): a recording wrapper would make +/// [`tests::the_witness_agrees_with_the_permutation`] a tautology. +pub fn permutation_witness(state: [FE; HASH_STATE_FELTS]) -> RpxWitness { + let zero = [FE::zero(); HASH_STATE_FELTS]; + let mut w = RpxWitness { + fb: [FbRound { + u: zero, + u2: zero, + u3: zero, + v: zero, + y: zero, + y2: zero, + y3: zero, + }; 3], + ext: [ExtRound { + x: zero, + t2: zero, + t3: zero, + out: zero, + }; 3], + final_out: zero, + }; + let mut s = state; + for r in 0..NUM_ROUNDS { + if is_fb_round(r) { + let round = &mut w.fb[kind_index(r)]; + let mixed = Rpo256::mds(&s); + round.u = core::array::from_fn(|i| &mixed[i] + FE::from(ARK1[r][i])); + let mut x = [FE::zero(); HASH_STATE_FELTS]; + for (lane, x_lane) in x.iter_mut().enumerate() { + let u = &round.u[lane]; + round.u2[lane] = u * u; + round.u3[lane] = &round.u2[lane] * u; + *x_lane = &(&round.u3[lane] * &round.u3[lane]) * u; + } + let mixed = Rpo256::mds(&x); + round.v = core::array::from_fn(|i| &mixed[i] + FE::from(ARK2[r][i])); + round.y = round.v; + Rpo256::inv_sbox_layer(&mut round.y); + for lane in 0..HASH_STATE_FELTS { + let y = &round.y[lane]; + round.y2[lane] = y * y; + round.y3[lane] = &round.y2[lane] * y; + } + s = round.y; + } else if is_ext_round(r) { + let round = &mut w.ext[kind_index(r)]; + round.x = core::array::from_fn(|i| &s[i] + FE::from(ARK1[r][i])); + for e in 0..EXT_ELEMENTS { + let base = e * EXT_DEGREE; + let x: cubic_ext::Ext = core::array::from_fn(|k| round.x[base + k]); + let t2 = cubic_ext::square(&x); + let t3 = cubic_ext::mul(&t2, &x); + let t6 = cubic_ext::square(&t3); + let out = cubic_ext::mul(&t6, &x); + round.t2[base..base + EXT_DEGREE].copy_from_slice(&t2); + round.t3[base..base + EXT_DEGREE].copy_from_slice(&t3); + round.out[base..base + EXT_DEGREE].copy_from_slice(&out); + } + s = round.out; + } else { + debug_assert!(is_final_round(r)); + let mixed = Rpo256::mds(&s); + w.final_out = core::array::from_fn(|i| &mixed[i] + FE::from(ARK1[r][i])); + s = w.final_out; + } + } + w +} + +/// Rescue-Prime eXtended at width 12, rate 8, capacity 4, 7 rounds — RPX256. +pub struct Rpx256; + +impl LfmHasher for Rpx256 { + /// The schedule: `FB E FB E FB E M`. + /// + /// Note the E round has **no linear layer** — its only mixing is the + /// extension multiplication inside each triple, and diffusion across triples + /// is the FB rounds' job. That is the design, not an omission + /// (✓ miden `Rpx256::apply_ext_round_ref`), and it is why an E round costs + /// the AIR 36 columns where an FB round costs 60. + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + let mut s = state; + for r in 0..NUM_ROUNDS { + if is_fb_round(r) { + s = Rpo256::mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += FE::from(ARK1[r][lane]); + } + for v in s.iter_mut() { + *v = Rpo256::sbox(v); + } + s = Rpo256::mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += FE::from(ARK2[r][lane]); + } + Rpo256::inv_sbox_layer(&mut s); + } else if is_ext_round(r) { + for (lane, v) in s.iter_mut().enumerate() { + *v += FE::from(ARK1[r][lane]); + } + let mut next = [FE::zero(); HASH_STATE_FELTS]; + for e in 0..EXT_ELEMENTS { + let base = e * EXT_DEGREE; + let x: cubic_ext::Ext = core::array::from_fn(|k| s[base + k]); + let p = cubic_ext::power7(&x); + next[base..base + EXT_DEGREE].copy_from_slice(&p); + } + s = next; + } else { + s = Rpo256::mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += FE::from(ARK1[r][lane]); + } + } + } + s + } + + /// The same capacity-domain separation RPO uses, over the same lanes — the + /// geometry is identical, so the convention carries across unchanged. + fn compress_iv(&self) -> LfmWord { + domain_iv(DOMAIN_COMPRESS).map(FE::from) + } + + fn transcript_iv(&self) -> LfmWord { + domain_iv(DOMAIN_TRANSCRIPT).map(FE::from) + } + + fn leaf_iv(&self) -> LfmWord { + domain_iv(DOMAIN_LEAF).map(FE::from) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Naive polynomial multiplication reduced mod `φ³ − φ − 1` — an + /// INDEPENDENT algorithm for [`cubic_ext::mul`]'s function, written the + /// obvious slow way precisely so it shares no structure with the closed + /// form under test. + fn naive_ext_mul(a: &cubic_ext::Ext, b: &cubic_ext::Ext) -> cubic_ext::Ext { + let mut c = [FE::zero(); 5]; + for (i, ai) in a.iter().enumerate() { + for (j, bj) in b.iter().enumerate() { + c[i + j] = &c[i + j] + &(ai * bj); + } + } + // φ³ = φ + 1, φ⁴ = φ² + φ. + let c3 = c[3]; + let c4 = c[4]; + [&c[0] + &c3, &(&c[1] + &c3) + &c4, &c[2] + &c4] + } + + /// Generic square-and-multiply in the extension — an INDEPENDENT algorithm + /// for [`cubic_ext::power7`]'s function. + fn naive_ext_pow(a: &cubic_ext::Ext, mut e: u32) -> cubic_ext::Ext { + let mut result: cubic_ext::Ext = [FE::one(), FE::zero(), FE::zero()]; + let mut base = *a; + while e > 0 { + if e & 1 == 1 { + result = naive_ext_mul(&result, &base); + } + base = naive_ext_mul(&base, &base); + e >>= 1; + } + result + } + + fn sample_ext(seed: u64) -> cubic_ext::Ext { + core::array::from_fn(|k| { + FE::from( + seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(k as u64 + 1), + ) + }) + } + + /// ★ The pin that carries RPX's NEW arithmetic, since miden publishes no + /// known-answer table for it. Two different algorithms, same answer. + #[test] + fn the_extension_product_matches_naive_polynomial_arithmetic() { + for seed in 0..64u64 { + let a = sample_ext(seed); + let b = sample_ext(seed.wrapping_add(1_000)); + assert_eq!( + cubic_ext::mul(&a, &b), + naive_ext_mul(&a, &b), + "mul at {seed}" + ); + assert_eq!( + cubic_ext::square(&a), + naive_ext_mul(&a, &a), + "square at {seed}" + ); + } + } + + /// `power7` is the seventh power, checked against generic exponentiation. + #[test] + fn the_extension_power7_matches_generic_exponentiation() { + for seed in 0..32u64 { + let a = sample_ext(seed); + assert_eq!( + cubic_ext::power7(&a), + naive_ext_pow(&a, 7), + "power7 at {seed}" + ); + } + } + + /// ★ The AIR's degree-3 lowering, checked in the extension: committing `a²` + /// and `a³` and writing the output as `(a³)²·a` must be the seventh power. + /// If this ever fails, `eval_rpx`'s E-round arm is proving a different map. + #[test] + fn the_air_lowering_computes_the_seventh_power_in_the_extension() { + for seed in 0..32u64 { + let a = sample_ext(seed); + let a2 = cubic_ext::square(&a); + let a3 = cubic_ext::mul(&a2, &a); + let lowered = cubic_ext::mul(&cubic_ext::square(&a3), &a); + assert_eq!(lowered, naive_ext_pow(&a, 7), "lowering at {seed}"); + } + } + + /// The extension must be a FIELD over the polynomial claimed, or `x ↦ x^7` + /// is not a permutation of it. `φ³ − φ − 1` is irreducible over Goldilocks + /// iff it has no root; checked by exhaustion over the only cheap witness we + /// have — that `power7` is injective on a sample — plus the algebraic + /// identity that makes the S-box invertible: `gcd(7, p³ − 1) = 1`. + #[test] + fn the_extension_sbox_is_a_permutation() { + const P: u128 = (1u128 << 64) - (1u128 << 32) + 1; + // `x ↦ x^7` permutes GF(p³) iff gcd(7, p³ − 1) = 1, i.e. iff 7 ∤ p³ − 1. + // Computed mod 7 rather than over p³, which does not fit. + let p_mod_7 = (P % 7) as u64; + // p³ − 1 ≡ p_mod_7³ − 1 (mod 7) + let cube_minus_one = (p_mod_7 * p_mod_7 % 7 * p_mod_7 % 7 + 7 - 1) % 7; + assert_ne!( + cube_minus_one, 0, + "7 must not divide p³ − 1, or x^7 is not a permutation of GF(p³)" + ); + // And injectivity on a sample, as the concrete counterpart. + let mut seen = std::collections::BTreeSet::new(); + for seed in 0..128u64 { + let a = sample_ext(seed); + assert!( + seen.insert(cubic_ext::power7(&a).map(|f| *f.value())), + "power7 collided at seed {seed}" + ); + } + } + + /// The schedule is the one miden runs: FB E FB E FB E M. + #[test] + fn the_schedule_is_three_fb_three_ext_and_one_final() { + let kinds: Vec<&str> = (0..NUM_ROUNDS) + .map(|r| { + if is_fb_round(r) { + "FB" + } else if is_ext_round(r) { + "E" + } else { + "M" + } + }) + .collect(); + assert_eq!(kinds, vec!["FB", "E", "FB", "E", "FB", "E", "M"]); + // Exactly one kind per round — no round is two things, none is nothing. + for r in 0..NUM_ROUNDS { + let n = usize::from(is_fb_round(r)) + + usize::from(is_ext_round(r)) + + usize::from(is_final_round(r)); + assert_eq!(n, 1, "round {r} must have exactly one kind"); + } + assert_eq!(EXT_ELEMENTS, 4); + assert_eq!(EXT_DEGREE * EXT_ELEMENTS, HASH_STATE_FELTS); + } + + /// ★ RPX shares RPO's constants LITERALLY, and this asserts the import + /// rather than trusting it: a future edit that gave RPX its own tables would + /// silently make it a different hash from the one miden ships. + #[test] + fn rpx_uses_rpos_constant_tables() { + assert_eq!(ARK1.len(), NUM_ROUNDS); + assert_eq!(ARK2.len(), NUM_ROUNDS); + assert_eq!(ARK1[0][0], super::super::rpo::ARK1[0][0]); + assert_eq!( + super::super::rpo::MDS_CIRC_ROW, + [7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8] + ); + // Only the FB rounds consume ARK2; the E and M rounds use ARK1 alone. + // Asserted so the schedule and the constant usage cannot drift apart. + assert_eq!((0..NUM_ROUNDS).filter(|r| is_fb_round(*r)).count(), 3); + } + + /// A genuine differential: two independently written round loops. + #[test] + fn the_witness_agrees_with_the_permutation() { + for seed in 0..8u64 { + let input: [FE; HASH_STATE_FELTS] = + core::array::from_fn(|i| FE::from(seed.wrapping_mul(0x9E37_79B9) + i as u64)); + let w = permutation_witness(input); + assert_eq!( + w.final_out, + Rpx256.permute(input), + "witness and permute must agree at seed {seed}" + ); + } + } + + /// The witness records the association the AIR constrains, in both round + /// kinds. + #[test] + fn the_witness_records_the_degree_three_associations() { + let input: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(5 * i as u64 + 3)); + let w = permutation_witness(input); + for round in w.fb.iter() { + for lane in 0..HASH_STATE_FELTS { + assert_eq!(round.u2[lane], &round.u[lane] * &round.u[lane]); + assert_eq!(round.u3[lane], &round.u2[lane] * &round.u[lane]); + assert_eq!(round.y2[lane], &round.y[lane] * &round.y[lane]); + assert_eq!(round.y3[lane], &round.y2[lane] * &round.y[lane]); + assert_eq!( + &(&round.y3[lane] * &round.y3[lane]) * &round.y[lane], + round.v[lane], + "the FB fold must hold" + ); + } + } + for round in w.ext.iter() { + for e in 0..EXT_ELEMENTS { + let base = e * EXT_DEGREE; + let x: cubic_ext::Ext = core::array::from_fn(|k| round.x[base + k]); + let t2: cubic_ext::Ext = core::array::from_fn(|k| round.t2[base + k]); + let t3: cubic_ext::Ext = core::array::from_fn(|k| round.t3[base + k]); + let out: cubic_ext::Ext = core::array::from_fn(|k| round.out[base + k]); + assert_eq!(t2, cubic_ext::square(&x)); + assert_eq!(t3, cubic_ext::mul(&t2, &x)); + assert_eq!(out, cubic_ext::mul(&cubic_ext::square(&t3), &x)); + } + } + } + + /// ★ The padding row, in all three round kinds. With the mode sum zero the + /// chip scales every round constant away, and the all-zero state must be a + /// fixed point of what remains — MDS, both S-boxes, and the extension power. + #[test] + fn an_all_zero_state_is_a_fixed_point_without_constants() { + let mut s = [FE::zero(); HASH_STATE_FELTS]; + for r in 0..NUM_ROUNDS { + if is_fb_round(r) { + s = Rpo256::mds(&s); + for v in s.iter_mut() { + *v = Rpo256::sbox(v); + } + s = Rpo256::mds(&s); + Rpo256::inv_sbox_layer(&mut s); + } else if is_ext_round(r) { + let mut next = [FE::zero(); HASH_STATE_FELTS]; + for e in 0..EXT_ELEMENTS { + let base = e * EXT_DEGREE; + let x: cubic_ext::Ext = core::array::from_fn(|k| s[base + k]); + next[base..base + EXT_DEGREE].copy_from_slice(&cubic_ext::power7(&x)); + } + s = next; + } else { + s = Rpo256::mds(&s); + } + } + assert_eq!(s, [FE::zero(); HASH_STATE_FELTS]); + } + + /// ⚠ RPX must not accidentally BE RPO. The two share constants, an MDS and + /// three of seven rounds, so a schedule bug could plausibly collapse one + /// into the other; this is the check that says it did not. + #[test] + fn rpx_and_rpo_are_different_functions() { + let input: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(i as u64)); + assert_ne!(Rpx256.permute(input), Rpo256.permute(input)); + } + + /// The three socket domains carry across from RPO unchanged, and remain + /// three different functions. + #[test] + fn the_three_socket_domains_are_distinct() { + let a: LfmWord = core::array::from_fn(|i| FE::from(11 * i as u64 + 1)); + let b: LfmWord = core::array::from_fn(|i| FE::from(7 * i as u64 + 2)); + let parent = Rpx256.compress(&a, &b); + let step = Rpx256.transcript(&a, &b); + let leaf = Rpx256.leaf(&a, &b); + assert_ne!(parent, step); + assert_ne!(parent, leaf); + assert_ne!(step, leaf); + assert_eq!(Rpx256.compress_iv(), Rpo256.compress_iv()); + } +} diff --git a/prover/src/lfm/rpx_chip_tests.rs b/prover/src/lfm/rpx_chip_tests.rs new file mode 100644 index 000000000..1b5531aaf --- /dev/null +++ b/prover/src/lfm/rpx_chip_tests.rs @@ -0,0 +1,458 @@ +//! The RPX256 `LFM_HASH` chip: its mixed-schedule layout, its degree bound, and +//! the prove+verify that turns a predicted cell count into a measured one. +//! +//! ## What pins what +//! +//! The permutation is pinned in [`super::rpx`], and ⚠ **more weakly than RPO's** +//! — miden publishes no RPX known-answer table, so the anchors are (a) the +//! shared constants, MDS and FB round, externally anchored through RPO's +//! nineteen vectors, and (b) the new extension arithmetic, pinned against naive +//! polynomial multiplication and generic exponentiation. Nothing here re-checks +//! the algebra; this module checks the *chip*. +//! +//! ## What is different from the RPO arm +//! +//! The schedule. RPX's seven rounds are three kinds — FB, E, M — so the layout's +//! block width depends on the round, the constraint count is not a multiple of +//! anything, and the padding-by-zero argument has to hold in all three. Those +//! are the properties worth testing, and they are what this file tests. + +use math::field::element::FieldElement; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, ProverEvalFolder, RootKind, num_base_from_meta, +}; +use stark::frame::Frame; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; + +use super::airs::lfm_chip_census_with_hasher; +use super::chips::hash::{self, HashConstraints, rpx_cols as rc}; +use super::fixture::fixture_prove_with_hasher; +use super::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use super::instr::HashMode; +use super::programs::{fri_toy_program, trivial_program}; +use super::proof::{lfm_prove_with_hasher, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::rpx::{EXT_DEGREE, EXT_ELEMENTS, NUM_ROUNDS, Rpx256, is_fb_round, is_final_round}; +use super::trace::fill_rpx_witness; +use super::word::LfmWord; + +type Gl = GoldilocksField; +type Gl3 = GoldilocksExtension; + +/// The predicted layout width: `28 + 3·60 + 3·36`. +const PINNED_VALUE_COLUMNS: usize = 316; +/// The predicted constraint count: `4 + 1 + 3·60 + 3·36 + 12 + 4`. +const PINNED_CONSTRAINTS: usize = 309; +/// Base-equivalent cells per permutation: `316 + 3·3`. +const PINNED_CELLS_PER_PERMUTATION: u64 = 325; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +fn mode_selector(mode: HashMode) -> usize { + match mode { + HashMode::Compress => rc::MODE_C, + HashMode::Transcript => rc::MODE_T, + HashMode::Leaf => rc::MODE_L, + HashMode::Permute => rc::MODE_P, + } +} + +fn hash_row(state: [FE; HASH_STATE_FELTS], mode: HashMode) -> Vec { + let mut row = vec![FE::zero(); rc::NUM_COLUMNS]; + row[mode_selector(mode)] = FE::one(); + let mut permuted_input = state; + if mode == HashMode::Permute { + row[hash::cols::IN0..hash::cols::IN0 + HASH_STATE_FELTS].copy_from_slice(&state); + row[hash::cols::S8..hash::cols::S8 + 4].copy_from_slice(&state[8..12]); + } else { + row[hash::cols::IN0..hash::cols::IN0 + 8].copy_from_slice(&state[0..8]); + let iv = Rpx256.mode_iv(mode); + row[hash::cols::S8..hash::cols::S8 + 4].copy_from_slice(&iv); + permuted_input[8..12].copy_from_slice(&iv); + } + let permuted = Rpx256.permute(permuted_input); + row[hash::cols::OUT0..hash::cols::OUT0 + HASH_STATE_FELTS].copy_from_slice(&permuted); + fill_rpx_witness(&mut row); + row +} + +fn sample_row() -> Vec { + hash_row( + core::array::from_fn(|i| FE::from(0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1))), + HashMode::Permute, + ) +} + +fn evaluate(row: &[FE]) -> Vec { + let set = HashConstraints::RPX; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out +} + +fn violations(row: &[FE]) -> Vec { + evaluate(row) + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +// ========================================================================= +// The layout +// ========================================================================= + +/// ★ The width, and the headline against RPO: **316 against 436, 27% narrower.** +/// +/// Both sides are stated independently — the left is the AIR's own width, the +/// right is the closed form the round schedule implies. +#[test] +fn the_rpx_layout_is_316_value_columns() { + assert_eq!(rc::NUM_COLUMNS - rc::PREP_WIDTH, PINNED_VALUE_COLUMNS); + // Three FB rounds at 48 ladder + 12 output, three E rounds at 24 extension + // intermediates + 12 output, and the M round at nothing. + assert_eq!(PINNED_VALUE_COLUMNS, 28 + 3 * (48 + 12) + 3 * (24 + 12)); + assert_eq!(rc::PREP_WIDTH, 13, "the preprocessed prefix does not move"); +} + +/// The layout is injective and gapless across a MIXED schedule — the property +/// most at risk when block width depends on the round kind. +#[test] +fn the_rpx_layout_assigns_every_column_exactly_once() { + assert_eq!( + (0..HASH_STATE_FELTS) + .map(|j| rc::out(NUM_ROUNDS - 1, j)) + .collect::>(), + (0..HASH_STATE_FELTS) + .map(|j| hash::cols::OUT0 + j) + .collect::>(), + "the M round's output must BE the frozen OUT columns" + ); + + let mut seen = vec![0usize; rc::NUM_COLUMNS]; + let mut claim = |c: usize| seen[c] += 1; + for i in 0..HASH_STATE_FELTS { + claim(hash::cols::IN0 + i); + } + for k in 0..4 { + claim(hash::cols::S8 + k); + } + for j in 0..HASH_STATE_FELTS { + claim(hash::cols::OUT0 + j); + } + for r in 0..NUM_ROUNDS { + if is_final_round(r) { + continue; + } + if is_fb_round(r) { + for lane in 0..HASH_STATE_FELTS { + claim(rc::u2(r, lane)); + claim(rc::u3(r, lane)); + claim(rc::y2(r, lane)); + claim(rc::y3(r, lane)); + } + } else { + for e in 0..EXT_ELEMENTS { + for k in 0..EXT_DEGREE { + claim(rc::t2(r, e, k)); + claim(rc::t3(r, e, k)); + } + } + } + for j in 0..HASH_STATE_FELTS { + claim(rc::out(r, j)); + } + } + for (c, &n) in seen.iter().enumerate().skip(rc::PREP_WIDTH) { + assert_eq!( + n, 1, + "value column {c} is claimed {n} times, want exactly 1" + ); + } + for (c, &n) in seen.iter().enumerate().take(rc::PREP_WIDTH) { + assert_eq!(n, 0, "preprocessed column {c} must not be claimed"); + } +} + +#[test] +fn rpx_adds_no_bus_interactions() { + assert_eq!(hash::bus_interactions(HasherKind::Rpx).len(), 6); + assert_eq!(hash::num_columns(HasherKind::Rpx), rc::NUM_COLUMNS); +} + +// ========================================================================= +// The degree bound +// ========================================================================= + +/// ★ Degree 3 across all three round kinds — including the EXTENSION seventh +/// power, whose lowering is the one genuinely new thing in this chip. +#[test] +fn every_rpx_constraint_is_degree_three_or_less() { + let set = HashConstraints::RPX; + let meta = ConstraintSet::::meta(&set); + let n = meta.len(); + assert_eq!(n, PINNED_CONSTRAINTS); + // 4 capacity + 1 mode + FB 5/lane + E 3 per extension op on 4 triples + + // M 1/lane + the shared unread-IN pins. + assert_eq!( + PINNED_CONSTRAINTS, + 4 + 1 + 3 * (5 * 12) + 3 * (3 * 3 * 4) + 12 + super::chips::hash::NUM_UNREAD_INPUT_PINS + ); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "meta must be dense and idx-ordered"); + assert_eq!(m.kind, RootKind::Base); + } + + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert_eq!(degrees.len(), n, "one emit per constraint"); + let mut emitted: Vec = degrees.iter().map(|&(idx, _)| idx).collect(); + emitted.sort_unstable(); + assert!(emitted.iter().enumerate().all(|(i, &idx)| i == idx)); + + let declared = ConstraintSet::::max_degree(&set); + assert_eq!(declared, 3, "the wrap's blowup 2 depends on this staying 3"); + for &(idx, measured) in °rees { + assert!( + measured <= declared, + "constraint {idx}: measured degree {measured} EXCEEDS declared {declared}" + ); + } + assert_eq!(degrees.iter().map(|&(_, d)| d).max(), Some(3)); +} + +// ========================================================================= +// Satisfaction and rejection +// ========================================================================= + +#[test] +fn a_real_rpx_row_satisfies_every_constraint() { + for mode in [ + HashMode::Compress, + HashMode::Transcript, + HashMode::Leaf, + HashMode::Permute, + ] { + let state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(7 * i as u64 + 1)); + let row = hash_row(state, mode); + assert_eq!( + violations(&row), + Vec::::new(), + "an honest row ({mode:?}) must satisfy every constraint" + ); + } +} + +/// Perturbing any single witness column fires — one per structural role, so +/// both round kinds and the chaining are covered. +#[test] +fn perturbing_one_column_is_rejected() { + let base = sample_row(); + assert!( + violations(&base).is_empty(), + "the unperturbed row is honest" + ); + + let cases: [(&str, usize); 6] = [ + ("FB u2 (round 0, lane 5)", rc::u2(0, 5)), + ("FB y3 (round 2, lane 9)", rc::y3(2, 9)), + ("FB output (round 4, lane 3)", rc::out(4, 3)), + ("E t2 (round 1, element 2, coeff 1)", rc::t2(1, 2, 1)), + ("E t3 (round 3, element 0, coeff 2)", rc::t3(3, 0, 2)), + ("E output (round 5, lane 7)", rc::out(5, 7)), + ]; + for (label, col) in cases { + let mut row = base.clone(); + row[col] = &row[col] + FE::one(); + assert!( + !violations(&row).is_empty(), + "perturbing {label} (column {col}) must fire at least one constraint" + ); + } +} + +/// ★ The EXTENSION fold is load-bearing: a consistent `t²`/`t³` ladder for the +/// wrong extension element must still be rejected, because `x ↦ x^7` permutes +/// `GF(p³)` and the output is pinned to the seventh power of the round input. +#[test] +fn a_consistent_extension_ladder_for_the_wrong_value_is_rejected() { + use super::rpx::cubic_ext; + let mut row = sample_row(); + let (r, e) = (1usize, 2usize); + let wrong: cubic_ext::Ext = + core::array::from_fn(|k| &row[rc::t2(r, e, k)] + FE::from(k as u64 + 1)); + let t3 = cubic_ext::mul(&wrong, &wrong); + for k in 0..EXT_DEGREE { + row[rc::t2(r, e, k)] = wrong[k]; + row[rc::t3(r, e, k)] = t3[k]; + } + assert!( + !violations(&row).is_empty(), + "a consistent extension ladder for the wrong value must be rejected" + ); +} + +/// ★ Padding by zero must hold in ALL THREE round kinds — the FB rounds' RPO +/// argument, the E rounds' extension products, and the M round's linear map. +#[test] +fn the_all_zero_padding_row_satisfies_every_constraint() { + let row = vec![FE::zero(); rc::NUM_COLUMNS]; + assert_eq!(violations(&row), Vec::::new()); +} + +#[test] +fn a_padding_row_claiming_to_be_real_is_rejected() { + for mode in [ + HashMode::Compress, + HashMode::Transcript, + HashMode::Leaf, + HashMode::Permute, + ] { + let mut row = vec![FE::zero(); rc::NUM_COLUMNS]; + row[mode_selector(mode)] = FE::one(); + assert!(!violations(&row).is_empty(), "{mode:?}"); + } +} + +/// The three domains separate under RPX exactly as under RPO — same prefix +/// emitter, so this is a check that sharing it did not lose anything. +#[test] +fn a_row_carrying_another_modes_capacity_is_rejected() { + let two_cell = [HashMode::Compress, HashMode::Transcript, HashMode::Leaf]; + let state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| { + if i < 8 { + FE::from(5 * i as u64 + 3) + } else { + FE::zero() + } + }); + for claimed in two_cell { + for carried in two_cell { + if claimed == carried { + continue; + } + let mut row = hash_row(state, carried); + row[mode_selector(carried)] = FE::zero(); + row[mode_selector(claimed)] = FE::one(); + assert!( + !violations(&row).is_empty(), + "a {claimed:?} row carrying the {carried:?} capacity must be rejected" + ); + } + } +} + +// ========================================================================= +// Prove, verify, and end to end +// ========================================================================= + +#[test] +fn the_rpx_chip_proves_and_verifies() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, HasherKind::Rpx); + let proved = lfm_prove_with_hasher(&program, &artifacts, &arenas(), &opts, HasherKind::Rpx) + .expect("proving under RPX must succeed"); + assert!(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + )); +} + +/// ★★★ The same end-to-end that RPO passes, under RPX — and on the SAME +/// unmodified verifier program, which is the point of the exercise. +#[test] +fn the_machine_verifies_a_fixture_fri_proof_end_to_end_under_rpx() { + let opts = options(); + let program = fri_toy_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, HasherKind::Rpx); + let inner = fixture_prove_with_hasher(HasherKind::Rpx); + let proved = lfm_prove_with_hasher( + &program, + &artifacts, + &[inner.commitments.clone(), inner.openings.clone()], + &opts, + HasherKind::Rpx, + ) + .expect("the machine must accept an RPX-committed inner proof"); + assert_eq!(proved.public_words[0].1, inner.commitments[0]); + assert!(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + )); +} + +#[test] +fn the_hasher_tags_are_stable_and_distinct() { + assert_eq!(HasherKind::Test.as_tag(), 0); + assert_eq!(HasherKind::Blake3.as_tag(), 2); + assert_eq!(HasherKind::Rpx.as_tag(), 4); + assert_eq!(HasherKind::Poseidon1W16.as_tag(), 5); +} + +// ========================================================================= +// The measurement +// ========================================================================= + +/// ★★ Base-equivalent cells per permutation, on the census instrument that +/// measured the BLAKE3 column. +#[test] +fn the_measured_cells_per_permutation_match_the_pinned_prediction() { + let program = trivial_program(); + let census = lfm_chip_census_with_hasher(&program, HasherKind::Rpx); + let chip = census + .iter() + .find(|c| c.name == "LFM_HASH") + .expect("LFM_HASH is slot-registered"); + assert_eq!(chip.main_cols, PINNED_VALUE_COLUMNS); + assert_eq!( + chip.aux_cols, 3, + "six LfmMem interactions ⇒ three aux columns" + ); + let per_permutation = chip.main_cols as u64 + 3 * chip.aux_cols as u64; + assert_eq!(per_permutation, PINNED_CELLS_PER_PERMUTATION); + + let rpx = per_permutation; + assert_eq!(rpx, 325); + // An order of magnitude under the slot-11 BLAKE3 incumbent, whose 4,946 is + // quoted from the wrap census rather than recomputed here. + const BLAKE3_CELLS_PER_COMPRESSION: u64 = 4_946; + assert!(BLAKE3_CELLS_PER_COMPRESSION / rpx >= 15); +} diff --git a/prover/src/lfm/statement.rs b/prover/src/lfm/statement.rs new file mode 100644 index 000000000..8dd109b4b --- /dev/null +++ b/prover/src/lfm/statement.rs @@ -0,0 +1,348 @@ +//! LFM program identity and statement binding. +//! +//! `lfm_program_id` binds the instruction column groups (roots + heights), +//! the machine version, the preset and the hash those roots were committed +//! under — it is the digest the registry pins and the consumer's attestation +//! folds. The digest FUNCTION here is keccak regardless of what the roots were +//! committed with: it is a host-side program identity, not a commitment, and +//! `_V2` rides the ecosystem hash migration. +//! +//! The statement absorb seeds the Fiat–Shamir transcript before +//! `multi_prove` / `multi_verify_views`, exactly like the RV64 VM's +//! `statement.rs`: any divergence in the absorbed bytes changes every derived +//! challenge and verification rejects. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::hash::platform_keccak::PlatformKeccak256 as Keccak256; +use digest::Digest; +use math::field::traits::IsPrimeField; +use stark::config::{Commitment, CommitmentHash}; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +use super::airs::{ChipSet, NUM_LFM_CHIPS}; +use super::hash::HasherKind; +use super::word::LfmWord; + +type E = GoldilocksExtension; + +pub const LFM_MACHINE_VERSION: u32 = 1; +/// Single preset in v0; becomes the preset ladder tag later. +pub const LFM_PRESET_TAG: u32 = 0; + +const LFM_PROGRAM_TAG: &[u8] = b"LAMBDAVM_LFM_PROGRAM_V1"; +/// Separates the `LFM_BLAKE3` chunk tail from the fixed part of the preimage. +/// See the suffix note in [`lfm_program_id`]. +const LFM_BLAKE3_CHUNK_TAG: &[u8] = b"LAMBDAVM_LFM_BLAKE3_CHUNKS_V1"; + +/// Domain tag of the `LFM_HASH` chunk tail in [`lfm_program_id_chunked`]. +const LFM_HASH_CHUNK_TAG: &[u8] = b"LAMBDAVM_LFM_HASH_CHUNKS_V1"; +/// Tags the merged-AIR tail of [`lfm_program_id_merged_under`]. +const LFM_MERGE_TAG: &[u8] = b"LAMBDAVM_LFM_MERGED_TABLES_V1"; +/// `pub(crate)`: the aggregation layer's emitted verifier replays +/// [`absorb_lfm_statement`] byte for byte and needs the same tag bytes, and the +/// legacy proof family (`hash_pin::LegacyFamily`) names it as its statement tag. +pub(crate) const LFM_STATEMENT_TAG: &[u8] = b"LAMBDAVM_LFM_STATEMENT_V1"; + +/// The byte that names a commitment hash inside [`lfm_program_id`]. +/// +/// Exhaustive on purpose, and that is the whole of what remains of the tripwire +/// `build_artifacts_with_hasher` used to carry: a third commitment hash cannot +/// be added without choosing a tag for it here, and choosing a tag is the act of +/// deciding what program identity says about it. The old guard made that +/// decision unskippable by refusing to compile; this makes it unskippable by +/// having no default. +/// +/// Tags are frozen. Changing one re-blesses every `LFM_REGISTRY` entry. Tags 2 +/// and 4 belonged to the retired RPO and Poseidon-original configurations and +/// are never reused. +pub(crate) const fn commitment_hash_tag(hash: CommitmentHash) -> u8 { + match hash { + CommitmentHash::Keccak256 => 0, + CommitmentHash::Blake3 => 1, + CommitmentHash::Rpx256 => 3, + CommitmentHash::Poseidon1 => 5, + } +} + +/// The program digest over the frozen chip order. +/// +/// `keccak_rnd_chunks` is bound alongside the roots and heights because it is +/// program shape too: it decides how many `KECCAK_RND` instances the verifier +/// builds. Binding it here is what makes the registry entry — rather than the +/// proof — the authority on that shape. `blake3_chunk_roots` / +/// `blake3_chunk_log_heights` are the same decision for `LFM_BLAKE3`, and they +/// are slices rather than a count because that chip's chunks are not identical: +/// each commits its own slice of the instruction group. They are absorbed as a +/// tail, for the reason spelled out at the absorb itself. +/// +/// `hasher` is bound for the same reason and is the one piece of program shape +/// the roots cannot carry: `LFM_HASH`'s preprocessed group is its INSTRUCTION +/// group — addresses, mode selectors and multiplicities — which no candidate +/// changes, so every hasher commits the same width (13 since `MODE_L`) and the +/// commitments are hasher-independent by construction (`airs.rs`). Without this +/// tag the only thing separating one permutation's machine from another's would +/// be a main-trace width coincidence, which a third candidate could collide +/// with. The tag is what makes two hashers two programs. +/// +/// ★ **The COMMITMENT hash is bound too, and it is a different axis from +/// `hasher`.** `hasher` names the `LFM_HASH` chip the machine RUNS; +/// [`commitment_hash_tag`] names the hash the `roots` above were BUILT with. The +/// two were separate axes with only the first one named, which is what +/// `build_artifacts_with_hasher`'s guard existed to force a decision about. The +/// decision is here: a build committing under a different hash is a different +/// program identity by name, not merely by value. +/// +/// Binding it by value alone would not have been enough. The roots do move when +/// the commitment hash moves, so the digest already changed — but "changed" and +/// "says which" are different properties, and only the second one lets a +/// mismatch be reported as *what it is* rather than as an unrecognised root. +/// +/// It is read from the global rather than taken as a parameter because the three +/// commit helpers in `registry.rs` are hard-wired to `stark`'s default aliases, +/// which is exactly what `stark::config::COMMITMENT_HASH` names. Should those +/// helpers ever become generic over `H`, this read moves with them. +#[allow(clippy::too_many_arguments)] +pub fn lfm_program_id( + roots: &[Commitment; NUM_LFM_CHIPS], + log_heights: &[u8; NUM_LFM_CHIPS], + keccak_rnd_chunks: usize, + hasher: HasherKind, + chip_set: ChipSet, + blake3_chunk_roots: &[Commitment], + blake3_chunk_log_heights: &[u8], +) -> Commitment { + lfm_program_id_chunked( + roots, + log_heights, + keccak_rnd_chunks, + hasher, + chip_set, + blake3_chunk_roots, + blake3_chunk_log_heights, + &[], + &[], + ) +} + +/// [`lfm_program_id`] for a program whose `LFM_HASH` may be split: +/// `hash_chunk_roots` / `hash_chunk_log_heights` list every chunk, chunk 0 being +/// slot 5's own entry. A one-chunk (or empty) list absorbs NOTHING, so an +/// unsplit program keeps the digest it always had; a split one absorbs chunks +/// 1.. as a length-prefixed tail under its own tag, after the `LFM_BLAKE3` tail. +#[allow(clippy::too_many_arguments)] +pub fn lfm_program_id_chunked( + roots: &[Commitment; NUM_LFM_CHIPS], + log_heights: &[u8; NUM_LFM_CHIPS], + keccak_rnd_chunks: usize, + hasher: HasherKind, + chip_set: ChipSet, + blake3_chunk_roots: &[Commitment], + blake3_chunk_log_heights: &[u8], + hash_chunk_roots: &[Commitment], + hash_chunk_log_heights: &[u8], +) -> Commitment { + lfm_program_id_chunked_under( + crate::hash_pin::BLOCK_COMMITMENT_HASH, + roots, + log_heights, + keccak_rnd_chunks, + hasher, + chip_set, + blake3_chunk_roots, + blake3_chunk_log_heights, + hash_chunk_roots, + hash_chunk_log_heights, + ) +} + +/// [`lfm_program_id_chunked`] for roots built under the commitment hash +/// `commitment` (a proof family's, `registry::build_artifacts_under`) rather +/// than the legacy pin's: the hash that BUILT the roots is the one the +/// identity names. +#[allow(clippy::too_many_arguments)] +pub fn lfm_program_id_chunked_under( + commitment: CommitmentHash, + roots: &[Commitment; NUM_LFM_CHIPS], + log_heights: &[u8; NUM_LFM_CHIPS], + keccak_rnd_chunks: usize, + hasher: HasherKind, + chip_set: ChipSet, + blake3_chunk_roots: &[Commitment], + blake3_chunk_log_heights: &[u8], + hash_chunk_roots: &[Commitment], + hash_chunk_log_heights: &[u8], +) -> Commitment { + lfm_program_id_merged_under( + commitment, + roots, + log_heights, + keccak_rnd_chunks, + hasher, + chip_set, + blake3_chunk_roots, + blake3_chunk_log_heights, + hash_chunk_roots, + hash_chunk_log_heights, + &super::merge::LfmMerge::none(), + ) +} + +/// [`lfm_program_id_chunked_under`] for a program whose equal-height chips are +/// proved as merged AIRs (`merge`, [`super::merge`]). An empty `merge` absorbs +/// NOTHING, so every program built without the merge keeps the digest it +/// always had; a merged one absorbs, after the hash tail, a length-prefixed +/// tail under its own tag: each merged AIR's chip slots, its root and its +/// height. The merged chips' own slots above hold the all-zero root (their +/// columns are committed only inside the merged root), so the tail is what +/// binds them, and the layout by name. +#[allow(clippy::too_many_arguments)] +pub fn lfm_program_id_merged_under( + commitment: CommitmentHash, + roots: &[Commitment; NUM_LFM_CHIPS], + log_heights: &[u8; NUM_LFM_CHIPS], + keccak_rnd_chunks: usize, + hasher: HasherKind, + chip_set: ChipSet, + blake3_chunk_roots: &[Commitment], + blake3_chunk_log_heights: &[u8], + hash_chunk_roots: &[Commitment], + hash_chunk_log_heights: &[u8], + merge: &super::merge::LfmMerge, +) -> Commitment { + let mut h = Keccak256::new(); + h.update(LFM_PROGRAM_TAG); + h.update(LFM_MACHINE_VERSION.to_le_bytes()); + h.update(LFM_PRESET_TAG.to_le_bytes()); + h.update([hasher.as_tag()]); + h.update([commitment_hash_tag(commitment)]); + // ★ The chip set is program shape and is bound by NAME, for the reason the + // commitment hash is: the roots of an absent family are still in the array + // (a hole, like KECCAK_RND's), so nothing else in this digest distinguishes + // a program that instantiates a family from one that omits it. Without this + // byte a verifier resolving the wrong mask would build a different AIR set + // and report an unrecognised shape rather than the mismatch it is. + h.update([chip_set.as_tag()]); + for i in 0..NUM_LFM_CHIPS { + h.update([i as u8]); + h.update(roots[i]); + h.update([log_heights[i]]); + } + h.update((keccak_rnd_chunks as u64).to_le_bytes()); + // ★ `LFM_BLAKE3` chunking rides as a SUFFIX, and only when it is on. + // + // The chip's chunks each commit their own instruction group, so a split + // program has roots and heights the fifteen-wide arrays cannot hold. Chunk 0 + // is already bound above — it IS `roots[BLAKE3_SLOT]` — so what is left is + // chunks 1.. , and a program with none of them absorbs NOTHING here. That is + // deliberate: a single-table program is the machine as it stood before + // chunking existed, and every blessed `LFM_REGISTRY` digest must stay the + // byte-identical value it was blessed at. + // + // Ambiguity between "no tail" and "a tail" is what a suffix encoding has to + // rule out, and the length prefix does it: a non-empty tail contributes at + // least the tag and eight length bytes, so no empty-tail preimage can equal + // a non-empty one, and two non-empty tails of different lengths cannot + // collide either. The chunk COUNT is bound by that same length — `n` tail + // entries is `n + 1` chunks — so nothing about the split is left unnamed. + debug_assert_eq!( + blake3_chunk_roots.len(), + blake3_chunk_log_heights.len(), + "an LFM_BLAKE3 chunk has exactly one root and one height" + ); + if blake3_chunk_roots.len() > 1 { + h.update(LFM_BLAKE3_CHUNK_TAG); + h.update(((blake3_chunk_roots.len() - 1) as u64).to_le_bytes()); + for (root, height) in blake3_chunk_roots + .iter() + .zip(blake3_chunk_log_heights) + .skip(1) + { + h.update(root); + h.update([*height]); + } + } + // The hash tail, on the same length-prefixed terms: chunk 0 is bound above + // as slot 5, so only chunks 1.. are absorbed, and a distinct tag keeps it + // apart from the BLAKE3 tail when both are present. + debug_assert_eq!( + hash_chunk_roots.len(), + hash_chunk_log_heights.len(), + "an LFM_HASH chunk has exactly one root and one height" + ); + if hash_chunk_roots.len() > 1 { + h.update(LFM_HASH_CHUNK_TAG); + h.update(((hash_chunk_roots.len() - 1) as u64).to_le_bytes()); + for (root, height) in hash_chunk_roots.iter().zip(hash_chunk_log_heights).skip(1) { + h.update(root); + h.update([*height]); + } + } + // The merged AIRs, on the same length-prefixed terms, after the hash tail: + // nothing at all for a program without them. + if !merge.is_empty() { + h.update(LFM_MERGE_TAG); + h.update((merge.tables.len() as u64).to_le_bytes()); + for table in &merge.tables { + h.update((table.chips.len() as u64).to_le_bytes()); + for chip in &table.chips { + h.update([chip.slot() as u8]); + } + h.update(table.root); + h.update([table.log_height]); + } + } + h.finalize().into() +} + +/// Binds the LFM statement: program identity, machine version, the claimed +/// public words and the FRI terminal degree. Exhaustive by construction — +/// extending the statement means extending this function, in one place. +/// +/// Generic over the transcript because the statement bind is hash-agnostic: it +/// only absorbs, so it is the same sequence of `append_bytes` calls whichever +/// sponge the proof runs on. Pinning it to `DefaultTranscript` would have made +/// the machine's own transcript a fork of this function rather than a caller of +/// it, and two copies of a statement encoding is exactly the drift the +/// "exhaustive by construction" note above exists to prevent. +pub fn absorb_lfm_statement( + transcript: &mut impl IsTranscript, + program_id: &Commitment, + public_words: &[(u32, LfmWord)], + fri_final_poly_log_degree: u8, +) { + absorb_lfm_statement_tagged( + transcript, + LFM_STATEMENT_TAG, + program_id, + public_words, + fri_final_poly_log_degree, + ); +} + +/// [`absorb_lfm_statement`] under a proof family's statement tag +/// (`hash_pin::ProofFamily::STATEMENT_TAG`), absorbed IN PLACE of +/// [`LFM_STATEMENT_TAG`] as the statement's first `append_bytes` call, then the +/// rest of the statement unchanged. The legacy family's tag is +/// [`LFM_STATEMENT_TAG`] itself, so its statements are today's bytes exactly. +/// Domain separation, not the binding: a statement names the family its proof +/// was made under, so no statement of one family can coincide with another's. +pub fn absorb_lfm_statement_tagged( + transcript: &mut impl IsTranscript, + tag: &[u8], + program_id: &Commitment, + public_words: &[(u32, LfmWord)], + fri_final_poly_log_degree: u8, +) { + transcript.append_bytes(tag); + transcript.append_bytes(program_id); + transcript.append_bytes(&LFM_MACHINE_VERSION.to_le_bytes()); + transcript.append_bytes(&(public_words.len() as u64).to_le_bytes()); + for (index, word) in public_words { + transcript.append_bytes(&index.to_le_bytes()); + for lane in word { + transcript.append_bytes(&GoldilocksField::canonical(lane.value()).to_le_bytes()); + } + } + transcript.append_bytes(&[fri_final_poly_log_degree]); +} diff --git a/prover/src/lfm/statement_replay.rs b/prover/src/lfm/statement_replay.rs new file mode 100644 index 000000000..f67688990 --- /dev/null +++ b/prover/src/lfm/statement_replay.rs @@ -0,0 +1,330 @@ +//! The continuation-epoch statement and Phase A, replayed in the machine. +//! +//! This is the first leg of a REAL verifier: everything a `multi_verify` does to +//! its transcript before the per-table forks. Two pieces, in order: +//! +//! 1. `absorb_statement(StatementKind::ContinuationEpoch { .. })` — the +//! canonical, domain-separated statement encoding from `crate::statement`; +//! 2. the Phase-A commitment absorbs from `crate::replay_transcript_phase_a_view` +//! — per air an optional preprocessed root then the main trace root — followed +//! by the two shared LogUp challenges `z` and `α`. +//! +//! The target is a continuation EPOCH, not a monolithic proof (see +//! `others/lfm-target-shape.md`), so the tag is `LAMBDAVM_CONTINUATION_EPOCH_V2` +//! and the encoding carries a trailing `epoch_label` the monolithic variant +//! lacks. +//! +//! ## Why this leg is misaligned end to end +//! +//! The tag is 30 bytes, `≡ 2 (mod 4)`, so the ELF digest immediately after it +//! straddles half boundaries; the one-byte `fri_final_poly_log_degree` later +//! moves the cursor again. The whole statement is +//! `264 + public_output_len + 16·page_ranges` bytes. +//! +//! ★★ THE TRAILING MISALIGNMENT IS GONE, and not because anyone aimed at it. +//! At fifteen table counts the total was `215 + …`, which is `≡ 3 (mod 4)` +//! whenever `public_output_len ≡ 0 (mod 4)` — so every Phase-A root absorb was +//! spliced at shift 3, at about one `BitDec` and 34 `BALU` rows per half, and +//! this note said "a single pad byte at the end of the statement encoding would +//! make all of Phase A free". The main-sync port supplies that byte: the counts +//! grew by six (`+48 ≡ 0 mod 4`, which moves nothing) and the statement gained +//! a trailing `is_final` byte, so `264 ≡ 0 (mod 4)` and the roots that follow +//! start on a half boundary. `Counts::transcript_misaligned_absorbs_after_statement` +//! is what reads it; the internal straddles (the digest after the 30-byte tag, +//! the FRI byte) are untouched, since only the statement's END moved. +//! +//! ## Which fields are program constants +//! +//! Shape-static fields are emitted as constants, not read from an arena, because +//! they DETERMINE the program's shape: the table counts and the page-range list +//! fix how many sub-proofs Phase A absorbs, and `num_private_input_pages` fixes +//! the AIR layout. A program that read them from an arena would be claiming to +//! verify a shape it was not compiled for. Only the genuinely per-proof +//! values — the ELF digest, the public output and the epoch label — come from +//! the arena. + +use crate::statement::CONTINUATION_EPOCH_TAG; + +use crate::tables::types::FE; + +use super::builder::{Ext, Felt, LfmBuilder}; +use super::keccak_host::BYTES_PER_HALF; +use super::transcript_replay::TranscriptReplay; + +/// Counts `TableCounts` absorbs: fourteen split-table families, the six +/// accelerator chips, the 0-or-1 BLAKE3 presence count and the 0-or-1 IS_WH one. The guest must +/// absorb exactly what the host's `statement::absorb_statement_with_digest` +/// does — one count too few and every challenge downstream diverges, so this +/// tracks that encoding, not a structural property of the machine. +/// +/// ⚠ A SECOND COPY OF `statement::NUM_TABLE_KINDS`, and nothing in the type +/// system ties them together. What catches a drift is the host-vs-machine +/// challenge equality in `algebraic_transcript`'s replay test, plus the +/// hand-written `[u64; NUM_TABLE_COUNTS]` literal it builds — which fails to +/// compile when this number moves and the literal does not. +pub const NUM_TABLE_COUNTS: usize = 23; + +/// The shape-static half of the statement — emitted as program constants. +#[derive(Debug, Clone)] +pub struct EpochStatementShape { + /// Length of the public output in bytes. Shape-static: it fixes how many + /// arena halves the program reads. + pub public_output_len: usize, + /// The absorbed counts, in `TableCounts` declaration order: the fourteen + /// split-table chunk counts, then BLAKE3's 0-or-1. + pub table_counts: [u64; NUM_TABLE_COUNTS], + pub num_private_input_pages: u64, + pub fri_final_poly_log_degree: u8, + /// `(base, count)` per runtime page range. + pub page_ranges: Vec<(u64, u64)>, + /// Whether this is the run's final epoch. Shape-static for the same reason + /// the counts are: it decides whether HALT is in the epoch's AIR set, so it + /// fixes how many sub-proofs Phase A absorbs. The host appends it as the + /// statement's LAST byte. + pub is_final: bool, +} + +impl EpochStatementShape { + /// Total bytes the statement absorbs — the emitter's own accounting, so a + /// test can pin the resulting misalignment instead of trusting prose. + pub fn byte_len(&self) -> usize { + CONTINUATION_EPOCH_TAG.len() + + 32 + + 8 + + self.public_output_len + + 8 * NUM_TABLE_COUNTS + + 8 + + 1 + + 8 + + 16 * self.page_ranges.len() + + 8 + + 1 + } +} + +/// The per-proof half of the statement — arena halves, four bytes each, +/// little-endian, in absorb order. +pub struct EpochStatementVars<'a> { + /// The 32-byte ELF digest: 8 halves. + pub elf_digest: &'a [Felt], + /// `public_output_len / 4` halves. + pub public_output: &'a [Felt], + /// The `u64` epoch label, little-endian: `[low32, high32]`. + pub epoch_label: &'a [Felt], +} + +/// Emits `absorb_statement(ContinuationEpoch)` byte for byte — and, since the +/// algebraic arm landed, **call for call**. +/// +/// ⚠ The call sequence below tracks +/// `statement::absorb_statement_with_digest`'s `append_bytes` calls one for one. +/// That is a real obligation rather than tidiness: a byte transcript concatenates +/// and cannot see the boundaries, an algebraic one length-prefixes every call and +/// sees nothing else. +/// +/// Every multi-byte field in this encoding is LITTLE-endian (`to_le_bytes`), +/// unlike `append_field_element`'s big-endian rendering — so a `u64` carried as +/// `[low32, high32]` halves needs no byte manipulation at all, and the only cost +/// here is the misalignment splice. +pub fn absorb_epoch_statement( + t: &mut TranscriptReplay, + shape: &EpochStatementShape, + vars: &EpochStatementVars, +) { + assert_eq!( + vars.public_output.len(), + shape.public_output_len.div_ceil(BYTES_PER_HALF), + "public_output halves must match the declared length" + ); + assert_eq!(vars.elf_digest.len(), 8, "the ELF digest is 32 bytes"); + assert_eq!(vars.epoch_label.len(), 2, "the epoch label is one u64"); + + t.append_const_bytes(CONTINUATION_EPOCH_TAG); + t.append_halves_misaligned(vars.elf_digest); + t.append_const_bytes(&(shape.public_output_len as u64).to_le_bytes()); + // Byte-granular on purpose. `public_output` is collected one byte per COMMIT + // operation (`trace_builder`), so an epoch's length is whatever the workload + // produced — nothing aligns it, and the trailing half must be masked rather + // than absorbed whole. + t.append_bytes_misaligned(vars.public_output, shape.public_output_len); + + // ⚠ ONE APPEND PER HOST CALL, not one run. The counts, the page total, the + // FRI byte and the range list are all shape-static, so a byte transcript + // cannot tell a single concatenated run from this sequence — the packer + // chunks consecutive constants together either way, and the emitted halves + // are identical. An ALGEBRAIC transcript can: it prefixes every + // `append_bytes` call with that call's LENGTH, so coalescing here would + // absorb one long field where the host absorbed twenty short ones, and + // every challenge downstream would diverge. See `transcript_replay::Append` + // for the general statement of this. + for count in shape.table_counts { + t.append_const_bytes(&count.to_le_bytes()); + } + t.append_const_bytes(&shape.num_private_input_pages.to_le_bytes()); + // A single byte, no endianness concern — and its own call. + t.append_const_bytes(&[shape.fri_final_poly_log_degree]); + t.append_const_bytes(&(shape.page_ranges.len() as u64).to_le_bytes()); + for (base, count) in &shape.page_ranges { + t.append_const_bytes(&base.to_le_bytes()); + t.append_const_bytes(&count.to_le_bytes()); + } + + // Continuation epochs bind their position, then their role (replay + // protection). One call each, matching the host's two `append_bytes`. + t.append_halves_misaligned(vars.epoch_label); + t.append_const_bytes(&[u8::from(shape.is_final)]); +} + +/// A preprocessed commitment as Phase A absorbs it — and the distinction is +/// which SOURCE the root has, not how it is encoded. +/// +/// Production reads every one of these from the AIR and never from the proof +/// (`verifier.rs:1187`), so what the machine must reproduce is the root's +/// provenance: a commitment that is a function of the proof options alone is +/// program text and absorbs as literal bytes; one that is a function of per-proof +/// data (an ELF, a register boundary) is cells, and something else in the program +/// owes their binding (assembly ledger entry 7). +pub enum PhaseAPreprocessed<'a> { + /// Program text — 32 literal bytes, absorbed with no arithmetic at all. + Constant(&'a [u8; 32]), + /// Cells: eight `u32` halves, derived in-machine or read from the arena. + Cells(&'a [Felt]), +} + +/// One sub-proof's Phase-A commitments, as arena halves (8 per 32-byte root). +pub struct PhaseATable<'a> { + /// Present exactly when the air is preprocessed — the verifier absorbs the + /// precomputed commitment only then. + pub preprocessed_root: Option>, + pub main_root: &'a [Felt], +} + +/// Absorb one root supplied as the configuration's ROOT FELTS — eight `u32` +/// halves on a byte hash, the digest's four felts on an algebraic one. +/// +/// ⚠ The byte arm's emission is deliberately unchanged: the felts go straight to +/// `append_halves_misaligned` with no packing, because `statement_replay` is a +/// REGISTRY program and one extra instruction would drift every blessed +/// `program_id`. The algebraic arm packs the four felts into the one digest cell +/// they already are, which is what `RootCells::absorb` does for a root the +/// emitter holds as cells rather than as arena felts. +fn absorb_root_felts(b: &mut LfmBuilder, t: &mut TranscriptReplay, felts: &[Felt]) { + // Four felts per digest cell: two `u32` halves each on a byte hash, the four + // Goldilocks felts themselves on an algebraic one. + let words = super::edsl::digest_words(b); + assert_eq!( + felts.len(), + 4 * words as usize, + "a commitment is one root's felts" + ); + if words == 1 { + let cell = b.pack_word([felts[0], felts[1], felts[2], felts[3]]); + t.append_root_cells(b, &[cell]); + } else { + t.append_halves_misaligned(felts); + } +} + +/// Replays Phase A: the commitment absorbs, then the two shared LogUp +/// challenges. +/// +/// Mirrors `crate::replay_transcript_phase_a_view` — for each air, the +/// preprocessed commitment when it has one, then the main trace root, and +/// finally `z` and `α` sampled as cubic-extension elements in that order. +/// +/// The absorbs use the misaligned path because the statement leaves the cursor +/// at `≡ 3 (mod 4)`; nothing about a 32-byte root is itself misaligned. +pub fn replay_phase_a( + t: &mut TranscriptReplay, + b: &mut LfmBuilder, + tables: &[PhaseATable], +) -> (Ext, Ext) { + for table in tables { + match &table.preprocessed_root { + Some(PhaseAPreprocessed::Constant(bytes)) => t.append_const_bytes(&bytes[..]), + Some(PhaseAPreprocessed::Cells(prep)) => absorb_root_felts(b, t, prep), + None => {} + } + absorb_root_felts(b, t, table.main_root); + } + let z = t.sample_ext(b); + let alpha = t.sample_ext(b); + (z, alpha) +} + +/// The domain the commitment-time point `z0` must avoid, as the leg's shape: +/// the largest trace domain (`log2`) and the largest LDE coset over a bound +/// proof's sub-proofs, and their one coset offset +/// (`stark::commit_ood::point_domain`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CommitOodDomain { + pub log2_trace_length: u32, + pub log2_lde_length: u32, + pub coset_offset: FE, +} + +impl CommitOodDomain { + /// From each sub-proof's `(log2 trace length, log2 blowup, coset offset)`. + /// Panics on sub-proofs with different offsets, which the host verifier + /// refuses (emit-time shape, never proof data). + pub fn over(tables: impl IntoIterator) -> Self { + let mut out: Option = None; + for (log2_trace_length, log2_blowup, coset_offset) in tables { + let log2_lde_length = log2_trace_length + log2_blowup; + out = Some(match out { + None => Self { + log2_trace_length, + log2_lde_length, + coset_offset, + }, + Some(d) => { + assert_eq!( + d.coset_offset, coset_offset, + "a bound proof's sub-proofs share one coset offset" + ); + Self { + log2_trace_length: d.log2_trace_length.max(log2_trace_length), + log2_lde_length: d.log2_lde_length.max(log2_lde_length), + coset_offset, + } + } + }); + } + out.expect("a bound proof has a sub-proof") + } +} + +/// [`replay_phase_a`] for a bound proof (`stark::commit_ood`): the roots, then +/// `z0` (asserted outside `domain`), then every answer, then `z` and `α`. +/// Returns `(z, α, z0)`. `answers` are the cells the per-table legs then +/// fold, so the walk cannot read a value Phase A did not absorb. +pub fn replay_phase_a_answered( + t: &mut TranscriptReplay, + b: &mut LfmBuilder, + tables: &[PhaseATable], + answers: &[Ext], + domain: CommitOodDomain, +) -> (Ext, Ext, Ext) { + for table in tables { + match &table.preprocessed_root { + Some(PhaseAPreprocessed::Constant(bytes)) => t.append_const_bytes(&bytes[..]), + Some(PhaseAPreprocessed::Cells(prep)) => absorb_root_felts(b, t, prep), + None => {} + } + absorb_root_felts(b, t, table.main_root); + } + let z0 = t.sample_ext(b); + super::epoch::assert_z_outside_domains_raw( + b, + z0, + domain.log2_trace_length, + domain.log2_lde_length - domain.log2_trace_length, + domain.coset_offset, + ); + for answer in answers { + super::epoch::append_ext_cell(b, t, *answer); + } + let z = t.sample_ext(b); + let alpha = t.sample_ext(b); + (z, alpha, z0) +} diff --git a/prover/src/lfm/step_size_tests.rs b/prover/src/lfm/step_size_tests.rs new file mode 100644 index 000000000..90d262a2c --- /dev/null +++ b/prover/src/lfm/step_size_tests.rs @@ -0,0 +1,773 @@ +//! Assembly ledger entries 8 and 9 — the two OOD-grid blindnesses, witnessed. +//! +//! Both entries are members of the phase's degenerate-parameter family: a defect +//! that no test can see because every proof the phase has shares a parameter +//! value. Their shared cause is `num_eval_points = transition_offsets.len() · +//! step_size` (`verifier.rs:179`, `prover.rs:1452`) being 2 for every production +//! AIR, which collapses the OOD grid to one row per block. +//! +//! - **Entry 8, the ABSORB ORDER.** Production absorbs both pruned OOD blocks +//! COLUMN-major (`verifier.rs:1421-1431`). Row-major is indistinguishable while +//! every block is one row tall. Witnessed here by a THREE-offset AIR: the +//! next-row block's height is `num_eval_points − step_size`, so three offsets at +//! `step_size = 1` make it two rows tall. +//! - **Entry 9, the FRAME-STEP VIEW.** `Op::Var{offset}` indexes the constraint +//! frame's evaluation STEP, and a step is `step_size` grid rows. Witnessed here +//! by comparing [`super::epoch_verify::frame_step_view`] against production's +//! own `StarkTableView::into_frame` at `step_size = 2`. +//! +//! ## Two different witnesses, and why one AIR could not carry both +//! +//! The brief asked for ONE synthetic AIR with three transition offsets AND +//! `step_size > 1`, on the grounds that a witness for one entry closes the other +//! only if it exercises both. That is right about the requirement and wrong about +//! the vehicle, for two reasons found by reading and then measured: +//! +//! 1. **Three offsets and `step_size > 1` cannot coexist in a provable AIR.** +//! `AirWithBuses::new` hardcodes `transition_offsets: vec![0, 1]` +//! (`lookup.rs:922`), so three offsets means an `AIR` impl, and the only ones +//! outside `crypto/**`'s example tree are the examples themselves — none of +//! which has `step_size > 1`. Writing one means adding to `crypto/**`, which is +//! on the standing always-stop list. +//! 2. **`step_size > 1` is not provable at all** — a framework ceiling, measured +//! by [`the_prover_cannot_prove_a_step_size_two_air`] rather than argued (in +//! debug the prover panics on the `RowFrame` shape assert; in release, with +//! that assert compiled out, it emits a proof production's own verifier +//! rejects). So entry 9 cannot be closed by a proof of any kind, from any AIR. +//! +//! What closes entry 9 instead is that it does not need a proof. The defect is in +//! how the machine maps a reconstructed grid onto frame steps, and production has +//! its own function for exactly that mapping (`into_frame`), which is a pure +//! function of a grid and a `step_size`. Differentialling against it needs no +//! prover, and it is a stronger oracle than a proof would have been: it is the +//! very code the real verifier runs. +//! +//! So the two entries get two witnesses, each with a production oracle, and +//! neither witness is of the other's defect. What is NOT covered, stated plainly: +//! no test here runs the ASSEMBLED verifier at `step_size > 1`, because nothing +//! can produce such a proof. Entry 9's closure is therefore about the emitter's +//! grid indexing, not about an end-to-end run. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::traits::IsField; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::examples::fibonacci_multi_column::{ + FibonacciMultiColumnAIR, FibonacciMultiColumnPublicInputs, compute_trace, +}; +use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::{MultiProofView, StarkProofView, StarkTableView}; +use stark::table::Table; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::epoch::{ + RootCells, TableAbsorbs, TableChallengeShape, emit_table_challenges, fork_table, +}; +use super::executor::execute; +use super::fri::FriShape; +use super::transcript_replay::TranscriptReplay; +use super::validator::validate; +use super::word::{base_word, ext_word, word_as_base, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("fixture options") +} + +// ============================================================================= +// Entry 9 — the frame-step view, against production's own frame assembly +// ============================================================================= + +/// A `num_eval_points × width` grid of distinct extension values, so a +/// mis-indexed read is caught by VALUE and not merely by shape. +fn distinct_grid(rows: usize, width: usize) -> Vec> { + (0..rows) + .map(|r| { + (0..width) + .map(|c| { + FEE::new([ + FE::from((100 * r + c + 1) as u64), + FE::from((7 * r + 2 * c + 3) as u64), + FE::from((13 * r + 5 * c + 11) as u64), + ]) + }) + .collect() + }) + .collect() +} + +/// What production's constraint interpreter resolves `Op::Var{offset, col}` to, +/// for every offset and column of a grid — via its OWN frame assembly. +/// +/// `into_frame` groups the grid into `step_size`-row steps +/// (`proof/view.rs:269-294`) and the interpreter takes row 0 of the step +/// (`constraint_ir/interp.rs:240-242`, which asserts `row == 0`). Nothing here is +/// our arithmetic: the grid goes in, production decides which value each offset +/// sees. +fn production_frame_values( + grid: &[Vec], + main_width: usize, + step_size: usize, +) -> Vec> { + let width = grid[0].len(); + let flat: Vec = grid.iter().flat_map(|r| r.iter().cloned()).collect(); + let table = Table::new(flat, width); + let frame = StarkTableView::Owned(&table).into_frame(main_width, step_size); + (0..grid.len() / step_size) + .map(|offset| { + let step = frame.get_evaluation_step(offset); + (0..width) + .map(|col| { + if col < main_width { + *step.get_main_evaluation_element(0, col) + } else { + *step.get_aux_evaluation_element(0, col - main_width) + } + }) + .collect() + }) + .collect() +} + +/// ★ ENTRY 9: the machine's frame-step view is production's, at a `step_size` +/// where the two possible answers differ. +/// +/// The oracle is `StarkTableView::into_frame` — the function the real verifier +/// calls on the reconstructed grid (`verifier.rs:320-321`) — so this is not a +/// comparison of two of our own passes. +/// +/// The `step_size = 1` case is included deliberately and it is the point of the +/// entry: there the strided view and the whole grid are the SAME vector, so the +/// test passes for a correct emitter and for the defective one alike. `step_size = +/// 2` separates them, and the negative half below is what shows it. +#[test] +fn the_frame_step_view_matches_productions_own_frame_assembly() { + use super::epoch_verify::frame_step_view; + + let main_width = 3usize; + let width = 4usize; // one aux column, so the aux branch is exercised too + for (offsets, step_size) in [(2usize, 1usize), (3, 1), (2, 2), (3, 2), (2, 4)] { + let rows = offsets * step_size; + let grid = distinct_grid(rows, width); + let expected = production_frame_values(&grid, main_width, step_size); + let got = frame_step_view(&grid, step_size); + assert_eq!( + got.len(), + offsets, + "offsets {offsets}, step_size {step_size}: one view row per frame step" + ); + assert_eq!( + got, expected, + "offsets {offsets}, step_size {step_size}: the machine's frame-step \ + view must be the values production's own frame assembly hands the \ + interpreter" + ); + } + + // ---- ★ the negative half: the wave-5 defect, and the fact that only + // step_size > 1 can see it. + // + // M2 passed the WHOLE grid to the constraint fold. At step_size 1 that is + // literally the same vector, so the mutation was invisible to every test in + // the suite. At step_size 2 it is a different vector, and this is the + // comparison that says so. + for step_size in [1usize, 2] { + let grid = distinct_grid(3 * step_size, width); + let expected = production_frame_values(&grid, main_width, step_size); + let whole_grid = grid.clone(); + if step_size == 1 { + assert_eq!( + whole_grid, expected, + "at step_size 1 the whole grid IS the frame view — this is the \ + blindness the entry records, not a bug" + ); + } else { + assert_ne!( + whole_grid, expected, + "at step_size {step_size} the whole grid must NOT be the frame \ + view, or this witness sees nothing" + ); + // And precisely which rows differ: production sees rows 0, 2, 4. + assert_eq!( + expected, + vec![grid[0].clone(), grid[2].clone(), grid[4].clone()], + "production's frame reads every step_size-th row" + ); + } + } +} + +// ============================================================================= +// The framework ceiling: step_size > 1 is not provable +// ============================================================================= + +/// The parameter the ceiling test uses. +const STEP_SIZE: usize = 2; +const STRIDED_COLS: usize = 3; +const STRIDED_ROWS: usize = 64; + +/// Reads both transition steps, so the AIR would have a non-empty next-row +/// column set if it could be proved. +struct StridedConstraints; + +type StridedAir = AirWithBuses; + +impl ConstraintSet for StridedConstraints { + fn eval>(&self, b: &mut B) { + let here = b.main(0, 0); + let there = b.main(1, 0); + b.emit_base(0, there - here); + } +} + +/// The `step_size = 2` fixture, shared by both halves of the ceiling test. +/// +/// Column 0 — the only column `StridedConstraints` reads — is CONSTANT, so +/// `main(1, 0) − main(0, 0)` is zero under ANY choice of which rows the two +/// transition offsets resolve to. That is what makes the release half below +/// meaningful: the proof it produces is rejected for a structural +/// prover/verifier disagreement, not because some other frame reading would +/// violate the constraint. +fn strided_fixture() -> (StridedAir, TraceTable) { + let air = StridedAir::new( + STRIDED_COLS, + AuxiliaryTraceBuildData { + interactions: vec![], + }, + &options(), + STEP_SIZE, + StridedConstraints, + ) + .with_name("STRIDED"); + + let mut main = Vec::with_capacity(STRIDED_ROWS * STRIDED_COLS); + for r in 0..STRIDED_ROWS as u64 { + main.push(FE::from(7u64)); + main.push(FE::from(1_000 + r)); + main.push(FE::from(2_000 + 3 * r)); + } + let trace = TraceTable::new_main(main, STRIDED_COLS, STEP_SIZE); + + assert_eq!(air.step_size(), STEP_SIZE, "the fixture's step size"); + assert_eq!( + air.context().transition_offsets.len() * air.step_size(), + 4, + "num_eval_points is offsets x step_size, so this AIR WOULD have two-row \ + blocks and a stride of two — the shape both entries want" + ); + + (air, trace) +} + +/// ★ A FRAMEWORK CEILING, measured rather than asserted from reading, and +/// reported as a finding (standing decisions: report ceilings, do not work around +/// them silently). +/// +/// The production prover cannot prove ANY AIR with `step_size > 1`, but it fails +/// in two DIFFERENT ways depending on the build profile, so the ceiling is +/// witnessed twice — once per profile — rather than in a single `should_panic` +/// that only holds in one of them: +/// +/// - **Debug (this body).** The CPU transition evaluator borrows one row per +/// transition offset (`RowFrame::from_lde`, `evaluator.rs:72`) and asserts the +/// single-row shape outright: `debug_assert_eq!(lde_trace.lde_step_size, +/// lde_trace.blowup_factor, "RowFrame requires single-row steps (step_size 1)")` +/// — and `lde_step_size = trace_step_size · blowup_factor`, so the equality IS +/// `step_size == 1`. The prover panics before emitting anything. +/// - **Release (the sibling body below, selected by `cfg(not(debug_assertions))`).** +/// `debug_assert` is compiled out, so the prover runs to completion and returns +/// `Ok(proof)` — and production's own verifier REJECTS that proof. Measured, not +/// read: the ceiling is a completeness failure, not a soundness one, and it is +/// NOT the one assert. Relaxing the assert alone would not lift it. +/// +/// Nothing production-reachable is affected either way: every AIR in the tree +/// reports `step_size = 1` — the VM tables and the LFM chips pass it through +/// their `build_air` helpers, the continuation AIRs pass it to +/// `AirWithBuses::new` directly, and every example AIR's `step_size` impl returns +/// the literal `1` — so this shape exists only in this fixture. +/// +/// What this costs the ledger: entry 9 can have no end-to-end witness, from any +/// AIR, until the ceiling lifts — +/// [`the_frame_step_view_matches_productions_own_frame_assembly`] closes the +/// emitter's half against production's own frame assembly instead. +/// +/// Both halves are self-updating: if the ceiling is ever lifted, the debug half +/// stops panicking and the release half starts verifying, and each fails saying +/// entry 9 became closeable end to end. +#[cfg(debug_assertions)] +#[test] +#[should_panic(expected = "RowFrame requires single-row steps")] +fn the_prover_cannot_prove_a_step_size_two_air() { + use crate::test_utils::multi_prove_ram; + + let (air, mut trace) = strided_fixture(); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let _ = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])); +} + +/// ★ The same ceiling, as release actually reaches it — see the debug body above +/// for the full finding. +/// +/// With the `RowFrame` `debug_assert` compiled out the prover does NOT stop: it +/// emits a proof. What still holds is the claim the test's name makes, one level +/// out — that proof does not round-trip, because production's own verifier +/// rejects it. Asserting the rejection (rather than skipping the test in release) +/// is what keeps the required release CI gate covering this path. +#[cfg(not(debug_assertions))] +#[test] +fn the_prover_cannot_prove_a_step_size_two_air() { + use crate::test_utils::multi_prove_ram; + + let (air, mut trace) = strided_fixture(); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .expect("with the debug_assert compiled out the prover runs to completion"); + + let refs: Vec<&dyn AIR> = vec![&air]; + assert!( + !crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(&proof), + &mut crate::hash_pin::block_transcript(&[]), + &FEE::zero(), + ), + "production accepted a step_size = 2 proof — the framework ceiling lifted, \ + so entry 9 is now closeable end to end and this test should be replaced by \ + the end-to-end witness" + ); +} + +// ============================================================================= +// Entry 8 — the absorb order, on a real proof with a multi-row OOD block +// ============================================================================= + +type FibAir = FibonacciMultiColumnAIR; +type FibPi = FibonacciMultiColumnPublicInputs; + +/// Columns the three-offset fixture carries. More than one, or a column-major and +/// a row-major absorb of the block coincide. +const FIB_COLS: usize = 3; +const FIB_ROWS: usize = 64; + +fn fib_initial_values() -> Vec<(FE, FE)> { + (0..FIB_COLS as u64) + .map(|c| (FE::from(1 + c), FE::from(3 + 2 * c))) + .collect() +} + +/// A real proof of a real three-offset AIR, produced and accepted by production. +fn fib_proof() -> ( + FibAir, + FibPi, + stark::proof::stark::MultiProof, +) { + use crate::test_utils::multi_prove_ram; + + let opts = options(); + let air = FibAir::with_num_columns(&opts, FIB_COLS); + let initial_values = fib_initial_values(); + let pi = FibPi { + initial_values: initial_values.clone(), + }; + let mut trace = compute_trace::(&initial_values, FIB_ROWS); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &pi)]; + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .expect("the three-offset fixture must prove"); + (air, pi, proof) +} + +/// The challenges production derives from this proof, and the shape the machine +/// must replay. +struct FibReplay { + shape: TableChallengeShape, + main_root: stark::config::Commitment, + composition_root: stark::config::Commitment, + ood_current: Vec, + ood_next: Vec, + parts: Vec, + fri_roots: Vec, + fri_coeffs: Vec, + nonce: Option, + beta: FEE, + z: FEE, + gamma: FEE, + zetas: Vec, + iotas: Vec, +} + +fn fib_replay( + air: &FibAir, + pi: &FibPi, + proof: &stark::proof::stark::MultiProof, +) -> FibReplay { + use stark::domain::new_verifier_domain; + + let view = StarkProofView::Owned(&proof.proofs[0]); + let opts = air.options(); + let trace_length = view.trace_length(); + let log2_trace_length = trace_length.trailing_zeros(); + let log2_blowup = (opts.blowup_factor as usize).trailing_zeros(); + + // Single-table Phase A, transcribed from `multi_verify_views`: this AIR is not + // preprocessed and has no aux trace, so it is the main root and nothing else. + assert!(!air.is_preprocessed(), "the fixture is not preprocessed"); + assert!(!air.has_aux_trace(), "the fixture has no aux trace"); + let mut transcript = crate::hash_pin::block_transcript(&[]); + transcript.append_bytes(view.lde_trace_main_merkle_root()); + + let domain = new_verifier_domain(air, trace_length); + let layout = Verifier::::ood_layout(air); + let challenges = Verifier::::replay_rounds_after_round_1( + air, + view, + pi, + &domain, + &mut transcript, + Vec::new(), + &layout, + ); + + let nt = challenges.transition_coeffs.len(); + let beta = if nt > 1 { + challenges.transition_coeffs[1] + } else { + challenges.boundary_coeffs[0] + }; + let gamma = challenges.trace_term_coeffs[1][0]; + + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + let shape = TableChallengeShape { + index: 0, + num_tables: 1, + has_aux_root: view.lde_trace_aux_merkle_root().is_some(), + has_contribution: view.bus_table_contribution().is_some(), + log2_trace_length, + log2_blowup, + coset_offset: FE::from(opts.coset_offset), + ood_current_dims: (ood_c.width(), ood_c.height()), + ood_next_dims: (ood_n.width(), ood_n.height()), + num_parts: view.composition_poly_parts_ood_evaluation().len(), + fri: FriShape::from_options(opts, log2_trace_length + log2_blowup), + grinding_factor: opts.grinding_factor, + num_queries: opts.fri_number_of_queries, + }; + + FibReplay { + shape, + main_root: *view.lde_trace_main_merkle_root(), + composition_root: *view.composition_poly_root(), + ood_current: ood_c.row_major_data().to_vec(), + ood_next: ood_n.row_major_data().to_vec(), + parts: view.composition_poly_parts_ood_evaluation().to_vec(), + fri_roots: view.fri_layers_merkle_roots().to_vec(), + fri_coeffs: view.fri_final_poly_coeffs().to_vec(), + nonce: view.nonce(), + beta, + z: challenges.z, + gamma, + zetas: challenges.zetas.clone(), + iotas: challenges.iotas.clone(), + } +} + +/// The machine's replay of `r`'s rounds, publishing every challenge. +fn fib_challenge_program( + r: &FibReplay, +) -> (super::compiler::LfmProgram, Vec>) { + let s = &r.shape; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + + let a_main = b.declare_arena(super::proof_arena::words_per_root() as u32); + let a_composition = b.declare_arena(super::proof_arena::words_per_root() as u32); + let a_current = b.declare_arena((s.ood_current_dims.0 * s.ood_current_dims.1) as u32); + let a_next = b.declare_arena((s.ood_next_dims.0 * s.ood_next_dims.1) as u32); + let a_parts = b.declare_arena(s.num_parts as u32); + let a_fri_roots = + b.declare_arena(super::proof_arena::words_per_root() as u32 * s.fri.num_committed() as u32); + let a_fri_coeffs = b.declare_arena(s.fri.num_terminal_coeffs() as u32); + let a_nonce = (s.grinding_factor > 0).then(|| b.declare_arena(1)); + + let mut t = TranscriptReplay::new(&[]); + let main = RootCells::hint(&mut b, a_main, 0); + main.absorb(&mut b, &mut t); + + let composition = RootCells::hint(&mut b, a_composition, 0); + let current: Vec<_> = (0..(s.ood_current_dims.0 * s.ood_current_dims.1) as u32) + .map(|i| b.hint_word(a_current, i).as_ext()) + .collect(); + let next: Vec<_> = (0..(s.ood_next_dims.0 * s.ood_next_dims.1) as u32) + .map(|i| b.hint_word(a_next, i).as_ext()) + .collect(); + let parts: Vec<_> = (0..s.num_parts as u32) + .map(|i| b.hint_word(a_parts, i).as_ext()) + .collect(); + let fri_roots: Vec<_> = (0..s.fri.num_committed()) + .map(|i| { + RootCells::hint( + &mut b, + a_fri_roots, + super::proof_arena::words_per_root() as u32 * i as u32, + ) + }) + .collect(); + let fri_coeffs: Vec<_> = (0..s.fri.num_terminal_coeffs() as u32) + .map(|i| b.hint_word(a_fri_coeffs, i).as_ext()) + .collect(); + let nonce = a_nonce.map(|id| b.hint_felt(id, 0)); + + let mut fork = fork_table(&t, s.index, s.num_tables); + let ch = emit_table_challenges( + &mut b, + &mut fork, + s, + &TableAbsorbs { + aux_root: None, + contribution: None, + composition_root: &composition, + ood_current: ¤t, + ood_next: &next, + parts: &parts, + fri_roots: &fri_roots, + fri_coeffs: &fri_coeffs, + nonce, + }, + ); + b.public(ch.beta.as_cell()); + b.public(ch.z.as_cell()); + b.public(ch.gamma.as_cell()); + for zeta in &ch.zetas { + b.public(zeta.as_cell()); + } + for bits in &ch.iota_bits { + let felt = super::edsl::bits_to_felt(&mut b, bits); + b.public(felt.as_cell()); + } + + let program = compile(b.finish()); + validate(&program).expect("the three-offset replay must be admissible"); + + let mut arenas = vec![ + super::proof_arena::commitments_to_arena(&[r.main_root]), + super::proof_arena::commitments_to_arena(&[r.composition_root]), + r.ood_current.iter().map(ext_word).collect(), + r.ood_next.iter().map(ext_word).collect(), + r.parts.iter().map(ext_word).collect(), + super::proof_arena::commitments_to_arena(&r.fri_roots), + r.fri_coeffs.iter().map(ext_word).collect(), + ]; + if let Some(n) = r.nonce { + arenas.push(vec![base_word(FE::from(n))]); + } + (program, arenas) +} + +/// ★ ENTRY 8: the machine absorbs a MULTI-ROW OOD block in production's order. +/// +/// The fixture is `FibonacciMultiColumnAIR` with three columns — three transition +/// offsets at `step_size = 1`, so `num_eval_points = 3` and the next-row block is +/// 2 rows × 3 columns. That is the first block in this phase where column-major +/// and row-major absorbs differ. +/// +/// The oracle is production's own `replay_rounds_after_round_1` on a proof the +/// production verifier accepts, so this is the same differential the epoch spine +/// runs — on a shape the epoch cannot produce. +/// +/// The negative half is not optional: without it a green test here would be +/// consistent with the block still being one row tall. So the same proof's block +/// is absorbed ROW-major through production's own transcript, and the challenge +/// that follows must MOVE. +#[test] +fn the_machine_absorbs_a_multi_row_ood_block_in_productions_order() { + let (air, pi, proof) = fib_proof(); + + // Production must accept it, or the blocks below are not a real proof's. + let refs: Vec<&dyn AIR> = vec![&air]; + assert!( + crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + MultiProofView::Owned(&proof), + &mut crate::hash_pin::block_transcript(&[]), + &FEE::zero(), + ), + "production must accept the three-offset fixture" + ); + + let r = fib_replay(&air, &pi, &proof); + + // ---- ★ the blindness this fixture removes, ASSERTED before it is relied on. + assert_eq!( + air.context().transition_offsets.len(), + 3, + "the fixture must have three transition offsets" + ); + assert_eq!( + air.context().transition_offsets.len() * air.step_size(), + 3, + "num_eval_points" + ); + println!( + " three-offset fixture: offsets {:?}, step_size {}, ood_current {:?}, \ + ood_next {:?}, next_row_cols {:?}, parts {}", + air.context().transition_offsets, + air.step_size(), + r.shape.ood_current_dims, + r.shape.ood_next_dims, + air.trace_ood_next_row_columns(), + r.shape.num_parts, + ); + assert!( + r.shape.ood_next_dims.1 > 1 && r.shape.ood_next_dims.0 > 1, + "the next-row OOD block must be taller than one row AND wider than one \ + column, or a row-major absorb is indistinguishable: got {:?}", + r.shape.ood_next_dims + ); + + // ---- the differential: every challenge, against production's own replay. + let (program, arenas) = fib_challenge_program(&r); + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the three-offset replay must execute"); + + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), r.beta, "beta"); + assert_eq!(pub_ext(1), r.z, "z"); + assert_eq!( + pub_ext(2), + r.gamma, + "gamma — the first challenge AFTER the OOD absorb" + ); + let mut cursor = 3usize; + for (k, want) in r.zetas.iter().enumerate() { + assert_eq!(pub_ext(cursor + k), *want, "zeta {k}"); + } + cursor += r.zetas.len(); + for q in 0..r.shape.num_queries { + let got = word_as_base(&exec.public_words[cursor + q].1).expect("an index is a base felt"); + assert_eq!(got, FE::from(r.iotas[q] as u64), "iota {q}"); + } + cursor += r.shape.num_queries; + assert_eq!( + cursor, + exec.public_words.len(), + "every published challenge must be checked" + ); + + // ---- ★ the negative half: INJECT the row-major absorb and watch it fail. + // + // Not a comparison of two of my own orders — a control program that replays + // the same rounds with the blocks absorbed row-major, checked against the SAME + // production challenge the positive half matched. One side is production's. + // + // The control stops at `gamma`, the first challenge drawn after the OOD + // absorb: everything downstream of a wrong `gamma` is wrong for a derived + // reason, and stopping here says the divergence begins exactly at the absorb. + let control_gamma = row_major_control_gamma(&r); + assert_ne!( + control_gamma, r.gamma, + "a row-major absorb of this proof's OOD blocks must move the first \ + challenge drawn after them — if it does not, the fixture is as blind as \ + the epoch and entry 8 stays open" + ); + println!( + " row-major control: gamma moves ({} != production's), so the absorb \ + order is load-bearing on this fixture and the differential above covers it", + control_gamma == r.gamma + ); +} + +/// The DENIED absorb order, emitted: Phase A, round 2, `z`, both OOD blocks +/// ROW-major, the parts, then `γ`. +/// +/// This is the mutation entry 8 says nothing can catch — deliberately built so it +/// CAN be caught, on a fixture whose blocks are more than one row tall. It stops +/// at `γ` because that is the first value the absorb order can move, so a +/// difference here is attributable to the order and to nothing else. +/// +/// It duplicates the round structure of [`super::epoch::emit_table_challenges`] +/// rather than calling it with a flag: a production emitter should not carry a +/// switch for its own denied behaviour, and the duplication is bounded because +/// the control needs nothing past `γ`. +fn row_major_control_gamma(r: &FibReplay) -> FEE { + let s = &r.shape; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + + let a_main = b.declare_arena(super::proof_arena::words_per_root() as u32); + let a_composition = b.declare_arena(super::proof_arena::words_per_root() as u32); + let a_current = b.declare_arena((s.ood_current_dims.0 * s.ood_current_dims.1) as u32); + let a_next = b.declare_arena((s.ood_next_dims.0 * s.ood_next_dims.1) as u32); + let a_parts = b.declare_arena(s.num_parts as u32); + + let mut t = TranscriptReplay::new(&[]); + let main = RootCells::hint(&mut b, a_main, 0); + main.absorb(&mut b, &mut t); + let mut fork = fork_table(&t, s.index, s.num_tables); + + let _beta = fork.sample_ext(&mut b); + let composition = RootCells::hint(&mut b, a_composition, 0); + composition.absorb(&mut b, &mut fork); + + let _z = super::epoch::emit_z_ood(&mut b, &mut fork, s); + let current: Vec<_> = (0..(s.ood_current_dims.0 * s.ood_current_dims.1) as u32) + .map(|i| b.hint_word(a_current, i).as_ext()) + .collect(); + let next: Vec<_> = (0..(s.ood_next_dims.0 * s.ood_next_dims.1) as u32) + .map(|i| b.hint_word(a_next, i).as_ext()) + .collect(); + let parts: Vec<_> = (0..s.num_parts as u32) + .map(|i| b.hint_word(a_parts, i).as_ext()) + .collect(); + + // ★ THE MUTATION: rows outside, columns inside — production has it the other + // way round (`verifier.rs:1421-1431`). + for (dims, block) in [(s.ood_current_dims, ¤t), (s.ood_next_dims, &next)] { + let (width, height) = dims; + for row in 0..height { + for col in 0..width { + let coords = b.unpack(block[row * width + col].as_cell()); + fork.append_ext(&mut b, [coords[0], coords[1], coords[2]]); + } + } + } + for part in &parts { + let coords = b.unpack(part.as_cell()); + fork.append_ext(&mut b, [coords[0], coords[1], coords[2]]); + } + let gamma = fork.sample_ext(&mut b); + b.public(gamma.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the control must be admissible"); + let arenas = vec![ + super::proof_arena::commitments_to_arena(&[r.main_root]), + super::proof_arena::commitments_to_arena(&[r.composition_root]), + r.ood_current.iter().map(ext_word).collect(), + r.ood_next.iter().map(ext_word).collect(), + r.parts.iter().map(ext_word).collect(), + ]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the control must execute"); + word_as_ext(&exec.public_words[0].1).expect("gamma is ext") +} diff --git a/prover/src/lfm/sub_proof.rs b/prover/src/lfm/sub_proof.rs new file mode 100644 index 000000000..3f45ad809 --- /dev/null +++ b/prover/src/lfm/sub_proof.rs @@ -0,0 +1,1007 @@ +//! One sub-proof's query verification: DEEP reconstruction over the SAME arena +//! cells the Merkle authentication authenticates. +//! +//! The [constraint](super::constraints) and [DEEP](super::deep) legs consume +//! opened values; the [Merkle walk](super::edsl::wrap_merkle_walk) +//! authenticates them. Built separately the two are each correct and neither +//! says anything about the other — a program could fold one set of values and +//! authenticate a different set, and every test that fed both halves the same +//! data would pass. This module is the join, and it is a join by CONSTRUCTION +//! rather than by convention: [`emit_group_authentication`] takes cells and +//! cannot hint, so the only values it can authenticate are the caller's, and +//! [`emit_query`] hands those same cells to the DEEP fold. +//! +//! # The two consumers disagree about layout, which is the whole difficulty +//! +//! A query opens four committed matrices — precomputed, main, aux, composition +//! — and each is a SEPARATE Merkle tree with its own root and its own path. The +//! leaf of one tree is that matrix's own row pair: +//! +//! ```text +//! leaf(main) = keccak( main[υ] ‖ main[−υ] ) +//! ``` +//! +//! while DEEP walks one POINT across all matrices: +//! +//! ```text +//! DEEP(υ) folds precomputed[υ] ‖ main[υ] ‖ aux[υ] +//! ``` +//! +//! So the authentication groups by matrix and the fold groups by point. The +//! two orders cross, which is exactly the situation that invites two parallel +//! copies of the same values in two arenas — sound only as long as the host +//! filling them agrees with itself, which no in-machine constraint requires. +//! Here `values` is one vector of cells per matrix, in LEAF order, and DEEP +//! indexes into it: column `c` at the regular point is `values[c]`, at the +//! symmetric point `values[num_columns + c]`. +//! +//! # The query point is derived from the index bits, not hinted +//! +//! `DEEP(υ)` is meaningless unless `υ` is the point the authenticated leaf +//! sits at. Production derives both from one challenge `iota` +//! (`query_challenge_to_evaluation_point`); a machine that hinted the point +//! separately would let a prover authenticate a leaf at one index and evaluate +//! DEEP at another. [`emit_query`] decomposes the hinted index ONCE and uses +//! the same bits for the walk and for the point, so the two cannot disagree. +//! +//! `υ = offset · g^{br(2·iota)}` where `br` is the bit reversal over the LDE +//! domain. Reversing `2·iota` maps index bit `i` to weight `2^{depth-1-i}`, so +//! the point is `offset · Π (g^{2^{depth-1-i}})^{b_i}` — one `Select` and one +//! `Mul` per bit against program constants, via [`super::edsl::pow_bits`]. The +//! symmetric point is `−υ`: `br(2·iota+1) = br(2·iota) + L/2` and `g^{L/2} = +//! −1`, so it costs one subtraction rather than a second derivation. +//! +//! # One-row leaves (S2) +//! +//! Under [`SubProofShape::layout`] = `LeafLayout::Row` every committed matrix +//! holds ONE row per leaf: a query index `r` has `log2(lde)` bits (uniform over +//! the whole LDE, not a pair index), every tree is `log2(lde)` deep, a group's +//! opening is `num_columns` cells (no symmetric row), the point is +//! `x_r = offset · g^{br(r)}` alone ([`emit_point_from_row_bits`]), and DEEP +//! is evaluated ONCE. The FRI leg then starts at the committed DEEP codeword +//! (the input tree) with the input-slot check `group₀[slot] == DEEP(x_r)` +//! (`super::fri::emit_query_fri`). + +use math::field::traits::IsFFTField; + +use crate::tables::types::{FE, GoldilocksField}; + +use stark::leaf_layout::LeafLayout; + +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::deep::{DeepInvariants, DeepOpening, DeepShape, emit_deep_point}; +use super::edsl::{self, WrapDigest}; +use super::merkle_cap::CapCells; + +/// Rows a Merkle leaf covers at today's layout — `crypto/stark`'s +/// `ROWS_PER_LEAF`, mirrored here because it fixes program shape: a leaf holds +/// a row PAIR, which is why one path authenticates both of a query's two +/// points. One-row leaves (S2) are [`SubProofShape::layout`]'s other value. +pub const ROWS_PER_LEAF: usize = 2; + +/// The compile-time shape of one committed matrix of a sub-proof. +/// +/// `is_ext` is the element kind, and it is not cosmetic: a base element is +/// rendered into the leaf as 8 big-endian bytes and an extension element as 24 +/// (components 0, 1, 2, each big-endian — `write_bytes_be` for +/// `FieldElement`). Getting it wrong changes +/// the byte string and therefore the leaf. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GroupShape { + /// Columns at ONE point. A leaf covers `ROWS_PER_LEAF · num_columns`. + pub num_columns: usize, + pub is_ext: bool, +} + +impl GroupShape { + /// Cells one query's opening of this group occupies under row-pair + /// leaves — both points. [`Self::values_at`] is the layout-generic form. + pub fn num_values(&self) -> usize { + self.values_at(ROWS_PER_LEAF) + } + + /// Cells one query's opening of this group occupies when a leaf holds + /// `rows_per_leaf` rows: `2·num_columns` for row pairs, `num_columns` + /// under one-row leaves (S2). + pub fn values_at(&self, rows_per_leaf: usize) -> usize { + rows_per_leaf * self.num_columns + } + + /// Bytes the row-pair leaf hash covers. + pub fn leaf_bytes(&self) -> usize { + self.leaf_bytes_at(ROWS_PER_LEAF) + } + + /// Bytes the leaf hash covers at `rows_per_leaf` rows per leaf. + pub fn leaf_bytes_at(&self, rows_per_leaf: usize) -> usize { + self.values_at(rows_per_leaf) * if self.is_ext { 24 } else { 8 } + } +} + +/// One sub-proof's per-query verification shape. +/// +/// Every field is program SHAPE. In particular the group list is: a proof that +/// carried an aux opening where the program expects none would not match the +/// arena schema, which is the straight-line discipline standing in for +/// production's `(Some(root), Some(opening)) | (None, None)` presence check. +#[derive(Clone, Debug)] +pub struct SubProofShape { + /// The DEEP fold's shape — column count, OOD grid, part count. + pub deep: DeepShape, + /// The TRACE matrices in DEEP column order: precomputed, then main, then + /// aux. Absent groups are omitted, exactly as the proof omits them. Their + /// widths must sum to `deep.num_total_cols`. + pub trace_groups: Vec, + /// Merkle depth — `log2(lde_length) − 1` when a leaf is a row pair, + /// `log2(lde_length)` under one-row leaves ([`Self::layout`]). All four + /// trees commit over the same LDE domain at the same layout, so one depth + /// serves them all and one index addresses them all. + pub merkle_depth: usize, + /// `log2` of the LDE domain — `log2_trace_length + log2(blowup)`. + pub log2_lde_length: u32, + /// The LDE coset offset, `ProofOptions::coset_offset`. + pub coset_offset: FE, + /// The Merkle-cap height of every committed matrix's tree (they share a + /// depth and an opening count, so one height): `0` = uncapped, today's + /// format. With `c > 0` each tree's `2^c` cap digests are hinted ONCE per + /// sub-proof and authenticated against the root ([`CapCells`]), and every + /// query's path stops `c` levels short. A verifier + /// constant: `CapPolicy::height(num_queries, merkle_depth)` of the inner + /// proof's options, never read from the proof. + pub trace_cap: usize, + /// Children per node of every committed matrix's tree: 2, or 4 for a proof + /// committed under an arity-4 configuration (Poseidon1). `trace_cap` then + /// counts 4-ary levels, the host's units, and a path carries three + /// siblings per walked 4-ary level ([`super::merkle_cap::path_hints`]), in + /// the walk's hint order. A verifier constant: the child's configuration + /// (`stark::config::TreeGeometry`), never the proof. + pub arity: usize, + /// The trace trees' leaf layout (S2): today's row + /// pairs, or one row per leaf. A verifier constant — the table's + /// `stark::leaf_layout::table_leaf_layout`, resolved from the AIR's + /// widths and the trace length, never read from the proof. Under + /// [`LeafLayout::Row`] a query opens ONE row per tree, its index ranges + /// over the whole LDE, and DEEP is evaluated at the one point `x_r`. + pub layout: LeafLayout, +} + +impl SubProofShape { + /// Rows one leaf of every committed matrix holds (2, or 1 under S2). + pub fn rows_per_leaf(&self) -> usize { + self.layout.rows_per_leaf() + } + + /// Cells one query's opening of `g` occupies at this shape's layout. + pub fn group_values(&self, g: &GroupShape) -> usize { + g.values_at(self.rows_per_leaf()) + } + + /// The composition-parts group. Its width is the part count and its + /// elements are extension, both of which are already DEEP shape. + pub fn parts_group(&self) -> GroupShape { + GroupShape { + num_columns: self.deep.num_composition_parts, + is_ext: true, + } + } + + /// Every group a query authenticates: the trace matrices then the parts. + pub fn groups(&self) -> Vec { + let mut all = self.trace_groups.clone(); + all.push(self.parts_group()); + all + } + + /// Arena words one query's openings occupy — every group's values, plus + /// the index and the sibling digests (`digest_words` per level per group). + /// + /// ★ `digest_words` is the DIGEST's width in arena words, and it is an + /// argument rather than a read of the configuration: the machine side + /// passes `edsl::digest_words(b)` — the BUILDER's width, the one every + /// emitter advances its cursor by — and the host side passes + /// `proof_arena::words_per_root()`, the width it serialises roots at. A + /// shape that read the configuration here would agree with a builder at + /// `WrapHash::production()` and disagree with any other, and the + /// disagreement would surface as the emitter's own stride assertion. + pub fn query_words(&self, digest_words: usize) -> usize { + 1 + self.opening_words(digest_words) + } + + /// [`Self::query_words`] WITHOUT the index word. + /// + /// The assembled verifier's stride: its query index is not proof data at + /// all but the transcript's own bits, so the arena carries only the opened + /// values and the paths. An arena that still carried an index would be + /// offering the prover a second one. + pub fn opening_words(&self, digest_words: usize) -> usize { + let values: usize = self.groups().iter().map(|g| self.group_values(g)).sum(); + let siblings = digest_words * self.path_len() * self.groups().len(); + values + siblings + } + + /// Sibling digests one query's path carries per group: the tree's depth + /// less its cap height at arity 2, three per walked 4-ary level at arity 4 + /// ([`super::merkle_cap::path_hints`]; the owner path's cap is split off + /// into the caps arena). + pub fn path_len(&self) -> usize { + super::merkle_cap::path_hints(self.merkle_depth, self.trace_cap, self.arity) + } + + /// Node hashes one query's walk costs per group: [`Self::path_len`] at + /// arity 2, one per walked 4-ary level at arity 4. + pub fn path_permutations(&self) -> usize { + super::merkle_cap::path_permutations(self.merkle_depth, self.trace_cap, self.arity) + } + + /// Digests in one committed matrix's cap ([`super::merkle_cap::cap_nodes`]: + /// `2^c`, or the 4-ary level's real nodes). + pub fn cap_nodes(&self) -> usize { + super::merkle_cap::cap_nodes(self.merkle_depth, self.trace_cap, self.arity) + } + + /// Arena words the committed matrices' caps occupy, once per sub-proof: + /// [`Self::cap_nodes`] digests per group when capped, nothing otherwise. + pub fn cap_words(&self, digest_words: usize) -> usize { + if self.trace_cap == 0 { + 0 + } else { + self.groups().len() * self.cap_nodes() * digest_words + } + } + + /// Permutations the committed matrices' cap checks cost, once per + /// sub-proof: the cap hashed to its root per group (nothing uncapped). + pub fn cap_permutations(&self) -> usize { + self.groups().len() * super::merkle_cap::cap_root_permutations(self.cap_nodes(), self.arity) + } + + /// Checked invariants of a shape, so a caller cannot assemble one whose + /// groups do not cover the fold. + fn check(&self) { + assert!( + matches!(self.arity, 2 | 4), + "a tree has 2 or 4 children per node, not {}", + self.arity + ); + assert!( + self.trace_cap <= crypto::merkle_tree::cap::tree_levels(self.merkle_depth, self.arity), + "a cap is at most the tree: height {} over depth {} at arity {}", + self.trace_cap, + self.merkle_depth, + self.arity + ); + let width: usize = self.trace_groups.iter().map(|g| g.num_columns).sum(); + assert_eq!( + width, self.deep.num_total_cols, + "the trace groups must cover exactly the DEEP column set" + ); + assert_eq!( + self.merkle_depth, + self.layout.tree_depth(self.log2_lde_length as usize), + "a row-pair tree is one level shallower than the LDE domain, a \ + one-row tree is as deep as it: depth {} against log2(lde) {} at {:?}", + self.merkle_depth, + self.log2_lde_length, + self.layout + ); + // ⚠ NO `merkle_depth >= 1`. A ONE-PAIR domain — a one-row trace at blowup + // 2 — has a single leaf, so the tree has no levels and the LEAF HASH IS + // THE ROOT. That is a legitimate degenerate shape, and both sides already + // handle it without a special case: + // + // · the host's `verify_merkle_path_from_leaf_hash` loops over an empty + // path and returns `root_hash == hashed_value`; + // · `emit_group_authentication` hashes the leaf, walks zero levels, and + // asserts the result equals the committed root. + // + // So the walk at depth 0 is NOT a no-op — it is exactly the binding, and + // the old assert refused a shape the code below verifies correctly. It + // read "a tree with no levels has no path to walk", which is true and + // beside the point: there is no path, none is walked, and the leaf-versus + // -root compare still happens. + // + // Found by the leaf-node gate at a real continuation epoch's sub-proof + // #10; gated by + // `per_table_aggregator_tests::a_depth_zero_walk_still_binds_leaf_to_root`, + // whose REJECTION arm is what proves the compare survives. + } +} + +/// A committed matrix's root, unpacked once and shared by every query. +/// +/// Hoisting the unpack is what `fri_toy_program` already does per query: the +/// root is a per-sub-proof value and a 219-query proof would otherwise pay +/// 219 redundant `Unpack`s per group. +pub struct GroupCommitment { + /// The root's two words as lanes. + /// ⚠ A `Vec`, ONE ENTRY PER DIGEST CELL, not `[_; 2]`: the array wrote the + /// byte digest's cell COUNT into the type, and an algebraic root is one cell + /// of four felts. `edsl::assert_digest_eq_lanes` zips a digest against these + /// and asserts the widths agree, so it works at either width unchanged. + pub root_lanes: Vec<[Felt; 4]>, + pub shape: GroupShape, + /// The tree's authenticated Merkle cap, when the format caps it: every + /// query's opening is then checked against THESE cells + /// ([`CapCells::verify_path`]) instead of the root lanes. `None` = today. + pub cap: Option, +} + +impl GroupCommitment { + /// Reads a root out of the arena and hoists its unpack. + pub fn hint( + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + base: u32, + shape: GroupShape, + ) -> Self { + // ⚠ The configuration's word count, not two: an algebraic root is ONE + // arena word of four felts. `RootCells::words_per_root` is the single + // definition; a second `+ 1` here would be a second definition. + let n = super::epoch::RootCells::words_per_root(b); + let root_lanes: Vec<[Felt; 4]> = (0..n) + .map(|i| { + let w = b.hint_word(arena, base + i); + b.unpack(w) + }) + .collect(); + GroupCommitment { + root_lanes, + shape, + cap: None, + } + } + + /// A commitment over lanes the caller already holds — the assembled + /// verifier's route, where a root reaches this leg as the SAME cells the + /// transcript absorbed rather than as a second hint. + /// + /// A root has two consumers (`epoch::RootCells`' doc comment names them): + /// the Fiat-Shamir absorb and this comparison. Hinting it twice is the + /// two-consumer hazard — a prover would absorb one root and authenticate + /// against another, and no differential over honest data could see it, + /// because the host packs the same bytes into both. This constructor is the + /// join, and it takes lanes rather than words precisely so there is nothing + /// left to hint. + pub fn from_lanes(root_lanes: Vec<[Felt; 4]>, shape: GroupShape) -> Self { + GroupCommitment { + root_lanes, + shape, + cap: None, + } + } + + /// Hint this tree's height-`c` cap out of `arena` at `base`, authenticate + /// it against the root lanes (once per tree), and check every later + /// opening against it. Returns the next free word. `c = 0` hints nothing + /// and leaves the root check in place. + pub fn hint_cap( + &mut self, + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + base: u32, + c: usize, + depth: usize, + ) -> u32 { + if c == 0 { + return base; + } + let (cap, next) = + super::merkle_cap::hint_and_authenticate(b, arena, base, c, depth, &self.root_lanes); + self.cap = Some(cap); + next + } + + /// The cap height openings of this tree are checked at (0 = the root). + pub fn cap_height(&self) -> usize { + self.cap.as_ref().map_or(0, CapCells::height) + } +} + +/// One query's opening of one committed matrix, as CELLS. +/// +/// There is deliberately no constructor that hints: the values are whatever the +/// caller already holds, which is what makes the authentication and the fold +/// share them rather than agree about them. +pub struct GroupOpening { + /// `evaluations ‖ evaluations_sym` in LEAF order — the row pair written + /// column by column, the regular point first. + pub values: Vec, + /// Sibling digests, LEAF LEVEL FIRST — the order + /// `verify_merkle_path_from_leaf_hash` consumes them in. + pub siblings: Vec, +} + +/// The leaf hash of one group's row pair, in the production commitment layout. +/// +/// Base groups go through [`edsl::wrap_leaf_hash`] unchanged. Extension +/// groups render each element as its three components, each big-endian — +/// `write_bytes_be` writes components 0, 1, 2 in that order, so the machine +/// unpacks the word and byteswaps lanes 0, 1, 2. +/// +/// Lane 3 is NOT hashed, which is correct (production hashes three components) +/// and worth stating: an extension cell whose lane 3 is nonzero would hash the +/// same as one whose lane 3 is zero. It cannot arise here because every +/// extension value a query opens is also consumed as an ext operand by the DEEP +/// fold, and an ext read of a word with a nonzero lane 3 is unprovable. A +/// caller that authenticated an extension group WITHOUT folding it would owe +/// that check itself. +pub fn emit_leaf_hash(b: &mut LfmBuilder, shape: GroupShape, values: &[Cell]) -> WrapDigest { + emit_leaf_hash_rows(b, shape, ROWS_PER_LEAF, values) +} + +/// [`emit_leaf_hash`] for a leaf of `rows_per_leaf` rows: the same stream +/// (every value's felts in the order given — row-major across the leaf's +/// rows, `hash_data_from_slices(evaluations, evaluations_sym)` on the host), +/// sized by the layout. At `rows_per_leaf = 2` it IS [`emit_leaf_hash`], +/// instruction for instruction. +pub fn emit_leaf_hash_rows( + b: &mut LfmBuilder, + shape: GroupShape, + rows_per_leaf: usize, + values: &[Cell], +) -> WrapDigest { + use super::keccak_host::BYTES_PER_HALF; + use super::transcript_replay::felt_be_halves; + + assert_eq!( + values.len(), + shape.values_at(rows_per_leaf), + "a leaf covers the whole row pair (or the one row)" + ); + if !shape.is_ext { + let felts: Vec = values.iter().map(|c| Felt(c.addr())).collect(); + return edsl::wrap_leaf_hash(b, &felts); + } + + // ★ The ALGEBRAIC path absorbs the felts. The byte stream below exists only + // because the incumbent hashes are byte-oriented: what it serialises IS + // three field elements per value, so an algebraic leaf deletes the + // serialisation rather than reimplementing it. ✓ The host's decomposition + // agrees by construction — `write_bytes_be` for an Fp3 element writes + // components 0, 1, 2 in order, which is the lane order `unpack` returns. + let Some(byte_hash) = b.wrap_hash().byte_hash() else { + let mut felts = Vec::with_capacity(3 * values.len()); + for v in values { + let lanes = b.unpack(*v); + felts.extend_from_slice(&lanes[..3]); + } + return edsl::wrap_leaf_hash(b, &felts); + }; + + let mut stream = Vec::with_capacity(6 * values.len()); + for v in values { + let lanes = b.unpack(*v); + for lane in lanes.iter().take(3) { + stream.extend(felt_be_halves(b, *lane)); + } + } + let len_bytes = BYTES_PER_HALF * stream.len(); + debug_assert_eq!(len_bytes, shape.leaf_bytes_at(rows_per_leaf)); + edsl::wrap_hash_bytes(b, byte_hash, &stream, len_bytes) +} + +/// Authenticate one group's opened values against its committed root. +/// +/// Takes the caller's cells and never hints a value, so what it authenticates +/// is what the caller folds. The assert is the binding; `bits` are shared with +/// every other group of the same query, which is what makes the four trees +/// agree about WHICH leaf they opened. +pub fn emit_group_authentication( + b: &mut LfmBuilder, + commitment: &GroupCommitment, + opening: &GroupOpening, + bits: &[Bit], +) { + emit_group_authentication_at(b, commitment, ROWS_PER_LEAF, opening, bits); +} + +/// [`emit_group_authentication`] for a leaf of `rows_per_leaf` rows (the +/// sub-proof's [`SubProofShape::rows_per_leaf`]). +pub fn emit_group_authentication_at( + b: &mut LfmBuilder, + commitment: &GroupCommitment, + rows_per_leaf: usize, + opening: &GroupOpening, + bits: &[Bit], +) { + let leaf = emit_leaf_hash_rows(b, commitment.shape, rows_per_leaf, &opening.values); + authenticate_leaf( + b, + leaf, + &commitment.root_lanes, + commitment.cap.as_ref(), + bits, + &opening.siblings, + ); +} + +/// Authenticate one opened leaf digest at the tree's whole leaf index `bits` +/// (low first, one per binary level): against the root by walking every level +/// when the tree is uncapped, against the cap's node otherwise +/// ([`CapCells::verify_path`]: the cap splits the index, walk low, mux top). +/// At arity 4 the siblings are in the walk's HINT order +/// ([`super::p1w16_emit::walk4`]: the partner, then the other pair, per +/// level), laid out by the harvest at the transcript's index +/// (`merkle_cap::path_to_cap`): every placement is a `Select` on an index bit, +/// so the node hashed is the one the transcript's index names, whatever the +/// hints — three `Select`s a level against the child order's five. +pub(crate) fn authenticate_leaf( + b: &mut LfmBuilder, + leaf: WrapDigest, + root_lanes: &[[Felt; 4]], + cap: Option<&CapCells>, + bits: &[Bit], + siblings: &[WrapDigest], +) { + match cap { + None => { + assert_eq!( + siblings.len(), + b.wrap_hash().path_hints(bits.len()), + "a path's hints for every level, and every group walks the same index" + ); + let root = edsl::wrap_merkle_walk(b, leaf, bits, siblings); + edsl::assert_digest_eq_lanes(b, root, root_lanes); + } + // The whole index goes in; the cap splits it (walk low, mux top). + Some(cap) => cap.verify_path(b, leaf, bits, siblings), + } +} + +/// The LDE-domain constants the point derivation multiplies together for an +/// index of `nbits` bits: `factors[i] = g^{2^{nbits-1-i}}`, matching index bit +/// `i`'s weight after the bit reversal. +/// +/// Row pairs: the index `ι` has `nbits = log2(lde) − 1` bits and the point is +/// at bit-reversed position `2ι`, so bit `i` of `ι` is bit `i + 1` of `2ι`, +/// weight `2^{log2(lde)−2−i} = 2^{nbits−1−i}`. One row: the index `r` has +/// `nbits = log2(lde)` bits and the point is at position `r` itself, weight +/// `2^{log2(lde)−1−i} = 2^{nbits−1−i}`. One formula, keyed on the bit count. +fn point_factors(log2_lde_length: u32, nbits: usize) -> Vec { + let g = ::get_primitive_root_of_unity(log2_lde_length as u64) + .expect("a power-of-two LDE length has a root of unity"); + (0..nbits).map(|i| g.pow(1u64 << (nbits - 1 - i))).collect() +} + +/// `x_r` — the LDE point at bit-reversed position `r` — from the ONE-ROW +/// query index bits (`log2(lde)` of them, S2; `r` uniform over the whole LDE). +/// The one-row counterpart of [`emit_points_from_bits`]: no symmetric point, +/// because a one-row leaf holds one point. +pub fn emit_point_from_row_bits( + b: &mut LfmBuilder, + log2_lde_length: u32, + coset_offset: FE, + bits: &[Bit], +) -> Felt { + assert_eq!( + bits.len(), + log2_lde_length as usize, + "a one-row index ranges over the whole LDE domain" + ); + edsl::pow_bits( + b, + bits, + &point_factors(log2_lde_length, bits.len()), + coset_offset, + ) +} + +/// `(υ, −υ)` from the query index bits, for the LDE domain given by its size and +/// coset offset. Shape-only inputs: the factors are program constants. +/// +/// Keyed on the domain rather than on a [`SubProofShape`] because the FRI leg +/// needs the same derivation and has no trace shape to hand — it holds a +/// [`super::fri::FriShape`], which carries both of these fields. One derivation +/// serves both, which is the point: `join_tests::the_join_premises_hold_on_a_real_proof` +/// checks THIS function against production's +/// `query_challenge_to_evaluation_point` at every index of a real proof, and a +/// second copy would not be covered by that check. +pub fn emit_points_from_bits( + b: &mut LfmBuilder, + log2_lde_length: u32, + coset_offset: FE, + bits: &[Bit], +) -> (Felt, Felt) { + assert_eq!( + bits.len(), + log2_lde_length as usize - 1, + "a leaf is a row pair, so the index is one bit narrower than the domain" + ); + let point = edsl::pow_bits( + b, + bits, + &point_factors(log2_lde_length, bits.len()), + coset_offset, + ); + let zero = b.felt_const(FE::zero()); + (point, b.sub(zero, point)) +} + +/// `(υ, −υ)` from the query index bits (row-pair shapes). +pub fn emit_query_points(b: &mut LfmBuilder, shape: &SubProofShape, bits: &[Bit]) -> (Felt, Felt) { + assert_eq!(bits.len(), shape.merkle_depth); + assert_eq!( + shape.layout, + LeafLayout::RowPair, + "a one-row query has one point (emit_point_from_row_bits)" + ); + emit_points_from_bits(b, shape.log2_lde_length, shape.coset_offset, bits) +} + +/// Everything one query of one sub-proof contributes, emitted. +/// +/// Order of business: decompose the index, authenticate every group against +/// its root, derive the two points from the same bits, then fold DEEP at both. +/// Returns `(DEEP(υ), DEEP(−υ))` for the FRI leg to consume. +/// +/// `trace_openings` is parallel to [`SubProofShape::trace_groups`]; the parts +/// opening is separate because DEEP treats it separately. +pub fn emit_query( + b: &mut LfmBuilder, + shape: &SubProofShape, + gamma: Ext, + inv: &DeepInvariants, + commitments: &[GroupCommitment], + index: Felt, + openings: &[GroupOpening], +) -> (Ext, Ext) { + let out = emit_query_with_bits(b, shape, gamma, inv, commitments, index, openings); + ( + out.deep, + out.deep_sym + .expect("emit_query returns the DEEP pair: a row-pair shape"), + ) +} + +/// What one query contributes when the caller needs more than the DEEP pair. +pub struct QueryOutput { + /// `DEEP(υ)` — or, under one-row leaves, `DEEP(x_r)`, the ONE point the + /// query opens. + pub deep: Ext, + /// `DEEP(−υ)` for a row-pair shape; `None` under one-row leaves (S2), + /// where the query opens no symmetric row and DEEP runs once. + pub deep_sym: Option, + /// The query index decomposed low-to-high — the SAME cells the Merkle walk + /// consumed and the query points were derived from. + /// + /// Handing these out is what lets a later leg join to this one rather than + /// run beside it. FRI reuses the index per layer (leaf position `index >> 1`, + /// partner `index ^ 1`, halving each layer), and a leg that decomposed its + /// own copy would authenticate one index while folding at another — the + /// exact gap this module exists to close, reopened one level up. There is + /// no way to return a DIFFERENT decomposition from here: `bit_dec` is + /// called once and its result feeds the walk, the points and this field. + pub bits: Vec, + /// `υ` — the cell the DEEP fold above evaluated at. + /// + /// Exposed for the same reason as [`Self::bits`], one step further along. + /// FRI needs `υ⁻¹` for its first fold and `υ^(2^total_folds)` for its + /// terminal check; both are functions of this cell, and a leg that + /// re-derived the point from `bits` would pay `merkle_depth` `Select`s and + /// `Mul`s per query for a value it was already holding. Handing the cell + /// over is not just cheaper, it removes the question: there is exactly one + /// `emit_query_points` call in this function and its outputs go to DEEP and + /// to these fields, so no second point EXISTS to disagree. + /// + /// The structural guard is a count, not a comparison — see + /// `fri_tests::the_fri_join_adds_no_second_point_derivation`. + pub point: Felt, + /// `−υ`, likewise. The zero-fold FRI shape checks the terminal polynomial + /// at both points (production's `zetas.is_empty()` branch tests + /// `terminal[2·iota]` AND `terminal[2·iota+1]`). `None` under one-row + /// leaves: there is no second point. + pub point_sym: Option, +} + +/// [`emit_query`], additionally returning the index bits — see [`QueryOutput`]. +/// +/// The index arrives as a FELT here, which is the isolation drivers' route: the +/// differential supplies production's own `iota` and the emitter decomposes it. +/// The assembled verifier does not have a felt to supply — its index is +/// `TranscriptReplay::sample_u64_pow2`'s bits — and takes +/// [`emit_query_from_bits`] instead, which is the same emitter minus this one +/// `bit_dec`. +#[allow(clippy::too_many_arguments)] +pub fn emit_query_with_bits( + b: &mut LfmBuilder, + shape: &SubProofShape, + gamma: Ext, + inv: &DeepInvariants, + commitments: &[GroupCommitment], + index: Felt, + openings: &[GroupOpening], +) -> QueryOutput { + let bits = b.bit_dec(index, shape.merkle_depth); + emit_query_from_bits(b, shape, gamma, inv, commitments, bits, openings) +} + +/// [`emit_query_with_bits`] over an index the caller already holds as BITS. +/// +/// This is the entry point the assembled epoch verifier uses. Production's query +/// index is `sample_u64(lde_length >> 1)`, whose output is `index_bits()` bits by +/// construction (`verifier.rs:138-141`), and the machine's +/// `TranscriptReplay::sample_u64_pow2` produces exactly those bits. Routing them +/// straight in — rather than recomposing a felt and decomposing it again — is +/// what makes the assembled machine's query index in-range by construction and +/// closes ledger entry 5: with no felt in the program, `ι` and `ι + 2^(n−1)` +/// cannot be the same query, because neither is ever a number. +#[allow(clippy::too_many_arguments)] +pub fn emit_query_from_bits( + b: &mut LfmBuilder, + shape: &SubProofShape, + gamma: Ext, + inv: &DeepInvariants, + commitments: &[GroupCommitment], + bits: Vec, + openings: &[GroupOpening], +) -> QueryOutput { + shape.check(); + assert_eq!( + b.wrap_hash().arity(), + shape.arity, + "the builder hashes the trees the shape describes" + ); + let groups = shape.groups(); + assert_eq!(commitments.len(), groups.len(), "one commitment per group"); + assert_eq!(openings.len(), groups.len(), "one opening per group"); + for (c, g) in commitments.iter().zip(&groups) { + assert_eq!(c.shape, *g, "commitment shapes must match the sub-proof"); + assert_eq!( + c.cap_height(), + shape.trace_cap, + "every committed matrix is capped at the shape's height" + ); + } + assert_eq!( + bits.len(), + shape.merkle_depth, + "a query index is exactly the tree's depth in bits" + ); + + let rows = shape.rows_per_leaf(); + for (commitment, opening) in commitments.iter().zip(openings) { + emit_group_authentication_at(b, commitment, rows, opening, &bits); + } + + if shape.layout.is_one_row() { + return emit_one_row_deep(b, shape, gamma, inv, openings, &groups, bits); + } + + let (point, point_sym) = emit_query_points(b, shape, &bits); + + // The crossing: the authenticated cells, re-read by POINT instead of by + // matrix. Nothing is hinted here, so `trace` cannot hold anything the walk + // above did not fold into a leaf. + let mut trace = Vec::with_capacity(shape.deep.num_total_cols); + let mut trace_sym = Vec::with_capacity(shape.deep.num_total_cols); + for (opening, g) in openings.iter().zip(&groups).take(shape.trace_groups.len()) { + for c in 0..g.num_columns { + trace.push(opening.values[c].as_ext()); + trace_sym.push(opening.values[g.num_columns + c].as_ext()); + } + } + + let parts_opening = openings.last().expect("the parts group is always present"); + let num_parts = shape.deep.num_composition_parts; + let parts: Vec = (0..num_parts) + .map(|j| parts_opening.values[j].as_ext()) + .collect(); + let parts_sym: Vec = (0..num_parts) + .map(|j| parts_opening.values[num_parts + j].as_ext()) + .collect(); + + let regular = DeepOpening { + point, + trace, + parts, + }; + let symmetric = DeepOpening { + point: point_sym, + trace: trace_sym, + parts: parts_sym, + }; + QueryOutput { + deep: emit_deep_point(b, &shape.deep, gamma, inv, ®ular), + deep_sym: Some(emit_deep_point(b, &shape.deep, gamma, inv, &symmetric)), + bits, + point, + point_sym: Some(point_sym), + } +} + +/// The one-row half of [`emit_query_from_bits`] (S2), after +/// every group was authenticated at leaf `r`: `x_r` from the SAME bits, then +/// DEEP ONCE, over the authenticated cells — column `c` is `values[c]` (a +/// one-row leaf holds no symmetric row, so there is no `values[w + c]`). +fn emit_one_row_deep( + b: &mut LfmBuilder, + shape: &SubProofShape, + gamma: Ext, + inv: &DeepInvariants, + openings: &[GroupOpening], + groups: &[GroupShape], + bits: Vec, +) -> QueryOutput { + let point = emit_point_from_row_bits(b, shape.log2_lde_length, shape.coset_offset, &bits); + let mut trace = Vec::with_capacity(shape.deep.num_total_cols); + for (opening, g) in openings.iter().zip(groups).take(shape.trace_groups.len()) { + assert_eq!(opening.values.len(), g.num_columns, "one row per leaf"); + trace.extend(opening.values.iter().map(|v| v.as_ext())); + } + let parts_opening = openings.last().expect("the parts group is always present"); + let parts: Vec = parts_opening.values.iter().map(|v| v.as_ext()).collect(); + assert_eq!(parts.len(), shape.deep.num_composition_parts); + let at = DeepOpening { + point, + trace, + parts, + }; + QueryOutput { + deep: emit_deep_point(b, &shape.deep, gamma, inv, &at), + deep_sym: None, + bits, + point, + point_sym: None, + } +} + +// ===================== the whole sub-proof ===================== + +/// The arenas one sub-proof's verification reads, in declaration order. +/// +/// Each field is packed into its OWN arena rather than one concatenated +/// stream — the packing rule [`super::proof_arena`] exists to enforce, applied +/// one level up: a query whose group widths shifted would otherwise silently +/// slide every query behind it. +pub struct SubProofArenas { + /// `γ`, then `ζ`. + pub uniforms: super::instr::ArenaId, + /// The reconstructed OOD grid, row-major, `num_eval_points × + /// num_total_cols` — the same values the constraint leg folds. + pub ood: super::instr::ArenaId, + /// The composition parts claimed at `z^P`. + pub parts: super::instr::ArenaId, + /// Two words per group's committed root, in [`SubProofShape::groups`] order. + pub roots: super::instr::ArenaId, + /// Per query, in order: the index, then per group the row-pair values + /// followed by the sibling digests (two words per level). + pub queries: super::instr::ArenaId, + /// Per group, its [`SubProofShape::cap_nodes`] cap digests — declared + /// only when the shape caps the trees ([`SubProofShape::trace_cap`] `> 0`). + pub caps: Option, +} + +/// Emit a whole sub-proof's query verification: the invariants once, then every +/// query authenticated and folded. +/// +/// Returns `(DEEP(υ), DEEP(−υ))` per query (a row-pair shape). The invariant +/// hoist is the reason a 219-query proof is affordable, and it is production's +/// own hoist — the OOD row sums and the block scalars do not depend on the query. +pub fn emit_sub_proof( + b: &mut LfmBuilder, + shape: &SubProofShape, + num_queries: usize, +) -> (SubProofArenas, Vec<(Ext, Ext)>) { + let (arenas, out) = emit_sub_proof_with_bits(b, shape, num_queries); + ( + arenas, + out.into_iter() + .map(|q| { + ( + q.deep, + q.deep_sym + .expect("emit_sub_proof returns DEEP pairs: a row-pair shape"), + ) + }) + .collect(), + ) +} + +/// [`emit_sub_proof`], additionally returning each query's index bits — see +/// [`QueryOutput`]. The FRI leg folds from these same cells. +pub fn emit_sub_proof_with_bits( + b: &mut LfmBuilder, + shape: &SubProofShape, + num_queries: usize, +) -> (SubProofArenas, Vec) { + use super::deep::emit_deep_invariants; + + shape.check(); + assert!(num_queries > 0, "a proof carries at least one query"); + let groups = shape.groups(); + + let uniforms = b.declare_arena(2); + let ood = b.declare_arena((shape.deep.num_eval_points * shape.deep.num_total_cols) as u32); + let parts = b.declare_arena(shape.deep.num_composition_parts as u32); + let roots = b.declare_arena(edsl::digest_words(b) * groups.len() as u32); + let queries = + b.declare_arena((num_queries * shape.query_words(edsl::digest_words(b) as usize)) as u32); + let cap_words = shape.cap_words(edsl::digest_words(b) as usize); + let caps = (cap_words > 0).then(|| b.declare_arena(cap_words as u32)); + let arenas = SubProofArenas { + uniforms, + ood, + parts, + roots, + queries, + caps, + }; + + let gamma = b.hint_word(uniforms, 0).as_ext(); + let zeta = b.hint_word(uniforms, 1).as_ext(); + + let mut next = 0u32; + let ood_steps: Vec> = (0..shape.deep.num_eval_points) + .map(|_| { + (0..shape.deep.num_total_cols) + .map(|_| { + let c = b.hint_word(ood, next).as_ext(); + next += 1; + c + }) + .collect() + }) + .collect(); + let claimed_parts: Vec = (0..shape.deep.num_composition_parts as u32) + .map(|j| b.hint_word(parts, j).as_ext()) + .collect(); + + let mut commitments: Vec = groups + .iter() + .enumerate() + .map(|(i, g)| GroupCommitment::hint(b, roots, edsl::digest_words(b) * i as u32, *g)) + .collect(); + if let Some(caps) = caps { + let mut at = 0u32; + for c in &mut commitments { + at = c.hint_cap(b, caps, at, shape.trace_cap, shape.merkle_depth); + } + assert_eq!(at as usize, cap_words, "the caps arena is filled exactly"); + } + + let inv = emit_deep_invariants(b, &shape.deep, gamma, zeta, &ood_steps, &claimed_parts); + + let mut cursor = 0u32; + let mut out = Vec::with_capacity(num_queries); + for _ in 0..num_queries { + let index = b.hint_felt(queries, cursor); + cursor += 1; + let openings: Vec = groups + .iter() + .map(|g| { + let values: Vec = (0..shape.group_values(g)) + .map(|_| { + let c = b.hint_word(queries, cursor); + cursor += 1; + c + }) + .collect(); + let siblings: Vec = (0..shape.path_len()) + .map(|_| { + // The stride follows the DIGEST's width, not a literal. + let d = edsl::hint_digest(b, queries, cursor); + cursor += edsl::digest_words(b); + d + }) + .collect(); + GroupOpening { values, siblings } + }) + .collect(); + out.push(emit_query_with_bits( + b, + shape, + gamma, + &inv, + &commitments, + index, + &openings, + )); + } + assert_eq!( + cursor as usize, + num_queries * shape.query_words(edsl::digest_words(b) as usize), + "the emitter's cursor must agree with the declared query stride" + ); + + (arenas, out) +} diff --git a/prover/src/lfm/testdata/poc_rodata_commit.elf b/prover/src/lfm/testdata/poc_rodata_commit.elf new file mode 100644 index 000000000..4288f04bf Binary files /dev/null and b/prover/src/lfm/testdata/poc_rodata_commit.elf differ diff --git a/prover/src/lfm/tests.rs b/prover/src/lfm/tests.rs new file mode 100644 index 000000000..189f59763 --- /dev/null +++ b/prover/src/lfm/tests.rs @@ -0,0 +1,857 @@ +//! Milestone A suite: the software layer round-trips — build → compile → +//! validate → execute — plus the negative paths (validator rejections, +//! executor runtime checks, compiler invariant panics). + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::builder::{LfmBuilder, LfmProgramSource}; +use super::compiler::{LfmProgram, compile}; +use super::executor::{LfmExecError, LfmExecution, execute}; +use super::hash::{LfmHasher, TestPermutation}; +use super::instr::{Addr, HashMode, Instr}; +use super::layout; +use super::validator::{LfmViolation, validate}; +use super::word::{LfmWord, base_word}; + +const GOLDILOCKS_P: u64 = 0xFFFF_FFFF_0000_0001; + +fn fe(v: u64) -> FE { + FE::from(v) +} + +fn ext(a: u64, b: u64, c: u64) -> FEE { + FEE::new([fe(a), fe(b), fe(c)]) +} + +fn run(program: &LfmProgram, arenas: &[Vec]) -> LfmExecution { + validate(program).expect("valid program"); + execute(program, arenas, &TestPermutation).expect("execution succeeds") +} + +fn cell(exec: &LfmExecution, addr: Addr) -> LfmWord { + exec.memory.get(addr).expect("cell written") +} + +fn base_at(exec: &LfmExecution, addr: Addr) -> FE { + super::word::word_as_base(&cell(exec, addr)).expect("base word") +} + +fn ext_at(exec: &LfmExecution, addr: Addr) -> FEE { + super::word::word_as_ext(&cell(exec, addr)).expect("ext word") +} + +// ---- base ALU ---- + +#[test] +fn base_alu_round_trip() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(7)); + let y = b.felt_const(fe(5)); + let s = b.add(x, y); + let d = b.sub(x, y); + let m = b.mul(x, y); + let q = b.div(m, y); + let h = b.mul_add(x, y, s); // 7·5 + 12 = 47 + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(base_at(&exec, s.addr()), fe(12)); + assert_eq!(base_at(&exec, d.addr()), fe(2)); + assert_eq!(base_at(&exec, m.addr()), fe(35)); + assert_eq!(base_at(&exec, q.addr()), fe(7)); + assert_eq!(base_at(&exec, h.addr()), fe(47)); +} + +#[test] +fn div_zero_conventions() { + // 0/0 = 1 (the assert mechanism's accepting case). + let mut b = LfmBuilder::new(); + let z = b.felt_const(FE::zero()); + let q = b.div(z, z); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(base_at(&exec, q.addr()), FE::one()); + + // x/0 with x ≠ 0 errors. + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(3)); + let z = b.felt_const(FE::zero()); + let _ = b.div(x, z); + let program = compile(b.finish()); + validate(&program).expect("structurally valid"); + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::DivByZero { .. })); +} + +#[test] +fn assert_lowering_pass_and_fail() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(6)); + let y = b.felt_const(fe(2)); + let three = b.felt_const(fe(3)); + let q = b.mul(y, three); + b.assert_eq(x, q); + let program = compile(b.finish()); + run(&program, &[]); // passes + + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(6)); + let y = b.felt_const(fe(5)); + b.assert_eq(x, y); + let program = compile(b.finish()); + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::DivByZero { .. })); +} + +// ---- Fp3 ALU ---- + +#[test] +fn ext_alu_matches_field_reference() { + let av = ext(3, 11, 2026); + let bv = ext(9, 1, 77); + let cv = ext(5, 4, 3); + let f = fe(13); + + let mut b = LfmBuilder::new(); + let a = b.ext_const(&av); + let bb = b.ext_const(&bv); + let c = b.ext_const(&cv); + let s = b.eadd(a, bb); + let d = b.esub(a, bb); + let p = b.emul(a, bb); + let q = b.ediv(p, bb); + let ma = b.emul_add(a, bb, c); + let fl = b.felt_const(f); + let mb = b.emul_base(a, fl); + let program = compile(b.finish()); + let exec = run(&program, &[]); + + assert_eq!(ext_at(&exec, s.addr()), &av + &bv); + assert_eq!(ext_at(&exec, d.addr()), &av - &bv); + assert_eq!(ext_at(&exec, p.addr()), &av * &bv); + assert_eq!(ext_at(&exec, q.addr()), av.clone()); + assert_eq!(ext_at(&exec, ma.addr()), &av * &bv + &cv); + let [a0, a1, a2] = *av.value(); + assert_eq!( + ext_at(&exec, mb.addr()), + FEE::new([&a0 * &f, &a1 * &f, &a2 * &f]) + ); +} + +#[test] +fn ext_assert_and_horner() { + // Horner: evaluate 5x² + 3x + 7 at x = (0,1,0) (i.e. w) via mul_add. + let x = ext(0, 1, 0); + let mut b = LfmBuilder::new(); + let xv = b.ext_const(&x); + let c2 = b.ext_const(&ext(5, 0, 0)); + let c1 = b.ext_const(&ext(3, 0, 0)); + let c0 = b.ext_const(&ext(7, 0, 0)); + let acc = b.emul_add(c2, xv, c1); // 5x + 3 + let acc = b.emul_add(acc, xv, c0); // 5x² + 3x + 7 + let expected = &(&(&ext(5, 0, 0) * &x) + &ext(3, 0, 0)) * &x + &ext(7, 0, 0); + let ex = b.ext_const(&expected); + b.assert_eq_ext(acc, ex); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(ext_at(&exec, acc.addr()), expected); +} + +// ---- select / bitdec ---- + +#[test] +fn select_swaps_on_bit() { + let mut b = LfmBuilder::new(); + let l = b.felt_const(fe(100)); + let r = b.felt_const(fe(200)); + let b0 = b.bit_const(false); + let b1 = b.bit_const(true); + let (l0, r0) = b.select(b0, l.as_cell(), r.as_cell()); + let (l1, r1) = b.select(b1, l.as_cell(), r.as_cell()); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(cell(&exec, l0.addr()), base_word(fe(100))); + assert_eq!(cell(&exec, r0.addr()), base_word(fe(200))); + assert_eq!(cell(&exec, l1.addr()), base_word(fe(200))); + assert_eq!(cell(&exec, r1.addr()), base_word(fe(100))); +} + +#[test] +fn non_boolean_select_bit_rejected() { + let mut b = LfmBuilder::new(); + let l = b.felt_const(fe(1)); + let r = b.felt_const(fe(2)); + let two = b.felt_const(fe(2)); + let (_, _) = b.select(super::builder::Bit(two.addr()), l.as_cell(), r.as_cell()); + let program = compile(b.finish()); + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::NonBooleanBit(_))); +} + +#[test] +fn bit_dec_edge_values() { + // Canonical decomposition + the p-specific gadget witnesses at the edges. + for v in [ + 0u64, + 1, + (1 << 32) - 1, + GOLDILOCKS_P - 1, + 0x1234_5678_9ABC_DEF0, + ] { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(v)); + let bits = b.bit_dec(x, 64); + let program = compile(b.finish()); + let exec = run(&program, &[]); + for (i, bit) in bits.iter().enumerate() { + assert_eq!( + base_at(&exec, bit.addr()), + fe((v >> i) & 1), + "bit {i} of {v:#x}" + ); + } + let row = &exec.records.bitdec[0]; + let top = (v >> 32) as u32; + if top == u32::MAX { + assert_eq!(row.z, FE::one(), "z for {v:#x}"); + assert_eq!(row.ginv, FE::zero()); + } else { + assert_eq!(row.z, FE::zero(), "z for {v:#x}"); + let g = fe(0xFFFF_FFFFu64 - top as u64); + assert_eq!(&row.ginv * &g, FE::one(), "ginv·g = 1 for {v:#x}"); + } + } +} + +#[test] +fn bit_dec_partial_width_allocates_only_requested_cells() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(0b1011)); + let bits = b.bit_dec(x, 4); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(bits.len(), 4); + let vals: Vec = bits.iter().map(|bit| base_at(&exec, bit.addr())).collect(); + assert_eq!(vals, vec![fe(1), fe(1), fe(0), fe(1)]); + // All 64 witness bits still recorded for the constraint columns. + assert_eq!( + exec.records.bitdec[0].bits[4..] + .iter() + .filter(|b| **b == FE::one()) + .count(), + 0 + ); +} + +// ---- hash ---- + +#[test] +fn hash_compress_and_permute_match_reference() { + let hasher = TestPermutation; + let a: LfmWord = core::array::from_fn(|i| fe(10 + i as u64)); + let c: LfmWord = core::array::from_fn(|i| fe(20 + i as u64)); + + let mut b = LfmBuilder::new(); + let da = b.digest_const(a); + let dc = b.digest_const(c); + let d = b.compress(da, dc); + let s0 = b.digest_const(core::array::from_fn(|i| fe(30 + i as u64))); + let s1 = b.digest_const(core::array::from_fn(|i| fe(40 + i as u64))); + let s2 = b.digest_const(core::array::from_fn(|i| fe(50 + i as u64))); + let out = b.permute([s0.as_cell(), s1.as_cell(), s2.as_cell()]); + let program = compile(b.finish()); + let exec = run(&program, &[]); + + assert_eq!(cell(&exec, d.addr()), hasher.compress(&a, &c)); + + let mut state: [FE; 12] = core::array::from_fn(|_| FE::zero()); + for i in 0..4 { + state[i] = fe(30 + i as u64); + state[4 + i] = fe(40 + i as u64); + state[8 + i] = fe(50 + i as u64); + } + let expected = hasher.permute(state); + for (j, o) in out.iter().enumerate() { + let w = cell(&exec, o.addr()); + for l in 0..4 { + assert_eq!(w[l], expected[4 * j + l]); + } + } +} + +// ---- hints / public ---- + +#[test] +fn hint_and_public_round_trip() { + let w0: LfmWord = core::array::from_fn(|i| fe(100 + i as u64)); + let w1: LfmWord = core::array::from_fn(|i| fe(200 + i as u64)); + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let h0 = b.hint_word(arena, 0); + let h1 = b.hint_word(arena, 1); + b.public(h0); + b.public(h1); + let program = compile(b.finish()); + let exec = run(&program, &[vec![w0, w1]]); + + assert_eq!(cell(&exec, h0.addr()), w0); + assert_eq!(exec.public_words, vec![(0, w0), (1, w1)]); +} + +#[test] +fn arena_out_of_bounds_rejected_by_validator_and_executor() { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let h = b.hint_word(arena, 5); // past the declared length + b.public(h); + let program = compile(b.finish()); + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::ArenaOutOfBounds { arena: 0, index: 5 } + ); + let err = execute(&program, &[vec![base_word(fe(1)); 2]], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::ArenaOutOfBounds { .. })); +} + +// ---- validator negatives (via mutation of a valid program) ---- + +fn small_valid_program() -> LfmProgram { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(4)); + let y = b.felt_const(fe(9)); + let s = b.add(x, y); + let p = b.mul(s, x); + b.public(p.as_cell()); + compile(b.finish()) +} + +#[test] +fn validator_rejects_double_write() { + let mut program = small_valid_program(); + // Point the mul's destination at the add's (already-written) cell. + let add_out = program + .instrs + .iter() + .find_map(|i| match i { + Instr::BaseAlu { + op: super::instr::BaseOp::Add, + out, + .. + } => Some(*out), + _ => None, + }) + .unwrap(); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Mul, + out, + .. + } = i + { + *out = add_out; + } + } + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::DoubleWrite { addr: add_out.0 } + ); + // The executor independently catches it. + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert_eq!(err, LfmExecError::DoubleWrite(add_out.0)); +} + +/// ★ The check the split memory layout made load-bearing. +/// +/// The value array is zero-filled and the occupancy bit is now the ONLY thing +/// separating "never written" from "written zero" — `Option` used to +/// carry that distinction inside the cell. So both halves are asserted here: a +/// genuine zero reads back as a zero, and an address no instruction has written +/// is an error rather than that same zero. +/// +/// ⚠ Nothing else in the suite asserted `ReadBeforeWrite` from the executor +/// (the cycle test stops at the validator), so deleting the bit test would have +/// been silent. +#[test] +fn executor_rejects_read_before_write() { + // A written zero is a value. + let mut b = LfmBuilder::new(); + let z = b.felt_const(fe(0)); + let one = b.felt_const(fe(1)); + let s = b.add(z, one); + b.public(s.as_cell()); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(base_at(&exec, z.addr()), fe(0), "a written zero is a value"); + assert_eq!(base_at(&exec, s.addr()), fe(1)); + + // An address only a LATER instruction writes is not that zero. + let mut program = small_valid_program(); + let mul_out = program + .instrs + .iter() + .find_map(|i| match i { + Instr::BaseAlu { + op: super::instr::BaseOp::Mul, + out, + .. + } => Some(*out), + _ => None, + }) + .unwrap(); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Add, + a, + .. + } = i + { + *a = mul_out; + } + } + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert_eq!(err, LfmExecError::ReadBeforeWrite(mul_out.0)); +} + +/// An address past `num_addrs` is out of range in both directions, and the two +/// directions report differently: a read is indistinguishable from an unwritten +/// cell, a write is a caller bug the executor names. +#[test] +fn executor_rejects_addresses_outside_the_program() { + let mut program = small_valid_program(); + let past = Addr(program.num_addrs); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Add, + a, + .. + } = i + { + *a = past; + } + } + assert_eq!( + execute(&program, &[], &TestPermutation).unwrap_err(), + LfmExecError::ReadBeforeWrite(past.0) + ); + + let mut program = small_valid_program(); + let past = Addr(program.num_addrs); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Add, + out, + .. + } = i + { + *out = past; + } + } + assert_eq!( + execute(&program, &[], &TestPermutation).unwrap_err(), + LfmExecError::Internal("address out of range") + ); +} + +/// The memory cell divides the cache line. That is the whole mechanism behind +/// splitting the occupancy flag out of the cell: `Option` is 40 bytes +/// (a Goldilocks felt has no niche), so a cell straddled two lines at half of +/// all addresses. If the word ever grows past 32 bytes the argument is gone, and +/// this says so rather than letting the layout quietly regress. +#[test] +fn the_memory_cell_divides_the_cache_line() { + assert_eq!(core::mem::size_of::(), 32); + assert_eq!(64 % core::mem::size_of::(), 0); + assert_eq!(core::mem::size_of::>(), 40); +} + +/// A program touching most of the chips, with an arena, for the record-sizing +/// test below. Row counts are deliberately not powers of two, so a vector grown +/// by `push` doubling lands on a different capacity than a sized one. +fn multi_chip_program() -> (LfmProgram, Vec>) { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(1); + let h0 = b.hint_felt(arena, 0); + let x = b.felt_const(fe(7)); + let y = b.felt_const(fe(5)); + let s = b.add(x, y); + let d = b.sub(s, h0); + let m = b.mul(d, y); + let e0 = b.ext_const(&ext(1, 2, 3)); + let e1 = b.ext_const(&ext(4, 5, 6)); + let ep = b.emul(e0, e1); + let bit = b.bit_const(true); + let (l, r) = b.select(bit, x.as_cell(), y.as_cell()); + let da = b.digest_const([fe(1), fe(2), fe(3), fe(4)]); + let db = b.digest_const([fe(5), fe(6), fe(7), fe(8)]); + let dc = b.compress(da, db); + let lanes = b.unpack(dc.as_cell()); + let packed = b.pack_word(lanes); + b.public(packed); + b.public(l); + b.public(r); + b.public(ep.as_cell()); + b.public(m.as_cell()); + (compile(b.finish()), vec![vec![base_word(fe(3))]]) +} + +/// ★ The records are SIZED from the compiler's census, not grown. +/// +/// Pass 2 opens one column-group row per instruction and the executor pushes one +/// record per instruction of the same chip, so `real_rows` is each vector's +/// final length exactly — which is what lets the executor allocate once instead +/// of copying every record vector ~log2(rows) times per proof. +/// +/// Both halves are asserted: the length identity (the contract), and +/// `capacity == real_rows` (that the allocation was actually made up front — +/// push doubling would overshoot on these deliberately non-power-of-two counts). +#[test] +fn the_records_are_sized_from_the_census() { + let (program, arenas) = multi_chip_program(); + let exec = run(&program, &arenas); + let g = &program.groups; + let r = &exec.records; + + // Non-degenerate: several chips have to actually be exercised, or the test + // would pass on a program that allocates nothing. + assert_eq!(r.balu.len(), 3, "add, sub, mul"); + assert_eq!(r.lanes.len(), 2, "unpack, pack"); + assert_eq!(r.public.len(), 5); + assert_eq!(r.hash.len(), 1); + + let counts: [(&str, usize, usize, usize); 10] = [ + ("balu", r.balu.len(), r.balu.capacity(), g.balu.real_rows), + ("xalu", r.xalu.len(), r.xalu.capacity(), g.xalu.real_rows), + ( + "select", + r.select.len(), + r.select.capacity(), + g.select.real_rows, + ), + ( + "bitdec", + r.bitdec.len(), + r.bitdec.capacity(), + g.bitdec.real_rows, + ), + ("hash", r.hash.len(), r.hash.capacity(), g.hash.real_rows), + ( + "keccak", + r.keccak.len(), + r.keccak.capacity(), + g.keccak.real_rows, + ), + ( + "blake3", + r.blake3.len(), + r.blake3.capacity(), + g.blake3.real_rows, + ), + ( + "lanes", + r.lanes.len(), + r.lanes.capacity(), + g.lanes.real_rows, + ), + ("hint", r.hint.len(), r.hint.capacity(), g.hint.real_rows), + ( + "public", + r.public.len(), + r.public.capacity(), + g.public.real_rows, + ), + ]; + for (chip, len, cap, real_rows) in counts { + assert_eq!(len, real_rows, "{chip}: record count vs emitted rows"); + assert_eq!(cap, real_rows, "{chip}: sized up front, not grown"); + } + assert_eq!( + r.num_consts, g.const_.real_rows, + "const: record count vs emitted rows" + ); +} + +#[test] +fn validator_rejects_cycle() { + let mut program = small_valid_program(); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Mul, + out, + a, + .. + } = i + { + *a = *out; // a := f(a) — balances for any value; must die here + } + } + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::CyclicRead { .. } + )); +} + +#[test] +fn validator_rejects_wrong_mult() { + let mut program = small_valid_program(); + for i in &mut program.instrs { + if let Instr::Const { mult, .. } = i { + *mult += 1; + break; + } + } + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::MultMismatch { .. } + )); +} + +#[test] +fn validator_rejects_non_one_hot_selector() { + let mut program = small_valid_program(); + // Turn a second selector on in the first real BALU row. + program.groups.balu.set(0, layout::balu::SEL_SUB, FE::one()); + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::NonOneHotSelector { + chip: "LFM_BALU", + row: 0 + } + ); +} + +#[test] +fn validator_rejects_dirty_padding() { + let mut program = small_valid_program(); + let row = program.groups.balu.real_rows; // first padding row + program.groups.balu.set(row, layout::balu::MULT, fe(1)); + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::DirtyPadding { + chip: "LFM_BALU", + row + } + ); +} + +/// Check 9 at the level it operates: a multiplicity column of the COMMITTED +/// group, with the instruction list left honest. +/// +/// `p − 1` is what `−1` looks like as a canonical field element, and a +/// negative send is the one stray multiplicity the LogUp count argument cannot +/// catch: it cancels an honest write instead of adding an unmatched token. +#[test] +fn validator_rejects_negative_multiplicity() { + let mut program = small_valid_program(); + // Honest-path control: the program is otherwise admissible, so the + // rejection below is about the multiplicity and nothing else. + validate(&program).expect("the untampered program must pass admission"); + + program + .groups + .const_ + .set(0, layout::const_::MULT, fe(GOLDILOCKS_P - 1)); + assert!( + matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_CONST", + row: 0, + col: layout::const_::MULT, + .. + } + ), + "a field-negative multiplicity must fail admission" + ); +} + +/// The bound is a bound, not merely a sign test: a multiplicity far above any +/// read count the program can emit is rejected even though it is positive. +#[test] +fn validator_rejects_oversized_multiplicity() { + let mut program = small_valid_program(); + validate(&program).expect("the untampered program must pass admission"); + + program.groups.balu.set(0, layout::balu::MULT, fe(1 << 40)); + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_BALU", + row: 0, + .. + } + )); +} + +/// The forgery this pair of checks exists for: a `Compress` row whose two +/// *spare* output slots carry a `−1` / `+1` pair aimed at one address. +/// +/// `Instr::writes()` hides those slots for `Compress`, so checks 1 and 4 never +/// look at them — yet `emit_column_groups` copies them into the committed +/// group and `chips::hash` sends all three slots gated only by their own +/// `MULT`, with no mode factor. The negative send cancels the victim cell's +/// honest write and the positive one re-supplies it with the row's own +/// permutation output, so the token count is preserved exactly and the reader +/// observes a word nobody committed. This was executed end to end against the +/// real prover and verifier, and accepted, before these checks existed. +#[test] +fn validator_rejects_compress_ghost_slot_forgery() { + let konst = |out: u64, v: u64| Instr::Const { + out: Addr(out), + value: [fe(v), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }; + let source = LfmProgramSource { + instrs: vec![ + konst(0, 1), + konst(1, 2), + konst(2, 3), // the victim: a program constant the ghost pair replaces + Instr::Hash { + mode: HashMode::Compress, + ins: [Addr(0), Addr(1), Addr(0)], + outs: [Addr(3), Addr(2), Addr(2)], + mults: [0, GOLDILOCKS_P - 1, 1], + }, + Instr::Public { + addr: Addr(2), + index: 0, + }, + ], + num_addrs: 4, + read_counts: vec![1, 1, 1, 0], + arena_schema: Default::default(), + public_len: 1, + }; + let mut program = compile(source); + + // Leg 1: the point check on `instr.rs`'s placeholder convention. + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::CompressSlotNotPlaceholder { instr: 3 } + ); + + // Leg 2: and check 9 denies it on its own. Repair the INSTRUCTION list — + // the object checks 1–4 read — and leave the COMMITTED group hostile, + // which is precisely the divergence that made the forgery admissible. + let Instr::Hash { outs, mults, .. } = &mut program.instrs[3] else { + panic!("instruction 3 is the hash row"); + }; + *outs = [Addr(3), Addr(0), Addr(0)]; + *mults = [0, 0, 0]; + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_HASH", + row: 0, + col: layout::hash::MULT1, + .. + } + )); +} + +#[test] +fn validator_rejects_read_of_unwritten() { + let mut program = small_valid_program(); + let bogus = Addr(program.num_addrs - 1); // allocated range, but rewire below + // Extend the address space by one and point an operand at the unwritten slot. + program.num_addrs += 1; + let unwritten = Addr(program.num_addrs - 1); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Mul, + b, + .. + } = i + { + *b = unwritten; + } + } + let _ = bogus; + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::ReadOfUnwritten { addr: unwritten.0 } + ); +} + +// ---- compiler invariant panics (tripwires behind the validator) ---- + +#[test] +fn compiler_panics_on_double_assignment() { + let source = LfmProgramSource { + instrs: vec![ + Instr::Const { + out: Addr(0), + value: [FE::zero(), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }, + Instr::Const { + out: Addr(0), + value: [FE::one(), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }, + ], + num_addrs: 1, + read_counts: vec![0], + arena_schema: Default::default(), + public_len: 0, + }; + let result = std::panic::catch_unwind(|| compile(source)); + assert!(result.is_err()); +} + +#[test] +fn compiler_panics_on_undrained_read_counts() { + let mut read_counts = vec![0u64; 8]; + read_counts[7] = 1; // a read of an address nothing writes + let source = LfmProgramSource { + instrs: vec![Instr::Const { + out: Addr(0), + value: [FE::zero(), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }], + num_addrs: 8, + read_counts, + arena_schema: Default::default(), + public_len: 0, + }; + let result = std::panic::catch_unwind(|| compile(source)); + assert!(result.is_err()); +} + +// ---- misc structural ---- + +#[test] +fn const_pool_interns_and_counts_reads() { + let mut b = LfmBuilder::new(); + let x1 = b.felt_const(fe(42)); + let x2 = b.felt_const(fe(42)); // same cell + assert_eq!(x1.addr(), x2.addr()); + let s = b.add(x1, x2); // two reads of the shared cell + b.public(s.as_cell()); + let program = compile(b.finish()); + let mult = program + .instrs + .iter() + .find_map(|i| match i { + Instr::Const { out, mult, .. } if *out == x1.addr() => Some(*mult), + _ => None, + }) + .unwrap(); + assert_eq!(mult, 2); + run(&program, &[]); +} + +#[test] +fn canonical_helper_sanity() { + // p ≡ 0: the canonical map the executor's BitDec relies on. + assert_eq!(GoldilocksField::canonical(fe(GOLDILOCKS_P).value()), 0); + assert_eq!(GoldilocksField::canonical(fe(5).value()), 5); +} + +#[test] +fn digest_packing_round_trips() { + let w: LfmWord = core::array::from_fn(|i| fe(0xDEAD_0000 + i as u64)); + let packed = super::word::pack_digest(&w); + assert_eq!(super::word::unpack_digest(&packed), w); +} diff --git a/prover/src/lfm/trace.rs b/prover/src/lfm/trace.rs new file mode 100644 index 000000000..c6c895bf0 --- /dev/null +++ b/prover/src/lfm/trace.rs @@ -0,0 +1,784 @@ +//! LFM trace generation: instruction column group (preprocessed, leading) +//! plus value columns from the executor's records. Aux/LogUp columns are +//! entirely framework-built; heights equal each group's padded height, so the +//! prover's preprocessed-subset recommit matches the registry root exactly. + +use stark::trace::TraceTable; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField, zeroed_fe_vec}; + +use crate::tables::{bitwise, keccak_rc, keccak_rnd}; + +use super::blake3_chip; +use super::blake3_socket; +use super::chips::{balu, bitdec, const_, hash, hint, keccak, lanes, public, select, xalu}; +use super::compiler::{ColumnGroup, LfmProgram}; +use super::executor::LfmRecords; +use super::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use super::instr::{HashMode, Instr}; +use super::keccak_adapter::{self, KeccakAdapterOperation}; +use super::layout; +use super::merge::{MergeChip, MergePlan, MergedLayout, PartShape}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +pub struct LfmTraces { + pub const_: TraceTable, + pub balu: TraceTable, + pub xalu: TraceTable, + pub select: TraceTable, + pub bitdec: TraceTable, + pub hash: TraceTable, + /// `LFM_HASH` chunks 1.. — empty unless the program's policy split + /// the table (`chunking::HashChunking`); [`Self::hash`] is then chunk 0. + pub hash_tail: Vec>, + pub keccak: TraceTable, + /// One trace per `LFM_BLAKE3` chunk (see [`super::chunking::Blake3Chunking`]). + /// A `Vec` even when the policy is the default single table, so the prove and + /// verify paths have one shape to walk rather than two. + pub blake3: Vec>, + pub lanes: TraceTable, + pub hint: TraceTable, + pub public: TraceTable, + pub range: TraceTable, + /// The three production keccak-family tables, proved unchanged. They carry + /// no LFM instruction column group: `KECCAK_RND` has no preprocessed + /// columns at all, and the other two have fixed, program-independent ones. + /// + /// `KECCAK_RND` is one trace per chunk (see [`super::chunking`]); the + /// other two stay single shared instances whose multiplicities count the + /// lookups from *every* chunk — including `LFM_BLAKE3`'s, which `BITWISE` + /// receives whole however that chip's rows were split. + pub keccak_rnd: Vec>, + pub keccak_rc: TraceTable, + pub bitwise: TraceTable, + /// The merged tables (`LAMBDA_VM_ZF_LFM_MERGE`), one per group of + /// [`Self::merge_plan`], in its order: the group's chips side by side + /// ([`super::merge`]). A merged chip's own field above is then an empty + /// table. Both empty by default. + pub merged: Vec>, + /// Which chips each merged table holds. + pub merge_plan: MergePlan, +} + +impl LfmTraces { + /// This set under `plan`: the chips each group of `plan` names, taken out + /// of their own tables and laid side by side, row for row. A copy — the + /// production fill writes the merged tables directly + /// ([`build_traces_with_merge`]); this is for a set filled under another + /// plan than its artifacts' (`proof::prove_traces_under`): a set merged + /// under another plan is split back first. Nothing a proof commits to + /// depends on which way the set got there. + pub fn merge_to(&mut self, plan: &MergePlan) -> Result<(), String> { + if self.merge_plan == *plan { + return Ok(()); + } + self.split_merged(); + let mut merged = Vec::with_capacity(plan.len()); + for chips in plan { + let tables: Vec> = chips + .iter() + .map(|&c| { + let width = PartShape::of(c).width; + std::mem::replace(self.chip_mut(c), TraceTable::new_main(Vec::new(), width, 1)) + }) + .collect(); + merged.push(side_by_side(chips, &tables)); + } + self.merged = merged; + self.merge_plan = plan.clone(); + Ok(()) + } + + /// Every merged table split back into its chips' own tables. + fn split_merged(&mut self) { + let plan = std::mem::take(&mut self.merge_plan); + let merged = std::mem::take(&mut self.merged); + for (chips, table) in plan.iter().zip(merged) { + let layout = MergedLayout::of(chips); + let rows = table.num_rows(); + let (cells, _) = table.main_data_row_major(); + for part in &layout.parts { + let shape = PartShape::of(part.chip); + let mut data = zeroed_fe_vec(rows * shape.width); + for (row, out) in data.chunks_mut(shape.width.max(1)).enumerate() { + let src = &cells[row * layout.num_columns..(row + 1) * layout.num_columns]; + out[..shape.prep] + .copy_from_slice(&src[part.prep_at..part.prep_at + shape.prep]); + out[shape.prep..].copy_from_slice( + &src[part.main_at..part.main_at + shape.width - shape.prep], + ); + } + *self.chip_mut(part.chip) = TraceTable::new_main(data, shape.width, 1); + } + } + } + + fn chip_mut(&mut self, chip: MergeChip) -> &mut TraceTable { + match chip { + MergeChip::Balu => &mut self.balu, + MergeChip::Select => &mut self.select, + MergeChip::Lanes => &mut self.lanes, + MergeChip::Hint => &mut self.hint, + } + } +} + +/// One chip's row filler, as both walks run it: a chip's own table or its part +/// of a merged one. +type RowFill<'a> = &'a (dyn Fn(usize, &mut [FE]) + Sync); + +/// The widest merge candidate (`LFM_SELECT`, 25 columns): the scratch row a +/// merged fill builds one part's row in. +const MAX_PART_WIDTH: usize = 32; + +/// `tables` (the chips of `chips`, in order, at one height) side by side in the +/// merged layout ([`MergedLayout`]). [`LfmTraces::merge_to`]'s copy. +fn side_by_side(chips: &[MergeChip], tables: &[TraceTable]) -> TraceTable { + let layout = MergedLayout::of(chips); + let rows = tables.first().map_or(0, TraceTable::num_rows); + let width = layout.num_columns; + let mut data = zeroed_fe_vec(rows * width); + let body = |row: usize, out: &mut [FE]| { + for (part, table) in layout.parts.iter().zip(tables) { + let shape = PartShape::of(part.chip); + let (cells, _) = table.main_data_row_major(); + let cells = &cells[row * shape.width..(row + 1) * shape.width]; + out[part.prep_at..part.prep_at + shape.prep].copy_from_slice(&cells[..shape.prep]); + out[part.main_at..part.main_at + shape.width - shape.prep] + .copy_from_slice(&cells[shape.prep..]); + } + }; + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + data.par_chunks_mut(width.max(1)) + .enumerate() + .for_each(|(row, out)| body(row, out)); + } + #[cfg(not(feature = "parallel"))] + for (row, out) in data.chunks_mut(width.max(1)).enumerate() { + body(row, out); + } + TraceTable::new_main(data, width, 1) +} + +/// A merged table filled directly: each row writes each part's prefix and its +/// `fill` into a scratch row of the part's width — exactly [`chip_row`]'s row — +/// then places the two halves where the merged layout puts them. So the merged +/// trace is, cell for cell, the parts' [`chip_trace`] rows side by side, with no +/// second copy of them. +fn merged_trace( + walk: Walk, + chips: &[MergeChip], + groups: &[&ColumnGroup], + fills: &[RowFill<'_>], +) -> TraceTable { + let layout = MergedLayout::of(chips); + let rows = groups.first().map_or(0, |g| g.padded_rows); + let width = layout.num_columns; + let mut data = zeroed_fe_vec(rows * width); + let body = |row: usize, out: &mut [FE]| { + for ((part, group), fill) in layout.parts.iter().zip(groups).zip(fills) { + let shape = PartShape::of(part.chip); + debug_assert!(shape.width <= MAX_PART_WIDTH); + let mut scratch = [FE::zero(); MAX_PART_WIDTH]; + let cells = &mut scratch[..shape.width]; + chip_row(group, fill, row, cells); + out[part.prep_at..part.prep_at + shape.prep].copy_from_slice(&cells[..shape.prep]); + out[part.main_at..part.main_at + shape.width - shape.prep] + .copy_from_slice(&cells[shape.prep..]); + } + }; + match walk { + Walk::Parallel => { + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + data.par_chunks_mut(width.max(1)) + .enumerate() + .for_each(|(row, out)| body(row, out)); + } + #[cfg(not(feature = "parallel"))] + for (row, out) in data.chunks_mut(width.max(1)).enumerate() { + body(row, out); + } + } + #[cfg(test)] + Walk::SerialReference => { + for (row, out) in data.chunks_mut(width.max(1)).enumerate() { + body(row, out); + } + } + } + TraceTable::new_main(data, width, 1) +} + +/// The `LFM_RANGE` fixed table's column group (program-independent). +pub fn range_group() -> ColumnGroup { + ColumnGroup { + width: layout::range::PREP_WIDTH, + real_rows: layout::range::NUM_ROWS, + padded_rows: layout::range::NUM_ROWS, + data: (0..layout::range::NUM_ROWS as u64).map(FE::from).collect(), + } +} + +/// Which row walk [`chip_trace`] runs. +/// +/// Production is always [`Walk::Parallel`], and it is the only variant a +/// production build has: `SerialReference` is the two-loop walk this module had +/// before the rows were split across threads, kept verbatim under `cfg(test)` so +/// the identity gate in `trace_identity_tests` compares against the code the +/// parallel walk replaced rather than a second spelling of the walk under test. +/// Gating it means no shipped build can reach the old walk by accident. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum Walk { + Parallel, + #[cfg(test)] + SerialReference, +} + +/// Builds one chip's trace: copy the (already padded) group into the leading +/// columns, then let `fill` write the value columns of each real row. +/// +/// The walk is over rows, and rows are independent, so it runs across the +/// prover's rayon pool. Two facts make the split byte-identical to the serial +/// one, and both are load-bearing: +/// +/// * A row's body is its prefix copy *then* its `fill` — the order the two +/// serial loops gave any single row. The hash arm reads back what the copy +/// wrote: `blake3_socket`'s filler takes its domain off the row's own mode +/// columns, and every `fill_*_witness` reads the `IN`/`S` cells. +/// * No chip's body touches another row. Each closure reads `records[row]` and +/// writes only the slice it is handed, so the chunks' writes are disjoint. +/// `fill` is `Fn + Sync`, not `FnMut`, so a closure that wants to carry state +/// from row to row fails to compile here instead of racing on the box. +pub(super) fn chip_trace( + walk: Walk, + group: &ColumnGroup, + num_columns: usize, + fill: impl Fn(usize, &mut [FE]) + Sync, +) -> TraceTable { + let rows = group.padded_rows; + // The serial walk got this for free: its second loop ran to `real_rows` and + // would have panicked slicing past the buffer. The parallel one walks the + // PADDED rows and asks each whether it is real, so an over-long real count + // would quietly fill nothing instead. Said once, per chip, rather than lost. + assert!( + group.real_rows <= rows, + "a column group's real rows ({}) cannot outrun its padded rows ({rows})", + group.real_rows + ); + // calloc rather than `vec![FE::zero(); n]`'s element-wise sweep. A chip + // trace runs to gigabytes — `LFM_HASH` at 2^20 rows is 2.6 GiB — and the + // sweep writes every one of those bytes before the fill overwrites most of + // them. The calloc path hands back demand-zeroed pages instead, so the only + // cost is the faults the fill would take anyway. `zeroed_fe_vec` carries the + // soundness argument (`FE` is `repr(transparent)` over `u64` and Goldilocks + // has no Montgomery form) and its own guard test. + let mut data = zeroed_fe_vec(rows * num_columns); + match walk { + Walk::Parallel => fill_rows(&mut data, group, num_columns, &fill), + #[cfg(test)] + Walk::SerialReference => fill_rows_serial_reference(&mut data, group, num_columns, fill), + } + TraceTable::new_main(data, num_columns, 1) +} + +/// One row of [`chip_trace`]'s walk: the preprocessed prefix, then the value +/// columns if this is a real row. `row` indexes the whole table; `out` is that +/// row's `num_columns` cells and nothing else. +#[inline] +fn chip_row( + group: &ColumnGroup, + fill: &(impl Fn(usize, &mut [FE]) + ?Sized), + row: usize, + out: &mut [FE], +) { + out[..group.width].copy_from_slice(&group.data[row * group.width..(row + 1) * group.width]); + if row < group.real_rows { + fill(row, out); + } +} + +/// [`Walk::Parallel`]: the rows, chunked across the pool the rest of the prover +/// already uses. No second pool — `rayon`'s global one, entered through the +/// crate's `parallel` feature, exactly as `tables::bitwise` and +/// `tables::trace_builder` enter it. +#[cfg(feature = "parallel")] +fn fill_rows( + data: &mut [FE], + group: &ColumnGroup, + num_columns: usize, + fill: &(impl Fn(usize, &mut [FE]) + Sync), +) { + use rayon::prelude::*; + + data.par_chunks_mut(num_columns) + .enumerate() + .for_each(|(row, out)| chip_row(group, fill, row, out)); +} + +/// [`Walk::Parallel`] without the feature: the same row bodies in order. This is +/// the walk, not the reference — `--no-default-features` builds still have to +/// produce the trace, and they produce the same one. +#[cfg(not(feature = "parallel"))] +fn fill_rows( + data: &mut [FE], + group: &ColumnGroup, + num_columns: usize, + fill: &(impl Fn(usize, &mut [FE]) + Sync), +) { + for (row, out) in data.chunks_mut(num_columns).enumerate() { + chip_row(group, fill, row, out); + } +} + +/// [`Walk::SerialReference`]: the pre-parallel body, verbatim — the whole +/// table's prefixes in one loop, then the real rows' values in a second. +/// +/// It deliberately does not go through [`chip_row`]. The gate's whole value is +/// that the two sides were written independently, so a bug introduced into the +/// shared row body would have to be introduced twice to go unnoticed. +#[cfg(test)] +fn fill_rows_serial_reference( + data: &mut [FE], + group: &ColumnGroup, + num_columns: usize, + mut fill: impl FnMut(usize, &mut [FE]), +) { + let rows = group.padded_rows; + for row in 0..rows { + data[row * num_columns..row * num_columns + group.width] + .copy_from_slice(&group.data[row * group.width..(row + 1) * group.width]); + } + for row in 0..group.real_rows { + fill(row, &mut data[row * num_columns..(row + 1) * num_columns]); + } +} + +/// Writes the RPX round witness into a hash row whose `IN`/`S`/`OUT` columns are +/// already filled. +/// +/// The permutation input is read back out of the row's own `IN`/`S` columns — +/// the exact cells round 0's constraints read — rather than from the executor +/// record, so the witness cannot describe a different input than the one the +/// AIR constrains. +/// +/// The one thing that differs is that RPX's rounds are not alike — the FB rounds +/// write two ladders per lane, the E rounds two extension intermediates per +/// triple, and the M round writes only `OUT` — so this walks the schedule rather +/// than a uniform loop. +pub(super) fn fill_rpx_witness(out: &mut [FE]) { + use super::chips::hash::rpx_cols as rc; + use super::rpx::{ + EXT_DEGREE, EXT_ELEMENTS, NUM_ROUNDS, is_fb_round, is_final_round, kind_index, + permutation_witness, + }; + + let state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| { + if i < 8 { + out[hash::cols::IN0 + i] + } else { + out[hash::cols::S8 + (i - 8)] + } + }); + let w = permutation_witness(state); + for r in 0..NUM_ROUNDS { + if is_final_round(r) { + continue; // the M round's output IS `OUT`, already written. + } + if is_fb_round(r) { + let round = &w.fb[kind_index(r)]; + for lane in 0..HASH_STATE_FELTS { + out[rc::u2(r, lane)] = round.u2[lane]; + out[rc::u3(r, lane)] = round.u3[lane]; + out[rc::y2(r, lane)] = round.y2[lane]; + out[rc::y3(r, lane)] = round.y3[lane]; + out[rc::out(r, lane)] = round.y[lane]; + } + } else { + let round = &w.ext[kind_index(r)]; + for e in 0..EXT_ELEMENTS { + for k in 0..EXT_DEGREE { + out[rc::t2(r, e, k)] = round.t2[e * EXT_DEGREE + k]; + out[rc::t3(r, e, k)] = round.t3[e * EXT_DEGREE + k]; + } + } + for j in 0..HASH_STATE_FELTS { + out[rc::out(r, j)] = round.out[j]; + } + } + } + debug_assert_eq!( + &out[hash::cols::OUT0..hash::cols::OUT0 + HASH_STATE_FELTS], + w.final_out.as_slice(), + "the M round's output is the OUT columns the executor already wrote" + ); +} + +/// Writes one `LFM_HASH` row: the record's state in and out, the capacity cells +/// its MODE calls for, and then the hasher's round witness. +/// +/// A free function rather than a closure inside [`build_traces_walked`] so the +/// scale probe in `trace_identity_tests` times the filler production runs +/// instead of a copy of it. Row-local, like every other chip's fill: it reads +/// one record and writes only `out`. +pub(super) fn fill_hash_row( + hasher: HasherKind, + r: &super::executor::HashRow, + mode: HashMode, + out: &mut [FE], +) { + out[hash::cols::IN0..hash::cols::IN0 + 12].copy_from_slice(&r.ins); + for k in 0..4 { + // S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i, materialized. + // Every mode but the permutation takes the IV. The capacity a row takes + // is its MODE's, not one value for the whole trace: RPO separates its + // socket domains through the capacity, so a transcript row and a parent + // row carry different constants. `mode_iv` is the one rule the executor + // reads too. + out[hash::cols::S8 + k] = if mode == HashMode::Permute { + r.ins[8 + k] + } else { + hasher.mode_iv(mode)[k] + }; + } + out[hash::cols::OUT0..hash::cols::OUT0 + 12].copy_from_slice(&r.outs); + match hasher { + HasherKind::Test => {} + // The domain is read off the row's own mode columns, which `chip_trace` + // populated before calling this — the same discipline + // `fill_poseidon_witness` follows for its input. + HasherKind::Blake3 => blake3_socket::fill_socket_witness(out), + HasherKind::Rpx => fill_rpx_witness(out), + // No twelve-felt row exists under the width-16 socket: the executor's + // `admits` refuses one, so its program's `hash` records are empty and + // `LFM_HASH` fills from `hash16` instead (`build_traces_walked`). + HasherKind::Poseidon1W16 => {} + } +} + +pub fn build_traces(program: &LfmProgram, records: &LfmRecords) -> LfmTraces { + build_traces_with_hasher(program, records, HasherKind::default()) +} + +/// [`build_traces`] for a proof under `hasher`. +/// +/// `hasher` must be the one the executor ran (`proof::lfm_prove_with_hasher` +/// passes the same value to both) and the one the AIR set was built with: the +/// hash chip's width and witness columns are its layout's, and the constraints +/// bake its round constants. +pub fn build_traces_with_hasher( + program: &LfmProgram, + records: &LfmRecords, + hasher: HasherKind, +) -> LfmTraces { + build_traces_walked(program, records, hasher, Walk::Parallel) +} + +/// [`build_traces_with_hasher`] with the chips of each group of `plan` filled +/// straight into one merged table ([`super::merge`]), their own tables left +/// empty. An empty `plan` is [`build_traces_with_hasher`]. +pub fn build_traces_with_merge( + program: &LfmProgram, + records: &LfmRecords, + hasher: HasherKind, + plan: &MergePlan, +) -> LfmTraces { + build_traces_merged(program, records, hasher, Walk::Parallel, plan) +} + +/// [`build_traces_with_hasher`] with the row walk named. +/// +/// The walk is a parameter for exactly one reason: so `trace_identity_tests` can +/// build the whole `LfmTraces` both ways, through these closures and no others, +/// and compare the results cell for cell. Nothing outside this module chooses; +/// every production entry point above takes [`Walk::Parallel`]. +pub(super) fn build_traces_walked( + program: &LfmProgram, + records: &LfmRecords, + hasher: HasherKind, + walk: Walk, +) -> LfmTraces { + build_traces_merged(program, records, hasher, walk, &Vec::new()) +} + +/// [`build_traces_walked`] under the merge `plan` ([`build_traces_with_merge`]). +pub(super) fn build_traces_merged( + program: &LfmProgram, + records: &LfmRecords, + hasher: HasherKind, + walk: Walk, + plan: &MergePlan, +) -> LfmTraces { + let g = &program.groups; + + let hash_modes: Vec = program + .instrs + .iter() + .filter_map(|i| match i { + Instr::Hash { mode, .. } => Some(*mode), + _ => None, + }) + .collect(); + + // The keccak family's traces are driven by the executor's records; the tag + // is the row ordinal, exactly as the compiler emitted it into the + // preprocessed group (one rule, `layout::keccak::tag_for_row`, two callers). + // The family sees `perm_in` — post-XOR on absorb rows — not the state as + // read from memory. + let keccak_ops: Vec = records + .keccak + .iter() + .enumerate() + .map(|(row, r)| KeccakAdapterOperation { + tag: layout::keccak::tag_for_row(row), + input: r.perm_in, + }) + .collect(); + + // The round operations split across `KECCAK_RND` chunks; the chip has no + // row-to-row constraints, so a chunk is just a slice of the permutations + // (see `chunking`). The chunk *count* is program shape, so this uses the + // program's pinned policy rather than anything derived here. + let round_ops = keccak_adapter::round_operations(&keccak_ops); + let keccak_rnd_traces: Vec<_> = program + .chunking + .split(&round_ops) + .into_iter() + .map(keccak_rnd::generate_keccak_rnd_trace) + .collect(); + + // `LFM_BLAKE3` splits on the same principle, with one step `KECCAK_RND` does + // not need: the chip carries a preprocessed instruction group, so a chunk is + // its own slice of that group as well as its own slice of the records. Each + // chunk's group is materialized, copied into the trace and dropped, so peak + // residency is one chunk rather than a second whole group. + let blake3_traces: Vec> = (0..program.blake3_chunk_count()) + .map(|c| { + let group = program.blake3_chunk_group(c); + let base = program + .blake3_chunking + .chunk_range(g.blake3.real_rows, c) + .start; + chip_trace(walk, &group, blake3_chip::cols::NUM_COLUMNS, |row, out| { + blake3_chip::fill_blake3_witness(out, &records.blake3[base + row]); + }) + }) + .collect(); + + // KECCAK_RC and BITWISE are single shared tables: their multiplicities are + // totals over the whole proof, so they are fed the complete operation list + // regardless of how the round rows were chunked. + let mut keccak_rc_trace = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut keccak_rc_trace, keccak_ops.len()); + + let mut histogram = bitwise::BitwiseHistogram::new(); + histogram.add_ops(&keccak_adapter::bitwise_ops_for(&keccak_ops)); + // A KECCAK_RND padding row sends the round's byte checks as `ARE_BYTES[0, 0]` + // (`keccak_rnd::ARE_BYTES_PER_ROW` a row): each chunk's padding rows. A + // program without keccak ops has one empty chunk, which counts none: the + // chip is not instantiated then ([`super::airs::ChipSet::keccak`]). + for ops in program.chunking.split(&round_ops) { + histogram.bump_n( + bitwise::BitwiseOperation::byte_op(bitwise::BitwiseOperationType::AreBytes, 0, 0), + keccak_rnd::padding_are_bytes(ops.len()), + ); + } + // `LFM_BLAKE3` is a BITWISE consumer on every row — ~1,248 lookups per + // compression. Unconditional, unlike the socket's below: the chip is a + // member of the fixed set, so its lookups are a property of the program, + // not of the hash choice. + histogram.add_ops(&blake3_chip::bitwise_ops_for(&records.blake3)); + // Absorb rows additionally send one BYTE_ALU[XOR] lookup per rate byte. + histogram.add_ops(&keccak_adapter::absorb_bitwise_ops(&records.keccak)); + // Under BLAKE3 the hash chip is a BITWISE consumer too — over a thousand + // lookups per compression. Every other hasher sends none, so this is the + // one place the shared table's multiplicities depend on the hash choice. + if hasher == HasherKind::Blake3 { + let rows: Vec<([u32; blake3_socket::cols::NUM_LANES], u32)> = records + .hash + .iter() + .zip(&hash_modes) + .map(|(r, mode)| { + let cell = + |k: usize| -> super::word::LfmWord { core::array::from_fn(|i| r.ins[k + i]) }; + // The message lanes, read the way this row's MODE reads them: + // digest cells throughout, or an accumulator cell followed by + // four felts split into halves. A leaf row sends lookups over + // its halves, so the histogram has to split the row exactly the + // way the witness filler does — which is why both come through + // `lanes_from_cells` instead of each carrying its own split. + let lanes = blake3_socket::lanes_from_cells( + *mode == HashMode::Leaf, + &[cell(0), cell(4), cell(8)], + ); + ( + lanes, + // The row's DOMAIN, not a fixed tag: the lookups a row sends + // are values downstream of the tag word, so a transcript row and a + // compress row over the same cells send different bytes. + blake3_socket::tag_for_mode(*mode) + .expect("BLAKE3 admits no permute row (its AIR pins MODE_P = 0)"), + ) + }) + .collect(); + histogram.add_ops(&blake3_socket::bitwise_ops_for(&rows)); + } + // A program whose mask drops BITWISE skips the `2^20`-row fill, and an + // empty placeholder stands in the struct. The histogram above is empty in + // exactly that case — the mask drops the table only when no instantiated + // chip sends it a lookup — so the table it stands for is all zero, which is + // what `LfmAirs::air_trace_pairs` pairs if the artifacts keep BITWISE. + let bitwise_trace = if super::airs::ChipSet::for_program_with_hasher(program, hasher).bitwise { + let mut trace = bitwise::generate_bitwise_trace(); + histogram.fill_multiplicities(&mut trace); + trace + } else { + TraceTable::new_main(Vec::new(), bitwise::cols::NUM_COLUMNS, 1) + }; + + // `LFM_HASH` is filled per chunk, each from its own slice of the + // group and of the records. An unsplit program fills the whole group in + // place, as before; a split one materializes one chunk group at a time. + let hash_count = program.hash_chunk_count(); + let hash_trace = |c: usize| { + let chunk = (hash_count > 1).then(|| program.hash_chunk_group(c)); + let base = program.hash_chunking.chunk_range(g.hash.real_rows, c).start; + let group = chunk.as_ref().unwrap_or(&g.hash); + #[cfg_attr(not(feature = "cuda"), allow(unused_mut))] + let mut trace = chip_trace(walk, group, hash::num_columns(hasher), |row, out| { + if program.hash16() { + super::p1w16_socket::fill_row( + super::p1w16_socket::SOCKET_FORM, + records.hash16[base + row].ins, + out, + ); + } else { + fill_hash_row( + hasher, + &records.hash[base + row], + hash_modes[base + row], + out, + ) + } + }); + // The socket's witness is a function of each row's inputs: the card + // rebuilds it from the prefix and the inputs instead of receiving it. + #[cfg(feature = "cuda")] + if program.hash16() && super::p1w16_socket::lean_upload_enabled() { + trace.set_main_dev_producer(super::p1w16_socket::lean_upload_producer( + super::p1w16_socket::SOCKET_FORM, + group.real_rows, + )); + } + trace + }; + + // The four chips a merge may take, as fillers both walks can run: a chip's + // own `chip_trace`, or its part of a merged table. + let balu_fill = |row: usize, out: &mut [FE]| { + let r = &records.balu[row]; + out[balu::cols::A] = r.a; + out[balu::cols::B] = r.b; + out[balu::cols::C] = r.c; + out[balu::cols::OUT] = r.out; + }; + let select_fill = |row: usize, out: &mut [FE]| { + let r = &records.select[row]; + out[select::cols::BIT] = r.bit; + out[select::cols::INL0..select::cols::INL0 + 4].copy_from_slice(&r.in_l); + out[select::cols::INR0..select::cols::INR0 + 4].copy_from_slice(&r.in_r); + out[select::cols::OUTL0..select::cols::OUTL0 + 4].copy_from_slice(&r.out_l); + out[select::cols::OUTR0..select::cols::OUTR0 + 4].copy_from_slice(&r.out_r); + }; + let lanes_fill = |row: usize, out: &mut [FE]| { + out[lanes::cols::V0..lanes::cols::V0 + 4].copy_from_slice(&records.lanes[row]); + }; + let hint_fill = |row: usize, out: &mut [FE]| { + out[hint::cols::V0..hint::cols::V0 + 4].copy_from_slice(&records.hint[row]); + }; + let fill_of = |chip: MergeChip| -> RowFill<'_> { + match chip { + MergeChip::Balu => &balu_fill, + MergeChip::Select => &select_fill, + MergeChip::Lanes => &lanes_fill, + MergeChip::Hint => &hint_fill, + } + }; + let merged: Vec> = plan + .iter() + .map(|chips| { + let groups: Vec<&ColumnGroup> = chips + .iter() + .map(|&c| super::merge::group_of_program(program, c)) + .collect(); + let fills: Vec> = chips.iter().map(|&c| fill_of(c)).collect(); + merged_trace(walk, chips, &groups, &fills) + }) + .collect(); + // A merged chip's own table: empty, its rows are in its merged table. + let own = |chip: MergeChip, group: &ColumnGroup| -> TraceTable { + let width = PartShape::of(chip).width; + if super::merge::group_of(plan, chip.slot()).is_some() { + TraceTable::new_main(Vec::new(), width, 1) + } else { + chip_trace(walk, group, width, fill_of(chip)) + } + }; + + LfmTraces { + const_: chip_trace(walk, &g.const_, const_::cols::NUM_COLUMNS, |_, _| {}), + balu: own(MergeChip::Balu, &g.balu), + xalu: chip_trace(walk, &g.xalu, xalu::cols::NUM_COLUMNS, |row, out| { + let r = &records.xalu[row]; + out[xalu::cols::A0..xalu::cols::A0 + 3].copy_from_slice(&r.a); + out[xalu::cols::B0..xalu::cols::B0 + 3].copy_from_slice(&r.b); + out[xalu::cols::C0..xalu::cols::C0 + 3].copy_from_slice(&r.c); + out[xalu::cols::OUT0..xalu::cols::OUT0 + 3].copy_from_slice(&r.out); + }), + select: own(MergeChip::Select, &g.select), + bitdec: chip_trace(walk, &g.bitdec, bitdec::cols::NUM_COLUMNS, |row, out| { + let r = &records.bitdec[row]; + out[bitdec::cols::BITS0..bitdec::cols::BITS0 + 64].copy_from_slice(&r.bits); + out[bitdec::cols::Z] = r.z; + out[bitdec::cols::GINV] = r.ginv; + }), + hash: hash_trace(0), + hash_tail: (1..hash_count).map(hash_trace).collect(), + keccak: chip_trace(walk, &g.keccak, keccak::cols::NUM_COLUMNS, |row, out| { + let r = &records.keccak[row]; + for lane in 0..25 { + for b in 0..8 { + let byte = |v: u64| FE::from(u64::from((v >> (8 * b)) as u8)); + out[keccak::cols::state_byte(lane, b)] = byte(r.state[lane]); + out[keccak::cols::perm_in_byte(lane, b)] = byte(r.perm_in[lane]); + out[keccak::cols::out_byte(lane, b)] = byte(r.output[lane]); + } + } + for (k, &v) in r.block.iter().enumerate() { + out[keccak::cols::BLOCK + k] = FE::from(u64::from(v)); + } + }), + blake3: blake3_traces, + lanes: own(MergeChip::Lanes, &g.lanes), + hint: own(MergeChip::Hint, &g.hint), + public: chip_trace(walk, &g.public, public::cols::NUM_COLUMNS, |row, out| { + out[public::cols::V0..public::cols::V0 + 4].copy_from_slice(&records.public[row]); + }), + range: chip_trace( + walk, + &range_group(), + super::chips::range::cols::NUM_COLUMNS, + |_, _| {}, + ), + keccak_rnd: keccak_rnd_traces, + keccak_rc: keccak_rc_trace, + bitwise: bitwise_trace, + merged, + merge_plan: plan.clone(), + } +} diff --git a/prover/src/lfm/trace_identity_tests.rs b/prover/src/lfm/trace_identity_tests.rs new file mode 100644 index 000000000..b9d635dfb --- /dev/null +++ b/prover/src/lfm/trace_identity_tests.rs @@ -0,0 +1,431 @@ +//! The gate on `trace`'s row walk: a chip's trace is a pure function of the +//! records, so **which** walk fills it may not change one cell of it. +//! +//! The two sides are [`Walk::Parallel`] — the rows chunked across rayon's pool, +//! what production runs — and [`Walk::SerialReference`], the two plain row loops +//! `trace.rs` had before. They are driven through `build_traces_walked`, so both +//! sides go through the *production* fill closures and no others: a test that +//! re-spelled the closures would gate its own copy instead of the machine. +//! +//! ## What is checked +//! +//! 1. **Byte identity**, cell for cell, over every table in `LfmTraces` — the +//! chip traces the walk fills, and also the keccak-family ones it does not, +//! since a regression there would be just as fatal and costs nothing to see. +//! 2. **Coverage**: the cases must, between them, give every chip with a +//! non-empty fill at least one real row. Without this the identity assertion +//! passes vacuously on a chip nothing exercised. +//! 3. **The census** — total cells and instruction count — computed on each side +//! and asserted equal, then printed. +//! +//! ## What is NOT checked +//! +//! Proof bytes. Grinding picks a nonce that is not reproducible run to run +//! (`grinding-nonce-nondeterminism`), so comparing proofs would gate the wrong +//! thing; the trace is the object the walk produces and the trace is what this +//! compares. +//! +//! Under `--no-default-features` both walks are serial (`fill_rows` has a +//! non-rayon twin, since the crate still has to build there), so the gate is +//! true by construction on that arm and says nothing. It is the default build it +//! is written for. +//! +//! The scale probe at the bottom is a measurement, not a gate: it drives the +//! `LFM_HASH` filler at a production-shaped height so the walk's speedup has a +//! number attached. It asserts identity too, because that is free. + +use std::time::Instant; + +use stark::trace::TraceTable; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; + +use super::chips::hash; +use super::compiler::{ColumnGroup, LfmProgram}; +use super::executor::{HashRow, execute}; +use super::hash::HasherKind; +use super::instr::{HashMode, Instr}; +use super::keccak_host::pack_stream; +use super::trace::{LfmTraces, Walk, build_traces_walked, chip_trace, fill_hash_row}; +use super::word::{LfmWord, base_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// One thing to fill traces for: a compiled program, the arenas it reads, and +/// the hasher both the executor and the trace builder are handed. +/// +/// Visible to the rest of `lfm` so [`super::exec_identity_tests`] gates the +/// executor over exactly these cases rather than over a second, drifting copy of +/// them: the two gates are consecutive stages of one pipeline, and a case list +/// that stopped covering a chip would otherwise go quiet on one of them only. +pub(super) struct Case { + pub(super) name: &'static str, + pub(super) program: LfmProgram, + pub(super) arenas: Vec>, + pub(super) hasher: HasherKind, +} + +/// The message the sponge cases run over: byte `i` is `37i + 11`, the generator +/// `crypto`'s own chain KATs use. +fn message(len: usize) -> Vec { + (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect() +} + +fn sponge_arenas(msg: &[u8]) -> Vec> { + vec![pack_stream(msg).into_iter().map(base_word).collect()] +} + +fn fri_arenas(inner: &super::fixture::FriToyProof) -> Vec> { + vec![inner.commitments.clone(), inner.openings.clone()] +} + +/// The cases, chosen for coverage rather than for size. +/// +/// `FriToyV0` is a real verification program over real FRI data and is what +/// gives the value chips their rows; it appears twice because the `LFM_HASH` +/// fill DISPATCHES on the hasher, so RPX (the production pin, and the most +/// expensive filler — twelve lanes of `x^{1/7}` per round) and BLAKE3 (the +/// socket filler, the one that reads its domain back off the row) are two +/// different bodies under test. The two sponge programs are the smallest things +/// that give `LFM_BLAKE3` and `LFM_KECCAK` rows. +pub(super) fn cases() -> Vec { + let msg = message(202); + vec![ + Case { + name: "FriToyV0 @ RPX", + program: super::programs::fri_toy_program(), + arenas: fri_arenas(&super::fixture::fixture_prove_with_hasher(HasherKind::Rpx)), + hasher: HasherKind::Rpx, + }, + Case { + name: "FriToyV0 @ BLAKE3", + program: super::programs::fri_toy_program(), + arenas: fri_arenas(&super::fixture::fixture_prove_with_hasher( + HasherKind::Blake3, + )), + hasher: HasherKind::Blake3, + }, + Case { + name: "Blake3Chain(202)", + program: super::programs::blake3_sponge_program(202), + arenas: sponge_arenas(&msg), + hasher: HasherKind::Test, + }, + Case { + name: "Keccak256(202)", + program: super::programs::keccak_sponge_program(202), + arenas: sponge_arenas(&msg), + hasher: HasherKind::Test, + }, + ] +} + +/// Every table in an `LfmTraces`, paired with the chip's name, in one flat list +/// so the comparison walks both sides in lockstep and can say which chip moved. +fn tables(t: &LfmTraces) -> Vec<(String, &TraceTable)> { + let mut out: Vec<(String, &TraceTable)> = vec![ + ("LFM_CONST".into(), &t.const_), + ("LFM_BALU".into(), &t.balu), + ("LFM_XALU".into(), &t.xalu), + ("LFM_SELECT".into(), &t.select), + ("LFM_BITDEC".into(), &t.bitdec), + ("LFM_HASH".into(), &t.hash), + ("LFM_KECCAK".into(), &t.keccak), + ("LFM_LANES".into(), &t.lanes), + ("LFM_HINT".into(), &t.hint), + ("LFM_PUBLIC".into(), &t.public), + ("LFM_RANGE".into(), &t.range), + ("KECCAK_RC".into(), &t.keccak_rc), + ("BITWISE".into(), &t.bitwise), + ]; + for (c, tr) in t.blake3.iter().enumerate() { + out.push((format!("LFM_BLAKE3[{c}]"), tr)); + } + for (c, tr) in t.keccak_rnd.iter().enumerate() { + out.push((format!("KECCAK_RND[{c}]"), tr)); + } + out +} + +/// Total main-trace cells across every table — the census figure. +fn cells(t: &LfmTraces) -> u64 { + tables(t) + .iter() + .map(|(_, tr)| tr.main_table.row_major_data().len() as u64) + .sum() +} + +/// The share of [`cells`] the row walk actually fills. +/// +/// The three keccak-family tables do not go through `chip_trace` — `BITWISE` +/// alone is a fixed 2^20 rows whatever the program is — so at fixture scale they +/// are most of the census and all of the wall. Printing the split is what keeps +/// the timing column below from being read as a fill measurement; the fill +/// measurement is the scale probe. +fn walked_cells(t: &LfmTraces) -> u64 { + tables(t) + .iter() + .filter(|(name, _)| name.starts_with("LFM_")) + .map(|(_, tr)| tr.main_table.row_major_data().len() as u64) + .sum() +} + +/// Asserts two trace sets are the same object, and says where they first differ +/// if they are not. +fn assert_identical(case: &str, a: &LfmTraces, b: &LfmTraces) { + let (ta, tb) = (tables(a), tables(b)); + assert_eq!( + ta.len(), + tb.len(), + "{case}: the two walks produced different table counts" + ); + for ((name, x), (other, y)) in ta.iter().zip(&tb) { + assert_eq!(name, other, "{case}: table order diverged"); + let width = x.num_main_columns; + assert_eq!( + width, y.num_main_columns, + "{case}/{name}: the column count moved" + ); + let (dx, dy) = (x.main_table.row_major_data(), y.main_table.row_major_data()); + assert_eq!(dx.len(), dy.len(), "{case}/{name}: the row count moved"); + if dx == dy { + continue; + } + let at = dx + .iter() + .zip(dy) + .position(|(p, q)| p != q) + .expect("the slices differ, so some index differs"); + panic!( + "{case}/{name}: the parallel walk and the serial reference disagree \ + at row {}, column {} — {:?} vs {:?}", + at / width, + at % width, + dx[at], + dy[at] + ); + } +} + +/// ★ The gate. Both walks, every case, cell for cell. +#[test] +fn the_parallel_row_walk_is_byte_identical_to_the_serial_reference() { + // Chips whose fill closure writes something. `LFM_CONST` and `LFM_RANGE` + // take `|_, _| {}` — their rows are preprocessed data only — so there is + // nothing for a case to "cover" on them. + let mut covered: Vec<&str> = Vec::new(); + + for case in cases() { + let exec = execute(&case.program, &case.arenas, &case.hasher) + .unwrap_or_else(|e| panic!("{}: the case must execute: {e:?}", case.name)); + + let t0 = Instant::now(); + let par = build_traces_walked(&case.program, &exec.records, case.hasher, Walk::Parallel); + let t_par = t0.elapsed(); + + let t1 = Instant::now(); + let seq = build_traces_walked( + &case.program, + &exec.records, + case.hasher, + Walk::SerialReference, + ); + let t_seq = t1.elapsed(); + + assert_identical(case.name, &par, &seq); + + // The census, both ways. Byte identity already implies this; it is + // asserted and printed separately because the census is the number the + // campaign quotes, and a silent change to it is the failure mode that + // would otherwise be found on a box rather than here. + let instrs = case.program.instrs.len(); + assert_eq!( + cells(&par), + cells(&seq), + "{}: the census moved with the walk", + case.name + ); + assert_eq!( + walked_cells(&par), + walked_cells(&seq), + "{}: the walked share of the census moved", + case.name + ); + println!( + "{:<18} {:>3}t cells {:>10} (walked {:>9}) instrs {:>5} \ + build: parallel {:>7.1} ms serial {:>7.1} ms", + case.name, + threads(), + cells(&par), + walked_cells(&par), + instrs, + t_par.as_secs_f64() * 1e3, + t_seq.as_secs_f64() * 1e3, + ); + + for (name, group) in named_groups(&case.program) { + if group.real_rows > 0 && !covered.contains(&name) { + covered.push(name); + } + } + } + + // Vacuity guard: without this the identity assertion above is satisfied by + // two empty traces. + for chip in FILLED_CHIPS { + assert!( + covered.contains(chip), + "no case gave {chip} a real row, so its fill closure was never run \ + on either walk — the gate is vacuous on it. Add a case that \ + exercises it rather than dropping it from this list." + ); + } +} + +/// The chips whose fill closure writes value columns. `LFM_CONST` and +/// `LFM_RANGE` are absent on purpose: their fills are `|_, _| {}`. +const FILLED_CHIPS: &[&str] = &[ + "LFM_BALU", + "LFM_XALU", + "LFM_SELECT", + "LFM_BITDEC", + "LFM_HASH", + "LFM_KECCAK", + "LFM_BLAKE3", + "LFM_LANES", + "LFM_HINT", + "LFM_PUBLIC", +]; + +fn named_groups(p: &LfmProgram) -> Vec<(&'static str, &ColumnGroup)> { + let g = &p.groups; + vec![ + ("LFM_CONST", &g.const_), + ("LFM_BALU", &g.balu), + ("LFM_XALU", &g.xalu), + ("LFM_SELECT", &g.select), + ("LFM_BITDEC", &g.bitdec), + ("LFM_HASH", &g.hash), + ("LFM_KECCAK", &g.keccak), + ("LFM_BLAKE3", &g.blake3), + ("LFM_LANES", &g.lanes), + ("LFM_HINT", &g.hint), + ("LFM_PUBLIC", &g.public), + ] +} + +#[cfg(feature = "parallel")] +fn threads() -> usize { + rayon::current_num_threads() +} + +#[cfg(not(feature = "parallel"))] +fn threads() -> usize { + 1 +} + +// ========================================================================= +// The scale probe +// ========================================================================= + +/// log2 of the row count the probe fills. 2^18 rows of RPX witness is 0.64 GiB +/// of trace, which a laptop can hold twice; the box arm can raise it to the +/// production 2^20 through the environment without a rebuild. +fn probe_log_rows() -> u32 { + std::env::var("LFM_TRACE_PROBE_LOG_ROWS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(18) +} + +/// `group` with its real rows repeated, cyclically, up to `rows`. +/// +/// Repetition is sound for a timing probe because the `LFM_HASH` fill reads the +/// row's own record and, under BLAKE3, the row's own mode columns — never a +/// neighbour, never a row ordinal. So a repeated row costs what the row it +/// repeats costs. The resulting group is not a provable trace and is never +/// proved; it exists to be filled. +fn repeated(group: &ColumnGroup, rows: usize) -> ColumnGroup { + let src = group.real_rows; + assert!(src > 0, "nothing to repeat"); + let mut data = Vec::with_capacity(rows * group.width); + for row in 0..rows { + let at = (row % src) * group.width; + data.extend_from_slice(&group.data[at..at + group.width]); + } + ColumnGroup { + width: group.width, + real_rows: rows, + padded_rows: rows, + data, + } +} + +fn hash_modes(p: &LfmProgram) -> Vec { + p.instrs + .iter() + .filter_map(|i| match i { + Instr::Hash { mode, .. } => Some(*mode), + _ => None, + }) + .collect() +} + +/// The measurement the handback quotes: `LFM_HASH` at a production-shaped +/// height, filled both ways, on this machine's pool. +/// +/// It drives `chip_trace` directly rather than a program, because no cheap +/// program has a million hash rows — the one that does is the recursion leaf, +/// and building one is a box job. The filler is the production `fill_hash_row`, +/// so the work per row is the real work. +#[test] +fn the_row_walk_scales_on_the_hash_chip() { + let hasher = HasherKind::Rpx; + let program = super::programs::fri_toy_program(); + let inner = super::fixture::fixture_prove_with_hasher(hasher); + let exec = execute(&program, &fri_arenas(&inner), &hasher).expect("the fixture must execute"); + + let rows = 1usize << probe_log_rows(); + let group = repeated(&program.groups.hash, rows); + let modes = hash_modes(&program); + let src = program.groups.hash.real_rows; + assert_eq!( + modes.len(), + src, + "one mode per hash row, or the repetition would read the wrong one" + ); + let records: &Vec = &exec.records.hash; + let num_columns = hash::num_columns(hasher); + + let fill = |row: usize, out: &mut [FE]| { + let at = row % src; + fill_hash_row(hasher, &records[at], modes[at], out) + }; + + let t0 = Instant::now(); + let par = chip_trace(Walk::Parallel, &group, num_columns, fill); + let t_par = t0.elapsed(); + + let t1 = Instant::now(); + let seq = chip_trace(Walk::SerialReference, &group, num_columns, fill); + let t_seq = t1.elapsed(); + + assert_eq!( + par.main_table.row_major_data(), + seq.main_table.row_major_data(), + "the probe's two walks must agree too" + ); + + println!( + "LFM_HASH @ RPX {rows} rows x {num_columns} cols ({:.2} GiB) {} threads \ + parallel {:.3} s serial {:.3} s speedup {:.2}x", + (rows * num_columns * 8) as f64 / (1u64 << 30) as f64, + threads(), + t_par.as_secs_f64(), + t_seq.as_secs_f64(), + t_seq.as_secs_f64() / t_par.as_secs_f64(), + ); +} diff --git a/prover/src/lfm/transcript_diff_tests.rs b/prover/src/lfm/transcript_diff_tests.rs new file mode 100644 index 000000000..db41388dd --- /dev/null +++ b/prover/src/lfm/transcript_diff_tests.rs @@ -0,0 +1,252 @@ +//! ★★★ **THE TRANSCRIPT ABSORB DIFFERENTIAL** — the machine's +//! [`TranscriptReplay`] against the host's [`AlgebraicTranscript`], step by +//! step. +//! +//! The machine re-derives the host's Fiat–Shamir chain. If the two absorb +//! anything differently the states part, every later challenge is a different +//! value, and the first thing that NOTICES is whatever assert consumes a +//! challenge — in a wrap leg that is the grinding check, failing as a +//! `DivByZero` that names neither the absorb nor the step. +//! +//! ⚠ **Why this compares STATE AFTER EACH STEP.** Comparing only the final +//! challenge says THAT the chains disagree; it cannot say which absorb did it, +//! and the absorbs have different fixes — a length prefix, a felt's byte order, +//! a root's cell count, a payload grouping. So both sides expose their state +//! after every step and the assertion names the first step at which they part. +//! +//! ⚠ **The host side is driven through the HOST's own API**, never a +//! reimplementation of it: a root goes in as `append_bytes(&commitment)`, a +//! felt as the bytes `FieldElement::stream_bytes` produces (production's own +//! serialisation, the one `algebraic_commit::element_felts` also goes through), +//! and an extension element as `append_field_element`. What this file chooses +//! is the script, not the encodings. +//! +//! Absorbs only, deliberately. The spine's first squeezed challenge is the +//! shared LogUp `z`, so everything that can move it is an absorb; squeezes have +//! their own asymmetries (`sample` is defined on the algebraic arm rather than +//! mirrored, per its doc) and gating them means deciding what they should mean, +//! which is not this gate's job. + +use math::traits::AsBytes; + +use crypto::fiat_shamir::is_transcript::IsTranscript; + +use super::algebraic_commit::{AlgebraicHasher, RpxCommit, commitment_to_digest}; +use super::algebraic_transcript::AlgebraicTranscript; +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::edsl::WrapHash; +use super::executor::execute; +use super::transcript_replay::TranscriptReplay; +use super::word::{LfmWord, base_word, ext_word}; +use crate::tables::types::{FE, FEE}; + +macro_rules! for_each_tenant { + ($body:ident) => { + $body::("Rpx"); + }; +} + +/// A commitment whose bytes are distinctive in every position, so a regrouping +/// or a reversal is visible rather than accidentally symmetric. +fn probe_root() -> [u8; 32] { + core::array::from_fn(|i| (i as u8).wrapping_mul(37).wrapping_add(3)) +} + +/// Production's own serialisation of a base element — the bytes the host +/// transcript absorbs for a felt. Through `AsBytes::stream_bytes` rather than +/// spelled out, for the reason `algebraic_commit::element_felts` gives. +fn felt_bytes(v: FE) -> Vec { + let mut out = Vec::new(); + v.stream_bytes(&mut |bytes| out.extend_from_slice(bytes)); + out +} + +/// ★★★ The gate: every absorb the batched spine uses, machine against host, +/// compared after each step. +#[test] +fn the_machine_transcript_tracks_the_host_absorb_for_absorb() { + fn check(tenant: &str) { + let root = probe_root(); + let root_word = commitment_to_digest(&root); + let felt_v = FE::from(0x0123_4567_89ab_cdefu64); + let ext_v = FEE::new([FE::from(11u64), FE::from(22), FE::from(33)]); + + // ---- the host chain, driven through the host's own API ---- + let mut host = AlgebraicTranscript::new(H::KIND); + let mut want: Vec = Vec::new(); + let mut steps: Vec = Vec::new(); + fn record( + host: &AlgebraicTranscript, + want: &mut Vec, + steps: &mut Vec, + label: &str, + ) { + want.push(host.state_word()); + steps.push(label.to_string()); + } + + // `TranscriptReplay::new(seed)` absorbs the seed as its first append, + // and `AlgebraicTranscript::with_seed` is `new` plus that same call. + host.append_bytes(&[]); + record(&host, &mut want, &mut steps, "seed (empty)"); + + // Const byte strings across the cell-grouping boundary in both + // directions — 32 bytes is one payload cell, so 31/32/33 straddle it, + // and the empty string is the degenerate case a length prefix has to + // separate from every other. + let consts: Vec> = vec![ + vec![], + vec![0xa5], + (0..7u8).collect(), + (0..8u8).collect(), + (0..9u8).collect(), + (0..31u8).collect(), + (0..32u8).collect(), + (0..33u8).collect(), + (0..40u8).collect(), + ]; + for c in &consts { + host.append_bytes(c); + record( + &host, + &mut want, + &mut steps, + &format!("const bytes len {}", c.len()), + ); + } + + host.append_bytes(&root); + record(&host, &mut want, &mut steps, "root (32 bytes)"); + + host.append_bytes(&felt_bytes(felt_v)); + record(&host, &mut want, &mut steps, "felt"); + + host.append_field_element(&ext_v); + record(&host, &mut want, &mut steps, "ext"); + + // ★ The PHASE A path: a root absorbed through the halves family rather + // than through `append_root_cells`. `replay_phase_a` takes its roots as + // `&[Felt]` and calls `append_halves_misaligned`, whose declared byte + // length is `4 · halves.len()` — so what it is handed must be the eight + // halves of the root's THIRTY-TWO bytes on both arms. Handing it + // `RootCells::lanes_flat` gives four FULL FELTS on the algebraic arm, + // declaring sixteen bytes where the host declared thirty-two. + host.append_bytes(&root); + record( + &host, + &mut want, + &mut steps, + "root via the Phase A halves path", + ); + + // ★ SQUEEZES. `sample_ext` is where every shared challenge comes from — + // the LogUp pair, every beta, every z, every gamma — so the first + // challenge a spine draws is one of these. Its host counterpart is + // unambiguous (`sample_field_element`, one squeezed cell read as lanes + // 0-2), unlike `sample()`, which the algebraic arm DEFINES rather than + // mirrors and which is therefore still out of scope here. + // + // Both the drawn VALUE and the state after it, because they fail + // differently: a wrong value with a right state is a read of the wrong + // lanes, a right value with a wrong state is a wrong advance, and only + // the second corrupts everything downstream. + for k in 0..3 { + let e = host.sample_field_element(); + want.push(ext_word(&e)); + steps.push(format!("sample_ext {k} — the drawn value")); + record( + &host, + &mut want, + &mut steps, + &format!("sample_ext {k} — the state after"), + ); + } + + // ---- the machine chain, same script ---- + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Algebraic); + let arena = b.declare_arena(3); + let c_root = b.hint_word(arena, 0); + let c_felt = b.hint_felt(arena, 1); + let c_ext = b.hint_word(arena, 2); + let ext_lanes = b.unpack(c_ext); + + let mut t = TranscriptReplay::new(&[]); + let publish = |b: &mut LfmBuilder, t: &mut TranscriptReplay| { + let s = t.state(b); + assert_eq!(s.len(), 1, "{tenant}: an algebraic state is ONE cell"); + b.public(s[0]); + }; + publish(&mut b, &mut t); + for c in &consts { + t.append_const_bytes(c); + publish(&mut b, &mut t); + } + t.append_root_cells(&mut b, &[c_root]); + publish(&mut b, &mut t); + t.append_felt(&mut b, c_felt); + publish(&mut b, &mut t); + t.append_ext(&mut b, [ext_lanes[0], ext_lanes[1], ext_lanes[2]]); + publish(&mut b, &mut t); + + let root_arena = b.declare_arena(super::epoch::RootCells::words_per_root(&b)); + let root_cells = super::epoch::RootCells::hint(&mut b, root_arena, 0); + let phase_a_halves = root_cells.byte_halves(&mut b); + t.append_halves_misaligned(&phase_a_halves); + publish(&mut b, &mut t); + + for _ in 0..3 { + let e = t.sample_ext(&mut b); + b.public(e.as_cell()); + publish(&mut b, &mut t); + } + + let program = compile(b.finish()); + let mut arenas = vec![vec![root_word, base_word(felt_v), ext_word(&ext_v)]]; + // ⚠ The ALGEBRAIC form directly, NOT `proof_arena::commitment_words`. + // That helper reads `WrapHash::production()` — the workspace PIN — while + // this program's reader is `RootCells::words_per_root`, which reads THIS + // BUILDER's arm, and the builder is unconditionally `Algebraic` here. + // The two coincide only on an algebraic pin, so mixing them makes the + // gate pass on one branch and fail on another for a reason that has + // nothing to do with what it tests. + arenas.push(vec![commitment_to_digest(&root)]); + let exec = execute(&program, &arenas, &H::KIND).expect("the transcript program executes"); + + assert_eq!( + exec.public_words.len(), + want.len(), + "{tenant}: the two chains must have the same number of steps" + ); + for (i, (got, expect)) in exec.public_words.iter().zip(&want).enumerate() { + assert_eq!( + &got.1, expect, + "{tenant}: transcript states part at step {i} ({}) — \ + every later challenge is a different value", + steps[i] + ); + } + } + for_each_tenant!(check); +} + +/// ⚠ The differential's own control: `state()` must actually TRACK the chain, +/// or a gate comparing a constant against a constant would pass. Absorbing +/// anything must move the state. +#[test] +fn the_state_moves_on_every_absorb() { + fn check(tenant: &str) { + let mut host = AlgebraicTranscript::new(H::KIND); + let mut seen = vec![host.state_word()]; + for c in [vec![], vec![1u8], (0..32u8).collect::>()] { + host.append_bytes(&c); + let s = host.state_word(); + assert!( + !seen.contains(&s), + "{tenant}: absorbing must move the state to a fresh value" + ); + seen.push(s); + } + } + for_each_tenant!(check); +} diff --git a/prover/src/lfm/transcript_kats.rs b/prover/src/lfm/transcript_kats.rs new file mode 100644 index 000000000..c78758c1a --- /dev/null +++ b/prover/src/lfm/transcript_kats.rs @@ -0,0 +1,164 @@ +//! Transcript KATs for the LFM compress-chain Fiat–Shamir transcript, at 6 and +//! 7 rounds. +//! +//! GENERATED — do not hand-edit. The INPUTS come from +//! `thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.json`, which +//! the oracle produced from a Python reference written **before any Rust +//! existed**. That ordering is the point: these vectors are a specification the +//! implementation is checked against, not a recording of what the +//! implementation happened to do. +//! +//! ⚠ **The results were re-pinned when the socket widened to twelve lanes**, by +//! `leaf-spec/rate4_kat_gen.py` out of the same oracle. All 12 moved and no +//! input did: `block_len` is `v[14]` and cannot be made mode-dependent, so the +//! transcript domain re-blesses alongside the leaf domain that needed the width +//! (COMMIT.md §1.4.4 H9). +//! +//! Framing (transcript spec §1.2 at the COMMIT.md §1.2 width): identical to the +//! Merkle socket in every respect except the tag word, which is `"LFMT"` +//! instead of `"LFMC"`. So h = BLAKE3_IV, m[0..4] = state, m[4..8] = operand, +//! m[8..12] = 0 — the third input cell, which the unread-`IN` pins force to +//! zero — m[12] = "LFMT", m[13..16] = 0, t = 0, block_len = 52, flags = 0x0B, +//! digest = out[0..4]. At 7 rounds a step is still literally +//! `blake3::hash(state ‖ operand ‖ 0^16 ‖ "LFMT")[..16]`. + +/// One transcript step: state, operand, and the resulting state at each round +/// count. +pub struct StepVector { + pub name: &'static str, + pub state: [u32; 4], + pub operand: [u32; 4], + /// Result at 6 rounds (the A6R variant; no library computes it). + pub result_6: [u32; 4], + /// Result at 7 rounds — `blake3::hash(state ‖ operand ‖ "LFMT")[..16]`. + pub result_7: [u32; 4], +} + +pub const STEP_VECTORS: [StepVector; 6] = [ + StepVector { + name: "zero_state_zero_operand", + state: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + operand: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + result_6: [0x58A784C6, 0xCA20122A, 0x574D1385, 0x4C7F61AC], + result_7: [0xBB5DF0AD, 0xBB660FC6, 0x401C1FAD, 0x651C297C], + }, + StepVector { + name: "zero_state_main_root", + state: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + operand: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + result_6: [0xBFD9E2ED, 0x726EDE27, 0x91805DE1, 0xC11F0DA8], + result_7: [0x503FDDF4, 0x48633531, 0x8EEA401C, 0x213213C8], + }, + StepVector { + name: "ramp_state_ramp_operand", + state: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + operand: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + result_6: [0x00A8B31B, 0x0C48A09A, 0x1D06A9A8, 0x6C27BD61], + result_7: [0x29D95598, 0x69E4FD73, 0x243BFCE9, 0x14598F96], + }, + StepVector { + name: "max_state", + state: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + operand: [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE], + result_6: [0xA710A43E, 0x62E96839, 0xE00D7CA2, 0x1E054FEF], + result_7: [0xBA98C5EF, 0xAFDC8C3E, 0xFA425A12, 0xF35C1B47], + }, + StepVector { + name: "squeeze_operand_0", + state: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + operand: [0x305A5153, 0x00000000, 0x00000000, 0x00000000], + result_6: [0xE1F1E0DF, 0x9B1491E4, 0x26F46CE4, 0x644BA9F0], + result_7: [0x2EBCFDA8, 0x2F7C4E72, 0xAE841641, 0x6751FE80], + }, + StepVector { + name: "squeeze_operand_255", + state: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + operand: [0x305A5153, 0x000000FF, 0x00000000, 0x00000000], + result_6: [0xDD47DC57, 0x9AC95714, 0xE774A0DA, 0xD4703C0B], + result_7: [0x33396F61, 0x832BA04F, 0x2BB788AB, 0xE9FE006B], + }, +]; + +/// The END-TO-END vector: a `FriToyV0`-preamble-shaped transcript, op by op. +/// +/// The operation sequence is fixed and lives in the test that replays it — +/// `absorb(main_root), squeeze, squeeze, absorb(l1_root), squeeze, +/// absorb_felts(t0w), absorb_felts(t1w), 4× squeeze_bits`, ✓ VERIFIED against +/// `programs::fri_toy_program_source`. What is pinned here is the STATE after +/// every recorded op, so a divergence is located at the step it happened rather +/// than at the end. +/// +/// The last two absorbs are `absorb_felts`, not `absorb2`: the terminal +/// coefficients are field DATA, so each is leaf-hashed under `"LFML"` and the +/// DIGEST is absorbed. The transcript's step count is the same either way, which +/// is why this vector had to be re-pointed deliberately when the program moved +/// rather than caught by a red test. +pub struct EndToEndVector { + /// State after each recorded op, in order. + pub states: [[u32; 4]; 11], + /// The three ext challenges (lanes 0–2 of a squeezed cell). + pub alpha: [u32; 3], + pub zeta0: [u32; 3], + pub zeta1: [u32; 3], + /// `QUERY_BITS` index bits per query, low-to-high. + pub query_bits: [[u8; 4]; 4], +} + +/// The transcript's inputs — the four cells the preamble absorbs. +pub const MAIN_ROOT: [u32; 4] = [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10]; + +pub const L1_ROOT: [u32; 4] = [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20]; + +pub const T0W: [u32; 4] = [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE]; + +pub const T1W: [u32; 4] = [0x0BADC0DE, 0xD15EA5E5, 0xC0FFEE00, 0xBAAAAAAD]; + +/// Compressions the whole preamble costs — the oracle's cost claim. +/// +/// **13, not 11.** Eleven are TRANSCRIPT steps (5 absorbs, 6 squeezes); the +/// other two are the LEAF rows `absorb_felts` adds, one per data cell. Counting +/// them here is the oracle's convention and it is the one that closes the +/// `FriToyV0` total: 4 queries × 20 + 13 = 93. +pub const FRI_TOY_COMPRESSIONS: usize = 13; + +/// The end-to-end vector at 7 rounds (the default build). +pub const FRI_TOY_7: EndToEndVector = EndToEndVector { + states: [ + [0x503FDDF4, 0x48633531, 0x8EEA401C, 0x213213C8], + [0x02FEA9A6, 0xE6BF885C, 0xF174E65F, 0x4EC9AB10], + [0xF91F56DE, 0x62F37956, 0xE67D5421, 0xD82727D0], + [0x7A31B840, 0xAD7F2625, 0xE27D1C56, 0xCDB0E9A7], + [0xA6790B7B, 0x00695D49, 0xA663DC33, 0x2E849F0C], + [0xFC40465E, 0x0092B147, 0x2FA48645, 0x9755608B], + [0x6BD6F0B0, 0x634CF1C6, 0x3CBD9D2D, 0x349F278B], + [0xD86E1D3F, 0xDD1CFBC3, 0x1C8E8F14, 0x22D35494], + [0x5B449138, 0x3435B7D5, 0x7CFE4C06, 0x1C022FCF], + [0xE8D9848C, 0x0429B6F7, 0xDD5CBA1A, 0xBD465F16], + [0xD1E3472D, 0xD945386A, 0xC746A9B3, 0x8FD73C31], + ], + alpha: [0x503FDDF4, 0x48633531, 0x8EEA401C], + zeta0: [0x02FEA9A6, 0xE6BF885C, 0xF174E65F], + zeta1: [0x7A31B840, 0xAD7F2625, 0xE27D1C56], + query_bits: [[0, 0, 0, 0], [1, 1, 1, 1], [0, 0, 0, 1], [0, 0, 1, 1]], +}; + +/// The end-to-end vector at 6 rounds (`--features blake3-6round`). +pub const FRI_TOY_6: EndToEndVector = EndToEndVector { + states: [ + [0xBFD9E2ED, 0x726EDE27, 0x91805DE1, 0xC11F0DA8], + [0x9DDE5E52, 0xD7018142, 0x978528FF, 0xA01782B7], + [0x189BAD31, 0x364E3C30, 0x6B4D5516, 0x78D7FE7B], + [0x8E3E80BB, 0xFCA1AF96, 0xA59E0F41, 0x18C9AB19], + [0x4DF9BD75, 0x2E131E7D, 0x2DEB348E, 0x62BC30F3], + [0x202DB868, 0x8FF72AC4, 0x452D536A, 0x78DECD7B], + [0xA3D1E95E, 0x8C68ECF6, 0x91D7DF5E, 0x28D8CBB2], + [0xFA4222A2, 0x18DA7862, 0x3BBDA144, 0xAB453013], + [0x525CE059, 0x0E9B9EAB, 0xFB57633B, 0xE43490A7], + [0x261309AA, 0x84C675B7, 0xD5BBF0EB, 0x4AF40850], + [0x4D62F602, 0x2CC5D660, 0x99C44AF7, 0xEB502D8A], + ], + alpha: [0xBFD9E2ED, 0x726EDE27, 0x91805DE1], + zeta0: [0x9DDE5E52, 0xD7018142, 0x978528FF], + zeta1: [0x8E3E80BB, 0xFCA1AF96, 0xA59E0F41], + query_bits: [[0, 1, 1, 1], [0, 1, 0, 0], [1, 0, 0, 1], [0, 1, 0, 1]], +}; diff --git a/prover/src/lfm/transcript_replay.rs b/prover/src/lfm/transcript_replay.rs new file mode 100644 index 000000000..709d12f47 --- /dev/null +++ b/prover/src/lfm/transcript_replay.rs @@ -0,0 +1,1519 @@ +//! `TranscriptReplay` — the production `DefaultTranscript` replayed inside the +//! machine. +//! +//! This is an eDSL library, not a chip: it is ordinary Rust that tracks the +//! transcript's state AT EMIT TIME and emits the instructions that reproduce +//! the transcript's VALUES at run time. The split matters. Which squeeze a +//! challenge comes from, where a refill lands, which absorb invalidates the +//! output buffer — all of that is decided by the emitter and baked into the +//! program's shape. Only the field arithmetic and the keccak rows are machine +//! work. +//! +//! The mirror it must match is `crypto::fiat_shamir::default_transcript` +//! (post-#841): a keccak sponge with a Plonky3-style duplex output buffer. +//! [`super::keccak_host::TranscriptModel`] is the host model of the same state +//! machine and is checked against the real thing in `machine_tests`; this type +//! tracks the identical `segment` / `out_pos` pair. +//! +//! ## What makes the replay cheap +//! +//! Two identities do all the work: +//! +//! 1. **The reversal cancels.** `sample()` returns the digest byte-REVERSED, and +//! candidates are read big-endian out of those reversed bytes. The two +//! reversals cancel exactly: candidate `i` is the PLAIN digest's `u64` lane +//! `3 − i` (see [`super::keccak_host::candidate_from_state`]). So sampling +//! never reverses anything — it reads `u32` halves straight off the keccak +//! state words, two `Unpack`s per squeeze. The reversed digest is emitted +//! only for the RE-ABSORB (and for a raw [`TranscriptReplay::sample`], whose +//! return value *is* those bytes). +//! +//! 2. **Canonicity is one instruction.** `p = (2^32 − 1)·2^32 + 1`, so a +//! candidate `hi·2^32 + lo` is out of range exactly when +//! `hi = 2^32 − 1 ∧ lo ≠ 0` — and `div` is constrained as `OUT·B = A`, which +//! is provable with `B = 0` only when `A = 0`. See [`assert_canonical`]. +//! +//! A third property is about the emitter rather than the machine: the segment is +//! packed into `u32` halves per SEGMENT, never per append, which is what makes +//! constants of arbitrary length safe anywhere in the stream. The argument is in +//! [`TranscriptReplay::append_const_bytes`] and should be read before touching +//! the append path. +//! +//! ## The zero-rejection restriction +//! +//! A straight-line program has one shape. The production sampler rejects +//! out-of-range candidates and draws again, so the number of candidates a draw +//! consumes — and therefore every later draw's buffer position — is +//! DATA-DEPENDENT. A machine with no branches cannot follow that. The emitted +//! program therefore encodes the no-rejection schedule and is unprovable for the +//! (vanishingly rare) transcript that rejects. See `SOUNDNESS.md` §6.3 for the +//! completeness bound, for why this costs COMPLETENESS only — a rejecting +//! transcript yields no proof, never a wrong one — and for why supporting one +//! rejection is NOT an emitter parameter but a change to the production +//! sampler. + +use crate::tables::types::FE; + +use super::algebraic_transcript::AlgebraicTranscript; +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::edsl::{self, ByteWrapHash, SpongeVar, WrapHash}; +use super::keccak_host::{BYTES_PER_HALF, SQUEEZE_LEN}; +use super::layout::keccak::DIGEST_WORDS; + +/// `u32` halves in one 32-byte squeeze. +const SQUEEZE_HALVES: usize = SQUEEZE_LEN / BYTES_PER_HALF; + +/// Bytes in one 64-bit candidate. +const CANDIDATE_BYTES: usize = 8; + +/// Candidates one squeeze yields. +const CANDIDATES_PER_SQUEEZE: usize = SQUEEZE_LEN / CANDIDATE_BYTES; + +/// `2^32 − 1` — the only `hi` half that can put a candidate at or above `p`. +const HI_MAX: u64 = 0xFFFF_FFFF; + +/// One host `append_bytes` call — the unit BOTH arms replay, and the reason it +/// is represented rather than implied. +/// +/// ★ **The call boundary is invisible on the byte arm and LOAD-BEARING on the +/// algebraic one.** A keccak or BLAKE3 transcript absorbs into one flat segment, +/// so two appends and one append of their concatenation are the same bytes and +/// the same digest — [`TranscriptReplay::append_const_bytes`] relies on exactly +/// that to concatenate constant runs before chunking. An ALGEBRAIC transcript +/// prefixes every call with its LENGTH +/// ([`AlgebraicTranscript::append_bytes_cells`]), so those same two appends are +/// a DIFFERENT transcript. +/// +/// The consequence for callers, stated once here: **on the algebraic arm the +/// machine's append calls must correspond one-to-one with the host's +/// `append_bytes` calls.** A coalescing that is free under keccak is a wrong +/// challenge under RPO, and it fails as a diverged challenge rather than as +/// anything that names the coalescing. +#[derive(Clone)] +enum Append { + /// Compile-time bytes, any length, any alignment. + Const(Vec), + /// `byte_len` machine-computed bytes carried as `ceil(byte_len / 4)` `u32` + /// halves, four bytes each little-endian. The trailing half carries only + /// its low `byte_len % 4` bytes. + Bytes { halves: Vec, byte_len: usize }, +} + +impl Append { + /// The byte arm's view: appends flattened into packer pieces, boundaries + /// discarded because the byte stream does not have them. + fn push_pieces(&self, out: &mut Vec) { + match self { + Append::Const(bytes) => out.push(SegPiece::Const(bytes.clone())), + Append::Bytes { halves, byte_len } => { + let full = byte_len / BYTES_PER_HALF; + let rem = byte_len % BYTES_PER_HALF; + if full > 0 { + out.push(SegPiece::Halves(halves[..full].to_vec())); + } + if rem > 0 { + out.push(SegPiece::Partial(halves[full], rem)); + } + } + } + } +} + +/// A piece of the pending segment, held UNPACKED until the squeeze. +/// +/// Packing is per SEGMENT, never per append — see +/// [`TranscriptReplay::append_const_bytes`] for why that distinction is the +/// whole design. +#[derive(Clone)] +enum SegPiece { + /// Compile-time bytes. Consecutive runs of these are concatenated before + /// being chunked into halves, so a constant of any length may sit anywhere. + Const(Vec), + /// Machine-computed `u32` halves, four bytes each little-endian. Opaque + /// felts, so they must land on a 4-byte boundary of the segment. + Halves(Vec), + /// A machine half carrying only its low `n` bytes (`n` in `1..4`) — the + /// trailing piece of a byte string whose length is not a multiple of four. + /// The packer masks it and pins the unused high bytes to zero. + Partial(Felt, usize), +} + +/// A 64-bit candidate as the two `u32` halves the machine actually holds: +/// `value = hi·2^32 + lo`. +#[derive(Debug, Clone, Copy)] +pub struct Candidate { + pub lo: Felt, + pub hi: Felt, +} + +/// The squeeze currently backing the output buffer. +/// +/// Held as the PLAIN digest's two words with their lane unpacks memoized: +/// candidates 0 and 1 live in word 1 and candidates 2 and 3 in word 0, so a +/// draw that consumes one or two candidates emits a single `Unpack`. +#[derive(Clone)] +struct SqueezeBuf { + // Carries its own width, so an algebraic squeeze (ONE cell) fits unchanged. + words: edsl::WrapDigest, + lanes: [Option<[Felt; 4]>; DIGEST_WORDS], +} + +/// Emit-time replay of `DefaultTranscript`. +/// +/// `Clone` is production's `transcript.clone()` — the per-table fork +/// (`verifier.rs:1263`), and the only reason this type is cloneable. It is a +/// pure emitter-state copy: the pending segment is a list of program constants +/// and CELL handles, so a fork shares the cells its prefix already produced and +/// emits nothing. That is exactly the production semantics — the shared prefix +/// is hashed once and each fork diverges only past its domain separator. +#[derive(Clone)] +pub struct TranscriptReplay { + /// The pending appends — the hasher's unfinalized input. Turned into halves + /// (byte arm) or driven into the chain (algebraic arm) at the squeeze, never + /// at the append. + segment: Vec, + /// The segment's length in BYTES: what drives keccak's length-dependent + /// padding, and what decides where every half boundary falls. + segment_len: usize, + buf: Option, + /// Bytes already handed out of the buffer; `SQUEEZE_LEN` means "empty, the + /// next candidate forces a squeeze". + out_pos: usize, + /// ★ The ALGEBRAIC arm's compress chain, `None` until the first + /// builder-taking call creates it. + /// + /// Lazily, and deliberately: [`TranscriptReplay::new`] takes no builder, and + /// the arm is a property of the BUILDER — the hash-pinned instruments + /// override it with `with_wrap_hash` — so it cannot be decided at + /// construction from the global configuration. + sponge: Option, + /// ★ The POSEIDON1 arm's sponge (ZisK's transcript, `p1w16_emit`), lazily + /// like [`Self::sponge`] and for the same reason. Every append drains into + /// it in host order at the next builder-taking call. + p1: Option, +} + +impl TranscriptReplay { + /// `DefaultTranscript::new(seed)` — an empty sponge with `seed` absorbed. + /// + /// The seed is a program constant, interned by the builder. Anything the + /// machine COMPUTES is absorbed with [`TranscriptReplay::append_halves`]. + pub fn new(seed: &[u8]) -> Self { + let mut t = Self { + segment: Vec::new(), + segment_len: 0, + buf: None, + out_pos: SQUEEZE_LEN, + sponge: None, + p1: None, + }; + t.append_const_bytes(seed); + t + } + + /// Absorb machine-computed data: `4 · halves.len()` bytes, four per half, + /// little-endian. + /// + /// Whole halves only, and there is no length parameter. A half is four + /// consecutive bytes of the SEGMENT, and these felts are opaque to the + /// emitter — it cannot split or shift their bytes — so machine data must + /// land on a 4-byte boundary. [`TranscriptReplay::assert_appendable`] is the + /// loud check. + /// + /// This is not a real restriction for the FRI-verifier scope: every + /// production rendering is a multiple of four bytes. A commitment root is + /// 32, a Goldilocks felt streams as 8, a cubic-extension felt as 24. + pub fn append_halves(&mut self, halves: &[Felt]) { + self.assert_appendable(); + self.segment.push(Append::Bytes { + halves: halves.to_vec(), + byte_len: BYTES_PER_HALF * halves.len(), + }); + self.segment_len += BYTES_PER_HALF * halves.len(); + // Absorbing invalidates the buffer: a later challenge must depend on + // this input, so bytes squeezed before it are dropped. + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + /// Absorb one machine word — its four lanes as four halves, 16 bytes. + /// + /// The word must be a `u32`-half word (a keccak state/digest word, which is + /// where transcript-bound data comes from). Feeding one whose lanes are full + /// felts is not a silent miscoding: the halves end up as lanes of a keccak + /// input word, and the adapter refuses anything at or above `2^32` with + /// `LfmExecError::NotU32Half` — and would be unprovable regardless, since the + /// bus range-checks them. + pub fn append_word(&mut self, b: &mut LfmBuilder, w: Cell) { + let lanes = b.unpack(w); + self.append_halves(&lanes); + } + + /// Absorb a 32-byte keccak digest carried as two machine words — the shape a + /// commitment root arrives in. + /// Absorb one 32-byte commitment ROOT, carried as the configuration's digest + /// cells — two on a byte hash, ONE on an algebraic one. + /// + /// ★ **FREE on the algebraic arm, by the cancellation the whole felt↔byte + /// convention exists for.** The host absorbs a root with + /// `append_bytes(root)`, whose payload cell is `bytes_to_cell(root)`; an + /// algebraic backend serialises its digest as four canonical big-endian + /// felts; so that call recovers exactly the digest cell the backend started + /// from. No decomposition, no regrouping — one `absorb_cell` of the root's + /// own cell. + /// + /// ⚠ Precondition, exactly as `AlgebraicTranscript::bytes_to_cell` states + /// it: the backend's serialisation is canonical. A non-canonical eight-byte + /// group would reduce, and reduction is what makes two different roots + /// absorb identically. + pub fn append_root_cells(&mut self, b: &mut LfmBuilder, cells: &[Cell]) { + if Self::is_p1(b) { + // `P1Transcript::append_bytes(root)`: the length 32, then the root's + // 8-byte big-endian groups — its four digest lanes, by the node + // encoding's cancellation (`digest_to_commitment`). + assert_eq!(cells.len(), 1, "a Poseidon1 root is ONE cell"); + let mut sponge = self.drive_p1(b); + sponge.put_const(b, FE::from(32u64)); + for lane in b.unpack(cells[0]) { + sponge.put_felt(b, lane); + } + self.p1 = Some(sponge); + return; + } + if Self::is_algebraic(b) { + assert_eq!(cells.len(), 1, "an algebraic root is ONE cell"); + let mut sponge = self.drive_chain(b); + // The prefix from the rule, applied at a 32-byte payload length. + let rule = AlgebraicTranscript::append_bytes_cells(&[0u8; 32]); + assert_eq!(rule.len(), 2, "32 bytes is one prefix and one cell"); + let prefix = b.digest_const(rule[0]); + sponge.absorb(b, prefix.as_cell()); + sponge.absorb(b, cells[0]); + self.sponge = Some(sponge); + return; + } + let mut halves = Vec::with_capacity(4 * cells.len()); + for c in cells { + halves.extend_from_slice(&b.unpack(*c)); + } + self.append_halves(&halves); + } + + /// [`Self::append_root_cells`] where the cursor is not 4-byte aligned — the + /// statement leg's position. Identical on the algebraic arm, which has no + /// cursor at all. + pub fn append_root_cells_misaligned(&mut self, b: &mut LfmBuilder, cells: &[Cell]) { + if Self::is_algebraic(b) || Self::is_p1(b) { + self.append_root_cells(b, cells); + return; + } + let mut halves = Vec::with_capacity(4 * cells.len()); + for c in cells { + halves.extend_from_slice(&b.unpack(*c)); + } + self.append_halves_misaligned(&halves); + } + + /// ⚠ ONE append, not two, and on the algebraic arm that is the difference + /// between the host's transcript and a different one: the host absorbs a + /// root with a single `append_bytes(root)`, so a machine that absorbed the + /// two words separately would emit two length prefixes. On the byte arm the + /// two spellings are the same 32 bytes, which is exactly why the mistake + /// would have been invisible until an algebraic challenge diverged. + pub fn append_digest(&mut self, b: &mut LfmBuilder, words: &[Cell; DIGEST_WORDS]) { + let mut halves = Vec::with_capacity(4 * DIGEST_WORDS); + for w in words { + halves.extend_from_slice(&b.unpack(*w)); + } + self.append_halves(&halves); + } + + /// Absorb one base field element the way production streams it: the + /// canonical `u64` in BIG-endian byte order, 8 bytes. + /// + /// `FieldElement::stream_bytes` is + /// `sink(&self.canonical_u64().to_be_bytes())`, so the endianness flip is + /// real work for this machine — see [`felt_be_halves`] for the gadget and + /// its cost. + /// + /// ★ FREE on the algebraic arm, and by the same cancellation a root gets. + /// The host absorbs `append_bytes(&canonical(v).to_be_bytes())`, and + /// `bytes_to_cell` over exactly eight bytes puts that `u64` in lane 0 and + /// zero everywhere else — so the payload cell IS `[v, 0, 0, 0]`, with no + /// decomposition and no byte swap. The byte arm's `felt_be_halves` was the + /// price of a byte-oriented hash, not of the value. + pub fn append_felt(&mut self, b: &mut LfmBuilder, v: Felt) { + if Self::is_p1(b) { + // `append_bytes(&canonical(v).to_be_bytes())`: the length 8, then the + // one 8-byte big-endian group, which is `v`. + let mut sponge = self.drive_p1(b); + sponge.put_const(b, FE::from(8u64)); + sponge.put_felt(b, v); + self.p1 = Some(sponge); + return; + } + if Self::is_algebraic(b) { + let mut sponge = self.drive_chain(b); + // Eight bytes of payload: the rule fixes the prefix and says there + // is exactly one payload cell. + // A probe rather than a zero payload: zero would cellify to zero + // whatever the placement, and PLACEMENT is the thing being relied + // on. This pins that a felt's eight big-endian bytes land in lane 0 + // and nowhere else, which is what makes the pack below the rule + // rather than a restatement of it. + const PROBE: u64 = 0x0123_4567_89ab_cdef; + let rule = AlgebraicTranscript::append_bytes_cells(&PROBE.to_be_bytes()); + assert_eq!(rule.len(), 2, "eight bytes is one prefix and one cell"); + assert_eq!( + rule[1], + [FE::from(PROBE), FE::zero(), FE::zero(), FE::zero()], + "a felt's eight big-endian bytes must cellify to lane 0 alone" + ); + let prefix = b.digest_const(rule[0]); + sponge.absorb(b, prefix.as_cell()); + let zero = b.felt_const(FE::zero()); + let cell = b.pack_word([v, zero, zero, zero]); + sponge.absorb(b, cell); + self.sponge = Some(sponge); + return; + } + let halves = felt_be_halves(b, v); + self.append_halves(&halves); + } + + /// Absorb one cubic-extension element: coordinates 0, 1, 2, each as its own + /// 8 big-endian bytes — 24 bytes in total. + /// + /// Coordinate order is FORWARD and was verified against the source, because + /// the file offers both orders and picking the wrong one is invisible until + /// a challenge diverges: `FieldElement`'s + /// `write_bytes_be` (which `stream_bytes` calls) writes components 0, 1, 2, + /// while the REVERSED 2, 1, 0 order belongs to the raw `[FpE; 3]` array + /// type. Different types, no contradiction — but do not "fix" this to match + /// the other impl. + /// + /// ★ On the ALGEBRAIC arm this is `append_field_element`, which is a + /// DIFFERENT host method rather than the same one over different bytes: one + /// DATA cell `[x0, x1, x2, 0]` absorbed through the leaf domain, against the + /// byte arm's three separate 8-byte renderings. So the arm is not an + /// optimisation of the loop below — routing through it would emit three + /// length-prefixed `append_bytes` calls where the host made one + /// `append_field_element`, and that is exactly what the emitter gate caught. + pub fn append_ext(&mut self, b: &mut LfmBuilder, coords: [Felt; 3]) { + if Self::is_p1(b) { + // `append_field_element`: the three coefficients, no length. + let mut sponge = self.drive_p1(b); + for c in coords { + sponge.put_felt(b, c); + } + self.p1 = Some(sponge); + return; + } + if Self::is_algebraic(b) { + let mut sponge = self.drive_chain(b); + // `field_element_cell`'s layout, in the machine's spelling: lanes + // 0-2 the coordinates, lane 3 zero. `pack_ext` already IS that + // packing, so the layout has one definition rather than two. + let cell = b.pack_ext(coords[0], coords[1], coords[2]); + sponge.absorb_felts(b, cell.as_cell()); + self.sponge = Some(sponge); + return; + } + for c in coords { + self.append_felt(b, c); + } + } + + /// Absorb a compile-time constant byte string of ANY length, anywhere in the + /// segment. No alignment requirement, no builder — the bytes are stored + /// unpacked and interned at the squeeze. + /// + /// ## Why packing is per SEGMENT, not per append + /// + /// Append boundaries are not machine-visible. Between two finalize points + /// the production hasher sees one concatenated byte stream; `append_bytes` + /// boundaries leave no trace in the digest input. Every length here is + /// compile-time. So the emitter's correct unit of packing is the segment, + /// and it packs by concatenating consecutive constant runs and only THEN + /// chunking into halves. + /// + /// That is what makes "a partial half in the middle of a segment that the + /// next append must continue into" impossible rather than merely rejected: + /// when the emitter packs, it already holds every later constant in the + /// segment. Appending `b"abc"` then `b"de"` yields the five-byte run + /// `abcde`, chunked as two halves — it is not two independently packed + /// pieces. **Do not reintroduce per-append packing.** + /// + /// Segment prefixes are safe by construction too: every segment after the + /// first begins with the 32-byte reversed digest, a multiple of four. + /// + /// ## The one case that remains, deliberately unbuilt + /// + /// A constant of length ≢ 0 (mod 4) followed by MACHINE data — a 27-byte + /// domain tag ahead of a root word, say — leaves the dynamic value straddling + /// a half boundary, and re-aligning opaque felts by 1–3 bytes needs a + /// byte-level splice (BitDec-32 per affected half, or a byte-table route). + /// [`TranscriptReplay::append_halves`] rejects it loudly. It arises only in + /// the statement-absorb leg, at a volume of a few dozen halves per proof, and + /// never in FRI or Merkle traffic; when that leg is built it gets a + /// `splice_misaligned(constant_prefix_len, dynamic_halves)` helper. That is + /// an extension point, not a redesign. + pub fn append_const_bytes(&mut self, bytes: &[u8]) { + self.segment.push(Append::Const(bytes.to_vec())); + self.segment_len += bytes.len(); + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + fn assert_appendable(&self) { + assert_eq!( + self.segment_len % BYTES_PER_HALF, + 0, + "machine-computed data must start on a 4-byte boundary of the segment, \ + but {} bytes are already absorbed: a constant of length not a multiple \ + of four leaves the dynamic value straddling a half, which needs the \ + byte-level splice that only the statement-absorb leg will build", + self.segment_len + ); + } + + /// Whether this builder's configuration replays the ALGEBRAIC chain. + /// + /// Read from the BUILDER and never from `WrapHash::production()`: the + /// hash-pinned instruments override the configuration with + /// `with_wrap_hash(WrapHash::Blake3)` and must keep the byte arm even in a + /// build whose default is algebraic. + fn is_algebraic(b: &LfmBuilder) -> bool { + b.wrap_hash() == WrapHash::Algebraic + } + + /// Whether this builder replays ZisK's Poseidon1 transcript. + fn is_p1(b: &LfmBuilder) -> bool { + b.wrap_hash() == WrapHash::Poseidon1 + } + + /// The Poseidon1 arm's [`Self::drive_chain`]: every pending append into + /// the sponge, in order, under `P1Transcript::append_bytes`' encoding — + /// the length, then the bytes as 8-byte big-endian felts, the last + /// zero-padded. + fn drive_p1(&mut self, b: &mut LfmBuilder) -> super::p1w16_emit::P1SpongeVar { + let mut sponge = self + .p1 + .take() + .unwrap_or_else(|| super::p1w16_emit::P1SpongeVar::new(b)); + for append in core::mem::take(&mut self.segment) { + match append { + Append::Const(bytes) => { + for v in super::p1_commit::P1Transcript::append_bytes_felts(&bytes) { + sponge.put_const(b, v); + } + } + Append::Bytes { halves, byte_len } => { + sponge.put_const(b, FE::from(byte_len as u64)); + // The 32-byte cells of the big-endian grouping, cut back to + // the host's felt count (the rule's 8-byte groups). + let felts = byte_len.div_ceil(8); + let cells = cells_from_halves_be(b, &halves, byte_len); + let lanes: Vec = cells.iter().flat_map(|c| b.unpack(*c)).collect(); + for lane in lanes.into_iter().take(felts) { + sponge.put_felt(b, lane); + } + } + } + } + self.segment_len = 0; + sponge + } + + /// Drive every pending append into the algebraic chain, creating it on first + /// use, and hand the chain back. + /// + /// Deferred rather than eager because two of the append methods + /// ([`Self::append_const_bytes`] and the `halves` family) take no builder + /// and so have nothing to emit with. The deferral is invisible: nothing + /// between two appends can emit, so draining at the next builder-taking call + /// preserves the absorb ORDER exactly, which is the only thing the chain + /// binds. + /// + /// Every constant it needs comes from [`AlgebraicTranscript`]'s exported + /// rules rather than being restated here — the length prefix, the 32-byte + /// grouping and the felt encoding alike. That discipline is not decoration: + /// twice in this lane a machine side hand-wrote a constant that agreed with + /// the host until the host's rule moved. + fn drive_chain(&mut self, b: &mut LfmBuilder) -> SpongeVar { + let mut sponge = self.sponge.take().unwrap_or_else(|| SpongeVar::new(b)); + for append in core::mem::take(&mut self.segment) { + match append { + Append::Const(bytes) => { + for cell in AlgebraicTranscript::append_bytes_cells(&bytes) { + let c = b.digest_const(cell); + sponge.absorb(b, c.as_cell()); + } + } + Append::Bytes { halves, byte_len } => { + // The rule applied to a payload of the same LENGTH: it fixes + // the prefix cell and the payload cell COUNT, neither of + // which depends on the payload's value. Asserting the count + // is what keeps the gadget below honest if the rule moves. + let rule = AlgebraicTranscript::append_bytes_cells(&vec![0u8; byte_len]); + let prefix = b.digest_const(rule[0]); + sponge.absorb(b, prefix.as_cell()); + let cells = cells_from_halves_be(b, &halves, byte_len); + assert_eq!( + cells.len(), + rule.len() - 1, + "the machine's payload cell count must be the host rule's" + ); + for cell in cells { + sponge.absorb(b, cell); + } + } + } + } + self.segment_len = 0; + sponge + } + + /// Absorb machine-computed data that does NOT start on a 4-byte boundary. + /// + /// Same bytes as [`TranscriptReplay::append_halves`], but it permits the + /// misalignment that method rejects, and pays for it: each half then + /// straddles two output halves and has to be split byte-wise (see + /// [`split_half`] for the gadget and its cost). Use the aligned method + /// wherever the encoding allows — this one exists for the statement leg, + /// where a 30-byte domain tag and a 1-byte `fri` field between fixed-width + /// fields make misalignment unavoidable. + /// + /// The splice itself happens in [`TranscriptReplay::pack_segment`], not + /// here, because only the packer knows the byte cursor. + pub fn append_halves_misaligned(&mut self, halves: &[Felt]) { + self.segment.push(Append::Bytes { + halves: halves.to_vec(), + byte_len: BYTES_PER_HALF * halves.len(), + }); + self.segment_len += BYTES_PER_HALF * halves.len(); + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + /// Absorb `byte_len` machine-computed bytes carried in + /// `ceil(byte_len / 4)` halves — the general case, where the byte string's + /// length need not be a multiple of four. + /// + /// The trailing half is masked to its live bytes and its unused high bytes + /// are pinned to zero (see [`Packer::push_masked`]). That matters for any + /// length-prefixed field: `public_output` in the epoch statement is + /// collected one byte per COMMIT operation, so its length is whatever the + /// workload produced and is not aligned in general. + pub fn append_bytes_misaligned(&mut self, halves: &[Felt], byte_len: usize) { + assert_eq!( + halves.len(), + byte_len.div_ceil(BYTES_PER_HALF), + "byte_len must match the supplied halves" + ); + self.segment.push(Append::Bytes { + halves: halves.to_vec(), + byte_len, + }); + self.segment_len += byte_len; + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + /// Packs the segment into `u32` halves, walking it at BYTE granularity. + /// + /// Constant bytes accumulate host-side; a machine half drops straight in + /// when the cursor is 4-byte aligned — the path every aligned program takes, + /// which must stay instruction-free — and is split when it is not. The + /// packer is the only place that knows the cursor, which is why the splice + /// lives here rather than at the append. + fn pack_segment(&self, b: &mut LfmBuilder) -> Vec { + let mut pieces = Vec::with_capacity(self.segment.len()); + for append in &self.segment { + append.push_pieces(&mut pieces); + } + pack_pieces(&pieces, b) + } + + /// `DefaultTranscript::sample()` — finalize, reverse the 32 digest bytes, + /// re-absorb them, return them. + /// + /// The returned bytes and the re-absorbed bytes are the SAME 32 bytes; one + /// keccak row produces both. Also invalidates the output buffer, exactly as + /// production does. + /// + /// ★ On the ALGEBRAIC arm this is one `squeeze_cell` — the state, then an + /// advance past it. That is a DEFINITION rather than a claim of fidelity: + /// `sample()` mirrors `DefaultTranscript::sample()`, a keccak-duplex + /// primitive with no counterpart on [`AlgebraicTranscript`], so there is + /// nothing on the host for it to agree with. It is defined as the chain's + /// own canonical self-advance so that the method is total on both arms + /// instead of being a hole one configuration falls into. + pub fn sample(&mut self, b: &mut LfmBuilder) -> edsl::WrapDigest { + let Some(h) = b.wrap_hash().byte_hash() else { + let mut sponge = self.drive_chain(b); + let c = sponge.squeeze_cell(b); + self.sponge = Some(sponge); + return edsl::WrapDigest::from_cell(c); + }; + let (_plain, rev) = self.squeeze(b, h); + self.buf = None; + self.out_pos = SQUEEZE_LEN; + edsl::WrapDigest::from_pair(rev[0], rev[1]) + } + + /// One squeeze: emits the keccak row over the current segment, sets the + /// segment to the reversed digest, and hands back both digests — the plain + /// one because candidates are read off it, the reversed one because it is + /// what `sample()` returns. + fn squeeze( + &mut self, + b: &mut LfmBuilder, + h: ByteWrapHash, + ) -> (edsl::WrapDigest, [Cell; DIGEST_WORDS]) { + let packed = self.pack_segment(b); + let (plain, rev) = edsl::wrap_hash_bytes_with_rev(b, h, &packed, self.segment_len); + // The transcript absorbs the reversed bytes into a freshly reset hasher, + // so they are the WHOLE of the next segment, not a suffix of this one. + let mut halves = Vec::with_capacity(SQUEEZE_HALVES); + for w in rev { + halves.extend_from_slice(&b.unpack(w)); + } + self.segment = vec![Append::Bytes { + halves, + byte_len: SQUEEZE_LEN, + }]; + self.segment_len = SQUEEZE_LEN; + (plain, rev) + } + + /// Refill the output buffer with one squeeze, as `next_sample_u64` does. + fn refill(&mut self, b: &mut LfmBuilder, h: ByteWrapHash) { + let (plain, _rev) = self.squeeze(b, h); + self.buf = Some(SqueezeBuf { + words: plain, + lanes: [None; DIGEST_WORDS], + }); + self.out_pos = 0; + } + + /// The next 64-bit candidate, refilling when fewer than 8 bytes remain. + /// + /// Returns the candidate as its two `u32` halves rather than a felt: a + /// candidate is a 64-BIT integer and values in `[p, 2^64)` are not + /// felt-representable, so it cannot be one cell until it has been range- + /// checked. Consumers either check it ([`TranscriptReplay::sample_felt`]) or + /// use only the low half ([`TranscriptReplay::sample_u64_pow2`]). + /// + /// ⚠ **The BYTE duplex's primitive.** The algebraic arm has no candidate + /// stream at all — its squeezes are canonical felts by construction, so + /// there is nothing to carve and nothing to reject — and each of + /// [`Self::sample_felt`], [`Self::sample_ext`] and [`Self::sample_u64_pow2`] + /// takes its own path there. None of them reaches here, which is why this + /// method needs no arm rather than needing a guard. + pub fn next_candidate(&mut self, b: &mut LfmBuilder, h: ByteWrapHash) -> Candidate { + if self.out_pos + CANDIDATE_BYTES > SQUEEZE_LEN { + self.refill(b, h); + } + debug_assert_eq!( + self.out_pos % CANDIDATE_BYTES, + 0, + "candidates are the buffer's only consumer, so out_pos moves in 8s" + ); + let i = self.out_pos / CANDIDATE_BYTES; + debug_assert!(i < CANDIDATES_PER_SQUEEZE); + // Candidate i is the plain digest's u64 lane 3 − i (the reversal + // cancellation), and lane j is halves 2j (low) and 2j + 1 (high). + let lo = self.half(b, 2 * (CANDIDATES_PER_SQUEEZE - 1 - i)); + let hi = self.half(b, 2 * (CANDIDATES_PER_SQUEEZE - 1 - i) + 1); + self.out_pos += CANDIDATE_BYTES; + Candidate { lo, hi } + } + + /// Half `h` of the buffered digest: lane `h % 4` of word `h / 4`, unpacking + /// that word on first use. + fn half(&mut self, b: &mut LfmBuilder, h: usize) -> Felt { + let buf = self + .buf + .as_mut() + .expect("next_candidate refills before reading"); + let (w, l) = (h / 4, h % 4); + let lanes = match buf.lanes[w] { + Some(lanes) => lanes, + None => { + let lanes = b.unpack(buf.words[w]); + buf.lanes[w] = Some(lanes); + lanes + } + }; + lanes[l] + } + + /// One base-field challenge, under the configuration's **draw schedule**. + /// + /// ★ This is not a hash swap with a different sponge underneath; the two + /// configurations consume different numbers of candidates per coordinate, + /// which moves every refill boundary and therefore the squeeze count. The + /// schedule is read from `crypto`'s own `TranscriptHash` impls + /// ([`candidates_per_coordinate`]), never restated here. + /// + /// - **n = 1** (keccak, whose schedule is the unbounded rejection loop and + /// whose modal cost is one candidate): draw one, ASSERT it canonical, use + /// it. A straight-line machine cannot emit an unbounded loop, so it emits + /// the modal schedule and constrains the draw to have landed — the + /// standing unprovability restriction of `SOUNDNESS.md` §6.3, at ≈ 2⁻³² + /// per coordinate. + /// - **n ≥ 2** (BLAKE3, rider 1): draw all `n`, SELECT the first in range, + /// assert only the selection. Asserting candidate 0 instead would + /// reinstate exactly the restriction the rider was adopted to remove. + /// + /// **The claim the review should check** (stated, not self-certified): the + /// emitted program is complete except when EVERY one of the `n` candidates + /// misses — ≈ 2⁻³²ⁿ per coordinate, so ≈ 2⁻⁶⁴ at n = 2 — and on that event + /// the host draws another `n` while the machine has no program to run. The + /// restriction shrinks by a factor of 2³² and does not vanish. Consumption + /// is exactly `n` per coordinate on every path, which is the property the + /// replay's refill bookkeeping depends on. + /// + /// `sample_u64_pow2` is unaffected in both configurations: production's + /// `sample_u64` reaches the raw candidate stream, not the fixed schedule, + /// and at a power-of-two bound it accepts its first candidate. + pub fn sample_felt(&mut self, b: &mut LfmBuilder) -> Felt { + if Self::is_p1(b) { + let mut sponge = self.drive_p1(b); + let f = sponge.squeeze(b); + self.p1 = Some(sponge); + return f; + } + let Some(h) = b.wrap_hash().byte_hash() else { + // Lane 0 of one squeezed cell. ⚠ No host counterpart: an algebraic + // transcript has no base-field draw — `sample_field_element` returns + // an Fp3 element and reads three lanes at once. This exists so the + // byte arm's `sample_ext` and the hash-pinned instruments keep one + // API, and nothing in production reaches it on this arm. + let mut sponge = self.drive_chain(b); + let c = sponge.squeeze_cell(b); + self.sponge = Some(sponge); + let [lane0, _, _, _] = b.unpack(c); + return lane0; + }; + let n = candidates_per_coordinate(b.wrap_hash()); + let candidates: Vec = (0..n).map(|_| self.next_candidate(b, h)).collect(); + + // The fallback is the LAST candidate, matching the host's + // `chosen.unwrap_or(last)`: when everything missed it hands back an + // out-of-range value, its field rejects it, and the draw continues. + // Here "the field rejects it" is `assert_canonical` below, i.e. the + // program is unprovable — see the claim above. + let mut chosen = *candidates.last().expect("at least one candidate"); + for candidate in candidates[..n - 1].iter().rev() { + let in_range = candidate_in_range(b, *candidate); + // `select(bit, l, r)` yields `l` first when the bit is 0 and `r` + // first when it is 1, so `l = chosen`, `r = candidate` gives + // "this candidate if it is in range, else what we had". + let (lo, _) = b.select(in_range, chosen.lo.as_cell(), candidate.lo.as_cell()); + let (hi, _) = b.select(in_range, chosen.hi.as_cell(), candidate.hi.as_cell()); + // Both halves are `Select` outputs of `u32` cells, so they are the + // `u32`s the hash chip range-checked — which is what + // `assert_canonical`'s derivation needs. + chosen = Candidate { + lo: Felt(lo.addr()), + hi: Felt(hi.addr()), + }; + } + + assert_canonical(b, chosen); + candidate_to_felt(b, chosen) + } + + /// One cubic-extension challenge: three independent base draws in + /// coordinate order 0, 1, 2 — which is what + /// `Degree3GoldilocksExtensionField::sample_field_element_from` does + /// (`core::array::from_fn` evaluates in index order). + /// + /// This is the production shape: the STARK verifier's challenges are + /// extension elements, so an ext draw is where the completeness bound is + /// paid three times over. + pub fn sample_ext(&mut self, b: &mut LfmBuilder) -> Ext { + if Self::is_p1(b) { + // `sample_field_element`: three consecutive squeezed felts. + let mut sponge = self.drive_p1(b); + let c: [Felt; 3] = core::array::from_fn(|_| sponge.squeeze(b)); + self.p1 = Some(sponge); + return b.pack_ext(c[0], c[1], c[2]); + } + if Self::is_algebraic(b) { + // ★ ONE squeeze, not three. `sample_field_element` reads all three + // coordinates off a single squeezed cell (lanes 0-2) where the byte + // transcript carves three independent 64-bit candidates — so the + // squeeze count of a whole wrap program falls by a factor of three + // on this arm, and the completeness restriction of `SOUNDNESS.md` + // §6.3 does not apply to it at all. + let mut sponge = self.drive_chain(b); + let e = sponge.squeeze_ext(b); + self.sponge = Some(sponge); + return e; + } + let a0 = self.sample_felt(b); + let a1 = self.sample_felt(b); + let a2 = self.sample_felt(b); + b.pack_ext(a0, a1, a2) + } + + /// `sample_u64(1 << nbits)` — the low `nbits` bits of one candidate, as bits + /// low-to-high. + /// + /// No canonicity guard and no rejection, because production has none here: + /// `threshold = upper_bound.wrapping_neg() % upper_bound` is 0 at every + /// power of two, so the loop in `sample_u64` accepts its first candidate + /// unconditionally and returns `candidate % 2^nbits`. An out-of-range + /// candidate is perfectly legal for this draw — which is why `u64` draws + /// contribute NOTHING to the completeness bound. + /// + /// `nbits ≤ 32` keeps the answer inside the candidate's low half. The bound + /// is real rather than defensive: FRI query indices are bounded by the LDE + /// domain, which is ≤ 2^25 here. + /// + /// ## `nbits = 0` is legal, and it still CONSUMES + /// + /// A one-row trace at blowup 2 has a two-leaf LDE, so `sample_query_indexes` + /// calls `sample_u64(domain_size >> 1)` = `sample_u64(1)`: one pair, one + /// index, and that index is 0. Production does not skip the draw for it. + /// + /// - `DefaultTranscript::sample_u64` computes + /// `threshold = 1u64.wrapping_neg() % 1 = 0`, so the loop calls + /// `next_sample_u64()` ONCE — advancing `out_pos` by eight — and returns + /// `candidate % 1 = 0`. + /// - `AlgebraicTranscript::sample_u64`, the PINNED path, squeezes a cell and + /// then masks: `squeeze_cell()`, then `canonical(c[0]) & (upper_bound − 1)`. + /// The squeeze happens first and unconditionally; `& 0` is what yields 0. + /// + /// So an emitter that skipped the draw would be one squeeze short of the host + /// for that table, and every challenge after it in that fork would diverge. + /// Both arms below already consume BEFORE reading `nbits` — + /// `SpongeVar::squeeze_bits` calls `squeeze_cell` first, and the byte arm + /// calls `next_candidate` first — so zero bits needs no special case beyond + /// letting it through. The resulting `bit_dec(_, 0)` is a well-formed row + /// that exposes no bits; it is left unspecial-cased on purpose, because the + /// smallest correct change is the right one in a Fiat–Shamir file. + /// + /// ⚠ A desync here is INVISIBLE to a value differential on the table itself: + /// every index is 0 whether the draw happened or not, and `iota_bits` is the + /// last thing `epoch::emit_table_challenges` samples, so nothing downstream + /// in that fork would disagree either. The gate that holds this is + /// `per_table_aggregator_tests::a_zero_bit_query_draw_consumes_what_the_host_does`, + /// which samples an extension element AFTER the zero-bit draw on both sides — + /// the only place the missing squeeze shows up. + pub fn sample_u64_pow2(&mut self, b: &mut LfmBuilder, nbits: usize) -> Vec { + assert!( + nbits <= 32, + "sample_u64_pow2: nbits must be at most 32, got {nbits} — above 32 the \ + answer would span both halves of the candidate" + ); + if Self::is_p1(b) { + // `P1Transcript::sample_u64`: one squeezed felt, canonical, masked — + // the low `nbits` of its canonical decomposition. + let mut sponge = self.drive_p1(b); + let f = sponge.squeeze(b); + self.p1 = Some(sponge); + return b.bit_dec(f, nbits); + } + let Some(h) = b.wrap_hash().byte_hash() else { + // `sample_u64` at a power-of-two bound is `canonical(cell[0]) & + // (bound − 1)` — the low `nbits` of lane 0, which is exactly what + // `squeeze_bits` decomposes. Constant consumption either way: one + // cell, no rejection. + let mut sponge = self.drive_chain(b); + let bits = sponge.squeeze_bits(b, nbits); + self.sponge = Some(sponge); + return bits; + }; + let c = self.next_candidate(b, h); + b.bit_dec(c.lo, nbits) + } + + /// `DefaultTranscript::state()` — the sponge's digest RIGHT NOW, without + /// advancing it (production finalizes a CLONE of the hasher, + /// `default_transcript.rs:128-130`). + /// + /// Only grinding needs this: the seed it hashes is the state before the + /// nonce is absorbed, and the nonce is then absorbed into the live + /// transcript. Neither the segment nor the output buffer moves here, so a + /// later `sample` still hashes `segment ‖ nonce` exactly as production does. + /// + /// The segment is packed twice as a result (once here, once at that later + /// squeeze). Packing is free for an aligned segment — which this one is, + /// every caller reaching grinding through a `sample` — and a re-emitted + /// splice would only be redundant work, never a different value. + pub fn state(&mut self, b: &mut LfmBuilder) -> edsl::WrapDigest { + if Self::is_p1(b) { + // `P1Transcript::state`: lanes 0..4 after flushing a COPY. + let sponge = self.drive_p1(b); + let cell = sponge.digest(b); + self.p1 = Some(sponge); + return edsl::WrapDigest::from_cell(cell); + } + let Some(h) = b.wrap_hash().byte_hash() else { + // The chain's state cell, which is what `AlgebraicTranscript::state` + // serialises. Draining the pending appends here is not the extra + // step the byte arm's double pack is — the chain absorbs them once + // and a later squeeze sees the same state, so the two arms agree + // that `state()` observes without advancing. + let sponge = self.drive_chain(b); + let cell = sponge.state(); + self.sponge = Some(sponge); + return edsl::WrapDigest::from_cell(cell); + }; + let packed = self.pack_segment(b); + edsl::wrap_hash_bytes(b, h, &packed, self.segment_len) + } + + /// Emit-time buffer position, for tests that pin the consumption schedule. + pub fn out_pos(&self) -> usize { + self.out_pos + } + + /// Emit-time segment length in bytes, for the same reason. + pub fn segment_len(&self) -> usize { + self.segment_len + } +} + +/// Constrains a candidate to be a canonical field element, i.e. `< p`. +/// +/// `p = 2^64 − 2^32 + 1`, so `p − 1 = (2^32 − 1)·2^32` and +/// `p = (2^32 − 1)·2^32 + 1`. For `candidate = hi·2^32 + lo` with both halves +/// below `2^32`: +/// +/// - `hi < 2^32 − 1` ⇒ `candidate ≤ (2^32 − 2)·2^32 + (2^32 − 1)` +/// `= (2^32 − 1)·2^32 − 1 < p`, always in range; +/// - `hi = 2^32 − 1` ⇒ `candidate = (p − 1) + lo`, in range iff `lo = 0`. +/// +/// So `candidate ≥ p ⟺ hi = 2^32 − 1 ∧ lo ≠ 0`. (The `LFM_BITDEC` chip proves +/// canonicity of a 64-bit decomposition with the same predicate over the top and +/// bottom 32 bits — see `chips::bitdec`.) +/// +/// The guard is then a single division. `g = (2^32 − 1) − hi` is zero exactly +/// when `hi = 2^32 − 1`, and `LFM_BALU` constrains division as +/// `SEL_DIV·(B·OUT − A) = 0`: with `B = 0` that reads `−A = 0`, forcing `A = 0` +/// and leaving `OUT` free. So `div(lo, g)` is provable iff `g ≠ 0 ∨ lo = 0` — +/// the exact negation of the reject condition, in one instruction with nothing +/// hinted and nothing to verify. It is the same assert-via-division mechanism +/// `LfmBuilder::assert_eq` is built from. +/// +/// Both halves must be canonical `u32`s for the derivation to hold. They are: +/// they come from `Unpack` of a hash chip's output word, and both chips +/// recompose each `u32` from four BITWISE-range-checked byte columns — +/// `LFM_KECCAK` (`keccak_rejects_non_u32_half`) and `LFM_BLAKE3` alike. A +/// half that reached here through a `Select` is one of those values by the +/// select chip's own semantics. +pub fn assert_canonical(b: &mut LfmBuilder, c: Candidate) { + let hi_max = b.felt_const(FE::from(HI_MAX)); + let g = b.sub(hi_max, c.hi); + let _ = b.div(c.lo, g); +} + +/// `hi·2^32 + lo` as a field element. +/// +/// Only equal to the candidate's INTEGER value once [`assert_canonical`] has +/// pinned that value below `p`; without the guard this silently wraps (a +/// candidate of `p` becomes `0`). +pub fn candidate_to_felt(b: &mut LfmBuilder, c: Candidate) -> Felt { + let two32 = b.felt_const(FE::from(1u64 << 32)); + b.mul_add(c.hi, two32, c.lo) +} + +// =============================== the packer =============================== + +/// The half currently under construction. +enum Partial { + /// Its bytes so far, all compile-time. Always fewer than four. + Const(Vec), + /// A machine value occupying the LOW `filled` bytes of the half, with + /// `filled` in `1..4`. Its unfilled high bytes are zero, so completing it is + /// an addition rather than an or. + Mixed(Felt, usize), +} + +/// Emits a segment's `u32` halves from a byte-granular walk of its pieces. +struct Packer { + out: Vec, + partial: Partial, +} + +/// The little-endian value of up to four bytes. +fn le_value(bytes: &[u8]) -> u64 { + bytes + .iter() + .enumerate() + .fold(0u64, |acc, (i, &v)| acc | (u64::from(v) << (8 * i))) +} + +impl Packer { + fn filled(&self) -> usize { + match &self.partial { + Partial::Const(v) => v.len(), + Partial::Mixed(_, f) => *f, + } + } + + fn push_const(&mut self, b: &mut LfmBuilder, bytes: &[u8]) { + for &byte in bytes { + match core::mem::replace(&mut self.partial, Partial::Const(Vec::new())) { + Partial::Const(mut v) => { + v.push(byte); + if v.len() == BYTES_PER_HALF { + let c = b.felt_const(FE::from(le_value(&v))); + self.out.push(c); + v.clear(); + } + self.partial = Partial::Const(v); + } + Partial::Mixed(m, filled) => { + // The byte lands above what is already there, and the high + // bytes are zero, so `add` is exactly an or. + let w = b.felt_const(FE::from(u64::from(byte) << (8 * filled))); + let m = b.add(m, w); + if filled + 1 == BYTES_PER_HALF { + self.out.push(m); + self.partial = Partial::Const(Vec::new()); + } else { + self.partial = Partial::Mixed(m, filled + 1); + } + } + } + } + } + + /// Merges `v` into the half under construction at byte offset `filled`. + /// The destination's higher bytes are zero, so `mul_add` is exactly an or. + fn merge(&mut self, b: &mut LfmBuilder, v: Felt, filled: usize) -> Felt { + let shift = b.felt_const(FE::from(1u64 << (8 * filled))); + match core::mem::replace(&mut self.partial, Partial::Const(Vec::new())) { + Partial::Const(c) => { + let base = b.felt_const(FE::from(le_value(&c))); + b.mul_add(v, shift, base) + } + Partial::Mixed(m, _) => b.mul_add(v, shift, m), + } + } + + /// Places a machine value carrying `nbytes` live bytes at the cursor. + /// + /// One routine covers both a whole half (`nbytes == 4`) and the masked tail + /// of an odd-length byte string, because they differ only in width. + fn push_partial(&mut self, b: &mut LfmBuilder, v: Felt, nbytes: usize) { + debug_assert!((1..=BYTES_PER_HALF).contains(&nbytes)); + let filled = self.filled(); + if filled == 0 { + // Nothing to merge with: `v` already sits in the low bytes of a + // fresh half and its high bytes are zero. No instructions — the path + // every aligned program takes, which must stay free or every + // registered digest moves. + if nbytes == BYTES_PER_HALF { + self.out.push(v); + } else { + self.partial = Partial::Mixed(v, nbytes); + } + return; + } + let room = BYTES_PER_HALF - filled; + if nbytes < room { + let merged = self.merge(b, v, filled); + self.partial = Partial::Mixed(merged, filled + nbytes); + } else if nbytes == room { + let merged = self.merge(b, v, filled); + self.out.push(merged); + self.partial = Partial::Const(Vec::new()); + } else { + // Crosses the boundary: the low `room` bytes finish this half and + // the rest opens the next. + let (lo, hi) = split_half(b, v, room); + let merged = self.merge(b, lo, filled); + self.out.push(merged); + self.partial = Partial::Mixed(hi, nbytes - room); + } + } + + fn push_half(&mut self, b: &mut LfmBuilder, d: Felt) { + self.push_partial(b, d, BYTES_PER_HALF); + } + + /// Masks a trailing half to its `nbytes` live bytes and PINS the rest to + /// zero, then places it. + /// + /// The zero-pin is a soundness obligation, not tidiness: the high bytes of + /// an arena-supplied felt are otherwise unconstrained, and without it a + /// prover could put arbitrary content there. Those bytes are past the + /// encoding's length prefix, so they would change the absorbed byte string + /// while the length said otherwise. + fn push_masked(&mut self, b: &mut LfmBuilder, v: Felt, nbytes: usize) { + let (lo, hi) = split_half(b, v, nbytes); + let zero = b.felt_const(FE::zero()); + b.assert_eq(hi, zero); + self.push_partial(b, lo, nbytes); + } + + fn finish(mut self, b: &mut LfmBuilder) -> Vec { + match self.partial { + // A trailing partial half's unused high bytes are zero either way, + // which is the property `edsl::keccak256` needs to merge the padding + // constant with an `add`. + Partial::Const(v) if v.is_empty() => {} + Partial::Const(v) => { + let c = b.felt_const(FE::from(le_value(&v))); + self.out.push(c); + } + Partial::Mixed(m, _) => self.out.push(m), + } + self.out + } +} + +/// Splits a `u32` half into its low `k` bytes and its high `4 − k` bytes. +/// +/// This is the byte-level splice the misaligned statement encoding needs. A byte +/// split is not field arithmetic, so it goes through the canonical bit +/// decomposition and two weighted sums over disjoint bit ranges. +/// +/// The recomposition assert is load-bearing, not a belt: `bit_dec` bounds its +/// input by `p`, not by `2^32`, and a "half" at or above `2^32` has no four-byte +/// rendering at all. Pinning `d = lo + hi·2^(8k)` forces `d < 2^32` and the +/// split's correctness in the same constraint. +/// +/// Cost: one `LFM_BITDEC` row and ~33 `LFM_BALU` rows per spliced half. It only +/// ever runs on the statement leg — a few dozen halves per proof — and never in +/// FRI or Merkle traffic. +pub fn split_half(b: &mut LfmBuilder, d: Felt, k: usize) -> (Felt, Felt) { + assert!( + (1..BYTES_PER_HALF).contains(&k), + "split_half: k must be in 1..4, got {k}" + ); + let bits = b.bit_dec(d, 8 * BYTES_PER_HALF); + let lo = edsl::bits_to_felt(b, &bits[..8 * k]); + let hi = edsl::bits_to_felt(b, &bits[8 * k..]); + let shift = b.felt_const(FE::from(1u64 << (8 * k))); + let recomposed = b.mul_add(hi, shift, lo); + b.assert_eq(d, recomposed); + (lo, hi) +} + +/// The two `u32` halves of a base felt's 8-byte BIG-endian rendering — what +/// `append_field_element` puts on the wire, expressed in the machine's +/// little-endian half convention. +/// +/// ## The derivation +/// +/// Write `v = hi·2^32 + lo`. Big-endian, `v`'s bytes are `hi`'s four bytes +/// most-significant-first, then `lo`'s. Half `h` of the segment is the LE `u32` +/// of segment bytes `4h..4h+4`, so +/// +/// - half 0 = `byteswap32(hi)` — the HIGH word leads in big-endian order, +/// - half 1 = `byteswap32(lo)`. +/// +/// A byte swap is not field arithmetic, so it goes through the canonical bit +/// decomposition: bit `j` of byte `k` must land at bit `j` of byte `3 − k`, +/// which is just a different constant weight per bit. Each half is one +/// 32-term linear form over the row's bit columns — and since the chip sends +/// those forms on the bus directly ([`LfmBuilder::bit_dec_be_halves`]), the +/// whole byte permutation lives in the BUS COEFFICIENTS rather than in any +/// emitted instruction. +/// +/// ## Cost +/// +/// One `LFM_BITDEC` row. Nothing else: the 64 base-ALU recomposition rows +/// this gadget used to cost — 77% of the whole wrap program at the real +/// epoch, measured 2026-08-21 — are gone, which is the point. +/// +/// The row also enforces canonicity (`< p`), which is exactly right: +/// production renders `canonical_u64()`. +/// +/// Note for callers re-absorbing a value the transcript just produced: a +/// challenge from [`TranscriptReplay::sample_felt`] arrives as a recomposed +/// `Felt` and is decomposed again here. That round trip is one redundant +/// `BitDec`; carrying the halves through would avoid it, and is worth doing only +/// if a profile says so. +pub fn felt_be_halves(b: &mut LfmBuilder, v: Felt) -> [Felt; 2] { + b.bit_dec_be_halves(v) +} + +/// The payload cells the host's `append_bytes` produces, computed from the `u32` +/// halves the machine holds — the inverse direction of [`felt_be_halves`]. +/// +/// ★ **This is the ONLY place the algebraic arm still pays for a byte encoding, +/// and what does NOT reach it is the point.** A commitment root costs nothing: +/// the host absorbs it with `append_bytes(root)`, whose payload cell is +/// `AlgebraicTranscript::bytes_to_cell(root)`, and an algebraic backend +/// serialises its digest as four canonical big-endian felts — so that call +/// recovers exactly the digest cell the backend started from and the +/// serialisation CANCELS. What arrives here is the genuinely byte-shaped data: +/// a keccak ELF digest, a `public_output` run, a big-endian nonce. +/// +/// ⚠ The cancellation's precondition is the backend's canonicity, exactly as +/// `bytes_to_cell` states it — a non-canonical eight-byte group reduces, and +/// reduction is what would make two different roots absorb identically. The same +/// reduction applies here, and it applies to both sides: this gadget accumulates +/// in the field and `bytes_to_cell` calls `FE::from(u64)`, so a group at or above +/// `p` reduces the same way on the host and in the machine. +/// +/// ## The encoding, byte for byte +/// +/// Half `h` carries stream bytes `4h..4h+4` LITTLE-endian, so byte `t` of that +/// half is bits `[8t, 8t+8)` of it — and the trailing half is decomposed to its +/// LIVE bytes only, which is precisely the zero-pinning +/// [`TranscriptReplay::append_bytes_misaligned`] promises rather than a separate +/// mask. Each output felt is one 8-byte group read BIG-endian and LEFT-justified +/// when the group is short, because `bytes_to_cell` reads a zero-filled buffer +/// whose live bytes sit at the low indices. +/// +/// ## Cost, and the cheaper route deliberately not taken +/// +/// One `LFM_BITDEC` row per half, plus ~63 `LFM_BALU` rows per output felt. +/// Hinting the felt and asserting `bit_dec_be_halves(felt) == [hi, lo]` would be +/// roughly 30× cheaper — one row instead of a decomposition and a recomposition +/// — and it is the right move IF a profile ever asks for it. It is not taken +/// here because it costs an arena word per felt, i.e. a host-side layout change, +/// to save on the order of a thousand rows per epoch statement against a +/// 45.7M-instruction aggregation program. Written down so the trade is on record +/// rather than rediscovered. +fn cells_from_halves_be(b: &mut LfmBuilder, halves: &[Felt], byte_len: usize) -> Vec { + assert_eq!( + halves.len(), + byte_len.div_ceil(BYTES_PER_HALF), + "byte_len must match the supplied halves" + ); + + let mut bytes: Vec = Vec::with_capacity(byte_len); + for (h, half) in halves.iter().enumerate() { + let live = (byte_len - h * BYTES_PER_HALF).min(BYTES_PER_HALF); + let bits = b.bit_dec(*half, 8 * live); + for t in 0..live { + bytes.push(edsl::bits_to_felt(b, &bits[8 * t..8 * t + 8])); + } + } + + let byte_radix = b.felt_const(FE::from(1u64 << 8)); + let mut felts: Vec = Vec::with_capacity(bytes.len().div_ceil(8)); + for group in bytes.chunks(8) { + let mut acc = group[0]; + for byte in &group[1..] { + acc = b.mul_add(acc, byte_radix, *byte); + } + if group.len() < 8 { + let shift = b.felt_const(FE::from(1u64 << (8 * (8 - group.len())))); + acc = b.mul(acc, shift); + } + felts.push(acc); + } + + // A cell is four felts and a short final cell is zero-filled, again matching + // `bytes_to_cell`'s buffer rather than restating a convention. + let zero = b.felt_const(FE::zero()); + while !felts.len().is_multiple_of(4) { + felts.push(zero); + } + felts + .chunks(4) + .map(|c| b.pack_word([c[0], c[1], c[2], c[3]])) + .collect() +} + +/// Per-candidate probability that the production sampler rejects: there are +/// `2^64 − p = 2^32 − 1` out-of-range values among the `2^64` a candidate can +/// take. +/// +/// Only `sample_field_element` draws are exposed — `sample_u64` at a power-of-two +/// bound never rejects. +pub fn reject_probability_per_candidate() -> f64 { + ((1u64 << 32) - 1) as f64 / 2f64.powi(64) +} + +/// Upper bound on the probability that a transcript with `base_draws` base-field +/// challenge draws rejects at least once — i.e. that the emitted zero-rejection +/// program cannot prove it. +/// +/// A cubic-extension challenge is THREE base draws, so pass `3 · ext_draws`. +/// The union bound is what makes this an upper bound; the exact value is +/// `1 − (1 − q)^n`, indistinguishable at these magnitudes. +pub fn reject_probability_per_proof(base_draws: usize) -> f64 { + base_draws as f64 * reject_probability_per_candidate() +} + +/// Packs a piece list into `u32` halves, walking it at BYTE granularity. +/// +/// Constant bytes accumulate host-side; a machine half drops straight in when +/// the cursor is 4-byte aligned — the path every aligned program takes, which +/// must stay instruction-free — and is split when it is not. The packer is the +/// only place that knows the cursor, which is why the splice lives here rather +/// than at the append. +fn pack_pieces(pieces: &[SegPiece], b: &mut LfmBuilder) -> Vec { + let mut p = Packer { + out: Vec::new(), + partial: Partial::Const(Vec::new()), + }; + for piece in pieces { + match piece { + SegPiece::Const(bytes) => p.push_const(b, bytes), + SegPiece::Halves(halves) => { + for h in halves { + p.push_half(b, *h); + } + } + SegPiece::Partial(v, nbytes) => p.push_masked(b, *v, *nbytes), + } + } + p.finish(b) +} + +/// A structured byte string of compile-time constants and machine values, +/// hashed directly rather than absorbed into a transcript. +/// +/// Same byte-granular packer the transcript's segments use — one implementation, +/// so the splice semantics cannot drift between the two callers. This exists for +/// folds like the recursion attestation's `program_id`, which is a plain +/// `keccak256` over `tag ‖ fields`, not a Fiat-Shamir absorb. +/// +/// The alignment lesson from R1e applies unchanged and is why this is not just +/// "concatenate then hash": alignment is a property of the CURSOR, not of the +/// field. `PROGRAM_ID_TAG` is 22 bytes, so every machine value after it lands +/// mid-half and must be spliced. +#[derive(Default)] +pub struct ByteString { + pieces: Vec, + len: usize, +} + +impl ByteString { + pub fn new() -> Self { + Self::default() + } + + /// Append compile-time constant bytes, any length, any alignment. + pub fn push_const(&mut self, bytes: &[u8]) { + self.pieces.push(SegPiece::Const(bytes.to_vec())); + self.len += bytes.len(); + } + + /// Append `4 · halves.len()` machine-computed bytes at any alignment. + pub fn push_halves(&mut self, halves: &[Felt]) { + self.pieces.push(SegPiece::Halves(halves.to_vec())); + self.len += BYTES_PER_HALF * halves.len(); + } + + /// Append `byte_len` machine-computed bytes carried in + /// `ceil(byte_len / 4)` halves, masking the trailing partial half. + pub fn push_bytes(&mut self, halves: &[Felt], byte_len: usize) { + assert_eq!( + halves.len(), + byte_len.div_ceil(BYTES_PER_HALF), + "byte_len must match the supplied halves" + ); + let full = byte_len / BYTES_PER_HALF; + let rem = byte_len % BYTES_PER_HALF; + if full > 0 { + self.pieces.push(SegPiece::Halves(halves[..full].to_vec())); + } + if rem > 0 { + self.pieces.push(SegPiece::Partial(halves[full], rem)); + } + self.len += byte_len; + } + + /// Bytes the string will hash — its own accounting, so a test can pin the + /// resulting alignment rather than trust prose. + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// ⛔ `keccak256` over the assembled bytes, **pinned** — it does not follow + /// the configured wrap hash. + /// + /// For folds whose host counterpart names `PlatformKeccak256` explicitly + /// rather than the configuration's hash. There is exactly one such caller, + /// `programs::emit_program_id`, mirroring + /// `recursion::program_id_from_digest`; switching it would make the + /// attestation join disagree with every host consumer of a `program_id`. + /// + /// ★ This method and [`ByteString::wrap_hash`] are the reason `ByteString` + /// needed a SECOND hashing method rather than a substitution. The naive + /// sweep — "replace every `edsl::keccak256` call" — produces a wrong proof + /// exactly here, because grinding and `program_id` share this type and have + /// opposite fates. + pub fn keccak256(&self, b: &mut LfmBuilder) -> [Cell; DIGEST_WORDS] { + let packed = pack_pieces(&self.pieces, b); + edsl::keccak256(b, &packed, self.len) + } + + /// The CONFIGURED wrap hash over the assembled bytes. + /// + /// For folds that follow the configuration — grinding, whose host side + /// reaches the digest through `GrindingDigest` (P-a Stage 3, + /// `crypto/stark/src/config.rs`) and therefore moves with `H`. + /// + /// ⚠ Takes the byte hash rather than reading the builder's, because a + /// `ByteString` IS a byte stream: there is no algebraic reading of one, and + /// the caller that has an algebraic configuration must have handled it + /// before assembling any bytes at all. The parameter is where that shows up. + pub fn wrap_hash(&self, b: &mut LfmBuilder, h: ByteWrapHash) -> edsl::WrapDigest { + let packed = pack_pieces(&self.pieces, b); + edsl::wrap_hash_bytes(b, h, &packed, self.len) + } +} + +/// Candidates one base coordinate draws, under the configuration `hash` names. +/// +/// Read from `crypto`'s own `TranscriptHash` impls rather than restated: the +/// schedule is half of what a transcript configuration IS, and a machine that +/// replayed a different one would produce challenges the host never drew. +/// +/// `None` is the host's UNBOUNDED rejection loop. Its modal cost is one +/// candidate and a straight-line machine cannot emit an unbounded loop, so the +/// emission is one draw plus a canonicity constraint — see +/// [`TranscriptReplay::sample_felt`] for exactly what that costs. +fn candidates_per_coordinate(hash: WrapHash) -> usize { + use crypto::fiat_shamir::transcript_hash::{ + Blake3TranscriptHash, KeccakTranscriptHash, TranscriptHash, + }; + let schedule = match hash { + WrapHash::Keccak => KeccakTranscriptHash::CANDIDATES_PER_COORDINATE, + WrapHash::Blake3 => Blake3TranscriptHash::CANDIDATES_PER_COORDINATE, + // ★ One candidate, GUARANTEED rather than probabilistic: an algebraic + // squeeze returns felts canonical by construction, so a carved `u64` + // cannot miss (`SOUNDNESS.md` §6.4). + WrapHash::Algebraic => { + super::algebraic_commit::AlgebraicRpxTranscriptHash::CANDIDATES_PER_COORDINATE + } + // ZisK's transcript squeezes canonical felts too: one per coordinate. + WrapHash::Poseidon1 => { + ::CANDIDATES_PER_COORDINATE + } + }; + schedule.map_or(1, core::num::NonZeroUsize::get) +} + +/// Whether a candidate is a canonical field element, as a BIT. +/// +/// The predicate form of [`assert_canonical`], and strictly more work: that one +/// is a single division whose provability IS the answer, which is enough when +/// the answer must be yes and useless when the answer has to be selected on. +/// +/// `candidate ≥ p ⟺ hi = 2^32 − 1 ∧ lo ≠ 0` (the derivation is in +/// [`assert_canonical`]). So with `a = [hi = 2^32 − 1]`, the product `a · lo` is +/// zero exactly when the candidate is in range — one `is_zero` rather than two, +/// because `a` collapses the conjunction. `a · lo < 2^32` since `a` is boolean +/// and `lo` is a `u32` half, which is the bound +/// [`edsl::is_zero_bounded`] needs. +/// +/// Cost: two `LFM_BITDEC` rows and ~95 `LFM_BALU` rows per coordinate, against +/// the two rows [`assert_canonical`] costs. Small next to the squeeze that +/// produced the candidate (a keccak permutation is 24 `KECCAK_RND` rows of +/// 1,480 columns), but it is not nothing and it is paid per coordinate. +pub fn candidate_in_range(b: &mut LfmBuilder, c: Candidate) -> Bit { + let hi_is_max = edsl::is_all_ones_bounded(b, c.hi, 32); + let witness = b.mul(hi_is_max.as_felt(), c.lo); + edsl::is_zero_bounded(b, witness, 32) +} diff --git a/prover/src/lfm/transcript_tests.rs b/prover/src/lfm/transcript_tests.rs new file mode 100644 index 000000000..5ff6ee791 --- /dev/null +++ b/prover/src/lfm/transcript_tests.rs @@ -0,0 +1,756 @@ +//! The compress-chain Fiat–Shamir transcript (option B1): its vectors, its +//! domain separation, its cost, and the machine that computes it. +//! +//! ## What pins what +//! +//! Four layers, deliberately different evidence: +//! +//! 1. **The step function** is `blake3::hash(state ‖ operand ‖ "LFMT")` +//! truncated — asserted against the *crate*, not against an oracle, so the +//! external anchor the 7-round decision was bought for is inherited rather +//! than claimed. +//! 2. **The vectors** are [`super::transcript_kats`], rendered from a Python +//! reference the oracle wrote before any of this Rust existed. Both round +//! counts, per-op and end-to-end. +//! 3. **The host chain** (`fixture::HostSponge`) reproduces them, which is what +//! "bit-exact mirror" has to mean to be checkable. +//! 4. **The machine** (`edsl::SpongeVar` through `LFM_HASH`) reproduces the same +//! challenges *inside a proof the production verifier accepts* — the layer +//! that would catch a host and chip that agree with the spec separately and +//! with each other not at all. +//! +//! Every rejection test here is paired with an honest-path assertion. A test +//! that only checks "the bad thing is rejected" passes just as well when +//! everything is rejected, which is the failure mode a soundness fix has. + +use crate::tables::types::{FE, FEE, GoldilocksField}; +use math::field::traits::IsPrimeField; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use super::blake3_socket::{ + SOCKET_ROUNDS, TAG_LFMT, leaf_digest_rounds, socket_digest_rounds, transcript_digest, + transcript_digest_rounds, word_of, +}; +use super::builder::{Cell, LfmBuilder, LfmProgramSource}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::{SQUEEZE_MARK, SpongeVar}; +use super::executor::execute; +use super::fixture::HostSponge; +use super::hash::HasherKind; +use super::instr::{HashMode, Instr}; +use super::proof::{lfm_prove_with_hasher, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::transcript_kats::{ + EndToEndVector, FRI_TOY_6, FRI_TOY_7, FRI_TOY_COMPRESSIONS, L1_ROOT, MAIN_ROOT, STEP_VECTORS, + T0W, T1W, +}; +use super::word::LfmWord; + +const KIND: HasherKind = HasherKind::Blake3; + +/// Query shape of the `FriToyV0` preamble the end-to-end vector is shaped like +/// (✓ VERIFIED `fixture::shape`). +const NUM_QUERIES: usize = 4; +const QUERY_BITS: usize = 4; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// The end-to-end vector for the round count this build compiled. +fn compiled_vector() -> &'static EndToEndVector { + if SOCKET_ROUNDS == 7 { + &FRI_TOY_7 + } else { + &FRI_TOY_6 + } +} + +fn lanes(w: &LfmWord) -> [u32; 4] { + core::array::from_fn(|i| { + u32::try_from(GoldilocksField::canonical(w[i].value())).expect("a u32 lane") + }) +} + +// ========================================================================= +// K1 — the step function, and its external anchor +// ========================================================================= + +/// The per-op vectors, at BOTH round counts in one run. +#[test] +fn every_step_vector_reproduces_at_both_round_counts() { + for v in STEP_VECTORS.iter() { + assert_eq!( + transcript_digest_rounds(&v.state, &v.operand, 6), + v.result_6, + "6-round step vector {}", + v.name + ); + assert_eq!( + transcript_digest_rounds(&v.state, &v.operand, 7), + v.result_7, + "7-round step vector {}", + v.name + ); + } +} + +/// The compiled-in entry point agrees with the vector for its round count — so +/// the knob and the table cannot drift apart. +#[test] +fn the_compiled_step_matches_its_round_counts_vectors() { + for v in STEP_VECTORS.iter() { + let expected = if SOCKET_ROUNDS == 7 { + v.result_7 + } else { + v.result_6 + }; + assert_eq!( + transcript_digest(&v.state, &v.operand), + expected, + "vector {}", + v.name + ); + } +} + +/// ★ **The external anchor, direct.** At 7 rounds a transcript step is +/// literally `blake3::hash(state ‖ operand ‖ 0^16 ‖ "LFMT")` truncated to 16 +/// bytes. +/// +/// The message is re-derived from the byte-level framing rather than from +/// `socket_message`, so the word-level and byte-level forms are two statements +/// that can disagree. This is the property option B was chosen for: the +/// transcript inherits the compress socket's anchor because the tag is the only +/// thing that moved — and it kept it through the leaf RATE's widening, because +/// 52 bytes is still one block. +#[test] +fn seven_rounds_is_blake3_of_the_transcript_message() { + for v in STEP_VECTORS.iter() { + let mut msg = Vec::with_capacity(52); + for lane in v.state.iter().chain(v.operand.iter()) { + msg.extend_from_slice(&lane.to_le_bytes()); + } + // The third input cell, pinned to zero on every row that does not read + // it — a transcript step reads two. + msg.extend_from_slice(&[0u8; 16]); + msg.extend_from_slice(b"LFMT"); + assert_eq!(msg.len(), 52, "a transcript step is one 52-byte block"); + + let full = blake3::hash(&msg); + let want: [u32; 4] = core::array::from_fn(|i| { + u32::from_le_bytes(full.as_bytes()[4 * i..4 * i + 4].try_into().unwrap()) + }); + assert_eq!( + transcript_digest_rounds(&v.state, &v.operand, 7), + want, + "7-round step {} must be blake3::hash of its message", + v.name + ); + assert_eq!(want, v.result_7, "the table itself agrees with the crate"); + } +} + +/// The tag word is the ASCII, little-endian — the one place a byte order slip +/// would silently redefine the domain. +#[test] +fn the_transcript_tag_is_lfmt_little_endian() { + assert_eq!(TAG_LFMT, u32::from_le_bytes(*b"LFMT")); + assert_eq!(TAG_LFMT.to_le_bytes(), *b"LFMT"); + assert_eq!(SQUEEZE_MARK.to_le_bytes(), *b"SQZ0"); +} + +// ========================================================================= +// K3 — domain separation, in both directions +// ========================================================================= + +/// A transcript step and a Merkle parent over the SAME two cells are different +/// digests. Without this the chain would be replayable as a tree and vice +/// versa, and the `MODE_T` column would be buying nothing. +#[test] +fn a_transcript_step_is_not_a_merkle_parent() { + for v in STEP_VECTORS.iter() { + for rounds in [6, 7] { + assert_ne!( + transcript_digest_rounds(&v.state, &v.operand, rounds), + socket_digest_rounds(&v.state, &v.operand, rounds), + "vector {} at {rounds} rounds: the tag is not separating the domains", + v.name + ); + } + } +} + +/// The honest-path control for the test above: the two ARE the same function +/// apart from the tag, so a bug that made them differ for some other reason +/// would show up here. +#[test] +fn the_two_domains_differ_only_in_the_tag() { + for v in STEP_VECTORS.iter() { + use super::blake3_socket::{TAG_LFMC, socket_digest_rounds_tagged}; + assert_eq!( + socket_digest_rounds_tagged(&v.state, &v.operand, 7, TAG_LFMT), + transcript_digest_rounds(&v.state, &v.operand, 7) + ); + assert_eq!( + socket_digest_rounds_tagged(&v.state, &v.operand, 7, TAG_LFMC), + socket_digest_rounds(&v.state, &v.operand, 7) + ); + } +} + +// ========================================================================= +// K2 — the end-to-end vector, host side +// ========================================================================= + +/// Replays the `FriToyV0` preamble against the reference step function at an +/// explicit round count, so BOTH vectors are checkable from one build. +/// +/// ✓ VERIFIED sequence, `programs::fri_toy_program_source`: absorb(main_root), +/// squeeze_ext, squeeze_ext, absorb(l1_root), squeeze_ext, **absorb_felts(t0w), +/// absorb_felts(t1w)**, then `NUM_QUERIES` × squeeze_bits. +/// +/// The last two are `absorb_felts`, not `absorb2`: the terminal coefficients are +/// field DATA, so they are leaf-hashed and the DIGEST is absorbed. The step +/// count is the same either way, which is exactly why this had to be re-pointed +/// deliberately rather than caught by a red test. +fn replay_reference(rounds: usize) -> (Vec<[u32; 4]>, Vec<[u32; 4]>, usize) { + let mut state = [0u32; 4]; + let mut squeeze_index = 0u32; + let mut compressions = 0usize; + let mut states = Vec::new(); + let mut outputs = Vec::new(); + + let absorb = |state: &mut [u32; 4], c: &[u32; 4], compressions: &mut usize| { + *state = transcript_digest_rounds(state, c, rounds); + *compressions += 1; + }; + let squeeze = |state: &mut [u32; 4], i: &mut u32, compressions: &mut usize| -> [u32; 4] { + let out = *state; + let sq = [SQUEEZE_MARK, *i, 0, 0]; + *state = transcript_digest_rounds(state, &sq, rounds); + *i += 1; + *compressions += 1; + out + }; + + absorb(&mut state, &MAIN_ROOT, &mut compressions); + states.push(state); + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + absorb(&mut state, &L1_ROOT, &mut compressions); + states.push(state); + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + // DATA, so each goes through the leaf encoding before it is absorbed. + for cell in [&T0W, &T1W] { + let felts: LfmWord = core::array::from_fn(|i| FE::from(u64::from(cell[i]))); + // From the chain start, exactly as `absorb_felts` does: one leaf row per + // data cell, absorbing four felts and chaining in the same compression. + let d = leaf_digest_rounds(&super::fixture::leaf_chain_start(), &felts, rounds) + .expect("the KAT inputs are canonical"); + absorb(&mut state, &d, &mut compressions); + states.push(state); + } + for _ in 0..NUM_QUERIES { + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + } + (states, outputs, compressions) +} + +fn check_end_to_end(rounds: usize, want: &EndToEndVector) { + let (states, outputs, compressions) = replay_reference(rounds); + assert_eq!(states.len(), want.states.len()); + for (i, (got, expected)) in states.iter().zip(want.states.iter()).enumerate() { + assert_eq!(got, expected, "state after op {i} at {rounds} rounds"); + } + // Challenges are read off the PRE-advance outputs, not off the states. + let ext = |o: &[u32; 4]| [o[0], o[1], o[2]]; + assert_eq!(ext(&outputs[0]), want.alpha, "alpha at {rounds} rounds"); + assert_eq!(ext(&outputs[1]), want.zeta0, "zeta0 at {rounds} rounds"); + assert_eq!(ext(&outputs[2]), want.zeta1, "zeta1 at {rounds} rounds"); + for (q, bits) in want.query_bits.iter().enumerate() { + let lane0 = outputs[3 + q][0]; + let got: [u8; QUERY_BITS] = core::array::from_fn(|k| ((lane0 >> k) & 1) as u8); + assert_eq!(&got, bits, "query {q} bits at {rounds} rounds"); + } + // The reference replay counts TRANSCRIPT steps; the oracle's constant counts + // the leaf rows too, so the two differ by exactly the two data absorbs. + assert_eq!( + compressions + 2, + FRI_TOY_COMPRESSIONS, + "the preamble's compression count is a cost claim, not an accident" + ); +} + +/// K2 at 7 rounds — the default build. +#[test] +fn the_end_to_end_vector_reproduces_at_seven_rounds() { + check_end_to_end(7, &FRI_TOY_7); +} + +/// K2 at 6 rounds — the `blake3-6round` variant, pinned unconditionally. +#[test] +fn the_end_to_end_vector_reproduces_at_six_rounds() { + check_end_to_end(6, &FRI_TOY_6); +} + +/// ★ The HOST CHAIN — `fixture::HostSponge`, the thing the fixture prover and +/// every host-side replay use — reproduces the vector op for op. +/// +/// This is the mirror property stated as something checkable. It runs at the +/// compiled-in round count, which is why the two `check_end_to_end` tests above +/// carry the other one. +#[test] +fn the_host_sponge_reproduces_the_end_to_end_vector() { + let want = compiled_vector(); + let mut sponge = HostSponge::with_hasher(KIND); + let cell = |w: &[u32; 4]| word_of(w); + let mut states = Vec::new(); + + sponge.absorb(&cell(&MAIN_ROOT)); + states.push(sponge.state()); + let alpha = sponge.squeeze_ext(); + states.push(sponge.state()); + let zeta0 = sponge.squeeze_ext(); + states.push(sponge.state()); + sponge.absorb(&cell(&L1_ROOT)); + states.push(sponge.state()); + let zeta1 = sponge.squeeze_ext(); + states.push(sponge.state()); + sponge.absorb_felts(&felts_of(&T0W)); + states.push(sponge.state()); + sponge.absorb_felts(&felts_of(&T1W)); + states.push(sponge.state()); + let mut queries = Vec::new(); + for _ in 0..NUM_QUERIES { + queries.push(sponge.squeeze_index(QUERY_BITS)); + states.push(sponge.state()); + } + + for (i, (got, expected)) in states.iter().zip(want.states.iter()).enumerate() { + assert_eq!(lanes(got), *expected, "host state after op {i}"); + } + + let ext_lanes = |e: &FEE| -> [u32; 3] { + let v = e.value(); + core::array::from_fn(|i| { + u32::try_from(GoldilocksField::canonical(v[i].value())).expect("a u32 lane") + }) + }; + assert_eq!(ext_lanes(&alpha), want.alpha); + assert_eq!(ext_lanes(&zeta0), want.zeta0); + assert_eq!(ext_lanes(&zeta1), want.zeta1); + for (q, bits) in want.query_bits.iter().enumerate() { + let index: u64 = bits + .iter() + .enumerate() + .map(|(k, &b)| u64::from(b) << k) + .sum(); + assert_eq!(queries[q], index, "host query {q}"); + } +} + +// ========================================================================= +// K4/K5 — the counter and the ordering are load-bearing +// ========================================================================= + +/// Without the counter every advance uses ONE fixed operand, so a run of +/// squeezes iterates one fixed map — precisely the structure the FSE-2014 +/// T-sponge attacks exploit. The vectors must notice. +#[test] +fn the_squeeze_counter_is_load_bearing() { + let rounds = 7; + let start = transcript_digest_rounds(&[0; 4], &MAIN_ROOT, rounds); + + let mut with_counter = Vec::new(); + let mut s = start; + for i in 0..4u32 { + with_counter.push(s); + s = transcript_digest_rounds(&s, &[SQUEEZE_MARK, i, 0, 0], rounds); + } + + let mut without = Vec::new(); + let mut s = start; + let fixed = [SQUEEZE_MARK, 0, 0, 0]; + for _ in 0..4 { + without.push(s); + s = transcript_digest_rounds(&s, &fixed, rounds); + } + + // Honest-path half: squeeze 0 uses counter 0, so the two MUST agree there. + // Without this the test would pass for a chain that simply produced noise. + assert_eq!( + with_counter[0], without[0], + "the first squeeze is the same either way — counter 0 is counter 0" + ); + assert_ne!( + with_counter[1..], + without[1..], + "counter-free squeezes must diverge: they iterate one fixed map" + ); +} + +/// Absorbing two cells in the other order is a different transcript. Cheap to +/// state, and the property every Fiat–Shamir argument silently assumes. +#[test] +fn absorb_order_is_load_bearing() { + let mut a = HostSponge::with_hasher(KIND); + a.absorb(&word_of(&MAIN_ROOT)); + a.absorb(&word_of(&L1_ROOT)); + + let mut b = HostSponge::with_hasher(KIND); + b.absorb(&word_of(&L1_ROOT)); + b.absorb(&word_of(&MAIN_ROOT)); + + assert_ne!(a.state(), b.state()); + + // Honest-path control: the same order gives the same state. + let mut c = HostSponge::with_hasher(KIND); + c.absorb(&word_of(&MAIN_ROOT)); + c.absorb(&word_of(&L1_ROOT)); + assert_eq!(a.state(), c.state()); +} + +// ========================================================================= +// K6 + the machine — the emitted program +// ========================================================================= + +/// A program shaped exactly like `FriToyV0`'s preamble, with the absorbed cells +/// as arena words so a test can feed the vector's inputs in. +/// +/// Its public output is every challenge the preamble derives, so a proof of it +/// carries the transcript's answers where a verifier can check them. +fn preamble_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(4); + let h: Vec = (0..4).map(|i| b.hint_word(arena, i)).collect(); + + let mut sponge = SpongeVar::new(&mut b); + sponge.absorb(&mut b, h[0]); + let alpha = sponge.squeeze_ext(&mut b); + let zeta0 = sponge.squeeze_ext(&mut b); + sponge.absorb(&mut b, h[1]); + let zeta1 = sponge.squeeze_ext(&mut b); + // The last two arena cells stand for the terminal coefficients — DATA — so + // the program absorbs them the way `FriToyV0` does. + sponge.absorb_felts(&mut b, h[2]); + sponge.absorb_felts(&mut b, h[3]); + + b.public(alpha.as_cell()); + b.public(zeta0.as_cell()); + b.public(zeta1.as_cell()); + for _ in 0..NUM_QUERIES { + let bits = sponge.squeeze_bits(&mut b, QUERY_BITS); + let index = super::edsl::bits_to_felt(&mut b, &bits); + b.public(index.as_cell()); + } + b.finish() +} + +fn preamble_program() -> LfmProgram { + compile(preamble_program_source()) +} + +fn preamble_arena() -> Vec> { + vec![vec![ + word_of(&MAIN_ROOT), + word_of(&L1_ROOT), + felts_of(&T0W), + felts_of(&T1W), + ]] +} + +/// The KAT's `u32` inputs read as FIELD ELEMENTS — what the leaf encoding +/// consumes. `word_of` reads the same values as digest lanes; both are the same +/// four numbers, and which reading applies is the mode's business. +fn felts_of(lanes: &[u32; 4]) -> LfmWord { + core::array::from_fn(|i| FE::from(u64::from(lanes[i]))) +} + +/// K6 — the preamble costs exactly the compressions the spec priced it at, and +/// every one of them is a TRANSCRIPT row rather than a Merkle one. +#[test] +fn the_preamble_costs_eleven_transcript_steps() { + let program = preamble_program(); + let modes: Vec = program + .instrs + .iter() + .filter_map(|i| match i { + Instr::Hash { mode, .. } => Some(*mode), + _ => None, + }) + .collect(); + let steps = modes.iter().filter(|m| **m == HashMode::Transcript).count(); + let leaves = modes.iter().filter(|m| **m == HashMode::Leaf).count(); + assert_eq!(steps, 11, "the transcript itself is 11 steps"); + assert_eq!(leaves, 2, "one leaf row per data cell absorbed"); + // The oracle's `FRI_TOY_COMPRESSIONS` counts BOTH kinds — it is the + // preamble's total socket cost, which is the number that closes `FriToyV0` + // at 93 (4 queries × 20 + 13). + assert_eq!( + steps + leaves, + FRI_TOY_COMPRESSIONS, + "the preamble costs {FRI_TOY_COMPRESSIONS} compressions in total" + ); + assert_eq!( + steps + leaves, + modes.len(), + "a transcript preamble emits transcript steps and leaf rows, nothing else" + ); +} + +/// ★ **The machine computes the specified transcript.** The emitted program, +/// executed under BLAKE3, produces the vector's challenges. +/// +/// This is the layer the host tests cannot reach: `SpongeVar` and `HostSponge` +/// are separate code, and this is where they are made to answer the same +/// question. +#[test] +fn the_machine_reproduces_the_end_to_end_vector() { + let want = compiled_vector(); + let program = preamble_program(); + let exec = execute(&program, &preamble_arena(), &KIND).expect("the preamble executes"); + + let public: Vec = exec.public_words.iter().map(|(_, w)| *w).collect(); + assert_eq!(public.len(), 3 + NUM_QUERIES); + + let ext3 = |w: &LfmWord| -> [u32; 3] { + let l = lanes(w); + [l[0], l[1], l[2]] + }; + assert_eq!(ext3(&public[0]), want.alpha, "alpha"); + assert_eq!(ext3(&public[1]), want.zeta0, "zeta0"); + assert_eq!(ext3(&public[2]), want.zeta1, "zeta1"); + for (q, bits) in want.query_bits.iter().enumerate() { + let index: u64 = bits + .iter() + .enumerate() + .map(|(k, &b)| u64::from(b) << k) + .sum(); + assert_eq!( + GoldilocksField::canonical(public[3 + q][0].value()), + index, + "query {q}" + ); + } +} + +/// The same program, PROVED under BLAKE3 and accepted by the production +/// verifier — the transcript is not merely computed, it is constrained. +#[test] +fn the_transcript_proves_and_verifies_under_blake3() { + let opts = options(); + let program = preamble_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &preamble_arena(), &opts, KIND) + .expect("a transcript program must prove under BLAKE3"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "an honest BLAKE3 transcript proof must verify" + ); + + // The public challenges are checked against the SPEC's vector, not against + // the executor — so the proof's outputs answer to the specification. + let want = compiled_vector(); + let alpha = lanes(&proved.public_words[0].1); + assert_eq!([alpha[0], alpha[1], alpha[2]], want.alpha); +} + +/// The same program under every hasher: B1 changed the transcript for ALL of +/// them, so all of them must still prove and verify. +#[test] +fn the_transcript_proves_and_verifies_under_every_hasher() { + let opts = options(); + let program = preamble_program(); + for kind in [HasherKind::Test, HasherKind::Rpx, HasherKind::Blake3] { + let artifacts = build_artifacts_with_hasher(&program, &opts, kind); + let proved = lfm_prove_with_hasher(&program, &artifacts, &preamble_arena(), &opts, kind) + .unwrap_or_else(|e| panic!("prove under {kind:?}: {e:?}")); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "an honest transcript proof must verify under {kind:?}" + ); + } +} + +/// The machine's chain and the host's chain agree under every hasher — the +/// property `fixture_prove` depends on and the one a rewrite of either side +/// would break. +#[test] +fn the_machine_and_the_host_chain_agree_under_every_hasher() { + let program = preamble_program(); + for kind in [HasherKind::Test, HasherKind::Rpx, HasherKind::Blake3] { + // BLAKE3 needs u32 lanes (O1); the vector's inputs are u32 either way, + // so one arena serves all three hashers. + let exec = execute(&program, &preamble_arena(), &kind).expect("executes"); + let public: Vec = exec.public_words.iter().map(|(_, w)| *w).collect(); + + let mut sponge = HostSponge::with_hasher(kind); + sponge.absorb(&word_of(&MAIN_ROOT)); + let alpha = sponge.squeeze_ext(); + let zeta0 = sponge.squeeze_ext(); + sponge.absorb(&word_of(&L1_ROOT)); + let zeta1 = sponge.squeeze_ext(); + sponge.absorb_felts(&felts_of(&T0W)); + sponge.absorb_felts(&felts_of(&T1W)); + + for (i, want) in [alpha, zeta0, zeta1].iter().enumerate() { + let v = want.value(); + for l in 0..3 { + assert_eq!(public[i][l], v[l], "{kind:?} challenge {i} lane {l}"); + } + } + for q in 0..NUM_QUERIES { + let index = sponge.squeeze_index(QUERY_BITS); + assert_eq!( + public[3 + q][0], + FE::from(index), + "{kind:?} query {q} index" + ); + } + } +} + +// ========================================================================= +// The cost claims the decision was made on +// ========================================================================= + +/// Hash rows in `program`, split by mode. +fn hash_row_modes(program: &LfmProgram) -> (usize, usize, usize) { + let mut compress = 0; + let mut transcript = 0; + let mut leaf = 0; + for i in &program.instrs { + if let Instr::Hash { mode, .. } = i { + match mode { + HashMode::Compress => compress += 1, + HashMode::Transcript => transcript += 1, + HashMode::Leaf => leaf += 1, + HashMode::Permute => panic!("no registered program may contain a permute"), + } + } + } + (compress, transcript, leaf) +} + +/// ★ The ratified cost claims, measured on the emitted programs. +/// +/// `leaf-spec/LEAF.md` §5 prices `TrivialV0` at **16,551** cell-equiv at 7 +/// rounds, which reproduces exactly. It prices `FriToyV0` at **502,047**, and +/// the built machine costs **513,081** — see the ⚠ below. Both are +/// `rows × cells_per_compression`, so this asserts the row counts and the +/// per-row price separately: a product that came out right for two wrong +/// reasons is the failure mode. +/// +/// ⚠ **The spec's `FriToyV0` figure rests on a premise that does not hold.** +/// §5 has "transcript unchanged at 11", but two of the four cells `FriToyV0` +/// absorbs are the terminal polynomial's COEFFICIENTS — arbitrary field +/// elements, not digests — so absorbing them raw hands the socket lanes that +/// are not `u32` and the row is unprovable. They now enter through the leaf +/// encoding (`SpongeVar::absorb_felts`), which adds **two `LFML` rows**: 93 +/// rows rather than 91. The transcript's own step count is unchanged at 11, so +/// the spec's sentence is right about the transcript and wrong about the total. +/// +/// ⚠ Both numbers MOVED with the leaf mode, and `TrivialV0`'s moved even though +/// its row count did not: the canonicity witness columns are part of the AIR, so +/// they exist on every compress row, leaf or not. Option B priced the same two +/// programs at 369,103 and 16,527 against a 5,509-cell row; the row went to +/// 5,517 and `FriToyV0` to 91 rows from 67, because each of its three data +/// leaves became two `LFML` rows and a parent. +/// +/// ★★ **And both moved again with the leaf RATE, in opposite directions — which +/// is the whole trade, priced.** The row grew by 16 columns / 28 census cells +/// (5,517 → 5,545) to carry four more lanes, and `FriToyV0` LOST twelve rows +/// (93 → 81): a data leaf is now one two-row `LFML` chain instead of two `LFML` +/// rows plus an `LFMC` fold, so each of the three leaves per query drops its +/// parent. −12.9% on the program against +0.5% on the row. +/// +/// ⚠ Do not read −12.9% as the tower's number. `FriToyV0` is Merkle-walk-heavy +/// at a toy width — 11 of its 17 per-query hashes are path steps the RATE does +/// not touch. What the RATE halves is leaf ABSORPTION, which is ~70% of a +/// recursion tower node's bill (COMMIT.md §1.4.1) and a rounding error here. +/// +/// The per-compression price is `blake3_socket_tests`' own census formula +/// (`main + 3·⌈interactions/2⌉`), 5,545 at 7 rounds and 4,777 at 6. +#[test] +fn the_programs_cost_what_the_leaf_spec_priced_them_at() { + const CELLS_PER_COMPRESSION_7R: usize = 5_545; + const CELLS_PER_COMPRESSION_6R: usize = 4_777; + let price = if SOCKET_ROUNDS == 7 { + CELLS_PER_COMPRESSION_7R + } else { + CELLS_PER_COMPRESSION_6R + }; + + // The price, from the census rather than from a literal. + let census = super::airs::lfm_chip_census_with_hasher( + &super::programs::trivial_program(), + HasherKind::Blake3, + ); + let hash_chip = census + .iter() + .find(|c| c.name == "LFM_HASH") + .expect("the census names the hash chip"); + assert_eq!( + hash_chip.main_cols + 3 * hash_chip.aux_cols, + price, + "the per-compression price must be the census's, not a literal" + ); + + // TrivialV0: three compressions, no transcript, no leaves. Its row COUNT + // is unchanged by the leaf mode and its PRICE is not — see the doc above. + let (c, t, l) = hash_row_modes(&super::programs::trivial_program()); + assert_eq!((c, t, l), (3, 0, 0)); + if SOCKET_ROUNDS == 7 { + assert_eq!((c + t + l) * price, 16_635, "TrivialV0 at 7 rounds"); + } + + // FriToyV0, per query: three data leaves at two chained `LFML` rows each + // (6) and 11 Merkle-walk steps — 11 `LFMC` and 6 `LFML`, i.e. 17. Times 4 + // queries, plus the preamble's 13. The three `LFMC` folds that used to + // combine each leaf's two halves are gone: the chain does that work inside + // the rows it was already paying for. + let (c, t, l) = hash_row_modes(&super::programs::fri_toy_program()); + assert_eq!((c, t, l), (44, 11, 26)); + assert_eq!( + t + 2, + FRI_TOY_COMPRESSIONS, + "the preamble's share: 11 transcript steps plus its 2 leaf rows" + ); + assert_eq!( + c + t + l, + 4 * 17 + FRI_TOY_COMPRESSIONS, + "the decomposition: 4 queries × 17 + the preamble's 13" + ); + assert_eq!(c + t + l, 81, "was 93 before the leaf RATE"); + // ★ The LEAF ROW count is unchanged — 26 either way — which is the point: + // the same felts are absorbed by the same number of `LFML` rows, and what + // disappeared is the FOLD. A change that had merely moved work from the + // parents into more leaf rows would show up right here. + assert_eq!(l, 26, "the RATE removes folds, it does not add leaf rows"); + if SOCKET_ROUNDS == 7 { + assert_eq!((c + t + l) * price, 449_145, "FriToyV0 at 7 rounds"); + } +} diff --git a/prover/src/lfm/tree_plan_property_tests.rs b/prover/src/lfm/tree_plan_property_tests.rs new file mode 100644 index 000000000..51b94e921 --- /dev/null +++ b/prover/src/lfm/tree_plan_property_tests.rs @@ -0,0 +1,177 @@ +//! Properties of a block tree's plan: the leaf partition's exact cover and its +//! doubling threshold, the tree shape's coverage, and what the published +//! state word is. + +use crate::hash_pin::{BlockFamily, ProofFamily}; +use crate::tables::types::{FE, FEE}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::executor::execute; +use super::per_table_aggregator::tree_shape; +use super::whir_block::{ + BlockPartition, LeafScale, P1_LEAF_HEIGHTS, P1_LEAF_HEIGHTS_2X, scaled_leaf_partition, +}; +use super::whir_chain::ChipRows; +use super::whir_epoch::emit_absorb_roots; +use super::whir_transcript::WhirTranscript; +use super::word::{LfmWord, word_as_ext}; + +struct Lcg(u64); +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + self.0 >> 33 + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } +} + +fn random_load(r: &mut Lcg, room: ChipRows, frac_den: usize) -> ChipRows { + let a = room.to_array(); + ChipRows::from_array(core::array::from_fn(|c| { + if a[c] == 0 { + 0 + } else if a[c] == usize::MAX { + r.below(64) + } else { + r.below(a[c] / frac_den + 1) + } + })) +} + +/// The partition under either scale is an exact cover of the groups, every +/// leaf (front included) under the heights it reports, the same on a re-run, +/// and doubled exactly when the single partition reaches the threshold. +#[test] +fn the_partition_is_an_exact_cover_and_doubles_on_an_integer_compare() { + let mut r = Lcg(0x5eed_0606); + let front = ChipRows { + hash: 9_000, + select: 20_000, + balu: 30_000, + xalu: 40_000, + lanes: 10_000, + hint: 5_000, + bitdec: 100, + consts: 500, + public: 12, + accel: 0, + }; + let room = ChipRows::from_array(core::array::from_fn(|c| { + P1_LEAF_HEIGHTS.to_array()[c].saturating_sub(front.to_array()[c]) + })); + let mut doubled_seen = [false; 2]; + for case in 0..400 { + let n = 1 + r.below(160); + let den = 2 + r.below(20); + let loads: Vec = (0..n).map(|_| random_load(&mut r, room, den)).collect(); + let t = 1 + r.below(14); + for scale in [LeafScale::Fixed, LeafScale::DoubleFrom(t)] { + let got = + scaled_leaf_partition(&loads, front, scale, P1_LEAF_HEIGHTS, P1_LEAF_HEIGHTS_2X); + let (p, bound) = got.unwrap_or_else(|e| panic!("case {case}: {e}")); + let again = + scaled_leaf_partition(&loads, front, scale, P1_LEAF_HEIGHTS, P1_LEAF_HEIGHTS_2X) + .expect("re-run"); + assert_eq!(p, again.0, "case {case}: deterministic"); + assert_eq!(bound, again.1); + BlockPartition::new(p.leaves().to_vec(), n).expect("an exact cover"); + assert_eq!(p.num_units(), n); + for leaf in p.leaves() { + let sum = leaf.iter().fold(front, |acc, &g| acc.plus(loads[g])); + assert!( + sum.under(bound.heights), + "case {case}: leaf over its heights" + ); + } + let want_doubled = + matches!(scale, LeafScale::DoubleFrom(t) if bound.single_leaves >= t); + assert_eq!(bound.doubled, want_doubled, "case {case}"); + assert_eq!( + bound.heights, + if want_doubled { + P1_LEAF_HEIGHTS_2X + } else { + P1_LEAF_HEIGHTS + } + ); + doubled_seen[usize::from(bound.doubled)] = true; + } + } + assert_eq!(doubled_seen, [true, true], "both outcomes exercised"); +} + +/// Every level's arities consume exactly the level below, each in +/// `1..=fan_in`, and the last level is one node: every leaf reaches the top. +#[test] +fn the_tree_shape_covers_every_leaf_and_closes_to_one_node() { + for n in 2..=400 { + let shape = tree_shape(n, 3); + let mut below = n; + for level in &shape { + assert!(level.arities.iter().all(|&a| (1..=3).contains(&a))); + assert_eq!(level.arities.iter().sum::(), below, "n {n}"); + below = level.arities.len(); + } + assert_eq!(below, 1, "n {n} closes to one node"); + } + assert!( + tree_shape(1, 3).is_empty(), + "one leaf: the plan adds the top" + ); +} + +/// The leaf's published state word is lanes 0..3 of the permutation output the +/// draws come from: its first three lanes are `z`, its fourth `α`'s first. +/// Equal states across leaves therefore pin `z` and `α₀` directly, and `α₁`, +/// `α₂`, `β` only through the sponge's 4-felt truncated-output collision +/// resistance. Two different carried roots give two different states. +#[test] +fn the_state_word_is_the_draw_permutations_first_four_lanes() { + let run = |roots: &[LfmWord]| -> Vec { + let mut b = LfmBuilder::new().with_wrap_hash(::WRAP); + let arena = b.declare_arena(roots.len() as u32); + let carried: Vec<_> = (0..roots.len() as u32) + .map(|i| b.hint_word(arena, i)) + .collect(); + let mut t = WhirTranscript::for_builder(&mut b); + t.absorb_const_bytes(b"a statement call"); + emit_absorb_roots(&mut b, &mut t, &carried, &[]); + let z = t.sample_ext(&mut b); + let alpha = t.sample_ext(&mut b); + let beta = t.sample_ext(&mut b); + let state = t.state(&mut b); + for e in [z, alpha, beta] { + b.public(e.as_cell()); + } + b.public(state); + let program = compile(b.finish()); + execute( + &program, + &[roots.to_vec()], + &program.hasher(::SOCKET), + ) + .expect("executes") + .public_words + .iter() + .map(|(_, w)| *w) + .collect() + }; + let root = |s: u64| -> LfmWord { core::array::from_fn(|i| FE::from(s * 10 + i as u64)) }; + let a = run(&[root(1), root(2), root(3)]); + let b = run(&[root(1), root(2), root(4)]); + for words in [&a, &b] { + let z: FEE = word_as_ext(&words[0]).expect("z"); + let alpha: FEE = word_as_ext(&words[1]).expect("alpha"); + let state = words[3]; + assert_eq!(&state[..3], z.value(), "state lanes 0..3 are z"); + assert_eq!(state[3], alpha.value()[0], "state lane 3 is alpha's first"); + } + assert_ne!(a[3], b[3], "another carried root, another state"); + assert_ne!(a[2], b[2], "and another beta"); +} diff --git a/prover/src/lfm/tree_probe.rs b/prover/src/lfm/tree_probe.rs new file mode 100644 index 000000000..79b96c8a9 --- /dev/null +++ b/prover/src/lfm/tree_probe.rs @@ -0,0 +1,298 @@ +//! ⛔ ROUND-3 TREE/WRAP DISCRIMINATOR (diagnostic; on NO production decision +//! path, OFF by default, byte-identical when off). +//! +//! # The question it answers +//! +//! Rounds 1 and 2 landed on the WHIR BASE and round 3 measured the base at two +//! floors (the RPX permutation's arithmetic floor and argue's Fiat-Shamir +//! sequencing floor). What was never examined is the ~55 s ABOVE the base: the +//! recursion tree — level 0's epoch wraps, the cross-epoch GLOBAL child, the +//! interior fold levels, and the block-artifact root. Either that phase has +//! CONCURRENCY SLACK (a lever: overlap stages that are independent) or it is +//! already saturated and bound by the same permutation (a floor). +//! +//! # ⛔ Why this measures the PERMIT and not SM occupancy +//! +//! The tree phase's serialising resource is not the card's utilisation, it is +//! [`super::device_permit`] — MUTUAL EXCLUSION over the two device phases of a +//! proof, because two proofs in flight can ask the card for twice its budget +//! from two directions (`build_artifacts`' per-dispatch admission has no running +//! total; each `multi_prove` builds its own full-budget `VramGate`). So even a +//! card that idles INSIDE a device phase cannot be filled by another sibling: +//! the permit excludes it, and the acquire panics rather than allowing it. +//! +//! ⇒ the ceiling on every concurrency lever in this phase is `Σ held`, the time +//! spent inside a device phase. `held / stage wall` is therefore the number that +//! discriminates: +//! +//! near 1 ⇒ the card is the wall; more siblings buy nothing and the only +//! lever left is making the device phases themselves cheaper — +//! which is the permutation, already at its arithmetic floor; +//! well under 1 ⇒ the phase idles the card while workers walk the host, and +//! overlapping independent work recovers it. +//! +//! # What is missing from the shipped instruments, and why this exists +//! +//! ✓ `PermitStats::describe` already prints exactly this reading — but only +//! where a level calls `take_stats()`, and there are three such sites: the +//! interior's BARRIER loop and the two drivers' level 0. The production harness +//! runs with `LFM_TREE_LEVEL_POOL=1` and `LFM_TREE_LEVEL_POOL_FROM` at its +//! default of 1, so `barrier_levels` is 0, the barrier loop runs ZERO levels and +//! the whole interior goes through the POOLED span — which prints no permit +//! line. And the GLOBAL child and the ROOT run with the permit armed at 1, where +//! `hold` returns early with no guard and `Drop` accumulates nothing, so their +//! held time exists only on the per-hold `LFM_CARD_TRACE` lines and in no +//! summary at all. +//! +//! ⇒ this probe accumulates on EVERY hold, armed or not, into counters of its +//! own — it never touches `HELD_NANOS` or `take_stats()`, so every line the +//! drivers already print keeps its exact current meaning. +//! +//! # ★ The second reading: how much of the `build_artifacts` hold is DEVICE +//! +//! ✓ The permit is held across the WHOLE of `build_artifacts_with_hasher`, +//! while the only device work inside it is one `try_commit_row_major` per column +//! group that clears the admission floor (`super::commit::commit_group_device_or_host`, +//! and ✓ its only two production call sites are both inside that build — +//! `registry.rs`'s prep-group and blake3-chunk walks). If the dispatch total is +//! much smaller than the hold, the permit is held over HOST work and narrowing +//! its scope would free the card — a second, independent lever with a mechanism. +//! Measured with a host stopwatch around a host-blocking call: ⛔ NO CUDA event +//! and NO added synchronize, so the device path cannot be perturbed by the +//! measurement even when it is on. +//! +//! ⚠⚠ THE DISPATCH TOTAL IS WORKER-SECONDS, NOT A WALL, and mixing the two is +//! the way to misread this line. ✓ Both walks go through `map_maybe_parallel`, +//! which runs on rayon when `LFM_ARTIFACT_PARALLEL` is not `0` (default ON) over +//! windows of `LFM_ARTIFACT_GROUPS_IN_FLIGHT` groups (default 4). So up to four +//! dispatches overlap and the total can EXCEED the hold it is quoted against: +//! above 100% is concurrency, not a bug. What the reading still says is the +//! thing it is for — a ratio WELL UNDER 100% means the card was idle inside an +//! exclusive hold even allowing for that concurrency. +//! +//! # Cost when off +//! +//! [`enabled`] is a `OnceLock` load. Every call site reads it once per +//! HOLD or per device dispatch — tens of times in a whole block run, never in a +//! loop — and does nothing else. Off, no counter moves and no line prints. + +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// `LAMBDA_VM_TREE_BUSY_PROBE=1` (any non-empty value other than `0`) turns it +/// on, read ONCE and cached. +/// +/// ⚠ Unset, empty and `0` all read as OFF — `trace_enabled`'s spelling, not the +/// argue probe's `is_some()`. `FOO=` is the shell clearing a variable and +/// `FOO=0` is somebody switching a knob off; a probe that turned ON for either +/// would put a diagnostic line into a run nobody asked for one in. +pub fn enabled() -> bool { + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| match std::env::var("LAMBDA_VM_TREE_BUSY_PROBE") { + Ok(v) => !v.is_empty() && v != "0", + Err(_) => false, + }) +} + +/// Held nanoseconds inside `build_artifacts` device phases, and how many. +static BUILD_NS: AtomicU64 = AtomicU64::new(0); +static BUILD_HOLDS: AtomicU64 = AtomicU64::new(0); +/// Held nanoseconds inside `multi_prove` device phases, and how many. +static PROVE_NS: AtomicU64 = AtomicU64::new(0); +static PROVE_HOLDS: AtomicU64 = AtomicU64::new(0); +/// Held nanoseconds under any other label, so a new hold site cannot be lost. +static OTHER_NS: AtomicU64 = AtomicU64::new(0); +static OTHER_HOLDS: AtomicU64 = AtomicU64::new(0); +/// Nanoseconds workers spent QUEUED on the card — nonzero only where the permit +/// is armed above one. +static WAITED_NS: AtomicU64 = AtomicU64::new(0); +/// Nanoseconds inside the device commit dispatch, and how many groups reached it. +static DEVICE_COMMIT_NS: AtomicU64 = AtomicU64::new(0); +static DEVICE_COMMIT_CALLS: AtomicU64 = AtomicU64::new(0); + +/// Record one released card permit: which phase it was, how long it was held, +/// and how long its holder queued for it. +/// +/// Called from `device_permit::CardPermit::drop` for EVERY hold — including the +/// unarmed ones a `K = 1` stage takes, which is the whole reason this exists. +pub fn note_hold(phase: &'static str, held_nanos: u64, waited_nanos: u64) { + let (ns, holds) = match phase { + "build_artifacts" => (&BUILD_NS, &BUILD_HOLDS), + "multi_prove" => (&PROVE_NS, &PROVE_HOLDS), + _ => (&OTHER_NS, &OTHER_HOLDS), + }; + ns.fetch_add(held_nanos, Ordering::Relaxed); + holds.fetch_add(1, Ordering::Relaxed); + WAITED_NS.fetch_add(waited_nanos, Ordering::Relaxed); +} + +/// Record one device commit dispatch — the admitted `try_commit_row_major` +/// inside `build_artifacts`. +pub fn note_device_commit(nanos: u64) { + DEVICE_COMMIT_NS.fetch_add(nanos, Ordering::Relaxed); + DEVICE_COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); +} + +/// One stage's counters. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Stats { + pub build_holds: u64, + pub build_nanos: u64, + pub prove_holds: u64, + pub prove_nanos: u64, + pub other_holds: u64, + pub other_nanos: u64, + pub waited_nanos: u64, + pub device_commit_calls: u64, + pub device_commit_nanos: u64, +} + +impl Stats { + /// Seconds this stage spent inside a device phase — the serialised total + /// every concurrency lever is bounded by. + pub fn held_secs(&self) -> f64 { + (self.build_nanos + self.prove_nanos + self.other_nanos) as f64 / 1e9 + } + + /// How many holds the stage took. + pub fn holds(&self) -> u64 { + self.build_holds + self.prove_holds + self.other_holds + } + + /// ★ THE LINE THE DISCRIMINATOR IS READ OFF. + /// + /// ⛔ NO DEVICE-MEMORY FIELD, and its removal is a finding rather than a + /// simplification. Splitting `math_cuda::device::reserved_high_water` per + /// stage means CUTTING a process-wide counter at every boundary, which + /// costs the harness's own whole-run `reserved high-water` line its + /// meaning — and the free read of wt27/wt28 showed the number was not worth + /// that: the whole-run figure is 25,003 MiB against a 32,607 MiB card, the + /// 10 Hz `nvidia-smi` trace the harness already keeps splits device memory + /// by phase, and no tree stage is anywhere near the budget. A measurement + /// that degrades an existing one has to buy more than that. + pub fn describe(&self, stage: &str, wall_secs: f64) -> String { + let held = self.held_secs(); + let pct = |x: f64| { + if wall_secs > 0.0 { + 100.0 * x / wall_secs + } else { + 0.0 + } + }; + format!( + "TREE PROBE {stage}: wall {wall_secs:.1}s · card-held {held:.1}s ({:.0}%) over {} hold(s) \ + [build_artifacts {:.1}s/{} · multi_prove {:.1}s/{} · other {:.1}s/{}] · \ + queued {:.1}s · device commit dispatch {:.1}s WORKER-SEC over {} group(s) \ + ({:.0}% of the build_artifacts hold — worker-seconds over a wall, so \ + above 100% is concurrency)", + pct(held), + self.holds(), + self.build_nanos as f64 / 1e9, + self.build_holds, + self.prove_nanos as f64 / 1e9, + self.prove_holds, + self.other_nanos as f64 / 1e9, + self.other_holds, + self.waited_nanos as f64 / 1e9, + self.device_commit_nanos as f64 / 1e9, + self.device_commit_calls, + if self.build_nanos > 0 { + 100.0 * self.device_commit_nanos as f64 / self.build_nanos as f64 + } else { + 0.0 + }, + ) + } +} + +/// Read and CLEAR every counter, so a stage reports its own. +pub fn take() -> Stats { + Stats { + build_holds: BUILD_HOLDS.swap(0, Ordering::Relaxed), + build_nanos: BUILD_NS.swap(0, Ordering::Relaxed), + prove_holds: PROVE_HOLDS.swap(0, Ordering::Relaxed), + prove_nanos: PROVE_NS.swap(0, Ordering::Relaxed), + other_holds: OTHER_HOLDS.swap(0, Ordering::Relaxed), + other_nanos: OTHER_NS.swap(0, Ordering::Relaxed), + waited_nanos: WAITED_NS.swap(0, Ordering::Relaxed), + device_commit_calls: DEVICE_COMMIT_CALLS.swap(0, Ordering::Relaxed), + device_commit_nanos: DEVICE_COMMIT_NS.swap(0, Ordering::Relaxed), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// ⛔ The counters are process-global, so the two tests that WRITE them run + /// one at a time. Without this they interleave and each reads the other's + /// holds — a flake that would read as the probe losing time. + /// + /// ⓘ No other test in this binary can reach them: `note_hold` is called + /// only from `CardPermit::drop` and only when the permit carries a `probe` + /// tag, which needs `LAMBDA_VM_TREE_BUSY_PROBE` in the environment, and a + /// test binary does not set it. + static COUNTERS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// `take` must CLEAR, so two stages cannot report each other's holds — the + /// failure that makes a late stage look card-bound because an early one was. + #[test] + fn take_clears_so_a_stage_reports_its_own() { + let _g = COUNTERS.lock().unwrap_or_else(|e| e.into_inner()); + let _ = take(); + note_hold("build_artifacts", 3_000_000_000, 1_000_000_000); + note_hold("multi_prove", 7_000_000_000, 0); + note_device_commit(2_000_000_000); + let first = take(); + assert_eq!(first.build_holds, 1); + assert_eq!(first.prove_holds, 1); + assert_eq!(first.device_commit_calls, 1); + assert!((first.held_secs() - 10.0).abs() < 1e-9); + let second = take(); + assert_eq!(second, Stats::default(), "the second read must be empty"); + } + + /// An unrecognised phase label lands in `other` rather than being dropped — + /// a third hold site added later must show up as time somewhere, or the + /// held fraction silently understates and reads as slack that is not there. + #[test] + fn an_unknown_phase_is_counted_not_dropped() { + let _g = COUNTERS.lock().unwrap_or_else(|e| e.into_inner()); + let _ = take(); + note_hold("a phase nobody has written yet", 5_000_000_000, 0); + let s = take(); + assert_eq!(s.other_holds, 1); + assert_eq!(s.build_holds + s.prove_holds, 0); + assert!((s.held_secs() - 5.0).abs() < 1e-9); + } + + /// The line carries the fraction the verdict is read off, and the + /// build-hold's device share beside it. + #[test] + fn the_line_reports_the_held_fraction_and_the_dispatch_share() { + let s = Stats { + build_holds: 2, + build_nanos: 4_000_000_000, + prove_holds: 2, + prove_nanos: 6_000_000_000, + device_commit_calls: 9, + device_commit_nanos: 1_000_000_000, + ..Stats::default() + }; + let line = s.describe("level 0", 20.0); + assert!(line.contains("card-held 10.0s (50%)"), "{line}"); + assert!(line.contains("over 4 hold(s)"), "{line}"); + assert!( + line.contains("dispatch 1.0s WORKER-SEC over 9 group(s) (25% of the"), + "{line}" + ); + } + + /// ⛔ A stage with no holds must not divide by zero into a fake percentage. + #[test] + fn an_empty_stage_reports_zero_rather_than_nan() { + let line = Stats::default().describe("nothing", 0.0); + assert!(line.contains("card-held 0.0s (0%)"), "{line}"); + assert!(line.contains("over 0 group(s) (0% of the"), "{line}"); + } +} diff --git a/prover/src/lfm/tree_run.rs b/prover/src/lfm/tree_run.rs new file mode 100644 index 000000000..155f61428 --- /dev/null +++ b/prover/src/lfm/tree_run.rs @@ -0,0 +1,88 @@ +//! What running a recursion tree needs besides its programs: proving a level's +//! proofs a bounded number at once and handing them back in index order +//! ([`in_index_order`]). The W3 driver ([`super::whir_block_tree`]) and the +//! tree suites share it. + +/// Run `task` over `0..n` on `workers` threads, and return the results **in +/// index order** whatever order they finished in. +/// +/// ★★★ THE ORDER IS THE SOUNDNESS PROPERTY, not a convenience. A level's +/// children, layouts and label runs are three parallel vectors, and a node at +/// the level above takes a contiguous SUBSLICE of each — which is exactly what +/// makes contiguity across sibling subtrees a consequence of the label pins +/// rather than a check of its own. Drain them in completion order and the pins +/// still verify, one subtree at a time, while the tree they describe is not the +/// tree that was built. ⇒ results land in per-index slots and are drained by +/// index, so nothing downstream can observe that a scheduler ran at all. +/// +/// `workers <= 1` runs `task` inline, on this thread, in order: the control arm +/// is the original path and not this function with one worker. +/// +/// # Panics +/// +/// Re-raises the FIRST worker panic on the caller's thread, payload intact. +/// ⚠ `std::thread::scope` otherwise propagates with the fixed string "a scoped +/// thread panicked", which names neither the cause nor its location, and +/// libtest's global hook files a spawned thread's own message against no test +/// and drops it on the floor. The prover's `run_admitted` learned that the +/// expensive way — eleven anonymous failures in one suite run. +pub(crate) fn in_index_order( + n: usize, + workers: usize, + task: impl Fn(usize) -> T + Sync, +) -> Vec { + let slots: Vec>> = + (0..n).map(|_| std::sync::Mutex::new(None)).collect(); + if workers <= 1 { + for (j, slot) in slots.iter().enumerate() { + *slot.lock().unwrap_or_else(|e| e.into_inner()) = Some(task(j)); + } + } else { + let cursor = std::sync::atomic::AtomicUsize::new(0); + let first_panic: std::sync::Mutex>> = + std::sync::Mutex::new(None); + std::thread::scope(|scope| { + for _ in 0..workers { + let (cursor, slots, first_panic, task) = (&cursor, &slots, &first_panic, &task); + scope.spawn(move || { + // ⛔ THIS THREAD IS PART OF THIS LEVEL. Without the enrolment + // its artifact builds are not counted and the level reports + // fewer proofs than it made. + let _enrolled = super::program_census::enrol(); + loop { + let j = cursor.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if j >= slots.len() { + break; + } + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| task(j))) { + Ok(out) => { + *slots[j].lock().unwrap_or_else(|e| e.into_inner()) = Some(out); + } + Err(payload) => { + let mut first = + first_panic.lock().unwrap_or_else(|e| e.into_inner()); + if first.is_none() { + *first = Some(payload); + } + break; + } + } + } + }); + } + }); + if let Some(payload) = first_panic.into_inner().unwrap_or_else(|e| e.into_inner()) { + std::panic::resume_unwind(payload); + } + } + slots + .into_iter() + .enumerate() + .map(|(j, slot)| { + // Every index ran (a panicking task was re-raised above). + slot.into_inner() + .unwrap_or_else(|e| e.into_inner()) + .unwrap_or_else(|| panic!("index {j} produced no result")) + }) + .collect() +} diff --git a/prover/src/lfm/validator.rs b/prover/src/lfm/validator.rs new file mode 100644 index 000000000..e144ed798 --- /dev/null +++ b/prover/src/lfm/validator.rs @@ -0,0 +1,827 @@ +//! The registry-admission validator — release-mode, always on. +//! +//! A program digest enters the `LFM_REGISTRY` only after this passes. The +//! AIR checks per-op algebra and bus balance; the *registrar* vouches for the +//! structural well-formedness below, and this validator is what makes that +//! vouching real (the reference machine checks less, and only in dev builds). +//! Together: uniqueness + acyclicity + bounded multiplicities + balance ⇒ +//! every read observes the unique written value. +//! +//! "Bounded multiplicities" is load-bearing and easy to lose: balance is a +//! *field* identity, so a send gated by `p − 1` subtracts a token, and a +//! subtract-then-re-add pair over one address keeps the count while changing +//! the value. Every check below that reads `program.instrs` is therefore +//! backed by one that reads the committed `program.groups` directly — the +//! groups are what the AIR sees, and the two are joined by nothing but the +//! emitter. +//! +//! The compiler's invariant panics are tripwires; this validator is the gate; +//! the registry is the record. There is no off-switch, and there must never +//! be one. + +use std::collections::{HashMap, HashSet}; + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::compiler::{ColumnGroup, LfmColumnGroups, LfmProgram}; +use super::instr::{Addr, Instr}; +use super::layout; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LfmViolation { + /// Check 1 — write-once uniqueness. + DoubleWrite { addr: u64 }, + /// Check 2 — every read has a writer. + ReadOfUnwritten { addr: u64 }, + /// Check 2 (range) — an address outside the allocated space. + AddressOutOfRange { addr: u64 }, + /// Check 3 — acyclicity: an operand not strictly below its destination. + CyclicRead { instr: usize, addr: u64 }, + /// Check 4 — a write's `mult` differs from the emitted read count. + MultMismatch { + addr: u64, + expected: u64, + found: u64, + }, + /// Check 4 — a `Compress` row whose spare output slots are not the + /// documented placeholders. + /// + /// `Instr::writes()` reports only `outs[0]` for `Compress` (`instr.rs`), + /// so checks 1 and 4 never see slots 1–2 — but `emit_column_groups` copies + /// them into the committed group unconditionally and `chips::hash` sends + /// all three slots. `instr.rs`'s field conventions call them "`Addr(0)` + /// placeholders with `mults` fixed to 0"; this is what makes that a rule + /// rather than a comment. + CompressSlotNotPlaceholder { instr: usize }, + /// A program with both twelve-felt (`Hash`) and width-16 (`Hash16`) + /// `LFM_HASH` rows: one chip proves one of the two, so the other half could + /// never be proved. `instr` is the first row of the minority width met. + MixedHashWidths { instr: usize }, + /// Check 5 — opcode selectors not one-hot / flags not boolean on a real row. + NonOneHotSelector { chip: &'static str, row: usize }, + /// Check 6 — nonzero data beyond the program length. + DirtyPadding { chip: &'static str, row: usize }, + /// Check 7 — a `Hint` outside the declared arena schema. + ArenaOutOfBounds { arena: u32, index: u32 }, + /// Check 8 — two `LFM_KECCAK` rows carry the same tag. + /// + /// The tag is the only thing binding a permutation's request token to its + /// reply token: with a duplicate, a prover can swap the two rows' output + /// states and the `Keccak` bus still balances. This is not theoretical — + /// `keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` + /// exhibits the forgery against the raw family. Tags are preprocessed + /// program data, so this check is what makes them trustworthy. + DuplicateKeccakTag { tag: (u64, u64) }, + /// Check 8 — a tag half at or above `2^32`, so it cannot equal the + /// `DWordWL` timestamp any `KECCAK_RND` row carries. + MalformedKeccakTag { row: usize }, + /// Check 9 — a multiplicity column outside the legal range. + /// + /// A multiplicity is a static read count: a small non-negative integer. + /// The field is not ordered, so nothing about `Multiplicity::Column` stops + /// a preprocessed mult from holding `p − 1`, which the bus reads as `−1` — + /// a **negative** send. That is the one shape the LogUp count argument + /// cannot see: a negative ghost send cancels an honest write and a + /// positive one replaces it at the same address, leaving the token count + /// exactly balanced while the cell's value is prover-chosen. Surplus + /// *positive* sends are already denied by the count (they have no matching + /// receive); this check is what denies the negative half. + MultOutOfRange { + chip: &'static str, + row: usize, + col: usize, + mult: u64, + bound: u64, + }, + /// Cross-check — group shape does not match the instruction partition. + GroupShapeMismatch { chip: &'static str }, +} + +pub fn validate(program: &LfmProgram) -> Result<(), LfmViolation> { + check_writes_and_reads(program)?; + check_multiplicities(program)?; + check_arenas(program)?; + check_groups(program)?; + check_keccak_tags(&program.groups.keccak)?; + Ok(()) +} + +/// [`validate`]'s checks one by one, each with its seconds — an instrument +/// (`node_emit_timing_tests`), test-only. +#[cfg(test)] +pub(crate) fn validate_split(program: &LfmProgram) -> Vec<(&'static str, f64)> { + let mut out = Vec::new(); + let mut time = |name: &'static str, f: &dyn Fn() -> Result<(), LfmViolation>| { + let t = std::time::Instant::now(); + f().expect("the program is admissible"); + out.push((name, t.elapsed().as_secs_f64())); + }; + time("writes_and_reads", &|| check_writes_and_reads(program)); + time("multiplicities (map)", &|| { + check_multiplicities_with(program, false) + }); + time("multiplicities (dense)", &|| { + check_multiplicities_with(program, true) + }); + time("arenas", &|| check_arenas(program)); + time("groups", &|| check_groups(program)); + time("keccak_tags", &|| check_keccak_tags(&program.groups.keccak)); + out +} + +/// Check 8: `LFM_KECCAK` tags are well-formed and pairwise distinct. +/// +/// Padding rows are skipped: their `IS_REAL` is zero, so they emit no bus +/// tokens and their all-zero tag binds nothing. +fn check_keccak_tags(group: &ColumnGroup) -> Result<(), LfmViolation> { + let mut seen = HashSet::new(); + for row in 0..group.real_rows { + let lo = GoldilocksField::canonical(group.at(row, layout::keccak::TAG_LO).value()); + let hi = GoldilocksField::canonical(group.at(row, layout::keccak::TAG_HI).value()); + if lo >= 1u64 << 32 || hi >= 1u64 << 32 { + return Err(LfmViolation::MalformedKeccakTag { row }); + } + if !seen.insert((lo, hi)) { + return Err(LfmViolation::DuplicateKeccakTag { tag: (lo, hi) }); + } + } + Ok(()) +} + +/// Checks 1–3: uniqueness, read-has-writer, acyclicity. +fn check_writes_and_reads(program: &LfmProgram) -> Result<(), LfmViolation> { + let n = program.num_addrs as usize; + let mut written = vec![false; n]; + for instr in &program.instrs { + for Addr(w) in instr.writes() { + let slot = written + .get_mut(w as usize) + .ok_or(LfmViolation::AddressOutOfRange { addr: w })?; + if *slot { + return Err(LfmViolation::DoubleWrite { addr: w }); + } + *slot = true; + } + } + for (idx, instr) in program.instrs.iter().enumerate() { + let min_write = instr.writes().iter().map(|a| a.0).min(); + // A `HintOn`'s bit is held to the same order as a read, though it is + // not one (`Instr::schedule_reads_into`). + for Addr(r) in instr.schedule_reads() { + if !*written + .get(r as usize) + .ok_or(LfmViolation::AddressOutOfRange { addr: r })? + { + return Err(LfmViolation::ReadOfUnwritten { addr: r }); + } + if let Some(w) = min_write + && r >= w + { + return Err(LfmViolation::CyclicRead { + instr: idx, + addr: r, + }); + } + } + } + Ok(()) +} + +/// Check 4 recounts reads in a table indexed by address ([`ReadCounts`]) by +/// default; `LFM_VALIDATE_DENSE_COUNTS=0` restores the hash map (the rollback). +/// The same counts, so the same verdict and the same first violation; only the +/// time differs. The map was ≈ 0.8 s of a 1.2 s node emission (laptop +/// instrument `node_emit_timing_tests`), and on the box the dense recount took +/// the median block's recursion down 0.27 s and the tree verify from 7.96 to +/// 3.77 s. +pub const DENSE_COUNTS_ENV: &str = "LFM_VALIDATE_DENSE_COUNTS"; + +/// Whether [`DENSE_COUNTS_ENV`] is on, read once. +pub fn dense_counts() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| parse_dense_counts(std::env::var(DENSE_COUNTS_ENV).ok().as_deref())) +} + +/// [`DENSE_COUNTS_ENV`]'s value: unset, empty or `1` on (the default); `0` +/// off; anything else refused. +pub fn parse_dense_counts(value: Option<&str>) -> bool { + match value { + None | Some("") | Some("1") => true, + Some("0") => false, + Some(other) => panic!("{DENSE_COUNTS_ENV} must be `0` or `1`, got `{other}`"), + } +} + +/// Check 4's independent recount of every address's reads. +/// +/// `Dense` holds one count per allocated address. Check 1 runs first and +/// refuses any read or write outside `0..num_addrs`, so every read it counts is +/// in range; a read past the table would be a program check 1 already refused, +/// and a write past it reads zero, as the map's missing entry does. +enum ReadCounts { + Map(HashMap), + Dense(Vec), +} + +impl ReadCounts { + fn of(program: &LfmProgram, dense: bool) -> Self { + if dense { + let mut counts = vec![0u64; program.num_addrs as usize]; + let mut reads = Vec::new(); + for instr in &program.instrs { + reads.clear(); + instr.reads_into(&mut reads); + for Addr(r) in &reads { + if let Some(c) = counts.get_mut(*r as usize) { + *c += 1; + } + } + } + Self::Dense(counts) + } else { + let mut counts: HashMap = HashMap::new(); + for instr in &program.instrs { + for r in instr.reads() { + *counts.entry(r).or_insert(0) += 1; + } + } + Self::Map(counts) + } + } + + fn get(&self, addr: Addr) -> u64 { + match self { + Self::Map(m) => m.get(&addr).copied().unwrap_or(0), + Self::Dense(v) => v.get(addr.0 as usize).copied().unwrap_or(0), + } + } +} + +/// Check 4: every write's `mult` equals an independent recount of its reads. +fn check_multiplicities(program: &LfmProgram) -> Result<(), LfmViolation> { + check_multiplicities_with(program, dense_counts()) +} + +/// [`check_multiplicities`] with the recount's form named. +fn check_multiplicities_with(program: &LfmProgram, dense: bool) -> Result<(), LfmViolation> { + let counts = ReadCounts::of(program, dense); + let check = |addr: Addr, found: u64| -> Result<(), LfmViolation> { + let expected = counts.get(addr); + if expected != found { + return Err(LfmViolation::MultMismatch { + addr: addr.0, + expected, + found, + }); + } + Ok(()) + }; + for (idx, instr) in program.instrs.iter().enumerate() { + match instr { + Instr::Const { out, mult, .. } + | Instr::BaseAlu { out, mult, .. } + | Instr::ExtAlu { out, mult, .. } + | Instr::Hint { out, mult, .. } + | Instr::HintOn { out, mult, .. } + | Instr::Pack { out, mult, .. } => check(*out, *mult)?, + Instr::Unpack { outs, mults, .. } => { + for i in 0..4 { + check(outs[i], mults[i])?; + } + } + Instr::Select { + out_l, + out_r, + mult_l, + mult_r, + .. + } => { + check(*out_l, *mult_l)?; + check(*out_r, *mult_r)?; + } + Instr::BitDec { bits, halves, .. } => { + for (addr, mult) in bits { + check(*addr, *mult)?; + } + for (addr, mult) in halves.iter().flatten() { + check(*addr, *mult)?; + } + } + Instr::Hash { + mode, + ins, + outs, + mults, + } => { + let num_outs = mode.num_output_cells(); + for i in 0..num_outs { + check(outs[i], mults[i])?; + } + // A one-output row's spare slots are outside `writes()` and so + // outside checks 1 and 4 — but they are inside the committed + // group and inside the bus. Pin them to the placeholders + // `instr.rs` documents, so "slot 0 only" is a checked property + // of the program and not a convention the emitter happens to + // follow. + if mode.num_output_cells() == 1 + && (mults[1] != 0 || mults[2] != 0 || outs[1] != Addr(0) || outs[2] != Addr(0)) + { + return Err(LfmViolation::CompressSlotNotPlaceholder { instr: idx }); + } + // The same for the INPUT slots a mode does not read. A leaf row + // reads one cell, so its second and third slots reach the bus as + // addresses nothing receives; pinning them keeps "reads exactly + // `num_input_cells`" a checked property too. + if ins[mode.num_input_cells()..].iter().any(|a| *a != Addr(0)) { + return Err(LfmViolation::CompressSlotNotPlaceholder { instr: idx }); + } + if program.hash16() { + return Err(LfmViolation::MixedHashWidths { instr: idx }); + } + } + Instr::Hash16(h) => { + for i in 0..4 { + check(h.outs[i], h.mults[i])?; + } + if !program.hash16() { + return Err(LfmViolation::MixedHashWidths { instr: idx }); + } + } + Instr::KeccakF(k) => { + for i in 0..layout::keccak::NUM_WORDS { + check(k.outs[i], k.mults[i])?; + } + if let Some(rev) = &k.rev { + for i in 0..layout::keccak::DIGEST_WORDS { + check(rev.outs[i], rev.mults[i])?; + } + } + } + Instr::Blake3(k) => { + for i in 0..layout::blake3::OUT_WORDS { + check(k.outs[i], k.mults[i])?; + } + if let Some(rev) = &k.rev { + for i in 0..layout::blake3::DIGEST_WORDS { + check(rev.outs[i], rev.mults[i])?; + } + } + } + Instr::Public { .. } => {} + } + } + Ok(()) +} + +/// Check 7: arena discipline. +fn check_arenas(program: &LfmProgram) -> Result<(), LfmViolation> { + let lens = &program.arena_schema.lens; + for instr in &program.instrs { + let (arena, indices) = match instr { + Instr::Hint { arena, index, .. } => (arena, [Some(index), None]), + Instr::HintOn { + arena, index, alt, .. + } => (arena, [Some(index), Some(alt)]), + _ => continue, + }; + for index in indices.into_iter().flatten() { + let ok = lens.get(*arena as usize).is_some_and(|&len| *index < len); + if !ok { + return Err(LfmViolation::ArenaOutOfBounds { + arena: *arena, + index: *index, + }); + } + } + } + Ok(()) +} + +fn is_bool(v: &FE) -> bool { + *v == FE::zero() || *v == FE::one() +} + +/// Checks 5–6 on the emitted column groups: selector one-hot-ness on real +/// rows, all-zero padding beyond the program length — plus the shape +/// cross-check against the instruction partition. +fn check_groups(program: &LfmProgram) -> Result<(), LfmViolation> { + let g = &program.groups; + + let chip_real = |chip: &'static str, group: &ColumnGroup, count: usize| { + if group.real_rows == count { + Ok(()) + } else { + Err(LfmViolation::GroupShapeMismatch { chip }) + } + }; + let counts = partition_counts(&program.instrs); + chip_real("LFM_CONST", &g.const_, counts.const_)?; + chip_real("LFM_BALU", &g.balu, counts.balu)?; + chip_real("LFM_XALU", &g.xalu, counts.xalu)?; + chip_real("LFM_SELECT", &g.select, counts.select)?; + chip_real("LFM_BITDEC", &g.bitdec, counts.bitdec)?; + chip_real("LFM_HASH", &g.hash, counts.hash)?; + chip_real("LFM_KECCAK", &g.keccak, counts.keccak)?; + chip_real("LFM_BLAKE3", &g.blake3, counts.blake3)?; + chip_real("LFM_LANES", &g.lanes, counts.lanes)?; + chip_real("LFM_HINT", &g.hint, counts.hint)?; + chip_real("LFM_PUBLIC", &g.public, counts.public)?; + + // Selector one-hot / is_real flags on real rows. + one_hot( + &g.balu, + "LFM_BALU", + layout::balu::SEL_ADD, + layout::balu::NUM_SELECTORS, + )?; + one_hot( + &g.xalu, + "LFM_XALU", + layout::xalu::SEL_ADD, + layout::xalu::NUM_SELECTORS, + )?; + if program.hash16() { + // The width-16 socket has ONE mode, so its is-real flag is a flag. + flag_is_one(&g.hash, "LFM_HASH", super::p1w16_socket::cols::IS_REAL)?; + } else { + one_hot( + &g.hash, + "LFM_HASH", + layout::hash::MODE_C, + layout::hash::NUM_SELECTORS, + )?; + } + one_hot(&g.lanes, "LFM_LANES", layout::lanes::MODE_PACK, 2)?; + one_hot(&g.keccak, "LFM_KECCAK", layout::keccak::MODE_PERM, 2)?; + // `LFM_BLAKE3` has ONE mode, so its is-real flag is a flag rather than a + // one-hot run — the same shape `LFM_SELECT` and `LFM_BITDEC` have. The AIR + // keeps its own ungated `IS_BIT(MU)` belt-over-suspenders; this is what + // makes MU = 1 on every real row a checked property of the program. + flag_is_one(&g.blake3, "LFM_BLAKE3", layout::blake3::MU)?; + flag_is_one(&g.select, "LFM_SELECT", layout::select::IS_REAL)?; + flag_is_one(&g.bitdec, "LFM_BITDEC", layout::bitdec::IS_REAL)?; + flag_is_one(&g.public, "LFM_PUBLIC", layout::public::IS_REAL)?; + + // Padding: everything beyond the real rows is zero. + for (chip, group) in [ + ("LFM_CONST", &g.const_), + ("LFM_BALU", &g.balu), + ("LFM_XALU", &g.xalu), + ("LFM_SELECT", &g.select), + ("LFM_BITDEC", &g.bitdec), + ("LFM_HASH", &g.hash), + ("LFM_KECCAK", &g.keccak), + ("LFM_BLAKE3", &g.blake3), + ("LFM_LANES", &g.lanes), + ("LFM_HINT", &g.hint), + ("LFM_PUBLIC", &g.public), + ] { + for row in group.real_rows..group.padded_rows { + for col in 0..group.width { + if *group.at(row, col) != FE::zero() { + return Err(LfmViolation::DirtyPadding { chip, row }); + } + } + } + } + + check_mult_ranges(g, program.hash16())?; + Ok(()) +} + +/// The hard ceiling on any multiplicity, independent of the per-chip +/// accounting below and of how that accounting might drift: `2^32`, the same +/// canonical-range shape check 8 uses on the keccak tags. Under it no +/// multiplicity can reach the half of the field the bus reads as negative, and +/// no sum of them over a program of any buildable size can wrap the modulus — +/// which is what the LogUp argument needs in order to mean what it says. +const MULT_HARD_CAP: u64 = 1 << 32; + +/// The largest value a multiplicity may legally take. +/// +/// A multiplicity is the number of times its cell is read, so it cannot exceed +/// the number of reads the *whole program* emits — which is bounded by the +/// committed row counts times the per-chip count of `LfmMem` receivers. Those +/// counts mirror `chips::*::bus_interactions`; over-counting only widens an +/// upper bound (safe), under-counting would reject honest programs, so where +/// two receivers are mutually exclusive the count rounds up. +fn mult_bound(g: &LfmColumnGroups, hash16: bool) -> u64 { + let receives_per_row: [(&ColumnGroup, u64); 11] = [ + (&g.const_, 0), // reads nothing + (&g.balu, 3), // A, B, C + (&g.xalu, 3), // A, B, C + (&g.select, 3), // BIT, IN_L, IN_R + (&g.bitdec, 1), // IN + (&g.hash, if hash16 { 4 } else { 3 }), // IN0..IN2, or the socket's four cells + ( + &g.keccak, + (layout::keccak::NUM_WORDS + layout::keccak::BLOCK_WORDS) as u64, + ), // state + rate block + (&g.blake3, layout::blake3::IN_WORDS as u64), // h | m | params + (&g.lanes, 5), // the word (Unpack) or the four lanes (Pack) + (&g.hint, 0), // reads nothing + (&g.public, 1), // the published cell + ]; + let reads = receives_per_row.iter().fold(0u64, |acc, (group, per_row)| { + acc.saturating_add((group.real_rows as u64).saturating_mul(*per_row)) + }); + reads.min(MULT_HARD_CAP - 1) +} + +/// Every preprocessed column that gates an `LfmMem` **send** — i.e. every +/// write multiplicity — per chip, mirroring `chips::*::bus_interactions`. +/// +/// `LFM_PUBLIC` contributes none: it only receives, gated by `IS_REAL`, which +/// check 5 pins to 1. Every other receive gate is likewise a selector already +/// pinned to `{0,1}` on real rows by check 5 and to 0 on padding rows by +/// check 6 — the send gates listed here are the only unbounded ones. +fn mult_columns( + g: &LfmColumnGroups, + hash16: bool, +) -> Vec<(&'static str, &ColumnGroup, Vec)> { + use layout::{balu, bitdec, blake3, const_, hash, hint, keccak, lanes, select, xalu}; + vec![ + ("LFM_CONST", &g.const_, vec![const_::MULT]), + ("LFM_BALU", &g.balu, vec![balu::MULT]), + ("LFM_XALU", &g.xalu, vec![xalu::MULT]), + ( + "LFM_SELECT", + &g.select, + vec![select::MULT_L, select::MULT_R], + ), + ( + "LFM_BITDEC", + &g.bitdec, + (0..bitdec::NUM_BITS).map(bitdec::bit_mult).collect(), + ), + ( + "LFM_HASH", + &g.hash, + if hash16 { + (0..4) + .map(|k| super::p1w16_socket::cols::MULT0 + k) + .collect() + } else { + vec![hash::MULT0, hash::MULT1, hash::MULT2] + }, + ), + ( + "LFM_KECCAK", + &g.keccak, + (0..keccak::NUM_WORDS) + .map(keccak::mult) + .chain((0..keccak::DIGEST_WORDS).map(keccak::rev_mult)) + .collect(), + ), + ( + "LFM_BLAKE3", + &g.blake3, + (0..blake3::OUT_WORDS) + .map(blake3::mult) + .chain((0..blake3::DIGEST_WORDS).map(blake3::rev_mult)) + .collect(), + ), + ( + "LFM_LANES", + &g.lanes, + (0..4) + .map(|i| lanes::LANE_MULT0 + i) + .chain([lanes::WORD_MULT]) + .collect(), + ), + ("LFM_HINT", &g.hint, vec![hint::MULT]), + ] +} + +/// Check 9: every multiplicity column of every group holds a canonically +/// small non-negative integer. +/// +/// This is the one check on this list that is deliberately +/// **instrs-independent**. Checks 1–4 reach the committed columns only through +/// `program.instrs`, so any slot the `Instr` accessors do not expose is +/// invisible to them while still being emitted onto the bus, and the two +/// objects are joined by nothing but the emitter. Reading the committed +/// columns directly is what makes this hold whatever the instruction list +/// claims — and a field-negative multiplicity, the shape the LogUp count +/// argument cannot see (see [`LfmViolation::MultOutOfRange`]), is a canonical +/// value near `p` and dies here. +/// +/// Padding rows are skipped: check 6 already pins them to all-zero. +fn check_mult_ranges(g: &LfmColumnGroups, hash16: bool) -> Result<(), LfmViolation> { + let bound = mult_bound(g, hash16); + for (chip, group, cols) in mult_columns(g, hash16) { + for row in 0..group.real_rows { + for &col in &cols { + let mult = GoldilocksField::canonical(group.at(row, col).value()); + if mult > bound { + return Err(LfmViolation::MultOutOfRange { + chip, + row, + col, + mult, + bound, + }); + } + } + } + } + Ok(()) +} + +fn one_hot( + group: &ColumnGroup, + chip: &'static str, + first_sel: usize, + num_sels: usize, +) -> Result<(), LfmViolation> { + for row in 0..group.real_rows { + let mut ones = 0usize; + for s in 0..num_sels { + let v = group.at(row, first_sel + s); + if !is_bool(v) { + return Err(LfmViolation::NonOneHotSelector { chip, row }); + } + if *v == FE::one() { + ones += 1; + } + } + if ones != 1 { + return Err(LfmViolation::NonOneHotSelector { chip, row }); + } + } + Ok(()) +} + +fn flag_is_one(group: &ColumnGroup, chip: &'static str, col: usize) -> Result<(), LfmViolation> { + for row in 0..group.real_rows { + if *group.at(row, col) != FE::one() { + return Err(LfmViolation::NonOneHotSelector { chip, row }); + } + } + Ok(()) +} + +struct PartitionCounts { + const_: usize, + balu: usize, + xalu: usize, + select: usize, + bitdec: usize, + hash: usize, + keccak: usize, + blake3: usize, + lanes: usize, + hint: usize, + public: usize, +} + +fn partition_counts(instrs: &[Instr]) -> PartitionCounts { + let mut c = PartitionCounts { + const_: 0, + balu: 0, + xalu: 0, + select: 0, + bitdec: 0, + hash: 0, + keccak: 0, + blake3: 0, + lanes: 0, + hint: 0, + public: 0, + }; + for i in instrs { + match i { + Instr::Const { .. } => c.const_ += 1, + Instr::BaseAlu { .. } => c.balu += 1, + Instr::ExtAlu { .. } => c.xalu += 1, + Instr::Select { .. } => c.select += 1, + Instr::BitDec { .. } => c.bitdec += 1, + Instr::Hash { .. } | Instr::Hash16(_) => c.hash += 1, + Instr::KeccakF(_) => c.keccak += 1, + Instr::Blake3(_) => c.blake3 += 1, + Instr::Pack { .. } | Instr::Unpack { .. } => c.lanes += 1, + Instr::Hint { .. } | Instr::HintOn { .. } => c.hint += 1, + Instr::Public { .. } => c.public += 1, + } + } + c +} + +/// [`validate`] with check 4's recount named rather than read from +/// [`DENSE_COUNTS_ENV`], test-only. +#[cfg(test)] +fn validate_with(program: &LfmProgram, dense: bool) -> Result<(), LfmViolation> { + check_writes_and_reads(program)?; + check_multiplicities_with(program, dense)?; + check_arenas(program)?; + check_groups(program)?; + check_keccak_tags(&program.groups.keccak)?; + Ok(()) +} + +#[cfg(test)] +mod dense_count_tests { + use super::*; + use crate::hash_pin::{BlockFamily, ProofFamily}; + + /// A block node over two trivial children: every instruction kind a node + /// emits (width-16 hashes, unpacks, selects, bit-decs), at a laptop's size. + fn small_node() -> LfmProgram { + let opts = crate::lfm::proof::aggregation_wrap_options(); + let child = crate::lfm::programs::trivial_program(); + let artifacts = + crate::lfm::registry::census_artifacts(&child, ::SOCKET); + let layout = crate::lfm::block_node::BlockLayout::child(1, 2); + let derived = crate::lfm::per_table_aggregator::DerivedChild::from_artifacts_under::< + BlockFamily, + >(&artifacts, &opts, layout.total()) + .expect("the child derives"); + let shapes = [derived.shape(), derived.shape()]; + let mut b = crate::lfm::builder::LfmBuilder::new() + .with_wrap_hash(::WRAP); + crate::lfm::block_node::emit_block_node( + &mut b, + &crate::lfm::block_node::BlockNodeInputs { + children: &shapes, + layout, + top: false, + }, + ); + crate::lfm::compiler::compile(b.finish()) + } + + fn programs() -> Vec { + vec![crate::lfm::programs::trivial_program(), small_node()] + } + + /// Both recounts hold the same count at every address of a valid program, + /// and both admit it. + #[test] + fn dense_and_map_recounts_agree() { + for program in programs() { + let map = ReadCounts::of(&program, false); + let dense = ReadCounts::of(&program, true); + for a in 0..program.num_addrs + 2 { + assert_eq!(map.get(Addr(a)), dense.get(Addr(a)), "address {a}"); + } + assert_eq!(validate_with(&program, false), Ok(())); + assert_eq!(validate_with(&program, true), Ok(())); + } + } + + /// The dense recount refuses what the map refuses, with the same first + /// violation: a write's count raised, a count lowered, and a read moved to + /// another cell (one count up, one down). + #[test] + fn dense_and_map_refuse_alike() { + let mut tampered: Vec = Vec::new(); + let mut raised = small_node(); + for i in &mut raised.instrs { + if let Instr::Const { mult, .. } = i { + *mult += 1; + break; + } + } + tampered.push(raised); + let mut lowered = small_node(); + for i in lowered.instrs.iter_mut().rev() { + if let Instr::Hash16(h) = i + && h.mults[0] > 0 + { + h.mults[0] -= 1; + break; + } + } + tampered.push(lowered); + let mut moved = small_node(); + let last = moved.instrs.len() - 1; + for i in moved.instrs[..last].iter_mut().rev() { + if let Instr::BaseAlu { a, b, .. } = i + && a != b + { + *a = *b; + break; + } + } + tampered.push(moved); + for (k, bad) in tampered.iter().enumerate() { + let map = validate_with(bad, false); + assert!( + matches!(map, Err(LfmViolation::MultMismatch { .. })), + "tamper {k}: {map:?}" + ); + assert_eq!(validate_with(bad, true), map, "tamper {k}"); + } + } + + #[test] + fn the_dense_counts_knob_is_on_by_default_and_zero_rolls_back() { + assert!(parse_dense_counts(None)); + assert!(parse_dense_counts(Some(""))); + assert!(parse_dense_counts(Some("1"))); + assert!(!parse_dense_counts(Some("0"))); + assert!(std::panic::catch_unwind(|| parse_dense_counts(Some("yes"))).is_err()); + } +} diff --git a/prover/src/lfm/whir_air.rs b/prover/src/lfm/whir_air.rs new file mode 100644 index 000000000..8b81551ef --- /dev/null +++ b/prover/src/lfm/whir_air.rs @@ -0,0 +1,126 @@ +//! `IrShape::combine` as a machine leg — the zerocheck rule's value. +//! +//! The mirror is `stark::multilinear_air::IrShape::combine` +//! (`multilinear_air.rs:898-916`): the table's constraint DAG run over the +//! factor values, then one root per constraint batched by the powers of `beta`, +//! each root optionally gated by its selector. +//! +//! ```text +//! combine(β, v) = Σ_i β^i · node[root_i] · (selector_i ? v[slot_i] : 1) +//! ``` +//! +//! # ⛔ Why this is not `emit_program` pointed at `IrShape::program` +//! +//! `IrShape::program(&betas, weight)` builds the same expression as a +//! `Program`, and a `Program` is what [`super::whir_program::emit_program`] +//! already emits and is gated on. It cannot be used here, and the reason is the +//! whole shape of this leg: `Builder::weighted_sum` interns each batching +//! coefficient as a `Fixed` STEP (`program.rs:304-316`), and those coefficients +//! are the powers of `beta` — a challenge the verifier draws from the +//! transcript. `emit_program` interns every `Fixed` as an `LFM_CONST`, which is +//! right for the AIR's own constants and wrong for a value that does not exist +//! until the proof is being read. +//! +//! So the DAG half goes through `emit_steps` — the same walk, the same +//! constants, already gated — and the batching half is emitted here with the +//! beta powers as WIRES. The accessors that make the structure readable +//! (`steps_as_ops`, `root_steps`, `selector_of_root`) were added to `IrShape` +//! for exactly this and nothing else. +//! +//! # The factor ordering, and what `values` must hold +//! +//! `Var(i)` reads factor `i`, and a selector names a factor slot, so `values` is +//! the woven factor vector `constraint_argument::verify_core` builds: the +//! committed factors in kind order, then the public ones, then the two weight +//! tables (`multilinear_table.rs:616-618`). The rule's own value is this times +//! the weight at `weights(kinds.len()).0`, which the CALLER applies — +//! `combine` does not, and the gate is on `combine`. +//! +//! # ⚠ One commutative reordering, stated rather than hidden +//! +//! The host multiplies the node by `β^i` and then by the selector; this +//! multiplies by the selector and then folds `β^i` in with a `MulAdd`. Same +//! value, same row count — a field multiplication is commutative and the fold +//! is one row either way. It is a rewrite of the host's expression, not a +//! difference, and it is written down because "the operands are in the other +//! order" is exactly what a reader should be able to check rather than wonder +//! about. + +use stark::multilinear_air::IrShape; + +use crate::tables::types::{FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::{Ext, LfmBuilder}; +use super::whir_program::{emit_steps, steps_rows}; + +type Shape = IrShape; + +/// INSTRUCTIONS [`emit_combine`] emits, constants included. +/// +/// Every term by the shape it comes from: +/// +/// - the DAG's own rows, `steps_rows` — one `LFM_XALU` per operation node, one +/// `LFM_CONST` per distinct `Fixed` value, and the zero a `Neg` subtracts +/// from; +/// - one `Mul` per root that carries a SELECTOR, and none for a root that +/// applies on every step; +/// - one row per root to batch it: a `Mul` for the first, which opens the +/// accumulator, and a `MulAdd` for every later one. +/// +/// A shape with no roots emits one `LFM_CONST` zero and nothing else, which is +/// what `combine`'s fold from zero comes to. +pub fn combine_rows(shape: &Shape) -> usize { + let roots = shape.root_steps().len(); + if roots == 0 { + return 1; + } + let selectors = shape + .selector_of_root() + .iter() + .filter(|selector| selector.is_some()) + .count(); + steps_rows(&shape.steps_as_ops()) + selectors + roots +} + +/// ★ `IrShape::combine(beta_powers, values)`, emitted with the beta powers as +/// runtime wires. +/// +/// Panics if the number of beta powers is not the number of roots, or if a step +/// or a selector reads a factor the caller did not supply — every one of those +/// is an emit-time length, so a mismatch is a bug in the caller rather than a +/// condition to carry at runtime. The host would return +/// `VariableCountMismatch` for the first and index out of bounds for the rest. +pub fn emit_combine(b: &mut LfmBuilder, shape: &Shape, betas: &[Ext], values: &[Ext]) -> Ext { + assert_eq!( + betas.len(), + shape.num_roots(), + "one beta power per batched root — the host's `beta_powers` length check" + ); + let ops = shape.steps_as_ops(); + let nodes = emit_steps(b, &ops, values); + + let mut batched: Option = None; + for ((&root, &beta), selector) in shape + .root_steps() + .iter() + .zip(betas) + .zip(shape.selector_of_root()) + { + let mut term = nodes[root as usize]; + if let Some(slot) = selector { + let gate = *values.get(*slot).unwrap_or_else(|| { + panic!( + "a root's selector reads factor {slot}, but only {} were supplied", + values.len() + ) + }); + term = b.emul(term, gate); + } + batched = Some(match batched { + None => b.emul(beta, term), + Some(accumulated) => b.emul_add(beta, term, accumulated), + }); + } + + batched.unwrap_or_else(|| b.ext_const(&FEE::zero())) +} diff --git a/prover/src/lfm/whir_air_tests.rs b/prover/src/lfm/whir_air_tests.rs new file mode 100644 index 000000000..7ac854d60 --- /dev/null +++ b/prover/src/lfm/whir_air_tests.rs @@ -0,0 +1,304 @@ +//! Gates for the zerocheck rule's value. +//! +//! The shape under test is a REAL one: `l2g_memory_air`'s, laid out by the same +//! `TableLayout::new` both sides of the argument call (`multilinear_prove.rs`'s +//! `layout_of`), so the DAG, its roots, its selectors and its factor kinds are +//! the production ones. Its `num_vars` is small because the SHAPE is what the +//! emitter reproduces and the shape does not vary with the row count — epoch-0 +//! heights belong to the census and to the box, not to this gate. +//! +//! ⚠ MEASURED, and the reason a second shape is here: the l2g memory AIR +//! compiles to ONE root with NO selector, so on it alone the batching fold and +//! the selector multiply are terms the form states and the gate never reaches. +//! [`ThreeRootsOneSelector`] carries both, through the same compiler. + +use stark::multilinear_air::{IrShape, Uniforms, beta_powers}; +use stark::multilinear_table::TableLayout; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_air::{combine_rows, emit_combine}; +use super::word::{ext_word, word_as_ext}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type Shape = IrShape; + +fn pseudo(seed: u64, count: usize) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..count) + .map(|_| FEE::new([next(), next(), next()])) + .collect() +} + +fn options() -> stark::proof::options::ProofOptions { + stark::proof::options::GoldilocksCubicProofOptions::with_params(4, 128, 20) + .expect("valid options") +} + +/// The production shape: the local-to-global memory AIR, laid out exactly as +/// both sides of a real proof lay it out. +/// +/// Chosen because it is a real AIR that this laptop can build — most of the +/// VM's tables need the guest ELF, and the two global AIRs beside this one +/// carry `EmptyConstraints` and so have no roots to batch at all. +fn production_shape(num_vars: usize) -> Shape { + let opts = options(); + let air = + crate::continuation::l2g_memory_air(&opts, crate::tables::local_to_global::epoch_label(1)); + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + crate::tables::local_to_global::cols::NUM_COLUMNS, + num_vars, + Uniforms::default(), + ) + .expect("the table lays out"); + layout.shape().clone() +} + +/// Three roots, one of them carrying a selector — the terms the production AIR +/// above does not have. +/// +/// ⚠ A TEST constraint set, and said so: what makes it worth gating against is +/// that it goes through the SAME pipeline — `AirWithBuses::new` and +/// `TableLayout::new` — so its `IrShape` is built by the real compiler, with +/// real roots, a real selector and real factor slots. The production AIR says +/// the pipeline is the production one; this says the form is right about the +/// roots that AIR happens not to have. Neither alone would do. +/// +/// `except_last(1)` is what gives a root a selector: a constraint that reads one +/// row ahead cannot apply on the last row (`builder.rs:177-195`). +struct ThreeRootsOneSelector; + +impl stark::constraints::builder::ConstraintSet for ThreeRootsOneSelector { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + let x = b.main(0, 0); + let y = b.main(0, 1); + let z = b.main(0, 2); + b.emit_base(0, x * y - z); + + let x = b.main(0, 0); + let y = b.main(0, 1); + let z = b.main(0, 2); + b.emit_base(1, x + y - z); + + let next = b.main(1, 0); + let here = b.main(0, 0); + b.emit_base_rows( + 2, + stark::constraints::builder::RowDomain::except_last(1), + next - here, + ); + } +} + +fn stress_shape(num_vars: usize) -> Shape { + let opts = options(); + let air = stark::lookup::AirWithBuses::< + F, + E, + stark::lookup::NullBoundaryConstraintBuilder, + (), + ThreeRootsOneSelector, + >::new( + crate::tables::local_to_global::cols::NUM_COLUMNS, + stark::lookup::AuxiliaryTraceBuildData { + // ⚠ The layout needs a bus — `TableLayout::new` refuses a table with + // no interactions (`EmptyPolynomial`, measured). These are the + // production AIR's own, so the column indices they read exist. + interactions: crate::tables::local_to_global::memory_bus_interactions(), + }, + &opts, + 1, + ThreeRootsOneSelector, + ); + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + crate::tables::local_to_global::cols::NUM_COLUMNS, + num_vars, + Uniforms::default(), + ) + .expect("the table lays out"); + layout.shape().clone() +} + +/// The shapes under gate, with a name for the failure message. +/// +/// ⚠ Both are the SAME AIR at two heights, because it is the only real AIR this +/// laptop can build with constraints on it: every VM table needs the guest ELF, +/// and the two global AIRs beside it carry `EmptyConstraints` and have no roots +/// to batch. Whatever roots and selectors this one does not have are terms the +/// form states and this gate does not exercise — said here rather than left to +/// read as full coverage. +fn shapes() -> Vec<(&'static str, Shape)> { + vec![ + ("l2g_memory num_vars=4", production_shape(4)), + ("l2g_memory num_vars=7", production_shape(7)), + ("three roots, one selector, num_vars=4", stress_shape(4)), + ("three roots, one selector, num_vars=6", stress_shape(6)), + ] +} + +/// Values for every factor the shape can read, plus the two weight slots the +/// woven factor vector carries (`multilinear_table.rs:616-618`). +fn factor_values(shape: &Shape, seed: u64) -> Vec { + let slots = shape + .steps_as_ops() + .iter() + .filter_map(|op| match op { + multilinear::program::Op::Var(i) => Some(*i as usize), + _ => None, + }) + .chain(shape.selector_of_root().iter().filter_map(|s| *s)) + .max() + .map(|m| m + 1) + .unwrap_or(0); + pseudo(seed, slots.max(1) + 2) +} + +/// `combine` over hinted betas and hinted factor values, the answer published. +fn combine_program(shape: &Shape, num_values: usize) -> LfmProgram { + let roots = shape.num_roots(); + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena((roots + num_values) as u32); + let betas: Vec<_> = (0..roots) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let values: Vec<_> = (0..num_values) + .map(|i| b.hint_word(arena, (roots + i) as u32).as_ext()) + .collect(); + let combined = emit_combine(&mut b, shape, &betas, &values); + b.public(combined.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the combine leg must be admissible"); + program +} + +/// The leg's marginal cost: the same program without it. +fn marginal_rows(shape: &Shape, num_values: usize) -> usize { + let roots = shape.num_roots(); + let with = combine_program(shape, num_values); + let without = { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena((roots + num_values) as u32); + let first = b.hint_word(arena, 0).as_ext(); + for i in 1..roots + num_values { + let _ = b.hint_word(arena, i as u32); + } + b.public(first.as_cell()); + compile(b.finish()) + }; + with.instrs.len() - without.instrs.len() +} + +/// ★ The leg computes what the host computes, at RANDOM betas. +/// +/// The betas are the point: the host takes them as an argument, and the whole +/// reason this leg exists rather than `emit_program` is that they are drawn +/// after the commitment and cannot be interned. A gate that fixed them at one +/// value would pass for an emitter that had baked them in. +#[test] +fn the_combine_leg_computes_what_the_host_computes() { + for (name, shape) in shapes() { + let values = factor_values(&shape, 0xC0FFEE); + let program = combine_program(&shape, values.len()); + for seed in [0x11u64, 0x22, 0x33] { + let betas_free = pseudo(seed, shape.num_roots()); + // Both a free vector and a real power ladder: `beta_powers` is what + // the caller actually supplies, and its first element is ONE, which + // is the case an emitter that dropped the first multiply would pass. + let ladder = beta_powers(&pseudo(seed ^ 0xFF, 1)[0], shape.num_roots()); + for betas in [betas_free, ladder] { + let arenas = vec![ + betas + .iter() + .chain(values.iter()) + .map(ext_word) + .collect::>(), + ]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("{name}: the combine leg must execute: {e:?}")); + let got = + word_as_ext(&exec.public_words[0].1).expect("a published extension value"); + let want = shape.combine(&betas, &values); + assert_eq!(got, want, "{name} at seed {seed:#x}"); + } + } + println!( + "combine {name}: {} roots, {} with selectors, {} factors supplied", + shape.num_roots(), + shape + .selector_of_root() + .iter() + .filter(|s| s.is_some()) + .count(), + values.len() + ); + } +} + +/// ★ F1 for the leg: its rows against the closed form. +#[test] +fn the_combine_leg_emits_its_closed_form() { + for (name, shape) in shapes() { + let values = factor_values(&shape, 0xC0FFEE); + let measured = marginal_rows(&shape, values.len()); + let predicted = combine_rows(&shape); + println!( + "combine {name}: {measured} rows emitted, {predicted} predicted \ + ({} DAG steps, {} roots)", + shape.steps_as_ops().len(), + shape.num_roots() + ); + assert_eq!(measured, predicted, "{name}"); + } +} + +/// The batching is a function of the BETAS and not a sum: scaling one beta +/// scales exactly that root's contribution. +/// +/// A leg that summed the roots and multiplied once, or that reused one beta for +/// every root, agrees with the host wherever the betas happen to be equal. This +/// is the linearity that says each root got its own. +#[test] +fn each_root_carries_its_own_beta() { + let shape = stress_shape(4); + let roots = shape.num_roots(); + assert!(roots > 1, "this shape exists to batch more than one root"); + let values = factor_values(&shape, 0xBEEF); + + let base = vec![FEE::one(); roots]; + let combined = shape.combine(&base, &values); + for i in 0..roots { + let mut bumped = base.clone(); + bumped[i] = FEE::one() + FEE::one(); + // Doubling one beta adds exactly that root's own term. + let delta = &shape.combine(&bumped, &values) - &combined; + let mut only = vec![FEE::zero(); roots]; + only[i] = FEE::one(); + assert_eq!( + delta, + shape.combine(&only, &values), + "root {i}'s contribution must be what its own beta scales" + ); + } +} diff --git a/prover/src/lfm/whir_batch.rs b/prover/src/lfm/whir_batch.rs new file mode 100644 index 000000000..5fb2fc33e --- /dev/null +++ b/prover/src/lfm/whir_batch.rs @@ -0,0 +1,893 @@ +//! The batched argue as a machine leg (D-BATCH B-4a): the in-guest mirror of +//! `stark::multilinear_table::batched::verify_argue`. +//! +//! One lockstep GKR ladder per bin, one front-loaded constraint sumcheck over +//! every table, and a claim reduction only for a table with a shifted read — +//! where the per-table leg ([`super::whir_table::emit_table_verify`]) runs all +//! three per table. The plan is the host's own `argue_plan`, called, never +//! restated: the bins are part of the transcript. +//! +//! Like the per-table leg this DERIVES every challenge from the transcript it +//! drives, so executing an honest proof is the challenge-stream comparison: +//! a wrong challenge anywhere leaves one of the refusals below unsatisfiable — +//! each ladder step's relation, the constraint sumcheck's final equation, a +//! shifted table's reduction, and a column read twice claimed twice the same. +//! +//! ⛔ `check_preprocessed`, the bus balance and the openings are the caller's, +//! exactly as the host's `verify_argue` leaves them. + +use multilinear::claim_reduce::FactorSource; +use multilinear::constraint_argument::FactorKind; +use stark::multilinear_table::{ArguePlan, weight_slots}; + +use crate::tables::types::FEE; + +use super::algebraic_commit::leaf_capacity; +use super::builder::{Ext, LfmBuilder}; +use super::whir_air::{combine_rows, emit_combine}; +use super::whir_bus::{ + BusInputs, Cost, claim_statements_cost, emit_claim_statements, emit_eq_evals, eq_evals_cost, +}; +use super::whir_gkr::GKR_SUMCHECK_DEGREE; +use super::whir_poly::{ + challenge_powers_rows, emit_challenge_powers, emit_eq_eval, emit_sumcheck_rounds, + eq_eval_rows_again, sumcheck_round_rows, +}; +use super::whir_reduce::{REDUCE_DEGREE, ReduceWires, claim_reduce_rows, emit_claim_reduce_verify}; +use super::whir_table::{ + TableCost, TableShape, dag_constant_rows, emit_selector, newton_constants, selector_cost, weave, +}; +use super::whir_transcript::{ + COORDINATES_PER_EXT, SpongeEntry, SpongeSchedule, WhirTranscript, absorb_unpack_rows, + sample_ext_rows, +}; + +/// Rows an `assert_eq_ext` lowers to (`builder.rs:289-292`). +const ASSERT_ROWS: usize = 2; + +/// One ladder step as wires: the shared sumcheck's rounds (`i` of them at step +/// `i`, three evaluations each) and every active tree's four halves. +pub struct LadderStepWires { + pub sumcheck: Vec>, + /// `[p_lo, p_hi, q_lo, q_hi]` per active tree, in table order. + pub halves: Vec<[Ext; 4]>, +} + +/// The `BatchedArgue` as wires. Every field is proof data. +pub struct BatchedArgueWires<'a> { + /// Per tree, `(p, q)`: one a table, or one for each chunk of a table the + /// plan cuts ([`ArguePlan::chunks`]), table order then chunk order. + pub bus_outputs: &'a [(Ext, Ext)], + /// Per bin, per step. + pub gkr: &'a [Vec], + /// `n_max` rounds of `D_max` evaluations. + pub constraint: &'a [Vec], + /// Per table, its committed factors' values at `r_{], + /// Per table, its reduction: present exactly for a table with a shift. + pub reduces: &'a [Option>], +} + +/// What the leg leaves per table. +pub struct BatchedVerdictWires { + /// Per table, its bus output: its one tree's, or — a cut table — its + /// chunks' fractions added, `Σ p_j/q_j` as one `(p, q)`. + pub bus_outputs: Vec<(Ext, Ext)>, + /// Per table, the point its columns are claimed at and their values. + pub tables: Vec<(Vec, Vec)>, +} + +/// `Σ_a w_{2a}·(p_lo q_hi + p_hi q_lo) + w_{2a+1}·q_lo q_hi` with `w_0` the +/// literal one: `5|A| − 1` rows. +fn emit_relation(b: &mut LfmBuilder, weights: &[Ext], halves: &[[Ext; 4]]) -> Ext { + let mut acc: Option = None; + for (a, h) in halves.iter().enumerate() { + let [p_lo, p_hi, q_lo, q_hi] = *h; + let cross = b.emul(p_lo, q_hi); + let numerator = b.emul_add(p_hi, q_lo, cross); + let denominator = b.emul(q_lo, q_hi); + let with_numerator = match acc { + None => numerator, + Some(running) => b.emul_add(weights[2 * a], numerator, running), + }; + acc = Some(b.emul_add(weights[2 * a + 1], denominator, with_numerator)); + } + acc.expect("a ladder step has an active tree") +} + +/// `lo + c·(hi − lo)`. +fn emit_line(b: &mut LfmBuilder, lo: Ext, hi: Ext, c: Ext) -> Ext { + let spread = b.esub(hi, lo); + b.emul_add(c, spread, lo) +} + +/// A cut table's chunk claims, all at one point `P`, combined at `s'` into +/// the claim of its one padded tree at `s' ‖ P` — the host's +/// `combine_chunk_claims`: `p̃ = Σ_{j, Ext, Ext)], + combine: &[Ext], +) -> (Vec, Ext, Ext) { + let weights = emit_eq_evals(b, combine); + assert!( + chunks.len() <= weights.len(), + "the chunks fit the combination" + ); + // One height in one bin: the ladder left every chunk at the same wires. + assert!( + chunks.iter().all(|(at, _, _)| at.len() == chunks[0].0.len() + && at.iter().zip(&chunks[0].0).all(|(x, y)| x.0 == y.0)), + "a cut table's chunks end at one point" + ); + let mut p: Option = None; + let mut q: Option = None; + for ((_, pj, qj), w) in chunks.iter().zip(&weights) { + p = Some(match p { + None => b.emul(*w, *pj), + Some(acc) => b.emul_add(*w, *pj, acc), + }); + q = Some(match q { + None => b.emul(*w, *qj), + Some(acc) => b.emul_add(*w, *qj, acc), + }); + } + let p = p.expect("a cut table has chunks"); + let mut q = q.expect("a cut table has chunks"); + for w in &weights[chunks.len()..] { + q = b.eadd(q, *w); + } + let point = combine.iter().chain(&chunks[0].0).copied().collect(); + (point, p, q) +} + +/// A cut table's chunk outputs as one fraction: `p/q = Σ_j p_j/q_j`, three rows +/// a chunk after the first. +fn emit_add_fractions(b: &mut LfmBuilder, outputs: &[(Ext, Ext)]) -> (Ext, Ext) { + let (mut p, mut q) = outputs[0]; + for &(pj, qj) in &outputs[1..] { + let scaled = b.emul(p, qj); + p = b.emul_add(pj, q, scaled); + q = b.emul(q, qj); + } + (p, q) +} + +/// ★ `verify_argue`, emitted. +/// +/// `z`, `alpha_powers` and `beta` are the proof's shared challenges, drawn by +/// the roots block before this leg. `shapes` are the tables' structures, in +/// table order, and `plan` the host's `argue_plan` over them. +#[allow(clippy::too_many_arguments)] +pub fn emit_batched_argue( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + proof: &BatchedArgueWires<'_>, + shapes: &[TableShape<'_>], + plan: &ArguePlan, + z: Ext, + alpha_powers: &[Ext], + beta: Ext, +) -> BatchedVerdictWires { + let n = shapes.len(); + assert_eq!(plan.shapes.len(), n, "the plan is over these tables"); + assert_eq!( + proof.bus_outputs.len(), + plan.trees(), + "one bus output per tree" + ); + assert_eq!(proof.gkr.len(), plan.bins.len(), "one ladder per bin"); + assert_eq!(proof.factor_values.len(), n, "factor values per table"); + assert_eq!(proof.reduces.len(), n, "a reduction slot per table"); + + // ── G: per bin, its trees' outputs, then its ladder, then each cut + // table's chunks combined ── + let mut claims: Vec, Ext, Ext)>> = vec![None; n]; + let mut outputs: Vec> = vec![None; n]; + for (bin, ladder) in plan.bins.iter().zip(proof.gkr) { + // The bin's trees: each table's chunks, in order, as `(table, tree)`. + let members: Vec<(usize, usize)> = bin + .iter() + .flat_map(|&t| { + let first = plan.first_tree(t); + (0..plan.chunks[t].count).map(move |j| (t, first + j)) + }) + .collect(); + for &(_, tree) in &members { + transcript.absorb_ext(b, proof.bus_outputs[tree].0); + transcript.absorb_ext(b, proof.bus_outputs[tree].1); + } + let heights: Vec = members.iter().map(|&(t, _)| plan.tree_vars(t)).collect(); + let steps = heights.iter().copied().max().unwrap_or(0); + assert_eq!(ladder.len(), steps, "the ladder runs to the tallest tree"); + let mut running: Vec<(Ext, Ext)> = members + .iter() + .map(|&(_, tree)| proof.bus_outputs[tree]) + .collect(); + let mut leaves: Vec, Ext, Ext)>> = vec![None; members.len()]; + for a in 0..members.len() { + if heights[a] == 0 { + leaves[a] = Some((Vec::new(), running[a].0, running[a].1)); + } + } + let mut point: Vec = Vec::new(); + for (i, step) in ladder.iter().enumerate() { + let active: Vec = (0..members.len()).filter(|&a| heights[a] > i).collect(); + assert_eq!(step.halves.len(), active.len(), "halves per active tree"); + assert_eq!(step.sumcheck.len(), i, "step i's sumcheck has i rounds"); + let mu = transcript.sample_ext(b); + let mut rounds = Vec::with_capacity(i); + for evaluations in &step.sumcheck { + assert_eq!(evaluations.len(), GKR_SUMCHECK_DEGREE); + for e in evaluations { + transcript.absorb_ext(b, *e); + } + rounds.push(transcript.sample_ext(b)); + } + for h in &step.halves { + for v in h { + transcript.absorb_ext(b, *v); + } + } + let c = transcript.sample_ext(b); + + let weights = emit_challenge_powers(b, mu, 2 * active.len()); + let mut claimed: Option = None; + for (k, &a) in active.iter().enumerate() { + let (p, q) = running[a]; + let with_p = match claimed { + None => p, + Some(acc) => b.emul_add(weights[2 * k], p, acc), + }; + claimed = Some(b.emul_add(weights[2 * k + 1], q, with_p)); + } + let claimed = claimed.expect("a step has an active tree"); + let residual = emit_sumcheck_rounds(b, claimed, &step.sumcheck, &rounds); + let eq_at = emit_eq_eval(b, &point, &rounds); + let relation = emit_relation(b, &weights, &step.halves); + let expected = b.emul(eq_at, relation); + b.assert_eq_ext(expected, residual); + + point = std::iter::once(c).chain(rounds).collect(); + for (k, &a) in active.iter().enumerate() { + let [p_lo, p_hi, q_lo, q_hi] = step.halves[k]; + running[a] = (emit_line(b, p_lo, p_hi, c), emit_line(b, q_lo, q_hi, c)); + if heights[a] == i + 1 { + leaves[a] = Some((point.clone(), running[a].0, running[a].1)); + } + } + } + // Each table's claim and output: its one tree's, or its chunks' + // combined at `s'`, drawn after the bin's ladder (the host's + // `table_claim`). + let mut at_tree = 0; + for &t in bin { + let count = plan.chunks[t].count; + let chunks: Vec<(Vec, Ext, Ext)> = leaves[at_tree..at_tree + count] + .iter() + .map(|c| c.clone().expect("every tree reaches its input layer")) + .collect(); + let first = plan.first_tree(t); + let tree_outputs = &proof.bus_outputs[first..first + count]; + at_tree += count; + if count == 1 { + claims[t] = chunks.into_iter().next(); + outputs[t] = Some(tree_outputs[0]); + continue; + } + let combine: Vec = (0..plan.chunks[t].combine_vars()) + .map(|_| transcript.sample_ext(b)) + .collect(); + claims[t] = Some(emit_combine_chunks(b, &chunks, &combine)); + outputs[t] = Some(emit_add_fractions(b, tree_outputs)); + } + } + let claims: Vec<(Vec, Ext, Ext)> = claims + .into_iter() + .map(|c| c.expect("every table's tree reaches its input layer")) + .collect(); + let outputs: Vec<(Ext, Ext)> = outputs + .into_iter() + .map(|o| o.expect("every table has its output")) + .collect(); + + // ── C: one front-loaded sumcheck ── + let xi: Vec = (0..plan.num_vars) + .map(|_| transcript.sample_ext(b)) + .collect(); + for (_, p, q) in &claims { + transcript.absorb_ext(b, *p); + transcript.absorb_ext(b, *q); + } + let lambda = transcript.sample_ext(b); + assert_eq!(proof.constraint.len(), plan.num_vars, "n_max rounds"); + let mut r: Vec = Vec::with_capacity(plan.num_vars); + for evaluations in proof.constraint { + assert_eq!(evaluations.len(), plan.degree, "D_max evaluations a round"); + for e in evaluations { + transcript.absorb_ext(b, *e); + } + r.push(transcript.sample_ext(b)); + } + + let lambda2 = b.emul(lambda, lambda); + let lambda3 = b.emul(lambda2, lambda); + let weights = emit_challenge_powers(b, lambda3, n); + let mut claimed: Option = None; + for (t, (_, p, q)) in claims.iter().enumerate() { + let inner = b.emul(lambda, *p); + let inner = b.emul_add(lambda2, *q, inner); + claimed = Some(match claimed { + None => inner, + Some(acc) => b.emul_add(weights[t], inner, acc), + }); + } + let claimed = claimed.expect("a proof argues a table"); + let residual = emit_sumcheck_rounds(b, claimed, proof.constraint, &r); + + // `Π_{j ≥ m} r_j` for each table's `m = n_T < n_max`, from the end. + let lowest = shapes.iter().map(|s| s.num_vars).min().unwrap_or(0); + let mut suffix: Vec> = vec![None; plan.num_vars + 1]; + for j in (lowest..plan.num_vars).rev() { + suffix[j] = Some(match suffix[j + 1] { + None => r[j], + Some(rest) => b.emul(r[j], rest), + }); + } + let max_roots = shapes.iter().map(|s| s.ir.num_roots()).max().unwrap_or(0); + let betas = emit_challenge_powers(b, beta, max_roots); + let mut rebuilt: Option = None; + for (t, shape) in shapes.iter().enumerate() { + let num_vars = shape.num_vars; + let at = &r[..num_vars]; + let (weight_r, weight_z) = weight_slots(shape.kinds.len()); + let (claim_point, _, _) = &claims[t]; + let row_point = &claim_point[claim_point.len() - num_vars..]; + let mut public: Vec = shape + .ir + .public_selectors() + .iter() + .map(|selector| emit_selector(b, *selector, at)) + .collect(); + public.push(emit_eq_eval(b, &xi[..num_vars], at)); + public.push(emit_eq_eval(b, row_point, at)); + let values = weave(shape.kinds, &proof.factor_values[t], &public); + let combined = emit_combine(b, shape.ir, &betas[..shape.ir.num_roots()], &values); + let zerocheck = b.emul(values[weight_r], combined); + let bus = emit_claim_statements( + b, + shape.bus, + &BusInputs { + claim_point, + num_row_vars: num_vars, + z, + alpha_powers, + values: &values, + weight: weight_z, + }, + ); + let value = b.emul_add(lambda, bus.numerator, zerocheck); + let value = b.emul_add(lambda2, bus.denominator, value); + let term = match suffix[num_vars] { + Some(padding) if num_vars < plan.num_vars => b.emul(value, padding), + _ => value, + }; + rebuilt = Some(match rebuilt { + None => term, + Some(acc) => b.emul_add(weights[t], term, acc), + }); + } + let rebuilt = rebuilt.expect("a proof argues a table"); + // The host's `BatchMismatch`. + b.assert_eq_ext(rebuilt, residual); + for values in proof.factor_values { + for v in values { + transcript.absorb_ext(b, *v); + } + } + + // ── Rd ── + let mut tables = Vec::with_capacity(n); + for (t, shape) in shapes.iter().enumerate() { + let at = &r[..shape.num_vars]; + let sources = shape.sources(); + let factor_values = &proof.factor_values[t]; + match (plan.shapes[t].shifted, &proof.reduces[t]) { + (true, Some(reduce)) => { + let reduced = emit_claim_reduce_verify( + b, + transcript, + reduce, + &sources, + factor_values, + at, + shape.num_columns, + ); + tables.push((reduced.point, reduced.column_values)); + } + (false, None) => { + let columns = emit_direct_columns(b, &sources, factor_values, shape.num_columns); + tables.push((at.to_vec(), columns)); + } + _ => panic!("table {t}: a reduction is present exactly for a table with a shift"), + } + } + + BatchedVerdictWires { + bus_outputs: outputs, + tables, + } +} + +/// A table's column claims when every committed factor reads its column +/// unshifted: each column's claim is its factor's value, and a column read +/// twice must be claimed the same both times (an assert each). +fn emit_direct_columns( + b: &mut LfmBuilder, + sources: &[FactorSource], + factor_values: &[Ext], + num_columns: usize, +) -> Vec { + assert_eq!(sources.len(), factor_values.len(), "one value per factor"); + let mut columns: Vec> = vec![None; num_columns]; + for (source, value) in sources.iter().zip(factor_values) { + assert_eq!(source.offset, 0, "an unshifted table"); + match columns[source.column] { + Some(claimed) => b.assert_eq_ext(claimed, *value), + None => columns[source.column] = Some(*value), + } + } + columns + .into_iter() + .map(|c| c.expect("every column is read by a factor")) + .collect() +} + +/// Each of a bin's trees' input variables, in the ladder's order: each +/// table's chunks, in table order. +fn tree_heights(plan: &ArguePlan, bin: &[usize]) -> Vec { + bin.iter() + .flat_map(|&t| std::iter::repeat_n(plan.tree_vars(t), plan.chunks[t].count)) + .collect() +} + +/// `absorb_ext` on both halves: one `Unpack` and three felts into the sponge. +fn absorb_ext(leg: &mut Cost, schedule: &mut SpongeSchedule, count: usize) { + leg.ops(count * absorb_unpack_rows()); + for _ in 0..count { + schedule.absorb(COORDINATES_PER_EXT); + } +} + +/// `sample_ext` on both halves: one `Pack` and three candidates. +fn draw_ext(leg: &mut Cost, schedule: &mut SpongeSchedule, count: usize) { + leg.ops(count * sample_ext_rows()); + for _ in 0..count { + schedule.draw_ext(); + } +} + +/// ★ INSTRUCTIONS and permutations [`emit_batched_argue`] costs, from the +/// shapes and the plan alone — every term by the emitter's own shape: +/// +/// - G, per bin: two absorbs per table; per step `i` with `A` active trees, a +/// draw, `i` rounds of three absorbs and a draw, `4A` absorbs and a draw; +/// then `challenge_powers_rows(2A)`, `2A − 1` rows for the claim, `i` degree-3 +/// rounds, `eq` over `i`, `5A − 1` for the relation, one `Mul` by `eq`, the +/// assert, and `4A` for the two lines per tree; +/// - C: `n_max` draws for `ξ`, `2|T|` absorbs, the `λ` draw, `n_max` rounds of +/// `D_max` absorbs and a draw; `λ²`, `λ³`, `challenge_powers_rows(|T|)`, +/// `3|T| − 1` rows for the claim, `n_max` degree-`D_max` rounds, the suffix +/// products (`n_max − min n_T − 1` rows when some table is short), the beta +/// ladder once at the widest table's roots; per table its selectors, two +/// `eq`s, `combine_rows` less its constants, one `Mul`, the bus statements, +/// two rows for the rule, one for the padding when short, one to fold (none +/// for the first); the assert; one absorb per factor value; +/// - Rd: a shifted table's `claim_reduce_rows` and sponge; an unshifted one's +/// asserts for columns read twice. +pub fn batched_argue_cost( + shapes: &[TableShape<'_>], + plan: &ArguePlan, + entry: SpongeEntry, +) -> TableCost { + let n = shapes.len(); + let mut leg = Cost::default(); + let mut schedule = SpongeSchedule::new(entry); + let one = FEE::one(); + let zero = FEE::zero(); + let mut degrees: Vec = Vec::new(); + + // ── G ── + for bin in &plan.bins { + let heights = tree_heights(plan, bin); + absorb_ext(&mut leg, &mut schedule, 2 * heights.len()); + let steps = heights.iter().copied().max().unwrap_or(0); + for i in 0..steps { + let active = heights.iter().filter(|&&h| h > i).count(); + draw_ext(&mut leg, &mut schedule, 1); + for _ in 0..i { + absorb_ext(&mut leg, &mut schedule, GKR_SUMCHECK_DEGREE); + draw_ext(&mut leg, &mut schedule, 1); + } + absorb_ext(&mut leg, &mut schedule, 4 * active); + draw_ext(&mut leg, &mut schedule, 1); + + leg.constant(one); + leg.ops(challenge_powers_rows(2 * active)); + leg.ops(2 * active - 1); + leg.ops(i * sumcheck_round_rows(GKR_SUMCHECK_DEGREE)); + if i > 0 { + degrees.push(GKR_SUMCHECK_DEGREE); + } + leg.ops(eq_eval_rows_again(i)); + leg.ops(5 * active - 1); + leg.op(); + leg.ops(ASSERT_ROWS); + leg.constant(zero); + leg.ops(4 * active); + } + // A cut table's `ℓ` draws, its eq table over them, the two sums and + // the padding chunks' weights, and its outputs added as fractions. + for &t in bin { + let chunks = &plan.chunks[t]; + if chunks.count == 1 { + continue; + } + let ell = chunks.combine_vars(); + draw_ext(&mut leg, &mut schedule, ell); + eq_evals_cost(ell, &mut leg); + leg.ops(2 * chunks.count + ((1usize << ell) - chunks.count)); + leg.ops(3 * (chunks.count - 1)); + } + } + + // ── C ── + draw_ext(&mut leg, &mut schedule, plan.num_vars); + absorb_ext(&mut leg, &mut schedule, 2 * n); + draw_ext(&mut leg, &mut schedule, 1); + for _ in 0..plan.num_vars { + absorb_ext(&mut leg, &mut schedule, plan.degree); + draw_ext(&mut leg, &mut schedule, 1); + } + leg.ops(2); + leg.constant(one); + leg.ops(challenge_powers_rows(n)); + leg.ops(3 * n - 1); + leg.ops(plan.num_vars * sumcheck_round_rows(plan.degree)); + if plan.num_vars > 0 { + degrees.push(plan.degree); + } + let lowest = shapes.iter().map(|s| s.num_vars).min().unwrap_or(0); + if lowest < plan.num_vars { + leg.ops(plan.num_vars - lowest - 1); + } + let max_roots = shapes.iter().map(|s| s.ir.num_roots()).max().unwrap_or(0); + leg.ops(challenge_powers_rows(max_roots)); + for (t, shape) in shapes.iter().enumerate() { + let num_vars = shape.num_vars; + for selector in shape.ir.public_selectors() { + selector_cost(*selector, num_vars, &mut leg); + } + leg.ops(2 * eq_eval_rows_again(num_vars)); + leg.ops(combine_rows(shape.ir) - dag_constant_rows(shape.ir)); + for step in shape.ir.steps_as_ops() { + if let multilinear::program::Op::Fixed(value) = step { + leg.constant(value); + } + } + if shape.ir.root_steps().is_empty() { + leg.constant(zero); + } + let negates = shape + .ir + .steps_as_ops() + .iter() + .any(|step| matches!(step, multilinear::program::Op::Neg(_))); + if negates { + leg.constant(zero); + } + leg.op(); + leg.merge(&claim_statements_cost(shape.bus, num_vars)); + leg.ops(2); + if num_vars < plan.num_vars { + leg.op(); + } + if t > 0 { + leg.op(); + } + } + leg.ops(ASSERT_ROWS); + leg.constant(zero); + let factor_values: usize = shapes.iter().map(|s| s.sources().len()).sum(); + absorb_ext(&mut leg, &mut schedule, factor_values); + + // ── Rd ── + for (t, shape) in shapes.iter().enumerate() { + let sources = shape.sources(); + if plan.shapes[t].shifted { + leg.ops(claim_reduce_rows( + &sources, + shape.num_columns, + shape.num_vars, + )); + leg.constant(one); + leg.constant(zero); + degrees.push(REDUCE_DEGREE); + for _ in 0..sources.len() { + schedule.absorb(COORDINATES_PER_EXT); + } + schedule.draw_ext(); + for _ in 0..shape.num_vars { + for _ in 0..REDUCE_DEGREE { + schedule.absorb(COORDINATES_PER_EXT); + } + schedule.draw_ext(); + } + for _ in 0..shape.num_columns { + schedule.absorb(COORDINATES_PER_EXT); + } + } else { + let mut seen = vec![false; shape.num_columns]; + for source in &sources { + if seen[source.column] { + leg.ops(ASSERT_ROWS); + leg.constant(zero); + } + seen[source.column] = true; + } + } + } + + for d in degrees { + for value in newton_constants(d) { + leg.constant(value); + } + } + for hash in schedule.hashes() { + leg.constant_word(leaf_capacity(hash.felts())); + } + TableCost { leg, schedule } +} + +/// The kinds a table's factors read — for a caller holding only the layout. +pub fn is_shifted(kinds: &[FactorKind]) -> bool { + kinds + .iter() + .filter_map(FactorKind::source) + .any(|source| source.offset != 0) +} + +// ============================ inside a block leaf =========================== + +/// A batched argue's wires, hinted from an arena with every count taken from +/// the tables' SHAPES and the plan — never from a proof — and OWNED, for +/// [`BatchedStore::reduces`] and [`BatchedArgueWires`] to borrow. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) struct BatchedStore { + pub(crate) bus_outputs: Vec<(Ext, Ext)>, + pub(crate) gkr: Vec>, + pub(crate) constraint: Vec>, + pub(crate) factor_values: Vec>, + /// Per table, a shifted table's reduction rounds and column values. + reduce_parts: Vec>, +} + +/// A reduction's rounds and column values, as wires. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +type ReduceParts = (Vec>, Vec); + +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +impl BatchedStore { + /// The reductions as wires, one slot per table. + pub(crate) fn reduces(&self) -> Vec>> { + self.reduce_parts + .iter() + .map(|part| { + part.as_ref().map(|(rounds, columns)| ReduceWires { + sumcheck: rounds, + column_values: columns, + }) + }) + .collect() + } +} + +/// How many words a batched argue over `shapes` binned by `plan` takes, in +/// [`hint_batched_shaped`]'s order: what [`push_batched_words`] must write. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn batched_words(shapes: &[TableShape<'_>], plan: &ArguePlan) -> usize { + assert_eq!( + plan.shapes.len(), + shapes.len(), + "the plan is over these tables" + ); + let mut words = 2 * plan.trees(); + for bin in &plan.bins { + let heights = tree_heights(plan, bin); + let steps = heights.iter().copied().max().unwrap_or(0); + for i in 0..steps { + let active = heights.iter().filter(|&&h| h > i).count(); + words += i * GKR_SUMCHECK_DEGREE + 4 * active; + } + } + words += plan.num_vars * plan.degree; + words += shapes.iter().map(|s| s.sources().len()).sum::(); + for (t, shape) in shapes.iter().enumerate() { + if plan.shapes[t].shifted { + words += shape.num_vars * REDUCE_DEGREE + shape.num_columns; + } + } + words +} + +/// Hints a batched argue's wires from `arena` at `at`, in [`push_batched_words`]' +/// order: the bus outputs, each bin's ladder (each step's rounds, then its active +/// trees' halves), the constraint rounds, the factor values, then each shifted +/// table's reduction. Every count is `emit_batched_argue`'s own, from the shapes +/// and the plan, so a proof of another shape cannot be read into it. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn hint_batched_shaped( + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + at: &mut u32, + shapes: &[TableShape<'_>], + plan: &ArguePlan, +) -> BatchedStore { + let mut take = |b: &mut LfmBuilder, count: usize| -> Vec { + (0..count) + .map(|_| { + let wire = b.hint_word(arena, *at).as_ext(); + *at += 1; + wire + }) + .collect() + }; + let bus_outputs = (0..plan.trees()) + .map(|_| { + let pq = take(b, 2); + (pq[0], pq[1]) + }) + .collect(); + let gkr = plan + .bins + .iter() + .map(|bin| { + let heights = tree_heights(plan, bin); + let steps = heights.iter().copied().max().unwrap_or(0); + (0..steps) + .map(|i| { + let active = heights.iter().filter(|&&h| h > i).count(); + LadderStepWires { + sumcheck: (0..i).map(|_| take(b, GKR_SUMCHECK_DEGREE)).collect(), + halves: (0..active) + .map(|_| { + let h = take(b, 4); + [h[0], h[1], h[2], h[3]] + }) + .collect(), + } + }) + .collect() + }) + .collect(); + let constraint = (0..plan.num_vars).map(|_| take(b, plan.degree)).collect(); + let factor_values = shapes.iter().map(|s| take(b, s.sources().len())).collect(); + let reduce_parts = shapes + .iter() + .enumerate() + .map(|(t, shape)| { + plan.shapes[t].shifted.then(|| { + let rounds = (0..shape.num_vars) + .map(|_| take(b, REDUCE_DEGREE)) + .collect(); + (rounds, take(b, shape.num_columns)) + }) + }) + .collect(); + BatchedStore { + bus_outputs, + gkr, + constraint, + factor_values, + reduce_parts, + } +} + +/// A batched argue's words, in [`hint_batched_shaped`]'s order. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn push_batched_words( + words: &mut Vec, + argue: &stark::multilinear_table::BatchedArgue, +) { + use super::word::ext_word; + for (p, q) in &argue.bus_outputs { + words.extend([ext_word(p), ext_word(q)]); + } + for ladder in &argue.gkr { + for step in &ladder.layers { + for round in &step.sumcheck.rounds { + words.extend(round.evaluations.iter().map(ext_word)); + } + for h in &step.halves { + words.extend([h.p_lo, h.p_hi, h.q_lo, h.q_hi].iter().map(ext_word)); + } + } + } + for round in &argue.constraint.rounds { + words.extend(round.evaluations.iter().map(ext_word)); + } + for values in &argue.factor_values { + words.extend(values.iter().map(ext_word)); + } + for reduce in argue.reduces.iter().flatten() { + for round in &reduce.sumcheck.rounds { + words.extend(round.evaluations.iter().map(ext_word)); + } + words.extend(reduce.column_values.iter().map(ext_word)); + } +} + +/// ★ A block group's tables, batched: [`emit_batched_argue`] on the group's +/// fork, then each table's preprocessed leg at the point its columns are +/// claimed at — the host's `check_preprocessed` after `verify_argue`, which +/// `block_verify_with` runs per table — leaving what the per-table walk leaves. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +#[allow(clippy::too_many_arguments)] +pub(crate) fn emit_batched_walk( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + store: &BatchedStore, + shapes: &[TableShape<'_>], + plan: &ArguePlan, + plans: &[super::whir_epoch::PreprocessedPlan<'_>], + slots: &[&[usize]], + z: Ext, + alpha_powers: &[Ext], + beta: Ext, +) -> super::whir_epoch::TableWalk { + assert_eq!(shapes.len(), plans.len(), "one preprocessed plan per table"); + assert_eq!(shapes.len(), slots.len(), "one slot map per table"); + let reduces = store.reduces(); + let verdict = emit_batched_argue( + b, + transcript, + &BatchedArgueWires { + bus_outputs: &store.bus_outputs, + gkr: &store.gkr, + constraint: &store.constraint, + factor_values: &store.factor_values, + reduces: &reduces, + }, + shapes, + plan, + z, + alpha_powers, + beta, + ); + let mut walk = super::whir_epoch::TableWalk { + outputs: Vec::with_capacity(shapes.len()), + points: Vec::with_capacity(shapes.len()), + widths: Vec::with_capacity(shapes.len()), + values: Vec::new(), + }; + for (t, ((point, columns), output)) in verdict + .tables + .into_iter() + .zip(verdict.bus_outputs) + .enumerate() + { + let table = super::whir_table::TableVerdictWires { + bus_output: output, + point, + column_values: columns, + }; + super::whir_epoch::emit_preprocessed_leg(b, &plans[t], slots[t], &shapes[t], &table); + walk.outputs.push(table.bus_output); + walk.widths.push(table.column_values.len()); + walk.values.extend(table.column_values.iter().copied()); + walk.points.push(table.point); + } + walk +} diff --git a/prover/src/lfm/whir_batch_tests.rs b/prover/src/lfm/whir_batch_tests.rs new file mode 100644 index 000000000..f064a928e --- /dev/null +++ b/prover/src/lfm/whir_batch_tests.rs @@ -0,0 +1,1007 @@ +//! Gates for the batched argue's leg (D-BATCH B-4a), against REAL batched +//! argues from the host reference (`stark::multilinear_table::batched`). +//! +//! Three tables of different heights — two with a shifted read (so a claim +//! reduction), one without — argued in one bin and in several, the leg run +//! over the proof values, and its verdict compared with the host's +//! `verify_argue`. The closed form is gated as the per-table leg's is: the +//! program's marginal rows against `batched_argue_cost`. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; +use multilinear::whir_chain::{ArgueFormat, GkrPack}; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet, RowDomain}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, Multiplicity, + NullBoundaryConstraintBuilder, Packing, +}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_logup::{InteractionShape, interaction_shapes}; +use stark::multilinear_table::batched::{ + BatchedArgue, ProverFaults, VerifierChecks, Where, prove_argue, verify_argue, +}; +use stark::multilinear_table::{ + ArguePlan, ArgueShape, CommittedTable, TableLayout, argue_plan, argue_plan_packed, +}; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_batch::{ + BatchedArgueWires, LadderStepWires, batched_argue_cost, emit_batched_argue, +}; +use super::whir_bus::alpha_powers_read; +use super::whir_reduce::ReduceWires; +use super::whir_table::TableShape; +use super::whir_transcript::{SpongeEntry, WhirTranscript}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +type F = crate::tables::types::GoldilocksField; +type E = crate::tables::types::GoldilocksExtension; +type HostTranscript = DefaultTranscript; + +/// One `Public` instruction per published value. +const PUBLISH_ROW: usize = 1; + +fn pseudo(seed: u64, count: usize) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..count) + .map(|_| FEE::new([next(), next(), next()])) + .collect() +} + +/// `c = a·b`, `d = a + b` on every row and — when `shifted` — `next(a) = a + 1` +/// off the last row: a public selector and a shifted read. +struct Rules { + shifted: bool, +} + +const COLUMNS: usize = 4; + +impl ConstraintSet for Rules { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, 0); + let y = b.main(0, 1); + let c = b.main(0, 2); + b.emit_base(0, a * y - c); + let a = b.main(0, 0); + let y = b.main(0, 1); + let d = b.main(0, 3); + b.emit_base(1, a + y - d); + if self.shifted { + let one = b.one(); + let next = b.main(1, 0); + let here = b.main(0, 0); + b.emit_base_rows(2, RowDomain::except_last(1), next - here - one); + } + } +} + +fn buses() -> Vec { + vec![ + BusInteraction::sender( + 0u64, + Multiplicity::Column(1), + Packing::Direct.columns(&[0, 2]), + ), + BusInteraction::receiver( + 0u64, + Multiplicity::Column(1), + Packing::Direct.columns(&[0, 3]), + ), + ] +} + +fn columns(num_vars: usize) -> Vec> { + let rows = 1usize << num_vars; + let b = 7u64; + vec![ + (0..rows).map(|i| FE::from(i as u64)).collect(), + (0..rows).map(|_| FE::from(b)).collect(), + (0..rows).map(|i| FE::from(i as u64 * b)).collect(), + (0..rows).map(|i| FE::from(i as u64 + b)).collect(), + ] +} + +type Air = AirWithBuses; + +fn air(shifted: bool) -> Air { + air_with(shifted, buses()) +} + +/// An unshifted table with `m` interactions, each `(a, c)` sent on its own +/// bus: enough of them for the GKR packing to cut. Its bus does not balance, +/// which the leg does not check (the caller's, as the host's). +fn fan(m: usize) -> Air { + let interactions = (0..m) + .map(|i| { + BusInteraction::sender( + 10 + i as u64, + Multiplicity::Column(1), + Packing::Direct.columns(&[0, 2]), + ) + }) + .collect(); + air_with(false, interactions) +} + +fn air_with(shifted: bool, interactions: Vec) -> Air { + AirWithBuses::new( + COLUMNS, + AuxiliaryTraceBuildData { interactions }, + &stark::proof::options::GoldilocksCubicProofOptions::with_params(4, 128, 20) + .expect("valid options"), + 1, + Rules { shifted }, + ) +} + +fn challenges() -> (FEE, FEE, FEE) { + let drawn = pseudo(0xBA7C4, 3); + (drawn[0], drawn[1], drawn[2]) +} + +/// The fixture: `(air, num_vars)` per table, and how its argue lays them +/// into trees. +struct Fixture { + airs: Vec<(Air, usize)>, + pack: GkrPack, +} + +/// The packed fixture's price: every fan below is cut. +const PACKED: GkrPack = GkrPack::Chunks { tree_log_cells: 2 }; + +impl Fixture { + /// A shifted table of 2^3 rows, an unshifted one of 2^4, a shifted one of + /// 2^5: every table shorter than another but the last. + fn new() -> Self { + Self { + airs: vec![(air(true), 3), (air(false), 4), (air(true), 5)], + pack: GkrPack::Off, + } + } + + /// [`Self::new`]'s tables with three fans between them, packed at + /// [`PACKED`]: FAN(5) and FAN(9) at 2^3 rows in chunks of two slots (the + /// last of each one interaction, padded to its slots), FAN(7) at 2^5 in + /// seven chunks of one, each with all-padding chunks after its last. + fn packed() -> Self { + Self { + airs: vec![ + (air(true), 3), + (fan(5), 3), + (air(false), 4), + (fan(9), 3), + (fan(7), 5), + (air(true), 5), + ], + pack: PACKED, + } + } + + fn layouts(&self) -> Vec> { + self.airs + .iter() + .map(|(air, n)| { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + COLUMNS, + *n, + Uniforms::default(), + ) + .expect("the table lays out") + }) + .collect() + } + + fn buses(&self, layouts: &[TableLayout<'_, F, E>]) -> Vec>> { + self.airs + .iter() + .zip(layouts) + .map(|((air, _), layout)| { + let slots = layout.slot_of().to_vec(); + interaction_shapes(air.bus_interactions(), COLUMNS, |column| { + slots + .get(column) + .copied() + .ok_or(multilinear::Error::UnknownPolynomial { + index: column, + len: slots.len(), + }) + }) + .expect("the bus probes") + }) + .collect() + } + + fn argue(&self, cap: u8) -> BatchedArgue { + let tables: Vec> = self + .airs + .iter() + .enumerate() + .map(|(t, (air, n))| { + let mut cols = columns(*n); + if t == 1 && VIOLATE_TABLE_1.with(std::cell::Cell::get) { + cols[2][1] += FE::one(); + } + CommittedTable::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + COLUMNS, + *n, + Uniforms::default(), + move |col| cols[col as usize].clone(), + ) + .expect("the table lays out") + }) + .collect(); + let (z, alpha, beta) = challenges(); + let mut prover = HostTranscript::new(&[]); + prove_argue( + &tables, + &z, + &alpha, + &beta, + cap, + self.pack, + &mut prover, + ARGUE_FAULTS.with(std::cell::Cell::get), + Where::Host, + ) + .expect("the tables argue") + .0 + } +} + +fn plan_packed(layouts: &[TableLayout<'_, F, E>], cap: u8, pack: GkrPack) -> ArguePlan { + let shapes: Vec = layouts + .iter() + .map(|l| ArgueShape::of(&l.statement())) + .collect(); + let plan = argue_plan_packed(&shapes, cap, pack); + if pack == GkrPack::Off { + assert_eq!(plan, argue_plan(&shapes, cap), "Off plans today's bins"); + } + plan +} + +fn alpha_ladder(buses: &[Vec>], alpha: FEE) -> Vec { + let count = buses + .iter() + .map(|b| alpha_powers_read(b)) + .max() + .unwrap_or(0); + let mut powers = Vec::with_capacity(count); + let mut power = FEE::one(); + for _ in 0..count { + powers.push(power); + power *= alpha; + } + powers +} + +/// Every value the program hints, in the order [`batched_program`] takes them. +fn flatten(argue: &BatchedArgue, z: FEE, alpha_powers: &[FEE], beta: FEE) -> Vec { + let mut values = vec![z]; + values.extend_from_slice(alpha_powers); + values.push(beta); + for (p, q) in &argue.bus_outputs { + values.extend([*p, *q]); + } + for ladder in &argue.gkr { + for step in &ladder.layers { + for round in &step.sumcheck.rounds { + values.extend(round.evaluations.iter().copied()); + } + for h in &step.halves { + values.extend([h.p_lo, h.p_hi, h.q_lo, h.q_hi]); + } + } + } + for round in &argue.constraint.rounds { + values.extend(round.evaluations.iter().copied()); + } + for fv in &argue.factor_values { + values.extend(fv.iter().copied()); + } + for reduce in argue.reduces.iter().flatten() { + for round in &reduce.sumcheck.rounds { + values.extend(round.evaluations.iter().copied()); + } + values.extend(reduce.column_values.iter().copied()); + } + values +} + +/// A reduction's rounds and column values, as wires. +type ReduceParts = (Vec>, Vec); + +/// The assembled program over the hinted proof, or — with `leg` false — the +/// same arena with no leg, so the difference is the leg. +#[allow(clippy::too_many_arguments)] +fn batched_program( + argue: &BatchedArgue, + shapes: &[TableShape<'_>], + plan: &ArguePlan, + alpha_count: usize, + total: usize, + leg: bool, +) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(total as u32); + let mut index = 0u32; + let mut take = |b: &mut LfmBuilder, count: usize| -> Vec { + (0..count) + .map(|_| { + let wire = b.hint_word(arena, index).as_ext(); + index += 1; + wire + }) + .collect() + }; + let z = take(&mut b, 1)[0]; + let alpha_powers = take(&mut b, alpha_count); + let beta = take(&mut b, 1)[0]; + let outputs: Vec<(Ext, Ext)> = argue + .bus_outputs + .iter() + .map(|_| { + let pq = take(&mut b, 2); + (pq[0], pq[1]) + }) + .collect(); + let gkr: Vec> = argue + .gkr + .iter() + .map(|ladder| { + ladder + .layers + .iter() + .map(|step| LadderStepWires { + sumcheck: step + .sumcheck + .rounds + .iter() + .map(|round| take(&mut b, round.evaluations.len())) + .collect(), + halves: step + .halves + .iter() + .map(|_| { + let h = take(&mut b, 4); + [h[0], h[1], h[2], h[3]] + }) + .collect(), + }) + .collect() + }) + .collect(); + let constraint: Vec> = argue + .constraint + .rounds + .iter() + .map(|round| take(&mut b, round.evaluations.len())) + .collect(); + let factor_values: Vec> = argue + .factor_values + .iter() + .map(|fv| take(&mut b, fv.len())) + .collect(); + let reduce_parts: Vec> = argue + .reduces + .iter() + .map(|reduce| { + reduce.as_ref().map(|reduce| { + let rounds: Vec> = reduce + .sumcheck + .rounds + .iter() + .map(|round| take(&mut b, round.evaluations.len())) + .collect(); + let columns = take(&mut b, reduce.column_values.len()); + (rounds, columns) + }) + }) + .collect(); + + if !leg { + b.public(z.as_cell()); + b.public(beta.as_cell()); + return compile(b.finish()); + } + let reduces: Vec>> = reduce_parts + .iter() + .map(|part| { + part.as_ref().map(|(rounds, columns)| ReduceWires { + sumcheck: rounds, + column_values: columns, + }) + }) + .collect(); + let mut transcript = WhirTranscript::new(); + let verdict = emit_batched_argue( + &mut b, + &mut transcript, + &BatchedArgueWires { + bus_outputs: &outputs, + gkr: &gkr, + constraint: &constraint, + factor_values: &factor_values, + reduces: &reduces, + }, + shapes, + plan, + z, + &alpha_powers, + beta, + ); + for (p, q) in &verdict.bus_outputs { + b.public(p.as_cell()); + b.public(q.as_cell()); + } + for (point, columns) in &verdict.tables { + for wire in point.iter().chain(columns) { + b.public(wire.as_cell()); + } + } + compile(b.finish()) +} + +fn words(values: &[FEE]) -> Vec> { + vec![values.iter().map(ext_word).collect()] +} + +/// Everything one run needs, built from the fixture at `cap`. +fn with_run( + cap: u8, + tamper: impl FnOnce(&mut BatchedArgue), + check: impl FnOnce(RunView<'_>) -> R, +) -> R { + with_run_on(Fixture::new(), cap, tamper, check) +} + +/// [`with_run`] over `fixture`. +fn with_run_on( + fixture: Fixture, + cap: u8, + tamper: impl FnOnce(&mut BatchedArgue), + check: impl FnOnce(RunView<'_>) -> R, +) -> R { + let layouts = fixture.layouts(); + let buses = fixture.buses(&layouts); + let plan = plan_packed(&layouts, cap, fixture.pack); + let shapes: Vec> = layouts + .iter() + .zip(&buses) + .map(|(layout, bus)| TableShape { + ir: layout.shape(), + bus, + kinds: layout.kinds(), + num_columns: layout.num_columns(), + num_vars: layout.num_vars(), + }) + .collect(); + let mut argue = fixture.argue(cap); + tamper(&mut argue); + let (z, alpha, beta) = challenges(); + let alpha_powers = alpha_ladder(&buses, alpha); + let statements: Vec<_> = layouts.iter().map(|l| l.statement()).collect(); + let host = { + let mut verifier = HostTranscript::new(&[]); + verify_argue( + &argue, + &statements, + cap, + fixture.pack, + &z, + &alpha, + &beta, + &mut verifier, + HOST_CHECKS.with(std::cell::Cell::get), + ) + .map_err(|e| format!("{e:?}")) + }; + check(RunView { + argue: &argue, + shapes: &shapes, + plan: &plan, + z, + beta, + alpha_powers: &alpha_powers, + host, + }) +} + +struct RunView<'a> { + argue: &'a BatchedArgue, + shapes: &'a [TableShape<'a>], + plan: &'a ArguePlan, + z: FEE, + beta: FEE, + alpha_powers: &'a [FEE], + host: Result>, String>, +} + +impl RunView<'_> { + fn values(&self) -> Vec { + flatten(self.argue, self.z, self.alpha_powers, self.beta) + } + + fn program(&self, leg: bool) -> LfmProgram { + batched_program( + self.argue, + self.shapes, + self.plan, + self.alpha_powers.len(), + self.values().len(), + leg, + ) + } + + fn execute(&self) -> Result, String> { + let program = self.program(true); + validate(&program).map_err(|e| format!("invalid: {e:?}"))?; + execute( + &program, + &words(&self.values()), + &crate::hash_pin::BLOCK_HASHER, + ) + .map(|exec| { + exec.public_words + .iter() + .map(|(_, word)| word_as_ext(word).expect("a published extension value")) + .collect() + }) + .map_err(|e| format!("{e:?}")) + } +} + +/// Each table's bus output as the leg leaves it: its one tree's, or its +/// chunks' fractions added in chunk order, `(p·q_j + p_j·q, q·q_j)`. +fn table_outputs(run: &RunView<'_>) -> Vec<(FEE, FEE)> { + (0..run.plan.shapes.len()) + .map(|t| { + let first = run.plan.first_tree(t); + let chunks = &run.argue.bus_outputs[first..first + run.plan.chunks[t].count]; + chunks[1..] + .iter() + .fold(chunks[0], |(p, q), (pj, qj)| (p * qj + pj * q, q * qj)) + }) + .collect() +} + +/// ★ B-4a G1 — the leg computes what the host's `verify_argue` computes, in +/// one bin and with every table alone: the bus outputs, and every table's +/// column point and values. +#[test] +fn the_batched_leg_computes_what_the_host_computes() { + for cap in [27u8, 0] { + with_run( + cap, + |_| {}, + |run| { + let host = run.host.clone().expect("the host verifies its own argue"); + let got = run + .execute() + .unwrap_or_else(|e| panic!("cap {cap}: the leg must execute: {e}")); + let mut want: Vec = Vec::new(); + for (p, q) in table_outputs(&run) { + want.extend([p, q]); + } + for claim in &host { + want.extend(claim.point.iter().copied()); + want.extend(claim.column_values.iter().copied()); + } + assert_eq!(got, want, "cap {cap}: the leg's verdict is the host's"); + println!( + "batched leg cap={cap}: bins {:?}, n_max {}, D_max {}, reductions {}", + run.plan.bins, + run.plan.num_vars, + run.plan.degree, + run.argue.reduces.iter().flatten().count() + ); + }, + ); + } +} + +/// ★ B-4a G2 — the closed form is the emitted census: the program's marginal +/// rows against `batched_argue_cost`, and the form names every constant. +#[test] +fn the_batched_leg_emits_its_closed_form() { + for cap in [27u8, 0] { + with_run( + cap, + |_| {}, + |run| { + let with = run.program(true); + let without = run.program(false); + let published = 2 * run.shapes.len() + + run + .host + .as_ref() + .expect("host") + .iter() + .map(|c| c.point.len() + c.column_values.len()) + .sum::(); + let measured = + with.instrs.len() - without.instrs.len() - (published - 2) * PUBLISH_ROW; + let cost = batched_argue_cost(run.shapes, run.plan, SpongeEntry::fresh()); + let predicted = cost.leg.constant_values(); + let mut unnamed: Vec = Vec::new(); + for instr in &with.instrs { + if let super::instr::Instr::Const { value, .. } = instr + && !predicted.contains(value) + && !unnamed.contains(value) + { + unnamed.push(*value); + } + } + assert!( + unnamed.is_empty(), + "cap {cap}: the form does not name {unnamed:?}" + ); + println!( + "batched leg cap={cap}: {measured} rows emitted, {} predicted ({} ops + {} \ + constants + {} sponge rows, {} permutations)", + cost.rows(), + cost.leg.operations(), + cost.leg.constants(), + cost.schedule.rows(), + cost.perms() + ); + assert_eq!(measured, cost.rows(), "cap {cap}"); + }, + ); + } +} + +fn bump(value: &mut FEE) { + *value += FEE::one(); +} + +/// ★ B-4a G3 — every tamper the host refuses, the leg refuses: the untouched +/// argue executes, the host rejects each forgery, and the machine cannot +/// execute it. +#[test] +fn the_batched_leg_refuses_what_the_host_rejects() { + type Site = (&'static str, fn(&mut BatchedArgue)); + let sites: Vec = vec![ + ("a bus output", |a| bump(&mut a.bus_outputs[1].0)), + ("a mid-step half", |a| { + bump(&mut a.gkr[0].layers[2].halves[1].q_lo) + }), + ("a ladder round", |a| { + bump(&mut a.gkr[0].layers[3].sumcheck.rounds[1].evaluations[0]) + }), + ("a constraint round", |a| { + bump(&mut a.constraint.rounds[1].evaluations[1]) + }), + ("a factor value", |a| bump(&mut a.factor_values[1][0])), + ("a reduced column value", |a| { + bump(&mut a.reduces[2].as_mut().expect("shifted").column_values[0]) + }), + ]; + with_run( + 27, + |_| {}, + |run| { + run.host + .clone() + .expect("the honest argue verifies on the host"); + run.execute().expect("the honest argue executes"); + }, + ); + for (name, tamper) in sites { + with_run(27, tamper, |run| { + assert!(run.host.is_err(), "{name}: the HOST accepted it"); + assert!( + run.execute().is_err(), + "{name}: the machine executed a forgery the host rejects" + ); + }); + } +} + +/// The format constant is what plans: `ArgueFormat::BATCHED`'s cap and +/// packing put these three tables in one bin, none of them cut. +#[test] +fn the_production_cap_bins_the_fixture_once() { + let ArgueFormat::Batched { + bin_log_cells, + pack, + } = ArgueFormat::BATCHED + else { + unreachable!() + }; + let fixture = Fixture::new(); + let layouts = fixture.layouts(); + let plan = plan_packed(&layouts, bin_log_cells, pack); + assert_eq!(plan.bins, vec![vec![0, 1, 2]]); + assert_eq!(plan.trees(), 3); +} + +// ─── The leaf's two asserts, each against a forgery only it refuses ── + +thread_local! { + static ARGUE_FAULTS: std::cell::Cell = std::cell::Cell::new(ProverFaults::default()); + static VIOLATE_TABLE_1: std::cell::Cell = const { std::cell::Cell::new(false) }; + static HOST_CHECKS: std::cell::Cell = const { std::cell::Cell::new(VerifierChecks::ALL) }; +} + +/// The host verdict under `checks`, and whether the machine executes. +fn run_under(checks: VerifierChecks) -> (Result<(), String>, bool) { + HOST_CHECKS.with(|c| c.set(checks)); + let out = with_run( + 27, + |_| {}, + |run| (run.host.clone().map(|_| ()), run.execute().is_ok()), + ); + HOST_CHECKS.with(|c| c.set(VerifierChecks::ALL)); + out +} + +/// Table 1's output numerator moved (its ladder on the true tree): only the +/// ladder residual refuses it on the host, and the machine must refuse it. +#[test] +fn the_leaf_refuses_a_forgery_only_the_residual_catches() { + ARGUE_FAULTS.with(|f| { + f.set(ProverFaults { + balance_with: Some(1), + ..ProverFaults::default() + }) + }); + let (all, executes) = run_under(VerifierChecks::ALL); + let (inert, _) = run_under(VerifierChecks { + residual: false, + ..VerifierChecks::ALL + }); + ARGUE_FAULTS.with(|f| f.set(ProverFaults::default())); + assert!(all.is_err(), "the host refuses it"); + assert_eq!(inert, Ok(()), "premise: only the residual refuses it"); + assert!( + !executes, + "the machine executed a forgery only the residual catches" + ); +} + +/// A violated constraint in table 1, argued honestly: only the constraint +/// equation refuses it on the host, and the machine must refuse it. +#[test] +fn the_leaf_refuses_a_forgery_only_the_constraint_equation_catches() { + VIOLATE_TABLE_1.with(|v| v.set(true)); + let (all, executes) = run_under(VerifierChecks::ALL); + let (inert, _) = run_under(VerifierChecks { + constraint: false, + ..VerifierChecks::ALL + }); + VIOLATE_TABLE_1.with(|v| v.set(false)); + assert!(all.is_err(), "the host refuses it"); + assert_eq!( + inert, + Ok(()), + "premise: only the constraint equation refuses it" + ); + assert!( + !executes, + "the machine executed a forgery only the constraint equation catches" + ); +} + +// ─── The GKR slot packing (`GkrPack::Chunks`), in-guest ────────────────── + +/// The packed fixture cuts every fan, in one bin and with every table alone. +fn packed_cuts(cap: u8) -> Vec<(usize, usize)> { + let fixture = Fixture::packed(); + let layouts = fixture.layouts(); + plan_packed(&layouts, cap, fixture.pack) + .chunks + .iter() + .map(|c| (c.slot_vars, c.count)) + .collect() +} + +/// ★ The leg computes what the host computes under the packing: each table's +/// claim from its chunks combined at `s'` (drawn after its bin's ladder), and +/// each table's output its chunks' fractions added — in one bin and with +/// every table alone. +#[test] +fn the_packed_leg_computes_what_the_host_computes() { + for cap in [27u8, 0] { + assert_eq!( + packed_cuts(cap), + vec![(1, 1), (1, 3), (1, 1), (1, 5), (0, 7), (1, 1)], + "cap {cap}: every fan is cut" + ); + with_run_on( + Fixture::packed(), + cap, + |_| {}, + |run| { + assert_eq!(run.argue.bus_outputs.len(), run.plan.trees()); + assert_eq!(run.plan.trees(), 1 + 3 + 1 + 5 + 7 + 1); + let host = run.host.clone().expect("the host verifies its own argue"); + let got = run + .execute() + .unwrap_or_else(|e| panic!("cap {cap}: the leg must execute: {e}")); + let mut want: Vec = Vec::new(); + for (p, q) in table_outputs(&run) { + want.extend([p, q]); + } + for claim in &host { + want.extend(claim.point.iter().copied()); + want.extend(claim.column_values.iter().copied()); + } + assert_eq!(got, want, "cap {cap}: the leg's verdict is the host's"); + }, + ); + } +} + +/// ★ The closed form is the emitted census under the packing too: the +/// combination's draws, its eq table, its sums and the added fractions. +#[test] +fn the_packed_leg_emits_its_closed_form() { + for cap in [27u8, 0] { + with_run_on( + Fixture::packed(), + cap, + |_| {}, + |run| { + let with = run.program(true); + let without = run.program(false); + let published = 2 * run.shapes.len() + + run + .host + .as_ref() + .expect("host") + .iter() + .map(|c| c.point.len() + c.column_values.len()) + .sum::(); + let measured = + with.instrs.len() - without.instrs.len() - (published - 2) * PUBLISH_ROW; + let cost = batched_argue_cost(run.shapes, run.plan, SpongeEntry::fresh()); + let predicted = cost.leg.constant_values(); + let mut unnamed: Vec = Vec::new(); + for instr in &with.instrs { + if let super::instr::Instr::Const { value, .. } = instr + && !predicted.contains(value) + && !unnamed.contains(value) + { + unnamed.push(*value); + } + } + assert!( + unnamed.is_empty(), + "cap {cap}: the form does not name {unnamed:?}" + ); + assert_eq!(measured, cost.rows(), "cap {cap}"); + }, + ); + } +} + +/// ★ Every tamper the host refuses under the packing, the leg refuses: a cut +/// table's chunk output (either side), a chunk's halves at a mid step, two +/// chunks' outputs swapped, and a ladder round. +#[test] +fn the_packed_leg_refuses_what_the_host_rejects() { + // FAN(5) is table 1, its trees 1–3; FAN(9) is table 3, its trees 5–9. + type Site = (&'static str, fn(&mut BatchedArgue)); + let sites: Vec = vec![ + ("a chunk's output numerator", |a| { + bump(&mut a.bus_outputs[2].0) + }), + ("a chunk's output denominator", |a| { + bump(&mut a.bus_outputs[6].1) + }), + ("two chunks' outputs swapped", |a| a.bus_outputs.swap(1, 2)), + ("a chunk's mid-step half", |a| { + bump(&mut a.gkr[0].layers[2].halves[3].p_hi) + }), + ("a ladder round", |a| { + bump(&mut a.gkr[0].layers[3].sumcheck.rounds[1].evaluations[2]) + }), + ]; + with_run_on( + Fixture::packed(), + 27, + |_| {}, + |run| { + run.host + .clone() + .expect("the honest packed argue verifies on the host"); + run.execute().expect("the honest packed argue executes"); + }, + ); + for (name, tamper) in sites { + with_run_on(Fixture::packed(), 27, tamper, |run| { + assert!(run.host.is_err(), "{name}: the HOST accepted it"); + assert!( + run.execute().is_err(), + "{name}: the machine executed a forgery the host rejects" + ); + }); + } +} + +/// A cut table's first two chunks traded by the prover (honest trees over the +/// chunks in the wrong order): the host's constraint equation refuses it, and +/// so must the machine. +#[test] +fn the_packed_leg_refuses_swapped_chunks() { + for t in [1usize, 3, 4] { + ARGUE_FAULTS.with(|f| { + f.set(ProverFaults { + swap_chunks: Some(t), + ..ProverFaults::default() + }) + }); + let (host, executes) = with_run_on( + Fixture::packed(), + 27, + |_| {}, + |run| (run.host.clone().map(|_| ()), run.execute().is_ok()), + ); + ARGUE_FAULTS.with(|f| f.set(ProverFaults::default())); + assert_eq!( + host, + Err(format!("{:?}", multilinear::Error::BatchMismatch)), + "table {t}: the host's constraint equation" + ); + assert!(!executes, "table {t}: the machine executed swapped chunks"); + } +} + +/// A forged output on a cut table's first chunk (its ladder on the true +/// tree): only the ladder residual refuses it on the host, and the machine +/// must refuse it. +#[test] +fn the_packed_leg_refuses_a_forgery_only_the_residual_catches() { + ARGUE_FAULTS.with(|f| { + f.set(ProverFaults { + balance_with: Some(3), + ..ProverFaults::default() + }) + }); + let run_with = |checks: VerifierChecks| { + HOST_CHECKS.with(|c| c.set(checks)); + let out = with_run_on( + Fixture::packed(), + 27, + |_| {}, + |run| (run.host.clone().map(|_| ()), run.execute().is_ok()), + ); + HOST_CHECKS.with(|c| c.set(VerifierChecks::ALL)); + out + }; + let (all, executes) = run_with(VerifierChecks::ALL); + let (inert, _) = run_with(VerifierChecks { + residual: false, + ..VerifierChecks::ALL + }); + ARGUE_FAULTS.with(|f| f.set(ProverFaults::default())); + assert!(all.is_err(), "the host refuses it"); + assert_eq!(inert, Ok(()), "premise: only the residual refuses it"); + assert!( + !executes, + "the machine executed a forgery only the residual catches" + ); +} diff --git a/prover/src/lfm/whir_block.rs b/prover/src/lfm/whir_block.rs new file mode 100644 index 000000000..307984ebc --- /dev/null +++ b/prover/src/lfm/whir_block.rs @@ -0,0 +1,2748 @@ +//! The no-epoch WHIR BLOCK's recursion: the tree plan, the leaf program and its +//! arena (D-NOEPOCH §12.5, I-NOEPOCH-W §9). +//! +//! A block proof ([`crate::block_whir`]) is ONE multilinear proof: the +//! statement and the partition, every group's roots and every prepared table's +//! derived roots, then `(z, α, β)`, then each group on its own fork `S_post ‖ g` +//! — its tables' arguments, its stacked opening, its prepared openings. The +//! recursion splits the GROUPS across leaves: a group's opening covers all its +//! tables, so a group is the smallest unit a leaf can verify alone. +//! +//! # What a leaf does +//! +//! 1. The statement, as program constants (`block_whir::block_statement_calls`), then the +//! roots block over EVERY root of the block — the groups' (arena) and the +//! prepared tables' (constants the plan derived) — then EVERY carried +//! polynomial's commit-time answer (arena), then `z, α, β` and the +//! transcript's state digest. Every leaf replays the same front, so every leaf +//! forks from the same `S_post`. +//! 2. Per group of its own: its commit-time point, drawn from a fresh sponge +//! over its own roots — the front's cells — and refused in its domain +//! ([`emit_commit_ood_point`]); the fork, each table's argument with its +//! preprocessed leg ([`emit_table_walk`]), the group's opening with its +//! answers carried — the front's cells again ([`emit_group_walk_chains`]), +//! each prepared table's opening — #1010's epoch legs, on the fork. +//! 3. Publishes the block layout ([`BlockLayout::child`]): the block's id, the +//! state, the public output's halves, and `Σ p/q` over its tables — minus the +//! COMMIT-bus target on the leaf that carries it ([`CARRIER`]). +//! +//! The nodes are [`super::block_node`]'s, unchanged: they check the id, the +//! state and the output equal across children, add the sums, and the top +//! asserts the total is zero — the host's single bus check, where every table's +//! share is in scope. A leaf divides `p/q` itself, as #1010's epoch closure does, +//! so the node needs no fraction arithmetic ([`emit_share`]: the share is +//! `p · (1/q)`, so a zero denominator has no satisfying assignment whatever `p` +//! is, the host's `None`; `LFM_WHIR_SHARE_INVERSE=0` restores the former +//! `ediv(p, q)`, which leaves the share free when `p = q = 0`). +//! +//! # Never from a proof +//! +//! [`WhirBlockPlan::derive`] takes the statement's fields through the host +//! verifier's own checks ([`crate::block_whir::block_frame`]); every shape comes +//! from the AIR at the stated height, every count a leaf hints from that shape +//! ([`hint_table_wires_shaped`]), every derived root from the program; the leaf +//! partition and its carrier are the plan's. The proof is read only to fill a +//! leaf's arena — the witness. + +use multilinear::stacking::StackedLayout; +use multilinear::whir::Domain; +use multilinear::whir_chain::GroupFold; +use multilinear::whir_commit::Commitment; +use stark::multilinear_logup::InteractionShape; +use stark::multilinear_table::TableLayout; + +use crate::block_whir::{ + BlockFormat, BlockFrame, BlockStatement, BlockWhirProof, block_frame, block_statement_bytes, + commit_group, first_page_index, group_stacks, prepared_tables, +}; +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::block_node::{BlockLayout, BlockNodeInputs}; +use super::builder::{Cell, Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::per_table_aggregator::DerivedChild; +use super::registry::LfmArtifacts; +use super::whir_chain::{ChainShape, ChipRows, LeafQuery}; +use super::whir_epoch::{ + BITWISE_NAME, DECODE_NAME, GroupChainWires, KECCAK_RC_NAME, PreprocessedPlan, + PreprocessedRoute, REGISTER_NAME, TableWires, emit_absorb_roots, emit_expected, + emit_group_walk_chains, emit_table_walk, group_columns, hint_group_chains_shaped, + hint_table_wires_shaped, push_table_words, table_words, +}; +use super::whir_stacked::{ + OodWires, StackedChainWires, stacked_verify_cost, stacked_verify_cost_ood, +}; +use super::whir_table::{TableProofWires, TableShape, table_verify_cost}; +use super::whir_transcript::{SpongeEntry, WhirTranscript}; +use super::word::{LfmWord, base_word}; + +/// The leaf-load cap, in socket (`Hash16`) rows: one +/// 2^18 table, all of it — seven full groups a leaf, 13 leaves at the median +/// (I-WHIR-P1 §S6b). The recursion's time is the card's: the leaves prove one +/// at a time, each ≈ 0.28 s plus ≈ 0.0122 s a million padded cells outside the +/// socket (RYZEN 062), so fewer, fuller leaves win over four groups a leaf +/// (S6), which padded `Select`, base ALU and words to 2^20 for half of them. +pub const LEAF_P1_CAP: usize = 1 << 18; + +/// The padded heights a Poseidon1 leaf keeps every chip under: the +/// [`LEAF_P1_CAP`] socket rows, 2^20 rows of `Select`, base ALU, lanes and +/// hinted words, 2^21 of extension ALU (seven full groups' heights, +/// I-WHIR-P1 §S6b), and the small chips at heights no leaf of the median +/// reaches. The constants' bound is loose by construction — a leaf's distinct +/// constants (≈ 1.1 k at six groups) are far fewer than its parts' sum +/// (≈ 3.6 k), which is what the partition can add up — so its height sits +/// where that sum does not bind before the socket. The publishes are the leaf +/// layout's, the same in every leaf and no group's, so nothing the partition +/// places moves them: unbounded. No accelerator: a WHIR leaf has none. The +/// partition admits no leaf past any of them ([`leaf_partition_rows`]), so no +/// chip doubles silently. +pub const P1_LEAF_HEIGHTS: ChipRows = ChipRows { + hash: LEAF_P1_CAP, + select: 1 << 20, + balu: 1 << 20, + xalu: 1 << 21, + lanes: 1 << 20, + hint: 1 << 20, + bitdec: 1 << 15, + consts: 1 << 13, + public: usize::MAX, + accel: 0, +}; + +/// The socket rows of a doubled leaf ([`LeafScale::DoubleFrom`]): twice +/// [`LEAF_P1_CAP`]. +pub const LEAF_P1_CAP_2X: usize = 2 * LEAF_P1_CAP; + +/// [`P1_LEAF_HEIGHTS`] with every bounded chip twice as tall: the heights a +/// block's leaves are held under when [`LeafScale::DoubleFrom`] doubles them. +/// The same rule packs about twice the groups a leaf, so about half the leaves +/// and half the children for the nodes to verify: 18 leaves → 9 on a large +/// block at blowup 2, and one node level fewer. A leaf's card time follows its +/// padded heights, so the leaves' own work stays about the same. +pub const P1_LEAF_HEIGHTS_2X: ChipRows = ChipRows { + hash: LEAF_P1_CAP_2X, + select: 1 << 21, + balu: 1 << 21, + xalu: 1 << 22, + lanes: 1 << 21, + hint: 1 << 21, + bitdec: 1 << 16, + consts: 1 << 14, + public: usize::MAX, + accel: 0, +}; + +// The conjectured LFM bits stay at 128 or above only while a Poseidon1 leaf's +// socket and bit-dec tables stay at 2^19 rows or fewer (REV-P1W-JUDGE V3: the +// calculator reads 128.005 for the socket and 128.51 for bit-dec at 2^20), at +// either leaf scale. A wider chip re-runs the calculator before this bound +// moves. +const _: () = assert!( + P1_LEAF_HEIGHTS.hash <= 1 << 19 + && P1_LEAF_HEIGHTS.bitdec <= 1 << 19 + && P1_LEAF_HEIGHTS_2X.hash <= 1 << 19 + && P1_LEAF_HEIGHTS_2X.bitdec <= 1 << 19 +); + +// The doubled heights are the single ones doubled, chip by chip: neither set +// moves without the other. +const _: () = { + let (one, two) = (P1_LEAF_HEIGHTS, P1_LEAF_HEIGHTS_2X); + assert!( + two.hash == 2 * one.hash + && two.select == 2 * one.select + && two.balu == 2 * one.balu + && two.xalu == 2 * one.xalu + && two.lanes == 2 * one.lanes + && two.hint == 2 * one.hint + && two.bitdec == 2 * one.bitdec + && two.consts == 2 * one.consts + && two.public == one.public + && two.accel == one.accel + ); +}; + +/// How big a block's leaves are ([`BlockFormat::leaf_scale`]): a format +/// constant the prover and the verifier each hold, never read from a proof. +/// Both derive the partition from the statement under it +/// ([`WhirBlockPlan::derive`]), so the leaf count, every tree program and the +/// top are the verifier's own whichever scale applies. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LeafScale { + /// Every leaf under [`P1_LEAF_HEIGHTS`]: the format before the doubling, + /// and its rollback (`off`). + Fixed, + /// The leaves under [`P1_LEAF_HEIGHTS`]; if that takes `T` leaves or more, + /// partitioned again under [`P1_LEAF_HEIGHTS_2X`]. Below `T` the partition, + /// the tree and the proof are [`Self::Fixed`]'s, byte for byte. Doubling + /// saves node children, which a small tree has few of, while its leaves + /// pad more and its first leaf starts later (measured on one RTX 5090 with + /// the leaves forced to double: −2.57 s of recursion on a large block's 18 + /// leaves, +0.33 s on a median block's 9). + DoubleFrom(usize), + /// Every leaf under [`P1_LEAF_HEIGHTS_2X`], whatever the leaf count, and, + /// when that takes at most [`ONE_TOP_MAX_LEAVES`] leaves, one top over all + /// of them: no node level between the leaves and the top. Past that count + /// the tree is [`Self::DoubleFrom`]'s, nodes of the plan's fan-in. A top + /// over three doubled leaves stays within 2^19 socket rows, the LFM proofs' + /// bits bound, and is refused past it ([`ONE_TOP_SOCKET_ROWS`]). The + /// default ([`Self::DEFAULT`], `one-top`). + DoubledOneTop, +} + +/// The most leaves [`LeafScale::DoubledOneTop`] puts under one top. With the +/// recursion proofs' commitment-time answers, a top over four doubled leaves +/// takes its LANES chip past 2^19 rows (99.2 % of 2^19 without them), a padded +/// height of 2^20; a top over three sits near 76 % of 2^19. +pub const ONE_TOP_MAX_LEAVES: usize = 3; + +/// The socket rows a node with more children than the plan's fan-in may take: +/// the LFM proofs' bits bound (socket at most 2^19 rows). Past it the node is +/// refused, not proved. +pub const ONE_TOP_SOCKET_ROWS: usize = 1 << 19; + +/// The doubling threshold the default ran at before [`LeafScale::DoubledOneTop`] +/// (the fewest leaves at the single heights from which a block's leaves +/// double): `LAMBDA_VM_WHIR_LEAF_DOUBLE_FROM=10` or `--leaf-double-from 10` +/// rolls back to it. +pub const LEAF_DOUBLE_FROM_DEFAULT: usize = 10; + +impl LeafScale { + /// ★ The default: every leaf doubled, and one top over at most + /// [`ONE_TOP_MAX_LEAVES`] of them ([`Self::DoubledOneTop`]). Measured on one + /// RTX 5090 against [`Self::DoubleFrom`]`(`[`LEAF_DOUBLE_FROM_DEFAULT`]`)`, + /// three runs an arm: the median block's recursion −0.78 s and whole block + /// −0.71 s (6 single leaves → 3 doubled under one top, its level of nodes + /// gone); the large block, already doubled to more than four leaves, keeps + /// its partition, tree and proof byte for byte. + pub const DEFAULT: Self = Self::DoubledOneTop; +} + +impl std::fmt::Display for LeafScale { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Fixed => f.write_str("fixed"), + Self::DoubleFrom(t) => write!(f, "doubled from {t} leaves"), + Self::DoubledOneTop => write!( + f, + "doubled, one top over at most {ONE_TOP_MAX_LEAVES} leaves" + ), + } + } +} + +/// [`leaf_scale_from_env`]'s knob: unset is [`LeafScale::DEFAULT`], `off` is +/// [`LeafScale::Fixed`], a count `T` is [`LeafScale::DoubleFrom`]`(T)`, +/// `one-top` is [`LeafScale::DoubledOneTop`]. +pub const LEAF_DOUBLE_FROM_ENV: &str = "LAMBDA_VM_WHIR_LEAF_DOUBLE_FROM"; + +/// A value of [`LEAF_DOUBLE_FROM_ENV`] (or the CLI's `--leaf-double-from`): +/// unset or empty is [`LeafScale::DEFAULT`]; `off` is [`LeafScale::Fixed`]; a +/// positive count `T` is [`LeafScale::DoubleFrom`]`(T)`; `one-top` is +/// [`LeafScale::DoubledOneTop`]; anything else is an error. +pub fn leaf_scale_setting(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("") => Ok(LeafScale::DEFAULT), + Some("off") => Ok(LeafScale::Fixed), + Some("one-top") => Ok(LeafScale::DoubledOneTop), + Some(v) => v + .parse::() + .ok() + .filter(|&t| t >= 1) + .map(LeafScale::DoubleFrom) + .ok_or_else(|| { + format!( + "{LEAF_DOUBLE_FROM_ENV} must be `off`, `one-top` or a leaf count, got `{v}`" + ) + }), + } +} + +/// The environment's leaf scale ([`LEAF_DOUBLE_FROM_ENV`]). A format knob: +/// the block's verifier must be given the same one. +pub fn leaf_scale_from_env() -> Result { + leaf_scale_setting(std::env::var(LEAF_DOUBLE_FROM_ENV).ok().as_deref()) +} + +/// [`leaf_scale_from_env`], read once per process: what +/// [`BlockFormat::production`] holds. A value it cannot take is refused here, +/// as `LFM_WHIR_SHARE_INVERSE`'s is ([`share_inverse`]); the harness reads the +/// knob first ([`super::whir_block_tree::WhirTreeConfig::from_env`]) and +/// returns that refusal as an error. +pub(crate) fn process_leaf_scale() -> LeafScale { + static SCALE: std::sync::OnceLock = std::sync::OnceLock::new(); + *SCALE.get_or_init(|| leaf_scale_from_env().unwrap_or_else(|e| panic!("{e}"))) +} + +/// [`leaf_query_from_env`]'s knob: unset or `lean` is [`LeafQuery::Lean`] (the +/// default), `v1` is [`LeafQuery::V1`]. +pub const LEAF_QUERY_ENV: &str = "LAMBDA_VM_WHIR_LEAF_QUERY"; + +/// A value of [`LEAF_QUERY_ENV`]: unset or empty is the default +/// ([`LeafQuery::Lean`]); `lean` is [`LeafQuery::Lean`], `v1` is +/// [`LeafQuery::V1`]; anything else is an error. +pub fn leaf_query_setting(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("") => Ok(LeafQuery::default()), + Some("v1") => Ok(LeafQuery::V1), + Some("lean") => Ok(LeafQuery::Lean), + Some(v) => Err(format!( + "{LEAF_QUERY_ENV} must be `v1` or `lean`, got `{v}`" + )), + } +} + +/// The environment's leaf query check ([`LEAF_QUERY_ENV`]). A format knob: the +/// block's verifier must be given the same one. +pub fn leaf_query_from_env() -> Result { + leaf_query_setting(std::env::var(LEAF_QUERY_ENV).ok().as_deref()) +} + +/// [`leaf_query_from_env`], read once per process: what +/// [`BlockFormat::production`] holds, refused as [`process_leaf_scale`] refuses. +pub(crate) fn process_leaf_query() -> LeafQuery { + static QUERY: std::sync::OnceLock = std::sync::OnceLock::new(); + *QUERY.get_or_init(|| leaf_query_from_env().unwrap_or_else(|e| panic!("{e}"))) +} + +/// The leaf that subtracts the COMMIT-bus target. +pub const CARRIER: usize = 0; + +/// `LFM_WHIR_SHARE_INVERSE`: unset or `1` emits a leaf's bus share as +/// `p · (1/q)` ([`emit_share`]); `0` keeps the former `ediv(p, q)` leaf programs +/// (and their tree ids). Read once per process. Both sides derive the leaf +/// programs, so the verifier's setting is part of the tree identity it derives. +pub fn share_inverse() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init( + || match std::env::var("LFM_WHIR_SHARE_INVERSE").as_deref() { + Err(_) | Ok("1") => true, + Ok("0") => false, + Ok(v) => panic!("LFM_WHIR_SHARE_INVERSE must be 0 or 1, got `{v}`"), + }, + ) +} + +/// One table's share `p/q` of the block's bus sum. XALU division constrains +/// `q · out = p`, so `ediv(p, q)` has no satisfying assignment when `q = 0` +/// and `p ≠ 0`, but leaves `out` free when `p = q = 0`. With `inverse`, the +/// share is `p · ediv(1, q)`: `q · inv = 1` has no assignment for any `q = 0`, +/// whatever `p` is — the host's `contribution() = None` refusal. +pub fn emit_share(b: &mut LfmBuilder, p: Ext, q: Ext, inverse: bool) -> Ext { + if inverse { + let one = b.ext_const(&FEE::one()); + let inv = b.ediv(one, q); + b.emul(p, inv) + } else { + b.ediv(p, q) + } +} + +/// Children a node verifies: three, so the block's three leaves close in ONE +/// node, the top, instead of two levels. +pub const BLOCK_FAN_IN: usize = 3; + +// =============================== the partition ============================ + +/// Which groups each leaf verifies: lists of group indices that together cover +/// `0..num_groups` exactly once. (The no-epoch STARK block's `BlockPartition`, +/// over groups.) +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockPartition { + leaves: Vec>, + num_units: usize, +} + +impl BlockPartition { + /// Validate and sort each list. Refuses an empty leaf, an index out of + /// range, an index in two leaves (or twice in one), and a unit in none. + pub fn new(leaves: Vec>, num_units: usize) -> Result { + let mut seen = vec![None::; num_units]; + let mut sorted = Vec::with_capacity(leaves.len()); + for (k, mut list) in leaves.into_iter().enumerate() { + if list.is_empty() { + return Err(format!("leaf {k} verifies nothing")); + } + list.sort_unstable(); + for &i in &list { + let slot = seen + .get_mut(i) + .ok_or_else(|| format!("leaf {k}: {i} is out of range 0..{num_units}"))?; + if let Some(other) = slot.replace(k) { + return Err(format!("{i} is in leaf {other} and leaf {k}")); + } + } + sorted.push(list); + } + if let Some(i) = seen.iter().position(Option::is_none) { + return Err(format!("{i} is in no leaf")); + } + Ok(Self { + leaves: sorted, + num_units, + }) + } + + /// Lists taken as given, with none of [`Self::new`]'s checks: what an + /// adversarial prover may emit leaves over. Only the tests that show such a + /// tree is refused at the final check build one. + #[cfg(test)] + pub(crate) fn unvalidated(leaves: Vec>, num_units: usize) -> Self { + Self { leaves, num_units } + } + + pub fn num_leaves(&self) -> usize { + self.leaves.len() + } + + pub fn num_units(&self) -> usize { + self.num_units + } + + /// Leaf `k`'s units, ascending. + pub fn leaf(&self, k: usize) -> &[usize] { + &self.leaves[k] + } + + pub fn leaves(&self) -> &[Vec] { + &self.leaves + } +} + +/// The groups over `num_leaves` leaves: heaviest first, each onto the leaf with +/// the least load so far (ties: the lower group, then the lower leaf). Pure — +/// every emitter derives the same partition from the same costs — and never +/// more leaves than groups. +pub fn partition_groups(costs: &[usize], num_leaves: usize) -> BlockPartition { + let k = num_leaves.clamp(1, costs.len().max(1)); + let mut order: Vec = (0..costs.len()).collect(); + order.sort_by_key(|&g| (std::cmp::Reverse(costs[g]), g)); + let mut leaves: Vec> = vec![Vec::new(); k]; + let mut load = vec![0usize; k]; + for g in order { + let leaf = (0..k) + .min_by_key(|&l| (load[l], l)) + .expect("at least one leaf"); + leaves[leaf].push(g); + load[leaf] += costs[g]; + } + BlockPartition::new(leaves, costs.len()) + .expect("the rule places every group exactly once by construction") +} + +/// The leaf partition of groups costing `costs` permutations, under a leaf cap +/// of `cap`: over `num_leaves` leaves when the caller fixes the count (the +/// tests', never a statement's), else over the fewest leaves from +/// `⌈Σ cost / cap⌉` at which no leaf is over the cap — one more leaf while the +/// heaviest is over it, the STARK block's rule (`block_plan::partition_for`). +/// Refuses a group over the cap, which no leaf can hold: a group is atomic for +/// a leaf (its stacked opening covers all its tables), and without the bound a +/// statement could grow one leaf, and its LFM chips, without limit. +pub fn leaf_partition( + costs: &[usize], + num_leaves: Option, + cap: usize, +) -> Result { + if let Some((g, &cost)) = costs.iter().enumerate().find(|&(_, &c)| c > cap) { + return Err(format!( + "group {g} costs {cost} permutations — a leaf takes at most {cap}" + )); + } + if let Some(k) = num_leaves { + return Ok(partition_groups(costs, k.max(1))); + } + let mut k = costs.iter().sum::().div_ceil(cap).max(1); + loop { + let partition = partition_groups(costs, k); + let heaviest = partition + .leaves() + .iter() + .map(|leaf| leaf.iter().map(|&g| costs[g]).sum::()) + .max() + .unwrap_or(0); + // Every group is under the cap, so one group a leaf always fits. + if heaviest <= cap || k >= costs.len() { + return Ok(partition); + } + k += 1; + } +} + +/// ★ The Poseidon1 leaf partition: no leaf past any chip's padded height +/// (`heights`, each less the per-leaf `front`), over the fewest leaves that +/// admit one. The groups go heaviest first by their binding share (the largest +/// of their chips' rows over each chip's room). At each leaf count from the +/// rows' lower bound up, two placements: the balancing one (each group onto +/// the admitting leaf with the least binding share so far, ties the lower +/// leaf), then first fit (each onto the lowest admitting leaf), which packs +/// the heavy groups together where balancing would spread them over one leaf +/// more than the rows need; neither admitting every group, one leaf more. +/// Refuses a group that no leaf can hold, and a front past a height. Pure: +/// every emitter derives the same partition from the same rows. +pub fn leaf_partition_rows( + loads: &[ChipRows], + front: ChipRows, + heights: ChipRows, +) -> Result { + const N: usize = ChipRows::NAMES.len(); + let (h, f) = (heights.to_array(), front.to_array()); + if (0..N).any(|c| f[c] > h[c]) { + return Err(format!( + "the leaf front {front:?} is past a chip of {heights:?}" + )); + } + let room: [usize; N] = core::array::from_fn(|c| h[c] - f[c]); + let rows: Vec<[usize; N]> = loads.iter().map(|l| l.to_array()).collect(); + for (g, load) in rows.iter().enumerate() { + for c in 0..N { + if load[c] > room[c] { + return Err(format!( + "group {g} spends {} {} rows — a leaf takes at most {}", + load[c], + ChipRows::NAMES[c], + room[c] + )); + } + } + } + // A share in 2^-32 units of each chip's room; the binding one is the max. + // A chip with no room has no rows to share (refused above). + let share = |r: &[usize; N]| -> u64 { + (0..N) + .filter(|&c| room[c] > 0) + .map(|c| ((r[c] as u128) << 32).div_ceil(room[c] as u128) as u64) + .max() + .unwrap_or(0) + }; + let weights: Vec = rows.iter().map(share).collect(); + let mut order: Vec = (0..loads.len()).collect(); + order.sort_by_key(|&g| (std::cmp::Reverse(weights[g]), g)); + let place = |k: usize, first_fit: bool| -> Option>> { + let mut leaves: Vec> = vec![Vec::new(); k]; + let mut sums = vec![[0usize; N]; k]; + for &g in &order { + let add = &rows[g]; + let fits = |s: &[usize; N]| (0..N).all(|c| s[c] + add[c] <= room[c]); + let mut open = (0..k).filter(|&l| fits(&sums[l])); + let leaf = if first_fit { + open.next()? + } else { + open.min_by_key(|&l| (share(&sums[l]), l))? + }; + leaves[leaf].push(g); + for c in 0..N { + sums[leaf][c] += add[c]; + } + } + Some(leaves) + }; + let mut k = (0..N) + .filter(|&c| room[c] > 0) + .map(|c| rows.iter().map(|r| r[c]).sum::().div_ceil(room[c])) + .max() + .unwrap_or(1) + .max(1); + while k <= loads.len().max(1) { + if let Some(mut leaves) = place(k, false).or_else(|| place(k, true)) { + // A leaf left empty (more leaves than the placement needed) is + // dropped. + leaves.retain(|l| !l.is_empty()); + return BlockPartition::new(leaves, loads.len()); + } + k += 1; + } + Err("no partition admits every group".to_string()) +} + +/// What a plan's leaf partition held its leaves under: the leaf count at the +/// single heights, and the heights every leaf of the partition is under — the +/// doubled ones when the format's [`LeafScale`] doubled them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LeafBound { + pub single_leaves: usize, + pub heights: ChipRows, + pub doubled: bool, +} + +/// ★ The leaf partition under `scale`: [`leaf_partition_rows`] at `heights`, +/// and when [`LeafScale::DoubleFrom`]`(t)` and that takes `t` leaves or more, +/// or under [`LeafScale::DoubledOneTop`], [`leaf_partition_rows`] again at +/// `doubled`. Pure, so the prover and the +/// verifier derive one partition from one statement; below `t` it is the one +/// at `heights`, unchanged. Refuses what [`leaf_partition_rows`] refuses at +/// `heights`, so every group a doubled leaf holds fits a single one too, and +/// a `doubled` lower than `heights` in any chip. +pub fn scaled_leaf_partition( + loads: &[ChipRows], + front: ChipRows, + scale: LeafScale, + heights: ChipRows, + doubled: ChipRows, +) -> Result<(BlockPartition, LeafBound), String> { + if !heights.under(doubled) { + return Err(format!( + "the doubled heights {doubled:?} are below {heights:?}" + )); + } + let single = leaf_partition_rows(loads, front, heights)?; + let single_leaves = single.num_leaves(); + let double = match scale { + LeafScale::Fixed => false, + LeafScale::DoubleFrom(t) => single_leaves >= t, + LeafScale::DoubledOneTop => true, + }; + Ok(if double { + ( + leaf_partition_rows(loads, front, doubled)?, + LeafBound { + single_leaves, + heights: doubled, + doubled: true, + }, + ) + } else { + ( + single, + LeafBound { + single_leaves, + heights, + doubled: false, + }, + ) + }) +} + +/// The workers [`chip_loads_by_emission`] emits on at most. +const PROBE_WORKERS: usize = 16; + +/// ★ Each group's rows in every chip, and the per-leaf front's, measured by +/// emission — exact by construction, whatever the walk, the opening or the +/// fold emit (a closed form would be a second derivation to drift). Group `g` +/// is emitted alone as a leaf: its rows are the instructions between the front +/// and the tail (its walk, opening, prepared stack and bus shares), plus the +/// one `eadd` that joins its shares to a leaf's running sum. The front's are a +/// carrier leaf of no group's (the front, the COMMIT-bus target and the +/// publishes), less that one `eadd`. The builder emits each group on its own +/// fork and interns nothing but constants, so a leaf's rows are the front's +/// plus its groups' — exactly the carrier's, a non-carrier's less the target's +/// — and its distinct constants at most that sum. Each probe's front is held +/// to the others'. On plain threads: a prover derives beside phase B's pool. +pub(crate) fn chip_loads_by_emission( + plan: &WhirBlockPlan, +) -> Result<(Vec, ChipRows), String> { + let probe = |groups: &[usize], carrier: bool| -> Result<[ChipRows; 3], String> { + let mut b = plan.leaf_builder(); + let mut marks = LeafMarks::default(); + emit_leaf_over( + &mut b, + plan, + groups, + carrier, + LeafChecks::ALL, + Some(&mut marks), + )?; + let instrs = b.instrs(); + Ok([ + ChipRows::of_instrs(&instrs[..marks.front]), + ChipRows::of_instrs(&instrs[marks.front..marks.shares]), + ChipRows::of_instrs(&instrs[marks.shares..]), + ]) + }; + let n = plan.groups.len(); + let [front, _, tail] = probe(&[], true)?; + let workers = std::thread::available_parallelism() + .map_or(1, |p| p.get()) + .min(PROBE_WORKERS) + .min(n) + .max(1); + let next = std::sync::atomic::AtomicUsize::new(0); + let measured: Vec<(usize, Result<[ChipRows; 3], String>)> = std::thread::scope(|scope| { + let handles: Vec<_> = (0..workers) + .map(|_| { + scope.spawn(|| { + let mut out = Vec::new(); + loop { + let g = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if g >= n { + return out; + } + out.push((g, probe(&[g], false))); + } + }) + }) + .collect(); + handles + .into_iter() + .flat_map(|h| h.join().unwrap_or_default()) + .collect() + }); + let mut loads = vec![None; n]; + for (g, parts) in measured { + let [probe_front, rows, _] = parts?; + if probe_front != front { + return Err(format!( + "group {g}'s probe leaf has another front: {probe_front:?} against {front:?}" + )); + } + loads[g] = Some(ChipRows { + xalu: rows.xalu + 1, + ..rows + }); + } + let loads = loads + .into_iter() + .enumerate() + .map(|(g, l)| l.ok_or_else(|| format!("group {g}'s probe never ran"))) + .collect::, _>>()?; + let front = front.plus(tail); + Ok(( + loads, + ChipRows { + xalu: front.xalu.saturating_sub(1), + ..front + }, + )) +} + +// ================================= the plan =============================== + +/// One group's prepared stack, as the plan derived it. +pub struct PlannedPrepared { + pub group: usize, + /// `(slot in the group's table order, AIR index, prefix length)`, in stack + /// order. + pub tables: Vec<(usize, usize, usize)>, + /// Derived from the program and the partition, never read from a proof. + pub roots: Vec, + pub layout: StackedLayout, + pub domain: Domain, +} + +impl PlannedPrepared { + /// The prefix the stack settles of table `air`, when it holds it. + pub fn settles(&self, air: usize) -> Option { + self.tables + .iter() + .find(|&&(_, table, _)| table == air) + .map(|&(_, _, n)| n) + } +} + +/// Every program of a block's tree, derived from the trusted ELF, the options, +/// the format and the statement. See the module docs. +pub struct WhirBlockPlan { + frame: BlockFrame, + /// The statement's `append_bytes` calls: the base's sponge (ZisK's) + /// binds each call's length, so the leaves replay them call by call. + statement_calls: Vec>, + /// The tree's wrap hash, the block proof family's (`hash_pin::BlockFamily`): + /// the leaves replay the base's transcript in it and the nodes their + /// children's LFM proofs. + wrap_hash: super::edsl::WrapHash, + public_output: Vec, + /// The statement's groups: AIR indices in each group's order. + groups: Vec>, + /// The prepared tables, in the proof's table order (the order their roots + /// are absorbed and their openings carried). + prepared: Vec, + /// Each group's in-guest cost, in permutations. + costs: Vec, + /// Each group's rows in every bounded chip, and the per-leaf front's + /// ([`leaf_partition_rows`]). + loads: Vec, + front: ChipRows, + partition: BlockPartition, + /// What the partition held its leaves under (`None` under a fixed leaf + /// count, which bounds no chip). + leaf_bound: Option, + /// How the leaves check each chain's queries: the format's. + leaf_query: LeafQuery, + /// Whether a leaf's bus share is `p · (1/q)` ([`emit_share`]): the + /// process's knob for a prover ([`share_inverse`]), always on for the + /// pinned verifier. + share_inverse: bool, + /// One top over every leaf when they are at most [`ONE_TOP_MAX_LEAVES`] + /// ([`LeafScale::DoubledOneTop`]). + one_top: bool, + /// Children a node verifies. + fan_in: usize, + /// A digest of the statement run: the block's identity, published by every + /// leaf and node. + id: [u8; 32], +} + +/// Every program of a block's tree with its artifacts and derived shape, level +/// by level (the leaves first, the top last): what a prover proves and a +/// verifier derives, built from the plan alone. +pub struct TreePrograms { + pub levels: Vec>, +} + +pub struct TreeProgram { + pub program: LfmProgram, + pub artifacts: LfmArtifacts, + pub derived: DerivedChild, +} + +impl WhirBlockPlan { + /// Derive the plan over `num_leaves` leaves (when `None`, the fewest that + /// keep every chip of every leaf under [`P1_LEAF_HEIGHTS`], or under + /// [`P1_LEAF_HEIGHTS_2X`] where `format`'s [`LeafScale`] doubles them: + /// [`scaled_leaf_partition`]), or refuse a statement the host verifier + /// refuses or whose group no leaf holds. + pub fn derive( + elf_bytes: &[u8], + proof_options: &crate::ProofOptions, + format: &BlockFormat, + statement: BlockStatement<'_>, + num_leaves: Option, + ) -> Result { + Self::derive_with( + elf_bytes, + proof_options, + format, + statement, + num_leaves, + BLOCK_FAN_IN, + None, + ) + } + + /// [`Self::derive`] with the nodes' fan-in named, and optionally the + /// prepared tables' roots in `prepared_tables` order — a PROVER's own, + /// computed by the same function from the same program, so it need not + /// commit them twice. A verifier passes `None` and derives them itself; + /// wrong roots give other leaf programs, and another top. + pub fn derive_with( + elf_bytes: &[u8], + proof_options: &crate::ProofOptions, + format: &BlockFormat, + statement: BlockStatement<'_>, + num_leaves: Option, + fan_in: usize, + prepared_roots: Option<&[(usize, Vec)]>, + ) -> Result { + Self::derive_capped( + elf_bytes, + proof_options, + format, + statement, + num_leaves, + fan_in, + prepared_roots, + LEAF_P1_CAP, + ) + } + + /// [`Self::derive_with`] under a leaf cap of `leaf_cap` socket rows (the + /// partition's `hash` height; twice that, at most [`LEAF_P1_CAP_2X`], for + /// doubled leaves); everything but the tests' refusals and censuses runs + /// at [`LEAF_P1_CAP`]. + #[allow(clippy::too_many_arguments)] + pub(crate) fn derive_capped( + elf_bytes: &[u8], + proof_options: &crate::ProofOptions, + format: &BlockFormat, + statement: BlockStatement<'_>, + num_leaves: Option, + fan_in: usize, + prepared_roots: Option<&[(usize, Vec)]>, + leaf_cap: usize, + ) -> Result { + Self::derive_full( + elf_bytes, + proof_options, + format, + statement, + num_leaves, + fan_in, + prepared_roots, + leaf_cap, + share_inverse(), + ) + } + + /// [`Self::derive_capped`] with the leaves' share form named + /// (`share_inverse`, [`emit_share`]) rather than the process's knob: the + /// pinned verifier's derivation. + #[allow(clippy::too_many_arguments)] + fn derive_full( + elf_bytes: &[u8], + proof_options: &crate::ProofOptions, + format: &BlockFormat, + statement: BlockStatement<'_>, + num_leaves: Option, + fan_in: usize, + prepared_roots: Option<&[(usize, Vec)]>, + leaf_cap: usize, + share_inverse: bool, + ) -> Result { + if fan_in < 2 { + return Err(format!("a fan-in of {fan_in} closes no tree")); + } + // The machine replays the base's transcript, ZisK's sponge: the block + // family's hash, which the base proof was made under too + // (`with_block_hash!` admits no other base and reads no knob). + let base = &format.zf.base; + crate::block_whir::checked_base(base).map_err(|e| format!("{e:?}"))?; + let wrap_hash = ::WRAP; + let frame = block_frame(statement, elf_bytes, proof_options, format) + .map_err(|e| format!("statement: {e:?}"))?; + let elf_digest = crate::statement::elf_digest(elf_bytes); + let tag = crate::block_whir::block_statement_tag(base); + let statement_bytes = block_statement_bytes(statement, &tag, &elf_digest, &frame.config); + let statement_calls = + crate::block_whir::block_statement_calls(statement, &tag, elf_bytes, &frame.config)?; + let groups: Vec> = statement + .groups + .iter() + .map(|g| g.iter().map(|&t| t as usize).collect()) + .collect(); + + // The prepared tables: their layouts and domains from their shapes, + // their roots committed here (or the prover's own, see + // [`Self::derive_with`]). + if !format.prepared { + return Err( + "the block's recursion needs the prepared openings (BlockFormat::prepared)" + .to_string(), + ); + } + let columns = prepared_tables(&frame.airs, &frame.page_configs, format) + .map_err(|e| format!("prepared tables: {e:?}"))?; + let config = &frame.config; + let stacks = group_stacks(columns, statement.groups) + .map_err(|e| format!("prepared stacks: {e:?}"))?; + if let Some(roots) = prepared_roots + && roots + .iter() + .map(|(g, _)| *g) + .ne(stacks.iter().map(|s| s.group)) + { + return Err("the supplied prepared roots are for other groups".to_string()); + } + let mut prepared = Vec::with_capacity(stacks.len()); + for (index, stack) in stacks.into_iter().enumerate() { + let group = stack.group; + let start = groups[..group].iter().map(Vec::len).sum::(); + let tables: Vec<(usize, usize, usize)> = stack + .tables + .iter() + .map(|&(position, air, n)| (position - start, air, n)) + .collect(); + let layout = + stark::multilinear_table::global_layout(&stack.shapes(), config.format.stack) + .map_err(|e| format!("prepared group {group}: {e:?}"))?; + let domain = Domain::::new(layout.n_stack() + config.log_blowup) + .map_err(|e| format!("prepared group {group}: {e:?}"))?; + let roots = match prepared_roots { + Some(roots) => roots[index].1.clone(), + None => commit_prepared(stack, config, base) + .map_err(|e| format!("prepared group {group}: {e:?}"))?, + }; + if roots.len() != layout.num_polys() { + return Err(format!( + "prepared group {group}: {} roots for {} polynomials", + roots.len(), + layout.num_polys() + )); + } + prepared.push(PlannedPrepared { + group, + tables, + roots, + layout, + domain, + }); + } + + // Each group's cost in socket rows: its tables' legs, its opening, its + // prepared ones. + let refs = frame.airs.air_refs(); + let arity = wrap_hash.arity(); + let mut costs = Vec::with_capacity(groups.len()); + for (g, list) in groups.iter().enumerate() { + let owned = Shapes::build(&refs, &frame.shapes, list)?; + let shapes = owned.table_shapes(); + let entry = SpongeEntry::fresh_for(wrap_hash); + let tables: usize = match batched_cap(config)? { + None => shapes + .iter() + .map(|shape| table_verify_cost(shape, entry).perms()) + .sum(), + Some(planning) => super::whir_batch::batched_argue_cost( + &shapes, + &owned.argue_plan(planning), + entry, + ) + .perms(), + }; + let group_shapes: Vec<(usize, usize)> = list.iter().map(|&t| frame.shapes[t]).collect(); + let (_, group_of) = group_columns(&group_shapes); + let layout = &frame.stack_layouts[g]; + let chain = ChainShape { + query: format.leaf_query, + ..ChainShape::new_at(config, layout.n_stack(), arity) + }; + let opening = stacked_verify_cost_ood(layout, &group_of, &chain, entry, true).perms() + + commit_ood_point_perms(wrap_hash, layout.num_polys()); + let prepared_perms: usize = prepared + .iter() + .filter(|p| p.group == g) + .map(|p| { + let group_of: Vec = p + .tables + .iter() + .enumerate() + .flat_map(|(k, &(_, _, n))| std::iter::repeat_n(k, n)) + .collect(); + let chain = ChainShape { + query: format.leaf_query, + ..ChainShape::new_at(config, p.layout.n_stack(), arity) + }; + stacked_verify_cost(&p.layout, &group_of, &chain, entry).perms() + }) + .sum(); + costs.push(tables + opening + prepared_perms); + } + drop(refs); + let id = crate::statement::elf_digest(&statement_bytes); + // The partition comes after the rows are measured, over a plan whose + // partition the measuring never reads. + let partition = partition_groups(&costs, 1); + let mut plan = Self { + frame, + statement_calls, + wrap_hash, + public_output: statement.public_output.to_vec(), + groups, + prepared, + costs, + loads: Vec::new(), + front: ChipRows::default(), + partition, + leaf_bound: None, + leaf_query: format.leaf_query, + share_inverse, + one_top: format.leaf_scale == LeafScale::DoubledOneTop, + fan_in, + id, + }; + // Every chip's rows by group, emitted; the partition bounds them all, + // at the format's leaf scale (a fixed leaf count, the tests', bounds + // none). + (plan.loads, plan.front) = chip_loads_by_emission(&plan)?; + plan.partition = match num_leaves { + None => { + let heights = ChipRows { + hash: leaf_cap, + ..P1_LEAF_HEIGHTS + }; + let doubled = ChipRows { + hash: (2 * leaf_cap).min(LEAF_P1_CAP_2X), + ..P1_LEAF_HEIGHTS_2X + }; + let (partition, bound) = scaled_leaf_partition( + &plan.loads, + plan.front, + format.leaf_scale, + heights, + doubled, + )?; + plan.leaf_bound = Some(bound); + partition + } + Some(_) => leaf_partition(&plan.costs, num_leaves, leaf_cap)?, + }; + Ok(plan) + } + + pub fn partition(&self) -> &BlockPartition { + &self.partition + } + + /// What the partition held its leaves under — the leaf count at the + /// single heights and the heights every leaf is under — or `None` under a + /// fixed leaf count, which bounds no chip. + pub fn leaf_bound(&self) -> Option { + self.leaf_bound + } + + /// Each group's in-guest cost, in socket rows. + pub fn costs(&self) -> &[usize] { + &self.costs + } + + /// Each group's rows in every bounded chip, and the per-leaf front's — + /// what the partition admitted each leaf by. + pub fn chip_loads(&self) -> (&[ChipRows], ChipRows) { + (&self.loads, self.front) + } + + /// The tree's wrap hash: the block family's. + pub fn wrap_hash(&self) -> super::edsl::WrapHash { + self.wrap_hash + } + + /// A leaf's builder: the tree's wrap hash. + fn leaf_builder(&self) -> LfmBuilder { + LfmBuilder::new().with_wrap_hash(self.wrap_hash) + } + + /// The chain shape of a stack of `n_stack` variables, at the base's tree + /// arity. + pub fn chain_shape(&self, n_stack: usize) -> ChainShape { + ChainShape { + query: self.leaf_query, + ..ChainShape::new_at(&self.frame.config, n_stack, self.wrap_hash.arity()) + } + } + + pub fn carrier(&self) -> usize { + CARRIER + } + + pub fn num_groups(&self) -> usize { + self.groups.len() + } + + /// The prepared tables, in the proof's table order. + pub fn prepared(&self) -> &[PlannedPrepared] { + &self.prepared + } + + /// The block's identity: a digest of the statement run. + pub fn id(&self) -> &[u8; 32] { + &self.id + } + + /// What a leaf and a non-top node publish. + pub fn child_layout(&self) -> BlockLayout { + BlockLayout::child(STATE_WORDS, out_halves(&self.public_output).len()) + } + + /// What the top node publishes. + pub fn top_layout(&self) -> BlockLayout { + BlockLayout::top(STATE_WORDS, out_halves(&self.public_output).len()) + } + + /// Leaf `k`, emitted and validated. + pub fn leaf_program(&self, k: usize) -> Result { + if k >= self.partition.num_leaves() { + return Err(format!("no leaf {k}")); + } + let mut b = self.leaf_builder(); + emit_block_leaf(&mut b, self, k)?; + let program = finish(b)?; + // The partition held this leaf's planned rows (the front plus its + // groups') under the heights; the emitted rows are held there too, so + // the bound on every leaf chip holds by construction, not only by the + // rows' additivity (checked by the census instruments). + if let Some(bound) = self.leaf_bound { + let rows = ChipRows::of(&program); + if !rows.under(bound.heights) { + return Err(format!( + "leaf {k} emits {rows:?}, past the heights its partition holds it under, {:?}", + bound.heights + )); + } + } + Ok(program) + } + + /// The plan with every leaf held under `heights` instead of its + /// partition's: a test's tightened bound, to show the emitted rows are + /// checked against it ([`Self::leaf_program`]). + #[cfg(test)] + pub(crate) fn with_leaf_heights(mut self, heights: ChipRows) -> Self { + self.leaf_bound = self.leaf_bound.map(|bound| LeafBound { heights, ..bound }); + self + } + + /// The node over `children`, the top when `top`, emitted and validated: + /// its legs replay the children under the block's proof family + /// ([`crate::hash_pin::BlockFamily`]). + pub fn node_program( + &self, + children: &[&DerivedChild], + top: bool, + ) -> Result { + self.node_program_under::(children, top) + } + + /// [`Self::node_program`] for children proved under the family `Fam`: the + /// node's legs replay them under `Fam`'s wrap hash (its hash rows on + /// `Fam`'s socket). The children's shapes carry their own trees and tag + /// (`DerivedChild::from_artifacts_under::`). + pub(crate) fn node_program_under( + &self, + children: &[&DerivedChild], + top: bool, + ) -> Result { + let shapes: Vec<_> = children.iter().map(|c| c.shape()).collect(); + let mut b = LfmBuilder::new().with_wrap_hash(Fam::WRAP); + super::block_node::emit_block_node( + &mut b, + &BlockNodeInputs { + children: &shapes, + layout: self.child_layout(), + top, + }, + ); + let program = finish(b)?; + // A node wider than the plan's fan-in (one top over every leaf) is held + // to the LFM proofs' bits bound on its socket and bit-dec rows. + if children.len() > self.fan_in { + let rows = ChipRows::of(&program); + if rows.hash > ONE_TOP_SOCKET_ROWS || rows.bitdec > ONE_TOP_SOCKET_ROWS { + return Err(format!( + "a node over {} children emits {} socket and {} bit-dec rows — at most {ONE_TOP_SOCKET_ROWS}", + children.len(), + rows.hash, + rows.bitdec + )); + } + } + Ok(program) + } + + /// The levels above the leaves: the plan's fan-in, leftovers in arity-1 + /// nodes, the last level's single node the top. A one-leaf block still gets + /// a top node: the bus closes there. Under [`LeafScale::DoubledOneTop`], at + /// most [`ONE_TOP_MAX_LEAVES`] leaves go under one top. + pub fn levels(&self) -> Vec { + use super::per_table_aggregator::{Level, tree_shape}; + let leaves = self.partition.num_leaves(); + if self.one_top && leaves <= ONE_TOP_MAX_LEAVES { + return vec![Level { + arities: vec![leaves.max(1)], + }]; + } + let mut shape = tree_shape(leaves, self.fan_in); + if shape.is_empty() { + shape.push(Level { arities: vec![1] }); + } + shape + } + + /// Every program of the tree with its artifacts and derived shape, with no + /// proof: every leaf emitted (in parallel) and its artifacts built, each + /// node emitted over its children's derived shapes, level by level, under + /// the tree's options. + pub fn programs(&self, wrap_opts: &crate::ProofOptions) -> Result { + let words = self.child_layout().total(); + let build = |program: LfmProgram| -> Result { + let artifacts = artifacts_of(&program, wrap_opts); + let derived = derived_of(&artifacts, wrap_opts, words)?; + Ok(TreeProgram { + program, + artifacts, + derived, + }) + }; + let leaf = |k: usize| build(self.leaf_program(k)?); + #[cfg(feature = "parallel")] + let first: Vec = { + use rayon::prelude::*; + (0..self.partition.num_leaves()) + .into_par_iter() + .map(leaf) + .collect::>()? + }; + #[cfg(not(feature = "parallel"))] + let first: Vec = (0..self.partition.num_leaves()) + .map(leaf) + .collect::>()?; + let mut levels = vec![first]; + let shape = self.levels(); + for (lv, arities) in shape.iter().enumerate() { + let top = lv + 1 == shape.len(); + let below = levels.last().expect("the leaves"); + let mut at = 0usize; + let mut next = Vec::with_capacity(arities.arities.len()); + for &a in &arities.arities { + let kids = below + .get(at..at + a) + .ok_or_else(|| format!("level {}: arities overrun the children", lv + 1))?; + let derived: Vec<&DerivedChild> = kids.iter().map(|k| &k.derived).collect(); + next.push(build(self.node_program(&derived, top)?)?); + at += a; + } + if at != below.len() { + return Err(format!("level {}: arities leave children over", lv + 1)); + } + levels.push(next); + } + match levels.last().map(Vec::len) { + Some(1) => Ok(TreePrograms { levels }), + other => Err(format!("the tree closes to {other:?} nodes")), + } + } + + /// The top program's artifacts, with no proof: [`Self::derive_top_in`] + /// with the leaves built in windows of `LAMBDA_VM_BLOCK_DERIVE_BUILDS` + /// ([`derive_builds_bound`]), or under `LAMBDA_VM_BLOCK_DERIVE_HOLD=level` + /// (the A/B control, [`derive_holds_levels`]) every program of the tree + /// held to the end ([`Self::programs`]), as the verifier did before. + pub fn derive_top(&self, wrap_opts: &crate::ProofOptions) -> Result { + if derive_holds_levels()? { + let mut programs = self.programs(wrap_opts)?; + let top = programs + .levels + .pop() + .and_then(|mut level| level.pop()) + .ok_or("no top")?; + return Ok(top.artifacts); + } + self.derive_top_in(wrap_opts, derive_builds_bound()?) + } + + /// The top program's artifacts, derived as a verifier needs them: the + /// programs of [`Self::programs`], built in the same order, but each leaf + /// and node keeps only its child's shape ([`DerivedChild`], what the node + /// above is emitted over): its program and artifacts are dropped as soon + /// as that shape is derived, and only the top's artifacts are kept. The + /// leaves are built in windows of `window` (the whole level at once when + /// `None`), each window in parallel; the nodes one at a time. Holding + /// every level ([`Self::programs`]) took the p90 block's verifier to + /// 41.17 GiB live (RYZEN 071, I-MEMFIT §6.24). Scheduling and retention + /// only: every artifact is a pure function of its program and the + /// options, so the top is [`Self::programs`]' top. + pub fn derive_top_in( + &self, + wrap_opts: &crate::ProofOptions, + window: Option, + ) -> Result { + let words = self.child_layout().total(); + let derive = |program: LfmProgram| -> Result { + let artifacts = artifacts_of(&program, wrap_opts); + derived_of(&artifacts, wrap_opts, words) + }; + let leaves: Vec = (0..self.partition.num_leaves()).collect(); + let mut below = map_in_windows(&leaves, window, |&k| derive(self.leaf_program(k)?))?; + let shape = self.levels(); + for (lv, arities) in shape.iter().enumerate() { + let top = lv + 1 == shape.len(); + let mut at = 0usize; + let mut next = Vec::with_capacity(arities.arities.len()); + for &a in &arities.arities { + let kids: Vec<&DerivedChild> = below + .get(at..at + a) + .ok_or_else(|| format!("level {}: arities overrun the children", lv + 1))? + .iter() + .collect(); + let program = self.node_program(&kids, top)?; + if top { + if arities.arities.len() != 1 { + return Err(format!( + "the tree closes to {:?} nodes", + Some(arities.arities.len()) + )); + } + let artifacts = artifacts_of(&program, wrap_opts); + // The top's shape too, as [`Self::programs`] derives it: the + // same refusals. + derived_of(&artifacts, wrap_opts, words)?; + if at + a != below.len() { + return Err(format!("level {}: arities leave children over", lv + 1)); + } + return Ok(artifacts); + } + next.push(derive(program)?); + at += a; + } + if at != below.len() { + return Err(format!("level {}: arities leave children over", lv + 1)); + } + below = next; + } + Err("no top".to_string()) + } + + /// Replace the partition — another tree, test-only: what a prover may emit + /// leaves over, refused at the final check. The derived partition's leaf + /// bound goes with it: the replacement's leaves are held under none. + #[cfg(test)] + pub(crate) fn with_partition(mut self, partition: BlockPartition) -> Self { + assert_eq!(partition.num_units(), self.groups.len()); + self.partition = partition; + self.leaf_bound = None; + self + } + + /// Where group `g`'s first table sits in the proof's table order. + fn group_start(&self, g: usize) -> usize { + self.groups[..g].iter().map(Vec::len).sum() + } + + /// Where group `g`'s first root sits among the proof's roots. + fn root_start(&self, g: usize) -> usize { + self.frame.stack_layouts[..g] + .iter() + .map(StackedLayout::num_polys) + .sum() + } +} + +/// Words the transcript's state digest occupies: one, on the algebraic sponge. +const STATE_WORDS: usize = 1; + +/// The public output as the statement's 32-bit halves, little-endian, the last +/// one zero-padded. +pub fn out_halves(bytes: &[u8]) -> Vec { + super::keccak_host::pack_stream(bytes) +} + +/// The two published words of the block's id: its eight little-endian `u32` +/// halves, four to a word. +pub fn id_words(id: &[u8; 32]) -> [LfmWord; 2] { + let halves = super::keccak_host::pack_stream(id); + [ + [halves[0], halves[1], halves[2], halves[3]], + [halves[4], halves[5], halves[6], halves[7]], + ] +} + +/// `LAMBDA_VM_BLOCK_DERIVE_BUILDS=`: the block verifier builds a tree's +/// leaves in windows of `n` ([`WhirBlockPlan::derive_top`]), each window in +/// parallel; unset, the whole level is one window: #1013's knob of the same +/// name, on the WHIR tree. Scheduling only: every +/// artifact is a pure function of its program and the options, so the derived +/// top does not depend on it. A value that is not a positive integer is an +/// error. +pub fn derive_builds_bound() -> Result, String> { + parse_derive_builds(std::env::var(DERIVE_BUILDS_ENV).ok().as_deref()) +} + +/// [`derive_builds_bound`]'s knob. +pub const DERIVE_BUILDS_ENV: &str = "LAMBDA_VM_BLOCK_DERIVE_BUILDS"; + +/// `LAMBDA_VM_BLOCK_DERIVE_HOLD=level`: [`WhirBlockPlan::derive_top`] holds +/// every program and its artifacts to the end ([`WhirBlockPlan::programs`]), +/// as the verifier did before (the A/B control; #1013's knob of the same +/// name). Unset or empty, each is dropped once its child's shape is derived. +/// Any other value is an error. +pub fn derive_holds_levels() -> Result { + parse_derive_hold(std::env::var(DERIVE_HOLD_ENV).ok().as_deref()) +} + +/// [`derive_holds_levels`]' knob. +pub const DERIVE_HOLD_ENV: &str = "LAMBDA_VM_BLOCK_DERIVE_HOLD"; + +/// A value of [`DERIVE_HOLD_ENV`]. +pub fn parse_derive_hold(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("") => Ok(false), + Some("level") => Ok(true), + Some(v) => Err(format!("{DERIVE_HOLD_ENV} must be `level`, got `{v}`")), + } +} + +/// A value of [`DERIVE_BUILDS_ENV`]: unset or empty is the whole level. +pub fn parse_derive_builds(value: Option<&str>) -> Result, String> { + match value.map(str::trim) { + None | Some("") => Ok(None), + Some(v) => v + .parse::() + .ok() + .filter(|&n| n >= 1) + .map(Some) + .ok_or_else(|| format!("{DERIVE_BUILDS_ENV} must be a positive integer, got `{v}`")), + } +} + +/// `f` over `items` in order, in windows of `window` (all of them at once when +/// `None`), each window in parallel where there is a pool; the first error +/// stops it. At most `window` items are in `f` at once. +pub(crate) fn map_in_windows( + items: &[T], + window: Option, + f: impl Fn(&T) -> Result + Sync + Send, +) -> Result, String> { + let mut done = Vec::with_capacity(items.len()); + for part in items.chunks(window.unwrap_or(items.len()).max(1)) { + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + done.extend( + part.par_iter() + .map(&f) + .collect::, String>>()?, + ); + } + #[cfg(not(feature = "parallel"))] + done.extend(part.iter().map(&f).collect::, String>>()?); + } + Ok(done) +} + +/// A tree program's artifacts, committed under the block's proof family +/// ([`crate::hash_pin::BlockFamily`]'s STARK configuration) and the socket +/// its hash rows run on — the width-16 socket's for a program with `Hash16` +/// rows ([`LfmProgram::hasher`]). +pub fn artifacts_of(program: &LfmProgram, wrap_opts: &crate::ProofOptions) -> LfmArtifacts { + super::program_census::build_artifacts_counted_under::( + program, + wrap_opts, + program.hasher(::SOCKET), + ) +} + +/// The block family's LFM commitment configuration. +type BlockStark = ::Stark; + +/// A tree program's shape as its parent's emitter reads it: [`artifacts`] +/// proved under the block's proof family. +pub(crate) fn derived_of( + artifacts: &LfmArtifacts, + wrap_opts: &crate::ProofOptions, + words: usize, +) -> Result { + DerivedChild::from_artifacts_under::(artifacts, wrap_opts, words) +} + +fn finish(b: LfmBuilder) -> Result { + let program = compile(b.finish()); + super::validator::validate(&program).map_err(|e| format!("not admissible: {e:?}"))?; + Ok(program) +} + +/// Whether `words`, a top proof's published words, claim the block: the plan's +/// id and the public output, in the top layout. The state is bound across the +/// children by the nodes. +pub fn top_claims(plan: &WhirBlockPlan, words: &[(u32, LfmWord)]) -> bool { + let layout = plan.top_layout(); + if words.len() != layout.total() + || words + .iter() + .enumerate() + .any(|(i, (at, _))| *at as usize != i) + { + return false; + } + let id = id_words(&plan.id); + (0..2).all(|h| words[layout.id(h)].1 == id[h]) + && out_halves(&plan.public_output) + .iter() + .enumerate() + .all(|(i, v)| words[layout.out_half(i)].1 == base_word(*v)) +} + +/// ★ The no-epoch WHIR block's verifier over its tree's top proof: derive the +/// plan and the top program from the trusted ELF and the statement, verify +/// `top` against that program, and check it claims the block. +/// +/// Every preset is pinned here as a constant, none is a caller's or the +/// environment's ([`pinned_verifier_presets`]): the block format +/// ([`BlockFormat::pinned`]: [`crate::zf_format::ZfFormat::DEFAULT`]'s WHIR +/// format, the default leaf scale, the lean leaf check, whose base is the +/// block family's, `hash_pin::BLOCK_BASE`, never read from the proof), the +/// base's proof options and the tree's options under that format +/// ([`super::proof::block_base_options_under`], +/// [`super::proof::aggregation_wrap_options_under`]), the leaves' share form +/// (`p · (1/q)`), the plan's leaf rule at the format's leaf scale and its +/// fan-in ([`BLOCK_FAN_IN`]). A caller, or an environment, that could set +/// weaker ones would choose the programs the top is checked against and the +/// security it is checked at. +pub fn verify_block_tree( + elf_bytes: &[u8], + statement: BlockStatement<'_>, + top: &super::proof::LfmProof, +) -> Result<(), String> { + verify_block_tree_grouped(elf_bytes, statement, top, BlockFormat::pinned().group) +} + +/// [`verify_block_tree`] for a block whose groups' stacked polynomials were +/// opened per `group` ([`BlockFormat::pinned_with`]): the one format choice a +/// caller makes, and the verifier's constant — the prover's must match, the +/// statement binds it, and nothing reads it from the proof. Every other preset +/// is [`verify_block_tree`]'s. +pub fn verify_block_tree_grouped( + elf_bytes: &[u8], + statement: BlockStatement<'_>, + top: &super::proof::LfmProof, + group: GroupFold, +) -> Result<(), String> { + verify_block_tree_as(elf_bytes, statement, top, &pinned_verifier_presets(group)) +} + +/// [`verify_block_tree`] at the pinned format with the caller's NAMED choices: +/// its group opening, its WHIR blowup, its round-0 leaf order, its leaf scale +/// and its leaves' query check (`verify-block --group / --blowup / --leaves / +/// --leaf-double-from / --leaf-query`), every other preset pinned as before +/// (the folds, stack, grind and caps are [`crate::zf_format::ZfFormat::DEFAULT`]'s, +/// never the environment's). All five are the verifier's constants; the proof +/// carries none of them. A caller cannot hand in a format of its own: +/// `verify_block_tree_as` is crate-internal. The coset order is refused by name +/// under a base whose trees do not take it. +#[allow(clippy::too_many_arguments)] +pub fn verify_block_tree_at( + elf_bytes: &[u8], + statement: BlockStatement<'_>, + top: &super::proof::LfmProof, + group: GroupFold, + blowup: crate::zf_format::WhirBlowup, + leaves: multilinear::whir_chain::LeafOrder, + leaf_scale: LeafScale, + leaf_query: LeafQuery, + pack: multilinear::whir_chain::GkrPack, +) -> Result<(), String> { + let presets = verifier_presets_at(group, blowup, leaves, leaf_scale, leaf_query, pack)?; + verify_block_tree_as(elf_bytes, statement, top, &presets) +} + +/// [`verify_block_tree_at`]'s presets: [`BlockFormat::pinned_with`] `group` +/// with the five named choices set (the GKR packing through +/// [`BlockFormat::with_pack`]), the coset order refused under a base whose +/// trees do not take it. +pub(crate) fn verifier_presets_at( + group: GroupFold, + blowup: crate::zf_format::WhirBlowup, + leaves: multilinear::whir_chain::LeafOrder, + leaf_scale: LeafScale, + leaf_query: LeafQuery, + pack: multilinear::whir_chain::GkrPack, +) -> Result { + let mut format = BlockFormat::pinned_with(group).with_pack(pack); + format.zf.whir_blowup = blowup.factor(); + format.zf.whir_leaves = leaves; + format.leaf_scale = leaf_scale; + format.leaf_query = leaf_query; + format.zf.check_leaves()?; + Ok(verifier_presets(format)) +} + +/// What the public block verifiers check a top against, every field a constant +/// of the code ([`verifier_presets`]): the block format, the base's and the +/// tree's proof options under its WHIR format, and the leaves' share form. +pub(crate) struct VerifierPresets { + pub(crate) format: BlockFormat, + pub(crate) base_options: crate::ProofOptions, + pub(crate) wrap_options: crate::ProofOptions, + pub(crate) share_inverse: bool, +} + +/// The presets for a verifier that checks under `format`: the options under +/// `format.zf` (never the process's format) and the share form `p · (1/q)` +/// (never `LFM_WHIR_SHARE_INVERSE`). +pub(crate) fn verifier_presets(format: BlockFormat) -> VerifierPresets { + VerifierPresets { + base_options: super::proof::block_base_options_under(&format.zf), + wrap_options: super::proof::aggregation_wrap_options_under(&format.zf), + share_inverse: true, + format, + } +} + +/// [`verify_block_tree_grouped`]'s presets: [`BlockFormat::pinned_with`] +/// `group`. +pub(crate) fn pinned_verifier_presets(group: GroupFold) -> VerifierPresets { + verifier_presets(BlockFormat::pinned_with(group)) +} + +/// [`verify_block_tree`] under `presets` of the crate's choosing. +/// Crate-internal: a public caller who could pass any format would choose the +/// programs the top is checked against ([`verify_block_tree`]); the public +/// entry points build the pinned presets from named choices. +pub(crate) fn verify_block_tree_as( + elf_bytes: &[u8], + statement: BlockStatement<'_>, + top: &super::proof::LfmProof, + presets: &VerifierPresets, +) -> Result<(), String> { + let plan = verifier_plan(elf_bytes, statement, presets)?; + verify_top_of(&plan, &presets.wrap_options, top) +} + +/// [`verify_block_tree`] under presets of the caller's: the fixture formats, +/// leaf counts and fan-ins the tests run at. Test-only. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) fn verify_block_tree_under( + elf_bytes: &[u8], + proof_options: &crate::ProofOptions, + format: &BlockFormat, + statement: BlockStatement<'_>, + num_leaves: Option, + fan_in: usize, + wrap_opts: &crate::ProofOptions, + top: &super::proof::LfmProof, +) -> Result<(), String> { + verify_block_tree_with( + elf_bytes, + proof_options, + format, + statement, + num_leaves, + fan_in, + wrap_opts, + top, + LEAF_P1_CAP, + share_inverse(), + ) +} + +/// [`verify_block_tree_under`] with the leaves partitioned by the rule under a +/// leaf cap of `leaf_cap` socket rows ([`WhirBlockPlan::derive_capped`]): a +/// small block spans several leaves only under a cap far below +/// [`LEAF_P1_CAP`]. Test-only. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) fn verify_block_tree_capped( + elf_bytes: &[u8], + proof_options: &crate::ProofOptions, + format: &BlockFormat, + statement: BlockStatement<'_>, + fan_in: usize, + wrap_opts: &crate::ProofOptions, + top: &super::proof::LfmProof, + leaf_cap: usize, +) -> Result<(), String> { + verify_block_tree_with( + elf_bytes, + proof_options, + format, + statement, + None, + fan_in, + wrap_opts, + top, + leaf_cap, + share_inverse(), + ) +} + +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +fn verify_block_tree_with( + elf_bytes: &[u8], + proof_options: &crate::ProofOptions, + format: &BlockFormat, + statement: BlockStatement<'_>, + num_leaves: Option, + fan_in: usize, + wrap_opts: &crate::ProofOptions, + top: &super::proof::LfmProof, + leaf_cap: usize, + share_inverse: bool, +) -> Result<(), String> { + let plan = WhirBlockPlan::derive_full( + elf_bytes, + proof_options, + format, + statement, + num_leaves, + fan_in, + None, + leaf_cap, + share_inverse, + )?; + verify_top_of(&plan, wrap_opts, top) +} + +/// The plan the public verifiers derive for `statement` under `presets`: the +/// one whose top program and block id a top proof is checked against +/// ([`verify_block_tree_as`]). +pub(crate) fn verifier_plan( + elf_bytes: &[u8], + statement: BlockStatement<'_>, + presets: &VerifierPresets, +) -> Result { + WhirBlockPlan::derive_full( + elf_bytes, + &presets.base_options, + &presets.format, + statement, + None, + BLOCK_FAN_IN, + None, + LEAF_P1_CAP, + presets.share_inverse, + ) +} + +/// `top` verified against `plan`'s derived top program under `wrap_opts`, and +/// claiming `plan`'s block. +fn verify_top_of( + plan: &WhirBlockPlan, + wrap_opts: &crate::ProofOptions, + top: &super::proof::LfmProof, +) -> Result<(), String> { + let artifacts = plan.derive_top(wrap_opts)?; + if !super::proof::verify_against_artifacts_under::( + &artifacts, + &top.proof, + &top.public_words, + wrap_opts, + ) { + return Err("the top proof does not verify against the derived top program".to_string()); + } + if !top_claims(plan, &top.public_words) { + return Err("the top proof does not claim this block".to_string()); + } + Ok(()) +} + +// ================================ the leaf ================================ + +/// Some tables' layouts and buses, owned, for [`TableShape`]s to borrow. +struct Shapes<'a> { + layouts: Vec>, + buses: Vec>>, +} + +type DynAir<'a> = dyn stark::traits::AIR< + Field = GoldilocksField, + FieldExtension = GoldilocksExtension, + PublicInputs = (), + > + 'a; + +impl<'a> Shapes<'a> { + /// The layouts of `tables` (AIR indices) at their stated shapes. + fn build( + refs: &[&'a DynAir<'a>], + shapes: &[(usize, usize)], + tables: &[usize], + ) -> Result { + let mut layouts = Vec::with_capacity(tables.len()); + let mut buses = Vec::with_capacity(tables.len()); + for &t in tables { + let air = refs[t]; + let (width, num_vars) = shapes[t]; + let layout = crate::multilinear_prove::layout_of(air, width, num_vars) + .map_err(|e| format!("{}: {e:?}", air.name()))?; + let slots = layout.slot_of().to_vec(); + let bus = stark::multilinear_logup::interaction_shapes( + air.bus_interactions(), + slots.len(), + |column| { + slots + .get(column) + .copied() + .ok_or(multilinear::Error::UnknownPolynomial { + index: column, + len: slots.len(), + }) + }, + ) + .map_err(|e| format!("{}: bus: {e:?}", air.name()))?; + layouts.push(layout); + buses.push(bus); + } + Ok(Self { layouts, buses }) + } + + fn table_shapes(&self) -> Vec> { + self.layouts + .iter() + .zip(&self.buses) + .map(|(layout, bus)| TableShape { + ir: layout.shape(), + bus, + kinds: layout.kinds(), + num_columns: layout.num_columns(), + num_vars: layout.num_vars(), + }) + .collect() + } + + /// The batched argue's plan over these tables, the host's own + /// [`stark::multilinear_table::argue_plan`] at the format's cap: the bins + /// are part of the transcript, so the leaf never restates them. + fn argue_plan(&self, planning: BatchedPlanning) -> stark::multilinear_table::ArguePlan { + let shapes: Vec = self + .layouts + .iter() + .map(|layout| stark::multilinear_table::ArgueShape::of(&layout.statement())) + .collect(); + stark::multilinear_table::argue_plan_packed(&shapes, planning.bin_log_cells, planning.pack) + } +} + +/// The batched argue's planning constants when the block's format batches its +/// groups: its bin cap and its GKR packing, both the format's. +fn batched_cap( + config: &multilinear::whir_chain::ChainConfig, +) -> Result, String> { + use multilinear::whir_chain::ArgueFormat; + match config.format.argue { + ArgueFormat::Batched { + bin_log_cells, + pack, + } => Ok(Some(BatchedPlanning { + bin_log_cells, + pack, + })), + ArgueFormat::PerTable => Ok(None), + } +} + +/// What plans a batched group, the format's two constants: the bin cap and +/// how a table's interactions are laid into trees (the host's +/// `argue_plan_packed`, called, never restated). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct BatchedPlanning { + bin_log_cells: u8, + pack: multilinear::whir_chain::GkrPack, +} + +/// How a table's preprocessed columns are discharged in a leaf: the epoch's +/// routes by name, the page route for a page table, and nothing left for a +/// prepared table, whose opening settles every one of them. A table with +/// preprocessed columns and no route is refused, never skipped. +enum Route { + None, + Bitwise, + ConstMle(Vec>), + Page(Vec>), + Prepared(usize), + /// A CPU table's TIMESTAMP, by its closed form. + Timestamp, +} + +fn route_of( + plan: &WhirBlockPlan, + air: &DynAir<'_>, + table: usize, + first_page: usize, +) -> Result { + if let Some(n) = plan.prepared.iter().find_map(|p| p.settles(table)) { + return Ok(Route::Prepared(n)); + } + let pages = first_page..first_page + plan.frame.page_configs.len(); + if pages.contains(&table) { + let config = &plan.frame.page_configs[table - first_page]; + let columns = if config.is_private_input { + vec![crate::tables::page::offset_column()] + } else { + crate::tables::page::preprocessed_columns(config) + }; + return Ok(Route::Page(columns)); + } + // A CPU table's TIMESTAMP is checked by its closed form, by name and with its + // AIR's one preprocessed column: never materialised (2^21 rows a table). + if air.name().starts_with(super::whir_epoch::CPU_NAME_PREFIX) + && air.num_precomputed_columns() == crate::tables::cpu::NUM_PRECOMPUTED_COLS + { + return Ok(Route::Timestamp); + } + let columns = air.precomputed_columns(); + if columns.is_empty() { + return Ok(Route::None); + } + match air.name() { + BITWISE_NAME => Ok(Route::Bitwise), + KECCAK_RC_NAME | REGISTER_NAME => Ok(Route::ConstMle(columns)), + DECODE_NAME => Err("DECODE is not prepared".to_string()), + other => Err(format!( + "table {table} ({other}) carries {} preprocessed columns and no route covers them", + columns.len() + )), + } +} + +/// What every leaf's front leaves: the carried roots and their commit-time +/// answers, the transcript after the draws, the challenges and the state +/// digest. +struct Front { + carried: Vec, + answers: Vec, + transcript: WhirTranscript, + z: Ext, + alpha: Ext, + beta: Ext, + state: Cell, +} + +/// The front every leaf replays: every group's roots and every carried +/// polynomial's commit-time answer (hinted, in that order), the statement +/// (constants), the roots block with the derived prepared roots and then the +/// answers (`multilinear_block::absorb_roots_answers_and_challenge`), +/// `z, α, β`, and the state digest (which does not advance the transcript). +fn emit_front( + b: &mut LfmBuilder, + plan: &WhirBlockPlan, + arena: super::instr::ArenaId, + at: &mut u32, +) -> Front { + let num_roots: usize = plan + .frame + .stack_layouts + .iter() + .map(StackedLayout::num_polys) + .sum(); + let carried: Vec = (0..num_roots) + .map(|_| { + let cell = b.hint_word(arena, *at); + *at += 1; + cell + }) + .collect(); + let answers: Vec = (0..num_roots) + .map(|_| { + let cell = b.hint_word(arena, *at); + *at += 1; + cell.as_ext() + }) + .collect(); + let mut transcript = WhirTranscript::for_builder(b); + // ZisK's sponge binds each call's length: the statement call by call. + for call in &plan.statement_calls { + transcript.absorb_const_bytes(call); + } + let derived: Vec = plan + .prepared + .iter() + .flat_map(|p| { + p.roots + .iter() + .map(super::algebraic_commit::commitment_to_digest) + }) + .collect(); + emit_absorb_roots(b, &mut transcript, &carried, &derived); + #[cfg(test)] + let late = ood_faults::get() == ood_faults::Fault::AnswersAfterDraws; + #[cfg(not(test))] + let late = false; + #[cfg(test)] + let skipped = match ood_faults::get() { + ood_faults::Fault::SkipFirstGroupsAnswers => plan + .frame + .stack_layouts + .first() + .map_or(0, StackedLayout::num_polys), + _ => 0, + }; + #[cfg(not(test))] + let skipped = 0; + if !late { + for answer in &answers[skipped..] { + transcript.absorb_ext(b, *answer); + } + } + let z = transcript.sample_ext(b); + let alpha = transcript.sample_ext(b); + let beta = transcript.sample_ext(b); + if late { + for answer in &answers { + transcript.absorb_ext(b, *answer); + } + } + let state = transcript.state(b); + Front { + carried, + answers, + transcript, + z, + alpha, + beta, + state, + } +} + +/// The front alone, publishing `z, α, β` and then group `g`'s fork's first +/// draw — what a test compares against the host's transcript. +#[cfg(test)] +pub(crate) fn front_program(plan: &WhirBlockPlan, g: usize) -> LfmProgram { + let mut b = plan.leaf_builder(); + let arena = b.declare_arena(0); + let mut at = 0u32; + let front = emit_front(&mut b, plan, arena, &mut at); + for e in [front.z, front.alpha, front.beta] { + b.public(e.as_cell()); + } + let mut fork = front.transcript.clone(); + fork.absorb_const_bytes(&(g as u64).to_le_bytes()); + let first = fork.sample_ext(&mut b); + b.public(first.as_cell()); + // Group `g`'s commit-time point, from the front's root cells. + let layout = &plan.frame.stack_layouts[g]; + let root_at = plan.root_start(g); + let point = emit_commit_ood_point( + &mut b, + &stark::multilinear_block::commit_ood_prefix(g, layout, &plan.frame.config), + &front.carried[root_at..root_at + layout.num_polys()], + layout.n_stack(), + layout.n_stack() + plan.frame.config.log_blowup, + ); + b.public(point[0].as_cell()); + b.set_arena_len(arena, at); + compile(b.finish()) +} + +/// ★ `multilinear_block::commit_ood_point`, emitted: a fresh sponge over the +/// group's prefix (program constants, ONE `append_bytes`) and its roots (the +/// front's cells), one draw `z0`, refused inside the group's domain of +/// `2^log_size` points; returns the point over the stack, `z0^(2^j)` for +/// `j < n_stack`. One squaring chain serves both: the point takes its first +/// `n_stack` entries and the domain check its last. +pub(crate) fn emit_commit_ood_point( + b: &mut LfmBuilder, + prefix: &[u8], + roots: &[Cell], + n_stack: usize, + log_size: usize, +) -> Vec { + assert!(n_stack <= log_size, "the stack lies inside its domain"); + let mut transcript = WhirTranscript::for_builder(b); + transcript.absorb_const_bytes(prefix); + for root in roots { + transcript.absorb_digest(b, *root); + } + let z0 = transcript.sample_ext(b); + let mut powers = Vec::with_capacity(log_size + 1); + powers.push(z0); + for j in 0..log_size { + let squared = b.emul(powers[j], powers[j]); + powers.push(squared); + } + let one = b.ext_const(&FEE::one()); + super::whir_chain::emit_require_out_of_domain(b, powers[log_size], one); + powers.truncate(n_stack); + powers +} + +/// PERMUTATIONS [`emit_commit_ood_point`] costs a group of `polys` roots: its +/// own sponge over the prefix — under Poseidon1 one `append_bytes`, its length +/// and nine felts — the roots, and one draw. +pub(crate) fn commit_ood_point_perms(hash: super::edsl::WrapHash, polys: usize) -> usize { + let mut schedule = super::whir_transcript::SpongeSchedule::new(SpongeEntry::fresh_for(hash)); + let prefix_felts = COMMIT_OOD_PREFIX_BYTES / 8; + schedule.absorb(usize::from(hash == super::edsl::WrapHash::Poseidon1) + prefix_felts); + for _ in 0..polys { + schedule.absorb_root(); + } + schedule.draw_ext(); + schedule.perms() +} + +/// [`stark::multilinear_block::commit_ood_prefix`]'s length: the label and +/// five words. +const COMMIT_OOD_PREFIX_BYTES: usize = 32 + 5 * 8; + +/// Which of a leaf's own checks it emits. Production emits all of them +/// ([`LeafChecks::ALL`]); a weakened set exists so a negative test can show the +/// check it names is the one refusing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LeafChecks { + /// The prepared openings (their chains are hinted either way, so the arena + /// does not move). + pub(crate) prepared: bool, + /// Whether the carrier subtracts the COMMIT-bus target. + pub(crate) target: bool, +} + +impl LeafChecks { + pub const ALL: Self = Self { + prepared: true, + target: true, + }; +} + +/// ★ Emit block leaf `k`: the shared front, then its groups on their forks, +/// then its share of the bus and the block layout's publishes. +pub fn emit_block_leaf(b: &mut LfmBuilder, plan: &WhirBlockPlan, k: usize) -> Result<(), String> { + emit_leaf(b, plan, k, LeafChecks::ALL) +} + +/// [`emit_block_leaf`] under a weakened check set — a mutation, test-only. +#[cfg(test)] +pub(crate) fn leaf_program_with( + plan: &WhirBlockPlan, + k: usize, + checks: LeafChecks, +) -> Result { + let mut b = plan.leaf_builder(); + emit_leaf(&mut b, plan, k, checks)?; + finish(b) +} + +fn emit_leaf( + b: &mut LfmBuilder, + plan: &WhirBlockPlan, + k: usize, + checks: LeafChecks, +) -> Result<(), String> { + emit_leaf_over(b, plan, plan.partition.leaf(k), k == CARRIER, checks, None) +} + +/// Where a leaf's parts end among its instructions: the front, and its +/// groups' walks with their bus shares; the rest is the carrier's target and +/// the publishes ([`chip_loads_by_emission`]). +#[derive(Default)] +struct LeafMarks { + front: usize, + shares: usize, +} + +/// A leaf over `groups`, the carrier when `carrier`: [`emit_leaf`]'s body, which +/// [`chip_loads_by_emission`] also runs over one group or none, marking where +/// the parts end. +fn emit_leaf_over( + b: &mut LfmBuilder, + plan: &WhirBlockPlan, + groups: &[usize], + carrier: bool, + checks: LeafChecks, + mut marks: Option<&mut LeafMarks>, +) -> Result<(), String> { + let frame = &plan.frame; + let config = &frame.config; + let refs = frame.airs.air_refs(); + let first_page = first_page_index(&frame.airs).map_err(|e| format!("{e:?}"))?; + let arena = b.declare_arena(0); + let mut at = 0u32; + + // ---- 1. the front. + let Front { + carried, + answers, + transcript, + z, + alpha, + beta, + state, + } = emit_front(b, plan, arena, &mut at); + if let Some(m) = marks.as_deref_mut() { + m.front = b.instrs().len(); + } + + // ---- 2. the leaf's groups, each on its fork. + let mut outputs: Vec<(Ext, Ext)> = Vec::new(); + for &g in groups { + let tables = &plan.groups[g]; + let owned = Shapes::build(&refs, &frame.shapes, tables)?; + let shapes = owned.table_shapes(); + let routes: Vec = tables + .iter() + .map(|&t| route_of(plan, refs[t], t, first_page)) + .collect::>()?; + // BITWISE's closed form ([`super::preprocessed::emit_bitwise_preprocessed`]) + // takes its fixed height alone. The frame refuses any other height + // first (`block_whir::check_bitwise_height`); this keeps the leaf's + // refusal an error, not the emitter's assertion, if that ever moves. + if let Some(shape) = routes + .iter() + .zip(&shapes) + .find(|(route, shape)| { + matches!(route, Route::Bitwise) + && shape.num_vars != crate::tables::bitwise::NUM_VARS + }) + .map(|(_, shape)| shape) + { + return Err(format!( + "group {g}: BITWISE at 2^{} rows — its table is 2^{}", + shape.num_vars, + crate::tables::bitwise::NUM_VARS + )); + } + let views: Vec> = routes + .iter() + .map(|route| match route { + Route::ConstMle(columns) | Route::Page(columns) => { + columns.iter().map(Vec::as_slice).collect() + } + _ => Vec::new(), + }) + .collect(); + let plans: Vec> = routes + .iter() + .zip(&views) + .enumerate() + .map(|(index, (route, view))| match route { + Route::None => PreprocessedPlan { + settled: 0, + route: PreprocessedRoute::None, + }, + Route::Bitwise => PreprocessedPlan { + settled: 0, + route: PreprocessedRoute::Bitwise, + }, + Route::ConstMle(_) => PreprocessedPlan { + settled: 0, + route: PreprocessedRoute::ConstMle(view), + }, + Route::Page(_) => PreprocessedPlan { + settled: 0, + route: PreprocessedRoute::Page { + offset: view[0], + init: view.get(1).copied(), + }, + }, + Route::Prepared(settled) => PreprocessedPlan { + settled: *settled, + route: PreprocessedRoute::None, + }, + Route::Timestamp => PreprocessedPlan { + settled: 0, + route: PreprocessedRoute::Timestamp { + num_vars: shapes[index].num_vars, + }, + }, + }) + .collect(); + let slots: Vec<&[usize]> = owned.layouts.iter().map(|l| l.slot_of()).collect(); + let ladder_len = shapes + .iter() + .map(|shape| super::whir_bus::alpha_powers_read(shape.bus)) + .max() + .unwrap_or(1); + let ladder = super::whir_poly::emit_challenge_powers(b, alpha, ladder_len); + + let mut fork = transcript.clone(); + fork.absorb_const_bytes(&(g as u64).to_le_bytes()); + let walk = match batched_cap(config)? { + None => { + let store: Vec = shapes + .iter() + .map(|shape| hint_table_wires_shaped(b, arena, &mut at, shape)) + .collect(); + let wires: Vec> = + store.iter().map(TableWires::borrow).collect(); + emit_table_walk( + b, &mut fork, &wires, &shapes, &plans, &slots, z, &ladder, beta, + ) + } + // The group's tables in one batched argue, its bins the host's. + Some(planning) => { + let argue_plan = owned.argue_plan(planning); + let store = + super::whir_batch::hint_batched_shaped(b, arena, &mut at, &shapes, &argue_plan); + super::whir_batch::emit_batched_walk( + b, + &mut fork, + &store, + &shapes, + &argue_plan, + &plans, + &slots, + z, + &ladder, + beta, + ) + } + }; + + // The group's opening, against its own carried roots, its answers + // carried at its own commit-time point — the front's root and answer + // cells, never a second copy. + let layout = &frame.stack_layouts[g]; + let root_at = plan.root_start(g); + let group_roots = &carried[root_at..root_at + layout.num_polys()]; + let point = emit_commit_ood_point( + b, + &stark::multilinear_block::commit_ood_prefix(g, layout, config), + group_roots, + layout.n_stack(), + layout.n_stack() + config.log_blowup, + ); + #[cfg(test)] + let fresh: Vec = match ood_faults::get() { + ood_faults::Fault::FreshAnswerHint => (0..layout.num_polys()) + .map(|_| { + let cell = b.hint_word(arena, at); + at += 1; + cell.as_ext() + }) + .collect(), + _ => Vec::new(), + }; + #[cfg(test)] + let group_answers: &[Ext] = if fresh.is_empty() { + &answers[root_at..root_at + layout.num_polys()] + } else { + &fresh + }; + #[cfg(not(test))] + let group_answers: &[Ext] = &answers[root_at..root_at + layout.num_polys()]; + let chain = plan.chain_shape(layout.n_stack()); + let held = hint_group_chains_shaped(b, arena, &mut at, layout.num_polys(), &chain); + let openings: Vec<_> = held + .storage + .iter() + .map(super::whir_chain::RoundStorage::openings) + .collect(); + let rounds: Vec>> = held + .storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + // Each chain against its run's slice of the carried roots: tree `p` of + // a run is checked against the root at `root_at + run.start + p`, a + // program constant (REV-B2-JUDGE B2-4). + let chains: Vec> = held + .runs + .iter() + .enumerate() + .map(|(i, run)| StackedChainWires { + rounds: &rounds[i], + roots: &carried[root_at + run.start..root_at + run.end], + final_value: held.finals[i], + }) + .collect(); + emit_group_walk_chains( + b, + &mut fork, + &[GroupChainWires { + layout, + chains: &chains, + shape: &chain, + domain: &frame.domains[g], + ood: Some(OodWires { + point: &point, + answers: group_answers, + }), + }], + &[tables.len()], + &walk, + ); + + // Its prepared stack, against the derived roots: each table's block at + // that table's point. + for p in plan.prepared.iter().filter(|p| p.group == g) { + let shape = plan.chain_shape(p.layout.n_stack()); + let held = hint_group_chains_shaped(b, arena, &mut at, p.layout.num_polys(), &shape); + let roots: Vec = p + .roots + .iter() + .map(|root| { + b.digest_const(super::algebraic_commit::commitment_to_digest(root)) + .as_cell() + }) + .collect(); + let openings: Vec<_> = held + .storage + .iter() + .map(super::whir_chain::RoundStorage::openings) + .collect(); + let rounds: Vec>> = held + .storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + let chains: Vec> = held + .runs + .iter() + .enumerate() + .map(|(i, run)| StackedChainWires { + rounds: &rounds[i], + roots: &roots[run.clone()], + final_value: held.finals[i], + }) + .collect(); + if !checks.prepared { + continue; + } + let mut points: Vec<&[Ext]> = Vec::new(); + let mut values: Vec = Vec::new(); + for &(slot, _, n) in &p.tables { + let column_at = walk.column_at(slot); + points.extend(std::iter::repeat_n(walk.points[slot].as_slice(), n)); + values.extend_from_slice(&walk.values[column_at..column_at + n]); + } + // Prepared roots carry no answers: the verifier derives them, so + // each is a codeword. + super::whir_stacked::emit_stacked_verify_chains( + b, &mut fork, &p.layout, &chains, &points, &values, None, &shape, &p.domain, + ); + } + outputs.extend(walk.outputs.iter().copied()); + } + + // ---- 3. the leaf's share of the bus: Σ p/q, less the COMMIT-bus target on + // the carrier (commits are indexed from 0 in a block). + let mut sum: Option = None; + let inverse = plan.share_inverse; + for (p, q) in &outputs { + let share = emit_share(b, *p, *q, inverse); + sum = Some(match sum { + None => share, + Some(running) => b.eadd(running, share), + }); + } + if let Some(m) = marks { + m.shares = b.instrs().len(); + } + let mut sum = sum.unwrap_or_else(|| b.ext_const(&FEE::zero())); + if carrier && checks.target { + let target = emit_expected(b, &plan.public_output, 0, z, alpha); + sum = b.esub(sum, target); + } + + // ---- the publishes, in `BlockLayout::child`'s order. + for word in id_words(&plan.id) { + let cell = b.digest_const(word).as_cell(); + b.public(cell); + } + b.public(state); + for half in out_halves(&plan.public_output) { + let felt = b.felt_const(half); + b.public(felt.as_cell()); + } + b.public(sum.as_cell()); + b.set_arena_len(arena, at); + Ok(()) +} + +// ================================ the arena =============================== + +/// Leaf `k`'s arena from the block proof — the witness — in the order +/// [`emit_block_leaf`] hints it. Refuses a proof whose argument does not have +/// the shapes the plan derived (a count the leaf would misread). +pub fn block_leaf_arena( + plan: &WhirBlockPlan, + proof: &BlockWhirProof, + k: usize, +) -> Result>, String> { + let mut groups = Vec::with_capacity(plan.partition.leaf(k).len()); + for &g in plan.partition.leaf(k) { + let start = plan.group_start(g); + let tables = match batched_cap(&plan.frame.config)? { + None => proof + .proof + .tables + .get(start..start + plan.groups[g].len()) + .ok_or("the proof is short of tables")?, + Some(_) => &[], + }; + let prepared = match plan.prepared.iter().position(|p| p.group == g) { + Some(index) => Some( + proof + .prepared + .get(index) + .ok_or("the proof is short of prepared openings")?, + ), + None => None, + }; + let opening = proof + .proof + .columns + .get(g) + .ok_or("the proof is short of openings")?; + groups.push(group_arena_words( + plan, + g, + tables, + proof.argues.get(g), + opening, + prepared, + )?); + } + if proof.commit_ood.len() != proof.proof.roots.len() { + return Err(format!( + "the proof carries {} commit-time answers for {} roots", + proof.commit_ood.len(), + proof.proof.roots.len() + )); + } + Ok(leaf_arena(&proof.proof.roots, &proof.commit_ood, groups)) +} + +/// A leaf's arena from the block's roots, their commit-time answers and its +/// groups' words ([`group_arena_words`]), in the leaf's group order. +pub fn leaf_arena( + roots: &[Commitment], + answers: &[FEE], + groups: Vec>, +) -> Vec> { + let mut words: Vec = roots + .iter() + .map(super::algebraic_commit::commitment_to_digest) + .collect(); + words.extend(answers.iter().map(super::word::ext_word)); + for group in groups { + words.extend(group); + } + vec![words] +} + +/// Group `g`'s words of a leaf's arena, from its share of the proof alone: its +/// tables' proofs (the per-table format) or its batched argue, its opening and +/// its prepared opening. A group's words depend on nothing else, so a leaf can +/// be fed each group as phase B finishes it. Refuses the shapes the plan does +/// not derive, as [`block_leaf_arena`] does. +pub fn group_arena_words( + plan: &WhirBlockPlan, + g: usize, + tables: &[stark::multilinear_table::TableProof], + argue: Option<&stark::multilinear_table::BatchedArgue>, + opening: &multilinear::stacked_eval::StackedProof, + prepared: Option< + &multilinear::stacked_eval::StackedProof, + >, +) -> Result, String> { + let frame = &plan.frame; + let refs = frame.airs.air_refs(); + let group_tables = plan.groups.get(g).ok_or("a group the plan does not hold")?; + let owned = Shapes::build(&refs, &frame.shapes, group_tables)?; + let mut words = Vec::new(); + match batched_cap(&frame.config)? { + None => { + for (slot, shape) in owned.table_shapes().iter().enumerate() { + let table = tables.get(slot).ok_or("the proof is short of tables")?; + let before = words.len(); + push_table_words(&mut words, table); + if words.len() - before != table_words(shape) { + return Err(format!( + "table {} carries {} words, its shape {}", + group_tables[slot], + words.len() - before, + table_words(shape) + )); + } + } + } + Some(planning) => { + let argue = argue.ok_or("the proof is short of batched argues")?; + let shapes = owned.table_shapes(); + let expected = super::whir_batch::batched_words(&shapes, &owned.argue_plan(planning)); + super::whir_batch::push_batched_words(&mut words, argue); + if words.len() != expected { + return Err(format!( + "group {g}'s batched argue carries {} words, its shapes {expected}", + words.len() + )); + } + } + } + let layout = &frame.stack_layouts[g]; + let chain = plan.chain_shape(layout.n_stack()); + push_chains(&mut words, opening, layout.num_polys(), &chain)?; + let mut planned = plan.prepared.iter().filter(|p| p.group == g); + match (planned.next(), prepared) { + (Some(p), Some(opening)) => { + let shape = plan.chain_shape(p.layout.n_stack()); + push_chains(&mut words, opening, p.layout.num_polys(), &shape)?; + } + (Some(_), None) => return Err("the proof is short of prepared openings".to_string()), + (None, Some(_)) => return Err(format!("a prepared opening for group {g}, which has none")), + (None, None) => {} + } + if planned.next().is_some() { + return Err(format!("group {g} plans more than one prepared stack")); + } + Ok(words) +} + +/// One opening's chains: per chain (a run of [`ChainShape::chains`]) its final +/// value, then its rounds at the run's shape. +fn push_chains( + words: &mut Vec, + opening: &multilinear::stacked_eval::StackedProof, + polys: usize, + shape: &ChainShape, +) -> Result<(), String> { + let runs = shape.chains(polys); + if opening.polys.len() != runs.len() { + return Err(format!( + "an opening of {} chains where the layout's {polys} polynomials take {}", + opening.polys.len(), + runs.len() + )); + } + for (chain, run) in opening.polys.iter().zip(&runs) { + let shape = shape.run(run.len()); + words.push(super::word::ext_word(&chain.final_value)); + let before = words.len(); + super::whir_chain::try_push_round_words(words, &shape, chain)?; + if words.len() - before != super::whir_chain::RoundStorage::words(&shape) as usize { + return Err("a chain whose rounds are not its shape's".to_string()); + } + } + Ok(()) +} + +/// A prepared stack's roots, committed under the base's hash. +fn commit_prepared( + stack: crate::block_whir::GroupStack, + config: &multilinear::whir_chain::ChainConfig, + base: &stark::proof::options::BaseFormat, +) -> Result, crate::Error> { + crate::with_block_hash!(*base, |H| { Ok(commit_group::(stack, config)?.roots) }) +} + +/// Test only: the leaf's mutations of its commit-time answers, one at a time, +/// each of which a test must catch. Thread-local, so a test that sets one +/// moves no other test's program. +#[cfg(test)] +pub(crate) mod ood_faults { + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub(crate) enum Fault { + #[default] + None, + /// The front absorbs the answers after `(z, α, β)`. + AnswersAfterDraws, + /// The front leaves the first group's answers out of the transcript. + SkipFirstGroupsAnswers, + /// Each group's walk reads its answers from a fresh hint, not the + /// front's cells. + FreshAnswerHint, + } + + std::thread_local! { + static FAULT: std::cell::Cell = const { std::cell::Cell::new(Fault::None) }; + } + + pub(crate) fn get() -> Fault { + FAULT.with(std::cell::Cell::get) + } + + /// Runs `f` under `fault`, restoring the previous one after. + pub(crate) fn with(fault: Fault, f: impl FnOnce() -> R) -> R { + struct Restore(Fault); + impl Drop for Restore { + fn drop(&mut self) { + FAULT.with(|c| c.set(self.0)); + } + } + let _restore = Restore(FAULT.with(|c| c.replace(fault))); + f() + } +} + +#[cfg(test)] +mod pack_tests { + use multilinear::whir_chain::{ArgueFormat, ChainConfig, ChainFormat, GkrPack, GrindBits}; + + fn config(argue: ArgueFormat) -> ChainConfig { + ChainConfig { + log_blowup: 1, + log_folding: 1, + num_queries: 1, + grind: GrindBits::default(), + format: ChainFormat { + argue, + ..ChainFormat::DEFAULT + }, + } + } + + /// ★ The in-guest verifier plans a batched group by the format's two + /// constants — its cap and its packing — and reads no other. + #[test] + fn the_recursion_plans_by_the_formats_cap_and_packing() { + let production = super::batched_cap(&config(ArgueFormat::BATCHED)).expect("batched"); + assert_eq!( + production, + Some(super::BatchedPlanning { + bin_log_cells: 26, + pack: GkrPack::PRODUCTION + }) + ); + assert_eq!(super::batched_cap(&config(ArgueFormat::PerTable)), Ok(None)); + let pack = GkrPack::Off; + let packed = ArgueFormat::Batched { + bin_log_cells: 26, + pack, + }; + assert_eq!( + super::batched_cap(&config(packed)), + Ok(Some(super::BatchedPlanning { + bin_log_cells: 26, + pack + })) + ); + } +} diff --git a/prover/src/lfm/whir_block_tests.rs b/prover/src/lfm/whir_block_tests.rs new file mode 100644 index 000000000..4f7512379 --- /dev/null +++ b/prover/src/lfm/whir_block_tests.rs @@ -0,0 +1,4502 @@ +//! The no-epoch WHIR block's recursion ([`super::whir_block`]): the group +//! partition; the leaves over small block proofs, closing the bus between them; +//! a refusal per check a leaf or a node adds, each shown load-bearing by its +//! mutation; the tree to the top the plan derives; and the real block (box). + +use crate::block_whir::{ + self, BlockFormat, BlockOptions, BlockWhirProof, prove_block_whir, verify_block_whir, +}; +use crate::hash_pin::{BlockFamily, ProofFamily}; +use crate::tables::MaxRowsConfig; +use crate::tables::types::{FE, FEE}; +use crate::test_utils::E as FEE_FIELD; +use crate::test_utils::asm_elf_bytes; +use crate::zf_format::ZfFormat; +use multilinear::whir_chain::{ArgueFormat, StackVars}; +use stark::proof::options::ProofOptions; + +use super::block_node::{ + BlockBindings, BlockLayout, BlockNodeInputs, bind_and_publish_with, emit_block_node_with, +}; +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::per_table_aggregator::{DerivedChild, LegCells, hint_public_words, publics_arena}; +use super::per_table_aggregator_tests::{RealChild, child_arena_words}; +use super::proof::{LfmProof, aggregation_wrap_options, lfm_prove_under}; +use super::whir_block::{ + BLOCK_FAN_IN, BlockPartition, LeafChecks, WhirBlockPlan, artifacts_of, block_leaf_arena, + emit_share, group_arena_words, id_words, leaf_arena, leaf_partition, leaf_program_with, + out_halves, partition_groups, verify_block_tree_under, +}; +use super::whir_block_tree::{ + FailEmpty, FailUnpublished, GroupMsg, ProgramSlot, Published, TreeAt, TreeBudget, build_levels, + in_arrival_order, node_flow, prove_dataflow, publish_each, top_streams, tree_order, wait_all, +}; +use super::word::{LfmWord, base_word, word_as_ext}; + +/// A stack of 2^10 and two polynomials a group: a small program spans several +/// groups, which is the block's shape at a size a test can prove. +fn small_format() -> BlockFormat { + let mut zf = ZfFormat::DEFAULT; + zf.whir_stack = StackVars::new(10).expect("a valid stack"); + BlockFormat { + zf, + group_polys: 2, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: ArgueFormat::PerTable, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +/// `format` at blowup 4, for the tests whose blocks the leaf planner holds to a +/// leaf's rows. The test format's 2^10 stack makes a lone wide table (BITWISE's +/// 21 columns of 2^20 rows are 21 polynomials on their own), whose leaf rows at +/// blowup 2 pass a leaf's room: a shape production's 2^27 stack cannot form. +/// These tests check the leaves' and the tree's mechanics, which the rate does +/// not change. +fn at_blowup_4(format: BlockFormat) -> BlockFormat { + BlockFormat { + zf: ZfFormat { + whir_blowup: 4, + ..format.zf + }, + ..format + } +} + +/// The same with three polynomials a group: the same program packs into other +/// groups, so its statement — and every leaf's state and id — differ. +fn other_format() -> BlockFormat { + BlockFormat { + group_polys: 3, + ..small_format() + } +} + +fn small_block(name: &str, format: &BlockFormat) -> (Vec, BlockWhirProof) { + small_block_at(name, format, block_whir::BLOCK_ECDAS_ROWS_LOG2) +} + +/// [`small_block`] with ECDAS cut at 2^`ecdas_rows_log2` rows. +fn small_block_at( + name: &str, + format: &BlockFormat, + ecdas_rows_log2: usize, +) -> (Vec, BlockWhirProof) { + small_block_cut(name, format, |o| o.ecdas_rows_log2 = ecdas_rows_log2) +} + +/// [`small_block`] with its options changed by `cut` (the chunked tables' +/// heights). +fn small_block_cut( + name: &str, + format: &BlockFormat, + cut: impl FnOnce(&mut BlockOptions), +) -> (Vec, BlockWhirProof) { + let elf = asm_elf_bytes(name); + let opts = ProofOptions::default_test_options(); + let mut options = BlockOptions { + max_rows: MaxRowsConfig::small(), + keccak_rnd_rows_log2: 3, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: block_whir::BLOCK_COPY_ROWS_LOG2, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(3), + window_log2: None, + drop_streamed_ops: false, + layout_workers: 0, + layout_ahead: Some(2), + // Every table packed on a card, so a box run walks the narrow path at a + // test's size. + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + upload_ahead: true, + memlog: false, + // The rest laid out in waves of 1 MiB: several at a test's size. + finish_keccak_rnd_chunks: true, + finish_cuts: true, + finish_stream: true, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + pack_finished: true, + gpack: true, + regen: None, + phase_b_overlap: stark::multilinear_block::PhaseBOverlap::Off, + }; + cut(&mut options); + let proof = prove_block_whir(&elf, &[], &opts, format, &options) + .expect("the block proves") + .0; + assert!( + verify_block_whir(&proof, &elf, &opts, format).expect("verify runs"), + "the host verifier accepts the block" + ); + (elf, proof) +} + +fn plan_of( + elf: &[u8], + proof: &BlockWhirProof, + format: &BlockFormat, + leaves: Option, +) -> WhirBlockPlan { + WhirBlockPlan::derive( + elf, + &ProofOptions::default_test_options(), + format, + proof.statement(), + leaves, + ) + .expect("the plan derives") +} + +/// Leaf `k` executed over the proof's arena: its published words, or why it +/// did not execute. +fn run_leaf( + plan: &WhirBlockPlan, + proof: &BlockWhirProof, + k: usize, + checks: LeafChecks, +) -> Result, String> { + let program = leaf_program_with(plan, k, checks)?; + let arenas = block_leaf_arena(plan, proof, k)?; + assert_eq!( + program.arena_schema.lens, + vec![arenas[0].len() as u32], + "the leaf hints exactly the words its arena holds" + ); + execute( + &program, + &arenas, + &program.hasher(::SOCKET), + ) + .map(|e| e.public_words.iter().map(|(_, w)| *w).collect()) + .map_err(|e| format!("{e:?}")) +} + +fn leaves_words(plan: &WhirBlockPlan, proof: &BlockWhirProof) -> Vec> { + (0..plan.partition().num_leaves()) + .map(|k| run_leaf(plan, proof, k, LeafChecks::ALL).expect("an honest leaf executes")) + .collect() +} + +fn ext_of(word: &LfmWord) -> FEE { + word_as_ext(word).expect("an extension word") +} + +/// ★ What phase B hands over as each group's opening ends is the finished +/// proof's share of that group: built from the observer's groups alone, every +/// leaf's arena is the one the proof gives, under both argue formats, for a +/// streamed block (the production path). +/// +/// At blowup 4 ([`at_blowup_4`]): the test format's small stack makes a lone +/// wide table that production cannot form, and at blowup 2 its leaf rows +/// pass a leaf's room. The lean arm stays at blowup 2, the shipped rate: the +/// lean leaf holds the format's widest groups at 0.821 (BITWISE) and 0.802 +/// (CPU) of a leaf (`leaf_room_small_format_census`), so a rise past a leaf +/// refuses here first. +#[test] +#[ignore = "proves small blocks; box tier"] +fn the_groups_phase_b_hands_over_build_every_leafs_arena() { + for (format, name) in [ + (at_blowup_4(small_format()), "all_instructions_64"), + (at_blowup_4(small_batched()), "all_instructions_64"), + (at_blowup_4(small_batched()), "test_keccak"), + // At blowup 2, the shipped rate: the lean leaf holds this format's + // widest groups (0.82 of a leaf), where V1's do not. + (lean_format(small_format()), "all_instructions_64"), + ] { + let elf = asm_elf_bytes(name); + let opts = ProofOptions::default_test_options(); + let mut options = BlockOptions::production(); + options.max_rows = MaxRowsConfig::small(); + options.keccak_rnd_rows_log2 = 3; + options.drop_levels = multilinear::whir_commit::TreeDrop::uniform(3); + options.window_log2 = Some(4); + options.narrow = stark::multilinear_block::Narrowing::Card { min_cells: 0 }; + let seen = std::sync::Mutex::new(Vec::::new()); + let on_group = |opened: stark::multilinear_block::GroupOpened<'_, _, _>| { + seen.lock().expect("lock").push(GroupMsg { + group: opened.group, + roots: opened.roots.to_vec(), + commit_ood: opened.commit_ood.to_vec(), + tables: opened.tables.to_vec(), + argue: opened.argue.cloned(), + opening: opened.opening.clone(), + prepared: opened.prepared.cloned(), + at: 0.0, + }); + }; + let (proof, _) = block_whir::prove_block_whir_observed_groups( + &elf, + &[], + &opts, + &format, + &options, + &|_, _| {}, + &on_group, + ) + .expect("the block proves"); + let seen = seen.into_inner().expect("lock"); + assert_eq!( + seen.len(), + proof.groups.len(), + "{name}: one hand-over a group" + ); + assert!( + seen.iter().enumerate().all(|(g, m)| m.group == g), + "{name}: the groups in order" + ); + let plan = plan_of(&elf, &proof, &format, None); + let words: Vec> = seen + .iter() + .map(|m| { + group_arena_words( + &plan, + m.group, + &m.tables, + m.argue.as_ref(), + &m.opening, + m.prepared.as_ref(), + ) + .expect("the group's words") + }) + .collect(); + for k in 0..plan.partition().num_leaves() { + let streamed = leaf_arena( + &seen[0].roots, + &seen[0].commit_ood, + plan.partition() + .leaf(k) + .iter() + .map(|&g| words[g].clone()) + .collect(), + ); + assert_eq!( + streamed, + block_leaf_arena(&plan, &proof, k).expect("the arena"), + "{name} leaf {k}: the streamed arena is the proof's" + ); + } + // A group handed over with another group's opening is refused or + // differs: the arena is a function of each group's own share. + if seen.len() >= 2 { + let swapped = group_arena_words( + &plan, + 0, + &seen[0].tables, + seen[0].argue.as_ref(), + &seen[1].opening, + seen[0].prepared.as_ref(), + ); + assert!( + swapped.as_ref().map_or(true, |w| *w != words[0]), + "{name}: group 1's opening does not pass for group 0's" + ); + } + } +} + +// ============================== the partition ============================= + +#[test] +fn the_group_partition_is_heaviest_first_onto_the_least_loaded_leaf() { + // 9 → leaf 0, 7 → leaf 1, 5 → leaf 1 (7 < 9), 3 → leaf 0 (9 < 12), + // 1 → leaf 0 (12 = 12, the lower leaf). + let p = partition_groups(&[5, 9, 1, 7, 3], 2); + assert_eq!(p.leaves(), &[vec![1, 2, 4], vec![0, 3]]); + // Never more leaves than groups, and one leaf takes everything. + assert_eq!(partition_groups(&[5, 9], 7).num_leaves(), 2); + assert_eq!(partition_groups(&[5, 9, 1], 1).leaves(), &[vec![0, 1, 2]]); +} + +/// ★ G3: no leaf over the cap. A group over it is refused (a group is atomic +/// for a leaf); without a fixed count, leaves are added from `⌈Σ/cap⌉` while +/// the heaviest is over it; and the block's own costs (FAST 421's W3 plan) keep +/// today's three leaves, so the default tree is unchanged. +#[test] +fn the_leaf_partition_keeps_every_leaf_under_the_cap() { + let refused = leaf_partition(&[200, 10], None, 150); + assert!( + refused + .as_ref() + .is_err_and(|e| e.contains("group 0 costs 200")), + "a group over the cap must be refused, got {refused:?}" + ); + assert!(leaf_partition(&[200, 10], Some(2), 150).is_err()); + // ⌈300/150⌉ = 2 leaves would carry 200: one more leaf, 100 each. + let p = leaf_partition(&[100, 100, 100], None, 150).expect("partitions"); + assert_eq!(p.num_leaves(), 3, "{:?}", p.leaves()); + // A fixed count (the tests', never a statement's) is the caller's. + assert_eq!( + leaf_partition(&[100, 100, 100], Some(1), 150) + .expect("partitions") + .num_leaves(), + 1 + ); + // A cost partition over one block's group costs, at the retired RPX leaf + // cap of 279 k permutations (the function serves a fixed leaf count now). + let block = [ + 65825, 64356, 64405, 64474, 66559, 65146, 81574, 68482, 105991, + ]; + let p = leaf_partition(&block, None, 279_000).expect("partitions"); + assert_eq!(p.leaves(), partition_groups(&block, 3).leaves()); + assert_eq!(p.leaves(), &[vec![1, 5, 8], vec![0, 2, 6], vec![3, 4, 7]]); +} + +#[test] +fn the_group_partition_refuses_a_gap_an_overlap_and_an_empty_leaf() { + assert!(BlockPartition::new(vec![vec![0], vec![2]], 3).is_err()); + assert!(BlockPartition::new(vec![vec![0, 1], vec![1, 2]], 3).is_err()); + assert!(BlockPartition::new(vec![vec![0, 1, 2], vec![]], 3).is_err()); + assert!(BlockPartition::new(vec![vec![3]], 3).is_err()); + assert!(BlockPartition::new(vec![vec![2, 0], vec![1]], 3).is_ok()); +} + +// ================================ the leaves ============================== + +/// ★ Every leaf executes over an honest block proof — on the width-16 socket, +/// with no twelve-felt hash row — all of them publish the same id, state and +/// output — the plan's — and their shares cancel: the block's bus, checked +/// once over every table, closes across the leaves. +/// +/// At blowup 4 ([`at_blowup_4`]): the test format's small stack makes a lone +/// wide table that production cannot form, and at blowup 2 its leaf rows +/// pass a leaf's room. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn whir_block_leaves_execute_and_close_the_bus() { + let format = at_blowup_4(small_format()); + for (name, leaves) in [ + ("all_instructions_64", Some(3)), + ("test_commit_4", Some(2)), + ("test_keccak", None), + ("test_commit_4", Some(1)), + ] { + let (elf, proof) = small_block(name, &format); + let plan = plan_of(&elf, &proof, &format, leaves); + let layout = plan.child_layout(); + println!( + "WHIR BLOCK LEAVES {name}: {} groups, costs {:?}, partition {:?}, {} prepared", + plan.num_groups(), + plan.costs(), + plan.partition().leaves(), + plan.prepared().len() + ); + for k in 0..plan.partition().num_leaves() { + let program = plan.leaf_program(k).expect("the leaf emits"); + assert!(program.hash16(), "leaf {k} proves on the socket"); + } + let words = leaves_words(&plan, &proof); + let id = id_words(plan.id()); + let out: Vec = out_halves(&proof.public_output) + .into_iter() + .map(base_word) + .collect(); + let mut sum = FEE::zero(); + for w in &words { + assert_eq!(w.len(), layout.total()); + assert_eq!(w[layout.id(0)], id[0]); + assert_eq!(w[layout.id(1)], id[1]); + assert_eq!( + w[layout.state(0)], + words[0][layout.state(0)], + "one state across the leaves" + ); + for (i, half) in out.iter().enumerate() { + assert_eq!(w[layout.out_half(i)], *half); + } + sum += ext_of(&w[layout.sum()]); + } + assert_eq!(sum, FEE::zero(), "{name}: the leaves' shares cancel"); + } +} + +/// ★ W1: the leaf's bus share ([`emit_share`]). Under the inverse form (the +/// default) a zero `q` has no satisfying assignment whatever `p` is — +/// `q · inv = 1` — which is the host's `contribution() = None` refusal. The +/// former `ediv(p, q)` (`LFM_WHIR_SHARE_INVERSE=0`) refuses `q = 0` only when +/// `p ≠ 0`: at `p = q = 0` its constraint `q · out = p` holds for every `out` +/// (the executor writes 1), so the share is free — the gap the inverse form +/// closes. The process default is checked to be the inverse form. +#[test] +fn a_zero_denominator_has_no_satisfying_assignment() { + let run = |p: u64, q: u64, inverse: bool| { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let a = b.declare_arena(2); + let p_wire = b.hint_word(a, 0).as_ext(); + let q_wire = b.hint_word(a, 1).as_ext(); + let share = emit_share(&mut b, p_wire, q_wire, inverse); + b.public(share.as_cell()); + let program = compile(b.finish()); + let words = vec![ + super::word::ext_word(&FEE::from(p)), + super::word::ext_word(&FEE::from(q)), + ]; + execute(&program, &[words], &crate::hash_pin::BLOCK_HASHER).map(|_| ()) + }; + for inverse in [false, true] { + assert!(run(7, 3, inverse).is_ok(), "inverse {inverse}: 7/3"); + assert!(run(7, 0, inverse).is_err(), "inverse {inverse}: 7/0"); + } + assert!( + run(0, 0, true).is_err(), + "the inverse form must refuse p = q = 0" + ); + if std::env::var_os("LFM_WHIR_SHARE_INVERSE").is_none() { + assert!( + super::whir_block::share_inverse(), + "the default share is the inverse form" + ); + } + assert!( + run(0, 0, false).is_ok(), + "the default ediv(p, q) executes p = q = 0 (its share is free): the gap" + ); +} + +/// The production format with three polynomials a group: the dense fixture's +/// pages share groups with tables of their height. +fn dense_format() -> BlockFormat { + BlockFormat { + zf: ZfFormat::DEFAULT, + group_polys: block_whir::BLOCK_GROUP_POLYS, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: ArgueFormat::PerTable, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +fn dense_block_with( + deviations: &block_whir::Deviations, +) -> (Vec, BlockWhirProof, Vec) { + let elf = asm_elf_bytes("test_dense_pages"); + let roots = std::sync::Mutex::new(Vec::new()); + let proof = block_whir::prove_block_whir_observed_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &dense_format(), + &BlockOptions { + max_rows: MaxRowsConfig::default(), + keccak_rnd_rows_log2: 16, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: block_whir::BLOCK_COPY_ROWS_LOG2, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(3), + window_log2: None, + drop_streamed_ops: false, + layout_workers: 0, + layout_ahead: Some(2), + // Every table packed on a card, so a box run walks the narrow path at a + // test's size. + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + upload_ahead: true, + memlog: false, + // The rest laid out in waves of 1 MiB: several at a test's size. + finish_keccak_rnd_chunks: true, + finish_cuts: true, + finish_stream: true, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + pack_finished: true, + gpack: true, + regen: None, + phase_b_overlap: stark::multilinear_block::PhaseBOverlap::Off, + }, + deviations, + &|_, r| *roots.lock().expect("lock") = r.to_vec(), + ) + .expect("the block proves") + .0; + let roots = roots.into_inner().expect("lock"); + (elf, proof, roots) +} + +/// ★ A leaf refuses a prepared block opened at another table's point (a valid +/// opening of its stack, of the wrong claim), and with the prepared openings +/// left out (the mutation) it executes. +/// +/// Then stacks committed wrongly, each opened consistently — two pages' blocks +/// swapped, a page's block holding the other page's columns, DECODE's block +/// another program's: +/// - refused by a leaf of the verifier's plan (it absorbs the stacks it +/// derives); +/// - refused by a leaf of a plan given the prover's own roots (the opening no +/// longer matches the tables' claims); +/// - executing only with those roots trusted AND the openings left out — the +/// two together bind the prepared columns to the program. +#[test] +#[ignore = "proves the dense fixture and executes its leaf; box tier"] +fn a_block_leaf_refuses_wrong_prepared_openings() { + use block_whir::{Deviations, StackTamper}; + let format = dense_format(); + let opts = ProofOptions::default_test_options(); + let skip = LeafChecks { + prepared: false, + ..LeafChecks::ALL + }; + let (elf, honest, _) = dense_block_with(&Deviations::default()); + let plan = plan_of(&elf, &honest, &format, Some(1)); + let tables: Vec = plan + .prepared() + .iter() + .flat_map(|p| p.tables.iter().map(|&(_, air, _)| air)) + .collect(); + assert!(tables.len() >= 3, "DECODE and two pages: {tables:?}"); + run_leaf(&plan, &honest, 0, LeafChecks::ALL).expect("the honest leaf executes"); + + // AIR indices: DECODE is the lowest prepared table, then the pages. + let mut pages: Vec = tables.clone(); + pages.sort_unstable(); + let (a, b) = (pages[1], pages[2]); + let group = |t: usize| { + honest + .groups + .iter() + .position(|g| g.contains(&(t as u32))) + .expect("in a group") + }; + let height = honest.table_num_vars[a]; + let other = honest.groups[group(a)] + .iter() + .map(|&t| t as usize) + .find(|&t| t != a && honest.table_num_vars[t] == height) + .expect("a table of the page's height in its group"); + let (_, wrong, _) = dense_block_with(&Deviations { + prepared_points: vec![(a, other)], + ..Default::default() + }); + assert!( + run_leaf(&plan, &wrong, 0, LeafChecks::ALL).is_err(), + "wrong point" + ); + assert!( + run_leaf(&plan, &wrong, 0, skip).is_ok(), + "wrong point, mutation" + ); + + let mut stack_tampers = vec![ + ( + "columns of another page", + Deviations { + prepared_stack: vec![StackTamper::ColumnsOf { table: a, from: b }], + ..Default::default() + }, + ), + ( + "another program's DECODE", + Deviations { + other_prepared: true, + ..Default::default() + }, + ), + ]; + if group(a) == group(b) { + stack_tampers.push(( + "two pages swapped", + Deviations { + prepared_stack: vec![StackTamper::Swap(a, b)], + ..Default::default() + }, + )); + } + for (name, deviations) in &stack_tampers { + let (_, wrong, prover_roots) = dense_block_with(deviations); + assert!( + run_leaf(&plan, &wrong, 0, LeafChecks::ALL).is_err(), + "{name}: verifier's roots" + ); + let trusting = WhirBlockPlan::derive_with( + &elf, + &opts, + &format, + wrong.statement(), + Some(1), + BLOCK_FAN_IN, + Some(&prover_roots), + ) + .expect("a plan over the prover's roots"); + assert!( + run_leaf(&trusting, &wrong, 0, LeafChecks::ALL).is_err(), + "{name}: prover's roots" + ); + assert!( + run_leaf(&trusting, &wrong, 0, skip).is_ok(), + "{name}: the prover's roots trusted and the openings left out admit it" + ); + } +} + +// ================================= the nodes ============================== + +/// A node's binding step over HINTED child words: the checks and publishes +/// `emit_block_node` runs after verifying its children, with the verification +/// left out so each check can be driven on its own. +fn bindings_program( + layout: &BlockLayout, + children: usize, + top: bool, + checks: BlockBindings, +) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arenas: Vec<_> = (0..children) + .map(|_| b.declare_arena(8 * layout.total() as u32)) + .collect(); + let zero = b.ext_const(&FEE::zero()); + let legs: Vec = arenas + .iter() + .map(|&a| LegCells { + publics: hint_public_words(&mut b, a, layout.total()), + z_alpha: (zero, zero), + }) + .collect(); + bind_and_publish_with(&mut b, &legs, layout, top, checks); + compile(b.finish()) +} + +fn run_bindings( + layout: &BlockLayout, + words: &[Vec], + top: bool, + checks: BlockBindings, +) -> Result, String> { + let program = bindings_program(layout, words.len(), top, checks); + let arenas: Vec> = words.iter().map(|w| publics_arena(w)).collect(); + execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .map(|e| e.public_words.iter().map(|(_, w)| *w).collect()) + .map_err(|e| format!("{e:?}")) +} + +/// ★ Over REAL leaves' published words, every cross-child check refuses its +/// tamper and is the one refusing: with that check removed the same words bind. +/// Then the two block-level refusals: a leaf of another block's proof (another +/// statement, so another id and state), and a carrier that subtracted nothing +/// (the shares no longer cancel at the top). +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn the_node_bindings_refuse_each_tamper_over_real_leaves() { + let format = small_format(); + let (elf, proof) = small_block("test_commit_4", &format); + let plan = plan_of(&elf, &proof, &format, Some(2)); + let layout = plan.child_layout(); + let honest = leaves_words(&plan, &proof); + let top = run_bindings(&layout, &honest, true, BlockBindings::ALL).expect("honest leaves bind"); + assert_eq!( + top[..], + honest[0][..layout.total() - 1], + "the top republishes the claim" + ); + + for (check, at) in [ + ("id", layout.id(1)), + ("state", layout.state(0)), + ("out", layout.out_half(0)), + ("bus", layout.sum()), + ] { + let mut words = honest.clone(); + words[1][at][0] += FE::from(1u64); + assert!( + run_bindings(&layout, &words, true, BlockBindings::ALL).is_err(), + "`{check}`: the tamper is refused" + ); + assert!( + run_bindings(&layout, &words, true, BlockBindings::without(check)).is_ok(), + "`{check}`: with the check removed the tamper binds" + ); + } + + // A leaf of another proof of the same program, packed into other groups: + // another statement, so another id and another state. With its id words + // made this block's, only the state tells it apart, and the state check is + // what refuses it. + let other_format = other_format(); + let (_, other) = small_block("test_commit_4", &other_format); + let other_plan = plan_of(&elf, &other, &other_format, Some(2)); + assert_ne!(other_plan.id(), plan.id(), "the two statements differ"); + let foreign = leaves_words(&other_plan, &other); + assert_ne!(foreign[1][layout.state(0)], honest[1][layout.state(0)]); + let mut posing = foreign[1].clone(); + posing[layout.id(0)] = honest[0][layout.id(0)]; + posing[layout.id(1)] = honest[0][layout.id(1)]; + let mixed = vec![honest[0].clone(), posing]; + assert!(run_bindings(&layout, &mixed, false, BlockBindings::ALL).is_err()); + assert!( + run_bindings(&layout, &mixed, false, BlockBindings::without("state")).is_ok(), + "with the state check removed the foreign leaf binds" + ); + + // The carrier subtracting nothing: the shares sum to the target, not zero. + let mut no_target = honest.clone(); + no_target[plan.carrier()] = run_leaf( + &plan, + &proof, + plan.carrier(), + LeafChecks { + target: false, + ..LeafChecks::ALL + }, + ) + .expect("the leaf executes"); + assert!(run_bindings(&layout, &no_target, true, BlockBindings::ALL).is_err()); + assert!(run_bindings(&layout, &no_target, true, BlockBindings::without("bus")).is_ok()); +} + +// ================================= the tree =============================== + +/// One tree program proved: the proof (what a parent's arena and the final +/// check read), the child a parent emits over, and its derived shape. +struct Proved { + proof: LfmProof, + child: RealChild, + derived: DerivedChild, +} + +fn prove_tree_program( + label: &str, + program: &LfmProgram, + arenas: &[Vec], + words: usize, +) -> Result<(Proved, f64), String> { + let opts = aggregation_wrap_options(); + let t = std::time::Instant::now(); + let artifacts = artifacts_of(program, &opts); + let t_artifacts = t.elapsed().as_secs_f64(); + let derived = super::whir_block::derived_of(&artifacts, &opts, words)?; + let t = std::time::Instant::now(); + let proof = lfm_prove_under::(program, &artifacts, arenas, &opts) + .map_err(|e| format!("{label}: {e:?}"))?; + let t_prove = t.elapsed().as_secs_f64(); + let (child, verify) = + super::harvest::harvest_child_verified_under::(artifacts, opts, &proof)?; + println!( + " {label} TIMING: {} instructions · build_artifacts {t_artifacts:.2}s · prove {t_prove:.2}s · harvest verify {verify:.2}s", + program.instrs.len() + ); + Ok(( + Proved { + proof, + child, + derived, + }, + t_artifacts + t_prove, + )) +} + +/// The tree over `plan`'s leaves to its top. Returns the top's proof and, per +/// level (the leaves first), its wall and the sum of its programs' artifact and +/// prove seconds. +fn compose( + plan: &WhirBlockPlan, + proof: &BlockWhirProof, + tag: &str, +) -> Result<(LfmProof, Vec<(f64, f64)>), String> { + let words = plan.child_layout().total(); + let mut walls = Vec::new(); + let t = std::time::Instant::now(); + let mut busy = 0.0; + let mut level: Vec = Vec::with_capacity(plan.partition().num_leaves()); + for k in 0..plan.partition().num_leaves() { + let te = std::time::Instant::now(); + let program = plan.leaf_program(k)?; + let arenas = block_leaf_arena(plan, proof, k)?; + println!( + " {tag} LEAF {k}: groups {:?} · emitted in {:.2}s · arena {} words", + plan.partition().leaf(k), + te.elapsed().as_secs_f64(), + arenas[0].len() + ); + let (proved, secs) = + prove_tree_program(&format!("{tag} LEAF {k}"), &program, &arenas, words)?; + busy += secs; + level.push(proved); + } + let wall = t.elapsed().as_secs_f64(); + println!( + " {tag} LEVEL 0: {} leaves in {wall:.2}s (artifacts + prove {busy:.2}s)", + level.len() + ); + walls.push((wall, busy)); + let levels = plan.levels(); + for (lv, arities) in levels.iter().enumerate() { + let top = lv + 1 == levels.len(); + let t = std::time::Instant::now(); + let mut busy = 0.0; + let mut next = Vec::with_capacity(arities.arities.len()); + let mut rest = level.into_iter(); + for (j, &a) in arities.arities.iter().enumerate() { + let kids: Vec = rest.by_ref().take(a).collect(); + let arenas: Vec> = kids + .iter() + .flat_map(|k| child_arena_words(&k.child)) + .collect(); + let derived: Vec<&DerivedChild> = kids.iter().map(|k| &k.derived).collect(); + let program = plan.node_program(&derived, top)?; + let (proved, secs) = prove_tree_program( + &format!("{tag} L{}N{j}{}", lv + 1, if top { " TOP" } else { "" }), + &program, + &arenas, + words, + )?; + busy += secs; + next.push(proved); + } + let wall = t.elapsed().as_secs_f64(); + println!( + " {tag} LEVEL {}: {} node(s) in {wall:.2}s (artifacts + prove {busy:.2}s)", + lv + 1, + next.len(), + ); + walls.push((wall, busy)); + level = next; + } + let [top] = <[Proved; 1]>::try_from(level).map_err(|l| format!("{} tops", l.len()))?; + Ok((top.proof, walls)) +} + +/// ★ ECDAS cut through the block's recursion: test_ecsm_multi with ECDAS in +/// 16-row tables (three at the default, the fourth of padding rows alone not +/// built; the 0xABCDEF call through the three) proves as a block +/// whose tree the verifier accepts. The same statement with one ECDAS table +/// stated over [`block_whir::BLOCK_ECDAS_MAX_VARS`] is refused before any +/// program is derived. +/// +/// At blowup 4 ([`at_blowup_4`]): the test format's small stack makes a lone +/// wide table that production cannot form, and at blowup 2 its leaf rows +/// pass a leaf's room. +#[test] +#[ignore = "proves block leaves and nodes over a small block; box tier"] +fn the_whir_block_tree_verifies_a_split_ecdas() { + super::device_permit::arm(1); + let format = at_blowup_4(small_format()); + let (elf, proof) = small_block_at("test_ecsm_multi", &format, 4); + assert_eq!(proof.table_counts.ecdas, 3, "three ECDAS tables"); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + let plan = plan_of(&elf, &proof, &format, None); + let (top, _) = compose(&plan, &proof, "ECDAS TREE").expect("the honest tree proves"); + let verify = |statement: block_whir::BlockStatement<'_>| { + verify_block_tree_under( + &elf, + &opts, + &format, + statement, + None, + BLOCK_FAN_IN, + &wrap, + &top, + ) + }; + verify(proof.statement()).expect("the block's verifier accepts the split ECDAS"); + + let mut tall = proof.statement().to_owned(); + let (_, range, cap) = block_whir::chunked_table_ranges(&tall.table_counts) + .into_iter() + .find(|(name, ..)| *name == "ECDAS") + .expect("the ECDAS range"); + tall.table_num_vars[range.start] = (cap + 1) as u8; + let refused = verify(tall.view()); + assert!( + refused.is_err(), + "an ECDAS table stated over its cap must be refused" + ); + println!( + "WHIR BLOCK TREE ECDAS SPLIT: {} groups, 3 ECDAS tables, accepted; an ECDAS over its cap \ + refused ({})", + plan.num_groups(), + refused.err().unwrap_or_default() + ); +} + +/// ★ KECCAK, ECSM and COPY cut through the block's recursion: +/// test_keccak_multi's three permutations and test_ecsm_multi's three scalar +/// multiplications, one row a table (three tables each at the default: the +/// three calls, the padding row's table not built), and test_copy_ecalls' +/// 17 COPY rows, one row a table (17 tables, calls cut inside), prove as +/// blocks whose trees the verifier derives and accepts. Each statement with +/// its first table stated over its cap ([`block_whir::BLOCK_KECCAK_MAX_VARS`], +/// [`block_whir::BLOCK_ECSM_MAX_VARS`], [`block_whir::BLOCK_COPY_MAX_VARS`]) +/// is refused before any program is derived. +/// +/// At blowup 4 ([`at_blowup_4`]): the test format's small stack makes a lone +/// wide table that production cannot form, and at blowup 2 its leaf rows +/// pass a leaf's room. +#[test] +#[ignore = "proves three small blocks and their trees; box tier"] +fn the_whir_block_tree_verifies_split_keccak_and_ecsm() { + super::device_permit::arm(1); + let format = at_blowup_4(small_format()); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + for (guest, name, tables) in [ + ("test_keccak_multi", "KECCAK", 3), + ("test_ecsm_multi", "ECSM", 3), + ("test_copy_ecalls", "COPY", 17), + ] { + let (elf, proof) = small_block_cut(guest, &format, |o| match name { + "KECCAK" => o.keccak_rows_log2 = 0, + "COPY" => o.copy_rows_log2 = 0, + _ => o.ecsm_rows_log2 = 0, + }); + let (_, range, cap) = block_whir::chunked_table_ranges(&proof.table_counts) + .into_iter() + .find(|(n, ..)| *n == name) + .expect("the table's range"); + assert_eq!(range.len(), tables, "{name}: one row a table"); + let plan = plan_of(&elf, &proof, &format, None); + let (top, _) = + compose(&plan, &proof, &format!("{name} TREE")).expect("the honest tree proves"); + let verify = |statement: block_whir::BlockStatement<'_>| { + verify_block_tree_under( + &elf, + &opts, + &format, + statement, + None, + BLOCK_FAN_IN, + &wrap, + &top, + ) + }; + verify(proof.statement()) + .unwrap_or_else(|e| panic!("the block's verifier accepts the split {name}: {e}")); + + let mut tall = proof.statement().to_owned(); + tall.table_num_vars[range.start] = (cap + 1) as u8; + let refused = verify(tall.view()); + assert!( + refused.is_err(), + "a {name} table stated over its cap must be refused" + ); + println!( + "WHIR BLOCK TREE {name} SPLIT: {} groups, {tables} {name} tables, accepted; a {name} \ + over its cap refused ({})", + plan.num_groups(), + refused.err().unwrap_or_default() + ); + } +} + +/// The verifier's derive windows: at most `n` items at once, every item once, +/// in order, and the first error stops the level. +#[test] +fn a_derive_window_holds_its_bound_and_its_order() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let items: Vec = (0..23).collect(); + for window in [None, Some(1), Some(2), Some(5), Some(64)] { + let inside = AtomicUsize::new(0); + let most = AtomicUsize::new(0); + let done = super::whir_block::map_in_windows(&items, window, |&i| { + let now = inside.fetch_add(1, Ordering::SeqCst) + 1; + most.fetch_max(now, Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(2)); + inside.fetch_sub(1, Ordering::SeqCst); + Ok(i * 3) + }) + .expect("no item fails"); + assert_eq!(done, items.iter().map(|i| i * 3).collect::>()); + if let Some(n) = window { + assert!(most.load(Ordering::SeqCst) <= n, "window {n}"); + } + } + let failed = super::whir_block::map_in_windows(&items, Some(4), |&i| { + if i == 9 { + Err(format!("item {i}")) + } else { + Ok(i) + } + }); + assert_eq!(failed, Err("item 9".to_string())); +} + +/// The derive's knobs read a positive window and `level`, and refuse nonsense +/// with an error, not a panic. +#[test] +fn the_derive_knobs_read_their_values_and_refuse_nonsense() { + use super::whir_block::{parse_derive_builds, parse_derive_hold}; + assert_eq!(parse_derive_builds(None), Ok(None)); + assert_eq!(parse_derive_builds(Some(" ")), Ok(None)); + assert_eq!(parse_derive_builds(Some("8")), Ok(Some(8))); + for bad in ["0", "-1", "eight"] { + assert!(parse_derive_builds(Some(bad)).is_err(), "{bad}"); + } + assert_eq!(parse_derive_hold(None), Ok(false)); + assert_eq!(parse_derive_hold(Some("level")), Ok(true)); + assert!(parse_derive_hold(Some("all")).is_err()); +} + +/// The leaf scale's knob reads nothing as the default (doubled from 10 +/// leaves), `off` as the single heights and a positive count as the +/// threshold, refuses anything else with an error, and the production format +/// holds what the environment says. +#[test] +fn the_leaf_scale_knob_reads_off_or_a_count() { + use super::whir_block::{LeafScale, leaf_scale_from_env, leaf_scale_setting}; + assert_eq!(LeafScale::DEFAULT, LeafScale::DoubledOneTop); + for unset in [None, Some(""), Some(" ")] { + assert_eq!( + leaf_scale_setting(unset), + Ok(LeafScale::DEFAULT), + "{unset:?}" + ); + } + for off in [Some("off"), Some(" off ")] { + assert_eq!(leaf_scale_setting(off), Ok(LeafScale::Fixed), "{off:?}"); + } + assert_eq!( + leaf_scale_setting(Some("12")), + Ok(LeafScale::DoubleFrom(12)) + ); + assert_eq!( + leaf_scale_setting(Some(" 10 ")), + Ok(LeafScale::DoubleFrom(10)) + ); + assert_eq!(leaf_scale_setting(Some("1")), Ok(LeafScale::DoubleFrom(1))); + for bad in ["0", "-1", "twelve", "on", "12 leaves", "OFF"] { + let err = leaf_scale_setting(Some(bad)).expect_err(bad); + assert!( + err.contains(super::whir_block::LEAF_DOUBLE_FROM_ENV), + "{err}" + ); + } + assert_eq!( + Ok(BlockFormat::production().leaf_scale), + leaf_scale_from_env(), + "the production format holds the environment's scale" + ); +} + +/// ★ The verifier's lean derive ([`WhirBlockPlan::derive_top_in`]: each +/// program and its artifacts dropped once its child's shape is derived, the +/// leaves in windows) derives the held derive's top ([`WhirBlockPlan:: +/// programs`]) byte for byte, at a window of 1, of 8 and the whole level, over +/// a tree with a node level below the top. +#[test] +#[ignore = "proves a small block's base; box tier"] +fn the_lean_derive_is_the_held_derives_top_at_every_window() { + let format = small_format(); + let (elf, proof) = small_block("test_commit_4", &format); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + // The group count is the statement's, whatever the leaf count. Under the + // chip bound (`None`) the plan refuses this block at blowup 2: its group 0 + // spends more lanes rows than a single leaf holds. + let groups = plan_of(&elf, &proof, &format, Some(1)).num_groups(); + let leaves = groups.min(5); + assert!( + leaves >= 3, + "{groups} groups: too few for a node level below the top" + ); + let plan = WhirBlockPlan::derive_with( + &elf, + &opts, + &format, + proof.statement(), + Some(leaves), + 2, + None, + ) + .expect("the plan derives"); + assert!(plan.levels().len() >= 2, "a node level below the top"); + let mut held = plan.programs(&wrap).expect("the held derive"); + let held_top = held + .levels + .pop() + .and_then(|mut level| level.pop()) + .expect("a top") + .artifacts; + for window in [Some(1), Some(8), None] { + let lean = plan.derive_top_in(&wrap, window).expect("the lean derive"); + assert_eq!( + lean, held_top, + "window {window:?}: the lean top is the held top" + ); + } + println!( + "LEAN DERIVE IDENTITY: {leaves} leaves, {} node levels, the top equal at windows 1, 8 and the whole level", + plan.levels().len() + ); +} + +/// ★ Under the lean derive the verifier still refuses: a plan with one leaf +/// over a tampered prepared root derives another top, against which the +/// honest top proof does not verify (the leaf case), and a top proof with one +/// published word changed is refused by the block's verifier (the node case), +/// at every window; the honest top verifies at every window. +#[test] +#[ignore = "proves block leaves and nodes over a small block; box tier"] +fn the_lean_derive_still_refuses_a_tampered_leaf_and_node() { + super::device_permit::arm(1); + let format = small_format(); + let (elf, proof) = small_block("test_commit_4", &format); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + let plan = plan_of(&elf, &proof, &format, Some(2)); + let (top, _) = compose(&plan, &proof, "LEAN").expect("the honest tree proves"); + let mut roots: Vec = plan + .prepared() + .iter() + .map(|p| (p.group, p.roots.clone())) + .collect(); + roots[0].1[0][0] ^= 1; + let bad = WhirBlockPlan::derive_with( + &elf, + &opts, + &format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + Some(&roots), + ) + .expect("a plan over a tampered root"); + for window in [Some(1), Some(8), None] { + let honest = plan.derive_top_in(&wrap, window).expect("the honest top"); + assert!( + super::proof::verify_against_artifacts_under::( + &honest, + &top.proof, + &top.public_words, + &wrap + ), + "window {window:?}: the honest top verifies" + ); + let other = bad + .derive_top_in(&wrap, window) + .expect("the tampered plan's top"); + assert_ne!(other.program_id, honest.program_id, "window {window:?}"); + assert!( + !super::proof::verify_against_artifacts_under::( + &other, + &top.proof, + &top.public_words, + &wrap + ), + "window {window:?}: the honest top does not verify against a tampered leaf's tree" + ); + } + let mut words = top.public_words.clone(); + words[0].1[0] += FE::from(1u64); + let node = LfmProof { + proof: top.proof.clone(), + public_words: words, + }; + assert!( + verify_block_tree_under( + &elf, + &opts, + &format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + &wrap, + &node, + ) + .is_err(), + "a top proof with one published word changed is refused" + ); + assert!( + verify_block_tree_under( + &elf, + &opts, + &format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + &wrap, + &top, + ) + .is_ok(), + "the honest top is accepted by the block's verifier" + ); +} + +/// ★ The tree over a small block proves to a top, and the block's verifier — +/// which derives the top program from the ELF and the statement, never from +/// the proof — accepts it. +/// +/// Two trees the plan would not derive: +/// - the same groups with the leaves' lists swapped (the carrier now holds the +/// other half): every group once and the bus closes, so it proves, and the +/// verifier refuses it; checked against its OWN top program it is accepted, +/// so the refusal is the derived program's identity; +/// - a group verified twice and another never: refused, at the latest by the +/// verifier. +#[test] +#[ignore = "proves block leaves and nodes over a small block; box tier"] +fn the_whir_block_tree_proves_to_the_derived_top() { + super::device_permit::arm(1); + let format = small_format(); + let (elf, proof) = small_block("test_commit_4", &format); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + let plan = plan_of(&elf, &proof, &format, Some(2)); + let (top, _) = compose(&plan, &proof, "TREE").expect("the honest tree proves"); + verify_block_tree_under( + &elf, + &opts, + &format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + &wrap, + &top, + ) + .expect("the block's verifier accepts the honest top"); + // The block's base is its proof family's: under RPX's the verifier refuses + // the honest top, and no RPX plan derives at all. + let rpx = BlockFormat { + zf: format.zf.with_base(stark::proof::options::BaseFormat::RPX), + ..format + }; + assert!( + verify_block_tree_under( + &elf, + &opts, + &rpx, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + &wrap, + &top, + ) + .is_err(), + "the top under an RPX base" + ); + assert!( + WhirBlockPlan::derive(&elf, &opts, &rpx, proof.statement(), Some(2)).is_err(), + "no plan under an RPX base" + ); + + // The prover's roots shortcut only the prover's side. A plan built from + // tampered ones holds other leaf programs, so another top; its leaves + // refuse the honest proof (every leaf absorbs the roots), so no tree can be + // built over them; and the plan built from the true roots is the + // verifier's own. + let roots: Vec = plan + .prepared() + .iter() + .map(|p| (p.group, p.roots.clone())) + .collect(); + let mut tampered = roots.clone(); + tampered[0].1[0][0] ^= 1; + let from = |roots: &[block_whir::PreparedRoots]| { + WhirBlockPlan::derive_with( + &elf, + &opts, + &format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + Some(roots), + ) + .expect("a plan over supplied roots") + }; + let honest_top = plan.derive_top(&wrap).expect("the top derives").program_id; + assert_eq!( + from(&roots).derive_top(&wrap).expect("top").program_id, + honest_top + ); + let bad = from(&tampered); + assert_ne!(bad.derive_top(&wrap).expect("top").program_id, honest_top); + for k in 0..bad.partition().num_leaves() { + assert!( + run_leaf(&bad, &proof, k, LeafChecks::ALL).is_err(), + "leaf {k} over tampered roots" + ); + } + + let groups = plan.num_groups(); + assert!(groups >= 2, "the block spans at least two groups"); + let lists = plan.partition().leaves().to_vec(); + let swapped = plan_of(&elf, &proof, &format, Some(2)).with_partition( + BlockPartition::new(vec![lists[1].clone(), lists[0].clone()], groups) + .expect("still a partition"), + ); + let (other, _) = + compose(&swapped, &proof, "SWAPPED").expect("a tree over another partition proves"); + assert!( + verify_block_tree_under( + &elf, + &opts, + &format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + &wrap, + &other + ) + .is_err(), + "a tree over another partition is refused" + ); + let own = swapped.derive_top(&wrap).expect("its own top derives"); + assert!( + super::proof::verify_against_artifacts_under::( + &own, + &other.proof, + &other.public_words, + &wrap + ), + "against its own top it verifies: the derived program's identity is what refuses it" + ); + + // A carrier that subtracts nothing: both leaves prove, and the top node + // cannot — unless its bus check is removed. + let words = plan.child_layout().total(); + let leaves: Vec = (0..2) + .map(|k| { + let checks = if k == plan.carrier() { + LeafChecks { + target: false, + ..LeafChecks::ALL + } + } else { + LeafChecks::ALL + }; + let program = leaf_program_with(&plan, k, checks).expect("the leaf emits"); + let arenas = block_leaf_arena(&plan, &proof, k).expect("the arena"); + prove_tree_program(&format!("NO-TARGET LEAF {k}"), &program, &arenas, words) + .expect("each leaf proves on its own") + .0 + }) + .collect(); + let derived: Vec<_> = leaves.iter().map(|l| l.derived.shape()).collect(); + let arenas: Vec> = leaves + .iter() + .flat_map(|l| child_arena_words(&l.child)) + .collect(); + let node = |checks: BlockBindings| -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(::WRAP); + emit_block_node_with( + &mut b, + &BlockNodeInputs { + children: &derived, + layout: plan.child_layout(), + top: true, + }, + checks, + ); + compile(b.finish()) + }; + assert!( + prove_tree_program("NO-TARGET TOP", &node(BlockBindings::ALL), &arenas, words).is_err(), + "the top refuses shares that do not cancel" + ); + assert!( + prove_tree_program( + "NO-TARGET TOP (no bus check)", + &node(BlockBindings::without("bus")), + &arenas, + words + ) + .is_ok(), + "with the bus check removed the same leaves close a top" + ); + + let skewed = plan_of(&elf, &proof, &format, Some(2)).with_partition( + BlockPartition::unvalidated(vec![vec![0], (0..groups - 1).collect()], groups), + ); + match compose(&skewed, &proof, "SKEWED") { + Err(e) => println!(" SKEWED: refused while proving: {e}"), + Ok((bad, _)) => assert!( + verify_block_tree_under( + &elf, + &opts, + &format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + &wrap, + &bad + ) + .is_err(), + "a tree that verifies a group twice and another never is refused" + ), + } +} + +/// The leaf query knob reads `lean` (or nothing) and `v1`, and refuses +/// anything else with an error; the production format holds what the +/// environment says. +#[test] +fn the_leaf_query_knob_reads_v1_or_lean() { + use super::whir_block::{LEAF_QUERY_ENV, leaf_query_from_env, leaf_query_setting}; + use super::whir_chain::LeafQuery; + for lean in [None, Some(""), Some("lean"), Some(" lean ")] { + assert_eq!(leaf_query_setting(lean), Ok(LeafQuery::Lean), "{lean:?}"); + } + assert_eq!(leaf_query_setting(Some(" v1 ")), Ok(LeafQuery::V1)); + for bad in ["Lean", "on", "1", "v2"] { + let err = leaf_query_setting(Some(bad)).expect_err(bad); + assert!(err.contains(LEAF_QUERY_ENV), "{err}"); + } + assert_eq!( + Ok(BlockFormat::production().leaf_query), + leaf_query_from_env(), + "the production format holds the environment's check" + ); +} + +/// The leaves check lean by default: the type's default, the knob unset, the +/// production format with the knob unset, and both block commands' flag +/// (`bin/cli`). `v1` stays a named choice, and every chain outside a block +/// leaf keeps the per-query check (`ChainShape::new`). +#[test] +fn the_leaves_check_lean_by_default() { + use super::whir_block::{LEAF_QUERY_ENV, leaf_query_setting}; + use super::whir_chain::{ChainShape, LeafQuery}; + assert_eq!(LeafQuery::default(), LeafQuery::Lean); + assert_eq!(leaf_query_setting(None), Ok(LeafQuery::Lean)); + assert_eq!(leaf_query_setting(Some("v1")), Ok(LeafQuery::V1)); + if std::env::var_os(LEAF_QUERY_ENV).is_none() { + assert_eq!(BlockFormat::production().leaf_query, LeafQuery::Lean); + } + let config = BlockFormat::production().chain_config(&[(1, 12)]); + assert_eq!(ChainShape::new(&config, 12).query, LeafQuery::V1); +} + +/// ★ The leaf scale is the verifier's, both ways, on a small block whose +/// groups each need a leaf of their own under a leaf cap of the heaviest +/// group's socket rows ([`WhirBlockPlan::derive_capped`]): `n` leaves at the +/// single heights. At [`LeafScale::DoubleFrom`]`(n)` the plan doubles them, +/// to fewer leaves; at `DoubleFrom(n + 1)` it keeps the single partition. +/// Both trees prove, and the block's verifier, which derives the partition +/// from the statement under ITS scale: +/// - refuses the doubled tree under the single heights, and the single tree +/// under `DoubleFrom(n)`: the cross-mode refusals; +/// - refuses the doubled tree under `DoubleFrom(n + 1)`: a prover cannot +/// double below the threshold; +/// - accepts each tree under its own scale, and the single tree under +/// `DoubleFrom(n + 1)`: below the threshold the tree is the single one. +/// +/// The refusals come first, each the one a verifier-side mutant turns red +/// (one that always doubles, one that never does, one that doubles from one +/// leaf fewer). Against its own derived top the doubled tree verifies: the +/// refusal is the derived program's identity. +/// +/// At blowup 4, the rate this block's shape is sized at: at blowup 2 the +/// small format's group 0 spends more lanes rows than a single leaf holds +/// (1,051,383 against 1,048,482 at Q 229), so every scale's partition refuses +/// the block. +#[test] +#[ignore = "proves two trees over a small block; box tier"] +fn the_leaf_scale_is_the_verifiers_both_ways() { + use super::whir_block::{LeafScale, verify_block_tree_capped}; + super::device_permit::arm(1); + let format = BlockFormat { + zf: ZfFormat { + whir_blowup: 4, + ..small_format().zf + }, + ..small_format() + }; + let (elf, proof) = small_block("test_commit_4", &format); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + let probe = plan_of(&elf, &proof, &format, Some(1)); + let (loads, front) = probe.chip_loads(); + let cap = front.hash + loads.iter().map(|l| l.hash).max().expect("groups"); + let at = |leaf_scale: LeafScale| BlockFormat { + leaf_scale, + ..format + }; + let plan_at = |scale: LeafScale| { + WhirBlockPlan::derive_capped( + &elf, + &opts, + &at(scale), + proof.statement(), + None, + BLOCK_FAN_IN, + None, + cap, + ) + .expect("the plan derives") + }; + let single = plan_at(LeafScale::Fixed); + let n = single.partition().num_leaves(); + assert!(n >= 2, "{n} leaves at a cap of {cap} socket rows"); + let doubled = plan_at(LeafScale::DoubleFrom(n)); + let m = doubled.partition().num_leaves(); + let bound = doubled.leaf_bound().expect("the rule's partition"); + assert!( + bound.doubled && bound.single_leaves == n && m < n, + "doubled from {n}: {m} leaves, {bound:?}" + ); + let below = plan_at(LeafScale::DoubleFrom(n + 1)); + assert_eq!(below.partition(), single.partition(), "below the threshold"); + println!( + " LEAF SCALE: {} groups · cap {cap} · {n} single leaves {:?} · {m} doubled {:?}", + single.num_groups(), + single.partition().leaves(), + doubled.partition().leaves() + ); + let (top_single, _) = compose(&single, &proof, "SINGLE").expect("the single tree proves"); + let (top_doubled, _) = compose(&doubled, &proof, "DOUBLED").expect("the doubled tree proves"); + let verify = |scale: LeafScale, top: &LfmProof| { + verify_block_tree_capped( + &elf, + &opts, + &at(scale), + proof.statement(), + BLOCK_FAN_IN, + &wrap, + top, + cap, + ) + }; + assert!( + verify(LeafScale::Fixed, &top_doubled).is_err(), + "the doubled tree under the single heights" + ); + assert!( + verify(LeafScale::DoubleFrom(n), &top_single).is_err(), + "the single tree under doubled leaves from {n}" + ); + assert!( + verify(LeafScale::DoubleFrom(n + 1), &top_doubled).is_err(), + "the doubled tree below the threshold {}", + n + 1 + ); + verify(LeafScale::Fixed, &top_single).expect("the single tree under its own scale"); + verify(LeafScale::DoubleFrom(n), &top_doubled).expect("the doubled tree under its own scale"); + verify(LeafScale::DoubleFrom(n + 1), &top_single) + .expect("below the threshold the single tree is the verifier's"); + let own = doubled.derive_top(&wrap).expect("its own top derives"); + assert!( + super::proof::verify_against_artifacts_under::( + &own, + &top_doubled.proof, + &top_doubled.public_words, + &wrap + ), + "against its own top the doubled tree verifies" + ); +} + +// ===================== the GKR slot packing through the tree ===================== + +/// ★ A block argued with its GKR slots packed (`GkrPack::Chunks`): its leaves +/// execute and their shares cancel — a cut table's chunk outputs added as one +/// fraction in-guest — and its tree proves to the top the packed verifier +/// derives. The packing is the verifier's: the unpacked verifier (`Off`) +/// refuses the packed top, and the packed verifier refuses the unpacked top +/// over the same program. (τ = 8 cuts tables of this small block.) +#[test] +#[ignore = "proves two trees over a small block; box tier"] +fn the_packed_block_proves_to_the_derived_top_and_only_there() { + use multilinear::whir_chain::GkrPack; + super::device_permit::arm(1); + let today = at_blowup_4(BlockFormat { + argue: ArgueFormat::BATCHED, + ..small_format() + }) + .with_pack(GkrPack::Off); + let packed = today.with_pack(GkrPack::Chunks { tree_log_cells: 8 }); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + let mut tops = Vec::new(); + for (tag, format) in [("OFF", today), ("PACKED", packed)] { + let (elf, proof) = small_block("test_keccak", &format); + let outputs: usize = proof.argues.iter().map(|a| a.bus_outputs.len()).sum(); + if tag == "PACKED" { + assert!( + outputs > proof.table_num_vars.len(), + "{outputs} bus outputs for {} tables: something is cut", + proof.table_num_vars.len() + ); + } else { + assert_eq!(outputs, proof.table_num_vars.len(), "one output a table"); + } + let plan = plan_of(&elf, &proof, &format, None); + let layout = plan.child_layout(); + let mut sum = FEE::zero(); + for w in &leaves_words(&plan, &proof) { + sum += ext_of(&w[layout.sum()]); + } + assert_eq!(sum, FEE::zero(), "{tag}: the leaves' shares cancel"); + let (top, _) = compose(&plan, &proof, tag).expect("the honest tree proves"); + verify_block_tree_under( + &elf, + &opts, + &format, + proof.statement(), + None, + BLOCK_FAN_IN, + &wrap, + &top, + ) + .unwrap_or_else(|e| panic!("{tag}: its own verifier accepts its top: {e}")); + println!( + " PACKING {tag}: {} tables · {outputs} bus outputs · {} groups · leaves {:?}", + proof.table_num_vars.len(), + plan.num_groups(), + plan.partition().leaves() + ); + tops.push((elf, proof, format, top)); + } + let under = |at: usize, format: &BlockFormat| { + let (elf, proof, _, top) = &tops[at]; + verify_block_tree_under( + elf, + &opts, + format, + proof.statement(), + None, + BLOCK_FAN_IN, + &wrap, + top, + ) + }; + assert!( + under(1, &today).is_err(), + "the unpacked verifier refuses the packed top" + ); + assert!( + under(0, &packed).is_err(), + "the packed verifier refuses the unpacked top" + ); +} + +// ===================== the lean query check over a block ===================== + +/// `format` with its leaves checking each chain's queries the lean way +/// ([`LeafQuery::Lean`](super::whir_chain::LeafQuery::Lean)). +fn lean_format(format: BlockFormat) -> BlockFormat { + BlockFormat { + leaf_query: super::whir_chain::LeafQuery::Lean, + ..format + } +} + +/// ★ Under the lean query check every leaf of a small block executes over the +/// honest proof, per polynomial and with grouped round-0 trees, and the +/// leaves' shares cancel; its leaf programs are not V1's (fewer `Select`s); +/// and a round-0 Merkle sibling or value tampered at query 1 is refused by the +/// leaf holding the group. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn lean_block_leaves_execute_and_refuse_a_tampered_witness() { + use super::instr::Instr; + use multilinear::whir_chain::RoundOpenings; + let indexed = indexed_format(crate::hash_pin::BLOCK_BASE, ArgueFormat::PerTable); + for (format, name) in [ + (small_format(), "test_commit_4"), + (indexed, "all_instructions_64"), + ] { + let lean = lean_format(format); + let (elf, proof) = small_block(name, &lean); + let plan = plan_of(&elf, &proof, &lean, Some(2)); + let words = leaves_words(&plan, &proof); + let layout = plan.child_layout(); + let sum = words + .iter() + .fold(FEE::zero(), |acc, w| acc + ext_of(&w[layout.sum()])); + assert_eq!(sum, FEE::zero(), "{name}: the leaves' shares cancel"); + let v1 = plan_of(&elf, &proof, &format, Some(2)); + let selects = |p: &WhirBlockPlan| { + let program = p.leaf_program(0).expect("the leaf emits"); + program + .instrs + .iter() + .filter(|i| matches!(i, Instr::Select { .. })) + .count() + }; + assert!( + selects(&plan) < selects(&v1), + "{name}: lean {} Selects against V1 {}", + selects(&plan), + selects(&v1) + ); + + let g = 0; + let k = (0..plan.partition().num_leaves()) + .find(|&k| plan.partition().leaf(k).contains(&g)) + .expect("a leaf holds the group"); + run_leaf(&plan, &proof, k, LeafChecks::ALL).expect("the honest leaf executes"); + for value in [false, true] { + let mut forged = proof.clone(); + match &mut forged.proof.columns[g].polys[0].rounds[0].openings { + RoundOpenings::Base(p) if value => p.current[1].values[0] += FE::one(), + RoundOpenings::Base(p) => p.current[1].proof.merkle_path[0][0] ^= 1, + RoundOpenings::BaseGrouped(p) if value => p.current[1][0].values[0] += FE::one(), + RoundOpenings::BaseGrouped(p) => p.current[1][0].proof.merkle_path[0][0] ^= 1, + RoundOpenings::Extension(_) => unreachable!("round 0 is base-valued"), + } + assert!( + run_leaf(&plan, &forged, k, LeafChecks::ALL).is_err(), + "{name}: a round-0 {} at query 1", + if value { "value" } else { "sibling" } + ); + } + } +} + +/// ★ The leaves' query check is the verifier's, both ways, on proved trees: a +/// tree proved under `Lean` is refused by a `V1` verifier and a `V1` tree by a +/// `Lean` one, and each verifies under its own check. Each refusal is the one +/// a verifier-side mutant turns red (one that always takes the lean leaf, one +/// that never does). +#[test] +#[ignore = "proves two trees over a small block; box tier"] +fn the_leaf_query_is_the_verifiers_both_ways() { + use super::whir_block::{LEAF_P1_CAP, verify_block_tree_capped}; + use super::whir_chain::LeafQuery; + super::device_permit::arm(1); + // At blowup 4 ([`at_blowup_4`]): at blowup 2 this small format's V1 + // groups pass a single leaf's room, so the rule's partition refuses. + let format = at_blowup_4(small_format()); + let (elf, proof) = small_block("test_commit_4", &format); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + let at = |leaf_query: LeafQuery| BlockFormat { + leaf_query, + ..format + }; + let plan_at = |query: LeafQuery| plan_of(&elf, &proof, &at(query), None); + let (v1, lean) = (plan_at(LeafQuery::V1), plan_at(LeafQuery::Lean)); + println!( + " LEAF QUERY: {} groups · V1 {:?} · lean {:?}", + v1.num_groups(), + v1.partition().leaves(), + lean.partition().leaves() + ); + let (top_v1, _) = compose(&v1, &proof, "V1").expect("the V1 tree proves"); + let (top_lean, _) = compose(&lean, &proof, "LEAN").expect("the lean tree proves"); + let verify = |query: LeafQuery, top: &LfmProof| { + verify_block_tree_capped( + &elf, + &opts, + &at(query), + proof.statement(), + BLOCK_FAN_IN, + &wrap, + top, + LEAF_P1_CAP, + ) + }; + assert!( + verify(LeafQuery::V1, &top_lean).is_err(), + "the lean tree under V1" + ); + assert!( + verify(LeafQuery::Lean, &top_v1).is_err(), + "the V1 tree under lean" + ); + verify(LeafQuery::V1, &top_v1).expect("the V1 tree under its own check"); + verify(LeafQuery::Lean, &top_lean).expect("the lean tree under its own check"); +} + +// ============================== the real block ============================ + +/// A node builder that stops leaves an error in every slot it had not +/// filled, so no level waits forever, and keeps what it published. +#[test] +fn a_stopped_node_builder_fails_every_empty_slot() { + let slots: Vec>> = vec![ + vec![Published::new(), Published::new()], + vec![Published::new()], + ]; + { + let _fail = FailUnpublished(&slots); + let _ = slots[0][1].0.set(Ok(7)); + } + // `get`, not `wait`: an empty slot must fail the test, not hang it. + assert_eq!(slots[0][1].0.get(), Some(&Ok(7))); + for (lv, j) in [(0, 0), (1, 0)] { + assert_eq!( + slots[lv][j].0.get(), + Some(&Err("the node builder stopped".to_string())), + "slot ({lv}, {j})" + ); + } +} + +/// [`build_levels`] over a toy tree whose programs are their texts (`L3` a +/// leaf, `N(..)` / `T(..)` a node over its children's): each node's build +/// sleeps longer the earlier it is in its level, so on a pool the level's +/// nodes tend to finish in reverse. On a pool, each even node of level 0 +/// finishes only after the odd node after it: the sleeps alone do not order +/// them, since a sleep can overrun the 4 ms between two neighbours (on a +/// loaded macOS host a 28 ms sleep took 38 ms, and the level then finished in +/// order). Returns every slot's content and the order the builds finished in. +#[allow(clippy::type_complexity)] +fn toy_tree( + leaves: usize, + arities: &[Vec], + threads: Option, + fail_at: Option<(usize, usize)>, +) -> ( + Vec>>>, + Vec<(usize, usize)>, +) { + use super::per_table_aggregator::Level; + let shape: Vec = arities + .iter() + .map(|a| Level { arities: a.clone() }) + .collect(); + let leaf_text: Vec = (0..leaves).map(|k| format!("L{k}")).collect(); + let leaf_refs: Vec<&String> = leaf_text.iter().collect(); + let slots: Vec>> = shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect(); + let programs: Vec>> = shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect(); + let finished = std::sync::Mutex::new(Vec::new()); + // Level 0's builds that have finished, for the even nodes to wait on. + let done = ( + std::sync::Mutex::new(vec![false; shape[0].arities.len()]), + std::sync::Condvar::new(), + ); + let pool = threads.map(|n| { + rayon::ThreadPoolBuilder::new() + .num_threads(n) + .build() + .expect("a pool") + }); + build_levels( + &shape, + &|i: usize| -> Result<&String, String> { Ok(leaf_refs[i]) }, + leaf_refs.len(), + &programs, + &slots, + |node: &String| node, + |_, _| Ok(()), + |lv, j, kids: &[&String], top, ()| { + let late = shape[lv].arities.len() - j; + std::thread::sleep(std::time::Duration::from_millis(4 * late as u64)); + if lv == 0 { + let (marks, landed) = &done; + let mut marks = marks.lock().expect("the marks"); + // Its odd neighbour is handed out next, to another thread, and + // never waits: no two even nodes wait at once. Bounded, so a + // neighbour that never comes fails the order check, not the + // run. + if threads.is_some() && j % 2 == 0 && j + 1 < marks.len() { + marks = landed + .wait_timeout_while(marks, std::time::Duration::from_secs(10), |m| { + !m[j + 1] + }) + .expect("the marks") + .0; + } + finished.lock().expect("the log").push((lv, j)); + marks[j] = true; + landed.notify_all(); + } else { + finished.lock().expect("the log").push((lv, j)); + } + if fail_at == Some((lv, j)) { + return Err(format!("node ({lv}, {j}) does not build")); + } + let kids: Vec<&str> = kids.iter().map(|k| k.as_str()).collect(); + let text = format!("{}({})", if top { "T" } else { "N" }, kids.join(",")); + Ok((text.clone(), text)) + }, + // The finish is where a node takes the card: never on a rayon worker. + // Its program is already published when it starts. + |lv, j, program: String| match rayon::current_thread_index() { + None if programs[lv][j].0.get() == Some(&Ok(program.clone())) => Ok(program), + None => Err(format!( + "node ({lv}, {j}) finished before its program was published" + )), + Some(w) => Err(format!("node ({lv}, {j}) finished on rayon worker {w}")), + }, + pool.as_ref(), + ); + // Every slot's program is published too, and agrees with its node (the + // error included). + for (lv, level) in slots.iter().enumerate() { + for (j, slot) in level.iter().enumerate() { + match (slot.0.get(), programs[lv][j].0.get()) { + (Some(Ok(node)), program) => { + assert_eq!(program, Some(&Ok(node.clone())), "program ({lv}, {j})") + } + (_, program) => assert!(program.is_some(), "program ({lv}, {j}) never published"), + } + } + } + let got = slots + .iter() + .map(|level| level.iter().map(|s| s.0.get().cloned()).collect()) + .collect(); + (got, finished.into_inner().expect("the log")) +} + +/// The median's tree shape: 23 leaves at fan-in 3. +fn median_arities() -> Vec> { + vec![vec![3, 3, 3, 3, 3, 3, 3, 2], vec![3, 3, 2], vec![3]] +} + +/// The toy tree's nodes over `leaves` leaves, level by level, as [`toy_tree`] +/// names them. +fn toy_want(leaves: usize, arities: &[Vec]) -> Vec>>> { + let group = + |kids: &[String], top: bool| format!("{}({})", if top { "T" } else { "N" }, kids.join(",")); + let mut want: Vec> = Vec::new(); + let mut below: Vec = (0..leaves).map(|k| format!("L{k}")).collect(); + for (lv, a) in arities.iter().enumerate() { + let top = lv + 1 == arities.len(); + let mut at = 0; + let level: Vec = a + .iter() + .map(|&n| { + at += n; + group(&below[at - n..at], top) + }) + .collect(); + want.push(level.clone()); + below = level; + } + want.into_iter() + .map(|l| l.into_iter().map(|p| Some(Ok(p))).collect()) + .collect() +} + +/// ★ The node pipe builds the serial builder's nodes, each in its own slot, +/// whatever order a level's emissions finish in, and finishes every node (where +/// it takes the card) off the rayon workers: the median's shape (23 leaves at +/// fan-in 3), on pools of 2 to 4 threads, where each level's emissions finish +/// out of order, and serially. +#[test] +fn the_node_pipe_builds_the_serial_builders_nodes_in_any_completion_order() { + let arities = median_arities(); + let want = toy_want(23, &arities); + for threads in [None, Some(2), Some(3), Some(4)] { + let (got, finished) = toy_tree(23, &arities, threads, None); + assert_eq!(got, want, "{threads:?} threads"); + let level0: Vec = finished + .iter() + .filter(|(lv, _)| *lv == 0) + .map(|(_, j)| *j) + .collect(); + let in_order = level0.windows(2).all(|w| w[0] < w[1]); + assert_eq!( + in_order, + threads.is_none(), + "{threads:?} threads: level 0 finished in {level0:?}" + ); + } +} + +/// A node build that fails leaves its error in its slot and an error in every +/// slot above it, promptly: no level waits for a node that will never come. +#[test] +fn a_failing_node_build_fails_every_node_above_it() { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let arities = vec![vec![3, 3, 3, 3, 3, 3, 3, 2], vec![3, 3, 2], vec![3]]; + let _ = tx.send(toy_tree(23, &arities, Some(3), Some((0, 2))).0); + }); + let got = rx + .recv_timeout(std::time::Duration::from_secs(20)) + .expect("the builder hung after a failed build"); + assert_eq!( + got[0][2], + Some(Err("node (0, 2) does not build".to_string())) + ); + for (lv, level) in got.iter().enumerate().skip(1) { + for (j, slot) in level.iter().enumerate() { + assert!( + matches!(slot, Some(Err(_))), + "slot ({lv}, {j}) above a failed build: {slot:?}" + ); + } + } +} + +/// [`build_levels`] at the median's shape over toy leaves (`L3`) that +/// [`publish_each`] builds one after another on another thread, each waited +/// for where it is read, as [`build_nodes`] waits for a leaf's artifacts. While +/// it builds the last leaf the publisher waits up to `poll` for the first +/// node's program; the leaf at `fail_leaf` does not build. Returns every node +/// slot's content and whether that program was out before the last leaf was +/// built — or `None` if the two did not finish within 20 s. +#[allow(clippy::type_complexity)] +fn toy_tree_over_published_leaves( + threads: Option, + together: bool, + fail_leaf: Option, + poll: std::time::Duration, +) -> Option<(Vec>>>, bool)> { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + use super::per_table_aggregator::Level; + let n = 23; + let shape: Vec = median_arities() + .into_iter() + .map(|arities| Level { arities }) + .collect(); + let node_slots = || -> Vec>> { + shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect() + }; + let (programs, slots) = (node_slots(), node_slots()); + let leaves: Vec> = (0..n).map(|_| Published::new()).collect(); + let pool = threads.map(|t| { + rayon::ThreadPoolBuilder::new() + .num_threads(t) + .build() + .expect("a pool") + }); + let early = std::sync::atomic::AtomicBool::new(false); + std::thread::scope(|scope| { + scope.spawn(|| { + let ids: Vec = (0..n).collect(); + publish_each( + &ids, + &leaves, + !together, + |_| true, + |k, _| { + std::thread::sleep(std::time::Duration::from_millis(2)); + if fail_leaf == Some(k) { + return Err(format!("leaf {k} does not build")); + } + if k + 1 == n { + let t = std::time::Instant::now(); + while programs[0][0].0.get().is_none() && t.elapsed() < poll { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + let out = programs[0][0].0.get().is_some(); + early.store(out, std::sync::atomic::Ordering::SeqCst); + } + Ok(format!("L{k}")) + }, + ) + }); + build_levels( + &shape, + &|i: usize| leaves[i].wait(), + n, + &programs, + &slots, + |node: &String| node, + |_, _| Ok(()), + |_, _, kids: &[&String], top, ()| { + let kids: Vec<&str> = kids.iter().map(|k| k.as_str()).collect(); + let text = format!("{}({})", if top { "T" } else { "N" }, kids.join(",")); + Ok((text.clone(), text)) + }, + |_, _, program: String| Ok(program), + pool.as_ref(), + ); + }); + let got: Vec>>> = slots + .iter() + .map(|level| level.iter().map(|s| s.0.get().cloned()).collect()) + .collect(); + let _ = tx.send((got, early.into_inner())); + }); + rx.recv_timeout(std::time::Duration::from_secs(20)).ok() +} + +/// ★ Leaves published one at a time build the tree all of them published at +/// once build, and nothing waits forever: serially and on pools of 1 to 4 +/// threads, where a node over leaves waits for its own on a pool worker. +#[test] +fn leaves_published_one_at_a_time_build_the_same_tree() { + let want = toy_want(23, &median_arities()); + for threads in [None, Some(1), Some(2), Some(4)] { + for together in [false, true] { + let got = + toy_tree_over_published_leaves(threads, together, None, std::time::Duration::ZERO); + let (got, _) = got.unwrap_or_else(|| { + panic!("{threads:?} threads, together {together}: the tree hung") + }); + assert_eq!(got, want, "{threads:?} threads, together {together}"); + } + } +} + +/// ★ A node over leaves is emitted once its own leaves are published, before +/// the last leaf is built; with every leaf held back to the end (the default +/// while every leaf's program is there), it cannot be. +#[test] +fn a_node_over_leaves_is_emitted_before_the_last_leaf_is_built() { + let want = toy_want(23, &median_arities()); + for threads in [None, Some(2), Some(4)] { + for (together, poll) in [(false, 10_000), (true, 200)] { + let poll = std::time::Duration::from_millis(poll); + let (got, early) = toy_tree_over_published_leaves(threads, together, None, poll) + .unwrap_or_else(|| { + panic!("{threads:?} threads, together {together}: the tree hung") + }); + assert_eq!(got, want, "{threads:?} threads, together {together}"); + assert_eq!( + early, !together, + "{threads:?} threads, together {together}: the first node's program \ + out before the last leaf was built" + ); + } + } +} + +/// A leaf that does not build fails the nodes over it and every node above, +/// and the leaves after it fail too: nothing waits forever. +#[test] +fn a_leaf_that_does_not_build_fails_the_nodes_over_it() { + for threads in [None, Some(4)] { + for together in [false, true] { + let (got, _) = toy_tree_over_published_leaves( + threads, + together, + Some(4), + std::time::Duration::ZERO, + ) + .unwrap_or_else(|| panic!("{threads:?} threads, together {together}: the tree hung")); + let at = format!("{threads:?} threads, together {together}"); + assert_eq!( + got[0][0], + Some(Ok("N(L0,L1,L2)".to_string())), + "{at}: the node before it" + ); + assert_eq!( + got[0][1], + Some(Err("leaf 4 does not build".to_string())), + "{at}: the node over it" + ); + for (lv, level) in got.iter().enumerate() { + for (j, slot) in level.iter().enumerate().skip(usize::from(lv == 0) * 2) { + assert!( + matches!(slot, Some(Err(_))), + "{at}: slot ({lv}, {j}) after a failed leaf: {slot:?}" + ); + } + } + } + } +} + +/// ★ [`publish_each`] holds what it built while each next input is there, +/// publishing it all after the last; before an input that is not there yet it +/// publishes what it holds, then each value as it is built. Here input 3 +/// arrives only once output 0 is out, as a leaf program the budget admits only +/// once the provers take the leaves before it: holding across it (the mutation: +/// ignore `ready`) waits forever, and the bounded wait fails the test instead. +#[test] +fn the_leaves_artifacts_are_published_before_a_wait_for_a_missing_program() { + let wait_for = |slot: &Published| -> Result { + let t = std::time::Instant::now(); + while slot.0.get().is_none() { + if t.elapsed() > std::time::Duration::from_secs(10) { + return Err("an input that never came".to_string()); + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + slot.wait().copied() + }; + // Every input there: one publication, after the last. + let inputs: Vec> = (0..6).map(|_| Published::new()).collect(); + for (k, input) in inputs.iter().enumerate() { + let _ = input.0.set(Ok(k)); + } + let out: Vec> = (0..6).map(|_| Published::new()).collect(); + let from = publish_each( + &inputs, + &out, + false, + |k| inputs[k].0.get().is_some(), + |k, input| { + assert!( + out[0].0.get().is_none(), + "output 0 out before the last build ({k})" + ); + wait_for(input).map(|v| v * 10) + }, + ); + assert_eq!(from, None); + let got: Vec<_> = out.iter().map(|s| s.0.get().cloned()).collect(); + assert_eq!(got, (0..6).map(|k| Some(Ok(k * 10))).collect::>()); + // Input 3 comes only once output 0 is out. + let inputs: Vec> = (0..6).map(|_| Published::new()).collect(); + for (k, input) in inputs.iter().enumerate().take(3) { + let _ = input.0.set(Ok(k)); + } + let out: Vec> = (0..6).map(|_| Published::new()).collect(); + let from = std::thread::scope(|scope| { + scope.spawn(|| { + if wait_for(&out[0]).is_ok() { + for (k, input) in inputs.iter().enumerate().skip(3) { + let _ = input.0.set(Ok(k)); + } + } + }); + publish_each( + &inputs, + &out, + false, + |k| inputs[k].0.get().is_some(), + |_, input| wait_for(input).map(|v| v * 10), + ) + }); + assert_eq!( + from, + Some(3), + "per-item publication begins at the missing input" + ); + let got: Vec<_> = out.iter().map(|s| s.0.get().cloned()).collect(); + assert_eq!(got, (0..6).map(|k| Some(Ok(k * 10))).collect::>()); +} + +/// The W3 tree's programs composed as `prove_tree_pipelined` composes them, +/// over toy programs (texts), at the median's shape: under `room`, the leaves +/// the budget admits now are emitted at once and the rest by the streaming +/// emitter as it admits them; the leaves' "artifacts" (their texts) published +/// by [`publish_each`] against "is the next program there"; the nodes admitted +/// in prove order on the builder's plain threads ([`build_levels`]); the +/// provers taking programs in tree order, claiming each at the take and +/// letting it go after its proof ([`prove_dataflow`], [`node_flow`]). Returns +/// every node's text, how many leaves came in at once, where per-leaf +/// publication began, the budget's summary and how many programs were never +/// let go — or `None` after 30 s. +#[allow(clippy::type_complexity)] +fn toy_budget_tree( + threads: Option, + room: super::program_budget::Room, + ahead: usize, +) -> Option<( + Vec>>>, + usize, + Option, + String, + usize, +)> { + use super::per_table_aggregator::Level; + use super::program_budget::{Permit, ProgramBudget, emit_ordered}; + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let n = 23; + let shape: Vec = median_arities() + .into_iter() + .map(|arities| Level { arities }) + .collect(); + let budget = ProgramBudget::with_reading(room, ahead, Box::new(|| None)); + let tb = TreeBudget::new(budget, &vec![1; n], &shape); + // Beside the base: the leaves admitted now, in order. + let permits: Vec = (0..n) + .map_while(|k| tb.budget.try_acquire(k, tb.leaf_estimate(k))) + .collect(); + let upto = permits.len(); + let leaves: Vec>> = + (0..n).map(|_| Published::new()).collect(); + for (k, mut permit) in permits.into_iter().enumerate() { + permit.emitted(1); + let _ = leaves[k] + .0 + .set(Ok(ProgramSlot::new(format!("L{k}"), Some(permit)))); + } + let built: Vec> = (0..n).map(|_| Published::new()).collect(); + let node_slots = || -> Vec>> { + shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect() + }; + let slots = node_slots(); + let programs: Vec>>> = shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect(); + let results: Vec>> = std::iter::once(n) + .chain(shape.iter().map(|l| l.arities.len())) + .map(|m| (0..m).map(|_| Published::new()).collect()) + .collect(); + let pool = threads.map(|t| { + rayon::ThreadPoolBuilder::new() + .num_threads(t) + .build() + .expect("a pool") + }); + let tb = &tb; + let from = std::thread::scope(|outer| { + if upto < n { + let leaves = &leaves; + outer.spawn(move || { + let _fail = FailEmpty(&leaves[upto..], "the leaf emitter stopped"); + let emit = |k: usize| -> Result, String> { + let mut permit = tb.budget.acquire(k, tb.leaf_estimate(k))?; + permit.emitted(1); + Ok(ProgramSlot::new(format!("L{k}"), Some(permit))) + }; + emit_ordered(upto..n, 4, &emit, &mut |k, slot| { + let emitted = slot.is_ok(); + let _ = leaves[k].0.set(slot); + emitted + }); + }); + } + outer.spawn(|| { + build_levels( + &shape, + &|i: usize| built[i].wait(), + n, + &programs, + &slots, + |node: &String| node, + |lv, j| { + tb.budget + .acquire(tb.node_order(lv, j), tb.node_estimate()) + .map(Some) + }, + |_, _, kids: &[&String], top, permit: Option| { + let kids: Vec<&str> = kids.iter().map(|k| k.as_str()).collect(); + let text = format!("{}({})", if top { "T" } else { "N" }, kids.join(",")); + let mut permit = permit; + if let Some(p) = permit.as_mut() { + p.emitted(1); + } + Ok((ProgramSlot::new(text.clone(), permit), text)) + }, + |_, _, text: String| Ok(text), + pool.as_ref(), + ) + }); + std::thread::scope(|scope| { + let artifacts = scope.spawn(|| { + publish_each( + &leaves, + &built, + false, + |k| leaves[k].0.get().is_some(), + |_, slot| Ok(slot.wait()?.get()?.program.clone()), + ) + }); + prove_dataflow( + &shape, + &tree_order(n, &shape), + &results, + 3, + false, + |at, kids: &[&Published]| -> Result { + if at.lv == 0 { + let slot = leaves[at.j].wait()?; + slot.claim(); + let program = slot.get()?.program.clone(); + built[at.j].wait()?; + std::thread::sleep(std::time::Duration::from_millis(1)); + slot.release(); + return Ok(program); + } + node_flow( + &programs, + &slots, + at, + true, + |slot: &ProgramSlot| -> Result { + slot.claim(); + let program = slot.get()?.program.clone(); + wait_all(kids)?; + slot.release(); + Ok(program) + }, + |program, _: &String| Ok(program), + ) + }, + ); + tb.budget.fail(); + artifacts.join().expect("the artifacts thread") + }) + }); + let got: Vec>>> = slots + .iter() + .map(|level| level.iter().map(|s| s.0.get().cloned()).collect()) + .collect(); + let kept = leaves + .iter() + .chain(programs.iter().flatten()) + .filter(|slot| slot.wait().is_ok_and(|p| p.get().is_ok())) + .count(); + let _ = tx.send((got, upto, from, tb.budget.summary(), kept)); + }); + rx.recv_timeout(std::time::Duration::from_secs(30)).ok() +} + +/// ★ No room at all (a one-byte budget; every program comes in among the next +/// `ahead` the provers will take): the tree still completes with the same +/// nodes, serially and on pools of 1 to 4 threads, ahead 1 and +/// [`super::program_budget::AHEAD`]. The leaves' artifacts go out before the +/// first missing program; holding them across it (or provers that never claim) +/// deadlocks, which the bounded run turns into a failure. With room, every leaf +/// comes in at once and the artifacts go out together, as before the budget. +#[test] +fn a_one_byte_budget_cannot_deadlock_the_w3_tree() { + use super::program_budget::{AHEAD, Room}; + let want = toy_want(23, &median_arities()); + for threads in [None, Some(1), Some(2), Some(4)] { + for ahead in [1, AHEAD] { + let (got, upto, from, summary, kept) = toy_budget_tree(threads, Room::Bytes(1), ahead) + .unwrap_or_else(|| panic!("{threads:?} threads, ahead {ahead}: the tree hung")); + assert_eq!(got, want, "{threads:?} threads, ahead {ahead}"); + assert_eq!(kept, 0, "{threads:?} threads: programs never let go"); + assert_eq!(upto, ahead, "{threads:?} threads: leaves in at once"); + // The provers claim the leaves they take before those leaves' + // artifacts are out, so a few more programs may come in first; at + // the latest, the leaf after the three in flight is missing. + assert!( + from.is_some_and(|f| (ahead..=ahead + 3).contains(&f)), + "{threads:?} threads, ahead {ahead}: per-leaf from {from:?}" + ); + assert!( + summary.contains(&format!("0 by room, 35 among the next {ahead}")), + "{threads:?} threads, ahead {ahead}: {summary}" + ); + } + } + let (got, upto, from, summary, kept) = + toy_budget_tree(Some(4), Room::Unbounded, AHEAD).expect("the roomy tree hung"); + assert_eq!(got, want); + assert_eq!(kept, 0, "programs never let go"); + assert_eq!( + (upto, from), + (23, None), + "every leaf at once, published together" + ); + assert!(summary.contains("35 by room, 0 among"), "{summary}"); +} + +/// One node's stamps in [`split_slot_flow`], in ms since the tree started: +/// when a worker took it, when its program was emitted, when the worker got +/// the program, when it executed and proved; and whose artifacts it proved +/// with, and when those were built. +#[derive(Clone, Debug)] +struct FlowStamps { + taken: u128, + emitted: u128, + got: u128, + exec: u128, + prove: u128, + artifacts: String, + ready: u128, +} + +/// The tree's split slots over a toy at the median's shape (23 leaves, fan-in +/// 3): [`build_levels`] emits each level's programs on a pool in reverse (the +/// last node first), publishes each program and then each node's artifacts +/// after a finish that holds "the card" — 120 ms for the first node to arrive, +/// 5 ms for the rest; [`prove_dataflow`] proves the leaves and runs each node +/// through [`node_flow`]. Returns every node's stamps, by level then index. +fn split_slot_flow(workers: usize) -> Vec> { + use super::per_table_aggregator::Level; + let shape: Vec = [vec![3, 3, 3, 3, 3, 3, 3, 2], vec![3, 3, 2], vec![3]] + .into_iter() + .map(|arities| Level { arities }) + .collect(); + let leaf_text: Vec = (0..23).map(|k| format!("L{k}")).collect(); + let leaf_refs: Vec<&String> = leaf_text.iter().collect(); + let node_slots = || -> Vec>> { + shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect() + }; + // A program is (its text, when emitted); a node is (its artifacts' name, + // when built). + let (programs, slots) = (node_slots(), node_slots()); + let results: Vec>> = std::iter::once(23) + .chain(shape.iter().map(|l| l.arities.len())) + .map(|n| (0..n).map(|_| Published::new()).collect()) + .collect(); + let t0 = std::time::Instant::now(); + let ms = || t0.elapsed().as_millis(); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .expect("a pool"); + std::thread::scope(|scope| { + scope.spawn(|| { + build_levels( + &shape, + &|i: usize| -> Result<&String, String> { Ok(leaf_refs[i]) }, + leaf_refs.len(), + &programs, + &slots, + |node: &(String, u128)| &node.0, + |_, _| Ok(()), + |lv, j, kids: &[&String], _, ()| { + let late = shape[lv].arities.len() - j; + std::thread::sleep(std::time::Duration::from_millis(3 * late as u64)); + let text = format!( + "N({})", + kids.iter() + .map(|k| k.as_str()) + .collect::>() + .join(",") + ); + Ok(((text.clone(), ms()), text)) + }, + |lv, j, _: String| { + let first = j + 1 == shape[lv].arities.len(); + let hold = if first { 120 } else { 5 }; + std::thread::sleep(std::time::Duration::from_millis(hold)); + Ok((format!("A{lv}.{j}"), ms())) + }, + Some(&pool), + ) + }); + prove_dataflow( + &shape, + &tree_order(23, &shape), + &results, + workers, + false, + |at, kids: &[&Published]| { + let taken = ms(); + if at.lv == 0 { + std::thread::sleep(std::time::Duration::from_millis(1)); + let now = ms(); + return Ok(FlowStamps { + taken, + emitted: 0, + got: taken, + exec: now, + prove: now, + artifacts: String::new(), + ready: 0, + }); + } + node_flow( + &programs, + &slots, + at, + true, + |program: &(String, u128)| { + let got = ms(); + wait_all(kids)?; + let exec = ms(); + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok((program.1, got, exec)) + }, + |(emitted, got, exec), node: &(String, u128)| { + Ok(FlowStamps { + taken, + emitted, + got, + exec, + prove: ms(), + artifacts: node.0.clone(), + ready: node.1, + }) + }, + ) + }, + ); + }); + results + .into_iter() + .skip(1) + .map(|level| { + level + .into_iter() + .map(|s| s.take().expect("every node proves")) + .collect() + }) + .collect() +} + +/// ★ A node gets its program as soon as it is emitted — not behind another +/// node's finish holding the card — and executes from it before its artifacts +/// are built, but proves only with its OWN artifacts, after they are built: at +/// 1 to 6 workers, without a hang. (Publishing a program after a sibling's +/// finish, waiting on another node's artifact slot, or waiting for the +/// artifacts before the execute each fails this.) +#[test] +fn a_node_proves_only_after_its_own_artifacts_and_executes_before_them() { + for workers in 1..=6 { + let got = within_20s(move || split_slot_flow(workers)); + let mut early = 0usize; + for (l, level) in got.iter().enumerate() { + for (j, node) in level.iter().enumerate() { + let at = format!("{workers} workers: node ({}, {j})", l + 1); + assert_eq!(node.artifacts, format!("A{l}.{j}"), "{at}"); + assert!( + node.prove >= node.ready, + "{at} proved at {} ms, before its artifacts at {} ms", + node.prove, + node.ready + ); + assert!( + node.got <= node.taken.max(node.emitted) + 40, + "{at} got its program at {} ms: emitted at {} ms, taken at {} ms", + node.got, + node.emitted, + node.taken + ); + early += usize::from(node.exec < node.ready); + } + } + assert!( + early > 0, + "{workers} workers: no node executed before its artifacts were built" + ); + } +} + +/// The top streams when its program is ready before its last child is proved +/// (as at the median), and executes whole when it is ready after (as at 1×). +#[test] +fn the_top_streams_only_while_a_child_is_unproved() { + let kids: Vec> = (0..3).map(|_| Published::new()).collect(); + let refs: Vec<&Published> = kids.iter().collect(); + assert!(top_streams(&refs), "ready before any child: streamed"); + let _ = kids[2].0.set(Ok(20)); + let _ = kids[0].0.set(Ok(0)); + assert!(top_streams(&refs), "ready before the last child: streamed"); + let _ = kids[1].0.set(Ok(10)); + assert!(!top_streams(&refs), "ready after the last child: whole"); + + let kids: Vec> = (0..2).map(|_| Published::new()).collect(); + let _ = kids[0].0.set(Ok(0)); + let _ = kids[1].0.set(Err("child 1 failed".to_string())); + let refs: Vec<&Published> = kids.iter().collect(); + assert!(!top_streams(&refs), "a failed child is done: whole"); + assert_eq!( + wait_all(&refs).map(|_| ()), + Err("child 1 failed".to_string()), + "and the whole path reports the failure" + ); +} + +/// The streamed top's children land in the order they are proved, each once, +/// and a failed child fails the top with its error, without a hang. +#[test] +fn a_streamed_top_takes_its_children_as_they_are_proved() { + let kids: Vec> = (0..3).map(|_| Published::new()).collect(); + let order = std::thread::scope(|scope| { + let kids = &kids; + scope.spawn(move || { + for c in [2usize, 0, 1] { + std::thread::sleep(std::time::Duration::from_millis(15)); + let _ = kids[c].0.set(Ok(10 * c as u32)); + } + }); + let refs: Vec<&Published> = kids.iter().collect(); + let mut order = Vec::new(); + in_arrival_order(&refs, |c, v| { + assert_eq!(*v, 10 * c as u32); + order.push(c); + Ok(()) + }) + .expect("every child lands"); + order + }); + assert_eq!(order, vec![2, 0, 1]); + + let kids: Vec> = (0..3).map(|_| Published::new()).collect(); + let _ = kids[1].0.set(Ok(10)); + let _ = kids[0].0.set(Err("child 0 failed".to_string())); + let refs: Vec<&Published> = kids.iter().collect(); + // Child 0's error is already published, so this returns at once. + let got = in_arrival_order(&refs, |_, _| Ok(())); + assert_eq!(got, Err("child 0 failed".to_string())); +} + +/// [`prove_dataflow`] over a toy tree at the median's shape (23 leaves, fan-in +/// 3, levels of 8, 3 and 1) whose programs are their texts (`L3` a leaf, +/// `N(..)` / `T(..)` a node over its children's): every program sleeps longer +/// the earlier it is in its level, so a level's programs finish in reverse. +/// Returns each slot's text (or error) and each program's (start, end), in +/// milliseconds since the tree started. +#[allow(clippy::type_complexity)] +fn toy_flow( + workers: usize, + barrier: bool, + fail_at: Option, + panic_at: Option, + order: impl Fn(&[TreeAt]) -> Vec, +) -> Vec>> { + use super::per_table_aggregator::Level; + let shape: Vec = [vec![3, 3, 3, 3, 3, 3, 3, 2], vec![3, 3, 2], vec![3]] + .into_iter() + .map(|arities| Level { arities }) + .collect(); + let results: Vec>> = std::iter::once(23) + .chain(shape.iter().map(|l| l.arities.len())) + .map(|n| (0..n).map(|_| Published::new()).collect()) + .collect(); + let width = |lv: usize| { + if lv == 0 { + 23 + } else { + shape[lv - 1].arities.len() + } + }; + let t0 = std::time::Instant::now(); + prove_dataflow( + &shape, + &order(&tree_order(23, &shape)), + &results, + workers, + barrier, + |at, kids: &[&Published<(String, u128, u128)>]| { + let kids = wait_all(kids)?; + let start = t0.elapsed().as_millis(); + let late = width(at.lv) - at.j; + std::thread::sleep(std::time::Duration::from_millis(3 * late as u64)); + if fail_at == Some(at) { + return Err(format!("({}, {}) does not prove", at.lv, at.j)); + } + assert!(panic_at != Some(at), "({}, {}) panics", at.lv, at.j); + let text = match at.lv { + 0 => format!("L{}", at.j), + _ => { + let kids: Vec<&str> = kids.iter().map(|k| k.0.as_str()).collect(); + let top = at.lv == shape.len(); + format!("{}({})", if top { "T" } else { "N" }, kids.join(",")) + } + }; + Ok((text, start, t0.elapsed().as_millis())) + }, + ); + results + .iter() + .map(|level| { + level + .iter() + .map(|s| s.0.get().cloned().unwrap_or(Err("empty".to_string()))) + .collect() + }) + .collect() +} + +/// Runs `f` on a thread of its own and gives up after 20 s: a deadlocked +/// scheduler fails the test instead of hanging it. +fn within_20s(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(std::time::Duration::from_secs(20)) + .expect("the tree's scheduler deadlocked (no result in 20 s)") +} + +/// [`prove_dataflow`] over the toy's shape where the last leaf, once taken, +/// holds its worker until some node starts (or 1 s passes): whether a node +/// started while that leaf was still proving. No clock decides it, only who +/// signalled whom. +fn a_node_starts_while_the_last_leaf_proves(workers: usize, barrier: bool) -> bool { + use super::per_table_aggregator::Level; + let shape: Vec = [vec![3, 3, 3, 3, 3, 3, 3, 2], vec![3, 3, 2], vec![3]] + .into_iter() + .map(|arities| Level { arities }) + .collect(); + let results: Vec>> = std::iter::once(23) + .chain(shape.iter().map(|l| l.arities.len())) + .map(|n| (0..n).map(|_| Published::new()).collect()) + .collect(); + let (started_tx, started_rx) = std::sync::mpsc::channel::<()>(); + let (started_tx, started_rx) = ( + std::sync::Mutex::new(started_tx), + std::sync::Mutex::new(started_rx), + ); + let overlapped = std::sync::atomic::AtomicBool::new(false); + prove_dataflow( + &shape, + &tree_order(23, &shape), + &results, + workers, + barrier, + |at, kids: &[&Published<()>]| { + wait_all(kids)?; + match (at.lv, at.j) { + (0, 22) => { + let rx = started_rx.lock().expect("one receiver"); + if rx.recv_timeout(std::time::Duration::from_secs(1)).is_ok() { + overlapped.store(true, std::sync::atomic::Ordering::SeqCst); + } + } + (0, _) => {} + _ => { + let _ = started_tx.lock().expect("never poisoned").send(()); + } + } + Ok(()) + }, + ); + overlapped.load(std::sync::atomic::Ordering::SeqCst) +} + +/// ★ Dataflow over the median's shape proves every program of the serial +/// level-by-level order, each in its own slot, whatever order they finish in +/// (each level in reverse), at 1 to 4 workers: and with two or more workers a +/// node proves while the level below is still proving, which the barrier +/// (the control) and a single worker never let it. The overlap is decided by +/// signals, not by timestamps: the last leaf waits for a node to start. +#[test] +fn dataflow_proves_the_level_by_level_programs_in_any_completion_order() { + let leaves: Vec = (0..23).map(|k| format!("L{k}")).collect(); + let mut want = vec![leaves]; + for (lv, arities) in [vec![3, 3, 3, 3, 3, 3, 3, 2], vec![3, 3, 2], vec![3]] + .iter() + .enumerate() + { + let top = lv == 2; + let below = want.last().expect("a level").clone(); + let mut at = 0; + let level: Vec = arities + .iter() + .map(|&n| { + at += n; + let kids = below[at - n..at].join(","); + format!("{}({kids})", if top { "T" } else { "N" }) + }) + .collect(); + want.push(level); + } + for workers in 1..=4 { + for barrier in [false, true] { + let got = within_20s(move || toy_flow(workers, barrier, None, None, |o| o.to_vec())); + let texts: Vec> = got + .iter() + .map(|l| { + l.iter() + .map(|r| r.as_ref().expect("proved").0.clone()) + .collect() + }) + .collect(); + assert_eq!(texts, want, "{workers} workers, barrier {barrier}"); + // Does a node start while the level below is still proving? + let early = + within_20s(move || a_node_starts_while_the_last_leaf_proves(workers, barrier)); + assert_eq!( + early, + workers >= 2 && !barrier, + "{workers} workers, barrier {barrier}: a node started before its level below ended" + ); + } + } +} + +/// The no-deadlock invariant: workers take the programs in topological order, +/// so the earliest unfinished program never waits on an untaken one — at any +/// number of workers, with every level finishing in reverse. (Taking the nodes +/// first deadlocks this test: every worker waits on children no one has taken.) +#[test] +fn dataflow_in_topological_order_never_deadlocks() { + for workers in 1..=6 { + let got = within_20s(move || toy_flow(workers, false, None, None, |o| o.to_vec())); + assert!(got.iter().flatten().all(Result::is_ok), "{workers} workers"); + } +} + +/// A failed or panicking prove leaves its error in its slot and an error in +/// every slot above it, promptly: no node waits for a child that will never +/// come. +#[test] +fn a_failed_program_fails_every_program_above_it_without_a_hang() { + for (fail, panic) in [ + (Some(TreeAt { lv: 0, j: 4 }), None), + (None, Some(TreeAt { lv: 1, j: 2 })), + ] { + let got = within_20s(move || toy_flow(3, false, fail, panic, |o| o.to_vec())); + let at = fail.or(panic).expect("one"); + assert!(got[at.lv][at.j].is_err(), "{at:?}: {:?}", got[at.lv][at.j]); + let mut j = at.j; + let arities = [vec![3, 3, 3, 3, 3, 3, 3, 2], vec![3, 3, 2], vec![3]]; + for lv in at.lv + 1..got.len() { + let mut start = 0; + j = arities[lv - 1] + .iter() + .position(|&n| { + start += n; + j < start + }) + .expect("a parent"); + assert!( + got[lv][j].is_err(), + "{at:?}: its ancestor ({lv}, {j}) proved" + ); + } + assert!( + got[0] + .iter() + .enumerate() + .all(|(k, r)| r.is_ok() || k == at.j), + "{at:?}: another leaf failed" + ); + } +} + +/// ★ W3's readout on a real block (box only, `--ignored`, `--features cuda`; +/// the block path reads no WHIR hash knob), through the one driver the CLI's +/// `prove-block` +/// runs too ([`super::whir_block_tree::prove_whir_block_tree`]), run the way a +/// prover would: +/// - the block proved in one proof at the production format; +/// - the moment its statement is final (after phase A), the plan derived and +/// the leaves' programs emitted on threads of their own, beside phase B; +/// - then the tree ([`prove_tree_pipelined`]): the leaves `W3_SIBLINGS` at a +/// time (3 by default), the nodes' programs built beside them. +/// +/// Off the clock, after the whole block: every proof verified against its +/// artifacts, the base verified on the host, and the top checked by the block's +/// verifier against the top program it derives from the ELF and the statement. +/// Reads `BLOCK_WHIR_ELF`, `BLOCK_WHIR_INPUT`, `W3_LEAVES` (the rule's count +/// when unset), `W3_SIBLINGS`, `W3_FAN_IN` ([`BLOCK_FAN_IN`] when unset) and +/// `W3_CROSS_SCALE` (unset: no cross-scale verify). +/// Prints `W3 …` lines. +#[test] +#[ignore = "a real block and its recursion: box only"] +fn the_whir_block_tree_on_a_real_block() { + use super::whir_block_tree::{StdoutSink, WhirTreeConfig, prove_whir_block_tree}; + let elf = std::fs::read(std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF")) + .expect("read the ELF"); + let input = std::fs::read(std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT")) + .expect("read the input"); + let cfg = WhirTreeConfig::from_env().unwrap_or_else(|e| panic!("{e}")); + // The block's base is its proof family's (D-P3B P3). `BLOCK_WHIR_BASE` + // was this harness's word for it: `p1`/`p1w` are still accepted and + // change nothing (an A/B against an older binary keeps equal environment + // bytes); any other value — `rpx` — is refused rather than ignored. + if let Ok(word) = std::env::var("BLOCK_WHIR_BASE") { + assert!( + matches!(word.trim(), "p1" | "p1w"), + "BLOCK_WHIR_BASE={word}: the block's base is its proof family's (p1)" + ); + } + println!("W3 BASE HASH: {:?}", cfg.format.zf.base.hash); + let run = + prove_whir_block_tree(&elf, &input, &cfg, &StdoutSink).unwrap_or_else(|e| panic!("{e}")); + let (proof, plan, proofs, early_out, base) = ( + &run.base_proof, + &run.plan, + &run.proofs, + &run.early, + run.times.base, + ); + let top = run.top().expect("a top"); + let (leaves, fan_in, argue, format, early_on) = + (cfg.leaves, cfg.fan_in, cfg.argue, cfg.format, cfg.early_on); + let opts = super::proof::block_base_options(); + let wrap = aggregation_wrap_options(); + // Off the clock: the early leaf's arena, built from the groups as phase B + // handed them over, is the one the finished proof gives. + match early_out { + Some((k, arena, complete_at, started, secs)) => { + let same = block_leaf_arena(plan, proof, *k).expect("the leaf's arena") == *arena; + println!( + "W3 EARLY LEAF: leaf {k} complete at {complete_at:.2}s · execute + fill from {started:.2}s for {secs:.2}s (base ended {base:.2}s) · arena = the proof's {same}" + ); + assert!(same, "the early leaf's arena is the finished proof's"); + } + None => println!("W3 EARLY LEAF: none (W3_LEAF_DURING_PHASE_B {early_on})"), + } + // The recursion proofs' domains and twiddles built before their card holds + // (`LFM_WARM_DOMAINS`): how many lengths, how long, and when the early + // leaf's were warm on the tree's clock. + println!("W3 WARM DOMAINS: {}", super::domain_warm::line()); + // The first prove's warm-up beside the base (`LFM_WARM_FIRST_PROVE`), and + // every prove's host preparation before it took the card, in both arms. + println!("W3 FIRST PROVE WARM: {}", super::first_prove_warm::line()); + // Off the clock: what the device declined, and the card's peaks as the + // ledger and the memory pool count them (not total − free, which reads a + // retaining pool's blocks as used). + #[cfg(feature = "cuda")] + let (reserved_peak, pool_peak) = ( + format!( + "{:.2}", + math_cuda::device::reserved_high_water() as f64 / (1u64 << 30) as f64 + ), + math_cuda::device::pool_used_bytes().map_or("-".to_string(), |(_, high)| { + format!("{:.2}", high as f64 / (1u64 << 30) as f64) + }), + ); + #[cfg(not(feature = "cuda"))] + let (reserved_peak, pool_peak) = ("-".to_string(), "-".to_string()); + #[cfg(feature = "cuda")] + let argue_fallbacks = math_cuda::device::device_fallbacks(); + #[cfg(not(feature = "cuda"))] + let argue_fallbacks = 0u64; + println!( + "W3 DEVICE: commit/encode to the host {} · device commit errors {} · openings on the host {} · argue device fallbacks {argue_fallbacks} · GKR tree refusals {} · fused declines {} · reservation peak {reserved_peak} GiB · pool live peak {pool_peak} GiB", + multilinear::gpu::host_fallbacks(), + multilinear::gpu::commit_errors(), + multilinear::gpu::open_host_fallbacks(), + multilinear::gpu::gkr_tree_refusals(), + multilinear::gpu_fused::fused_declines(), + ); + // Only under the coset round-0 order, so every other run's lines are + // today's: the round-0 openings served from the stack and the codewords + // re-encoded after all. + if crate::zf_format::ZfFormat::global().whir_leaves == multilinear::whir_chain::LeafOrder::Coset + { + // Then the openings that read the stack where it lies (the default; + // `LAMBDA_VM_WHIR_COSET_INPLACE=0` assembles), those asked to whose + // parts the kernels could not place (assembled after all), and their + // tiles folded to zero unread and read value by value. + let (zero, mixed) = multilinear::gpu::coset_inplace_tiles(); + println!( + "W3 WHIR COSET: round-0 leaves in the coset order · openings from the stack {} · re-encoded {} · in place {} ({}) · assembled after all {} · tiles zero {} · mixed {}", + multilinear::gpu::coset_openings(), + multilinear::gpu::coset_fallbacks(), + multilinear::gpu::coset_inplace_opens(), + if multilinear::gpu::coset_inplace_in_effect() { + "on" + } else { + "off" + }, + multilinear::gpu::coset_inplace_assembled(), + zero, + mixed, + ); + } + // Each table's input layer kept on the card without its padding + // (`LAMBDA_VM_ARGUE_INPUT_REAL`), and under `check` how many were compared + // round by round with the whole layer. + println!( + "W3 ARGUE INPUT REAL: {} · {}", + multilinear::gkr_input_real::input_real_in_effect(), + multilinear::gkr_input_real::counts().line() + ); + // The argue's round sums through a slot (`LAMBDA_VM_ARGUE_ROUND_SLOT`): + // rounds read from it, waits that fell back to the stream, today's second + // launches left, and under `check` the rounds compared with today's. + println!( + "W3 ARGUE ROUND SLOT: {} · {}", + multilinear::round_slot::round_slot_in_effect(), + multilinear::round_slot::counts_line() + ); + // The precomputed trees the run downloaded on cache misses, by path, and + // the staging pairs' waits (I-COPIES L1): the recursion's LFM proofs are + // where they happen. + #[cfg(feature = "cuda")] + println!( + "W3 TREE DOWNLOAD ({}): {} · staging pair waits {}", + if math_cuda::device::tree_download_staged() { + "staged" + } else { + "pageable" + }, + math_cuda::device::tree_download_totals().line(), + math_cuda::device::staging_totals().pair_waits + ); + // The base's stacked openings: the lean rounds in effect + // (`LAMBDA_VM_WHIR_LEAN_ROUNDS`, default 3) and how many openings took them. + println!( + "W3 WHIR LEAN: {} rounds · lean opens {} of {}", + multilinear::gpu::lean_rounds_in_effect(), + multilinear::gpu::lean_open_calls(), + multilinear::gpu::open_calls() + ); + // How those rounds were computed (`LAMBDA_VM_WHIR_LEAN_TILES`): from one + // pass over the message (the default, `wide`), or over every stack + // position in every round (`0`); under `check`, the rounds and the bound + // tables both ways, and how many differed. Then the most one opening held + // on the card, by form. + { + let (compared, differing) = multilinear::gpu::lean_tile_checks(); + let (tables, positions) = multilinear::gpu::lean_tile_table_checks(); + let mib = |bytes: u64| bytes as f64 / (1u64 << 20) as f64; + let [share, tile, check] = multilinear::gpu::lean_open_peaks(); + println!( + "W3 WHIR LEAN TILES: {} · tile opens {} of {} lean · rounds compared {compared} · differing {differing} · tables compared {tables} · positions differing {positions} · opening peak MiB share {:.1} tile {:.1} check {:.1}", + multilinear::gpu::lean_tiles_in_effect(), + multilinear::gpu::lean_tile_opens(), + multilinear::gpu::lean_open_calls(), + mib(share), + mib(tile), + mib(check) + ); + } + // Each chain's first successor: encoded from the bound message (the + // default) or folded (`LAMBDA_VM_WHIR_ENCODE_SUCCESSOR=0`), and how many + // folds ran. + println!( + "W3 WHIR SUCCESSORS: encode {} · {} encoded · {} folds ({} fused) · encodes the card declined {}", + if multilinear::gpu::encode_successor_in_effect() { + "on" + } else { + "off" + }, + multilinear::gpu::encoded_successors(), + multilinear::gpu::resident_fold_calls(), + multilinear::gpu::fused_fold_calls(), + multilinear::gpu::encode_declines() + ); + // Coset-ordered round-0 successors encoded on the host because the card + // declined the opening: 0 on a run whose openings all went to the card. + println!( + "W3 WHIR HOST ENCODES: {}", + multilinear::gpu::coset_host_encodes() + ); + // How each group's stacked polynomials were opened (`format.group`): one + // chain a polynomial, or one a run of up to four whose round 0 ran on the + // card, or fell to the host over the card's codewords (a fallback, each of + // its polynomials also among the openings on the host above); and the + // grouped successors folded and combined on the card rather than encoded. + println!( + "W3 WHIR GROUPED: {:?} · grouped chains on the card {} · on the host (fallback) {} · successors combined {}", + format.group, + multilinear::gpu::grouped_card_chains(), + multilinear::gpu::grouped_host_chains(), + multilinear::gpu::grouped_combined_successors() + ); + // The top proof's bytes: two runs prove the same ones under + // LAMBDA_VM_FIXED_TRACE_HASH=1 and LAMBDA_VM_DETERMINISTIC_GRIND=1. + let top_bytes = rkyv::to_bytes::(&top.proof).expect("serialize the top"); + println!( + "W3 TOP DIGEST: {} (blake3 of the top proof's {} bytes)", + &blake3::hash(&top_bytes).to_hex()[..32], + top_bytes.len() + ); + // What phase A fixes before any challenge or grind: the group roots, the + // partition, the tables' heights and counts, the pages and the public + // output. Two runs over the same tables placed alike give the same one + // under LAMBDA_VM_FIXED_TRACE_HASH=1 alone, so a block too large to prove + // twice under the deterministic grind (≈ 23 s of serial grinding a group in + // phase B) gates its tables' identity on this. + let mut statement = blake3::Hasher::new(); + for root in &proof.proof.roots { + statement.update(root); + } + for group in &proof.groups { + statement.update(&(group.len() as u64).to_le_bytes()); + for table in group { + statement.update(&table.to_le_bytes()); + } + } + statement.update(&proof.table_num_vars); + statement.update(format!("{:?}", proof.table_counts).as_bytes()); + statement.update(format!("{:?}", proof.runtime_page_ranges).as_bytes()); + statement.update(&proof.public_output); + statement.update(&(proof.num_private_input_pages as u64).to_le_bytes()); + println!( + "W3 STATEMENT DIGEST: {} (blake3 of {} group roots, {} groups, {} tables' heights and counts)", + &statement.finalize().to_hex()[..32], + proof.proof.roots.len(), + proof.groups.len(), + proof.table_num_vars.len() + ); + + // The tree's freed pages back to the OS before the verifiers, as + // `LAMBDA_VM_ALLOC_PURGE` names it (`tree` is not an `auto` point). + crate::alloc_purge::purge_point("tree"); + + // Off the clock: the harness's checks, the permit disarmed first. The + // verifier derives the tree's programs and artifacts as rayon jobs + // (`WhirBlockPlan::programs`), which must not take an armed card from a + // rayon worker (BIG 569); nothing proves beside them now. + super::device_permit::arm(1); + let t = std::time::Instant::now(); + for (i, (artifacts, lfm)) in proofs.iter().enumerate() { + assert!( + super::proof::verify_against_artifacts_under::( + artifacts, + &lfm.proof, + &lfm.public_words, + &wrap + ), + "tree proof {i} verifies against its own program" + ); + } + let ok = verify_block_whir(proof, &elf, &opts, &format).expect("the verifier runs"); + println!( + "W3 HARNESS VERIFY: {} tree proofs and the base {} in {:.2}s", + proofs.len(), + if ok { "ACCEPTED" } else { "REJECTED" }, + t.elapsed().as_secs_f64() + ); + assert!(ok, "the block proof must verify"); + // Off the clock: the top's rows and padded heights by chip, emitted again + // from its children's shapes, and its LANES rows against 2^19 — the chips a + // wider top must keep under the LFM proofs' bits bound. + { + use super::airs::{LFM_CHIP_NAMES, NUM_LFM_CHIPS}; + use super::whir_chain::ChipRows; + let children = plan.levels().last().map_or(1, |level| level.arities[0]); + let below = &proofs[proofs.len() - 1 - children..proofs.len() - 1]; + let words = plan.child_layout().total(); + let derived: Vec = below + .iter() + .map(|(artifacts, _)| super::whir_block::derived_of(artifacts, &wrap, words)) + .collect::>() + .expect("the top's children's shapes"); + let top = plan + .node_program(&derived.iter().collect::>(), true) + .expect("the top emits again"); + let rows = ChipRows::of(&top); + let heights = &proofs.last().expect("a top").0.log_heights; + let padded: Vec = (0..NUM_LFM_CHIPS) + .map(|c| format!("{}=2^{}", LFM_CHIP_NAMES[c], heights[c])) + .collect(); + println!( + "W3 TOP ROWS: {children} children · {rows:?} · lanes {} / 2^19 = {:.1} % · padded {}", + rows.lanes, + 100.0 * rows.lanes as f64 / (1u64 << 19) as f64, + padded.join(" ") + ); + } + // Off the clock: every tree proof's AIRs in proof order, their heights and + // the merged ones (`LAMBDA_VM_ZF_LFM_MERGE`), its bytes; then what one node + // spends verifying ONE child of each kind (the first leaf, and the last + // proof below the top when it is another node): the rows of a one-child + // node emitted over that child's shape, the socket first. + { + use super::airs::LfmAirs; + use super::whir_chain::ChipRows; + for (k, (artifacts, lfm)) in proofs.iter().enumerate() { + let Ok(airs) = LfmAirs::for_artifacts(artifacts, &wrap) else { + println!("W3 AIRS {k}: the artifacts' merge is refused"); + continue; + }; + let heights = LfmAirs::log_heights_in_air_order(artifacts).unwrap_or_default(); + let tables: Vec = airs + .air_refs() + .iter() + .zip(heights.iter().map(Some).chain(std::iter::repeat(None))) + .map(|(air, h)| match h { + Some(h) => format!("{}=2^{h}", air.name()), + None => air.name().to_string(), + }) + .collect(); + let bytes = rkyv::to_bytes::(&lfm.proof) + .map(|b| b.len()) + .unwrap_or(0); + println!( + "W3 AIRS {k}: {} sub-proofs · {bytes} bytes · merged {:?} · {}", + tables.len(), + artifacts.merge.plan(), + tables.join(" ") + ); + } + let words = plan.child_layout().total(); + let mut kinds = vec![(0usize, "leaf")]; + let children = plan.levels().last().map_or(1, |level| level.arities[0]); + if proofs.len() > plan.partition().num_leaves() + 1 { + kinds.push((proofs.len() - 1 - children, "node")); + } + // A census line, never a gate: an emission that fails says so and the + // run goes on to its verdicts. + for (k, kind) in kinds { + let one = super::whir_block::derived_of(&proofs[k].0, &wrap, words) + .and_then(|derived| plan.node_program(&[&derived], false)); + match one { + Ok(one) => { + let rows = ChipRows::of(&one); + println!( + "W3 CHILD COST {kind} (tree proof {k}): socket {} rows for one child · {rows:?}", + rows.hash + ); + } + Err(e) => println!("W3 CHILD COST {kind} (tree proof {k}): not emitted: {e}"), + } + } + } + let t = std::time::Instant::now(); + // The production verifier when the run is at its pinned presets (the + // public grouped verifier at the production packing), given the run's group + // opening (`LAMBDA_VM_WHIR_GROUP`) and GKR packing (`BLOCK_WHIR_ARGUE_PACK`) + // as the CLI's `verify-block --group --argue-pack` give them; the fixture form, at the run's own format, when + // a knob moved the tree or the format off them (the pinned verifier would + // refuse such a run: its format is constants, not the environment's). + let pack = match argue { + ArgueFormat::Batched { pack, .. } => pack, + ArgueFormat::PerTable => multilinear::whir_chain::GkrPack::Off, + }; + let pinned = BlockFormat::pinned_with(format.group).with_pack(pack); + let at_pinned = + format!("{format:?}") == format!("{pinned:?}") && super::whir_block::share_inverse(); + let verdict = if leaves.is_none() + && fan_in == BLOCK_FAN_IN + && argue == pinned.argue + && at_pinned + { + if pack == multilinear::whir_chain::GkrPack::PRODUCTION { + super::whir_block::verify_block_tree_grouped(&elf, proof.statement(), top, format.group) + } else { + super::whir_block::verify_block_tree_as( + &elf, + proof.statement(), + top, + &super::whir_block::verifier_presets(pinned), + ) + } + } else { + verify_block_tree_under( + &elf, + &opts, + &format, + proof.statement(), + leaves, + fan_in, + &wrap, + top, + ) + }; + println!( + "W3 TREE VERIFY: {} in {:.2}s (derives every program and its artifacts)", + if verdict.is_ok() { + "ACCEPTED" + } else { + "REJECTED" + }, + t.elapsed().as_secs_f64() + ); + verdict.expect("the block's verifier accepts the top"); + + // `W3_CROSS_SCALE=`, off the clock: the top verified again under the + // other leaf scale — the single heights when the run's format doubles + // from some threshold, doubled from `T` when it is fixed. Refused when + // that scale derives another partition; accepted when it derives the same + // one (below the threshold). + if let Ok(word) = std::env::var("W3_CROSS_SCALE") { + use super::whir_block::LeafScale; + let t: usize = word + .trim() + .parse() + .unwrap_or_else(|_| panic!("W3_CROSS_SCALE is a leaf count, got `{word}`")); + assert!( + leaves.is_none(), + "W3_CROSS_SCALE: a forced leaf count has no scale" + ); + let other = BlockFormat { + leaf_scale: match format.leaf_scale { + LeafScale::Fixed => LeafScale::DoubleFrom(t), + LeafScale::DoubleFrom(_) | LeafScale::DoubledOneTop => LeafScale::Fixed, + }, + ..format + }; + let clock = std::time::Instant::now(); + let other_plan = WhirBlockPlan::derive(&elf, &opts, &other, proof.statement(), None) + .expect("the other scale's plan derives"); + let (same, other_leaves) = ( + other_plan.partition() == plan.partition(), + other_plan.partition().num_leaves(), + ); + drop(other_plan); + let crossed = verify_block_tree_under( + &elf, + &opts, + &other, + proof.statement(), + None, + fan_in, + &wrap, + top, + ); + println!( + "W3 CROSS SCALE: the top under {}: {} · that scale's partition is {} ({other_leaves} leaves against {}) · as expected {} in {:.2}s", + other.leaf_scale, + if crossed.is_ok() { + "ACCEPTED" + } else { + "REJECTED" + }, + if same { "the same" } else { "another" }, + plan.partition().num_leaves(), + crossed.is_ok() == same, + clock.elapsed().as_secs_f64() + ); + assert_eq!( + crossed.is_ok(), + same, + "under {}: accepted exactly when it derives the run's partition", + other.leaf_scale + ); + } +} + +/// The real block's trace alone, as the streamed prove builds it under the +/// harness's options, each closed group drained instead of committed +/// ([`block_whir::trace_block_host_only`]): the host's own pace, with no card +/// to wait for. The words of [`the_whir_block_tree_on_a_real_block`]; prints +/// the build's stamps and a `BLOCK HOST ONLY` line. +#[test] +#[ignore = "a real block's trace: box only"] +fn the_whir_block_trace_host_only_on_a_real_block() { + use super::whir_block_tree::WhirTreeConfig; + let elf = std::fs::read(std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF")) + .expect("read the ELF"); + let input = std::fs::read(std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT")) + .expect("read the input"); + let cfg = WhirTreeConfig::from_env().unwrap_or_else(|e| panic!("{e}")); + let opts = super::proof::block_base_options(); + let (stamps, drained) = + block_whir::trace_block_host_only(&elf, &input, &opts, &cfg.format, &cfg.options) + .unwrap_or_else(|e| panic!("{e}")); + print!("{}", stamps.report()); + println!("BLOCK HOST ONLY: {drained}"); +} + +// ========================= the batched argue (N-4) ======================== + +/// [`small_format`] with each group's tables argued together at the format's +/// cap. +fn small_batched() -> BlockFormat { + BlockFormat { + argue: ArgueFormat::BATCHED, + ..small_format() + } +} + +/// ★ Under the batched format every leaf executes over an honest block proof, +/// publishing the plan's id, state and output, and their shares cancel: the +/// batched groups close the block's bus across the leaves. Each group costs the +/// leaf less than its tables argued one by one. +/// +/// At blowup 4 ([`at_blowup_4`]): the test format's small stack makes a lone +/// wide table that production cannot form, and at blowup 2 its leaf rows +/// pass a leaf's room. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn batched_block_leaves_execute_and_close_the_bus() { + let format = at_blowup_4(small_batched()); + for (name, leaves) in [ + ("all_instructions_64", Some(3)), + ("test_commit_4", Some(2)), + ("test_keccak", None), + ("test_commit_4", Some(1)), + ] { + let (elf, proof) = small_block(name, &format); + assert_eq!(proof.argues.len(), proof.groups.len()); + let plan = plan_of(&elf, &proof, &format, leaves); + let per_table = plan_of(&elf, &proof, &at_blowup_4(small_format()), leaves); + println!( + "WHIR BLOCK BATCHED LEAVES {name}: {} groups, costs {:?} (per table {:?}), partition {:?}", + plan.num_groups(), + plan.costs(), + per_table.costs(), + plan.partition().leaves(), + ); + for (g, (batched, each)) in plan.costs().iter().zip(per_table.costs()).enumerate() { + assert!(batched <= each, "{name} group {g}: {batched} > {each}"); + } + let layout = plan.child_layout(); + let words = leaves_words(&plan, &proof); + let id = id_words(plan.id()); + let mut sum = FEE::zero(); + for w in &words { + assert_eq!(w[layout.id(0)], id[0]); + assert_eq!(w[layout.id(1)], id[1]); + assert_eq!(w[layout.state(0)], words[0][layout.state(0)]); + sum += ext_of(&w[layout.sum()]); + } + assert_eq!(sum, FEE::zero(), "{name}: the leaves' shares cancel"); + // The format is the plan's: a per-table plan has no reading of a + // batched proof, and a batched plan none of a per-table one. + assert!(block_leaf_arena(&per_table, &proof, 0).is_err()); + } +} + +/// ★ A batched leaf refuses each forgery of its group's argue — a bus output, +/// a ladder step's half, a ladder round, a constraint round, a factor value — +/// and a tampered prepared opening, which executes with the prepared openings +/// left out (the mutation): the preprocessed and prepared checks stay in place +/// under the batched argue. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn a_batched_block_leaf_refuses_a_tampered_witness() { + let format = small_batched(); + let (elf, proof) = small_block("test_commit_4", &format); + let plan = plan_of(&elf, &proof, &format, Some(1)); + run_leaf(&plan, &proof, 0, LeafChecks::ALL).expect("the honest leaf executes"); + let one = FEE::one(); + // The tallest bin of group 0, whose ladder has a step with rounds. + let (bin, steps) = proof.argues[0] + .gkr + .iter() + .enumerate() + .map(|(i, l)| (i, l.layers.len())) + .max_by_key(|&(_, steps)| steps) + .expect("a ladder"); + assert!(steps >= 2, "a ladder with a step that has rounds"); + type Forge = fn(&mut BlockWhirProof, usize, usize); + let forgeries: [(&str, Forge); 5] = [ + ("bus output", |p, _, _| { + p.argues[0].bus_outputs[0].0 += FEE::one() + }), + ("ladder half", |p, bin, steps| { + p.argues[0].gkr[bin].layers[steps - 1].halves[0].p_lo += FEE::one() + }), + ("ladder round", |p, bin, steps| { + p.argues[0].gkr[bin].layers[steps - 1].sumcheck.rounds[0].evaluations[0] += FEE::one() + }), + ("constraint round", |p, _, _| { + p.argues[0].constraint.rounds[0].evaluations[0] += FEE::one() + }), + ("factor value", |p, _, _| { + p.argues[0].factor_values[0][0] += FEE::one() + }), + ]; + for (name, forge) in forgeries { + let mut forged = proof.clone(); + forge(&mut forged, bin, steps); + assert!( + run_leaf(&plan, &forged, 0, LeafChecks::ALL).is_err(), + "a forged {name}" + ); + } + let mut prepared = proof.clone(); + prepared.prepared[0].polys[0].final_value += one; + assert!(run_leaf(&plan, &prepared, 0, LeafChecks::ALL).is_err()); + let without = LeafChecks { + prepared: false, + ..LeafChecks::ALL + }; + assert!( + run_leaf(&plan, &prepared, 0, without).is_ok(), + "with the prepared openings left out the tamper executes" + ); +} + +/// ★ The batched block's tree proves to the top its plan derives, and the +/// format is the verifier's: the batched top is accepted under the batched +/// format and refused under the per-table one. +#[test] +#[ignore = "proves a block tree; box tier"] +fn the_batched_whir_block_tree_proves_to_the_derived_top() { + super::device_permit::arm(1); + let format = small_batched(); + let (elf, proof) = small_block("test_commit_4", &format); + let opts = ProofOptions::default_test_options(); + let wrap = aggregation_wrap_options(); + let plan = plan_of(&elf, &proof, &format, Some(2)); + let (top, _) = compose(&plan, &proof, "BATCHED TREE").expect("the honest tree proves"); + let under = |format: &BlockFormat| { + verify_block_tree_under( + &elf, + &opts, + format, + proof.statement(), + Some(2), + BLOCK_FAN_IN, + &wrap, + &top, + ) + }; + under(&format).expect("the batched verifier accepts the honest top"); + assert!( + under(&small_format()).is_err(), + "the per-table verifier refuses the batched top" + ); +} + +/// ★ The grind bits (`LAMBDA_VM_ZF_WHIR_GRIND_BITS`) are a verifier constant +/// on the block's recursion too: the statement, Q and the grind are the leaf +/// program's constants. Over a block ground at 18 bits the plan at 18 emits a +/// leaf that executes; the plan at 20 emits one that refuses the same proof, +/// and executes over a block ground at 20 (the control: the refusal is the +/// bits', not the leaf's). Every leaf runs on the family's socket, so no +/// refusal is the hasher's (REV-P3B14-A A4). Q moves with the bits (114 at 18, +/// 112 at 20), so the refusal may be the arena's layout before any grind is +/// checked; the line `GRIND LEAF REFUSAL:` says which. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn a_block_leaf_at_20_bits_refuses_a_block_ground_at_18() { + let at = |bits: u8| BlockFormat { + zf: ZfFormat { + whir_grind_bits: bits, + ..small_format().zf + }, + ..small_format() + }; + let (elf, proof) = small_block("test_commit_4", &at(18)); + let plan = plan_of(&elf, &proof, &at(18), Some(1)); + run_leaf(&plan, &proof, 0, LeafChecks::ALL).expect("the leaf at 18 bits executes"); + + let strict = plan_of(&elf, &proof, &at(20), Some(1)); + assert_ne!( + strict.id(), + plan.id(), + "the statement at 20 bits is another statement" + ); + let program = leaf_program_with(&strict, 0, LeafChecks::ALL).expect("the leaf at 20 emits"); + let socket = program.hasher(::SOCKET); + assert_eq!( + socket, + super::hash::HasherKind::Poseidon1W16, + "the leaf's socket" + ); + let refusal = match block_leaf_arena(&strict, &proof, 0) { + Err(e) => format!("the arena: {e}"), + Ok(arenas) if program.arena_schema.lens != vec![arenas[0].len() as u32] => format!( + "the arena's length: {} words for a leaf hinting {:?}", + arenas[0].len(), + program.arena_schema.lens + ), + Ok(arenas) => match execute(&program, &arenas, &socket) { + Err(e) => { + assert!( + !matches!( + e, + super::executor::LfmExecError::HasherRejected(_) + | super::executor::LfmExecError::Internal(_) + ), + "the leaf at 20 refused for another reason than the proof: {e:?}" + ); + format!("the execution: {e:?}") + } + Ok(_) => panic!("a leaf at 20 bits executed over a block ground at 18"), + }, + }; + println!("GRIND LEAF REFUSAL: {refusal}"); + // The control: the same leaf at 20 executes over a block ground at 20. + let (elf20, proof20) = small_block("test_commit_4", &at(20)); + let plan20 = plan_of(&elf20, &proof20, &at(20), Some(1)); + run_leaf(&plan20, &proof20, 0, LeafChecks::ALL) + .expect("the leaf at 20 executes over a block at 20"); +} + +/// ★ The base code's rate (`LAMBDA_VM_ZF_WHIR_BLOWUP`) is a verifier constant +/// on the block's recursion too: the statement, the domains, the tree depths +/// and Q are the leaf program's constants. Each way round, the plan at the +/// block's own blowup emits a leaf that executes over it, and the plan at the +/// other blowup is another statement whose leaf refuses the same proof — at its +/// arena (other path lengths and Q) or in its execution, never at the hasher +/// (every leaf runs on the family's socket). The line `RATE LEAF REFUSAL:` +/// says where. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn a_block_leaf_at_one_blowup_refuses_a_block_at_the_other() { + let at = |blowup: u8| BlockFormat { + zf: ZfFormat { + whir_blowup: blowup, + ..small_format().zf + }, + ..small_format() + }; + for (own, other) in [(2u8, 4u8), (4, 2)] { + let (elf, proof) = small_block("test_commit_4", &at(own)); + let plan = plan_of(&elf, &proof, &at(own), Some(1)); + run_leaf(&plan, &proof, 0, LeafChecks::ALL) + .unwrap_or_else(|e| panic!("the leaf at blowup {own} executes over its block: {e}")); + + let wrong = plan_of(&elf, &proof, &at(other), Some(1)); + assert_ne!( + wrong.id(), + plan.id(), + "the statement at blowup {other} is another statement" + ); + let program = + leaf_program_with(&wrong, 0, LeafChecks::ALL).expect("the leaf at the other blowup"); + let socket = program.hasher(::SOCKET); + let refusal = match block_leaf_arena(&wrong, &proof, 0) { + Err(e) => format!("the arena: {e}"), + Ok(arenas) if program.arena_schema.lens != vec![arenas[0].len() as u32] => format!( + "the arena's length: {} words for a leaf hinting {:?}", + arenas[0].len(), + program.arena_schema.lens + ), + Ok(arenas) => match execute(&program, &arenas, &socket) { + Err(e) => { + assert!( + !matches!( + e, + super::executor::LfmExecError::HasherRejected(_) + | super::executor::LfmExecError::Internal(_) + ), + "the leaf at blowup {other} refused for another reason than the \ + proof: {e:?}" + ); + format!("the execution: {e:?}") + } + Ok(_) => panic!("a leaf at blowup {other} executed over a block at blowup {own}"), + }, + }; + println!("RATE LEAF REFUSAL: a block at blowup {own}, a leaf at {other}: {refusal}"); + } +} + +/// I-GRIND2 instrument (not a gate): a block group's in-guest opening cost +/// ([`super::whir_stacked::stacked_verify_cost`], what the plan charges a +/// group for its stacked opening and each prepared one) at 20 grind bits +/// (Q 112) and at `LAMBDA_VM_ZF_WHIR_GRIND_BITS=18` (Q 114), per stack height +/// and polynomial count. The per-group deltas predict the W3 plan's costs and +/// leaf count under the arm; the argue's cost does not read Q. +/// +/// cargo test -p lambda-vm-prover --lib grind_bits_group_cost_census -- --ignored --nocapture +#[test] +#[ignore = "instrument: prints the group opening cost per setting, asserts nothing"] +fn grind_bits_group_cost_census() { + for bits in crate::zf_format::WHIR_GRIND_BITS { + let format = ZfFormat { + whir_grind_bits: bits, + ..ZfFormat::DEFAULT + }; + for (polys, n) in [1usize, 2, 3] + .into_iter() + .map(|p| (p, 27usize)) + .chain((15..=26).map(|n| (1, n))) + { + // Tables of 2^n cells, one per polynomial: at 27 the stack cap + // keeps them apart, below it one table is one polynomial. + let height = n.min(21); + let shapes = vec![(1usize << (n - height), height); polys]; + let config = crate::multilinear_prove::chain_config_under(&format, &shapes); + let layout = stark::multilinear_table::global_layout(&shapes, config.format.stack) + .expect("a layout"); + let (_, group_of) = super::whir_epoch::group_columns(&shapes); + let chain = super::whir_chain::ChainShape::new(&config, layout.n_stack()); + let cost = super::whir_stacked::stacked_verify_cost( + &layout, + &group_of, + &chain, + super::whir_transcript::SpongeEntry::fresh(), + ); + println!( + "GRINDGROUP bits={bits} Q={} n_stack={} polys={} perms={} instrs={}", + config.num_queries, + layout.n_stack(), + layout.num_polys(), + cost.perms(), + cost.operations(), + ); + } + } +} + +// ======================== the Poseidon1 base (S5) ========================= + +/// ★ The Poseidon1 leaves' front draws the host verifier's challenges: +/// `P1Transcript` over the statement (call by call), every root, every +/// commit-time answer, `z, α, β`, a group fork's first draw, and that group's +/// commit-time point (`multilinear_block::commit_ood_point`). The host side is +/// spelled out here, so a misplaced answer on either side is a mismatch: each +/// of the front's mutations — the answers after the draws, the first group's +/// answers left out — moves the draws off the host's. +#[test] +#[ignore = "executes a block leaf's front over a small block proof; box tier"] +fn the_p1_block_front_draws_the_hosts_challenges() { + let format = small_format(); + let (elf, proof) = small_block("test_commit_4", &format); + // The front is the same under either query check; each plan's own. + for format in [format, lean_format(format)] { + front_draws_the_hosts_challenges(&elf, &proof, &format); + } +} + +/// [`the_p1_block_front_draws_the_hosts_challenges`] under `format`. +fn front_draws_the_hosts_challenges(elf: &[u8], proof: &BlockWhirProof, format: &BlockFormat) { + use super::whir_block::ood_faults::{self, Fault}; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use crypto::fiat_shamir::p1_transcript::P1Transcript; + let (elf, format) = (elf.to_vec(), *format); + let plan = plan_of(&elf, proof, &format, Some(1)); + assert_eq!(plan.wrap_hash(), super::edsl::WrapHash::Poseidon1); + let g = plan.num_groups() - 1; + let mut arena: Vec = proof + .proof + .roots + .iter() + .map(super::algebraic_commit::commitment_to_digest) + .collect(); + assert_eq!(proof.commit_ood.len(), proof.proof.roots.len()); + arena.extend(proof.commit_ood.iter().map(super::word::ext_word)); + let draws = |fault: Fault| -> Vec { + let program = ood_faults::with(fault, || super::whir_block::front_program(&plan, g)); + assert!(program.hash16(), "the front's hash rows are the socket's"); + execute( + &program, + std::slice::from_ref(&arena), + &program.hasher(::SOCKET), + ) + .expect("the front executes") + .public_words + .iter() + .map(|(_, w)| ext_of(w)) + .collect() + }; + let machine = draws(Fault::None); + let opts = ProofOptions::default_test_options(); + let frame = block_whir::block_frame(proof.statement(), &elf, &opts, &format).expect("frame"); + let derived: Vec = plan + .prepared() + .iter() + .flat_map(|p| p.roots.iter().copied()) + .collect(); + let mut t = P1Transcript::new(); + block_whir::absorb_block( + &mut t, + &block_whir::block_statement_tag(&format.zf.base), + &elf, + &proof.public_output, + &proof.table_counts, + proof.num_private_input_pages, + &proof.runtime_page_ranges, + &proof.table_num_vars, + &frame.config, + &proof.groups, + ); + for root in proof.proof.roots.iter().chain(&derived) { + t.append_bytes(root); + } + for answer in &proof.commit_ood { + t.append_field_element(answer); + } + let mut host: Vec = (0..3).map(|_| t.sample_field_element()).collect(); + let mut fork = t.clone(); + fork.append_bytes(&(g as u64).to_le_bytes()); + host.push(fork.sample_field_element()); + let layout = &frame.stack_layouts[g]; + let root_at: usize = frame.stack_layouts[..g] + .iter() + .map(multilinear::stacking::StackedLayout::num_polys) + .sum(); + host.push( + stark::multilinear_block::commit_ood_point::( + g, + layout, + &proof.proof.roots[root_at..root_at + layout.num_polys()], + &frame.config, + ) + .expect("the group's point"), + ); + assert_eq!( + machine, host, + "{:?}: z, α, β, the fork's first draw and the group's point", + format.leaf_query + ); + for fault in [Fault::AnswersAfterDraws, Fault::SkipFirstGroupsAnswers] { + assert_ne!( + draws(fault)[..3], + host[..3], + "{:?} {fault:?}: the front's draws must move off the host's", + format.leaf_query + ); + } +} + +/// ★ Under either query check, every leaf of a small block executes over the +/// honest proof, and a commit-time answer altered — the first group's, a +/// middle one, the last — gives every leaf no execution: each absorbs every +/// answer in its front, and the group's chains carry its own. At blowup 4, +/// as every test here whose leaves the planner holds to a leaf's rows. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn every_leaf_refuses_an_altered_answer_under_either_query_check() { + let format = at_blowup_4(small_format()); + let (elf, proof) = small_block("test_commit_4", &format); + let n = proof.commit_ood.len(); + assert!(n >= 3, "the block carries a few polynomials"); + for format in [format, lean_format(format)] { + let plan = plan_of(&elf, &proof, &format, None); + let leaves = plan.partition().num_leaves(); + for k in 0..leaves { + run_leaf(&plan, &proof, k, LeafChecks::ALL).unwrap_or_else(|e| { + panic!("{:?} leaf {k}: the honest leaf: {e}", format.leaf_query) + }); + } + for at in [0, n / 2, n - 1] { + let mut altered = proof.clone(); + altered.commit_ood[at] += FEE::one(); + for k in 0..leaves { + assert!( + run_leaf(&plan, &altered, k, LeafChecks::ALL).is_err(), + "{:?} leaf {k}: answer {at} altered executed", + format.leaf_query + ); + } + } + println!( + "COMMIT OOD LEAVES {:?}: {leaves} leaves execute; answers 0, {}, {} altered refused by each", + format.leaf_query, + n / 2, + n - 1 + ); + } +} + +/// ★ A Poseidon1 leaf refuses a tampered witness: a group root, a table's +/// argument, a Merkle sibling and a grind nonce of a group opening, and a +/// prepared opening (the prepared check is the one refusing it). +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn a_p1_block_leaf_refuses_a_tampered_witness() { + use multilinear::whir_chain::RoundOpenings; + let format = small_format(); + let (elf, proof) = small_block("test_commit_4", &format); + let plan = plan_of(&elf, &proof, &format, Some(1)); + run_leaf(&plan, &proof, 0, LeafChecks::ALL).expect("the honest leaf executes"); + + let mut root = proof.clone(); + root.proof.roots[0][0] ^= 1; + assert!( + run_leaf(&plan, &root, 0, LeafChecks::ALL).is_err(), + "a group root" + ); + + let mut table = proof.clone(); + table.proof.tables[0].bus_output.0 += FEE::one(); + assert!( + run_leaf(&plan, &table, 0, LeafChecks::ALL).is_err(), + "a table argument" + ); + + let mut sibling = proof.clone(); + match &mut sibling.proof.columns[0].polys[0].rounds[0].openings { + RoundOpenings::Base(p) => p.current[1].proof.merkle_path[0][0] ^= 1, + RoundOpenings::Extension(p) => p.current[1].proof.merkle_path[0][0] ^= 1, + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + assert!( + run_leaf(&plan, &sibling, 0, LeafChecks::ALL).is_err(), + "a Merkle sibling" + ); + + let mut nonce = proof.clone(); + nonce.proof.columns[0].polys[0].rounds[0].nonces.query ^= 1; + assert!( + run_leaf(&plan, &nonce, 0, LeafChecks::ALL).is_err(), + "a grind nonce" + ); + + // A commit-time answer, first and last: every leaf absorbs every answer + // in its front, and a group's chains carry its own. + for at in [0, proof.commit_ood.len() - 1] { + let mut answer = proof.clone(); + answer.commit_ood[at] += FEE::one(); + assert!( + run_leaf(&plan, &answer, 0, LeafChecks::ALL).is_err(), + "commit-time answer {at}" + ); + } + + let mut prepared = proof.clone(); + prepared.prepared[0].polys[0].final_value += FEE::one(); + assert!( + run_leaf(&plan, &prepared, 0, LeafChecks::ALL).is_err(), + "a prepared opening" + ); + let without = LeafChecks { + prepared: false, + ..LeafChecks::ALL + }; + assert!( + run_leaf(&plan, &prepared, 0, without).is_ok(), + "with the prepared openings left out the tamper executes" + ); +} + +/// ★ Mutation: a group's walk that reads +/// its commit-time answers from a fresh hint rather than the cells the front +/// absorbed hints words the block's arena does not hold: the leaf's arena is +/// the proof's, one answer a root, read once, by the front. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn a_walk_reads_its_answers_from_the_fronts_cells() { + use super::whir_block::ood_faults::{self, Fault}; + let format = small_format(); + let (elf, proof) = small_block("test_commit_4", &format); + for format in [format, lean_format(format)] { + let plan = plan_of(&elf, &proof, &format, Some(1)); + let honest = leaf_program_with(&plan, 0, LeafChecks::ALL).expect("the leaf"); + let arenas = block_leaf_arena(&plan, &proof, 0).expect("the arena"); + assert_eq!(honest.arena_schema.lens, vec![arenas[0].len() as u32]); + let fresh = ood_faults::with(Fault::FreshAnswerHint, || { + leaf_program_with(&plan, 0, LeafChecks::ALL).expect("the mutated leaf") + }); + assert_ne!( + fresh.arena_schema.lens, + vec![arenas[0].len() as u32], + "{:?}: a walk reading its answers anew hints words the proof's arena does not hold", + format.leaf_query + ); + } +} + +// ============ D-ONECHAIN B2: the leaves over an `Indexed` block ============ + +/// [`small_format`] at three polynomials a group under `base`, each group's +/// tables argued per `argue` and its stacked polynomials opened by one chain +/// per run of them (`GroupFold::Indexed`). +fn indexed_format(base: stark::proof::options::BaseFormat, argue: ArgueFormat) -> BlockFormat { + let small = small_format(); + BlockFormat { + zf: small.zf.with_base(base), + group_polys: 3, + argue, + group: multilinear::whir_chain::GroupFold::Indexed, + ..small + } +} + +/// ★ REV-B2-JUDGE B2-6 at the block (D-ONECHAIN B2), under the block's base: +/// the plan's split is the proof's (one chain per run), every leaf of an +/// `Indexed` block executes over the honest proof and the leaves' shares +/// cancel; a grouped round-0 value tampered in a group is refused by the host +/// and by that group's leaf; and the plan prices each group at most at its +/// per-polynomial cost, below it wherever a run opens several trees. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn an_indexed_blocks_leaves_execute_and_refuse_a_grouped_forgery() { + indexed_leaves_execute_and_refuse_a_grouped_forgery(ArgueFormat::PerTable); +} + +/// [`an_indexed_blocks_leaves_execute_and_refuse_a_grouped_forgery`] with each +/// group's tables argued together (`ArgueFormat::BATCHED`), the argue +/// production runs: the pairing a default would ship, every leaf executed. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn an_indexed_blocks_leaves_execute_under_the_batched_argue() { + indexed_leaves_execute_and_refuse_a_grouped_forgery(ArgueFormat::BATCHED); +} + +/// ★ The base code's rate is the leaf's constant under one chain a run too, at +/// the batched argue (the composed format): each way round, every leaf of the +/// plan at the block's own blowup executes, and the plan at the other blowup — +/// another statement — has a first leaf that refuses the block at its arena or +/// in its execution, never at the hasher. The line `RATE INDEXED LEAF REFUSAL:` +/// says where. +#[test] +#[ignore = "executes block leaves over small block proofs; box tier"] +fn an_indexed_blocks_leaves_at_one_blowup_refuse_a_block_at_the_other() { + let at = |blowup: u8| { + let mut f = indexed_format(crate::hash_pin::BLOCK_BASE, ArgueFormat::BATCHED); + f.zf.whir_blowup = blowup; + f + }; + for (own, other) in [(2u8, 4u8), (4, 2)] { + let (elf, proof) = small_block("all_instructions_64", &at(own)); + let plan = plan_of(&elf, &proof, &at(own), Some(2)); + for k in 0..plan.partition().num_leaves() { + run_leaf(&plan, &proof, k, LeafChecks::ALL).unwrap_or_else(|e| { + panic!("leaf {k} at blowup {own} executes over its Indexed block: {e}") + }); + } + let wrong = plan_of(&elf, &proof, &at(other), Some(2)); + assert_ne!(wrong.id(), plan.id(), "another statement at blowup {other}"); + let program = + leaf_program_with(&wrong, 0, LeafChecks::ALL).expect("the leaf at the other blowup"); + let socket = program.hasher(::SOCKET); + let refusal = match block_leaf_arena(&wrong, &proof, 0) { + Err(e) => format!("the arena: {e}"), + Ok(arenas) if program.arena_schema.lens != vec![arenas[0].len() as u32] => format!( + "the arena's length: {} words for a leaf hinting {:?}", + arenas[0].len(), + program.arena_schema.lens + ), + Ok(arenas) => match execute(&program, &arenas, &socket) { + Err(e) => { + assert!( + !matches!( + e, + super::executor::LfmExecError::HasherRejected(_) + | super::executor::LfmExecError::Internal(_) + ), + "the leaf at blowup {other} refused for another reason than the \ + proof: {e:?}" + ); + format!("the execution: {e:?}") + } + Ok(_) => panic!( + "an Indexed leaf at blowup {other} executed over a block at blowup {own}" + ), + }, + }; + println!( + "RATE INDEXED LEAF REFUSAL: a block at blowup {own}, a leaf at {other}: {refusal}" + ); + } +} + +/// ★ The group opening is the leaf's constant too: a plan derived for the +/// other mode, over a block proved under one, has no leaf that executes the +/// proof's groups — the leaf programs of one mode refuse the other mode's +/// base, both ways, whichever check or arena shape refuses first. The honest +/// pairs execute. +#[test] +#[ignore = "proves small blocks and executes their leaves; box tier"] +fn a_leaf_of_one_group_opening_refuses_the_others_base() { + use multilinear::whir_chain::GroupFold; + let base = crate::hash_pin::BLOCK_BASE; + let indexed = indexed_format(base, ArgueFormat::BATCHED); + let per_poly = BlockFormat { + group: GroupFold::PerPoly, + ..indexed + }; + for (label, proved, other) in [ + ("an Indexed base under PerPoly leaves", &indexed, &per_poly), + ("a PerPoly base under Indexed leaves", &per_poly, &indexed), + ] { + let (elf, proof) = small_block("all_instructions_64", proved); + let honest = plan_of(&elf, &proof, proved, Some(2)); + for k in 0..honest.partition().num_leaves() { + run_leaf(&honest, &proof, k, LeafChecks::ALL) + .unwrap_or_else(|e| panic!("{label}: the honest leaf {k} refused: {e}")); + } + let crossed = plan_of(&elf, &proof, other, Some(2)); + // `run_leaf` without its assertion: an arena of another length than + // the program hints is one more way the crossed leaf refuses. + let crossed_leaf = |k: usize| -> Result<(), String> { + let program = leaf_program_with(&crossed, k, LeafChecks::ALL)?; + let arenas = block_leaf_arena(&crossed, &proof, k)?; + if program.arena_schema.lens != vec![arenas[0].len() as u32] { + return Err(format!( + "the arena holds {} words, the program hints {:?}", + arenas[0].len(), + program.arena_schema.lens + )); + } + execute( + &program, + &arenas, + &program.hasher(::SOCKET), + ) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) + }; + // A hint builder that cannot read the other mode's openings may stop + // by a panic rather than an error: the leaf has no arena either way. + let crossed_leaf = |k: usize| -> Result<(), String> { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| crossed_leaf(k))) + .unwrap_or_else(|_| Err("the hint builder stopped (a panic)".to_string())) + }; + let refused = (0..crossed.partition().num_leaves()) + .filter(|&k| crossed_leaf(k).is_err()) + .count(); + println!( + "ONECHAIN CROSS LEAF {label}: {refused} of {} leaves refuse", + crossed.partition().num_leaves() + ); + assert!( + refused > 0, + "{label}: a leaf of the other mode executed the base" + ); + // Every leaf over a group the two modes open differently — one of + // two or more polynomials — refuses. + let frame = block_whir::block_frame( + proof.statement(), + &elf, + &ProofOptions::default_test_options(), + proved, + ) + .expect("the frame"); + let differ: Vec = frame + .stack_layouts + .iter() + .enumerate() + .filter(|(_, layout)| layout.num_polys() >= 2) + .map(|(g, _)| g) + .collect(); + assert!( + !differ.is_empty(), + "{label}: a group of several polynomials" + ); + for k in 0..crossed.partition().num_leaves() { + if crossed + .partition() + .leaf(k) + .iter() + .any(|g| differ.contains(g)) + { + assert!( + crossed_leaf(k).is_err(), + "{label}: leaf {k} holds a group the modes open differently and executed" + ); + } + } + } +} + +fn indexed_leaves_execute_and_refuse_a_grouped_forgery(argue: ArgueFormat) { + use multilinear::whir_chain::RoundOpenings; + let base = crate::hash_pin::BLOCK_BASE; + let format = indexed_format(base, argue); + let (elf, proof) = small_block("all_instructions_64", &format); + let plan = plan_of(&elf, &proof, &format, Some(2)); + let grouped: Vec<(usize, usize)> = proof + .proof + .columns + .iter() + .enumerate() + .flat_map(|(g, stack)| { + stack + .polys + .iter() + .enumerate() + .filter(|(_, chain)| { + matches!(chain.rounds[0].openings, RoundOpenings::BaseGrouped(_)) + }) + .map(move |(i, _)| (g, i)) + }) + .collect(); + assert!(!grouped.is_empty(), "{base:?}: a grouped chain"); + + let words = leaves_words(&plan, &proof); + let layout = plan.child_layout(); + let sum = words + .iter() + .fold(FEE::zero(), |acc, w| acc + ext_of(&w[layout.sum()])); + assert_eq!(sum, FEE::zero(), "{base:?}: the leaves' shares cancel"); + + let per_poly = BlockFormat { + group: multilinear::whir_chain::GroupFold::PerPoly, + ..format + }; + let apart = plan_of(&elf, &proof, &per_poly, Some(2)); + for (g, (&one, &many)) in plan.costs().iter().zip(apart.costs()).enumerate() { + let runs = proof.proof.columns[g].polys.len(); + assert!(one <= many, "{base:?} group {g}: {one} against {many}"); + if grouped.iter().any(|&(gg, _)| gg == g) { + assert!(one < many, "{base:?} group {g}: {runs} chains"); + } + } + println!( + "ONECHAIN BLOCK {base:?}: {} groups, grouped chains {grouped:?}, costs {:?} (per \ + polynomial {:?}), leaves {:?}", + plan.num_groups(), + plan.costs(), + apart.costs(), + plan.partition().leaves() + ); + + // A value of tree 1's block at query 1 in the first grouped chain: the + // host refuses it at that chain's opening (reported as the run's first + // polynomial), and the leaf holding the group at tree 1's opening at + // query 1 — the first failing check in program order, against the + // honest leaf, which executes. + let (g, i) = grouped[0]; + let mut forged = proof.clone(); + match &mut forged.proof.columns[g].polys[i].rounds[0].openings { + RoundOpenings::BaseGrouped(round) => round.current[1][1].values[0] += FE::one(), + _ => unreachable!("a grouped round 0"), + } + let opts = ProofOptions::default_test_options(); + let frame = block_whir::block_frame(proof.statement(), &elf, &opts, &format).expect("frame"); + let run = multilinear::stacked_eval::index_chains( + frame.stack_layouts[g].num_polys(), + multilinear::whir_chain::GroupFold::Indexed, + )[i] + .clone(); + let refusal = block_whir::verify_block_whir_traced( + &forged, + &elf, + &opts, + &format, + false, + stark::multilinear_table::batched::VerifierChecks::ALL, + ) + .expect("the statement frames"); + assert!( + matches!( + refusal, + Err(block_whir::BlockRefusal::Proof( + multilinear::Error::ColumnOpeningRejected { column } + )) if column == run.start + ), + "{base:?}: the host refuses group {g} chain {i}'s opening, got {refusal:?}" + ); + let k = (0..plan.partition().num_leaves()) + .find(|&k| plan.partition().leaf(k).contains(&g)) + .expect("a leaf holds the group"); + use super::whir_chain::onechain_steps::{self, Step}; + let (program, marks) = onechain_steps::record(|| { + leaf_program_with(&plan, k, LeafChecks::ALL).expect("the leaf emits") + }); + let hasher = program.hasher(crate::hash_pin::BLOCK_HASHER); + let honest = block_leaf_arena(&plan, &proof, k).expect("the honest arena"); + super::executor::execute_serial(&program, &honest, &hasher).expect("the honest leaf executes"); + let forged_arena = block_leaf_arena(&plan, &forged, k).expect("the forged arena"); + let error = super::executor::execute_serial(&program, &forged_arena, &hasher) + .expect_err("the leaf refuses the forgery"); + assert_eq!( + onechain_steps::failed_at(&program, &marks, &error), + Some(( + Some(run.start), + Step::Opening { + round: 0, + query: 1, + tree: 1 + } + )), + "{base:?}: leaf {k} refuses group {g} chain {i} at tree 1's opening, got {error:?}" + ); + assert!( + run_leaf(&plan, &forged, k, LeafChecks::ALL).is_err(), + "{base:?}: and on the production schedule" + ); +} + +/// The code review's M-g: a small `Indexed` block under a stack of +/// `2^stack` and `group_polys` polynomials a group, its prepared stack forged +/// in one chain, at the block and in its leaf. +/// +/// The block proves and verifies ([`small_block`]); every leaf executes and the +/// shares cancel. `pick` chooses, from the prepared stacks' chains — `(stack, +/// chain, width)`, a width of 1 being a per-polynomial chain — the one to +/// forge: a value of its round-0 block at query 1, in tree 1 of a grouped +/// chain or the one tree of another. The host must refuse at that stack's +/// opening, reported as the run's first polynomial, and the leaf holding the +/// stack's group at that tree's opening at query 1 under the chain's own mark. +/// The forgery touches the prepared opening alone, and the honest leaf +/// executes. +fn forged_prepared_chain( + base: stark::proof::options::BaseFormat, + argue: ArgueFormat, + stack: usize, + group_polys: usize, + pick: impl Fn(&[(usize, usize, usize)]) -> Option<(usize, usize, usize)>, +) { + use super::whir_chain::onechain_steps::{self, Step}; + use multilinear::whir_chain::{GroupFold, RoundOpenings}; + let small = indexed_format(base, argue); + let mut zf = small.zf; + zf.whir_stack = StackVars::new(stack).expect("a valid stack"); + let format = BlockFormat { + zf, + group_polys, + ..small + }; + let (elf, proof) = small_block("all_instructions_64", &format); + let chains: Vec<(usize, usize, usize)> = proof + .prepared + .iter() + .enumerate() + .flat_map(|(k, opening)| { + opening.polys.iter().enumerate().map(move |(i, chain)| { + let width = match &chain.rounds[0].openings { + RoundOpenings::BaseGrouped(g) => g.current.first().map_or(0, Vec::len), + _ => 1, + }; + (k, i, width) + }) + }) + .collect(); + println!( + "ONECHAIN PREPARED {:?} stack 2^{stack}: {} groups, prepared chains (stack, chain, width) \ + {chains:?}", + base.hash, + proof.groups.len() + ); + let (k, i, width) = pick(&chains).unwrap_or_else(|| { + panic!( + "{:?} stack 2^{stack}: the prepared stacks' chains {chains:?} hold none to forge", + base.hash + ) + }); + + let plan = plan_of(&elf, &proof, &format, None); + let words = leaves_words(&plan, &proof); + let layout = plan.child_layout(); + let sum = words + .iter() + .fold(FEE::zero(), |acc, w| acc + ext_of(&w[layout.sum()])); + assert_eq!( + sum, + FEE::zero(), + "{:?}: the leaves' shares cancel", + base.hash + ); + + let prepared = &plan.prepared()[k]; + let run = + multilinear::stacked_eval::index_chains(prepared.layout.num_polys(), GroupFold::Indexed)[i] + .clone(); + assert_eq!(run.len(), width, "the proof's chain is the derived run"); + let mut forged = proof.clone(); + let tree = match &mut forged.prepared[k].polys[i].rounds[0].openings { + RoundOpenings::BaseGrouped(round) => { + round.current[1][1].values[0] += FE::one(); + 1 + } + RoundOpenings::Base(round) => { + round.current[1].values[0] += FE::one(); + 0 + } + RoundOpenings::Extension(_) => unreachable!("round 0 is base-field"), + }; + let opts = ProofOptions::default_test_options(); + let refusal = block_whir::verify_block_whir_traced( + &forged, + &elf, + &opts, + &format, + false, + stark::multilinear_table::batched::VerifierChecks::ALL, + ) + .expect("the statement frames"); + assert!( + matches!( + refusal, + Err(block_whir::BlockRefusal::Proof( + multilinear::Error::ColumnOpeningRejected { column } + )) if column == run.start + ), + "{:?}: the host refuses prepared stack {k} chain {i}'s opening, got {refusal:?}", + base.hash + ); + let g = prepared.group; + let leaf = (0..plan.partition().num_leaves()) + .find(|&l| plan.partition().leaf(l).contains(&g)) + .expect("a leaf holds the group"); + let (program, marks) = onechain_steps::record(|| { + leaf_program_with(&plan, leaf, LeafChecks::ALL).expect("the leaf emits") + }); + let hasher = program.hasher(crate::hash_pin::BLOCK_HASHER); + let honest = block_leaf_arena(&plan, &proof, leaf).expect("the honest arena"); + super::executor::execute_serial(&program, &honest, &hasher).expect("the honest leaf executes"); + let forged_arena = block_leaf_arena(&plan, &forged, leaf).expect("the forged arena"); + let error = super::executor::execute_serial(&program, &forged_arena, &hasher) + .expect_err("the leaf refuses the forgery"); + assert_eq!( + onechain_steps::failed_at(&program, &marks, &error), + Some(( + Some(run.start), + Step::Opening { + round: 0, + query: 1, + tree + } + )), + "{:?}: leaf {leaf} refuses prepared stack {k} chain {i} at tree {tree}'s opening, got \ + {error:?}", + base.hash + ); + println!( + "ONECHAIN PREPARED {:?} stack 2^{stack}: REFUSED prepared stack {k} chain {i} (run {run:?}) \ + — host at its opening, leaf {leaf} of {} at tree {tree}'s opening at query 1", + base.hash, + plan.partition().num_leaves() + ); +} + +/// The code review's M-g: a PREPARED stack opened by a GROUPED chain. Under a +/// stack of 2^8, DECODE's five ELF-derived columns (2^7 rows for this program) +/// stack into three polynomials — one chain of three, slot 3 absent — while +/// twelve polynomials a group keep each group's cells those of the 2^10 +/// format. The grouped prepared chain is forged in tree 1. +#[test] +#[ignore = "proves small blocks and executes their leaves; box tier"] +fn a_grouped_prepared_stack_verifies_and_its_leaf_refuses_a_forgery() { + for argue in [ArgueFormat::PerTable, ArgueFormat::BATCHED] { + forged_prepared_chain(crate::hash_pin::BLOCK_BASE, argue, 8, 12, |chains| { + chains.iter().copied().find(|&(_, _, width)| width >= 2) + }); + } +} + +/// The code review's M-g: a prepared stack SPLIT into two chains. Under a stack +/// of 2^7 each of DECODE's five 2^7-row columns is a polynomial of its own: +/// runs [4, 1]. The LAST chain — whose roots are the stack's derived roots from +/// polynomial 4 on — is forged, so each chain's slice of the derived roots +/// (`roots[run]`) is what refuses it, by position. +#[test] +#[ignore = "proves small blocks and executes their leaves; box tier"] +fn a_split_prepared_stack_refuses_a_forgery_in_its_last_chain() { + for argue in [ArgueFormat::PerTable, ArgueFormat::BATCHED] { + split_prepared_stack_refuses_a_forgery_in_its_last_chain(argue); + } +} + +fn split_prepared_stack_refuses_a_forgery_in_its_last_chain(argue: ArgueFormat) { + forged_prepared_chain(crate::hash_pin::BLOCK_BASE, argue, 7, 24, |chains| { + chains.iter().copied().rfind(|&(k, i, _)| { + i >= 1 + && chains + .iter() + .any(|&(kk, ii, w)| kk == k && ii == 0 && w >= 2) + }) + }); +} diff --git a/prover/src/lfm/whir_block_tree.rs b/prover/src/lfm/whir_block_tree.rs new file mode 100644 index 000000000..f7521d969 --- /dev/null +++ b/prover/src/lfm/whir_block_tree.rs @@ -0,0 +1,3219 @@ +//! ★ The no-epoch WHIR block, base to top node (W3): the one driver the +//! whole-block harness (`whir_block_tests::the_whir_block_tree_on_a_real_block`) +//! and the CLI's `prove-block` both run, so their numbers cannot diverge. +//! +//! [`prove_whir_block_tree`] proves the block in one proof at the production +//! format ([`crate::block_whir::prove_block_whir_observed_groups`]); the moment +//! its statement is final it derives the plan and emits the leaves' programs +//! beside phase B, and the first leaf whose groups phase B finished executes +//! and fills right then; then it proves the tree ([`prove_tree_pipelined`]): +//! the leaves a bounded number at once, the nodes' programs and artifacts built +//! beside them, each node proved as soon as its own children are. Every line +//! goes through a [`WhirTreeSink`]: the harness prints to stdout, the CLI to +//! stderr. What the harness checks off the clock (every tree proof against its +//! artifacts, the base on the host, the top through the block verifier) stays +//! in the harness. +//! +//! Every knob is read once, by [`WhirTreeConfig::from_env`], under the names +//! and defaults the harness always read. [`POSTURE`] is the production posture +//! the measurements run under; the CLI sets it for knobs the environment +//! leaves unset, and the driver's first line says how the running process +//! compares with it. + +use multilinear::whir_chain::{ArgueFormat, GkrPack, GroupFold, LeafOrder}; + +use crate::block_whir::{self, BlockFormat, BlockOptions, BlockWhirProof}; +use crate::zf_format::WhirBlowup; + +use super::compiler::LfmProgram; +use super::harvest::{HarvestedChild, harvest_child_under, try_child_arena_words}; +use super::per_table_aggregator::DerivedChild; +use super::proof::{ + LfmFilled, LfmProof, aggregation_wrap_options, decide_lfm_residency, lfm_execute_and_fill, + take_prove_split, +}; +use super::whir_block::{ + BLOCK_FAN_IN, LeafScale, WhirBlockPlan, artifacts_of, block_leaf_arena, group_arena_words, + leaf_arena, leaf_scale_from_env, +}; +use super::whir_chain::LeafQuery; +use super::word::LfmWord; +use crate::hash_pin::{BlockFamily, ProofFamily}; + +// ================================ the sink ================================ + +/// Where the driver's lines go. +pub trait WhirTreeSink: Send + Sync { + /// Whole lines, each ending in `\n`, written at once. + fn write(&self, text: &str); + + /// One line. + fn line(&self, line: &str) { + self.write(&format!("{line}\n")); + } +} + +/// Lines to stdout (the harness's). +pub struct StdoutSink; + +impl WhirTreeSink for StdoutSink { + fn write(&self, text: &str) { + print!("{text}"); + } +} + +/// Lines to stderr (the CLI's: its stdout carries the results). +pub struct StderrSink; + +impl WhirTreeSink for StderrSink { + fn write(&self, text: &str) { + eprint!("{text}"); + } +} + +// =============================== the posture ============================== + +/// The production posture the block's measurements run under: the knobs the +/// suite env sets that change behaviour or performance, at its values (ULTRA's +/// and BIG's W3 env). The CLI sets each one the environment leaves unset, and +/// the harness never does: its arms rely on unset meaning the library default. +/// The base hash is not posture: it is the build's (`hash_pin::BLOCK_BASE`, the +/// only base `block_whir::checked_base` admits), and `LAMBDA_VM_WHIR_HASH` is +/// not read on the block path. The measurement-only `LAMBDA_VM_BASE_SPLIT` is not posture, and +/// the allocator's never-purge posture is compiled into the binary. +/// `LFM_PRECOMPUTED_TREE_CACHE_CAP`'s posture follows the host target +/// ([`posture_tree_cache_cap`]): the table's 64 from a 64 GiB target up, 16 +/// below it. +pub const POSTURE: &[(&str, &str)] = &[ + ("TABLE_PARALLELISM", "4"), + ("LAMBDA_VM_MAX_ROWS_LOG2", "21"), + ("LFM_WHIR_RETENTION", "1"), + ("LFM_PRECOMPUTED_TREE_CACHE_CAP", "64"), + ("LFM_EXEC_PARALLEL", "1"), + ("LAMBDA_VM_GRIND_SCAN_FACTOR", "8"), + ("LAMBDA_VM_GRIND_GRID", "1024"), +]; + +/// The precomputed-tree cache's knob, whose posture follows the host target. +pub const POSTURE_TREE_CACHE_KNOB: &str = "LFM_PRECOMPUTED_TREE_CACHE_CAP"; + +/// The host target (GiB, [`block_whir::memory_target_bytes`]) from which the +/// posture keeps [`POSTURE`]'s 64 precomputed-tree cache entries. +pub const POSTURE_TREE_CACHE_FULL_TARGET_GIB: u64 = 64; + +/// `LFM_PRECOMPUTED_TREE_CACHE_CAP`'s posture for a host `target` in bytes: +/// 64 entries (≈ 100 MiB each, ≈ 6.2 GiB once full) from a +/// [`POSTURE_TREE_CACHE_FULL_TARGET_GIB`] target up, 16 below it, where the +/// ≈ 4.7 GiB they free count for more than the hits they lose. A miss only +/// rebuilds the tree: the cache's key is the root, so nothing a proof commits +/// to moves (#1013's rule, 49c4bb512). +pub fn posture_tree_cache_cap(target: u64) -> &'static str { + if target >= POSTURE_TREE_CACHE_FULL_TARGET_GIB << 30 { + "64" + } else { + "16" + } +} + +/// The `BLOCK POSTURE:` line: each posture knob as the process has it — equal +/// to the posture, unset, or another value. +pub fn posture_line() -> String { + let words: Vec = POSTURE + .iter() + .map(|&(name, want)| { + if name == POSTURE_TREE_CACHE_KNOB { + ( + name, + posture_tree_cache_cap(block_whir::memory_target_bytes()), + ) + } else { + (name, want) + } + }) + .map(|(name, want)| match std::env::var(name) { + Ok(v) if v == *want => format!("{name}={v}"), + Ok(v) => format!("{name}={v} (≠ posture {want})"), + Err(_) => format!("{name} unset (≠ posture {want})"), + }) + .collect(); + format!("BLOCK POSTURE: {}", words.join(" · ")) +} + +/// The `BLOCK BASE:` line: the base hash and 4-ary cap the format names, the +/// block path's only source for its hash (`LAMBDA_VM_WHIR_HASH` is not read +/// there, D-WHIR-P1 S7). +pub fn base_line(base: &stark::proof::options::BaseFormat) -> String { + format!( + "BLOCK BASE: {:?} · 4-ary cap {:?} · the format's ({} is not read on the block path)", + base.hash, + base.arity4_cap, + crate::whir_hash_knob::ENV + ) +} + +/// Phase A's two knobs beside the finish, as the run read them: +/// `LAMBDA_VM_KR_PACKED_WAVE` (KECCAK_RND chunks written packed at once) and +/// `LAMBDA_VM_BLOCK_COMMIT_OFF_POOL` (the committer's pairs off the global +/// pool). +pub fn finish_idle_line() -> String { + let wave = match crate::tables::trace_builder::kr_packed_wave() { + usize::MAX => "all".to_string(), + n => n.to_string(), + }; + format!( + "BLOCK FINISH IDLE: KECCAK_RND packed wave {wave} · commit pairs off the pool {}", + stark::multilinear_block::commit_pairs_off_pool() + ) +} + +// ================================ the knobs =============================== + +/// Every knob the W3 block reads, once. +#[derive(Clone, Debug)] +pub struct WhirTreeConfig { + /// `W3_LEAVES`: the leaf count; unset is the plan's rule. A forced count + /// is another tree than the production verifier derives. + pub leaves: Option, + /// `W3_SIBLINGS` (3 by default): programs of the tree proved at once. + pub siblings: usize, + /// `W3_FAN_IN` ([`BLOCK_FAN_IN`] by default): another fan-in is another + /// tree than the production verifier derives. + pub fan_in: usize, + /// `W3_EXEC_BESIDE_ARTIFACTS=0|1` (default 1): the leaves execute and fill + /// while their artifacts are built, or (0, the control) after all of them. + pub beside: bool, + /// `W3_LEAF_ARTIFACTS_EACH=0|1` (default 0): the leaves' artifacts are + /// published all at once after the last is built while every leaf's + /// program is there, and each as it is built from the first leaf whose + /// program is not ([`publish_each`]); or (1, a measurement arm) each as it + /// is built throughout. + pub leaves_each: bool, + /// `LAMBDA_VM_TREE_PROGRAM_BUDGET` (`auto` by default): how many bytes of + /// tree programs may exist ahead of their provers + /// ([`super::program_budget`]), against the memory target. + pub program_budget: super::program_budget::BudgetSetting, + /// `W3_LEAF_DURING_PHASE_B=0|1` (default 1): the first leaf whose groups + /// phase B finished while another group was still to come executes and + /// fills right then, or (0, the control) with the others after the base. + pub early_on: bool, + /// `W3_LEAF_EMITTERS` ([`LEAF_EMITTERS`] by default): how many of the + /// leaves' programs are emitted at once beside phase B, in leaf order, + /// each published as it is ready while the planner takes phase B's groups; + /// or (0, the control: `None`) one thread a leaf, all at once, the first + /// group taken after the last is emitted. + pub leaf_emitters: Option, + /// `BLOCK_WHIR_ARGUE=batched|per-table` (production: batched, at the + /// format's bin cap; `BLOCK_WHIR_ARGUE_CAP=k` for 2^k). Another argue is + /// another format than the production verifier's. + pub argue: ArgueFormat, + /// The block format: production, with `argue`, its groups' stacked + /// polynomials opened per `LAMBDA_VM_WHIR_GROUP=per-poly|indexed` + /// ([`group_setting`]; production: per-poly). + pub format: BlockFormat, + /// The base's options: production, with the `BLOCK_WHIR_*` knobs. + pub options: BlockOptions, + /// `W3_NODE_PIPE=0|1` (default 1): the nodes' programs and artifacts built + /// on a pool of `W3_EMIT_THREADS` threads of their own (default 4), each + /// level proving as its own nodes arrive; or (0, the control) one after + /// another on one thread, level 1 waiting for the whole tree's builds. + pub node_pipe: Option, + /// `W3_DATAFLOW=0|1` (default 1): each node proves as soon as its own + /// children are proved, or (0, the control) once its whole level below is. + pub dataflow: bool, + /// `W3_STREAM_TOP=0|1` (default 1): when a child is still unproved as the + /// top node can execute, the top executes each child's share as that child + /// is proved; otherwise, and at 0 (the control), all of it after the last. + pub stream_top: bool, + /// `W3_EXEC_EARLY=0|1` (default 1): a node executes and fills from its + /// program as soon as the builder emits it, and only its prove waits for + /// its artifacts; or (0, the control) it executes once its artifacts are + /// built too. + pub exec_early: bool, + /// `W3_NODE_TIMES=1`: the per-program stamps readout after the tree. + pub node_times: bool, + /// `W3_LEAF_FIRST=0|1` (default 1): with the early leaf on, the planner + /// offers the leaves to phase B as soon as it has derived the plan, and + /// phase B takes the groups of the leaf it can finish soonest next + /// ([`block_whir::PhaseBLead`]); the tree's workers take that early leaf + /// first. Or (0, the control) phase B keeps its own order and the tree + /// takes the leaves in index order. Prover-only: the bytes are the same. + pub leaf_first: bool, +} + +/// A count knob: unset is `None`; anything but a count is an error. +fn count_knob(name: &str) -> Result, String> { + match std::env::var(name) { + Err(_) => Ok(None), + Ok(v) => v + .trim() + .parse() + .map(Some) + .map_err(|_| format!("{name} is a count, got `{v}`")), + } +} + +/// `LAMBDA_VM_WHIR_GROUP`'s reading: unset or `indexed` is one chain per run +/// of up to four stacked polynomials ([`GroupFold::Indexed`], production); +/// `per-poly` one chain per polynomial ([`GroupFold::PerPoly`]); anything else +/// an error. A format choice: the block's verifier must be given the same one. +pub fn group_setting(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("indexed") => Ok(GroupFold::Indexed), + Some("per-poly") => Ok(GroupFold::PerPoly), + Some(other) => Err(format!("LAMBDA_VM_WHIR_GROUP={other}: per-poly or indexed")), + } +} + +/// The CLI's `--blowup` reading: unset or `2` is the production WHIR base code +/// ([`WhirBlowup::Two`]), `4` is the legacy blowup ([`WhirBlowup::Four`]); +/// anything else an error. A format choice: the block's verifier must be given +/// the same one, and the proof file does not carry it. +pub fn blowup_setting(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("2") => Ok(WhirBlowup::Two), + Some("4") => Ok(WhirBlowup::Four), + Some(word) => Err(format!("--blowup {word}: 4 or 2")), + } +} + +/// The CLI's `--leaves` reading: unset or `coset` is the production round-0 +/// leaf order ([`LeafOrder::Coset`], Poseidon1 trees only), `natural` the +/// codeword's own order ([`LeafOrder::Natural`]); anything else an error. A +/// format choice: the block's verifier must be given the same one, and the +/// proof file does not carry it. +pub fn leaves_setting(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("coset") => Ok(LeafOrder::Coset), + Some("natural") => Ok(LeafOrder::Natural), + Some(word) => Err(format!("--leaves {word}: natural or coset")), + } +} + +/// `W3 LEAF SCALE`: the format's leaf scale, the leaf count at the single +/// heights and whether the partition doubled them; a fixed leaf count +/// (`W3_LEAVES`) bounds no chip, so no scale applies. +pub fn leaf_scale_line(scale: LeafScale, plan: &WhirBlockPlan) -> String { + let leaves = plan.partition().num_leaves(); + match plan.leaf_bound() { + None => { + format!("W3 LEAF SCALE: {scale} · {leaves} leaves forced (W3_LEAVES): no scale applies") + } + Some(bound) if bound.doubled => format!( + "W3 LEAF SCALE: {scale} · {} leaves at the single heights · doubled to {leaves} leaves", + bound.single_leaves + ), + Some(bound) => format!( + "W3 LEAF SCALE: {scale} · {} leaves at the single heights · not doubled", + bound.single_leaves + ), + } +} + +/// How many leaf programs are emitted at once beside phase B by default +/// (`W3_LEAF_EMITTERS`). One thread a leaf, all started at the statement, took +/// the host from phase B's first groups (measured on a 16-core host with one +/// RTX 5090: at the median 0.61 s a group while 13 emitters ran against +/// 0.45 s after them; on a large block 0.95 s with 26), and +/// nothing needs a leaf's program before the base ends but the early leaf, +/// which is among the first emitted. +pub const LEAF_EMITTERS: usize = 3; + +/// `BLOCK_WHIR_ARGUE_PACK`'s reading: unset is the production packing +/// ([`GkrPack::PRODUCTION`]); `off` is one GKR tree a table ([`GkrPack::Off`]); +/// a count `τ` in `0..=40` cuts a table into equal chunks where that saves +/// input cells, each extra tree priced at `2^τ` ([`GkrPack::Chunks`]). A format +/// value: the prover and the verifier of one proof take the same one. +pub fn pack_setting(knob: Option<&str>) -> Result { + match knob.map(str::trim) { + None => Ok(GkrPack::PRODUCTION), + Some("off") => Ok(GkrPack::Off), + Some(v) => v + .parse::() + .ok() + .filter(|tau| *tau <= 40) + .map(|tree_log_cells| GkrPack::Chunks { tree_log_cells }) + .ok_or_else(|| format!("BLOCK_WHIR_ARGUE_PACK={v}: off or a count in 0..=40")), + } +} + +/// `argue` with its trees laid out per `pack`: only the batched argue has +/// trees to pack. +fn with_pack(argue: ArgueFormat, pack: GkrPack) -> Result { + match (argue, pack) { + (ArgueFormat::PerTable, GkrPack::Off) => Ok(argue), + (ArgueFormat::Batched { bin_log_cells, .. }, pack) => Ok(ArgueFormat::Batched { + bin_log_cells, + pack, + }), + (ArgueFormat::PerTable, _) => { + Err("BLOCK_WHIR_ARGUE_PACK packs the batched argue's trees: not per-table".to_string()) + } + } +} + +/// `W3_LEAF_EMITTERS`'s reading: unset is [`LEAF_EMITTERS`]; 0 is the control, +/// one thread a leaf. +fn leaf_emitters_setting(knob: Option) -> Option { + match knob { + Some(0) => None, + other => Some(other.unwrap_or(LEAF_EMITTERS)), + } +} + +impl WhirTreeConfig { + /// Every knob from the environment, under the names and defaults the + /// harness always read. A value a knob cannot take is an error (the + /// `BLOCK_WHIR_*` readers of [`crate::block_whir`] keep their own refusals). + pub fn from_env() -> Result { + // The leaves' scale first: the production format reads the same knob + // and refuses what it cannot take, which this returns as an error. + let leaf_scale = leaf_scale_from_env()?; + let leaf_query = super::whir_block::leaf_query_from_env()?; + let leaves = count_knob("W3_LEAVES")?; + let siblings = count_knob("W3_SIBLINGS")?.unwrap_or(3); + let fan_in = count_knob("W3_FAN_IN")?.unwrap_or(BLOCK_FAN_IN); + let beside = count_knob("W3_EXEC_BESIDE_ARTIFACTS")?.is_none_or(|v| v != 0); + let leaves_each = count_knob("W3_LEAF_ARTIFACTS_EACH")? == Some(1); + let early_on = count_knob("W3_LEAF_DURING_PHASE_B")?.is_none_or(|v| v != 0); + let leaf_emitters = leaf_emitters_setting(count_knob("W3_LEAF_EMITTERS")?); + let argue = match std::env::var("BLOCK_WHIR_ARGUE").as_deref().map(str::trim) { + Ok("batched") => match count_knob("BLOCK_WHIR_ARGUE_CAP")? { + Some(cap) => ArgueFormat::batched( + u8::try_from(cap) + .map_err(|_| format!("BLOCK_WHIR_ARGUE_CAP={cap}: a cap below 2^256"))?, + ), + None => ArgueFormat::BATCHED, + }, + Ok("per-table") => ArgueFormat::PerTable, + Err(_) => BlockFormat::production().argue, + Ok(other) => return Err(format!("BLOCK_WHIR_ARGUE={other}: per-table or batched")), + }; + // Unset, the argue keeps its own packing (the production one when + // batched); set, the knob's packing replaces it. + let argue = match std::env::var("BLOCK_WHIR_ARGUE_PACK") { + Ok(word) => with_pack(argue, pack_setting(Some(&word))?)?, + Err(_) => argue, + }; + let format = BlockFormat { + argue, + leaf_scale, + leaf_query, + ..BlockFormat::production_with(group_setting( + std::env::var("LAMBDA_VM_WHIR_GROUP").ok().as_deref(), + )?) + }; + let mut options = BlockOptions::production(); + // `BLOCK_WHIR_LAYOUT_WORKERS=n`: the streamed chunks laid out on n threads + // (production 6; 0 is the inline layout). + if let Some(n) = count_knob("BLOCK_WHIR_LAYOUT_WORKERS")? { + options.layout_workers = n; + } + // `BLOCK_WHIR_LAYOUT_AHEAD=k` bounds the chunks unpacked at k + 1; + // `none` lifts the bound (the reverted version, BIG 390). + match std::env::var("BLOCK_WHIR_LAYOUT_AHEAD") + .as_deref() + .map(str::trim) + { + Ok("none") => options.layout_ahead = None, + Ok(k) => { + if let Ok(k) = k.parse() { + options.layout_ahead = Some(k); + } + } + Err(_) => {} + } + // `BLOCK_WHIR_DROP_OPS=0`: the builder keeps the streamed chunks' ops + // (production drops them). + match std::env::var("BLOCK_WHIR_DROP_OPS") + .as_deref() + .map(str::trim) + { + Ok("0") => options.drop_streamed_ops = false, + Ok("1") => options.drop_streamed_ops = true, + _ => {} + } + // `BLOCK_WHIR_NARROW=wide|card|host` (production card): how phase A holds + // each group's columns once committed. + if let Some(narrow) = block_whir::narrow_from_env() { + options.narrow = narrow; + } + // `BLOCK_WHIR_KECCAK_LOG2=k`, `BLOCK_WHIR_ECSM_LOG2=k`: force a KECCAK or + // ECSM split (production 2^18 / 2^17). + block_whir::chunk_cuts_from_env(&mut options); + // `BLOCK_WHIR_WINDOW_LOG2=20` (production): the streamed prove's window; + // any other value is refused (it would move the groups). + if let Some(log2) = block_whir::window_log2_from_env() { + options.window_log2 = Some(log2); + } + // `BLOCK_WHIR_UPLOAD_AHEAD=0|1` (production 1): phase A puts each group's + // columns on the card beside the previous group's commit. + if let Some(ahead) = block_whir::upload_ahead_from_env() { + options.upload_ahead = ahead; + } + // `BLOCK_WHIR_REST_LAYOUT=all|` (production 4096), + // `BLOCK_WHIR_REST_BUILD=` (production 2048), + // `BLOCK_WHIR_KR_FINISH_CHUNKS=0|1` and `BLOCK_WHIR_PACK_FINISHED=0|1` + // (production 1): the rest's gate (its waves where it is laid out in + // waves) and the bytes it builds at once, KECCAK_RND built as its + // tables, the finish's tables packed as they are built. + block_whir::rest_layout_from_env(&mut options); + let node_pipe = count_knob("W3_NODE_PIPE")? + .is_none_or(|v| v != 0) + .then(|| count_knob("W3_EMIT_THREADS").map(|t| t.unwrap_or(4).max(1))) + .transpose()?; + Ok(Self { + leaves, + siblings, + fan_in, + beside, + leaves_each, + program_budget: super::program_budget::setting_from_env()?, + early_on, + leaf_emitters, + argue, + format, + options, + node_pipe, + dataflow: count_knob("W3_DATAFLOW")?.is_none_or(|v| v != 0), + stream_top: count_knob("W3_STREAM_TOP")?.is_none_or(|v| v != 0), + exec_early: count_knob("W3_EXEC_EARLY")?.is_none_or(|v| v != 0), + node_times: count_knob("W3_NODE_TIMES")? == Some(1), + leaf_first: count_knob("W3_LEAF_FIRST")?.is_none_or(|v| v != 0), + }) + } + + /// Whether the tree is the production verifier's: the plan's leaf count, + /// [`BLOCK_FAN_IN`] and the production argue — with its GKR trees laid out + /// as the format's packing says, a named constant the block verifier is + /// given too ([`BlockFormat::with_pack`]). + pub fn at_presets(&self) -> bool { + let pack = match self.argue { + ArgueFormat::Batched { pack, .. } => pack, + ArgueFormat::PerTable => GkrPack::Off, + }; + self.leaves.is_none() + && self.fan_in == BLOCK_FAN_IN + && self.argue == BlockFormat::production().with_pack(pack).argue + && self.format.argue == self.argue + } +} + +// ============================ the tree's machinery ======================== + +/// A tree program as its emitter made it, with its admission under the tree's +/// program budget (none without one). The program is declared first, so it is +/// dropped before the permit that accounts for it. +pub(crate) struct Admitted

{ + pub(crate) program: P, + permit: Option, +} + +/// One program's place from its emission to its proof: shared by its +/// artifacts' build and its execute, and let go once its proof is done, which +/// frees the program (with the last of its users) and gives its bytes back to +/// the budget. +pub(crate) struct ProgramSlot

( + std::sync::Mutex>>>, +); + +impl

ProgramSlot

{ + pub(crate) fn new(program: P, permit: Option) -> Self { + Self(std::sync::Mutex::new(Some(std::sync::Arc::new(Admitted { + program, + permit, + })))) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Option>>> { + self.0.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// The program, until its proof lets it go. + pub(crate) fn get(&self) -> Result>, String> { + self.lock() + .clone() + .ok_or_else(|| "the program was let go after its proof".to_string()) + } + + /// A prover took the program ([`super::program_budget::Permit::claim`]). + pub(crate) fn claim(&self) { + if let Some(permit) = self.lock().as_ref().and_then(|a| a.permit.as_ref()) { + permit.claim(); + } + } + + /// Its proof is done: the slot lets go of the program. + pub(crate) fn release(&self) { + let program = self.lock().take(); + drop(program); + } +} + +/// A node's program as the builder emits it, before its artifacts: all its +/// execute and fill need, and when it was emitted (seconds since the tree +/// started). +pub(crate) struct NodeProgram { + pub(crate) program: ProgramSlot, + pub(crate) at: f64, + /// Seconds its emission took, after its admission: the builder's walk, + /// the compile and the admission validator. + pub(crate) emit: f64, + /// How many of the program's arenas each child supplies, in child order + /// ([`super::per_table_aggregator::leg_arena_count`] over the shapes the + /// program was emitted over): where a streamed execution splits them + /// ([`execute_streamed`]). + pub(crate) child_arenas: Vec, +} + +/// A leaf program's bytes before it is emitted, a group of it: 0.157 GiB at +/// T200-R (130 leaves, 81.25 GiB of programs; I-T200 §11b), 0.55–0.63 GiB a +/// leaf of 3–4 groups at p90 (I-WFULL §1.3). The budget learns the real bytes +/// when the program is emitted. +pub(crate) const LEAF_BYTES_PER_GROUP: u64 = 161 << 20; + +/// The streaming threads that emit, under a bounded budget, the leaf programs +/// not admitted beside phase B (#1013's `DEFERRED_LEAF_EMITTERS`). +const DEFERRED_LEAF_EMITTERS: usize = 4; + +/// A program's host bytes as held (capacities): its instructions with what +/// they box, its column groups and its arena schema (#1013's +/// `ProgramBytes::total`). +pub(crate) fn program_bytes(p: &LfmProgram) -> u64 { + use super::instr::{Addr, Blake3Operands, Instr, KeccakOperands}; + let mut bytes = p.instrs.capacity() * size_of::(); + for instr in &p.instrs { + bytes += match instr { + Instr::BitDec { bits, .. } => bits.capacity() * size_of::<(Addr, u64)>(), + Instr::KeccakF(_) => size_of::(), + Instr::Blake3(_) => size_of::(), + _ => 0, + }; + } + let g = &p.groups; + for group in [ + &g.const_, &g.balu, &g.xalu, &g.select, &g.bitdec, &g.hash, &g.keccak, &g.blake3, &g.lanes, + &g.hint, &g.public, + ] { + bytes += group.data.capacity() * size_of::(); + } + (bytes + p.arena_schema.lens.capacity() * size_of::()) as u64 +} + +/// The tree's program budget as its emitters use it: each program admitted in +/// prove order ([`tree_order`]'s numbering: the leaves, then each node level) +/// with its estimated bytes before it is emitted. +pub(crate) struct TreeBudget { + pub(crate) budget: std::sync::Arc, + /// Each leaf's estimate: [`LEAF_BYTES_PER_GROUP`] a group. + leaf_estimates: Vec, + /// Where each node level starts in prove order. + level_starts: Vec, + /// The largest node program emitted so far. + node_seen: std::sync::atomic::AtomicU64, +} + +impl TreeBudget { + /// The budget of a tree of `shape` over leaves of `leaf_groups` groups each. + pub(crate) fn new( + budget: std::sync::Arc, + leaf_groups: &[usize], + shape: &[super::per_table_aggregator::Level], + ) -> Self { + let mut level_starts = Vec::with_capacity(shape.len()); + let mut next = leaf_groups.len(); + for level in shape { + level_starts.push(next); + next += level.arities.len(); + } + Self { + budget, + leaf_estimates: leaf_groups + .iter() + .map(|&g| g as u64 * LEAF_BYTES_PER_GROUP) + .collect(), + level_starts, + node_seen: std::sync::atomic::AtomicU64::new(0), + } + } + + pub(crate) fn leaf_estimate(&self, k: usize) -> u64 { + self.leaf_estimates.get(k).copied().unwrap_or(0) + } + + /// Node `j` of node level `lv`'s place in prove order (0 = the level above + /// the leaves). + pub(crate) fn node_order(&self, lv: usize, j: usize) -> usize { + self.level_starts[lv] + j + } + + /// A node program's estimate before it is emitted: the largest node seen, + /// or twice the largest leaf before the first (#1013's rule). + pub(crate) fn node_estimate(&self) -> u64 { + let leaf = self.leaf_estimates.iter().max().copied().unwrap_or(0); + self.node_seen + .load(std::sync::atomic::Ordering::SeqCst) + .max(2 * leaf) + } + + pub(crate) fn node_emitted(&self, bytes: u64) { + self.node_seen + .fetch_max(bytes, std::sync::atomic::Ordering::SeqCst); + } +} + +/// A node built ahead of the proofs below it: its artifacts (what its prove +/// needs; its program was published before them, [`NodeProgram`]). +pub(crate) struct TreeNode { + pub(crate) artifacts: super::registry::LfmArtifacts, + pub(crate) derived: DerivedChild, + /// Seconds emitting it and building its artifacts. + pub(crate) built: f64, + /// When its builder finished it, in seconds since the tree started. + pub(crate) built_at: f64, +} + +/// One level's readout: its wall, and per program (artifacts, prove) seconds +/// and its [`ProgramTimes`]. +pub(crate) struct LevelTiming { + pub(crate) wall: f64, + pub(crate) programs: Vec<(f64, f64)>, + pub(crate) times: Vec, +} + +/// When one program of the tree ran, in seconds since the tree started, and +/// its prove's split ([`take_prove_split`]: execute, fill, the prove net of +/// the card wait, the card wait) — the `W3 TIMES` readout. +#[derive(Clone, Copy, Default)] +pub(crate) struct ProgramTimes { + /// Its program emitted (nodes; the leaves' exist before the tree). + pub(crate) program_at: f64, + /// Seconds its emission took after its admission (nodes). + pub(crate) program_emit: f64, + /// Its program (nodes) and artifacts built. + pub(crate) built_at: f64, + /// Its worker took it. + pub(crate) start: f64, + /// Its proof done. + pub(crate) end: f64, + pub(crate) split: Option<(f64, f64, f64, f64)>, +} + +/// One `ALLOC PEAK ` line, when the watermark monitor runs: the phase's +/// highest VmRSS reading with jemalloc's allocated (live) and resident bytes +/// at it ([`crate::alloc_purge::peak_window`]), and the precomputed-tree +/// cache's live bytes now. Under the pipelined tree a level's window runs from +/// the line before to its last proof, so it overlaps the level above. +fn alloc_peak_line(say: &dyn Fn(&str), phase: &str) { + if let Some(peak) = crate::alloc_purge::peak_window() { + say(&format!( + "ALLOC PEAK {phase}: {peak} · tree cache live {:.2} GiB ({} entries)", + stark::prover::precomputed_tree_cache_live_bytes() as f64 / (1u64 << 30) as f64, + stark::prover::precomputed_tree_cache_entries() + )); + } +} + +/// The split of the prove that just ran on this thread. +pub(crate) fn prove_split_now() -> Option<(f64, f64, f64, f64)> { + take_prove_split().map(|s| (s.execute, s.fill, s.multi_prove, s.permit_wait)) +} + +/// One group's share of the proof, owned, as phase B hands it over +/// ([`block_whir::GroupObserver`]). +pub(crate) struct GroupMsg { + pub(crate) group: usize, + pub(crate) roots: Vec, + /// Every carried polynomial's commit-time answer, in root order. + pub(crate) commit_ood: Vec, + pub(crate) tables: + Vec>, + pub(crate) argue: + Option>, + pub(crate) opening: multilinear::stacked_eval::StackedProof< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >, + pub(crate) prepared: Option< + multilinear::stacked_eval::StackedProof< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >, + >, + pub(crate) at: f64, +} + +/// What the early leaf's thread hands back: the filled leaf, its arena (for +/// the off-the-clock check against the finished proof's), and when its +/// execute + fill ran. +pub(crate) type EarlyOut = (LfmFilled, Vec>, f64, f64); + +/// The first leaf whose groups phase B finished while another group was +/// still to come, executing and filling on a thread of its own. +pub(crate) struct EarlyLeaf { + pub(crate) leaf: usize, + /// When its last group's opening ended (seconds since the prove started). + pub(crate) complete_at: f64, + pub(crate) handle: std::thread::JoinHandle>, +} + +/// A value one thread publishes once and others wait for. A publisher that +/// unwinds before publishing leaves an error behind ([`FailUnpublished`]), so +/// no waiter blocks on a value that will never come. +pub(crate) struct Published(pub(crate) std::sync::OnceLock>); + +impl Published { + pub(crate) fn new() -> Self { + Self(std::sync::OnceLock::new()) + } + + pub(crate) fn wait(&self) -> Result<&T, String> { + self.0.wait().as_ref().map_err(Clone::clone) + } + + pub(crate) fn take(self) -> Result { + self.0 + .into_inner() + .unwrap_or_else(|| Err("never published".to_string())) + } +} + +/// The leaves' artifacts as their builder publishes them: per leaf, its +/// artifacts, its derived shape and the seconds building them. +pub(crate) type LeafBuilt = (super::registry::LfmArtifacts, DerivedChild, f64); + +/// Publishes an error into every slot still empty when it drops: a node +/// builder that stops early (an error, or a panic) leaves no level waiting +/// for a node that will never come. +pub(crate) struct FailUnpublished<'a, T>(pub(crate) &'a [Vec>]); + +impl Drop for FailUnpublished<'_, T> { + fn drop(&mut self) { + for slot in self.0.iter().flatten() { + let _ = slot.0.set(Err("the node builder stopped".to_string())); + } + } +} + +/// Publishes `why` into every slot of a row still empty when it drops: a +/// publisher that stops early (an error, or a panic) leaves no waiter hanging +/// on a value that will never come. +pub(crate) struct FailEmpty<'a, T>(pub(crate) &'a [Published], pub(crate) &'static str); + +impl Drop for FailEmpty<'_, T> { + fn drop(&mut self) { + for slot in self.0 { + let _ = slot.0.set(Err(self.1.to_string())); + } + } +} + +/// The leaves admitted beside phase B (one permit each, in leaf order; `None` +/// without a program budget): emitted on `threads` plain threads, off the rayon +/// pool phase B is using, and each published to its slot strictly in leaf +/// order, as soon as it and every leaf before it are emitted +/// ([`super::program_budget::emit_ordered`]). `published(k)` is called on this +/// thread after leaf k's slot is set. Returns once every admitted leaf's slot +/// holds a program or an error: after an emitter fails, its error is in its +/// slot and the slots after it say the emitter stopped, so no waiter blocks on +/// a program that will never come. The first error is the result. +pub(crate) fn emit_admitted_leaves( + slots: &[Published>], + permits: Vec>, + threads: usize, + emit: &(dyn Fn(usize) -> Result + Sync), + bytes: &(dyn Fn(&P) -> u64 + Sync), + published: &mut dyn FnMut(usize), +) -> Result<(), String> { + let upto = permits.len(); + let permits: Vec>> = + permits.into_iter().map(std::sync::Mutex::new).collect(); + let _fail = FailEmpty(&slots[..upto], "the leaf emitter stopped"); + let admitted = |k: usize| -> Result, String> { + let program = emit(k)?; + let mut permit = permits[k].lock().unwrap_or_else(|e| e.into_inner()).take(); + if let Some(permit) = permit.as_mut() { + permit.emitted(bytes(&program)); + } + Ok(ProgramSlot::new(program, permit)) + }; + let mut failed = None; + super::program_budget::emit_ordered(0..upto, threads, &admitted, &mut |k, slot| { + let emitted = match &slot { + Ok(_) => true, + Err(e) => { + failed = Some(e.clone()); + false + } + }; + let _ = slots[k].0.set(slot); + published(k); + emitted + }); + failed.map_or(Ok(()), Err) +} + +/// Each item's value built one after another on this thread and published to +/// its slot. While `ready(k)` says item k's input is already there, the values +/// are held and published all at once after the last (ULTRA 101: published one +/// by one, each build queued behind a sibling's prove on the card, and the 1× +/// tree's top waited 0.8 s for its last leaf). Before a build whose input is +/// not there yet, everything held is published, and each value from then on +/// as it is built (`each`: from the start), so nothing waits for an input +/// while holding what others need: under a program budget a missing input is +/// a program the budget admits only once the provers take the ones before it. +/// Returns the item where per-item publication began, if it did. A build that +/// fails is published at once and ends the run; every slot still empty when +/// this returns or unwinds gets an error ([`FailEmpty`]). +pub(crate) fn publish_each( + items: &[I], + slots: &[Published], + each: bool, + ready: impl Fn(usize) -> bool, + build: impl Fn(usize, &I) -> Result, +) -> Option { + let _fail = FailEmpty(slots, "the publisher stopped"); + let mut held = Vec::new(); + let mut from = each.then_some(0); + for ((k, item), slot) in items.iter().enumerate().zip(slots) { + if from.is_none() && !ready(k) { + from = Some(k); + for (one, slot) in held.drain(..).zip(slots) { + let _ = slot.0.set(one); + } + } + let one = build(k, item); + let failed = one.is_err(); + if from.is_none() && !failed { + held.push(one); + } else { + let _ = slot.0.set(one); + } + if failed { + break; + } + } + for (one, slot) in held.into_iter().zip(slots) { + let _ = slot.0.set(one); + } + from +} + +/// A tree's nodes built level by level from the leaves' shapes — leaf `i`'s +/// from `leaf(i)`, of `n_leaves`, which may wait for it to be published: a node +/// over leaves waits for its own children only — each node published to its +/// slot (level, node) as soon as it is built: `admit` lets a node in (its +/// level and index; under a program budget it may wait), then `emit` makes its +/// program from its level, index, children's shapes (in order), whether it is +/// the top and its admission, as the part published to `programs` at once (what +/// a node executes) and the part `finish` builds the rest of the node from (its +/// artifacts: it may take the card); `shape_of` gives a built node's shape for +/// the level above. A node depends on its children's shapes, never on their +/// proofs, so the whole tree can be built while the leaves prove. +/// +/// With `pool`, as many plain threads as the pool has take a level's nodes in +/// order, each admitting its node there and emitting it on the pool, and this +/// thread finishes each as its emission ends and publishes it into ITS OWN +/// slot, whatever order they finish in; without, admitted, emitted and finished +/// one after another here. An admission never waits on a rayon worker, nor +/// ahead of a finish another node's prover needs. `finish` always runs on this +/// thread, never on a rayon worker: a worker that waits inside rayon while it +/// holds the card runs queued jobs meanwhile, and a sibling that takes the card +/// is a second hold on one thread (BIG 569). A node's program is published as +/// its emission ends, before its finish starts and whatever the other nodes' +/// finishes are waiting for. A build that fails leaves its error in its slots, +/// and every slot still empty when this returns or unwinds gets one +/// ([`FailUnpublished`]). +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_levels<'l, C: Sync, A: Send, E: Send + Sync, P: Send, N: Send + Sync>( + shape: &[super::per_table_aggregator::Level], + leaf: &(dyn Fn(usize) -> Result<&'l C, String> + Sync), + n_leaves: usize, + programs: &[Vec>], + slots: &[Vec>], + shape_of: impl Fn(&N) -> &C + Sync, + admit: impl Fn(usize, usize) -> Result + Sync, + emit: impl Fn(usize, usize, &[&C], bool, A) -> Result<(E, P), String> + Sync, + finish: impl Fn(usize, usize, P) -> Result, + pool: Option<&rayon::ThreadPool>, +) { + let _fail_programs = FailUnpublished(programs); + let _fail = FailUnpublished(slots); + // Publishes an emitted node's program, and hands on what its finish takes. + let publish = |lv: usize, j: usize, emitted: Result<(E, P), String>| match emitted { + Ok((program, rest)) => { + let _ = programs[lv][j].0.set(Ok(program)); + Ok(rest) + } + Err(e) => { + let _ = programs[lv][j].0.set(Err(e.clone())); + Err(e) + } + }; + for (lv, arities) in shape.iter().enumerate() { + let top = lv + 1 == shape.len(); + // The level below's shapes. Over the leaves each node waits for its + // own children as it is emitted (`leaf`); a node level is whole by the + // time the next one starts. + let below: Option> = match lv { + 0 => None, + _ => match slots[lv - 1] + .iter() + .map(|s| s.wait().map(&shape_of)) + .collect::, String>>() + { + Ok(below) => Some(below), + Err(_) => return, + }, + }; + let kids_of = |range: std::ops::Range| -> Result, String> { + match &below { + Some(below) => Ok(below[range].to_vec()), + None => range.map(leaf).collect(), + } + }; + let n_below = below.as_ref().map_or(n_leaves, Vec::len); + let mut at = 0usize; + let groups: Vec> = arities + .arities + .iter() + .map(|&a| { + at += a; + at - a..at + }) + .collect(); + if groups.last().map_or(0, |g| g.end) != n_below { + let _ = slots[lv][0].0.set(Err(format!( + "level {}: arities cover {at} of {n_below} children", + lv + 1 + ))); + return; + } + // A node's kids (over the leaves, each waited for), its admission, + // then its emission (on the pool, when there is one). + let made = |j: usize, on: Option<&rayon::ThreadPool>| -> Result<(E, P), String> { + let kids = kids_of(groups[j].clone())?; + let admitted = admit(lv, j)?; + match on { + Some(pool) => pool.install(|| emit(lv, j, &kids, top, admitted)), + None => emit(lv, j, &kids, top, admitted), + } + }; + match pool { + Some(pool) => { + let (tx, rx) = std::sync::mpsc::channel::<(usize, Result)>(); + let next = std::sync::atomic::AtomicUsize::new(0); + std::thread::scope(|scope| { + for _ in 0..pool.current_num_threads().max(1) { + let (tx, made, publish, next, nodes) = + (tx.clone(), &made, &publish, &next, groups.len()); + // Each program is published here, as its emission + // ends: never behind another node's finish, which may + // wait for the card (BIG 622: a program queued 2.1 s + // behind a sibling's artifacts). + scope.spawn(move || { + loop { + let j = next.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if j >= nodes { + return; + } + if tx.send((j, publish(lv, j, made(j, Some(pool))))).is_err() { + return; + } + } + }); + } + drop(tx); + // This thread, not a pool worker, finishes each node as + // its program arrives. + for (j, rest) in rx { + let _ = slots[lv][j].0.set(rest.and_then(|p| finish(lv, j, p))); + } + }); + } + None => { + for (j, slot) in slots[lv].iter().enumerate() { + let rest = publish(lv, j, made(j, None)); + let _ = slot.0.set(rest.and_then(|p| finish(lv, j, p))); + } + } + } + } +} + +/// The tree's node programs and artifacts ([`build_levels`] over the leaves' +/// derived shapes): each node's program published to `programs` as soon as it +/// is emitted (what its execute and fill need), and the node with its +/// artifacts to `slots` once they are built (what its prove needs). Under a +/// program `budget` each node is admitted in prove order before it is emitted +/// ([`TreeBudget`]), its permit travelling with its program. +/// +/// With `pool` (`W3_NODE_PIPE`, on by default) a level's programs are emitted +/// together on the pool's threads — the builder's own, not the global pool, on +/// which a prover's join could steal an emission and leave the card idle +/// (#1013's FAST 454) — and this thread builds each one's artifacts (holding +/// the card) as it arrives; each level proves as its own nodes arrive. Without +/// it, one after another on this thread (the control: the builder before, +/// whose whole tree level 1 then waits for — at the median 32 s of serial +/// builds holding level 0 open 10.7 s past its last proof, BIG 565 / 568). +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_nodes( + plan: &WhirBlockPlan, + shape: &[super::per_table_aggregator::Level], + leaves: &[Published], + programs: &[Vec>], + slots: &[Vec>], + wrap: &crate::ProofOptions, + words: usize, + t_tree: std::time::Instant, + pool: Option<&rayon::ThreadPool>, + budget: Option<&TreeBudget>, +) { + type Emitted = (NodeProgram, (std::sync::Arc, f64)); + let leaf = |i: usize| -> Result<&DerivedChild, String> { + leaves + .get(i) + .ok_or_else(|| format!("no leaf {i}"))? + .wait() + .map(|(_, derived, _)| derived) + }; + build_levels( + shape, + &leaf, + leaves.len(), + programs, + slots, + |node: &TreeNode| &node.derived, + // Under a program budget each node is admitted in prove order before + // it is emitted: on an emitter's plain thread, never the pool's. + |lv, j| match budget { + Some(b) => b + .budget + .acquire(b.node_order(lv, j), b.node_estimate()) + .map(Some), + None => Ok(None), + }, + |_, _, kids: &[&DerivedChild], top, permit| -> Result { + let t = std::time::Instant::now(); + let program = plan.node_program(kids, top)?; + let child_arenas = kids + .iter() + .map(|kid| { + super::per_table_aggregator::leg_arena_count(&kid.shape(), BlockFamily::WRAP) + }) + .collect(); + let mut permit: Option = permit; + if let (Some(b), Some(permit)) = (budget, permit.as_mut()) { + let bytes = program_bytes(&program); + b.node_emitted(bytes); + permit.emitted(bytes); + } + let slot = ProgramSlot::new(program, permit); + let program = slot.get()?; + let at = t_tree.elapsed().as_secs_f64(); + let emit = t.elapsed().as_secs_f64(); + Ok(( + NodeProgram { + program: slot, + at, + emit, + child_arenas, + }, + (program, emit), + )) + }, + |_, _, (program, emitted): (std::sync::Arc, f64)| -> Result { + let t = std::time::Instant::now(); + let artifacts = artifacts_of(&program.program, wrap); + let derived = super::whir_block::derived_of(&artifacts, wrap, words)?; + Ok(TreeNode { + artifacts, + derived, + built: emitted + t.elapsed().as_secs_f64(), + built_at: t_tree.elapsed().as_secs_f64(), + }) + }, + pool, + ); +} + +/// A proved program of the tree, as [`prove_dataflow`] publishes it: its +/// proof, its harvest for the level above, its prove's seconds (artifacts +/// wait, execute, fill and prove), its build seconds (nodes) and its stamps. +pub(crate) struct ProvedProgram { + pub(crate) lfm: LfmProof, + pub(crate) child: HarvestedChild, + pub(crate) prove: f64, + pub(crate) built: f64, + pub(crate) times: ProgramTimes, +} + +/// One program of the tree: its level (0 = the leaves) and its index there. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TreeAt { + pub(crate) lv: usize, + pub(crate) j: usize, +} + +/// Every program of a tree of `leaves` leaves laid out as `shape`, in +/// topological order: the leaves, then each node level in turn. Workers that +/// take programs in this order reach a node only after every program before it +/// — its children among them — has been taken, so the earliest unfinished +/// program waits on nothing untaken: no deadlock, at any number of workers. +pub(crate) fn tree_order( + leaves: usize, + shape: &[super::per_table_aggregator::Level], +) -> Vec { + let mut order: Vec = (0..leaves).map(|j| TreeAt { lv: 0, j }).collect(); + for (l, level) in shape.iter().enumerate() { + order.extend((0..level.arities.len()).map(|j| TreeAt { lv: l + 1, j })); + } + order +} + +/// `order` ([`tree_order`]) with leaf `first`, when it is one, taken before +/// the other leaves: a permutation of the leaves alone, so every node still +/// comes after every leaf and the order stays topological. +pub(crate) fn early_leaf_first(mut order: Vec, first: Option) -> Vec { + let at = first.and_then(|j| order.iter().position(|&t| t == TreeAt { lv: 0, j })); + if let Some(at) = at { + order[..=at].rotate_right(1); + } + order +} + +/// `W3 LEAF FIRST`: whether the leaves were offered to phase B and when, what +/// phase B did with them (the turn it read the offer, the leaf it led with and +/// how many groups it moved) and which leaf was early. +fn leaf_first_line( + on: bool, + offered_at: Option, + stated_at: f64, + taken: Option, + early: Option, +) -> String { + if !on { + return "W3 LEAF FIRST: off (phase B in its own order; the tree takes the leaves in index \ + order)" + .to_string(); + } + let offered = offered_at.map_or("never".to_string(), |t| { + format!("{t:.2}s ({:.2}s after the statement)", t - stated_at) + }); + let phase_b = match taken { + None => "phase B ended before it read them".to_string(), + Some(t) => match t.leaf { + Some(leaf) => format!( + "phase B read them at its turn {} ({:.2}s in) and moved leaf {leaf}'s {} groups next", + t.turn, + t.after, + t.moved.len() + ), + None => format!( + "phase B read them at its turn {} ({:.2}s in); no leaf could lead (each holds a \ + rebuilt group, or none was left)", + t.turn, t.after + ), + }, + }; + format!( + "W3 LEAF FIRST: on · leaves offered at {offered} · {phase_b} · early leaf {} (taken first by \ + the tree)", + early.map_or("none".to_string(), |k| k.to_string()) + ) +} + +/// Node `at`'s children, as indices into level `at.lv − 1`. +pub(crate) fn tree_children( + shape: &[super::per_table_aggregator::Level], + at: TreeAt, +) -> std::ops::Range { + let arities = &shape[at.lv - 1].arities; + let start: usize = arities[..at.j].iter().sum(); + start..start + arities[at.j] +} + +/// Proves every program of a tree on `workers` threads, taking them in `order` +/// ([`tree_order`]) and publishing each result into its slot (`results[lv][j]`, +/// level 0 the leaves): `prove` gets the program and its children's slots in +/// order (none for a leaf), to wait for as it needs them ([`wait_all`]), so a +/// node whose children are done proves while the rest of the level below still +/// does — and one that streams can start on its first child. With +/// `barrier` (the control, `W3_DATAFLOW=0`), a node first waits for the whole +/// level below, as level-by-level proving did. A failed or panicking prove +/// leaves an error in its slot and so in every slot above it; every slot still +/// empty when this returns or unwinds gets one. +pub(crate) fn prove_dataflow( + shape: &[super::per_table_aggregator::Level], + order: &[TreeAt], + results: &[Vec>], + workers: usize, + barrier: bool, + prove: impl Fn(TreeAt, &[&Published]) -> Result + Sync, +) { + use super::tree_run::in_index_order; + let _fail = FailUnpublished(results); + in_index_order(order.len(), workers, |i| { + let at = order[i]; + let kids = match at.lv { + 0 => Ok(Vec::new()), + _ => { + let below = &results[at.lv - 1]; + let all = match barrier { + true => below.iter().try_for_each(|s| s.wait().map(drop)), + false => Ok(()), + }; + all.map(|()| { + tree_children(shape, at) + .map(|c| &below[c]) + .collect::>() + }) + } + }; + let result = kids.and_then(|kids| { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| prove(at, &kids))) + .unwrap_or_else(|payload| { + let why = payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_default(); + Err(format!("level {} program {}: panicked: {why}", at.lv, at.j)) + }) + }); + let _ = results[at.lv][at.j].0.set(result); + }); +} + +/// The top node streamed: its execution starts on the first of its children to +/// be proved and runs each child's share as that child lands +/// ([`super::executor::StreamedExecution`]: the witness is the unstreamed +/// one's, word for word), so only the last child's share and the fill remain +/// after the last child. A node's arenas are its children's, in child order: +/// child `c` supplies `child_arenas[c]` of them ([`NodeProgram::child_arenas`]) +/// and they are group `c`. The counts differ between children whose AIR sets +/// differ (under the equal-height merge each child's merged tables follow its +/// own heights), so they are never an equal split. The execute the split +/// reports is the part after the last child landed. Returns the filled traces, +/// for a prove under artifacts built for `hasher`. +pub(crate) fn execute_streamed( + program: &LfmProgram, + child_arenas: &[usize], + kids: &[&Published], + hasher: crate::lfm::hash::HasherKind, +) -> Result { + let (execution, after) = execute_streamed_with(program, child_arenas, kids, &hasher, |kid| { + try_child_arena_words(&kid.child) + })?; + Ok(super::proof::lfm_fill_executed( + program, execution, hasher, after, + )) +} + +/// [`execute_streamed`]'s execution, over children of any kind: `arenas_of` gives +/// a landed child's arenas. Returns the execution and the seconds it took +/// after the last child landed. Refuses counts that do not cover the +/// program's arenas, one count a child. +pub(crate) fn execute_streamed_with( + program: &LfmProgram, + child_arenas: &[usize], + kids: &[&Published], + hasher: &(impl super::hash::LfmHasher + Sync), + arenas_of: impl Fn(&R) -> Result>, String>, +) -> Result<(super::executor::LfmExecution, f64), String> { + use super::executor::StreamedExecution; + let k = kids.len(); + let arenas = program.arena_schema.lens.len(); + if k == 0 || child_arenas.len() != k || child_arenas.iter().sum::() != arenas { + return Err(format!( + "{arenas} arenas over {k} children of {child_arenas:?} arenas" + )); + } + let group_of: Vec = child_arenas + .iter() + .enumerate() + .flat_map(|(c, &n)| std::iter::repeat_n(c, n)) + .collect(); + let held: Vec>>> = + (0..k).map(|_| std::cell::OnceCell::new()).collect(); + let mut ex = StreamedExecution::new(program, group_of, hasher).map_err(|e| format!("{e:?}"))?; + let mut last = std::time::Instant::now(); + in_arrival_order(kids, |c, kid| { + let words = match held[c].get() { + Some(words) => words, + None => { + let words = arenas_of(kid)?; + held[c].get_or_init(|| words) + } + }; + last = std::time::Instant::now(); + ex.land(c, words) + .map_err(|e| format!("child {c} lands: {e:?}")) + })?; + let execution = ex.finish().map_err(|e| format!("{e:?}"))?; + Ok((execution, last.elapsed().as_secs_f64())) +} + +/// One node of the tree: its execute and fill (`execute`) from its program as +/// soon as the builder has emitted it, and its prove (`prove`) only once its +/// OWN artifacts are in its slot — the program in `programs[lv − 1][j]`, the +/// artifacts in `slots[lv − 1][j]`. With `early` off (the control, +/// `W3_EXEC_EARLY=0`), the execute also waits for the artifacts, as when one +/// slot held both. The artifacts are a card hold that queues behind the +/// proves below (BIG 617: the top's came 0.12-0.15 s after its last child +/// although its program was emitted 0.03-2.1 s before it). +pub(crate) fn node_flow( + programs: &[Vec>], + slots: &[Vec>], + at: TreeAt, + early: bool, + execute: impl FnOnce(&E) -> Result, + prove: impl FnOnce(F, &N) -> Result, +) -> Result { + let (program, node) = (&programs[at.lv - 1][at.j], &slots[at.lv - 1][at.j]); + let program = program.wait()?; + if !early { + node.wait()?; + } + let filled = execute(program)?; + prove(filled, node.wait()?) +} + +/// Whether the top streams: only if a child is still unproved when the top can +/// execute (its program emitted; with `W3_EXEC_EARLY=0`, its artifacts built +/// too). With every child already proved there is nothing to stream behind, +/// and the streamed path (its forward pass, then a wave per child) costs more +/// than one whole execution: at 1× the top's artifacts come after its last +/// child, and streaming it anyway cost 0.19 s (BIG 617 / 618). A failed child +/// counts as done: the whole path reports its error. +pub(crate) fn top_streams(kids: &[&Published]) -> bool { + kids.iter().any(|k| k.0.get().is_none()) +} + +/// Calls `land` on each child's result as it is published, in the order they +/// arrive, each exactly once; a child that failed fails this with its error. +pub(crate) fn in_arrival_order( + kids: &[&Published], + mut land: impl FnMut(usize, &R) -> Result<(), String>, +) -> Result<(), String> { + let mut landed = vec![false; kids.len()]; + while landed.contains(&false) { + let mut moved = false; + for (c, kid) in kids.iter().enumerate() { + if landed[c] { + continue; + } + match kid.0.get() { + None => {} + Some(Err(e)) => return Err(e.clone()), + Some(Ok(r)) => { + land(c, r)?; + landed[c] = true; + moved = true; + } + } + } + if !moved { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + Ok(()) +} + +/// Every child's result, each waited for in order. +pub(crate) fn wait_all<'r, R>(kids: &[&'r Published]) -> Result, String> { + kids.iter().map(|k| k.wait()).collect() +} + +/// The tree proved the way a prover would run it, with the leaves' programs in +/// `leaves`: those emitted beforehand (while phase B ran) already published, +/// the rest emitted here as the program `budget` admits them, on +/// [`DEFERRED_LEAF_EMITTERS`] streaming threads in leaf order: +/// 1. the leaves' artifacts, one after another on a thread of their own (each +/// holds the card), all published at once after the last while every +/// leaf's program is there, and each as it is built from the first whose +/// program is not ([`publish_each`]; `leaves_each`: each throughout); +/// 2. the nodes' programs and artifacts — functions of the leaves' artifacts, +/// not of any proof — on a thread of their own ([`build_nodes`]), each +/// admitted under the budget, then published to its slot as it is built, a +/// node over leaves once its own leaves' artifacts are, beside +/// 3. every program proved `siblings` at a time, leaves first, each harvested +/// without its verify ([`prove_dataflow`]): with `beside`, each leaf +/// executes and fills its traces while the artifacts are built (neither +/// reads them) and waits for its own only to prove; without, it waits for +/// every leaf's first (the order before, `W3_EXEC_BESIDE_ARTIFACTS=0`); each +/// node as soon as its own children are proved and its program is in its +/// slot — with `dataflow` off (the control), once its whole level below is +/// proved; without `node_pipe` (that control), once the builder has built +/// the whole tree. +/// +/// Each program's prover claims it as it takes it and lets it go once it is +/// done with it — a leaf once proved, a node once filled (its artifacts' build +/// holds its own reference) — so a program lives from its emission to its +/// proof, and its admission with it. After the last proof of a level below the +/// top, `LAMBDA_VM_TREE_LEVEL_PURGE` may purge the allocator +/// ([`crate::alloc_purge::purge_after_level`], its line through `say`). +/// +/// `node_pipe` is the builder's own pool ([`build_nodes`]); `early`, when +/// given, is a leaf that already executed and filled while phase B ran +/// ([`EarlyLeaf`]): its worker joins it instead. +/// +/// Returns each level's timing and every proof with its artifacts, level by +/// level, the top last — for the harness to verify off the clock — the early +/// leaf's arena, if any, and the leaf where per-leaf publication began, if it +/// did. +#[allow(clippy::type_complexity, clippy::too_many_arguments)] +pub(crate) fn prove_tree_pipelined( + plan: &WhirBlockPlan, + proof: &BlockWhirProof, + leaves: Vec>, + budget: Option<&TreeBudget>, + siblings: usize, + beside: bool, + leaves_each: bool, + early: Option, + early_first: bool, + node_pipe: Option<&rayon::ThreadPool>, + dataflow: bool, + stream_top: Option<&std::sync::OnceLock>, + exec_early: bool, + say: &(dyn Fn(&str) + Sync), +) -> Result< + ( + Vec, + Vec<(super::registry::LfmArtifacts, LfmProof)>, + Option<(usize, Vec>, f64, f64, f64)>, + Option, + ), + String, +> { + // With `early_first` (`W3_LEAF_FIRST`), the early leaf is taken first. + let first = early.as_ref().map(|e| e.leaf).filter(|_| early_first); + let early = std::sync::Mutex::new(early); + let early_out: std::sync::Mutex>, f64, f64, f64)>> = + std::sync::Mutex::new(None); + let wrap = aggregation_wrap_options(); + let words = plan.child_layout().total(); + let t_tree = std::time::Instant::now(); + let shape = plan.levels(); + let mut timings = Vec::with_capacity(shape.len() + 1); + let mut proofs = Vec::new(); + // Each leaf's artifacts ([`publish_each`]). + let built: Vec> = leaves.iter().map(|_| Published::new()).collect(); + let leaf_built_at: Vec> = + leaves.iter().map(|_| std::sync::Mutex::new(0.0)).collect(); + let slots: Vec>> = shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect(); + let programs: Vec>> = shape + .iter() + .map(|l| l.arities.iter().map(|_| Published::new()).collect()) + .collect(); + + let order = early_leaf_first(tree_order(leaves.len(), &shape), first); + let results: Vec>> = std::iter::once(leaves.len()) + .chain(shape.iter().map(|l| l.arities.len())) + .map(|n| (0..n).map(|_| Published::new()).collect()) + .collect(); + + // The programs a level still has to prove, for the purge after its last. + let remaining: Vec = results + .iter() + .map(|level| std::sync::atomic::AtomicUsize::new(level.len())) + .collect(); + + let (n_leaves, each_from) = std::thread::scope(|outer| -> Result<_, String> { + // 0. the leaves the budget left to the tree, emitted in leaf order as + // it admits them, on plain threads (an admission may wait). + let deferred = leaves.iter().position(|slot| slot.0.get().is_none()); + if let Some(from) = deferred { + let (leaves, emit_leaf) = (&leaves, move |k: usize| -> Result { + let b = budget.ok_or("a leaf left to the tree without a program budget")?; + let mut permit = b.budget.acquire(k, b.leaf_estimate(k))?; + let program = plan.leaf_program(k)?; + permit.emitted(program_bytes(&program)); + Ok(ProgramSlot::new(program, Some(permit))) + }); + outer.spawn(move || { + let _fail = FailEmpty(&leaves[from..], "the leaf emitter stopped"); + super::program_budget::emit_ordered( + from..leaves.len(), + DEFERRED_LEAF_EMITTERS, + &emit_leaf, + &mut |k, slot| { + let emitted = slot.is_ok(); + let _ = leaves[k].0.set(slot); + emitted + }, + ); + }); + } + // 2. the nodes' programs and artifacts, into their slots. + let builder = outer.spawn(|| { + build_nodes( + plan, &shape, &built, &programs, &slots, &wrap, words, t_tree, node_pipe, budget, + ) + }); + let each_from = std::thread::scope(|scope| { + // 1. the leaves' artifacts, one after another on this thread: each + // build holds the card throughout (armed), so building them as + // rayon jobs bought no overlap, and a holder on a rayon worker can + // run a sibling build that takes the card again (BIG 569). + let artifacts = scope.spawn(|| { + publish_each( + &leaves, + &built, + leaves_each, + |k| leaves[k].0.get().is_some(), + |k, slot| { + let program = slot.wait()?.get()?; + let t = std::time::Instant::now(); + let artifacts = artifacts_of(&program.program, &wrap); + drop(program); + let derived = super::whir_block::derived_of(&artifacts, &wrap, words)?; + *leaf_built_at[k] + .lock() + .map_err(|_| "a built-at stamp is poisoned")? = + t_tree.elapsed().as_secs_f64(); + Ok((artifacts, derived, t.elapsed().as_secs_f64())) + }, + ) + }); + // 3. every program of the tree, `siblings` at a time, leaves first; + // each node as soon as its children are proved and its program is + // in its slot ([`prove_dataflow`]). + let prove_one = |at: TreeAt, + kids: &[&Published]| + -> Result { + let start = t_tree.elapsed().as_secs_f64(); + if at.lv == 0 { + let k = at.j; + // The leaf's program, once emitted: this prover takes it. + let program = leaves[k].wait()?; + program.claim(); + let ahead = { + let mut slot = early.lock().map_err(|_| "the early slot is poisoned")?; + if slot.as_ref().is_some_and(|e| e.leaf == k) { + slot.take() + } else { + None + } + }; + let t = std::time::Instant::now(); + let filled = match ahead { + Some(e) => { + let (filled, arena, started, secs) = e + .handle + .join() + .map_err(|_| format!("leaf {k}: the early executor panicked"))??; + *early_out + .lock() + .map_err(|_| "the early readout is poisoned")? = + Some((k, arena, e.complete_at, started, secs)); + filled + } + None => { + let arenas = block_leaf_arena(plan, proof, k)?; + if !beside { + for slot in &built { + slot.wait()?; + } + } + let program = program.get()?; + lfm_execute_and_fill( + &program.program, + &arenas, + program.program.hasher(::SOCKET), + ) + .map_err(|e| format!("leaf {k}: {e:?}"))? + } + }; + let artifacts = &built[k].wait()?.0; + let lfm = + filled.prove_under::(artifacts, &wrap, decide_lfm_residency()); + // Proved, so its artifacts are built: nothing reads + // the program again. + program.release(); + let lfm = lfm.map_err(|e| format!("leaf {k}: {e:?}"))?; + let prove = t.elapsed().as_secs_f64(); + let times = ProgramTimes { + program_at: 0.0, + program_emit: 0.0, + built_at: 0.0, + start, + end: t_tree.elapsed().as_secs_f64(), + split: prove_split_now(), + }; + let child = + harvest_child_under::(artifacts.clone(), wrap.clone(), &lfm)?; + return Ok(ProvedProgram { + lfm, + child, + prove, + built: 0.0, + times, + }); + } + // The control of the node pipeline: no node proves before + // the whole tree is built, as when the builder joined level 0. + if node_pipe.is_none() { + for slot in slots.iter().flatten() { + slot.wait()?; + } + } + // Execute and fill from the program as soon as it is + // emitted, prove once the node's own artifacts are built + // ([`node_flow`]); traces filled under the socket its hash + // rows run on, which the node's artifacts are built for (the + // prove asserts it). + node_flow( + &programs, + &slots, + at, + exec_early, + |emitted: &NodeProgram| -> Result<_, String> { + let t = std::time::Instant::now(); + // This prover takes the node's program; once filled, + // nothing but its artifacts' build (which holds its + // own reference) reads it again. + emitted.program.claim(); + let program = emitted.program.get()?; + let hasher = program.program.hasher(::SOCKET); + // The top streams only if a child is still unproved + // now that it can execute; the choice is kept for + // the readout. + let streamed = match stream_top { + Some(chose) if at.lv == shape.len() => { + let streams = top_streams(kids); + let _ = chose.set(streams); + streams + } + _ => false, + }; + let filled = if streamed { + execute_streamed(&program.program, &emitted.child_arenas, kids, hasher) + .map_err(|e| format!("the top, streamed: {e}"))? + } else { + let kids = wait_all(kids)?; + let mut arenas: Vec> = Vec::new(); + for k in &kids { + arenas.extend(try_child_arena_words(&k.child)?); + } + lfm_execute_and_fill(&program.program, &arenas, hasher) + .map_err(|e| format!("level {} node {}: {e:?}", at.lv, at.j))? + }; + drop(program); + emitted.program.release(); + Ok((filled, t, (emitted.at, emitted.emit))) + }, + |(filled, t, (program_at, program_emit)), node: &TreeNode| { + let lfm = filled + .prove_under::( + &node.artifacts, + &wrap, + decide_lfm_residency(), + ) + .map_err(|e| format!("level {} node {}: {e:?}", at.lv, at.j))?; + let prove = t.elapsed().as_secs_f64(); + let times = ProgramTimes { + program_at, + program_emit, + built_at: node.built_at, + start, + end: t_tree.elapsed().as_secs_f64(), + split: prove_split_now(), + }; + let child = harvest_child_under::( + node.artifacts.clone(), + wrap.clone(), + &lfm, + )?; + Ok(ProvedProgram { + lfm, + child, + prove, + built: node.built, + times, + }) + }, + ) + }; + let target = block_whir::memory_target_bytes(); + prove_dataflow(&shape, &order, &results, siblings, !dataflow, |at, kids| { + let proved = prove_one(at, kids); + // The level's last program: its freed pages back to the OS + // before the levels above go on, where the host is short. + let left = remaining[at.lv].fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + if left == 1 { + alloc_peak_line(say, &format!("level-{}", at.lv)); + } + if left == 1 && at.lv + 1 < remaining.len() { + crate::alloc_purge::purge_after_level(at.lv, target, say); + } + proved + }); + // Every program is proved or failed: an emitter still waiting for + // an admission (only after a failure) returns an error instead. + if let Some(b) = budget { + b.budget.fail(); + } + artifacts + .join() + .map_err(|_| "the leaves' artifacts builder panicked".to_string()) + })?; + for slot in &built { + slot.wait()?; + } + builder + .join() + .map_err(|_| "the node builder panicked".to_string())?; + Ok((built.len(), each_from)) + })?; + // Each level's readout: a level's wall is from the level below's last + // proof to its own (level 0: from the tree's start), now that a node may + // prove before its level below is done. + let leaf_secs: Vec = built + .iter() + .map(|slot| slot.wait().map(|(_, _, s)| *s)) + .collect::>()?; + let mut level_proofs: Vec> = Vec::with_capacity(results.len()); + let mut below_end = 0.0f64; + for (lv, level) in results.into_iter().enumerate() { + let proved: Vec = level + .into_iter() + .map(Published::take) + .collect::>()?; + let mut times = Vec::with_capacity(proved.len()); + let mut programs = Vec::with_capacity(proved.len()); + for (j, p) in proved.iter().enumerate() { + let built = match lv { + 0 => leaf_secs[j], + _ => p.built, + }; + let built_at = match lv { + 0 => *leaf_built_at[j] + .lock() + .map_err(|_| "a built-at stamp is poisoned")?, + _ => p.times.built_at, + }; + times.push(ProgramTimes { + built_at, + ..p.times + }); + programs.push((built, p.prove)); + } + let end = proved.iter().map(|p| p.times.end).fold(0.0, f64::max); + timings.push(LevelTiming { + wall: end - below_end, + programs, + times, + }); + below_end = end; + level_proofs.push(proved.into_iter().map(|p| p.lfm).collect()); + } + if level_proofs.last().map(Vec::len) != Some(1) { + return Err(format!( + "the tree closes to {:?} nodes", + level_proofs.last().map(Vec::len) + )); + } + // Off the clock: every proof with its artifacts, level by level. + let mut level_proofs = level_proofs.into_iter(); + let leaf_lfms = level_proofs.next().unwrap_or_default(); + debug_assert_eq!(leaf_lfms.len(), n_leaves); + for (slot, lfm) in built.into_iter().zip(leaf_lfms) { + proofs.push((slot.take()?.0, lfm)); + } + for (level, lfms) in slots.into_iter().zip(level_proofs) { + for (slot, lfm) in level.into_iter().zip(lfms) { + proofs.push((slot.take()?.artifacts, lfm)); + } + } + let early_out = early_out + .into_inner() + .map_err(|_| "the early readout is poisoned")?; + Ok((timings, proofs, early_out, each_from)) +} + +// ================================ the driver ============================== + +/// The whole block's walls, in seconds. +#[derive(Clone, Debug, Default)] +pub struct WhirTreeTimes { + pub base: f64, + /// The tree, from its start to the top's proof. + pub tree: f64, + /// From before the base to the top's proof, the readouts inside. + pub whole: f64, + /// The readouts printed between the base and the tree (on the clock). + pub readouts: f64, +} + +/// The early leaf's readout: its index, its arena (for the harness's check +/// against the finished proof's), when its groups were complete, when its +/// execute + fill started and how long it took (seconds). +#[cfg(test)] +pub(crate) type EarlyReadout = (usize, Vec>, f64, f64, f64); + +/// What a whole W3 block run hands back. +pub struct WhirTreeRun { + /// The block's statement, as a consumer receives it with the top proof. + pub statement: block_whir::OwnedBlockStatement, + /// Every proof of the tree with its artifacts, level by level, the top + /// last. + pub proofs: Vec<(super::registry::LfmArtifacts, LfmProof)>, + pub times: WhirTreeTimes, + /// The base proof (the harness verifies it, and the early leaf's arena + /// against it, off the clock). + #[cfg(test)] + pub(crate) base_proof: BlockWhirProof, + #[cfg(test)] + pub(crate) plan: WhirBlockPlan, + #[cfg(test)] + pub(crate) early: Option, +} + +impl WhirTreeRun { + /// The top node's proof. + pub fn top(&self) -> Option<&LfmProof> { + self.proofs.last().map(|(_, p)| p) + } +} + +/// What the planner thread hands back: the plan, the leaves' programs, when +/// the statement came and when the programs were ready, and the early leaf. +type Planned = ( + WhirBlockPlan, + Vec>, + usize, + Option, + f64, + Option, + f64, + Option, +); + +/// ★ W3 on a real block, run the way a prover runs it: +/// - the block proved in one proof at the production format; +/// - the moment its statement is final (after phase A), the plan derived and +/// the leaves' programs emitted on threads of their own, beside phase B; +/// - then the tree ([`prove_tree_pipelined`]): the leaves `siblings` at a time, +/// the nodes' programs built beside them. +/// +/// Prints the harness's `W3 …` lines, up to `W3 RECURSION`. A failure anywhere +/// is an error. +pub fn prove_whir_block_tree( + elf: &[u8], + input: &[u8], + cfg: &WhirTreeConfig, + sink: &dyn WhirTreeSink, +) -> Result { + let (leaves, siblings, fan_in, beside, early_on, leaf_emitters) = ( + cfg.leaves, + cfg.siblings, + cfg.fan_in, + cfg.beside, + cfg.early_on, + cfg.leaf_emitters, + ); + let (argue, format, options) = (cfg.argue, cfg.format, &cfg.options); + sink.line(&posture_line()); + sink.line(&base_line(&format.zf.base)); + sink.line(&format!("W3 ARGUE: {argue:?}")); + sink.line(&format!("W3 GROUP OPENING: {:?}", format.group)); + sink.line(&format!("BLOCK UPLOAD AHEAD: {}", options.upload_ahead)); + sink.line(&finish_idle_line()); + sink.line(&format!( + "BLOCK REST LAYOUT CONFIG: {} · KECCAK_RND built as its tables {} · finish packed {} · building at most {}", + block_whir::rest_layout_label(options, block_whir::rest_slide()), + options.finish_keccak_rnd_chunks, + options.pack_finished, + options + .rest_build_bytes + .map_or("the gate".to_string(), |b| format!("{} MiB", b >> 20)), + )); + let opts = super::proof::block_base_options(); + super::device_permit::arm(siblings); + sink.line(&format!( + "W3 CONFIG: leaves {leaves:?} · siblings {siblings} · fan-in {fan_in} · layout workers {} (ahead {:?}) · streamed ops dropped {}", + options.layout_workers, + options.layout_ahead, + options.drop_streamed_ops + )); + + // `LAMBDA_VM_ALLOC_WATERMARK`: freed pages back to the OS inside a phase, + // only where the host is short (`alloc_purge`). + let watermark = crate::alloc_purge::start_watermark(block_whir::memory_target_bytes()); + let t0 = std::time::Instant::now(); + // The wall clock at the prove's start, to place a box's memory samples. + sink.line(&format!( + "W3 PROVE START: unix {:.3}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()) + )); + let sender = std::sync::Mutex::new(None::>); + let (tx, rx) = std::sync::mpsc::channel::<( + block_whir::OwnedBlockStatement, + Vec, + )>(); + *sender.lock().unwrap_or_else(|e| e.into_inner()) = Some(tx); + let observe = |statement: block_whir::BlockStatement<'_>, + roots: &[block_whir::PreparedRoots]| { + if let Some(tx) = sender.lock().unwrap_or_else(|e| e.into_inner()).as_ref() { + let _ = tx.send((statement.to_owned(), roots.to_vec())); + } + }; + // Each group's share of the proof as its opening ends, for the early leaf. + let group_sender = std::sync::Mutex::new(None::>); + let (gtx, grx) = std::sync::mpsc::channel::(); + // Without the early leaf no sender is kept, so the planner's loop ends + // at once. + *group_sender.lock().unwrap_or_else(|e| e.into_inner()) = early_on.then_some(gtx); + let on_group = |opened: stark::multilinear_block::GroupOpened<'_, _, _>| { + if let Some(tx) = group_sender + .lock() + .unwrap_or_else(|e| e.into_inner()) + .as_ref() + { + let _ = tx.send(GroupMsg { + group: opened.group, + roots: opened.roots.to_vec(), + commit_ood: opened.commit_ood.to_vec(), + tables: opened.tables.to_vec(), + argue: opened.argue.cloned(), + opening: opened.opening.clone(), + prepared: opened.prepared.cloned(), + at: t0.elapsed().as_secs_f64(), + }); + } + }; + // The tree programs' budget (`LAMBDA_VM_TREE_PROGRAM_BUDGET`): against the + // memory target, which `LAMBDA_VM_BLOCK_MEMORY_TARGET_GIB` moves too, `auto`'s + // room line at `LAMBDA_VM_TREE_PROGRAM_BUDGET_SHARE` of it, the host read + // as `LAMBDA_VM_TREE_PROGRAM_BUDGET_READING` says (less what a purge could + // return, unless it says `resident`); `off` emits every leaf beside phase + // B, as before. + let budget = match cfg.program_budget { + super::program_budget::BudgetSetting::Off => None, + setting => Some(super::program_budget::ProgramBudget::new( + super::program_budget::Room::of( + setting, + block_whir::memory_target_bytes(), + super::program_budget::share_from_env()?, + ), + super::program_budget::reading_from_env()?, + )), + }; + // The leaves offered to phase B (`W3_LEAF_FIRST`), and when. + let lead = block_whir::PhaseBLead::default(); + let offered_at = std::sync::Mutex::new(None::); + let leaf_first = cfg.leaf_first && early_on; + let (lead_ref, offered_ref) = (&lead, &offered_at); + let (elf_ref, opts_ref, format_ref) = (elf, &opts, &format); + let (proved, pre) = std::thread::scope(|scope| { + let pre = scope.spawn(move || -> Result { + let (statement, roots) = rx + .recv() + .map_err(|_| "the prover never stated".to_string())?; + let at = t0.elapsed().as_secs_f64(); + let plan = WhirBlockPlan::derive_with( + elf_ref, + opts_ref, + format_ref, + statement.view(), + leaves, + fan_in, + Some(&roots), + )?; + // Under a program budget, the leaves admitted now, in leaf order + // while it has room (`try_acquire` never waits: the next AHEAD + // always come in); the tree emits the rest as it admits them. + let n = plan.partition().num_leaves(); + let tree_budget = budget.map(|b| { + let groups: Vec = (0..n).map(|k| plan.partition().leaf(k).len()).collect(); + TreeBudget::new(b, &groups, &plan.levels()) + }); + let permits: Vec> = match &tree_budget { + Some(b) => (0..n) + .map_while(|k| b.budget.try_acquire(k, b.leaf_estimate(k))) + .map(Some) + .collect(), + None => (0..n).map(|_| None).collect(), + }; + let upto = permits.len(); + // The admitted leaves' groups to phase B, before their programs are + // emitted: it takes the soonest leaf's groups next, and that leaf + // is the early one below. + if leaf_first { + let leaves = (0..upto) + .map(|k| plan.partition().leaf(k).to_vec()) + .collect(); + if lead_ref.offer(leaves) { + *offered_ref.lock().unwrap_or_else(|e| e.into_inner()) = + Some(t0.elapsed().as_secs_f64()); + } + } + let programs: Vec> = (0..n).map(|_| Published::new()).collect(); + // The admitted leaves' programs, on plain threads off the rayon + // pool phase B is using; the stamps are leaf 0's and the last's. + let emit = |k: usize| plan.leaf_program(k); + let emit_leaves = |permits: Vec>, + threads: usize| + -> Result<(Option, f64), String> { + let mut first = None; + emit_admitted_leaves( + &programs, + permits, + threads, + &emit, + &program_bytes, + &mut |k| { + if k == 0 { + first = Some(t0.elapsed().as_secs_f64()); + } + }, + )?; + Ok((first, t0.elapsed().as_secs_f64())) + }; + // The control (`W3_LEAF_EMITTERS=0`): one thread a leaf, all at + // once, and the first group taken after the last is emitted. + // Otherwise `leaf_emitters` at a time beside the groups below. + let (control, beside) = match leaf_emitters { + None => (Some(emit_leaves(permits, upto)?), None), + Some(threads) => (None, Some((permits, threads))), + }; + let (looped, emitted) = std::thread::scope(|inner| { + let emitter = beside + .map(|(permits, threads)| inner.spawn(move || emit_leaves(permits, threads))); + let looped = (|| -> Result, String> { + // The groups as phase B finishes them (none unless `early_on`): + // the first leaf complete while a group is still to come runs + // its execute + fill now, on a thread of its own. + let mut early = None; + let mut words: Vec>> = + (0..plan.num_groups()).map(|_| None).collect(); + let mut block_roots = Vec::new(); + let mut block_answers = Vec::new(); + let mut done = 0usize; + while let Ok(msg) = grx.recv() { + done += 1; + let g = msg.group; + let slot = words.get_mut(g).ok_or("a group the plan does not hold")?; + *slot = Some(group_arena_words( + &plan, + g, + &msg.tables, + msg.argue.as_ref(), + &msg.opening, + msg.prepared.as_ref(), + )?); + if block_roots.is_empty() { + block_roots = msg.roots; + block_answers = msg.commit_ood; + } + if early.is_some() || done >= plan.num_groups() { + continue; + } + let partition = plan.partition(); + let complete = (0..partition.num_leaves()) + .find(|&k| partition.leaf(k).iter().all(|&g| words[g].is_some())) + .filter(|&k| k < upto); + if let Some(k) = complete { + let groups: Vec> = partition + .leaf(k) + .iter() + .map(|&g| words[g].clone().unwrap_or_default()) + .collect(); + let arena = leaf_arena(&block_roots, &block_answers, groups); + // This executor takes the leaf's program. + let slot = programs[k].wait()?; + slot.claim(); + let program = slot.get()?; + let handle = std::thread::spawn(move || -> Result { + let started = t0.elapsed().as_secs_f64(); + // The staging pairs the tree's first card holds + // would allocate, made now, beside the base + // (`LFM_WARM_FIRST_PROVE`). + let staging = super::first_prove_warm::on().then(|| { + std::thread::spawn(super::first_prove_warm::prewarm_staging) + }); + let t = std::time::Instant::now(); + let hasher = + program.program.hasher(::SOCKET); + let mut filled = + lfm_execute_and_fill(&program.program, &arena, hasher) + .map_err(|e| format!("early leaf {k}: {e:?}"))?; + let secs = t.elapsed().as_secs_f64(); + let wrap = super::proof::aggregation_wrap_options(); + // Its tables' domains and twiddles, warm before + // the tree's first card hold (`LFM_WARM_DOMAINS`). + if super::domain_warm::on() { + super::domain_warm::warm(filled.traces(), &wrap); + super::domain_warm::note_early(t0.elapsed().as_secs_f64()); + } + // And the rest of what its prove builds on first + // use: the per-domain prove constants and the + // DEEP's LDE cosets (`LFM_WARM_FIRST_PROVE`). + // A warm-up that fails leaves the prove to build + // what it needs, as with the knob off; the line + // says so. + if super::first_prove_warm::on() { + match super::first_prove_warm::warm( + &program.program, + filled.traces_mut(), + &wrap, + hasher, + ) { + Ok(_) => super::first_prove_warm::note_early( + t0.elapsed().as_secs_f64(), + ), + Err(e) => eprintln!( + "W3 FIRST PROVE WARM: early leaf {k}'s warm-up refused: {e}" + ), + } + } + if let Some(h) = staging + && h.join().is_err() + { + eprintln!("W3 FIRST PROVE WARM: the staging warm-up panicked"); + } + Ok((filled, arena, started, secs)) + }); + early = Some(EarlyLeaf { + leaf: k, + complete_at: msg.at, + handle, + }); + } + } + Ok(early) + })(); + // Every admitted leaf published before the tree may look: an + // empty slot is, to it, a leaf the budget left to the tree. + let emitted = emitter.map(|h| { + h.join() + .map_err(|_| "a leaf emitter panicked".to_string()) + .and_then(|stamps| stamps) + }); + (looped, emitted) + }); + let early = looped?; + let (first_ready, ready) = match control { + Some(stamps) => stamps, + None => emitted.ok_or("the leaf emitters never ran")??, + }; + Ok(( + plan, + programs, + upto, + tree_budget, + at, + first_ready, + ready, + early, + )) + }); + let proved = block_whir::prove_block_whir_observed_led( + elf, + input, + &opts, + &format, + options, + &observe, + &on_group, + leaf_first.then_some(&lead), + ); + // A prover that failed before stating must not leave the planner waiting. + *sender.lock().unwrap_or_else(|e| e.into_inner()) = None; + *group_sender.lock().unwrap_or_else(|e| e.into_inner()) = None; + (proved, pre.join()) + }); + let (proof, stamps) = proved.map_err(|e| format!("the block proves: {e}"))?; + let base = t0.elapsed().as_secs_f64(); + alloc_peak_line(&|l: &str| sink.line(l), "base"); + // The base's freed pages back to the OS before the tree allocates: when + // the block found its memory short, or as `LAMBDA_VM_ALLOC_PURGE` names it + // (`alloc_purge`). + crate::alloc_purge::purge_point("base"); + let (plan, programs, upto, tree_budget, stated_at, first_ready_at, ready_at, early) = pre + .map_err(|_| "the planner panicked".to_string())? + .map_err(|e| format!("the plan and the leaves derive: {e}"))?; + sink.write(&stamps.report()); + // LT's chunk heights (what its per-chunk deduplication leaves): equal across + // runs of one setting when the chunking is deterministic. + { + let frame = block_whir::block_frame(proof.statement(), elf, &opts, &format) + .map_err(|e| format!("the statement's frame: {e}"))?; + let refs = frame.airs.air_refs(); + let lt: Vec = refs + .iter() + .zip(&proof.table_num_vars) + .filter(|(air, _)| air.name().starts_with("LT[")) + .map(|(_, &n)| n) + .collect(); + sink.line(&format!("W3 LT HEIGHTS: {lt:?}")); + sink.line(&format!( + "W3 CHUNKED: {} (cuts KECCAK 2^{} · ECSM 2^{} · COPY 2^{})", + block_whir::chunked_census(&proof.table_counts, &proof.table_num_vars), + options.keccak_rows_log2, + options.ecsm_rows_log2, + options.copy_rows_log2, + )); + // Each group's tables by AIR, in the group's order: which tables the + // card waited for, and where the layout's chunks went. + for (g, group) in proof.groups.iter().enumerate() { + let mut kinds: Vec<(String, usize)> = Vec::new(); + for &index in group { + let name = refs[index as usize].name(); + let kind = name.split('[').next().unwrap_or(name).to_string(); + match kinds.last_mut() { + Some((last, n)) if *last == kind => *n += 1, + _ => kinds.push((kind, 1)), + } + } + let kinds: Vec = kinds + .into_iter() + .map(|(kind, n)| if n == 1 { kind } else { format!("{kind}×{n}") }) + .collect(); + sink.line(&format!("W3 GROUP TABLES {g}: {}", kinds.join(" "))); + } + } + sink.line(&format!( + "W3 BASE: {base:.2}s · statement at {stated_at:.2}s · plan + {} leaves emitted by {ready_at:.2}s ({})", + if upto == programs.len() { + format!("{upto}") + } else { + format!("{upto} of {}", programs.len()) + }, + if ready_at <= base { + "inside the base" + } else { + "after the base" + } + )); + // The card's upload traffic at the base's end, read against the same line at + // the tree's end: what the tree's proves uploaded. + #[cfg(feature = "cuda")] + sink.line(&format!( + "W3 STAGING AFTER BASE: {} · main traces built on the card {}", + stark::gpu_lde::staging_report(), + stark::trace::main_dev_producer_calls() + )); + sink.line(&format!( + "W3 LEAF EMITTERS: {} · leaf 0's program at {} · the last at {ready_at:.2}s", + match leaf_emitters { + Some(threads) => format!( + "{threads} at a time in leaf order, beside phase B's groups (W3_LEAF_EMITTERS)" + ), + None => "one thread a leaf, all at once, before the first group is taken \ + (W3_LEAF_EMITTERS=0, the control)" + .to_string(), + }, + first_ready_at.map_or("-".to_string(), |t| format!("{t:.2}s")), + )); + sink.line(&leaf_first_line( + leaf_first, + *offered_at.lock().unwrap_or_else(|e| e.into_inner()), + stated_at, + lead.taken(), + early.as_ref().map(|e| e.leaf), + )); + sink.line(&format!( + "W3 TREE PROGRAM BUDGET: {}", + match &tree_budget { + Some(b) => format!( + "{} · {upto} of {} leaf programs beside the base, the rest emitted in the tree as \ + the budget admits them", + b.budget.describe(), + programs.len() + ), + None => "off (every leaf program beside the base)".to_string(), + } + )); + sink.line(&format!( + "W3 PLAN: {} groups · costs {:?} (Σ {}) · {} leaves {:?} · {} prepared", + plan.num_groups(), + plan.costs(), + plan.costs().iter().sum::(), + plan.partition().num_leaves(), + plan.partition().leaves(), + plan.prepared().len() + )); + sink.line(&leaf_scale_line(format.leaf_scale, &plan)); + sink.line(&format!("W3 LEAF QUERY: {:?}", format.leaf_query)); + // Each leaf's chips, real / padded rows: how far each sits from its next + // doubling, and which one doubled when the in-guest work moves. + // (The leaves emitted beside the base: the others do not exist yet.) + for (k, slot) in programs.iter().take(upto).enumerate() { + let program = slot.wait()?.get()?; + let chips: Vec = super::airs::lfm_chip_census_with_hasher( + &program.program, + program.program.hasher(::SOCKET), + ) + .iter() + .filter(|c| c.real_rows > 0) + .map(|c| format!("{}={}/{}", c.name, c.real_rows, c.rows)) + .collect(); + sink.line(&format!("W3 LEAF CENSUS {k}: {}", chips.join(" "))); + } + + // The readouts above run on the clock, between the base and the tree: a + // prover prints none of them, so the whole block is also given without. + let readouts = t0.elapsed().as_secs_f64() - base; + let node_pipe = match cfg.node_pipe { + Some(threads) => Some( + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .thread_name(|i| format!("w3-emit-{i}")) + .build() + .map_err(|e| format!("the node builder's pool: {e}"))?, + ), + None => None, + }; + let (dataflow, stream_top, exec_early) = (cfg.dataflow, cfg.stream_top, cfg.exec_early); + let top_streamed = std::sync::OnceLock::new(); + let t = std::time::Instant::now(); + let tree_unix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + let (timings, proofs, early_out, each_from) = prove_tree_pipelined( + &plan, + &proof, + programs, + tree_budget.as_ref(), + siblings, + beside, + cfg.leaves_each, + early, + leaf_first, + node_pipe.as_ref(), + dataflow, + stream_top.then_some(&top_streamed), + exec_early, + &|line: &str| sink.line(line), + ) + .map_err(|e| format!("the tree proves: {e}"))?; + if let Some(b) = &tree_budget { + sink.line(&format!( + "W3 TREE PROGRAM BUDGET end: {}", + b.budget.summary() + )); + } + let tree = t.elapsed().as_secs_f64(); + let whole = t0.elapsed().as_secs_f64(); + for (lv, level) in timings.iter().enumerate() { + let programs: Vec = level + .programs + .iter() + .map(|(built, prove)| format!("{built:.2}+{prove:.2}")) + .collect(); + sink.line(&format!( + "W3 LEVEL {lv}: {:.2}s wall · per program build+prove {}", + level.wall, + programs.join(" · ") + )); + } + // `W3_NODE_TIMES=1`: when each program of the tree was built, taken and + // proved (seconds since the tree started), its prove's split, and for each + // node level when the level below was done and its nodes were built. Off + // the clock; the stamps are taken either way. + if cfg.node_times { + sink.line(&format!( + "W3 TIMES TREE START: unix {tree_unix:.3} (the card trace's clock)" + )); + let shape = plan.levels(); + let split = |t: &ProgramTimes| match t.split { + Some((x, f, p, w)) => format!("exec {x:.2} fill {f:.2} prove {p:.2} wait {w:.2}"), + None => "no split".to_string(), + }; + for (lv, level) in timings.iter().enumerate() { + let mut at = 0usize; + let rows: Vec = level + .times + .iter() + .enumerate() + .map(|(j, t)| { + let landed = match lv { + 0 => String::new(), + _ => { + let a = shape[lv - 1].arities[j]; + let kids: Vec = timings[lv - 1].times[at..at + a] + .iter() + .map(|c| format!("{:.2}", c.end)) + .collect(); + at += a; + format!(" kids done [{}]", kids.join(" ")) + } + }; + let program = match lv { + 0 => String::new(), + _ => format!(" program@{:.2} emit {:.2}", t.program_at, t.program_emit), + }; + format!( + "{j}: built@{:.2}{landed} took@{:.2} done@{:.2} ({}){program}", + t.built_at, + t.start, + t.end, + split(t) + ) + }) + .collect(); + sink.line(&format!("W3 TIMES L{lv}: {}", rows.join(" · "))); + if lv > 0 { + let last_kid = timings[lv - 1] + .times + .iter() + .map(|t| t.end) + .fold(0.0, f64::max); + let built = level.times.iter().map(|t| t.built_at).fold(0.0, f64::max); + let first = level + .times + .iter() + .map(|t| t.start) + .fold(f64::INFINITY, f64::min); + sink.line(&format!( + "W3 TIMES START L{lv}: level {} done@{last_kid:.2} · its nodes built by {built:.2} · first node took@{first:.2} ({:+.2} after the later)", + lv - 1, + first - last_kid.max(built) + )); + } + } + } + sink.line(&format!( + "W3 STREAM TOP: {}", + if stream_top { + "on (while a child is unproved as its program arrives, the top executes each child's share as that child is proved)" + } else { + "off (the top executes after its last child)" + } + )); + sink.line(&format!( + "W3 EXEC EARLY: {}", + if exec_early { + "on (a node executes and fills from its program as it is emitted; only its prove waits for its artifacts)" + } else { + "off (a node executes once its program and its artifacts are both built)" + } + )); + sink.line(&format!( + "W3 TOP EXECUTE: {}", + match (stream_top, top_streamed.get()) { + (false, _) => "whole (streaming off)", + (true, Some(true)) => + "streamed (a child was still unproved when the top could execute)", + (true, Some(false)) => "whole (every child was proved before the top could execute)", + (true, None) => "unknown (the top never chose)", + } + )); + sink.line(&format!( + "W3 DATAFLOW: {}", + if dataflow { + "on (each node proves as soon as its own children are proved)" + } else { + "off (each node waits for its whole level below)" + } + )); + sink.line(&format!( + "W3 NODE PIPE: {}", + match &node_pipe { + Some(pool) => format!( + "on ({} builder threads; each level proves as its own nodes arrive)", + pool.current_num_threads() + ), + None => "off (one builder thread; level 1 waits for the whole tree)".to_string(), + } + )); + // How the leaves' artifacts were published ([`publish_each`]), when the + // first leaf was proved and the first node over the leaves had its + // program, against when the last leaf's artifacts were built. + let (first_leaf, last_leaf) = timings.first().map_or((f64::NAN, 0.0), |level| { + let first = level + .times + .iter() + .map(|t| t.end) + .fold(f64::INFINITY, f64::min); + let last = level.times.iter().map(|t| t.built_at).fold(0.0, f64::max); + (first, last) + }); + let first_program = timings.get(1).map_or(f64::NAN, |level| { + level + .times + .iter() + .map(|t| t.program_at) + .fold(f64::INFINITY, f64::min) + }); + sink.line(&format!( + "W3 LEAF ARTIFACTS: {} · first leaf proved@{first_leaf:.2} · first level-1 program@{first_program:.2} · last leaf built@{last_leaf:.2}", + match each_from { + None => "together (every leaf's published after the last is built)".to_string(), + Some(_) if cfg.leaves_each => "each published as it is built (W3_LEAF_ARTIFACTS_EACH=1)".to_string(), + Some(k) => format!( + "together until leaf {k}'s program was not there, then each published as it is built" + ), + } + )); + #[cfg(feature = "cuda")] + sink.line(&format!( + "W3 STAGING AT END: {} · main traces built on the card {}", + stark::gpu_lde::staging_report(), + stark::trace::main_dev_producer_calls() + )); + #[cfg(feature = "cuda")] + sink.line(&format!( + "W3 DEEP HOISTED: {}", + stark::gpu_lde::deep_hoisted_report() + )); + sink.line(&format!( + "W3 RECURSION: {:.2}s after the base (tree {tree:.2}s) · whole block {whole:.2}s · whole excl. harness readouts {:.2}s (readouts {readouts:.2}s) · leaves execute beside their artifacts {beside}", + whole - base, + whole - readouts + )); + if let Some(w) = watermark { + sink.line(&w.finish()); + } + if proofs.is_empty() { + return Err("the tree has no top".to_string()); + } + // The early leaf's arena is checked against the proof's by the harness. + #[cfg(not(test))] + drop(early_out); + Ok(WhirTreeRun { + statement: proof.statement().to_owned(), + proofs, + times: WhirTreeTimes { + base, + tree, + whole, + readouts, + }, + #[cfg(test)] + base_proof: proof, + #[cfg(test)] + plan, + #[cfg(test)] + early: early_out, + }) +} + +// ============================== the proof file ============================ + +/// The proof file's first bytes (the STARK block tree's file shares them). +const PROOF_MAGIC: [u8; 8] = *b"LVMBLKTR"; + +/// The proof file's layout version. 2: the archived `TableCounts` gained +/// `is_wh`, so a version-1 file is a different layout. 3: the top proof +/// carries its commitment-time answers. +const PROOF_VERSION: u32 = 3; + +/// Which tree a proof file holds: the STARK block tree (0) or this one. A +/// verifier refuses the other's file by its tag, before reading it as its own. +pub const PIPELINE_WHIR: u8 = 1; + +/// A whole W3 block's proof as a consumer receives it: the block's statement +/// and the top node's proof. The block verifier ([`verify_whir_block_tree_proof`]) +/// takes the statement as a claim — the plan and the top program are derived +/// from the trusted ELF and the statement, under the block presets — so +/// nothing here is a format parameter: the file only carries what +/// [`super::whir_block::verify_block_tree`] already takes from its caller. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct WhirBlockTreeProof { + magic: [u8; 8], + version: u32, + pipeline: u8, + pub table_num_vars: Vec, + pub runtime_page_ranges: Vec, + pub table_counts: crate::TableCounts, + pub public_output: Vec, + pub num_private_input_pages: usize, + pub groups: Vec>, + pub top_proof: super::proof::LfmStarkProof, + pub top_public_words: Vec<(u32, LfmWord)>, +} + +impl WhirBlockTreeProof { + /// The file for a W3 block's statement and its top proof. + pub fn new(statement: block_whir::OwnedBlockStatement, top: LfmProof) -> Self { + Self { + magic: PROOF_MAGIC, + version: PROOF_VERSION, + pipeline: PIPELINE_WHIR, + table_num_vars: statement.table_num_vars, + runtime_page_ranges: statement.runtime_page_ranges, + table_counts: statement.table_counts, + public_output: statement.public_output, + num_private_input_pages: statement.num_private_input_pages, + groups: statement.groups, + top_proof: top.proof, + top_public_words: top.public_words, + } + } + + /// The claimed statement. + pub fn statement(&self) -> block_whir::BlockStatement<'_> { + block_whir::BlockStatement { + table_num_vars: &self.table_num_vars, + runtime_page_ranges: &self.runtime_page_ranges, + table_counts: &self.table_counts, + public_output: &self.public_output, + num_private_input_pages: self.num_private_input_pages, + groups: &self.groups, + } + } + + /// The file's bytes. + pub fn to_bytes(&self) -> Result { + rkyv::to_bytes::(self).map_err(|e| format!("serialize: {e}")) + } + + /// A file read back: refused unless it is this layout's W3 block tree. + /// `bytes` must be aligned for rkyv (an `AlignedVec`). + pub fn from_bytes(bytes: &[u8]) -> Result { + let proof = rkyv::from_bytes::(bytes) + .map_err(|e| format!("not a W3 block proof file: {e}"))?; + if proof.magic != PROOF_MAGIC { + return Err("not a block proof file (magic)".to_string()); + } + if proof.version != PROOF_VERSION { + return Err(format!( + "block proof file version {}, this build reads {PROOF_VERSION}", + proof.version + )); + } + if proof.pipeline != PIPELINE_WHIR { + return Err(format!( + "the file holds pipeline {} ({}), not the WHIR block tree", + proof.pipeline, + if proof.pipeline == 0 { + "the STARK block tree" + } else { + "unknown" + } + )); + } + Ok(proof) + } +} + +/// ★ A W3 block proof file verified: [`super::whir_block::verify_block_tree`] +/// over its claimed statement and its top proof, under the pinned block +/// presets (constants, never the environment's), against the trusted +/// `elf_bytes`. +pub fn verify_whir_block_tree_proof( + elf_bytes: &[u8], + proof: &WhirBlockTreeProof, +) -> Result<(), String> { + verify_whir_block_tree_proof_grouped(elf_bytes, proof, BlockFormat::pinned().group) +} + +/// [`verify_whir_block_tree_proof`] for a block whose groups were opened per +/// `group` — the verifier's constant, the caller's to give +/// ([`super::whir_block::verify_block_tree_grouped`]); the file does not +/// carry it. +pub fn verify_whir_block_tree_proof_grouped( + elf_bytes: &[u8], + proof: &WhirBlockTreeProof, + group: GroupFold, +) -> Result<(), String> { + let top = LfmProof { + proof: proof.top_proof.clone(), + public_words: proof.top_public_words.clone(), + }; + super::whir_block::verify_block_tree_grouped(elf_bytes, proof.statement(), &top, group) +} + +/// [`verify_whir_block_tree_proof`] at the pinned format with the caller's +/// named choices, its group opening, its WHIR blowup, its round-0 leaf order, +/// its leaf scale, its leaves' query check and its GKR packing +/// ([`super::whir_block::verify_block_tree_at`]); the file carries none of +/// them, and the environment sets none of the rest. No public entry point +/// takes a format of the caller's own. +/// +/// ```no_run +/// use lambda_vm_prover::lfm::whir_block::LeafScale; +/// use lambda_vm_prover::lfm::whir_block_tree::{ +/// WhirBlockTreeProof, verify_whir_block_tree_proof_at, +/// }; +/// use lambda_vm_prover::lfm::whir_chain::LeafQuery; +/// use lambda_vm_prover::zf_format::WhirBlowup; +/// use multilinear::whir_chain::{GkrPack, GroupFold, LeafOrder}; +/// +/// fn check(elf: &[u8], proof: &WhirBlockTreeProof) -> Result<(), String> { +/// verify_whir_block_tree_proof_at( +/// elf, +/// proof, +/// GroupFold::Indexed, +/// WhirBlowup::Two, +/// LeafOrder::Coset, +/// LeafScale::DoubleFrom(12), +/// LeafQuery::Lean, +/// GkrPack::Off, +/// ) +/// } +/// ``` +/// +/// The any-format verifier is crate-internal, so outside the crate this does +/// not compile: +/// +/// ```compile_fail +/// use lambda_vm_prover::lfm::whir_block::verify_block_tree_as; +/// ``` +#[allow(clippy::too_many_arguments)] +pub fn verify_whir_block_tree_proof_at( + elf_bytes: &[u8], + proof: &WhirBlockTreeProof, + group: GroupFold, + blowup: WhirBlowup, + leaves: LeafOrder, + leaf_scale: LeafScale, + leaf_query: LeafQuery, + pack: GkrPack, +) -> Result<(), String> { + let top = LfmProof { + proof: proof.top_proof.clone(), + public_words: proof.top_public_words.clone(), + }; + super::whir_block::verify_block_tree_at( + elf_bytes, + proof.statement(), + &top, + group, + blowup, + leaves, + leaf_scale, + leaf_query, + pack, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// With the early leaf first (`W3_LEAF_FIRST`), the tree's order takes + /// that leaf before every other and is otherwise [`tree_order`]'s: a + /// permutation of the leaves alone, every node after every leaf. With no + /// early leaf, leaf 0, or a leaf out of range it is [`tree_order`] + /// unchanged. Proved over it, with one worker (inline) or two, every + /// program proves, and one worker takes the early leaf first. + #[test] + fn the_early_leaf_is_taken_first() { + use super::super::per_table_aggregator::Level; + let shape = vec![ + Level { + arities: vec![3, 2], + }, + Level { arities: vec![2] }, + ]; + let base = tree_order(5, &shape); + for unchanged in [None, Some(0), Some(9)] { + assert_eq!( + early_leaf_first(base.clone(), unchanged), + base, + "{unchanged:?}" + ); + } + let led = early_leaf_first(base.clone(), Some(3)); + let leaves: Vec = (0..5).map(|j| TreeAt { lv: 0, j }).collect(); + assert_eq!( + led[..5], + [leaves[3], leaves[0], leaves[1], leaves[2], leaves[4]] + ); + assert_eq!(led[5..], base[5..]); + for workers in [1, 2] { + let results: Vec>> = [5usize, 2, 1] + .iter() + .map(|&n| (0..n).map(|_| Published::new()).collect()) + .collect(); + let taken = std::sync::Mutex::new(Vec::new()); + prove_dataflow(&shape, &led, &results, workers, false, |at, kids| { + taken.lock().expect("taken").push(at); + wait_all(kids)?; + Ok(at.lv) + }); + for (lv, level) in results.iter().enumerate() { + for slot in level { + assert_eq!(slot.wait(), Ok(&lv), "{workers} workers"); + } + } + let taken = taken.into_inner().expect("taken"); + assert_eq!(taken.len(), led.len(), "{workers} workers"); + if workers == 1 { + assert_eq!(taken, led); + } + } + } + + /// ★ No public block-tree verifier takes a format of the caller's own (a + /// caller who could would choose the programs the top is checked against): + /// every top-level `pub fn verify…` of this file and of `whir_block.rs` is + /// read off the source and its parameters must not name `BlockFormat`, and + /// the one any-format verifier, `whir_block::verify_block_tree_as`, is + /// declared crate-internal. From outside the crate the same is checked by + /// [`verify_whir_block_tree_proof_at`]'s `compile_fail` doc test. The + /// verifiers of named choices (`…_at`) take each choice typed: the group + /// opening, the WHIR blowup, the round-0 leaf order, the leaf scale and + /// the leaves' query check. + #[test] + fn no_public_tree_verifier_takes_a_block_format() { + let block = include_str!("whir_block.rs"); + for (file, src) in [ + ("whir_block.rs", block), + ("whir_block_tree.rs", include_str!("whir_block_tree.rs")), + ] { + let mut public = Vec::new(); + for (at, _) in src.match_indices("\npub fn verify") { + let sig = &src[at + 1..]; + let params = &sig[..sig.find(')').expect("a parameter list")]; + let name = &sig[7..sig.find('(').expect("a name")]; + assert!( + !params.contains("BlockFormat"), + "{file}: the public verifier {name} takes a BlockFormat" + ); + if name.ends_with("_at") { + for choice in [ + "GroupFold", + "WhirBlowup", + "LeafOrder", + "LeafScale", + "LeafQuery", + ] { + assert!( + params.contains(choice), + "{file}: {name} takes its {choice} as a named choice" + ); + } + } + public.push(name.to_string()); + } + assert!( + public.len() >= 2, + "{file}: the scan found its public verifiers: {public:?}" + ); + assert!( + public.iter().any(|name| name.ends_with("_at")), + "{file}: a verifier of named choices: {public:?}" + ); + } + assert!(block.contains("\npub(crate) fn verify_block_tree_as(")); + assert!(!block.contains("\npub fn verify_block_tree_as(")); + } + + /// The posture line names every posture knob, in the table's order, and + /// the WHIR hash (a format knob both sides must share) is one of them. + #[test] + fn the_posture_line_names_every_knob() { + let line = posture_line(); + assert!(line.starts_with("BLOCK POSTURE: ")); + let mut at = 0; + for (name, _) in POSTURE { + let found = line[at..] + .find(name) + .unwrap_or_else(|| panic!("{name} missing or out of order in `{line}`")); + at += found + name.len(); + } + assert!( + POSTURE + .iter() + .all(|(name, _)| *name != crate::whir_hash_knob::ENV), + "the base hash is the format's, not posture" + ); + } + + /// `LAMBDA_VM_WHIR_GROUP` reads two words, unset is production's one chain + /// per run, and anything else is refused rather than defaulted. + #[test] + fn the_group_opening_reads_two_words() { + assert_eq!(group_setting(None), Ok(GroupFold::Indexed)); + assert_eq!(BlockFormat::production().group, GroupFold::Indexed); + assert_eq!(group_setting(Some("per-poly")), Ok(GroupFold::PerPoly)); + assert_eq!(group_setting(Some(" indexed ")), Ok(GroupFold::Indexed)); + for word in ["index", "Indexed", "1", ""] { + assert!(group_setting(Some(word)).is_err(), "`{word}` is refused"); + } + assert_eq!( + BlockFormat::production_with(GroupFold::PerPoly).group, + GroupFold::PerPoly + ); + } + + /// A synthetic statement and an empty top proof. + fn synthetic_proof() -> WhirBlockTreeProof { + let counts = crate::TableCounts { + cpu: 3, + lt: 1, + memw: 2, + memw_aligned: 1, + load: 1, + mul: 1, + dvrm: 1, + shift: 1, + branch: 1, + memw_register: 1, + eq: 1, + bytewise: 1, + store: 1, + cpu32: 1, + keccak: 2, + keccak_rnd: 4, + ecsm: 1, + ecdas: 1, + hint: 0, + copy: 1, + blake3: 0, + is_wh: 0, + ecrec: 0, + }; + let statement = block_whir::OwnedBlockStatement { + table_num_vars: vec![21, 16, 6], + runtime_page_ranges: vec![crate::RuntimePageRange { + base: 0x7000_0000, + count: 3, + }], + table_counts: counts, + public_output: vec![1, 2, 3, 4, 5], + num_private_input_pages: 2, + groups: vec![vec![0, 2], vec![1]], + }; + let words: Vec<(u32, LfmWord)> = (0..3u32) + .map(|i| (i, [crate::tables::types::FE::from(u64::from(i) + 7); 4])) + .collect(); + let top = LfmProof { + proof: super::super::proof::LfmStarkProof { + multi: stark::proof::stark::MultiProof { proofs: vec![] }, + commit_ood: vec![], + }, + public_words: words, + }; + WhirBlockTreeProof::new(statement, top) + } + + /// The proof file reads back what was written; another tag, version or + /// magic, or bytes that are not a file, are refused. + #[test] + fn the_proof_file_round_trips_and_refuses_another_file() { + let proof = synthetic_proof(); + let bytes = proof.to_bytes().expect("serializes"); + let back = WhirBlockTreeProof::from_bytes(&bytes).expect("reads back"); + assert_eq!( + format!("{:?}", back.statement()), + format!("{:?}", proof.statement()) + ); + assert_eq!(back.top_public_words, proof.top_public_words); + assert_eq!(back.top_proof.proofs.len(), 0); + + for (what, tamper) in [ + ( + "pipeline", + (|p: &mut WhirBlockTreeProof| p.pipeline = 0) as fn(&mut WhirBlockTreeProof), + ), + ("version", |p| p.version = PROOF_VERSION + 1), + ("magic", |p| p.magic[0] ^= 1), + ] { + let mut other = synthetic_proof(); + tamper(&mut other); + let bytes = other.to_bytes().expect("serializes"); + assert!( + WhirBlockTreeProof::from_bytes(&bytes).is_err(), + "a file with another {what} is refused" + ); + } + let mut junk = rkyv::util::AlignedVec::<16>::new(); + junk.extend_from_slice(&[0u8; 7]); + assert!( + WhirBlockTreeProof::from_bytes(&junk).is_err(), + "seven zero bytes" + ); + } + + /// Only the production tree is at the presets: the plan's leaf count, the + /// block fan-in and the production argue. + #[test] + fn only_the_production_tree_is_at_the_presets() { + let production = WhirTreeConfig { + leaves: None, + siblings: 3, + fan_in: BLOCK_FAN_IN, + beside: true, + leaves_each: false, + program_budget: super::super::program_budget::BudgetSetting::Auto, + early_on: true, + leaf_emitters: Some(LEAF_EMITTERS), + argue: BlockFormat::production().argue, + format: BlockFormat::production(), + options: BlockOptions::production(), + node_pipe: Some(4), + dataflow: true, + stream_top: true, + exec_early: true, + node_times: false, + leaf_first: true, + }; + assert!(production.at_presets()); + let off = |f: fn(&mut WhirTreeConfig)| { + let mut c = production.clone(); + f(&mut c); + c.at_presets() + }; + assert!(!off(|c| c.leaves = Some(2)), "a forced leaf count"); + assert!(!off(|c| c.fan_in = BLOCK_FAN_IN + 1), "another fan-in"); + let other = match BlockFormat::production().argue { + ArgueFormat::PerTable => ArgueFormat::BATCHED, + _ => ArgueFormat::PerTable, + }; + let mut c = production.clone(); + c.argue = other; + assert!(!c.at_presets(), "another argue"); + assert!( + off(|c| c.siblings = 6), + "the sibling count is scheduling only" + ); + // A packing is a named constant the block verifier is given too: a + // tree argued under another (here `Off`), its format saying so, is at + // the presets; a config whose argue and format disagree is not. + let pack = GkrPack::Off; + let mut c = production.clone(); + c.format = c.format.with_pack(pack); + c.argue = c.format.argue; + assert!(c.at_presets(), "an unpacked argue, named on both"); + let mut c = production.clone(); + c.argue = BlockFormat::production().with_pack(pack).argue; + assert!(!c.at_presets(), "an argue its format does not carry"); + let mut c = production.clone(); + c.argue = ArgueFormat::batched(20); + c.format.argue = c.argue; + assert!(!c.at_presets(), "another bin cap"); + } + + /// `W3_LEAF_EMITTERS`: unset is the default count, 0 is the control (one + /// thread a leaf), any other count is itself. + #[test] + fn the_leaf_emitters_knob_reads_a_count_and_zero_as_the_control() { + assert_eq!(leaf_emitters_setting(None), Some(LEAF_EMITTERS)); + assert_eq!(leaf_emitters_setting(Some(0)), None); + assert_eq!(leaf_emitters_setting(Some(1)), Some(1)); + assert_eq!(leaf_emitters_setting(Some(26)), Some(26)); + assert_eq!(LEAF_EMITTERS, 3); + } + + /// The admitted leaves' programs are emitted at most `threads` at a time + /// (and more than one at a time), and each is published strictly in leaf + /// order although leaf 0 is the slowest; the leaves the budget did not + /// admit keep an empty slot, which is how the tree knows them. + #[test] + fn the_admitted_leaves_are_emitted_a_few_at_a_time_and_published_in_leaf_order() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let (n, upto, threads) = (9usize, 7usize, 3usize); + let slots: Vec>> = (0..n).map(|_| Published::new()).collect(); + let (active, peak) = (AtomicUsize::new(0), AtomicUsize::new(0)); + let emit = |k: usize| -> Result { + let now = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + if k == 0 { + // Leaf 0 waits for a second emitter (a bounded wait), then is + // the last of its batch to finish. + let t = std::time::Instant::now(); + while active.load(Ordering::SeqCst) < 2 && t.elapsed().as_secs() < 5 { + std::thread::yield_now(); + } + std::thread::sleep(std::time::Duration::from_millis(30)); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + active.fetch_sub(1, Ordering::SeqCst); + Ok(100 + k) + }; + let mut order = Vec::new(); + let mut set_when_published = true; + let emitted = emit_admitted_leaves( + &slots, + (0..upto).map(|_| None).collect(), + threads, + &emit, + &|_: &usize| 0, + &mut |k| { + set_when_published &= slots[k].0.get().is_some(); + order.push(k); + }, + ); + assert_eq!(emitted, Ok(())); + assert_eq!( + order, + (0..upto).collect::>(), + "strictly in leaf order" + ); + assert!( + set_when_published, + "a leaf's slot is set before it is announced" + ); + let peak = peak.load(Ordering::SeqCst); + assert!( + peak <= threads, + "{peak} emitters at once, the cap is {threads}" + ); + assert!(peak >= 2, "the emitters run beside each other"); + for (k, slot) in slots.iter().enumerate().take(upto) { + let program = slot + .0 + .get() + .map(|s| s.as_ref().map(|s| s.get().map(|a| a.program))); + assert_eq!(program, Some(Ok(Ok(100 + k)))); + } + for slot in &slots[upto..] { + assert!(slot.0.get().is_none(), "a leaf left to the tree"); + } + + // The control: one thread a leaf. + let slots: Vec>> = (0..n).map(|_| Published::new()).collect(); + let emitted = emit_admitted_leaves( + &slots, + (0..n).map(|_| None).collect(), + n, + &|k| Ok(k), + &|_: &usize| 0, + &mut |_| {}, + ); + assert_eq!(emitted, Ok(())); + assert!(slots.iter().all(|slot| slot.0.get().is_some())); + } + + /// An emitter that fails leaves its error in its own slot and no admitted + /// slot empty after it, so nothing waits for a program that will not come; + /// the leaves before it keep their programs, and the error is the result. + #[test] + fn a_failed_leaf_emitter_leaves_no_admitted_slot_empty() { + let (n, upto) = (8usize, 6usize); + let slots: Vec>> = (0..n).map(|_| Published::new()).collect(); + let emit = |k: usize| -> Result { + if k == 2 { + Err("leaf 2 does not emit".to_string()) + } else { + Ok(k) + } + }; + let emitted = emit_admitted_leaves( + &slots, + (0..upto).map(|_| None).collect(), + 2, + &emit, + &|_: &usize| 0, + &mut |_| {}, + ); + assert_eq!(emitted, Err("leaf 2 does not emit".to_string())); + // Read without waiting: a slot left empty fails here, it does not hang. + let state = |k: usize| { + slots[k] + .0 + .get() + .map(|r| r.as_ref().map(|_| ()).map_err(Clone::clone)) + }; + assert_eq!(state(0), Some(Ok(()))); + assert_eq!(state(1), Some(Ok(()))); + assert_eq!(state(2), Some(Err("leaf 2 does not emit".to_string()))); + for k in 3..upto { + assert_eq!(state(k), Some(Err("the leaf emitter stopped".to_string()))); + } + for slot in &slots[upto..] { + assert!(slot.0.get().is_none()); + } + } +} diff --git a/prover/src/lfm/whir_bus.rs b/prover/src/lfm/whir_bus.rs new file mode 100644 index 000000000..adaca1a21 --- /dev/null +++ b/prover/src/lfm/whir_bus.rs @@ -0,0 +1,492 @@ +//! The bus statements as a machine leg: `logup::claim_statements`, emitted. +//! +//! The mirrors are `multilinear::logup::claim_statements` +//! (`logup.rs:220-275`) and, behind it, `stark::multilinear_logup::interactions` +//! (`multilinear_logup.rs:84-127`). Together they turn a table's GKR +//! input-layer claim into the two rules the main sumcheck batches beside the +//! zerocheck: +//! +//! ```text +//! numerator = weight_row · Σ_i w_i · numerator_i(f) +//! denominator = weight_row · ( Σ_i w_i · denominator_i(f) + padding ) +//! w = eq_evals(interaction_point) — 2^ceil_log2(I) of them +//! padding = Σ of the weights past the live interactions +//! ``` +//! +//! # ⛔ Only ONE of the two affine forms has runtime coefficients +//! +//! `interactions` builds each interaction as a signed multiplicity over +//! `z − fingerprint`. The NUMERATOR is `multiplicity.evaluate_with`, negated +//! for a receiver (`multilinear_logup.rs:117-120`) — it reads neither `z` nor +//! `alpha`, so its coefficients and its constant are emit-time values and this +//! leg interns them. Only the DENOMINATOR's coefficients are wires, and they +//! are wires because the fingerprint rides the powers of `alpha`, a challenge +//! drawn after the commitment. +//! +//! # ★ The regrouping, with its cost, so it is not mistaken for a difference +//! +//! The host FUSES the fingerprint into one affine per interaction whose +//! coefficient for column `c` is `−Σ_p alpha^p·c_{p,c}` — `probe` recovers that +//! sum by evaluating at basis rows. Emitting THAT form means building every +//! column's alpha-polynomial as a wire first: `Σ_c (one MulAdd per power that +//! touches c) + |terms|`, which is `|columns|·(W+1)` when the interaction's +//! columns are spread over its bus elements. +//! +//! This emits the UNFUSED form, straight off `fingerprint_at`'s own loop +//! (`multilinear_logup.rs:24-38`): +//! +//! ```text +//! denominator_i = z − bus_id − Σ_p alpha^{p+1} · elements[p](f) +//! ``` +//! +//! which costs `Σ_p |terms_p| + W`. Identical in value; strictly fewer rows +//! whenever a bus element reads fewer columns than its interaction does, which +//! is every packed value. The structure comes from +//! `multilinear_logup::interaction_shapes`, whose own gate is that recombining +//! it at any `(z, alpha)` lands on what `interactions` fuses — so both +//! groupings are pinned and neither is re-derived here. +//! +//! # ⚠ What `interaction_shapes` already did, and this leg must not undo +//! +//! A column past the main width is PINNED TO ZERO in the recovered affine +//! rather than carried, because `interactions` filters its probe candidates to +//! `column < num_main_columns`. That happens in the accessor; this leg only +//! reads the affines it hands back. +//! +//! # The weights, and the one term that is exponential in anything +//! +//! `eq_evals(interaction_point)` is `2^ceil_log2(I)` values for a table with +//! `I` interactions (`logup.rs:235`). It is the only place a table's verify +//! cost grows as a power of anything, and it is small: over epoch 0's 34 tables +//! (ts1, 2026-09-18) `Σ 2^ceil_log2(I)` is 4,444 against `Σ I` of 2,531, so the +//! whole epoch's weights are ≈9,006 rows. + +use multilinear::logup::Affine; +use stark::multilinear_logup::InteractionShape; + +use crate::tables::types::{FEE, GoldilocksExtension}; + +use super::builder::{Ext, LfmBuilder}; +use super::word::{LfmWord, ext_word}; + +type Aff = Affine; +type Shape = InteractionShape; + +/// What a leg costs: its operation rows, and the DISTINCT constants it interns. +/// +/// Constants are pooled per PROGRAM (`builder.rs:180-190`), so counting them +/// per sub-form and adding would charge a shared value twice. This carries the +/// values and deduplicates once, the way `whir_program::steps_rows` counts a +/// DAG's `Fixed` steps. +/// +/// It lives here because the bus statements were the first leg that needed it, +/// and it is the accumulator every later per-table form uses — the whole table +/// shares one constant pool, so the terms have to be added into one of these +/// rather than summed as numbers. +#[derive(Default, Debug, Clone)] +pub struct Cost { + operations: usize, + /// The interned values as WORDS, not field elements: the pool is keyed on + /// the canonical word (`builder.rs:180-190`), and not every constant a leg + /// interns is an extension element — the sponge's leaf capacity is a + /// four-lane digest and shares the same pool. + constants: Vec, +} + +impl Cost { + /// One `LFM_XALU` row. + pub fn op(&mut self) { + self.operations += 1; + } + + /// `count` of them. + pub fn ops(&mut self, count: usize) { + self.operations += count; + } + + /// One `LFM_CONST` row, unless this value is already interned. + pub fn constant(&mut self, value: FEE) { + self.constant_word(ext_word(&value)); + } + + /// The same for a constant that is a whole word — a digest, or the sponge's + /// leaf capacity. + pub fn constant_word(&mut self, word: LfmWord) { + if !self.constants.contains(&word) { + self.constants.push(word); + } + } + + /// Another leg's cost folded in, with the constant pools UNIONED rather + /// than added — which is the whole reason this is a type and not a number. + pub fn merge(&mut self, other: &Cost) { + self.operations += other.operations; + for value in &other.constants { + self.constant_word(*value); + } + } + + /// The distinct words interned, for a caller that has to union them with + /// its own. + pub fn constant_values(&self) -> &[LfmWord] { + &self.constants + } + + /// The instructions the leg emits. + pub fn rows(&self) -> usize { + self.operations + self.constants.len() + } + + pub fn operations(&self) -> usize { + self.operations + } + + pub fn constants(&self) -> usize { + self.constants.len() + } +} + +/// The wire a factor slot reads. +fn read(values: &[Ext], slot: usize) -> Ext { + *values.get(slot).unwrap_or_else(|| { + panic!( + "the bus reads factor {slot}, but only {} were supplied", + values.len() + ) + }) +} + +/// `Σ c_j·v_{s_j} + k` over the factor wires, with the coefficients interned. +/// +/// A coefficient of ONE is folded away exactly as the host's +/// `Builder::weighted_sum` folds it (`program.rs:304-316`): it needs no +/// constant, and when it is the term that opens the fold it needs no row +/// either — the wire is already the running value. +pub fn emit_affine(b: &mut LfmBuilder, affine: &Aff, values: &[Ext]) -> Ext { + let mut accumulated: Option = None; + for (slot, coefficient) in affine.terms() { + let value = read(values, *slot); + let unit = *coefficient == FEE::one(); + accumulated = Some(match (accumulated, unit) { + (None, true) => value, + (None, false) => { + let c = b.ext_const(coefficient); + b.emul(c, value) + } + (Some(acc), true) => b.eadd(value, acc), + (Some(acc), false) => { + let c = b.ext_const(coefficient); + b.emul_add(c, value, acc) + } + }); + } + + let constant = affine.constant_term(); + match accumulated { + None => b.ext_const(constant), + Some(acc) if *constant == FEE::zero() => acc, + Some(acc) => { + let k = b.ext_const(constant); + b.eadd(acc, k) + } + } +} + +/// INSTRUCTIONS [`emit_affine`] emits, added to `cost`. +/// +/// One row per term, less the one a leading coefficient of one saves; one more +/// for a nonzero constant. The interned values are every coefficient that is +/// not one, and the constant when it is carried at all. +pub fn affine_cost(affine: &Aff, cost: &mut Cost) { + for (index, (_, coefficient)) in affine.terms().iter().enumerate() { + if *coefficient == FEE::one() { + // The leading term needs no row: the factor's wire IS the running + // value. Every later one is an add. + if index > 0 { + cost.op(); + } + } else { + cost.constant(*coefficient); + cost.op(); + } + } + + let constant = affine.constant_term(); + if affine.terms().is_empty() { + // A constant expression is one `LFM_CONST` and no arithmetic — and a + // zero constant still costs that row, because the wire has to exist. + cost.constant(*constant); + } else if *constant != FEE::zero() { + cost.constant(*constant); + cost.op(); + } +} + +/// ★ One interaction's two values: the signed multiplicity, and +/// `z − fingerprint`. +/// +/// `alpha_powers[p]` must be `alpha^p`; the element at index `p` rides +/// `alpha^{p+1}`, which is `fingerprint_at`'s `power` counter starting at one +/// with the bus id at `alpha^0`. +/// +/// Panics if an element has no power to ride — an emit-time length, so a +/// mismatch is a bug in the caller rather than a condition to carry at +/// runtime. +pub fn emit_interaction( + b: &mut LfmBuilder, + shape: &Shape, + z: Ext, + alpha_powers: &[Ext], + values: &[Ext], +) -> (Ext, Ext) { + let numerator = emit_affine(b, &shape.numerator, values); + + let mut fingerprint = b.ext_const(&shape.bus_id); + for (index, element) in shape.elements.iter().enumerate() { + let value = emit_affine(b, element, values); + let power = *alpha_powers.get(index + 1).unwrap_or_else(|| { + panic!( + "bus element {index} rides alpha^{}, but only {} powers were supplied", + index + 1, + alpha_powers.len() + ) + }); + fingerprint = b.emul_add(power, value, fingerprint); + } + let denominator = b.esub(z, fingerprint); + + (numerator, denominator) +} + +/// INSTRUCTIONS [`emit_interaction`] emits, added to `cost`. +pub fn interaction_cost(shape: &Shape, cost: &mut Cost) { + affine_cost(&shape.numerator, cost); + // The fingerprint opens at the bus id, one `LFM_CONST`. + cost.constant(shape.bus_id); + for element in &shape.elements { + affine_cost(element, cost); + // One `MulAdd` folding this element's alpha power in. + cost.op(); + } + // `z − fingerprint`. + cost.op(); +} + +/// How many powers of alpha [`emit_interaction`] READS over a whole bus: +/// `1 + max |elements|`. +/// +/// ⚠ ONE FEWER than the host builds. `interactions` sizes its ladder at +/// `width + 1` where `width` is the largest `num_bus_elements`, and +/// `num_bus_elements` COUNTS THE BUS ID (`lookup.rs:1779-1787`) — so the host +/// holds `max |elements| + 2` powers and its last is never read. The values +/// are identical; only the dead tail is missing, which is the same +/// relationship `challenge_powers_rows` already records. +/// +/// ⚠ `alpha^0` is in both ladders and read by neither: `fingerprint_at` seeds +/// its sum with the bus id directly rather than with `alpha_powers[0]·bus_id`. +/// It is kept so the index arithmetic stays the host's. +pub fn alpha_powers_read(shapes: &[Shape]) -> usize { + 1 + shapes + .iter() + .map(|shape| shape.elements.len()) + .max() + .unwrap_or(0) +} + +/// ★ `multilinear::eq::eq_evals`, emitted: the weight of every interaction +/// slot, live and padding alike. +/// +/// The host's own doubling (`eq.rs:49-61`): seed the table with one, then per +/// variable taken in REVERSE scale the live half by `r_i` into the upper half +/// and by `1 − r_i` in place. +pub fn emit_eq_evals(b: &mut LfmBuilder, point: &[Ext]) -> Vec { + let one = b.ext_const(&FEE::one()); + let mut table = vec![one; 1usize << point.len()]; + for (level, r) in point.iter().rev().enumerate() { + let one_minus = b.esub(one, *r); + let half = 1usize << level; + for index in 0..half { + let live = table[index]; + table[half + index] = b.emul(live, *r); + table[index] = b.emul(live, one_minus); + } + } + table +} + +/// INSTRUCTIONS [`emit_eq_evals`] emits over `n` variables, added to `cost`. +/// +/// One `LFM_CONST` for the seed `1`; one `esub` per variable for `1 − r_i`; +/// two multiplies per live entry per level, `Σ_{level { + /// The GKR claim's point, interaction half first. + pub claim_point: &'a [Ext], + /// The table's row variables — the tail of `claim_point`. + pub num_row_vars: usize, + /// The LogUp challenge `z`. + pub z: Ext, + /// `alpha_powers[p]` must be `alpha^p`, and there must be at least + /// [`alpha_powers_read`] of them. + pub alpha_powers: &'a [Ext], + /// The woven factor values the rules index + /// (`constraint_argument.rs:103-136`). + pub values: &'a [Ext], + /// The factor slot holding `eq(row_point, ·)`, which the caller registers + /// as a public factor. + pub weight: usize, +} + +/// The two rule values a table's bus leaves for the main sumcheck. +#[derive(Debug, Clone, Copy)] +pub struct BusValues { + pub numerator: Ext, + pub denominator: Ext, +} + +/// The variables the claim point spans — the host's own +/// `multilinear::logup::input_layer_vars`, called rather than restated. +pub fn claim_point_vars(interactions: usize, num_row_vars: usize) -> usize { + multilinear::logup::input_layer_vars(interactions, num_row_vars) +} + +/// ★ `logup::claim_statements`' two rules, emitted as their VALUES at the +/// factor wires. +/// +/// The host returns two `Program`s and `constraint_argument::verify_core` runs +/// them through `Rule::apply` (`batch.rs:79-96`); a straight-line emitter has +/// no reason to build the program and then walk it, so this is the walk. +/// +/// `claim_point` is the GKR claim's point as wires, interaction half first; +/// `weight` is the factor slot holding `eq(row_point, ·)`, which the caller +/// registers as a public factor. The row half of the claim point is the +/// caller's to turn into that factor, exactly as the host hands `row_point` +/// back. +/// +/// Panics on an empty bus, and on a claim point of the wrong arity. Both are +/// emit-time lengths. The empty case is where this REFUSES what the host +/// merely computes: `claim_statements` would return `row · 0` and `row · 1`, +/// but `TableLayout::new` rejects a table with no interactions (`EmptyPolynomial`, +/// measured by V1d), so there is no such table to emit and a zero-length bus +/// here is a caller's bug. +pub fn emit_claim_statements( + b: &mut LfmBuilder, + shapes: &[Shape], + inputs: &BusInputs<'_>, +) -> BusValues { + let &BusInputs { + claim_point, + num_row_vars, + z, + alpha_powers, + values, + weight, + } = inputs; + assert!( + !shapes.is_empty(), + "a table with no bus interactions does not lay out" + ); + let expected = claim_point_vars(shapes.len(), num_row_vars); + assert_eq!( + claim_point.len(), + expected, + "the claim point spans the interaction bits and the rows" + ); + + let interaction_point = &claim_point[..claim_point.len() - num_row_vars]; + let weights = emit_eq_evals(b, interaction_point); + + // The tail belongs to the 0/1 padding slots, whose denominators are one and + // whose numerators vanish — so it reaches the denominator as a sum and the + // numerator not at all. + let padding = weights[shapes.len()..] + .iter() + .copied() + .reduce(|acc, w| b.eadd(acc, w)); + + let mut numerator: Option = None; + let mut denominator: Option = None; + for (shape, w) in shapes.iter().zip(&weights) { + let (num, den) = emit_interaction(b, shape, z, alpha_powers, values); + numerator = Some(match numerator { + None => b.emul(*w, num), + Some(acc) => b.emul_add(*w, num, acc), + }); + denominator = Some(match denominator { + None => b.emul(*w, den), + Some(acc) => b.emul_add(*w, den, acc), + }); + } + + let mut numerator = numerator.expect("a non-empty bus batches at least one interaction"); + let mut denominator = denominator.expect("a non-empty bus batches at least one interaction"); + if let Some(padding) = padding { + denominator = b.eadd(denominator, padding); + } + + let row = read(values, weight); + numerator = b.emul(row, numerator); + denominator = b.emul(row, denominator); + + BusValues { + numerator, + denominator, + } +} + +/// INSTRUCTIONS [`emit_claim_statements`] emits. +/// +/// Every term by the shape it comes from, with `I = shapes.len()` and +/// `n = ceil_log2(I)`: +/// +/// - the weights, `eq_evals_cost(n)`; +/// - the padding fold, `2^n − I − 1` adds when there are padding slots at all; +/// - each interaction's two values, `interaction_cost`; +/// - `2I` rows batching them, one `Mul` opening each of the two folds and a +/// `MulAdd` for every later interaction; +/// - one add folding the padding into the denominator, when there is one; +/// - two multiplies by the row weight. +pub fn claim_statements_cost(shapes: &[Shape], num_row_vars: usize) -> Cost { + let mut cost = Cost::default(); + let interactions = shapes.len(); + let n = claim_point_vars(interactions, num_row_vars) - num_row_vars; + eq_evals_cost(n, &mut cost); + + let padding = (1usize << n) - interactions; + if padding > 0 { + cost.ops(padding - 1); + } + + for shape in shapes { + interaction_cost(shape, &mut cost); + } + cost.ops(2 * interactions); + if padding > 0 { + cost.op(); + } + cost.ops(2); + + cost +} diff --git a/prover/src/lfm/whir_bus_tests.rs b/prover/src/lfm/whir_bus_tests.rs new file mode 100644 index 000000000..c58b3e230 --- /dev/null +++ b/prover/src/lfm/whir_bus_tests.rs @@ -0,0 +1,726 @@ +//! Gates for the bus statements. +//! +//! Two bus sets, for the reason 2c needed two shapes: +//! +//! - the PRODUCTION one is `l2g_memory_air`'s own interactions through the real +//! `TableLayout::new`, so the slot map, the multiplicities and the packings +//! are the ones a real table hands `interactions`; +//! - the STRESS one carries what that bus has not: a `QuadHL` value worth FOUR +//! bus elements with `2^16` coefficients, a signed multi-term multiplicity +//! with a constant, an element that is a bare constant, and an interaction +//! count that is not a power of two — which is the only way the padding slots +//! exist at all. Said here rather than left to read as coverage. +//! +//! Unlike 2c's, the stress set needs no AIR: `interactions` and +//! `claim_statements` take `&[BusInteraction]`, which is exactly what the +//! production path hands them (`multilinear_table.rs:737-743`), so a hand-built +//! vector is the production input type and not a stand-in for one. + +use multilinear::Error as MlError; +use multilinear::eq::eq_evals; +use multilinear::logup::{self, Interaction, input_layer_vars}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_logup::{InteractionShape, interaction_shapes, interactions}; +use stark::multilinear_table::TableLayout; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_bus::{ + BusInputs, Cost, alpha_powers_read, claim_statements_cost, emit_claim_statements, + emit_eq_evals, emit_interaction, eq_evals_cost, interaction_cost, +}; +use super::word::{ext_word, word_as_ext}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type Shape = InteractionShape; + +fn pseudo(seed: u64, count: usize) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..count) + .map(|_| FEE::new([next(), next(), next()])) + .collect() +} + +fn options() -> stark::proof::options::ProofOptions { + stark::proof::options::GoldilocksCubicProofOptions::with_params(4, 128, 20) + .expect("valid options") +} + +/// A bus set with everything both sides of the gate need to read it. +struct BusCase { + name: &'static str, + buses: Vec, + /// The main width `interactions` filters its probe candidates against. + width: usize, + /// Main column -> the factor that reads it. + slots: Vec, + num_row_vars: usize, +} + +impl BusCase { + fn slot_of(&self) -> impl FnMut(usize) -> Result + '_ { + |column| { + self.slots + .get(column) + .copied() + .ok_or(MlError::UnknownPolynomial { + index: column, + len: self.slots.len(), + }) + } + } + + fn shapes(&self) -> Vec { + interaction_shapes(&self.buses, self.width, self.slot_of()).expect("the bus probes") + } + + fn fused(&self, z: &FEE, alpha: &FEE) -> Vec> { + interactions(&self.buses, self.width, z, alpha, self.slot_of()).expect("the bus probes") + } + + /// One factor value per slot the bus can read, plus the row weight the + /// statements multiply by, which sits in the LAST slot here. + fn num_values(&self) -> usize { + self.slots.iter().copied().max().unwrap_or(0) + 2 + } + + fn weight(&self) -> usize { + self.num_values() - 1 + } +} + +/// The production bus: `l2g_memory_air`'s interactions, laid out by the same +/// `TableLayout::new` both sides of a real proof call. +fn production_case() -> BusCase { + let opts = options(); + let air = + crate::continuation::l2g_memory_air(&opts, crate::tables::local_to_global::epoch_label(1)); + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + crate::tables::local_to_global::cols::NUM_COLUMNS, + 4, + Uniforms::default(), + ) + .expect("the table lays out"); + BusCase { + name: "l2g_memory (production bus, production slot map)", + buses: air.bus_interactions().to_vec(), + width: crate::tables::local_to_global::cols::NUM_COLUMNS, + slots: layout.slot_of().to_vec(), + num_row_vars: 4, + } +} + +/// The terms the production bus does not have. +fn stress_case() -> BusCase { + let buses = vec![ + // Four bus elements out of eight columns, with the packing's own 2^16 + // coefficients — and a multiplicity that is a bare one, so its + // numerator is an affine with NO terms at all. + BusInteraction::sender(7u64, Multiplicity::One, Packing::QuadHL.columns(&[0])), + // A receiver, so the numerator carries the sign; a multiplicity with + // signed coefficients and a constant; an element that is a scaled + // column plus a constant; and an element read with coefficient one, + // which is the term that opens a fold for free. + BusInteraction::receiver( + 7u64, + Multiplicity::Linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: 8, + }, + LinearTerm::Column { + coefficient: -3, + column: 9, + }, + LinearTerm::Constant(5), + ]), + vec![ + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 2, + column: 10, + }, + LinearTerm::Constant(11), + ]), + BusValue::column(11), + ], + ), + // A second bus id, and an element that is a constant and nothing else. + BusInteraction::sender(9u64, Multiplicity::Column(12), vec![BusValue::constant(42)]), + ]; + BusCase { + name: "stress (QuadHL, signed multiplicity, 3 interactions)", + buses, + width: 13, + slots: (0..13).collect(), + num_row_vars: 3, + } +} + +fn cases() -> Vec { + vec![production_case(), stress_case()] +} + +/// The arena a bus program reads: `z`, the alpha ladder, the claim point, then +/// the factor values. +struct Inputs { + z: FEE, + alpha_powers: Vec, + claim_point: Vec, + values: Vec, +} + +impl Inputs { + fn words(&self) -> Vec> { + vec![ + std::iter::once(&self.z) + .chain(&self.alpha_powers) + .chain(&self.claim_point) + .chain(&self.values) + .map(ext_word) + .collect(), + ] + } + + fn len(&self) -> usize { + 1 + self.alpha_powers.len() + self.claim_point.len() + self.values.len() + } +} + +fn inputs(case: &BusCase, shapes: &[Shape], z: FEE, alpha: FEE, seed: u64) -> Inputs { + let count = alpha_powers_read(shapes); + let mut alpha_powers = Vec::with_capacity(count); + let mut power = FEE::one(); + for _ in 0..count { + alpha_powers.push(power); + power *= alpha; + } + Inputs { + z, + alpha_powers, + claim_point: pseudo(seed, input_layer_vars(shapes.len(), case.num_row_vars)), + values: pseudo(seed ^ 0x5A5A, case.num_values()), + } +} + +/// The same hinted arena for every program under gate, so "with the leg" and +/// "without it" differ by the leg alone. +fn hint_inputs( + b: &mut LfmBuilder, + shape: &Inputs, +) -> ( + super::builder::Ext, + Vec, + Vec, + Vec, +) { + let arena = b.declare_arena(shape.len() as u32); + let mut index = 0u32; + let mut take = |b: &mut LfmBuilder, count: usize| -> Vec { + (0..count) + .map(|_| { + let wire = b.hint_word(arena, index).as_ext(); + index += 1; + wire + }) + .collect() + }; + let z = take(b, 1)[0]; + let alpha_powers = take(b, shape.alpha_powers.len()); + let claim_point = take(b, shape.claim_point.len()); + let values = take(b, shape.values.len()); + (z, alpha_powers, claim_point, values) +} + +/// The leg's whole program: both rule values published. +fn statements_program(case: &BusCase, shapes: &[Shape], shape: &Inputs) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (z, alpha_powers, claim_point, values) = hint_inputs(&mut b, shape); + let out = emit_claim_statements( + &mut b, + shapes, + &BusInputs { + claim_point: &claim_point, + num_row_vars: case.num_row_vars, + z, + alpha_powers: &alpha_powers, + values: &values, + weight: case.weight(), + }, + ); + b.public(out.numerator.as_cell()); + b.public(out.denominator.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the bus leg must be admissible"); + program +} + +/// The same program without the leg: the marginal is the leg. +fn empty_program(shape: &Inputs) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (z, _alpha, _point, values) = hint_inputs(&mut b, shape); + b.public(z.as_cell()); + b.public(values[0].as_cell()); + compile(b.finish()) +} + +fn run(program: &LfmProgram, shape: &Inputs, name: &str) -> Vec { + let exec = execute(program, &shape.words(), &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("{name}: the bus leg must execute: {e:?}")); + exec.public_words + .iter() + .map(|(_, word)| word_as_ext(word).expect("a published extension value")) + .collect() +} + +/// Three challenge sets, because an identity that only holds at one `alpha` is +/// not one and a gate at a fixed `z` passes an emitter that baked it in. +fn challenges() -> Vec<(FEE, FEE)> { + vec![ + (FEE::from(7u64), FEE::from(11u64)), + ( + FEE::new([FE::from(3u64), FE::from(5u64), FE::from(9u64)]), + FEE::new([FE::from(2u64), FE::from(0u64), FE::from(4u64)]), + ), + (pseudo(0xD00Du64, 1)[0], pseudo(0xFEEDu64, 1)[0]), + ] +} + +/// ★ G1 — the weights are the host's `eq_evals`, entry for entry. +/// +/// Every interaction's weight and every padding slot's comes out of this one +/// table, so a table that is right only on its live prefix would still settle +/// the wrong padding. +#[test] +fn the_weights_are_the_hosts_eq_table() { + for n in 0..=5usize { + let point = pseudo(0x1234 + n as u64, n); + let shape = Inputs { + z: FEE::zero(), + alpha_powers: Vec::new(), + claim_point: point.clone(), + values: Vec::new(), + }; + + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (_z, _alpha, wires, _values) = hint_inputs(&mut b, &shape); + let table = emit_eq_evals(&mut b, &wires); + for entry in &table { + b.public(entry.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the weight table must be admissible"); + + let got = run(&program, &shape, "eq_evals"); + let want = eq_evals(&point); + assert_eq!(got.len(), 1usize << n, "the table's length at n={n}"); + assert_eq!(got, want, "the weight table at n={n}"); + } +} + +/// ★ G1b — F1 for the weights: the closed form `1 + n + 2·(2^n − 1)`. +#[test] +fn the_weight_table_emits_its_closed_form() { + for n in 0..=5usize { + let point = pseudo(0x1234 + n as u64, n); + let shape = Inputs { + z: FEE::zero(), + alpha_powers: Vec::new(), + claim_point: point, + values: Vec::new(), + }; + let with = { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (z, _alpha, wires, _values) = hint_inputs(&mut b, &shape); + let table = emit_eq_evals(&mut b, &wires); + b.public(table[0].as_cell()); + b.public(z.as_cell()); + compile(b.finish()) + }; + let without = { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (z, _alpha, wires, _values) = hint_inputs(&mut b, &shape); + let first = wires.first().copied().unwrap_or(z); + b.public(first.as_cell()); + b.public(z.as_cell()); + compile(b.finish()) + }; + let measured = with.instrs.len() - without.instrs.len(); + + let mut cost = Cost::default(); + eq_evals_cost(n, &mut cost); + let closed = 1 + n + 2 * ((1usize << n) - 1); + assert_eq!( + cost.rows(), + closed, + "the accumulator and the closed form must agree at n={n}" + ); + println!("eq_evals n={n}: {measured} rows emitted, {closed} predicted"); + assert_eq!(measured, closed, "at n={n}"); + } +} + +/// ★ G2 — each interaction's two values are what `interactions` builds. +/// +/// Per interaction and not per sum: a leg that got the batch right by +/// compensating two errors passes a total and fails this. +#[test] +fn every_interaction_is_what_the_host_probes() { + for case in cases() { + let shapes = case.shapes(); + for (index, (z, alpha)) in challenges().into_iter().enumerate() { + let shape = inputs(&case, &shapes, z, alpha, 0xB0_5A11 + index as u64); + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let (z_wire, alpha_wires, _point, values) = hint_inputs(&mut b, &shape); + for interaction in &shapes { + let (num, den) = + emit_interaction(&mut b, interaction, z_wire, &alpha_wires, &values); + b.public(num.as_cell()); + b.public(den.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the interaction leg must be admissible"); + + let got = run(&program, &shape, case.name); + let fused = case.fused(&z, &alpha); + assert_eq!(fused.len(), shapes.len(), "{}", case.name); + for (i, interaction) in fused.iter().enumerate() { + assert_eq!( + got[2 * i], + interaction.numerator.evaluate(&shape.values), + "{} interaction {i} numerator at challenge set {index}", + case.name + ); + assert_eq!( + got[2 * i + 1], + interaction.denominator.evaluate(&shape.values), + "{} interaction {i} denominator at challenge set {index}", + case.name + ); + } + } + } +} + +/// ★ G3 — the two rule VALUES are what `claim_statements` applies. +/// +/// The host returns programs; `constraint_argument::verify_core` runs them +/// through `Rule::apply`, which is the quantity this leg has to land on. +#[test] +fn the_bus_statements_compute_what_the_host_computes() { + for case in cases() { + let shapes = case.shapes(); + for (index, (z, alpha)) in challenges().into_iter().enumerate() { + let shape = inputs(&case, &shapes, z, alpha, 0xC0_FFEE + index as u64); + let program = statements_program(&case, &shapes, &shape); + let got = run(&program, &shape, case.name); + + let fused = case.fused(&z, &alpha); + let statements = logup::claim_statements( + &fused, + &shape.claim_point, + case.num_row_vars, + case.weight(), + ) + .expect("the host builds its rules"); + assert_eq!( + got[0], + statements.numerator.apply(&shape.values), + "{} numerator at challenge set {index}", + case.name + ); + assert_eq!( + got[1], + statements.denominator.apply(&shape.values), + "{} denominator at challenge set {index}", + case.name + ); + assert_eq!( + statements.row_point.len(), + case.num_row_vars, + "the row half of the claim point" + ); + } + println!( + "bus {}: {} interactions, {} row vars", + case.name, + case.shapes().len(), + case.num_row_vars + ); + } +} + +/// ★ G4 — F1 for the whole leg. +#[test] +fn the_bus_statements_emit_their_closed_form() { + for case in cases() { + let shapes = case.shapes(); + let shape = inputs(&case, &shapes, FEE::from(13u64), FEE::from(17u64), 0xF00D); + let measured = statements_program(&case, &shapes, &shape).instrs.len() + - empty_program(&shape).instrs.len(); + let cost = claim_statements_cost(&shapes, case.num_row_vars); + let n = input_layer_vars(shapes.len(), case.num_row_vars) - case.num_row_vars; + println!( + "bus {}: {measured} rows emitted, {} predicted ({} ops + {} constants; \ + I={} 2^n={} alpha powers read={})", + case.name, + cost.rows(), + cost.operations(), + cost.constants(), + shapes.len(), + 1usize << n, + alpha_powers_read(&shapes), + ); + assert_eq!(measured, cost.rows(), "{}", case.name); + } +} + +/// The per-interaction form sums to the whole leg's, less the parts only the +/// batch has. +/// +/// Not a restatement: `interaction_cost` is what item 2e's per-table form will +/// call, and this is the statement that calling it that way reaches the same +/// number the whole-leg gate above pinned. +#[test] +fn the_per_interaction_cost_is_the_legs_own() { + for case in cases() { + let shapes = case.shapes(); + let whole = claim_statements_cost(&shapes, case.num_row_vars); + let mut parts = Cost::default(); + let n = input_layer_vars(shapes.len(), case.num_row_vars) - case.num_row_vars; + eq_evals_cost(n, &mut parts); + for shape in &shapes { + interaction_cost(shape, &mut parts); + } + let padding = (1usize << n) - shapes.len(); + let batch = 2 * shapes.len() + 2 + if padding > 0 { padding } else { 0 }; + assert_eq!( + whole.rows(), + parts.rows() + batch, + "{}: the batch's own rows are {batch}", + case.name + ); + } +} + +/// `Affine::constant_term` is the `k` of the expression and not some other +/// field: it is what the host's own evaluator returns at the origin. +#[test] +fn the_affine_getter_is_the_hosts_constant() { + for case in cases() { + let shapes = case.shapes(); + let zeros = vec![FEE::zero(); case.num_values()]; + for (i, shape) in shapes.iter().enumerate() { + assert_eq!( + shape.numerator.constant_term(), + &shape.numerator.evaluate(&zeros), + "{} interaction {i}'s numerator", + case.name + ); + for (p, element) in shape.elements.iter().enumerate() { + assert_eq!( + element.constant_term(), + &element.evaluate(&zeros), + "{} interaction {i}'s element {p}", + case.name + ); + } + } + } +} + +/// The stress bus exists to carry terms the production one does not; this is +/// the assertion that it actually does, so a later edit cannot quietly turn it +/// into a second copy of the first. +#[test] +fn the_stress_bus_carries_what_the_production_one_does_not() { + let stress = stress_case(); + let shapes = stress.shapes(); + assert_eq!( + shapes.len(), + 3, + "not a power of two, so padding slots exist" + ); + assert_eq!(shapes[0].elements.len(), 4, "QuadHL is four bus elements"); + assert!( + shapes[0].numerator.terms().is_empty(), + "a multiplicity of one is an affine with no terms" + ); + assert!( + shapes[1] + .numerator + .terms() + .iter() + .any(|(_, c)| *c != FEE::one() && *c != -FEE::one()), + "the signed multiplicity must carry a coefficient that is not a unit" + ); + assert_ne!( + shapes[1].numerator.constant_term(), + &FEE::zero(), + "the multiplicity's constant must survive the probe" + ); + assert!( + shapes[2].elements[0].terms().is_empty(), + "a constant bus value is an element with no terms" + ); + assert!( + shapes[0] + .elements + .iter() + .any(|e| e.terms().iter().any(|(_, c)| *c != FEE::one())), + "the packing's 2^16 coefficients must reach the affine" + ); +} + +/// ★ The row form recounted from the shapes, by a walk that is not +/// `affine_cost`'s. +/// +/// The operations an interaction costs are: one per affine TERM, less the one a +/// leading coefficient of one saves; one per affine that carries a nonzero +/// constant beside at least one term; one per bus element folding its alpha +/// power in; and one for `z − fingerprint`. Counted here off the shapes +/// directly, so a drift in the emitter's accumulator has a second opinion — +/// and the saving the leg actually takes is named rather than implied. +#[test] +fn the_row_form_recounts_from_the_shapes() { + for case in cases() { + let shapes = case.shapes(); + let affines = |shape: &Shape| -> Vec> { + std::iter::once(shape.numerator.clone()) + .chain(shape.elements.iter().cloned()) + .collect() + }; + + let mut leading_units = 0usize; + let mut recount = 0usize; + for shape in &shapes { + for affine in affines(shape) { + recount += affine.terms().len(); + if affine + .terms() + .first() + .is_some_and(|(_, c)| *c == FEE::one()) + { + leading_units += 1; + recount -= 1; + } + if !affine.terms().is_empty() && *affine.constant_term() != FEE::zero() { + recount += 1; + } + } + recount += shape.elements.len() + 1; + } + + let mut charged = Cost::default(); + for shape in &shapes { + interaction_cost(shape, &mut charged); + } + println!( + "bus {}: {leading_units} affines open on a unit coefficient and cost no row; \ + {} operations charged, {recount} recounted", + case.name, + charged.operations() + ); + assert_eq!(charged.operations(), recount, "{}", case.name); + assert!( + leading_units > 0, + "{}: some affine must open on a unit coefficient, or the saving is untested", + case.name + ); + } +} + +/// The alpha ladder's length is what the elements READ, and the host's is two +/// longer. +#[test] +fn the_alpha_ladder_is_as_long_as_the_widest_interaction() { + for case in cases() { + let shapes = case.shapes(); + let widest = shapes.iter().map(|s| s.elements.len()).max().unwrap(); + assert_eq!(alpha_powers_read(&shapes), widest + 1); + // `num_bus_elements` COUNTS THE BUS ID, and `interactions` sizes its + // ladder at `width + 1` — so the host holds one power past the last + // index `fingerprint_at` ever reads. + let host = case + .buses + .iter() + .map(BusInteraction::num_bus_elements) + .max() + .unwrap() + + 1; + assert_eq!(host, widest + 2, "{}", case.name); + assert_eq!( + host - alpha_powers_read(&shapes), + 1, + "{}: the ladder this leg emits is exactly one shorter", + case.name + ); + } +} + +/// A gate that can fail on the thing it names: the bus values must not be +/// independent of the factor values. +/// +/// Both rules multiply by the row weight and by the interaction weights, so a +/// leg that dropped the factors entirely would still produce something that +/// moves with the challenges. This moves a FACTOR and demands the answer +/// follows. +#[test] +fn the_statements_read_the_factor_values() { + for case in cases() { + let shapes = case.shapes(); + let (z, alpha) = challenges()[0]; + let base = inputs(&case, &shapes, z, alpha, 0xAB_CDEF); + let program = statements_program(&case, &shapes, &base); + let first = run(&program, &base, case.name); + + let mut moved = inputs(&case, &shapes, z, alpha, 0xAB_CDEF); + // A slot some element actually reads — moving one nothing reads would + // leave the answer alone for a correct leg too. + let slot = shapes + .iter() + .flat_map(|s| &s.elements) + .flat_map(|e| e.terms()) + .map(|(slot, _)| *slot) + .next() + .expect("the bus reads at least one factor"); + moved.values[slot] += FEE::one(); + let second = run(&program, &moved, case.name); + assert_ne!( + first[1], second[1], + "{}: moving factor {slot} must move the denominator", + case.name + ); + } +} + +/// `BusValues` is two distinct wires, not one published twice. +#[test] +fn the_two_rules_are_different_values() { + let case = stress_case(); + let shapes = case.shapes(); + let (z, alpha) = challenges()[0]; + let shape = inputs(&case, &shapes, z, alpha, 0x2468); + let out = run( + &statements_program(&case, &shapes, &shape), + &shape, + case.name, + ); + assert_ne!(out[0], out[1], "the numerator and denominator must differ"); +} diff --git a/prover/src/lfm/whir_chain.rs b/prover/src/lfm/whir_chain.rs new file mode 100644 index 000000000..d5222277e --- /dev/null +++ b/prover/src/lfm/whir_chain.rs @@ -0,0 +1,2620 @@ +//! `whir_chain::verify_weighted` as a machine leg — the WHIR chain, assembled. +//! +//! This is the whole of `crypto/multilinear/src/whir_chain.rs:938-1105` and it +//! is where the wrap's cost lives: about ninety per cent of a chain's +//! permutations are the query phase's openings, and a chain is run once per +//! stacked polynomial. +//! +//! Every brick is already gated on its own against the host function it +//! replaces — the grind against `is_valid_nonce` +//! ([`super::whir_transcript::emit_grind_check`]), the degree-2 sumcheck +//! against `verify_rounds` ([`super::whir_poly`]), `eq` against `eq_eval`, the +//! opening against `verify_opening` ([`super::whir_open`]), the fold against +//! `fold_coset` ([`super::whir_fold`]), and the transcript against the K1–K6 +//! vectors. What this module adds is the ORDER, and the order is exactly what +//! its own gate checks: the host's challenge stream, observed through a +//! recording transcript, against the stream the machine derives from the +//! sponge. +//! +//! # The round structure, and the three places a nonce is spent +//! +//! Per round: `check_grind(folding)`; `k_r` sumcheck rounds at degree two, each +//! absorbing its two evaluations and drawing one challenge; then the domain is +//! squared `k_r` times, which is emit-time and free. A round that has a +//! successor absorbs its root, draws `z0`, refuses an in-domain `z0`, absorbs +//! `y0`, spends the OOD nonce, draws `gamma`, folds `gamma·y0` into the claim, +//! spends the QUERY nonce, and opens `Q` queries against both trees. The last +//! round absorbs `final_value` instead, spends only the query nonce, and opens +//! `Q` queries against one tree — `verify_final` has no successor +//! (`whir_chain.rs:1125-1141`). That is why a uniformly ground chain has +//! `3R − 1` grinds. +//! +//! A grind of zero bits is not emitted at all. Which nonces the ARENA carries +//! is the config's [`NonceLayout`]: three a round under `Three` (today's +//! format, the unspent ones hinted and never read), only the spent ones under +//! `Spent`. P2 grinds before the queries alone (`GrindBits::query_only`) under +//! `Spent`, so a round carries ONE nonce word and an unspent nonce has no word +//! to be set in: [`ChainShape::carries`]. +//! +//! # ★ The out-of-domain squarings are SHARED, and exactly +//! +//! `require_out_of_domain` squares `z0` once per bit of the successor domain +//! (`:93-96`) and `ood_point` keeps `z0^(2^j)` for `j < num_vars − bound` +//! (`:72-81`). Those are the same chain of squarings, and the second is a +//! PREFIX of the first: `bound_r` variables are bound after round `r`, the +//! successor domain is `2^(num_vars + 2 − bound_r)`, so the domain check needs +//! exactly two more squarings than the point does — always, at every round and +//! every stack height. One chain of `D_{r+1}` multiplies serves both. Computing +//! them apart would cost `2·D_{r+1} − 2`, which at `S = 25` is 156 multiplies a +//! chain against 78. +//! +//! # ★ The successor slot is a MUX, and it is not free +//! +//! `leaf_and_slot` splits the query into a leaf index and a slot +//! (`whir_commit.rs:119-121`); both bounds are powers of two, so the split is a +//! partition of the `BitDec`'s bits and costs nothing. But `nxt.values[slot]` +//! (`whir_round.rs:160`) then selects one of `2^k` extension values by the high +//! bits, which is `2^k − 1` `Select` rows a query a round. At `k = 4`, `Q = +//! 112` and six successor rounds that is 8,400 rows a chain. The slot is always +//! in range — `slot < 2^{k_{r+1}}` follows from `q < 2^{D_{r+1}}` — so there is +//! no out-of-range branch to emit, which is the host's `QueryOutOfRange` made +//! unreachable rather than checked. +//! +//! # ⚠ The tail divides, and must +//! +//! The host computes `required = claim · weight⁻¹` and rejects a zero weight +//! with `DegenerateEvaluationPoint` (`:1095-1102`). Asserting +//! `final_value · weight == claim` instead would be one row cheaper and +//! strictly WEAKER: it accepts every `final_value` when `weight` and `claim` +//! are both zero, where the host rejects. So the tail inverts — +//! `div(one, weight)` has no satisfying assignment at zero, which IS the +//! refusal — and then compares. +//! +//! # What is emit-time, and therefore has no runtime check +//! +//! The host's shape rejections (`proof.rounds.len() != schedule.len()`, +//! `round.sumcheck.len() != k`, the base-versus-extension opening variant at +//! `:983`, `openings.current.len() != num_queries`) are all properties of the +//! PROGRAM here: the emitter reads a fixed number of wires at fixed offsets, so +//! a proof of another shape has no way to be supplied. That is the +//! `epoch_verify.rs:171-179` idiom — the absence of a second value rather than +//! an assert somebody could forget. + +use multilinear::stacked_eval::index_chains; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + ChainConfig, ChainProof, ChainRound, GroupFold, LeafOrder, MAX_INDEX_POLYS, NonceLayout, + RoundOpenings, index_vars, +}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; +use stark::config::Commitment; + +use super::algebraic_commit::commitment_to_digest; +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::edsl::WrapDigest; +use super::whir_fold::{RoundFold, fold_coset_rows_lean, lean_prepare_rows}; +use super::whir_open::{ + BlockValues, CapCells, Siblings, TreeAuth, cap_check_perms, cap_check_rows, + verify_opening_perms_capped, verify_opening_rows_capped, +}; +use super::whir_poly::{ + emit_eq_eval, emit_sumcheck_round, eq_eval_rows_again, sumcheck_round_rows, +}; +use super::whir_transcript::{ + COORDINATES_PER_EXT, SpongeEntry, SpongeHash, SpongeSchedule, WhirTranscript, emit_grind_check, +}; +use super::word::{LfmWord, ext_word}; + +/// The degree the weight raises the plain `f` term to (`whir_chain.rs:991`). +pub(crate) const SUMCHECK_DEGREE: usize = 2; + +/// One query's opening against one tree: the block, and its authentication +/// path. +pub struct QueryOpening<'a> { + pub values: BlockValues<'a>, + pub siblings: Siblings<'a>, +} + +/// A round's nonce wires, one per slot the arena carries +/// ([`ChainShape::carries`]). A slot is read only when its grind spends bits, +/// and every such slot is carried, so `None` is a slot nothing reads. Under +/// [`NonceLayout::Three`] all three are hinted and the unspent ones (zero bits, +/// and the out-of-domain one on the last round) are not read. +#[derive(Clone, Copy)] +pub struct RoundNonces { + pub folding: Option, + pub ood: Option, + pub query: Option, +} + +/// One round of the chain, as wires. +pub struct ChainRoundWires<'a> { + /// `k_r` sumcheck rounds, each carrying `SUMCHECK_DEGREE` evaluations. + pub sumcheck: &'a [Vec], + /// The successor's root as a word. `None` on the last round, where the + /// folded message is a constant and `final_value` is sent instead. + pub next_root: Option, + /// The successor's value at the out-of-domain point. `None` on the last + /// round. + pub ood_value: Option, + pub nonces: RoundNonces, + /// Per query: the block of the current codeword that folds onto the query. + /// A grouped round 0 holds one block a tree, query-major + /// (`num_queries · trees`, [`ChainShape::trees_at`]). + pub current: &'a [QueryOpening<'a>], + /// Per query: the successor block holding the folded value. Empty on the + /// last round. + pub next: &'a [QueryOpening<'a>], + /// The current tree's Merkle cap, when this round OWNS it: round 0 with + /// `caps[0] > 0` — a grouped round 0's trees' caps, in tree order. Empty + /// otherwise (W1; a later round's current tree was authenticated as the + /// round before's successor). + pub current_cap: &'a [WrapDigest], + /// The successor tree's Merkle cap, when it has one (`caps[r + 1] > 0`). + pub next_cap: &'a [WrapDigest], +} + +/// The shape of one chain: everything the closed forms below are a function of. +/// +/// Built from the same `ChainConfig::schedule` the host runs, so a change to +/// the schedule moves both sides together rather than only one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChainShape { + /// How the recursion leaf checks this chain's queries + /// ([`LeafQuery`]): [`LeafQuery::V1`] for every chain but a block leaf's + /// under its format's `leaf_query`. + pub query: LeafQuery, + /// Variables folded in each round. + pub schedule: Vec, + /// `log2` of the domain each round starts at: `D_0 = num_vars + + /// log_blowup`, then `D_{r+1} = D_r − k_r`. + pub domain_log: Vec, + pub num_vars: usize, + pub num_queries: usize, + pub grind: (usize, usize, usize), + /// Each tree's Merkle cap height (W1): tree `r` is round `r`'s current + /// tree. From the same `ChainConfig::tree_caps` the host prover and + /// verifier use; all zero at the default format. + pub caps: Vec, + /// Which nonces a round's arena words carry (P2): the config's own. + pub nonces: NonceLayout, + /// Children per Merkle node: 2, or 4 for a Poseidon1 base's trees, whose + /// caps then count 4-ary levels (`ChainConfig::tree_caps_at`). + pub arity: usize, + /// How a stack's polynomials are cut into chains (D-ONECHAIN, W-14): the + /// config's own, so the emitter, the arena and the plan split a stack as + /// the host does ([`Self::chains`]). + pub group: GroupFold, + /// The polynomials this chain opens: one, or a run of up to + /// `MAX_INDEX_POLYS` under [`GroupFold::Indexed`] ([`Self::run`]), whose + /// round 0 opens one tree each at one position set and binds `m = + /// ⌈log2 trees⌉` index variables after its `k0` data ones. + pub trees: usize, + /// Where the round-0 trees put their leaves: the config's own, so round + /// 0's walks authenticate the position the host prover placed the block + /// at ([`walk_bits`]). + pub leaves: LeafOrder, +} + +/// How a block's recursion leaf checks a chain's queries: a verifier +/// constant (`BlockFormat::leaf_query`) the prover's arena and the leaf +/// program both follow, never read from a proof. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum LeafQuery { + /// Every round's queries opened and folded one by one. Selectable by name + /// (`v1`); the leaves' check before [`Self::Lean`]. + V1, + /// As [`Self::V1`], except round 0's base values arrive four to a hinted + /// word ([`ChainShape::round0_packed`]) and the chain's tail rounds + /// ([`ChainShape::tail_round`]): a round whose current tree lies wholly in + /// its cap, with at least [`TAIL_QUERIES_A_LEAF`] queries a leaf, is + /// checked at EVERY leaf once, each leaf's block hashed against its cap + /// node, folded at its constant position and compared with the successor + /// block (or the final value). A superset of the sampled positions, so a + /// leaf accepts only what the host verifier's query checks for those + /// rounds accept. The rounds' query draws stay in the transcript, their + /// bits unused. The default. + #[default] + Lean, +} + +/// Queries a tail tree's leaf ([`ChainShape::tail_round`]) needs at least. +/// +/// The arena takes a tail tree's every block from the proof, which opens the +/// tree at `2Q` uniform positions (the round before's successor openings and +/// the round's own). A tree of `L` leaves misses one with probability at most +/// `L · (1 − 1/L)^(2Q) ≤ L · e^(−2Q/L)`; at `Q ≥ 24 L` that is `L · e^(−48)`, +/// below `2^−63` for every tree of at most 64 leaves. A miss leaves an honest +/// proof without an arena (completeness, never soundness). +pub const TAIL_QUERIES_A_LEAF: usize = 24; + +impl ChainShape { + pub fn new(config: &ChainConfig, num_vars: usize) -> Self { + Self::new_at(config, num_vars, 2) + } + + /// [`Self::new`] over trees of `arity` children per node: the hash's + /// arity, as the host's prover and verifier take it. + pub fn new_at(config: &ChainConfig, num_vars: usize, arity: usize) -> Self { + let schedule = config.schedule(num_vars); + let mut domain_log = Vec::with_capacity(schedule.len()); + let mut d = num_vars + config.log_blowup; + for &k in &schedule { + domain_log.push(d); + d -= k; + } + let caps = config.tree_caps_at(num_vars, arity); + debug_assert_eq!(caps.len(), schedule.len(), "one cap height per tree"); + Self { + query: LeafQuery::V1, + schedule, + domain_log, + num_vars, + num_queries: config.num_queries, + caps, + grind: ( + config.grind.folding as usize, + config.grind.ood as usize, + config.grind.query as usize, + ), + nonces: config.format.nonces, + arity, + group: config.format.group, + trees: 1, + leaves: config.format.leaves, + } + } + + pub fn rounds(&self) -> usize { + self.schedule.len() + } + + /// ★ Whether round `r` is a tail round under [`LeafQuery::Lean`]: checked + /// at every leaf of its current tree once instead of at each query. Round + /// `r ≥ 1`, and it and every later round have a single current tree lying + /// wholly in its 4-ary cap (no walked level, `mux_bits` = the depth) with + /// at least [`TAIL_QUERIES_A_LEAF`] queries a leaf: a suffix of the chain, + /// so a tail round's successor is a tail round too. Derived from the shape + /// alone, so the arena, the leaf and the cost model agree. + pub fn tail_round(&self, r: usize) -> bool { + self.query == LeafQuery::Lean + && r >= 1 + && r < self.rounds() + && (r..self.rounds()).all(|t| self.tail_shaped(t)) + } + + /// Round `r`'s current tree on its own: one tree, at arity 4, capped, its + /// every leaf a cap node, and [`TAIL_QUERIES_A_LEAF`] queries a leaf. + fn tail_shaped(&self, r: usize) -> bool { + let depth = self.current_depth(r); + let c = self.caps[r]; + self.arity == 4 + && self.trees_at(r) == 1 + && c > 0 + && super::merkle_cap::mux_bits(depth, c, 4) == depth + && TAIL_QUERIES_A_LEAF << depth <= self.num_queries + } + + /// Whether round 0's base values arrive four to a hinted word + /// ([`LeafQuery::Lean`] on a 4-ary tree): hashed as the leaf hash's rate + /// words and unpacked into the felts the fold reads. + pub fn round0_packed(&self) -> bool { + self.query == LeafQuery::Lean && self.arity == 4 + } + + /// Whether the openings' paths stay in the arena until their walk, which + /// hints each 4-ary level's siblings in hint order on its `b1` bit and + /// places them with three `Select`s ([`LeafQuery::Lean`] on a 4-ary tree, + /// [`Siblings::Rotated`]). The arena keeps the proof's child order. + pub fn paths_rotated(&self) -> bool { + self.query == LeafQuery::Lean && self.arity == 4 + } + + /// Hinted words one round-0 block of `block` values takes: one a value, or + /// four values a word when packed. + pub fn round0_block_words(&self, block: usize) -> usize { + if self.round0_packed() { + block.div_ceil(4) + } else { + block + } + } + + /// Leaves of round `r`'s current tree: the blocks a tail round checks. + pub fn current_leaves(&self, r: usize) -> usize { + 1usize << self.current_depth(r) + } + + /// The chains a stack of `num_polys` polynomials is opened by, each a range + /// of its polynomials: the host's own split (`stacked_eval::index_chains`, + /// REV-B2-JUDGE B2-1), which the emitter, the arena and the plan all cut by. + pub fn chains(&self, num_polys: usize) -> Vec> { + index_chains(num_polys, self.group) + } + + /// This shape for a chain over `trees` polynomials, a run of + /// [`Self::chains`]: the folds, trees, caps and `Q` stay the + /// per-polynomial chain's, and round 0 carries the index (REV-B2-JUDGE J1). + pub fn run(&self, trees: usize) -> Self { + assert_eq!(self.trees, 1, "a run is cut from a per-polynomial shape"); + assert!( + (1..=MAX_INDEX_POLYS).contains(&trees) + && (trees == 1 || self.group == GroupFold::Indexed), + "a run of {trees} polynomials under {:?}", + self.group + ); + // Lemma B's virtual domain `H_{n+m+lb}` must exist (REV-B2-JUDGE B1-1): + // the host refuses such a config before reading, and no program for + // one is emitted. + let two_adicity = + ::TWO_ADICITY as usize; + assert!( + self.domain_log[0] + index_vars(trees) <= two_adicity, + "a run of {trees} over 2^{} has no virtual domain", + self.domain_log[0] + ); + Self { + trees, + ..self.clone() + } + } + + /// `m`: the index variables round 0 binds after its data ones (0 for one + /// polynomial, today's chain). + pub fn index_vars(&self) -> usize { + index_vars(self.trees) + } + + /// Sumcheck messages round `r` carries: its folds, and at round 0 the `m` + /// index rounds after them (`k0 + m`, `ChainConfig::index_schedule`). + pub fn messages(&self, r: usize) -> usize { + self.schedule[r] + if r == 0 { self.index_vars() } else { 0 } + } + + /// The trees round `r` opens at each query position: the run's at round 0, + /// the one successor after it. + pub fn trees_at(&self, r: usize) -> usize { + if r == 0 { self.trees } else { 1 } + } + + /// Which of round `r`'s nonce slots (folding, out-of-domain, query) have an + /// arena word: all three under [`NonceLayout::Three`], the spent ones + /// under [`NonceLayout::Spent`], a grind of nonzero bits and the + /// out-of-domain one only where the round has a successor. + /// + /// ★ The one place the nonce layout is written: the arena's word count, + /// the hints that read it and the words that fill it all take it from here. + pub fn carries(&self, r: usize) -> [bool; 3] { + let (folding, ood, query) = self.grind; + match self.nonces { + NonceLayout::Three => [true; 3], + NonceLayout::Spent => [ + folding > 0, + ood > 0 && self.next_depth(r).is_some(), + query > 0, + ], + } + } + + /// The current tree's depth at round `r`: its leaves are the round's domain + /// folded by that round's `k`. + pub fn current_depth(&self, r: usize) -> usize { + self.domain_log[r] - self.schedule[r] + } + + /// The successor tree's depth at round `r`, which is the NEXT round's + /// current depth. Only rounds before the last have one. + pub fn next_depth(&self, r: usize) -> Option { + (r + 1 < self.rounds()).then(|| self.current_depth(r + 1)) + } + + /// Round `r`'s current tree's cap height. ⚠ [`current_depth`](Self::current_depth) + /// stays the index-bit count; the sibling count is + /// [`current_path`](Self::current_path). + pub fn current_cap(&self, r: usize) -> usize { + self.caps[r] + } + + /// Siblings on a path to round `r`'s current tree's cap. + pub fn current_path(&self, r: usize) -> usize { + self.path_words(self.current_depth(r), self.caps[r]) + } + + /// Sibling words one opening's path to a height-`c` cap carries in a tree + /// of `depth` index bits: `depth − c` at arity 2, three a walked 4-ary + /// level and one at an uncapped odd top at arity 4 + /// ([`super::merkle_cap::path_hints`]). + pub fn path_words(&self, depth: usize, c: usize) -> usize { + if self.arity == 4 { + super::merkle_cap::path_hints(depth, c, 4) + } else { + depth - c + } + } + + /// Words a height-`c` cap of a tree of `depth` index bits occupies: `2^c` + /// at arity 2, its real nodes at arity 4 + /// ([`super::merkle_cap::cap_nodes`]); none uncapped. + pub fn cap_words(&self, depth: usize, c: usize) -> usize { + match (c, self.arity) { + (0, _) => 0, + (_, 4) => super::merkle_cap::cap_nodes(depth, c, 4), + _ => 1usize << c, + } + } + + /// Permutations one opening of `felts` felts costs against a height-`c` + /// cap of a tree of `depth` index bits: the leaf's blocks and a node a + /// walked level (`Hash16` rows at arity 4: rate-12 blocks, 4-ary levels). + pub fn opening_perms(&self, felts: usize, depth: usize, c: usize) -> usize { + if self.arity == 4 { + felts.div_ceil(super::p1w16_emit::RATE_FELTS) + + super::merkle_cap::path_permutations(depth, c, 4) + } else { + verify_opening_perms_capped(felts, depth, c) + } + } + + /// Permutations a height-`c` cap of a tree of `depth` index bits costs up + /// to its root, once a tree. + pub fn cap_perms(&self, depth: usize, c: usize) -> usize { + match (c, self.arity) { + (0, _) => 0, + (_, 4) => super::merkle_cap::cap_root_permutations( + super::merkle_cap::cap_nodes(depth, c, 4), + 4, + ), + _ => cap_check_perms(c), + } + } + + /// The successor tree's cap height at round `r`. + pub fn next_cap(&self, r: usize) -> Option { + (r + 1 < self.rounds()).then(|| self.caps[r + 1]) + } + + /// Felts in round `r`'s current block: one per value in round 0, where the + /// codeword is still base-field, and three after. + pub fn current_felts(&self, r: usize) -> usize { + let values = 1usize << self.schedule[r]; + if r == 0 { values } else { 3 * values } + } + + /// Variables bound after round `r`, the index ones included (round 0 binds + /// them). + pub fn bound(&self, r: usize) -> usize { + self.schedule[..=r].iter().sum::() + self.index_vars() + } + + /// Variables the chain binds in all: the polynomial's and the index's. + pub fn chain_vars(&self) -> usize { + self.num_vars + self.index_vars() + } +} + +/// PERMUTATIONS one chain costs in its query phase and its openings — the term +/// the wrap's cost is dominated by, and a function of the tree depths and block +/// widths alone. +/// +/// The transcript's own permutations are NOT in this: they depend on the +/// sponge's buffer at each squeeze, which is a schedule and not a shape, and +/// mixing the two would make a form that no longer says where its cost is. +/// They are [`chain_schedule_perms`], and the grinds' two-apiece are +/// [`chain_grind_perms`]; [`chain_perms`] is the sum of the three. +pub fn chain_opening_perms(shape: &ChainShape) -> usize { + let mut per_query = 0; + // A tail round ([`ChainShape::tail_round`]) hashes each leaf's block once + // (no walked level) and opens no successor a query. + let mut tail = 0; + for r in 0..shape.rounds() { + let current = shape.opening_perms( + shape.current_felts(r), + shape.current_depth(r), + shape.caps[r], + ); + if shape.tail_round(r) { + tail += shape.current_leaves(r) * current; + continue; + } + per_query += shape.trees_at(r) * current; + if let Some(depth) = shape.next_depth(r) { + per_query += shape.opening_perms(3 << shape.schedule[r + 1], depth, shape.caps[r + 1]); + } + } + shape.num_queries * per_query + tail + chain_cap_perms(shape) +} + +/// PERMUTATIONS the chain's cap checks cost: each capped tree's cap hashed up +/// to its root once, `2^c − 1` parents (a grouped round 0's trees each). Zero +/// at the default. Part of [`chain_opening_perms`], stated apart so the +/// per-tree term is visible. +pub fn chain_cap_perms(shape: &ChainShape) -> usize { + (0..shape.rounds()) + .map(|r| shape.trees_at(r) * shape.cap_perms(shape.current_depth(r), shape.caps[r])) + .sum() +} + +/// INSTRUCTIONS one chain's query phase costs: per round, per query, the two +/// openings, the fold, the successor slot's mux and the assert that closes it. +pub fn chain_query_rows(shape: &ChainShape) -> usize { + let mut per_round = 0; + for r in 0..shape.rounds() { + let felts = shape.current_felts(r); + let unpacks = if r == 0 { + 0 + } else { + 1usize << shape.schedule[r] + }; + let depth = shape.current_depth(r); + if shape.tail_round(r) { + // A tail round ([`emit_tail_phase`]): the draws, then per leaf its + // block's leaf, the compare with its cap node (both cells + // unpacked, four lowered asserts), the fold at its constant + // position and `folded == claimed`. + let leaf = super::whir_open::block_leaf_rows(felts, unpacks) + 2 + 2 * 4; + per_round += shape.num_queries + + shape.current_leaves(r) + * (leaf + fold_coset_rows_lean(1usize << shape.schedule[r], depth) + 2); + per_round += lean_prepare_rows(shape.schedule[r]); + continue; + } + // The index draw, the current opening (one a tree), the trees' blocks + // combined (a grouped round 0), and the fold. + let mut q = 1 + + shape.trees_at(r) * verify_opening_rows_capped(felts, unpacks, depth, shape.caps[r]) + + combine_rows(shape, r) + + fold_coset_rows_lean(1usize << shape.schedule[r], depth); + match shape.next_depth(r) { + Some(next_depth) => { + let next_block = 1usize << shape.schedule[r + 1]; + // The successor opening, the slot mux, and `folded == claimed`. + q += verify_opening_rows_capped( + 3 * next_block, + next_block, + next_depth, + shape.caps[r + 1], + ) + (next_block - 1) + + 2; + } + // `folded == final_value`. + None => q += 2, + } + per_round += shape.num_queries * q; + // The lean fold's once-a-round `½·α`. + per_round += lean_prepare_rows(shape.schedule[r]); + } + per_round +} + +/// INSTRUCTIONS one chain costs OUTSIDE its query phase: the grinds' fixed +/// parts, the sumchecks, the out-of-domain chain, the claim updates, and the +/// tail. +/// +/// ⚠ The grinds' `state_rows` term is NOT here. A `state()` hashes whatever the +/// sponge is holding at that moment, which is a property of the schedule rather +/// than of the shape; it is in [`chain_hash_schedule`], as a `State` event, +/// beside the transcript's own squeezes. +pub fn chain_fixed_rows(shape: &ChainShape) -> usize { + let (folding, ood, query) = shape.grind; + let mut rows = 0; + for r in 0..shape.rounds() { + let k = shape.schedule[r]; + rows += grind_fixed(folding); + // Per sumcheck round: two absorbed evaluations (one `Unpack` each), one + // drawn challenge (one `Pack`), and the round itself. Round 0 of a + // grouped chain has `m` more, then its `eq(ρ, p)`s. + rows += shape.messages(r) * (SUMCHECK_DEGREE + 1 + sumcheck_round_rows(SUMCHECK_DEGREE)); + if r == 0 { + rows += eq_index_rows(shape.trees); + } + match shape.next_depth(r) { + Some(_) => { + let next_domain_log = shape.domain_log[r] - k; + // The successor root: one `Unpack` into four absorbed felts. + rows += 1; + // `z0`: one `Pack`. + rows += 1; + // The squaring chain, shared between the domain check and the + // out-of-domain point, plus the `!= 1` refusal. + rows += next_domain_log + 2; + // `y0` absorbed, the ood grind, `gamma` drawn, the claim update. + rows += 1 + grind_fixed(ood) + 1 + 1; + rows += grind_fixed(query); + } + None => { + // `final_value` absorbed, then the query grind. + rows += 1 + grind_fixed(query); + } + } + } + // The tail: one `eq` and one `MulAdd` per out-of-domain claim, then the + // inversion, the multiply and the assert. + for r in 0..shape.rounds() - 1 { + rows += eq_eval_rows_again(shape.chain_vars() - shape.bound(r)) + 1; + } + rows += 1 + 1 + 2; + // W1: each capped tree's cap check, once (its hinted words are arena + // words, counted with the arena like every other hint). + rows += shape + .caps + .iter() + .enumerate() + .map(|(r, &c)| shape.trees_at(r) * cap_check_rows(c)) + .sum::(); + rows +} + +/// INSTRUCTIONS round `r`'s query spends combining its trees' blocks: one +/// extension row a value a tree ([`emit_combine`]), none with one tree. +pub fn combine_rows(shape: &ChainShape, r: usize) -> usize { + match shape.trees_at(r) { + 1 => 0, + trees => trees << shape.schedule[r], + } +} + +/// INSTRUCTIONS [`emit_eq_index`] spends on `trees` slots: one `1 − ρ_i` a +/// position some slot reads as a zero bit (shared), and `m − 1` products a +/// slot; none for one polynomial. +pub fn eq_index_rows(trees: usize) -> usize { + let m = index_vars(trees); + let complements = (0..m) + .filter(|&i| (0..trees).any(|p| (p >> (m - 1 - i)) & 1 == 0)) + .count(); + if m == 0 { + 0 + } else { + complements + trees * (m - 1) + } +} + +/// A grind's rows beyond the sponge state it reads — see +/// [`super::whir_transcript::grind_check_rows`], whose `state_rows` term +/// belongs to the schedule. +const fn grind_fixed(bits: usize) -> usize { + super::whir_transcript::grind_check_rows(bits) +} + +/// PERMUTATIONS the grinds cost: two a check — one for the inner hash of +/// `PREFIX ‖ seed ‖ factor` and one for the outer hash of `inner ‖ nonce` +/// (`epoch.rs:614-640`). A check at zero bits costs none, because it is not +/// emitted at all. +/// +/// The count of checks is the round structure's `3R − 1`, written as the three +/// terms it comes from so a config that grinds in only one place still reads +/// correctly. +pub fn chain_grind_perms(shape: &ChainShape) -> usize { + if shape.arity == 4 { + // The width-8 grind is ALU instructions, not socket rows; the + // `state()` it reads is in the schedule. + return 0; + } + let (folding, ood, query) = shape.grind; + let rounds = shape.rounds(); + let checks = rounds * usize::from(folding > 0) + + (rounds - 1) * usize::from(ood > 0) + + rounds * usize::from(query > 0); + PERMS_PER_GRIND * checks +} + +/// Permutations one grind check spends: its two hashes. +const PERMS_PER_GRIND: usize = 2; + +/// ★ The chain's SCHEDULE: every hash its transcript performs, in order, and +/// how many felts each one hashes. +/// +/// This is the half of the chain's cost that the shape alone does not give, +/// because a sponge's cost is what its BUFFER holds at each hash and the buffer +/// is a running quantity. It is derived here from the round structure — the +/// absorbs, the draws, the `state()` reads and the candidates a squeeze hands +/// out — and NOT read off [`WhirTranscript`], which is a different state +/// machine written for a different purpose. Its gate is the host's own call +/// stream (`whir_chain_tests::the_schedule_is_the_host_transcripts`). +/// +/// Every absorb by the shape it comes from: `SUMCHECK_DEGREE · +/// COORDINATES_PER_EXT` felts a sumcheck round, `DIGEST_FELTS` for a successor +/// root, `COORDINATES_PER_EXT` for `y0` or `final_value`, `NONCE_FELTS` a +/// spent grind. Every draw: one extension challenge a sumcheck round, `z0` and +/// `gamma` on a round with a successor, and `num_queries` bounded draws in the +/// query phase — which come FIRST and together, because `sample_queries` draws +/// them all before any opening is checked (`whir_round.rs:68-76`). +pub fn chain_hash_schedule(shape: &ChainShape, entry: SpongeEntry) -> Vec { + let mut sponge = SpongeSchedule::new(entry); + chain_sponge(shape, &mut sponge); + sponge.hashes().to_vec() +} + +/// ★ The same round structure, driven into a schedule the CALLER owns — so the +/// sponge a chain leaves behind is reachable, and the next leg of the same +/// program continues from it. +/// +/// ⚠ **This is the one place the chain's round structure is written**, and +/// [`chain_hash_schedule`] is a thin wrapper over it rather than a second copy. +/// A caller that needs the successor entry — `stacked_eval`'s wrapper threading +/// one sponge through every polynomial's chain — could instead replay the +/// structure beside this one, and that replay would be a second thing to keep +/// in step: the schedule's gate +/// (`whir_chain_tests::the_schedule_is_the_host_transcripts`) compares THIS +/// walk against a real verifier's calls, and would say nothing about a copy. +/// Sharing the walk puts the caller inside that gate. +pub fn chain_sponge(shape: &ChainShape, sponge: &mut SpongeSchedule) { + let (folding, ood, query) = shape.grind; + + for r in 0..shape.rounds() { + sponge.grind(folding); + for _ in 0..shape.messages(r) { + sponge.absorb(SUMCHECK_DEGREE * COORDINATES_PER_EXT); + sponge.draw_ext(); + } + match shape.next_depth(r) { + Some(_) => { + sponge.absorb_root(); + sponge.draw_ext(); + sponge.absorb(COORDINATES_PER_EXT); + sponge.grind(ood); + sponge.draw_ext(); + sponge.grind(query); + } + None => { + sponge.absorb(COORDINATES_PER_EXT); + sponge.grind(query); + } + } + for _ in 0..shape.num_queries { + sponge.candidate(); + } + } +} + +/// INSTRUCTIONS the schedule costs: the `Pack`s that build each hash's words +/// and the `Unpack` a squeeze spends reading its digest. +pub fn chain_schedule_rows(shape: &ChainShape, entry: SpongeEntry) -> usize { + let mut sponge = SpongeSchedule::new(entry); + chain_sponge(shape, &mut sponge); + sponge.rows() +} + +/// PERMUTATIONS the schedule costs: one per rate-8 block of every hash (a +/// `Hash16` row per Poseidon1 permutation). +pub fn chain_schedule_perms(shape: &ChainShape, entry: SpongeEntry) -> usize { + let mut sponge = SpongeSchedule::new(entry); + chain_sponge(shape, &mut sponge); + sponge.perms() +} + +/// ★ INSTRUCTIONS one chain costs, whole: the shape half and the schedule half. +/// +/// The two are pinned apart and not only as this sum, because they move +/// independently — a form that shifted work from the sponge to the arithmetic +/// at constant total would fail one of them rather than neither. +pub fn chain_rows(shape: &ChainShape, entry: SpongeEntry) -> usize { + chain_shape_rows(shape) + chain_schedule_rows(shape, entry) +} + +/// ★ PERMUTATIONS one chain costs, whole, in the THREE terms it has: the query +/// phase's openings, the grinds' two hashes each, and the transcript's own. +/// +/// ⚠ Only the first was pinned before this; the sizing note's per-chain +/// permutation figure predates the other two and is not reproduced by them. +pub fn chain_perms(shape: &ChainShape, entry: SpongeEntry) -> usize { + chain_opening_perms(shape) + chain_grind_perms(shape) + chain_schedule_perms(shape, entry) +} + +/// ★ `whir_chain::verify_weighted`, emitted. +/// +/// `weight_at` is the caller's closure, mirroring the host's `W` parameter: it +/// is handed the concatenated round challenges and returns the weight at that +/// point. `whir_chain::verify` passes `eq_eval(z, ·)`, which is +/// [`emit_eq_eval`]; a stacked chain passes +/// [`super::whir_stacked::emit_weight_at`]. +/// +/// Every refusal is a division with no satisfying assignment rather than a +/// branch, so a proof the host rejects has no execution here. +#[allow(clippy::too_many_arguments)] +pub fn emit_verify_weighted( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + rounds: &[ChainRoundWires<'_>], + root_lanes: &[Felt; 4], + final_value: Ext, + y: Ext, + shape: &ChainShape, + domain: &Domain, + weight_at: impl FnOnce(&mut LfmBuilder, &[Ext]) -> Ext, +) { + assert_eq!( + shape.trees, 1, + "a chain over one polynomial; a run of several is emit_verify_grouped's" + ); + emit_chain( + b, + transcript, + rounds, + std::slice::from_ref(root_lanes), + final_value, + y, + shape, + domain, + |b, at, _| weight_at(b, at), + ); +} + +/// ★ `whir_chain::verify_grouped`, emitted: ONE chain over a run of +/// `shape.trees` stacked polynomials ([`GroupFold::Indexed`], D-ONECHAIN +/// §2.7), each committed under its own root in `roots`, in slot order. +/// +/// Round 0 runs `k0 + m` sumcheck rounds, the `m` index challenges `ρ` last; +/// at each query position it opens EVERY tree at the one drawn index, combines +/// the blocks as `Σ_p eq(ρ, p)·block_p` and folds that as today's `2^k0` +/// block. Every later round is [`emit_verify_weighted`]'s. +/// +/// `weight_at` is handed the chain's `n + m` challenges in binding order and +/// the run's `eq(ρ, p)`s, `p < trees` ([`emit_eq_index`]): the one evaluation +/// the combine reads too, from the sampled `ρ` wires. An absent slot `p ≥ +/// trees` has no `eq`, no block and no hint wire. +#[allow(clippy::too_many_arguments)] +pub fn emit_verify_grouped( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + rounds: &[ChainRoundWires<'_>], + roots: &[[Felt; 4]], + final_value: Ext, + y: Ext, + shape: &ChainShape, + domain: &Domain, + weight_at: impl FnOnce(&mut LfmBuilder, &[Ext], &[Ext]) -> Ext, +) { + assert!( + shape.trees >= 2, + "a run of one polynomial is emit_verify_weighted's" + ); + emit_chain( + b, + transcript, + rounds, + roots, + final_value, + y, + shape, + domain, + weight_at, + ); +} + +/// [`emit_verify_weighted`] and [`emit_verify_grouped`]: one chain over +/// `roots.len() = shape.trees` polynomials. At one polynomial every row it +/// emits, and their order, are [`emit_verify_weighted`]'s as they were. +#[allow(clippy::too_many_arguments)] +fn emit_chain( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + rounds: &[ChainRoundWires<'_>], + roots: &[[Felt; 4]], + final_value: Ext, + y: Ext, + shape: &ChainShape, + domain: &Domain, + weight_at: impl FnOnce(&mut LfmBuilder, &[Ext], &[Ext]) -> Ext, +) { + assert_eq!( + rounds.len(), + shape.rounds(), + "one set of wires per scheduled round" + ); + assert_eq!(roots.len(), shape.trees, "one root per round-0 tree"); + let one = b.ext_const(&FEE::one()); + let (grind_folding, grind_ood, grind_query) = shape.grind; + + let mut claim = y; + let mut alphas: Vec = Vec::with_capacity(shape.chain_vars()); + // Round 0's trees: each one's cap (when it has one) is authenticated + // against its root here, once; every later tree where its root is absorbed. + // Tree `p` is checked against `roots[p]` alone, by its position. + let depth0 = shape.current_depth(0); + let cap_words = shape.cap_words(depth0, shape.caps[0]); + assert_eq!( + rounds[0].current_cap.len(), + shape.trees * cap_words, + "one cap a round-0 tree" + ); + let mut current_trees: Vec = roots + .iter() + .enumerate() + .map(|(p, lanes)| { + tree_auth( + b, + shape, + depth0, + shape.caps[0], + &rounds[0].current_cap[p * cap_words..(p + 1) * cap_words], + lanes, + ) + }) + .collect(); + let mut current_domain = domain.clone(); + // Each out-of-domain claim: its batching weight, its point, and how many + // variables were bound when it entered. + let mut ood: Vec<(Ext, Vec, usize)> = Vec::new(); + // The run's `eq(ρ, p)`s, once round 0 has drawn `ρ`; none for one tree. + let mut eqs: Vec = Vec::new(); + + for (r, round) in rounds.iter().enumerate() { + let k = shape.schedule[r]; + let sent = shape.messages(r); + assert_eq!( + round.sumcheck.len(), + sent, + "round {r} carries {sent} sumcheck messages" + ); + + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::Step::Sumcheck { round: r }); + emit_spent_grind(b, transcript, grind_folding, round.nonces.folding); + + // The sumcheck, interleaved: a round's challenge is drawn only after + // its evaluations are absorbed, which is the order `verify_rounds` + // takes (`sumcheck.rs:386-390`) and the seam the round primitive was + // left with. + let mut point = Vec::with_capacity(sent); + for evaluations in round.sumcheck { + assert_eq!(evaluations.len(), SUMCHECK_DEGREE); + for e in evaluations { + transcript.absorb_ext(b, *e); + } + let challenge = transcript.sample_ext(b); + claim = emit_sumcheck_round(b, claim, evaluations, challenge); + point.push(challenge); + } + // A grouped round 0 bound its index LAST: the challenges past the + // first `k` are `ρ`, and only the first `k` fold a block. + let (data, rho) = point.split_at(k); + if !rho.is_empty() { + eqs = emit_eq_index(b, rho, shape.trees, one); + } + // The combine reads the same `eq(ρ, p)`s the weight does. + #[cfg(test)] + let faulted: Vec = + if r == 0 && onechain_faults::get() == onechain_faults::Fault::SwapRhoInCombine { + let swapped: Vec = rho.iter().rev().copied().collect(); + emit_eq_index(b, &swapped, shape.trees, one) + } else { + eqs.clone() + }; + #[cfg(test)] + let combine_eqs: &[Ext] = &faulted; + #[cfg(not(test))] + let combine_eqs: &[Ext] = &eqs; + + let mut next_domain = current_domain.clone(); + for _ in 0..k { + next_domain = next_domain + .squared() + .expect("the schedule never folds past the domain"); + } + let bound = alphas.len() + sent; + + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::Step::Ood { round: r }); + let next_tree = match (round.next_root, round.ood_value, shape.next_depth(r)) { + (Some(next_root), Some(y0), Some(_)) => { + let lanes = b.unpack(next_root); + transcript.absorb_root_lanes(b, &lanes); + + let z0 = transcript.sample_ext(b); + + // ONE squaring chain for two consumers. `powers[j] = z0^(2^j)`: + // the out-of-domain point takes the first `n + m − bound` of + // them and the domain check takes the last. + let depth = next_domain.log_size(); + let mut powers = Vec::with_capacity(depth + 1); + powers.push(z0); + for j in 0..depth { + let squared = b.emul(powers[j], powers[j]); + powers.push(squared); + } + emit_require_out_of_domain(b, powers[depth], one); + let ood_point: Vec = powers[..shape.chain_vars() - bound].to_vec(); + + transcript.absorb_ext(b, y0); + emit_spent_grind(b, transcript, grind_ood, round.nonces.ood); + let gamma = transcript.sample_ext(b); + claim = b.emul_add(gamma, y0, claim); + ood.push((gamma, ood_point, bound)); + + emit_spent_grind(b, transcript, grind_query, round.nonces.query); + Some(tree_auth( + b, + shape, + shape.current_depth(r + 1), + shape.caps[r + 1], + round.next_cap, + &lanes, + )) + } + (None, None, None) => { + assert!(round.next_cap.is_empty(), "the last round has no successor"); + transcript.absorb_ext(b, final_value); + emit_spent_grind(b, transcript, grind_query, round.nonces.query); + None + } + _ => panic!( + "round {r}: a successor root and an out-of-domain value exist together, and \ + exactly on the rounds the schedule gives a successor" + ), + }; + + if shape.tail_round(r) { + let next_blocks = rounds.get(r + 1).map(|next| next.current); + emit_tail_phase( + b, + transcript, + round, + shape, + r, + ¤t_domain, + data, + ¤t_trees, + next_blocks, + final_value, + ); + } else { + emit_query_phase( + b, + transcript, + round, + shape, + r, + ¤t_domain, + data, + ¤t_trees, + if r == 0 { combine_eqs } else { &[] }, + next_tree.as_ref(), + final_value, + ); + } + + if let Some(tree) = next_tree { + current_trees = vec![tree]; + } + alphas.extend(point); + current_domain = next_domain; + } + + // The accumulated weight: the caller's own, plus each out-of-domain claim's + // batched `eq` over the challenges that came after it. + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::Step::Final); + let mut weight = weight_at(b, &alphas, &eqs); + for (gamma, point, bound) in &ood { + let eq = emit_eq_eval(b, point, &alphas[*bound..]); + weight = b.emul_add(*gamma, eq, weight); + } + + emit_final_check(b, claim, weight, final_value, one); +} + +/// Test only: emitter faults a grouped-chain test switches on, one at a time, +/// to show each in-guest reading of the index is load-bearing (REV-B2-JUDGE +/// B2-5, the host's B1-3 negatives): under any of them the honest arena has no +/// execution. Thread-local, so a test that sets one moves no other test's +/// program. +#[cfg(test)] +pub(crate) mod onechain_faults { + /// Which piece of the emitted verifier reads the index wrongly. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub(crate) enum Fault { + #[default] + None, + /// Round 0's combine reads `ρ` reversed. + SwapRhoInCombine, + /// The final weight reads `ρ` reversed. + SwapRhoInWeight, + /// The final weight drops `eq(ρ, p)`: `Σ_p w_p` at the data point. + DropIndexFactor, + /// The final weight gives each absent slot `p ≥ P'` a weight of 1. + AbsentSlotWeightOne, + } + + std::thread_local! { + static FAULT: std::cell::Cell = const { std::cell::Cell::new(Fault::None) }; + } + + pub(crate) fn get() -> Fault { + FAULT.with(std::cell::Cell::get) + } + + /// Runs `f` under `fault`, restoring the previous one after. + pub(crate) fn with(fault: Fault, f: impl FnOnce() -> R) -> R { + struct Restore(Fault); + impl Drop for Restore { + fn drop(&mut self) { + FAULT.with(|c| c.set(self.0)); + } + } + let _restore = Restore(FAULT.with(|c| c.replace(fault))); + f() + } +} + +/// Test only: where each check of a chain's emission begins, as the builder's +/// instruction index, so a refusal is named by the check holding the assert +/// that failed — the executor reports a failed `assert_eq` as `DivByZero` at +/// the address of its difference (`executor::locate_addr`), and a negative +/// test names the step against its honest twin, which executes past it. +/// Recording only: nothing is emitted, and nothing is kept outside +/// [`onechain_steps::record`]. +#[cfg(test)] +pub(crate) mod onechain_steps { + use super::super::builder::LfmBuilder; + use super::super::compiler::LfmProgram; + use super::super::executor::LfmExecError; + use super::super::instr::Instr; + + /// A check of the emitted chain, in emission order. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) enum Step { + /// A stacked opening's chain whose first polynomial is this begins + /// (its roots, its claim). + Chain(usize), + /// A round's folding grind and sumcheck. + Sumcheck { round: usize }, + /// A round's successor root, out-of-domain point, grinds (or the final + /// value) and query draws. + Ood { round: usize }, + /// Tree `tree`'s opening at a query (a grouped round 0 has several). + Opening { + round: usize, + query: usize, + tree: usize, + }, + /// The successor's opening at a query. + Successor { round: usize, query: usize }, + /// A query's combine (a grouped round 0), fold and compare. + Fold { round: usize, query: usize }, + /// The weight and the final check. + Final, + } + + /// [`Step::Opening`]. + pub(crate) fn opening(round: usize, query: usize, tree: usize) -> Step { + Step::Opening { round, query, tree } + } + + std::thread_local! { + static MARKS: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + } + + pub(crate) fn mark(b: &LfmBuilder, step: Step) { + MARKS.with(|m| { + if let Some(marks) = m.borrow_mut().as_mut() { + marks.push((step, b.instrs().len())); + } + }); + } + + /// Runs `f` and returns the marks its emission left. + pub(crate) fn record(f: impl FnOnce() -> R) -> (R, Vec<(Step, usize)>) { + let previous = MARKS.with(|m| m.replace(Some(Vec::new()))); + let out = f(); + let marks = MARKS.with(|m| m.replace(previous)).unwrap_or_default(); + (out, marks) + } + + /// The chain (by its first polynomial) and the step whose assert failed: + /// the instruction that wrote a `DivByZero`'s address, under the last chain + /// mark and the last step mark at or before it. `None` for any other + /// refusal. + pub(crate) fn failed_at( + program: &LfmProgram, + marks: &[(Step, usize)], + error: &LfmExecError, + ) -> Option<(Option, Step)> { + let LfmExecError::DivByZero { addr } = error else { + return None; + }; + let index = program.instrs.iter().position(|i| match i { + Instr::BaseAlu { out, .. } | Instr::ExtAlu { out, .. } => out.0 == *addr, + _ => false, + })?; + let before = marks.iter().filter(|(_, at)| *at <= index); + let chain = before.clone().rev().find_map(|(s, _)| match s { + Step::Chain(c) => Some(*c), + _ => None, + }); + let step = before + .rev() + .find(|(s, _)| !matches!(s, Step::Chain(_))) + .map(|(s, _)| *s)?; + Some((chain, step)) + } +} + +/// `eq(ρ, p)` for each slot `p < trees`, `ρ_0` against the MOST significant of +/// `p`'s bits: the host's `whir_chain::eq_index`, emitted from the sampled `ρ` +/// wires and never hinted (REV-B2-JUDGE B2-5). An absent slot `p ≥ trees` has +/// no `eq` at all. `1 − ρ_i` is emitted once a position, when a slot reads it. +pub fn emit_eq_index(b: &mut LfmBuilder, rho: &[Ext], trees: usize, one: Ext) -> Vec { + let m = rho.len(); + assert_eq!( + m, + index_vars(trees), + "{trees} slots bind {m} index variables" + ); + let mut complement: Vec> = vec![None; m]; + let mut eqs = Vec::with_capacity(trees); + for p in 0..trees { + let mut acc: Option = None; + for (i, &r) in rho.iter().enumerate() { + let factor = if (p >> (m - 1 - i)) & 1 == 1 { + r + } else { + match complement[i] { + Some(c) => c, + None => { + let c = b.esub(one, r); + complement[i] = Some(c); + c + } + } + }; + acc = Some(match acc { + None => factor, + Some(a) => b.emul(a, factor), + }); + } + eqs.push(acc.expect("a run of two or more binds an index variable")); + } + eqs +} + +/// `Σ_p eqs[p]·block_p`, value by value: the one `2^k0` block a grouped round +/// 0 folds (the host's `whir_round::combine_blocks`). +/// +/// ★ THE BASE-LIFT PIN, PER SLOT (REV-B2-JUDGE B2-2, REV-B2-B RF-2). Each block +/// is read as EXACTLY the felt slice tree `p`'s `verify_opening` hashed — the +/// caller hands both the same [`QueryOpening`] — so its upper lanes are pinned +/// by that leaf hash's `Pack`s before the lift reads them ([`super::whir_open`]'s +/// obligation). A block not hashed by its tree has no way in. +fn emit_combine(b: &mut LfmBuilder, eqs: &[Ext], blocks: &[QueryOpening<'_>]) -> Vec { + assert_eq!(eqs.len(), blocks.len(), "one eq(ρ, p) a tree's block"); + let felts: Vec<&[Felt]> = blocks + .iter() + .map(|block| match block.values { + BlockValues::Base(felts) | BlockValues::BaseWords { felts, .. } => felts, + BlockValues::Ext(_) => panic!("a grouped round 0's blocks are base-field"), + }) + .collect(); + let width = felts[0].len(); + (0..width) + .map(|t| { + let mut acc = b.emul_base(eqs[0], felts[0][t]); + for (eq, block) in eqs.iter().zip(&felts).skip(1) { + acc = b.emul_add(*eq, block[t].as_ext(), acc); + } + acc + }) + .collect() +} + +/// One grind slot, emitted: [`emit_grind_check`] where the grind spends bits, +/// nothing where it does not (the host's `check_grind` reads nothing then). +/// +/// A spent grind always has its nonce wire, because [`ChainShape::carries`] +/// carries every slot of nonzero bits under either layout; a missing one is a +/// wire walk that disagrees with the shape it was built from, refused here at +/// emit time rather than emitted as a grind that checks nothing. +fn emit_spent_grind( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + bits: usize, + nonce: Option, +) { + if bits == 0 { + return; + } + let nonce = nonce.expect("a grind of nonzero bits has its nonce wire under every layout"); + emit_grind_check(b, transcript, bits as u8, nonce); +} + +/// How a tree's openings are checked: against its root lanes when its cap +/// height is 0 (today's emission, unchanged), else against its cap, hinted as +/// `cap` and authenticated against the root lanes HERE — the one place a +/// tree's [`CapCells`] are made. +fn tree_auth( + b: &mut LfmBuilder, + shape: &ChainShape, + depth: usize, + cap_height: usize, + cap: &[WrapDigest], + root_lanes: &[Felt; 4], +) -> TreeAuth { + if cap_height == 0 { + assert!(cap.is_empty(), "an uncapped tree carries no cap wires"); + TreeAuth::Root(*root_lanes) + } else if shape.arity == 4 { + assert_eq!( + cap.len(), + shape.cap_words(depth, cap_height), + "a 4-ary cap is its level's real nodes" + ); + TreeAuth::Cap(CapCells::authenticate_at( + b, + cap, + cap_height, + std::slice::from_ref(root_lanes), + )) + } else { + assert_eq!(cap.len(), 1usize << cap_height, "a cap is 2^c digests"); + TreeAuth::Cap(CapCells::authenticate( + b, + cap, + std::slice::from_ref(root_lanes), + )) + } +} + +/// ★ `require_out_of_domain` (`whir_chain.rs:88-101`), emitted as a REFUSAL. +/// +/// The host rejects `z0` whose `2^log_size`-th power is one, because such a +/// point lies inside the domain and constrains nothing beyond the in-domain +/// queries. Here `raised − 1` is inverted: `1/0` has no satisfying assignment, +/// so an in-domain `z0` has no execution. +/// +/// ⚠ **Its own function because no honest fixture reaches it.** A transcript +/// draws an in-domain `z0` with negligible probability, so a chain-level +/// mutation that deletes this check passes every gate — measured, not assumed: +/// mutation CC removed it and all four chain tests stayed green. A refusal that +/// nothing can exercise is a check that cannot fail, and the cure is to make it +/// callable so a test can hand it the input the protocol never will. +pub fn emit_require_out_of_domain(b: &mut LfmBuilder, raised: Ext, one: Ext) { + let gap = b.esub(raised, one); + let _ = b.ediv(one, gap); +} + +/// ★ The chain's last check: `final_value == claim · weight⁻¹`. +/// +/// Inverting rather than cross-multiplying, and the difference is soundness +/// rather than taste. `final_value · weight == claim` is one row cheaper and +/// accepts EVERY `final_value` when `weight` and `claim` are both zero, where +/// the host returns `DegenerateEvaluationPoint` (`whir_chain.rs:1095-1102`). +/// +/// ⚠ **Its own function for the same reason as above.** Weight and claim are +/// both zero only at a point a transcript does not produce, so mutation CD +/// replaced this with the cross-multiplied form and all four chain tests stayed +/// green. The gap is closed by calling this directly with the degenerate input. +pub fn emit_final_check(b: &mut LfmBuilder, claim: Ext, weight: Ext, final_value: Ext, one: Ext) { + let inv = b.ediv(one, weight); + let required = b.emul(claim, inv); + b.assert_eq_ext(final_value, required); +} + +/// `whir_round::verify` and `verify_final`: draw every query position, then +/// check every opening. +/// +/// The draws come first and together, because that is the order +/// `sample_queries` takes (`whir_round.rs:68-76`) and the transcript is +/// sequential — interleaving them with the openings would change every +/// challenge after the first. +/// +/// ★ A grouped round 0 (`current_trees.len() ≥ 2`, `whir_round::verify_grouped`) +/// reads `round.current` query-major, one opening a tree, and ONE draw a query +/// feeds every tree's `verify_opening`, the successor's leaf and slot, and the +/// fold point (REV-B2-JUDGE B2-3): a tree opened at another position has no +/// wire to arrive on. The trees' blocks are combined with `eqs` and folded as +/// one block. +#[allow(clippy::too_many_arguments)] +fn emit_query_phase( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + round: &ChainRoundWires<'_>, + shape: &ChainShape, + r: usize, + current_domain: &Domain, + alphas: &[Ext], + current_trees: &[TreeAuth], + eqs: &[Ext], + next_tree: Option<&TreeAuth>, + final_value: Ext, +) { + let depth = shape.current_depth(r); + let trees = shape.trees_at(r); + assert_eq!(current_trees.len(), trees, "round {r} opens {trees} trees"); + debug_assert!( + current_trees + .iter() + .all(|t| t.cap_height() == shape.caps[r]) + ); + assert_eq!(round.current.len(), shape.num_queries * trees); + assert_eq!( + eqs.len(), + if trees == 1 { 0 } else { trees }, + "one eq(ρ, p) a tree" + ); + let queries: Vec> = (0..shape.num_queries) + .map(|_| transcript.sample_u64_pow2(b, depth)) + .collect(); + // The current trees are walked at the block's tree position; the fold, the + // successor's leaf and its slot read the index itself. + let walks: Vec> = queries + .iter() + .map(|bits| walk_bits(shape, r, bits)) + .collect(); + // The lean fold halves the round's challenges once, here, rather + // than once per query. + let fold = RoundFold::prepare(b, alphas); + + match (next_tree, shape.next_depth(r)) { + (Some(next_tree), Some(next_depth)) => { + let next_block = 1usize << shape.schedule[r + 1]; + assert_eq!(round.next.len(), shape.num_queries); + for (q, bits) in queries.iter().enumerate() { + // Each tree's block at the query's one position, against that + // tree's own check. + let blocks = &round.current[q * trees..(q + 1) * trees]; + for (p, current) in blocks.iter().enumerate() { + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::opening(r, q, p)); + current_trees[p].verify_opening(b, current.values, &walks[q], current.siblings); + } + let next = &round.next[q]; + // `leaf_and_slot`: the low `next_depth` bits index the successor + // leaf and the high ones choose the slot inside it. Both bounds + // are powers of two, so this is a partition of the bits. + let (leaf_bits, slot_bits) = bits.split_at(next_depth); + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::Step::Successor { round: r, query: q }); + next_tree.verify_opening(b, next.values, leaf_bits, next.siblings); + + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::Step::Fold { round: r, query: q }); + let block = folded_block(b, blocks, eqs); + let folded = fold.fold(b, &block, current_domain, bits); + let claimed = emit_slot_mux(b, &block_ext(next.values), slot_bits); + b.assert_eq_ext(folded, claimed); + debug_assert_eq!(1usize << slot_bits.len(), next_block); + } + } + _ => { + for (q, bits) in queries.iter().enumerate() { + let blocks = &round.current[q * trees..(q + 1) * trees]; + for (p, current) in blocks.iter().enumerate() { + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::opening(r, q, p)); + current_trees[p].verify_opening(b, current.values, &walks[q], current.siblings); + } + #[cfg(test)] + onechain_steps::mark(b, onechain_steps::Step::Fold { round: r, query: q }); + let block = folded_block(b, blocks, eqs); + let folded = fold.fold(b, &block, current_domain, bits); + b.assert_eq_ext(folded, final_value); + } + } + } +} + +/// ★ A tail round's check ([`ChainShape::tail_round`], [`LeafQuery::Lean`]): +/// the query draws, as [`emit_query_phase`] makes them, then EVERY leaf of the +/// current tree at once. +/// +/// Each leaf's block is hashed and must equal its cap node: the tree lies +/// wholly in its cap, already authenticated against the root as the round +/// before's successor, so the positions are program constants and no node is +/// muxed. Each block folds at its constant position, and the fold must equal +/// the successor's value at that position — successor block `i mod 2^d'`, +/// slot `i >> d'`, read from the successor tail round's own hinted blocks +/// (`next_blocks`), which that round authenticates the same way — or the +/// final value at the last round. +/// +/// Every sampled position is one of these, so the leaf accepts only what the +/// host's checks at the sampled positions accept; an honest proof passes at +/// every position (WHIR's completeness is perfect). The draws stay where the +/// host's transcript takes them, and their bits go unused. +#[allow(clippy::too_many_arguments)] +fn emit_tail_phase( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + round: &ChainRoundWires<'_>, + shape: &ChainShape, + r: usize, + current_domain: &Domain, + alphas: &[Ext], + current_trees: &[TreeAuth], + next_blocks: Option<&[QueryOpening<'_>]>, + final_value: Ext, +) { + let depth = shape.current_depth(r); + let leaves = shape.current_leaves(r); + assert!(shape.tail_round(r), "round {r} is not a tail round"); + assert_eq!( + round.current.len(), + leaves, + "a tail round hints every leaf once" + ); + assert!( + round.next.is_empty(), + "a tail round opens no successor block a query" + ); + for _ in 0..shape.num_queries { + let _ = transcript.sample_u64_pow2(b, depth); + } + let fold = RoundFold::prepare(b, alphas); + let [TreeAuth::Cap(cap)] = current_trees else { + panic!("a tail round's one current tree is capped") + }; + let digests: Vec = round + .current + .iter() + .map(|block| super::whir_open::emit_block_leaf(b, block.values)) + .collect(); + #[cfg(test)] + let unchecked = tail_faults::leaves_unchecked(); + #[cfg(not(test))] + let unchecked = false; + if !unchecked { + cap.verify_every_leaf(b, &digests); + } + let next = match (next_blocks, shape.next_depth(r)) { + (Some(blocks), Some(next_depth)) => { + debug_assert!( + shape.tail_round(r + 1), + "a tail round's successor is a tail round" + ); + assert_eq!( + blocks.len(), + 1usize << next_depth, + "the successor's every leaf" + ); + Some((blocks, next_depth)) + } + (None, None) => None, + _ => panic!("round {r}: successor blocks exactly where the schedule has a successor"), + }; + for (i, block) in round.current.iter().enumerate() { + let bits: Vec = (0..depth).map(|t| b.bit_const((i >> t) & 1 == 1)).collect(); + let folded = fold.fold(b, &block_ext(block.values), current_domain, &bits); + let claimed = match next { + Some((blocks, next_depth)) => { + let successor = &blocks[i % (1usize << next_depth)]; + let slot = i >> next_depth; + #[cfg(test)] + let slot = if tail_faults::slot_off() { + slot ^ 1 + } else { + slot + }; + block_ext(successor.values)[slot] + } + None => final_value, + }; + b.assert_eq_ext(folded, claimed); + } +} + +/// Test only, two faults. `slot_off`: a tail round's compare reads the +/// successor's NEIGHBOURING slot, so an honest proof has no execution — the +/// compare is live and reads the slot the position names. `leaves_unchecked`: +/// a tail round's blocks are not compared with their cap nodes, so blocks +/// that only fold consistently execute — the cap compare is what binds them. +/// Thread-local, as [`onechain_faults`] is. +#[cfg(test)] +pub(crate) mod tail_faults { + std::thread_local! { + static SLOT_OFF: std::cell::Cell = const { std::cell::Cell::new(false) }; + static LEAVES_UNCHECKED: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + pub(crate) fn slot_off() -> bool { + SLOT_OFF.with(std::cell::Cell::get) + } + + pub(crate) fn leaves_unchecked() -> bool { + LEAVES_UNCHECKED.with(std::cell::Cell::get) + } + + /// Runs `f` with the fault on, restoring it after. + pub(crate) fn with_slot_off(f: impl FnOnce() -> R) -> R { + with(&SLOT_OFF, f) + } + + /// Runs `f` with the tail leaves' cap compare off, restoring it after. + pub(crate) fn with_leaves_unchecked(f: impl FnOnce() -> R) -> R { + with(&LEAVES_UNCHECKED, f) + } + + fn with( + fault: &'static std::thread::LocalKey>, + f: impl FnOnce() -> R, + ) -> R { + struct Restore(&'static std::thread::LocalKey>, bool); + impl Drop for Restore { + fn drop(&mut self) { + self.0.with(|c| c.set(self.1)); + } + } + let _restore = Restore(fault, fault.with(|c| c.replace(true))); + f() + } +} + +/// The bits round `r`'s current trees are walked on for a query of index bits +/// `bits` (low first): the index itself, or at round 0 under +/// [`LeafOrder::Coset`] the position the order puts the block at, the index +/// bits reversed (`LeafOrder::position`) — the same wires in the other order, +/// no row. +fn walk_bits<'a>(shape: &ChainShape, r: usize, bits: &'a [Bit]) -> std::borrow::Cow<'a, [Bit]> { + match (r, shape.leaves) { + (0, LeafOrder::Coset) => std::borrow::Cow::Owned(bits.iter().rev().copied().collect()), + _ => std::borrow::Cow::Borrowed(bits), + } +} + +/// The block a query folds: its one opening's values, or a grouped round 0's +/// trees' blocks combined with `eqs` ([`emit_combine`]). +fn folded_block(b: &mut LfmBuilder, blocks: &[QueryOpening<'_>], eqs: &[Ext]) -> Vec { + match blocks { + [one] => block_ext(one.values), + _ => emit_combine(b, eqs, blocks), + } +} + +/// A block as extension wires. A base block's values lift for free — a base +/// cell is `(v, 0, 0, 0)` — and the lift is sound because the leaf hash pinned +/// those lanes; [`super::whir_open`]'s module doc carries that obligation. +fn block_ext(values: BlockValues<'_>) -> Vec { + match values { + BlockValues::Base(felts) | BlockValues::BaseWords { felts, .. } => { + felts.iter().map(|f| f.as_ext()).collect() + } + BlockValues::Ext(cells) => cells.to_vec(), + } +} + +/// `values[slot]` where `slot` is given as bits, LOW first: a balanced mux, +/// `values.len() − 1` `Select` rows. +/// +/// `Select` returns both arms for one row (`builder.rs:298-314`), so a level +/// that halves `2m` values into `m` costs `m` rows and the whole mux costs +/// `values.len() − 1`. +/// +/// ⚠ The pairing is `(2t, 2t + 1)` and not `(t, t + half)`, because the bits +/// arrive low first. `t` against `t + half` differs in the HIGH bit, so pairing +/// that way while consuming the low bit first reads the index backwards — it +/// agrees at every palindromic slot and nowhere else, which is the shape of a +/// bug a small fixture hides. +fn emit_slot_mux(b: &mut LfmBuilder, values: &[Ext], bits: &[Bit]) -> Ext { + assert_eq!( + values.len(), + 1usize << bits.len(), + "a slot mux covers the whole successor block" + ); + let mut level: Vec = values.to_vec(); + for bit in bits { + level = level + .chunks_exact(2) + .map(|pair| { + let (chosen, _) = b.select(*bit, pair[0].as_cell(), pair[1].as_cell()); + chosen.as_ext() + }) + .collect(); + } + level[0] +} + +/// The half of a chain's cost that does not depend on the sponge's schedule. +/// +/// The other half is [`chain_schedule_rows`] and the sum is [`chain_rows`]. +pub fn chain_shape_rows(shape: &ChainShape) -> usize { + chain_fixed_rows(shape) + chain_query_rows(shape) +} + +// ============================================================================= +// One chain's wires in an arena — MOVED OUT OF `whir_chain_tests` (V1g) +// ============================================================================= +// +// These four were `pub(super)` inside the chain's own test module, which is +// `#[cfg(test)]`, so nothing in a production module could reach them. The +// assembled epoch verify (`whir_epoch`) is a production module that holds EIGHT +// chains plus the DECODE group's, and it has to hint their wires out of one +// arena; a second copy of this walk beside the one the chain suite gates would +// be exactly the drift the chain suite exists to prevent. So the walk moves +// here and the suite keeps gating it. +// +// Nothing about them changed but their visibility and `round_words` losing an +// `impl` block: the chain suite is the control that says so. + +/// An opening's path as [`RoundStorage`] holds it. +enum PathStore { + /// Hinted with the round's other wires, in the proof's child order. + Hinted(Vec), + /// Left in the arena from word `at`, for its walk to hint + /// ([`ChainShape::paths_rotated`]). + Arena { + arena: super::instr::ArenaId, + at: u32, + }, +} + +impl PathStore { + fn siblings(&self) -> Siblings<'_> { + match self { + PathStore::Hinted(path) => Siblings::Child(path), + PathStore::Arena { arena, at } => Siblings::Rotated { + arena: *arena, + at: *at, + }, + } + } +} + +/// A current block's wires, in the field its round holds them in. +pub enum CurrentBlock { + Base(Vec), + /// Round 0's base values hinted four a word, and the felts their + /// `Unpack`s write ([`ChainShape::round0_packed`]). + BaseWords { + words: Vec, + felts: Vec, + }, + Ext(Vec), +} + +impl CurrentBlock { + fn as_block(&self) -> BlockValues<'_> { + match self { + CurrentBlock::Base(v) => BlockValues::Base(v), + CurrentBlock::BaseWords { words, felts } => BlockValues::BaseWords { words, felts }, + CurrentBlock::Ext(v) => BlockValues::Ext(v), + } + } +} + +/// ★ One chain's round wires, hinted and OWNED — because `ChainRoundWires` +/// borrows them. +/// +/// Extracted from [`chain_program`] so a caller that emits SEVERAL chains in one +/// program — `stacked_eval`'s wrapper, one chain per stacked polynomial — builds +/// them from this walk rather than from a second copy of it. The chain suite is +/// the control that the extraction moved nothing. +pub struct RoundStorage { + shape: ChainShape, + sumchecks: Vec>>, + currents: Vec>, + nexts: Vec, PathStore)>>, + roots: Vec>, + oods: Vec>, + nonces: Vec, + /// Per round: the current tree's cap when the round owns it (round 0), + /// and the successor's cap. Empty where the tree is uncapped. + current_caps: Vec>, + next_caps: Vec>, +} + +impl RoundStorage { + /// Words one chain's rounds occupy, which is what a caller placing several + /// chains in one arena advances by. + pub fn words(shape: &ChainShape) -> u32 { + (0..shape.rounds()).map(|r| round_words(shape, r)).sum() + } + + /// Hints every round's wires out of `arena`, starting at `base`, in the + /// order [`push_round_words`] writes them. + pub fn hint( + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + base: u32, + shape: &ChainShape, + ) -> Self { + let mut sumchecks: Vec>> = Vec::new(); + let mut currents: Vec> = Vec::new(); + let mut nexts: Vec, PathStore)>> = Vec::new(); + let mut roots: Vec> = Vec::new(); + let mut oods: Vec> = Vec::new(); + let mut nonces: Vec = Vec::new(); + let mut current_caps: Vec> = Vec::new(); + let mut next_caps: Vec> = Vec::new(); + + let mut at = base; + for r in 0..shape.rounds() { + let next_word = |b: &mut LfmBuilder, at: &mut u32| { + let cell = b.hint_word(arena, *at); + *at += 1; + cell + }; + // A path of `words` words: hinted now, or left for its walk. + let next_path = |b: &mut LfmBuilder, at: &mut u32, words: usize| { + if shape.paths_rotated() { + let path = PathStore::Arena { arena, at: *at }; + *at += words as u32; + path + } else { + PathStore::Hinted( + (0..words) + .map(|_| WrapDigest::from_cell(next_word(b, at))) + .collect(), + ) + } + }; + let k = shape.schedule[r]; + let sumcheck: Vec> = (0..shape.messages(r)) + .map(|_| (0..2).map(|_| next_word(b, &mut at).as_ext()).collect()) + .collect(); + // The nonces are FELTS: `append_bytes(&nonce.to_be_bytes())` is one + // big-endian felt, which is how the grind absorbs them. One word a + // slot the layout carries, in slot order. + let [folding, ood_nonce, query] = shape.carries(r).map(|carried| { + carried.then(|| { + let felt = b.hint_felt(arena, at); + at += 1; + felt + }) + }); + + // W1: tree 0's cap, right after round 0's nonces (the words the + // owner path carried after its siblings) — a grouped round 0's + // trees' caps, in tree order. + let current_cap: Vec = if r == 0 { + (0..shape.trees * shape.cap_words(shape.current_depth(0), shape.caps[0])) + .map(|_| WrapDigest::from_cell(next_word(b, &mut at))) + .collect() + } else { + Vec::new() + }; + let depth = shape.current_path(r); + let block = 1usize << k; + // ★ ROUND 0's current codeword is BASE on the host + // (`whir_chain.rs:983`), so its block hashes ONE felt a value and + // not three. Hinting it as extension wires would hash forty-eight + // felts where the committer hashed sixteen and the root would never + // match — which is exactly how this test first failed. + // A grouped round 0: one block and path a tree, query-major. + // A tail round: every leaf's block once, in leaf order, no path. + let openings = if shape.tail_round(r) { + shape.current_leaves(r) + } else { + shape.num_queries * shape.trees_at(r) + }; + let current: Vec<(CurrentBlock, PathStore)> = (0..openings) + .map(|_| { + let values = if r == 0 && shape.round0_packed() { + // Four values a word: the leaf hash takes the words, + // the fold the felts their `Unpack`s write. + let words: Vec = (0..shape.round0_block_words(block)) + .map(|_| next_word(b, &mut at)) + .collect(); + let felts: Vec = words + .iter() + .flat_map(|w| b.unpack(*w)) + .take(block) + .collect(); + CurrentBlock::BaseWords { words, felts } + } else if r == 0 { + let felts: Vec = (0..block) + .map(|_| { + let f = b.hint_felt(arena, at); + at += 1; + f + }) + .collect(); + CurrentBlock::Base(felts) + } else { + CurrentBlock::Ext( + (0..block).map(|_| next_word(b, &mut at).as_ext()).collect(), + ) + }; + let path = if shape.tail_round(r) { + PathStore::Hinted(Vec::new()) + } else { + next_path(b, &mut at, depth) + }; + (values, path) + }) + .collect(); + + let (next_root, ood_value, next, next_cap) = match shape.next_depth(r) { + Some(next_depth) => { + let nr = next_word(b, &mut at); + let ov = next_word(b, &mut at).as_ext(); + // W1: the successor's cap, after its root and ood value. + let next_cap: Vec = (0..shape + .cap_words(next_depth, shape.caps[r + 1])) + .map(|_| WrapDigest::from_cell(next_word(b, &mut at))) + .collect(); + let next_depth = shape.path_words(next_depth, shape.caps[r + 1]); + let next_block = 1usize << shape.schedule[r + 1]; + // A tail round opens no successor block a query. + let next_openings = if shape.tail_round(r) { + 0 + } else { + shape.num_queries + }; + let next: Vec<(Vec, PathStore)> = (0..next_openings) + .map(|_| { + let values: Vec = (0..next_block) + .map(|_| next_word(b, &mut at).as_ext()) + .collect(); + let path = next_path(b, &mut at, next_depth); + (values, path) + }) + .collect(); + (Some(nr), Some(ov), next, next_cap) + } + None => (None, None, Vec::new(), Vec::new()), + }; + + sumchecks.push(sumcheck); + currents.push(current); + nexts.push(next); + roots.push(next_root); + oods.push(ood_value); + nonces.push(RoundNonces { + folding, + ood: ood_nonce, + query, + }); + current_caps.push(current_cap); + next_caps.push(next_cap); + } + assert_eq!( + at - base, + Self::words(shape), + "the round walk and the word count are one derivation" + ); + + Self { + shape: shape.clone(), + sumchecks, + currents, + nexts, + roots, + oods, + nonces, + current_caps, + next_caps, + } + } + + /// The query openings, current and successor, borrowing this storage. + #[allow(clippy::type_complexity)] + pub fn openings(&self) -> (Vec>>, Vec>>) { + let current: Vec>> = self + .currents + .iter() + .map(|round| { + round + .iter() + .map(|(values, path)| QueryOpening { + values: values.as_block(), + siblings: path.siblings(), + }) + .collect() + }) + .collect(); + let next: Vec>> = self + .nexts + .iter() + .map(|round| { + round + .iter() + .map(|(values, path)| QueryOpening { + values: BlockValues::Ext(values), + siblings: path.siblings(), + }) + .collect() + }) + .collect(); + (current, next) + } + + /// The wires `emit_verify_weighted` takes, borrowing this storage and the + /// openings built from it. + pub fn wires<'a>( + &'a self, + current: &'a [Vec>], + next: &'a [Vec>], + ) -> Vec> { + (0..self.shape.rounds()) + .map(|r| ChainRoundWires { + sumcheck: &self.sumchecks[r], + next_root: self.roots[r], + ood_value: self.oods[r], + nonces: self.nonces[r], + current: ¤t[r], + next: &next[r], + current_cap: &self.current_caps[r], + next_cap: &self.next_caps[r], + }) + .collect() + } +} + +pub fn round_words(shape: &ChainShape, r: usize) -> u32 { + let k = shape.schedule[r]; + // The sumcheck's two evaluations a round, the nonces the layout carries, + // and per query the current block plus its path (to the cap, when the tree + // has one). + let nonces = shape.carries(r).iter().filter(|&&carried| carried).count(); + let mut n = (2 * shape.messages(r) + nonces) as u32; + let depth = shape.current_path(r); + let block = 1usize << k; + if shape.tail_round(r) { + // A tail round: every leaf's block once, in leaf order, and no path. + n += (shape.current_leaves(r) * block) as u32; + } else { + // A grouped round 0 opens each of its trees at every query; round 0's + // base values four a word when packed. + let words = if r == 0 { + shape.round0_block_words(block) + } else { + block + }; + n += (shape.num_queries * shape.trees_at(r) * (words + depth)) as u32; + } + if r == 0 { + // W1: tree 0's cap words, a grouped round 0's trees' each. + n += (shape.trees * shape.cap_words(shape.current_depth(0), shape.caps[0])) as u32; + } + if let Some(next_depth) = shape.next_depth(r) { + // The successor root, its out-of-domain value, its cap, and per query + // its block and path. + let c = shape.caps[r + 1]; + n += 2 + shape.cap_words(next_depth, c) as u32; + // A tail round reads its successor's blocks where that tail round + // hints them, so it opens none a query. + if !shape.tail_round(r) { + let next_block = 1usize << shape.schedule[r + 1]; + n += (shape.num_queries * (next_block + shape.path_words(next_depth, c))) as u32; + } + } + n +} + +/// One chain's round wires, in the order [`RoundStorage::hint`] reads them. +/// +/// Split out of [`chain_arena`] for the same reason [`RoundStorage`] was split +/// out of [`chain_program`]: a program holding several chains fills one arena +/// with several of these, and a second copy of the order would be a second thing +/// to keep in step with the walk that reads it. +pub fn push_round_words( + words: &mut Vec, + shape: &ChainShape, + proof: &ChainProof, +) { + // A proof [`try_push_round_words`] cannot lay out leaves the arena short, + // and its length (a verifier constant) refuses it. + let _ = try_push_round_words(words, shape, proof); +} + +/// [`push_round_words`], refusing a proof whose tail round +/// ([`ChainShape::tail_round`]) misses a leaf's block: every leaf of a tail +/// tree is opened hundreds of times by an honest proof, so a missing one is a +/// completeness failure (probability ≈ 4·(3/4)^(2Q)), and the words written +/// before it stand. +pub fn try_push_round_words( + words: &mut Vec, + shape: &ChainShape, + proof: &ChainProof, +) -> Result<(), String> { + for (r, round) in proof.rounds.iter().enumerate() { + for sc in &round.sumcheck { + for e in &sc.evaluations { + words.push(ext_word(e)); + } + } + // The slots the layout carries. A slot it does not carry has no word, + // so its host field, which the host verifier requires to be 0 under + // `Spent`, has no way into the machine. + let nonces = [round.nonces.folding, round.nonces.ood, round.nonces.query]; + for (nonce, carried) in nonces.into_iter().zip(shape.carries(r)) { + if carried { + words.push([FE::from(nonce), FE::zero(), FE::zero(), FE::zero()]); + } + } + // W1: round 0 owns tree 0, whose cap rides at the end of its first + // current path; it goes to the arena here and the path goes without it. + let current = PathSide { + depth: shape.current_depth(r), + cap: shape.caps[r], + owner: r == 0, + }; + if shape.tail_round(r) { + push_tail_blocks(words, shape, proof, r)?; + } else { + push_openings(words, shape, round, true, current); + } + if let Some(next_depth) = shape.next_depth(r) { + words.push(commitment_to_digest( + round.next_root.as_ref().expect("a successor root"), + )); + words.push(ext_word( + round.ood_value.as_ref().expect("an out-of-domain value"), + )); + let next = PathSide { + depth: next_depth, + cap: shape.caps[r + 1], + owner: true, + }; + if shape.tail_round(r) { + // The successor's cap, for its tail round to authenticate; its + // blocks are that round's own. + for node in tail_cap(shape, round, r + 1)? { + words.push(commitment_to_digest(&node)); + } + } else { + push_openings(words, shape, round, false, next); + } + } + } + Ok(()) +} + +/// The successor openings round `r` carries, extension-valued at every round. +fn next_openings( + round: &ChainRound, +) -> &[multilinear::whir_commit::CosetOpening] { + match &round.openings { + RoundOpenings::Base(p) => &p.next, + RoundOpenings::Extension(p) => &p.next, + RoundOpenings::BaseGrouped(p) => &p.next, + } +} + +/// Tail round `t`'s tree's cap: the end of the first successor path of the +/// round before (`round`), its owner (W1). Every leaf of a tail tree is a cap +/// node, so the cap is the leaves' digests in leaf order. +fn tail_cap( + shape: &ChainShape, + round: &ChainRound, + t: usize, +) -> Result, String> { + let owner = next_openings(round) + .first() + .ok_or_else(|| format!("tail round {t}: no successor opening carries its cap"))?; + let (siblings, cap) = super::merkle_cap::child_path_to_cap( + &owner.proof.merkle_path, + shape.current_depth(t), + shape.caps[t], + shape.arity, + true, + )?; + match cap { + Some(cap) if siblings.is_empty() && cap.len() == shape.current_leaves(t) => Ok(cap), + _ => Err(format!( + "tail round {t}: the owner path is not a cap over every leaf" + )), + } +} + +/// ★ Tail round `r`'s blocks, one a leaf in leaf order ([`ChainShape::tail_round`]). +/// +/// The proof opens the tree at sampled positions only, and the host keeps no +/// position. Each distinct opened block — round `r − 1`'s successor openings +/// and round `r`'s own — is placed by its leaf hash among the cap's nodes, which +/// are the leaves' digests: no index is needed, and the arena stays a function +/// of the proof. A leaf no opening hits is a completeness failure, refused. +/// The leaf checks every placed block against its cap node, so a misplaced one +/// has no execution. +fn push_tail_blocks( + words: &mut Vec, + shape: &ChainShape, + proof: &ChainProof, + r: usize, +) -> Result<(), String> { + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + type Leaf = ::Backend< + GoldilocksExtension, + >; + let before = &proof.rounds[r - 1]; + let cap = tail_cap(shape, before, r)?; + let current: &[multilinear::whir_commit::CosetOpening] = + match &proof.rounds[r].openings { + RoundOpenings::Extension(p) => &p.current, + _ => { + return Err(format!( + "tail round {r}: its openings are not extension-valued" + )); + } + }; + let block = 1usize << shape.schedule[r]; + let mut placed: Vec]>> = + vec![None; cap.len()]; + for opening in next_openings(before).iter().chain(current) { + if opening.values.len() != block { + return Err(format!( + "tail round {r}: a block of {} values", + opening.values.len() + )); + } + let digest = Leaf::hash_data(&opening.values); + for (slot, node) in placed.iter_mut().zip(&cap) { + if slot.is_none() && *node == digest { + *slot = Some(&opening.values); + } + } + } + for (j, slot) in placed.iter().enumerate() { + let values = slot.ok_or_else(|| format!("tail round {r}: no opening of leaf {j}"))?; + for v in values { + words.push(ext_word(v)); + } + } + Ok(()) +} + +/// One side's tree, as its openings' paths are laid out: its index bits, its +/// cap height, and whether its first opening is the tree's owner (carries the +/// cap after its siblings, W1). +#[derive(Clone, Copy)] +struct PathSide { + depth: usize, + cap: usize, + owner: bool, +} + +/// One round's query openings, current or successor, block then path. +/// +/// The side's FIRST opening, when it is the owner, carries the cap's words at +/// the end of its path (the owner-path encoding, W1): they are written first, +/// ahead of every block, and the path is written without them. No cap at the +/// default. A malformed path is not repaired here: its words are written as +/// they are, and the arena's length (a verifier constant) refuses it. +/// +/// At arity 4 each path keeps the proof's child order and an uncapped +/// odd-depth top keeps only its partner ([`super::merkle_cap::child_path_to_cap`]). +fn push_openings( + words: &mut Vec, + shape: &ChainShape, + round: &ChainRound, + current: bool, + tree: PathSide, +) { + let owner_cap = if tree.owner { + shape.cap_words(tree.depth, tree.cap) + } else { + 0 + }; + if shape.arity == 4 { + push_openings4(words, round, current, tree, shape.round0_packed()); + return; + } + fn side( + words: &mut Vec, + openings: &[multilinear::whir_commit::CosetOpening], + owner_cap: usize, + value_word: impl Fn(&math::field::element::FieldElement) -> LfmWord, + ) where + V: math::field::traits::IsField, + { + let owner_split = openings + .first() + .map_or(0, |o| o.proof.merkle_path.len().saturating_sub(owner_cap)); + if let Some(owner) = openings.first() { + for node in &owner.proof.merkle_path[owner_split..] { + words.push(commitment_to_digest(node)); + } + } + for (i, opening) in openings.iter().enumerate() { + for v in &opening.values { + words.push(value_word(v)); + } + let path = if i == 0 { + &opening.proof.merkle_path[..owner_split] + } else { + &opening.proof.merkle_path[..] + }; + for node in path { + words.push(commitment_to_digest(node)); + } + } + } + match &round.openings { + RoundOpenings::Base(p) => { + if current { + // A base value arrives as `(v, 0, 0, 0)`. + side(words, &p.current, owner_cap, |v| { + [*v, FE::zero(), FE::zero(), FE::zero()] + }); + } else { + side(words, &p.next, owner_cap, ext_word); + } + } + RoundOpenings::Extension(p) => { + let openings = if current { &p.current } else { &p.next }; + side(words, openings, owner_cap, ext_word); + } + // A grouped round 0 (D-ONECHAIN): every tree's cap off its owner + // (query 0's opening of it), in tree order; then per query each + // tree's block and path. + RoundOpenings::BaseGrouped(p) => { + if current { + let owner_splits: Vec = p + .current + .first() + .map(|row| { + row.iter() + .map(|owner| { + let path = &owner.proof.merkle_path; + let split = path.len().saturating_sub(owner_cap); + for node in &path[split..] { + words.push(commitment_to_digest(node)); + } + split + }) + .collect() + }) + .unwrap_or_default(); + for (i, row) in p.current.iter().enumerate() { + for (tree, opening) in row.iter().enumerate() { + for v in &opening.values { + words.push([*v, FE::zero(), FE::zero(), FE::zero()]); + } + let path = &opening.proof.merkle_path; + let path = match (i, owner_splits.get(tree)) { + (0, Some(&split)) => &path[..split], + _ => &path[..], + }; + for node in path { + words.push(commitment_to_digest(node)); + } + } + } + } else { + side(words, &p.next, owner_cap, ext_word); + } + } + } +} + +/// Round 0's base block: a `(v, 0, 0, 0)` word a value, or four values a word, +/// the last zero-filled, when `packed` ([`ChainShape::round0_packed`]). +fn push_base_block(words: &mut Vec, values: &[FE], packed: bool) { + if packed { + for lanes in values.chunks(4) { + let mut word = [FE::zero(); 4]; + word[..lanes.len()].copy_from_slice(lanes); + words.push(word); + } + } else { + words.extend( + values + .iter() + .map(|v| [*v, FE::zero(), FE::zero(), FE::zero()]), + ); + } +} + +/// [`push_openings`] at arity 4: the owner's cap, then per opening its block +/// and its path in child order, cut to the cap. Only a current base side is +/// ever `packed`. +fn push_openings4( + words: &mut Vec, + round: &ChainRound, + current: bool, + tree: PathSide, + packed: bool, +) { + fn side( + words: &mut Vec, + openings: &[multilinear::whir_commit::CosetOpening], + tree: PathSide, + block: impl Fn(&mut Vec, &[math::field::element::FieldElement]), + ) where + V: math::field::traits::IsField, + { + let laid: Vec<(Vec<_>, Option>)> = openings + .iter() + .enumerate() + .map(|(i, o)| { + let path = &o.proof.merkle_path; + super::merkle_cap::child_path_to_cap( + path, + tree.depth, + tree.cap, + 4, + tree.owner && i == 0, + ) + // A path of another shape goes in as it is: the arena's length + // refuses it. + .unwrap_or_else(|_| (path.clone(), None)) + }) + .collect(); + if let Some((_, Some(cap))) = laid.first() { + for node in cap { + words.push(commitment_to_digest(node)); + } + } + for (opening, (path, _)) in openings.iter().zip(&laid) { + block(words, &opening.values); + for node in path { + words.push(commitment_to_digest(node)); + } + } + } + let ext_block = |words: &mut Vec, values: &[FEE]| { + words.extend(values.iter().map(ext_word)); + }; + match &round.openings { + RoundOpenings::Base(p) => { + if current { + side(words, &p.current, tree, |words, values| { + push_base_block(words, values, packed) + }); + } else { + side(words, &p.next, tree, ext_block); + } + } + RoundOpenings::Extension(p) => { + let openings = if current { &p.current } else { &p.next }; + side(words, openings, tree, ext_block); + } + // See `push_openings`: the trees' caps in tree order, then per query + // each tree's block and path. + RoundOpenings::BaseGrouped(p) => { + if current { + let laid: Vec, Option>)>> = p + .current + .iter() + .enumerate() + .map(|(i, row)| { + row.iter() + .map(|o| { + let path = &o.proof.merkle_path; + super::merkle_cap::child_path_to_cap( + path, + tree.depth, + tree.cap, + 4, + tree.owner && i == 0, + ) + .unwrap_or_else(|_| (path.clone(), None)) + }) + .collect() + }) + .collect(); + for (_, cap) in laid.first().into_iter().flatten() { + for node in cap.iter().flatten() { + words.push(commitment_to_digest(node)); + } + } + for (row, laid_row) in p.current.iter().zip(&laid) { + for (opening, (path, _)) in row.iter().zip(laid_row) { + push_base_block(words, &opening.values, packed); + for node in path { + words.push(commitment_to_digest(node)); + } + } + } + } else { + side(words, &p.next, tree, ext_block); + } + } + } +} + +// ============================================================================= +// A program's rows by chip — what a Poseidon1 leaf's partition bounds +// ============================================================================= + +/// Rows a leaf part spends in each chip of the LFM machine, one a chip's +/// instruction (`exec_schedule`'s census holds a program's chips to its +/// instruction counts), so a leaf's padded heights follow from its sum: what a +/// Poseidon1 leaf's partition bounds ([`super::whir_block::leaf_partition_rows`]). +/// `LFM_RANGE` is the one chip no instruction opens — a fixed table, the same in +/// every leaf. `consts` counts distinct constants (the builder interns them), so +/// a leaf's is at most its parts' sum. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ChipRows { + /// The socket (`Hash` or `Hash16`). + pub hash: usize, + pub select: usize, + pub balu: usize, + pub xalu: usize, + /// `Pack` and `Unpack`. + pub lanes: usize, + /// The hinted words. + pub hint: usize, + pub bitdec: usize, + pub consts: usize, + pub public: usize, + /// `KeccakF` and `Blake3`: none in a WHIR leaf. + pub accel: usize, +} + +impl ChipRows { + /// The chips' names, in [`Self::to_array`]'s order. + pub const NAMES: [&'static str; 10] = [ + "socket", + "Select", + "base-ALU", + "ext-ALU", + "lanes", + "hint", + "bit-dec", + "const", + "public", + "accelerator", + ]; + + pub fn to_array(self) -> [usize; 10] { + [ + self.hash, + self.select, + self.balu, + self.xalu, + self.lanes, + self.hint, + self.bitdec, + self.consts, + self.public, + self.accel, + ] + } + + pub fn from_array(a: [usize; 10]) -> Self { + let [ + hash, + select, + balu, + xalu, + lanes, + hint, + bitdec, + consts, + public, + accel, + ] = a; + Self { + hash, + select, + balu, + xalu, + lanes, + hint, + bitdec, + consts, + public, + accel, + } + } + + pub fn plus(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + Self::from_array(core::array::from_fn(|c| a[c] + b[c])) + } + + /// Whether every chip's rows are at most `heights`'. + pub fn under(self, heights: Self) -> bool { + self.to_array() + .iter() + .zip(heights.to_array()) + .all(|(&r, h)| r <= h) + } + + /// The rows by chip of some emitted instructions. + pub fn of_instrs(instrs: &[super::instr::Instr]) -> Self { + use super::instr::Instr; + let mut rows = Self::default(); + for instr in instrs { + let chip = match instr { + Instr::Hash { .. } | Instr::Hash16(_) => &mut rows.hash, + Instr::Select { .. } => &mut rows.select, + Instr::BaseAlu { .. } => &mut rows.balu, + Instr::ExtAlu { .. } => &mut rows.xalu, + Instr::Pack { .. } | Instr::Unpack { .. } => &mut rows.lanes, + Instr::Hint { .. } | Instr::HintOn { .. } => &mut rows.hint, + Instr::BitDec { .. } => &mut rows.bitdec, + Instr::Const { .. } => &mut rows.consts, + Instr::Public { .. } => &mut rows.public, + Instr::KeccakF(_) | Instr::Blake3(_) => &mut rows.accel, + }; + *chip += 1; + } + rows + } + + /// The rows by chip of a compiled program. + pub fn of(program: &super::compiler::LfmProgram) -> Self { + Self::of_instrs(&program.instrs) + } +} diff --git a/prover/src/lfm/whir_chain_tests.rs b/prover/src/lfm/whir_chain_tests.rs new file mode 100644 index 000000000..35a8b83c6 --- /dev/null +++ b/prover/src/lfm/whir_chain_tests.rs @@ -0,0 +1,2608 @@ +//! Gates for the assembled WHIR chain. +//! +//! The proof under test is a real one: `whir_chain::prove` over a real +//! polynomial and a real commitment, under the RPX transcript the machine +//! replays. Small — a 2^6 polynomial, three queries — because the chain's +//! shapes are what this checks and a 2^25 stack is a box's work, not a +//! laptop's. +//! +//! # ★ Why executing the honest proof IS the stream comparison +//! +//! The machine never compares its challenges to the host's. It USES them, and +//! every one of them feeds a refusal: a wrong query index opens the wrong leaf +//! and fails at the root, a wrong folding challenge folds to something the +//! successor block does not hold, a wrong sumcheck challenge leaves a claim the +//! tail's division does not close. So a proof that the host accepts executing +//! here is already the statement that every drawn value agrees. +//! +//! What the recording transcript IS used for is a second check, on the host and +//! not on the machine: the SHAPE of the draw stream. `verify_weighted` must draw +//! one extension challenge per sumcheck round plus `z0` and `gamma` on every +//! round but the last, and `Q` bounded draws per round — `num_vars + 2(R − 1)` +//! and `Q·R`. Those counts are what the emitter's order is built on, so if the +//! host ever reorders, this says so rather than leaving the machine to fail at +//! a root and look like a hashing bug. + +use core::cell::RefCell; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; +use math::traits::AsBytes; +use multilinear::mle::Mle; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, ChainProof, FirstFold, GrindBits, NonceLayout, + RoundOpenings, WhirFolds, commit, prove, verify, +}; +use multilinear::whir_hash::RpxWhir; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::commitment_to_digest; +use super::builder::{Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_chain::{ + ChainShape, RoundStorage, chain_cap_perms, chain_grind_perms, chain_hash_schedule, + chain_opening_perms, chain_perms, chain_rows, chain_schedule_perms, chain_schedule_rows, + chain_shape_rows, emit_verify_weighted, push_round_words, round_words, +}; +use super::whir_poly::{emit_eq_eval, eq_eval_rows_again}; +use super::whir_transcript::{SpongeEntry, SpongeHash, WhirTranscript}; +use super::word::{LfmWord, ext_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type HostTranscript = DefaultTranscript; + +/// Bytes one squeeze hands out (`default_transcript.rs:19`). +const SQUEEZE_BYTES: usize = 32; + +/// Bytes one candidate consumes (`:130-132`). +const CANDIDATE_BYTES: usize = 8; + +/// ★ The host's duplex, reconstructed from the calls the VERIFIER makes. +/// +/// This is the second derivation the chain's schedule half is gated against, +/// and it reads nothing of the emitter. Every rule is the host's own: +/// +/// - an absorb appends its bytes and invalidates the output buffer +/// (`default_transcript.rs:205-210`); +/// - a candidate is eight bytes, and refills with ONE `sample()` when fewer +/// than eight remain (`:125-134`); +/// - `sample()` hashes everything absorbed since the last one and RE-ABSORBS +/// its 32-byte digest, so the buffer afterwards is four felts and not none +/// (`:104-110`); +/// - `state()` hashes that same buffer and leaves it alone (`:236-239`). +/// +/// ★ **The reconstruction is not trusted, it is CHECKED.** `shadow` is a second +/// `DefaultTranscript` fed exactly the same bytes and squeezed at exactly the +/// points this reconstruction says the host squeezes; every value the real +/// transcript hands back is compared against the one the shadow's bytes give. +/// A squeeze in the wrong place leaves different bytes in the buffer, so the +/// very next draw disagrees — which is what makes this a check and not a +/// restatement of the same belief twice. +pub(super) struct HostDuplex { + shadow: HostTranscript, + out: [u8; SQUEEZE_BYTES], + out_pos: usize, + /// Bytes absorbed since the last squeeze. + buffered: usize, + pub(super) hashes: Vec, +} + +impl HostDuplex { + fn new() -> Self { + Self { + shadow: HostTranscript::new(&[]), + out: [0u8; SQUEEZE_BYTES], + out_pos: SQUEEZE_BYTES, + buffered: 0, + hashes: Vec::new(), + } + } + + /// Felts in the buffer. A hash reads whole 8-byte groups, so a buffer that + /// is not a multiple of eight would mean a value straddling two felts — + /// the alignment the emitter refuses by construction, asserted here rather + /// than rounded away. + fn felts(&self) -> usize { + assert_eq!( + self.buffered % CANDIDATE_BYTES, + 0, + "the chain absorbs whole felts: {} bytes buffered", + self.buffered + ); + self.buffered / CANDIDATE_BYTES + } + + fn absorb(&mut self, bytes: &[u8]) { + self.shadow.append_bytes(bytes); + self.buffered += bytes.len(); + self.out_pos = SQUEEZE_BYTES; + } + + /// `append_field_element` absorbs exactly the bytes the element streams + /// (`:193-206`), so the shadow is given those same bytes and their count is + /// MEASURED rather than assumed to be three felts. + fn absorb_element(&mut self, element: &FEE) { + let mut bytes = Vec::new(); + element.stream_bytes(&mut |chunk| bytes.extend_from_slice(chunk)); + self.absorb(&bytes); + } + + fn state(&mut self) { + let felts = self.felts(); + self.hashes.push(SpongeHash::State(felts)); + } + + fn candidate(&mut self) -> u64 { + if self.out_pos + CANDIDATE_BYTES > SQUEEZE_BYTES { + let felts = self.felts(); + self.hashes.push(SpongeHash::Squeeze(felts)); + self.out = self.shadow.sample(); + self.buffered = SQUEEZE_BYTES; + self.out_pos = 0; + } + let mut bytes = [0u8; CANDIDATE_BYTES]; + bytes.copy_from_slice(&self.out[self.out_pos..self.out_pos + CANDIDATE_BYTES]); + self.out_pos += CANDIDATE_BYTES; + u64::from_be_bytes(bytes) + } +} + +/// A transcript that records what it hands out, delegating everything. +/// +/// The chain's challenges are drawn interleaved with its sumchecks and never +/// returned, and re-deriving them would mirror the host rather than read it. +/// +/// It also carries [`HostDuplex`], which reconstructs the host's hash schedule +/// from these same calls. `state()` takes `&self`, so the reconstruction lives +/// behind a `RefCell`. +pub(super) struct Recording { + inner: HostTranscript, + pub(super) sampled: Vec, + pub(super) drawn_u64: Vec, + pub(super) duplex: RefCell, +} + +impl Recording { + pub(super) fn new() -> Self { + Self { + inner: HostTranscript::new(&[]), + sampled: Vec::new(), + drawn_u64: Vec::new(), + duplex: RefCell::new(HostDuplex::new()), + } + } +} + +/// ★ The recorder's hash, NAMED — because `multi_verify` will not take a +/// transcript that does not name one. +/// +/// This is not a convenience: the bound exists so that a keccak transcript +/// cannot be passed under an RPX `H`, which is the half-configured arm that ran +/// undetected for a day. It is TRUE here rather than asserted: `Recording`'s +/// inner transcript is `DefaultTranscript` (the alias at +/// the top of this file), so a recorder handed to an RPX verify runs the same +/// sponge the verify does. Changing that alias must change this line with it. +impl crypto::fiat_shamir::transcript_hash::HasTranscriptHash for Recording { + type Hash = RpxTranscriptHash; +} + +impl IsTranscript for Recording { + fn append_field_element(&mut self, element: &FEE) { + self.duplex.borrow_mut().absorb_element(element); + self.inner.append_field_element(element); + } + fn append_bytes(&mut self, new_bytes: &[u8]) { + self.duplex.borrow_mut().absorb(new_bytes); + self.inner.append_bytes(new_bytes); + } + fn state(&self) -> [u8; 32] { + self.duplex.borrow_mut().state(); + self.inner.state() + } + fn sample_field_element(&mut self) -> FEE { + // Three candidates, one a coordinate and in coordinate order + // (`extensions_goldilocks.rs:574-581`); each coordinate IS its + // candidate, because `candidate_in_range` accepted it. + let mine: [FE; 3] = { + let mut duplex = self.duplex.borrow_mut(); + core::array::from_fn(|_| FE::from(duplex.candidate())) + }; + let drawn = self.inner.sample_field_element(); + assert_eq!( + *drawn.value(), + mine, + "the reconstructed duplex must reproduce the host's challenge — a squeeze in the \ + wrong place is what this catches" + ); + self.sampled.push(drawn); + drawn + } + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + let mine = self.duplex.borrow_mut().candidate(); + let drawn = self.inner.sample_u64(upper_bound); + assert_eq!( + mine % upper_bound, + drawn, + "the reconstructed duplex must reproduce the host's query position" + ); + self.drawn_u64.push(drawn); + drawn + } +} + +fn pseudo_mle(num_vars: usize, seed: u64) -> Mle { + let vals = (0..1usize << num_vars) + .map(|i| { + let mixed = (i as u64) + .wrapping_mul(0x2545_F491_4F6C_DD1D) + .wrapping_add(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + FE::from(mixed >> 13) + }) + .collect(); + Mle::new(vals).expect("a cube") +} + +fn point(num_vars: usize, seed: u64) -> Vec { + (0..num_vars) + .map(|i| { + FEE::new([ + FE::from(101 + seed + i as u64), + FE::from(7 * i as u64 + 3), + FE::from(i as u64 + 1), + ]) + }) + .collect() +} + +fn config(num_queries: usize, grind: u8) -> ChainConfig { + config_with(num_queries, grind, CapPolicy::Off) +} + +/// [`config`] under a Merkle cap policy (W1). +fn config_with(num_queries: usize, grind: u8, cap: CapPolicy) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries, + grind: GrindBits::uniform(grind), + format: multilinear::whir_chain::ChainFormat { + cap, + ..multilinear::whir_chain::ChainFormat::DEFAULT + }, + } +} + +/// [`config`] under a first fold of `k0` (W2's `first5`/`first6` at `k0` = 5, 6). +fn first_fold_config(num_queries: usize, grind: u8, k0: usize) -> ChainConfig { + ChainConfig { + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(k0).expect("a tested first fold")), + ..ChainFormat::DEFAULT + }, + ..config(num_queries, grind) + } +} + +/// Everything one run needs on both sides. +struct Fixture { + proof: ChainProof, + z: Vec, + y: FEE, + root: LfmWord, + /// The same root as the host holds it — kept rather than rebuilt from the + /// lanes, because a second conversion is a second thing to get wrong. + root_bytes: [u8; 32], + domain: Domain, + shape: ChainShape, + /// Kept so a failing run can print which draw diverged; the counts it is + /// checked for are asserted in `fixture` above. + #[allow(dead_code)] + recorded: Recording, +} + +/// Proves and verifies one chain on the host, keeping the verifier's draws. +/// +/// ⚠ Under `RpxTranscriptHash`, not the default keccak one — the machine's +/// replay reproduces that hash and no other, so a fixture on the default +/// transcript would be a fixture of a different protocol. +fn fixture(num_vars: usize, num_queries: usize, grind: u8) -> Fixture { + fixture_with(&config(num_queries, grind), num_vars) +} + +/// [`fixture`] under a Merkle cap policy (W1). +fn fixture_capped(num_vars: usize, num_queries: usize, grind: u8, cap: CapPolicy) -> Fixture { + fixture_with(&config_with(num_queries, grind, cap), num_vars) +} + +/// [`fixture`] under any config — the knob-on shapes use it. +fn fixture_with(cfg: &ChainConfig, num_vars: usize) -> Fixture { + let num_queries = cfg.num_queries; + let f = pseudo_mle(num_vars, 11); + let z = point(num_vars, 0); + // `evaluate_in`, not `evaluate`: the claimed point is in the cubic + // extension, which is where every WHIR challenge lives. + let y = f.evaluate_in::(&z).expect("f takes its own point"); + + let (commitment, domain) = commit::(&f, cfg, true).expect("the polynomial commits"); + let mut proving = HostTranscript::new(&[]); + let proof = prove::(&f, &z, &commitment, &domain, cfg, &mut proving) + .expect("the chain proves"); + + let mut recorded = Recording::new(); + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + cfg, + &mut recorded, + ) + .expect("the control proof must verify"); + + // ★ The host's draw stream has the shape the emitter's order assumes: one + // challenge per sumcheck round, `z0` and `gamma` on every round but the + // last, and `Q` bounded draws a round. Derived from the schedule, not read + // off the recorder. + let shape = ChainShape::new(cfg, num_vars); + let rounds = shape.rounds(); + assert_eq!( + recorded.sampled.len(), + num_vars + 2 * (rounds - 1), + "extension draws: one a sumcheck round ({num_vars} in all), plus z0 and gamma on \ + each of the {} rounds with a successor", + rounds - 1 + ); + assert_eq!( + recorded.drawn_u64.len(), + num_queries * rounds, + "bounded draws: {num_queries} query positions in each of {rounds} rounds" + ); + + Fixture { + proof, + z, + y, + root: commitment_to_digest(&commitment.root()), + root_bytes: commitment.root(), + domain, + shape, + recorded, + } +} + +/// Where every wire of one round lives in the arena. Built once and used by +/// both the program and the arena filler, so the two cannot drift. +struct Layout { + shape: ChainShape, + /// The arena index each round's block of wires starts at. + round_at: Vec, + total: u32, +} + +impl Layout { + fn new(shape: &ChainShape) -> Self { + let mut round_at = Vec::with_capacity(shape.rounds()); + // z, y, the root, and the final value. + let mut at = (shape.num_vars + 3) as u32; + for r in 0..shape.rounds() { + round_at.push(at); + at += round_words(shape, r); + } + Self { + shape: shape.clone(), + round_at, + total: at, + } + } +} + +/// Builds the program for one shape, publishing every extension challenge the +/// machine draws in the order it draws them. +pub(super) fn chain_program(shape: &ChainShape) -> LfmProgram { + let program = chain_program_from( + shape, + LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()), + ); + validate(&program).expect("the chain leg must be admissible"); + program +} + +/// [`chain_program`]'s emission from a given (empty) builder, unvalidated — +/// the D-HASH census emits it under a different geometry. +pub(super) fn chain_program_from(shape: &ChainShape, b: LfmBuilder) -> LfmProgram { + let layout = Layout::new(shape); + let mut b = b; + let arena = b.declare_arena(layout.total); + let mut transcript = WhirTranscript::new(); + + let z: Vec = (0..shape.num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let y = b.hint_word(arena, shape.num_vars as u32).as_ext(); + let root = b.hint_word(arena, shape.num_vars as u32 + 1); + let root_lanes = b.unpack(root); + let final_value = b.hint_word(arena, shape.num_vars as u32 + 2).as_ext(); + + // Owned storage, because the wires borrow from it. + let storage = RoundStorage::hint(&mut b, arena, layout.round_at[0], shape); + let (current_openings, next_openings) = storage.openings(); + let wires = storage.wires(¤t_openings, &next_openings); + + emit_verify_weighted( + &mut b, + &mut transcript, + &wires, + &root_lanes, + final_value, + y, + shape, + &layout.shape_domain(), + |b, alphas| emit_eq_eval(b, &z, alphas), + ); + + compile(b.finish()) +} + +impl Layout { + fn shape_domain(&self) -> Domain { + Domain::::new(self.shape.domain_log[0]).expect("the chain's first domain") + } +} + +/// The arena in the order [`chain_program`] hints it. +fn chain_arena(fixture: &Fixture, proof: &ChainProof) -> Vec { + chain_arena_as(&fixture.shape, fixture, proof) +} + +/// [`chain_arena`] under another shape of the same chain: the same proof +/// written in another nonce layout. +fn chain_arena_as(shape: &ChainShape, fixture: &Fixture, proof: &ChainProof) -> Vec { + let mut words: Vec = fixture.z.iter().map(ext_word).collect(); + words.push(ext_word(&fixture.y)); + words.push(fixture.root); + words.push(ext_word(&proof.final_value)); + + push_round_words(&mut words, shape, proof); + words +} + +/// ★ The assembled chain executes on a proof the host accepts. +/// +/// Every challenge the machine draws feeds a refusal, so this is the stream +/// comparison — see the module doc. +#[test] +fn the_chain_executes_on_a_proof_the_host_accepts() { + // ★ `S = 9` is the three-round shape: the only one here carrying a round + // that is neither the first nor the last, and therefore the only one where + // a successor block is opened and then re-opened as a current one. + for (num_vars, num_queries) in [(6usize, 3usize), (6, 5), (5, 3), (9, 3)] { + let f = fixture(num_vars, num_queries, 0); + let program = chain_program(&f.shape); + let arena = chain_arena(&f, &f.proof); + println!( + "chain S={num_vars} Q={num_queries}: {} rounds, {} instructions, opening \ + permutations predicted {}", + f.shape.rounds(), + program.instrs.len(), + chain_opening_perms(&f.shape) + ); + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!("S={num_vars} Q={num_queries}: the machine refused an accepted proof: {e:?}") + }); + } +} + +/// ★ The tamper arm: three halves, five sites. +/// +/// ⚠ Run at a REAL grind width, and the reason is a catch this arm made on +/// itself. At zero grind bits `check_grind` returns before reading the nonce +/// (`whir_chain.rs:131-133`), so flipping one is not a forgery at all — the +/// host accepts it, and the arm's middle half said so: "the host must reject +/// the forgery, or the machine's refusal is a refusal of something valid". A +/// version without that half would have recorded the machine refusing a proof +/// the host ACCEPTS as a soundness success, when it is a completeness bug. +#[test] +fn a_tampered_chain_cannot_execute() { + let grind = 8u8; + let f = fixture(6, 3, grind); + let program = chain_program(&f.shape); + + assert!( + execute( + &program, + &[chain_arena(&f, &f.proof)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the untouched proof must execute, or the arm below proves nothing" + ); + + let host_rejects = |proof: &ChainProof, root_bytes: &[u8; 32]| -> bool { + let mut t = Recording::new(); + verify::( + proof, + root_bytes, + &f.z, + f.y, + &f.domain, + &config(3, grind), + &mut t, + ) + .is_err() + }; + let honest_root_bytes = f.root_bytes; + + let mut sites: Vec<(&str, ChainProof)> = Vec::new(); + + // A final value: the tail's division no longer closes. + let mut forged = f.proof.clone(); + forged.final_value += FEE::one(); + sites.push(("the final value", forged)); + + // A sumcheck evaluation: the claim the group leaves moves. + let mut forged = f.proof.clone(); + forged.rounds[0].sumcheck[0].evaluations[0] += FEE::one(); + sites.push(("a sumcheck evaluation", forged)); + + // An out-of-domain value: `gamma` moves, and with it every later draw. + let mut forged = f.proof.clone(); + if let Some(v) = forged.rounds[0].ood_value.as_mut() { + *v += FEE::one(); + } + sites.push(("an out-of-domain value", forged)); + + // A grind nonce: the range check fails. + let mut forged = f.proof.clone(); + forged.rounds[0].nonces.query ^= 1; + sites.push(("a grind nonce", forged)); + + // A Merkle sibling: the walk arrives somewhere else. + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => p.current[0].proof.merkle_path[0][0] ^= 1, + RoundOpenings::Extension(p) => p.current[0].proof.merkle_path[0][0] ^= 1, + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push(("a Merkle sibling", forged)); + + for (name, forged) in &sites { + assert!( + host_rejects(forged, &honest_root_bytes), + "{name}: the host must reject the forgery, or the machine's refusal is \ + a refusal of something valid" + ); + assert!( + execute( + &program, + &[chain_arena(&f, forged)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "{name}: the machine must refuse the forgery" + ); + } + println!("chain tamper arm: {} sites, all refused", sites.len()); +} + +/// ★ The grind is spent where the host spends it: `3R − 1` times, and a wrong +/// nonce at any of them has no execution. +/// +/// Run at a real grind width rather than zero, so the nonce search actually +/// happens and the machine's range check is exercised. Small bits, because the +/// prover's search is `2^bits` hashes. +#[test] +fn the_chain_spends_its_grinds_where_the_host_does() { + let bits = 8u8; + let f = fixture(6, 3, bits); + let program = chain_program(&f.shape); + let rounds = f.shape.rounds(); + + assert!( + execute( + &program, + &[chain_arena(&f, &f.proof)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the ground proof must execute" + ); + + let mut refused = 0; + for r in 0..rounds { + let last = r + 1 == rounds; + let mut spots: Vec<&str> = vec!["folding", "query"]; + if !last { + spots.push("ood"); + } + for spot in spots { + let mut forged = f.proof.clone(); + match spot { + "folding" => forged.rounds[r].nonces.folding ^= 1, + "query" => forged.rounds[r].nonces.query ^= 1, + _ => forged.rounds[r].nonces.ood ^= 1, + } + assert!( + execute( + &program, + &[chain_arena(&f, &forged)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "round {r}, the {spot} nonce: a wrong nonce must have no execution" + ); + refused += 1; + } + } + assert_eq!( + refused, + 3 * rounds - 1, + "a chain spends 3R − 1 nonces; every one of them must be checked" + ); + println!("chain grinds: {refused} nonces, all checked, at {bits} bits"); +} + +/// ★ The production shape, checked by arithmetic rather than by a run. +/// +/// The epoch's group-0 chains are `n_stack = 25` at `k = 4`, blowup 4, `Q = +/// 112` (sh1, 2026-09-18). That shape is far past what a laptop proves, but the +/// permutation form is a function of the tree depths and block widths alone, so +/// it can be evaluated without emitting anything — and it is the one number the +/// campaign's sizing carries, so it is worth pinning against a derivation done +/// a second way. +/// +/// The second derivation, by hand off `verify_weighted`'s round structure: +/// schedule `[4,4,4,4,4,4,1]`, domains 27/23/19/15/11/7/3; current depths +/// 23+19+15+11+7+3+2 = 80 and successor depths 19+15+11+7+3+2 = 57, so 137 +/// parents; current leaves 2 (round 0 is 16 BASE felts) + 5x6 (48 extension +/// felts) + 1 (the 6-felt tail block) = 33 and successor leaves 5x6 + 1 = 31, +/// so 64 leaf blocks. 201 a query, 22,512 a chain. +#[test] +fn the_production_shape_reproduces_the_campaigns_permutation_count() { + let shape = ChainShape::new(&config(112, 20), 25); + assert_eq!(shape.schedule, vec![4, 4, 4, 4, 4, 4, 1]); + assert_eq!(shape.domain_log, vec![27, 23, 19, 15, 11, 7, 3]); + + let parents: usize = (0..shape.rounds()) + .map(|r| shape.current_depth(r) + shape.next_depth(r).unwrap_or(0)) + .sum(); + assert_eq!(parents, 137, "Merkle parents a query"); + + let perms = chain_opening_perms(&shape); + println!("production chain S=25 k=4 Q=112: {perms} permutations, {parents} parents a query"); + assert_eq!( + perms, 22_512, + "the opening form at the production shape must reproduce the hand derivation" + ); + assert_eq!(perms % shape.num_queries, 0); + assert_eq!(perms / shape.num_queries, 201, "permutations a query"); + + // The tail round's block is the one whose felt count is not a multiple of + // eight: two extension values are six felts, and reading the capacity off + // the VALUE count instead would make it two. + assert_eq!(shape.current_felts(6), 6); + assert_eq!( + shape.current_felts(0), + 16, + "round 0 is base: one felt a value" + ); + assert_eq!(shape.current_felts(1), 48, "every later round is three"); +} + +/// A one-refusal program: hints its inputs, runs the refusal, publishes a +/// witness so the program has an output. +fn refusal_program(which: &str) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(3); + let one = b.ext_const(&FEE::one()); + let a = b.hint_word(arena, 0).as_ext(); + let c = b.hint_word(arena, 1).as_ext(); + let d = b.hint_word(arena, 2).as_ext(); + match which { + "ood" => super::whir_chain::emit_require_out_of_domain(&mut b, a, one), + _ => super::whir_chain::emit_final_check(&mut b, a, c, d, one), + } + b.public(a.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("a refusal leg must be admissible"); + program +} + +fn runs(program: &LfmProgram, words: [FEE; 3]) -> bool { + let arena: Vec = words.iter().map(ext_word).collect(); + execute(program, &[arena], &crate::hash_pin::BLOCK_HASHER).is_ok() +} + +/// ★ The two refusals a chain-level gate CANNOT exercise, driven directly. +/// +/// Mutations CC and CD deleted each of these from the assembled chain and all +/// four chain tests stayed green — measured, not feared. The reason is the same +/// for both: the input that would trip them is one a Fiat-Shamir transcript does +/// not produce. An in-domain `z0` has negligible probability, and a weight and +/// claim that are BOTH zero is a point no honest or tampered fixture reaches. +/// +/// So they are called with the inputs the protocol never will, each beside a +/// control that must execute — otherwise the refusal would be a refusal of +/// everything, which is the sibling failure. +#[test] +fn the_refusals_a_real_proof_cannot_reach() { + let ood = refusal_program("ood"); + let zero = FEE::zero(); + let two = FEE::one() + FEE::one(); + // `z0` raised to the domain's size: one means in-domain. + assert!( + runs(&ood, [two, zero, zero]), + "an out-of-domain point must execute, or the refusal below refuses everything" + ); + assert!( + !runs(&ood, [FEE::one(), zero, zero]), + "an IN-domain point must have no execution: this is `OodPointInDomain`, and a \ + transcript never draws one, so nothing else in this suite can see it" + ); + + let tail = refusal_program("tail"); + // (claim, weight, final_value): the honest relation is final = claim/weight. + assert!( + runs(&tail, [two + two, two, two]), + "4 / 2 == 2 must execute, or the refusal below refuses everything" + ); + assert!( + !runs(&tail, [two + two, two, FEE::one()]), + "a final value that is not claim/weight must have no execution" + ); + assert!( + !runs(&tail, [zero, zero, FEE::one()]), + "★ the DOUBLE ZERO: the host returns DegenerateEvaluationPoint here, and the \ + cross-multiplied form `final·weight == claim` would accept any final value. \ + This is the whole reason the tail inverts." + ); + assert!( + !runs(&tail, [two, zero, FEE::one()]), + "a zero weight with a nonzero claim must have no execution either" + ); +} + +/// The shapes the cost forms are gated at: `(num_vars, num_queries, grind)`. +/// +/// Grind 0 is not a smaller version of grind 8. With no query grind the query +/// phase is entered with a candidate still in hand, which is the `avail > 0` +/// branch of the schedule — a branch the production shape never takes, and one +/// a suite run only at a real grind width would never reach. `S = 9` is the +/// only shape here with three rounds, so it is the only one carrying a round +/// that is neither the first nor the last. +const COST_SHAPES: [(usize, usize, u8); 5] = + [(6, 3, 0), (6, 5, 0), (5, 3, 0), (6, 3, 8), (9, 3, 8)]; + +/// ★ The knob-on shapes (W2): `(num_vars, num_queries, grind, k0)`. +/// +/// `S = 9` under `first6` is `[6, 3]` and `S = 11` under `first5` is +/// `[5, 4, 2]`, each at grind 0 and 8 for the reason +/// [`COST_SHAPES`] gives. `S = 6` under `first6` is the one-round chain whose +/// only block is 64 base values, and `S = 7` is `[6, 1]`, a 64-wide base block +/// folded into a 2-wide extension tail. +const KNOB_COST_SHAPES: [(usize, usize, u8, usize); 6] = [ + (9, 3, 0, 6), + (9, 3, 8, 6), + (11, 3, 0, 5), + (11, 3, 8, 5), + (6, 3, 8, 6), + (7, 3, 8, 6), +]; + +/// Every shape the cost forms are gated at: [`COST_SHAPES`] under today's +/// schedule, then [`KNOB_COST_SHAPES`] under their first folds. +fn cost_configs() -> Vec<(ChainConfig, usize)> { + COST_SHAPES + .iter() + .map(|&(n, q, g)| (config(q, g), n)) + .chain( + KNOB_COST_SHAPES + .iter() + .map(|&(n, q, g, k0)| (first_fold_config(q, g, k0), n)), + ) + .collect() +} + +pub(super) fn count_rows(program: &LfmProgram, want: fn(&super::instr::Instr) -> bool) -> usize { + program.instrs.iter().filter(|instr| want(instr)).count() +} + +pub(super) fn const_rows(program: &LfmProgram) -> usize { + count_rows(program, |i| matches!(i, super::instr::Instr::Const { .. })) +} + +/// `LFM_HASH` invocations: one per sponge permutation, whether it is a leaf's +/// duplex block or a Merkle parent's compress. +pub(super) fn perm_rows(program: &LfmProgram) -> usize { + count_rows(program, |i| matches!(i, super::instr::Instr::Hash { .. })) +} + +pub(super) fn hint_rows(program: &LfmProgram) -> usize { + count_rows(program, |i| { + matches!( + i, + super::instr::Instr::Hint { .. } | super::instr::Instr::HintOn { .. } + ) + }) +} + +/// Rows the chain PROGRAM carries that are not the chain's own cost. +/// +/// Derived from the test's own [`Layout`] and not from the program's +/// histogram, so a form row that turned into a hint could not hide inside the +/// subtraction: one `Hint` per arena word the layout declares, the caller's +/// single `Unpack` of the root (shared by every query against it, which is why +/// the opening form does not charge it), and the weight closure — +/// `emit_eq_eval(z, alphas)` is the caller's `W`, and `chain_fixed_rows` +/// deliberately counts only the out-of-domain `eq`s. The closure's interned `1` +/// is a `Const` row and is subtracted with the other constants, so the +/// second-leg form is the right one here. +fn chain_plumbing(shape: &ChainShape) -> usize { + Layout::new(shape).total as usize + 1 + eq_eval_rows_again(shape.num_vars) +} + +/// ★ GATE ONE for the schedule: the emitter's hash schedule is the HOST's. +/// +/// `chain_hash_schedule` is derived from the round structure — the absorbs, the +/// draws, the `state()` reads, the candidates a squeeze hands out. The host's +/// is reconstructed by [`HostDuplex`] from the calls a real `whir_chain::verify` +/// makes on a real proof, and checked against the values that verify received. +/// Two derivations that share no code, compared event for event and felt count +/// for felt count. +/// +/// This is the half the previous instance could not write, and the reason its +/// row form was not quotable: a sponge's cost is what its BUFFER holds at each +/// hash, which is a running quantity and not a shape. +#[test] +fn the_schedule_is_the_host_transcripts() { + for (cfg, num_vars) in cost_configs() { + let (num_queries, grind) = (cfg.num_queries, cfg.grind.query); + let f = fixture_with(&cfg, num_vars); + let host = f.recorded.duplex.borrow().hashes.clone(); + let mine = chain_hash_schedule(&f.shape, SpongeEntry::fresh()); + + let squeezes = mine + .iter() + .filter(|h| matches!(h, SpongeHash::Squeeze(_))) + .count(); + let states = mine.len() - squeezes; + println!( + "schedule S={num_vars} Q={num_queries} grind={grind} {:?}: {squeezes} squeezes, \ + {states} state reads, {} rows, {} permutations", + f.shape.schedule, + chain_schedule_rows(&f.shape, SpongeEntry::fresh()), + chain_schedule_perms(&f.shape, SpongeEntry::fresh()), + ); + + assert_eq!( + mine.len(), + host.len(), + "S={num_vars} Q={num_queries} grind={grind}: the host performs {} transcript \ + hashes and the form says {}", + host.len(), + mine.len() + ); + for (i, (derived, observed)) in mine.iter().zip(&host).enumerate() { + assert_eq!( + derived, + observed, + "S={num_vars} Q={num_queries} grind={grind}: hash {i} of {} — the form says \ + {derived:?} and the host's own transcript did {observed:?}", + host.len() + ); + } + + // A state read happens exactly where a grind is spent, and nowhere + // else: `3R − 1` at a real width, none at zero bits. + let expected_states = if grind == 0 { + 0 + } else { + 3 * f.shape.rounds() - 1 + }; + assert_eq!( + states, expected_states, + "state reads are one per grind check spent" + ); + } +} + +/// ★ GATE TWO: the chain's rows and permutations against the program it emits. +/// +/// Pinned separately and on purpose (the opening leg's reason, one level up): a +/// form that moved work between the sponge and the arithmetic at constant total +/// fails one of the two rather than neither. The permutation form has THREE +/// terms — the query phase's openings, two per grind check, and the schedule's +/// own — and only the first of them was pinned before this test. +#[test] +fn the_chain_emits_its_closed_form() { + for (cfg, num_vars) in cost_configs() { + let (num_queries, grind) = (cfg.num_queries, cfg.grind.query); + let f = fixture_with(&cfg, num_vars); + let program = chain_program(&f.shape); + let entry = SpongeEntry::fresh(); + + let hints = hint_rows(&program); + assert_eq!( + hints, + Layout::new(&f.shape).total as usize, + "every arena word is hinted exactly once — the plumbing subtraction below is only \ + honest while this holds" + ); + + let consts = const_rows(&program); + let measured = program.instrs.len() - consts - chain_plumbing(&f.shape); + let predicted = chain_rows(&f.shape, entry); + let perms = perm_rows(&program); + let predicted_perms = chain_perms(&f.shape, entry); + + println!( + "chain S={num_vars} Q={num_queries} grind={grind} {:?}: {measured} rows \ + ({} shape + {} schedule predicted {predicted}); {perms} permutations \ + ({} openings + {} grind + {} schedule predicted {predicted_perms}); \ + {consts} constants, {hints} hints, {} instructions", + f.shape.schedule, + chain_shape_rows(&f.shape), + chain_schedule_rows(&f.shape, entry), + chain_opening_perms(&f.shape), + chain_grind_perms(&f.shape), + chain_schedule_perms(&f.shape, entry), + program.instrs.len(), + ); + + assert_eq!( + measured, predicted, + "S={num_vars} Q={num_queries} grind={grind}: rows" + ); + assert_eq!( + perms, predicted_perms, + "S={num_vars} Q={num_queries} grind={grind}: permutations" + ); + } +} + +/// ★ What a chain is ENTERED with reaches its first SQUEEZE, and stops there. +/// +/// The property the census rests on: a chain inside an assembled verifier finds +/// the sponge holding whatever the statement and the tables before it left, and +/// that could in principle change every later buffer length. It does not — but +/// the boundary is not where this test first claimed it was. +/// +/// ⚠ It was written as "the first HASH carries the entering buffer and nothing +/// after it does", and it failed: a `state()` hashes the buffer WITHOUT +/// clearing it (`default_transcript.rs:236-239`), so a grind's state read is a +/// hash that passes the entering felts straight through to the squeeze behind +/// it. The boundary is the first SQUEEZE, which is the first event that resets +/// the buffer to the digest. Recorded rather than quietly corrected, because +/// the wrong version is the one a reader would assume. +/// +/// The entering `out_pos` is never read at all: the first draw of the first +/// round always follows an absorb, and an absorb invalidates the output buffer. +#[test] +fn what_a_chain_is_entered_with_reaches_its_first_squeeze_and_stops() { + let shape = ChainShape::new(&config(4, 8), 6); + let fresh = chain_hash_schedule(&shape, SpongeEntry::fresh()); + let first_squeeze = fresh + .iter() + .position(|h| matches!(h, SpongeHash::Squeeze(_))) + .expect("a chain squeezes"); + + for (buffered, out_pos) in [(0, 0), (3, 2), (7, 4), (12, 1)] { + let entry = SpongeEntry { + buffered_felts: buffered, + out_pos, + p1: None, + }; + let entered = chain_hash_schedule(&shape, entry); + assert_eq!(entered.len(), fresh.len(), "the event count cannot move"); + for i in 0..=first_squeeze { + assert_eq!( + entered[i].felts(), + fresh[i].felts() + buffered, + "hash {i} is at or before the first squeeze, so it carries the entering buffer" + ); + assert_eq!( + core::mem::discriminant(&entered[i]), + core::mem::discriminant(&fresh[i]), + "hash {i} is the same KIND either way" + ); + } + assert_eq!( + &entered[first_squeeze + 1..], + &fresh[first_squeeze + 1..], + "every hash after the first squeeze is identical — that squeeze reset the buffer \ + to the digest" + ); + } + + // The entering `out_pos` changes nothing: at one buffer length, every + // position gives the same schedule. + for out_pos in 0..=4 { + assert_eq!( + chain_hash_schedule( + &shape, + SpongeEntry { + buffered_felts: 5, + out_pos, + p1: None, + } + ), + chain_hash_schedule( + &shape, + SpongeEntry { + buffered_felts: 5, + out_pos: 0, + p1: None, + } + ), + "a chain never draws before it absorbs, so what is in hand on entry is dropped" + ); + } +} + +/// ★ The production chain's cost, whole: the number the epoch census adds up. +/// +/// The forms are gated at [`COST_SHAPES`]; this evaluates them at the shape the +/// campaign runs and pins the answer, so a change to any term has to be +/// restated here before it can be quoted. Every number was derived by hand in +/// the design note before this test was written, which is what makes the first +/// run of it a measurement rather than a transcription. +/// +/// The census quoted 184,673 shape rows under the fold the emitter wrote then, +/// seven rows a folded value. The lean fold spends 52 fewer a sixteen-value +/// fold (`7·15 − 1` against `3·15 + 2·4 − 1`), 2 fewer a two-value one, and one +/// more a level a round for its halved challenges: `184,673 − 6·112·52 − +/// 112·2 + (6·4 + 1) = 149,530` shape rows, and 150,366 with the schedule's +/// 836. +/// +/// ⚠ The sizing note's per-chain figures are NOT these. Its 22,830 permutations +/// predate both the grind term and the schedule term; the 318 it implies for the +/// transcript is 42 above the 276 the schedule actually costs, and was never +/// re-derived. Quote these. +#[test] +fn the_production_chain_costs_what_the_census_quotes() { + // `multilinear_prove.rs:93`: blowup 2, fold 4, 128 bits, uniform 20-bit + // grinds — the config the block is proven under. + let shape = ChainShape::new(&config(112, 20), 25); + let entry = SpongeEntry::fresh(); + let schedule = chain_hash_schedule(&shape, entry); + let squeezes = schedule + .iter() + .filter(|h| matches!(h, SpongeHash::Squeeze(_))) + .count(); + + assert_eq!(squeezes, 233, "squeezes a chain"); + assert_eq!(schedule.len() - squeezes, 20, "state reads = 3R − 1 grinds"); + assert_eq!(chain_schedule_rows(&shape, entry), 836, "schedule rows"); + assert_eq!(chain_schedule_perms(&shape, entry), 276, "schedule perms"); + assert_eq!(chain_grind_perms(&shape), 40, "two permutations a grind"); + assert_eq!(chain_shape_rows(&shape), 149_530, "shape rows"); + + println!( + "production chain S=25 k=4 Q=112 grind=20: {} rows, {} permutations", + chain_rows(&shape, entry), + chain_perms(&shape, entry) + ); + assert_eq!(chain_rows(&shape, entry), 150_366, "rows a chain"); + assert_eq!(chain_perms(&shape, entry), 22_828, "permutations a chain"); +} + +/// ⛔ THE GENESIS THRESHOLD'S BUDGET, ASSERTED WHERE IT CAN BE COMPUTED. +/// +/// `crate::continuation::PREPARED_LEG_ROWS` is PART 2's term: the rows the +/// stacked chain costs ONCE, which the candidate set's total savings must +/// exceed before any page is carried. It is a CONSTANT there rather than a +/// call, because that module sits below `crate::lfm` — a routing rule the +/// prover, the verifier and the emitter must all agree on cannot depend on the +/// emitter's row accounting. This is the assertion that pays for the constant, +/// in the one place the form exists. +/// +/// ⚠ IT IS NOT WHAT A PAGE IS CHARGED. What one more carried page adds to the +/// leg is `continuation::marginal_stacked_rows`, a form evaluated at the run's +/// own bracket whose pin lives in `whir_stacked_tests`. The two terms are separate because the +/// costs are: one chain, however many pages ride it. +/// +/// ★ WHAT THE CONSTANT IS: 175,066 rows, the chain over a stacked family +/// polynomial at 24 variables. The block's stack is THREE columns of 2^18 — 20 +/// variables — so it is charged at a figure larger than it can cost, which is +/// the conservative direction: a page must be worth more than the stack could +/// possibly cost before it joins. +/// +/// ⚠ CONFIGURATION IS PART OF THE NUMBER. Chain rows move with blowup, folding +/// and the query count, so this asserts at the shape the campaign's production +/// figures are quoted at — the same `config(112, 20)` the chain's own F1 above +/// uses. A posture change reddens this rather than silently retuning a routing +/// rule nobody is looking at. +/// +/// ⛔ THE CONSTANT STAYS FIXED: a routing rule the prover, the verifier and the +/// emitter agree on, not a figure to re-tune with the emitter. It was read off +/// the 24-variable chain under the fold the emitter wrote then, at seven rows a +/// folded value; the lean fold's chain costs fewer, so the constant now bounds +/// the 24-variable chain rather than equalling it — the conservative direction. +/// It is pinned by value, so a drift either way reddens here. +#[test] +fn the_genesis_threshold_budget_is_in_band_at_the_production_shape() { + let at = |vars| chain_shape_rows(&ChainShape::new(&config(112, 20), vars)); + let (at_20, at_24) = (at(20), at(24)); + let budget = crate::continuation::PREPARED_LEG_ROWS; + println!( + "GENESIS BUDGET: {budget} rows against a chain of {at_20} at 20 variables and \ + {at_24} at 24, Q=112 grind=20 blowup=2 fold=4" + ); + assert_eq!(budget, 175_066, "the routing constant is fixed"); + assert!( + at_20 < at_24, + "a stack of fewer variables must cost fewer chain rows, or the budget's \ + conservatism argument does not hold ({at_20} at 20, {at_24} at 24)" + ); + assert!( + budget >= at_20, + "the threshold charges {budget} rows for a stack that costs {at_20} at the \ + block's 20 variables: pages would be left sparse that the opening could carry" + ); + assert!( + budget >= at_24, + "the threshold charges {budget} rows, under the {at_24} the 24-variable chain \ + it bounds costs" + ); +} + +/// ★ AND THE ROUTING DECISION DOES NOT SIT NEAR THAT BAND. +/// +/// The block's three dense pages cost 2,100,474, 4,127,238 and 4,020,858 rows +/// by the sparse form, and its 27 zero pages cost 18 each. Whatever the chain +/// figure moves to within any plausible posture, the same three pages are +/// selected — which is the argument that this is a routing rule rather than a +/// tuning knob, made against the cost form rather than against the constant. +#[test] +fn the_blocks_genesis_routing_is_insensitive_to_the_chain_figure() { + let cheapest_dense = 2_100_474usize; + let dearest_sparse = 18usize; + for stack_vars in 18..=25 { + let rows = chain_shape_rows(&ChainShape::new(&config(112, 20), stack_vars)); + assert!( + dearest_sparse <= rows && rows < cheapest_dense, + "at {stack_vars} stacked variables the chain costs {rows} rows, which falls \ + outside ({dearest_sparse}, {cheapest_dense}) — the block's selection would \ + change with the posture and the pre-registered three pages are no longer \ + a property of the run" + ); + } +} + +/// ⛔⛔ THE THRESHOLD AND THE SPARSE CAP MUST NOT DISAGREE, OR THE PROGRAM +/// CANNOT BE BUILT AT ALL. +/// +/// Two independent rules decide what happens to a genesis page. The two-part +/// routing rule in `crate::continuation` decides whether the PREPARED OPENING +/// carries it; [`super::preprocessed::MAX_SPARSE_INIT_ENTRIES`] decides whether +/// the sparse leg is willing to EMIT it, and refuses above the cap because a +/// column that dense has no cheap closed form. +/// +/// If the threshold ever left a page sparse that the cap then refused, the page +/// would have no route at all: too dense to emit, not dense enough to stack, +/// and the emit-time refusal would fire on a program nobody could fix by +/// changing either constant alone. The two rules must therefore overlap, with +/// the threshold strictly the tighter one. +/// +/// ⛔⛔ AND THE BOUND IS RE-DERIVED FROM THE LIVE RULE, NOT CARRIED OVER. Under +/// the retired single-page rule the densest page left sparse carried 9,724 +/// entries; under the two-part rule it carries 9,730, because a page is now +/// charged its own marginal on top of the chain. ⚠ BOTH ARE `<= 60,000`, so a +/// test left at the old number stays GREEN while measuring a rule that no +/// longer exists — which is the only reason this is worth saying twice. +/// [`crate::continuation::densest_sparse_entries`] is where the quantity now +/// lives, so it moves when the rule does. +/// +/// ★ THIS IS THE STATE THE BLOCK WAS ACTUALLY IN. V1j's block bundle arm +/// refused at that cap — "these preprocessed columns carry 116692 nonzero +/// entries ... the cap is 60000 entries" for page `0x0` — because the prepared +/// route did not exist yet and every genesis page went to the sparse leg. Once +/// the threshold routes the dense pages to the opening, no page reaching the +/// sparse leg can be within six times the cap, and ⛔ THE CAP SHOULD NEVER FIRE +/// AGAIN. A refusal from it after this lands is not a page that needs a bigger +/// cap; it is these two constants having drifted apart. +#[test] +fn every_page_the_threshold_leaves_sparse_is_one_the_sparse_leg_will_emit() { + use crate::continuation::{ + BLOCK_STACK_VARS, PAGE_NUM_VARS, PREPARED_LEG_ROWS, chain_is_paid, densest_sparse_entries, + fixed_stack_vars, marginal_stacked_rows, page_savings, sparse_leg_rows, + }; + let num_vars = PAGE_NUM_VARS; + let cap = super::preprocessed::MAX_SPARSE_INIT_ENTRIES; + // ⚠ AT THE BLOCK'S BRACKET, WHICH IS NOW PART OF THE QUESTION. The bound + // grows with the run's genesis page count — one entry per `num_vars` rows + // of marginal — so a bound quoted without its bracket is a bound for one + // run. The sweep below covers the rest. + let marginal = marginal_stacked_rows(num_vars, BLOCK_STACK_VARS); + let densest_sparse = densest_sparse_entries(num_vars, BLOCK_STACK_VARS); + + // ⛔ BOTH CONTRASTS WITH THE RETIRED RULE, because they are different + // quantities and both have been quoted. Its BREAK-EVEN (the least S it + // carried) was 9,725 against this rule's 9,732 at the block's bracket; its + // DENSEST SPARSE page carried 9,724 against this rule's 9,731. `<= 60,000` + // is true of all four, which is exactly why a test left at either old + // number stays green while measuring a rule that no longer exists. + let retired_break_even = (PREPARED_LEG_ROWS - num_vars) / num_vars + 1; + println!( + "ROUTE OVERLAP: at the block's bracket {BLOCK_STACK_VARS} (marginal {marginal}) \ + the densest page the rule can leave sparse carries {densest_sparse} entries \ + and the least it carries is {}, against a sparse-leg cap of {cap}. A LONE \ + page's bracket gives {}; the retired single-page rule's pair was {} and \ + {retired_break_even}.", + densest_sparse + 1, + densest_sparse_entries(num_vars, fixed_stack_vars(num_vars, 1)), + retired_break_even - 1, + ); + assert_eq!(retired_break_even, 9_725); + assert_eq!( + densest_sparse, 9_731, + "the bound the cap must clear is the two-part rule's AT THE BLOCK'S BRACKET; a \ + lone page's bracket gives 9,730 and the retired single-page rule's pair was \ + 9,724 / 9,725 — all four clear the cap, which is exactly why this must be \ + re-derived rather than re-read" + ); + + // ⚠ EXACT, BOTH WAYS — a bound asserted only from above could be any number + // larger than the truth. The page at the bound must route sparse and the one + // above it must not, or `densest_sparse_entries` is inverting the wrong form. + assert!(!chain_is_paid(page_savings( + num_vars, + BLOCK_STACK_VARS, + densest_sparse + ))); + assert!(chain_is_paid(page_savings( + num_vars, + BLOCK_STACK_VARS, + densest_sparse + 1 + ))); + + assert!( + densest_sparse <= cap, + "a page carrying {densest_sparse} entries routes SPARSE and is then REFUSED by \ + the {cap}-entry cap: it has no route at all, and neither constant can be fixed \ + without the other" + ); + + // ⛔ AND THE MARGIN MUST NOT CLOSE WHEN THE LITERAL MOVES. The marginal is + // a form of the run's own bracket, and the bound rises one entry per + // `num_vars` rows as a run stands taller. The overlap is asserted over every + // marginal up to four orders past the block's, which is where the + // arithmetic says it would finally close: + // `(PREPARED_LEG_ROWS + m - num_vars) / num_vars <= cap` fails above + // `m = 904,952`. ⚠ A run would need more genesis pages than there are + // addresses to reach that, so the sweep is deliberately far past anything + // reachable. + let mut m = marginal; + while m <= 900_000 { + let bound = (PREPARED_LEG_ROWS + m - num_vars) / num_vars; + assert!( + bound <= cap, + "at marginal {m} the rule can leave a page of {bound} entries sparse, and \ + the sparse leg refuses above {cap}" + ); + // The sparse leg it would then emit must be a leg, not a claim. + assert!(sparse_leg_rows(num_vars, bound) > 0); + m = (m * 2).max(m + 1); + } + // ⚠ AND THE CHECK CAN FAIL: one marginal past that point must breach the + // cap, or the sweep above proves nothing about where the margin closes. + assert!((PREPARED_LEG_ROWS + 1_000_000 - num_vars) / num_vars > cap); +} + +/// ⛔⛔ A LOWER BOUND ON PART 1's MARGINAL, READ OFF THE EMITTER — AND +/// DELIBERATELY NOT A PIN. +/// +/// `crate::continuation::marginal_stacked_rows` is a FORM whose last term is a +/// BOUND, and its pin — the form read against the differenced +/// `stacked_verify_cost` — lives in `whir_stacked_tests` and belongs to the lane +/// that owns that cost form. +/// +/// ⛔ THIS TEST MUST NOT ASSERT THAT EQUALITY, and the reason is the whole +/// point of a pin. [`super::whir_stacked::weight_at_rows`] is a PARTIAL view: +/// it carries the `eq` and the prefix indicators, while `stacked_verify_cost` +/// pays three more terms per column outside it — and a fourth, an absorb into a +/// THREADED sponge whose row cost depends on where the previous columns left +/// the buffer. A second equality here, against the partial form, would +/// CONTRADICT the real pin the day it lands, and two pins on one constant is +/// precisely the duplication a pin exists to prevent. +/// +/// ★ WHAT IT DOES ASSERT, and it is worth having: the two terms +/// `weight_at_rows` DOES account for, differenced across one more page inside +/// ONE power-of-two bracket so the stack height does not move — `eq` 90 plus +/// two indicators of six — and that the literal is at least that plus one. An +/// inequality cannot conflict with a measurement. +/// +/// ⚠ THE `+ 1` IS THE SHARED `Sub`, AND DIFFERENCING IS WHAT SHOWS IT IS NOT A +/// MARGINAL. `weight_at_rows` emits one per prefix POSITION any column reads as +/// a zero bit, which is per-POLYNOMIAL: across 29 and 30 pages at one height +/// the term contributes ZERO, which is why `billed` is 102 and not 103. The +/// literal charges it anyway, so `>= billed + 1` is exactly the statement that +/// the literal contains every term this form can see. +#[test] +fn the_marginal_is_at_least_what_the_weight_term_bills() { + use super::whir_stacked::weight_at_rows; + use crate::continuation::{ + BLOCK_STACK_VARS, MAX_SPONGE_MARGINAL, PAGE_NUM_VARS, PAGE_PREPROCESSED_COLUMNS, + STACKED_ROWS_PER_COLUMN, marginal_stacked_rows, + }; + let marginal = marginal_stacked_rows(PAGE_NUM_VARS, BLOCK_STACK_VARS); + + // A page's two preprocessed columns settle at ONE point — its table's — so + // the groups are the pages. + let layout_for = |pages: usize| { + let columns = pages * PAGE_PREPROCESSED_COLUMNS; + let layout = stark::multilinear_table::global_layout( + &[(columns, PAGE_NUM_VARS)], + crate::zf_format::ZfFormat::DEFAULT.whir_stack, + ) + .expect("a stack of whole pages"); + let group_of: Vec = (0..columns) + .map(|column| column / PAGE_PREPROCESSED_COLUMNS) + .collect(); + (layout, group_of) + }; + + // 29 and 30 pages both stand at the block's bracket, and in ONE polynomial + // — or the difference would span two chains. + let (at_29, groups_29) = layout_for(29); + let (at_30, groups_30) = layout_for(30); + assert_eq!(at_29.n_stack(), BLOCK_STACK_VARS); + assert_eq!(at_30.n_stack(), BLOCK_STACK_VARS); + assert_eq!(at_30.num_polys(), 1, "or the difference spans two chains"); + + let rows_29 = weight_at_rows(&at_29, 0, &groups_29); + let rows_30 = weight_at_rows(&at_30, 0, &groups_30); + let billed = rows_30 - rows_29; + println!( + "MARGINAL: the weight closure alone bills {billed} rows for the thirtieth page \ + ({rows_29} -> {rows_30} at {} variables); the form charges {marginal}", + at_30.n_stack() + ); + + // The two terms this form accounts for, to the row. + assert_eq!( + billed, + 5 * PAGE_NUM_VARS + PAGE_PREPROCESSED_COLUMNS * (BLOCK_STACK_VARS - PAGE_NUM_VARS) + ); + assert_eq!(billed, 102); + + // ⛔ THE GAP, DECOMPOSED — a statement about which terms the form CONTAINS, + // never about what the stack bills. Everything `weight_at_rows` cannot see + // is the per-column rows and the sponge bound, and the shared `Sub` is in + // neither number: it is per-POLYNOMIAL and differences to zero. + assert_eq!( + marginal - billed, + PAGE_PREPROCESSED_COLUMNS * STACKED_ROWS_PER_COLUMN + MAX_SPONGE_MARGINAL, + "the form charges {marginal} where the weight closure bills {billed}; the gap \ + must be exactly the per-column rows plus the sponge bound" + ); + assert!( + marginal > billed, + "the form charges {marginal} rows for a page the weight closure alone bills \ + {billed} for: it would be below a term it must contain" + ); +} + +/// ⚠ A LIMIT OF PART 2's SINGLE CHAIN TERM, STATED WHERE IT CAN BE MEASURED. +/// +/// [`crate::continuation::PREPARED_LEG_ROWS`] prices ONE chain, and one chain +/// is what the stack costs while its columns fit one polynomial. +/// `global_layout` caps a stack at the production format's stack (`whir_stack`) +/// and spills the rest into another polynomial, each of which runs its own +/// chain — so past that width part 2 is weighing a set against a fraction of +/// what carrying it costs. +/// +/// ⛔ THIS IS A PRICING LIMIT AND NOT A SOUNDNESS ONE. All three parties still +/// reach the same set from the same data; the rule would simply buy pages whose +/// keep it had under-quoted. It is recorded rather than guarded because the +/// block sits 85x under it and no run this prover has seen comes near — and if +/// one ever does, the fix is a chain term that counts polynomials, not a bigger +/// constant. +/// +/// The limit is the stack's: 64 pages at the legacy 25, 256 at the production +/// format's 27, because `2 * pages * 2^18` cells must fit one polynomial. +#[test] +fn the_single_chain_term_prices_a_stack_of_at_most_two_hundred_fifty_six_pages() { + use crate::continuation::{PAGE_NUM_VARS, PAGE_PREPROCESSED_COLUMNS}; + let per_page = PAGE_PREPROCESSED_COLUMNS; + let max_stack_vars = crate::zf_format::ZfFormat::DEFAULT.whir_stack.get(); + + let polys_at = |pages: usize| { + stark::multilinear_table::global_layout( + &[(pages * per_page, PAGE_NUM_VARS)], + crate::zf_format::ZfFormat::DEFAULT.whir_stack, + ) + .expect("a stack of whole pages") + .num_polys() + }; + // `2 * pages * 2^18` cells must fit `2^27`. + let widest = 1 << (max_stack_vars - PAGE_NUM_VARS - 1); + assert_eq!(widest, 256); + assert_eq!(polys_at(widest), 1, "the last width that is one chain"); + assert!( + polys_at(widest + 1) > 1, + "one page past it must spill, or this boundary is not where it is claimed" + ); + // The block carries three. + assert_eq!(polys_at(3), 1); +} + +// ============================================================================= +// W1: the chain under a Merkle cap +// ============================================================================= + +/// The policies the capped gates run under. At `Q = 3` `Auto` caps tree 0 not +/// at all (3 openings) and every later tree at 2 (6 openings) — a chain with an +/// uncapped owner round and capped successors, the mixed case. +const CAP_POLICIES: [CapPolicy; 3] = [CapPolicy::Fixed(1), CapPolicy::Fixed(2), CapPolicy::Auto]; + +/// ★ The capped chain executes on a proof the host accepts, at every policy, +/// one- and three-round shapes, and `Q` large enough for `Auto` to cap the +/// first tree at 3. +#[test] +fn a_capped_chain_executes_on_a_proof_the_host_accepts() { + for (num_vars, num_queries) in [(6usize, 3usize), (5, 3), (9, 3), (6, 25), (9, 25)] { + for cap in CAP_POLICIES { + let f = fixture_capped(num_vars, num_queries, 0, cap); + assert_eq!( + f.shape.caps, + config_with(num_queries, 0, cap).tree_caps(num_vars), + "the shape's caps are the host's" + ); + assert!( + f.shape.caps.iter().any(|&c| c > 0), + "{cap}: something is capped" + ); + let program = chain_program(&f.shape); + let arena = chain_arena(&f, &f.proof); + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!( + "S={num_vars} Q={num_queries} {cap} caps {:?}: the machine refused an \ + accepted proof: {e:?}", + f.shape.caps + ) + }); + } + } +} + +/// ★ GATE TWO under the cap: emitted rows and permutations against the closed +/// forms, which now carry the cap terms. +#[test] +fn a_capped_chain_emits_its_closed_form() { + for (num_vars, num_queries, grind) in COST_SHAPES.into_iter().chain([(9, 25, 0)]) { + for cap in [CapPolicy::Fixed(2), CapPolicy::Fixed(3), CapPolicy::Auto] { + let shape = ChainShape::new(&config_with(num_queries, grind, cap), num_vars); + let program = chain_program(&shape); + let entry = SpongeEntry::fresh(); + assert_eq!( + hint_rows(&program), + Layout::new(&shape).total as usize, + "every arena word, cap words included, is hinted exactly once" + ); + let measured = program.instrs.len() - const_rows(&program) - chain_plumbing(&shape); + let tag = format!( + "S={num_vars} Q={num_queries} grind={grind} {cap} {:?}", + shape.caps + ); + assert_eq!(measured, chain_rows(&shape, entry), "{tag}: rows"); + assert_eq!( + perm_rows(&program), + chain_perms(&shape, entry), + "{tag}: permutations" + ); + } + } +} + +/// ★ The transcript does not move with the cap: the host's hash schedule on a +/// capped proof is the same form the default follows. +#[test] +fn the_schedule_is_the_host_transcripts_under_the_cap() { + for (num_vars, num_queries, grind) in COST_SHAPES { + let f = fixture_capped(num_vars, num_queries, grind, CapPolicy::Auto); + let host = f.recorded.duplex.borrow().hashes.clone(); + assert_eq!( + chain_hash_schedule(&f.shape, SpongeEntry::fresh()), + host, + "S={num_vars} Q={num_queries} grind={grind}" + ); + } +} + +/// ★ The tamper arm under the cap: a cap node of tree 0 that NO query reaches +/// (so only the in-guest cap-to-root check can refuse it), +/// a reached one, and a successor tree's cap node. Each: the host rejects it +/// and the machine has no execution. +#[test] +fn a_tampered_capped_chain_cannot_execute() { + // S = 6, k = 4: schedule [4, 2], trees of depth 4 and 2. Fixed(3): caps + // [3, 2] — tree 0 has eight cap nodes and three queries, so at least five + // are unreached. + let cap = CapPolicy::Fixed(3); + let f = fixture_capped(6, 3, 0, cap); + assert_eq!(f.shape.caps, vec![3, 2]); + let program = chain_program(&f.shape); + assert!( + execute( + &program, + &[chain_arena(&f, &f.proof)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the untouched proof must execute, or the arm proves nothing" + ); + let cfg = config_with(3, 0, cap); + let host_rejects = |proof: &ChainProof| -> bool { + verify::( + proof, + &f.root_bytes, + &f.z, + f.y, + &f.domain, + &cfg, + &mut Recording::new(), + ) + .is_err() + }; + + // Round 0's query positions, as the host drew them: the leaf is the + // position itself, its cap node the top three of its four bits. + let reached: Vec = f.recorded.drawn_u64[..3].iter().map(|q| q >> 1).collect(); + let unreached = (0..8u64).find(|j| !reached.contains(j)).unwrap() as usize; + let reached = reached[0] as usize; + let depth0 = f.shape.current_depth(0); + + let mut sites: Vec<(String, ChainProof)> = Vec::new(); + for (name, j) in [("unreached", unreached), ("reached", reached)] { + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => p.current[0].proof.merkle_path[depth0 - 3 + j][9] ^= 1, + RoundOpenings::Extension(_) => unreachable!("round 0 is base"), + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push((format!("tree-0 cap node {j} ({name})"), forged)); + } + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => { + let path = &mut p.next[0].proof.merkle_path; + let last = path.len() - 1; + path[last][0] ^= 1; + } + RoundOpenings::Extension(_) => unreachable!("round 0 is base"), + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push(("tree-1 cap node 3".to_string(), forged)); + + for (name, forged) in &sites { + assert!( + host_rejects(forged), + "{name}: the host must reject the forgery" + ); + assert!( + execute( + &program, + &[chain_arena(&f, forged)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "{name}: the machine must refuse the forgery" + ); + } +} + +/// ★ The production chain under `Auto`, evaluated — the knob-on twin of +/// [`the_production_chain_costs_what_the_census_quotes`] and +/// [`the_production_shape_reproduces_the_campaigns_permutation_count`]. +/// +/// Hand derivation: trees of depth 23, 19, 15, 11, 7, 3, 2 +/// opened 112, then 224 times each, capped 3, 3, 3, 3, 3, 3, 2. Openings save +/// `112·3 + 5·224·3 + 224·2 = 4,144` parents; the caps cost `6·7 + 3 = 45`: +/// 22,512 → 18,413 opening permutations, 22,828 → 18,729 in all. Rows: `+2` +/// an opening at `c = 3` (`7 − 6 + 1`), `0` at `c = 2` (`3 − 4 + 1`), so +/// `2·(112 + 5·224) = 2,464`, plus the cap checks `6·16 + 12 = 108`: 2,572 +/// more shape rows than the uncapped chain, and as many more in all (the +/// schedule does not move, and nothing the cap changes touches the fold). +#[test] +fn the_production_chain_under_the_auto_cap_costs_its_hand_derivation() { + let shape = ChainShape::new(&config_with(112, 20, CapPolicy::Auto), 25); + let uncapped = ChainShape::new(&config(112, 20), 25); + assert_eq!(shape.caps, vec![3, 3, 3, 3, 3, 3, 2]); + let entry = SpongeEntry::fresh(); + assert_eq!(chain_cap_perms(&shape), 45, "cap permutations"); + assert_eq!(chain_opening_perms(&shape), 18_413, "opening permutations"); + assert_eq!( + chain_schedule_rows(&shape, entry), + 836, + "schedule rows unmoved" + ); + assert_eq!( + chain_schedule_perms(&shape, entry), + 276, + "schedule perms unmoved" + ); + assert_eq!(chain_grind_perms(&shape), 40); + assert_eq!( + chain_shape_rows(&shape), + chain_shape_rows(&uncapped) + 2_572, + "shape rows" + ); + assert_eq!( + chain_rows(&shape, entry), + chain_rows(&uncapped, entry) + 2_572, + "rows a chain" + ); + assert_eq!(chain_perms(&shape, entry), 18_729, "permutations a chain"); + println!( + "production chain S=25 k=4 Q=112 grind=20 cap=auto: {} rows, {} permutations", + chain_rows(&shape, entry), + chain_perms(&shape, entry) + ); +} + +/// ⛔ `PREPARED_LEG_ROWS` is a ROUTING constant and stays fixed +/// across formats. Under the `Auto` cap a chain costs slightly more rows (+2 an +/// opening at `c = 3`, plus the cap checks), and the constant still covers the +/// block's 20-variable stack and bounds the 24-variable chain. +#[test] +fn the_genesis_threshold_budget_still_bounds_the_chain_under_the_auto_cap() { + let auto = |vars| { + chain_shape_rows(&ChainShape::new( + &config_with(112, 20, CapPolicy::Auto), + vars, + )) + }; + let (at_20, at_24) = (auto(20), auto(24)); + let budget = crate::continuation::PREPARED_LEG_ROWS; + println!( + "GENESIS BUDGET cap=auto: {budget} rows against {at_20} at 20 variables, {at_24} at 24" + ); + assert!( + budget >= at_20, + "the budget must still cover the 20-variable stack" + ); + assert!( + budget >= at_24, + "the budget must still bound the 24-variable chain ({at_24})" + ); +} + +// --------------------------------------------------------------------------- +// W2: the first-fold schedules (`LAMBDA_VM_ZF_WHIR_FOLDS=first5 | first6`). +// --------------------------------------------------------------------------- + +/// ★ A first-fold chain executes on a proof the host accepts — the stream +/// comparison of [`the_chain_executes_on_a_proof_the_host_accepts`], with the +/// round-0 block 32 or 64 base values wide. +#[test] +fn a_first_fold_chain_executes_on_a_proof_the_host_accepts() { + for (num_vars, num_queries, grind, k0) in KNOB_COST_SHAPES { + let cfg = first_fold_config(num_queries, grind, k0); + let f = fixture_with(&cfg, num_vars); + assert_eq!(f.shape.schedule, cfg.schedule(num_vars)); + assert_eq!(f.shape.schedule[0], k0.min(num_vars)); + assert_eq!(f.shape.current_felts(0), 1 << k0.min(num_vars)); + let program = chain_program(&f.shape); + execute( + &program, + &[chain_arena(&f, &f.proof)], + &crate::hash_pin::BLOCK_HASHER, + ) + .unwrap_or_else(|e| { + panic!( + "S={num_vars} first{k0} {:?}: the machine refused an accepted proof: {e:?}", + f.shape.schedule + ) + }); + } +} + +/// ★ The tamper arm on the wide base block: the LAST value of round 0's +/// 64-value block and one of its Merkle siblings. The host must reject each +/// forgery (so the refusal is of something invalid) and the machine must +/// refuse it. +#[test] +fn a_tampered_first_fold_chain_cannot_execute() { + let grind = 8u8; + let cfg = first_fold_config(3, grind, 6); + let f = fixture_with(&cfg, 9); + assert_eq!(f.shape.schedule, vec![6, 3]); + let program = chain_program(&f.shape); + assert!( + execute( + &program, + &[chain_arena(&f, &f.proof)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the untouched proof must execute, or the arm below proves nothing" + ); + + let host_rejects = |proof: &ChainProof| -> bool { + let mut t = Recording::new(); + verify::(proof, &f.root_bytes, &f.z, f.y, &f.domain, &cfg, &mut t) + .is_err() + }; + + let mut sites: Vec<(&str, ChainProof)> = Vec::new(); + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => { + assert_eq!(p.current[0].values.len(), 64, "round 0 opens 64 values"); + p.current[0].values[63] += FE::one(); + } + RoundOpenings::Extension(_) => panic!("round 0 is base"), + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push(("the last value of a 64-wide base block", forged)); + + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => p.current[0].proof.merkle_path[0][0] ^= 1, + RoundOpenings::Extension(_) => panic!("round 0 is base"), + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push(("a round-0 Merkle sibling", forged)); + + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => p.next[0].values[0] += FEE::one(), + RoundOpenings::Extension(_) => panic!("round 0 is base"), + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push(( + "the successor block round 0 checks its fold against", + forged, + )); + + for (name, forged) in &sites { + assert!( + host_rejects(forged), + "{name}: the host must reject the forgery" + ); + assert!( + execute( + &program, + &[chain_arena(&f, forged)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "{name}: the machine must refuse the forgery" + ); + } +} + +/// ★ The knob-on production pins, at `S = 25, Q = 112`, 20-bit grinds. +/// +/// Derived by hand first, off `verify_weighted`'s round structure, as +/// [`the_production_shape_reproduces_the_campaigns_permutation_count`] did: +/// +/// - `first6` `[6,4,4,4,4,3]`: domains 27/21/17/13/9/5; current depths +/// 21+17+13+9+5+2 = 67, successor depths 17+13+9+5+2 = 46, so 113 parents; +/// current leaves 8 (64 BASE felts) + 4×6 + 3 (the 24-felt tail) = 35 and +/// successor leaves 4×6 + 3 = 27, so 62 leaf blocks. 175 a query, 19,600 a +/// chain. +/// - `first5` `[5,4,4,4,4,4]`: domains 27/22/18/14/10/6; current depths +/// 22+18+14+10+6+2 = 72, successor 18+14+10+6+2 = 50, so 122 parents; leaves +/// 4 (32 base felts) + 5×6 + 5×6 = 64. 186 a query, 20,832 a chain. +/// +/// The whole-chain figures (grind + schedule terms) come from an independent +/// Python re-implementation of these forms, which reproduces today's 22,828 +/// permutations: first6 19,877, first5 21,109. R = 6 under both, so +/// `3R − 1 = 17` grinds, 34 permutations. The fold hashes nothing, so the +/// permutations do not depend on it. Its rows were derived under the fold the +/// emitter wrote then, seven rows a folded value — first6 201,318, first5 +/// 189,028 — and the lean fold's follow by the fold's difference a query +/// (`(7·(B − 1) − 1) − (3·(B − 1) + 2·log B − 1)`: 240 at 64 values, 114 at 32, +/// 52 at 16, 22 at 8), plus one row a level a round for the halved challenges: +/// first6 `201,318 − 112·(240 + 4·52 + 22) + 25 = 148,703`, first5 +/// `189,028 − 112·(114 + 5·52) + 25 = 147,165`. +#[test] +fn the_first_fold_production_chains_cost_what_the_design_derived() { + let entry = SpongeEntry::fresh(); + for (k0, schedule, parents, per_query, perms, rows) in [ + (6, vec![6, 4, 4, 4, 4, 3], 113, 175, 19_877, 148_703), + (5, vec![5, 4, 4, 4, 4, 4], 122, 186, 21_109, 147_165), + ] { + let shape = ChainShape::new(&first_fold_config(112, 20, k0), 25); + assert_eq!(shape.schedule, schedule, "first{k0}"); + let got_parents: usize = (0..shape.rounds()) + .map(|r| shape.current_depth(r) + shape.next_depth(r).unwrap_or(0)) + .sum(); + assert_eq!(got_parents, parents, "first{k0}: Merkle parents a query"); + assert_eq!( + shape.current_felts(0), + 1 << k0, + "first{k0}: round 0 is base" + ); + let opening = chain_opening_perms(&shape); + assert_eq!(opening, per_query * 112, "first{k0}: opening permutations"); + assert_eq!(chain_grind_perms(&shape), 34, "first{k0}: 17 grinds"); + println!( + "production chain S=25 first{k0} Q=112 grind=20: {opening} opening permutations, \ + {} permutations, {} rows ({} schedule perms, {} schedule rows)", + chain_perms(&shape, entry), + chain_rows(&shape, entry), + chain_schedule_perms(&shape, entry), + chain_schedule_rows(&shape, entry), + ); + assert_eq!( + chain_perms(&shape, entry), + perms, + "first{k0}: permutations a chain" + ); + assert_eq!(chain_rows(&shape, entry), rows, "first{k0}: rows a chain"); + } +} + +/// One production chain's pinned costs at `S = 25`: the auto caps per tree and +/// their permutations, the grinds' permutations, every permutation, the +/// arena's words, and the shape rows and rows a chain. +struct ChainPins { + caps: &'static [usize], + cap_perms: usize, + grind_perms: usize, + opening_perms: usize, + perms: usize, + words: u32, + rows: (usize, usize), +} + +/// ★ THE PRODUCTION DEFAULT CHAIN — what `chain_config` builds with no knob +/// set — is `first6` under the `Auto` cap, at blowup 2^1 and Q = 229, grinding +/// before its queries alone: P2's `whir_grind=query`, one 18-bit grind and one +/// nonce word a round. The legacy chain keeps its own pins above +/// (`the_production_chain_costs…`, 22,828 permutations). At blowup 2^2 +/// (`LAMBDA_VM_ZF_WHIR_BLOWUP=4`, Q 114) the chain keeps the pins the default +/// had before the blowup moved +/// ([`the_blowup_opt_out_chain_keeps_the_pins_from_before_the_flip`]); at 20 +/// bits there too (`LAMBDA_VM_ZF_WHIR_GRIND_BITS=20`, Q 112) the pins it had +/// before the bits moved ([`the_grind_bits_opt_out_chain_keeps_its_pins`]); and +/// the opt-out of all three (`whir_grind=all` at 20 bits) the pins it had +/// before P2 ([`the_grind_opt_out_chain_keeps_the_pins_from_before_p2`]). +/// +/// The format also carries the block base's 4-ary cap +/// ([`crate::hash_pin::BLOCK_BASE`]), under every grind knob: a 4-ary chain +/// alone reads it (`ChainFormat::cap_policy_at`). The shape and the pins of +/// these three tests are the binary chain's ([`ChainShape::new`]), which it +/// does not enter. +#[test] +fn the_production_default_chain_is_first6_under_the_auto_cap() { + let production = crate::multilinear_prove::chain_config_under( + &crate::zf_format::ZfFormat::DEFAULT, + &[(1, 25)], + ); + let want = ChainConfig { + grind: GrindBits::query_only(18), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).expect("6")), + stack: multilinear::whir_chain::StackVars::new(27).expect("27"), + nonces: NonceLayout::Spent, + argue: multilinear::whir_chain::ArgueFormat::PerTable, + arity4_cap: crate::hash_pin::BLOCK_BASE.arity4_cap, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaves: multilinear::whir_chain::LeafOrder::Natural, + }, + log_blowup: 1, + ..config(229, 0) + }; + assert_eq!(production, want, "the production default's chain config"); + let shape = ChainShape::new(&production, 25); + for r in 0..shape.rounds() { + assert_eq!(shape.carries(r), [false, false, true], "round {r}"); + } + check_production_chain("DEFAULT", &shape, &DEFAULT_CHAIN_PINS); +} + +/// ★ THE BLOWUP OPT-OUT'S CHAIN IS THE DEFAULT'S BEFORE THE FLIP. +/// `LAMBDA_VM_ZF_WHIR_BLOWUP=4` builds the chain at blowup 2^2 and Q 114, 18 +/// bits, with every cost the default had before its blowup moved to 2^1: these +/// pins are that default's, unmoved. +#[test] +fn the_blowup_opt_out_chain_keeps_the_pins_from_before_the_flip() { + let at4 = crate::multilinear_prove::chain_config_under( + &crate::zf_format::ZfFormat { + whir_blowup: 4, + ..crate::zf_format::ZfFormat::DEFAULT + }, + &[(1, 25)], + ); + let want = ChainConfig { + grind: GrindBits::query_only(18), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).expect("6")), + stack: multilinear::whir_chain::StackVars::new(27).expect("27"), + nonces: NonceLayout::Spent, + argue: multilinear::whir_chain::ArgueFormat::PerTable, + arity4_cap: crate::hash_pin::BLOCK_BASE.arity4_cap, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaves: multilinear::whir_chain::LeafOrder::Natural, + }, + ..config(114, 0) + }; + assert_eq!(at4, want, "the blowup-4 chain config"); + let shape = ChainShape::new(&at4, 25); + check_production_chain("BLOWUP-4", &shape, &BLOWUP_4_CHAIN_PINS); + // The two queries the 18-bit grind buys back cost the verifier their + // openings (2 × 144 permutations, plus the extra queries' schedule terms): + // the in-circuit price of two fewer grind bits on the prover. + const { + assert!(BLOWUP_4_CHAIN_PINS.perms > GRIND_BITS_20_CHAIN_PINS.perms); + assert!(BLOWUP_4_CHAIN_PINS.grind_perms == GRIND_BITS_20_CHAIN_PINS.grind_perms); + }; +} + +/// ★ THE GRIND-BITS OPT-OUT'S CHAIN IS THE DEFAULT'S BEFORE THE BITS MOVED. +/// `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20` at blowup 2^2 +/// (`LAMBDA_VM_ZF_WHIR_BLOWUP=4`, the rate the default had then) builds P2's +/// chain at 20 bits and Q 112, with every cost it had then: these pins are +/// that default's, unmoved. P2 and first6 under the auto cap both pay at this +/// Q: fewer permutations than either alone, and fewer than the same chain +/// grinding all three. +#[test] +fn the_grind_bits_opt_out_chain_keeps_its_pins() { + let at20 = crate::multilinear_prove::chain_config_under( + &crate::zf_format::ZfFormat { + whir_blowup: 4, + whir_grind_bits: crate::zf_format::LEGACY_WHIR_GRIND_BITS, + ..crate::zf_format::ZfFormat::DEFAULT + }, + &[(1, 25)], + ); + let want = ChainConfig { + grind: GrindBits::query_only(20), + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).expect("6")), + stack: multilinear::whir_chain::StackVars::new(27).expect("27"), + nonces: NonceLayout::Spent, + argue: multilinear::whir_chain::ArgueFormat::PerTable, + arity4_cap: crate::hash_pin::BLOCK_BASE.arity4_cap, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaves: multilinear::whir_chain::LeafOrder::Natural, + }, + ..config(112, 0) + }; + assert_eq!(at20, want, "the 20-bit chain config"); + let shape = ChainShape::new(&at20, 25); + check_production_chain("GRIND-BITS-20", &shape, &GRIND_BITS_20_CHAIN_PINS); + const { + assert!(GRIND_BITS_20_CHAIN_PINS.perms < 18_729 && GRIND_BITS_20_CHAIN_PINS.perms < 19_877); + assert!(GRIND_BITS_20_CHAIN_PINS.perms < GRIND_ALL_CHAIN_PINS.perms); + }; +} + +/// ★ THE OPT-OUT'S CHAIN IS TODAY'S. `LAMBDA_VM_ZF_WHIR_GRIND=all` with +/// `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20` at blowup 2^2 (`LAMBDA_VM_ZF_WHIR_BLOWUP=4`) +/// builds the production default's chain exactly as it was before P2, 20-bit +/// grinds before all three challenges and three nonce words a round, and every +/// cost it had then: these pins are the pre-P2 default's, unmoved. +#[test] +fn the_grind_opt_out_chain_keeps_the_pins_from_before_p2() { + let opt_out = crate::multilinear_prove::chain_config_under( + &crate::zf_format::ZfFormat { + whir_blowup: 4, + whir_grind: crate::zf_format::WhirGrind::All, + whir_grind_bits: crate::zf_format::LEGACY_WHIR_GRIND_BITS, + ..crate::zf_format::ZfFormat::DEFAULT + }, + &[(1, 25)], + ); + let want = ChainConfig { + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).expect("6")), + stack: multilinear::whir_chain::StackVars::new(27).expect("27"), + nonces: NonceLayout::Three, + argue: multilinear::whir_chain::ArgueFormat::PerTable, + arity4_cap: crate::hash_pin::BLOCK_BASE.arity4_cap, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaves: multilinear::whir_chain::LeafOrder::Natural, + }, + ..config(112, 20) + }; + assert_eq!(opt_out, want, "the opt-out's chain config"); + let shape = ChainShape::new(&opt_out, 25); + check_production_chain("GRIND-ALL", &shape, &GRIND_ALL_CHAIN_PINS); +} + +/// The closed forms of one production chain against its pins. +fn check_production_chain(label: &str, shape: &ChainShape, pins: &ChainPins) { + assert_eq!( + shape.schedule, + vec![6, 4, 4, 4, 4, 3], + "{label}: first6 at 25" + ); + let entry = SpongeEntry::fresh(); + println!( + "production {label} chain S=25 first6 cap=auto Q={} grind={:?}: caps {:?}, {} opening \ + permutations, {} cap permutations, {} grind permutations, {} permutations, {} words, \ + {} rows ({} shape rows)", + shape.num_queries, + shape.grind, + shape.caps, + chain_opening_perms(shape), + chain_cap_perms(shape), + chain_grind_perms(shape), + chain_perms(shape, entry), + RoundStorage::words(shape), + chain_rows(shape, entry), + chain_shape_rows(shape), + ); + assert_eq!( + chain_grind_perms(shape), + pins.grind_perms, + "{label}: grind permutations" + ); + assert_eq!( + chain_opening_perms(shape), + pins.opening_perms, + "{label}: opening permutations" + ); + assert_eq!( + chain_cap_perms(shape), + pins.cap_perms, + "{label}: cap permutations" + ); + assert_eq!(shape.caps, pins.caps, "{label}: the auto caps per tree"); + assert_eq!( + RoundStorage::words(shape), + pins.words, + "{label}: arena words" + ); + let (shape_rows, rows) = pins.rows; + assert_eq!(chain_shape_rows(shape), shape_rows, "{label}: shape rows"); + assert_eq!(chain_rows(shape, entry), rows, "{label}: rows a chain"); + assert_eq!( + chain_perms(shape, entry), + pins.perms, + "{label}: permutations a chain" + ); +} + +/// The production default chain's pins, derived by the closed forms and +/// checked against the EMITTED program by +/// [`the_production_chains_emit_their_closed_forms`]. P2's: one grind a round +/// (6 × 2 permutations), one nonce word a round; blowup 2^1 and Q 229 at 18 +/// grind bits, the last tree's auto cap one lower than at 2^2 (its domain a bit +/// smaller). +const DEFAULT_CHAIN_PINS: ChainPins = ChainPins { + caps: &[3, 3, 3, 3, 3, 1], + cap_perms: 36, + grind_perms: 12, + opening_perms: 30_951, + perms: 31_376, + words: 64_457, + rows: (296_872, 298_123), +}; + +/// The blowup opt-out's pins (blowup 2^2, Q 114 at 18 bits), which are the +/// production default's from before the blowup moved to 2^1. +const BLOWUP_4_CHAIN_PINS: ChainPins = ChainPins { + caps: &[3, 3, 3, 3, 3, 2], + cap_perms: 38, + grind_perms: 12, + opening_perms: 16_454, + perms: 16_705, + words: 33_170, + rows: (152_175, 152_904), +}; + +/// The grind-bits opt-out's pins (blowup 2^2, 20 bits, Q 112), which are the +/// production default's from before the bits moved. +const GRIND_BITS_20_CHAIN_PINS: ChainPins = ChainPins { + caps: &[3, 3, 3, 3, 3, 2], + cap_perms: 38, + grind_perms: 12, + opening_perms: 16_166, + perms: 16_411, + words: 32_590, + rows: (149_547, 150_258), +}; + +/// The opt-out's pins, which are the production default's from before P2: +/// `3R − 1 = 17` grinds (34 permutations), three nonce words a round. +const GRIND_ALL_CHAIN_PINS: ChainPins = ChainPins { + caps: &[3, 3, 3, 3, 3, 2], + cap_perms: 38, + grind_perms: 34, + opening_perms: 16_166, + perms: 16_443, + words: 32_602, + rows: (150_075, 150_811), +}; + +/// ⛔ `PREPARED_LEG_ROWS` stays FIXED under the fold knob, and +/// this is what makes that safe: under each first fold the constant still +/// covers the block's 20-variable stack, so no page is left sparse that the +/// opening could carry. The bound on the 24-variable chain above is the +/// default schedule's only. +#[test] +fn the_genesis_threshold_budget_still_covers_the_stack_under_each_first_fold() { + let budget = crate::continuation::PREPARED_LEG_ROWS; + for k0 in [5, 6] { + let rows = chain_shape_rows(&ChainShape::new(&first_fold_config(112, 20, k0), 20)); + println!("GENESIS BUDGET first{k0}: {budget} rows against a chain of {rows} at 20"); + assert!( + budget >= rows, + "first{k0}: the threshold charges {budget} rows for a stack that costs {rows}" + ); + } +} + +// ================ P2: a grind before the queries alone (`whir_grind=query`) ================ +// +// The production default grinds before each round's queries only, under +// `NonceLayout::Spent`: one grind and one nonce word a round. The machine sees +// the config only through `ChainShape::new(config, …)`, so the gates below +// prove, verify, execute and tamper it at a laptop's width. + +/// P2's grind and layout at a test width. +fn query_only_config(num_queries: usize, bits: u8) -> ChainConfig { + ChainConfig { + grind: GrindBits::query_only(bits), + format: ChainFormat { + nonces: NonceLayout::Spent, + ..ChainFormat::DEFAULT + }, + ..config(num_queries, 0) + } +} + +fn host_accepts(f: &Fixture, cfg: &ChainConfig, proof: &ChainProof) -> bool { + let mut t = Recording::new(); + verify::(proof, &f.root_bytes, &f.z, f.y, &f.domain, cfg, &mut t).is_ok() +} + +fn machine_accepts(program: &LfmProgram, arena: Vec) -> bool { + execute(program, &[arena], &crate::hash_pin::BLOCK_HASHER).is_ok() +} + +/// ★ A query-only chain executes on a proof the host accepts, and a wrong query +/// nonce is refused in every round on BOTH sides: over four forgeries a round +/// the machine agrees with the host on each, and at least one is refused (a +/// forgery passes the PoW with probability `2^-8`, so all four do with +/// `2^-32`). Its only grinds are the `R` query checks, and the uniform verifier +/// refuses its proofs: the grind bits are a verifier constant. +#[test] +fn a_query_only_chain_checks_its_query_nonce_in_every_round_on_both_sides() { + let bits = 8u8; + for num_vars in [6usize, 9] { + let cfg = query_only_config(3, bits); + let f = fixture_with(&cfg, num_vars); + assert_eq!(f.shape.grind, (0, 0, bits as usize)); + let program = chain_program(&f.shape); + assert!( + machine_accepts(&program, chain_arena(&f, &f.proof)), + "S={num_vars}: the honest proof must execute" + ); + + let rounds = f.shape.rounds(); + for r in 0..rounds { + assert_eq!( + f.shape.carries(r), + [false, false, true], + "S={num_vars}, round {r}: one nonce word, the query's" + ); + let mut refused = 0; + for flip in [1u64, 2, 4, 8] { + let mut forged = f.proof.clone(); + forged.rounds[r].nonces.query ^= flip; + let host = host_accepts(&f, &cfg, &forged); + assert_eq!( + machine_accepts(&program, chain_arena(&f, &forged)), + host, + "S={num_vars}, round {r}, nonce ^ {flip}: the machine and the host disagree" + ); + refused += usize::from(!host); + } + assert!( + refused > 0, + "S={num_vars}, round {r}: no wrong query nonce was refused" + ); + } + assert_eq!( + chain_grind_perms(&f.shape), + 2 * rounds, + "two permutations a query check, none anywhere else" + ); + assert!( + !host_accepts(&f, &config(3, bits), &f.proof), + "the uniform verifier must refuse a query-only proof" + ); + } +} + +/// ★ The default's 18-bit grind (against `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20`), at a test width: the grind +/// bits and Q are verifier constants. A chain ground at fewer bits and opened +/// at more queries executes on the machine built for its own config, and both +/// the host and the machine refuse it under a stricter grind at the same Q; the +/// host refuses it under the smaller Q too. +#[test] +fn a_chain_ground_at_fewer_bits_is_refused_by_the_stricter_verifier() { + for num_vars in [6usize, 9] { + let cheap = query_only_config(4, 2); + let f = fixture_with(&cheap, num_vars); + assert!( + host_accepts(&f, &cheap, &f.proof), + "S={num_vars}: the control" + ); + assert!( + machine_accepts(&chain_program(&f.shape), chain_arena(&f, &f.proof)), + "S={num_vars}: the honest proof must execute" + ); + + let strict = query_only_config(4, 12); + let strict_shape = ChainShape::new(&strict, num_vars); + assert!( + !host_accepts(&f, &strict, &f.proof), + "S={num_vars}: the host must refuse a 2-bit proof under 12 bits" + ); + assert!( + !machine_accepts( + &chain_program(&strict_shape), + chain_arena_as(&strict_shape, &f, &f.proof) + ), + "S={num_vars}: the machine must refuse a 2-bit proof under 12 bits" + ); + assert!( + !host_accepts(&f, &query_only_config(3, 2), &f.proof), + "S={num_vars}: the host must refuse a proof opened at a Q it does not expect" + ); + } +} + +/// ★ The base code's rate (`LAMBDA_VM_ZF_WHIR_BLOWUP`) is a verifier constant, +/// at a test width, under today's chain format and the production one (first6, +/// the auto cap, one nonce a round): a chain committed at one blowup executes on +/// the machine built for its own config, and both the host and the machine built +/// for the other blowup refuse it, each way round. Under the other rate every +/// tree is a level deeper or shallower, so the machine's arena (a verifier +/// constant) no longer fits the proof's paths. +#[test] +fn a_chain_at_one_blowup_is_refused_by_the_verifier_at_the_other() { + let formats = [ + ("today's chain format", query_only_config(4, 2).format), + ( + "the production chain format", + crate::zf_format::ZfFormat::DEFAULT.chain_format(), + ), + ]; + for (label, format) in formats { + let at = |log_blowup: usize| ChainConfig { + log_blowup, + format, + ..query_only_config(4, 2) + }; + for num_vars in [6usize, 9] { + for (own, other) in [(1usize, 2usize), (2, 1)] { + let tag = format!("{label}, S={num_vars}, blowup 2^{own}"); + let cfg = at(own); + let f = fixture_with(&cfg, num_vars); + assert_eq!(f.domain.log_size(), num_vars + own, "{tag}"); + assert!(host_accepts(&f, &cfg, &f.proof), "{tag}: the control"); + assert!( + machine_accepts(&chain_program(&f.shape), chain_arena(&f, &f.proof)), + "{tag}: the honest proof must execute" + ); + + let wrong = at(other); + let domain = Domain::::new(num_vars + other).expect("the other rate's domain"); + let mut t = Recording::new(); + assert!( + verify::( + &f.proof, + &f.root_bytes, + &f.z, + f.y, + &domain, + &wrong, + &mut t + ) + .is_err(), + "{tag}: the host at blowup 2^{other} must refuse it" + ); + let shape = ChainShape::new(&wrong, num_vars); + assert_ne!(shape.domain_log, f.shape.domain_log, "{tag}"); + assert!( + !machine_accepts(&chain_program(&shape), chain_arena_as(&shape, &f, &f.proof)), + "{tag}: the machine at blowup 2^{other} must refuse it" + ); + } + } + } +} + +/// ★ An unspent nonce has no word in the arena. Under `Spent`, a set folding or +/// out-of-domain nonce, in any round and the last round's out-of-domain slot +/// included, is refused by the host and cannot reach the machine: the arena +/// built from the forgery IS the honest one. Under `Three`, the same bits in +/// the legacy layout, the field has a word and neither side reads it: the +/// unbound field `Spent` removes, two of them a round. +#[test] +fn an_unspent_nonce_has_no_word_in_the_arena() { + let cfg = query_only_config(3, 8); + let f = fixture_with(&cfg, 9); + let rounds = f.shape.rounds(); + let honest = chain_arena(&f, &f.proof); + let three_cfg = ChainConfig { + format: ChainFormat::DEFAULT, + ..cfg + }; + let three = ChainShape::new(&three_cfg, 9); + let three_program = chain_program(&three); + assert_eq!( + RoundStorage::words(&three) - RoundStorage::words(&f.shape), + 2 * rounds as u32, + "two nonce words fewer a round" + ); + assert_eq!( + honest.len() + 2 * rounds, + chain_arena_as(&three, &f, &f.proof).len() + ); + + let mut refused = 0; + for r in 0..rounds { + for slot in ["folding", "ood"] { + let mut forged = f.proof.clone(); + match slot { + "folding" => forged.rounds[r].nonces.folding = 1, + _ => forged.rounds[r].nonces.ood = 1, + } + assert!( + !host_accepts(&f, &cfg, &forged), + "round {r}, the {slot} slot: the host must refuse a set unspent nonce" + ); + assert_eq!( + chain_arena(&f, &forged), + honest, + "round {r}, the {slot} slot: the forgery reached the arena" + ); + refused += 1; + + // The legacy layout carries it, unread on both sides. + assert!( + host_accepts(&f, &three_cfg, &forged), + "round {r}, {slot}: Three reads it" + ); + let arena = chain_arena_as(&three, &f, &forged); + assert_ne!(arena, chain_arena_as(&three, &f, &f.proof)); + assert!( + machine_accepts(&three_program, arena), + "round {r}, {slot}: the Three machine reads it" + ); + } + } + assert_eq!(refused, 2 * rounds, "two unspent slots a round"); +} + +/// ★ Gates one and two under P2: the emitter's hash schedule is the host's, +/// event for event; a state read happens exactly where a grind is spent (the +/// `R` query checks); every arena word is hinted once; and the rows and +/// permutations are the closed form's, which the census and the wrap sizing +/// read. Under the production format's cap and first fold as well. +#[test] +fn the_query_only_chains_are_the_host_transcripts_and_emit_their_closed_forms() { + let entry = SpongeEntry::fresh(); + let production_like = |num_queries| ChainConfig { + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).expect("6")), + ..query_only_config(num_queries, 8).format + }, + ..query_only_config(num_queries, 8) + }; + for (cfg, num_vars) in [ + (query_only_config(3, 8), 6), + (query_only_config(5, 8), 9), + (production_like(3), 9), + (production_like(4), 13), + ] { + let f = fixture_with(&cfg, num_vars); + let host = f.recorded.duplex.borrow().hashes.clone(); + let mine = chain_hash_schedule(&f.shape, entry); + assert_eq!( + mine, host, + "{cfg:?} S={num_vars}: the form's hash schedule is not the host's" + ); + let states = mine + .iter() + .filter(|h| !matches!(h, SpongeHash::Squeeze(_))) + .count(); + assert_eq!( + states, + f.shape.rounds(), + "{cfg:?} S={num_vars}: one state read a round, the query grind's" + ); + + let program = chain_program(&f.shape); + assert_eq!( + hint_rows(&program), + Layout::new(&f.shape).total as usize, + "{cfg:?} S={num_vars}: every arena word hinted once" + ); + let measured = program.instrs.len() - const_rows(&program) - chain_plumbing(&f.shape); + assert_eq!( + measured, + chain_rows(&f.shape, entry), + "{cfg:?} S={num_vars}: rows" + ); + assert_eq!( + perm_rows(&program), + chain_perms(&f.shape, entry), + "{cfg:?} S={num_vars}: permutations" + ); + assert!( + machine_accepts(&program, chain_arena(&f, &f.proof)), + "{cfg:?} S={num_vars}: the honest proof must execute" + ); + } +} + +// ================ the opt-out reproduces the bytes from before P2 ================ + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Keccak over every instruction and the arena schema, as `Debug` prints them. +pub(super) fn program_digest(program: &LfmProgram) -> String { + use digest::Digest; + let mut h = crypto::hash::platform_keccak::PlatformKeccak256::new(); + for instr in &program.instrs { + h.update(format!("{instr:?}\n").as_bytes()); + } + h.update(format!("{:?}", program.arena_schema).as_bytes()); + hex(&h.finalize()) +} + +fn arena_digest(words: &[LfmWord]) -> String { + use digest::Digest; + hex(&crypto::hash::platform_keccak::PlatformKeccak256::digest( + format!("{words:?}").as_bytes(), + )) +} + +/// The proof's rkyv length and keccak. +fn proof_digest(proof: &ChainProof) -> (usize, String) { + use digest::Digest; + let bytes = rkyv::to_bytes::(proof).expect("the proof serializes"); + ( + bytes.len(), + hex(&crypto::hash::platform_keccak::PlatformKeccak256::digest( + bytes.as_ref(), + )), + ) +} + +/// ★ THE LEGACY NONCE LAYOUT IS BYTE FOR BYTE THE ONE BEFORE P2. Every value +/// below was printed by the code at 0428c393b, before P2 existed: the chain +/// programs (instructions, a digest of all of them, arena words), the arenas and +/// the host proofs' rkyv bytes at zero bits (the only width where two proves +/// agree: a search returns any valid nonce), under `Three` with uniform grinds. +/// Also the production chain under the opt-out, `whir_grind=all` at 20 bits +/// and blowup 2^2, at the stacks the block reaches (25 and 27): the program the +/// pre-P2 default emitted. +#[test] +fn the_legacy_nonce_layout_reproduces_the_bytes_from_before_p2() { + const PROGRAMS: [(usize, u8, usize, u32, &str); 4] = [ + ( + 6, + 0, + 772, + 116, + "d99454d21b008a7cf6ac3d633b97b38176ea77a8c1c7dd08e85d49c7aa807341", + ), + ( + 9, + 0, + 1514, + 238, + "2435342f0fee1d177a5327440488045f3313f0f121cb3174a9fd52dec8545fd9", + ), + ( + 6, + 8, + 907, + 116, + "99c2d0b71e767efc63e71d942519910bd77e257f82248faa1d3bfea12ac5bf4b", + ), + ( + 9, + 8, + 1729, + 238, + "4a4d35af69c1997306486762356ce33007200fa366bfce66f8065d2218bb021f", + ), + ]; + const ARENAS: [(usize, &str, usize, &str); 2] = [ + ( + 6, + "223a11955672049c529f2d8c467bc7f3789f961d7516c2323a9a2459c745369a", + 2744, + "1226279cd1a1019770c62e5357174c6ae8b153d2ba5150b2a33d46d9a179f5bb", + ), + ( + 9, + "5f52e35dff3e56d4da419c89a52afa25a4783daaec7d29c62047e7c5762db98e", + 6160, + "6616efadcac28bc004aae828c38495ba866775d4288eaa2f6f05e338c13d4a9d", + ), + ]; + for (num_vars, grind, instrs, words, digest) in PROGRAMS { + let f = fixture(num_vars, 3, grind); + assert_eq!(f.shape.nonces, NonceLayout::Three); + let program = chain_program(&f.shape); + assert_eq!( + (program.instrs.len(), RoundStorage::words(&f.shape)), + (instrs, words), + "S={num_vars} grind {grind}" + ); + assert_eq!( + program_digest(&program), + digest, + "S={num_vars} grind {grind}: the program" + ); + if grind == 0 { + let (_, arena, len, proof) = ARENAS + .iter() + .find(|a| a.0 == num_vars) + .copied() + .expect("an arena golden"); + assert_eq!( + arena_digest(&chain_arena(&f, &f.proof)), + arena, + "S={num_vars}: the arena" + ); + assert_eq!( + proof_digest(&f.proof), + (len, proof.to_string()), + "S={num_vars}: the proof" + ); + } + } + + let opt_out = crate::zf_format::ZfFormat { + whir_blowup: 4, + whir_grind: crate::zf_format::WhirGrind::All, + whir_grind_bits: crate::zf_format::LEGACY_WHIR_GRIND_BITS, + ..crate::zf_format::ZfFormat::DEFAULT + }; + for (num_vars, instrs, words, digest) in [ + ( + 25usize, + 183_631usize, + 32_602u32, + "97b10ca20df8a7e4b4da359220786ce2fe70004af519a532c8c696119ab4d601", + ), + ( + 27, + 208_671, + 36_875, + "ceccaf1e696c0a71e199f4b1a726bd113b77c9fc4037cf3eea05c7561e1a0eee", + ), + ] { + let config = crate::multilinear_prove::chain_config_under(&opt_out, &[(1, num_vars)]); + let shape = ChainShape::new(&config, num_vars); + let program = chain_program(&shape); + assert_eq!( + (program.instrs.len(), RoundStorage::words(&shape)), + (instrs, words), + "production S={num_vars} under whir_grind=all" + ); + assert_eq!( + program_digest(&program), + digest, + "production S={num_vars} under whir_grind=all: the program" + ); + } +} diff --git a/prover/src/lfm/whir_commit_ood_leg_tests.rs b/prover/src/lfm/whir_commit_ood_leg_tests.rs new file mode 100644 index 000000000..a72534e71 --- /dev/null +++ b/prover/src/lfm/whir_commit_ood_leg_tests.rs @@ -0,0 +1,510 @@ +//! The block leaf's group opening with its commit-time answers +//! (`stacked_eval::verify_ood`), as a machine leg under Poseidon1, gated +//! against the host's `stacked_eval::prove_mapped_ood` proofs: the point +//! drawn in-guest from the group's root cells is the host's +//! (`multilinear_block::commit_ood_point`), an honest opening executes, a +//! false or traded answer has no execution, and each half of the batching — +//! the answers on the claims, their `eq` on the weights — is load-bearing. +//! +//! Under each of the leaf's query checks: V1 on a small chain, and V1 and +//! lean on the production's chain shape (first fold 6, the automatic 4-ary +//! cap, coset leaves, enough queries that the lean check's tail rounds run). + +use crypto::fiat_shamir::p1_transcript::P1Transcript; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{Claimed, OodClaims, StackedCommitment, StackedProof}; +use multilinear::stacking::StackedLayout; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, LeafOrder, WhirFolds, +}; +use multilinear::whir_commit::Commitment; +use multilinear::whir_hash::P1Whir; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::commitment_to_digest; +use super::builder::{Cell, Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::WrapHash; +use super::executor::execute; +use super::hash::HasherKind; +use super::validator::validate; +use super::whir_block::emit_commit_ood_point; +use super::whir_chain::{ChainShape, LeafQuery, RoundStorage, try_push_round_words}; +use super::whir_stacked::{ + OodWires, StackedChainWires, emit_stacked_verify_chains, ood_faults, stacked_verify_cost_ood, +}; +use super::whir_transcript::{SpongeEntry, WhirTranscript}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// The group's index in the block: a constant of its point's prefix. +const GROUP: usize = 2; + +/// Mixed heights with gaps, seven polynomials of ten variables: under +/// `Indexed` a run of four and a run of three. +const HEIGHTS: [usize; 14] = [8, 9, 7, 9, 6, 9, 9, 8, 5, 9, 9, 4, 9, 7]; +const N_STACK: usize = 10; + +/// A chain shape and the leaf's query check over it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Case { + /// A small chain: first fold 3, folds of 2, three queries; V1. + Small, + /// The production's chain shape at this stack — first fold 6, folds of 4, + /// blowup 2, the automatic 4-ary cap, coset leaves — at 96 queries, under + /// the leaf's query check `LeafQuery`. + Production(LeafQuery), +} + +const CASES: [Case; 3] = [ + Case::Small, + Case::Production(LeafQuery::V1), + Case::Production(LeafQuery::Lean), +]; + +fn config(case: Case, group: GroupFold) -> ChainConfig { + match case { + Case::Small => ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::query_only(4), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(3).expect("a tested first fold")), + group, + ..ChainFormat::DEFAULT + }, + }, + Case::Production(_) => ChainConfig { + log_blowup: 1, + log_folding: 4, + num_queries: 96, + grind: GrindBits::query_only(4), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).expect("a tested first fold")), + arity4_cap: CapPolicy::Auto, + leaves: LeafOrder::Coset, + group, + ..ChainFormat::DEFAULT + }, + }, + } +} + +fn query(case: Case) -> LeafQuery { + match case { + Case::Small => LeafQuery::V1, + Case::Production(query) => query, + } +} + +fn fee(v: u64) -> FEE { + FEE::new([ + FE::from(v.wrapping_mul(6364136223846793005) >> 11), + FE::from(v ^ 0xA5A5), + FE::from(v.wrapping_add(7)), + ]) +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..1u64 << num_vars) + .map(|i| { + FE::from( + i.wrapping_mul(0x2545_F491_4F6C_DD1D) + .wrapping_add(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)) + >> 13, + ) + }) + .collect(), + ) + .expect("a cube") +} + +/// A group committed and opened on the host with its answers, every column +/// at a point of its own. +struct Fixture { + case: Case, + cfg: ChainConfig, + layout: StackedLayout, + points: Vec>, + values: Vec, + roots: Vec, + claims: OodClaims, + proof: StackedProof, + domain: Domain, + shape: ChainShape, +} + +fn fixture(case: Case, group: GroupFold) -> Fixture { + moved_fixture(case, group, FEE::zero()) +} + +/// [`fixture`] with `delta` moved from polynomial 0's answer onto column 0's +/// stated value, the opening still proved from the true polynomials: the +/// honest fixture at a zero `delta`. +fn moved_fixture(case: Case, group: GroupFold, delta: FEE) -> Fixture { + let cfg = config(case, group); + let layout = StackedLayout::build(&HEIGHTS, N_STACK).expect("the layout packs"); + assert_eq!(layout.num_polys(), 7, "the stack this test means"); + let columns: Vec> = HEIGHTS + .iter() + .enumerate() + .map(|(c, &h)| column(h, 17 + 13 * c as u64)) + .collect(); + let borrowed = multilinear::stacking::borrow(&columns); + let points: Vec> = HEIGHTS + .iter() + .enumerate() + .map(|(c, &h)| (0..h).map(|i| fee(0x5eed + (c * 37 + i) as u64)).collect()) + .collect(); + let mut values: Vec = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in::(p).expect("a column takes its own point")) + .collect(); + values[0] += delta; + let stacked = StackedCommitment::::commit(layout.clone(), &borrowed, None, &cfg) + .expect("the group commits"); + let roots = stacked.roots(); + let z0 = + stark::multilinear_block::commit_ood_point::(GROUP, &layout, &roots, &cfg) + .expect("the group's point"); + let (mut claims, _) = + multilinear::stacked_eval::ood_claims::(&layout, &borrowed, None, &z0) + .expect("the answers"); + claims.answers[0] = claims.answers[0] - delta; + let proof = multilinear::stacked_eval::prove_mapped_ood::( + &stacked, + &borrowed, + None, + &Claimed::PerColumn(&points), + &values, + Some(&claims), + &cfg, + &mut P1Transcript::new(), + ) + .expect("the group proves"); + Fixture { + case, + cfg, + shape: ChainShape { + query: query(case), + ..ChainShape::new_at(&cfg, layout.n_stack(), 4) + }, + layout, + points, + values, + roots, + claims, + proof, + domain: stacked.domain().clone(), + } +} + +/// The host's verdict on `answers`, read at the group's point. +fn host_accepts(f: &Fixture, answers: &[FEE]) -> bool { + multilinear::stacked_eval::verify_ood::( + &f.proof, + &f.layout, + &f.roots, + &Claimed::PerColumn(&f.points), + &f.values, + Some(multilinear::stacked_eval::CommitOod { + z0: &f.claims.z0, + answers, + }), + &f.domain, + &f.cfg, + &mut P1Transcript::new(), + ) + .is_ok() +} + +/// The leg as a program: the group's roots and answers hinted, then each +/// column's point, the values and the chains; the group's point drawn from +/// the root cells ([`emit_commit_ood_point`]); the opening with the answers +/// carried; the point's first coordinate `z0` and the batching challenge +/// published. +fn program(f: &Fixture) -> LfmProgram { + emit_leg(f, Leg::Derived) +} + +/// What a leg emits around the opening. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Leg { + /// The point drawn from the root cells, the answers carried. + Derived, + /// The point's coordinates hinted, the answers carried: the opening's own + /// rows alone. + Hinted, + /// No answers: today's opening. + Without, +} + +fn emit_leg(f: &Fixture, leg: Leg) -> LfmProgram { + let polys = f.layout.num_polys(); + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1); + let arena = b.declare_arena(0); + let mut at = 0u32; + let next = |b: &mut LfmBuilder, at: &mut u32| { + let cell = b.hint_word(arena, *at); + *at += 1; + cell + }; + let roots: Vec = (0..polys).map(|_| next(&mut b, &mut at)).collect(); + let answers: Vec = (0..polys).map(|_| next(&mut b, &mut at).as_ext()).collect(); + let points: Vec> = f + .points + .iter() + .map(|p| p.iter().map(|_| next(&mut b, &mut at).as_ext()).collect()) + .collect(); + let values: Vec = f + .values + .iter() + .map(|_| next(&mut b, &mut at).as_ext()) + .collect(); + let runs = f.shape.chains(polys); + let mut finals = Vec::new(); + let mut storage = Vec::new(); + for run in &runs { + finals.push(next(&mut b, &mut at).as_ext()); + let shape = f.shape.run(run.len()); + storage.push(RoundStorage::hint(&mut b, arena, at, &shape)); + at += RoundStorage::words(&shape); + } + let point: Vec = match leg { + Leg::Derived => emit_commit_ood_point( + &mut b, + &stark::multilinear_block::commit_ood_prefix(GROUP, &f.layout, &f.cfg), + &roots, + f.layout.n_stack(), + f.layout.n_stack() + f.cfg.log_blowup, + ), + Leg::Hinted | Leg::Without => (0..f.layout.n_stack()) + .map(|_| next(&mut b, &mut at).as_ext()) + .collect(), + }; + let openings: Vec<_> = storage.iter().map(RoundStorage::openings).collect(); + let rounds: Vec> = storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + let chains: Vec> = runs + .iter() + .enumerate() + .map(|(i, run)| StackedChainWires { + rounds: &rounds[i], + roots: &roots[run.clone()], + final_value: finals[i], + }) + .collect(); + let claimed_at: Vec<&[Ext]> = points.iter().map(Vec::as_slice).collect(); + let mut transcript = WhirTranscript::for_builder(&mut b); + let gamma = emit_stacked_verify_chains( + &mut b, + &mut transcript, + &f.layout, + &chains, + &claimed_at, + &values, + (leg != Leg::Without).then_some(OodWires { + point: &point, + answers: &answers, + }), + &f.shape, + &f.domain, + ); + b.public(point[0].as_cell()); + b.public(gamma.as_cell()); + b.set_arena_len(arena, at); + let program = compile(b.finish()); + validate(&program).expect("the leg must be admissible"); + program +} + +/// The arena from the fixture with `answers`, in [`program`]'s order. +fn arena(f: &Fixture, answers: &[FEE]) -> Vec { + let mut words: Vec = f.roots.iter().map(commitment_to_digest).collect(); + words.extend(answers.iter().map(ext_word)); + words.extend(f.points.iter().flatten().map(ext_word)); + words.extend(f.values.iter().map(ext_word)); + for (chain, run) in f + .proof + .polys + .iter() + .zip(f.shape.chains(f.layout.num_polys())) + { + words.push(ext_word(&chain.final_value)); + try_push_round_words(&mut words, &f.shape.run(run.len()), chain) + .expect("the proof covers every leaf the lean check reads"); + } + words +} + +/// Every case and split, with the lean case's tail rounds asserted present: +/// the fixtures the tests below walk. +fn fixtures() -> Vec { + let mut out = Vec::new(); + for case in CASES { + for group in GROUPS { + let f = fixture(case, group); + if case == Case::Production(LeafQuery::Lean) { + let runs = f.shape.chains(f.layout.num_polys()); + assert!( + runs.iter().any(|run| { + let shape = f.shape.run(run.len()); + (0..shape.rounds()).any(|r| shape.tail_round(r)) + }), + "{case:?} {group:?}: the lean check's tail rounds run" + ); + } + out.push(f); + } + } + out +} + +fn runs(program: &LfmProgram, words: Vec) -> Result, String> { + let exec = + execute(program, &[words], &HasherKind::Poseidon1W16).map_err(|e| format!("{e:?}"))?; + exec.public_words + .iter() + .map(|(_, w)| word_as_ext(w).ok_or_else(|| "a published extension".to_string())) + .collect() +} + +const GROUPS: [GroupFold; 2] = [GroupFold::PerPoly, GroupFold::Indexed]; + +/// ★ The leg executes the host's honest opening, and the point it draws from +/// the root cells is the host's. +#[test] +fn an_opening_with_its_answers_executes_at_the_hosts_point() { + for f in fixtures() { + let group = (f.case, f.cfg.format.group); + assert!( + host_accepts(&f, &f.claims.answers), + "{group:?}: the control" + ); + let published = runs(&program(&f), arena(&f, &f.claims.answers)) + .unwrap_or_else(|e| panic!("{group:?}: the machine refused the host's opening: {e}")); + assert_eq!(published[0], f.claims.z0, "{group:?}: the group's point"); + } +} + +/// ★ A false answer — any one of them — and two answers of one run traded, +/// their sum kept, have no execution, as the host refuses them. +#[test] +fn a_false_or_traded_answer_has_no_execution() { + for f in fixtures() { + let group = (f.case, f.cfg.format.group); + let program = program(&f); + let answers = &f.claims.answers; + for at in 0..answers.len() { + let mut wrong = answers.clone(); + wrong[at] += FEE::one(); + assert!(!host_accepts(&f, &wrong), "{group:?}: answer {at}, host"); + assert!( + runs(&program, arena(&f, &wrong)).is_err(), + "{group:?}: answer {at} false executed" + ); + } + let mut traded = answers.clone(); + let delta = fee(99); + traded[0] += delta; + traded[1] = traded[1] - delta; + assert!(!host_accepts(&f, &traded), "{group:?}: traded, host"); + assert!( + runs(&program, arena(&f, &traded)).is_err(), + "{group:?}: two answers of one run traded" + ); + } +} + +/// ★ No answer rides at `γ'^0`. Column 0's claim is carried at power one, so +/// were polynomial 0's answer carried there too, `Δ` moved from the answer +/// onto column 0's value would keep every chain's claim. With `γ'^(i+1)` the +/// moved pair, proved by the host from the true polynomials, has no +/// execution, as the host refuses it. +#[test] +fn an_answer_traded_against_column_zero_has_no_execution() { + for case in CASES { + for group in GROUPS { + let f = moved_fixture(case, group, fee(1234)); + assert_eq!( + f.layout.placements()[0].poly, + 0, + "column 0 lies in polynomial 0" + ); + let host = host_accepts(&f, &f.claims.answers); + assert!( + runs(&program(&f), arena(&f, &f.claims.answers)).is_err(), + "{case:?} {group:?}: answer 0 traded against column 0's value executed" + ); + assert!( + !host, + "{case:?} {group:?}: answer 0 traded against column 0, host" + ); + } + } +} + +/// ★ Mutations: with the answers left out of the chains' claims, or their +/// `eq` out of the chains' weights, the honest opening has no execution — +/// each half of the batching is what the chain checks. +#[test] +fn each_half_of_the_batching_is_load_bearing() { + for f in fixtures() { + let group = (f.case, f.cfg.format.group); + for fault in [ + ood_faults::Fault::ClaimWithoutAnswers, + ood_faults::Fault::WeightWithoutAnswers, + ] { + let program = ood_faults::with(fault, || program(&f)); + assert!( + runs(&program, arena(&f, &f.claims.answers)).is_err(), + "{group:?} {fault:?}: the honest opening executed" + ); + } + } +} + +/// Poseidon1 permutations of a program: its socket's `Hash16` rows. +fn hash16_rows(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Hash16(_))) + .count() +} + +/// ★ The plan prices what the leaf emits: the opening's permutations with +/// and without its answers are the cost model's +/// (`stacked_verify_cost_ood`), and the point's own sponge is +/// `commit_ood_point_perms`'s, at either split and either query check. +#[test] +fn the_answers_cost_what_the_plan_prices() { + for f in fixtures() { + let group = (f.case, f.cfg.format.group); + let group_of: Vec = (0..HEIGHTS.len()).collect(); + let entry = SpongeEntry::fresh_for(WrapHash::Poseidon1); + for (leg, ood) in [(Leg::Hinted, true), (Leg::Without, false)] { + let cost = stacked_verify_cost_ood(&f.layout, &group_of, &f.shape, entry, ood); + assert_eq!( + hash16_rows(&emit_leg(&f, leg)), + cost.perms(), + "{group:?} answers {ood}: permutations" + ); + } + assert_eq!( + hash16_rows(&emit_leg(&f, Leg::Derived)) - hash16_rows(&emit_leg(&f, Leg::Hinted)), + super::whir_block::commit_ood_point_perms(WrapHash::Poseidon1, f.layout.num_polys()), + "{group:?}: the point's sponge" + ); + } +} diff --git a/prover/src/lfm/whir_epoch.rs b/prover/src/lfm/whir_epoch.rs new file mode 100644 index 000000000..25a252f31 --- /dev/null +++ b/prover/src/lfm/whir_epoch.rs @@ -0,0 +1,652 @@ +//! The table and commitment-group walks of `multilinear_table::multi_verify`, +//! as machine legs: the shared emitters a block leaf assembles its verify from +//! ([`super::whir_block`]). +//! +//! Every leg is gated on its own against the host function it replaces — a +//! table against `multilinear_table::verify`, a commitment group against +//! `stacked_eval::verify`, a preprocessed column against `Mle::evaluate_in`. +//! What a caller adds is the ORDER, and the order is the only part of a +//! Fiat-Shamir verifier that no per-leg gate can see: each leg derives its own +//! challenges from the transcript it is handed, so a leg run at the wrong point +//! in the stream is internally consistent and globally wrong. +//! +//! The order, read from `multilinear_table::multi_verify`: +//! +//! 1. the CARRIED group roots, then the DERIVED roots ([`emit_absorb_roots`]), +//! then `z`, `alpha`, `beta`; +//! 2. per table, in sub-proof order and threading the sponge table to table, +//! `emit_table_verify` ([`emit_table_walk`]), with `settled_out_of_band` +//! for the tables a prepared opening covers; +//! 3. the bus balance against the expected value ([`emit_expected`]); +//! 4. per commitment group, in group order, its opening +//! ([`emit_group_walk_chains`]). + +use multilinear::constraint_argument::FactorKind; +use multilinear::stacking::StackedLayout; +use multilinear::whir::Domain; + +use crate::tables::bitwise::NUM_PRECOMPUTED_COLS; +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::{Cell, Ext, LfmBuilder}; +use super::preprocessed::{emit_bitwise_preprocessed, emit_const_mle_at}; +use super::whir_chain::ChainShape; +use super::whir_stacked::{StackedChainWires, emit_stacked_verify_chains}; +use super::whir_table::{TableProofWires, TableShape, TableVerdictWires, emit_table_verify}; +use super::whir_transcript::WhirTranscript; +use super::word::LfmWord; + +/// The shapes a group's wrapper needs, derived from what the epoch states. +/// +/// `heights` is one entry per COLUMN — a table of width `w` at `v` variables +/// contributes `w` entries of `v` — and `group_of` names which columns share a +/// claimed point, which in an epoch is the table they belong to. Both are read +/// off `(width, num_vars)` and nothing else, which is why a census can run on a +/// laptop from a shape log while the proof itself is the box's. +pub fn group_columns(shapes: &[(usize, usize)]) -> (Vec, Vec) { + let mut heights = Vec::new(); + let mut group_of = Vec::new(); + for (table, &(width, num_vars)) in shapes.iter().enumerate() { + for _ in 0..width { + heights.push(num_vars); + group_of.push(table); + } + } + (heights, group_of) +} + +// ============================================================================= +// The roots block +// ============================================================================= + +/// `multilinear_table::absorb_roots`, emitted: the carried roots, then the +/// derived ones. No draws: the block's front follows the roots with its +/// commit-time answers before it draws `z`, `alpha`, `beta`. +pub fn emit_absorb_roots( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + carried: &[Cell], + derived: &[LfmWord], +) { + for root in carried { + transcript.absorb_digest(b, *root); + } + for word in derived { + let root = b.digest_const(*word); + transcript.absorb_digest(b, root.as_cell()); + } +} + +/// `compute_commit_bus_offset`, emitted: `alpha^2` and `z - bus_id` once, then +/// per published byte two `MulAdd`s for `- index*alpha` and `- value*alpha^2`, +/// one `Div` for the inverse, and one `Add` into the running sum, which the +/// first byte does not need. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn emit_expected( + b: &mut LfmBuilder, + published: &[u8], + start_index: u64, + z: Ext, + alpha: Ext, +) -> Ext { + if published.is_empty() { + // ⛔ The host's own early return, and it reads NEITHER challenge. + return b.ext_const(&FEE::zero()); + } + let alpha_sq = b.emul(alpha, alpha); + let bus_id = b.ext_const(&FEE::from(crate::tables::types::BusId::Commit as u64)); + let base = b.esub(z, bus_id); + let one = b.ext_const(&FEE::one()); + let mut sum: Option = None; + for (offset, value) in published.iter().enumerate() { + // The two coefficients are NEGATED at emit time, so the fingerprint is + // two `MulAdd`s rather than two multiplies and two subtractions. + let index = b.ext_const(&-FEE::from(start_index + offset as u64)); + let byte = b.ext_const(&-FEE::from(u64::from(*value))); + let term = b.emul_add(index, alpha, base); + let term = b.emul_add(byte, alpha_sq, term); + let inverse = b.ediv(one, term); + sum = Some(match sum { + None => inverse, + Some(running) => b.eadd(running, inverse), + }); + } + sum.expect("a non-empty public output") +} + +// ============================================================================= +// The per-table walk +// ============================================================================= + +/// How one table's preprocessed columns are discharged. +/// +/// The host has ONE route — rebuild each column's multilinear extension and +/// evaluate it at the reduced point (`multilinear_table.rs:959`) — and the +/// machine cannot take it: BITWISE alone is eleven columns of `2^20`. So the +/// machine has four, and which one a table takes is a property of the TABLE and +/// never of its position: they are selected by `air.name()`, the same rule +/// `block_whir::decode_table_index` applies. +pub enum PreprocessedRoute<'a> { + /// Nothing left to check — either the table has no preprocessed columns, or + /// a prepared opening settled all of them. + None, + /// BITWISE: the closed form, `O(NUM_VARS)` where the fold is `O(2^NUM_VARS)`. + Bitwise, + /// KECCAK_RC and REGISTER: one shared `eq(point, ·)` per table with each + /// column folded against it, one row per NONZERO entry. + ConstMle(&'a [&'a [FE]]), + /// ★ A CROSS-EPOCH PAGE TABLE, and the only route an epoch never takes. + /// + /// `GLOBAL_MEMORY` carries OFFSET — the identity ramp, whose extension is + /// `Σ_k 2^k · r_k` and costs `num_vars − 1` rows — and, on any page that is + /// not a private-input page, INIT: that page's genesis bytes, discharged by + /// [`super::preprocessed::emit_sparse_mle_at`] because they are a function + /// of the program rather than of the row index and the cross-epoch proof + /// has no prepared opening to settle them with. + /// + /// ⚠ BOTH COLUMNS ARE CARRIED, not just the one that needs deciding. The + /// leg asserts each against the table's own claimed value through + /// [`preprocessed_targets`], and the cost form reads `num_vars` off OFFSET's + /// own length rather than taking it from a caller — a page whose columns + /// were not `2^num_vars` tall would be a table nobody meant to build. + /// + /// `init` is `None` exactly on a private-input page, where INIT is a + /// committed main column the verifier never recomputes + /// (`continuation.rs:241-251`). That `None` is the ROUTE TABLE's decision, + /// taken from the page's own config, and never inferred from how many + /// columns the AIR happened to present. + Page { + offset: &'a [FE], + init: Option<&'a [FE]>, + }, + /// A CPU table's TIMESTAMP (main's preprocessed timestamp): the column + /// `4·(i + 1)` of `2^num_vars` rows, whose extension is `4 + 4·Σ_k 2^k·r_k` + /// ([`super::preprocessed::emit_timestamp_ramp`]) — no opening, no fold. + Timestamp { num_vars: usize }, +} + +/// One table's preprocessed plan: what a prepared opening settled, and how +/// whatever is left is checked. +/// +/// ⚠ `settled` must be the SAME value that drives the opening, never a knob a +/// caller sets on its own — the host's own warning at +/// `multilinear_table.rs:943`. A second, independent knob would be a way to +/// switch off a check with nothing put in its place. +pub struct PreprocessedPlan<'a> { + /// Leading columns the prepared opening covers: `settled_out_of_band`. + pub settled: usize, + /// How the columns past those are checked. + pub route: PreprocessedRoute<'a>, +} + +/// Where each preprocessed column's claimed value sits among a table's reduced +/// column values. +/// +/// ★ THIS IS THE HOST'S OWN INDIRECTION, AND IT IS NOT THE IDENTITY. +/// `check_preprocessed` does not compare preprocessed column `c` against +/// `column_values[c]`. It reads `slot(slot_of, c)` to get a factor, takes that +/// factor's SOURCE, requires the source unshifted, and indexes +/// `column_values[source.column]` (`multilinear_table.rs:1060-1080`). A leg that +/// indexed by `c` would agree with the host on every table whose layout happens +/// to be the identity and disagree silently on the rest, which is why the +/// mapping is derived here from the same two slices the host reads rather than +/// assumed. +/// +/// Both of the host's refusals are EMIT-TIME assertions, the +/// `epoch_verify.rs:171-179` idiom: a column with no factor slot, or a factor +/// read at an offset, describes a layout nobody meant to build, and a program +/// emitted for it could not be supplied a proof of the right shape anyway. +pub fn preprocessed_targets(slot_of: &[usize], kinds: &[FactorKind], columns: usize) -> Vec { + (0..columns) + .map(|column| { + let factor = *slot_of.get(column).unwrap_or_else(|| { + panic!( + "preprocessed column {column} has no factor slot in a layout of {} columns", + slot_of.len() + ) + }); + let source = kinds + .get(factor) + .and_then(FactorKind::source) + .filter(|source| source.offset == 0) + .unwrap_or_else(|| { + panic!( + "preprocessed column {column} rides factor {factor}, which is not an \ + unshifted committed source" + ) + }); + source.column + }) + .collect() +} + +/// What the per-table walk leaves for the two legs below it. +/// +/// The collections are the host's own under the host's names: `outputs` is what +/// the closure sums, `points` and `values` are what each commitment group +/// settles. ⚠ `points` is kept ONE PER TABLE and expanded to one per COLUMN by +/// [`Self::column_points`], because one per column is the shape +/// `emit_stacked_verify` takes and the expansion is where the host's +/// `for _ in 0..statement.slot_of.len()` (`multilinear_table.rs:1226`) lives. +pub struct TableWalk { + /// `(p, q)` per table, in table order. + pub outputs: Vec<(Ext, Ext)>, + /// The reduced point, one per TABLE. + pub points: Vec>, + /// Committed columns per table — what the group walk advances by. + pub widths: Vec, + /// Every table's reduced column values, concatenated in table order. + pub values: Vec, +} + +impl TableWalk { + /// The host's `points`: one entry per COLUMN, a table's columns sharing that + /// table's point. + pub fn column_points(&self) -> Vec<&[Ext]> { + let mut out = Vec::with_capacity(self.values.len()); + for (point, &width) in self.points.iter().zip(&self.widths) { + for _ in 0..width { + out.push(point.as_slice()); + } + } + out + } + + /// Where table `index`'s columns start in the global column order — the + /// host's `prepared_at` when `index` is the prepared table. + pub fn column_at(&self, index: usize) -> usize { + self.widths[..index].iter().sum() + } +} + +/// ★ The per-table walk, emitted: `multi_verify`'s first loop +/// (`multilinear_table.rs:1218`). +/// +/// Per table in index order, [`emit_table_verify`] with the sponge entry +/// threaded table to table, then that table's preprocessed leg IMMEDIATELY +/// after. The leg sits inside the same step rather than in a pass of its own +/// because `check_preprocessed` is the LAST statement of the host's `verify` +/// (`multilinear_table.rs:936`) and moves no transcript operation — a pass of +/// its own would compute the same values and say something different about +/// where they belong. +#[allow(clippy::too_many_arguments)] +pub fn emit_table_walk( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + tables: &[TableProofWires<'_>], + shapes: &[TableShape<'_>], + plans: &[PreprocessedPlan<'_>], + slots: &[&[usize]], + z: Ext, + alpha_powers: &[Ext], + beta: Ext, +) -> TableWalk { + assert_eq!(tables.len(), shapes.len(), "one shape per table proof"); + assert_eq!(tables.len(), plans.len(), "one preprocessed plan per table"); + assert_eq!(tables.len(), slots.len(), "one slot map per table"); + + let mut walk = TableWalk { + outputs: Vec::with_capacity(tables.len()), + points: Vec::with_capacity(tables.len()), + widths: Vec::with_capacity(tables.len()), + values: Vec::new(), + }; + + for (index, ((proof, shape), plan)) in tables.iter().zip(shapes).zip(plans).enumerate() { + let verdict = emit_table_verify(b, transcript, proof, shape, z, alpha_powers, beta); + emit_preprocessed_leg(b, plan, slots[index], shape, &verdict); + walk.outputs.push(verdict.bus_output); + walk.widths.push(verdict.column_values.len()); + walk.values.extend(verdict.column_values.iter().copied()); + walk.points.push(verdict.point); + } + walk +} + +/// One table's preprocessed columns, checked against the values its own +/// argument settled on. +pub(crate) fn emit_preprocessed_leg( + b: &mut LfmBuilder, + plan: &PreprocessedPlan<'_>, + slot_of: &[usize], + shape: &TableShape<'_>, + verdict: &TableVerdictWires, +) { + let columns: &[&[FE]] = match &plan.route { + PreprocessedRoute::None => return, + PreprocessedRoute::Bitwise => { + // The closed form covers all eleven at once, so a settled prefix + // over them would be two checks on one column rather than none. + assert_eq!( + plan.settled, 0, + "BITWISE's columns are covered by the closed form, not by an opening" + ); + let targets = preprocessed_targets(slot_of, shape.kinds, NUM_PRECOMPUTED_COLS); + let values = emit_bitwise_preprocessed(b, &verdict.point); + for (value, &target) in values.iter().zip(&targets) { + b.assert_eq_ext(*value, verdict.column_values[target]); + } + return; + } + PreprocessedRoute::Page { offset, init } => { + assert_eq!( + plan.settled, 0, + "a cross-epoch page's columns are checked in the program, not settled \ + by an opening the cross-epoch proof does not carry" + ); + let columns = 1 + usize::from(init.is_some()); + let targets = preprocessed_targets(slot_of, shape.kinds, columns); + assert_eq!( + offset.len(), + 1usize << verdict.point.len(), + "OFFSET must have one entry per row of its table" + ); + let ramp = super::preprocessed::emit_offset_ramp(b, &verdict.point); + b.assert_eq_ext(ramp, verdict.column_values[targets[0]]); + if let Some(init) = init { + let values = super::preprocessed::emit_sparse_mle_at(b, &[*init], &verdict.point); + b.assert_eq_ext(values[0], verdict.column_values[targets[1]]); + } + return; + } + PreprocessedRoute::Timestamp { num_vars } => { + assert_eq!( + plan.settled, 0, + "a CPU table's TIMESTAMP is checked by its closed form, not settled by an opening" + ); + assert_eq!( + *num_vars, + verdict.point.len(), + "the TIMESTAMP column is as tall as its table" + ); + let targets = preprocessed_targets( + slot_of, + shape.kinds, + crate::tables::cpu::NUM_PRECOMPUTED_COLS, + ); + let timestamp = super::preprocessed::emit_timestamp_ramp(b, &verdict.point); + b.assert_eq_ext(timestamp, verdict.column_values[targets[0]]); + return; + } + PreprocessedRoute::ConstMle(columns) => columns, + }; + + // The host's own assert, at emit time: an opening may not cover more + // columns than the table has (`multilinear_table.rs:1069`). + assert!( + plan.settled <= columns.len(), + "a prepared opening settled {} of {} preprocessed columns", + plan.settled, + columns.len() + ); + let remaining = &columns[plan.settled..]; + if remaining.is_empty() { + return; + } + let targets = preprocessed_targets(slot_of, shape.kinds, columns.len()); + let values = emit_const_mle_at(b, remaining, &verdict.point); + for (value, &target) in values.iter().zip(&targets[plan.settled..]) { + b.assert_eq_ext(*value, verdict.column_values[target]); + } +} + +// ============================================================================= +// The commitment-group walk +// ============================================================================= + +/// One commitment group's wires by CHAIN: its layout, the domain the chains +/// run over, and one chain per run of the shape's split +/// ([`ChainShape::chains`]), which is one per polynomial under +/// `GroupFold::PerPoly` and a run of up to four under `Indexed` (D-ONECHAIN). +pub struct GroupChainWires<'a> { + pub layout: &'a StackedLayout, + pub chains: &'a [StackedChainWires<'a>], + pub shape: &'a ChainShape, + pub domain: &'a Domain, + /// The group's commit-time answers, where its proof carries them (the + /// block's groups; an epoch's carry none). + pub ood: Option>, +} + +/// ★ The commitment-group walk, emitted over each group's chains +/// ([`GroupChainWires`]): `multi_verify`'s second loop +/// (`multilinear_table.rs:1254`), each group's opening +/// [`emit_stacked_verify_chains`]. +/// +/// The three counters are the host's own: `statement_at` advances by the +/// group's table count, `column_at` by the sum of those tables' COLUMN counts, +/// and the roots by `layout.num_polys()`. A group handed the wrong column slice +/// settles one table's values against another table's commitment, which is +/// precisely what a counter that advanced by tables rather than columns would +/// do. +pub fn emit_group_walk_chains( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + groups: &[GroupChainWires<'_>], + sizes: &[usize], + walk: &TableWalk, +) -> Vec { + assert_eq!(groups.len(), sizes.len(), "one size per commitment group"); + assert_eq!( + sizes.iter().sum::(), + walk.widths.len(), + "the groups' sizes must cover every table exactly once" + ); + let points = walk.column_points(); + let mut statement_at = 0usize; + let mut column_at = 0usize; + let mut gammas = Vec::with_capacity(groups.len()); + for (group, &size) in groups.iter().zip(sizes) { + let width: usize = walk.widths[statement_at..statement_at + size].iter().sum(); + gammas.push(emit_stacked_verify_chains( + b, + transcript, + group.layout, + group.chains, + &points[column_at..column_at + width], + &walk.values[column_at..column_at + width], + group.ood, + group.shape, + group.domain, + )); + statement_at += size; + column_at += width; + } + gammas +} + +// ============================================================================= +// The tables a verify settles by name +// ============================================================================= + +/// The four tables whose preprocessed columns a block carries, and the route +/// each one takes. +/// +/// ⛔ A TABLE WITH PREPROCESSED COLUMNS AND NONE OF THESE NAMES IS A REFUSAL, +/// not a skip. Skipping it would drop a `check_preprocessed` with nothing put in +/// its place, which is exactly what `settled_out_of_band` is documented never to +/// become (`multilinear_table.rs:943`). A fifth preprocessed table added +/// upstream must fail this build rather than go quietly unchecked. +pub const BITWISE_NAME: &str = "BITWISE"; +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) const DECODE_NAME: &str = "DECODE"; +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) const KECCAK_RC_NAME: &str = "KECCAK_RC"; +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) const REGISTER_NAME: &str = "REGISTER"; +/// What every CPU table's name starts with (`CPU[k]`): its TIMESTAMP takes the +/// closed-form route. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) const CPU_NAME_PREFIX: &str = "CPU["; + +/// One table's proof words, in the order a leaf hints them. +/// +/// ⚠ The order is a CONTRACT with [`hint_table_wires_shaped`]: a second +/// spelling of it is exactly the drift that hands the machine somebody else's +/// field element with every value gate still green. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn push_table_words( + words: &mut Vec, + table: &stark::multilinear_table::TableProof, +) { + let ext = super::word::ext_word; + words.push(ext(&table.bus_output.0)); + words.push(ext(&table.bus_output.1)); + for layer in &table.gkr.layers { + for round in &layer.sumcheck.rounds { + words.extend(round.evaluations.iter().map(ext)); + } + for value in [&layer.p_lo, &layer.p_hi, &layer.q_lo, &layer.q_hi] { + words.push(ext(value)); + } + } + for round in &table.constraint.sumcheck.rounds { + words.extend(round.evaluations.iter().map(ext)); + } + words.extend(table.constraint.factor_values.iter().map(ext)); + for round in &table.constraint.reduce.sumcheck.rounds { + words.extend(round.evaluations.iter().map(ext)); + } + words.extend(table.constraint.reduce.column_values.iter().map(ext)); +} + +/// One table's hinted wires, OWNED, because `TableProofWires` borrows them. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) struct TableWires { + bus_output: (Ext, Ext), + gkr: Vec, + sumcheck: Vec>, + factor_values: Vec, + reduce_sumcheck: Vec>, + column_values: Vec, +} + +impl TableWires { + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn borrow(&self) -> TableProofWires<'_> { + TableProofWires { + bus_output: self.bus_output, + gkr: &self.gkr, + sumcheck: &self.sumcheck, + factor_values: &self.factor_values, + reduce: super::whir_reduce::ReduceWires { + sumcheck: &self.reduce_sumcheck, + column_values: &self.column_values, + }, + } + } +} + +/// How many words [`push_table_words`] writes for a table of `shape`: the +/// counts [`hint_table_wires_shaped`] hints, from the shape alone. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn table_words(shape: &TableShape<'_>) -> usize { + use super::whir_gkr::GKR_SUMCHECK_DEGREE; + use super::whir_reduce::REDUCE_DEGREE; + let layers = shape.gkr_layers(); + let gkr: usize = (0..layers).map(|i| i * GKR_SUMCHECK_DEGREE + 4).sum(); + 2 + gkr + + shape.num_vars * shape.sumcheck_degree() + + shape.sources().len() + + shape.num_vars * REDUCE_DEGREE + + shape.num_columns +} + +/// A table's wires, hinted with every count taken from the table's SHAPE — the +/// AIR at the stated height — rather than from a proof: what an emitter that +/// derives its program from the statement alone hints. The counts are +/// `emit_table_verify`'s own (one GKR layer per input variable, layer `i` with +/// `i` rounds; one main round per row variable at the batch's degree; one factor +/// value per committed source; one reduce round per row variable; one value per +/// committed column), so a proof of another shape cannot be read into it. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn hint_table_wires_shaped( + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + at: &mut u32, + shape: &TableShape<'_>, +) -> TableWires { + use super::whir_gkr::GKR_SUMCHECK_DEGREE; + use super::whir_reduce::REDUCE_DEGREE; + let mut take = |b: &mut LfmBuilder, count: usize| -> Vec { + (0..count) + .map(|_| { + let wire = b.hint_word(arena, *at).as_ext(); + *at += 1; + wire + }) + .collect() + }; + let output = take(b, 2); + let gkr = (0..shape.gkr_layers()) + .map(|i| { + let sumcheck: Vec> = (0..i).map(|_| take(b, GKR_SUMCHECK_DEGREE)).collect(); + let halves = take(b, 4); + super::whir_gkr::GkrLayerWires { + sumcheck, + p_lo: halves[0], + p_hi: halves[1], + q_lo: halves[2], + q_hi: halves[3], + } + }) + .collect(); + let degree = shape.sumcheck_degree(); + let sumcheck: Vec> = (0..shape.num_vars).map(|_| take(b, degree)).collect(); + let factor_values = take(b, shape.sources().len()); + let reduce_sumcheck: Vec> = (0..shape.num_vars) + .map(|_| take(b, REDUCE_DEGREE)) + .collect(); + let column_values = take(b, shape.num_columns); + TableWires { + bus_output: (output[0], output[1]), + gkr, + sumcheck, + factor_values, + reduce_sumcheck, + column_values, + } +} + +/// A group's chains, hinted for a stack of `polys` polynomials under `shape`: one +/// chain per run of [`ChainShape::chains`], each at its run's shape — the count +/// and the shapes the verifier's own layout and format give rather than the +/// proof's. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn hint_group_chains_shaped( + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + at: &mut u32, + polys: usize, + shape: &ChainShape, +) -> GroupChains { + let runs = shape.chains(polys); + let mut finals = Vec::with_capacity(runs.len()); + let mut storage = Vec::with_capacity(runs.len()); + for run in &runs { + let shape = shape.run(run.len()); + finals.push(b.hint_word(arena, *at).as_ext()); + *at += 1; + storage.push(super::whir_chain::RoundStorage::hint(b, arena, *at, &shape)); + *at += super::whir_chain::RoundStorage::words(&shape); + } + GroupChains { + finals, + storage, + runs, + } +} + +/// One commitment group's hinted chains, OWNED for the same reason. +/// +/// ⛔ The ROOTS ARE NOT HERE. A group's roots are the cells the roots block +/// already hinted, and [`Self::polys`] takes them by reference: a second copy +/// would let a prover absorb one root into the transcript and open the chain +/// against another, with an honest arena looking identical to every value gate. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) struct GroupChains { + pub(crate) finals: Vec, + pub(crate) storage: Vec, + /// Each chain's polynomials, a range of the stack's ([`ChainShape::chains`]). + pub(crate) runs: Vec>, +} diff --git a/prover/src/lfm/whir_fold.rs b/prover/src/lfm/whir_fold.rs new file mode 100644 index 000000000..5aaf6d7f9 --- /dev/null +++ b/prover/src/lfm/whir_fold.rs @@ -0,0 +1,283 @@ +//! `fold_coset` as a machine leg — the arithmetic half of the query phase. +//! +//! `whir_commit::fold_coset` (`crypto/multilinear/src/whir_commit.rs:475-511`) +//! folds one opened block down to the single value it contributes, and the +//! verifier runs it once per query per round: at the round's own successor +//! block in `whir_round::verify` (`:159`) and against the final constant in +//! `verify_final` (`:1138`). The hashing half is [`super::whir_open`] beside +//! it. +//! +//! Each level halves the block by `fold_block_level` (`:434-465`): +//! +//! ```text +//! out[t] = ½·(a + b) + α·(½·x⁻¹)·(a − b), a = v[t], b = v[t + half] +//! ``` +//! +//! with `x` running over the coset: slot `t` sits at `g^p·η^t`, where `p` is +//! the block's position in the level's domain and `η` is that level's stride. +//! +//! # ★ Why only ONE exponentiation, and the identity that buys it +//! +//! `x` starts at `generator^position`, which is RUNTIME — `position` is the +//! query index the transcript drew — so it is built from the index's BITS by +//! [`super::edsl::pow_bits`], whose factors `g^(2^i)` are program constants. +//! +//! Every LATER level is free of a second exponentiation. The host squares the +//! domain and reduces the position into it (`:501-507`), so its point is +//! `(g²)^(p mod n/2)`; and +//! +//! ```text +//! (g²)^(p mod n/2) = g^(2p − k·n) = g^(2p) = (g^p)² +//! ``` +//! +//! because `g` has order `n` (`Domain::squared` squares the generator and drops +//! one from the log size, `whir.rs:50-58`). So the emitter squares the point it +//! already has, and a squaring is one row where a second `pow_bits` would be +//! `2·index_bits`. `η` is a program constant at every level, because the domain +//! and the block width are both emit-time. +//! +//! ⚠ The reduction reads as the hard part of that identity and is not part of +//! it at all: `Domain::new` takes a PRIMITIVE `2^log_size`-th root +//! (`whir.rs:36-47`), so each level's generator has order exactly that level's +//! size and `g_l^p = g_l^(p mod n_l)` for every `p`. The host's `position %=` +//! (`whir_commit.rs:502`, `:507`) keeps the exponent small for `pow` and +//! changes no value. Recorded because an earlier draft of this doc — and a test +//! written against it — treated the reduction as something the emitter had to +//! survive. +//! +//! # The base round +//! +//! Round 0's current codeword is base-field, and the host folds its first level +//! in the base field before the multiply by `α` lifts it. This emits that level +//! in the extension instead: the values agree exactly — a base element embedded +//! in the cubic extension is `(v, 0, 0)`, and every operation here is the +//! embedding of the host's — so it is a COST difference and not a value one. It +//! is worth naming rather than hiding: an `LFM_XALU` row is 18 base-equivalent +//! cells against `LFM_BALU`'s 10, so folding round 0's first level in the base +//! field would save `8 · (18 − 10)` cells a query and is a lever for the +//! optimisation rounds, not a correctness matter. +//! +//! ⚠ Lifting a HINTED base value is only sound because the block's leaf hash +//! pins its upper lanes to zero — see [`super::whir_open`]'s module doc, which +//! carries the obligation. + +use multilinear::whir::Domain; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::builder::{Bit, Ext, LfmBuilder}; +use super::edsl::{pow_bits, pow_bits_alu}; +use super::word::{LfmWord, base_word}; + +// ============================================================================ +// The lean fold +// ============================================================================ + +/// A round's folding challenges as the lean fold consumes them: `½·α` per level, +/// computed ONCE per round rather than once per query, plus the half itself. +pub struct LeanAlphas { + half_alphas: Vec, + half: super::builder::Felt, +} + +/// [`LeanAlphas`] for one round: one `MulBase` a level. +pub fn prepare_lean_alphas(b: &mut LfmBuilder, alphas: &[Ext]) -> LeanAlphas { + let half = b.felt_const(two_inv()); + LeanAlphas { + half_alphas: alphas.iter().map(|a| b.emul_base(*a, half)).collect(), + half, + } +} + +fn two_inv() -> FE { + (FE::one() + FE::one()) + .inv() + .expect("2 is invertible in Goldilocks") +} + +/// ★ `whir_commit::fold_coset`, emitted lean — the same value, in three +/// extension rows a folded value and none in the base field. +/// +/// Each level's output is rewritten around the SECOND value of its pair: +/// +/// ```text +/// ½·(a + c) + ½·α·x⁻¹·(a − c) = (a − c)·w + c, w = ½ + ½·α·x⁻¹ +/// ``` +/// +/// and along a level `x_t = x_0·η^t`, so `½·α·x_t⁻¹ = u·η^(−t)` with +/// `u = ½·α·x_0⁻¹` — ONE extension product a level. Per slot that leaves +/// `w = u·η^(−t) + ½` (a `MulAdd` against the program constant `η^(−t)`), +/// `a − c`, and the `MulAdd` that joins them: three `LFM_XALU` rows, where the +/// form above as written pays five plus a base `Div` for `½·x⁻¹` and a base +/// `Mul` to step `x`. No reciprocal is taken at all: `x_0⁻¹ = g^(−p)` comes +/// straight out of [`pow_bits`] over the INVERSE factors `g^(−2^i)`, and each +/// level after the first squares it (`(x²)⁻¹ = (x⁻¹)²`, the module doc's +/// identity inverted). +/// +/// Rows: `2·index_bits` (the point) + `levels − 1` (squarings) + `levels` (the +/// `u` of each level) + `3·(block − 1)`; see [`fold_coset_rows_lean`]. The +/// `½·α` of [`prepare_lean_alphas`] are per ROUND and not counted here. +pub fn emit_fold_coset_lean( + b: &mut LfmBuilder, + values: &[Ext], + domain: &Domain, + index_bits: &[Bit], + prepared: &LeanAlphas, +) -> Ext { + let levels = prepared.half_alphas.len(); + assert_eq!( + values.len(), + 1usize << levels, + "a block holds one value per folded variable" + ); + if levels == 0 { + return values[0]; + } + + // `x_0⁻¹ = g^(−p)`, from the index's bits against the inverse factors. + let generator_inv = domain + .generator() + .inv() + .expect("a domain generator is nonzero"); + let factors: Vec = (0..index_bits.len()) + .map(|i| generator_inv.pow(1u64 << i)) + .collect(); + // A Poseidon1 leaf's `Select` table binds its leaves (I-WHIR-P1 §S5.4): its + // point reads the bits through the base ALU instead. + let mut x_inv = match b.wrap_hash() { + super::edsl::WrapHash::Poseidon1 => pow_bits_alu(b, index_bits, &factors), + _ => pow_bits(b, index_bits, &factors, FE::one()), + }; + let half = prepared.half.as_ext(); + + let mut current: Vec = values.to_vec(); + let mut current_domain = domain.clone(); + for (level, half_alpha) in prepared.half_alphas.iter().enumerate() { + if level > 0 { + x_inv = b.mul(x_inv, x_inv); + } + let pairs = current.len() / 2; + // This level's stride `η`, inverted: `η^(−t)` is a program constant per + // slot, so the point never steps at run time. + let eta_inv = current_domain + .generator() + .pow((current_domain.size() / current.len()) as u64) + .inv() + .expect("a coset stride is nonzero"); + let u = b.emul_base(*half_alpha, x_inv); + + let mut next = Vec::with_capacity(pairs); + for t in 0..pairs { + let (a, c) = (current[t], current[t + pairs]); + let w = if t == 0 { + b.eadd(u, half) + } else { + let step = b.felt_const(eta_inv.pow(t as u64)); + b.emul_add(u, step.as_ext(), half) + }; + let difference = b.esub(a, c); + next.push(b.emul_add(difference, w, c)); + } + current = next; + current_domain = current_domain + .squared() + .expect("a domain of at least two points squares"); + } + + current[0] +} + +/// INSTRUCTIONS [`emit_fold_coset_lean`] emits for a block of `block` values +/// with a query index of `index_bits` bits: +/// +/// ```text +/// 2·index_bits + 3·(block − 1) + 2·levels − 1 +/// ``` +/// +/// `2·index_bits` for the point, three rows an output slot (`block − 1` of them +/// across the levels), one `u` a level and one squaring a level after the first. +pub const fn fold_coset_rows_lean(block: usize, index_bits: usize) -> usize { + if block <= 1 { + // `fold_coset` returns `values[0]` lifted when there is nothing to fold + // and the emitter returns the wire. No chain reaches it — a round folds + // at least one variable — and it is stated rather than left to + // underflow `block − 1`. + return 0; + } + let levels = block.trailing_zeros() as usize; + 2 * index_bits + 3 * (block - 1) + 2 * levels - 1 +} + +/// Rows [`prepare_lean_alphas`] emits for a round of `levels` levels: one +/// `MulBase` each. +pub const fn lean_prepare_rows(levels: usize) -> usize { + levels +} + +/// ★ THE VALUES ONE LEAN FOLD INTERNS, deduplicated by value: `g^(−e)` for +/// `e` in `{0} ∪ {2^i : i < index_bits}` (the scale and the inverse factors of +/// the point) and `{s·N/block : 1 ≤ s < block/2}` (every `η_l^(−t)` of every +/// level is one of level 0's `η_0^(−s)`, since `η_l = η_0^(2^l)`), plus the +/// half. +/// +/// ⛔ A program's fold pool is a UNION over its distinct domains, never a +/// maximum: the same exponents under a different generator are different field +/// elements. The builder interns on the canonical word, so two folds sharing a +/// value pay for it once. +pub fn fold_coset_constants_lean( + domain: &Domain, + levels: usize, + index_bits: usize, +) -> Vec { + if levels == 0 { + return Vec::new(); + } + let n = domain.size() as u128; + let block = 1u128 << levels; + let generator_inv = domain + .generator() + .inv() + .expect("a domain generator is nonzero"); + let mut exponents = vec![0u128]; + for i in 0..index_bits { + exponents.push((1u128 << i) % n); + } + for s in 1..block / 2 { + exponents.push((s * (n / block)) % n); + } + let mut words: Vec = Vec::new(); + for e in exponents { + let word = base_word(generator_inv.pow(e as u64)); + if !words.contains(&word) { + words.push(word); + } + } + let half = base_word(two_inv()); + if !words.contains(&half) { + words.push(half); + } + words +} + +/// A round's fold, prepared once per round and applied once per query: the +/// round's `½·α` ([`prepare_lean_alphas`]), then [`emit_fold_coset_lean`] a +/// query. +pub struct RoundFold(LeanAlphas); + +impl RoundFold { + /// Prepare the round's challenges: one `MulBase` a level. + pub fn prepare(b: &mut LfmBuilder, alphas: &[Ext]) -> Self { + RoundFold(prepare_lean_alphas(b, alphas)) + } + + /// Fold one opened block at one query. + pub fn fold( + &self, + b: &mut LfmBuilder, + values: &[Ext], + domain: &Domain, + index_bits: &[Bit], + ) -> Ext { + emit_fold_coset_lean(b, values, domain, index_bits, &self.0) + } +} diff --git a/prover/src/lfm/whir_fold_tests.rs b/prover/src/lfm/whir_fold_tests.rs new file mode 100644 index 000000000..17673890a --- /dev/null +++ b/prover/src/lfm/whir_fold_tests.rs @@ -0,0 +1,265 @@ +//! Gates for the coset fold. + +use multilinear::whir::Domain; +use multilinear::whir_commit::fold_coset; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::{Bit, Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_fold::{ + emit_fold_coset_lean, fold_coset_constants_lean, fold_coset_rows_lean, lean_prepare_rows, + prepare_lean_alphas, +}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +fn fee(v: u64) -> FEE { + FEE::new([ + FE::from(v.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 7), + FE::from(v ^ 0x1234), + FE::from(v.wrapping_add(5)), + ]) +} + +fn fe(v: u64) -> FE { + FE::from(v.wrapping_mul(0xD1B5_4A32_D192_ED03) >> 5) +} + +/// Rows the plumbing contributes and the leg does not: the hints, the one +/// `BitDec`, and the public. Counted by construction. +fn fold_plumbing(block: usize, levels: usize) -> usize { + (block + levels + 1) + 1 + 1 +} + +fn fold_arena(values: &[LfmWord], alphas: &[FEE], index: usize) -> Vec { + let mut arena = values.to_vec(); + arena.extend(alphas.iter().map(ext_word)); + arena.push([FE::from(index as u64), FE::zero(), FE::zero(), FE::zero()]); + arena +} + +fn const_rows(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|instr| matches!(instr, super::instr::Instr::Const { .. })) + .count() +} + +/// `(log_domain, levels, index_bits)`. +/// +/// The first four are the shapes a chain reaches: the index is the BLOCK index, +/// bounded by `domain.size() >> log_folding`, so `index_bits = log_domain − +/// levels` — the production fold of four at two tree sizes, the `S mod k` tail +/// round's fold of one, and a fold of two. +/// +/// The fifth is deliberately off that relation. `2·index_bits` is the only term +/// that moves with the index width, so pinning it needs two widths at the SAME +/// block: `(8, 4, 4)` and `(8, 4, 6)` are that pair, and a form that folded the +/// index term into the block term would fit one and miss the other. +/// +/// The last two are W2's first folds (`first5`, `first6`): blocks of 32 and 64, +/// on the chain relation, the widest folds the stack runs (`MAX_FOLD`). +const SHAPES: &[(usize, usize, usize)] = &[ + (5, 1, 4), + (6, 2, 4), + (8, 4, 4), + (10, 4, 6), + (8, 4, 6), + (12, 5, 7), + (13, 6, 7), +]; + +/// The block's values and the folding challenges hinted, the query index's bits +/// hinted as ONE word and decomposed in one `BitDec` — which is exactly what +/// `WhirTranscript::sample_u64_pow2` emits, so the assembled chain hands these +/// bits over rather than rebuilding them — then the round's `½·α` prepared +/// once and one lean fold. +fn lean_fold_program(log_domain: usize, levels: usize, index_bits: usize) -> LfmProgram { + let block = 1usize << levels; + let domain = Domain::::new(log_domain).expect("a domain of that size"); + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena((block + levels + 1) as u32); + + let values: Vec = (0..block) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let alphas: Vec = (0..levels) + .map(|i| b.hint_word(arena, (block + i) as u32).as_ext()) + .collect(); + let index = b.hint_felt(arena, (block + levels) as u32); + let bits: Vec = b.bit_dec(index, index_bits); + + let prepared = prepare_lean_alphas(&mut b, &alphas); + let folded = emit_fold_coset_lean(&mut b, &values, &domain, &bits, &prepared); + b.public(folded.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the lean fold leg must be admissible"); + program +} + +/// ★ The lean fold's F1: rows and interned constants, by count and by value. +#[test] +fn the_lean_fold_emits_its_closed_form() { + for &(log_domain, levels, index_bits) in SHAPES { + let block = 1usize << levels; + let program = lean_fold_program(log_domain, levels, index_bits); + let constants = const_rows(&program); + let measured = program.instrs.len() + - fold_plumbing(block, levels) + - constants + - lean_prepare_rows(levels); + assert_eq!( + measured, + fold_coset_rows_lean(block, index_bits), + "a lean block of {block} on 2^{log_domain} at {index_bits} index bits" + ); + let named = fold_coset_constants_lean( + &Domain::::new(log_domain).expect("a domain"), + levels, + index_bits, + ); + let interned: Vec = program + .instrs + .iter() + .filter_map(|i| match i { + super::instr::Instr::Const { value, .. } => Some(*value), + _ => None, + }) + .collect(); + assert_eq!(named.len(), interned.len(), "a lean block of {block}"); + for word in &named { + assert!( + interned.contains(word), + "a lean block of {block}: named, not interned" + ); + } + for word in &interned { + assert!( + named.contains(word), + "a lean block of {block}: interned, not named" + ); + } + } +} + +/// ★ The lean leg computes what `whir_commit::fold_coset` computes — over +/// extension blocks, over round 0's base blocks, and at the widest indices. +#[test] +fn the_lean_fold_computes_what_the_host_computes() { + for &(log_domain, levels, index_bits) in SHAPES { + let block = 1usize << levels; + let domain = Domain::::new(log_domain).expect("a domain"); + let program = lean_fold_program(log_domain, levels, index_bits); + for seed in [0x21u64, 0x63] { + let alphas: Vec = (0..levels).map(|i| fee(seed * 91 + 1 + i as u64)).collect(); + let ext_values: Vec = (0..block).map(|i| fee(seed * 17 + i as u64)).collect(); + let base_values: Vec = (0..block).map(|i| fe(seed * 13 + i as u64)).collect(); + for index in [0usize, 1, 3, (1 << index_bits) - 1] { + let want_ext = fold_coset::(&ext_values, &domain, index, &alphas) + .expect("the host folds the block"); + let words: Vec = ext_values.iter().map(ext_word).collect(); + let exec = execute( + &program, + &[fold_arena(&words, &alphas, index)], + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("the lean fold executes"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("a value"), + want_ext, + "ext block {block} on 2^{log_domain}, index {index}, seed {seed:#x}" + ); + + let want_base = fold_coset::(&base_values, &domain, index, &alphas) + .expect("the host folds the base block"); + let words: Vec = base_values + .iter() + .map(|v| [*v, FE::zero(), FE::zero(), FE::zero()]) + .collect(); + let exec = execute( + &program, + &[fold_arena(&words, &alphas, index)], + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("the lean fold executes over a base block"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("a value"), + want_base, + "base block {block} on 2^{log_domain}, index {index}, seed {seed:#x}" + ); + } + } + } + // The deepest fold at indices spanning the whole domain. + let (log_domain, levels, index_bits) = (6usize, 4usize, 6usize); + let block = 1usize << levels; + let domain = Domain::::new(log_domain).expect("a domain"); + let program = lean_fold_program(log_domain, levels, index_bits); + let values: Vec = (0..block).map(|i| fee(0x99 + i as u64)).collect(); + let alphas: Vec = (0..levels).map(|i| fee(0x5150 + i as u64)).collect(); + let words: Vec = values.iter().map(ext_word).collect(); + let half = 1usize << (log_domain - 1); + for index in [half - 1, half, half + 1, (1 << log_domain) - 1] { + let want = fold_coset::(&values, &domain, index, &alphas).expect("host"); + let exec = execute( + &program, + &[fold_arena(&words, &alphas, index)], + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("the lean fold executes"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("a value"), + want, + "the lean point chain at index {index}" + ); + } +} + +/// ★ Load-bearing, not incidental: one changed opened value or one changed +/// challenge moves the lean fold's output off the host's fold of the originals. +#[test] +fn the_lean_fold_moves_with_every_input() { + let (log_domain, levels, index_bits) = (8usize, 4usize, 4usize); + let block = 1usize << levels; + let domain = Domain::::new(log_domain).expect("a domain"); + let program = lean_fold_program(log_domain, levels, index_bits); + let values: Vec = (0..block).map(|i| fee(0x777 + i as u64)).collect(); + let alphas: Vec = (0..levels).map(|i| fee(0x888 + i as u64)).collect(); + let index = 5usize; + let honest = fold_coset::(&values, &domain, index, &alphas).expect("host"); + let run = |values: &[FEE], alphas: &[FEE]| -> FEE { + let words: Vec = values.iter().map(ext_word).collect(); + let exec = execute( + &program, + &[fold_arena(&words, alphas, index)], + &crate::hash_pin::BLOCK_HASHER, + ) + .expect("the lean fold executes"); + word_as_ext(&exec.public_words[0].1).expect("a value") + }; + assert_eq!(run(&values, &alphas), honest); + for slot in 0..block { + let mut tampered = values.clone(); + tampered[slot] += FEE::one(); + assert_ne!( + run(&tampered, &alphas), + honest, + "value {slot} must reach the output" + ); + } + for level in 0..levels { + let mut tampered = alphas.clone(); + tampered[level] += FEE::one(); + assert_ne!( + run(&values, &tampered), + honest, + "alpha {level} must reach the output" + ); + } +} diff --git a/prover/src/lfm/whir_gkr.rs b/prover/src/lfm/whir_gkr.rs new file mode 100644 index 000000000..606910698 --- /dev/null +++ b/prover/src/lfm/whir_gkr.rs @@ -0,0 +1,164 @@ +//! The LogUp-GKR closure as a machine leg: `gkr::verify`, layer by layer. +//! +//! This is the WHIR verifier's largest fixed term per epoch — one tree per +//! table, `m = n + ceil_log2(I)` layers each — and it is where the degree-3 +//! sumcheck round and `eq` earn their keep, so it is built on top of them +//! rather than beside them. + +use super::builder::{Ext, LfmBuilder}; +use super::whir_poly::{ + emit_eq_eval, emit_sumcheck_rounds, eq_eval_rows_again, sumcheck_round_consts, + sumcheck_round_rows, +}; + +/// The degree the GKR ladder's sumchecks run at (`gkr.rs:522`). +pub const GKR_SUMCHECK_DEGREE: usize = 3; + +/// One layer's proof values as wires: the sumcheck's round evaluations and the +/// four bound values of the layer relation. +pub struct GkrLayerWires { + /// `i` rounds of `GKR_SUMCHECK_DEGREE` evaluations at layer `i`. + pub sumcheck: Vec>, + pub p_lo: Ext, + pub p_hi: Ext, + pub q_lo: Ext, + pub q_hi: Ext, +} + +/// The challenges one layer consumes, in the order `gkr::verify` draws them: +/// the batching λ, one per sumcheck round, then the halves' folding point. +pub struct GkrLayerChallenges { + pub lambda: Ext, + pub rounds: Vec, + pub c: Ext, +} + +/// What the ladder leaves: the input layer's claim. +pub struct GkrClaimWires { + pub point: Vec, + pub p: Ext, + pub q: Ext, +} + +/// INSTRUCTIONS layer `i` emits — the layer whose point carries `i` variables, +/// which is the `i`th from the output. +/// +/// The `12` that does not depend on `i`, each term by its shape: one `MulAdd` +/// for `claimed_sum = p + λ·q`; two for the numerator `p_lo·q_hi + p_hi·q_lo` +/// (a `Mul` then a `MulAdd`); one for the denominator `q_lo·q_hi`; one `MulAdd` +/// for `numerator + λ·denominator`; one `Mul` by `eq_at`; **two for the +/// layer-relation assert**, which lowers to `diff = a − b; _ = diff / 0` and is +/// the machine's `LayerRelationMismatch`; and four for the two +/// `combine_halves`, a `Sub` and a `MulAdd` each. +/// +/// The rest is `i` degree-3 sumcheck rounds and one `eq` over `i` variables. +/// +/// ⚠ At layer 0 the `eq` is the empty product and the `Mul` by it is a wasted +/// row — about 34 rows an epoch across the tables. It is emitted anyway, so the +/// shape stays uniform with the host's and the form has no special case. +pub const fn gkr_layer_rows(i: usize) -> usize { + let fixed = 12; + fixed + i * sumcheck_round_rows(GKR_SUMCHECK_DEGREE) + eq_eval_rows_again(i) +} + +/// INSTRUCTIONS a whole ladder of `layers` layers emits, constants excluded. +/// +/// Layer `i` carries a point of `i` variables and a sumcheck of `i` rounds, so +/// the ladder's shape follows from its length alone. +pub fn gkr_verify_rows(layers: usize) -> usize { + (0..layers).map(gkr_layer_rows).sum() +} + +/// `LFM_CONST` rows the ladder interns, once per program: `eq`'s `1`, the +/// assert's `0`, and — only once a layer actually runs a sumcheck round, which +/// takes two layers — the degree-3 round's Newton constants. +pub const fn gkr_verify_consts(layers: usize) -> usize { + let eq_one_and_assert_zero = 2; + let newton = if layers >= 2 { + sumcheck_round_consts(GKR_SUMCHECK_DEGREE) + } else { + 0 + }; + eq_one_and_assert_zero + newton +} + +/// ★ `gkr::verify` (`crypto/multilinear/src/gkr.rs:505-546`), emitted. +/// +/// Per layer: draw λ and form the batched claim `p + λ·q`; run the layer's +/// sumcheck; check the residual against the layer relation +/// +/// ```text +/// eq(point, r) · (p_lo·q_hi + p_hi·q_lo + λ·q_lo·q_hi) +/// ``` +/// +/// absorb the four values and draw `c`; and fold each half at `c`. The point +/// the next layer carries is `c` followed by this layer's sumcheck challenges, +/// which is why layer `i`'s sumcheck has exactly `i` rounds. +/// +/// ⚠ The challenges are INPUTS, as in [`super::whir_poly::emit_sumcheck_round`] +/// — this leg neither absorbs nor draws. What the replay owes it, in order: +/// draw λ; per round absorb the three evaluations then draw; absorb `p_lo`, +/// `p_hi`, `q_lo`, `q_hi` in that order; draw `c`. +/// +/// ★ The layer-relation check is a REAL REJECT, and the only one in the legs so +/// far: `assert_eq_ext` has no satisfying assignment when the two differ +/// (`builder.rs:289-292`), so a proof whose relation fails cannot be executed, +/// let alone proven. +pub fn emit_gkr_verify( + b: &mut LfmBuilder, + output: (Ext, Ext), + layers: &[GkrLayerWires], + challenges: &[GkrLayerChallenges], +) -> GkrClaimWires { + assert_eq!( + layers.len(), + challenges.len(), + "every GKR layer draws its own challenges" + ); + let (mut p_claim, mut q_claim) = output; + let mut point: Vec = Vec::new(); + + for (layer, drawn) in layers.iter().zip(challenges) { + assert_eq!( + layer.sumcheck.len(), + point.len(), + "layer {}'s sumcheck runs over the point it carries", + point.len() + ); + assert_eq!( + drawn.rounds.len(), + point.len(), + "one challenge per sumcheck round" + ); + + let claimed_sum = b.emul_add(drawn.lambda, q_claim, p_claim); + let residual = emit_sumcheck_rounds(b, claimed_sum, &layer.sumcheck, &drawn.rounds); + + // The sumcheck's point IS the challenges it drew. + let eq_at = emit_eq_eval(b, &point, &drawn.rounds); + let cross = b.emul(layer.p_lo, layer.q_hi); + let numerator = b.emul_add(layer.p_hi, layer.q_lo, cross); + let denominator = b.emul(layer.q_lo, layer.q_hi); + let inner = b.emul_add(drawn.lambda, denominator, numerator); + let expected = b.emul(eq_at, inner); + b.assert_eq_ext(expected, residual); + + p_claim = emit_combine_halves(b, layer.p_lo, layer.p_hi, drawn.c); + q_claim = emit_combine_halves(b, layer.q_lo, layer.q_hi, drawn.c); + point = std::iter::once(drawn.c) + .chain(drawn.rounds.iter().copied()) + .collect(); + } + + GkrClaimWires { + point, + p: p_claim, + q: q_claim, + } +} + +/// `combine_halves(lo, hi, c) = lo + c·(hi − lo)` (`gkr.rs:384-390`). +fn emit_combine_halves(b: &mut LfmBuilder, lo: Ext, hi: Ext, c: Ext) -> Ext { + let spread = b.esub(hi, lo); + b.emul_add(c, spread, lo) +} diff --git a/prover/src/lfm/whir_gkr_tests.rs b/prover/src/lfm/whir_gkr_tests.rs new file mode 100644 index 000000000..3724d1056 --- /dev/null +++ b/prover/src/lfm/whir_gkr_tests.rs @@ -0,0 +1,317 @@ +//! Gates for the GKR closure. +//! +//! The proof under test is a real one: `gkr::prove` over a real `FractionTree`, +//! so the layer relation actually holds and the emitted assert is exercised on +//! a proof that passes as well as on one that must not. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use multilinear::gkr::{FractionLayer, FractionTree, GkrProof, prove, verify}; +use multilinear::mle::Mle; + +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +use super::builder::{Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_gkr::{ + GKR_SUMCHECK_DEGREE, GkrLayerChallenges, GkrLayerWires, emit_gkr_verify, gkr_verify_consts, + gkr_verify_rows, +}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +/// A transcript that records what it hands out. +/// +/// The GKR verifier's challenges are drawn interleaved with its sumchecks and +/// never returned, and re-deriving them would be a mirror of the host rather +/// than a reading of it. This observes instead: every call is delegated, and +/// `sample_field_element`'s answers are kept in order. +struct Recording { + inner: DefaultTranscript, + sampled: Vec, +} + +impl Recording { + fn new(label: &[u8]) -> Self { + Self { + inner: DefaultTranscript::::new(label), + sampled: Vec::new(), + } + } +} + +impl IsTranscript for Recording { + fn append_field_element(&mut self, element: &FEE) { + self.inner.append_field_element(element); + } + fn append_bytes(&mut self, new_bytes: &[u8]) { + self.inner.append_bytes(new_bytes); + } + fn state(&self) -> [u8; 32] { + self.inner.state() + } + fn sample_field_element(&mut self) -> FEE { + let drawn = self.inner.sample_field_element(); + self.sampled.push(drawn); + drawn + } + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + self.inner.sample_u64(upper_bound) + } +} + +fn fee(v: u64) -> FEE { + FEE::new([ + FE::from(v), + FE::from(v ^ 0x9E37), + FE::from(v.wrapping_mul(31)), + ]) +} + +/// A fraction tree over `num_vars` variables, with no zero denominator. +fn tree(num_vars: usize, seed: u64) -> FractionTree { + let size = 1usize << num_vars; + let p = Mle::new((0..size).map(|i| fee(seed + i as u64)).collect()).expect("p is a cube"); + let q = Mle::new( + (0..size) + .map(|i| fee(seed.wrapping_mul(7) + 1 + i as u64)) + .collect(), + ) + .expect("q is a cube"); + FractionTree::build(FractionLayer::new(p, q).expect("equal widths")).expect("the tree builds") +} + +/// The arena the ladder reads: the output fraction, then per layer its +/// sumcheck evaluations, its four bound values, λ, its round challenges and +/// `c`. The program below hints in exactly this order. +fn gkr_arena_words(layers: usize) -> usize { + 2 + (0..layers) + .map(|i| i * GKR_SUMCHECK_DEGREE + 4 + 1 + i + 1) + .sum::() +} + +/// Publics: `p`, `q`, and the point the ladder leaves. +fn gkr_public_words(layers: usize) -> usize { + 2 + if layers == 0 { 0 } else { layers } +} + +fn gkr_program(layers: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(gkr_arena_words(layers) as u32); + let mut idx = 0u32; + let next = |b: &mut LfmBuilder, idx: &mut u32| -> Ext { + let cell = b.hint_word(arena, *idx).as_ext(); + *idx += 1; + cell + }; + + let output_p = next(&mut b, &mut idx); + let output_q = next(&mut b, &mut idx); + + let mut wires = Vec::with_capacity(layers); + let mut drawn = Vec::with_capacity(layers); + for layer in 0..layers { + let mut sumcheck = Vec::with_capacity(layer); + for _ in 0..layer { + let mut round = Vec::with_capacity(GKR_SUMCHECK_DEGREE); + for _ in 0..GKR_SUMCHECK_DEGREE { + round.push(next(&mut b, &mut idx)); + } + sumcheck.push(round); + } + let p_lo = next(&mut b, &mut idx); + let p_hi = next(&mut b, &mut idx); + let q_lo = next(&mut b, &mut idx); + let q_hi = next(&mut b, &mut idx); + let lambda = next(&mut b, &mut idx); + let mut rounds = Vec::with_capacity(layer); + for _ in 0..layer { + rounds.push(next(&mut b, &mut idx)); + } + let c = next(&mut b, &mut idx); + wires.push(GkrLayerWires { + sumcheck, + p_lo, + p_hi, + q_lo, + q_hi, + }); + drawn.push(GkrLayerChallenges { lambda, rounds, c }); + } + + let claim = emit_gkr_verify(&mut b, (output_p, output_q), &wires, &drawn); + b.public(claim.p.as_cell()); + b.public(claim.q.as_cell()); + for coordinate in &claim.point { + b.public(coordinate.as_cell()); + } + + let program = compile(b.finish()); + validate(&program).expect("the GKR leg must be admissible"); + program +} + +/// The arena values, in the order [`gkr_program`] hints them. `sampled` is the +/// recorder's challenge stream, which `gkr::verify` draws as λ, then one per +/// sumcheck round, then `c`, layer by layer. +fn gkr_arena( + output: (FEE, FEE), + proof: &GkrProof, + sampled: &[FEE], +) -> Vec { + let mut words = vec![ext_word(&output.0), ext_word(&output.1)]; + let mut drawn = sampled.iter(); + for (layer, proof_layer) in proof.layers.iter().enumerate() { + for round in &proof_layer.sumcheck.rounds { + for value in &round.evaluations { + words.push(ext_word(value)); + } + } + for value in [ + &proof_layer.p_lo, + &proof_layer.p_hi, + &proof_layer.q_lo, + &proof_layer.q_hi, + ] { + words.push(ext_word(value)); + } + // λ, one challenge per round, then c. + for _ in 0..(layer + 2) { + words.push(ext_word(drawn.next().expect("the recorder saw every draw"))); + } + } + assert!( + drawn.next().is_none(), + "the recorder's stream must be exactly the ladder's challenges" + ); + words +} + +/// ★ F1 for the ladder: every layer's rows, against the closed form. +#[test] +fn the_gkr_ladder_emits_its_closed_form() { + for layers in [1usize, 2, 3, 6, 12] { + let program = gkr_program(layers); + let measured = program.instrs.len() - gkr_arena_words(layers) - gkr_public_words(layers); + let predicted = gkr_verify_rows(layers) + gkr_verify_consts(layers); + println!( + "gkr over {layers:>2} layers: {measured:>5} rows emitted, {predicted:>5} predicted" + ); + assert_eq!( + measured, predicted, + "a {layers}-layer ladder must emit its closed form" + ); + } +} + +/// ★ The leg computes what `gkr::verify` computes — all three parts of the +/// claim it returns, on a proof `gkr::prove` actually produced. +#[test] +fn the_gkr_leg_computes_what_the_host_computes() { + for num_vars in [1usize, 2, 4] { + let tree = tree(num_vars, 0x1234); + let output = tree.output(); + + let mut proving = Recording::new(b"v1-gkr-gate"); + let proved = prove(&tree, &mut proving).expect("the tree proves"); + + let mut verifying = Recording::new(b"v1-gkr-gate"); + let host = verify(&proved.proof, output, &mut verifying).expect("the host accepts"); + + let layers = proved.proof.layers.len(); + let program = gkr_program(layers); + let arena = gkr_arena(output, &proved.proof, &verifying.sampled); + let exec = execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER) + .expect("the GKR leg executes on a proof the host accepts"); + + let published: Vec = exec + .public_words + .iter() + .map(|(_, word)| word_as_ext(word).expect("a published extension value")) + .collect(); + assert_eq!(published.len(), gkr_public_words(layers)); + assert_eq!( + published[0], host.p, + "{num_vars} variables: the p the ladder leaves" + ); + assert_eq!( + published[1], host.q, + "{num_vars} variables: the q the ladder leaves" + ); + assert_eq!( + published[2..], + host.point[..], + "{num_vars} variables: the point the ladder leaves" + ); + } +} + +/// ★ THE TAMPER ARM. One corrupted proof value and the emitted program cannot +/// execute — the layer-relation assert has no satisfying assignment. +/// +/// The forged arena keeps the HONEST challenge stream, which is the sharper +/// model of what a forger can do here: every input is hinted, so nothing stops +/// one from leaving the transcript alone and changing a proof value. (The host +/// cannot be asked for the forgery's challenges at all — it rejects at the +/// tampered layer and stops drawing, which is how this arm first failed.) +/// +/// Three halves matter, not two: the untouched proof must EXECUTE, or the arm +/// is passing because the program never runs; the host must REJECT the same +/// forged proof under its own transcript, so the arm is about a proof that is +/// genuinely bad; and the machine must refuse it. +#[test] +fn a_corrupted_proof_value_cannot_execute() { + let tree = tree(3, 0x99); + let output = tree.output(); + let mut proving = Recording::new(b"v1-gkr-tamper"); + let proved = prove(&tree, &mut proving).expect("the tree proves"); + + let mut verifying = Recording::new(b"v1-gkr-tamper"); + verify(&proved.proof, output, &mut verifying).expect("the control proof must verify"); + + let layers = proved.proof.layers.len(); + // The corruption sites below are `[0, 1, layers − 1]`, which is three + // DISTINCT layers only while the ladder has at least three. At two they + // collapse to `[0, 1, 1]`: the loop still runs six times, still passes, and + // silently stops covering the last layer — the coverage would be a property + // of the fixture rather than of the test. `tree(3, ..)` gives four layers + // today; this is what says so, and what fires if the fixture shrinks. + assert!( + layers >= 3, + "the tamper arm needs at least three layers for [0, 1, layers−1] to be three \ + distinct sites; this ladder has {layers}" + ); + let program = gkr_program(layers); + let honest = gkr_arena(output, &proved.proof, &verifying.sampled); + assert!( + execute(&program, &[honest], &crate::hash_pin::BLOCK_HASHER).is_ok(), + "the untouched proof must execute, or the arm below proves nothing" + ); + + for layer in [0usize, 1, layers - 1] { + for which in ["p_lo", "a sumcheck evaluation"] { + let mut forged = proved.proof.clone(); + match which { + "p_lo" => forged.layers[layer].p_lo += FEE::one(), + _ => match forged.layers[layer].sumcheck.rounds.first_mut() { + Some(round) => round.evaluations[0] += FEE::one(), + // Layer 0 has no sumcheck round to corrupt. + None => continue, + }, + } + + let mut fresh = Recording::new(b"v1-gkr-tamper"); + assert!( + verify(&forged, output, &mut fresh).is_err(), + "layer {layer}, {which}: the host must reject the forgery" + ); + + let arena = gkr_arena(output, &forged, &verifying.sampled); + assert!( + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).is_err(), + "layer {layer}, {which}: the machine must refuse the forgery too" + ); + } + } +} diff --git a/prover/src/lfm/whir_lean_tests.rs b/prover/src/lfm/whir_lean_tests.rs new file mode 100644 index 000000000..3f7f76b0c --- /dev/null +++ b/prover/src/lfm/whir_lean_tests.rs @@ -0,0 +1,602 @@ +//! Gates for the leaf's cheaper query check ([`LeafQuery::Lean`]) on Poseidon1 +//! chains: the tail rounds checked at every leaf once +//! ([`ChainShape::tail_round`]), round 0's values four a word +//! ([`ChainShape::round0_packed`]) and the paths walked in three `Select`s a +//! level on rotated hints ([`ChainShape::paths_rotated`]), against the host's +//! own proofs. +//! +//! The fixtures are `whir_p1_leaf_tests`' small chains with enough queries that +//! their last trees, lying wholly in their caps, take [`TAIL_QUERIES_A_LEAF`] +//! queries a leaf. + +use multilinear::whir_chain::{CapPolicy, ChainConfig, ChainProof, LeafOrder, RoundOpenings}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::commitment_to_digest; +use super::edsl::WrapHash; +use super::executor::execute; +use super::hash::HasherKind; +use super::whir_chain::{ + ChainShape, LeafQuery, TAIL_QUERIES_A_LEAF, chain_perms, round_words, tail_faults, + try_push_round_words, +}; +use super::whir_chain_tests::hint_rows; +use super::whir_p1_leaf_tests::{ + Fixture, fixture, hash16_rows, host_accepts, p1_chain_program, p1_config, p1_ordered_config, +}; +use super::whir_transcript::SpongeEntry; +use super::word::{LfmWord, ext_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +const SOCKET: HasherKind = HasherKind::Poseidon1W16; + +/// `shape` under [`LeafQuery::Lean`]. +fn lean(shape: &ChainShape) -> ChainShape { + ChainShape { + query: LeafQuery::Lean, + ..shape.clone() + } +} + +/// The chain's arena laid out at `shape` (the fixture's own or its lean +/// twin), as `whir_p1_leaf_tests::p1_chain_arena` lays it at the fixture's. +fn arena_at( + f: &Fixture, + proof: &ChainProof, + shape: &ChainShape, +) -> Result, String> { + let mut words: Vec = f.z.iter().map(ext_word).collect(); + words.push(ext_word(&f.y)); + words.push(commitment_to_digest(&f.root_bytes)); + words.push(ext_word(&proof.final_value)); + try_push_round_words(&mut words, shape, proof)?; + Ok(words) +} + +/// Chains with tail rounds: uniform folds of 4 at the C2 cap (rounds 1 and 2, +/// 8 and 4 leaves, at 192 queries; round 2 alone at 96), and the production's +/// first fold of 6 under the production cap (its last round). +fn tail_cases() -> Vec<(usize, ChainConfig)> { + vec![ + (9, p1_config(192, 0, CapPolicy::Fixed(2))), + (9, p1_config(96, 6, CapPolicy::Fixed(2))), + ( + 10, + p1_ordered_config(96, 0, CapPolicy::Auto, Some(6), LeafOrder::Coset), + ), + ( + 11, + p1_ordered_config(100, 6, CapPolicy::Auto, Some(6), LeafOrder::Coset), + ), + ] +} + +/// The tail rounds of `shape`, which must have some. +fn tails(shape: &ChainShape) -> Vec { + let tails: Vec = (0..shape.rounds()) + .filter(|&r| shape.tail_round(r)) + .collect(); + assert!( + !tails.is_empty(), + "depths {:?} caps {:?} at {} queries: no tail round", + (0..shape.rounds()) + .map(|r| shape.current_depth(r)) + .collect::>(), + shape.caps, + shape.num_queries + ); + tails +} + +/// The arena offset of tail round `r`'s first block: the fixture's prefix +/// (`z`, `y`, the root, the final value), the rounds before, then round `r`'s +/// sumcheck words and nonces. +fn tail_block_at(shape: &ChainShape, r: usize) -> usize { + let prefix = shape.num_vars + 3; + let before: u32 = (0..r).map(|t| round_words(shape, t)).sum(); + let nonces = shape.carries(r).iter().filter(|&&c| c).count(); + prefix + before as usize + 2 * shape.messages(r) + nonces +} + +/// The arena offset of round 0's block at query `q`: the fixture's prefix, +/// round 0's sumcheck words and nonces, the owner's cap, then per query its +/// block (four values a word under `Lean`) and its path. +fn round0_block_at(shape: &ChainShape, q: usize) -> usize { + let prefix = shape.num_vars + 3; + let nonces = shape.carries(0).iter().filter(|&&c| c).count(); + let cap = shape.cap_words(shape.current_depth(0), shape.caps[0]); + let words = shape.round0_block_words(1usize << shape.schedule[0]); + prefix + 2 * shape.messages(0) + nonces + cap + q * (words + shape.current_path(0)) +} + +/// Fewer than [`TAIL_QUERIES_A_LEAF`] queries a leaf of any tree: a lean +/// chain with no tail round. +fn untailed() -> (usize, ChainConfig) { + (9, p1_config(12, 0, CapPolicy::Fixed(2))) +} + +/// ★ Under `Lean` the tail rule picks a suffix of rounds ≥ 1, every one of +/// them wholly in its cap with [`TAIL_QUERIES_A_LEAF`] queries a leaf, and +/// none under `V1`. +#[test] +fn the_tail_rounds_are_a_capped_suffix_and_only_under_lean() { + for (num_vars, cfg) in tail_cases() { + let shape = ChainShape::new_at(&cfg, num_vars, 4); + assert!( + (0..shape.rounds()).all(|r| !shape.tail_round(r)), + "V1 has no tail round" + ); + let lean = lean(&shape); + let tails = tails(&lean); + assert!(tails[0] >= 1, "round 0 is never a tail round"); + assert_eq!( + *tails.last().unwrap(), + lean.rounds() - 1, + "a suffix of the chain" + ); + assert!(tails.windows(2).all(|w| w[1] == w[0] + 1), "contiguous"); + for &r in &tails { + assert!(TAIL_QUERIES_A_LEAF * lean.current_leaves(r) <= lean.num_queries); + assert_eq!(lean.current_path(r), 0, "no walked level"); + } + } + // One query short of round 1's bound (8 leaves): round 2 alone is a tail. + let (num_vars, cfg) = tail_cases().remove(0); + let short = lean(&ChainShape::new_at( + &ChainConfig { + num_queries: cfg.num_queries - 1, + ..cfg + }, + num_vars, + 4, + )); + assert_eq!(tails(&short), vec![2], "8 leaves need 192 queries"); + let (num_vars, cfg) = untailed(); + let few = lean(&ChainShape::new_at(&cfg, num_vars, 4)); + assert!((0..few.rounds()).all(|r| !few.tail_round(r))); +} + +/// ★ The lean chain executes on every proof the host accepts, and spends its +/// closed form of socket rows, fewer than the V1 chain's. +#[test] +fn a_lean_p1_chain_executes_on_a_proof_the_host_accepts() { + let entry = SpongeEntry::fresh_for(WrapHash::Poseidon1); + for (num_vars, cfg) in tail_cases() { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let tails = tails(&shape); + let program = p1_chain_program(&shape); + let arena = arena_at(&f, &f.proof, &shape).expect("an honest proof lays out every leaf"); + execute(&program, &[arena], &SOCKET).unwrap_or_else(|e| { + panic!("S={num_vars} tails {tails:?}: the machine refused an accepted proof: {e:?}") + }); + let v1 = p1_chain_program(&f.shape); + assert_eq!( + hash16_rows(&program), + chain_perms(&shape, entry), + "S={num_vars}: Hash16 rows against the lean closed form" + ); + assert!( + hash16_rows(&program) < hash16_rows(&v1), + "S={num_vars}: lean {} socket rows against V1 {}", + hash16_rows(&program), + hash16_rows(&v1) + ); + } +} + +/// ★ A tail block that is not its leaf (one word off in the arena) misses its +/// cap node and has no execution; the final value one off has none either. +#[test] +fn a_lean_chain_refuses_a_tampered_tail_block() { + for (num_vars, cfg) in tail_cases() { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let program = p1_chain_program(&shape); + let honest = arena_at(&f, &f.proof, &shape).expect("laid out"); + assert!(execute(&program, std::slice::from_ref(&honest), &SOCKET).is_ok()); + for r in tails(&shape) { + let at = tail_block_at(&shape, r); + for leaf in [0, shape.current_leaves(r) - 1] { + let mut forged = honest.clone(); + let word = at + leaf * (1usize << shape.schedule[r]); + forged[word][1] += FE::from(1u64); + assert!( + execute(&program, &[forged], &SOCKET).is_err(), + "S={num_vars}: tail round {r}'s leaf {leaf} one lane off must be refused" + ); + } + } + let mut forged = f.proof.clone(); + forged.final_value += FEE::one(); + assert!(!host_accepts(&f, &forged)); + let arena = arena_at(&f, &forged, &shape).expect("laid out"); + assert!( + execute(&program, &[arena], &SOCKET).is_err(), + "S={num_vars}: the final value one off must be refused" + ); + } +} + +/// ★ Every tail round's blocks replaced by blocks of the final value: they fold +/// to the final value at every position, so every tail round's fold compare +/// holds, and only the cap compare refuses them — with it off (a test fault) +/// the forgery executes. +#[test] +fn a_lean_chain_refuses_tail_blocks_that_only_fold_consistently() { + for (num_vars, cfg) in tail_cases() { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let mut forged = arena_at(&f, &f.proof, &shape).expect("laid out"); + for r in tails(&shape) { + let at = tail_block_at(&shape, r); + let words = shape.current_leaves(r) << shape.schedule[r]; + for w in &mut forged[at..at + words] { + *w = ext_word(&f.proof.final_value); + } + } + let program = p1_chain_program(&shape); + assert!( + execute(&program, std::slice::from_ref(&forged), &SOCKET).is_err(), + "S={num_vars}: blocks off their cap nodes must be refused" + ); + let unchecked = tail_faults::with_leaves_unchecked(|| p1_chain_program(&shape)); + execute(&unchecked, &[forged], &SOCKET).unwrap_or_else(|e| { + panic!("S={num_vars}: without the cap compare the forgery folds consistently: {e:?}") + }); + } +} + +/// ★ The tail compare is live and reads the slot its position names: comparing +/// with the neighbouring slot of the successor block, an honest proof has no +/// execution. +#[test] +fn a_tail_compare_at_the_wrong_slot_refuses_an_honest_proof() { + let mut compared = 0; + for (num_vars, cfg) in tail_cases() { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + // Only a tail round with a successor compares a slot. + if !tails(&shape).iter().any(|&r| shape.next_depth(r).is_some()) { + continue; + } + let arena = arena_at(&f, &f.proof, &shape).expect("laid out"); + let faulted = tail_faults::with_slot_off(|| p1_chain_program(&shape)); + assert!( + execute(&faulted, std::slice::from_ref(&arena), &SOCKET).is_err(), + "S={num_vars}: the neighbouring slot must be refused" + ); + assert!(execute(&p1_chain_program(&shape), &[arena], &SOCKET).is_ok()); + compared += 1; + } + assert!(compared > 0, "some case has a tail round with a successor"); +} + +/// Every opening of tail round `r`'s leaf `leaf` in `proof` (the round +/// before's successor openings and the round's own), each value one off: no +/// opening hashes to that leaf's cap node any more. +fn spoil_leaf(proof: &mut ChainProof, shape: &ChainShape, r: usize, digest: [u8; 32]) { + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + type Leaf = ::Backend; + let _ = shape; + let spoil = |openings: &mut Vec>| { + for o in openings.iter_mut() { + if Leaf::hash_data(&o.values) == digest { + o.values[0] += FEE::one(); + } + } + }; + match &mut proof.rounds[r - 1].openings { + RoundOpenings::Base(p) => spoil(&mut p.next), + RoundOpenings::Extension(p) => spoil(&mut p.next), + RoundOpenings::BaseGrouped(p) => spoil(&mut p.next), + } + match &mut proof.rounds[r].openings { + RoundOpenings::Extension(p) => spoil(&mut p.current), + _ => panic!("a tail round's openings are extension-valued"), + } +} + +/// ★ A proof that opens no block of some tail leaf (every such opening spoilt) +/// makes the arena writer return an error naming the leaf, not a panic, and +/// lays out no arena; under V1 the same proof still lays out (its paths, not +/// the tail, are what refuse it). +#[test] +fn a_lean_arena_refuses_a_proof_missing_a_tail_leaf() { + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + type Leaf = ::Backend; + let (num_vars, cfg) = tail_cases().remove(0); + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let r = *tails(&shape).last().expect("a tail round"); + // Leaf 0's digest: the hash of any opened block placed first. + let honest = arena_at(&f, &f.proof, &shape).expect("laid out"); + let at = tail_block_at(&shape, r); + let block = 1usize << shape.schedule[r]; + let values: Vec> = honest[at..at + block] + .iter() + .map(|w| FEE::new([w[0], w[1], w[2]])) + .collect(); + let digest = Leaf::hash_data(&values); + let mut spoilt = f.proof.clone(); + spoil_leaf(&mut spoilt, &shape, r, digest); + let err = arena_at(&f, &spoilt, &shape).expect_err("a missing leaf is refused"); + assert!( + err.contains(&format!("tail round {r}: no opening of leaf 0")), + "{err}" + ); + assert!( + arena_at(&f, &spoilt, &f.shape).is_ok(), + "V1 lays the paths out" + ); +} + +/// ★ Equal tail blocks (a constant polynomial folds to a constant codeword) +/// share one cap digest; each leaf takes a block that hashes to it, and the +/// lean chain executes. +#[test] +fn equal_tail_blocks_are_each_placed() { + let cfg = p1_config(96, 0, CapPolicy::Fixed(2)); + let f = super::whir_p1_leaf_tests::constant_fixture(9, cfg); + let shape = lean(&f.shape); + let program = p1_chain_program(&shape); + let arena = arena_at(&f, &f.proof, &shape).expect("every leaf placed"); + execute(&program, &[arena], &SOCKET).expect("the lean chain executes"); +} + +/// ★ Under `Lean` round 0's blocks arrive four values a word, in order, the +/// last word zero-filled; with no tail round the lean chain executes, costs +/// the V1 chain's socket rows, and hints `Q · (block − ⌈block / 4⌉)` fewer +/// words. +#[test] +fn a_lean_round_zero_block_arrives_four_values_a_word() { + let mut cases = tail_cases(); + cases.push(untailed()); + for (num_vars, cfg) in cases { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + assert!(shape.round0_packed() && !f.shape.round0_packed()); + let block = 1usize << shape.schedule[0]; + let arena = arena_at(&f, &f.proof, &shape).expect("laid out"); + let RoundOpenings::Base(p) = &f.proof.rounds[0].openings else { + panic!("round 0 is base-valued") + }; + for q in [0, shape.num_queries - 1] { + let at = round0_block_at(&shape, q); + let lanes: Vec<_> = arena[at..at + block.div_ceil(4)] + .iter() + .flatten() + .copied() + .collect(); + assert_eq!( + lanes[..block], + p.current[q].values[..], + "S={num_vars} q={q}" + ); + assert!(lanes[block..].iter().all(|v| *v == FE::zero())); + } + } + let (num_vars, cfg) = untailed(); + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + assert!((0..shape.rounds()).all(|r| !shape.tail_round(r))); + let program = p1_chain_program(&shape); + let arena = arena_at(&f, &f.proof, &shape).expect("laid out"); + execute(&program, &[arena], &SOCKET).expect("the lean chain executes"); + let v1 = p1_chain_program(&f.shape); + assert_eq!( + hash16_rows(&program), + hash16_rows(&v1), + "packing costs no permutation" + ); + let block = 1usize << shape.schedule[0]; + assert_eq!( + hint_rows(&v1) - hint_rows(&program), + shape.num_queries * (block - block.div_ceil(4)), + "round 0's hinted words" + ); +} + +/// ★ A round-0 value one off in any lane of a block's first or last word, at +/// the first or last query, misses its leaf and has no execution. +#[test] +fn a_lean_chain_refuses_a_round_zero_lane_off() { + let mut cases = tail_cases(); + cases.push(untailed()); + for (num_vars, cfg) in cases { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let program = p1_chain_program(&shape); + let honest = arena_at(&f, &f.proof, &shape).expect("laid out"); + assert!(execute(&program, std::slice::from_ref(&honest), &SOCKET).is_ok()); + let words = shape.round0_block_words(1usize << shape.schedule[0]); + for q in [0, shape.num_queries - 1] { + for word in [0, words - 1] { + for lane in 0..4 { + let mut forged = honest.clone(); + forged[round0_block_at(&shape, q) + word][lane] += FE::from(1u64); + assert!( + execute(&program, &[forged], &SOCKET).is_err(), + "S={num_vars}: query {q}'s word {word} lane {lane} one off must be \ + refused" + ); + } + } + } + } +} + +/// ★ Under `Lean` every walked 4-ary level hints its siblings rotated on its +/// `b1` (three `HintOn` rows) and places them with `walk4`'s three `Select`s, +/// two fewer than the child-order walk; the socket rows stay the closed +/// form's. +#[test] +fn a_lean_chain_walks_each_level_in_three_selects() { + use super::instr::Instr; + let count = |p: &super::compiler::LfmProgram, f: fn(&Instr) -> bool| { + p.instrs.iter().filter(|i| f(i)).count() + }; + let entry = SpongeEntry::fresh_for(WrapHash::Poseidon1); + let mut cases = tail_cases(); + cases.push(untailed()); + let mut walked = 0; + for (num_vars, cfg) in cases { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + assert!(shape.paths_rotated() && !f.shape.paths_rotated()); + let (v1, lean) = (p1_chain_program(&f.shape), p1_chain_program(&shape)); + let hint_ons = count(&lean, |i| matches!(i, Instr::HintOn { .. })); + // A chain whose trees all lie in their caps walks no level. + assert_eq!(hint_ons % 3, 0, "S={num_vars}: three a walked level"); + walked += hint_ons; + assert_eq!(count(&v1, |i| matches!(i, Instr::HintOn { .. })), 0); + if (0..shape.rounds()).all(|r| !shape.tail_round(r)) { + let selects = |p| count(p, |i| matches!(i, Instr::Select { .. })); + assert_eq!( + selects(&v1) - selects(&lean), + 2 * hint_ons / 3, + "S={num_vars}: two Selects fewer a walked level" + ); + } + assert_eq!(hash16_rows(&lean), chain_perms(&shape, entry)); + } + assert!(walked > 0, "some case walks a level"); +} + +/// ★ A sibling one off in a lean arena (the first and the last word of round +/// 0's first path) moves the walked root and has no execution. +#[test] +fn a_lean_chain_refuses_a_path_word_off() { + let mut tampered = 0; + for (num_vars, cfg) in tail_cases().into_iter().chain([untailed()]) { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + // Round 0's tree wholly in its cap walks no level. + if shape.current_path(0) == 0 { + continue; + } + tampered += 1; + let program = p1_chain_program(&shape); + let honest = arena_at(&f, &f.proof, &shape).expect("laid out"); + assert!(execute(&program, std::slice::from_ref(&honest), &SOCKET).is_ok()); + let words = shape.round0_block_words(1usize << shape.schedule[0]); + let path = round0_block_at(&shape, 0) + words; + for at in [path, path + shape.current_path(0) - 1] { + let mut forged = honest.clone(); + forged[at][2] += FE::from(1u64); + assert!( + execute(&program, &[forged], &SOCKET).is_err(), + "S={num_vars}: path word {} one off must be refused", + at - path + ); + } + } + assert!(tampered > 0, "some case walks round 0"); +} + +/// ★ Beside the tail rounds the sampled checks still run. The lean leaf draws +/// every query position V1 draws (as many `BitDec` rows: the tail rounds' +/// draws stay in the transcript, their bits unused), and the round before the +/// first tail round still opens the tail tree at each sampled position: its +/// successor opening one off at the first or the last query, which the host +/// refuses, has no execution. +#[test] +fn the_sampled_checks_still_run_beside_the_tail_rounds() { + use super::instr::Instr; + let bitdecs = |p: &super::compiler::LfmProgram| { + p.instrs + .iter() + .filter(|i| matches!(i, Instr::BitDec { .. })) + .count() + }; + for (num_vars, cfg) in tail_cases() { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let program = p1_chain_program(&shape); + assert_eq!( + bitdecs(&program), + bitdecs(&p1_chain_program(&f.shape)), + "S={num_vars}: every query position drawn" + ); + let r = tails(&shape)[0] - 1; + for q in [0, shape.num_queries - 1] { + let mut forged = f.proof.clone(); + match &mut forged.rounds[r].openings { + RoundOpenings::Base(p) => p.next[q].values[0] += FEE::one(), + RoundOpenings::Extension(p) => p.next[q].values[0] += FEE::one(), + RoundOpenings::BaseGrouped(p) => p.next[q].values[0] += FEE::one(), + } + assert!(!host_accepts(&f, &forged), "S={num_vars}: the host refuses"); + let arena = arena_at(&f, &forged, &shape).expect("the other openings place every leaf"); + assert!( + execute(&program, &[arena], &SOCKET).is_err(), + "S={num_vars}: round {r}'s successor opening at query {q} one off must be refused" + ); + } + } +} + +/// ★ The tail rounds' fold compare is load-bearing on its own: with the cap +/// compare off (a test fault, so the hinted tail blocks are bound to nothing +/// else), one value of the last tail round's first block one off folds away +/// from the final value and has no execution, while the honest arena under the +/// same fault executes. Dropping the fold compare turns this red: it is what +/// makes the leaf's acceptance imply the host's sampled fold checks. +#[test] +fn a_lean_chain_refuses_a_tail_value_off_by_its_fold() { + for (num_vars, cfg) in tail_cases() { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let unchecked = tail_faults::with_leaves_unchecked(|| p1_chain_program(&shape)); + let honest = arena_at(&f, &f.proof, &shape).expect("laid out"); + execute(&unchecked, std::slice::from_ref(&honest), &SOCKET) + .unwrap_or_else(|e| panic!("S={num_vars}: the honest arena under the fault: {e:?}")); + let r = *tails(&shape).last().expect("a tail round"); + assert!( + shape.next_depth(r).is_none(), + "the last tail round folds to the final value" + ); + let mut forged = honest; + forged[tail_block_at(&shape, r)][0] += FE::from(1u64); + assert!( + execute(&unchecked, &[forged], &SOCKET).is_err(), + "S={num_vars}: a tail value one off must be refused by its fold" + ); + } +} + +/// ★ The intermediate tail rounds' successor compare is load-bearing on its +/// own: with the cap compare off, one value of the first tail round's first or +/// last block one off (a round with a tail successor) folds away from its +/// successor's slot and has no execution. No other compare reads that block +/// (the round before checks its own successor openings, separate words). +#[test] +fn a_lean_chain_refuses_a_first_tail_value_off_by_its_successor_compare() { + let mut checked = 0; + for (num_vars, cfg) in tail_cases() { + let f = fixture(num_vars, cfg); + let shape = lean(&f.shape); + let r = tails(&shape)[0]; + if shape.next_depth(r).is_none() { + continue; + } + let unchecked = tail_faults::with_leaves_unchecked(|| p1_chain_program(&shape)); + let honest = arena_at(&f, &f.proof, &shape).expect("laid out"); + execute(&unchecked, std::slice::from_ref(&honest), &SOCKET) + .unwrap_or_else(|e| panic!("S={num_vars}: the honest arena under the fault: {e:?}")); + for leaf in [0, shape.current_leaves(r) - 1] { + let mut forged = honest.clone(); + forged[tail_block_at(&shape, r) + leaf * (1usize << shape.schedule[r])][0] += + FE::from(1u64); + assert!( + execute(&unchecked, &[forged], &SOCKET).is_err(), + "S={num_vars}: tail round {r} leaf {leaf} one off must be refused by its successor compare" + ); + } + checked += 1; + } + assert!(checked > 0, "some case has two tail rounds"); +} diff --git a/prover/src/lfm/whir_onechain_tests.rs b/prover/src/lfm/whir_onechain_tests.rs new file mode 100644 index 000000000..e8894f1dc --- /dev/null +++ b/prover/src/lfm/whir_onechain_tests.rs @@ -0,0 +1,1988 @@ +//! D-ONECHAIN B2: the WHIR leaf's leg for one chain per run of a group's +//! stacked polynomials (`GroupFold::Indexed`), gated against +//! `stacked_eval::verify` on real `stacked_eval::prove` proofs, under RPX and +//! under Poseidon1's 4-ary capped trees. The gates are REV-B2-JUDGE §4's B2-1 +//! to B2-6, each named where it is tested; B2-7, the median census by +//! emission, is `whir_p1_census_tests::whir_onechain_census`. +//! +//! As in `whir_stacked_tests`, executing an honest proof IS the stream +//! comparison: every value the machine draws feeds a refusal. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::p1_transcript::P1Transcript; +use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{Claimed, Refusal, StackedCommitment, StackedProof}; +use multilinear::stacking::StackedLayout; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, LeafOrder, RoundOpenings, + WhirFolds, +}; +use multilinear::whir_commit::Commitment; +use multilinear::whir_hash::{P1Whir, RpxWhir, WhirHash}; +use multilinear::whir_round::GroupedRoundProof; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::commitment_to_digest; +use super::builder::{Cell, Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::WrapHash; +use super::executor::{LfmExecError, execute, execute_serial}; +use super::hash::HasherKind; +use super::instr::Instr; +use super::validator::validate; +use super::whir_chain::onechain_steps::{self, Step}; +use super::whir_chain::{ + ChainShape, LeafQuery, RoundStorage, chain_opening_perms, onechain_faults, push_round_words, +}; +use super::whir_chain_tests::{Recording, const_rows, hint_rows, perm_rows}; +use super::whir_stacked::{StackedChainWires, emit_stacked_verify_chains, stacked_verify_cost}; +use super::whir_transcript::{SpongeEntry, WhirTranscript}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type RpxHost = DefaultTranscript; + +/// The base a case runs under: RPX's binary trees and byte transcript, or +/// Poseidon1's 4-ary trees and field sponge (the block's production base). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Base { + Rpx, + P1, +} + +/// One case: columns' heights in a stack of `n_stack`, the fold schedule, +/// `Q`, a uniform grind, the base's cap (binary under RPX, 4-ary under +/// Poseidon1) and the round-0 trees' leaf order. Every column is claimed at a +/// point of its own. +struct Case { + name: &'static str, + base: Base, + heights: Vec, + n_stack: usize, + first: Option, + log_folding: usize, + log_blowup: usize, + num_queries: usize, + grind: u8, + cap: CapPolicy, + leaves: LeafOrder, +} + +fn rpx(name: &'static str, heights: &[usize], n_stack: usize, grind: u8) -> Case { + Case { + name, + base: Base::Rpx, + heights: heights.to_vec(), + n_stack, + first: None, + log_folding: 2, + log_blowup: 2, + num_queries: 3, + grind, + cap: CapPolicy::Off, + leaves: LeafOrder::Natural, + } +} + +/// The cases: a run of three (slot 3 absent), of two (`m = 1`) and of four (no +/// slot absent); a split group (five → [4, 1]); mixed heights whose short +/// columns' selectors run past `k0`; a first fold of six over eight variables +/// (round 0 carries 6 + 2 messages); a grouped chain of one round (`n ≤ k0`, +/// R = 1, REV-B2-JUDGE J2); no grind; a binary cap; and under Poseidon1 a run +/// of three at the C2 cap and a split group uncapped. +fn cases() -> Vec { + vec![ + rpx("three polynomials, one chain", &[6; 3], 6, 8), + rpx("two polynomials, one index variable", &[6; 2], 6, 8), + rpx("four polynomials, no slot absent", &[6; 4], 6, 8), + rpx("five polynomials split [4, 1]", &[6; 5], 6, 8), + rpx( + "mixed heights, selectors past k0", + &[6, 3, 6, 2, 4, 6], + 6, + 8, + ), + Case { + first: Some(6), + log_folding: 4, + ..rpx("first fold 6 over eight variables", &[8; 3], 8, 8) + }, + // Blowup 2^6: a one-round chain's tree has 2^log_blowup leaves, and a + // forgery from another position needs positions that differ. + Case { + first: Some(6), + log_folding: 4, + log_blowup: 6, + ..rpx("one round (n ≤ k0)", &[5; 3], 5, 8) + }, + rpx("three polynomials, no grind", &[6; 3], 6, 0), + Case { + cap: CapPolicy::Fixed(1), + ..rpx("three polynomials under a binary cap", &[6; 3], 6, 8) + }, + Case { + base: Base::P1, + first: Some(6), + log_folding: 4, + cap: CapPolicy::Fixed(2), + ..rpx("P1, three polynomials at C2", &[10; 3], 10, 6) + }, + Case { + base: Base::P1, + first: Some(6), + log_folding: 4, + ..rpx("P1, five polynomials split [4, 1]", &[8; 5], 8, 0) + }, + ] +} + +fn config(case: &Case) -> ChainConfig { + let mut format = ChainFormat { + folds: case.first.map_or(WhirFolds::Uniform, |k| { + WhirFolds::First(FirstFold::new(k).expect("a tested first fold")) + }), + group: GroupFold::Indexed, + leaves: case.leaves, + ..ChainFormat::DEFAULT + }; + match case.base { + Base::Rpx => format.cap = case.cap, + Base::P1 => format.arity4_cap = case.cap, + } + ChainConfig { + log_blowup: case.log_blowup, + log_folding: case.log_folding, + num_queries: case.num_queries, + grind: GrindBits::uniform(case.grind), + format, + } +} + +fn fee(v: u64) -> FEE { + FEE::new([ + FE::from(v.wrapping_mul(6364136223846793005) >> 11), + FE::from(v ^ 0xA5A5), + FE::from(v.wrapping_add(7)), + ]) +} + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..1u64 << num_vars) + .map(|i| { + FE::from( + i.wrapping_mul(0x2545_F491_4F6C_DD1D) + .wrapping_add(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)) + >> 13, + ) + }) + .collect(), + ) + .expect("a cube") +} + +/// Everything one case's run needs on both sides. +struct Fixture { + name: &'static str, + base: Base, + cfg: ChainConfig, + layout: StackedLayout, + /// One claimed point per column. + points: Vec>, + values: Vec, + roots: Vec, + proof: StackedProof, + domain: Domain, + /// The per-polynomial shape the stack is cut from ([`ChainShape::run`]). + shape: ChainShape, + /// RPX only: the batching challenge and the hash events the HOST verifier's + /// own transcript performed (`Recording`). + gamma: Option, + host_hashes: Option>, +} + +/// Commits and proves one case on the host under its base, opened per +/// `group`. +fn build(case: &Case, group: GroupFold) -> Fixture { + build_over(case, group, None) +} + +/// [`build`], column `slot` replaced by another polynomial (`seed`) when +/// `other` is `Some((slot, seed))`: the same points, another stack. +fn build_over(case: &Case, group: GroupFold, other: Option<(usize, u64)>) -> Fixture { + let mut cfg = config(case); + cfg.format.group = group; + let layout = StackedLayout::build(&case.heights, case.n_stack).expect("the layout packs"); + let columns: Vec> = case + .heights + .iter() + .enumerate() + .map(|(c, &h)| match other { + Some((slot, seed)) if slot == c => column(h, seed), + _ => column(h, 17 + 13 * c as u64), + }) + .collect(); + let borrowed = multilinear::stacking::borrow(&columns); + let points: Vec> = case + .heights + .iter() + .enumerate() + .map(|(c, &h)| (0..h).map(|i| fee(0x5eed + (c * 37 + i) as u64)).collect()) + .collect(); + let values: Vec = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in::(p).expect("a column takes its own point")) + .collect(); + fn run>( + layout: &StackedLayout, + borrowed: &[&Mle], + points: &[Vec], + values: &[FEE], + cfg: &ChainConfig, + transcript: fn() -> T, + ) -> (Vec, StackedProof, Domain) { + let stacked = StackedCommitment::::commit(layout.clone(), borrowed, None, cfg) + .expect("the group commits"); + let proof = multilinear::stacked_eval::prove::( + &stacked, + borrowed, + None, + &Claimed::PerColumn(points), + values, + cfg, + &mut transcript(), + ) + .expect("the group proves"); + (stacked.roots(), proof, stacked.domain().clone()) + } + let (roots, proof, domain) = match case.base { + Base::Rpx => run::(&layout, &borrowed, &points, &values, &cfg, || { + RpxHost::new(&[]) + }), + Base::P1 => run::( + &layout, + &borrowed, + &points, + &values, + &cfg, + P1Transcript::new, + ), + }; + let arity = match case.base { + Base::Rpx => 2, + Base::P1 => 4, + }; + Fixture { + name: case.name, + base: case.base, + cfg, + shape: ChainShape::new_at(&cfg, layout.n_stack(), arity), + layout, + points, + values, + roots, + proof, + domain, + gamma: None, + host_hashes: None, + } +} + +/// [`build`] under `Indexed`, with a grouped chain, accepted by the host — +/// under RPX through `Recording`, keeping the host's batching challenge and +/// hash events. +fn fixture(case: &Case) -> Fixture { + let mut f = build(case, GroupFold::Indexed); + assert!( + f.shape + .chains(f.layout.num_polys()) + .iter() + .any(|r| r.len() >= 2), + "{}: a grouped chain", + f.name + ); + match f.base { + Base::Rpx => { + let mut recorded = Recording::new(); + host_verify(&f, &f.proof, &mut recorded).expect("the control proof must verify"); + f.gamma = Some(recorded.sampled[0]); + f.host_hashes = Some(recorded.duplex.borrow().hashes.clone()); + } + Base::P1 => assert!(host_accepts(&f, &f.proof), "{}: the control", f.name), + } + f +} + +fn host_verify>( + f: &Fixture, + proof: &StackedProof, + transcript: &mut T, +) -> Result<(), multilinear::Error> { + macro_rules! verify_under { + ($h:ty) => { + multilinear::stacked_eval::verify::( + proof, + &f.layout, + &f.roots, + &Claimed::PerColumn(&f.points), + &f.values, + &f.domain, + &f.cfg, + transcript, + ) + }; + } + match f.base { + Base::Rpx => verify_under!(RpxWhir), + Base::P1 => verify_under!(P1Whir), + } +} + +fn host_accepts(f: &Fixture, proof: &StackedProof) -> bool { + match f.base { + Base::Rpx => host_verify(f, proof, &mut RpxHost::new(&[])).is_ok(), + Base::P1 => host_verify(f, proof, &mut P1Transcript::new()).is_ok(), + } +} + +fn socket(base: Base) -> HasherKind { + match base { + Base::Rpx => crate::hash_pin::BLOCK_HASHER, + Base::P1 => HasherKind::Poseidon1W16, + } +} + +/// Where every wire lives: each column's point, the values, then per chain +/// its run's roots, its final value and its rounds at the run's shape. Built +/// once for the program and the arena filler alike. +struct Arena { + values_at: u32, + chain_at: Vec, + total: u32, +} + +impl Arena { + fn new(f: &Fixture) -> Self { + let mut at: u32 = f.points.iter().map(|p| p.len() as u32).sum(); + let values_at = at; + at += f.values.len() as u32; + let mut chain_at = Vec::new(); + for run in f.shape.chains(f.layout.num_polys()) { + chain_at.push(at); + at += run.len() as u32 + 1 + RoundStorage::words(&f.shape.run(run.len())); + } + Self { + values_at, + chain_at, + total: at, + } + } +} + +/// The leg as a program: the hints in [`Arena`]'s order, then +/// [`emit_stacked_verify_chains`] — the block leaf's group opening — and the +/// batching challenge published. +fn program(f: &Fixture) -> LfmProgram { + traced_program(f).0 +} + +/// [`program`] with the marks of its checks ([`onechain_steps`]), so a refusal +/// can be named by the check that refused. +fn traced_program(f: &Fixture) -> (LfmProgram, Vec<(Step, usize)>) { + onechain_steps::record(|| emit_program(f)) +} + +fn emit_program(f: &Fixture) -> LfmProgram { + let at = Arena::new(f); + let hash = match f.base { + Base::Rpx => WrapHash::production(), + Base::P1 => WrapHash::Poseidon1, + }; + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let arena = b.declare_arena(at.total); + let mut transcript = WhirTranscript::for_builder(&mut b); + let mut next = 0u32; + let points: Vec> = f + .points + .iter() + .map(|p| { + p.iter() + .map(|_| { + let w = b.hint_word(arena, next).as_ext(); + next += 1; + w + }) + .collect() + }) + .collect(); + let values: Vec = (0..f.values.len() as u32) + .map(|i| b.hint_word(arena, at.values_at + i).as_ext()) + .collect(); + let runs = f.shape.chains(f.layout.num_polys()); + let mut roots: Vec> = Vec::new(); + let mut finals: Vec = Vec::new(); + let mut storage: Vec = Vec::new(); + for (run, &base) in runs.iter().zip(&at.chain_at) { + let n = run.len() as u32; + roots.push((0..n).map(|i| b.hint_word(arena, base + i)).collect()); + finals.push(b.hint_word(arena, base + n).as_ext()); + storage.push(RoundStorage::hint( + &mut b, + arena, + base + n + 1, + &f.shape.run(run.len()), + )); + } + let openings: Vec<_> = storage.iter().map(RoundStorage::openings).collect(); + let rounds: Vec> = storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + let chains: Vec> = (0..runs.len()) + .map(|i| StackedChainWires { + rounds: &rounds[i], + roots: &roots[i], + final_value: finals[i], + }) + .collect(); + let claimed_at: Vec<&[Ext]> = points.iter().map(Vec::as_slice).collect(); + let gamma = emit_stacked_verify_chains( + &mut b, + &mut transcript, + &f.layout, + &chains, + &claimed_at, + &values, + None, + &f.shape, + &f.domain, + ); + b.public(gamma.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the grouped leg must be admissible"); + program +} + +/// The arena from a proof, in [`program`]'s order. +fn arena(f: &Fixture, proof: &StackedProof) -> Vec { + arena_under(f, &f.roots, proof) +} + +/// [`arena`] laid out by `f`'s shape over `roots` and `proof`, which may be +/// another fixture's: a proof written into the arena of another config. +fn arena_under(f: &Fixture, roots: &[Commitment], proof: &StackedProof) -> Vec { + let mut words: Vec = f.points.iter().flatten().map(ext_word).collect(); + words.extend(f.values.iter().map(ext_word)); + let runs = f.shape.chains(f.layout.num_polys()); + for (chain, run) in proof.polys.iter().zip(&runs) { + words.extend(roots[run.clone()].iter().map(commitment_to_digest)); + words.push(ext_word(&chain.final_value)); + push_round_words(&mut words, &f.shape.run(run.len()), chain); + } + words +} + +fn machine_runs(f: &Fixture, program: &LfmProgram, words: Vec) -> Result { + let exec = execute(program, &[words], &socket(f.base)).map_err(|e| format!("{e:?}"))?; + word_as_ext(&exec.public_words[0].1).ok_or_else(|| "no published challenge".to_string()) +} + +/// How the machine refused an arena: at an emitted check (a failed assert, +/// named by its chain and step), or with another executor error. +#[derive(Debug)] +enum Refused { + At(Option, Step), + Other(LfmExecError), +} + +/// The machine's refusal of `words`, or `None` when it executes. On the serial +/// schedule, so the refusal reported is the FIRST failing check in program +/// order — the host's order; the level schedule may meet a later one first (it +/// refuses all the same: [`machine_runs`] runs it). +fn machine_refusal( + f: &Fixture, + (program, marks): &(LfmProgram, Vec<(Step, usize)>), + words: Vec, +) -> Option { + assert_eq!( + execute(program, std::slice::from_ref(&words), &socket(f.base)).is_ok(), + execute_serial(program, std::slice::from_ref(&words), &socket(f.base)).is_ok(), + "{}: both schedules give one verdict", + f.name + ); + let error = execute_serial(program, &[words], &socket(f.base)).err()?; + Some(match onechain_steps::failed_at(program, marks, &error) { + Some((chain, step)) => Refused::At(chain, step), + None => Refused::Other(error), + }) +} + +/// The host's refusal of `proof` with its cause, or `None` when it accepts. +fn host_refusal(f: &Fixture, proof: &StackedProof) -> Option { + macro_rules! traced_under { + ($h:ty, $t:expr) => { + multilinear::stacked_eval::verify_traced::( + proof, + &f.layout, + &f.roots, + &Claimed::PerColumn(&f.points), + &f.values, + &f.domain, + &f.cfg, + &mut $t, + ) + }; + } + match f.base { + Base::Rpx => traced_under!(RpxWhir, RpxHost::new(&[])), + Base::P1 => traced_under!(P1Whir, P1Transcript::new()), + } + .err() +} + +/// Whether the machine's step is the check the host's cause names: a grouped +/// round-0 tree's opening, another opening, a fold, a grind, or the final +/// check — in the same chain, at the same query and tree where the host says. +fn same_check(host: &Refusal, machine: &Refused) -> bool { + use multilinear::Error as H; + let (Refusal::Chain { chain, cause, .. }, Refused::At(Some(at_chain), step)) = (host, machine) + else { + return false; + }; + chain == at_chain + && match (cause, *step) { + ( + H::GroupedOpeningRejected { query, tree }, + Step::Opening { + round, + query: q, + tree: p, + }, + ) => round == 0 && *query == q && *tree == p, + ( + H::OpeningRejected { query }, + Step::Opening { + round, query: q, .. + }, + ) => round > 0 && *query == q, + (H::OpeningRejected { query }, Step::Successor { query: q, .. }) => *query == q, + (H::FoldInconsistent { query }, Step::Fold { query: q, .. }) => *query == q, + (H::GrindingRejected { .. }, Step::Sumcheck { .. } | Step::Ood { .. }) => true, + (H::EvaluationMismatch, Step::Final) => true, + _ => false, + } +} + +fn grouped(proof: &mut StackedProof, chain: usize) -> &mut GroupedRoundProof { + match &mut proof.polys[chain].rounds[0].openings { + RoundOpenings::BaseGrouped(g) => g, + _ => panic!("chain {chain} is not grouped"), + } +} + +/// The arena index of value `t` of tree `p`'s block at query `q`, in chain 0 +/// (a grouped one): past its roots and final value, round 0's messages and +/// nonces and its trees' caps, then query-major blocks and paths. +fn round0_value_at(f: &Fixture, q: usize, p: usize, t: usize) -> usize { + let at = Arena::new(f); + let runs = f.shape.chains(f.layout.num_polys()); + let shape = f.shape.run(runs[0].len()); + let nonces = shape.carries(0).iter().filter(|&&c| c).count(); + let caps = shape.trees * shape.cap_words(shape.current_depth(0), shape.caps[0]); + let block = 1usize << shape.schedule[0]; + at.chain_at[0] as usize + + runs[0].len() + + 1 + + 2 * shape.messages(0) + + nonces + + caps + + (q * shape.trees + p) * (block + shape.current_path(0)) + + t +} + +// ─────────────── B2-1, B2-6: the leg executes what the host accepts ─────────────── + +/// ★ REV-B2-JUDGE B2-6 (host acceptance implies guest acceptance) at every +/// case, and B2-1's split: the machine executes each proof the host accepts, +/// one chain per run of the verifier's own split, and draws the batching +/// challenge the HOST drew. +#[test] +fn grouped_openings_execute_on_proofs_the_host_accepts() { + for case in cases() { + let f = fixture(&case); + let program = program(&f); + let gamma = machine_runs(&f, &program, arena(&f, &f.proof)) + .unwrap_or_else(|e| panic!("{}: the machine refused an accepted proof: {e}", f.name)); + if let Some(host) = f.gamma { + assert_eq!( + gamma, host, + "{}: the batching challenge is the host's", + f.name + ); + } + println!( + "onechain [{}]: runs {:?}, {} instructions", + f.name, + f.shape.chains(f.layout.num_polys()), + program.instrs.len() + ); + } +} + +/// `case` at blowup `2^log_blowup`, everything else the same. +fn at_blowup(case: &Case, log_blowup: usize) -> Case { + Case { + name: case.name, + base: case.base, + heights: case.heights.clone(), + n_stack: case.n_stack, + first: case.first, + log_folding: case.log_folding, + log_blowup, + num_queries: case.num_queries, + grind: case.grind, + cap: case.cap, + leaves: case.leaves, + } +} + +/// ★ The base code's rate (`LAMBDA_VM_ZF_WHIR_BLOWUP`) under `Indexed`: every +/// case at blowup 2 — runs of two, three and four, a split group, mixed +/// heights, both bases, capped and not — executes on the machine built for it +/// over a proof the host accepts. The host and the machine built for blowup 4 +/// refuse the same proof over the same roots: every tree is a level shallower, +/// so the other rate's domain and arena (verifier constants) do not fit it. +#[test] +fn grouped_openings_at_blowup_2_execute_and_are_refused_at_blowup_4() { + for case in cases().into_iter().filter(|c| c.log_blowup == 2) { + let half = fixture(&at_blowup(&case, 1)); + assert_eq!( + half.domain.log_size(), + half.layout.n_stack() + 1, + "{}: the stack's domain at blowup 2", + half.name + ); + machine_runs(&half, &program(&half), arena(&half, &half.proof)).unwrap_or_else(|e| { + panic!( + "{}: the machine at blowup 2 refused an accepted proof: {e}", + half.name + ) + }); + + let four = build(&case, GroupFold::Indexed); + macro_rules! refused_by_four { + ($h:ty, $t:expr) => { + multilinear::stacked_eval::verify::( + &half.proof, + &half.layout, + &half.roots, + &Claimed::PerColumn(&half.points), + &half.values, + &four.domain, + &four.cfg, + &mut $t, + ) + .is_err() + }; + } + let host = match half.base { + Base::Rpx => refused_by_four!(RpxWhir, RpxHost::new(&[])), + Base::P1 => refused_by_four!(P1Whir, P1Transcript::new()), + }; + assert!(host, "{}: the host at blowup 4 must refuse it", half.name); + assert!( + machine_runs( + &four, + &program(&four), + arena_under(&four, &half.roots, &half.proof) + ) + .is_err(), + "{}: the machine at blowup 4 must refuse it", + half.name + ); + } +} + +/// `case` with its round-0 trees in `leaves`' order, everything else the same. +fn in_order(case: &Case, leaves: LeafOrder) -> Case { + Case { + leaves, + ..at_blowup(case, case.log_blowup) + } +} + +/// Whether the host verifier of `f`'s config accepts `proof` over `roots`, +/// which may be another fixture's. +fn host_accepts_under(f: &Fixture, roots: &[Commitment], proof: &StackedProof) -> bool { + macro_rules! accepts { + ($h:ty, $t:expr) => { + multilinear::stacked_eval::verify::( + proof, + &f.layout, + roots, + &Claimed::PerColumn(&f.points), + &f.values, + &f.domain, + &f.cfg, + &mut $t, + ) + .is_ok() + }; + } + match f.base { + Base::Rpx => accepts!(RpxWhir, RpxHost::new(&[])), + Base::P1 => accepts!(P1Whir, P1Transcript::new()), + } +} + +/// ★ The composed format at laptop width — one chain a run, the coset round-0 +/// order and blowup 2 (`LAMBDA_VM_ZF_WHIR_LEAVES=coset`, +/// `LAMBDA_VM_ZF_WHIR_BLOWUP=2`): every Poseidon1 case executes on the machine +/// built for it over a proof the host accepts, and the host and the machine of +/// each format one setting away — the coset order at blowup 4, the natural +/// order at blowup 2 — refuse it over its roots. +#[test] +fn grouped_openings_under_the_composed_format_execute_and_are_refused_one_setting_away() { + for case in cases().into_iter().filter(|c| c.base == Base::P1) { + let composed = fixture(&in_order(&at_blowup(&case, 1), LeafOrder::Coset)); + assert_eq!( + composed.domain.log_size(), + composed.layout.n_stack() + 1, + "{}: blowup 2", + composed.name + ); + assert_eq!(composed.shape.leaves, LeafOrder::Coset); + machine_runs( + &composed, + &program(&composed), + arena(&composed, &composed.proof), + ) + .unwrap_or_else(|e| { + panic!( + "{}: the machine under the composed format refused an accepted proof: {e}", + composed.name + ) + }); + let coset_four = fixture(&in_order(&case, LeafOrder::Coset)); + let natural_two = fixture(&at_blowup(&case, 1)); + for (label, verifier) in [ + ("the coset order at blowup 4", &coset_four), + ("the natural order at blowup 2", &natural_two), + ] { + assert!( + !host_accepts_under(verifier, &composed.roots, &composed.proof), + "{}: {label} (host)", + case.name + ); + assert!( + machine_runs( + verifier, + &program(verifier), + arena_under(verifier, &composed.roots, &composed.proof) + ) + .is_err(), + "{}: {label} (machine)", + case.name + ); + } + } +} + +/// ★ The coset round-0 leaf order (`LAMBDA_VM_ZF_WHIR_LEAVES`, Poseidon1 only) +/// under `Indexed`: every Poseidon1 case — a run of three at the C2 cap, a +/// split group — executes on the machine built for it over a proof the host +/// accepts, so EVERY round-0 tree of a run is walked at the position the order +/// puts the block at (one tree walked at the index would refuse the honest +/// proof). Across the two orders the host and the machine each refuse the +/// other order's proof over its roots, both ways. +#[test] +fn grouped_openings_under_the_coset_order_execute_and_are_refused_in_the_natural_one() { + for case in cases().into_iter().filter(|c| c.base == Base::P1) { + let coset = fixture(&in_order(&case, LeafOrder::Coset)); + let natural = fixture(&case); + assert_eq!(coset.shape.leaves, LeafOrder::Coset); + assert_ne!( + coset.roots, natural.roots, + "{}: the order moves the roots", + coset.name + ); + machine_runs(&coset, &program(&coset), arena(&coset, &coset.proof)).unwrap_or_else(|e| { + panic!( + "{}: the machine under the coset order refused an accepted proof: {e}", + coset.name + ) + }); + for (label, verifier, prover) in [ + ("natural refuses coset", &natural, &coset), + ("coset refuses natural", &coset, &natural), + ] { + assert!( + !host_accepts_under(verifier, &prover.roots, &prover.proof), + "{}: {label} (host)", + case.name + ); + assert!( + machine_runs( + verifier, + &program(verifier), + arena_under(verifier, &prover.roots, &prover.proof) + ) + .is_err(), + "{}: {label} (machine)", + case.name + ); + } + } +} + +/// ★ REV-B2-JUDGE B2-1: the grouped legs' THREADED schedule is the host's own +/// (`the_schedule_is_the_host_transcripts` at `P' = 2, 3, 4` and a split +/// group): `stacked_verify_cost` walks the runs of the split, each at its +/// shape, and the host's `stacked_eval::verify` performs exactly its hashes. +#[test] +fn the_grouped_schedule_is_the_host_transcripts() { + for case in cases().into_iter().filter(|c| c.base == Base::Rpx) { + let f = fixture(&case); + let group_of: Vec = (0..f.values.len()).collect(); + let cost = stacked_verify_cost(&f.layout, &group_of, &f.shape, SpongeEntry::fresh()); + let mine = cost.schedule().hashes(); + let host = f.host_hashes.as_ref().expect("an RPX fixture records"); + assert_eq!( + cost.chains(), + f.shape.chains(f.layout.num_polys()).len(), + "{}: one chain a run", + f.name + ); + assert_eq!(mine.len(), host.len(), "{}: hash count", f.name); + for (i, (derived, observed)) in mine.iter().zip(host).enumerate() { + assert_eq!(derived, observed, "{}: hash {i}", f.name); + } + } +} + +/// ★ REV-B2-JUDGE B2-1: the closed forms are the emitted counts at every case — +/// rows (const-free, `chain_rows`' convention) and permutations under RPX, the +/// socket's `Hash16` rows under Poseidon1 — so the W3 plan prices exactly the +/// leg the leaf emits. +#[test] +fn grouped_openings_emit_their_closed_form() { + for case in cases() { + let f = fixture(&case); + let program = program(&f); + let at = Arena::new(&f); + let group_of: Vec = (0..f.values.len()).collect(); + let entry = match f.base { + Base::Rpx => SpongeEntry::fresh(), + Base::P1 => SpongeEntry::fresh_for(WrapHash::Poseidon1), + }; + let cost = stacked_verify_cost(&f.layout, &group_of, &f.shape, entry); + assert_eq!( + hint_rows(&program), + at.total as usize, + "{}: every arena word hinted once", + f.name + ); + let perms = match f.base { + Base::Rpx => perm_rows(&program), + Base::P1 => program + .instrs + .iter() + .filter(|i| matches!(i, Instr::Hash16(_))) + .count(), + }; + assert_eq!(perms, cost.perms(), "{}: permutations", f.name); + if f.base == Base::Rpx { + // One `Hint` a word and the published challenge are plumbing. + let measured = program.instrs.len() - const_rows(&program) - (at.total as usize + 1); + assert_eq!(measured, cost.operations(), "{}: rows", f.name); + } + } +} + +/// ★ A run of ONE polynomial under `Indexed` emits today's program, row for +/// row, and reads today's arena: the singleton chain of a split is the +/// per-polynomial chain exactly. The program at a real grind width; the +/// arena unground, because a ground proof's nonces are the grinder's choice +/// (`whir_identity_tests`). +#[test] +fn one_polynomial_under_indexed_emits_todays_program() { + let case = rpx("one polynomial", &[6], 6, 8); + let (indexed, per_poly) = ( + build(&case, GroupFold::Indexed), + build(&case, GroupFold::PerPoly), + ); + assert!(host_accepts(&indexed, &indexed.proof)); + assert_eq!( + format!("{:?}", program(&indexed).instrs), + format!("{:?}", program(&per_poly).instrs) + ); + let case = rpx("one polynomial, unground", &[6], 6, 0); + let (indexed, per_poly) = ( + build(&case, GroupFold::Indexed), + build(&case, GroupFold::PerPoly), + ); + assert_eq!( + arena(&indexed, &indexed.proof), + arena(&per_poly, &per_poly.proof) + ); +} + +// ───────────────────── B2-2: the base-lift pin, per slot ───────────────────── + +/// ★ REV-B2-JUDGE B2-2 (REV-B2-B RF-2): in EVERY slot `p < P'`, a hinted +/// round-0 value with a nonzero upper lane has no execution, refused as +/// `NotBaseWord` at tree `p`'s leaf `Pack` — the slice the combine lifts is the +/// slice that leaf hashed (`whir_open_tests:: +/// the_leaf_pins_a_hinted_base_value_to_its_low_lane`, for one tree). +#[test] +fn every_slots_round_zero_value_is_pinned_to_its_low_lane() { + let pinned = + |c: &Case| (c.n_stack == 6 && c.heights.len() <= 4) || c.cap == CapPolicy::Fixed(2); + for case in cases().into_iter().filter(|c| pinned(c) && c.grind > 0) { + let f = fixture(&case); + let program = program(&f); + let honest = arena(&f, &f.proof); + assert!(machine_runs(&f, &program, honest.clone()).is_ok()); + let trees = f.shape.chains(f.layout.num_polys())[0].len(); + for p in 0..trees { + let mut smuggled = honest.clone(); + let at = round0_value_at(&f, 1, p, 2); + assert_eq!(smuggled[at][1], FE::zero(), "{}: a base word", f.name); + smuggled[at][1] = FE::one(); + let refusal = execute(&program, &[smuggled], &socket(f.base)) + .expect_err("a lifted value with a nonzero lane 1 must have no execution"); + assert!( + matches!(refusal, LfmExecError::NotBaseWord(_)), + "{} slot {p}: the refusal is the base-word check at the leaf's Pack, got \ + {refusal:?}", + f.name + ); + } + } +} + +/// The arena word and lane of value `t` of tree `p`'s block at query `q` in +/// chain 0, under [`LeafQuery::Lean`]: [`round0_value_at`]'s layout with each +/// block four values a word. +fn round0_lane_at(f: &Fixture, q: usize, p: usize, t: usize) -> (usize, usize) { + let at = Arena::new(f); + let runs = f.shape.chains(f.layout.num_polys()); + let shape = f.shape.run(runs[0].len()); + assert!(shape.round0_packed(), "a lean P1 shape"); + let nonces = shape.carries(0).iter().filter(|&&c| c).count(); + let caps = shape.trees * shape.cap_words(shape.current_depth(0), shape.caps[0]); + let words = shape.round0_block_words(1usize << shape.schedule[0]); + let word = at.chain_at[0] as usize + + runs[0].len() + + 1 + + 2 * shape.messages(0) + + nonces + + caps + + (q * shape.trees + p) * (words + shape.current_path(0)) + + t / 4; + (word, t % 4) +} + +/// ★ Under [`LeafQuery::Lean`] a grouped round 0's blocks arrive four values a +/// word: every Poseidon1 case executes on the proof the host accepts, with +/// fewer hinted words and the same socket rows as under V1, and a value one +/// off in any lane of any slot's tree has no execution. +#[test] +fn grouped_round_zero_words_under_the_lean_query_execute_and_refuse_a_lane_off() { + for case in cases().into_iter().filter(|c| c.base == Base::P1) { + let mut f = fixture(&case); + let v1 = program(&f); + f.shape.query = LeafQuery::Lean; + let lean = program(&f); + let honest = arena(&f, &f.proof); + machine_runs(&f, &lean, honest.clone()) + .unwrap_or_else(|e| panic!("{}: the machine refused an accepted proof: {e}", f.name)); + assert!( + hint_rows(&lean) < hint_rows(&v1), + "{}: fewer hinted words", + f.name + ); + let block = 1usize << f.shape.schedule[0]; + let trees = f.shape.chains(f.layout.num_polys())[0].len(); + for p in 0..trees { + for t in [0, 1, 2, 3, block - 1] { + let (word, lane) = round0_lane_at(&f, 1, p, t); + let mut forged = honest.clone(); + forged[word][lane] += FE::one(); + assert!( + execute(&lean, &[forged], &socket(f.base)).is_err(), + "{} slot {p}: value {t} one off must have no execution", + f.name + ); + } + } + } +} + +// ────────────── B2-3, B2-4, B2-6: every forgery has no execution ────────────── + +/// A forgery: what it is, the forged proof, the host's cause when it is fixed +/// (`None` when the forgery moves the transcript), the machine's step likewise. +type Forgery = ( + String, + StackedProof, + Option, + Option, +); + +/// ★ REV-B2-JUDGE B2-3, B2-4 and B2-6: every forgery the host refuses has no +/// execution, and the machine refuses it at the SAME check the host does +/// ([`same_check`], against the honest twin, which executes): +/// - a value in slot `p` at query 1 — tree `p`'s opening there; +/// - blocks swapped between trees 0 and 1 — tree 0's opening (each tree +/// against its own root, by position); +/// - the last tree's block from another query's position — that tree's +/// opening (one draw feeds every tree); +/// - a round-0 sibling of tree 1 at query 2 — tree 1's opening there; +/// - the successor's block at query 0 — the successor's opening; +/// - an index round's message and the final value, which move the +/// transcript: wherever the host's refusal lands (a grind, an opening at a +/// moved position, a fold), the machine's is the same check; +/// - B1-2's shapes (a block short, one in the absent slot, `k0` round-0 +/// messages): the host's named shape refusal, and an arena of another length, +/// which the program — reading a fixed number of wires — refuses as such. +#[test] +fn every_grouped_forgery_the_host_refuses_has_no_execution() { + use multilinear::Error as H; + use multilinear::whir_chain::GroupedShape as S; + let three = |c: &Case| c.heights.len() == 3 && c.cap == CapPolicy::Off; + for case in cases() + .into_iter() + .filter(|c| three(c) || c.base == Base::P1) + { + // Eight queries: a block from another position needs one whose + // position differs from query 1's. + let f = fixture(&Case { + num_queries: 8, + ..case + }); + let traced = traced_program(&f); + assert!(machine_refusal(&f, &traced, arena(&f, &f.proof)).is_none()); + let k0 = f.shape.schedule[0]; + let trees = f.shape.chains(f.layout.num_polys())[0].len(); + let opening = |query, tree| Some(H::GroupedOpeningRejected { query, tree }); + // (what, the forgery, the host's cause when it is fixed, the machine's step) + let mut forgeries: Vec = Vec::new(); + for p in 0..trees { + let mut proof = f.proof.clone(); + grouped(&mut proof, 0).current[1][p].values[2] += FE::one(); + forgeries.push(( + format!("a value in slot {p}"), + proof, + opening(1, p), + Some(onechain_steps::opening(0, 1, p)), + )); + } + let mut swapped = f.proof.clone(); + grouped(&mut swapped, 0).current[1].swap(0, 1); + forgeries.push(( + "blocks swapped between trees 0 and 1".into(), + swapped, + opening(1, 0), + Some(onechain_steps::opening(0, 1, 0)), + )); + let g = grouped(&mut f.proof.clone(), 0).clone(); + let last = trees - 1; + // From a query past the owner (query 0, whose paths carry the caps), so + // the arena keeps its length and only the position differs. + let other = (2..g.current.len()) + .find(|&j| g.current[j][last].values != g.current[1][last].values) + .unwrap_or_else(|| panic!("{}: no query at another position", f.name)); + let mut moved = f.proof.clone(); + grouped(&mut moved, 0).current[1][last] = g.current[other][last].clone(); + forgeries.push(( + "the last tree's block from another position".into(), + moved, + opening(1, last), + Some(onechain_steps::opening(0, 1, last)), + )); + let mut sibling = f.proof.clone(); + grouped(&mut sibling, 0).current[2][1] + .proof + .merkle_path + .first_mut() + .unwrap_or_else(|| panic!("{}: a walked level to tamper", f.name))[0] ^= 1; + forgeries.push(( + "a round-0 sibling of tree 1".into(), + sibling, + opening(2, 1), + Some(onechain_steps::opening(0, 2, 1)), + )); + if f.shape.rounds() > 1 { + let mut next = f.proof.clone(); + grouped(&mut next, 0).next[0].values[0] += FEE::one(); + forgeries.push(( + "the successor's block".into(), + next, + Some(H::OpeningRejected { query: 0 }), + Some(Step::Successor { round: 0, query: 0 }), + )); + } + let mut index = f.proof.clone(); + index.polys[0].rounds[0].sumcheck[k0].evaluations[0] += FEE::one(); + forgeries.push(("an index round's message".into(), index, None, None)); + let mut final_value = f.proof.clone(); + final_value.polys[0].final_value += FEE::one(); + forgeries.push(("the final value".into(), final_value, None, None)); + + for (what, forged, cause, step) in forgeries { + let host = host_refusal(&f, &forged) + .unwrap_or_else(|| panic!("{} [{what}]: the host must refuse", f.name)); + let machine = machine_refusal(&f, &traced, arena(&f, &forged)) + .unwrap_or_else(|| panic!("{} [{what}]: the machine executed it", f.name)); + println!( + "ONECHAIN FORGERY [{}] {what}: host {host:?} · machine {machine:?}", + f.name + ); + if let Some(cause) = cause { + assert_eq!( + host, + Refusal::Chain { + chain: 0, + poly: 0, + cause + }, + "{} [{what}]: the host's check", + f.name + ); + } + if let Some(step) = step { + assert!( + matches!(machine, Refused::At(Some(0), s) if s == step), + "{} [{what}]: the machine refused at {machine:?}, not {step:?}", + f.name + ); + } + assert!( + same_check(&host, &machine), + "{} [{what}]: the host refused at {host:?}, the machine at {machine:?}", + f.name + ); + } + + // B1-2's shapes: the host names them; the arena has another length. + let mut short = f.proof.clone(); + grouped(&mut short, 0).current[1].pop(); + let mut absent = f.proof.clone(); + let extra = grouped(&mut absent, 0).current[1][0].clone(); + grouped(&mut absent, 0).current[1].push(extra); + let mut messages = f.proof.clone(); + messages.polys[0].rounds[0].sumcheck.truncate(k0); + let m = f.shape.run(trees).index_vars(); + for (what, forged, shape) in [ + ( + "a block short at a query", + short, + S::BlocksPerQuery { + query: 1, + expected: trees, + got: trees - 1, + }, + ), + ( + "a block in the absent slot", + absent, + S::BlocksPerQuery { + query: 1, + expected: trees, + got: trees + 1, + }, + ), + ( + "k0 round-0 messages", + messages, + S::Round0Messages { + expected: k0 + m, + got: k0, + }, + ), + ] { + assert_eq!( + host_refusal(&f, &forged), + Some(Refusal::Chain { + chain: 0, + poly: 0, + cause: H::GroupedShape(shape) + }), + "{} [{what}]: the host's named shape", + f.name + ); + let machine = machine_refusal(&f, &traced, arena(&f, &forged)); + assert!( + matches!( + machine, + Some(Refused::Other(LfmExecError::ArenaLenMismatch { .. })) + ), + "{} [{what}]: refused as another arena length, got {machine:?}", + f.name + ); + } + } +} + +// ─────────────────── B2-5: each reading of the index ─────────────────── + +/// ★ REV-B2-JUDGE B2-5, the host's B1-3 negatives in-guest: each place the +/// leg reads the index is load-bearing, and each refuses at its own check. +/// With `ρ` swapped in the combine only, the honest arena is refused at round +/// 0's first fold; with `ρ` swapped in the weight only, the index factor +/// dropped, or the absent slot given weight 1, at the final check; with none of +/// them it executes. (`eq(ρ, p)` is computed from the sampled `ρ` wires: there +/// is no hint wire to set it by.) +#[test] +fn each_in_guest_reading_of_the_index_is_load_bearing() { + use onechain_faults::Fault; + // Runs of three and four: at two, one index variable, swapping `ρ` is the + // identity. + let wide = |c: &Case| { + (c.base == Base::Rpx && c.n_stack == 6 && [3, 4].contains(&c.heights.len()) && c.grind > 0) + || c.cap == CapPolicy::Fixed(2) + }; + for case in cases().into_iter().filter(wide) { + let f = fixture(&case); + let words = arena(&f, &f.proof); + let honest = onechain_faults::with(Fault::None, || traced_program(&f)); + assert!(machine_refusal(&f, &honest, words.clone()).is_none()); + let trees = f.shape.chains(f.layout.num_polys())[0].len(); + for (fault, step) in [ + (Fault::SwapRhoInCombine, Step::Fold { round: 0, query: 0 }), + (Fault::SwapRhoInWeight, Step::Final), + (Fault::DropIndexFactor, Step::Final), + (Fault::AbsentSlotWeightOne, Step::Final), + ] { + if fault == Fault::AbsentSlotWeightOne && trees.is_power_of_two() { + continue; + } + let faulty = onechain_faults::with(fault, || traced_program(&f)); + let refused = machine_refusal(&f, &faulty, words.clone()); + assert!( + matches!(refused, Some(Refused::At(Some(0), s)) if s == step), + "{} {fault:?}: refused at {refused:?}, not {step:?}", + f.name + ); + } + } +} + +/// REV-B2-JUDGE B1-1 in guest: a run whose virtual domain `H_{n+m+lb}` would +/// pass the field's two-adicity has no program — the shape refuses it at emit +/// time, as the host refuses the config before reading. +#[test] +#[should_panic(expected = "has no virtual domain")] +fn a_run_past_the_two_adicity_has_no_program() { + let mut cfg = config(&rpx("wide", &[6; 3], 6, 0)); + cfg.log_blowup = 32 - 6 - 2 + 1; + let shape = ChainShape::new_at(&cfg, 6, 2); + let _ = shape.run(3); +} + +// ─────────────────── the production shape's cost (B2-7) ─────────────────── + +/// ★ The production grouped chain's in-guest openings, from the closed form +/// the W3 plan prices with: Poseidon1, C2, Q 114, schedule [6, 4, 4, 4, 4, 4, +/// 1] at a 2^27 stack — D-ONECHAIN §4.1's B2-sep column (`size_b2.py`), to the +/// permutation: 11,191 for one polynomial (today's chain), 13,018 / 14,845 / +/// 16,672 for runs of two, three and four, against today's 22,382 / 33,573 / +/// 44,764 for the same polynomials chained apart. +#[test] +fn the_production_grouped_chain_costs_what_the_design_sized() { + let mut cfg = ChainConfig::with_security_folds( + 2, + 4, + WhirFolds::First(FirstFold::new(6).expect("first6")), + 27, + 128, + GrindBits::query_only(18), + ); + cfg.format.arity4_cap = CapPolicy::Fixed(2); + cfg.format.group = GroupFold::Indexed; + let shape = ChainShape::new_at(&cfg, 27, 4); + assert_eq!(shape.schedule, [6, 4, 4, 4, 4, 4, 1]); + assert_eq!(shape.num_queries, 114); + let one = chain_opening_perms(&shape.run(1)); + assert_eq!(one, 11_191, "today's chain"); + for (trees, want) in [(2usize, 13_018usize), (3, 14_845), (4, 16_672)] { + let grouped = chain_opening_perms(&shape.run(trees)); + println!( + "P1 C2 Q114 n27: a run of {trees} opens for {grouped} permutations (apart {})", + trees * one + ); + assert_eq!(grouped, want, "a run of {trees}"); + } + // ★ The B2 gate's per-group figure (D-ONECHAIN §5, "14,845 + sponge ± + // 3 %"): a three-polynomial group's whole opening leg is the chain's + // openings and its threaded sponge, and the sponge is inside the 3 %. The + // design's ≈ 720 a chain [I] (`size_b2.py`, read off W3 PLAN costs that + // also hold the tables' legs) is not the sponge: one grouped chain's is + // 99 here, three per-polynomial chains' 289. + let layout = StackedLayout::build(&[27; 3], 27).expect("three 2^27 polynomials"); + let entry = SpongeEntry::fresh_for(WrapHash::Poseidon1); + let opening = stacked_verify_cost(&layout, &[0, 1, 2], &shape, entry).perms(); + let mut per_poly = shape.clone(); + per_poly.group = GroupFold::PerPoly; + let today = stacked_verify_cost(&layout, &[0, 1, 2], &per_poly, entry).perms(); + println!( + "P1 C2 Q114 n27, a three-polynomial group's opening leg: {opening} permutations \ + Indexed (sponge {}), {today} PerPoly (sponge {}), {:+.1} %", + opening - 14_845, + today - 3 * 11_191, + 100.0 * (opening as f64 / today as f64 - 1.0) + ); + assert_eq!((opening, today), (14_845 + 99, 33_573 + 289)); + assert!( + (opening - 14_845) * 100 <= 3 * 14_845, + "the sponge within the gate's 3 %" + ); +} + +// ─────────────────── from the code review (REV-OC-B) ─────────────────── + +/// The code review's M-a, in guest: each round-0 tree's cap is authenticated +/// against ITS root, by position. Under the binary cap and Poseidon1's C2 cap, +/// the arena words of trees 0's and 1's caps traded have no execution, and +/// the failing assert lies in the chain's cap authentication — after its +/// `Chain` mark and before round 0's first check; the honest arena executes. +#[test] +fn round_zero_caps_are_bound_per_tree_in_guest() { + for case in cases().into_iter().filter(|c| c.cap != CapPolicy::Off) { + let f = fixture(&case); + let (program, marks) = traced_program(&f); + let honest = arena(&f, &f.proof); + assert!(machine_runs(&f, &program, honest.clone()).is_ok()); + let at = Arena::new(&f); + let runs = f.shape.chains(f.layout.num_polys()); + let shape = f.shape.run(runs[0].len()); + let words = shape.cap_words(shape.current_depth(0), shape.caps[0]); + assert!(words > 0, "{}: a capped round-0 tree", f.name); + let nonces = shape.carries(0).iter().filter(|&&c| c).count(); + let caps = at.chain_at[0] as usize + runs[0].len() + 1 + 2 * shape.messages(0) + nonces; + let mut traded = honest.clone(); + for w in 0..words { + traded.swap(caps + w, caps + words + w); + } + assert_ne!(traded, honest, "{}: two trees, two caps", f.name); + let error = execute_serial(&program, &[traded], &socket(f.base)) + .expect_err("traded caps must have no execution"); + let LfmExecError::DivByZero { addr } = error else { + panic!("{}: a failed assert, got {error:?}", f.name); + }; + let index = program + .instrs + .iter() + .position(|i| match i { + Instr::BaseAlu { out, .. } | Instr::ExtAlu { out, .. } => out.0 == addr, + _ => false, + }) + .expect("the assert's difference"); + let chain0 = marks + .iter() + .find(|(s, _)| *s == Step::Chain(0)) + .map(|&(_, at)| at) + .expect("chain 0's mark"); + let round0 = marks + .iter() + .find(|(s, _)| *s == Step::Sumcheck { round: 0 }) + .map(|&(_, at)| at) + .expect("round 0's mark"); + println!( + "ONECHAIN CAPS [{}]: {words} cap words a tree; refused at instruction {index} in \ + [{chain0}, {round0})", + f.name + ); + assert!( + chain0 <= index && index < round0, + "{}: refused at {index}, outside the cap authentication [{chain0}, {round0})", + f.name + ); + } +} + +/// The leg with its roots as PROGRAM CONSTANTS, as the block leaf's prepared +/// branch wires them (`whir_block`: derived roots as `digest_const`, each chain +/// its run's slice): tree `p` is checked against the constant of root +/// `order[p]`. Its arena holds no root word. +fn const_root_program(f: &Fixture, order: &[usize]) -> (LfmProgram, Vec<(Step, usize)>) { + onechain_steps::record(|| { + let runs = f.shape.chains(f.layout.num_polys()); + let values_at: u32 = f.points.iter().map(|p| p.len() as u32).sum(); + let mut total = values_at + f.values.len() as u32; + let chain_at: Vec = runs + .iter() + .map(|run| { + let at = total; + total += 1 + RoundStorage::words(&f.shape.run(run.len())); + at + }) + .collect(); + let hash = match f.base { + Base::Rpx => WrapHash::production(), + Base::P1 => WrapHash::Poseidon1, + }; + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let arena = b.declare_arena(total); + let mut transcript = WhirTranscript::for_builder(&mut b); + let mut next = 0u32; + let points: Vec> = f + .points + .iter() + .map(|p| { + p.iter() + .map(|_| { + let w = b.hint_word(arena, next).as_ext(); + next += 1; + w + }) + .collect() + }) + .collect(); + let values: Vec = (0..f.values.len() as u32) + .map(|i| b.hint_word(arena, values_at + i).as_ext()) + .collect(); + let mut finals: Vec = Vec::new(); + let mut storage: Vec = Vec::new(); + for (run, &base) in runs.iter().zip(&chain_at) { + finals.push(b.hint_word(arena, base).as_ext()); + storage.push(RoundStorage::hint( + &mut b, + arena, + base + 1, + &f.shape.run(run.len()), + )); + } + let roots: Vec = order + .iter() + .map(|&p| b.digest_const(commitment_to_digest(&f.roots[p])).as_cell()) + .collect(); + let openings: Vec<_> = storage.iter().map(RoundStorage::openings).collect(); + let rounds: Vec> = storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + let chains: Vec> = runs + .iter() + .enumerate() + .map(|(i, run)| StackedChainWires { + rounds: &rounds[i], + roots: &roots[run.clone()], + final_value: finals[i], + }) + .collect(); + let claimed_at: Vec<&[Ext]> = points.iter().map(Vec::as_slice).collect(); + let gamma = emit_stacked_verify_chains( + &mut b, + &mut transcript, + &f.layout, + &chains, + &claimed_at, + &values, + None, + &f.shape, + &f.domain, + ); + b.public(gamma.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the constant-root leg must be admissible"); + program + }) +} + +/// The arena [`const_root_program`] reads: the points, the values, then per +/// chain its final value and its rounds. +fn const_root_arena(f: &Fixture, proof: &StackedProof) -> Vec { + let mut words: Vec = f.points.iter().flatten().map(ext_word).collect(); + words.extend(f.values.iter().map(ext_word)); + let runs = f.shape.chains(f.layout.num_polys()); + for (chain, run) in proof.polys.iter().zip(&runs) { + words.push(ext_word(&chain.final_value)); + push_round_words(&mut words, &f.shape.run(run.len()), chain); + } + words +} + +/// The code review's M-g at the stacked level: grouped chains opened +/// against roots that are PROGRAM CONSTANTS, the wiring of the block leaf's +/// prepared branch. The honest arena executes; with the constants of trees 0 +/// and 1 exchanged it has no execution (tree 0's opening at query 0, or its cap +/// where the tree is capped); a value of tree 1's block at query 1 is refused +/// at tree 1's opening there. Under RPX (a run of three, a split [4, 1]) and +/// Poseidon1 (C2, and a split). +#[test] +fn grouped_chains_open_against_constant_roots() { + let picked = |c: &Case| { + c.base == Base::P1 + || (c.n_stack == 6 + && [3, 5].contains(&c.heights.len()) + && c.grind > 0 + && c.cap == CapPolicy::Off) + }; + for case in cases().into_iter().filter(picked) { + let f = fixture(&Case { + num_queries: 8, + ..case + }); + let order: Vec = (0..f.roots.len()).collect(); + let traced = const_root_program(&f, &order); + let words = const_root_arena(&f, &f.proof); + assert!( + machine_refusal(&f, &traced, words.clone()).is_none(), + "{}: the honest arena executes against constant roots", + f.name + ); + + let mut exchanged = order.clone(); + exchanged.swap(0, 1); + let wrong = const_root_program(&f, &exchanged); + let refused = machine_refusal(&f, &wrong, words.clone()); + let capped = f.shape.caps[0] > 0; + assert!( + match &refused { + Some(Refused::At(Some(0), s)) => !capped && *s == onechain_steps::opening(0, 0, 0), + // A capped tree's cap is authenticated before the chain's first + // marked check. + Some(Refused::Other(LfmExecError::DivByZero { .. })) => capped, + _ => false, + }, + "{}: constants exchanged, refused at {refused:?}", + f.name + ); + + let mut forged = f.proof.clone(); + grouped(&mut forged, 0).current[1][1].values[2] += FE::one(); + let refused = machine_refusal(&f, &traced, const_root_arena(&f, &forged)); + assert!( + matches!(&refused, Some(Refused::At(Some(0), s)) if *s == onechain_steps::opening(0, 1, 1)), + "{}: a value of tree 1's block, refused at {refused:?}", + f.name + ); + println!( + "ONECHAIN CONST ROOTS [{}]: runs {:?}, capped {capped}", + f.name, + f.shape.chains(f.layout.num_polys()) + ); + } +} + +// ─────────── each binding of a grouped round 0, on its own ─────────── + +/// The chain whose PROLOGUE holds the assert a `DivByZero` failed at: after +/// that chain's `Chain` mark and before its round 0's `Sumcheck` mark — its +/// roots unpacked, its claim batched and its round-0 trees' caps +/// authenticated, of which only the caps assert. `None` for any other refusal. +fn refused_in_prologue( + (program, marks): &(LfmProgram, Vec<(Step, usize)>), + error: &LfmExecError, +) -> Option { + let LfmExecError::DivByZero { addr } = error else { + return None; + }; + let index = program.instrs.iter().position(|i| match i { + Instr::BaseAlu { out, .. } | Instr::ExtAlu { out, .. } => out.0 == *addr, + _ => false, + })?; + match marks.iter().rev().find(|(_, at)| *at <= index)? { + (Step::Chain(c), _) => Some(*c), + _ => None, + } +} + +/// EACH round-0 tree of a grouped chain is bound, in guest, to the root at its +/// own position, its cap included — tree 0 as much as a later one. Trading +/// trees 0's and 1's caps ([`round_zero_caps_are_bound_per_tree_in_guest`]) +/// breaks two bindings at once, so it cannot show either tree bound alone; +/// this forgery moves one slot and leaves every other tree as committed. +/// +/// For every slot `p` in turn, slot `p` commits another polynomial; the proof +/// is that stack's honest one (its own trees, caps and paths, +/// self-consistent), claimed against the stated roots, which differ from its +/// own at slot `p` alone. The claim is false, and every check but tree `p`'s +/// binding to its root passes. A capped tree refuses at its cap (host +/// `CapRejected`; the machine in the chain's prologue, where the caps are +/// authenticated), an uncapped one at its first opening +/// (`GroupedOpeningRejected { 0, p }`; the machine at tree `p`'s opening at +/// query 0). +#[test] +fn a_tree_committing_another_polynomial_has_no_execution() { + let cases = [ + Case { + cap: CapPolicy::Fixed(1), + ..rpx("three polynomials under a binary cap", &[6; 3], 6, 8) + }, + rpx("three polynomials, uncapped", &[6; 3], 6, 8), + Case { + base: Base::P1, + first: Some(6), + log_folding: 4, + cap: CapPolicy::Fixed(2), + ..rpx("P1, three polynomials at C2", &[10; 3], 10, 6) + }, + ]; + for case in cases { + let capped = case.cap != CapPolicy::Off; + let honest = build(&case, GroupFold::Indexed); + for slot in 0..3 { + let mut forged = + build_over(&case, GroupFold::Indexed, Some((slot, 0xBAD + slot as u64))); + assert_eq!(forged.points, honest.points); + assert_ne!(forged.values, honest.values); + for p in 0..3 { + assert_eq!( + forged.roots[p] == honest.roots[p], + p != slot, + "{}: root {p}", + case.name + ); + } + let traced = traced_program(&forged); + assert!( + host_accepts(&forged, &forged.proof), + "{}: the control", + case.name + ); + assert!( + machine_refusal(&forged, &traced, arena(&forged, &forged.proof)).is_none(), + "{}: the control executes", + case.name + ); + let proof = forged.proof.clone(); + forged.roots = honest.roots.clone(); + let cause = if capped { + multilinear::Error::CapRejected + } else { + multilinear::Error::GroupedOpeningRejected { + query: 0, + tree: slot, + } + }; + assert_eq!( + host_refusal(&forged, &proof), + Some(Refusal::Chain { + chain: 0, + poly: 0, + cause + }), + "{} slot {slot}: the host's check", + case.name + ); + let words = arena(&forged, &proof); + if capped { + let Err(error) = execute_serial( + &traced.0, + std::slice::from_ref(&words), + &socket(forged.base), + ) else { + panic!( + "{} slot {slot}: a tree's cap off its root was executed", + case.name + ); + }; + assert!(execute(&traced.0, &[words], &socket(forged.base)).is_err()); + assert_eq!( + refused_in_prologue(&traced, &error), + Some(0), + "{} slot {slot}: refused at the chain's caps, got {error:?}", + case.name + ); + } else { + let refused = machine_refusal(&forged, &traced, words); + assert!( + matches!( + refused, + Some(Refused::At(Some(0), Step::Opening { round: 0, query: 0, tree })) if tree == slot + ), + "{} slot {slot}: refused at {refused:?}", + case.name + ); + } + } + } +} + +/// The grouped leg's instruction stream, emitted over hinted wires alone — it +/// is a function of the layout and the config, never of a proof — as +/// `(instructions, FNV-1a of their Debug text)`. ★ The SAME body sits in +/// `prover/tests/onechain_release_stream.rs`, built against the library +/// WITHOUT `cfg(test)` (no faults, no step marks), and both builds are held to +/// the same pins: the test build emits exactly the release build's program. +fn grouped_leg_stream(p1: bool, heights: &[usize], n_stack: usize) -> (usize, u64) { + use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, WhirFolds, + }; + let mut format = ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).expect("first6")), + group: GroupFold::Indexed, + ..ChainFormat::DEFAULT + }; + if p1 { + format.arity4_cap = CapPolicy::Fixed(2); + } else { + format.cap = CapPolicy::Fixed(1); + } + let cfg = ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 3, + grind: GrindBits::uniform(8), + format, + }; + let layout = multilinear::stacking::StackedLayout::build(heights, n_stack).expect("packs"); + let shape = ChainShape::new_at(&cfg, n_stack, if p1 { 4 } else { 2 }); + let domain = multilinear::whir::Domain::::new(n_stack + cfg.log_blowup) + .expect("domain"); + let hash = if p1 { + WrapHash::Poseidon1 + } else { + WrapHash::production() + }; + let runs = shape.chains(layout.num_polys()); + let mut total: u32 = heights.iter().sum::() as u32 + heights.len() as u32; + for run in &runs { + total += run.len() as u32 + 1 + RoundStorage::words(&shape.run(run.len())); + } + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let arena = b.declare_arena(total); + let mut transcript = WhirTranscript::for_builder(&mut b); + let mut at = 0u32; + let next = |b: &mut LfmBuilder, at: &mut u32| { + let cell = b.hint_word(arena, *at); + *at += 1; + cell + }; + let points: Vec> = heights + .iter() + .map(|&h| (0..h).map(|_| next(&mut b, &mut at).as_ext()).collect()) + .collect(); + let values: Vec = heights + .iter() + .map(|_| next(&mut b, &mut at).as_ext()) + .collect(); + let mut roots: Vec> = Vec::new(); + let mut finals: Vec = Vec::new(); + let mut storage: Vec = Vec::new(); + for run in &runs { + roots.push(run.clone().map(|_| next(&mut b, &mut at)).collect()); + finals.push(next(&mut b, &mut at).as_ext()); + let run_shape = shape.run(run.len()); + let base = at; + storage.push(RoundStorage::hint(&mut b, arena, base, &run_shape)); + at += RoundStorage::words(&run_shape); + } + let openings: Vec<_> = storage.iter().map(RoundStorage::openings).collect(); + let rounds: Vec> = storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + let chains: Vec> = (0..runs.len()) + .map(|i| StackedChainWires { + rounds: &rounds[i], + roots: &roots[i], + final_value: finals[i], + }) + .collect(); + let claimed_at: Vec<&[Ext]> = points.iter().map(Vec::as_slice).collect(); + let gamma = emit_stacked_verify_chains( + &mut b, + &mut transcript, + &layout, + &chains, + &claimed_at, + &values, + None, + &shape, + &domain, + ); + b.public(gamma.as_cell()); + let program = compile(b.finish()); + let text = format!("{:?}", program.instrs); + let digest = text.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, byte| { + (h ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3) + }); + (program.instrs.len(), digest) +} + +/// One pinned case: the base (`true` for Poseidon1), the columns' heights, the +/// stack's height, and the stream's `(instructions, FNV-1a)`. +type StreamPin = (bool, &'static [usize], usize, (usize, u64)); + +/// The cases both builds are held to, each with its stream's pin: RPX and +/// Poseidon1, five polynomials split [4, 1] and six split [4, 2] whose short +/// columns' selectors run past `k0`. A pin moves only with the emitted +/// program; move it in both files together. +const GROUPED_LEG_STREAMS: [StreamPin; 4] = [ + (false, &[8; 5], 8, (5241, 0x8d95_4e39_3cca_f387)), + (false, &[8, 3, 8, 2, 4, 8], 8, (5229, 0xac5d_a244_80ae_4271)), + (true, &[10; 5], 10, (28895, 0x52ec_ae97_ae80_6593)), + ( + true, + &[10, 3, 10, 2, 4, 10], + 10, + (28868, 0x02e6_e3fd_5c51_0e98), + ), +]; + +/// Under `Fault::None`, and with the step marks recorded or not, the test +/// build emits one program — and it is the pinned one, which +/// `prover/tests/onechain_release_stream.rs` holds a build without `cfg(test)` +/// to as well: the test build's program is the release build's. +#[test] +fn the_grouped_leg_stream_of_a_test_build() { + for (p1, heights, n, pin) in GROUPED_LEG_STREAMS { + let plain = grouped_leg_stream(p1, heights, n); + let (marked, marks) = onechain_steps::record(|| { + onechain_faults::with(onechain_faults::Fault::None, || { + grouped_leg_stream(p1, heights, n) + }) + }); + assert_eq!(plain, marked, "the marks and Fault::None emit nothing"); + assert!(!marks.is_empty(), "the marks were recorded"); + println!( + "ONECHAIN STREAM p1={p1} heights={heights:?}: {} instructions, digest {:016x}", + plain.0, plain.1 + ); + assert_eq!(plain, pin, "p1={p1} heights={heights:?}: the pinned stream"); + } +} + +/// In guest, a grouped round 0 that is the chain's last round (R = 1) compares +/// each query's combined fold with the final value, as its own check ahead of +/// the final weight check. Unground, so a moved final +/// value moves only the query positions; the forgery is the first such value +/// whose first position does not move (two leaves a tree), so that query's +/// openings stay valid and the host refuses at its fold — and the machine at +/// the same fold. +#[test] +fn a_one_round_grouped_chain_folds_to_its_final_value_in_guest() { + let case = Case { + first: Some(6), + log_folding: 4, + log_blowup: 1, + ..rpx("one round, unground", &[5; 3], 5, 0) + }; + let f = fixture(&case); + assert_eq!(f.shape.rounds(), 1, "R = 1"); + let traced = traced_program(&f); + assert!(machine_refusal(&f, &traced, arena(&f, &f.proof)).is_none()); + let fold = || Refusal::Chain { + chain: 0, + poly: 0, + cause: multilinear::Error::FoldInconsistent { query: 0 }, + }; + let forged = (1u64..512) + .map(|delta| { + let mut proof = f.proof.clone(); + proof.polys[0].final_value += FEE::from(delta); + proof + }) + .find(|proof| host_refusal(&f, proof) == Some(fold())) + .expect("a final value whose first query position does not move, refused at the fold"); + let refused = machine_refusal(&f, &traced, arena(&f, &forged)); + assert!( + matches!( + refused, + Some(Refused::At(Some(0), Step::Fold { round: 0, query: 0 })) + ), + "the machine refused at {refused:?}, not round 0's first fold" + ); +} + +// ─────────── the grouped round 0's successor, bound to its root in guest ─────────── + +/// A grouped round 0's SUCCESSOR tree is bound, through its cap, to the root +/// the round absorbed — in guest as on the host (`whir_round::verify_grouped`). +/// Each node of the successor's cap moved in turn (in the proof, so the host +/// reads the same forgery) is refused by the host at the cap, and has no +/// execution: the failing assert lies in round 0's out-of-domain step, where +/// the successor's cap is authenticated — never at an opening, which a cap +/// node no query reaches would not meet. +#[test] +fn a_grouped_successor_cap_off_its_root_has_no_execution() { + let cases = [ + // A cap of eight nodes, unground: the successor is opened at most six + // times (three as round 0's successor, three as round 1's current + // tree), so some node is met by no opening whatever the draw, and the + // draw is the same every build. Were the cap not authenticated, the + // arenas that move those nodes would execute. + Case { + cap: CapPolicy::Fixed(3), + ..rpx( + "three polynomials under a binary cap of eight, unground", + &[6; 3], + 6, + 0, + ) + }, + Case { + cap: CapPolicy::Fixed(1), + ..rpx("three polynomials under a binary cap", &[6; 3], 6, 8) + }, + Case { + base: Base::P1, + first: Some(6), + log_folding: 4, + cap: CapPolicy::Fixed(2), + ..rpx("P1, three polynomials at C2", &[10; 3], 10, 6) + }, + ]; + for case in cases { + let f = build(&case, GroupFold::Indexed); + let traced = traced_program(&f); + let run = f.shape.run(f.shape.chains(f.layout.num_polys())[0].len()); + assert!(run.trees >= 2, "{}: a grouped chain", case.name); + let depth = run.next_depth(0).expect("round 0 has a successor"); + let cap_words = run.cap_words(depth, run.caps[1]); + assert!( + cap_words >= 2, + "{}: a capped successor, {cap_words} cap words", + case.name + ); + assert!( + machine_refusal(&f, &traced, arena(&f, &f.proof)).is_none(), + "{}: the control executes", + case.name + ); + // Every node first, then the verdict: a node no query reaches is the + // one only the cap's own check refuses. + let mut elsewhere = Vec::new(); + for node in 0..cap_words { + let mut forged = f.proof.clone(); + let owner = &mut grouped(&mut forged, 0).next[0].proof.merkle_path; + let at = owner.len() - cap_words + node; + owner[at][0] ^= 1; + let host = host_refusal(&f, &forged); + let refused = machine_refusal(&f, &traced, arena(&f, &forged)); + println!( + "ONECHAIN SUCCESSOR CAP [{}] node {node} of {cap_words}: host {host:?}, machine \ + {refused:?}", + case.name + ); + let at_the_cap = + host == Some(Refusal::Chain { + chain: 0, + poly: 0, + cause: multilinear::Error::CapRejected, + }) && matches!(refused, Some(Refused::At(Some(0), Step::Ood { round: 0 }))); + if !at_the_cap { + elsewhere.push((node, host, refused)); + } + } + assert!( + elsewhere.is_empty(), + "{}: a successor cap node off its root is refused at the cap by the host and where \ + the cap is authenticated by the machine; not so: {elsewhere:?}", + case.name + ); + } +} diff --git a/prover/src/lfm/whir_open.rs b/prover/src/lfm/whir_open.rs new file mode 100644 index 000000000..74dd1a7ba --- /dev/null +++ b/prover/src/lfm/whir_open.rs @@ -0,0 +1,393 @@ +//! A WHIR query opening as a machine leg — the hashing half of the chain's +//! dominant term. +//! +//! `whir_commit::verify_opening` (`crypto/multilinear/src/whir_commit.rs:416`) +//! is one line: `opening.proof.verify::>(root, index, &values)`. +//! Under the RPX pin that unfolds into exactly two primitives this machine +//! already has — +//! +//! - the LEAF is `sponge_leaf` over the block's felts +//! (`merkle_tree/backends/rpx.rs:80-87`), which is +//! [`WrapHash::Algebraic`](super::edsl::WrapHash)'s leaf hash; and +//! - a PARENT is `compress`, one permutation of `[left ‖ right ‖ 0⁴]` +//! (`:116-126`), which is `b.compress` +//! +//! — walked by `verify_merkle_path_from_leaf_hash` +//! (`crypto/crypto/src/merkle_tree/proof.rs:30-50`): level by level, the +//! current node is the LEFT child when the index's low bit is zero, then the +//! index shifts right. That is [`edsl::wrap_merkle_walk`]'s contract verbatim, +//! so the walk is a reuse and not a port. +//! +//! # ⚠ The felt decomposition, and why it is not a choice +//! +//! The leaf hashes `element_felts` of each value in order (`rpx/mod.rs:470`), +//! which streams the element's big-endian bytes: ONE felt for a base element, +//! THREE for a cubic extension element in coordinate order. Coordinate order is +//! the order `unpack` returns lanes 0–2 in, which is why an extension value +//! costs one `Unpack` here and no byte work at all. +//! +//! ★ **The capacity is keyed on the FELT count, not the value count.** A block +//! of sixteen extension values is forty-eight felts, and `leaf_capacity` reads +//! `len % 8` off that (`rpx/mod.rs:349-354`). The two counts agree modulo eight +//! for a sixteen-wide block and DISAGREE for the two-wide tail block the last +//! round opens (six felts against two values), which is the shape that catches +//! the mistake. +//! +//! # ★ Why a round-0 base value may be lifted into the fold, and what pins it +//! +//! Round 0's current codeword is base-field (`whir_chain.rs:983` rejects any +//! other arrangement), so its opened values arrive as hinted felts and the fold +//! then treats them as extension elements through [`Felt::as_ext`]. That lift +//! is only sound if lanes 1–3 of the hinted word are zero, and a hint +//! constrains nothing: `hint_felt` is `hint_word` retyped +//! (`builder.rs:632-634`) and the arena chip sends a full `word_token` +//! (`chips.rs:105`). +//! +//! What pins them is the leaf hash itself, and it is pinned TWICE by two +//! independent mechanisms. `algebraic_leaf_hash` feeds every value to +//! `pack_word`; in the AIR, `Pack` RECEIVES each lane as a `base_token`, +//! `(addr, v, 0, 0, 0)` with three tuple-constant zeros (`chips.rs:64-66`, +//! `:1934-1937`), and the memory bus is a multiset, so the arena's send of +//! `(addr, l0, l1, l2, l3)` can only balance against that receive when +//! `l1 = l2 = l3 = 0` — that is the one that carries the soundness. In the +//! executor, the same `Pack` reads each lane through `read_base`, which refuses +//! a word with a nonzero upper lane outright (`executor.rs:871` → `:398-401`, +//! `LfmExecError::NotBaseWord`) — that is the one a test can watch, and +//! `whir_open_tests::the_leaf_pins_a_hinted_base_value_to_its_low_lane` names +//! the variant rather than settling for `is_err`. The block is leaf-hashed in +//! full by construction here, so every base value the fold reads has been +//! pinned before it reads it. +//! +//! ⚠ The obligation that follows, written down because it is exactly the kind a +//! later caller can forget: **a base value that is NOT part of a leaf this leg +//! hashes is not pinned, and must not be lifted.** The bad state is unreachable +//! only while the values folded and the values hashed are the same slice. +//! +//! # Under Poseidon1 +//! +//! A [`WrapHash::Poseidon1`](super::edsl::WrapHash) builder verifies a +//! Poseidon1 base's 4-ary trees: the leaf is `poseidon1_stark::linear_hash` +//! (rate 12, the width tag; [`edsl::wrap_leaf_hash`] dispatches), and the walk +//! is [`super::p1w16_emit::walk4_child`] over the path in the proof's own child +//! order, three siblings a 4-ary level and one at an uncapped odd top +//! ([`super::merkle_cap::child_path_to_cap`] lays the arena out). The `Pack` +//! pin above holds the same way: the leaf packs every value lane. +//! +//! # Under the lean query +//! +//! A lean leaf (`LeafQuery::Lean`) hints round 0's base values four to a word +//! ([`BlockValues::BaseWords`]): the leaf hash takes the words as its rate +//! words ([`super::p1w16_emit::leaf_hash_words`]) and the fold reads the felts +//! each word's `Unpack` writes. The pin moves from `Pack` to `Unpack`, which +//! receives the word token and sends each lane as a `base_token` +//! `(addr, v, 0, 0, 0)` from the same value columns (`chips::lanes`): every +//! felt the fold lifts is a base cell and a lane of a hashed word. A word's +//! lanes past the block's last value are hashed as given: a nonzero fill is +//! bound by the leaf hash, though no committer's zero-filled leaf matches it, +//! and the fold never reads it. No block chain has such lanes (round 0's block +//! is 16, 32 or 64 values). The obligation above holds unchanged: the felts +//! folded are the unpacked lanes of the words hashed. +//! +//! Its paths stay in the arena in the proof's child order until their walk, +//! which hints each level's siblings in hint order — the executor picks the +//! word on the level's `b1` (`Instr::HintOn`) — and places them with +//! [`super::p1w16_emit::walk4`]'s three `Select`s ([`Siblings::Rotated`]). +//! Which word a hint reads is witness: every placement is still a `Select` on +//! an index bit, so the node hashed is the one the index names. + +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::edsl::{self, WrapDigest}; +use super::instr::ArenaId; + +/// Base felts a cubic extension element decomposes to. +const FELTS_PER_EXT: usize = 3; + +/// The sponge's rate, in felts — the block size `leaf_capacity` is keyed on. +/// +/// Shared with [`super::whir_transcript`], whose sponge is the same one: a +/// second spelling of the rate is a second thing to keep in step. +pub(super) const RATE_FELTS: usize = 8; + +/// Felts a `Pack` assembles into one sponge word. +const FELTS_PER_WORD: usize = 4; + +/// A queried block's values, in the field the round holds them in. +/// +/// Two variants rather than one slice of `Ext`, because the FIELD is what the +/// leaf hash reads: a base block is one felt per value and an extension block +/// is three, and hashing sixteen base values as though they were extension +/// elements produces a digest no committer ever computed. +#[derive(Clone, Copy)] +pub enum BlockValues<'a> { + /// Round 0's current codeword, before any extension challenge has touched + /// it — `RoundOpenings::Base` on the host. + Base(&'a [Felt]), + /// [`Self::Base`] as a lean leaf hints it (`LeafQuery::Lean`): the + /// values four a hinted word, which the leaf hash takes as its rate words, + /// and the felts each word's `Unpack` writes, which the fold reads (the + /// module doc's "Under the lean query"). + BaseWords { + words: &'a [Cell], + felts: &'a [Felt], + }, + /// Every successor block, and every current block after round 0. + Ext(&'a [Ext]), +} + +impl BlockValues<'_> { + /// Values in the block: `2^k` for a fold of `k` variables. + pub fn len(&self) -> usize { + match self { + BlockValues::Base(v) => v.len(), + BlockValues::BaseWords { felts, .. } => felts.len(), + BlockValues::Ext(v) => v.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Base felts the leaf hashes — the number `leaf_capacity` is keyed on. + pub fn felts(&self) -> usize { + match self { + BlockValues::Base(v) => v.len(), + BlockValues::BaseWords { felts, .. } => felts.len(), + BlockValues::Ext(v) => FELTS_PER_EXT * v.len(), + } + } +} + +/// INSTRUCTIONS [`emit_block_leaf`] emits over a block of `felts` felts reached +/// through `unpacks` `Unpack` rows. +/// +/// Every term by its shape: one `Unpack` per EXTENSION value and none for a +/// base one (a base value is already a lane); `ceil(felts/4)` `Pack` rows, one +/// per sponge word, the tail zero-padded; and `ceil(words/2)` permutations, +/// because a rate block is two words. The two ceilings compose to +/// `ceil(felts/8)` at every width. +pub const fn block_leaf_rows(felts: usize, unpacks: usize) -> usize { + if felts == 0 { + // `sponge_leaf` returns the zero digest without permuting, and so does + // the emitter. Unreachable from a chain — a block is `2^k` values with + // `k >= 1` — and stated rather than left to underflow. + return unpacks; + } + unpacks + felts.div_ceil(FELTS_PER_WORD) + felts.div_ceil(RATE_FELTS) +} + +/// INSTRUCTIONS [`emit_verify_opening`] emits for one query. +/// +/// The leaf, then the walk at two rows a level — one `Select` and one +/// `compress`, because an algebraic digest is ONE cell and the whole digest +/// swaps on the same bit (`edsl.rs:756-771`) — then the root comparison, which +/// is one `Unpack` of the walked cell and four lowered asserts at two rows each +/// (`edsl.rs:846-851`, `builder.rs:283-287`). The root's own lanes are unpacked +/// once by the caller and shared across every query against that root, so they +/// are not charged here. +pub const fn verify_opening_rows(felts: usize, unpacks: usize, depth: usize) -> usize { + verify_opening_rows_capped(felts, unpacks, depth, 0) +} + +/// [`verify_opening_rows`] against a tree capped at height `cap` +/// ([`CapCells`]): the walk stops `cap` levels short (`2·cap` rows fewer), +/// the cap mux picks the node with `2^cap − 1` `Select` rows, and the +/// comparison is of two VARIABLE cells, so it unpacks both (one `Unpack` more +/// than against the hoisted root lanes). At `cap = 0` it is the root form. +pub const fn verify_opening_rows_capped( + felts: usize, + unpacks: usize, + depth: usize, + cap: usize, +) -> usize { + let leaf = block_leaf_rows(felts, unpacks); + let walk = 2 * (depth - cap); + let compare = 1 + 2 * FELTS_PER_WORD; + if cap == 0 { + leaf + walk + compare + } else { + leaf + walk + ((1usize << cap) - 1) + 1 + compare + } +} + +/// PERMUTATIONS one query's opening costs: the leaf's blocks plus one parent a +/// level. This is the term the chain's cost is dominated by, and it is a +/// function of the block's felts and the tree's depth alone — no row +/// bookkeeping enters it. +pub const fn verify_opening_perms(felts: usize, depth: usize) -> usize { + verify_opening_perms_capped(felts, depth, 0) +} + +/// [`verify_opening_perms`] against a tree capped at height `cap`: `cap` +/// parents fewer. The cap's own `2^cap − 1` parents are paid once per TREE, +/// by [`cap_check_perms`]. +pub const fn verify_opening_perms_capped(felts: usize, depth: usize, cap: usize) -> usize { + felts.div_ceil(RATE_FELTS) + depth - cap +} + +/// PERMUTATIONS one tree's cap check costs: the cap hashed up to its root, +/// `2^cap − 1` parents. Nothing at `cap = 0`. +pub const fn cap_check_perms(cap: usize) -> usize { + (1usize << cap) - 1 +} + +/// INSTRUCTIONS one tree's cap check costs beyond its hinted words: the +/// `2^cap − 1` parents (one `compress` each) and the root comparison (one +/// `Unpack` and four lowered asserts). Nothing at `cap = 0`: an uncapped tree +/// is compared against its root lanes query by query. +pub const fn cap_check_rows(cap: usize) -> usize { + if cap == 0 { + 0 + } else { + cap_check_perms(cap) + 1 + 2 * FELTS_PER_WORD + } +} + +/// ★ The block's Merkle leaf: `sponge_leaf` over its felts. +/// +/// Goes through [`edsl::wrap_leaf_hash`] rather than restating the duplex, so +/// the leaf this authenticates against and the leaf the commitment builder +/// computes have one definition between them. +pub fn emit_block_leaf(b: &mut LfmBuilder, values: BlockValues<'_>) -> WrapDigest { + match values { + BlockValues::Base(felts) => edsl::wrap_leaf_hash(b, felts), + BlockValues::BaseWords { words, felts } => { + assert_eq!( + b.wrap_hash(), + edsl::WrapHash::Poseidon1, + "packed round-0 words hash under the Poseidon1 leaf" + ); + super::p1w16_emit::leaf_hash_words(b, words, felts.len()) + } + BlockValues::Ext(cells) => { + let mut felts = Vec::with_capacity(FELTS_PER_EXT * cells.len()); + for value in cells { + let lanes = b.unpack(value.as_cell()); + felts.extend_from_slice(&lanes[..FELTS_PER_EXT]); + } + edsl::wrap_leaf_hash(b, &felts) + } + } +} + +/// ★ `whir_commit::verify_opening`, emitted — as a REFUSAL rather than a bool. +/// +/// The host returns `false` and its caller turns that into +/// `Error::OpeningRejected`. Here the comparison IS the rejection: +/// `assert_digest_eq_lanes` lowers to `diff / 0`, which has no satisfying +/// assignment unless the two agree (`builder.rs:283-292`), so an opening that +/// does not authenticate has no execution rather than a `false` somebody could +/// forget to branch on. +/// +/// `index_bits` are the leaf index low-to-high with one bit per level, which is +/// both what `sample_u64_pow2` hands back and what the walk consumes; `depth` +/// is therefore `index_bits.len()` and is not passed separately. `root_lanes` +/// is the root unpacked ONCE by the caller: a round opens `Q` blocks against +/// the same root, and unpacking it per query would charge `Q − 1` rows for a +/// value that never changes. +pub fn emit_verify_opening( + b: &mut LfmBuilder, + values: BlockValues<'_>, + index_bits: &[Bit], + siblings: &[WrapDigest], + root_lanes: &[Felt; 4], +) { + let hash = b.wrap_hash(); + assert_eq!( + hash.path_hints(index_bits.len()), + siblings.len(), + "the path's siblings for every level of the tree" + ); + let leaf = emit_block_leaf(b, values); + let walked = if hash.arity() == 4 { + super::p1w16_emit::walk4_child(b, leaf, index_bits, siblings) + } else { + edsl::wrap_merkle_walk(b, leaf, index_bits, siblings) + }; + edsl::assert_digest_eq_lanes(b, walked, std::slice::from_ref(root_lanes)); +} + +/// ★ One tree's authenticated Merkle cap — the gadget the STARK verifier +/// shares ([`super::merkle_cap`]): its only constructor checks the cap against +/// the tree's root, and its one entry point walks, muxes and compares. +pub use super::merkle_cap::CapCells; + +/// An opening's authentication path as the leaf takes it. +#[derive(Clone, Copy)] +pub enum Siblings<'a> { + /// Hinted with the round's other wires, in the proof's child order + /// ([`super::p1w16_emit::walk4_child`] at arity 4). + Child(&'a [WrapDigest]), + /// Left in `arena` from word `at`, in the proof's child order, and hinted + /// at the walk in hint order (`LeafQuery::Lean`, + /// [`super::p1w16_emit::hint_path_rotated`]) for + /// [`super::p1w16_emit::walk4`]'s three `Select`s a level. Arity 4 only. + Rotated { arena: ArenaId, at: u32 }, +} + +/// How one tree's openings are authenticated in-guest: against its root +/// lanes (no cap — today's emission, instruction for instruction), or +/// against its authenticated [`CapCells`]. +pub enum TreeAuth { + Root([Felt; 4]), + Cap(CapCells), +} + +impl TreeAuth { + /// The cap height the openings are cut to (0 for a root). + pub fn cap_height(&self) -> usize { + match self { + TreeAuth::Root(_) => 0, + TreeAuth::Cap(cap) => cap.height(), + } + } + + /// ★ `whir_commit::verify_opening_capped`, emitted as a refusal. + /// + /// `index_bits` is the WHOLE leaf index, low first, one bit per tree + /// level; `siblings` is the path to the cap, `index_bits.len() − c` long. + /// The low bits are walked and the top `c` pick the cap node. With a + /// [`TreeAuth::Root`] this is [`emit_verify_opening`] exactly. + pub fn verify_opening( + &self, + b: &mut LfmBuilder, + values: BlockValues<'_>, + index_bits: &[Bit], + siblings: Siblings<'_>, + ) { + if let Siblings::Rotated { .. } = siblings { + assert_eq!(b.wrap_hash().arity(), 4, "a rotated path is 4-ary"); + } + match (self, siblings) { + (TreeAuth::Root(lanes), Siblings::Child(siblings)) => { + emit_verify_opening(b, values, index_bits, siblings, lanes) + } + (TreeAuth::Root(lanes), Siblings::Rotated { arena, at }) => { + let leaf = emit_block_leaf(b, values); + let hints = super::p1w16_emit::hint_path_rotated(b, arena, at, index_bits); + let walked = super::p1w16_emit::walk4(b, leaf, index_bits, &hints); + edsl::assert_digest_eq_lanes(b, walked, std::slice::from_ref(lanes)); + } + (TreeAuth::Cap(cap), siblings) => { + // The cap checks the path's length against the index and its + // height at the builder's arity. + let leaf = emit_block_leaf(b, values); + match siblings { + Siblings::Child(siblings) => { + cap.verify_path_child(b, leaf, index_bits, siblings) + } + Siblings::Rotated { arena, at } => { + let walked = index_bits.len().saturating_sub(cap.mux_bits()); + let hints = super::p1w16_emit::hint_path_rotated( + b, + arena, + at, + &index_bits[..walked], + ); + cap.verify_path(b, leaf, index_bits, &hints); + } + } + } + } + } +} diff --git a/prover/src/lfm/whir_open_tests.rs b/prover/src/lfm/whir_open_tests.rs new file mode 100644 index 000000000..e87313a3e --- /dev/null +++ b/prover/src/lfm/whir_open_tests.rs @@ -0,0 +1,772 @@ +//! Gates for the WHIR query opening. +//! +//! The commitment under test is a real one — `CodewordCommitment` over a real +//! codeword, opened at a real index — so the leaf the machine recomputes is the +//! leaf a committer actually hashed, and the path is a path the tree actually +//! produced. `from_codeword_on_host` rather than `from_codeword`, so no test +//! here asks for a card. + +use multilinear::whir_commit::{CodewordCommitment, CosetOpening, verify_opening}; +use multilinear::whir_hash::RpxWhir; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::commitment_to_digest; +use super::builder::{Ext, Felt, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::WrapDigest; +use super::executor::execute; +use super::validator::validate; +use super::whir_open::{ + BlockValues, CapCells, TreeAuth, cap_check_perms, cap_check_rows, emit_verify_opening, + verify_opening_perms, verify_opening_perms_capped, verify_opening_rows, + verify_opening_rows_capped, +}; +use super::word::{LfmWord, ext_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +fn base(v: u64) -> FE { + FE::from(v.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 3) +} + +fn ext(v: u64) -> FEE { + FEE::new([base(v), base(v ^ 0x5151), base(v.wrapping_add(7))]) +} + +/// A base word `(v, 0, 0, 0)` — what a hinted base opening value must be, and +/// what [`the_leaf_pins_a_hinted_base_value_to_its_low_lane`] shows the bus +/// forces it to be. +fn base_word(v: FE) -> LfmWord { + [v, FE::zero(), FE::zero(), FE::zero()] +} + +/// One shape: the codeword's log size, the fold, and which field the block is +/// held in. `(log_domain, log_folding)` gives a tree of `log_domain − +/// log_folding` levels and a block of `2^log_folding` values. +struct Shape { + log_domain: usize, + log_folding: usize, + name: &'static str, +} + +/// The shapes a chain actually reaches, small enough to commit on a laptop: a +/// block of two (the `S mod k` tail round's width), of four, and the production +/// sixteen; and a one-level tree, where the walk's loop runs once. +const SHAPES: &[Shape] = &[ + Shape { + log_domain: 6, + log_folding: 1, + name: "block 2, 5 levels", + }, + Shape { + log_domain: 6, + log_folding: 2, + name: "block 4, 4 levels", + }, + Shape { + log_domain: 8, + log_folding: 4, + name: "block 16, 4 levels", + }, + Shape { + log_domain: 5, + log_folding: 4, + name: "block 16, 1 level", + }, +]; + +/// The arena, in the order [`opening_program`] hints it: the block's values, +/// then one word per sibling, then the root, then the index. +fn opening_arena( + values: &[LfmWord], + siblings: &[LfmWord], + root: LfmWord, + index: usize, +) -> Vec { + let mut words = values.to_vec(); + words.extend_from_slice(siblings); + words.push(root); + words.push([FE::from(index as u64), FE::zero(), FE::zero(), FE::zero()]); + words +} + +/// Hints in that order, decomposes the index in ONE `BitDec` — which is what +/// `WhirTranscript::sample_u64_pow2` emits, so the assembled verifier hands +/// these bits over rather than rebuilding them — and publishes the root so the +/// program has an output. +fn opening_program(block: usize, is_ext: bool, depth: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena((block + depth + 2) as u32); + + // ⚠ Only the field this round holds is hinted. Building both vectors and + // choosing between them afterwards hints the block TWICE, which F1 caught + // as `Hint 11` where the shape says nine. + let mut base_values: Vec = Vec::new(); + let mut ext_values: Vec = Vec::new(); + if is_ext { + ext_values = (0..block) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + } else { + base_values = (0..block).map(|i| b.hint_felt(arena, i as u32)).collect(); + } + + let siblings: Vec = (0..depth) + .map(|i| WrapDigest::from_cell(b.hint_word(arena, (block + i) as u32))) + .collect(); + let root = b.hint_word(arena, (block + depth) as u32); + let root_lanes = b.unpack(root); + let index = b.hint_felt(arena, (block + depth + 1) as u32); + let bits = b.bit_dec(index, depth); + + let values = if is_ext { + BlockValues::Ext(&ext_values) + } else { + BlockValues::Base(&base_values) + }; + emit_verify_opening(&mut b, values, &bits, &siblings, &root_lanes); + b.public(root); + + let program = compile(b.finish()); + validate(&program).expect("the opening leg must be admissible"); + program +} + +/// Rows the plumbing above contributes and the leg does not: the hints, the +/// root's one `Unpack`, the one `BitDec`, and the public. Counted by +/// construction rather than by differencing against a second program. +fn opening_plumbing(block: usize, depth: usize) -> usize { + (block + depth + 2) + 1 + 1 + 1 +} + +fn const_rows(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|instr| matches!(instr, super::instr::Instr::Const { .. })) + .count() +} + +/// The instruction histogram, by variant. F1 twice named a term I had written +/// down before measuring; this is what names them. +fn histogram(program: &LfmProgram) -> String { + use super::instr::Instr; + let mut counts = [0usize; 11]; + for instr in &program.instrs { + let i = match instr { + Instr::Const { .. } => 0, + Instr::BaseAlu { .. } => 1, + Instr::ExtAlu { .. } => 2, + Instr::Select { .. } => 3, + Instr::BitDec { .. } => 4, + Instr::Hash { .. } | Instr::Hash16(_) => 5, + Instr::Hint { .. } | Instr::HintOn { .. } => 6, + Instr::Pack { .. } => 7, + Instr::Unpack { .. } => 8, + Instr::Public { .. } => 9, + // No byte hash is reachable from this leg under the algebraic pin; + // counted so a change that reached one would show here. + Instr::KeccakF(_) | Instr::Blake3(_) => 10, + }; + counts[i] += 1; + } + let names = [ + "Const", "BaseAlu", "ExtAlu", "Select", "BitDec", "Hash", "Hint", "Pack", "Unpack", + "Public", "ByteHash", + ]; + names + .iter() + .zip(counts) + .filter(|(_, c)| *c > 0) + .map(|(n, c)| format!("{n} {c}")) + .collect::>() + .join(", ") +} + +/// `LFM_HASH` invocations: one per sponge permutation, whether it is a leaf's +/// duplex block or a Merkle parent's compress. Both lower to `Instr::Hash` and +/// differ only in `HashMode`, so counting the variant counts permutations. +fn perm_rows(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|instr| matches!(instr, super::instr::Instr::Hash { .. })) + .count() +} + +/// Commits a base-field codeword and opens one block. +fn base_commitment(shape: &Shape, seed: u64) -> (CodewordCommitment, Vec) { + let codeword: Vec = (0..1usize << shape.log_domain) + .map(|i| base(seed + i as u64)) + .collect(); + let commitment = CodewordCommitment::::from_codeword_on_host( + codeword.clone(), + shape.log_folding, + ) + .expect("the codeword commits"); + (commitment, codeword) +} + +/// The same over the cubic extension — every block but round 0's current one. +fn ext_commitment(shape: &Shape, seed: u64) -> CodewordCommitment { + let codeword: Vec = (0..1usize << shape.log_domain) + .map(|i| ext(seed + i as u64)) + .collect(); + CodewordCommitment::::from_codeword_on_host(codeword, shape.log_folding) + .expect("the codeword commits") +} + +fn siblings_of(opening: &CosetOpening) -> Vec { + opening + .proof + .merkle_path + .iter() + .map(commitment_to_digest) + .collect() +} + +/// ★ F1 for the opening: rows and PERMUTATIONS, both against their closed +/// forms, at every shape and in both fields. +/// +/// The permutation count is pinned separately and on purpose. It is the term +/// the chain's cost is dominated by, it is a function of the block's felts and +/// the tree's depth alone, and pinning it apart from the rows means a form that +/// moved work between hashing and arithmetic at constant total would fail one +/// of the two rather than neither. +#[test] +fn the_opening_emits_its_closed_form() { + for shape in SHAPES { + let block = 1usize << shape.log_folding; + let depth = shape.log_domain - shape.log_folding; + for is_ext in [false, true] { + let felts = if is_ext { 3 * block } else { block }; + let unpacks = if is_ext { block } else { 0 }; + let program = opening_program(block, is_ext, depth); + let constants = const_rows(&program); + let measured = program.instrs.len() - opening_plumbing(block, depth) - constants; + let predicted = verify_opening_rows(felts, unpacks, depth); + let perms = perm_rows(&program); + let predicted_perms = verify_opening_perms(felts, depth); + println!( + "{} {}: {measured:>3} rows emitted, {predicted:>3} predicted; \ + {perms:>2} permutations, {predicted_perms:>2} predicted \ + ({constants} interned constants)", + shape.name, + if is_ext { "ext " } else { "base" }, + ); + println!(" [{}]", histogram(&program)); + assert_eq!( + measured, + predicted, + "{} {}: the emitted row count must equal the closed form", + shape.name, + if is_ext { "ext" } else { "base" } + ); + assert_eq!( + perms, + predicted_perms, + "{} {}: the permutation count must equal the closed form", + shape.name, + if is_ext { "ext" } else { "base" } + ); + } + } +} + +/// ★ The leg accepts exactly what `whir_commit::verify_opening` accepts. +/// +/// The host runs first and must return `true`, so the arm is about an opening +/// that genuinely authenticates; then the machine must EXECUTE it. Nothing here +/// compares the emitter against the emitter: the leaf the machine recomputes +/// comes out of `prover::lfm::rpo`, and the leaf the tree hashed came out of +/// `crypto::hash::rpx` — two implementations that have to agree. +#[test] +fn the_opening_accepts_what_the_host_accepts() { + for shape in SHAPES { + let block = 1usize << shape.log_folding; + let depth = shape.log_domain - shape.log_folding; + let num_leaves = 1usize << depth; + + let (commitment, _) = base_commitment(shape, 0x31); + let program = opening_program(block, false, depth); + for index in [0usize, 1, num_leaves / 2, num_leaves - 1] { + let opening = commitment.open(index).expect("the block opens"); + assert!( + verify_opening::(&commitment.root(), depth, index, &opening), + "{}: the host must accept its own opening at {index}", + shape.name + ); + let values: Vec = opening.values.iter().copied().map(base_word).collect(); + let arena = opening_arena( + &values, + &siblings_of(&opening), + commitment_to_digest(&commitment.root()), + index, + ); + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!( + "{} base at {index}: the machine refused a valid opening: {e:?}", + shape.name + ) + }); + } + + let commitment = ext_commitment(shape, 0x77); + let program = opening_program(block, true, depth); + for index in [0usize, 1, num_leaves - 1] { + let opening = commitment.open(index).expect("the block opens"); + assert!( + verify_opening::(&commitment.root(), depth, index, &opening), + "{}: the host must accept its own opening at {index}", + shape.name + ); + let values: Vec = opening.values.iter().map(ext_word).collect(); + let arena = opening_arena( + &values, + &siblings_of(&opening), + commitment_to_digest(&commitment.root()), + index, + ); + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!( + "{} ext at {index}: the machine refused a valid opening: {e:?}", + shape.name + ) + }); + } + } +} + +/// ★ The tamper arm, in the three halves this lane's legs use: the untouched +/// opening EXECUTES (or the refusals below are refusals of nothing), the HOST +/// rejects each forgery, and the machine refuses it. +/// +/// Four sites, each a different part of the argument: a value (the leaf moves), +/// a sibling (the path moves), the root (what the walk is compared against), +/// and the INDEX (the same leaf against the same root at the wrong position — +/// the one a walk that ignored its bits would still accept). +#[test] +fn a_tampered_opening_cannot_execute() { + let shape = &SHAPES[2]; + let block = 1usize << shape.log_folding; + let depth = shape.log_domain - shape.log_folding; + let num_leaves = 1usize << depth; + let index = 3usize; + + let commitment = ext_commitment(shape, 0xB1); + let root = commitment.root(); + let opening = commitment.open(index).expect("the block opens"); + let program = opening_program(block, true, depth); + + let honest_values: Vec = opening.values.iter().map(ext_word).collect(); + let honest_siblings = siblings_of(&opening); + let honest_root = commitment_to_digest(&root); + + assert!( + verify_opening::(&root, depth, index, &opening), + "the control opening must authenticate" + ); + assert!( + execute( + &program, + &[opening_arena( + &honest_values, + &honest_siblings, + honest_root, + index + )], + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the untouched opening must execute, or the arm below proves nothing" + ); + + // A value: the leaf hashes to something else. + let mut forged = opening.clone(); + forged.values[block / 2] += FEE::one(); + assert!( + !verify_opening::(&root, depth, index, &forged), + "the host must reject a corrupted value" + ); + let values: Vec = forged.values.iter().map(ext_word).collect(); + assert!( + execute( + &program, + &[opening_arena(&values, &honest_siblings, honest_root, index)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "the machine must refuse a corrupted value" + ); + + // A sibling: the path walks somewhere else. + let mut forged = opening.clone(); + forged.proof.merkle_path[0][0] ^= 1; + assert!( + !verify_opening::(&root, depth, index, &forged), + "the host must reject a corrupted sibling" + ); + assert!( + execute( + &program, + &[opening_arena( + &honest_values, + &siblings_of(&forged), + honest_root, + index + )], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "the machine must refuse a corrupted sibling" + ); + + // The root: the walk arrives, at the wrong place. + let mut wrong_root = root; + wrong_root[0] ^= 1; + assert!( + !verify_opening::(&wrong_root, depth, index, &opening), + "the host must reject a wrong root" + ); + assert!( + execute( + &program, + &[opening_arena( + &honest_values, + &honest_siblings, + commitment_to_digest(&wrong_root), + index + )], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "the machine must refuse a wrong root" + ); + + // The INDEX: the right leaf and the right path, at the wrong position. A + // walk that took the same order at every level would pass everything above + // and fail only here. + let elsewhere = (index + 1) % num_leaves; + assert!( + !verify_opening::(&root, depth, elsewhere, &opening), + "the host must reject an opening claimed at the wrong index" + ); + assert!( + execute( + &program, + &[opening_arena( + &honest_values, + &honest_siblings, + honest_root, + elsewhere + )], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "the machine must refuse an opening claimed at the wrong index" + ); +} + +/// ★ The bus pins a hinted base value's upper lanes, which is what makes it +/// safe for the fold to lift one. +/// +/// A hint constrains nothing — `hint_felt` is `hint_word` retyped — so lanes +/// 1–3 of a round-0 opening value are free as far as the arena is concerned, +/// and `Felt::as_ext` would carry them into the fold as coordinates. The leaf's +/// `Pack` is what forbids it: `Pack` receives each lane as a `base_token` +/// `(addr, v, 0, 0, 0)`, and the memory bus is a multiset, so the arena's +/// word-token send can only balance when those lanes are zero. +/// +/// This is a property of the MACHINE, not of the emitter, so it is checked the +/// only way it can be: the same program, the same valid opening, one arena word +/// with a nonzero upper lane. The control is the same word with a zero one. +/// +/// ⚠ What this actually watches is the EXECUTOR's half — `Pack` reads its lanes +/// through `read_base` and refuses a word with a nonzero upper lane +/// (`executor.rs:871`), which is why the variant is named here rather than +/// settling for `is_err`: a refusal for some other reason would be a refusal +/// this test could not tell apart from the one it exists for. The half that +/// carries the SOUNDNESS is the AIR's, and no test reaches it — it is the +/// `base_token` on `Pack`'s receive, read at `chips.rs:1934-1937`. +#[test] +fn the_leaf_pins_a_hinted_base_value_to_its_low_lane() { + let shape = &SHAPES[1]; + let block = 1usize << shape.log_folding; + let depth = shape.log_domain - shape.log_folding; + let index = 2usize; + + let (commitment, _) = base_commitment(shape, 0xC3); + let opening = commitment.open(index).expect("the block opens"); + let program = opening_program(block, false, depth); + let siblings = siblings_of(&opening); + let root = commitment_to_digest(&commitment.root()); + + let honest: Vec = opening.values.iter().copied().map(base_word).collect(); + assert!( + execute( + &program, + &[opening_arena(&honest, &siblings, root, index)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the control must execute, or the refusal below is a refusal of nothing" + ); + + let mut smuggled = honest.clone(); + smuggled[0][1] = FE::one(); + let refusal = execute( + &program, + &[opening_arena(&smuggled, &siblings, root, index)], + &crate::hash_pin::BLOCK_HASHER, + ) + .expect_err( + "a base opening value with a nonzero lane 1 must have no execution: the leaf's \ + Pack is what pins it, and the fold lifts these values through Felt::as_ext", + ); + println!("smuggled lane 1 refused with: {refusal:?}"); + assert!( + matches!(refusal, super::executor::LfmExecError::NotBaseWord(_)), + "the refusal must be the base-word check at the leaf's Pack, not some other \ + failure that happens to also stop the program: got {refusal:?}" + ); +} + +// ============================================================================= +// W1: openings against a Merkle cap +// ============================================================================= + +/// A tree of 64 leaves (depth 6) over ext blocks of two, capped at `c`. +fn capped_commitment() -> CodewordCommitment { + ext_commitment( + &Shape { + log_domain: 7, + log_folding: 1, + name: "block 2, 6 levels", + }, + 0xCA9, + ) +} + +/// Arena: the cap (`2^c` words), the root, then per opening its block, its +/// `depth − c` siblings and its index. +fn capped_arena( + commitment: &CodewordCommitment, + c: usize, + openings: &[(usize, CosetOpening)], + cap: &[[u8; 32]], +) -> Vec { + let mut words: Vec = cap.iter().map(commitment_to_digest).collect(); + words.push(commitment_to_digest(&commitment.root())); + let depth = commitment.depth(); + for (index, opening) in openings { + words.extend(opening.values.iter().map(ext_word)); + words.extend( + opening.proof.merkle_path[..depth - c] + .iter() + .map(commitment_to_digest), + ); + words.push([FE::from(*index as u64), FE::zero(), FE::zero(), FE::zero()]); + } + words +} + +/// The program [`capped_arena`] feeds: one cap check, then `n` openings through +/// [`TreeAuth::verify_opening`]. Returns the program and the rows of its cap +/// check alone (measured by building the same prefix twice). +fn capped_program(depth: usize, c: usize, n: usize) -> LfmProgram { + let block = 2usize; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let per = block + (depth - c) + 1; + let arena = b.declare_arena(((1 << c) + 1 + n * per) as u32); + let cap: Vec = (0..1u32 << c) + .map(|i| WrapDigest::from_cell(b.hint_word(arena, i))) + .collect(); + let root = b.hint_word(arena, 1 << c); + let root_lanes = b.unpack(root); + let tree = TreeAuth::Cap(CapCells::authenticate( + &mut b, + &cap, + std::slice::from_ref(&root_lanes), + )); + for q in 0..n { + let at = ((1 << c) + 1 + q * per) as u32; + let values: Vec = (0..block) + .map(|i| b.hint_word(arena, at + i as u32).as_ext()) + .collect(); + let siblings: Vec = (0..depth - c) + .map(|i| WrapDigest::from_cell(b.hint_word(arena, at + (block + i) as u32))) + .collect(); + let index = b.hint_felt(arena, at + (block + depth - c) as u32); + let bits = b.bit_dec(index, depth); + tree.verify_opening( + &mut b, + BlockValues::Ext(&values), + &bits, + super::whir_open::Siblings::Child(&siblings), + ); + } + b.public(root); + let program = compile(b.finish()); + validate(&program).expect("the capped opening leg must be admissible"); + program +} + +/// The host's owner encoding: every path cut to `depth − c`, the first one +/// carrying the cap. Returns the openings and the cap. +#[allow(clippy::type_complexity)] +fn open_capped( + commitment: &CodewordCommitment, + c: usize, + indices: &[usize], +) -> (Vec<(usize, CosetOpening)>, Vec<[u8; 32]>) { + let depth = commitment.depth(); + let openings = commitment + .open_many_capped(indices, c, true) + .expect("the blocks open"); + let cap = openings[0].proof.merkle_path[depth - c..].to_vec(); + (indices.iter().copied().zip(openings).collect(), cap) +} + +/// ★ The cap mux selects the right node for EVERY index: all 64 leaves of a +/// depth-6 tree opened against a height-3 cap, so all eight top-bit patterns +/// are exercised. A mux level fed a constant (or the wrong bit) picks the +/// wrong node for half the indices and this refuses to execute. And an +/// opening claimed under another top-bit pattern — right leaf, right path, +/// wrong subtree — is refused, host and machine. +#[test] +fn the_cap_mux_selects_every_index() { + let commitment = capped_commitment(); + let depth = commitment.depth(); + assert_eq!(depth, 6); + for c in 1..=3usize { + let all: Vec = (0..64).collect(); + let (openings, cap) = open_capped(&commitment, c, &all); + let program = capped_program(depth, c, all.len()); + execute( + &program, + &[capped_arena(&commitment, c, &openings, &cap)], + &crate::hash_pin::BLOCK_HASHER, + ) + .unwrap_or_else(|e| panic!("c={c}: the machine refused honest capped openings: {e:?}")); + + // Right leaf and path, claimed in another subtree. + let (index, opening) = &openings[5]; + let elsewhere = index ^ (1 << (depth - 1)); + let root = commitment.root(); + let (check, _) = crypto::merkle_tree::cap::CappedRoot::from_owner::< + ::Backend, + >(&root, &openings[0].1.proof.merkle_path, depth, c) + .expect("the owner's cap authenticates"); + let siblings = &opening.proof.merkle_path[..depth - c]; + assert!( + multilinear::whir_commit::verify_opening_capped::( + &check, *index, opening, siblings + ) + ); + assert!( + !multilinear::whir_commit::verify_opening_capped::( + &check, elsewhere, opening, siblings + ), + "c={c}: the host must refuse the wrong subtree" + ); + let program = capped_program(depth, c, 1); + let forged = vec![(elsewhere, opening.clone())]; + assert!( + execute( + &program, + &[capped_arena(&commitment, c, &forged, &cap)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_err(), + "c={c}: the machine must refuse the wrong subtree" + ); + } +} + +/// ★ In-guest: a cap word NO opening reaches, tampered. The +/// walk and the mux of every opening are unaffected, so only the cap-to-root +/// check can refuse it — and it does. A cap word an opening does reach is +/// refused too. +#[test] +fn a_tampered_cap_word_cannot_execute() { + let commitment = capped_commitment(); + let depth = commitment.depth(); + let c = 3; + // Two openings, both under cap node 0 (indices < 8). + let (openings, cap) = open_capped(&commitment, c, &[1, 6]); + let program = capped_program(depth, c, 2); + let honest = capped_arena(&commitment, c, &openings, &cap); + assert!( + execute( + &program, + std::slice::from_ref(&honest), + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the untouched arena must execute, or the arm proves nothing" + ); + for node in [5usize, 0] { + let mut forged = honest.clone(); + forged[node][1] += FE::one(); + assert!( + execute(&program, &[forged], &crate::hash_pin::BLOCK_HASHER).is_err(), + "cap word {node} tampered: the machine must refuse" + ); + } +} + +/// ★ F1 for the capped opening and the cap check: the emitted rows and +/// permutations against [`verify_opening_rows_capped`], +/// [`verify_opening_perms_capped`], [`cap_check_rows`] and +/// [`cap_check_perms`], at every cap height of a depth-6 tree. +#[test] +fn the_capped_opening_emits_its_closed_form() { + let depth = 6; + let (block, felts, unpacks) = (2usize, 6usize, 2usize); + for c in 1..=depth { + let one = capped_program(depth, c, 1); + let two = capped_program(depth, c, 2); + // The second opening's own rows: the difference, less its plumbing + // (its hints and its `BitDec`). + let per_opening_plumbing = block + (depth - c) + 1 + 1; + let opening_rows = (two.instrs.len() - const_rows(&two)) + - (one.instrs.len() - const_rows(&one)) + - per_opening_plumbing; + assert_eq!( + opening_rows, + verify_opening_rows_capped(felts, unpacks, depth, c), + "c={c}: rows a capped opening" + ); + assert_eq!( + perm_rows(&two) - perm_rows(&one), + verify_opening_perms_capped(felts, depth, c), + "c={c}: permutations a capped opening" + ); + // The one-opening program: plumbing (cap hints, root hint, its + // `Unpack`, the public) + the cap check + one opening. + let fixed_plumbing = (1 << c) + 1 + 1 + 1; + let check_rows = one.instrs.len() + - const_rows(&one) + - fixed_plumbing + - per_opening_plumbing + - opening_rows; + assert_eq!(check_rows, cap_check_rows(c), "c={c}: rows the cap check"); + assert_eq!( + perm_rows(&one) - verify_opening_perms_capped(felts, depth, c), + cap_check_perms(c), + "c={c}: permutations the cap check" + ); + } + // At c = 0 the capped forms are the root forms. + assert_eq!( + verify_opening_rows_capped(felts, unpacks, depth, 0), + verify_opening_rows(felts, unpacks, depth) + ); + assert_eq!( + verify_opening_perms_capped(felts, depth, 0), + verify_opening_perms(felts, depth) + ); + assert_eq!((cap_check_rows(0), cap_check_perms(0)), (0, 0)); +} diff --git a/prover/src/lfm/whir_p1_census_tests.rs b/prover/src/lfm/whir_p1_census_tests.rs new file mode 100644 index 000000000..91fa37e3c --- /dev/null +++ b/prover/src/lfm/whir_p1_census_tests.rs @@ -0,0 +1,1629 @@ +//! D-WHIR-P1 S5's laptop census: the Poseidon1 leaf against RPX's at the +//! median block's shape, by emission alone (a WHIR block statement is shape +//! only, so no proof). The statement is I-PADLEAF §7's instrument's (its +//! measurement branch `padleaf/whir-census-pow`, 18d7d6b20). +//! +//! Run alone: `LAMBDA_VM_WHIR_HASH=rpx … --ignored --exact +//! lfm::whir_p1_census_tests::whir_p1_leaf_census --nocapture`. Prints +//! `CENSUS …` lines: each base's plan, and leaves of one to four full groups +//! and the plan's own heaviest leaf, rows by instruction and padded cells by +//! chip. + +use std::collections::BTreeMap; + +use super::instr::{BaseOp, Instr}; +use super::whir_block::{BlockPartition, WhirBlockPlan}; +use crate::block_whir::{BlockFormat, BlockStatement, OwnedBlockStatement}; +use stark::proof::options::BaseFormat; + +/// The median 25475471's tables (ULTRA mc12's BLOCK CENSUS, #1013), AIR order +/// after the fixed tables: `(kind, [(rows, count)])`, and its 198 pages. +const MEDIAN_MIX: &[(&str, &[(usize, usize)])] = &[ + // COMMIT's table then; COPY sits in its place now, and the old guest wrote its + // 160-byte output through it in one table of this height. + ("COPY", &[(256, 1)]), + ("KECCAK", &[(1 << 17, 1)]), + ("KECCAK_RND", &[(1 << 16, 35), (1 << 15, 1)]), + ("ECSM", &[(2048, 1)]), + ("ECDAS", &[(1 << 17, 4)]), + ("HINT", &[(1024, 1)]), + ("CPU", &[(1 << 21, 142), (1 << 20, 1)]), + ("LT", &[(1 << 21, 49), (1 << 20, 49)]), + ("SHIFT", &[(1 << 21, 10), (1 << 20, 1)]), + ("MEMW", &[(1 << 21, 5)]), + ("MEMW_A", &[(1 << 21, 50), (1 << 17, 1)]), + ("LOAD", &[(1 << 21, 31)]), + ("MUL", &[(1 << 20, 1), (1 << 17, 1)]), + ("DVRM", &[(1 << 14, 1)]), + ( + "BRANCH", + &[(1 << 19, 11), (1 << 18, 1), (1 << 17, 1), (1 << 14, 2)], + ), + ("PAGE", &[]), + ("MEMW_R", &[(1 << 21, 292), (1 << 19, 1)]), + ( + "EQ", + &[(1 << 18, 4), (1 << 17, 1), (1 << 15, 1), (1 << 14, 2)], + ), + ( + "BYTEWISE", + &[(1 << 20, 7), (1 << 19, 1), (1 << 18, 1), (1 << 17, 2)], + ), + ("STORE", &[(1 << 21, 23), (1 << 17, 1)]), + ("CPU32", &[(1 << 20, 1)]), +]; +const MEDIAN_PAGES: usize = 198; +const RUNTIME_PAGE_BASE: u64 = 0x4000_0000; + +/// The statement of [`MEDIAN_MIX`] over `elf_bytes`, its groups packed as the +/// prover packs them ([`stark::multilinear_block::block_groups`]). +pub(crate) fn median_statement( + elf_bytes: &[u8], + opts: &crate::ProofOptions, + format: &BlockFormat, +) -> OwnedBlockStatement { + let elf = executor::elf::Elf::load(elf_bytes).expect("load the ELF"); + let elf_pages = crate::tables::trace_builder::Traces::page_configs_from_elf(&elf).len(); + let mut counts = crate::TableCounts { + cpu: 0, + lt: 0, + memw: 0, + memw_aligned: 0, + load: 0, + mul: 0, + dvrm: 0, + shift: 0, + branch: 0, + memw_register: 0, + eq: 0, + bytewise: 0, + store: 0, + cpu32: 0, + keccak: 0, + keccak_rnd: 0, + ecsm: 0, + ecdas: 0, + hint: 0, + copy: 0, + blake3: 0, + is_wh: 0, + ecrec: 0, + }; + let mut rows: Vec = vec![32; crate::FIXED_TABLE_COUNT]; + for &(kind, runs) in MEDIAN_MIX { + if kind == "PAGE" { + // Every page table is one row a byte of its page (`page::DEFAULT_PAGE_SIZE`). + rows.extend(std::iter::repeat_n( + crate::tables::page::DEFAULT_PAGE_SIZE, + elf_pages, + )); + rows.extend(std::iter::repeat_n(1 << 18, MEDIAN_PAGES - elf_pages)); + continue; + } + let n: usize = runs.iter().map(|&(_, c)| c).sum(); + rows.extend(runs.iter().flat_map(|&(r, c)| std::iter::repeat_n(r, c))); + let slot = match kind { + "COPY" => &mut counts.copy, + "KECCAK" => &mut counts.keccak, + "KECCAK_RND" => &mut counts.keccak_rnd, + "ECSM" => &mut counts.ecsm, + "ECDAS" => &mut counts.ecdas, + "HINT" => &mut counts.hint, + "CPU" => &mut counts.cpu, + "LT" => &mut counts.lt, + "SHIFT" => &mut counts.shift, + "MEMW" => &mut counts.memw, + "MEMW_A" => &mut counts.memw_aligned, + "LOAD" => &mut counts.load, + "MUL" => &mut counts.mul, + "DVRM" => &mut counts.dvrm, + "BRANCH" => &mut counts.branch, + "MEMW_R" => &mut counts.memw_register, + "EQ" => &mut counts.eq, + "BYTEWISE" => &mut counts.bytewise, + "STORE" => &mut counts.store, + "CPU32" => &mut counts.cpu32, + other => panic!("{other}"), + }; + *slot = n; + } + let ranges = vec![crate::RuntimePageRange { + base: RUNTIME_PAGE_BASE, + count: (MEDIAN_PAGES - elf_pages) as u64, + }]; + let page_configs = crate::tables::trace_builder::Traces::page_configs_from_elf_and_runtime( + &elf, + &ranges, + 0, + rows.len(), + ) + .expect("page configs"); + let airs = crate::VmAirs::new( + &elf, + opts, + false, + &page_configs, + &counts, + &vec![4; counts.cpu], + None, + true, + None, + crate::test_utils::halted_register_commitment(opts, &elf, &[]).preprocessed(), + ); + // The fixed tables at their preprocessed columns' heights, so group 0 + // emits too (HALT, which has none, stays at the placeholder). + for (i, air) in airs + .air_refs() + .iter() + .enumerate() + .take(crate::FIXED_TABLE_COUNT) + { + if let Some(column) = air.precomputed_columns().first() { + rows[i] = column.len(); + } + } + println!("CENSUS FIXED ROWS: {:?}", &rows[..crate::FIXED_TABLE_COUNT]); + let shapes: Vec<(usize, usize)> = airs + .air_refs() + .iter() + .zip(&rows) + .map(|(air, &r)| (air.trace_layout().0, r.trailing_zeros() as usize)) + .collect(); + let config = format.chain_config(&shapes); + let sizes = + stark::multilinear_block::block_groups(&shapes, config.format.stack, format.group_polys) + .expect("groups"); + let mut at = 0u32; + let groups: Vec> = sizes + .iter() + .map(|&size| { + let g = (at..at + size as u32).collect(); + at += size as u32; + g + }) + .collect(); + OwnedBlockStatement { + table_num_vars: rows.iter().map(|r| r.trailing_zeros() as u8).collect(), + runtime_page_ranges: ranges, + table_counts: counts, + public_output: vec![0; 4], + num_private_input_pages: 0, + groups, + } +} + +fn kind(instr: &Instr) -> &'static str { + match instr { + Instr::Const { .. } => "CONST", + Instr::BaseAlu { op, .. } => match op { + BaseOp::Add => "BALU.add", + BaseOp::Sub => "BALU.sub", + BaseOp::Mul => "BALU.mul", + BaseOp::Div => "BALU.div", + BaseOp::MulAdd => "BALU.muladd", + }, + Instr::ExtAlu { .. } => "XALU", + Instr::Select { .. } => "SELECT", + Instr::BitDec { .. } => "BITDEC", + Instr::Hash { .. } => "HASH", + Instr::Hash16(_) => "HASH16", + Instr::KeccakF(_) => "KECCAK", + Instr::Blake3(_) => "BLAKE3", + Instr::Hint { .. } | Instr::HintOn { .. } => "HINT", + Instr::Pack { .. } => "LANES.pack", + Instr::Unpack { .. } => "LANES.unpack", + Instr::Public { .. } => "PUBLIC", + } +} + +/// One leaf's census line: instruction rows by kind, then padded cells by chip +/// under the program's own hasher. +fn census_leaf(label: &str, program: &super::compiler::LfmProgram) -> u64 { + let mut rows: BTreeMap<&'static str, usize> = BTreeMap::new(); + for instr in &program.instrs { + *rows.entry(kind(instr)).or_default() += 1; + } + let balu: usize = rows + .iter() + .filter(|(k, _)| k.starts_with("BALU")) + .map(|(_, r)| *r) + .sum(); + let chips = super::airs::lfm_chip_census_with_hasher( + program, + program.hasher(crate::hash_pin::BLOCK_HASHER), + ); + let cells: u64 = chips.iter().map(|c| c.main_cells() + c.aux_cells()).sum(); + println!( + "CENSUS {label} ROWS: BALU={balu} {}", + rows.iter() + .map(|(k, r)| format!("{k}={r}")) + .collect::>() + .join(" ") + ); + println!( + "CENSUS {label} CHIPS: cells {cells} · {}", + chips + .iter() + .filter(|c| c.real_rows > 0) + .map(|c| format!( + "{}={}/{}x{}", + c.name, + c.real_rows, + c.rows, + c.main_cols + c.aux_cols + )) + .collect::>() + .join(" ") + ); + cells +} + +/// ★ The S5 census (laptop instrument, `--exact`): per base, the median mix's +/// plan, leaves of 1–4 full groups and the plan's heaviest leaf. +#[test] +#[ignore = "laptop instrument: run alone with --exact"] +fn whir_p1_leaf_census() { + let elf_bytes = crate::test_utils::asm_elf_bytes("poc_rodata_commit"); + let opts = super::proof::block_base_options(); + let mut four: Vec<(String, u64)> = Vec::new(); + // The block admits its proof family's base only (D-P3B P3): the RPX arm is gone. + for (name, base) in [("P1", BaseFormat::P1_WHIR)] { + let production = BlockFormat::production(); + let format = BlockFormat { + zf: production.zf.with_base(base), + ..production + }; + let owned = median_statement(&elf_bytes, &opts, &format); + let statement: BlockStatement<'_> = owned.view(); + let plan = WhirBlockPlan::derive_with( + &elf_bytes, + &opts, + &format, + statement, + None, + super::whir_block::BLOCK_FAN_IN, + None, + ) + .expect("the plan derives"); + let costs = plan.costs().to_vec(); + println!( + "CENSUS {name} PLAN: {} tables · {} groups · {} leaves · cap {} · group costs {:?}", + owned.table_num_vars.len(), + plan.num_groups(), + plan.partition().num_leaves(), + super::whir_block::LEAF_P1_CAP, + costs + ); + let heaviest = (0..plan.partition().num_leaves()) + .max_by_key(|&k| { + plan.partition() + .leaf(k) + .iter() + .map(|&g| costs[g]) + .sum::() + }) + .expect("a leaf"); + let program = plan.leaf_program(heaviest).expect("the leaf emits"); + census_leaf( + &format!( + "{name} PLAN LEAF {heaviest} (groups {:?})", + plan.partition().leaf(heaviest) + ), + &program, + ); + let n = plan.num_groups(); + let mut lists: Vec> = + vec![vec![1], vec![2, 3], vec![4, 5, 6], vec![7, 8, 9, 10]]; + let used: Vec = lists.iter().flatten().copied().collect(); + lists.push((0..n).filter(|g| !used.contains(g)).collect()); + let plan = plan.with_partition(BlockPartition::new(lists.clone(), n).expect("covers once")); + for (k, list) in lists.iter().enumerate().take(4) { + let program = plan.leaf_program(k).expect("the leaf emits"); + let cells = census_leaf(&format!("{name} LEAF {} GROUPS", list.len()), &program); + if list.len() == 4 { + four.push((name.to_string(), cells)); + } + } + } + // The Poseidon1 plan at a 2^17 socket-row cap (three full groups a leaf): + // the hash table one doubling down from D8a's. + { + let production = BlockFormat::production(); + let format = BlockFormat { + zf: production.zf.with_base(BaseFormat::P1_WHIR), + ..production + }; + let owned = median_statement(&elf_bytes, &opts, &format); + let plan = WhirBlockPlan::derive_capped( + &elf_bytes, + &opts, + &format, + owned.view(), + None, + super::whir_block::BLOCK_FAN_IN, + None, + 1 << 17, + ) + .expect("the plan derives at 2^17"); + let costs = plan.costs().to_vec(); + let heaviest = (0..plan.partition().num_leaves()) + .max_by_key(|&k| { + plan.partition() + .leaf(k) + .iter() + .map(|&g| costs[g]) + .sum::() + }) + .expect("a leaf"); + println!( + "CENSUS P1@2^17 PLAN: {} leaves · heaviest leaf {heaviest} groups {:?}", + plan.partition().num_leaves(), + plan.partition().leaf(heaviest) + ); + let program = plan.leaf_program(heaviest).expect("the leaf emits"); + census_leaf(&format!("P1@2^17 PLAN LEAF {heaviest}"), &program); + } + if let [(_, rpx), (_, p1)] = four.as_slice() { + println!( + "CENSUS 4-GROUP LEAF CELLS: RPX {rpx} · P1 {p1} · P1/RPX {:.4}", + *p1 as f64 / *rpx as f64 + ); + } +} + +/// A leaf's emitted rows against the plan's: every chip but the constants +/// exactly on the carrier, and on any other leaf less one vector — the COMMIT +/// bus target's rows, the same for every such leaf (`target` keeps the first); +/// the constants (interned) at most the plan's. +fn assert_planned( + label: &str, + carrier: bool, + emitted: ChipRows, + planned: ChipRows, + target: &mut Option, +) { + assert!( + emitted.consts <= planned.consts, + "{label}: {} constants against the plan's {}", + emitted.consts, + planned.consts + ); + let (e, p) = ( + ChipRows { + consts: 0, + ..emitted + }, + ChipRows { + consts: 0, + ..planned + }, + ); + if carrier { + assert_eq!(e, p, "{label}: the carrier's rows are the plan's"); + return; + } + assert!(e.under(p), "{label}: {e:?} past the plan's {p:?}"); + let (ea, pa) = (e.to_array(), p.to_array()); + let gap = ChipRows::from_array(core::array::from_fn(|c| pa[c] - ea[c])); + match target { + None => *target = Some(gap), + Some(t) => assert_eq!(*t, gap, "{label}: a leaf less another target"), + } +} + +/// ★ Every group of the median mix as a one-group Poseidon1 leaf, against the +/// plan's rows for it — the lead's acceptance criterion (10-06; every chip +/// since S6b): the rows the partition admits leaves by ARE the emitted ones, +/// for every group shape the mix has, both prepared kinds included (DECODE's +/// stack in group 0, the ELF pages' genesis), with no fallback +/// ([`assert_planned`]). Then every leaf the plan builds, emitted, stays under +/// every padded height. +/// +/// Both prepared kinds need a real guest (a small one's pages are not dense +/// enough for a genesis stack): `CENSUS_ELF=` or `BLOCK_WHIR_ELF` (the +/// box's), which then also asserts both kinds are present; else the small one. +/// Prints `CENSUS GROUP …` and `CENSUS PLAN LEAF …` lines. +#[test] +#[ignore = "instrument (laptop with CENSUS_ELF, box with BLOCK_WHIR_ELF): run with --exact"] +fn whir_p1_group_census() { + let real = std::env::var("CENSUS_ELF") + .or_else(|_| std::env::var("BLOCK_WHIR_ELF")) + .ok(); + let elf_bytes = match &real { + Some(path) => std::fs::read(path).expect("read the guest ELF"), + None => crate::test_utils::asm_elf_bytes("poc_rodata_commit"), + }; + let opts = super::proof::block_base_options(); + let format = p1_format(); + let owned = median_statement(&elf_bytes, &opts, &format); + let derive = || { + WhirBlockPlan::derive_with( + &elf_bytes, + &opts, + &format, + owned.view(), + None, + super::whir_block::BLOCK_FAN_IN, + None, + ) + .expect("the plan derives") + }; + let started = std::time::Instant::now(); + let plan = derive(); + println!( + "CENSUS PLAN DERIVED in {:.2} s (every group emitted once)", + started.elapsed().as_secs_f64() + ); + let started = std::time::Instant::now(); + let again = super::whir_block::chip_loads_by_emission(&plan).expect("the probe"); + println!( + "CENSUS PROBE: {} groups measured in {:.2} s", + plan.num_groups(), + started.elapsed().as_secs_f64() + ); + let n = plan.num_groups(); + let prepared: Vec = plan.prepared().iter().map(|p| p.group).collect(); + let (loads, front) = plan.chip_loads(); + assert_eq!((loads.to_vec(), front), again, "the probe is deterministic"); + let (loads, front) = (loads.to_vec(), front); + if real.is_some() { + assert!( + prepared.contains(&0) && prepared.iter().any(|&g| g != 0), + "a real guest's mix has both prepared kinds, got groups {prepared:?}" + ); + } + let singles = plan.with_partition( + BlockPartition::new((0..n).map(|g| vec![g]).collect(), n).expect("one group a leaf"), + ); + let mut target = None; + let mut max = ChipRows::default(); + for (g, &load) in loads.iter().enumerate() { + let emitted = ChipRows::of(&singles.leaf_program(g).expect("the leaf emits")); + assert_planned( + &format!("group {g} (prepared {})", prepared.contains(&g)), + g == CARRIER, + emitted, + front.plus(load), + &mut target, + ); + println!( + "CENSUS GROUP {g}: tables {} · prepared {} · cost {} · {load:?}", + owned.groups[g].len(), + prepared.contains(&g), + singles.costs()[g], + ); + let (m, l) = (max.to_array(), load.to_array()); + max = ChipRows::from_array(core::array::from_fn(|c| m[c].max(l[c]))); + } + println!("CENSUS GROUP MAX: {max:?}"); + println!("CENSUS FRONT: {front:?} · a non-carrier leaf less {target:?}"); + + // The plan at the production cap: every leaf, against its chips' heights — + // the partition's own (doubled when the production leaf scale doubles the + // median's leaves, as it does at blowup 2). + let plan = derive(); + let heights = plan.leaf_bound().expect("the rule's partition").heights; + let mut target = None; + for k in 0..plan.partition().num_leaves() { + let leaf = plan.partition().leaf(k); + let emitted = ChipRows::of(&plan.leaf_program(k).expect("the leaf emits")); + let planned = leaf.iter().fold(front, |acc, &g| acc.plus(loads[g])); + assert_planned( + &format!("leaf {k}"), + k == CARRIER, + emitted, + planned, + &mut target, + ); + println!("CENSUS PLAN LEAF {k}: groups {leaf:?} · {emitted:?}"); + assert!( + emitted.under(heights), + "leaf {k} is past a chip's height: {emitted:?}" + ); + } +} + +/// ★ D-ONECHAIN B2-7 (REV-B2-JUDGE §4): the median block's census by +/// emission under `GroupFold::Indexed`, beside `PerPoly`, on the Poseidon1 +/// base at the production format: one plan each over the same statement (the +/// groups do not depend on the opening's mode), every group emitted +/// ([`super::whir_block::chip_loads_by_emission`]), the partition at +/// [`P1_LEAF_HEIGHTS`], and every leaf's emitted rows under every chip's +/// height — the socket and bit-dec within the LFM bits guard's 2^19 by the +/// heights' own const-assert. +/// +/// Prints `ONECHAIN CENSUS …` lines: per mode the leaves, the L0 socket rows +/// (Σ group costs), the group costs' spread, the heaviest group's chips, and +/// each leaf's emitted chips. Pre-registered (D-ONECHAIN §5, B2): Indexed +/// 6–7 leaves (RED at 8, STOP at ≥ 10), every chip under its height. +#[test] +#[ignore = "instrument (box with BLOCK_WHIR_ELF, or CENSUS_ELF): run with --exact"] +fn whir_onechain_census() { + let real = std::env::var("CENSUS_ELF") + .or_else(|_| std::env::var("BLOCK_WHIR_ELF")) + .ok(); + let elf_bytes = match &real { + Some(path) => std::fs::read(path).expect("read the guest ELF"), + None => crate::test_utils::asm_elf_bytes("poc_rodata_commit"), + }; + let opts = super::proof::block_base_options(); + let per_poly = p1_format(); + let indexed = BlockFormat { + group: multilinear::whir_chain::GroupFold::Indexed, + ..p1_format() + }; + let owned = median_statement(&elf_bytes, &opts, &per_poly); + let mut leaves_of = Vec::new(); + for (label, format) in [("PerPoly", &per_poly), ("Indexed", &indexed)] { + let started = std::time::Instant::now(); + let plan = WhirBlockPlan::derive_with( + &elf_bytes, + &opts, + format, + owned.view(), + None, + super::whir_block::BLOCK_FAN_IN, + None, + ) + .expect("the plan derives"); + let (loads, front) = plan.chip_loads(); + let costs = plan.costs(); + let mut sorted = costs.to_vec(); + sorted.sort_unstable(); + let heaviest = loads.iter().fold(ChipRows::default(), |m, l| { + let (m, l) = (m.to_array(), l.to_array()); + ChipRows::from_array(core::array::from_fn(|c| m[c].max(l[c]))) + }); + let leaves = plan.partition().num_leaves(); + // The partition's own heights: doubled when the production leaf scale + // doubles the median's leaves (at blowup 2). + let heights = plan.leaf_bound().expect("the rule's partition").heights; + println!( + "ONECHAIN CENSUS {label}: {} groups, {leaves} leaves, L0 socket {} (derived in {:.1} \ + s); group costs min {} median {} max {}; heaviest group {heaviest:?}; front {front:?}", + plan.num_groups(), + costs.iter().sum::(), + started.elapsed().as_secs_f64(), + sorted[0], + sorted[sorted.len() / 2], + sorted[sorted.len() - 1], + ); + let mut target = None; + for k in 0..leaves { + let leaf = plan.partition().leaf(k); + let emitted = ChipRows::of(&plan.leaf_program(k).expect("the leaf emits")); + let planned = leaf.iter().fold(front, |acc, &g| acc.plus(loads[g])); + assert_planned( + &format!("{label} leaf {k}"), + k == CARRIER, + emitted, + planned, + &mut target, + ); + println!( + "ONECHAIN CENSUS {label} LEAF {k}: {} groups · {emitted:?}", + leaf.len() + ); + assert!( + emitted.under(heights), + "{label} leaf {k} is past a chip's height: {emitted:?}" + ); + } + leaves_of.push(leaves); + } + println!( + "ONECHAIN CENSUS LEAVES: PerPoly {} Indexed {}", + leaves_of[0], leaves_of[1] + ); +} + +// ======================== the leaves' chip heights ======================== + +use super::whir_block::{ + CARRIER, LeafScale, P1_LEAF_HEIGHTS, P1_LEAF_HEIGHTS_2X, leaf_partition_rows, + scaled_leaf_partition, +}; +use super::whir_chain::ChipRows; + +fn p1_format() -> BlockFormat { + let production = BlockFormat::production(); + BlockFormat { + zf: production.zf.with_base(BaseFormat::P1_WHIR), + ..production + } +} + +/// [`p1_format`] at blowup 2 (the production rate), every other field its own. +fn p1_format_at_blowup_2() -> BlockFormat { + let mut format = p1_format(); + format.zf.whir_blowup = 2; + format +} + +fn rows(hash: usize, select: usize, balu: usize, hint: usize) -> ChipRows { + ChipRows { + hash, + select, + balu, + hint, + ..ChipRows::default() + } +} + +/// The census's shapes (median mix, an ethrex ELF; `whir_p1_group_census`): a +/// full group, a full one heavy in the extension ALU (KECCAK_RND's tables), a +/// prepared one (the ELF pages' genesis), and the front. +const FULL: ChipRows = ChipRows { + hash: 35_744, + select: 139_878, + balu: 142_236, + xalu: 193_562, + lanes: 129_555, + hint: 135_333, + bitdec: 2_415, + consts: 527, + public: 0, + accel: 0, +}; +const HEAVY: ChipRows = ChipRows { + hash: 38_358, + xalu: 321_505, + lanes: 147_538, + hint: 140_290, + consts: 519, + ..FULL +}; +const PREPARED: ChipRows = ChipRows { + hash: 48_367, + select: 172_938, + balu: 180_090, + xalu: 350_252, + lanes: 190_294, + hint: 185_114, + bitdec: 3_105, + consts: 706, + public: 0, + accel: 0, +}; +const FRONT: ChipRows = ChipRows { + hash: 287, + select: 0, + balu: 0, + xalu: 17, + lanes: 544, + hint: 237, + bitdec: 0, + consts: 554, + public: 5, + accel: 0, +}; + +fn sums(loads: &[ChipRows], leaf: &[usize], front: ChipRows) -> ChipRows { + leaf.iter().fold(front, |acc, &g| acc.plus(loads[g])) +} + +/// ★ The partition admits no leaf past any chip's height, whichever chip +/// binds: groups light in the socket but heavy in `Select`, base ALU, hints, +/// lanes, bit decompositions or extension ALU are split where a partition +/// blind to that chip would double it silently. Seven full groups share a +/// leaf, an eighth does not (one 2^18 socket table); six heavy ones do, a +/// seventh not. Every group is placed once, over no more leaves than the rows +/// need, and a group no leaf holds is refused. +#[test] +fn the_p1_partition_keeps_every_chip_under_its_height() { + let front = FRONT; + let small = ChipRows { + xalu: 90_000, + lanes: 50_000, + ..rows(9_000, 60_000, 40_000, 50_000) + }; + for loads in [ + vec![PREPARED, FULL, FULL], + vec![PREPARED, FULL, HEAVY, FULL, HEAVY, FULL, PREPARED, small], + [ + vec![PREPARED; 2], + vec![HEAVY; 8], + vec![FULL; 68], + vec![small], + ] + .concat(), + ] { + let partition = leaf_partition_rows(&loads, front, P1_LEAF_HEIGHTS).expect("partitions"); + for leaf in partition.leaves() { + assert!( + sums(&loads, leaf, front).under(P1_LEAF_HEIGHTS), + "leaf {leaf:?} of {} groups is past a height", + loads.len() + ); + } + // No fewer than the socket's sum allows. + let hashes: usize = loads.iter().map(|l| l.hash).sum(); + assert!(partition.num_leaves() >= hashes.div_ceil(P1_LEAF_HEIGHTS.hash - front.hash)); + // Deterministic. + assert_eq!( + partition, + leaf_partition_rows(&loads, front, P1_LEAF_HEIGHTS).expect("again") + ); + } + let leaves = |loads: &[ChipRows]| { + leaf_partition_rows(loads, front, P1_LEAF_HEIGHTS) + .expect("fits") + .num_leaves() + }; + assert_eq!(leaves(&[FULL; 7]), 1, "seven full groups are one leaf"); + assert_eq!(leaves(&[FULL; 8]), 2, "an eighth is past the socket"); + assert_eq!(leaves(&[HEAVY; 6]), 1, "six heavy groups are one leaf"); + assert_eq!(leaves(&[HEAVY; 7]), 2, "a seventh is past the socket"); + // The 1× block's nine groups close in two leaves; the median's ninety in + // thirteen, the socket's lower bound — where balancing alone puts the two + // prepared groups on two leaves of five full ones and takes fourteen, + // first fit packs them on one with four. + let nine = [vec![FULL; 7], vec![PREPARED; 2]].concat(); + assert_eq!(leaves(&nine), 2, "the 1× block's nine groups"); + let ninety = [vec![PREPARED; 2], vec![FULL; 88]].concat(); + assert_eq!(leaves(&ninety), 13, "the median's ninety groups"); + // Any one chip binds: groups light in the socket are split. + for heavy in [ + rows(10_000, 600_000, 10_000, 10_000), + rows(10_000, 10_000, 600_000, 10_000), + rows(10_000, 10_000, 10_000, 600_000), + ChipRows { + lanes: 600_000, + ..rows(10_000, 10_000, 10_000, 10_000) + }, + ChipRows { + xalu: 1_100_000, + ..rows(10_000, 10_000, 10_000, 10_000) + }, + ChipRows { + bitdec: 17_000, + ..rows(10_000, 10_000, 10_000, 10_000) + }, + ChipRows { + consts: 4_200, + ..rows(10_000, 10_000, 10_000, 10_000) + }, + ] { + let p = leaf_partition_rows(&[heavy, heavy], front, P1_LEAF_HEIGHTS).expect("fits"); + assert_eq!( + p.num_leaves(), + 2, + "{heavy:?} twice is past a height: split, not doubled" + ); + } + // ★ Four groups that fit every chip S6 bounded (socket, Select, base ALU, + // words) but not the extension ALU (2.8 M): the partition splits them; with + // the XALU bound off — S6's partition — they share a leaf whose XALU + // crosses 2^21 (arm C on RYZEN 062 padded seven groups' to 2^21 unbounded). + let xaluy = ChipRows { + xalu: 700_000, + ..rows(40_000, 150_000, 150_000, 150_000) + }; + let set = [xaluy; 4]; + let p = leaf_partition_rows(&set, front, P1_LEAF_HEIGHTS).expect("fits"); + assert_eq!(p.num_leaves(), 2, "split by the extension ALU"); + for leaf in p.leaves() { + assert!(sums(&set, leaf, front).under(P1_LEAF_HEIGHTS)); + } + let unbounded = ChipRows { + xalu: usize::MAX / 2, + ..P1_LEAF_HEIGHTS + }; + let blind = leaf_partition_rows(&set, front, unbounded).expect("the XALU-blind partition"); + assert!( + blind + .leaves() + .iter() + .any(|leaf| sums(&set, leaf, front).xalu > P1_LEAF_HEIGHTS.xalu), + "the XALU bound is what keeps it under 2^21: {:?}", + blind.leaves() + ); + // Refusals: a group past a height, an accelerator row (a WHIR leaf has + // none), a front past a height. + let huge = rows(1_000, 1_200_000, 0, 0); + assert!(leaf_partition_rows(&[FULL, huge], front, P1_LEAF_HEIGHTS).is_err()); + let accel = ChipRows { accel: 1, ..FULL }; + assert!(leaf_partition_rows(&[FULL, accel], front, P1_LEAF_HEIGHTS).is_err()); + assert!(leaf_partition_rows(&[FULL], rows(0, 1 << 20, 0, 0), P1_LEAF_HEIGHTS).is_err()); + assert!( + leaf_partition_rows( + &[FULL], + ChipRows { + xalu: (1 << 21) + 1, + ..front + }, + P1_LEAF_HEIGHTS + ) + .is_err() + ); +} + +/// The doubling rule at its threshold, on the census's group shapes: `loads` +/// built of two prepared groups, four heavy ones and as many full ones as +/// take the single heights to `leaves` leaves. +fn loads_at(leaves: usize) -> Vec { + (1..1024) + .map(|full| [vec![PREPARED; 2], vec![HEAVY; 4], vec![FULL; full]].concat()) + .find(|loads| { + leaf_partition_rows(loads, FRONT, P1_LEAF_HEIGHTS) + .expect("fits") + .num_leaves() + == leaves + }) + .expect("some count of full groups takes that many leaves") +} + +/// ★ [`LeafScale::DoubleFrom`]`(T)` keeps the single partition while the +/// single heights take `T - 1` leaves, and from `T` partitions again under the +/// doubled heights, to fewer leaves (about half). [`LeafScale::Fixed`] never +/// doubles, and a threshold one above the count does not either. +#[test] +fn the_leaves_double_from_their_threshold() { + let scaled = |loads: &[ChipRows], scale: LeafScale| { + scaled_leaf_partition(loads, FRONT, scale, P1_LEAF_HEIGHTS, P1_LEAF_HEIGHTS_2X) + .expect("partitions") + }; + for t in [10usize, 12, 18] { + let under = loads_at(t - 1); + let single = leaf_partition_rows(&under, FRONT, P1_LEAF_HEIGHTS).expect("fits"); + let (partition, bound) = scaled(&under, LeafScale::DoubleFrom(t)); + assert_eq!( + partition, + single, + "{} leaves keep the single partition", + t - 1 + ); + assert_eq!( + (bound.single_leaves, bound.heights, bound.doubled), + (t - 1, P1_LEAF_HEIGHTS, false) + ); + + let at = loads_at(t); + let single = leaf_partition_rows(&at, FRONT, P1_LEAF_HEIGHTS).expect("fits"); + assert_eq!(single.num_leaves(), t); + let (partition, bound) = scaled(&at, LeafScale::DoubleFrom(t)); + assert_eq!( + partition, + leaf_partition_rows(&at, FRONT, P1_LEAF_HEIGHTS_2X).expect("fits doubled"), + "{t} leaves partition again under the doubled heights" + ); + assert_eq!( + (bound.single_leaves, bound.heights, bound.doubled), + (t, P1_LEAF_HEIGHTS_2X, true) + ); + assert!( + partition.num_leaves() <= t.div_ceil(2) + 1, + "{t} single leaves double to {}", + partition.num_leaves() + ); + assert_eq!(scaled(&at, LeafScale::Fixed).0, single, "fixed at {t}"); + assert_eq!( + scaled(&at, LeafScale::DoubleFrom(t + 1)).0, + single, + "a threshold of {} at {t} leaves", + t + 1 + ); + } +} + +/// ★ Every doubled leaf stays under every doubled height, whichever chip +/// binds, and some are past a single height (the doubling is real). Groups a +/// single leaf holds are refused under either scale when one is past a single +/// height, even if a doubled leaf would hold it; and doubled heights below +/// the single ones are refused. +#[test] +fn every_doubled_leaf_stays_under_the_doubled_heights() { + let doubled = |loads: &[ChipRows]| { + let (partition, bound) = scaled_leaf_partition( + loads, + FRONT, + LeafScale::DoubleFrom(1), + P1_LEAF_HEIGHTS, + P1_LEAF_HEIGHTS_2X, + ) + .expect("partitions"); + assert!(bound.doubled); + partition + }; + for loads in [ + loads_at(12), + loads_at(18), + [vec![PREPARED; 2], vec![HEAVY; 8], vec![FULL; 68]].concat(), + ] { + let partition = doubled(&loads); + for leaf in partition.leaves() { + assert!( + sums(&loads, leaf, FRONT).under(P1_LEAF_HEIGHTS_2X), + "leaf {leaf:?} past a doubled height" + ); + } + assert!( + partition + .leaves() + .iter() + .any(|leaf| !sums(&loads, leaf, FRONT).under(P1_LEAF_HEIGHTS)), + "some doubled leaf is past a single height" + ); + let hashes: usize = loads.iter().map(|l| l.hash).sum(); + assert!(partition.num_leaves() >= hashes.div_ceil(P1_LEAF_HEIGHTS_2X.hash - FRONT.hash)); + } + // Each chip binds a doubled leaf: four groups that fit one doubled leaf + // in every other chip are split by the one each is heavy in. + let light = ChipRows { + xalu: 50_000, + lanes: 50_000, + bitdec: 1_000, + consts: 100, + ..rows(50_000, 50_000, 50_000, 50_000) + }; + for heavy in [ + ChipRows { + hash: 200_000, + ..light + }, + ChipRows { + select: 900_000, + ..light + }, + ChipRows { + balu: 900_000, + ..light + }, + ChipRows { + xalu: 1_900_000, + ..light + }, + ChipRows { + lanes: 900_000, + ..light + }, + ChipRows { + hint: 900_000, + ..light + }, + ChipRows { + bitdec: 30_000, + ..light + }, + ChipRows { + consts: 7_000, + ..light + }, + ] { + let set = [heavy; 4]; + let partition = doubled(&set); + assert_eq!(partition.num_leaves(), 2, "{heavy:?} four times"); + for leaf in partition.leaves() { + assert!( + sums(&set, leaf, FRONT).under(P1_LEAF_HEIGHTS_2X), + "{heavy:?}" + ); + } + } + // Refusals: a group past a single height, under either scale; doubled + // heights below the single ones. + let past = rows(1_000, 1_200_000, 0, 0); + assert!(past.plus(FRONT).under(P1_LEAF_HEIGHTS_2X)); + for scale in [LeafScale::Fixed, LeafScale::DoubleFrom(1)] { + assert!( + scaled_leaf_partition( + &[FULL, past], + FRONT, + scale, + P1_LEAF_HEIGHTS, + P1_LEAF_HEIGHTS_2X + ) + .is_err(), + "{scale}" + ); + } + let lower = ChipRows { + xalu: P1_LEAF_HEIGHTS.xalu - 1, + ..P1_LEAF_HEIGHTS_2X + }; + assert!( + scaled_leaf_partition( + &[FULL], + FRONT, + LeafScale::DoubleFrom(1), + P1_LEAF_HEIGHTS, + lower + ) + .is_err() + ); +} + +/// ★ The plan doubles the median mix's leaves from its threshold and not +/// below it, through [`WhirBlockPlan::derive`]: at [`LeafScale::Fixed`] the +/// single heights take `n` leaves; at `DoubleFrom(n + 1)` the partition is the +/// same one, so every leaf program, every node and the top are too (a leaf's +/// program is a function of the plan's other fields, which the scale does not +/// move, and its groups); at `DoubleFrom(n)` fewer leaves, each under the +/// doubled heights by its planned rows, so another tree — the one a prover +/// cannot make the verifier derive below the threshold. +#[test] +fn the_plan_doubles_the_median_mix_only_from_its_threshold() { + let elf_bytes = crate::test_utils::asm_elf_bytes("poc_rodata_commit"); + let opts = super::proof::block_base_options(); + let at = |leaf_scale: LeafScale| BlockFormat { + leaf_scale, + ..p1_format() + }; + let owned = median_statement(&elf_bytes, &opts, &at(LeafScale::Fixed)); + let plan = |scale: LeafScale| { + WhirBlockPlan::derive(&elf_bytes, &opts, &at(scale), owned.view(), None) + .expect("the plan derives") + }; + let single = plan(LeafScale::Fixed); + let n = single.partition().num_leaves(); + assert!(n >= 2, "the median mix takes {n} leaves"); + let bound = single.leaf_bound().expect("the rule's partition"); + assert_eq!( + (bound.single_leaves, bound.heights, bound.doubled), + (n, P1_LEAF_HEIGHTS, false) + ); + + let below = plan(LeafScale::DoubleFrom(n + 1)); + assert_eq!(below.partition(), single.partition()); + assert_eq!(below.chip_loads(), single.chip_loads()); + assert_eq!(below.costs(), single.costs()); + assert_eq!(below.id(), single.id()); + assert!(!below.leaf_bound().expect("bounded").doubled); + + let doubled = plan(LeafScale::DoubleFrom(n)); + let bound = doubled.leaf_bound().expect("the rule's partition"); + assert_eq!( + (bound.single_leaves, bound.heights, bound.doubled), + (n, P1_LEAF_HEIGHTS_2X, true) + ); + let m = doubled.partition().num_leaves(); + assert!(m < n, "{n} single leaves double to {m}"); + let (loads, front) = doubled.chip_loads(); + for leaf in doubled.partition().leaves() { + assert!( + sums(loads, leaf, front).under(P1_LEAF_HEIGHTS_2X), + "leaf {leaf:?}" + ); + } + assert_ne!(doubled.levels(), single.levels()); + println!( + "LEAF SCALE MEDIAN MIX: {} groups · {n} single leaves · {m} doubled · node levels {} → {}", + single.num_groups(), + single.levels().len(), + doubled.levels().len() + ); +} + +/// ★ Each query check's leaf programs on the median mix at blowup 2 (the +/// production rate), pinned: the leaf count (the leaf scale's own), and the +/// digest of the first and the last leaf (every instruction and the arena schema, as `Debug` prints +/// them) under [`LeafQuery::V1`] and [`LeafQuery::Lean`], each named, at the +/// production GKR packing and at `Off` (one tree a table: the programs before +/// the packing became production's, unchanged); the default's first leaf is +/// the lean one at the production packing. A change of which check is the +/// default moves no pin. +/// +/// Every leaf absorbs the ELF (its digest and its pages), and the built +/// `poc_rodata_commit.elf` differs by the assembler that made it, so the +/// pins are taken over a checked-in copy of it: the same bytes on every +/// machine. +/// +/// The pins move with any AIR whose constraint leg a leaf emits: last moved by +/// the ECREC table and the 23-count statement, after KECCAK's sponge mode +/// (Keccak-256 of a buffer) and its round-chip tokens. +#[test] +fn the_median_mix_leaves_are_pinned_under_each_query_check() { + use super::whir_chain::LeafQuery; + use super::whir_chain_tests::program_digest; + use multilinear::whir_chain::GkrPack; + const PINS: [(LeafQuery, GkrPack, usize, &str, &str); 4] = [ + ( + LeafQuery::V1, + GkrPack::PRODUCTION, + 5, + "24dc068769f1a38fdda271559c3918d8146666c6c24dc015d70782bb2f113b5c", + "491998ddcd396c7703a3a9d7dcd8a72cb29a7f8191da110aea796d53757ebd3e", + ), + ( + LeafQuery::Lean, + GkrPack::PRODUCTION, + 5, + "055bc62eba89a898a8e8728401370a90edbf8122d565de88f05310127a33d1b4", + "ce51c07238f53afb5deb58551f918ff39d3816ad3d717ad70b5e441bc867e454", + ), + ( + LeafQuery::V1, + GkrPack::Off, + 5, + "07f558619e111cca75018ac12c244fb6b0319e60eb8bf36695119b71a66fdc11", + "93fcbcf64a9fb55c08bb75dd27c7241aef553bf875b678a358050398c8663340", + ), + ( + LeafQuery::Lean, + GkrPack::Off, + 5, + "3f94f9beb7a983b88c3873b0f137f197ef00b07e797978d2da3f9e6b2cbf4a78", + "207745f8b67e3bd04f4ad051dc35194b4376facffe12d3869f6ae979b7ea0017", + ), + ]; + let elf_bytes: &[u8] = include_bytes!("testdata/poc_rodata_commit.elf"); + let opts = super::proof::block_base_options(); + let at = |leaf_query: LeafQuery, pack: GkrPack| { + let mut format = p1_format(); + format.zf.whir_blowup = 2; + BlockFormat { + leaf_query, + ..format + } + .with_pack(pack) + }; + let owned = median_statement(elf_bytes, &opts, &at(LeafQuery::V1, GkrPack::PRODUCTION)); + // Every value measured and printed before any is compared, so one run shows + // all of them when the leaves move. + let measured: Vec<(usize, String, String)> = PINS + .iter() + .map(|&(query, pack, ..)| { + let plan = + WhirBlockPlan::derive(elf_bytes, &opts, &at(query, pack), owned.view(), None) + .expect("the plan derives"); + let n = plan.partition().num_leaves(); + let digest = |k: usize| program_digest(&plan.leaf_program(k).expect("the leaf emits")); + let (d_first, d_last) = (digest(0), digest(n - 1)); + println!("LEAF PINS {query:?} {pack:?}: {n} leaves · first {d_first} · last {d_last}"); + (n, d_first, d_last) + }) + .collect(); + for ((query, pack, leaves, first, last), (n, d_first, d_last)) in PINS.iter().zip(&measured) { + assert_eq!(n, leaves, "{query:?} {pack:?}: leaves"); + assert_eq!( + d_first, first, + "{query:?} {pack:?}: the first leaf's program" + ); + assert_eq!(d_last, last, "{query:?} {pack:?}: the last leaf's program"); + } + let default = WhirBlockPlan::derive( + elf_bytes, + &opts, + &p1_format_at_blowup_2(), + owned.view(), + None, + ) + .expect("the plan derives"); + assert_eq!( + program_digest(&default.leaf_program(0).expect("the leaf emits")), + PINS[1].3, + "the default's first leaf is the lean check's at the production packing" + ); +} + +/// ★ The lean leaf's census on the median mix at blowup 2 (the production +/// rate): each group's emitted chip rows under [`LeafQuery::Lean`] against +/// `V1`, the same statement and partition rule. The tail rounds cut every +/// group's socket, hint and select rows, round 0's packed words its hint rows +/// again, and the leaf's front stays as it is. Prints `LEAN CENSUS` lines: the +/// per-group deltas by chip (min, median, max) — the measured deltas the +/// packing replay takes — and the tail rounds of the production chain at each +/// stack height. +#[test] +fn the_lean_leaf_cuts_every_groups_rows_on_the_median_mix() { + use super::whir_chain::LeafQuery; + let elf_bytes = crate::test_utils::asm_elf_bytes("poc_rodata_commit"); + let opts = super::proof::block_base_options(); + let at = |leaf_query: LeafQuery| { + let mut format = p1_format(); + format.zf.whir_blowup = 2; + BlockFormat { + leaf_query, + ..format + } + }; + let owned = median_statement(&elf_bytes, &opts, &at(LeafQuery::V1)); + let plan = |query: LeafQuery| { + WhirBlockPlan::derive(&elf_bytes, &opts, &at(query), owned.view(), None) + .expect("the plan derives") + }; + let (v1, lean) = (plan(LeafQuery::V1), plan(LeafQuery::Lean)); + let ((v1_loads, v1_front), (lean_loads, lean_front)) = (v1.chip_loads(), lean.chip_loads()); + assert_eq!(v1_front, lean_front, "the leaf's front is the same"); + assert_eq!(v1_loads.len(), lean_loads.len()); + let deltas: Vec<[i64; 10]> = v1_loads + .iter() + .zip(lean_loads) + .map(|(a, b)| { + let (a, b) = (a.to_array(), b.to_array()); + core::array::from_fn(|c| b[c] as i64 - a[c] as i64) + }) + .collect(); + for (g, d) in deltas.iter().enumerate() { + assert!( + d[0] < 0 && d[1] < 0 && d[5] < 0, + "group {g}: socket {} select {} hint {} — the tail rounds cut all three", + d[0], + d[1], + d[5] + ); + } + for (c, name) in ChipRows::NAMES.iter().enumerate() { + let mut col: Vec = deltas.iter().map(|d| d[c]).collect(); + col.sort_unstable(); + let v1_median = { + let mut v: Vec = v1_loads.iter().map(|l| l.to_array()[c]).collect(); + v.sort_unstable(); + v[v.len() / 2] + }; + println!( + "LEAN CENSUS {name}: per group min {} median {} max {} (V1 median {v1_median})", + col[0], + col[col.len() / 2], + col[col.len() - 1] + ); + } + println!( + "LEAN CENSUS LEAVES: V1 {} · lean {} ({} groups)", + v1.partition().num_leaves(), + lean.partition().num_leaves(), + v1.num_groups() + ); + for n_stack in 16..=24 { + let shape = lean.chain_shape(n_stack); + let tails: Vec<(usize, usize)> = (0..shape.rounds()) + .filter(|&r| shape.tail_round(r)) + .map(|r| (r, shape.current_leaves(r))) + .collect(); + println!( + "LEAN CENSUS TAILS n_stack {n_stack}: {} queries, (round, leaves) {tails:?}", + shape.num_queries + ); + } +} + +/// ★ The heights hold on the median mix, as emitted: the plan's rows are the +/// emitted ones ([`assert_planned`]: groups 0 (prepared, the carrier) and 1 +/// (heavy) alone, and a leaf of the heavy groups from 2 on, as many as take its +/// planned rows past a height — a leaf the partition never builds, its rows +/// still the sum), and every leaf the plan builds stays under every chip's +/// height. One chain a polynomial reaches a height at seven heavy groups; one +/// chain a run (the production opening) verifies fewer chains a group and +/// takes more. At the single heights ([`LeafScale::Fixed`]): at the default +/// blowup 2 the median mix takes enough leaves that the production scale +/// doubles them, and the doubled partition is the next test's. +#[test] +fn the_p1_leaves_stay_under_their_heights_on_the_median_mix() { + let elf_bytes = crate::test_utils::asm_elf_bytes("poc_rodata_commit"); + let opts = super::proof::block_base_options(); + let format = BlockFormat { + leaf_scale: LeafScale::Fixed, + ..p1_format() + }; + let owned = median_statement(&elf_bytes, &opts, &format); + let plan = WhirBlockPlan::derive_with( + &elf_bytes, + &opts, + &format, + owned.view(), + None, + super::whir_block::BLOCK_FAN_IN, + None, + ) + .expect("the plan derives"); + let (loads, front) = plan.chip_loads(); + let (loads, n) = (loads.to_vec(), plan.num_groups()); + assert_eq!(loads.len(), n); + // Every leaf the plan built, by its rows. + for leaf in plan.partition().leaves() { + assert!( + sums(&loads, leaf, front).under(P1_LEAF_HEIGHTS), + "leaf {leaf:?}" + ); + } + // The rows are the emitted ones. The third leaf: the heavy groups from 2 + // on, up to the first whose planned rows pass a height. + let past = (3..n) + .find(|&end| !sums(&loads, &(2..end).collect::>(), front).under(P1_LEAF_HEIGHTS)) + .expect("the heavy groups from 2 on pass a height before the last group"); + let lists = [ + vec![0usize], + vec![1], + (2..past).collect(), + (past..n).collect(), + ]; + let probe = plan.with_partition(BlockPartition::new(lists.to_vec(), n).expect("covers once")); + let mut target = None; + for (k, list) in lists.iter().enumerate().take(3) { + let emitted = ChipRows::of(&probe.leaf_program(k).expect("the leaf emits")); + assert_planned( + &format!("groups {list:?}"), + k == CARRIER, + emitted, + sums(&loads, list, front), + &mut target, + ); + if k == 2 { + println!( + "P1 HEAVY GROUPS PAST A HEIGHT: {} groups {emitted:?} against {P1_LEAF_HEIGHTS:?}", + list.len() + ); + assert!( + !emitted.under(P1_LEAF_HEIGHTS), + "{} heavy groups: {emitted:?}", + list.len() + ); + } + } +} + +/// The LFM chip a [`ChipRows`] field counts, by [`ChipRows::NAMES`] order. +const CHIP_NAMES: [&str; 10] = [ + "LFM_HASH", + "LFM_SELECT", + "LFM_BALU", + "LFM_XALU", + "LFM_LANES", + "LFM_HINT", + "LFM_BITDEC", + "LFM_CONST", + "LFM_PUBLIC", + "", +]; + +/// RYZEN 062's card model of a leaf (I-WHIR-P1 §S6.4): ≈ 0.28 s plus +/// ≈ 0.0122 s a million padded cells outside the socket. +fn card_seconds(non_hash_cells: u64) -> f64 { + 0.28 + 0.0122 * non_hash_cells as f64 / 1e6 +} + +/// ★ The S6b census (instrument, `--exact`; `CENSUS_ELF` for a real guest): +/// the median mix's leaves under RPX's partition, S6's (178 k socket rows, four +/// groups a leaf) and S6b's (every chip bounded) — every leaf emitted: its +/// padded chip heights, its cells (all, and outside the socket), the tree above +/// it, and 062's card model summed over the leaves (L0 is card-serial). Then +/// the 1× block's shape, nine groups (seven full, both prepared) of the same +/// mix, under S6 and S6b, by the plan's rows and the chips' widths. Prints +/// `S6b …` lines; asserts S6b's leaves under every height. +#[test] +#[ignore = "instrument: run alone with --exact (CENSUS_ELF for a real guest)"] +fn whir_s6b_census() { + let elf_bytes = match std::env::var("CENSUS_ELF") { + Ok(path) => std::fs::read(path).expect("read CENSUS_ELF"), + Err(_) => crate::test_utils::asm_elf_bytes("poc_rodata_commit"), + }; + let opts = super::proof::block_base_options(); + let production = BlockFormat::production(); + let arms: [(&str, BaseFormat, Option); 2] = [ + ("P1-S6 (178 k)", BaseFormat::P1_WHIR, Some(178_000)), + ("P1-S6b", BaseFormat::P1_WHIR, None), + ]; + // Each chip's columns a row (main + aux), from the leaves' census. + let mut widths: BTreeMap<&'static str, u64> = BTreeMap::new(); + let mut s6b: Option<(Vec, ChipRows, Vec)> = None; + for (name, base, cap) in arms { + let format = BlockFormat { + zf: production.zf.with_base(base), + ..production + }; + let owned = median_statement(&elf_bytes, &opts, &format); + let started = std::time::Instant::now(); + let plan = WhirBlockPlan::derive_capped( + &elf_bytes, + &opts, + &format, + owned.view(), + None, + super::whir_block::BLOCK_FAN_IN, + None, + cap.unwrap_or(super::whir_block::LEAF_P1_CAP), + ) + .expect("the plan derives"); + let derived = started.elapsed().as_secs_f64(); + let levels = plan.levels(); + let nodes: usize = levels.iter().map(|l| l.arities.len()).sum(); + let mut max: BTreeMap<&'static str, u64> = BTreeMap::new(); + let (mut cells_sum, mut non_hash_sum, mut card) = (0u64, 0u64, 0f64); + let mut groups_max = 0usize; + let started = std::time::Instant::now(); + for k in 0..plan.partition().num_leaves() { + let program = plan.leaf_program(k).expect("the leaf emits"); + let chips = super::airs::lfm_chip_census_with_hasher( + &program, + program.hasher(crate::hash_pin::BLOCK_HASHER), + ); + let cells: u64 = chips.iter().map(|c| c.main_cells() + c.aux_cells()).sum(); + let hash: u64 = chips + .iter() + .filter(|c| c.name == "LFM_HASH") + .map(|c| c.main_cells() + c.aux_cells()) + .sum(); + for c in &chips { + let e = max.entry(c.name).or_default(); + *e = (*e).max(c.rows); + if c.rows > 0 { + widths.insert(c.name, (c.main_cols + c.aux_cols) as u64); + } + } + if base == BaseFormat::P1_WHIR && cap.is_none() { + // The partition's own heights: doubled when the production leaf + // scale doubles the median's leaves (at blowup 2). + let heights = plan.leaf_bound().expect("the rule's partition").heights; + let rows = ChipRows::of(&program); + assert!( + rows.under(heights), + "S6b leaf {k} past a bounded height: {rows:?}" + ); + } + cells_sum += cells; + non_hash_sum += cells - hash; + card += card_seconds(cells - hash); + groups_max = groups_max.max(plan.partition().leaf(k).len()); + } + println!( + "S6b {name}: {} groups · {} leaves (≤ {groups_max} groups) · {nodes} nodes over {} node levels · leaf cells Σ {:.1} M, outside the socket Σ {:.1} M · card model Σ {card:.2} s · plan derived {derived:.2} s, leaves emitted {:.2} s · padded max {}", + plan.num_groups(), + plan.partition().num_leaves(), + levels.len(), + cells_sum as f64 / 1e6, + non_hash_sum as f64 / 1e6, + started.elapsed().as_secs_f64(), + max.iter() + .map(|(n, r)| format!("{n}={r}")) + .collect::>() + .join(" ") + ); + if base == BaseFormat::P1_WHIR && cap.is_none() { + let (loads, front) = plan.chip_loads(); + let prepared = plan.prepared().iter().map(|p| p.group).collect(); + s6b = Some((loads.to_vec(), front, prepared)); + } + } + println!("S6b CHIP WIDTHS: {widths:?}"); + + // The 1× shape: nine groups of the mix — both prepared and seven full — + // under S6's heights and S6b's, by the plan's rows. + let (loads, front, prepared) = s6b.expect("the S6b arm ran"); + let full: Vec = (0..loads.len()) + .filter(|g| !prepared.contains(g)) + .take(7) + .collect(); + let nine: Vec = prepared.iter().chain(&full).map(|&g| loads[g]).collect(); + let leaf_cells = |r: ChipRows| -> u64 { + r.to_array() + .iter() + .zip(CHIP_NAMES) + .filter(|&(&rows, name)| rows > 0 && !name.is_empty() && name != "LFM_HASH") + .map(|(&rows, name)| { + rows.next_power_of_two() as u64 * widths.get(name).copied().unwrap_or(0) + }) + .sum::() + + 65_536 * widths.get("LFM_RANGE").copied().unwrap_or(0) + }; + for (name, hash) in [("P1-S6 (178 k)", 178_000), ("P1-S6b", P1_LEAF_HEIGHTS.hash)] { + let heights = ChipRows { + hash, + ..P1_LEAF_HEIGHTS + }; + let p = leaf_partition_rows(&nine, front, heights).expect("the nine partition"); + let leaves: Vec = p.leaves().iter().map(|l| sums(&nine, l, front)).collect(); + let card: f64 = leaves.iter().map(|&r| card_seconds(leaf_cells(r))).sum(); + println!( + "S6b X1-SHAPE {name}: {} leaves {:?} · outside the socket {:?} M · card model Σ {card:.2} s", + p.num_leaves(), + p.leaves(), + leaves + .iter() + .map(|&r| format!("{:.1}", leaf_cells(r) as f64 / 1e6)) + .collect::>(), + ); + } +} + +/// ★ Every leaf's emitted rows are checked against the heights its partition +/// holds it under ([`WhirBlockPlan::leaf_program`]): the median mix's leaves +/// emit under them, and with the bound's `Select` height one below leaf 0's +/// emitted `Select` rows, that leaf is refused by name rather than emitted. +#[test] +fn every_emitted_leaf_is_held_under_its_partitions_heights() { + let elf_bytes: &[u8] = include_bytes!("testdata/poc_rodata_commit.elf"); + let opts = super::proof::block_base_options(); + let format = p1_format(); + let owned = median_statement(elf_bytes, &opts, &format); + let plan = WhirBlockPlan::derive(elf_bytes, &opts, &format, owned.view(), None) + .expect("the plan derives"); + let bound = plan + .leaf_bound() + .expect("a derived partition bounds its leaves"); + for k in 0..plan.partition().num_leaves() { + let rows = ChipRows::of(&plan.leaf_program(k).expect("the leaf emits")); + assert!(rows.under(bound.heights), "leaf {k}: {rows:?}"); + } + let rows = ChipRows::of(&plan.leaf_program(0).expect("the leaf emits")); + let tight = ChipRows { + select: rows.select - 1, + ..bound.heights + }; + let refused = plan + .with_leaf_heights(tight) + .leaf_program(0) + .expect_err("a leaf past its heights is refused"); + assert!(refused.starts_with("leaf 0 emits"), "{refused}"); +} + +/// ★ The one-top scale ([`LeafScale::DoubledOneTop`]): its knob reads +/// `one-top` and is the default; its partition is the doubled one at any leaf +/// count, where the threshold scale keeps the single one below its threshold; +/// and its +/// tree puts at most [`super::whir_block::ONE_TOP_MAX_LEAVES`] leaves under +/// one top, past that the plan's fan-in, where the threshold scale always +/// builds nodes of the fan-in. On the median mix (forced leaf counts, and the rule's +/// own partition, which doubles it to five leaves and so keeps a node level). +#[test] +fn the_one_top_scale_doubles_and_puts_at_most_three_leaves_under_one_top() { + use super::whir_block::{ONE_TOP_MAX_LEAVES, leaf_scale_setting, scaled_leaf_partition}; + assert_eq!( + leaf_scale_setting(Some("one-top")), + Ok(LeafScale::DoubledOneTop) + ); + assert_eq!( + LeafScale::DoubledOneTop.to_string(), + "doubled, one top over at most 3 leaves" + ); + // One group, one single leaf: the default keeps it, the one-top scale + // doubles it. + let loads = [ChipRows { + hash: 1_000, + ..ChipRows::default() + }]; + let front = ChipRows::default(); + let bound = |scale| { + scaled_leaf_partition(&loads, front, scale, P1_LEAF_HEIGHTS, P1_LEAF_HEIGHTS_2X) + .expect("a partition") + .1 + }; + let threshold = LeafScale::DoubleFrom(super::whir_block::LEAF_DOUBLE_FROM_DEFAULT); + assert_eq!(LeafScale::DEFAULT, LeafScale::DoubledOneTop); + assert!(!bound(threshold).doubled); + assert!(bound(LeafScale::DoubledOneTop).doubled); + assert_eq!(bound(LeafScale::DoubledOneTop).heights, P1_LEAF_HEIGHTS_2X); + + let elf_bytes: &[u8] = include_bytes!("testdata/poc_rodata_commit.elf"); + let opts = super::proof::block_base_options(); + let at = |scale: LeafScale| BlockFormat { + leaf_scale: scale, + ..p1_format() + }; + let owned = median_statement(elf_bytes, &opts, &at(threshold)); + let levels = |scale: LeafScale, leaves: Option| -> Vec> { + WhirBlockPlan::derive_with( + elf_bytes, + &opts, + &at(scale), + owned.view(), + leaves, + super::whir_block::BLOCK_FAN_IN, + None, + ) + .expect("the plan derives") + .levels() + .into_iter() + .map(|level| level.arities) + .collect() + }; + for leaves in 1..=ONE_TOP_MAX_LEAVES { + assert_eq!( + levels(LeafScale::DoubledOneTop, Some(leaves)), + vec![vec![leaves]], + "{leaves} leaves under one top" + ); + } + assert_eq!(levels(threshold, Some(4)), vec![vec![3, 1], vec![2]]); + assert_eq!( + levels(LeafScale::DoubledOneTop, Some(4)), + vec![vec![3, 1], vec![2]] + ); + assert_eq!( + levels(LeafScale::DoubledOneTop, Some(5)), + vec![vec![3, 2], vec![2]] + ); + assert_eq!( + levels(LeafScale::DoubledOneTop, None), + vec![vec![3, 2], vec![2]] + ); +} diff --git a/prover/src/lfm/whir_p1_leaf_tests.rs b/prover/src/lfm/whir_p1_leaf_tests.rs new file mode 100644 index 000000000..0e70babb8 --- /dev/null +++ b/prover/src/lfm/whir_p1_leaf_tests.rs @@ -0,0 +1,928 @@ +//! Gates for the WHIR leaf's Poseidon1 arm (D-WHIR-P1 S5): the transcript +//! replay, the child-order 4-ary walk and the assembled chain, each against +//! the host function it mirrors. +//! +//! Every program here is built by a [`WrapHash::Poseidon1`] builder and run on +//! the width-16 socket (`HasherKind::Poseidon1W16`, the executor's host +//! permutation). As in `whir_chain_tests`, executing an honest proof IS the +//! stream comparison: every drawn value feeds a refusal. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::p1_transcript::P1Transcript; +use crypto::hash::poseidon1_stark::Merkle4; +use multilinear::mle::Mle; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, ChainProof, FirstFold, GrindBits, LeafOrder, + RoundOpenings, WhirFolds, commit, prove, verify, +}; +use multilinear::whir_hash::P1Whir; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::commitment_to_digest; +use super::builder::{Ext, Felt, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::{self, WrapDigest, WrapHash}; +use super::executor::execute; +use super::hash::HasherKind; +use super::instr::Instr; +use super::p1w16_emit::{hint_order, walk4, walk4_child}; +use super::validator::validate; +use super::whir_chain::{ + ChainShape, RoundStorage, chain_opening_perms, chain_perms, emit_verify_weighted, + push_round_words, round_words, +}; +use super::whir_poly::emit_eq_eval; +use super::whir_transcript::{SpongeEntry, WhirTranscript}; +use super::word::{LfmWord, base_word, ext_word}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +const SOCKET: HasherKind = HasherKind::Poseidon1W16; + +pub(super) fn p1_builder() -> LfmBuilder { + LfmBuilder::new().with_wrap_hash(WrapHash::Poseidon1) +} + +fn felt(seed: u64, i: usize) -> FE { + let mut z = seed + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(i as u64 + 1); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + FE::from(z ^ (z >> 31)) +} + +fn fee(seed: u64, i: usize) -> FEE { + FEE::new([ + felt(seed, 3 * i), + felt(seed, 3 * i + 1), + felt(seed, 3 * i + 2), + ]) +} + +/// Compile, validate and run `b`'s program; the published words, flattened. +fn run(b: LfmBuilder, arenas: &[Vec]) -> Result, String> { + let program = compile(b.finish()); + validate(&program).map_err(|e| format!("not admissible: {e:?}"))?; + let exec = execute(&program, arenas, &SOCKET).map_err(|e| format!("{e:?}"))?; + Ok(exec.public_words.iter().flat_map(|(_, w)| *w).collect()) +} + +// ============================== the transcript ============================= + +/// One call of a transcript stream, both sides. +#[derive(Clone)] +enum Call { + Bytes(Vec), + Ext(FEE), + Root([u8; 32]), + Nonce(u64), + DrawExt, + DrawBits(usize), + State, +} + +/// ★ The in-guest Poseidon1 transcript replays `P1Transcript` call for call: +/// constant `append_bytes` of every length class (empty, short, a felt, not a +/// felt multiple, more than a rate), field elements, roots, nonces, extension +/// and bounded draws, and `state()` reads at every buffer fill — each output +/// published and compared with the host's. +#[test] +fn the_p1_whir_transcript_replays_the_hosts_stream() { + let root = |seed: u64| -> [u8; 32] { + crypto::hash::rpx::digest_to_commitment(&core::array::from_fn::<_, 4, _>(|k| felt(seed, k))) + }; + let mut calls = vec![ + Call::Bytes(b"LAMBDAVM_MULTILINEAR_BLOCK_STATEMENT_V2/P1W16/C2".to_vec()), + Call::Bytes(Vec::new()), + Call::Bytes(vec![7; 8]), + Call::Bytes(vec![1, 2, 3]), + Call::State, + Call::Root(root(1)), + Call::Root(root(2)), + Call::DrawExt, + Call::DrawExt, + Call::Bytes((0..100u8).collect()), + Call::Ext(fee(3, 0)), + Call::State, + Call::Ext(fee(3, 1)), + Call::Nonce(0x1234_5678_9abc), + Call::DrawBits(10), + Call::DrawBits(17), + ]; + // Twenty draws in a row cross a permutation's sixteen output lanes. + calls.extend((0..20).map(|i| Call::DrawBits(1 + i % 20))); + for i in 0..13 { + calls.push(Call::Ext(fee(4, i))); + if i % 4 == 3 { + calls.push(Call::State); + calls.push(Call::DrawExt); + } + } + calls.push(Call::State); + + // Host. + let mut host = P1Transcript::new(); + let mut want: Vec = Vec::new(); + let mut arena: Vec = Vec::new(); + for call in &calls { + match call { + Call::Bytes(bytes) => host.append_bytes(bytes), + Call::Ext(v) => { + host.append_field_element(v); + arena.push(ext_word(v)); + } + Call::Root(r) => { + host.append_bytes(r); + arena.push(commitment_to_digest(r)); + } + Call::Nonce(n) => { + host.append_bytes(&n.to_be_bytes()); + arena.push(base_word(FE::from(*n))); + } + Call::DrawExt => want.extend(ext_word(&host.sample_field_element())), + Call::DrawBits(n) => { + let v = host.sample_u64(1u64 << n); + want.extend((0..*n).map(|j| FE::from((v >> j) & 1))); + } + Call::State => { + let s = host.state(); + want.extend(commitment_to_digest(&s)); + } + } + } + + // Guest. + let mut b = p1_builder(); + let a = b.declare_arena(arena.len() as u32); + let mut t = WhirTranscript::for_builder(&mut b); + let mut at = 0u32; + for call in &calls { + match call { + Call::Bytes(bytes) => t.absorb_const_bytes(bytes), + Call::Ext(_) => { + let v = b.hint_word(a, at).as_ext(); + at += 1; + t.absorb_ext(&mut b, v); + } + Call::Root(_) => { + let r = b.hint_word(a, at); + at += 1; + t.absorb_digest(&mut b, r); + } + Call::Nonce(_) => { + let n = b.hint_felt(a, at); + at += 1; + t.absorb_nonce(&mut b, n); + } + Call::DrawExt => { + let e = t.sample_ext(&mut b); + b.public(e.as_cell()); + } + Call::DrawBits(n) => { + for bit in t.sample_u64_pow2(&mut b, *n) { + b.public(Felt(bit.addr()).as_cell()); + } + } + Call::State => { + let s = t.state(&mut b); + b.public(s); + } + } + } + // A bit publishes as a word whose low lane is the bit. + let got = run(b, &[arena]).expect("the replay executes"); + let mut expect_words: Vec = Vec::new(); + let mut k = 0usize; + for call in &calls { + let n = match call { + Call::DrawExt | Call::State => 4, + Call::DrawBits(n) => *n, + _ => 0, + }; + for _ in 0..n { + let v = want[k]; + k += 1; + if matches!(call, Call::DrawBits(_)) { + expect_words.extend([v, FE::zero(), FE::zero(), FE::zero()]); + } else { + expect_words.push(v); + } + } + } + assert_eq!( + got.len(), + expect_words.len(), + "one published word per output" + ); + assert_eq!( + got, expect_words, + "the replay's draws and states are the host's" + ); +} + +/// The byte arm is untouched: a [`WhirTranscript::for_builder`] of an RPX +/// builder emits exactly what [`WhirTranscript::new`] does. +#[test] +fn an_rpx_builder_keeps_the_byte_transcript() { + let emit = |for_builder: bool| -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(WrapHash::production()); + let a = b.declare_arena(3); + let mut t = if for_builder { + WhirTranscript::for_builder(&mut b) + } else { + WhirTranscript::new() + }; + t.absorb_const_bytes(b"statement"); + t.absorb_const_bytes(&[0u8; 7]); + let r = b.hint_word(a, 0); + t.absorb_digest(&mut b, r); + let lanes = b.unpack(r); + t.absorb_root_lanes(&mut b, &lanes); + let n = b.hint_felt(a, 1); + t.absorb_nonce(&mut b, n); + let e = t.sample_ext(&mut b); + b.public(e.as_cell()); + let s = t.state(&mut b); + b.public(s); + compile(b.finish()) + }; + assert_eq!( + format!("{:?}", emit(true).instrs), + format!("{:?}", emit(false).instrs) + ); +} + +// ================================= the walk ================================ + +/// The child-order walk folds a leaf onto `Merkle4`'s root at every index of +/// trees of 1 to 6 index bits (odd tops included), equals the hint-order walk +/// over the same path, and refuses a moved sibling. +#[test] +fn the_child_order_walk_reaches_the_hosts_root() { + for bits in 1..=6usize { + let n = 1usize << bits; + let leaves: Vec<[FE; 4]> = (0..n) + .map(|i| core::array::from_fn(|k| felt(bits as u64, 4 * i + k))) + .collect(); + let tree = Merkle4::new(&leaves).expect("a tree"); + let root = tree.root(); + let indices: Vec = if n <= 16 { + (0..n).collect() + } else { + vec![0, 1, 2, 3, n / 2 + 1, n - 2, n - 1] + }; + for index in indices { + let path = tree.path(index).expect("a leaf"); + let triples: Vec<[[FE; 4]; 3]> = path + .iter() + .map(|s| core::array::from_fn(|j| core::array::from_fn(|k| s[4 * j + k]))) + .collect(); + // The proof's own order, the odd top cut to its partner. + let mut child: Vec<[FE; 4]> = triples.iter().flatten().copied().collect(); + if bits % 2 == 1 { + child.truncate(child.len() - 2); + } + let hints = hint_order(index, bits, &triples); + for moved in [None, Some(0usize), Some(child.len() - 1)] { + let mut sib = child.clone(); + if let Some(at) = moved { + sib[at][1] += FE::one(); + } + let mut b = p1_builder(); + let a = b.declare_arena((1 + 1 + sib.len() + hints.len()) as u32); + let leaf = edsl::hint_digest(&mut b, a, 0); + let idx = b.hint_felt(a, 1); + let index_bits = b.bit_dec(idx, bits); + let s: Vec = (0..sib.len()) + .map(|i| edsl::hint_digest(&mut b, a, 2 + i as u32)) + .collect(); + let h: Vec = (0..hints.len()) + .map(|i| edsl::hint_digest(&mut b, a, (2 + sib.len() + i) as u32)) + .collect(); + let walked = walk4_child(&mut b, leaf, &index_bits, &s); + let reference = walk4(&mut b, leaf, &index_bits, &h); + b.public(walked.cells()[0]); + b.public(reference.cells()[0]); + let mut arena = vec![leaves[index], base_word(FE::from(index as u64))]; + arena.extend(sib.iter().copied()); + arena.extend(hints.iter().copied()); + let got = run(b, &[arena]).expect("the walks execute"); + let (child_root, hint_root) = (&got[..4], &got[4..8]); + assert_eq!( + hint_root, + &root[..], + "bits {bits} index {index}: hint order" + ); + if moved.is_none() { + assert_eq!( + child_root, + &root[..], + "bits {bits} index {index}: child order" + ); + } else { + assert_ne!( + child_root, + &root[..], + "bits {bits} index {index}: a moved sibling moves the root" + ); + } + } + } + } +} + +// ================================= the chain =============================== + +pub(super) fn pseudo_mle(num_vars: usize, seed: u64) -> Mle { + let vals = (0..1usize << num_vars) + .map(|i| { + let mixed = (i as u64) + .wrapping_mul(0x2545_F491_4F6C_DD1D) + .wrapping_add(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + FE::from(mixed >> 13) + }) + .collect(); + Mle::new(vals).expect("a cube") +} + +pub(super) fn point(num_vars: usize, seed: u64) -> Vec { + (0..num_vars) + .map(|i| { + FEE::new([ + FE::from(101 + seed + i as u64), + FE::from(7 * i as u64 + 3), + FE::from(i as u64 + 1), + ]) + }) + .collect() +} + +/// A small chain config at arity 4: blowup 4, folds of 4, `cap4` the 4-ary +/// cap policy (`ChainFormat::arity4_cap`). +pub(super) fn p1_config(num_queries: usize, grind: u8, cap4: CapPolicy) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries, + grind: GrindBits::uniform(grind), + format: ChainFormat { + arity4_cap: cap4, + ..ChainFormat::DEFAULT + }, + } +} + +pub(super) struct Fixture { + pub(super) cfg: ChainConfig, + pub(super) proof: ChainProof, + pub(super) z: Vec, + pub(super) y: FEE, + pub(super) root_bytes: [u8; 32], + pub(super) domain: Domain, + pub(super) shape: ChainShape, +} + +/// Proves one chain over `P1Whir` and checks the host verifier accepts it. +pub(super) fn fixture(num_vars: usize, cfg: ChainConfig) -> Fixture { + fixture_over(pseudo_mle(num_vars, 11), num_vars, cfg) +} + +/// [`fixture`] over a constant polynomial: every codeword is constant, so the +/// blocks a tree opens are equal. +pub(super) fn constant_fixture(num_vars: usize, cfg: ChainConfig) -> Fixture { + let f = Mle::new(vec![FE::from(7u64); 1usize << num_vars]).expect("a cube"); + fixture_over(f, num_vars, cfg) +} + +/// [`fixture`] over the polynomial `f`. +fn fixture_over(f: Mle, num_vars: usize, cfg: ChainConfig) -> Fixture { + let z = point(num_vars, 0); + let y = f.evaluate_in::(&z).expect("f takes its own point"); + let (commitment, domain) = commit::(&f, &cfg, true).expect("the polynomial commits"); + let mut proving = P1Transcript::new(); + let proof = prove::(&f, &z, &commitment, &domain, &cfg, &mut proving) + .expect("the chain proves"); + let root_bytes = commitment.root(); + let mut checking = P1Transcript::new(); + verify::(&proof, &root_bytes, &z, y, &domain, &cfg, &mut checking) + .expect("the host accepts its own proof"); + let shape = ChainShape::new_at(&cfg, num_vars, 4); + Fixture { + cfg, + proof, + z, + y, + root_bytes, + domain, + shape, + } +} + +pub(super) fn host_accepts(f: &Fixture, proof: &ChainProof) -> bool { + let mut t = P1Transcript::new(); + verify::(proof, &f.root_bytes, &f.z, f.y, &f.domain, &f.cfg, &mut t).is_ok() +} + +/// The chain program for `shape` under Poseidon1: `z`, `y`, the root and the +/// final value hinted, then the rounds, as `whir_chain_tests::chain_program`. +pub(super) fn p1_chain_program(shape: &ChainShape) -> LfmProgram { + let mut b = p1_builder(); + let mut total = (shape.num_vars + 3) as u32; + let first_round = total; + for r in 0..shape.rounds() { + total += round_words(shape, r); + } + let arena = b.declare_arena(total); + let mut transcript = WhirTranscript::for_builder(&mut b); + let z: Vec = (0..shape.num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let y = b.hint_word(arena, shape.num_vars as u32).as_ext(); + let root = b.hint_word(arena, shape.num_vars as u32 + 1); + let root_lanes = b.unpack(root); + let final_value = b.hint_word(arena, shape.num_vars as u32 + 2).as_ext(); + let storage = RoundStorage::hint(&mut b, arena, first_round, shape); + let (current, next) = storage.openings(); + let wires = storage.wires(¤t, &next); + let domain = Domain::::new(shape.domain_log[0]).expect("the first domain"); + emit_verify_weighted( + &mut b, + &mut transcript, + &wires, + &root_lanes, + final_value, + y, + shape, + &domain, + |b, alphas| emit_eq_eval(b, &z, alphas), + ); + let program = compile(b.finish()); + validate(&program).expect("the P1 chain leg is admissible (no twelve-felt hash row)"); + program +} + +fn p1_chain_arena(f: &Fixture, proof: &ChainProof) -> Vec { + let mut words: Vec = f.z.iter().map(ext_word).collect(); + words.push(ext_word(&f.y)); + words.push(commitment_to_digest(&f.root_bytes)); + words.push(ext_word(&proof.final_value)); + push_round_words(&mut words, &f.shape, proof); + words +} + +pub(super) fn hash16_rows(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|i| matches!(i, Instr::Hash16(_))) + .count() +} + +/// The shapes the chain gates run: odd and even tree depths, one to three +/// rounds, uncapped and at the C2 cap, grind off and on. +fn chain_cases() -> Vec<(usize, ChainConfig)> { + let mut cases = Vec::new(); + for cap4 in [CapPolicy::Off, CapPolicy::Fixed(2)] { + for (num_vars, num_queries, grind) in [ + (5usize, 3usize, 0u8), + (6, 4, 0), + (9, 3, 0), + (6, 3, 6), + (9, 4, 6), + ] { + cases.push((num_vars, p1_config(num_queries, grind, cap4))); + } + } + cases +} + +/// ★ The assembled Poseidon1 chain executes on every proof the host accepts. +#[test] +fn a_p1_chain_executes_on_a_proof_the_host_accepts() { + for (num_vars, cfg) in chain_cases() { + let f = fixture(num_vars, cfg); + let program = p1_chain_program(&f.shape); + let arena = p1_chain_arena(&f, &f.proof); + execute(&program, &[arena], &SOCKET).unwrap_or_else(|e| { + panic!( + "S={num_vars} caps {:?} grind {:?}: the machine refused an accepted proof: {e:?}", + f.shape.caps, f.shape.grind + ) + }); + } +} + +/// ★ The socket rows the chain emits are its closed form: openings, cap +/// checks and the sponge's own permutations (the width-8 grind is ALU work). +#[test] +fn a_p1_chain_emits_its_permutation_closed_form() { + for (num_vars, cfg) in chain_cases() { + let f = fixture(num_vars, cfg); + let program = p1_chain_program(&f.shape); + let entry = SpongeEntry::fresh_for(WrapHash::Poseidon1); + assert_eq!( + hash16_rows(&program), + chain_perms(&f.shape, entry), + "S={num_vars} caps {:?} grind {:?}: Hash16 rows against the closed form \ + (openings {})", + f.shape.caps, + f.shape.grind, + chain_opening_perms(&f.shape) + ); + } +} + +/// ★ Every forgery the host rejects, the machine refuses: a final value, a +/// sumcheck evaluation, an out-of-domain value, a grind nonce, a current and a +/// successor Merkle sibling, and (capped) a cap node. +#[test] +fn a_tampered_p1_chain_cannot_execute() { + for cap4 in [CapPolicy::Off, CapPolicy::Fixed(2)] { + let f = fixture(9, p1_config(3, 6, cap4)); + let program = p1_chain_program(&f.shape); + assert!( + execute(&program, &[p1_chain_arena(&f, &f.proof)], &SOCKET).is_ok(), + "the untouched proof must execute, or the arm below proves nothing" + ); + let mut sites: Vec<(&str, ChainProof)> = Vec::new(); + let mut forged = f.proof.clone(); + forged.final_value += FEE::one(); + sites.push(("the final value", forged)); + let mut forged = f.proof.clone(); + forged.rounds[0].sumcheck[0].evaluations[0] += FEE::one(); + sites.push(("a sumcheck evaluation", forged)); + let mut forged = f.proof.clone(); + if let Some(v) = forged.rounds[0].ood_value.as_mut() { + *v += FEE::one(); + } + sites.push(("an out-of-domain value", forged)); + let mut forged = f.proof.clone(); + forged.rounds[0].nonces.query ^= 1; + sites.push(("a grind nonce", forged)); + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => p.current[1].proof.merkle_path[0][0] ^= 1, + RoundOpenings::Extension(p) => p.current[1].proof.merkle_path[0][0] ^= 1, + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push(("a current Merkle sibling", forged)); + // A successor sibling: the first round whose successor paths walk at + // all (a tree shorter than its cap has none). + let mut forged = f.proof.clone(); + let moved = forged.rounds.iter_mut().any(|round| { + let next = match &mut round.openings { + RoundOpenings::Base(p) => &mut p.next, + RoundOpenings::Extension(p) => &mut p.next, + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + }; + match next.get_mut(1) { + Some(o) if !o.proof.merkle_path.is_empty() => { + o.proof.merkle_path[0][0] ^= 1; + true + } + _ => false, + } + }); + if moved { + sites.push(("a successor Merkle sibling", forged)); + } + if cap4 != CapPolicy::Off { + // The owner path ends with the cap: its last node is a cap node. + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => { + let path = &mut p.current[0].proof.merkle_path; + let last = path.len() - 1; + path[last][0] ^= 1; + } + RoundOpenings::Extension(p) => { + let path = &mut p.current[0].proof.merkle_path; + let last = path.len() - 1; + path[last][0] ^= 1; + } + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + sites.push(("a cap node", forged)); + } + for (name, forged) in &sites { + assert!( + !host_accepts(&f, forged), + "{name} (caps {:?}): the host must reject the forgery", + f.shape.caps + ); + assert!( + execute(&program, &[p1_chain_arena(&f, forged)], &SOCKET).is_err(), + "{name} (caps {:?}): the machine must refuse the forgery", + f.shape.caps + ); + } + } +} + +/// [`p1_config`] with its round-0 trees in `leaves`' order, round 0 folding +/// `first` variables when given. +pub(super) fn p1_ordered_config( + num_queries: usize, + grind: u8, + cap4: CapPolicy, + first: Option, + leaves: LeafOrder, +) -> ChainConfig { + let mut cfg = p1_config(num_queries, grind, cap4); + cfg.format.leaves = leaves; + if let Some(k0) = first { + cfg.format.folds = WhirFolds::First(FirstFold::new(k0).expect("a tested first fold")); + } + cfg +} + +/// The coset gates' shapes: [`chain_cases`] in the coset order, and the +/// production's first fold of 6 under the production cap (`Auto`) and at the C2 +/// cap, at odd and even round-0 depths. +fn coset_chain_cases() -> Vec<(usize, ChainConfig)> { + let mut cases: Vec<(usize, ChainConfig)> = chain_cases() + .into_iter() + .map(|(num_vars, mut cfg)| { + cfg.format.leaves = LeafOrder::Coset; + (num_vars, cfg) + }) + .collect(); + for cap4 in [CapPolicy::Auto, CapPolicy::Fixed(2)] { + for (num_vars, num_queries, grind) in [(9usize, 3usize, 0u8), (10, 4, 6), (11, 4, 0)] { + cases.push(( + num_vars, + p1_ordered_config(num_queries, grind, cap4, Some(6), LeafOrder::Coset), + )); + } + } + cases +} + +/// ★ A Poseidon1 chain whose round-0 tree places block `j` at `bitrev(j)` +/// executes on every proof the host accepts: round 0's 4-ary walks take the +/// index bits reversed, the position the host prover placed the block at. +#[test] +fn a_coset_p1_chain_executes_on_a_proof_the_host_accepts() { + for (num_vars, cfg) in coset_chain_cases() { + let f = fixture(num_vars, cfg); + assert_eq!(f.shape.leaves, LeafOrder::Coset); + let program = p1_chain_program(&f.shape); + execute(&program, &[p1_chain_arena(&f, &f.proof)], &SOCKET).unwrap_or_else(|e| { + panic!( + "S={num_vars} {:?} caps {:?}: the machine refused an accepted coset proof: {e:?}", + f.shape.schedule, f.shape.caps + ) + }); + } +} + +/// ★ The round-0 rewire adds no row and is load-bearing. At every coset shape +/// the coset program and the natural one have the same rows chip by chip and +/// different instructions. Each refuses the other's proofs (and so does the +/// host, so each refusal is of a proof invalid under that order). The refusals +/// are run where round 0's tree has 2^7 leaves, 16 of them their own bit +/// reversal, with nine queries and no grind: a natural proof whose every query +/// lands on such a leaf is valid under both orders, as it should be, and turns +/// up with probability 8^-9. +#[test] +fn a_coset_p1_program_and_a_natural_one_refuse_each_others_proofs() { + for (num_vars, cfg) in coset_chain_cases() { + let mut natural = cfg; + natural.format.leaves = LeafOrder::Natural; + let coset_program = p1_chain_program(&ChainShape::new_at(&cfg, num_vars, 4)); + let natural_program = p1_chain_program(&ChainShape::new_at(&natural, num_vars, 4)); + assert_eq!( + super::whir_chain::ChipRows::of(&coset_program), + super::whir_chain::ChipRows::of(&natural_program), + "S={num_vars}: no row added" + ); + assert_ne!( + format!("{:?}", coset_program.instrs), + format!("{:?}", natural_program.instrs), + "S={num_vars}: the round-0 walks are wired otherwise" + ); + } + for cap4 in [CapPolicy::Off, CapPolicy::Fixed(2)] { + let coset = p1_ordered_config(9, 0, cap4, Some(6), LeafOrder::Coset); + let natural = p1_ordered_config(9, 0, cap4, Some(6), LeafOrder::Natural); + let made_coset = fixture(11, coset); + let made_natural = fixture(11, natural); + assert_eq!( + made_coset.shape.domain_log[0] - made_coset.shape.schedule[0], + 7 + ); + let coset_program = p1_chain_program(&made_coset.shape); + let natural_program = p1_chain_program(&made_natural.shape); + let mut t = P1Transcript::new(); + let natural_under_coset = verify::( + &made_natural.proof, + &made_natural.root_bytes, + &made_natural.z, + made_natural.y, + &made_natural.domain, + &coset, + &mut t, + ); + assert!( + natural_under_coset.is_err(), + "{cap4:?}: the host, a natural proof" + ); + assert!( + !host_accepts(&made_natural, &made_coset.proof), + "{cap4:?}: the host, a coset proof" + ); + assert!( + execute( + &coset_program, + &[p1_chain_arena(&made_natural, &made_natural.proof)], + &SOCKET + ) + .is_err(), + "{cap4:?}: a coset program accepted a natural proof" + ); + assert!( + execute( + &natural_program, + &[p1_chain_arena(&made_coset, &made_coset.proof)], + &SOCKET + ) + .is_err(), + "{cap4:?}: a natural program accepted a coset proof" + ); + } +} + +/// ★ Under the coset order the machine refuses what the host rejects: a round-0 +/// block value and a round-0 Merkle sibling, uncapped and capped. +#[test] +fn a_tampered_coset_p1_chain_cannot_execute() { + for cap4 in [CapPolicy::Off, CapPolicy::Fixed(2)] { + let f = fixture(11, p1_ordered_config(4, 0, cap4, Some(6), LeafOrder::Coset)); + let program = p1_chain_program(&f.shape); + assert!( + execute(&program, &[p1_chain_arena(&f, &f.proof)], &SOCKET).is_ok(), + "the untouched proof must execute, or the arm below proves nothing" + ); + let mut sites: Vec<(&str, ChainProof)> = Vec::new(); + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => p.current[1].values[63] += FE::one(), + _ => unreachable!("a per-polynomial chain opens base blocks in round 0"), + } + sites.push(("a round-0 block value", forged)); + let mut forged = f.proof.clone(); + match &mut forged.rounds[0].openings { + RoundOpenings::Base(p) => p.current[1].proof.merkle_path[0][0] ^= 1, + _ => unreachable!("a per-polynomial chain opens base blocks in round 0"), + } + sites.push(("a round-0 Merkle sibling", forged)); + for (name, forged) in &sites { + assert!( + !host_accepts(&f, forged), + "{name} ({cap4:?}): the host must reject it" + ); + assert!( + execute(&program, &[p1_chain_arena(&f, forged)], &SOCKET).is_err(), + "{name} ({cap4:?}): the machine must refuse it" + ); + } + } +} + +/// The fold point read through the base ALU (`edsl::pow_bits_alu`, the P1 +/// leaf's) is `pow_bits`' value at every index, with no `Select`. +#[test] +fn the_alu_point_is_pow_bits() { + let factors: Vec = (0..12).map(|i| felt(9, i)).collect(); + for index in [0u64, 1, 2, 5, 0x5a5, 0xfff] { + let mut b = p1_builder(); + let a = b.declare_arena(1); + let idx = b.hint_felt(a, 0); + let bits = b.bit_dec(idx, factors.len()); + let alu = edsl::pow_bits_alu(&mut b, &bits, &factors); + let reference = edsl::pow_bits(&mut b, &bits, &factors, FE::one()); + b.public(alu.as_cell()); + b.public(reference.as_cell()); + let got = run(b, &[vec![base_word(FE::from(index))]]).expect("executes"); + let want = (0..factors.len()) + .filter(|i| index >> i & 1 == 1) + .fold(FE::one(), |acc, i| acc * factors[i]); + assert_eq!((got[0], got[4]), (want, want), "index {index:#x}"); + } + let mut b = p1_builder(); + let a = b.declare_arena(1); + let idx = b.hint_felt(a, 0); + let bits = b.bit_dec(idx, factors.len()); + let alu = edsl::pow_bits_alu(&mut b, &bits, &factors); + b.public(alu.as_cell()); + let program = compile(b.finish()); + let rows = super::whir_chain::ChipRows::of(&program); + assert_eq!((rows.select, rows.balu), (0, 2 * factors.len() - 1)); +} + +/// ★ The ALU fold point is sound only while each index bit is a bit, and the +/// machine refuses a non-bit (the lead's item on `pow_bits_alu`, 10-06). +/// +/// The index bits are `LFM_BITDEC`'s: it receives the drawn felt as the +/// recomposition `Σ 2^i·B_i` of its 64 bit columns (`chips.rs`, `bitdec:: +/// bus_interactions`), constrains each `B_i·(1 − B_i) = 0` and canonicity +/// (`BitDecConstraints`), and sends each kept bit as the cell's value. So a +/// prover who wants another fold point must change a bit and keep the +/// recomposition: here the index 2 = (b0, b1) = (0, 1) is rewritten as +/// (2, 0), which recomposes to 2 as well, and every row reading those cells +/// is recomputed so memory stays consistent. The point then reads +/// `2·f0 − 1` where the transcript drew `f1`. Only booleanity is left +/// violated, and the proof is refused; the untouched trace proves. +#[test] +fn a_non_boolean_index_bit_is_refused() { + use super::proof::{prove_traces_with_hasher, verify_against}; + use super::registry::build_artifacts_with_hasher; + use super::trace::build_traces_with_hasher; + use crate::tables::types::VmTable; + use stark::proof::options::GoldilocksCubicProofOptions; + + const KIND: HasherKind = crate::hash_pin::BLOCK_HASHER; + let (f0, f1) = (felt(11, 0), felt(11, 1)); + let program = { + let mut b = p1_builder(); + let a = b.declare_arena(1); + let idx = b.hint_felt(a, 0); + let bits = b.bit_dec(idx, 2); + let x = edsl::pow_bits_alu(&mut b, &bits, &[f0, f1]); + b.public(x.as_cell()); + compile(b.finish()) + }; + validate(&program).expect("admissible"); + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("options"); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let arena = vec![base_word(FE::from(2u64))]; + let exec = execute(&program, std::slice::from_ref(&arena), &KIND).expect("executes"); + assert_eq!(exec.public_words[0].1[0], f1, "index 2 reads f1"); + + let round_trip = |tamper: bool| -> Result { + let mut traces = build_traces_with_hasher(&program, &exec.records, KIND); + let mut public = exec.public_words.clone(); + if tamper { + use super::chips::{balu, bitdec, public as pubc}; + let set = |t: &mut stark::trace::TraceTable, r: usize, c: usize, v: FE| { + t.main_table.set_fe(r, c, v) + }; + let get = |t: &stark::trace::TraceTable, r: usize, c: usize| *t.get_main(r, c); + // BITDEC: (0, 1) → (2, 0), the recomposition unchanged. + assert_eq!(get(&traces.bitdec, 0, bitdec::cols::BITS0), FE::zero()); + assert_eq!(get(&traces.bitdec, 0, bitdec::cols::BITS0 + 1), FE::one()); + set(&mut traces.bitdec, 0, bitdec::cols::BITS0, FE::from(2u64)); + set(&mut traces.bitdec, 0, bitdec::cols::BITS0 + 1, FE::zero()); + // BALU, in program order: chosen0 = 2·(f0 − 1) + 1, chosen1 = 1, + // the point = chosen0 · chosen1. + let chosen0 = FE::from(2u64) * (f0 - FE::one()) + FE::one(); + assert_eq!(get(&traces.balu, 0, balu::cols::OUT), FE::one()); + assert_eq!(get(&traces.balu, 1, balu::cols::OUT), f1); + set(&mut traces.balu, 0, balu::cols::A, FE::from(2u64)); + set(&mut traces.balu, 0, balu::cols::OUT, chosen0); + set(&mut traces.balu, 1, balu::cols::A, FE::zero()); + set(&mut traces.balu, 1, balu::cols::OUT, FE::one()); + set(&mut traces.balu, 2, balu::cols::A, chosen0); + set(&mut traces.balu, 2, balu::cols::B, FE::one()); + set(&mut traces.balu, 2, balu::cols::OUT, chosen0); + set(&mut traces.public, 0, pubc::cols::V0, chosen0); + public[0].1[0] = chosen0; + } + match prove_traces_with_hasher( + &artifacts, + &mut traces, + &public, + &opts, + KIND, + stark::residency_mode::ResidencyMode::Retain, + ) { + Ok(proof) => Ok(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &public, + &opts, + KIND, + artifacts.chip_set, + )), + Err(e) => Err(format!("{e:?}")), + } + }; + assert_eq!( + round_trip(false), + Ok(true), + "the honest trace proves and verifies" + ); + let forged = round_trip(true); + assert_ne!( + forged, + Ok(true), + "a non-boolean index bit must be refused: {forged:?}" + ); + println!("non-boolean index bit: {forged:?}"); +} diff --git a/prover/src/lfm/whir_poly.rs b/prover/src/lfm/whir_poly.rs new file mode 100644 index 000000000..530f05178 --- /dev/null +++ b/prover/src/lfm/whir_poly.rs @@ -0,0 +1,526 @@ +//! Polynomial primitives the WHIR verifier's legs share. +//! +//! `eq` is the one everything reaches for: the GKR layer relation multiplies by +//! it once per layer, the stacked-evaluation weight is a product of two of +//! them per column, and every sumcheck closes against one. So it is worth +//! emitting once, in the cheapest shape, with its cost written down. + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, LfmBuilder}; +use super::word::{LfmWord, ext_word}; + +/// INSTRUCTIONS [`emit_eq_eval`] emits over `n` variables, as the FIRST leg in +/// its program. +/// +/// Four rows per variable and one product to fold each into the running +/// accumulator, except the first, which *is* the accumulator. Plus the interned +/// `1`, counted the way [`super::preprocessed::bitwise_preprocessed_rows`] +/// counts it — an `LFM_CONST` row, paid once per program rather than once per +/// leg (`LfmBuilder::ext_const` interns by canonical word), so a second leg in +/// the same program costs [`eq_eval_rows_again`] and this number is an upper +/// bound there. +/// +/// ⚠ The three terms sum to exactly `5n`, so a measurement of ONE leg cannot +/// tell them apart: it forces `per_variable + fold_per_variable = 5` and +/// `interned_one = fold_per_variable`, and leaves the split free. What pins the +/// split is a second leg in the same program, which pays `5n − interned_one`. +/// That is why the cost has two pins rather than one. +/// +/// Pinned against the emitter by +/// `whir_poly_tests::the_eq_leg_emits_its_closed_form` (this form) and +/// `whir_poly_tests::the_interned_one_is_paid_once_per_program` (the split). +pub const fn eq_eval_rows(n: usize) -> usize { + if n == 0 { + // `eq` over no variables is the empty product. Nothing is emitted but + // the constant it returns. + return 1; + } + INTERNED_ONE + eq_eval_rows_again(n) +} + +/// INSTRUCTIONS a FURTHER [`emit_eq_eval`] emits in a program that already has +/// one: the same rows less the constant, which is already interned. +/// +/// This is the form the assembled verifier's census adds up — it emits `eq` +/// once per GKR layer, twice per stacked column and once per sumcheck close, +/// and only the first of those pays for the `1`. +pub const fn eq_eval_rows_again(n: usize) -> usize { + if n == 0 { + return 0; + } + let per_variable = 4 * n; // r·x, 1−r, 1−x, and the MulAdd that joins them + let fold = n - 1; // one product per variable past the first + per_variable + fold +} + +/// The `LFM_CONST` row holding `1`, paid once per program. +const INTERNED_ONE: usize = 1; + +/// ★ `eq(r, x) = Π_i [ r_i·x_i + (1−r_i)(1−x_i) ]`, emitted. +/// +/// The factor is written as `(1−r)(1−x) + r·x` rather than as the sum of two +/// products, so the join is one `MulAdd` and a variable costs four rows instead +/// of five. `MulAdd` is the same one row as `Mul` on `LFM_XALU`, which is what +/// makes the shape free to choose. +/// +/// Panics if the two points differ in length — the host returns an error there, +/// but a straight-line program's lengths are emit-time constants, so a mismatch +/// is a bug in the emitter rather than a condition to carry at runtime. +pub fn emit_eq_eval(b: &mut LfmBuilder, r: &[Ext], x: &[Ext]) -> Ext { + assert_eq!( + r.len(), + x.len(), + "eq's two points must have the same number of variables" + ); + let one = b.ext_const(&FEE::one()); + let mut acc = one; + for (i, (&ri, &xi)) in r.iter().zip(x).enumerate() { + let rx = b.emul(ri, xi); + let not_r = b.esub(one, ri); + let not_x = b.esub(one, xi); + let term = b.emul_add(not_r, not_x, rx); + acc = if i == 0 { term } else { b.emul(acc, term) }; + } + acc +} + +/// INSTRUCTIONS one [`emit_sumcheck_round`] emits at degree `d`, once the +/// program has interned its constants. +/// +/// One subtract to recover `g(0)`; `d(d+1)/2` subtracts for the forward +/// difference triangle (`d` at the first level down to one at the last, and +/// every entry of it is used); `d − 1` `MulAdd`s for the Newton steps `u_1 .. +/// u_{d−1}` (`u_0` is the challenge itself and costs nothing); and `d` more for +/// the nest that folds them. So **7 rows at degree 2, 12 at degree 3, and 42 at +/// degree 7** — the degrees the verifier reaches are 3 in the GKR ladder +/// (`gkr.rs:522`), 2 in `claim_reduce` (`claim_reduce.rs:291`) and in the WHIR +/// chain (`whir_chain.rs:991`), and the worst rule's degree in the main batched +/// sumcheck (`batch.rs:431`, `degree_of`), which is table-dependent. +/// +/// Pinned as a SLOPE — one more round in the same program — by +/// `whir_poly_tests::a_sumcheck_round_costs_its_closed_form`, so the constants +/// below cancel out of it and are pinned separately. +pub const fn sumcheck_round_rows(degree: usize) -> usize { + let d = clamp_degree(degree); + let recover_g0 = 1; + let triangle = d * (d + 1) / 2; + let newton_steps = d - 1; // u_0 is the challenge; only u_1 .. u_{d−1} cost + let nest = d; // one MulAdd per level of the nest + recover_g0 + triangle + newton_steps + nest +} + +/// `LFM_CONST` rows a degree-`d` round interns, paid once per program however +/// many rounds share the degree: the pair `(1/(j+1), −j/(j+1))` for each Newton +/// step `u_1 .. u_{d−1}`. +/// +/// ⛔⛔ **NEVER SUM THIS ACROSS ROUNDS OR LEGS**, and that is not a style note — +/// it is the reason a program-level pool could not be formed from it for as long +/// as this function was the only one here. Counts ADD where values MERGE: the +/// builder interns on the canonical word, so a degree-13 round and a degree-3 +/// round in one program SHARE the steps `u_1, u_2`, and adding their counts +/// charges those four words twice. There is no scalar that composes. +/// +/// ⇒ The program-level answer is [`sumcheck_round_constants`] at the program's +/// MAXIMUM degree — see that function for why the max alone is the whole pool. +/// This form remains correct for exactly one thing: what ONE round of this +/// degree would intern in a program containing nothing else. +pub const fn sumcheck_round_consts(degree: usize) -> usize { + 2 * (clamp_degree(degree) - 1) +} + +/// ★★ THE NEWTON PAIRS A DEGREE-`d` ROUND INTERNS, BY VALUE — and, at a +/// program's maximum degree, THE WHOLE PROGRAM'S NEWTON POOL. +/// +/// ★ **THE NESTING LAW, which is what makes one call enough.** Step `j` interns +/// `(1/(j+1), −j/(j+1))` and a degree-`d` round runs steps `u_1 .. u_{d−1}`, so +/// the set for `d` is a SUBSET of the set for any larger degree. The union over +/// every round of every leg is therefore exactly the set for the largest degree +/// among them, and no summation is involved anywhere: +/// +/// ```text +/// pool(program) = sumcheck_round_constants(max over rounds of d) +/// ``` +/// +/// ⚠ **`≤ 2(d − 1)`, NOT `=`, AND THIS RETURNS A SET.** These are field +/// elements: nothing forbids `1/(a+1) == −b/(b+1)` for some pair in Goldilocks, +/// and a collision would make the true pool smaller than the count form says. +/// So the words are deduplicated here and a caller compares SETS. The identity +/// against [`sumcheck_round_consts`] is worth asserting precisely because it is +/// a claim that can fail rather than a restatement. +/// +/// ⚠ `u_0` is the challenge itself and interns nothing, which is why the range +/// starts at 1 — a form that started at 0 would name `1/1` and `−0/1`, the +/// already-interned `one` and `zero`, and over-count every program by two. +pub fn sumcheck_round_constants(degree: usize) -> Vec { + let d = clamp_degree(degree); + let mut words: Vec = Vec::new(); + for j in 1..d { + for value in newton_step_constants(j) { + let word = ext_word(&value); + if !words.contains(&word) { + words.push(word); + } + } + } + words +} + +/// The two constants Newton step `j` interns: `(1/(j+1), −j/(j+1))`. +/// +/// ★ **ONE DERIVATION, TWO CALLERS.** [`emit_newton_step`] interns exactly these +/// and [`sumcheck_round_constants`] names exactly these, so the pool a program +/// PAYS and the pool a form PREDICTS are the same expression rather than two +/// that have to be kept in step. The gap this closes existed because the only +/// form here returned a count, and a count cannot be compared against a value. +fn newton_step_constants(j: usize) -> [FEE; 2] { + let inv = FE::from((j + 1) as u64) + .inv() + .expect("j + 1 is a small nonzero Goldilocks element"); + [ + FEE::new([inv, FE::zero(), FE::zero()]), + FEE::new([ + FE::zero() - FE::from(j as u64) * inv, + FE::zero(), + FE::zero(), + ]), + ] +} + +/// `verify_rounds` clamps the degree to at least one (`sumcheck.rs:366`), so a +/// leg emitted for degree 0 must cost what degree 1 costs rather than underflow +/// the forms above. +const fn clamp_degree(degree: usize) -> usize { + if degree == 0 { 1 } else { degree } +} + +/// ★ One sumcheck round's claim recursion, as `sumcheck::verify_rounds` +/// computes it (`crypto/multilinear/src/sumcheck.rs:357-397`). +/// +/// The round polynomial travels as its evaluations at `1 .. d`; `g(0)` is not +/// sent, because `g(0) + g(1)` is the claim carried in. So the nodes are +/// `0 .. d` with values `[claim − e_0, e_0, …, e_{d−1}]`, and the new claim is +/// that polynomial at the round's challenge. +/// +/// ★ **Why Newton rather than Lagrange.** The nodes are `0 .. d`, equally +/// spaced and known at emit time, so +/// +/// ```text +/// g(r) = y_0 + u_0·(Δ¹ + u_1·(Δ² + u_2·(Δ³ + …))), u_j = (r − j)/(j + 1) +/// ``` +/// +/// with `Δ^k` the forward differences of the values. `u_0 = r`, and every other +/// `u_j` is `r·(1/(j+1)) + (−j/(j+1))` — one `MulAdd` against two interned +/// constants, not a subtract and a scale. The host's `interpolate` +/// (`sumcheck.rs:71-92`) rebuilds Lagrange denominators and batch-inverts them +/// every round; that is a host inefficiency this does not copy. Newton and +/// Lagrange are the same polynomial through the same `d + 1` points, so the two +/// agree identically — and unlike a barycentric form this never divides by +/// `r − node`, so there is no special case when a challenge lands on one. +/// +/// ⚠ **The challenge is an INPUT here: this leg neither absorbs nor draws.** +/// What the transcript replay owes it, stated so the two cannot drift: absorb +/// `e_0 … e_{d−1}` in that order, as field elements, then draw exactly one +/// challenge. Until the replay supplies it, a caller's challenge is hinted, and +/// a hinted challenge is a forgery by the arena rule (`builder.rs:617-620`: +/// never derive challenges from arenas) — test-only, never a verifier. +/// +/// The host's length check (`evaluations.len() == degree`) has no counterpart +/// here and needs none: the emitter reads exactly `d` words at fixed offsets, +/// so a proof carrying a different count has no way to be supplied. That is the +/// `epoch_verify.rs:171-179` idiom — the absence of a second value, not an +/// assert somebody could forget. +pub fn emit_sumcheck_round( + b: &mut LfmBuilder, + claim: Ext, + evaluations: &[Ext], + challenge: Ext, +) -> Ext { + let d = evaluations.len(); + assert!( + d >= 1, + "a sumcheck round sends at least one evaluation; the host clamps the degree to 1" + ); + + // `g(0)` is recovered, not sent. + let mut level = Vec::with_capacity(d + 1); + level.push(b.esub(claim, evaluations[0])); + level.extend_from_slice(evaluations); + + // The difference triangle, keeping `Δ^k y_0` — the head of every level. + let mut deltas = Vec::with_capacity(d + 1); + deltas.push(level[0]); + while level.len() > 1 { + let next: Vec = level + .windows(2) + .map(|pair| b.esub(pair[1], pair[0])) + .collect(); + deltas.push(next[0]); + level = next; + } + + // The nest, from the innermost level out. + let mut acc = deltas[d]; + for k in (0..d).rev() { + let u = if k == 0 { + challenge + } else { + emit_newton_step(b, challenge, k) + }; + acc = b.emul_add(u, acc, deltas[k]); + } + acc +} + +/// A group of rounds against a running claim, which is what every caller of +/// `sumcheck::verify_rounds` actually asks for. Returns the claim the group +/// leaves; the point is the challenges it was handed, so there is nothing to +/// return for it. +pub fn emit_sumcheck_rounds( + b: &mut LfmBuilder, + claim: Ext, + rounds: &[Vec], + challenges: &[Ext], +) -> Ext { + assert_eq!( + rounds.len(), + challenges.len(), + "a sumcheck group draws exactly one challenge per round" + ); + let mut current = claim; + for (evaluations, &r) in rounds.iter().zip(challenges) { + current = emit_sumcheck_round(b, current, evaluations, r); + } + current +} + +/// `u_j = (r − j)/(j + 1)`, one `MulAdd` against two interned constants. +fn emit_newton_step(b: &mut LfmBuilder, r: Ext, j: usize) -> Ext { + let [scale, shift] = newton_step_constants(j); + let scale = b.ext_const(&scale); + let shift = b.ext_const(&shift); + b.emul_add(r, scale, shift) +} + +/// The three kernels a shift step can read. +/// +/// `eq_j` accepts the bit unchanged, `carry` accepts `x_j = 1, y_j = 0` (the +/// only way a `+1` carries out of this bit), and `no_carry` accepts +/// `x_j = 0, y_j = 1` (the only way it does not). Named rather than indexed +/// because which of them a step reads is what its cost is made of. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum ShiftKernel { + Eq, + Carry, + NoCarry, +} + +/// One contribution a step makes: which carry state weights it, which kernel it +/// multiplies, and which carry state it lands on. +type ShiftStep = (usize, ShiftKernel, usize); + +/// The contributions a step makes, given the constant's bit and which carry +/// states can be nonzero. +/// +/// This IS the host's `match ((k >> t) & 1) + c` (`eq.rs:163-172`), read as +/// structure rather than as arithmetic: a zero bit with no carry in keeps the +/// bit equal and the carry out zero; a one bit with no carry in (or a zero bit +/// with a carry in) either lands and carries or lands and does not; a one bit +/// with a carry in must equal and carry. +fn shift_contributions(bit: usize, live: [bool; 2]) -> Vec { + let mut out = Vec::new(); + for (c, &alive) in live.iter().enumerate() { + if !alive { + continue; + } + match bit + c { + 0 => out.push((c, ShiftKernel::Eq, 0)), + 1 => { + out.push((c, ShiftKernel::NoCarry, 0)); + out.push((c, ShiftKernel::Carry, 1)); + } + _ => out.push((c, ShiftKernel::Eq, 1)), + } + } + out +} + +/// INSTRUCTIONS [`emit_shift_eval`] emits over `n` variables at shift `k`, +/// once the program has interned its `1`. +/// +/// Counted from the recursion's structure rather than from the emitter. At each +/// variable, from the LEAST significant end: +/// +/// - the kernels that step's contributions read cost `1` for `1 − x_j` if any +/// of them is `eq` or `no_carry`, `1` for `1 − y_j` if any is `eq` or +/// `carry`, then `2` for `eq` (one `Mul` and one `MulAdd`), `1` for `carry` +/// and `1` for `no_carry`; +/// - each contribution costs one row — a `Mul` for the first one landing on a +/// state and a `MulAdd` for every later one — except a contribution weighted +/// by a state that is still the literal `1`, whose first landing is the +/// kernel wire itself and costs nothing; +/// - the two states are added at the end, which the shift's wrap is the reason +/// for, unless only one of them can be nonzero. +/// +/// ★ **At `k = 0` this must be [`eq_eval_rows_again`], and that is a check and +/// not a coincidence.** With every bit zero the second carry state is never +/// reached, every step reads `eq` alone, and the recursion IS `eq`'s: four rows +/// a variable and one fold per variable past the first, `5n − 1`. The two forms +/// are derived from different code and pinned against each other. +/// +/// Bits of `k` at or above `n` are never read (`eq.rs:155`), so this is a +/// function of `k mod 2^n` — pinned in the tests rather than assumed. +pub fn shift_eval_rows(n: usize, k: usize) -> usize { + if n == 0 { + return 0; + } + let mut live = [true, false]; + let mut unit = [true, false]; + let mut rows = 0; + + for t in 0..n { + let steps = shift_contributions((k >> t) & 1, live); + let reads = |kernel: ShiftKernel| steps.iter().any(|&(_, used, _)| used == kernel); + let uses_eq = reads(ShiftKernel::Eq); + let uses_carry = reads(ShiftKernel::Carry); + let uses_no_carry = reads(ShiftKernel::NoCarry); + + rows += usize::from(uses_eq || uses_no_carry); // 1 − x_j + rows += usize::from(uses_eq || uses_carry); // 1 − y_j + rows += 2 * usize::from(uses_eq); + rows += usize::from(uses_carry); + rows += usize::from(uses_no_carry); + + let mut next_live = [false; 2]; + for &(source, _, target) in &steps { + let first = !next_live[target]; + rows += usize::from(!(first && unit[source])); + next_live[target] = true; + } + live = next_live; + // A state that has been through a step is a product of kernels, never + // the literal one again. + unit = [false, false]; + } + + rows + usize::from(live[0] && live[1]) +} + +/// ★ `shift_k(x, y)`, emitted — `multilinear::eq::shift_eval` +/// (`eq.rs:141-176`), the kernel `claim_reduce` settles a shifted column with. +/// +/// `shift_k(x, y) = 1` exactly when `index(y) = index(x) + k mod 2^n`, extended +/// multilinearly. The host adds the constant `k` bit by bit from the least +/// significant end, carrying; `state[c]` is the weight of the bits handled so +/// far having produced carry `c`, and because the shift WRAPS both carries are +/// accepted at the end. +/// +/// `k` is emit-time — it is a factor's frame offset, which is program structure +/// — so the emitter knows each bit and specialises: a step whose carry state +/// cannot be nonzero emits nothing for it, and the first step's weight is the +/// literal `1` and costs no multiply. That specialisation is why `k = 0` +/// collapses to exactly [`emit_eq_eval`]'s shape. +pub fn emit_shift_eval(b: &mut LfmBuilder, x: &[Ext], y: &[Ext], k: usize) -> Ext { + assert_eq!( + x.len(), + y.len(), + "a shift's two points must have the same number of variables" + ); + let one = b.ext_const(&FEE::one()); + let n = x.len(); + if n == 0 { + // The empty product: every index agrees with itself, at any shift. + return one; + } + + // `None` is the zero weight; `Some(None)` is the literal one, which costs no + // multiply; `Some(Some(w))` is a wire. + let mut state: [Option>; 2] = [Some(None), None]; + + for t in 0..n { + // `x.iter().zip(y).rev()`: `t` counts from the LAST variable, which is + // the least significant bit of an index. + let j = n - 1 - t; + let (xj, yj) = (x[j], y[j]); + let live = [state[0].is_some(), state[1].is_some()]; + let steps = shift_contributions((k >> t) & 1, live); + let reads = |kernel: ShiftKernel| steps.iter().any(|&(_, used, _)| used == kernel); + let uses_eq = reads(ShiftKernel::Eq); + let uses_carry = reads(ShiftKernel::Carry); + let uses_no_carry = reads(ShiftKernel::NoCarry); + + let not_x = (uses_eq || uses_no_carry).then(|| b.esub(one, xj)); + let not_y = (uses_eq || uses_carry).then(|| b.esub(one, yj)); + let eq = uses_eq.then(|| { + let xy = b.emul(xj, yj); + b.emul_add( + not_x.expect("eq reads 1 − x"), + not_y.expect("eq reads 1 − y"), + xy, + ) + }); + let carry = uses_carry.then(|| b.emul(xj, not_y.expect("carry reads 1 − y"))); + let no_carry = uses_no_carry.then(|| b.emul(not_x.expect("no_carry reads 1 − x"), yj)); + let kernel = |which: ShiftKernel| match which { + ShiftKernel::Eq => eq.expect("the step reads eq"), + ShiftKernel::Carry => carry.expect("the step reads carry"), + ShiftKernel::NoCarry => no_carry.expect("the step reads no_carry"), + }; + + let mut next: [Option; 2] = [None, None]; + for &(source, which, target) in &steps { + let value = kernel(which); + let weight = state[source].expect("a dead state makes no contribution"); + next[target] = Some(match (next[target], weight) { + // The first landing, weighted by the literal one: the kernel + // itself, with nothing to emit. + (None, None) => value, + (None, Some(w)) => b.emul(w, value), + (Some(acc), None) => b.eadd(acc, value), + (Some(acc), Some(w)) => b.emul_add(w, value, acc), + }); + } + state = [next[0].map(Some), next[1].map(Some)]; + } + + match (state[0], state[1]) { + (Some(a), Some(c)) => { + let a = a.expect("a state past the first step is a wire"); + let c = c.expect("a state past the first step is a wire"); + b.eadd(a, c) + } + (Some(only), None) | (None, Some(only)) => { + only.expect("a state past the first step is a wire") + } + (None, None) => unreachable!("some carry state survives every step"), + } +} + +/// INSTRUCTIONS [`emit_challenge_powers`] emits, once the program has interned +/// its `1`: one multiply per power past the first. +/// +/// ⚠ One FEWER than the host runs. `multilinear::challenge_powers` +/// (`lib.rs:60-69`) multiplies once per element and throws the last product +/// away, because it accumulates before returning the current power. The values +/// are identical; only the dead row is missing. +pub const fn challenge_powers_rows(count: usize) -> usize { + count.saturating_sub(1) +} + +/// ★ `[1, γ, γ², …]` — `multilinear::challenge_powers`, emitted. +/// +/// The batching weights of every batched statement in the verifier: the three +/// rules of the main sumcheck, the factor values of `claim_reduce`, and the +/// stacked evaluation's columns. It starts at ONE and not at γ, which is what +/// makes the first term of every batch free. +pub fn emit_challenge_powers(b: &mut LfmBuilder, gamma: Ext, count: usize) -> Vec { + let mut powers = Vec::with_capacity(count); + let mut acc = b.ext_const(&FEE::one()); + for i in 0..count { + if i > 0 { + acc = b.emul(acc, gamma); + } + powers.push(acc); + } + powers +} diff --git a/prover/src/lfm/whir_poly_tests.rs b/prover/src/lfm/whir_poly_tests.rs new file mode 100644 index 000000000..f6e38ba8c --- /dev/null +++ b/prover/src/lfm/whir_poly_tests.rs @@ -0,0 +1,782 @@ +//! Gates for the shared polynomial primitives. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use multilinear::eq::{eq_eval, rot_eval, shift_eval}; +use multilinear::sumcheck::{RoundProof, verify_rounds}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_poly::{ + emit_eq_eval, emit_shift_eval, emit_sumcheck_rounds, eq_eval_rows, eq_eval_rows_again, + shift_eval_rows, sumcheck_round_constants, sumcheck_round_consts, sumcheck_round_rows, +}; +use super::word::{ext_word, word_as_ext}; + +fn sample(seed: u64, n: usize) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..n).map(|_| FEE::new([next(), next(), next()])).collect() +} + +/// `eq` over `n` variables, both points hinted, the result published. +fn eq_only_program(n: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(2 * n as u32); + let r: Vec<_> = (0..n) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let x: Vec<_> = (0..n) + .map(|i| b.hint_word(arena, (n + i) as u32).as_ext()) + .collect(); + let v = emit_eq_eval(&mut b, &r, &x); + b.public(v.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the eq leg must be admissible"); + program +} + +/// The leg's marginal cost: the same program without it. +fn marginal_rows(n: usize) -> usize { + let with = eq_only_program(n); + let without = { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(2 * n as u32); + let first = b.hint_word(arena, 0).as_ext(); + for i in 1..2 * n { + let _ = b.hint_word(arena, i as u32); + } + b.public(first.as_cell()); + compile(b.finish()) + }; + with.instrs.len() - without.instrs.len() +} + +/// ★ F1 for `eq`, across the widths the verifier actually reaches: a table's +/// height in variables, a GKR layer's, and the stack's 25. +#[test] +fn the_eq_leg_emits_its_closed_form() { + for n in [1usize, 2, 4, 13, 20, 21, 25, 31] { + let measured = marginal_rows(n); + let predicted = eq_eval_rows(n); + println!("eq over {n:>2} variables: {measured:>4} rows emitted, {predicted:>4} predicted"); + assert_eq!( + measured, predicted, + "eq over {n} variables: the emitted row count must equal the closed form" + ); + } +} + +/// `legs` independent `eq` legs over their own hinted points, and the rows they +/// emit BETWEEN them — the program's total less the `2n + 1` hints and public +/// each leg's plumbing contributes, which is countable by construction rather +/// than by differencing against another program. +fn legs_rows(n: usize, legs: usize) -> usize { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena((2 * n * legs) as u32); + for leg in 0..legs { + let base = (2 * n * leg) as u32; + let r: Vec<_> = (0..n) + .map(|i| b.hint_word(arena, base + i as u32).as_ext()) + .collect(); + let x: Vec<_> = (0..n) + .map(|i| b.hint_word(arena, base + (n + i) as u32).as_ext()) + .collect(); + let v = emit_eq_eval(&mut b, &r, &x); + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("a multi-leg eq program must be admissible"); + program.instrs.len() - legs * (2 * n + 1) +} + +/// ★ The second pin on the cost, and the reason there are two. +/// +/// `eq_eval_rows(n)` is `5n` at every width, so the single-leg measurement in +/// [`the_eq_leg_emits_its_closed_form`] forces only the SUM of the form's +/// terms. A second leg in the same program separates them: it pays no second +/// constant, so it costs `5n − 1`. Between the two measurements all three terms +/// are determined, and a form that moved a row from the constant into the +/// per-variable count would pass the first pin and fail this one. +/// +/// It also carries a control on the first pin's method — `legs_rows(n, 1)` +/// counts the leg directly, where `marginal_rows(n)` differences two programs, +/// and the two agree. +#[test] +fn the_interned_one_is_paid_once_per_program() { + for n in [1usize, 4, 20, 25] { + let one_leg = legs_rows(n, 1); + let two_legs = legs_rows(n, 2); + let second = two_legs - one_leg; + println!( + "eq over {n:>2} variables: first leg {one_leg:>4} rows emitted, \ + {:>4} predicted; second leg {second:>4} rows emitted, {:>4} predicted", + eq_eval_rows(n), + eq_eval_rows_again(n) + ); + assert_eq!( + one_leg, + eq_eval_rows(n), + "eq over {n} variables: counted directly, the first leg must equal the closed form" + ); + assert_eq!( + one_leg, + marginal_rows(n), + "eq over {n} variables: the direct count and the differenced count must agree" + ); + assert_eq!( + second, + eq_eval_rows_again(n), + "eq over {n} variables: a second leg must pay every row but the interned constant" + ); + } +} + +/// ★ The zero-variable entry, which the assembled census REACHES rather than +/// avoids: `gkr_layer_rows(i)` adds `eq_eval_rows_again(i)`, so the ladder's +/// layer 0 asks for `eq` over no variables. Both cost branches (`= 1` for a +/// first leg, `= 0` for a further one) and the value are gated here; the loops +/// above start at one variable and could not have. +/// +/// The value side is against `eq_eval(&[], &[])` — the host's own empty +/// product, not the emitter's claim about it. +#[test] +fn eq_over_no_variables_is_the_empty_product() { + let one_leg = legs_rows(0, 1); + let second = legs_rows(0, 2) - one_leg; + println!( + "eq over 0 variables: first leg {one_leg} rows emitted, {} predicted; \ + second leg {second} rows emitted, {} predicted", + eq_eval_rows(0), + eq_eval_rows_again(0) + ); + assert_eq!( + one_leg, + eq_eval_rows(0), + "the empty product costs exactly the constant it returns" + ); + assert_eq!( + second, + eq_eval_rows_again(0), + "a further empty product costs nothing: the constant is already interned" + ); + + let program = eq_only_program(0); + let exec = execute(&program, &[Vec::new()], &crate::hash_pin::BLOCK_HASHER) + .expect("the eq leg executes over an empty point"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published extension value"); + let want = eq_eval::(&[], &[]).expect("the host agrees on the width"); + assert_eq!( + got, want, + "eq over no variables: the emitted leg and the host disagree on the empty product" + ); +} + +/// ★ The leg computes what `multilinear::eq::eq_eval` computes — the function +/// the verifier calls, not a restatement of the emitter's own algebra. +#[test] +fn the_eq_leg_computes_what_the_host_computes() { + for n in [1usize, 4, 20] { + let program = eq_only_program(n); + for seed in [0x11u64, 0x22, 0x33] { + let r = sample(seed, n); + let x = sample(seed ^ 0xFFFF, n); + let arenas = vec![r.iter().chain(x.iter()).map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the eq leg executes"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published extension value"); + let want = eq_eval(&r, &x).expect("the host agrees on the width"); + assert_eq!( + got, want, + "eq over {n} variables at seed {seed:#x}: the emitted leg and \ + the host disagree" + ); + } + } +} + +/// `eq` is an indicator on the cube: one when the two points are equal corners, +/// zero otherwise. A leg that got the factor backwards still agrees with a +/// host that got it backwards the same way; this does not. +#[test] +fn eq_is_one_on_the_diagonal_and_zero_off_it() { + let n = 6; + let program = eq_only_program(n); + let corner = |mask: usize| -> Vec { + (0..n) + .map(|i| { + let bit = (mask >> i) & 1; + FEE::new([FE::from(bit as u64), FE::zero(), FE::zero()]) + }) + .collect() + }; + let run = |r: &[FEE], x: &[FEE]| -> FEE { + let arenas = vec![r.iter().chain(x.iter()).map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER).expect("executes"); + word_as_ext(&exec.public_words[0].1).expect("a published extension value") + }; + for mask in [0usize, 1, 0b101010, 0b111111] { + let c = corner(mask); + assert_eq!( + run(&c, &c), + FEE::one(), + "eq must be one where the corners agree ({mask:#08b})" + ); + let other = corner(mask ^ 0b000100); + assert_eq!( + run(&c, &other), + FEE::zero(), + "eq must be zero where the corners differ ({mask:#08b})" + ); + } +} + +/// A group of `rounds` sumcheck rounds at `degree`, everything hinted: the +/// claim carried in at word 0, then each round's `degree` evaluations followed +/// by its challenge. The claim the group leaves is published. +/// +/// ⚠ Hinting the challenges is what makes this a TEST and not a verifier — the +/// arena rule forbids deriving a challenge from an arena. The replay supplies +/// them in the assembled program. +fn sumcheck_program(degree: usize, rounds: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(sumcheck_arena_words(degree, rounds) as u32); + let claim = b.hint_word(arena, 0).as_ext(); + let mut evaluations = Vec::with_capacity(rounds); + let mut challenges = Vec::with_capacity(rounds); + for round in 0..rounds { + let base = (1 + round * (degree + 1)) as u32; + evaluations.push( + (0..degree) + .map(|j| b.hint_word(arena, base + j as u32).as_ext()) + .collect::>(), + ); + challenges.push(b.hint_word(arena, base + degree as u32).as_ext()); + } + let out = emit_sumcheck_rounds(&mut b, claim, &evaluations, &challenges); + b.public(out.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the sumcheck leg must be admissible"); + program +} + +/// The claim, then `degree` evaluations and one challenge per round. +fn sumcheck_arena_words(degree: usize, rounds: usize) -> usize { + 1 + rounds * (degree + 1) +} + +/// The leg's rows: the program's total less the hints and the one public, which +/// are countable by construction. +fn sumcheck_leg_rows(degree: usize, rounds: usize) -> usize { + let program = sumcheck_program(degree, rounds); + program.instrs.len() - sumcheck_arena_words(degree, rounds) - 1 +} + +/// ★ F1 for the sumcheck round, as a SLOPE: one more round in the same program +/// costs exactly `sumcheck_round_rows(degree)`, with the interned constants +/// cancelling out of the difference. The intercept pins those constants. +#[test] +fn a_sumcheck_round_costs_its_closed_form() { + for degree in [1usize, 2, 3, 5, 7] { + for rounds in [1usize, 2, 5] { + let slope = sumcheck_leg_rows(degree, rounds + 1) - sumcheck_leg_rows(degree, rounds); + let predicted = sumcheck_round_rows(degree); + println!( + "sumcheck degree {degree}, round {} → {}: {slope:>3} rows emitted, \ + {predicted:>3} predicted", + rounds, + rounds + 1 + ); + assert_eq!( + slope, predicted, + "degree {degree}: one more round must cost the closed form" + ); + } + let first = sumcheck_leg_rows(degree, 1); + let predicted = sumcheck_round_rows(degree) + sumcheck_round_consts(degree); + println!( + "sumcheck degree {degree}, first round: {first:>3} rows emitted, \ + {predicted:>3} predicted (round + interned constants)" + ); + assert_eq!( + first, predicted, + "degree {degree}: the first round pays for its Newton constants too" + ); + } +} + +/// ★ The degree-0 entry, which is the caller's degree and not a shape the +/// emitter can be handed. +/// +/// `verify_rounds` clamps with `degree.max(1)` (`sumcheck.rs:366`), so a rule +/// whose degree is zero is verified at degree ONE and its round proof carries +/// one evaluation. A census adding up `sumcheck_round_rows(degree_of(rules))` +/// holds the RAW degree, so the cost form has to be callable there — and +/// `clamp_degree` is only right if it matches the host's clamp. Nothing above +/// can see that: the loops start at one and the emitter refuses zero +/// evaluations outright. +/// +/// Three observations, each able to fail: the host ACCEPTS a one-evaluation +/// round at degree 0 (the clamp, watched rather than assumed); it REJECTS a +/// two-evaluation one, which is what says the clamp is a clamp and not an +/// unchecked pass; and the cost form at the raw 0 predicts the program emitted +/// at the width the clamp implies. +#[test] +fn a_degree_zero_rule_is_verified_at_degree_one() { + let rounds = 3; + let values = sample(0x0D, 1 + rounds); + let claim = values[0]; + let proof: Vec> = (0..rounds) + .map(|round| RoundProof { + evaluations: vec![values[1 + round]], + }) + .collect(); + + let mut transcript = DefaultTranscript::::new(b"v1-sumcheck-degree0"); + let host = verify_rounds(&proof, claim, 0, &mut transcript) + .expect("asked for degree 0, the host clamps to 1 and accepts one evaluation a round"); + assert_eq!(host.point.len(), rounds, "one challenge per round"); + + let mut wider = DefaultTranscript::::new(b"v1-sumcheck-degree0"); + let two: Vec> = (0..rounds) + .map(|round| RoundProof { + evaluations: vec![values[1 + round], values[1 + round]], + }) + .collect(); + assert!( + verify_rounds(&two, claim, 0, &mut wider).is_err(), + "the clamp is a clamp: at degree 0 a two-evaluation round must be rejected, or \ + the acceptance above says nothing about the width" + ); + + // The cost form at the RAW degree, against the program emitted at the width + // the clamp implies. + let slope = sumcheck_leg_rows(1, rounds + 1) - sumcheck_leg_rows(1, rounds); + let first = sumcheck_leg_rows(1, 1); + println!( + "sumcheck degree 0 (clamped to 1): {slope} rows a round emitted, \ + {} predicted; first round {first}, {} predicted", + sumcheck_round_rows(0), + sumcheck_round_rows(0) + sumcheck_round_consts(0) + ); + assert_eq!( + slope, + sumcheck_round_rows(0), + "a degree-0 rule's round must cost what the clamped width emits" + ); + assert_eq!( + first, + sumcheck_round_rows(0) + sumcheck_round_consts(0), + "and its constants must be the clamped width's constants" + ); + + // The value, against the host's own claim. + let mut arena = vec![ext_word(&claim)]; + for (round, r) in host.point.iter().enumerate() { + arena.extend(proof[round].evaluations.iter().map(ext_word)); + arena.push(ext_word(r)); + } + let program = sumcheck_program(1, rounds); + let exec = execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER) + .expect("the sumcheck leg executes at the clamped width"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published claim"); + assert_eq!( + got, host.expected_evaluation, + "degree 0 over {rounds} rounds: the emitted leg and the host disagree" + ); +} + +/// ★ The leg computes what `sumcheck::verify_rounds` computes, driven by that +/// function's OWN challenges. +/// +/// The host is run first over a real transcript on arbitrary evaluations — it +/// never rejects, so any values are a legal input, and generic values exercise +/// the recursion harder than a satisfiable proof would. The challenges it +/// sampled come back as `SumcheckClaim::point`; those are hinted into the +/// machine, and the machine's final claim is compared against the host's +/// `expected_evaluation`. Nothing here compares the emitter against my own +/// interpolation. +#[test] +fn the_sumcheck_leg_computes_what_the_host_computes() { + for degree in [1usize, 2, 3, 5, 7] { + for rounds in [1usize, 4] { + for seed in [0xA5u64, 0x5A, 0xC3] { + let values = sample(seed, 1 + rounds * degree); + let claim = values[0]; + let proof: Vec> = (0..rounds) + .map(|round| RoundProof { + evaluations: values[1 + round * degree..1 + (round + 1) * degree].to_vec(), + }) + .collect(); + + let mut transcript = + DefaultTranscript::::new(b"v1-sumcheck-gate"); + let host = verify_rounds(&proof, claim, degree, &mut transcript) + .expect("verify_rounds accepts a well-shaped group"); + assert_eq!(host.point.len(), rounds, "one challenge per round"); + + let mut arena = vec![ext_word(&claim)]; + for (round, r) in host.point.iter().enumerate() { + arena.extend(proof[round].evaluations.iter().map(ext_word)); + arena.push(ext_word(r)); + } + let program = sumcheck_program(degree, rounds); + let exec = execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER) + .expect("the sumcheck leg executes"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published claim"); + assert_eq!( + got, host.expected_evaluation, + "degree {degree}, {rounds} rounds, seed {seed:#x}: the emitted leg and \ + the host disagree on the claim the group leaves" + ); + } + } + } +} + +/// `shift_k` over `n` variables, both points hinted, the result published. +fn shift_only_program(n: usize, k: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(2 * n as u32); + let x: Vec<_> = (0..n) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let y: Vec<_> = (0..n) + .map(|i| b.hint_word(arena, (n + i) as u32).as_ext()) + .collect(); + let v = emit_shift_eval(&mut b, &x, &y, k); + b.public(v.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the shift leg must be admissible"); + program +} + +/// The shift leg's marginal cost, measured the way the `eq` leg's is: the same +/// program without it. +fn shift_marginal_rows(n: usize, k: usize) -> usize { + let with = shift_only_program(n, k); + let without = { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(2 * n as u32); + let first = b.hint_word(arena, 0).as_ext(); + for i in 1..2 * n { + let _ = b.hint_word(arena, i as u32); + } + b.public(first.as_cell()); + compile(b.finish()) + }; + with.instrs.len() - without.instrs.len() +} + +/// The shifts `claim_reduce` reaches, and the ones that exercise the carry. +/// +/// A VM table's factors are read at frame offsets — 0 and 1 in the tables this +/// campaign proves — but the kernel takes any `k`, and a form that only ever +/// saw 0 and 1 would never see a carry ripple past one bit. +const SHIFTS: [(usize, usize); 12] = [ + (1, 0), + (1, 1), + (4, 0), + (4, 1), + (4, 2), + (4, 5), + (4, 15), + (7, 0), + (7, 1), + (7, 3), + (7, 64), + (7, 127), +]; + +/// ★ F1 for the shift kernel, against a form derived from the recursion rather +/// than from the emitter. +#[test] +fn the_shift_leg_emits_its_closed_form() { + for (n, k) in SHIFTS { + let measured = shift_marginal_rows(n, k); + // The leg's own rows, plus the `LFM_CONST` holding `1` — the same + // accounting `eq_eval_rows` uses, where the constant is paid once per + // program and `shift_eval_rows` is the second-leg form. + let predicted = shift_eval_rows(n, k) + 1; + println!("shift n={n} k={k}: {measured} rows emitted, {predicted} predicted"); + assert_eq!(measured, predicted, "shift over {n} variables at k={k}"); + } +} + +/// ★★ At `k = 0` the shift IS `eq`, in VALUE and in COST, and the two forms +/// were derived from different code. +/// +/// The second carry state is never reached with every bit zero, so the +/// recursion collapses to `eq`'s: this pins `shift_eval_rows(n, 0)` against +/// `eq_eval_rows_again(n)`, a number this module pinned another way entirely +/// (against a second leg in one program, to fix the split of `5n`). +#[test] +fn the_shift_form_is_eqs_at_a_zero_shift() { + for n in 1..=12 { + assert_eq!( + shift_eval_rows(n, 0), + eq_eval_rows_again(n), + "a zero shift is eq over {n} variables, and must cost exactly that" + ); + } +} + +/// ★ The shift leg against the host, at every shift in [`SHIFTS`]. +/// +/// `shift_eval` is the function `claim_reduce` settles a shifted column with, +/// and it is the only kernel in the per-table verify whose answer depends on a +/// constant the AIR chose rather than on a challenge. +#[test] +fn the_shift_leg_computes_what_the_host_computes() { + for (n, k) in SHIFTS { + let program = shift_only_program(n, k); + for seed in [0x11u64, 0x5eed] { + let x = sample(seed, n); + let y = sample(seed ^ 0xFFFF, n); + let arenas = vec![x.iter().chain(y.iter()).map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the shift leg executes"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published extension value"); + let want = shift_eval(&x, &y, k).expect("the host agrees on the width"); + assert_eq!(got, want, "shift_{k} over {n} variables at seed {seed:#x}"); + if k == 0 { + assert_eq!(want, eq_eval(&x, &y).expect("eq agrees"), "shift_0 is eq"); + } + if k == 1 { + assert_eq!( + want, + rot_eval(&x, &y).expect("rot agrees"), + "shift_1 is rot" + ); + } + } + } +} + +/// ★ The shift is an indicator on the cube, and that is what says the kernel +/// counts in the right DIRECTION. +/// +/// A leg that added the constant the other way round agrees with the host's own +/// arithmetic at random points only if it is wrong in the same direction; the +/// corners say which way it goes. `index(y) = index(x) + k mod 2^n`, with the +/// index read big-endian — variable 0 is the most significant, which is why the +/// recursion walks the variables in reverse. +#[test] +fn the_shift_is_one_exactly_where_the_index_advances() { + let n = 4; + let size = 1usize << n; + for k in [0usize, 1, 3, 13] { + let program = shift_only_program(n, k); + let corner = |index: usize| -> Vec { + (0..n) + .map(|bit| { + if index >> (n - 1 - bit) & 1 == 1 { + FEE::one() + } else { + FEE::zero() + } + }) + .collect() + }; + for from in 0..size { + for to in 0..size { + let x = corner(from); + let y = corner(to); + let arenas = vec![x.iter().chain(y.iter()).map(ext_word).collect::>()]; + let exec = execute(&program, &arenas, &crate::hash_pin::BLOCK_HASHER) + .expect("the shift leg executes"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published value"); + let want = if to == (from + k) % size { + FEE::one() + } else { + FEE::zero() + }; + assert_eq!(got, want, "shift_{k}: corner {from} to corner {to}"); + } + } + } +} + +/// Bits of `k` at or above `n` are never read, so the kernel is a function of +/// `k mod 2^n` — the host's loop only ever asks for `t < n` (`eq.rs:155`). +/// +/// `claim_reduce` is handed a factor's raw offset, and `materialize` reduces it +/// on the prover's side (`claim_reduce.rs:357`); this is what says the two +/// agree without the verifier reducing anything. +#[test] +fn the_shift_reads_only_the_bits_it_has_variables_for() { + let n = 4; + let size = 1usize << n; + for k in [0usize, 3, 9] { + assert_eq!(shift_eval_rows(n, k), shift_eval_rows(n, k + size)); + let x = sample(0xA1, n); + let y = sample(0xB2, n); + assert_eq!( + shift_eval(&x, &y, k).expect("the host agrees"), + shift_eval(&x, &y, k + 3 * size).expect("the host agrees"), + ); + } +} + +// ============================================================================= +// The Newton pool — the constants a program's sumchecks intern +// ============================================================================= + +/// The Newton pairs for degree `d`, computed a SECOND WAY. +/// +/// ⛔ NOT `newton_step_constants`, and not its spelling either. That helper and +/// the emitter now share one derivation on purpose, so a test comparing them +/// would be agreeing with itself — the oracle-shares-the-function trap. This +/// works in the EXTENSION field throughout (`1/(j+1)` and `−j·(1/(j+1))` as +/// extension elements) where the production form embeds a BASE-field inverse, +/// so the two routes meet only if the arithmetic is right. +fn newton_pairs_second_source(degree: usize) -> Vec { + let d = degree.max(1); + let mut out = Vec::new(); + for j in 1..d { + let scale = FEE::from((j + 1) as u64) + .inv() + .expect("j + 1 is nonzero in the extension too"); + let shift = FEE::zero() - FEE::from(j as u64) * scale; + out.push(scale); + out.push(shift); + } + out +} + +/// Every `LFM_CONST` value a program holds, in order. +fn const_values(program: &LfmProgram) -> Vec<[FE; 4]> { + program + .instrs + .iter() + .filter_map(|i| match i { + super::instr::Instr::Const { value, .. } => Some(*value), + _ => None, + }) + .collect() +} + +/// ★★ THE VALUES FORM NAMES EXACTLY WHAT THE ROUND INTERNS — as a DELTA against +/// degree one, which runs no Newton step at all and so isolates them. +/// +/// This is the gate the pool needed and did not have: `sumcheck_round_consts` +/// counted these words and a count cannot be compared against a value, so the +/// program-level pool carried them as an unnamed remainder. +#[test] +fn the_values_form_names_what_a_sumcheck_round_interns() { + for degree in [2usize, 3, 5, 7, 13] { + let leg = const_values(&sumcheck_program(degree, 1)); + let base = const_values(&sumcheck_program(1, 1)); + let added: Vec<[FE; 4]> = leg.iter().filter(|w| !base.contains(w)).copied().collect(); + + let named = sumcheck_round_constants(degree); + let expected: Vec<[FE; 4]> = newton_pairs_second_source(degree) + .iter() + .map(ext_word) + .collect(); + + println!( + "degree {degree}: {} words interned beyond degree 1, {} named by the form, {} from the second source", + added.len(), + named.len(), + expected.len(), + ); + // ⛔ ANTI-VACUITY: degree 1 runs no step, so a degree that added nothing + // would make every comparison below trivially true. + assert!( + !added.is_empty(), + "degree {degree} interned no constant beyond degree 1, so this arm compares three empty sets" + ); + for word in &expected { + assert!( + added.contains(word), + "degree {degree}: a Newton word the definition requires is not interned" + ); + assert!( + named.contains(word), + "degree {degree}: a Newton word the definition requires is not named" + ); + } + assert_eq!( + added.len(), + named.len(), + "degree {degree}: the form names a different number of words than the round interns beyond degree one" + ); + } +} + +/// ★★ THE NESTING LAW, which is what lets ONE call at the maximum degree be a +/// whole program's pool: the set for `d` is a subset of the set for any larger +/// degree, so a union over a mixed set of degrees IS the maximum's set. +/// +/// ⛔ This is the property the cross-epoch F1 rests on. Without it a pool would +/// have to be accumulated round by round, and `sumcheck_round_consts` — a count +/// — would still be useless for it. +#[test] +fn the_newton_pool_of_a_mixed_program_is_its_maximum_degrees_set() { + let degrees = [1usize, 2, 3, 2, 7, 5, 13, 3]; + let max = *degrees.iter().max().expect("a non-empty set"); + + let mut union: Vec<[FE; 4]> = Vec::new(); + for d in degrees { + for word in sumcheck_round_constants(d) { + if !union.contains(&word) { + union.push(word); + } + } + } + let at_max = sumcheck_round_constants(max); + println!( + "degrees {degrees:?}: union {} words, max {max} alone {} words", + union.len(), + at_max.len() + ); + + // ⛔ ANTI-VACUITY on the SHAPE of the test: a set of equal degrees would make + // the union trivially the max's, and the law would go untested. + assert!( + degrees.iter().any(|d| *d < max), + "every degree equals the maximum, so this arm tests nothing" + ); + assert_eq!( + union.len(), + at_max.len(), + "the union over a mixed degree set is not the maximum's set, so the nesting law the pool rests on does not hold" + ); + for word in &at_max { + assert!( + union.contains(word), + "a word of the maximum's set is not in the union" + ); + } +} + +/// ★ THE TWO FORMS AGREE — the values form's LENGTH against the count form. +/// +/// ⚠ It can genuinely fail, which is why it is worth running over a range. The +/// values are field elements and nothing forbids `1/(a+1) == −b/(b+1)` for some +/// pair in Goldilocks; the values form deduplicates and would come back SHORT, +/// and that collision is a fact worth discovering here rather than as an +/// unexplained two-word gap in a program's pool. +#[test] +fn the_newton_values_form_agrees_with_the_count_form() { + for degree in 1usize..=20 { + let named = sumcheck_round_constants(degree); + assert_eq!( + named.len(), + sumcheck_round_consts(degree), + "degree {degree}: the values form deduplicated to {} words where the count form says {} — two Newton constants collide in Goldilocks at this degree, which is a finding and not a rounding", + named.len(), + sumcheck_round_consts(degree), + ); + } + println!("the Newton pairs are distinct for every degree through 20"); +} diff --git a/prover/src/lfm/whir_program.rs b/prover/src/lfm/whir_program.rs new file mode 100644 index 000000000..171fd760f --- /dev/null +++ b/prover/src/lfm/whir_program.rs @@ -0,0 +1,145 @@ +//! The constraint DAG as a machine leg: `multilinear::program::Program`, +//! emitted. +//! +//! Every table's constraint argument reduces, at the end, to evaluating one +//! straight-line program at the point its sumcheck left — `Program::eval` +//! (`crypto/multilinear/src/program.rs:52-71`). It is walked ONCE per table, +//! not once per round and not once per constraint, which is why the sizing +//! note's census puts the whole term at a few percent of a wrap. +//! +//! ⚠ **This is not `constraints::emit_analyzed` re-aimed**, which is what my +//! sizing note §5 recommended. That emitter consumes an `Analysis` over the +//! UNIVARIATE `Op`, whose `Var` carries `{main, offset, col}` and which arrives +//! with liveness, folding and `MulAdd` fusion already computed. The multilinear +//! `Program` is a different and much smaller type: six variants over a flat +//! `steps` vector with a `root`. So this walks it directly. +//! +//! ⚠ **One row per operation node, and the number that costs.** The univariate +//! emitter fuses a `Mul` feeding a single-use `Add` into one `MulAdd` and is +//! MEASURED at 0.75–0.87 rows per IR node with folding and DCE. This sits at +//! 1.0. The term is small — the note prices the whole constraint DAG at ~70k +//! rows for all 34 sub-proofs plus ~15k for the two bus programs, 1.6% of +//! today's wrap — so the fusion is worth roughly 0.3% of an epoch. It is left +//! as a named lever rather than taken here, because its closed form would have +//! to count fusions the same way the emitter does, which turns the row pin into +//! a round trip on exactly the term the optimisation moves. Doing it properly +//! means an independent fusion count, which is a leg of its own. + +use multilinear::program::{Op, Program}; + +use crate::tables::types::{FEE, GoldilocksExtension}; + +use super::builder::{Ext, LfmBuilder}; + +/// INSTRUCTIONS [`emit_program`] emits for `program`. +/// +/// One `LFM_XALU` row per operation node — `Add`, `Sub`, `Mul` and `Neg` — plus +/// one `LFM_CONST` row per DISTINCT `Fixed` value and one more for the zero a +/// `Neg` subtracts from, when the program has any. `Var` costs nothing: it is +/// already a wire. +/// +/// Countable straight off `steps()` without simulating the walk, which is what +/// makes the pin independent of the emitter rather than a restatement of it. +pub fn program_rows(program: &Program) -> usize { + steps_rows(program.steps()) +} + +/// The same count over a DAG's steps alone. +/// +/// Split out because a caller that reads SEVERAL roots out of one DAG — the +/// zerocheck rule batches one per constraint root — has no single `Program` to +/// hand over, and counting it a second way would be a second form to keep in +/// step. +pub fn steps_rows(steps: &[Op]) -> usize { + let mut operations = 0; + let mut negates = false; + let mut constants: Vec = Vec::new(); + for step in steps { + match step { + Op::Fixed(value) => { + if !constants.contains(value) { + constants.push(*value); + } + } + Op::Var(_) => {} + Op::Add(_, _) | Op::Sub(_, _) | Op::Mul(_, _) => operations += 1, + Op::Neg(_) => { + operations += 1; + negates = true; + } + } + } + // The zero a `Neg` subtracts from is interned like any other constant, so a + // program that already carries `Fixed(0)` would pay for it once. + // + // ⚠ MEASURED: that guard is unreachable with the host's own `Builder`, which + // folds identities at CONSTRUCTION — `sub(x, zero)` returns `x` and + // `add(x, zero)` returns `x`, so a `Fixed(0)` never becomes a step. It is + // kept because the form should be right about the type rather than about + // the one producer, and it is written down as unexercised rather than left + // looking like a tested branch. + // + // ⚠ The same measurement retires a mutation this leg was pre-registered + // with. Emitting `Neg` as a multiply by an interned `−1` instead of a + // subtraction from an interned zero is EQUIVALENT in value and in cost — + // one row against one constant either way — on every program this builder + // can produce, because the only shape that could tell them apart is one + // holding `Fixed(0)`, and it folds away. It is not a mutation; it is a + // choice with no observable difference. + let zero_for_neg = usize::from(negates && !constants.contains(&FEE::zero())); + operations + constants.len() + zero_for_neg +} + +/// ★ `Program::eval`, emitted. +/// +/// The walk is the host's, step for step: each step's value is kept as a wire, +/// `Var(i)` reads `values[i]`, and the answer is the wire at `root()`. `Neg` is +/// `0 − a` rather than a multiply by `−1`, so it costs the same one row without +/// a second constant. +/// +/// Panics if a step reads a slot the caller did not supply — the host indexes +/// `values` directly and would panic too, but here the lengths are emit-time +/// constants, so a mismatch is a bug in the caller rather than a condition to +/// carry at runtime. +pub fn emit_program( + b: &mut LfmBuilder, + program: &Program, + values: &[Ext], +) -> Ext { + let wires = emit_steps(b, program.steps(), values); + wires[program.root() as usize] +} + +/// Every step's value as a wire, in step order. +/// +/// [`emit_program`] is this and then one index. The zerocheck rule needs the +/// whole vector, because it reads one node per constraint root out of a single +/// DAG and batches them. +pub fn emit_steps( + b: &mut LfmBuilder, + steps: &[Op], + values: &[Ext], +) -> Vec { + let mut wires: Vec = Vec::with_capacity(steps.len()); + let mut zero: Option = None; + for (index, step) in steps.iter().enumerate() { + let wire = match *step { + Op::Fixed(value) => b.ext_const(&value), + Op::Var(slot) => *values.get(slot as usize).unwrap_or_else(|| { + panic!( + "step {index} reads factor {slot}, but only {} were supplied", + values.len() + ) + }), + Op::Add(a, c) => b.eadd(wires[a as usize], wires[c as usize]), + Op::Sub(a, c) => b.esub(wires[a as usize], wires[c as usize]), + Op::Mul(a, c) => b.emul(wires[a as usize], wires[c as usize]), + Op::Neg(a) => { + let unit = *zero.get_or_insert_with(|| b.ext_const(&FEE::zero())); + b.esub(unit, wires[a as usize]) + } + }; + wires.push(wire); + } + wires +} diff --git a/prover/src/lfm/whir_program_tests.rs b/prover/src/lfm/whir_program_tests.rs new file mode 100644 index 000000000..9b66e01c2 --- /dev/null +++ b/prover/src/lfm/whir_program_tests.rs @@ -0,0 +1,194 @@ +//! Gates for the constraint-DAG leg. + +use multilinear::program::{Builder, Program}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +use super::builder::{Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_program::{emit_program, program_rows}; +use super::word::{ext_word, word_as_ext}; + +fn fee(v: u64) -> FEE { + FEE::new([ + FE::from(v.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 9), + FE::from(v ^ 0x5A5A), + FE::from(v.wrapping_add(11)), + ]) +} + +/// One shape under test: a name, the program, and how many factors it reads. +struct Shape { + name: &'static str, + program: Program, + factors: usize, +} + +/// Programs with the structure a real constraint DAG has: shared +/// subexpressions, a constant that appears twice, a negation, a degree +/// stack, and one that `simplify` has been through. +fn shapes() -> Vec { + let mut out = Vec::new(); + + // A single factor read straight to the root: no operation at all. + let mut b = Builder::::new(); + let v = b.var(0); + out.push(Shape { + name: "one factor, no work", + program: b.finish(v).expect("root is a step"), + factors: 1, + }); + + // A constant used twice, a negation, and a shared subexpression. + let mut b = Builder::::new(); + let x = b.var(0); + let y = b.var(1); + let k = b.fixed(fee(7)); + let k_again = b.fixed(fee(7)); + let shared = b.mul(x, y); + let left = b.add(shared, k); + let right = b.sub(shared, k_again); + let negated = b.neg(right); + let root = b.mul(left, negated); + out.push(Shape { + name: "shared subexpression, repeated constant, a negation", + program: b.finish(root).expect("root is a step"), + factors: 2, + }); + + // A degree stack: the shape a high-degree rule takes. + let mut b = Builder::::new(); + let mut acc = b.var(0); + for slot in 1..6 { + let next = b.var(slot); + let scaled = b.mul(acc, next); + let shifted = b.fixed(fee(slot as u64 * 13)); + acc = b.add(scaled, shifted); + } + let root = b.neg(acc); + out.push(Shape { + name: "a degree stack with six factors", + program: b.finish(root).expect("root is a step"), + factors: 6, + }); + + // ★ An attempt to build a program holding `Fixed(0)` beside a `Neg` — the + // only shape that could tell `0 − a` from `(−1)·a`. + // + // ⚠ It cannot be built: the printed step count is TWO, not five, because the + // host's `Builder` folds identities at construction and the zero never + // becomes a step. The shape is kept for exactly that reading — it pins that + // the program the verifier evaluates carries no `Fixed(0)`, which is what + // makes the two spellings of a negation indistinguishable and retires a + // mutation this leg was pre-registered with. + let mut b = Builder::::new(); + let x = b.var(0); + let z = b.fixed(FEE::zero()); + let shifted = b.sub(x, z); + let negated = b.neg(shifted); + let root = b.add(negated, z); + out.push(Shape { + name: "a zero the builder folds away, and a negation", + program: b.finish(root).expect("root is a step"), + factors: 1, + }); + + // The host's own simplification of each, which is the form the verifier + // actually evaluates. + // + // ⚠ MEASURED, not assumed: `simplify` leaves BOTH of these at the step count + // they already have — the builder interns a repeated `Fixed` before + // `simplify` ever sees it, and neither program has a dead step, an add of + // zero or a multiply by one. An earlier version of this comment claimed the + // repeated constant was deduped here; the printed step counts say otherwise. + // What these two shapes pin, then, is that an already-minimal program stays + // minimal through `simplify` and that the leg costs the same either way — + // which is worth pinning, because the verifier evaluates the simplified + // form and the census is taken on the unsimplified one. + let simplified: Vec = out + .iter() + .skip(1) + .map(|shape| Shape { + name: shape.name, + program: shape.program.simplify(), + factors: shape.factors, + }) + .collect(); + for (index, shape) in simplified.into_iter().enumerate() { + out.push(Shape { + name: match index { + 0 => "the repeated constant, simplified by the host", + 1 => "the stack, simplified by the host", + _ => "the folded zero, simplified by the host", + }, + program: shape.program, + factors: shape.factors, + }); + } + + out +} + +/// The program's factors hinted, its root published. +fn program_of(shape: &Shape) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(shape.factors as u32); + let values: Vec = (0..shape.factors) + .map(|slot| b.hint_word(arena, slot as u32).as_ext()) + .collect(); + let root = emit_program(&mut b, &shape.program, &values); + b.public(root.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the constraint leg must be admissible"); + program +} + +/// ★ F1: every row named, and the prediction read off the HOST's program rather +/// than off the emitter's walk. +#[test] +fn the_constraint_leg_emits_its_closed_form() { + for shape in shapes() { + let emitted = program_of(&shape); + let measured = emitted.instrs.len() - shape.factors - 1; + let predicted = program_rows(&shape.program); + println!( + "constraint [{}]: {measured:>3} rows emitted, {predicted:>3} predicted \ + ({} steps)", + shape.name, + shape.program.steps().len() + ); + assert_eq!( + measured, predicted, + "{}: the emitted row count must equal the closed form", + shape.name + ); + } +} + +/// ★ The leg computes what `Program::eval` computes — the function the +/// constraint argument itself calls. +#[test] +fn the_constraint_leg_computes_what_the_host_computes() { + for shape in shapes() { + let emitted = program_of(&shape); + for seed in [0x13u64, 0x27, 0x41] { + let values: Vec = (0..shape.factors) + .map(|slot| fee(seed * 101 + slot as u64)) + .collect(); + let mut scratch = Vec::new(); + let want = shape.program.eval(&values, &mut scratch); + + let arena: Vec<_> = values.iter().map(ext_word).collect(); + let exec = execute(&emitted, &[arena], &crate::hash_pin::BLOCK_HASHER) + .expect("the constraint leg executes"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published value"); + assert_eq!( + got, want, + "{} at seed {seed:#x}: the emitted leg and the host disagree", + shape.name + ); + } + } +} diff --git a/prover/src/lfm/whir_reduce.rs b/prover/src/lfm/whir_reduce.rs new file mode 100644 index 000000000..7d991fe73 --- /dev/null +++ b/prover/src/lfm/whir_reduce.rs @@ -0,0 +1,228 @@ +//! `claim_reduce::verify` as a machine leg — the last step of a table's +//! argument, and the one that hands the stacked opening its claims. +//! +//! The mirror is `multilinear::claim_reduce::verify` +//! (`crypto/multilinear/src/claim_reduce.rs:262-321`). The main sumcheck leaves +//! one claim per FACTOR, and a factor is a column read at a frame offset; this +//! reduces all of them to one claim per COLUMN, at a single point, so the whole +//! epoch settles in one stacked opening. +//! +//! # What it does, in the order the transcript sees it +//! +//! Absorb every factor value; draw γ and raise it to the factor count; batch +//! the factor values by those weights into one claim; run a DEGREE-2 sumcheck +//! over the table's variables (`:291` — two, always, because the summand is a +//! kernel times a column and each is multilinear); rebuild that sumcheck's +//! residual as `Σ_offset shift_k(α, y)·(Σ_{factors at that offset} γ^i·c_i)`; +//! refuse if it disagrees; then absorb the column values. +//! +//! # ★ The kernel is per OFFSET, not per factor +//! +//! `shift_eval(α, y, k)` (`:308`) is evaluated once per DISTINCT offset, and +//! every factor reading that offset shares it. The VM's tables read at two +//! offsets — the row and the row after — so a table with dozens of factors +//! spends two kernels, and the per-factor cost is one `MulAdd`. That is the +//! whole reason this leg is cheap, and it is why the form below is a function +//! of the offset SET rather than of the factor count alone. +//! +//! # The refusal +//! +//! `rebuilt != claim.expected_evaluation` is `Error::ShiftedReadMismatch` on +//! the host (`:310-312`) and an `assert_eq_ext` here — a division with no +//! satisfying assignment, so a proof the host rejects has no execution. +//! +//! # ⛔ A non-mutation, recorded rather than dropped (instance 52) +//! +//! Moving the column-value absorbs from AFTER the refusal to before it was run +//! as a mutation and passed every gate — because it is a rewrite. The refusal +//! emits no transcript operation, so carrying the absorbs across it leaves the +//! absorb sequence, every drawn challenge and every row exactly as they were; +//! a straight-line program is a DAG and the order two independent groups are +//! emitted in is not a property of it. The claim written down before running it +//! — "it moves every challenge a later leg draws" — was simply false. +//! +//! The mutation that IS one moves the absorbs above the SUMCHECK, where a draw +//! sits between: every round challenge then comes from a transcript that +//! already holds the column values, and three of the four gates fail. + +use multilinear::claim_reduce::FactorSource; + +use super::builder::{Ext, LfmBuilder}; +use super::whir_poly::{ + challenge_powers_rows, emit_challenge_powers, emit_shift_eval, emit_sumcheck_rounds, + shift_eval_rows, sumcheck_round_rows, +}; +use super::whir_transcript::{WhirTranscript, absorb_unpack_rows, sample_ext_rows}; + +/// The degree the reduce sumcheck runs at (`claim_reduce.rs:291`). +/// +/// Not a parameter: the summand is one multilinear kernel times one multilinear +/// column, and two is what that is. +pub const REDUCE_DEGREE: usize = 2; + +/// Rows an `assert_eq_ext` lowers to: the difference and the division by zero +/// (`builder.rs:289-292`). +const ASSERT_ROWS: usize = 2; + +/// The proof `claim_reduce::verify` reads, as wires. +pub struct ReduceWires<'a> { + /// One sumcheck round per variable, each carrying [`REDUCE_DEGREE`] + /// evaluations. + pub sumcheck: &'a [Vec], + /// Every committed column's value at the reduced point. + pub column_values: &'a [Ext], +} + +/// What the leg leaves for the stacked opening to settle. +pub struct ReducedClaimWires { + pub point: Vec, + pub column_values: Vec, +} + +/// The distinct offsets the factors read, ascending — `claim_reduce::offsets` +/// (`:84-89`), which is emit-time structure because a factor's offset is the +/// AIR's own. +pub fn distinct_offsets(sources: &[FactorSource]) -> Vec { + let mut all: Vec = sources.iter().map(|s| s.offset).collect(); + all.sort_unstable(); + all.dedup(); + all +} + +/// INSTRUCTIONS [`emit_claim_reduce_verify`] emits, once the program has +/// interned its `1` and its `0`. +/// +/// Every term by the shape it comes from: +/// +/// - one `Unpack` per factor value absorbed, and one `Pack` for γ; +/// - `challenge_powers_rows(|sources|)` for the weights; +/// - `|sources| − 1` `MulAdd`s to batch the factor values, because the first +/// weight is the literal one and costs no multiply; +/// - per variable, the sumcheck's `REDUCE_DEGREE` absorbed evaluations, its +/// drawn challenge and `sumcheck_round_rows(REDUCE_DEGREE)`; +/// - per DISTINCT offset, one `shift_eval` over the table's variables, plus one +/// row per factor at that offset (the first is free when its weight is the +/// literal one) and one more to fold the offset's term into the total (free +/// for the first offset, which starts it); +/// - the refusal, and one `Unpack` per column value absorbed. +pub fn claim_reduce_rows(sources: &[FactorSource], num_columns: usize, num_vars: usize) -> usize { + let factors = sources.len(); + let mut rows = factors * absorb_unpack_rows() + sample_ext_rows(); + rows += challenge_powers_rows(factors); + rows += factors.saturating_sub(1); + rows += num_vars + * (REDUCE_DEGREE * absorb_unpack_rows() + + sample_ext_rows() + + sumcheck_round_rows(REDUCE_DEGREE)); + + for offset in distinct_offsets(sources) { + rows += shift_eval_rows(num_vars, offset); + let members: Vec = (0..factors) + .filter(|&i| sources[i].offset == offset) + .collect(); + // The first member costs nothing when its weight is `γ^0 = 1`, which + // only the very first factor can be. + rows += members.len() - usize::from(members[0] == 0); + // One row to join the kernel to the batch, whichever offset it is: a + // `Mul` for the first, which opens the total, and a `MulAdd` for every + // later one, which folds into it. + rows += 1; + } + + rows + ASSERT_ROWS + num_columns * absorb_unpack_rows() +} + +/// ★ `claim_reduce::verify`, emitted. +/// +/// `alpha` is the point the main sumcheck left the factors claimed at; the +/// point this returns is the one every COLUMN is claimed at, which is what the +/// stacked opening settles. +pub fn emit_claim_reduce_verify( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + proof: &ReduceWires<'_>, + sources: &[FactorSource], + factor_values: &[Ext], + alpha: &[Ext], + num_columns: usize, +) -> ReducedClaimWires { + assert_eq!( + sources.len(), + factor_values.len(), + "one value per factor — the host's `check_shape`, made emit-time" + ); + assert!(!sources.is_empty(), "a table has factors"); + assert_eq!( + proof.column_values.len(), + num_columns, + "one value per committed column" + ); + assert_eq!( + proof.sumcheck.len(), + alpha.len(), + "the reduce sumcheck runs over the table's variables" + ); + assert!( + sources.iter().all(|s| s.column < num_columns), + "every factor reads a column the table committed" + ); + + for &value in factor_values { + transcript.absorb_ext(b, value); + } + let gamma = transcript.sample_ext(b); + let weights = emit_challenge_powers(b, gamma, sources.len()); + + // `Σ γ^i · v_i`, with the first weight the literal one. + let mut claimed = factor_values[0]; + for (weight, &value) in weights.iter().zip(factor_values).skip(1) { + claimed = b.emul_add(*weight, value, claimed); + } + + let mut point = Vec::with_capacity(alpha.len()); + for round in proof.sumcheck { + assert_eq!( + round.len(), + REDUCE_DEGREE, + "the reduce sumcheck is degree two" + ); + for &evaluation in round { + transcript.absorb_ext(b, evaluation); + } + point.push(transcript.sample_ext(b)); + } + let residual = emit_sumcheck_rounds(b, claimed, proof.sumcheck, &point); + + let mut rebuilt: Option = None; + for offset in distinct_offsets(sources) { + let members: Vec = (0..sources.len()) + .filter(|&i| sources[i].offset == offset) + .collect(); + let mut batched: Option = None; + for &i in &members { + let column = proof.column_values[sources[i].column]; + batched = Some(match batched { + // `γ^0` is the literal one: the column value IS the term. + None if i == 0 => column, + None => b.emul(weights[i], column), + Some(acc) => b.emul_add(weights[i], column, acc), + }); + } + let batched = batched.expect("an offset in the list has a factor reading it"); + let kernel = emit_shift_eval(b, alpha, &point, offset); + rebuilt = Some(match rebuilt { + None => b.emul(kernel, batched), + Some(acc) => b.emul_add(kernel, batched, acc), + }); + } + b.assert_eq_ext(rebuilt.expect("a table has at least one offset"), residual); + + for &value in proof.column_values { + transcript.absorb_ext(b, value); + } + + ReducedClaimWires { + point, + column_values: proof.column_values.to_vec(), + } +} diff --git a/prover/src/lfm/whir_reduce_tests.rs b/prover/src/lfm/whir_reduce_tests.rs new file mode 100644 index 000000000..ae705f956 --- /dev/null +++ b/prover/src/lfm/whir_reduce_tests.rs @@ -0,0 +1,427 @@ +//! Gates for the claim-reduce leg. +//! +//! The proof under test is a real one: `claim_reduce::prove` over real columns, +//! under the RPX transcript the machine replays, with the factor values taken +//! from the host's own `evaluate_source` so the claim being reduced is the one +//! the columns actually hold. +//! +//! # Why executing the honest proof pins the transcript SEQUENCE too +//! +//! The leg draws γ and one challenge per sumcheck round, and every one of them +//! feeds a refusal: γ weights the batch the sumcheck is run on, and a round's +//! challenge is where the next round's claim is taken. A leg that absorbed in +//! the wrong order, or drew a challenge one absorb early, would draw different +//! values from the same proof and the residual would not close — so the +//! machine would refuse a proof the host accepts, which is what the first gate +//! asserts cannot happen. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; +use multilinear::claim_reduce::{self, FactorSource, ReduceProof, ReducedClaim}; +use multilinear::mle::Mle; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_reduce::{REDUCE_DEGREE, ReduceWires, claim_reduce_rows, emit_claim_reduce_verify}; +use super::whir_transcript::{ + COORDINATES_PER_EXT, SpongeEntry, SpongeSchedule, WhirTranscript, absorb_unpack_rows, +}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type HostTranscript = DefaultTranscript; + +fn pseudo(seed: u64, count: usize) -> Vec { + let mut state = seed | 1; + (0..count) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }) + .collect() +} + +fn ext_point(seed: u64, n: usize) -> Vec { + pseudo(seed, 3 * n) + .chunks(3) + .map(|c| FEE::new([c[0], c[1], c[2]])) + .collect() +} + +/// The factor layouts the gate runs at. +/// +/// A VM table reads its columns at the row and at the row after, so offsets +/// `{0, 1}` is the production case; `{0, 2, 3}` is here because a form that +/// only ever saw one kernel per offset could not tell "per offset" from "per +/// factor", and because an offset whose first factor is not factor zero is the +/// case where the `γ^0 = 1` saving does NOT apply. +fn layouts() -> Vec<(usize, usize, Vec)> { + vec![ + (3, 2, vec![FactorSource::direct(0), FactorSource::direct(1)]), + ( + 4, + 3, + vec![ + FactorSource::direct(0), + FactorSource::direct(1), + FactorSource::shifted(1, 1), + FactorSource::direct(2), + FactorSource::shifted(2, 1), + ], + ), + ( + 5, + 4, + vec![ + FactorSource::direct(0), + FactorSource::shifted(0, 2), + FactorSource::direct(1), + FactorSource::shifted(2, 3), + FactorSource::direct(3), + FactorSource::shifted(3, 2), + ], + ), + ] +} + +struct Fixture { + proof: ReduceProof, + factor_values: Vec, + alpha: Vec, + claim: ReducedClaim, +} + +/// Proves and verifies one reduction on the host, keeping what both sides saw. +fn fixture(num_vars: usize, num_columns: usize, sources: &[FactorSource]) -> Fixture { + let columns: Vec> = (0..num_columns) + .map(|c| { + Mle::new(pseudo(0x51ced + c as u64, 1 << num_vars)).expect("a power-of-two column") + }) + .collect(); + let alpha = ext_point(0x5EED_u64, num_vars); + // The claims being reduced must be the ones the columns hold, or the + // reduction is a proof about nothing. + let factor_values: Vec = sources + .iter() + .map(|source| { + claim_reduce::evaluate_source::(&columns, source, &alpha) + .expect("the host evaluates its own factor") + }) + .collect(); + + let mut proving = HostTranscript::new(&[]); + let (proof, _) = claim_reduce::prove::( + &columns, + sources, + &factor_values, + &alpha, + None, + &mut proving, + ) + .expect("the reduction proves"); + + let mut verifying = HostTranscript::new(&[]); + let claim = claim_reduce::verify::( + &proof, + sources, + &factor_values, + &alpha, + num_columns, + &mut verifying, + ) + .expect("the control proof must verify"); + + Fixture { + proof, + factor_values, + alpha, + claim, + } +} + +/// Where each wire lives in the arena, used by the program and the filler so +/// the two cannot drift. +struct Layout { + factors: usize, + num_vars: usize, + num_columns: usize, +} + +impl Layout { + fn sumcheck_at(&self, round: usize, which: usize) -> u32 { + (self.factors + REDUCE_DEGREE * round + which) as u32 + } + fn column_at(&self, column: usize) -> u32 { + (self.factors + REDUCE_DEGREE * self.num_vars + column) as u32 + } + fn alpha_at(&self, variable: usize) -> u32 { + (self.factors + REDUCE_DEGREE * self.num_vars + self.num_columns + variable) as u32 + } + fn total(&self) -> u32 { + self.alpha_at(self.num_vars) + } +} + +fn reduce_program(num_vars: usize, num_columns: usize, sources: &[FactorSource]) -> LfmProgram { + let layout = Layout { + factors: sources.len(), + num_vars, + num_columns, + }; + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(layout.total()); + let mut transcript = WhirTranscript::new(); + + let factor_values: Vec<_> = (0..sources.len()) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let sumcheck: Vec> = (0..num_vars) + .map(|round| { + (0..REDUCE_DEGREE) + .map(|which| { + b.hint_word(arena, layout.sumcheck_at(round, which)) + .as_ext() + }) + .collect() + }) + .collect(); + let column_values: Vec<_> = (0..num_columns) + .map(|c| b.hint_word(arena, layout.column_at(c)).as_ext()) + .collect(); + let alpha: Vec<_> = (0..num_vars) + .map(|v| b.hint_word(arena, layout.alpha_at(v)).as_ext()) + .collect(); + + let wires = ReduceWires { + sumcheck: &sumcheck, + column_values: &column_values, + }; + let reduced = emit_claim_reduce_verify( + &mut b, + &mut transcript, + &wires, + sources, + &factor_values, + &alpha, + num_columns, + ); + for value in &reduced.point { + b.public(value.as_cell()); + } + + let program = compile(b.finish()); + validate(&program).expect("the reduce leg must be admissible"); + program +} + +fn reduce_arena(f: &Fixture, proof: &ReduceProof, num_columns: usize) -> Vec { + let mut words: Vec = f.factor_values.iter().map(ext_word).collect(); + for round in &proof.sumcheck.rounds { + assert_eq!(round.evaluations.len(), REDUCE_DEGREE); + words.extend(round.evaluations.iter().map(ext_word)); + } + assert_eq!(proof.column_values.len(), num_columns); + words.extend(proof.column_values.iter().map(ext_word)); + words.extend(f.alpha.iter().map(ext_word)); + words +} + +/// The sponge's own hashing for this leg, derived from its absorbs and draws. +/// +/// Every absorb here is one extension element — the factor values, the +/// sumcheck's evaluations, the column values — and every draw is one extension +/// challenge. The leg reads no `state()`, because it spends no grind. +fn reduce_schedule(num_vars: usize, num_columns: usize, factors: usize) -> SpongeSchedule { + let mut sponge = SpongeSchedule::new(SpongeEntry::fresh()); + for _ in 0..factors { + sponge.absorb(COORDINATES_PER_EXT); + } + sponge.draw_ext(); + for _ in 0..num_vars { + for _ in 0..REDUCE_DEGREE { + sponge.absorb(COORDINATES_PER_EXT); + } + sponge.draw_ext(); + } + for _ in 0..num_columns { + sponge.absorb(COORDINATES_PER_EXT); + } + sponge +} + +fn const_rows(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Const { .. })) + .count() +} + +/// ★ The leg executes on a proof the host accepts, and lands on the host's +/// point. +/// +/// The point is what the stacked opening settles every column at, so a leg that +/// executed but reduced to a different point would be a verifier that proves +/// the wrong claim. It is published and compared rather than left to the +/// execution. +#[test] +fn the_reduce_leg_executes_and_lands_where_the_host_does() { + for (num_vars, num_columns, sources) in layouts() { + let f = fixture(num_vars, num_columns, &sources); + let program = reduce_program(num_vars, num_columns, &sources); + let arena = reduce_arena(&f, &f.proof, num_columns); + let exec = + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!( + "{num_vars} vars, {} factors: the machine refused an accepted proof: {e:?}", + sources.len() + ) + }); + let point: Vec = exec + .public_words + .iter() + .map(|(_, w)| word_as_ext(w).expect("a published extension value")) + .collect(); + assert_eq!( + point, f.claim.point, + "{num_vars} vars: the reduced point must be the host's" + ); + println!( + "reduce {num_vars} vars, {} factors, {num_columns} columns: {} instructions", + sources.len(), + program.instrs.len() + ); + } +} + +/// ★ The tamper arm, in three halves at three sites. +/// +/// Each half earns its place: the untouched proof must execute, or a refusal +/// below is a refusal of everything; the HOST must reject the same forgery, or +/// the machine is refusing something valid and the finding is a completeness +/// bug rather than a soundness success; and only then does the machine's +/// refusal say anything. +#[test] +fn a_tampered_reduction_cannot_execute() { + let (num_vars, num_columns, sources) = layouts()[1].clone(); + let f = fixture(num_vars, num_columns, &sources); + let program = reduce_program(num_vars, num_columns, &sources); + + assert!( + execute( + &program, + &[reduce_arena(&f, &f.proof, num_columns)], + &crate::hash_pin::BLOCK_HASHER + ) + .is_ok(), + "the ground proof must execute" + ); + + let mut refused = 0; + for site in ["column", "evaluation", "factor"] { + let mut forged = f.proof.clone(); + let mut factor_values = f.factor_values.clone(); + match site { + "column" => forged.column_values[1] += FEE::one(), + "evaluation" => forged.sumcheck.rounds[0].evaluations[1] += FEE::one(), + _ => factor_values[2] += FEE::one(), + } + + let mut verifying = HostTranscript::new(&[]); + assert!( + claim_reduce::verify::( + &forged, + &sources, + &factor_values, + &f.alpha, + num_columns, + &mut verifying, + ) + .is_err(), + "the host must reject the forgery at {site}, or the machine's refusal refuses \ + something the host accepts" + ); + + let mut arena = reduce_arena(&f, &forged, num_columns); + for (i, value) in factor_values.iter().enumerate() { + arena[i] = ext_word(value); + } + assert!( + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).is_err(), + "the machine must refuse the forgery at {site}" + ); + refused += 1; + } + println!("reduce tamper arm: {refused} sites, all refused"); +} + +/// ★ F1 for the leg: its rows against the closed form, with the sponge's own +/// hashing derived separately from the absorbs and draws. +/// +/// The two halves are separated for the reason the chain's are: the schedule is +/// what the sponge's BUFFER holds at each hash, which the leg's arithmetic +/// cannot see. +#[test] +fn the_reduce_leg_emits_its_closed_form() { + for (num_vars, num_columns, sources) in layouts() { + let program = reduce_program(num_vars, num_columns, &sources); + let layout = Layout { + factors: sources.len(), + num_vars, + num_columns, + }; + let schedule = reduce_schedule(num_vars, num_columns, sources.len()); + // The plumbing: one `Hint` per arena word, and one `Public` per + // published coordinate of the reduced point. + let plumbing = layout.total() as usize + num_vars; + let consts = const_rows(&program); + let measured = program.instrs.len() - consts - plumbing; + let predicted = claim_reduce_rows(&sources, num_columns, num_vars) + schedule.rows(); + println!( + "reduce {num_vars} vars, {} factors, {num_columns} columns: {measured} rows \ + ({} leg + {} schedule predicted {predicted}), {consts} constants, \ + {} permutations", + sources.len(), + claim_reduce_rows(&sources, num_columns, num_vars), + schedule.rows(), + schedule.perms(), + ); + assert_eq!( + measured, predicted, + "{num_vars} vars, {num_columns} columns" + ); + } +} + +/// The absorb count the schedule is built from is the one the leg emits: one +/// `Unpack` per absorbed extension element, and the leg absorbs the factor +/// values, the sumcheck's evaluations and the column values and nothing else. +/// +/// Stated as its own pin because the schedule above would otherwise be a +/// second, unchecked description of the same sequence. +#[test] +fn the_reduce_leg_absorbs_exactly_what_it_reads() { + let (num_vars, num_columns, sources) = layouts()[2].clone(); + let program = reduce_program(num_vars, num_columns, &sources); + let unpacks = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Unpack { .. })) + .count(); + let schedule = reduce_schedule(num_vars, num_columns, sources.len()); + let absorbed = sources.len() + REDUCE_DEGREE * num_vars + num_columns; + // One `Unpack` per absorbed element, plus one per squeeze reading its + // digest — the two are the only `Unpack`s the leg emits. + assert_eq!( + unpacks, + absorbed * absorb_unpack_rows() + schedule.hashes().len(), + "every `Unpack` is either an absorbed element or a squeeze's digest" + ); +} diff --git a/prover/src/lfm/whir_stacked.rs b/prover/src/lfm/whir_stacked.rs new file mode 100644 index 000000000..451708c38 --- /dev/null +++ b/prover/src/lfm/whir_stacked.rs @@ -0,0 +1,791 @@ +//! `stacked_eval::verify` as a machine leg: the weight, and the wrapper over it. +//! +//! `stacked_eval::verify` hands `whir_chain::verify_weighted` a closure — the +//! weight of one stacked polynomial at the chain's final point — and that +//! closure is `weight_at` (`crypto/multilinear/src/stacked_eval.rs:287-317`): +//! +//! ```text +//! Σ_c w_c · eq(corner_c, high) · eq(point_c, low) +//! ``` +//! +//! where `corner_c` is the column's prefix bits and `(high, low)` splits the +//! point at the prefix length. Written naively that is two `eq`s per column per +//! chain, which the sizing note priced as the term to collapse. Two structural +//! facts collapse it. +//! +//! [`emit_stacked_verify`] is the whole of `stacked_eval::verify` +//! (`stacked_eval.rs:409-459`) around it: the column values absorbed, one +//! batching draw, and one chain per stacked polynomial with this weight as its +//! `W`. It is the boundary an epoch's group is verified at, and the place the +//! transcript is THREADED — see [`stacked_verify_cost`]. + +use multilinear::stacking::StackedLayout; +use multilinear::whir::Domain; + +use crate::tables::types::{FEE, GoldilocksField}; + +use super::algebraic_commit::leaf_capacity; +use super::builder::{Cell, Ext, LfmBuilder}; +use super::instr::Addr; +use super::whir_chain::{ + ChainRoundWires, ChainShape, chain_grind_perms, chain_opening_perms, chain_shape_rows, + chain_sponge, emit_verify_grouped, emit_verify_weighted, +}; +use super::whir_poly::{ + challenge_powers_rows, emit_challenge_powers, emit_eq_eval, eq_eval_rows_again, +}; + +/// A stack's commit-time out-of-domain claims as wires +/// (`stacked_eval::CommitOod`): the point over the stack, `z0^(2^j)` for +/// `j < n_stack`, and one answer per stacked polynomial in commitment order. +/// +/// ⛔ The answers are the CELLS the leaf's front absorbed before `(z, α, β)`, +/// never a second hint: a chain reading another copy could open a value the +/// transcript never bound. +#[derive(Clone, Copy)] +pub struct OodWires<'a> { + pub point: &'a [Ext], + pub answers: &'a [Ext], +} +use super::whir_transcript::{ + COORDINATES_PER_EXT, SpongeEntry, SpongeSchedule, WhirTranscript, absorb_unpack_rows, + sample_ext_rows, +}; +use super::word::{LfmWord, ext_word}; + +/// One column's claim, as wires: the point it is claimed at and the batching +/// weight it carries (`gamma^column`). +/// +/// ⚠ The weights are INPUTS here: whoever assembles the group emits the `n − 1` +/// multiplies. `multilinear::challenge_powers` is the host function that leg is +/// gated against — it was `pub(crate)` when this type was written, which is why +/// the weights arrive from outside rather than being built in place. +#[derive(Clone, Copy)] +pub struct ColumnClaim<'a> { + pub point: &'a [Ext], + pub weight: Ext, +} + +/// INSTRUCTIONS [`emit_weight_at`] emits for one stacked polynomial, given +/// which columns share a claimed point. +/// +/// `group_of[column]` names that point: one table's sumcheck leaves every one +/// of its columns at the same point (`stacked_eval.rs:185-188`), so in a real +/// epoch this is the table index and the group is a table's columns inside this +/// polynomial. +/// +/// The three terms, each by its shape: +/// - one `Sub` per prefix position that ANY of the polynomial's columns reads as +/// a zero bit — `1 − at_j` depends on the position alone, so it is emitted +/// once and shared; +/// - per distinct claimed point, one `eq` over that group's columns' own +/// variables plus one `MulAdd` to join the group into the running total; +/// - per column, `p − 1` products for its prefix indicator plus one `MulAdd` +/// for its weighted fold, where `p = n_stack − num_vars`; a column that fills +/// the whole stack has no indicator and pays only the fold, hence `max(p, 1)`. +/// +/// An empty polynomial is one interned zero. +pub fn weight_at_rows(layout: &StackedLayout, poly: usize, group_of: &[usize]) -> usize { + let placements = layout.placements(); + let columns: Vec = (0..placements.len()) + .filter(|&column| placements[column].poly == poly) + .collect(); + if columns.is_empty() { + return 1; + } + + let mut reads_a_zero = vec![false; layout.n_stack()]; + for &column in &columns { + for (position, bit) in placements[column].prefix_bits().iter().enumerate() { + if !bit { + reads_a_zero[position] = true; + } + } + } + let shared_subs = reads_a_zero.iter().filter(|seen| **seen).count(); + + let mut seen_groups: Vec = Vec::new(); + let mut group_rows = 0; + for &column in &columns { + if !seen_groups.contains(&group_of[column]) { + seen_groups.push(group_of[column]); + group_rows += eq_eval_rows_again(placements[column].num_vars) + 1; + } + } + + let column_rows: usize = columns + .iter() + .map(|&column| prefix_len(layout, column).max(1)) + .sum(); + + shared_subs + group_rows + column_rows +} + +/// `LFM_CONST` rows the leg interns, once per program: the `1`, which both +/// `eq` and a full-height column's empty indicator reach for. +pub const fn weight_at_consts() -> usize { + 1 +} + +/// ★ `weight_at` for one stacked polynomial, emitted. +/// +/// ★ **Why this is not two `eq`s per column.** First, `corner_c` is a CONSTANT +/// zero/one vector, so its `eq` is just the prefix indicator +/// `Π_j (at_j or 1 − at_j)` — `p` factors, not a `5p`-row `eq` — and `1 − at_j` +/// depends only on the position, so every column that reads position `j` as +/// zero shares one `Sub`. Second, columns claimed at the same point share their +/// whole low-half `eq`; one table's sumcheck leaves all of its columns at one +/// point, so a real epoch pays one `eq` per (table, polynomial), not one per +/// column. Grouping is by the point's WIRES, which is exact at emit time and +/// needs no notion of a table. +/// +/// The total is then `Σ_groups eq_low · Σ_{c in group} w_c · indicator_c`, which +/// also saves one multiply per column against multiplying each term out. +pub fn emit_weight_at( + b: &mut LfmBuilder, + layout: &StackedLayout, + poly: usize, + claims: &[ColumnClaim<'_>], + at: &[Ext], +) -> Ext { + assert_eq!( + at.len(), + layout.n_stack(), + "the weight is evaluated on the stacked cube" + ); + assert_eq!( + claims.len(), + layout.placements().len(), + "one claim per column of the layout" + ); + + let placements = layout.placements(); + let columns: Vec = (0..placements.len()) + .filter(|&column| placements[column].poly == poly) + .collect(); + if columns.is_empty() { + return b.ext_const(&FEE::zero()); + } + + // Columns sharing a claimed point share its `eq`. The key is the point's + // wires, in order. + let mut groups: Vec<(Vec, Vec)> = Vec::new(); + for &column in &columns { + let key: Vec = claims[column].point.iter().map(Ext::addr).collect(); + match groups.iter_mut().find(|(seen, _)| *seen == key) { + Some((_, members)) => members.push(column), + None => groups.push((key, vec![column])), + } + } + + let mut not_at: Vec> = vec![None; at.len()]; + let mut one: Option = None; + let mut total: Option = None; + + for (_, members) in &groups { + let leader = members[0]; + let prefix = prefix_len(layout, leader); + assert_eq!( + claims[leader].point.len(), + placements[leader].num_vars, + "a column is claimed on its own cube" + ); + let eq_low = emit_eq_eval(b, claims[leader].point, &at[prefix..]); + + let mut inner: Option = None; + for &column in members { + let mut indicator: Option = None; + for (position, bit) in placements[column].prefix_bits().iter().enumerate() { + let factor = if *bit { + at[position] + } else { + match not_at[position] { + Some(cached) => cached, + None => { + let unit = *one.get_or_insert_with(|| b.ext_const(&FEE::one())); + let complement = b.esub(unit, at[position]); + not_at[position] = Some(complement); + complement + } + } + }; + indicator = Some(match indicator { + None => factor, + Some(acc) => b.emul(acc, factor), + }); + } + // A column filling the whole stack has an empty indicator, which is + // the interned `1` — so the fold below is one row either way. + let indicator = + indicator.unwrap_or_else(|| *one.get_or_insert_with(|| b.ext_const(&FEE::one()))); + let weight = claims[column].weight; + inner = Some(match inner { + None => b.emul(indicator, weight), + Some(acc) => b.emul_add(indicator, weight, acc), + }); + } + let inner = inner.expect("a group holds at least one column"); + total = Some(match total { + None => b.emul(eq_low, inner), + Some(acc) => b.emul_add(eq_low, inner, acc), + }); + } + + total.expect("a non-empty polynomial has at least one group") +} + +/// How many high variables of the stack a column's prefix selects. +fn prefix_len(layout: &StackedLayout, column: usize) -> usize { + let place = &layout.placements()[column]; + place.n_stack - place.num_vars +} + +/// The columns one stacked polynomial holds — `columns_in` +/// (`stacked_eval.rs:200-206`), which is private to the multilinear crate and +/// is exactly this filter over the public `placements()`. +fn columns_of(layout: &StackedLayout, poly: usize) -> Vec { + let placements = layout.placements(); + (0..placements.len()) + .filter(|&column| placements[column].poly == poly) + .collect() +} + +/// One stacked polynomial's side of the proof, as wires. +pub struct StackedPolyWires<'a> { + /// Its chain, one entry per scheduled round. + pub rounds: &'a [ChainRoundWires<'a>], + /// Its commitment root, as a word. Unpacked by the wrapper, because the + /// wrapper is what holds it. + pub root: Cell, + /// The value its chain folds down to. + pub final_value: Ext, +} + +impl<'a> StackedPolyWires<'a> { + /// The one-polynomial chain these wires are. + pub fn as_chain(&'a self) -> StackedChainWires<'a> { + StackedChainWires { + rounds: self.rounds, + roots: std::slice::from_ref(&self.root), + final_value: self.final_value, + } + } +} + +/// One chain of a stacked opening, as wires: per polynomial under +/// [`multilinear::whir_chain::GroupFold::PerPoly`], or per run of +/// [`ChainShape::chains`] under `Indexed` (D-ONECHAIN). +pub struct StackedChainWires<'a> { + /// Its rounds, one entry per scheduled round of the run's shape. + pub rounds: &'a [ChainRoundWires<'a>], + /// The roots of the polynomials it opens, as words, in slot order: the + /// run's slice of the group's carried roots, so tree `p` is checked against + /// the root at a position the program fixes (REV-B2-JUDGE B2-4). + pub roots: &'a [Cell], + /// The value its chain folds down to. + pub final_value: Ext, +} + +/// `claimed` (`stacked_eval.rs:320-331`): one stacked polynomial's columns, +/// batched by their `gamma` powers. +/// +/// ⚠ The first term is a MULTIPLY and not a free wire, even though +/// `weights[0]` is the literal one: the weights arrive as WIRES out of +/// [`emit_challenge_powers`], and a polynomial past the first does not begin at +/// column zero anyway. This is where the analogy with `challenge_powers` — whose +/// first power IS free — misleads, so the form says `|columns|` and not +/// `|columns| − 1`. +/// +/// A grouped chain's claim is its run's polynomials' claims summed, which is +/// the same accumulation over the run's columns in order: one `γ` for the +/// whole group, its powers indexed by the group's columns (D-ONECHAIN §3.2). +fn emit_claimed( + b: &mut LfmBuilder, + layout: &StackedLayout, + run: std::ops::Range, + values: &[Ext], + weights: &[Ext], +) -> Ext { + let mut claimed: Option = None; + for column in run.flat_map(|poly| columns_of(layout, poly)) { + claimed = Some(match claimed { + None => b.emul(weights[column], values[column]), + Some(acc) => b.emul_add(weights[column], values[column], acc), + }); + } + claimed.unwrap_or_else(|| b.ext_const(&FEE::zero())) +} + +/// ★ `stacked_eval::verify` (`stacked_eval.rs:409-459`), emitted. +/// +/// Every column value is absorbed BEFORE the batching challenge is drawn, which +/// is the host's order and the reason the claims cannot be picked after seeing +/// it; then one [`emit_verify_weighted`] per stacked polynomial, with +/// [`emit_weight_at`] as its `W` and [`emit_claimed`] as its claim. +/// +/// ★ **One transcript, threaded.** The chains share `transcript`: the γ draw +/// precedes the first of them, so a chain here enters holding the digest that +/// draw re-absorbed rather than an empty sponge, and each chain enters where +/// the previous one left off. That is a cost as well as a value — see +/// [`stacked_verify_cost`]. +/// +/// `points` is the emitter's `Claimed`: one entry per COLUMN. Columns settled at +/// the same point pass the same WIRES, which is what [`emit_weight_at`] groups +/// on, so `Claimed::Shared` and `Claimed::PerColumn` are one object here. +/// +/// The host's three shape refusals are emit-time assertions, the +/// `epoch_verify.rs:171-179` idiom: a proof of another shape has no way to be +/// supplied to a program that reads a fixed number of wires. +/// +/// Returns the batching challenge. `stacked_eval::verify` returns only its +/// verdict, and every refusal here is already a division with no satisfying +/// assignment; γ comes back so a gate can compare it against the element the +/// HOST verifier sampled, which turns "the draw is in the right place" from an +/// absence of execution into a named assertion. +/// +/// One chain per polynomial: [`emit_stacked_verify_chains`] over +/// [`StackedPolyWires::as_chain`], whose rows at one polynomial a chain are +/// these. +#[allow(clippy::too_many_arguments)] +pub fn emit_stacked_verify( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + layout: &StackedLayout, + polys: &[StackedPolyWires<'_>], + points: &[&[Ext]], + values: &[Ext], + shape: &ChainShape, + domain: &Domain, +) -> Ext { + assert_eq!( + polys.len(), + layout.num_polys(), + "one chain per stacked polynomial" + ); + let chains: Vec> = polys.iter().map(StackedPolyWires::as_chain).collect(); + emit_stacked_verify_chains( + b, transcript, layout, &chains, points, values, None, shape, domain, + ) +} + +/// ★ [`emit_stacked_verify`] over the stack's CHAINS: one per run of +/// [`ChainShape::chains`], the host's own split (`stacked_eval::index_chains`, +/// REV-B2-JUDGE B2-1). A run of one is [`emit_verify_weighted`] with +/// [`emit_weight_at`] as its `W`, today's chain; a run of `P' ≥ 2` under +/// `GroupFold::Indexed` is ONE [`emit_verify_grouped`] whose claim is the run's +/// columns batched and whose weight is `Σ_p eq(ρ, p)·weight_at_p` read at the +/// data challenges `at[..k0] ‖ at[k0+m..]` (REV-B2-JUDGE B1-3, B2-5). +/// +/// With `ood` (`stacked_eval::verify_ood`): after the batching draw one more, +/// `γ'`; polynomial `i`'s answer rides its chain's claim at `γ'^(i+1)`, and its +/// weight gains `γ'^(i+1)·eq(point, ·)` — at the data challenges, under +/// `eq(ρ, p)`, in a run of several. +#[allow(clippy::too_many_arguments)] +pub fn emit_stacked_verify_chains( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + layout: &StackedLayout, + chains: &[StackedChainWires<'_>], + points: &[&[Ext]], + values: &[Ext], + ood: Option>, + shape: &ChainShape, + domain: &Domain, +) -> Ext { + assert_eq!( + values.len(), + layout.placements().len(), + "one claimed value per column of the layout" + ); + assert_eq!( + points.len(), + layout.placements().len(), + "one claimed point per column of the layout" + ); + let runs = shape.chains(layout.num_polys()); + assert_eq!( + chains.len(), + runs.len(), + "one chain per run of the verifier's split" + ); + assert_eq!( + shape.num_vars, + layout.n_stack(), + "every stacked polynomial has the stack's variables" + ); + + for value in values { + transcript.absorb_ext(b, *value); + } + let gamma = transcript.sample_ext(b); + let weights = emit_challenge_powers(b, gamma, values.len()); + + let claims: Vec> = (0..values.len()) + .map(|column| ColumnClaim { + point: points[column], + weight: weights[column], + }) + .collect(); + // The commit-time answers' batching, drawn after the claims' as the host + // draws it, and each polynomial's scale `γ'^(i+1)`. + let ood = ood.map(|ood| { + assert_eq!( + ood.answers.len(), + layout.num_polys(), + "one answer per stacked polynomial" + ); + assert_eq!( + ood.point.len(), + layout.n_stack(), + "the point spans the stack" + ); + let gamma = transcript.sample_ext(b); + let mut scales = emit_challenge_powers(b, gamma, layout.num_polys() + 1); + scales.remove(0); + (ood, scales) + }); + + for (chain, run) in chains.iter().zip(runs) { + #[cfg(test)] + super::whir_chain::onechain_steps::mark( + b, + super::whir_chain::onechain_steps::Step::Chain(run.start), + ); + assert_eq!( + chain.roots.len(), + run.len(), + "one root per polynomial of the run" + ); + let roots: Vec<[super::builder::Felt; 4]> = + chain.roots.iter().map(|&root| b.unpack(root)).collect(); + let mut claimed = emit_claimed(b, layout, run.clone(), values, &weights); + #[cfg(test)] + let claim_fault = ood_faults::get() == ood_faults::Fault::ClaimWithoutAnswers; + #[cfg(not(test))] + let claim_fault = false; + if let Some((wires, scales)) = ood.as_ref().filter(|_| !claim_fault) { + for i in run.clone() { + claimed = b.emul_add(scales[i], wires.answers[i], claimed); + } + } + #[cfg(test)] + let weight_fault = ood_faults::get() == ood_faults::Fault::WeightWithoutAnswers; + #[cfg(not(test))] + let weight_fault = false; + let ood = &ood; + if run.len() == 1 { + let i = run.start; + emit_verify_weighted( + b, + transcript, + chain.rounds, + &roots[0], + chain.final_value, + claimed, + shape, + domain, + |b, at| { + let weight = emit_weight_at(b, layout, i, &claims, at); + match ood.as_ref().filter(|_| !weight_fault) { + Some((wires, scales)) => { + let eq = emit_eq_eval(b, wires.point, at); + b.emul_add(scales[i], eq, weight) + } + None => weight, + } + }, + ); + continue; + } + let run_shape = shape.run(run.len()); + let (k0, m) = (run_shape.schedule[0], run_shape.index_vars()); + emit_verify_grouped( + b, + transcript, + chain.rounds, + &roots, + chain.final_value, + claimed, + &run_shape, + domain, + |b, at, eqs| { + // Each polynomial's weight at the data challenges, the index + // left out where round 0 bound it. + let data: Vec = at[..k0].iter().chain(&at[k0 + m..]).copied().collect(); + #[cfg(test)] + let faulted = weight_faults(b, &at[k0..k0 + m], eqs); + #[cfg(test)] + let eqs: &[Ext] = &faulted; + let mut total: Option = None; + for (eq, poly) in eqs.iter().zip(run.clone()) { + let weight = emit_weight_at(b, layout, poly, &claims, &data); + total = Some(match total { + None => b.emul(*eq, weight), + Some(acc) => b.emul_add(*eq, weight, acc), + }); + } + let mut total = total.expect("a run of two or more"); + // The answers' share: `eq(point, data)·Σ_p eq(ρ, p)·γ'^(i_p+1)`. + if let Some((wires, scales)) = ood.as_ref().filter(|_| !weight_fault) { + let eq = emit_eq_eval(b, wires.point, &data); + let mut scale: Option = None; + for (index, poly) in eqs.iter().zip(run) { + scale = Some(match scale { + None => b.emul(*index, scales[poly]), + Some(acc) => b.emul_add(*index, scales[poly], acc), + }); + } + let scale = scale.expect("a run of two or more"); + total = b.emul_add(scale, eq, total); + } + #[cfg(test)] + let total = absent_slot_fault(b, &at[k0..k0 + m], eqs.len(), total); + total + }, + ); + } + + gamma +} + +/// Test only: the final weight's `eq(ρ, p)`s under the weight faults of +/// [`super::whir_chain::onechain_faults`] — `ρ` reversed, or the index factor +/// dropped (every slot's `eq` one). +#[cfg(test)] +fn weight_faults(b: &mut LfmBuilder, rho: &[Ext], eqs: &[Ext]) -> Vec { + use super::whir_chain::onechain_faults::{Fault, get}; + let one = b.ext_const(&FEE::one()); + match get() { + Fault::SwapRhoInWeight => { + let swapped: Vec = rho.iter().rev().copied().collect(); + super::whir_chain::emit_eq_index(b, &swapped, eqs.len(), one) + } + Fault::DropIndexFactor => vec![one; eqs.len()], + _ => eqs.to_vec(), + } +} + +/// Test only: under `AbsentSlotWeightOne`, the weight plus `eq(ρ, p)` for +/// every absent slot `p ≥ trees`, as if each held a weight of one. +#[cfg(test)] +fn absent_slot_fault(b: &mut LfmBuilder, rho: &[Ext], trees: usize, total: Ext) -> Ext { + use super::whir_chain::onechain_faults::{Fault, get}; + if get() != Fault::AbsentSlotWeightOne { + return total; + } + let one = b.ext_const(&FEE::one()); + let slots = 1usize << rho.len(); + super::whir_chain::emit_eq_index(b, rho, slots, one)[trees..] + .iter() + .fold(total, |acc, eq| b.eadd(acc, *eq)) +} + +/// What [`emit_stacked_verify`] costs: its rows, its permutations, and the +/// sponge it hands on. +/// +/// ⚠ **The row count is CONST-FREE**, in exactly the sense `chain_rows` is — +/// [`operations`](Self::operations) counts rows that are neither `LFM_CONST` nor +/// the arena's hints (`whir_chain_tests.rs:1062`). The per-table form +/// (`whir_table::table_verify_cost`) is the OTHER convention and includes its +/// constants, so a census that sums the two must say which it is in. +/// [`own_constants`](Self::own_constants) is the constants this leg is +/// responsible for, by VALUE, so a caller can union them into one pool. +pub struct StackedCost { + ops: usize, + /// The openings' and grinds' permutations, summed over the polynomials. + chain_perms: usize, + schedule: SpongeSchedule, + chains: usize, +} + +impl StackedCost { + /// INSTRUCTIONS, excluding every `LFM_CONST`: the straight-line rows and + /// the threaded sponge's `Pack`s and `Unpack`s. + pub fn operations(&self) -> usize { + self.ops + self.schedule.rows() + } + + /// PERMUTATIONS, whole: each chain's openings and grinds, and the threaded + /// transcript's own. + pub fn perms(&self) -> usize { + self.chain_perms + self.schedule.perms() + } + + /// Where the sponge is left, so the next leg of the same program continues + /// from it. + pub fn entry(&self) -> SpongeEntry { + self.schedule.entry() + } + + /// Chains threaded — what a census multiplies a per-chain constant count by, + /// and the number the pool assertion says it should NOT. + pub fn chains(&self) -> usize { + self.chains + } + + pub fn schedule(&self) -> &SpongeSchedule { + &self.schedule + } + + /// ★ The `LFM_CONST` words this leg OWNS, deduplicated — the two kinds it + /// can name: + /// + /// - `FEE::one()`, ONE row for the whole program. [`emit_challenge_powers`] + /// seeds its accumulator with it, [`emit_weight_at`] reaches for it for + /// `1 − at_j` and for a full-height column's empty indicator, and every + /// chain's `emit_verify_weighted` interns it as its `one`; the pool is + /// keyed on the canonical word (`builder.rs:170-181`), so they are one row + /// between them. + /// - `leaf_capacity(felts)` per hash of the THREADED schedule. + /// `algebraic_leaf_hash` interns it for the leaf it is about to hash + /// (`edsl.rs:676-692`), so a program pays one per distinct leaf length — + /// and, because the word carries `felts % RATE_FELTS`, lengths sharing + /// that residue share the row. No per-hash form can see this; it is taken + /// off the finished schedule, the way `table_verify_cost` takes it. + /// + /// The chains' OTHER constants are not named here, for the same reason + /// `chain_rows` does not name them. + pub fn own_constants(&self) -> Vec { + let mut words: Vec = vec![ext_word(&FEE::one())]; + for hash in self.schedule.hashes() { + let word = leaf_capacity(hash.felts()); + if !words.contains(&word) { + words.push(word); + } + } + words + } +} + +/// ★ INSTRUCTIONS and permutations [`emit_stacked_verify`] costs, every term by +/// the shape it comes from. +/// +/// With `C` the layout's columns and `P` its polynomials: +/// +/// - `C` absorbs, one `Unpack` each, and `C` times three felts into the sponge; +/// - the batching draw: one `Pack`, and three candidates; +/// - `challenge_powers_rows(C)` for the weights — `C − 1`, the first power being +/// the interned one; +/// - per polynomial: one `Unpack` for its root, `|columns in poly|` rows for the +/// claimed fold (a `Mul` then `MulAdd`s — see [`emit_claimed`] for why the +/// first is not free), `weight_at_rows` for its weight closure, and +/// `chain_shape_rows` for its chain; +/// - the sponge, THREADED: the chains run `chain_sponge` into this one schedule +/// in order, so each enters what the last left. That is why this takes an +/// entry and hands one back, and why the chains' schedule halves are NOT +/// summed here as standalone numbers — a chain entered fresh reads a different +/// buffer at its first `state()`. +/// +/// `group_of[column]` names which columns share a claimed point, as +/// [`weight_at_rows`] takes it. +pub fn stacked_verify_cost( + layout: &StackedLayout, + group_of: &[usize], + shape: &ChainShape, + entry: SpongeEntry, +) -> StackedCost { + stacked_verify_cost_ood(layout, group_of, shape, entry, false) +} + +/// [`stacked_verify_cost`] with the stack's commit-time answers carried +/// (`ood`, [`OodWires`]): one more draw and its `P` powers; per chain one +/// `MulAdd` a polynomial on the claim; on the weight one `eq` over the stack +/// and one `MulAdd`, and in a run of several `P'` more rows for the scales +/// under `eq(ρ, p)`. Without, [`stacked_verify_cost`]'s rows. +pub fn stacked_verify_cost_ood( + layout: &StackedLayout, + group_of: &[usize], + shape: &ChainShape, + entry: SpongeEntry, + ood: bool, +) -> StackedCost { + let columns = layout.placements().len(); + let polys = layout.num_polys(); + + let mut schedule = SpongeSchedule::new(entry); + let mut ops = + columns * absorb_unpack_rows() + sample_ext_rows() + challenge_powers_rows(columns); + for _ in 0..columns { + schedule.absorb(COORDINATES_PER_EXT); + } + schedule.draw_ext(); + if ood { + // `γ'` and its powers `γ'^1 ..= γ'^P` (the `1` interned). + ops += sample_ext_rows() + challenge_powers_rows(polys + 1); + schedule.draw_ext(); + } + + // One chain per run of the verifier's split (B2-1): its roots' `Unpack`s, + // its claim over the run's columns, each polynomial's weight (and, for a + // run of several, the `eq(ρ, p)` combination of them), and its chain at + // the run's shape. + let runs = shape.chains(polys); + let mut chain_perms = 0; + for run in &runs { + let chain = shape.run(run.len()); + ops += run.len(); + ops += run + .clone() + .map(|poly| columns_of(layout, poly).len()) + .sum::(); + ops += run + .clone() + .map(|poly| weight_at_rows(layout, poly, group_of)) + .sum::(); + if run.len() >= 2 { + ops += run.len(); + } + if ood { + ops += run.len() + eq_eval_rows_again(layout.n_stack()) + 1; + if run.len() >= 2 { + ops += run.len(); + } + } + ops += chain_shape_rows(&chain); + chain_sponge(&chain, &mut schedule); + chain_perms += chain_opening_perms(&chain) + chain_grind_perms(&chain); + } + + StackedCost { + ops, + chain_perms, + schedule, + chains: runs.len(), + } +} + +/// Test only: the leg's mutations of the commit-time answers' batching, one at +/// a time; with any of them an honest opening must have no execution. +/// Thread-local, so a test that sets one moves no other test's program. +#[cfg(test)] +pub(crate) mod ood_faults { + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub(crate) enum Fault { + #[default] + None, + /// The chains' claims leave the answers out. + ClaimWithoutAnswers, + /// The chains' weights leave the answers' `eq` out. + WeightWithoutAnswers, + } + + std::thread_local! { + static FAULT: std::cell::Cell = const { std::cell::Cell::new(Fault::None) }; + } + + pub(crate) fn get() -> Fault { + FAULT.with(std::cell::Cell::get) + } + + /// Runs `f` under `fault`, restoring the previous one after. + pub(crate) fn with(fault: Fault, f: impl FnOnce() -> R) -> R { + struct Restore(Fault); + impl Drop for Restore { + fn drop(&mut self) { + FAULT.with(|c| c.set(self.0)); + } + } + let _restore = Restore(FAULT.with(|c| c.replace(fault))); + f() + } +} diff --git a/prover/src/lfm/whir_stacked_tests.rs b/prover/src/lfm/whir_stacked_tests.rs new file mode 100644 index 000000000..52ee651dc --- /dev/null +++ b/prover/src/lfm/whir_stacked_tests.rs @@ -0,0 +1,1122 @@ +//! Gates for the stacked-evaluation weight, and for the wrapper around it. +//! +//! `weight_at` is private to the multilinear crate, so the gate is against the +//! public pair that DEFINES it — `weight_table`, the table the prover actually +//! commits, evaluated at the same point. That is a stronger comparison than the +//! host's own closed form would be: it catches an error the two closed forms +//! could share. +//! +//! The wrapper's gates are further down and are against `stacked_eval::verify` +//! itself, on a real `stacked_eval::prove` proof. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; +use multilinear::mle::Mle; +use multilinear::stacked_eval::{ + Claimed, StackedCommitment, StackedProof, WeightShare, weight_table, +}; +use multilinear::stacking::StackedLayout; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::RpxWhir; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::{commitment_to_digest, leaf_capacity}; +use super::builder::{Cell, Ext, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_chain::{ + ChainRoundWires, ChainShape, RoundStorage, chain_hash_schedule, push_round_words, +}; +use super::whir_chain_tests::{Recording, chain_program, const_rows, hint_rows, perm_rows}; +use super::whir_stacked::{ + ColumnClaim, StackedPolyWires, emit_stacked_verify, emit_weight_at, stacked_verify_cost, + weight_at_consts, weight_at_rows, +}; +use super::whir_transcript::{SpongeEntry, SpongeHash, WhirTranscript}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type HostTranscript = DefaultTranscript; + +fn fee(v: u64) -> FEE { + FEE::new([ + FE::from(v.wrapping_mul(6364136223846793005) >> 11), + FE::from(v ^ 0xA5A5), + FE::from(v.wrapping_add(7)), + ]) +} + +/// One shape under test: a layout, which claimed point each column shares, and +/// the number of variables each of those points carries. +struct Shape { + name: &'static str, + layout: StackedLayout, + group_of: Vec, + group_vars: Vec, +} + +fn shape(name: &'static str, heights: &[usize], n_stack: usize, group_of: Vec) -> Shape { + let layout = StackedLayout::build(heights, n_stack).expect("the layout packs"); + let groups = group_of.iter().copied().max().map(|g| g + 1).unwrap_or(0); + let mut group_vars = vec![usize::MAX; groups]; + for (column, &group) in group_of.iter().enumerate() { + let vars = layout.placements()[column].num_vars; + if group_vars[group] == usize::MAX { + group_vars[group] = vars; + } + assert_eq!( + group_vars[group], vars, + "{name}: columns sharing a point must share a height" + ); + } + Shape { + name, + layout, + group_of, + group_vars, + } +} + +/// The shapes: one polynomial with every column the same height and two +/// distinct points; columns of MIXED heights, so prefix lengths differ inside +/// one polynomial; and a stack that spills into more than one polynomial. +fn shapes() -> Vec { + vec![ + shape( + "eight equal columns, two points", + &[3; 8], + 6, + (0..8).map(|c| c % 2).collect(), + ), + shape( + "mixed heights, one point each", + &[4, 3, 3, 2, 2, 5], + 6, + vec![0, 1, 1, 2, 2, 3], + ), + shape( + "twelve columns spilling into two polynomials", + &[3; 12], + 6, + (0..12).map(|c| c % 3).collect(), + ), + shape("one column filling the stack", &[5, 5], 5, vec![0, 1]), + ] +} + +fn arena_words(shape: &Shape) -> usize { + shape.layout.n_stack() + shape.group_vars.iter().sum::() + shape.group_of.len() +} + +/// Hints `at`, then each group's point, then every column's weight — the order +/// [`weight_arena`] fills. +fn weight_program(shape: &Shape, poly: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(arena_words(shape) as u32); + let mut idx = 0u32; + let next = |b: &mut LfmBuilder, idx: &mut u32| -> Ext { + let cell = b.hint_word(arena, *idx).as_ext(); + *idx += 1; + cell + }; + + let mut at = Vec::with_capacity(shape.layout.n_stack()); + for _ in 0..shape.layout.n_stack() { + at.push(next(&mut b, &mut idx)); + } + let mut points: Vec> = Vec::with_capacity(shape.group_vars.len()); + for &vars in &shape.group_vars { + let mut point = Vec::with_capacity(vars); + for _ in 0..vars { + point.push(next(&mut b, &mut idx)); + } + points.push(point); + } + let mut weights = Vec::with_capacity(shape.group_of.len()); + for _ in 0..shape.group_of.len() { + weights.push(next(&mut b, &mut idx)); + } + + let claims: Vec> = shape + .group_of + .iter() + .enumerate() + .map(|(column, &group)| ColumnClaim { + point: &points[group], + weight: weights[column], + }) + .collect(); + + let weight = emit_weight_at(&mut b, &shape.layout, poly, &claims, &at); + b.public(weight.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the weight leg must be admissible"); + program +} + +fn weight_arena(at: &[FEE], points: &[Vec], weights: &[FEE]) -> Vec { + let mut words: Vec = at.iter().map(ext_word).collect(); + for point in points { + words.extend(point.iter().map(ext_word)); + } + words.extend(weights.iter().map(ext_word)); + words +} + +/// ★ F1 for the weight, on every shape and every polynomial of it. +#[test] +fn the_weight_leg_emits_its_closed_form() { + for shape in shapes() { + for poly in 0..shape.layout.num_polys() { + let program = weight_program(&shape, poly); + let measured = program.instrs.len() - arena_words(&shape) - 1; + let predicted = + weight_at_rows(&shape.layout, poly, &shape.group_of) + weight_at_consts(); + println!( + "weight [{}] poly {poly}: {measured:>4} rows emitted, {predicted:>4} predicted", + shape.name + ); + assert_eq!( + measured, predicted, + "{}: polynomial {poly} must emit its closed form", + shape.name + ); + } + } +} + +/// ★ The leg computes what the committed weight table evaluates to. +#[test] +fn the_weight_leg_computes_what_the_host_computes() { + for shape in shapes() { + let n_stack = shape.layout.n_stack(); + for seed in [0x31u64, 0x77] { + let at: Vec = (0..n_stack).map(|i| fee(seed + i as u64)).collect(); + let points: Vec> = shape + .group_vars + .iter() + .enumerate() + .map(|(group, &vars)| { + (0..vars) + .map(|i| fee(seed * 13 + (group * 100 + i) as u64)) + .collect() + }) + .collect(); + let weights: Vec = (0..shape.group_of.len()) + .map(|c| fee(seed * 29 + 1 + c as u64)) + .collect(); + + for poly in 0..shape.layout.num_polys() { + let shares: Vec> = shape + .layout + .placements() + .iter() + .enumerate() + .filter(|(_, place)| place.poly == poly) + .map(|(column, place)| WeightShare { + offset: place.offset, + point: &points[shape.group_of[column]], + scale: weights[column], + }) + .collect(); + let table: Mle<_> = + weight_table(&shares, n_stack).expect("the weight table builds"); + let want = table + .evaluate(&at) + .expect("the table takes the stack's point"); + + let program = weight_program(&shape, poly); + let arena = weight_arena(&at, &points, &weights); + let exec = execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER) + .expect("the weight leg executes"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published weight"); + assert_eq!( + got, want, + "{}: polynomial {poly} at seed {seed:#x} disagrees with the committed \ + weight table", + shape.name + ); + } + } + } +} + +/// ★ `multilinear::challenge_powers` is REACHABLE from this crate, and means +/// what the batching weights need it to mean. +/// +/// Without this, item 0's visibility change is a check that cannot fail: a +/// `pub` item with no consumer draws no warning and no test, so "the emitter +/// can gate against it" would be a claim nothing stands behind until the +/// stacked wiring lands. This is the cheapest thing that fails if the +/// visibility is reverted — it would not compile — and it pins the semantics +/// the γ-powers leg will be gated against: `[1, γ, γ², …]`, the FIRST weight +/// one and not γ, which is the end that is easy to get wrong. +#[test] +fn challenge_powers_is_reachable_and_starts_at_one() { + let gamma = fee(0x51); + let powers = multilinear::challenge_powers::< + math::field::extensions_goldilocks::Degree3GoldilocksExtensionField, + >(&gamma, 4); + let want = [FEE::one(), gamma, gamma * gamma, gamma * gamma * gamma]; + assert_eq!(powers.len(), want.len(), "one weight per source"); + for (i, (got, expected)) in powers.iter().zip(&want).enumerate() { + assert_eq!(got, expected, "gamma^{i}"); + } +} + +// ───────────────────────── `stacked_eval::verify`, whole ───────────────────── +// +// The weight above is one closure inside it. What follows gates the wrapper: +// the column values absorbed, the batching draw, and one chain per stacked +// polynomial with the transcript THREADED through them. + +/// One group under test: its columns' heights, the stack they pack into, which +/// columns settle at a shared point, and the chain posture. +struct Group { + name: &'static str, + heights: Vec, + n_stack: usize, + /// `group_of[column]` names the claimed point the column shares. Columns in + /// one group must share a height, because a point is on the column's own + /// cube. + group_of: Vec, + num_queries: usize, + grind: u8, +} + +/// The groups: one polynomial with a single shared point (the epoch's common +/// case — one table's sumcheck leaves all of its columns at one point); twelve +/// columns SPILLING into two polynomials, which is the only shape that pays the +/// threading twice; mixed heights, so prefix lengths differ inside one +/// polynomial and three distinct points are live at once; and one column +/// FILLING the stack, whose prefix indicator is empty. +/// +/// ⚠ Grind 0 is not a smaller grind 8, for the chain suite's own reason: with no +/// query grind the query phase is entered with a candidate still in hand, which +/// is a branch of the schedule the production posture never takes. +fn groups() -> Vec { + vec![ + Group { + name: "six equal columns, one polynomial, one shared point", + heights: vec![3; 6], + n_stack: 6, + group_of: vec![0; 6], + num_queries: 3, + grind: 8, + }, + Group { + name: "twelve columns spilling into two polynomials, three points", + heights: vec![3; 12], + n_stack: 6, + group_of: (0..12).map(|c| c % 3).collect(), + num_queries: 3, + grind: 8, + }, + Group { + name: "mixed heights, three points, no grind", + heights: vec![4, 3, 3, 2, 2], + n_stack: 6, + group_of: vec![0, 1, 1, 2, 2], + num_queries: 3, + grind: 0, + }, + Group { + name: "one column filling the stack", + heights: vec![5], + n_stack: 5, + group_of: vec![0], + num_queries: 3, + grind: 8, + }, + ] +} + +fn group_config(group: &Group) -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: group.num_queries, + grind: GrindBits::uniform(group.grind), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + } +} + +fn pseudo_column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..1u64 << num_vars) + .map(|i| { + FE::from( + i.wrapping_mul(0x2545_F491_4F6C_DD1D) + .wrapping_add(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)) + >> 13, + ) + }) + .collect(), + ) + .expect("a cube") +} + +/// Everything one group's run needs on both sides. +struct Fixture { + name: &'static str, + layout: StackedLayout, + group_of: Vec, + /// One claimed point per GROUP; columns of a group share its wires, which + /// is what `emit_weight_at` groups on. + points: Vec>, + values: Vec, + root_words: Vec, + proof: StackedProof, + domain: Domain, + shape: ChainShape, + /// The batching challenge the HOST verifier drew, read off the recorder + /// rather than recomputed. + gamma: FEE, + /// The hashes the host's own transcript performed, reconstructed from the + /// calls a real `stacked_eval::verify` made. + host_hashes: Vec, +} + +/// Commits, proves and verifies one group on the host, keeping the verifier's +/// draws and its hash events. +/// +/// ⚠ Under `RpxTranscriptHash` and `RpxWhir`, not the default keccak pair — the +/// machine replays that hash and no other, so a fixture on the default +/// transcript would be a fixture of a different protocol. +fn fixture(group: &Group) -> Fixture { + let config = group_config(group); + let layout = StackedLayout::build(&group.heights, group.n_stack).expect("the layout packs"); + let columns: Vec> = group + .heights + .iter() + .enumerate() + .map(|(c, &height)| pseudo_column(height, 17 + 13 * c as u64)) + .collect(); + let borrowed = multilinear::stacking::borrow(&columns); + + let count = group.group_of.iter().copied().max().map_or(0, |g| g + 1); + let mut points: Vec> = Vec::with_capacity(count); + for g in 0..count { + let first = group + .group_of + .iter() + .position(|&x| x == g) + .expect("every group has a column"); + let vars = group.heights[first]; + for (column, &of) in group.group_of.iter().enumerate() { + assert!( + of != g || group.heights[column] == vars, + "{}: columns sharing a point share a height", + group.name + ); + } + points.push( + (0..vars) + .map(|i| fee(0x5eed + (g * 37 + i) as u64)) + .collect(), + ); + } + let per_column: Vec> = group.group_of.iter().map(|&g| points[g].clone()).collect(); + let values: Vec = columns + .iter() + .enumerate() + .map(|(c, column)| { + column + .evaluate_in::(&per_column[c]) + .expect("a column takes its own point") + }) + .collect(); + + let stacked = StackedCommitment::::commit(layout.clone(), &borrowed, None, &config) + .expect("the group commits"); + let roots = stacked.roots(); + let mut proving = HostTranscript::new(&[]); + let proof = multilinear::stacked_eval::prove::( + &stacked, + &borrowed, + None, + &Claimed::PerColumn(&per_column), + &values, + &config, + &mut proving, + ) + .expect("the group proves"); + + let mut recorded = Recording::new(); + multilinear::stacked_eval::verify::( + &proof, + stacked.layout(), + &roots, + &Claimed::PerColumn(&per_column), + &values, + stacked.domain(), + &config, + &mut recorded, + ) + .expect("the control proof must verify"); + + // ★ The host's draw stream has the shape the emitter's order assumes: the + // batching challenge FIRST, then each chain's own. Derived from the + // structure, not read off the recorder. + let shape = ChainShape::new(&config, layout.n_stack()); + let polys = layout.num_polys(); + assert_eq!( + recorded.sampled.len(), + 1 + polys * (shape.num_vars + 2 * (shape.rounds() - 1)), + "{}: extension draws are gamma, then one a sumcheck round plus z0 and gamma on each \ + round with a successor, in each of the {polys} chains", + group.name + ); + + let host_hashes = recorded.duplex.borrow().hashes.clone(); + Fixture { + name: group.name, + layout, + group_of: group.group_of.clone(), + points, + values, + root_words: roots.iter().map(commitment_to_digest).collect(), + proof, + domain: stacked.domain().clone(), + shape, + gamma: recorded.sampled[0], + host_hashes, + } +} + +/// Where every wire of the wrapper lives. Built once and used by both the +/// program and the arena filler, so the two cannot drift. +struct Arena { + group_at: Vec, + values_at: u32, + poly_at: Vec, + total: u32, +} + +impl Arena { + fn new(fixture: &Fixture) -> Self { + let mut at = 0u32; + let mut group_at = Vec::with_capacity(fixture.points.len()); + for point in &fixture.points { + group_at.push(at); + at += point.len() as u32; + } + let values_at = at; + at += fixture.values.len() as u32; + let mut poly_at = Vec::with_capacity(fixture.layout.num_polys()); + for _ in 0..fixture.layout.num_polys() { + poly_at.push(at); + // The root, the final value, then the chain's rounds. + at += 2 + RoundStorage::words(&fixture.shape); + } + Self { + group_at, + values_at, + poly_at, + total: at, + } + } +} + +/// Rows the wrapper PROGRAM carries that are not the leg's own cost: one `Hint` +/// per arena word, and the published batching challenge. +/// +/// The root `Unpack`s and the weight closure are NOT subtracted here — unlike +/// the chain's plumbing, they belong to this leg and its form charges them. +fn plumbing(arena: &Arena) -> usize { + arena.total as usize + 1 +} + +fn stacked_program(fixture: &Fixture) -> LfmProgram { + let at = Arena::new(fixture); + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(at.total); + let mut transcript = WhirTranscript::new(); + + let points: Vec> = fixture + .points + .iter() + .enumerate() + .map(|(g, point)| { + (0..point.len() as u32) + .map(|i| b.hint_word(arena, at.group_at[g] + i).as_ext()) + .collect() + }) + .collect(); + let values: Vec = (0..fixture.values.len() as u32) + .map(|i| b.hint_word(arena, at.values_at + i).as_ext()) + .collect(); + + let mut roots: Vec = Vec::new(); + let mut finals: Vec = Vec::new(); + let mut storage: Vec = Vec::new(); + for poly in 0..fixture.layout.num_polys() { + let base = at.poly_at[poly]; + roots.push(b.hint_word(arena, base)); + finals.push(b.hint_word(arena, base + 1).as_ext()); + storage.push(RoundStorage::hint(&mut b, arena, base + 2, &fixture.shape)); + } + + let openings: Vec<_> = storage.iter().map(RoundStorage::openings).collect(); + let wires: Vec>> = storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + let polys: Vec> = (0..fixture.layout.num_polys()) + .map(|poly| StackedPolyWires { + rounds: &wires[poly], + root: roots[poly], + final_value: finals[poly], + }) + .collect(); + let claimed_at: Vec<&[Ext]> = fixture + .group_of + .iter() + .map(|&g| points[g].as_slice()) + .collect(); + + let gamma = emit_stacked_verify( + &mut b, + &mut transcript, + &fixture.layout, + &polys, + &claimed_at, + &values, + &fixture.shape, + &fixture.domain, + ); + b.public(gamma.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the stacked leg must be admissible"); + program +} + +/// The arena in the order [`stacked_program`] hints it. +fn stacked_arena(fixture: &Fixture) -> Vec { + let mut words: Vec = Vec::new(); + for point in &fixture.points { + words.extend(point.iter().map(ext_word)); + } + words.extend(fixture.values.iter().map(ext_word)); + for (poly, chain) in fixture.proof.polys.iter().enumerate() { + words.push(fixture.root_words[poly]); + words.push(ext_word(&chain.final_value)); + push_round_words(&mut words, &fixture.shape, chain); + } + words +} + +fn const_words(program: &LfmProgram) -> Vec { + program + .instrs + .iter() + .filter_map(|instr| match instr { + super::instr::Instr::Const { value, .. } => Some(*value), + _ => None, + }) + .collect() +} + +/// ★ GATE ONE: the wrapper executes on a proof `stacked_eval::verify` accepts, +/// and draws the batching challenge the HOST drew. +/// +/// Every challenge the machine derives feeds a refusal — the chain's final +/// check, its out-of-domain checks, its openings — so executing IS the stream +/// comparison. The published gamma turns the one thing that would otherwise +/// show up only as an absence of execution, the position of the draw relative +/// to the column absorbs, into a named assertion. +#[test] +fn the_stacked_verify_executes_on_a_proof_the_host_accepts() { + for group in groups() { + let fixture = fixture(&group); + let program = stacked_program(&fixture); + let arena = stacked_arena(&fixture); + let exec = + execute(&program, &[arena], &crate::hash_pin::BLOCK_HASHER).unwrap_or_else(|e| { + panic!( + "{}: the machine must execute the host's proof: {e:?}", + fixture.name + ) + }); + let gamma = word_as_ext(&exec.public_words[0].1).expect("a published challenge"); + assert_eq!( + gamma, fixture.gamma, + "{}: the batching challenge must be the one the host verifier sampled", + fixture.name + ); + } +} + +/// ★ GATE TWO: the THREADED schedule is the host's own. +/// +/// `HostDuplex` reconstructs the host's hash events from the calls a real +/// `stacked_eval::verify` makes, and is itself checked against a shadow +/// transcript squeezed at exactly those points. This compares the wrapper's +/// schedule against it event for event and felt count for felt count — which is +/// what makes the threading a measurement rather than a claim: a chain entered +/// fresh reads a different buffer at its very first hash. +#[test] +fn the_threaded_schedule_is_the_host_transcripts() { + for group in groups() { + let fixture = fixture(&group); + let cost = stacked_verify_cost( + &fixture.layout, + &fixture.group_of, + &fixture.shape, + SpongeEntry::fresh(), + ); + let mine = cost.schedule().hashes(); + let squeezes = mine + .iter() + .filter(|h| matches!(h, SpongeHash::Squeeze(_))) + .count(); + println!( + "stacked [{}]: {} chains, {squeezes} squeezes, {} state reads, {} schedule rows, \ + {} schedule permutations", + fixture.name, + cost.chains(), + mine.len() - squeezes, + cost.schedule().rows(), + cost.schedule().perms(), + ); + + assert_eq!( + mine.len(), + fixture.host_hashes.len(), + "{}: the host performs {} transcript hashes and the form says {}", + fixture.name, + fixture.host_hashes.len(), + mine.len() + ); + for (i, (derived, observed)) in mine.iter().zip(&fixture.host_hashes).enumerate() { + assert_eq!( + derived, + observed, + "{}: hash {i} of {} — the form says {derived:?} and the host's own transcript \ + did {observed:?}", + fixture.name, + fixture.host_hashes.len() + ); + } + } +} + +/// ★ GATE THREE: the rows and the permutations against the program emitted. +/// +/// Pinned apart for the chain leg's reason: a form that moved work between the +/// sponge and the arithmetic at constant total would fail one of them rather +/// than neither. +#[test] +fn the_stacked_verify_emits_its_closed_form() { + for group in groups() { + let fixture = fixture(&group); + let program = stacked_program(&fixture); + let at = Arena::new(&fixture); + let cost = stacked_verify_cost( + &fixture.layout, + &fixture.group_of, + &fixture.shape, + SpongeEntry::fresh(), + ); + + let hints = hint_rows(&program); + assert_eq!( + hints, at.total as usize, + "{}: every arena word is hinted exactly once — the subtraction below is only honest \ + while this holds", + fixture.name + ); + let consts = const_rows(&program); + let measured = program.instrs.len() - consts - plumbing(&at); + let perms = perm_rows(&program); + + println!( + "stacked [{}]: {measured} rows emitted ({consts} constants, {hints} hints), \ + {} predicted; {perms} permutations, {} predicted; {} instructions", + fixture.name, + cost.operations(), + cost.perms(), + program.instrs.len(), + ); + + assert_eq!( + measured, + cost.operations(), + "{}: rows (const-free, the convention `chain_rows` is in)", + fixture.name + ); + assert_eq!(perms, cost.perms(), "{}: permutations", fixture.name); + } +} + +/// ★ GATE THREE's other half: the program interns EXACTLY the constants the +/// chain's arithmetic does, plus the two kinds this leg names. +/// +/// Instance 63's question, asked at the wrapper: a count that comes up short is +/// a question and not a fudge, so this asserts the SET rather than the number. +/// Both sides are derived independently — the left from a one-chain program at +/// the same shape, with its own sponge's leaf capacities removed because a chain +/// entered FRESH hashes different lengths than one entered mid-transcript; the +/// right from `StackedCost::own_constants`. +/// +/// ★ What it establishes for a census: the pool does NOT grow with the number of +/// chains. Every chain of a group has the same shape and the same domain, so +/// their constants collide in the one pool a program has — which is why a sum +/// over a group's chains adds a chain's constants ONCE and not once each. +#[test] +fn the_wrapper_interns_the_chains_constants_once_and_names_its_own() { + for group in groups() { + let fixture = fixture(&group); + let program = stacked_program(&fixture); + let cost = stacked_verify_cost( + &fixture.layout, + &fixture.group_of, + &fixture.shape, + SpongeEntry::fresh(), + ); + + let standalone = chain_program(&fixture.shape); + let chain_leaves: Vec = chain_hash_schedule(&fixture.shape, SpongeEntry::fresh()) + .iter() + .map(|hash| leaf_capacity(hash.felts())) + .collect(); + let arithmetic: Vec = const_words(&standalone) + .into_iter() + .filter(|word| !chain_leaves.contains(word)) + .collect(); + + let mut predicted = arithmetic.clone(); + for word in cost.own_constants() { + if !predicted.contains(&word) { + predicted.push(word); + } + } + + let interned = const_words(&program); + println!( + "stacked [{}]: {} constants interned, {} predicted ({} the chain's arithmetic + {} \ + named by this leg), over {} chains", + fixture.name, + interned.len(), + predicted.len(), + arithmetic.len(), + cost.own_constants().len(), + cost.chains(), + ); + + for word in &interned { + assert!( + predicted.contains(word), + "{}: the program interns {word:?}, which the form does not name — ask which \ + constant it is before adding a number", + fixture.name + ); + } + for word in &predicted { + assert!( + interned.contains(word), + "{}: the form names {word:?} and the program does not intern it", + fixture.name + ); + } + assert_eq!( + interned.len(), + predicted.len(), + "{}: the pools are the same size", + fixture.name + ); + } +} + +/// ★ The PRODUCTION cross-check, from the form alone: the DECODE group's +/// transcript counters as the box measured them. +/// +/// Opening DECODE's out-of-band commitment is a WHIR chain per epoch inside the +/// stacked wrapper (`n_stack` 23, five columns, one polynomial), and the box's +/// `hash-metrics` counters read +202 squeezes and +17 state reads per epoch over +/// the run without it — the second production point recorded in +/// `V1-handoff-2026-09-18-V1d.md`. Those are `stacked_eval::verify`'s totals and +/// not the chain's: a chain-only form is one squeeze short at that boundary, +/// which is the boundary and not a discrepancy. +/// +/// Needs no proof, no commitment and no guest ELF — the schedule is a function +/// of the round structure, the column count and the polynomial count. ⚠ The +/// column HEIGHTS below are not DECODE's; they are any heights that pack five +/// columns into one polynomial of the right stack, and no term of the schedule +/// reads them. +#[test] +fn the_decode_groups_threaded_schedule_reproduces_the_box() { + // The posture the whole VM proof runs at, read from `chain_config` + // (`multilinear_prove.rs:93`): blowup 2, fold 4, 128 bits, 20-bit grinds, + // with the query count sized by the TALLEST stack in the proof, 25. + let config = ChainConfig::with_security(2, 4, 25, 128, GrindBits::uniform(20)); + let layout = StackedLayout::build(&[20; 5], 23).expect("five columns pack into one stack"); + assert_eq!( + layout.num_polys(), + 1, + "the DECODE group is one stacked polynomial" + ); + let shape = ChainShape::new(&config, 23); + assert_eq!(shape.rounds(), 6, "n_stack 23 at fold 4 is six rounds"); + assert_eq!( + shape.num_queries, 112, + "the shipped posture's query count for a seven-round tallest stack" + ); + + let cost = stacked_verify_cost(&layout, &[0; 5], &shape, SpongeEntry::fresh()); + let hashes = cost.schedule().hashes(); + let squeezes = hashes + .iter() + .filter(|hash| matches!(hash, SpongeHash::Squeeze(_))) + .count(); + let states = hashes.len() - squeezes; + println!( + "DECODE group: {squeezes} squeezes, {states} state reads, {} schedule rows, \ + {} permutations whole, chain rows {}", + cost.schedule().rows(), + cost.perms(), + cost.operations(), + ); + + assert_eq!( + squeezes, 202, + "the chain's 201 plus the wrapper's own batching draw" + ); + assert_eq!(states, 17, "3R - 1 grinds, each reading the sponge once"); +} + +/// ★★ THE PIN: what the routing rule CHARGES a carried page, against what the +/// stack BILLS for one — read off [`stacked_verify_cost`], never asserted +/// against the form's own arithmetic. +/// +/// `continuation::marginal_stacked_rows` is part 1's whole term, and until this +/// ran it was four terms READ off this file's cost form by hand. Two earlier +/// readings of it disagreed, each missing a term the other had, so the form is +/// pinned here by DIFFERENCING the cost of one more carried page — the only +/// definition of "marginal" that cannot miss a term. +/// +/// ⛔⛔ WHY IT IS A MAXIMUM AND NOT AN EQUALITY AT EVERY PAGE. The wrapper +/// absorbs `COORDINATES_PER_EXT` per column into ONE threaded sponge and +/// squeezes once, at `f.div_ceil(4) + f.div_ceil(8) + 1`. A page is two +/// columns, six felts, which divides neither 4 nor 8 — so the cost STEPS +/// irregularly as pages are added and the per-page marginal is a SPREAD, not a +/// number. The form charges the dearest position, which is what +/// `continuation::MAX_SPONGE_MARGINAL` bounds; this walks every position in a +/// bracket and reads the spread out loud. +/// +/// ⛔ AND A MARGINAL IS ONLY DEFINED INSIDE A BRACKET. The prefix-indicator term +/// grows two rows per prefix bit, so the difference across a power-of-two +/// boundary carries a stack-height change that is not a page's cost at all. +/// Every difference here is taken between two page counts whose stack stands at +/// the SAME height and in ONE polynomial, both asserted per page count rather +/// than assumed from the range. +/// +/// Every bracket a block-shaped run can stand in as ONE polynomial is walked: +/// from 24 variables (17..=32 genesis pages, where the block's thirty sit) to +/// the production format's stack, `whir_stack` (27: 129..=256 pages, the last +/// bracket that fits one polynomial). +#[test] +fn the_marginal_the_routing_rule_charges_is_the_one_the_stack_bills() { + use crate::continuation::{ + BLOCK_STACK_VARS, MAX_SPONGE_MARGINAL, PAGE_NUM_VARS, PAGE_PREPROCESSED_COLUMNS, + STACKED_ROWS_PER_COLUMN, candidate_threshold_entries, fixed_stack_vars, + marginal_stacked_rows, + }; + + // The posture the whole VM proof runs at (`multilinear_prove.rs:93`), and a + // deliberately different one: the marginal must not move with either, since + // every chain term is per-POLYNOMIAL and there is one polynomial throughout. + // Both stack under the production format's cap, so every single-chain + // bracket under it is reachable. + let stack = crate::zf_format::ZfFormat::DEFAULT.whir_stack; + let at_stack = |mut config: ChainConfig| { + config.format.stack = stack; + config + }; + let shipped = at_stack(ChainConfig::with_security( + 2, + 4, + 25, + 128, + GrindBits::uniform(20), + )); + let other = at_stack(ChainConfig::with_security( + 1, + 3, + 25, + 100, + GrindBits::uniform(16), + )); + // The most whole pages one polynomial holds: `2 * pages * 2^18` cells fit + // `2^stack`. + let widest = 1usize << (stack.get() - PAGE_NUM_VARS - 1); + + let cost_at = |pages: usize, config: &ChainConfig| -> (usize, usize) { + let columns = pages * PAGE_PREPROCESSED_COLUMNS; + let layout = stark::multilinear_table::global_layout( + &[(columns, PAGE_NUM_VARS)], + config.format.stack, + ) + .expect("a stack of whole pages"); + // ⛔ THE BRACKET GUARD, PER PAGE COUNT. A difference taken across a + // height change or a polynomial split is not a page's marginal. + assert_eq!( + layout.num_polys(), + 1, + "{pages} pages spill into {} polynomials, so this difference would span \ + two chains", + layout.num_polys() + ); + // A page's two preprocessed columns settle at ONE point — its own + // table's — so the groups are the pages. + let group_of: Vec = (0..columns) + .map(|column| column / PAGE_PREPROCESSED_COLUMNS) + .collect(); + let shape = ChainShape::new(config, layout.n_stack()); + let cost = stacked_verify_cost(&layout, &group_of, &shape, SpongeEntry::fresh()); + (layout.n_stack(), cost.operations()) + }; + + for bracket in BLOCK_STACK_VARS..=stack.get() { + let counts: Vec = (1..=widest) + .filter(|&pages| fixed_stack_vars(PAGE_NUM_VARS, pages) == bracket) + .collect(); + assert!( + counts.len() > MAX_SPONGE_MARGINAL + 1, + "a bracket of {} page counts cannot show a spread of {MAX_SPONGE_MARGINAL} \ + values", + counts.len() + ); + + // The deterministic part, which is every term but the sponge's. + let deterministic = 5 * PAGE_NUM_VARS + + PAGE_PREPROCESSED_COLUMNS * (bracket - PAGE_NUM_VARS) + + PAGE_PREPROCESSED_COLUMNS * STACKED_ROWS_PER_COLUMN; + + let mut marginals: Vec = Vec::with_capacity(counts.len() - 1); + let mut sponge: Vec = Vec::with_capacity(counts.len() - 1); + let mut previous: Option = None; + for &pages in &counts { + let (n_stack, cost) = cost_at(pages, &shipped); + assert_eq!( + n_stack, bracket, + "{pages} pages stand at {n_stack}, not the \ + bracket being walked" + ); + // ⚠ THE POSTURE MUST NOT MOVE THE DIFFERENCE, executed rather than + // argued: the same page count under a different blowup, folding, + // security level and grind. + let (_, elsewhere) = cost_at(pages, &other); + if let Some(before) = previous { + marginals.push(cost - before); + sponge.push(cost - before - deterministic); + } + previous = Some(cost); + let _ = elsewhere; + } + + // The same walk under the other posture, differenced on its own. + let mut elsewhere_marginals: Vec = Vec::with_capacity(counts.len() - 1); + let mut previous: Option = None; + for &pages in &counts { + let (_, cost) = cost_at(pages, &other); + if let Some(before) = previous { + elsewhere_marginals.push(cost - before); + } + previous = Some(cost); + } + assert_eq!( + marginals, elsewhere_marginals, + "the per-page marginal moved with the chain posture, so a term that is \ + supposed to be per-polynomial is being charged per page" + ); + + let charged = marginal_stacked_rows(PAGE_NUM_VARS, bracket); + let measured_max = *marginals.iter().max().expect("a walked bracket"); + let measured_min = *marginals.iter().min().expect("a walked bracket"); + println!( + "MARGINAL PIN at bracket {bracket} ({} pages, {} differences): charged \ + {charged}, measured {measured_min}..={measured_max}, deterministic \ + {deterministic}, sponge term s = {sponge:?}", + counts.len(), + marginals.len(), + ); + + // (i) THE FORM IS THE MAXIMUM THE STACK BILLS — the assertion the whole + // rule rests on, and it is EXACT rather than a bound on a bound: the + // charge is reached at some position in the bracket, not merely never + // exceeded. + assert_eq!( + measured_max, charged, + "at bracket {bracket} the form charges {charged} and the dearest page the \ + stack bills is {measured_max}. If this is LARGER, part 1 undercharges \ + and the form must move; if SMALLER, the bound is loose and the entry \ + threshold is one too eager" + ); + + // (ii) THE SPONGE TERM IS INSIDE ITS BOUND AT EVERY POSITION, and the + // bound is reached: a bound never reached would be a charge no page can + // justify. + for (index, &s) in sponge.iter().enumerate() { + assert!( + (1..=MAX_SPONGE_MARGINAL).contains(&s), + "the {}th page of the bracket adds {s} sponge rows, outside \ + 1..={MAX_SPONGE_MARGINAL}", + counts[index + 1] + ); + } + assert_eq!( + *sponge.iter().max().expect("a walked bracket"), + MAX_SPONGE_MARGINAL + ); + + // (iii) τ IS INVARIANT ACROSS THE SPREAD, which is what makes charging + // the maximum cost nothing the rule can see: every position in the + // bracket yields the same entry threshold as the charged marginal does. + let tau_of = |m: usize| m.saturating_sub(PAGE_NUM_VARS) / PAGE_NUM_VARS + 1; + let tau = candidate_threshold_entries(PAGE_NUM_VARS, bracket); + assert_eq!( + tau_of(charged), + tau, + "the closure and the live function disagree" + ); + for &m in &marginals { + assert_eq!( + tau_of(m), + tau, + "at bracket {bracket} a page billed {m} rows implies τ = {}, against \ + the charged {charged}'s τ = {tau}: the choice of the maximum is no \ + longer free", + tau_of(m) + ); + } + } + + // ⛔ WHERE THIS PIN'S COVERAGE STOPS, READ RATHER THAN ASSUMED. One page + // past `widest` the stack wants one variable more than the production cap, + // `global_layout` caps it and the columns spill into a second polynomial — + // which is a second chain, so part 2's single-chain price stops being the + // cost. `whir_chain_tests::the_single_chain_term_prices_a_stack_of_at_most_ + // two_hundred_fifty_six_pages` is where that limit is stated; here it is + // only the end of the range a marginal is defined on. + let spilled = stark::multilinear_table::global_layout( + &[((widest + 1) * PAGE_PREPROCESSED_COLUMNS, PAGE_NUM_VARS)], + stack, + ) + .expect("a stack of whole pages"); + assert_eq!( + spilled.num_polys(), + 2, + "{} genesis pages must spill, or the bracket walk above stopped one short of \ + its own boundary", + widest + 1 + ); + assert_eq!(spilled.n_stack(), stack.get()); +} diff --git a/prover/src/lfm/whir_table.rs b/prover/src/lfm/whir_table.rs new file mode 100644 index 000000000..25c683e15 --- /dev/null +++ b/prover/src/lfm/whir_table.rs @@ -0,0 +1,715 @@ +//! The per-table verify as a machine leg. +//! +//! The mirror is `stark::multilinear_table::verify` +//! (`multilinear_table.rs:721-785`): the bus output absorbed, the GKR ladder, +//! the row challenges, the bus statements, the zerocheck rule, one batched +//! sumcheck over all three, and `claim_reduce` to leave every COLUMN claimed at +//! one point. +//! +//! This module starts with the one kernel none of the earlier legs has: the row +//! SELECTOR, which is what a public factor's value is. + +use multilinear::claim_reduce::FactorSource; +use multilinear::constraint_argument::FactorKind; +use multilinear::logup::input_layer_vars; +use multilinear::selector::Selector; +use stark::multilinear_air::IrShape; +use stark::multilinear_logup::InteractionShape; +use stark::multilinear_table::weight_slots; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::algebraic_commit::leaf_capacity; +use super::builder::{Ext, LfmBuilder}; +use super::whir_air::{combine_rows, emit_combine}; +use super::whir_bus::{BusInputs, Cost, claim_statements_cost, emit_claim_statements}; +use super::whir_gkr::{ + GKR_SUMCHECK_DEGREE, GkrLayerChallenges, GkrLayerWires, emit_gkr_verify, gkr_verify_rows, +}; +use super::whir_poly::{ + challenge_powers_rows, emit_challenge_powers, emit_eq_eval, emit_sumcheck_rounds, + eq_eval_rows_again, sumcheck_round_rows, +}; +use super::whir_reduce::{REDUCE_DEGREE, ReduceWires, claim_reduce_rows, emit_claim_reduce_verify}; +use super::whir_transcript::{ + COORDINATES_PER_EXT, SpongeEntry, SpongeSchedule, WhirTranscript, absorb_unpack_rows, + sample_ext_rows, +}; + +/// ★ `Selector::evaluate` (`crypto/multilinear/src/selector.rs:61-100`), +/// emitted. +/// +/// A constraint that reads the next row cannot apply on the last one, so it +/// carries `Selector { end_exemptions }` and the verifier needs that +/// indicator's value at the point its sumcheck settled on. The host computes it +/// in `O(num_vars)` rather than over the cube: +/// +/// ```text +/// s(x) = 1 − geq(x, cutoff), cutoff = 2^n − end_exemptions +/// ``` +/// +/// where `geq` is the multilinear indicator of `index(x) >= cutoff`: either `x` +/// matches `cutoff` bit for bit, or the two first differ at a position where +/// `cutoff` has a zero and `x` a one. `prefix` is the running "everything above +/// this bit matched"; VARIABLE 0 IS THE MOST SIGNIFICANT BIT. +/// +/// # ⚠ Why the literal values are tracked rather than emitted +/// +/// `prefix` starts at ONE and `geq` at ZERO. A row that multiplies a wire by a +/// literal one, or adds a literal zero to it, computes nothing — so this +/// carries each as `Option`, where `None` IS the literal the host starts +/// from, and spends a row only once the value is a wire. It is not an +/// optimisation bolted onto the host's expression; it is the host's expression +/// with the identity steps left out, and the cost form below counts the same +/// cases. +/// +/// Panics when there are more exemptions than rows — the host returns +/// `TooManyExemptions`, and here the two are emit-time numbers, so a mismatch +/// is a bug in the caller. +pub fn emit_selector(b: &mut LfmBuilder, selector: Selector, point: &[Ext]) -> Ext { + let num_vars = point.len(); + let size = 1usize << num_vars; + assert!( + selector.end_exemptions <= size, + "a selector cannot exempt more rows than the table has: {} of {size}", + selector.end_exemptions + ); + + let one = b.ext_const(&FEE::one()); + if selector.is_trivial() { + return one; + } + let cutoff = size - selector.end_exemptions; + if cutoff == 0 { + // Every row is exempt: the constraint applies nowhere. + return b.ext_const(&FEE::zero()); + } + + // `None` is the literal the host starts from: `prefix = 1`, `geq = 0`. + let mut prefix: Option = None; + let mut geq: Option = None; + for (i, x) in point.iter().enumerate() { + let bit = (cutoff >> (num_vars - 1 - i)) & 1; + if bit == 0 { + // `x` exceeds `cutoff` here when everything above matched and + // `x_i = 1`. + geq = Some(match (prefix, geq) { + (None, None) => *x, + (None, Some(running)) => b.eadd(*x, running), + (Some(above), None) => b.emul(above, *x), + (Some(above), Some(running)) => b.emul_add(above, *x, running), + }); + let one_minus = b.esub(one, *x); + prefix = Some(match prefix { + None => one_minus, + Some(above) => b.emul(above, one_minus), + }); + } else { + prefix = Some(match prefix { + None => *x, + Some(above) => b.emul(above, *x), + }); + } + } + + // Both branches write `prefix`, and a selector that is neither trivial nor + // all-exempt has at least one variable — `num_vars = 0` leaves `cutoff = 0` + // for every non-trivial exemption count, which returned above. + let prefix = prefix.expect("a live selector reads at least one variable"); + // The remaining prefix is the `x == cutoff` case. + let total = match geq { + None => prefix, + Some(running) => b.eadd(running, prefix), + }; + b.esub(one, total) +} + +/// INSTRUCTIONS [`emit_selector`] emits over `num_vars` variables, added to +/// `cost`. +/// +/// The same case analysis, counted rather than emitted: +/// +/// - a trivial selector is the interned `1` and nothing else; +/// - an all-exempt one is the interned `0` and nothing else; +/// - otherwise, per variable, a `1 − x_i` subtract at every zero bit of the +/// cutoff, plus one row for each of `geq` and `prefix` once that value has +/// stopped being a literal; +/// - then the `geq + prefix` add, which the all-ones cutoff does not need, and +/// the final subtract from one. +pub fn selector_cost(selector: Selector, num_vars: usize, cost: &mut Cost) { + let size = 1usize << num_vars; + assert!(selector.end_exemptions <= size, "more exemptions than rows"); + + cost.constant(FEE::one()); + if selector.is_trivial() { + return; + } + let cutoff = size - selector.end_exemptions; + if cutoff == 0 { + cost.constant(FEE::zero()); + return; + } + + let mut prefix_live = false; + let mut geq_live = false; + for i in 0..num_vars { + let bit = (cutoff >> (num_vars - 1 - i)) & 1; + if bit == 0 { + if prefix_live || geq_live { + cost.op(); + } + geq_live = true; + // `1 − x_i` is a row whatever the running product holds. + cost.op(); + if prefix_live { + cost.op(); + } + prefix_live = true; + } else { + if prefix_live { + cost.op(); + } + prefix_live = true; + } + } + + if geq_live { + cost.op(); + } + cost.op(); +} + +// --------------------------------------------------------------------------- +// The assembly: `multilinear_table::verify`, emitted. +// --------------------------------------------------------------------------- + +/// The `TableProof` as wires. +/// +/// Every field is proof data — a value the prover chose — which is what makes +/// the challenges below DERIVED rather than supplied: nothing the transcript +/// absorbs is computed by this leg. +pub struct TableProofWires<'a> { + /// The table's share of the bus, `(p, q)`. + pub bus_output: (Ext, Ext), + /// One per GKR ladder layer, output first; layer `i` carries `i` sumcheck + /// rounds. + pub gkr: &'a [GkrLayerWires], + /// The main batched sumcheck's rounds: `num_vars` of them, each carrying + /// `degree_of(rules)` evaluations. + pub sumcheck: &'a [Vec], + /// Each COMMITTED factor's value at the sumcheck point. The public factors + /// are absent — this leg computes those. + pub factor_values: &'a [Ext], + /// `claim_reduce`'s own proof. + pub reduce: ReduceWires<'a>, +} + +/// The table's emit-time structure: what `TableStatement` carries that is not +/// proof data. +pub struct TableShape<'a> { + /// The constraint DAG, its roots and their selectors. + pub ir: &'a IrShape, + /// The bus, with the challenges factored out (`interaction_shapes`). + pub bus: &'a [InteractionShape], + /// The factor list, committed and public interleaved as the rules index it. + pub kinds: &'a [FactorKind], + /// The table's committed columns. + pub num_columns: usize, + pub num_vars: usize, +} + +impl TableShape<'_> { + /// The GKR ladder's length — `logup::input_layer_vars`, the host's own. + pub fn gkr_layers(&self) -> usize { + input_layer_vars(self.bus.len(), self.num_vars) + } + + /// The main batched sumcheck's degree: `degree_of([zerocheck, numerator, + /// denominator])` (`batch.rs:264`), where the zerocheck rule is compiled at + /// `shape.degree() + 1` and both bus rules at 2. + pub fn sumcheck_degree(&self) -> usize { + (self.ir.degree() + 1).max(2) + } + + /// The committed factors' sources — `constraint_argument::sources_of`. + pub fn sources(&self) -> Vec { + self.kinds.iter().filter_map(FactorKind::source).collect() + } +} + +/// What a table's verify leaves for the caller. +pub struct TableVerdictWires { + /// The table's share of the bus, for the caller to sum. + pub bus_output: (Ext, Ext), + /// The point every COLUMN is now claimed at. + pub point: Vec, + pub column_values: Vec, +} + +/// `constraint_argument::weave` (`:103-136`), at emit time. +/// +/// The committed factors' values come out of the proof and the public ones are +/// computed; this puts them back in factor order, with any weight tables a +/// statement appended landing at the end. Costs NO rows: it is a reordering of +/// wires that already exist. +pub(crate) fn weave(kinds: &[FactorKind], committed: &[Ext], public: &[Ext]) -> Vec { + let want_committed = kinds.iter().filter(|k| k.source().is_some()).count(); + assert_eq!( + committed.len(), + want_committed, + "one proof value per committed factor" + ); + let want_public = kinds.len() - want_committed; + assert!( + public.len() >= want_public, + "one computed value per public factor, plus the weight tables" + ); + + let (public, weights) = public.split_at(want_public); + let mut committed = committed.iter(); + let mut public = public.iter(); + let mut values: Vec = kinds + .iter() + .map(|kind| match kind { + FactorKind::Committed(_) => committed.next(), + FactorKind::Public => public.next(), + }) + .map(|v| *v.expect("the counts were checked")) + .collect(); + values.extend(weights); + values +} + +/// ★ `stark::multilinear_table::verify` (`multilinear_table.rs:721-785`), +/// emitted. +/// +/// `z`, `alpha_powers` and `beta` are the LogUp and batching challenges, drawn +/// ONCE for the whole epoch and shared by every table, so they arrive as wires. +/// Everything else this leg needs it draws itself. +/// +/// # ★ The challenges are DERIVED, which is what makes an honest proof the gate +/// +/// The leg drives the transcript, so a wrong challenge anywhere leaves a +/// refusal the machine cannot satisfy: the GKR layer relation, the batch's +/// residual, and `claim_reduce`'s shifted read are three independent +/// divisions-by-zero. Executing a real proof is therefore the challenge-stream +/// comparison, and the verdict compared against the host's is the value gate. +/// +/// # ⚠ One reordering, stated so it is not mistaken for a difference +/// +/// The host interleaves the GKR ladder's transcript operations with its +/// arithmetic; this draws a layer's challenges and then runs +/// [`emit_gkr_verify`] over all the layers. The ORDER OF TRANSCRIPT OPERATIONS +/// is unchanged, which is the only order that is a property of the protocol: +/// `gkr::verify` absorbs nothing it computed — every absorb is proof data — so +/// where the arithmetic sits between the absorbs is not observable. A +/// straight-line program is a DAG, and this is the same non-difference V1d +/// recorded when moving two independent emissions past each other. +/// +/// # ⛔ `check_preprocessed` IS NOT EMITTED, and this is the seam +/// +/// The host evaluates each preprocessed column's MLE at the reduced point +/// (`check_preprocessed`), a full pass per column. A verifier that is itself +/// proven cannot pay it. The caller supplies the replacement and the two are +/// named here rather than anywhere else: BITWISE by +/// [`super::preprocessed::emit_bitwise_preprocessed`], DECODE by its own pinned +/// commitment group. A table with no preprocessed columns owes nothing, and the +/// gate below is on such a table — said so rather than left to read as +/// coverage. +/// +/// ★ The host now names half of that itself: `verify`'s `settled_out_of_band` +/// is how many LEADING preprocessed columns a prepared opening already settled +/// at this very point, and those are skipped. So DECODE's replacement is not a +/// thing this emitter invents — it is the argument the host takes, and the two +/// sides have to agree on the same count. What stays owed here is the +/// REMAINDER: every preprocessed column past that prefix is still an MLE pass +/// the host makes and this leg does not. +pub fn emit_table_verify( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + proof: &TableProofWires<'_>, + shape: &TableShape<'_>, + z: Ext, + alpha_powers: &[Ext], + beta: Ext, +) -> TableVerdictWires { + let num_vars = shape.num_vars; + let layers = shape.gkr_layers(); + assert_eq!( + proof.gkr.len(), + layers, + "one GKR layer per variable of the input layer" + ); + assert_eq!( + proof.sumcheck.len(), + num_vars, + "the main sumcheck runs over the table's rows" + ); + let degree = shape.sumcheck_degree(); + + // The table's share of the bus, before anything is drawn from it. + transcript.absorb_ext(b, proof.bus_output.0); + transcript.absorb_ext(b, proof.bus_output.1); + + // `gkr::verify`'s draws, layer by layer: the batching lambda, one challenge + // per sumcheck round, then the folding point behind the four halves. + let mut drawn: Vec = Vec::with_capacity(layers); + for layer in proof.gkr { + let lambda = transcript.sample_ext(b); + let mut rounds = Vec::with_capacity(layer.sumcheck.len()); + for evaluations in &layer.sumcheck { + for evaluation in evaluations { + transcript.absorb_ext(b, *evaluation); + } + rounds.push(transcript.sample_ext(b)); + } + for value in [layer.p_lo, layer.p_hi, layer.q_lo, layer.q_hi] { + transcript.absorb_ext(b, value); + } + let c = transcript.sample_ext(b); + drawn.push(GkrLayerChallenges { lambda, rounds, c }); + } + let gkr_claim = emit_gkr_verify(b, proof.bus_output, proof.gkr, &drawn); + + // The zerocheck's own weight point. + let r: Vec = (0..num_vars).map(|_| transcript.sample_ext(b)).collect(); + let (weight_r, weight_z) = weight_slots(shape.kinds.len()); + + // The row half of the GKR claim's point is where the bus's weight table + // belongs (`logup.rs:234`). + let row_point = &gkr_claim.point[gkr_claim.point.len() - num_vars..]; + let betas = emit_challenge_powers(b, beta, shape.ir.num_roots()); + + // `batch::verify`: the three claims, then the batching challenge. The + // zerocheck's claim is the LITERAL zero — the constraints must vanish — so + // it is a constant on both sides of the transcript. + let zero = b.ext_const(&FEE::zero()); + transcript.absorb_ext(b, zero); + transcript.absorb_ext(b, gkr_claim.p); + transcript.absorb_ext(b, gkr_claim.q); + let batching = transcript.sample_ext(b); + let lambdas = emit_challenge_powers(b, batching, RULES); + // `Σ lambda^i · claim_i`, and the first claim contributes nothing. + let claimed = b.emul(lambdas[1], gkr_claim.p); + let claimed = b.emul_add(lambdas[2], gkr_claim.q, claimed); + + let mut point: Vec = Vec::with_capacity(num_vars); + for evaluations in proof.sumcheck { + assert_eq!( + evaluations.len(), + degree, + "a round carries the batch's degree in evaluations" + ); + for evaluation in evaluations { + transcript.absorb_ext(b, *evaluation); + } + point.push(transcript.sample_ext(b)); + } + let residual = emit_sumcheck_rounds(b, claimed, proof.sumcheck, &point); + + // `values_at`: the selectors the AIR made public, then the two weight + // tables, woven back together with the committed values. + let mut public: Vec = shape + .ir + .public_selectors() + .iter() + .map(|selector| emit_selector(b, *selector, &point)) + .collect(); + public.push(emit_eq_eval(b, &r, &point)); + public.push(emit_eq_eval(b, row_point, &point)); + let values = weave(shape.kinds, proof.factor_values, &public); + + // The three rules at that point. `IrShape::program` applies the zerocheck's + // own weight, which `combine` does not (`multilinear_air.rs:889-916`). + let combined = emit_combine(b, shape.ir, &betas, &values); + let zerocheck = b.emul(values[weight_r], combined); + let bus = emit_claim_statements( + b, + shape.bus, + &BusInputs { + claim_point: &gkr_claim.point, + num_row_vars: num_vars, + z, + alpha_powers, + values: &values, + weight: weight_z, + }, + ); + + // `Σ lambda^i · rule_i`, and the first weight is the literal one. + let rebuilt = b.emul_add(lambdas[1], bus.numerator, zerocheck); + let rebuilt = b.emul_add(lambdas[2], bus.denominator, rebuilt); + // The host's `BatchMismatch`: a division by zero has no satisfying + // assignment, so a proof that fails it cannot be executed. + b.assert_eq_ext(rebuilt, residual); + + let reduced = emit_claim_reduce_verify( + b, + transcript, + &proof.reduce, + &shape.sources(), + proof.factor_values, + &point, + shape.num_columns, + ); + + TableVerdictWires { + bus_output: proof.bus_output, + point: reduced.point, + column_values: reduced.column_values, + } +} + +/// The three statements one table batches: the zerocheck and the bus's two. +const RULES: usize = 3; + +/// Rows an `assert_eq_ext` lowers to: the difference and the division by zero +/// (`builder.rs:289-292`). +const ASSERT_ROWS: usize = 2; + +/// What a table's verify costs: the straight-line rows, and the sponge. +/// +/// Two halves because they are two different measurements — the leg's rows are +/// a function of the shape alone, the sponge's of what the transcript was +/// holding when the table started, which is why [`table_verify_cost`] takes an +/// entry and hands one back. +pub struct TableCost { + pub leg: Cost, + pub schedule: SpongeSchedule, +} + +impl TableCost { + pub fn rows(&self) -> usize { + self.leg.rows() + self.schedule.rows() + } + + pub fn perms(&self) -> usize { + self.schedule.perms() + } + + /// Where the sponge is left, so the next table in the same program + /// continues from it. + pub fn entry(&self) -> SpongeEntry { + self.schedule.entry() + } +} + +/// INSTRUCTIONS and permutations [`emit_table_verify`] costs. +/// +/// Every term by the shape it comes from, with `L` the ladder's length, `V` the +/// table's variables and `d` the batch's degree: +/// +/// - the bus output: two absorbs; +/// - the ladder: per layer `i`, one draw, `i` rounds of three absorbs and a +/// draw, four absorbs and a draw — plus `gkr_verify_rows(L)` and its +/// constants; +/// - `V` draws for the zerocheck's weight point; +/// - the beta ladder, `challenge_powers_rows(num_roots)`; +/// - three claim absorbs and the batching draw, then `challenge_powers_rows(3)` +/// and TWO rows for `Σ lambda^i·claim_i`, because the zerocheck's claim is the +/// literal zero and contributes nothing; +/// - `V` rounds of `d` absorbs, a draw and `sumcheck_round_rows(d)`, plus the +/// round's interned Newton constants once; +/// - one selector per public factor, and two `eq`s for the weight tables; +/// - `combine_rows` and one multiply for the zerocheck rule's own weight; +/// - the bus statements, `claim_statements_cost`; +/// - two rows for `Σ lambda^i·rule_i`, the first weight being one, and the +/// refusal; +/// - `claim_reduce_rows`. +/// +/// The `LFM_CONST` for the zero claim is interned like any other, and shares +/// with whatever else in the program needs a zero. +pub fn table_verify_cost(shape: &TableShape<'_>, entry: SpongeEntry) -> TableCost { + let num_vars = shape.num_vars; + let layers = shape.gkr_layers(); + let degree = shape.sumcheck_degree(); + let mut leg = Cost::default(); + let mut schedule = SpongeSchedule::new(entry); + + // The bus output. + absorb_ext(&mut leg, &mut schedule, 2); + + // The ladder's transcript, then its arithmetic. + for i in 0..layers { + draw_ext(&mut leg, &mut schedule, 1); + for _ in 0..i { + absorb_ext(&mut leg, &mut schedule, GKR_SUMCHECK_DEGREE); + draw_ext(&mut leg, &mut schedule, 1); + } + absorb_ext(&mut leg, &mut schedule, 4); + draw_ext(&mut leg, &mut schedule, 1); + } + leg.ops(gkr_verify_rows(layers)); + + // The zerocheck's weight point. + draw_ext(&mut leg, &mut schedule, num_vars); + + // The beta ladder. + leg.ops(challenge_powers_rows(shape.ir.num_roots())); + + // The batch: three claims, the batching draw, its powers, and the fold — + // whose first term is the literal zero and costs nothing. + absorb_ext(&mut leg, &mut schedule, RULES); + draw_ext(&mut leg, &mut schedule, 1); + leg.ops(challenge_powers_rows(RULES)); + leg.ops(RULES - 1); + + // The main sumcheck. + for _ in 0..num_vars { + absorb_ext(&mut leg, &mut schedule, degree); + draw_ext(&mut leg, &mut schedule, 1); + leg.ops(sumcheck_round_rows(degree)); + } + + // `values_at`. + for selector in shape.ir.public_selectors() { + selector_cost(*selector, num_vars, &mut leg); + } + leg.ops(2 * eq_eval_rows_again(num_vars)); + + // The three rules, their fold and the refusal. `combine_rows` folds the + // DAG's own constants in as ROWS; this form owns every constant in one pool + // instead, so they come back out here and go in as values below — a DAG + // constant that is also the `1` every `eq` seeds costs one row, not two. + leg.ops(combine_rows(shape.ir) - dag_constant_rows(shape.ir)); + leg.op(); + leg.merge(&claim_statements_cost(shape.bus, num_vars)); + leg.ops(RULES - 1); + leg.ops(ASSERT_ROWS); + + // `claim_reduce`. Its own form already carries its absorbs and draws as + // ROWS, so only the sponge is replayed here. + let sources = shape.sources(); + leg.ops(claim_reduce_rows(&sources, shape.num_columns, num_vars)); + for _ in 0..sources.len() { + schedule.absorb(COORDINATES_PER_EXT); + } + schedule.draw_ext(); + for _ in 0..num_vars { + for _ in 0..REDUCE_DEGREE { + schedule.absorb(COORDINATES_PER_EXT); + } + schedule.draw_ext(); + } + for _ in 0..shape.num_columns { + schedule.absorb(COORDINATES_PER_EXT); + } + + // The constants, by VALUE, because the pool is one per program and the + // landed forms report their own as counts that cannot know what collides. + for value in table_constants(shape, degree) { + leg.constant(value); + } + // ★ And the SPONGE's own: `algebraic_leaf_hash` interns + // `leaf_capacity(felts)` for the leaf it is about to hash + // (`edsl.rs:676-692`), so the program pays one `LFM_CONST` per DISTINCT + // leaf length it ever hashes. No per-hash form can carry this — the pool is + // per PROGRAM and a hash does not know what other lengths the program will + // use — which is why it is added here, off the finished schedule, and why + // `SpongeHash::rows()` is right to leave it out. + for hash in schedule.hashes() { + leg.constant_word(leaf_capacity(hash.felts())); + } + + TableCost { leg, schedule } +} + +/// `absorb_ext` on both halves: one `Unpack` and three felts into the sponge. +fn absorb_ext(leg: &mut Cost, schedule: &mut SpongeSchedule, count: usize) { + leg.ops(count * absorb_unpack_rows()); + for _ in 0..count { + schedule.absorb(COORDINATES_PER_EXT); + } +} + +/// `sample_ext` on both halves: one `Pack` and three candidates. +fn draw_ext(leg: &mut Cost, schedule: &mut SpongeSchedule, count: usize) { + leg.ops(count * sample_ext_rows()); + for _ in 0..count { + schedule.draw_ext(); + } +} + +/// `u_j = (r − j)/(j + 1)`'s two interned constants, for `j = 1 .. d−1` — the +/// same values `whir_poly::emit_newton_step` interns, written out here so the +/// pool can be predicted BY VALUE rather than by count. +/// +/// Counting them would not do: `claim_reduce` runs at degree 2 and the GKR +/// ladder at 3, so `j = 1`'s pair is the same pair twice and the program pays +/// for it once. `sumcheck_round_consts` is the count for a leg standing alone. +pub(crate) fn newton_constants(degree: usize) -> Vec { + let d = degree.max(1); + let mut values = Vec::with_capacity(2 * d.saturating_sub(1)); + for j in 1..d { + let inv = FE::from((j + 1) as u64) + .inv() + .expect("j + 1 is a small nonzero Goldilocks element"); + values.push(FEE::new([inv, FE::zero(), FE::zero()])); + values.push(FEE::new([ + FE::zero() - FE::from(j as u64) * inv, + FE::zero(), + FE::zero(), + ])); + } + values +} + +/// Every constant a table's verify interns, by value. +/// +/// The `1` that seeds every `eq` and every challenge ladder; the `0` the +/// refusals divide by and the zerocheck's claim is; the Newton pairs of the +/// three sumcheck degrees a table runs — the ladder's 3, the batch's, and +/// `claim_reduce`'s 2 — which overlap; and the constraint DAG's own `Fixed` +/// values, which the AIR chose. +/// The rows `whir_program::steps_rows` charges for the DAG's CONSTANTS: one per +/// distinct `Fixed` value, and the zero a `Neg` subtracts from. +/// +/// Counted the same way it counts them, so that subtracting it from +/// `combine_rows` leaves exactly the operations. +/// +/// ⚠ AND THE EMPTY DAG'S ZERO, which is the one case where `combine_rows`' +/// single row is not an operation at all. A shape with NO ROOTS takes +/// `combine_rows`' early return of 1 (`whir_air.rs:74`), and what +/// `emit_combine` emits for it is `b.ext_const(&FEE::zero())` +/// (`whir_air.rs:125`) — an `LFM_CONST`, not an `LFM_XALU`. `combine_rows` is +/// in a rows-INCLUSIVE convention and is right; this function is the conversion +/// to the const-free one, so the zero has to be named HERE or the conversion +/// charges an operation nobody emits. The zero itself is already in +/// [`table_constants`], so naming it costs the pool nothing. +/// +/// Found by the assembled walk's F1 over three tables whose AIRs carry +/// `EmptyConstraints`: it read exactly three rows under its prediction, one per +/// table. No fixture with an empty DAG existed before it. +pub(crate) fn dag_constant_rows(ir: &IrShape) -> usize { + if ir.root_steps().is_empty() { + return 1; + } + let mut values: Vec = Vec::new(); + let mut negates = false; + for step in ir.steps_as_ops() { + match step { + multilinear::program::Op::Fixed(value) => { + if !values.contains(&value) { + values.push(value); + } + } + multilinear::program::Op::Neg(_) => negates = true, + _ => {} + } + } + values.len() + usize::from(negates && !values.contains(&FEE::zero())) +} + +fn table_constants(shape: &TableShape<'_>, degree: usize) -> Vec { + let mut values = vec![FEE::one(), FEE::zero()]; + for d in [GKR_SUMCHECK_DEGREE, degree, REDUCE_DEGREE] { + values.extend(newton_constants(d)); + } + for step in shape.ir.steps_as_ops() { + if let multilinear::program::Op::Fixed(value) = step { + values.push(value); + } + } + // The zero a `Neg` subtracts from is already in the list above. + values +} diff --git a/prover/src/lfm/whir_table_tests.rs b/prover/src/lfm/whir_table_tests.rs new file mode 100644 index 000000000..832b81389 --- /dev/null +++ b/prover/src/lfm/whir_table_tests.rs @@ -0,0 +1,771 @@ +//! Gates for the per-table verify's legs. +//! +//! The selector is gated against `Selector::evaluate` itself, at EVERY +//! exemption count rather than at a sample of them: the cost and the value both +//! turn on the bit pattern of `cutoff`, so a gate that picked a few counts +//! would be picking a few bit patterns. + +use multilinear::selector::Selector; + +use crate::tables::types::{FE, FEE}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_bus::Cost; +use super::whir_table::{emit_selector, selector_cost}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +/// One `Public` instruction per published value (`builder.rs:637-642`), which +/// the marginal below has to take out to be about the leg. +const PUBLISH_ROW: usize = 1; + +fn pseudo(seed: u64, count: usize) -> Vec { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + FE::from(state >> 2) + }; + (0..count) + .map(|_| FEE::new([next(), next(), next()])) + .collect() +} + +fn words(point: &[FEE]) -> Vec> { + vec![point.iter().map(ext_word).collect()] +} + +/// Every exemption count a table of `2^num_vars` rows can carry, the trivial +/// and the all-exempt included. +fn every_selector(num_vars: usize) -> Vec { + (0..=(1usize << num_vars)) + .map(Selector::except_last) + .collect() +} + +/// The first `count` selectors emitted over one hinted point, each published. +fn selectors_program(num_vars: usize, count: usize) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(num_vars.max(1) as u32); + let point: Vec<_> = (0..num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + // A program with no variables still needs something in its arena to hint, + // and something to publish. + let anchor = if num_vars == 0 { + b.hint_word(arena, 0).as_ext() + } else { + point[0] + }; + b.public(anchor.as_cell()); + for selector in every_selector(num_vars).into_iter().take(count) { + let value = emit_selector(&mut b, selector, &point); + b.public(value.as_cell()); + } + compile(b.finish()) +} + +/// ★ The selector leg computes what the host computes, at every cutoff. +#[test] +fn the_selector_is_the_hosts_indicator() { + for num_vars in 1..=5usize { + let selectors = every_selector(num_vars); + let program = selectors_program(num_vars, selectors.len()); + validate(&program).expect("the selector leg must be admissible"); + for seed in [0x5E1EC7u64, 0xB17, 0xC0FFEE] { + let point = pseudo(seed, num_vars); + let exec = execute(&program, &words(&point), &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("num_vars {num_vars}: must execute: {e:?}")); + let got: Vec = exec.public_words[1..] + .iter() + .map(|(_, word)| word_as_ext(word).expect("a published extension value")) + .collect(); + assert_eq!(got.len(), selectors.len()); + for (selector, value) in selectors.iter().zip(&got) { + let want = selector + .evaluate::(&point) + .expect("the host evaluates it"); + assert_eq!( + *value, want, + "num_vars {num_vars}, end_exemptions {} at seed {seed:#x}", + selector.end_exemptions + ); + } + } + } +} + +/// The two cases the general form does not reach by arithmetic: a selector that +/// applies everywhere is the literal one, and one that applies nowhere is the +/// literal zero. +/// +/// Named separately because they are the branches that RETURN before the loop, +/// and a gate that only swept the middle would leave both unexercised at the +/// value level. +#[test] +fn the_trivial_and_the_all_exempt_selectors_are_literals() { + let num_vars = 3; + let point = pseudo(0xAB1E, num_vars); + for (selector, want) in [ + (Selector::ALL, FEE::one()), + (Selector::except_last(1usize << num_vars), FEE::zero()), + ] { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(num_vars as u32); + let wires: Vec<_> = (0..num_vars) + .map(|i| b.hint_word(arena, i as u32).as_ext()) + .collect(); + let value = emit_selector(&mut b, selector, &wires); + b.public(value.as_cell()); + let program = compile(b.finish()); + let exec = execute(&program, &words(&point), &crate::hash_pin::BLOCK_HASHER) + .expect("the literal cases must execute"); + let got = word_as_ext(&exec.public_words[0].1).expect("a published extension value"); + assert_eq!(got, want, "end_exemptions {}", selector.end_exemptions); + assert_eq!( + got, + selector + .evaluate::(&point) + .expect("the host evaluates it"), + "and it must be the host's answer, not just the literal I expected" + ); + } +} + +/// ★ F1, incrementally: each selector's own marginal rows against its form. +/// +/// Incremental rather than total, because the `1` is shared across the whole +/// set — a total would let one selector's over-count cancel another's, and the +/// first selector is the only one that pays for the constant. +#[test] +fn each_selector_emits_its_closed_form() { + for num_vars in 1..=4usize { + let selectors = every_selector(num_vars); + let mut previous = selectors_program(num_vars, 0).instrs.len(); + let mut pool = Cost::default(); + for (index, selector) in selectors.iter().enumerate() { + let before = pool.rows(); + selector_cost(*selector, num_vars, &mut pool); + let predicted = pool.rows() - before; + + let now = selectors_program(num_vars, index + 1).instrs.len(); + let measured = now - previous - PUBLISH_ROW; + previous = now; + assert_eq!( + measured, predicted, + "num_vars {num_vars}, end_exemptions {}", + selector.end_exemptions + ); + } + println!( + "selectors num_vars={num_vars}: {} selectors, {} rows pooled ({} ops + {} constants)", + selectors.len(), + pool.rows(), + pool.operations(), + pool.constants() + ); + } +} + +/// The pooled total is what the incremental steps add up to, and the whole set +/// costs it once. +/// +/// This is the statement the incremental test cannot make: that the ACCUMULATOR +/// shares its constants rather than charging each selector its own `1`. +#[test] +fn the_selectors_share_one_interned_constant() { + let num_vars = 4; + let selectors = every_selector(num_vars); + let mut pool = Cost::default(); + for selector in &selectors { + selector_cost(*selector, num_vars, &mut pool); + } + let mut separately = 0usize; + for selector in &selectors { + let mut alone = Cost::default(); + selector_cost(*selector, num_vars, &mut alone); + separately += alone.rows(); + } + // Every selector interns the `1`; the all-exempt one also interns a `0`. + // Pooled, those are two rows for the whole set. + let shared = separately - pool.rows(); + assert_eq!( + shared, + selectors.len() - 1, + "the `1` is paid once and not {} times", + selectors.len() + ); + + let measured = selectors_program(num_vars, selectors.len()).instrs.len() + - selectors_program(num_vars, 0).instrs.len() + - selectors.len() * PUBLISH_ROW; + assert_eq!(measured, pool.rows(), "the pooled form is the emitted set"); +} + +// --------------------------------------------------------------------------- +// The assembly, against a REAL per-table proof. +// --------------------------------------------------------------------------- + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet, RowDomain}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, Multiplicity, + NullBoundaryConstraintBuilder, Packing, +}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_logup::{InteractionShape, interaction_shapes}; +use stark::multilinear_table::{CommittedTable, TableProof}; +use stark::traits::AIR; + +use super::whir_bus::alpha_powers_read; +use super::whir_gkr::GkrLayerWires; +use super::whir_reduce::ReduceWires; +use super::whir_table::{ + TableCost, TableProofWires, TableShape, emit_table_verify, table_verify_cost, +}; +use super::whir_transcript::{SpongeEntry, WhirTranscript}; + +type F = crate::tables::types::GoldilocksField; +type E = crate::tables::types::GoldilocksExtension; +type HostTranscript = DefaultTranscript; + +/// The table under gate: three constraint roots, one of them carrying a +/// selector, over a trace that SATISFIES them. +/// +/// ⚠ A test constraint set, said so. What makes it worth gating against is that +/// it goes through the same `AirWithBuses` + `TableLayout` pipeline a VM table +/// does, so its `IrShape`, its factor kinds and its public selector are built +/// by the production compiler — and unlike 2c's shape this one is SATISFIABLE, +/// which is what lets a real `multilinear_table::prove` run over it. +/// +/// - `c = a·b` and `d = a + b` apply on every row; +/// - `next(a) = a + 1` cannot apply on the last, so it carries +/// `except_last(1)` — and that selector is what puts a PUBLIC FACTOR in the +/// woven factor vector, which is how the assembly's gate reaches the selector +/// leg as well as its own. +struct ThreeRootsOneSelector; + +const COLUMNS: usize = 4; + +impl ConstraintSet for ThreeRootsOneSelector { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, 0); + let y = b.main(0, 1); + let c = b.main(0, 2); + b.emit_base(0, a * y - c); + + let a = b.main(0, 0); + let y = b.main(0, 1); + let d = b.main(0, 3); + b.emit_base(1, a + y - d); + + let one = b.one(); + let next = b.main(1, 0); + let here = b.main(0, 0); + b.emit_base_rows(2, RowDomain::except_last(1), next - here - one); + } +} + +/// Two interactions over the same four columns: a real bus, and one the layout +/// will not refuse. +fn buses() -> Vec { + vec![ + BusInteraction::sender( + 0u64, + Multiplicity::Column(1), + Packing::Direct.columns(&[0, 2]), + ), + BusInteraction::receiver( + 0u64, + Multiplicity::Column(1), + Packing::Direct.columns(&[0, 3]), + ), + ] +} + +/// The trace the constraints hold on: `a` counts, `b` is fixed, `c = a·b`, +/// `d = a + b`. +fn columns(num_vars: usize) -> Vec> { + let rows = 1usize << num_vars; + let b = 7u64; + vec![ + (0..rows).map(|i| FE::from(i as u64)).collect(), + (0..rows).map(|_| FE::from(b)).collect(), + (0..rows).map(|i| FE::from(i as u64 * b)).collect(), + (0..rows).map(|i| FE::from(i as u64 + b)).collect(), + ] +} + +fn air() -> AirWithBuses { + AirWithBuses::new( + COLUMNS, + AuxiliaryTraceBuildData { + interactions: buses(), + }, + &table_options(), + 1, + ThreeRootsOneSelector, + ) +} + +fn table_options() -> stark::proof::options::ProofOptions { + stark::proof::options::GoldilocksCubicProofOptions::with_params(4, 128, 20) + .expect("valid options") +} + +/// The LogUp challenges and the zerocheck's batching challenge, as the epoch +/// would have drawn them. +fn table_challenges() -> (FEE, FEE, FEE) { + let drawn = pseudo(0x7AB1E, 3); + (drawn[0], drawn[1], drawn[2]) +} + +/// A real per-table proof. `multilinear_table::prove` reads no commitment root +/// (`multilinear_table.rs:630-700`), so this needs no WHIR commitment and no +/// guest ELF. +fn real_proof( + air: &AirWithBuses, + num_vars: usize, +) -> (TableProof, Vec, Vec>) { + let cols = columns(num_vars); + let lifted = cols.clone(); + let table = CommittedTable::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + COLUMNS, + num_vars, + Uniforms::default(), + move |col| lifted[col as usize].clone(), + ) + .expect("the table lays out"); + + let (z, alpha, beta) = table_challenges(); + let mut prover = HostTranscript::new(&[]); + let (proof, _point) = + stark::multilinear_table::prove(&table, &z, &alpha, &beta, &mut prover, None) + .expect("the table proves"); + (proof, Vec::new(), cols) +} + +/// The bus, with the challenges factored out, and the layout it came from. +fn shape_of( + air: &AirWithBuses, + num_vars: usize, +) -> ( + stark::multilinear_table::TableLayout<'_, F, E>, + Vec>, +) { + let layout = stark::multilinear_table::TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + COLUMNS, + num_vars, + Uniforms::default(), + ) + .expect("the table lays out"); + let slots = layout.slot_of().to_vec(); + let bus = interaction_shapes(air.bus_interactions(), COLUMNS, |column| { + slots + .get(column) + .copied() + .ok_or(multilinear::Error::UnknownPolynomial { + index: column, + len: slots.len(), + }) + }) + .expect("the bus probes"); + (layout, bus) +} + +/// The alpha ladder the emitter reads. +fn alpha_ladder(bus: &[InteractionShape], alpha: FEE) -> Vec { + let mut powers = Vec::with_capacity(alpha_powers_read(bus)); + let mut power = FEE::one(); + for _ in 0..alpha_powers_read(bus) { + powers.push(power); + power *= alpha; + } + powers +} + +/// Every value the program hints, in arena order. +fn flatten(proof: &TableProof, z: FEE, alpha_powers: &[FEE], beta: FEE) -> Vec { + let mut values = vec![z]; + values.extend_from_slice(alpha_powers); + values.push(beta); + values.push(proof.bus_output.0); + values.push(proof.bus_output.1); + for layer in &proof.gkr.layers { + for round in &layer.sumcheck.rounds { + values.extend(round.evaluations.iter().copied()); + } + values.extend([layer.p_lo, layer.p_hi, layer.q_lo, layer.q_hi]); + } + for round in &proof.constraint.sumcheck.rounds { + values.extend(round.evaluations.iter().copied()); + } + values.extend(proof.constraint.factor_values.iter().copied()); + for round in &proof.constraint.reduce.sumcheck.rounds { + values.extend(round.evaluations.iter().copied()); + } + values.extend(proof.constraint.reduce.column_values.iter().copied()); + values +} + +/// The assembled program, or — with `leg` false — the same hinted arena with no +/// leg at all, so the difference is the leg. +fn table_program( + proof: &TableProof, + shape: &TableShape<'_>, + alpha_count: usize, + total: usize, + leg: bool, +) -> LfmProgram { + let mut b = LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()); + let arena = b.declare_arena(total as u32); + let mut index = 0u32; + let mut take = |b: &mut LfmBuilder, count: usize| -> Vec { + (0..count) + .map(|_| { + let wire = b.hint_word(arena, index).as_ext(); + index += 1; + wire + }) + .collect() + }; + + let z = take(&mut b, 1)[0]; + let alpha_powers = take(&mut b, alpha_count); + let beta = take(&mut b, 1)[0]; + let output = take(&mut b, 2); + let mut gkr = Vec::with_capacity(proof.gkr.layers.len()); + for layer in &proof.gkr.layers { + let sumcheck: Vec> = layer + .sumcheck + .rounds + .iter() + .map(|round| take(&mut b, round.evaluations.len())) + .collect(); + let halves = take(&mut b, 4); + gkr.push(GkrLayerWires { + sumcheck, + p_lo: halves[0], + p_hi: halves[1], + q_lo: halves[2], + q_hi: halves[3], + }); + } + let sumcheck: Vec> = proof + .constraint + .sumcheck + .rounds + .iter() + .map(|round| take(&mut b, round.evaluations.len())) + .collect(); + let factor_values = take(&mut b, proof.constraint.factor_values.len()); + let reduce_sumcheck: Vec> = proof + .constraint + .reduce + .sumcheck + .rounds + .iter() + .map(|round| take(&mut b, round.evaluations.len())) + .collect(); + let column_values = take(&mut b, proof.constraint.reduce.column_values.len()); + + if !leg { + b.public(z.as_cell()); + b.public(beta.as_cell()); + return compile(b.finish()); + } + + let mut transcript = WhirTranscript::new(); + let verdict = emit_table_verify( + &mut b, + &mut transcript, + &TableProofWires { + bus_output: (output[0], output[1]), + gkr: &gkr, + sumcheck: &sumcheck, + factor_values: &factor_values, + reduce: ReduceWires { + sumcheck: &reduce_sumcheck, + column_values: &column_values, + }, + }, + shape, + z, + &alpha_powers, + beta, + ); + b.public(verdict.bus_output.0.as_cell()); + b.public(verdict.bus_output.1.as_cell()); + for wire in &verdict.point { + b.public(wire.as_cell()); + } + for wire in &verdict.column_values { + b.public(wire.as_cell()); + } + compile(b.finish()) +} + +/// ★ G1 — the assembled per-table verify computes what the host computes. +/// +/// The machine derives every challenge itself, so executing an honest proof IS +/// the challenge-stream comparison: a wrong challenge anywhere leaves one of +/// three refusals the machine cannot satisfy. The verdict compared against the +/// host's is the value gate on top of that. +#[test] +fn the_table_verify_computes_what_the_host_computes() { + for num_vars in [3usize, 4] { + let built = air(); + let (proof, _, _) = real_proof(&built, num_vars); + let (layout, bus) = shape_of(&built, num_vars); + let (z, alpha, beta) = table_challenges(); + + let mut verifier = HostTranscript::new(&[]); + let (bus_output, reduced) = stark::multilinear_table::verify( + &proof, + layout.statement(), + &z, + &alpha, + &beta, + &mut verifier, + // Nothing settled out of band: this fixture has no preprocessed + // columns at all, so every one of them (none) stays inside + // `check_preprocessed`. The argument arrived with W1-B, which lets + // DECODE's prepared opening settle its leading preprocessed columns + // and skip their MLE evaluations here — the very seam + // `emit_table_verify` says it does not reproduce. + 0, + ) + .expect("the host must verify its own proof — the fixture is the precondition"); + + let shape = TableShape { + ir: layout.shape(), + bus: &bus, + kinds: layout.kinds(), + num_columns: layout.num_columns(), + num_vars, + }; + let alpha_powers = alpha_ladder(&bus, alpha); + let values = flatten(&proof, z, &alpha_powers, beta); + let program = table_program(&proof, &shape, alpha_powers.len(), values.len(), true); + validate(&program).expect("the table leg must be admissible"); + + let exec = execute(&program, &words(&values), &crate::hash_pin::BLOCK_HASHER) + .unwrap_or_else(|e| panic!("num_vars {num_vars}: the table leg must execute: {e:?}")); + let got: Vec = exec + .public_words + .iter() + .map(|(_, word)| word_as_ext(word).expect("a published extension value")) + .collect(); + + assert_eq!(got[0], bus_output.0, "num_vars {num_vars}: bus output p"); + assert_eq!(got[1], bus_output.1, "num_vars {num_vars}: bus output q"); + assert_eq!( + &got[2..2 + num_vars], + reduced.point.as_slice(), + "num_vars {num_vars}: the reduced point" + ); + assert_eq!( + &got[2 + num_vars..], + reduced.column_values.as_slice(), + "num_vars {num_vars}: the column values" + ); + println!( + "table verify num_vars={num_vars}: {} GKR layers, degree {}, {} factors, \ + {} columns, {} public selectors", + proof.gkr.layers.len(), + shape.sumcheck_degree(), + layout.kinds().len(), + layout.num_columns(), + layout.shape().public_selectors().len() + ); + } +} + +/// ★ G2 — F1 for the whole table: its marginal rows and its permutations. +#[test] +fn the_table_verify_emits_its_closed_form() { + for num_vars in [3usize, 4] { + let built = air(); + let (proof, _, _) = real_proof(&built, num_vars); + let (layout, bus) = shape_of(&built, num_vars); + let (z, alpha, beta) = table_challenges(); + let shape = TableShape { + ir: layout.shape(), + bus: &bus, + kinds: layout.kinds(), + num_columns: layout.num_columns(), + num_vars, + }; + let alpha_powers = alpha_ladder(&bus, alpha); + let values = flatten(&proof, z, &alpha_powers, beta); + let with = table_program(&proof, &shape, alpha_powers.len(), values.len(), true); + let without = table_program(&proof, &shape, alpha_powers.len(), values.len(), false); + // The leg publishes the whole verdict; the bare program publishes two. + let published = 2 + num_vars + layout.num_columns(); + let measured = with.instrs.len() - without.instrs.len() - (published - 2) * PUBLISH_ROW; + + let cost: TableCost = table_verify_cost(&shape, SpongeEntry::fresh()); + let consts = |p: &LfmProgram| { + p.instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Const { .. })) + .count() + }; + let measured_consts = consts(&with) - consts(&without); + { + // Diagnostic: which VALUES the program interns that the form does + // not name. A residual here is a leg whose own constants are not in + // the pool, and naming it is the point. + let predicted = cost.leg.constant_values(); + let mut unnamed: Vec = Vec::new(); + for instr in &with.instrs { + if let super::instr::Instr::Const { value, .. } = instr + && !predicted.contains(value) + && !unnamed.contains(value) + { + unnamed.push(*value); + } + } + assert!( + unnamed.is_empty(), + "the form must NAME every constant the program interns, and it does not name {:?}", + unnamed + ); + } + println!( + "table num_vars={num_vars}: {measured} rows emitted ({} of them LFM_CONST), \ + {} predicted ({} leg ops + {} constants + {} sponge rows, {} permutations)", + measured_consts, + cost.rows(), + cost.leg.operations(), + cost.leg.constants(), + cost.schedule.rows(), + cost.perms() + ); + assert_eq!(measured, cost.rows(), "num_vars {num_vars}"); + } +} + +/// One forgery: what it is called, and the single value it moves. +type TamperSite = (&'static str, Box)>); + +/// ★ G3 — the tamper arm, in THREE halves (instance 49). +/// +/// Every site runs all three: the untouched proof EXECUTES, the HOST rejects +/// the forgery, and the machine REFUSES it. The middle half is not decoration — +/// without it a machine that refused something the host accepts would score as +/// a soundness success when it is a completeness bug. +/// +/// The three sites are chosen to land on three different refusals: a GKR half +/// breaks the layer relation, a main-sumcheck evaluation breaks the batch's +/// residual, and a column value breaks `claim_reduce`'s shifted read. Each is +/// reached only if the machine drew the same challenges the host did, which is +/// the other thing this arm witnesses. +#[test] +fn the_tamper_arm_refuses_what_the_host_rejects() { + let num_vars = 3usize; + let built = air(); + let (honest, _, _) = real_proof(&built, num_vars); + let (layout, bus) = shape_of(&built, num_vars); + let (z, alpha, beta) = table_challenges(); + let shape = TableShape { + ir: layout.shape(), + bus: &bus, + kinds: layout.kinds(), + num_columns: layout.num_columns(), + num_vars, + }; + let alpha_powers = alpha_ladder(&bus, alpha); + + let run = |proof: &TableProof| -> Result<(), String> { + let values = flatten(proof, z, &alpha_powers, beta); + let program = table_program(proof, &shape, alpha_powers.len(), values.len(), true); + execute(&program, &words(&values), &crate::hash_pin::BLOCK_HASHER) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) + }; + let host = |proof: &TableProof| -> Result<(), String> { + let mut verifier = HostTranscript::new(&[]); + stark::multilinear_table::verify( + proof, + layout.statement(), + &z, + &alpha, + &beta, + &mut verifier, + // Nothing settled out of band: this fixture has no preprocessed + // columns at all, so every one of them (none) stays inside + // `check_preprocessed`. The argument arrived with W1-B, which lets + // DECODE's prepared opening settle its leading preprocessed columns + // and skip their MLE evaluations here — the very seam + // `emit_table_verify` says it does not reproduce. + 0, + ) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) + }; + + // HALF ONE: the untouched proof executes, on both sides. + host(&honest).expect("the honest proof must verify on the host"); + run(&honest).expect("the honest proof must EXECUTE on the machine"); + + // An item, not a closure: a `TamperSite` outlives the function body, so a + // boxed closure in it cannot borrow a local. + fn bump(value: &mut FEE) { + *value += FEE::one(); + } + let sites: Vec = vec![ + ( + "a GKR layer's q_lo", + Box::new(|p: &mut TableProof| bump(&mut p.gkr.layers[0].q_lo)), + ), + ( + "the main sumcheck's first evaluation", + Box::new(|p: &mut TableProof| { + bump(&mut p.constraint.sumcheck.rounds[0].evaluations[0]) + }), + ), + ( + "a claimed column value", + Box::new(|p: &mut TableProof| bump(&mut p.constraint.reduce.column_values[0])), + ), + ]; + + for (name, tamper) in sites { + let mut forged = honest.clone(); + tamper(&mut forged); + // HALF TWO: the host must reject it, or it is not a forgery and the + // machine's refusal would say nothing. + let rejected = host(&forged); + assert!( + rejected.is_err(), + "{name}: the HOST accepted the tampered proof, so this site is not a forgery" + ); + // HALF THREE: the machine refuses it. + let refused = run(&forged); + assert!( + refused.is_err(), + "{name}: the machine EXECUTED a proof the host rejected with {:?}", + rejected.unwrap_err() + ); + println!( + "tamper {name}: host {} / machine {}", + rejected.unwrap_err(), + refused.unwrap_err() + ); + } +} diff --git a/prover/src/lfm/whir_transcript.rs b/prover/src/lfm/whir_transcript.rs new file mode 100644 index 000000000..9276c4edc --- /dev/null +++ b/prover/src/lfm/whir_transcript.rs @@ -0,0 +1,740 @@ +//! The WHIR Fiat–Shamir transcript replayed inside the machine. +//! +//! The mirror is `DefaultTranscript` +//! (`crypto/crypto/src/fiat_shamir/default_transcript.rs`): a BUFFERING sponge +//! — every absorb appends to a byte buffer, and a squeeze hashes the whole +//! accumulated buffer with `rpx::sponge_leaf_bytes`, clears it, and re-absorbs +//! the digest to advance the chain. +//! +//! Like [`super::transcript_replay`] this is an eDSL library rather than a chip: +//! ordinary Rust tracking the transcript's state AT EMIT TIME and emitting the +//! instructions that reproduce its VALUES at run time. Which squeeze a challenge +//! comes from, where a refill lands, which absorb invalidates the output buffer +//! — all decided by the emitter and baked into the program's shape. +//! +//! # Why this replay is nearly free +//! +//! Three facts, each read off the host rather than assumed. +//! +//! 1. **The squeeze is not reversed and needs no rejection.** +//! `RpxTranscriptHash` sets `REVERSES_SQUEEZE = false` and +//! `CANDIDATES_PER_COORDINATE = Some(1)` (`transcript_hash.rs:169-180`): a +//! squeeze is four canonical felts and one candidate per coordinate is +//! structural, not probabilistic. The transcript never leaves the field. +//! +//! 2. **An absorbed value IS its felts.** The cubic extension streams 24 bytes +//! big-endian in coordinate order (`extensions_goldilocks.rs:566-572`), the +//! base field streams `canonical_u64().to_be_bytes()`, and +//! `sponge_leaf_bytes` reads 8-byte groups with `u64::from_be_bytes` +//! (`rpx/mod.rs:404`). So three felts per extension element and four per +//! 32-byte commitment, with no byte work — provided a runtime value starts at +//! an offset that is a multiple of eight. +//! +//! 3. **The machine's leaf hash IS the host's sponge.** +//! `WrapHash::algebraic_leaf_hash` (`edsl.rs:676-708`) is `rpx::sponge_leaf` +//! structurally: rate-8 overwrite, `leaf_capacity` with the padding flag in +//! lane 0, only the capacity carried between blocks, the digest from the last +//! permutation, the same empty case. The replay needs no new hash operation. +//! +//! # Constants are free; runtime values must be aligned +//! +//! A run of constant bytes is packed into felts at emit time by exactly +//! `sponge_leaf_bytes`' rule — 8-byte big-endian groups, the trailing partial +//! group zero-extended on the LOW side — at any length. So the epoch statement's +//! fixed part costs no rows and imposes no alignment. +//! +//! What needs alignment is the first RUNTIME value after such a run, which in +//! the epoch statement is the roots. [`WhirTranscript::absorb_felts`] REFUSES an +//! unaligned runtime absorb rather than emitting a byte shift: a value straddling +//! two felts would have to be bit-decomposed, and a shape-dependent shift +//! computed from three lengths is the bug class that is correct on a fixture and +//! wrong on a block. The statement padding that makes the offset a multiple of +//! eight is W1's, computed from the accumulated length. +//! +//! # The Poseidon1 arm +//! +//! A [`WrapHash::Poseidon1`] builder replays ZisK's field sponge instead +//! ([`WhirTranscript::for_builder`]), the host's `P1Transcript` +//! (`crypto/crypto/src/fiat_shamir/p1_transcript.rs`), through +//! [`P1SpongeVar`]. That sponge has no byte buffer: every `append_bytes` is the +//! LENGTH felt, then the bytes as 8-byte big-endian felts (the last zero-padded +//! on the low side, `felts_from_bytes`), and every `append_field_element` its +//! three coordinates. So the call boundaries are part of the stream, and the +//! callers name what they absorb: [`WhirTranscript::absorb_digest`] and +//! [`WhirTranscript::absorb_root_lanes`] are a 32-byte `append_bytes`, +//! [`WhirTranscript::absorb_nonce`] an 8-byte one, and each +//! [`WhirTranscript::absorb_const_bytes`] call is one `append_bytes` of its +//! bytes. A candidate is one squeezed lane; `state()` is the sponge's digest +//! after flushing a copy. The byte arm above is unchanged, call for call. + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::edsl::WrapHash; +use super::p1w16_emit::P1SpongeVar; + +/// Felts a squeeze hands out before it must refill — `SQUEEZE_LEN / 8` in the +/// host's bytes (`default_transcript.rs:19`). +pub const CANDIDATES_PER_SQUEEZE: usize = 4; + +/// Coordinates in a cubic extension element, one candidate each. +pub const COORDINATES_PER_EXT: usize = 3; + +/// Bytes one felt occupies in the sponge's stream. +const BYTES_PER_FELT: usize = 8; + +/// The WHIR transcript, replayed. A clone is a fork: the host's +/// `transcript.clone()`, as the block's per-group forks take it. +#[derive(Default, Clone)] +pub struct WhirTranscript { + /// Constant bytes absorbed but not yet packed into felts. Only a trailing + /// run of constants can be pending: any runtime absorb flushes. + pending: Vec, + /// Felts absorbed since the last squeeze, in order. + buf: Vec, + /// The last squeeze's four lanes, and how many have been handed out. + /// `out_pos == CANDIDATES_PER_SQUEEZE` means "empty, squeeze to refill", + /// which is also what every absorb resets it to. + out: Option<[Felt; CANDIDATES_PER_SQUEEZE]>, + out_pos: usize, + /// Squeezes emitted, for the cost pins. + squeezes: usize, + /// The Poseidon1 sponge, when the builder verifies a Poseidon1 base: every + /// call below goes to it and the byte buffer above stays empty. + p1: Option, +} + +/// The Poseidon1 arm's state: the sponge, and constant lanes absorbed by a +/// call that has no builder ([`WhirTranscript::absorb_const_bytes`]), put in +/// order before anything else touches the sponge. Putting a lane later is the +/// same stream: the sponge is a function of its lanes' order alone. +#[derive(Clone)] +struct P1Arm { + sponge: P1SpongeVar, + consts: Vec, +} + +impl P1Arm { + fn ready(&mut self, b: &mut LfmBuilder) -> &mut P1SpongeVar { + for v in self.consts.drain(..) { + self.sponge.put_const(b, v); + } + &mut self.sponge + } +} + +impl WhirTranscript { + /// `DefaultTranscript::new(&[])` — the empty seed is absorbed, which + /// appends nothing and invalidates the (already empty) output buffer. + pub fn new() -> Self { + Self { + pending: Vec::new(), + buf: Vec::new(), + out: None, + out_pos: CANDIDATES_PER_SQUEEZE, + squeezes: 0, + p1: None, + } + } + + /// The transcript of a program built by `b`: [`Self::new`], or under a + /// [`WrapHash::Poseidon1`] builder the empty `P1Transcript` (`P1Whir`'s + /// `WhirHash::sponge()`, which absorbs no seed). + pub fn for_builder(b: &mut LfmBuilder) -> Self { + let mut t = Self::new(); + if b.wrap_hash() == WrapHash::Poseidon1 { + t.p1 = Some(P1Arm { + sponge: P1SpongeVar::new(b), + consts: Vec::new(), + }); + } + t + } + + /// `append_bytes` of a PROGRAM CONSTANT: free, and unconstrained in length. + /// + /// Under Poseidon1 it is ONE `append_bytes` call: the length, then the + /// bytes' felts, all program constants. + pub fn absorb_const_bytes(&mut self, bytes: &[u8]) { + if let Some(arm) = &mut self.p1 { + arm.consts.push(FE::from(bytes.len() as u64)); + arm.consts.extend(const_byte_felts(bytes)); + return; + } + self.pending.extend_from_slice(bytes); + self.invalidate(); + } + + /// `append_bytes` / `append_field_element` of RUNTIME felts. + /// + /// Panics when the constant run before it does not end on a felt boundary: + /// see the module header. Under Poseidon1 these are field-element lanes + /// with no length: a byte string goes through the named absorbs below. + pub fn absorb_felts(&mut self, b: &mut LfmBuilder, felts: &[Felt]) { + if let Some(arm) = &mut self.p1 { + let sponge = arm.ready(b); + for felt in felts { + sponge.put_felt(b, *felt); + } + return; + } + assert_eq!( + self.pending.len() % BYTES_PER_FELT, + 0, + "a runtime value must start on a felt boundary — {} constant bytes are pending, \ + which is {} short. The statement's computed padding is what guarantees this; \ + emitting a byte shift here instead is the bug class that is correct on a fixture \ + and wrong on a block.", + self.pending.len(), + BYTES_PER_FELT - self.pending.len() % BYTES_PER_FELT + ); + self.flush_pending(b); + self.buf.extend_from_slice(felts); + self.invalidate(); + } + + /// `append_field_element` of a cubic extension element: its three + /// coordinates, big-endian and in order, are three felts of the stream. + pub fn absorb_ext(&mut self, b: &mut LfmBuilder, value: Ext) { + let lanes = b.unpack(value.as_cell()); + self.absorb_felts(b, &lanes[..COORDINATES_PER_EXT]); + } + + /// `append_bytes(root, 32)`: a commitment is four canonical felts, big-endian + /// (`digest_to_commitment`, `rpx/mod.rs:428`). + pub fn absorb_digest(&mut self, b: &mut LfmBuilder, digest: Cell) { + let lanes = b.unpack(digest); + self.absorb_root_lanes(b, &lanes); + } + + /// `append_bytes(root, 32)` of a root already unpacked into its four lanes. + pub fn absorb_root_lanes(&mut self, b: &mut LfmBuilder, lanes: &[Felt; 4]) { + self.absorb_byte_felts(b, DIGEST_FELTS * BYTES_PER_FELT, lanes); + } + + /// `append_bytes(&nonce.to_be_bytes())`: eight big-endian bytes, one felt. + pub fn absorb_nonce(&mut self, b: &mut LfmBuilder, nonce: Felt) { + self.absorb_byte_felts(b, BYTES_PER_FELT, &[nonce]); + } + + /// One `append_bytes` of `len` RUNTIME bytes that are exactly `felts`, big- + /// endian: the felts on the byte arm, the length and then the felts under + /// Poseidon1. + fn absorb_byte_felts(&mut self, b: &mut LfmBuilder, len: usize, felts: &[Felt]) { + if let Some(arm) = &mut self.p1 { + arm.consts.push(FE::from(len as u64)); + } + self.absorb_felts(b, felts); + } + + /// `DefaultTranscript::sample()` — hash the accumulated buffer, clear it, + /// re-absorb the digest, and refill the output buffer. + /// + /// ⚠ The re-absorb is the chain advance, and it is what makes consecutive + /// draws differ. Dropping it leaves every draw after the first identical, + /// which a single pinned value cannot see. + pub fn squeeze(&mut self, b: &mut LfmBuilder) -> [Felt; CANDIDATES_PER_SQUEEZE] { + if let Some(arm) = &mut self.p1 { + // Four candidates: the Poseidon1 sponge squeezes lanes, not digests. + let sponge = arm.ready(b); + return core::array::from_fn(|_| sponge.squeeze(b)); + } + let felts = self.felts_now(b); + let digest = WrapHash::Algebraic.leaf_hash(b, &felts); + let lanes = b.unpack(digest.cells()[0]); + self.pending.clear(); + self.buf = lanes.to_vec(); + self.out = Some(lanes); + self.out_pos = 0; + self.squeezes += 1; + lanes + } + + /// `next_sample_u64` — one candidate, refilling with a squeeze when the + /// buffer is spent. A candidate is a squeeze lane: `digest_to_commitment` + /// writes felt `i` as the big-endian bytes `8i..8i+8` and the sampler reads + /// exactly those with `from_be_bytes`, so the two conversions cancel. + pub fn next_candidate(&mut self, b: &mut LfmBuilder) -> Felt { + if let Some(arm) = &mut self.p1 { + return arm.ready(b).squeeze(b); + } + if self.out.is_none() || self.out_pos >= CANDIDATES_PER_SQUEEZE { + self.squeeze(b); + } + let lane = self.out.expect("a squeeze just filled the buffer")[self.out_pos]; + self.out_pos += 1; + lane + } + + /// `sample_field_element` on the cubic extension: three candidates, no + /// rejection. + /// + /// Every candidate is a canonical felt of an RPX squeeze, so + /// `candidate_in_range` holds by construction and the host's single-draw + /// schedule (`CANDIDATES_PER_COORDINATE = Some(1)`) has no rejected branch + /// to emit. + pub fn sample_ext(&mut self, b: &mut LfmBuilder) -> Ext { + let a0 = self.next_candidate(b); + let a1 = self.next_candidate(b); + let a2 = self.next_candidate(b); + b.pack_ext(a0, a1, a2) + } + + /// `sample_u64(2^nbits)`, as the BITS the Merkle walk consumes. + /// + /// The bound is always a power of two here — `domain.size() >> k` — so + /// `threshold = (-2^k) mod 2^k = 0` and no candidate is ever rejected: the + /// schedule is fixed and there is no branch to emit. The answer is + /// `candidate mod 2^nbits`, the low `nbits` bits, which is one `BitDec` row + /// and needs no recomposition because its consumer wants bits. + pub fn sample_u64_pow2(&mut self, b: &mut LfmBuilder, nbits: usize) -> Vec { + let candidate = self.next_candidate(b); + b.bit_dec(candidate, nbits) + } + + /// `state()` — the digest of everything absorbed so far, WITHOUT advancing + /// the chain: production finalizes a clone, with no reset and no re-absorb + /// (`default_transcript.rs:236-239`). Only grinding needs it. + /// + /// Neither the buffer nor the output position moves, so a later squeeze + /// hashes the same bytes again. The packing is therefore emitted twice, + /// which is redundant work and never a different value. + pub fn state(&mut self, b: &mut LfmBuilder) -> Cell { + if let Some(arm) = &mut self.p1 { + return arm.ready(b).digest(b); + } + let felts = self.felts_now(b); + WrapHash::Algebraic.leaf_hash(b, &felts).cells()[0] + } + + /// Squeezes emitted so far. + pub fn squeezes(&self) -> usize { + self.squeezes + } + + /// Emit-time output position, for tests that pin the consumption schedule. + pub fn out_pos(&self) -> usize { + self.out_pos + } + + /// Felts currently in the sponge's stream: what has been absorbed so far, + /// then the trailing constant run packed by `sponge_leaf_bytes`' rule — + /// pending constants come AFTER the felts, never before. Does not mutate, so + /// `state` can call it without disturbing a partial constant group that a + /// later absorb will continue. + fn felts_now(&self, b: &mut LfmBuilder) -> Vec { + let mut felts = self.buf.clone(); + felts.extend(pack_const_bytes(b, &self.pending)); + felts + } + + /// Moves the pending constants into the felt buffer. Only called where the + /// run is known to end on a felt boundary, so no group is left partial. + fn flush_pending(&mut self, b: &mut LfmBuilder) { + if self.pending.is_empty() { + return; + } + let packed = pack_const_bytes(b, &self.pending); + self.pending.clear(); + self.buf.extend(packed); + } + + /// Every absorb drops the buffered squeeze output, so a challenge can never + /// predate the input it must depend on (`default_transcript.rs:206-210`). + fn invalidate(&mut self) { + self.out_pos = CANDIDATES_PER_SQUEEZE; + } +} + +/// Constant bytes as the felts `sponge_leaf_bytes` would read: 8-byte +/// BIG-endian groups, the trailing partial group zero-extended on the LOW side +/// (`rpx/mod.rs:396-406` writes the available bytes at the FRONT of a zeroed +/// eight-byte buffer, so the missing ones are the low bytes). +fn pack_const_bytes(b: &mut LfmBuilder, bytes: &[u8]) -> Vec { + const_byte_felts(bytes) + .into_iter() + .map(|felt| b.felt_const(felt)) + .collect() +} + +/// The values [`pack_const_bytes`] interns: `rpx::felts_from_bytes`, the rule +/// both sponges read bytes by. +fn const_byte_felts(bytes: &[u8]) -> Vec { + bytes + .chunks(BYTES_PER_FELT) + .map(|group| { + let mut whole = [0u8; BYTES_PER_FELT]; + whole[..group.len()].copy_from_slice(group); + FE::from(u64::from_be_bytes(whole)) + }) + .collect() +} + +/// INSTRUCTIONS a squeeze emits over `felts` buffered felts. +/// +/// `ceil(felts/4)` `Pack` rows to build the sponge's words, `ceil(felts/8)` +/// permutations (a block is two words), and one `Unpack` to read the digest's +/// four lanes. An empty buffer never permutes — `sponge_leaf_bytes` returns the +/// zero digest — so only the `Unpack` remains. +pub const fn squeeze_rows(felts: usize) -> usize { + felts.div_ceil(4) + felts.div_ceil(8) + 1 +} + +/// INSTRUCTIONS a `state()` emits over `felts` buffered felts: the same hash +/// without the digest `Unpack`, because the grind consumes the digest as a word. +pub const fn state_rows(felts: usize) -> usize { + felts.div_ceil(4) + felts.div_ceil(8) +} + +/// PERMUTATIONS a transcript hash over `felts` buffered felts costs, whether it +/// is a squeeze or a `state()`: one per rate-8 block, and NONE for an empty +/// buffer, which returns the zero digest without permuting +/// (`edsl.rs:694-697`). +/// +/// The rate is [`super::whir_open::RATE_FELTS`] rather than a literal, because +/// it is the same sponge the leaves are hashed with. +pub const fn sponge_perms(felts: usize) -> usize { + felts.div_ceil(super::whir_open::RATE_FELTS) +} + +/// INSTRUCTIONS an absorbed extension element or commitment emits: the one +/// `Unpack` that turns the cell into lanes. +pub const fn absorb_unpack_rows() -> usize { + 1 +} + +/// INSTRUCTIONS `sample_ext` emits beyond its squeezes: one `Pack`. +pub const fn sample_ext_rows() -> usize { + 1 +} + +/// INSTRUCTIONS `sample_u64_pow2` emits beyond its squeezes: one `BitDec`. +pub const fn sample_u64_rows() -> usize { + 1 +} + +/// Felts a squeeze's digest occupies when it is re-absorbed: 32 bytes. +pub const DIGEST_FELTS: usize = 4; + +/// The transcript state a chain is ENTERED with. +/// +/// The schedule below is a function of the round structure and of this, and of +/// nothing else. A standalone chain enters [`SpongeEntry::fresh`]; a chain +/// inside an assembled verifier enters whatever the statement and the tables +/// before it left behind, which is why this is a parameter rather than an +/// assumption. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SpongeEntry { + /// Felts the sponge is holding but has not hashed. + pub buffered_felts: usize, + /// Candidates of the last squeeze already handed out; + /// `CANDIDATES_PER_SQUEEZE` means "none in hand, the next draw squeezes". + pub out_pos: usize, + /// The Poseidon1 sponge's state instead, when the program verifies a + /// Poseidon1 base (the two fields above are then unused). + pub p1: Option, +} + +/// Where a Poseidon1 sponge stands (`poseidon1_stark::Transcript`): lanes +/// pending in its rate, and output lanes not yet squeezed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct P1SpongeEntry { + pub pending: usize, + pub unread: usize, +} + +impl SpongeEntry { + /// `WhirTranscript::new()`, which is `DefaultTranscript::new(&[])`: an empty + /// buffer and no squeeze in hand (`default_transcript.rs:67-77`). + pub const fn fresh() -> Self { + Self { + buffered_felts: 0, + out_pos: CANDIDATES_PER_SQUEEZE, + p1: None, + } + } + + /// The empty transcript of a program built under `hash`: [`Self::fresh`], + /// or the empty Poseidon1 sponge (nothing pending, nothing to read). + pub const fn fresh_for(hash: WrapHash) -> Self { + match hash { + WrapHash::Poseidon1 => Self { + p1: Some(P1SpongeEntry { + pending: 0, + unread: 0, + }), + ..Self::fresh() + }, + _ => Self::fresh(), + } + } +} + +/// One hash the transcript performs, and how many felts it hashes. +/// +/// Told apart because they cost differently and because a `state()` does NOT +/// advance the chain: the buffer it hashed is still there for the next squeeze +/// to hash again. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SpongeHash { + /// `sample()`: hash the buffer, hand out four candidates, re-absorb the + /// digest. + Squeeze(usize), + /// `state()`: hash the buffer and leave it alone. + State(usize), +} + +impl SpongeHash { + pub const fn felts(self) -> usize { + match self { + Self::Squeeze(felts) | Self::State(felts) => felts, + } + } + + pub const fn rows(self) -> usize { + match self { + Self::Squeeze(felts) => squeeze_rows(felts), + Self::State(felts) => state_rows(felts), + } + } + + pub const fn perms(self) -> usize { + sponge_perms(self.felts()) + } +} + +/// The sponge as the SCHEDULE sees it: what it holds and what it has in hand. +/// +/// Emits nothing. Every transition is one of the host's +/// (`default_transcript.rs`), named after it. +pub struct SpongeSchedule { + buffered: usize, + out_pos: usize, + hashes: Vec, + /// The Poseidon1 sponge instead, and the permutations it has spent. + p1: Option<(P1SpongeEntry, usize)>, +} + +/// Lanes of rate in a Poseidon1 sponge block, and lanes one permutation's +/// output hands out. +const P1_RATE: usize = super::p1w16_emit::RATE_FELTS; +const P1_OUT: usize = 16; + +impl SpongeSchedule { + pub fn new(entry: SpongeEntry) -> Self { + Self { + buffered: entry.buffered_felts, + out_pos: entry.out_pos, + hashes: Vec::new(), + p1: entry.p1.map(|p| (p, 0)), + } + } + + /// `append_bytes` / `append_field_element`: the buffer grows and any + /// buffered squeeze output is dropped (`:205-210`). Under Poseidon1, each + /// felt is a lane: a full rate permutes (`Transcript::put1`). + pub fn absorb(&mut self, felts: usize) { + if let Some((p, perms)) = &mut self.p1 { + for _ in 0..felts { + p.pending += 1; + p.unread = 0; + if p.pending == P1_RATE { + *perms += 1; + p.pending = 0; + p.unread = P1_OUT; + } + } + return; + } + self.buffered += felts; + self.out_pos = CANDIDATES_PER_SQUEEZE; + } + + /// `append_bytes(root, 32)`: four felts, and under Poseidon1 the length + /// before them. + pub fn absorb_root(&mut self) { + let len = usize::from(self.p1.is_some()); + self.absorb(len + DIGEST_FELTS); + } + + /// `sample()`: the buffer is hashed, and the digest is re-absorbed — which + /// is why the buffer is FOUR felts afterwards and not none (`:104-110`). + pub fn squeeze(&mut self) { + self.hashes.push(SpongeHash::Squeeze(self.buffered)); + self.buffered = DIGEST_FELTS; + self.out_pos = 0; + } + + /// `next_sample_u64`: refill only when nothing is in hand. + pub fn candidate(&mut self) { + if let Some((p, perms)) = &mut self.p1 { + if p.unread == 0 { + *perms += 1; + p.pending = 0; + p.unread = P1_OUT; + } + p.unread -= 1; + return; + } + if self.out_pos >= CANDIDATES_PER_SQUEEZE { + self.squeeze(); + } + self.out_pos += 1; + } + + /// `sample_field_element` on the cubic extension: one candidate a + /// coordinate, no rejection (`CANDIDATES_PER_COORDINATE = Some(1)`). + pub fn draw_ext(&mut self) { + for _ in 0..COORDINATES_PER_EXT { + self.candidate(); + } + } + + /// `check_grind`: at zero bits it returns before reading the state and + /// before absorbing the nonce (`whir_chain.rs:131-133`), so it is not in the + /// schedule at all. Otherwise `state()` over the buffer, then the nonce's + /// eight big-endian bytes. + pub fn grind(&mut self, bits: usize) { + if bits == 0 { + return; + } + if let Some((p, perms)) = &mut self.p1 { + // `state()` flushes a COPY: one permutation when lanes are pending, + // none adopted. Then the nonce's `append_bytes`, length and felt. + if p.pending > 0 { + *perms += 1; + } + self.absorb(1 + NONCE_FELTS); + return; + } + self.hashes.push(SpongeHash::State(self.buffered)); + self.absorb(NONCE_FELTS); + } +} + +/// Felts the grind's nonce occupies: `nonce.to_be_bytes()` is eight. +const NONCE_FELTS: usize = 1; + +impl SpongeSchedule { + /// The hashes performed, in order — what a leg's schedule half is summed + /// from. + pub fn hashes(&self) -> &[SpongeHash] { + &self.hashes + } + + /// Where the sponge is left, so one leg's schedule can be continued by the + /// next in the same program. + pub fn entry(&self) -> SpongeEntry { + SpongeEntry { + buffered_felts: self.buffered, + out_pos: self.out_pos, + p1: self.p1.map(|(p, _)| p), + } + } + + /// INSTRUCTIONS the hashes cost. Under Poseidon1 an estimate, four a + /// permutation (its rate words and the `Hash16` row): the census reads the + /// emitted program for the exact count. + pub fn rows(&self) -> usize { + if let Some((_, perms)) = self.p1 { + return 4 * perms; + } + self.hashes.iter().map(|hash| hash.rows()).sum() + } + + /// PERMUTATIONS the hashes cost: under Poseidon1, `Hash16` rows. + pub fn perms(&self) -> usize { + if let Some((_, perms)) = self.p1 { + return perms; + } + self.hashes.iter().map(|hash| hash.perms()).sum() + } +} + +/// The cubic extension element a squeeze's lanes would produce, for tests that +/// need the host's value beside the machine's. +pub fn ext_from_lanes(a0: FE, a1: FE, a2: FE) -> FEE { + FEE::new([a0, a1, a2]) +} + +/// ★ `whir_chain::check_grind` (`crypto/multilinear/src/whir_chain.rs:125-139`), +/// emitted — and it is a REUSE, not a port. +/// +/// The check is +/// `is_valid_nonce::>(&transcript.state(), nonce, bits)`, and +/// `GrindingDigest` is the TRANSCRIPT's digest (`whir_hash.rs:54`), which on +/// this arm is `Rpx256Digest` — the same leaf-domain sponge the replay above +/// uses. So `H(H(PREFIX ‖ seed ‖ factor) ‖ nonce_be)` is exactly what +/// [`super::epoch::emit_grinding_check`]'s algebraic arm already computes: both +/// preimages are already felts (41 bytes is six big-endian groups, 40 is five), +/// the inner digest cell IS the outer preimage's first rate cell so nothing +/// repacks it, and the range check is one `BitDec` whose top `bits` bits are +/// asserted zero. ⚠ My own sizing note called this "a port, not a reuse"; that +/// was wrong, and reusing it is also what keeps ONE definition of the lane +/// placement rather than two. +/// +/// `bits == 0` returns immediately, as the host does — with NO state read and +/// NO nonce absorb. Both are observable: a state read is counted separately +/// from a squeeze, and an absorb would move every later challenge. +/// +/// ⚠ A nonce at or above `p` reduces, on both sides: the host's sponge reads the +/// eight big-endian bytes through `Fp::from(u64)` exactly as the machine holds +/// the felt. It is a completeness restriction, not a disagreement — and search +/// returns small nonces, so it is unreachable in practice. +pub fn emit_grind_check( + b: &mut LfmBuilder, + transcript: &mut WhirTranscript, + bits: u8, + nonce: Felt, +) { + if bits == 0 { + return; + } + let seed = transcript.state(b); + super::epoch::emit_grinding_check(b, super::edsl::WrapDigest::from_cell(seed), nonce, bits); + // `append_bytes(&nonce.to_be_bytes())`: eight big-endian bytes are one felt. + transcript.absorb_nonce(b, nonce); +} + +/// INSTRUCTIONS [`emit_grind_check`] emits beyond the transcript state's own +/// hash, which costs [`state_rows`] of whatever the buffer holds. +/// +/// One `Unpack` of the seed; two `Pack`s and one permutation for the inner +/// hash; one `Pack` and one permutation for the outer hash; one `Unpack` of the +/// result; and the range check, one `BitDec` plus two rows per asserted zero +/// bit. The nonce absorb is free — it is already a felt. So `8 + 2·bits`. +/// +/// ★ The inner digest is NEVER unpacked, and that is the shape's one real +/// saving: it IS the outer preimage's first rate cell, so nothing repacks it +/// (`epoch.rs:624-626`). An earlier version of this form charged an `Unpack` +/// for it, and F1 said so. +pub const fn grind_check_rows(bits: usize) -> usize { + if bits == 0 { + return 0; + } + let seed_unpack = 1; + let inner = 2 + 1; + let outer = 1 + 1; + let digest_unpack = 1; + let range = 1 + 2 * bits; + seed_unpack + inner + outer + digest_unpack + range +} + +/// `LFM_CONST` rows the grind interns BEYOND the capacity words of its two +/// hashes (which are `leaf_capacity(6)` for the 41-byte inner preimage and +/// `leaf_capacity(5)` for the 40-byte outer one, and are shared with any other +/// hash of those widths): the PREFIX felt and the factor felt. +/// +/// ⛔⛔ **NEVER SUM THIS ACROSS GRINDS, AND NEVER USE IT FOR A POOL.** It is a +/// COUNT, and counts ADD where values MERGE: nineteen grinds at one width share +/// all four words and pay for them once. Worse, the factor felt is keyed on the +/// BIT COUNT, so two widths intern five words and not eight — an arithmetic no +/// scalar can express. +/// +/// ⇒ The values form is the words themselves, and the program-level answer is +/// their union over the distinct bit counts. This +/// function stays correct for exactly one thing: what ONE grind of a width no +/// other leg shares would intern. +pub const fn grind_check_const_felts() -> usize { + 2 +} diff --git a/prover/src/lfm/whir_transcript_kat_tests.rs b/prover/src/lfm/whir_transcript_kat_tests.rs new file mode 100644 index 000000000..a7ef37743 --- /dev/null +++ b/prover/src/lfm/whir_transcript_kat_tests.rs @@ -0,0 +1,344 @@ +//! The four sequence vectors, and what each one is for. + +use super::whir_transcript_kats::{Script, Step, run}; + +/// K1 — ORDER. Distinct, asymmetric values, so swapping any two absorbs moves +/// the result. A script of equal values would pass a swap. +const K1_ORDER: Script = Script { + name: "order", + steps: &[ + Step::AbsorbBytes { + seed: 0x11, + len: 32, + }, + Step::AbsorbExt { seed: 0x2222_2222 }, + Step::SampleExt, + Step::AbsorbBytes { seed: 0x33, len: 8 }, + Step::SampleExt, + Step::SampleU64 { bound: 1 << 20 }, + ], +}; + +/// K2 — WIDTH AND PADDING. Every length class mod 8 and mod 64, because the +/// leaf sponge's padding flag is `len mod 8` and its block is 64 bytes. A +/// trailing partial group zero-extended on the wrong side lands here. +const K2_WIDTH: Script = Script { + name: "width", + steps: &[ + Step::AbsorbBytes { seed: 0x41, len: 0 }, + Step::SampleExt, + Step::AbsorbBytes { seed: 0x42, len: 1 }, + Step::SampleExt, + Step::AbsorbBytes { seed: 0x43, len: 7 }, + Step::SampleExt, + Step::AbsorbBytes { seed: 0x44, len: 8 }, + Step::SampleExt, + Step::AbsorbBytes { seed: 0x45, len: 9 }, + Step::SampleExt, + Step::AbsorbBytes { + seed: 0x46, + len: 63, + }, + Step::SampleExt, + Step::AbsorbBytes { + seed: 0x47, + len: 64, + }, + Step::SampleExt, + Step::AbsorbBytes { + seed: 0x48, + len: 65, + }, + Step::SampleExt, + Step::AbsorbBytes { + seed: 0x49, + len: 72, + }, + Step::SampleExt, + ], +}; + +/// K3 — CHAIN ADVANCE. Consecutive draws with no absorb between them. A +/// squeeze re-absorbs its own output; drop that and every draw after the first +/// repeats, which one squeeze cannot show. +const K3_CHAIN: Script = Script { + name: "chain", + steps: &[ + Step::AbsorbBytes { + seed: 0x51, + len: 32, + }, + Step::SampleExt, + Step::SampleExt, + Step::SampleExt, + Step::SampleExt, + Step::SampleExt, + Step::SampleExt, + ], +}; + +/// K4 — SAMPLER OFFSET. An asymmetric interleaving of the two samplers and an +/// absorb. An extension element takes three of a squeeze's four candidates and +/// an absorb discards the rest, so a one-candidate slip still produces +/// plausible field elements and is invisible to anything but pinned values. +const K4_OFFSET: Script = Script { + name: "offset", + steps: &[ + Step::AbsorbBytes { + seed: 0x61, + len: 24, + }, + Step::SampleU64 { bound: 1 << 23 }, + Step::SampleExt, + Step::SampleU64 { bound: 1 << 19 }, + Step::SampleU64 { bound: 1 << 19 }, + Step::SampleExt, + Step::AbsorbBytes { + seed: 0x62, + len: 32, + }, + Step::SampleExt, + Step::SampleU64 { bound: 4 }, + ], +}; + +const SCRIPTS: &[&Script] = &[&K1_ORDER, &K2_WIDTH, &K3_CHAIN, &K4_OFFSET]; + +/// The same four scripts, for the machine replay's gates: the emitter is +/// checked against the host on exactly the sequences these vectors pin. +pub fn scripts() -> &'static [&'static Script] { + SCRIPTS +} + +/// Prints the vectors in the form the table below takes. Run after a +/// deliberate change to the transcript, never to make a red test green. +#[test] +#[ignore = "generator, not a test"] +fn print_whir_transcript_vectors() { + for s in SCRIPTS { + let o = run(s); + let ext: Vec = o + .ext + .iter() + .map(|e| { + let [a, b, c] = *e.value(); + format!("({}, {}, {})", a.value(), b.value(), c.value()) + }) + .collect(); + println!("// {}", s.name); + println!("state {}", hex(&o.state)); + println!("ext [{}]", ext.join(", ")); + println!("u64s {:?}\n", o.u64s); + } +} + +fn hex(b: &[u8; 32]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +/// ★ The vectors. GENERATED by the test above — recordings of this +/// transcript, not a specification of it (see the module header). +/// +/// ⚠ The SAMPLED VALUES are pinned, not only the state, and that is +/// load-bearing rather than thorough. A squeeze re-absorbs its output, so the +/// chain advances once per SQUEEZE — a sampler reading the wrong eight bytes of +/// the RIGHT squeeze returns a different value from an identical chain, leaving +/// the final state untouched. Pinning the state alone would have been blind to +/// exactly the failure K4 exists to catch. +pub struct Vector { + pub name: &'static str, + pub state: &'static str, + /// Sampled extension elements, three coordinates each, flattened. + pub ext: &'static [u64], + pub u64s: &'static [u64], +} + +const VECTORS: &[Vector] = &[ + Vector { + name: "order", + state: "c59cae7a0b68bd9979abad5ccdfb56a6fe41376e33b9c6af33c9d3abbf771e8b", + ext: &[ + 5076344684520935901, + 13481200151466509943, + 16683032029069390868, + 1376745799081504414, + 17851777139959444118, + 1864588735162070927, + ], + u64s: &[144076], + }, + Vector { + name: "width", + state: "3d35cdbb8dd36f6937a25b596c077a630c3b5d8917073d61cbf82ebfd7076f19", + ext: &[ + 0, + 0, + 0, + 13779361758518057625, + 12861723740627776268, + 1850467284123532435, + 1483117093021199727, + 15817517280666576414, + 9696778708142809560, + 12000993751906614372, + 8022334130801498047, + 11287794890553842288, + 9755754461561997648, + 11710191894463407300, + 122339190123564887, + 4801124081631589251, + 210538569640777557, + 11301051852487502001, + 3437677669383980337, + 15204956286457165234, + 15610975559828333221, + 14734763767558745501, + 9832650469355206763, + 5246191761053816184, + 9584292669402132547, + 18430272915700838563, + 4262435669781350388, + ], + u64s: &[], + }, + Vector { + name: "chain", + state: "17ed4136b20cb6bf075fb0a69bcb70bd2f84e85d0d616b777f86dc4472bdb11f", + ext: &[ + 12482777032374998597, + 17123056680038681210, + 15285331874205903840, + 11971505191909278852, + 17560771582447933866, + 9573896076222905442, + 8298914713805243577, + 11930008621129790039, + 8209673665337805593, + 11264933134106569355, + 15636484629095859538, + 2339114142658380897, + 9932494386427677841, + 17740759830813518525, + 5773334258638364270, + 3369130016219959262, + 9334649995654086438, + 15117490715794284312, + ], + u64s: &[], + }, + Vector { + name: "offset", + state: "73bf7c069df5f3d26760c9fd3d3c5259aaddc92a8667170ace9fb98407ef50b9", + ext: &[ + 5510470753300211043, + 6140529258356735583, + 13714250910034146583, + 14904270912001888584, + 8534709010995452892, + 10669800597048532156, + 10638644543122064108, + 4984655170130837497, + 9626235538098858159, + ], + u64s: &[785996, 520380, 330140, 1], + }, +]; + +#[test] +fn the_whir_transcript_scripts_reproduce_their_vectors() { + for (s, want) in SCRIPTS.iter().zip(VECTORS) { + assert_eq!( + s.name, want.name, + "the script table and the vector table must agree in order" + ); + let got = run(s); + let flat: Vec = got + .ext + .iter() + .flat_map(|e| { + let [a, b, c] = *e.value(); + [*a.value(), *b.value(), *c.value()] + }) + .collect(); + assert_eq!( + flat.as_slice(), + want.ext, + "{}: a sampled extension element moved — the draw schedule or the \ + buffer offset changed", + want.name + ); + assert_eq!( + got.u64s.as_slice(), + want.u64s, + "{}: a sampled u64 moved", + want.name + ); + assert_eq!( + hex(&got.state), + want.state, + "{}: the transcript's sequence moved. If that was deliberate, rerun \ + `print_whir_transcript_vectors` and say in the commit which absorb \ + or draw changed; if it was not, the emitter and the host have \ + diverged", + want.name + ); + } +} + +/// ⚠ An edge the vectors above happen to record, worth its own name. +/// +/// `sponge_leaf_bytes` returns the capacity's low lanes without permuting when +/// there is nothing to absorb, and those lanes are zero — so squeezing a +/// transcript that has absorbed nothing yields the ZERO digest and hence a zero +/// challenge. Production never reaches it: every WHIR transcript absorbs its +/// statement tag before drawing anything. This test is here so that a path +/// which one day samples first fails loudly rather than drawing a constant. +#[test] +fn an_empty_rpx_transcript_squeezes_to_zero_which_production_never_reaches() { + let empty = Script { + name: "empty", + steps: &[Step::SampleExt], + }; + let o = run(&empty); + let [a, b, c] = *o.ext[0].value(); + assert_eq!( + (*a.value(), *b.value(), *c.value()), + (0, 0, 0), + "the empty squeeze is documented as the un-permuted capacity; if it is \ + no longer zero the sponge's empty case changed" + ); + // And the guard that makes it harmless: a statement tag first. + let tagged = Script { + name: "tagged", + steps: &[ + Step::AbsorbBytes { + seed: 0x71, + len: 41, + }, + Step::SampleExt, + ], + }; + let t = run(&tagged); + let [a, b, c] = *t.ext[0].value(); + assert_ne!( + (*a.value(), *b.value(), *c.value()), + (0, 0, 0), + "a challenge drawn after a statement absorb must not be the zero element" + ); +} + +/// K3's own property, which no single pinned value can express: a squeeze +/// re-absorbs its output, so consecutive draws cannot repeat. +#[test] +fn consecutive_draws_never_repeat_because_a_squeeze_advances_the_chain() { + let o = run(&K3_CHAIN); + for i in 0..o.ext.len() { + for j in (i + 1)..o.ext.len() { + assert_ne!( + o.ext[i], o.ext[j], + "draws {i} and {j} are equal — the squeeze did not chain its \ + output back in, and every draw after the first is a repeat" + ); + } + } +} diff --git a/prover/src/lfm/whir_transcript_kats.rs b/prover/src/lfm/whir_transcript_kats.rs new file mode 100644 index 000000000..cb1bbce0a --- /dev/null +++ b/prover/src/lfm/whir_transcript_kats.rs @@ -0,0 +1,107 @@ +//! Sequence KATs for the WHIR transcript the LFM replay has to reproduce. +//! +//! [`crypto::fiat_shamir::transcript_hash`]'s own tests pin what an RPX squeeze +//! IS — unreversed, four canonical felts, consumed in whole groups, one +//! candidate per coordinate. They say nothing about ORDER, because order is not +//! a property of the sponge; it is a property of the verifier that drives it. +//! The replay's failure mode is a sequence that differs from the host's by one +//! absorb or one draw, which every one of those tests passes. +//! +//! So these are scripts. A [`Script`] is a list of [`Step`]s, running one +//! produces the values it sampled and the state it left, and a vector is a +//! script beside what it produced. When the emitter exists it replays the same +//! scripts and is checked against the same vectors — the host is Rust driving a +//! byte sponge and the emitter is a straight-line field machine, so agreement +//! between them is a comparison of two constructions rather than a round trip +//! through shared code. +//! +//! # ⚠ What these vectors are, and what they are not +//! +//! They are **RECORDINGS**, not a specification. The LFM's other transcript +//! KATs ([`super::transcript_kats`]) come from an oracle written before any +//! Rust existed, which is why they can say the implementation is right. These +//! cannot: there is no second implementation of this sponge to take them from. +//! +//! What they catch is a transcript order that CHANGES — any edit to the +//! verifier's sequence moves a vector, and an emitter that drifts from the host +//! fails against one. What they cannot catch is the host and the emitter being +//! wrong together, i.e. an order that was never right. That is guarded by prose +//! instead: the sequence is written out in `V1-sizing-note.md` part (4) and the +//! preprocessed root's position in W1-B's design note §8, and the vectors here +//! are what makes an unintended divergence from those loud. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; + +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +type Transcript = DefaultTranscript; + +/// One step of a transcript script. +/// +/// Field elements are named by a seed rather than written out, so a script +/// stays readable as data and a vector's inputs cannot silently disagree with +/// its outputs. +#[derive(Clone, Copy, Debug)] +pub enum Step { + /// `append_bytes` of `n` bytes, filled deterministically from `seed`. + AbsorbBytes { seed: u8, len: usize }, + /// `append_field_element` of the element `seed` names. + AbsorbExt { seed: u64 }, + /// `sample_field_element`, recorded. + SampleExt, + /// `sample_u64(bound)`, recorded. + SampleU64 { bound: u64 }, +} + +/// A named sequence of steps. +pub struct Script { + pub name: &'static str, + pub steps: &'static [Step], +} + +/// What running a script produced. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Observed { + /// Every sampled extension element, in order. + pub ext: Vec, + /// Every sampled `u64`, in order. + pub u64s: Vec, + /// The transcript state the script left. + pub state: [u8; 32], +} + +pub fn bytes_from(seed: u8, len: usize) -> Vec { + (0..len) + .map(|i| seed.wrapping_mul(31).wrapping_add(i as u8)) + .collect() +} + +pub fn ext_from(seed: u64) -> FEE { + FEE::new([ + FE::from(seed), + FE::from(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)), + FE::from(seed.wrapping_add(0x1234_5678_9ABC_DEF0)), + ]) +} + +/// Runs a script against the production transcript type. +pub fn run(script: &Script) -> Observed { + let mut t = Transcript::new(&[]); + let mut ext = Vec::new(); + let mut u64s = Vec::new(); + for step in script.steps { + match *step { + Step::AbsorbBytes { seed, len } => t.append_bytes(&bytes_from(seed, len)), + Step::AbsorbExt { seed } => t.append_field_element(&ext_from(seed)), + Step::SampleExt => ext.push(t.sample_field_element()), + Step::SampleU64 { bound } => u64s.push(t.sample_u64(bound)), + } + } + Observed { + ext, + u64s, + state: t.state(), + } +} diff --git a/prover/src/lfm/whir_transcript_replay_tests.rs b/prover/src/lfm/whir_transcript_replay_tests.rs new file mode 100644 index 000000000..d28c694c6 --- /dev/null +++ b/prover/src/lfm/whir_transcript_replay_tests.rs @@ -0,0 +1,561 @@ +//! The replay against the vectors — the machine on one side, the host on the +//! other. +//! +//! [`super::whir_transcript_kat_tests`] pins what the HOST produces for four +//! scripts (K1 order, K2 width and padding, K3 chain advance, K4 sampler +//! offset). These run the same scripts through the emitter and compare against +//! the host directly, so the two sides are a Rust byte sponge and a +//! straight-line field machine rather than two halves of one implementation. +//! +//! W1's position KAT — where the derived DECODE preprocessed root is absorbed — +//! is CITED, not duplicated: that is a property of the epoch program's order, +//! not of this object. + +use crypto::hash::rpx::{digest_to_commitment, sponge_leaf_bytes}; + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Bit, Ext, Felt, LfmBuilder}; +use super::compiler::{LfmProgram, compile}; +use super::executor::execute; +use super::validator::validate; +use super::whir_transcript::{ + WhirTranscript, absorb_unpack_rows, sample_ext_rows, sample_u64_rows, squeeze_rows, state_rows, +}; +use super::whir_transcript_kats::{Observed, Script, Step, bytes_from, ext_from, run}; +use super::word::{LfmWord, ext_word, word_as_ext}; + +/// `LFM_CONST` rows any program containing one leaf hash interns: the zero word +/// — which `felt_const(0)`, the empty digest and `pack_ext`'s lane 3 all share, +/// because interning is by canonical word — and the capacity word, which +/// carries the padding flag and so is one per distinct buffer length. +fn leaf_hash_consts(distinct_lengths: usize) -> usize { + 1 + distinct_lengths +} + +fn builder() -> LfmBuilder { + LfmBuilder::new().with_wrap_hash(super::edsl::WrapHash::production()) +} + +fn run_program(program: &LfmProgram, arenas: &[Vec]) -> Vec { + let exec = + execute(program, arenas, &crate::hash_pin::BLOCK_HASHER).expect("the replay must execute"); + exec.public_words.iter().map(|(_, word)| *word).collect() +} + +// --------------------------------------------------------------------------- +// The gate: the scripts the host vectors pin, replayed by the machine. +// --------------------------------------------------------------------------- + +/// Every `AbsorbExt` value of a script, HINTED rather than folded in as a +/// constant, so the runtime absorb path and its felt-alignment rule are what +/// run. +fn runtime_inputs(script: &Script) -> Vec { + script + .steps + .iter() + .filter_map(|step| match step { + Step::AbsorbExt { seed } => Some(ext_from(*seed)), + _ => None, + }) + .collect() +} + +struct Emitted { + program: LfmProgram, + ext_count: usize, + u64_bits: Vec, + rows: usize, +} + +fn emit(script: &Script) -> Emitted { + let inputs = runtime_inputs(script); + let mut b = builder(); + let arena = b.declare_arena(inputs.len().max(1) as u32); + let hinted: Vec = (0..inputs.len()) + .map(|index| b.hint_word(arena, index as u32).as_ext()) + .collect(); + + let mut transcript = WhirTranscript::new(); + let mut next_input = 0usize; + let mut sampled: Vec = Vec::new(); + let mut drawn: Vec> = Vec::new(); + let mut u64_bits: Vec = Vec::new(); + + for step in script.steps { + match *step { + Step::AbsorbBytes { seed, len } => { + transcript.absorb_const_bytes(&bytes_from(seed, len)); + } + Step::AbsorbExt { .. } => { + transcript.absorb_ext(&mut b, hinted[next_input]); + next_input += 1; + } + Step::SampleExt => sampled.push(transcript.sample_ext(&mut b)), + Step::SampleU64 { bound } => { + let nbits = bound.trailing_zeros() as usize; + assert_eq!( + bound, + 1u64 << nbits, + "{}: the replay's bounded sampler is the power-of-two one", + script.name + ); + u64_bits.push(nbits); + drawn.push(transcript.sample_u64_pow2(&mut b, nbits)); + } + } + } + let state = transcript.state(&mut b); + + for value in &sampled { + b.public(value.as_cell()); + } + for bits in &drawn { + for bit in bits { + b.public(bit.as_cell()); + } + } + b.public(state); + + let publics = sampled.len() + u64_bits.iter().sum::() + 1; + let program = compile(b.finish()); + validate(&program).expect("the replay must be admissible"); + let rows = program.instrs.len() - inputs.len() - publics; + Emitted { + program, + ext_count: sampled.len(), + u64_bits, + rows, + } +} + +fn arena_for(script: &Script) -> Vec { + let inputs = runtime_inputs(script); + if inputs.is_empty() { + return vec![[FE::zero(); 4]]; + } + inputs.iter().map(ext_word).collect() +} + +fn observe(script: &Script, emitted: &Emitted) -> Observed { + let words = run_program(&emitted.program, &[arena_for(script)]); + let mut at = 0usize; + let ext: Vec = (0..emitted.ext_count) + .map(|_| { + let value = word_as_ext(&words[at]).expect("a published challenge"); + at += 1; + value + }) + .collect(); + let u64s: Vec = emitted + .u64_bits + .iter() + .map(|&nbits| { + let mut value = 0u64; + for bit in 0..nbits { + let lane = words[at][0]; + assert!( + lane == FE::zero() || lane == FE::one(), + "a published bit must be zero or one" + ); + if lane == FE::one() { + value |= 1 << bit; + } + at += 1; + } + value + }) + .collect(); + let digest = words[at]; + let state = digest_to_commitment(&[digest[0], digest[1], digest[2], digest[3]]); + Observed { ext, u64s, state } +} + +/// ★ THE GATE. Every script, three streams: the sampled extension elements, the +/// sampled `u64`s, and the transcript state left at the end. +/// +/// The state is the strong one — a function of the whole buffer history — so a +/// wrong clear, a missing chain advance or a mis-packed felt moves it even when +/// every challenge happens to look plausible. +#[test] +fn the_replay_reproduces_what_the_host_transcript_produces() { + for script in super::whir_transcript_kat_tests::scripts() { + let emitted = emit(script); + let machine = observe(script, &emitted); + let host = run(script); + assert_eq!( + machine.ext, host.ext, + "{}: the extension challenges must match the host's", + script.name + ); + assert_eq!( + machine.u64s, host.u64s, + "{}: the bounded draws must match the host's", + script.name + ); + assert_eq!( + machine.state, host.state, + "{}: the transcript state must match the host's", + script.name + ); + println!( + "{:>6}: {:>2} challenges, {} bounded draws, {:>3} rows emitted", + script.name, + machine.ext.len(), + machine.u64s.len(), + emitted.rows + ); + } +} + +/// ★ K5, in the world where the reversal is gone: a squeeze's four lanes ARE +/// the leaf digest of the buffer, unreversed. +/// +/// `RpxTranscriptHash::REVERSES_SQUEEZE` is `false`, so `sample()` hands back +/// `digest_to_commitment(sponge_leaf_bytes(buf))` as it stands. Pinned against +/// the host's own two functions at every length class mod 8 and mod 64. +#[test] +fn a_squeeze_is_the_unreversed_leaf_digest_of_the_buffer() { + for len in [0usize, 1, 7, 8, 9, 63, 64, 65, 72] { + let bytes = bytes_from(0x5A, len); + let mut b = builder(); + let mut transcript = WhirTranscript::new(); + transcript.absorb_const_bytes(&bytes); + for lane in transcript.squeeze(&mut b) { + b.public(lane.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("admissible"); + let words = run_program(&program, &[]); + + let want = sponge_leaf_bytes(&bytes); + let machine: [FE; 4] = core::array::from_fn(|i| words[i][0]); + assert_eq!( + machine, want, + "{len} bytes: the squeeze's lanes must be the leaf digest's felts, unreversed" + ); + assert_eq!( + digest_to_commitment(&machine), + digest_to_commitment(&want), + "{len} bytes: and so must the commitment bytes a byte consumer would see" + ); + } +} + +/// ★ The alignment rule is ENFORCED, not remembered. +/// +/// This is what "hold the first window" looks like in code: the emitter cannot +/// silently produce a byte shift, so a statement whose computed padding has not +/// landed fails loudly at emit time rather than producing a program that is +/// right on a fixture and wrong on a block. +#[test] +#[should_panic(expected = "must start on a felt boundary")] +fn a_runtime_absorb_at_an_unaligned_offset_refuses() { + let mut b = builder(); + let arena = b.declare_arena(1); + let value = b.hint_word(arena, 0).as_ext(); + let mut transcript = WhirTranscript::new(); + transcript.absorb_const_bytes(&[0u8; 5]); + transcript.absorb_ext(&mut b, value); +} + +// --------------------------------------------------------------------------- +// The cost pins: four purpose-built programs whose every row is named. +// --------------------------------------------------------------------------- + +/// `n` hinted felts absorbed, then one squeeze, then the four lanes published. +fn squeeze_only(n: usize) -> LfmProgram { + let mut b = builder(); + let arena = b.declare_arena(n.max(1) as u32); + let felts: Vec = (0..n) + .map(|index| b.hint_felt(arena, index as u32)) + .collect(); + let mut transcript = WhirTranscript::new(); + transcript.absorb_felts(&mut b, &felts); + for lane in transcript.squeeze(&mut b) { + b.public(lane.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("admissible"); + program +} + +/// ★ F1 for the squeeze, across the width classes: every row named. +/// +/// `n` hints, the leaf hash's two interned constants, the squeeze itself, and +/// four publics. The three terms of `squeeze_rows` grow differently in `n` — one +/// `Pack` per four felts, one permutation per eight, one `Unpack` always — so a +/// sweep separates them rather than fixing only their sum. +#[test] +fn the_squeeze_costs_its_closed_form() { + for n in [0usize, 1, 3, 4, 5, 8, 9, 16, 17] { + let program = squeeze_only(n); + let measured = program.instrs.len(); + let predicted = n + leaf_hash_consts(1) + squeeze_rows(n) + 4; + println!("squeeze over {n:>2} felts: {measured:>3} rows emitted, {predicted:>3} predicted"); + assert_eq!( + measured, predicted, + "a squeeze over {n} felts must emit its closed form" + ); + } +} + +/// ★ F1 for the three one-row operations, each isolated in its own program. +/// +/// An absorbed extension element is one `Unpack` and contributes three felts to +/// the buffer; a challenge is one `Pack` on top of the squeeze its candidates +/// force; a bounded draw is one `BitDec`, whose bits are the answer and need no +/// recomposition. +#[test] +fn the_one_row_operations_cost_one_row() { + // An extension element absorbed, then squeezed: one Unpack, and a buffer of + // three felts. + let mut b = builder(); + let arena = b.declare_arena(1); + let value = b.hint_word(arena, 0).as_ext(); + let mut transcript = WhirTranscript::new(); + transcript.absorb_ext(&mut b, value); + for lane in transcript.squeeze(&mut b) { + b.public(lane.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("admissible"); + assert_eq!( + program.instrs.len(), + 1 + absorb_unpack_rows() + leaf_hash_consts(1) + squeeze_rows(3) + 4, + "an absorbed extension element is one Unpack and three buffered felts" + ); + + // A challenge: the squeeze its three candidates force, plus one Pack. + let mut b = builder(); + let arena = b.declare_arena(4); + let felts: Vec = (0..4).map(|index| b.hint_felt(arena, index)).collect(); + let mut transcript = WhirTranscript::new(); + transcript.absorb_felts(&mut b, &felts); + let challenge = transcript.sample_ext(&mut b); + b.public(challenge.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("admissible"); + assert_eq!( + program.instrs.len(), + 4 + leaf_hash_consts(1) + squeeze_rows(4) + sample_ext_rows() + 1, + "a challenge is one Pack on top of its squeeze" + ); + + // A bounded draw: the same squeeze, plus one BitDec. + let nbits = 20usize; + let mut b = builder(); + let arena = b.declare_arena(4); + let felts: Vec = (0..4).map(|index| b.hint_felt(arena, index)).collect(); + let mut transcript = WhirTranscript::new(); + transcript.absorb_felts(&mut b, &felts); + for bit in transcript.sample_u64_pow2(&mut b, nbits) { + b.public(bit.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("admissible"); + assert_eq!( + program.instrs.len(), + 4 + leaf_hash_consts(1) + squeeze_rows(4) + sample_u64_rows() + nbits, + "a bounded draw is one BitDec, and its bits are the answer" + ); +} + +/// ★ `state()` observes without advancing: the same hash, no digest `Unpack`, +/// and the buffer it leaves is the one a later squeeze hashes again. +#[test] +fn state_costs_the_hash_without_the_unpack() { + for n in [1usize, 4, 9] { + let mut b = builder(); + let arena = b.declare_arena(n as u32); + let felts: Vec = (0..n) + .map(|index| b.hint_felt(arena, index as u32)) + .collect(); + let mut transcript = WhirTranscript::new(); + transcript.absorb_felts(&mut b, &felts); + let observed = transcript.state(&mut b); + b.public(observed); + let program = compile(b.finish()); + validate(&program).expect("admissible"); + assert_eq!( + program.instrs.len(), + n + leaf_hash_consts(1) + state_rows(n) + 1, + "state over {n} felts is the hash without the Unpack" + ); + assert_eq!( + transcript.out_pos(), + super::whir_transcript::CANDIDATES_PER_SQUEEZE, + "state must not fill the output buffer" + ); + assert_eq!(transcript.squeezes(), 0, "state must not advance the chain"); + } +} + +// --------------------------------------------------------------------------- +// The grind check: the machine's only reject in this file. +// --------------------------------------------------------------------------- + +/// A program that absorbs `seed_felts` hinted felts, grinds at `bits` with a +/// hinted nonce, and publishes the transcript state afterwards. +/// +/// The nonce is hinted, which is what a forger controls; the check is what +/// stops one choosing it freely. +fn grind_program(seed_felts: usize, bits: u8) -> (LfmProgram, usize) { + let mut b = builder(); + let arena = b.declare_arena((seed_felts + 1) as u32); + let felts: Vec = (0..seed_felts) + .map(|index| b.hint_felt(arena, index as u32)) + .collect(); + let nonce = b.hint_felt(arena, seed_felts as u32); + let mut transcript = WhirTranscript::new(); + transcript.absorb_felts(&mut b, &felts); + let before = transcript.squeezes(); + super::whir_transcript::emit_grind_check(&mut b, &mut transcript, bits, nonce); + assert_eq!( + transcript.squeezes(), + before, + "a grind reads the state; it must not advance the chain" + ); + let after = transcript.state(&mut b); + b.public(after); + let program = compile(b.finish()); + validate(&program).expect("admissible"); + (program, seed_felts + 1) +} + +/// ★ THE GATE AND THE TAMPER ARM IN ONE. The machine accepts exactly the nonces +/// `crypto::grinding::is_valid_nonce` accepts, and the transcript it leaves is +/// the host's. +/// +/// The valid nonce comes from the host's own search, so nothing here decides +/// what "valid" means; the machine only has to agree. The invalid arm walks +/// nonces the host rejects and requires every one of them to fail to execute — +/// the grind is a real reject, exactly like the GKR layer relation. +#[test] +fn the_grind_accepts_what_the_host_accepts_and_refuses_the_rest() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use crypto::fiat_shamir::transcript_hash::RpxTranscriptHash; + use crypto::hash::rpx::Rpx256Digest; + + // A few bits: the search is cheap and the arm below can find rejects. + let bits = 8u8; + let seed_felts = 4usize; + let values: Vec = (0..seed_felts) + .map(|i| FE::from(7 + i as u64 * 31)) + .collect(); + + // The host's transcript over the same absorbs, and its state — the grind's + // seed. + let mut host = DefaultTranscript::::new(&[]); + let mut seed_bytes = Vec::new(); + for value in &values { + seed_bytes.extend_from_slice(&value.to_raw().to_be_bytes()); + } + host.append_bytes(&seed_bytes); + let seed = host.state(); + + let nonce = crypto::grinding::generate_nonce_smallest::(&seed, bits) + .expect("a nonce exists at eight bits"); + assert!( + crypto::grinding::is_valid_nonce::(&seed, nonce, bits), + "the host's own search must produce a nonce the host accepts" + ); + + let (program, _) = grind_program(seed_felts, bits); + let mut arena: Vec = values + .iter() + .map(|v| [*v, FE::zero(), FE::zero(), FE::zero()]) + .collect(); + arena.push([FE::from(nonce), FE::zero(), FE::zero(), FE::zero()]); + let words = run_program(&program, &[arena.clone()]); + + // The host's transcript absorbs the nonce, exactly as `check_grind` does. + host.append_bytes(&nonce.to_be_bytes()); + let want = host.state(); + let left = words[0]; + assert_eq!( + digest_to_commitment(&[left[0], left[1], left[2], left[3]]), + want, + "after the grind the machine's transcript must be the host's" + ); + + // ★ The tamper arm: every nonce the host rejects must fail to execute. + let mut refused = 0usize; + for candidate in 0..64u64 { + if crypto::grinding::is_valid_nonce::(&seed, candidate, bits) { + continue; + } + let mut forged = arena.clone(); + forged[seed_felts] = [FE::from(candidate), FE::zero(), FE::zero(), FE::zero()]; + assert!( + execute(&program, &[forged], &crate::hash_pin::BLOCK_HASHER).is_err(), + "nonce {candidate}: the host rejects it, so the machine must refuse to execute" + ); + refused += 1; + } + assert!( + refused >= 32, + "the arm must actually exercise rejects; it found only {refused}" + ); + println!("grind at {bits} bits: nonce {nonce} accepted, {refused} rejects refused"); +} + +/// ★ `bits == 0` does nothing at all — no state read, no absorb. +/// +/// The host returns before touching the transcript (`whir_chain.rs:131-133`), +/// and both halves are observable: an absorb would move every later challenge, +/// and a state read is counted apart from a squeeze. A grind that "does nothing" +/// but still absorbs is a desync that no value in the round itself would show. +#[test] +fn a_zero_bit_grind_touches_nothing() { + let (with_zero, _) = grind_program(4, 0); + let mut b = builder(); + let arena = b.declare_arena(5); + let felts: Vec = (0..4).map(|index| b.hint_felt(arena, index)).collect(); + let _unused = b.hint_felt(arena, 4); + let mut transcript = WhirTranscript::new(); + transcript.absorb_felts(&mut b, &felts); + let state = transcript.state(&mut b); + b.public(state); + let without = compile(b.finish()); + validate(&without).expect("admissible"); + assert_eq!( + with_zero.instrs.len(), + without.instrs.len(), + "a zero-bit grind must emit nothing" + ); + assert_eq!(super::whir_transcript::grind_check_rows(0), 0); +} + +/// ★ F1 for the grind: every row named, at three factors. +/// +/// The program hashes buffers of four felts (the state the grind seeds from), +/// six and five (the grind's two preimages) and five again (the state it +/// publishes) — three distinct capacity words, shared across those hashes, +/// plus the grind's own two constant felts. +#[test] +fn the_grind_costs_its_closed_form() { + for bits in [1u8, 8, 20] { + let seed_felts = 4usize; + let (program, plumbing) = grind_program(seed_felts, bits); + let distinct_capacities = 3; // widths 4, 5 and 6 + let predicted = plumbing + + leaf_hash_consts(distinct_capacities) + + super::whir_transcript::grind_check_const_felts() + + state_rows(seed_felts) + + super::whir_transcript::grind_check_rows(bits as usize) + + state_rows(seed_felts + 1) + + 1; + println!( + "grind at {bits:>2} bits: {:>3} rows emitted, {predicted:>3} predicted", + program.instrs.len() + ); + assert_eq!( + program.instrs.len(), + predicted, + "a {bits}-bit grind must emit its closed form" + ); + } +} diff --git a/prover/src/lfm/word.rs b/prover/src/lfm/word.rs new file mode 100644 index 000000000..4d66f5908 --- /dev/null +++ b/prover/src/lfm/word.rs @@ -0,0 +1,61 @@ +//! The LFM machine word: `[F; 4]`, four Goldilocks elements. +//! +//! The word is digest-aligned, not extension-aligned: a Goldilocks-native hash +//! at a 128-bit target uses a 4-felt digest and a 12-felt state, so a digest is +//! exactly one cell, the sponge rate two cells and the state three cells. Base +//! values occupy lane 0 with lanes 1–3 zero; extension values (Fp3) occupy +//! lanes 0–2 with lane 3 zero. The zero lanes are enforced on the bus as +//! constant tuple entries, never as trace columns, so a base value cannot +//! smuggle a phantom extension element. + +use crate::tables::types::{FE, FEE, GoldilocksField}; +use math::field::traits::IsPrimeField; + +/// One machine word / memory cell: four Goldilocks elements. +pub type LfmWord = [FE; 4]; + +/// Number of felt lanes in a word. +pub const WORD_LANES: usize = 4; + +/// A base field value embedded as a word: `(v, 0, 0, 0)`. +pub fn base_word(v: FE) -> LfmWord { + [v, FE::zero(), FE::zero(), FE::zero()] +} + +/// An Fp3 extension value embedded as a word: `(a0, a1, a2, 0)`. +pub fn ext_word(e: &FEE) -> LfmWord { + let [a0, a1, a2] = *e.value(); + [a0, a1, a2, FE::zero()] +} + +/// Reads a word as a base value. `None` unless lanes 1–3 are zero — mirrors +/// the bus-level rule that a base receive carries constant zero high lanes. +pub fn word_as_base(w: &LfmWord) -> Option { + (w[1] == FE::zero() && w[2] == FE::zero() && w[3] == FE::zero()).then(|| w[0]) +} + +/// Reads a word as an Fp3 value. `None` unless lane 3 is zero. +pub fn word_as_ext(w: &LfmWord) -> Option { + (w[3] == FE::zero()).then(|| FEE::new([w[0], w[1], w[2]])) +} + +/// Packs a digest word into the 32-byte commitment format: four canonical +/// u64 lanes, little-endian, in lane order. Exact: 4 × 8 bytes. +pub fn pack_digest(w: &LfmWord) -> [u8; 32] { + let mut out = [0u8; 32]; + for (lane, chunk) in w.iter().zip(out.chunks_exact_mut(8)) { + chunk.copy_from_slice(&GoldilocksField::canonical(lane.value()).to_le_bytes()); + } + out +} + +/// Inverse of [`pack_digest`]. Lanes are reduced mod p on the way in. +pub fn unpack_digest(bytes: &[u8; 32]) -> LfmWord { + let mut lanes = [FE::zero(), FE::zero(), FE::zero(), FE::zero()]; + for (lane, chunk) in lanes.iter_mut().zip(bytes.chunks_exact(8)) { + let mut raw = [0u8; 8]; + raw.copy_from_slice(chunk); + *lane = FE::from(u64::from_le_bytes(raw)); + } + lanes +} diff --git a/prover/src/lfm/wrap_tests.rs b/prover/src/lfm/wrap_tests.rs new file mode 100644 index 000000000..6a6afdc70 --- /dev/null +++ b/prover/src/lfm/wrap_tests.rs @@ -0,0 +1,1802 @@ +//! ★ THE WRAP — the assembled epoch verifier PROVED, not just executed. +//! +//! [`super::epoch_verify_tests`] runs the whole epoch verifier under +//! [`super::executor::execute`]: every check is an assert inside the program, so +//! reaching the end of the execution is the verification passing. What that says +//! nothing about is the CHIPS (standing-decisions method rule 2 — where the +//! executor mirrors a computation the chip also does, only a prove+verify test +//! sees the chip). This module is the wrap: the same program, the same real +//! epoch, through [`lfm_prove`] and [`verify_against`]. +//! +//! ## What the numbers here are, and are not +//! +//! Every cost figure names the epoch's trace-length profile (assembly ledger +//! entry 10). Two different epoch shapes are involved and they must never be +//! conflated: +//! +//! - the INNER epoch's shape — the proof being verified: its per-table trace +//! lengths, its blowup and its query count. This is what makes the emitted +//! verifier program big or small. +//! - the WRAP's own [`ProofOptions`] — blowup 2, the same options every other +//! LFM prove test uses. It fixes what proving the verifier costs, not what the +//! verifier does. +//! +//! ## What this module cannot see +//! +//! The hash — MOSTLY. This module was written when every permutation here was +//! `TestPermutation` inside the LFM chips plus the production keccak family +//! hosted for `keccak256`, and the point of measuring cells at all was to have +//! the first column of a matrix whose other columns did not exist yet. Those +//! columns exist now (BLAKE3, RPO, RPX, Poseidon), so +//! ⚠ [`the_census_agrees_with_the_traces_the_prover_builds`] takes its tenant +//! from `artifacts.hasher` rather than defaulting: it compares a census, a trace +//! set and an AIR set, and the socket chip's WIDTH is tenant-dependent, so three +//! defaults against one pinned artifact set is an out-of-bounds index rather +//! than a disagreement. Everything else here is still tenant-agnostic. It also +//! cannot see prove time or +//! peak memory as a property of the machine — those are measured around the +//! process, by the harness that runs it, and are reported as observations of one +//! box rather than as machine invariants. + +use std::time::Instant; + +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use super::airs::{HeightRule, LfmChipCells, lfm_cell_counts, lfm_chip_census}; +use super::compiler::LfmProgram; +use super::edsl::WrapHash; +use super::epoch_tests::EpochInputs; +use super::executor::execute; +use super::instr::Instr; +use super::proof::{LfmProveError, lfm_prove, lfm_prove_with_residency, verify_against}; +use super::registry::{build_artifacts, build_artifacts_with_hasher}; + +use crate::tables::types::FE; + +/// The WRAP proof's own options: blowup 2, the framework's 128-bit query count. +/// +/// Deliberately the same `prove_options()` every leg suite proved under +/// (`join_tests`, `fri_tests`, `constraint_tests`), so a wrap cost is comparable +/// with a leg cost. The inner epoch's options are a different thing entirely and +/// are named per measurement. +fn wrap_options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// Instructions of each kind, for the shape line every measurement prints. +fn instruction_mix(program: &LfmProgram) -> String { + let count = |f: fn(&Instr) -> bool| program.instrs.iter().filter(|i| f(i)).count(); + format!( + "const {} / base-alu {} / ext-alu {} / select {} / bitdec {} / hash {} / \ + keccak {} / hint {} / pack {} / unpack {} / public {}", + count(|i| matches!(i, Instr::Const { .. })), + count(|i| matches!(i, Instr::BaseAlu { .. })), + count(|i| matches!(i, Instr::ExtAlu { .. })), + count(|i| matches!(i, Instr::Select { .. })), + count(|i| matches!(i, Instr::BitDec { .. })), + count(|i| matches!(i, Instr::Hash { .. })), + count(|i| matches!(i, Instr::KeccakF(_))), + count(|i| matches!(i, Instr::Hint { .. })), + count(|i| matches!(i, Instr::Pack { .. })), + count(|i| matches!(i, Instr::Unpack { .. })), + count(|i| matches!(i, Instr::Public { .. })), + ) +} + +/// Keccak permutations a program requests. +pub(super) fn permutations(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|i| matches!(i, Instr::KeccakF(_))) + .count() +} + +/// The production wrap hash's own operations in a program — keccak +/// permutations or BLAKE3 compressions, whichever `hash` names. The census's +/// closed-form check compares like with like: `query_permutations_for` counts +/// compressions under the same hash's block rule. +pub(super) fn hash_ops(program: &LfmProgram, hash: WrapHash) -> usize { + program + .instrs + .iter() + .filter(|i| match hash { + WrapHash::Keccak => matches!(i, Instr::KeccakF(_)), + WrapHash::Blake3 => matches!(i, Instr::Blake3(_)), + WrapHash::Algebraic => matches!(i, Instr::Hash { .. }), + WrapHash::Poseidon1 => matches!(i, Instr::Hash16(_)), + }) + .count() +} + +/// Arena words a program declares. +pub(super) fn arena_words(program: &LfmProgram) -> usize { + program.arena_schema.lens.iter().map(|l| *l as usize).sum() +} + +/// A chip's headroom as the census table prints it: a percentage for the chips +/// that can actually grow into it, and the reason otherwise. +fn headroom_cell(c: &LfmChipCells) -> String { + match c.height_rule { + HeightRule::Workload => format!("{:.1}%", 100.0 * c.headroom()), + HeightRule::Fixed => "fixed".to_string(), + HeightRule::Chunked => "chunked".to_string(), + } +} + +/// ★ THE ROW-CLIFF PANEL — what the census could not say before. +/// +/// The wrap's cell count is a STEP function of the instruction mix: every +/// workload-sized chip commits `real_rows.next_power_of_two()` rows, so a chip +/// sitting just under a step doubles its entire contribution on the next 1% of +/// growth, while one just over a step absorbs a near-doubling for free. +/// +/// The census has always computed both heights and printed only the padded one, +/// which is why #903's cost was invisible until a rented box measured it: four +/// near-empty BLAKE3 rows grew the mix ~21%, and the campaign happened to be +/// standing 1.2% under a step on `LFM_LANES` and within 18% on four more chips, +/// so that growth tripped FIVE simultaneous doublings and turned +21% of +/// permutations into +31% of cells +/// (`thoughts/shared/block-compression/WRAP-GROWTH-BISECT.md`). +/// +/// This panel makes that legible at emission time. It reports the tightest +/// chips, what crossing each would cost, and the total exposure — the cells the +/// wrap would gain if every chip within the warning band crossed at once, which +/// is exactly the quantity Stage 4 spent without anyone pricing it. +fn report_row_cliffs(census: &[LfmChipCells]) { + /// Headroom under which a chip is called out. A change of this order is + /// routine — Stage 4's was ~21% — so anything inside the band should be + /// read as "will cross on the next ordinary change", not as a safe margin. + const WARN: f64 = 0.20; + + let total: u64 = census.iter().map(|c| c.cliff_cost()).sum(); + let mut at_risk: Vec<&LfmChipCells> = census.iter().filter(|c| c.at_risk()).collect(); + at_risk.sort_by(|a, b| a.headroom().total_cmp(&b.headroom())); + + println!("\n ★ ROW-CLIFF PANEL — cells are a step function of the mix"); + if at_risk.is_empty() { + println!(" no workload-sized chips in this census"); + return; + } + + let exposed: Vec<&&LfmChipCells> = at_risk.iter().filter(|c| c.headroom() < WARN).collect(); + let pct = |n: u64| 100.0 * n as f64 / total as f64; + + for c in at_risk.iter().take(5) { + let flag = if c.headroom() < WARN { "⚠" } else { " " }; + println!( + " {flag} {:>12} {:>7.1}% headroom ({} of {} rows) — crossing costs \ + {} base-field equivalents ({:.1}% of the total)", + c.name, + 100.0 * c.headroom(), + c.real_rows, + c.rows, + c.cliff_cost(), + pct(c.cliff_cost()), + ); + } + + let exposure: u64 = exposed.iter().map(|c| c.cliff_cost()).sum(); + println!( + " {} of {} workload-sized chips are within {:.0}% of a step; if all of them \ + crossed the wrap would gain {} base-field equivalents (+{:.1}%)", + exposed.len(), + at_risk.len(), + 100.0 * WARN, + exposure, + pct(exposure), + ); +} + +/// ★ The registry-entry shape record: what the machine proves for one program. +/// +/// Prints the chip census — one line per SUB-PROOF, since `KECCAK_RND`'s chunks +/// are separate AIRs at separate heights — the per-chip row-cliff headroom, and +/// the totals the hash matrix wants. +/// Returns `(main_cells, aux_cells)` so a caller can assert on them. +pub(super) fn report_census(label: &str, program: &LfmProgram) -> (u64, u64) { + let census = lfm_chip_census(program); + let (main, aux) = lfm_cell_counts(program); + // The census is `lfm_cell_counts`' own decomposition, so summing it is not an + // independent check of the total — it is the same arithmetic. What IS + // independent is that the sub-proof COUNT the census implies must equal the + // AIR count the verifier builds from the program's chunk policy. + // Mask-aware, like the census itself: an absent family contributes no + // AIRs, so the expected count comes from the program's own chip set and + // its family-gated chunk count — `num_lfm_airs` is the FULL-mask count and + // no real program is FULL (a program uses at most one hash family). A split + // `LFM_HASH` is one sub-proof more, as `proof::verify_against_chunked_with` + // counts it. + let chip_set = super::airs::ChipSet::for_program(program); + assert_eq!( + census.len(), + chip_set.num_airs( + chip_set.keccak_rnd_chunks( + program + .chunking + .chunk_count(program.groups.keccak.real_rows) + ), + program.blake3_chunk_count(), + ) + program.hash_chunk_count() + - 1, + "the census must have one entry per sub-proof the AIR set builds" + ); + println!("\n★ CHIP CENSUS — {label}"); + println!( + " {:>12} {:>10} {:>12} {:>9} {:>6} {:>6} {:>16} {:>14}", + "chip", "rows", "used", "headroom", "main", "aux", "main cells", "aux cells" + ); + for c in &census { + println!( + " {:>12} {:>10} {:>12} {:>9} {:>6} {:>6} {:>16} {:>14}", + c.name, + c.rows, + c.real_rows, + headroom_cell(c), + c.main_cols, + c.aux_cols, + c.main_cells(), + c.aux_cells() + ); + } + println!( + " {:>12} {:>10} {:>12} {:>9} {:>6} {:>6} {:>16} {:>14}", + "TOTAL", + census.iter().map(|c| c.rows).sum::(), + census.iter().map(|c| c.real_rows).sum::(), + "", + "", + "", + main, + aux + ); + report_row_cliffs(&census); + println!( + " cells per verify = {main} main + {aux} aux ext = {} base-field equivalents \ + (an ext element is 3 base felts)", + main + 3 * aux + ); + + // ★ THE HASH SHARE — what the matrix is about. + // + // The whole reason to count cells is to price the hash, so the census says + // outright how much of the machine IS the hash. `LFM_KECCAK` (the adapter row + // that requests a permutation) and `KECCAK_RND` (its 24 rounds) are the + // permutation itself; `KECCAK_RC` and `BITWISE` are the lookup tables it reads, + // and they are reported separately because they are FIXED-height — a different + // hash would delete the first pair and shrink but not necessarily remove the + // second. + let share = |names: &[&str]| -> (u64, u64) { + census + .iter() + .filter(|c| names.contains(&c.name)) + .fold((0u64, 0u64), |(m, a), c| { + (m + c.main_cells(), a + c.aux_cells()) + }) + }; + let (perm_main, perm_aux) = share(&["LFM_KECCAK", "KECCAK_RND"]); + let (tab_main, tab_aux) = share(&["KECCAK_RC", "BITWISE"]); + let total = (main + 3 * aux) as f64; + println!( + " keccak permutation chips (LFM_KECCAK + KECCAK_RND): {perm_main} main + \ + {perm_aux} aux = {:.1}% of cells\n \ + its lookup tables (KECCAK_RC + BITWISE, fixed height): {tab_main} main + \ + {tab_aux} aux = {:.1}%\n \ + everything else (the verifier's own arithmetic): {:.1}%", + 100.0 * (perm_main + 3 * perm_aux) as f64 / total, + 100.0 * (tab_main + 3 * tab_aux) as f64 / total, + 100.0 * (main + 3 * aux - perm_main - 3 * perm_aux - tab_main - 3 * tab_aux) as f64 / total, + ); + println!(" instruction mix: {}", instruction_mix(program)); + (main, aux) +} + +/// The three headline shape numbers, printed with the epoch profile that fixes +/// them (ledger entry 10). +pub(super) fn report_program(label: &str, profile: &str, program: &LfmProgram) { + println!( + "\n★ {label}\n epoch trace lengths (log2): {profile}\n \ + {} instructions / {} keccak permutations / {} arena words / {} chunks", + program.instrs.len(), + permutations(program), + arena_words(program), + program + .chunking + .chunk_count(program.groups.keccak.real_rows), + ); +} + +/// The INNER epoch's own committed trace cells — `(main, aux ext)` — summed over +/// its sub-proofs. +/// +/// The denominator of the recursion ratio, and the only honest one available from +/// shapes alone: `rows x main_width` and `rows x aux_width` per sub-proof, which is +/// the same accounting [`lfm_chip_census`] applies to the machine (value columns +/// plus aux, one ext element per aux column per row). +/// +/// What it CANNOT see, on both sides equally: preprocessed columns, the +/// composition polynomial's own commitment, the LDE, and the Merkle trees. So the +/// ratio it feeds is "trace cells to verify one epoch's trace cells", not "total +/// prover work", and it is quoted that way. +fn inner_epoch_cells(e: &super::epoch_tests::RealEpoch) -> (u64, u64) { + e.legs + .iter() + .map(|l| { + let rows = 1u64 << l.verify.sub.deep.log2_trace_length; + let aux_width = l.verify.sub.deep.num_total_cols - l.verify.main_width; + (rows * l.verify.main_width as u64, rows * aux_width as u64) + }) + .fold((0, 0), |(m, a), (dm, da)| (m + dm, a + da)) +} + +/// Prints the recursion ratio: machine cells per verify against the verified +/// epoch's own cells. The kill-risk-3 question, asked of a real epoch at last. +fn report_ratio(e: &super::epoch_tests::RealEpoch, main: u64, aux: u64) { + let (inner_main, inner_aux) = inner_epoch_cells(e); + let inner = inner_main + 3 * inner_aux; + let outer = main + 3 * aux; + println!( + " the epoch VERIFIED carries {inner_main} main + {inner_aux} aux ext = {inner} \ + base-field-equivalent trace cells\n \ + so verifying it costs {:.1}x its own trace cells (trace-to-trace; neither \ + side counts preprocessed columns, LDEs or trees)", + outer as f64 / inner as f64, + ); +} + +/// The epoch's trace-length profile as ledger entry 10 wants it printed. +pub(super) fn epoch_profile(e: &super::epoch_tests::RealEpoch) -> String { + let mut lengths: Vec = e + .legs + .iter() + .map(|l| l.verify.sub.deep.log2_trace_length) + .collect(); + lengths.sort_unstable(); + let mut runs: Vec = Vec::new(); + for len in &lengths { + let n = lengths.iter().filter(|l| *l == len).count(); + let entry = if n > 1 { + format!("{len} x{n}") + } else { + format!("{len}") + }; + if !runs.contains(&entry) { + runs.push(entry); + } + } + format!("[{}]", runs.join(", ")) +} + +/// ★ SLICE 0 — the wrap on the min-preset fixture epoch: prove, verify, tamper. +/// +/// `#[ignore]`d, and the reason is the cost: the assembled verifier is ~2.25M +/// instructions, so the LFM traces are an order of magnitude past anything else +/// in this suite and the run is minutes of CPU and tens of gigabytes. It is the +/// wrap run's own harness, not a test the suite can afford on every PR. +/// +/// ⚠⚠ **BOTH FIGURES ARE UNVERIFIED FOR THE SHAPE THIS SIGNATURE DESCRIBES, +/// and the same doubt covers [`MEASURED_BYTES_PER_CELL`]'s 481,327,124-cell +/// point, which is attributed to this test.** +/// +/// [`wrap_run`] passes [`EpochInputs::from_env`], and with no `LFM_CENSUS_*` +/// set that is byte-for-byte [`EpochInputs::fixture`] — so in a CLEAN +/// ENVIRONMENT this test and [`the_fixture_epoch_wraps`] build the same epoch +/// through the same `real_epoch_from` and emit the same +/// `epoch_program(&e, true)`. They are ONE program. That program measures +/// **210,782 instructions** and 82,059,828 base-field-equivalent cells, an +/// order of magnitude under both numbers above. +/// +/// ⚖ The likely explanation is that these were recorded under `LFM_CENSUS_*` +/// overrides — this test reads them, and a real-block run through this harness +/// is what they exist for — but that is a HYPOTHESIS and is written as one. It +/// has not been checked, and "the shape shrank since" is not excluded. The +/// numbers are left in place rather than corrected, because correcting them +/// would mean inventing replacements: what they need is one `--ignored` run of +/// this test in a clean environment, after which all three move together or +/// none do. +/// +/// ⛔ Until that run exists, do not size a wrap run from anything on this +/// signature. [`the_fixture_epoch_wraps`] carries measured numbers for the +/// clean-environment shape. +/// +/// Run with: +/// `cargo test --release -p lambda-vm-prover --lib lfm::wrap_tests::the_wrap_proves_and_verifies -- --ignored --nocapture` +#[test] +#[ignore] +fn the_wrap_proves_and_verifies() { + wrap_run(super::proof_fixture::fixture_options()); +} + +/// ★★ THE SUITE-GATED WRAP — the one assembled epoch verifier a suite run +/// PROVES, and NOT `#[ignore]`d. +/// +/// # The gap it closes +/// +/// Every other epoch-verifier wrap is `#[ignore]`d +/// ([`the_wrap_proves_and_verifies`], [`the_real_block_epoch_wraps`], +/// [`the_from_proof_final_epoch_wraps`], +/// [`the_real_block_proves_and_wraps_end_to_end`]). The proof FORMAT is covered +/// without this test — the leg suites prove it — but the assembled epoch +/// verifier PROGRAM is not. This is the arm that keeps it from being the +/// untested one. +/// +/// # What proving adds to an execution +/// +/// [`super::epoch_verify_tests::the_assembled_epoch_verifier_runs`] already +/// EXECUTES this exact program on every suite run, and by the method's rule 2 +/// that says nothing about the chips: where the executor mirrors a computation +/// the chip also does, only a prove+verify test sees the chip. So the delta here +/// is the whole LFM machine — traces, AIRs, commitments and verifier — over the +/// per-table assembled epoch verifier. +/// +/// # The pin, NAMED rather than implied +/// +/// The classification rule, stated here rather than cited — the document that +/// carries it lands with the RPX pin and does not exist on this branch: +/// +/// > A program built at `WrapHash::production()` emits `Instr::Hash` and must +/// > be proved under `BLOCK_HASHER`. A program that pins a byte hash on its own +/// > builder emits none, never consults the socket, and is correct at the +/// > registry's blessed default under every pin. +/// +/// `epoch_tests::epoch_program` builds at [`WrapHash::production()`], so this +/// site is in the first class and must name [`crate::hash_pin::BLOCK_HASHER`]. +/// It does, twice over: `build_artifacts_with_hasher` for the artifacts and +/// `lfm_cell_counts_with_hasher` for the numbers, so neither the proof nor the +/// census can be taken at `HasherKind::default()`. +/// +/// ⚠ **On a BYTE pin that naming is inert, and that is exactly the trap.** +/// `ByteWrapHash` lowers to the dedicated KECCAK / `LFM_BLAKE3` chips and emits +/// no `Instr::Hash` at all, so a defaulted socket hasher is free and correct +/// here and becomes wrong only when the pin moves. Written pinned now so the +/// flip is not a bug-hunt later — which is what it was the last three times +/// (`f6ca405c`, then `fri_tests` and `join_tests` at v10). +/// +/// # The shape +/// +/// The smallest shape that still exercises every leg: the min preset +/// ([`super::proof_fixture::fixture_options`] — blowup 2, ONE query) over the +/// continuation fixture epoch ([`EpochInputs::fixture`], `FIXTURE_EPOCH_LOG2`). +/// `EpochInputs::fixture` rather than `from_env`, deliberately: a measurement +/// run's `LFM_CENSUS_*` variables must not be able to turn a suite gate into a +/// real-block run. +/// +/// # What it costs, MEASURED +/// +/// One run of this test, 48-core box, at the default (BLAKE3) pin: +/// +/// | | | +/// |---|---| +/// | inner epoch | 25 sub-proofs, trace lengths (log2) `[2 x18, 3, 4 x2, 5 x2, 7, 20]` | +/// | emitted program | **210,782 instructions**, 16,461 arena words | +/// | census at the pin | 42,096,912 main + 13,320,972 aux ext = 82,059,828 base-field equivalents | +/// | wall | 9.20s for the whole test: 3.1s to build and host-verify the epoch, 3.8s to prove, 0.16s to verify | +/// | wrap proof | 45,953,352 bytes over 15 sub-proofs | +/// +/// ⚠ On CI, expect **30-60s** rather than 9.2s: the runners are 2-4 vCPU and +/// the suite runs `--test-threads=1`, so almost none of the box's parallelism +/// is there. `? INFERRED` — scaled from the box wall, not measured on a runner. +/// This is the only test of its class, so that bill is paid once per suite run. +/// +/// ⚠ These numbers, and NOT the `~2.25M` instructions the slice-0 doc quotes, +/// describe this shape. In a clean environment the two tests are the SAME +/// PROGRAM: `wrap_run` passes `EpochInputs::from_env`, which with no +/// `LFM_CENSUS_*` set is byte-for-byte [`EpochInputs::fixture`]. Why slice 0's +/// doc carries a figure an order of magnitude higher for one program is an open +/// question and is written up on [`the_wrap_proves_and_verifies`]; it is not +/// settled here, and this table is not evidence about which environment that +/// number came from. +/// +/// The 46 MB proof is a consequence of the WRAP proving at +/// [`wrap_options`]' framework query count over 15 sub-proofs, not of anything +/// about the inner epoch, whose own preset is one query. It is fine for a gate: +/// the proof is built, verified and dropped inside the test and is never +/// serialized to disk or carried anywhere. A wrap proof meant to be SHIPPED is +/// the aggregator's problem and is sized by different levers. +#[test] +fn the_fixture_epoch_wraps() { + fixture_wrap_run( + super::proof_fixture::fixture_options(), + EpochInputs::fixture(), + ); +} + +/// [`the_fixture_epoch_wraps`]' body: build the inner epoch, emit the per-table +/// assembled verifier, prove it under the pin, verify it, and run the two +/// falsification arms. +/// +/// **Two, and they are not one check twice.** A flipped root makes the wrap +/// UNBUILDABLE and never reaches the verifier at all; a moved claimed word +/// leaves the proof untouched and must be REJECTED by it. Only the second +/// exercises `verify_against`'s reject path, so a gate carrying only the first +/// would show that the machine will not lie without ever showing that the +/// verifier catches a lie. +/// +/// Deliberately NOT [`wrap_run_from`], which is the MEASUREMENT harness: that +/// one emits the program a second time (the spine, for the closed-form +/// permutation check) and walks the chip census and the row-cliff panel. Both +/// earn their cost in a measurement run and neither is what this gate claims, +/// so the gate pays for neither. Its third arm, the moved PROGRAM DIGEST, is +/// the registry premise rather than this program's, and +/// `machine_tests::verify_against_artifacts_agrees_with_the_registry_path` +/// already holds it on a trivial program at a fraction of the cost. +fn fixture_wrap_run(inner: ProofOptions, inputs: EpochInputs) { + let t = Instant::now(); + let e = super::epoch_tests::real_epoch_from(inner.clone(), inputs); + let profile = epoch_profile(&e); + println!( + "per-table inner epoch: {} sub-proofs, blowup {}, {} quer{} per table, \ + grinding {} — built and HOST-VERIFIED in {:.1}s", + e.legs.len(), + 1 << e.tables[0].shape.log2_blowup, + e.legs[0].verify.num_queries, + if e.legs[0].verify.num_queries == 1 { + "y" + } else { + "ies" + }, + e.tables[0].shape.grinding_factor, + t.elapsed().as_secs_f64() + ); + + let t = Instant::now(); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + println!( + " emitted the assembled PER-TABLE verifier in {:.1}s", + t.elapsed().as_secs_f64() + ); + report_program("THE PER-TABLE WRAPPED PROGRAM", &profile, &program); + + // ---- the prediction, registered BEFORE the measurement. + // + // Counted at the pin's own permutation, not at `HasherKind::default()`: the + // `LFM_HASH` chip's width is tenant-dependent (Poseidon 612 value columns, + // RPO 436, RPX 316), so a defaulted census does not report a smaller number + // under an algebraic pin — it reports the wrong chip's. + let (main, aux) = + super::airs::lfm_cell_counts_with_hasher(&program, crate::hash_pin::BLOCK_HASHER); + println!( + " census at {:?}: {main} main + {aux} aux ext = {} base-field equivalents; \ + PROJECTED peak RSS {:.1} GiB (a projection from slice 0's coefficient, \ + not a measurement of this run)", + crate::hash_pin::BLOCK_HASHER, + main + 3 * aux, + projected_peak_bytes(main, aux) / (1u64 << 30) as f64, + ); + + // ---- the artifacts, at the PINNED socket permutation. + // + // `wrap_options` UNREDUCED, and deliberately with no knob to reduce it. The + // wrap's own query count is the one lever that would buy wall time without + // touching the emitted program, and at 9.2s for the whole test there is + // nothing to buy — so the gate proves under the same options every leg suite + // proved under, which is what keeps a wrap cost comparable with a leg cost. + // A dormant setting that quietly weakens a suite gate is worse than no + // setting; if this shape ever does need trimming, the honest lever is the + // one `the_wrap_proves_at_blowup_8_geometry` documents, named per run. + let opts = wrap_options(); + let artifacts = super::registry::build_artifacts_with_hasher( + &program, + &opts, + crate::hash_pin::BLOCK_HASHER, + ); + println!( + " wrap options: blowup {}, {} queries (the framework's 128-bit count), \ + grinding {}\n chip log-heights: {:?}", + opts.blowup_factor, opts.fri_number_of_queries, opts.grinding_factor, artifacts.log_heights + ); + + // ---- PROVE. + let t = Instant::now(); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts) + .expect("the per-table fixture wrap must prove"); + let prove_secs = t.elapsed().as_secs_f64(); + let size = rkyv::to_bytes::(&proved.proof) + .expect("the wrap proof must serialize") + .len(); + + // ---- VERIFY. + let t = Instant::now(); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the per-table fixture wrap proof must verify" + ); + let verify_secs = t.elapsed().as_secs_f64(); + println!( + "\n★ PER-TABLE WRAP PROVED AND VERIFIED (inner epoch {profile}, blowup {}, \ + {} quer{})\n prove {prove_secs:.1}s / verify {verify_secs:.2}s / \ + proof {size} bytes / {} published words / {} sub-proofs", + inner.blowup_factor, + inner.fri_number_of_queries, + if inner.fri_number_of_queries == 1 { + "y" + } else { + "ies" + }, + proved.public_words.len(), + proved.proof.proofs.len(), + ); + + // ---- the PROVED run published the epoch's own oracles, so "it proved" is + // "it proved the right thing" rather than "some program proved". Three + // reads of `public_words`, which cost nothing next to the prove. + let pub_ext = + |i: usize| super::word::word_as_ext(&proved.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the proved run publishes z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the proved run publishes alpha"); + assert_eq!( + super::word::word_as_ext(&proved.public_words[proved.public_words.len() - 1].1) + .expect("the bus total is ext"), + e.expected_bus_balance, + "the proved run reaches production's own COMMIT-bus target" + ); + + // ---- THE TAMPER ARM: a flipped MAIN ROOT must fail. + // + // Arena 2 is the epoch's main roots — statement, then the ELF-dependent + // preprocessed roots, then these (declaration order IS absorb order in + // `epoch_tests::epoch_arena_words`). The index is asserted against the + // shape rather than trusted, because a declaration-order change would + // otherwise leave this tampering some other arena and still reporting a + // pass, which is how `arena_index`' own first version came to tamper an + // empty arena. + const MAIN_ROOTS_ARENA: usize = 2; + assert!( + MAIN_ROOTS_ARENA < super::epoch_tests::num_epoch_wide_arenas(&e), + "the main roots are an EPOCH-WIDE arena" + ); + assert_eq!( + arenas[MAIN_ROOTS_ARENA].len(), + super::proof_arena::words_per_root() * e.tables.len(), + "arena {MAIN_ROOTS_ARENA} must be the main roots — one root per sub-proof, \ + at the production wrap hash's root width" + ); + + let mut tampered = arenas.clone(); + // Assigned rather than incremented: a root word is a packed u32 half on the + // byte arm, so `+= 1` on a lane holding `u32::MAX` would be refused for its + // RANGE instead of for the root mismatch this arm is about. + tampered[MAIN_ROOTS_ARENA][0] = super::word::base_word(FE::from(999_999u64)); + assert_ne!( + tampered[MAIN_ROOTS_ARENA][0], arenas[MAIN_ROOTS_ARENA][0], + "the tamper must actually move the root word — an arm that tampers \ + nothing passes while asserting nothing" + ); + + // UNBUILDABLE, not unverifiable: every check is an assert inside a + // straight-line program, so a false statement has no execution at all and + // `lfm_prove` fails in `execute` before a trace exists. A failure in the + // PROVER instead would mean the machine admitted the forgery and the + // rejection came from somewhere else, so the two are distinguished. + match lfm_prove(&program, &artifacts, &tampered, &opts) { + Err(LfmProveError::Exec(err)) => println!( + " TAMPERED main root of sub-proof 0: the per-table wrap is \ + UNBUILDABLE ({err:?})" + ), + Err(LfmProveError::Prover(err)) => { + panic!("a tampered main root must fail in EXECUTION, not in the prover: {err:?}") + } + Err(err @ LfmProveError::HasherMismatch { .. }) => panic!("{err:?}"), + Ok(_) => panic!("a tampered main root must not produce a wrap proof"), + } + + // ---- ARM 2: the honest proof against a MOVED claimed statement must be + // REJECTED. + // + // The other half of the pair, and it is not redundant with arm 1 — the two + // exercise different machinery. Arm 1 never reaches the verifier at all: a + // false statement has no execution, so `verify_against`'s REJECT path is + // untouched by it, and a gate with only that arm would prove the machine + // refuses to lie without ever showing that the verifier catches one. This + // arm hands the verifier the REAL proof under a claim it does not answer; + // `absorb_lfm_statement` binds the proof to its published words, so it must + // reject. Costs one verify (~0.16s against a 9s test), so there was no case + // for leaving it out. + let mut moved = proved.public_words.clone(); + moved[0].1[0] += FE::one(); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &moved, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a moved claimed public word must make the wrap proof UNVERIFIABLE" + ); + println!(" MOVED claimed public word 0: the wrap proof is UNVERIFIABLE"); +} + +/// ★ SLICE 1 (local rung) — the wrap at the inner proof's BLOWUP-8 GEOMETRY. +/// +/// The standing decision is that the inner proof is at blowup 8, and blowup is +/// not a rescaling of blowup 2: the LDE is four times deeper, so every Merkle walk +/// climbs two more levels, the FRI chain commits more layers, and the terminal +/// polynomial is reached from further away. None of that is exercised by slice 0. +/// +/// The QUERY count is the one thing reduced, and reduced for a stated reason: at +/// the real 73 queries the wrap's own trace does not fit in any box we have (see +/// [`the_wrap_census_at_blowup_8`], which measures the program and prints what +/// proving it would need). ONE query is what a 36 GiB local box holds, and it is +/// enough to make every blowup-8 structure real — the deeper walk, the longer fold +/// chain, the terminal polynomial reached from further away — since what falls out +/// at one query is only the REPETITION of that structure. An honest partial: the +/// GEOMETRY is proved, the query COUNT is not, and the two are separable because +/// per-query cost is a closed form over the shapes that +/// [`the_wrap_census_at_blowup_8`] asserts the emitted program against. +/// `LFM_WRAP_QUERIES` raises the inner query count above the 1 this asserts at, +/// which is how the residency ladder walks the wrap up until a box refuses it. +/// Unset — every CI and local run — it is exactly the one-query test described +/// above. +#[test] +#[ignore] +fn the_wrap_proves_at_blowup_8_geometry() { + let queries = match std::env::var("LFM_WRAP_QUERIES") { + Ok(v) => v.parse().expect("LFM_WRAP_QUERIES must be an integer"), + Err(_) => 1, + }; + wrap_run(inner_blowup_8_with_queries(queries)); +} + +/// The inner proof's blowup-8 options with the query count overridden. +/// +/// NOT a security parameter set at anything below 73 queries, and never used as +/// one: the query count is what this reduces and every measurement taken under it +/// says so in its label. +fn inner_blowup_8_with_queries(queries: usize) -> ProofOptions { + let mut o = crate::recursion::Preset::Blowup8.options(); + o.fri_number_of_queries = queries; + o +} + +/// The wrap, end to end, under supplied INNER proof options, over whatever epoch +/// [`EpochInputs::from_env`] names — the continuation fixture unless a measurement +/// run overrode it. +fn wrap_run(inner: ProofOptions) { + wrap_run_from(inner, EpochInputs::from_env()); +} + +/// [`wrap_run`] over an explicitly supplied epoch: build it, emit the verifier, +/// prove it, verify it, and run the three falsifications. +fn wrap_run_from(inner: ProofOptions, inputs: EpochInputs) { + let t_epoch = Instant::now(); + let e = super::epoch_tests::real_epoch_from(inner.clone(), inputs); + let profile = epoch_profile(&e); + println!( + "inner epoch: {} sub-proofs, blowup {}, {} quer{} per table, grinding {} — built in {:.1}s", + e.legs.len(), + 1 << e.tables[0].shape.log2_blowup, + e.legs[0].verify.num_queries, + if e.legs[0].verify.num_queries == 1 { + "y" + } else { + "ies" + }, + e.tables[0].shape.grinding_factor, + t_epoch.elapsed().as_secs_f64() + ); + + // The GEOMETRY the blowup fixes, stated per run: what the walks climb and what + // the FRI chain folds. This is what separates a blowup-8 run from a blowup-2 one + // at the same query count, so it is printed rather than left to the label. + let big = e + .legs + .iter() + .max_by_key(|l| l.verify.sub.deep.log2_trace_length) + .expect("the epoch has sub-proofs"); + println!( + " geometry: widest sub-proof 2^{} trace -> 2^{} LDE, {} Merkle levels per group, \ + {} committed FRI layers ({} across the epoch); widest leaf {} bytes", + big.verify.sub.deep.log2_trace_length, + big.verify.sub.log2_lde_length, + big.verify.sub.merkle_depth, + big.verify.fri.num_committed(), + e.legs + .iter() + .map(|l| l.verify.fri.num_committed()) + .sum::(), + e.legs + .iter() + .flat_map(|l| l.verify.sub.groups()) + .map(|g| g.leaf_bytes()) + .max() + .expect("the epoch has groups"), + ); + + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + report_program("THE WRAPPED PROGRAM", &profile, &program); + let (main, aux) = report_census( + &format!("assembled epoch verifier, epoch {profile}"), + &program, + ); + + // ---- the spine/legs split, and the legs' permutations against a CLOSED FORM. + // + // Both halves matter and for different reasons. The split is what makes two + // runs at different query counts comparable at all: the SPINE also grows with + // the query count (it samples an index per query, and every sample is + // transcript work), so "permutations per query" taken from the total is wrong + // and taken from the difference is right. The closed form is the absolute + // check rule 7's refinement demands — `query_permutations` is arithmetic over + // byte widths and tree depths, not a second pass of this emitter, so a leg + // that quietly stopped hashing a group fails here rather than printing a + // smaller number. + let spine = super::epoch_tests::epoch_program(&e, false); + // The legs hash with the PRODUCTION wrap hash — keccak pre-flip, BLAKE3 + // after it — so both sides of the closed-form check follow it: the emitted + // count is that hash's own instruction, and the prediction uses its block + // rule. Counting keccak against a BLAKE3 leg reads 0 and fails spuriously. + let wrap_hash = WrapHash::production(); + let leg_hash_ops = hash_ops(&program, wrap_hash) - hash_ops(&spine, wrap_hash); + let predicted: usize = e + .legs + .iter() + .map(|l| super::epoch_verify::query_permutations_for(&l.verify, wrap_hash)) + .sum(); + assert_eq!( + leg_hash_ops, predicted, + "the emitted leg {wrap_hash:?} operations must equal the closed form over the shapes" + ); + let queries = e.legs[0].verify.num_queries; + println!( + " spine {} instr / {} {:?} ops / {} words legs {} / {} / {} \ + per query: {:.1} ops ({} queries, closed form checked)", + spine.instrs.len(), + hash_ops(&spine, wrap_hash), + wrap_hash, + arena_words(&spine), + program.instrs.len() - spine.instrs.len(), + leg_hash_ops, + arena_words(&program) - arena_words(&spine), + leg_hash_ops as f64 / queries as f64, + queries, + ); + + report_ratio(&e, main, aux); + + let opts = wrap_options(); + let artifacts = build_artifacts_with_hasher(&program, &opts, crate::hash_pin::BLOCK_HASHER); + println!( + " wrap options: blowup {}, {} queries, grinding {}\n chip log-heights: {:?}", + opts.blowup_factor, opts.fri_number_of_queries, opts.grinding_factor, artifacts.log_heights + ); + + // ---- PROVE. + let t = Instant::now(); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("the wrap must prove"); + let prove_secs = t.elapsed().as_secs_f64(); + + let size = rkyv::to_bytes::(&proved.proof) + .expect("the wrap proof must serialize") + .len(); + + // ---- VERIFY. + let t = Instant::now(); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the wrap proof must verify" + ); + let verify_secs = t.elapsed().as_secs_f64(); + println!( + "\n★ WRAP PROVED AND VERIFIED (inner epoch {profile}, blowup {}, {} quer{})\n \ + prove {prove_secs:.1}s / verify {verify_secs:.2}s / proof {size} bytes / \ + {} published words / {} sub-proofs\n \ + cells {main} main + {aux} aux ext; the projection for this run was \ + {:.1} GiB of peak RSS — compare against what the harness measured around \ + the process", + inner.blowup_factor, + inner.fri_number_of_queries, + if inner.fri_number_of_queries == 1 { + "y" + } else { + "ies" + }, + proved.public_words.len(), + proved.proof.proofs.len(), + projected_peak_bytes(main, aux) / (1u64 << 30) as f64, + ); + + // ---- the published words are the ones the execution produced, so the + // spine's differential still holds of the PROVED run and not only of an + // execution. Checked by value against the epoch's own oracles. + let pub_ext = + |i: usize| super::word::word_as_ext(&proved.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the proved run publishes z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the proved run publishes alpha"); + assert_eq!( + super::word::word_as_ext(&proved.public_words[proved.public_words.len() - 1].1) + .expect("the bus total is ext"), + e.expected_bus_balance, + "the proved run reaches production's own COMMIT-bus target" + ); + + // ---- FALSIFICATION 1: a tampered inner proof makes the wrap UNBUILDABLE. + // + // Not "unverifiable": every check is an assert inside a straight-line + // program, so a false statement has no execution at all — there is no branch + // to take and no error path to return, and `lfm_prove` fails in `execute` + // before a trace exists. That is the designed behaviour of the machine, and + // it is why the positive result above ("it proved") is the verification. + let ix = super::epoch_verify_tests::arena_index(&e, 0); + let mut tampered = arenas.clone(); + tampered[ix.openings][0][0] += FE::one(); + match lfm_prove(&program, &artifacts, &tampered, &opts) { + Err(LfmProveError::Exec(err)) => { + println!(" TAMPERED opened value 0 of table 0: the wrap is UNBUILDABLE ({err:?})") + } + Err(LfmProveError::Prover(err)) => { + panic!("a tampered inner proof must fail in execution, not in the prover: {err:?}") + } + Err(err @ LfmProveError::HasherMismatch { .. }) => panic!("{err:?}"), + Ok(_) => panic!("a tampered opened value must not produce a wrap proof"), + } + + // ---- FALSIFICATION 2: the honest proof against a MOVED claimed statement. + // + // The other half of the pair: the wrap proof is bound to the public words it + // published (`absorb_lfm_statement`), so a verifier handed the real proof and + // a different claim must reject. This is the path that rejects rather than + // failing to build, and both must exist — a machine where only the first + // existed would prove nothing about what the proof says. + let mut moved = proved.public_words.clone(); + moved[0].1[0] += FE::one(); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &moved, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a moved claimed public word must make the wrap proof UNVERIFIABLE" + ); + println!(" MOVED claimed public word 0: the wrap proof is UNVERIFIABLE"); + + // ---- FALSIFICATION 3: the same proof against another program's identity. + // + // The registry premise. `verify_against` takes the roots and the digest, and + // a proof of THIS program must not verify as a proof of a different one. + let mut other = artifacts.program_id; + other[0] ^= 1; + assert!( + !verify_against( + &artifacts.roots, + &other, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "a moved program digest must make the wrap proof UNVERIFIABLE" + ); + println!(" MOVED program digest: the wrap proof is UNVERIFIABLE"); +} + +/// ★ THE RESIDENCY ORACLE — `ResidencyMode::RecomputeLde` produces the same wrap +/// commitments as `Retain`. +/// +/// [`crate::tests::residency_mode_tests`] in the stark crate pins the mechanism +/// on a three-table toy, byte for byte. This pins it on the workload the mode +/// exists for: the wrap has preprocessed tables (whose main LDE carries the +/// precomputed columns the split trees were built from), `KECCAK_RND` chunks +/// (the family whose retention the mode drops), and the real transcript. +/// +/// ## Why this compares commitments and not proof bytes +/// +/// Proving is **not** reproducible run to run, and it is worth being exact +/// about why, because the fixture-blob note in +/// [`super::proof_fixture`] attributes it to sub-proofs committing to different +/// roots — which is not what happens here. Measured on this test: two runs +/// build byte-identical LFM traces, produce identical roots at every stage, and +/// still serialize to different proof bytes. The cause is the grinding search, +/// `grinding::generate_nonce`, which under the `parallel` feature is +/// `into_par_iter().find_any(..)` — *any* valid nonce, so which one comes back +/// depends on thread scheduling. The nonce is absorbed before the query indices +/// are sampled, so a different nonce opens different leaves. Both proofs are +/// valid; grinding is a proof of work and any witness satisfies it. +/// +/// So everything the nonce cannot reach is compared, which is everything the +/// recomputed LDE feeds: the main, aux, precomputed, composition and FRI-layer +/// roots, the out-of-domain evaluations, the final polynomial, and the bus +/// public inputs. A recomputed LDE that disagreed with the tree Round 1 +/// committed would move the composition root and the OOD evaluations — the two +/// values in that set derived from the LDE rather than from the trace. +/// +/// The run is its own control: `Retain` is proved twice, and the first +/// comparison is `Retain` against `Retain`. If that one ever fails, the +/// nondeterminism has reached the commitments and this oracle — not the +/// residency mode — is what needs fixing. +/// +/// `#[ignore]`d for the same reason as [`the_wrap_proves_and_verifies`], three +/// times over: it proves the wrap three times. +/// +/// Run with: +/// `cargo test --release -p lambda-vm-prover --lib lfm::wrap_tests::the_wrap_commitments_match_across_residency_modes -- --ignored --nocapture` +#[test] +#[ignore] +fn the_wrap_commitments_match_across_residency_modes() { + use stark::residency_mode::ResidencyMode; + + let e = super::epoch_tests::real_epoch_with(super::proof_fixture::fixture_options()); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let opts = wrap_options(); + let artifacts = build_artifacts_with_hasher(&program, &opts, crate::hash_pin::BLOCK_HASHER); + + let prove_under = |residency: ResidencyMode| { + let t = Instant::now(); + let proved = lfm_prove_with_residency( + &program, + &artifacts, + &arenas, + &opts, + artifacts.hasher, + residency, + ) + .expect("the wrap must prove"); + let secs = t.elapsed().as_secs_f64(); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the wrap proof must verify under {residency:?}" + ); + println!(" {residency:?}: proved in {secs:.1}s, verified"); + proved + }; + + // Everything the grinding nonce cannot reach. + let commitments = |p: &super::proof::LfmProof| { + p.proof + .proofs + .iter() + .map(|q| { + ( + q.trace_length, + q.lde_trace_main_merkle_root, + q.lde_trace_aux_merkle_root, + q.lde_trace_precomputed_merkle_root, + q.composition_poly_root, + q.composition_poly_parts_ood_evaluation.clone(), + q.trace_ood_evaluations.row_major_data().to_vec(), + q.trace_ood_next_evaluations.row_major_data().to_vec(), + q.fri_layers_merkle_roots.clone(), + q.fri_final_poly_coeffs.clone(), + q.bus_public_inputs.as_ref().map(|b| b.table_contribution), + ) + }) + .collect::>() + }; + + let retained = prove_under(ResidencyMode::Retain); + let control = prove_under(ResidencyMode::Retain); + assert!( + commitments(&retained) == commitments(&control), + "CONTROL FAILED: two Retain runs disagree on their commitments, so this \ + oracle cannot say anything about the residency mode" + ); + println!(" control: two Retain runs agree on every commitment"); + + let recomputed = prove_under(ResidencyMode::RecomputeLde); + assert!( + commitments(&retained) == commitments(&recomputed), + "the wrap commitments moved between residency modes" + ); + assert_eq!( + retained.public_words, recomputed.public_words, + "the published words moved between residency modes" + ); + + // Not asserted — recorded. The nonces are expected to differ; printing them + // keeps the reason this test compares commitments visible in its own output + // rather than only in its doc comment. + let nonces = |p: &super::proof::LfmProof| { + p.proof + .proofs + .iter() + .filter_map(|q| q.nonce) + .collect::>() + }; + println!( + " grinding nonces equal across the two Retain runs: {} (expected false under `parallel`)", + nonces(&retained) == nonces(&control) + ); + println!("\n★ RESIDENCY ORACLE: the wrap commits identically under Retain and RecomputeLde"); +} + +/// ★ GATE B — a REAL Ethereum-block epoch, wrapped. +/// +/// Everything else in this module wraps the 47-cycle continuation fixture, which +/// exercises every structure but at a size no production workload has. This +/// wraps one epoch of a real mainnet block at a SECURE inner preset +/// (blowup 4 / 110 queries, grinding as the preset sets it): one real block +/// epoch proof, compressed into one LFM proof. +/// +/// The guest and the block input are multi-megabyte binaries that cannot be +/// checked in, so the test requires them by path and says so rather than +/// quietly proving the fixture and reporting it as a block: +/// +/// ```text +/// LFM_CENSUS_ELF=/path/to/ethrex.elf \ +/// LFM_CENSUS_INPUT=/path/to/ethrex_mainnet_25368371.bin \ +/// LFM_CENSUS_EPOCH_LOG2=16 \ +/// cargo test --release -p lambda-vm-prover --lib \ +/// lfm::wrap_tests::the_real_block_epoch_wraps -- --ignored --nocapture +/// ``` +/// +/// ## The two knobs, and which one actually binds +/// +/// `LFM_CENSUS_EPOCH_LOG2` sets the epoch size and `LFM_WRAP_QUERIES` overrides +/// the inner query count (default: the preset's 110, which is the secure one — +/// anything lower is NOT a security parameter set and every number taken under +/// it carries the count, exactly as +/// [`the_wrap_proves_at_blowup_8_geometry`] does). +/// +/// Measured on a 60 GiB box: the epoch size is the *weak* knob and the query +/// count is the strong one. What decides whether a run fits is the number of +/// `KECCAK_RND` chunks the wrap's own trace needs, and that is +/// `(spine + per_query x queries) / 21,845` permutations. Per-query cost is +/// dominated by leaf absorption, which is set by table WIDTH and so barely +/// moves with epoch size — shrinking the epoch does not meaningfully shrink the +/// chunk count, and shrinking the query count does, linearly. +#[test] +#[ignore] +fn the_real_block_epoch_wraps() { + for var in ["LFM_CENSUS_ELF", "LFM_CENSUS_INPUT"] { + assert!( + std::env::var(var).is_ok(), + "{var} must name a file: this test wraps a REAL block epoch, and \ + without it the harness would build the continuation fixture and report \ + it under this test's name" + ); + } + let inputs = EpochInputs::from_env(); + let mut inner = crate::recursion::Preset::Blowup4.options(); + if let Ok(v) = std::env::var("LFM_WRAP_QUERIES") { + inner.fri_number_of_queries = v.parse().expect("LFM_WRAP_QUERIES must be an integer"); + } + println!( + "★ REAL-BLOCK WRAP: guest {}, {} bytes of private input, 2^{} cycles/epoch, \ + inner blowup {} / {} queries{}", + inputs.label, + inputs.private_input.len(), + inputs.epoch_log2, + inner.blowup_factor, + inner.fri_number_of_queries, + if inner.fri_number_of_queries < 110 { + " (REDUCED — not a security parameter set)" + } else { + " (the secure preset)" + }, + ); + wrap_run_from(inner, inputs); +} + +/// Peak prover memory the census implies, in bytes, from a MEASURED coefficient. +/// +/// The measured point is slice 0: 481,327,124 base-field-equivalent cells peaked +/// at 16,228,499,456 bytes of RSS (15.1 GiB), i.e. 33.7 bytes per cell — a trace word, its +/// blowup-2 LDE, and the Merkle/quotient working set on top. Stated as a +/// coefficient rather than derived from first principles because the derivation +/// would be a guess about the prover's allocation pattern and this is an +/// observation of it. What it CANNOT see: whether the coefficient holds at ten +/// times the size (allocator behaviour, and the fact that a bigger program is +/// bigger in different chips), so it is a projection and is labelled as one +/// wherever it is printed. +/// +/// ⚠ **The PROVENANCE line is in doubt, the RATIO is not.** Slice 0 in a clean +/// environment is the same program as [`the_fixture_epoch_wraps`], which +/// measures 82,059,828 cells — so the 481,327,124 above cannot be that run, and +/// "the measured point is slice 0" is under-specified about which environment +/// slice 0 was in. See [`the_wrap_proves_and_verifies`] for what is and is not +/// known. The value is deliberately UNCHANGED: a ratio of two numbers taken +/// together on one run stays a valid coefficient whichever shape that run was, +/// and re-deriving it from a shape nobody measured the RSS of would replace a +/// misfiled observation with a fabricated one. +const MEASURED_BYTES_PER_CELL: f64 = 16_228_499_456.0 / 481_327_124.0; + +fn projected_peak_bytes(main: u64, aux: u64) -> f64 { + (main + 3 * aux) as f64 * MEASURED_BYTES_PER_CELL +} + +/// Falsification of the census instrument itself: it must agree with what the +/// PROVER actually builds and with what the VERIFIER's AIR set declares. +/// +/// A census computed from the program alone would report the same numbers under a +/// broken trace builder, which is the "measures nothing" failure the method rules +/// name. Two independent oracles, both of which the census is not derived from: +/// +/// - the real [`super::trace::LfmTraces`] — the tables `multi_prove` receives — +/// for the heights; +/// - the AIR set [`super::airs::LfmAirs`] builds, for the NAMES and the widths. +/// The names matter more than they look: the census maps `per_chip` array slots +/// onto `LFM_CHIP_NAMES` across the `KECCAK_RND` slot, and nothing about a +/// height or a width can see that mapping being off by one. `air_refs` is the +/// frozen order's own definition, so comparing against it is what catches it. +#[test] +fn the_census_agrees_with_the_traces_the_prover_builds() { + // The two-permutation keccak chain: small, and it exercises every chip class + // the census names except `LFM_PUBLIC`'s value columns. + let program = super::programs::keccak_chain_program(); + let state: [u64; 25] = + core::array::from_fn(|i| 0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1)); + let arenas = vec![super::keccak_adapter::state_to_words(&state).to_vec()]; + // ⚠ ONE tenant for all four of execution, traces, census and AIRs, taken + // from the artifacts this test is about to compare against. The four used to + // default to `HasherKind::Test` while `build_artifacts` named the pin, and a + // trace built for one tenant against constraints built for another is an + // out-of-bounds index inside `HashConstraints::eval` — the socket chip's + // width is tenant-dependent (436 columns for RPO against 3,056 for BLAKE3). + // It surfaces as a bounds panic in a rayon worker, which reaches the test as + // "a scoped thread panicked" and names nothing at all. + let opts = wrap_options(); + let artifacts = build_artifacts(&program, &opts); + let exec = execute(&program, &arenas, &artifacts.hasher).expect("the chain program runs"); + let traces = super::trace::build_traces_with_hasher(&program, &exec.records, artifacts.hasher); + let census = super::airs::lfm_chip_census_with_hasher(&program, artifacts.hasher); + + // The frozen AIR order, as the census emits it and `air_trace_pairs` proves + // it. Built from the trace set so a chip whose height the census got from the + // wrong group shows up here. + let dims = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| (t.num_rows(), t.num_main_columns); + // Gated exactly as `air_trace_pairs` gates: this list is what the PROVER + // proves, and the chain program hashes with keccak, so it carries the + // keccak family and omits LFM_BLAKE3. + let chip_set = super::airs::ChipSet::for_program(&program); + assert_eq!( + (chip_set.keccak, chip_set.blake3), + (true, false), + "the keccak chain must be a keccak-family program with no BLAKE3 work" + ); + let mut built: Vec<(usize, usize)> = vec![ + dims(&traces.const_), + dims(&traces.balu), + dims(&traces.xalu), + dims(&traces.select), + dims(&traces.bitdec), + dims(&traces.hash), + ]; + if chip_set.keccak { + built.push(dims(&traces.keccak)); + } + built.push(dims(&traces.lanes)); + built.push(dims(&traces.hint)); + built.push(dims(&traces.public)); + built.push(dims(&traces.range)); + if chip_set.blake3 { + built.extend(traces.blake3.iter().map(dims)); + } + if chip_set.keccak { + built.extend(traces.keccak_rnd.iter().map(dims)); + built.push(dims(&traces.keccak_rc)); + } + built.push(dims(&traces.bitwise)); + + assert_eq!( + census.len(), + built.len(), + "the census must have one entry per trace the prover proves" + ); + for (c, (rows, width)) in census.iter().zip(&built) { + assert_eq!( + c.rows, *rows as u64, + "{}: the census height must be the trace's own", + c.name + ); + // The census counts VALUE columns, so the trace's full width less the + // preprocessed prefix must be what it reports. + assert!( + c.main_cols <= *width, + "{}: the census cannot count more value columns than the trace has", + c.name + ); + } + + // ---- the AIR set: the names and the widths, in the frozen order. + let airs = super::airs::LfmAirs::new_with_hasher( + &artifacts.roots, + &opts, + artifacts.keccak_rnd_chunks, + artifacts.hasher, + artifacts.chip_set, + ); + let refs = airs.air_refs(); + assert_eq!( + census.len(), + refs.len(), + "the census must have one entry per AIR the verifier builds" + ); + for (c, air) in census.iter().zip(&refs) { + assert_eq!( + c.name, + air.name(), + "the census and the AIR set disagree about the frozen chip order" + ); + let (main_width, aux_width) = air.trace_layout(); + let prep = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + assert_eq!( + c.main_cols, + main_width - prep, + "{}: the census must count the AIR's value columns", + c.name + ); + assert_eq!( + c.aux_cols, aux_width, + "{}: the census must count the AIR's aux columns", + c.name + ); + } + + let cells_of = |c: &[LfmChipCells], name: &str| -> u64 { + c.iter() + .filter(|e| e.name == name) + .map(|e| e.main_cells()) + .sum() + }; + assert!( + cells_of(&census, "KECCAK_RND") > 0, + "the chain program hashes, so KECCAK_RND must carry rows" + ); +} + +/// ★ R3 — the row-cliff instrument reports the two heights honestly. +/// +/// The panel's whole value is that `real_rows` is the height the workload +/// actually occupies and `rows` is the one the prover pays for. If those two +/// were ever the same number the headroom column would read 0% everywhere and +/// silently stop warning, which is exactly the failure mode the panel exists to +/// end — so this pins the relationship rather than the values. +#[test] +fn the_census_reports_real_and_committed_heights_separately() { + use super::airs::HeightRule; + use super::layout::MIN_GROUP_ROWS; + + let program = super::programs::keccak_chain_program(); + let census = lfm_chip_census(&program); + let chunk_perms = super::airs::keccak_rnd_chunk_permutations(&program); + let mut chunk = chunk_perms.iter(); + + for c in &census { + assert!( + c.real_rows <= c.rows, + "{}: a chip cannot occupy more rows than it commits ({} > {})", + c.name, + c.real_rows, + c.rows + ); + // The headroom column is this subtraction and nothing else, so an entry + // whose two heights are inconsistent with the padding rule would print a + // number that means nothing. + let expected = (c.rows - c.real_rows) as f64 / c.rows as f64; + assert!( + (c.headroom() - expected).abs() < 1e-12, + "{}: headroom must be the padding fraction", + c.name + ); + + match c.height_rule { + HeightRule::Workload => { + assert_eq!( + c.rows, + (c.real_rows as usize) + .next_power_of_two() + .max(MIN_GROUP_ROWS) as u64, + "{}: a workload-sized chip commits its real height padded to the \ + next power of two — if this ever stops holding, the headroom is \ + not a distance to a cliff", + c.name + ); + assert!( + c.at_risk(), + "{}: workload-sized chips are watchable", + c.name + ); + } + HeightRule::Fixed => { + assert_eq!( + c.real_rows, c.rows, + "{}: a lookup table is full, so its two heights coincide", + c.name + ); + assert!( + !c.at_risk(), + "{}: a fixed table reads 0% headroom because it is full, not \ + because it is about to double — the panel must not warn on it", + c.name + ); + } + HeightRule::Chunked => { + let perms = chunk.next().expect("one census chunk per policy chunk"); + assert_eq!( + c.real_rows, + (perms * super::chunking::KECCAK_RND_ROWS_PER_PERMUTATION) as u64, + "a chunk occupies 24 rows per permutation it carries" + ); + assert!( + !c.at_risk(), + "a full chunk sits permanently just under a power of two and can \ + never cross it — the policy emits another chunk instead" + ); + } + } + } + assert!( + chunk.next().is_none(), + "every policy chunk must appear in the census" + ); + assert!( + census.iter().any(|c| c.height_rule == HeightRule::Workload), + "the census must contain workload-sized chips for the panel to watch" + ); +} + +/// ★ R3 — the headroom the panel would have printed for the artifact. +/// +/// The bisect measured `LFM_LANES` at 4,141,992 of 4,194,304 rows on the real +/// block at 110 queries and called it 1.2% — the margin that let #903's ~21% +/// growth trip five chip doublings at once. That number is the instrument's +/// reason to exist, so it is pinned here against the recorded measurement +/// (`thoughts/shared/block-compression/WRAP-GROWTH-BISECT.md`, "Headroom at the +/// artifact"). A 110q census needs ~30 GiB to emit, so the arithmetic is pinned +/// on the recorded row counts rather than by re-running it. +#[test] +fn the_row_cliff_panel_reproduces_the_artifacts_measured_headroom() { + use super::airs::HeightRule; + + // (chip, rows the mix occupied, rows committed, headroom the bisect reports) + let artifact = [ + ("LFM_LANES", 4_141_992u64, 4_194_304u64, 1.2), + ("LFM_SELECT", 463_650, 524_288, 11.6), + ("LFM_BALU", 110_147_086, 134_217_728, 17.9), + ("LFM_BITDEC", 1_717_958, 2_097_152, 18.1), + ("LFM_CONST", 1_656, 2_048, 19.1), + ("LFM_HINT", 1_523_011, 2_097_152, 27.4), + ("LFM_XALU", 1_485_219, 2_097_152, 29.2), + ]; + + for (name, real_rows, rows, expected_pct) in artifact { + let c = LfmChipCells { + name, + rows, + real_rows, + height_rule: HeightRule::Workload, + main_cols: 1, + aux_cols: 0, + }; + assert_eq!( + rows, + (real_rows as usize).next_power_of_two() as u64, + "{name}: the recorded committed height must be the recorded real height padded", + ); + let got = 100.0 * c.headroom(); + assert!( + (got - expected_pct).abs() < 0.05, + "{name}: panel would print {got:.1}%, the bisect measured {expected_pct}%", + ); + // Crossing doubles the height, so the chip adds exactly what it already + // contributes — the quantity the panel prices. + assert_eq!(c.cliff_cost(), c.main_cells() + 3 * c.aux_cells()); + } +} + +/// ★ GATE B (P1) — a from-proof epoch wraps end to end, and it is the FINAL +/// epoch of its continuation (HALT on board): the shape the real block's last +/// epoch has, which the session harness cannot build. The epoch reaches the +/// wrap through [`super::epoch_tests::real_epoch_from_continuation`] — proofs +/// alone, no live proving session — and the proved run must publish the +/// epoch's own oracles. +/// +/// Run with: +/// `cargo test --release -p lambda-vm-prover --lib lfm::wrap_tests::the_from_proof_final_epoch_wraps -- --ignored --exact --nocapture` +#[test] +#[ignore] +fn the_from_proof_final_epoch_wraps() { + let elf_bytes = super::proof_fixture::read_inner_elf(); + let inner = super::epoch_tests::from_proof_gate_options(); + let bundle = crate::continuation::prove_continuation( + &elf_bytes, + &[], + super::proof_fixture::FIXTURE_EPOCH_LOG2, + &inner, + ) + .expect("the fixture continuation must prove"); + let n = bundle.num_epochs(); + assert!(n >= 2, "the fixture continuation must have a final epoch"); + + let e = + super::epoch_tests::real_epoch_from_continuation(&inner, &elf_bytes, &bundle, n - 1, None) + .expect("the final epoch must reconstruct from proofs alone"); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let opts = wrap_options(); + let artifacts = build_artifacts_with_hasher(&program, &opts, crate::hash_pin::BLOCK_HASHER); + + let t = Instant::now(); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("the wrap must prove"); + let prove_secs = t.elapsed().as_secs_f64(); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + artifacts.chip_set, + ), + "the wrap of the final epoch must verify" + ); + + // The proved run publishes the epoch's own oracles. + let pub_ext = + |i: usize| super::word::word_as_ext(&proved.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the proved run publishes z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the proved run publishes alpha"); + assert_eq!( + super::word::word_as_ext(&proved.public_words[proved.public_words.len() - 1].1) + .expect("the bus total is ext"), + e.expected_bus_balance, + "the proved run reaches production's own COMMIT-bus target" + ); + println!( + "\n★ P1 GATE B: the FINAL epoch (epoch {} of {n}), wrapped from proofs alone — \ + proved in {prove_secs:.1}s, verified, {} sub-proofs, {} published words", + n - 1, + proved.proof.proofs.len(), + proved.public_words.len(), + ); +} + +/// Peak RSS high-water mark of this process, GiB — `VmHWM` on Linux (the box); +/// `None` elsewhere. +pub(super) fn peak_rss_gib() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|l| l.starts_with("VmHWM:"))?; + let kb: f64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb / (1024.0 * 1024.0)) +} + +/// ★ P1 — THE BLOCK, end to end, in one process: the production continuation +/// prove (every epoch + the global proof), full bundle host verification +/// (every epoch, the global proof, and the L2G root-equality binding), then +/// EVERY epoch wrapped from the proofs alone. The numbers this prints are +/// BLOCK numbers; per-epoch lines are supporting detail. +/// +/// Residency: the epoch proves are `Retain` (hardcoded in +/// `prove_continuation`); the wrap proves are `Retain` (the `lfm_prove` +/// default). Proof BYTES are not run-reproducible (grinding nonces); roots +/// are. +/// +/// env (required): `LFM_CENSUS_ELF`, `LFM_CENSUS_INPUT`. Optional: +/// `LFM_CENSUS_EPOCH_LOG2` (epoch size), `LFM_WRAP_QUERIES` (inner query +/// count override; the default is the secure preset's). +/// +/// Run at the 2^24 posture: +/// ```text +/// LFM_CENSUS_ELF=/path/to/ethrex.elf \ +/// LFM_CENSUS_INPUT=/path/to/ethrex_mainnet_25368371.bin \ +/// LFM_CENSUS_EPOCH_LOG2=24 LAMBDA_VM_MAX_ROWS_LOG2=24 \ +/// cargo test --release -p lambda-vm-prover --lib \ +/// lfm::wrap_tests::the_real_block_proves_and_wraps_end_to_end -- --ignored --exact --nocapture +/// ``` +#[test] +#[ignore] +fn the_real_block_proves_and_wraps_end_to_end() { + for var in ["LFM_CENSUS_ELF", "LFM_CENSUS_INPUT"] { + assert!( + std::env::var(var).is_ok(), + "{var} must name a file: this test proves a REAL block, and without \ + it the harness would build the continuation fixture and report it \ + under this test's name" + ); + } + let inputs = EpochInputs::from_env(); + let mut inner = crate::recursion::Preset::Blowup4.options(); + if let Ok(v) = std::env::var("LFM_WRAP_QUERIES") { + inner.fri_number_of_queries = v.parse().expect("LFM_WRAP_QUERIES must be an integer"); + } + println!( + "★ P1 BLOCK RUN: guest {}, {} bytes of private input, 2^{} cycles/epoch, \ + inner blowup {} / {} queries{} — epoch residency Retain, wrap residency Retain", + inputs.label, + inputs.private_input.len(), + inputs.epoch_log2, + inner.blowup_factor, + inner.fri_number_of_queries, + if inner.fri_number_of_queries < 110 { + " (REDUCED — not a security parameter set)" + } else { + " (the secure preset)" + }, + ); + + let t_total = Instant::now(); + + // ---- the base layer: every epoch + the global proof, production's path. + let t = Instant::now(); + let bundle = crate::continuation::prove_continuation( + &inputs.elf_bytes, + &inputs.private_input, + inputs.epoch_log2, + &inner, + ) + .expect("the block must prove"); + let base_secs = t.elapsed().as_secs_f64(); + let n = bundle.num_epochs(); + let bundle_bytes = rkyv::to_bytes::(&bundle) + .expect("the bundle must serialize") + .len(); + println!( + " base: {n} epochs + global proof in {base_secs:.1}s ({bundle_bytes} bundle bytes), \ + peak RSS so far {:?} GiB", + peak_rss_gib(), + ); + + // ---- full host verification: every epoch, the global proof, the binding. + let t = Instant::now(); + let out = crate::continuation::verify_continuation(&inputs.elf_bytes, &bundle, &inner) + .expect("the bundle must be well-formed"); + assert!( + out.is_some(), + "the block bundle must host-verify (epochs + global + L2G root binding)" + ); + let host_verify_secs = t.elapsed().as_secs_f64(); + println!(" host verify (epochs + global + binding): {host_verify_secs:.1}s"); + + // ---- every epoch, wrapped from the proofs alone. + // ⓘ The DECODE commitment was already hoisted here; the ELF PARSE was not — + // `real_epoch_from_continuation` reloaded 3.4 MB of it per epoch. Both now + // ride one `EpochConstants`, which is what makes the pair impossible to + // hoist by halves. + let epoch_konsts = super::epoch_tests::EpochConstants::load(&inputs.elf_bytes, &inner, None) + .expect("the inner ELF and its DECODE commitment must build once"); + let wrap_opts = wrap_options(); + let (mut construct_secs, mut wrap_prove_secs, mut wrap_verify_secs) = (0f64, 0f64, 0f64); + let mut wrap_sizes = Vec::new(); + for i in 0..n { + let t = Instant::now(); + let e = super::epoch_tests::real_epoch_from_constants(&inner, &epoch_konsts, &bundle, i) + .unwrap_or_else(|err| panic!("epoch {i} must reconstruct from the bundle: {err}")); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let artifacts = + build_artifacts_with_hasher(&program, &wrap_opts, crate::hash_pin::BLOCK_HASHER); + let c = t.elapsed().as_secs_f64(); + construct_secs += c; + + let t = Instant::now(); + let proved = lfm_prove(&program, &artifacts, &arenas, &wrap_opts) + .unwrap_or_else(|err| panic!("epoch {i}'s wrap must prove: {err:?}")); + let p = t.elapsed().as_secs_f64(); + wrap_prove_secs += p; + + let t = Instant::now(); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &wrap_opts, + artifacts.hasher, + artifacts.chip_set, + ), + "epoch {i}'s wrap must verify" + ); + let v = t.elapsed().as_secs_f64(); + wrap_verify_secs += v; + + let size = rkyv::to_bytes::(&proved.proof) + .expect("the wrap proof must serialize") + .len(); + wrap_sizes.push(size); + println!( + " epoch {i}: reconstruct+emit {c:.1}s, wrap prove {p:.1}s, verify {v:.2}s, \ + {size} bytes, {} sub-proofs", + proved.proof.proofs.len(), + ); + } + + let total = t_total.elapsed().as_secs_f64(); + println!( + "\n★★★ P1 BLOCK RECORD (per-table): {n} epochs @2^{} cycles, inner blowup {} / {}q, \ + wrap blowup {} / {}q, residency Retain both layers\n \ + base prove {base_secs:.1}s + host verify {host_verify_secs:.1}s + wrap constructs \ + {construct_secs:.1}s + wrap proves {wrap_prove_secs:.1}s + wrap verifies \ + {wrap_verify_secs:.1}s\n TOTAL WALL {total:.1}s ({:.1} min)\n \ + proofs: bundle {bundle_bytes} B, wraps {wrap_sizes:?} B\n \ + peak RSS (VmHWM): {:?} GiB", + inputs.epoch_log2, + inner.blowup_factor, + inner.fri_number_of_queries, + wrap_opts.blowup_factor, + wrap_opts.fri_number_of_queries, + total / 60.0, + peak_rss_gib(), + ); +} + +/// +/// The real block's cross-epoch PAGE CENSUS — execution and collection only, +/// nothing proven. Prints the numbers the aggregator's closed-form census +/// consumes: the global memory proof carries one GLOBAL_MEMORY table per +/// touched page, so the aggregation program's global-verify legs scale with +/// exactly what this prints. Runs locally in minutes (same env contract as +/// the block drivers). +#[test] +#[ignore] +fn the_real_blocks_page_census() { + for var in ["LFM_CENSUS_ELF", "LFM_CENSUS_INPUT"] { + assert!( + std::env::var(var).is_ok(), + "{var} must name a file: this census is only meaningful on a real block" + ); + } + let inputs = EpochInputs::from_env(); + let census = crate::continuation::block_page_census( + &inputs.elf_bytes, + &inputs.private_input, + inputs.epoch_log2, + ) + .expect("the census run must execute"); + let mut hist: std::collections::BTreeMap = std::collections::BTreeMap::new(); + for &cells in &census.page_cells { + *hist + .entry(cells.next_power_of_two().max(4).trailing_zeros()) + .or_default() += 1; + } + println!( + "★ PAGE CENSUS {}: {} epochs, {} touched pages ({} private-input), \ + crossing cells per epoch {:?}", + inputs.label, + census.num_epochs, + census.touched_page_bases.len(), + census.num_private_input_pages, + census.l2g_cells, + ); + println!(" page-table height histogram (log2 padded rows -> pages): {hist:?}"); +} diff --git a/prover/src/lib.rs b/prover/src/lib.rs index f24bc7a61..d045a6257 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -10,35 +10,55 @@ //! assert!(lambda_vm_prover::verify(&vm_proof, &elf_bytes).unwrap()); //! ``` -#[cfg(feature = "disk-spill")] -pub mod auto_storage; +pub mod alloc_purge; +// The block prover runs its phases on threads and rayon: a host prover, built +// with `parallel` (the guests take the crate without it). +#[cfg(feature = "parallel")] +pub mod block_whir; pub mod constraints; pub mod continuation; #[cfg(feature = "debug-checks")] mod debug_report; +pub mod hash_pin; #[cfg(feature = "instruments")] pub mod instruments; +pub mod lfm; +pub mod multilinear_prove; mod paged_mem; pub use stark::profile_markers; pub mod recursion; +#[cfg(feature = "shape-profile")] +pub mod shape_profile; mod statement; pub mod tables; pub mod test_utils; #[cfg(test)] pub mod tests; +pub mod whir_hash_knob; +pub mod whir_identity; +pub mod zf_format; + +// The lib's test harness runs the allocator the shipped binary runs +// (`bin/cli/src/main.rs` installs the same one), so every host-memory number a +// `cargo test --lib` measurement produces is a production-allocator number. +// Under the platform allocator the same proves read up to 13 GiB higher at the +// wrap's q=41 rung: glibc kept freed arena chunks resident, and the run +// measured the allocator, not the prover. Integration tests are separate +// crates and install their own (`tests/calibration.rs` already does). +#[cfg(test)] +#[global_allocator] +static TEST_ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; use std::fmt; +use std::sync::Arc; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use crypto::fiat_shamir::is_transcript::IsTranscript; use executor::elf::Elf; use executor::vm::execution::Executor; use math::field::element::FieldElement; -use stark::prover::{IsStarkProver, Prover}; -#[cfg(feature = "disk-spill")] -use stark::storage_mode::StorageMode; +use stark::prover::IsStarkProver; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::statement::{StatementKind, absorb_statement, absorb_statement_with_digest}; pub use crate::tables::MaxRowsConfig; @@ -47,16 +67,15 @@ use crate::tables::decode; use crate::tables::page; use crate::tables::register; use crate::tables::trace_builder::Traces; -#[cfg(feature = "disk-spill")] -use crate::tables::trace_builder::count_table_lengths; use crate::tables::types::BusId; use crate::test_utils::{ - E, F, VmAir, create_bitwise_air, create_branch_air, create_bytewise_air, create_copy_air, - create_cpu_air, create_cpu32_air, create_decode_air, create_dvrm_air, create_ecdas_air, - create_ecsm_air, create_eq_air, create_halt_air, create_hint_air, create_is_wh_air, - create_keccak_air, create_keccak_rc_air, create_keccak_rnd_air, create_load_air, create_lt_air, - create_memw_air, create_memw_aligned_air, create_memw_register_air, create_mul_air, - create_page_air, create_register_air, create_shift_air, create_store_air, + E, F, VmAir, create_bitwise_air, create_blake3_air, create_branch_air, create_bytewise_air, + create_copy_air, create_cpu_air, create_cpu32_air, create_decode_air, create_dvrm_air, + create_ecdas_air, create_ecrec_air, create_ecsm_air, create_eq_air, create_halt_air, + create_hint_air, create_is_wh_air, create_keccak_air, create_keccak_rc_air, + create_keccak_rnd_air, create_load_air, create_lt_air, create_memw_air, + create_memw_aligned_air, create_memw_register_air, create_mul_air, create_page_air, + create_register_air, create_shift_air, create_store_air, }; // Re-exported for downstream hosts and verifier guests (e.g. the in-VM @@ -81,19 +100,28 @@ pub struct RuntimePageRange { } /// Number of tables that contribute exactly one sub-proof regardless of -/// `TableCounts`: bitwise, decode, halt, keccak_rc, register, copy. The other -/// accelerator chips are counted instead — a run that never calls one carries no -/// table for it. COPY stays fixed because it receives the `write` ECALL itself, so -/// any run that produces output reaches it. +/// `TableCounts`: bitwise, decode, halt, keccak_rc, register. The accelerator +/// chips are counted instead — a run that never calls one carries no table for +/// it. COPY is one of them here (main keeps it fixed): a block proof cuts it into +/// tables of a fixed height, so it needs a count ([`TableCounts::copy`]). /// /// HALT is the exception, and the reason this is not simply "always": a /// continuation epoch carries it only when it is the final one, so `verify_epoch` /// sizes non-final epochs with `FIXED_TABLE_COUNT - 1`. Any caller computing an /// expected sub-proof count for a continuation epoch must do the same. -pub const FIXED_TABLE_COUNT: usize = 6; +/// +/// ⚠ Every always-on table costs every proof a near-empty AIR even when the +/// workload never touches it (the EC-campaign lesson, PR #871). BLAKE3 was +/// always-on for one release and that bill came due: on a real block it cost +/// the recursion wrap +31.2% of its trace cells and +44% of its peak memory, +/// out of a table carrying 0.035% of the epoch's own cells. It left this count +/// for [`TableCounts::blake3`], and the six accelerator chips followed it for +/// the same reason — see that field for why omitting such a table is sound +/// (`thoughts/shared/block-compression/WRAP-GROWTH-BISECT.md`). +pub const FIXED_TABLE_COUNT: usize = 5; /// How many sub-proofs each counted table contributes. Chunked chips report -/// their chunk count; the five accelerators and IS_WH are not chunked and report +/// their chunk count; the six accelerators and IS_WH are not chunked and report /// 0 or 1 (see `validate`). The verifier needs this to reconstruct matching AIRs. #[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct TableCounts { @@ -120,9 +148,74 @@ pub struct TableCounts { pub ecsm: usize, pub ecdas: usize, pub hint: usize, - // Range-check chip for the top 48 bits of `DWordWHH` addresses. One table - // when a chip sends `IS_WH` (today only KECCAK), none otherwise. + /// COPY tables: memcpy, memmove, memset and the `write` syscall's bytes. + /// 0 or 1 here ([`TableCounts::validate`]); a block proof cuts COPY into + /// tables of a fixed height and bounds the count itself. Zero is sound by + /// the argument [`TableCounts::blake3`] spells out: the CPU's Ecall sends + /// for syscalls 64, `-30` and `-31` have no receiver but COPY, and the + /// verifier closes the Commit bus from the public output, so a run that + /// wrote or copied anything cannot omit the table. + pub copy: usize, + /// ★ BLAKE3 tables: **0 or 1**, not a chunk count — 1 iff the workload + /// executed at least one BLAKE3 syscall. + /// + /// ## Why a zero here is sound, when a zero anywhere else is not + /// + /// Every other field in this struct is rejected at zero by + /// [`TableCounts::validate`], because dropping (say) the LT table would + /// delete its constraints while the CPU rows that depend on them stay in + /// the proof — the classic remove-the-checker forgery. BLAKE3 is different, + /// and the difference is a bus argument, not a convention: + /// + /// - CPU is the sole SENDER on [`tables::types::BusId::Ecall`], with + /// multiplicity `Column(cols::ECALL)` and tuple + /// `[timestamp, 0, rv1]`, where `rv1` is the syscall number read out of + /// the guest's register — **trace data**, not a constant + /// (`tables/cpu.rs`, the ECALL sender). + /// - Each syscall table RECEIVES on that same bus with its own syscall + /// number as a hardcoded constant in the tuple. The BLAKE3 table has TWO + /// such receivers, one per mode: the single-compression receive carries + /// `BLAKE3_SYSCALL_NUMBER` (`tables/blake3.rs`, interaction 1) and the + /// absorb receive carries `BLAKE3_ABSORB_SYSCALL_NUMBER` (gated on the + /// group's FIRST row). No other receiver in the machine carries either + /// constant, so the argument covers both syscalls: omitting the table + /// strands compress sends and absorb sends alike. + /// + /// So a workload that executes a BLAKE3 syscall — either mode — necessarily + /// puts a send on the Ecall bus whose tuple only the BLAKE3 table can match. Omit the + /// table and that send has no receiver: the global LogUp sum is non-zero, + /// and `multi_verify` rejects on bus balance before any constraint is + /// evaluated. A prover claiming `blake3: 0` for an epoch that used BLAKE3 + /// is therefore not saving work — it is producing a proof that cannot + /// verify. The count is a size hint the bus already enforces, not a + /// permission the verifier grants. + /// + /// ## Why it must still be bound into the statement + /// + /// `include_halt` is NOT a precedent for leaving this unbound: `is_final` + /// is known to the verifier a priori, whereas "this epoch used BLAKE3" is + /// asserted by the prover. Unbound, prover and verifier could build + /// different AIR sets from the same bytes. It is bound the same way every + /// other count is — absorbed into the Fiat-Shamir transcript by + /// `statement::absorb_statement_with_digest`, so a disagreement diverges + /// every derived challenge — and cross-checked against the sub-proof count + /// in `verify_proof_parts`. + /// + /// Bounded above by 1 in [`TableCounts::validate`]: extra BLAKE3 tables buy + /// an attacker nothing, but an unbounded count would let a malformed proof + /// make the verifier build arbitrarily many AIRs before any other check. + pub blake3: usize, + /// IS_WH: the range check for the top 48 bits of the `DWordWHH` addresses + /// a chip sends it (today only KECCAK's state address). One table when any + /// is sent, none otherwise; at most one ([`TableCounts::validate`]), which is + /// what bounds its multiplicity sum. Zero is sound by the same bus argument + /// as COPY's: KECCAK's `IS_WH` sends have no other receiver. pub is_wh: usize, + /// ECREC tables (ECDSA public-key recovery): 0 or 1, like the other + /// accelerators. Zero is sound by the same Ecall-bus argument: ECREC is the + /// only receiver of the syscall `-12`, so a run that called it cannot omit + /// the table. + pub ecrec: usize, } impl TableCounts { @@ -154,6 +247,9 @@ impl TableCounts { self.ecsm, self.ecdas, self.hint, + self.ecrec, + self.copy, + self.blake3, self.is_wh, ] .into_iter() @@ -171,12 +267,14 @@ impl TableCounts { /// /// What keeps a zero count honest is the LogUp bus, not this check. A chip /// influences the run only through its bus interactions, and none of the - /// eighteen optional chips carries boundary constraints of its own, so one - /// with no rows contributes zero to the bus and removing it changes nothing. - /// (The mandatory tables do constrain boundaries — that is part of why they - /// are mandatory.) A prover that omits a table whose operations *did* - /// execute leaves its counterparty's sends unmatched, and the bus-balance - /// check — summed over the tables that are present — rejects the proof. + /// optional chips carries boundary constraints of its own, so one with no + /// rows contributes zero to the bus and removing it changes nothing. (The + /// mandatory tables do constrain boundaries — that is part of why they are + /// mandatory.) A prover that omits a table whose operations *did* execute + /// leaves its counterparty's sends unmatched, and the bus-balance check — + /// summed over the tables that are present — rejects the proof. For BLAKE3 + /// the same argument is spelled out end to end on [`TableCounts::blake3`], + /// down to the two Ecall receivers that make it airtight. /// /// Two things that argument needs, both pinned by tests. Each omitted table /// must be a bus participant (`every_table_participates_in_the_bus`), and @@ -206,15 +304,23 @@ impl TableCounts { // lets an inflated count through whenever another count is // lowered to match; this pins the per-field shape instead. It is not // load-bearing for memory safety — both verifiers run the cross-check - // before any count sizes an AIR set. For the accelerators it is only - // shape, and if one of them ever becomes chunked, this list is what - // changes. IS_WH is the exception, see its entry. + // before any count sizes an AIR set — and if one of them ever becomes + // chunked, this list is what changes. IS_WH is the exception, see its + // entry: for it the bound is soundness, not shape. + // + // BLAKE3 belongs to the same list for the same reason, and had its own + // bound before these six existed: nothing is gained by claiming more + // than one BLAKE3 table, and an unbounded count would have the verifier + // allocate AIRs off a number the proof has not been checked against. let at_most_one = [ ("keccak", self.keccak), ("keccak_rnd", self.keccak_rnd), ("ecsm", self.ecsm), ("ecdas", self.ecdas), ("hint", self.hint), + ("ecrec", self.ecrec), + ("copy", self.copy), + ("blake3", self.blake3), // Load-bearing for soundness: this is what bounds the IS_WH // multiplicity sum per proof. One table of at most 2^k rows, each // μ < 2^20, stays far below p, so no stack of repeated rows reaches @@ -285,14 +391,37 @@ pub struct GuestInput { /// 4-byte magic identifying a lambda-vm recursion input blob ("LVMR"). pub const RECURSION_INPUT_MAGIC: [u8; 4] = *b"LVMR"; -/// Wire-format version of the recursion input blob. v5: `TableCounts` gained -/// `is_wh`. v4: `TableCounts` lost its `commit` field when COPY absorbed the -/// COMMIT table, which shortens the archive. v3: `TableCounts` gained the six -/// accelerator fields, which moves every field after it in the archive. v2: -/// rkyv pointer_width_64 (64-bit rel-ptrs) — v1 archives use 32-bit offsets. -/// Any change to a type reachable from `GuestInput` belongs here: the guest -/// reads the archive in place, so a stale blob is misread rather than rejected. -pub const RECURSION_INPUT_VERSION: u32 = 5; +/// Wire-format version of the recursion input blob. +/// +/// Any change to a type reachable from `GuestInput` belongs here. The prefix IS +/// checked — `recursion_archive_bytes` rejects a version it does not recognise +/// before the guest's in-place read — so a blob carrying an OLD number is +/// refused legibly. The hazard this constant exists for is the other one: a +/// layout change that does NOT bump it passes that check and is then read in +/// place at offsets the reader does not expect. +/// +/// - v1: rkyv 32-bit rel-ptrs. +/// - v2: rkyv pointer_width_64 (64-bit rel-ptrs) — v1 archives are incompatible. +/// - v3: one epoch proof format. `EpochProof::proof` is a `MultiProof` where v2 +/// had an `EpochProofBody` enum, so every v2 archive carries a discriminant at +/// an offset a v3 reader does not expect. +/// - v4: `TableCounts` gained the six accelerator fields, which moves every +/// field after it in the archive. +/// +/// ⚠ v4 EXISTS BECAUSE TWO BRANCHES BOTH CALLED THEIR CHANGE v3, which is the +/// unbumped case in disguise. The per-table lineage's v3 is the epoch proof +/// format; main's v3 is the accelerator fields. Both are reachable from +/// `GuestInput` and neither is a superset of the other, so two mutually +/// unreadable archives carried one number — and a prefix check cannot tell them +/// apart. Keeping either suffix would have been a silent misread; the merged +/// format carries both changes and is v4. +/// - v5: `TableCounts`' `commit` field became `copy` when COPY absorbed the +/// COMMIT table: the same offset, counting another table. +/// - v6: `TableCounts` gained `is_wh` after `blake3` (main's IS_WH table). Main's +/// own v5 is `is_wh` on a layout with no `copy` or `blake3` field: a different +/// archive, so this one takes a number neither line has used (the v4 lesson). +/// - v7: `TableCounts` gained `ecrec` after `is_wh` (the ECDSA-recovery table). +pub const RECURSION_INPUT_VERSION: u32 = 7; /// Required alignment (bytes) of the archive's first byte in guest memory. pub const RECURSION_INPUT_ALIGN: usize = 16; @@ -540,6 +669,9 @@ pub enum Error { /// layout: unaligned or overflowing base, zero count, more pages than the /// proof can hold, or two pages covering the same address. MalformedPageLayout(String), + /// A timestamp would reach 2^32, past the CPU's 32-bit clock: the run's + /// buses cannot balance (`tables::trace_builder::clock`). + ClockPastLimit(String), } impl fmt::Display for Error { @@ -570,6 +702,7 @@ impl fmt::Display for Error { } Error::Recursion(msg) => write!(f, "recursion helper error: {msg}"), Error::MalformedPageLayout(msg) => write!(f, "malformed page layout: {msg}"), + Error::ClockPastLimit(msg) => write!(f, "clock past its limit: {msg}"), } } } @@ -577,17 +710,18 @@ impl fmt::Display for Error { impl std::error::Error for Error {} /// Type alias for AIR-trace-public-inputs triples used in multi-table proving. -type AirTracePair<'a> = ( +pub type AirTracePair<'a> = ( &'a dyn AIR, &'a mut stark::trace::TraceTable, &'a (), ); -/// Sub-proof index of the first CPU table: [`VmAirs::air_refs`] puts the five -/// fixed tables (BITWISE, DECODE, KECCAK_RC, COPY, REGISTER), HALT (when included), -/// the accelerator tables and IS_WH first. -fn cpu_start(include_halt: bool, accelerator_counts: [usize; 6]) -> usize { - 5 + usize::from(include_halt) + accelerator_counts.iter().sum::() +/// Sub-proof index of the first CPU table: [`VmAirs::air_refs`] puts the four +/// fixed tables (BITWISE, DECODE, KECCAK_RC, REGISTER), HALT (when included), +/// the counted COPY, KECCAK, KECCAK_RND, ECSM, ECDAS, HINT and ECREC tables, BLAKE3 +/// and IS_WH first. +fn cpu_start(include_halt: bool, counted: [usize; 9]) -> usize { + 4 + usize::from(include_halt) + counted.iter().sum::() } /// [`cpu_start`] for a proof's declared `table_counts`. @@ -595,11 +729,14 @@ pub(crate) fn cpu_proof_start(include_halt: bool, table_counts: &TableCounts) -> cpu_start( include_halt, [ + table_counts.copy, table_counts.keccak, table_counts.keccak_rnd, table_counts.ecsm, table_counts.ecdas, table_counts.hint, + table_counts.ecrec, + table_counts.blake3, table_counts.is_wh, ], ) @@ -637,13 +774,15 @@ pub(crate) struct VmAirs { pub dvrms: Vec, pub branches: Vec, pub halt: VmAir, + pub copies: Vec, pub keccaks: Vec, pub keccak_rnds: Vec, pub keccak_rc: VmAir, + pub blake3: VmAir, pub ecsms: Vec, pub ecdases: Vec, pub hints: Vec, - pub copy: VmAir, + pub ecrecs: Vec, pub is_whs: Vec, pub register: VmAir, pub pages: Vec, @@ -651,6 +790,16 @@ pub(crate) struct VmAirs { /// Whether the HALT table participates in this proof. False for intermediate /// continuation epochs, which do not terminate the program. pub include_halt: bool, + /// Whether the BLAKE3 table participates in this proof — `table_counts + /// .blake3 == 1`, i.e. the workload executed a BLAKE3 syscall. + /// + /// Unlike `include_halt`, this is NOT verifier-known a priori: `is_final` + /// follows from the epoch's position, while "did this workload use BLAKE3" + /// is asserted by the prover. Honouring that assertion is safe only because + /// a false one cannot verify — see [`TableCounts::blake3`] for the + /// Ecall-bus argument. Both sides derive this from the same bound count, so + /// they cannot silently build different AIR sets. + pub include_blake3: bool, // Auxiliary ALU / memory / CPU32 dispatch chips pub eqs: Vec, pub bytewises: Vec, @@ -658,6 +807,133 @@ pub(crate) struct VmAirs { pub cpu32s: Vec, } +/// What REGISTER is preprocessed against: the commitment **and** the two public +/// vectors it commits — a continuation epoch's INIT = `R_i` and FINI = `R_{i+1}`, +/// or a monolithic proof's (a block's) entry file and halted file +/// ([`RegisterFiles::halted`]). +/// +/// ★ THE VECTORS TRAVEL WITH THE COMMITMENT BECAUSE THE TWO VERIFIERS BIND +/// DIFFERENT THINGS. The univariate verifier recomputes the root and compares +/// it; the multilinear one has no precomputed tree and instead checks the +/// proof's claimed openings of columns `0..n` against the values the program +/// implies, which it obtains from +/// [`stark::traits::AIR::precomputed_columns`]. Handing an AIR a commitment +/// with no columns closure therefore binds the first path and nothing at all on +/// the second. Carrying both here makes "preprocessed" mean the same thing on +/// both, and makes the omission a missing field rather than a silent `None`. +pub struct RegisterPreprocessed<'a> { + /// The root, for the univariate verifier. + pub commitment: Commitment, + /// `R_i`, the epoch's starting register file — the verifier's own value: + /// the ELF's entry point for epoch 0, the previous epoch's proved `reg_fini` + /// after that. + pub init: &'a [u32], + /// `R_{i+1}`, the epoch's final register file as the proof states it. + pub fini: &'a [u32], +} + +/// REGISTER's preprocessing, owned: the two register files and their root, so a +/// call site hands [`Self::preprocessed`] to [`VmAirs::new`]. A monolithic proof +/// or a block takes [`Self::halted`]: INIT from the ELF's entry point, FINI +/// pinned to the halted file (`register::halted_fini`: every register zero but +/// x254, the committed byte count, and the PC at 1). +pub(crate) struct RegisterFiles { + commitment: Commitment, + init: Vec, + fini: Vec, +} + +impl RegisterFiles { + /// INIT `init`, FINI `fini`, and their root. + pub(crate) fn new(proof_options: &ProofOptions, init: Vec, fini: Vec) -> Self { + let commitment = register::compute_precomputed_commitment(proof_options, &init, &fini); + Self { + commitment, + init, + fini, + } + } + + /// The halted files of a run from `entry_point` that committed + /// `committed_bytes` bytes. + pub(crate) fn halted( + proof_options: &ProofOptions, + entry_point: u64, + committed_bytes: u32, + ) -> Self { + Self::new( + proof_options, + register::register_init_from_entry_point(entry_point), + register::halted_fini(committed_bytes), + ) + } + + /// The files' root (the univariate verifier's). + #[cfg(test)] + pub(crate) fn commitment(&self) -> Commitment { + self.commitment + } + + pub(crate) fn preprocessed(&self) -> RegisterPreprocessed<'_> { + RegisterPreprocessed { + commitment: self.commitment, + init: &self.init, + fini: &self.fini, + } + } +} + +/// The CPU tables' heights a block states and REGISTER's halted file, checked: +/// what [`block_airs`] builds the block's AIRs from. The heights fix every CPU +/// timestamp (each table's base, `tables::cpu::timestamp_bases`) and the final PC +/// timestamp, so they must respect the per-run row bound before they size +/// anything; x254's final value is the committed byte count, a `u32`. +pub(crate) struct BlockAirInputs { + cpu_heights: Vec, + register: RegisterFiles, +} + +impl BlockAirInputs { + pub(crate) fn new( + opts: &ProofOptions, + program: &Elf, + cpu_heights: Vec, + public_output: &[u8], + ) -> Result { + crate::tables::cpu::validate_heights(&cpu_heights)?; + let committed = u32::try_from(public_output.len()).map_err(|_| { + Error::InvalidTableCounts("the public output exceeds u32 range".to_string()) + })?; + Ok(Self { + cpu_heights, + register: RegisterFiles::halted(opts, program.entry_point, committed), + }) + } +} + +/// A block's AIRs: every table of `table_counts`, HALT included, the CPU +/// tables at `inputs`' heights and REGISTER's FINI pinned to the halted file. +pub(crate) fn block_airs( + program: &Elf, + opts: &ProofOptions, + page_configs: &[crate::tables::page::PageConfig], + table_counts: &TableCounts, + inputs: &BlockAirInputs, +) -> VmAirs { + VmAirs::new( + program, + opts, + false, + page_configs, + table_counts, + &inputs.cpu_heights, + None, + true, + None, + inputs.register.preprocessed(), + ) +} + impl VmAirs { /// Build `(air, trace, public_inputs)` triples for [`Prover::multi_prove`]. pub fn air_trace_pairs<'a>(&'a self, traces: &'a mut Traces) -> Vec> { @@ -676,11 +952,13 @@ impl VmAirs { self.muls.len(), self.dvrms.len(), self.branches.len(), + self.copies.len(), self.keccaks.len(), self.keccak_rnds.len(), self.ecsms.len(), self.ecdases.len(), self.hints.len(), + self.ecrecs.len(), self.is_whs.len(), self.pages.len(), self.memw_registers.len(), @@ -699,11 +977,13 @@ impl VmAirs { traces.muls.len(), traces.dvrms.len(), traces.branches.len(), + traces.copies.len(), traces.keccaks.len(), traces.keccak_rnds.len(), traces.ecsms.len(), traces.ecdases.len(), traces.hints.len(), + traces.ecrecs.len(), traces.is_whs.len(), traces.pages.len(), traces.memw_registers.len(), @@ -718,12 +998,14 @@ impl VmAirs { (self.bitwise.as_ref(), &mut traces.bitwise, &()), (self.decode.as_ref(), &mut traces.decode, &()), (self.keccak_rc.as_ref(), &mut traces.keccak_rc, &()), - (self.copy.as_ref(), &mut traces.copy, &()), (self.register.as_ref(), &mut traces.register, &()), ]; if self.include_halt { pairs.push((self.halt.as_ref(), &mut traces.halt, &())); } + for (air, trace) in self.copies.iter().zip(traces.copies.iter_mut()) { + pairs.push((air.as_ref(), trace, &())); + } for (air, trace) in self.keccaks.iter().zip(traces.keccaks.iter_mut()) { pairs.push((air.as_ref(), trace, &())); } @@ -739,6 +1021,21 @@ impl VmAirs { for (air, trace) in self.hints.iter().zip(traces.hints.iter_mut()) { pairs.push((air.as_ref(), trace, &())); } + for (air, trace) in self.ecrecs.iter().zip(traces.ecrecs.iter_mut()) { + pairs.push((air.as_ref(), trace, &())); + } + // ★ BLAKE3 closes the accelerator group rather than keeping the slot it + // held between KECCAK_RC and ECSM. That slot no longer exists: the six + // accelerators stopped being always-on tables in the leading block and + // became counted ones here, and BLAKE3 is the same kind of chip — 0 or 1 + // by `TableCounts::blake3`. Its position is a layout choice, and the only + // obligation is that `air_refs` below makes the identical one: those two + // orders ARE the proof's sub-proof layout. + if self.include_blake3 { + pairs.push((self.blake3.as_ref(), &mut traces.blake3, &())); + } + // IS_WH (0 or 1, main's range check for KECCAK's state addresses) after + // the accelerators, in both orders. for (air, trace) in self.is_whs.iter().zip(traces.is_whs.iter_mut()) { pairs.push((air.as_ref(), trace, &())); } @@ -800,18 +1097,44 @@ impl VmAirs { pairs } + /// Refuses a table set whose CPU tables are not where [`cpu_proof_start`] + /// reads their heights for `table_counts`: a verifier takes each CPU table's + /// height, and so its timestamp base and the final PC timestamp, at that + /// position, so the two must agree in every build, not only under + /// `debug_assert`. + pub(crate) fn check_cpu_position(&self, table_counts: &TableCounts) -> Result<(), Error> { + let refs = self.air_refs(); + let start = cpu_proof_start(self.include_halt, table_counts); + let in_place = self.cpus.len() == table_counts.cpu + && start + .checked_add(self.cpus.len()) + .is_some_and(|end| end <= refs.len()) + && self.cpus.iter().enumerate().all(|(k, cpu)| { + std::ptr::addr_eq(refs[start + k] as *const _, cpu.as_ref() as *const _) + }); + if in_place { + Ok(()) + } else { + Err(Error::InvalidTableCounts(format!( + "the CPU tables are not at sub-proof {start}, where their heights are read" + ))) + } + } + /// Collect AIR references for [`Verifier::multi_verify`]. pub fn air_refs(&self) -> Vec<&dyn AIR> { let mut refs: Vec<&dyn AIR> = vec![ self.bitwise.as_ref(), self.decode.as_ref(), self.keccak_rc.as_ref(), - self.copy.as_ref(), self.register.as_ref(), ]; if self.include_halt { refs.push(self.halt.as_ref()); } + for air in &self.copies { + refs.push(air.as_ref()); + } for air in &self.keccaks { refs.push(air.as_ref()); } @@ -827,6 +1150,15 @@ impl VmAirs { for air in &self.hints { refs.push(air.as_ref()); } + for air in &self.ecrecs { + refs.push(air.as_ref()); + } + // The same slot as `air_trace_pairs`: these two orders ARE the proof's + // layout, and must move together. + if self.include_blake3 { + refs.push(self.blake3.as_ref()); + } + // IS_WH after the accelerators that send to it, in both orders. for air in &self.is_whs { refs.push(air.as_ref()); } @@ -836,11 +1168,14 @@ impl VmAirs { cpu_start( self.include_halt, [ + self.copies.len(), self.keccaks.len(), self.keccak_rnds.len(), self.ecsms.len(), self.ecdases.len(), self.hints.len(), + self.ecrecs.len(), + usize::from(self.include_blake3), self.is_whs.len(), ], ), @@ -931,7 +1266,7 @@ impl VmAirs { decode_commitment: Option, include_halt: bool, page_commitments: Option<&[(u64, Commitment)]>, - register_commitment: Commitment, + register_preprocessed: RegisterPreprocessed<'_>, ) -> Self { assert_eq!( cpu_heights.len(), @@ -943,12 +1278,15 @@ impl VmAirs { .zip(tables::cpu::timestamp_bases(cpu_heights)) .enumerate() .map(|(i, (&height, ts_base))| { + // TIMESTAMP's column too, for the multilinear verifier (see the + // REGISTER note below); its root only on the univariate path. Box::new( create_cpu_air(proof_options, ts_base) .with_name(&format!("CPU[{}]", i)) - .with_preprocessed( - tables::cpu::timestamp_commitment(proof_options, height), + .with_lazy_preprocessed_columns( + tables::cpu::lazy_timestamp_commitment(proof_options, height), tables::cpu::NUM_PRECOMPUTED_COLS, + Arc::new(move || vec![tables::cpu::timestamp_column(height)]), ), ) as VmAir }) @@ -972,10 +1310,13 @@ impl VmAirs { // own preprocessed commitment first. Box::new(create_bitwise_air(proof_options)) } else { - Box::new(create_bitwise_air(proof_options).with_preprocessed( - bitwise::preprocessed_commitment(proof_options), - bitwise::NUM_PRECOMPUTED_COLS, - )) + Box::new( + create_bitwise_air(proof_options).with_lazy_preprocessed_columns( + bitwise::lazy_commitment(proof_options), + bitwise::NUM_PRECOMPUTED_COLS, + Arc::new(bitwise::preprocessed_columns), + ), + ) }; let lts: Vec<_> = (0..table_counts.lt) .map(|i| { @@ -1005,14 +1346,27 @@ impl VmAirs { Box::new(create_load_air(proof_options).with_name(&format!("LOAD[{}]", i))) as VmAir }) .collect(); - let decode_root = decode_commitment.unwrap_or_else(|| { - decode::commitment_from_elf(elf, proof_options) - .expect("Failed to compute decode commitment") - }); - let decode: VmAir = Box::new( - create_decode_air(proof_options) - .with_preprocessed(decode_root, decode::NUM_PRECOMPUTED_COLS), - ); + let decode: VmAir = { + // The instruction map, decoded once here rather than on every call: + // only the multilinear verifier asks, but it asks per proof. + let instructions = Arc::new( + decode::instructions_from_elf(elf).expect("the ELF decodes into instructions"), + ); + // Deferred: the commitment is an LDE and a Merkle tree over the + // program's whole instruction table, and only the univariate path + // compares it — the multilinear one checks the columns instead. + // Both leaf layouts (S2): the one-row root is computed on first + // use, never taken from a supplied row-pair root. + let decode_root = + decode::lazy_commitment(instructions.clone(), proof_options, decode_commitment); + Box::new( + create_decode_air(proof_options).with_lazy_preprocessed_columns( + decode_root, + decode::NUM_PRECOMPUTED_COLS, + Arc::new(move || decode::preprocessed_columns(&instructions)), + ), + ) + }; let muls: Vec<_> = (0..table_counts.mul) .map(|i| { Box::new(create_mul_air(proof_options).with_name(&format!("MUL[{}]", i))) as VmAir @@ -1030,6 +1384,11 @@ impl VmAirs { }) .collect(); let halt: VmAir = Box::new(create_halt_air(proof_options)); + let copies: Vec<_> = (0..table_counts.copy) + .map(|i| { + Box::new(create_copy_air(proof_options).with_name(&format!("COPY[{i}]"))) as VmAir + }) + .collect(); let keccaks: Vec<_> = (0..table_counts.keccak) .map(|i| { Box::new(create_keccak_air(proof_options).with_name(&format!("KECCAK[{i}]"))) @@ -1043,10 +1402,18 @@ impl VmAirs { ) as VmAir }) .collect(); - let keccak_rc: VmAir = Box::new(create_keccak_rc_air(proof_options).with_preprocessed( - tables::keccak_rc::preprocessed_commitment(proof_options), - tables::keccak_rc::NUM_PRECOMPUTED_COLS, - )); + let blake3: VmAir = Box::new(create_blake3_air(proof_options)); + // The columns, not just the commitment: the multilinear path checks the + // precomputed columns' claimed openings rather than comparing a root, so + // without the generator it cannot tell a real preprocessed table from a + // forged one. The univariate path ignores the extra argument. + let keccak_rc: VmAir = Box::new( + create_keccak_rc_air(proof_options).with_lazy_preprocessed_columns( + tables::keccak_rc::lazy_commitment(proof_options), + tables::keccak_rc::NUM_PRECOMPUTED_COLS, + Arc::new(tables::keccak_rc::preprocessed_columns), + ), + ); let ecsms: Vec<_> = (0..table_counts.ecsm) .map(|i| { Box::new(create_ecsm_air(proof_options).with_name(&format!("ECSM[{i}]"))) as VmAir @@ -1062,15 +1429,38 @@ impl VmAirs { Box::new(create_hint_air(proof_options).with_name(&format!("HINT[{i}]"))) as VmAir }) .collect(); - let copy: VmAir = Box::new(create_copy_air(proof_options)); + let ecrecs: Vec<_> = (0..table_counts.ecrec) + .map(|i| { + Box::new(create_ecrec_air(proof_options).with_name(&format!("ECREC[{i}]"))) as VmAir + }) + .collect(); let is_whs: Vec<_> = (0..table_counts.is_wh) .map(|i| { Box::new(create_is_wh_air(proof_options).with_name(&format!("IS_WH[{i}]"))) as VmAir }) .collect(); + // ⚠ THE COLUMNS, NOT ONLY THE COMMITMENT. `with_preprocessed` alone + // leaves `precomputed_columns()` empty, and the multilinear verifier + // checks exactly that list — an empty one in zero iterations. INIT and + // FINI (the epoch's `R_i` and `R_{i+1}`, or a monolithic proof's entry + // file and halted file) bind only through the columns on that path. + let RegisterPreprocessed { + commitment, + init, + fini, + } = register_preprocessed; + let root = register::lazy_commitment(proof_options, commitment, init, fini); + let (init, fini) = (init.to_vec(), fini.to_vec()); + // Named "REGISTER" whatever `final_pc_ts` is, as CPU tables are named by + // position: the block's routes, stream names and families read the name. let register: VmAir = Box::new( create_register_air(proof_options, final_pc_ts) - .with_preprocessed(register_commitment, register::NUM_PREPROCESSED_COLS), + .with_name("REGISTER") + .with_lazy_preprocessed_columns( + root, + register::NUM_PREPROCESSED_COLS, + Arc::new(move || register::preprocessed_columns(&init, &fini)), + ), ); // Every zero-init page shares one preprocessed commitment: OFFSET is // page-relative and INIT is all-zero, so it depends only on @@ -1099,28 +1489,38 @@ impl VmAirs { // Committing OFFSET alone publishes nothing: it is the dense // `0..page_size-1` enumeration, byte-identical for every page // regardless of program or input. - Box::new(air.with_preprocessed( - page::private_page_preprocessed_commitment(proof_options), + Box::new(air.with_lazy_preprocessed_columns( + page::private_page_lazy_commitment(proof_options), page::NUM_PREPROCESSED_COLS_PRIVATE, + Arc::new(|| vec![page::offset_column()]), )) } else if config.init_values.is_none() { // Zero-init pages: the shared commitment computed once above. - Box::new( - air.with_preprocessed(zero_init_commitment, page::NUM_PREPROCESSED_COLS), - ) + let config = config.clone(); + Box::new(air.with_lazy_preprocessed_columns( + page::zero_init_lazy_commitment_from(zero_init_commitment, proof_options), + page::NUM_PREPROCESSED_COLS, + Arc::new(move || page::preprocessed_columns(&config)), + )) } else { // ELF data pages: INIT is program-specific, so the commitment is // per-page. Prefer a caller-supplied `(page_base, commitment)` // (recursion guest); otherwise recompute from the ELF. - let commitment = page_commitments + // Deferred when it has to be computed: two dozen pages of + // LDE and Merkle that only the univariate path compares. + let supplied = page_commitments .unwrap_or(&[]) .iter() .find(|(pb, _)| *pb == config.page_base) - .map(|(_, c)| *c) - .unwrap_or_else(|| { - page::compute_precomputed_commitment(config, proof_options) - }); - Box::new(air.with_preprocessed(commitment, page::NUM_PREPROCESSED_COLS)) + .map(|(_, c)| *c); + let commitment = + page::data_page_lazy_commitment(config, proof_options, supplied); + let config = config.clone(); + Box::new(air.with_lazy_preprocessed_columns( + commitment, + page::NUM_PREPROCESSED_COLS, + Arc::new(move || page::preprocessed_columns(&config)), + )) } }) .collect(); @@ -1171,18 +1571,21 @@ impl VmAirs { dvrms, branches, halt, + copies, keccaks, keccak_rnds, keccak_rc, + blake3, ecsms, ecdases, hints, - copy, + ecrecs, is_whs, register, pages, memw_registers, include_halt, + include_blake3: table_counts.blake3 == 1, eqs, bytewises, stores, @@ -1252,20 +1655,39 @@ pub(crate) fn compute_commit_bus_offset( /// Replay the prover's Phase A (main trace commitments) to recover the shared /// LogUp challenges (z, alpha), over a proof view (owned or archived-in-place) /// — no `MultiProof` deserialization required either way. +/// +/// Generic over the transcript for the same reason as `absorb_lfm_statement`: +/// the replay is `append_bytes` plus `sample_field_element`, both on +/// `IsTranscript`, so it is the same replay under any sponge. +/// +/// ★ The preprocessed root absorbed is the one of the table's LEAF LAYOUT +/// (S2), resolved exactly as the STARK prover and verifier resolve it — +/// `stark::leaf_layout::table_leaf_layout(air, proof.trace_length())`, then +/// `air.precomputed_commitment_for(layout)` — because that is the root the +/// prover absorbed before sampling `z` and `α`. Absorbing the row-pair root +/// for a one-row table replays a different transcript: the recovered `z`, `α` +/// differ from the prover's, so every expected bus balance that depends on +/// them (the LFM public words, the VM commit bus) is wrong and an honest proof +/// is rejected. At the default format every layout is row pairs and this is +/// the row-pair root, byte for byte what was absorbed before. +/// +/// `None` = a preprocessed table has no root for its layout (never recomputed): the +/// caller rejects, exactly as the STARK verifier would. pub(crate) fn replay_transcript_phase_a_view<'p>( airs: &[&dyn AIR], proofs: impl ProofViewSource<'p, F, E, ()>, - transcript: &mut DefaultTranscript, -) -> (FieldElement, FieldElement) { + transcript: &mut impl IsTranscript, +) -> Option<(FieldElement, FieldElement)> { for (air, proof) in airs.iter().zip(proofs.view_iter()) { if air.is_preprocessed() { - transcript.append_bytes(&air.precomputed_commitment()); + let layout = stark::leaf_layout::table_leaf_layout(*air, proof.trace_length()); + transcript.append_bytes(&air.precomputed_commitment_for(layout)?); } transcript.append_bytes(proof.lde_trace_main_merkle_root()); } let z: FieldElement = transcript.sample_field_element(); let alpha: FieldElement = transcript.sample_field_element(); - (z, alpha) + Some((z, alpha)) } /// Computes the expected COMMIT bus balance for a proof view slice (owned or @@ -1275,9 +1697,12 @@ pub(crate) fn compute_expected_commit_bus_balance_view<'p>( proofs: impl ProofViewSource<'p, F, E, ()>, public_output_bytes: &[u8], start_index: u64, - transcript: &mut DefaultTranscript, + // Any transcript, not the byte one by name: the block path's transcript is + // whatever `hash_pin` pins, and on an algebraic branch that is a different + // TYPE rather than the same type over a different digest. + transcript: &mut impl crypto::fiat_shamir::is_transcript::IsTranscript, ) -> Option> { - let (z, alpha) = replay_transcript_phase_a_view(airs, proofs, transcript); + let (z, alpha) = replay_transcript_phase_a_view(airs, proofs, transcript)?; compute_commit_bus_offset(public_output_bytes, start_index, &z, &alpha) } @@ -1344,8 +1769,6 @@ pub fn count_elements(elf_bytes: &[u8], private_inputs: &[u8]) -> Result<(u64, u &result.logs, &MaxRowsConfig::default(), private_inputs, - #[cfg(feature = "disk-spill")] - StorageMode::Ram, )?; Ok(( traces.total_field_elements(), @@ -1410,20 +1833,7 @@ pub fn prove_with_options_and_inputs( #[cfg(feature = "instruments")] let __sp = stark::instruments::span("trace_build"); - #[cfg(feature = "disk-spill")] - let storage_mode = { - let lengths = count_table_lengths(&program, &result.logs, max_rows, private_inputs)?; - auto_storage::decide(&lengths, proof_options.blowup_factor) - }; - - let mut traces = Traces::from_elf_and_logs( - &program, - &result.logs, - max_rows, - private_inputs, - #[cfg(feature = "disk-spill")] - storage_mode, - )?; + let mut traces = Traces::from_elf_and_logs(&program, &result.logs, max_rows, private_inputs)?; debug_assert_eq!( traces.public_output_bytes, result.return_values.memory_values, "public output diverged between executor view and trace reconstruction" @@ -1458,7 +1868,7 @@ pub fn prove_with_options_and_inputs( None, true, None, - register::halted_commitment(proof_options, program.entry_point, committed_bytes), + RegisterFiles::halted(proof_options, program.entry_point, committed_bytes).preprocessed(), ); #[cfg(feature = "instruments")] @@ -1478,7 +1888,7 @@ pub fn prove_with_options_and_inputs( // Bind the full statement (program, public output, table layout) into the // Fiat-Shamir transcript so every challenge depends on it. - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -1493,11 +1903,15 @@ pub fn prove_with_options_and_inputs( // Phase 4: Prove (multi_prove) #[cfg(feature = "instruments")] let __sp = stark::instruments::span("proving"); - let proof = Prover::multi_prove( - airs.air_trace_pairs(&mut traces), + let pairs = airs.air_trace_pairs(&mut traces); + #[cfg(feature = "shape-profile")] + shape_profile::capture(pairs.iter().map(|(air, trace, _)| (*air, trace.num_rows()))); + // ★ The block path's PIN, not `stark`'s default alias — the monolithic + // production prove, the twin of the batched one in `continuation.rs`. + let proof = crate::hash_pin::BlockProver::multi_prove( + pairs, &mut transcript, - #[cfg(feature = "disk-spill")] - storage_mode, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| Error::Prover(format!("{e:?}")))?; #[cfg(feature = "instruments")] @@ -1712,8 +2126,10 @@ fn verify_proof_parts( decode_commitment, true, page_commitments, - register::halted_commitment(proof_options, program.entry_point, committed_bytes), + RegisterFiles::halted(proof_options, program.entry_point, committed_bytes).preprocessed(), ); + // The CPU heights were read at `cpu_proof_start`: the tables must be there. + airs.check_cpu_position(table_counts)?; // Recompute the COMMIT output bus offset from the public output. // If public_output was tampered, the recomputed offset won't match the @@ -1723,7 +2139,7 @@ fn verify_proof_parts( // Bind the statement into the verifier's transcript. A tampered statement // field makes this diverge from the prover's transcript state, so every // derived challenge differs and verification rejects. - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement_with_digest( &mut transcript, StatementKind::Monolithic, @@ -1754,7 +2170,7 @@ fn verify_proof_parts( stark::profile_markers::step_marker::<{ stark::profile_markers::STEP_AIRS_AND_BUS_BALANCE_DONE }>( ); - Ok(Verifier::multi_verify_views( + Ok(crate::hash_pin::BlockVerifier::multi_verify_views( &air_refs, proofs, &mut transcript, diff --git a/prover/src/multilinear_prove.rs b/prover/src/multilinear_prove.rs new file mode 100644 index 000000000..33c112843 --- /dev/null +++ b/prover/src/multilinear_prove.rs @@ -0,0 +1,631 @@ +//! The VM proved with the multilinear argument, end to end. +//! +//! Same executor, same traces and the same AIRs as [`crate::prove`]; what +//! changes is how each table is argued. Instead of a composition polynomial +//! committed by FRI, every table's own constraints and every one of its buses +//! are discharged in **one** sumcheck against one WHIR commitment for its whole +//! trace, and the LogUp argument is a fraction tree rather than auxiliary +//! columns. +//! +//! The verifier here never sees a trace. It rebuilds each table's +//! [`TableLayout`] from the AIR and the table's shape, which is the same +//! construction the prover ran, and takes the commitment roots from the proof. +//! +//! # How the preprocessed tables are bound +//! +//! BITWISE, DECODE, KECCAK_RC, REGISTER and the PAGE tables have +//! **preprocessed** columns — columns `0..num_precomputed_columns()`, fully +//! determined by the program and known to both sides. Nothing in the argument +//! itself pins them: a commitment says the prover stayed consistent with what +//! it committed, not that it committed the right thing, and a forged bitwise +//! table would otherwise prove forged lookups. +//! +//! The univariate path pins them with a second Merkle root +//! ([`AIR::precomputed_commitment`]) the verifier recomputes from the ELF. Here +//! there is no second root to compare, so the verifier does it one step later: +//! it rebuilds the columns ([`AIR::precomputed_columns`]) and evaluates its own +//! copy at the point the proof settled on, demanding the value the proof claims +//! for that column. One pass per column, which is what recomputing that root +//! costs anyway. **This is what binds a proof to its program** — DECODE's +//! preprocessed columns *are* the instruction table, and REGISTER's carry the +//! entry point. +//! +//! **The bus balance is not zero.** The COMMIT table sends the program's public +//! output on a bus whose counterparty is the statement rather than another +//! table, so what the tables must sum to is +//! [`compute_commit_bus_offset`](crate::compute_commit_bus_offset) — computed +//! against the very challenges the verifier draws, which means replaying the +//! transcript up to that point. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use executor::elf::Elf; +use executor::vm::execution::Executor; +use math::field::element::FieldElement; +use multilinear::mle::Mle; +use multilinear::whir_chain::ChainConfig; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{ + self, CommittedTable, CommittedTables, MultiProof, TableLayout, TableStatement, +}; +use stark::traits::AIR; + +use crate::statement::{self, MULTILINEAR_TAG}; +use crate::tables::trace_builder::Traces; +use crate::test_utils::{E, F}; +use crate::{Error, FIXED_TABLE_COUNT, MaxRowsConfig, ProofOptions, RuntimePageRange, TableCounts}; + +/// One table's height and main width — the shape the verifier needs to rebuild +/// its layout, and all it needs. +type Shape = (usize, usize); + +/// The multilinear counterpart of [`crate::VmProof`]. +/// +/// Carries the same statement metadata, plus every table's height. The +/// univariate path reads heights off its FRI domains; here they are stated, and +/// bound into the transcript so restating them changes every challenge. +#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct MultilinearVmProof { + /// Every table's argument and the one opening that settles all of them, + /// in [`VmAirs::air_refs`] order. + pub proof: MultiProof, + /// Each table's height in variables, same order. + pub table_num_vars: Vec, + pub runtime_page_ranges: Vec, + pub table_counts: TableCounts, + pub public_output: Vec, + pub num_private_input_pages: usize, +} + +/// The blowup, fold factor and grinding the whole VM proof runs at. +/// +/// The query count comes from the tallest stacked polynomial in the proof, so +/// one config covers every table: a taller stack means more rounds, and more +/// rounds is what the union bound charges for. +/// +/// ★ A PRODUCTION FORMAT SITE: the process's +/// [`ZfFormat`](crate::zf_format::ZfFormat) WHIR fields (`LAMBDA_VM_ZF_WHIR_CAP`, +/// `_WHIR_FOLDS`, `_WHIR_STACK`, `_WHIR_GRIND`, `_WHIR_GRIND_BITS`) are stamped on +/// here. Unset knobs give [`ZfFormat::DEFAULT`](crate::zf_format::ZfFormat::DEFAULT)'s +/// WHIR fields (`whir_cap=auto`, `whir_folds=first6`, `whir_stack=27`, +/// `whir_grind=query`, `whir_grind_bits=18`); the five knobs at their off +/// spellings give the legacy config. The stack reaches every layout +/// through this config ([`stacks`], `CommittedTables::commit_grouped`), on the +/// prover's side and the verifier's alike. +pub fn chain_config(shapes: &[Shape]) -> ChainConfig { + chain_config_under(crate::zf_format::ZfFormat::global(), shapes) +} + +/// [`chain_config`] under an explicit format, so a test can build a knob-on +/// production config without setting the environment. +/// +/// The query count is charged the fold schedule's worst round count +/// (`with_security_folds`): the schedule is part of the security accounting, +/// not a label stamped on afterwards. +/// +/// ★ AND THE TALLEST STACKED POLYNOMIAL. A chain runs over a stacked +/// polynomial, and a group stacks many tables into one: its height can exceed +/// every table's own `one_stack` (narrow tables filling one polynomial). So the +/// height charged is the larger of the widest table's `one_stack` and the +/// height all the shapes would stack to together under the format's cap, which +/// bounds every group's from above ([`multilinear_table::stack_height`] is +/// monotone). At the default format (first6, blowup 2, 18 query-grind bits) Q +/// is 229 at every height of 23 or more, 228 from 15 to 22, 227 from 8 to 14, +/// 226 at 7 and 224 at 6 (`the_query_count_by_tallest_stack_at_the_default`). +/// On a block the charge does not move Q: BITWISE (21 columns of 2^20 rows) +/// stacks to 25, and the frame holds it to that height +/// (`block_whir::check_bitwise_height`). Q moves only for a statement whose +/// tallest stack is under 23, which only small test programs make. +pub fn chain_config_under(format: &crate::zf_format::ZfFormat, shapes: &[Shape]) -> ChainConfig { + let widest_table = shapes + .iter() + .map(|&(width, num_vars)| multilinear::constraint_argument::one_stack(num_vars, width)) + .max() + .unwrap_or(1); + let tallest = widest_table.max(multilinear_table::stack_height(shapes, format.whir_stack)); + // The grind's bits come from the format with its nonce layout: P2's query + // grind alone by default, all three under `LAMBDA_VM_ZF_WHIR_GRIND=all`. Q + // reads the query grind only, so it is the same under both placements; at + // the default 18 bits it is two above `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20`'s, + // to buy back the bits. + // The rate is the format's (`LAMBDA_VM_ZF_WHIR_BLOWUP`, blowup 2 by + // default), and Q follows from it by the same rule. + let config = ChainConfig::with_security_folds( + format.whir_log_blowup(), + crate::zf_format::PRODUCTION_WHIR_LOG_FOLDING, + format.whir_folds, + tallest, + 128, + format.whir_grind_bits(), + ); + format.chain(config) +} + +/// Binds the statement into the transcript before any challenge is drawn. +/// +/// The univariate encoding, under a tag of its own — a WHIR proof and a FRI +/// proof must never share a transcript prefix — followed by what only this path +/// states: the table heights and the parameters the argument runs at. +/// +/// ⚠ Padded like the continuation statements, and for the same reason: the +/// roots absorbed next have to start on a field element boundary, and two +/// variable-length fields (`public_output`, `table_num_vars`) sit between them +/// and the fixed prefix. `runtime_page_ranges` is sixteen bytes an entry and +/// does not move the alignment. +#[allow(clippy::too_many_arguments)] +pub(crate) fn absorb( + t: &mut impl crypto::fiat_shamir::is_transcript::IsTranscript, + elf_digest: &[u8; 32], + public_output: &[u8], + table_counts: &TableCounts, + num_private_input_pages: usize, + runtime_page_ranges: &[RuntimePageRange], + table_num_vars: &[u8], + config: &ChainConfig, +) { + absorb_tagged( + t, + MULTILINEAR_TAG, + "monolithic", + elf_digest, + public_output, + table_counts, + num_private_input_pages, + runtime_page_ranges, + table_num_vars, + config, + ); +} + +/// [`absorb`] under `tag` — the block prover's statement is this one under a +/// tag of its own (`crate::block_whir`), so a block proof and a monolithic one +/// never share a transcript prefix. `kind` names it in the padding diagnostic. +#[allow(clippy::too_many_arguments)] +pub(crate) fn absorb_tagged( + t: &mut impl crypto::fiat_shamir::is_transcript::IsTranscript, + tag: &[u8], + kind: &str, + elf_digest: &[u8; 32], + public_output: &[u8], + table_counts: &TableCounts, + num_private_input_pages: usize, + runtime_page_ranges: &[RuntimePageRange], + table_num_vars: &[u8], + config: &ChainConfig, +) { + let mut len = 0usize; + + t.append_bytes(tag); + len += tag.len(); + t.append_bytes(elf_digest); + len += elf_digest.len(); + + t.append_bytes(&(public_output.len() as u64).to_le_bytes()); + len += size_of::(); + t.append_bytes(public_output); + len += public_output.len(); + + len += statement::absorb_table_counts(t, table_counts); + t.append_bytes(&(num_private_input_pages as u64).to_le_bytes()); + len += size_of::(); + + t.append_bytes(&(runtime_page_ranges.len() as u64).to_le_bytes()); + len += size_of::(); + for r in runtime_page_ranges { + let &RuntimePageRange { base, count } = r; + t.append_bytes(&base.to_le_bytes()); + len += size_of_val(&base); + t.append_bytes(&count.to_le_bytes()); + len += size_of_val(&count); + } + + // Every table's height. A prover who shrank a table would have to state the + // smaller height here, which moves every challenge. + t.append_bytes(&(table_num_vars.len() as u64).to_le_bytes()); + len += size_of::(); + t.append_bytes(table_num_vars); + len += table_num_vars.len(); + + // The parameters the argument runs at: derived from the heights above, but + // absorbed rather than assumed, so the two sides agree in the transcript and + // not only in the code. + let &ChainConfig { + log_blowup, + // ★ Absorbed as `fold_word()`: `log_folding` itself (4u64) at the + // default fold schedule, a tagged word binding the schedule otherwise. + log_folding: _, + num_queries, + grind, + // ⚠ Format, NOT absorbed except the fold schedule, through the word + // above: verifier-side constants (see `lfm::whir_statement::push_config`, + // the emitter's twin of this). The nonce layout among them; the grind + // bits it follows from are the trailer below. + format: _, + } = config; + for value in [log_blowup as u64, config.fold_word(), num_queries as u64] { + t.append_bytes(&value.to_le_bytes()); + len += size_of_val(&value); + } + let trailer = [grind.folding, grind.ood, grind.query]; + t.append_bytes(&trailer); + len += trailer.len(); + + statement::absorb_statement_padding( + t, + kind, + len, + &[ + ("public_output", public_output.len()), + ("runtime_page_ranges", runtime_page_ranges.len()), + ("table_num_vars", table_num_vars.len()), + ], + ); +} + +/// Every table's `(main width, height in variables)`, checked to be what the +/// AIR says and a power of two tall. +pub(crate) fn shapes_of(pairs: &[crate::AirTracePair<'_>]) -> Result, Error> { + pairs + .iter() + .map(|(air, trace, _)| { + // The table's own shape, not its columns: transposing the trace to + // count it is the whole trace copied for two numbers. + let width = trace.main_table.width; + let rows = trace.main_table.height; + if width == 0 || !rows.is_power_of_two() { + return Err(Error::Prover(format!( + "{}: {width} columns of {rows} rows, which the hypercube cannot hold", + air.name(), + ))); + } + if width != air.trace_layout().0 { + return Err(Error::Prover(format!( + "{}: trace has {width} main columns, the AIR declares {}", + air.name(), + air.trace_layout().0, + ))); + } + Ok((width, rows.trailing_zeros() as usize)) + }) + .collect() +} + +/// Proves an ELF the multilinear way. +pub fn prove_with_options_and_inputs( + elf_bytes: &[u8], + private_inputs: &[u8], + proof_options: &ProofOptions, + max_rows: &MaxRowsConfig, +) -> Result { + let program = Elf::load(elf_bytes).map_err(|e| Error::ElfLoad(format!("{e}")))?; + let result = Executor::new(&program, private_inputs.to_vec()) + .map_err(|e| Error::Execution(format!("{e}")))? + .run() + .map_err(|e| Error::Execution(format!("{e}")))?; + + let mut traces = Traces::from_elf_and_logs(&program, &result.logs, max_rows, private_inputs)?; + drop(result); + + let table_counts = traces.table_counts(); + let runtime_page_ranges = traces.runtime_page_ranges(); + let num_private_input_pages = traces + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(); + let public_output = traces.public_output_bytes.clone(); + + let inputs = crate::BlockAirInputs::new( + proof_options, + &program, + crate::cpu_heights_from_traces(&traces), + &public_output, + )?; + let airs = crate::block_airs( + &program, + proof_options, + &traces.page_configs, + &table_counts, + &inputs, + ); + + let mut pairs = airs.air_trace_pairs(&mut traces); + let shapes = shapes_of(&pairs)?; + let table_num_vars: Vec = shapes.iter().map(|&(_, n)| n as u8).collect(); + let config = chain_config(&shapes); + + // Commit every table against the layout the verifier will rebuild. + let mut committed = Vec::with_capacity(pairs.len()); + for ((air, trace, _), &(width, num_vars)) in pairs.iter_mut().zip(&shapes) { + let layout = layout_of(*air, width, num_vars) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name())))?; + let mut columns = trace.columns_main(); + // The verifier will rebuild these and demand the proof open to them, so a + // trace that disagrees produces a proof nobody can verify. Better to say + // so here than to hand out that proof. + for (col, expected) in air.precomputed_columns().iter().enumerate() { + if columns.get(col) != Some(expected) { + return Err(Error::Prover(format!( + "{}: preprocessed column {col} is not what the program implies", + air.name(), + ))); + } + } + committed.push( + // Moved, not cloned: `columns` is this iteration's own transpose + // of the trace and nothing reads it afterwards. + CommittedTable::from_layout(layout, |col| core::mem::take(&mut columns[col as usize])) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name())))?, + ); + } + + // One commitment for every table in the proof: the opening is nearly all of + // a proof's bytes, and one settles them all. + let proof = crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb( + &mut transcript, + &statement::elf_digest(elf_bytes), + &public_output, + &table_counts, + num_private_input_pages, + &runtime_page_ranges, + &table_num_vars, + &config, + ); + let committed = CommittedTables::<_, _, H>::commit(committed, &config) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + multilinear_table::multi_prove(&committed, &config, &mut transcript, None) + .map_err(|e| Error::Prover(format!("{e:?}")))? + }); + + Ok(MultilinearVmProof { + proof, + table_num_vars, + runtime_page_ranges, + table_counts, + public_output, + num_private_input_pages, + }) +} + +/// [`prove_with_options_and_inputs`] with no private input. +pub fn prove_with_options( + elf_bytes: &[u8], + proof_options: &ProofOptions, + max_rows: &MaxRowsConfig, +) -> Result { + prove_with_options_and_inputs(elf_bytes, &[], proof_options, max_rows) +} + +/// One table's layout, from the AIR and its shape alone. Both sides call this. +pub(crate) fn layout_of<'a>( + air: &'a dyn AIR, + width: usize, + num_vars: usize, +) -> Result, multilinear::Error> { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + width, + num_vars, + Uniforms::default(), + ) +} + +/// Each commitment group's stack and the domain it is committed over, rebuilt +/// from the shapes and the split alone — the verifier never takes either from +/// the proof. +pub(crate) fn stacks( + shapes: &[Shape], + sizes: &[usize], + config: &ChainConfig, +) -> Result< + ( + Vec, + Vec>, + ), + Error, +> { + let layouts = multilinear_table::global_layouts(shapes, sizes, config.format.stack) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + let domains = layouts + .iter() + .map(|layout| { + multilinear::whir::Domain::::new(layout.n_stack() + config.log_blowup) + .map_err(|e| Error::Prover(format!("{e:?}"))) + }) + .collect::>()?; + Ok((layouts, domains)) +} + +/// A table's preprocessed columns as MLEs, empty for a table that has none. +pub(crate) fn preprocessed_mles( + air: &dyn AIR, +) -> Result>, Error> { + air.precomputed_columns() + .into_iter() + .map(|values| Mle::new(values).map_err(|e| Error::Prover(format!("{}: {e:?}", air.name())))) + .collect() +} + +/// Verifies a proof from [`prove_with_options_and_inputs`]. +pub fn verify_with_options( + proof: &MultilinearVmProof, + elf_bytes: &[u8], + proof_options: &ProofOptions, +) -> Result { + let program = Elf::load(elf_bytes).map_err(|e| Error::ElfLoad(format!("{e}")))?; + + // A prover choosing the counts chooses the constraint sets, so they are + // checked before an AIR is built off them. + proof.table_counts.validate()?; + let max_pages = crate::tables::page::max_private_input_pages(); + if proof.num_private_input_pages > max_pages { + return Err(Error::InvalidTableCounts(format!( + "num_private_input_pages ({}) exceeds max ({max_pages})", + proof.num_private_input_pages, + ))); + } + + let page_configs = Traces::page_configs_from_elf_and_runtime( + &program, + &proof.runtime_page_ranges, + proof.num_private_input_pages, + proof.proof.tables.len(), + )?; + + // Checked: the counts are prover-supplied and release builds wrap on + // overflow, so a plain sum would let one astronomically large field still + // match the sub-proof count and reach `VmAirs::new`, which sizes a `Vec` + // from that field directly. + let Some(expected) = proof + .table_counts + .total() + .and_then(|t| t.checked_add(FIXED_TABLE_COUNT)) + .and_then(|t| t.checked_add(page_configs.len())) + else { + return Err(Error::InvalidTableCounts( + "the table counts do not sum to a usize".to_string(), + )); + }; + if expected != proof.proof.tables.len() { + return Err(Error::InvalidTableCounts(format!( + "table_counts total ({}) + {FIXED_TABLE_COUNT} fixed + {} pages = {expected}, but the proof carries {} tables", + expected - FIXED_TABLE_COUNT - page_configs.len(), + page_configs.len(), + proof.proof.tables.len(), + ))); + } + if proof.table_num_vars.len() != proof.proof.tables.len() { + return Err(Error::InvalidTableCounts(format!( + "the proof carries {} tables but {} heights", + proof.proof.tables.len(), + proof.table_num_vars.len(), + ))); + } + + // The CPU heights the proof states fix the CPU timestamps and the final PC + // timestamp; REGISTER's FINI is the halted file of the public output. + let cpu_start = crate::cpu_proof_start(true, &proof.table_counts); + let cpu_heights: Vec = cpu_start + .checked_add(proof.table_counts.cpu) + .and_then(|end| proof.table_num_vars.get(cpu_start..end)) + .ok_or_else(|| { + Error::InvalidTableCounts("the proof states fewer heights than CPU tables".into()) + })? + .iter() + .map(|&vars| 1usize << vars.min(63)) + .collect(); + let inputs = + crate::BlockAirInputs::new(proof_options, &program, cpu_heights, &proof.public_output)?; + let airs = crate::block_airs( + &program, + proof_options, + &page_configs, + &proof.table_counts, + &inputs, + ); + // The CPU heights were read at `cpu_proof_start`: the tables must be there. + airs.check_cpu_position(&proof.table_counts)?; + let air_refs = airs.air_refs(); + if air_refs.len() != proof.proof.tables.len() { + return Err(Error::InvalidTableCounts(format!( + "the layout has {} tables, the proof carries {}", + air_refs.len(), + proof.proof.tables.len(), + ))); + } + + // The width is the AIR's, never the proof's; only the height is stated. + let shapes: Vec = air_refs + .iter() + .zip(&proof.table_num_vars) + .map(|(air, &num_vars)| (air.trace_layout().0, num_vars as usize)) + .collect(); + let config = chain_config(&shapes); + + let layouts: Vec> = air_refs + .iter() + .zip(&shapes) + .map(|(air, &(width, num_vars))| { + layout_of(*air, width, num_vars) + .map_err(|e| Error::Prover(format!("{}: {e:?}", air.name()))) + }) + .collect::>()?; + // The preprocessed columns, rebuilt from the ELF. Nothing else in the + // argument says a preprocessed table is the one the program implies: the + // commitment binds the prover to what it committed, not to the right thing. + let preprocessed: Vec>> = air_refs + .iter() + .map(|air| preprocessed_mles(*air)) + .collect::>()?; + let statements: Vec> = layouts + .iter() + .zip(&preprocessed) + .map(|(layout, cols)| layout.statement_with_preprocessed(cols)) + .collect(); + + // The stack every table's columns share, rebuilt from the shapes alone. A + // monolithic proof commits them all together, so there is one group. + let sizes = [shapes.len()]; + let (layouts, domains) = stacks(&shapes, &sizes, &config)?; + + Ok(crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb( + &mut transcript, + &statement::elf_digest(elf_bytes), + &proof.public_output, + &proof.table_counts, + proof.num_private_input_pages, + &proof.runtime_page_ranges, + &proof.table_num_vars, + &config, + ); + // What the tables owe: the COMMIT bus's counterparty is the statement, + // and its offset depends on the very challenges `multi_verify` is about + // to draw — so the transcript is replayed to that point on a fork. + // + // ★ Inside the dispatch with the transcript it forks. Those challenges + // are a function of the configuration's sponge, so computing them + // against a transcript of a different hash is the same defect one level + // down, and just as quiet. + // + // ★ The block is CALLED, not re-spelled: a replay is a third side of the + // roots-block agreement, and the epoch path's `owed` broke precisely by + // being a second spelling that did not grow a list the block had grown. + // This path opens nothing out of band, so `derived` is empty and not one + // byte of any transcript moves; what changes is that it cannot drift. + let mut probe = transcript.clone(); + multilinear_table::absorb_roots::(&mut probe, &proof.proof.roots, &[]); + let z: FieldElement = probe.sample_field_element(); + let alpha: FieldElement = probe.sample_field_element(); + // `start_index` is the carried x254: zero for a monolithic proof. + let Some(owed) = crate::compute_commit_bus_offset(&proof.public_output, 0, &z, &alpha) + else { + return Ok(false); + }; + multilinear_table::multi_verify::<_, _, _, H>( + &proof.proof, + &statements, + &layouts, + &domains, + &sizes, + &owed, + &config, + &mut transcript, + None, + ) + .is_ok() + })) +} diff --git a/prover/src/paged_mem.rs b/prover/src/paged_mem.rs index 978dba9e3..42abefbc6 100644 --- a/prover/src/paged_mem.rs +++ b/prover/src/paged_mem.rs @@ -61,6 +61,26 @@ impl PagedMem { (addr & !mask, (addr & mask) as usize) } + /// What a cell never `set` reads. + #[inline] + pub fn fill(&self) -> T { + self.fill + } + + /// Keeps only the pages whose base `keep` accepts; the rest are dropped, + /// their cells reading `fill` again. Moves page indices. + pub fn retain_pages(&mut self, mut keep: impl FnMut(u64) -> bool) { + self.pages.retain(|(base, _)| keep(*base)); + } + + /// Each allocated page, to be written: distinct pages can be written at + /// once, from several threads. + pub fn pages_mut(&mut self) -> impl Iterator> { + self.pages + .iter_mut() + .map(|(base, page)| PageMut { base: *base, page }) + } + /// Value at `addr`, or `fill` if never `set`. #[inline] pub fn get(&self, addr: u64) -> T { @@ -75,7 +95,28 @@ impl PagedMem { #[inline] pub fn set(&mut self, addr: u64, val: T) { let (base, off) = Self::split(addr); - let i = match self.pages.binary_search_by_key(&base, |(b, _)| *b) { + let i = self.page_index_or_insert(base); + self.page_set(i, off, val); + } + + /// `addr`'s page base and its offset in the page. + #[inline] + pub fn page_of(addr: u64) -> (u64, usize) { + Self::split(addr) + } + + /// The index of the page at `base` (page-aligned), if it is allocated. An + /// index stays valid until a page is allocated. + #[inline] + pub fn page_index(&self, base: u64) -> Option { + self.pages.binary_search_by_key(&base, |(b, _)| *b).ok() + } + + /// The index of the page at `base` (page-aligned), allocating it (filled) + /// on first touch, which moves the index of every page above it. + #[inline] + pub fn page_index_or_insert(&mut self, base: u64) -> usize { + match self.pages.binary_search_by_key(&base, |(b, _)| *b) { Ok(i) => i, Err(i) => { self.pages.insert( @@ -90,12 +131,46 @@ impl PagedMem { ); i } - }; - let page = &mut self.pages[i].1; + } + } + + /// The value at offset `off` of page `index`. + #[inline] + pub fn page_get(&self, index: usize, off: usize) -> T { + self.pages[index].1.data[off] + } + + /// Sets offset `off` of page `index` to `val`, as [`Self::set`] does. + #[inline] + pub fn page_set(&mut self, index: usize, off: usize, val: T) { + let page = &mut self.pages[index].1; page.data[off] = val; page.occupied[off / WORD_BITS] |= 1u64 << (off % WORD_BITS); } + /// The store with each cell's value mapped through `f`, page by page: the + /// same pages and the same `set` cells, each offset holding `f` of what it + /// held. `f(fill)` should be `fill` (the new store's), so that an offset + /// never `set` reads alike in an allocated page and outside every page. + pub fn map(&self, fill: U, f: impl Fn(T) -> U) -> PagedMem { + PagedMem { + pages: self + .pages + .iter() + .map(|(base, page)| { + ( + *base, + Page { + data: page.data.iter().map(|&value| f(value)).collect(), + occupied: page.occupied.clone(), + }, + ) + }) + .collect(), + fill, + } + } + /// Base addresses of the pages that hold at least one `set` cell, ascending. /// (For a `DEFAULT_PAGE_SIZE`-aligned page, this equals `page_base_for_address` /// of every cell in it, so it replaces `cells.keys().map(page_base)`.) @@ -122,6 +197,13 @@ impl PagedMem { } } + /// The bytes its pages take on the heap. + pub fn heap_bytes(&self) -> usize { + self.pages.capacity() * std::mem::size_of::<(u64, Page)>() + + self.pages.len() + * (DEFAULT_PAGE_SIZE * std::mem::size_of::() + OCC_WORDS * size_of::()) + } + /// Number of cells that were explicitly `set`. pub fn len(&self) -> usize { self.pages @@ -159,6 +241,27 @@ impl PagedMem { } } +/// One allocated page of a [`PagedMem`], to be written +/// ([`PagedMem::pages_mut`]). +pub struct PageMut<'a, T> { + base: u64, + page: &'a mut Page, +} + +impl PageMut<'_, T> { + /// The page's base address. + pub fn base(&self) -> u64 { + self.base + } + + /// Sets offset `off` to `val`, as [`PagedMem::page_set`] does. + #[inline] + pub fn set(&mut self, off: usize, val: T) { + self.page.data[off] = val; + self.page.occupied[off / WORD_BITS] |= 1u64 << (off % WORD_BITS); + } +} + /// A read-only initial-memory image: `addr -> byte`, with an iterator over the /// bytes it holds. Implemented for both `HashMap` (the monolithic /// prover's image) and [`PagedMem`] (the continuation's carried image), so @@ -207,3 +310,50 @@ impl Iterator for BitIter { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// `map` keeps every page and every `set` cell, maps each offset's value, + /// and reads unset cells as the new fill inside and outside the pages: the + /// store `set` cell by cell with the mapped values. + #[test] + fn a_mapped_store_is_the_store_set_with_the_mapped_values() { + let page = DEFAULT_PAGE_SIZE as u64; + let mut bytes = PagedMem::new(0u8); + let cells = [ + (0x10, 7u8), + (page - 1, 0), + (page, 255), + (5 * page + 3, 1), + (u64::MAX, 9), + ]; + for (addr, value) in cells { + bytes.set(addr, value); + } + let mapped = bytes.map((0u8, 0u64), |value| (value, 0)); + let mut set = PagedMem::new((0u8, 0u64)); + for (addr, value) in cells { + set.set(addr, (value, 0)); + } + assert_eq!( + mapped.page_bases().collect::>(), + set.page_bases().collect::>() + ); + for base in set.page_bases() { + assert_eq!( + mapped.page_data(base), + set.page_data(base), + "page {base:#x}" + ); + } + assert_eq!( + mapped.iter().collect::>(), + set.iter().collect::>() + ); + assert_eq!(mapped.len(), cells.len()); + assert_eq!(mapped.get(page + 1), (0, 0)); + assert_eq!(mapped.get(3 * page), (0, 0)); + } +} diff --git a/prover/src/recursion.rs b/prover/src/recursion.rs index efca722c9..ab7065a36 100644 --- a/prover/src/recursion.rs +++ b/prover/src/recursion.rs @@ -42,6 +42,9 @@ pub const MIN_PROOF_OPTIONS: ProofOptions = ProofOptions { coset_offset: 3, grinding_factor: 1, fri_final_poly_log_degree: 7, + // The RV64 guest verifies the LEGACY format, named here rather + // than inherited from a default. + format: stark::proof::options::ProofFormat::LEGACY, }; /// The recursion verifier's build presets. Each fixes the guest's @@ -73,8 +76,14 @@ impl Preset { ]; /// The fixed `ProofOptions` this preset's guest verifies with. + /// + /// ★ Always the LEGACY proof format ([`ProofFormat::LEGACY`](stark::proof::options::ProofFormat::LEGACY)), + /// stamped explicitly: the RV64 guest's archived verifier is + /// not threaded with the ZF format levers, so its presets name the format + /// it was built for instead of inheriting the process's production format + /// ([`crate::zf_format::ZfFormat::DEFAULT`]). pub fn options(&self) -> ProofOptions { - match self { + let mut options = match self { Preset::Min => MIN_PROOF_OPTIONS, Preset::Blowup2 => crate::GoldilocksCubicProofOptions::with_blowup(2) .expect("blowup=2 is always valid"), @@ -82,7 +91,9 @@ impl Preset { .expect("blowup=4 is always valid"), Preset::Blowup8 => crate::GoldilocksCubicProofOptions::with_blowup(8) .expect("blowup=8 is always valid"), - } + }; + options.format = stark::proof::options::ProofFormat::LEGACY; + options } /// Artifact stem under `executor/program_artifacts/recursion/` @@ -194,7 +205,12 @@ pub fn encode_continuation_guest_input( } /// Domain tag for [`program_id`]. -const PROGRAM_ID_TAG: &[u8] = b"LAMBDAVM_PROGRAM_ID_V1"; +/// Domain tag for the attestation's program id. +/// +/// `pub(crate)` so the LFM emitter binds the same literal instead of +/// duplicating it — the precedent `CONTINUATION_EPOCH_TAG` set in R1e. 22 bytes, +/// so it is `≡ 2 (mod 4)` and every machine value folded after it is spliced. +pub(crate) const PROGRAM_ID_TAG: &[u8] = b"LAMBDAVM_PROGRAM_ID_V1"; /// [`program_id`] from a precomputed ELF digest and entry point — the guest /// path, sharing one full-ELF Keccak pass with the verify-side statement @@ -260,6 +276,23 @@ pub fn program_id_from_elf( )) } +/// The RV64 recursion guest verifies the LEGACY proof format only: its +/// presets fix the options at build time and +/// name the legacy format, and the archived verifier it runs is not threaded +/// with the ZF format levers. Any other format — including the production +/// default [`crate::zf_format::ZfFormat::DEFAULT`] — must never reach it, so +/// both guest entry points refuse one up front instead of verifying a proof +/// under a format the guest was not built for. +fn require_legacy_format(proof_options: &ProofOptions) -> Result<(), Error> { + if proof_options.has_legacy_format() { + Ok(()) + } else { + Err(Error::Execution(String::from( + "the recursion guest verifies legacy-format proofs only (every ZF format lever off)", + ))) + } +} + /// Verify the guest's private-input blob ([`encode_guest_input`]) in place and, /// on success, produce the attestation bytes the recursion guest commits: /// `program_id(elf, roots) || inner_public_output`. `Ok(None)` means the @@ -274,6 +307,7 @@ pub fn verify_and_attest_blob( blob: &[u8], proof_options: &ProofOptions, ) -> Result>, Error> { + require_legacy_format(proof_options)?; let verification = crate::verify_recursion_blob(blob, proof_options)?; if !verification.ok { return Ok(None); @@ -309,6 +343,8 @@ pub fn verify_continuation_and_attest( ) -> Result>, Error> { use rkyv::rancor::Error as RkyvError; + require_legacy_format(proof_options)?; + let archive_bytes = crate::recursion_archive_bytes(blob).ok_or_else(|| { Error::Execution(String::from( "continuation recursion blob: bad magic or version", diff --git a/prover/src/shape_profile.rs b/prover/src/shape_profile.rs new file mode 100644 index 000000000..ba4147db3 --- /dev/null +++ b/prover/src/shape_profile.rs @@ -0,0 +1,529 @@ +//! Emits a per-proof trace-shape profile as JSONL — heights, widths, constraint +//! and interaction counts, never trace values. One line per proof; a +//! continuation epoch maps onto one line. +//! +//! Four deliberate mismatches with the consumer's model, none of which affect +//! the prover-cost dimensions being measured: `count_weight` is emitted as 1; +//! bus ids are remapped to a dense `u16` range; message lengths drop our leading +//! bus-id element; and preprocessed columns are reported in `preprocessed`, with +//! `cached_mains` always empty. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use stark::lookup::BusInteraction; +use stark::traits::AIR; + +/// Schema version of the emitted JSONL. Matches what the consumer expects. +const SCHEMA: &str = "v2"; + +/// Per-AIR column widths, split the way the consumer's keygen splits them. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ProfileTraceWidth { + pub preprocessed: Option, + pub cached_mains: Vec, + pub common_main: usize, + pub after_challenge: Vec, +} + +/// Shape of one AIR within a segment. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AirShapeRecord { + pub air_name: String, + pub air_id: usize, + pub log_height: usize, + pub height: usize, + pub width: ProfileTraceWidth, + pub num_constraints: usize, + pub num_interactions: usize, + pub max_constraint_degree: usize, + pub buses: Vec, + pub interaction_message_lens: Vec, + pub interaction_count_weights: Vec, +} + +/// One proof's worth of AIR shapes — one JSONL line. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SegmentProfile { + pub schema: String, + pub segment_idx: usize, + pub global_max_constraint_degree: usize, + pub airs: Vec, +} + +/// Assigns dense `u16` bus indices to our sparse `u64` bus ids. +/// +/// Shared across a run so an id keeps its index across segments; interactions +/// on the same bus must land on the same index for the replayed LogUp to have +/// the same bus structure. +#[derive(Default)] +pub struct BusIndexMap { + map: HashMap, +} + +impl BusIndexMap { + pub fn new() -> Self { + Self::default() + } + + fn index_of(&mut self, bus_id: u64) -> u16 { + let next = self.map.len(); + *self.map.entry(bus_id).or_insert_with(|| { + u16::try_from(next).expect("bus count exceeds u16 — the bus index is u16") + }) + } +} + +/// The shape inputs one AIR contributes, decoupled from the [`AIR`] trait so +/// the mapping below is testable without standing up a full AIR. +pub struct AirShapeInput<'a> { + pub name: &'a str, + pub height: usize, + pub common_main: usize, + pub after_challenge_width: usize, + pub num_precomputed: usize, + pub num_constraints: usize, + pub max_constraint_degree: usize, + pub interactions: &'a [BusInteraction], +} + +impl<'a> AirShapeInput<'a> { + /// Reads every shape dimension off a live AIR. + pub fn from_air(air: &'a A, height: usize) -> Self + where + A: AIR + ?Sized, + { + let (common_main, after_challenge_width) = air.trace_layout(); + Self { + name: air.name(), + height, + common_main, + after_challenge_width, + num_precomputed: air.num_precomputed_columns(), + num_constraints: air.num_transition_constraints(), + max_constraint_degree: air.max_constraint_degree(), + interactions: air.bus_interactions(), + } + } +} + +/// Builds the shape record for one AIR. +fn record_for(input: &AirShapeInput<'_>, air_id: usize, buses: &mut BusIndexMap) -> AirShapeRecord { + let preprocessed = match input.num_precomputed { + 0 => None, + n => Some(n), + }; + let after_challenge = match input.after_challenge_width { + 0 => Vec::new(), + n => vec![n], + }; + + let mut bus_indices = Vec::with_capacity(input.interactions.len()); + let mut message_lens = Vec::with_capacity(input.interactions.len()); + for interaction in input.interactions { + bus_indices.push(buses.index_of(interaction.bus_id)); + // Ours counts the bus id as element 0; theirs counts message fields only. + message_lens.push(interaction.num_bus_elements().saturating_sub(1)); + } + + AirShapeRecord { + air_name: input.name.to_string(), + air_id, + log_height: input.height.max(1).trailing_zeros() as usize, + height: input.height, + width: ProfileTraceWidth { + preprocessed, + cached_mains: Vec::new(), + common_main: input.common_main, + after_challenge, + }, + num_constraints: input.num_constraints, + num_interactions: input.interactions.len(), + max_constraint_degree: input.max_constraint_degree, + buses: bus_indices, + interaction_message_lens: message_lens, + interaction_count_weights: vec![1; input.interactions.len()], + } +} + +/// Builds a segment profile from per-AIR shapes in proving order. +pub fn build_segment<'a>( + segment_idx: usize, + inputs: impl IntoIterator>, + buses: &mut BusIndexMap, +) -> SegmentProfile { + let airs: Vec = inputs + .into_iter() + .enumerate() + .map(|(air_id, input)| record_for(&input, air_id, buses)) + .collect(); + + let global_max_constraint_degree = airs + .iter() + .map(|a| a.max_constraint_degree) + .max() + .unwrap_or(0); + + SegmentProfile { + schema: SCHEMA.to_string(), + segment_idx, + global_max_constraint_degree, + airs, + } +} + +/// Scales every AIR's `common_main` to estimate the width the same table would +/// need over a 31-bit field, where our 64-bit-field limbs no longer fit. +/// +/// The result is an estimate for a second, pessimistic replay — it is not a +/// claim about any real BabyBear arithmetization. Emit it as a separate profile +/// and compare the two runs separately. +pub fn scale_widths(profile: &SegmentProfile, factor_for: impl Fn(&str) -> f64) -> SegmentProfile { + let mut scaled = profile.clone(); + for air in &mut scaled.airs { + let factor = factor_for(&air.air_name); + air.width.common_main = ((air.width.common_main as f64) * factor).ceil() as usize; + if let Some(pre) = air.width.preprocessed.as_mut() { + *pre = ((*pre as f64) * factor).ceil() as usize; + } + } + scaled +} + +/// Appends one segment profile as a JSONL line. +pub fn append_jsonl(path: &std::path::Path, profile: &SegmentProfile) -> std::io::Result<()> { + use std::io::Write; + + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent)?; + } + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + let line = serde_json::to_string(profile) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + writeln!(file, "{line}") +} + +/// Environment variable holding the output path. Unset means no capture. +pub const PROFILE_PATH_ENV: &str = "LAMBDA_VM_SHAPE_PROFILE"; + +/// Bus indices and the segment counter, shared by every capture in a process so +/// a bus id keeps one index across segments and each proof gets its own line. +static CAPTURE_STATE: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// Appends one line for this proof, if `LAMBDA_VM_SHAPE_PROFILE` is set. +/// +/// A capture failure is logged and ignored: profiling must never fail a proof. +pub fn capture<'a, A>(airs: impl IntoIterator) +where + A: AIR + ?Sized + 'a, +{ + let Ok(path) = std::env::var(PROFILE_PATH_ENV) else { + return; + }; + + let inputs: Vec<(&'a A, usize)> = airs.into_iter().collect(); + let mut guard = match CAPTURE_STATE.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + let state = guard.get_or_insert_with(|| (BusIndexMap::new(), 0)); + let (buses, next_idx) = state; + + let segment_idx = *next_idx; + *next_idx += 1; + + let profile = build_segment( + segment_idx, + inputs + .iter() + .map(|(air, height)| AirShapeInput::from_air(*air, *height)), + buses, + ); + drop(guard); + + if let Err(e) = append_jsonl(std::path::Path::new(&path), &profile) { + log::warn!("shape profile: could not append to {path}: {e}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use stark::lookup::{BusValue, Multiplicity}; + + fn interaction(bus_id: u64, num_values: usize) -> BusInteraction { + BusInteraction::sender( + bus_id, + Multiplicity::One, + (0..num_values).map(BusValue::column).collect(), + ) + } + + fn shape<'a>( + name: &'a str, + height: usize, + interactions: &'a [BusInteraction], + ) -> AirShapeInput<'a> { + AirShapeInput { + name, + height, + common_main: 64, + after_challenge_width: 3, + num_precomputed: 0, + num_constraints: 12, + max_constraint_degree: 3, + interactions, + } + } + + #[test] + fn per_interaction_vectors_agree_with_num_interactions() { + let ints = vec![interaction(7, 4), interaction(9, 2)]; + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [shape("CPU", 1 << 19, &ints)], &mut buses); + + let rec = &profile.airs[0]; + assert_eq!(rec.num_interactions, 2); + assert_eq!(rec.buses.len(), rec.num_interactions); + assert_eq!(rec.interaction_message_lens.len(), rec.num_interactions); + assert_eq!(rec.interaction_count_weights.len(), rec.num_interactions); + } + + #[test] + fn message_len_drops_our_leading_bus_id_element() { + // `num_bus_elements()` counts the bus id plus one element per value. + let ints = vec![interaction(7, 4)]; + assert_eq!(ints[0].num_bus_elements(), 5); + + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [shape("CPU", 1 << 19, &ints)], &mut buses); + + assert_eq!(profile.airs[0].interaction_message_lens, vec![4]); + } + + #[test] + fn height_and_log_height_agree() { + let ints = Vec::new(); + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [shape("MUL", 1 << 20, &ints)], &mut buses); + + let rec = &profile.airs[0]; + assert_eq!(rec.height, 1 << 20); + assert_eq!(rec.log_height, 20); + assert_eq!(1usize << rec.log_height, rec.height); + } + + #[test] + fn single_row_table_has_log_height_zero() { + let ints = Vec::new(); + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [shape("HALT", 1, &ints)], &mut buses); + + assert_eq!(profile.airs[0].log_height, 0); + assert_eq!(profile.airs[0].height, 1); + } + + #[test] + fn same_bus_id_keeps_one_index_across_airs_and_segments() { + let a_ints = vec![interaction(7, 1), interaction(9, 1)]; + let b_ints = vec![interaction(9, 1), interaction(7, 1)]; + let mut buses = BusIndexMap::new(); + + let first = build_segment( + 0, + [ + shape("CPU", 1 << 19, &a_ints), + shape("MEMW", 1 << 19, &b_ints), + ], + &mut buses, + ); + let second = build_segment(1, [shape("MEMW", 1 << 19, &b_ints)], &mut buses); + + // Each bus id gets exactly one index, and the order within an AIR is the + // declaration order, not the discovery order. + assert_eq!(first.airs[0].buses, vec![0, 1]); + assert_eq!(first.airs[1].buses, vec![1, 0]); + assert_eq!(second.airs[0].buses, vec![1, 0]); + } + + #[test] + fn air_id_follows_proving_order() { + let ints = Vec::new(); + let mut buses = BusIndexMap::new(); + let profile = build_segment( + 0, + [ + shape("BITWISE", 1 << 20, &ints), + shape("DECODE", 1 << 21, &ints), + shape("CPU", 1 << 19, &ints), + ], + &mut buses, + ); + + let ids: Vec = profile.airs.iter().map(|a| a.air_id).collect(); + assert_eq!(ids, vec![0, 1, 2]); + assert_eq!(profile.airs[1].air_name, "DECODE"); + } + + #[test] + fn widths_split_preprocessed_and_after_challenge() { + let ints = Vec::new(); + let mut input = shape("BITWISE", 1 << 20, &ints); + input.num_precomputed = 3; + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [input], &mut buses); + + let w = &profile.airs[0].width; + assert_eq!(w.preprocessed, Some(3)); + assert_eq!(w.common_main, 64); + assert_eq!(w.after_challenge, vec![3]); + assert!(w.cached_mains.is_empty()); + } + + #[test] + fn no_aux_columns_emits_empty_after_challenge() { + let ints = Vec::new(); + let mut input = shape("HALT", 1, &ints); + input.after_challenge_width = 0; + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [input], &mut buses); + + assert!(profile.airs[0].width.after_challenge.is_empty()); + assert_eq!(profile.airs[0].width.preprocessed, None); + } + + #[test] + fn global_degree_is_the_max_over_airs() { + let ints = Vec::new(); + let mut a = shape("CPU", 1 << 19, &ints); + a.max_constraint_degree = 3; + let mut b = shape("MUL", 1 << 20, &ints); + b.max_constraint_degree = 5; + let mut buses = BusIndexMap::new(); + + let profile = build_segment(0, [a, b], &mut buses); + assert_eq!(profile.global_max_constraint_degree, 5); + } + + #[test] + fn width_scaling_applies_per_air_and_leaves_the_original() { + let ints = Vec::new(); + let mut buses = BusIndexMap::new(); + let profile = build_segment( + 0, + [shape("MUL", 1 << 20, &ints), shape("EQ", 1 << 20, &ints)], + &mut buses, + ); + + let scaled = scale_widths(&profile, |name| if name == "MUL" { 2.0 } else { 1.0 }); + assert_eq!(scaled.airs[0].width.common_main, 128); + assert_eq!(scaled.airs[1].width.common_main, 64); + assert_eq!(profile.airs[0].width.common_main, 64); + } + + #[test] + fn width_scaling_rounds_up() { + let ints = Vec::new(); + let mut input = shape("LT", 1 << 20, &ints); + input.common_main = 41; + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [input], &mut buses); + + let scaled = scale_widths(&profile, |_| 1.5); + // 41 * 1.5 = 61.5 — a fractional column is still a whole column. + assert_eq!(scaled.airs[0].width.common_main, 62); + } + + #[test] + fn serialized_line_carries_every_consumer_field() { + let ints = vec![interaction(7, 4)]; + let mut buses = BusIndexMap::new(); + let profile = build_segment(0, [shape("CPU", 1 << 19, &ints)], &mut buses); + + let json = serde_json::to_string(&profile).unwrap(); + for key in [ + "\"schema\"", + "\"segment_idx\"", + "\"global_max_constraint_degree\"", + "\"air_name\"", + "\"air_id\"", + "\"log_height\"", + "\"height\"", + "\"preprocessed\"", + "\"cached_mains\"", + "\"common_main\"", + "\"after_challenge\"", + "\"num_constraints\"", + "\"num_interactions\"", + "\"max_constraint_degree\"", + "\"buses\"", + "\"interaction_message_lens\"", + "\"interaction_count_weights\"", + ] { + assert!(json.contains(key), "missing {key} in {json}"); + } + assert!(json.contains("\"schema\":\"v2\"")); + } + + /// A line of the consumer's own captured profile must deserialize into our + /// mirror of the schema — this is what pins the two definitions together. + #[test] + fn parses_a_line_from_a_real_capture() { + let line = r#"{"schema":"v2","segment_idx":0,"global_max_constraint_degree":4,"airs":[{"air_name":"VmConnectorAir","air_id":1,"log_height":1,"height":2,"width":{"preprocessed":null,"cached_mains":[],"common_main":6,"after_challenge":[]},"num_constraints":8,"num_interactions":5,"max_constraint_degree":3,"buses":[0,0,2,3,3],"interaction_message_lens":[2,2,9,2,2],"interaction_count_weights":[1,1,1,1,1]}]}"#; + + let parsed: SegmentProfile = serde_json::from_str(line).unwrap(); + assert_eq!(parsed.schema, SCHEMA); + assert_eq!(parsed.global_max_constraint_degree, 4); + + let air = &parsed.airs[0]; + assert_eq!(air.air_name, "VmConnectorAir"); + assert_eq!(air.num_interactions, 5); + assert_eq!(air.buses.len(), air.num_interactions); + assert_eq!(air.interaction_message_lens, vec![2, 2, 9, 2, 2]); + assert_eq!(air.width.preprocessed, None); + assert!(air.width.cached_mains.is_empty()); + } + + #[test] + fn round_trips_through_the_consumer_shape() { + let ints = vec![interaction(7, 4), interaction(9, 2)]; + let mut buses = BusIndexMap::new(); + let profile = build_segment(3, [shape("CPU", 1 << 19, &ints)], &mut buses); + + let json = serde_json::to_string(&profile).unwrap(); + let back: SegmentProfile = serde_json::from_str(&json).unwrap(); + + assert_eq!(back.segment_idx, 3); + assert_eq!(back.airs[0].buses, profile.airs[0].buses); + assert_eq!( + back.airs[0].interaction_message_lens, + profile.airs[0].interaction_message_lens + ); + } + + #[test] + fn appends_one_line_per_segment() { + let dir = std::env::temp_dir().join("lambda_vm_shape_profile_test"); + let _ = std::fs::remove_dir_all(&dir); + let path = dir.join("profile.jsonl"); + + let ints = Vec::new(); + let mut buses = BusIndexMap::new(); + for idx in 0..3 { + let profile = build_segment(idx, [shape("CPU", 1 << 19, &ints)], &mut buses); + append_jsonl(&path, &profile).unwrap(); + } + + let contents = std::fs::read_to_string(&path).unwrap(); + let lines: Vec<&str> = contents.lines().collect(); + assert_eq!(lines.len(), 3); + for (idx, line) in lines.iter().enumerate() { + let parsed: SegmentProfile = serde_json::from_str(line).unwrap(); + assert_eq!(parsed.segment_idx, idx); + } + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/prover/src/statement.rs b/prover/src/statement.rs index bb94817c2..3e8a0fb76 100644 --- a/prover/src/statement.rs +++ b/prover/src/statement.rs @@ -17,7 +17,25 @@ use crate::test_utils::E; use crate::{RuntimePageRange, TableCounts}; /// Domain-separation tag. Bump the suffix (`_V2`, ...) on any encoding change. -const DOMAIN_TAG: &[u8] = b"LAMBDAVM_STARK_STATEMENT_V5"; +/// V4 appended `TableCounts::blake3`, which made the BLAKE3 table conditional. +/// [`CONTINUATION_EPOCH_TAG`] moved with it and for the same reason: the count +/// list below is SHARED, so a continuation epoch absorbs the new u64 too. +/// Bumping only the monolithic tag would have left continuation proofs from two +/// encodings sharing a transcript prefix. +/// +/// ⚠ V5 is a MERGE OF TWO INDEPENDENT V4s, and that is why it exists. The +/// per-table lineage reached V4 by appending `blake3`; main reached its own V4 +/// by appending the six accelerator counts. Two different encodings were both +/// called V4, so keeping either suffix would have let proofs from the two +/// branches share a transcript prefix — the exact thing this tag prevents. The +/// merged encoding appends all seven counts and is V5. +/// +/// V6 appends `TableCounts::is_wh` (main's IS_WH table). Main calls its own +/// append of `is_wh` V5, on an encoding without `blake3`: a different encoding +/// again, so this one takes the next number neither line has used. +/// +/// V7 appends `TableCounts::ecrec` (the ECDSA-recovery table). +pub(crate) const DOMAIN_TAG: &[u8] = b"LAMBDAVM_STARK_STATEMENT_V7"; /// Canonical full-ELF identity digest — exactly what [`absorb_statement`] binds /// into the transcript. The recursion attestation folds the same digest into @@ -68,6 +86,13 @@ pub(crate) fn absorb_statement( /// hold the digest reuse it instead of a second full-ELF Keccak pass — the /// recursion attestation path shares one digest between the transcript absorb /// and the `program_id` fold (a full-ELF hash is expensive in-guest). +/// +/// ⛔ **This statement is NOT padded to a field element boundary**, unlike the +/// three WHIR ones — see [`absorb_statement_padding`]. Its transcript is +/// keccak, which absorbs a byte stream and never re-slices it into field +/// elements, so there is no straddling value to prevent; and its bytes are the +/// univariate pipeline's, which the block identity lines pin. Padding here +/// would move a record for nothing. #[allow(clippy::too_many_arguments)] pub(crate) fn absorb_statement_with_digest( t: &mut impl IsTranscript, @@ -94,55 +119,13 @@ pub(crate) fn absorb_statement_with_digest( t.append_bytes(&(public_output.len() as u64).to_le_bytes()); t.append_bytes(public_output); - // table_counts: fixed-width u64s in declared order. The exhaustive - // destructure makes any field added to TableCounts a compile error here — - // that's the signal to extend the loop below and bump DOMAIN_TAG. - let &TableCounts { - cpu, - lt, - memw, - memw_aligned, - load, - mul, - dvrm, - shift, - branch, - memw_register, - eq, - bytewise, - store, - cpu32, - keccak, - keccak_rnd, - ecsm, - ecdas, - hint, - is_wh, - } = table_counts; - for count in [ - cpu, - lt, - memw, - memw_aligned, - load, - mul, - dvrm, - shift, - branch, - memw_register, - eq, - bytewise, - store, - cpu32, - keccak, - keccak_rnd, - ecsm, - ecdas, - hint, - is_wh, - ] { - t.append_bytes(&(count as u64).to_le_bytes()); - } + // The width is for callers that pad from it; this one does not. + // + // ⚠ The exhaustive destructure that turns a new `TableCounts` field into a + // compile error lives in [`table_count_values`], not here. It is the same + // tripwire one level down, and it is what the WHIR statements inherit by + // calling the same helper. + let _ = absorb_table_counts(t, table_counts); t.append_bytes(&(num_private_input_pages as u64).to_le_bytes()); @@ -187,10 +170,219 @@ pub(crate) fn absorb_statement_with_digest( } } +/// Bytes per field element, which is the granularity the algebraic transcript +/// slices its absorbed buffer at (`rpx::sponge_leaf_bytes`). +pub(crate) const FELT_BYTES: usize = 8; + +/// Zeroes for [`absorb_statement_padding`]. A pad is at most `FELT_BYTES - 1`. +const PAD_ZEROS: [u8; FELT_BYTES - 1] = [0u8; FELT_BYTES - 1]; + +/// Zero bytes that bring a statement of `len` bytes up to a multiple of +/// [`FELT_BYTES`]. +pub(crate) const fn statement_padding(len: usize) -> usize { + (FELT_BYTES - len % FELT_BYTES) % FELT_BYTES +} + +/// Closes a WHIR statement so that whatever is absorbed next starts on a field +/// element boundary, and returns the number of bytes it added. +/// +/// # Why a statement is padded at all +/// +/// The transcript hashes BYTES, and the algebraic configuration's sponge +/// re-slices everything absorbed since the last squeeze into field elements +/// every [`FELT_BYTES`] bytes (`crypto::hash::rpx::sponge_leaf_bytes`). A value +/// absorbed at an offset that is not a multiple of 8 therefore STRADDLES two +/// field elements, and a field-machine verifier replaying the transcript has to +/// bit-decompose it to reproduce the absorb. +/// +/// Every window after the first is already aligned: a squeeze leaves the buffer +/// holding its own 32-byte output, and everything absorbed afterwards — roots +/// 32, extension elements 24, grind nonces 8, final values 24 — is a multiple of +/// 8. The FIRST window is the exception, because it opens with the statement, +/// and the roots that follow it land wherever the statement ended. +/// +/// # Why the pad is COMPUTED and not a constant +/// +/// A statement's roots do not sit at the end of its fixed prefix: two +/// variable-length fields sit in between (the public output and the per-table +/// heights). Padding the fixed prefix to a multiple of 8 would leave the roots +/// at `(|public_output| + |table_num_vars|) mod 8` — 2 mod 8 at the shape this +/// system runs — so it would align nothing while moving every pinned constant. +/// The length is accumulated beside the absorbs that produce it, and the pad +/// follows from that length. +/// +/// # Why it is called even when the pad is empty +/// +/// So that "one padding absorb per statement" holds for every shape. The +/// transcript counts an empty `append_bytes` as an absorb, so an absorb count +/// stays a function of the statement's FIELDS rather than of its lengths, and +/// the pinned pair moves by exactly one per statement instead of by a number +/// nobody can predict without the shapes. +pub(crate) fn absorb_statement_padding( + t: &mut impl IsTranscript, + kind: &str, + len: usize, + shape: &[(&str, usize)], +) -> usize { + #[cfg(not(feature = "hash-metrics"))] + let _ = (kind, shape); + + let pad = statement_padding(len); + t.append_bytes(&PAD_ZEROS[..pad]); + // ★ The statement is over, and the transcript is told so. + // + // `Counts::transcript_misaligned_absorbs_after_statement` reports what + // happens from here on, and nothing else can define that boundary: a + // transcript sees a byte stream and a statement is not a type. Compiles to + // nothing without `hash-metrics`. + // + // ⚠ It is called AFTER the pad, not before, and the difference is the whole + // check: the pad is what makes the next absorb aligned, so a mark placed + // before it would count the pad's own absorb and a mark that reset the + // window would report zero whether the pad worked or not. + t.mark_statement_end(); + + // A diagnostic, not a gate: it prints every variable length the pad is a + // function of beside the pad itself, so a run that reports a total number of + // padding bytes can be checked against the shapes that produced it instead + // of against a premise nobody measured. + #[cfg(feature = "hash-metrics")] + { + // ⚠ Its own label, not `WHIR`: the bench's per-arm lines already start + // with that, and a box launcher counting `WHIR` lines would silently + // pick these up as well. + let mut line = format!("{:<12} statement {kind}", "WHIR-PAD"); + for (name, value) in shape { + line.push_str(&format!(" {name}={value}")); + } + println!("{line} len={len} pad={pad}"); + } + + pad +} + +/// How many per-table counts a statement binds — one `u64` each. +/// +/// ★ Read, never written as a literal by a caller. The transcript pin's +/// expected absorb counts are `base + epochs * NUM_TABLE_KINDS`, because the +/// per-table campaign added a count to this list (`TableCounts::blake3`) and a +/// pin carrying the total would be a constant describing one branch while +/// claiming to describe the protocol. +/// +/// Two compile errors guard it together, and neither alone is enough: the +/// exhaustive destructure in [`table_count_values`] fails when a field is added +/// to [`TableCounts`], and that function's return type fails when the new field +/// is pushed into the array without bumping this constant. +pub(crate) const NUM_TABLE_KINDS: usize = 23; + +/// Every per-table count, in declared order. +/// +/// The exhaustive destructure makes any field added to [`TableCounts`] a +/// compile error here — that's the signal to extend the array and bump the +/// domain tag of every statement that absorbs it. +pub(crate) fn table_count_values(table_counts: &TableCounts) -> [u64; NUM_TABLE_KINDS] { + let &TableCounts { + cpu, + lt, + memw, + memw_aligned, + load, + mul, + dvrm, + shift, + branch, + memw_register, + eq, + bytewise, + store, + cpu32, + keccak, + keccak_rnd, + ecsm, + ecdas, + hint, + copy, + blake3, + is_wh, + ecrec, + } = table_counts; + [ + cpu as u64, + lt as u64, + memw as u64, + memw_aligned as u64, + load as u64, + mul as u64, + dvrm as u64, + shift as u64, + branch as u64, + memw_register as u64, + eq as u64, + bytewise as u64, + store as u64, + cpu32 as u64, + // The six accelerator chips. Counted rather than fixed, so an epoch that + // never reaches one carries no table for it; each is 0 or 1 today and + // `TableCounts::validate` refuses anything higher. + keccak as u64, + keccak_rnd as u64, + ecsm as u64, + ecdas as u64, + hint as u64, + copy as u64, + // 0 or 1, and the one count the verifier cannot derive for itself — + // binding it is what stops prover and verifier building different AIR + // sets from the same bytes (see `TableCounts::blake3`). + blake3 as u64, + // IS_WH: 0 or 1, the range check for KECCAK's state addresses (main's + // #1022); its bound is what bounds IS_WH's multiplicity sum. + is_wh as u64, + // ECREC: 0 or 1, the ECDSA public-key recovery table. + ecrec as u64, + ] +} + +/// The table layout, as fixed-width u64s in declared order. +/// +/// Returns the number of BYTES it absorbed, so a caller accumulating a +/// statement's length does not have to know — or track — how many counts there +/// are. A caller that does not need the length (the univariate path) ignores it. +#[must_use] +pub(crate) fn absorb_table_counts( + t: &mut impl IsTranscript, + table_counts: &TableCounts, +) -> usize { + let counts = table_count_values(table_counts); + for count in counts { + t.append_bytes(&count.to_le_bytes()); + } + counts.len() * size_of::() +} + +/// Domain tag for the multilinear path. A WHIR proof and a FRI proof must +/// never share a transcript prefix. V3: the count list gained `is_wh`. +pub(crate) const MULTILINEAR_TAG: &[u8] = b"LAMBDAVM_MULTILINEAR_STATEMENT_V3"; + +/// The block prover's statement tag (`crate::block_whir`): the monolithic +/// multilinear statement, proved in groups on forks of one transcript. Distinct +/// from [`MULTILINEAR_TAG`] so a block proof is never read as a monolithic one. +/// V2: the count list gained `is_wh`. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) const MULTILINEAR_BLOCK_TAG: &[u8] = b"LAMBDAVM_MULTILINEAR_BLOCK_STATEMENT_V2"; + /// Continuation domain tags. Distinct from the monolithic `DOMAIN_TAG` so a /// monolithic proof and a continuation proof can never share a transcript prefix. -const CONTINUATION_EPOCH_TAG: &[u8] = b"LAMBDAVM_CONTINUATION_EPOCH_V5"; -const CONTINUATION_GLOBAL_TAG: &[u8] = b"LAMBDAVM_CONTINUATION_GLOBAL_V2"; +/// `pub(crate)` so the LFM statement replay emits the identical tag instead of +/// duplicating the literal: a second copy would drift silently on a version +/// bump, and the tag existing at all depends on both sides agreeing on it. +/// +/// ⚠ V5 for the reason [`DOMAIN_TAG`] is: the lineage's V3 and main's V4 are +/// different encodings of the same statement, and the merged one appends both +/// additions. V6 with it: the shared count list gained `is_wh`, V7 `ecrec`. The GLOBAL tag +/// stays at V2 — the cross-epoch statement binds no per-table counts, so +/// nothing in its encoding moved. +pub(crate) const CONTINUATION_EPOCH_TAG: &[u8] = b"LAMBDAVM_CONTINUATION_EPOCH_V7"; +pub(crate) const CONTINUATION_GLOBAL_TAG: &[u8] = b"LAMBDAVM_CONTINUATION_GLOBAL_V2"; /// Statement bound into the cross-epoch **global** proof's transcript before /// Phase A: the ELF (so the global proof is program-bound), the epoch count (so a diff --git a/prover/src/tables/bitwise.rs b/prover/src/tables/bitwise.rs index 8970daeaa..9bdf907bb 100644 --- a/prover/src/tables/bitwise.rs +++ b/prover/src/tables/bitwise.rs @@ -26,8 +26,8 @@ //! meaning other tables send to this table. use math::polynomial::Polynomial; -use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed}; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; use stark::proof::options::ProofOptions; use stark::prover::evaluate_polynomial_on_lde_domain; @@ -36,6 +36,9 @@ use stark::trace::TraceTable; #[cfg(feature = "parallel")] use rayon::prelude::*; +use math::field::element::FieldElement; +use math::field::traits::IsField; + use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; // ========================================================================= @@ -195,58 +198,78 @@ pub const fn is_preprocessed() -> bool { fn static_commitment(blowup_factor: u8) -> Option { match blowup_factor { 2 => Some([ - 0xfa, 0x3e, 0xcf, 0x80, 0xfd, 0x95, 0xe5, 0x09, 0x74, 0xd4, 0x55, 0x23, 0xf6, 0x42, - 0xb6, 0x4b, 0x05, 0xc4, 0xf9, 0x66, 0xc2, 0x4d, 0xff, 0xda, 0x31, 0x47, 0xab, 0x7b, - 0x0c, 0x6d, 0xc4, 0xcf, + 0x46, 0x8d, 0xfe, 0x65, 0xed, 0x50, 0x25, 0xbf, 0x7e, 0xfa, 0x04, 0x11, 0x28, 0x36, + 0x29, 0x06, 0xbd, 0x97, 0x29, 0xeb, 0x5b, 0x3a, 0xae, 0xbf, 0x66, 0xea, 0x5e, 0x59, + 0xe4, 0xd7, 0x17, 0x6c, ]), 4 => Some([ - 0xff, 0x76, 0x8e, 0x85, 0x4b, 0xdc, 0x32, 0x61, 0x96, 0x16, 0x15, 0x19, 0x73, 0x70, - 0xf0, 0x64, 0x81, 0xfd, 0x4f, 0x5c, 0xbd, 0x9c, 0x30, 0x26, 0xd5, 0xc0, 0x81, 0xf3, - 0xce, 0x38, 0x50, 0x3e, + 0x88, 0x7b, 0x04, 0x68, 0xdd, 0x29, 0xf5, 0x0a, 0x87, 0x05, 0xcc, 0xdb, 0x48, 0xce, + 0x38, 0x11, 0xf5, 0x49, 0x2f, 0xe2, 0x73, 0x15, 0x03, 0xa7, 0xb9, 0x86, 0x04, 0x60, + 0xcc, 0x28, 0xa5, 0x9b, ]), 8 => Some([ - 0x0e, 0x1b, 0xc1, 0x0d, 0xae, 0x64, 0xe7, 0xca, 0xe0, 0x2a, 0x3b, 0xab, 0xd7, 0xd2, - 0xbb, 0x80, 0xd5, 0x24, 0x5a, 0xce, 0x25, 0xb6, 0x84, 0x77, 0x9c, 0xb5, 0xeb, 0x67, - 0x61, 0x82, 0x78, 0x3d, + 0x7d, 0x85, 0xb2, 0x07, 0x0e, 0xdb, 0x9c, 0x89, 0xd1, 0x91, 0xda, 0x78, 0xe7, 0x11, + 0x13, 0x77, 0xe3, 0x1e, 0xe9, 0xbe, 0x3e, 0x3d, 0xd4, 0x26, 0x86, 0xbb, 0x4a, 0xe9, + 0x2c, 0x51, 0x1e, 0x44, ]), _ => None, } } -/// Computes the Merkle commitment over the precomputed bitwise table columns. -/// -/// This builds a Merkle tree over the LDE (Low Degree Extension) of the precomputed -/// columns, matching exactly how the prover commits to traces. The tree has -/// NUM_ROWS * blowup_factor leaves, enabling FRI queries at any index -/// in the extended domain. -/// -/// Critical for security: the commitment must be over LDE values (not raw values) -/// because FRI queries can target any index in [0, N*blowup). A raw-value commitment -/// would only have N leaves, unable to verify queries at indices >= N. +/// The ONE-ROW (S2) twin of [`static_commitment`]: the same columns committed +/// with one LDE row per leaf ([`LeafLayout::Row`]), per `blowup_factor` in +/// [`crate::tables::STATIC_BLOWUP_FACTORS_ONE_ROW`], generated by +/// `compute_static_commitments --layout row` and pinned by the one-row drift +/// test. The same regeneration rules as [`static_commitment`]. A blowup with +/// no arm here is a hard miss under one row: no recompute. +pub(crate) fn static_commitment_one_row(blowup_factor: u8) -> Option { + match blowup_factor { + 4 => Some([ + 0x34, 0x22, 0x21, 0x59, 0xc3, 0xe7, 0x92, 0x18, 0xb5, 0xf0, 0x3b, 0xd8, 0x73, 0x37, + 0xd8, 0x33, 0x62, 0xbb, 0xea, 0xe4, 0x1f, 0x1e, 0x0a, 0x15, 0x59, 0x19, 0x8c, 0xf2, + 0x13, 0xd4, 0x82, 0x09, + ]), + _ => None, + } +} + +/// The POSEIDON1 twin of [`static_commitment`]: the same columns in row pairs, +/// committed by a Poseidon1 configuration (4-ary trees of ZisK's leaf hash; a +/// configuration's cap moves path nodes, never a root), per `blowup_factor` +/// in [`crate::tables::STATIC_BLOWUP_FACTORS_P1`] at coset 3. Generated by +/// `compute_static_commitments --hash p1` and pinned by +/// `bitwise_p1_static_matches_recompute`. The same regeneration rules as +/// [`static_commitment`]. A blowup with no arm here is computed, as every +/// other Poseidon1 key is ([`crate::hash_pin::static_root_under`]). +pub(crate) fn static_commitment_p1(blowup_factor: u8) -> Option { + match blowup_factor { + 4 => Some([ + 0xd4, 0x82, 0x02, 0x4a, 0xaa, 0x5f, 0x16, 0x8e, 0x6c, 0x62, 0xf7, 0xb8, 0x3b, 0x95, + 0xe0, 0x38, 0xb5, 0x4c, 0xac, 0x75, 0x3b, 0x02, 0x3d, 0xa0, 0x08, 0xef, 0xcc, 0xa6, + 0xfe, 0x34, 0x4a, 0x21, + ]), + _ => None, + } +} + +/// The precomputed columns themselves, one per column, `NUM_ROWS` tall. /// -/// Exposed for the `compute_static_commitments` binary and the -/// drift-detection tests in `static_commitments_tests`. Production callers -/// should go through [`preprocessed_commitment`] so the static const-table -/// shortcut is used when applicable. -#[doc(hidden)] -pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { - // Step 1: Generate precomputed columns in parallel - // Each column is generated independently by iterating over all row indices +/// The multilinear path checks a proof's claimed openings against these instead +/// of comparing a commitment, so it needs the values and not just their root. +pub fn preprocessed_columns() -> Vec> { + // Each column is generated independently by iterating over all row indices. #[cfg(feature = "parallel")] - let columns: Vec> = (0..NUM_PRECOMPUTED_COLS) + return (0..NUM_PRECOMPUTED_COLS) .into_par_iter() .map(|col_idx| { (0..NUM_ROWS) - .map(|idx| { - let row = generate_bitwise_row(idx); - FE::from(row[col_idx]) - }) + .map(|idx| FE::from(generate_bitwise_row(idx)[col_idx])) .collect() }) .collect(); #[cfg(not(feature = "parallel"))] - let columns: Vec> = { + { let mut cols: Vec> = (0..NUM_PRECOMPUTED_COLS) .map(|_| Vec::with_capacity(NUM_ROWS)) .collect(); @@ -257,9 +280,178 @@ pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { } } cols + } +} + +// ========================================================================= +// The precomputed columns' multilinear extensions, in closed form +// ========================================================================= + +/// Variables in the precomputed table's hypercube: NUM_ROWS is 2^NUM_VARS. +pub const NUM_VARS: usize = 20; +const _: () = assert!(1usize << NUM_VARS == NUM_ROWS); + +/// ★ Every precomputed column's multilinear extension, evaluated at one point +/// in O(NUM_VARS) field operations instead of a 2^20 fold. +/// +/// # Why a closed form exists at all +/// +/// [`generate_bitwise_row`] reads its row index as three disjoint bit fields, +/// index = X + 256*Y + 65536*Z, and every column it returns is a MULTILINEAR +/// polynomial in the twenty index bits: +/// +/// - X, Y and Z are linear forms over disjoint bit ranges. +/// - AND, OR and XOR are per-bit functions of ONE bit of X and ONE bit of Y. +/// Two distinct variables, so each of x*y, x+y-x*y and x+y-2*x*y has degree +/// one in each of them. +/// - MSB8 is bit 7, and MSB16 is bit 15 (bit 15 of X + 256*Y is bit 7 of Y). +/// - ZERO is the product over all twenty bits of (1 - b), i.e. eq(0, .). +/// - SLL and SLLC are sums over the sixteen values of Z of an indicator in the +/// four Z bits times a linear form in the sixteen halfword bits. The two +/// variable sets are disjoint, so each product is again multilinear. +/// +/// The multilinear extension of a function that is ALREADY multilinear in the +/// index bits is that same polynomial, so this is an identity rather than an +/// approximation. It is what lets a recursive verifier bind these columns +/// without the 2^20 pass that [`preprocessed_columns`] plus an MLE fold costs. +/// +/// # The variable convention, which is the easy thing to get backwards +/// +/// Mle::evaluate_in binds its first coordinate to the HIGH half of the table, +/// so coordinate i carries index bit NUM_VARS - 1 - i. The two run in opposite +/// directions, and this function is the only place that reversal is written. +/// +/// Returns None if the point is not NUM_VARS long. Column order is +/// [`generate_bitwise_row`]'s. +pub fn preprocessed_mle_at( + point: &[FieldElement], +) -> Option<[FieldElement; NUM_PRECOMPUTED_COLS]> +where + E: IsField, +{ + if point.len() != NUM_VARS { + return None; + } + let zero = FieldElement::::zero(); + let one = FieldElement::::one(); + // vs[k] is the variable carried by index bit k. + let vs: Vec<&FieldElement> = (0..NUM_VARS).map(|k| &point[NUM_VARS - 1 - k]).collect(); + + // Powers of two by doubling: a generic field has no From. + let mut pow2 = Vec::with_capacity(16); + let mut p = one.clone(); + for _ in 0..16 { + pow2.push(p.clone()); + p = &p + &p; + } + + // A linear form over a run of bits: sum over i of 2^i * bit(lo + i). + let lin = |lo: usize, len: usize| -> FieldElement { + (0..len).fold(zero.clone(), |acc, i| &acc + &(&pow2[i] * vs[lo + i])) }; + let x = lin(0, 8); + let y = lin(8, 8); + let z = lin(16, 4); + + // AND, OR, XOR: one product of two DISTINCT variables per bit. + let mut and = zero.clone(); + let mut or = zero.clone(); + let mut xor = zero.clone(); + for i in 0..8 { + let (a, b) = (vs[i], vs[8 + i]); + let ab = a * b; + let sum = a + b; + and = &and + &(&pow2[i] * &ab); + or = &or + &(&pow2[i] * &(&sum - &ab)); + xor = &xor + &(&pow2[i] * &(&(&sum - &ab) - &ab)); + } + + let msb8 = vs[7].clone(); + // Bit 15 of X + 256*Y is bit 7 of Y, which is index bit 15. + let msb16 = vs[15].clone(); - // Step 2: Interpolate each column to a polynomial (parallel) + // ZERO is eq(0, .): the product over every bit of (1 - b). + let is_zero = (0..NUM_VARS).fold(one.clone(), |acc, k| &acc * &(&one - vs[k])); + + // Halfword bit i is index bit i for i in 0..16, so one prefix sweep serves + // both shifts. prefix[m] = sum over i <= m of 2^i * bit(i). + let mut prefix = Vec::with_capacity(16); + let mut running = zero.clone(); + for i in 0..16 { + running = &running + &(&pow2[i] * vs[i]); + prefix.push(running.clone()); + } + + let mut sll = zero.clone(); + let mut sllc = zero.clone(); + // suffix holds halfword >> (16 - j) on entry to iteration j, advanced by + // suffix_{j+1} = 2*suffix_j + bit(15 - j). It is zero for j = 0. + let mut suffix = zero.clone(); + for j in 0..16usize { + // eq(j, the four Z bits). + let indicator = (0..4).fold(one.clone(), |acc, b| { + let bit = vs[16 + b]; + if (j >> b) & 1 == 1 { + &acc * bit + } else { + &acc * &(&one - bit) + } + }); + // (halfword << j) mod 2^16 keeps bits 0 ..= 15 - j at weight 2^(i + j). + // generate_bitwise_row's Z = 0 branch is this same value: a halfword is + // already below 2^16, so masking it changes nothing. + sll = &sll + &(&indicator * &(&prefix[15 - j] * &pow2[j])); + if j > 0 { + // SLLC is zero at Z = 0 and halfword >> (16 - Z) otherwise. + sllc = &sllc + &(&indicator * &suffix); + } + let doubled = &suffix + &suffix; + suffix = &doubled + vs[15 - j]; + } + + Some([x, y, z, and, or, xor, msb8, msb16, is_zero, sll, sllc]) +} + +/// Computes the Merkle commitment over the precomputed bitwise table columns. +/// +/// This builds a Merkle tree over the LDE (Low Degree Extension) of the precomputed +/// columns, matching exactly how the prover commits to traces. The tree has +/// NUM_ROWS * blowup_factor leaves, enabling FRI queries at any index +/// in the extended domain. +/// +/// Critical for security: the commitment must be over LDE values (not raw values) +/// because FRI queries can target any index in [0, N*blowup). A raw-value commitment +/// would only have N leaves, unable to verify queries at indices >= N. +/// +/// Exposed for the `compute_static_commitments` binary and the +/// drift-detection tests in `static_commitments_tests`. Production callers +/// should go through [`preprocessed_commitment`] so the static const-table +/// shortcut is used when applicable. +#[doc(hidden)] +pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { + compute_preprocessed_commitment_with(options, LeafLayout::RowPair) +} + +/// [`compute_preprocessed_commitment`] under an explicit trace-tree leaf +/// layout (the generator and the one-row drift test; S2). +#[doc(hidden)] +pub fn compute_preprocessed_commitment_with( + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + compute_preprocessed_commitment_under::(options, layout) +} + +/// [`compute_preprocessed_commitment_with`] under the commitment configuration +/// `S` (a proof family's: `crate::lfm::registry::build_artifacts_under`). +#[doc(hidden)] +pub fn compute_preprocessed_commitment_under( + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + let columns = preprocessed_columns(); + + // Interpolate each column to a polynomial (parallel) #[cfg(feature = "parallel")] let polys: Vec> = columns .par_iter() @@ -278,7 +470,7 @@ pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { }) .collect(); - // Step 3: Evaluate polynomials on LDE domain (parallel) + // Evaluate polynomials on LDE domain (parallel) let blowup_factor = options.blowup_factor as usize; let coset_offset = FE::from(options.coset_offset); @@ -300,9 +492,13 @@ pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { }) .collect(); - let (_, root) = commit_bit_reversed(&lde_columns, ROWS_PER_LEAF) - .expect("Failed to build Merkle tree for bitwise LDE"); - root + // ★ Through the LFM commit helper, which commits under the BLOCK PATH's pin + // rather than `stark`'s default aliases. This root is a PREPROCESSED + // commitment the block path's verifier absorbs, so it has to be built with + // the hash that path commits under — on a branch that pins an algebraic + // hash, a root left on the alias would be the one BLAKE3 artifact in an RPO + // proof, and it would fail as a root nothing reconstructs. + crate::lfm::commit::commit_lde_columns_under::(&lde_columns, layout) } /// Returns the preprocessed commitment for the bitwise table. @@ -327,6 +523,72 @@ pub fn preprocessed_commitment(options: &ProofOptions) -> Commitment { compute_preprocessed_commitment(options) } +/// The preprocessed commitment under the table's resolved leaf `layout`: +/// today's [`preprocessed_commitment`] for row pairs; for one row the static +/// twin ([`static_commitment_one_row`]) at coset 3, and `None` otherwise — a +/// hard miss the prover refuses and the verifier rejects, never a +/// recompute of a 2^20-row table behind the operator's back. +pub fn preprocessed_commitment_for( + options: &ProofOptions, + layout: LeafLayout, +) -> Option { + match layout { + LeafLayout::RowPair => Some(preprocessed_commitment(options)), + LeafLayout::Row => (options.coset_offset == 3) + .then(|| static_commitment_one_row(options.blowup_factor)) + .flatten(), + } +} + +/// The row-pair preprocessed commitment under the commitment configuration +/// `S`: the legacy pin's blessed root ([`preprocessed_commitment`]); a +/// Poseidon1 configuration's blessed root where one ships +/// ([`static_commitment_p1`], coset 3); any other key computed once per +/// process ([`crate::hash_pin::static_root_under`]). +pub fn preprocessed_root_under(options: &ProofOptions) -> Commitment { + if S::COMMITMENT_HASH == crate::hash_pin::BLOCK_COMMITMENT_HASH { + return preprocessed_commitment(options); + } + if S::COMMITMENT_HASH == stark::config::CommitmentHash::Poseidon1 + && options.coset_offset == 3 + && let Some(commitment) = static_commitment_p1(options.blowup_factor) + { + return commitment; + } + crate::hash_pin::static_root_under::("bitwise", options, LeafLayout::RowPair, || { + compute_preprocessed_commitment_under::(options, LeafLayout::RowPair) + }) +} + +/// [`preprocessed_commitment_for`] under the commitment configuration `S`: the +/// legacy pin's static roots (a one-row miss stays a miss), any other +/// configuration's computed once per process. +pub fn preprocessed_commitment_under( + options: &ProofOptions, + layout: LeafLayout, +) -> Option { + match layout { + LeafLayout::RowPair => Some(preprocessed_root_under::(options)), + LeafLayout::Row if S::COMMITMENT_HASH == crate::hash_pin::BLOCK_COMMITMENT_HASH => { + preprocessed_commitment_for(options, layout) + } + LeafLayout::Row => Some(crate::hash_pin::static_root_under::( + "bitwise", + options, + layout, + || compute_preprocessed_commitment_under::(options, layout), + )), + } +} + +/// The AIR's commitment source for both leaf layouts: today's root now, the +/// one-row twin on demand ([`preprocessed_commitment_for`]). +pub fn lazy_commitment(options: &ProofOptions) -> stark::lookup::LazyCommitment { + let o = options.clone(); + stark::lookup::LazyCommitment::ready(preprocessed_commitment(options)) + .with_one_row(move || preprocessed_commitment_for(&o, LeafLayout::Row)) +} + // ========================================================================= // Trace generation // ========================================================================= @@ -425,6 +687,44 @@ pub fn update_multiplicities( } } +/// Takes `count` lookups of `op` back off the multiplicity columns: lookups the +/// histogram counted for rows a later cut leaves out of the proof (KECCAK_RND's +/// padding rows in the tables a block's split drops). The trace may be held +/// packed. Refuses a count above the multiplicity it would take from. +pub(crate) fn take_back_multiplicity( + trace: &mut TraceTable, + op: BitwiseOperation, + count: u64, +) -> Result<(), crate::Error> { + if count == 0 { + return Ok(()); + } + #[cfg(feature = "cuda")] + trace.clear_main_rowmajor_dev(); + let row = row_index(op.x, op.y, op.z); + let col = mu_column(op.lookup_type); + let short = |have: u64| { + crate::Error::Prover(format!( + "BITWISE {:?}[{}, {}, {}] counts {have}, {count} cannot be taken back", + op.lookup_type, op.x, op.y, op.z + )) + }; + if let Some(packed) = trace.narrow_main_mut() { + let have = packed.get(row, col); + if have < count || !packed.set(row, col, have - count) { + return Err(short(have)); + } + return Ok(()); + } + let current = trace.main_table.get_row(row)[col]; + let have = current.canonical(); + if have < count { + return Err(short(have)); + } + trace.main_table.set_fe(row, col, current - FE::from(count)); + Ok(()) +} + /// Number of distinct BITWISE lookup types (one multiplicity column each). /// Derived from [`BitwiseOperationType::ALL`], which the compile-time guard /// below keeps in lockstep with [`lookup_type_index`]. @@ -567,6 +867,11 @@ impl BitwiseHistogram { self.counters[idx] += n; } + /// The bytes its counters take on the heap. + pub(crate) fn heap_bytes(&self) -> usize { + std::mem::size_of_val(&*self.counters) + } + /// Fold a slice of lookups into the histogram. #[inline] pub(crate) fn add_ops(&mut self, ops: &[BitwiseOperation]) { @@ -582,6 +887,36 @@ impl BitwiseHistogram { } } + /// Add `plane`'s counts ([`BitwisePlane`]). + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn merge_plane(&mut self, plane: &BitwisePlane) { + for type_idx in 0..NUM_LOOKUP_TYPES { + let at = type_idx * PLANE_ROWS; + for (cell, &count) in plane.counters[at..at + PLANE_ROWS].iter().enumerate() { + if count != 0 { + self.counters[type_idx * NUM_ROWS + cell] += count; + } + } + } + for &(op, n) in &plane.off_plane { + self.bump_n(op, n); + } + } + + /// A digest of every count with its cell: equal digests, equal + /// multiplicities (a fold, for a log line that compares two builds). + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn digest(&self) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325u64; + for (cell, &count) in self.counters.iter().enumerate() { + if count != 0 { + h = (h ^ (cell as u64)).wrapping_mul(0x0100_0000_01b3); + h = (h ^ count).wrapping_mul(0x0100_0000_01b3).rotate_left(29); + } + } + h + } + /// Write the accumulated multiplicities into the BITWISE trace's MU columns. /// /// OVERWRITES each nonzero cell with its count (it does not add to what is @@ -608,6 +943,42 @@ impl BitwiseHistogram { } } +/// Cells of one lookup type with `z = 0`: every `(x, y)`. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +const PLANE_ROWS: usize = 256 * 256; + +/// A table's share of the BITWISE multiplicities, counted as the table is +/// built and added to the histogram after ([`BitwiseHistogram::merge_plane`]): +/// the lookups with `z = 0` of every type in a dense array (5 MiB, against the +/// histogram's 80), any other kept aside, so none is lost. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) struct BitwisePlane { + counters: Box<[u64]>, + off_plane: Vec<(BitwiseOperation, u64)>, +} + +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +impl BitwisePlane { + #[allow(clippy::new_without_default)] + pub(crate) fn new() -> Self { + Self { + counters: vec![0u64; PLANE_ROWS * NUM_LOOKUP_TYPES].into_boxed_slice(), + off_plane: Vec::new(), + } + } + + /// Adds `n` occurrences of one lookup. + #[inline] + pub(crate) fn bump_n(&mut self, op: BitwiseOperation, n: u64) { + if op.z == 0 { + let idx = lookup_type_index(op.lookup_type) * PLANE_ROWS + row_index(op.x, op.y, 0); + self.counters[idx] += n; + } else { + self.off_plane.push((op, n)); + } + } +} + /// Types of lookups the BITWISE table provides. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BitwiseOperationType { diff --git a/prover/src/tables/blake3.rs b/prover/src/tables/blake3.rs new file mode 100644 index 000000000..637127fa2 --- /dev/null +++ b/prover/src/tables/blake3.rs @@ -0,0 +1,3406 @@ +//! BLAKE3 6-round compression accelerator chip (syscall variant). +//! +//! One row per compression call, fully unrolled (Layout B of +//! `thoughts/blake3/blake3-chip/DESIGN.md`): all 6 rounds × 8 G-functions are +//! laid out in SSA form across the row, so the message schedule is a +//! compile-time permutation of the 16 committed message words and there is no +//! state/message handoff between rows. +//! +//! I/O follows the KECCAK core idiom (`keccak.rs`): an `Ecall` receiver binds +//! (timestamp, syscall#), a `Memw` register read binds the x10 state pointer, +//! and per-dword `Memw` ops read the 112 input bytes / write the 64 output +//! bytes. The 176-byte state region layout is documented on +//! [`executor::vm::instruction::execution::BLAKE3_SYSCALL_NUMBER`]. +//! +//! ## Two modes, one table +//! +//! The table serves two syscalls, selected per row by a pair of mutually +//! exclusive multiplicity columns with `MU = MU_S + MU_A`: +//! +//! * **`MU_S` — single compression** ([`BLAKE3_SYSCALL_NUMBER`]). One ecall, +//! one row, the 176-byte state region described above. Unchanged. +//! * **`MU_A` — chained absorb** +//! ([`executor::vm::instruction::execution::BLAKE3_ABSORB_SYSCALL_NUMBER`]). +//! One ecall folds `num_blocks` consecutive 64-byte blocks into a chaining +//! value, and occupies `num_blocks + 1` rows: one compression per block, then +//! an END row that does no work and writes `cv_out`. The rows are linked by +//! the self-referencing [`BusId::Blake3Absorb`] bus, COMMIT's idiom +//! (`commit.rs`) with the countdown carried in a single field element. +//! +//! Every row of one absorb group carries that ecall's timestamp, so the group +//! is identified on the bus by `(timestamp, …)`. `END` is never free: it is the +//! output of a `Zero[REMAINING]` lookup, so a prover cannot end a group early +//! (the lookup rejects) or late (`REMAINING = 0 ⇒ END = 1 ⇒ no send`, and a +//! send with no receiver unbalances the bus). +//! +//! The absorb mode's soundness ledger, beyond the compression core's: +//! +//! 11. `MU_S · MU_A = 0` and `MU − MU_S − MU_A = 0`, both `IS_BIT` — a row is in +//! exactly one mode, and neither mode's interactions fire in the other. +//! 12. `(FIRST + END)·(1 − MU_A) = 0` locks both boundary flags to absorb rows, +//! so a `μ = 0` padding row cannot mint a group boundary; `FIRST · END = 0` +//! forbids the zero-block group the executor also rejects. +//! 13. The chain bus tuple leads with `TIMESTAMP_0, TIMESTAMP_1` and carries the +//! control address and message base as well as the chaining value, so a row +//! cannot receive one group's state and send another's (DESIGN.md §1.1), nor +//! read a block of its own choosing. +//! 14. The END row's mixing core is gated OFF (`MU − END`), so its `h` bytes are +//! range-checked by an explicit `AreBytes` instead — without it the `cv_out` +//! write could place non-canonical "bytes" in memory. +//! 15. The block cap is enforced in-circuit (`IsB20[REM_DECR · 2^10]` on the +//! FIRST row), not inherited from the executor, so the chip accepts exactly +//! the `1..=1024` the VM semantics accept. +//! 16. The ABI's address conditions are checked here too, not only by the +//! executor: x11 is 8-aligned (`FIRST·(M_BASE[0] − 8·Q) = 0`, with `Q` a +//! halfword) and the message region does not wrap the address space except +//! on its last block (`MU_C − NEXT_IS_END` gating the successor's carry). +//! Without them the chip accepted absorbs the executor rejects. +//! +//! ## ⚠ Row budget: one cycle, up to 1,025 rows +//! +//! This is the only accelerator whose row count is not a function of cycles: one +//! absorb ecall is one CPU cycle and up to 1,025 rows plus ~8,196 MEMW ops, +//! where keccak is one cycle to one row. A capped-absorb loop therefore inflates +//! an epoch's BLAKE3 rows ~170× over what the cycle-keyed splitter expects. +//! +//! No per-epoch budget enforces this, which is safe only because of WHO calls +//! the ecall. The trust boundary, and the condition under which a hard budget +//! becomes required, are written at `crypto::hash::blake3::chain::bulk_absorb_blocks` +//! — the gate every guest absorb goes through, and so the place the next person +//! adding a call site will actually read. +//! +//! ## The single-dataflow rule +//! +//! The compression dataflow is written ONCE, in [`run_flow`], and interpreted +//! twice: [`WireFlow`] (columns — drives the constraints and bus senders) and +//! [`ValueFlow`] (u32 witness — drives trace filling and the BITWISE +//! multiplicity collection in `trace_builder.rs`). The two cannot diverge on +//! wiring, only on interpretation, which the e2e bus-balance gate checks. +//! +//! ## Soundness ledger (DESIGN.md §7, adapted to the syscall variant) +//! +//! 1. Every eval constraint is μ-gated; padding rows are all-zero (except the +//! keccak-style PTR pad) with μ=0. +//! 2. 3-op adds use TWO summed committed carry bits + the explicit sum +//! identity (a ternary carry would be degree 4 after gating). +//! 3. 2-op adds use the `emit_add_pair`-style expression carry (no committed +//! cell) with μ-gated booleanity; the output's bytes are range-checked by +//! the downstream XOR lookup that consumes them. +//! 4. Every add/shift output feeds a downstream `ByteAlu` XOR — that lookup is +//! its only byte range check. The last-round outputs are consumed by the +//! feed-forward XORs, closing the chain. +//! 5. The message words `m` are never XORed, so their 64 bytes get explicit +//! `AreBytes` sends. Same for the 64 `OLD_OUT` bytes (the previous memory +//! content of the out region, which appear only on the Memw bus) and the 8 +//! address bytes (aliasing — see keccak.rs's addr comment). +//! 6. rotr16/rotr8 are free byte relabels `[b2,b3,b0,b1]` / `[b1,b2,b3,b0]`. +//! 7. rotr12/rotr7 are inline μ-gated shift identities with `AreBytes` on all +//! four shift halfwords (`SLL_lo/SLLC_lo/SLL_hi/SLLC_hi`); soundness needs +//! the tight bound on the `SLL` pair (2^16 invertible mod p — the audited +//! Euclidean-division argument). +//! 8. The message schedule is `permute^r` wired from the ORIGINAL M columns. +//! 9. All identities stay < 2^35 ≪ p (non-overflow side conditions), given +//! byte-range operands and boolean carries. +//! 10. (Internal-bus binding — N/A here: the syscall variant has no `Blake3` +//! bus; a row's inputs and outputs are tied by being the same row.) +//! +//! ⚠ This chip implements the **6-round internal variant** — NOT standard +//! 7-round BLAKE3. Its collision resistance is a named assumption +//! (DESIGN.md "If this is picked up again"). + +use executor::vm::instruction::execution::{ + BLAKE3_ABSORB_SYSCALL_NUMBER, BLAKE3_IV, BLAKE3_MSG_PERMUTATION, BLAKE3_ROUNDS, + BLAKE3_SYSCALL_NUMBER, +}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; + +use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; +use crate::constraints::templates::{AddOperand, INV_SHIFT_32}; + +/// G-instances per compression: 8 per round × 6 rounds. +pub const NUM_G: usize = BLAKE3_ROUNDS * 8; + +/// Dwords in the state region: 14 input (h|m|t|len_flags) + 8 output. +pub const STATE_DWORDS: usize = 22; +/// Input dwords (read-only). +pub const IN_DWORDS: usize = 14; +/// Dwords of the state region that carry the chaining value `h`. In absorb mode +/// the same columns and the same `PTR[0..4]` addresses hold `cv_in`. +pub const CV_DWORDS: usize = 4; +/// Dwords of one 64-byte message block. +pub const MSG_DWORDS: usize = 8; +/// Dwords of the absorb control region: 4 for `cv_in`, 4 for `cv_out`. Only +/// `PTR[0..CTRL_DWORDS]` is meaningful on an absorb row. +pub const CTRL_DWORDS: usize = 8; +/// Dword offset of `cv_out` inside the absorb control region — pinned equal to +/// the executor's [`BLAKE3_ABSORB_CV_OUT_DWORD`] by `the_chip_agrees_on_the_absorb_abi`. +pub const CV_OUT_DWORD: usize = 4; +/// Largest `num_blocks` the chip's own range check admits, restated in-circuit +/// rather than inherited from the executor: see `absorb_cap_in_circuit`. +pub const ABSORB_MAX_BLOCKS: u64 = 1 << 10; +/// Multiplier that turns the `IsB20` 20-bit lookup into the exact `≤ 2^10` cap: +/// `REM_DECR · 2^10 < 2^20 ⟺ REM_DECR < 2^10`. +pub const ABSORB_CAP_SCALE: u64 = (1 << 20) / ABSORB_MAX_BLOCKS; + +/// The (a, b, c, d) state indices of the 8 G-calls of one round: +/// 4 column mixes then 4 diagonal mixes (BLAKE3 spec §2.1). +const G_INDICES: [(usize, usize, usize, usize); 8] = [ + (0, 4, 8, 12), + (1, 5, 9, 13), + (2, 6, 10, 14), + (3, 7, 11, 15), + (0, 5, 10, 15), + (1, 6, 11, 12), + (2, 7, 8, 13), + (3, 4, 9, 14), +]; + +/// Shift amounts of the two non-free rotations, as `rotl` inner shifts: +/// rotr12 = rotl20 = rotl16∘rotl4 (r=4); rotr7 = rotl25 = rotl16∘rotl9 (r=9). +const ROT_SHIFT_R: [u32; 2] = [4, 9]; + +// ========================================================================= +// Column indices +// ========================================================================= + +pub mod cols { + use super::NUM_G; + + pub const TIMESTAMP_0: usize = 0; + pub const TIMESTAMP_1: usize = 1; + + /// State address as 8 bytes (DWordBL). + pub const ADDR: usize = 2; + + /// Per-dword pointers [22][4] halfwords (DWordHL), ptr[k] = addr + 8k. + pub const PTR: usize = ADDR + 8; // 10 + + /// Input bytes: h[32] | m[64] | t_lo[4] | t_hi[4] | block_len[4] | flags[4]. + pub const IN: usize = PTR + super::STATE_DWORDS * 4; // 98 + + /// 48 G-blocks × 60 cells (56 bytes + 4 carry bits) — see `g` accessors. + pub const G: usize = IN + 112; // 210 + pub const G_SIZE: usize = 60; + + /// Feed-forward output bytes out[0..16] (64 bytes). + pub const OUT: usize = G + NUM_G * G_SIZE; // 3090 + + /// Previous memory content of the out region (64 bytes). Appears only in + /// the Memw write ops' `old` field; range-checked by AreBytes. + pub const OLD_OUT: usize = OUT + 64; // 3154 + + /// Multiplicity / gate flag. `MU = MU_S + MU_A`. + pub const MU: usize = OLD_OUT + 64; // 3218 + + // ------------------------------------------------------------------------- + // Absorb mode (see the module docs' "Two modes" section) + // ------------------------------------------------------------------------- + + /// Single-compression row (the `BLAKE3_SYSCALL_NUMBER` mode). + pub const MU_S: usize = MU + 1; // 3219 + /// Absorb row — a compression row of a group, or its END row. + pub const MU_A: usize = MU_S + 1; // 3220 + /// Absorb row that compresses: `MU_C = MU_A − END`. A column rather than an + /// expression because `emit_add_pair` gates on a SUM of columns. + pub const MU_C: usize = MU_A + 1; // 3221 + /// First row of an absorb group (the one that receives the `Ecall`). + pub const FIRST: usize = MU_C + 1; // 3222 + /// Last row of an absorb group: does no compression, writes `cv_out`. + pub const END: usize = FIRST + 1; // 3223 + /// Blocks left to fold INCLUDING this row's: `num_blocks` on FIRST, 0 on END. + pub const REMAINING: usize = END + 1; // 3224 + /// `REMAINING − 1`, the countdown the chain bus carries forward. + pub const REM_DECR: usize = REMAINING + 1; // 3225 + /// Message address of this row's block, as 4 halfwords (DWordHL). + pub const M_BASE: usize = REM_DECR + 1; // 3226 + /// `M_BASE + 64` — the next row's block address (DWordHL). + pub const M_BASE_INCR: usize = M_BASE + 4; // 3230 + /// Per-dword message pointers `[8][4]` halfwords, `msg_ptr[j] = M_BASE + 8j`. + pub const MSG_PTR: usize = M_BASE_INCR + 4; // 3234 + + /// `M_BASE[0] / 8` on the FIRST row — the witness that the message address + /// the caller passed in x11 is 8-aligned, which the ABI requires and the + /// executor rejects without. + pub const MSG_ALIGN_Q: usize = MSG_PTR + super::MSG_DWORDS * 4; // 3266 + /// `REM_DECR == 0`: this compressing row is the group's LAST one. Only that + /// row may let `M_BASE + 64` leave the address space. + pub const NEXT_IS_END: usize = MSG_ALIGN_Q + 1; // 3267 + + pub const NUM_COLUMNS: usize = NEXT_IS_END + 1; // 3268 + + /// `msg_ptr[j][hw]` — halfword `hw` of the pointer to message dword `j`. + #[inline] + pub const fn msg_ptr(j: usize, hw: usize) -> usize { + MSG_PTR + j * 4 + hw + } + + // ------------------------------------------------------------------------- + // Index helpers + // ------------------------------------------------------------------------- + + #[inline] + pub const fn addr(byte: usize) -> usize { + ADDR + byte + } + + /// ptr[k][hw] — halfword hw of the pointer to dword k. + #[inline] + pub const fn ptr(k: usize, hw: usize) -> usize { + PTR + k * 4 + hw + } + + /// Input word i (0..28: h[0..8], m[8..24], t_lo=24, t_hi=25, len=26, flags=27), + /// byte b. + #[inline] + pub const fn in_word(i: usize, b: usize) -> usize { + IN + i * 4 + b + } + + /// Base column of G-block g. + #[inline] + pub const fn g_base(g: usize) -> usize { + G + g * G_SIZE + } + + // Offsets inside one G block (56 byte cells + 4 carry bits = 60): + /// add3 #1 output word (4 bytes). + pub const G_A1: usize = 0; + /// add3 #1 carry bits c1, c2. + pub const G_A1_C: usize = 4; + /// X1 = vd ^ A1 (4 bytes). + pub const G_X1: usize = 6; + /// add2 #1 output word (4 bytes). + pub const G_C1: usize = 10; + /// X2 = vb ^ C1 (4 bytes). + pub const G_X2: usize = 14; + /// rotr12 block: SLL_lo(2) SLLC_lo(2) SLL_hi(2) SLLC_hi(2) Y(4). + pub const G_R1: usize = 18; + /// add3 #2 output word (4 bytes). + pub const G_A2: usize = 30; + /// add3 #2 carry bits. + pub const G_A2_C: usize = 34; + /// X3 = vd ^ A2 (4 bytes). + pub const G_X3: usize = 36; + /// add2 #2 output word (4 bytes). + pub const G_C2: usize = 40; + /// X4 = B1 ^ C2 (4 bytes). + pub const G_X4: usize = 44; + /// rotr7 block: same layout as G_R1. + pub const G_R2: usize = 48; + + /// Feed-forward output word i (0..16), byte b. + #[inline] + pub const fn out_word(i: usize, b: usize) -> usize { + OUT + i * 4 + b + } + + /// Previous-content byte b (0..64) of the out region. + #[inline] + pub const fn old_out(b: usize) -> usize { + OLD_OUT + b + } +} + +// ========================================================================= +// The single dataflow, interpreted twice +// ========================================================================= + +/// The BLAKE3 compression dataflow, abstracted over its word representation. +/// +/// [`run_flow`] is the only place the G-function wiring, message schedule and +/// feed-forward exist; implementors interpret the primitive ops either as +/// column wiring ([`WireFlow`]) or as u32 witness computation ([`ValueFlow`]). +pub(crate) trait Blake3Flow { + type Word: Copy; + + /// h[i] input word. + fn input_h(&mut self, i: usize) -> Self::Word; + /// v[12..16] init words: t_lo, t_hi, block_len, flags. + fn input_v12(&mut self, j: usize) -> Self::Word; + /// IV[i] constant (v[8..12]). + fn iv_const(&mut self, i: usize) -> Self::Word; + + /// 3-operand add `s = a + b + m[m_idx] mod 2^32` (half 0/1 = which add3 of G g). + fn add3( + &mut self, + g: usize, + half: usize, + a: Self::Word, + b: Self::Word, + m_idx: usize, + ) -> Self::Word; + /// 2-operand add `s = a + b mod 2^32`. + fn add2(&mut self, g: usize, half: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// XOR (slot 0..4 = X1..X4 of G g). Operand order is part of the wire format. + fn xor(&mut self, g: usize, slot: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// rotr16: free byte relabel [b2,b3,b0,b1]. + fn rotr16(&mut self, w: Self::Word) -> Self::Word; + /// rotr8: free byte relabel [b1,b2,b3,b0]. + fn rotr8(&mut self, w: Self::Word) -> Self::Word; + /// rotr12 (half=0) / rotr7 (half=1) via the inline shift identity. + fn rot_shift(&mut self, g: usize, half: usize, w: Self::Word) -> Self::Word; + /// Feed-forward XOR pair: out[i] = v[i] ^ v[i+8], out[i+8] = v[i+8] ^ h[i]. + fn feed_forward(&mut self, i: usize, vi: Self::Word, vi8: Self::Word, hi: Self::Word); +} + +/// Drive the full 6-round compression through `f`. The message schedule is +/// tracked as indices into the ORIGINAL m (permute^r composition), so both +/// interpretations reference original message words — never copies. +pub(crate) fn run_flow(f: &mut F) { + let h: [F::Word; 8] = core::array::from_fn(|i| f.input_h(i)); + let mut v: [F::Word; 16] = core::array::from_fn(|i| { + if i < 8 { + h[i] + } else if i < 12 { + f.iv_const(i - 8) + } else { + f.input_v12(i - 12) + } + }); + + // sched[i] = index into the original m of the word consumed at position i + // this round. permute: m'[i] = m[P[i]] ⇒ sched'[i] = sched[P[i]]. + let mut sched: [usize; 16] = core::array::from_fn(|i| i); + + for r in 0..BLAKE3_ROUNDS { + for (j, &(ia, ib, ic, id)) in G_INDICES.iter().enumerate() { + let g = r * 8 + j; + let (va, vb, vc, vd) = (v[ia], v[ib], v[ic], v[id]); + let mx = sched[2 * j]; + let my = sched[2 * j + 1]; + + let a1 = f.add3(g, 0, va, vb, mx); + let x1 = f.xor(g, 0, vd, a1); + let vd1 = f.rotr16(x1); + let c1 = f.add2(g, 0, vc, vd1); + let x2 = f.xor(g, 1, vb, c1); + let b1 = f.rot_shift(g, 0, x2); // rotr12 + let a2 = f.add3(g, 1, a1, b1, my); + let x3 = f.xor(g, 2, vd1, a2); + let vd2 = f.rotr8(x3); + let c2 = f.add2(g, 1, c1, vd2); + let x4 = f.xor(g, 3, b1, c2); + let b2 = f.rot_shift(g, 1, x4); // rotr7 + + v[ia] = a2; + v[ib] = b2; + v[ic] = c2; + v[id] = vd2; + } + if r < BLAKE3_ROUNDS - 1 { + let prev = sched; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + sched[i] = prev[p]; + } + } + } + + for i in 0..8 { + f.feed_forward(i, v[i], v[i + 8], h[i]); + } +} + +// ========================================================================= +// Wire interpretation (columns) +// ========================================================================= + +/// A 32-bit word as wiring: four byte columns (LSB first) or a constant. +/// Constants only ever appear as the IV `v[c]` operands of round-0 add2s. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WordRef { + Cols([usize; 4]), + Const(u32), +} + +impl WordRef { + fn byte(self, b: usize) -> ByteRef { + match self { + WordRef::Cols(c) => ByteRef::Col(c[b]), + WordRef::Const(w) => ByteRef::Const(((w >> (8 * b)) & 0xFF) as u8), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ByteRef { + Col(usize), + Const(u8), +} + +/// One recorded 3-op add: operands (a, b, m columns), output columns, carries. +pub(crate) struct Add3Wire { + pub a: WordRef, + pub b: WordRef, + pub m: [usize; 4], + pub s: [usize; 4], + pub c1: usize, + pub c2: usize, +} + +/// One recorded 2-op add: operands, output columns (carry is an expression). +pub(crate) struct Add2Wire { + pub a: WordRef, + pub b: WordRef, + pub s: [usize; 4], +} + +/// One recorded XOR: per-byte operands and output columns. +pub(crate) struct XorWire { + pub a: WordRef, + pub b: WordRef, + pub out: [usize; 4], +} + +/// One recorded shift rotation: input word, the 8 shift-halfword byte columns +/// (SLL_lo, SLLC_lo, SLL_hi, SLLC_hi — 2 bytes each), output columns, r. +pub(crate) struct RotWire { + pub input: WordRef, + pub sll_lo: [usize; 2], + pub sllc_lo: [usize; 2], + pub sll_hi: [usize; 2], + pub sllc_hi: [usize; 2], + pub y: [usize; 4], + pub r: u32, +} + +/// The full wiring of one compression row: everything the constraints and the +/// bus senders need, recorded in canonical order by [`run_flow`]. +pub(crate) struct WireFlow { + pub add3s: Vec, + pub add2s: Vec, + pub xors: Vec, + pub rots: Vec, +} + +impl WireFlow { + pub(crate) fn build() -> Self { + let mut w = WireFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + 16), + rots: Vec::with_capacity(NUM_G * 2), + }; + run_flow(&mut w); + w + } +} + +#[inline] +fn word_cols(start: usize) -> [usize; 4] { + [start, start + 1, start + 2, start + 3] +} + +impl Blake3Flow for WireFlow { + type Word = WordRef; + + fn input_h(&mut self, i: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(i, 0))) + } + fn input_v12(&mut self, j: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(24 + j, 0))) + } + fn iv_const(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + fn add3(&mut self, g: usize, half: usize, a: WordRef, b: WordRef, m_idx: usize) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_A1 } else { cols::G_A2 }; + let cbase = cols::g_base(g) + + if half == 0 { + cols::G_A1_C + } else { + cols::G_A2_C + }; + let s = word_cols(base); + self.add3s.push(Add3Wire { + a, + b, + m: word_cols(cols::in_word(8 + m_idx, 0)), + s, + c1: cbase, + c2: cbase + 1, + }); + WordRef::Cols(s) + } + + fn add2(&mut self, g: usize, half: usize, a: WordRef, b: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_C1 } else { cols::G_C2 }; + let s = word_cols(base); + self.add2s.push(Add2Wire { a, b, s }); + WordRef::Cols(s) + } + + fn xor(&mut self, g: usize, slot: usize, a: WordRef, b: WordRef) -> WordRef { + let off = match slot { + 0 => cols::G_X1, + 1 => cols::G_X2, + 2 => cols::G_X3, + _ => cols::G_X4, + }; + let out = word_cols(cols::g_base(g) + off); + self.xors.push(XorWire { a, b, out }); + WordRef::Cols(out) + } + + fn rotr16(&mut self, w: WordRef) -> WordRef { + match w { + WordRef::Cols([b0, b1, b2, b3]) => WordRef::Cols([b2, b3, b0, b1]), + WordRef::Const(v) => WordRef::Const(v.rotate_right(16)), + } + } + fn rotr8(&mut self, w: WordRef) -> WordRef { + match w { + WordRef::Cols([b0, b1, b2, b3]) => WordRef::Cols([b1, b2, b3, b0]), + WordRef::Const(v) => WordRef::Const(v.rotate_right(8)), + } + } + + fn rot_shift(&mut self, g: usize, half: usize, w: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_R1 } else { cols::G_R2 }; + let y = word_cols(base + 8); + self.rots.push(RotWire { + input: w, + sll_lo: [base, base + 1], + sllc_lo: [base + 2, base + 3], + sll_hi: [base + 4, base + 5], + sllc_hi: [base + 6, base + 7], + y, + r: ROT_SHIFT_R[half], + }); + WordRef::Cols(y) + } + + fn feed_forward(&mut self, i: usize, vi: WordRef, vi8: WordRef, hi: WordRef) { + let out_lo = word_cols(cols::out_word(i, 0)); + let out_hi = word_cols(cols::out_word(i + 8, 0)); + self.xors.push(XorWire { + a: vi, + b: vi8, + out: out_lo, + }); + self.xors.push(XorWire { + a: vi8, + b: hi, + out: out_hi, + }); + } +} + +// ========================================================================= +// Value interpretation (u32 witness) +// ========================================================================= + +/// Everything the trace filler and the BITWISE collector need for one +/// compression, recorded cell-exactly in the same canonical order as +/// [`WireFlow`]. `xor_ops` carries (a, b) operand VALUES per XOR word — the +/// per-byte lookups are `(a_byte, b_byte)` in the same operand order the +/// senders use. +pub(crate) struct ValueFlow { + /// (s, c1, c2) per add3, canonical order. + pub add3s: Vec<(u32, u8, u8)>, + /// s per add2 (the carry is an expression, not a cell). + pub add2s: Vec, + /// (a, b, out) per XOR word, canonical order (Gs then feed-forward). + pub xors: Vec<(u32, u32, u32)>, + /// (sll_lo, sllc_lo, sll_hi, sllc_hi, y) per shift rotation. + pub rots: Vec<(u16, u16, u16, u16, u32)>, + /// The 16-word output. + pub out: [u32; 16], + + h: [u32; 8], + m: [u32; 16], + v12: [u32; 4], +} + +impl ValueFlow { + pub(crate) fn compute(h: &[u32; 8], m: &[u32; 16], t: u64, block_len: u32, flags: u32) -> Self { + let mut f = ValueFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + 16), + rots: Vec::with_capacity(NUM_G * 2), + out: [0; 16], + h: *h, + m: *m, + v12: [t as u32, (t >> 32) as u32, block_len, flags], + }; + run_flow(&mut f); + f + } +} + +impl Blake3Flow for ValueFlow { + type Word = u32; + + fn input_h(&mut self, i: usize) -> u32 { + self.h[i] + } + fn input_v12(&mut self, j: usize) -> u32 { + self.v12[j] + } + fn iv_const(&mut self, i: usize) -> u32 { + BLAKE3_IV[i] + } + + fn add3(&mut self, _g: usize, _half: usize, a: u32, b: u32, m_idx: usize) -> u32 { + let m = self.m[m_idx]; + let wide = a as u64 + b as u64 + m as u64; + let s = wide as u32; + let carry = (wide >> 32) as u8; // 0, 1 or 2 + // Two summed carry bits: c1 + c2 = carry. + let (c1, c2) = match carry { + 0 => (0, 0), + 1 => (1, 0), + _ => (1, 1), + }; + self.add3s.push((s, c1, c2)); + s + } + + fn add2(&mut self, _g: usize, _half: usize, a: u32, b: u32) -> u32 { + let s = a.wrapping_add(b); + self.add2s.push(s); + s + } + + fn xor(&mut self, _g: usize, _slot: usize, a: u32, b: u32) -> u32 { + let out = a ^ b; + self.xors.push((a, b, out)); + out + } + + fn rotr16(&mut self, w: u32) -> u32 { + w.rotate_right(16) + } + fn rotr8(&mut self, w: u32) -> u32 { + w.rotate_right(8) + } + + fn rot_shift(&mut self, _g: usize, half: usize, w: u32) -> u32 { + let r = ROT_SHIFT_R[half]; + let xlo = w & 0xFFFF; + let xhi = w >> 16; + // xlo·2^r = SLLC_lo·2^16 + SLL_lo (and same for hi): Euclidean split. + let sll_lo = ((xlo << r) & 0xFFFF) as u16; + let sllc_lo = ((xlo << r) >> 16) as u16; + let sll_hi = ((xhi << r) & 0xFFFF) as u16; + let sllc_hi = ((xhi << r) >> 16) as u16; + // Recombine + halfword swap: Ylo = SLL_hi + SLLC_lo, Yhi = SLL_lo + SLLC_hi. + let ylo = sll_hi as u32 + sllc_lo as u32; + let yhi = sll_lo as u32 + sllc_hi as u32; + let y = ylo | (yhi << 16); + debug_assert_eq!(y, w.rotate_right(if r == 4 { 12 } else { 7 })); + self.rots.push((sll_lo, sllc_lo, sll_hi, sllc_hi, y)); + y + } + + fn feed_forward(&mut self, i: usize, vi: u32, vi8: u32, hi: u32) { + let lo = vi ^ vi8; + let hi_w = vi8 ^ hi; + self.xors.push((vi, vi8, lo)); + self.xors.push((vi8, hi, hi_w)); + self.out[i] = lo; + self.out[i + 8] = hi_w; + } +} + +// ========================================================================= +// Operation struct + trace generation +// ========================================================================= + +#[derive(Debug, Clone)] +pub struct Blake3Operation { + pub timestamp: u64, + pub state_addr: u64, + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, + /// Previous memory content of the 64-byte out region (for the Memw `old`). + pub old_out: [u8; 64], + /// The 16-word compression output (recomputed by the trace builder). + pub out: [u32; 16], +} + +/// One chained-absorb ecall: `num_blocks` compressions plus the END row that +/// writes `cv_out`, all at one timestamp. +#[derive(Debug, Clone)] +pub struct Blake3AbsorbOperation { + pub timestamp: u64, + /// x10 — the 64-byte control region (`cv_in` in dwords 0..4, `cv_out` 4..8). + pub ctrl_addr: u64, + /// x11 — the message, read in place. + pub msg_addr: u64, + /// x13 — the flag word of block 0; every later block carries 0. + pub first_flags: u32, + /// The incoming chaining value, read from `ctrl_addr`. + pub cv_in: [u32; 8], + /// The message, one entry per block. `num_blocks` is its length. + pub blocks: Vec<[u32; 16]>, + /// Previous memory content of the 32-byte `cv_out` region (the Memw `old`). + pub old_cv_out: [u8; 32], +} + +/// One row of an absorb group, in trace order: `num_blocks` compressing rows +/// then the END row. +/// +/// ★ This expansion is the single origin of the group's per-row witness. The +/// trace filler and `trace_builder::collect_bitwise_from_blake3` both iterate +/// it, so the BITWISE multiplicities cannot drift from the rows that send them. +#[derive(Debug, Clone)] +pub(crate) struct AbsorbRow { + /// Chaining value entering this row (`cv_in` on FIRST, the previous row's + /// output after that; on the END row, the group's result). + pub h: [u32; 8], + /// This row's message block; all-zero on the END row. + pub m: [u32; 16], + /// `first_flags` on the FIRST row, 0 elsewhere. + pub flags: u32, + /// Blocks left including this row's; 0 on the END row. + pub remaining: u32, + /// Address of this row's block. On the END row this is one past the last + /// block, which may WRAP when a message ends at the top of memory — the + /// chain carries it and nothing reads it, so no no-overflow constraint + /// applies to it. + pub m_base: u64, + pub first: bool, + pub end: bool, +} + +/// Expand one absorb ecall into its `num_blocks + 1` rows. +pub(crate) fn expand_absorb(op: &Blake3AbsorbOperation) -> Vec { + let n = op.blocks.len(); + let mut rows = Vec::with_capacity(n + 1); + let mut cv = op.cv_in; + for (i, block) in op.blocks.iter().enumerate() { + let flags = if i == 0 { op.first_flags } else { 0 }; + rows.push(AbsorbRow { + h: cv, + m: *block, + flags, + remaining: (n - i) as u32, + m_base: op.msg_addr.wrapping_add((i as u64) * 64), + first: i == 0, + end: false, + }); + // The chaining value the chip carries forward is the compression's first + // eight output words — the same `out[0..8]` the internal bus sends. + let out = ValueFlow::compute(&cv, block, 0, 64, flags).out; + cv = out[0..8].try_into().expect("out[0..8] is 8 words"); + } + rows.push(AbsorbRow { + h: cv, + m: [0; 16], + flags: 0, + remaining: 0, + m_base: op.msg_addr.wrapping_add((n as u64) * 64), + first: false, + end: true, + }); + rows +} + +/// Write a 32-bit word as 4 byte cells at `col..col+4`. +#[inline] +fn set_word_bytes(table: &mut T, row: usize, col: usize, w: u32) { + for b in 0..4 { + table.set_u64(row, col + b, ((w >> (8 * b)) & 0xFF) as u64); + } +} + +/// Fill one row's mixing core and feed-forward output from a computed flow, +/// cell-exactly in [`WireFlow`]'s canonical order. Shared by both modes: an +/// absorb compression row is the same 814-constraint core as a single one. +fn fill_mixing_core(table: &mut T, row: usize, flow: &ValueFlow) { + { + let mut a3 = flow.add3s.iter(); + let mut a2 = flow.add2s.iter(); + let mut xo = flow.xors.iter(); + let mut ro = flow.rots.iter(); + for g in 0..NUM_G { + let base = cols::g_base(g); + for half in 0..2 { + let (s_off, c_off, x_off, c2_off, x2_off, r_off) = if half == 0 { + ( + cols::G_A1, + cols::G_A1_C, + cols::G_X1, + cols::G_C1, + cols::G_X2, + cols::G_R1, + ) + } else { + ( + cols::G_A2, + cols::G_A2_C, + cols::G_X3, + cols::G_C2, + cols::G_X4, + cols::G_R2, + ) + }; + let &(s, c1, c2) = a3.next().expect("add3 count"); + set_word_bytes(table, row, base + s_off, s); + table.set_u64(row, base + c_off, c1 as u64); + table.set_u64(row, base + c_off + 1, c2 as u64); + + let &(_, _, x) = xo.next().expect("xor count"); + set_word_bytes(table, row, base + x_off, x); + + let &c = a2.next().expect("add2 count"); + set_word_bytes(table, row, base + c2_off, c); + + let &(_, _, x2) = xo.next().expect("xor count"); + set_word_bytes(table, row, base + x2_off, x2); + + let &(sll_lo, sllc_lo, sll_hi, sllc_hi, y) = ro.next().expect("rot count"); + table.set_u64(row, base + r_off, (sll_lo & 0xFF) as u64); + table.set_u64(row, base + r_off + 1, (sll_lo >> 8) as u64); + table.set_u64(row, base + r_off + 2, (sllc_lo & 0xFF) as u64); + table.set_u64(row, base + r_off + 3, (sllc_lo >> 8) as u64); + table.set_u64(row, base + r_off + 4, (sll_hi & 0xFF) as u64); + table.set_u64(row, base + r_off + 5, (sll_hi >> 8) as u64); + table.set_u64(row, base + r_off + 6, (sllc_hi & 0xFF) as u64); + table.set_u64(row, base + r_off + 7, (sllc_hi >> 8) as u64); + set_word_bytes(table, row, base + r_off + 8, y); + } + } + // Feed-forward outputs (the last 16 entries of flow.xors). + for i in 0..16 { + set_word_bytes(table, row, cols::out_word(i, 0), flow.out[i]); + } + } +} + +/// Fill the compression inputs `h | m | t_lo t_hi block_len flags` of one row. +fn fill_compression_inputs( + table: &mut T, + row: usize, + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) { + for (i, &w) in h.iter().enumerate() { + set_word_bytes(table, row, cols::in_word(i, 0), w); + } + for (i, &w) in m.iter().enumerate() { + set_word_bytes(table, row, cols::in_word(8 + i, 0), w); + } + set_word_bytes(table, row, cols::in_word(24, 0), t as u32); + set_word_bytes(table, row, cols::in_word(25, 0), (t >> 32) as u32); + set_word_bytes(table, row, cols::in_word(26, 0), block_len); + set_word_bytes(table, row, cols::in_word(27, 0), flags); +} + +/// Generate the BLAKE3 trace: the single-compression rows first, then each +/// absorb ecall's group of `num_blocks + 1` rows. +/// +/// Groups are laid down contiguously, but nothing depends on that — every +/// cross-row link goes through the [`BusId::Blake3Absorb`] bus, which is keyed +/// on the ecall's timestamp and is therefore order-free. +pub fn generate_blake3_trace( + ops: &[Blake3Operation], + absorb_ops: &[Blake3AbsorbOperation], +) -> TraceTable { + let absorb_rows: Vec<(&Blake3AbsorbOperation, AbsorbRow)> = absorb_ops + .iter() + .flat_map(|op| expand_absorb(op).into_iter().map(move |r| (op, r))) + .collect(); + let n = ops.len() + absorb_rows.len(); + let num_rows = n.next_power_of_two().max(4); + let mut trace = TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), + cols::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + + for (row, op) in ops.iter().enumerate() { + table.set_dword_wl(row, cols::TIMESTAMP_0, op.timestamp); + table.set_dword_bl(row, cols::addr(0), op.state_addr); + + // Pointers ptr[k] = addr + 8k. All 22: single mode reads the whole + // 176-byte state region, whose top the executor's range check binds. + for k in 0..STATE_DWORDS { + let ptr = op + .state_addr + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + table.set_dword_hl(row, cols::ptr(k, 0), ptr); + } + + fill_compression_inputs(table, row, &op.h, &op.m, op.t, op.block_len, op.flags); + + // The mixing core, cell-exactly in canonical order. + let flow = ValueFlow::compute(&op.h, &op.m, op.t, op.block_len, op.flags); + debug_assert_eq!( + flow.out, op.out, + "trace-builder output must match the executor" + ); + fill_mixing_core(table, row, &flow); + + // Previous content of the out region. + for b in 0..64 { + table.set_u64(row, cols::old_out(b), op.old_out[b] as u64); + } + + table.set_fe(row, cols::MU, FE::one()); + table.set_fe(row, cols::MU_S, FE::one()); + } + + for (i, (op, r)) in absorb_rows.iter().enumerate() { + let row = ops.len() + i; + table.set_dword_wl(row, cols::TIMESTAMP_0, op.timestamp); + table.set_dword_bl(row, cols::addr(0), op.ctrl_addr); + + // Only the control region's 8 pointers. ptr[8..22] stay 0 and their + // constraints and range checks are gated on MU_S — an absorb's x10 is + // bounded by the executor at ctrl + 63, NOT at ctrl + 168, so computing + // the upper pointers here would panic on an address the ABI accepts. + for k in 0..CTRL_DWORDS { + let ptr = op + .ctrl_addr + .checked_add(k as u64 * 8) + .expect("absorb control region range must be validated by the executor"); + table.set_dword_hl(row, cols::ptr(k, 0), ptr); + } + + // t = 0 and block_len = 64 on every absorbed block: the interior + // schedule, constrained on the row rather than assumed. + fill_compression_inputs(table, row, &r.h, &r.m, 0, 64, r.flags); + + table.set_fe(row, cols::MU, FE::one()); + table.set_fe(row, cols::MU_A, FE::one()); + table.set_bool(row, cols::FIRST, r.first); + table.set_bool(row, cols::END, r.end); + table.set_u64(row, cols::REMAINING, r.remaining as u64); + table.set_dword_hl(row, cols::M_BASE, r.m_base); + + if r.end { + // The END row does no compression; the chain receive lands the + // group's result in `h`, and the cv_out write reads those columns. + for (b, &old) in op.old_cv_out.iter().enumerate() { + table.set_u64(row, cols::old_out(b), old as u64); + } + } else { + table.set_fe(row, cols::MU_C, FE::one()); + table.set_u64(row, cols::REM_DECR, (r.remaining - 1) as u64); + table.set_dword_hl(row, cols::M_BASE_INCR, r.m_base.wrapping_add(64)); + // The last compressing row, and only it, may let M_BASE + 64 leave + // the address space. + table.set_bool(row, cols::NEXT_IS_END, r.remaining == 1); + for j in 0..MSG_DWORDS { + let ptr = r + .m_base + .checked_add(j as u64 * 8) + .expect("absorb message range must be validated by the executor"); + table.set_dword_hl(row, cols::msg_ptr(j, 0), ptr); + } + let flow = ValueFlow::compute(&r.h, &r.m, 0, 64, r.flags); + fill_mixing_core(table, row, &flow); + } + if r.first { + // The 8-alignment witness for x11: M_BASE's low halfword is 8·Q. + table.set_u64(row, cols::MSG_ALIGN_Q, (r.m_base & 0xFFFF) / 8); + } + } + + // Padding rows: ptr[k][0] = 8k (all fit in the low halfword), matching the + // keccak pad idiom. μ = 0 gates every constraint and interaction. + for row in n..num_rows { + for k in 0..STATE_DWORDS { + table.set_u64(row, cols::ptr(k, 0), (k as u64) * 8); + } + } + + trace +} + +// ========================================================================= +// Bus interactions +// ========================================================================= + +/// Order groups: I/O (Ecall + reg-read + 22 Memw), then the mixing core's +/// ByteAlu XORs (canonical WireFlow order), then the shift AreBytes, then the +/// message/old-out/addr AreBytes, the alignment AND and the pointer IS_HALFs, +/// then the absorb mode's own block (registers, message, `cv_out`, the chain +/// bus, the countdown lookups and the absorb-only range checks). +/// +/// ## Per-mode multiplicities +/// +/// | interaction | multiplicity | +/// |---|---| +/// | `Ecall` receive, single syscall | `MU_S` | +/// | `Ecall` receive, absorb syscall | `FIRST` | +/// | x10 read (state addr / control addr) | `MU_S + FIRST` | +/// | state dwords 0..4 (`h` / `cv_in`) | `MU_S + FIRST` | +/// | state dwords 4..22 (`m`, `t|len|flags`, `out`) | `MU_S` | +/// | x11/x12/x13 reads | `FIRST` | +/// | message block read (8 dwords) | `MU_C` | +/// | `cv_out` write (4 dwords) | `END` | +/// | chain send / receive | `MU_C` / `MU_A − FIRST` | +/// | `Zero[REMAINING] → END` | `MU_A` | +/// | mixing core (ByteAlu, AreBytes) | `MU − END` | +/// | `h` byte range check | `END` (the one row the mixing core does not cover) | +/// +/// ★ The two `Ecall` receives cannot be one interaction with multiplicity +/// `MU_S + FIRST`: the syscall number is a CONSTANT in the tuple and the two +/// modes have different ones. Merging them would make every single-compression +/// row claim the absorb syscall (or vice versa). +pub fn bus_interactions() -> Vec { + let syscall_lo = BLAKE3_SYSCALL_NUMBER & 0xFFFF_FFFF; + let syscall_hi = BLAKE3_SYSCALL_NUMBER >> 32; + let absorb_syscall_lo = BLAKE3_ABSORB_SYSCALL_NUMBER & 0xFFFF_FFFF; + let absorb_syscall_hi = BLAKE3_ABSORB_SYSCALL_NUMBER >> 32; + let wires = WireFlow::build(); + let mut interactions = Vec::with_capacity(1500); + + // Reusable multiplicities. `MU − END` covers both modes' compression rows: + // single rows have END = 0, so the single mode's gating is unchanged. + let mu_s_or_first = Multiplicity::Sum(cols::MU_S, cols::FIRST); + let compressing = Multiplicity::Diff(cols::MU, cols::END); + + let byte_bus_value = |b: ByteRef| -> BusValue { + match b { + ByteRef::Col(c) => BusValue::Packed { + start_column: c, + packing: Packing::Direct, + }, + ByteRef::Const(v) => BusValue::constant(v as u64), + } + }; + let direct = |c: usize| -> BusValue { + BusValue::Packed { + start_column: c, + packing: Packing::Direct, + } + }; + // A 32-bit word from 4 byte columns, LSB first. + let word_of_bytes = |start: usize| -> BusValue { + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: start, + }, + LinearTerm::Column { + coefficient: 256, + column: start + 1, + }, + LinearTerm::Column { + coefficient: 65536, + column: start + 2, + }, + LinearTerm::Column { + coefficient: 16777216, + column: start + 3, + }, + ]) + }; + // One 32-bit half of a DWordHL pair (2 halfword columns), LSB first. + let dword_hl_half = |start: usize| -> BusValue { + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: start, + }, + LinearTerm::Column { + coefficient: 65536, + column: start + 1, + }, + ]) + }; + let addr_word = |lo_byte: usize| -> BusValue { word_of_bytes(cols::addr(lo_byte)) }; + + // 1. ECALL receiver: [ts_lo, ts_hi, syscall_lo32, syscall_hi32]. + interactions.push(BusInteraction::receiver( + BusId::Ecall, + Multiplicity::Column(cols::MU_S), + vec![ + direct(cols::TIMESTAMP_0), + direct(cols::TIMESTAMP_1), + BusValue::constant(syscall_lo), + BusValue::constant(syscall_hi), + ], + )); + + // 2. MEMW read of register x10 binding the state address (keccak idiom): + // [old(8), is_register=1, base=20, value(8), ts(2), w2=1, w4=0, w8=0]. + // In absorb mode x10 is the control-region address and only the group's + // FIRST row reads it; the rest of the group receives it over the chain bus. + { + let mut values = Vec::with_capacity(24); + values.push(addr_word(0)); + values.push(addr_word(4)); + for _ in 2..8 { + values.push(BusValue::constant(0)); + } + values.push(BusValue::constant(1)); // is_register + values.push(BusValue::constant(20)); // x10 → address 2*10 + values.push(BusValue::constant(0)); + values.push(addr_word(0)); + values.push(addr_word(4)); + for _ in 2..8 { + values.push(BusValue::constant(0)); + } + values.push(BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }); + values.push(BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }); + values.push(BusValue::constant(1)); // w2 (register) + values.push(BusValue::constant(0)); + values.push(BusValue::constant(0)); + interactions.push(BusInteraction::sender( + BusId::Memw, + mu_s_or_first.clone(), + values, + )); + } + + // An 8-byte memory access: [old(8), is_register=0, addr(2), value(8), ts(2), + // w2=0, w4=0, w8=1], addressed by a DWordHL pointer. A pure read passes the + // same columns as `old` and `value`; a write passes the previous content as + // `old`. Shared by the state region, the absorb message and `cv_out`. + let mem_dword = + |ptr_col: usize, old_cols: &[usize; 8], val_cols: &[usize; 8]| -> Vec { + let mut values = Vec::with_capacity(24); + for &c in old_cols { + values.push(direct(c)); + } + values.push(BusValue::constant(0)); // is_register + values.push(dword_hl_half(ptr_col)); + values.push(dword_hl_half(ptr_col + 2)); + for &c in val_cols { + values.push(direct(c)); + } + values.push(direct(cols::TIMESTAMP_0)); + values.push(direct(cols::TIMESTAMP_1)); + values.push(BusValue::constant(0)); + values.push(BusValue::constant(0)); + values.push(BusValue::constant(1)); // w8 + values + }; + let eight_from = |base: usize| -> [usize; 8] { core::array::from_fn(|b| base + b) }; + + // 3. MEMW per state dword. Input dwords are pure reads (old = value = input + // bytes); output dwords write OUT over OLD_OUT. + // + // Dwords 0..4 are the chaining value. In absorb mode the SAME interaction + // reads `cv_in` into the same `h` columns from the same `PTR[0..4]` + // addresses — the control region's layout was chosen so it could. Only the + // group's FIRST row reads it; later rows take `h` off the chain bus. + for k in 0..STATE_DWORDS { + // (old bytes, value bytes) column bases for this dword. + let (old_base, val_base): ([usize; 8], [usize; 8]) = if k < IN_DWORDS { + let c = eight_from(cols::in_word(2 * k, 0)); + (c, c) + } else { + let o = k - IN_DWORDS; + ( + eight_from(cols::old_out(o * 8)), + eight_from(cols::out_word(2 * o, 0)), + ) + }; + let mult = if k < CV_DWORDS { + mu_s_or_first.clone() + } else { + Multiplicity::Column(cols::MU_S) + }; + interactions.push(BusInteraction::sender( + BusId::Memw, + mult, + mem_dword(cols::ptr(k, 0), &old_base, &val_base), + )); + } + + // 4. Mixing core + feed-forward: ByteAlu[XOR] per byte, canonical order. + // Gated on `MU − END`: the END row of an absorb group does no compression + // and has no witness for these. Single rows have END = 0, so `MU − END = MU` + // and the single mode's gating is bit-identical to before. + for xw in &wires.xors { + for b in 0..4 { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + compressing.clone(), + vec![ + BusValue::constant(alu_op::XOR as u64), + byte_bus_value(xw.a.byte(b)), + byte_bus_value(xw.b.byte(b)), + direct(xw.out[b]), + ], + )); + } + } + + // 5. Shift-halfword AreBytes: 4 pairs per rotation + // (SLL_lo, SLLC_lo, SLL_hi, SLLC_hi bytes). + for rw in &wires.rots { + for pair in [rw.sll_lo, rw.sllc_lo, rw.sll_hi, rw.sllc_hi] { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + compressing.clone(), + vec![direct(pair[0]), direct(pair[1])], + )); + } + } + + // 6. Message AreBytes (m is never XORed — DESIGN §4.7/§7.5): 32 pairs. + // Absorb rows read `m` from memory just as single rows do, and MEMW does not + // range-check what it carries, so the same check covers both modes. + for i in 0..16 { + for p in 0..2 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + compressing.clone(), + vec![ + direct(cols::in_word(8 + i, 2 * p)), + direct(cols::in_word(8 + i, 2 * p + 1)), + ], + )); + } + } + + // 6b. ★ `h` AreBytes on the END row ONLY — the one row whose mixing core is + // gated off, and therefore the one row where `h`'s bytes are not already + // range-checked as XOR operands (soundness ledger 14). Without it a prover + // could satisfy the chain receive with non-canonical bytes summing to the + // right word and have the `cv_out` write place a "byte" ≥ 256 in memory, + // which a later read would faithfully return. + for p in 0..16 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::END), + vec![direct(cols::IN + 2 * p), direct(cols::IN + 2 * p + 1)], + )); + } + + // 7. OLD_OUT AreBytes: those bytes only ride the Memw bus; without a byte + // range check their packed linear combinations alias (same argument as the + // addr bytes in keccak.rs). + for p in 0..32 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: cols::old_out(2 * p), + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::old_out(2 * p + 1), + packing: Packing::Direct, + }, + ], + )); + } + + // 8. Address byte range checks (4 pairs) + alignment addr[0] & 7 = 0. + for i in 0..4 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: cols::addr(2 * i), + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::addr(2 * i + 1), + packing: Packing::Direct, + }, + ], + )); + } + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(cols::MU), + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: cols::addr(0), + packing: Packing::Direct, + }, + BusValue::constant(7), + BusValue::constant(0), + ], + )); + + // 9. IS_HALF range checks on the 22 pointers' halfwords. The control + // region's 8 are live in both modes; the 14 above it only in single mode, + // where the executor's `state_addr + 175` check bounds them (an absorb's x10 + // is only bounded at `ctrl + 63`). + for k in 0..STATE_DWORDS { + let mult = if k < CTRL_DWORDS { + Multiplicity::Column(cols::MU) + } else { + Multiplicity::Column(cols::MU_S) + }; + for hw in 0..4 { + interactions.push(BusInteraction::sender( + BusId::IsHalfword, + mult.clone(), + vec![direct(cols::ptr(k, hw))], + )); + } + } + + // ===================================================================== + // 10. Absorb mode + // ===================================================================== + + // 10a. ECALL receiver for the absorb syscall, on the group's FIRST row. + interactions.push(BusInteraction::receiver( + BusId::Ecall, + Multiplicity::Column(cols::FIRST), + vec![ + direct(cols::TIMESTAMP_0), + direct(cols::TIMESTAMP_1), + BusValue::constant(absorb_syscall_lo), + BusValue::constant(absorb_syscall_hi), + ], + )); + + // 10b. Register reads x11/x12/x13 on the FIRST row (x10 is shared with the + // single mode above). Register accesses put [lo32, hi32] in the first two + // value slots and zero in the rest. + // + // The `hi32 = 0` constants are load-bearing, not padding: + // * x12 pins `num_blocks` to the single field element `REMAINING`, so a + // count above 2^32 cannot be smuggled in through the high half; + // * x13 pins `first_flags < 2^32`, which is what makes it fit the chip's + // one-word flags column — the same bound the executor rejects above. + { + let reg_read = |base_addr: u64, lo: BusValue, hi: BusValue| -> Vec { + let mut values = Vec::with_capacity(24); + values.push(lo.clone()); + values.push(hi.clone()); + for _ in 2..8 { + values.push(BusValue::constant(0)); + } + values.push(BusValue::constant(1)); // is_register + values.push(BusValue::constant(base_addr)); + values.push(BusValue::constant(0)); + values.push(lo); + values.push(hi); + for _ in 2..8 { + values.push(BusValue::constant(0)); + } + values.push(direct(cols::TIMESTAMP_0)); + values.push(direct(cols::TIMESTAMP_1)); + values.push(BusValue::constant(1)); // w2 (register) + values.push(BusValue::constant(0)); + values.push(BusValue::constant(0)); + values + }; + // x11 → message base. + interactions.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::FIRST), + reg_read( + 22, + dword_hl_half(cols::M_BASE), + dword_hl_half(cols::M_BASE + 2), + ), + )); + // x12 → num_blocks, which IS the group's initial countdown. + interactions.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::FIRST), + reg_read(24, direct(cols::REMAINING), BusValue::constant(0)), + )); + // x13 → first_flags, landing in the flags column the compression reads. + interactions.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::FIRST), + reg_read( + 26, + word_of_bytes(cols::in_word(27, 0)), + BusValue::constant(0), + ), + )); + } + + // 10c. The message block: 8 dword reads at MSG_PTR[j], into the same `m` + // columns the single mode reads out of its state region. Gated on `MU_C` — + // the END row reads no block. + for j in 0..MSG_DWORDS { + let c = eight_from(cols::in_word(8 + 2 * j, 0)); + interactions.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::MU_C), + mem_dword(cols::msg_ptr(j, 0), &c, &c), + )); + } + + // 10d. `cv_out`: 4 dword writes at PTR[4..8] on the END row, whose value is + // the `h` the chain receive delivered. Disjoint from `cv_in` by the ABI, so + // the two never touch one address at one timestamp. + for k in 0..CV_DWORDS { + interactions.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::END), + mem_dword( + cols::ptr(CV_OUT_DWORD + k, 0), + &eight_from(cols::old_out(k * 8)), + &eight_from(cols::in_word(2 * k, 0)), + ), + )); + } + + // 10e. ★ The chain. Both endpoints come from ONE builder, so the send and + // the receive cannot drift apart — the failure DESIGN.md §1.1 describes is + // exactly a drift between them. The tuple leads with the timestamp (which + // identifies the group) and carries the control address and message base as + // well as the chaining value: without `M_BASE` a prover could point any row + // at a block of its choosing and still balance MEMW, because reading some + // other address is a legitimate read. + // + // `ADDR` rides for a different reason. The END row needs the control address + // to write `cv_out`, and it cannot get it from a second x10 read, because no + // address may be touched twice at one timestamp. That claim rests on MEMW's + // token argument, and the short form of it ("LT(T, T) → 1 is unsatisfiable") + // is under-argued — a second access at T could consume an OLDER token rather + // than the one the first access produced. The argument that does hold: + // + // Per address, MEMW's genesis and finalization tokens have cardinality + // one, so the accesses to that address form a PERFECT MATCHING between + // produced and consumed tokens — a disjoint union of paths and cycles. + // Every edge carries `old_ts < ts` strictly, so no cycle can close, and + // the union is a single strictly increasing path. Timestamps along that + // path are therefore pairwise distinct, for ANY pair of accesses and not + // merely adjacent ones. Two accesses at one timestamp cannot both sit on + // it, so the trace has no such pair. + // + // That is also what makes the ABI's control/message disjointness enforceable + // without a chip constraint — see the note on the region overlap check. + let chain_values = |remaining_col: usize, m_base_col: usize, cv_base: usize| -> Vec { + let mut values = Vec::with_capacity(15); + values.push(direct(cols::TIMESTAMP_0)); + values.push(direct(cols::TIMESTAMP_1)); + values.push(direct(remaining_col)); + values.push(dword_hl_half(m_base_col)); + values.push(dword_hl_half(m_base_col + 2)); + values.push(addr_word(0)); + values.push(addr_word(4)); + for i in 0..8 { + values.push(word_of_bytes(cv_base + i * 4)); + } + values + }; + interactions.push(BusInteraction::sender( + BusId::Blake3Absorb, + Multiplicity::Column(cols::MU_C), + chain_values(cols::REM_DECR, cols::M_BASE_INCR, cols::OUT), + )); + interactions.push(BusInteraction::receiver( + BusId::Blake3Absorb, + Multiplicity::Diff(cols::MU_A, cols::FIRST), + chain_values(cols::REMAINING, cols::M_BASE, cols::IN), + )); + + // 10f. `END` is DERIVED, never free: ZERO[REMAINING] → END. The lookup also + // bounds `REMAINING < 2^20` (the BITWISE ZERO table's domain), which is what + // stops a countdown from wrapping the field back to zero, and with it the + // "chain of rows with no FIRST and no END" the bus would otherwise balance. + interactions.push(BusInteraction::sender( + BusId::Zero, + Multiplicity::Column(cols::MU_A), + vec![direct(cols::REMAINING), direct(cols::END)], + )); + + // 10g. ★ The block cap, in-circuit. `IsB20` admits [0, 2^20), so + // `REM_DECR · 2^10 ∈ IsB20 ⟺ REM_DECR < 2^10` — no wraparound, since the + // ZERO lookup already put `REMAINING` under 2^20 and the product is < 2^30. + // With `FIRST · END = 0` forcing `REMAINING ≠ 0`, the group's block count is + // exactly the `1..=1024` the executor accepts, rather than inherited from it. + interactions.push(BusInteraction::sender( + BusId::IsB20, + Multiplicity::Column(cols::FIRST), + vec![BusValue::linear(vec![LinearTerm::Column { + coefficient: ABSORB_CAP_SCALE as i64, + column: cols::REM_DECR, + }])], + )); + + // 10g-bis. ★ `NEXT_IS_END = (REM_DECR == 0)` — "this is the group's LAST + // compressing row". It exists to let exactly that row's `M_BASE + 64` leave + // the address space (a message may legally end at the top of memory) while + // every earlier row's may not; see the constraint that gates on it. + // + // The same send also bounds `REM_DECR < 2^20` directly on compressing rows, + // which is what makes the block cap's `IsB20[REM_DECR · 2^10]` above safe + // from field wraparound at its own site rather than only via `REMAINING`. + interactions.push(BusInteraction::sender( + BusId::Zero, + Multiplicity::Column(cols::MU_C), + vec![direct(cols::REM_DECR), direct(cols::NEXT_IS_END)], + )); + + // 10g-ter. ★ x11 is 8-aligned: `M_BASE[0] = 8·Q` with `Q` a halfword. + // + // The bound on `Q` has to come from a lookup whose ARGUMENT is `Q` itself. + // Scaling it the way the block cap scales `REM_DECR` — `IsB20[Q · 2^7]` — + // does NOT work here and is worth writing down, because it looks identical: + // with `Q` otherwise unconstrained a prover sets `Q = M_BASE[0] · 8⁻¹ mod p`, + // and then `Q · 2^7 = M_BASE[0] · 16`, which is under 2^20 for ANY halfword + // `M_BASE[0]` — aligned or not. The scaled form is vacuous. The block cap + // survives the same attack only because `REMAINING`'s domain bound arrives + // first from a different lookup. + // + // `IsHalfword[Q]` closes it: `Q < 2^16` as an INTEGER, so `8Q < 2^19 < p` is + // computed without reduction, and `8Q = M_BASE[0] < 2^16` then forces + // `Q < 2^13` and `8 | M_BASE[0]`. Since `8 | 65536`, a multiple of 8 in the + // low halfword is exactly an 8-aligned address. + interactions.push(BusInteraction::sender( + BusId::IsHalfword, + Multiplicity::Column(cols::FIRST), + vec![direct(cols::MSG_ALIGN_Q)], + )); + + // 10h. IS_HALF on the absorb-only pointer arithmetic: the message base (live + // on every absorb row, including END, where it rides the chain), its + // increment and the 8 message dword pointers. + for hw in 0..4 { + interactions.push(BusInteraction::sender( + BusId::IsHalfword, + Multiplicity::Column(cols::MU_A), + vec![direct(cols::M_BASE + hw)], + )); + } + for hw in 0..4 { + interactions.push(BusInteraction::sender( + BusId::IsHalfword, + Multiplicity::Column(cols::MU_C), + vec![direct(cols::M_BASE_INCR + hw)], + )); + } + for j in 0..MSG_DWORDS { + for hw in 0..4 { + interactions.push(BusInteraction::sender( + BusId::IsHalfword, + Multiplicity::Column(cols::MU_C), + vec![direct(cols::msg_ptr(j, hw))], + )); + } + } + + interactions +} + +// ========================================================================= +// Single-source constraint set +// ========================================================================= + +/// The BLAKE3 table's transition constraints (851 total): +/// - idx 0..44: 22 pointer `ADD` carry pairs (`ptr[k] = addr + 8k`); the +/// control region's 8 are μ-gated, the 14 above it MU_S-gated; +/// - idx 44, 45: top-pointer no-overflow, once per mode — `MU_S·carry_1` of +/// `addr + 168 = ptr[21]` and `MU_A·carry_1` of `addr + 56 = ptr[7]`; +/// - idx 46..334: all 96 add3 groups (sum identity + 2 carry booleanities); +/// - idx 334..430: all 96 add2 expression-carry booleanities; +/// - idx 430..814: all 96 rotations (2 shift identities + 2 recombine each). +/// NOTE the grouping is by op type across the whole row, NOT per G — G #g's +/// 16 constraints are scattered across the three bands. +/// - idx 814: `IS_BIT(MU)` — μ·(1−μ) = 0, ungated. The bus argument pins +/// μ to {0,1} indirectly (the Ecall receive anchors μ>0 rows to a CPU ecall +/// whose ECALL flag is IS_BIT; MEMW's width flags are boolean), but that is +/// an inter-table argument — this makes it local, matching ecsm/commit. +/// +/// Absorb mode (idx 815..848): +/// - idx 815..819: `IS_BIT` on MU_S, MU_A, FIRST, END; +/// - idx 819..824: the mode/boundary algebra — `μ = MU_S + MU_A`, +/// `MU_S·MU_A = 0`, `MU_C = MU_A − END`, the boundary lock +/// `(FIRST + END)·(1 − MU_A) = 0`, and `FIRST·END = 0`; +/// - idx 824: the countdown `MU_C·(REM_DECR + 1 − REMAINING) = 0`; +/// - idx 825..843: `M_BASE + 64 = M_BASE_INCR` and the 8 `msg_ptr[j]` ADD pairs; +/// - idx 843: `MU_C·carry_1` of `m_base + 56 = msg_ptr[7]`; +/// - idx 844..848: the interior schedule — `t = 0`, `block_len = 64`, and flags +/// zero on every compressing row but the FIRST; +/// - idx 848: x11 is 8-aligned — `FIRST·(M_BASE[0] − 8·Q) = 0`; +/// - idx 849: `NEXT_IS_END·(1 − MU_C) = 0`, so the flag is off the rows +/// whose `REM_DECR` the ZERO lookup does not pin; +/// - idx 850: `(MU_C − NEXT_IS_END)·carry_1` of `M_BASE + 64 = M_BASE_INCR` +/// — the message region wraps the address space on its LAST block or not at all. +/// +/// Every constraint is gated by the mode it belongs to, and the mixing core by +/// `μ − END`. Max degree 3 (the booleanities; identities are degree 2). +#[derive(Clone, Copy)] +pub struct Blake3Constraints; + +/// Word expression from a [`WordRef`]: b0 + 256·b1 + 2^16·b2 + 2^24·b3. +fn word_expr>( + b: &B, + w: &WordRef, +) -> B::Expr { + match w { + WordRef::Cols(c) => { + b.main(0, c[0]) + + b.main(0, c[1]) * b.const_base(256) + + b.main(0, c[2]) * b.const_base(65536) + + b.main(0, c[3]) * b.const_base(16777216) + } + WordRef::Const(v) => b.const_base(*v as u64), + } +} + +/// Halfword expression from 2 byte columns: b0 + 256·b1. +fn half_expr>( + b: &B, + c: &[usize; 2], +) -> B::Expr { + b.main(0, c[0]) + b.main(0, c[1]) * b.const_base(256) +} + +impl ConstraintSet for Blake3Constraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + use crate::constraints::templates::emit_add_pair; + + let wires = WireFlow::build(); + + // idx 0..44: ptr[k] = addr + 8k. The control region's 8 pointers are + // gated on μ (both modes address through them); the 14 above it on MU_S, + // because an absorb's x10 is bounded by the executor at ctrl + 63 and + // deriving ptr[21] = ctrl + 168 from it would demand a no-overflow the + // ABI never promised. + for k in 0..STATE_DWORDS { + let gate = if k < CTRL_DWORDS { + cols::MU + } else { + cols::MU_S + }; + emit_add_pair( + b, + k * 2, + &[gate], + &AddOperand::from_dword_bl(cols::ADDR), + &AddOperand::constant((k * 8) as i64), + &AddOperand::from_dword_hl(cols::ptr(k, 0)), + ); + } + + // idx 44, 45: top-pointer no-overflow, once per mode — MU_S over the + // 176-byte state region (`addr + 168 = ptr[21]`) and MU_A over the + // 64-byte control region (`addr + 56 = ptr[7]`). Each mode forbids the + // wrap only across the range it actually addresses, which is exactly the + // range the executor's own overflow check binds. + let mut idx = STATE_DWORDS * 2; + for (gate, last) in [ + (cols::MU_S, STATE_DWORDS - 1), + (cols::MU_A, CTRL_DWORDS - 1), + ] { + let c256 = b.const_base(256); + let c65536 = b.const_base(65536); + let c16777216 = b.const_base(16777216); + let addr_lo = b.main(0, cols::addr(0)) + + b.main(0, cols::addr(1)) * c256.clone() + + b.main(0, cols::addr(2)) * c65536.clone() + + b.main(0, cols::addr(3)) * c16777216.clone(); + let addr_hi = b.main(0, cols::addr(4)) + + b.main(0, cols::addr(5)) * c256 + + b.main(0, cols::addr(6)) * c65536.clone() + + b.main(0, cols::addr(7)) * c16777216; + let ptr_lo = + b.main(0, cols::ptr(last, 0)) + b.main(0, cols::ptr(last, 1)) * c65536.clone(); + let ptr_hi = b.main(0, cols::ptr(last, 2)) + b.main(0, cols::ptr(last, 3)) * c65536; + + let inv_2_32 = b.const_base(INV_SHIFT_32); + let off = b.const_base((8 * last) as u64); + let carry_0 = (addr_lo + off - ptr_lo) * inv_2_32.clone(); + let carry_1 = (addr_hi + carry_0 - ptr_hi) * inv_2_32; + let g = b.main(0, gate); + b.emit_base(idx, g * carry_1); + idx += 1; + } + + // Mixing core. Same canonical order as the wire builder records. + // + // ★ Gated on `μ − END`, not `μ`. The END row of an absorb group holds no + // compression witness — it exists to drain the chain and write `cv_out` + // — so it must not have to satisfy the 814 identities below. A single + // row has END = 0, so `μ − END = μ` and this mode is untouched. + let mu = |b: &B| b.main(0, cols::MU) - b.main(0, cols::END); + let two_32 = b.const_base(1u64 << 32); + let inv_2_32 = b.const_base(INV_SHIFT_32); + + // add3: μ·(a + b + m − s − 2^32·(c1+c2)) = 0; μ·ci·(1−ci) = 0. + for aw in &wires.add3s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let m_w = word_expr(b, &WordRef::Cols(aw.m)); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let c1 = b.main(0, aw.c1); + let c2 = b.main(0, aw.c2); + let sum_id = a + bb + m_w - s - (c1.clone() + c2.clone()) * two_32.clone(); + let m = mu(b); + b.emit_base(idx, m * sum_id); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c1.clone() * (one - c1)); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c2.clone() * (one - c2)); + idx += 1; + } + + // add2: carry = (a + b − s)·2^−32; μ·carry·(1−carry) = 0. + for aw in &wires.add2s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let carry = (a + bb - s) * inv_2_32.clone(); + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * carry.clone() * (one - carry)); + idx += 1; + } + + // Rotations: 2 shift identities + 2 recombine identities each. + for rw in &wires.rots { + let (xlo, xhi) = match &rw.input { + WordRef::Cols(c) => (half_expr(b, &[c[0], c[1]]), half_expr(b, &[c[2], c[3]])), + WordRef::Const(_) => unreachable!("shift inputs are always committed XOR outputs"), + }; + let sll_lo = half_expr(b, &rw.sll_lo); + let sllc_lo = half_expr(b, &rw.sllc_lo); + let sll_hi = half_expr(b, &rw.sll_hi); + let sllc_hi = half_expr(b, &rw.sllc_hi); + let ylo = half_expr(b, &[rw.y[0], rw.y[1]]); + let yhi = half_expr(b, &[rw.y[2], rw.y[3]]); + let two_r = b.const_base(1u64 << rw.r); + let two_16 = b.const_base(65536); + + // μ·(xlo·2^r − SLLC_lo·2^16 − SLL_lo) = 0 (and hi). + let m = mu(b); + b.emit_base( + idx, + m * (xlo * two_r.clone() - sllc_lo.clone() * two_16.clone() - sll_lo.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base( + idx, + m * (xhi * two_r - sllc_hi.clone() * two_16 - sll_hi.clone()), + ); + idx += 1; + // μ·(Ylo − SLL_hi − SLLC_lo) = 0; μ·(Yhi − SLL_lo − SLLC_hi) = 0. + let m = mu(b); + b.emit_base(idx, m * (ylo - sll_hi - sllc_lo)); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (yhi - sll_lo - sllc_hi)); + idx += 1; + } + + // IS_BIT(MU) — ungated booleanity, degree 2. See the struct doc for why + // this is emitted even though the bus argument already pins μ + // indirectly. + crate::constraints::templates::emit_is_bit(b, idx, cols::MU, None); + idx += 1; + + // ================================================================= + // Absorb mode + // ================================================================= + + // The mode and boundary flags are bits, ungated for the same reason + // IS_BIT(MU) is: the alternative is an inter-table argument. + for col in [cols::MU_S, cols::MU_A, cols::FIRST, cols::END] { + crate::constraints::templates::emit_is_bit(b, idx, col, None); + idx += 1; + } + + // μ = MU_S + MU_A: the two modes partition the real rows, so every + // interaction gated on μ fires exactly once per real row whichever + // mode it is in. + { + let root = b.main(0, cols::MU) - b.main(0, cols::MU_S) - b.main(0, cols::MU_A); + b.emit_base(idx, root); + idx += 1; + } + + // ★ The modes are exclusive. Without this a row could set both and + // receive BOTH `Ecall` tuples — one row answering two syscalls. + { + let root = b.main(0, cols::MU_S) * b.main(0, cols::MU_A); + b.emit_base(idx, root); + idx += 1; + } + + // MU_C = MU_A − END: "an absorb row that compresses", as a column so the + // audited `emit_add_pair` template can gate on it (its condition is a + // SUM of columns, not an arbitrary expression). + { + let root = b.main(0, cols::MU_C) + b.main(0, cols::END) - b.main(0, cols::MU_A); + b.emit_base(idx, root); + idx += 1; + } + + // ★ (FIRST + END)·(1 − MU_A) = 0 — COMMIT's boundary lock (commit.rs + // idx 3), retargeted from μ to MU_A. A padding row (μ = 0) or a + // single-compression row cannot mint a group boundary: FIRST would let + // it receive a second `Ecall`, END would let it write a `cv_out`. + { + let one = b.one(); + let root = + (b.main(0, cols::FIRST) + b.main(0, cols::END)) * (one - b.main(0, cols::MU_A)); + b.emit_base(idx, root); + idx += 1; + } + + // FIRST · END = 0 — no zero-block group. The executor rejects + // `num_blocks = 0`; a group that was its own END row would copy `cv_in` + // to `cv_out` and prove an ecall the VM semantics never accept. + { + let root = b.main(0, cols::FIRST) * b.main(0, cols::END); + b.emit_base(idx, root); + idx += 1; + } + + // The countdown: REM_DECR + 1 = REMAINING on every compressing row. + // Combined with the chain (which carries REM_DECR forward as the next + // row's REMAINING) and ZERO[REMAINING] → END, the group has exactly + // `num_blocks` compressing rows and one END row. + { + let one = b.one(); + let root = b.main(0, cols::MU_C) + * (b.main(0, cols::REM_DECR) + one - b.main(0, cols::REMAINING)); + b.emit_base(idx, root); + idx += 1; + } + + // Message pointer arithmetic, MU_C-gated: the next block's base and the + // 8 dword pointers into this one. + emit_add_pair( + b, + idx, + &[cols::MU_C], + &AddOperand::from_dword_hl(cols::M_BASE), + &AddOperand::constant(64), + &AddOperand::from_dword_hl(cols::M_BASE_INCR), + ); + idx += 2; + for j in 0..MSG_DWORDS { + emit_add_pair( + b, + idx, + &[cols::MU_C], + &AddOperand::from_dword_hl(cols::M_BASE), + &AddOperand::constant((j * 8) as i64), + &AddOperand::from_dword_hl(cols::msg_ptr(j, 0)), + ); + idx += 2; + } + + // No-overflow on the block's own 64 bytes (`m_base + 56 = msg_ptr[7]`), + // which the executor's `msg_addr + 64·num_blocks − 1` check bounds. + // + // ★ Deliberately NOT applied to M_BASE_INCR: a message ending exactly at + // the top of memory makes the last row's `m_base + 64` equal 2^64, and + // that value only rides the chain to the END row, which never reads it. + // Forbidding the wrap there would reject an absorb the ABI accepts. + { + let c65536 = b.const_base(65536); + let last = MSG_DWORDS - 1; + let base_lo = b.main(0, cols::M_BASE) + b.main(0, cols::M_BASE + 1) * c65536.clone(); + let base_hi = + b.main(0, cols::M_BASE + 2) + b.main(0, cols::M_BASE + 3) * c65536.clone(); + let ptr_lo = b.main(0, cols::msg_ptr(last, 0)) + + b.main(0, cols::msg_ptr(last, 1)) * c65536.clone(); + let ptr_hi = + b.main(0, cols::msg_ptr(last, 2)) + b.main(0, cols::msg_ptr(last, 3)) * c65536; + let inv_2_32 = b.const_base(INV_SHIFT_32); + let off = b.const_base((8 * last) as u64); + let carry_0 = (base_lo + off - ptr_lo) * inv_2_32.clone(); + let carry_1 = (base_hi + carry_0 - ptr_hi) * inv_2_32; + let c = b.main(0, cols::MU_C); + b.emit_base(idx, c * carry_1); + idx += 1; + } + + // ★ The interior schedule, constrained rather than assumed: every + // absorbed block runs at `t = 0` with `block_len = 64`, and only the + // FIRST block carries flags. These four columns are the compression + // inputs an absorb row does NOT read from memory, so nothing else pins + // them — a prover free to choose them could absorb the caller's blocks + // under a framing that produces a different digest and still balance + // every bus. + // + // Their bytes are range-checked as XOR operands by the mixing core, + // which is live on exactly the rows these gate, so a zero word + // expression means four zero bytes. + for (word, want) in [(24usize, 0u64), (25, 0), (26, 64)] { + let c = b.main(0, cols::MU_C); + let expr = word_expr(b, &WordRef::Cols(word_cols(cols::in_word(word, 0)))); + let target = b.const_base(want); + let root = c * (expr - target); + b.emit_base(idx, root); + idx += 1; + } + { + // Flags are zero on every compressing row but the first; the FIRST + // row's value is bound by the x13 register read. + let gate = b.main(0, cols::MU_C) - b.main(0, cols::FIRST); + let flags = word_expr(b, &WordRef::Cols(word_cols(cols::in_word(27, 0)))); + let root = gate * flags; + b.emit_base(idx, root); + idx += 1; + } + + // ★ x11 is 8-aligned: `M_BASE[0] = 8·Q`, with `Q` bounded to a halfword + // by its own `IsHalfword` send. The ABI requires the alignment and the + // executor rejects without it, so a chip that did not check it would + // accept absorbs the VM semantics reject. Only the FIRST row needs it: + // `M_BASE + 64` preserves 8-alignment down the group. + { + let eight = b.const_base(8); + let root = b.main(0, cols::FIRST) + * (b.main(0, cols::M_BASE) - b.main(0, cols::MSG_ALIGN_Q) * eight); + b.emit_base(idx, root); + idx += 1; + } + + // `NEXT_IS_END` is zero off the compressing rows, so the gate below is a + // bit there rather than a negative. On a compressing row the ZERO lookup + // pins it to `REM_DECR == 0`, so it needs no IS_BIT of its own — it is + // fully determined either way. + { + let one = b.one(); + let root = b.main(0, cols::NEXT_IS_END) * (one - b.main(0, cols::MU_C)); + b.emit_base(idx, root); + idx += 1; + } + + // ★ The message region does not wrap the address space — except that its + // LAST block may end exactly at the top of it. + // + // `M_BASE_INCR = M_BASE + 64` is the next block's address, so forbidding + // its carry on every compressing row would reject a message ending at + // 2^64, which the ABI accepts. Forbidding it on NONE of them (the + // previous state of this chip) let a group of N blocks wrap and fold + // blocks read from the bottom of memory, which the executor rejects. + // Gating on `MU_C − NEXT_IS_END` is the exact distinction: every + // compressing row but the last. + { + let c65536 = b.const_base(65536); + let base_lo = b.main(0, cols::M_BASE) + b.main(0, cols::M_BASE + 1) * c65536.clone(); + let base_hi = + b.main(0, cols::M_BASE + 2) + b.main(0, cols::M_BASE + 3) * c65536.clone(); + let incr_lo = + b.main(0, cols::M_BASE_INCR) + b.main(0, cols::M_BASE_INCR + 1) * c65536.clone(); + let incr_hi = + b.main(0, cols::M_BASE_INCR + 2) + b.main(0, cols::M_BASE_INCR + 3) * c65536; + let inv_2_32 = b.const_base(INV_SHIFT_32); + let off = b.const_base(64); + let carry_0 = (base_lo + off - incr_lo) * inv_2_32.clone(); + let carry_1 = (base_hi + carry_0 - incr_hi) * inv_2_32; + let gate = b.main(0, cols::MU_C) - b.main(0, cols::NEXT_IS_END); + b.emit_base(idx, gate * carry_1); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The value interpretation must reproduce the executor's compression — + /// same function the canonical oracle vectors validate. + #[test] + fn value_flow_matches_executor() { + use executor::vm::instruction::execution::blake3_compress_6round; + let h: [u32; 8] = core::array::from_fn(|i| 0x9E3779B9u32.wrapping_mul(i as u32 + 1)); + let m: [u32; 16] = core::array::from_fn(|i| 0x85EBCA6Bu32.wrapping_mul(i as u32 + 7)); + let t = 0x0123_4567_89AB_CDEFu64; + let (bl, fl) = (64u32, 11u32); + let flow = ValueFlow::compute(&h, &m, t, bl, fl); + assert_eq!(flow.out, blake3_compress_6round(&h, &m, t, bl, fl)); + } + + /// Canonical op counts: 96 add3s, 96 add2s, 96 rotations, 192+16 XORs. + #[test] + fn wire_flow_counts() { + let w = WireFlow::build(); + assert_eq!(w.add3s.len(), NUM_G * 2); + assert_eq!(w.add2s.len(), NUM_G * 2); + assert_eq!(w.rots.len(), NUM_G * 2); + assert_eq!(w.xors.len(), NUM_G * 4 + 16); + // Every output column lands exactly once, and inside the row. + use std::collections::HashSet; + let mut seen = HashSet::new(); + let mut claim = |c: usize| { + assert!(c < cols::NUM_COLUMNS, "column {c} out of range"); + assert!(seen.insert(c), "column {c} written twice"); + }; + for aw in &w.add3s { + for c in aw.s { + claim(c); + } + claim(aw.c1); + claim(aw.c2); + } + for aw in &w.add2s { + for c in aw.s { + claim(c); + } + } + for xw in &w.xors { + for c in xw.out { + claim(c); + } + } + for rw in &w.rots { + for c in rw + .sll_lo + .iter() + .chain(&rw.sllc_lo) + .chain(&rw.sll_hi) + .chain(&rw.sllc_hi) + .chain(&rw.y) + { + claim(*c); + } + } + // 48 G-blocks × 60 cells + 64 out bytes, all distinct. + assert_eq!(seen.len(), NUM_G * cols::G_SIZE + 64); + } +} + +/// ★ The executor's compression and `crypto`'s shared primitive are the same +/// function. +/// +/// #903 landed a second host transcription of the BLAKE3 compression: +/// `executor::vm::instruction::execution::blake3_compress_6round`, with its own +/// `blake3_g`, its own `BLAKE3_IV` and its own `BLAKE3_ROUNDS = 6`. `crypto`'s +/// `blake3_compress_rounds` is the one P-a Stage 1 hoisted precisely so there +/// would be a single definition — the same treatment the CUDA reference got. +/// +/// Two independently written encodings of one function is what PA-PLAN §1.4 +/// forbids ("do not prove that two … coincide; make them one function"), and +/// merging #903 reintroduced it. Until they are unified, this is the gate: the +/// executor is what the guest's syscall actually runs, so a divergence here is +/// a guest that hashes differently from the host prover — R5's invisible +/// failure, which surfaces only as in-guest proof rejection. +/// +/// Checked over the message schedule's structural edge cases plus a pseudo-random +/// sweep, at every `(t, block_len, flags)` shape the chain framing produces. +#[cfg(test)] +mod executor_primitive_parity { + use crypto::hash::blake3::{BLAKE3_SIX_ROUNDS, blake3_compress_rounds}; + use executor::vm::instruction::execution::blake3_compress_6round; + + #[test] + fn the_executor_compression_is_the_shared_primitive() { + // A cheap deterministic stream; no rand dependency in this crate. + let mut z = 0x243f_6a88_85a3_08d3u64; + let mut next = move || { + z ^= z << 13; + z ^= z >> 7; + z ^= z << 17; + z as u32 + }; + + // The flag/counter shapes `Blake3Chain` actually emits: first block + // (CHUNK_START), interior, and last (CHUNK_END|ROOT with the true byte + // count as block_len). t is 0 throughout for the chain, but the syscall + // takes a full 64-bit counter, so both halves are exercised. + let shapes: [(u64, u32, u32); 5] = [ + (0, 64, 1), // CHUNK_START + (0, 64, 0), // interior + (0, 7, 2 | 8), // CHUNK_END | ROOT, partial final block + (u64::MAX, 64, 0), // both counter halves set + (1 << 32, 0, 0xffff_ffff), // high half only; degenerate len/flags + ]; + + for (t, block_len, flags) in shapes { + for _ in 0..64 { + let h: [u32; 8] = core::array::from_fn(|_| next()); + let m: [u32; 16] = core::array::from_fn(|_| next()); + + assert_eq!( + blake3_compress_6round(&h, &m, t, block_len, flags), + blake3_compress_rounds(&h, &m, t, block_len, flags, BLAKE3_SIX_ROUNDS), + "executor and crypto disagree at t={t}, block_len={block_len}, flags={flags}" + ); + } + } + } + + /// CONTROL: the two would NOT agree at a different round count, so the test + /// above is comparing round counts as well as wiring. + #[test] + fn the_parity_is_round_count_sensitive() { + let h = [1u32, 2, 3, 4, 5, 6, 7, 8]; + let m: [u32; 16] = core::array::from_fn(|i| (i as u32) * 7 + 1); + assert_ne!( + blake3_compress_6round(&h, &m, 0, 64, 1), + blake3_compress_rounds(&h, &m, 0, 64, 1, BLAKE3_SIX_ROUNDS + 1), + "a 7-round reference must not match the 6-round executor" + ); + } +} + +/// ★ The state the guest hands the accelerator is the layout the executor +/// reads back. +/// +/// `Blake3Chain`'s riscv64 arm marshals a compression into +/// `crypto::hash::blake3::chain::pack_syscall_state`'s 22 dwords and reads the +/// result out of `unpack_syscall_out`. That marshaling is the last unchecked +/// link between the host prover's hash and the guest's: `executor_primitive_parity` +/// above gates the *compression*, and `crypto` gates the chain framing, but a +/// transposed dword or a swapped counter half would leave both of those green +/// and still make the guest hash differently — R5's invisible failure, visible +/// only as in-guest proof rejection. +/// +/// It needs no guest to check. The executor's syscall handler is ordinary host +/// code driven by a real `EcallEbreak`, so this lays the packed dwords into a VM +/// `Memory` exactly as the guest's `&mut [u64; 22]` presents them, runs the +/// instruction, and unpacks the result — closing the loop through the same two +/// functions the guest calls. +#[cfg(test)] +mod executor_syscall_packing { + use crypto::hash::blake3::chain::{SYSCALL_OUT_DWORD, pack_syscall_state, unpack_syscall_out}; + use crypto::hash::blake3::{BLAKE3_SIX_ROUNDS, blake3_compress_rounds}; + use executor::vm::instruction::decoding::Instruction; + use executor::vm::instruction::execution::BLAKE3_SYSCALL_NUMBER; + use executor::vm::memory::Memory; + use executor::vm::registers::Registers; + + /// Pre-filled into the output dwords, so a packing that pointed the + /// accelerator at the wrong part of the region cannot pass on stale data. + const SENTINEL: u64 = 0xDEAD_BEEF_DEAD_BEEF; + + /// One compression the way a guest performs it: pack, ecall, unpack. + fn through_the_accelerator( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, + ) -> [u32; 16] { + let addr = 0x1000u64; + let mut memory = Memory::default(); + let mut registers = Registers::default(); + + let mut state = pack_syscall_state(h, m, t, block_len, flags); + for dword in &mut state[SYSCALL_OUT_DWORD..] { + *dword = SENTINEL; + } + for (k, dword) in state.iter().enumerate() { + memory + .store_doubleword(addr + (k as u64) * 8, *dword) + .unwrap(); + } + + let mut pc = 0; + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + registers.write(10, addr).unwrap(); + Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap(); + + for (k, dword) in state.iter_mut().enumerate() { + *dword = memory.load_doubleword(addr + (k as u64) * 8).unwrap(); + } + for (k, dword) in state.iter().enumerate().skip(SYSCALL_OUT_DWORD) { + assert_ne!(*dword, SENTINEL, "output dword {k} was never written"); + } + unpack_syscall_out(&state) + } + + #[test] + fn the_packed_state_is_what_the_executor_reads() { + // A cheap deterministic stream; no rand dependency in this crate. + let mut z = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + z ^= z << 13; + z ^= z >> 7; + z ^= z << 17; + z as u32 + }; + + // The flag shapes `Blake3Chain` emits — first block, interior, and last + // with a partial `block_len` — plus two counters setting exactly one + // half. The chain always sends `t = 0`, so only these pin the split + // order the packing chose. + let shapes: [(u64, u32, u32); 5] = [ + (0, 64, 1), // CHUNK_START + (0, 64, 0), // interior + (0, 7, 2 | 8), // CHUNK_END | ROOT, partial block + (0x0000_0000_FFFF_FFFF, 64, 0), // low counter half only + (0xFFFF_FFFF_0000_0000, 0, !0u32), // high half; degenerate len/flags + ]; + + for (t, block_len, flags) in shapes { + for _ in 0..16 { + let h: [u32; 8] = core::array::from_fn(|_| next()); + let m: [u32; 16] = core::array::from_fn(|_| next()); + assert_eq!( + through_the_accelerator(&h, &m, t, block_len, flags), + blake3_compress_rounds(&h, &m, t, block_len, flags, BLAKE3_SIX_ROUNDS), + "packed state mismatch at t={t}, block_len={block_len}, flags={flags}" + ); + } + } + } + + /// CONTROL: the check above discriminates the *layout*, not merely the + /// compression. The counter's two halves share one dword and are the + /// likeliest thing to transpose — and the chain, which only ever sends + /// `t = 0`, would never notice. They must be distinguishable, or agreement + /// above would survive the swap. + #[test] + fn the_packing_check_is_layout_sensitive() { + let h: [u32; 8] = core::array::from_fn(|i| (i as u32).wrapping_mul(2_654_435_761)); + let m: [u32; 16] = + core::array::from_fn(|i| (i as u32).wrapping_mul(40_503).wrapping_add(7)); + let t = 0x0000_0001_0000_0000u64; + + assert_ne!( + through_the_accelerator(&h, &m, t, 64, 1), + blake3_compress_rounds(&h, &m, t.rotate_left(32), 64, 1, BLAKE3_SIX_ROUNDS), + "the counter halves must be distinguishable" + ); + } +} + +/// ★ The absorb ecall folds blocks the way `Blake3Chain` folds them. +/// +/// The absorb syscall exists so a long message costs one ecall instead of one +/// per 64 bytes, and paying for that means the executor applies a flag and +/// `block_len` schedule of its own: `t = 0` and `block_len = 64` on every block, +/// the caller's `first_flags` on block 0 and nothing after it +/// ([`executor::vm::instruction::execution::blake3_absorb_chain_6round`]). +/// `executor` cannot call `crypto`, so that schedule is a SECOND statement of +/// the framing `Blake3Chain` owns — precisely the thing PA-PLAN §1.4 forbids +/// leaving unchecked, and precisely the thing that fails invisibly: a guest that +/// absorbs under the wrong flags produces a perfectly valid proof of a digest +/// nobody else computes. +/// +/// This is the gate. It drives real `EcallEbreak` instructions through the +/// executor exactly as the guest's `bulk_absorb` arm does, finishes the hash +/// with `Blake3Chain` itself, and compares against `blake3_chain_rounds` — over +/// every length from the empty message past two blocks, plus the boundaries +/// that discriminate the schedule. Nothing here restates the framing: the run's +/// first flag word comes from [`block_flags`], and resuming after the run goes +/// through `Blake3Chain::resume_with_rounds`, the same call the guest makes. +#[cfg(test)] +mod executor_absorb_parity { + use crypto::hash::blake3::chain::{ + ABSORB_CV_OUT_DWORD, BLOCK_LEN, Blake3Chain, blake3_chain_rounds, block_flags, + bulk_absorb_blocks, kat_message_byte, pack_absorb_ctrl, unpack_absorb_cv, + }; + use crypto::hash::blake3::{BLAKE3_IV, BLAKE3_SIX_ROUNDS}; + use executor::vm::instruction::decoding::Instruction; + use executor::vm::instruction::execution::BLAKE3_ABSORB_SYSCALL_NUMBER; + use executor::vm::memory::Memory; + use executor::vm::registers::Registers; + + /// Pre-filled into `cv_out`, so an absorb that wrote nowhere near the + /// control region cannot pass on stale bytes. + const SENTINEL: u64 = 0xDEAD_BEEF_DEAD_BEEF; + /// Disjoint by construction — the ecall rejects overlapping regions, and + /// 0x2000 is past the control region's 64 bytes at 0x1000. + const CTRL_ADDR: u64 = 0x1000; + const MSG_ADDR: u64 = 0x2000; + + fn message(len: usize) -> Vec { + (0..len).map(kat_message_byte).collect() + } + + /// One absorb the way a guest performs it: pack the control region, lay the + /// blocks out in memory, ecall, read the chaining value back. + fn absorb_through_the_accelerator( + cv_in: &[u32; 8], + blocks: &[u8], + first_flags: u32, + ) -> [u32; 8] { + assert!(blocks.len().is_multiple_of(BLOCK_LEN) && !blocks.is_empty()); + let mut memory = Memory::default(); + let mut registers = Registers::default(); + + let mut ctrl = pack_absorb_ctrl(cv_in); + for dword in &mut ctrl[ABSORB_CV_OUT_DWORD..] { + *dword = SENTINEL; + } + for (k, dword) in ctrl.iter().enumerate() { + memory + .store_doubleword(CTRL_ADDR + (k as u64) * 8, *dword) + .unwrap(); + } + for (k, chunk) in blocks.chunks_exact(8).enumerate() { + let dword = u64::from_le_bytes(chunk.try_into().unwrap()); + memory + .store_doubleword(MSG_ADDR + (k as u64) * 8, dword) + .unwrap(); + } + + let mut pc = 0; + registers.write(17, BLAKE3_ABSORB_SYSCALL_NUMBER).unwrap(); + registers.write(10, CTRL_ADDR).unwrap(); + registers.write(11, MSG_ADDR).unwrap(); + registers + .write(12, (blocks.len() / BLOCK_LEN) as u64) + .unwrap(); + registers.write(13, first_flags as u64).unwrap(); + Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap(); + + for (k, dword) in ctrl.iter_mut().enumerate() { + *dword = memory.load_doubleword(CTRL_ADDR + (k as u64) * 8).unwrap(); + } + for (k, dword) in ctrl.iter().enumerate().skip(ABSORB_CV_OUT_DWORD) { + assert_ne!(*dword, SENTINEL, "cv_out dword {k} was never written"); + } + unpack_absorb_cv(&ctrl) + } + + /// `Blake3Chain::update`'s bulk path, driven against the real ecall: absorb + /// whole blocks while the schedule will take them, then let the hasher + /// finish the tail — which is where the final block's `CHUNK_END | ROOT` + /// and true byte count stay. + fn chain_through_the_accelerator(msg: &[u8]) -> [u8; 32] { + let mut cv = BLAKE3_IV; + let mut started = false; + let mut rest = msg; + loop { + // VM addresses here are 8-aligned by construction. + let blocks = bulk_absorb_blocks(0, rest.len(), true); + if blocks == 0 { + break; + } + // What `Blake3Chain::flags(false)` evaluates to at this point: the + // closed form at the next block's index, read rather than restated. + let index = usize::from(started); + let first_flags = block_flags(index, index + 2); + let taken = blocks * BLOCK_LEN; + cv = absorb_through_the_accelerator(&cv, &rest[..taken], first_flags); + started = true; + rest = &rest[taken..]; + } + let mut chain = if started { + Blake3Chain::resume_with_rounds(cv, BLAKE3_SIX_ROUNDS) + } else { + Blake3Chain::with_rounds(BLAKE3_SIX_ROUNDS) + }; + chain.update(rest); + chain.finalize_digest() + } + + /// ★ Exhaustive over every length through two blocks and past, plus the + /// lengths PA-PLAN §1.7.4 singles out. Anything the executor gets wrong + /// about the interior schedule moves a digest here. + #[test] + fn the_absorb_ecall_is_the_chain() { + let lengths = (0..=300usize).chain([511, 512, 513, 1023, 1024, 1025, 1088, 4096]); + for len in lengths { + let msg = message(len); + assert_eq!( + chain_through_the_accelerator(&msg), + blake3_chain_rounds(&msg, BLAKE3_SIX_ROUNDS), + "the absorb path must be the chain, at length {len}" + ); + } + } + + /// CONTROL: the test above must actually be taking the ecall. If + /// `bulk_absorb_blocks` returned 0 everywhere, every assertion would still + /// pass — `chain_through_the_accelerator` would degrade to plain + /// `Blake3Chain` and gate nothing at all. + #[test] + fn the_parity_test_really_reaches_the_accelerator() { + assert_eq!(bulk_absorb_blocks(0, 65, true), 1); + assert_eq!(bulk_absorb_blocks(0, 1024, true), 15); + // And an absorb changes the chaining value away from the IV, so the + // ecall is doing work rather than copying `cv_in` through. + let msg = message(128); + let cv = absorb_through_the_accelerator(&BLAKE3_IV, &msg[..64], block_flags(0, 2)); + assert_ne!(cv, BLAKE3_IV, "the absorb must fold the block in"); + } + + /// ★ A message past the block cap takes SEVERAL absorb ecalls, and the + /// second one is where the flag schedule can go wrong. + /// + /// Every other case here fits in one ecall, so `first_flags` is always + /// `CHUNK_START` and the "0 on every later block" half of the schedule is + /// exercised only *within* a run. Past `ABSORB_MAX_BLOCKS` the guest calls + /// again with `started` already true, and the second run's first block must + /// carry NO flags — a run that re-sent `CHUNK_START`, or an executor that + /// applied it because it is block 0 of *its* run, would produce a valid + /// proof of the wrong digest and nothing else here would notice. + /// + /// `(ABSORB_MAX_BLOCKS + 2) * 64` forces exactly two ecalls. Not the + /// *smallest* message that does — that is `ABSORB_MAX_BLOCKS * 64 + 1`, + /// one byte past a full cap's worth — but it leaves the second run a clean + /// whole block, which is the shape worth pinning. + #[test] + fn a_message_past_the_block_cap_takes_several_ecalls() { + use crypto::hash::blake3::chain::ABSORB_MAX_BLOCKS; + + let len = (ABSORB_MAX_BLOCKS + 2) * BLOCK_LEN; + // Two full runs plus the final block that `finalize_digest` keeps. + assert_eq!(bulk_absorb_blocks(0, len, true), ABSORB_MAX_BLOCKS); + let after_first = len - ABSORB_MAX_BLOCKS * BLOCK_LEN; + assert_eq!(bulk_absorb_blocks(0, after_first, true), 1); + + let msg = message(len); + assert_eq!( + chain_through_the_accelerator(&msg), + blake3_chain_rounds(&msg, BLAKE3_SIX_ROUNDS), + "a multi-ecall absorb must still be the chain" + ); + } + + /// ★ The two crates' copies of the absorb ABI's shape are the same numbers. + /// + /// `crypto` states the control-region layout and the block cap for the guest + /// (`ABSORB_*`), and `executor` states them again for the handler + /// (`BLAKE3_ABSORB_*`), because neither crate can see the other's. Nothing + /// above catches drift: the parity test drives well-formed absorbs, so a + /// guest whose cap exceeded the executor's would fault only on the long + /// messages no test reaches. `prover` sees both crates, so this is where + /// they can be pinned. + /// + /// The guest's own control array is already pinned to the syscall wrapper's + /// by the type system (`[u64; N]` on both sides) when the guest compiles; + /// this covers the half no type can see. + #[test] + fn the_two_crates_agree_on_the_absorb_abi() { + use crypto::hash::blake3::chain::{ABSORB_CTRL_DWORDS, ABSORB_MAX_BLOCKS}; + use executor::vm::instruction::execution::{ + BLAKE3_ABSORB_CTRL_DWORDS, BLAKE3_ABSORB_CV_OUT_DWORD, BLAKE3_ABSORB_MAX_BLOCKS, + BLAKE3_BLOCK_BYTES, + }; + + assert_eq!(ABSORB_CTRL_DWORDS as u64, BLAKE3_ABSORB_CTRL_DWORDS); + assert_eq!(ABSORB_CV_OUT_DWORD as u64, BLAKE3_ABSORB_CV_OUT_DWORD); + assert_eq!(ABSORB_MAX_BLOCKS as u64, BLAKE3_ABSORB_MAX_BLOCKS); + assert_eq!(BLOCK_LEN as u64, BLAKE3_BLOCK_BYTES); + // `cv_out` starts past `cv_in`'s four dwords and the region holds both — + // the disjointness the single-timestamp memory argument depends on. + assert_eq!(BLAKE3_ABSORB_CV_OUT_DWORD, 4); + assert_eq!(BLAKE3_ABSORB_CTRL_DWORDS, 8); + } + + /// CONTROL: the parity is sensitive to the run's FIRST flag word — the one + /// value the guest marshals rather than the executor deriving it. Absorbing + /// under interior flags where `CHUNK_START` belongs must not agree, or the + /// gate would survive dropping the flag entirely. + #[test] + fn the_absorb_parity_is_first_flag_sensitive() { + let msg = message(128); + let honest = absorb_through_the_accelerator(&BLAKE3_IV, &msg[..64], block_flags(0, 2)); + let tampered = absorb_through_the_accelerator(&BLAKE3_IV, &msg[..64], 0); + assert_ne!(honest, tampered, "CHUNK_START must change the absorb"); + } + + /// CONTROL: the executor applies `first_flags` to block 0 ONLY. A run of + /// two blocks must differ from one where the flag were applied to both, or + /// "and nothing after it" would be untested. + #[test] + fn the_absorb_applies_the_first_flag_to_one_block_only() { + let msg = message(128); + let two_at_once = + absorb_through_the_accelerator(&BLAKE3_IV, &msg[..128], block_flags(0, 2)); + // The same two blocks as two separate runs, each carrying the flag. + let first = absorb_through_the_accelerator(&BLAKE3_IV, &msg[..64], block_flags(0, 2)); + let flagged_twice = + absorb_through_the_accelerator(&first, &msg[64..128], block_flags(0, 2)); + assert_ne!( + two_at_once, flagged_twice, + "the flag must land on block 0 alone" + ); + // ...and chaining two single-block runs with the interior flag on the + // second IS the two-block run: that is what "chained" means. + let flagged_once = absorb_through_the_accelerator(&first, &msg[64..128], 0); + assert_eq!(two_at_once, flagged_once, "the run must chain its blocks"); + } +} + +/// ★ The absorb mode's soundness suite: every claim the chip makes, with an +/// honest-path control beside each attack. +/// +/// Two levels, because the mode's guarantees live at two levels: +/// +/// * **Row-local constraints** — checked with the cheap `eval_main_row` idiom +/// (`prover/src/tests/hint_tests.rs`): generate a trace, assert every +/// constraint is zero on every row, tamper one cell, assert a constraint +/// fires. This covers the schedule, the boundary algebra and the pointer +/// arithmetic. +/// * **Bus tuples** — the chain, the `Zero` derivation of END and the `Ecall` +/// anchoring are cross-row properties no constraint can see. They are checked +/// by evaluating the table's own `bus_interactions` over the trace and +/// asserting the multiset of `Blake3Absorb` tuples cancels, which is exactly +/// what LogUp asks of them. +#[cfg(test)] +mod absorb_tests { + use super::*; + use math::field::element::FieldElement; + use stark::constraints::builder::ProverEvalFolder; + use stark::frame::Frame; + use stark::lookup::{BusInteraction, Multiplicity}; + use stark::table::TableView; + use stark::traits::TransitionEvaluationContext; + use std::collections::BTreeMap; + + // --------------------------------------------------------------------- + // Fixtures and helpers + // --------------------------------------------------------------------- + + const CTRL: u64 = 0x1000; + const MSG: u64 = 0x8000; + + /// A deterministic absorb of `n` blocks. No `rand` dependency in this crate. + fn absorb_op(timestamp: u64, n: usize, first_flags: u32) -> Blake3AbsorbOperation { + let mut z = 0x243f_6a88_85a3_08d3u64 ^ (n as u64); + let mut next = move || { + z ^= z << 13; + z ^= z >> 7; + z ^= z << 17; + z as u32 + }; + Blake3AbsorbOperation { + timestamp, + ctrl_addr: CTRL, + msg_addr: MSG, + first_flags, + cv_in: core::array::from_fn(|_| next()), + blocks: (0..n).map(|_| core::array::from_fn(|_| next())).collect(), + old_cv_out: core::array::from_fn(|i| (i as u8).wrapping_mul(7)), + } + } + + fn single_op(timestamp: u64) -> Blake3Operation { + single_op_with_flags(timestamp, 11) + } + + /// A single compression with a chosen `flags` word. `flags = 0` is not a + /// detail: several absorb constraints multiply the flags word, so a fixture + /// with nonzero flags can make a test pass through the WRONG constraint. + fn single_op_with_flags(timestamp: u64, flags: u32) -> Blake3Operation { + let h: [u32; 8] = core::array::from_fn(|i| 0x9E3779B9u32.wrapping_mul(i as u32 + 1)); + let m: [u32; 16] = core::array::from_fn(|i| 0x85EBCA6Bu32.wrapping_mul(i as u32 + 7)); + let (t, block_len) = (0x0123_4567u64, 64u32); + Blake3Operation { + timestamp, + state_addr: 0x2000, + h, + m, + t, + block_len, + flags, + old_out: [0; 64], + out: executor::vm::instruction::execution::blake3_compress_6round( + &h, &m, t, block_len, flags, + ), + } + } + + fn trace_of( + singles: &[Blake3Operation], + absorbs: &[Blake3AbsorbOperation], + ) -> TraceTable { + generate_blake3_trace(singles, absorbs) + } + + fn row_of(trace: &TraceTable, row: usize) -> Vec { + (0..cols::NUM_COLUMNS) + .map(|c| *trace.main_table.get(row, c)) + .collect() + } + + /// Evaluate the BLAKE3 constraint set on one main-trace row. + fn eval_main_row(main: Vec) -> Vec { + let n = Blake3Constraints.meta().len(); + let frame = Frame::::new(vec![TableView::new( + vec![main], + vec![vec![]], + )]); + let no_e: Vec> = vec![]; + let offset_e = FieldElement::::zero(); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_e, &no_e, &offset_e); + let mut base = vec![FE::zero(); n]; + let mut ext = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base, &mut ext); + Blake3Constraints.eval(&mut folder); + base + } + + /// Assert every constraint holds on every row of `rows` (all of them when + /// `rows` is `None` — the constraint set rebuilds the wire flow per call, so + /// large traces are sampled instead). + fn assert_constraints_hold( + trace: &TraceTable, + rows: Option<&[usize]>, + ) { + let all: Vec = (0..trace.num_rows()).collect(); + for &row in rows.unwrap_or(&all) { + for (i, v) in eval_main_row(row_of(trace, row)).iter().enumerate() { + assert_eq!(*v, FE::zero(), "constraint {i} must hold at row {row}"); + } + } + } + + /// Evaluate a multiplicity expression on one row. + fn multiplicity_at(m: &Multiplicity, row: &[FE]) -> FE { + match m { + Multiplicity::One => FE::one(), + Multiplicity::Column(c) => row[*c], + Multiplicity::Sum(a, b) => row[*a] + row[*b], + Multiplicity::Negated(c) => FE::one() - row[*c], + Multiplicity::Diff(a, b) => row[*a] - row[*b], + Multiplicity::Sum3(a, b, c) => row[*a] + row[*b] + row[*c], + Multiplicity::Linear(_) => unreachable!("no Linear multiplicity in this chip"), + } + } + + /// The net multiset of tuples on `bus`, as `key → (senders − receivers)`. + /// An honest trace leaves this empty: that IS the LogUp condition. + /// A bus's net multiset, plus how many tuples were actually put on it. + /// + /// ★ `touched` exists so an `is_empty()` control cannot pass vacuously. An + /// empty net means "senders and receivers cancelled"; it says nothing about + /// whether anything was sent at all, and a mutation that gates the whole + /// chain off would satisfy the balance assertion while proving nothing. + struct BusNet { + net: BTreeMap, + touched: usize, + } + + impl BusNet { + fn is_empty(&self) -> bool { + self.net.is_empty() + } + fn len(&self) -> usize { + self.net.len() + } + } + + fn bus_net(trace: &TraceTable, bus: BusId) -> BusNet { + let interactions: Vec = bus_interactions() + .into_iter() + .filter(|i| i.bus_id == u64::from(bus)) + .collect(); + let mut net: BTreeMap = BTreeMap::new(); + let mut touched = 0usize; + for row_idx in 0..trace.num_rows() { + let row = row_of(trace, row_idx); + for it in &interactions { + let mult = multiplicity_at(&it.multiplicity, &row); + if mult == FE::zero() { + continue; + } + assert_eq!(mult, FE::one(), "chip multiplicities are 0 or 1"); + touched += 1; + let values: Vec = it + .values + .iter() + .flat_map(|v| v.combine_from::(|c| row[c])) + .collect(); + let key = format!("{values:?}"); + *net.entry(key).or_insert(0) += if it.is_sender { 1 } else { -1 }; + } + } + net.retain(|_, v| *v != 0); + BusNet { net, touched } + } + + /// The chain balances AND actually carried `expected` tuples. A group of N + /// blocks puts 2N on the bus: N sends from the compressing rows, N receives + /// from the rows after the first. + fn assert_chain_balances( + trace: &TraceTable, + expected: usize, + ) { + let n = bus_net(trace, BusId::Blake3Absorb); + assert!(n.is_empty(), "the honest chain must balance"); + assert_eq!( + n.touched, expected, + "the balance assertion must not pass vacuously" + ); + } + + /// Number of rows whose `Ecall` receive fires, split by syscall. + fn ecall_receives(trace: &TraceTable) -> (usize, usize) { + let mut single = 0; + let mut absorb = 0; + for row_idx in 0..trace.num_rows() { + let row = row_of(trace, row_idx); + if row[cols::MU_S] == FE::one() { + single += 1; + } + if row[cols::FIRST] == FE::one() { + absorb += 1; + } + } + (single, absorb) + } + + // --------------------------------------------------------------------- + // Honest-path controls + // --------------------------------------------------------------------- + + /// ★ CONTROL for everything below: a trace holding both modes at once — a + /// single compression, a one-block absorb and a three-block absorb — + /// satisfies every constraint on every row, padding included. + #[test] + fn both_modes_share_the_table() { + let trace = trace_of(&[single_op(4)], &[absorb_op(8, 1, 1), absorb_op(12, 3, 1)]); + assert_constraints_hold(&trace, None); + // 2N tuples per group: 2·1 for the one-block group, 2·3 for the other. + assert_chain_balances(&trace, 8); + assert_eq!(ecall_receives(&trace), (1, 2), "one Ecall per ecall"); + } + + /// The group is `num_blocks + 1` rows: N compressions and one END row that + /// does no work. This is CHANGE 1 of the design, and the row budget the + /// 64 KiB cap is chosen for depends on it. + #[test] + fn a_group_is_n_plus_one_rows() { + for n in [1usize, 2, 5] { + let rows = expand_absorb(&absorb_op(4, n, 1)); + assert_eq!(rows.len(), n + 1); + assert!(rows[0].first && !rows[0].end); + assert!(rows[n].end && !rows[n].first); + assert_eq!(rows[0].remaining, n as u32); + assert_eq!(rows[n].remaining, 0); + for (i, r) in rows.iter().enumerate() { + assert_eq!(r.flags == 0, i > 0, "flags land on block 0 alone"); + assert_eq!(r.m_base, MSG + 64 * i as u64); + } + } + } + + /// ★ The chip folds blocks the way the executor's ecall does. Without this + /// the chip could be internally consistent and still prove a different + /// digest from the one the VM semantics define. + #[test] + fn the_expansion_is_the_executors_absorb() { + use executor::vm::instruction::execution::blake3_absorb_chain_6round; + for n in [1usize, 2, 7] { + for flags in [0u32, 1, 1 | 8] { + let op = absorb_op(4, n, flags); + let flat: Vec<[u32; 16]> = op.blocks.clone(); + let rows = expand_absorb(&op); + assert_eq!( + rows.last().unwrap().h, + blake3_absorb_chain_6round(&op.cv_in, &flat, flags), + "the chip's chain must be the executor's, n={n} flags={flags}" + ); + } + } + } + + /// ★ The single-compression mode is untouched. Every absorb column is zero + /// on a single row, so no absorb interaction and no absorb constraint can + /// fire in that mode — the property a reviewer should check first. + #[test] + fn the_single_mode_is_untouched() { + let trace = trace_of(&[single_op(4), single_op(8)], &[]); + assert_constraints_hold(&trace, None); + // 0 touched is the POINT here: the single mode must not reach the bus. + assert_chain_balances(&trace, 0); + for row_idx in 0..2 { + let row = row_of(&trace, row_idx); + assert_eq!(row[cols::MU], FE::one()); + assert_eq!(row[cols::MU_S], FE::one()); + for (c, v) in row.iter().enumerate().skip(cols::MU_A) { + assert_eq!(*v, FE::zero(), "absorb column {c} must be 0 in single mode"); + } + } + } + + /// Both ends of the legal block range prove: the degenerate one-block group + /// and a group at the 1 024-block cap. + #[test] + fn the_cap_boundary_and_the_degenerate_group_both_hold() { + let one = trace_of(&[], &[absorb_op(4, 1, 1)]); + assert_constraints_hold(&one, None); + assert_chain_balances(&one, 2); + + let n = ABSORB_MAX_BLOCKS as usize; + let full = trace_of(&[], &[absorb_op(4, n, 1)]); + // The row budget the 64 KiB cap exists for: 1 024 compressions + END, + // padded to 2 048. This is the largest group the ABI admits. + assert_eq!(n + 1, 1025, "rows per ecall at the cap"); + assert_eq!(full.num_rows(), 2048); + assert_eq!( + expand_absorb(&absorb_op(4, n, 1)).len(), + 1025, + "the group really is at the cap" + ); + // The constraint set rebuilds the wire flow per call, so sample the + // boundaries rather than all 2 048 rows. + assert_constraints_hold(&full, Some(&[0, 1, n - 1, n, n + 1, full.num_rows() - 1])); + assert_chain_balances(&full, 2 * n); + } + + // --------------------------------------------------------------------- + // Falsification: the chain + // --------------------------------------------------------------------- + + /// ★ A tampered chaining value breaks the chain. The bus is what ties a + /// row's output to the next row's input; nothing row-local can see it. + #[test] + fn a_tampered_chained_cv_unbalances_the_chain() { + let mut trace = trace_of(&[], &[absorb_op(4, 3, 1)]); + assert_chain_balances(&trace, 6); + // Row 1's incoming chaining value, one byte off. + let c = cols::in_word(0, 0); + let old = *trace.main_table.get(1, c); + trace.main_table.set_fe(1, c, old + FE::one()); + assert_eq!( + bus_net(&trace, BusId::Blake3Absorb).len(), + 2, + "the send it no longer matches and the receive nobody sent" + ); + } + + /// ★ Running past the end: drop the END row and the last compression's send + /// has no receiver. This is why END must exist at all. + #[test] + fn a_group_without_its_end_row_leaves_a_dangling_send() { + let mut trace = trace_of(&[], &[absorb_op(4, 2, 1)]); + assert_chain_balances(&trace, 4); + // Blank the END row (row 2) exactly as a padding row. + for c in 0..cols::NUM_COLUMNS { + trace.main_table.set_fe(2, c, FE::zero()); + } + assert_eq!( + bus_net(&trace, BusId::Blake3Absorb).len(), + 1, + "the last compression's send must dangle" + ); + } + + /// ★ END is derived, not chosen: `ZERO[REMAINING] → END`. Claiming END early + /// leaves the chip sending a ZERO tuple the BITWISE table does not contain, + /// so the range-check bus cannot balance. + #[test] + fn an_early_end_sends_a_zero_tuple_that_does_not_exist() { + // Every ZERO tuple the chip sends, as (input, claimed output). BITWISE + // answers `input == 0`, so a tuple disagreeing with that is a lookup no + // row of the precomputed table can satisfy. + let zero_tuples = |trace: &TraceTable| { + let sends: Vec = bus_interactions() + .into_iter() + .filter(|i| i.bus_id == u64::from(BusId::Zero)) + .collect(); + let mut out = Vec::new(); + for row_idx in 0..trace.num_rows() { + let row = row_of(trace, row_idx); + for it in &sends { + if multiplicity_at(&it.multiplicity, &row) == FE::zero() { + continue; + } + let v: Vec = it + .values + .iter() + .flat_map(|x| x.combine_from::(|c| row[c])) + .collect(); + out.push((v[0], v[1])); + } + } + out + }; + let answerable = + |(input, output): &(FE, FE)| (*input == FE::zero()) == (*output == FE::one()); + + // CONTROL: a 3-block group sends 7 ZERO tuples — `ZERO[REMAINING] → END` + // on each of its 4 rows, and `ZERO[REM_DECR] → NEXT_IS_END` on the 3 + // compressing ones — and the precomputed table answers every one. + let trace = trace_of(&[], &[absorb_op(4, 3, 1)]); + let honest = zero_tuples(&trace); + assert_eq!(honest.len(), 7, "4 END lookups + 3 NEXT_IS_END lookups"); + assert!( + honest.iter().all(answerable), + "BITWISE answers every honest lookup" + ); + + // The attack: END = 1 while REMAINING = 3. The chip then sends ZERO[3] → 1, + // and the BITWISE row for 3 holds 0 — the range-check bus cannot balance. + let mut tampered = trace_of(&[], &[absorb_op(4, 3, 1)]); + tampered.main_table.set_fe(0, cols::END, FE::one()); + let attacked = zero_tuples(&tampered); + assert_eq!( + attacked.iter().filter(|t| !answerable(t)).count(), + 1, + "claiming END early must leave a ZERO lookup nothing can answer" + ); + } + + /// ★ A padding row cannot mint a group boundary — COMMIT's boundary lock, + /// retargeted to MU_A. Without it a `μ = 0` row could claim FIRST and + /// receive a second `Ecall`, or claim END and write a `cv_out`. + #[test] + fn a_padding_row_cannot_mint_first_or_end() { + let trace = trace_of(&[], &[absorb_op(4, 1, 1)]); + let pad = trace.num_rows() - 1; + assert_eq!(row_of(&trace, pad)[cols::MU], FE::zero(), "row is padding"); + + for col in [cols::FIRST, cols::END] { + let mut main = row_of(&trace, pad); + main[col] = FE::one(); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "the boundary lock must reject a padding row claiming {col}" + ); + } + } + + /// A single-compression row cannot claim a boundary either: the lock is on + /// MU_A, not μ, which is CHANGE 3 of the design. Copying COMMIT's `μ − FIRST` + /// receive multiplicity across would have made every single row receive from + /// the absorb bus. + #[test] + fn a_single_row_cannot_mint_a_boundary() { + // ★ flags = 0 on purpose. With the default fixture's flags = 11 the + // FIRST half of this test passes through `(MU_C − FIRST)·flags = 0` + // rather than through the boundary lock — the assertion would survive + // deleting the lock entirely. Zeroing the flags removes that path and + // leaves the lock as the only constraint that can fire. + for op in [single_op_with_flags(4, 0), single_op(4)] { + let trace = trace_of(&[op], &[]); + for col in [cols::FIRST, cols::END] { + let mut main = row_of(&trace, 0); + main[col] = FE::one(); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "a single-compression row must not claim {col}" + ); + } + } + + // ...and specifically the LOCK fires, not something downstream: on the + // flags = 0 fixture it is the only nonzero constraint. + let trace = trace_of(&[single_op_with_flags(4, 0)], &[]); + let mut main = row_of(&trace, 0); + main[cols::FIRST] = FE::one(); + let fired: Vec = eval_main_row(main) + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect(); + assert_eq!( + fired.len(), + 1, + "exactly the boundary lock should fire, got {fired:?}" + ); + } + + /// The two modes are exclusive, so one row cannot answer two syscalls. + #[test] + fn a_row_cannot_be_in_both_modes() { + let trace = trace_of(&[single_op(4)], &[]); + let mut main = row_of(&trace, 0); + main[cols::MU_A] = FE::one(); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "MU_S · MU_A = 0 must reject a row in both modes" + ); + } + + /// A zero-block group is rejected: `FIRST · END = 0`. The executor rejects + /// `num_blocks = 0`, and a group that were its own END row would copy + /// `cv_in` straight to `cv_out`. + #[test] + fn a_zero_block_group_is_rejected() { + let trace = trace_of(&[], &[absorb_op(4, 1, 1)]); + let mut main = row_of(&trace, 0); + assert_eq!(main[cols::FIRST], FE::one()); + main[cols::END] = FE::one(); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "FIRST · END = 0 must reject a zero-block group" + ); + } + + /// Two FIRST rows at one timestamp means two `Ecall` receives against the + /// CPU's single send. + #[test] + fn two_first_rows_receive_one_ecall_twice() { + let mut trace = trace_of(&[], &[absorb_op(4, 2, 1)]); + assert_eq!( + ecall_receives(&trace).1, + 1, + "control: one group, one receive" + ); + trace.main_table.set_fe(1, cols::FIRST, FE::one()); + assert_eq!( + ecall_receives(&trace).1, + 2, + "a second FIRST at the same timestamp doubles the receive" + ); + } + + // --------------------------------------------------------------------- + // Falsification: the schedule + // --------------------------------------------------------------------- + + /// ★ `first_flags` lands on block 0 and nowhere else. Flags on a later block + /// would produce a valid proof of a digest nobody else computes. + #[test] + fn flags_on_a_later_block_are_rejected() { + let trace = trace_of(&[], &[absorb_op(4, 3, 1)]); + let mut main = row_of(&trace, 1); + assert_eq!( + main[cols::FIRST], + FE::zero(), + "row 1 is not the first block" + ); + main[cols::in_word(27, 0)] = FE::one(); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "flags must be zero on every compressing row but the first" + ); + } + + /// ★ The FIRST row's flags are not free either — they are bound by the x13 + /// register read, so tampering them changes a tuple that must match the + /// register file. (Row-local constraints deliberately do NOT pin this: the + /// guest chooses its own flag word.) + #[test] + fn the_first_rows_flags_ride_the_x13_register_read() { + let trace = trace_of(&[], &[absorb_op(4, 2, 5)]); + let row = row_of(&trace, 0); + // The register reads gated on FIRST, evaluated on the group's first row. + // Slot 9 of a MEMW tuple is the register's word address (2·regno). + let reads: Vec> = bus_interactions() + .iter() + .filter(|i| { + i.bus_id == u64::from(BusId::Memw) + && matches!(i.multiplicity, Multiplicity::Column(c) if c == cols::FIRST) + }) + .map(|i| { + i.values + .iter() + .flat_map(|v| v.combine_from::(|c| row[c])) + .collect() + }) + .collect(); + assert_eq!(reads.len(), 3, "x11, x12 and x13 (x10 is shared with MU_S)"); + let x13 = reads + .iter() + .find(|v| v[9] == FE::from(26u64)) + .expect("x13 -> word address 26"); + assert_eq!(x13[0], FE::from(5u64), "x13 carries first_flags"); + assert_eq!(x13[1], FE::zero(), "and pins its high half to zero"); + // x12 carries the block count, which IS the countdown's initial value: + // nothing else pins the group's length to the ecall's argument. + let x12 = reads + .iter() + .find(|v| v[9] == FE::from(24u64)) + .expect("x12 -> word address 24"); + assert_eq!(x12[0], FE::from(2u64), "x12 is REMAINING on the FIRST row"); + assert_eq!(x12[1], FE::zero(), "no block count above 2^32"); + } + + /// The interior schedule is constrained, not assumed: `t = 0` and + /// `block_len = 64` on every absorbed block. + #[test] + fn the_interior_schedule_is_pinned() { + let trace = trace_of(&[], &[absorb_op(4, 2, 1)]); + for (col, delta) in [ + (cols::in_word(24, 0), FE::one()), // t_lo + (cols::in_word(25, 0), FE::one()), // t_hi + (cols::in_word(26, 0), FE::one()), // block_len 64 -> 65 + ] { + let mut main = row_of(&trace, 0); + main[col] += delta; + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "column {col} must be pinned on an absorb row" + ); + } + } + + /// ★ The message base advances by exactly 64 per block. A prover free to + /// choose it would read a block of its own choosing and still balance MEMW, + /// because reading some other address is a legitimate read. + #[test] + fn the_message_base_must_advance_by_64() { + let trace = trace_of(&[], &[absorb_op(4, 2, 1)]); + let mut main = row_of(&trace, 0); + // 32 instead of 64: the ADD template's carry stops being a bit. + main[cols::M_BASE_INCR] = main[cols::M_BASE_INCR] - FE::from(32u64); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "M_BASE + 64 = M_BASE_INCR must reject a short step" + ); + + // ...and the per-dword pointers into the block are pinned the same way. + let mut main = row_of(&trace, 0); + main[cols::msg_ptr(3, 0)] = main[cols::msg_ptr(3, 0)] + FE::from(8u64); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "msg_ptr[j] = M_BASE + 8j must reject a shifted pointer" + ); + } + + /// ★ x11 must be 8-aligned, the way the executor requires and the way the + /// chip's own MEMW dword reads assume. Before this constraint the chip + /// accepted an unaligned message and folded blocks the ABI forbids. + #[test] + fn an_unaligned_message_address_is_rejected() { + // CONTROL: the aligned fixture satisfies the alignment constraint, and + // its witness is the honest quotient. + let trace = trace_of(&[], &[absorb_op(4, 2, 1)]); + assert_constraints_hold(&trace, None); + let row = row_of(&trace, 0); + assert_eq!(row[cols::MSG_ALIGN_Q], FE::from(MSG / 8)); + + // The attack: an unaligned base. The witness a prover would need is + // `M_BASE[0] / 8`, and for an odd low halfword no HALFWORD value works — + // 8·Q would have to be odd, and 8·Q is even for every integer Q. + let mut main = row_of(&trace, 0); + main[cols::M_BASE] += FE::one(); // low halfword now MSG+1, unaligned + assert!( + eval_main_row(main.clone()).iter().any(|v| *v != FE::zero()), + "an unaligned M_BASE must fail with the honest quotient" + ); + + // ...and no small Q rescues it: the constraint is an equality over + // integers once Q is halfword-bounded, so an odd target is unreachable. + for q in 0u64..=0xFFFF { + main[cols::MSG_ALIGN_Q] = FE::from(q); + if eval_main_row(main.clone()).iter().all(|v| *v == FE::zero()) { + panic!("Q = {q} satisfied an unaligned M_BASE"); + } + } + } + + /// ★ The scaled form of the alignment check — `IsB20[Q · 2^7]`, mirroring + /// the block cap — is VACUOUS, and this test is why the shipped chip uses + /// `IsHalfword[Q]` instead. + /// + /// With `Q` bounded only through a scaled product, a prover sets + /// `Q = M_BASE[0] · 8⁻¹ mod p`; then `Q · 2^7 = M_BASE[0] · 16`, which is + /// under 2^20 for ANY halfword `M_BASE[0]`, aligned or not. The block cap + /// survives the identical attack only because `REMAINING`'s domain bound + /// arrives first from the ZERO lookup — a distinction easy to lose. + #[test] + fn the_scaled_alignment_check_would_have_been_vacuous() { + const B20: u64 = 1 << 20; + let eight_inv = (FE::one() / FE::from(8u64)).expect("8 is invertible"); + for base in [1u64, 3, 0xFFFF, 12345] { + let q = FE::from(base) * eight_inv; + // The bogus witness satisfies the equality... + assert_eq!(FE::from(8u64) * q, FE::from(base)); + // ...and the SCALED range check would have accepted it. + let scaled = q * FE::from(128u64); + assert_eq!(scaled, FE::from(base * 16)); + assert!( + base * 16 < B20, + "the scaled form admits unaligned base {base}" + ); + } + // The shipped form does not: Q would have to exceed a halfword. + for base in [1u64, 3, 12345] { + let q = FE::from(base) * eight_inv; + assert!( + *q.value() > 0xFFFF, + "the halfword bound must exclude the bogus witness for {base}" + ); + } + } + + /// ★ A message region may end at the top of memory, but it may not WRAP + /// through it. The distinction is the last block: only its `M_BASE + 64` + /// is allowed to carry. + #[test] + fn the_message_region_may_end_at_the_top_but_not_wrap() { + // The legal shape: a one-block group whose block is the last 64 bytes of + // the address space. `M_BASE + 64` = 2^64 wraps, and that is fine. + let mut op = absorb_op(4, 1, 1); + op.msg_addr = u64::MAX - 63; + let trace = trace_of(&[], &[op]); + assert_constraints_hold(&trace, None); + assert_eq!(row_of(&trace, 0)[cols::NEXT_IS_END], FE::one()); + + // The illegal shape: two blocks from the same base, so block 0's + // successor address wraps to 0 and block 1 is read from the bottom of + // memory. This is the executor's `Blake3AbsorbAddressOverflow` case. + let mut op = absorb_op(4, 2, 1); + op.msg_addr = u64::MAX - 63; + let rows = expand_absorb(&op); + assert_eq!(rows[1].m_base, 0, "the fixture really does wrap"); + let trace = trace_of(&[], &[op]); + let row0 = row_of(&trace, 0); + assert_eq!( + row0[cols::NEXT_IS_END], + FE::zero(), + "block 0 is not the last block" + ); + assert!( + eval_main_row(row0).iter().any(|v| *v != FE::zero()), + "a non-final block whose successor address wraps must be rejected" + ); + } + + /// The countdown cannot be re-based mid-group. + #[test] + fn the_countdown_must_decrement_by_one() { + let trace = trace_of(&[], &[absorb_op(4, 3, 1)]); + let mut main = row_of(&trace, 0); + main[cols::REM_DECR] += FE::one(); + assert!( + eval_main_row(main).iter().any(|v| *v != FE::zero()), + "REM_DECR + 1 = REMAINING must reject a skipped block" + ); + } + + // --------------------------------------------------------------------- + // Pins + // --------------------------------------------------------------------- + + /// ★ The chip, `crypto` and `executor` agree on the absorb ABI's shape. + /// The chip is a third statement of the control-region layout and the cap; + /// drift would fault only on inputs no test reaches. + #[test] + fn the_chip_agrees_on_the_absorb_abi() { + use executor::vm::instruction::execution::{ + BLAKE3_ABSORB_CTRL_DWORDS, BLAKE3_ABSORB_CV_OUT_DWORD, BLAKE3_ABSORB_MAX_BLOCKS, + BLAKE3_BLOCK_BYTES, + }; + assert_eq!(CTRL_DWORDS as u64, BLAKE3_ABSORB_CTRL_DWORDS); + assert_eq!(CV_OUT_DWORD as u64, BLAKE3_ABSORB_CV_OUT_DWORD); + assert_eq!(ABSORB_MAX_BLOCKS, BLAKE3_ABSORB_MAX_BLOCKS); + assert_eq!(MSG_DWORDS as u64 * 8, BLAKE3_BLOCK_BYTES); + // `cv_in` occupies the dwords below `cv_out`, which is what lets the + // FIRST row's read share the single mode's `h` interaction. + assert_eq!(CV_DWORDS, CV_OUT_DWORD); + } + + /// ★ The in-circuit cap admits exactly `1..=1024`. `IsB20` holds [0, 2^20), + /// so the scaled countdown is in the table iff the block count is legal — + /// and the product cannot wrap, being under 2^30. + #[test] + fn the_block_cap_is_enforced_in_circuit() { + const B20: u64 = 1 << 20; + + /// The value the chip actually puts on the `IsB20` bus for a group of + /// `n` blocks — read off a generated TRACE through the shipped + /// interaction, not recomputed from the constants. + fn cap_lookup_value(n: usize) -> u64 { + let trace = trace_of(&[], &[absorb_op(4, n, 1)]); + let row = row_of(&trace, 0); + assert_eq!(row[cols::FIRST], FE::one(), "row 0 is the group's FIRST"); + let sends: Vec> = bus_interactions() + .iter() + .filter(|i| { + i.bus_id == u64::from(BusId::IsB20) + && matches!(i.multiplicity, Multiplicity::Column(c) if c == cols::FIRST) + }) + .map(|i| { + i.values + .iter() + .flat_map(|v| v.combine_from::(|c| row[c])) + .collect() + }) + .collect(); + assert_eq!(sends.len(), 1, "exactly one cap lookup per group"); + assert_eq!(sends[0].len(), 1, "IsB20 takes one bus element"); + // BITWISE holds [0, 2^20); anything at or above it is answerable by + // no row. The value is a small integer, so its canonical + // representative IS the integer the lookup asks for. + *sends[0][0].value() + } + + // ★ The scale is in the emitted tuple, not just in the constant. At + // n = 2 the countdown is 1, so a chip that dropped ABSORB_CAP_SCALE + // would put 1 on the bus; the shipped one puts 1024. + assert_eq!(cap_lookup_value(2), ABSORB_CAP_SCALE); + assert_ne!(cap_lookup_value(2), 1, "dropping the scale must be visible"); + assert_eq!(cap_lookup_value(1), 0); + assert_eq!(cap_lookup_value(7), 6 * ABSORB_CAP_SCALE); + + // Legal groups land inside the table; the one at the cap lands at its + // very top row. + assert!(cap_lookup_value(ABSORB_MAX_BLOCKS as usize) < B20); + assert_eq!( + cap_lookup_value(ABSORB_MAX_BLOCKS as usize), + B20 - ABSORB_CAP_SCALE + ); + + // ★ One block past the cap leaves the table, so no BITWISE row answers + // the lookup and the range-check bus cannot balance. The trace builder + // will happily BUILD that group — the cap is enforced by this lookup, + // not by the witness generator, which is the point. + let over = cap_lookup_value(ABSORB_MAX_BLOCKS as usize + 1); + assert_eq!(over, B20); + assert!(over >= B20, "an over-cap group must fall outside IsB20"); + // ...and it fails by being out of range, not by wrapping the field. + assert!(over < 1u64 << 30); + } + + /// The table's shape, pinned so a change has to be deliberate. The absorb + /// mode costs 47 columns and 76 interactions on top of the single mode's + /// 3 219 / 1 397, and every one of those interactions costs a LogUp + /// denominator on single-compression rows too. + #[test] + fn the_tables_shape_is_pinned() { + assert_eq!(cols::NUM_COLUMNS, 3268); + assert_eq!(bus_interactions().len(), 1475); + assert_eq!(Blake3Constraints.meta().len(), 851); + assert_eq!(Blake3Constraints.max_degree(), 3); + } + + /// ★ The END-gated `AreBytes` sends cover the chaining value, and nothing + /// else — the mutation this catches is re-pointing them at columns that are + /// already range-checked. + /// + /// Those 16 sends are the ONLY byte range check `h` has on the END row (the + /// mixing core, which range-checks it everywhere else as an XOR operand, is + /// gated off there). Aim them at the message or the output columns and every + /// bus still balances, every constraint still holds, and the `cv_out` write + /// is free to put a non-byte in memory. Nothing behavioural distinguishes + /// the mutant, so the test has to be structural. + #[test] + fn the_end_row_range_checks_the_chaining_value_itself() { + let mut covered: Vec = bus_interactions() + .iter() + .filter(|i| { + i.bus_id == u64::from(BusId::AreBytes) + && matches!(i.multiplicity, Multiplicity::Column(c) if c == cols::END) + }) + .flat_map(|i| i.values.iter().flat_map(|v| v.column_indices())) + .collect(); + covered.sort_unstable(); + + // h[0..8] is 32 bytes at cols::IN — the columns the chain receive fills + // and the cv_out write reads. + let expected: Vec = (cols::IN..cols::IN + 32).collect(); + assert_eq!( + covered, expected, + "the END-gated AreBytes must range-check exactly h's 32 bytes" + ); + } + + /// Every interaction's columns are inside the row. + #[test] + fn interactions_stay_inside_the_row() { + for (i, it) in bus_interactions().iter().enumerate() { + for v in &it.values { + for c in v.column_indices() { + assert!(c < cols::NUM_COLUMNS, "interaction {i} reads column {c}"); + } + } + } + } +} diff --git a/prover/src/tables/branch.rs b/prover/src/tables/branch.rs index 828d91861..1a68037bd 100644 --- a/prover/src/tables/branch.rs +++ b/prover/src/tables/branch.rs @@ -31,6 +31,7 @@ use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{ BusId, GoldilocksExtension, GoldilocksField, SHIFT_16, VmTable, alu_op, dedup_rows, }; @@ -165,53 +166,65 @@ impl BranchOperation { pub fn generate_branch_trace( operations: &[BranchOperation], ) -> TraceTable { - let unique_ops = dedup_rows(operations); - let num_rows = unique_ops.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, (op, multiplicity)) in unique_ops.iter().enumerate() { - // Compute next_pc - let next_pc_unmasked = op.compute_next_pc_unmasked(); - let next_pc = op.compute_next_pc(); - - // Extract next_pc components - // next_pc_low[0]: bits 0-7 (masked) - // next_pc_low[1]: bits 8-15 - // next_pc_high[0]: bits 16-31 - // next_pc_high[1]: bits 32-47 - // next_pc_high[2]: bits 48-63 - let unmasked_low_byte = (next_pc_unmasked & 0xFF) as u8; - let next_pc_low_0 = (next_pc & 0xFF) as u8; // = unmasked_low_byte & 0xFE - let next_pc_low_1 = ((next_pc >> 8) & 0xFF) as u8; - let next_pc_high_0 = ((next_pc >> 16) & 0xFFFF) as u16; - let next_pc_high_1 = ((next_pc >> 32) & 0xFFFF) as u16; - let next_pc_high_2 = ((next_pc >> 48) & 0xFFFF) as u16; - - // Store columns - table.set_dword_wl(row_idx, cols::PC_0, op.pc); - table.set_dword_wl(row_idx, cols::OFFSET_0, op.offset); - table.set_dword_wl(row_idx, cols::REGISTER_0, op.register); - table.set_bool(row_idx, cols::JALR, op.jalr); - table.set_halves( - row_idx, - cols::NEXT_PC_HIGH_0, - &[next_pc_high_0, next_pc_high_1, next_pc_high_2], - ); - table.set_bytes( - row_idx, - cols::NEXT_PC_LOW_0, - &[next_pc_low_0, next_pc_low_1], - ); - table.set_byte(row_idx, cols::UNMASKED_LOW_BYTE, unmasked_low_byte); - table.set_u64(row_idx, cols::MU, *multiplicity); - } + generate_branch_trace_as(operations, TraceForm::Wide) +} - trace +/// The widths BRANCH traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_branch_trace`] in `form` (`tables::gpack`). +pub fn generate_branch_trace_as( + operations: &[BranchOperation], + form: TraceForm, +) -> TraceTable { + branch_trace_of_rows(&dedup_rows(operations), form) +} + +/// The BRANCH trace of `unique_ops`, the rows [`generate_branch_trace_as`] +/// deduplicates its operations into, in `form`. +pub(crate) fn branch_trace_of_rows( + unique_ops: &[(BranchOperation, u64)], + form: TraceForm, +) -> TraceTable { + let num_rows = unique_ops.len().next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, (op, multiplicity)) in unique_ops.iter().enumerate() { + // Compute next_pc + let next_pc_unmasked = op.compute_next_pc_unmasked(); + let next_pc = op.compute_next_pc(); + + // Extract next_pc components + // next_pc_low[0]: bits 0-7 (masked) + // next_pc_low[1]: bits 8-15 + // next_pc_high[0]: bits 16-31 + // next_pc_high[1]: bits 32-47 + // next_pc_high[2]: bits 48-63 + let unmasked_low_byte = (next_pc_unmasked & 0xFF) as u8; + let next_pc_low_0 = (next_pc & 0xFF) as u8; // = unmasked_low_byte & 0xFE + let next_pc_low_1 = ((next_pc >> 8) & 0xFF) as u8; + let next_pc_high_0 = ((next_pc >> 16) & 0xFFFF) as u16; + let next_pc_high_1 = ((next_pc >> 32) & 0xFFFF) as u16; + let next_pc_high_2 = ((next_pc >> 48) & 0xFFFF) as u16; + + // Store columns + table.set_dword_wl(row_idx, cols::PC_0, op.pc); + table.set_dword_wl(row_idx, cols::OFFSET_0, op.offset); + table.set_dword_wl(row_idx, cols::REGISTER_0, op.register); + table.set_bool(row_idx, cols::JALR, op.jalr); + table.set_halves( + row_idx, + cols::NEXT_PC_HIGH_0, + &[next_pc_high_0, next_pc_high_1, next_pc_high_2], + ); + table.set_bytes( + row_idx, + cols::NEXT_PC_LOW_0, + &[next_pc_low_0, next_pc_low_1], + ); + table.set_byte(row_idx, cols::UNMASKED_LOW_BYTE, unmasked_low_byte); + table.set_u64(row_idx, cols::MU, *multiplicity); + } + }) } // ========================================================================= diff --git a/prover/src/tables/bytewise.rs b/prover/src/tables/bytewise.rs index 3ac757a37..be9ab6767 100644 --- a/prover/src/tables/bytewise.rs +++ b/prover/src/tables/bytewise.rs @@ -19,6 +19,7 @@ use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, GoldilocksExtension, GoldilocksField, VmTable, alu_op, dedup_rows}; // ========================================================================= @@ -73,6 +74,16 @@ impl BytewiseOperation { /// The 8 `BYTE_ALU` lookups this op sends, for the BITWISE table's /// multiplicity bookkeeping (one per byte, keyed by opsel). pub fn collect_bitwise_ops(&self) -> Vec { + let mut ops = Vec::with_capacity(8); + self.for_each_bitwise_op(|op| ops.push(op)); + ops + } + + /// [`Self::collect_bitwise_ops`]' lookups, one by one. + pub(crate) fn for_each_bitwise_op( + &self, + mut emit: impl FnMut(super::bitwise::BitwiseOperation), + ) { use super::bitwise::{BitwiseOperation, BitwiseOperationType}; let kind = match self.op { alu_op::AND => BitwiseOperationType::ByteAluAnd, @@ -80,13 +91,11 @@ impl BytewiseOperation { alu_op::XOR => BitwiseOperationType::ByteAluXor, other => panic!("BYTEWISE only handles AND/OR/XOR, got opcode {other}"), }; - (0..8) - .map(|i| { - let a = ((self.a >> (i * 8)) & 0xFF) as u8; - let b = ((self.b >> (i * 8)) & 0xFF) as u8; - BitwiseOperation::byte_op(kind, a, b) - }) - .collect() + for i in 0..8 { + let a = ((self.a >> (i * 8)) & 0xFF) as u8; + let b = ((self.b >> (i * 8)) & 0xFF) as u8; + emit(BitwiseOperation::byte_op(kind, a, b)); + } } } @@ -99,26 +108,38 @@ impl BytewiseOperation { pub fn generate_bytewise_trace( operations: &[BytewiseOperation], ) -> TraceTable { - let unique_ops = dedup_rows(operations); - let num_rows = unique_ops.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, (op, multiplicity)) in unique_ops.iter().enumerate() { - let res = op.compute_res(); - - table.set_dword_bl(row_idx, cols::A[0], op.a); - table.set_dword_bl(row_idx, cols::B[0], op.b); - table.set_dword_bl(row_idx, cols::RES[0], res); - table.set_byte(row_idx, cols::OP, op.op); - table.set_u64(row_idx, cols::MU, *multiplicity); - } + generate_bytewise_trace_as(operations, TraceForm::Wide) +} + +/// The widths BYTEWISE traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - trace +/// [`generate_bytewise_trace`] in `form` (`tables::gpack`). +pub fn generate_bytewise_trace_as( + operations: &[BytewiseOperation], + form: TraceForm, +) -> TraceTable { + bytewise_trace_of_rows(&dedup_rows(operations), form) +} + +/// The BYTEWISE trace of `unique_ops`, the rows [`generate_bytewise_trace_as`] +/// deduplicates its operations into, in `form`. +pub(crate) fn bytewise_trace_of_rows( + unique_ops: &[(BytewiseOperation, u64)], + form: TraceForm, +) -> TraceTable { + let num_rows = unique_ops.len().next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, (op, multiplicity)) in unique_ops.iter().enumerate() { + let res = op.compute_res(); + + table.set_dword_bl(row_idx, cols::A[0], op.a); + table.set_dword_bl(row_idx, cols::B[0], op.b); + table.set_dword_bl(row_idx, cols::RES[0], res); + table.set_byte(row_idx, cols::OP, op.op); + table.set_u64(row_idx, cols::MU, *multiplicity); + } + }) } // ========================================================================= diff --git a/prover/src/tables/card_fill_inputs.rs b/prover/src/tables/card_fill_inputs.rs new file mode 100644 index 000000000..9d6e7f814 --- /dev/null +++ b/prover/src/tables/card_fill_inputs.rs @@ -0,0 +1,1447 @@ +//! The inputs of the card fill's tests: the records the card's fill kernels +//! read (`math_cuda::fill`), encoded from the prover's own; seeded synthetic +//! sets of each family's; the host fill's words in the card's layout; and the +//! vectors of the kernels' host known-answer harness +//! (`crypto/math-cuda/tests/host_kat/fill_kat_vectors.h`). +//! +//! The synthetic sets (splitmix64, seeds [`SEEDS`]): +//! - **MEMW_R:** 2^21 register rows. Address < 256, timestamp in [8, 2^34), +//! the old timestamp's low word 1..=2^16 below the timestamp's with no 32-bit +//! wrap, random values and read flags. +//! - **CPU:** 2^21 logs over a program's decode table from cycle 7·2^21. Each +//! log draws a decode row uniformly (the decode gathers' worst case) and takes +//! its pc. The next pc is pc + length, or a random aligned target on a branch +//! row. Register values are random, and one log in four has src2 = src1, so +//! conditional branches go both ways. +//! - **KECCAK_RND:** 2,732 keccak operations on random states, the output their +//! permutation, timestamps increasing across 2^32. The table is 2^16 rows cut +//! 16 rows into the first op, so the first and the last op are both cut. + +use executor::elf::Elf; +use executor::vm::instruction::decoding::Instruction; +use executor::vm::instruction::execution::keccak_f1600; +use executor::vm::logs::Log; +use executor::vm::memory::U64HashMap; +use math_cuda::fill::{ + AlignedRowC, DecodeEntryC, DecodeRunC, KeccakOpC, LoadOpC, LogC, MemwOpC, ShiftOpC, StoreOpC, +}; + +use super::cpu::{CpuOperation, generate_cpu_trace_as}; +use super::decode::{DecodeTable, instructions_from_elf}; +use super::gpack::{Trace, TraceForm}; +use super::keccak_rnd::{self, KeccakRoundOperation, generate_keccak_rnd_rows_as}; +use super::load::{self, LoadOperation}; +use super::memw::{self, MemwOperation}; +use super::memw_aligned::{self, AlignedRow}; +use super::memw_register::{RegRow, generate_memw_register_trace_from_rows_as}; +use super::shift::{self, ShiftOperation}; +use super::store::{self, StoreOperation}; + +/// The synthetic sets' seeds. +pub(crate) const SEEDS: [u64; 3] = [1, 2, 3]; +/// Rows of a streamed CPU or MEMW_R chunk. +pub(crate) const CHUNK_ROWS: usize = 1 << 21; +/// The synthetic CPU chunk's first cycle. +pub(crate) const CPU_FIRST_CYCLE: u64 = 7 << 21; +/// Rows of a KECCAK_RND table. +pub(crate) const KECCAK_RND_ROWS: usize = 1 << 16; +/// The synthetic KECCAK_RND table's ops and its cut into the first. +pub(crate) const KECCAK_RND_OPS: usize = 2732; +pub(crate) const KECCAK_RND_SKIP: usize = 16; +// The table's rows cut into the first and the last op, and the ops reach its +// end. +const _: () = { + assert!(!KECCAK_RND_SKIP.is_multiple_of(24)); + assert!(!(KECCAK_RND_SKIP + KECCAK_RND_ROWS).is_multiple_of(24)); + assert!(KECCAK_RND_OPS * 24 >= KECCAK_RND_SKIP + KECCAK_RND_ROWS); +}; + +/// splitmix64: the inputs are a function of the seed alone. +pub(crate) fn splitmix(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +// ========================================================================= +// The records +// ========================================================================= + +/// A log as the card's CPU fill reads it, field for field. +pub(crate) fn log_record(log: &Log) -> LogC { + LogC { + current_pc: log.current_pc, + next_pc: log.next_pc, + src1_val: log.src1_val, + src2_val: log.src2_val, + dst_val: log.dst_val, + } +} + +/// `decode`'s entries and runs as the card's CPU fill reads them. +pub(crate) fn decode_records(decode: &DecodeTable) -> (Vec, Vec) { + let entries = decode + .entries() + .iter() + .map(|e| { + let f = &e.fields; + DecodeEntryC { + pc: e.pc, + imm: e.imm, + read_register1: u8::from(f.read_register1), + read_register2: u8::from(f.read_register2), + write_register: u8::from(f.write_register), + word_instr: u8::from(f.word_instr), + alu: u8::from(f.alu), + add: u8::from(f.add), + sub: u8::from(f.sub), + memory: u8::from(f.memory), + branch: u8::from(f.branch), + ecall: u8::from(f.ecall), + rs1: f.rs1, + rs2: f.rs2, + rd: f.rd, + half_instruction_length: f.half_instruction_length, + alu_flags: f.alu_flags, + mem_flags: f.mem_flags, + } + }) + .collect(); + let runs = decode + .runs() + .iter() + .map(|&(first_pc, row, len)| DecodeRunC { first_pc, row, len }) + .collect(); + (entries, runs) +} + +/// A keccak operation as the card's KECCAK_RND fill reads it. +pub(crate) fn keccak_record(op: &KeccakRoundOperation) -> KeccakOpC { + KeccakOpC { + timestamp: op.timestamp, + input: op.input, + output: op.output, + } +} + +// ========================================================================= +// The synthetic sets +// ========================================================================= + +/// `n` register rows from `seed` (the module docs). +pub(crate) fn memw_r_rows(seed: u64, n: usize) -> Vec { + let mut rng = seed; + (0..n) + .map(|_| { + // The low word above 2^16, so the old one is below it with no wrap. + let timestamp = loop { + let t = 8 + splitmix(&mut rng) % ((1 << 34) - 8); + if t & 0xFFFF_FFFF > 1 << 16 { + break t; + } + }; + let delta = 1 + splitmix(&mut rng) % (1 << 16); + let old_timestamp = timestamp - delta; + let address = splitmix(&mut rng) % 256; + let [val0, val1, old0, old1] = std::array::from_fn(|_| splitmix(&mut rng) as u32); + let is_read = splitmix(&mut rng) & 1 == 1; + RegRow::new( + 2 * address, + timestamp, + val0, + val1, + old0, + old1, + old_timestamp, + is_read, + ) + }) + .collect() +} + +/// A log of the instruction at `entry` of `decode`, from `rng`; `equal` sets +/// src2 = src1. +fn cpu_log(rng: &mut u64, decode: &DecodeTable, entry: usize, equal: bool) -> Log { + let e = &decode.entries()[entry]; + let length = 2 * u64::from(e.fields.half_instruction_length); + let src1_val = splitmix(rng); + let src2_val = if equal { src1_val } else { splitmix(rng) }; + let next_pc = if e.fields.branch { + splitmix(rng) & !3 + } else { + e.pc.wrapping_add(length) + }; + Log { + current_pc: e.pc, + next_pc, + src1_val, + src2_val, + dst_val: splitmix(rng), + } +} + +/// `n` logs over `decode` from `seed` (the module docs). +pub(crate) fn cpu_logs(seed: u64, n: usize, decode: &DecodeTable) -> Vec { + let mut rng = seed; + let entries = decode.entries().len() as u64; + (0..n) + .map(|i| { + let entry = (splitmix(&mut rng) % entries) as usize; + cpu_log(&mut rng, decode, entry, i % 4 == 0) + }) + .collect() +} + +/// The CPU ops of `logs` from cycle `first_cycle`, as the walk makes them +/// (`collect_cpu_ops_into`: timestamp `4·cycle + 4`, the op read with its +/// DECODE row). +pub(crate) fn cpu_ops(logs: &[Log], first_cycle: u64, decode: &DecodeTable) -> Vec { + logs.iter() + .enumerate() + .map(|(i, log)| { + let row = decode + .row(log.current_pc) + .expect("every log's pc is the program's"); + CpuOperation::from_log( + log, + (first_cycle + i as u64) * 4 + 4, + decode.entry(row), + row, + ) + }) + .collect() +} + +/// `n` keccak operations from `seed` (the module docs). +pub(crate) fn keccak_ops(seed: u64, n: usize) -> Vec { + let mut rng = seed; + // 1,024 an op below 2^32, half the increments' mean (2,050): the run + // crosses 2^32 near its middle. + let mut timestamp = (1u64 << 32) - 1024 * n as u64; + (0..n) + .map(|_| { + timestamp += 4 * (1 + splitmix(&mut rng) % 1024); + let input: [u64; 25] = std::array::from_fn(|_| splitmix(&mut rng)); + let mut output = input; + keccak_f1600(&mut output); + KeccakRoundOperation { + timestamp, + input, + output, + } + }) + .collect() +} + +/// The RV64 major opcodes a decode table with no guest draws from: LOAD, +/// OP-IMM, AUIPC, OP-IMM-32, STORE, OP, LUI, OP-32, BRANCH, JALR, JAL. +const OPCODES: [u32; 11] = [ + 0x03, 0x13, 0x17, 0x1b, 0x23, 0x33, 0x37, 0x3b, 0x63, 0x67, 0x6f, +]; +/// The BRANCH opcode and its six conditions (BEQ, BNE, BLT, BGE, BLTU, BGEU). +const BRANCH: u32 = 0x63; +const BRANCH_FUNCT3: [u32; 6] = [0, 1, 4, 5, 6, 7]; +/// ECALL's word. +const ECALL: u32 = 0x0000_0073; +/// Where the runs of a decode table with no guest start: gaps between them, +/// so the CPU fill's search is across runs. +const RUN_BASES: [u64; 3] = [0x1_0000, 0x40_0000, 0x8000_0000]; + +/// A seeded random word of `opcode` (and `funct3`, when given) that parses. +fn random_instruction(rng: &mut u64, opcode: u32, funct3: Option) -> Instruction { + loop { + let mut word = (splitmix(rng) as u32 & !0x7f) | opcode; + if let Some(f) = funct3 { + word = (word & !(0x7 << 12)) | (f << 12); + } + if let Ok(instruction) = Instruction::parse(word) { + return instruction; + } + } +} + +/// A decode table with no guest: three runs of `per_run` instructions 4 +/// apart, each a seeded random word of a random major opcode that parses, one +/// in 64 an ECALL. +pub(crate) fn parsed_decode(seed: u64, per_run: usize) -> DecodeTable { + let mut rng = seed; + let mut instructions = U64HashMap::default(); + for base in RUN_BASES { + for i in 0..per_run { + let instruction = if i % 64 == 63 { + Instruction::parse(ECALL).expect("ECALL parses") + } else { + let opcode = OPCODES[(splitmix(&mut rng) % OPCODES.len() as u64) as usize]; + random_instruction(&mut rng, opcode, None) + }; + instructions.insert(base + 4 * i as u64, instruction); + } + } + DecodeTable::from_instructions(&instructions) +} + +/// The decode table the CPU sets are drawn over: the guest's at +/// `BLOCK_WHIR_ELF` (the median block's guest, on the box), or else +/// [`parsed_decode`]'s. +pub(crate) fn program_decode() -> DecodeTable { + match std::env::var("BLOCK_WHIR_ELF") { + Ok(path) => { + let bytes = + std::fs::read(&path).unwrap_or_else(|e| panic!("BLOCK_WHIR_ELF {path}: {e}")); + let elf = Elf::load(&bytes).expect("the guest ELF loads"); + DecodeTable::from_instructions( + &instructions_from_elf(&elf).expect("the guest's instructions parse"), + ) + } + Err(_) => parsed_decode(SEEDS[0], 4096), + } +} + +// ========================================================================= +// The host fills, in the card's layout +// ========================================================================= + +/// An aligned row's record: the card's struct, field for field. +pub(crate) fn aligned_record(row: &AlignedRow) -> AlignedRowC { + let r = &memw_aligned::card_rows(std::slice::from_ref(row))[0]; + AlignedRowC { + base_address: r.base_address, + timestamp: r.timestamp, + old_timestamp: r.old_timestamp, + value: r.value, + old: r.old, + width: r.width, + is_read: r.is_read, + is_register: r.is_register, + } +} + +/// A load op's record: the card's struct, field for field. +pub(crate) fn load_record(op: &LoadOperation) -> LoadOpC { + let r = &load::card_ops(std::slice::from_ref(op))[0]; + LoadOpC { + base_address: r.base_address, + timestamp: r.timestamp, + width: r.width, + is_signed: r.signed, + res: r.res, + } +} + +/// A store op's record: the card's struct, field for field. +pub(crate) fn store_record(op: &StoreOperation) -> StoreOpC { + let r = &store::card_ops(std::slice::from_ref(op))[0]; + StoreOpC { + base_address: r.base_address, + timestamp: r.timestamp, + value: r.value, + write2: r.write2, + write4: r.write4, + write8: r.write8, + } +} + +/// A shift op's record: the card's struct, field for field. +pub(crate) fn shift_record(op: &ShiftOperation) -> ShiftOpC { + let r = &shift::card_ops(std::slice::from_ref(op))[0]; + ShiftOpC { + shift_amount: r.shift_amount, + in_halves: r.in_halves, + shift: r.shift, + direction: r.direction, + is_signed: r.signed, + word_instr: r.word_instr, + } +} + +/// A MEMW op's record: the card's struct, field for field. +pub(crate) fn memw_record(op: &MemwOperation) -> MemwOpC { + let r = &memw::card_ops(std::slice::from_ref(op))[0]; + MemwOpC { + base_address: r.base_address, + timestamp: r.timestamp, + old_timestamp: r.old_timestamp, + value: r.value, + old: r.old, + is_register: r.is_register, + width: r.width, + is_read: r.is_read, + } +} + +/// `n` store ops from `seed`: widths 1, 2, 4 and 8 in turn, the values random. +pub(crate) fn store_ops(seed: u64, n: usize) -> Vec { + let mut rng = seed; + (0..n) + .map(|i| { + StoreOperation::new( + splitmix(&mut rng), + 8 + splitmix(&mut rng) % (1 << 40), + splitmix(&mut rng), + 1 << (i % 4), + ) + }) + .collect() +} + +/// The shift amounts of [`shift_ops`]: every bit shift and limb shift edge, +/// at 32 and 64 bits, and amounts whose high bits are set. +const SHIFT_AMOUNTS: [u64; 16] = [ + 0, + 1, + 4, + 15, + 16, + 17, + 31, + 32, + 33, + 47, + 48, + 63, + 64 + 7, + 255, + 0x1_0000_0103, + u64::MAX - 1, +]; + +/// Every direction, signedness and width at each of [`SHIFT_AMOUNTS`], the +/// values from `seed` (one in two with its top bit set), the first `n`. +pub(crate) fn shift_ops(seed: u64, n: usize) -> Vec { + let mut rng = seed; + let mut ops = Vec::new(); + for amount in SHIFT_AMOUNTS { + for k in 0..8u32 { + let value = splitmix(&mut rng) ^ (u64::from(k % 2) << 63); + ops.push(ShiftOperation::new( + value, + amount, + k & 1 == 1, + k & 2 == 2, + k & 4 == 4, + )); + } + } + ops.truncate(n); + ops +} + +/// `n` MEMW ops from `seed`: memory and register accesses (bytes or 32-bit +/// halves) at every width, reads and writes, low words about to carry, old +/// timestamps up to 2^64 - 1. +pub(crate) fn memw_ops(seed: u64, n: usize) -> Vec { + let mut rng = seed; + (0..n) + .map(|i| { + let is_register = i % 3 == 1; + let element = |w: u64| { + if is_register { + w as u32 + } else { + (w & 0xFF) as u32 + } + }; + let base_address = (splitmix(&mut rng) & !0xFFFF_FFFF) | (0xFFFF_FFF8 + (i % 8) as u64); + let timestamp = splitmix(&mut rng) >> 1; + let mut old_timestamp: [u64; 8] = std::array::from_fn(|_| splitmix(&mut rng) >> 2); + if i == 5 { + old_timestamp[3] = u64::MAX; + } + MemwOperation { + base_address, + timestamp, + old_timestamp, + value: std::array::from_fn(|_| element(splitmix(&mut rng))), + old: std::array::from_fn(|_| element(splitmix(&mut rng))), + is_register, + width: 1 << (i % 4), + is_read: i % 2 == 0, + } + }) + .collect() +} + +/// STORE's host fill of `ops`, in `form`. +pub(crate) fn store_host(ops: &[StoreOperation], form: TraceForm) -> Trace { + store::generate_store_trace_segments_as(&[ops], form) +} + +/// SHIFT's host fill of `ops`, in `form`. +pub(crate) fn shift_host(ops: &[ShiftOperation], form: TraceForm) -> Trace { + shift::generate_shift_trace_segments_as(&[ops], form) +} + +/// MEMW's host fill of `ops`, in `form`. +pub(crate) fn memw_host(ops: &[MemwOperation], form: TraceForm) -> Trace { + memw::generate_memw_trace_segments_as(&[ops], form) +} + +/// `n` aligned rows from `seed`: memory and register accesses, reads and +/// writes, of widths 1, 2, 4 and 8 in turn, the values random (a register +/// access's two 32-bit halves, a memory access's eight bytes). +pub(crate) fn memw_a_rows(seed: u64, n: usize) -> Vec { + let mut rng = seed; + (0..n) + .map(|i| { + let width = 1u8 << (i % 4); + let is_register = i % 3 == 1; + let timestamp = 8 + splitmix(&mut rng) % (1 << 40); + let old_timestamp = splitmix(&mut rng) % timestamp; + AlignedRow::from_parts( + splitmix(&mut rng) & !7, + timestamp, + old_timestamp, + splitmix(&mut rng), + splitmix(&mut rng), + width, + splitmix(&mut rng) & 1 == 1, + is_register, + ) + }) + .collect() +} + +/// `n` load ops from `seed`: widths 1, 2, 4 and 8 in turn, signed or not, the +/// result's bytes random (so each width's sign bit is set in some op and clear +/// in another). +pub(crate) fn load_ops(seed: u64, n: usize) -> Vec { + let mut rng = seed; + (0..n) + .map(|i| { + LoadOperation::new( + splitmix(&mut rng), + 8 + splitmix(&mut rng) % (1 << 40), + 1 << (i % 4), + (i / 4) % 2 == 1, + splitmix(&mut rng).to_le_bytes(), + ) + }) + .collect() +} + +/// MEMW_A's host fill of `rows`, in `form`. +pub(crate) fn memw_a_host(rows: &[AlignedRow], form: TraceForm) -> Trace { + memw_aligned::generate_memw_aligned_trace_segments_as(&[rows], form) +} + +/// LOAD's host fill of `ops`, in `form`. +pub(crate) fn load_host(ops: &[LoadOperation], form: TraceForm) -> Trace { + load::generate_load_trace_segments_as(&[ops], form) +} + +/// MEMW_R's host fill of `rows`, in `form`. +pub(crate) fn memw_r_host(rows: &[RegRow], form: TraceForm) -> Trace { + generate_memw_register_trace_from_rows_as(rows, form) +} + +/// CPU's host fill of `ops`, in `form`. +pub(crate) fn cpu_host(ops: &[CpuOperation], decode: &DecodeTable, form: TraceForm) -> Trace { + generate_cpu_trace_as(ops, decode, form) +} + +/// KECCAK_RND's host fill of rows `[skip, skip + rows)` of `ops`' table, in +/// `form`. +pub(crate) fn keccak_rnd_host( + ops: &[KeccakRoundOperation], + skip: usize, + rows: usize, + form: TraceForm, +) -> Trace { + generate_keccak_rnd_rows_as(ops, skip, rows, form) +} + +/// The words of `trace`'s main table column-major (`col · rows + row`), the +/// card's layout, whether it is held packed or at 8 bytes a cell. +pub(crate) fn col_major(trace: &Trace) -> Vec { + let (rows, cols) = (trace.num_rows(), trace.main_table.width); + if let Some(packed) = trace.narrow_main() { + return (0..cols).flat_map(|c| packed.column(c)).collect(); + } + // A block of rows at a time, so its rows stay in cache across the columns. + const BLOCK_ROWS: usize = 64; + let mut words = vec![0u64; rows * cols]; + for first in (0..rows).step_by(BLOCK_ROWS) { + let end = (first + BLOCK_ROWS).min(rows); + let block: Vec<&[super::types::FE]> = + (first..end).map(|r| trace.main_table.get_row(r)).collect(); + for c in 0..cols { + for (k, row) in block.iter().enumerate() { + words[c * rows + first + k] = *row[c].value(); + } + } + } + words +} + +/// Where two column-major tables of `rows` rows differ: how many words, and +/// the first (column, row, wanted, got). +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct Mismatches { + pub(crate) count: usize, + pub(crate) first: Option<(usize, usize, u64, u64)>, +} + +/// [`Mismatches`] of `got` against `want`; a length difference counts each +/// word past the shorter. +pub(crate) fn mismatches(want: &[u64], got: &[u64], rows: usize) -> Mismatches { + let mut out = Mismatches { + count: want.len().abs_diff(got.len()), + first: None, + }; + for (i, (&w, &g)) in want.iter().zip(got).enumerate() { + if w != g { + out.count += 1; + out.first + .get_or_insert((i / rows.max(1), i % rows.max(1), w, g)); + } + } + out +} + +// ========================================================================= +// The host known-answer harness's vectors +// ========================================================================= + +/// The harness's vectors, relative to the workspace root. +pub(crate) const HOST_KAT_VECTORS: &str = "crypto/math-cuda/tests/host_kat/fill_kat_vectors.h"; + +/// The known-answer MEMW_R table: rows, then zero rows. +const KAT_MEMW_R_ROWS: usize = 13; +const KAT_MEMW_R_HEIGHT: usize = 16; +/// The known-answer CPU table: three runs of eight instructions, every op +/// class among them, and logs that take each in turn, then padding rows. The +/// ops carry the run's timestamps from a cycle well into the run; the table's +/// own are its rows' `4·(row + 1)` whatever the ops carry. +const KAT_CPU_LOGS: usize = 29; +const KAT_CPU_HEIGHT: usize = 32; +const KAT_CPU_FIRST_CYCLE: u64 = (1 << 30) - 20; +/// The known-answer MEMW_A and LOAD tables: rows, then zero rows. +const KAT_MEMW_A_ROWS: usize = 13; +const KAT_MEMW_A_HEIGHT: usize = 16; +const KAT_LOAD_OPS: usize = 19; +const KAT_LOAD_HEIGHT: usize = 32; +/// The known-answer STORE, SHIFT and MEMW tables: ops, then padding rows. +const KAT_STORE_OPS: usize = 11; +const KAT_STORE_HEIGHT: usize = 16; +const KAT_SHIFT_OPS: usize = 123; +const KAT_SHIFT_HEIGHT: usize = 128; +const KAT_MEMW_OPS: usize = 13; +const KAT_MEMW_HEIGHT: usize = 16; +/// The known-answer KECCAK_RND tables, `(first op, ops, skip, rows)` of +/// [`KAT_KECCAK_OPS`] ops: cut into the first op, every round of the second, +/// zero rows after; and cut at both ends. +const KAT_KECCAK_OPS: usize = 4; +const KAT_KECCAK_CASES: [(usize, usize, usize, usize); 2] = [(0, 2, 21, 32), (2, 2, 10, 20)]; + +/// The known-answer decode table (the CPU table's docs). +fn kat_decode() -> DecodeTable { + let mut rng = 0xCA4D_F111; + let mut program: Vec = Vec::new(); + for opcode in OPCODES.into_iter().filter(|&o| o != BRANCH) { + program.push(random_instruction(&mut rng, opcode, None)); + } + for funct3 in BRANCH_FUNCT3 { + program.push(random_instruction(&mut rng, BRANCH, Some(funct3))); + } + program.push(Instruction::parse(ECALL).expect("ECALL parses")); + while program.len() < 24 { + let opcode = OPCODES[(splitmix(&mut rng) % OPCODES.len() as u64) as usize]; + program.push(random_instruction(&mut rng, opcode, None)); + } + let mut instructions = U64HashMap::default(); + for (i, instruction) in program.into_iter().enumerate() { + instructions.insert(RUN_BASES[i / 8] + 4 * (i % 8) as u64, instruction); + } + DecodeTable::from_instructions(&instructions) +} + +/// FNV-1a over `words`' little-endian bytes: the harness's digest of a row's +/// run of KECCAK_RND columns. +pub(crate) fn fnv1a(words: impl IntoIterator) -> u64 { + let mut h = 0xCBF2_9CE4_8422_2325u64; + for w in words { + for b in w.to_le_bytes() { + h = (h ^ u64::from(b)).wrapping_mul(0x0000_0100_0000_01B3); + } + } + h +} + +/// The KECCAK_RND column groups' ends, in column order (`keccak_rnd::cols`). +fn keccak_group_ends() -> [usize; 15] { + use keccak_rnd::cols; + [ + cols::ROUND, + cols::START, + cols::CXZ, + cols::CXZ_LEFT, + cols::CXZ_RIGHT, + cols::DXZ, + cols::THETA, + cols::ROT_LEFT, + cols::ROT_RIGHT, + cols::CHI_ANDS, + cols::CHI, + cols::RC, + cols::IOTA, + cols::MU, + cols::NUM_COLUMNS, + ] +} + +/// `values` as a C initializer list, `per_line` to a line. +fn c_list(values: &[String], per_line: usize) -> String { + values + .chunks(per_line) + .map(|line| format!(" {},\n", line.join(", "))) + .collect() +} + +fn u64s(values: impl IntoIterator) -> Vec { + values.into_iter().map(|v| format!("{v}ull")).collect() +} + +/// The harness's vectors (`HOST_KAT_VECTORS`): the records and the host +/// fill's words of the known-answer tables, as C++. +pub(crate) fn host_kat_vectors() -> String { + let mut s = String::from( + "// Known-answer vectors for the table fills (`kernels/fill.cu`), from the prover's host\n\ + // fill. GENERATED by `tables::card_fill_inputs` (prover); do not edit. Regenerate with\n\ + // CARD_FILL_KAT_WRITE=1 cargo test -p lambda-vm-prover --lib \\\n\ + // tables::card_fill_inputs::tests::the_host_kat_vectors_are_the_host_fills\n\ + // Include after `fill.cu` (it declares the record structs). The CPU table's decode is\n\ + // seeded random instruction words (no guest ELF): these are host-generated vectors,\n\ + // not a guest's rows.\n\ + #pragma once\n#include \n\n", + ); + + // MEMW_R. + let rows = memw_r_rows(0x4D45_4D57, KAT_MEMW_R_ROWS); + let words = col_major(&memw_r_host(&rows, TraceForm::Wide)); + assert_eq!( + words.len(), + KAT_MEMW_R_HEIGHT * math_cuda::fill::MEMW_R_COLUMNS + ); + s += &format!( + "// MEMW_R: {KAT_MEMW_R_ROWS} rows, then zero rows to {KAT_MEMW_R_HEIGHT}.\n\ + inline constexpr uint64_t KAT_MEMW_R_N = {KAT_MEMW_R_ROWS};\n\ + inline constexpr uint64_t KAT_MEMW_R_HEIGHT = {KAT_MEMW_R_HEIGHT};\n\ + inline const RegRowC KAT_MEMW_R_ROWS[KAT_MEMW_R_N] = {{\n" + ); + for r in &rows { + let c = r.card_record(); + s += &format!( + " {{{}ull, {}u, {}u, {}u, {}u, {}u, {}, {}, {}}},\n", + c.timestamp, c.val0, c.val1, c.old0, c.old1, c.old_ts_lo, c.address, c.is_read, c.pad + ); + } + s += "};\n// Column-major: column c's rows from c * KAT_MEMW_R_HEIGHT.\n"; + s += "inline const uint64_t KAT_MEMW_R_WORDS[] = {\n"; + s += &c_list(&u64s(words), 6); + s += "};\n\n"; + + // MEMW_A. + let rows = memw_a_rows(0x4D45_4D41, KAT_MEMW_A_ROWS); + let words = col_major(&memw_a_host(&rows, TraceForm::Wide)); + assert_eq!( + words.len(), + KAT_MEMW_A_HEIGHT * math_cuda::fill::MEMW_A_COLUMNS + ); + s += &format!( + "// MEMW_A: {KAT_MEMW_A_ROWS} rows, then zero rows to {KAT_MEMW_A_HEIGHT}.\n\ + inline constexpr uint64_t KAT_MEMW_A_N = {KAT_MEMW_A_ROWS};\n\ + inline constexpr uint64_t KAT_MEMW_A_HEIGHT = {KAT_MEMW_A_HEIGHT};\n\ + inline const AlignedRowC KAT_MEMW_A_ROWS[KAT_MEMW_A_N] = {{\n" + ); + for c in rows.iter().map(aligned_record) { + s += &format!( + " {{{}ull, {}ull, {}ull, {}ull, {}ull, {}, {}, {}}},\n", + c.base_address, + c.timestamp, + c.old_timestamp, + c.value, + c.old, + c.width, + c.is_read, + c.is_register + ); + } + s += "};\n// Column-major: column c's rows from c * KAT_MEMW_A_HEIGHT.\n"; + s += "inline const uint64_t KAT_MEMW_A_WORDS[] = {\n"; + s += &c_list(&u64s(words), 6); + s += "};\n\n"; + + // LOAD. + let ops = load_ops(0x4C4F_4144, KAT_LOAD_OPS); + let words = col_major(&load_host(&ops, TraceForm::Wide)); + assert_eq!(words.len(), KAT_LOAD_HEIGHT * math_cuda::fill::LOAD_COLUMNS); + s += &format!( + "// LOAD: {KAT_LOAD_OPS} ops, then zero rows to {KAT_LOAD_HEIGHT}.\n\ + inline constexpr uint64_t KAT_LOAD_N = {KAT_LOAD_OPS};\n\ + inline constexpr uint64_t KAT_LOAD_HEIGHT = {KAT_LOAD_HEIGHT};\n\ + inline const LoadOpC KAT_LOAD_OPS[KAT_LOAD_N] = {{\n" + ); + for c in ops.iter().map(load_record) { + s += &format!( + " {{{}ull, {}ull, {}, {}, {{{}}}}},\n", + c.base_address, + c.timestamp, + c.width, + c.is_signed, + c.res.map(|b| b.to_string()).join(", ") + ); + } + s += "};\n// Column-major: column c's rows from c * KAT_LOAD_HEIGHT.\n"; + s += "inline const uint64_t KAT_LOAD_WORDS[] = {\n"; + s += &c_list(&u64s(words), 6); + s += "};\n\n"; + + // STORE. + let ops = store_ops(0x5354_4F52, KAT_STORE_OPS); + let words = col_major(&store_host(&ops, TraceForm::Wide)); + assert_eq!( + words.len(), + KAT_STORE_HEIGHT * math_cuda::fill::STORE_COLUMNS + ); + s += &format!( + "// STORE: {KAT_STORE_OPS} ops, then zero rows to {KAT_STORE_HEIGHT}.\n\ + inline constexpr uint64_t KAT_STORE_N = {KAT_STORE_OPS};\n\ + inline constexpr uint64_t KAT_STORE_HEIGHT = {KAT_STORE_HEIGHT};\n\ + inline const StoreOpC KAT_STORE_OPS[KAT_STORE_N] = {{\n" + ); + for c in ops.iter().map(store_record) { + s += &format!( + " {{{}ull, {}ull, {}ull, {}, {}, {}}},\n", + c.base_address, c.timestamp, c.value, c.write2, c.write4, c.write8 + ); + } + s += "};\n// Column-major: column c's rows from c * KAT_STORE_HEIGHT.\n"; + s += "inline const uint64_t KAT_STORE_WORDS[] = {\n"; + s += &c_list(&u64s(words), 6); + s += "};\n\n"; + + // SHIFT. + let ops = shift_ops(0x5348_4946, KAT_SHIFT_OPS); + let words = col_major(&shift_host(&ops, TraceForm::Wide)); + assert_eq!( + words.len(), + KAT_SHIFT_HEIGHT * math_cuda::fill::SHIFT_COLUMNS + ); + s += &format!( + "// SHIFT: {KAT_SHIFT_OPS} ops (every direction, signedness and width at each edge amount),\n\ + // then padding rows (ZBS 1) to {KAT_SHIFT_HEIGHT}.\n\ + inline constexpr uint64_t KAT_SHIFT_N = {KAT_SHIFT_OPS};\n\ + inline constexpr uint64_t KAT_SHIFT_HEIGHT = {KAT_SHIFT_HEIGHT};\n\ + inline const ShiftOpC KAT_SHIFT_OPS[KAT_SHIFT_N] = {{\n" + ); + for c in ops.iter().map(shift_record) { + s += &format!( + " {{{}ull, {{{}}}, {}, {}, {}, {}}},\n", + c.shift_amount, + c.in_halves.map(|h| h.to_string()).join(", "), + c.shift, + c.direction, + c.is_signed, + c.word_instr + ); + } + s += "};\n// Column-major: column c's rows from c * KAT_SHIFT_HEIGHT.\n"; + s += "inline const uint64_t KAT_SHIFT_WORDS[] = {\n"; + s += &c_list(&u64s(words), 6); + s += "};\n\n"; + + // MEMW. + let ops = memw_ops(0x4D45_4D57, KAT_MEMW_OPS); + let words = col_major(&memw_host(&ops, TraceForm::Wide)); + assert_eq!(words.len(), KAT_MEMW_HEIGHT * math_cuda::fill::MEMW_COLUMNS); + s += &format!( + "// MEMW: {KAT_MEMW_OPS} ops, then zero rows to {KAT_MEMW_HEIGHT}.\n\ + inline constexpr uint64_t KAT_MEMW_N = {KAT_MEMW_OPS};\n\ + inline constexpr uint64_t KAT_MEMW_HEIGHT = {KAT_MEMW_HEIGHT};\n\ + inline const MemwOpC KAT_MEMW_OPS[KAT_MEMW_N] = {{\n" + ); + for c in ops.iter().map(memw_record) { + s += &format!( + " {{{}ull, {}ull,\n {{{}}},\n {{{}}},\n {{{}}}, {}, {}, {}}},\n", + c.base_address, + c.timestamp, + u64s(c.old_timestamp).join(", "), + c.value.map(|v| format!("{v}u")).join(", "), + c.old.map(|v| format!("{v}u")).join(", "), + c.is_register, + c.width, + c.is_read + ); + } + s += "};\n// Column-major: column c's rows from c * KAT_MEMW_HEIGHT.\n"; + s += "inline const uint64_t KAT_MEMW_WORDS[] = {\n"; + s += &c_list(&u64s(words), 6); + s += "};\n\n"; + + // CPU. + let decode = kat_decode(); + let (entries, runs) = decode_records(&decode); + let mut rng = 0x4350_5521u64; + let logs: Vec = (0..KAT_CPU_LOGS) + .map(|i| cpu_log(&mut rng, &decode, i % entries.len(), i % 3 == 0)) + .collect(); + let ops = cpu_ops(&logs, KAT_CPU_FIRST_CYCLE, &decode); + let words = col_major(&cpu_host(&ops, &decode, TraceForm::Wide)); + assert_eq!(words.len(), KAT_CPU_HEIGHT * math_cuda::fill::CPU_COLUMNS); + let absent = runs[0].first_pc + 2; + assert_eq!(decode.row(absent), None); + s += &format!( + "// CPU: {} decode entries in {} runs; {KAT_CPU_LOGS} logs (their ops from cycle\n\ + // {KAT_CPU_FIRST_CYCLE}), each entry in turn, then padding rows to {KAT_CPU_HEIGHT}; every row's\n\ + // timestamp the table's own 4·(row + 1). KAT_CPU_ABSENT_PC is no entry's.\n\ + inline constexpr uint64_t KAT_CPU_N = {KAT_CPU_LOGS};\n\ + inline constexpr uint64_t KAT_CPU_HEIGHT = {KAT_CPU_HEIGHT};\n\ + inline constexpr uint64_t KAT_CPU_ABSENT_PC = {absent}ull;\n\ + inline constexpr uint32_t KAT_CPU_ENTRIES_N = {};\n\ + inline constexpr uint32_t KAT_CPU_RUNS_N = {};\n\ + inline const DecodeEntryC KAT_CPU_ENTRIES[KAT_CPU_ENTRIES_N] = {{\n", + entries.len(), + runs.len(), + entries.len(), + runs.len() + ); + for e in &entries { + s += &format!( + " {{{}ull, {}ull, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}}},\n", + e.pc, + e.imm, + e.read_register1, + e.read_register2, + e.write_register, + e.word_instr, + e.alu, + e.add, + e.sub, + e.memory, + e.branch, + e.ecall, + e.rs1, + e.rs2, + e.rd, + e.half_instruction_length, + e.alu_flags, + e.mem_flags + ); + } + s += "};\ninline const DecodeRunC KAT_CPU_RUNS[KAT_CPU_RUNS_N] = {\n"; + for r in &runs { + s += &format!(" {{{}ull, {}u, {}u}},\n", r.first_pc, r.row, r.len); + } + s += "};\ninline const LogC KAT_CPU_LOGS[KAT_CPU_N] = {\n"; + for l in logs.iter().map(log_record) { + s += &format!( + " {{{}ull, {}ull, {}ull, {}ull, {}ull}},\n", + l.current_pc, l.next_pc, l.src1_val, l.src2_val, l.dst_val + ); + } + s += "};\n// Column-major: column c's rows from c * KAT_CPU_HEIGHT.\n"; + s += "inline const uint64_t KAT_CPU_WORDS[] = {\n"; + s += &c_list(&u64s(words), 6); + s += "};\n\n"; + + // KECCAK_RND. + let ops = keccak_ops(0x4B45_4343, KAT_KECCAK_OPS); + let ends = keccak_group_ends(); + s += &format!( + "// KECCAK_RND: {KAT_KECCAK_OPS} ops; each case a table of rows [skip, skip + rows) of ops\n\ + // [first_op, first_op + nops); KAT_KECCAK_DIGESTS: per case, per row, per column group\n\ + // (columns [KAT_KECCAK_GROUP_END[g - 1], KAT_KECCAK_GROUP_END[g])), FNV-1a of the words'\n\ + // little-endian bytes.\n\ + inline constexpr int KAT_KECCAK_GROUPS = {};\n\ + inline const uint32_t KAT_KECCAK_GROUP_END[KAT_KECCAK_GROUPS] = {{{}}};\n\ + struct KatKeccakCase {{\n uint32_t first_op;\n uint32_t nops;\n uint64_t skip;\n uint64_t rows;\n}};\n\ + inline constexpr int KAT_KECCAK_CASES_N = {};\n\ + inline const KatKeccakCase KAT_KECCAK_CASES[KAT_KECCAK_CASES_N] = {{\n", + ends.len(), + ends.map(|e| e.to_string()).join(", "), + KAT_KECCAK_CASES.len() + ); + for (first, nops, skip, rows) in KAT_KECCAK_CASES { + s += &format!(" {{{first}, {nops}, {skip}, {rows}}},\n"); + } + s += &format!("}};\ninline const KeccakOpC KAT_KECCAK_OPS[{KAT_KECCAK_OPS}] = {{\n"); + for op in ops.iter().map(keccak_record) { + s += &format!( + " {{{}ull,\n {{{}}},\n {{{}}}}},\n", + op.timestamp, + u64s(op.input).join(", "), + u64s(op.output).join(", ") + ); + } + s += "};\ninline const uint64_t KAT_KECCAK_DIGESTS[] = {\n"; + let mut digests = Vec::new(); + for (first, nops, skip, rows) in KAT_KECCAK_CASES { + let words = col_major(&keccak_rnd_host( + &ops[first..first + nops], + skip, + rows, + TraceForm::Wide, + )); + for row in 0..rows { + let mut start = 0; + for &end in &ends { + digests.push(fnv1a((start..end).map(|c| words[c * rows + row]))); + start = end; + } + } + } + s += &c_list(&u64s(digests), 4); + s += "};\n"; + s +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tables::{cpu, memw_register}; + + /// The card's column counts are the prover's tables', and a log is the + /// record it uploads as. + #[test] + fn the_card_layouts_are_the_prover_tables() { + use math_cuda::fill as f; + assert_eq!(f::MEMW_R_COLUMNS, memw_register::cols::NUM_COLUMNS); + assert_eq!(f::MEMW_A_COLUMNS, memw_aligned::cols::NUM_COLUMNS); + assert_eq!(f::LOAD_COLUMNS, load::cols::NUM_COLUMNS); + assert_eq!(f::STORE_COLUMNS, store::cols::NUM_COLUMNS); + assert_eq!(f::SHIFT_COLUMNS, shift::cols::NUM_COLUMNS); + assert_eq!(f::MEMW_COLUMNS, memw::cols::NUM_COLUMNS); + assert_eq!(f::CPU_COLUMNS, cpu::cols::NUM_COLUMNS); + assert_eq!(f::KECCAK_RND_COLUMNS, keccak_rnd::cols::NUM_COLUMNS); + assert_eq!(std::mem::size_of::(), std::mem::size_of::()); + assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 408); + } + + /// The bytes of `value` as the card receives them. + fn bytes_of(value: &T) -> Vec { + // SAFETY: `T` is one of the `#[repr(C)]` records, plain data with no + // padding (asserted in `math_cuda::fill`). + unsafe { + std::slice::from_raw_parts((value as *const T).cast::(), std::mem::size_of::()) + } + .to_vec() + } + + /// The first `n` bytes of `value` as the card receives them: `T`'s fields + /// up to its trailing padding. + fn field_bytes(value: &T, n: usize) -> Vec { + assert!(n <= std::mem::size_of::()); + // SAFETY: `T` is `#[repr(C)]` and its first `n` bytes are fields with + // no padding between them (asserted by the callers' offsets), so every + // byte read is initialized. + unsafe { std::slice::from_raw_parts((value as *const T).cast::(), n) }.to_vec() + } + + /// A register row's record: its fields, and its 32 bytes as uploaded. + #[test] + fn a_register_row_encodes_field_for_field() { + let row = RegRow::new( + 2 * 37, + 0x3_0000_0010, + 0x1111_2222, + 0x3333_4444, + 0x5555_6666, + 0x7777_8888, + 0x3_0000_0001, + true, + ); + let c = row.card_record(); + let want = math_cuda::fill::RegRowC { + timestamp: 0x3_0000_0010, + val0: 0x1111_2222, + val1: 0x3333_4444, + old0: 0x5555_6666, + old1: 0x7777_8888, + old_ts_lo: 1, + address: 37, + is_read: 1, + pad: 0, + }; + assert_eq!(c, want); + #[rustfmt::skip] + let bytes = [ + 0x10, 0, 0, 0, 3, 0, 0, 0, + 0x22, 0x22, 0x11, 0x11, 0x44, 0x44, 0x33, 0x33, + 0x66, 0x66, 0x55, 0x55, 0x88, 0x88, 0x77, 0x77, + 1, 0, 0, 0, 37, 0, 1, 0, + ]; + assert_eq!(bytes_of(&c), bytes); + let write = RegRow::new(0, 9, 0, 0, 0, 0, 8, false).card_record(); + assert_eq!((write.is_read, write.address, write.old_ts_lo), (0, 0, 8)); + } + + /// An aligned row's record: its fields, and its first 43 bytes as + /// uploaded (the rest is padding no kernel reads). + #[test] + fn an_aligned_row_encodes_field_for_field() { + let row = AlignedRow::from_parts( + 0x1122_3344_5566_7788, + 0x3_0000_0010, + 0x2_FFFF_FFF0, + 0x0102_0304_0506_0708, + u64::MAX, + 4, + true, + false, + ); + let c = aligned_record(&row); + let want = AlignedRowC { + base_address: 0x1122_3344_5566_7788, + timestamp: 0x3_0000_0010, + old_timestamp: 0x2_FFFF_FFF0, + value: 0x0102_0304_0506_0708, + old: u64::MAX, + width: 4, + is_read: 1, + is_register: 0, + }; + assert_eq!(c, want); + let bytes = field_bytes(&row, 43); + let words: Vec = bytes[..40] + .chunks_exact(8) + .map(|b| u64::from_le_bytes(b.try_into().unwrap())) + .collect(); + assert_eq!( + words, + [ + 0x1122_3344_5566_7788, + 0x3_0000_0010, + 0x2_FFFF_FFF0, + 0x0102_0304_0506_0708, + u64::MAX + ] + ); + assert_eq!(bytes[40..43], [4, 1, 0]); + let register = AlignedRow::from_parts(0, 9, 8, 0, 0, 8, false, true); + assert_eq!(field_bytes(®ister, 43)[40..43], [8, 0, 1]); + } + + /// A load op's record: its fields, and its first 26 bytes as uploaded + /// (the rest is padding no kernel reads). + #[test] + fn a_load_op_encodes_field_for_field() { + let op = LoadOperation::new( + 0xFF00_0000_0000_0008, + 0x1_0000_0004, + 2, + true, + [9, 8, 7, 6, 5, 4, 3, 2], + ); + let c = load_record(&op); + assert_eq!( + c, + LoadOpC { + base_address: 0xFF00_0000_0000_0008, + timestamp: 0x1_0000_0004, + width: 2, + is_signed: 1, + res: [9, 8, 7, 6, 5, 4, 3, 2], + } + ); + let bytes = field_bytes(&op, 26); + assert_eq!(bytes[..8], 0xFF00_0000_0000_0008u64.to_le_bytes()); + assert_eq!(bytes[8..16], 0x1_0000_0004u64.to_le_bytes()); + assert_eq!(bytes[16..26], [2, 1, 9, 8, 7, 6, 5, 4, 3, 2]); + let unsigned = LoadOperation::new(0, 8, 1, false, [0; 8]); + assert_eq!(field_bytes(&unsigned, 26)[16..18], [1, 0]); + } + + /// A store, shift and MEMW op's record: its fields, and its field bytes as + /// uploaded (the rest is padding no kernel reads). + #[test] + fn the_op_records_encode_field_for_field() { + let store = StoreOperation::new(0x1122_3344_5566_7788, 0x3_0000_0010, u64::MAX - 2, 4); + assert_eq!( + store_record(&store), + StoreOpC { + base_address: 0x1122_3344_5566_7788, + timestamp: 0x3_0000_0010, + value: u64::MAX - 2, + write2: 0, + write4: 1, + write8: 0, + } + ); + let bytes = field_bytes(&store, 27); + assert_eq!(bytes[16..24], (u64::MAX - 2).to_le_bytes()); + assert_eq!(bytes[24..27], [0, 1, 0]); + let shift = ShiftOperation::new(0x8001_0203_0405_0607, 0x1_0000_0123, true, true, false); + assert_eq!( + shift_record(&shift), + ShiftOpC { + shift_amount: 0x1_0000_0123, + in_halves: [0x0607, 0x0405, 0x0203, 0x8001], + shift: 0x23, + direction: 1, + is_signed: 1, + word_instr: 0, + } + ); + let bytes = field_bytes(&shift, 20); + assert_eq!(bytes[..8], 0x1_0000_0123u64.to_le_bytes()); + assert_eq!(bytes[8..10], [0x07, 0x06]); + assert_eq!(bytes[16..20], [0x23, 1, 1, 0]); + let memw = &memw_ops(9, 2)[1]; + let c = memw_record(memw); + assert_eq!( + (c.base_address, c.timestamp, c.old_timestamp), + (memw.base_address, memw.timestamp, memw.old_timestamp) + ); + assert_eq!((c.value, c.old), (memw.value, memw.old)); + assert_eq!( + (c.is_register, c.width, c.is_read), + ( + u8::from(memw.is_register), + memw.width, + u8::from(memw.is_read) + ) + ); + let bytes = field_bytes(memw, 147); + assert_eq!(bytes[16..24], memw.old_timestamp[0].to_le_bytes()); + assert_eq!(bytes[80..84], memw.value[0].to_le_bytes()); + assert_eq!(bytes[112..116], memw.old[0].to_le_bytes()); + assert_eq!( + bytes[144..147], + [ + u8::from(memw.is_register), + memw.width, + u8::from(memw.is_read) + ] + ); + } + + /// A log's record: its five words in order. + #[test] + fn a_log_encodes_field_for_field() { + let log = Log { + current_pc: 0x1000, + next_pc: 0x1004, + src1_val: u64::MAX, + src2_val: 7, + dst_val: 0x0102_0304_0506_0708, + }; + let c = log_record(&log); + assert_eq!( + (c.current_pc, c.next_pc, c.src1_val, c.src2_val, c.dst_val), + (0x1000, 0x1004, u64::MAX, 7, 0x0102_0304_0506_0708) + ); + let words: Vec = bytes_of(&c) + .chunks_exact(8) + .map(|b| u64::from_le_bytes(b.try_into().unwrap())) + .collect(); + assert_eq!(words, [0x1000, 0x1004, u64::MAX, 7, 0x0102_0304_0506_0708]); + } + + /// The known-answer decode table's records: every entry's fields, the + /// runs, and every pc found where the prover finds it (and the pcs around + /// them nowhere). + #[test] + fn a_decode_table_encodes_row_for_row() { + let decode = kat_decode(); + let (entries, runs) = decode_records(&decode); + assert_eq!(entries.len(), 24); + assert_eq!( + runs, + RUN_BASES.map(|first_pc| DecodeRunC { + first_pc, + row: entries.iter().position(|e| e.pc == first_pc).unwrap() as u32, + len: 8, + }) + ); + for (row, (e, c)) in decode.entries().iter().zip(&entries).enumerate() { + let f = &e.fields; + assert_eq!((c.pc, c.imm), (e.pc, e.imm), "row {row}"); + let flags = [ + f.read_register1, + f.read_register2, + f.write_register, + f.word_instr, + f.alu, + f.add, + f.sub, + f.memory, + f.branch, + f.ecall, + ] + .map(u8::from); + let bytes = bytes_of(c); + assert_eq!(bytes[16..26], flags, "row {row}"); + assert_eq!( + bytes[26..32], + [ + f.rs1, + f.rs2, + f.rd, + f.half_instruction_length, + f.alu_flags, + f.mem_flags + ], + "row {row}" + ); + assert_eq!(DecodeRunC::find(&runs, e.pc), Some(row as u32), "row {row}"); + for miss in [e.pc + 2, e.pc.wrapping_sub(2)] { + assert_eq!( + DecodeRunC::find(&runs, miss), + decode.row(miss), + "pc {miss:#x}" + ); + } + } + for pc in [0, RUN_BASES[0] - 4, RUN_BASES[1] + 32, !3] { + assert_eq!(DecodeRunC::find(&runs, pc), None, "pc {pc:#x}"); + assert_eq!(decode.row(pc), None, "pc {pc:#x}"); + } + // Every op class the CPU fill branches on is in the table. + let has = |p: &dyn Fn(&DecodeEntryC) -> bool| entries.iter().any(p); + assert!(has(&|e| e.word_instr == 1)); + assert!(has(&|e| e.memory == 1 && e.mem_flags & 1 == 1), "a store"); + assert!(has(&|e| e.memory == 1 && e.mem_flags & 1 == 0), "a load"); + assert!(has(&|e| e.branch == 1 && e.mem_flags & 1 == 1), "a JALR"); + assert!(has(&|e| e.rs1 == 255), "a pc read"); + assert!(has(&|e| e.ecall == 1)); + assert!(has(&|e| e.add == 1) && has(&|e| e.sub == 1) && has(&|e| e.alu == 1)); + } + + /// A keccak operation's record: the timestamp, then input and output. + #[test] + fn a_keccak_op_encodes_field_for_field() { + let op = &keccak_ops(9, 1)[0]; + let c = keccak_record(op); + assert_eq!( + (c.timestamp, c.input, c.output), + (op.timestamp, op.input, op.output) + ); + let words: Vec = bytes_of(&c) + .chunks_exact(8) + .map(|b| u64::from_le_bytes(b.try_into().unwrap())) + .collect(); + assert_eq!(words[0], op.timestamp); + assert_eq!(words[1..26], op.input); + assert_eq!(words[26..51], op.output); + } + + /// The synthetic sets are what the module docs say. + #[test] + fn the_synthetic_sets_have_their_shapes() { + for r in memw_r_rows(SEEDS[0], 1 << 12) + .iter() + .map(RegRow::card_record) + { + let ts_lo = (r.timestamp & 0xFFFF_FFFF) as u32; + assert!((8..1 << 34).contains(&r.timestamp)); + assert!(ts_lo > r.old_ts_lo && ts_lo - r.old_ts_lo <= 1 << 16); + } + let decode = parsed_decode(SEEDS[0], 256); + assert_eq!(decode.runs().len(), RUN_BASES.len()); + let logs = cpu_logs(SEEDS[0], 1 << 12, &decode); + assert!(logs.iter().all(|l| decode.row(l.current_pc).is_some())); + let ops = keccak_ops(SEEDS[0], KECCAK_RND_OPS); + assert!(ops.windows(2).all(|w| w[0].timestamp < w[1].timestamp)); + assert!(ops[0].timestamp < 1 << 32 && ops[KECCAK_RND_OPS - 1].timestamp > 1 << 32); + } + + /// [`mismatches`] of `packed`'s words against `wide`'s, a column at a + /// time (a 2^16-row KECCAK_RND table is 776 MB at 8 bytes a cell). + fn packed_and_wide(packed: &Trace, wide: &Trace) -> Mismatches { + let rows = wide.num_rows(); + let want = col_major(wide); + let Some(columns) = packed.narrow_main() else { + // A build that holds no table packed (`debug-checks`). + return mismatches(&want, &col_major(packed), rows); + }; + let mut out = Mismatches { + count: want.len().abs_diff(rows * columns.cols()), + first: None, + }; + for c in 0..columns.cols().min(wide.main_table.width) { + let m = mismatches(&want[c * rows..(c + 1) * rows], &columns.column(c), rows); + out.count += m.count; + if out.first.is_none() { + out.first = m.first.map(|(_, row, w, g)| (c, row, w, g)); + } + } + out + } + + /// Every synthetic input's host fill is the same words packed as at 8 + /// bytes a cell (G-pack's invariant, which the card's identity rests on). + #[test] + fn every_synthetic_input_is_the_same_words_wide_and_packed() { + let decode = program_decode(); + for seed in SEEDS { + let rows = memw_r_rows(seed, CHUNK_ROWS); + let m = packed_and_wide( + &memw_r_host(&rows, TraceForm::Narrow), + &memw_r_host(&rows, TraceForm::Wide), + ); + assert_eq!(m, Mismatches::default(), "MEMW_R seed {seed}"); + + let ops = cpu_ops( + &cpu_logs(seed, CHUNK_ROWS, &decode), + CPU_FIRST_CYCLE, + &decode, + ); + let m = packed_and_wide( + &cpu_host(&ops, &decode, TraceForm::Narrow), + &cpu_host(&ops, &decode, TraceForm::Wide), + ); + assert_eq!(m, Mismatches::default(), "CPU seed {seed}"); + + let ops = keccak_ops(seed, KECCAK_RND_OPS); + let m = packed_and_wide( + &keccak_rnd_host(&ops, KECCAK_RND_SKIP, KECCAK_RND_ROWS, TraceForm::Narrow), + &keccak_rnd_host(&ops, KECCAK_RND_SKIP, KECCAK_RND_ROWS, TraceForm::Wide), + ); + assert_eq!(m, Mismatches::default(), "KECCAK_RND seed {seed}"); + } + } + + /// The comparison the identity checks rest on finds one changed word, + /// where it is. + #[test] + fn one_changed_word_is_one_mismatch() { + let rows = memw_r_rows(SEEDS[1], 100); + let want = col_major(&memw_r_host(&rows, TraceForm::Wide)); + let height = 128; + let mut got = col_major(&memw_r_host(&rows, TraceForm::Narrow)); + assert_eq!(mismatches(&want, &got, height), Mismatches::default()); + let at = memw_register::cols::OLD_1 * height + 41; + got[at] ^= 1 << 7; + assert_eq!( + mismatches(&want, &got, height), + Mismatches { + count: 1, + first: Some((memw_register::cols::OLD_1, 41, want[at], got[at])), + } + ); + assert_eq!(mismatches(&want, &got[..got.len() - 3], height).count, 4); + } + + /// The harness's vectors are the host fills' of today: regenerated and + /// compared with the committed file (`CARD_FILL_KAT_WRITE=1` writes it). + #[test] + fn the_host_kat_vectors_are_the_host_fills() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("the workspace root") + .join(HOST_KAT_VECTORS); + let text = host_kat_vectors(); + if std::env::var("CARD_FILL_KAT_WRITE").is_ok_and(|v| v == "1") { + std::fs::write(&path, &text).expect("write the vectors"); + } + let committed = std::fs::read_to_string(&path).unwrap_or_default(); + assert!( + committed == text, + "{} is not the host fills' vectors; regenerate it (its header says how)", + path.display() + ); + } +} diff --git a/prover/src/tables/card_fill_tests.rs b/prover/src/tables/card_fill_tests.rs new file mode 100644 index 000000000..bbf0434e8 --- /dev/null +++ b/prover/src/tables/card_fill_tests.rs @@ -0,0 +1,789 @@ +//! ★ Box only (`--ignored`, a card): the card's table fills from records +//! (`math_cuda::fill`) against today's path, the host's packed fill widened on +//! the card, for MEMW_R, CPU and KECCAK_RND. +//! +//! Each case runs both paths on the same rows: +//! - **today:** the host fill in [`TraceForm::Narrow`] on this thread (timed), +//! its packed bytes uploaded and widened (`fill::widen_col_major_timed`); +//! - **the card:** the case's records uploaded and filled by the family's +//! kernel. +//! +//! Both slabs are downloaded and compared word for word, against each other +//! and against the host fill's [`TraceForm::Wide`] words. A mutant (one record +//! changed) must then differ from the host, which shows the comparison sees +//! the records. Timed cases repeat both paths 3 + 20 times and print the +//! median, min and max of the host fill, the uploads and the kernels. Each +//! case prints one `CARD FILL key=value …` line (KECCAK_RND's +//! with `minimal_bytes`, its records without the output the kernel never +//! reads). +//! +//! The inputs ([`super::card_fill_inputs`]): +//! - three seeded sets a family, CPU's over the decode table of the guest at +//! `BLOCK_WHIR_ELF`; +//! - real chunks of the block at `BLOCK_WHIR_ELF` / `BLOCK_WHIR_INPUT`. The run +//! is executed and walked with one walker (the split builder, as the +//! streamed prove does) through window 65. The CPU chunk is chunk 32: the +//! logs of windows 64 and 65 against the walk's ops. The MEMW_R chunk is the +//! first full register chunk handed out after window 64; +//! - one padded table a family, identity only. +//! +//! KECCAK_RND runs first, then prints its own `CARD FILL VERDICT KECCAK_RND`, +//! so its rows stand even if the real block's capture fails. Then come CPU +//! and MEMW_R, their real chunks before their synthetic sets, so the host +//! fill's widths are learned from real rows as in a prove. A CPU fill with +//! one log at a pc no DECODE row holds must be refused (`CARD FILL REFUSAL`), +//! and on every CPU case a fault of column 0 (the table's preprocessed +//! timestamp) alone on the card must be flagged there alone (`col0_fault`). +//! The loaded cases repeat the first KECCAK_RND set's and the real chunks' +//! uploads beside four threads copying 1 GiB buffers (a walker's memory +//! traffic), as `CARD FILL LOADED` lines. `CARD FILL VERDICT` closes the run. +//! +//! ```text +//! BLOCK_WHIR_ELF=…/ethrex_3e8f1ce1.elf BLOCK_WHIR_INPUT=…/block.bin \ +//! cargo test --release -p lambda-vm-prover --lib --features cuda \ +//! tables::card_fill_tests::the_card_fills_the_streamed_tables_as_the_host_does \ +//! -- --exact --ignored --nocapture +//! ``` + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::Relaxed}; +use std::time::Instant; + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; +use math_cuda::fill::{self, CardDecode, FillTimes, KeccakOpC, LogC, RegRowC}; +use math_cuda::narrow::NarrowInput; +use math_cuda::{CudaSlice, CudaStream}; + +use super::card_fill_inputs::{self as inputs, CHUNK_ROWS, KECCAK_RND_ROWS, KECCAK_RND_SKIP}; +use super::cpu::generate_cpu_trace_segments_as; +use super::gpack::{Trace, TraceForm}; +use super::keccak_rnd::KeccakRoundOperation; +use super::memw_register::generate_memw_register_trace_from_rows_segments_as; +use super::trace_builder::{ChunkJob, StreamTable, WindowedTraceBuilder}; + +const WARMUPS: usize = 3; +const REPS: usize = 20; +/// The real CPU chunk, and the window its MEMW_R chunk is handed out at or +/// after: chunk 32 is windows 64 and 65. +const REAL_CPU_CHUNK: usize = 32; +const REAL_FIRST_WINDOW: usize = 64; +/// Rows a padded case leaves off a full table. +const PADDED_SHORT: usize = 12_345; +/// The padded KECCAK_RND case's ops: its last rows are past them. +const PADDED_KECCAK_OPS: usize = 2700; +/// The loaded cases' copy threads and their buffers. +const LOAD_THREADS: usize = 4; +const LOAD_BYTES: usize = 1 << 30; + +/// What a case's card path fills from. +enum Records<'a> { + MemwR(Vec), + Cpu { + logs: Vec, + decode: &'a CardDecode, + }, + KeccakRnd { + ops: Vec, + skip: usize, + }, +} + +impl Records<'_> { + /// The card's fill of the case's `rows` rows into `out`. + fn fill(&self, stream: &Arc, rows: usize, out: &mut CudaSlice) -> FillTimes { + let dst = &mut out.slice_mut(..); + match self { + Records::MemwR(records) => fill::fill_memw_r_into(stream, records, rows, dst), + Records::Cpu { logs, decode } => { + fill::fill_cpu_from_logs_into(stream, logs, decode, rows, dst) + } + Records::KeccakRnd { ops, skip } => { + fill::fill_keccak_rnd_into(stream, ops, *skip, rows, dst) + } + } + .expect("the card's fill") + } + + /// The bytes the records would take without what the kernel never reads + /// (a keccak op's output): KECCAK_RND's only. + fn minimal_bytes(&self) -> Option { + match self { + Records::KeccakRnd { ops, .. } => { + Some(ops.len() * std::mem::offset_of!(KeccakOpC, output)) + } + _ => None, + } + } + + /// The records with one changed where it moves a word: a register row's + /// value, a log's pc (to another instruction's), a keccak op's input lane. + fn mutant(&self) -> Self { + match self { + Records::MemwR(records) => { + let mut records = records.clone(); + let at = records.len() / 2; + records[at].val0 ^= 1; + Records::MemwR(records) + } + Records::Cpu { logs, decode } => { + let mut logs = logs.clone(); + let at = (logs.len() / 2..logs.len() - 1) + .find(|&i| logs[i].current_pc != logs[i + 1].current_pc) + .expect("two logs at different pcs"); + logs[at].current_pc = logs[at + 1].current_pc; + Records::Cpu { logs, decode } + } + Records::KeccakRnd { ops, skip } => { + let mut ops = ops.clone(); + let at = ops.len() / 2; + ops[at].input[3] ^= 1 << 17; + Records::KeccakRnd { ops, skip: *skip } + } + } + } +} + +/// One family's table on one input. +struct Case<'a> { + family: &'static str, + input: String, + rows: usize, + cols: usize, + /// The host fill, in a form. + host: Box Trace + 'a>, + records: Records<'a>, + /// Timed, or identity only. + timed: bool, +} + +/// Milliseconds since `t`. +fn ms(t: Instant) -> f64 { + t.elapsed().as_secs_f64() * 1e3 +} + +/// `key=median key_min=… key_max=…` of `values`. +fn stat(key: &str, values: &[f64]) -> String { + let mut v = values.to_vec(); + v.sort_by(f64::total_cmp); + let median = if v.is_empty() { + f64::NAN + } else if v.len() % 2 == 1 { + v[v.len() / 2] + } else { + (v[v.len() / 2 - 1] + v[v.len() / 2]) / 2.0 + }; + format!( + "{key}={median:.4} {key}_min={:.4} {key}_max={:.4}", + v.first().copied().unwrap_or(f64::NAN), + v.last().copied().unwrap_or(f64::NAN) + ) +} + +/// One path's repetitions. +#[derive(Default)] +struct Reps { + upload: Vec, + kernel: Vec, + wall: Vec, + bytes: usize, +} + +impl Reps { + fn push(&mut self, start: Instant, times: FillTimes) { + self.wall.push(ms(start)); + self.upload.push(times.upload_ms); + self.kernel.push(times.kernel_ms); + self.bytes = times.upload_bytes; + } + + fn line(&self, path: &str) -> String { + format!( + "{} {} {} {path}_bytes={}", + stat(&format!("{path}_upload_ms"), &self.upload), + stat(&format!("{path}_kernel_ms"), &self.kernel), + stat(&format!("{path}_wall_ms"), &self.wall), + self.bytes + ) + } +} + +/// Four threads copying 1 GiB buffers until stopped: the host's memory +/// traffic beside the walkers. +struct HostLoad { + stop: Arc, + copied: Arc, + threads: Vec>, + started: Instant, + /// Bytes copied by then. + base: u64, +} + +impl HostLoad { + fn start() -> Self { + let stop = Arc::new(AtomicBool::new(false)); + let copied = Arc::new(AtomicU64::new(0)); + let threads = (0..LOAD_THREADS) + .map(|_| { + let (stop, copied) = (stop.clone(), copied.clone()); + std::thread::spawn(move || { + let src = vec![0xA5u8; LOAD_BYTES]; + let mut dst = vec![0u8; LOAD_BYTES]; + while !stop.load(Relaxed) { + dst.copy_from_slice(&src); + std::hint::black_box(&mut dst); + copied.fetch_add(LOAD_BYTES as u64, Relaxed); + } + }) + }) + .collect(); + // Every thread's buffers touched and copying. + while copied.load(Relaxed) < (LOAD_THREADS * LOAD_BYTES) as u64 { + std::thread::yield_now(); + } + let base = copied.load(Relaxed); + Self { + stop, + copied, + threads, + started: Instant::now(), + base, + } + } + + /// Stops the threads; the GiB/s they copied since started. + fn stop(self) -> f64 { + let secs = self.started.elapsed().as_secs_f64(); + let copied = self.copied.load(Relaxed) - self.base; + self.stop.store(true, Relaxed); + for t in self.threads { + t.join().expect("a copy thread"); + } + copied as f64 / secs / (1u64 << 30) as f64 + } +} + +/// What a case found: its words that differ (today's against the host, the +/// card's against the host, the card's against today's), its mutant's, and +/// for CPU whether a fault of column 0 alone was flagged there alone. +struct Found { + mismatches: usize, + mutant: usize, + col0: Option, +} + +/// A card buffer of `cells` words, each `poison` (so a cell no kernel writes +/// is a mismatch). +fn poisoned(stream: &Arc, poison: &[u64], cells: usize) -> CudaSlice { + let mut out = stream.alloc_zeros::(cells).expect("card memory"); + repoison(stream, poison, &mut out); + out +} + +fn repoison(stream: &Arc, poison: &[u64], out: &mut CudaSlice) { + let cells = out.len(); + stream + .memcpy_htod(&poison[..cells], out) + .expect("poison the slab"); +} + +fn download(stream: &Arc, out: &CudaSlice) -> Vec { + let words = stream.clone_dtoh(out).expect("download"); + stream.synchronize().expect("the stream"); + words +} + +fn first(m: &inputs::Mismatches) -> String { + match m.first { + Some((col, row, want, got)) => format!("{col}:{row}:{want}:{got}"), + None => "none".into(), + } +} + +/// Runs `case` (the module docs); with `load`, only its timed repetitions, +/// beside the copy threads. +fn run_case(case: &Case, stream: &Arc, poison: &[u64], load: bool) -> Found { + let cells = case.rows * case.cols; + assert!(poison.len() >= cells, "the poison covers the table"); + + // Today: the host fill, packed. + let mut host_ms = Vec::new(); + let mut packed = None; + let fills = if case.timed && !load { + WARMUPS + REPS + } else { + 1 + }; + for rep in 0..fills { + let t = Instant::now(); + let trace = (case.host)(TraceForm::Narrow); + if rep >= fills.saturating_sub(REPS) && case.timed { + host_ms.push(ms(t)); + } + packed = Some(trace); + } + let packed = packed.expect("a host fill"); + let columns = packed + .narrow_main() + .expect("the host fill packs (a build without debug-checks)"); + assert_eq!( + (columns.rows(), columns.cols()), + (case.rows, case.cols), + "{} {}: the host table's shape", + case.family, + case.input + ); + let narrow = NarrowInput::new( + columns.data(), + columns.offsets(), + columns.widths(), + columns.rows(), + columns.cols(), + ); + let mut today = poisoned(stream, poison, cells); + let mut card = poisoned(stream, poison, cells); + + if load { + let host_load = HostLoad::start(); + let (mut a, mut b) = (Reps::default(), Reps::default()); + for rep in 0..WARMUPS + REPS { + let t = Instant::now(); + let times = fill::widen_col_major_timed(stream, narrow, &mut today.slice_mut(..)) + .expect("the widen"); + if rep >= WARMUPS { + a.push(t, times); + } + let t = Instant::now(); + let times = case.records.fill(stream, case.rows, &mut card); + if rep >= WARMUPS { + b.push(t, times); + } + } + let gib_s = host_load.stop(); + println!( + "CARD FILL LOADED {} {} threads={LOAD_THREADS} copy_gib_s={gib_s:.2} {} {}", + case.family, + case.input, + a.line("widen"), + b.line("fill") + ); + return Found { + mismatches: 0, + mutant: 1, + col0: None, + }; + } + + // Identity: both slabs against the host's words and each other. + let want = inputs::col_major(&(case.host)(TraceForm::Wide)); + assert_eq!(want.len(), cells); + let widen = + fill::widen_col_major_timed(stream, narrow, &mut today.slice_mut(..)).expect("the widen"); + let filled = case.records.fill(stream, case.rows, &mut card); + let (a, b) = (download(stream, &today), download(stream, &card)); + let a_host = inputs::mismatches(&want, &a, case.rows); + let b_host = inputs::mismatches(&want, &b, case.rows); + let b_a = inputs::mismatches(&a, &b, case.rows); + drop((a, b)); + + // CPU's column 0 (the preprocessed TIMESTAMP) faulted alone on the card, + // one row: the comparison flags that word and no other. + let col0 = (case.family == "CPU").then(|| { + let row = 1; + let bad = want[row] ^ 4; + stream + .memcpy_htod(&[bad], &mut card.slice_mut(row..row + 1)) + .expect("fault column 0"); + let m = inputs::mismatches(&want, &download(stream, &card), case.rows); + m.count == 1 && m.first == Some((0, row, want[row], bad)) + }); + + // The mutant: one record changed must show. + repoison(stream, poison, &mut card); + case.records.mutant().fill(stream, case.rows, &mut card); + let mutant = inputs::mismatches(&want, &download(stream, &card), case.rows); + drop(want); + + let mut line = format!( + "CARD FILL {} {} rows={} cols={} packed_bytes={} record_bytes={} \ + mismatches_widen_host={} mismatches_fill_host={} mismatches_fill_widen={} \ + first_fill_host={} mutant_mismatches={} timed={}", + case.family, + case.input, + case.rows, + case.cols, + widen.upload_bytes, + filled.upload_bytes, + a_host.count, + b_host.count, + b_a.count, + first(&b_host), + mutant.count, + case.timed, + ); + if let Some(bytes) = case.records.minimal_bytes() { + line += &format!(" minimal_bytes={bytes}"); + } + if let Some(flagged) = col0 { + line += &format!(" col0_fault={}", if flagged { "ok" } else { "FAIL" }); + } + if case.timed { + let (mut a, mut b) = (Reps::default(), Reps::default()); + for rep in 0..WARMUPS + REPS { + let t = Instant::now(); + let times = fill::widen_col_major_timed(stream, narrow, &mut today.slice_mut(..)) + .expect("the widen"); + if rep >= WARMUPS { + a.push(t, times); + } + let t = Instant::now(); + let times = case.records.fill(stream, case.rows, &mut card); + if rep >= WARMUPS { + b.push(t, times); + } + } + line += &format!( + " {} {} {}", + stat("host_fill_ms", &host_ms), + a.line("widen"), + b.line("fill") + ); + } + println!("{line}"); + Found { + mismatches: a_host.count + b_host.count + b_a.count, + mutant: mutant.count, + col0, + } +} + +/// The real block's chunks (the module docs). +struct RealChunks { + cpu: ChunkJob, + /// Windows 64 and 65's logs. + logs: Vec, + first_cycle: u64, + memw_r: ChunkJob, +} + +fn real_chunks(elf: &[u8]) -> RealChunks { + let input = + std::fs::read(std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT")).unwrap(); + let program = Elf::load(elf).expect("the ELF loads"); + let options = crate::block_whir::BlockOptions::production(); + let window = 1usize << options.window_log2.expect("the block streams"); + assert_eq!(2 * window, CHUNK_ROWS, "a CPU chunk is two windows"); + let t = Instant::now(); + let mut builder = WindowedTraceBuilder::new(&program, &input, &options.max_rows) + .expect("the builder") + .drop_streamed_ops() + .expect("before any window"); + let mut executor = Executor::new(&program, input.clone()).expect("the executor starts"); + let (mut walker, mut accumulator) = builder.split(); + let (mut cpu, mut memw_r) = (None, None); + let mut logs = Vec::with_capacity(2 * window); + for w in 0..REAL_FIRST_WINDOW + 2 { + let window_logs = executor + .resume_with_limit(window) + .expect("the block runs") + .expect("the block runs past window 65"); + assert_eq!(window_logs.len(), window, "window {w} is full"); + if w >= REAL_FIRST_WINDOW { + logs.extend_from_slice(window_logs); + } + let walked = walker.walk(window_logs).expect("the walk"); + for job in accumulator.absorb(walked) { + match job.table { + StreamTable::Cpu if job.index == REAL_CPU_CHUNK => cpu = Some(job), + StreamTable::MemwRegister if w >= REAL_FIRST_WINDOW && memw_r.is_none() => { + memw_r = Some(job) + } + _ => {} + } + } + } + let (cpu, memw_r) = ( + cpu.expect("CPU chunk 32 handed out with window 65"), + memw_r.expect("a MEMW_R chunk handed out from window 64"), + ); + println!( + "CARD FILL CAPTURE windows={} secs={:.2} memw_r_chunk={} memw_r_rows={}", + REAL_FIRST_WINDOW + 2, + t.elapsed().as_secs_f64(), + memw_r.index, + memw_r.op_count() + ); + RealChunks { + cpu, + logs, + first_cycle: (REAL_FIRST_WINDOW * window) as u64, + memw_r, + } +} + +fn sha256(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// KECCAK_RND's table of `ops` (the synthetic sets' rows and cut). +fn keccak_case(input: String, ops: &[KeccakRoundOperation], timed: bool) -> Case<'_> { + Case { + family: "KECCAK_RND", + input, + rows: KECCAK_RND_ROWS, + cols: fill::KECCAK_RND_COLUMNS, + host: Box::new(move |form| { + inputs::keccak_rnd_host(ops, KECCAK_RND_SKIP, KECCAK_RND_ROWS, form) + }), + records: Records::KeccakRnd { + ops: ops.iter().map(inputs::keccak_record).collect(), + skip: KECCAK_RND_SKIP, + }, + timed, + } +} + +/// What the cases in `found` add up to: the words that differ, and the cases +/// whose mutant showed nothing. +fn tally(found: &[(String, Found)]) -> (usize, Vec<&String>) { + let mismatches = found.iter().map(|(_, f)| f.mismatches).sum(); + let blind = found + .iter() + .filter(|(_, f)| f.mutant == 0) + .map(|(name, _)| name) + .collect(); + (mismatches, blind) +} + +/// Every family's card fill is the host fill's words, on synthetic and real +/// rows, and both paths are timed (the module docs). +#[test] +#[ignore = "card: a box test (BLOCK_WHIR_ELF, BLOCK_WHIR_INPUT)"] +fn the_card_fills_the_streamed_tables_as_the_host_does() { + let elf_path = std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF names the guest"); + let elf = std::fs::read(&elf_path).unwrap_or_else(|e| panic!("{elf_path}: {e}")); + // The synthetic CPU sets are drawn over this guest's decode table. + println!( + "CARD FILL ENV elf_sha256={} decode_source=BLOCK_WHIR_ELF reps={REPS} warmups={WARMUPS}", + sha256(&elf) + ); + let be = math_cuda::device::backend().expect("a card"); + let stream = be.next_stream(); + let poison = vec![0x5A5A_5A5A_5A5A_5A5Au64; KECCAK_RND_ROWS * fill::KECCAK_RND_COLUMNS]; + // Each case's findings (a loaded case finds nothing). + let found = std::cell::RefCell::new(Vec::new()); + let run = |case: &Case, load: bool| { + let f = run_case(case, &stream, &poison, load); + if !load { + found + .borrow_mut() + .push((format!("{} {}", case.family, case.input), f)); + } + }; + let seed = inputs::SEEDS[0]; + + // KECCAK_RND first: its rows stand even if the real block's capture fails. + let sets: Vec<_> = inputs::SEEDS + .iter() + .map(|&s| inputs::keccak_ops(s, inputs::KECCAK_RND_OPS)) + .collect(); + for (s, ops) in inputs::SEEDS.iter().zip(&sets) { + run(&keccak_case(format!("seed{s}"), ops, true), false); + } + let short = inputs::keccak_ops(seed, PADDED_KECCAK_OPS); + run(&keccak_case("padded".into(), &short, false), false); + run(&keccak_case(format!("seed{seed}"), &sets[0], true), true); + drop((sets, short)); + { + let found = found.borrow(); + let (mismatches, blind) = tally(&found); + println!( + "CARD FILL VERDICT KECCAK_RND: identity={} mismatches={mismatches} mutants={} cases={}", + if mismatches == 0 { "ok" } else { "FAIL" }, + if blind.is_empty() { "ok" } else { "FAIL" }, + found.len() + ); + } + + // The real chunks, before the synthetic sets, so the host fill's widths + // are learned from real rows as in a prove. + let real = real_chunks(&elf); + let (cpu_parts, real_decode) = real.cpu.cpu_parts().expect("a CPU chunk"); + let real_ops: usize = cpu_parts.iter().map(|s| s.len()).sum(); + assert_eq!(real_ops, CHUNK_ROWS); + assert_eq!(real.logs.len(), CHUNK_ROWS); + assert_eq!( + cpu_parts[0][0].timestamp, + 4 * real.first_cycle + 4, + "chunk 32 starts at window 64" + ); + let (entries, runs) = inputs::decode_records(real_decode); + let real_card_decode = CardDecode::upload(&stream, &entries, &runs).expect("the decode"); + let real_cpu = Case { + family: "CPU", + input: "real".into(), + rows: CHUNK_ROWS, + cols: fill::CPU_COLUMNS, + host: Box::new(|form| generate_cpu_trace_segments_as(&cpu_parts, real_decode, form)), + records: Records::Cpu { + logs: real.logs.iter().map(inputs::log_record).collect(), + decode: &real_card_decode, + }, + timed: true, + }; + run(&real_cpu, false); + let reg_parts = real.memw_r.register_rows().expect("a MEMW_R chunk"); + assert_eq!( + reg_parts.iter().map(|s| s.len()).sum::(), + CHUNK_ROWS, + "a full register chunk" + ); + let real_memw_r = Case { + family: "MEMW_R", + input: "real".into(), + rows: CHUNK_ROWS, + cols: fill::MEMW_R_COLUMNS, + host: Box::new(|form| generate_memw_register_trace_from_rows_segments_as(®_parts, form)), + records: Records::MemwR( + reg_parts + .iter() + .flat_map(|s| s.iter()) + .map(|r| r.card_record()) + .collect(), + ), + timed: true, + }; + run(&real_memw_r, false); + + // The synthetic sets, over the guest's decode table. + let decode = inputs::program_decode(); + let (entries, runs) = inputs::decode_records(&decode); + let card_decode = CardDecode::upload(&stream, &entries, &runs).expect("the decode"); + for s in inputs::SEEDS { + let rows = inputs::memw_r_rows(s, CHUNK_ROWS); + run( + &Case { + family: "MEMW_R", + input: format!("seed{s}"), + rows: CHUNK_ROWS, + cols: fill::MEMW_R_COLUMNS, + host: Box::new(|form| inputs::memw_r_host(&rows, form)), + records: Records::MemwR(rows.iter().map(|r| r.card_record()).collect()), + timed: true, + }, + false, + ); + let logs = inputs::cpu_logs(s, CHUNK_ROWS, &decode); + let ops = inputs::cpu_ops(&logs, inputs::CPU_FIRST_CYCLE, &decode); + run( + &Case { + family: "CPU", + input: format!("seed{s}"), + rows: CHUNK_ROWS, + cols: fill::CPU_COLUMNS, + host: Box::new(|form| inputs::cpu_host(&ops, &decode, form)), + records: Records::Cpu { + logs: logs.iter().map(inputs::log_record).collect(), + decode: &card_decode, + }, + timed: true, + }, + false, + ); + } + + // A log whose pc no DECODE row holds: the fill is refused, as the host + // fill's `MissingInstruction`. + // The first instruction's pc + 2, unless the host finds a row there (a + // 2-byte instruction); then the pc past the last run's end. + let (last_pc, _, last_len) = *decode.runs().last().expect("a program"); + let absent = [ + decode.entries()[0].pc + 2, + last_pc + 4 * u64::from(last_len), + ] + .into_iter() + .find(|&pc| decode.row(pc).is_none()) + .expect("a pc no DECODE row holds"); + assert_eq!(decode.row(absent), None); + let refusal_rows = 1 << 12; + let mut logs: Vec = inputs::cpu_logs(seed, refusal_rows, &decode) + .iter() + .map(inputs::log_record) + .collect(); + logs[1234].current_pc = absent; + let mut out = poisoned(&stream, &poison, refusal_rows * fill::CPU_COLUMNS); + let refused = fill::fill_cpu_from_logs_into( + &stream, + &logs, + &card_decode, + refusal_rows, + &mut out.slice_mut(..), + ) + .is_err(); + println!("CARD FILL REFUSAL CPU absent_pc={absent} refused={refused}"); + + // Padded tables, identity only. + let short = CHUNK_ROWS - PADDED_SHORT; + let rows = inputs::memw_r_rows(seed, short); + run( + &Case { + family: "MEMW_R", + input: "padded".into(), + rows: CHUNK_ROWS, + cols: fill::MEMW_R_COLUMNS, + host: Box::new(|form| inputs::memw_r_host(&rows, form)), + records: Records::MemwR(rows.iter().map(|r| r.card_record()).collect()), + timed: false, + }, + false, + ); + let logs = inputs::cpu_logs(seed, short, &decode); + let ops = inputs::cpu_ops(&logs, inputs::CPU_FIRST_CYCLE, &decode); + run( + &Case { + family: "CPU", + input: "padded".into(), + rows: CHUNK_ROWS, + cols: fill::CPU_COLUMNS, + host: Box::new(|form| inputs::cpu_host(&ops, &decode, form)), + records: Records::Cpu { + logs: logs.iter().map(inputs::log_record).collect(), + decode: &card_decode, + }, + timed: false, + }, + false, + ); + + // The real chunks' uploads beside the host's memory traffic. + run(&real_cpu, true); + run(&real_memw_r, true); + + let found = found.into_inner(); + let (mismatches, blind) = tally(&found); + let col0_missed: Vec<&String> = found + .iter() + .filter(|(_, f)| f.col0 == Some(false)) + .map(|(name, _)| name) + .collect(); + println!( + "CARD FILL VERDICT: identity={} mismatches={mismatches} mutants={} refusal={} col0={} cases={}", + if mismatches == 0 { "ok" } else { "FAIL" }, + if blind.is_empty() { "ok" } else { "FAIL" }, + if refused { "ok" } else { "FAIL" }, + if col0_missed.is_empty() { "ok" } else { "FAIL" }, + found.len() + ); + assert_eq!(mismatches, 0, "a card fill differs from the host's"); + assert!(blind.is_empty(), "mutants the comparison missed: {blind:?}"); + assert!(refused, "a log with no DECODE row was filled"); + assert!( + col0_missed.is_empty(), + "a fault of CPU column 0 not flagged there alone: {col0_missed:?}" + ); +} diff --git a/prover/src/tables/copy.rs b/prover/src/tables/copy.rs index b41cce063..62959e3f0 100644 --- a/prover/src/tables/copy.rs +++ b/prover/src/tables/copy.rs @@ -136,6 +136,7 @@ use executor::vm::instruction::execution::{ DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_GAP, DMA_MEMSET_SYSCALL_NUMBER, }; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable}; /// `write(2)`, the one ECALL number this chip serves that is not an extension. @@ -266,66 +267,94 @@ impl CopyOperation { pub fn generate_copy_trace( ops: &[CopyOperation], ) -> TraceTable { - let n = ops.len(); - let num_rows = n.next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in ops.iter().enumerate() { - let width = u64::from(op.width); - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - - for (column, &byte) in cols::VALUE.iter().zip(&op.value) { - table.set_byte(row_idx, *column, byte); - } - - table.set_dword_wl(row_idx, cols::SRC_0, op.src); - table.set_dword_hl(row_idx, cols::SRC_INCR_0, op.src.wrapping_add(width)); - - table.set_dword_wl(row_idx, cols::DST_0, op.dst); - table.set_dword_hl(row_idx, cols::DST_INCR_0, op.dst.wrapping_add(width)); - - table.set_dword_wl(row_idx, cols::COUNT_0, op.count); - table.set_dword_hl(row_idx, cols::COUNT_DECR_0, op.count.wrapping_sub(width)); - - table.set_dword_hl(row_idx, cols::DST_RES_0, op.dst_res()); + generate_copy_trace_as(ops, TraceForm::Wide) +} - table.set_bool(row_idx, cols::FIRST, op.first); - table.set_bool(row_idx, cols::END, op.end); - table.set_bool(row_idx, cols::SINGLE, op.width == 1); - table.set_fe(row_idx, cols::MU, FE::one()); +/// The widths COPY traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - let is_set = op.functionality == Functionality::Set; - let is_write = op.functionality == Functionality::Write; - table.set_bool(row_idx, cols::IS_SET, is_set); - table.set_bool(row_idx, cols::IS_WRITE, is_write); - table.set_bool(row_idx, cols::FIRST_WRITE, op.first && is_write); +/// [`generate_copy_trace`] in `form` (`tables::gpack`). +pub fn generate_copy_trace_as( + ops: &[CopyOperation], + form: TraceForm, +) -> TraceTable { + generate_copy_rows_as(ops, ops.len().next_power_of_two().max(4), form) +} - // `mu - count_borrows[1]`: the row copies unless it is the degenerate - // zero-length row, which is the only one that borrows out of `count - step`. - let copies = op.count != 0; - table.set_bool(row_idx, cols::MU_COM, copies && is_write); - table.set_bool( - row_idx, - cols::MU_COM_WIDE, - copies && is_write && op.width == 8, - ); - } +/// Whether [`generate_copy_rows_as`] in `form` writes its rows packed, with no +/// 64-bit table on the way: in [`TraceForm::Narrow`] once an earlier COPY +/// trace of this process left its widths (`tables::gpack`). +pub(crate) fn rows_written_packed(form: TraceForm) -> bool { + matches!( + super::gpack::Plan::new(form, &WIDTHS, cols::NUM_COLUMNS), + super::gpack::Plan::Write(_) + ) +} - for row_idx in n..num_rows { - // `single = 0` on padding, so `step = 8` and the three unconditional ADD - // relations close with `count = src_incr = dst_incr = 8` and everything else - // zero. Both borrows are then zero, so no padding row touches the memory bus. - table.set_fe(row_idx, cols::COUNT_0, FE::from(8u64)); - table.set_fe(row_idx, cols::SRC_INCR_0, FE::from(8u64)); - table.set_fe(row_idx, cols::DST_INCR_0, FE::from(8u64)); - } +/// A COPY table of `num_rows` rows in `form`: `ops` (at most `num_rows`, a row +/// each), then the padding rows. A row reads its own op alone and the padding +/// rows are constants, so rows `[k·R, (k+1)·R)` of the whole padded table are +/// this over the ops in that range with `num_rows = R`: the block's cut of the +/// whole table, built on its own. A call's rows may straddle two tables: they +/// chain only through the CopyNext bus, keyed by the call's timestamp and the +/// running `(src, dst, count)`. +pub fn generate_copy_rows_as( + ops: &[CopyOperation], + num_rows: usize, + form: TraceForm, +) -> TraceTable { + debug_assert!(ops.len() <= num_rows); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in ops.iter().enumerate() { + let width = u64::from(op.width); + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + + for (column, &byte) in cols::VALUE.iter().zip(&op.value) { + table.set_byte(row_idx, *column, byte); + } + + table.set_dword_wl(row_idx, cols::SRC_0, op.src); + table.set_dword_hl(row_idx, cols::SRC_INCR_0, op.src.wrapping_add(width)); + + table.set_dword_wl(row_idx, cols::DST_0, op.dst); + table.set_dword_hl(row_idx, cols::DST_INCR_0, op.dst.wrapping_add(width)); + + table.set_dword_wl(row_idx, cols::COUNT_0, op.count); + table.set_dword_hl(row_idx, cols::COUNT_DECR_0, op.count.wrapping_sub(width)); + + table.set_dword_hl(row_idx, cols::DST_RES_0, op.dst_res()); + + table.set_bool(row_idx, cols::FIRST, op.first); + table.set_bool(row_idx, cols::END, op.end); + table.set_bool(row_idx, cols::SINGLE, op.width == 1); + table.set_fe(row_idx, cols::MU, FE::one()); + + let is_set = op.functionality == Functionality::Set; + let is_write = op.functionality == Functionality::Write; + table.set_bool(row_idx, cols::IS_SET, is_set); + table.set_bool(row_idx, cols::IS_WRITE, is_write); + table.set_bool(row_idx, cols::FIRST_WRITE, op.first && is_write); + + // `mu - count_borrows[1]`: the row copies unless it is the degenerate + // zero-length row, which is the only one that borrows out of `count - step`. + let copies = op.count != 0; + table.set_bool(row_idx, cols::MU_COM, copies && is_write); + table.set_bool( + row_idx, + cols::MU_COM_WIDE, + copies && is_write && op.width == 8, + ); + } - trace + for row_idx in ops.len()..num_rows { + // `single = 0` on padding, so `step = 8` and the three unconditional ADD + // relations close with `count = src_incr = dst_incr = 8` and everything else + // zero. Both borrows are then zero, so no padding row touches the memory bus. + table.set_fe(row_idx, cols::COUNT_0, FE::from(8u64)); + table.set_fe(row_idx, cols::SRC_INCR_0, FE::from(8u64)); + table.set_fe(row_idx, cols::DST_INCR_0, FE::from(8u64)); + } + }) } /// `-1 / 2^32`, `-1 / 2^16` and friends, for the two borrows of `count - step`. diff --git a/prover/src/tables/cpu.rs b/prover/src/tables/cpu.rs index ac81a9338..9cfcbbbd4 100644 --- a/prover/src/tables/cpu.rs +++ b/prover/src/tables/cpu.rs @@ -24,6 +24,8 @@ //! JALR bit (the memory-width bits are 0), so `mem_flags ∈ {0,1} = JALR` and the //! `mem_flags` column is used directly as `JALR` wherever it is gated by `BRANCH`. +use super::decode::DecodeTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, DecodeEntry, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; use crate::Error; use executor::vm::{ @@ -32,8 +34,8 @@ use executor::vm::{ memory::U64HashMap, }; use math::polynomial::Polynomial; -use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed}; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::proof::options::ProofOptions; use stark::prover::evaluate_polynomial_on_lde_domain; @@ -63,6 +65,17 @@ pub const TIMESTAMP_STRIDE: u64 = 4; /// `2^30` cycles; `N = 2^30` would put the last timestamp at `2^32`). pub const MAX_CPU_ROWS: u64 = (1 << 30) - 1; +/// Tallest CPU table, `log2`: the largest uniform chunk a prover may cut +/// (`LAMBDA_VM_MAX_ROWS_LOG2` takes at most 26; the block proves 2^21-row CPU +/// tables). Main bounds a table by `max_rows::CPU`, the heights its static +/// timestamp roots cover; here every root is computed for its height, so the +/// bound is the prover's range. The clock's own bound is [`MAX_CPU_ROWS`], over +/// all tables together. +pub const MAX_TABLE_ROWS_LOG2: u32 = 26; + +/// Tallest CPU table, in rows. +pub const MAX_TABLE_ROWS: usize = 1 << MAX_TABLE_ROWS_LOG2; + /// Largest continuation epoch, `log2` of its cycle count: `2^29` is the largest /// power of two within [`MAX_CPU_ROWS`]. pub const MAX_EPOCH_SIZE_LOG2: u32 = 29; @@ -87,15 +100,13 @@ pub fn timestamp_bases(heights: &[usize]) -> Vec { } /// Checks the CPU table heights of an epoch or monolithic proof: each a power -/// of two in `[4, max_rows::CPU]` (the range the static timestamp commitments -/// cover) and at most [`MAX_CPU_ROWS`] in total. +/// of two in `[4, MAX_TABLE_ROWS]` and at most [`MAX_CPU_ROWS`] in total. pub fn validate_heights(heights: &[usize]) -> Result<(), Error> { let mut total: u64 = 0; for &h in heights { - if !h.is_power_of_two() || !(4..=super::max_rows::CPU).contains(&h) { + if !h.is_power_of_two() || !(4..=MAX_TABLE_ROWS).contains(&h) { return Err(Error::InvalidTableCounts(format!( - "CPU table height must be a power of two in [4, {}], got {h}", - super::max_rows::CPU + "CPU table height must be a power of two in [4, {MAX_TABLE_ROWS}], got {h}" ))); } total += h as u64; @@ -127,47 +138,68 @@ pub fn validate_max_rows(max_rows_cpu: usize) -> Result<(), Error> { validate_heights(&[max_rows_cpu]) } -/// The preprocessed `TIMESTAMP` commitment of a CPU table of `height` rows. -/// Static for the standard options (see [`static_timestamp_commitment`]), -/// recomputed otherwise. -pub fn timestamp_commitment(options: &ProofOptions, height: usize) -> Commitment { +/// The `TIMESTAMP` column of a CPU table of `height` rows: `4·(i + 1)`. +/// +/// What the multilinear verifier reads ([`stark::traits::AIR::precomputed_columns`]): +/// it checks the table's claimed opening of column 0 against this column's value +/// at the reduced point (the recursion's leaf through the column's closed form, +/// `4 + 4·Σ_k 2^k·r_k`). The block never reads a timestamp root. +pub fn timestamp_column(height: usize) -> Vec { + (0..height as u64) + .map(|i| FE::from(TIMESTAMP_STRIDE * (i + 1))) + .collect() +} + +/// The preprocessed `TIMESTAMP` commitment of a CPU table of `height` rows, +/// for the univariate verifier: computed under the block path's pin on first +/// use, for both leaf layouts ([`LeafLayout`]). Main ships a table of static +/// roots for its own hash; under this branch's pin they would be other bytes, +/// so none is carried, and the multilinear path never asks for one. +pub fn lazy_timestamp_commitment( + options: &ProofOptions, + height: usize, +) -> stark::lookup::LazyCommitment { debug_assert!( height.is_power_of_two(), "CPU table height {height} must be checked by validate_heights first" ); - if options.coset_offset == 3 - && let Some(commitment) = - static_timestamp_commitment(options.blowup_factor, height.trailing_zeros()) - { - return commitment; - } - // Once per process: the verifier asks for one commitment per CPU table. - static WARNED: std::sync::Once = std::sync::Once::new(); - WARNED.call_once(|| { - log::warn!( - "CPU timestamp commitment not static for (blowup={}, coset={}); \ - falling back to recompute. Add it to `CPU_TIMESTAMP_COMMITMENTS` by running \ - `cargo run --bin compute_static_commitments --release`.", - options.blowup_factor, - options.coset_offset, - ); - }); - compute_timestamp_commitment(options, height) + let (o, one_row) = (options.clone(), options.clone()); + stark::lookup::LazyCommitment::deferred(move || { + compute_timestamp_commitment_with(&o, height, LeafLayout::RowPair) + }) + .with_one_row(move || { + Some(compute_timestamp_commitment_with( + &one_row, + height, + LeafLayout::Row, + )) + }) } -/// Recomputes the preprocessed `TIMESTAMP` commitment of a CPU table of `height` -/// rows: the column `4·(i + 1)`, low-degree extended and committed exactly as -/// the prover commits a table's precomputed columns. -pub fn compute_timestamp_commitment(options: &ProofOptions, height: usize) -> Commitment { - let column: Vec = (0..height as u64) - .map(|i| FE::from(TIMESTAMP_STRIDE * (i + 1))) - .collect(); - commit_timestamp_column(options, &column) +/// The preprocessed `TIMESTAMP` commitment of a CPU table of `height` rows: the +/// column `4·(i + 1)`, low-degree extended and committed exactly as the prover +/// commits a table's precomputed columns (row-pair leaves). +pub fn timestamp_commitment(options: &ProofOptions, height: usize) -> Commitment { + compute_timestamp_commitment_with(options, height, LeafLayout::RowPair) +} + +/// [`timestamp_commitment`] under an explicit trace-tree leaf layout. +pub fn compute_timestamp_commitment_with( + options: &ProofOptions, + height: usize, + layout: LeafLayout, +) -> Commitment { + commit_timestamp_column(options, ×tamp_column(height), layout) } /// Commits a CPU `TIMESTAMP` column as the prover commits a table's precomputed -/// columns. Soundness tests use it to commit a forged column. -pub(crate) fn commit_timestamp_column(options: &ProofOptions, column: &[FE]) -> Commitment { +/// columns, under the block path's pin. Soundness tests use it to commit a +/// forged column. +pub(crate) fn commit_timestamp_column( + options: &ProofOptions, + column: &[FE], + layout: LeafLayout, +) -> Commitment { let poly = Polynomial::interpolate_fft::(column) .expect("FFT interpolation failed for CPU timestamp column"); let lde = evaluate_polynomial_on_lde_domain( @@ -177,35 +209,12 @@ pub(crate) fn commit_timestamp_column(options: &ProofOptions, column: &[FE]) -> &FE::from(options.coset_offset), ) .expect("LDE evaluation failed for CPU timestamp column"); - let (_, root) = commit_bit_reversed(&[lde], ROWS_PER_LEAF) - .expect("Failed to build Merkle tree for CPU timestamp LDE"); - root -} - -/// Static CPU timestamp commitments for `log2(height)` in -/// `2..=log2(max_rows::CPU)`, per standard blowup factor (coset offset 3). -/// -/// # Regenerating -/// -/// Only after a deliberate change to the timestamp column or the FFT / LDE / -/// Merkle pipeline: `cargo run --bin compute_static_commitments --release`, -/// then paste the printed table over [`CPU_TIMESTAMP_COMMITMENTS`]. A failing -/// drift test (`static_commitments_tests`) is a reason to ask why the bytes -/// changed, not to regenerate them. -pub fn static_timestamp_commitment(blowup_factor: u8, log_height: u32) -> Option { - let (_, by_height) = CPU_TIMESTAMP_COMMITMENTS - .iter() - .find(|(blowup, _)| *blowup == blowup_factor)?; - by_height - .get(log_height.checked_sub(MIN_LOG_HEIGHT)? as usize) - .copied() + crate::lfm::commit::commit_lde_columns_with(&[lde], layout) } /// Smallest CPU table height is `2^2` (`generate_cpu_trace` pads to at least 4). pub const MIN_LOG_HEIGHT: u32 = 2; -include!("cpu_timestamp_commitments.rs"); - // ========================================================================= // Column indices for the CPU table // ========================================================================= @@ -315,61 +324,71 @@ pub mod cols { /// A single CPU cycle to be added to the trace. /// -/// Holds the decoded instruction (`DecodeEntry`) plus the runtime values needed -/// to fill a row: register values, the multiplexed `arg2`, the ALU result, and -/// the branch decision. For `word_instr` rows all operational values are 0 (the -/// row is a pure CPU32 delegate). +/// Holds the DECODE row of the decoded instruction plus the runtime values +/// needed to fill a row: register values, the ALU result, the next pc. Read +/// through a [`CpuOp`] ([`DecodeTable::op`]), which pairs it with its +/// [`DecodeEntry`]. For `word_instr` rows all operational values are 0 (the row +/// is a pure CPU32 delegate). #[derive(Debug, Clone, Default)] pub struct CpuOperation { - /// Static decode information (shared with the DECODE table). - pub decode: DecodeEntry, + /// The row of its instruction in the DECODE table ([`DecodeTable::row`]). + pub decode_row: u32, /// Timestamp for memory argument coordination. pub timestamp: u64, /// Next program counter. pub next_pc: u64, /// Value to write back to rd. pub rvd: u64, - /// Value of register rs1. + /// Value of register rs1 (for an ECALL, a7: the syscall number). pub rv1: u64, /// Value of register rs2. pub rv2: u64, - /// Multiplexed second ALU argument. - pub arg2: u64, /// ALU result (or memory address for LOAD/STORE). pub res: u64, - /// Whether the branch/jump is taken. - pub branch_cond: bool, + /// An ECALL's operands from its log: `[x11, x12]` for COMMIT (buffer + /// address, byte count), `[x10, 0]` for KECCAK and BLAKE3 (state address), + /// zero otherwise. ECSM, HINT, BLAKE3 absorb and the two copy ecalls read + /// theirs from the register state in the trace builder. + /// + /// The rest of what a row needs (the decode, `arg2`, the branch decision, + /// the ECALL's syscall) is a function of these fields: [`CpuOp`]'s methods, + /// so the op stays 72 bytes (the walk materializes one per cycle). + pub ecall_args: [u64; 2], +} - /// Whether this ECALL is a Commit syscall. - pub ecall_commit: bool, - /// For Commit ECALLs: buffer address from x11. - pub commit_buf_addr: u64, - /// For Commit ECALLs: byte count from x12. - pub commit_count: u64, - /// Whether this ECALL is a KeccakPermute syscall. - pub ecall_keccak: bool, - /// For KeccakPermute ECALLs: state address from x10. - pub keccak_state_addr: u64, +// The walk materializes one per cycle: keep it at 72 bytes. +const _: () = assert!(std::mem::size_of::() == 72); - /// Whether this ECALL is an ECSM (elliptic-curve scalar multiply) syscall - pub ecall_ecsm: bool, +impl CpuOperation { + /// Creates a new CPU operation with defaults. + pub fn new() -> Self { + Self::default() + } +} - /// Whether this ECALL is a non-constraining Hint syscall. The hint operand - /// addresses (x10/x11/x12) are recovered from the register state in the trace - /// builder, exactly like ECSM. - pub ecall_hint: bool, +/// A [`CpuOperation`] with its [`DecodeEntry`] ([`DecodeTable::op`]): what +/// every reader of a CPU op works from. Derefs to the op. +#[derive(Debug, Clone, Copy)] +pub struct CpuOp<'a> { + /// Static decode information (the op's DECODE row). + pub decode: &'a DecodeEntry, + op: &'a CpuOperation, +} - /// Whether this ECALL is a DMA memcpy. Operands are recovered from x10/x11/x12. - pub ecall_dma_memcpy: bool, +impl std::ops::Deref for CpuOp<'_> { + type Target = CpuOperation; - /// Whether this ECALL is a DMA memset. Operands are recovered from x10/x11/x12. - pub ecall_dma_memset: bool, + #[inline] + fn deref(&self) -> &CpuOperation { + self.op + } } -impl CpuOperation { - /// Creates a new CPU operation with defaults. - pub fn new() -> Self { - Self::default() +impl<'a> CpuOp<'a> { + /// `op` read with `decode`, the entry of its DECODE row. + #[inline] + pub fn new(decode: &'a DecodeEntry, op: &'a CpuOperation) -> Self { + Self { decode, op } } // ------- convenience accessors ------- @@ -391,32 +410,247 @@ impl CpuOperation { self.decode.fields.mem_flags & 1 == 1 } - /// Creates a CpuOperation from an executor Log and a DecodeEntry. - pub fn from_log(log: &Log, timestamp: u64, decode: DecodeEntry) -> Self { + /// Multiplexed second ALU argument (CPU-A1, `cpu.toml`): `imm` under + /// MEMORY, `rv2` under BRANCH (JAL/JALR read no rs2), `rv2 + imm` otherwise + /// (≤ 1 nonzero by decode A2); 0 on a word delegate row. + #[inline] + pub fn arg2(&self) -> u64 { + let f = &self.decode.fields; + if f.word_instr { + 0 + } else if f.memory { + self.decode.imm + } else if f.branch { + self.rv2 + } else { + self.rv2.wrapping_add(self.decode.imm) + } + } + + /// Whether the branch/jump is taken: JAL/JALR always, a conditional branch + /// by the EQ/LT comparison (with invert) in `alu_flags`; never off BRANCH + /// or on a word delegate row. + #[inline] + pub fn branch_cond(&self) -> bool { + let f = &self.decode.fields; + !f.word_instr + && f.branch + && (self.jalr() || CpuOperation::branch_taken(f, self.rv1, self.rv2)) + } + + /// Whether this is an ECALL of syscall `number` (`rv1` is a7 on an ECALL). + #[inline] + fn ecall_of(&self, number: u64) -> bool { + self.decode.fields.ecall && self.rv1 == number + } + + /// Whether this ECALL is a Commit syscall. + #[inline] + pub fn ecall_commit(&self) -> bool { + self.ecall_of(SyscallNumbers::Commit as u64) + } + + /// For Commit ECALLs: buffer address from x11. + #[inline] + pub fn commit_buf_addr(&self) -> u64 { + if self.ecall_commit() { + self.ecall_args[0] + } else { + 0 + } + } + + /// For Commit ECALLs: byte count from x12. + #[inline] + pub fn commit_count(&self) -> u64 { + if self.ecall_commit() { + self.ecall_args[1] + } else { + 0 + } + } + + /// Whether this ECALL is a KeccakPermute syscall. + #[inline] + pub fn ecall_keccak(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::KECCAK_SYSCALL_NUMBER) + } + + /// For KeccakPermute ECALLs: state address from x10. + #[inline] + pub fn keccak_state_addr(&self) -> u64 { + if self.ecall_keccak() { + self.ecall_args[0] + } else { + 0 + } + } + + /// Whether this ECALL is a Keccak256 (Keccak-256 of a buffer) syscall. Its + /// operands (x10..x12) are recovered from the register state in the trace + /// builder, like ECSM and HINT. + #[inline] + pub fn ecall_keccak256(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::KECCAK256_SYSCALL_NUMBER) + } + + /// Whether this ECALL is a Blake3Compress syscall. + #[inline] + pub fn ecall_blake3(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::BLAKE3_SYSCALL_NUMBER) + } + + /// For Blake3Compress ECALLs: state address from x10. + #[inline] + pub fn blake3_state_addr(&self) -> u64 { + if self.ecall_blake3() { + self.ecall_args[0] + } else { + 0 + } + } + + /// Whether this ECALL is a Blake3Absorb (chained-absorb) syscall. Its four + /// operands (x10..x13) are recovered from the register state in the trace + /// builder, exactly like ECSM and HINT. + #[inline] + pub fn ecall_blake3_absorb(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::BLAKE3_ABSORB_SYSCALL_NUMBER) + } + + /// Whether this ECALL is an ECSM (elliptic-curve scalar multiply) syscall. + #[inline] + pub fn ecall_ecsm(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::ECSM_SYSCALL_NUMBER) + } + + /// Whether this ECALL is an ECREC (ECDSA public-key recovery) syscall. Its + /// operands (x10..x13) are recovered from the register state in the trace + /// builder, like ECSM's. + #[inline] + pub fn ecall_ecrec(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::ECREC_SYSCALL_NUMBER) + } + + /// Whether this ECALL is a non-constraining Hint syscall. The hint operand + /// addresses (x10/x11/x12) are recovered from the register state in the + /// trace builder, exactly like ECSM. + #[inline] + pub fn ecall_hint(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::HINT_SYSCALL_NUMBER) + } + + /// Whether this ECALL is a DMA memcpy (also serving `memmove`). Operands are + /// recovered from x10/x11/x12 in the trace builder. + #[inline] + pub fn ecall_dma_memcpy(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::DMA_MEMCPY_SYSCALL_NUMBER) + } + + /// Whether this ECALL is a DMA memset. Operands are recovered from + /// x10/x11/x12 in the trace builder. + #[inline] + pub fn ecall_dma_memset(&self) -> bool { + self.ecall_of(executor::vm::instruction::execution::DMA_MEMSET_SYSCALL_NUMBER) + } + + /// Collects the BITWISE-table range-check lookups generated by this row, so + /// the BITWISE table can account for the matching multiplicities: + /// 3 `ARE_BYTES` (rs1/rs2, rd/half_instruction_length, alu_flags/mem_flags) and + /// 4 `IS_HALF` (the four halves of `res`). + pub fn collect_bitwise_ops(&self) -> Vec { + let mut ops = Vec::with_capacity(7); + self.for_each_bitwise_op(|op| ops.push(op)); + ops + } + + /// [`Self::collect_bitwise_ops`]'s lookups counted into `histogram`, with no + /// list built. + #[inline] + pub(crate) fn count_bitwise_into(&self, histogram: &mut super::bitwise::BitwiseHistogram) { + self.for_each_bitwise_op(|op| histogram.bump(op)); + } + + /// Each of [`Self::collect_bitwise_ops`]'s lookups, in its order. + #[inline] + fn for_each_bitwise_op(&self, mut emit: impl FnMut(super::bitwise::BitwiseOperation)) { + use super::bitwise::{BitwiseOperation, BitwiseOperationType}; + let f = self.decode.fields; + + // Must mirror the trace columns exactly. On word delegate rows the CPU + // zeroes rs1/rs2/rd/alu_flags/mem_flags and res (half_instruction_length stays); + // CPU32 emits its own range checks for the real decoded values. + let word = f.word_instr; + let z = |v: u8| if word { 0 } else { v }; + let res = if word { 0 } else { self.res }; + + emit(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + z(f.rs1), + z(f.rs2), + )); + emit(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + z(f.rd), + f.half_instruction_length, + )); + emit(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + z(f.alu_flags), + z(f.mem_flags), + )); + + for i in 0..4 { + let half = ((res >> (i * 16)) & 0xFFFF) as u16; + emit(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + (half >> 8) as u8, + )); + } + } +} + +/// Executor logs as the card's CPU fill reads them +/// (`multilinear::records::CpuLog`): the same bytes. +pub(crate) fn card_logs(logs: &[Log]) -> &[multilinear::records::CpuLog] { + use multilinear::records::CpuLog; + use std::mem::{align_of, offset_of, size_of}; + const _: () = { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(Log, current_pc) == offset_of!(CpuLog, current_pc)); + assert!(offset_of!(Log, next_pc) == offset_of!(CpuLog, next_pc)); + assert!(offset_of!(Log, src1_val) == offset_of!(CpuLog, src1_val)); + assert!(offset_of!(Log, src2_val) == offset_of!(CpuLog, src2_val)); + assert!(offset_of!(Log, dst_val) == offset_of!(CpuLog, dst_val)); + }; + // SAFETY: both are `#[repr(C)]`, five `u64`s in the same order (asserted + // above), so the same size, alignment and offsets. + unsafe { std::slice::from_raw_parts(logs.as_ptr() as *const CpuLog, logs.len()) } +} + +impl CpuOperation { + /// Creates a CpuOperation from an executor Log and its instruction's + /// DecodeEntry, the entry of DECODE row `decode_row`. + pub fn from_log(log: &Log, timestamp: u64, decode: &DecodeEntry, decode_row: u32) -> Self { let f = decode.fields; // Real byte length: the column stores half. let instruction_length = 2 * f.half_instruction_length as u64; - // ECALL syscall classification (rv1 = a7 = syscall number). - let ecall_commit = f.ecall && log.src1_val == SyscallNumbers::Commit as u64; - let (commit_buf_addr, commit_count) = if ecall_commit { - (log.src2_val, log.dst_val) + // An ECALL's operands from the log (rv1 = a7 = syscall number): COMMIT's + // buffer and count, KECCAK's and BLAKE3's state address. + let ecall_args = if !f.ecall { + [0, 0] + } else if log.src1_val == SyscallNumbers::Commit as u64 { + [log.src2_val, log.dst_val] + } else if log.src1_val == executor::vm::instruction::execution::KECCAK_SYSCALL_NUMBER + || log.src1_val == executor::vm::instruction::execution::BLAKE3_SYSCALL_NUMBER + { + [log.src2_val, 0] } else { - (0, 0) + [0, 0] }; - let ecall_keccak = - f.ecall && log.src1_val == executor::vm::instruction::execution::KECCAK_SYSCALL_NUMBER; - let keccak_state_addr = if ecall_keccak { log.src2_val } else { 0 }; - // The ECSM operand addresses (x10/x11/x12) are recovered from the register state - // in the trace builder. - let ecall_ecsm = - f.ecall && log.src1_val == executor::vm::instruction::execution::ECSM_SYSCALL_NUMBER; - let ecall_hint = - f.ecall && log.src1_val == executor::vm::instruction::execution::HINT_SYSCALL_NUMBER; - let ecall_dma_memcpy = f.ecall - && log.src1_val == executor::vm::instruction::execution::DMA_MEMCPY_SYSCALL_NUMBER; - let ecall_dma_memset = f.ecall - && log.src1_val == executor::vm::instruction::execution::DMA_MEMSET_SYSCALL_NUMBER; // Word instructions are fully handled by CPU32; the main CPU row is a // delegate that only advances the PC and sends the CPU32 lookup. We still @@ -429,12 +663,8 @@ impl CpuOperation { rv1: log.src1_val, rv2: if f.read_register2 { log.src2_val } else { 0 }, rvd: log.dst_val, - ecall_commit, - commit_buf_addr, - commit_count, - ecall_keccak, - keccak_state_addr, - decode, + ecall_args, + decode_row, timestamp, ..Default::default() }; @@ -520,24 +750,14 @@ impl CpuOperation { }; Self { - decode, + decode_row, timestamp, next_pc, rvd, rv1, rv2, - arg2, res, - branch_cond, - ecall_commit, - commit_buf_addr, - commit_count, - ecall_keccak, - keccak_state_addr, - ecall_ecsm, - ecall_hint, - ecall_dma_memcpy, - ecall_dma_memset, + ecall_args, } } @@ -560,56 +780,6 @@ impl CpuOperation { }; cmp ^ invert } - - /// Creates a CpuOperation from Log and Instruction (convenience). - pub fn from_log_and_instruction(log: &Log, timestamp: u64, instruction: Instruction) -> Self { - let decode = DecodeEntry::from_instruction(log.current_pc, instruction, 4); - Self::from_log(log, timestamp, decode) - } - - /// Collects the BITWISE-table range-check lookups generated by this row, so - /// the BITWISE table can account for the matching multiplicities: - /// 3 `ARE_BYTES` (rs1/rs2, rd/half_instruction_length, alu_flags/mem_flags) and - /// 4 `IS_HALF` (the four halves of `res`). - pub fn collect_bitwise_ops(&self) -> Vec { - use super::bitwise::{BitwiseOperation, BitwiseOperationType}; - let f = self.decode.fields; - let mut ops = Vec::with_capacity(7); - - // Must mirror the trace columns exactly. On word delegate rows the CPU - // zeroes rs1/rs2/rd/alu_flags/mem_flags and res (half_instruction_length stays); - // CPU32 emits its own range checks for the real decoded values. - let word = f.word_instr; - let z = |v: u8| if word { 0 } else { v }; - let res = if word { 0 } else { self.res }; - - ops.push(BitwiseOperation::byte_op( - BitwiseOperationType::AreBytes, - z(f.rs1), - z(f.rs2), - )); - ops.push(BitwiseOperation::byte_op( - BitwiseOperationType::AreBytes, - z(f.rd), - f.half_instruction_length, - )); - ops.push(BitwiseOperation::byte_op( - BitwiseOperationType::AreBytes, - z(f.alu_flags), - z(f.mem_flags), - )); - - for i in 0..4 { - let half = ((res >> (i * 16)) & 0xFFFF) as u16; - ops.push(BitwiseOperation::halfword( - BitwiseOperationType::IsHalf, - (half & 0xFF) as u8, - (half >> 8) as u8, - )); - } - - ops - } } // ========================================================================= @@ -621,22 +791,54 @@ fn local_timestamp(row: usize) -> u64 { TIMESTAMP_STRIDE * (row as u64 + 1) } -/// Generates the CPU trace table from a list of operations. +/// Generates the CPU trace table from a list of operations, each read with +/// its entry of `decode`. /// /// Each operation becomes one row; the table is padded to the next power of 2. pub fn generate_cpu_trace( operations: &[CpuOperation], + decode: &DecodeTable, ) -> TraceTable { - let n = operations.len(); + generate_cpu_trace_as(operations, decode, TraceForm::Wide) +} + +/// [`generate_cpu_trace`] in `form` (`tables::gpack`). +pub fn generate_cpu_trace_as( + operations: &[CpuOperation], + decode: &DecodeTable, + form: TraceForm, +) -> TraceTable { + generate_cpu_trace_segments_as(&[operations], decode, form) +} + +/// The widths CPU traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_cpu_trace_as`] over `segments`, the operations one after +/// another: a chunk handed out as the window parts it lies in. +pub(crate) fn generate_cpu_trace_segments_as( + segments: &[&[CpuOperation]], + decode: &DecodeTable, + form: TraceForm, +) -> TraceTable { + let n: usize = segments.iter().map(|s| s.len()).sum(); let num_rows = n.next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |t| { + fill_cpu_rows(t, segments, decode, num_rows) + }) +} - for (row_idx, op) in operations.iter().enumerate() { +/// Every row of a zeroed CPU table of `num_rows` rows: `segments`' operations, +/// then the padding. +fn fill_cpu_rows( + table: &mut T, + segments: &[&[CpuOperation]], + decode: &DecodeTable, + num_rows: usize, +) { + let n: usize = segments.iter().map(|s| s.len()).sum(); + for (row_idx, op) in segments.iter().flat_map(|s| s.iter()).enumerate() { + let op = decode.op(op); let f = &op.decode.fields; let word = f.word_instr; @@ -685,7 +887,7 @@ pub fn generate_cpu_trace( let (imm, rvd, rv1, rv2, arg2, res) = if word { (0, 0, 0, 0, 0, 0) } else { - (op.decode.imm, op.rvd, op.rv1, op.rv2, op.arg2, op.res) + (op.decode.imm, op.rvd, op.rv1, op.rv2, op.arg2(), op.res) }; table.set_dword_wl(row_idx, cols::IMM_0, imm); @@ -718,7 +920,7 @@ pub fn generate_cpu_trace( // res as DWordHL (4 × 16-bit halves). table.set_dword_hl(row_idx, cols::RES_0, res); - table.set_bool(row_idx, cols::BRANCH_COND, op.branch_cond); + table.set_bool(row_idx, cols::BRANCH_COND, op.branch_cond()); // Inline-PC coordination column. let pc_double_read = !word && f.read_register1 && f.rs1 == 255; @@ -738,34 +940,43 @@ pub fn generate_cpu_trace( table.set_u64(row_idx, cols::PC_0, CPU_PADDING_PC); table.set_u64(row_idx, cols::NEXT_PC_0, CPU_PADDING_PC); } - - trace } -/// Generates the CPU trace table directly from executor logs. -pub fn generate_cpu_trace_from_logs( - logs: &[Log], - instructions: &U64HashMap, -) -> Result, Error> { +/// The CPU ops of `logs` (timestamps from 4, the run's cadence) read against +/// `decode`. +fn ops_from_logs(logs: &[Log], decode: &DecodeTable) -> Result, Error> { let mut operations = Vec::with_capacity(logs.len()); for (i, log) in logs.iter().enumerate() { - let instruction = *instructions - .get(&log.current_pc) + let row = decode + .row(log.current_pc) .ok_or(Error::MissingInstruction(log.current_pc))?; - operations.push(CpuOperation::from_log_and_instruction( + operations.push(CpuOperation::from_log( log, (i as u64) * 4 + 4, - instruction, + decode.entry(row), + row, )); } - Ok(generate_cpu_trace(&operations)) + Ok(operations) +} + +/// Generates the CPU trace table directly from executor logs. +pub fn generate_cpu_trace_from_logs( + logs: &[Log], + instructions: &U64HashMap, +) -> Result, Error> { + let decode = DecodeTable::from_instructions(instructions); + Ok(generate_cpu_trace(&ops_from_logs(logs, &decode)?, &decode)) } /// Collects all BITWISE lookups generated by these CPU operations. -pub fn collect_bitwise_ops(operations: &[CpuOperation]) -> Vec { +pub fn collect_bitwise_ops( + operations: &[CpuOperation], + decode: &DecodeTable, +) -> Vec { operations .iter() - .flat_map(|op| op.collect_bitwise_ops()) + .flat_map(|op| decode.op(op).collect_bitwise_ops()) .collect() } @@ -774,18 +985,8 @@ pub fn collect_bitwise_ops_from_logs( logs: &[Log], instructions: &U64HashMap, ) -> Result, Error> { - let mut operations = Vec::with_capacity(logs.len()); - for (i, log) in logs.iter().enumerate() { - let instruction = *instructions - .get(&log.current_pc) - .ok_or(Error::MissingInstruction(log.current_pc))?; - operations.push(CpuOperation::from_log_and_instruction( - log, - (i as u64) * 4 + 4, - instruction, - )); - } - Ok(collect_bitwise_ops(&operations)) + let decode = DecodeTable::from_instructions(instructions); + Ok(collect_bitwise_ops(&ops_from_logs(logs, &decode)?, &decode)) } // ========================================================================= diff --git a/prover/src/tables/cpu32.rs b/prover/src/tables/cpu32.rs index 60b83e84e..3bead0dc8 100644 --- a/prover/src/tables/cpu32.rs +++ b/prover/src/tables/cpu32.rs @@ -20,6 +20,7 @@ use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{ BusId, FE, GoldilocksExtension, GoldilocksField, SHIFT_16, VmTable, alu_op, packed_decode_shrunk, @@ -195,61 +196,65 @@ impl Cpu32Operation { pub fn generate_cpu32_trace( operations: &[Cpu32Operation], ) -> TraceTable { - let num_rows = operations.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in operations.iter().enumerate() { - let aux = op.compute_aux(); - - // Inputs - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - table.set_dword_wl(row_idx, cols::PC_0, op.pc); - - // rv1 as DWordWHH: [Half, Half, Word] - table.set_byte(row_idx, cols::RS1, op.rs1); - table.set_bool(row_idx, cols::READ_REGISTER1, op.read_register1); - table.set_dword_whh(row_idx, cols::RV1_0, op.rv1); - table.set_bool(row_idx, cols::RV1_SIGN, aux.rv1_sign); - table.set_dword_wl(row_idx, cols::ARG1_0, aux.arg1); - - // rv2 as DWordWHH - table.set_byte(row_idx, cols::RS2, op.rs2); - table.set_bool(row_idx, cols::READ_REGISTER2, op.read_register2); - table.set_dword_whh(row_idx, cols::RV2_0, op.rv2); - table.set_bool(row_idx, cols::RV2_SIGN, aux.rv2_sign); - table.set_dword_wl(row_idx, cols::IMM_0, op.imm); - table.set_dword_wl(row_idx, cols::ARG2_0, aux.arg2); - - // res as DWordHL: 4 halves - table.set_dword_hl(row_idx, cols::RES_0, op.res); - table.set_bool(row_idx, cols::RES_SIGN, aux.res_sign); - - // rd write - table.set_byte(row_idx, cols::RD, op.rd); - table.set_bool(row_idx, cols::WRITE_REGISTER, op.write_register); - table.set_dword_wl(row_idx, cols::RVD_0, aux.rvd); - - // ALU control - table.set_bool(row_idx, cols::ALU, op.alu); - table.set_byte(row_idx, cols::ALU_FLAGS, op.alu_flags); - table.set_bool(row_idx, cols::ADD, op.add); - table.set_bool(row_idx, cols::SUB, op.sub); - table.set_byte( - row_idx, - cols::HALF_INSTRUCTION_LENGTH, - op.half_instruction_length, - ); - table.set_bool(row_idx, cols::SIGNED, aux.signed); + generate_cpu32_trace_as(operations, TraceForm::Wide) +} - table.set_fe(row_idx, cols::MU, FE::one()); - } +/// The widths CPU32 traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - trace +/// [`generate_cpu32_trace`] in `form` (`tables::gpack`). +pub fn generate_cpu32_trace_as( + operations: &[Cpu32Operation], + form: TraceForm, +) -> TraceTable { + let num_rows = operations.len().next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in operations.iter().enumerate() { + let aux = op.compute_aux(); + + // Inputs + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + table.set_dword_wl(row_idx, cols::PC_0, op.pc); + + // rv1 as DWordWHH: [Half, Half, Word] + table.set_byte(row_idx, cols::RS1, op.rs1); + table.set_bool(row_idx, cols::READ_REGISTER1, op.read_register1); + table.set_dword_whh(row_idx, cols::RV1_0, op.rv1); + table.set_bool(row_idx, cols::RV1_SIGN, aux.rv1_sign); + table.set_dword_wl(row_idx, cols::ARG1_0, aux.arg1); + + // rv2 as DWordWHH + table.set_byte(row_idx, cols::RS2, op.rs2); + table.set_bool(row_idx, cols::READ_REGISTER2, op.read_register2); + table.set_dword_whh(row_idx, cols::RV2_0, op.rv2); + table.set_bool(row_idx, cols::RV2_SIGN, aux.rv2_sign); + table.set_dword_wl(row_idx, cols::IMM_0, op.imm); + table.set_dword_wl(row_idx, cols::ARG2_0, aux.arg2); + + // res as DWordHL: 4 halves + table.set_dword_hl(row_idx, cols::RES_0, op.res); + table.set_bool(row_idx, cols::RES_SIGN, aux.res_sign); + + // rd write + table.set_byte(row_idx, cols::RD, op.rd); + table.set_bool(row_idx, cols::WRITE_REGISTER, op.write_register); + table.set_dword_wl(row_idx, cols::RVD_0, aux.rvd); + + // ALU control + table.set_bool(row_idx, cols::ALU, op.alu); + table.set_byte(row_idx, cols::ALU_FLAGS, op.alu_flags); + table.set_bool(row_idx, cols::ADD, op.add); + table.set_bool(row_idx, cols::SUB, op.sub); + table.set_byte( + row_idx, + cols::HALF_INSTRUCTION_LENGTH, + op.half_instruction_length, + ); + table.set_bool(row_idx, cols::SIGNED, aux.signed); + + table.set_fe(row_idx, cols::MU, FE::one()); + } + }) } // ========================================================================= diff --git a/prover/src/tables/cpu_timestamp_commitments.rs b/prover/src/tables/cpu_timestamp_commitments.rs deleted file mode 100644 index 81663a705..000000000 --- a/prover/src/tables/cpu_timestamp_commitments.rs +++ /dev/null @@ -1,345 +0,0 @@ -// Generated by `cargo run --bin compute_static_commitments --release`. -// Do not edit by hand; see `static_timestamp_commitment`. - -/// `(blowup_factor, [commitment for log2(height) = MIN_LOG_HEIGHT..])`. -const CPU_TIMESTAMP_COMMITMENTS: &[(u8, &[Commitment])] = &[ - ( - 2, - &[ - // 2^2 - [ - 0xfd, 0x8b, 0x4b, 0x51, 0xe3, 0x95, 0x89, 0x04, 0x74, 0xfe, 0x74, 0x1d, 0x7e, 0x87, - 0xf2, 0xfc, 0x93, 0x3e, 0x68, 0x4d, 0xc5, 0x9e, 0x5b, 0xdd, 0xdb, 0x72, 0xb9, 0xb3, - 0x28, 0xcb, 0x02, 0x6f, - ], - // 2^3 - [ - 0x49, 0x9a, 0x26, 0xf7, 0x40, 0x0a, 0x85, 0xa2, 0xa1, 0x91, 0x82, 0xf7, 0x04, 0xeb, - 0x24, 0x84, 0xf1, 0x4f, 0x9e, 0xd4, 0xe6, 0xd0, 0x92, 0x6b, 0xeb, 0xe3, 0x76, 0xfc, - 0xce, 0xd1, 0x74, 0xa0, - ], - // 2^4 - [ - 0x4c, 0x97, 0x30, 0x4d, 0x72, 0xa6, 0x2a, 0x87, 0xb3, 0xeb, 0x0d, 0x8e, 0x42, 0x49, - 0xf5, 0x93, 0x80, 0x92, 0x9a, 0x84, 0x65, 0xd5, 0x16, 0xc8, 0x37, 0xa8, 0xe5, 0xaa, - 0xf6, 0xd4, 0x34, 0x36, - ], - // 2^5 - [ - 0xbd, 0x1d, 0x27, 0xf8, 0x1b, 0x7e, 0x91, 0x64, 0x57, 0x53, 0xc2, 0x83, 0xc1, 0x14, - 0xb7, 0x19, 0x5a, 0x91, 0x85, 0xb6, 0xe4, 0x6a, 0x83, 0x99, 0x4a, 0x03, 0x7b, 0x5a, - 0xb2, 0x2e, 0x69, 0xd4, - ], - // 2^6 - [ - 0xf4, 0x06, 0xca, 0x8a, 0xf0, 0xad, 0xc9, 0x4b, 0x4f, 0x95, 0x10, 0x62, 0x01, 0x0d, - 0x5f, 0x17, 0xc8, 0xac, 0xe4, 0xaa, 0xc0, 0xfb, 0xfd, 0x05, 0x48, 0x65, 0xd2, 0x78, - 0xdc, 0xa5, 0xe3, 0x36, - ], - // 2^7 - [ - 0x75, 0xd3, 0x04, 0x50, 0x29, 0x46, 0xbb, 0x1c, 0x1e, 0xc4, 0xc8, 0xf3, 0x31, 0xce, - 0xdf, 0x75, 0x34, 0x9f, 0x29, 0xba, 0x45, 0xc2, 0xb8, 0xac, 0x39, 0xdf, 0xdd, 0xd7, - 0x3b, 0x94, 0x8f, 0x11, - ], - // 2^8 - [ - 0xf9, 0xc0, 0x17, 0x76, 0x62, 0x77, 0x37, 0xeb, 0x79, 0x2d, 0x69, 0x1a, 0xa3, 0x23, - 0xd2, 0x5d, 0x7c, 0xdc, 0x37, 0xec, 0xa0, 0x9d, 0x58, 0x88, 0x6e, 0xb2, 0x6e, 0x9d, - 0x10, 0xfe, 0xac, 0x05, - ], - // 2^9 - [ - 0xbd, 0x3e, 0x4f, 0xa7, 0x6b, 0x1c, 0x3f, 0x86, 0x7d, 0xb7, 0x22, 0xaa, 0x77, 0x22, - 0x19, 0xfc, 0xe7, 0x84, 0xe6, 0x49, 0x3f, 0xb2, 0x5d, 0x44, 0x05, 0xe7, 0x38, 0x3d, - 0x7f, 0x04, 0x6e, 0x6a, - ], - // 2^10 - [ - 0xca, 0x03, 0xf5, 0x3b, 0x2b, 0x4a, 0x22, 0x5c, 0x0f, 0x95, 0x83, 0x07, 0xc6, 0x84, - 0x01, 0xe7, 0x77, 0x6e, 0xc1, 0x76, 0x05, 0x27, 0xca, 0xe2, 0xd6, 0x9a, 0x8c, 0x7e, - 0xf7, 0x10, 0xd6, 0x9d, - ], - // 2^11 - [ - 0xa7, 0x7c, 0x7e, 0x08, 0xf2, 0x91, 0xae, 0x0e, 0x29, 0x79, 0x54, 0x49, 0x4a, 0x58, - 0x0a, 0x30, 0xf8, 0x15, 0x27, 0x04, 0x8f, 0xd5, 0x19, 0xc7, 0xea, 0xf3, 0xa4, 0x11, - 0x37, 0xee, 0x3c, 0x56, - ], - // 2^12 - [ - 0x5a, 0x10, 0x18, 0x2a, 0x89, 0xb6, 0x81, 0x40, 0xc9, 0xe8, 0xb1, 0x1f, 0x29, 0x2c, - 0x49, 0xfc, 0x6f, 0xdb, 0x7a, 0x9b, 0x0f, 0x59, 0x69, 0x29, 0x36, 0x01, 0x63, 0x69, - 0x2e, 0xc0, 0x57, 0x6a, - ], - // 2^13 - [ - 0x78, 0xad, 0xd9, 0xc4, 0x4b, 0x3f, 0x8c, 0xd7, 0xa9, 0x16, 0x1f, 0xea, 0xcc, 0xe3, - 0xed, 0xcb, 0x54, 0xe8, 0x97, 0xb2, 0x42, 0x0c, 0x91, 0x7a, 0x87, 0xc5, 0xf0, 0x92, - 0xdc, 0x8d, 0x72, 0x06, - ], - // 2^14 - [ - 0x91, 0x5a, 0xe0, 0xe9, 0x0a, 0x63, 0x07, 0x38, 0xcc, 0x37, 0x30, 0xbb, 0x34, 0x94, - 0x8f, 0xc9, 0x31, 0x08, 0x55, 0xad, 0xd6, 0x83, 0x23, 0xa8, 0x41, 0x76, 0xcb, 0x4e, - 0x4d, 0x43, 0xb2, 0x43, - ], - // 2^15 - [ - 0x79, 0x06, 0x8e, 0xcd, 0xba, 0x71, 0x66, 0xd0, 0x97, 0x4b, 0xa4, 0x15, 0xf7, 0xe7, - 0x13, 0xed, 0x74, 0xbb, 0xe8, 0x30, 0xc3, 0x1a, 0x3c, 0xfa, 0xac, 0x63, 0xb9, 0x2b, - 0x40, 0x85, 0xcb, 0x0e, - ], - // 2^16 - [ - 0x68, 0x8a, 0xbc, 0x2a, 0x54, 0x31, 0xd9, 0x6a, 0x06, 0xa7, 0x7d, 0xbd, 0xab, 0x20, - 0x85, 0xe8, 0x69, 0x0d, 0x8e, 0xa9, 0x9b, 0x48, 0x8e, 0x50, 0xad, 0xed, 0x7d, 0xc8, - 0x8e, 0x64, 0xb1, 0xeb, - ], - // 2^17 - [ - 0xd3, 0x14, 0xcb, 0x64, 0x69, 0x87, 0x5f, 0x17, 0xdb, 0x16, 0x52, 0x64, 0x73, 0xfd, - 0x80, 0xba, 0x8e, 0xe1, 0x99, 0xc2, 0x4a, 0x58, 0xbf, 0xc7, 0x7a, 0x70, 0x2f, 0x7c, - 0xec, 0x95, 0xc3, 0x90, - ], - // 2^18 - [ - 0x34, 0x6d, 0xaf, 0xc2, 0xcd, 0x01, 0x17, 0x5c, 0xd2, 0x94, 0xb8, 0x5e, 0x22, 0x26, - 0x52, 0x48, 0x37, 0xfe, 0xd0, 0x22, 0x33, 0x25, 0x2c, 0xe5, 0x76, 0x51, 0x47, 0xa0, - 0xf8, 0x2b, 0xc7, 0xe2, - ], - // 2^19 - [ - 0xcc, 0xcb, 0x01, 0xe3, 0xba, 0x8d, 0x22, 0x17, 0x3a, 0xf2, 0x27, 0xe0, 0x96, 0x4e, - 0x43, 0xc5, 0xad, 0xe2, 0xa5, 0x2f, 0xde, 0xf1, 0x2f, 0x06, 0xd8, 0xb6, 0x11, 0xa5, - 0x1c, 0x2e, 0x5f, 0x38, - ], - ], - ), - ( - 4, - &[ - // 2^2 - [ - 0x72, 0xfb, 0x27, 0x34, 0x7e, 0x41, 0x05, 0x71, 0x69, 0x09, 0x4b, 0xec, 0x84, 0xac, - 0xba, 0x2f, 0xc2, 0xc6, 0xe0, 0xc9, 0xd2, 0xe5, 0x31, 0xfb, 0x8b, 0x28, 0xab, 0xaf, - 0x98, 0x9b, 0xd7, 0x2a, - ], - // 2^3 - [ - 0x21, 0x02, 0x0b, 0x76, 0x14, 0xb4, 0x95, 0xe4, 0xd9, 0x12, 0x36, 0x10, 0xf9, 0x22, - 0xe7, 0xb5, 0x25, 0x85, 0x4d, 0xc8, 0x67, 0x7f, 0xf2, 0xb9, 0x48, 0xd1, 0x63, 0xd5, - 0x96, 0x2a, 0x22, 0x7a, - ], - // 2^4 - [ - 0xc6, 0x94, 0x7a, 0x1e, 0x5f, 0xd6, 0xfd, 0x1e, 0x74, 0x5a, 0xc1, 0xb9, 0x96, 0xa5, - 0xad, 0x1b, 0x22, 0x08, 0xcf, 0xe2, 0x63, 0x75, 0x27, 0x82, 0xb2, 0xdf, 0x34, 0x9b, - 0xaa, 0x79, 0xbf, 0x35, - ], - // 2^5 - [ - 0x16, 0x97, 0x32, 0x3b, 0x4d, 0x1e, 0x52, 0x9b, 0x15, 0xdf, 0x44, 0xad, 0x52, 0x06, - 0x8d, 0x75, 0xbc, 0x6b, 0xc9, 0xf6, 0x2c, 0x1c, 0x7b, 0x9a, 0x37, 0x50, 0x25, 0x12, - 0x75, 0x0b, 0xc1, 0xd0, - ], - // 2^6 - [ - 0x01, 0x20, 0x61, 0xdf, 0x58, 0xe2, 0xfa, 0x8e, 0x1b, 0xcf, 0x6e, 0xe7, 0xc9, 0x59, - 0xc7, 0x9b, 0xd8, 0xb3, 0xad, 0x72, 0x31, 0x0d, 0xe8, 0x8a, 0x86, 0xff, 0xea, 0x57, - 0xa6, 0xc3, 0x77, 0x3f, - ], - // 2^7 - [ - 0x72, 0x22, 0x3a, 0x03, 0x6b, 0x0c, 0x2f, 0x7f, 0x77, 0xe1, 0xce, 0xb5, 0x83, 0x07, - 0x00, 0xdc, 0xef, 0x01, 0x2b, 0xd0, 0x06, 0xbf, 0x68, 0x3d, 0x13, 0x04, 0x6d, 0x47, - 0x0c, 0x87, 0xd0, 0x80, - ], - // 2^8 - [ - 0x60, 0x65, 0x01, 0x60, 0xc8, 0xc4, 0x09, 0x68, 0x3d, 0x64, 0xbc, 0x88, 0xaa, 0xd0, - 0x80, 0xff, 0xb0, 0xac, 0x3b, 0x77, 0x25, 0x38, 0x09, 0x1d, 0xa2, 0x0e, 0xb5, 0x54, - 0x54, 0x81, 0x5c, 0x76, - ], - // 2^9 - [ - 0x8a, 0x15, 0x89, 0xb7, 0xef, 0xb3, 0xdb, 0x94, 0x14, 0x89, 0x51, 0x1f, 0x9d, 0x7d, - 0x08, 0xbf, 0xea, 0x3f, 0xb7, 0xa2, 0x79, 0xe8, 0xb4, 0x41, 0x6b, 0x7d, 0xd2, 0xa0, - 0x8b, 0x2d, 0xcc, 0xb5, - ], - // 2^10 - [ - 0xa3, 0xa6, 0x4e, 0x9e, 0x84, 0x87, 0x5d, 0x40, 0xb7, 0x2d, 0xe0, 0x01, 0x52, 0x64, - 0x3f, 0xa1, 0xa3, 0x3f, 0xd5, 0x16, 0x37, 0x79, 0x53, 0xcc, 0x0a, 0x92, 0x3c, 0x48, - 0xed, 0x44, 0x44, 0x28, - ], - // 2^11 - [ - 0x30, 0xca, 0x16, 0x42, 0xf4, 0xdf, 0x7b, 0xe6, 0x9e, 0xe1, 0x2b, 0xa7, 0x9f, 0x90, - 0x6a, 0x20, 0x29, 0x15, 0x9a, 0xdb, 0xca, 0xa4, 0x92, 0xe2, 0x46, 0x84, 0x75, 0x92, - 0x0d, 0x8d, 0x84, 0x41, - ], - // 2^12 - [ - 0xc5, 0x98, 0xfe, 0xdb, 0x5a, 0x88, 0x83, 0x9f, 0x6b, 0x56, 0x0c, 0xd2, 0x74, 0x14, - 0xbd, 0xa6, 0x24, 0xf0, 0x8e, 0x4d, 0x99, 0xc1, 0x19, 0x14, 0xc5, 0x60, 0x54, 0x09, - 0x62, 0x98, 0x50, 0xcb, - ], - // 2^13 - [ - 0x96, 0xd1, 0xc2, 0x6b, 0x13, 0x17, 0x13, 0x74, 0x55, 0xc3, 0xbd, 0x03, 0x12, 0x14, - 0xd0, 0x79, 0x58, 0x86, 0x6b, 0x6c, 0x3b, 0x96, 0xb7, 0x5b, 0x82, 0x91, 0x3f, 0x61, - 0x17, 0x00, 0xf1, 0x3d, - ], - // 2^14 - [ - 0xa5, 0xe7, 0xb9, 0xf8, 0x53, 0x00, 0x7a, 0xb8, 0xe8, 0x0f, 0x6d, 0xd7, 0x8e, 0x5e, - 0x03, 0x0f, 0x02, 0xb5, 0x54, 0x93, 0x9c, 0x79, 0x15, 0xe5, 0x23, 0xe0, 0x22, 0x77, - 0x2a, 0xca, 0x61, 0xc1, - ], - // 2^15 - [ - 0x88, 0x06, 0xdf, 0x8b, 0x12, 0xd1, 0x1a, 0x3e, 0x2e, 0x5f, 0x51, 0x04, 0x1d, 0xc5, - 0xcc, 0x59, 0xcf, 0x9a, 0x41, 0x6f, 0x01, 0x94, 0xb1, 0x72, 0xed, 0x8d, 0xa5, 0x42, - 0x76, 0xa3, 0x00, 0x41, - ], - // 2^16 - [ - 0xaf, 0x29, 0x3c, 0x38, 0xa8, 0x99, 0xff, 0x34, 0x03, 0x99, 0xd9, 0xf2, 0x0e, 0xd1, - 0xb8, 0x97, 0xbb, 0xfe, 0xdd, 0x7c, 0xde, 0x73, 0x58, 0xc7, 0xb4, 0x55, 0x9e, 0xd4, - 0xd2, 0x60, 0x52, 0x70, - ], - // 2^17 - [ - 0x50, 0x65, 0x02, 0xb2, 0x27, 0x17, 0xa4, 0x11, 0x20, 0xbd, 0xe7, 0x4c, 0xb2, 0x6e, - 0x60, 0x38, 0x43, 0x63, 0x8c, 0xac, 0x93, 0xae, 0x3e, 0x5f, 0xdf, 0x2a, 0x80, 0xcf, - 0x11, 0xad, 0x33, 0x22, - ], - // 2^18 - [ - 0xf1, 0x41, 0x4b, 0x26, 0xf4, 0x8b, 0x56, 0x57, 0x6c, 0xf2, 0xd2, 0x94, 0xb0, 0x14, - 0x96, 0xea, 0x1e, 0x29, 0x40, 0xf4, 0xbc, 0x39, 0xfa, 0x67, 0xbf, 0x95, 0xfc, 0xc7, - 0x97, 0xc7, 0xf4, 0x5f, - ], - // 2^19 - [ - 0x24, 0x8d, 0xb4, 0x2e, 0x6c, 0x0c, 0x82, 0xde, 0xdb, 0x42, 0x39, 0xc7, 0x2e, 0x7f, - 0x54, 0x2b, 0x43, 0xb7, 0x8b, 0xb0, 0xa3, 0x99, 0x0d, 0x27, 0x22, 0x8e, 0x0f, 0x3b, - 0xbe, 0x50, 0xba, 0x80, - ], - ], - ), - ( - 8, - &[ - // 2^2 - [ - 0x98, 0xc7, 0xf8, 0x35, 0x47, 0xce, 0x3d, 0x35, 0xd0, 0xc4, 0xf3, 0x9a, 0x73, 0x80, - 0x8d, 0x7e, 0x6a, 0xd5, 0xcd, 0x43, 0x77, 0x57, 0x10, 0x9f, 0xd8, 0xf6, 0x7f, 0xe8, - 0x99, 0xe3, 0x19, 0xe7, - ], - // 2^3 - [ - 0x5b, 0xb8, 0x15, 0xd7, 0x98, 0xc9, 0xd5, 0x64, 0x45, 0x13, 0x22, 0x04, 0x4c, 0xe4, - 0x8d, 0xac, 0xa3, 0x8b, 0x48, 0x3f, 0x44, 0xc5, 0xe0, 0xa2, 0xb0, 0xf9, 0x65, 0x91, - 0x75, 0x3f, 0xb7, 0xb3, - ], - // 2^4 - [ - 0x5a, 0x94, 0x7f, 0xd5, 0x59, 0xb3, 0xb6, 0x95, 0xc2, 0x26, 0x74, 0x06, 0x2b, 0x93, - 0x3b, 0xb9, 0xd6, 0x1e, 0x57, 0xb1, 0x0c, 0x11, 0x74, 0x53, 0xe3, 0x5e, 0x45, 0xb9, - 0x90, 0x61, 0x82, 0x7a, - ], - // 2^5 - [ - 0xb6, 0xe5, 0x3f, 0x5a, 0xc5, 0x88, 0x33, 0x63, 0xdb, 0xd8, 0x56, 0xc9, 0xf5, 0xc9, - 0x17, 0x35, 0x9f, 0x10, 0xd3, 0xd8, 0xfd, 0x76, 0x45, 0xd7, 0x0c, 0x08, 0x54, 0x25, - 0xc7, 0xaa, 0xec, 0x7b, - ], - // 2^6 - [ - 0xc2, 0x9b, 0x84, 0x12, 0x3c, 0x40, 0x5d, 0x4b, 0xf6, 0x5b, 0x02, 0x25, 0x04, 0x2e, - 0x1d, 0x05, 0x06, 0x25, 0x16, 0xfb, 0x19, 0x7f, 0xa1, 0x48, 0x13, 0x4f, 0xb2, 0xaf, - 0x36, 0x5c, 0x7d, 0xec, - ], - // 2^7 - [ - 0x77, 0x8e, 0x69, 0x8a, 0xde, 0x86, 0xee, 0xaf, 0x95, 0xa3, 0xfb, 0x01, 0x9f, 0x7b, - 0xbe, 0x1d, 0x67, 0xe2, 0x02, 0x83, 0x51, 0xb8, 0x83, 0xbf, 0x70, 0x8b, 0x95, 0x43, - 0x5c, 0x29, 0x4f, 0x5c, - ], - // 2^8 - [ - 0xbf, 0xc4, 0x7f, 0x00, 0xd9, 0x4b, 0x6a, 0xc3, 0x85, 0x01, 0xbe, 0x0f, 0x75, 0x8f, - 0xb4, 0xd6, 0x0e, 0xf2, 0x93, 0xc3, 0x92, 0x74, 0x96, 0xa5, 0xef, 0x05, 0xe3, 0x05, - 0xce, 0x57, 0x44, 0xb6, - ], - // 2^9 - [ - 0xc3, 0x28, 0x64, 0xac, 0xd1, 0xc7, 0x85, 0x0c, 0x9b, 0xf0, 0xdf, 0x3e, 0x55, 0x84, - 0xbb, 0x19, 0x6f, 0x9d, 0xe7, 0x12, 0x0f, 0x7d, 0x83, 0x8e, 0x2c, 0xd1, 0x2c, 0xda, - 0xfe, 0x93, 0xd1, 0xd9, - ], - // 2^10 - [ - 0xe0, 0x1c, 0x59, 0x0e, 0x4e, 0xf3, 0xbf, 0xb2, 0x94, 0xc1, 0xe0, 0xa5, 0x6d, 0x06, - 0x4e, 0xd6, 0x0e, 0x59, 0x63, 0x18, 0x24, 0xfb, 0x18, 0xea, 0xf6, 0x5d, 0xb9, 0x41, - 0x85, 0x6d, 0x85, 0x2e, - ], - // 2^11 - [ - 0xe7, 0x1e, 0x5b, 0x58, 0x79, 0x59, 0x2f, 0x67, 0x95, 0x23, 0x99, 0x51, 0x02, 0x4d, - 0xf8, 0x72, 0xd4, 0x3c, 0x41, 0x16, 0xaa, 0x01, 0xc4, 0xaf, 0x88, 0x30, 0x2d, 0x95, - 0xc7, 0xec, 0x81, 0xc3, - ], - // 2^12 - [ - 0x52, 0x36, 0x52, 0x5e, 0x1f, 0x1a, 0x6b, 0xbe, 0x39, 0x7f, 0x92, 0x93, 0x92, 0x2f, - 0xcc, 0x34, 0x8f, 0x16, 0x91, 0xfc, 0xd4, 0x56, 0xc2, 0xe3, 0x96, 0xa9, 0x4e, 0x31, - 0x9c, 0x42, 0x00, 0x36, - ], - // 2^13 - [ - 0xd9, 0xe3, 0xd8, 0x18, 0xaa, 0x8f, 0x5d, 0xbd, 0xc0, 0xf5, 0x43, 0x31, 0x8b, 0xe5, - 0x32, 0x06, 0xad, 0x54, 0xe1, 0x65, 0x61, 0xec, 0x42, 0x9c, 0x72, 0xac, 0x09, 0x7c, - 0x7d, 0xb7, 0xc0, 0x4c, - ], - // 2^14 - [ - 0xc7, 0x8e, 0x04, 0x79, 0xb7, 0x17, 0x4b, 0xdf, 0x50, 0xd4, 0x81, 0xa2, 0xa8, 0x23, - 0xcb, 0x13, 0x3e, 0xad, 0x7b, 0xe4, 0x3b, 0x41, 0x1b, 0x08, 0x81, 0x5e, 0xde, 0xf5, - 0xc4, 0x02, 0x2c, 0xdb, - ], - // 2^15 - [ - 0x78, 0xb3, 0x81, 0xf0, 0x7f, 0xa3, 0xda, 0x3d, 0xf8, 0xd8, 0x7a, 0x7c, 0xdb, 0x50, - 0x72, 0x2b, 0x13, 0x6a, 0x60, 0x96, 0x1e, 0xfb, 0xff, 0xf3, 0x2a, 0xe2, 0x7a, 0x4e, - 0x3a, 0xf5, 0xdc, 0x70, - ], - // 2^16 - [ - 0xcc, 0x75, 0x75, 0xba, 0x3f, 0x0f, 0xfe, 0x0e, 0x25, 0x2c, 0x66, 0xc0, 0x64, 0xfc, - 0xc5, 0xd3, 0x56, 0x67, 0xf9, 0x63, 0x7c, 0xe2, 0xc8, 0x4c, 0xa7, 0x3e, 0xca, 0xfa, - 0x9a, 0x6f, 0x99, 0x7c, - ], - // 2^17 - [ - 0x77, 0xfa, 0x2e, 0x51, 0xe5, 0x57, 0xbf, 0x89, 0xbd, 0x3f, 0xf2, 0x3f, 0x93, 0xde, - 0x89, 0xdb, 0xb5, 0x5e, 0xb3, 0x84, 0xca, 0xce, 0x33, 0xc5, 0x28, 0x27, 0x4e, 0x27, - 0x2d, 0x26, 0x39, 0xa9, - ], - // 2^18 - [ - 0xce, 0x49, 0x07, 0x32, 0xb5, 0x54, 0x0f, 0x96, 0xea, 0xad, 0x27, 0xff, 0x0c, 0x72, - 0xa5, 0xab, 0x1f, 0x72, 0xf0, 0x57, 0x96, 0x24, 0xb0, 0x7c, 0xce, 0x66, 0xe5, 0x2d, - 0xd7, 0xde, 0xf6, 0x29, - ], - // 2^19 - [ - 0xcb, 0xa5, 0xef, 0x03, 0x64, 0xe1, 0x49, 0xcb, 0x87, 0xa5, 0x66, 0x76, 0xf7, 0x71, - 0xb0, 0x80, 0x12, 0x6f, 0x85, 0xcb, 0x84, 0x9d, 0xac, 0xe1, 0x15, 0xeb, 0x49, 0x45, - 0x17, 0x3b, 0x06, 0x40, - ], - ], - ), -]; diff --git a/prover/src/tables/decode.rs b/prover/src/tables/decode.rs index bfd1ddb90..2a695d062 100644 --- a/prover/src/tables/decode.rs +++ b/prover/src/tables/decode.rs @@ -35,8 +35,8 @@ use executor::elf::Elf; use executor::vm::instruction::decoding::{Instruction, InstructionError}; use executor::vm::memory::U64HashMap; use math::polynomial::Polynomial; -use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed}; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; use stark::proof::options::ProofOptions; use stark::prover::evaluate_polynomial_on_lde_domain; @@ -100,13 +100,38 @@ pub type PcToRow = U64HashMap; pub fn generate_decode_trace( instructions: &U64HashMap, ) -> (TraceTable, PcToRow) { - // Build entries and PC-to-row mapping + // ★★ ROWS GO IN PC ORDER, and the sort is the whole point of this block. + // + // The rows used to come out of `instructions.iter()`, so their order was + // hashbrown's: a function of the hasher, the capacity the map happened to + // grow to, and the insertion sequence. Every one of those is stable for a + // given binary, which is why nothing has ever failed — prover and verifier + // both reach this through `instructions_from_elf`, so they agree with each + // other. What they agree on is a CONSTRUCTION PROCEDURE, not the ELF. + // + // That distinction is about to start mattering. These five columns are + // ELF-derived and their Merkle root is on its way to being a program + // constant pinned in a recursion guest (W1-B). A pinned root must be a + // function of the ELF ALONE: sorted by pc it is, and a hashbrown version + // bump, a capacity change or a reserve added upstream cannot move it. + // Unsorted it is not, and the failure mode is the bad kind — a toolchain + // update silently invalidates the pin, with no ELF change, no code change + // and no test that fails until a verifier rejects a valid proof. + // + // pc is unique (it is the map's key), so the order is total and the sort is + // not merely deterministic but canonical. + let mut entries: Vec<(u64, Instruction)> = instructions + .iter() + .map(|(&pc, &instr)| (pc, instr)) + .collect(); + entries.sort_unstable_by_key(|(pc, _)| *pc); + let mut pc_to_row = PcToRow::default(); pc_to_row.reserve(instructions.len() + 1); - let entries: Vec<_> = instructions - .iter() + let entries: Vec<_> = entries + .into_iter() .enumerate() - .map(|(row_idx, (&pc, &instr))| { + .map(|(row_idx, (pc, instr))| { pc_to_row.insert(pc, row_idx); // instruction_length = 4 (RV64C compressed decode is a separate workstream). DecodeEntry::from_instruction(pc, instr, 4) @@ -172,6 +197,167 @@ pub fn generate_decode_trace( (trace, pc_to_row) } +/// Every instruction of a program decoded once, in the DECODE table's row +/// order: the entry of row `r` is [`generate_decode_trace`]'s row `r` (both +/// sort the pcs), and the CPU padding entry's row follows the last +/// instruction's. A CPU op keeps its DECODE row instead of a copy of the entry +/// ([`super::cpu::CpuOperation`]), so its decode is one indexed read here and +/// its DECODE lookup is counted by row, with no pc search. +#[derive(Debug)] +pub struct DecodeTable { + /// The entries, in pc order. + entries: Vec, + /// `(first pc, its row, number of pcs)` of each run of pcs 4 apart, + /// ascending: a pc's row is found by run, not by hashing. + runs: Vec<(u64, u32, u32)>, +} + +#[cfg(test)] +impl DecodeTable { + /// The entries, in pc order: entry `r` is DECODE row `r`. + pub(crate) fn entries(&self) -> &[DecodeEntry] { + &self.entries + } + + /// The runs of pcs 4 apart, ascending: `(first pc, its row, number of pcs)`. + pub(crate) fn runs(&self) -> &[(u64, u32, u32)] { + &self.runs + } +} + +impl DecodeTable { + /// The table of `instructions`, each decoded as the DECODE trace decodes + /// it (length 4). + /// + /// # Panics + /// If the program has 2^32 − 1 instructions or more (rows are `u32`). + pub fn from_instructions(instructions: &U64HashMap) -> Self { + let mut pcs: Vec = instructions.keys().copied().collect(); + // The order of `generate_decode_trace`'s rows. + pcs.sort_unstable(); + assert!( + pcs.len() < u32::MAX as usize, + "{} instructions do not fit u32 DECODE rows", + pcs.len() + ); + let mut entries = Vec::with_capacity(pcs.len()); + let mut runs: Vec<(u64, u32, u32)> = Vec::new(); + for pc in pcs { + let row = entries.len() as u32; + entries.push(DecodeEntry::from_instruction(pc, instructions[&pc], 4)); + match runs.last_mut() { + Some((first, _, len)) if first.checked_add(4 * *len as u64) == Some(pc) => { + *len += 1 + } + _ => runs.push((pc, row, 1)), + } + } + Self { entries, runs } + } + + /// The DECODE row of the instruction at `pc`, if the program has one. + #[inline] + pub fn row(&self, pc: u64) -> Option { + self.runs.iter().find_map(|&(first, row, len)| { + let offset = pc.wrapping_sub(first); + (offset % 4 == 0 && offset / 4 < len as u64).then(|| row + (offset / 4) as u32) + }) + } + + /// The table as the card's CPU fill reads it + /// (`multilinear::records::DecodeRecords`): each entry's pc, immediate and + /// decoded fields as bytes, in row order, and the runs. `None` for a + /// program with no instruction. + pub(crate) fn card_records(&self) -> Option { + use multilinear::records::{DecodeRecords, DecodeRow, DecodeRun}; + let entries = self + .entries + .iter() + .map(|e| { + let f = &e.fields; + DecodeRow { + pc: e.pc, + imm: e.imm, + read_register1: u8::from(f.read_register1), + read_register2: u8::from(f.read_register2), + write_register: u8::from(f.write_register), + word_instr: u8::from(f.word_instr), + alu: u8::from(f.alu), + add: u8::from(f.add), + sub: u8::from(f.sub), + memory: u8::from(f.memory), + branch: u8::from(f.branch), + ecall: u8::from(f.ecall), + rs1: f.rs1, + rs2: f.rs2, + rd: f.rd, + half_instruction_length: f.half_instruction_length, + alu_flags: f.alu_flags, + mem_flags: f.mem_flags, + } + }) + .collect(); + let runs = self + .runs + .iter() + .map(|&(first_pc, row, len)| DecodeRun { first_pc, row, len }) + .collect(); + DecodeRecords::new(entries, runs) + } + + /// The entry of DECODE row `row` (a row [`Self::row`] returned). + #[inline] + pub fn entry(&self, row: u32) -> &DecodeEntry { + &self.entries[row as usize] + } + + /// The DECODE row of the CPU padding entry (`pc = CPU_PADDING_PC`). + pub fn padding_row(&self) -> usize { + self.entries.len() + } + + /// `op` with its decode. + #[inline] + pub fn op<'a>(&'a self, op: &'a super::cpu::CpuOperation) -> super::cpu::CpuOp<'a> { + super::cpu::CpuOp::new(self.entry(op.decode_row), op) + } +} + +/// Counts one DECODE lookup per row of `rows` into the MU column (rows a +/// [`DecodeTable`] gives). +pub fn count_rows( + trace: &mut TraceTable, + rows: impl IntoIterator, +) { + for row in rows { + let current = trace.main_table.get(row, cols::MU); + trace.main_table.set_fe(row, cols::MU, current + FE::one()); + } +} + +/// Adds `counts[row]` DECODE lookups to each row's MU column at once: the +/// column [`count_rows`] leaves after `counts[row]` lookups of that row, the +/// field's sums being exact. Rows with no count are not touched. The first +/// counted row the trace does not have is the error. +pub fn add_row_counts( + trace: &mut TraceTable, + counts: &[u64], +) -> Result<(), usize> { + for (row, &n) in counts.iter().enumerate() { + if n == 0 { + continue; + } + if row >= trace.main_table.height { + return Err(row); + } + let current = trace.main_table.get(row, cols::MU); + trace + .main_table + .set_fe(row, cols::MU, current + FE::from(n)); + } + Ok(()) +} + /// Updates multiplicities in the DECODE trace table. /// /// For each PC in `lookups`, increments the MU column in the corresponding row. @@ -229,6 +415,25 @@ pub fn bus_interactions() -> Vec { // Precomputed commitment // ========================================================================= +/// The precomputed columns themselves, `0..NUM_PRECOMPUTED_COLS` of the DECODE +/// trace: the program's instruction table. +/// +/// The multilinear path checks a proof's claimed openings against these instead +/// of comparing a commitment, so it needs the values and not just their root. +/// This is what binds a proof to the program it claims to run. +pub fn preprocessed_columns(instructions: &U64HashMap) -> Vec> { + // MU=0: only the precomputed columns are wanted. + let (trace, _pc_to_row) = generate_decode_trace(instructions); + let num_rows = trace.num_rows(); + (0..NUM_PRECOMPUTED_COLS) + .map(|col_idx| { + (0..num_rows) + .map(|row_idx| *trace.main_table.get(row_idx, col_idx)) + .collect() + }) + .collect() +} + /// Computes the LDE commitment for DECODE precomputed columns. /// /// This builds a Merkle tree over the LDE (Low Degree Extension) of the precomputed @@ -260,20 +465,21 @@ pub fn compute_precomputed_commitment( instructions: &U64HashMap, options: &ProofOptions, ) -> Commitment { - // Step 1: Generate trace (MU=0, we only need precomputed columns) - let (trace, _pc_to_row) = generate_decode_trace(instructions); - let num_rows = trace.num_rows(); + compute_precomputed_commitment_with(instructions, options, LeafLayout::RowPair) +} - // Step 2: Extract precomputed columns (0..NUM_PRECOMPUTED_COLS) - let columns: Vec> = (0..NUM_PRECOMPUTED_COLS) - .map(|col_idx| { - (0..num_rows) - .map(|row_idx| *trace.main_table.get(row_idx, col_idx)) - .collect() - }) - .collect(); +/// [`compute_precomputed_commitment`] under an explicit trace-tree leaf layout +/// (S2). DECODE is program-dependent, so a one-row DECODE root is computed at +/// run time, like the row-pair one. +pub fn compute_precomputed_commitment_with( + instructions: &U64HashMap, + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + let columns = preprocessed_columns(instructions); + let num_rows = columns[0].len(); - // Step 3: Interpolate each column to a polynomial + // Interpolate each column to a polynomial let polys: Vec> = columns .iter() .map(|col| { @@ -282,7 +488,7 @@ pub fn compute_precomputed_commitment( }) .collect(); - // Step 4: Evaluate polynomials on LDE domain (N * blowup_factor points) + // Evaluate polynomials on LDE domain (N * blowup_factor points) let blowup_factor = options.blowup_factor as usize; let coset_offset = FE::from(options.coset_offset); let lde_columns: Vec> = polys @@ -293,9 +499,40 @@ pub fn compute_precomputed_commitment( }) .collect(); - let (_, root) = commit_bit_reversed(&lde_columns, ROWS_PER_LEAF) - .expect("Failed to build Merkle tree for decode LDE"); - root + // ★ Through the LFM commit helper, which commits under the BLOCK PATH's pin + // rather than `stark`'s default aliases. This root is a PREPROCESSED + // commitment the prover recomputes and compares against, so building it with + // a different hash than the path commits under fails at prove time with + // `PrecomputedCommitmentMismatch` — which is exactly how it was found. + crate::lfm::commit::commit_lde_columns_with(&lde_columns, layout) +} + +/// DECODE's commitment source for both leaf layouts: the row-pair root +/// `supplied` by the caller (the recursion guest's) or computed on first use, +/// and the one-row root computed on first use (program-dependent: no static +/// twin; a supplied root never stands in for the other layout). +pub fn lazy_commitment( + instructions: std::sync::Arc>, + options: &ProofOptions, + supplied: Option, +) -> stark::lookup::LazyCommitment { + let base = match supplied { + Some(c) => stark::lookup::LazyCommitment::ready(c), + None => { + let (instructions, options) = (instructions.clone(), options.clone()); + stark::lookup::LazyCommitment::deferred(move || { + compute_precomputed_commitment(&instructions, &options) + }) + } + }; + let options = options.clone(); + base.with_one_row(move || { + Some(compute_precomputed_commitment_with( + &instructions, + &options, + LeafLayout::Row, + )) + }) } // ========================================================================= diff --git a/prover/src/tables/dvrm.rs b/prover/src/tables/dvrm.rs index 50c0faf27..773fb806b 100644 --- a/prover/src/tables/dvrm.rs +++ b/prover/src/tables/dvrm.rs @@ -33,8 +33,9 @@ use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; -use std::collections::HashMap; +use super::trace_hash::{OpMap, trace_hash_state}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{ BusId, GoldilocksExtension, GoldilocksField, MU_MAX, NEG_INV_2_16, NEG_INV_2_32, NEG_INV_2_48, NEG_INV_2_64, SHIFT_16, VmTable, alu_op, @@ -287,7 +288,8 @@ impl DvrmOperation { pub fn dedup_dvrm_rows( operations: &[(DvrmOperation, bool)], ) -> Vec<(DvrmOperation, DvrmMultiplicities)> { - let mut op_map: HashMap = HashMap::new(); + let mut op_map: OpMap = + OpMap::with_hasher(trace_hash_state()); for (op, wants_remainder) in operations { let entry = op_map.entry(op.clone()).or_default(); if *wants_remainder { @@ -322,58 +324,62 @@ pub fn dedup_dvrm_rows( pub fn generate_dvrm_trace( operations: &[(DvrmOperation, bool)], ) -> TraceTable { - let unique_ops = dedup_dvrm_rows(operations); - let num_rows = unique_ops.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; + generate_dvrm_trace_as(operations, TraceForm::Wide) +} - for (row_idx, (op, multiplicities)) in unique_ops.iter().enumerate() { - let q = op.compute_quotient(); - let r = op.compute_remainder(); - let n_sub_r = op.n_sub_r(); - let abs_r = op.abs_r(); - let abs_d = op.abs_d(); +/// The widths DVRM traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - // Fill n as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::N_0, op.n); +/// [`generate_dvrm_trace`] in `form` (`tables::gpack`). +pub fn generate_dvrm_trace_as( + operations: &[(DvrmOperation, bool)], + form: TraceForm, +) -> TraceTable { + let unique_ops = dedup_dvrm_rows(operations); + let num_rows = unique_ops.len().next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, (op, multiplicities)) in unique_ops.iter().enumerate() { + let q = op.compute_quotient(); + let r = op.compute_remainder(); + let n_sub_r = op.n_sub_r(); + let abs_r = op.abs_r(); + let abs_d = op.abs_d(); - // Fill d as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::D_0, op.d); + // Fill n as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::N_0, op.n); - table.set_bool(row_idx, cols::SIGNED, op.signed); + // Fill d as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::D_0, op.d); - // Fill q as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::Q_0, q); + table.set_bool(row_idx, cols::SIGNED, op.signed); - // Fill r as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::R_0, r); + // Fill q as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::Q_0, q); - // Fill auxiliary columns - table.set_bool(row_idx, cols::DIV_BY_ZERO, op.is_div_by_zero()); - table.set_bool(row_idx, cols::OVERFLOW, op.is_overflow()); + // Fill r as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::R_0, r); - table.set_dword_wl(row_idx, cols::ABS_R_0, abs_r); - table.set_dword_wl(row_idx, cols::ABS_D_0, abs_d); + // Fill auxiliary columns + table.set_bool(row_idx, cols::DIV_BY_ZERO, op.is_div_by_zero()); + table.set_bool(row_idx, cols::OVERFLOW, op.is_overflow()); - // Fill n_sub_r as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::N_SUB_R_0, n_sub_r); + table.set_dword_wl(row_idx, cols::ABS_R_0, abs_r); + table.set_dword_wl(row_idx, cols::ABS_D_0, abs_d); - table.set_bool(row_idx, cols::SIGN_N_SUB_R, op.sign_n_sub_r()); - table.set_bool(row_idx, cols::SIGN_N, op.sign_n()); - table.set_bool(row_idx, cols::SIGN_D, op.sign_d()); - table.set_bool(row_idx, cols::SIGN_Q, op.sign_q()); - table.set_bool(row_idx, cols::SIGN_R, op.sign_r()); + // Fill n_sub_r as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::N_SUB_R_0, n_sub_r); - // Multiplicities - table.set_u64(row_idx, cols::MU_Q, multiplicities.mu_q); - table.set_u64(row_idx, cols::MU_R, multiplicities.mu_r); - } + table.set_bool(row_idx, cols::SIGN_N_SUB_R, op.sign_n_sub_r()); + table.set_bool(row_idx, cols::SIGN_N, op.sign_n()); + table.set_bool(row_idx, cols::SIGN_D, op.sign_d()); + table.set_bool(row_idx, cols::SIGN_Q, op.sign_q()); + table.set_bool(row_idx, cols::SIGN_R, op.sign_r()); - trace + // Multiplicities + table.set_u64(row_idx, cols::MU_Q, multiplicities.mu_q); + table.set_u64(row_idx, cols::MU_R, multiplicities.mu_r); + } + }) } // ========================================================================= diff --git a/prover/src/tables/ecdas.rs b/prover/src/tables/ecdas.rs index 6d4b8a908..d6acdbd52 100644 --- a/prover/src/tables/ecdas.rs +++ b/prover/src/tables/ecdas.rs @@ -11,9 +11,12 @@ //! to 0 and `op = 1`. The `R·P` term in the λ, xR, and yR relations is gated with `μ`, so it //! vanishes on padding rows (μ=0) and all relations hold with zero carries. +use std::borrow::Cow; + use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable}; use crate::tables::ecsm::ecdas_tuple; use ecsm::{EcdasStep, P_BYTES}; @@ -92,50 +95,220 @@ fn fe_from_i64(c: i64) -> FE { pub fn generate_ecdas_trace( ops: &[EcdasOperation], ) -> TraceTable { - let n = ops.len(); - let num_rows = n.next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in ops.iter().enumerate() { + generate_ecdas_trace_as(ops, TraceForm::Wide) +} + +/// The widths ECDAS traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_ecdas_trace`] in `form` (`tables::gpack`). +pub fn generate_ecdas_trace_as( + ops: &[EcdasOperation], + form: TraceForm, +) -> TraceTable { + generate_ecdas_rows_as(ops, ops.len().next_power_of_two().max(4), form) +} + +/// Whether [`generate_ecdas_rows_as`] in `form` writes its rows packed, with no +/// 64-bit table on the way: in [`TraceForm::Narrow`] once an earlier ECDAS +/// trace of this process left its widths (`tables::gpack`). +pub(crate) fn rows_written_packed(form: TraceForm) -> bool { + matches!( + super::gpack::Plan::new(form, &WIDTHS, cols::NUM_COLUMNS), + super::gpack::Plan::Write(_) + ) +} + +/// A ECDAS table of `num_rows` rows in `form`: `ops` (at most `num_rows`, a row +/// each, one double/add step a row), then the padding rows. A row reads its own op alone and +/// the padding rows are constants, so rows `[k·R, (k+1)·R)` of the whole padded +/// table are this over the ops in that range with `num_rows = R`: the block's +/// cut of the whole table, built on its own. +pub fn generate_ecdas_rows_as( + ops: &[EcdasOperation], + num_rows: usize, + form: TraceForm, +) -> TraceTable { + generate_ecdas_rows_of(ops, num_rows, form, borrowed) +} + +/// An op read as itself. +fn borrowed(op: &EcdasOperation) -> Cow<'_, EcdasOperation> { + Cow::Borrowed(op) +} + +/// [`generate_ecdas_rows_as`] over rows held in another form, each read as +/// its op by `op` (a [`CompactEcdasOp`] widened back) as its row is filled. +pub(crate) fn generate_ecdas_rows_of( + rows: &[R], + num_rows: usize, + form: TraceForm, + op: impl for<'r> Fn(&'r R) -> Cow<'r, EcdasOperation> + Sync, +) -> TraceTable { + let n = rows.len(); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, row) in rows.iter().enumerate() { + let op = op(row); + let s = &op.step; + + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + table.set_bytes(row_idx, cols::XG, &s.x_g); + table.set_bytes(row_idx, cols::YG, &s.y_g); + table.set_bytes(row_idx, cols::XA, &s.x_a); + table.set_bytes(row_idx, cols::YA, &s.y_a); + table.set_byte(row_idx, cols::ROUND, s.round); + table.set_byte(row_idx, cols::OP, s.op); + table.set_bytes(row_idx, cols::XR, &s.x_r); + table.set_bytes(row_idx, cols::YR, &s.y_r); + table.set_bytes(row_idx, cols::LAMBDA, &s.lambda); + table.set_bytes(row_idx, cols::Q0, &s.q0); + table.set_bytes(row_idx, cols::Q1, &s.q1); + table.set_bytes(row_idx, cols::Q2, &s.q2); + for i in 0..64 { + debug_assert!((0..1 << 16).contains(&(s.c0[i] + CARRY_OFFSET_LAMBDA))); + debug_assert!((0..1 << 16).contains(&(s.c1[i] + CARRY_OFFSET_XR))); + debug_assert!((0..1 << 16).contains(&(s.c2[i] + CARRY_OFFSET_YR))); + table.set_fe(row_idx, cols::c0(i), fe_from_i64(s.c0[i])); + table.set_fe(row_idx, cols::c1(i), fe_from_i64(s.c1[i])); + table.set_fe(row_idx, cols::c2(i), fe_from_i64(s.c2[i])); + } + table.set_byte(row_idx, cols::NEXT_OP, s.next_op); + table.set_fe(row_idx, cols::MU, FE::one()); + } + + // Padding rows: q0 = q1 = q2 = 0, op = 1 (add), everything else 0. The μ-gated R·P + // term vanishes (μ=0), so all convolution relations hold with zero carries. + for row_idx in n..num_rows { + table.set_byte(row_idx, cols::OP, 1); + } + }) +} + +/// The carry offsets of `c0`, `c1` and `c2`, in that order. +const CARRY_OFFSETS: [i64; 3] = [CARRY_OFFSET_LAMBDA, CARRY_OFFSET_XR, CARRY_OFFSET_YR]; + +/// An ECDAS row as a builder keeps it until its table is built. Its carries — +/// `c0`, `c1`, `c2`, three `[i64; 64]`, 1.5 KB of a 1.87 KB row — are held as +/// their offset 16-bit values, the range the trace checks them in: ≈ 0.72 KB a +/// row. A row any of whose carries does not fit keeps all three as they are, +/// so a row widens back to exactly the op it was made from whatever the input +/// ([`Self::to_op`]); the release build has no range assert to rely on. +#[derive(Clone, Debug)] +pub(crate) struct CompactEcdasOp { + timestamp: u64, + x_a: [u8; 32], + y_a: [u8; 32], + x_g: [u8; 32], + y_g: [u8; 32], + round: u8, + op: u8, + next_op: u8, + lambda: [u8; 32], + x_r: [u8; 32], + y_r: [u8; 32], + q0: [u8; 33], + q1: [u8; 33], + q2: [u8; 33], + carries: Carries, +} + +/// A row's carries, `c0`, `c1`, `c2` in that order. The common row is held +/// inline; boxing it would cost an allocation a row, the rare one is boxed. +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug)] +enum Carries { + /// Each carry plus its array's offset ([`CARRY_OFFSETS`]), every one in + /// `0..2^16`. + Offset([[u16; 64]; 3]), + /// As they are: some carry's offset value is outside `0..2^16`. + Raw(Box<[[i64; 64]; 3]>), +} + +impl CompactEcdasOp { + /// `op`, its carries offset when they all fit 16 bits. + pub(crate) fn from_op(op: &EcdasOperation) -> Self { let s = &op.step; + let raw = [s.c0, s.c1, s.c2]; + let mut offset = [[0u16; 64]; 3]; + let mut fits = true; + for ((carries, packed), base) in raw.iter().zip(&mut offset).zip(CARRY_OFFSETS) { + for (&c, slot) in carries.iter().zip(packed.iter_mut()) { + match c.checked_add(base).and_then(|v| u16::try_from(v).ok()) { + Some(v) => *slot = v, + None => fits = false, + } + } + } + Self { + timestamp: op.timestamp, + x_a: s.x_a, + y_a: s.y_a, + x_g: s.x_g, + y_g: s.y_g, + round: s.round, + op: s.op, + next_op: s.next_op, + lambda: s.lambda, + x_r: s.x_r, + y_r: s.y_r, + q0: s.q0, + q1: s.q1, + q2: s.q2, + carries: if fits { + Carries::Offset(offset) + } else { + Carries::Raw(Box::new(raw)) + }, + } + } - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - table.set_bytes(row_idx, cols::XG, &s.x_g); - table.set_bytes(row_idx, cols::YG, &s.y_g); - table.set_bytes(row_idx, cols::XA, &s.x_a); - table.set_bytes(row_idx, cols::YA, &s.y_a); - table.set_byte(row_idx, cols::ROUND, s.round); - table.set_byte(row_idx, cols::OP, s.op); - table.set_bytes(row_idx, cols::XR, &s.x_r); - table.set_bytes(row_idx, cols::YR, &s.y_r); - table.set_bytes(row_idx, cols::LAMBDA, &s.lambda); - table.set_bytes(row_idx, cols::Q0, &s.q0); - table.set_bytes(row_idx, cols::Q1, &s.q1); - table.set_bytes(row_idx, cols::Q2, &s.q2); - for i in 0..64 { - debug_assert!((0..1 << 16).contains(&(s.c0[i] + CARRY_OFFSET_LAMBDA))); - debug_assert!((0..1 << 16).contains(&(s.c1[i] + CARRY_OFFSET_XR))); - debug_assert!((0..1 << 16).contains(&(s.c2[i] + CARRY_OFFSET_YR))); - table.set_fe(row_idx, cols::c0(i), fe_from_i64(s.c0[i])); - table.set_fe(row_idx, cols::c1(i), fe_from_i64(s.c1[i])); - table.set_fe(row_idx, cols::c2(i), fe_from_i64(s.c2[i])); + /// The op this row was made from, exactly. + pub(crate) fn to_op(&self) -> EcdasOperation { + let [c0, c1, c2] = match &self.carries { + Carries::Offset(offset) => { + let mut raw = [[0i64; 64]; 3]; + for ((carries, packed), base) in raw.iter_mut().zip(offset).zip(CARRY_OFFSETS) { + for (c, &v) in carries.iter_mut().zip(packed) { + *c = i64::from(v) - base; + } + } + raw + } + Carries::Raw(raw) => **raw, + }; + EcdasOperation { + timestamp: self.timestamp, + step: EcdasStep { + x_a: self.x_a, + y_a: self.y_a, + x_g: self.x_g, + y_g: self.y_g, + round: self.round, + op: self.op, + next_op: self.next_op, + lambda: self.lambda, + x_r: self.x_r, + y_r: self.y_r, + q0: self.q0, + q1: self.q1, + q2: self.q2, + c0, + c1, + c2, + }, } - table.set_byte(row_idx, cols::NEXT_OP, s.next_op); - table.set_fe(row_idx, cols::MU, FE::one()); } - // Padding rows: q0 = q1 = q2 = 0, op = 1 (add), everything else 0. The μ-gated R·P - // term vanishes (μ=0), so all convolution relations hold with zero carries. - for row_idx in n..num_rows { - table.set_byte(row_idx, cols::OP, 1); + /// Whether the row keeps its carries as they are. + #[cfg(test)] + pub(crate) fn is_raw(&self) -> bool { + matches!(self.carries, Carries::Raw(_)) } +} - trace +/// A [`CompactEcdasOp`] read as its op. +pub(crate) fn widen(row: &CompactEcdasOp) -> Cow<'_, EcdasOperation> { + Cow::Owned(row.to_op()) } // ========================================================================= @@ -451,3 +624,126 @@ impl ConstraintSet for EcdasConstraints { } } } + +#[cfg(test)] +mod compact_tests { + use super::*; + + /// A step whose every field differs from its neighbours', with these carries. + fn op(i: u64, c: [[i64; 64]; 3]) -> EcdasOperation { + let bytes = |f: u64| -> [u8; 32] { + std::array::from_fn(|b| ((i + 1) * 31 + f * 7 + b as u64) as u8) + }; + let wide = |f: u64| -> [u8; 33] { + std::array::from_fn(|b| ((i + 3) * 17 + f * 5 + b as u64) as u8) + }; + EcdasOperation { + timestamp: 4 * i + 3, + step: EcdasStep { + x_a: bytes(1), + y_a: bytes(2), + x_g: bytes(3), + y_g: bytes(4), + round: (i % 256) as u8, + op: (i % 2) as u8, + next_op: ((i + 1) % 2) as u8, + lambda: bytes(5), + x_r: bytes(6), + y_r: bytes(7), + q0: wide(8), + q1: wide(9), + q2: wide(10), + c0: c[0], + c1: c[1], + c2: c[2], + }, + } + } + + /// Carries at and around each array's 16-bit bounds, and far outside them; + /// `out` puts one carry of array `k` outside its range. + fn ops() -> Vec<(EcdasOperation, bool)> { + let mut ops = Vec::new(); + for (i, base) in CARRY_OFFSETS.iter().enumerate() { + let low = -base; + let high = (1 << 16) - 1 - base; + let fit: [[i64; 64]; 3] = std::array::from_fn(|k| { + std::array::from_fn(|j| { + let b = CARRY_OFFSETS[k]; + match j % 4 { + 0 => -b, + 1 => (1 << 16) - 1 - b, + 2 => (j as i64 * 37) % 200 - 100, + _ => 0, + } + }) + }); + ops.push((op(i as u64, fit), false)); + for (n, bad) in [low - 1, high + 1, i64::MIN, i64::MAX, -(1 << 40)] + .into_iter() + .enumerate() + { + let mut c = fit; + c[i][(n * 13) % 64] = bad; + ops.push((op((10 + 10 * i + n) as u64, c), true)); + } + } + ops + } + + fn same(a: &EcdasOperation, b: &EcdasOperation) -> bool { + format!("{a:?}") == format!("{b:?}") + } + + /// Every row widens back to exactly the op it was made from: carries at + /// both ends of their 16-bit range held offset, and a row with any carry + /// outside it (by one, or at either end of i64) held as it is. + #[test] + fn a_compact_row_widens_to_the_op_it_was_made_from() { + for (op, out) in ops() { + let row = CompactEcdasOp::from_op(&op); + assert_eq!(row.is_raw(), out, "{op:?}"); + assert!(same(&row.to_op(), &op), "{op:?}"); + } + // The compact row is well under half the op. + assert!( + std::mem::size_of::() * 5 < std::mem::size_of::() * 2, + "{} vs {}", + std::mem::size_of::(), + std::mem::size_of::() + ); + } + + fn words(t: &TraceTable) -> Vec { + match t.narrow_main() { + Some(p) => (0..p.cols()).flat_map(|c| p.column(c)).collect(), + None => { + let m = &t.main_table; + (0..m.width) + .flat_map(|c| (0..m.height).map(move |r| *m.get(r, c).value())) + .collect() + } + } + } + + /// A table built from compact rows, each widened as its row is filled, is + /// the table of the ops, wide and packed, the fallback rows included. + #[test] + fn a_table_of_compact_rows_is_the_table_of_the_ops() { + // The generator's debug checks reject a carry outside its range, so the + // fallback rows are built only without them. + let wide: Vec = ops() + .into_iter() + .filter(|(_, out)| !out || !cfg!(debug_assertions)) + .map(|(op, _)| op) + .collect(); + let compact: Vec = wide.iter().map(CompactEcdasOp::from_op).collect(); + for form in [TraceForm::Wide, TraceForm::Narrow, TraceForm::Narrow] { + for rows in [wide.len().next_power_of_two(), 64] { + let want = generate_ecdas_rows_as(&wide, rows, form); + let got = generate_ecdas_rows_of(&compact, rows, form, widen); + assert_eq!(words(&got), words(&want), "{form:?} {rows}"); + } + } + } +} diff --git a/prover/src/tables/ecrec.rs b/prover/src/tables/ecrec.rs new file mode 100644 index 000000000..fb4380c22 --- /dev/null +++ b/prover/src/tables/ecrec.rs @@ -0,0 +1,1086 @@ +//! ECREC chip — one ECDSA public-key recovery on secp256k1 per row. +//! +//! One row per `ECALL(-12)`. It reads the signature `(r, s)`, the message hash `z` +//! (big-endian, as the guest holds them) and the parity of `R`'s y, and either writes +//! the public key `Q = u1·G + u2·R` (`u1 = −z·r⁻¹`, `u2 = s·r⁻¹ mod N`) with status 0, +//! or *bails*: status 1 and a zero key, after which the guest recovers the key in +//! software. A bail proves nothing and needs nothing proven: the guest's software path +//! decides the answer. Everything below is gated on `OK` (the row computed the key). +//! +//! Under `OK` the row proves: +//! - `0 < r < N`, `0 < s < N` (overflow chains, `Zero` lookups); +//! - `w = r⁻¹`: `r·w − 1 − qw·N = 0`; `u1`: `z·w + u1 − qz·N = 0`; `u2`: +//! `s·w − u2 − qu·N = 0`, with `u1, u2` as 256 bit columns each, `0 < u1, u2 < N`; +//! - `R = (r, yR)` on the curve (ECSM's two relations on `(r, yR)`), `yR < p`, and +//! `yR[0] = 2·H + PAR` with `PAR` the `x13` value: the requested root; +//! - `A = u1·G` through the `Ecdas`/`Bit` chain keyed at `TS`, `B = u2·R` keyed at +//! `TS + 1` (ECDAS unchanged), canonical `xA, xB < p` and `xA ≠ xB` (an inverse +//! witness over the eight 32-bit word differences); +//! - `Q = A + B` as one ECDAS addition row keyed at `TS + 2`, with `xQ, yQ < p`. +//! +//! Every ECALL timestamp is a multiple of 4: the CPU's preprocessed `TIMESTAMP` +//! column is `4·(i + 1)`, and each CPU table adds its base `4·S_k`, four times the +//! rows of the tables before it (`cpu::timestamp_bases`, computed by the verifier +//! from the heights). So `TS`, `TS + 1` and `TS + 2` never meet another call's keys +//! on the two buses only ECSM, ECDAS and ECREC use (the offsets are tied to the +//! stride by a compile-time check below). +//! +//! The witness is `ecsm::compute_ecrec_witness`; padding rows are all zero. + +use executor::vm::instruction::execution::ECREC_SYSCALL_NUMBER; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use super::ecsm::{CARRY_OFFSET_X2, CARRY_OFFSET_YG, point_coord_busvalues}; +use super::gpack::{TraceForm, WidthHint, generate_main}; +use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable}; +use crate::constraints::templates::INV_SHIFT_32; +use ecsm::{B, EcdasStep, EcrecWitness, N_BYTES, P_BYTES}; + +/// The second chain's key on the `Ecdas` and `Bit` buses, as an offset from the +/// call's timestamp: the first chain is keyed at `TS`, this one at `TS + 1`, the +/// final addition at `TS + ADD_KEY`. Every ECALL timestamp is a multiple of 4, so +/// these never meet another call's keys; and the two chains must not share a key, +/// or their `Bit` messages become one multiset and their scalars can be swapped. +pub(crate) const CHAIN2_KEY: i64 = 1; + +/// The final addition's key offset (see [`CHAIN2_KEY`]). +pub(crate) const ADD_KEY: i64 = 2; + +// The three keys of one call, `TS`, `TS + CHAIN2_KEY` and `TS + ADD_KEY`, are +// distinct and stay below the next CPU row's timestamp `TS + TIMESTAMP_STRIDE`, +// so no two calls share a key; ECALL timestamps are multiples of the stride +// because every CPU table's base is too (`cpu::timestamp_bases`). A smaller +// stride or larger offsets would let one call's keys meet the next call's. +const _: () = assert!( + 0 < CHAIN2_KEY && CHAIN2_KEY < ADD_KEY && ADD_KEY < super::cpu::TIMESTAMP_STRIDE as i64, + "ECREC's key offsets must be distinct, nonzero and below the CPU timestamp stride" +); + +/// Offset of the three mod-N product relations' carries into `IsHalfword`: +/// their honest carries are within `±8161` (the `x2` relation's bound), and +/// `2^14` leaves room on both sides. +pub(crate) const CARRY_OFFSET_MODN: i64 = 1 << 14; + +// ========================================================================= +// Column indices +// ========================================================================= + +pub mod cols { + pub const TIMESTAMP_0: usize = 0; + pub const TIMESTAMP_1: usize = 1; + pub const ADDR_OUT_0: usize = 2; + pub const ADDR_OUT_1: usize = 3; + pub const ADDR_SIG_0: usize = 4; + pub const ADDR_SIG_1: usize = 5; + pub const ADDR_MSG_0: usize = 6; + pub const ADDR_MSG_1: usize = 7; + /// The `x13` value: the requested parity of `R`'s y. + pub const PAR: usize = 8; + pub const MU: usize = 9; + /// The row computed the key (status 0); `MU − OK` is the bail status. + pub const OK: usize = 10; + /// `yR[0] = 2·H + PAR`. + pub const H: usize = 11; + pub const LEN1: usize = 12; + pub const LEN2: usize = 13; + pub const R: usize = 14; // 32 bytes, little-endian limbs + pub const S: usize = R + 32; + pub const Z: usize = S + 32; + pub const W: usize = Z + 32; + pub const QW: usize = W + 32; + pub const CW: usize = QW + 32; // 64 carries + pub const U1: usize = CW + 64; // 256 bits, LSB first + pub const QZ: usize = U1 + 256; + pub const CZ: usize = QZ + 32; + pub const U2: usize = CZ + 64; // 256 bits + pub const QU: usize = U2 + 256; + pub const CU: usize = QU + 32; + pub const YR: usize = CU + 64; + pub const X2: usize = YR + 32; + pub const Q0: usize = X2 + 32; + pub const C0: usize = Q0 + 32; + pub const Q1: usize = C0 + 64; // 33 bytes + pub const C1: usize = Q1 + 33; + pub const XA: usize = C1 + 64; + pub const YA: usize = XA + 32; + pub const XB: usize = YA + 32; + pub const YB: usize = XB + 32; + pub const XQ: usize = YB + 32; + pub const YQ: usize = XQ + 32; + /// Inverse witnesses of the eight 32-bit word differences `xB − xA`. + pub const DINV: usize = YQ + 32; + pub const R_SUB_N: usize = DINV + 8; // 16 halfwords each + pub const S_SUB_N: usize = R_SUB_N + 16; + pub const U1_SUB_N: usize = S_SUB_N + 16; + pub const U2_SUB_N: usize = U1_SUB_N + 16; + pub const YR_SUB_P: usize = U2_SUB_N + 16; + pub const XA_SUB_P: usize = YR_SUB_P + 16; + pub const XB_SUB_P: usize = XA_SUB_P + 16; + pub const XQ_SUB_P: usize = XB_SUB_P + 16; + pub const YQ_SUB_P: usize = XQ_SUB_P + 16; + + pub const NUM_COLUMNS: usize = YQ_SUB_P + 16; +} + +// ========================================================================= +// Operation +// ========================================================================= + +/// One ECREC ecall: its witness, addresses and timestamp. +#[derive(Debug, Clone)] +pub struct EcrecOperation { + pub timestamp: u64, + pub addr_out: u64, + pub addr_sig: u64, + pub addr_msg: u64, + pub witness: EcrecWitness, +} + +impl EcrecOperation { + /// The heap its witness owns beyond the op: the ECDAS steps, which no ECREC table reads. + pub(crate) fn steps_heap_bytes(&self) -> usize { + (self.witness.chain1.capacity() + self.witness.chain2.capacity()) + * std::mem::size_of::() + } +} + +// ========================================================================= +// Trace generation +// ========================================================================= + +fn fe_from_i64(c: i64) -> FE { + if c >= 0 { + FE::from(c as u64) + } else { + FE::zero() - FE::from((-c) as u64) + } +} + +fn write_halfwords(table: &mut impl VmTable, row: usize, col: usize, bytes: &[u8; 32]) { + let halfwords: [u16; 16] = + std::array::from_fn(|j| u16::from_le_bytes([bytes[2 * j], bytes[2 * j + 1]])); + table.set_halves(row, col, &halfwords); +} + +fn write_bits(table: &mut impl VmTable, row: usize, col: usize, bytes: &[u8; 32]) { + for b in 0..256 { + table.set_fe( + row, + col + b, + FE::from(((bytes[b / 8] >> (b % 8)) & 1) as u64), + ); + } +} + +fn write_carries(table: &mut impl VmTable, row: usize, col: usize, c: &[i64; 64], offset: i64) { + for (i, &v) in c.iter().enumerate() { + debug_assert!((0..1 << 16).contains(&(v + offset)), "carry {i} = {v}"); + table.set_fe(row, col + i, fe_from_i64(v)); + } +} + +/// The eight 32-bit word differences `xB − xA` of a witness, as integers. +pub(crate) fn word_differences(w: &EcrecWitness) -> [i64; 8] { + std::array::from_fn(|c| { + let word = |b: &[u8; 32]| u32::from_le_bytes(b[4 * c..4 * c + 4].try_into().unwrap()); + i64::from(word(&w.x_b)) - i64::from(word(&w.x_a)) + }) +} + +pub fn generate_ecrec_trace( + ops: &[EcrecOperation], +) -> TraceTable { + generate_ecrec_trace_as(ops, TraceForm::Wide) +} + +static WIDTHS: WidthHint = WidthHint::new(); + +pub fn generate_ecrec_trace_as( + ops: &[EcrecOperation], + form: TraceForm, +) -> TraceTable { + generate_ecrec_rows_as(ops, ops.len().next_power_of_two().max(4), form) +} + +/// An ECREC table of `num_rows` rows: `ops`, then all-zero padding rows. +pub fn generate_ecrec_rows_as( + ops: &[EcrecOperation], + num_rows: usize, + form: TraceForm, +) -> TraceTable { + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row, op) in ops.iter().enumerate() { + let w = &op.witness; + table.set_dword_wl(row, cols::TIMESTAMP_0, op.timestamp); + table.set_dword_wl(row, cols::ADDR_OUT_0, op.addr_out); + table.set_dword_wl(row, cols::ADDR_SIG_0, op.addr_sig); + table.set_dword_wl(row, cols::ADDR_MSG_0, op.addr_msg); + table.set_u64(row, cols::PAR, w.parity as u64); + table.set_fe(row, cols::MU, FE::one()); + table.set_bytes(row, cols::R, &w.r); + table.set_bytes(row, cols::S, &w.s); + table.set_bytes(row, cols::Z, &w.z); + if !w.ok { + continue; + } + table.set_fe(row, cols::OK, FE::one()); + table.set_u64(row, cols::H, w.h as u64); + table.set_u64(row, cols::LEN1, w.len1 as u64); + table.set_u64(row, cols::LEN2, w.len2 as u64); + table.set_bytes(row, cols::W, &w.w); + table.set_bytes(row, cols::QW, &w.qw); + write_carries(table, row, cols::CW, &w.cw, CARRY_OFFSET_MODN); + write_bits(table, row, cols::U1, &w.u1); + table.set_bytes(row, cols::QZ, &w.qz); + write_carries(table, row, cols::CZ, &w.cz, CARRY_OFFSET_MODN); + write_bits(table, row, cols::U2, &w.u2); + table.set_bytes(row, cols::QU, &w.qu); + write_carries(table, row, cols::CU, &w.cu, CARRY_OFFSET_MODN); + table.set_bytes(row, cols::YR, &w.y_r); + table.set_bytes(row, cols::X2, &w.x2); + table.set_bytes(row, cols::Q0, &w.q0); + write_carries(table, row, cols::C0, &w.c0, CARRY_OFFSET_X2); + table.set_bytes(row, cols::Q1, &w.q1); + write_carries(table, row, cols::C1, &w.c1, CARRY_OFFSET_YG); + table.set_bytes(row, cols::XA, &w.x_a); + table.set_bytes(row, cols::YA, &w.y_a); + table.set_bytes(row, cols::XB, &w.x_b); + table.set_bytes(row, cols::YB, &w.y_b); + table.set_bytes(row, cols::XQ, &w.x_q); + table.set_bytes(row, cols::YQ, &w.y_q); + // One nonzero word difference gets its inverse; the sum Σ d·inv is then 1. + let d = word_differences(w); + let c = d + .iter() + .position(|&v| v != 0) + .expect("xA ≠ xB on a computed row"); + table.set_fe( + row, + cols::DINV + c, + fe_from_i64(d[c]).inv().expect("nonzero"), + ); + for (col, bytes) in [ + (cols::R_SUB_N, &w.r_sub_n), + (cols::S_SUB_N, &w.s_sub_n), + (cols::U1_SUB_N, &w.u1_sub_n), + (cols::U2_SUB_N, &w.u2_sub_n), + (cols::YR_SUB_P, &w.y_r_sub_p), + (cols::XA_SUB_P, &w.x_a_sub_p), + (cols::XB_SUB_P, &w.x_b_sub_p), + (cols::XQ_SUB_P, &w.x_q_sub_p), + (cols::YQ_SUB_P, &w.y_q_sub_p), + ] { + write_halfwords(table, row, col, bytes); + } + } + }) +} + +// ========================================================================= +// Bus interactions +// ========================================================================= + +fn packed(col: usize) -> BusValue { + BusValue::Packed { + start_column: col, + packing: Packing::Direct, + } +} + +fn col_plus(col: usize, k: i64) -> BusValue { + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: col, + }, + LinearTerm::Constant(k), + ]) +} + +/// `MU − OK`: the bail status (and the multiplicity of nothing). +fn bail_value() -> BusValue { + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: cols::MU, + }, + LinearTerm::Column { + coefficient: -1, + column: cols::OK, + }, + ]) +} + +/// `[old[8], is_register, base_lo, base_hi, value[8], ts_lo, ts_hi, w2, w4, w8]`, a read. +#[allow(clippy::too_many_arguments)] +fn memw_read( + value: [BusValue; 8], + is_register: u64, + base_lo: BusValue, + base_hi: BusValue, + ts_lo: BusValue, + ts_hi: BusValue, + w2: u64, + w8: u64, +) -> Vec { + let mut v = Vec::with_capacity(24); + v.extend(value.clone()); + v.push(BusValue::constant(is_register)); + v.push(base_lo); + v.push(base_hi); + v.extend(value); + v.push(ts_lo); + v.push(ts_hi); + v.push(BusValue::constant(w2)); + v.push(BusValue::constant(0)); + v.push(BusValue::constant(w8)); + v +} + +/// `[is_register, base_lo, base_hi, value[8], ts_lo, ts_hi, w2, w4, w8]`, a write. +fn memw_write( + value: [BusValue; 8], + base_lo: BusValue, + base_hi: BusValue, + ts_lo: BusValue, + ts_hi: BusValue, +) -> Vec { + let mut v = Vec::with_capacity(16); + v.push(BusValue::constant(0)); + v.push(base_lo); + v.push(base_hi); + v.extend(value); + v.push(ts_lo); + v.push(ts_hi); + v.push(BusValue::constant(0)); + v.push(BusValue::constant(0)); + v.push(BusValue::constant(1)); + v +} + +fn register_value(lo: BusValue, hi: BusValue) -> [BusValue; 8] { + let mut v: [BusValue; 8] = std::array::from_fn(|_| BusValue::constant(0)); + v[0] = lo; + v[1] = hi; + v +} + +/// Memory byte `m` (`0..32`) of a big-endian 256-bit value held as little-endian limbs +/// at `col`. +fn be_byte(col: usize, m: usize) -> BusValue { + packed(col + 31 - m) +} + +/// The `Ecdas` tuple `[0, ts_lo, ts_hi, acc_x, acc_y, gen_x, gen_y, round, op]` with +/// any bus values (ECSM's [`super::ecsm::ecdas_tuple`] takes columns only). +#[allow(clippy::too_many_arguments)] +fn ecdas_tuple( + acc_x: Vec, + acc_y: Vec, + gen_x: Vec, + gen_y: Vec, + round: BusValue, + op: BusValue, + ts_lo: BusValue, + ts_hi: BusValue, +) -> Vec { + let mut v = Vec::with_capacity(1 + 2 + 4 * 32 + 2); + v.push(BusValue::constant(0)); // curve id 0 = secp256k1, as ECSM and ECDAS + v.push(ts_lo); + v.push(ts_hi); + v.extend(acc_x); + v.extend(acc_y); + v.extend(gen_x); + v.extend(gen_y); + v.push(round); + v.push(op); + v +} + +/// A constant 256-bit value as 32 little-endian byte constants. +fn const_coord(le: &[u8; 32]) -> Vec { + le.iter().map(|&b| BusValue::constant(b as u64)).collect() +} + +/// The generator's coordinates, little-endian. +pub(crate) fn generator_le() -> ([u8; 32], [u8; 32]) { + let mut x = [ + 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, 0x0B, + 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, 0x16, 0xF8, + 0x17, 0x98, + ]; + let mut y = [ + 0x48, 0x3A, 0xDA, 0x77, 0x26, 0xA3, 0xC4, 0x65, 0x5D, 0xA4, 0xFB, 0xFC, 0x0E, 0x11, 0x08, + 0xA8, 0xFD, 0x17, 0xB4, 0x48, 0xA6, 0x85, 0x54, 0x19, 0x9C, 0x47, 0xD0, 0x8F, 0xFB, 0x10, + 0xD4, 0xB8, + ]; + x.reverse(); + y.reverse(); + (x, y) +} + +pub fn bus_interactions() -> Vec { + let mu = || Multiplicity::Column(cols::MU); + let ok = || Multiplicity::Column(cols::OK); + let ts = |k: i64| col_plus(cols::TIMESTAMP_0, k); + let ts_hi = || packed(cols::TIMESTAMP_1); + let mut out = Vec::new(); + + // ECALL receiver: [ts_lo, ts_hi, syscall_lo32, syscall_hi32]. + out.push(BusInteraction::receiver( + BusId::Ecall, + mu(), + vec![ + ts(0), + ts_hi(), + BusValue::constant(ECREC_SYSCALL_NUMBER & 0xFFFF_FFFF), + BusValue::constant(ECREC_SYSCALL_NUMBER >> 32), + ], + )); + + // A register read of `reg` with value `[lo, hi]` at ts + k. + let reg_read = |reg: u64, lo: BusValue, hi: BusValue, k: i64| { + BusInteraction::sender( + BusId::Memw, + mu(), + memw_read( + register_value(lo, hi), + 1, + BusValue::constant(2 * reg), + BusValue::constant(0), + ts(k), + ts_hi(), + 1, + 0, + ), + ) + }; + + // T: x11 -> addr_sig; the signature r ‖ s (8 doublewords, big-endian values). + out.push(reg_read( + 11, + packed(cols::ADDR_SIG_0), + packed(cols::ADDR_SIG_1), + 0, + )); + for i in 0..8usize { + let value: [BusValue; 8] = std::array::from_fn(|j| { + let m = 8 * i + j; + if m < 32 { + be_byte(cols::R, m) + } else { + be_byte(cols::S, m - 32) + } + }); + out.push(BusInteraction::sender( + BusId::Memw, + mu(), + memw_read( + value, + 0, + col_plus(cols::ADDR_SIG_0, (8 * i) as i64), + packed(cols::ADDR_SIG_1), + ts(0), + ts_hi(), + 0, + 1, + ), + )); + } + + // T + 1: x12 -> addr_msg, x13 = parity (a value), the hash z (4 doublewords). + out.push(reg_read( + 12, + packed(cols::ADDR_MSG_0), + packed(cols::ADDR_MSG_1), + 1, + )); + out.push(reg_read(13, packed(cols::PAR), BusValue::constant(0), 1)); + for i in 0..4usize { + let value: [BusValue; 8] = std::array::from_fn(|j| be_byte(cols::Z, 8 * i + j)); + out.push(BusInteraction::sender( + BusId::Memw, + mu(), + memw_read( + value, + 0, + col_plus(cols::ADDR_MSG_0, (8 * i) as i64), + packed(cols::ADDR_MSG_1), + ts(1), + ts_hi(), + 0, + 1, + ), + )); + } + + // T + 2: x10 -> addr_out; the key Q.x ‖ Q.y (big-endian) and the status doubleword. + out.push(reg_read( + 10, + packed(cols::ADDR_OUT_0), + packed(cols::ADDR_OUT_1), + 2, + )); + for i in 0..9usize { + let value: [BusValue; 8] = std::array::from_fn(|j| { + let m = 8 * i + j; + match m { + 0..32 => be_byte(cols::XQ, m), + 32..64 => be_byte(cols::YQ, m - 32), + 64 => bail_value(), + _ => BusValue::constant(0), + } + }); + out.push(BusInteraction::sender( + BusId::Memw, + mu(), + memw_write( + value, + col_plus(cols::ADDR_OUT_0, (8 * i) as i64), + packed(cols::ADDR_OUT_1), + ts(2), + ts_hi(), + ), + )); + } + + // Byte checks on the computed row's witness bytes. + let is_byte = |col: usize, len: usize, out: &mut Vec| { + for i in 0..len { + out.push(BusInteraction::sender( + BusId::AreBytes, + ok(), + vec![packed(col + i), BusValue::constant(0)], + )); + } + }; + for col in [ + cols::W, + cols::QW, + cols::QZ, + cols::QU, + cols::YR, + cols::X2, + cols::Q0, + ] { + is_byte(col, 32, &mut out); + } + is_byte(cols::Q1, 33, &mut out); + is_byte(cols::H, 1, &mut out); + + // Halfword checks: the five relations' carries (offset), the nine subtraction witnesses. + for (col, off) in [ + (cols::CW, CARRY_OFFSET_MODN), + (cols::CZ, CARRY_OFFSET_MODN), + (cols::CU, CARRY_OFFSET_MODN), + (cols::C0, CARRY_OFFSET_X2), + (cols::C1, CARRY_OFFSET_YG), + ] { + for i in 0..63 { + out.push(BusInteraction::sender( + BusId::IsHalfword, + ok(), + vec![col_plus(col + i, off)], + )); + } + } + for col in [ + cols::R_SUB_N, + cols::S_SUB_N, + cols::U1_SUB_N, + cols::U2_SUB_N, + cols::YR_SUB_P, + cols::XA_SUB_P, + cols::XB_SUB_P, + cols::XQ_SUB_P, + cols::YQ_SUB_P, + ] { + for i in 0..16 { + out.push(BusInteraction::sender( + BusId::IsHalfword, + ok(), + vec![packed(col + i)], + )); + } + } + + // Zero: r, s, u1, u2 nonzero (the sums of their bytes). + let byte_sum = |col: usize, bits: bool| { + if bits { + BusValue::linear( + (0..256) + .map(|b| LinearTerm::Column { + coefficient: 1i64 << (b % 8), + column: col + b, + }) + .collect(), + ) + } else { + BusValue::linear( + (0..32) + .map(|b| LinearTerm::Column { + coefficient: 1, + column: col + b, + }) + .collect(), + ) + } + }; + for (col, bits) in [ + (cols::R, false), + (cols::S, false), + (cols::U1, true), + (cols::U2, true), + ] { + out.push(BusInteraction::sender( + BusId::Zero, + ok(), + vec![byte_sum(col, bits), BusValue::constant(0)], + )); + } + + // The first chain, u1·G, keyed at ts. + let (gx, gy) = generator_le(); + for i in 0..256 { + out.push(BusInteraction::receiver( + BusId::Bit, + Multiplicity::Column(cols::U1 + i), + vec![ts(0), ts_hi(), BusValue::constant(i as u64)], + )); + } + out.push(BusInteraction::sender( + BusId::Bit, + ok(), + vec![ts(0), ts_hi(), packed(cols::LEN1)], + )); + out.push(BusInteraction::sender( + BusId::Ecdas, + ok(), + ecdas_tuple( + const_coord(&gx), + const_coord(&gy), + const_coord(&gx), + const_coord(&gy), + col_plus(cols::LEN1, -1), + BusValue::constant(0), + ts(0), + ts_hi(), + ), + )); + out.push(BusInteraction::receiver( + BusId::Ecdas, + ok(), + ecdas_tuple( + point_coord_busvalues(cols::XA), + point_coord_busvalues(cols::YA), + const_coord(&gx), + const_coord(&gy), + BusValue::linear(vec![LinearTerm::Constant(-1)]), + BusValue::constant(0), + ts(0), + ts_hi(), + ), + )); + + // The second chain, u2·R with R = (r, yR), keyed at ts + 1. + for i in 0..256 { + out.push(BusInteraction::receiver( + BusId::Bit, + Multiplicity::Column(cols::U2 + i), + vec![ts(CHAIN2_KEY), ts_hi(), BusValue::constant(i as u64)], + )); + } + out.push(BusInteraction::sender( + BusId::Bit, + ok(), + vec![ts(CHAIN2_KEY), ts_hi(), packed(cols::LEN2)], + )); + out.push(BusInteraction::sender( + BusId::Ecdas, + ok(), + ecdas_tuple( + point_coord_busvalues(cols::R), + point_coord_busvalues(cols::YR), + point_coord_busvalues(cols::R), + point_coord_busvalues(cols::YR), + col_plus(cols::LEN2, -1), + BusValue::constant(0), + ts(CHAIN2_KEY), + ts_hi(), + ), + )); + out.push(BusInteraction::receiver( + BusId::Ecdas, + ok(), + ecdas_tuple( + point_coord_busvalues(cols::XB), + point_coord_busvalues(cols::YB), + point_coord_busvalues(cols::R), + point_coord_busvalues(cols::YR), + BusValue::linear(vec![LinearTerm::Constant(-1)]), + BusValue::constant(0), + ts(CHAIN2_KEY), + ts_hi(), + ), + )); + + // The final addition Q = A + B, one ECDAS add row keyed at ts + 2. + out.push(BusInteraction::sender( + BusId::Ecdas, + ok(), + ecdas_tuple( + point_coord_busvalues(cols::XA), + point_coord_busvalues(cols::YA), + point_coord_busvalues(cols::XB), + point_coord_busvalues(cols::YB), + BusValue::constant(0), + BusValue::constant(1), + ts(ADD_KEY), + ts_hi(), + ), + )); + out.push(BusInteraction::receiver( + BusId::Ecdas, + ok(), + ecdas_tuple( + point_coord_busvalues(cols::XQ), + point_coord_busvalues(cols::YQ), + point_coord_busvalues(cols::XB), + point_coord_busvalues(cols::YB), + BusValue::linear(vec![LinearTerm::Constant(-1)]), + BusValue::constant(0), + ts(ADD_KEY), + ts_hi(), + ), + )); + + out +} + +// ========================================================================= +// Constraints +// ========================================================================= + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; + +/// The `< m` checks, as ECSM's overflow chains: `m + sub = v + 2^256` with a carry out. +/// Each is load-bearing (`tests::ecrec_tests::*_is_refused_only_by_its_bound`). +#[derive(Clone, Copy, Debug)] +pub enum Bound { + RLtN, + /// Without it `(r, s + N)`, which Ethereum refuses, recovers the same key. + SLtN, + /// `u1 < N` and `u2 < N` keep each chain off ECDAS's chord branch at `acc = gen`, + /// where λ is free: a prover who makes the true scalar 2 (`r = −z/2`, or + /// `s = 2r`) and claims `2 + N` would end that chain in `G + G` by the chord. + U1LtN, + /// See [`Bound::U1LtN`]. + U2LtN, + YrLtP, + /// Canonical `xA`, `xB` make the `xA ≠ xB` witness mean `xA ≢ xB (mod p)`. + XaLtP, + /// See [`Bound::XaLtP`]. + XbLtP, + XqLtP, + YqLtP, +} + +impl Bound { + pub const ALL: [Bound; 9] = [ + Bound::RLtN, + Bound::SLtN, + Bound::U1LtN, + Bound::U2LtN, + Bound::YrLtP, + Bound::XaLtP, + Bound::XbLtP, + Bound::XqLtP, + Bound::YqLtP, + ]; + + fn modulus(self) -> &'static [u8; 32] { + match self { + Bound::RLtN | Bound::SLtN | Bound::U1LtN | Bound::U2LtN => &N_BYTES, + _ => &P_BYTES, + } + } + + fn sub_col(self) -> usize { + match self { + Bound::RLtN => cols::R_SUB_N, + Bound::SLtN => cols::S_SUB_N, + Bound::U1LtN => cols::U1_SUB_N, + Bound::U2LtN => cols::U2_SUB_N, + Bound::YrLtP => cols::YR_SUB_P, + Bound::XaLtP => cols::XA_SUB_P, + Bound::XbLtP => cols::XB_SUB_P, + Bound::XqLtP => cols::XQ_SUB_P, + Bound::YqLtP => cols::YQ_SUB_P, + } + } + + /// The bounded value's column and whether it is held as bits (else bytes). + fn value_col(self) -> (usize, bool) { + match self { + Bound::RLtN => (cols::R, false), + Bound::SLtN => (cols::S, false), + Bound::U1LtN => (cols::U1, true), + Bound::U2LtN => (cols::U2, true), + Bound::YrLtP => (cols::YR, false), + Bound::XaLtP => (cols::XA, false), + Bound::XbLtP => (cols::XB, false), + Bound::XqLtP => (cols::XQ, false), + Bound::YqLtP => (cols::YQ, false), + } + } +} + +/// Which product relation a carry chain enforces. +#[derive(Clone, Copy, Debug)] +pub enum Relation { + /// `r·w − 1 − qw·N = 0` + W, + /// `z·w + u1 − qz·N = 0` + U1, + /// `s·w − u2 − qu·N = 0` + U2, + /// `r² − x2 − q0·p = 0` + X2, + /// `yR² + p² − r·x2 − b − q1·p = 0` + Yr, +} + +/// ECREC constraints. Index map: +/// 0, 1, 2 : IS_BIT(MU), IS_BIT(OK), IS_BIT(PAR) +/// 3 : OK·(1 − MU) +/// 4..516 : IS_BIT of the 512 scalar bits +/// 516 : (Σ u1 bits + Σ u2 bits)·(1 − OK) +/// 517..842 : for W, U1, U2, X2, Yr: 64 OK-gated carry steps + OK·c₆₃ +/// 842 : OK·IS_BIT(q1[32]) +/// 843 : OK·(yR[0] − 2·H − PAR) +/// 844..916 : for the nine bounds: 7 OK·carry-bit + OK·(1 − c₇) +/// 916 : Σ_c d_c·DINV_c − OK +/// 917..981 : (MU − OK)·xQ_i, (MU − OK)·yQ_i +#[derive(Clone, Copy)] +pub struct EcrecConstraints; + +pub const NUM_CONSTRAINTS: usize = 981; + +type Gl = GoldilocksField; +type Ext = GoldilocksExtension; + +impl EcrecConstraints { + fn byte>(b: &B, base: usize, len: usize, j: usize) -> B::Expr { + if j < len { + b.main(0, base + j) + } else { + b.zero() + } + } + + /// Byte `j` of a scalar held as 256 bits at `base`. + fn bits_byte>(b: &B, base: usize, j: usize) -> B::Expr { + let mut e = b.zero(); + if j < 32 { + for k in 0..8 { + e = e + b.main(0, base + 8 * j + k) * b.const_base(1u64 << k); + } + } + e + } + + fn const_byte>(b: &B, bytes: &[u8; 32], j: usize) -> B::Expr { + if j < 32 { + b.const_base(bytes[j] as u64) + } else { + b.zero() + } + } + + /// `S_i` of `relation` at limb `i`. + fn s_i>(b: &B, relation: Relation, i: usize) -> B::Expr { + let by = |base: usize, j: usize| Self::byte(b, base, 32, j); + let n_ = |j: usize| Self::const_byte(b, &N_BYTES, j); + let p_ = |j: usize| Self::const_byte(b, &P_BYTES, j); + let mut s = b.zero(); + match relation { + Relation::W => { + for j in 0..=i { + s = s + by(cols::R, j) * by(cols::W, i - j) - by(cols::QW, j) * n_(i - j); + } + if i == 0 { + s = s - b.one(); + } + } + Relation::U1 => { + for j in 0..=i { + s = s + by(cols::Z, j) * by(cols::W, i - j) - by(cols::QZ, j) * n_(i - j); + } + s = s + Self::bits_byte(b, cols::U1, i); + } + Relation::U2 => { + for j in 0..=i { + s = s + by(cols::S, j) * by(cols::W, i - j) - by(cols::QU, j) * n_(i - j); + } + s = s - Self::bits_byte(b, cols::U2, i); + } + Relation::X2 => { + for j in 0..=i { + s = s + by(cols::R, j) * by(cols::R, i - j) - by(cols::Q0, j) * p_(i - j); + } + s = s - by(cols::X2, i); + } + Relation::Yr => { + let mut p2 = b.zero(); + for j in 0..=i { + s = s + by(cols::YR, j) * by(cols::YR, i - j); + p2 = p2 + p_(j) * p_(i - j); + s = s - by(cols::X2, j) * by(cols::R, i - j); + s = s - Self::byte(b, cols::Q1, 33, j) * p_(i - j); + } + s = s + p2; + if i == 0 { + s = s - b.const_base(B); + } + } + } + s + } + + fn carry_base(relation: Relation) -> usize { + match relation { + Relation::W => cols::CW, + Relation::U1 => cols::CZ, + Relation::U2 => cols::CU, + Relation::X2 => cols::C0, + Relation::Yr => cols::C1, + } + } + + /// The eight virtual word carries of `bound`'s addition `m + sub = v + 2^256`. + fn bound_carries>(b: &B, bound: Bound) -> [B::Expr; 8] { + let m = bound.modulus(); + let hl = bound.sub_col(); + let (vcol, bits) = bound.value_col(); + let mut c: [B::Expr; 8] = std::array::from_fn(|_| b.zero()); + let mut prev = b.zero(); + for (i, slot) in c.iter_mut().enumerate() { + let sub = b.main(0, hl + 2 * i) + b.main(0, hl + 2 * i + 1) * b.const_base(1 << 16); + let mut v = b.zero(); + if bits { + for k in 0..32 { + v = v + b.main(0, vcol + 32 * i + k) * b.const_base(1u64 << k); + } + } else { + for k in 0..4 { + v = v + b.main(0, vcol + 4 * i + k) * b.const_base(1u64 << (8 * k)); + } + } + let word = (0..4).fold(0u64, |acc, k| acc + ((m[4 * i + k] as u64) << (8 * k))); + let ci = (b.const_base(word) + sub + prev.clone() - v) * b.const_base(INV_SHIFT_32); + *slot = ci.clone(); + prev = ci; + } + c + } +} + +impl ConstraintSet for EcrecConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let mut idx = 0; + let mut emit = |b: &mut B, e: B::Expr| { + b.emit_base(idx, e); + idx += 1; + }; + let main = |b: &B, c: usize| b.main(0, c); + + for c in [cols::MU, cols::OK, cols::PAR] { + let x = main(b, c); + let e = x.clone() * (b.one() - x); + emit(b, e); + } + let e = main(b, cols::OK) * (b.one() - main(b, cols::MU)); + emit(b, e); + + let mut bit_sum = b.zero(); + for base in [cols::U1, cols::U2] { + for i in 0..256 { + let x = main(b, base + i); + bit_sum = bit_sum + x.clone(); + let e = x.clone() * (b.one() - x); + emit(b, e); + } + } + let e = bit_sum * (b.one() - main(b, cols::OK)); + emit(b, e); + + for relation in [ + Relation::W, + Relation::U1, + Relation::U2, + Relation::X2, + Relation::Yr, + ] { + let base = Self::carry_base(relation); + for i in 0..64 { + let prev = if i == 0 { + b.zero() + } else { + main(b, base + i - 1) + }; + let step = main(b, base + i) * b.const_base(256) - prev - Self::s_i(b, relation, i); + let e = main(b, cols::OK) * step; + emit(b, e); + } + let e = main(b, cols::OK) * main(b, base + 63); + emit(b, e); + } + + let q = main(b, cols::Q1 + 32); + let e = main(b, cols::OK) * q.clone() * (b.one() - q); + emit(b, e); + + let e = main(b, cols::OK) + * (main(b, cols::YR) - main(b, cols::H) * b.const_base(2) - main(b, cols::PAR)); + emit(b, e); + + for bound in Bound::ALL { + let c = Self::bound_carries(b, bound); + for ci in c.iter().take(7) { + let e = main(b, cols::OK) * ci.clone() * (b.one() - ci.clone()); + emit(b, e); + } + let e = main(b, cols::OK) * (b.one() - c[7].clone()); + emit(b, e); + } + + // xA ≠ xB: Σ_c (xB − xA)_c · DINV_c = OK over the eight 32-bit word differences. + let mut sum = b.zero(); + for c in 0..8 { + let mut d = b.zero(); + for k in 0..4 { + d = d + + (main(b, cols::XB + 4 * c + k) - main(b, cols::XA + 4 * c + k)) + * b.const_base(1u64 << (8 * k)); + } + sum = sum + d * main(b, cols::DINV + c); + } + let e = sum - main(b, cols::OK); + emit(b, e); + + // A bail writes a zero key. + for base in [cols::XQ, cols::YQ] { + for i in 0..32 { + let e = (main(b, cols::MU) - main(b, cols::OK)) * main(b, base + i); + emit(b, e); + } + } + + debug_assert_eq!(idx, NUM_CONSTRAINTS); + } +} diff --git a/prover/src/tables/ecsm.rs b/prover/src/tables/ecsm.rs index 746bef91c..90b873fad 100644 --- a/prover/src/tables/ecsm.rs +++ b/prover/src/tables/ecsm.rs @@ -19,9 +19,10 @@ use executor::vm::instruction::execution::ECSM_SYSCALL_NUMBER; use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable}; use crate::constraints::templates::INV_SHIFT_32; -use ecsm::{B, EcsmWitness, N_BYTES, P_BYTES}; +use ecsm::{B, EcdasStep, EcsmWitness, N_BYTES, P_BYTES}; // Bias signed convolution carries into IsHalfword [0, 2^16); see spec ecsm.typ "Carry offset" (@ecsm-limb_carry). pub(crate) const CARRY_OFFSET_X2: i64 = 8160; @@ -124,6 +125,14 @@ pub struct EcsmOperation { pub witness: EcsmWitness, } +impl EcsmOperation { + /// The heap its witness owns beyond the op: the double/add steps' buffer + /// (one ECDAS row each), which no ECSM table reads. + pub(crate) fn steps_heap_bytes(&self) -> usize { + self.witness.steps.capacity() * std::mem::size_of::() + } +} + // ========================================================================= // Trace generation // ========================================================================= @@ -149,50 +158,76 @@ fn write_halfwords(table: &mut impl VmTable, row: usize, col: usize, bytes: &[u8 pub fn generate_ecsm_trace( ops: &[EcsmOperation], ) -> TraceTable { - let n = ops.len(); - let num_rows = n.next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in ops.iter().enumerate() { - let w = &op.witness; - - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - table.set_dword_wl(row_idx, cols::ADDR_XG_0, op.addr_xg); - table.set_dword_wl(row_idx, cols::ADDR_K_0, op.addr_k); - table.set_dword_wl(row_idx, cols::ADDR_XR_0, op.addr_xr); - - table.set_bytes(row_idx, cols::XR, &w.x_r); - table.set_bytes(row_idx, cols::YR, &w.y_r); - for b in 0..256 { - let bit = (w.k[b / 8] >> (b % 8)) & 1; - table.set_fe(row_idx, cols::k_bit(b), FE::from(bit as u64)); - } - table.set_u64(row_idx, cols::LEN_K, w.len_k as u64); - table.set_bytes(row_idx, cols::XG, &w.x_g); - table.set_bytes(row_idx, cols::YG, &w.y_g); - table.set_bytes(row_idx, cols::X2, &w.x2); - table.set_bytes(row_idx, cols::Q0, &w.q0); - table.set_bytes(row_idx, cols::Q1, &w.q1); - write_halfwords(table, row_idx, cols::XG_SUB_P, &w.x_g_sub_p); - write_halfwords(table, row_idx, cols::K_SUB_N, &w.k_sub_n); - write_halfwords(table, row_idx, cols::XR_SUB_P, &w.x_r_sub_p); + generate_ecsm_trace_as(ops, TraceForm::Wide) +} - for i in 0..64 { - debug_assert!((0..1 << 16).contains(&(w.c0[i] + CARRY_OFFSET_X2))); - debug_assert!((0..1 << 16).contains(&(w.c1[i] + CARRY_OFFSET_YG))); - table.set_fe(row_idx, cols::c0(i), fe_from_i64(w.c0[i])); - table.set_fe(row_idx, cols::c1(i), fe_from_i64(w.c1[i])); - } +/// The widths ECSM traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - table.set_fe(row_idx, cols::MU, FE::one()); - } +/// [`generate_ecsm_trace`] in `form` (`tables::gpack`). +pub fn generate_ecsm_trace_as( + ops: &[EcsmOperation], + form: TraceForm, +) -> TraceTable { + generate_ecsm_rows_as(ops, ops.len().next_power_of_two().max(4), form) +} + +/// Whether [`generate_ecsm_rows_as`] in `form` writes its rows packed, with no +/// 64-bit table on the way: in [`TraceForm::Narrow`] once an earlier ECSM +/// trace of this process left its widths (`tables::gpack`). +pub(crate) fn rows_written_packed(form: TraceForm) -> bool { + matches!( + super::gpack::Plan::new(form, &WIDTHS, cols::NUM_COLUMNS), + super::gpack::Plan::Write(_) + ) +} + +/// A ECSM table of `num_rows` rows in `form`: `ops` (at most `num_rows`, a row +/// each, one scalar multiplication a row), then the padding rows. A row reads its own op alone and +/// the padding rows are constants, so rows `[k·R, (k+1)·R)` of the whole padded +/// table are this over the ops in that range with `num_rows = R`: the block's +/// cut of the whole table, built on its own. +pub fn generate_ecsm_rows_as( + ops: &[EcsmOperation], + num_rows: usize, + form: TraceForm, +) -> TraceTable { + // Padding rows are all zero. + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in ops.iter().enumerate() { + let w = &op.witness; + + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + table.set_dword_wl(row_idx, cols::ADDR_XG_0, op.addr_xg); + table.set_dword_wl(row_idx, cols::ADDR_K_0, op.addr_k); + table.set_dword_wl(row_idx, cols::ADDR_XR_0, op.addr_xr); + + table.set_bytes(row_idx, cols::XR, &w.x_r); + table.set_bytes(row_idx, cols::YR, &w.y_r); + for b in 0..256 { + let bit = (w.k[b / 8] >> (b % 8)) & 1; + table.set_fe(row_idx, cols::k_bit(b), FE::from(bit as u64)); + } + table.set_u64(row_idx, cols::LEN_K, w.len_k as u64); + table.set_bytes(row_idx, cols::XG, &w.x_g); + table.set_bytes(row_idx, cols::YG, &w.y_g); + table.set_bytes(row_idx, cols::X2, &w.x2); + table.set_bytes(row_idx, cols::Q0, &w.q0); + table.set_bytes(row_idx, cols::Q1, &w.q1); + write_halfwords(table, row_idx, cols::XG_SUB_P, &w.x_g_sub_p); + write_halfwords(table, row_idx, cols::K_SUB_N, &w.k_sub_n); + write_halfwords(table, row_idx, cols::XR_SUB_P, &w.x_r_sub_p); + + for i in 0..64 { + debug_assert!((0..1 << 16).contains(&(w.c0[i] + CARRY_OFFSET_X2))); + debug_assert!((0..1 << 16).contains(&(w.c1[i] + CARRY_OFFSET_YG))); + table.set_fe(row_idx, cols::c0(i), fe_from_i64(w.c0[i])); + table.set_fe(row_idx, cols::c1(i), fe_from_i64(w.c1[i])); + } - trace + table.set_fe(row_idx, cols::MU, FE::one()); + } + }) } // ========================================================================= diff --git a/prover/src/tables/eq.rs b/prover/src/tables/eq.rs index b3287f6d8..4d0e09012 100644 --- a/prover/src/tables/eq.rs +++ b/prover/src/tables/eq.rs @@ -26,6 +26,7 @@ use stark::trace::TraceTable; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; use crate::constraints::templates::{AddOperand, emit_add_pair, emit_is_bit}; @@ -120,31 +121,35 @@ impl EqOperation { pub fn generate_eq_trace( operations: &[EqOperation], ) -> TraceTable { - let num_rows = operations.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; + generate_eq_trace_as(operations, TraceForm::Wide) +} - for (row_idx, op) in operations.iter().enumerate() { - // a, b as DWordWL (2 words each) - table.set_dword_wl(row_idx, cols::A_0, op.a); - table.set_dword_wl(row_idx, cols::B_0, op.b); +/// The widths EQ traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - table.set_bool(row_idx, cols::INVERT, op.invert); - table.set_bool(row_idx, cols::RES, op.compute_res()); +/// [`generate_eq_trace`] in `form` (`tables::gpack`). +pub fn generate_eq_trace_as( + operations: &[EqOperation], + form: TraceForm, +) -> TraceTable { + let num_rows = operations.len().next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in operations.iter().enumerate() { + // a, b as DWordWL (2 words each) + table.set_dword_wl(row_idx, cols::A_0, op.a); + table.set_dword_wl(row_idx, cols::B_0, op.b); - // diff = a - b (wrapping) as DWordHL (4 halves) - let diff = op.a.wrapping_sub(op.b); - table.set_dword_hl(row_idx, cols::DIFF_0, diff); + table.set_bool(row_idx, cols::INVERT, op.invert); + table.set_bool(row_idx, cols::RES, op.compute_res()); - table.set_bool(row_idx, cols::EQ, op.compute_eq()); - table.set_u64(row_idx, cols::MU, 1); - } + // diff = a - b (wrapping) as DWordHL (4 halves) + let diff = op.a.wrapping_sub(op.b); + table.set_dword_hl(row_idx, cols::DIFF_0, diff); - trace + table.set_bool(row_idx, cols::EQ, op.compute_eq()); + table.set_u64(row_idx, cols::MU, 1); + } + }) } // ========================================================================= diff --git a/prover/src/tables/gpack.rs b/prover/src/tables/gpack.rs new file mode 100644 index 000000000..dcae9c26d --- /dev/null +++ b/prover/src/tables/gpack.rs @@ -0,0 +1,268 @@ +//! G-pack: a generator writes its main trace straight into the packed form the +//! block holds it in (`stark::narrow`), with no 64-bit copy. +//! +//! A generator's fill is generic over [`VmTable`], and [`generate_main!`] runs +//! it on the writer its caller's [`TraceForm`] asks for: +//! - [`TraceForm::Wide`]: the table at 8 bytes a cell, as every build outside +//! the block's narrow storage holds it; +//! - [`TraceForm::Narrow`]: the packed trace, the `NarrowMain::pack` of the same +//! words byte for byte. When the table's kind has a [`WidthHint`] (the widths +//! its earlier traces needed in this process), the fill writes the packed +//! columns directly at those widths ([`NarrowWriter`]): a column that needed +//! fewer bytes is narrowed in place, and a word wider than its column has the +//! fill run again at the widths the trace needs. Without a hint (the kind's +//! first trace) the fill writes the 64-bit table and packs it, as before, +//! and the widths it needed become the hint. +//! +//! The packed bytes are a function of the words alone: a hint changes the +//! time a trace takes, never a byte of it. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + +use multilinear::narrow::NarrowColumns; +use stark::narrow::NarrowWriter; +use stark::trace::TraceTable; + +use super::types::{FE, GoldilocksExtension, GoldilocksField, VmTable}; + +/// A generator's trace. +pub type Trace = TraceTable; + +/// How a generator holds the main trace it writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TraceForm { + /// At 8 bytes a cell. + Wide, + /// Packed at the bytes its columns need (`TraceTable::is_main_narrow`), + /// where a trace can be held packed; at 8 bytes a cell where it cannot (the + /// `debug-checks` build). + Narrow, +} + +/// The widths a table kind's traces needed so far in this process, column by +/// column (the most each column needed): where a [`TraceForm::Narrow`] +/// generator writes the kind's next trace. +pub struct WidthHint(Mutex>); + +impl WidthHint { + /// No widths yet. + pub const fn new() -> Self { + Self(Mutex::new(Vec::new())) + } + + /// The widths, when there are some for `cols` columns. + fn get(&self, cols: usize) -> Option> { + let widths = self.0.lock().unwrap_or_else(|e| e.into_inner()); + (widths.len() == cols && cols > 0).then(|| widths.clone()) + } + + /// Whether there are widths for `cols` columns. + pub fn known(&self, cols: usize) -> bool { + let widths = self.0.lock().unwrap_or_else(|e| e.into_inner()); + widths.len() == cols && cols > 0 + } + + /// Take `widths` (a trace's) into the most each column needed. + pub fn learn(&self, widths: &[u8]) { + let mut most = self.0.lock().unwrap_or_else(|e| e.into_inner()); + if most.len() == widths.len() { + for (m, &w) in most.iter_mut().zip(widths) { + *m = (*m).max(w); + } + } else { + *most = widths.to_vec(); + } + } +} + +impl Default for WidthHint { + fn default() -> Self { + Self::new() + } +} + +/// The packed cells: the raw word of each element, as `NarrowMain` holds it. +impl VmTable for NarrowWriter { + #[inline] + fn set_fe(&mut self, row: usize, col: usize, value: FE) { + self.set(row, col, *value.value()); + } +} + +/// How [`generate_main!`] builds one trace. +#[doc(hidden)] +pub enum Plan { + /// The 64-bit table. + Wide, + /// The 64-bit table, then packed, its widths learned. + WideThenPack, + /// Written packed at these widths. + Write(Vec), +} + +impl Plan { + #[doc(hidden)] + pub fn new(form: TraceForm, hint: &WidthHint, cols: usize) -> Self { + match form { + TraceForm::Wide => Plan::Wide, + // The debug checks read the 64-bit table (`TraceTable::pack_main_narrow`). + TraceForm::Narrow if cfg!(feature = "debug-checks") => Plan::Wide, + TraceForm::Narrow => hint.get(cols).map_or(Plan::WideThenPack, Plan::Write), + } + } +} + +/// Traces a [`TraceForm::Narrow`] generator wrote packed and finished as +/// written, narrowed some columns of in place, wrote again after a word missed +/// its column, and built at 8 bytes a cell then packed (no hint yet); since +/// [`reset_counts`]. +static COUNTS: [AtomicUsize; 4] = [const { AtomicUsize::new(0) }; 4]; + +/// G-pack's counts since [`reset_counts`]: traces written packed as they +/// were, written packed with some columns narrowed afterwards, written packed +/// a second time (a word missed its column), and built wide then packed. +pub fn counts() -> [usize; 4] { + std::array::from_fn(|i| COUNTS[i].load(Relaxed)) +} + +/// Zero [`counts`]. +pub fn reset_counts() { + for c in &COUNTS { + c.store(0, Relaxed); + } +} + +#[cfg(test)] +thread_local! { + /// [`counts`] of this thread's traces, never reset. + static THREAD_COUNTS: std::cell::Cell<[usize; 4]> = const { std::cell::Cell::new([0; 4]) }; +} + +/// [`counts`] of the traces this thread built, since the thread began. +#[cfg(test)] +pub(crate) fn thread_counts() -> [usize; 4] { + THREAD_COUNTS.with(std::cell::Cell::get) +} + +/// One more trace in [`counts`] slot `i`. +fn count(i: usize) { + COUNTS[i].fetch_add(1, Relaxed); + #[cfg(test)] + THREAD_COUNTS.with(|c| { + let mut n = c.get(); + n[i] += 1; + c.set(n); + }); +} + +/// A zeroed 64-bit trace of `rows` × `cols`. +#[doc(hidden)] +pub fn wide_trace(rows: usize, cols: usize) -> Trace { + TraceTable::new_main(super::types::zeroed_fe_vec(rows * cols), cols, 1) +} + +/// `trace` packed (where it can be), its widths taken into `hint`. +#[doc(hidden)] +pub fn packed(mut trace: Trace, hint: &WidthHint) -> Trace { + if trace.pack_main_narrow() + && let Some(narrow) = trace.narrow_main() + { + hint.learn(narrow.widths()); + count(3); + } + trace +} + +/// The trace `writer` holds, its widths taken into `hint`; or the widths it +/// needs, when a word missed its column. +#[doc(hidden)] +pub fn written(writer: NarrowWriter, hint: &WidthHint, again: bool) -> Result> { + let narrowed = writer.needed_widths() != writer.widths(); + match writer.finish() { + Ok(narrow) => { + hint.learn(narrow.widths()); + count(if again { + 2 + } else if narrowed { + 1 + } else { + 0 + }); + // The trace holds it as `multilinear`'s columns, the same layout. + // An `Err` of no widths has the caller build the trace wide (not + // reached: the writer's widths are 1, 2, 4 or 8 and its bytes are + // rows × Σ widths). + let (rows, widths, data) = narrow.into_parts(); + let columns = NarrowColumns::from_parts(rows, widths, data).ok_or_else(Vec::new)?; + Ok(TraceTable::try_from_narrow_main(columns, 1) + .unwrap_or_else(|columns| widened(&columns))) + } + Err(need) => { + hint.learn(&need); + Err(need) + } + } +} + +/// `columns`' words as a 64-bit trace. +fn widened(columns: &NarrowColumns) -> Trace { + let (rows, cols) = (columns.rows(), columns.cols()); + let mut data = vec![FE::zero(); rows * cols]; + for c in 0..cols { + for (r, w) in columns.column(c).into_iter().enumerate() { + data[r * cols + c] = FE::from(w); + } + } + TraceTable::new_main(data, cols, 1) +} + +/// Build a `rows` × `cols` main trace (step size 1) in `form`: `fill`, an +/// expression over `$t: &mut impl VmTable` that writes every non-zero cell +/// once, runs on the writer the [`Plan`] picks, and again on a second writer +/// when a word missed its column (so it must only read its inputs). +/// +/// ```ignore +/// generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |t| fill(t, ops)) +/// ``` +macro_rules! generate_main { + ($form:expr, $hint:expr, $rows:expr, $cols:expr, |$t:ident| $fill:expr) => {{ + use $crate::tables::gpack::{Plan, packed, wide_trace, written}; + let (rows, cols, hint): (usize, usize, &$crate::tables::gpack::WidthHint) = + ($rows, $cols, $hint); + let wide = |then_pack: bool| { + let mut trace = wide_trace(rows, cols); + { + let $t = &mut trace.main_table; + $fill; + } + if then_pack { + packed(trace, hint) + } else { + trace + } + }; + let write = |widths: &[u8], again: bool| { + let mut writer = ::stark::narrow::NarrowWriter::new(rows, widths); + { + let $t = &mut writer; + $fill; + } + written(writer, hint, again) + }; + match Plan::new($form, hint, cols) { + Plan::Wide => wide(false), + Plan::WideThenPack => wide(true), + Plan::Write(widths) => match write(&widths, false) { + Ok(trace) => trace, + Err(need) if need.len() == cols => match write(&need, true) { + Ok(trace) => trace, + // Not reached: every word fits the widths of every word. + Err(_) => wide(true), + }, + Err(_) => wide(true), + }, + } + }}; +} +pub(crate) use generate_main; diff --git a/prover/src/tables/hint.rs b/prover/src/tables/hint.rs index aff37a4d8..0d9e2c16a 100644 --- a/prover/src/tables/hint.rs +++ b/prover/src/tables/hint.rs @@ -64,6 +64,7 @@ use stark::trace::TraceTable; use crate::constraints::templates::emit_is_bit; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; /// One past the largest valid hint selector (`a0 ∈ {0, 1, 2}` = FIELD_INV / SCALAR_INV / @@ -136,29 +137,33 @@ pub struct HintOperation { pub fn generate_hint_trace( ops: &[HintOperation], ) -> TraceTable { - let num_rows = ops.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; + generate_hint_trace_as(ops, TraceForm::Wide) +} - for (row, op) in ops.iter().enumerate() { - debug_assert!( - op.timestamp <= u32::MAX as u64, - "HINT timestamp {} exceeds u32", - op.timestamp - ); - table.set_dword_wl(row, cols::TIMESTAMP_0, op.timestamp); - table.set_dword_wl(row, cols::ADDR_OUT_0, op.out_addr); - table.set_bytes(row, cols::OUT, &op.out_bytes); - table.set_dword_wl(row, cols::SEL_0, op.hint_id); - table.set_dword_wl(row, cols::ADDR_IN_0, op.in_addr); - table.set_fe(row, cols::MU, FE::one()); - } +/// The widths HINT traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - trace +/// [`generate_hint_trace`] in `form` (`tables::gpack`). +pub fn generate_hint_trace_as( + ops: &[HintOperation], + form: TraceForm, +) -> TraceTable { + let num_rows = ops.len().next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row, op) in ops.iter().enumerate() { + debug_assert!( + op.timestamp <= u32::MAX as u64, + "HINT timestamp {} exceeds u32", + op.timestamp + ); + table.set_dword_wl(row, cols::TIMESTAMP_0, op.timestamp); + table.set_dword_wl(row, cols::ADDR_OUT_0, op.out_addr); + table.set_bytes(row, cols::OUT, &op.out_bytes); + table.set_dword_wl(row, cols::SEL_0, op.hint_id); + table.set_dword_wl(row, cols::ADDR_IN_0, op.in_addr); + table.set_fe(row, cols::MU, FE::one()); + } + }) } // ========================================================================= diff --git a/prover/src/tables/keccak.rs b/prover/src/tables/keccak.rs index 92a54f524..8b66af20f 100644 --- a/prover/src/tables/keccak.rs +++ b/prover/src/tables/keccak.rs @@ -1,26 +1,86 @@ //! KECCAK core chip — handles ECALL, memory I/O, and delegation to the round chip. //! -//! One row per keccak permutation call. Reads/writes 25 u64 lanes from/to memory, -//! sends input state to the round chip via the Keccak bus, and receives the output -//! state after 24 rounds. +//! One row per keccak permutation. The chip serves two syscalls, selected per +//! row by two mutually exclusive mode bits, `MU = MU_P + MU_S`: //! -//! ## Column layout (486 columns) +//! * **`MU_P` — permute** ([`KECCAK_SYSCALL_NUMBER`]). One ecall, one row: reads +//! the 25 lanes of the state at `x10`, sends them to the round chip on the +//! Keccak bus, receives the output after 24 rounds and writes it back. +//! * **`MU_S` — Keccak-256 of a buffer** ([`KECCAK256_SYSCALL_NUMBER`]). One +//! ecall of `len` bytes is `⌊len / 136⌋ + 1` rows, one per absorbed block: row +//! `j` reads block `j` from memory, XORs the padded block into the rate of the +//! state the previous row's permutation ended in, and sends that to the round +//! chip; the last row writes the first four lanes of its own output (the +//! digest) to `x12`. +//! +//! ## The sponge mode +//! +//! The rows of one call are linked by the [`BusId::KeccakSponge`] bus: a row +//! that is not the call's last sends `(T, j + 1, count − 136, src + 136, +//! src_hi, out, output state)` and the next row receives it as `(T, j, count, +//! src, src_hi, out, previous state)`. The previous state's rate lands in +//! `PREV`, its capacity directly in the capacity bytes of `INPUT_STATE`. The +//! first row receives the call on the Ecall bus instead and reads `x10`, `x11`, +//! `x12` (input address, length, output address) through the memory argument. +//! +//! On a sponge row `TIMESTAMP_0` is the call's timestamp `T` and `TIMESTAMP_1` +//! is the block index `j`, so the Keccak-bus key `(T, j)` is distinct for every +//! permutation of every call (a permute row's key is `(T', 0)` at its own +//! ecall's timestamp). Nothing but that key binds a round-chip request to its +//! reply, so its uniqueness is what makes the reply this row's permutation. +//! Every memory access of a sponge row uses the call's own timestamps +//! `(T, 0)` (reads) and `(T + 1, 0)` (the digest writes), never `TIMESTAMP_1`. +//! +//! Which bytes of a block are input is a prefix mask `s`: `s[8k + b] = +//! FULL[k] + PART[k][b]` for `b < 7` and `s[8k + 7] = FULL[k]`. A full lane is +//! one 8-byte read (multiplicity `FULL[k]`), a lane the input ends inside is +//! read a byte at a time (`PART[k][b]`). No byte past the input is read, and +//! its `MSG` cell is forced to 0. The padded block is then a linear form of +//! the columns, `m'[i] = MSG[i] + (s[i−1] − s[i]) + [i = 135]·0x80·LAST` +//! (`s[−1] = 1`): the `0x01` lands right after the input and the `0x80` on +//! byte 135 (`0x81` when the two meet). +//! +//! The byte count `COUNT` starts at `x11` (its high word forced 0), falls by +//! 136 per row, and the last row's mask must hold exactly `COUNT` bytes, at +//! most 135. In integers `len − 136·J` is in `(−2^48, 2^32)` for any chain of +//! `J <` [`SPONGE_MAX_ROWS`] rows, far inside the field, so `COUNT = Σ s` on the +//! last row forces `J = ⌊len/136⌋`. The same bound keeps the block index `j` +//! from wrapping, which the keys' uniqueness and the chain's acyclicity rest on. +//! Both proof shapes stay below it: a block proof holds at most +//! `BLOCK_MAX_KECCAK` tables of at most `2^BLOCK_KECCAK_MAX_VARS` rows (2^29, +//! asserted at compile time in `block_whir`), and a STARK proof one KECCAK table +//! (`TableCounts::validate`) of at most 2^32 rows (the field's two-adicity). +//! +//! ## Column layout (907 columns) //! //! | Group | Size | Description | //! |----------------|------|------------------------------------------------| -//! | timestamp | 2 | DWordWL | -//! | addr | 8 | State address as DWordBL (8 bytes) | -//! | input_state | 200 | Input state bytes [5][5][8] | -//! | output_state | 200 | Output state bytes [5][5][8] | -//! | state_ptr | 75 | Per-lane DWordWHH addresses [25][3] | -//! | mu | 1 | Multiplicity flag | - -use executor::vm::instruction::execution::KECCAK_SYSCALL_NUMBER; +//! | timestamp | 2 | DWordWL; sponge: `(T, j)` | +//! | addr | 8 | State address as DWordBL (permute) | +//! | input_state | 200 | Permutation input bytes [5][5][8] | +//! | output_state | 200 | Permutation output bytes [5][5][8] | +//! | state_ptr | 75 | Per-lane DWordWHH addresses [25][3] (permute) | +//! | mu | 1 | Real row, `MU_P + MU_S` | +//! | mu_p, mu_s | 2 | Mode bits | +//! | first, last | 2 | First / last row of a sponge call | +//! | count | 1 | Input bytes left before this row | +//! | src, out | 4 | Block address and digest address, DWordWL | +//! | src_end | 2 | Halves of `src_lo + 135` | +//! | out_end | 2 | Halves of `out_lo + 31` | +//! | prev | 136 | Rate of the previous permutation's output | +//! | msg | 136 | Block bytes read (0 past the input) | +//! | full | 17 | Lane read as one 8-byte access | +//! | part | 119 | Byte `b < 7` of a lane read as a 1-byte access | + +use executor::vm::instruction::execution::{ + KECCAK_SYSCALL_NUMBER, KECCAK256_RATE_BYTES, KECCAK256_SYSCALL_NUMBER, +}; use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::is_wh::{IsWhKey, is_whh}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; use crate::constraints::templates::{AddOperand, INV_SHIFT_32}; @@ -47,7 +107,85 @@ pub mod cols { pub const MU: usize = STATE_PTR + 75; // 485 - pub const NUM_COLUMNS: usize = MU + 1; // 486 + // --------------------------------------------------------------------- + // Mode bits (`MU = MU_P + MU_S`) + // --------------------------------------------------------------------- + /// Permute row (`keccak_permute`). + pub const MU_P: usize = MU + 1; // 486 + /// Sponge row (`keccak256`). + pub const MU_S: usize = MU_P + 1; // 487 + + // --------------------------------------------------------------------- + // Sponge mode + // --------------------------------------------------------------------- + /// First row of a call: receives the ecall and reads x10..x12. + pub const FIRST: usize = MU_S + 1; // 488 + /// Last row of a call: writes the digest, sends no link. + pub const LAST: usize = FIRST + 1; // 489 + /// Input bytes left before this row's block. + pub const COUNT: usize = LAST + 1; // 490 + /// Address of this row's block, DWordWL. + pub const SRC_LO: usize = COUNT + 1; // 491 + pub const SRC_HI: usize = SRC_LO + 1; // 492 + /// Digest address, DWordWL. + pub const OUT_LO: usize = SRC_HI + 1; // 493 + pub const OUT_HI: usize = OUT_LO + 1; // 494 + /// `SRC_LO + 135` as two halves (the block's last byte stays below 2^32). + /// Also the next block's address: the link sends `SRC_END + 1`, so the + /// equation `MU_S·(SRC_LO + 135 − SRC_END) = 0` is what places every block + /// after the first, not only a range check. + pub const SRC_END: usize = OUT_HI + 1; // 495 + /// `OUT_LO + 31` as two halves (the digest's last byte stays below 2^32). + pub const OUT_END: usize = SRC_END + 2; // 497 + /// Rate bytes (lanes 0..17) of the previous permutation's output. + pub const PREV: usize = OUT_END + 2; // 499 + /// The block's bytes as read (0 past the input). + pub const MSG: usize = PREV + RATE; // 635 + /// `FULL[k]`: lane `k` of the block is input and read as one 8-byte access. + pub const FULL: usize = MSG + RATE; // 771 + /// `PART[k][b]` (`b < 7`): byte `b` of lane `k` is input and read alone. + pub const PART: usize = FULL + RATE_LANES; // 788 + + pub const NUM_COLUMNS: usize = PART + RATE_LANES * 7; // 907 + + /// Bytes of a block (the rate). + pub const RATE: usize = 136; + /// Lanes of a block. + pub const RATE_LANES: usize = 17; + + #[inline] + pub const fn prev(i: usize) -> usize { + PREV + i + } + + #[inline] + pub const fn msg(i: usize) -> usize { + MSG + i + } + + #[inline] + pub const fn full(lane: usize) -> usize { + FULL + lane + } + + /// `PART[lane][byte]`, `byte < 7`. + #[inline] + pub const fn part(lane: usize, byte: usize) -> usize { + PART + lane * 7 + byte + } + + /// Byte `i` (`0..200`, lane-major: lane `i / 8`, byte `i % 8`) of the + /// permutation input. Lane `L = x + 5y`, so this is `input_state(x, y, b)`. + #[inline] + pub const fn input_byte(i: usize) -> usize { + INPUT_STATE + i + } + + /// Byte `i` of the permutation output, as [`input_byte`]. + #[inline] + pub const fn output_byte(i: usize) -> usize { + OUTPUT_STATE + i + } // ------------------------------------------------------------------------- // Index helpers @@ -81,18 +219,125 @@ pub mod cols { // Operation struct // ========================================================================= +/// One KECCAK row: one permutation. +/// +/// `timestamp` is the row's Keccak-bus key as a `DWordWL`: the ecall's +/// timestamp for a permute row, `T | j << 32` for block `j` of a Keccak-256 +/// call at `T` (`TIMESTAMP_0 = T`, `TIMESTAMP_1 = j`). The round chip's rows are +/// built from the same field, so they carry the same key. #[derive(Debug, Clone)] pub struct KeccakOperation { pub timestamp: u64, + /// The state address of a permute row; 0 on a sponge row. pub state_addr: u64, pub input: [u64; 25], pub output: [u64; 25], + /// `Some` on a sponge row (block of a `keccak256` call). + pub sponge: Option, +} + +/// What a sponge row holds beyond the permutation. +#[derive(Debug, Clone)] +pub struct SpongeRow { + /// Address of this row's block (`ptr + 136·j`). + pub src: u64, + /// Digest address (`x12`). + pub out: u64, + /// Input bytes left before this block (`len − 136·j`). + pub count: u64, + pub first: bool, + pub last: bool, + /// The block's input bytes, zero past the input. + pub msg: [u8; 136], + /// The read mask `s`: `s[i]` = byte `i` of the block is input and read. All + /// 136 on a row that is not the last, the first `count` on the last. + pub mask: [bool; 136], +} + +impl SpongeRow { + /// The read mask of a block holding `taken` input bytes. + pub fn prefix(taken: usize) -> [bool; 136] { + std::array::from_fn(|i| i < taken) + } + + /// Input bytes the block reads. + pub fn taken(&self) -> usize { + self.mask.iter().filter(|&&s| s).count() + } + + /// `FULL[k]`: the whole lane is read as one access. + pub fn full(&self, lane: usize) -> bool { + self.mask[8 * lane..8 * lane + 8].iter().all(|&s| s) + } + + /// `PART[k][b]`: byte `b < 7` of lane `k` is read alone (the lane is not whole). + pub fn part(&self, lane: usize, byte: usize) -> bool { + !self.full(lane) && self.mask[8 * lane + byte] + } + + /// The padded block, the chip's linear form `m'_i = msg_i + s_{i−1} − s_i + /// + [i = 135]·0x80·last` (`s_{−1} = 1`): on a prefix mask, the input bytes, + /// then on the last row `0x01` right after them and `0x80` on byte 135. + pub fn padded(&self) -> [u8; 136] { + std::array::from_fn(|i| { + let before = if i == 0 { true } else { self.mask[i - 1] }; + let mut v = i32::from(self.msg[i]) + i32::from(before) - i32::from(self.mask[i]); + if i == cols::RATE - 1 && self.last { + v += 0x80; + } + v as u8 + }) + } +} + +impl KeccakOperation { + /// The previous permutation's output rate on a sponge row: this row's input + /// with the padded block XORed back out (zero on the first row). + pub fn sponge_prev(&self, sponge: &SpongeRow) -> [u8; 136] { + let padded = sponge.padded(); + std::array::from_fn(|i| ((self.input[i / 8] >> (8 * (i % 8))) as u8) ^ padded[i]) + } +} + +/// The KECCAK rows of one `keccak256` call at timestamp `t` over `input`, at +/// `ptr` with the digest at `out`: one per permutation, from the executor's own +/// [`executor::vm::instruction::execution::keccak256_permutations`]. +pub fn sponge_ops(t: u64, ptr: u64, out: u64, input: &[u8]) -> Vec { + use executor::vm::instruction::execution::keccak256_permutations; + let rate = KECCAK256_RATE_BYTES as usize; + let perms = keccak256_permutations(input); + let n = perms.len(); + perms + .into_iter() + .enumerate() + .map(|(j, (start, end))| { + let begin = j * rate; + let take = input.len().saturating_sub(begin).min(rate); + let mut msg = [0u8; 136]; + msg[..take].copy_from_slice(&input[begin..begin + take]); + KeccakOperation { + timestamp: t | ((j as u64) << 32), + state_addr: 0, + input: start, + output: end, + sponge: Some(SpongeRow { + src: ptr.wrapping_add((j * rate) as u64), + out, + count: (input.len() - begin) as u64, + first: j == 0, + last: j + 1 == n, + msg, + mask: SpongeRow::prefix(take), + }), + } + }) + .collect() } -/// The `IS_WH` sends of the KECCAK table: one per lane of every call, the top +/// The `IS_WH` sends of the KECCAK table: one per lane of every permute row, the top /// 48 bits of `state_ptr[lane] = state_addr + 8·lane` (`keccak:c:range_state_ptr`). pub fn is_wh_keys(ops: &[KeccakOperation]) -> impl Iterator + '_ { - ops.iter().flat_map(|op| { + ops.iter().filter(|op| op.sponge.is_none()).flat_map(|op| { (0..25u64).map(move |lane_idx| { let ptr = op .state_addr @@ -109,63 +354,107 @@ pub fn is_wh_keys(ops: &[KeccakOperation]) -> impl Iterator + '_ pub fn generate_keccak_trace( ops: &[KeccakOperation], +) -> TraceTable { + generate_keccak_trace_as(ops, TraceForm::Wide) +} + +/// The widths KECCAK traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_keccak_trace`] in `form` (`tables::gpack`). +pub fn generate_keccak_trace_as( + ops: &[KeccakOperation], + form: TraceForm, +) -> TraceTable { + generate_keccak_rows_as(ops, ops.len().next_power_of_two().max(4), form) +} + +/// Whether [`generate_keccak_rows_as`] in `form` writes its rows packed, with no +/// 64-bit table on the way: in [`TraceForm::Narrow`] once an earlier KECCAK +/// trace of this process left its widths (`tables::gpack`). +pub(crate) fn rows_written_packed(form: TraceForm) -> bool { + matches!( + super::gpack::Plan::new(form, &WIDTHS, cols::NUM_COLUMNS), + super::gpack::Plan::Write(_) + ) +} + +/// A KECCAK table of `num_rows` rows in `form`: `ops` (at most `num_rows`, a row +/// each, one permutation call a row), then the padding rows. A row reads its own op alone and +/// the padding rows are constants, so rows `[k·R, (k+1)·R)` of the whole padded +/// table are this over the ops in that range with `num_rows = R`: the block's +/// cut of the whole table, built on its own. +pub fn generate_keccak_rows_as( + ops: &[KeccakOperation], + num_rows: usize, + form: TraceForm, ) -> TraceTable { let n = ops.len(); - let num_rows = n.next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in ops.iter().enumerate() { - // Timestamp - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - - // Address as 8 bytes - table.set_dword_bl(row_idx, cols::addr(0), op.state_addr); - - // Input state as bytes - for x in 0..5 { - for y in 0..5 { - let lane = op.input[x + 5 * y]; - table.set_dword_bl(row_idx, cols::input_state(x, y, 0), lane); - } - } + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in ops.iter().enumerate() { + // Timestamp (sponge: `(T, j)`) + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - // Output state as bytes - for x in 0..5 { - for y in 0..5 { - let lane = op.output[x + 5 * y]; - table.set_dword_bl(row_idx, cols::output_state(x, y, 0), lane); + // Input and output state as bytes + for x in 0..5 { + for y in 0..5 { + table.set_dword_bl(row_idx, cols::input_state(x, y, 0), op.input[x + 5 * y]); + table.set_dword_bl(row_idx, cols::output_state(x, y, 0), op.output[x + 5 * y]); + } } - } - - // State pointers: state_ptr[lane] = addr + 8 * lane_idx - for lane_idx in 0..25 { - let ptr = op - .state_addr - .checked_add(lane_idx as u64 * 8) - .expect("keccak state address range must be validated by the executor"); - table.set_dword_whh(row_idx, cols::state_ptr(lane_idx, 0), ptr); - } + table.set_fe(row_idx, cols::MU, FE::one()); - // mu = 1 (real row) - table.set_fe(row_idx, cols::MU, FE::one()); - } + let Some(sponge) = &op.sponge else { + // Permute row. + table.set_dword_bl(row_idx, cols::addr(0), op.state_addr); + // State pointers: state_ptr[lane] = addr + 8 * lane_idx + for lane_idx in 0..25 { + let ptr = op + .state_addr + .checked_add(lane_idx as u64 * 8) + .expect("keccak state address range must be validated by the executor"); + table.set_dword_whh(row_idx, cols::state_ptr(lane_idx, 0), ptr); + } + table.set_fe(row_idx, cols::MU_P, FE::one()); + continue; + }; - // Padding rows: state_ptr[lane][0] = 8 * lane_idx (per spec keccak.toml pad). - // half_mid and word_hi stay zero since 8*24 = 192 fits in half_lo. - // mu = 0 gates all bus interactions and the ADD constraint, so these values - // only need to satisfy the pad requirement, not reconstruct a real address. - for row_idx in n..num_rows { - for lane_idx in 0..25 { - table.set_u64(row_idx, cols::state_ptr(lane_idx, 0), (lane_idx as u64) * 8); + // Sponge row. `ADDR` and `STATE_PTR` stay 0: their constraints and + // sends are gated by `MU_P`. + table.set_fe(row_idx, cols::MU_S, FE::one()); + table.set_bool(row_idx, cols::FIRST, sponge.first); + table.set_bool(row_idx, cols::LAST, sponge.last); + table.set_u64(row_idx, cols::COUNT, sponge.count); + table.set_dword_wl(row_idx, cols::SRC_LO, sponge.src); + table.set_dword_wl(row_idx, cols::OUT_LO, sponge.out); + let src_end = (sponge.src & 0xFFFF_FFFF) + 135; + table.set_u64(row_idx, cols::SRC_END, src_end & 0xFFFF); + table.set_u64(row_idx, cols::SRC_END + 1, src_end >> 16); + if sponge.last { + let out_end = (sponge.out & 0xFFFF_FFFF) + 31; + table.set_u64(row_idx, cols::OUT_END, out_end & 0xFFFF); + table.set_u64(row_idx, cols::OUT_END + 1, out_end >> 16); + } + table.set_bytes(row_idx, cols::PREV, &op.sponge_prev(sponge)); + table.set_bytes(row_idx, cols::MSG, &sponge.msg); + for lane in 0..cols::RATE_LANES { + table.set_bool(row_idx, cols::full(lane), sponge.full(lane)); + for byte in 0..7 { + table.set_bool(row_idx, cols::part(lane, byte), sponge.part(lane, byte)); + } + } } - } - trace + // Padding rows: state_ptr[lane][0] = 8 * lane_idx (per spec keccak.toml pad). + // half_mid and word_hi stay zero since 8*24 = 192 fits in half_lo. + // mu = 0 gates all bus interactions and the ADD constraint, so these values + // only need to satisfy the pad requirement, not reconstruct a real address. + for row_idx in n..num_rows { + for lane_idx in 0..25 { + table.set_u64(row_idx, cols::state_ptr(lane_idx, 0), (lane_idx as u64) * 8); + } + } + }) } // ========================================================================= @@ -175,14 +464,14 @@ pub fn generate_keccak_trace( pub fn bus_interactions() -> Vec { let syscall_lo = KECCAK_SYSCALL_NUMBER & 0xFFFF_FFFF; let syscall_hi = KECCAK_SYSCALL_NUMBER >> 32; - let mut interactions = Vec::with_capacity(84); + let mut interactions = Vec::with_capacity(84 + SPONGE_INTERACTIONS); // 1. ECALL receiver (shared bus, per spec keccak:c:output) // Payload: [ts_lo, ts_hi, syscall_lo32, syscall_hi32] in DWordWL [lo, hi] // ordering, matching the CPU ECALL sender shared with HALT/COPY/ECSM/HINT. interactions.push(BusInteraction::receiver( BusId::Ecall, - Multiplicity::Column(cols::MU), + Multiplicity::Column(cols::MU_P), vec![ BusValue::Packed { start_column: cols::TIMESTAMP_0, @@ -271,7 +560,7 @@ pub fn bus_interactions() -> Vec { values.push(BusValue::constant(0)); interactions.push(BusInteraction::sender( BusId::Memw, - Multiplicity::Column(cols::MU), + Multiplicity::Column(cols::MU_P), values, )); } @@ -304,7 +593,7 @@ pub fn bus_interactions() -> Vec { } interactions.push(BusInteraction::sender( BusId::Keccak, - Multiplicity::Column(cols::MU), + round_chip_tokens(), values, )); } @@ -334,25 +623,25 @@ pub fn bus_interactions() -> Vec { } interactions.push(BusInteraction::receiver( BusId::Keccak, - Multiplicity::Column(cols::MU), + round_chip_tokens(), values, )); } // 4. IS_WHH range checks on state_ptr (keccak:c:range_state_ptr): IS_WH on the - // top 48 bits and IS_HALF on half_lo, 50 interactions. μ is a bit (idx 51 below), - // as the template's `cond: Bit` requires. + // top 48 bits and IS_HALF on half_lo, 50 interactions. `MU_P` is a bit, as the + // template's `cond: Bit` requires. for lane_idx in 0..25 { interactions.extend(is_whh( cols::state_ptr(lane_idx, 0), - Multiplicity::Column(cols::MU), + Multiplicity::Column(cols::MU_P), )); } // 5. Alignment: addr[0] & 7 = 0, which enforces addr % 8 == 0. interactions.push(BusInteraction::sender( BusId::ByteAlu, - Multiplicity::Column(cols::MU), + Multiplicity::Column(cols::MU_P), vec![ BusValue::constant(alu_op::AND as u64), BusValue::Packed { @@ -375,7 +664,7 @@ pub fn bus_interactions() -> Vec { for i in 0..4 { interactions.push(BusInteraction::sender( BusId::AreBytes, - Multiplicity::Column(cols::MU), + Multiplicity::Column(cols::MU_P), vec![ BusValue::Packed { start_column: cols::addr(2 * i), @@ -448,50 +737,518 @@ pub fn bus_interactions() -> Vec { interactions.push(BusInteraction::sender( BusId::Memw, - Multiplicity::Column(cols::MU), + Multiplicity::Column(cols::MU_P), values, )); } + interactions.extend(sponge_bus_interactions()); interactions } +/// The most KECCAK rows one proof may hold for the sponge mode's counting +/// argument (module docs): with `N` rows, `len − 136·J` stays in +/// `(−136·N, 2^32)`, which must not reach `p ≈ 2^64`; and the block index must +/// not wrap. 2^40 leaves a margin of 2^16 over what either needs. A cap that +/// lets one proof hold more KECCAK rows has to stay under this; `block_whir` +/// asserts its own at compile time. +pub const SPONGE_MAX_ROWS: u128 = 1 << 40; + +/// Interactions of the sponge mode, after the 84 of the permute mode. +pub const SPONGE_INTERACTIONS: usize = 1 + 3 + 2 + RATE_READS + cols::RATE + 4 + 4; + +/// Memory reads of a block: 17 lane reads and 7 byte reads per lane. +const RATE_READS: usize = cols::RATE_LANES * 8; + +fn direct(column: usize) -> BusValue { + BusValue::Packed { + start_column: column, + packing: Packing::Direct, + } +} + +fn column_term(column: usize, coefficient: i64) -> LinearTerm { + LinearTerm::Column { + coefficient, + column, + } +} + +/// `column + constant`. +fn offset(column: usize, constant: i64) -> BusValue { + BusValue::linear(vec![column_term(column, 1), LinearTerm::Constant(constant)]) +} + +/// The read mask `s_i` as linear terms with `coefficient` (`s_{8k+b} = +/// FULL_k + PART_{k,b}` for `b < 7`, `s_{8k+7} = FULL_k`). `i = -1` is the +/// constant 1. +fn mask_terms(i: isize, coefficient: i64) -> Vec { + if i < 0 { + return vec![LinearTerm::Constant(coefficient)]; + } + let i = i as usize; + let (lane, byte) = (i / 8, i % 8); + let mut terms = vec![column_term(cols::full(lane), coefficient)]; + if byte < 7 { + terms.push(column_term(cols::part(lane, byte), coefficient)); + } + terms +} + +/// The padded message byte `m'_i = MSG_i + (s_{i-1} − s_i) + [i = 135]·0x80·LAST`. +pub(crate) fn padded_byte(i: usize) -> BusValue { + let mut terms = vec![column_term(cols::msg(i), 1)]; + terms.extend(mask_terms(i as isize - 1, 1)); + terms.extend(mask_terms(i as isize, -1)); + if i == cols::RATE - 1 { + terms.push(column_term(cols::LAST, 0x80)); + } + BusValue::linear(terms) +} + +/// A MEMW register read of `x{reg}` at `(T, 0)` binding `(lo, hi)`. +fn sponge_register_read(reg: u64, lo: BusValue, hi: BusValue) -> Vec { + let zeros = || std::iter::repeat_n(BusValue::constant(0), 6); + let mut v = Vec::with_capacity(24); + v.push(lo.clone()); + v.push(hi.clone()); + v.extend(zeros()); // old[2..8] + v.push(BusValue::constant(1)); // is_register + v.push(BusValue::constant(2 * reg)); + v.push(BusValue::constant(0)); + v.push(lo); + v.push(hi); + v.extend(zeros()); // value[2..8] + v.push(direct(cols::TIMESTAMP_0)); + v.push(BusValue::constant(0)); + v.push(BusValue::constant(1)); // w2: a register + v.push(BusValue::constant(0)); + v.push(BusValue::constant(0)); + v +} + +/// A MEMW memory read at `(T, 0)` of `bytes` (1 or 8 of them) at `address_lo` +/// (`SRC_LO + offset`) and `SRC_HI`: old = value = the bytes. +fn sponge_memory_read(offset_lo: i64, bytes: &[usize]) -> Vec { + let value = || { + let mut v: Vec = bytes.iter().map(|&c| direct(c)).collect(); + v.resize_with(8, || BusValue::constant(0)); + v + }; + let mut v = Vec::with_capacity(24); + v.extend(value()); // old + v.push(BusValue::constant(0)); // is_register + v.push(offset(cols::SRC_LO, offset_lo)); + v.push(direct(cols::SRC_HI)); + v.extend(value()); // value == old + v.push(direct(cols::TIMESTAMP_0)); + v.push(BusValue::constant(0)); + v.push(BusValue::constant(0)); // w2 + v.push(BusValue::constant(0)); // w4 + v.push(BusValue::constant(u64::from(bytes.len() == 8))); // w8 + v +} + +/// The multiplicity of the two Keccak-bus tokens (the round chip's request and +/// reply): the row's mode bits, `MU_P + MU_S`. A row of either mode asks the +/// round chip because of its mode bit itself, so its output is a permutation of +/// its input whatever `μ` holds; idx 56 (`μ = MU_P + MU_S`) then only keeps `μ` +/// consistent for the readers that use it (the cuts). With the tokens on `μ`, +/// idx 56 alone would tie a mode to the round chip, and a row with `μ = 0` but a +/// mode bit set would write a free digest (`keccak256_tests:: +/// the_mode_bits_put_a_row_on_the_round_chip`). `SpongeCheck::TokensOnModes` +/// puts them back on `μ` in a test. +fn round_chip_tokens() -> Multiplicity { + if dropped(SpongeCheck::TokensOnModes) { + Multiplicity::Column(cols::MU) + } else { + Multiplicity::Sum(cols::MU_P, cols::MU_S) + } +} + +/// The digest address on the link between rows (`SpongeCheck::LinkOut` leaves it +/// out in a test). +fn link_out() -> [BusValue; 2] { + if dropped(SpongeCheck::LinkOut) { + [BusValue::constant(0), BusValue::constant(0)] + } else { + [direct(cols::OUT_LO), direct(cols::OUT_HI)] + } +} + +/// The sponge mode's checks by what they stop, so a test can leave one out and +/// show the forgery it stops then passes (`keccak256_tests`). Outside the tests +/// [`dropped`] is always `false`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SpongeCheck { + /// `μ = MU_P + MU_S` (idx 56). With the Keccak-bus tokens on the mode bits + /// it is defensive; with them on `μ` it alone ties a mode to the round chip. + ModeSum, + /// `(FIRST + LAST)(1 − MU_S) = 0` (idx 57). + FlagsOnSponge, + /// `FULL`, `PART` bits. + MaskBits, + /// `FULL_k · PART_{k,b} = 0`. + OneRead, + /// The mask is a prefix. + Prefix, + /// No read off a sponge row. + ReadsOnSponge, + /// A row that is not last reads a whole block. + FullUnlessLast, + /// The last block holds at most 135 bytes. + LastNotFull, + /// The last block holds exactly `COUNT` bytes. + CountOnLast, + /// Bytes past the input are 0. + MsgZero, + /// The first row is block 0. + FirstBlock, + /// The first row starts from the zero state. + FirstState, + /// `SRC_END` and `OUT_END` hold the addresses' ends; `SRC_END + 1` is also + /// the next block's address on the link. + AddressHalves, + /// The digest address rides the link between rows. + LinkOut, + /// Lane 0's 8-byte memory read. + LaneRead, + /// The Keccak-bus tokens on the mode bits `MU_P + MU_S` rather than on `μ` + /// ([`round_chip_tokens`]). + TokensOnModes, +} + +#[cfg(test)] +thread_local! { + /// The checks a test leaves out ([`SpongeCheck`]), one bit each; set + /// through [`leave_out`]. + pub(crate) static DROPPED: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// A test's forgery of a call's rows before the walk emits their memory + /// traffic (`trace_builder::collect_keccak256_ops`). + #[allow(clippy::type_complexity)] + pub(crate) static FORGE: std::cell::RefCell)>>> = + const { std::cell::RefCell::new(None) }; +} + +/// Whether a test left `check` out; never outside the tests. +#[inline(always)] +pub(crate) fn dropped(check: SpongeCheck) -> bool { + #[cfg(test)] + { + let bits = DROPPED.with(|d| d.get()) + | DROPPED_EVERYWHERE.load(std::sync::atomic::Ordering::Relaxed); + bits & (1 << check as u32) != 0 + } + #[cfg(not(test))] + { + let _ = check; + false + } +} + +/// Leaves `checks` out on this thread until the next call (an empty slice puts +/// every check back). +#[cfg(test)] +pub(crate) fn leave_out(checks: &[SpongeCheck]) { + let bits = checks.iter().fold(0u32, |a, c| a | (1 << *c as u32)); + DROPPED.with(|d| d.set(bits)); +} + +/// The checks left out on EVERY thread of the process, one bit each: the box +/// test's real prove and verify, whose table workers are plain scoped threads +/// that no thread-local reaches. Set only through [`LeftOutEverywhere`], by a +/// test that runs alone in its process (it would loosen any other KECCAK proof +/// made meanwhile). +#[cfg(test)] +static DROPPED_EVERYWHERE: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + +/// Leaves checks out process-wide while it lives; dropping it (also on a +/// panic) puts every check back. +#[cfg(test)] +pub(crate) struct LeftOutEverywhere; + +#[cfg(test)] +impl LeftOutEverywhere { + pub(crate) fn new(checks: &[SpongeCheck]) -> Self { + let bits = checks.iter().fold(0u32, |a, c| a | (1 << *c as u32)); + DROPPED_EVERYWHERE.store(bits, std::sync::atomic::Ordering::SeqCst); + LeftOutEverywhere + } +} + +#[cfg(test)] +impl Drop for LeftOutEverywhere { + fn drop(&mut self) { + DROPPED_EVERYWHERE.store(0, std::sync::atomic::Ordering::SeqCst); + } +} + +/// A test's forgery of one call's rows (see [`FORGE`]). +#[cfg(test)] +pub(crate) fn forge(rows: &mut Vec) { + FORGE.with(|f| { + if let Some(f) = f.borrow().as_ref() { + f(rows) + } + }); +} + +/// Recomputes a call's permutations from its rows' masks and messages, starting +/// from `start`: a test's forgery made consistent with the round chip. +#[cfg(test)] +pub(crate) fn rechain(rows: &mut [KeccakOperation], start: [u64; 25]) { + use executor::vm::instruction::execution::keccak_f1600; + let mut state = start; + for row in rows { + let padded = row.sponge.as_ref().expect("a sponge row").padded(); + for (lane, chunk) in state.iter_mut().zip(padded.chunks_exact(8)) { + *lane ^= u64::from_le_bytes(chunk.try_into().expect("8 bytes")); + } + row.input = state; + keccak_f1600(&mut state); + row.output = state; + } +} + +/// The sponge mode's interactions (see the module docs). +/// +/// - Ecall receive (`FIRST`): `[T, 0, KECCAK256 lo, hi]`. +/// - x10, x11, x12 read at `(T, 0)` (`FIRST`): `(SRC_LO, SRC_HI)`, `(COUNT, 0)`, +/// `(OUT_LO, OUT_HI)`. The 0 high word of x11 bounds the length below 2^32. +/// - KeccakSponge send (`MU_S − LAST`) and receive (`MU_S − FIRST`). +/// - 17 lane reads (`FULL_k`) and 119 byte reads (`PART_{k,b}`) at `(T, 0)`. +/// - 136 XORs (`MU_S`): `INPUT_STATE_i = PREV_i ⊕ m'_i` over the rate. +/// - IS_HALF on `SRC_END` (`MU_S`) and `OUT_END` (`LAST`). +/// - The digest: lanes 0..4 of `OUTPUT_STATE` written at `OUT + 8k`, `(T + 1, 0)` +/// (`LAST`). +pub fn sponge_bus_interactions() -> Vec { + let mut out = Vec::with_capacity(SPONGE_INTERACTIONS); + let first = || Multiplicity::Column(cols::FIRST); + let last = || Multiplicity::Column(cols::LAST); + let mu_s = || Multiplicity::Column(cols::MU_S); + + // Ecall receive on the call's first row. + out.push(BusInteraction::receiver( + BusId::Ecall, + first(), + vec![ + direct(cols::TIMESTAMP_0), + BusValue::constant(0), + BusValue::constant(KECCAK256_SYSCALL_NUMBER & 0xFFFF_FFFF), + BusValue::constant(KECCAK256_SYSCALL_NUMBER >> 32), + ], + )); + + // The operands, read on the first row. + for (reg, lo, hi) in [ + (10, direct(cols::SRC_LO), direct(cols::SRC_HI)), + (11, direct(cols::COUNT), BusValue::constant(0)), + (12, direct(cols::OUT_LO), direct(cols::OUT_HI)), + ] { + out.push(BusInteraction::sender( + BusId::Memw, + first(), + sponge_register_read(reg, lo, hi), + )); + } + + // The link to the next row: (T, j + 1, COUNT − 136, SRC_END + 1, SRC_HI, + // OUT_LO, OUT_HI, OUTPUT_STATE). The next block's address is `SRC_END + 1`, + // which is `SRC_LO + 136` only through the equation on `SRC_END` (idx + // `AddressHalves`): without it a block could be read from anywhere + // (`keccak256_tests::the_block_end_defines_the_next_block_address`). + { + let mut v = Vec::with_capacity(7 + 200); + v.push(direct(cols::TIMESTAMP_0)); + v.push(offset(cols::TIMESTAMP_1, 1)); + v.push(offset(cols::COUNT, -(cols::RATE as i64))); + v.push(BusValue::linear(vec![ + column_term(cols::SRC_END, 1), + column_term(cols::SRC_END + 1, 1 << 16), + LinearTerm::Constant(1), + ])); + v.push(direct(cols::SRC_HI)); + v.extend(link_out()); + v.extend((0..200).map(|i| direct(cols::output_byte(i)))); + out.push(BusInteraction::sender( + BusId::KeccakSponge, + Multiplicity::Diff(cols::MU_S, cols::LAST), + v, + )); + } + // ... and from the previous one: the previous output's rate into PREV, its + // capacity straight into this permutation's input. + { + let mut v = Vec::with_capacity(7 + 200); + v.push(direct(cols::TIMESTAMP_0)); + v.push(direct(cols::TIMESTAMP_1)); + v.push(direct(cols::COUNT)); + v.push(direct(cols::SRC_LO)); + v.push(direct(cols::SRC_HI)); + v.extend(link_out()); + v.extend((0..cols::RATE).map(|i| direct(cols::prev(i)))); + v.extend((cols::RATE..200).map(|i| direct(cols::input_byte(i)))); + out.push(BusInteraction::receiver( + BusId::KeccakSponge, + Multiplicity::Diff(cols::MU_S, cols::FIRST), + v, + )); + } + + // The block's reads: a whole lane at once, or the bytes of the lane the + // input ends inside one at a time. Byte 7 of a lane is never read alone. + for lane in 0..cols::RATE_LANES { + let bytes: Vec = (0..8).map(|b| cols::msg(8 * lane + b)).collect(); + // A test leaves lane 0's read out with `SpongeCheck::LaneRead` (a zero + // multiplicity, so the interaction count stays). + let multiplicity = if lane == 0 && dropped(SpongeCheck::LaneRead) { + Multiplicity::Linear(vec![LinearTerm::Constant(0)]) + } else { + Multiplicity::Column(cols::full(lane)) + }; + out.push(BusInteraction::sender( + BusId::Memw, + multiplicity, + sponge_memory_read(8 * lane as i64, &bytes), + )); + for byte in 0..7 { + out.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::part(lane, byte)), + sponge_memory_read((8 * lane + byte) as i64, &[cols::msg(8 * lane + byte)]), + )); + } + } + + // The absorb: INPUT_STATE_i = PREV_i XOR m'_i over the rate. + for i in 0..cols::RATE { + out.push(BusInteraction::sender( + BusId::ByteAlu, + mu_s(), + vec![ + BusValue::constant(alu_op::XOR as u64), + direct(cols::prev(i)), + padded_byte(i), + direct(cols::input_byte(i)), + ], + )); + } + + // Address range checks: the block's last byte and the digest's last byte + // stay below 2^32 in the low address word. + for half in 0..2 { + out.push(BusInteraction::sender( + BusId::IsHalfword, + mu_s(), + vec![direct(cols::SRC_END + half)], + )); + } + for half in 0..2 { + out.push(BusInteraction::sender( + BusId::IsHalfword, + last(), + vec![direct(cols::OUT_END + half)], + )); + } + + // The digest: lanes 0..4 of this row's output, at T + 1. + for lane in 0..4 { + let mut v = Vec::with_capacity(16); + v.push(BusValue::constant(0)); // is_register + v.push(offset(cols::OUT_LO, 8 * lane as i64)); + v.push(direct(cols::OUT_HI)); + v.extend((0..8).map(|b| direct(cols::output_byte(8 * lane + b)))); + v.push(offset(cols::TIMESTAMP_0, 1)); + v.push(BusValue::constant(0)); + v.push(BusValue::constant(0)); // w2 + v.push(BusValue::constant(0)); // w4 + v.push(BusValue::constant(1)); // w8 + out.push(BusInteraction::sender(BusId::Memw, last(), v)); + } + + debug_assert_eq!(out.len(), SPONGE_INTERACTIONS); + out +} + // ========================================================================= // Single-source constraint set (ConstraintBuilder front-end) // ========================================================================= -/// The KECCAK core table's 52 transition constraints as a single [`ConstraintSet`]: -/// - idx 0-49: for `lane_idx ∈ 0..25`, the `ADD` carry pair (gated on `μ`) +/// The KECCAK core table's transition constraints as a single [`ConstraintSet`]. +/// +/// Permute mode (as before the sponge mode, now gated on `MU_P`): +/// - idx 0-49: for `lane_idx ∈ 0..25`, the `ADD` carry pair (gated on `MU_P`) /// enforcing `state_ptr[lane] = addr + 8·lane_idx` (`addr` DWordBL, /// `state_ptr` DWordWHH); -/// - idx 50: `μ · carry_1 = 0` (top-lane no-overflow), where `carry_1` is the -/// high carry of `addr + 192 = state_ptr[24]`; +/// - idx 50: `MU_P · carry_1 = 0` (top-lane no-overflow), where `carry_1` is +/// the high carry of `addr + 192 = state_ptr[24]`; /// - idx 51: `μ · (1 − μ) = 0` (MuIsBit). +/// +/// Modes (idx 52-57): `MU_P`, `MU_S`, `FIRST`, `LAST` bits; `μ = MU_P + MU_S` +/// (with idx 51 a row is in at most one mode); `FIRST` and `LAST` only on +/// sponge rows. Every multiplicity of the permute mode is `MU_P` and every one +/// of the sponge mode is `FIRST`, `LAST`, `MU_S` (or a difference of these with +/// `MU_S`) or a mask bit that idx 58+ forces to 0 off a sponge row, so a row +/// sends nothing of the mode it is not in. +/// +/// Sponge mode, from idx 58 (see [`SPONGE_CONSTRAINTS`] for the count): +/// - `FULL_k`, `PART_{k,b}` bits; `FULL_k · PART_{k,b} = 0` (each `s_i` a bit); +/// - prefix: `(1 − s_i) · s_{i+1} = 0`; +/// - no read off a sponge row: `(1 − MU_S) · s_0 = 0`; +/// - a row that is not last reads a whole block: `(MU_S − LAST)(1 − FULL_16) = 0`; +/// - the last block holds at most 135 input bytes: `LAST · FULL_16 = 0`; +/// - the last block holds exactly `COUNT` bytes: `LAST · (COUNT − Σ s_i) = 0`; +/// - bytes past the input are 0: `(1 − s_i) · MSG_i = 0`; +/// - the first row starts from the zero state at block 0: +/// `FIRST · TIMESTAMP_1 = 0`, `FIRST · PREV_i = 0`, `FIRST · INPUT_i = 0` on +/// the 64 capacity bytes; +/// - `MU_S · (SRC_LO + 135 − SRC_END_0 − 2^16 SRC_END_1) = 0`, which both +/// bounds the block below 2^32 and, through the link's `SRC_END + 1`, places +/// the next block right after this one; +/// `LAST · (OUT_LO + 31 − OUT_END_0 − 2^16 OUT_END_1) = 0`. #[derive(Clone, Copy)] pub struct KeccakConstraints; +/// Constraints of the permute mode and the mode bits (idx 0-57). +const MODE_CONSTRAINTS: usize = 58; + +/// Constraints of the sponge mode (idx 58..). +pub const SPONGE_CONSTRAINTS: usize = cols::RATE_LANES * 8 // FULL, PART bits + + cols::RATE_LANES * 7 // FULL · PART + + (cols::RATE - 1) // prefix + + 1 // no read off a sponge row + + 1 // full block unless last + + 1 // last block < 136 + + 1 // count on the last row + + cols::RATE // MSG zero past the input + + 1 + cols::RATE + (200 - cols::RATE) // first row: j, PREV, capacity + + 2; // address halves + +/// All of KECCAK's constraints. +pub const NUM_CONSTRAINTS: usize = MODE_CONSTRAINTS + SPONGE_CONSTRAINTS; + impl ConstraintSet for KeccakConstraints { fn max_degree(&self) -> usize { 3 } fn eval>(&self, b: &mut B) { - use crate::constraints::templates::emit_add_pair; + use crate::constraints::templates::{emit_add_pair, emit_is_bit}; - // idx 0-49: state_ptr[lane] = addr + 8*lane_idx (gated on μ). + // idx 0-49: state_ptr[lane] = addr + 8*lane_idx (gated on MU_P). for lane_idx in 0..25 { let offset = (lane_idx * 8) as i64; emit_add_pair( b, lane_idx * 2, - &[cols::MU], + &[cols::MU_P], &AddOperand::from_dword_bl(cols::ADDR), &AddOperand::constant(offset), &AddOperand::from_dword_whh(cols::state_ptr(lane_idx, 0)), ); } - // idx 50: μ · carry_1 (top-lane no-overflow). + // idx 50: MU_P · carry_1 (top-lane no-overflow). let c256 = b.const_base(256); let c65536 = b.const_base(65536); let c16777216 = b.const_base(16777216); @@ -510,16 +1267,131 @@ impl ConstraintSet for KeccakConstraints { let c192 = b.const_base(192); let carry_0 = (addr_lo + c192 - ptr_lo) * inv_2_32.clone(); let carry_1 = (addr_hi + carry_0 - ptr_hi) * inv_2_32; - let mu = b.main(0, cols::MU); - b.emit_base(50, mu * carry_1); + let mu_p = b.main(0, cols::MU_P); + b.emit_base(50, mu_p * carry_1); - // idx 51: MuIsBit — μ·(1−μ). Every other constraint and bus send is - // weighted by μ, so μ ≠ 0 is indistinguishable from μ = 1 at the + // idx 51: MuIsBit — μ·(1−μ). Every bus send is weighted by μ or by a + // mode's own bits, so μ ≠ 0 is indistinguishable from μ = 1 at the // constraint level, while on the bus a μ = −1 row inverts a range check - // (carrier). KECCAK does not deduplicate (one row per hash, μ ∈ {0,1}), - // so the bit constraint is honest and forbids μ = −1. + // (carrier). KECCAK does not deduplicate (one row per permutation, μ ∈ + // {0,1}), so the bit constraint is honest and forbids μ = −1. let mu = b.main(0, cols::MU); let one = b.one(); - b.emit_base(51, mu.clone() * (one - mu)); + b.emit_base(51, mu.clone() * (one.clone() - mu.clone())); + + // idx 52-55: the mode and boundary bits. + emit_is_bit(b, 52, cols::MU_P, None); + emit_is_bit(b, 53, cols::MU_S, None); + emit_is_bit(b, 54, cols::FIRST, None); + emit_is_bit(b, 55, cols::LAST, None); + // idx 56: μ = MU_P + MU_S. With μ, MU_P, MU_S bits, at most one mode. + let mu_p = b.main(0, cols::MU_P); + let mu_s = b.main(0, cols::MU_S); + let e = if dropped(SpongeCheck::ModeSum) { + b.zero() + } else { + mu - mu_p - mu_s.clone() + }; + b.emit_base(56, e); + // idx 57: FIRST and LAST only on sponge rows: (FIRST + LAST)(1 − MU_S). + // Both are bits and both products are ≥ 0 in the integers, so the sum + // vanishing over the bits {0,1} means each does (2·1 ≠ 0 in the field). + let first = b.main(0, cols::FIRST); + let last = b.main(0, cols::LAST); + let e = if dropped(SpongeCheck::FlagsOnSponge) { + b.zero() + } else { + (first.clone() + last.clone()) * (one.clone() - mu_s.clone()) + }; + b.emit_base(57, e); + + // ---------------- sponge mode ---------------- + let mut idx = MODE_CONSTRAINTS; + // Every sponge constraint names its check, so a test can leave one out + // (`SpongeCheck`); outside the tests none is ever left out. + let mut emit = |b: &mut B, check: SpongeCheck, e: B::Expr| { + let e = if dropped(check) { b.zero() } else { e }; + b.emit_base(idx, e); + idx += 1; + }; + + // FULL_k, PART_{k,b} bits. + for lane in 0..cols::RATE_LANES { + let f = b.main(0, cols::full(lane)); + let e = f.clone() * (b.one() - f); + emit(b, SpongeCheck::MaskBits, e); + for byte in 0..7 { + let p = b.main(0, cols::part(lane, byte)); + let e = p.clone() * (b.one() - p); + emit(b, SpongeCheck::MaskBits, e); + } + } + // FULL_k · PART_{k,b} = 0: each s_i is a bit. + for lane in 0..cols::RATE_LANES { + for byte in 0..7 { + let f = b.main(0, cols::full(lane)); + let p = b.main(0, cols::part(lane, byte)); + emit(b, SpongeCheck::OneRead, f * p); + } + } + // s_i as an expression. + let s = |b: &B, i: usize| -> B::Expr { + let (lane, byte) = (i / 8, i % 8); + let f = b.main(0, cols::full(lane)); + if byte < 7 { + f + b.main(0, cols::part(lane, byte)) + } else { + f + } + }; + // Prefix: (1 − s_i) · s_{i+1} = 0. + for i in 0..cols::RATE - 1 { + let e = (b.one() - s(b, i)) * s(b, i + 1); + emit(b, SpongeCheck::Prefix, e); + } + // No read off a sponge row: (1 − MU_S) · s_0 = 0. + let e = (b.one() - mu_s.clone()) * s(b, 0); + emit(b, SpongeCheck::ReadsOnSponge, e); + // A row that is not last reads a whole block: (MU_S − LAST)(1 − FULL_16). + let full_last = b.main(0, cols::full(cols::RATE_LANES - 1)); + let e = (mu_s.clone() - last.clone()) * (b.one() - full_last.clone()); + emit(b, SpongeCheck::FullUnlessLast, e); + // The last block holds at most 135 input bytes: LAST · FULL_16 = 0. + emit(b, SpongeCheck::LastNotFull, last.clone() * full_last); + // The last block holds exactly COUNT bytes: LAST · (COUNT − Σ s_i) = 0. + let mut taken = b.zero(); + for i in 0..cols::RATE { + taken = taken + s(b, i); + } + let count = b.main(0, cols::COUNT); + emit(b, SpongeCheck::CountOnLast, last.clone() * (count - taken)); + // Bytes past the input are 0: (1 − s_i) · MSG_i = 0. + for i in 0..cols::RATE { + let e = (b.one() - s(b, i)) * b.main(0, cols::msg(i)); + emit(b, SpongeCheck::MsgZero, e); + } + // The first row is block 0 and starts from the zero state. + let e = first.clone() * b.main(0, cols::TIMESTAMP_1); + emit(b, SpongeCheck::FirstBlock, e); + for i in 0..cols::RATE { + let e = first.clone() * b.main(0, cols::prev(i)); + emit(b, SpongeCheck::FirstState, e); + } + for i in cols::RATE..200 { + let e = first.clone() * b.main(0, cols::input_byte(i)); + emit(b, SpongeCheck::FirstState, e); + } + // SRC_LO + 135 and OUT_LO + 31 as halves (each half range-checked). + // SRC_END + 1 is also the next block's address on the link, so this + // equation is load-bearing at any address, not only near 2^32. + let c65536 = b.const_base(1 << 16); + let src_end = b.main(0, cols::SRC_END) + b.main(0, cols::SRC_END + 1) * c65536.clone(); + let e = mu_s * (b.main(0, cols::SRC_LO) + b.const_base(135) - src_end); + emit(b, SpongeCheck::AddressHalves, e); + let out_end = b.main(0, cols::OUT_END) + b.main(0, cols::OUT_END + 1) * c65536; + let e = last * (b.main(0, cols::OUT_LO) + b.const_base(31) - out_end); + emit(b, SpongeCheck::AddressHalves, e); + + debug_assert_eq!(idx, NUM_CONSTRAINTS); } } diff --git a/prover/src/tables/keccak_rc.rs b/prover/src/tables/keccak_rc.rs index 142b5bdde..d3e32601a 100644 --- a/prover/src/tables/keccak_rc.rs +++ b/prover/src/tables/keccak_rc.rs @@ -9,8 +9,8 @@ //! `ProofOptions` not covered by the static table). use math::polynomial::Polynomial; -use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed}; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; use stark::proof::options::ProofOptions; use stark::prover::evaluate_polynomial_on_lde_domain; @@ -97,31 +97,65 @@ pub const fn generate_row(round: usize) -> [u64; NUM_PRECOMPUTED_COLS] { fn static_commitment(blowup_factor: u8) -> Option { match blowup_factor { 2 => Some([ - 0xab, 0x7a, 0xad, 0xf5, 0xbf, 0xa2, 0xd5, 0x5c, 0x29, 0x83, 0x83, 0xe6, 0x2e, 0x47, - 0xa0, 0xa5, 0x22, 0xf9, 0x57, 0x89, 0x5a, 0x5c, 0xbb, 0x1f, 0x34, 0xbc, 0x21, 0x72, - 0xa9, 0x2c, 0x85, 0xe3, + 0x0a, 0x3c, 0x86, 0xe2, 0xd3, 0x37, 0x98, 0xf2, 0x68, 0x45, 0x42, 0xbf, 0x0b, 0x3d, + 0x52, 0x02, 0x4b, 0x08, 0x8b, 0x7a, 0x4b, 0xc8, 0xb0, 0x2a, 0xe4, 0x14, 0x86, 0x09, + 0x35, 0xdc, 0xa6, 0x7c, ]), 4 => Some([ - 0xfb, 0x42, 0x58, 0x76, 0xf4, 0x30, 0x98, 0x04, 0xef, 0x8c, 0x4e, 0x65, 0xf7, 0x1a, - 0x29, 0x03, 0xd2, 0xc6, 0x12, 0x0d, 0x18, 0xe1, 0x28, 0x6e, 0x70, 0xeb, 0xa8, 0x11, - 0x3c, 0x3e, 0xe1, 0xdd, + 0xe9, 0xd8, 0x8a, 0xfe, 0x76, 0xf2, 0x68, 0x30, 0x55, 0xf6, 0xe7, 0x7c, 0x5e, 0x21, + 0x91, 0x7e, 0xa6, 0x59, 0x3b, 0xb1, 0x4e, 0xaa, 0xc0, 0x29, 0xf5, 0x00, 0xc0, 0x13, + 0xca, 0xc6, 0x6d, 0x10, ]), 8 => Some([ - 0x9f, 0x91, 0xaf, 0xb6, 0x5b, 0x75, 0x1e, 0xfb, 0x73, 0x93, 0x2c, 0xc4, 0xa8, 0xe1, - 0xb5, 0x21, 0x91, 0x5d, 0x6a, 0x19, 0x2e, 0x1d, 0xa8, 0x80, 0x21, 0x1f, 0x36, 0x76, - 0x9b, 0x8e, 0x3d, 0xb6, + 0x78, 0xfe, 0x1a, 0x99, 0xd6, 0x8b, 0x7d, 0x79, 0xd7, 0x3c, 0xd9, 0x8d, 0xc4, 0x81, + 0x82, 0x63, 0x82, 0x9a, 0x01, 0x3f, 0xed, 0x73, 0xab, 0xf9, 0x9d, 0xfa, 0xc7, 0xa1, + 0x99, 0x0f, 0xdf, 0xc2, ]), _ => None, } } -/// Exposed for the `compute_static_commitments` binary and the -/// drift-detection tests in `static_commitments_tests`. Production callers -/// should go through [`preprocessed_commitment`] so the static const-table -/// shortcut is used when applicable. -#[doc(hidden)] -pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { - // Generate precomputed columns +/// The ONE-ROW (S2) twin of [`static_commitment`]: the same columns committed with one +/// LDE row per leaf ([`LeafLayout::Row`]), per `blowup_factor` in +/// [`crate::tables::STATIC_BLOWUP_FACTORS_ONE_ROW`], generated by +/// `compute_static_commitments --layout row` and pinned by the one-row drift +/// test. The same regeneration rules as [`static_commitment`]. A blowup with no arm here +/// is a hard miss under one row: no recompute. +pub(crate) fn static_commitment_one_row(blowup_factor: u8) -> Option { + match blowup_factor { + 4 => Some([ + 0xe6, 0x9e, 0xfc, 0xec, 0x0d, 0x6f, 0x04, 0x22, 0xfc, 0xfe, 0x7c, 0x8b, 0x44, 0xcd, + 0x6d, 0x60, 0x27, 0x6f, 0x3b, 0x9a, 0x78, 0xf6, 0x89, 0x7b, 0x26, 0x40, 0x4b, 0x2a, + 0x65, 0x6d, 0x48, 0x79, + ]), + _ => None, + } +} + +/// The POSEIDON1 twin of [`static_commitment`]: the same columns in row pairs, +/// committed by a Poseidon1 configuration (4-ary trees of ZisK's leaf hash; a +/// configuration's cap moves path nodes, never a root), per `blowup_factor` +/// in [`crate::tables::STATIC_BLOWUP_FACTORS_P1`] at coset 3. Generated by +/// `compute_static_commitments --hash p1` and pinned by +/// `keccak_rc_p1_static_matches_recompute`. The same regeneration rules as +/// [`static_commitment`]. A blowup with no arm here is computed, as every +/// other Poseidon1 key is ([`crate::hash_pin::static_root_under`]). +pub(crate) fn static_commitment_p1(blowup_factor: u8) -> Option { + match blowup_factor { + 4 => Some([ + 0x61, 0x5b, 0x6d, 0x21, 0x29, 0x76, 0xb4, 0x35, 0xdb, 0x4e, 0x64, 0x1b, 0x44, 0x1a, + 0x35, 0xd8, 0xda, 0xfe, 0x82, 0x04, 0x08, 0x87, 0x06, 0xd7, 0x67, 0xfe, 0x7f, 0x13, + 0xc5, 0x90, 0x82, 0xad, + ]), + _ => None, + } +} + +/// The precomputed columns themselves, one per column, `NUM_ROWS` tall. +/// +/// The multilinear path checks a proof's claimed openings against these instead +/// of comparing a commitment, so it needs the values and not just their root. +pub fn preprocessed_columns() -> Vec> { let mut columns: Vec> = (0..NUM_PRECOMPUTED_COLS) .map(|_| Vec::with_capacity(NUM_ROWS)) .collect(); @@ -131,6 +165,36 @@ pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { columns[col_idx].push(FE::from(value)); } } + columns +} + +/// Exposed for the `compute_static_commitments` binary and the +/// drift-detection tests in `static_commitments_tests`. Production callers +/// should go through [`preprocessed_commitment`] so the static const-table +/// shortcut is used when applicable. +#[doc(hidden)] +pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { + compute_preprocessed_commitment_with(options, LeafLayout::RowPair) +} + +/// [`compute_preprocessed_commitment`] under an explicit trace-tree leaf +/// layout (the generator and the one-row drift test; S2). +#[doc(hidden)] +pub fn compute_preprocessed_commitment_with( + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + compute_preprocessed_commitment_under::(options, layout) +} + +/// [`compute_preprocessed_commitment_with`] under the commitment configuration +/// `S` (a proof family's: `crate::lfm::registry::build_artifacts_under`). +#[doc(hidden)] +pub fn compute_preprocessed_commitment_under( + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + let columns = preprocessed_columns(); // Interpolate each column to a polynomial let polys: Vec> = columns @@ -152,9 +216,13 @@ pub fn compute_preprocessed_commitment(options: &ProofOptions) -> Commitment { }) .collect(); - let (_, root) = commit_bit_reversed(&lde_columns, ROWS_PER_LEAF) - .expect("Failed to build Merkle tree for keccak_rc LDE"); - root + // ★ Through the LFM commit helper, which commits under the BLOCK PATH's pin + // rather than `stark`'s default aliases. This root is a PREPROCESSED + // commitment the block path's verifier absorbs, so it has to be built with + // the hash that path commits under — on a branch that pins an algebraic + // hash, a root left on the alias would be the one BLAKE3 artifact in an RPO + // proof, and it would fail as a root nothing reconstructs. + crate::lfm::commit::commit_lde_columns_under::(&lde_columns, layout) } /// Returns the preprocessed commitment for the keccak_rc table. @@ -180,6 +248,71 @@ pub fn preprocessed_commitment(options: &ProofOptions) -> Commitment { compute_preprocessed_commitment(options) } +/// The preprocessed commitment under the table's resolved leaf `layout`: +/// today's [`preprocessed_commitment`] for row pairs; for one row the static +/// twin ([`static_commitment_one_row`]) at coset 3, and `None` otherwise (a +/// hard miss, never a recompute). +pub fn preprocessed_commitment_for( + options: &ProofOptions, + layout: LeafLayout, +) -> Option { + match layout { + LeafLayout::RowPair => Some(preprocessed_commitment(options)), + LeafLayout::Row => (options.coset_offset == 3) + .then(|| static_commitment_one_row(options.blowup_factor)) + .flatten(), + } +} + +/// The row-pair preprocessed commitment under the commitment configuration +/// `S`: the legacy pin's blessed root ([`preprocessed_commitment`]); a +/// Poseidon1 configuration's blessed root where one ships +/// ([`static_commitment_p1`], coset 3); any other key computed once per +/// process ([`crate::hash_pin::static_root_under`]). +pub fn preprocessed_root_under(options: &ProofOptions) -> Commitment { + if S::COMMITMENT_HASH == crate::hash_pin::BLOCK_COMMITMENT_HASH { + return preprocessed_commitment(options); + } + if S::COMMITMENT_HASH == stark::config::CommitmentHash::Poseidon1 + && options.coset_offset == 3 + && let Some(commitment) = static_commitment_p1(options.blowup_factor) + { + return commitment; + } + crate::hash_pin::static_root_under::("keccak_rc", options, LeafLayout::RowPair, || { + compute_preprocessed_commitment_under::(options, LeafLayout::RowPair) + }) +} + +/// [`preprocessed_commitment_for`] under the commitment configuration `S`: the +/// legacy pin's static roots (a one-row miss stays a miss), any other +/// configuration's computed once per process. +pub fn preprocessed_commitment_under( + options: &ProofOptions, + layout: LeafLayout, +) -> Option { + match layout { + LeafLayout::RowPair => Some(preprocessed_root_under::(options)), + LeafLayout::Row if S::COMMITMENT_HASH == crate::hash_pin::BLOCK_COMMITMENT_HASH => { + preprocessed_commitment_for(options, layout) + } + LeafLayout::Row => Some(crate::hash_pin::static_root_under::( + "keccak_rc", + options, + layout, + || compute_preprocessed_commitment_under::(options, layout), + )), + } +} + +/// The AIR's commitment source for both leaf layouts: today's root now, the +/// one-row twin on demand ([`preprocessed_commitment_for`]). +pub fn lazy_commitment(options: &ProofOptions) -> stark::lookup::LazyCommitment { + let o = options.clone(); + stark::lookup::LazyCommitment::ready(preprocessed_commitment(options)) + .with_one_row(move || preprocessed_commitment_for(&o, LeafLayout::Row)) +} + // ========================================================================= // Trace generation // ========================================================================= diff --git a/prover/src/tables/keccak_rnd.rs b/prover/src/tables/keccak_rnd.rs index 1b4f0a318..8adb3a3d8 100644 --- a/prover/src/tables/keccak_rnd.rs +++ b/prover/src/tables/keccak_rnd.rs @@ -37,6 +37,7 @@ use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; // ========================================================================= @@ -263,189 +264,318 @@ pub fn generate_keccak_rnd_trace( ops: &[KeccakRoundOperation], ) -> TraceTable { let n_rows = (ops.len() * 24).next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(n_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (op_idx, op) in ops.iter().enumerate() { - // Execute round-by-round, tracking the state - let mut state = op.input; - - for round in 0..24 { - let row_idx = op_idx * 24 + round; - - // Timestamp & round - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - table.set_u64(row_idx, cols::ROUND, round as u64); - - // start = current state as bytes - for x in 0..5 { - for y in 0..5 { - let lane = state[x + 5 * y]; - table.set_dword_bl(row_idx, cols::start(x, y, 0), lane); - } - } + generate_keccak_rnd_rows(ops, 0, n_rows) +} - // === θ (theta) === - // Column parities: C[x] = XOR of all 5 lanes in column x - // Computed as a chain: Cxz[x][0] = start[x,0] XOR start[x,1] - // Cxz[x][k] = Cxz[x][k-1] XOR start[x,k+1] - let mut c_bytes = [[0u8; 8]; 5]; // C[x][byte] = final parity - let mut cxz = [[[0u8; 8]; 4]; 5]; // Cxz[x][stage][byte] - for x in 0..5 { - // Stage 0: XOR(start[x,0], start[x,1]) - for b in 0..8 { - let v0 = byte_of(state[x], b); - let v1 = byte_of(state[x + 5], b); - cxz[x][0][b] = v0 ^ v1; - } - table.set_bytes(row_idx, cols::cxz(x, 0, 0), &cxz[x][0]); - - // Stages 1..3: XOR(Cxz[x][k-1], start[x, k+1]) - for stage in 1..4 { - let y = stage + 1; - for b in 0..8 { - let prev = cxz[x][stage - 1][b]; - let sv = byte_of(state[x + 5 * y], b); - cxz[x][stage][b] = prev ^ sv; +/// Rows `skip..skip + rows` of the table [`generate_keccak_rnd_trace`] builds +/// from `ops` — the same rows, zero past the ops' last — as a table of `rows` +/// rows. Each op's 24 rows depend on that op alone, so a slice of the table is +/// the slice's ops generated with their row offset: a KECCAK_RND chunk can be +/// built from the ops that reach it, before the run's last op exists. +pub fn generate_keccak_rnd_rows( + ops: &[KeccakRoundOperation], + skip: usize, + rows: usize, +) -> TraceTable { + generate_keccak_rnd_rows_as(ops, skip, rows, TraceForm::Wide) +} + +/// The widths KECCAK_RND traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// Whether [`generate_keccak_rnd_rows_as`] in `form` writes its rows packed, +/// with no 64-bit table on the way: in [`TraceForm::Narrow`] once an earlier +/// KECCAK_RND trace of this process left its widths (`tables::gpack`). +pub(crate) fn rows_written_packed(form: TraceForm) -> bool { + matches!( + super::gpack::Plan::new(form, &WIDTHS, cols::NUM_COLUMNS), + super::gpack::Plan::Write(_) + ) +} + +/// [`generate_keccak_rnd_rows`] in `form` (`tables::gpack`). +pub fn generate_keccak_rnd_rows_as( + ops: &[KeccakRoundOperation], + skip: usize, + rows: usize, + form: TraceForm, +) -> TraceTable { + // Padding rows have mu=0 and all zeros (default) + generate_main!(form, &WIDTHS, rows, cols::NUM_COLUMNS, |table| { + for (op_idx, op) in ops.iter().enumerate() { + let first = op_idx * 24; + if first + 24 <= skip || first >= skip + rows { + continue; + } + if first >= skip && first + 24 <= skip + rows { + fill_op(table, op, first - skip); + } else { + // An op the slice cuts: its 24 rows on the side, the part in + // range copied over. + let mut scratch = TraceTable::::new_main( + crate::tables::types::zeroed_fe_vec(24 * cols::NUM_COLUMNS), + cols::NUM_COLUMNS, + 1, + ); + fill_op(&mut scratch.main_table, op, 0); + for round in 0..24 { + let row = first + round; + if row >= skip && row < skip + rows { + for col in 0..cols::NUM_COLUMNS { + table.set_fe(row - skip, col, *scratch.main_table.get(round, col)); + } } - table.set_bytes(row_idx, cols::cxz(x, stage, 0), &cxz[x][stage]); } - c_bytes[x] = cxz[x][3]; } + } + }) +} - // Rotate C left by 1 bit using HWSL decomposition. - // HWSL shifts each halfword (u16) independently. For shift=1, the - // carry is a single bit (top bit of the halfword); we store it in - // one column per halfword (Cxz_right[x][hw], spec d75944ee). - // rotated_Cxz[z] = Cxz_left[z] + (1 - z%2) * Cxz_right[(z/2 - 1) mod 4] - let mut cxz_left_bytes = [[0u8; 8]; 5]; - let mut cxz_right_bits = [[0u8; 4]; 5]; - let mut rotated_c = [[0u8; 8]; 5]; - for x in 0..5 { - for hw in 0..4 { - let lo = c_bytes[x][hw * 2] as u16; - let hi = c_bytes[x][hw * 2 + 1] as u16; - let halfword = lo | (hi << 8); - let (shifted, carry) = hwsl(halfword, 1); - cxz_left_bytes[x][hw * 2] = (shifted & 0xFF) as u8; - cxz_left_bytes[x][hw * 2 + 1] = (shifted >> 8) as u8; - // For shift=1, carry ∈ {0, 1}. - cxz_right_bits[x][hw] = carry as u8; - } - table.set_bytes(row_idx, cols::cxz_left(x, 0), &cxz_left_bytes[x]); - table.set_bytes(row_idx, cols::cxz_right_bit(x, 0), &cxz_right_bits[x]); +/// A KECCAK_RND table held as the keccak operations its rows come from +/// (`multilinear::records`), about 208 bytes an op against about 1,480 a row +/// packed: rows `skip..skip + rows` of the table the ops make, as +/// [`generate_keccak_rnd_rows`] builds them. The card fills the columns from +/// the ops; a host reader gets this module's fill of them. +#[derive(Debug)] +pub struct KeccakRndRecords { + ops: Vec, + skip: usize, + rows: usize, +} - // Reconstruct: left[b] + (1 - b%2) * right[(b/2 + 3) mod 4] - for b in 0..8 { - let right_contribution = match cols::cxz_right_bit_for_byte(b) { - Some(hw) => cxz_right_bits[x][hw], - None => 0, - }; - rotated_c[x][b] = cxz_left_bytes[x][b].wrapping_add(right_contribution); - } +impl KeccakRndRecords { + /// Rows `skip..skip + rows` of the table `ops` make. Each op's rows read + /// its timestamp and input alone (`fill_op`), so those are what is kept. + pub fn new( + ops: impl IntoIterator, + skip: usize, + rows: usize, + ) -> Self { + Self { + ops: ops.into_iter().collect(), + skip, + rows, + } + } + + /// The table held as these records. `None` where a trace cannot be held + /// so (not reached: KECCAK_RND has columns, and its field is Goldilocks). + pub fn into_trace(self) -> Option> { + TraceTable::try_from_records_main( + multilinear::records::Records(std::sync::Arc::new(self)), + 1, + ) + .ok() + } +} + +impl multilinear::records::TableRecords for KeccakRndRecords { + fn rows(&self) -> usize { + self.rows + } + + fn cols(&self) -> usize { + cols::NUM_COLUMNS + } + + fn fill(&self) -> multilinear::records::RecordsFill<'_> { + multilinear::records::RecordsFill::KeccakRnd { + ops: &self.ops, + skip: self.skip, + } + } + + fn host_fill(&self) -> Option { + let ops: Vec = self + .ops + .iter() + .map(|op| KeccakRoundOperation { + timestamp: op.timestamp, + input: op.input, + output: [0; 25], + }) + .collect(); + let mut trace = generate_keccak_rnd_rows_as(&ops, self.skip, self.rows, TraceForm::Narrow); + // Written packed, except in a build that holds every trace wide. + trace.pack_main_narrow(); + trace.take_narrow_main() + } +} + +/// One op's 24 rows, from row `first_row` on. +#[allow(clippy::needless_range_loop)] +fn fill_op(table: &mut T, op: &KeccakRoundOperation, first_row: usize) { + // Execute round-by-round, tracking the state + let mut state = op.input; + + for round in 0..24 { + let row_idx = first_row + round; + + // Timestamp & round + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + table.set_u64(row_idx, cols::ROUND, round as u64); + + // start = current state as bytes + for x in 0..5 { + for y in 0..5 { + let lane = state[x + 5 * y]; + table.set_dword_bl(row_idx, cols::start(x, y, 0), lane); } + } - // D[x] = C[(x-1)%5] XOR rotated_C[(x+1)%5] - let mut d_bytes = [[0u8; 8]; 5]; - for x in 0..5 { + // === θ (theta) === + // Column parities: C[x] = XOR of all 5 lanes in column x + // Computed as a chain: Cxz[x][0] = start[x,0] XOR start[x,1] + // Cxz[x][k] = Cxz[x][k-1] XOR start[x,k+1] + let mut c_bytes = [[0u8; 8]; 5]; // C[x][byte] = final parity + let mut cxz = [[[0u8; 8]; 4]; 5]; // Cxz[x][stage][byte] + for x in 0..5 { + // Stage 0: XOR(start[x,0], start[x,1]) + for b in 0..8 { + let v0 = byte_of(state[x], b); + let v1 = byte_of(state[x + 5], b); + cxz[x][0][b] = v0 ^ v1; + } + table.set_bytes(row_idx, cols::cxz(x, 0, 0), &cxz[x][0]); + + // Stages 1..3: XOR(Cxz[x][k-1], start[x, k+1]) + for stage in 1..4 { + let y = stage + 1; for b in 0..8 { - let val = c_bytes[(x + 4) % 5][b] ^ rotated_c[(x + 1) % 5][b]; - d_bytes[x][b] = val; + let prev = cxz[x][stage - 1][b]; + let sv = byte_of(state[x + 5 * y], b); + cxz[x][stage][b] = prev ^ sv; } - table.set_bytes(row_idx, cols::dxz(x, 0), &d_bytes[x]); + table.set_bytes(row_idx, cols::cxz(x, stage, 0), &cxz[x][stage]); } + c_bytes[x] = cxz[x][3]; + } - // theta[x][y] = start[x][y] XOR D[x] - let mut theta_lanes = [0u64; 25]; - for x in 0..5 { - for y in 0..5 { - let lane = state[x + 5 * y]; - let mut d_lane = 0u64; - for b in 0..8 { - d_lane |= (d_bytes[x][b] as u64) << (b * 8); - } - theta_lanes[x + 5 * y] = lane ^ d_lane; - table.set_dword_bl(row_idx, cols::theta(x, y, 0), theta_lanes[x + 5 * y]); - } + // Rotate C left by 1 bit using HWSL decomposition. + // HWSL shifts each halfword (u16) independently. For shift=1, the + // carry is a single bit (top bit of the halfword); we store it in + // one column per halfword (Cxz_right[x][hw], spec d75944ee). + // rotated_Cxz[z] = Cxz_left[z] + (1 - z%2) * Cxz_right[(z/2 - 1) mod 4] + let mut cxz_left_bytes = [[0u8; 8]; 5]; + let mut cxz_right_bits = [[0u8; 4]; 5]; + let mut rotated_c = [[0u8; 8]; 5]; + for x in 0..5 { + for hw in 0..4 { + let lo = c_bytes[x][hw * 2] as u16; + let hi = c_bytes[x][hw * 2 + 1] as u16; + let halfword = lo | (hi << 8); + let (shifted, carry) = hwsl(halfword, 1); + cxz_left_bytes[x][hw * 2] = (shifted & 0xFF) as u8; + cxz_left_bytes[x][hw * 2 + 1] = (shifted >> 8) as u8; + // For shift=1, carry ∈ {0, 1}. + cxz_right_bits[x][hw] = carry as u8; } + table.set_bytes(row_idx, cols::cxz_left(x, 0), &cxz_left_bytes[x]); + table.set_bytes(row_idx, cols::cxz_right_bit(x, 0), &cxz_right_bits[x]); - // === ρ (rho) === - // For each lane, rotate theta[x][y] by KECCAK_RHO[x][y] bits. - // Decompose rotation as: rnc (nibble, 0..15) + 16*rbc[0] + 32*rbc[1]. - // rnc and rbc are inlined as compile-time constants per spec - // [[variables.constant]]; only HWSL outputs are stored in the trace. - for x in 0..5 { - for y in 0..5 { - let rho_offset = KECCAK_RHO[x][y] as usize; - let rnc_val = (rho_offset % 16) as u8; - let theta_lane = theta_lanes[x + 5 * y]; - let mut rot_left_bytes = [0u8; 8]; - let mut rot_right_bytes = [0u8; 8]; - for hw in 0..4 { - let halfword = ((theta_lane >> (hw * 16)) & 0xFFFF) as u16; - let (shifted, carry) = hwsl(halfword, rnc_val); - rot_left_bytes[hw * 2] = (shifted & 0xFF) as u8; - rot_left_bytes[hw * 2 + 1] = (shifted >> 8) as u8; - rot_right_bytes[hw * 2] = (carry & 0xFF) as u8; - rot_right_bytes[hw * 2 + 1] = (carry >> 8) as u8; - } - table.set_bytes(row_idx, cols::rot_left(x, y, 0), &rot_left_bytes); - table.set_bytes(row_idx, cols::rot_right(x, y, 0), &rot_right_bytes); - } + // Reconstruct: left[b] + (1 - b%2) * right[(b/2 + 3) mod 4] + for b in 0..8 { + let right_contribution = match cols::cxz_right_bit_for_byte(b) { + Some(hw) => cxz_right_bits[x][hw], + None => 0, + }; + rotated_c[x][b] = cxz_left_bytes[x][b].wrapping_add(right_contribution); } + } - // === π (pi) === - // pi[x][y] = rho[(x+3y)%5][x] where rho is the rotated theta. - // pi is a spec [[variables.virtual]] — not stored as trace columns. - // It's reconstructed inline in chi bus interactions as - // pi[x][y][z] = rot_left[sx,sy,l_byte] + rot_right[sx,sy,r_byte] - // with (sx, sy) = ((x+3y)%5, x) and (l_byte, r_byte) resolved from - // the compile-time rbc constant. pi_lanes is still computed here - // for the chi step below. - let mut pi_lanes = [0u64; 25]; - for x in 0..5 { - for y in 0..5 { - let rotated = theta_lanes[x + 5 * y].rotate_left(KECCAK_RHO[x][y]); - let dst_x = y; - let dst_y = (2 * x + 3 * y) % 5; - pi_lanes[dst_x + 5 * dst_y] = rotated; - } + // D[x] = C[(x-1)%5] XOR rotated_C[(x+1)%5] + let mut d_bytes = [[0u8; 8]; 5]; + for x in 0..5 { + for b in 0..8 { + let val = c_bytes[(x + 4) % 5][b] ^ rotated_c[(x + 1) % 5][b]; + d_bytes[x][b] = val; } + table.set_bytes(row_idx, cols::dxz(x, 0), &d_bytes[x]); + } - // === χ (chi) === - let mut chi_lanes = [0u64; 25]; - for x in 0..5 { - for y in 0..5 { - let not_next = !pi_lanes[(x + 1) % 5 + 5 * y]; - let next2 = pi_lanes[(x + 2) % 5 + 5 * y]; - let and_val = not_next & next2; - chi_lanes[x + 5 * y] = pi_lanes[x + 5 * y] ^ and_val; - table.set_dword_bl(row_idx, cols::chi_ands(x, y, 0), and_val); - table.set_dword_bl(row_idx, cols::chi(x, y, 0), chi_lanes[x + 5 * y]); + // theta[x][y] = start[x][y] XOR D[x] + let mut theta_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let lane = state[x + 5 * y]; + let mut d_lane = 0u64; + for b in 0..8 { + d_lane |= (d_bytes[x][b] as u64) << (b * 8); } + theta_lanes[x + 5 * y] = lane ^ d_lane; + table.set_dword_bl(row_idx, cols::theta(x, y, 0), theta_lanes[x + 5 * y]); } + } - // === ι (iota) === - let rc_val = KECCAK_RC[round]; - let iota_lane = chi_lanes[0] ^ rc_val; - table.set_dword_bl(row_idx, cols::rc(0), rc_val); - table.set_dword_bl(row_idx, cols::iota(0), iota_lane); + // === ρ (rho) === + // For each lane, rotate theta[x][y] by KECCAK_RHO[x][y] bits. + // Decompose rotation as: rnc (nibble, 0..15) + 16*rbc[0] + 32*rbc[1]. + // rnc and rbc are inlined as compile-time constants per spec + // [[variables.constant]]; only HWSL outputs are stored in the trace. + for x in 0..5 { + for y in 0..5 { + let rho_offset = KECCAK_RHO[x][y] as usize; + let rnc_val = (rho_offset % 16) as u8; + let theta_lane = theta_lanes[x + 5 * y]; + let mut rot_left_bytes = [0u8; 8]; + let mut rot_right_bytes = [0u8; 8]; + for hw in 0..4 { + let halfword = ((theta_lane >> (hw * 16)) & 0xFFFF) as u16; + let (shifted, carry) = hwsl(halfword, rnc_val); + rot_left_bytes[hw * 2] = (shifted & 0xFF) as u8; + rot_left_bytes[hw * 2 + 1] = (shifted >> 8) as u8; + rot_right_bytes[hw * 2] = (carry & 0xFF) as u8; + rot_right_bytes[hw * 2 + 1] = (carry >> 8) as u8; + } + table.set_bytes(row_idx, cols::rot_left(x, y, 0), &rot_left_bytes); + table.set_bytes(row_idx, cols::rot_right(x, y, 0), &rot_right_bytes); + } + } - // Update state for next round - chi_lanes[0] = iota_lane; - state = chi_lanes; + // === π (pi) === + // pi[x][y] = rho[(x+3y)%5][x] where rho is the rotated theta. + // pi is a spec [[variables.virtual]] — not stored as trace columns. + // It's reconstructed inline in chi bus interactions as + // pi[x][y][z] = rot_left[sx,sy,l_byte] + rot_right[sx,sy,r_byte] + // with (sx, sy) = ((x+3y)%5, x) and (l_byte, r_byte) resolved from + // the compile-time rbc constant. pi_lanes is still computed here + // for the chi step below. + let mut pi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let rotated = theta_lanes[x + 5 * y].rotate_left(KECCAK_RHO[x][y]); + let dst_x = y; + let dst_y = (2 * x + 3 * y) % 5; + pi_lanes[dst_x + 5 * dst_y] = rotated; + } + } - // mu = 1 (real row) - table.set_fe(row_idx, cols::MU, FE::one()); + // === χ (chi) === + let mut chi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let not_next = !pi_lanes[(x + 1) % 5 + 5 * y]; + let next2 = pi_lanes[(x + 2) % 5 + 5 * y]; + let and_val = not_next & next2; + chi_lanes[x + 5 * y] = pi_lanes[x + 5 * y] ^ and_val; + table.set_dword_bl(row_idx, cols::chi_ands(x, y, 0), and_val); + table.set_dword_bl(row_idx, cols::chi(x, y, 0), chi_lanes[x + 5 * y]); + } } - } - // Padding rows have mu=0 and all zeros (default) - trace + // === ι (iota) === + let rc_val = KECCAK_RC[round]; + let iota_lane = chi_lanes[0] ^ rc_val; + table.set_dword_bl(row_idx, cols::rc(0), rc_val); + table.set_dword_bl(row_idx, cols::iota(0), iota_lane); + + // Update state for next round + chi_lanes[0] = iota_lane; + state = chi_lanes; + + // mu = 1 (real row) + table.set_fe(row_idx, cols::MU, FE::one()); + } } // ========================================================================= @@ -949,3 +1079,7 @@ impl ConstraintSet for KeccakRndConstraint b.emit_base(idx, mu.clone() * (one - mu)); } } + +#[cfg(all(test, feature = "cuda"))] +#[path = "keccak_rnd_card_tests.rs"] +mod card_tests; diff --git a/prover/src/tables/keccak_rnd_card_tests.rs b/prover/src/tables/keccak_rnd_card_tests.rs new file mode 100644 index 000000000..daff37aab --- /dev/null +++ b/prover/src/tables/keccak_rnd_card_tests.rs @@ -0,0 +1,168 @@ +//! ★ The card fills a KECCAK_RND table from its records +//! ([`super::KeccakRndRecords`]) with the words the host fill writes, at the +//! places its group's store holds them: alone, and between a wide and a packed +//! table; for the synthetic 2^16-row tables cut into an op at both ends, a +//! table cut into an op at either end, one of padding rows alone and a whole +//! padded table. A fill a row late is caught (with `test-cuda-faults`). +//! +//! Card only: without a device every upload is declined, and these tests say +//! so instead of passing. + +use std::sync::Arc; + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu::{TableColumns, upload_tables}; +use multilinear::mle::Mle; +use multilinear::narrow::NarrowColumns; +use multilinear::records::{KeccakRndOp, Records}; + +use super::{KeccakRndRecords, KeccakRoundOperation}; +use crate::tables::card_fill_inputs::{ + KECCAK_RND_OPS, KECCAK_RND_ROWS, KECCAK_RND_SKIP, SEEDS, keccak_ops, +}; + +/// Rows `skip..skip + rows` of the table `ops` make, held as their records. +fn records(ops: &[KeccakRoundOperation], skip: usize, rows: usize) -> Records { + Records(Arc::new(KeccakRndRecords::new( + ops.iter().map(|op| KeccakRndOp { + timestamp: op.timestamp, + input: op.input, + }), + skip, + rows, + ))) +} + +/// The tables: the synthetic sets, and three ops (72 rows) cut as the +/// finish cuts them at 32 rows a table (into the second op; from its ninth +/// row on; padding rows alone) or held whole (128 rows, 56 of padding). +fn cases() -> Vec<(String, Records)> { + let mut cases: Vec<(String, Records)> = SEEDS + .iter() + .map(|&seed| { + ( + format!("synthetic seed {seed}"), + records( + &keccak_ops(seed, KECCAK_RND_OPS), + KECCAK_RND_SKIP, + KECCAK_RND_ROWS, + ), + ) + }) + .collect(); + let three = keccak_ops(7, 3); + cases.push(("first cut".into(), records(&three[..2], 0, 32))); + cases.push(("second cut".into(), records(&three[1..], 8, 32))); + cases.push(("padding alone".into(), records(&three[3..], 24, 32))); + cases.push(("whole".into(), records(&three, 0, 128))); + cases +} + +/// The group's store, column by column, as the card holds it. +fn uploaded(tables: &[TableColumns<'_, F>]) -> Vec> { + upload_tables(tables) + .expect("the card takes the group") + .download() + .expect("the store comes back") +} + +/// Mismatched words between two stores, and the first. +fn mismatches(got: &[Vec], want: &[Vec]) -> (usize, Option<(usize, usize)>) { + let mut count = 0; + let mut first = None; + for (c, (g, w)) in got.iter().zip(want).enumerate() { + for (r, (a, b)) in g.iter().zip(w).enumerate() { + if a != b { + count += 1; + first.get_or_insert((c, r)); + } + } + } + (count, first) +} + +fn host_words(packed: &NarrowColumns) -> Vec> { + (0..packed.cols()).map(|c| packed.column(c)).collect() +} + +#[test] +fn the_card_fills_keccak_rnd_from_its_records_as_the_host_does() { + for (what, records) in cases() { + let packed = records.host_fill().expect("the host fills it"); + let fills = multilinear::records::fills().0; + let card = uploaded(&[TableColumns::Records(&records)]); + assert_eq!( + multilinear::records::fills().0, + fills + 1, + "{what}: filled on the card" + ); + let widened = uploaded(&[TableColumns::Narrow(&packed)]); + assert_eq!(card.len(), super::cols::NUM_COLUMNS, "{what}: the columns"); + assert!( + card.iter().all(|c| c.len() == records.rows()), + "{what}: the rows" + ); + assert_eq!( + mismatches(&card, &widened), + (0, None), + "{what}: against the widened host fill" + ); + assert_eq!( + mismatches(&card, &host_words(&packed)), + (0, None), + "{what}: against the host's words" + ); + } +} + +/// Between a wide table and a packed one, the filled table's columns sit +/// where its widened columns would, and the others are as uploaded. +#[test] +fn the_card_fills_keccak_rnd_in_place_among_a_groups_tables() { + let three = keccak_ops(7, 3); + let held = records(&three[..2], 0, 32); + let other = records(&three[1..], 8, 32) + .host_fill() + .expect("the host fills it"); + let wide: Vec> = (0..4u64) + .map(|c| { + Mle::new( + (0..16u64) + .map(|r| FieldElement::from(c * 1000 + r)) + .collect(), + ) + .expect("a column") + }) + .collect(); + let packed = held.host_fill().expect("the host fills it"); + let card = uploaded(&[ + TableColumns::Wide(&wide), + TableColumns::Records(&held), + TableColumns::Narrow(&other), + ]); + let host = uploaded(&[ + TableColumns::Wide(&wide), + TableColumns::Narrow(&packed), + TableColumns::Narrow(&other), + ]); + assert_eq!(card.len(), 4 + 2 * super::cols::NUM_COLUMNS); + assert_eq!(mismatches(&card, &host), (0, None)); +} + +/// A fill a row late is caught word by word: the comparison above is not +/// vacuous. +#[cfg(feature = "test-cuda-faults")] +#[test] +fn a_fill_a_row_late_is_caught() { + use std::sync::atomic::Ordering::Relaxed; + let (what, records) = cases().swap_remove(0); + let packed = records.host_fill().expect("the host fills it"); + math_cuda::fill::FAULT_RECORDS_SKIP.store(1, Relaxed); + let card = + upload_tables::(&[TableColumns::Records(&records)]).and_then(|store| store.download()); + math_cuda::fill::FAULT_RECORDS_SKIP.store(0, Relaxed); + let card = card.expect("the card takes it"); + let (count, _) = mismatches(&card, &host_words(&packed)); + assert!(count > 0, "{what}: a row late, and every word the host's"); +} diff --git a/prover/src/tables/load.rs b/prover/src/tables/load.rs index d914fd394..45f134ca7 100644 --- a/prover/src/tables/load.rs +++ b/prover/src/tables/load.rs @@ -26,6 +26,7 @@ use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable}; // ========================================================================= @@ -72,7 +73,9 @@ pub mod cols { // Trace generation // ========================================================================= -/// A single LOAD operation to be added to the trace. +/// A single LOAD operation to be added to the trace. `#[repr(C)]`: a chunk's +/// ops are the records the card fills its LOAD table from ([`card_ops`]). +#[repr(C)] #[derive(Debug, Clone)] pub struct LoadOperation { /// Base address (64-bit) @@ -84,12 +87,35 @@ pub struct LoadOperation { /// Whether to sign-extend (true) or zero-extend (false) pub signed: bool, /// Result bytes (8 bytes, extended) - pub res: [u64; 8], + pub res: [u8; 8], +} + +// The walk keeps one per load: keep it at 32 bytes. +const _: () = assert!(std::mem::size_of::() == 32); + +/// LOAD ops as the card's LOAD fill reads them (`multilinear::records::LoadOp`): +/// the same bytes. +pub(crate) fn card_ops(ops: &[LoadOperation]) -> &[multilinear::records::LoadOp] { + use multilinear::records::LoadOp; + use std::mem::{align_of, offset_of, size_of}; + const _: () = { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(LoadOperation, base_address) == offset_of!(LoadOp, base_address)); + assert!(offset_of!(LoadOperation, timestamp) == offset_of!(LoadOp, timestamp)); + assert!(offset_of!(LoadOperation, width) == offset_of!(LoadOp, width)); + assert!(offset_of!(LoadOperation, signed) == offset_of!(LoadOp, signed)); + assert!(offset_of!(LoadOperation, res) == offset_of!(LoadOp, res)); + }; + // SAFETY: both are `#[repr(C)]` with the same fields in the same order + // (asserted above); `signed` is a `bool`, 0 or 1, read as a `u8`, and no + // reader looks at the padding after `res`. + unsafe { std::slice::from_raw_parts(ops.as_ptr() as *const LoadOp, ops.len()) } } impl LoadOperation { /// Create a new LOAD operation. - pub fn new(base_address: u64, timestamp: u64, width: u8, signed: bool, res: [u64; 8]) -> Self { + pub fn new(base_address: u64, timestamp: u64, width: u8, signed: bool, res: [u8; 8]) -> Self { Self { base_address, timestamp, @@ -151,11 +177,26 @@ impl LoadOperation { /// - read4: MSB8[res[3]] -> sign_bit /// - read8: no MSB8 lookup needed (all 8 bytes are used) pub fn collect_bitwise_ops(&self) -> Vec { + self.bitwise_op().into_iter().collect() + } + + /// [`Self::collect_bitwise_ops`]'s lookup counted into `histogram`, with no + /// list built. + #[inline] + pub(crate) fn count_bitwise_into(&self, histogram: &mut super::bitwise::BitwiseHistogram) { + if let Some(op) = self.bitwise_op() { + histogram.bump(op); + } + } + + /// The one lookup of [`Self::collect_bitwise_ops`], if any. + #[inline] + fn bitwise_op(&self) -> Option { use super::bitwise::{BitwiseOperation, BitwiseOperationType}; // For width 8, no sign extension is needed if self.width == 8 { - return Vec::new(); + return None; } // Get the byte index for the MSB8 lookup based on width @@ -163,14 +204,14 @@ impl LoadOperation { 1 => 0, // res[0] for read1 2 => 1, // res[1] for read2 4 => 3, // res[3] for read4 - _ => return Vec::new(), + _ => return None, }; - let input_byte = self.res[byte_idx] as u8; - vec![BitwiseOperation::single_byte( + let input_byte = self.res[byte_idx]; + Some(BitwiseOperation::single_byte( BitwiseOperationType::Msb8, input_byte, - )] + )) } } @@ -178,41 +219,55 @@ impl LoadOperation { pub fn generate_load_trace( operations: &[LoadOperation], ) -> TraceTable { - let num_rows = operations.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in operations.iter().enumerate() { - // Input columns - table.set_dword_wl(row_idx, cols::BASE_ADDRESS_0, op.base_address); - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - - // read flags - let (r2, r4, r8) = op.read_flags(); - table.set_bool(row_idx, cols::READ2, r2); - table.set_bool(row_idx, cols::READ4, r4); - table.set_bool(row_idx, cols::READ8, r8); - - // signed - table.set_bool(row_idx, cols::SIGNED, op.signed); - - // Output: res[8] - for i in 0..8 { - table.set_u64(row_idx, cols::RES[i], op.res[i]); - } + generate_load_trace_as(operations, TraceForm::Wide) +} - // Auxiliary: sign_bit - table.set_bool(row_idx, cols::SIGN_BIT, op.compute_sign_bit()); +/// The widths LOAD traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - // Multiplicity: active row - table.set_fe(row_idx, cols::MU, FE::one()); - } +/// [`generate_load_trace`] in `form` (`tables::gpack`). +pub fn generate_load_trace_as( + operations: &[LoadOperation], + form: TraceForm, +) -> TraceTable { + generate_load_trace_segments_as(&[operations], form) +} - trace +/// [`generate_load_trace_as`] over `segments`, the operations one after another: a +/// chunk handed out as the window parts it lies in. +pub(crate) fn generate_load_trace_segments_as( + segments: &[&[LoadOperation]], + form: TraceForm, +) -> TraceTable { + let len: usize = segments.iter().map(|s| s.len()).sum(); + let num_rows = len.next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in segments.iter().flat_map(|s| s.iter()).enumerate() { + // Input columns + table.set_dword_wl(row_idx, cols::BASE_ADDRESS_0, op.base_address); + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + + // read flags + let (r2, r4, r8) = op.read_flags(); + table.set_bool(row_idx, cols::READ2, r2); + table.set_bool(row_idx, cols::READ4, r4); + table.set_bool(row_idx, cols::READ8, r8); + + // signed + table.set_bool(row_idx, cols::SIGNED, op.signed); + + // Output: res[8] + for i in 0..8 { + table.set_u64(row_idx, cols::RES[i], u64::from(op.res[i])); + } + + // Auxiliary: sign_bit + table.set_bool(row_idx, cols::SIGN_BIT, op.compute_sign_bit()); + + // Multiplicity: active row + table.set_fe(row_idx, cols::MU, FE::one()); + } + }) } // ========================================================================= diff --git a/prover/src/tables/lt.rs b/prover/src/tables/lt.rs index e4cbf457c..fdc1a6ace 100644 --- a/prover/src/tables/lt.rs +++ b/prover/src/tables/lt.rs @@ -30,8 +30,9 @@ use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{ - BusId, GoldilocksExtension, GoldilocksField, SHIFT_16, VmTable, alu_op, dedup_rows, + BusId, GoldilocksExtension, GoldilocksField, SHIFT_16, VmTable, alu_op, dedup_rows_of, }; // ========================================================================= @@ -158,52 +159,75 @@ impl LtOperation { /// Duplicate operations (same lhs, rhs, signed, invert) are merged into a single /// row with their multiplicities summed, split at /// [`MU_MAX`](crate::tables::types::MU_MAX) (see -/// [`dedup_rows`]). The table is then padded +/// [`dedup_rows`](crate::tables::types::dedup_rows)). The table is then padded /// to the next power of 2. pub fn generate_lt_trace( operations: &[LtOperation], ) -> TraceTable { - let unique_ops = dedup_rows(operations); - let num_rows = unique_ops.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, (op, multiplicity)) in unique_ops.iter().enumerate() { - // Store input columns - table.set_dword_hhw(row_idx, cols::LHS_0, op.lhs); - table.set_dword_hhw(row_idx, cols::RHS_0, op.rhs); - table.set_bool(row_idx, cols::SIGNED, op.signed); - - // Compute lt result - let lt = op.compute_lt(); - table.set_bool(row_idx, cols::LT, lt); - - // Compute lhs_sub_rhs = lhs - rhs (wrapping) - // Note: We compute this as a 64-bit wrapping subtraction - let lhs_sub_rhs = op.lhs.wrapping_sub(op.rhs); - - // Store lhs_sub_rhs as DWordHL: [Half, Half, Half, Half] - table.set_dword_hl(row_idx, cols::LHS_SUB_RHS_0, lhs_sub_rhs); - - // Compute MSBs (bit 63 of each value) - let lhs_msb = (op.lhs >> 63) & 1; - let rhs_msb = (op.rhs >> 63) & 1; - table.set_u64(row_idx, cols::LHS_MSB, lhs_msb); - table.set_u64(row_idx, cols::RHS_MSB, rhs_msb); - - // ALU-bus fields: invert + the inverted output. - table.set_bool(row_idx, cols::INVERT, op.invert); - table.set_bool(row_idx, cols::OUT, op.compute_out()); - - // All LT lookups go through the unified ALU bus → single multiplicity. - table.set_u64(row_idx, cols::MU, *multiplicity); - } + generate_lt_trace_as(operations, TraceForm::Wide) +} + +/// The widths LT traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_lt_trace`] in `form` (`tables::gpack`). +pub fn generate_lt_trace_as( + operations: &[LtOperation], + form: TraceForm, +) -> TraceTable { + generate_lt_trace_segments_as(&[operations], form) +} + +/// [`generate_lt_trace_as`] over `segments`, the operations one after another +/// (deduplicated as their concatenation): a chunk handed out as the window +/// parts it lies in. +pub(crate) fn generate_lt_trace_segments_as( + segments: &[&[LtOperation]], + form: TraceForm, +) -> TraceTable { + // Deduplicate operations: (lhs, rhs, signed) -> multiplicity, split at `MU_MAX`. + lt_trace_of_rows(&dedup_rows_of(segments.iter().flat_map(|s| s.iter())), form) +} - trace +/// The LT trace of `unique_ops`, the rows [`generate_lt_trace_segments_as`] +/// deduplicates its operations into, in `form`. +pub(crate) fn lt_trace_of_rows( + unique_ops: &[(LtOperation, u64)], + form: TraceForm, +) -> TraceTable { + let num_rows = unique_ops.len().next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, (op, multiplicity)) in unique_ops.iter().enumerate() { + // Store input columns + table.set_dword_hhw(row_idx, cols::LHS_0, op.lhs); + table.set_dword_hhw(row_idx, cols::RHS_0, op.rhs); + table.set_bool(row_idx, cols::SIGNED, op.signed); + + // Compute lt result + let lt = op.compute_lt(); + table.set_bool(row_idx, cols::LT, lt); + + // Compute lhs_sub_rhs = lhs - rhs (wrapping) + // Note: We compute this as a 64-bit wrapping subtraction + let lhs_sub_rhs = op.lhs.wrapping_sub(op.rhs); + + // Store lhs_sub_rhs as DWordHL: [Half, Half, Half, Half] + table.set_dword_hl(row_idx, cols::LHS_SUB_RHS_0, lhs_sub_rhs); + + // Compute MSBs (bit 63 of each value) + let lhs_msb = (op.lhs >> 63) & 1; + let rhs_msb = (op.rhs >> 63) & 1; + table.set_u64(row_idx, cols::LHS_MSB, lhs_msb); + table.set_u64(row_idx, cols::RHS_MSB, rhs_msb); + + // ALU-bus fields: invert + the inverted output. + table.set_bool(row_idx, cols::INVERT, op.invert); + table.set_bool(row_idx, cols::OUT, op.compute_out()); + + // All LT lookups go through the unified ALU bus → single multiplicity. + table.set_u64(row_idx, cols::MU, *multiplicity); + } + }) } // ========================================================================= diff --git a/prover/src/tables/memw.rs b/prover/src/tables/memw.rs index ee17a4c8f..819a41b44 100644 --- a/prover/src/tables/memw.rs +++ b/prover/src/tables/memw.rs @@ -35,6 +35,7 @@ use stark::trace::TraceTable; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; use crate::constraints::templates::emit_is_bit; @@ -100,29 +101,58 @@ pub mod cols { // Trace generation // ========================================================================= -/// A single MEMW operation to be added to the trace. +/// A single MEMW operation to be added to the trace. `#[repr(C)]`, its fields +/// ordered so it stays 152 bytes: a chunk's ops are the records the card fills +/// its MEMW table from ([`card_ops`]). +#[repr(C)] #[derive(Debug, Clone)] pub struct MemwOperation { - /// Whether this is a register access (true) or memory access (false) - pub is_register: bool, /// Base address (64-bit) pub base_address: u64, + /// Timestamp of this access + pub timestamp: u64, + /// Previous timestamps at the addresses (filled by memory model). + pub old_timestamp: [u64; 8], /// Values to write. Each element is one memory byte (0-255) or, for register /// accesses, a 32-bit half of the register word — both fit in u32, so this is /// `[u32; 8]` rather than `[u64; 8]` to halve the struct's footprint (the walk /// materializes tens of millions of these; it is memory-bandwidth-bound). pub value: [u32; 8], - /// Timestamp of this access - pub timestamp: u64, + /// Previous values at the addresses (filled by memory model). Same element + /// domain as `value` (byte or 32-bit register half) → `[u32; 8]`. + pub old: [u32; 8], + /// Whether this is a register access (true) or memory access (false) + pub is_register: bool, /// Access width: 1, 2, 4, or 8 bytes pub width: u8, /// Whether this is a read (true) or write (false) pub is_read: bool, - /// Previous values at the addresses (filled by memory model). Same element - /// domain as `value` (byte or 32-bit register half) → `[u32; 8]`. - pub old: [u32; 8], - /// Previous timestamps at the addresses (filled by memory model). - pub old_timestamp: [u64; 8], +} + +// The walk keeps one per unaligned access: keep it at 152 bytes. +const _: () = assert!(std::mem::size_of::() == 152); + +/// MEMW ops as the card's MEMW fill reads them (`multilinear::records::MemwOp`): +/// the same bytes. +pub(crate) fn card_ops(ops: &[MemwOperation]) -> &[multilinear::records::MemwOp] { + use multilinear::records::MemwOp; + use std::mem::{align_of, offset_of, size_of}; + const _: () = { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(MemwOperation, base_address) == offset_of!(MemwOp, base_address)); + assert!(offset_of!(MemwOperation, timestamp) == offset_of!(MemwOp, timestamp)); + assert!(offset_of!(MemwOperation, old_timestamp) == offset_of!(MemwOp, old_timestamp)); + assert!(offset_of!(MemwOperation, value) == offset_of!(MemwOp, value)); + assert!(offset_of!(MemwOperation, old) == offset_of!(MemwOp, old)); + assert!(offset_of!(MemwOperation, is_register) == offset_of!(MemwOp, is_register)); + assert!(offset_of!(MemwOperation, width) == offset_of!(MemwOp, width)); + assert!(offset_of!(MemwOperation, is_read) == offset_of!(MemwOp, is_read)); + }; + // SAFETY: both are `#[repr(C)]` with the same fields in the same order + // (asserted above); the flags are `bool`s, 0 or 1, read as `u8`s, and no + // reader looks at the padding after them. + unsafe { std::slice::from_raw_parts(ops.as_ptr() as *const MemwOp, ops.len()) } } impl MemwOperation { @@ -177,60 +207,74 @@ impl MemwOperation { pub fn generate_memw_trace( operations: &[MemwOperation], ) -> TraceTable { - let num_rows = operations.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in operations.iter().enumerate() { - // Input columns - table.set_bool(row_idx, cols::IS_REGISTER, op.is_register); - - // base_address as DWordWL (2 words) - let base_addr_lo = op.base_address & 0xFFFF_FFFF; - table.set_dword_wl(row_idx, cols::BASE_ADDRESS_0, op.base_address); - - // value[8] - for i in 0..8 { - table.set_u64(row_idx, cols::VALUE[i], op.value[i] as u64); - } + generate_memw_trace_as(operations, TraceForm::Wide) +} - // timestamp as DWordWL (2 words) - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); +/// The widths MEMW traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - // write flags - let (w2, w4, w8) = op.write_flags(); - table.set_bool(row_idx, cols::WRITE2, w2); - table.set_bool(row_idx, cols::WRITE4, w4); - table.set_bool(row_idx, cols::WRITE8, w8); +/// [`generate_memw_trace`] in `form` (`tables::gpack`). +pub fn generate_memw_trace_as( + operations: &[MemwOperation], + form: TraceForm, +) -> TraceTable { + generate_memw_trace_segments_as(&[operations], form) +} - // Output: old[8] - for i in 0..8 { - table.set_u64(row_idx, cols::OLD[i], op.old[i] as u64); - } +/// [`generate_memw_trace_as`] over `segments`, the operations one after another: a +/// chunk handed out as the window parts it lies in. +pub(crate) fn generate_memw_trace_segments_as( + segments: &[&[MemwOperation]], + form: TraceForm, +) -> TraceTable { + let len: usize = segments.iter().map(|s| s.len()).sum(); + let num_rows = len.next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in segments.iter().flat_map(|s| s.iter()).enumerate() { + // Input columns + table.set_bool(row_idx, cols::IS_REGISTER, op.is_register); - // Auxiliary: carry[7] - // carry[i] = 1 if (base_address_lo + i+1) >= 2^32 - for i in 0..7 { - let overflows = base_addr_lo + (i as u64 + 1) >= (1u64 << 32); - table.set_bool(row_idx, cols::CARRY[i], overflows); - } + // base_address as DWordWL (2 words) + let base_addr_lo = op.base_address & 0xFFFF_FFFF; + table.set_dword_wl(row_idx, cols::BASE_ADDRESS_0, op.base_address); - // Auxiliary: old_timestamp[8] - each as DWordWL (2 words) - for i in 0..8 { - let cols_i = cols::old_timestamp(i); - table.set_dword_wl(row_idx, cols_i[0], op.old_timestamp[i]); - } + // value[8] + for i in 0..8 { + table.set_u64(row_idx, cols::VALUE[i], op.value[i] as u64); + } - // Multiplicity - table.set_bool(row_idx, cols::MU_READ, op.is_read); - table.set_bool(row_idx, cols::MU_WRITE, !op.is_read); - } + // timestamp as DWordWL (2 words) + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - trace + // write flags + let (w2, w4, w8) = op.write_flags(); + table.set_bool(row_idx, cols::WRITE2, w2); + table.set_bool(row_idx, cols::WRITE4, w4); + table.set_bool(row_idx, cols::WRITE8, w8); + + // Output: old[8] + for i in 0..8 { + table.set_u64(row_idx, cols::OLD[i], op.old[i] as u64); + } + + // Auxiliary: carry[7] + // carry[i] = 1 if (base_address_lo + i+1) >= 2^32 + for i in 0..7 { + let overflows = base_addr_lo + (i as u64 + 1) >= (1u64 << 32); + table.set_bool(row_idx, cols::CARRY[i], overflows); + } + + // Auxiliary: old_timestamp[8] - each as DWordWL (2 words) + for i in 0..8 { + let cols_i = cols::old_timestamp(i); + table.set_dword_wl(row_idx, cols_i[0], op.old_timestamp[i]); + } + + // Multiplicity + table.set_bool(row_idx, cols::MU_READ, op.is_read); + table.set_bool(row_idx, cols::MU_WRITE, !op.is_read); + } + }) } // ========================================================================= diff --git a/prover/src/tables/memw_aligned.rs b/prover/src/tables/memw_aligned.rs index 47678f541..59bc2812c 100644 --- a/prover/src/tables/memw_aligned.rs +++ b/prover/src/tables/memw_aligned.rs @@ -41,6 +41,7 @@ use stark::trace::TraceTable; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::memw::MemwOperation; use super::types::{BusId, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; use crate::constraints::templates::emit_is_bit; @@ -90,45 +91,237 @@ pub mod cols { /// /// Reuses `MemwOperation` — the trace generator uses `old_timestamp[0]` /// (verified equal for all accessed bytes by the routing logic). -pub fn generate_memw_aligned_trace( - operations: &[MemwOperation], -) -> TraceTable { - let num_rows = operations.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in operations.iter().enumerate() { - table.set_bool(row_idx, cols::IS_REGISTER, op.is_register); +/// One MEMW_A row as the walk keeps it: 48 bytes where a [`MemwOperation`] +/// takes 152 (the walk materializes about one every three cycles). An aligned +/// access's bytes share one old timestamp ([`MemwOperation`]'s +/// `old_timestamp[0]`), and its eight value (old) elements fit one `u64`: eight +/// bytes for a memory access, the two 32-bit register halves (the rest zero) +/// for a register access that missed MEMW_R. `#[repr(C)]`: a chunk's rows are +/// the records the card fills its MEMW_A table from ([`card_rows`]). +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct AlignedRow { + base_address: u64, + timestamp: u64, + old_timestamp: u64, + value: u64, + old: u64, + width: u8, + is_read: bool, + is_register: bool, +} - table.set_dword_whh(row_idx, cols::BASE_ADDRESS[0], op.base_address); +impl AlignedRow { + /// The row of an op [`super::trace_builder`] routed to MEMW_A. + #[inline] + pub(crate) fn from_memw(op: &MemwOperation) -> Self { + let pack = |elements: &[u32; 8]| -> u64 { + if op.is_register { + elements[0] as u64 | (elements[1] as u64) << 32 + } else { + elements.iter().enumerate().fold(0, |packed, (i, &byte)| { + packed | (byte as u64 & 0xFF) << (8 * i) + }) + } + }; + let row = Self { + base_address: op.base_address, + timestamp: op.timestamp, + old_timestamp: op.old_timestamp[0], + value: pack(&op.value), + old: pack(&op.old), + width: op.width, + is_read: op.is_read, + is_register: op.is_register, + }; + debug_assert!( + row.value() == op.value && row.old() == op.old, + "a MEMW_A op's elements do not fit its row: {op:?}" + ); + row + } - for i in 0..8 { - table.set_u64(row_idx, cols::VALUE[i], op.value[i] as u64); + #[inline] + fn unpack(&self, packed: u64) -> [u32; 8] { + let mut elements = [0u32; 8]; + if self.is_register { + elements[0] = packed as u32; + elements[1] = (packed >> 32) as u32; + } else { + for (i, element) in elements.iter_mut().enumerate() { + *element = (packed >> (8 * i)) as u8 as u32; + } } + elements + } - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + /// [`MemwOperation::value`]. + #[inline] + pub(crate) fn value(&self) -> [u32; 8] { + self.unpack(self.value) + } - let (w2, w4, w8) = op.write_flags(); - table.set_bool(row_idx, cols::WRITE2, w2); - table.set_bool(row_idx, cols::WRITE4, w4); - table.set_bool(row_idx, cols::WRITE8, w8); + /// [`MemwOperation::old`]. + #[inline] + pub(crate) fn old(&self) -> [u32; 8] { + self.unpack(self.old) + } + + #[inline] + pub(crate) fn base_address(&self) -> u64 { + self.base_address + } + + #[inline] + pub(crate) fn timestamp(&self) -> u64 { + self.timestamp + } - for i in 0..8 { - table.set_u64(row_idx, cols::OLD[i], op.old[i] as u64); + /// The accessed bytes' (register words') shared old timestamp. + #[inline] + pub(crate) fn old_timestamp(&self) -> u64 { + self.old_timestamp + } + + #[inline] + pub(crate) fn width(&self) -> u8 { + self.width + } + + #[inline] + pub(crate) fn is_read(&self) -> bool { + self.is_read + } + + #[inline] + pub(crate) fn is_register(&self) -> bool { + self.is_register + } + + /// A row of these fields, the value and old value packed as the row keeps + /// them (a test's: the walk makes its rows with [`Self::from_memw`]). + #[cfg(test)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn from_parts( + base_address: u64, + timestamp: u64, + old_timestamp: u64, + value: u64, + old: u64, + width: u8, + is_read: bool, + is_register: bool, + ) -> Self { + Self { + base_address, + timestamp, + old_timestamp, + value, + old, + width, + is_read, + is_register, } + } - // Single old_timestamp (from old_timestamp[0], verified equal for all bytes) - table.set_dword_wl(row_idx, cols::OLD_TIMESTAMP_0, op.old_timestamp[0]); + /// A test's fault: the same row, its timestamp `cycles` later. + #[cfg(test)] + pub(crate) fn later(mut self, cycles: u64) -> Self { + self.timestamp += cycles; + self + } - table.set_bool(row_idx, cols::MU_READ, op.is_read); - table.set_bool(row_idx, cols::MU_WRITE, !op.is_read); + /// [`MemwOperation::write_flags`]. + #[inline] + pub(crate) fn write_flags(&self) -> (bool, bool, bool) { + match self.width { + 2 => (true, false, false), + 4 => (false, true, false), + 8 => (false, false, true), + _ => (false, false, false), + } } +} + +const _: () = assert!(std::mem::size_of::() == 48); + +/// Aligned rows as the card's MEMW_A fill reads them +/// (`multilinear::records::AlignedRow`): the same bytes. +pub(crate) fn card_rows(rows: &[AlignedRow]) -> &[multilinear::records::AlignedRow] { + use multilinear::records::AlignedRow as Card; + use std::mem::{align_of, offset_of, size_of}; + const _: () = { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(AlignedRow, base_address) == offset_of!(Card, base_address)); + assert!(offset_of!(AlignedRow, timestamp) == offset_of!(Card, timestamp)); + assert!(offset_of!(AlignedRow, old_timestamp) == offset_of!(Card, old_timestamp)); + assert!(offset_of!(AlignedRow, value) == offset_of!(Card, value)); + assert!(offset_of!(AlignedRow, old) == offset_of!(Card, old)); + assert!(offset_of!(AlignedRow, width) == offset_of!(Card, width)); + assert!(offset_of!(AlignedRow, is_read) == offset_of!(Card, is_read)); + assert!(offset_of!(AlignedRow, is_register) == offset_of!(Card, is_register)); + }; + // SAFETY: both are `#[repr(C)]` with the same fields in the same order + // (asserted above); the flags are `bool`s, 0 or 1, read as `u8`s, and no + // reader looks at the padding after them. + unsafe { std::slice::from_raw_parts(rows.as_ptr() as *const Card, rows.len()) } +} - trace +#[cfg(test)] +pub(crate) fn generate_memw_aligned_trace( + operations: &[AlignedRow], +) -> TraceTable { + generate_memw_aligned_trace_as(operations, TraceForm::Wide) +} + +/// The widths MEMW_A traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_memw_aligned_trace`] in `form` (`tables::gpack`). +pub(crate) fn generate_memw_aligned_trace_as( + operations: &[AlignedRow], + form: TraceForm, +) -> TraceTable { + generate_memw_aligned_trace_segments_as(&[operations], form) +} + +/// [`generate_memw_aligned_trace_as`] over `segments`, the operations one after another: a +/// chunk handed out as the window parts it lies in. +pub(crate) fn generate_memw_aligned_trace_segments_as( + segments: &[&[AlignedRow]], + form: TraceForm, +) -> TraceTable { + let len: usize = segments.iter().map(|s| s.len()).sum(); + let num_rows = len.next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in segments.iter().flat_map(|s| s.iter()).enumerate() { + table.set_bool(row_idx, cols::IS_REGISTER, op.is_register()); + + table.set_dword_whh(row_idx, cols::BASE_ADDRESS[0], op.base_address()); + + for (column, element) in cols::VALUE.into_iter().zip(op.value()) { + table.set_u64(row_idx, column, element as u64); + } + + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp()); + + let (w2, w4, w8) = op.write_flags(); + table.set_bool(row_idx, cols::WRITE2, w2); + table.set_bool(row_idx, cols::WRITE4, w4); + table.set_bool(row_idx, cols::WRITE8, w8); + + for (column, element) in cols::OLD.into_iter().zip(op.old()) { + table.set_u64(row_idx, column, element as u64); + } + + // Single old_timestamp (verified equal for all bytes when routed here) + table.set_dword_wl(row_idx, cols::OLD_TIMESTAMP_0, op.old_timestamp()); + + table.set_bool(row_idx, cols::MU_READ, op.is_read()); + table.set_bool(row_idx, cols::MU_WRITE, !op.is_read()); + } + }) } // ========================================================================= diff --git a/prover/src/tables/memw_register.rs b/prover/src/tables/memw_register.rs index 99d8819f9..f42fdc1d5 100644 --- a/prover/src/tables/memw_register.rs +++ b/prover/src/tables/memw_register.rs @@ -44,6 +44,7 @@ use stark::trace::TraceTable; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; use super::bitwise::{BitwiseHistogram, BitwiseOperation, BitwiseOperationType}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::memw::MemwOperation; use super::types::{BusId, GoldilocksExtension, GoldilocksField, VmTable}; use crate::constraints::templates::emit_is_bit; @@ -89,7 +90,7 @@ pub mod cols { /// Compact, already-decomposed record for one MEMW_R (register fast-path) access. /// /// This is the "direct-to-column" carrier: it holds exactly the fields the MEMW_R -/// column fill ([`generate_memw_register_trace_from_rows`]) and its IS_HALFWORD +/// column fill ([`generate_memw_register_trace_from_rows_as`]) and its IS_HALFWORD /// bitwise collector ([`collect_bitwise_from_memw_register`]) need, and nothing /// else. It replaces the full `MemwOperation` (~152 B after the `[u32; 8]` /// value/old shrink, but still 8-element arrays) for register accesses — the @@ -103,18 +104,63 @@ pub mod cols { /// - `old0/old1` = `old[0]`/`old[1]` /// - `old_ts_lo` = `old_timestamp[0] & 0xFFFF_FFFF` (the two words share old_timestamp, /// enforced by `is_register_op`; the upper limb is TIMESTAMP_1 = timestamp>>32) +/// +/// `#[repr(C)]`, its fields in this order with the one spare byte a named +/// zero, so its 32 bytes are all set and a chunk's rows can cross to a card as +/// they lie (`multilinear::records::RegisterRow`, the card's MEMW_R fill). #[derive(Debug, Clone, Copy)] +#[repr(C)] pub(crate) struct RegRow { - /// Register index 0..=255 (`base_address / 2`); u16 keeps the struct at - /// 32 bytes — it is the largest persisted array of the walk. - address: u16, timestamp: u64, val0: u32, val1: u32, old0: u32, old1: u32, old_ts_lo: u32, + /// Register index 0..=255 (`base_address / 2`); u16 keeps the struct at + /// 32 bytes — it is the largest persisted array of the walk. + address: u16, is_read: bool, + /// Always 0. + pad: u8, +} + +const _: () = { + assert!(std::mem::size_of::() == 32); + assert!(std::mem::align_of::() == 8); + assert!(std::mem::offset_of!(RegRow, timestamp) == 0); + assert!(std::mem::offset_of!(RegRow, val0) == 8); + assert!(std::mem::offset_of!(RegRow, val1) == 12); + assert!(std::mem::offset_of!(RegRow, old0) == 16); + assert!(std::mem::offset_of!(RegRow, old1) == 20); + assert!(std::mem::offset_of!(RegRow, old_ts_lo) == 24); + assert!(std::mem::offset_of!(RegRow, address) == 28); + assert!(std::mem::offset_of!(RegRow, is_read) == 30); + assert!(std::mem::offset_of!(RegRow, pad) == 31); +}; + +/// Register rows as the card's MEMW_R fill reads them +/// (`multilinear::records::RegisterRow`): the same bytes. +pub(crate) fn card_rows(rows: &[RegRow]) -> &[multilinear::records::RegisterRow] { + use multilinear::records::RegisterRow; + use std::mem::{align_of, offset_of, size_of}; + const _: () = { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(RegRow, timestamp) == offset_of!(RegisterRow, timestamp)); + assert!(offset_of!(RegRow, val0) == offset_of!(RegisterRow, val0)); + assert!(offset_of!(RegRow, val1) == offset_of!(RegisterRow, val1)); + assert!(offset_of!(RegRow, old0) == offset_of!(RegisterRow, old0)); + assert!(offset_of!(RegRow, old1) == offset_of!(RegisterRow, old1)); + assert!(offset_of!(RegRow, old_ts_lo) == offset_of!(RegisterRow, old_ts_lo)); + assert!(offset_of!(RegRow, address) == offset_of!(RegisterRow, address)); + assert!(offset_of!(RegRow, is_read) == offset_of!(RegisterRow, is_read)); + assert!(offset_of!(RegRow, pad) == offset_of!(RegisterRow, pad)); + }; + // SAFETY: both are `#[repr(C)]` with the same fields in the same order + // (asserted above); `is_read` is a `bool`, 0 or 1, read as a `u8`, and + // `pad` is always written. + unsafe { std::slice::from_raw_parts(rows.as_ptr() as *const RegisterRow, rows.len()) } } impl RegRow { @@ -155,6 +201,7 @@ impl RegRow { old1, old_ts_lo: (old_ts & 0xFFFF_FFFF) as u32, is_read, + pad: 0, } } @@ -189,6 +236,24 @@ impl RegRow { } } +#[cfg(test)] +impl RegRow { + /// The row as the card's MEMW_R fill reads it, field for field. + pub(crate) fn card_record(&self) -> math_cuda::fill::RegRowC { + math_cuda::fill::RegRowC { + timestamp: self.timestamp, + val0: self.val0, + val1: self.val1, + old0: self.old0, + old1: self.old1, + old_ts_lo: self.old_ts_lo, + address: self.address, + is_read: u8::from(self.is_read), + pad: 0, + } + } +} + /// Generates the MEMW_R trace table from register operations. /// /// Thin wrapper over [`generate_memw_register_trace_from_rows`] (via @@ -204,39 +269,55 @@ pub(crate) fn generate_memw_register_trace( generate_memw_register_trace_from_rows(&rows) } -/// The MEMW_R column fill from compact [`RegRow`]s. This is the single source of -/// truth for the MEMW_R trace layout; both the walk's direct fast path and the -/// `MemwOperation`-based `generate_memw_register_trace` test wrapper land here. +/// [`generate_memw_register_trace_from_rows_as`] at 8 bytes a cell. +#[cfg(test)] pub(crate) fn generate_memw_register_trace_from_rows( rows: &[RegRow], ) -> TraceTable { - let num_rows = rows.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, r) in rows.iter().enumerate() { - // ADDRESS = base_address / 2 (already divided in RegRow). - table.set_u64(row_idx, cols::ADDRESS, r.address as u64); - // Timestamp split into lo/hi 32-bit words. - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, r.timestamp); - // Value: registers are DWordWL = 2 words. - table.set_u64(row_idx, cols::VAL_0, r.val0 as u64); - table.set_u64(row_idx, cols::VAL_1, r.val1 as u64); - // Old value. - table.set_u64(row_idx, cols::OLD_0, r.old0 as u64); - table.set_u64(row_idx, cols::OLD_1, r.old1 as u64); - // Old timestamp low (upper limb shared with TIMESTAMP_1). - table.set_u64(row_idx, cols::OLD_TIMESTAMP_LO, r.old_ts_lo as u64); - // Multiplicity. - table.set_bool(row_idx, cols::MU_READ, r.is_read); - table.set_bool(row_idx, cols::MU_WRITE, !r.is_read); - } + generate_memw_register_trace_from_rows_as(rows, TraceForm::Wide) +} + +/// The widths MEMW_R traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - trace +/// The MEMW_R column fill from compact [`RegRow`]s, in `form` +/// (`tables::gpack`). This is the single source of truth for the MEMW_R trace +/// layout; both the walk's direct fast path and the `MemwOperation`-based +/// `generate_memw_register_trace` test wrapper land here. +pub(crate) fn generate_memw_register_trace_from_rows_as( + rows: &[RegRow], + form: TraceForm, +) -> TraceTable { + generate_memw_register_trace_from_rows_segments_as(&[rows], form) +} + +/// [`generate_memw_register_trace_from_rows_as`] over `segments`, the rows one after another: a +/// chunk handed out as the window parts it lies in. +pub(crate) fn generate_memw_register_trace_from_rows_segments_as( + segments: &[&[RegRow]], + form: TraceForm, +) -> TraceTable { + let len: usize = segments.iter().map(|s| s.len()).sum(); + let num_rows = len.next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, r) in segments.iter().flat_map(|s| s.iter()).enumerate() { + // ADDRESS = base_address / 2 (already divided in RegRow). + table.set_u64(row_idx, cols::ADDRESS, r.address as u64); + // Timestamp split into lo/hi 32-bit words. + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, r.timestamp); + // Value: registers are DWordWL = 2 words. + table.set_u64(row_idx, cols::VAL_0, r.val0 as u64); + table.set_u64(row_idx, cols::VAL_1, r.val1 as u64); + // Old value. + table.set_u64(row_idx, cols::OLD_0, r.old0 as u64); + table.set_u64(row_idx, cols::OLD_1, r.old1 as u64); + // Old timestamp low (upper limb shared with TIMESTAMP_1). + table.set_u64(row_idx, cols::OLD_TIMESTAMP_LO, r.old_ts_lo as u64); + // Multiplicity. + table.set_bool(row_idx, cols::MU_READ, r.is_read); + table.set_bool(row_idx, cols::MU_WRITE, !r.is_read); + } + }) } /// The single IS_HALFWORD lookup a MEMW_R access sends: proves the timestamp delta diff --git a/prover/src/tables/mod.rs b/prover/src/tables/mod.rs index 97d46b90d..491728ccd 100644 --- a/prover/src/tables/mod.rs +++ b/prover/src/tables/mod.rs @@ -22,7 +22,10 @@ pub mod types; +pub mod gpack; + pub mod bitwise; +pub mod blake3; pub mod branch; pub mod bytewise; pub mod copy; @@ -31,6 +34,7 @@ pub mod cpu32; pub mod decode; pub mod dvrm; pub mod ecdas; +pub mod ecrec; pub mod ecsm; pub mod eq; pub mod global_memory; @@ -52,6 +56,15 @@ pub mod register; pub mod shift; pub mod store; pub mod trace_builder; +pub mod trace_hash; + +/// The card fill's records, synthetic sets and host known-answer vectors. +#[cfg(test)] +mod card_fill_inputs; +/// The card's table fills against the host's, word for word, and timed: a +/// box test (`--ignored`), with a card. +#[cfg(all(test, feature = "cuda", feature = "parallel"))] +mod card_fill_tests; pub use types::BusId; @@ -62,6 +75,23 @@ pub use types::BusId; /// silently skip a test. pub const STATIC_BLOWUP_FACTORS: &[u8] = &[2, 4, 8]; +/// Blowup factors for which the ONE-ROW (S2) twins of those static +/// commitments ship (`static_commitment_one_row` and the page twins), emitted +/// by `compute_static_commitments --layout row` and pinned by the one-row drift +/// tests. Only the blowup the knob is measured at (4 for +/// the base and for the LFM chips): under one row any other blowup is a hard +/// miss, never a recompute. +pub const STATIC_BLOWUP_FACTORS_ONE_ROW: &[u8] = &[4]; + +/// Blowup factors for which the POSEIDON1 twins of the bitwise and keccak_rc +/// static commitments ship (`static_commitment_p1`), emitted by +/// `compute_static_commitments --hash p1` and pinned by the Poseidon1 drift +/// tests. Only the blowup the block's proof family commits its LFM tables at +/// (4), in row pairs: the one key its provers and verifiers ask for. Any +/// other key under Poseidon1 is computed +/// ([`crate::hash_pin::static_root_under`]). +pub const STATIC_BLOWUP_FACTORS_P1: &[u8] = &[4]; + /// Per-table maximum rows, sized so each chunk uses roughly the same memory. /// /// Effective width = main_cols + 3 × bus_interactions (extension field = 3× cost). @@ -124,8 +154,44 @@ pub struct MaxRowsConfig { pub cpu32: usize, } +/// The uniform table cap this process proves at, or `None` for the production +/// per-table values. +/// +/// ★ ONE READER, AND IT IS THIS ONE. [`MaxRowsConfig::default`] is what decides +/// how an epoch is chunked, so anything that wants to describe the posture a +/// run was proven at has to ask the same question the same way. A second parse +/// of `LAMBDA_VM_MAX_ROWS_LOG2` somewhere else is how a label comes to name a +/// posture the epochs were not chunked at — and the transcript pin is exactly +/// such a label, since its counts ARE the chunking. +pub(crate) fn max_rows_log2_override() -> Option { + let v = std::env::var("LAMBDA_VM_MAX_ROWS_LOG2").ok()?; + let n: u32 = v + .parse() + .expect("LAMBDA_VM_MAX_ROWS_LOG2 must be an integer"); + assert!( + (5..=26).contains(&n), + "LAMBDA_VM_MAX_ROWS_LOG2 must be in 5..=26, got {n}" + ); + Some(n) +} + impl Default for MaxRowsConfig { + /// The production values from [`max_rows`], unless + /// `LAMBDA_VM_MAX_ROWS_LOG2` overrides them with one uniform cap. + /// + /// The env knob is a prover-side SHAPE choice, like `TABLE_PARALLELISM` is + /// a resource one: chunk counts already ride the statement (the verifier + /// reads them from the proof it checks, never from this config), so two + /// provers with different caps produce differently-chunked but equally + /// verifiable epochs. It exists for compression-posture measurement — the + /// production 2^19/2^20 values are sized for equal-memory parallel chunks, + /// which multiplies SUB-PROOFS per epoch, and every extra sub-proof is a + /// leg the recursion wrap pays for. Tall-table postures (2^24) trade chunk + /// parallelism for fewer legs. fn default() -> Self { + if let Some(n) = max_rows_log2_override() { + return Self::uniform(1 << n); + } Self { cpu: max_rows::CPU, memw: max_rows::MEMW, @@ -146,6 +212,26 @@ impl Default for MaxRowsConfig { } impl MaxRowsConfig { + /// One cap for every table — the tall-table posture the env override uses. + pub fn uniform(rows: usize) -> Self { + Self { + cpu: rows, + memw: rows, + memw_aligned: rows, + dvrm: rows, + mul: rows, + lt: rows, + shift: rows, + load: rows, + branch: rows, + memw_register: rows, + eq: rows, + bytewise: rows, + store: rows, + cpu32: rows, + } + } + /// Small limits for low-memory testing. Generates multiple chunks /// per table even for tiny programs (~32 rows per chunk). pub fn small() -> Self { diff --git a/prover/src/tables/mul.rs b/prover/src/tables/mul.rs index 8b9ee5865..086cccad2 100644 --- a/prover/src/tables/mul.rs +++ b/prover/src/tables/mul.rs @@ -34,8 +34,9 @@ use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; -use std::collections::HashMap; +use super::trace_hash::{OpMap, trace_hash_state}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{ BusId, GoldilocksExtension, GoldilocksField, INV_2_32, INV_2_64, INV_2_96, INV_2_128, MU_MAX, NEG_INV_2_16, NEG_INV_2_32, NEG_INV_2_48, NEG_INV_2_64, NEG_INV_2_80, NEG_INV_2_96, @@ -295,7 +296,7 @@ impl MulOperation { pub fn dedup_mul_rows( operations: &[(MulOperation, bool)], ) -> Vec<(MulOperation, MulMultiplicities)> { - let mut op_map: HashMap = HashMap::new(); + let mut op_map: OpMap = OpMap::with_hasher(trace_hash_state()); for (op, wants_hi) in operations { let entry = op_map.entry(op.clone()).or_default(); if *wants_hi { @@ -329,51 +330,55 @@ pub fn dedup_mul_rows( /// * `operations` - List of (MulOperation, wants_hi) pairs pub fn generate_mul_trace( operations: &[(MulOperation, bool)], +) -> TraceTable { + generate_mul_trace_as(operations, TraceForm::Wide) +} + +/// The widths MUL traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_mul_trace`] in `form` (`tables::gpack`). +pub fn generate_mul_trace_as( + operations: &[(MulOperation, bool)], + form: TraceForm, ) -> TraceTable { let unique_ops = dedup_mul_rows(operations); let num_rows = unique_ops.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, (op, multiplicities)) in unique_ops.iter().enumerate() { - // Compute product - let (lo, hi) = op.compute_product(); - - // Fill lhs as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::LHS_0, op.lhs); - table.set_bool(row_idx, cols::LHS_SIGNED, op.lhs_signed); - - // Fill rhs as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::RHS_0, op.rhs); - table.set_bool(row_idx, cols::RHS_SIGNED, op.rhs_signed); - - // Fill lo as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::LO_0, lo); - - // Fill hi as DWordHL (4 halfwords) - table.set_dword_hl(row_idx, cols::HI_0, hi); - - // Fill auxiliary columns - table.set_bool(row_idx, cols::LHS_IS_NEGATIVE, op.lhs_is_negative()); - table.set_bool(row_idx, cols::RHS_IS_NEGATIVE, op.rhs_is_negative()); - - // Fill raw_product columns - let raw = op.compute_raw_products(); - table.set_u64(row_idx, cols::RAW_PRODUCT_0, raw[0]); - table.set_u64(row_idx, cols::RAW_PRODUCT_1, raw[1]); - table.set_u64(row_idx, cols::RAW_PRODUCT_2, raw[2]); - table.set_u64(row_idx, cols::RAW_PRODUCT_3, raw[3]); - - // Fill multiplicities (ALU bus, lo/hi) - table.set_u64(row_idx, cols::MU_LO, multiplicities.mu_lo); - table.set_u64(row_idx, cols::MU_HI, multiplicities.mu_hi); - } - - trace + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, (op, multiplicities)) in unique_ops.iter().enumerate() { + // Compute product + let (lo, hi) = op.compute_product(); + + // Fill lhs as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::LHS_0, op.lhs); + table.set_bool(row_idx, cols::LHS_SIGNED, op.lhs_signed); + + // Fill rhs as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::RHS_0, op.rhs); + table.set_bool(row_idx, cols::RHS_SIGNED, op.rhs_signed); + + // Fill lo as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::LO_0, lo); + + // Fill hi as DWordHL (4 halfwords) + table.set_dword_hl(row_idx, cols::HI_0, hi); + + // Fill auxiliary columns + table.set_bool(row_idx, cols::LHS_IS_NEGATIVE, op.lhs_is_negative()); + table.set_bool(row_idx, cols::RHS_IS_NEGATIVE, op.rhs_is_negative()); + + // Fill raw_product columns + let raw = op.compute_raw_products(); + table.set_u64(row_idx, cols::RAW_PRODUCT_0, raw[0]); + table.set_u64(row_idx, cols::RAW_PRODUCT_1, raw[1]); + table.set_u64(row_idx, cols::RAW_PRODUCT_2, raw[2]); + table.set_u64(row_idx, cols::RAW_PRODUCT_3, raw[3]); + + // Fill multiplicities (ALU bus, lo/hi) + table.set_u64(row_idx, cols::MU_LO, multiplicities.mu_lo); + table.set_u64(row_idx, cols::MU_HI, multiplicities.mu_hi); + } + }) } // ========================================================================= diff --git a/prover/src/tables/page.rs b/prover/src/tables/page.rs index 6788bee08..ac53351d3 100644 --- a/prover/src/tables/page.rs +++ b/prover/src/tables/page.rs @@ -33,8 +33,8 @@ use std::collections::HashMap; use math::polynomial::Polynomial; -use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed}; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::proof::options::ProofOptions; use stark::prover::evaluate_polynomial_on_lde_domain; @@ -411,19 +411,36 @@ pub fn generate_page_trace_from_dense( pub(crate) fn static_zero_page_commitment(blowup_factor: u8) -> Option { match blowup_factor { 2 => Some([ - 0x7d, 0x74, 0x85, 0xf0, 0x2b, 0x74, 0xe0, 0x3f, 0x14, 0x99, 0xb3, 0xa0, 0x5f, 0x1d, - 0x6e, 0xf2, 0x21, 0xff, 0xaf, 0x24, 0x7e, 0x30, 0xb0, 0xda, 0x48, 0x79, 0xe1, 0x43, - 0xee, 0xea, 0x6a, 0x0f, + 0x3b, 0x8d, 0xfb, 0xfc, 0xab, 0xcb, 0x13, 0x98, 0x1b, 0xec, 0xe7, 0x72, 0x38, 0x50, + 0xcc, 0xd1, 0x9a, 0x95, 0xe6, 0xcc, 0x6c, 0xf7, 0xf0, 0x4a, 0x49, 0xc9, 0x5d, 0x52, + 0x42, 0x2b, 0x8c, 0x4b, ]), 4 => Some([ - 0x5c, 0xcc, 0x5b, 0xb1, 0xe8, 0x11, 0x91, 0x81, 0xbd, 0xdd, 0x39, 0x40, 0x77, 0x87, - 0xdc, 0x98, 0x06, 0x06, 0x8c, 0x63, 0xcd, 0xfd, 0xf1, 0xda, 0x4a, 0x55, 0x31, 0x4d, - 0x6a, 0x16, 0x18, 0xd0, + 0x0b, 0x95, 0xb7, 0xe1, 0x62, 0xe4, 0xec, 0x59, 0xd5, 0xb7, 0x41, 0x7f, 0xc6, 0xe7, + 0x03, 0x7c, 0xee, 0x3c, 0xc2, 0x10, 0x29, 0xea, 0xbf, 0xae, 0x5a, 0xfd, 0x5e, 0x5d, + 0x00, 0x0b, 0x41, 0xdb, ]), 8 => Some([ - 0xf0, 0xc0, 0x69, 0xed, 0xf8, 0x59, 0xd6, 0x56, 0x15, 0x3c, 0x2f, 0x93, 0x65, 0xd6, - 0xe9, 0xe9, 0x8e, 0xd1, 0x83, 0x94, 0xf9, 0x75, 0x59, 0xd1, 0xec, 0x16, 0xe1, 0x37, - 0xd5, 0x32, 0xd6, 0xd9, + 0x20, 0x86, 0x24, 0x23, 0x30, 0x0a, 0xea, 0xdc, 0x58, 0x37, 0x88, 0xf7, 0x33, 0x39, + 0x89, 0x2c, 0xf5, 0x2b, 0xfa, 0x89, 0x8b, 0x06, 0x60, 0x4d, 0x6f, 0x21, 0xc7, 0xfb, + 0x6f, 0xcb, 0xac, 0x84, + ]), + _ => None, + } +} + +/// The ONE-ROW (S2) twin of [`static_zero_page_commitment`]: the same columns committed with one +/// LDE row per leaf ([`LeafLayout::Row`]), per `blowup_factor` in +/// [`crate::tables::STATIC_BLOWUP_FACTORS_ONE_ROW`], generated by +/// `compute_static_commitments --layout row` and pinned by the one-row drift +/// test. The same regeneration rules as [`static_zero_page_commitment`]. A blowup with no arm here +/// is a hard miss under one row: no recompute. +pub(crate) fn static_zero_page_commitment_one_row(blowup_factor: u8) -> Option { + match blowup_factor { + 4 => Some([ + 0x19, 0x19, 0x77, 0x25, 0x76, 0x36, 0xfc, 0x2e, 0xa9, 0xaf, 0xb5, 0x0a, 0x11, 0x93, + 0xe7, 0x8f, 0xe2, 0x58, 0x38, 0x7a, 0x36, 0x4d, 0xf5, 0xad, 0x72, 0x91, 0x2d, 0x43, + 0xee, 0xca, 0xfa, 0xe4, ]), _ => None, } @@ -437,19 +454,36 @@ pub(crate) fn static_zero_page_commitment(blowup_factor: u8) -> Option Option { match blowup_factor { 2 => Some([ - 0x4a, 0x36, 0x1a, 0x29, 0x02, 0xc8, 0x21, 0x8e, 0xc0, 0xfd, 0x6d, 0xbe, 0xb3, 0x5f, - 0x70, 0x54, 0xcb, 0xa3, 0xa7, 0x8c, 0xa2, 0x37, 0xdc, 0xa3, 0x51, 0x29, 0xd8, 0xb8, - 0x94, 0x2d, 0x91, 0x3d, + 0x04, 0x31, 0x10, 0xb3, 0x61, 0xbd, 0xd4, 0x6d, 0x92, 0x65, 0x0e, 0x2d, 0xc9, 0xe6, + 0x1d, 0x20, 0x78, 0xc3, 0xd6, 0xc6, 0x8f, 0x12, 0xb0, 0x45, 0xf0, 0x76, 0x68, 0x27, + 0x5e, 0x2b, 0xa7, 0xbd, ]), 4 => Some([ - 0xa6, 0x53, 0x01, 0xd0, 0x2f, 0x47, 0xca, 0xe8, 0x7a, 0xbd, 0xb7, 0x14, 0x69, 0x28, - 0xaf, 0x67, 0xc9, 0xe5, 0x2d, 0xd6, 0x41, 0x5f, 0x76, 0xd8, 0xc4, 0x59, 0xdd, 0xaa, - 0xd2, 0x32, 0x1f, 0x6f, + 0xef, 0x71, 0xa3, 0x11, 0x4a, 0xb3, 0xf7, 0xd6, 0x77, 0xe4, 0x79, 0x0e, 0xd1, 0x96, + 0xf3, 0x36, 0x19, 0x70, 0x6b, 0xd7, 0x2e, 0x04, 0x86, 0x13, 0xfd, 0x81, 0xd1, 0xf4, + 0x26, 0xca, 0xd2, 0x41, ]), 8 => Some([ - 0xe7, 0x13, 0xe3, 0x59, 0xd6, 0xa5, 0xb9, 0xd5, 0xfa, 0xcb, 0x51, 0x8a, 0x42, 0x52, - 0xaa, 0x25, 0xf9, 0x0d, 0x94, 0xf5, 0xdf, 0x93, 0x56, 0x63, 0x77, 0x2c, 0x08, 0x75, - 0xb7, 0x68, 0xb0, 0x57, + 0xf7, 0x2d, 0x07, 0xc5, 0x84, 0x5c, 0xc8, 0x92, 0x79, 0x75, 0xf3, 0x56, 0xb7, 0xf9, + 0xe5, 0xd5, 0xb5, 0x8f, 0x62, 0x63, 0x36, 0x8f, 0xd0, 0x8c, 0x42, 0xce, 0xfa, 0x2c, + 0xd2, 0x79, 0xd2, 0x8e, + ]), + _ => None, + } +} + +/// The ONE-ROW (S2) twin of [`static_private_page_commitment`]: the same columns committed with one +/// LDE row per leaf ([`LeafLayout::Row`]), per `blowup_factor` in +/// [`crate::tables::STATIC_BLOWUP_FACTORS_ONE_ROW`], generated by +/// `compute_static_commitments --layout row` and pinned by the one-row drift +/// test. The same regeneration rules as [`static_private_page_commitment`]. A blowup with no arm here +/// is a hard miss under one row: no recompute. +pub(crate) fn static_private_page_commitment_one_row(blowup_factor: u8) -> Option { + match blowup_factor { + 4 => Some([ + 0x59, 0x6a, 0x3c, 0xc9, 0x79, 0x61, 0x9d, 0x33, 0xa2, 0xcc, 0xfc, 0xba, 0xc3, 0xd9, + 0x6f, 0x84, 0xb0, 0x4a, 0x48, 0x88, 0x87, 0x5d, 0x37, 0x18, 0xa3, 0xfb, 0xd7, 0xce, + 0x2f, 0x5d, 0x96, 0x39, ]), _ => None, } @@ -464,33 +498,62 @@ pub(crate) fn static_private_page_commitment(blowup_factor: u8) -> Option Commitment { - let page_size = DEFAULT_PAGE_SIZE; - let num_rows = page_size; + compute_precomputed_commitment_with(config, options, LeafLayout::RowPair) +} - // Precomputed columns: OFFSET and INIT. - // - // OFFSET (col 0): deterministic row index 0..page_size-1, the same for every - // page of a given size regardless of the program being proven. - // - // INIT (col 1): the initial byte value at each offset. For zero-init pages - // (stack, heap, BSS) this is all zeros. For ELF data pages it holds the - // bytes loaded from the binary. Either way the column is fully determined - // before execution, so the verifier can check it against a preprocessed - // commitment instead of including it in the main trace. - let mut offset_col = crate::tables::types::zeroed_fe_vec(num_rows); - let mut init_col = crate::tables::types::zeroed_fe_vec(num_rows); +/// [`compute_precomputed_commitment`] under an explicit trace-tree leaf layout +/// (S2). ELF data pages have no static root, so a one-row data page computes +/// this at run time. +pub fn compute_precomputed_commitment_with( + config: &PageConfig, + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { + commit_preprocessed_columns( + &preprocessed_columns(config), + DEFAULT_PAGE_SIZE, + options, + layout, + ) +} + +/// The precomputed columns themselves, `DEFAULT_PAGE_SIZE` tall. +/// +/// The multilinear path checks a proof's claimed openings against these instead +/// of comparing a commitment, so it needs the values and not just their root. +/// +/// OFFSET (col 0): deterministic row index 0..page_size-1, the same for every +/// page of a given size regardless of the program being proven. +/// +/// INIT (col 1): the initial byte value at each offset. For zero-init pages +/// (stack, heap, BSS) this is all zeros. For ELF data pages it holds the +/// bytes loaded from the binary. Either way the column is fully determined +/// before execution, so the verifier can check it rather than trust it. +pub fn preprocessed_columns(config: &PageConfig) -> Vec> { + let page_size = DEFAULT_PAGE_SIZE; + let mut init_col = crate::tables::types::zeroed_fe_vec(page_size); - for i in 0..page_size { - offset_col[i] = FE::from(i as u64); + for (i, slot) in init_col.iter_mut().enumerate() { let init_byte = config .init_values .as_ref() .and_then(|v| v.get(i).copied()) .unwrap_or(0); - init_col[i] = FE::from(init_byte as u64); + *slot = FE::from(init_byte as u64); } - commit_preprocessed_columns(&[offset_col, init_col], num_rows, options) + vec![offset_column(), init_col] +} + +/// OFFSET alone: the dense `0..page_size-1` enumeration, byte-identical for +/// every page regardless of program or input. +/// +/// A **private-input** page's INIT holds the private input and stays a main +/// column nobody recomputes, but its OFFSET is still preprocessed and still has +/// to be pinned — nothing else constrains it, and a row pointed at another +/// address forges a second memory history for it. +pub fn offset_column() -> Vec { + (0..DEFAULT_PAGE_SIZE as u64).map(FE::from).collect() } /// LDE + Merkle-commit a set of preprocessed PAGE columns. Shared by @@ -502,6 +565,7 @@ fn commit_preprocessed_columns( columns: &[Vec], num_rows: usize, options: &ProofOptions, + layout: LeafLayout, ) -> Commitment { let polys: Vec> = columns .iter() @@ -521,9 +585,13 @@ fn commit_preprocessed_columns( }) .collect(); - let (_, root) = commit_bit_reversed(&lde_columns, ROWS_PER_LEAF) - .expect("Failed to build Merkle tree for page LDE"); - root + // ★ Through the LFM commit helper, which commits under the BLOCK PATH's pin + // rather than `stark`'s default aliases. This root is a PREPROCESSED + // commitment the block path's verifier absorbs, so it has to be built with + // the hash that path commits under — on a branch that pins an algebraic + // hash, a root left on the alias would be the one BLAKE3 artifact in an RPO + // proof, and it would fail as a root nothing reconstructs. + crate::lfm::commit::commit_lde_columns_with(&lde_columns, layout) } /// Commitment over the OFFSET column **alone** — the preprocessed anchor for @@ -540,12 +608,20 @@ fn commit_preprocessed_columns( /// Memory-bus address is `page_base_lo + OFFSET`, so a free OFFSET names an /// arbitrary address and forges that address's memory history. pub fn compute_offset_only_commitment(options: &ProofOptions) -> Commitment { + compute_offset_only_commitment_with(options, LeafLayout::RowPair) +} + +/// [`compute_offset_only_commitment`] under an explicit leaf layout (S2). +pub fn compute_offset_only_commitment_with( + options: &ProofOptions, + layout: LeafLayout, +) -> Commitment { let num_rows = DEFAULT_PAGE_SIZE; let mut offset_col = crate::tables::types::zeroed_fe_vec(num_rows); for (i, cell) in offset_col.iter_mut().enumerate() { *cell = FE::from(i as u64); } - commit_preprocessed_columns(&[offset_col], num_rows, options) + commit_preprocessed_columns(&[offset_col], num_rows, options, layout) } /// Returns the zero-init PAGE preprocessed commitment. @@ -596,6 +672,85 @@ pub fn private_page_preprocessed_commitment(options: &ProofOptions) -> Commitmen compute_offset_only_commitment(options) } +/// The zero-init PAGE commitment under the table's resolved leaf `layout`: +/// today's [`zero_init_preprocessed_commitment`] for row pairs; for one row the +/// static twin at coset 3, and `None` otherwise (a hard miss, never a recompute). +pub fn zero_init_preprocessed_commitment_for( + options: &ProofOptions, + layout: LeafLayout, +) -> Option { + match layout { + LeafLayout::RowPair => Some(zero_init_preprocessed_commitment(options)), + LeafLayout::Row => (options.coset_offset == 3) + .then(|| static_zero_page_commitment_one_row(options.blowup_factor)) + .flatten(), + } +} + +/// The private-input PAGE commitment under the table's resolved leaf `layout` +/// (see [`zero_init_preprocessed_commitment_for`]). +pub fn private_page_preprocessed_commitment_for( + options: &ProofOptions, + layout: LeafLayout, +) -> Option { + match layout { + LeafLayout::RowPair => Some(private_page_preprocessed_commitment(options)), + LeafLayout::Row => (options.coset_offset == 3) + .then(|| static_private_page_commitment_one_row(options.blowup_factor)) + .flatten(), + } +} + +/// The zero-init page's commitment source for both leaf layouts. +pub fn zero_init_lazy_commitment(options: &ProofOptions) -> stark::lookup::LazyCommitment { + zero_init_lazy_commitment_from(zero_init_preprocessed_commitment(options), options) +} + +/// [`zero_init_lazy_commitment`] with the row-pair root already in hand. +pub fn zero_init_lazy_commitment_from( + row_pair: Commitment, + options: &ProofOptions, +) -> stark::lookup::LazyCommitment { + let o = options.clone(); + stark::lookup::LazyCommitment::ready(row_pair) + .with_one_row(move || zero_init_preprocessed_commitment_for(&o, LeafLayout::Row)) +} + +/// The private-input page's (OFFSET-only) commitment source for both layouts. +pub fn private_page_lazy_commitment(options: &ProofOptions) -> stark::lookup::LazyCommitment { + let o = options.clone(); + stark::lookup::LazyCommitment::ready(private_page_preprocessed_commitment(options)) + .with_one_row(move || private_page_preprocessed_commitment_for(&o, LeafLayout::Row)) +} + +/// An ELF data page's commitment source: the row-pair root `supplied` by the +/// caller or computed on first use, and the one-row root computed on first use +/// (program-dependent, so there is no static twin; a supplied root is a +/// row-pair root and never stands in for the other layout). +pub fn data_page_lazy_commitment( + config: &PageConfig, + options: &ProofOptions, + supplied: Option, +) -> stark::lookup::LazyCommitment { + let base = match supplied { + Some(c) => stark::lookup::LazyCommitment::ready(c), + None => { + let (config, options) = (config.clone(), options.clone()); + stark::lookup::LazyCommitment::deferred(move || { + compute_precomputed_commitment(&config, &options) + }) + } + }; + let (config, options) = (config.clone(), options.clone()); + base.with_one_row(move || { + Some(compute_precomputed_commitment_with( + &config, + &options, + LeafLayout::Row, + )) + }) +} + // ========================================================================= // Bus interactions // ========================================================================= diff --git a/prover/src/tables/register.rs b/prover/src/tables/register.rs index 93b8742f4..0f41dc3d6 100644 --- a/prover/src/tables/register.rs +++ b/prover/src/tables/register.rs @@ -28,8 +28,8 @@ use std::collections::HashMap; use math::polynomial::Polynomial; -use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed}; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; use stark::proof::options::ProofOptions; use stark::prover::evaluate_polynomial_on_lde_domain; @@ -341,19 +341,36 @@ pub fn fini_from_final_state(final_state: &FinalRegisterStateMap, init: &[u32]) // Preprocessed commitment // ========================================================================= -/// Computes the Merkle root commitment over the LDE of the REGISTER preprocessed -/// columns: OFFSET (the Word address: 0..63 for x0-x31, 508-509 for x254, 510-511 for -/// x255), INIT, FINI and IS_PC, so the verifier recomputes it from the public `init` -/// and `fini` and the proof's FINI column is locked to `fini`. `fini` is the vector -/// produced by `fini_from_trace` (entry `i` = the register at -/// `register_word_address_list()[i]`). Must match the column order of the REGISTER -/// trace (OFFSET, INIT, FINI, IS_PC), and FINI and IS_PC on padding rows are 0 (as -/// the trace builds them). -pub fn compute_precomputed_commitment( - options: &ProofOptions, - init: &[u32], - fini: &[u32], -) -> Commitment { +/// The preprocessed columns themselves: OFFSET (the Word address: 0..63 for +/// x0-x31, 508-509 for x254, 510-511 for x255), INIT, FINI and IS_PC, padded to a +/// power of two. FINI and IS_PC are 0 on padding rows, as the trace builds them. +/// +/// ★ THE COLUMNS, NOT ONLY THEIR ROOT, because the two verifiers bind different +/// things. The univariate verifier compares [`compute_precomputed_commitment`]'s +/// root against the proof's precomputed tree (`stark::verifier`, the +/// `is_preprocessed()` branch). The multilinear verifier has no such tree: it +/// checks the claimed openings of columns `0..num_precomputed_columns()` against +/// the values the program implies (`stark::multilinear_table::check_preprocessed`), +/// and it reads them through [`stark::traits::AIR::precomputed_columns`]. An AIR +/// built with a commitment and no columns closure hands that check an EMPTY list, +/// which it walks in zero iterations — so this function is what makes INIT and +/// FINI binding on the multilinear path at all. +/// +/// `fini` is the vector produced by `fini_from_trace` (entry `i` = the register +/// at `register_word_address_list()[i]`): a continuation epoch's `R_{i+1}`, or a +/// monolithic proof's (and a block's) halted file ([`halted_fini`]). +/// +/// # ★ Why the epoch statement does not absorb `init` or `fini`, and must not +/// +/// The chain of custody is already closed without an absorb. The commitment +/// ROOTS go into the transcript before any challenge is drawn, so the FINI +/// column is fixed before the prover learns the reduced point; `check_preprocessed` +/// then binds the verifier's own `(init, fini)` to that committed column AT that +/// point. A prover therefore cannot choose the claim after the fact: the column +/// is committed first and the value it is compared against is the verifier's, +/// not the proof's. Adding these vectors to `absorb_epoch` would bind nothing +/// further and would move every pinned transcript constant. +pub fn preprocessed_columns(init: &[u32], fini: &[u32]) -> Vec> { debug_assert_eq!(fini.len(), NUM_REGISTER_ADDRESSES); let num_rows = NUM_REGISTER_ADDRESSES.next_power_of_two(); let addr_list = register_word_address_list(); @@ -366,16 +383,44 @@ pub fn compute_precomputed_commitment( for i in 0..NUM_REGISTER_ADDRESSES { offset_col[i] = FE::from(addr_list[i]); init_col[i] = FE::from(init.get(i).copied().unwrap_or(0) as u64); - fini_col[i] = FE::from(fini[i] as u64); + fini_col[i] = FE::from(fini.get(i).copied().unwrap_or(0) as u64); is_pc_col[i] = FE::from(u64::from(is_pc_address(addr_list[i]))); } - commit_register_columns(options, vec![offset_col, init_col, fini_col, is_pc_col]) + vec![offset_col, init_col, fini_col, is_pc_col] +} + +/// Computes the Merkle root commitment over the LDE of the REGISTER preprocessed +/// columns ([`preprocessed_columns`]), so the univariate verifier recomputes it +/// from the public `init` and `fini` and the proof's FINI column is locked to +/// `fini`. The multilinear path never reads this root. +pub fn compute_precomputed_commitment( + options: &ProofOptions, + init: &[u32], + fini: &[u32], +) -> Commitment { + compute_precomputed_commitment_with(options, init, fini, LeafLayout::RowPair) +} + +/// [`compute_precomputed_commitment`] under an explicit leaf layout (S2) — the +/// host twin of the in-circuit register commitment +/// (`lfm::programs::emit_register_commitment` at the same `rows_per_leaf`). +pub fn compute_precomputed_commitment_with( + options: &ProofOptions, + init: &[u32], + fini: &[u32], + layout: LeafLayout, +) -> Commitment { + commit_register_columns(options, preprocessed_columns(init, fini), layout) } /// LDE + bit-reverse + Merkle-commit the given preprocessed columns (in column /// order). -fn commit_register_columns(options: &ProofOptions, columns: Vec>) -> Commitment { +fn commit_register_columns( + options: &ProofOptions, + columns: Vec>, + layout: LeafLayout, +) -> Commitment { let num_rows = NUM_REGISTER_ADDRESSES.next_power_of_two(); let polys: Vec> = columns .iter() @@ -395,9 +440,32 @@ fn commit_register_columns(options: &ProofOptions, columns: Vec>) -> Com }) .collect(); - let (_, root) = commit_bit_reversed(&lde_columns, ROWS_PER_LEAF) - .expect("Failed to build Merkle tree for register LDE"); - root + // ★ Through the LFM commit helper, which commits under the BLOCK PATH's pin + // rather than `stark`'s default aliases. This root is a PREPROCESSED + // commitment the prover recomputes and compares against, so building it with + // a different hash than the path commits under fails at prove time with + // `PrecomputedCommitmentMismatch` — which is exactly how it was found. + crate::lfm::commit::commit_lde_columns_with(&lde_columns, layout) +} + +/// REGISTER's commitment source for both leaf layouts, from `init` and `fini`: +/// the row-pair root `row_pair` the caller holds, and the one-row root computed +/// from the same files on first use. +pub fn lazy_commitment( + options: &ProofOptions, + row_pair: Commitment, + init: &[u32], + fini: &[u32], +) -> stark::lookup::LazyCommitment { + let (o, init, fini) = (options.clone(), init.to_vec(), fini.to_vec()); + stark::lookup::LazyCommitment::ready(row_pair).with_one_row(move || { + Some(compute_precomputed_commitment_with( + &o, + &init, + &fini, + LeafLayout::Row, + )) + }) } // ========================================================================= @@ -507,7 +575,7 @@ pub fn bus_interactions(final_pc_ts: u64) -> Vec { // ========================================================================= /// Whether `word_addr` is one of the two x255 (PC) Word addresses. -fn is_pc_address(word_addr: u64) -> bool { +pub(crate) fn is_pc_address(word_addr: u64) -> bool { word_addr == 510 || word_addr == 511 } diff --git a/prover/src/tables/shift.rs b/prover/src/tables/shift.rs index 891396405..dcf14eb98 100644 --- a/prover/src/tables/shift.rs +++ b/prover/src/tables/shift.rs @@ -22,6 +22,7 @@ use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, GoldilocksExtension, GoldilocksField, SHIFT_16, VmTable, alu_op}; // ========================================================================= @@ -103,15 +104,18 @@ pub mod cols { // Trace generation // ========================================================================= -/// A single SHIFT operation (hashable for deduplication). +/// A single SHIFT operation (hashable for deduplication). `#[repr(C)]`, its +/// fields ordered so it stays 24 bytes: a chunk's ops are the records the card +/// fills its SHIFT table from ([`card_ops`]). +#[repr(C)] #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct ShiftOperation { + /// Full shift amount `arg2` (the unified ALU bus carries this as in2). + pub shift_amount: u64, /// Input value as 4 halfwords (DWordHL) pub in_halves: [u16; 4], /// Shift amount low byte (used by the computation; effective = mod 32/64). pub shift: u8, - /// Full shift amount `arg2` (the unified ALU bus carries this as in2). - pub shift_amount: u64, /// 0 = left, 1 = right pub direction: bool, /// Whether arithmetic (signed) right shift @@ -120,6 +124,30 @@ pub struct ShiftOperation { pub word_instr: bool, } +// The walk keeps one per shift: keep it at 24 bytes. +const _: () = assert!(std::mem::size_of::() == 24); + +/// SHIFT ops as the card's SHIFT fill reads them +/// (`multilinear::records::ShiftOp`): the same bytes. +pub(crate) fn card_ops(ops: &[ShiftOperation]) -> &[multilinear::records::ShiftOp] { + use multilinear::records::ShiftOp; + use std::mem::{align_of, offset_of, size_of}; + const _: () = { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(ShiftOperation, shift_amount) == offset_of!(ShiftOp, shift_amount)); + assert!(offset_of!(ShiftOperation, in_halves) == offset_of!(ShiftOp, in_halves)); + assert!(offset_of!(ShiftOperation, shift) == offset_of!(ShiftOp, shift)); + assert!(offset_of!(ShiftOperation, direction) == offset_of!(ShiftOp, direction)); + assert!(offset_of!(ShiftOperation, signed) == offset_of!(ShiftOp, signed)); + assert!(offset_of!(ShiftOperation, word_instr) == offset_of!(ShiftOp, word_instr)); + }; + // SAFETY: both are `#[repr(C)]` with the same fields in the same order + // (asserted above); the flags are `bool`s, 0 or 1, read as `u8`s, and no + // reader looks at the padding after them. + unsafe { std::slice::from_raw_parts(ops.as_ptr() as *const ShiftOp, ops.len()) } +} + impl ShiftOperation { /// `shift_amount` is the full (un-reduced) shift operand `arg2`; only its low /// byte feeds the computation (the result depends on `arg2 mod 32/64`). @@ -353,65 +381,79 @@ struct ShiftAux { pub fn generate_shift_trace( operations: &[ShiftOperation], ) -> TraceTable { + generate_shift_trace_as(operations, TraceForm::Wide) +} + +/// The widths SHIFT traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); + +/// [`generate_shift_trace`] in `form` (`tables::gpack`). +pub fn generate_shift_trace_as( + operations: &[ShiftOperation], + form: TraceForm, +) -> TraceTable { + generate_shift_trace_segments_as(&[operations], form) +} + +/// [`generate_shift_trace_as`] over `segments`, the operations one after another: a +/// chunk handed out as the window parts it lies in. +pub(crate) fn generate_shift_trace_segments_as( + segments: &[&[ShiftOperation]], + form: TraceForm, +) -> TraceTable { + let len: usize = segments.iter().map(|s| s.len()).sum(); // No deduplication: each operation gets its own row with μ=1. // Spec declares μ: Bit. - let num_rows = operations.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; - - for (row_idx, op) in operations.iter().enumerate() { - let aux = op.compute_aux(); + let num_rows = len.next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in segments.iter().flat_map(|s| s.iter()).enumerate() { + let aux = op.compute_aux(); + + // Input columns + table.set_halves(row_idx, cols::IN_0, &op.in_halves); + table.set_byte(row_idx, cols::SHIFT_AMOUNT, op.shift); + // High bits of the full shift amount (for the ALU bus in2 = arg2). + table.set_byte( + row_idx, + cols::SHIFT_B1, + ((op.shift_amount >> 8) & 0xFF) as u8, + ); + table.set_half( + row_idx, + cols::SHIFT_H1, + ((op.shift_amount >> 16) & 0xFFFF) as u16, + ); + table.set_word(row_idx, cols::SHIFT_HIGH, (op.shift_amount >> 32) as u32); + table.set_bool(row_idx, cols::DIRECTION, op.direction); + table.set_bool(row_idx, cols::SIGNED, op.signed); + table.set_bool(row_idx, cols::WORD_INSTR, op.word_instr); + + // Output columns + table.set_words(row_idx, cols::OUT_0, &aux.out); + + // Auxiliary columns + table.set_bool(row_idx, cols::IS_NEGATIVE, aux.is_negative); + table.set_byte(row_idx, cols::BIT_SHIFT, aux.bit_shift); + table.set_bool(row_idx, cols::ZBS, aux.zbs); + + table.set_halves(row_idx, cols::X_0, &aux.x); + table.set_halves(row_idx, cols::Y_0, &aux.y); + for i in 0..3 { + table.set_bool(row_idx, cols::LIMB_SHIFT_RAW[i], aux.limb_shift[i]); + } + // limb_shift[3] is virtual: not stored in the trace - // Input columns - table.set_halves(row_idx, cols::IN_0, &op.in_halves); - table.set_byte(row_idx, cols::SHIFT_AMOUNT, op.shift); - // High bits of the full shift amount (for the ALU bus in2 = arg2). - table.set_byte( - row_idx, - cols::SHIFT_B1, - ((op.shift_amount >> 8) & 0xFF) as u8, - ); - table.set_half( - row_idx, - cols::SHIFT_H1, - ((op.shift_amount >> 16) & 0xFFFF) as u16, - ); - table.set_word(row_idx, cols::SHIFT_HIGH, (op.shift_amount >> 32) as u32); - table.set_bool(row_idx, cols::DIRECTION, op.direction); - table.set_bool(row_idx, cols::SIGNED, op.signed); - table.set_bool(row_idx, cols::WORD_INSTR, op.word_instr); - - // Output columns - table.set_words(row_idx, cols::OUT_0, &aux.out); - - // Auxiliary columns - table.set_bool(row_idx, cols::IS_NEGATIVE, aux.is_negative); - table.set_byte(row_idx, cols::BIT_SHIFT, aux.bit_shift); - table.set_bool(row_idx, cols::ZBS, aux.zbs); - - table.set_halves(row_idx, cols::X_0, &aux.x); - table.set_halves(row_idx, cols::Y_0, &aux.y); - for i in 0..3 { - table.set_bool(row_idx, cols::LIMB_SHIFT_RAW[i], aux.limb_shift[i]); + // μ = 1 for all active rows (Bit) + table.set_bool(row_idx, cols::MU, true); } - // limb_shift[3] is virtual: not stored in the trace - // μ = 1 for all active rows (Bit) - table.set_bool(row_idx, cols::MU, true); - } - - // Padding rows: set ZBS=1 per spec. All other columns remain 0. - // μ=0 so C13 (limb_shift encoding) is inactive. left=right=0 so shifted=0, - // making C14 (out=shifted) trivially satisfied regardless of limb_shift. - for row_idx in operations.len()..num_rows { - table.set_bool(row_idx, cols::ZBS, true); - } - - trace + // Padding rows: set ZBS=1 per spec. All other columns remain 0. + // μ=0 so C13 (limb_shift encoding) is inactive. left=right=0 so shifted=0, + // making C14 (out=shifted) trivially satisfied regardless of limb_shift. + for row_idx in len..num_rows { + table.set_bool(row_idx, cols::ZBS, true); + } + }) } // ========================================================================= diff --git a/prover/src/tables/store.rs b/prover/src/tables/store.rs index 6509dbe54..b0365a381 100644 --- a/prover/src/tables/store.rs +++ b/prover/src/tables/store.rs @@ -24,6 +24,7 @@ use stark::trace::TraceTable; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use super::gpack::{TraceForm, WidthHint, generate_main}; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable}; use crate::constraints::templates::emit_is_bit; @@ -54,7 +55,9 @@ pub mod cols { // ========================================================================= /// A single STORE operation. Exactly one of `write2/write4/write8` is set, or -/// none for a single-byte store. +/// none for a single-byte store. `#[repr(C)]`: a chunk's ops are the records +/// the card fills its STORE table from ([`card_ops`]). +#[repr(C)] #[derive(Debug, Clone, Default, Hash, PartialEq, Eq)] pub struct StoreOperation { pub base_address: u64, @@ -65,6 +68,30 @@ pub struct StoreOperation { pub write8: bool, } +// The walk keeps one per store: keep it at 32 bytes. +const _: () = assert!(std::mem::size_of::() == 32); + +/// STORE ops as the card's STORE fill reads them +/// (`multilinear::records::StoreOp`): the same bytes. +pub(crate) fn card_ops(ops: &[StoreOperation]) -> &[multilinear::records::StoreOp] { + use multilinear::records::StoreOp; + use std::mem::{align_of, offset_of, size_of}; + const _: () = { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + assert!(offset_of!(StoreOperation, base_address) == offset_of!(StoreOp, base_address)); + assert!(offset_of!(StoreOperation, timestamp) == offset_of!(StoreOp, timestamp)); + assert!(offset_of!(StoreOperation, value) == offset_of!(StoreOp, value)); + assert!(offset_of!(StoreOperation, write2) == offset_of!(StoreOp, write2)); + assert!(offset_of!(StoreOperation, write4) == offset_of!(StoreOp, write4)); + assert!(offset_of!(StoreOperation, write8) == offset_of!(StoreOp, write8)); + }; + // SAFETY: both are `#[repr(C)]` with the same fields in the same order + // (asserted above); the flags are `bool`s, 0 or 1, read as `u8`s, and no + // reader looks at the padding after them. + unsafe { std::slice::from_raw_parts(ops.as_ptr() as *const StoreOp, ops.len()) } +} + impl StoreOperation { pub fn new(base_address: u64, timestamp: u64, value: u64, bytes: u8) -> Self { Self { @@ -95,25 +122,39 @@ impl StoreOperation { pub fn generate_store_trace( operations: &[StoreOperation], ) -> TraceTable { - let num_rows = operations.len().next_power_of_two().max(4); - let mut trace = TraceTable::new_main( - crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), - cols::NUM_COLUMNS, - 1, - ); - let table = &mut trace.main_table; + generate_store_trace_as(operations, TraceForm::Wide) +} - for (row_idx, op) in operations.iter().enumerate() { - table.set_dword_wl(row_idx, cols::BASE_ADDRESS_0, op.base_address); - table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - table.set_bool(row_idx, cols::WRITE2, op.write2); - table.set_bool(row_idx, cols::WRITE4, op.write4); - table.set_bool(row_idx, cols::WRITE8, op.write8); - table.set_dword_bl(row_idx, cols::VALUE[0], op.value); - table.set_fe(row_idx, cols::MU, FE::one()); - } +/// The widths STORE traces needed so far in this process (`tables::gpack`). +static WIDTHS: WidthHint = WidthHint::new(); - trace +/// [`generate_store_trace`] in `form` (`tables::gpack`). +pub fn generate_store_trace_as( + operations: &[StoreOperation], + form: TraceForm, +) -> TraceTable { + generate_store_trace_segments_as(&[operations], form) +} + +/// [`generate_store_trace_as`] over `segments`, the operations one after +/// another. +pub(crate) fn generate_store_trace_segments_as( + segments: &[&[StoreOperation]], + form: TraceForm, +) -> TraceTable { + let len: usize = segments.iter().map(|s| s.len()).sum(); + let num_rows = len.next_power_of_two().max(4); + generate_main!(form, &WIDTHS, num_rows, cols::NUM_COLUMNS, |table| { + for (row_idx, op) in segments.iter().flat_map(|s| s.iter()).enumerate() { + table.set_dword_wl(row_idx, cols::BASE_ADDRESS_0, op.base_address); + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + table.set_bool(row_idx, cols::WRITE2, op.write2); + table.set_bool(row_idx, cols::WRITE4, op.write4); + table.set_bool(row_idx, cols::WRITE8, op.write8); + table.set_dword_bl(row_idx, cols::VALUE[0], op.value); + table.set_fe(row_idx, cols::MU, FE::one()); + } + }) } // ========================================================================= diff --git a/prover/src/tables/trace_builder.rs b/prover/src/tables/trace_builder.rs index 46ea8029d..a53f680bc 100644 --- a/prover/src/tables/trace_builder.rs +++ b/prover/src/tables/trace_builder.rs @@ -26,8 +26,8 @@ //! ``` use std::collections::HashMap; -#[cfg(feature = "disk-spill")] -use std::collections::HashSet; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; use executor::elf::Elf; use executor::vm::instruction::decoding::Instruction; @@ -35,21 +35,23 @@ use executor::vm::logs::Log; use executor::vm::memory::U64HashMap; #[cfg(feature = "parallel")] use rayon::prelude::*; -#[cfg(feature = "disk-spill")] -use stark::storage_mode::StorageMode; use stark::trace::TraceTable; use super::bitwise::{self, BitwiseOperation, BitwiseOperationType}; +use super::blake3::{self, Blake3Operation}; use super::branch::{self, BranchOperation}; use super::bytewise; use super::copy; -use super::cpu::{self, CpuOperation}; +use super::cpu::{self, CpuOp, CpuOperation}; use super::cpu32; use super::decode; +use super::decode::DecodeTable; use super::dvrm::{self, DvrmOperation}; use super::ecdas; +use super::ecrec; use super::ecsm; use super::eq; +use super::gpack::TraceForm; use super::halt; use super::hint; use super::is_wh; @@ -67,10 +69,910 @@ use super::page::{self, PageConfig}; use super::register::{self, FinalRegisterStateMap, FinalRegisterWordState}; use super::shift::{self, ShiftOperation}; use super::store; -use super::types::{GoldilocksExtension, GoldilocksField, dedup_rows}; +use super::types::{GoldilocksExtension, GoldilocksField, dedup_rows, dedup_rows_of}; use crate::Error; use crate::paged_mem::{ImageSource, PagedMem}; +mod blocks; +#[cfg(test)] +mod bus_net_tests; +mod clock; +mod compact_branch; +mod compact_bytewise; +mod compact_eq; +#[cfg(test)] +mod keccak256_tests; +#[cfg(test)] +mod direct_count_tests { + //! The direct counts of LT's and KECCAK's lookups against the lists the + //! finish built before them, kept here word for word as the reference. + use super::*; + + fn reference_lt_ops(lt_ops: &[LtOperation]) -> Vec { + let mut bitwise_ops = Vec::with_capacity(lt_ops.len() * 8); + + for op in lt_ops { + // MSB16 lookups for lhs[2] and rhs[2] + let lhs_2 = ((op.lhs >> 48) & 0xFFFF) as u16; + let rhs_2 = ((op.rhs >> 48) & 0xFFFF) as u16; + + bitwise_ops.push(BitwiseOperation::halfword( + BitwiseOperationType::Msb16, + (lhs_2 & 0xFF) as u8, + (lhs_2 >> 8) as u8, + )); + bitwise_ops.push(BitwiseOperation::halfword( + BitwiseOperationType::Msb16, + (rhs_2 & 0xFF) as u8, + (rhs_2 >> 8) as u8, + )); + + // IS_HALFWORD lookups for lhs_sub_rhs[0..4] + let lhs_sub_rhs = op.lhs.wrapping_sub(op.rhs); + for shift in [0, 16, 32, 48] { + let half = ((lhs_sub_rhs >> shift) & 0xFFFF) as u16; + bitwise_ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + (half >> 8) as u8, + )); + } + + // IS_HALFWORD lookups for lhs[1] and rhs[1] + let lhs_1 = ((op.lhs >> 32) & 0xFFFF) as u16; + let rhs_1 = ((op.rhs >> 32) & 0xFFFF) as u16; + bitwise_ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (lhs_1 & 0xFF) as u8, + (lhs_1 >> 8) as u8, + )); + bitwise_ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (rhs_1 & 0xFF) as u8, + (rhs_1 >> 8) as u8, + )); + } + + bitwise_ops + } + + #[allow(clippy::needless_range_loop)] + fn reference_keccak(keccak_ops: &[KeccakOperation]) -> Vec { + use executor::vm::instruction::execution::{KECCAK_RC, KECCAK_RHO}; + + let mut ops = Vec::new(); + + for kop in keccak_ops { + let state_addr = kop.state_addr; + + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluAnd, + (state_addr & 0xFF) as u8, + 7, + )); + + // Range-check addr bytes (paired with the ARE_BYTES sends in + // keccak::bus_interactions): without this the field-element value of + // the addr_lo / addr_hi linear combinations is unconstrained per byte. + // 4 paired ops matching the (addr[2i], addr[2i+1]) sender pairing. + for i in 0..4 { + let lo = ((state_addr >> (2 * i * 8)) & 0xFF) as u8; + let hi = ((state_addr >> ((2 * i + 1) * 8)) & 0xFF) as u8; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + lo, + hi, + )); + } + + // IS_HALF for state_ptr half_lo (25 per call). The top 48 bits go to + // IS_WH through `IS_WHH`; that table counts its own checks. + for lane_idx in 0..25 { + let ptr = state_addr + .checked_add(lane_idx as u64 * 8) + .expect("keccak state address range must be validated by the executor"); + let half = (ptr & 0xFFFF) as u16; + ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + ((half >> 8) & 0xFF) as u8, + )); + } + + // Replay keccak round computation to extract bitwise lookups + let mut state = kop.input; + for round in 0..24 { + // --- theta: Cxz chain BYTE_ALU[XOR] (160) --- + let mut cxz = [[[0u8; 8]; 4]; 5]; + for x in 0..5 { + for b in 0..8 { + let v0 = ((state[x] >> (b * 8)) & 0xFF) as u8; + let v1 = ((state[x + 5] >> (b * 8)) & 0xFF) as u8; + cxz[x][0][b] = v0 ^ v1; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + v0, + v1, + )); + } + for stage in 1..4usize { + let y = stage + 1; + for b in 0..8 { + let prev = cxz[x][stage - 1][b]; + let sv = ((state[x + 5 * y] >> (b * 8)) & 0xFF) as u8; + cxz[x][stage][b] = prev ^ sv; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + prev, + sv, + )); + } + } + } + + // theta: ARE_BYTES on Cxz_left (20 pairs). The rotate-by-1 shift is + // enforced by the keccak_rnd inline μ-gated identity, so no HWSL + // lookup is emitted here. Cxz_right is range-checked via IS_BIT + // polynomial constraints on the keccak_rnd chip (spec d75944ee). + let mut rotated_c = [[0u8; 8]; 5]; + for x in 0..5 { + let c = cxz[x][3]; + for hw in 0..4 { + let halfword = (c[hw * 2] as u16) | ((c[hw * 2 + 1] as u16) << 8); + let shifted = halfword << 1; // u16 wraps + // ARE_BYTES for cxz_left bytes: paired (low, high) of the halfword, + // matching `(cxz_left[x][2i], cxz_left[x][2i+1])` sender pairing. + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (shifted & 0xFF) as u8, + ((shifted >> 8) & 0xFF) as u8, + )); + } + // Reconstruct rotated_c using the bit-typed Cxz_right. + let mut left_bytes = [0u8; 8]; + let mut right_bits = [0u8; 4]; + for hw in 0..4 { + let halfword = (c[hw * 2] as u16) | ((c[hw * 2 + 1] as u16) << 8); + let shifted = halfword << 1; + left_bytes[hw * 2] = (shifted & 0xFF) as u8; + left_bytes[hw * 2 + 1] = ((shifted >> 8) & 0xFF) as u8; + right_bits[hw] = (halfword >> 15) as u8; + } + for b in 0usize..8 { + let right_contribution = if b.is_multiple_of(2) { + right_bits[(b / 2 + 3) % 4] + } else { + 0 + }; + rotated_c[x][b] = left_bytes[b].wrapping_add(right_contribution); + } + } + + // theta: Dxz BYTE_ALU[XOR] (40) + let mut d_bytes = [[0u8; 8]; 5]; + for x in 0..5 { + for b in 0..8 { + let a = cxz[(x + 4) % 5][3][b]; + let rb = rotated_c[(x + 1) % 5][b]; + d_bytes[x][b] = a ^ rb; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + a, + rb, + )); + } + } + + // theta final: BYTE_ALU[XOR] (200) + let mut theta_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let lane = state[x + 5 * y]; + let mut d_lane = 0u64; + for b in 0..8 { + d_lane |= (d_bytes[x][b] as u64) << (b * 8); + } + theta_lanes[x + 5 * y] = lane ^ d_lane; + for b in 0..8 { + let s = ((lane >> (b * 8)) & 0xFF) as u8; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + s, + d_bytes[x][b], + )); + } + } + } + + // rho: ARE_BYTES (200 pairs). The per-lane shift is enforced by the + // keccak_rnd inline μ-gated identities, so no HWSL lookup is emitted. + for x in 0..5 { + for y in 0..5 { + let rho_offset = KECCAK_RHO[x][y] as usize; + let rnc_val = (rho_offset % 16) as u8; + let theta_lane = theta_lanes[x + 5 * y]; + for hw in 0..4 { + let halfword = ((theta_lane >> (hw * 16)) & 0xFFFF) as u16; + let (shifted, carry) = if rnc_val == 0 { + (halfword, 0u16) + } else { + (halfword << rnc_val, halfword >> (16 - rnc_val)) + }; + // ARE_BYTES paired as (rot_left[b], rot_right[b]) for + // each byte of the halfword, matching the sender pairing + // in keccak_rnd::bus_interactions. + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (shifted & 0xFF) as u8, + (carry & 0xFF) as u8, + )); + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((shifted >> 8) & 0xFF) as u8, + ((carry >> 8) & 0xFF) as u8, + )); + } + } + } + + // pi: compute pi_lanes + let mut pi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let rotated = theta_lanes[x + 5 * y].rotate_left(KECCAK_RHO[x][y]); + let dst_x = y; + let dst_y = (2 * x + 3 * y) % 5; + pi_lanes[dst_x + 5 * dst_y] = rotated; + } + } + + // chi: BYTE_ALU[AND] (200) + BYTE_ALU[XOR] (200) + let mut chi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let not_next = !pi_lanes[(x + 1) % 5 + 5 * y]; + let next2 = pi_lanes[(x + 2) % 5 + 5 * y]; + let and_val = not_next & next2; + chi_lanes[x + 5 * y] = pi_lanes[x + 5 * y] ^ and_val; + for b in 0..8 { + let not_byte = ((not_next >> (b * 8)) & 0xFF) as u8; + let n2_byte = ((next2 >> (b * 8)) & 0xFF) as u8; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluAnd, + not_byte, + n2_byte, + )); + let pi_byte = ((pi_lanes[x + 5 * y] >> (b * 8)) & 0xFF) as u8; + let and_byte = ((and_val >> (b * 8)) & 0xFF) as u8; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + pi_byte, + and_byte, + )); + } + } + } + + // iota: BYTE_ALU[XOR] (8) + let rc_val = KECCAK_RC[round]; + for b in 0..8 { + let chi_byte = ((chi_lanes[0] >> (b * 8)) & 0xFF) as u8; + let rc_byte = ((rc_val >> (b * 8)) & 0xFF) as u8; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + chi_byte, + rc_byte, + )); + } + + // Update state + chi_lanes[0] ^= rc_val; + state = chi_lanes; + } + } + + ops + } + + /// A splitmix stream for the inputs. + fn mix(x: &mut u64) -> u64 { + *x = x.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = *x; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + /// ★ At the finish's slice (2^20 ops), LT's lookups counted straight into + /// the histogram are the reference lists', and so are the lists the + /// finish still builds on the rollback. + #[test] + fn lt_lookups_counted_directly_are_the_lists() { + let mut x = 7u64; + let edge = [ + 0u64, + 1, + 0xFFFF, + 0x1_0000, + u64::MAX, + 1 << 63, + 0xFFFF_0000_FFFF_0000, + ]; + let ops: Vec = (0..1usize << 20) + .map(|k| { + let (a, b) = if k < edge.len() * edge.len() { + (edge[k % edge.len()], edge[k / edge.len()]) + } else { + (mix(&mut x), mix(&mut x) >> (k % 64)) + }; + LtOperation::new_with_invert(a, b, k % 3 == 0, k % 5 == 0) + }) + .collect(); + let mut want = bitwise::BitwiseHistogram::new(); + want.add_ops(&reference_lt_ops(&ops)); + let mut direct = bitwise::BitwiseHistogram::new(); + count_bitwise_from_lt_ops(&ops, &mut direct); + assert!(direct == want, "LT counted directly"); + let mut lists = bitwise::BitwiseHistogram::new(); + lists.add_ops(&collect_bitwise_from_lt_ops(&ops)); + assert!(lists == want, "LT's lists"); + } + + /// ★ At the finish's slice (2^11 permutations), KECCAK's lookups counted + /// straight into the histogram are the reference lists' (every round of + /// every permutation replayed), and so are the rollback's lists. + #[test] + fn keccak_lookups_counted_directly_are_the_lists() { + let mut x = 11u64; + let ops: Vec = (0..1usize << 11) + .map(|k| { + let mut input = [0u64; 25]; + for lane in input.iter_mut() { + *lane = if k == 0 { 0 } else { mix(&mut x) }; + } + KeccakOperation { + timestamp: k as u64, + state_addr: (mix(&mut x) >> 2) & !7, + input, + output: [0; 25], + sponge: None, + } + }) + .collect(); + let mut want = bitwise::BitwiseHistogram::new(); + want.add_ops(&reference_keccak(&ops)); + let mut direct = bitwise::BitwiseHistogram::new(); + count_bitwise_from_keccak(&ops, &mut direct); + assert!(direct == want, "KECCAK counted directly"); + let mut lists = bitwise::BitwiseHistogram::new(); + lists.add_ops(&collect_bitwise_from_keccak(&ops)); + assert!(lists == want, "KECCAK's lists"); + } +} + +#[cfg(test)] +mod row_count_tests { + use super::*; + use crate::tables::types::{MU_MAX, alu_op}; + + /// A list as the finish keeps it: one block of `ops`. + fn one_block(ops: &[T]) -> Segmented<'_, T> { + Segmented { + parts: vec![Part::Ops(ops)], + } + } + + /// `n` ops from `make(i)` over a few hundred distinct values, each repeated + /// so that counts land on and around [`MU_MAX`] and its multiples. + fn skewed(n: usize, mut make: impl FnMut(u64) -> T) -> Vec { + let mut x = 0x9e37_79b9_7f4a_7c15u64; + let mut out = Vec::with_capacity(n); + let repeats = [1usize, 2, 254, 255, 256, 509, 510, 511, 600]; + let mut k = 0u64; + while out.len() < n { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + let times = repeats[(x % repeats.len() as u64) as usize]; + for _ in 0..times.min(n - out.len()) { + out.push(make(k)); + } + k += 1; + } + // Interleave so a chunk's repeats are not all adjacent. + let len = out.len(); + for i in 0..len { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + out.swap(i, (x % len as u64) as usize); + } + out + } + + /// ★ Counted on the pool, chunk by chunk with no lists, the per-row + /// lookups of LT, BRANCH and BYTEWISE are the multiplicities the lists + /// fold into the histogram: counts on, below and past μ's split at + /// 255 (several rows of one op), every chunk's padding rows, BRANCH's next + /// pc byte, at chunk sizes that leave a short last chunk. + #[test] + fn the_per_row_lookups_counted_on_the_pool_are_the_lists() { + assert_eq!(MU_MAX, 255); + for (n, rows) in [ + (0usize, 64usize), + (5, 64), + (3_000, 64), + (40_000, 1 << 12), + (70_000, 1 << 15), + ] { + let lt = skewed(n, |k| { + LtOperation::new_with_invert(k.wrapping_mul(3), k ^ 0x55, k % 3 == 0, k % 5 == 0) + }); + let branch = skewed(n, |k| { + BranchOperation::new( + 0x1000 + 4 * (k % 97), + k.wrapping_mul(0x1_0004), + k << 9, + k % 2 == 1, + ) + }); + let ops = [alu_op::AND, alu_op::OR, alu_op::XOR]; + let bytewise = skewed(n, |k| { + bytewise::BytewiseOperation::new(k & 0xFF, (k >> 3) & 0xFF, ops[(k % 3) as usize]) + }); + let max_rows = crate::tables::MaxRowsConfig::uniform(rows); + let (lt_s, branch_s, bytewise_s) = + (one_block(<), one_block(&branch), one_block(&bytewise)); + let mut lists = bitwise::BitwiseHistogram::new(); + for (start, end) in list_chunks(lt.len(), max_rows.lt) { + lists.add_ops(<_row_lookups(lt_s.range(start, end).iter())); + } + for (start, end) in list_chunks(branch.len(), max_rows.branch) { + lists.add_ops(&branch_row_lookups(&branch_s.range(start, end))); + } + for (start, end) in list_chunks(bytewise.len(), max_rows.bytewise) { + lists.add_ops(&bytewise_row_lookups(&bytewise_s.range(start, end))); + } + for by in [RowCount::Map, RowCount::Parts] { + let mut counted = bitwise::BitwiseHistogram::new(); + count_row_lookups(<_s, &branch_s, &bytewise_s, &max_rows, by) + .bump_into(&mut counted); + assert!(counted == lists, "{n} ops, {rows} a chunk, {by:?}"); + } + } + } + + /// The same over lists the finish keeps in several segments, LT's middle + /// one compact: chunks within a segment, across segments and inside the + /// compact one are counted from slices, never copied whole, and still give + /// the lists' multiplicities. Each map is reserved for the chunk's distinct + /// ops, never more than for its length, and LT's chunks note the copies + /// `range` would have made. + #[test] + fn the_per_row_lookups_over_split_and_compact_lists_are_the_lists() { + for (n, rows) in [(3_000usize, 64usize), (40_000, 1 << 12), (70_000, 1 << 15)] { + // Compact LT ops are unsigned and not inverted (MEMW's checks). + let lt = skewed(n, |k| LtOperation::new(k.wrapping_mul(3), k ^ 0x55, false)); + let branch = skewed(n, |k| { + BranchOperation::new( + 0x1000 + 4 * (k % 97), + k.wrapping_mul(0x1_0004), + k << 9, + k % 2 == 1, + ) + }); + let ops = [alu_op::AND, alu_op::OR, alu_op::XOR]; + let bytewise = skewed(n, |k| { + bytewise::BytewiseOperation::new(k & 0xFF, (k >> 3) & 0xFF, ops[(k % 3) as usize]) + }); + let (a, b) = (n / 3, 2 * n / 3 + 17); + let mut compact = CompactLt::default(); + compact.extend(<[a..b]); + let lt_s = Segmented { + parts: vec![ + Part::Ops(<[..a]), + Part::Compact(&compact), + Part::Ops(<[b..]), + ], + }; + let branch_s = Segmented { + parts: vec![Part::Ops(&branch[..a]), Part::Ops(&branch[a..])], + }; + let bytewise_s = Segmented { + parts: vec![Part::Ops(&bytewise[..b]), Part::Ops(&bytewise[b..])], + }; + let max_rows = crate::tables::MaxRowsConfig::uniform(rows); + let mut lists = bitwise::BitwiseHistogram::new(); + for (start, end) in list_chunks(lt.len(), max_rows.lt) { + lists.add_ops(<_row_lookups(lt_s.range(start, end).iter())); + } + for (start, end) in list_chunks(branch.len(), max_rows.branch) { + lists.add_ops(&branch_row_lookups(&branch_s.range(start, end))); + } + for (start, end) in list_chunks(bytewise.len(), max_rows.bytewise) { + lists.add_ops(&bytewise_row_lookups(&bytewise_s.range(start, end))); + } + let counts = count_row_lookups(<_s, &branch_s, &bytewise_s, &max_rows, RowCount::Map); + let mut counted = bitwise::BitwiseHistogram::new(); + counts.bump_into(&mut counted); + assert!(counted == lists, "{n} ops, {rows} a chunk"); + for (k, scan) in counts.scans.iter().enumerate() { + assert_eq!(scan.ops, n, "kind {k}"); + assert!( + scan.reserved <= scan.by_length, + "kind {k}: reserved {} by length {} ({} chunks, {} distinct)", + scan.reserved, + scan.by_length, + scan.chunks, + scan.distinct + ); + assert!(scan.distinct <= scan.ops, "kind {k}"); + // Chunks of a few dozen ops hold a handful of distinct ones, where + // one op is many percent; read the estimate's miss on larger ones. + if rows >= 1 << 12 { + assert!(scan.miss <= 0.05, "kind {k}: missed by {:.3}", scan.miss); + // A few hundred repeats an op: the maps hold a sliver of + // what reserving whole chunks would. + assert!( + scan.reserved * 4 <= scan.by_length, + "kind {k}: reserved {} against {} by length", + scan.reserved, + scan.by_length + ); + } + } + assert!( + counts.scans[0].copies > 0, + "LT's compact chunks would have been copied" + ); + assert!(counts.line().starts_with("BLOCK P4 ROWS: LT ")); + // By parts: the same multiplicities and distinct ops, never more + // bytes than a whole chunk's map. + let parts = + count_row_lookups(<_s, &branch_s, &bytewise_s, &max_rows, RowCount::Parts); + let mut by_parts = bitwise::BitwiseHistogram::new(); + parts.bump_into(&mut by_parts); + assert!(by_parts == lists, "{n} ops, {rows} a chunk, by parts"); + for (k, (map, part)) in counts.scans.iter().zip(&parts.scans).enumerate() { + assert_eq!( + (part.ops, part.distinct, part.parted, map.parted), + (map.ops, map.distinct, part.chunks, 0), + "kind {k}" + ); + assert!(part.reserved <= part.by_length, "kind {k}"); + } + assert!(parts.line().contains("counted by 64 parts a chunk")); + } + } + + /// ★ At a block's chunk size (2^21 ops a chunk, LT's about half of them + /// distinct as on a block), counting by parts gives each family the one + /// map's multiplicities and distinct ops, and the parts share the ops + /// evenly: no part holds twice its share. + #[test] + fn counted_by_parts_a_real_sized_chunk_is_the_one_maps() { + let n = 1usize << 21; + let pick = + |k: usize, distinct: usize| (k as u64).wrapping_mul(0x9e37_79b9) % distinct as u64; + let lt: Vec = (0..n) + .map(|k| { + let v = pick(k, n * 47 / 100); + LtOperation::new_with_invert( + v.wrapping_mul(0x1_0001), + v >> 3, + v % 3 == 0, + v % 7 == 0, + ) + }) + .collect(); + let branch: Vec = (0..n) + .map(|k| { + let v = pick(k, n / 20); + BranchOperation::new( + 0x1000 + 4 * (v % 4099), + v.wrapping_mul(0x1_0004), + v << 9, + v % 2 == 1, + ) + }) + .collect(); + let ops = [alu_op::AND, alu_op::OR, alu_op::XOR]; + let bytewise: Vec = (0..n) + .map(|k| { + let v = pick(k, n / 5); + bytewise::BytewiseOperation::new(v & 0xFF, (v >> 8) & 0xFF, ops[(v % 3) as usize]) + }) + .collect(); + let max_rows = crate::tables::MaxRowsConfig::uniform(n); + let (lt_s, branch_s, bytewise_s) = + (one_block(<), one_block(&branch), one_block(&bytewise)); + let map = count_row_lookups(<_s, &branch_s, &bytewise_s, &max_rows, RowCount::Map); + let parts = count_row_lookups(<_s, &branch_s, &bytewise_s, &max_rows, RowCount::Parts); + let (mut a, mut b) = ( + bitwise::BitwiseHistogram::new(), + bitwise::BitwiseHistogram::new(), + ); + map.bump_into(&mut a); + parts.bump_into(&mut b); + assert!(a == b, "by parts the multiplicities moved"); + for (k, (m, p)) in map.scans.iter().zip(&parts.scans).enumerate() { + assert_eq!( + (p.chunks, p.ops, p.distinct), + (m.chunks, m.ops, m.distinct), + "kind {k}" + ); + } + // Each count ran the way it was asked to, and by parts LT's chunk + // held less than its one map (every op copied once, a small map). + for (k, (m, p)) in map.scans.iter().zip(&parts.scans).enumerate() { + assert_eq!((m.parted, p.parted), (0, p.chunks), "kind {k}"); + } + assert!(parts.scans[0].reserved < map.scans[0].reserved); + assert!(parts.line().contains("counted by 64 parts a chunk")); + assert!(map.line().contains("counted by one map a chunk")); + assert!( + map.scans[0].distinct * 2 > n * 9 / 10, + "LT about half distinct" + ); + let mut sizes = vec![0usize; 1 << ROW_PARTS_LOG2]; + for op in < { + sizes[row_part(op)] += 1; + } + let share = n >> ROW_PARTS_LOG2; + assert!( + sizes.iter().all(|&s| s < 2 * share && s > share / 2), + "parts uneven: {:?} to {:?}", + sizes.iter().min(), + sizes.iter().max() + ); + } + + /// A list's ops handed over a slice at a time are the ops `range` gives, + /// over in-memory and compact segments, within and across them, at any + /// block size; the flag says when `range` copies (a range that spans + /// segments or lies in a compact one). + #[test] + fn a_chunk_read_a_slice_at_a_time_is_its_range() { + let lt: Vec = (0..10_000u64) + .map(|k| LtOperation::new(k.wrapping_mul(7), k ^ 0x3c, false)) + .collect(); + let mut compact = CompactLt::default(); + compact.extend(<[3_000..7_000]); + let list = Segmented { + parts: vec![ + Part::Ops(<[..3_000]), + Part::Compact(&compact), + Part::Ops(<[7_000..]), + ], + }; + for (start, end) in [ + (0usize, 0usize), + (10, 2_000), + (2_500, 3_500), + (3_100, 6_900), + (6_000, 9_000), + (0, 10_000), + (7_000, 10_000), + ] { + for block in [1usize, 7, 4_096, 1 << 16] { + let mut got = Vec::new(); + let copied = + list.for_each_slice(start, end, block, &mut |ops| got.extend_from_slice(ops)); + let want = list.range(start, end); + assert!(got == want.as_ref(), "{start}..{end} at {block}"); + assert_eq!( + copied, + matches!(want, std::borrow::Cow::Owned(_)), + "{start}..{end} at {block}" + ); + } + } + } + + /// The distinct-op estimate stays within 2 % (or 2 ops) of the count, and + /// never above the ops, from all-equal to all-distinct chunks. + #[test] + fn the_distinct_estimate_is_close_and_never_above_the_ops() { + for (n, d) in [ + (0usize, 1usize), + (1, 1), + (100, 1), + (1_000, 10), + (5_000, 3_000), + (1 << 16, 1 << 12), + (1 << 16, 1 << 16), + (100_000, 99_000), + ] { + let ops: Vec = (0..n) + .map(|i| ((i % d) as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15)) + .collect(); + let visit = |f: &mut Slices<'_, u64>| f(&ops); + let estimate = distinct_estimate(&visit, n); + let truth = d.min(n); + assert!(estimate <= n, "{n} ops: {estimate}"); + let off = estimate.abs_diff(truth); + assert!( + off <= 2 || off as f64 <= 0.02 * truth as f64, + "{n} ops, {truth} distinct: estimated {estimate}" + ); + } + } +} + +/// ★ BITWISE's lookups from a table's rows ([`count_lt_table_lookups`] and +/// its BRANCH and BYTEWISE twins) are the lookups of the table's ops: per op +/// and per row, padding included, cell by cell, for chunks with no op, one op, +/// rows exactly a power of two, rows padded, ops repeated past [`MU_MAX`] +/// (split rows), and a block's chunk size. +#[cfg(test)] +mod table_count_tests { + use super::*; + use crate::tables::types::{MU_MAX, alu_op, dedup_rows}; + + /// The histogram of what `count` puts in a plane. + fn from_plane(count: impl FnOnce(&mut bitwise::BitwisePlane)) -> bitwise::BitwiseHistogram { + let mut plane = bitwise::BitwisePlane::new(); + count(&mut plane); + let mut h = bitwise::BitwiseHistogram::new(); + h.merge_plane(&plane); + h + } + + /// Chunks of ops from `op(v)`: none, one, four distinct values (four + /// rows, no padding), five (padded to eight), one value 300 and 600 + /// times (rows split at `MU_MAX`), and 5,000 ops over 37 values. + fn chunks(op: impl Fn(u64) -> T) -> Vec<(String, Vec)> { + let repeat = |v: u64, n: usize| std::iter::repeat_n(op(v), n).collect::>(); + vec![ + ("none".into(), Vec::new()), + ("one".into(), vec![op(3)]), + ("four rows".into(), (0..4).map(&op).collect()), + ("five rows".into(), (0..5).map(&op).collect()), + ("300 of one".into(), repeat(9, 300)), + ("600 of one, two more".into(), { + let mut ops = repeat(11, 600); + ops.extend([op(1), op(2)]); + ops + }), + ( + "5000 over 37".into(), + (0..5000u64) + .map(|k| op(k.wrapping_mul(0x9e37) % 37)) + .collect(), + ), + ] + } + + fn lt_op(v: u64) -> LtOperation { + LtOperation::new_with_invert( + v.wrapping_mul(0x1_0001_0003), + (v << 47) ^ 0xFFFF_0000_0000, + v.is_multiple_of(3), + v.is_multiple_of(5), + ) + } + + fn branch_op(v: u64) -> BranchOperation { + BranchOperation::new( + 0x1000 + 4 * v, + v.wrapping_mul(0x1_0004), + v << 33, + v % 2 == 1, + ) + } + + fn bytewise_op(v: u64) -> bytewise::BytewiseOperation { + let ops = [alu_op::AND, alu_op::OR, alu_op::XOR]; + bytewise::BytewiseOperation::new(v.wrapping_mul(0x0101_0307), !v, ops[(v % 3) as usize]) + } + + fn lt_want(ops: &[LtOperation]) -> bitwise::BitwiseHistogram { + let mut h = bitwise::BitwiseHistogram::new(); + h.add_ops(&collect_bitwise_from_lt_ops(ops)); + h.add_ops(<_row_lookups(ops.iter())); + h + } + + fn branch_want(ops: &[BranchOperation]) -> bitwise::BitwiseHistogram { + let mut h = bitwise::BitwiseHistogram::new(); + h.add_ops(&collect_bitwise_from_branch_ops(ops)); + h.add_ops(&branch_row_lookups(ops)); + h + } + + fn bytewise_want(ops: &[bytewise::BytewiseOperation]) -> bitwise::BitwiseHistogram { + let mut h = bitwise::BitwiseHistogram::new(); + for op in ops { + h.add_ops(&op.collect_bitwise_ops()); + } + h.add_ops(&bytewise_row_lookups(ops)); + h + } + + #[test] + fn the_lookups_from_a_tables_rows_are_its_ops() { + const { assert!(MU_MAX < 300, "a value 300 times spans rows") }; + for (what, ops) in chunks(lt_op) { + let got = from_plane(|p| count_lt_table_lookups(&dedup_rows(&ops), p)); + assert!(got == lt_want(&ops), "LT, {what}"); + } + for (what, ops) in chunks(branch_op) { + let got = from_plane(|p| count_branch_table_lookups(&dedup_rows(&ops), p)); + assert!(got == branch_want(&ops), "BRANCH, {what}"); + } + for (what, ops) in chunks(bytewise_op) { + let got = from_plane(|p| count_bytewise_table_lookups(&dedup_rows(&ops), p)); + assert!(got == bytewise_want(&ops), "BYTEWISE, {what}"); + } + } + + /// At a block's chunk size (2^21 ops, LT's about half distinct as on a + /// block, BRANCH's a twentieth, BYTEWISE's a fifth). + #[test] + fn the_lookups_from_a_real_sized_tables_rows_are_its_ops() { + let n = 1usize << 21; + let pick = + |k: usize, distinct: usize| (k as u64).wrapping_mul(0x9e37_79b9) % distinct as u64; + let lt: Vec = (0..n).map(|k| lt_op(pick(k, n * 47 / 100))).collect(); + let got = from_plane(|p| count_lt_table_lookups(&dedup_rows(<), p)); + assert!(got == lt_want(<), "LT"); + drop(lt); + let branch: Vec = (0..n).map(|k| branch_op(pick(k, n / 20))).collect(); + let got = from_plane(|p| count_branch_table_lookups(&dedup_rows(&branch), p)); + assert!(got == branch_want(&branch), "BRANCH"); + drop(branch); + let bytewise: Vec = + (0..n).map(|k| bytewise_op(pick(k, n / 5))).collect(); + let got = from_plane(|p| count_bytewise_table_lookups(&dedup_rows(&bytewise), p)); + assert!(got == bytewise_want(&bytewise), "BYTEWISE"); + } + + /// A plane keeps a lookup off its `z = 0` cells aside, and adds it as the + /// histogram would. + #[test] + fn a_plane_keeps_the_lookups_past_z_0() { + let shift = BitwiseOperation::shift_op(BitwiseOperationType::Hwsl, 7, 9, 5); + let byte = BitwiseOperation::byte_op(BitwiseOperationType::ByteAluXor, 7, 9); + let got = from_plane(|p| { + p.bump_n(shift, 3); + p.bump_n(byte, 2); + }); + let mut want = bitwise::BitwiseHistogram::new(); + want.bump_n(shift, 3); + want.bump_n(byte, 2); + assert!(got == want); + } +} + +#[cfg(test)] +mod cut_padding_tests; +mod delta; +mod finish; +#[cfg(test)] +mod finish_oracle; +#[cfg(test)] +mod finish_tests; +pub(crate) mod gate; +#[cfg(test)] +mod lean_walk_tests; +mod windowed; +use blocks::{BlockVec, block_len}; +use compact_branch::CompactBranch; +use compact_bytewise::CompactBytewise; +use compact_eq::CompactEq; +#[cfg(feature = "parallel")] +pub(crate) use finish::hand_on_order; +pub(crate) use finish::{Emitted, RegenFamily, RegenJob, RestHeader, RestRegen}; +#[cfg(test)] +pub(crate) use windowed::DECODE_TRACE_DELAY; +pub use windowed::{ + Accumulator, CardFillStamps, ChunkJob, CountThreadStamps, RecordsRoom, RegenBuilder, + RowFamilies, SetupStamps, StreamTable, StreamedChunk, WalkedWindow, Walker, WindowStamps, + WindowedTraceBuilder, +}; +#[cfg(feature = "parallel")] +pub(crate) use windowed::{head_log, parallel}; + // ============================================================================= // Memory and Register State Tracking // ============================================================================= @@ -89,12 +991,35 @@ struct MemoryState { /// page-map lookup + dense indexing, no per-cell hashing or rehash-on-grow) /// is both lighter and faster than a per-cell `HashMap`. cells: PagedMem, + /// Lean accesses ([`WalkLean::memory`]): a multi-byte access inside one + /// page finds the page once, and the last page found is remembered (`hot`). + /// The cells read and written are the same. + lean: bool, + /// `(page base, page index)` of the last page a lean access found, or + /// [`NO_PAGE`]. While `lean`, every page allocation goes through + /// [`Self::page_or_insert`], which leaves the page it allocated here, so an + /// index here is never one an allocation moved. + hot: (u64, usize), + /// Set when a read finds an [`ABSENT`] timestamp. Only a window's state + /// holds such cells (the bytes its entries did not seed), so a run walked + /// whole never sets it. + missing: AtomicBool, } +/// No page: page bases are multiples of the page size, so never odd. +const NO_PAGE: (u64, usize) = (1, 0); + +/// The timestamp of a byte a window's state does not hold: a window walked +/// from its entries reads one only if the entries miss a byte it touches. +const ABSENT: u64 = u64::MAX; + impl MemoryState { fn new() -> Self { Self { cells: PagedMem::new((0, 0)), + lean: false, + hot: NO_PAGE, + missing: AtomicBool::new(false), } } @@ -108,53 +1033,143 @@ impl MemoryState { for (addr, value) in image.image_iter() { cells.set(addr, (value, 0)); } - Self { cells } + Self { + cells, + lean: false, + hot: NO_PAGE, + missing: AtomicBool::new(false), + } } - /// Number of distinct pages that contain at least one cell. - #[cfg(feature = "disk-spill")] - fn unique_page_count(&self, page_size: u64) -> u64 { - debug_assert!( - page_size.is_power_of_two(), - "page_size must be a power of two for the bitmask to work" - ); - let mask = !(page_size - 1); - let pages: HashSet = self.cells.iter().map(|(a, _)| a & mask).collect(); - pages.len() as u64 + /// [`Self::from_image`] of an image held as pages, seeded page by page: the + /// same pages, cells and set cells as seeding it byte by byte. + fn from_pages(image: &PagedMem) -> Self { + Self { + cells: image.map((0, 0), |value| (value, 0)), + lean: false, + hot: NO_PAGE, + missing: AtomicBool::new(false), + } + } + + /// The bytes its pages take on the heap. + pub(crate) fn heap_bytes(&self) -> usize { + self.cells.heap_bytes() } /// Read a byte from memory. Returns (value, timestamp) or (0, 0) if never written. fn read_byte(&self, address: u64) -> MemoryCell { - self.cells.get(address) + let cell = self.cells.get(address); + if cell.1 == ABSENT { + self.missing.store(true, Relaxed); + } + cell + } + + /// Whether a read found a byte the state does not hold, since the last + /// call; clears the mark. + #[cfg(feature = "parallel")] + fn take_missing(&self) -> bool { + self.missing.swap(false, Relaxed) + } + + /// Whether a read found a byte the state does not hold (the mark kept). + fn is_missing(&self) -> bool { + self.missing.load(Relaxed) } /// Write a byte to memory with the given timestamp. fn write_byte(&mut self, address: u64, value: u8, timestamp: u64) { - self.cells.set(address, (value, timestamp)); + if self.lean { + let (base, off) = PagedMem::::page_of(address); + let index = self.page_or_insert(base); + self.cells.page_set(index, off, (value, timestamp)); + } else { + self.cells.set(address, (value, timestamp)); + } } - /// Read multiple bytes. Returns arrays of values and timestamps. - fn read_bytes(&self, base_address: u64, count: usize) -> ([u32; 8], [u64; 8]) { - let mut values = [0u32; 8]; - let mut timestamps = [0u64; 8]; - for i in 0..count { - let (val, ts) = self.read_byte(base_address.wrapping_add(i as u64)); - values[i] = val as u32; - timestamps[i] = ts; + /// The index of the allocated page at `base`, through `hot` (lean only). + #[inline] + fn page(&mut self, base: u64) -> Option { + if self.hot.0 == base { + return Some(self.hot.1); } - (values, timestamps) + let index = self.cells.page_index(base)?; + self.hot = (base, index); + Some(index) } - /// Write multiple bytes from a value. - fn write_bytes(&mut self, base_address: u64, value: u64, count: usize, timestamp: u64) { - for i in 0..count { - let byte = ((value >> (i * 8)) & 0xFF) as u8; - self.write_byte(base_address.wrapping_add(i as u64), byte, timestamp); + /// The index of the page at `base`, allocated on first touch, through `hot` + /// (lean only). + #[inline] + fn page_or_insert(&mut self, base: u64) -> usize { + if self.hot.0 == base { + return self.hot.1; + } + let index = self.cells.page_index_or_insert(base); + self.hot = (base, index); + index + } + + /// `count` (≤ 8) bytes from `base_address` within one page: the access a + /// lean walk makes in one page lookup. `None` when it is not lean or the + /// bytes cross a page (or wrap past `u64::MAX`). + #[inline] + fn one_page(&self, base_address: u64, count: usize) -> Option<(u64, usize)> { + let (base, off) = PagedMem::::page_of(base_address); + (self.lean && count > 0 && off + count <= page::DEFAULT_PAGE_SIZE).then_some((base, off)) + } + + /// Read multiple bytes. Returns arrays of values and timestamps. + fn read_bytes(&mut self, base_address: u64, count: usize) -> ([u32; 8], [u64; 8]) { + let mut values = [0u32; 8]; + let mut timestamps = [0u64; 8]; + if let Some((base, off)) = self.one_page(base_address, count) { + // A page never allocated reads the fill throughout, as `read_byte`. + if let Some(index) = self.page(base) { + for i in 0..count { + let (val, ts) = self.cells.page_get(index, off + i); + values[i] = val as u32; + timestamps[i] = ts; + } + } else { + let (val, ts) = self.cells.fill(); + values[..count].fill(val as u32); + timestamps[..count].fill(ts); + } + if timestamps[..count].contains(&ABSENT) { + self.missing.store(true, Relaxed); + } + return (values, timestamps); + } + for i in 0..count { + let (val, ts) = self.read_byte(base_address.wrapping_add(i as u64)); + values[i] = val as u32; + timestamps[i] = ts; + } + (values, timestamps) + } + + /// Write multiple bytes from a value. + fn write_bytes(&mut self, base_address: u64, value: u64, count: usize, timestamp: u64) { + if let Some((base, off)) = self.one_page(base_address, count) { + let index = self.page_or_insert(base); + for i in 0..count { + let byte = ((value >> (i * 8)) & 0xFF) as u8; + self.cells.page_set(index, off + i, (byte, timestamp)); + } + return; + } + for i in 0..count { + let byte = ((value >> (i * 8)) & 0xFF) as u8; + self.write_byte(base_address.wrapping_add(i as u64), byte, timestamp); } } } /// Register state tracker for generating MEMW register traces. +#[derive(Clone)] struct RegisterState { /// Register file: (value, last_write_timestamp) regs: [RegisterCell; 32], @@ -313,7 +1328,7 @@ impl RegisterState { // ============================================================================= /// Get byte count and signed flag from CpuOperation memory flags. -fn cpu_op_to_bytes_and_signed(op: &CpuOperation) -> (usize, bool) { +fn cpu_op_to_bytes_and_signed(op: CpuOp<'_>) -> (usize, bool) { let f = &op.decode.fields; (f.mem_bytes(), f.mem_signed()) } @@ -338,31 +1353,120 @@ fn pack_register_value(value: u64) -> [u32; 8] { // Phase 1: Logs → CPU ops // ============================================================================= -/// Collects CPU operations from execution logs. +/// Collects CPU operations from execution logs, each holding its DECODE row +/// in `decode`. /// /// Returns a vector of CpuOperation, one per log entry. -fn collect_cpu_ops( +fn collect_cpu_ops(logs: &[Log], decode: &DecodeTable) -> Result, Error> { + collect_cpu_ops_from(logs, decode, 0) +} + +/// [`collect_cpu_ops`] for logs that start at cycle `first` of the run: the +/// timestamps are the run's, so a run collected window by window gets the ops a +/// whole-run collect would. +fn collect_cpu_ops_from( logs: &[Log], - instructions: &U64HashMap, + decode: &DecodeTable, + first: usize, ) -> Result, Error> { let mut cpu_ops = Vec::with_capacity(logs.len()); + collect_cpu_ops_into(logs, decode, first, &mut cpu_ops)?; + Ok(cpu_ops) +} +/// [`collect_cpu_ops_from`], appending to `cpu_ops`. +fn collect_cpu_ops_into( + logs: &[Log], + decode: &DecodeTable, + first: usize, + cpu_ops: &mut Vec, +) -> Result<(), Error> { + clock::check_clock( + clock::window_max_timestamp(first, logs.len()), + "the run's walk", + )?; // Timestamps start at 4 (not 0) to ensure old_timestamp < timestamp holds // for the first access to any register/memory location. The +4 stride reserves // per-cycle slots for M1/M3/M5 register accesses and the inline PC read. // Exactly 4 so that inline PC's prev_ts = timestamp - 3 = 1 on the first row, // matching the REGISTER table's initial PC token at timestamp 1 (per spec/memory.typ). for (i, log) in logs.iter().enumerate() { - let timestamp = (i as u64) * 4 + 4; - let instruction = instructions - .get(&log.current_pc) - .copied() + let timestamp = ((first + i) as u64) * 4 + 4; + let row = decode + .row(log.current_pc) .ok_or(Error::MissingInstruction(log.current_pc))?; + cpu_ops.push(CpuOperation::from_log( + log, + timestamp, + decode.entry(row), + row, + )); + } + Ok(()) +} - let op = CpuOperation::from_log_and_instruction(log, timestamp, instruction); - cpu_ops.push(op); +/// The parts of the lean walk a windowed build uses (D-EXEC E1). The tables are +/// the same with any of them; the whole-run build uses none. (Each cycle's +/// decode is always read from the [`DecodeTable`]: a CPU op holds its DECODE +/// row.) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct WalkLean { + /// Memory accesses through [`MemoryState`]'s page-found paths. + pub(crate) memory: bool, + /// The in-walk BITWISE lookups counted by the accumulator from the CPU and + /// LOAD ops instead of listed by the walk. + pub(crate) lookups: bool, + /// The routing in one pass over a window's CPU ops ([`route_ops_one_pass`]). + pub(crate) route: bool, +} + +impl WalkLean { + pub(crate) const ALL: Self = Self { + memory: true, + lookups: true, + route: true, + }; + pub(crate) const NONE: Self = Self { + memory: false, + lookups: false, + route: false, + }; + + /// `LAMBDA_VM_WALK_LEAN`: unset or `1` every part, `0` none, or a comma list + /// of `memory`, `lookups`, `route` (`decode` is accepted and always on). + /// Read once; any other value is refused. + pub(crate) fn from_env() -> Result { + static LEAN: std::sync::OnceLock> = std::sync::OnceLock::new(); + LEAN.get_or_init(|| match std::env::var("LAMBDA_VM_WALK_LEAN") { + Err(_) => Ok(Self::ALL), + Ok(v) => Self::parse(&v).ok_or_else(|| { + format!( + "LAMBDA_VM_WALK_LEAN={v}: 1, 0, or a comma list of decode, memory, lookups, route" + ) + }), + }) + .clone() + .map_err(Error::Prover) + } + + fn parse(v: &str) -> Option { + match v.trim() { + "1" => return Some(Self::ALL), + "0" => return Some(Self::NONE), + _ => {} + } + let mut lean = Self::NONE; + for part in v.split(',').map(str::trim) { + match part { + "decode" => {} + "memory" => lean.memory = true, + "lookups" => lean.lookups = true, + "route" => lean.route = true, + _ => return None, + } + } + Some(lean) } - Ok(cpu_ops) } // ============================================================================= @@ -382,8 +1486,7 @@ enum MemwRoute { } /// The single classification used everywhere a `MemwOperation` is routed to a -/// table — the walk's [`MemwBuckets`] and the sizing pass (`count_table_lengths`) -/// share it, so their routing cannot drift. +/// table, so the routing cannot drift between them. #[inline] fn classify_memw(op: &MemwOperation) -> MemwRoute { if is_register_op(op) { @@ -412,7 +1515,7 @@ fn classify_memw(op: &MemwOperation) -> MemwRoute { struct MemwBuckets { /// Compact register rows (filled directly into the MEMW_R columns). register_rows: Vec, - aligned: Vec, + aligned: Vec, general: Vec, } @@ -425,11 +1528,17 @@ impl MemwBuckets { } } + fn heap_bytes(&self) -> usize { + vec_heap_bytes(&self.register_rows) + + vec_heap_bytes(&self.aligned) + + vec_heap_bytes(&self.general) + } + #[inline] fn push(&mut self, op: MemwOperation) { match classify_memw(&op) { MemwRoute::Register => self.register_rows.push(RegRow::from_memw(&op)), - MemwRoute::Aligned => self.aligned.push(op), + MemwRoute::Aligned => self.aligned.push(memw_aligned::AlignedRow::from_memw(&op)), MemwRoute::General => self.general.push(op), } } @@ -441,8 +1550,7 @@ impl MemwBuckets { } /// Sink for `MemwOperation`s so `collect_register_ops_from_cpu` can feed either a plain -/// `Vec` (the `count_table_lengths` trace-sizing pass) or the classifying -/// [`MemwBuckets`] (the walk). +/// `Vec` or the classifying [`MemwBuckets`] (the walk). trait MemwSink { fn push_op(&mut self, op: MemwOperation); @@ -545,6 +1653,7 @@ fn build_reg_fallback( #[allow(clippy::type_complexity)] fn collect_ops_from_cpu( cpu_ops: &[CpuOperation], + decode: &DecodeTable, memory_state: &mut MemoryState, register_state: &mut RegisterState, ) -> Result< @@ -555,25 +1664,254 @@ fn collect_ops_from_cpu( Vec, Vec, Vec, + Vec, + Vec, Vec, Vec, Vec, Vec, Vec, + Vec, ), Error, > { - let mut memw = MemwBuckets::with_register_capacity(cpu_ops.len() * 3); - let mut load_ops = Vec::with_capacity(cpu_ops.len() / 8 + 1); - let mut lt_ops = Vec::with_capacity(cpu_ops.len() / 10 + 1); - let mut shift_ops = Vec::with_capacity(cpu_ops.len() / 10 + 1); - let mut bitwise_ops = Vec::with_capacity(cpu_ops.len() * 4); - let mut keccak_ops = Vec::new(); - let mut cpu32_ops = Vec::new(); - let mut ecsm_ops = Vec::new(); - let mut ecdas_ops = Vec::new(); - let mut copy_ops: Vec = Vec::new(); - let mut hint_ops = Vec::new(); + let mut out = WalkOutputs::with_capacity(cpu_ops.len()); + collect_ops_from_cpu_into( + cpu_ops, + decode, + memory_state, + register_state, + &mut out, + true, + )?; + let WalkOutputs { + memw, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + copy_ops, + hint_ops, + ecrec_ops, + } = out; + Ok(( + memw, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + copy_ops, + hint_ops, + ecrec_ops, + )) +} + +/// Everything the walk emits. [`collect_ops_from_cpu_into`] APPENDS to one of +/// these, so a run walked a window at a time into the same accumulator gets the +/// lists a single walk over the run gets: the walk is a left fold over the CPU +/// ops that reads and advances only the state its caller carries. +pub(crate) struct WalkOutputs { + memw: MemwBuckets, + load_ops: Vec, + lt_ops: Vec, + shift_ops: Vec, + bitwise_ops: Vec, + keccak_ops: Vec, + blake3_ops: Vec, + blake3_absorb_ops: Vec, + cpu32_ops: Vec, + ecsm_ops: Vec, + ecdas_ops: Vec, + copy_ops: Vec, + hint_ops: Vec, + ecrec_ops: Vec, +} + +impl WalkOutputs { + /// Sized for a walk over `cpu_ops` CPU ops, as the one-call walk sizes it. + fn with_capacity(cpu_ops: usize) -> Self { + Self::for_walk(cpu_ops, true) + } + + /// [`Self::with_capacity`], with no room for the in-walk lookups when the + /// walk leaves them out (`lookups` false, see [`collect_ops_from_cpu_into`]). + fn for_walk(cpu_ops: usize, lookups: bool) -> Self { + Self { + memw: MemwBuckets::with_register_capacity(cpu_ops * 3), + load_ops: Vec::with_capacity(cpu_ops / 8 + 1), + lt_ops: Vec::with_capacity(cpu_ops / 10 + 1), + shift_ops: Vec::with_capacity(cpu_ops / 10 + 1), + bitwise_ops: Vec::with_capacity(if lookups { cpu_ops * 4 } else { 0 }), + keccak_ops: Vec::new(), + blake3_ops: Vec::new(), + blake3_absorb_ops: Vec::new(), + cpu32_ops: Vec::new(), + ecsm_ops: Vec::new(), + ecdas_ops: Vec::new(), + copy_ops: Vec::new(), + hint_ops: Vec::new(), + ecrec_ops: Vec::new(), + } + } + + /// [`Self::for_walk`] with the streamed lists sized from an earlier walk's + /// lengths per cycle (`sizes`, an eighth over), so a window's lists are + /// allocated once instead of grown from empty or from a fixed guess. + pub(crate) fn for_walk_sized(cpu_ops: usize, lookups: bool, sizes: &WalkSizes) -> Self { + let mut out = Self::for_walk(cpu_ops, lookups); + if sizes.cycles == 0 { + return out; + } + let size = |len: usize| { + let scaled = (len as u128 * cpu_ops as u128 / sizes.cycles as u128) as usize; + scaled + scaled / 8 + 16 + }; + out.memw.register_rows = Vec::with_capacity(size(sizes.register_rows)); + out.memw.aligned = Vec::with_capacity(size(sizes.aligned)); + out.memw.general = Vec::with_capacity(size(sizes.general)); + out.load_ops = Vec::with_capacity(size(sizes.load)); + out.lt_ops = Vec::with_capacity(size(sizes.lt)); + out.shift_ops = Vec::with_capacity(size(sizes.shift)); + out + } + + /// The bytes its lists take on the heap (capacities, not lengths). + pub(crate) fn heap_bytes(&self) -> usize { + self.memw.heap_bytes() + + vec_heap_bytes(&self.load_ops) + + vec_heap_bytes(&self.lt_ops) + + vec_heap_bytes(&self.shift_ops) + + vec_heap_bytes(&self.bitwise_ops) + + vec_heap_bytes(&self.keccak_ops) + + vec_heap_bytes(&self.blake3_ops) + + vec_heap_bytes(&self.blake3_absorb_ops) + + vec_heap_bytes(&self.cpu32_ops) + + vec_heap_bytes(&self.ecsm_ops) + + self + .ecsm_ops + .iter() + .map(ecsm::EcsmOperation::steps_heap_bytes) + .sum::() + + vec_heap_bytes(&self.ecdas_ops) + + vec_heap_bytes(&self.copy_ops) + + vec_heap_bytes(&self.hint_ops) + + vec_heap_bytes(&self.ecrec_ops) + + self + .ecrec_ops + .iter() + .map(ecrec::EcrecOperation::steps_heap_bytes) + .sum::() + } +} + +/// The lengths of a walk's streamed lists and the cycles it walked: what +/// [`WalkOutputs::for_walk_sized`] sizes the next window's lists from. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct WalkSizes { + cycles: usize, + register_rows: usize, + aligned: usize, + general: usize, + load: usize, + lt: usize, + shift: usize, +} + +impl WalkSizes { + /// `walk`'s lengths over `cycles` cycles. + pub(crate) fn of(walk: &WalkOutputs, cycles: usize) -> Self { + Self { + cycles, + register_rows: walk.memw.register_rows.len(), + aligned: walk.memw.aligned.len(), + general: walk.memw.general.len(), + load: walk.load_ops.len(), + lt: walk.lt_ops.len(), + shift: walk.shift_ops.len(), + } + } +} + +/// The bytes `list`'s buffer takes on the heap: its capacity, not its length. +/// Elements that own heap memory of their own are counted at their inline size. +pub(crate) fn vec_heap_bytes(list: &Vec) -> usize { + list.capacity() * std::mem::size_of::() +} + +/// The walk over `cpu_ops` (read with `decode`), appended to `out` (see +/// [`WalkOutputs`]). Without `lookups` the walk leaves out the in-walk BITWISE +/// lookups (each CPU op's and each LOAD op's), which are a function of the CPU +/// and LOAD ops it emits: the caller counts them from those +/// ([`CpuOp::count_bitwise_into`], [`LoadOperation::count_bitwise_into`]). +#[inline] +fn collect_ops_from_cpu_into( + cpu_ops: &[CpuOperation], + decode: &DecodeTable, + memory_state: &mut MemoryState, + register_state: &mut RegisterState, + out: &mut WalkOutputs, + lookups: bool, +) -> Result<(), Error> { + walk_cpu_ops::(cpu_ops, decode, memory_state, register_state, out, lookups) +} + +/// The regeneration walk (D-REGEN §2.5): [`collect_ops_from_cpu_into`] without +/// the in-walk lookups, reading and advancing the same state through the same +/// code, but emitting only the lists of the tables a windowed build streams +/// (MEMW_R, MEMW_A, MEMW, LOAD, LT, SHIFT). Everything only the finish reads +/// (CPU32, COPY, KECCAK, BLAKE3, ECSM, ECDAS and HINT ops) is left out, and +/// those lists stay empty. +fn collect_streamed_ops_from_cpu_into( + cpu_ops: &[CpuOperation], + decode: &DecodeTable, + memory_state: &mut MemoryState, + register_state: &mut RegisterState, + out: &mut WalkOutputs, +) -> Result<(), Error> { + walk_cpu_ops::(cpu_ops, decode, memory_state, register_state, out, false) +} + +/// The walk's one body: with `REST` every list ([`collect_ops_from_cpu_into`]), +/// without it only the streamed tables' ([`collect_streamed_ops_from_cpu_into`]). +/// The state advances the same either way. +fn walk_cpu_ops( + cpu_ops: &[CpuOperation], + decode: &DecodeTable, + memory_state: &mut MemoryState, + register_state: &mut RegisterState, + out: &mut WalkOutputs, + lookups: bool, +) -> Result<(), Error> { + let WalkOutputs { + memw, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + copy_ops, + hint_ops, + ecrec_ops, + } = out; + // This call's COPY rows start here; earlier calls' are already in the list. + let copy_ops_before = copy_ops.len(); // Seed from the carried x254 (0 for a monolithic run or the first epoch) so a // continuation epoch indexes its commits globally, matching the x254 the // register binding transports across epochs. Resetting to 0 here would drift @@ -582,10 +1920,11 @@ fn collect_ops_from_cpu( let mut current_commit_index = start_commit_index; for op in cpu_ops { + let op = decode.op(op); // Word (`*W`) instructions delegate to the CPU32 table (built in program // order; its register accesses are still emitted via the shared register // collector below so the MEMW table balances). - if op.decode.fields.word_instr { + if REST && op.decode.fields.word_instr { cpu32_ops.push(build_cpu32_op(op)); } @@ -593,44 +1932,49 @@ fn collect_ops_from_cpu( // Collect memory operations for Load/Store instructions if op.decode.fields.is_load() { - let (memw_op, load_op, lookups) = collect_load_op_from_cpu(op, memory_state); + let (memw_op, load_op) = collect_load_op_from_cpu(op, memory_state); memw.push(memw_op); + if lookups { + // MSB8 lookups for the sign bit extraction. + bitwise_ops.extend(load_op.collect_bitwise_ops()); + } load_ops.push(load_op); - bitwise_ops.extend(lookups); } else if op.decode.fields.is_store() { let memw_op = collect_store_op_from_cpu(op, memory_state); memw.push(memw_op); } // Collect register operations (M1, M3, M5) - collect_register_ops_from_cpu(op, register_state, &mut memw); + collect_register_ops_from_cpu(op, register_state, memw); - // The `write` syscall: the COPY chip receives the ECALL, reads the operand + // The commit syscall (ecall 64): the COPY chip receives the ECALL, reads the operand // registers, advances x254 and moves the bytes into the COMMIT domain. - if op.ecall_commit { + if op.ecall_commit() { let index = u64::from(current_commit_index); memw.extend_ops(collect_copy_register_ops( copy::Functionality::Write, op.timestamp, - op.commit_buf_addr, + op.commit_buf_addr(), index, - op.commit_count, + op.commit_count(), register_state, )?); let (write_memw, rows) = collect_copy_ops( copy::Functionality::Write, op.timestamp, - op.commit_buf_addr, + op.commit_buf_addr(), index, - op.commit_count, + op.commit_count(), memory_state, ); memw.extend_ops(write_memw); - copy_ops.extend(rows); - let count = u32::try_from(op.commit_count).map_err(|_| { + if REST { + copy_ops.extend(rows); + } + let count = u32::try_from(op.commit_count()).map_err(|_| { Error::Prover(format!( "commit_count {} exceeds u32 range", - op.commit_count + op.commit_count() )) })?; current_commit_index = current_commit_index @@ -644,8 +1988,8 @@ fn collect_ops_from_cpu( } // Collect KeccakPermute ECALL operations - if op.ecall_keccak { - let state_addr = op.keccak_state_addr; + if op.ecall_keccak() { + let state_addr = op.keccak_state_addr(); let mut input = [0u64; 25]; for (i, lane) in input.iter_mut().enumerate() { let addr = state_addr @@ -667,26 +2011,119 @@ fn collect_ops_from_cpu( let keccak_memw_ops = collect_keccak_memw_ops(op, &input, &output, memory_state, register_state); memw.extend_ops(keccak_memw_ops); - keccak_ops.push(KeccakOperation { - timestamp: op.timestamp, - state_addr, - input, - output, - }); + if REST { + keccak_ops.push(KeccakOperation { + timestamp: op.timestamp, + state_addr, + input, + output, + sponge: None, + }); + } + } + + // Keccak256 ECALL: one KECCAK row per block of the input. + if op.ecall_keccak256() { + let (sponge_memw, rows) = collect_keccak256_ops(op, memory_state, register_state); + memw.extend_ops(sponge_memw); + if REST { + keccak_ops.extend(rows); + } + } + + // Collect Blake3Compress ECALL operations + if op.ecall_blake3() { + let state_addr = op.blake3_state_addr(); + // 14 input dwords: h | m | t | (block_len, flags), LE words. + let mut words = [0u32; 28]; + for k in 0..14usize { + let dword_addr = state_addr + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + let mut dw = 0u64; + for b in 0..8 { + let byte_addr = dword_addr + .checked_add(b as u64) + .expect("blake3 state address range must be validated by the executor"); + let (byte_val, _ts) = memory_state.read_byte(byte_addr); + dw |= (byte_val as u64) << (b * 8); + } + words[2 * k] = dw as u32; + words[2 * k + 1] = (dw >> 32) as u32; + } + let h: [u32; 8] = words[0..8].try_into().unwrap(); + let m: [u32; 16] = words[8..24].try_into().unwrap(); + let t = (words[24] as u64) | ((words[25] as u64) << 32); + let block_len = words[26]; + let flags = words[27]; + let out = executor::vm::instruction::execution::blake3_compress_6round( + &h, &m, t, block_len, flags, + ); + // Previous content of the out region, read BEFORE the write ops + // below advance memory_state (only the BLAKE3 op holds it). + let mut old_out = [0u8; 64]; + if REST { + for (b, byte) in old_out.iter_mut().enumerate() { + let byte_addr = state_addr + .checked_add(112 + b as u64) + .expect("blake3 state address range must be validated by the executor"); + let (v, _ts) = memory_state.read_byte(byte_addr); + *byte = v; + } + } + let blake3_memw_ops = + collect_blake3_memw_ops(op, &words, &out, memory_state, register_state); + if !strip_blake3_side_effects() { + memw.extend_ops(blake3_memw_ops); + } + if REST { + blake3_ops.push(Blake3Operation { + timestamp: op.timestamp, + state_addr, + h, + m, + t, + block_len, + flags, + old_out, + out, + }); + } + } + + // Collect Blake3Absorb ECALL operations. One ecall becomes a whole GROUP + // of `num_blocks + 1` BLAKE3 rows, so unlike every other accelerator the + // chip op collected here is a group rather than a row. + if op.ecall_blake3_absorb() { + let (absorb_memw, absorb_op) = + collect_blake3_absorb_ops(op, memory_state, register_state); + // Same strip gate as the single-compression MEMW above: the switch + // promises "MEMW and BITWISE left out" for BOTH modes, and an + // ungated arm here would silently reintroduce a MEMW imbalance the + // moment the omission forgery is retargeted at an absorb workload. + if !strip_blake3_side_effects() { + memw.extend_ops(absorb_memw); + } + if REST { + blake3_absorb_ops.push(absorb_op); + } } // Collect ECSM ecall operations (memory I/O + the two table row sets) - if op.ecall_ecsm { - let (ecsm_memw, ecsm_op, ecdas_rows) = - collect_ecsm_ops(op, memory_state, register_state); + if op.ecall_ecsm() + && let Some((ecsm_memw, ecsm_op, ecdas_rows)) = + collect_ecsm_ops(op, memory_state, register_state) + { memw.extend_ops(ecsm_memw); - ecsm_ops.push(ecsm_op); - ecdas_ops.extend(ecdas_rows); + if REST { + ecsm_ops.push(ecsm_op); + ecdas_ops.extend(ecdas_rows); + } } // DMA memcpy: authenticate x10/x11/x12, snapshot all source bytes at // T, then write all destination bytes at T+1. - if op.ecall_dma_memcpy { + if op.ecall_dma_memcpy() { let dst = register_state.read(10).0; let src = register_state.read(11).0; let count = register_state.read(12).0; @@ -707,7 +2144,9 @@ fn collect_ops_from_cpu( memory_state, ); memw.extend_ops(mm_memw); - copy_ops.extend(rows); + if REST { + copy_ops.extend(rows); + } } // DMA memset: authenticate x10/x11/x12, then run the copy primitive with the @@ -715,7 +2154,7 @@ fn collect_ops_from_cpu( // phase, and the self-overlap is the point: the stub seeds eight bytes and // calls with `dst = src + 8`, so each row's read observes the write eight // bytes back and the seed propagates. That is what `copy:c:set_gap` pins. - if op.ecall_dma_memset { + if op.ecall_dma_memset() { // memset is a copy call whose only distinguishing feature is the // inverted timestamp order; the stub already seeded the first eight // bytes and passed dst = seed_end, src = seed_start, count = n - 8. @@ -739,14 +2178,31 @@ fn collect_ops_from_cpu( memory_state, ); memw.extend_ops(memset_memw); - copy_ops.extend(rows); + if REST { + copy_ops.extend(rows); + } + } + + // Collect ECREC ecall operations (memory I/O, the ECREC row, and the ECDAS rows of its + // two chains and its final addition). + if op.ecall_ecrec() + && let Some((ecrec_memw, ecrec_op, ecrec_ecdas)) = + collect_ecrec_ops(op, memory_state, register_state) + { + memw.extend_ops(ecrec_memw); + if REST { + ecrec_ops.push(ecrec_op); + ecdas_ops.extend(ecrec_ecdas); + } } // Collect Hint ecall operations (the 32-byte output write). - if op.ecall_hint { + if op.ecall_hint() { let (hint_memw, hint_op) = collect_hint_ops(op, memory_state, register_state); memw.extend_ops(hint_memw); - hint_ops.push(hint_op); + if REST { + hint_ops.push(hint_op); + } } // --- ALU chip dispatch (no state tracking) --- @@ -760,7 +2216,7 @@ fn collect_ops_from_cpu( if f.is_lt() { lt_ops.push(LtOperation::new_with_invert( op.rv1, - op.arg2, + op.arg2(), f.alu_signed(), f.alu_signed2_or_invert(), )); @@ -771,7 +2227,7 @@ fn collect_ops_from_cpu( if f.is_shift() { shift_ops.push(ShiftOperation::new( op.rv1, - op.arg2, + op.arg2(), f.alu_signed2_or_invert(), f.alu_signed(), f.word_instr, @@ -782,44 +2238,36 @@ fn collect_ops_from_cpu( // Collect CPU range-check bitwise lookups (ARE_BYTES + IS_HALF). Kept serial here: // it's only ~110 ms (a serial `.extend` into one growing Vec), and moving it to a // rayon `flat_map`-collect over 6.8 M per-op Vecs regressed p4 ~4× (alloc + merge). - bitwise_ops.extend(op.collect_bitwise_ops()); + if lookups { + bitwise_ops.extend(op.collect_bitwise_ops()); + } } // There is no COMMIT table any more: COPY receives the `write` ECALL itself. The // committed length is checked against the rows that actually move the bytes, and - // a zero-length `write` contributes a single row that moves none. - debug_assert_eq!( - copy_ops - .iter() - .filter(|op| op.functionality == copy::Functionality::Write && op.count != 0) - .map(|op| op.width as u64) - .sum::(), - u64::from(current_commit_index - start_commit_index), + // a zero-length `write` contributes a single row that moves none. Only this call's + // rows count, so the carried start index is subtracted. + debug_assert!( + !REST + || copy_ops[copy_ops_before..] + .iter() + .filter(|op| op.functionality == copy::Functionality::Write && op.count != 0) + .map(|op| op.width as u64) + .sum::() + == u64::from(current_commit_index - start_commit_index), "COPY should move exactly the committed byte count" ); - - Ok(( - memw, - load_ops, - lt_ops, - shift_ops, - bitwise_ops, - keccak_ops, - cpu32_ops, - ecsm_ops, - ecdas_ops, - copy_ops, - hint_ops, - )) + Ok(()) } /// Collects a LOAD operation and corresponding MEMW read from CpuOperation. /// -/// Returns: (memw_op, load_op, bitwise_ops) +/// Returns: (memw_op, load_op); the LOAD op's BITWISE lookups are +/// [`LoadOperation::collect_bitwise_ops`]. fn collect_load_op_from_cpu( - op: &CpuOperation, + op: CpuOp<'_>, memory_state: &mut MemoryState, -) -> (MemwOperation, LoadOperation, Vec) { +) -> (MemwOperation, LoadOperation) { // res contains the effective address (base + offset) let base_address = op.res; let (byte_count, signed) = cpu_op_to_bytes_and_signed(op); @@ -863,22 +2311,20 @@ fn collect_load_op_from_cpu( op.timestamp, byte_count as u8, signed, - res_bytes.map(u64::from), + // Bytes: each `& 0xFF` or the 0/0xFF fill. + res_bytes.map(|byte| byte as u8), ); - // Collect MSB8 lookups for sign bit extraction - let bitwise_ops = load_op.collect_bitwise_ops(); - // Update memory state memory_state.write_bytes(base_address, loaded_value, byte_count, op.timestamp); - (memw_op, load_op, bitwise_ops) + (memw_op, load_op) } /// Collects a STORE operation as a MEMW write from CpuOperation. /// /// Returns: memw_op -fn collect_store_op_from_cpu(op: &CpuOperation, memory_state: &mut MemoryState) -> MemwOperation { +fn collect_store_op_from_cpu(op: CpuOp<'_>, memory_state: &mut MemoryState) -> MemwOperation { // res contains the effective address (base + offset) let base_address = op.res; let (byte_count, _) = cpu_op_to_bytes_and_signed(op); @@ -919,7 +2365,11 @@ fn collect_store_op_from_cpu(op: &CpuOperation, memory_state: &mut MemoryState) memw_op } -/// Collects all MEMW ops and the ECSM / ECDAS table ops for one ECSM ecall. +/// Collects all MEMW ops and the ECSM / ECDAS table ops for one ECSM ecall; +/// `None`, with nothing collected, when an operand byte is one the memory state +/// does not hold (a window walked from entries that miss it: the window is +/// refused, and its witness is not computed from operands that are not the +/// run's). /// /// Timestamp scheme: `x11` register read and `xG` memory reads at `T`; /// `x12` register read and `k` memory reads at `T + 1`; `x10` register read and @@ -928,14 +2378,14 @@ fn collect_store_op_from_cpu(op: &CpuOperation, memory_state: &mut MemoryState) /// accesses always observe a strictly smaller old timestamp. #[allow(clippy::needless_range_loop)] fn collect_ecsm_ops( - op: &CpuOperation, + op: CpuOp<'_>, memory_state: &mut MemoryState, register_state: &mut RegisterState, -) -> ( +) -> Option<( Vec, ecsm::EcsmOperation, Vec, -) { +)> { let t = op.timestamp; let addr_xr = register_state.read(10).0; let addr_xg = register_state.read(11).0; @@ -948,8 +2398,11 @@ fn collect_ecsm_ops( xg[i] = memory_state.read_byte(addr_xg.wrapping_add(i as u64)).0; k[i] = memory_state.read_byte(addr_k.wrapping_add(i as u64)).0; } + if memory_state.is_missing() { + return None; + } - let witness = ::ecsm::compute_witness(&k, &xg) + let mut witness = ::ecsm::compute_witness(&k, &xg) .expect("ECSM witness: executor validates 0 < k < N and xG on curve"); let mut memw_ops = Vec::with_capacity(15); @@ -1032,10 +2485,10 @@ fn collect_ecsm_ops( memory_state.write_bytes(addr, dword, 8, t + 2); } - let ecdas_ops = witness - .steps - .iter() - .cloned() + // The steps become the ECDAS ops; the ECSM op keeps none (its table and + // its lookups read the witness's other fields alone). + let ecdas_ops = std::mem::take(&mut witness.steps) + .into_iter() .map(|step| ecdas::EcdasOperation { timestamp: t, step }) .collect(); let ecsm_op = ecsm::EcsmOperation { @@ -1046,7 +2499,187 @@ fn collect_ecsm_ops( witness, }; - (memw_ops, ecsm_op, ecdas_ops) + Some((memw_ops, ecsm_op, ecdas_ops)) +} + +/// Test-only forgery switch: while set, the ECREC call whose `r` is +/// [`FORGED_ECREC_R`] runs its two chains with their scalars swapped — the first +/// chain `u2·G`, the second `u1·R` — and writes the key `u2·G + u1·R` that gives. +/// The row's scalar columns, relations and `LEN` swap stay consistent, so the only +/// thing that refuses it is the second chain's key `TS + 1`: with both chains keyed +/// at `TS`, the `Bit` messages of the two are one multiset and balance either way. +/// Keyed on `test_ecrec`'s first `r`, so tests beside the forging one are unaffected. +#[cfg(test)] +pub(crate) static FORGE_ECREC_SWAPPED_CHAINS: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// `r` of `test_ecrec`'s first call, little-endian. +#[cfg(test)] +pub(crate) const FORGED_ECREC_R: [u8; 32] = { + let be: [u8; 32] = [ + 0x32, 0x84, 0xba, 0xe4, 0xa1, 0xb6, 0x93, 0xdf, 0xbe, 0x8a, 0xc6, 0x74, 0x00, 0xe4, 0x89, + 0x1f, 0x3b, 0x53, 0x68, 0x1c, 0x51, 0x59, 0xa3, 0x15, 0x56, 0x10, 0x51, 0xf3, 0xd2, 0xc5, + 0x71, 0x39, + ]; + let mut le = [0u8; 32]; + let mut i = 0; + while i < 32 { + le[i] = be[31 - i]; + i += 1; + } + le +}; + +/// Applies [`FORGE_ECREC_SWAPPED_CHAINS`] to a computed witness. +#[cfg(test)] +fn forge_ecrec_swapped_chains(w: &mut ::ecsm::EcrecWitness) { + use std::sync::atomic::Ordering; + if !FORGE_ECREC_SWAPPED_CHAINS.load(Ordering::SeqCst) || !w.ok || w.r != FORGED_ECREC_R { + return; + } + swap_ecrec_chains(w); +} + +/// The forgery of [`FORGE_ECREC_SWAPPED_CHAINS`] on a computed witness, unconditionally. +#[cfg(test)] +pub(crate) fn swap_ecrec_chains(w: &mut ::ecsm::EcrecWitness) { + let sub_p = |v: &[u8; 32]| { + let mut out = [0u8; 32]; + let mut borrow = 0i16; + for i in 0..32 { + let d = v[i] as i16 - ::ecsm::P_BYTES[i] as i16 - borrow; + borrow = i16::from(d < 0); + out[i] = d.rem_euclid(256) as u8; + } + out + }; + let (gx, gy) = ::ecsm::generator_le(); + let (c1, xa, ya) = ::ecsm::ecdas_chain(&w.u2, &gx, &gy).expect("u2·G"); + let (c2, xb, yb) = ::ecsm::ecdas_chain(&w.u1, &w.r, &w.y_r).expect("u1·R"); + let add = ::ecsm::ecdas_add_step(&xa, &ya, &xb, &yb).expect("A ≠ ±B"); + (w.x_a, w.y_a, w.x_b, w.y_b) = (xa, ya, xb, yb); + (w.x_q, w.y_q) = (add.x_r, add.y_r); + w.x_a_sub_p = sub_p(&xa); + w.x_b_sub_p = sub_p(&xb); + w.x_q_sub_p = sub_p(&add.x_r); + w.y_q_sub_p = sub_p(&add.y_r); + (w.len1, w.len2) = (w.len2, w.len1); + (w.chain1, w.chain2, w.add) = (c1, c2, Some(add)); +} + +/// Collects all MEMW ops, the ECREC op and the ECDAS ops for one ECREC ecall; +/// `None`, with nothing collected, when an operand byte is one the memory state +/// does not hold (as [`collect_ecsm_ops`]). +/// +/// Timestamp scheme: `x11` register read and the signature reads (8 doublewords) +/// at `T`; `x12` and `x13` register reads and the hash reads (4 doublewords) at +/// `T + 1`; `x10` register read and the output writes (9 doublewords: the key, +/// then the status) at `T + 2`. The ECDAS rows: the first chain at `T`, the +/// second at `T + 1`, the final addition at `T + 2`. +fn collect_ecrec_ops( + op: CpuOp<'_>, + memory_state: &mut MemoryState, + register_state: &mut RegisterState, +) -> Option<( + Vec, + ecrec::EcrecOperation, + Vec, +)> { + let t = op.timestamp; + let addr_out = register_state.read(10).0; + let addr_sig = register_state.read(11).0; + let addr_msg = register_state.read(12).0; + let parity = register_state.read(13).0; + + let mut sig = [0u8; 64]; + let mut msg = [0u8; 32]; + for (i, b) in sig.iter_mut().enumerate() { + *b = memory_state.read_byte(addr_sig.wrapping_add(i as u64)).0; + } + for (i, b) in msg.iter_mut().enumerate() { + *b = memory_state.read_byte(addr_msg.wrapping_add(i as u64)).0; + } + if memory_state.is_missing() { + return None; + } + let mut witness = ::ecsm::compute_ecrec_witness(&sig, &msg, parity as u8); + #[cfg(test)] + forge_ecrec_swapped_chains(&mut witness); + + let mut memw_ops = Vec::with_capacity(25); + let mut reg_read = |reg: u8, at: u64, memw_ops: &mut Vec| { + let (val, old_ts) = register_state.read(reg); + let value = pack_register_value(val); + memw_ops.push( + MemwOperation::new(true, 2 * reg as u64, value, at, 2, true) + .with_old(value, [old_ts, old_ts, 0, 0, 0, 0, 0, 0]), + ); + register_state.write(reg, val, at); + }; + let mut reads = |base: u64, bytes: &[u8], at: u64, memw_ops: &mut Vec| { + for (i, chunk) in bytes.chunks_exact(8).enumerate() { + let addr = base.wrapping_add(8 * i as u64); + let value: [u32; 8] = std::array::from_fn(|j| chunk[j] as u32); + let dword = u64::from_le_bytes(chunk.try_into().expect("8 bytes")); + let (_old, old_ts) = memory_state.read_bytes(addr, 8); + memw_ops + .push(MemwOperation::new(false, addr, value, at, 8, true).with_old(value, old_ts)); + memory_state.write_bytes(addr, dword, 8, at); + } + }; + reg_read(11, t, &mut memw_ops); + reads(addr_sig, &sig, t, &mut memw_ops); + reg_read(12, t + 1, &mut memw_ops); + reg_read(13, t + 1, &mut memw_ops); + reads(addr_msg, &msg, t + 1, &mut memw_ops); + reg_read(10, t + 2, &mut memw_ops); + + // The written bytes: the key big-endian (zero on a bail), then the status doubleword. + let mut out = [0u8; 72]; + for i in 0..32 { + out[i] = witness.x_q[31 - i]; + out[32 + i] = witness.y_q[31 - i]; + } + out[64] = witness.status(); + for (i, chunk) in out.chunks_exact(8).enumerate() { + let addr = addr_out.wrapping_add(8 * i as u64); + let value: [u32; 8] = std::array::from_fn(|j| chunk[j] as u32); + let dword = u64::from_le_bytes(chunk.try_into().expect("8 bytes")); + let (old_vals, old_ts) = memory_state.read_bytes(addr, 8); + memw_ops.push( + MemwOperation::new(false, addr, value, t + 2, 8, false).with_old(old_vals, old_ts), + ); + memory_state.write_bytes(addr, dword, 8, t + 2); + } + + // The ECDAS rows move out of the witness; the ECREC op keeps none. + let mut ecdas_ops: Vec = Vec::new(); + for (steps, at) in [ + (std::mem::take(&mut witness.chain1), t), + ( + std::mem::take(&mut witness.chain2), + t.wrapping_add_signed(ecrec::CHAIN2_KEY), + ), + ] { + ecdas_ops.extend(steps.into_iter().map(|step| ecdas::EcdasOperation { + timestamp: at, + step, + })); + } + if let Some(step) = witness.add.take() { + ecdas_ops.push(ecdas::EcdasOperation { + timestamp: t.wrapping_add_signed(ecrec::ADD_KEY), + step, + }); + } + let ecrec_op = ecrec::EcrecOperation { + timestamp: t, + addr_out, + addr_sig, + addr_msg, + witness, + }; + Some((memw_ops, ecrec_op, ecdas_ops)) } /// The register accesses one copy ecall makes, all at the raw ECALL timestamp. @@ -1257,33 +2890,9 @@ fn collect_copy_ops( (memw_ops, rows) } -/// Sizing-pass replay of one COPY-driven ecall. -/// -/// Deliberately delegates to [`collect_copy_ops`] rather than re-deriving the -/// schedule: the two used to be separate implementations pinned together by an -/// assertion, and the schedule is now a function of `dst` as well as `count`, which -/// is exactly the kind of thing that drifts. The cost is one allocation per ecall. -#[cfg(feature = "disk-spill")] -fn replay_copy_for_sizing( - functionality: copy::Functionality, - timestamp: u64, - src: u64, - dst: u64, - count: u64, - memory_state: &mut MemoryState, - mut visit_memw: impl FnMut(&MemwOperation), -) -> u64 { - let (memw_ops, rows) = - collect_copy_ops(functionality, timestamp, src, dst, count, memory_state); - for op in &memw_ops { - visit_memw(op); - } - rows.len() as u64 -} - -/// Test hook for the schedule, so the COPY unit tests can pin it. -#[cfg(test)] -pub fn copy_row_width_for_test( +/// Test hook for the schedule, so the COPY unit tests can pin it. +#[cfg(test)] +pub fn copy_row_width_for_test( src: u64, dst: u64, offset: u64, @@ -1309,6 +2918,102 @@ pub fn copy_ops_for_test( collect_copy_ops(functionality, 100, src, dst, count, &mut memory_state).1 } +/// Collects the memory operations and KECCAK rows of a `keccak256` ecall. +/// +/// Recomputes the call from the replayed state, as the walk does for every +/// accelerator: x10/x11/x12 (input address, length, digest address) read back +/// at `T`; the input read from memory; one KECCAK row per block +/// ([`keccak::sponge_ops`], the executor's own permutations). The memory +/// traffic mirrors the chip's sends: per block, each lane wholly inside the +/// input as one 8-byte read and each input byte of the lane the input ends in +/// as a 1-byte read, all at `T`; then the 32-byte digest as four 8-byte writes +/// at `T + 1`. +fn collect_keccak256_ops( + op: CpuOp<'_>, + memory_state: &mut MemoryState, + register_state: &mut RegisterState, +) -> (Vec, Vec) { + let t = op.timestamp; + let ptr = register_state.read(10).0; + let len = register_state.read(11).0; + let out = register_state.read(12).0; + debug_assert!( + executor::vm::instruction::execution::keccak256_addresses_ok(ptr, len, out), + "keccak256 operands must be validated by the executor" + ); + + let blocks = executor::vm::instruction::execution::keccak256_blocks(len) as usize; + let mut memw_ops = Vec::with_capacity(3 + 17 * blocks + 7 + 4); + + // Bind x10/x11/x12 at T through the memory argument (register reads). + for (reg, value) in [(10u8, ptr), (11, len), (12, out)] { + let reg_value = pack_register_value(value); + let (_old_val, old_ts) = register_state.read(reg); + memw_ops.push( + MemwOperation::new(true, 2 * reg as u64, reg_value, t, 2, true) + .with_old(reg_value, [old_ts, old_ts, 0, 0, 0, 0, 0, 0]), + ); + register_state.write(reg, value, t); + } + + let input: Vec = (0..len) + .map(|i| memory_state.read_byte(ptr.wrapping_add(i)).0) + .collect(); + #[allow(unused_mut)] + let mut rows = keccak::sponge_ops(t, ptr, out, &input); + #[cfg(test)] + keccak::forge(&mut rows); + + // The reads, block by block, as the chip sends them. + for row in &rows { + let sponge = row.sponge.as_ref().expect("a sponge row"); + for lane in 0..keccak::cols::RATE_LANES { + if sponge.full(lane) { + if lane == 0 && keccak::dropped(keccak::SpongeCheck::LaneRead) { + continue; + } + let addr = sponge.src.wrapping_add(8 * lane as u64); + let (values, old_ts) = memory_state.read_bytes(addr, 8); + let dword = values + .iter() + .enumerate() + .fold(0u64, |acc, (b, &v)| acc | (u64::from(v) << (8 * b))); + memw_ops.push( + MemwOperation::new(false, addr, values, t, 8, true).with_old(values, old_ts), + ); + memory_state.write_bytes(addr, dword, 8, t); + continue; + } + for byte in 0..7 { + if !sponge.part(lane, byte) { + continue; + } + let addr = sponge.src.wrapping_add((8 * lane + byte) as u64); + let (values, old_ts) = memory_state.read_bytes(addr, 1); + memw_ops.push( + MemwOperation::new(false, addr, values, t, 1, true).with_old(values, old_ts), + ); + memory_state.write_bytes(addr, u64::from(values[0]), 1, t); + } + } + } + + // The digest: the last row's lanes 0..4, at T + 1. + let last = rows.last().expect("at least one block"); + let out = last.sponge.as_ref().expect("a sponge row").out; + for (lane, &dword) in last.output.iter().take(4).enumerate() { + let addr = out.wrapping_add(8 * lane as u64); + let value: [u32; 8] = std::array::from_fn(|b| ((dword >> (8 * b)) & 0xFF) as u32); + let (old_vals, old_ts) = memory_state.read_bytes(addr, 8); + memw_ops.push( + MemwOperation::new(false, addr, value, t + 1, 8, false).with_old(old_vals, old_ts), + ); + memory_state.write_bytes(addr, dword, 8, t + 1); + } + + (memw_ops, rows) +} + /// Collects the memory operations for a `Hint` ecall. /// /// The `hint` ecall writes a 32-byte value (a modular inverse / sqrt) to guest @@ -1323,7 +3028,7 @@ pub fn copy_ops_for_test( /// the guest supplied the input via ordinary stores). The value itself is /// unconstrained — soundness lives in the guest's in-circuit verify. fn collect_hint_ops( - op: &CpuOperation, + op: CpuOp<'_>, memory_state: &mut MemoryState, register_state: &mut RegisterState, ) -> (Vec, hint::HintOperation) { @@ -1387,7 +3092,7 @@ fn collect_hint_ops( /// Collects register read/write operations (M1, M3, M5) from CpuOperation, /// pushing them into `memw_ops`. fn collect_register_ops_from_cpu( - op: &CpuOperation, + op: CpuOp<'_>, register_state: &mut RegisterState, memw_ops: &mut S, ) { @@ -1510,7 +3215,7 @@ fn cpu32_res(c: &cpu32::Cpu32Operation, arg1: u64, arg2: u64) -> u64 { /// Builds the CPU32 row for a word (`*W`) instruction. `op.rv1/rv2/rvd` carry the /// real register values (the main CPU delegate row zeroes its own columns). -fn build_cpu32_op(op: &CpuOperation) -> cpu32::Cpu32Operation { +fn build_cpu32_op(op: CpuOp<'_>) -> cpu32::Cpu32Operation { let f = &op.decode.fields; let mut c = cpu32::Cpu32Operation { timestamp: op.timestamp, @@ -1684,14 +3389,14 @@ fn collect_halt_ops(register_state: &mut RegisterState) -> Vec { /// Generates 25 read operations (input lanes at timestamp) and 25 write /// operations (output lanes at timestamp+1). Each operation is 8 bytes wide. fn collect_keccak_memw_ops( - op: &CpuOperation, + op: CpuOp<'_>, input: &[u64; 25], output: &[u64; 25], memory_state: &mut MemoryState, register_state: &mut RegisterState, ) -> Vec { let ts = op.timestamp; - let state_addr = op.keccak_state_addr; + let state_addr = op.keccak_state_addr(); let mut memw_ops = Vec::with_capacity(26); // 1 register read + 25 lane ops // Per spec (keccak:c:read_addr): read register x10 to get state_addr @@ -1746,6 +3451,284 @@ fn collect_keccak_memw_ops( memw_ops } +#[cfg(test)] +thread_local! { + /// ★ TEST-ONLY — suppresses BLAKE3's *side* contributions to the shared tables. + /// + /// A BLAKE3 syscall touches four things: the BLAKE3 table itself, the Ecall bus + /// (CPU sends, BLAKE3 receives), MEMW (it reads and writes the state region), + /// and BITWISE (its XORs and byte checks). Omitting the BLAKE3 table from a + /// proof therefore unbalances FOUR buses at once, and the resulting rejection + /// says nothing about which one did the work — the omission forgery would be + /// rejected identically if the Ecall receiver did not exist at all. + /// + /// Setting this builds the trace with the MEMW and BITWISE contributions left + /// out, so those two balance without the table and **Ecall is the only + /// unmatched interaction left**. That is what makes the forgery test in + /// `prove_elfs_tests` a test of the Ecall argument rather than of arithmetic + /// that would hold anyway. + /// + /// It exists behind `cfg(test)` because it builds a trace that does not + /// describe the execution: nothing may prove under it except a deliberate + /// forgery. + pub(crate) static STRIP_BLAKE3_SIDE_EFFECTS: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +pub(crate) fn strip_blake3_side_effects() -> bool { + STRIP_BLAKE3_SIDE_EFFECTS.with(|c| c.get()) +} + +#[cfg(not(test))] +pub(crate) fn strip_blake3_side_effects() -> bool { + false +} + +/// `LAMBDA_VM_P4_SLICED`: phase 4 cuts its per-op BITWISE sources into slices of whole ops +/// (unset or anything but `0`, #1013's c36984343), or counts each source as one collector +/// (`0`, the path before, kept for the A/B). The multiplicities are the same either way. +fn p4_sliced() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| std::env::var("LAMBDA_VM_P4_SLICED").as_deref() != Ok("0")) +} + +/// Phase 4's per-op BITWISE sources, cut into slices of whole ops ([`p4_slice_len`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum P4Source { + Lt, + Mul, + Dvrm, + Branch, + Shift, + Bytewise, + Eq, + Store, + MemwAligned, + Keccak, + Ecdas, +} + +/// Ops per slice of a phase-4 source. MUL and DVRM deduplicate their bit-gated lookups per +/// instance, so each of their slices is one instance (`mul_chunk` / `dvrm_chunk` rows, from the +/// first op); the others are sums over their ops, so any cut counts the same and the length +/// only bounds a slice's list of lookups (KECCAK sends about 5,000 per permutation). +fn p4_slice_len(source: P4Source, mul_chunk: usize, dvrm_chunk: usize) -> usize { + match source { + P4Source::Mul => mul_chunk.max(1), + P4Source::Dvrm => dvrm_chunk.max(1), + P4Source::MemwAligned => 1 << 22, + P4Source::Keccak => 1 << 11, + P4Source::Ecdas => 1 << 16, + P4Source::Lt + | P4Source::Branch + | P4Source::Shift + | P4Source::Bytewise + | P4Source::Eq + | P4Source::Store => 1 << 20, + } +} + +/// Collect the MEMW operations and the chip group for one Blake3Absorb ECALL. +/// +/// The four operands come from the register state the way ECSM's and HINT's do +/// — the CPU row carries nothing useful for an ecall that expands to a group. +/// Every access happens at the ecall's single timestamp, which is why the ABI +/// requires the control and message regions to be disjoint — an overlap would +/// touch one address twice at one timestamp, and no trace can do that: +/// +/// Per address, MEMW's genesis and finalization tokens have cardinality one, +/// so that address's accesses form a perfect matching between produced and +/// consumed tokens — a disjoint union of paths and cycles. Every edge carries +/// `old_ts < ts` strictly, so no cycle closes, leaving a single strictly +/// increasing path. Its timestamps are pairwise distinct, for ANY pair of +/// accesses rather than only adjacent ones. +/// +/// (The shorter form — "the pair would need `LT(T, T) → 1`" — is not enough on +/// its own: the second access could consume an OLDER token instead of the one +/// the first produced. The matching argument is what rules that out.) +/// +/// The message is read but never written, so `memory_state` is untouched by the +/// block reads; only `cv_out` advances it. +fn collect_blake3_absorb_ops( + op: CpuOp<'_>, + memory_state: &mut MemoryState, + register_state: &mut RegisterState, +) -> (Vec, blake3::Blake3AbsorbOperation) { + use executor::vm::instruction::execution::{BLAKE3_ABSORB_CV_OUT_DWORD, BLAKE3_BLOCK_BYTES}; + + let ts = op.timestamp; + let ctrl_addr = register_state.read(10).0; + let msg_addr = register_state.read(11).0; + let num_blocks = register_state.read(12).0; + let first_flags = register_state.read(13).0; + let num_blocks_usize = + usize::try_from(num_blocks).expect("absorb block count must be validated by the executor"); + + // 4 register reads + 4 cv_in dwords + 8 dwords per block + 4 cv_out dwords. + let mut memw_ops = Vec::with_capacity(12 + num_blocks_usize * 8); + + // Bind x10..x13 at ts through the memory argument (reads: old == value). + for (reg, value) in [ + (10u8, ctrl_addr), + (11, msg_addr), + (12, num_blocks), + (13, first_flags), + ] { + let reg_value = pack_register_value(value); + let (_old_val, old_ts) = register_state.read(reg); + memw_ops.push( + MemwOperation::new(true, 2 * reg as u64, reg_value, ts, 2, true) + .with_old(reg_value, [old_ts, old_ts, 0, 0, 0, 0, 0, 0]), + ); + register_state.write(reg, value, ts); + } + + // A pure 8-byte read: `old == value`, the content unchanged. It still + // advances the address's timestamp, because MEMW models every access as + // consuming the (addr, old_ts, old_value) token and producing (addr, ts, + // value) — a read that left the token alone would leave the next access + // claiming an `old_timestamp` no longer in flight. + let mut read_dword = |addr: u64, memw_ops: &mut Vec| -> u64 { + let (vals, old_ts) = memory_state.read_bytes(addr, 8); + let mut dw = 0u64; + for (b, &v) in vals.iter().enumerate() { + dw |= (v as u64) << (8 * b); + } + memw_ops.push(MemwOperation::new(false, addr, vals, ts, 8, true).with_old(vals, old_ts)); + memory_state.write_bytes(addr, dw, 8, ts); + dw + }; + + // cv_in: control dwords 0..4, read into the same `h` columns single mode uses. + let mut cv_in = [0u32; 8]; + for k in 0..4u64 { + let dw = read_dword(ctrl_addr + k * 8, &mut memw_ops); + cv_in[2 * k as usize] = dw as u32; + cv_in[2 * k as usize + 1] = (dw >> 32) as u32; + } + + // The message blocks, read in place. + let mut blocks = Vec::with_capacity(num_blocks_usize); + for i in 0..num_blocks { + let block_addr = msg_addr + i * BLAKE3_BLOCK_BYTES; + let mut m = [0u32; 16]; + for k in 0..8u64 { + let dw = read_dword(block_addr + k * 8, &mut memw_ops); + m[2 * k as usize] = dw as u32; + m[2 * k as usize + 1] = (dw >> 32) as u32; + } + blocks.push(m); + } + + let mut absorb_op = blake3::Blake3AbsorbOperation { + timestamp: ts, + ctrl_addr, + msg_addr, + first_flags: first_flags as u32, + cv_in, + blocks, + old_cv_out: [0; 32], + }; + + // cv_out: control dwords 4..8, written on the group's END row with the + // chaining value the chain bus delivers there. `expand_absorb` is the same + // function the trace filler runs, so this cannot disagree with the row it + // is the write for. + let rows = blake3::expand_absorb(&absorb_op); + let final_cv = rows.last().expect("an absorb group has at least 2 rows").h; + for k in 0..4u64 { + let addr = ctrl_addr + (BLAKE3_ABSORB_CV_OUT_DWORD + k) * 8; + let dw = (final_cv[2 * k as usize] as u64) | ((final_cv[2 * k as usize + 1] as u64) << 32); + let mut value = [0u32; 8]; + for (j, slot) in value.iter_mut().enumerate() { + *slot = ((dw >> (8 * j)) & 0xFF) as u32; + } + let (old_vals, old_ts) = memory_state.read_bytes(addr, 8); + for (j, &v) in old_vals.iter().enumerate() { + absorb_op.old_cv_out[k as usize * 8 + j] = v as u8; + } + memw_ops + .push(MemwOperation::new(false, addr, value, ts, 8, true).with_old(old_vals, old_ts)); + memory_state.write_bytes(addr, dw, 8, ts); + } + + (memw_ops, absorb_op) +} + +/// Collect MEMW operations for a Blake3Compress ECALL. +/// +/// One register read of x10 plus 22 dword ops at the call's timestamp: the 14 +/// input dwords are pure reads (old = value = the input bytes, re-written at +/// `ts` like a LOAD), the 8 output dwords write the compression output over +/// the previous content. +fn collect_blake3_memw_ops( + op: CpuOp<'_>, + words: &[u32; 28], + out: &[u32; 16], + memory_state: &mut MemoryState, + register_state: &mut RegisterState, +) -> Vec { + let ts = op.timestamp; + let state_addr = op.blake3_state_addr(); + let mut memw_ops = Vec::with_capacity(23); // 1 register read + 22 dword ops + + // Read register x10 to bind state_addr (same as keccak:c:read_addr). + { + let reg_value = pack_register_value(state_addr); + let reg_addr = 2 * 10u64; // x10 -> address 20 + let (_old_val, old_ts) = register_state.read(10); + let old_timestamps = [old_ts, old_ts, 0, 0, 0, 0, 0, 0]; + let memw_op = MemwOperation::new(true, reg_addr, reg_value, ts, 2, true) + .with_old(reg_value, old_timestamps); + memw_ops.push(memw_op); + register_state.write(10, state_addr, ts); + } + + for k in 0..22usize { + let dword_addr = state_addr + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + + // The dword's new value: input dwords re-write their own bytes, output + // dwords write the compression output. + let dw = if k < 14 { + (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32) + } else { + let o = k - 14; + (out[2 * o] as u64) | ((out[2 * o + 1] as u64) << 32) + }; + let mut value_bytes = [0u32; 8]; + for (b, byte) in value_bytes.iter_mut().enumerate() { + *byte = ((dw >> (b * 8)) & 0xFF) as u32; + } + + let mut old_bytes = [0u32; 8]; + let mut old_timestamps = [0u64; 8]; + for b in 0..8 { + let byte_addr = dword_addr + .checked_add(b as u64) + .expect("blake3 state address range must be validated by the executor"); + let (old_val, old_ts) = memory_state.read_byte(byte_addr); + old_bytes[b] = old_val as u32; + old_timestamps[b] = old_ts; + } + + let memw_op = MemwOperation::new(false, dword_addr, value_bytes, ts, 8, true) + .with_old(old_bytes, old_timestamps); + memw_ops.push(memw_op); + + for (b, &val) in value_bytes.iter().enumerate() { + let byte_addr = dword_addr + .checked_add(b as u64) + .expect("blake3 state address range must be validated by the executor"); + memory_state.write_byte(byte_addr, val as u8, ts); + } + } + + memw_ops +} + /// /// From spec memw.md: /// - MEMW-C4 through MEMW-C7: old_timestamp[i] < timestamp (based on width) @@ -1803,13 +3786,13 @@ fn collect_lt_from_memw(memw_ops: &[MemwOperation]) -> Vec { /// Collects LT operations from MEMW_A for timestamp ordering. /// /// Each aligned operation has a single old_timestamp < timestamp check. -fn collect_lt_from_memw_aligned(memw_aligned_ops: &[MemwOperation]) -> Vec { +fn collect_lt_from_memw_aligned(memw_aligned_ops: &[memw_aligned::AlignedRow]) -> Vec { // Address overflow LT checks (R1-R3 in MEMW) are intentionally absent. // Alignment guarantees addr + (width-1) never wraps: the largest width-N // aligned address is 2^64-N, and 2^64-N+(N-1) = 2^64-1, so no u64 overflow. memw_aligned_ops .iter() - .map(|op| LtOperation::new(op.old_timestamp[0], op.timestamp, false)) + .map(|op| LtOperation::new(op.old_timestamp(), op.timestamp(), false)) .collect() } @@ -1844,12 +3827,12 @@ fn is_aligned_op(op: &MemwOperation) -> bool { /// /// IS_HALF[base_address[i]] for i ∈ [0, 1] and IS_WORD[base_address[2]] are /// assumptions — the caller's (CPU's) responsibility. -fn collect_bitwise_from_memw_aligned(ops: &[MemwOperation]) -> Vec { +fn collect_bitwise_from_memw_aligned(ops: &[memw_aligned::AlignedRow]) -> Vec { let mut bitwise_ops = Vec::with_capacity(ops.len()); for op in ops { - let low_half = (op.base_address & 0xFFFF) as u32; - let mask: u32 = match op.width { + let low_half = (op.base_address() & 0xFFFF) as u32; + let mask: u32 = match op.width() { 2 => 1, 4 => 3, 8 => 7, @@ -1929,23 +3912,49 @@ fn reg_ts_delta_in_range(timestamp: u64, old_ts: u64) -> bool { /// `chunk_and_generate_optional`. /// /// Returns: Vec of bitwise lookups +#[cfg(test)] pub(crate) fn collect_bitwise_from_lt( lt_ops: &[LtOperation], max_rows_lt: usize, ) -> Vec { + let mut bitwise_ops = collect_bitwise_from_lt_ops(lt_ops); + for chunk in op_chunks(lt_ops, max_rows_lt) { + bitwise_ops.extend(lt_row_lookups(chunk.iter())); + } + bitwise_ops +} + +/// [`collect_bitwise_from_lt`]'s per-op lookups (MSB16 and IS_HALFWORD): a sum +/// over the ops, so any slicing of the list counts them all. +pub(crate) fn collect_bitwise_from_lt_ops(lt_ops: &[LtOperation]) -> Vec { let mut bitwise_ops = Vec::with_capacity(lt_ops.len() * 8); + for_each_lt_lookup(lt_ops, |op| bitwise_ops.push(op)); + bitwise_ops +} + +/// [`collect_bitwise_from_lt_ops`]'s lookups counted into `histogram`, with no +/// list built: a block's LT ops are hundreds of millions, eight lookups each. +pub(crate) fn count_bitwise_from_lt_ops( + lt_ops: &[LtOperation], + histogram: &mut bitwise::BitwiseHistogram, +) { + for_each_lt_lookup(lt_ops, |op| histogram.bump(op)); +} +/// Every per-op BITWISE lookup of the LT ops `lt_ops` (MSB16 and IS_HALFWORD), +/// op by op. +fn for_each_lt_lookup(lt_ops: &[LtOperation], mut emit: impl FnMut(BitwiseOperation)) { for op in lt_ops { // MSB16 lookups for lhs[2] and rhs[2] let lhs_2 = ((op.lhs >> 48) & 0xFFFF) as u16; let rhs_2 = ((op.rhs >> 48) & 0xFFFF) as u16; - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::Msb16, (lhs_2 & 0xFF) as u8, (lhs_2 >> 8) as u8, )); - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::Msb16, (rhs_2 & 0xFF) as u8, (rhs_2 >> 8) as u8, @@ -1955,7 +3964,7 @@ pub(crate) fn collect_bitwise_from_lt( let lhs_sub_rhs = op.lhs.wrapping_sub(op.rhs); for shift in [0, 16, 32, 48] { let half = ((lhs_sub_rhs >> shift) & 0xFFFF) as u16; - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::IsHalf, (half & 0xFF) as u8, (half >> 8) as u8, @@ -1965,31 +3974,33 @@ pub(crate) fn collect_bitwise_from_lt( // IS_HALFWORD lookups for lhs[1] and rhs[1] let lhs_1 = ((op.lhs >> 32) & 0xFFFF) as u16; let rhs_1 = ((op.rhs >> 32) & 0xFFFF) as u16; - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::IsHalf, (lhs_1 & 0xFF) as u8, (lhs_1 >> 8) as u8, )); - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::IsHalf, (rhs_1 & 0xFF) as u8, (rhs_1 >> 8) as u8, )); } +} - // IS_BYTE[μ] | 1: every row sends its own μ once, padding rows (μ = 0) - // included, over exactly the rows `generate_lt_trace` builds for each chunk. +/// LT's per-row lookup over one table's ops (one chunk): IS_BYTE[μ] | 1, every +/// row its own μ once, padding rows (μ = 0) included, over exactly the rows +/// `generate_lt_trace` builds for the chunk. +pub(crate) fn lt_row_lookups<'a>( + chunk: impl Iterator, +) -> Vec { let mu_byte = |v: u64| BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, v as u8); - for chunk in op_chunks(lt_ops, max_rows_lt) { - let rows = dedup_rows(chunk); - bitwise_ops.extend(rows.iter().map(|(_, mu)| mu_byte(*mu))); - bitwise_ops.extend(std::iter::repeat_n( - mu_byte(0), - chunk_padding_rows(rows.len()), - )); - } - - bitwise_ops + let rows = dedup_rows_of(chunk); + let mut lookups: Vec = rows.iter().map(|(_, mu)| mu_byte(*mu)).collect(); + lookups.extend(std::iter::repeat_n( + mu_byte(0), + chunk_padding_rows(rows.len()), + )); + lookups } /// Collects bitwise lookups from MUL operations (MSB16 for sign bits). @@ -2291,13 +4302,33 @@ pub(crate) fn collect_bitwise_from_dvrm( /// - IS_HALFWORD[next_pc_high[0..3]] - range checks for bits 16-63 /// /// Returns: Vec of bitwise lookups +#[cfg(test)] pub(crate) fn collect_bitwise_from_branch( branch_ops: &[BranchOperation], max_rows_branch: usize, ) -> Vec { - let mut bitwise_ops = Vec::with_capacity(branch_ops.len() * 5); + let mut bitwise_ops = collect_bitwise_from_branch_ops(branch_ops); + for chunk in op_chunks(branch_ops, max_rows_branch) { + bitwise_ops.extend(branch_row_lookups(chunk)); + } + bitwise_ops +} +/// [`collect_bitwise_from_branch`]'s per-op lookups (the BYTE_ALU mask and the +/// three IS_HALFWORD checks): a sum over the ops, so any slicing counts them all. +pub(crate) fn collect_bitwise_from_branch_ops( + branch_ops: &[BranchOperation], +) -> Vec { + let mut bitwise_ops = Vec::with_capacity(branch_ops.len() * 4); for op in branch_ops { + for_each_branch_lookup(op, |lookup| bitwise_ops.push(lookup)); + } + bitwise_ops +} + +/// [`collect_bitwise_from_branch_ops`]' lookups of one op, one by one. +fn for_each_branch_lookup(op: &BranchOperation, mut emit: impl FnMut(BitwiseOperation)) { + { let next_pc = op.compute_next_pc(); let next_pc_unmasked = op.compute_next_pc_unmasked(); @@ -2311,53 +4342,53 @@ pub(crate) fn collect_bitwise_from_branch( // BYTE_ALU[AND, unmasked_low_byte, 254] → next_pc_low[0] // Verifies: next_pc_low[0] = unmasked_low_byte & 0xFE - bitwise_ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluAnd, unmasked_low_byte, 254, // 0xFE mask )); // IS_HALFWORD[next_pc_high[0]] - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::IsHalf, (next_pc_high_0 & 0xFF) as u8, (next_pc_high_0 >> 8) as u8, )); // IS_HALFWORD[next_pc_high[1]] - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::IsHalf, (next_pc_high_1 & 0xFF) as u8, (next_pc_high_1 >> 8) as u8, )); // IS_HALFWORD[next_pc_high[2]] - bitwise_ops.push(BitwiseOperation::halfword( + emit(BitwiseOperation::halfword( BitwiseOperationType::IsHalf, (next_pc_high_2 & 0xFF) as u8, (next_pc_high_2 >> 8) as u8, )); } +} - // ARE_BYTES[μ, next_pc_low[1]] | 1 (the batched IS_BYTE bounds): every row - // sends it once, padding rows included as ARE_BYTES[0, 0], over exactly the - // rows `generate_branch_trace` builds for each chunk. +/// BRANCH's per-row lookup over one table's ops (one chunk): ARE_BYTES[μ, +/// next_pc_low[1]] | 1 (the batched IS_BYTE bounds), every row once, padding rows +/// included as ARE_BYTES[0, 0], over exactly the rows `generate_branch_trace` +/// builds for the chunk. +pub(crate) fn branch_row_lookups(chunk: &[BranchOperation]) -> Vec { let row_bytes = |mu: u64, low_1: u8| { BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, mu as u8, low_1) }; - for chunk in op_chunks(branch_ops, max_rows_branch) { - let rows = dedup_rows(chunk); - bitwise_ops.extend( - rows.iter() - .map(|(op, mu)| row_bytes(*mu, ((op.compute_next_pc() >> 8) & 0xFF) as u8)), - ); - bitwise_ops.extend(std::iter::repeat_n( - row_bytes(0, 0), - chunk_padding_rows(rows.len()), - )); - } - - bitwise_ops + let rows = dedup_rows(chunk); + let mut lookups: Vec = rows + .iter() + .map(|(op, mu)| row_bytes(*mu, ((op.compute_next_pc() >> 8) & 0xFF) as u8)) + .collect(); + lookups.extend(std::iter::repeat_n( + row_bytes(0, 0), + chunk_padding_rows(rows.len()), + )); + lookups } /// Padding rows of a chip instance with `rows` real rows: `generate_*_trace` pads to @@ -2379,6 +4410,7 @@ fn op_chunks(ops: &[T], max_rows: usize) -> std::slice::Chunks<'_, T> { /// SHIFT's `ARE_BYTES[shift[0], shift[1]] | 1` on the padding rows of every /// `max_rows` instance: each one sends `ARE_BYTES[0, 0]` (padding cells are 0). /// The real rows are counted by `shift::collect_bitwise_from_shift`. +#[cfg(test)] pub(crate) fn shift_padding_are_bytes(shift_ops: &[ShiftOperation], max_rows: usize) -> u64 { op_chunks(shift_ops, max_rows) .map(|chunk| chunk_padding_rows(chunk.len()) as u64) @@ -2389,67 +4421,694 @@ pub(crate) fn shift_padding_are_bytes(shift_ops: &[ShiftOperation], max_rows: us /// `max_rows` instance sends its own μ once, padding rows (μ = 0) included, over /// exactly the rows `generate_bytewise_trace` builds. The per-op BYTE_ALU lookups /// are counted by `bytewise::BytewiseOperation::collect_bitwise_ops`. +#[cfg(test)] pub(crate) fn collect_bytewise_mu_bounds( bytewise_ops: &[bytewise::BytewiseOperation], max_rows: usize, ) -> Vec { - let mu_byte = |v: u64| BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, v as u8); let mut bounds = Vec::new(); for chunk in op_chunks(bytewise_ops, max_rows) { - let rows = dedup_rows(chunk); - bounds.extend(rows.iter().map(|(_, mu)| mu_byte(*mu))); - bounds.extend(std::iter::repeat_n( - mu_byte(0), - chunk_padding_rows(rows.len()), - )); + bounds.extend(bytewise_row_lookups(chunk)); } bounds } -/// Generates ARE_BYTES ops for CPU padding rows. -/// -/// CPU padding rows have all byte columns = 0 (RS1=0, RS2=0, RD=0, etc.). -/// Since the CPU bus interactions use Multiplicity::One for range checks, -/// padding rows also send, so we need matching bitwise ops. -/// -/// Add the BITWISE lookups every CPU padding row sends. A padding row has all -/// values zero, so per row it sends 3× ARE_BYTES[0,0] (rs1/rs2, -/// rd/instruction_length, alu_flags/mem_flags) and 4× IS_HALF[0] (the four `res` -/// halves). Every padding row sends the same lookups, so their whole -/// contribution is a per-cell count: two `bump_n`s, no per-row work. -fn add_padding_byte_checks(hist: &mut bitwise::BitwiseHistogram, num_padding_rows: usize) { - let n = num_padding_rows as u64; - hist.bump_n( - BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, 0, 0), - 3 * n, +/// [`collect_bytewise_mu_bounds`] over one table's ops (one chunk). +pub(crate) fn bytewise_row_lookups(chunk: &[bytewise::BytewiseOperation]) -> Vec { + let mu_byte = |v: u64| BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, v as u8); + let rows = dedup_rows(chunk); + let mut bounds: Vec = rows.iter().map(|(_, mu)| mu_byte(*mu)).collect(); + bounds.extend(std::iter::repeat_n( + mu_byte(0), + chunk_padding_rows(rows.len()), + )); + bounds +} + +/// LT's BITWISE lookups from one table's rows `(op, μ)`, as +/// [`lt::lt_trace_of_rows`] lays them out and the AIR sends them: each row's +/// per-op lookups ([`for_each_lt_lookup`]) μ times, its IS_BYTE[μ] bound once, +/// and the padding rows' bound. Over a chunk's deduplicated rows, the same +/// multiplicities as [`collect_bitwise_from_lt_ops`] and [`lt_row_lookups`] of +/// its ops: the rows are the ops' distinct values, each μ times. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn count_lt_table_lookups( + rows: &[(LtOperation, u64)], + plane: &mut bitwise::BitwisePlane, +) { + for (op, mu) in rows { + for_each_lt_lookup(std::slice::from_ref(op), |lookup| plane.bump_n(lookup, *mu)); + plane.bump_n( + BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, *mu as u8), + 1, + ); + } + plane.bump_n( + BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, 0), + chunk_padding_rows(rows.len()) as u64, ); - hist.bump_n( - BitwiseOperation::halfword(BitwiseOperationType::IsHalf, 0, 0), - 4 * n, +} + +/// BRANCH's BITWISE lookups from one table's rows, as +/// [`count_lt_table_lookups`] counts LT's: [`collect_bitwise_from_branch_ops`] +/// and [`branch_row_lookups`] of its ops. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn count_branch_table_lookups( + rows: &[(BranchOperation, u64)], + plane: &mut bitwise::BitwisePlane, +) { + for (op, mu) in rows { + for_each_branch_lookup(op, |lookup| plane.bump_n(lookup, *mu)); + plane.bump_n( + BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + *mu as u8, + ((op.compute_next_pc() >> 8) & 0xFF) as u8, + ), + 1, + ); + } + plane.bump_n( + BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, 0, 0), + chunk_padding_rows(rows.len()) as u64, ); } -/// Collects ARE_BYTES lookups from PAGE data (init and fini values). -/// -/// Each PAGE row generates 1 batched ARE_BYTES lookup: -/// - C1+C2: ARE_BYTES[init, fini] — range-checks both bytes in one interaction -/// -/// This must be called BEFORE bitwise multiplicities are updated. -/// -/// Encode private input as `[len_u32_LE][data]` — the canonical wire format. -/// Must match `executor::vm::memory::Memory::store_private_inputs`. -fn private_input_bytes(private_input: &[u8]) -> Vec { - let len_bytes = (private_input.len() as u32).to_le_bytes(); - len_bytes - .iter() - .chain(private_input.iter()) - .copied() - .collect() +/// BYTEWISE's BITWISE lookups from one table's rows, as +/// [`count_lt_table_lookups`] counts LT's: each op's BYTE_ALU lookups and +/// [`bytewise_row_lookups`] of its ops. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn count_bytewise_table_lookups( + rows: &[(bytewise::BytewiseOperation, u64)], + plane: &mut bitwise::BitwisePlane, +) { + for (op, mu) in rows { + op.for_each_bitwise_op(|lookup| plane.bump_n(lookup, *mu)); + plane.bump_n( + BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, *mu as u8), + 1, + ); + } + plane.bump_n( + BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, 0), + chunk_padding_rows(rows.len()) as u64, + ); } -/// Build the initial-memory image (byte address -> value) from the ELF segments -/// and the private-input region. Single source of "what memory starts as", read -/// by both `MemoryState` seeding and PAGE/bitwise init. +/// A count-only hash for [`count_row_lookups`]: the rows' order is not read +/// there, so the trace's process-wide state ([`super::trace_hash`]) is not +/// needed. A multiply-rotate mix, finished with a murmur3 finalizer. +#[derive(Default, Clone, Copy)] +struct RowHasher(u64); + +impl std::hash::Hasher for RowHasher { + fn write(&mut self, bytes: &[u8]) { + for &b in bytes { + self.write_u8(b); + } + } + + fn write_u8(&mut self, n: u8) { + self.write_u64(u64::from(n)); + } + + fn write_u64(&mut self, n: u64) { + self.0 = (self.0.rotate_left(5) ^ n).wrapping_mul(0x517c_c1b7_2722_0a95); + } + + fn write_usize(&mut self, n: usize) { + self.write_u64(n as u64); + } + + fn finish(&self) -> u64 { + let mut x = self.0; + x ^= x >> 33; + x = x.wrapping_mul(0xff51_afd7_ed55_8ccd); + x ^ (x >> 33) + } +} + +/// The per-row lookups of the deduplicating tables' chunks, counted: how +/// many of each, with no list of them ([`count_row_lookups`]). LT's and +/// BYTEWISE's are `ARE_BYTES[μ]` ([`lt_row_lookups`], +/// [`bytewise_row_lookups`]), BRANCH's `ARE_BYTES[μ, next pc's byte 1]` +/// ([`branch_row_lookups`]), the padding rows' at μ = 0. +struct RowCounts { + /// By μ. + single: [u64; 256], + /// By `μ · 256 + byte`, allocated by the first BRANCH chunk. + branch: Vec, + /// What LT's, BRANCH's and BYTEWISE's chunks cost the count. + scans: [RowScan; 3], + /// The count's wall seconds, for the line. + wall: f64, +} + +impl RowCounts { + fn new() -> Self { + Self { + single: [0; 256], + branch: Vec::new(), + scans: [RowScan::default(); 3], + wall: 0.0, + } + } + + /// The `BLOCK P4 ROWS` line: what each kind's chunks cost the count, how + /// its chunks were counted (as the scans saw them) and its wall seconds. + fn line(&self) -> String { + let (chunks, parted) = self + .scans + .iter() + .fold((0, 0), |(c, p), s| (c + s.chunks, p + s.parted)); + format!( + "BLOCK P4 ROWS: {} · {} · {} · counted {} in {:.2} s", + self.scans[0].line("LT"), + self.scans[1].line("BRANCH"), + self.scans[2].line("BYTEWISE"), + if parted == 0 { + "by one map a chunk".to_string() + } else if parted == chunks { + format!("by {} parts a chunk", 1usize << ROW_PARTS_LOG2) + } else { + format!("{parted} of {chunks} chunks by parts") + }, + self.wall, + ) + } + + fn branch_mut(&mut self) -> &mut [u64] { + if self.branch.is_empty() { + self.branch = vec![0; 1 << 16]; + } + &mut self.branch + } + + fn merge(mut self, other: Self) -> Self { + for (a, b) in self.single.iter_mut().zip(other.single) { + *a += b; + } + for (a, b) in self.scans.iter_mut().zip(&other.scans) { + a.add(b); + } + if !other.branch.is_empty() { + for (a, b) in self.branch_mut().iter_mut().zip(other.branch) { + *a += b; + } + } + self + } + + /// Into `histogram`: the multiplicities the lookup lists add. + fn bump_into(&self, histogram: &mut bitwise::BitwiseHistogram) { + for (mu, &n) in self.single.iter().enumerate() { + if n > 0 { + histogram.bump_n( + BitwiseOperation::single_byte(BitwiseOperationType::AreBytes, mu as u8), + n, + ); + } + } + for (k, &n) in self.branch.iter().enumerate() { + if n > 0 { + histogram.bump_n( + BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (k >> 8) as u8, + (k & 0xFF) as u8, + ), + n, + ); + } + } + } +} + +/// A chunk's ops as slices, handed one at a time to the closure it is given +/// ([`Segmented::for_each_slice`]); it may be called more than once. +type Visit<'v, T> = dyn Fn(&mut Slices<'_, T>) + 'v; + +/// What takes a chunk's slices ([`Visit`]). +type Slices<'s, T> = dyn FnMut(&[T]) + 's; + +/// How many distinct ops `visit` hands over (`len` of them), estimated by +/// linear counting: each op's [`RowHasher`] hash sets a bit of a bitmap of +/// `len` bits (a power of two, at least 64), and the share of bits left clear +/// gives the count. With at most as many distinct ops as bits, its error is a +/// fraction of a percent; it never exceeds `len`. +fn distinct_estimate(visit: &Visit<'_, T>, len: usize) -> usize { + use std::hash::BuildHasher; + let bits = len.next_power_of_two().max(64); + let mut map = vec![0u64; bits / 64]; + let hasher = std::hash::BuildHasherDefault::::default(); + visit(&mut |ops: &[T]| { + for op in ops { + let i = (hasher.hash_one(op) as usize) & (bits - 1); + map[i / 64] |= 1u64 << (i % 64); + } + }); + let set: usize = map.iter().map(|w| w.count_ones() as usize).sum(); + let clear = bits - set; + if clear == 0 { + return len; + } + let estimate = -(bits as f64) * (clear as f64 / bits as f64).ln(); + (estimate.ceil() as usize).min(len) +} + +/// The bytes a map of `(T, u64)` reserved for `capacity` entries holds: its +/// buckets by the map's rule (4 or 8 below 8 entries, else the power of two +/// at or above eight sevenths of them), an entry and a control byte each. +fn map_bytes(capacity: usize) -> usize { + let buckets = match capacity { + 0 => 0, + 1..=3 => 4, + 4..=7 => 8, + _ => (capacity * 8 / 7).next_power_of_two(), + }; + buckets * (std::mem::size_of::<(T, u64)>() + 1) +} + +/// What one kind of chunk ([`RowChunk`]) cost the count: chunks, ops, distinct +/// ops, the estimate's largest miss, the bytes its maps reserved against what +/// reserving a whole chunk would have, and the bytes copying the ranges would +/// have taken ([`Segmented::range`]). +#[derive(Clone, Copy, Default)] +struct RowScan { + chunks: usize, + ops: usize, + distinct: usize, + /// The least and the most distinct ops per op of a chunk. + least: f64, + most: f64, + /// The largest |estimate − distinct| / distinct of a chunk. + miss: f64, + reserved: usize, + by_length: usize, + copies: usize, + /// The chunks counted by parts ([`count_rows_by_parts`]). + parted: usize, +} + +impl RowScan { + fn add(&mut self, other: &Self) { + let first = self.chunks == 0; + self.chunks += other.chunks; + self.ops += other.ops; + self.distinct += other.distinct; + if other.chunks > 0 { + self.least = if first { + other.least + } else { + self.least.min(other.least) + }; + self.most = self.most.max(other.most); + } + self.miss = self.miss.max(other.miss); + self.reserved += other.reserved; + self.by_length += other.by_length; + self.copies += other.copies; + self.parted += other.parted; + } + + fn line(&self, name: &str) -> String { + let gib = |b: usize| b as f64 / (1u64 << 30) as f64; + let per = |b: usize| b as f64 / (1u64 << 20) as f64 / self.chunks.max(1) as f64; + format!( + "{name} {} chunks, {:.2} M ops, {:.3} distinct an op (chunks {:.3}–{:.3}), the estimate off by at most \ + {:.2} %; maps reserved {:.2} GiB, {:.1} MiB a chunk, against {:.2} GiB, {:.1} MiB, by length; copies \ + not made {:.2} GiB", + self.chunks, + self.ops as f64 / 1e6, + self.distinct as f64 / self.ops.max(1) as f64, + self.least, + self.most, + self.miss * 100.0, + gib(self.reserved), + per(self.reserved), + gib(self.by_length), + per(self.by_length), + gib(self.copies), + ) + } +} + +/// The ops a count reads a slice at a time ([`Segmented::for_each_slice`]). +const ROW_SCAN_BLOCK: usize = 1 << 16; + +/// [`dedup_rows_of`]'s rows of the ops `visit` hands over (`len` of them, and +/// it may be called twice), counted only: `each(op, μ, rows)` for every +/// distinct op and μ, then the number of rows. The map is reserved for the +/// distinct ops [`distinct_estimate`] finds, not for the whole chunk; `scan` +/// notes the chunk. +fn count_rows_of( + visit: &Visit<'_, T>, + len: usize, + mut each: impl FnMut(&T, u64, u64), + scan: &mut RowScan, +) -> usize { + let estimate = distinct_estimate(visit, len); + let capacity = (estimate + estimate / 32 + 16).min(len); + let mut counts: HashMap> = + HashMap::with_capacity_and_hasher(capacity, Default::default()); + visit(&mut |ops: &[T]| { + for op in ops { + *counts.entry(op.clone()).or_insert(0) += 1; + } + }); + let distinct = counts.len(); + let ratio = distinct as f64 / len.max(1) as f64; + scan.add(&RowScan { + chunks: 1, + ops: len, + distinct, + least: ratio, + most: ratio, + miss: (estimate as f64 - distinct as f64).abs() / distinct.max(1) as f64, + reserved: map_bytes::(capacity), + by_length: map_bytes::(len), + copies: 0, + parted: 0, + }); + let mut rows = 0u64; + for (op, &count) in &counts { + let (full, left) = (count / super::types::MU_MAX, count % super::types::MU_MAX); + if full > 0 { + each(op, super::types::MU_MAX, full); + } + if left > 0 { + each(op, left, 1); + } + rows += full + u64::from(left > 0); + } + rows as usize +} + +/// How [`count_row_lookups`] counts a chunk's distinct ops. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum RowCount { + /// One map a chunk, reserved for its distinct ops ([`count_rows_of`]). + #[default] + Map, + /// The chunk's ops split into parts by their hash, each counted with one + /// small map that stays in a core's cache ([`count_rows_by_parts`]). + Parts, +} + +/// The parts [`count_rows_by_parts`] splits a chunk into: `2^ROW_PARTS_LOG2` +/// of them, so the distinct ops of a part fit a core's own cache. A block's +/// LT chunk holds about a million distinct ops, 32 bytes each with its count. +const ROW_PARTS_LOG2: u32 = 6; + +/// Where an op's part is read from its [`RowHasher`] hash: above the low bits +/// a map indexes its buckets by and below the top seven its control bytes +/// hold, so the ops of one part still spread over their map's buckets and +/// still differ in their control bytes. +const ROW_PART_SHIFT: u32 = 40; + +/// The part of [`count_rows_by_parts`] `op` falls in. Equal ops hash alike, +/// so every op of one value falls in one part. +fn row_part(op: &T) -> usize { + use std::hash::BuildHasher; + let hash = std::hash::BuildHasherDefault::::default().hash_one(op); + ((hash >> ROW_PART_SHIFT) as usize) & ((1 << ROW_PARTS_LOG2) - 1) +} + +/// [`count_rows_of`], each chunk counted by parts. A first pass notes each +/// op's part ([`row_part`]) and each part's size; a second copies each op into +/// its part; then each part is counted with one map, cleared between parts. +/// A chunk's map holds a million distinct ops far beyond any cache, so each +/// insert waits on memory; a part's holds a sixty-fourth of them. Every op of +/// one value is in one part, so each value's count, and so `each`'s calls, +/// are the one map's (in another order, which nothing reads). `scan` notes +/// the chunk, its reserved bytes the parts' and the map's. +fn count_rows_by_parts( + visit: &Visit<'_, T>, + len: usize, + mut each: impl FnMut(&T, u64, u64), + scan: &mut RowScan, +) -> usize { + let parts = 1usize << ROW_PARTS_LOG2; + let mut tags: Vec = Vec::with_capacity(len); + let mut sizes = vec![0usize; parts]; + visit(&mut |ops: &[T]| { + for op in ops { + let part = row_part(op); + tags.push(part as u8); + sizes[part] += 1; + } + }); + let mut split: Vec> = sizes.iter().map(|&n| Vec::with_capacity(n)).collect(); + let mut at = 0usize; + visit(&mut |ops: &[T]| { + for op in ops { + split[tags[at] as usize].push(op.clone()); + at += 1; + } + }); + let tag_bytes = tags.capacity(); + drop(tags); + let mut counts: HashMap> = + HashMap::with_capacity_and_hasher( + sizes.iter().max().copied().unwrap_or(0) / 2, + Default::default(), + ); + let (mut distinct, mut rows, mut most) = (0usize, 0u64, 0usize); + for part in split { + counts.clear(); + for op in part { + *counts.entry(op).or_insert(0) += 1; + } + distinct += counts.len(); + most = most.max(counts.capacity()); + for (op, &count) in &counts { + let (full, left) = (count / super::types::MU_MAX, count % super::types::MU_MAX); + if full > 0 { + each(op, super::types::MU_MAX, full); + } + if left > 0 { + each(op, left, 1); + } + rows += full + u64::from(left > 0); + } + } + let ratio = distinct as f64 / len.max(1) as f64; + scan.add(&RowScan { + chunks: 1, + ops: len, + distinct, + least: ratio, + most: ratio, + miss: 0.0, + reserved: len * std::mem::size_of::() + tag_bytes + map_bytes::(most), + by_length: map_bytes::(len), + copies: 0, + parted: 1, + }); + rows as usize +} + +/// [`count_rows_of`] or [`count_rows_by_parts`], as `by` says. +fn count_rows( + by: RowCount, + visit: &Visit<'_, T>, + len: usize, + each: impl FnMut(&T, u64, u64), + scan: &mut RowScan, +) -> usize { + match by { + RowCount::Map => count_rows_of(visit, len, each, scan), + RowCount::Parts => count_rows_by_parts(visit, len, each, scan), + } +} + +/// One table's chunk of [`count_row_lookups`]. +#[derive(Clone, Copy)] +enum RowChunk { + Lt(usize, usize), + Branch(usize, usize), + Bytewise(usize, usize), +} + +/// The per-row lookups of every LT, BRANCH and BYTEWISE chunk of these lists +/// ([`list_chunks`] at `max_rows`), counted on the whole pool, a chunk a task, +/// with no list of lookups and no histogram per task: the same multiplicities +/// as folding [`lt_row_lookups`], [`branch_row_lookups`] and +/// [`bytewise_row_lookups`] of each chunk into the histogram. +fn count_row_lookups( + lt: &Segmented<'_, LtOperation>, + branch: &Segmented<'_, BranchOperation>, + bytewise: &Segmented<'_, bytewise::BytewiseOperation>, + max_rows: &crate::tables::MaxRowsConfig, + by: RowCount, +) -> RowCounts { + let started = std::time::Instant::now(); + let tasks: Vec = list_chunks(lt.len(), max_rows.lt) + .into_iter() + .map(|(s, e)| RowChunk::Lt(s, e)) + .chain( + list_chunks(branch.len(), max_rows.branch) + .into_iter() + .map(|(s, e)| RowChunk::Branch(s, e)), + ) + .chain( + list_chunks(bytewise.len(), max_rows.bytewise) + .into_iter() + .map(|(s, e)| RowChunk::Bytewise(s, e)), + ) + .collect(); + // Each chunk's ops a slice at a time, never copied whole; whether + // `range` would have copied them is noted as bytes not copied. + fn visit_of<'s, T: Clone>( + list: &'s Segmented<'_, T>, + start: usize, + end: usize, + copied: &'s std::cell::Cell, + ) -> impl Fn(&mut Slices<'_, T>) + 's { + move |f: &mut Slices<'_, T>| { + copied.set(list.for_each_slice(start, end, ROW_SCAN_BLOCK, f)); + } + } + let count = |task: &RowChunk| -> RowCounts { + let mut counts = RowCounts::new(); + let copied = std::cell::Cell::new(false); + let (kind, ops, size) = match *task { + RowChunk::Lt(start, end) => { + let mut scan = RowScan::default(); + let rows = count_rows( + by, + &visit_of(lt, start, end, &copied), + end - start, + |_, mu, n| counts.single[mu as usize] += n, + &mut scan, + ); + counts.single[0] += chunk_padding_rows(rows) as u64; + counts.scans[0] = scan; + (0, end - start, std::mem::size_of::()) + } + RowChunk::Bytewise(start, end) => { + let mut scan = RowScan::default(); + let rows = count_rows( + by, + &visit_of(bytewise, start, end, &copied), + end - start, + |_, mu, n| counts.single[mu as usize] += n, + &mut scan, + ); + counts.single[0] += chunk_padding_rows(rows) as u64; + counts.scans[2] = scan; + ( + 2, + end - start, + std::mem::size_of::(), + ) + } + RowChunk::Branch(start, end) => { + let mut scan = RowScan::default(); + let table = counts.branch_mut(); + let rows = count_rows( + by, + &visit_of(branch, start, end, &copied), + end - start, + |op, mu, n| { + let byte = ((op.compute_next_pc() >> 8) & 0xFF) as usize; + table[((mu as usize) << 8) | byte] += n + }, + &mut scan, + ); + table[0] += chunk_padding_rows(rows) as u64; + counts.scans[1] = scan; + (1, end - start, std::mem::size_of::()) + } + }; + if copied.get() { + counts.scans[kind].copies += ops * size; + } + counts + }; + #[cfg(feature = "parallel")] + let counts = { + use rayon::prelude::*; + tasks + .par_iter() + .map(count) + .reduce(RowCounts::new, RowCounts::merge) + }; + #[cfg(not(feature = "parallel"))] + let counts = tasks + .iter() + .map(count) + .fold(RowCounts::new(), RowCounts::merge); + let mut counts = counts; + counts.wall = started.elapsed().as_secs_f64(); + if let Ok(mut line) = ROW_SCAN_LINE.lock() { + *line = Some(counts.line()); + } + counts +} + +/// The `(start, end)` op ranges of a list of `len` ops at `max_rows` a table, as +/// [`chunk_and_generate_segmented`] cuts it: the tables whose per-row lookups +/// (the μ bounds, the padding rows' byte checks) the table phase counts. The +/// list holds the streamed chunks' ops unless the build kept only the tails +/// (`StreamSkip::tails`); then the walk counted those chunks' lookups as it +/// handed them out, and the list's chunks are exactly the ones left to build. +pub(crate) fn list_chunks(len: usize, max_rows: usize) -> Vec<(usize, usize)> { + let max_rows = max_rows.max(1); + (0..len.div_ceil(max_rows)) + .map(|k| (k * max_rows, ((k + 1) * max_rows).min(len))) + .collect() +} + +/// [`chunk_padding_rows`] for a table of `rows` real rows. +pub(crate) fn padding_rows_of(rows: usize) -> usize { + chunk_padding_rows(rows) +} + +/// Generates ARE_BYTES ops for CPU padding rows. +/// +/// CPU padding rows have all byte columns = 0 (RS1=0, RS2=0, RD=0, etc.). +/// Since the CPU bus interactions use Multiplicity::One for range checks, +/// padding rows also send, so we need matching bitwise ops. +/// +/// Add the BITWISE lookups every CPU padding row sends. A padding row has all +/// values zero, so per row it sends 3× ARE_BYTES[0,0] (rs1/rs2, +/// rd/instruction_length, alu_flags/mem_flags) and 4× IS_HALF[0] (the four `res` +/// halves). Every padding row sends the same lookups, so their whole +/// contribution is a per-cell count: two `bump_n`s, no per-row work. +fn add_padding_byte_checks(hist: &mut bitwise::BitwiseHistogram, num_padding_rows: usize) { + let n = num_padding_rows as u64; + hist.bump_n( + BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, 0, 0), + 3 * n, + ); + hist.bump_n( + BitwiseOperation::halfword(BitwiseOperationType::IsHalf, 0, 0), + 4 * n, + ); +} + +/// Collects ARE_BYTES lookups from PAGE data (init and fini values). +/// +/// Each PAGE row generates 1 batched ARE_BYTES lookup: +/// - C1+C2: ARE_BYTES[init, fini] — range-checks both bytes in one interaction +/// +/// This must be called BEFORE bitwise multiplicities are updated. +/// +/// Encode private input as `[len_u32_LE][data]` — the canonical wire format. +/// Must match `executor::vm::memory::Memory::store_private_inputs`. +fn private_input_bytes(private_input: &[u8]) -> Vec { + let len_bytes = (private_input.len() as u32).to_le_bytes(); + len_bytes + .iter() + .chain(private_input.iter()) + .copied() + .collect() +} + +/// Build the initial-memory image (byte address -> value) from the ELF segments +/// and the private-input region. Single source of "what memory starts as", read +/// by both `MemoryState` seeding and PAGE/bitwise init. pub(crate) fn build_initial_image(elf: &Elf, private_input: &[u8]) -> HashMap { let mut image: HashMap = HashMap::new(); for segment in &elf.data { @@ -2507,11 +5166,12 @@ pub(crate) fn epoch_touched_cells( ) -> Result, Error> { let instructions = decode::instructions_from_elf(elf) .map_err(|e| Error::Execution(format!("Failed to parse instructions: {e}")))?; - let cpu_ops = collect_cpu_ops(logs, &instructions)?; + let decode = DecodeTable::from_instructions(&instructions); + let cpu_ops = collect_cpu_ops(logs, &decode)?; let mut memory_state = MemoryState::from_image(initial_image); let mut register_state = RegisterState::from_init(register_init); - let _ = collect_ops_from_cpu(&cpu_ops, &mut memory_state, &mut register_state)?; + let _ = collect_ops_from_cpu(&cpu_ops, &decode, &mut memory_state, &mut register_state)?; Ok(touched_cells_from_memory_state(&memory_state)) } @@ -2587,14 +5247,29 @@ fn collect_bitwise_from_page( } } -// ============================================================================= /// BITWISE lookups sent by the COPY table: twelve `IS_HALF` for the three chained /// dwords on every row, plus four more for `dst_res` on a `write` row. /// /// The `ZERO` end detection is gone with the lookup it fed: `end` is now pinned by /// the `restrict_end` constraint group against the borrow out of `count - step`. +#[cfg(test)] fn collect_bitwise_from_copy(ops: &[copy::CopyOperation]) -> Vec { let mut lookups = Vec::with_capacity(ops.len() * 16); + for_each_copy_lookup(ops, |op| lookups.push(op)); + lookups +} + +/// [`collect_bitwise_from_copy`]'s lookups counted into `histogram`, with no list +/// built: a block's COPY rows are millions, sixteen lookups at most each. +pub(crate) fn count_bitwise_from_copy( + ops: &[copy::CopyOperation], + histogram: &mut bitwise::BitwiseHistogram, +) { + for_each_copy_lookup(ops, |op| histogram.bump(op)); +} + +/// Every BITWISE lookup the COPY rows `ops` send, in row order. +fn for_each_copy_lookup(ops: &[copy::CopyOperation], mut emit: impl FnMut(BitwiseOperation)) { for op in ops { let width = u64::from(op.width); let count_decr = op.count.wrapping_sub(width); @@ -2604,18 +5279,17 @@ fn collect_bitwise_from_copy(ops: &[copy::CopyOperation]) -> Vec> shift) & 0xFFFF) as u16; - lookups.push(is_half_op(half)); + emit(is_half_op(half)); } } if op.functionality == copy::Functionality::Write { for shift in [0, 16, 32, 48] { let half = ((op.dst_res() >> shift) & 0xFFFF) as u16; - lookups.push(is_half_op(half)); + emit(is_half_op(half)); } } } - lookups } /// BITWISE lookups sent by the HINT table: `ARE_BYTES[out[2i], out[2i+1]]` for the @@ -2695,11 +5369,75 @@ pub(crate) fn collect_bitwise_from_ecsm(ops: &[ecsm::EcsmOperation]) -> Vec Vec { +/// BITWISE lookups sent by the ECREC table, in the order and under the gate +/// (`OK`, a computed row) of `ecrec::bus_interactions`: byte checks on the +/// witness bytes, halfword checks on the five relations' offset carries and the +/// nine subtraction witnesses, and the four nonzero checks. +pub(crate) fn collect_bitwise_from_ecrec(ops: &[ecrec::EcrecOperation]) -> Vec { let mut out = Vec::new(); + let half = |bytes: &[u8; 32], i: usize| bytes[2 * i] as u16 + ((bytes[2 * i + 1] as u16) << 8); for op in ops { + let w = &op.witness; + if !w.ok { + continue; + } + for bytes in [&w.w, &w.qw, &w.qz, &w.qu, &w.y_r, &w.x2, &w.q0] { + out.extend(bytes.iter().map(|&b| is_byte_op(b))); + } + out.extend(w.q1.iter().map(|&b| is_byte_op(b))); + out.push(is_byte_op(w.h)); + for (carries, offset) in [ + (&w.cw, ecrec::CARRY_OFFSET_MODN), + (&w.cz, ecrec::CARRY_OFFSET_MODN), + (&w.cu, ecrec::CARRY_OFFSET_MODN), + (&w.c0, ecsm::CARRY_OFFSET_X2), + (&w.c1, ecsm::CARRY_OFFSET_YG), + ] { + for c in carries.iter().take(63) { + out.push(is_half_op((c + offset) as u16)); + } + } + for bytes in [ + &w.r_sub_n, + &w.s_sub_n, + &w.u1_sub_n, + &w.u2_sub_n, + &w.y_r_sub_p, + &w.x_a_sub_p, + &w.x_b_sub_p, + &w.x_q_sub_p, + &w.y_q_sub_p, + ] { + for i in 0..16 { + out.push(is_half_op(half(bytes, i))); + } + } + for bytes in [&w.r, &w.s, &w.u1, &w.u2] { + out.push(BitwiseOperation::zero( + bytes.iter().map(|&b| b as u32).sum(), + )); + } + } + out +} + +/// BITWISE lookups sent by every ECDAS row (range checks on the byte limbs + +/// carries), over the rows a build keeps compact, each widened back to its op +/// as it is read ([`ecdas::CompactEcdasOp`]). +pub(crate) fn collect_bitwise_from_compact_ecdas( + rows: &[ecdas::CompactEcdasOp], +) -> Vec { + let mut out = Vec::new(); + for row in rows { + ecdas_lookups(&row.to_op(), &mut out); + } + out +} + +/// One ECDAS row's BITWISE lookups, appended to `out`. +#[allow(clippy::needless_range_loop)] +fn ecdas_lookups(op: &ecdas::EcdasOperation, out: &mut Vec) { + { let s = &op.step; out.push(is_byte_op(s.round)); for i in 0..32 { @@ -2718,34 +5456,48 @@ pub(crate) fn collect_bitwise_from_ecdas(ops: &[ecdas::EcdasOperation]) -> Vec Vec { - use executor::vm::instruction::execution::{KECCAK_RC, KECCAK_RHO}; - +/// Mirrors `blake3::bus_interactions` send-for-send: the alignment AND, the +/// addr AreBytes pairs, the pointer IS_HALFs, the mixing-core / feed-forward +/// ByteAlu XORs and shift-halfword AreBytes (both via `blake3::ValueFlow`, +/// the same canonical enumeration the senders are built from), the message +/// AreBytes and the OLD_OUT AreBytes. +/// +/// The absorb mode's rows are mirrored the same way, off the same +/// `blake3::expand_absorb` the trace filler uses, so a row and the +/// multiplicities it consumes cannot come from two different enumerations. +pub(crate) fn collect_bitwise_from_blake3( + blake3_ops: &[Blake3Operation], + absorb_ops: &[blake3::Blake3AbsorbOperation], +) -> Vec { let mut ops = Vec::new(); - for kop in keccak_ops { - let state_addr = kop.state_addr; + /// The four halfword range checks of one DWordHL pointer. + fn is_half_dword(ops: &mut Vec, ptr: u64) { + for shift in [0, 16, 32, 48] { + let half = ((ptr >> shift) & 0xFFFF) as u16; + ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + ((half >> 8) & 0xFF) as u8, + )); + } + } + + for bop in blake3_ops { + let state_addr = bop.state_addr; + // Alignment: addr[0] & 7 = 0. ops.push(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluAnd, (state_addr & 0xFF) as u8, 7, )); - // Range-check addr bytes (paired with the ARE_BYTES sends in - // keccak::bus_interactions): without this the field-element value of - // the addr_lo / addr_hi linear combinations is unconstrained per byte. - // 4 paired ops matching the (addr[2i], addr[2i+1]) sender pairing. + // Addr byte range checks: (addr[2i], addr[2i+1]) pairs. for i in 0..4 { let lo = ((state_addr >> (2 * i * 8)) & 0xFF) as u8; let hi = ((state_addr >> ((2 * i + 1) * 8)) & 0xFF) as u8; @@ -2756,31 +5508,320 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec )); } - // IS_HALF for state_ptr half_lo (25 per call). The top 48 bits go to - // IS_WH through `IS_WHH`; that table counts its own checks. - for lane_idx in 0..25 { + // IS_HALF for the 22 pointers' halfwords. + for k in 0..blake3::STATE_DWORDS { let ptr = state_addr - .checked_add(lane_idx as u64 * 8) - .expect("keccak state address range must be validated by the executor"); - let half = (ptr & 0xFFFF) as u16; - ops.push(BitwiseOperation::halfword( - BitwiseOperationType::IsHalf, - (half & 0xFF) as u8, - ((half >> 8) & 0xFF) as u8, + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + for shift in [0, 16, 32, 48] { + let half = ((ptr >> shift) & 0xFFFF) as u16; + ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + ((half >> 8) & 0xFF) as u8, + )); + } + } + + // Mixing core + feed-forward, in the senders' canonical order. + let flow = blake3::ValueFlow::compute(&bop.h, &bop.m, bop.t, bop.block_len, bop.flags); + for &(a, b, _out) in &flow.xors { + for byte in 0..4 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + ((a >> (8 * byte)) & 0xFF) as u8, + ((b >> (8 * byte)) & 0xFF) as u8, + )); + } + } + for &(sll_lo, sllc_lo, sll_hi, sllc_hi, _y) in &flow.rots { + for hw in [sll_lo, sllc_lo, sll_hi, sllc_hi] { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (hw & 0xFF) as u8, + (hw >> 8) as u8, + )); + } + } + + // Message AreBytes: (byte 2p, byte 2p+1) of each m word. + for m in bop.m { + for p in 0..2 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((m >> (16 * p)) & 0xFF) as u8, + ((m >> (16 * p + 8)) & 0xFF) as u8, + )); + } + } + + // OLD_OUT AreBytes pairs. + for p in 0..32 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + bop.old_out[2 * p], + bop.old_out[2 * p + 1], )); } + } - // Replay keccak round computation to extract bitwise lookups - let mut state = kop.input; - for round in 0..24 { - // --- theta: Cxz chain BYTE_ALU[XOR] (160) --- - let mut cxz = [[[0u8; 8]; 4]; 5]; - for x in 0..5 { - for b in 0..8 { + // ---- absorb mode ---- + for aop in absorb_ops { + let rows = blake3::expand_absorb(aop); + let n_rows = rows.len(); + for (i, r) in rows.iter().enumerate() { + let compressing = !r.end; + // Gated on μ: every row of the group. + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluAnd, + (aop.ctrl_addr & 0xFF) as u8, + 7, + )); + for p in 0..4 { + let lo = ((aop.ctrl_addr >> (2 * p * 8)) & 0xFF) as u8; + let hi = ((aop.ctrl_addr >> ((2 * p + 1) * 8)) & 0xFF) as u8; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + lo, + hi, + )); + } + // Only the control region's pointers: ptr[8..22] are MU_S-gated. + for k in 0..blake3::CTRL_DWORDS { + is_half_dword( + &mut ops, + aop.ctrl_addr + .checked_add(k as u64 * 8) + .expect("absorb control region range must be validated by the executor"), + ); + } + // OLD_OUT is all-zero except on the END row, where it carries the + // previous content of cv_out; the check itself is μ-gated either way. + for p in 0..32 { + let (lo, hi) = if r.end { + (aop.old_cv_out.get(2 * p), aop.old_cv_out.get(2 * p + 1)) + } else { + (None, None) + }; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + lo.copied().unwrap_or(0), + hi.copied().unwrap_or(0), + )); + } + // ZERO[REMAINING] -> END, on every absorb row. + ops.push(BitwiseOperation::zero(r.remaining)); + // M_BASE halfwords, on every absorb row (it rides the chain to END). + is_half_dword(&mut ops, r.m_base); + if r.first { + // IS_HALF[Q]: the x11 8-alignment witness, `M_BASE[0] / 8`. + let q = (r.m_base & 0xFFFF) / 8; + ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (q & 0xFF) as u8, + ((q >> 8) & 0xFF) as u8, + )); + } + + if compressing { + // ZERO[REM_DECR] -> NEXT_IS_END, on compressing rows only. + ops.push(BitwiseOperation::zero(r.remaining - 1)); + // The mixing core, in the senders' canonical order. + let flow = blake3::ValueFlow::compute(&r.h, &r.m, 0, 64, r.flags); + for &(a, b, _out) in &flow.xors { + for byte in 0..4 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + ((a >> (8 * byte)) & 0xFF) as u8, + ((b >> (8 * byte)) & 0xFF) as u8, + )); + } + } + for &(sll_lo, sllc_lo, sll_hi, sllc_hi, _y) in &flow.rots { + for hw in [sll_lo, sllc_lo, sll_hi, sllc_hi] { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (hw & 0xFF) as u8, + (hw >> 8) as u8, + )); + } + } + for m in r.m { + for p in 0..2 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((m >> (16 * p)) & 0xFF) as u8, + ((m >> (16 * p + 8)) & 0xFF) as u8, + )); + } + } + // M_BASE_INCR and the 8 message dword pointers. + is_half_dword(&mut ops, r.m_base.wrapping_add(64)); + for j in 0..blake3::MSG_DWORDS { + is_half_dword( + &mut ops, + r.m_base + .checked_add(j as u64 * 8) + .expect("absorb message range must be validated by the executor"), + ); + } + } else { + // END row: `h`'s bytes are range-checked here instead of by the + // mixing core, which is gated off (soundness ledger 14). + for p in 0..16 { + let w = r.h[p / 2]; + let shift = 16 * (p % 2); + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((w >> shift) & 0xFF) as u8, + ((w >> (shift + 8)) & 0xFF) as u8, + )); + } + } + + if r.first { + // The in-circuit block cap: IsB20[REM_DECR · 2^10]. + let value = (r.remaining as u64 - 1) * blake3::ABSORB_CAP_SCALE; + ops.push(BitwiseOperation::b20( + (value & 0xFF) as u8, + ((value >> 8) & 0xFF) as u8, + ((value >> 16) & 0xF) as u8, + )); + } + debug_assert_eq!(r.first, i == 0, "FIRST is the group's first row"); + debug_assert_eq!(r.end, i + 1 == n_rows, "END is the group's last row"); + } + } + + ops +} + +/// Collect BITWISE lookups generated by the keccak chips. +/// +/// The keccak round chip sends BYTE_ALU and ARE_BYTES interactions (the θ/ρ +/// halfword shifts are enforced by inline constraints, not HWSL lookups); the +/// keccak core chip sends IS_HALF interactions. +/// All of these must be registered so the BITWISE table's multiplicities are correct. +#[allow(clippy::needless_range_loop)] +pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec { + let mut ops = Vec::new(); + for_each_keccak_lookup(keccak_ops, |op| ops.push(op)); + ops +} + +/// [`collect_bitwise_from_keccak`]'s lookups counted into `histogram`, with no +/// list built: about five thousand a permutation. +pub(crate) fn count_bitwise_from_keccak( + keccak_ops: &[KeccakOperation], + histogram: &mut bitwise::BitwiseHistogram, +) { + for_each_keccak_lookup(keccak_ops, |op| histogram.bump(op)); +} + +/// Every BITWISE lookup the keccak chips send for `keccak_ops`, permutation by +/// permutation, its rounds replayed. +#[allow(clippy::needless_range_loop)] +fn for_each_keccak_lookup(keccak_ops: &[KeccakOperation], mut emit: impl FnMut(BitwiseOperation)) { + for kop in keccak_ops { + if let Some(sponge) = &kop.sponge { + emit_sponge_lookups(kop, sponge, &mut emit); + } else { + emit_permute_lookups(kop, &mut emit); + } + emit_round_lookups(kop, &mut emit); + } +} + +/// The BITWISE lookups a sponge row of KECCAK sends (`keccak::sponge_bus_interactions`): +/// the 136 rate XORs `INPUT = PREV ⊕ m'`, the halves of `SRC_LO + 135`, and on the +/// last row the halves of `OUT_LO + 31`. +fn emit_sponge_lookups( + kop: &KeccakOperation, + sponge: &keccak::SpongeRow, + emit: &mut impl FnMut(BitwiseOperation), +) { + let prev = kop.sponge_prev(sponge); + let padded = sponge.padded(); + for i in 0..keccak::cols::RATE { + emit(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + prev[i], + padded[i], + )); + } + let halves = |value: u64, emit: &mut dyn FnMut(BitwiseOperation)| { + for half in [value & 0xFFFF, value >> 16] { + emit(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + ((half >> 8) & 0xFF) as u8, + )); + } + }; + halves((sponge.src & 0xFFFF_FFFF) + 135, &mut *emit); + if sponge.last { + halves((sponge.out & 0xFFFF_FFFF) + 31, &mut *emit); + } +} + +/// The BITWISE lookups a permute row of KECCAK sends: the alignment AND, the +/// address bytes and the low halves of the lane pointers. +fn emit_permute_lookups(kop: &KeccakOperation, emit: &mut impl FnMut(BitwiseOperation)) { + { + let state_addr = kop.state_addr; + + emit(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluAnd, + (state_addr & 0xFF) as u8, + 7, + )); + + // Range-check addr bytes (paired with the ARE_BYTES sends in + // keccak::bus_interactions): without this the field-element value of + // the addr_lo / addr_hi linear combinations is unconstrained per byte. + // 4 paired ops matching the (addr[2i], addr[2i+1]) sender pairing. + for i in 0..4 { + let lo = ((state_addr >> (2 * i * 8)) & 0xFF) as u8; + let hi = ((state_addr >> ((2 * i + 1) * 8)) & 0xFF) as u8; + emit(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + lo, + hi, + )); + } + + // IS_HALF for state_ptr half_lo (25 per call). The top 48 bits go to + // IS_WH through `IS_WHH`; that table counts its own checks. + for lane_idx in 0..25 { + let ptr = state_addr + .checked_add(lane_idx as u64 * 8) + .expect("keccak state address range must be validated by the executor"); + let half = (ptr & 0xFFFF) as u16; + emit(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + ((half >> 8) & 0xFF) as u8, + )); + } + } +} + +/// The BITWISE lookups the 24 KECCAK_RND rows of one permutation send. +#[allow(clippy::needless_range_loop)] +fn emit_round_lookups(kop: &KeccakOperation, emit: &mut impl FnMut(BitwiseOperation)) { + use executor::vm::instruction::execution::{KECCAK_RC, KECCAK_RHO}; + { + // Replay keccak round computation to extract bitwise lookups + let mut state = kop.input; + for round in 0..24 { + // --- theta: Cxz chain BYTE_ALU[XOR] (160) --- + let mut cxz = [[[0u8; 8]; 4]; 5]; + for x in 0..5 { + for b in 0..8 { let v0 = ((state[x] >> (b * 8)) & 0xFF) as u8; let v1 = ((state[x + 5] >> (b * 8)) & 0xFF) as u8; cxz[x][0][b] = v0 ^ v1; - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluXor, v0, v1, @@ -2792,7 +5833,7 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec let prev = cxz[x][stage - 1][b]; let sv = ((state[x + 5 * y] >> (b * 8)) & 0xFF) as u8; cxz[x][stage][b] = prev ^ sv; - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluXor, prev, sv, @@ -2813,7 +5854,7 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec let shifted = halfword << 1; // u16 wraps // ARE_BYTES for cxz_left bytes: paired (low, high) of the halfword, // matching `(cxz_left[x][2i], cxz_left[x][2i+1])` sender pairing. - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::AreBytes, (shifted & 0xFF) as u8, ((shifted >> 8) & 0xFF) as u8, @@ -2846,7 +5887,7 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec let a = cxz[(x + 4) % 5][3][b]; let rb = rotated_c[(x + 1) % 5][b]; d_bytes[x][b] = a ^ rb; - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluXor, a, rb, @@ -2866,7 +5907,7 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec theta_lanes[x + 5 * y] = lane ^ d_lane; for b in 0..8 { let s = ((lane >> (b * 8)) & 0xFF) as u8; - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluXor, s, d_bytes[x][b], @@ -2892,12 +5933,12 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec // ARE_BYTES paired as (rot_left[b], rot_right[b]) for // each byte of the halfword, matching the sender pairing // in keccak_rnd::bus_interactions. - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::AreBytes, (shifted & 0xFF) as u8, (carry & 0xFF) as u8, )); - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::AreBytes, ((shifted >> 8) & 0xFF) as u8, ((carry >> 8) & 0xFF) as u8, @@ -2928,14 +5969,14 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec for b in 0..8 { let not_byte = ((not_next >> (b * 8)) & 0xFF) as u8; let n2_byte = ((next2 >> (b * 8)) & 0xFF) as u8; - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluAnd, not_byte, n2_byte, )); let pi_byte = ((pi_lanes[x + 5 * y] >> (b * 8)) & 0xFF) as u8; let and_byte = ((and_val >> (b * 8)) & 0xFF) as u8; - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluXor, pi_byte, and_byte, @@ -2949,7 +5990,7 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec for b in 0..8 { let chi_byte = ((chi_lanes[0] >> (b * 8)) & 0xFF) as u8; let rc_byte = ((rc_val >> (b * 8)) & 0xFF) as u8; - ops.push(BitwiseOperation::byte_op( + emit(BitwiseOperation::byte_op( BitwiseOperationType::ByteAluXor, chi_byte, rc_byte, @@ -2961,12 +6002,11 @@ pub(crate) fn collect_bitwise_from_keccak(keccak_ops: &[KeccakOperation]) -> Vec state = chi_lanes; } } - - ops } /// every address accessed during execution (ELF init + runtime stores/loads). /// ELF pages get their init data from the binary; all others are zero-init. +#[cfg(test)] fn generate_page_tables( image: &I, memory_state: &MemoryState, @@ -2976,6 +6016,41 @@ fn generate_page_tables( Vec>, Vec, ) { + let page_configs = page_configs_of(image, memory_state, private_input); + let pages = page_configs + .iter() + .map(|config| page_trace(config, memory_state, exclude_touched)) + .collect(); + (pages, page_configs) +} + +/// The PAGE table of `config`'s page. +/// +/// The per-page final `(value, timestamp)` is read straight from the dense +/// `memory_state.cells` store (one indexed read per offset) rather than routing +/// through a sparse `FinalStateMap` whose per-page (mostly-miss) lookups dominated +/// PAGE generation. `exclude_touched` (continuation epoch with L2G bookend) drops +/// runtime-written cells (ts > 0) so PAGE self-cancels them (init == fini, ts == 0) +/// and the local-to-global table owns their Memory-bus init/fini instead — applied +/// per offset inside `generate_page_trace_from_dense`. +fn page_trace( + config: &PageConfig, + memory_state: &MemoryState, + exclude_touched: bool, +) -> TraceTable { + let final_page = memory_state.cells.page_data(config.page_base); + page::generate_page_trace_from_dense(config, final_page, exclude_touched) +} + +/// The PAGE configs of a run, one a page the memory state holds, in base +/// order: private-input pages, then pages the initial image fills, then +/// zero-init pages, as [`generate_page_tables`] makes them — the PAGE layout, +/// known before any PAGE table is generated. +fn page_configs_of( + image: &I, + memory_state: &MemoryState, + private_input: &[u8], +) -> Vec { use std::collections::BTreeSet; // Per-page init bytes from the initial-memory image. @@ -2984,39 +6059,23 @@ fn generate_page_tables( // Derive ALL page bases from memory_state (includes ELF + runtime pages) let page_bases: BTreeSet = memory_state.cells.page_bases().collect(); - // The per-page final `(value, timestamp)` is read straight from the dense - // `memory_state.cells` store (one indexed read per offset) rather than routing - // through a sparse `FinalStateMap` whose per-page (mostly-miss) lookups dominated - // PAGE generation. `exclude_touched` (continuation epoch with L2G bookend) drops - // runtime-written cells (ts > 0) so PAGE self-cancels them (init == fini, ts == 0) - // and the local-to-global table owns their Memory-bus init/fini instead — applied - // per offset inside `generate_page_trace_from_dense`. - - // Generate PAGE tables and configs - let mut pages = Vec::new(); - let mut page_configs = Vec::new(); - // Determine which page bases hold private input data — count-based, via the // shared helpers (single source of truth with the continuation path). let num_private_input_pages = page::private_input_page_count(private_input); - for &page_base in &page_bases { - let config = if page::is_private_input_page(page_base, num_private_input_pages) { - let init_data = init_page_data.get(&page_base).cloned().unwrap_or_default(); - PageConfig::with_private_input(page_base, init_data) - } else if let Some(init_data) = init_page_data.get(&page_base) { - PageConfig::with_data(page_base, init_data.clone()) - } else { - PageConfig::zero_init(page_base) - }; - - let final_page = memory_state.cells.page_data(page_base); - let trace = page::generate_page_trace_from_dense(&config, final_page, exclude_touched); - pages.push(trace); - page_configs.push(config); - } - - (pages, page_configs) + page_bases + .into_iter() + .map(|page_base| { + if page::is_private_input_page(page_base, num_private_input_pages) { + let init_data = init_page_data.get(&page_base).cloned().unwrap_or_default(); + PageConfig::with_private_input(page_base, init_data) + } else if let Some(init_data) = init_page_data.get(&page_base) { + PageConfig::with_data(page_base, init_data.clone()) + } else { + PageConfig::zero_init(page_base) + } + }) + .collect() } // ============================================================================= @@ -3030,9 +6089,10 @@ fn generate_page_tables( /// build ([`Traces::from_image_and_logs_with_decode`]) instead of re-parsing /// the ELF and regenerating the trace per epoch. pub struct DecodeArtifacts { - instructions: U64HashMap, decode_trace: TraceTable, - decode_pc_to_row: decode::PcToRow, + /// Every instruction's decode by DECODE row: what the CPU ops are read + /// with (shared with the threads that generate CPU chunks). + decode: Arc, } impl DecodeArtifacts { @@ -3044,11 +6104,11 @@ impl DecodeArtifacts { pub fn from_elf(elf: &Elf) -> Result { let instructions = decode::instructions_from_elf(elf) .map_err(|e| Error::Execution(format!("Failed to parse instructions: {e}")))?; - let (decode_trace, decode_pc_to_row) = decode::generate_decode_trace(&instructions); + let (decode_trace, _) = decode::generate_decode_trace(&instructions); + let decode = Arc::new(DecodeTable::from_instructions(&instructions)); Ok(Self { - instructions, decode_trace, - decode_pc_to_row, + decode, }) } } @@ -3143,6 +6203,12 @@ pub struct Traces { /// HALT single-row table for program termination pub halt: TraceTable, + /// Unified COPY tables: one streaming copy primitive for memcpy/memmove, + /// memset and the commit syscall's bytes, selected by decoded functionality + /// columns. Empty when the run neither commits nor copies; a block proof cuts + /// the rows into tables of a fixed height (`block_whir::split_copy`). + pub copies: Vec>, + /// KECCAK core table (one row per keccak permutation call). Empty when the /// run makes no keccak call. pub keccaks: Vec>, @@ -3153,6 +6219,21 @@ pub struct Traces { /// KECCAK_RC precomputed round constant table (32 rows) pub keccak_rc: TraceTable, + /// BLAKE3 6-round compression table (one row per compression call) + pub blake3: TraceTable, + + /// BLAKE3 compressions the workload actually performed. + /// + /// Not derivable from `blake3` above: that trace pads to a 4-row minimum, + /// so an unused table and a table with four compressions are the same + /// height. This is what decides whether the proof carries the table at all + /// (`crate::TableCounts::blake3`). + /// + /// ⚠ The six accelerator tables beside it are `Vec`s and say the same thing + /// with their length, so they need no companion count. BLAKE3 keeps one + /// because its trace is still a single padded table rather than a vector. + pub num_blake3_ops: usize, + /// ECSM core table (one row per scalar-multiplication ecall). Empty when the /// run makes no ECSM call. pub ecsms: Vec>, @@ -3164,18 +6245,14 @@ pub struct Traces { /// makes no hint ecall. pub hints: Vec>, + /// ECREC table (one row per ECDSA public-key recovery ecall). Empty when the + /// run makes no ECREC call; its ECDAS rows are in `ecdases`. + pub ecrecs: Vec>, + /// IS_WH table (one row per distinct `(Word, Half)` address pair sent through /// `IS_WHH`). Empty when no chip sends one. pub is_whs: Vec>, - /// Unified COPY table: one streaming copy primitive for memcpy/memmove, - /// memset and the commit byte loop, selected by decoded functionality columns. - /// Always present: COPY receives the `write` ECALL, so any run that produces - /// output reaches it. A run that neither commits nor copies still ships an - /// all-padding COPY sub-proof — it could be made optional like the other - /// accelerators, which is tracked separately. - pub copy: TraceTable, - /// MEMW_R register-only fast-path traces (split into chunks of max_rows::MEMW_R) pub memw_registers: Vec>, /// Local-to-global boundary table for continuation epochs. Empty unless the @@ -3195,137 +6272,1421 @@ pub struct Traces { /// Intermediate state from Phase 2: all ops collected from CPU, ready for /// Phases 3-5 (LT extension, bitwise, trace generation). +#[derive(Default)] +/// +/// The lists a windowed build accumulates over the run (and those assembled +/// from segments) are [`BlockVec`]s: the blocks they were appended in, never +/// concatenated ([`blocks`]); a whole-run build's are one block each. struct CollectedOps { cpu_ops: Vec, memw_ops: Vec, - memw_aligned_ops: Vec, + memw_aligned_ops: Vec, /// Direct-fill MEMW_R rows (register fast path). memw_register_rows: Vec, load_ops: Vec, - lt_ops: Vec, - shift_ops: Vec, - bitwise_ops: Vec, - branch_ops: Vec, - mul_ops: Vec<(MulOperation, bool)>, - dvrm_ops: Vec<(DvrmOperation, bool)>, - keccak_ops: Vec, + lt_ops: BlockVec, + shift_ops: BlockVec, + bitwise_ops: BlockVec, + /// Delta-coded ([`CompactBranch`]). + branch_ops: CompactBranch, + mul_ops: BlockVec<(MulOperation, bool)>, + dvrm_ops: BlockVec<(DvrmOperation, bool)>, + keccak_ops: BlockVec, + blake3_ops: BlockVec, + blake3_absorb_ops: BlockVec, // Auxiliary ALU / memory / CPU32 dispatch chips (driven by the CPU ALU/MEMORY dispatch). - eq_ops: Vec, - bytewise_ops: Vec, - store_ops: Vec, - cpu32_ops: Vec, - // EC scalar-multiplication accelerator chips. - ecsm_ops: Vec, - ecdas_ops: Vec, + /// Delta-coded ([`CompactEq`]). + eq_ops: CompactEq, + /// Delta-coded ([`CompactBytewise`]). + bytewise_ops: CompactBytewise, + store_ops: BlockVec, + cpu32_ops: BlockVec, + // EC scalar-multiplication accelerator chips. ECDAS's rows held compact + // (their carries in 16 bits), widened as each is read. + ecsm_ops: BlockVec, + ecdas_ops: BlockVec, // Unified COPY rows: memcpy/memmove, memset and the `write` byte loop. - copy_ops: Vec, + copy_ops: BlockVec, // Non-constraining hint ecall. - hint_ops: Vec, + hint_ops: BlockVec, + // ECDSA public-key recovery. + ecrec_ops: BlockVec, } -/// Chunk raw ops and generate one trace table per chunk, padding an empty `ops` -/// to a single chunk so the table is always present in the proof. -/// -/// For tables that may be omitted entirely, use [`chunk_and_generate_optional`]. -fn chunk_and_generate( - ops: &[T], - max_rows: usize, - generate: impl Fn(&[T]) -> TraceTable + Send + Sync, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, -) -> Result>, Error> { - let op_chunks: Vec<&[T]> = if ops.is_empty() { - vec![&[][..]] - } else { - ops.chunks(max_rows).collect() - }; - generate_chunks( - op_chunks, - generate, - #[cfg(feature = "disk-spill")] - storage_mode, - ) +/// How many leading chunks of each streamed table a windowed build already +/// generated and handed out (`WindowedTraceBuilder`). Those chunks are the +/// same op slices a whole-run build would chunk; the final build leaves an +/// empty placeholder in their slots ([`Traces::insert_streamed`] puts them +/// back). All zero — every build but the windowed one — changes nothing. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StreamSkip { + pub cpu: usize, + pub memw_register: usize, + pub memw_aligned: usize, + pub memw: usize, + pub load: usize, + pub lt: usize, + pub shift: usize, + pub store: usize, + /// Rows per KECCAK_RND chunk, when the build cuts KECCAK_RND into chunks of + /// that many rows (the split a block proof makes); zero builds it whole, as + /// every build but the block's. + pub keccak_rnd_rows: usize, + /// KECCAK_RND chunks already handed out (with `keccak_rnd_rows`). + pub keccak_rnd: usize, + /// Rows per KECCAK, ECSM, ECDAS and COPY table when the build cuts them + /// into the tables the block's split makes ([`cut_tables`], each packed as + /// it is built with [`Self::pack`]); zero builds each whole, as every build + /// but the block's. + pub keccak_rows: usize, + pub ecsm_rows: usize, + pub ecdas_rows: usize, + pub copy_rows: usize, + /// MEMW (general) and MEMW_A ops whose phase-3 LT ops a windowed build has + /// already put into the windows' LT lists (block path only, which may chunk + /// LT differently from the whole-run build): the table phase derives LT + /// only from the ops past these. Zero everywhere else. + pub memw_lt_done: usize, + pub memw_aligned_lt_done: usize, + /// Pack each table the build generates at the bytes its columns need as + /// soon as it is generated (`TraceTable::pack_main_narrow`), so the build + /// never holds its tables at eight bytes a cell; the words are the same. + /// A whole KECCAK_RND, and KECCAK, ECSM, ECDAS and COPY when not cut at + /// the build (`keccak_rows` and the rest zero), stay wide: the block cuts + /// them after the build. `false` everywhere but the WHIR block's windowed + /// finish. + pub pack: bool, + /// With [`Self::pack`]: each generator that can writes its table packed as + /// it generates it, with no 64-bit copy (G-pack, [`super::gpack`]), instead + /// of building it at 8 bytes a cell and packing it. The bytes are the same. + pub gpack: bool, + /// With [`Self::pack`] and KECCAK_RND cut at `keccak_rnd_rows`: each + /// KECCAK_RND table the build makes holds the keccak operations its rows + /// come from instead of its columns (`keccak_rnd::KeccakRndRecords`), and + /// the card fills the columns from them. The words are the same. Such a + /// table is never dropped, so phase B never builds KECCAK_RND again. + pub card_fill: bool, + /// A test's fault: with [`Self::card_fill`], each KECCAK_RND table's + /// records start this many rows late, so every fill of them (the card's at + /// both uploads, a host reader's) writes the same wrong table. + #[cfg(test)] + pub card_fill_fault: usize, + /// The streamed tables' op lists hold only the ops past their streamed + /// chunks (a windowed build that drops each chunk's ops as it hands the + /// chunk out, [`WindowedTraceBuilder::drop_streamed_ops`]), and + /// `memw_lt_done` / `memw_aligned_lt_done` count within those. `false` + /// everywhere else: the lists are the whole run's. + /// + /// [`WindowedTraceBuilder::drop_streamed_ops`]: windowed::WindowedTraceBuilder::drop_streamed_ops + pub tails: bool, + /// Phase 3 concatenates LT's ops into one list before chunking them, as it + /// did before keeping them as the segments they come from (the A arm of + /// the segments, `LAMBDA_VM_BLOCK_LT_CONCAT=1` on the block). The tables + /// are the same. + pub concat_lt: bool, + /// Phase 4 counts the deduplicating tables' per-row lookups (LT, BRANCH, + /// BYTEWISE) on the whole pool, a chunk a task, counts only + /// ([`count_row_lookups`]), instead of each chunk's list of lookups among + /// its other collectors (`LAMBDA_VM_BLOCK_P4_ROWS=pool` on the block). + /// The multiplicities and the tables are the same. + pub p4_rows: bool, + /// How that count counts each chunk's distinct ops ([`RowCount`]): one + /// map a chunk, or by parts (`LAMBDA_VM_BLOCK_P4_ROWS=parts` on the + /// block). The multiplicities are the same. + pub p4_by: RowCount, + /// Phase 4 counts LT's per-op lookups and KECCAK's straight into its + /// histograms ([`count_bitwise_from_lt_ops`], [`count_bitwise_from_keccak`]), + /// instead of building each slice's list of them first + /// (`LAMBDA_VM_BLOCK_P4_LOOKUPS` unset or `direct` on the block). The + /// multiplicities are the same. + pub p4_direct: bool, + /// The streamed finish counts the BITWISE lookups (phase 4) in BITWISE's + /// own table's job, beside the rest's builds, instead of before any of + /// them (`LAMBDA_VM_BLOCK_P4_BESIDE=1` on the block). The tables are the + /// same; a finish that builds every table at once counts first, as ever. + pub p4_beside: bool, + /// With [`Self::p4_beside`] and the slide, the rest is handed on in AIR + /// order but BITWISE, which goes after every other table, so the tables + /// that do not read the count reach the packer while it counts + /// (`LAMBDA_VM_BLOCK_REST_BITWISE_LAST=1` on the block). The tables are the + /// same; the groups they are packed into, and so the proof's bytes, are not. + pub bitwise_last: bool, + /// With the slide and only the streamed chunks' tails kept + /// ([`Self::tails`]), BITWISE's lookups from LT's, BRANCH's and BYTEWISE's + /// tables are counted from each table's rows as it is built, instead of + /// from their ops in phase 4, and BITWISE is built after them and handed + /// on last, as [`Self::bitwise_last`] hands it (the block's default; + /// `LAMBDA_VM_BLOCK_BITWISE_FROM_TABLES=0` rolls it back). Elsewhere it + /// changes nothing. The tables are the same; their order is not. + pub bitwise_from_tables: bool, + /// How the tables cut at `keccak_rnd_rows`, `keccak_rows`, `ecsm_rows`, + /// `ecdas_rows` and `copy_rows` are padded ([`CutPadding`]): the table + /// phase's counts, its + /// tables and the tables built again from its lists all read it here. The + /// default stops at the table that holds the last real row. + pub cut_padding: CutPadding, } -/// Like [`chunk_and_generate`], but an empty `ops` yields no table at all: the -/// chip is left out of the proof instead of costing a padded sub-proof. -/// -/// The empty case short-circuits rather than falling through to `ops.chunks`, -/// which panics on a zero chunk size even for an empty slice. `max_rows` comes -/// from a caller-supplied [`MaxRowsConfig`] whose fields are public. -/// -/// Sound because a chip contributes to the run only through its LogUp bus, and a -/// chip with no rows contributes zero. A prover that omits a table whose ops did -/// execute leaves its counterparty's sends unmatched, and the bus-balance check -/// over the tables that *are* present rejects the proof. See -/// `TableCounts::validate`. -fn chunk_and_generate_optional( - ops: &[T], - max_rows: usize, - generate: impl Fn(&[T]) -> TraceTable + Send + Sync, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, -) -> Result>, Error> { - let op_chunks: Vec<&[T]> = if ops.is_empty() { - vec![] - } else { - ops.chunks(max_rows).collect() - }; - generate_chunks( - op_chunks, - generate, - #[cfg(feature = "disk-spill")] - storage_mode, - ) +/// How the tables a block cuts at a fixed number of rows (KECCAK_RND, KECCAK, +/// ECSM, ECDAS, COPY) are padded. A prover's choice: a block statement states each +/// family's count and every table's height, whatever they are. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum CutPadding { + /// The whole run's rows padded to a power of two, then cut: the tables the + /// block's split makes of the whole padded table (`split_rows`). Past the + /// table that holds the last real row they are padding rows alone. + WholeRun, + /// Cut, and stop at the table that holds the last real row, padded to the + /// cut: [`Self::WholeRun`]'s tables without the ones of padding rows + /// alone. A padding row of each of the four families has every bus + /// multiplicity at zero, so a table of them adds nothing to the bus. + #[default] + LastTable, } -/// Generate a single trace table for `ops`, or none at all when `ops` is empty. -/// -/// The accelerator chips are not chunked: one call means one table. What they do -/// share with the chunked chips is that an empty op list should cost nothing, so -/// this returns an empty `Vec` and the table drops out of the proof. Soundness -/// rests on the same LogUp argument as [`chunk_and_generate_optional`]. -fn generate_optional( - ops: &[T], - generate: impl Fn(&[T]) -> TraceTable + Send + Sync, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, -) -> Result>, Error> { - let op_chunks: Vec<&[T]> = if ops.is_empty() { vec![] } else { vec![ops] }; - generate_chunks( - op_chunks, - generate, - #[cfg(feature = "disk-spill")] - storage_mode, - ) -} +impl CutPadding { + /// `LAMBDA_VM_BLOCK_CUT_PADDING`: unset or `last`, [`Self::LastTable`] (the + /// default); `run`, [`Self::WholeRun`]. Read once; any other value is + /// refused. It is the rule of the tables the table phase cuts + /// ([`StreamSkip::cut_padding`]) and of the block's split of a table built + /// whole (its comparison arms), so every arm builds the same tables. + pub(crate) fn from_env() -> Result { + static PADDING: std::sync::OnceLock> = + std::sync::OnceLock::new(); + PADDING + .get_or_init(|| { + Self::parse(std::env::var("LAMBDA_VM_BLOCK_CUT_PADDING").ok().as_deref()) + }) + .clone() + .map_err(Error::Prover) + } -/// Generate one trace table per already-chunked op slice. When `storage_mode` is -/// `Disk`, each chunk's main table is spilled to mmap before the next chunk is -/// built so peak heap usage stays bounded. -fn generate_chunks( - op_chunks: Vec<&[T]>, - generate: impl Fn(&[T]) -> TraceTable + Send + Sync, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, -) -> Result>, Error> { - // Disk mode generates one chunk at a time so each spills before the next - // allocates, keeping trace memory bounded. - #[cfg(feature = "disk-spill")] - if storage_mode == StorageMode::Disk { - let mut tables = Vec::with_capacity(op_chunks.len()); - for chunk in op_chunks { - let mut t = generate(chunk); - t.main_table - .spill_to_disk() - .map_err(|e| Error::Prover(format!("disk-spill trace: {e}")))?; - tables.push(t); + /// [`Self::from_env`]'s reading of its variable. + fn parse(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("last") => Ok(Self::LastTable), + Some("run") => Ok(Self::WholeRun), + Some(v) => Err(format!("LAMBDA_VM_BLOCK_CUT_PADDING={v}: run or last")), + } + } + + /// How many tables `n` rows cut at `rows` rows a table make: none without + /// rows; one when the whole padded table (`n.next_power_of_two().max(4)` + /// rows) is no taller than `rows`; else that table's `total / rows` cuts + /// ([`Self::WholeRun`]), or the `n.div_ceil(rows)` of them that hold a + /// real row ([`Self::LastTable`]). + pub(crate) fn tables(self, n: usize, rows: usize) -> usize { + if n == 0 { + return 0; + } + let total = n.next_power_of_two().max(4); + if total <= rows || rows == 0 { + return 1; + } + match self { + Self::WholeRun => total / rows, + Self::LastTable => n.div_ceil(rows), + } + } +} + +/// BITWISE lookups a windowed build counted while the run was still being +/// walked: the histogram of the first `bitwise_ops` in-walk lookups and of the +/// first `memw_register_rows` MEMW_R rows (the table phase's two dominant +/// sources). The table phase counts the rest of those two lists and every other +/// source as always; the histogram is a commutative monoid, so the +/// multiplicities are the same. +pub(crate) struct PreCounted { + pub(crate) histogram: bitwise::BitwiseHistogram, + pub(crate) bitwise_ops: usize, + pub(crate) memw_register_rows: usize, + /// The (timestamp, next pc) of the last ECALL among CPU ops that are no + /// longer in the CPU list ([`StreamSkip::tails`]): the HALT search's answer + /// when the list left holds none. + pub(crate) last_ecall: Option<(u64, u64)>, + /// The phase-3 LT ops of the MEMW (general) and MEMW_A ops that are no + /// longer in their lists ([`StreamSkip::tails`] without the MEMW-derived LT + /// ops streamed), in list order: phase 3 puts each before the LT ops of the + /// list it derives from the ops left. + pub(crate) memw_lt: CompactLt, + pub(crate) memw_aligned_lt: CompactLt, +} + +impl PreCounted { + /// The bytes it takes on the heap: the histogram and the derived LT ops. + pub(crate) fn heap_bytes(&self) -> usize { + self.histogram.heap_bytes() + self.memw_lt.heap_bytes() + self.memw_aligned_lt.heap_bytes() + } + + /// [`Self::heap_bytes`] part by part. + pub(crate) fn heap_parts(&self) -> Vec<(String, usize)> { + vec![ + ("counted histogram".to_string(), self.histogram.heap_bytes()), + ("counted memw_lt".to_string(), self.memw_lt.heap_bytes()), + ( + "counted memw_a_lt".to_string(), + self.memw_aligned_lt.heap_bytes(), + ), + ] + } + + /// Each part's largest single allocation, named as in [`Self::heap_parts`]. + pub(crate) fn largest_parts(&self) -> Vec<(String, usize)> { + vec![ + ("counted histogram".to_string(), self.histogram.heap_bytes()), + ("counted memw_lt".to_string(), self.memw_lt.largest_bytes()), + ( + "counted memw_a_lt".to_string(), + self.memw_aligned_lt.largest_bytes(), + ), + ] + } +} + +/// KECCAK_RND cut into tables of `rows` rows, the first `streamed` of them left +/// as placeholders: the tables `split_rows` makes of the whole table +/// ([`keccak_rnd::generate_keccak_rnd_trace`]), each built from the ops that +/// reach it, as many of them as `padding` counts ([`CutPadding::tables`]). A +/// table no taller than `rows` stays one table, as the split leaves it. +pub(crate) fn keccak_rnd_chunks( + ops: &[KeccakRoundOperation], + rows: usize, + streamed: usize, + padding: CutPadding, + pack: bool, + form: TraceForm, +) -> Result>, Error> { + keccak_rnd_chunks_in_waves(ops, rows, streamed, padding, pack, form, kr_packed_wave()) +} + +/// `LAMBDA_VM_KR_PACKED_WAVE`: how many KECCAK_RND chunks [`keccak_rnd_chunks`] +/// builds at once while it writes them packed (G-pack, once the widths are +/// known): unset or `0`, every chunk left in one parallel pass (the default); +/// `n ≥ 1`, `n` (`4`, [`KECCAK_RND_PACK_WAVE`], is the wave as it was). A chunk +/// written packed holds no 64-bit table, so the wave bounds nothing but the +/// parallelism. The tables and their order are the same. Read once; any other +/// value is the default. At the median block the finish's KECCAK_RND tail and +/// the card's wait for the rest go with it (ULTRA 096: finish 15.8 → 9.1 s, +/// phase A −6.7 s with the commit pairs off the pool, −5.7 s alone). +pub(crate) fn kr_packed_wave() -> usize { + static WAVE: std::sync::OnceLock = std::sync::OnceLock::new(); + *WAVE.get_or_init(|| { + kr_packed_wave_from(std::env::var("LAMBDA_VM_KR_PACKED_WAVE").ok().as_deref()) + }) +} + +/// [`kr_packed_wave`]'s reading of its variable. +fn kr_packed_wave_from(value: Option<&str>) -> usize { + match value.map(str::trim).map(str::parse::) { + Some(Ok(n)) if n > 0 => n, + _ => usize::MAX, + } +} + +/// [`keccak_rnd_chunks`], the chunks written packed built `packed_wave` at a +/// time ([`kr_packed_wave`]). +fn keccak_rnd_chunks_in_waves( + ops: &[KeccakRoundOperation], + rows: usize, + streamed: usize, + padding: CutPadding, + pack: bool, + form: TraceForm, + packed_wave: usize, +) -> Result>, Error> { + if ops.is_empty() { + return Ok(Vec::new()); + } + let total = (ops.len() * 24).next_power_of_two().max(4); + if total <= rows { + if streamed > 0 { + return Err(Error::Prover(format!( + "{streamed} KECCAK_RND chunks were streamed but the table is one of {total} rows" + ))); + } + return Ok(vec![packed_if( + pack, + keccak_rnd::generate_keccak_rnd_rows_as(ops, 0, total, form), + )]); + } + let chunks = padding.tables(ops.len() * 24, rows); + if streamed > chunks { + return Err(Error::Prover(format!( + "{streamed} KECCAK_RND chunks were streamed but the run has {chunks}" + ))); + } + let chunk = |c: usize| { + if c < streamed { + return streamed_placeholder(); + } + let (first, end) = keccak_rnd_op_range(c, rows, ops.len()); + packed_if( + pack, + keccak_rnd::generate_keccak_rnd_rows_as( + &ops[first..end], + c * rows - first * 24, + rows, + form, + ), + ) + }; + // Packed, a few chunks at a time in parallel: each chunk's pack is serial, + // and one task building them all one after another would be the finish's + // last; at most `KECCAK_RND_PACK_WAVE` chunks are wide at once. Once the + // chunks are written packed (the first wave left G-pack their widths) none + // is wide, and `packed_wave` go at once. + #[cfg(feature = "parallel")] + if pack { + let mut tables = Vec::with_capacity(chunks); + while tables.len() < chunks { + let wave = if keccak_rnd::rows_written_packed(form) { + packed_wave + } else { + KECCAK_RND_PACK_WAVE + }; + let first = tables.len(); + let end = first.saturating_add(wave.max(1)).min(chunks); + tables.par_extend((first..end).into_par_iter().map(chunk)); } return Ok(tables); } + #[cfg(not(feature = "parallel"))] + let _ = packed_wave; + Ok((0..chunks).map(chunk).collect()) +} + +/// The padding rows KECCAK_RND's tables hold for `n` KECCAK ops (24 rows an op): +/// the whole table's when it is built whole (`rows == 0`) or fits one table of +/// `rows`, else the tables `padding` makes of `rows` rows each ([`keccak_rnd_chunks`]) +/// less the real rows. Each sends KECCAK_RND's byte checks as `ARE_BYTES[0, 0]` +/// (`keccak_rnd::ARE_BYTES_PER_ROW`), so BITWISE counts them. +pub(crate) fn keccak_rnd_padding_rows(n: usize, rows: usize, padding: CutPadding) -> usize { + if n == 0 { + return 0; + } + let real = n * 24; + let total = real.next_power_of_two().max(4); + if rows == 0 || total <= rows { + return total - real; + } + padding.tables(real, rows) * rows - real +} + +/// KECCAK_RND chunks built and packed at once ([`keccak_rnd_chunks`]): each is +/// ≈ 0.8 GiB at eight bytes a cell (1,480 columns of 2^16 rows) before its pack. +const KECCAK_RND_PACK_WAVE: usize = 4; + +/// A table of one row an op (KECCAK, ECSM, ECDAS), built as the tables of +/// `rows` rows the block's split makes of the whole padded table +/// (`split_rows`): one table when the whole is no taller than `rows`, else +/// `total / rows` cuts, cut k the ops of rows `[k·rows, (k+1)·rows)` and the +/// padding to `rows` rows (`generate(ops, rows, form)`, a family's +/// `generate_*_rows_as`), as many of them as `padding` counts +/// ([`CutPadding::tables`]). No ops, no table. Each cut is packed as it is +/// built when `pack`. The words are the split's; the whole table and its split +/// copy are never held. A cut is read from the list's blocks: borrowed when it +/// is a block (ECDAS's blocks are its cut), copied when it straddles two. +fn cut_tables( + ops: &BlockVec, + rows: usize, + padding: CutPadding, + pack: bool, + form: TraceForm, + written_packed: fn(TraceForm) -> bool, + generate: impl Fn(&[T], usize, TraceForm) -> TraceTable + + Send + + Sync, +) -> Result>, Error> { + if ops.is_empty() { + return Ok(Vec::new()); + } + if !rows.is_power_of_two() { + return Err(Error::Prover(format!( + "tables cut at {rows} rows: a power of two is needed" + ))); + } + let n = ops.len(); + let total = n.next_power_of_two().max(4); + if total <= rows { + return Ok(vec![packed_if(pack, generate(&ops.whole(), total, form))]); + } + let cut = |k: usize| { + let (first, end) = ((k * rows).min(n), ((k + 1) * rows).min(n)); + packed_if(pack, generate(&ops.range(first, end), rows, form)) + }; + let cuts = padding.tables(n, rows); + // Written packed once the family's widths are known (`tables::gpack`): + // until then a cut is built at 8 bytes a cell, so the first goes alone and + // leaves its widths for the rest. + #[cfg(feature = "parallel")] + { + let mut tables = Vec::with_capacity(cuts); + while tables.len() < cuts { + let first = tables.len(); + let end = if written_packed(form) { + cuts + } else { + first + 1 + }; + tables.par_extend((first..end).into_par_iter().map(cut)); + } + Ok(tables) + } + #[cfg(not(feature = "parallel"))] + { + let _ = written_packed; + Ok((0..cuts).map(cut).collect()) + } +} + +/// The ops whose rows reach KECCAK_RND chunk `chunk` of `rows` rows (24 rows an +/// op), clamped to the `len` ops there are. +pub(crate) fn keccak_rnd_op_range(chunk: usize, rows: usize, len: usize) -> (usize, usize) { + let first = (chunk * rows / 24).min(len); + let end = ((chunk + 1) * rows).div_ceil(24).min(len); + (first, end) +} + +/// The slot a streamed chunk leaves in the final build: no rows, no columns. +pub(crate) fn streamed_placeholder() -> TraceTable { + TraceTable::from_columns_main(Vec::new(), 1) +} + +/// `generate`, each table it makes packed at the bytes its columns need as it +/// is made when `pack` ([`StreamSkip::pack`]); `generate` itself otherwise. +fn packed_by( + pack: bool, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> impl Fn(&[T]) -> TraceTable + Send + Sync { + move |ops| packed_if(pack, generate(ops)) +} + +/// `table`, packed ([`TraceTable::pack_main_narrow`]) when `pack`. +fn packed_if( + pack: bool, + mut table: TraceTable, +) -> TraceTable { + if pack { + table.pack_main_narrow(); + } + table +} + +/// [`chunk_and_generate`] / [`chunk_and_generate_optional`] (`optional`) with +/// the first `skip` chunks already handed out: those slots get placeholders +/// and are not generated again. `skip == 0` is the plain call. With `tails`, +/// `ops` holds only the ops past those chunks ([`StreamSkip::tails`]). +fn chunk_and_generate_skipping( + ops: &[T], + max_rows: usize, + skip: usize, + tails: bool, + optional: bool, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> Result>, Error> { + if skip == 0 { + return if optional { + chunk_and_generate_optional(ops, max_rows, generate) + } else { + chunk_and_generate(ops, max_rows, generate) + }; + } + let chunks: Vec<&[T]> = ops.chunks(max_rows).collect(); + let rest = if tails { + chunks + } else { + if skip > chunks.len() { + return Err(Error::Prover(format!( + "{skip} chunks were streamed but the run has {} of this table", + chunks.len() + ))); + } + chunks[skip..].to_vec() + }; + let mut tables: Vec<_> = (0..skip).map(|_| streamed_placeholder()).collect(); + tables.extend(generate_chunks(rest, generate)?); + Ok(tables) +} + +/// Chunk raw ops and generate one trace table per chunk, padding an empty `ops` +/// to a single chunk so the table is always present in the proof. +/// +/// For tables that may be omitted entirely, use [`chunk_and_generate_optional`]. +fn chunk_and_generate( + ops: &[T], + max_rows: usize, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> Result>, Error> { + let op_chunks: Vec<&[T]> = if ops.is_empty() { + vec![&[][..]] + } else { + ops.chunks(max_rows).collect() + }; + generate_chunks(op_chunks, generate) +} + +/// Like [`chunk_and_generate`], but an empty `ops` yields no table at all: the +/// chip is left out of the proof instead of costing a padded sub-proof. +/// +/// The empty case short-circuits rather than falling through to `ops.chunks`, +/// which panics on a zero chunk size even for an empty slice. `max_rows` comes +/// from a caller-supplied [`MaxRowsConfig`] whose fields are public. +/// +/// Sound because a chip contributes to the run only through its LogUp bus, and a +/// chip with no rows contributes zero. A prover that omits a table whose ops did +/// execute leaves its counterparty's sends unmatched, and the bus-balance check +/// over the tables that *are* present rejects the proof. See +/// `TableCounts::validate`. +fn chunk_and_generate_optional( + ops: &[T], + max_rows: usize, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> Result>, Error> { + let op_chunks: Vec<&[T]> = if ops.is_empty() { + vec![] + } else { + ops.chunks(max_rows).collect() + }; + generate_chunks(op_chunks, generate) +} + +/// Generate a single trace table for `ops`, or none at all when `ops` is empty. +/// +/// The accelerator chips are not chunked: one call means one table. What they do +/// share with the chunked chips is that an empty op list should cost nothing, so +/// this returns an empty `Vec` and the table drops out of the proof. Soundness +/// rests on the same LogUp argument as [`chunk_and_generate_optional`]. +fn generate_optional( + ops: &[T], + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> Result>, Error> { + let op_chunks: Vec<&[T]> = if ops.is_empty() { vec![] } else { vec![ops] }; + generate_chunks(op_chunks, generate) +} + +/// Generate one trace table per already-chunked op slice. +fn generate_chunks( + op_chunks: Vec<&[T]>, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> Result>, Error> { + generate_chunks_with( + op_chunks, + |chunk: &&[T], generate: &dyn Fn(&[T]) -> TraceTable<_, _>| generate(chunk), + generate, + ) +} + +/// [`generate_chunks`] over chunks that `view` hands to the generator as op +/// slices, each made only when its generation starts (a chunk that straddles +/// two segments is copied, or expanded, then, and freed with its table). +fn generate_chunks_with( + chunks: Vec, + view: impl Fn( + &C, + &dyn Fn(&[T]) -> TraceTable, + ) -> TraceTable + + Sync, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> Result>, Error> { + let generate = |chunk: C| view(&chunk, &generate); #[cfg(feature = "parallel")] - let tables = op_chunks.into_par_iter().map(generate).collect(); + let tables = chunks.into_par_iter().map(generate).collect(); #[cfg(not(feature = "parallel"))] - let tables = op_chunks.into_iter().map(generate).collect(); + let tables = chunks.into_iter().map(generate).collect(); + Ok(tables) +} + +/// A list kept as the segments it is the concatenation of, never concatenated +/// (phase 3's LT ops: the kept tail, the MEMW and MEMW_A prefixes and what the +/// finish derives, each a list of its own). +struct Segmented<'a, T> { + parts: Vec>, +} + +/// One segment of a [`Segmented`] list: ops in memory, or ops kept in a +/// compact form that expands any range of them ([`CompactLt`]). +enum Part<'a, T> { + Ops(&'a [T]), + Compact(&'a (dyn Expand + Sync)), +} + +impl Clone for Part<'_, T> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Part<'_, T> {} + +impl Part<'_, T> { + fn len(&self) -> usize { + match self { + Part::Ops(ops) => ops.len(), + Part::Compact(c) => c.len(), + } + } +} + +/// Ops kept in a compact form: how many, and any range of them expanded. +trait Expand { + fn len(&self) -> usize; + /// Ops `start..end`, appended to `out`. + fn expand_into(&self, start: usize, end: usize, out: &mut Vec); +} + +impl<'a, T: Clone> Segmented<'a, T> { + fn len(&self) -> usize { + self.parts.iter().map(Part::len).sum() + } + + /// Ops `start..end` of the concatenation, handed to `f` a slice at a time + /// with no copy of the range: an in-memory segment's ops as they lie, a + /// compact segment's expanded `block` ops at a time into one small buffer. + /// Returns whether [`Self::range`] would have copied them (they span + /// segments, or a compact one holds them). + fn for_each_slice( + &self, + start: usize, + end: usize, + block: usize, + f: &mut dyn FnMut(&[T]), + ) -> bool { + let (mut at, mut pieces, mut compact) = (0usize, 0usize, false); + let mut buf: Vec = Vec::new(); + for part in &self.parts { + let (lo, hi) = (at, at + part.len()); + at = hi; + if lo >= end { + break; + } + if hi <= start { + continue; + } + let (a, b) = (start.max(lo) - lo, end.min(hi) - lo); + pieces += 1; + match part { + Part::Ops(ops) => f(&ops[a..b]), + Part::Compact(c) => { + compact = true; + let mut k = a; + while k < b { + let e = (k + block.max(1)).min(b); + buf.clear(); + c.expand_into(k, e, &mut buf); + f(&buf); + k = e; + } + } + } + } + pieces > 1 || compact + } + + /// Ops `start..end` of the concatenation: borrowed when one in-memory + /// segment holds them, copied (or expanded) otherwise. + fn range(&self, start: usize, end: usize) -> std::borrow::Cow<'a, [T]> { + let mut at = 0usize; + let mut pieces: Vec<(Part<'a, T>, usize, usize)> = Vec::new(); + for part in &self.parts { + let (lo, hi) = (at, at + part.len()); + if hi > start && lo < end { + pieces.push((*part, start.max(lo) - lo, end.min(hi) - lo)); + } + at = hi; + if at >= end { + break; + } + } + match pieces.as_slice() { + [] => std::borrow::Cow::Borrowed(&[]), + [(Part::Ops(ops), lo, hi)] => std::borrow::Cow::Borrowed(&ops[*lo..*hi]), + _ => { + let mut out = Vec::with_capacity(end.saturating_sub(start)); + for (part, lo, hi) in pieces { + match part { + Part::Ops(ops) => out.extend_from_slice(&ops[lo..hi]), + Part::Compact(c) => c.expand_into(lo, hi, &mut out), + } + } + std::borrow::Cow::Owned(out) + } + } + } +} + +/// LT ops with `signed` and `invert` false — the timestamp checks phase 3 +/// derives from MEMW and MEMW_A ops — kept as a stream of zigzag LEB128 +/// deltas, `rhs` from the previous op's and `lhs` from its own `rhs`: a few +/// bytes an op instead of 24. Every [`COMPACT_LT_BLOCK`] ops a sparse index +/// records where the stream stands, so any range expands on its own. The +/// ops expand exactly as pushed, in order. The stream is stored in blocks of +/// at most [`blocks::BLOCK_BYTES`] that are never reallocated once full +/// ([`blocks`]); an op's two varints never straddle two blocks. +#[derive(Default)] +pub(crate) struct CompactLt { + bytes: Vec>, + /// (block, byte offset in it, previous `rhs`) at op `k * COMPACT_LT_BLOCK`. + index: Vec<(u32, u32, u64)>, + len: usize, + last_rhs: u64, + /// The ops as they are, 24 bytes each, instead of the stream (the A arm, + /// `LAMBDA_VM_BLOCK_COMPACT_LT=0` on the block). + raw: Option>, + /// Bytes a block of the stream holds; 0 is [`blocks::BLOCK_BYTES`] (tests + /// set a small one, to cross blocks). + block_bytes: usize, +} + +const COMPACT_LT_BLOCK: usize = 4096; + +/// The most bytes one compact LT op takes: two varints of at most ten. +const COMPACT_LT_OP_BYTES: usize = 20; + +impl CompactLt { + /// A list whose stream is cut into blocks of `bytes` (at least an op's). + #[cfg(test)] + pub(crate) fn with_block_bytes(bytes: usize) -> Self { + Self { + block_bytes: bytes.max(COMPACT_LT_OP_BYTES), + ..Self::default() + } + } + + /// Bytes a block of the stream holds. + fn block_bytes(&self) -> usize { + if self.block_bytes == 0 { + blocks::BLOCK_BYTES + } else { + self.block_bytes + } + } + + /// An empty list that keeps its ops as this one does (as they are, or as + /// the stream in blocks of the same bytes). + pub(crate) fn empty_like(&self) -> Self { + Self { + raw: self.raw.as_ref().map(|_| Vec::new()), + block_bytes: self.block_bytes, + ..Self::default() + } + } + + /// Whether it holds no op. + pub(crate) fn holds_none(&self) -> bool { + self.raw.as_ref().map_or(self.len == 0, Vec::is_empty) + } + + /// A list that keeps its ops as they are (the A arm of the compact form). + pub(crate) fn raw() -> Self { + Self { + raw: Some(Vec::new()), + ..Self::default() + } + } + + /// Append `ops` (each with `signed` and `invert` false). + pub(crate) fn extend(&mut self, ops: &[LtOperation]) { + if let Some(raw) = &mut self.raw { + raw.extend_from_slice(ops); + return; + } + fn put(out: &mut Vec, mut v: u64) { + while v >= 0x80 { + out.push(v as u8 | 0x80); + v >>= 7; + } + out.push(v as u8); + } + let zigzag = |d: u64| (d << 1) ^ ((d as i64 >> 63) as u64); + let block_bytes = self.block_bytes(); + for op in ops { + debug_assert!( + !op.signed && !op.invert, + "a compact LT op is unsigned, not inverted" + ); + // Room for the op in the last block: grown as a `Vec` grows up to a + // block's bytes, else a new block, allocated whole. + let room = self + .bytes + .last() + .is_some_and(|last| last.len() + COMPACT_LT_OP_BYTES <= block_bytes); + if !room { + let capacity = if self.bytes.is_empty() { + 4096.min(block_bytes) + } else { + block_bytes + }; + self.bytes.push(Vec::with_capacity(capacity)); + } + let block = self.bytes.len() - 1; + let Some(last) = self.bytes.last_mut() else { + continue; + }; + if last.capacity() - last.len() < COMPACT_LT_OP_BYTES { + let want = (last.capacity() * 2) + .min(block_bytes) + .max(last.len() + COMPACT_LT_OP_BYTES); + last.reserve_exact(want - last.len()); + } + if self.len.is_multiple_of(COMPACT_LT_BLOCK) { + self.index + .push((block as u32, last.len() as u32, self.last_rhs)); + } + put(last, zigzag(op.rhs.wrapping_sub(self.last_rhs))); + put(last, zigzag(op.rhs.wrapping_sub(op.lhs))); + self.last_rhs = op.rhs; + self.len += 1; + } + } + + /// The bytes it takes on the heap (capacities). + pub(crate) fn heap_bytes(&self) -> usize { + self.raw.as_ref().map_or(0, vec_heap_bytes) + + self.bytes.iter().map(Vec::capacity).sum::() + + self.bytes.capacity() * std::mem::size_of::>() + + self.index.capacity() * std::mem::size_of::<(u32, u32, u64)>() + } + + /// The bytes its largest single allocation takes. + pub(crate) fn largest_bytes(&self) -> usize { + let blocks = self.bytes.iter().map(Vec::capacity).max().unwrap_or(0); + let raw = self.raw.as_ref().map_or(0, vec_heap_bytes); + let index = self.index.capacity() * std::mem::size_of::<(u32, u32, u64)>(); + blocks.max(raw).max(index) + } +} + +impl Expand for CompactLt { + fn len(&self) -> usize { + self.raw.as_ref().map_or(self.len, Vec::len) + } + + fn expand_into(&self, start: usize, end: usize, out: &mut Vec) { + if let Some(raw) = &self.raw { + out.extend_from_slice(&raw[start.min(raw.len())..end.min(raw.len())]); + return; + } + let end = end.min(self.len); + if start >= end { + return; + } + let get = |bytes: &[u8], at: &mut usize| { + let (mut v, mut shift) = (0u64, 0u32); + loop { + let b = bytes[*at]; + *at += 1; + v |= u64::from(b & 0x7f) << shift; + if b < 0x80 { + return v; + } + shift += 7; + } + }; + let unzigzag = |z: u64| (z >> 1) ^ (z & 1).wrapping_neg(); + let first = start / COMPACT_LT_BLOCK; + let (block, at, mut prev) = self.index[first]; + let (mut block, mut at) = (block as usize, at as usize); + out.reserve(end - start); + for k in first * COMPACT_LT_BLOCK..end { + // An op lies in one block: the next op starts the next block once + // this one ends. + if at >= self.bytes[block].len() { + block += 1; + at = 0; + } + let bytes = &self.bytes[block]; + let rhs = prev.wrapping_add(unzigzag(get(bytes, &mut at))); + let lhs = rhs.wrapping_sub(unzigzag(get(bytes, &mut at))); + prev = rhs; + if k >= start { + out.push(LtOperation::new(lhs, rhs, false)); + } + } + } +} + +/// [`chunk_and_generate_skipping`] over a [`Segmented`] list: the same chunks +/// (whole multiples of `max_rows` of the concatenation, the same placeholders +/// and padding rules), each made from the segments only as it is generated. +fn chunk_and_generate_segmented( + ops: &Segmented<'_, T>, + max_rows: usize, + skip: usize, + tails: bool, + optional: bool, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, +) -> Result>, Error> { + let len = ops.len(); + let max_rows = max_rows.max(1); + let chunks: Vec<(usize, usize)> = (0..len.div_ceil(max_rows)) + .map(|k| (k * max_rows, ((k + 1) * max_rows).min(len))) + .collect(); + let (placeholders, chunks) = if skip == 0 { + // As `chunk_and_generate` / `chunk_and_generate_optional`: an empty + // list is one empty chunk, or no table at all when the table is + // optional. + let chunks = if len == 0 && !optional { + vec![(0, 0)] + } else { + chunks + }; + (0, chunks) + } else if tails { + (skip, chunks) + } else { + if skip > chunks.len() { + return Err(Error::Prover(format!( + "{skip} chunks were streamed but the run has {} of this table", + chunks.len() + ))); + } + (skip, chunks[skip..].to_vec()) + }; + let mut tables: Vec<_> = (0..placeholders).map(|_| streamed_placeholder()).collect(); + tables.extend(generate_chunks_with( + chunks, + |&(start, end): &(usize, usize), generate: &dyn Fn(&[T]) -> TraceTable<_, _>| { + generate(&ops.range(start, end)) + }, + generate, + )?); Ok(tables) } +/// What [`collect_all_ops`] derives from the CPU ops, kept as the SEGMENTS its +/// lists are concatenations of: a filter over every CPU op, then what CPU32 +/// dispatches, then what the DVRM ops imply (over the filter's DVRM ops, then +/// CPU32's). A run routed window by window appends each segment per window and +/// concatenates the segments at the end, which gives exactly the lists a +/// whole-run routing gives (the windowed builder). +#[derive(Default)] +struct RoutedSegments { + branch_ops: Vec, + mul_filter: Vec<(MulOperation, bool)>, + dvrm_filter: Vec<(DvrmOperation, bool)>, + eq_ops: Vec, + bytewise_ops: Vec, + store_ops: Vec, + shift_cpu32: Vec, + mul_cpu32: Vec<(MulOperation, bool)>, + dvrm_cpu32: Vec<(DvrmOperation, bool)>, + bitwise_cpu32: Vec, + lt_dvrm_filter: Vec, + lt_dvrm_cpu32: Vec, + mul_dvrm_filter: Vec<(MulOperation, bool)>, + mul_dvrm_cpu32: Vec<(MulOperation, bool)>, +} + +/// [`RoutedSegments`] accumulated over a run, window by window: each segment +/// a [`BlockVec`] in blocks of its table's chunk (or [`blocks::BLOCK_BYTES`]), +/// never reallocated as it grows. The same segments, in the same order. +struct RoutedBlocks { + /// Delta-coded ([`CompactBranch`]). + branch_ops: CompactBranch, + mul_filter: BlockVec<(MulOperation, bool)>, + dvrm_filter: BlockVec<(DvrmOperation, bool)>, + /// Delta-coded ([`CompactEq`]). + eq_ops: CompactEq, + /// Delta-coded ([`CompactBytewise`]). + bytewise_ops: CompactBytewise, + store_ops: BlockVec, + shift_cpu32: BlockVec, + mul_cpu32: BlockVec<(MulOperation, bool)>, + dvrm_cpu32: BlockVec<(DvrmOperation, bool)>, + bitwise_cpu32: BlockVec, + lt_dvrm_filter: BlockVec, + lt_dvrm_cpu32: BlockVec, + mul_dvrm_filter: BlockVec<(MulOperation, bool)>, + mul_dvrm_cpu32: BlockVec<(MulOperation, bool)>, +} + +impl RoutedBlocks { + /// Empty segments, each in blocks of its table's chunk at `m`. + fn new(m: &super::MaxRowsConfig) -> Self { + fn list(chunk: usize) -> BlockVec { + BlockVec::new(block_len::(chunk)) + } + Self { + branch_ops: CompactBranch::default(), + mul_filter: list(m.mul), + dvrm_filter: list(m.dvrm), + eq_ops: CompactEq::default(), + bytewise_ops: CompactBytewise::default(), + store_ops: list(m.store), + shift_cpu32: list(m.shift), + mul_cpu32: list(m.mul), + dvrm_cpu32: list(m.dvrm), + bitwise_cpu32: list(usize::MAX), + lt_dvrm_filter: list(m.lt), + lt_dvrm_cpu32: list(m.lt), + mul_dvrm_filter: list(m.mul), + mul_dvrm_cpu32: list(m.mul), + } + } + + /// Appends a later window's segments, segment by segment. + fn append(&mut self, other: RoutedSegments) { + self.branch_ops.extend(&other.branch_ops); + self.mul_filter.extend(other.mul_filter); + self.dvrm_filter.extend(other.dvrm_filter); + self.eq_ops.extend(&other.eq_ops); + self.bytewise_ops.extend(&other.bytewise_ops); + self.store_ops.extend(other.store_ops); + self.shift_cpu32.extend(other.shift_cpu32); + self.mul_cpu32.extend(other.mul_cpu32); + self.dvrm_cpu32.extend(other.dvrm_cpu32); + self.bitwise_cpu32.extend(other.bitwise_cpu32); + self.lt_dvrm_filter.extend(other.lt_dvrm_filter); + self.lt_dvrm_cpu32.extend(other.lt_dvrm_cpu32); + self.mul_dvrm_filter.extend(other.mul_dvrm_filter); + self.mul_dvrm_cpu32.extend(other.mul_dvrm_cpu32); + } + + /// Each segment's (name, heap bytes, largest single allocation). + fn sizes(&self) -> [(&'static str, usize, usize); 14] { + fn of(name: &'static str, list: &BlockVec) -> (&'static str, usize, usize) { + (name, list.heap_bytes(), list.largest_bytes()) + } + [ + ( + "branch", + self.branch_ops.heap_bytes(), + self.branch_ops.largest_bytes(), + ), + of("mul_filter", &self.mul_filter), + of("dvrm_filter", &self.dvrm_filter), + ("eq", self.eq_ops.heap_bytes(), self.eq_ops.largest_bytes()), + ( + "bytewise", + self.bytewise_ops.heap_bytes(), + self.bytewise_ops.largest_bytes(), + ), + of("store", &self.store_ops), + of("shift_cpu32", &self.shift_cpu32), + of("mul_cpu32", &self.mul_cpu32), + of("dvrm_cpu32", &self.dvrm_cpu32), + of("bitwise_cpu32", &self.bitwise_cpu32), + of("lt_dvrm_filter", &self.lt_dvrm_filter), + of("lt_dvrm_cpu32", &self.lt_dvrm_cpu32), + of("mul_dvrm_filter", &self.mul_dvrm_filter), + of("mul_dvrm_cpu32", &self.mul_dvrm_cpu32), + ] + } + + /// The bytes its segments take on the heap. + fn heap_bytes(&self) -> usize { + self.sizes().iter().map(|&(_, bytes, _)| bytes).sum() + } + + /// [`Self::heap_bytes`] segment by segment, each named `segments {name}`. + fn heap_parts(&self) -> Vec<(String, usize)> { + self.sizes() + .iter() + .map(|&(name, bytes, _)| (format!("segments {name}"), bytes)) + .collect() + } + + /// Each segment's largest single allocation, named `segments {name}`. + fn largest_parts(&self) -> Vec<(String, usize)> { + self.sizes() + .iter() + .map(|&(name, _, largest)| (format!("segments {name}"), largest)) + .collect() + } +} + +/// The routing of [`collect_all_ops`], segment by segment ([`RoutedSegments`]), +/// each CPU op read with `decode`. +fn route_ops( + cpu_ops: &[CpuOperation], + decode: &DecodeTable, + cpu32_ops: &[cpu32::Cpu32Operation], +) -> RoutedSegments { + let ops = || cpu_ops.iter().map(|op| decode.op(op)); + // Collect BRANCH operations from CPU ops where branch_cond = true + let branch_ops: Vec = ops() + .filter(|op| op.branch_cond()) + .map(route_branch) + .collect(); + + // Collect MUL operations from non-word MUL instructions. lhs_signed = `signed` + // (alu_flags bit 5); rhs_signed = `signed2` (bit 6); wants_hi = `muldiv` (bit 7). + let mul_filter: Vec<(MulOperation, bool)> = ops() + .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_mul()) + .map(route_mul) + .collect(); + + // Collect DVRM operations from non-word DIV/REM instructions. + let dvrm_filter: Vec<(DvrmOperation, bool)> = ops() + .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_divrem()) + .map(route_dvrm) + .collect(); + + // Collect the ALU/MEMORY chip ops (non-word rows). + // EQ: BEQ/BNE (invert = alu_flags bit 6). BYTEWISE: AND/OR/XOR (op = alu_op). + let eq_ops: Vec = ops() + .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_eq()) + .map(route_eq) + .collect(); + let bytewise_ops: Vec = ops() + .filter(|op| { + let f = &op.decode.fields; + !f.word_instr && (f.is_and() || f.is_or() || f.is_xor()) + }) + .map(route_bytewise) + .collect(); + // STORE: receives MEMORY(memory_op=1) from the CPU and sends the MEMW write + // at timestamp+1 (mirrors `collect_store_op_from_cpu`, which records the MEMW + // table row). + let store_ops: Vec = ops() + .filter(|op| op.decode.fields.is_store()) + .map(route_store) + .collect(); + + route_from_cpu_segments( + branch_ops, + mul_filter, + dvrm_filter, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ) +} + +/// [`route_ops`] in one pass over the CPU ops instead of one per segment: each +/// op goes to every segment whose filter it passes, in CPU order, which is each +/// filter's order — the same segments. +fn route_ops_one_pass( + cpu_ops: &[CpuOperation], + decode: &DecodeTable, + cpu32_ops: &[cpu32::Cpu32Operation], +) -> RoutedSegments { + let mut branch_ops = Vec::new(); + let mut mul_filter = Vec::new(); + let mut dvrm_filter = Vec::new(); + let mut eq_ops = Vec::new(); + let mut bytewise_ops = Vec::new(); + let mut store_ops = Vec::new(); + for op in cpu_ops { + let op = decode.op(op); + let f = &op.decode.fields; + if op.branch_cond() { + branch_ops.push(route_branch(op)); + } + if !f.word_instr { + if f.is_mul() { + mul_filter.push(route_mul(op)); + } + if f.is_divrem() { + dvrm_filter.push(route_dvrm(op)); + } + if f.is_eq() { + eq_ops.push(route_eq(op)); + } + if f.is_and() || f.is_or() || f.is_xor() { + bytewise_ops.push(route_bytewise(op)); + } + } + if f.is_store() { + store_ops.push(route_store(op)); + } + } + route_from_cpu_segments( + branch_ops, + mul_filter, + dvrm_filter, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ) +} + +/// [`route_ops_one_pass`] appended to the run's `segments` in place (no +/// window's segments to copy in), with each CPU op's BITWISE lookups counted +/// into `histogram` in the same pass ([`CpuOp::count_bitwise_into`]): the same +/// segments `segments.append(route_ops_one_pass(..))` gives, and the same +/// counts. CPU32's dispatch goes through lists of the window's own (few ops), +/// and the DVRM-implied LT and MUL ops are derived from the window's part of +/// each DVRM segment. +fn route_ops_into( + cpu_ops: &[CpuOperation], + decode: &DecodeTable, + cpu32_ops: &[cpu32::Cpu32Operation], + segments: &mut RoutedBlocks, + histogram: &mut bitwise::BitwiseHistogram, +) { + let dvrm_before = segments.dvrm_filter.len(); + for op in cpu_ops { + let op = decode.op(op); + op.count_bitwise_into(histogram); + let f = &op.decode.fields; + if op.branch_cond() { + segments + .branch_ops + .extend(std::slice::from_ref(&route_branch(op))); + } + if !f.word_instr { + if f.is_mul() { + segments.mul_filter.push(route_mul(op)); + } + if f.is_divrem() { + segments.dvrm_filter.push(route_dvrm(op)); + } + if f.is_eq() { + segments.eq_ops.extend(std::slice::from_ref(&route_eq(op))); + } + if f.is_and() || f.is_or() || f.is_xor() { + segments + .bytewise_ops + .extend(std::slice::from_ref(&route_bytewise(op))); + } + } + if f.is_store() { + segments.store_ops.push(route_store(op)); + } + } + // As `route_from_cpu_segments`, over this window's part of each list. + let (mut shift_cpu32, mut mul_cpu32, mut dvrm_cpu32) = (Vec::new(), Vec::new(), Vec::new()); + for c in cpu32_ops { + cpu32_chip_op(c, &mut shift_cpu32, &mut mul_cpu32, &mut dvrm_cpu32); + segments.bitwise_cpu32.extend(collect_cpu32_bitwise(c)); + } + let dvrm_lt = |dvrm: &[(DvrmOperation, bool)], out: &mut BlockVec| { + out.extend( + dvrm.iter() + .map(|(op, _wants_remainder)| LtOperation::new(op.abs_r(), op.abs_d(), false)), + ) + }; + let dvrm_mul = |dvrm: &[(DvrmOperation, bool)], out: &mut BlockVec<(MulOperation, bool)>| { + for (op, _wants_remainder) in dvrm { + let mul_op = MulOperation::new(op.d, op.signed, op.compute_quotient(), op.sign_q()); + out.push((mul_op.clone(), false)); // C13: lo (muldiv_selector=0) + out.push((mul_op, true)); // C14: hi (muldiv_selector=1) + } + }; + let dvrm_filter = segments + .dvrm_filter + .range(dvrm_before, segments.dvrm_filter.len()); + dvrm_lt(&dvrm_filter, &mut segments.lt_dvrm_filter); + dvrm_lt(&dvrm_cpu32, &mut segments.lt_dvrm_cpu32); + dvrm_mul(&dvrm_filter, &mut segments.mul_dvrm_filter); + dvrm_mul(&dvrm_cpu32, &mut segments.mul_dvrm_cpu32); + drop(dvrm_filter); + segments.shift_cpu32.extend(shift_cpu32); + segments.mul_cpu32.extend(mul_cpu32); + segments.dvrm_cpu32.extend(dvrm_cpu32); +} + +// [`route_ops`]'s op for each segment it filters out of the CPU ops. +fn route_branch(op: CpuOp<'_>) -> BranchOperation { + BranchOperation::new( + op.decode.pc, + op.decode.imm, // offset as full 64-bit DWordWL (already sign-extended) + op.rv1, // register value must match the CPU's BRANCH bus signature + op.decode.fields.jalr(), + ) +} +fn route_mul(op: CpuOp<'_>) -> (MulOperation, bool) { + let f = op.decode.fields; + ( + MulOperation::new(op.rv1, f.alu_signed(), op.arg2(), f.alu_signed2_or_invert()), + f.alu_muldiv(), + ) +} +fn route_dvrm(op: CpuOp<'_>) -> (DvrmOperation, bool) { + let f = op.decode.fields; + ( + DvrmOperation::new(op.rv1, op.arg2(), f.alu_signed()), + f.alu_muldiv(), + ) +} +fn route_eq(op: CpuOp<'_>) -> eq::EqOperation { + eq::EqOperation::new(op.rv1, op.arg2(), op.decode.fields.alu_signed2_or_invert()) +} +fn route_bytewise(op: CpuOp<'_>) -> bytewise::BytewiseOperation { + bytewise::BytewiseOperation::new(op.rv1, op.arg2(), op.decode.fields.alu_op()) +} +fn route_store(op: CpuOp<'_>) -> store::StoreOperation { + // The MEMORY bus and the STORE chip's MEMW write share the base + // timestamp (spec store.toml uses one `timestamp` for both). + store::StoreOperation::new( + op.res, + op.timestamp, + op.rv2, + op.decode.fields.mem_bytes() as u8, + ) +} + +/// The rest of [`route_ops`] once the CPU ops' segments are filtered: CPU32's +/// dispatch and what the DVRM ops imply. +fn route_from_cpu_segments( + branch_ops: Vec, + mul_filter: Vec<(MulOperation, bool)>, + dvrm_filter: Vec<(DvrmOperation, bool)>, + eq_ops: Vec, + bytewise_ops: Vec, + store_ops: Vec, + cpu32_ops: &[cpu32::Cpu32Operation], +) -> RoutedSegments { + // CPU32 (word `*W`) dispatch: each CPU32 row that uses the full ALU sends to + // the SHIFT/MUL/DVRM chips (ADDW/SUBW are the CPU32 ADD/SUB fast-path). These + // word DVRM ops are added before the DVRM→LT/MUL loops so they get their own + // internal consistency lookups. CPU32 also sends its own BITWISE range checks. + let mut shift_cpu32 = Vec::new(); + let mut mul_cpu32 = Vec::new(); + let mut dvrm_cpu32 = Vec::new(); + let mut bitwise_cpu32 = Vec::new(); + for c in cpu32_ops { + cpu32_chip_op(c, &mut shift_cpu32, &mut mul_cpu32, &mut dvrm_cpu32); + bitwise_cpu32.extend(collect_cpu32_bitwise(c)); + } + + // Collect LT operations from DVRM: |r| < |d| (unsigned comparison) + let lt_of = |dvrm: &[(DvrmOperation, bool)]| -> Vec { + dvrm.iter() + .map(|(op, _wants_remainder)| LtOperation::new(op.abs_r(), op.abs_d(), false)) + .collect() + }; + let lt_dvrm_filter = lt_of(&dvrm_filter); + let lt_dvrm_cpu32 = lt_of(&dvrm_cpu32); + + // Collect MUL operations from DVRM: d * q = n_sub_r (C13 lo, C14 hi) + let mul_of = |dvrm: &[(DvrmOperation, bool)]| -> Vec<(MulOperation, bool)> { + let mut out = Vec::with_capacity(2 * dvrm.len()); + for (op, _wants_remainder) in dvrm { + let d = op.d; + let d_signed = op.signed; + let q = op.compute_quotient(); + let q_signed = op.sign_q(); + let mul_op = MulOperation::new(d, d_signed, q, q_signed); + out.push((mul_op.clone(), false)); // C13: lo (muldiv_selector=0) + out.push((mul_op, true)); // C14: hi (muldiv_selector=1) + } + out + }; + let mul_dvrm_filter = mul_of(&dvrm_filter); + let mul_dvrm_cpu32 = mul_of(&dvrm_cpu32); + + RoutedSegments { + branch_ops, + mul_filter, + dvrm_filter, + eq_ops, + bytewise_ops, + store_ops, + shift_cpu32, + mul_cpu32, + dvrm_cpu32, + bitwise_cpu32, + lt_dvrm_filter, + lt_dvrm_cpu32, + mul_dvrm_filter, + mul_dvrm_cpu32, + } +} + /// Phase 2: Collect and route all operations from CPU ops. /// /// Takes the raw output of `collect_ops_from_cpu` plus `register_state` @@ -3333,17 +7694,21 @@ fn generate_chunks( #[allow(clippy::too_many_arguments)] fn collect_all_ops( cpu_ops: Vec, + decode: &DecodeTable, mut memw: MemwBuckets, load_ops: Vec, - mut lt_ops: Vec, - mut shift_ops: Vec, - mut bitwise_ops: Vec, + lt_ops: Vec, + shift_ops: Vec, + bitwise_ops: Vec, keccak_ops: Vec, + blake3_ops: Vec, + blake3_absorb_ops: Vec, cpu32_ops: Vec, ecsm_ops: Vec, ecdas_ops: Vec, copy_ops: Vec, hint_ops: Vec, + ecrec_ops: Vec, register_state: &mut RegisterState, is_final: bool, ) -> CollectedOps { @@ -3367,130 +7732,215 @@ fn collect_all_ops( general: memw_ops, } = memw; - // Collect BRANCH operations from CPU ops where branch_cond = true - let branch_ops: Vec = cpu_ops - .iter() - .filter(|op| op.branch_cond) - .map(|op| { - BranchOperation::new( - op.decode.pc, - op.decode.imm, // offset as full 64-bit DWordWL (already sign-extended) - op.rv1, // register value must match the CPU's BRANCH bus signature - op.decode.fields.jalr(), - ) - }) - .collect(); + let RoutedSegments { + branch_ops, + mul_filter, + dvrm_filter, + eq_ops, + bytewise_ops, + store_ops, + shift_cpu32, + mul_cpu32, + dvrm_cpu32, + bitwise_cpu32, + lt_dvrm_filter, + lt_dvrm_cpu32, + mul_dvrm_filter, + mul_dvrm_cpu32, + } = route_ops(&cpu_ops, decode, &cpu32_ops); + // The lists are the segments in the order the routing always produced them: + // each CPU-op filter, then what CPU32 dispatches, then what DVRM implies, + // each segment a block of its own (no copies). + let shift_ops = BlockVec::from_vec(shift_ops).chain(BlockVec::from_vec(shift_cpu32)); + let bitwise_ops = BlockVec::from_vec(bitwise_ops).chain(BlockVec::from_vec(bitwise_cpu32)); + let mul_ops = BlockVec::from_vec(mul_filter) + .chain(BlockVec::from_vec(mul_cpu32)) + .chain(BlockVec::from_vec(mul_dvrm_filter)) + .chain(BlockVec::from_vec(mul_dvrm_cpu32)); + let dvrm_ops = BlockVec::from_vec(dvrm_filter).chain(BlockVec::from_vec(dvrm_cpu32)); + let lt_ops = BlockVec::from_vec(lt_ops) + .chain(BlockVec::from_vec(lt_dvrm_filter)) + .chain(BlockVec::from_vec(lt_dvrm_cpu32)); - // Collect MUL operations from non-word MUL instructions. lhs_signed = `signed` - // (alu_flags bit 5); rhs_signed = `signed2` (bit 6); wants_hi = `muldiv` (bit 7). - let mut mul_ops: Vec<(MulOperation, bool)> = cpu_ops - .iter() - .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_mul()) - .map(|op| { - let f = op.decode.fields; - ( - MulOperation::new(op.rv1, f.alu_signed(), op.arg2, f.alu_signed2_or_invert()), - f.alu_muldiv(), - ) - }) - .collect(); + CollectedOps { + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + branch_ops: CompactBranch::from_ops(&branch_ops), + mul_ops, + dvrm_ops, + keccak_ops: BlockVec::from_vec(keccak_ops), + blake3_ops: BlockVec::from_vec(blake3_ops), + blake3_absorb_ops: BlockVec::from_vec(blake3_absorb_ops), + eq_ops: CompactEq::from_ops(&eq_ops), + bytewise_ops: CompactBytewise::from_ops(&bytewise_ops), + store_ops: BlockVec::from_vec(store_ops), + cpu32_ops: BlockVec::from_vec(cpu32_ops), + ecsm_ops: BlockVec::from_vec(ecsm_ops), + ecdas_ops: BlockVec::from_vec( + ecdas_ops + .iter() + .map(ecdas::CompactEcdasOp::from_op) + .collect(), + ), + copy_ops: BlockVec::from_vec(copy_ops), + hint_ops: BlockVec::from_vec(hint_ops), + ecrec_ops: BlockVec::from_vec(ecrec_ops), + } +} - // Collect DVRM operations from non-word DIV/REM instructions. - let mut dvrm_ops: Vec<(DvrmOperation, bool)> = cpu_ops - .iter() - .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_divrem()) - .map(|op| { - let f = op.decode.fields; - ( - DvrmOperation::new(op.rv1, op.arg2, f.alu_signed()), - f.alu_muldiv(), - ) - }) - .collect(); +/// When each phase of a trace build ended, and when each table type's phase-5 +/// generator finished — seconds since [`build_stamps::start`]. Off (one atomic +/// load a mark) unless the block prover turns it on for its readout; the +/// epoch pipeline never does. +/// The last pool count's `BLOCK P4 ROWS` line ([`RowCounts::line`]), for the +/// block's log to print once the finish is done. +static ROW_SCAN_LINE: std::sync::Mutex> = std::sync::Mutex::new(None); - // Collect the ALU/MEMORY chip ops (non-word rows). - // EQ: BEQ/BNE (invert = alu_flags bit 6). BYTEWISE: AND/OR/XOR (op = alu_op). - let eq_ops: Vec = cpu_ops - .iter() - .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_eq()) - .map(|op| eq::EqOperation::new(op.rv1, op.arg2, op.decode.fields.alu_signed2_or_invert())) - .collect(); - let bytewise_ops: Vec = cpu_ops - .iter() - .filter(|op| { - let f = &op.decode.fields; - !f.word_instr && (f.is_and() || f.is_or() || f.is_xor()) - }) - .map(|op| bytewise::BytewiseOperation::new(op.rv1, op.arg2, op.decode.fields.alu_op())) - .collect(); - // STORE: receives MEMORY(memory_op=1) from the CPU and sends the MEMW write - // at timestamp+1 (mirrors `collect_store_op_from_cpu`, which records the MEMW - // table row). - let store_ops: Vec = cpu_ops - .iter() - .filter(|op| op.decode.fields.is_store()) - .map(|op| { - // The MEMORY bus and the STORE chip's MEMW write share the base - // timestamp (spec store.toml uses one `timestamp` for both). - store::StoreOperation::new( - op.res, - op.timestamp, - op.rv2, - op.decode.fields.mem_bytes() as u8, - ) - }) - .collect(); +/// Adds `secs` to `name`'s seconds in `by` (the order is first seen). +#[cfg(feature = "parallel")] +fn add_source_seconds(by: &mut Vec<(&'static str, f64)>, name: &'static str, secs: f64) { + match by.iter_mut().find(|(n, _)| *n == name) { + Some((_, total)) => *total += secs, + None => by.push((name, secs)), + } +} - // CPU32 (word `*W`) dispatch: each CPU32 row that uses the full ALU sends to - // the SHIFT/MUL/DVRM chips (ADDW/SUBW are the CPU32 ADD/SUB fast-path). These - // word DVRM ops are added before the DVRM→LT/MUL loops so they get their own - // internal consistency lookups. CPU32 also sends its own BITWISE range checks. - for c in &cpu32_ops { - cpu32_chip_op(c, &mut shift_ops, &mut mul_ops, &mut dvrm_ops); - bitwise_ops.extend(collect_cpu32_bitwise(c)); +/// Starts a `BLOCK P4 ROWS` line with `line`, for the walls to be added to. +#[cfg(feature = "parallel")] +fn start_row_scan_line(line: String) { + if let Ok(mut slot) = ROW_SCAN_LINE.lock() { + *slot = Some(line); + } +} + +/// Adds `more` to the last pool count's `BLOCK P4 ROWS` line, if there is one. +#[cfg(feature = "parallel")] +fn add_to_row_scan_line(more: &str) { + if let Ok(mut line) = ROW_SCAN_LINE.lock() + && let Some(line) = line.as_mut() + { + line.push_str(more); + } +} + +/// Takes the last pool count's `BLOCK P4 ROWS` line, if one ran since. +pub(crate) fn take_row_scan_line() -> Option { + ROW_SCAN_LINE.lock().ok().and_then(|mut line| line.take()) +} + +/// The last BITWISE table's `BLOCK BITWISE` line (its multiplicities' digest), +/// for the block's log, while a build records its marks. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +static BITWISE_LINE: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// Notes the BITWISE multiplicities `histogram` for [`take_bitwise_line`], +/// while a build records its marks (the block's readout). +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +fn note_bitwise(histogram: &bitwise::BitwiseHistogram, from_tables: bool) { + if build_stamps::since_start().is_none() { + return; + } + let line = format!( + "BLOCK BITWISE: multiplicities digest {:016x} · LT's, BRANCH's and BYTEWISE's lookups \ + counted {}", + histogram.digest(), + if from_tables { + "from their tables' rows" + } else { + "from their ops" + } + ); + if let Ok(mut slot) = BITWISE_LINE.lock() { + *slot = Some(line); + } +} + +/// Takes the last `BLOCK BITWISE` line, if a BITWISE table was built since. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +pub(crate) fn take_bitwise_line() -> Option { + BITWISE_LINE.lock().ok().and_then(|mut line| line.take()) +} + +pub(crate) mod build_stamps { + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Instant; + + /// A build's start and its marks, `(label, seconds since the start)`. + type Marks = (Instant, Vec<(String, f64)>); + + static ON: AtomicBool = AtomicBool::new(false); + static LOG: Mutex> = Mutex::new(None); + /// Called with each mark's label while a build records: a memory log's + /// line at each phase end (`LAMBDA_VM_BLOCK_MEMLOG`). + static HOOK: Mutex> = Mutex::new(None); + + pub(crate) type Hook = std::sync::Arc; + + /// Sets (or, `None`, clears) the hook every mark calls. + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn set_hook(hook: Option) { + if let Ok(mut slot) = HOOK.lock() { + *slot = hook; + } + } + + /// Starts recording, clearing what an earlier build left. + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn start() { + if let Ok(mut log) = LOG.lock() { + *log = Some((Instant::now(), Vec::new())); + } + ON.store(true, Ordering::Relaxed); } - // Collect LT operations from DVRM: |r| < |d| (unsigned comparison) - for (op, _wants_remainder) in &dvrm_ops { - lt_ops.push(LtOperation::new(op.abs_r(), op.abs_d(), false)); + pub(crate) fn mark(label: &str) { + if !ON.load(Ordering::Relaxed) { + return; + } + if let Ok(mut log) = LOG.lock() + && let Some((t0, marks)) = log.as_mut() + { + let at = t0.elapsed().as_secs_f64(); + marks.push((label.to_string(), at)); + } + let hook = HOOK.lock().ok().and_then(|slot| slot.clone()); + if let Some(hook) = hook { + hook(label); + } } - // Collect MUL operations from DVRM: d * q = n_sub_r (C13 lo, C14 hi) - for (op, _wants_remainder) in &dvrm_ops { - let d = op.d; - let d_signed = op.signed; - let q = op.compute_quotient(); - let q_signed = op.sign_q(); - let mul_op = MulOperation::new(d, d_signed, q, q_signed); - mul_ops.push((mul_op.clone(), false)); // C13: lo (muldiv_selector=0) - mul_ops.push((mul_op, true)); // C14: hi (muldiv_selector=1) + /// Seconds since the build's start while it records, else `None`. + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn since_start() -> Option { + if !ON.load(Ordering::Relaxed) { + return None; + } + LOG.lock() + .ok() + .and_then(|log| log.as_ref().map(|(t0, _)| t0.elapsed().as_secs_f64())) } - CollectedOps { - cpu_ops, - memw_ops, - memw_aligned_ops, - memw_register_rows, - load_ops, - lt_ops, - shift_ops, - bitwise_ops, - branch_ops, - mul_ops, - dvrm_ops, - keccak_ops, - eq_ops, - bytewise_ops, - store_ops, - cpu32_ops, - ecsm_ops, - ecdas_ops, - hint_ops, - copy_ops, + /// Stops recording and hands back the marks. + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + pub(crate) fn take() -> Vec<(String, f64)> { + ON.store(false, Ordering::Relaxed); + LOG.lock() + .ok() + .and_then(|mut log| log.take()) + .map(|(_, marks)| marks) + .unwrap_or_default() } } -/// Phases 3-5: From routed ops, produce all traces and assemble `Traces`. +/// Phases 3-5: From routed ops, produce all traces and assemble `Traces`: the +/// finish's plan ([`finish::FinishPlan`]) and every table it counted. /// /// `initial_image` controls PAGE table generation: `Some(image)` generates real /// PAGE tables and PAGE bitwise lookups seeded from the initial-memory image; @@ -3502,999 +7952,180 @@ fn build_traces( memory_state: &MemoryState, register_init: &[u32], decode_trace: TraceTable, - decode_pc_to_row: &decode::PcToRow, - mut register_state: RegisterState, + decode_table: &DecodeTable, + register_state: RegisterState, max_rows: &super::MaxRowsConfig, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, private_input: &[u8], is_final: bool, l2g_memory_bookend: bool, + skip: &StreamSkip, + pre: Option, ) -> Result { - let CollectedOps { - cpu_ops, - memw_ops, - memw_aligned_ops, - memw_register_rows, - load_ops, - mut lt_ops, - shift_ops, - bitwise_ops, - branch_ops, - mul_ops, - dvrm_ops, - keccak_ops, - eq_ops, - bytewise_ops, - store_ops, - cpu32_ops, - ecsm_ops, - ecdas_ops, - copy_ops, - hint_ops, - } = ops; - - // ===================================================================== - // PHASE 3: MEMW → LT (timestamp ordering and overflow checks) - // ===================================================================== - lt_ops.extend(collect_lt_from_memw(&memw_ops)); - lt_ops.extend(collect_lt_from_memw_aligned(&memw_aligned_ops)); - // COPY sends no ALU lookups any more. The `count < 8` check on wide rows and the - // per-ecall `count <= 256` bound are both gone: a chain that overshoots can no - // longer reach a legal `end`, so the termination argument subsumes them. The - // executor still refuses an ecall above 256 bytes, which is where the guest ABI - // needs that bound. - // HINT range-checks: selector < 3 and both address low limbs < 2^32 - 31 (matching - // the executor's HintUnknownSelector / HintAddressOverflow rejections). Three LT ops - // per hint call; the HINT table sends the matching ALU LT interactions. - lt_ops.extend(hint_ops.iter().flat_map(|op| { - [ - LtOperation::new(op.hint_id, hint::HINT_SELECTOR_BOUND, false), - LtOperation::new(op.in_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), - LtOperation::new(op.out_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), - ] - })); - - // ===================================================================== - // PHASE 4: All → Bitwise lookups - // ===================================================================== - #[cfg(feature = "instruments")] - let __sp = stark::instruments::span("p4_bitwise_collect"); - - // The committed bytes flow through the COPY chip, so the public output is read - // off its `write` rows. A zero-length `write` contributes a row that moves no - // bytes; `count != 0` is exactly the `mu - count_borrows[1]` gate the AIR uses. - let public_output_bytes: Vec = { - let mut rows: Vec<©::CopyOperation> = copy_ops - .iter() - .filter(|op| op.functionality == copy::Functionality::Write && op.count != 0) - .collect(); - // `dst` is the COMMIT-domain address, i.e. the running global byte index. - rows.sort_by_key(|op| op.dst); - rows.iter() - .flat_map(|op| op.value[..op.width as usize].iter().copied()) - .collect() - }; - - // CPU padding rows send ARE_BYTES with all-zero values. - // Add corresponding ops so the bitwise table multiplicities balance. - let num_padding_rows: usize = cpu_ops - .chunks(max_rows.cpu) - .map(|chunk| chunk.len().next_power_of_two().max(4) - chunk.len()) - .sum(); - - // The per-source bitwise collectors are all pure functions of their inputs, and the - // BITWISE multiplicities are order-independent (they ride a permutation-invariant bus), - // so every source can be collected in parallel and the per-worker histograms summed in - // any order. - // - // LT/MUL/DVRM/BRANCH/BYTEWISE count their per-row lookups (μ bound, bit-gated - // MSB16/ZERO) and SHIFT its padding rows PER CHIP INSTANCE, so pass the same chunk size - // used to split them into instances so multiplicities match the per-instance sends. - // MEMW_R sends IS_HALFWORD[timestamp_0 - old_timestamp_lo - 1]. PAGE does a - // batched ARE_BYTES[init, fini] per row (skipped in continuation epochs, which the L2G - // table owns). COPY sends IsHalfword; KECCAK_RND sends XOR/AND/ARE_BYTES; - // HINT sends ARE_BYTES for its 32 output cells. - // We never concatenate the lookups into one giant `Vec` (~140 M ops / - // ~560 MB at 10-tx whose only consumer is the multiplicity count). Each collector bumps - // the `BitwiseHistogram` it is handed: the heavy sources (MEMW_R one-per-row, PAGE - // one-per-byte, padding) count directly with no per-source Vec at all, and the small - // sources fold their transient `collect_*` Vec in and drop it. The histogram is a - // commutative monoid, so per-worker histograms tree-reduce to multiplicities that are - // independent of accumulation order. - // IS_WH rows from the `IS_WH` sends of every `IS_WHH` caller (today only KECCAK's - // state_ptr), deduplicated once and shared by the BITWISE collector below and the - // IS_WH trace in phase 5, so the two read the same rows by construction. - let is_wh_rows = is_wh::dedup_is_wh_rows(keccak::is_wh_keys(&keccak_ops)); - - type Collector<'a> = Box; - let lt_chunk = max_rows.lt; - let branch_chunk = max_rows.branch; - let mul_chunk = max_rows.mul; - let dvrm_chunk = max_rows.dvrm; - let bytewise_chunk = max_rows.bytewise; - let shift_chunk = max_rows.shift; - // Every source except the two dominant ones (the in-walk lookups and MEMW_R, which are - // split into row-ranges in the parallel path below) stays a single whole-source collector. - let mut collectors: Vec = vec![ - Box::new(|h| h.add_ops(&collect_bitwise_from_lt(<_ops, lt_chunk))), - Box::new(|h| h.add_ops(&collect_bitwise_from_mul(&mul_ops, mul_chunk))), - Box::new(|h| h.add_ops(&collect_bitwise_from_dvrm(&dvrm_ops, dvrm_chunk))), - Box::new(|h| h.add_ops(&collect_bitwise_from_branch(&branch_ops, branch_chunk))), - Box::new(|h| { - h.add_ops(&shift::collect_bitwise_from_shift(&shift_ops)); - h.bump_n( - BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, 0, 0), - shift_padding_are_bytes(&shift_ops, shift_chunk), - ); - }), - Box::new(|h| { - for op in &bytewise_ops { - h.add_ops(&op.collect_bitwise_ops()); - } - h.add_ops(&collect_bytewise_mu_bounds(&bytewise_ops, bytewise_chunk)); - }), - Box::new(|h| { - for op in &eq_ops { - h.add_ops(&op.collect_bitwise_ops()); - } - }), - Box::new(|h| { - for op in &store_ops { - h.add_ops(&op.collect_bitwise_ops()); - } - }), - Box::new(|h| h.add_ops(&collect_bitwise_from_memw_aligned(&memw_aligned_ops))), - Box::new(|h| h.add_ops(&collect_bitwise_from_copy(©_ops))), - Box::new(|h| { - h.add_ops(&collect_bitwise_from_keccak(&keccak_ops)); - // KECCAK_RND's byte checks have multiplicity 1, so its padding rows - // (all cells 0) send ARE_BYTES[0, 0] too. - h.bump_n( - BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, 0, 0), - keccak_rnd::padding_are_bytes(keccak_ops.len()), - ); - }), - Box::new(|h| h.add_ops(&collect_bitwise_from_ecsm(&ecsm_ops))), - Box::new(|h| h.add_ops(&collect_bitwise_from_ecdas(&ecdas_ops))), - Box::new(|h| h.add_ops(&collect_bitwise_from_hint(&hint_ops))), - Box::new(|h| is_wh::collect_bitwise_from_is_wh(&is_wh_rows, h)), - Box::new(|h| add_padding_byte_checks(h, num_padding_rows)), - ]; - if let Some(image) = initial_image - && !l2g_memory_bookend - { - collectors.push(Box::new(move |h| { - collect_bitwise_from_page(image, memory_state, l2g_memory_bookend, h) - })); - } - - let mut base = bitwise::BitwiseHistogram::new(); - - #[cfg(feature = "parallel")] - { - use rayon::prelude::*; - // Cap concurrent 80 MiB histograms at `cap` to bound peak memory. The two dominant - // sources — the in-walk lookups and MEMW_R (each tens of millions of items) — are - // split into ~`cap` row-range slices so they parallelize INTERNALLY instead of each - // pinning one core while the rest idle. Every unit (whole collectors + the heavy - // slices) is round-robined into exactly `cap` buckets, one histogram each, so the - // split heavy work is spread across buckets rather than piled into one. - // add_ops/bump/merge form a commutative monoid, so any partition yields - // byte-identical multiplicities (same as the serial fallback below). - let cap = rayon::current_num_threads().clamp(1, 8); - let mut units: Vec = Vec::with_capacity(collectors.len() + 2 * cap); - let iw_chunk = bitwise_ops.len().div_ceil(cap).max(1); - for slice in bitwise_ops.chunks(iw_chunk) { - units.push(Box::new(move |h| h.add_ops(slice))); - } - let reg_chunk = memw_register_rows.len().div_ceil(cap).max(1); - for slice in memw_register_rows.chunks(reg_chunk) { - units.push(Box::new(move |h| { - memw_register::collect_bitwise_from_memw_register(slice, h) - })); - } - units.extend(collectors); - - let mut buckets: Vec> = (0..cap).map(|_| Vec::new()).collect(); - for (i, unit) in units.into_iter().enumerate() { - buckets[i % cap].push(unit); - } - if let Some(reduced) = buckets - .par_iter() - .map(|bucket| { - let mut h = bitwise::BitwiseHistogram::new(); - for f in bucket { - f(&mut h); - } - h - }) - .reduce_with(|mut a, b| { - a.merge(&b); - a - }) - { - base.merge(&reduced); - } + #[cfg(test)] + if finish_oracle::on() { + return finish_oracle::build_traces_oracle( + ops, + initial_image, + memory_state, + register_init, + decode_trace, + decode_table, + register_state, + max_rows, + private_input, + is_final, + l2g_memory_bookend, + skip, + pre, + ); } - #[cfg(not(feature = "parallel"))] - { - base.add_ops(&bitwise_ops); - memw_register::collect_bitwise_from_memw_register(&memw_register_rows, &mut base); - for f in &collectors { - f(&mut base); - } - } - let bitwise_histogram = base; - // The in-walk lookup Vec has been counted into the histogram; free it now. - drop(bitwise_ops); - #[cfg(feature = "instruments")] - drop(__sp); - - // ===================================================================== - // PHASE 5: Generate final traces (parallelized) - // ===================================================================== - #[cfg(feature = "instruments")] - let __sp = stark::instruments::span("p5_generate_tables"); - - // A monolithic run or the final continuation epoch terminates on the program's - // halt ECALL. Intermediate continuation epochs do not halt, so fall back to the - // last cycle's timestamp and skip HALT-based PC finalization — the PC is carried - // to the next epoch via the register snapshot, and HALT is excluded from the - // proof (`include_halt = false`) so `halt_trace` is unused there. - let (halt_timestamp, halt_next_pc) = if is_final { - let halt_op = cpu_ops - .iter() - .rev() - .find(|op| op.decode.fields.ecall) - .ok_or(Error::MissingHaltEcall)?; - // Finalize the PC (x255) on the REGISTER table. The CPU padding rows carry - // pc=1 and chain the inline-PC `memory` tokens with a +4 timestamp cadence - // starting from the HALT chip's emit_pc at `halt_timestamp + 1`; the last - // padding write therefore lands at `halt_timestamp + 4*num_padding_rows + 1` - // (= `halt_timestamp + 1` when there is no padding), which is the - // verifier-known `cpu::final_pc_timestamp` REGISTER consumes x255 at. Only - // the value 1 reaches the bus (as FINI); REGISTER does not read the x255 - // rows' timestamp columns. - register_state.write_pc(1, halt_op.timestamp + 4 * num_padding_rows as u64 + 1); - (halt_op.timestamp, halt_op.next_pc) - } else { - (cpu_ops.last().map(|op| op.timestamp).unwrap_or(0), 0) - }; - - let register_final_state = register_state.to_final_state_map(); - - // Each build below reads disjoint op lists and writes its own table, so - // they all run in one rayon scope. Disk-spill stays sequential: its - // generate→spill order keeps trace memory bounded. - let cpu_ops_ref = &cpu_ops; - let gen_cpus = || { - chunk_and_generate( - cpu_ops_ref, - max_rows.cpu, - cpu::generate_cpu_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_memws = || { - chunk_and_generate_optional( - &memw_ops, - max_rows.memw, - memw::generate_memw_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_memw_aligneds = || { - chunk_and_generate_optional( - &memw_aligned_ops, - max_rows.memw_aligned, - memw_aligned::generate_memw_aligned_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_memw_registers = || { - // Direct-to-column fill from compact RegRows — the register fast path never - // materializes a `Vec`. - chunk_and_generate( - &memw_register_rows, - max_rows.memw_register, - memw_register::generate_memw_register_trace_from_rows, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_loads = || { - chunk_and_generate_optional( - &load_ops, - max_rows.load, - load::generate_load_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_lts = || { - chunk_and_generate_optional( - <_ops, - max_rows.lt, - lt::generate_lt_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_shifts = || { - chunk_and_generate_optional( - &shift_ops, - max_rows.shift, - shift::generate_shift_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_muls = || { - chunk_and_generate_optional( - &mul_ops, - max_rows.mul, - mul::generate_mul_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_dvrms = || { - chunk_and_generate_optional( - &dvrm_ops, - max_rows.dvrm, - dvrm::generate_dvrm_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_branches = || { - chunk_and_generate_optional( - &branch_ops, - max_rows.branch, - branch::generate_branch_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - // Auxiliary ALU / memory / CPU32 dispatch chips, each filtered out of the CPU - // ops above. - let gen_eqs = || { - chunk_and_generate_optional::( - &eq_ops, - max_rows.eq, - eq::generate_eq_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_bytewises = || { - chunk_and_generate_optional::( - &bytewise_ops, - max_rows.bytewise, - bytewise::generate_bytewise_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_stores = || { - chunk_and_generate_optional::( - &store_ops, - max_rows.store, - store::generate_store_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_cpu32s = || { - chunk_and_generate_optional::( - &cpu32_ops, - max_rows.cpu32, - cpu32::generate_cpu32_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_bitwise = || { - let mut bitwise = bitwise::generate_bitwise_trace(); - // Fill the MU columns (11..=20) from the accumulated histogram. - bitwise_histogram.fill_multiplicities(&mut bitwise); - bitwise - }; - // Each CPU operation looks up the DECODE table once; padding rows look up - // pc=1 (the CPU padding entry). When CPU is split, each chunk pads - // independently. - let gen_decode = move || { - let mut decode = decode_trace; - let mut decode_lookups: Vec = cpu_ops_ref.iter().map(|op| op.decode.pc).collect(); - decode_lookups.extend(std::iter::repeat_n(cpu::CPU_PADDING_PC, num_padding_rows)); - decode::update_multiplicities(&mut decode, decode_pc_to_row, &decode_lookups); - decode - }; - let gen_keccaks = || { - generate_optional( - &keccak_ops, - keccak::generate_keccak_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_keccak_rnds = || { - let keccak_rnd_ops: Vec = keccak_ops - .iter() - .map(|op| KeccakRoundOperation { - timestamp: op.timestamp, - input: op.input, - output: op.output, - }) - .collect(); - generate_optional( - &keccak_rnd_ops, - keccak_rnd::generate_keccak_rnd_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_keccak_rc = || { - let mut keccak_rc_trace = keccak_rc::generate_keccak_rc_trace(); - keccak_rc::update_multiplicities(&mut keccak_rc_trace, keccak_ops.len()); - keccak_rc_trace - }; - let gen_pages = || match initial_image { - // Continuation epochs (l2g_memory_bookend) skip PAGE: the L2G table owns - // every touched cell's Memory init/fini, and every untouched PAGE row - // self-cancels (init==fini, ts=0), so PAGE contributes nothing here. - Some(image) if !l2g_memory_bookend => { - generate_page_tables(image, memory_state, private_input, l2g_memory_bookend) - } - _ => (Vec::new(), Vec::new()), - }; - let gen_register = || register::generate_register_trace(®ister_final_state, register_init); - let gen_halt = || halt::generate_halt_trace(halt_timestamp, halt_next_pc); - // ECSM accelerator traces. A program that does not use ECSM carries no ECSM - // and no ECDAS table at all — not a padded one. - let gen_ecsms = || { - generate_optional( - &ecsm_ops, - ecsm::generate_ecsm_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_ecdases = || { - generate_optional( - &ecdas_ops, - ecdas::generate_ecdas_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - // HINT table. Absent entirely for programs that make no hint ecalls. - let gen_hints = || { - generate_optional( - &hint_ops, - hint::generate_hint_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - // IS_WH table. Absent entirely when no chip sends `IS_WH`. - let gen_is_whs = || { - generate_optional( - &is_wh_rows, - is_wh::generate_is_wh_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - // COPY is not optional: it receives the `write` ECALL directly. - let gen_copy = || copy::generate_copy_trace(©_ops); - - let (mut cpus_slot, mut memws_slot, mut memw_aligneds_slot, mut memw_registers_slot) = - (None, None, None, None); - let (mut loads_slot, mut lts_slot, mut shifts_slot, mut muls_slot) = (None, None, None, None); - let (mut dvrms_slot, mut branches_slot, mut bitwise_slot, mut decode_slot) = - (None, None, None, None); - let (mut keccaks_slot, mut keccak_rnds_slot, mut keccak_rc_slot) = (None, None, None); - let (mut pages_slot, mut register_slot, mut halt_slot) = (None, None, None); - let (mut eqs_slot, mut bytewises_slot, mut stores_slot, mut cpu32s_slot) = - (None, None, None, None); - let (mut ecsms_slot, mut ecdases_slot) = (None, None); - let mut hints_slot = None; - let mut is_whs_slot = None; - let mut copy_slot = None; - - #[cfg(feature = "disk-spill")] - let sequential = storage_mode == StorageMode::Disk || cfg!(not(feature = "parallel")); - #[cfg(not(feature = "disk-spill"))] - let sequential = cfg!(not(feature = "parallel")); - - if !sequential { - #[cfg(feature = "parallel")] - rayon::scope(|s| { - macro_rules! spawn_into { - ($slot:ident, $gen:ident) => {{ - let slot = &mut $slot; - s.spawn(move |_| *slot = Some($gen())); - }}; - } - // Heaviest builds first so the scheduler overlaps them with the rest. - spawn_into!(memw_registers_slot, gen_memw_registers); - spawn_into!(cpus_slot, gen_cpus); - spawn_into!(memws_slot, gen_memws); - spawn_into!(lts_slot, gen_lts); - spawn_into!(decode_slot, gen_decode); - spawn_into!(branches_slot, gen_branches); - spawn_into!(bitwise_slot, gen_bitwise); - spawn_into!(muls_slot, gen_muls); - spawn_into!(memw_aligneds_slot, gen_memw_aligneds); - spawn_into!(loads_slot, gen_loads); - spawn_into!(shifts_slot, gen_shifts); - spawn_into!(dvrms_slot, gen_dvrms); - spawn_into!(pages_slot, gen_pages); - spawn_into!(keccaks_slot, gen_keccaks); - spawn_into!(keccak_rnds_slot, gen_keccak_rnds); - spawn_into!(keccak_rc_slot, gen_keccak_rc); - spawn_into!(register_slot, gen_register); - spawn_into!(halt_slot, gen_halt); - spawn_into!(eqs_slot, gen_eqs); - spawn_into!(bytewises_slot, gen_bytewises); - spawn_into!(stores_slot, gen_stores); - spawn_into!(cpu32s_slot, gen_cpu32s); - spawn_into!(ecsms_slot, gen_ecsms); - spawn_into!(ecdases_slot, gen_ecdases); - spawn_into!(hints_slot, gen_hints); - spawn_into!(is_whs_slot, gen_is_whs); - spawn_into!(copy_slot, gen_copy); - }); - } else { - cpus_slot = Some(gen_cpus()); - memws_slot = Some(gen_memws()); - memw_aligneds_slot = Some(gen_memw_aligneds()); - memw_registers_slot = Some(gen_memw_registers()); - loads_slot = Some(gen_loads()); - lts_slot = Some(gen_lts()); - shifts_slot = Some(gen_shifts()); - muls_slot = Some(gen_muls()); - dvrms_slot = Some(gen_dvrms()); - branches_slot = Some(gen_branches()); - bitwise_slot = Some(gen_bitwise()); - decode_slot = Some(gen_decode()); - keccaks_slot = Some(gen_keccaks()); - keccak_rnds_slot = Some(gen_keccak_rnds()); - keccak_rc_slot = Some(gen_keccak_rc()); - pages_slot = Some(gen_pages()); - register_slot = Some(gen_register()); - halt_slot = Some(gen_halt()); - eqs_slot = Some(gen_eqs()); - bytewises_slot = Some(gen_bytewises()); - stores_slot = Some(gen_stores()); - cpu32s_slot = Some(gen_cpu32s()); - ecsms_slot = Some(gen_ecsms()); - ecdases_slot = Some(gen_ecdases()); - hints_slot = Some(gen_hints()); - is_whs_slot = Some(gen_is_whs()); - copy_slot = Some(gen_copy()); - } - - const PHASE5_RAN: &str = "phase 5 generation ran in one of the branches above"; - let cpus = cpus_slot.expect(PHASE5_RAN)?; - let memws = memws_slot.expect(PHASE5_RAN)?; - let memw_aligneds = memw_aligneds_slot.expect(PHASE5_RAN)?; - let memw_registers = memw_registers_slot.expect(PHASE5_RAN)?; - let loads = loads_slot.expect(PHASE5_RAN)?; - let lts = lts_slot.expect(PHASE5_RAN)?; - let shifts = shifts_slot.expect(PHASE5_RAN)?; - let muls = muls_slot.expect(PHASE5_RAN)?; - let dvrms = dvrms_slot.expect(PHASE5_RAN)?; - let branches = branches_slot.expect(PHASE5_RAN)?; - let eqs = eqs_slot.expect(PHASE5_RAN)?; - let bytewises = bytewises_slot.expect(PHASE5_RAN)?; - let stores = stores_slot.expect(PHASE5_RAN)?; - let cpu32s = cpu32s_slot.expect(PHASE5_RAN)?; - #[allow(unused_mut)] - let mut bitwise = bitwise_slot.expect(PHASE5_RAN); - #[allow(unused_mut)] - let mut decode = decode_slot.expect(PHASE5_RAN); - let keccaks = keccaks_slot.expect(PHASE5_RAN)?; - let keccak_rnds = keccak_rnds_slot.expect(PHASE5_RAN)?; - let keccak_rc_trace = keccak_rc_slot.expect(PHASE5_RAN); - #[allow(unused_mut)] - let (mut pages, page_configs) = pages_slot.expect(PHASE5_RAN); - #[allow(unused_mut)] - let mut register_trace = register_slot.expect(PHASE5_RAN); - #[allow(unused_mut)] - let mut halt_trace = halt_slot.expect(PHASE5_RAN); - let ecsms = ecsms_slot.expect(PHASE5_RAN)?; - let ecdases = ecdases_slot.expect(PHASE5_RAN)?; - let hints = hints_slot.expect(PHASE5_RAN)?; - let is_whs = is_whs_slot.expect(PHASE5_RAN)?; - #[allow(unused_mut)] - let mut copy_trace = copy_slot.expect(PHASE5_RAN); - - // Fixed-size and per-page tables aren't built through `chunk_and_generate`, - // so spill them here before returning. - #[cfg(feature = "disk-spill")] - if storage_mode == StorageMode::Disk { - bitwise - .main_table - .spill_to_disk() - .map_err(|e| Error::Prover(format!("disk-spill bitwise: {e}")))?; - decode - .main_table - .spill_to_disk() - .map_err(|e| Error::Prover(format!("disk-spill decode: {e}")))?; - register_trace - .main_table - .spill_to_disk() - .map_err(|e| Error::Prover(format!("disk-spill register: {e}")))?; - halt_trace - .main_table - .spill_to_disk() - .map_err(|e| Error::Prover(format!("disk-spill halt: {e}")))?; - // COPY joined this list when it became a fixed table: it carries one row per - // eight committed bytes, so it is not small on an output-heavy run. - copy_trace - .main_table - .spill_to_disk() - .map_err(|e| Error::Prover(format!("disk-spill copy: {e}")))?; - for page in &mut pages { - page.main_table - .spill_to_disk() - .map_err(|e| Error::Prover(format!("disk-spill page: {e}")))?; - } - } - - // Continuation callers derive the real cross-epoch boundary from this set and - // install its L2G trace after provenance is applied. Avoid building a - // throwaway genesis-only L2G trace here. - let touched_memory_cells = if l2g_memory_bookend { - touched_cells_from_memory_state(memory_state) - } else { - Vec::new() - }; - let local_to_global = local_to_global::generate_local_to_global_trace(&[]); - - #[cfg(feature = "instruments")] - drop(__sp); - Ok(Traces { - cpus, - bitwise, - lts, - shifts, - memws, - memw_aligneds, - loads, - decode, - muls, - dvrms, - pages, - page_configs, - register: register_trace, - public_output_bytes, - branches, - halt: halt_trace, - keccaks, - keccak_rnds, - keccak_rc: keccak_rc_trace, - ecsms, - ecdases, - hints, - is_whs, - copy: copy_trace, - memw_registers, - local_to_global, - touched_memory_cells, - eqs, - bytewises, - stores, - cpu32s, - }) + finish::FinishPlan::new( + ops, + initial_image, + memory_state, + register_init, + decode_trace, + decode_table, + register_state, + max_rows, + private_input, + is_final, + l2g_memory_bookend, + skip, + pre, + )? + .emit_all() } -/// Padded row count after chunking, for a table that is always present: an -/// empty op list still allocates one 4-row padded chunk. -#[cfg(feature = "disk-spill")] -fn padded_chunked_rows(ops_count: usize, max_rows: usize) -> u64 { - padded_chunked_rows_optional(ops_count, max_rows).max(4) -} - -/// Padded row count after chunking, for a table left out of the proof when -/// unused. Mirrors [`chunk_and_generate_optional`]: no ops, no rows. -#[cfg(feature = "disk-spill")] -fn padded_chunked_rows_optional(ops_count: usize, max_rows: usize) -> u64 { - // `max_rows <= 0` would loop forever. Called internally with const values > 0. - assert!(max_rows > 0, "max_rows must be positive"); - let mut total: u64 = 0; - let mut remaining = ops_count; - while remaining > 0 { - let chunk_size = remaining.min(max_rows); - total += chunk_size.next_power_of_two().max(4) as u64; - remaining -= chunk_size; - } - total -} - -/// Per-table padded row counts plus auxiliary metrics for peak-heap estimation. -#[cfg(feature = "disk-spill")] -#[derive(Debug, Default, Clone)] -pub struct TableLengths { - pub cpu_padded_rows: u64, - pub memw_padded_rows: u64, - pub memw_aligned_padded_rows: u64, - pub memw_register_padded_rows: u64, - pub load_padded_rows: u64, - pub lt_padded_rows: u64, - pub shift_padded_rows: u64, - pub mul_padded_rows: u64, - pub dvrm_padded_rows: u64, - pub branch_padded_rows: u64, - pub copy_padded_rows: u64, - pub decode_rows: u64, - pub unique_page_count: u64, - pub cycle_count: u64, - pub unique_byte_count: u64, -} - -/// Per-table row counts from `logs`, without building op vectors. -/// Exact for tables that don't dedup; upper bound for LT, MUL, DVRM, BRANCH. -/// Must stay in sync with `Traces::from_elf_and_logs`. -#[cfg(feature = "disk-spill")] -pub fn count_table_lengths( - elf: &Elf, - logs: &[Log], - max_rows: &super::MaxRowsConfig, - private_input: &[u8], -) -> Result { - // Phase 0: ELF → instructions + DECODE row count. - let instructions = decode::instructions_from_elf(elf) - .map_err(|e| Error::Execution(format!("Failed to parse instructions: {e}")))?; - // Mirrors the padding inside `generate_decode_trace`. - let decode_rows = (instructions.len() as u64 + 1).next_power_of_two().max(2); - - // Memory + register state for partition predicates that need timestamps. - let mut memory_state = MemoryState::from_image(&build_initial_image(elf, private_input)); - let mut register_state = RegisterState::new(elf.entry_point); - - // Raw counts (pre-chunking + pre-padding). - let mut cpu_count = 0usize; - // Wide-MEMW counts bucketed by width, used by the LT-from-MEMW derivation. - let mut memw_by_width: [usize; 4] = [0; 4]; - let mut memw_aligned_count = 0usize; - let mut memw_register_count = 0usize; - let mut load_count = 0usize; - let mut lt_count = 0usize; - let mut shift_count = 0usize; - let mut mul_count = 0usize; - let mut dvrm_count = 0usize; - let mut branch_count = 0usize; - let mut copy_count = 0usize; - // Zero, not the carried x254 the generation walk seeds from: `count_table_lengths` - // takes no register-init vector, so it is only usable for a whole run or a first - // epoch anyway. It is sound today because no row count depends on `dst` — wire a - // `register_init` through before that stops being true. - let mut current_commit_index = 0u32; - - let partition_memw = |op: &MemwOperation, - by_width: &mut [usize; 4], - aligned: &mut usize, - register: &mut usize| { - // Same classifier as the walk's MemwBuckets, so the sizing pass counts - // exactly the rows trace generation will produce. - match classify_memw(op) { - MemwRoute::Register => *register += 1, - MemwRoute::Aligned => *aligned += 1, - MemwRoute::General => { - let idx = match op.width { - 1 => 0, - 2 => 1, - 4 => 2, - 8 => 3, - _ => return, - }; - by_width[idx] += 1; - } - } - }; - - let mut reg_memw_scratch: Vec = Vec::with_capacity(4); - for (i, log) in logs.iter().enumerate() { - let timestamp = (i as u64) * 4 + 4; - let instruction = instructions - .get(&log.current_pc) - .copied() - .ok_or(Error::MissingInstruction(log.current_pc))?; - let cpu_op = CpuOperation::from_log_and_instruction(log, timestamp, instruction); - cpu_count += 1; - - // Memory ops from load/store - if cpu_op.decode.fields.is_load() { - let (memw_op, _load_op, _bitwise) = - collect_load_op_from_cpu(&cpu_op, &mut memory_state); - partition_memw( - &memw_op, - &mut memw_by_width, - &mut memw_aligned_count, - &mut memw_register_count, - ); - load_count += 1; - } else if cpu_op.decode.fields.is_store() { - let memw_op = collect_store_op_from_cpu(&cpu_op, &mut memory_state); - partition_memw( - &memw_op, - &mut memw_by_width, - &mut memw_aligned_count, - &mut memw_register_count, - ); - } +/// [`Traces::runtime_page_ranges`] of these page configs: the runtime pages +/// (zero-init, not covered by ELF segments), contiguous ones merged. +pub(crate) fn runtime_page_ranges_of(page_configs: &[PageConfig]) -> Vec { + let page_size = page::DEFAULT_PAGE_SIZE as u64; - // Register accesses. - reg_memw_scratch.clear(); - collect_register_ops_from_cpu(&cpu_op, &mut register_state, &mut reg_memw_scratch); - for memw_op in ®_memw_scratch { - partition_memw( - memw_op, - &mut memw_by_width, - &mut memw_aligned_count, - &mut memw_register_count, - ); - } + // Collect sorted non-ELF page bases (zero-init pages are runtime pages) + let runtime_bases: Vec = page_configs + .iter() + .filter(|config| config.init_values.is_none()) + .map(|config| config.page_base) + .collect(); - // Every copy ecall — `write`, `memcpy`/`memmove` and `memset` — goes through - // the same chip, so it is sized through the same path. - if cpu_op.ecall_commit || cpu_op.ecall_dma_memcpy || cpu_op.ecall_dma_memset { - let (functionality, src, dst, count) = if cpu_op.ecall_commit { - ( - copy::Functionality::Write, - cpu_op.commit_buf_addr, - u64::from(current_commit_index), - cpu_op.commit_count, - ) - } else { - let functionality = if cpu_op.ecall_dma_memset { - copy::Functionality::Set - } else { - copy::Functionality::Copy - }; - ( - functionality, - register_state.read(11).0, - register_state.read(10).0, - register_state.read(12).0, - ) - }; + // Run-length encode contiguous pages into (base, count) ranges + let mut ranges = Vec::new(); + if runtime_bases.is_empty() { + return ranges; + } - for reg_op in collect_copy_register_ops( - functionality, - cpu_op.timestamp, - src, - dst, - count, - &mut register_state, - )? { - partition_memw( - ®_op, - &mut memw_by_width, - &mut memw_aligned_count, - &mut memw_register_count, - ); - } + let mut start = runtime_bases[0]; + let mut count = 1u64; - let rows = replay_copy_for_sizing( - functionality, - cpu_op.timestamp, - src, - dst, - count, - &mut memory_state, - |memw_op| { - partition_memw( - memw_op, - &mut memw_by_width, - &mut memw_aligned_count, - &mut memw_register_count, - ); - }, - ); - copy_count += rows as usize; - - if cpu_op.ecall_commit { - let delta = u32::try_from(count) - .map_err(|_| Error::Execution("commit_count exceeds u32 range".into()))?; - current_commit_index = current_commit_index - .checked_add(delta) - .ok_or_else(|| Error::Execution("commit index exceeds u32 range".into()))?; - } + for &base in &runtime_bases[1..] { + if base == start + count * page_size { + count += 1; + } else { + ranges.push(crate::RuntimePageRange { base: start, count }); + start = base; + count = 1; } + } + ranges.push(crate::RuntimePageRange { base: start, count }); - if cpu_op.ecall_hint { - // Mirror `collect_hint_ops`: three register reads (a0/a1/a2) and four - // 8-byte output writes go through the memory argument, plus the three LT - // range-checks (selector < 3, in_addr and out_addr low limbs). Replaying it - // here keeps memory/register state in sync with generation, exactly like - // commit above. - let (hint_memw, _hint_op) = - collect_hint_ops(&cpu_op, &mut memory_state, &mut register_state); - for memw_op in &hint_memw { - partition_memw( - memw_op, - &mut memw_by_width, - &mut memw_aligned_count, - &mut memw_register_count, - ); - } - lt_count += 3; - } + ranges +} - // CPU-side per-instruction-kind counters (non-word; word → CPU32, B5b) - let f = &cpu_op.decode.fields; - if !f.word_instr && f.is_lt() { - lt_count += 1; - } - if !f.word_instr && f.is_shift() { - shift_count += 1; - } - if !f.word_instr && f.is_mul() { - mul_count += 1; - } - if !f.word_instr && f.is_divrem() { - dvrm_count += 1; +impl Traces { + /// The bytes the main tables take: eight a cell, or their packed bytes + /// (the page configs and the touched cells left out). A measurement (`LAMBDA_VM_BLOCK_MEMLOG`): + /// every table field is named, so a new one does not compile until it is + /// counted here. + pub fn main_bytes(&self) -> usize { + type T = TraceTable; + fn one(t: &T) -> usize { + t.narrow_main().map_or_else( + || t.main_table.width * t.main_table.height * std::mem::size_of::(), + |packed| packed.data().len(), + ) } - if cpu_op.branch_cond { - branch_count += 1; + fn all(ts: &[T]) -> usize { + ts.iter().map(one).sum() } + let Traces { + cpus, + bitwise, + lts, + shifts, + memws, + memw_aligneds, + loads, + decode, + muls, + dvrms, + pages, + page_configs: _, + register, + public_output_bytes: _, + branches, + halt, + copies, + keccaks, + keccak_rnds, + keccak_rc, + blake3, + num_blake3_ops: _, + ecsms, + ecdases, + hints, + ecrecs, + is_whs, + memw_registers, + local_to_global, + touched_memory_cells: _, + eqs, + bytewises, + stores, + cpu32s, + } = self; + [ + cpus, + lts, + shifts, + memws, + memw_aligneds, + loads, + muls, + dvrms, + pages, + branches, + copies, + keccaks, + keccak_rnds, + ecsms, + ecdases, + hints, + ecrecs, + is_whs, + memw_registers, + eqs, + bytewises, + stores, + cpu32s, + ] + .into_iter() + .map(|ts| all(ts)) + .sum::() + + [ + bitwise, + decode, + register, + halt, + keccak_rc, + blake3, + local_to_global, + ] + .into_iter() + .map(one) + .sum::() } - // HALT finalization. Halt ops fall through to wide MEMW. - let halt_memw_ops = collect_halt_ops(&mut register_state); - for memw_op in &halt_memw_ops { - partition_memw( - memw_op, - &mut memw_by_width, - &mut memw_aligned_count, - &mut memw_register_count, - ); - } - - // LT ops derived from wide-MEMW and memw_aligned ops. - let memw_count = memw_by_width.iter().sum::(); - let lt_from_memw = - memw_by_width[0] + 2 * memw_by_width[1] + 4 * memw_by_width[2] + 8 * memw_by_width[3]; - lt_count += lt_from_memw + memw_aligned_count; - - // DVRM derives mul and lt ops. - mul_count += 2 * dvrm_count; - lt_count += dvrm_count; - - let unique_page_count = memory_state.unique_page_count(page::DEFAULT_PAGE_SIZE as u64); - let unique_byte_count = memory_state.cells.len() as u64; - let cycle_count = logs.len() as u64; - - Ok(TableLengths { - cpu_padded_rows: padded_chunked_rows(cpu_count, max_rows.cpu), - memw_padded_rows: padded_chunked_rows_optional(memw_count, max_rows.memw), - memw_aligned_padded_rows: padded_chunked_rows_optional( - memw_aligned_count, - max_rows.memw_aligned, - ), - memw_register_padded_rows: padded_chunked_rows(memw_register_count, max_rows.memw_register), - load_padded_rows: padded_chunked_rows_optional(load_count, max_rows.load), - lt_padded_rows: padded_chunked_rows_optional(lt_count, max_rows.lt), - shift_padded_rows: padded_chunked_rows_optional(shift_count, max_rows.shift), - mul_padded_rows: padded_chunked_rows_optional(mul_count, max_rows.mul), - dvrm_padded_rows: padded_chunked_rows_optional(dvrm_count, max_rows.dvrm), - branch_padded_rows: padded_chunked_rows_optional(branch_count, max_rows.branch), - copy_padded_rows: copy_count.next_power_of_two().max(4) as u64, - decode_rows, - unique_page_count, - cycle_count, - unique_byte_count, - }) -} - -impl Traces { /// Pre-upload the epoch's biggest main traces to device, called from the /// epoch pipeline's builder thread (idle slack ahead of the prover) so the /// R1 main commits D2D-copy instead of paying the H2D inside their chains. @@ -4557,7 +8188,7 @@ impl Traces { // out, it is the only one whose height scales with the workload rather than // being one row per event. Nothing mutates it after the build, so unlike // BITWISE it cannot go stale on the device. - tables.push(&mut self.copy); + tables.extend(self.copies.iter_mut()); let bytes_of = |t: &TraceTable| { t.num_rows() * t.num_main_columns * 8 @@ -4588,6 +8219,7 @@ impl Traces { pub fn total_field_elements(&self) -> u64 { use super::bitwise::NUM_PRECOMPUTED_COLS as BITWISE_PRECOMPUTED; use super::bitwise::cols::NUM_COLUMNS as BITWISE_COLS; + use super::blake3::cols::NUM_COLUMNS as BLAKE3_COLS; use super::branch::cols::NUM_COLUMNS as BRANCH_COLS; use super::bytewise::cols::NUM_COLUMNS as BYTEWISE_COLS; use super::cpu::cols::NUM_COLUMNS as CPU_COLS; @@ -4620,6 +8252,7 @@ impl Traces { let Traces { cpus, + num_blake3_ops, bitwise, lts, shifts, @@ -4636,11 +8269,13 @@ impl Traces { keccaks, keccak_rnds, keccak_rc, + blake3, ecsms, ecdases, hints, + ecrecs, + copies, is_whs, - copy, memw_registers, eqs, bytewises, @@ -4697,6 +8332,11 @@ impl Traces { total += (t.num_rows() * KECCAK_RND_COLS) as u64; } total += (keccak_rc.num_rows() * (KECCAK_RC_COLS - KECCAK_RC_PRECOMPUTED)) as u64; + // Counted only when the proof carries the table (`table_counts().blake3`); + // an unused BLAKE3 trace is padding the prover never commits. + if *num_blake3_ops > 0 { + total += (blake3.num_rows() * BLAKE3_COLS) as u64; + } for t in eqs { total += (t.num_rows() * EQ_COLS) as u64; } @@ -4718,10 +8358,15 @@ impl Traces { for t in hints { total += (t.num_rows() * HINT_COLS) as u64; } + for t in ecrecs { + total += (t.num_rows() * super::ecrec::cols::NUM_COLUMNS) as u64; + } + for t in copies { + total += (t.num_rows() * super::copy::cols::NUM_COLUMNS) as u64; + } for t in is_whs { total += (t.num_rows() * IS_WH_COLS) as u64; } - total += (copy.num_rows() * super::copy::cols::NUM_COLUMNS) as u64; total } @@ -4756,6 +8401,7 @@ impl Traces { let n_keccak = aux_cols(super::keccak::bus_interactions().len()); let n_keccak_rnd = aux_cols(super::keccak_rnd::bus_interactions().len()); let n_keccak_rc = aux_cols(super::keccak_rc::bus_interactions().len()); + let n_blake3 = aux_cols(super::blake3::bus_interactions().len()); let n_eq = aux_cols(super::eq::bus_interactions().len()); let n_bytewise = aux_cols(super::bytewise::bus_interactions().len()); let n_store = aux_cols(super::store::bus_interactions().len()); @@ -4764,10 +8410,12 @@ impl Traces { let n_ecdas = aux_cols(super::ecdas::bus_interactions().len()); let n_copy = aux_cols(super::copy::bus_interactions().len()); let n_hint = aux_cols(super::hint::bus_interactions().len()); + let n_ecrec = aux_cols(super::ecrec::bus_interactions().len()); let n_is_wh = aux_cols(super::is_wh::bus_interactions().len()); let Traces { cpus, + num_blake3_ops, bitwise, lts, shifts, @@ -4784,11 +8432,13 @@ impl Traces { keccaks, keccak_rnds, keccak_rc, + blake3, ecsms, ecdases, hints, + ecrecs, + copies, is_whs, - copy, memw_registers, eqs, bytewises, @@ -4845,6 +8495,9 @@ impl Traces { total += (t.num_rows() * n_keccak_rnd) as u64; } total += (keccak_rc.num_rows() * n_keccak_rc) as u64; + if *num_blake3_ops > 0 { + total += (blake3.num_rows() * n_blake3) as u64; + } for t in eqs { total += (t.num_rows() * n_eq) as u64; } @@ -4866,10 +8519,15 @@ impl Traces { for t in hints { total += (t.num_rows() * n_hint) as u64; } + for t in ecrecs { + total += (t.num_rows() * n_ecrec) as u64; + } + for t in copies { + total += (t.num_rows() * n_copy) as u64; + } for t in is_whs { total += (t.num_rows() * n_is_wh) as u64; } - total += (copy.num_rows() * n_copy) as u64; total } @@ -4895,6 +8553,12 @@ impl Traces { ecsm: self.ecsms.len(), ecdas: self.ecdases.len(), hint: self.hints.len(), + ecrec: self.ecrecs.len(), + copy: self.copies.len(), + // 0 or 1: the table is carried only when the workload used it. The + // six above read their own `Vec` length; this one cannot, because a + // BLAKE3 trace pads to four rows whether or not it was used. + blake3: usize::from(self.num_blake3_ops > 0), is_wh: self.is_whs.len(), } } @@ -5047,37 +8711,7 @@ impl Traces { /// Runtime (non-ELF) pages are identified by `init_values == None` /// (zero-init), avoiding a redundant ELF segment scan. pub fn runtime_page_ranges(&self) -> Vec { - let page_size = page::DEFAULT_PAGE_SIZE as u64; - - // Collect sorted non-ELF page bases (zero-init pages are runtime pages) - let runtime_bases: Vec = self - .page_configs - .iter() - .filter(|config| config.init_values.is_none()) - .map(|config| config.page_base) - .collect(); - - // Run-length encode contiguous pages into (base, count) ranges - let mut ranges = Vec::new(); - if runtime_bases.is_empty() { - return ranges; - } - - let mut start = runtime_bases[0]; - let mut count = 1u64; - - for &base in &runtime_bases[1..] { - if base == start + count * page_size { - count += 1; - } else { - ranges.push(crate::RuntimePageRange { base: start, count }); - start = base; - count = 1; - } - } - ranges.push(crate::RuntimePageRange { base: start, count }); - - ranges + runtime_page_ranges_of(&self.page_configs) } /// Generates all traces from ELF and execution logs using phased collection. @@ -5094,7 +8728,6 @@ impl Traces { logs: &[Log], max_rows: &super::MaxRowsConfig, private_input: &[u8], - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, ) -> Result { let initial_image = build_initial_image(elf, private_input); let register_init = register::register_init_from_entry_point(elf.entry_point); @@ -5107,8 +8740,6 @@ impl Traces { private_input, true, false, - #[cfg(feature = "disk-spill")] - storage_mode, ) } @@ -5134,7 +8765,6 @@ impl Traces { private_input: &[u8], is_final: bool, l2g_memory_bookend: bool, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, ) -> Result { let artifacts = DecodeArtifacts::from_elf(elf)?; Self::from_image_and_logs_with_decode( @@ -5146,8 +8776,6 @@ impl Traces { private_input, is_final, l2g_memory_bookend, - #[cfg(feature = "disk-spill")] - storage_mode, ) } @@ -5167,7 +8795,6 @@ impl Traces { private_input: &[u8], is_final: bool, l2g_memory_bookend: bool, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, ) -> Result { let collected = Self::collect_epoch(artifacts, initial_image, register_init, logs, is_final)?; @@ -5180,8 +8807,6 @@ impl Traces { private_input, is_final, l2g_memory_bookend, - #[cfg(feature = "disk-spill")] - storage_mode, ) } @@ -5209,16 +8834,18 @@ impl Traces { // Phase 1: Logs → CPU operations #[cfg(feature = "instruments")] let __sp = stark::instruments::span("p1_cpu_ops"); - let cpu_ops = collect_cpu_ops(logs, &artifacts.instructions)?; + let cpu_ops = collect_cpu_ops(logs, &artifacts.decode)?; #[cfg(feature = "instruments")] drop(__sp); + build_stamps::mark("p1 cpu ops"); - Self::collect_epoch_from_cpu_ops(cpu_ops, initial_image, register_init, is_final) + Self::collect_epoch_from_cpu_ops(cpu_ops, artifacts, initial_image, register_init, is_final) } /// Phase 2 of [`Self::collect_epoch`], over already-built CPU operations. fn collect_epoch_from_cpu_ops( cpu_ops: Vec, + artifacts: &DecodeArtifacts, initial_image: &I, register_init: &[u32], is_final: bool, @@ -5235,35 +8862,49 @@ impl Traces { shift_ops, bitwise_ops, keccak_ops, + blake3_ops, + blake3_absorb_ops, cpu32_ops, ecsm_ops, ecdas_ops, - hint_ops, copy_ops, - ) = collect_ops_from_cpu(&cpu_ops, &mut memory_state, &mut register_state)?; + hint_ops, + ecrec_ops, + ) = collect_ops_from_cpu( + &cpu_ops, + &artifacts.decode, + &mut memory_state, + &mut register_state, + )?; #[cfg(feature = "instruments")] drop(__sp); + build_stamps::mark("p2a collect"); #[cfg(feature = "instruments")] let __sp = stark::instruments::span("p2b_collect_all"); let ops = collect_all_ops( cpu_ops, + &artifacts.decode, memw_ops, load_ops, lt_ops, shift_ops, bitwise_ops, keccak_ops, + blake3_ops, + blake3_absorb_ops, cpu32_ops, ecsm_ops, ecdas_ops, - hint_ops, copy_ops, + hint_ops, + ecrec_ops, &mut register_state, is_final, ); #[cfg(feature = "instruments")] drop(__sp); + build_stamps::mark("p2b route"); Ok(CollectedEpoch { ops, @@ -5284,8 +8925,13 @@ impl Traces { let initial_image = build_initial_image(elf, &[]); let register_init = register::register_init_from_entry_point(elf.entry_point); let artifacts = DecodeArtifacts::from_elf(elf)?; - let collected = - Self::collect_epoch_from_cpu_ops(cpu_ops, &initial_image, ®ister_init, true)?; + let collected = Self::collect_epoch_from_cpu_ops( + cpu_ops, + &artifacts, + &initial_image, + ®ister_init, + true, + )?; Self::build_from_collected( &artifacts, collected, @@ -5295,8 +8941,6 @@ impl Traces { &[], true, false, - #[cfg(feature = "disk-spill")] - StorageMode::Ram, ) } @@ -5316,14 +8960,12 @@ impl Traces { private_input: &[u8], is_final: bool, l2g_memory_bookend: bool, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, ) -> Result { // Phase 0 (cached): the pristine DECODE trace is cloned so // `build_traces` can fill this epoch's multiplicities. #[cfg(feature = "instruments")] let __sp = stark::instruments::span("p0_decode"); let decode_trace = artifacts.decode_trace.clone(); - let decode_pc_to_row = &artifacts.decode_pc_to_row; #[cfg(feature = "instruments")] drop(__sp); @@ -5336,14 +8978,14 @@ impl Traces { &collected.memory_state, register_init, decode_trace, - decode_pc_to_row, + &artifacts.decode, collected.register_state, max_rows, - #[cfg(feature = "disk-spill")] - storage_mode, private_input, is_final, l2g_memory_bookend, + &StreamSkip::default(), + None, ); #[cfg(feature = "instruments")] drop(__sp); @@ -5362,11 +9004,14 @@ impl Traces { max_rows: &super::MaxRowsConfig, ) -> Result { // Phase 1: Logs → CPU operations - let cpu_ops = collect_cpu_ops(logs, &instructions)?; + let decode_table = DecodeTable::from_instructions(&instructions); + let cpu_ops = collect_cpu_ops(logs, &decode_table)?; // Phase 2: Collect + route all ops let mut memory_state = MemoryState::new(); - let entry_point = cpu_ops.first().map_or(0, |op| op.decode.pc); + let entry_point = cpu_ops + .first() + .map_or(0, |op| decode_table.entry(op.decode_row).pc); let register_init = register::register_init_from_entry_point(entry_point); let mut register_state = RegisterState::new(entry_point); let ( @@ -5376,32 +9021,44 @@ impl Traces { shift_ops, bitwise_ops, keccak_ops, + blake3_ops, + blake3_absorb_ops, cpu32_ops, ecsm_ops, ecdas_ops, - hint_ops, copy_ops, - ) = collect_ops_from_cpu(&cpu_ops, &mut memory_state, &mut register_state)?; + hint_ops, + ecrec_ops, + ) = collect_ops_from_cpu( + &cpu_ops, + &decode_table, + &mut memory_state, + &mut register_state, + )?; let ops = collect_all_ops( cpu_ops, + &decode_table, memw_ops, load_ops, lt_ops, shift_ops, bitwise_ops, keccak_ops, + blake3_ops, + blake3_absorb_ops, cpu32_ops, ecsm_ops, ecdas_ops, - hint_ops, copy_ops, + hint_ops, + ecrec_ops, &mut register_state, true, ); // DECODE (from_elf_and_logs does this in Phase 0; same result either way) - let (decode_trace, decode_pc_to_row) = decode::generate_decode_trace(&instructions); + let (decode_trace, _) = decode::generate_decode_trace(&instructions); // Phases 3-5 (elf=None → empty PAGE tables) build_traces( @@ -5410,14 +9067,913 @@ impl Traces { &memory_state, ®ister_init, decode_trace, - &decode_pc_to_row, + &decode_table, register_state, max_rows, - #[cfg(feature = "disk-spill")] - StorageMode::Ram, &[], true, false, + &StreamSkip::default(), + None, + ) + } +} + +#[cfg(test)] +mod p4_slice_tests { + use super::{ + BitwiseOperation, DvrmOperation, LtOperation, MulOperation, P4Source, + collect_bitwise_from_dvrm, collect_bitwise_from_lt_ops, collect_bitwise_from_mul, + p4_slice_len, + }; + use std::collections::HashMap; + + fn counts(ops: &[BitwiseOperation], into: &mut HashMap) { + for op in ops { + *into.entry(*op).or_default() += 1; + } + } + + /// Counts `ops` whole, and slice by slice at phase 4's length for `source`. + fn whole_and_sliced( + ops: &[T], + source: P4Source, + mul_chunk: usize, + dvrm_chunk: usize, + collect: impl Fn(&[T]) -> Vec, + ) -> ( + HashMap, + HashMap, + usize, + ) { + let mut whole = HashMap::new(); + counts(&collect(ops), &mut whole); + let mut sliced = HashMap::new(); + let mut slices = 0; + for slice in ops.chunks(p4_slice_len(source, mul_chunk, dvrm_chunk)) { + counts(&collect(slice), &mut sliced); + slices += 1; + } + (whole, sliced, slices) + } + + /// Phase 4's slices count every lookup the whole sources count. MUL and DVRM deduplicate + /// their signed lookups per instance, so their slices must be whole instances: the ops + /// below repeat inside and across instances, so a cut one row off (checked here) changes + /// the counts, and a `p4_slice_len` that misaligns them fails the equality. LT is a sum + /// over its ops, cut across its 2^20-op slices. + #[test] + fn phase_4_slices_count_what_the_whole_sources_count() { + const CHUNK: usize = 8; + let mul: Vec<(MulOperation, bool)> = (0..203u64) + .map(|i| { + // Three distinct signed ops, repeating: each instance deduplicates its own. + let op = MulOperation { + lhs: (i % 3 + 1) << 61 | 5, + lhs_signed: true, + rhs: 3 << 61 | 7, + rhs_signed: true, + }; + (op, i % 3 == 0) + }) + .collect(); + let (whole, sliced, slices) = whole_and_sliced(&mul, P4Source::Mul, CHUNK, CHUNK, |s| { + collect_bitwise_from_mul(s, CHUNK) + }); + assert!(slices > 1); + assert_eq!(whole, sliced, "MUL"); + let mut off = HashMap::new(); + for slice in mul.chunks(CHUNK + 1) { + counts(&collect_bitwise_from_mul(slice, CHUNK), &mut off); + } + assert_ne!(whole, off, "MUL's ops do not tell a misaligned cut apart"); + + let dvrm: Vec<(DvrmOperation, bool)> = (0..197u64) + .map(|i| { + let op = DvrmOperation { + n: (i % 3 + 1) << 61 | 9, + d: 5 << 60 | 3, + signed: true, + }; + (op, i % 2 == 0) + }) + .collect(); + let (whole, sliced, slices) = whole_and_sliced(&dvrm, P4Source::Dvrm, CHUNK, CHUNK, |s| { + collect_bitwise_from_dvrm(s, CHUNK) + }); + assert!(slices > 1); + assert_eq!(whole, sliced, "DVRM"); + let mut off = HashMap::new(); + for slice in dvrm.chunks(CHUNK + 1) { + counts(&collect_bitwise_from_dvrm(slice, CHUNK), &mut off); + } + assert_ne!(whole, off, "DVRM's ops do not tell a misaligned cut apart"); + + let lt: Vec = (0..(1u64 << 20) + 37) + .map(|i| LtOperation { + lhs: i.wrapping_mul(0x9e37_79b9_7f4a_7c15), + rhs: i.rotate_left(17), + signed: i % 2 == 0, + invert: i % 3 == 0, + }) + .collect(); + let (whole, sliced, slices) = + whole_and_sliced(<, P4Source::Lt, CHUNK, CHUNK, collect_bitwise_from_lt_ops); + assert_eq!(slices, 2); + assert_eq!(whole, sliced, "LT"); + } +} + +#[cfg(test)] +mod kr_wave_tests { + use super::{ + CutPadding, KECCAK_RND_PACK_WAVE, KeccakRoundOperation, TraceForm, keccak_rnd, + keccak_rnd_chunks_in_waves, kr_packed_wave_from, + }; + + /// `count` ops with distinct states, each 24 rows. + fn ops(count: usize) -> Vec { + (0..count as u64) + .map(|i| KeccakRoundOperation { + timestamp: 4 * i + 1, + input: std::array::from_fn(|l| { + (i + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ ((l as u64) << 7) + }), + output: std::array::from_fn(|l| { + (i + 3).wrapping_mul(0xc2b2_ae3d_27d4_eb4f) ^ l as u64 + }), + }) + .collect() + } + + /// A table's main trace as words, row-major, whether it is held packed or wide. + fn words( + table: &stark::trace::TraceTable, + ) -> Vec { + match table.narrow_main() { + Some(packed) => { + let columns: Vec> = (0..packed.cols()).map(|c| packed.column(c)).collect(); + (0..packed.rows()) + .flat_map(|r| columns.iter().map(move |column| column[r])) + .collect() + } + None => { + let main = &table.main_table; + (0..main.height) + .flat_map(|r| (0..main.width).map(move |c| *main.get(r, c).value())) + .collect() + } + } + } + + /// The finish's KECCAK_RND tables are the same tables, in the same order, however many of + /// the chunks written packed go at once: the wave moves time, never a word. + #[test] + fn keccak_rnd_chunks_are_the_same_at_every_packed_wave() { + // 21 ops = 504 rows in a 512-row table: eight chunks of 64 rows, the last part padding, + // and ops cut across chunk edges. + let ops = ops(21); + let padding = CutPadding::default(); + let reference = keccak_rnd_chunks_in_waves( + &ops, + 64, + 0, + padding, + true, + TraceForm::Narrow, + KECCAK_RND_PACK_WAVE, + ) + .expect("the chunks"); + assert_eq!(reference.len(), 8); + for wave in [1, 3, usize::MAX] { + let tables = + keccak_rnd_chunks_in_waves(&ops, 64, 0, padding, true, TraceForm::Narrow, wave) + .expect("the chunks"); + assert_eq!(tables.len(), reference.len(), "wave {wave}"); + for (i, (got, want)) in tables.iter().zip(&reference).enumerate() { + assert_eq!( + got.narrow_main().map(|p| p.widths().to_vec()), + want.narrow_main().map(|p| p.widths().to_vec()), + "wave {wave}, chunk {i}: widths" + ); + assert_eq!(words(got), words(want), "wave {wave}, chunk {i}"); + } + } + // The wide build is the same words too. + let wide = + keccak_rnd_chunks_in_waves(&ops, 64, 0, padding, false, TraceForm::Wide, usize::MAX) + .expect("the chunks"); + for (i, (got, want)) in wide.iter().zip(&reference).enumerate() { + assert_eq!(words(got), words(want), "wide, chunk {i}"); + } + // Once a KECCAK_RND trace was written in this process, the next is written packed + // (the wave's switch), except where every trace is wide. + assert_eq!( + keccak_rnd::rows_written_packed(TraceForm::Narrow), + !cfg!(feature = "debug-checks") + ); + assert!(!keccak_rnd::rows_written_packed(TraceForm::Wide)); + } + + /// The default builds every chunk written packed in one pass; `4` is the opt-out to the + /// wave as it was. + #[test] + fn the_packed_wave_is_all_by_default_and_four_opts_out() { + assert_eq!(kr_packed_wave_from(None), usize::MAX); + assert_eq!(kr_packed_wave_from(Some("0")), usize::MAX); + assert_eq!(kr_packed_wave_from(Some("all")), usize::MAX); + assert_eq!(kr_packed_wave_from(Some("-1")), usize::MAX); + assert_eq!(KECCAK_RND_PACK_WAVE, 4); + assert_eq!(kr_packed_wave_from(Some("4")), KECCAK_RND_PACK_WAVE); + assert_eq!(kr_packed_wave_from(Some(" 16 ")), 16); + } +} + +#[cfg(test)] +mod segmented_tests { + use super::*; + + type Table = TraceTable; + + /// One column holding the ops, padded to a power of two. + fn table_of(ops: &[u64]) -> Table { + let rows = ops.len().next_power_of_two().max(4); + let mut data = crate::tables::types::zeroed_fe_vec(rows); + for (cell, &op) in data.iter_mut().zip(ops) { + *cell = crate::tables::types::FE::from(op); + } + TraceTable::new_main(data, 1, 1) + } + + fn words(tables: &[Table]) -> Vec<(usize, Vec)> { + tables + .iter() + .map(|t| { + ( + t.main_table.width, + t.main_table + .row_major_data() + .iter() + .map(|v| v.canonical()) + .collect(), + ) + }) + .collect() + } + + /// LT ops with every kind of delta: increasing and decreasing `rhs`, `lhs` + /// above and below it, both ends of `u64`, and runs sharing one `rhs` (a + /// MEMW op's checks). + fn lt_ops(n: usize) -> Vec { + let mut x = 0x9e37_79b9_7f4a_7c15u64; + let mut rhs = 1000u64; + (0..n) + .map(|i| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + let (lhs, rhs) = match i % 7 { + 0 => (0, u64::MAX), + 1 => (u64::MAX, 0), + 2 => (x, x >> 3), + 3 | 4 => { + rhs += x % 9; + (rhs - x % 1000, rhs) + } + 5 => (rhs - x % (1 << 40), rhs), + _ => (x >> 1, rhs), + }; + LtOperation::new(lhs, rhs, false) + }) + .collect() + } + + fn same_lt(a: &[LtOperation], b: &[LtOperation]) -> bool { + a.len() == b.len() + && a.iter().zip(b).all(|(x, y)| { + (x.lhs, x.rhs, x.signed, x.invert) == (y.lhs, y.rhs, y.signed, y.invert) + }) + } + + /// The compact LT list expands to exactly the ops pushed, over any range: + /// within one index block, across blocks, from the start, to the end, past + /// it, and empty; pushed all at once or in pieces. The raw list (the A arm) + /// expands to them too. + #[test] + fn compact_lt_ops_expand_to_what_was_pushed() { + let n = 3 * COMPACT_LT_BLOCK + 123; + let ops = lt_ops(n); + let mut whole = CompactLt::default(); + whole.extend(&ops); + let mut pieces = CompactLt::default(); + for piece in ops.chunks(1000) { + pieces.extend(piece); + } + let mut raw = CompactLt::raw(); + for piece in ops.chunks(777) { + raw.extend(piece); + } + // The stream cut into blocks of a few ops: an op never straddles two, + // and every range crosses some. + let mut blocked = CompactLt::with_block_bytes(64); + for piece in ops.chunks(333) { + blocked.extend(piece); + } + assert!( + blocked.bytes.len() > n / 8, + "{} blocks", + blocked.bytes.len() + ); + assert!( + blocked + .bytes + .iter() + .all(|b| b.len() <= 64 && b.capacity() <= 64) + ); + assert!(blocked.largest_bytes() <= 64.max(blocked.index.capacity() * 16)); + assert!(whole.heap_bytes() < n * std::mem::size_of::()); + assert!(raw.heap_bytes() >= n * std::mem::size_of::()); + for compact in [&whole, &pieces, &raw, &blocked] { + assert_eq!(Expand::len(compact), n); + for (start, end) in [ + (0, n), + (0, 1), + (5, 17), + (COMPACT_LT_BLOCK - 3, COMPACT_LT_BLOCK + 3), + (COMPACT_LT_BLOCK, 2 * COMPACT_LT_BLOCK), + (2 * COMPACT_LT_BLOCK + 1, n), + (n - 1, n + 50), + (7, 7), + (n + 3, n + 9), + ] { + let mut out = Vec::new(); + compact.expand_into(start, end, &mut out); + assert!( + same_lt(&out, &ops[start.min(n)..end.min(n)]), + "{start}..{end}" + ); + } + } + // Empty lists expand to nothing. + for empty in [CompactLt::default(), CompactLt::raw()] { + assert_eq!(Expand::len(&empty), 0); + let mut out = Vec::new(); + empty.expand_into(0, 10, &mut out); + assert!(out.is_empty()); + } + } + + /// LT generated as a table per chunk, its first column the lhs and its + /// second the rhs. + fn lt_table(ops: &[LtOperation]) -> Table { + let rows = ops.len().next_power_of_two().max(4); + let mut data = crate::tables::types::zeroed_fe_vec(rows * 2); + for (r, op) in ops.iter().enumerate() { + data[2 * r] = crate::tables::types::FE::from(op.lhs); + data[2 * r + 1] = crate::tables::types::FE::from(op.rhs); + } + TraceTable::new_main(data, 2, 1) + } + + /// A list with compact segments chunks as its concatenation, as the + /// phase-3 LT list does with its MEMW-derived segments kept compact. + #[test] + fn a_list_with_compact_segments_chunks_as_its_concatenation() { + let all = lt_ops(2 * COMPACT_LT_BLOCK + 77); + let (a, b, c) = ( + &all[..100], + &all[100..COMPACT_LT_BLOCK + 300], + &all[COMPACT_LT_BLOCK + 300..], + ); + let mut compact_b = CompactLt::default(); + compact_b.extend(b); + let mut compact_c = CompactLt::with_block_bytes(40); + compact_c.extend(c); + let segmented = Segmented { + parts: vec![ + Part::Ops(a), + Part::Compact(&compact_b), + Part::Ops(&[]), + Part::Compact(&compact_c), + ], + }; + for max in [1, 64, 1000, COMPACT_LT_BLOCK, 1 << 20] { + for (skip, tails) in [(0, false), (1, false)] { + let whole = chunk_and_generate_skipping(&all, max, skip, tails, true, lt_table) + .expect("whole"); + let split = + chunk_and_generate_segmented(&segmented, max, skip, tails, true, lt_table) + .expect("segmented"); + assert_eq!(words(&whole), words(&split), "max {max} skip {skip}"); + } + } + } + + /// Chunked from its segments, a list gives the tables its concatenation + /// gives: every split of the segments (empty ones too), chunk sizes that + /// cut inside and across them, with and without chunks streamed ahead + /// (tails or whole lists), optional or not, and an empty list. + #[test] + fn a_segmented_list_chunks_as_its_concatenation() { + let all: Vec = (1..=37).collect(); + let layouts: [&[usize]; 4] = [&[37], &[0, 10, 0, 27], &[5, 5, 5, 22], &[1, 36, 0]]; + for layout in layouts { + let mut parts = Vec::new(); + let mut at = 0; + for &n in layout { + parts.push(Part::Ops(&all[at..at + n])); + at += n; + } + let segmented = Segmented { parts }; + for max in [1, 4, 5, 10, 37, 100] { + for (skip, tails) in [(0, false), (2, true), (2, false)] { + for optional in [true, false] { + // With `tails`, the list is what is left past the + // streamed chunks: the same list either way here. + let whole = + chunk_and_generate_skipping(&all, max, skip, tails, optional, table_of); + let split = chunk_and_generate_segmented( + &segmented, max, skip, tails, optional, table_of, + ); + match (whole, split) { + (Ok(a), Ok(b)) => assert_eq!( + words(&a), + words(&b), + "{layout:?} max {max} skip {skip} tails {tails} optional {optional}" + ), + (Err(_), Err(_)) => {} + (a, b) => panic!( + "{layout:?} max {max} skip {skip}: {:?} vs {:?}", + a.is_ok(), + b.is_ok() + ), + } + } + } + } + } + let empty: Segmented<'_, u64> = Segmented { + parts: vec![Part::Ops(&[]), Part::Ops(&[])], + }; + for optional in [true, false] { + let a = chunk_and_generate_skipping(&[] as &[u64], 4, 0, false, optional, table_of) + .expect("whole"); + let b = chunk_and_generate_segmented(&empty, 4, 0, false, optional, table_of) + .expect("segmented"); + assert_eq!(words(&a), words(&b), "empty, optional {optional}"); + } + } +} + +#[cfg(all(test, feature = "parallel"))] +mod cut_tests { + use super::cut_padding_tests::is_today_without_its_padding_tables; + use super::{ + BlockVec, CutPadding, Error, GoldilocksExtension, GoldilocksField, KeccakOperation, + TraceForm, cut_tables, ecdas, ecsm, keccak, + }; + use stark::lookup::BusInteraction; + use stark::trace::TraceTable; + + type Table = TraceTable; + + /// A table's main trace as words, row-major, whether it is held packed or wide. + fn words(table: &Table) -> Vec { + match table.narrow_main() { + Some(packed) => { + let columns: Vec> = (0..packed.cols()).map(|c| packed.column(c)).collect(); + (0..packed.rows()) + .flat_map(|r| columns.iter().map(move |column| column[r])) + .collect() + } + None => { + let main = &table.main_table; + (0..main.height) + .flat_map(|r| (0..main.width).map(move |c| *main.get(r, c).value())) + .collect() + } + } + } + + /// Distinct bytes a value `i` and a field `f`, so a row in the wrong place differs. + fn bytes(i: usize, f: u64) -> [u8; N] { + std::array::from_fn(|b| { + ((i as u64 + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ f.wrapping_mul(31) ^ b as u64) + as u8 + }) + } + + /// In-range carries (the generators' debug checks read them). + fn carries(i: usize) -> [i64; 64] { + std::array::from_fn(|k| ((i + k) % 7) as i64 - 3) + } + + fn keccak_ops(n: usize) -> Vec { + (0..n) + .map(|i| KeccakOperation { + timestamp: 4 * i as u64 + 1, + state_addr: 0x1000 + 0x100 * i as u64, + input: std::array::from_fn(|l| (i as u64 + 1).wrapping_mul(0x9e37_79b9) ^ l as u64), + output: std::array::from_fn(|l| { + (i as u64 + 7).wrapping_mul(0xc2b2_ae3d) ^ l as u64 + }), + sponge: None, + }) + .collect() + } + + fn ecdas_step(i: usize) -> ::ecsm::EcdasStep { + ::ecsm::EcdasStep { + x_a: bytes(i, 1), + y_a: bytes(i, 2), + x_g: bytes(i, 3), + y_g: bytes(i, 4), + round: (i % 256) as u8, + op: (i % 2) as u8, + next_op: ((i + 1) % 2) as u8, + lambda: bytes(i, 5), + x_r: bytes(i, 6), + y_r: bytes(i, 7), + q0: bytes(i, 8), + q1: bytes(i, 9), + q2: bytes(i, 10), + c0: carries(i), + c1: carries(i + 1), + c2: carries(i + 2), + } + } + + fn ecdas_ops(n: usize) -> Vec { + (0..n) + .map(|i| ecdas::EcdasOperation { + timestamp: 4 * i as u64 + 3, + step: ecdas_step(i), + }) + .collect() + } + + fn ecsm_ops(n: usize) -> Vec { + (0..n) + .map(|i| ecsm::EcsmOperation { + timestamp: 4 * i as u64 + 2, + addr_xg: 0x2000 + 0x40 * i as u64, + addr_k: 0x3000 + 0x40 * i as u64, + addr_xr: 0x4000 + 0x40 * i as u64, + witness: ::ecsm::EcsmWitness { + x_g: bytes(i, 11), + y_g: bytes(i, 12), + k: bytes(i, 13), + x2: bytes(i, 14), + q0: bytes(i, 15), + c0: carries(i), + q1: bytes(i, 16), + c1: carries(i + 3), + x_g_sub_p: bytes(i, 17), + k_sub_n: bytes(i, 18), + x_r_sub_p: bytes(i, 19), + len_k: (i % 256) as u8, + x_r: bytes(i, 20), + y_r: bytes(i, 21), + steps: Vec::new(), + }, + }) + .collect() + } + + /// `cut_tables` against the block's split of the whole wide table, for one + /// family: every op count around the cut sizes, every cut size from one row + /// to past the whole table, built wide, packed after the build, and + /// written packed. With the whole run padded, the same tables, word for + /// word, in the same order; cut at the last real row, those tables without + /// the ones of padding rows alone, which carry nothing on the family's + /// bus `interactions`. + fn cuts_are_the_split( + name: &str, + ops: impl Fn(usize) -> Vec, + whole: impl Fn(&[T]) -> Table, + written_packed: fn(TraceForm) -> bool, + rows_as: impl Fn(&[T], usize, TraceForm) -> Table + Send + Sync + Copy, + interactions: &[BusInteraction], + ) { + let whole_run = CutPadding::WholeRun; + let mut left_out = 0usize; + for n in [1usize, 2, 3, 4, 5, 7, 8, 9, 16, 17, 31, 33, 64, 65] { + let ops = ops(n); + // The ops in one block, and in blocks of three that cuts straddle. + let one = BlockVec::from_vec(ops.clone()); + let mut threes = BlockVec::new(3); + threes.extend(ops.iter().cloned()); + for log2 in 0..=7 { + let rows = 1usize << log2; + let split = crate::block_whir::split_rows(whole(&ops), rows); + for (pack, form, blocks) in [ + (false, TraceForm::Wide, &one), + (true, TraceForm::Wide, &threes), + (true, TraceForm::Narrow, &one), + (true, TraceForm::Narrow, &threes), + ] { + let cut = + cut_tables(blocks, rows, whole_run, pack, form, written_packed, rows_as) + .expect("the cuts"); + assert_eq!( + cut.len(), + split.len(), + "{name} n {n} rows {rows}: table count" + ); + for (k, (got, want)) in cut.iter().zip(&split).enumerate() { + assert_eq!( + got.num_rows(), + want.num_rows(), + "{name} n {n} rows {rows} cut {k}: height" + ); + assert_eq!( + got.narrow_main().is_some(), + pack, + "{name} n {n} rows {rows} cut {k}: packed" + ); + assert!( + words(got) == words(want), + "{name} n {n} rows {rows} cut {k} (pack {pack}, {form:?}): words" + ); + } + let last = cut_tables( + blocks, + rows, + CutPadding::LastTable, + pack, + form, + written_packed, + rows_as, + ) + .expect("the cuts"); + assert_eq!( + last.len(), + CutPadding::LastTable.tables(n, rows), + "{name} n {n} rows {rows}: tables with a real row" + ); + is_today_without_its_padding_tables(&last, &cut, interactions).unwrap_or_else( + |e| panic!("{name} n {n} rows {rows} (pack {pack}, {form:?}): {e}"), + ); + left_out += cut.len() - last.len(); + } + } + } + assert!( + left_out > 0, + "{name}: no table of padding rows alone was left out" + ); + for padding in [whole_run, CutPadding::LastTable] { + // No ops, no table, as the whole build's `generate_optional`. + assert!( + cut_tables( + &BlockVec::from_vec(ops(0)), + 4, + padding, + true, + TraceForm::Narrow, + written_packed, + rows_as + ) + .expect("no cuts") + .is_empty() + ); + // A cut that is not a power of two is refused, not split some other way. + assert!(matches!( + cut_tables( + &BlockVec::from_vec(ops(9)), + 6, + padding, + true, + TraceForm::Narrow, + written_packed, + rows_as + ), + Err(Error::Prover(_)) + )); + } + } + + #[test] + fn keccak_cuts_are_the_split_of_the_whole_table() { + cuts_are_the_split( + "KECCAK", + keccak_ops, + keccak::generate_keccak_trace, + keccak::rows_written_packed, + keccak::generate_keccak_rows_as, + &keccak::bus_interactions(), + ); + } + + #[test] + fn ecsm_cuts_are_the_split_of_the_whole_table() { + cuts_are_the_split( + "ECSM", + ecsm_ops, + ecsm::generate_ecsm_trace, + ecsm::rows_written_packed, + ecsm::generate_ecsm_rows_as, + &ecsm::bus_interactions(), + ); + } + + /// Phase 4 counts a compact ECDAS row's lookups as the op's. + #[test] + fn compact_ecdas_rows_count_the_ops_lookups() { + let wide = ecdas_ops(37); + let compact: Vec = + wide.iter().map(ecdas::CompactEcdasOp::from_op).collect(); + let mut want = Vec::new(); + for op in &wide { + super::ecdas_lookups(op, &mut want); + } + assert!(!want.is_empty()); + assert_eq!(super::collect_bitwise_from_compact_ecdas(&compact), want); + } + + #[test] + fn ecdas_cuts_are_the_split_of_the_whole_table() { + cuts_are_the_split( + "ECDAS", + ecdas_ops, + ecdas::generate_ecdas_trace, + ecdas::rows_written_packed, + ecdas::generate_ecdas_rows_as, + &ecdas::bus_interactions(), + ); + } +} + +#[cfg(test)] +mod ecrec_ops_tests { + use super::{CpuOperation, MemoryState, RegisterState, collect_ecrec_ops}; + + /// An ECREC call's ECDAS rows are its witness's first chain at `T`, second chain + /// at `T + 1` and final addition at `T + 2`; the ECREC op keeps none. Its MEMW + /// ops are the 25 the table sends (four register reads, eight signature reads, + /// four hash reads, nine output writes), and the written bytes are the key + /// big-endian, then status 0. + #[test] + fn an_ecrec_calls_rows_are_keyed_at_t_t1_t2() { + let hex = |s: &str| -> Vec { + (0..s.len() / 2) + .map(|i| u8::from_str_radix(&s[2 * i..2 * i + 2], 16).unwrap()) + .collect() + }; + let sig: Vec = hex(concat!( + "3284bae4a1b693dfbe8ac67400e4891f3b53681c5159a315561051f3d2c57139", + "97d4b2f2f728a54ed562a12237f6b11d32d626e7fcc24c3470b0729e095bd057" + )); + let msg = hex("c0ffee0011223344556677889900aabbccddeeff00112233445566778899aabb"); + let key = hex(concat!( + "8733750fdf82edb525a2c9f9d201ad1b26e163b8093f3c8f6eb83377d157edb2", + "ce023840e38085bfcf2b27bd66be6a877f0ec281b073ca0f9ab31cfc0733b62a" + )); + let (addr_out, addr_sig, addr_msg) = (0x1000u64, 0x2000u64, 0x3000u64); + let mut memory_state = MemoryState::new(); + let mut register_state = RegisterState::new(0); + register_state.write(10, addr_out, 1); + register_state.write(11, addr_sig, 1); + register_state.write(12, addr_msg, 1); + register_state.write(13, 0, 1); + for (base, bytes) in [(addr_sig, &sig), (addr_msg, &msg)] { + for (i, chunk) in bytes.chunks_exact(8).enumerate() { + let dword = u64::from_le_bytes(chunk.try_into().unwrap()); + memory_state.write_bytes(base + 8 * i as u64, dword, 8, 1); + } + } + let t = 400; + let decode = crate::tables::types::DecodeEntry::default(); + let op = CpuOperation { + timestamp: t, + ..Default::default() + }; + let (memw, ecrec_op, ecdas_ops) = collect_ecrec_ops( + super::CpuOp::new(&decode, &op), + &mut memory_state, + &mut register_state, ) + .expect("the operands are held"); + assert_eq!(memw.len(), 25); + let w = &ecrec_op.witness; + assert!(w.ok && w.chain1.is_empty() && w.chain2.is_empty() && w.add.is_none()); + let fresh = + ::ecsm::compute_ecrec_witness(&sig.try_into().unwrap(), &msg.try_into().unwrap(), 0); + let (n1, n2) = (fresh.chain1.len(), fresh.chain2.len()); + assert_eq!(ecdas_ops.len(), n1 + n2 + 1); + assert!(ecdas_ops[..n1].iter().all(|o| o.timestamp == t)); + assert!(ecdas_ops[n1..n1 + n2].iter().all(|o| o.timestamp == t + 1)); + assert_eq!(ecdas_ops[n1 + n2].timestamp, t + 2); + assert_eq!( + format!("{:?}", ecdas_ops[n1 + n2].step), + format!("{:?}", fresh.add.unwrap()) + ); + let written: Vec = (0..9) + .flat_map(|i| { + memory_state + .read_bytes(addr_out + 8 * i, 8) + .0 + .iter() + .map(|&b| b as u8) + .collect::>() + }) + .collect(); + assert_eq!(&written[..64], &key[..]); + assert_eq!(&written[64..], &[0u8; 8]); + } +} + +#[cfg(test)] +mod ecsm_steps_tests { + use super::{ + CpuOperation, GoldilocksExtension, GoldilocksField, MemoryState, RegisterState, + collect_ecsm_ops, ecdas, ecsm, + }; + use stark::trace::TraceTable; + + /// A table's main trace as words, row-major. + fn words(table: &TraceTable) -> Vec { + let main = &table.main_table; + (0..main.height) + .flat_map(|r| (0..main.width).map(move |c| *main.get(r, c).value())) + .collect() + } + + /// An ECSM call's ECDAS rows are its witness's steps, moved out of the ECSM + /// op: the ECDAS table of the ops the builder collects is the table of the + /// steps a fresh witness computes, the ECSM table is the fresh witness's, + /// and the ECSM op keeps no steps. + #[test] + fn an_ecsm_calls_ecdas_rows_are_its_witness_steps() { + let mut gx = [ + 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, + 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, + 0x16, 0xF8, 0x17, 0x98, + ]; + gx.reverse(); + let mut big = [0xA5u8; 32]; + big[31] = 0x7F; + let scalars: Vec<[u8; 32]> = [1u64, 2, 5, 0xFFFF, 1_000_003] + .iter() + .map(|v| { + let mut k = [0u8; 32]; + k[..8].copy_from_slice(&v.to_le_bytes()); + k + }) + .chain([big]) + .collect(); + let (addr_xr, addr_xg, addr_k) = (0x1000u64, 0x2000u64, 0x3000u64); + for (i, k) in scalars.iter().enumerate() { + let mut memory_state = MemoryState::new(); + let mut register_state = RegisterState::new(0); + register_state.write(10, addr_xr, 1); + register_state.write(11, addr_xg, 1); + register_state.write(12, addr_k, 1); + for d in 0..4 { + let dword = |bytes: &[u8; 32]| { + u64::from_le_bytes(bytes[8 * d..8 * d + 8].try_into().expect("8 bytes")) + }; + memory_state.write_bytes(addr_xg + 8 * d as u64, dword(&gx), 8, 1); + memory_state.write_bytes(addr_k + 8 * d as u64, dword(k), 8, 1); + } + let t = 100 + 10 * i as u64; + let decode = crate::tables::types::DecodeEntry::default(); + let op = CpuOperation { + timestamp: t, + ..Default::default() + }; + let (_, ecsm_op, ecdas_ops) = collect_ecsm_ops( + super::CpuOp::new(&decode, &op), + &mut memory_state, + &mut register_state, + ) + .expect("the operands are held"); + let fresh = ::ecsm::compute_witness(k, &gx).expect("a valid scalar and point"); + assert!(ecsm_op.witness.steps.is_empty(), "scalar {i}: steps kept"); + assert_eq!(ecdas_ops.len(), fresh.steps.len(), "scalar {i}"); + let want: Vec = fresh + .steps + .iter() + .cloned() + .map(|step| ecdas::EcdasOperation { timestamp: t, step }) + .collect(); + assert_eq!( + words(&ecdas::generate_ecdas_trace(&ecdas_ops)), + words(&ecdas::generate_ecdas_trace(&want)), + "scalar {i}: ECDAS rows" + ); + let whole = ecsm::EcsmOperation { + timestamp: t, + addr_xg, + addr_k, + addr_xr, + witness: fresh, + }; + assert_eq!( + words(&ecsm::generate_ecsm_trace(&[ecsm_op])), + words(&ecsm::generate_ecsm_trace(&[whole])), + "scalar {i}: ECSM row" + ); + } } } diff --git a/prover/src/tables/trace_builder/blocks.rs b/prover/src/tables/trace_builder/blocks.rs new file mode 100644 index 000000000..a8e4e5b14 --- /dev/null +++ b/prover/src/tables/trace_builder/blocks.rs @@ -0,0 +1,317 @@ +//! Lists appended to window by window without ever moving what they hold. +//! +//! A builder list grown with `Vec::extend` doubles its capacity when it fills: +//! a new buffer of twice the size, the old one copied over, then freed. At the +//! end of a large block's walk that is one allocation as large as the list and +//! a multi-GiB copy on the accumulator thread (RYZEN 015 at 195 Mgas: +24.2 GiB +//! live in one step, into the cgroup's limit). A [`BlockVec`] stores its +//! elements in blocks of a fixed length instead: the first grows as a `Vec` +//! does up to that length, every later one is allocated whole, and a full +//! block is never touched again. A block holds at most [`BLOCK_BYTES`] (or a +//! table's chunk, when that is smaller), so no single allocation, and no +//! copy, is larger. The elements and their order are the list's: readers see +//! it as the concatenation of its blocks ([`Segmented`]), and a chunk the +//! block length divides lies in one block. + +use std::borrow::Cow; +use std::collections::VecDeque; + +use super::{Part, Segmented}; + +/// The most bytes one block holds (rounded down to a power-of-two element +/// count): at or above the allocator's 8 MiB class, whose freed extents other +/// threads reuse. +pub(super) const BLOCK_BYTES: usize = 64 << 20; + +/// Elements a block of `T` holds for a table chunked at `chunk` rows: the +/// largest power of two that fits [`BLOCK_BYTES`], and no more than `chunk`, +/// so that a power-of-two chunk is whole blocks (one block at the production +/// chunks for ops of ≤ 32 bytes). At least one. +pub(super) fn block_len(chunk: usize) -> usize { + let fits = (BLOCK_BYTES / std::mem::size_of::().max(1)).max(1); + let pow2 = 1usize << (usize::BITS - 1 - fits.leading_zeros()); + pow2.min(chunk.max(1)) +} + +/// A list stored in blocks of `block` elements: see the module docs. +pub(super) struct BlockVec { + blocks: VecDeque>, + /// Elements in a full block. + block: usize, + len: usize, +} + +impl BlockVec { + /// An empty list of `block`-element blocks (at least one). + pub(super) fn new(block: usize) -> Self { + Self { + blocks: VecDeque::new(), + block: block.max(1), + len: 0, + } + } + + /// `v` as a list of one block, as it is (no copy): a list built whole. + pub(super) fn from_vec(v: Vec) -> Self { + let len = v.len(); + let mut blocks = VecDeque::new(); + if len > 0 { + blocks.push_back(v); + } + Self { + blocks, + block: len.max(1), + len, + } + } + + pub(super) fn len(&self) -> usize { + self.len + } + + pub(super) fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Appends `x`: into the last block while it has room (growing it, up to + /// the block length, as a `Vec` grows), else into a new block allocated + /// whole. Nothing already held moves once its block is full. + pub(super) fn push(&mut self, x: T) { + match self.blocks.back_mut() { + Some(last) if last.len() < self.block => { + if last.len() == last.capacity() { + let want = (last.capacity() * 2).max(16).min(self.block); + last.reserve_exact(want - last.len()); + } + last.push(x); + } + _ => { + // The first block starts small; once one block is full the + // list is a long one, and each next block is allocated whole. + let capacity = if self.blocks.is_empty() { + 16.min(self.block) + } else { + self.block + }; + let mut block = Vec::with_capacity(capacity); + block.push(x); + self.blocks.push_back(block); + } + } + self.len += 1; + } + + /// Appends every element of `items`, in order. + pub(super) fn extend(&mut self, items: impl IntoIterator) { + for x in items { + self.push(x); + } + } + + /// `other`'s elements after this list's, by moving its blocks (a list + /// assembled from segments; no element is copied). + pub(super) fn chain(mut self, other: Self) -> Self { + self.len += other.len; + self.blocks.extend(other.blocks); + self + } + + /// The blocks, in order: the list is their concatenation. + pub(super) fn parts(&self) -> impl Iterator { + self.blocks.iter().map(Vec::as_slice) + } + + /// The elements, in order. + pub(super) fn iter(&self) -> impl DoubleEndedIterator { + self.blocks.iter().flat_map(|block| block.iter()) + } + + /// The list as the segments it is the concatenation of. + pub(super) fn segments(&self) -> Segmented<'_, T> { + Segmented { + parts: self.parts().map(Part::Ops).collect(), + } + } + + /// The first `n` elements (at most the list), taken out: the front block + /// as it is when it holds exactly `n` (a chunk the block length is), + /// copied out of the front blocks otherwise. + pub(super) fn take_front(&mut self, n: usize) -> Vec { + let n = n.min(self.len); + self.len -= n; + if self.blocks.front().is_some_and(|front| front.len() == n) { + return self.blocks.pop_front().unwrap_or_default(); + } + let mut out = Vec::with_capacity(n); + while out.len() < n { + let Some(front) = self.blocks.front_mut() else { + break; + }; + let want = n - out.len(); + if front.len() <= want { + out.append(front); + self.blocks.pop_front(); + } else { + out.extend(front.drain(..want)); + } + } + out + } + + /// The bytes its blocks take on the heap (capacities). + pub(super) fn heap_bytes(&self) -> usize { + self.blocks.iter().map(Vec::capacity).sum::() * std::mem::size_of::() + + self.blocks.capacity() * std::mem::size_of::>() + } + + /// The bytes its largest single allocation takes (a block's capacity). + pub(super) fn largest_bytes(&self) -> usize { + self.blocks.iter().map(Vec::capacity).max().unwrap_or(0) * std::mem::size_of::() + } +} + +impl BlockVec { + /// Elements `start..end` (clamped to the list): borrowed when one block + /// holds them, copied otherwise. + pub(super) fn range(&self, start: usize, end: usize) -> Cow<'_, [T]> { + let end = end.min(self.len); + if start >= end { + return Cow::Borrowed(&[]); + } + self.segments().range(start, end) + } + + /// The whole list as one slice: borrowed when it is one block, copied + /// otherwise. + pub(super) fn whole(&self) -> Cow<'_, [T]> { + self.range(0, self.len) + } + + /// The elements as one `Vec`. + #[cfg(test)] + pub(super) fn to_vec(&self) -> Vec { + self.whole().into_owned() + } + + /// The elements as one `Vec`: the block itself when there is one, the + /// blocks concatenated (each freed as it is copied) otherwise. + pub(super) fn into_vec(mut self) -> Vec { + if self.blocks.len() <= 1 { + return self.blocks.pop_front().unwrap_or_default(); + } + let mut out = Vec::with_capacity(self.len); + while let Some(mut block) = self.blocks.pop_front() { + out.append(&mut block); + } + out + } +} + +impl Default for BlockVec { + /// An empty list of single-element blocks: a list nothing is appended to. + fn default() -> Self { + Self::new(1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Pushed one at a time, or appended a window at a time, a list holds the + /// elements in order, every block but the last full at the block length, + /// no block larger; and any range, and chunk k of any chunk size, is the + /// `Vec`'s. Empty lists, exactly full blocks and one-element blocks too. + #[test] + fn a_block_list_is_the_vec_it_was_appended() { + for block in [1usize, 2, 3, 7, 16, 64] { + for n in [ + 0usize, + 1, + block - 1, + block, + block + 1, + 2 * block, + 5 * block + 3, + 1000, + ] { + let want: Vec = (0..n as u64).map(|i| (i * 0x9e37_79b9) ^ 5).collect(); + let mut pushed = BlockVec::new(block); + for &x in &want { + pushed.push(x); + } + let mut windows = BlockVec::new(block); + for window in want.chunks(13) { + windows.extend(window.iter().copied()); + } + for list in [&pushed, &windows] { + assert_eq!(list.len(), n); + assert_eq!(list.to_vec(), want, "block {block} n {n}"); + assert_eq!(list.iter().copied().collect::>(), want); + let lens: Vec = list.parts().map(<[u64]>::len).collect(); + let full = lens.len().saturating_sub(1); + assert!( + lens[..full].iter().all(|&l| l == block), + "block {block} n {n}: {lens:?}" + ); + assert!( + lens.iter().all(|&l| 0 < l && l <= block), + "block {block} n {n}: {lens:?}" + ); + assert!(list.largest_bytes() <= block.max(16) * 8); + for chunk in [1usize, 2, block, 2 * block, 3 * block + 1, 4096] { + for (k, expected) in want.chunks(chunk).enumerate() { + let got = list.range(k * chunk, (k + 1) * chunk); + assert_eq!(&*got, expected, "block {block} n {n} chunk {chunk} k {k}"); + // A chunk the block length divides lies in one block. + if chunk % block == 0 && chunk == block { + assert!(matches!(got, Cow::Borrowed(_)), "chunk {k} copied"); + } + } + } + assert!(list.range(n, n + 5).is_empty()); + } + } + } + } + + /// Lists chained and taken from the front keep the order; a front block + /// that is exactly the chunk taken moves out whole. + #[test] + fn chained_and_taken_from_the_front_in_order() { + let a: Vec = (0..50).collect(); + let b: Vec = (50..77).collect(); + let mut first = BlockVec::new(8); + first.extend(a.iter().copied()); + let mut second = BlockVec::new(8); + second.extend(b.iter().copied()); + let mut all = first.chain(second).chain(BlockVec::from_vec(vec![77, 78])); + assert_eq!(all.to_vec(), (0..79).collect::>()); + let front = all.take_front(8); + assert_eq!(front, (0..8).collect::>()); + let mut taken = front; + while !all.is_empty() { + taken.extend(all.take_front(5)); + } + assert_eq!(taken, (0..79).collect::>()); + assert!(all.take_front(3).is_empty()); + assert_eq!(all.len(), 0); + } + + /// The block length: a power of two that fits 64 MiB, no more than the + /// chunk; 2^21 for 32-byte ops at the production chunk. + #[test] + fn a_block_is_at_most_64_mib_and_at_most_a_chunk() { + assert_eq!(block_len::<[u8; 32]>(1 << 21), 1 << 21); + assert_eq!(block_len::<[u8; 24]>(1 << 21), 1 << 21); + assert_eq!(block_len::<[u8; 40]>(1 << 21), 1 << 20); + assert_eq!(block_len::<[u8; 1870]>(1 << 21), 1 << 15); + assert_eq!(block_len::(24), 24); + assert_eq!(block_len::(0), 1); + for size_log2 in 0..12 { + let bytes = 1usize << size_log2; + let len = BLOCK_BYTES / bytes; + assert!(len.is_power_of_two() && len * bytes <= BLOCK_BYTES); + } + } +} diff --git a/prover/src/tables/trace_builder/bus_net_tests.rs b/prover/src/tables/trace_builder/bus_net_tests.rs new file mode 100644 index 000000000..5f4f66dad --- /dev/null +++ b/prover/src/tables/trace_builder/bus_net_tests.rs @@ -0,0 +1,265 @@ +//! ★ The buses the tables close among themselves balance on the traces the +//! builders make, row by row, with no proof: every tuple a table sends is +//! received as often, padding rows included. The multiplicities BITWISE answers +//! with are counted by collectors that mirror each table's rows (the μ bounds +//! the deduplicating tables send once a row, the byte checks SHIFT's and +//! KECCAK_RND's padding rows send); a collector that miscounts leaves a tuple +//! here that only a proof would otherwise find. +//! +//! The buses a block's statement closes from outside the tables (the committed +//! output, memory's and registers' first and last values) are left out. + +use std::collections::{BTreeMap, HashMap}; + +use executor::elf::Elf; +use stark::lookup::{BusInteraction, LinearTerm, Multiplicity}; +use stark::proof::options::ProofOptions; + +use super::finish_tests::{run, windowed}; +use super::*; +use crate::VmAirs; +use crate::tables::MaxRowsConfig; +use crate::tables::types::{BusId, FE}; + +/// The buses a statement closes from outside the tables: a net on them is the +/// verifier's to cancel, not a miscount. +const OUTSIDE: [BusId; 4] = [ + BusId::Commit, + BusId::Memory, + BusId::GlobalMemory, + BusId::Decode, +]; + +fn multiplicity(m: &Multiplicity, g: &dyn Fn(usize) -> FE) -> FE { + match m { + Multiplicity::One => FE::one(), + Multiplicity::Column(c) => g(*c), + Multiplicity::Sum(a, b) => g(*a) + g(*b), + Multiplicity::Negated(c) => FE::one() - g(*c), + Multiplicity::Diff(a, b) => g(*a) - g(*b), + Multiplicity::Sum3(a, b, c) => g(*a) + g(*b) + g(*c), + Multiplicity::Linear(ts) => ts.iter().fold(FE::zero(), |acc, t| match *t { + LinearTerm::Column { + coefficient, + column, + } => acc + g(column) * FE::from(coefficient), + LinearTerm::ColumnUnsigned { + coefficient, + column, + } => acc + g(column) * FE::from(coefficient), + LinearTerm::Constant(v) => acc + FE::from(v), + }), + } +} + +/// Every bus's tuples with a nonzero net over all of `traces`' tables, by bus, +/// the buses in [`OUTSIDE`] left out: `(bus, tuples)`. +pub(super) fn unbalanced( + program: &Elf, + traces: &mut Traces, +) -> BTreeMap, i128)>> { + unbalanced_with(program, traces, &|_| None) +} + +/// [`unbalanced`] with a table's interactions taken from `over` when it names +/// them (by the table's name), else from its AIR: a test's mutant interactions, +/// built on the test's own thread. +pub(super) fn unbalanced_with( + program: &Elf, + traces: &mut Traces, + over: &dyn Fn(&str) -> Option>, +) -> BTreeMap, i128)>> { + let options = ProofOptions::default_test_options(); + let table_counts = traces.table_counts(); + let airs = VmAirs::new( + program, + &options, + false, + &traces.page_configs, + &table_counts, + &crate::cpu_heights_from_traces(traces), + None, + true, + None, + crate::test_utils::halted_register_commitment( + &options, + program, + &traces.public_output_bytes, + ) + .preprocessed(), + ); + let outside: Vec = OUTSIDE.iter().map(|b| u64::from(*b)).collect(); + let mut net: HashMap<(u64, Vec), FE> = HashMap::new(); + // Each table's share of each tuple, to name the tables of a tuple left over. + let mut by_table: HashMap<(u64, Vec), BTreeMap> = HashMap::new(); + for (air, trace, _) in airs.air_trace_pairs(traces) { + let owned = over(air.name()); + let interactions: &[BusInteraction] = match &owned { + Some(v) => v, + None => air.bus_interactions(), + }; + for row in 0..trace.num_rows() { + let g = |c: usize| *trace.get_main(row, c); + for bi in interactions + .iter() + .filter(|bi| !outside.contains(&bi.bus_id)) + { + let m = multiplicity(&bi.multiplicity, &g); + if m == FE::zero() { + continue; + } + let tuple: Vec = bi + .values + .iter() + .flat_map(|v| v.combine_from(g)) + .map(|x: FE| x.canonical_u64()) + .collect(); + let family = air.name().split('[').next().unwrap_or("").to_string(); + let share = by_table.entry((bi.bus_id, tuple.clone())).or_default(); + let small = m.canonical_u64().min(1 << 20) as i128; + *share.entry(family).or_insert(0) += if bi.is_sender { small } else { -small }; + let e = net.entry((bi.bus_id, tuple)).or_insert(FE::zero()); + *e = if bi.is_sender { *e + m } else { *e - m }; + } + } + } + let mut by_bus: BTreeMap, i128)>> = BTreeMap::new(); + for ((bus, tuple), v) in net { + if v != FE::zero() { + // The net as a small signed count: a send left over is positive. + let v = v.canonical_u64(); + let signed = if v > u64::MAX / 2 { + -((FE::zero() - FE::from(v)).canonical_u64() as i128) + } else { + v as i128 + }; + let tables = by_table + .get(&(bus, tuple.clone())) + .map(|shares| { + shares + .iter() + .filter(|(_, n)| **n != 0) + .map(|(t, n)| format!("{t} {n:+}")) + .collect::>() + .join(" ") + }) + .unwrap_or_default(); + eprintln!("left over: bus {bus} {tuple:?} net {signed:+} ({tables})"); + by_bus.entry(bus).or_default().push((tuple, signed)); + } + } + by_bus +} + +pub(super) fn named(by_bus: &BTreeMap, i128)>>) -> String { + by_bus + .iter() + .map(|(bus, tuples)| { + let name = BusId::try_from(*bus) + .map_or_else(|_| format!("bus {bus}"), |b| b.name().to_string()); + let mut tuples = tuples.clone(); + tuples.sort(); + let shown: Vec = tuples + .iter() + .take(4) + .map(|(tuple, net)| format!("{tuple:?} net {net:+}")) + .collect(); + format!("{name}: {} tuple(s): {}", tuples.len(), shown.join(", ")) + }) + .collect::>() + .join(" · ") +} + +/// The programs: a loop that repeats its ops past a row's `MU_MAX` (BRANCH, LT +/// and the ALU tables split rows), every instruction, the shifts, divisions, +/// loads and comparisons main's multiplicity fix bounded, KECCAK (KECCAK_RND's +/// padding; a state across a 64 KiB boundary, two IS_WH keys), Keccak-256 of +/// buffers (KECCAK's sponge mode), COPY's writes. +const PROGRAMS: [&str; 12] = [ + "loop_4096", + "all_instructions_64", + "misalign_sd", + "poc_mul_commit", + "poc_divu_commit", + "poc_sll_commit", + "poc_slt_commit", + "poc_lb_commit", + "test_keccak_multi", + "test_keccak_64k", + "test_keccak256", + "test_commit_split", +]; + +/// Whole-run builds at small chunks (every chunked table split) and at the +/// production sizes. +#[test] +fn the_whole_run_tables_balance_every_inner_bus() { + for max_rows in [MaxRowsConfig::small(), MaxRowsConfig::default()] { + for name in PROGRAMS { + let (program, logs) = run(name); + let mut traces = + Traces::from_elf_and_logs(&program, &logs, &max_rows, &[]).expect("the build"); + let left = unbalanced(&program, &mut traces); + assert!(left.is_empty(), "{name}: {}", named(&left)); + } + } +} + +/// Windowed builds: streamed chunks counted as they are handed out, KECCAK_RND +/// cut at the finish under either padding rule, KECCAK / ECSM / ECDAS / COPY cut. +#[test] +fn the_windowed_tables_balance_every_inner_bus() { + type Configure = Box) -> WindowedTraceBuilder<'_>>; + let configs: Vec<(&str, Configure)> = vec![ + ("plain", Box::new(|b| b)), + ( + "dropped", + Box::new(|b| b.drop_streamed_ops().expect("before any window")), + ), + ( + "dropped, KECCAK_RND at the finish, cuts, last table padded", + Box::new(|b| { + b.keccak_rnd_chunks_at_finish(32) + .expect("rows") + .cuts_at_finish(2, 4, 2, 2) + .expect("rows") + .cut_padding(CutPadding::LastTable) + .drop_streamed_ops() + .expect("before any window") + }), + ), + ( + "KECCAK_RND streamed, whole run padded", + Box::new(|b| { + b.keccak_rnd_chunks(32) + .expect("rows") + .cut_padding(CutPadding::WholeRun) + .drop_streamed_ops() + .expect("before any window") + }), + ), + ]; + let mut failed = Vec::new(); + for (sizes, max_rows) in [ + ("small", MaxRowsConfig::small()), + ("2^4", MaxRowsConfig::uniform(4)), + ] { + for name in PROGRAMS { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs { + let mut traces = + windowed(&program, &logs, &max_rows, window, configure.as_ref()); + let left = unbalanced(&program, &mut traces); + if !left.is_empty() { + failed.push(format!( + "{name} {sizes} window {window} {config}: {}", + named(&left) + )); + } + } + } + } + } + assert!(failed.is_empty(), "{}", failed.join("\n")); +} diff --git a/prover/src/tables/trace_builder/clock.rs b/prover/src/tables/trace_builder/clock.rs new file mode 100644 index 000000000..824a47dfc --- /dev/null +++ b/prover/src/tables/trace_builder/clock.rs @@ -0,0 +1,211 @@ +//! The CPU's clock bound. +//! +//! A cycle's timestamp is `4·cycle + 4` over the whole run, and its accesses +//! sit at `+0..=+3`; a CPU padding row continues the `+4` cadence, and the +//! last padding row's PC write (`+1`) is the REGISTER table's final PC token. +//! The CPU holds the timestamp in one column and sends it with a zero high word +//! ("CPU timestamps fit in 32 bits", `cpu.rs`), while MEMW_R, MEMW, REGISTER +//! and the other tables it talks to take a timestamp as two 32-bit words. Below +//! 2^32 the two encodings are one message; at or past it they differ, the +//! buses cannot balance, and the proof cannot verify. A run is refused once a +//! timestamp would reach 2^32 — the walk's window that reaches cycle 2^30 − 1, +//! or the final PC token, which the last chunk's padding rows can put past +//! 2^32 a little below 2^30 cycles — rather than proved and then rejected. +//! +//! `LAMBDA_VM_BLOCK_UNVERIFIABLE_CLOCK=1` proves past the bound anyway, for +//! memory and timing measurements only: the proof does not verify, and the log +//! says so. A release build only: HALT and HINT debug-assert 32-bit timestamps +//! (`halt.rs`, `hint.rs`), so a build with debug assertions refuses instead of +//! proving into them. + +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; + +use crate::Error; + +/// The first timestamp the CPU cannot send: its high word is lost from here. +pub(crate) const CLOCK_LIMIT: u64 = 1 << 32; + +/// The knob that proves past [`CLOCK_LIMIT`] anyway (a proof that does not +/// verify). +const UNVERIFIABLE_CLOCK: &str = "LAMBDA_VM_BLOCK_UNVERIFIABLE_CLOCK"; + +/// Whether the knob's value asks to prove past the bound: `1`, `true` or `yes`. +fn unverifiable_clock_from(value: Option<&str>) -> bool { + value.is_some_and(|v| matches!(v.trim(), "1" | "true" | "yes")) +} + +/// The largest timestamp a window of `cycles` cycles from cycle `first` uses: +/// its last cycle's `4·cycle + 4`, plus the cycle's accesses up to `+3`. +pub(crate) fn window_max_timestamp(first: usize, cycles: usize) -> u64 { + 4 * (first as u64 + cycles as u64) + 3 +} + +/// The REGISTER table's final PC token: the last CPU padding row's PC write, +/// `halt_timestamp + 4·padding_rows + 1` (`halt_timestamp + 1` without +/// padding) — the largest timestamp the run's trace holds. +pub(crate) fn final_pc_timestamp(halt_timestamp: u64, padding_rows: usize) -> u64 { + halt_timestamp + 4 * padding_rows as u64 + 1 +} + +/// Refuses `max_timestamp` at or past [`CLOCK_LIMIT`] (`what` names where it +/// was reached), unless [`UNVERIFIABLE_CLOCK`] is set in a release build: then +/// it warns, once a process, and lets the run go on. +pub(crate) fn check_clock(max_timestamp: u64, what: &str) -> Result<(), Error> { + check_clock_with( + max_timestamp, + what, + unverifiable_clock_from(std::env::var(UNVERIFIABLE_CLOCK).ok().as_deref()), + ) +} + +/// [`check_clock`] with the knob's answer given. +fn check_clock_with(max_timestamp: u64, what: &str, unverifiable: bool) -> Result<(), Error> { + if max_timestamp < CLOCK_LIMIT { + return Ok(()); + } + if unverifiable && cfg!(debug_assertions) { + return Err(Error::ClockPastLimit(format!( + "{what} reaches timestamp {max_timestamp}, past the CPU's 32-bit clock (2^32); \ + {UNVERIFIABLE_CLOCK}=1 proves past it only in a release build (HALT and HINT \ + debug-assert 32-bit timestamps)" + ))); + } + if unverifiable { + static WARNED: AtomicBool = AtomicBool::new(false); + if !WARNED.swap(true, Relaxed) { + eprintln!( + "BLOCK CLOCK: UNVERIFIABLE — {what} reaches timestamp {max_timestamp} ≥ 2^32 \ + (the CPU sends a zero high word); {UNVERIFIABLE_CLOCK}=1 proves anyway, and \ + THE PROOF WILL NOT VERIFY" + ); + } + return Ok(()); + } + Err(Error::ClockPastLimit(format!( + "{what} reaches timestamp {max_timestamp}, past the CPU's 32-bit clock (2^32): \ + its buses cannot balance, so the proof would not verify (a run's CPU rows, \ + padding included, number at most 2^30 − 1; {UNVERIFIABLE_CLOCK}=1 proves \ + anyway, for measurements)" + ))) +} + +#[cfg(test)] +mod tests { + use super::{ + CLOCK_LIMIT, check_clock_with, final_pc_timestamp, unverifiable_clock_from, + window_max_timestamp, + }; + use crate::Error; + + /// A window ending at cycle 2^30 − 2 (2^30 − 1 cycles) stays under 2^32; + /// one more cycle puts its timestamp at 2^32. + #[test] + fn the_clock_takes_2_30_minus_1_cycles() { + let last = (1usize << 30) - 1; + assert_eq!(window_max_timestamp(0, last), CLOCK_LIMIT - 1); + assert_eq!(window_max_timestamp(last - 5, 5), CLOCK_LIMIT - 1); + assert!(check_clock_with(window_max_timestamp(0, last), "run", false).is_ok()); + assert_eq!(window_max_timestamp(last, 1), CLOCK_LIMIT + 3); + assert!(matches!( + check_clock_with(window_max_timestamp(last, 1), "run", false), + Err(Error::ClockPastLimit(_)) + )); + // The final PC token of a run whose padding crosses: refused too. + assert!(check_clock_with(CLOCK_LIMIT - 1, "pc", false).is_ok()); + assert!(check_clock_with(CLOCK_LIMIT + 1, "pc", false).is_err()); + } + + /// The walk refuses a window whose last cycle's timestamp would reach 2^32, + /// and takes the window that ends one cycle before. + #[test] + fn the_walk_refuses_the_window_that_crosses_2_32() { + use super::super::{DecodeArtifacts, Elf, collect_cpu_ops_from}; + use crate::test_utils::asm_elf_bytes; + use executor::vm::execution::Executor; + + let program = Elf::load(&asm_elf_bytes("lw_sw_offset_odd")).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let artifacts = DecodeArtifacts::from_elf(&program).expect("the decode artifacts"); + let n = logs.len(); + // The last cycle at 2^30 − 2: timestamp 2^32 − 4. + let fits = (1usize << 30) - 1 - n; + let ops = collect_cpu_ops_from(&logs, &artifacts.decode, fits).expect("it fits"); + assert_eq!(ops.last().map(|op| op.timestamp), Some(CLOCK_LIMIT - 4)); + // One cycle later: the last cycle's timestamp is 2^32. + assert!(matches!( + collect_cpu_ops_from(&logs, &artifacts.decode, fits + 1), + Err(Error::ClockPastLimit(_)) + )); + } + + /// A run just under 2^30 cycles whose last chunk's padding rows cross: + /// every window fits, but the CPU trace's last padded row writes the PC at + /// 2^32 or past, and that final PC token is refused. + #[test] + fn padding_rows_that_cross_2_32_are_refused() { + use super::super::{DecodeArtifacts, Elf, collect_cpu_ops_from}; + use crate::tables::cpu; + use crate::test_utils::asm_elf_bytes; + use executor::vm::execution::Executor; + + let program = Elf::load(&asm_elf_bytes("lw_sw_offset_odd")).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let artifacts = DecodeArtifacts::from_elf(&program).expect("the decode artifacts"); + let n = logs.len(); + let padding = n.next_power_of_two().max(4) - n; + assert!(padding > 0, "the program's {n} cycles leave padding rows"); + // The halting cycle at 2^30 − 2: the walk takes every window. + let first = (1usize << 30) - 1 - n; + let ops = collect_cpu_ops_from(&logs, &artifacts.decode, first).expect("it fits"); + let halt = ops.last().expect("ops").timestamp; + assert_eq!(halt, CLOCK_LIMIT - 4); + // The trace's last padded row writes the PC at the final PC token. + // TIMESTAMP is the table's own clock `4·(row + 1)`; the AIR's base, four + // times the rows before the table, puts it on the run's clock. + let trace = cpu::generate_cpu_trace(&ops, &artifacts.decode); + assert_eq!(trace.num_rows(), n + padding); + let base = cpu::TIMESTAMP_STRIDE * first as u64; + let last_row = base + + *trace + .get_main(trace.num_rows() - 1, cpu::cols::TIMESTAMP) + .value(); + let final_pc = final_pc_timestamp(halt, padding); + assert_eq!(last_row + 1, final_pc); + assert!(final_pc >= CLOCK_LIMIT); + assert!(matches!( + check_clock_with(final_pc, "the CPU's last padding row", false), + Err(Error::ClockPastLimit(_)) + )); + // Without padding the same halt fits. + assert!(check_clock_with(final_pc_timestamp(halt, 0), "pc", false).is_ok()); + } + + /// The knob lets a run past the bound go on in a release build; a build + /// with debug assertions refuses instead (HALT and HINT assert 32-bit + /// timestamps). + #[test] + fn the_unverifiable_clock_knob_proves_anyway() { + for ts in [CLOCK_LIMIT, u64::MAX] { + let run = check_clock_with(ts, "run", true); + if cfg!(debug_assertions) { + assert!(matches!(run, Err(Error::ClockPastLimit(_)))); + } else { + assert!(run.is_ok()); + } + } + assert!(unverifiable_clock_from(Some("1"))); + assert!(unverifiable_clock_from(Some(" true "))); + assert!(unverifiable_clock_from(Some("yes"))); + assert!(!unverifiable_clock_from(Some("0"))); + assert!(!unverifiable_clock_from(Some(""))); + assert!(!unverifiable_clock_from(None)); + } +} diff --git a/prover/src/tables/trace_builder/compact_branch.rs b/prover/src/tables/trace_builder/compact_branch.rs new file mode 100644 index 000000000..9b8c3dfde --- /dev/null +++ b/prover/src/tables/trace_builder/compact_branch.rs @@ -0,0 +1,236 @@ +//! BRANCH ops as the builder keeps them until their tables are built: a +//! [`DeltaStream`] instead of 32 bytes an op (S3c-BRANCH). +//! +//! An op is a flags byte (`jalr`, and how its register is coded) and three +//! varints: the pc from the previous op's, the offset itself, and the register +//! the cheapest of three ways (as it is, from the previous op's register, or +//! from the op's own pc: a JALR's return address sits near it). Every op takes +//! at most 31 bytes, so any op, whatever its values, is kept losslessly and +//! none is held raw. + +use super::BranchOperation; +use super::delta::{Codec, DeltaStream, cheapest, get, put, unzigzag, zigzag}; + +/// BRANCH's [`Codec`]: the previous op's pc and register. +#[derive(Clone, Copy, Default)] +pub(super) struct BranchCodec { + pc: u64, + register: u64, +} + +pub(super) type CompactBranch = DeltaStream; + +/// `flags`: the op is a JALR. +const JALR: u8 = 1; +/// `flags` bits 1–2: how the register is coded. +const REGISTER_SHIFT: u8 = 1; +const REGISTER_ABSOLUTE: u8 = 0; +const REGISTER_FROM_LAST: u8 = 1; +const REGISTER_FROM_PC: u8 = 2; + +impl Codec for BranchCodec { + type Op = BranchOperation; + /// The flags byte and three varints of at most ten. + const MAX_BYTES: usize = 31; + + fn encode(&mut self, op: &BranchOperation, out: &mut Vec) { + let (mode, register) = cheapest([ + (REGISTER_ABSOLUTE, zigzag(op.register)), + ( + REGISTER_FROM_LAST, + zigzag(op.register.wrapping_sub(self.register)), + ), + (REGISTER_FROM_PC, zigzag(op.register.wrapping_sub(op.pc))), + ]); + out.push((u8::from(op.jalr) * JALR) | (mode << REGISTER_SHIFT)); + put(out, zigzag(op.pc.wrapping_sub(self.pc))); + put(out, zigzag(op.offset)); + put(out, register); + self.pc = op.pc; + self.register = op.register; + } + + fn decode(&mut self, bytes: &[u8], at: &mut usize) -> BranchOperation { + let flags = bytes[*at]; + *at += 1; + self.pc = self.pc.wrapping_add(unzigzag(get(bytes, at))); + let offset = unzigzag(get(bytes, at)); + let coded = unzigzag(get(bytes, at)); + self.register = match (flags >> REGISTER_SHIFT) & 3 { + REGISTER_FROM_LAST => self.register.wrapping_add(coded), + REGISTER_FROM_PC => self.pc.wrapping_add(coded), + _ => coded, + }; + BranchOperation::new(self.pc, offset, self.register, flags & JALR != 0) + } +} + +#[cfg(test)] +mod tests { + use super::super::delta::{Codec, with_index_every}; + use super::{BranchCodec, CompactBranch}; + + const INDEX_EVERY: usize = 4096; + const OP_BYTES: usize = ::MAX_BYTES; + use crate::tables::branch::{BranchOperation, generate_branch_trace}; + + /// Ops whose fields reach both ends of u64 and i64 and sit near each other, + /// in a pattern that changes every few ops. + fn ops(n: usize) -> Vec { + let edges = [ + 0u64, + 1, + 2, + 0x7fff_ffff_ffff_ffff, + 0x8000_0000_0000_0000, + u64::MAX, + u64::MAX - 1, + ]; + (0..n) + .map(|i| { + let k = i as u64; + let pc = match i % 5 { + 0 => 0x1000 + 4 * k, + 1 => edges[i % edges.len()], + 2 => 0x2000_0000u64.wrapping_sub(4 * k), + _ => 0x1_0000_0000 + 8 * k, + }; + let offset = match i % 3 { + 0 => (-(k as i64 % 4096)) as u64, + 1 => edges[(i / 3) % edges.len()], + _ => 4 * (k % 1024), + }; + let register = match i % 7 { + 0 => pc.wrapping_add(4), + 1 => edges[(i / 7) % edges.len()], + 2 => k.wrapping_mul(0x9e37_79b9_7f4a_7c15), + _ => k / 3, + }; + BranchOperation::new(pc, offset, register, i % 4 == 1) + }) + .collect() + } + + /// Every range of ops, across the sparse index and across small blocks, + /// expands to the ops pushed, fields at both ends of their ranges + /// included; no op takes more than 31 bytes, so none needs a raw form. + #[test] + fn compact_branch_ops_expand_to_what_was_pushed() { + let all = ops(3 * INDEX_EVERY + 77); + for block_bytes in [0usize, 64, 1000] { + let mut list = if block_bytes == 0 { + CompactBranch::default() + } else { + CompactBranch::with_block_bytes(block_bytes) + }; + // Appended in uneven windows, as the builder appends them. + for window in all.chunks(997) { + list.extend(window); + } + assert_eq!(list.len(), all.len()); + assert_eq!( + list.range(0, all.len()), + all, + "the whole list, blocks of {block_bytes}" + ); + for (start, end) in [ + (0, 1), + (INDEX_EVERY - 1, INDEX_EVERY + 1), + (INDEX_EVERY, 2 * INDEX_EVERY), + (5, 3 * INDEX_EVERY + 77), + (2 * INDEX_EVERY + 3, 2 * INDEX_EVERY + 3), + (3 * INDEX_EVERY + 70, 9 * INDEX_EVERY), + ] { + assert_eq!( + list.range(start, end), + all[start.min(all.len())..end.min(all.len())], + "ops {start}..{end}, blocks of {block_bytes}" + ); + } + let stream = list.stream_bytes(); + assert!( + stream <= OP_BYTES * all.len(), + "{stream} bytes for {} ops", + all.len() + ); + } + } + + /// Chunks of every size around the index's spacing, each starting past an + /// index mark, expand to their ops; so does a stream marked every three + /// ops ([`with_index_every`]), where nearly every range starts past one. + #[test] + fn chunks_starting_past_an_index_mark_expand_to_their_ops() { + let all = ops(3 * INDEX_EVERY + 77); + let list = CompactBranch::from_ops(&all); + assert_eq!(list.every(), INDEX_EVERY); + for chunk in [1000usize, INDEX_EVERY - 1, INDEX_EVERY + 1, 5000] { + for (k, want) in all.chunks(chunk).enumerate() { + assert_eq!( + list.range(k * chunk, (k + 1) * chunk), + want, + "chunk {k} of {chunk}" + ); + } + } + let dense = with_index_every(3, || CompactBranch::from_ops(&all)); + assert_eq!(dense.every(), 3); + for chunk in [1usize, 2, 7, 100] { + for (k, want) in all.chunks(chunk).enumerate().take(500) { + assert_eq!( + dense.range(k * chunk, (k + 1) * chunk), + want, + "dense chunk {k} of {chunk}" + ); + } + } + } + + /// The worst op (every field a full ten-byte varint) round trips, at 31 + /// bytes; the common op (a nearby pc, a short offset, a return address + /// near the pc) takes a few. + #[test] + fn the_worst_op_is_kept_and_the_common_one_is_short() { + let worst = BranchOperation::new( + 0x8000_0000_0000_0000, + 0x8000_0000_0000_0000, + 0x5555_5555_5555_5555, + true, + ); + let list = CompactBranch::from_ops(std::slice::from_ref(&worst)); + assert_eq!(list.range(0, 1), vec![worst]); + assert!(list.stream_bytes() <= OP_BYTES); + let common: Vec = (0..1000u64) + .map(|k| { + BranchOperation::new( + 0x10_0000 + 16 * k, + (-64i64) as u64, + 0x10_0004 + 16 * k, + k % 2 == 0, + ) + }) + .collect(); + let list = CompactBranch::from_ops(&common); + assert_eq!(list.range(0, 1000), common); + let stream = list.stream_bytes(); + assert!( + stream <= 6 * common.len(), + "{stream} bytes for 1000 common ops" + ); + } + + /// The BRANCH table of the expanded ops is the table of the ops. + #[test] + fn the_table_of_compact_branch_ops_is_the_table_of_the_ops() { + let all = ops(2 * INDEX_EVERY + 5); + let list = CompactBranch::from_ops(&all); + let words = |ops: &[BranchOperation]| { + let t = generate_branch_trace(ops); + let m = &t.main_table; + (0..m.height) + .flat_map(|r| (0..m.width).map(move |c| *m.get(r, c).value())) + .collect::>() + }; + assert_eq!(words(&list.range(0, all.len())), words(&all)); + } +} diff --git a/prover/src/tables/trace_builder/compact_bytewise.rs b/prover/src/tables/trace_builder/compact_bytewise.rs new file mode 100644 index 000000000..86bbb312c --- /dev/null +++ b/prover/src/tables/trace_builder/compact_bytewise.rs @@ -0,0 +1,224 @@ +//! BYTEWISE ops as the builder keeps them until their tables are built: a +//! [`DeltaStream`] instead of 24 bytes an op (S3c-BYTEWISE). +//! +//! An op is a flags byte (how each operand is coded, and the opcode when it is +//! under 8; a larger one follows in a byte of its own) and two varints: `a` as +//! it is or from the previous op's `a`, and `b` as it is (a mask), from the +//! op's own `a` or from the previous op's `b`, each the cheapest. Every op +//! takes at most 22 bytes, so any op, whatever its values, is kept losslessly +//! and none is held raw. + +use super::bytewise::BytewiseOperation; +use super::delta::{Codec, DeltaStream, cheapest, get, put, unzigzag, zigzag}; + +/// BYTEWISE's [`Codec`]: the previous op's operands. +#[derive(Clone, Copy, Default)] +pub(super) struct BytewiseCodec { + a: u64, + b: u64, +} + +pub(super) type CompactBytewise = DeltaStream; + +/// `flags` bit 0: how `a` is coded; bits 1–2: how `b` is; bits 3–5: the +/// opcode; bit 6: the opcode is too large for bits 3–5 and follows. +const A_SHIFT: u8 = 0; +const B_SHIFT: u8 = 1; +const OP_SHIFT: u8 = 3; +const OP_FOLLOWS: u8 = 1 << 6; +const ABSOLUTE: u8 = 0; +const FROM_LAST: u8 = 1; +const FROM_A: u8 = 2; + +impl Codec for BytewiseCodec { + type Op = BytewiseOperation; + /// The flags byte, an opcode byte and two varints of at most ten. + const MAX_BYTES: usize = 22; + + fn encode(&mut self, op: &BytewiseOperation, out: &mut Vec) { + let (a_mode, a) = cheapest([ + (ABSOLUTE, zigzag(op.a)), + (FROM_LAST, zigzag(op.a.wrapping_sub(self.a))), + ]); + let (b_mode, b) = cheapest([ + (ABSOLUTE, zigzag(op.b)), + (FROM_A, zigzag(op.b.wrapping_sub(op.a))), + (FROM_LAST, zigzag(op.b.wrapping_sub(self.b))), + ]); + let modes = (a_mode << A_SHIFT) | (b_mode << B_SHIFT); + if op.op < 8 { + out.push(modes | (op.op << OP_SHIFT)); + } else { + out.push(modes | OP_FOLLOWS); + out.push(op.op); + } + put(out, a); + put(out, b); + self.a = op.a; + self.b = op.b; + } + + fn decode(&mut self, bytes: &[u8], at: &mut usize) -> BytewiseOperation { + let flags = bytes[*at]; + *at += 1; + let opcode = if flags & OP_FOLLOWS != 0 { + let opcode = bytes[*at]; + *at += 1; + opcode + } else { + (flags >> OP_SHIFT) & 7 + }; + let a = unzigzag(get(bytes, at)); + self.a = match (flags >> A_SHIFT) & 1 { + FROM_LAST => self.a.wrapping_add(a), + _ => a, + }; + let b = unzigzag(get(bytes, at)); + self.b = match (flags >> B_SHIFT) & 3 { + FROM_A => self.a.wrapping_add(b), + FROM_LAST => self.b.wrapping_add(b), + _ => b, + }; + BytewiseOperation::new(self.a, self.b, opcode) + } +} + +#[cfg(test)] +mod tests { + use super::super::delta::{Codec, with_index_every}; + use super::{BytewiseCodec, CompactBytewise}; + use crate::tables::bytewise::{BytewiseOperation, generate_bytewise_trace}; + use crate::tables::types::alu_op; + + const INDEX_EVERY: usize = 4096; + const OP_BYTES: usize = ::MAX_BYTES; + + /// Operands at both ends of u64 / i64, masks, near and far, every opcode + /// the table takes, in a pattern that changes every few ops. + fn ops(n: usize) -> Vec { + let edges = [ + 0u64, + 1, + 0xff, + 0xffff, + 0x7fff_ffff_ffff_ffff, + 0x8000_0000_0000_0000, + u64::MAX, + ]; + let opcodes = [alu_op::AND, alu_op::OR, alu_op::XOR]; + (0..n) + .map(|i| { + let k = i as u64; + let a = match i % 4 { + 0 => k, + 1 => edges[i % edges.len()], + 2 => k.wrapping_mul(0x9e37_79b9_7f4a_7c15), + _ => 0x10_0000 + 8 * k, + }; + let b = match i % 5 { + 0 => 0xff, + 1 => edges[(i / 5) % edges.len()], + 2 => a, + 3 => !a, + _ => k.wrapping_mul(0x2545_f491_4f6c_dd1d), + }; + BytewiseOperation::new(a, b, opcodes[i % 3]) + }) + .collect() + } + + /// Every range, across the sparse index and small blocks, and every chunk + /// around the index's spacing expands to the ops pushed, an opcode too + /// large for the flags byte included; no op takes more than 22 bytes, so + /// none needs a raw form. + #[test] + fn compact_bytewise_ops_expand_to_what_was_pushed() { + let mut all = ops(3 * INDEX_EVERY + 77); + // Opcodes past XOR are not the table's, but the list keeps any byte: + // the ones the flags byte holds (3–7) and the ones that follow it. + for (i, op) in [3u8, 4, 5, 6, 7].into_iter().enumerate() { + all[11 + 2 * i].op = op; + } + all[17].op = 8; + all[INDEX_EVERY + 5].op = u8::MAX; + for block_bytes in [0usize, 64, 1000] { + let mut list = if block_bytes == 0 { + CompactBytewise::default() + } else { + CompactBytewise::with_block_bytes(block_bytes) + }; + for window in all.chunks(997) { + list.extend(window); + } + assert_eq!(list.len(), all.len()); + assert_eq!( + list.range(0, all.len()), + all, + "the whole list, blocks of {block_bytes}" + ); + for (start, end) in [ + (0, 1), + (INDEX_EVERY - 1, INDEX_EVERY + 1), + (INDEX_EVERY, 2 * INDEX_EVERY), + (2 * INDEX_EVERY + 3, 2 * INDEX_EVERY + 3), + (3 * INDEX_EVERY + 70, 9 * INDEX_EVERY), + ] { + assert_eq!( + list.range(start, end), + all[start.min(all.len())..end.min(all.len())], + "ops {start}..{end}, blocks of {block_bytes}" + ); + } + assert!(list.stream_bytes() <= OP_BYTES * all.len()); + } + let list = CompactBytewise::from_ops(&all); + for chunk in [1000usize, INDEX_EVERY - 1, INDEX_EVERY + 1, 5000] { + for (k, want) in all.chunks(chunk).enumerate() { + assert_eq!( + list.range(k * chunk, (k + 1) * chunk), + want, + "chunk {k} of {chunk}" + ); + } + } + let dense = with_index_every(3, || CompactBytewise::from_ops(&all)); + for (k, want) in all.chunks(7).enumerate().take(500) { + assert_eq!(dense.range(k * 7, (k + 1) * 7), want, "dense chunk {k}"); + } + } + + /// The worst op round trips at 22 bytes; a byte mask on a small value + /// takes three (the first op, with no mask before it, four). + #[test] + fn the_worst_bytewise_op_is_kept_and_the_common_one_is_short() { + let worst = BytewiseOperation::new(0x5555_5555_5555_5555, 0x8000_0000_0000_0000, 200); + let list = CompactBytewise::from_ops(std::slice::from_ref(&worst)); + assert_eq!(list.range(0, 1), vec![worst]); + assert!(list.stream_bytes() <= OP_BYTES); + let common: Vec = (0..1000u64) + .map(|k| BytewiseOperation::new(k % 60, 0xff, alu_op::AND)) + .collect(); + let list = CompactBytewise::from_ops(&common); + assert_eq!(list.range(0, 1000), common); + assert!( + list.stream_bytes() <= 3 * common.len() + 1, + "{} bytes", + list.stream_bytes() + ); + } + + /// The BYTEWISE table of the expanded ops is the table of the ops. + #[test] + fn the_table_of_compact_bytewise_ops_is_the_table_of_the_ops() { + let all = ops(2 * INDEX_EVERY + 5); + let list = CompactBytewise::from_ops(&all); + let words = |ops: &[BytewiseOperation]| { + let t = generate_bytewise_trace(ops); + let m = &t.main_table; + (0..m.height) + .flat_map(|r| (0..m.width).map(move |c| *m.get(r, c).value())) + .collect::>() + }; + assert_eq!(words(&list.range(0, all.len())), words(&all)); + } +} diff --git a/prover/src/tables/trace_builder/compact_eq.rs b/prover/src/tables/trace_builder/compact_eq.rs new file mode 100644 index 000000000..186e11288 --- /dev/null +++ b/prover/src/tables/trace_builder/compact_eq.rs @@ -0,0 +1,199 @@ +//! EQ ops as the builder keeps them until their tables are built: a +//! [`DeltaStream`] instead of 24 bytes an op (S3c-EQ). +//! +//! An op is a flags byte (`invert`, and how each operand is coded) and two +//! varints: `a` as it is or from the previous op's `a`, and `b` as it is, from +//! the op's own `a` (equal operands cost a byte) or from the previous op's +//! `b`, each the cheapest. Every op takes at most 21 bytes, so any op, +//! whatever its values, is kept losslessly and none is held raw. + +use super::delta::{Codec, DeltaStream, cheapest, get, put, unzigzag, zigzag}; +use super::eq::EqOperation; + +/// EQ's [`Codec`]: the previous op's operands. +#[derive(Clone, Copy, Default)] +pub(super) struct EqCodec { + a: u64, + b: u64, +} + +pub(super) type CompactEq = DeltaStream; + +/// `flags`: the op inverts. +const INVERT: u8 = 1; +/// `flags` bits 1–2: how `a` is coded; bits 3–4: how `b` is. +const A_SHIFT: u8 = 1; +const B_SHIFT: u8 = 3; +const ABSOLUTE: u8 = 0; +const FROM_LAST: u8 = 1; +const FROM_A: u8 = 2; + +impl Codec for EqCodec { + type Op = EqOperation; + /// The flags byte and two varints of at most ten. + const MAX_BYTES: usize = 21; + + fn encode(&mut self, op: &EqOperation, out: &mut Vec) { + let (a_mode, a) = cheapest([ + (ABSOLUTE, zigzag(op.a)), + (FROM_LAST, zigzag(op.a.wrapping_sub(self.a))), + ]); + let (b_mode, b) = cheapest([ + (ABSOLUTE, zigzag(op.b)), + (FROM_A, zigzag(op.b.wrapping_sub(op.a))), + (FROM_LAST, zigzag(op.b.wrapping_sub(self.b))), + ]); + out.push((u8::from(op.invert) * INVERT) | (a_mode << A_SHIFT) | (b_mode << B_SHIFT)); + put(out, a); + put(out, b); + self.a = op.a; + self.b = op.b; + } + + fn decode(&mut self, bytes: &[u8], at: &mut usize) -> EqOperation { + let flags = bytes[*at]; + *at += 1; + let a = unzigzag(get(bytes, at)); + self.a = match (flags >> A_SHIFT) & 3 { + FROM_LAST => self.a.wrapping_add(a), + _ => a, + }; + let b = unzigzag(get(bytes, at)); + self.b = match (flags >> B_SHIFT) & 3 { + FROM_A => self.a.wrapping_add(b), + FROM_LAST => self.b.wrapping_add(b), + _ => b, + }; + EqOperation::new(self.a, self.b, flags & INVERT != 0) + } +} + +#[cfg(test)] +mod tests { + use super::super::delta::{Codec, with_index_every}; + use super::{CompactEq, EqCodec}; + use crate::tables::eq::{EqOperation, generate_eq_trace}; + + const INDEX_EVERY: usize = 4096; + const OP_BYTES: usize = ::MAX_BYTES; + + /// Operands at both ends of u64 / i64, equal, near each other and far + /// apart, in a pattern that changes every few ops. + fn ops(n: usize) -> Vec { + let edges = [ + 0u64, + 1, + 2, + 0x7fff_ffff_ffff_ffff, + 0x8000_0000_0000_0000, + u64::MAX, + u64::MAX - 1, + ]; + (0..n) + .map(|i| { + let k = i as u64; + let a = match i % 4 { + 0 => k, + 1 => edges[i % edges.len()], + 2 => k.wrapping_mul(0x9e37_79b9_7f4a_7c15), + _ => 0x10_0000 + 8 * k, + }; + let b = match i % 5 { + 0 => a, + 1 => edges[(i / 5) % edges.len()], + 2 => a.wrapping_add(1), + 3 => 0, + _ => k.wrapping_mul(0x2545_f491_4f6c_dd1d), + }; + EqOperation::new(a, b, i % 3 == 1) + }) + .collect() + } + + /// Every range, across the sparse index and small blocks, and every chunk + /// around the index's spacing expands to the ops pushed; no op takes more + /// than 21 bytes, so none needs a raw form. + #[test] + fn compact_eq_ops_expand_to_what_was_pushed() { + let all = ops(3 * INDEX_EVERY + 77); + for block_bytes in [0usize, 64, 1000] { + let mut list = if block_bytes == 0 { + CompactEq::default() + } else { + CompactEq::with_block_bytes(block_bytes) + }; + for window in all.chunks(997) { + list.extend(window); + } + assert_eq!(list.len(), all.len()); + assert_eq!( + list.range(0, all.len()), + all, + "the whole list, blocks of {block_bytes}" + ); + for (start, end) in [ + (0, 1), + (INDEX_EVERY - 1, INDEX_EVERY + 1), + (INDEX_EVERY, 2 * INDEX_EVERY), + (2 * INDEX_EVERY + 3, 2 * INDEX_EVERY + 3), + (3 * INDEX_EVERY + 70, 9 * INDEX_EVERY), + ] { + assert_eq!( + list.range(start, end), + all[start.min(all.len())..end.min(all.len())], + "ops {start}..{end}, blocks of {block_bytes}" + ); + } + assert!(list.stream_bytes() <= OP_BYTES * all.len()); + } + let list = CompactEq::from_ops(&all); + for chunk in [1000usize, INDEX_EVERY - 1, INDEX_EVERY + 1, 5000] { + for (k, want) in all.chunks(chunk).enumerate() { + assert_eq!( + list.range(k * chunk, (k + 1) * chunk), + want, + "chunk {k} of {chunk}" + ); + } + } + let dense = with_index_every(3, || CompactEq::from_ops(&all)); + for (k, want) in all.chunks(7).enumerate().take(500) { + assert_eq!(dense.range(k * 7, (k + 1) * 7), want, "dense chunk {k}"); + } + } + + /// The worst op round trips at 21 bytes; equal operands and small ones + /// take three. + #[test] + fn the_worst_eq_op_is_kept_and_the_common_one_is_short() { + let worst = EqOperation::new(0x5555_5555_5555_5555, 0x8000_0000_0000_0000, true); + let list = CompactEq::from_ops(std::slice::from_ref(&worst)); + assert_eq!(list.range(0, 1), vec![worst]); + assert!(list.stream_bytes() <= OP_BYTES); + let common: Vec = (0..1000u64) + .map(|k| EqOperation::new(k % 50, if k % 2 == 0 { k % 50 } else { 0 }, k % 3 == 0)) + .collect(); + let list = CompactEq::from_ops(&common); + assert_eq!(list.range(0, 1000), common); + assert!( + list.stream_bytes() <= 3 * common.len(), + "{} bytes", + list.stream_bytes() + ); + } + + /// The EQ table of the expanded ops is the table of the ops. + #[test] + fn the_table_of_compact_eq_ops_is_the_table_of_the_ops() { + let all = ops(2 * INDEX_EVERY + 5); + let list = CompactEq::from_ops(&all); + let words = |ops: &[EqOperation]| { + let t = generate_eq_trace(ops); + let m = &t.main_table; + (0..m.height) + .flat_map(|r| (0..m.width).map(move |c| *m.get(r, c).value())) + .collect::>() + }; + assert_eq!(words(&list.range(0, all.len())), words(&all)); + } +} diff --git a/prover/src/tables/trace_builder/cut_padding_tests.rs b/prover/src/tables/trace_builder/cut_padding_tests.rs new file mode 100644 index 000000000..4a0e2f197 --- /dev/null +++ b/prover/src/tables/trace_builder/cut_padding_tests.rs @@ -0,0 +1,352 @@ +//! The padding of the tables a block cuts at a fixed number of rows +//! ([`CutPadding`]): the default and its knob, the count against a model of +//! the block's split, and KECCAK_RND's tables under each rule. Cut at the +//! last real row, a family's tables are the whole run's without the ones of +//! padding rows alone, and a table left out has every bus multiplicity at +//! zero. + +use stark::lookup::{BusInteraction, Multiplicity}; + +use crate::tables::types::FE; + +use super::*; + +type Table = TraceTable; + +/// A table as (columns, rows, words row-major), held packed or wide. +pub(super) fn shape_and_words(t: &Table) -> (usize, usize, Vec) { + match t.narrow_main() { + Some(packed) => { + let columns: Vec> = (0..packed.cols()).map(|c| packed.column(c)).collect(); + let words = (0..packed.rows()) + .flat_map(|r| columns.iter().map(move |column| column[r])) + .collect(); + (packed.cols(), packed.rows(), words) + } + None => { + let main = &t.main_table; + let words = (0..main.height) + .flat_map(|r| (0..main.width).map(move |c| *main.get(r, c).value())) + .collect(); + (main.width, main.height, words) + } + } +} + +/// `Ok` when every interaction of `interactions` has multiplicity zero on +/// every row of `table`: the table adds nothing to any bus but its padding +/// rows' byte checks. A multiplicity is read as the four call families' one +/// column, KECCAK's sum of its two mode bits, COPY's difference of two columns, +/// or COPY's linear form of columns and constants; another kind is an error, +/// never read as zero. +/// +/// The one exception: a byte check of multiplicity 1 whose values vanish on a +/// row of zeros (KECCAK_RND's θ/ρ bytes, main's multiplicity fix) is sent by +/// every row, padding included, as `ARE_BYTES[0, 0]`. Its only counterparty is +/// BITWISE, which counts it per table the build makes +/// (`keccak_rnd_padding_rows`), so a table left out takes its sends with it. +pub(super) fn carries_nothing( + table: &Table, + interactions: &[BusInteraction], +) -> Result<(), String> { + use stark::lookup::{BusValue, LinearTerm}; + let (cols, rows, words) = shape_and_words(table); + let fe = |row: usize, column: usize| FE::from(words[row * cols + column]); + for (i, interaction) in interactions.iter().enumerate() { + let zero_on_padding = interaction.values.iter().all(|value| match value { + BusValue::Packed { .. } => true, + BusValue::Linear(terms) => terms + .iter() + .all(|term| !matches!(term, LinearTerm::Constant(c) if *c != 0)), + }); + if interaction.bus_id == u64::from(crate::tables::types::BusId::AreBytes) + && interaction.is_sender + && matches!(interaction.multiplicity, Multiplicity::One) + && zero_on_padding + { + continue; + } + let read: Vec = match &interaction.multiplicity { + Multiplicity::Column(c) => vec![*c], + Multiplicity::Sum(a, b) | Multiplicity::Diff(a, b) => vec![*a, *b], + Multiplicity::Linear(terms) => terms + .iter() + .filter_map(|t| match t { + LinearTerm::Column { column, .. } + | LinearTerm::ColumnUnsigned { column, .. } => Some(*column), + LinearTerm::Constant(_) => None, + }) + .collect(), + _ => { + return Err(format!( + "interaction {i}: its multiplicity is not a column, a sum, a difference or a \ + linear form" + )); + } + }; + if let Some(&column) = read.iter().find(|&&c| c >= cols) { + return Err(format!( + "interaction {i}: multiplicity column {column} of {cols}" + )); + } + let at = |row: usize| -> FE { + match &interaction.multiplicity { + Multiplicity::Column(c) => fe(row, *c), + Multiplicity::Sum(a, b) => fe(row, *a) + fe(row, *b), + Multiplicity::Diff(a, b) => fe(row, *a) - fe(row, *b), + Multiplicity::Linear(terms) => terms.iter().fold(FE::zero(), |acc, t| { + acc + match t { + LinearTerm::Column { + coefficient, + column, + } => { + let c = FE::from(coefficient.unsigned_abs()); + let c = if *coefficient < 0 { -c } else { c }; + c * fe(row, *column) + } + LinearTerm::ColumnUnsigned { + coefficient, + column, + } => FE::from(*coefficient) * fe(row, *column), + LinearTerm::Constant(k) => { + let k_fe = FE::from(k.unsigned_abs()); + if *k < 0 { -k_fe } else { k_fe } + } + } + }), + _ => unreachable!("refused above"), + } + }; + if let Some(row) = (0..rows).find(|&row| at(row) != FE::zero()) { + return Err(format!( + "interaction {i}: its multiplicity is not zero at row {row}" + )); + } + } + Ok(()) +} + +/// `Ok` when `kept` is `today` (a family's tables with the whole run padded) +/// without its tables of padding rows alone: `kept` is `today`'s first +/// tables word for word, every table left out carries nothing on the bus, and +/// the last table kept holds a real row. +pub(super) fn is_today_without_its_padding_tables( + kept: &[Table], + today: &[Table], + interactions: &[BusInteraction], +) -> Result<(), String> { + if kept.len() > today.len() { + return Err(format!( + "{} tables kept of today's {}", + kept.len(), + today.len() + )); + } + for (k, (got, want)) in kept.iter().zip(today).enumerate() { + if got.narrow_main().is_some() != want.narrow_main().is_some() { + return Err(format!("table {k} is not held as today's")); + } + if shape_and_words(got) != shape_and_words(want) { + return Err(format!("table {k} is not today's")); + } + } + for (k, table) in today.iter().enumerate().skip(kept.len()) { + carries_nothing(table, interactions).map_err(|e| format!("table {k}, left out: {e}"))?; + } + match kept.last() { + None if today.is_empty() => Ok(()), + None => Err("every table left out".to_string()), + Some(last) if carries_nothing(last, interactions).is_ok() => Err(format!( + "table {}, the last kept, is padding alone", + kept.len() - 1 + )), + Some(_) => Ok(()), + } +} + +/// `count` permutations with distinct states, each 24 KECCAK_RND rows. +fn round_ops(count: usize) -> Vec { + (0..count as u64) + .map(|i| KeccakRoundOperation { + timestamp: 4 * i + 1, + input: std::array::from_fn(|l| { + (i + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ ((l as u64) << 7) + }), + output: std::array::from_fn(|l| (i + 3).wrapping_mul(0xc2b2_ae3d_27d4_eb4f) ^ l as u64), + }) + .collect() +} + +/// KECCAK_RND's tables of `rows` rows under `padding`. +fn round_tables( + ops: &[KeccakRoundOperation], + rows: usize, + padding: CutPadding, + pack: bool, + form: TraceForm, +) -> Vec

{ + keccak_rnd_chunks_in_waves(ops, rows, 0, padding, pack, form, usize::MAX).expect("the chunks") +} + +/// The default stops at the last real row: the type's default, a build's +/// default settings, and what an unset variable reads as. The knob reads two +/// words (`run`, the whole run padded) and refuses the rest. +#[test] +fn the_default_is_the_last_real_row_and_the_knob_reads_two_words() { + assert_eq!(CutPadding::default(), CutPadding::LastTable); + assert_eq!(StreamSkip::default().cut_padding, CutPadding::LastTable); + assert_eq!(CutPadding::parse(None), Ok(CutPadding::LastTable)); + assert_eq!(CutPadding::parse(Some(" last ")), Ok(CutPadding::LastTable)); + assert_eq!(CutPadding::parse(Some("run")), Ok(CutPadding::WholeRun)); + for other in ["", "0", "1", "Last", "whole", "last,run"] { + let refused = CutPadding::parse(Some(other)).expect_err("another word"); + assert!( + refused.contains("LAMBDA_VM_BLOCK_CUT_PADDING") && refused.contains("run or last"), + "{other}: {refused}" + ); + } +} + +/// The count against a model of the block's split: the whole table is +/// `n.next_power_of_two().max(4)` rows; no taller than the cut it is one +/// table; else it is cut into pieces of `rows` rows, and piece k holds a real +/// row when its first row is one (`k·rows < n`). The whole run padded counts +/// the pieces, the last-table rule the pieces with a real row. +#[test] +fn the_count_is_the_splits_pieces_and_those_with_a_real_row() { + for log2 in 0..=8 { + let rows = 1usize << log2; + for n in 0..=600usize { + let (pieces, real) = if n == 0 { + (0, 0) + } else { + let total = n.next_power_of_two().max(4); + if total <= rows { + (1, 1) + } else { + let pieces = total / rows; + (pieces, (0..pieces).filter(|k| k * rows < n).count()) + } + }; + assert_eq!( + CutPadding::WholeRun.tables(n, rows), + pieces, + "n {n} rows {rows}" + ); + assert_eq!( + CutPadding::LastTable.tables(n, rows), + real, + "n {n} rows {rows}" + ); + assert!(real <= pieces && (real > 0) == (n > 0), "n {n} rows {rows}"); + } + } + // A cut of no rows is no cut: one table, never a division. + assert_eq!(CutPadding::WholeRun.tables(9, 0), 1); + assert_eq!(CutPadding::LastTable.tables(9, 0), 1); +} + +/// The counts at the sizes of three mainnet blocks (permutations × 24 +/// KECCAK_RND rows at 2^16 rows a table, ECDAS steps at 2^17), with the whole +/// run padded and cut at the last real row. +#[test] +fn the_counts_at_three_blocks_sizes() { + for (n, rows, whole_run, last_table) in [ + (10_659 * 24, 1usize << 16, 4, 4), + (96_384 * 24, 1 << 16, 64, 36), + (194_333 * 24, 1 << 16, 128, 72), + (790_528, 1 << 17, 8, 7), + ] { + assert_eq!(CutPadding::WholeRun.tables(n, rows), whole_run, "{n}"); + assert_eq!(CutPadding::LastTable.tables(n, rows), last_table, "{n}"); + } +} + +/// KECCAK_RND with the whole run padded is the block's split of the whole +/// table; cut at the last real row it is that split without its tables of +/// padding rows alone — every op count around the cut sizes, wide, packed +/// after the build and written packed. (The split is the block prover's, a +/// host build's.) +#[cfg(feature = "parallel")] +#[test] +fn keccak_rnd_cut_at_the_last_real_row_is_the_split_without_its_padding_tables() { + let interactions = keccak_rnd::bus_interactions(); + let mut left_out = 0usize; + for count in [1usize, 2, 3, 5, 6, 11, 21, 22, 43] { + let ops = round_ops(count); + for rows in [32usize, 64, 128] { + let what = format!("{count} permutations at {rows} rows"); + let split = + crate::block_whir::split_rows(keccak_rnd::generate_keccak_rnd_trace(&ops), rows); + for (pack, form) in [ + (false, TraceForm::Wide), + (true, TraceForm::Wide), + (true, TraceForm::Narrow), + ] { + let today = round_tables(&ops, rows, CutPadding::WholeRun, pack, form); + assert_eq!(today.len(), split.len(), "{what}: the split's count"); + for (k, (got, want)) in today.iter().zip(&split).enumerate() { + assert_eq!( + shape_and_words(got), + shape_and_words(want), + "{what}: table {k} with the whole run padded" + ); + } + let last = round_tables(&ops, rows, CutPadding::LastTable, pack, form); + assert_eq!( + last.len(), + CutPadding::LastTable.tables(count * 24, rows), + "{what}: the count" + ); + is_today_without_its_padding_tables(&last, &today, &interactions) + .unwrap_or_else(|e| panic!("{what} (pack {pack}, {form:?}): {e}")); + left_out += today.len() - last.len(); + } + } + } + assert!(left_out > 0, "no table of padding rows alone was left out"); +} + +/// The check the tests above lean on refuses a cut one table short of the +/// real rows (the table it leaves out holds real rounds) and a cut one table +/// long (the last table it keeps is padding alone). +#[test] +fn a_cut_one_table_short_of_the_real_rows_is_caught() { + let interactions = keccak_rnd::bus_interactions(); + // 11 permutations = 264 rows: 512 with the run padded, eight tables of 64 + // rows, the first five with a real row. + let ops = round_ops(11); + let today = round_tables(&ops, 64, CutPadding::WholeRun, false, TraceForm::Wide); + let last = round_tables(&ops, 64, CutPadding::LastTable, false, TraceForm::Wide); + assert_eq!((today.len(), last.len()), (8, 5)); + is_today_without_its_padding_tables(&last, &today, &interactions).expect("the rule's tables"); + + let short = is_today_without_its_padding_tables(&today[..4], &today, &interactions) + .expect_err("a table with real rows left out"); + assert!( + short.starts_with("table 4, left out:") && short.contains("is not zero at row"), + "{short}" + ); + let long = is_today_without_its_padding_tables(&today[..6], &today, &interactions) + .expect_err("a table of padding rows kept"); + assert_eq!(long, "table 5, the last kept, is padding alone"); + // A multiplicity the check cannot read as a column, a sum, a difference or a + // linear form is an error, not a zero. + let mut unread = keccak_rnd::bus_interactions(); + unread[0].multiplicity = Multiplicity::One; + assert!( + carries_nothing(&today[7], &unread) + .expect_err("a constant multiplicity") + .contains("is not a column, a sum, a difference or a linear form") + ); + // A linear form with a constant term is read, not assumed zero: `1 - 0` on a + // padding row is a send. + let mut constant = keccak_rnd::bus_interactions(); + constant[0].multiplicity = Multiplicity::Linear(vec![stark::lookup::LinearTerm::Constant(1)]); + assert!( + carries_nothing(&today[7], &constant) + .expect_err("a constant term") + .contains("is not zero at row 0") + ); + carries_nothing(&today[7], &interactions).expect("a table of padding rows alone"); +} diff --git a/prover/src/tables/trace_builder/delta.rs b/prover/src/tables/trace_builder/delta.rs new file mode 100644 index 000000000..dc2d743fe --- /dev/null +++ b/prover/src/tables/trace_builder/delta.rs @@ -0,0 +1,269 @@ +//! Op lists the builder keeps until their tables are built, as a stream of +//! varints instead of fixed-size ops (S3c): a [`Codec`] writes each op from +//! the ops before it (deltas, zigzag LEB128) and reads it back exactly. +//! +//! Every [`DeltaStream::every`] ops a sparse index records where the stream +//! stands (the block, the byte, and the codec's state), so any range expands +//! on its own. The stream is stored in blocks of at most +//! [`super::blocks::BLOCK_BYTES`] that are never reallocated once full, and an +//! op never straddles two blocks (as [`super::CompactLt`]). The ops expand +//! exactly as pushed, in order, so the tables are the same. + +use super::{Expand, Part, Segmented, blocks}; + +/// How a family's ops are written: each from the ops before it, whose part +/// the codec keeps as its state (a `Copy` value the index records). +pub(super) trait Codec: Copy + Default + Send + Sync { + type Op: Clone + Send + Sync; + /// The most bytes one op takes, whatever its values. + const MAX_BYTES: usize; + /// Write `op` to `out`, after the ops this state has seen. + fn encode(&mut self, op: &Self::Op, out: &mut Vec); + /// Read the next op from `bytes` at `at`, after the ops this state has seen. + fn decode(&mut self, bytes: &[u8], at: &mut usize) -> Self::Op; +} + +/// Where the stream stands at op `k * every`: its block, the byte in it, and +/// the codec's state there. +#[derive(Clone, Copy)] +struct Mark { + block: u32, + at: u32, + state: C, +} + +pub(super) struct DeltaStream { + bytes: Vec>, + index: Vec>, + len: usize, + state: C, + /// Bytes a block of the stream holds; 0 is [`blocks::BLOCK_BYTES`] (tests + /// set a small one, to cross blocks). + block_bytes: usize, + /// Ops between two index marks. + every: usize, +} + +/// The index's spacing a new stream takes: 4096 ops, or a test's +/// ([`with_index_every`]). +fn default_every() -> usize { + #[cfg(test)] + { + let every = EVERY.with(std::cell::Cell::get); + if every > 0 { + return every; + } + } + 4096 +} + +#[cfg(test)] +thread_local! { + static EVERY: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// `f` with every stream it makes on this thread marked each `every` ops (a +/// test's spacing, so chunks and slices start past index marks). +#[cfg(test)] +pub(super) fn with_index_every(every: usize, f: impl FnOnce() -> T) -> T { + let before = EVERY.with(|cell| cell.replace(every.max(1))); + let out = f(); + EVERY.with(|cell| cell.set(before)); + out +} + +impl Default for DeltaStream { + fn default() -> Self { + Self { + bytes: Vec::new(), + index: Vec::new(), + len: 0, + state: C::default(), + block_bytes: 0, + every: default_every(), + } + } +} + +pub(super) fn zigzag(d: u64) -> u64 { + (d << 1) ^ ((d as i64 >> 63) as u64) +} + +pub(super) fn unzigzag(z: u64) -> u64 { + (z >> 1) ^ (z & 1).wrapping_neg() +} + +/// The bytes LEB128 takes for `v`. +pub(super) fn varint_len(v: u64) -> usize { + (64 - (v | 1).leading_zeros() as usize).div_ceil(7) +} + +pub(super) fn put(out: &mut Vec, mut v: u64) { + while v >= 0x80 { + out.push(v as u8 | 0x80); + v >>= 7; + } + out.push(v as u8); +} + +pub(super) fn get(bytes: &[u8], at: &mut usize) -> u64 { + let (mut v, mut shift) = (0u64, 0u32); + loop { + let b = bytes[*at]; + *at += 1; + v |= u64::from(b & 0x7f) << shift; + if b < 0x80 { + return v; + } + shift += 7; + } +} + +/// The cheapest of `codings` (mode, zigzag value): the mode and the value. +pub(super) fn cheapest(codings: [(u8, u64); N]) -> (u8, u64) { + let mut best = codings[0]; + for coding in codings { + if varint_len(coding.1) < varint_len(best.1) { + best = coding; + } + } + best +} + +impl DeltaStream { + /// A list whose stream is cut into blocks of `bytes` (at least an op's). + #[cfg(test)] + pub(super) fn with_block_bytes(bytes: usize) -> Self { + Self { + block_bytes: bytes.max(C::MAX_BYTES), + ..Self::default() + } + } + + /// `ops`, in order. + pub(super) fn from_ops(ops: &[C::Op]) -> Self { + let mut list = Self::default(); + list.extend(ops); + list + } + + fn block_bytes(&self) -> usize { + if self.block_bytes == 0 { + blocks::BLOCK_BYTES + } else { + self.block_bytes + } + } + + pub(super) fn len(&self) -> usize { + self.len + } + + /// Ops between two index marks. + #[cfg(test)] + pub(super) fn every(&self) -> usize { + self.every + } + + /// The stream's bytes (not its capacity). + #[cfg(test)] + pub(super) fn stream_bytes(&self) -> usize { + self.bytes.iter().map(Vec::len).sum() + } + + /// Append `ops`, in order. + pub(super) fn extend(&mut self, ops: &[C::Op]) { + let block_bytes = self.block_bytes(); + for op in ops { + // Room for the op in the last block: grown as a `Vec` grows up to + // a block's bytes, else a new block, allocated whole. + let room = self + .bytes + .last() + .is_some_and(|last| last.len() + C::MAX_BYTES <= block_bytes); + if !room { + let capacity = if self.bytes.is_empty() { + 4096.min(block_bytes) + } else { + block_bytes + }; + self.bytes.push(Vec::with_capacity(capacity)); + } + let block = self.bytes.len() - 1; + let Some(last) = self.bytes.last_mut() else { + continue; + }; + if last.capacity() - last.len() < C::MAX_BYTES { + let want = (last.capacity() * 2) + .min(block_bytes) + .max(last.len() + C::MAX_BYTES); + last.reserve_exact(want - last.len()); + } + if self.len.is_multiple_of(self.every) { + self.index.push(Mark { + block: block as u32, + at: last.len() as u32, + state: self.state, + }); + } + self.state.encode(op, last); + self.len += 1; + } + } + + /// The bytes it takes on the heap (capacities). + pub(super) fn heap_bytes(&self) -> usize { + self.bytes.iter().map(Vec::capacity).sum::() + + self.bytes.capacity() * std::mem::size_of::>() + + self.index.capacity() * std::mem::size_of::>() + } + + /// The bytes its largest single allocation takes. + pub(super) fn largest_bytes(&self) -> usize { + let blocks = self.bytes.iter().map(Vec::capacity).max().unwrap_or(0); + blocks.max(self.index.capacity() * std::mem::size_of::>()) + } + + /// The list as one compact segment. + pub(super) fn segments(&self) -> Segmented<'_, C::Op> { + Segmented { + parts: vec![Part::Compact(self)], + } + } + + /// Ops `start..end`, expanded. + pub(super) fn range(&self, start: usize, end: usize) -> Vec { + let mut out = Vec::new(); + self.expand_into(start, end, &mut out); + out + } +} + +impl Expand for DeltaStream { + fn len(&self) -> usize { + self.len + } + + fn expand_into(&self, start: usize, end: usize, out: &mut Vec) { + let end = end.min(self.len); + if start >= end { + return; + } + let first = start / self.every; + let mark = self.index[first]; + let (mut block, mut at, mut state) = (mark.block as usize, mark.at as usize, mark.state); + out.reserve(end - start); + for k in first * self.every..end { + // An op lies in one block: the next op starts the next block once + // this one ends. + if at >= self.bytes[block].len() { + block += 1; + at = 0; + } + let op = state.decode(&self.bytes[block], &mut at); + if k >= start { + out.push(op); + } + } + } +} diff --git a/prover/src/tables/trace_builder/finish.rs b/prover/src/tables/trace_builder/finish.rs new file mode 100644 index 000000000..f23ed57d0 --- /dev/null +++ b/prover/src/tables/trace_builder/finish.rs @@ -0,0 +1,3529 @@ +//! The table phase of a build (phases 3–5) as a plan and its tables. +//! +//! [`FinishPlan::new`] runs what needs the whole run before any table: LT's +//! segments (phase 3), the BITWISE histogram (phase 4), HALT and the REGISTER +//! final state, the public output and the PAGE layout; and counts every table +//! it will build without generating one ([`RestHeader`], what a statement is +//! read from). [`FinishPlan::emit_all`] then generates the tables (phase 5), +//! as [`super::build_traces`] always did, and refuses tables the header did +//! not count. The tables are the same as before the split, which the frozen +//! oracle (`finish_oracle`, test-only) checks. + +use super::*; + +/// What a statement reads off a build before any of its tables exists. +#[derive(Clone)] +pub(crate) struct RestHeader { + /// How many tables each counted chip makes ([`Traces::table_counts`] of the + /// tables the plan builds). + pub(crate) table_counts: crate::TableCounts, + /// One a PAGE table, in their order (none in a continuation epoch). + pub(crate) page_configs: Vec, + pub(crate) public_output_bytes: Vec, + pub(crate) num_blake3_ops: usize, + /// The CPU tables' heights, in order: what fixes each one's timestamp base + /// and the final PC timestamp (`tables::cpu::timestamp_bases`), so a block's + /// AIRs can be built before its tables are. + pub(crate) cpu_heights: Vec, + /// The rest is handed on in AIR order but BITWISE, which comes after every + /// other ([`StreamSkip::bitwise_last`], [`hand_on_order`]): the order the + /// layout admits. + pub(crate) bitwise_last: bool, + /// BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables, counted as + /// each is built ([`StreamSkip::bitwise_from_tables`]), BITWISE last. + pub(crate) bitwise_from_tables: bool, +} + +impl RestHeader { + /// [`Traces::runtime_page_ranges`] of the tables the plan builds. + pub(crate) fn runtime_page_ranges(&self) -> Vec { + runtime_page_ranges_of(&self.page_configs) + } +} + +/// [`crate::TableCounts`] field by field, to compare two. +pub(super) fn counts_of(c: &crate::TableCounts) -> [usize; 23] { + [ + c.cpu, + c.lt, + c.memw, + c.memw_aligned, + c.load, + c.mul, + c.dvrm, + c.shift, + c.branch, + c.memw_register, + c.eq, + c.bytewise, + c.store, + c.cpu32, + c.keccak, + c.keccak_rnd, + c.ecsm, + c.ecdas, + c.hint, + c.copy, + c.blake3, + c.is_wh, + c.ecrec, + ] +} + +/// LT's ops as phase 3 leaves them: the segments its list is the +/// concatenation of, in this order ([`Segmented`]); with +/// [`StreamSkip::concat_lt`], all of them in `walk`. +#[derive(Default)] +struct LtLists { + /// The walk's LT ops not streamed and what DVRM implies, in blocks. + walk: BlockVec, + /// The phase-3 LT ops of the MEMW and MEMW_A ops a windowed build dropped. + memw_lt: CompactLt, + /// The phase-3 LT ops of the MEMW ops left. + from_memw: Vec, + memw_aligned_lt: CompactLt, + from_memw_aligned: Vec, + /// HINT's range checks. + from_hints: Vec, +} + +impl LtLists { + fn segments(&self) -> Segmented<'_, LtOperation> { + let mut parts: Vec> = self.walk.parts().map(Part::Ops).collect(); + parts.extend([ + Part::Compact(&self.memw_lt), + Part::Ops(&self.from_memw), + Part::Compact(&self.memw_aligned_lt), + Part::Ops(&self.from_memw_aligned), + Part::Ops(&self.from_hints), + ]); + Segmented { parts } + } +} + +/// How many tables a chunked list of `len` ops makes at `max` ops a table, the +/// first `skip` of them streamed ahead: [`chunk_and_generate_skipping`]'s and +/// [`chunk_and_generate_segmented`]'s count (`tails`: the list holds only the +/// ops past the streamed chunks; `optional`: an empty list makes no table). +fn chunked( + len: usize, + max: usize, + skip: usize, + tails: bool, + optional: bool, +) -> Result { + let chunks = len.div_ceil(max.max(1)); + if skip == 0 { + return Ok(if len == 0 && !optional { 1 } else { chunks }); + } + if tails { + return Ok(skip + chunks); + } + if skip > chunks { + return Err(Error::Prover(format!( + "{skip} chunks were streamed but the run has {chunks} of this table" + ))); + } + Ok(chunks) +} + +/// A build's table phase, planned: every list phase 5 reads, what phases 3 and +/// 4 derived from them, and the [`RestHeader`]. See the module docs. +pub(super) struct FinishPlan<'a> { + memory_state: &'a MemoryState, + register_init: &'a [u32], + decode_trace: TraceTable, + decode_table: &'a DecodeTable, + max_rows: &'a crate::tables::MaxRowsConfig, + l2g_memory_bookend: bool, + skip: StreamSkip, + cpu_ops: Vec, + memw_ops: Vec, + memw_aligned_ops: Vec, + memw_register_rows: Vec, + load_ops: Vec, + lt: LtLists, + shift_ops: BlockVec, + branch_ops: CompactBranch, + mul_ops: BlockVec<(MulOperation, bool)>, + dvrm_ops: BlockVec<(DvrmOperation, bool)>, + copy_ops: BlockVec, + keccak_ops: BlockVec, + blake3_ops: BlockVec, + blake3_absorb_ops: BlockVec, + eq_ops: CompactEq, + bytewise_ops: CompactBytewise, + store_ops: BlockVec, + cpu32_ops: BlockVec, + ecsm_ops: BlockVec, + ecdas_ops: BlockVec, + hint_ops: BlockVec, + ecrec_ops: BlockVec, + /// IS_WH's rows ([`is_wh::dedup_is_wh_rows`] over KECCAK's state addresses), + /// shared by its BITWISE lookups and its table. + is_wh_rows: Vec<(is_wh::IsWhKey, u64)>, + bitwise: BitwiseCount<'a>, + num_padding_rows: usize, + halt_timestamp: u64, + halt_next_pc: u64, + register_final_state: FinalRegisterStateMap, + header: RestHeader, + /// [`Self::cut_line`]. + cut_line: Option, +} + +/// The lists phase 4 counts the BITWISE lookups of ([`count_bitwise`]). +#[derive(Clone, Copy)] +struct P4Lists<'s> { + lt: &'s LtLists, + shift_ops: &'s BlockVec, + store_ops: &'s BlockVec, + memw_aligned_ops: &'s [memw_aligned::AlignedRow], + memw_register_rows: &'s [RegRow], + branch_ops: &'s CompactBranch, + mul_ops: &'s BlockVec<(MulOperation, bool)>, + dvrm_ops: &'s BlockVec<(DvrmOperation, bool)>, + eq_ops: &'s CompactEq, + bytewise_ops: &'s CompactBytewise, + keccak_ops: &'s BlockVec, + ecdas_ops: &'s BlockVec, + copy_ops: &'s BlockVec, + blake3_ops: &'s BlockVec, + blake3_absorb_ops: &'s BlockVec, + ecsm_ops: &'s BlockVec, + hint_ops: &'s BlockVec, + ecrec_ops: &'s BlockVec, + is_wh_rows: &'s [(is_wh::IsWhKey, u64)], + max_rows: &'s crate::tables::MaxRowsConfig, + num_padding_rows: usize, + /// The byte checks SHIFT's and KECCAK_RND's padding rows send + /// ([`padding_byte_checks`]). + padding_byte_checks: u64, + /// How the count runs ([`P4Mode`]). + mode: P4Mode, +} + +/// How phase 4 counts: the per-row lookups on the pool and how +/// ([`StreamSkip::p4_rows`], [`StreamSkip::p4_by`]), and LT's and KECCAK's +/// lookups straight into the histograms ([`StreamSkip::p4_direct`]). +#[derive(Clone, Copy)] +struct P4Mode { + rows_on_pool: bool, + by: RowCount, + direct: bool, + /// LT's, BRANCH's and BYTEWISE's lookups are left to their tables' jobs + /// ([`StreamSkip::bitwise_from_tables`]): none of their ops is read here. + from_tables: bool, +} + +impl P4Mode { + fn of(skip: &StreamSkip) -> Self { + Self { + rows_on_pool: skip.p4_rows, + by: skip.p4_by, + direct: skip.p4_direct, + from_tables: false, + } + } +} + +/// PAGE's BITWISE lookups, from the initial image and the memory state. +type PageCount<'a> = Box; + +/// Phase 4: the BITWISE histogram of every source, the in-walk lookups +/// (`bitwise_ops`) and PAGE's (`page`) with them, on top of what the windows +/// counted (`pre`). With `own_threads`, the buckets run on threads of their +/// own instead of the pool. +fn count_bitwise( + lists: P4Lists<'_>, + bitwise_ops: &BlockVec, + page: Option<&(dyn Fn(&mut bitwise::BitwiseHistogram) + Sync)>, + pre: Option, + #[cfg_attr(not(feature = "parallel"), allow(unused_variables))] own_threads: bool, +) -> bitwise::BitwiseHistogram { + let P4Lists { + lt, + shift_ops, + store_ops, + memw_aligned_ops, + memw_register_rows, + branch_ops, + mul_ops, + dvrm_ops, + eq_ops, + bytewise_ops, + keccak_ops, + ecdas_ops, + copy_ops, + blake3_ops, + blake3_absorb_ops, + ecsm_ops, + hint_ops, + ecrec_ops, + is_wh_rows, + max_rows, + num_padding_rows, + padding_byte_checks, + mode: + P4Mode { + rows_on_pool, + by, + direct, + from_tables, + }, + } = lists; + let lt_ops = lt.segments(); + let branch_segments = branch_ops.segments(); + let bytewise_segments = bytewise_ops.segments(); + // The per-source bitwise collectors are all pure functions of their inputs, and the + // BITWISE multiplicities are order-independent (they ride a permutation-invariant bus), + // so every source can be collected in parallel and the per-worker histograms summed in + // any order. + // + // MUL/DVRM dedup their per-unique bit-gated lookups PER CHIP INSTANCE, so pass the same + // chunk size used to split them into instances so multiplicities match the per-instance + // sends. MEMW_R sends IS_HALFWORD[timestamp_0 - old_timestamp_lo - 1]. PAGE does a + // batched ARE_BYTES[init, fini] per row (skipped in continuation epochs, which the L2G + // table owns). COPY sends IsHalfword; KECCAK_RND sends XOR/AND/ARE_BYTES/HWSL; + // HINT sends ARE_BYTES for its 32 output cells. + // We never concatenate the lookups into one giant `Vec` (~140 M ops / + // ~560 MB at 10-tx whose only consumer is the multiplicity count). Each collector bumps + // the `BitwiseHistogram` it is handed: the heavy sources (MEMW_R one-per-row, PAGE + // one-per-byte, padding) count directly with no per-source Vec at all, and the small + // sources fold their transient `collect_*` Vec in and drop it. The histogram is a + // commutative monoid, so per-worker histograms tree-reduce to multiplicities that are + // independent of accumulation order. + type Collector<'a> = Box; + let mul_chunk = max_rows.mul; + let dvrm_chunk = max_rows.dvrm; + // The sources that are a sum over their ops are cut into slices of whole ops, so phase 4's + // buckets share them (a whole source was one bucket's long pole) and no slice's list of + // lookups grows with the run. MUL and DVRM deduplicate per instance, so each of their + // slices is one instance. The in-walk lookups and MEMW_R are split in the parallel path + // below; the rest stay one collector each. + let mut collectors: Vec = Vec::new(); + // Where each source's collectors start in `collectors`, in order, so the + // parallel path can say what each source cost (the `BLOCK P4 ROWS` line). + let mut starts: Vec<(&'static str, usize)> = Vec::new(); + if p4_sliced() { + // LT's segments slice by slice; a compact one through a temporary of a + // slice's ops at a time. + starts.push(("LT", collectors.len())); + // Counted straight into the histogram (`skip.p4_direct`), or + // through each slice's list of lookups, as before. + let lt_lookups = move |slice: &[LtOperation], h: &mut bitwise::BitwiseHistogram| { + if direct { + count_bitwise_from_lt_ops(slice, h) + } else { + h.add_ops(&collect_bitwise_from_lt_ops(slice)) + } + }; + let lt_slice = p4_slice_len(P4Source::Lt, mul_chunk, dvrm_chunk); + for part in lt_ops.parts.iter().filter(|_| !from_tables) { + match *part { + Part::Ops(ops) => { + for slice in ops.chunks(lt_slice) { + collectors.push(Box::new(move |h| lt_lookups(slice, h))); + } + } + Part::Compact(compact) => { + for start in (0..compact.len()).step_by(lt_slice) { + collectors.push(Box::new(move |h| { + let mut slice = Vec::new(); + compact.expand_into(start, start + lt_slice, &mut slice); + lt_lookups(&slice, h) + })); + } + } + } + } + // MUL and DVRM deduplicate per instance: their slices are the + // instances of the concatenation (copied only where one straddles two + // blocks). The other sources are sums over their ops: block by block. + starts.push(("MUL", collectors.len())); + let mul_slice = p4_slice_len(P4Source::Mul, mul_chunk, dvrm_chunk); + for k in 0..mul_ops.len().div_ceil(mul_slice) { + let mul_ops = &mul_ops; + collectors.push(Box::new(move |h| { + let slice = mul_ops.range(k * mul_slice, (k + 1) * mul_slice); + h.add_ops(&collect_bitwise_from_mul(&slice, mul_chunk)) + })); + } + starts.push(("DVRM", collectors.len())); + let dvrm_slice = p4_slice_len(P4Source::Dvrm, mul_chunk, dvrm_chunk); + for k in 0..dvrm_ops.len().div_ceil(dvrm_slice) { + let dvrm_ops = &dvrm_ops; + collectors.push(Box::new(move |h| { + let slice = dvrm_ops.range(k * dvrm_slice, (k + 1) * dvrm_slice); + h.add_ops(&collect_bitwise_from_dvrm(&slice, dvrm_chunk)) + })); + } + starts.push(("BRANCH", collectors.len())); + let branch_slice = p4_slice_len(P4Source::Branch, mul_chunk, dvrm_chunk); + let branch_slices = if from_tables { + 0 + } else { + branch_ops.len().div_ceil(branch_slice.max(1)) + }; + for k in 0..branch_slices { + let branch_ops = &branch_ops; + collectors.push(Box::new(move |h| { + let slice = branch_ops.range(k * branch_slice, (k + 1) * branch_slice); + h.add_ops(&collect_bitwise_from_branch_ops(&slice)) + })); + } + starts.push(("SHIFT", collectors.len())); + let shift_slice = p4_slice_len(P4Source::Shift, mul_chunk, dvrm_chunk); + for slice in shift_ops.parts().flat_map(|part| part.chunks(shift_slice)) { + collectors.push(Box::new(move |h| { + h.add_ops(&shift::collect_bitwise_from_shift(slice)) + })); + } + starts.push(("BYTEWISE", collectors.len())); + let bytewise_slice = p4_slice_len(P4Source::Bytewise, mul_chunk, dvrm_chunk); + let bytewise_slices = if from_tables { + 0 + } else { + bytewise_ops.len().div_ceil(bytewise_slice.max(1)) + }; + for k in 0..bytewise_slices { + let bytewise_ops = &bytewise_ops; + collectors.push(Box::new(move |h| { + for op in bytewise_ops.range(k * bytewise_slice, (k + 1) * bytewise_slice) { + h.add_ops(&op.collect_bitwise_ops()); + } + })); + } + starts.push(("EQ", collectors.len())); + let eq_slice = p4_slice_len(P4Source::Eq, mul_chunk, dvrm_chunk); + for k in 0..eq_ops.len().div_ceil(eq_slice.max(1)) { + let eq_ops = &eq_ops; + collectors.push(Box::new(move |h| { + for op in eq_ops.range(k * eq_slice, (k + 1) * eq_slice) { + h.add_ops(&op.collect_bitwise_ops()); + } + })); + } + starts.push(("STORE", collectors.len())); + let store_slice = p4_slice_len(P4Source::Store, mul_chunk, dvrm_chunk); + for slice in store_ops.parts().flat_map(|part| part.chunks(store_slice)) { + collectors.push(Box::new(move |h| { + for op in slice { + h.add_ops(&op.collect_bitwise_ops()); + } + })); + } + starts.push(("MEMW_A", collectors.len())); + for slice in + memw_aligned_ops.chunks(p4_slice_len(P4Source::MemwAligned, mul_chunk, dvrm_chunk)) + { + collectors.push(Box::new(move |h| { + h.add_ops(&collect_bitwise_from_memw_aligned(slice)) + })); + } + starts.push(("KECCAK", collectors.len())); + let keccak_slice = p4_slice_len(P4Source::Keccak, mul_chunk, dvrm_chunk); + for slice in keccak_ops + .parts() + .flat_map(|part| part.chunks(keccak_slice)) + { + collectors.push(Box::new(move |h| { + if direct { + count_bitwise_from_keccak(slice, h) + } else { + h.add_ops(&collect_bitwise_from_keccak(slice)) + } + })); + } + starts.push(("ECDAS", collectors.len())); + let ecdas_slice = p4_slice_len(P4Source::Ecdas, mul_chunk, dvrm_chunk); + for slice in ecdas_ops.parts().flat_map(|part| part.chunks(ecdas_slice)) { + collectors.push(Box::new(move |h| { + h.add_ops(&collect_bitwise_from_compact_ecdas(slice)) + })); + } + } else { + for (k, name) in [ + "LT", "MUL", "DVRM", "BRANCH", "SHIFT", "BYTEWISE", "EQ", "STORE", "MEMW_A", "KECCAK", + "ECDAS", + ] + .into_iter() + .enumerate() + { + starts.push((name, collectors.len() + k)); + } + // `LAMBDA_VM_P4_SLICED=0`: every source one collector, as before the slices (the A/B's + // control). The same multiplicities: the histogram is a commutative sum. + collectors.extend([ + Box::new(|h: &mut bitwise::BitwiseHistogram| { + for part in lt_ops.parts.iter().filter(|_| !from_tables) { + match *part { + Part::Ops(ops) => h.add_ops(&collect_bitwise_from_lt_ops(ops)), + Part::Compact(compact) => { + for start in (0..compact.len()).step_by(1 << 20) { + let mut slice = Vec::new(); + compact.expand_into(start, start + (1 << 20), &mut slice); + h.add_ops(&collect_bitwise_from_lt_ops(&slice)); + } + } + } + } + }) as Collector, + // MUL and DVRM deduplicate per instance: instance by instance. + Box::new(|h| { + for k in 0..mul_ops.len().div_ceil(mul_chunk.max(1)) { + let slice = mul_ops.range(k * mul_chunk, (k + 1) * mul_chunk); + h.add_ops(&collect_bitwise_from_mul(&slice, mul_chunk)) + } + }), + Box::new(|h| { + for k in 0..dvrm_ops.len().div_ceil(dvrm_chunk.max(1)) { + let slice = dvrm_ops.range(k * dvrm_chunk, (k + 1) * dvrm_chunk); + h.add_ops(&collect_bitwise_from_dvrm(&slice, dvrm_chunk)) + } + }), + Box::new(|h| { + for k in 0..branch_ops.len().div_ceil(1 << 20) * usize::from(!from_tables) { + let slice = branch_ops.range(k << 20, (k + 1) << 20); + h.add_ops(&collect_bitwise_from_branch_ops(&slice)) + } + }), + Box::new(|h| { + for part in shift_ops.parts() { + h.add_ops(&shift::collect_bitwise_from_shift(part)) + } + }), + Box::new(|h| { + for k in 0..bytewise_ops.len().div_ceil(1 << 20) * usize::from(!from_tables) { + for op in bytewise_ops.range(k << 20, (k + 1) << 20) { + h.add_ops(&op.collect_bitwise_ops()); + } + } + }), + Box::new(|h| { + for k in 0..eq_ops.len().div_ceil(1 << 20) { + for op in eq_ops.range(k << 20, (k + 1) << 20) { + h.add_ops(&op.collect_bitwise_ops()); + } + } + }), + Box::new(|h| { + for op in store_ops.iter() { + h.add_ops(&op.collect_bitwise_ops()); + } + }), + Box::new(|h| h.add_ops(&collect_bitwise_from_memw_aligned(memw_aligned_ops))), + Box::new(|h| { + for part in keccak_ops.parts() { + h.add_ops(&collect_bitwise_from_keccak(part)) + } + }), + Box::new(|h| { + for part in ecdas_ops.parts() { + h.add_ops(&collect_bitwise_from_compact_ecdas(part)) + } + }), + ]); + } + // COPY's rows are millions at a block's size: a collector a block of + // them, each counted straight into its histogram. + starts.push(("COPY", collectors.len())); + for part in copy_ops.parts() { + collectors.push(Box::new(move |h| count_bitwise_from_copy(part, h))); + } + for (k, name) in ["BLAKE3", "ECSM", "HINT", "ECREC", "PADDING"] + .into_iter() + .enumerate() + { + starts.push((name, collectors.len() + k)); + } + collectors.extend([ + // Small lists, one block each at a block's size: whole. + Box::new(|h: &mut bitwise::BitwiseHistogram| { + if !strip_blake3_side_effects() { + h.add_ops(&collect_bitwise_from_blake3( + &blake3_ops.whole(), + &blake3_absorb_ops.whole(), + )); + } + }) as Collector, + Box::new(|h| h.add_ops(&collect_bitwise_from_ecsm(&ecsm_ops.whole()))), + Box::new(|h| h.add_ops(&collect_bitwise_from_hint(&hint_ops.whole()))), + Box::new(|h| h.add_ops(&collect_bitwise_from_ecrec(&ecrec_ops.whole()))), + Box::new(|h| add_padding_byte_checks(h, num_padding_rows)), + ]); + // The per-row lookups (`types::MU_MAX`): the μ bound every row of a + // deduplicating table sends, padding rows included, and the byte checks + // SHIFT's and KECCAK_RND's padding rows send, over exactly the tables + // this build generates, a collector a table: the chunks of the lists here + // ([`list_chunks`]); with only the tails kept, the streamed chunks' were + // counted as they were handed out (their full chunks' padding here). + // With `skip.p4_rows` they are counted on the whole pool instead + // (`count_row_lookups`, below). + starts.push(("LT_ROWS", collectors.len())); + for (start, end) in list_chunks(lt_ops.len(), max_rows.lt) + .into_iter() + .filter(|_| !rows_on_pool && !from_tables) + { + let lt_ops = <_ops; + collectors.push(Box::new(move |h| { + h.add_ops(<_row_lookups(lt_ops.range(start, end).iter())) + })); + } + starts.push(("BRANCH_ROWS", collectors.len())); + for (start, end) in list_chunks(branch_segments.len(), max_rows.branch) + .into_iter() + .filter(|_| !rows_on_pool && !from_tables) + { + let branch_segments = &branch_segments; + collectors.push(Box::new(move |h| { + h.add_ops(&branch_row_lookups(&branch_segments.range(start, end))) + })); + } + starts.push(("BYTEWISE_ROWS", collectors.len())); + for (start, end) in list_chunks(bytewise_segments.len(), max_rows.bytewise) + .into_iter() + .filter(|_| !rows_on_pool && !from_tables) + { + let bytewise_segments = &bytewise_segments; + collectors.push(Box::new(move |h| { + h.add_ops(&bytewise_row_lookups(&bytewise_segments.range(start, end))) + })); + } + let count_rows = || { + (rows_on_pool && !from_tables) + .then(|| count_row_lookups(<_ops, &branch_segments, &bytewise_segments, max_rows, by)) + }; + // Without a row count there is no `BLOCK P4 ROWS` line to say the + // collectors' walls on: one that says why. + #[cfg(feature = "parallel")] + if from_tables { + start_row_scan_line( + "BLOCK P4 ROWS: none in phase 4 · LT's, BRANCH's and BYTEWISE's lookups counted from \ + their tables' rows" + .to_string(), + ); + } + starts.push(("PADDING", collectors.len())); + collectors.push(Box::new(move |h| { + h.bump_n( + BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, 0, 0), + padding_byte_checks, + ) + })); + starts.push(("IS_WH", collectors.len())); + collectors.push(Box::new(|h| { + is_wh::collect_bitwise_from_is_wh(is_wh_rows, h) + })); + starts.push(("PAGE", collectors.len())); + if let Some(page) = page { + collectors.push(Box::new(move |h| page(h))); + } + let (mut base, counted_iw, counted_reg) = match pre { + Some(pre) => (pre.histogram, pre.bitwise_ops, pre.memw_register_rows), + None => (bitwise::BitwiseHistogram::new(), 0, 0), + }; + // The in-walk lookups past the counted ones, block by block. + let mut skip_iw = counted_iw; + let uncounted_iw: Vec<&[BitwiseOperation]> = bitwise_ops + .parts() + .filter_map(|part| { + let from = skip_iw.min(part.len()); + skip_iw -= from; + (from < part.len()).then(|| &part[from..]) + }) + .collect(); + let uncounted_reg = &memw_register_rows[counted_reg..]; + + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + // Cap concurrent 80 MiB histograms at `cap` to bound peak memory. The two dominant + // sources — the in-walk lookups and MEMW_R (each tens of millions of items) — are + // split into ~`cap` row-range slices so they parallelize INTERNALLY instead of each + // pinning one core while the rest idle. Every unit (whole collectors + the heavy + // slices) is round-robined into exactly `cap` buckets, one histogram each, so the + // split heavy work is spread across buckets rather than piled into one. + // add_ops/bump/merge form a commutative monoid, so any partition yields + // byte-identical multiplicities (same as the serial fallback below). + let cap = rayon::current_num_threads().clamp(1, 8); + let mut units: Vec<(&'static str, Collector)> = + Vec::with_capacity(collectors.len() + 2 * cap); + let iw_total: usize = uncounted_iw.iter().map(|part| part.len()).sum(); + let iw_chunk = iw_total.div_ceil(cap).max(1); + for slice in uncounted_iw.iter().flat_map(|part| part.chunks(iw_chunk)) { + units.push(("IN_WALK", Box::new(move |h| h.add_ops(slice)))); + } + let reg_chunk = uncounted_reg.len().div_ceil(cap).max(1); + for slice in uncounted_reg.chunks(reg_chunk) { + units.push(( + "MEMW_R", + Box::new(move |h| memw_register::collect_bitwise_from_memw_register(slice, h)), + )); + } + let mut at = 0usize; + for (k, unit) in collectors.into_iter().enumerate() { + while at + 1 < starts.len() && starts[at + 1].1 <= k { + at += 1; + } + let name = starts.get(at).filter(|s| s.1 <= k).map_or("OTHER", |s| s.0); + units.push((name, unit)); + } + + let mut buckets: Vec> = + (0..cap).map(|_| Vec::new()).collect(); + for (i, unit) in units.into_iter().enumerate() { + buckets[i % cap].push(unit); + } + // The row counts on the pool beside the buckets; the line says how + // long the buckets took beside them, and what each source cost. + type Run = ( + bitwise::BitwiseHistogram, + Vec<(&'static str, f64)>, + f64, + f64, + ); + let run_bucket = |bucket: &Vec<(&'static str, Collector)>| -> Run { + let begun = std::time::Instant::now(); + let mut h = bitwise::BitwiseHistogram::new(); + let mut by: Vec<(&'static str, f64)> = Vec::new(); + for (name, f) in bucket { + let t = std::time::Instant::now(); + f(&mut h); + add_source_seconds(&mut by, name, t.elapsed().as_secs_f64()); + } + let wall = begun.elapsed().as_secs_f64(); + (h, by, wall, wall) + }; + let merge_runs = |(mut a, mut by, most, least): Run, (b, more, m, l): Run| -> Run { + a.merge(&b); + for (name, secs) in more { + add_source_seconds(&mut by, name, secs); + } + (a, by, most.max(m), least.min(l)) + }; + let run_bucket = &run_bucket; + let buckets = &buckets; + // On the pool, or beside the rest's builds on threads of their own, so + // the pool's queue of builds cannot hold the buckets back. + let run_buckets = |own: bool| { + let started = std::time::Instant::now(); + let reduced = if own { + std::thread::scope(|scope| { + let running: Vec<_> = buckets + .iter() + .map(|bucket| scope.spawn(move || run_bucket(bucket))) + .collect(); + running + .into_iter() + .map(|t| t.join().unwrap_or_else(|e| std::panic::resume_unwind(e))) + .reduce(merge_runs) + }) + } else { + buckets.par_iter().map(run_bucket).reduce_with(merge_runs) + }; + (reduced, started.elapsed().as_secs_f64()) + }; + let (reduced, rows) = if own_threads { + std::thread::scope(|scope| { + let counting = scope.spawn(count_rows); + let reduced = run_buckets(true); + let rows = counting + .join() + .unwrap_or_else(|e| std::panic::resume_unwind(e)); + (reduced, rows) + }) + } else { + rayon::join(|| run_buckets(false), count_rows) + }; + let (reduced, collectors) = reduced; + let reduced = reduced.map(|(h, by, most, least)| { + if rows.is_some() || from_tables { + add_to_row_scan_line(&format!( + " · beside it, phase 4's other collectors in {collectors:.2} s on {cap} \ + buckets (the longest {most:.2} s, the shortest {least:.2} s) · by source, \ + summed over the buckets: {}", + by.iter() + .map(|(name, secs)| format!("{name} {secs:.2} s")) + .collect::>() + .join(" · ") + )); + add_to_row_scan_line(if direct { + " · LT's and KECCAK's lookups counted directly" + } else { + " · LT's and KECCAK's lookups through lists" + }); + } + h + }); + if let Some(reduced) = reduced { + base.merge(&reduced); + } + if let Some(rows) = rows { + rows.bump_into(&mut base); + } + } + #[cfg(not(feature = "parallel"))] + { + for part in &uncounted_iw { + base.add_ops(part); + } + memw_register::collect_bitwise_from_memw_register(uncounted_reg, &mut base); + for f in &collectors { + f(&mut base); + } + if let Some(rows) = count_rows() { + rows.bump_into(&mut base); + } + } + if rows_on_pool && !from_tables { + build_stamps::mark("p4 rows pool"); + } + base +} + +/// The BITWISE histogram, counted in [`FinishPlan::new`] or, beside the +/// rest's builds ([`StreamSkip::p4_beside`]), left for BITWISE's own table to +/// count: with it what phase 4 reads that the plan does not keep. +enum BitwiseCount<'a> { + Counted(bitwise::BitwiseHistogram), + Beside(Box>), +} + +struct P4Pending<'a> { + bitwise_ops: BlockVec, + page: Option>, + pre: Option, +} + +/// The lists phase 4 reads, each shared with the units that build its +/// tables ([`FinishPlan::units_keeping`]). +#[derive(Clone)] +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +struct P4Shared { + lt: std::sync::Arc, + shift_ops: std::sync::Arc>, + store_ops: std::sync::Arc>, + memw_aligned_ops: std::sync::Arc>, + memw_register_rows: std::sync::Arc>, + branch_ops: std::sync::Arc, + mul_ops: std::sync::Arc>, + dvrm_ops: std::sync::Arc>, + eq_ops: std::sync::Arc, + bytewise_ops: std::sync::Arc, + keccak_ops: std::sync::Arc>, + ecdas_ops: std::sync::Arc>, + copy_ops: std::sync::Arc>, + blake3_ops: std::sync::Arc>, + blake3_absorb_ops: std::sync::Arc>, + ecsm_ops: std::sync::Arc>, + hint_ops: std::sync::Arc>, + ecrec_ops: std::sync::Arc>, + is_wh_rows: std::sync::Arc>, +} + +impl P4Shared { + #[cfg_attr(not(feature = "parallel"), allow(dead_code))] + fn lists<'s>( + &'s self, + max_rows: &'s crate::tables::MaxRowsConfig, + num_padding_rows: usize, + padding_byte_checks: u64, + mode: P4Mode, + ) -> P4Lists<'s> { + P4Lists { + lt: &self.lt, + shift_ops: &self.shift_ops, + store_ops: &self.store_ops, + memw_aligned_ops: &self.memw_aligned_ops, + memw_register_rows: &self.memw_register_rows, + branch_ops: &self.branch_ops, + mul_ops: &self.mul_ops, + dvrm_ops: &self.dvrm_ops, + eq_ops: &self.eq_ops, + bytewise_ops: &self.bytewise_ops, + keccak_ops: &self.keccak_ops, + ecdas_ops: &self.ecdas_ops, + copy_ops: &self.copy_ops, + blake3_ops: &self.blake3_ops, + blake3_absorb_ops: &self.blake3_absorb_ops, + ecsm_ops: &self.ecsm_ops, + hint_ops: &self.hint_ops, + ecrec_ops: &self.ecrec_ops, + is_wh_rows: &self.is_wh_rows, + max_rows, + num_padding_rows, + padding_byte_checks, + mode, + } + } +} + +impl P4Pending<'_> { + /// The histogram, counted now over `lists` (`own_threads` as + /// [`count_bitwise`]). + fn count(self, lists: P4Lists<'_>, own_threads: bool) -> bitwise::BitwiseHistogram { + let Self { + bitwise_ops, + page, + pre, + } = self; + let page = page.as_deref().map(|p| p as _); + count_bitwise(lists, &bitwise_ops, page, pre, own_threads) + } +} + +/// The byte checks SHIFT's and KECCAK_RND's padding rows send: the tables of +/// the lists here ([`list_chunks`]) and, with only the tails kept, SHIFT's +/// streamed chunks' full tables. +fn padding_byte_checks( + shift_ops: usize, + keccak_ops: usize, + max_rows: &crate::tables::MaxRowsConfig, + skip: &StreamSkip, +) -> u64 { + let shift_padding: usize = list_chunks(shift_ops, max_rows.shift) + .iter() + .map(|&(start, end)| padding_rows_of(end - start)) + .sum::() + + if skip.tails { + skip.shift * padding_rows_of(max_rows.shift) + } else { + 0 + }; + let keccak_rnd_padding = keccak_rnd::ARE_BYTES_PER_ROW + * keccak_rnd_padding_rows(keccak_ops, skip.keccak_rnd_rows, skip.cut_padding); + (shift_padding + keccak_rnd_padding) as u64 +} + +/// BITWISE's job: the histogram as counted, or what to count it from. +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +enum BitwiseJob<'a> { + Counted(bitwise::BitwiseHistogram), + Beside(Box>, P4Shared), + /// The other sources' count, sent by the finish's side count ([`Side`]), + /// and LT's, BRANCH's and BYTEWISE's from their tables ([`FromTables`]). + FromTables( + std::sync::mpsc::Receiver, + std::sync::Arc, + ), +} + +/// Work the sliding finish runs on a thread of its own while it builds the +/// rest: phase 4's count of the sources BITWISE's job does not get from the +/// tables ([`StreamSkip::bitwise_from_tables`]). BITWISE's job waits for it on +/// a pool thread, so it must not need the pool: its collectors run on threads +/// of their own ([`count_bitwise`] with `own_threads`) and none of them uses +/// the pool. +pub(super) type Side<'a> = Box; + +/// BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables +/// ([`StreamSkip::bitwise_from_tables`]): each table's job adds its rows' +/// lookups as it builds the table, and BITWISE's job, built after every one +/// of them, takes the sum. A table that adds after BITWISE took it, or a sum +/// short of a table, is refused. +pub(super) struct FromTables { + counted: std::sync::Mutex>, + /// The tables that add to it: LT's, BRANCH's and BYTEWISE's built units. + tables: std::sync::atomic::AtomicUsize, +} + +impl FromTables { + fn new() -> Self { + Self { + counted: std::sync::Mutex::new(Some((bitwise::BitwiseHistogram::new(), 0))), + tables: std::sync::atomic::AtomicUsize::new(0), + } + } + + /// Adds one table's lookups. + fn add(&self, plane: &bitwise::BitwisePlane) -> Result<(), Error> { + let mut counted = self.counted.lock().unwrap_or_else(|e| e.into_inner()); + let (histogram, tables) = counted.as_mut().ok_or_else(|| { + Error::Prover("a table's BITWISE lookups came after BITWISE was built".into()) + })?; + histogram.merge_plane(plane); + *tables += 1; + Ok(()) + } + + /// The sum, once every table has added to it. + fn take(&self) -> Result { + let (histogram, added) = self + .counted + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take() + .ok_or_else(|| { + Error::Prover("BITWISE's lookups from the tables were taken twice".into()) + })?; + let tables = self.tables.load(std::sync::atomic::Ordering::Relaxed); + if added != tables { + return Err(Error::Prover(format!( + "BITWISE was built from {added} of the {tables} tables that send it lookups" + ))); + } + Ok(histogram) + } +} + +impl<'a> FinishPlan<'a> { + /// Phases 3 and 4 and the header, from routed ops ([`super::build_traces`]'s + /// arguments). `initial_image` controls PAGE: `Some(image)` plans real PAGE + /// tables and their BITWISE lookups seeded from the initial-memory image; + /// `None` plans none. + #[allow(clippy::too_many_arguments)] + pub(super) fn new( + ops: CollectedOps, + initial_image: Option<&'a I>, + memory_state: &'a MemoryState, + register_init: &'a [u32], + decode_trace: TraceTable, + decode_table: &'a DecodeTable, + mut register_state: RegisterState, + max_rows: &'a crate::tables::MaxRowsConfig, + private_input: &[u8], + is_final: bool, + l2g_memory_bookend: bool, + skip: &StreamSkip, + mut pre: Option, + ) -> Result { + let CollectedOps { + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt_ops: lt_walk, + shift_ops, + bitwise_ops, + branch_ops, + mul_ops, + dvrm_ops, + copy_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + ecrec_ops, + } = ops; + + // ===================================================================== + // PHASE 3: MEMW → LT (timestamp ordering and overflow checks) + // ===================================================================== + // LT's list is these segments in this order, kept apart: concatenating them + // into one list reallocated multi-GiB copies at a block's finish. + let memw_lt = pre + .as_mut() + .map(|pre| std::mem::take(&mut pre.memw_lt)) + .unwrap_or_default(); + let lt_from_memw = collect_lt_from_memw(&memw_ops[skip.memw_lt_done.min(memw_ops.len())..]); + build_stamps::mark("p3a lt from memw"); + let memw_aligned_lt = pre + .as_mut() + .map(|pre| std::mem::take(&mut pre.memw_aligned_lt)) + .unwrap_or_default(); + let lt_from_memw_aligned = collect_lt_from_memw_aligned( + &memw_aligned_ops[skip.memw_aligned_lt_done.min(memw_aligned_ops.len())..], + ); + build_stamps::mark("p3b lt from memw_a"); + // HINT range-checks: selector < 3 and both address low limbs < 2^32 - 31 (matching + // the executor's HintUnknownSelector / HintAddressOverflow rejections). Three LT ops + // per hint call; the HINT table sends the matching ALU LT interactions. + let lt_from_hints: Vec = hint_ops + .iter() + .flat_map(|op| { + [ + LtOperation::new(op.hint_id, hint::HINT_SELECTOR_BOUND, false), + LtOperation::new(op.in_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), + LtOperation::new(op.out_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), + ] + }) + .collect(); + let lt = if skip.concat_lt { + // The A arm: one list, each segment appended and then freed, as before. + let mut all = lt_walk.into_vec(); + memw_lt.expand_into(0, memw_lt.len(), &mut all); + drop(memw_lt); + all.extend(lt_from_memw); + memw_aligned_lt.expand_into(0, memw_aligned_lt.len(), &mut all); + drop(memw_aligned_lt); + all.extend(lt_from_memw_aligned); + all.extend(lt_from_hints); + LtLists { + walk: BlockVec::from_vec(all), + ..LtLists::default() + } + } else { + LtLists { + walk: lt_walk, + memw_lt, + from_memw: lt_from_memw, + memw_aligned_lt, + from_memw_aligned: lt_from_memw_aligned, + from_hints: lt_from_hints, + } + }; + + build_stamps::mark("p3 lt"); + + // ===================================================================== + // PHASE 4: All → Bitwise lookups + // ===================================================================== + #[cfg(feature = "instruments")] + let __sp = stark::instruments::span("p4_bitwise_collect"); + + // The committed bytes flow through the COPY chip, so the public output is read + // off its `write` rows. A zero-length `write` contributes a row that moves no + // bytes; `count != 0` is exactly the `mu - count_borrows[1]` gate the AIR uses. + // `dst` is the COMMIT-domain address, the running global byte index. + let public_output_bytes: Vec = { + let mut rows: Vec<©::CopyOperation> = copy_ops + .iter() + .filter(|op| op.functionality == copy::Functionality::Write && op.count != 0) + .collect(); + rows.sort_by_key(|op| op.dst); + rows.iter() + .flat_map(|op| op.value[..op.width as usize].iter().copied()) + .collect() + }; + + // CPU padding rows send ARE_BYTES with all-zero values. + // Add corresponding ops so the bitwise table multiplicities balance. + // Without the streamed chunks' ops (`skip.tails`), their full chunks pad too. + let padding = |len: usize| len.next_power_of_two().max(4) - len; + let num_padding_rows: usize = cpu_ops + .chunks(max_rows.cpu) + .map(|chunk| padding(chunk.len())) + .sum::() + + if skip.tails { + skip.cpu * padding(max_rows.cpu) + } else { + 0 + }; + + let streamed_last_ecall = pre.as_ref().and_then(|pre| pre.last_ecall); + // IS_WH's rows from the `IS_WH` sends of every `IS_WHH` caller (today only + // KECCAK's state pointers), deduplicated once and shared by its BITWISE + // lookups and its table, so the two read the same rows by construction. + let is_wh_rows = is_wh::dedup_is_wh_rows(keccak_ops.parts().flat_map(keccak::is_wh_keys)); + // PAGE's lookups, from the initial image (a continuation epoch's L2G + // table owns them). + let page = initial_image + .filter(|_| !l2g_memory_bookend) + .map(|image| -> PageCount<'a> { + Box::new(move |h: &mut bitwise::BitwiseHistogram| { + collect_bitwise_from_page(image, memory_state, l2g_memory_bookend, h) + }) + }); + let bitwise = if skip.p4_beside { + // Counted by BITWISE's own table, beside the rest's builds: the + // in-walk lookups wait for it. + BitwiseCount::Beside(Box::new(P4Pending { + bitwise_ops, + page, + pre, + })) + } else { + let lists = P4Lists { + lt: <, + shift_ops: &shift_ops, + store_ops: &store_ops, + memw_aligned_ops: &memw_aligned_ops, + memw_register_rows: &memw_register_rows, + branch_ops: &branch_ops, + mul_ops: &mul_ops, + dvrm_ops: &dvrm_ops, + eq_ops: &eq_ops, + bytewise_ops: &bytewise_ops, + keccak_ops: &keccak_ops, + ecdas_ops: &ecdas_ops, + copy_ops: ©_ops, + blake3_ops: &blake3_ops, + blake3_absorb_ops: &blake3_absorb_ops, + ecsm_ops: &ecsm_ops, + hint_ops: &hint_ops, + ecrec_ops: &ecrec_ops, + is_wh_rows: &is_wh_rows, + max_rows, + num_padding_rows, + padding_byte_checks: padding_byte_checks( + shift_ops.len(), + keccak_ops.len(), + max_rows, + skip, + ), + mode: P4Mode::of(skip), + }; + let page = page.as_deref().map(|p| p as _); + let histogram = count_bitwise(lists, &bitwise_ops, page, pre, false); + // The in-walk lookups have been counted into the histogram; free them now. + drop(bitwise_ops); + BitwiseCount::Counted(histogram) + }; + #[cfg(feature = "instruments")] + drop(__sp); + build_stamps::mark("p4 bitwise"); + + // A monolithic run or the final continuation epoch terminates on the program's + // halt ECALL. Intermediate continuation epochs do not halt, so fall back to the + // last cycle's timestamp and skip HALT-based PC finalization — the PC is carried + // to the next epoch via the register snapshot, and HALT is excluded from the + // proof (`include_halt = false`) so `halt_trace` is unused there. + let (halt_timestamp, halt_next_pc) = if is_final { + let (halt_timestamp, halt_next_pc) = cpu_ops + .iter() + .rev() + .find(|op| decode_table.entry(op.decode_row).fields.ecall) + .map(|op| (op.timestamp, op.next_pc)) + .or(streamed_last_ecall) + .ok_or(Error::MissingHaltEcall)?; + // Finalize the PC (x255) on the REGISTER table. The CPU padding rows carry + // pc=1 and chain the inline-PC `memory` tokens with a +4 timestamp cadence + // starting from the HALT chip's emit_pc at `halt_timestamp + 1`; the last + // padding write therefore lands at `halt_timestamp + 4*num_padding_rows + 1` + // (= `halt_timestamp + 1` when there is no padding). The REGISTER final token + // must match that last write to balance the memory argument. + let final_pc = super::clock::final_pc_timestamp(halt_timestamp, num_padding_rows); + super::clock::check_clock(final_pc, "the CPU's last padding row")?; + register_state.write_pc(1, final_pc); + (halt_timestamp, halt_next_pc) + } else { + (cpu_ops.last().map(|op| op.timestamp).unwrap_or(0), 0) + }; + + let register_final_state = register_state.to_final_state_map(); + + // The PAGE layout: none in a continuation epoch (the L2G table owns every + // touched cell's Memory init/fini there) or without an image. + let page_configs = match initial_image { + Some(image) if !l2g_memory_bookend => { + page_configs_of(image, memory_state, private_input) + } + _ => Vec::new(), + }; + let num_blake3_ops = blake3_ops.len() + blake3_absorb_ops.len(); + + // Every table the plan builds, counted without building one. + let tails = skip.tails; + let keccak_rnd = match (keccak_ops.len(), skip.keccak_rnd_rows) { + (n, 0) => usize::from(n > 0), + (0, _) => 0, + (n, rows) => { + let total = (n * 24).next_power_of_two().max(4); + let chunks = skip.cut_padding.tables(n * 24, rows); + if skip.keccak_rnd > 0 && (total <= rows || skip.keccak_rnd > chunks) { + return Err(Error::Prover(format!( + "{} KECCAK_RND chunks were streamed but the run has {chunks}", + skip.keccak_rnd + ))); + } + chunks + } + }; + // [`cut_tables`]' count, or the one whole table. + let accelerator = |n: usize, rows: usize| { + if rows > 0 { + skip.cut_padding.tables(n, rows) + } else { + usize::from(n > 0) + } + }; + // The CPU tables' heights, as `chunk_and_generate_skipping` cuts them: with + // only the tails kept, the streamed chunks (full, `max_rows.cpu` rows) and + // then the list's; else the list's chunks past the streamed ones. + let cpu_heights: Vec = { + let height = |len: usize| len.next_power_of_two().max(4); + let max = max_rows.cpu.max(1); + let chunks: Vec = cpu_ops.chunks(max).map(|c| height(c.len())).collect(); + if skip.cpu == 0 && chunks.is_empty() { + // An empty list is one empty table (CPU is not optional). + vec![height(0)] + } else if skip.cpu > 0 && tails { + std::iter::repeat_n(height(max), skip.cpu) + .chain(chunks) + .collect() + } else { + chunks + } + }; + let table_counts = crate::TableCounts { + cpu: chunked(cpu_ops.len(), max_rows.cpu, skip.cpu, tails, false)?, + lt: chunked(lt.segments().len(), max_rows.lt, skip.lt, tails, true)?, + memw: chunked(memw_ops.len(), max_rows.memw, skip.memw, tails, true)?, + memw_aligned: chunked( + memw_aligned_ops.len(), + max_rows.memw_aligned, + skip.memw_aligned, + tails, + true, + )?, + load: chunked(load_ops.len(), max_rows.load, skip.load, tails, true)?, + mul: chunked(mul_ops.len(), max_rows.mul, 0, false, true)?, + dvrm: chunked(dvrm_ops.len(), max_rows.dvrm, 0, false, true)?, + shift: chunked(shift_ops.len(), max_rows.shift, skip.shift, tails, true)?, + branch: chunked(branch_ops.len(), max_rows.branch, 0, false, true)?, + memw_register: chunked( + memw_register_rows.len(), + max_rows.memw_register, + skip.memw_register, + tails, + false, + )?, + eq: chunked(eq_ops.len(), max_rows.eq, 0, false, true)?, + bytewise: chunked(bytewise_ops.len(), max_rows.bytewise, 0, false, true)?, + store: chunked(store_ops.len(), max_rows.store, skip.store, tails, true)?, + cpu32: chunked(cpu32_ops.len(), max_rows.cpu32, 0, false, true)?, + keccak: accelerator(keccak_ops.len(), skip.keccak_rows), + keccak_rnd, + ecsm: accelerator(ecsm_ops.len(), skip.ecsm_rows), + ecdas: accelerator(ecdas_ops.len(), skip.ecdas_rows), + hint: usize::from(!hint_ops.is_empty()), + copy: accelerator(copy_ops.len(), skip.copy_rows), + blake3: usize::from(num_blake3_ops > 0), + is_wh: usize::from(!is_wh_rows.is_empty()), + ecrec: usize::from(!ecrec_ops.is_empty()), + }; + if cpu_heights.len() != table_counts.cpu { + return Err(Error::InvalidTableCounts(format!( + "{} CPU heights for {} CPU tables", + cpu_heights.len(), + table_counts.cpu + ))); + } + // The families cut at a fixed number of rows: the tables counted, and + // how many of them hold a real row. + let cut: Vec = [ + ( + "KECCAK", + skip.keccak_rows, + keccak_ops.len(), + table_counts.keccak, + ), + ( + "KECCAK_RND", + skip.keccak_rnd_rows, + keccak_ops.len() * 24, + table_counts.keccak_rnd, + ), + ("ECSM", skip.ecsm_rows, ecsm_ops.len(), table_counts.ecsm), + ( + "ECDAS", + skip.ecdas_rows, + ecdas_ops.len(), + table_counts.ecdas, + ), + ("COPY", skip.copy_rows, copy_ops.len(), table_counts.copy), + ] + .into_iter() + .filter(|&(_, rows, _, _)| rows > 0) + .map(|(name, rows, n, tables)| { + format!( + "{name} {tables} ({})", + CutPadding::LastTable.tables(n, rows) + ) + }) + .collect(); + let cut_line = (!cut.is_empty()).then(|| { + format!( + "BLOCK CUT PADDING: {} · tables (of them with a real row): {}", + match skip.cut_padding { + CutPadding::WholeRun => "the whole run padded, then cut", + CutPadding::LastTable => "cut, the last table padded (the default)", + }, + cut.join(" · "), + ) + }); + + Ok(Self { + memory_state, + register_init, + decode_trace, + decode_table, + max_rows, + l2g_memory_bookend, + skip: *skip, + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt, + shift_ops, + branch_ops, + mul_ops, + dvrm_ops, + copy_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + ecrec_ops, + is_wh_rows, + bitwise, + num_padding_rows, + halt_timestamp, + halt_next_pc, + register_final_state, + header: RestHeader { + table_counts, + page_configs, + public_output_bytes, + num_blake3_ops, + cpu_heights, + bitwise_last: false, + bitwise_from_tables: false, + }, + cut_line, + }) + } + + /// What a statement reads off the build, before any table is built. + pub(crate) fn header(&self) -> &RestHeader { + &self.header + } + + /// The rest handed on with BITWISE after every other table + /// ([`RestHeader::bitwise_last`]): for a finish that counts BITWISE's + /// multiplicities beside the rest's builds and slides. + pub(crate) fn hand_bitwise_last(&mut self) { + self.header.bitwise_last = true; + } + + /// BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables as each is + /// built ([`RestHeader::bitwise_from_tables`]): for a finish that hands + /// BITWISE on last and keeps only the streamed chunks' tails. + pub(crate) fn count_bitwise_from_tables(&mut self) { + self.header.bitwise_from_tables = true; + } + + /// A `BLOCK CUT PADDING` line for the readout, when the build cuts a + /// family at a fixed number of rows: the rule ([`StreamSkip::cut_padding`]) + /// and, per cut family, the tables it makes and how many hold a real row. + pub(crate) fn cut_line(&self) -> Option<&str> { + self.cut_line.as_deref() + } + + /// The header's counts replaced (a test's wrong plan, which + /// [`Self::emit_all`] must refuse). + #[cfg(test)] + pub(super) fn with_counts(mut self, table_counts: crate::TableCounts) -> Self { + self.header.table_counts = table_counts; + self + } + + /// Phase 5: every table, as [`super::build_traces`] always built them, in + /// one parallel scope; refused unless they are the ones the header counted. + pub(super) fn emit_all(self) -> Result { + let Self { + memory_state, + register_init, + decode_trace, + decode_table, + max_rows, + l2g_memory_bookend, + skip, + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt, + shift_ops, + branch_ops, + mul_ops, + dvrm_ops, + copy_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + ecrec_ops, + is_wh_rows, + bitwise, + num_padding_rows, + halt_timestamp, + halt_next_pc, + register_final_state, + header, + cut_line: _, + } = self; + let bitwise_histogram = match bitwise { + BitwiseCount::Counted(histogram) => histogram, + BitwiseCount::Beside(pending) => pending.count( + P4Lists { + lt: <, + shift_ops: &shift_ops, + store_ops: &store_ops, + memw_aligned_ops: &memw_aligned_ops, + memw_register_rows: &memw_register_rows, + branch_ops: &branch_ops, + mul_ops: &mul_ops, + dvrm_ops: &dvrm_ops, + eq_ops: &eq_ops, + bytewise_ops: &bytewise_ops, + keccak_ops: &keccak_ops, + ecdas_ops: &ecdas_ops, + copy_ops: ©_ops, + blake3_ops: &blake3_ops, + blake3_absorb_ops: &blake3_absorb_ops, + ecsm_ops: &ecsm_ops, + hint_ops: &hint_ops, + ecrec_ops: &ecrec_ops, + is_wh_rows: &is_wh_rows, + max_rows, + num_padding_rows, + padding_byte_checks: padding_byte_checks( + shift_ops.len(), + keccak_ops.len(), + max_rows, + &skip, + ), + mode: P4Mode::of(&skip), + }, + false, + ), + }; + let lt_ops = lt.segments(); + + // ===================================================================== + // PHASE 5: Generate final traces (parallelized) + // ===================================================================== + #[cfg(feature = "instruments")] + let __sp = stark::instruments::span("p5_generate_tables"); + + // Each build below reads disjoint op lists and writes its own table, so + // they all run in one rayon scope. Disk-spill stays sequential: its + // generate→spill order keeps trace memory bounded. + let cpu_ops_ref = &cpu_ops; + let pack = skip.pack; + // A packing build writes each table packed as it generates it where the + // generator can (G-pack, `StreamSkip::gpack`); `packed_by` then finds it + // packed. + let gpack = pack && skip.gpack; + let form = if gpack { + TraceForm::Narrow + } else { + TraceForm::Wide + }; + let gen_cpus = || { + chunk_and_generate_skipping( + cpu_ops_ref, + max_rows.cpu, + skip.cpu, + skip.tails, + false, + packed_by(pack, |ops| { + cpu::generate_cpu_trace_as(ops, decode_table, form) + }), + ) + }; + let gen_memws = || { + chunk_and_generate_skipping( + &memw_ops, + max_rows.memw, + skip.memw, + skip.tails, + true, + packed_by(pack, |ops| memw::generate_memw_trace_as(ops, form)), + ) + }; + let gen_memw_aligneds = || { + chunk_and_generate_skipping( + &memw_aligned_ops, + max_rows.memw_aligned, + skip.memw_aligned, + skip.tails, + true, + packed_by(pack, |ops| { + memw_aligned::generate_memw_aligned_trace_as(ops, form) + }), + ) + }; + let gen_memw_registers = || { + // Direct-to-column fill from compact RegRows — the register fast path never + // materializes a `Vec`. + chunk_and_generate_skipping( + &memw_register_rows, + max_rows.memw_register, + skip.memw_register, + skip.tails, + false, + packed_by(pack, |ops| { + memw_register::generate_memw_register_trace_from_rows_as(ops, form) + }), + ) + }; + let gen_loads = || { + chunk_and_generate_skipping( + &load_ops, + max_rows.load, + skip.load, + skip.tails, + true, + packed_by(pack, |ops| load::generate_load_trace_as(ops, form)), + ) + }; + let gen_lts = || { + chunk_and_generate_segmented( + <_ops, + max_rows.lt, + skip.lt, + skip.tails, + true, + packed_by(pack, |ops| lt::generate_lt_trace_as(ops, form)), + ) + }; + let gen_shifts = || { + chunk_and_generate_segmented( + &shift_ops.segments(), + max_rows.shift, + skip.shift, + skip.tails, + true, + packed_by(pack, |ops| shift::generate_shift_trace_as(ops, form)), + ) + }; + let gen_muls = || { + chunk_and_generate_segmented( + &mul_ops.segments(), + max_rows.mul, + 0, + false, + true, + packed_by(pack, |ops| mul::generate_mul_trace_as(ops, form)), + ) + }; + let gen_dvrms = || { + chunk_and_generate_segmented( + &dvrm_ops.segments(), + max_rows.dvrm, + 0, + false, + true, + packed_by(pack, |ops| dvrm::generate_dvrm_trace_as(ops, form)), + ) + }; + let gen_branches = || { + chunk_and_generate_segmented( + &branch_ops.segments(), + max_rows.branch, + 0, + false, + true, + packed_by(pack, |ops| branch::generate_branch_trace_as(ops, form)), + ) + }; + // Auxiliary ALU / memory / CPU32 dispatch chips, each filtered out of the CPU + // ops above. + let gen_eqs = || { + chunk_and_generate_segmented::( + &eq_ops.segments(), + max_rows.eq, + 0, + false, + true, + packed_by(pack, |ops| eq::generate_eq_trace_as(ops, form)), + ) + }; + let gen_bytewises = || { + chunk_and_generate_segmented::( + &bytewise_ops.segments(), + max_rows.bytewise, + 0, + false, + true, + packed_by(pack, |ops| bytewise::generate_bytewise_trace_as(ops, form)), + ) + }; + let gen_stores = || { + chunk_and_generate_segmented( + &store_ops.segments(), + max_rows.store, + skip.store, + skip.tails, + true, + packed_by(pack, |ops| store::generate_store_trace_as(ops, form)), + ) + }; + let gen_cpu32s = || { + chunk_and_generate_segmented::( + &cpu32_ops.segments(), + max_rows.cpu32, + 0, + false, + true, + packed_by(pack, |ops| cpu32::generate_cpu32_trace_as(ops, form)), + ) + }; + let gen_bitwise = || { + let mut bitwise = bitwise::generate_bitwise_trace(); + // Fill the MU columns (11..=20) from the accumulated histogram. + bitwise_histogram.fill_multiplicities(&mut bitwise); + packed_if(pack, bitwise) + }; + // Each CPU operation looks up the DECODE table once; padding rows look up + // pc=1 (the CPU padding entry). When CPU is split, each chunk pads + // independently. + let gen_decode = move || { + let mut decode = decode_trace; + let rows = cpu_ops_ref.iter().map(|op| op.decode_row as usize); + let padding = std::iter::repeat_n(decode_table.padding_row(), num_padding_rows); + decode::count_rows(&mut decode, rows.chain(padding)); + packed_if(pack, decode) + }; + let gen_copies = || { + if skip.copy_rows > 0 { + return cut_tables( + ©_ops, + skip.copy_rows, + skip.cut_padding, + pack, + form, + copy::rows_written_packed, + copy::generate_copy_rows_as, + ); + } + generate_optional(©_ops.whole(), copy::generate_copy_trace) + }; + let gen_keccaks = || { + if skip.keccak_rows > 0 { + return cut_tables( + &keccak_ops, + skip.keccak_rows, + skip.cut_padding, + pack, + form, + keccak::rows_written_packed, + keccak::generate_keccak_rows_as, + ); + } + generate_optional(&keccak_ops.whole(), keccak::generate_keccak_trace) + }; + let gen_keccak_rnds = || { + let keccak_rnd_ops: Vec = keccak_ops + .iter() + .map(|op| KeccakRoundOperation { + timestamp: op.timestamp, + input: op.input, + output: op.output, + }) + .collect(); + if skip.keccak_rnd_rows == 0 { + // Whole, it may be cut after the build: it stays wide. + return generate_optional(&keccak_rnd_ops, keccak_rnd::generate_keccak_rnd_trace); + } + keccak_rnd_chunks( + &keccak_rnd_ops, + skip.keccak_rnd_rows, + skip.keccak_rnd, + skip.cut_padding, + pack, + form, + ) + }; + let gen_blake3 = || { + packed_if( + pack, + blake3::generate_blake3_trace(&blake3_ops.whole(), &blake3_absorb_ops.whole()), + ) + }; + let gen_keccak_rc = || { + let mut keccak_rc_trace = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut keccak_rc_trace, keccak_ops.len()); + packed_if(pack, keccak_rc_trace) + }; + // The plan's page configs: none in a continuation epoch + // (l2g_memory_bookend) or without an image ([`FinishPlan::new`]). + let gen_pages = || -> Vec<_> { + header + .page_configs + .iter() + .map(|config| packed_if(pack, page_trace(config, memory_state, l2g_memory_bookend))) + .collect() + }; + let gen_register = || { + packed_if( + pack, + register::generate_register_trace(®ister_final_state, register_init), + ) + }; + let gen_halt = || { + packed_if( + pack, + halt::generate_halt_trace(halt_timestamp, halt_next_pc), + ) + }; + // ECSM accelerator traces. A program that does not use ECSM carries no ECSM + // and no ECDAS table at all — not a padded one. + let gen_ecsms = || { + if skip.ecsm_rows > 0 { + return cut_tables( + &ecsm_ops, + skip.ecsm_rows, + skip.cut_padding, + pack, + form, + ecsm::rows_written_packed, + ecsm::generate_ecsm_rows_as, + ); + } + generate_optional(&ecsm_ops.whole(), ecsm::generate_ecsm_trace) + }; + let gen_ecdases = || { + if skip.ecdas_rows > 0 { + return cut_tables( + &ecdas_ops, + skip.ecdas_rows, + skip.cut_padding, + pack, + form, + ecdas::rows_written_packed, + |rows, num_rows, form| { + ecdas::generate_ecdas_rows_of(rows, num_rows, form, ecdas::widen) + }, + ); + } + generate_optional(&ecdas_ops.whole(), |rows| { + let num_rows = rows.len().next_power_of_two().max(4); + ecdas::generate_ecdas_rows_of(rows, num_rows, TraceForm::Wide, ecdas::widen) + }) + }; + // HINT table. Absent entirely for programs that make no hint ecalls. + let gen_hints = || { + generate_optional( + &hint_ops.whole(), + packed_by(pack, |ops| hint::generate_hint_trace_as(ops, form)), + ) + }; + // ECREC table. Absent entirely for programs that make no ECREC call. + let gen_ecrecs = || { + generate_optional( + &ecrec_ops.whole(), + packed_by(pack, |ops| ecrec::generate_ecrec_trace_as(ops, form)), + ) + }; + + let (mut cpus_slot, mut memws_slot, mut memw_aligneds_slot, mut memw_registers_slot) = + (None, None, None, None); + let (mut loads_slot, mut lts_slot, mut shifts_slot, mut muls_slot) = + (None, None, None, None); + let (mut dvrms_slot, mut branches_slot, mut bitwise_slot, mut decode_slot) = + (None, None, None, None); + let (mut copies_slot, mut keccaks_slot, mut keccak_rnds_slot, mut keccak_rc_slot) = + (None, None, None, None); + let mut blake3_slot = None; + let (mut pages_slot, mut register_slot, mut halt_slot) = (None, None, None); + let (mut eqs_slot, mut bytewises_slot, mut stores_slot, mut cpu32s_slot) = + (None, None, None, None); + let (mut ecsms_slot, mut ecdases_slot) = (None, None); + let mut hints_slot = None; + let mut ecrecs_slot = None; + + let sequential = cfg!(not(feature = "parallel")); + + if !sequential { + #[cfg(feature = "parallel")] + rayon::scope(|s| { + macro_rules! spawn_into { + ($slot:ident, $gen:ident) => {{ + let slot = &mut $slot; + s.spawn(move |_| { + *slot = Some($gen()); + build_stamps::mark(concat!("p5 ", stringify!($gen))); + }); + }}; + } + // Heaviest builds first so the scheduler overlaps them with the rest. + spawn_into!(memw_registers_slot, gen_memw_registers); + spawn_into!(cpus_slot, gen_cpus); + spawn_into!(memws_slot, gen_memws); + spawn_into!(lts_slot, gen_lts); + spawn_into!(decode_slot, gen_decode); + spawn_into!(branches_slot, gen_branches); + spawn_into!(bitwise_slot, gen_bitwise); + spawn_into!(muls_slot, gen_muls); + spawn_into!(memw_aligneds_slot, gen_memw_aligneds); + spawn_into!(loads_slot, gen_loads); + spawn_into!(shifts_slot, gen_shifts); + spawn_into!(dvrms_slot, gen_dvrms); + spawn_into!(pages_slot, gen_pages); + spawn_into!(keccaks_slot, gen_keccaks); + spawn_into!(keccak_rnds_slot, gen_keccak_rnds); + spawn_into!(keccak_rc_slot, gen_keccak_rc); + spawn_into!(blake3_slot, gen_blake3); + spawn_into!(copies_slot, gen_copies); + spawn_into!(register_slot, gen_register); + spawn_into!(halt_slot, gen_halt); + spawn_into!(eqs_slot, gen_eqs); + spawn_into!(bytewises_slot, gen_bytewises); + spawn_into!(stores_slot, gen_stores); + spawn_into!(cpu32s_slot, gen_cpu32s); + spawn_into!(ecsms_slot, gen_ecsms); + spawn_into!(ecdases_slot, gen_ecdases); + spawn_into!(hints_slot, gen_hints); + spawn_into!(ecrecs_slot, gen_ecrecs); + }); + } else { + cpus_slot = Some(gen_cpus()); + memws_slot = Some(gen_memws()); + memw_aligneds_slot = Some(gen_memw_aligneds()); + memw_registers_slot = Some(gen_memw_registers()); + loads_slot = Some(gen_loads()); + lts_slot = Some(gen_lts()); + shifts_slot = Some(gen_shifts()); + muls_slot = Some(gen_muls()); + dvrms_slot = Some(gen_dvrms()); + branches_slot = Some(gen_branches()); + bitwise_slot = Some(gen_bitwise()); + decode_slot = Some(gen_decode()); + copies_slot = Some(gen_copies()); + keccaks_slot = Some(gen_keccaks()); + keccak_rnds_slot = Some(gen_keccak_rnds()); + keccak_rc_slot = Some(gen_keccak_rc()); + blake3_slot = Some(gen_blake3()); + pages_slot = Some(gen_pages()); + register_slot = Some(gen_register()); + halt_slot = Some(gen_halt()); + eqs_slot = Some(gen_eqs()); + bytewises_slot = Some(gen_bytewises()); + stores_slot = Some(gen_stores()); + cpu32s_slot = Some(gen_cpu32s()); + ecsms_slot = Some(gen_ecsms()); + ecdases_slot = Some(gen_ecdases()); + hints_slot = Some(gen_hints()); + ecrecs_slot = Some(gen_ecrecs()); + } + + build_stamps::mark("p5 end"); + const PHASE5_RAN: &str = "phase 5 generation ran in one of the branches above"; + let cpus = cpus_slot.expect(PHASE5_RAN)?; + let memws = memws_slot.expect(PHASE5_RAN)?; + let memw_aligneds = memw_aligneds_slot.expect(PHASE5_RAN)?; + let memw_registers = memw_registers_slot.expect(PHASE5_RAN)?; + let loads = loads_slot.expect(PHASE5_RAN)?; + let lts = lts_slot.expect(PHASE5_RAN)?; + let shifts = shifts_slot.expect(PHASE5_RAN)?; + let muls = muls_slot.expect(PHASE5_RAN)?; + let dvrms = dvrms_slot.expect(PHASE5_RAN)?; + let branches = branches_slot.expect(PHASE5_RAN)?; + let eqs = eqs_slot.expect(PHASE5_RAN)?; + let bytewises = bytewises_slot.expect(PHASE5_RAN)?; + let stores = stores_slot.expect(PHASE5_RAN)?; + let cpu32s = cpu32s_slot.expect(PHASE5_RAN)?; + #[allow(unused_mut)] + let mut bitwise = bitwise_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let mut decode = decode_slot.expect(PHASE5_RAN); + let copies = copies_slot.expect(PHASE5_RAN)?; + let keccaks = keccaks_slot.expect(PHASE5_RAN)?; + let keccak_rnds = keccak_rnds_slot.expect(PHASE5_RAN)?; + let keccak_rc_trace = keccak_rc_slot.expect(PHASE5_RAN); + let blake3_trace = blake3_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let mut pages = pages_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let mut register_trace = register_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let mut halt_trace = halt_slot.expect(PHASE5_RAN); + let ecsms = ecsms_slot.expect(PHASE5_RAN)?; + let ecdases = ecdases_slot.expect(PHASE5_RAN)?; + let hints = hints_slot.expect(PHASE5_RAN)?; + let ecrecs = ecrecs_slot.expect(PHASE5_RAN)?; + // IS_WH: one table of its rows, none when nothing sent `IS_WH`. + let is_whs = generate_optional(&is_wh_rows, packed_by(pack, is_wh::generate_is_wh_trace))?; + + // Continuation callers derive the real cross-epoch boundary from this set and + // install its L2G trace after provenance is applied. Avoid building a + // throwaway genesis-only L2G trace here. + let touched_memory_cells = if l2g_memory_bookend { + touched_cells_from_memory_state(memory_state) + } else { + Vec::new() + }; + let local_to_global = local_to_global::generate_local_to_global_trace(&[]); + #[cfg(feature = "instruments")] + drop(__sp); + let RestHeader { + table_counts, + page_configs, + public_output_bytes, + num_blake3_ops, + cpu_heights: _, + bitwise_last: _, + bitwise_from_tables: _, + } = header; + let traces = Traces { + cpus, + bitwise, + lts, + shifts, + memws, + memw_aligneds, + loads, + decode, + muls, + dvrms, + pages, + page_configs, + register: register_trace, + public_output_bytes, + branches, + halt: halt_trace, + copies, + keccaks, + keccak_rnds, + blake3: blake3_trace, + num_blake3_ops, + keccak_rc: keccak_rc_trace, + ecsms, + ecdases, + hints, + ecrecs, + is_whs, + memw_registers, + local_to_global, + touched_memory_cells, + eqs, + bytewises, + stores, + cpu32s, + }; + // The tables are the ones the header counted, or the build is refused: the + // header is what a statement can be read from before the tables exist. + let built = traces.table_counts(); + if counts_of(&built) != counts_of(&table_counts) { + return Err(Error::Prover(format!( + "the finish built {built:?}, its plan counted {table_counts:?}" + ))); + } + Ok(traces) + } +} + +type Table = TraceTable; + +/// A table of the rest, not yet built: building it is calling this. +type Job<'a> = Box Result + Send + 'a>; + +/// A one-row-an-op family's cut generator: (ops, rows, form) → table. +type CutGenerator<'a, T> = + std::sync::Arc Table + Send + Sync + 'a>; + +/// A wave's slot: a unit's AIR position and, for a table, its number among +/// the rest's tables, its family and its job. +type WaveSlot<'a> = (usize, Option<(usize, usize, Job<'a>)>); + +/// One table of the rest, in AIR order ([`FinishPlan::units`]). +pub(super) enum Unit<'a> { + /// A streamed chunk's slot of a family ([`FAMILIES`]): its table went out + /// with the windows. + Streamed { family: usize }, + /// A table to build: its family ([`FAMILIES`]), an estimate of its rows and + /// the job that builds it. + Build { + family: usize, + rows: usize, + job: Job<'a>, + }, +} + +/// What the streamed finish hands on, in AIR order ([`FinishPlan::emit_streamed`]): +/// the table at AIR `position`, or `None` for a streamed chunk's slot. +pub(crate) struct Emitted { + pub(crate) position: usize, + pub(crate) table: Option
, +} + +/// The rest's families in AIR order (`VmAirs::air_refs`), each with its +/// columns at eight bytes a cell: what a family's first table is estimated +/// at, before one of them has been built. +const FAMILIES: [(&str, usize); 29] = [ + ("BITWISE", bitwise::cols::NUM_COLUMNS), + ("DECODE", decode::cols::NUM_COLUMNS), + ("KECCAK_RC", keccak_rc::cols::NUM_COLUMNS), + ("REGISTER", register::cols::NUM_COLUMNS), + ("HALT", halt::cols::NUM_COLUMNS), + ("COPY", copy::cols::NUM_COLUMNS), + ("KECCAK", keccak::cols::NUM_COLUMNS), + ("KECCAK_RND", keccak_rnd::cols::NUM_COLUMNS), + ("ECSM", ecsm::cols::NUM_COLUMNS), + ("ECDAS", ecdas::cols::NUM_COLUMNS), + ("HINT", hint::cols::NUM_COLUMNS), + ("ECREC", ecrec::cols::NUM_COLUMNS), + ("BLAKE3", blake3::cols::NUM_COLUMNS), + ("IS_WH", is_wh::cols::NUM_COLUMNS), + ("CPU", cpu::cols::NUM_COLUMNS), + ("LT", lt::cols::NUM_COLUMNS), + ("SHIFT", shift::cols::NUM_COLUMNS), + ("MEMW", memw::cols::NUM_COLUMNS), + ("MEMW_A", memw_aligned::cols::NUM_COLUMNS), + ("LOAD", load::cols::NUM_COLUMNS), + ("MUL", mul::cols::NUM_COLUMNS), + ("DVRM", dvrm::cols::NUM_COLUMNS), + ("BRANCH", branch::cols::NUM_COLUMNS), + ("PAGE", page::cols::NUM_COLUMNS), + ("MEMW_R", memw_register::cols::NUM_COLUMNS), + ("EQ", eq::cols::NUM_COLUMNS), + ("BYTEWISE", bytewise::cols::NUM_COLUMNS), + ("STORE", store::cols::NUM_COLUMNS), + ("CPU32", cpu32::cols::NUM_COLUMNS), +]; + +/// [`FAMILIES`]' index of a family. +fn family(name: &str) -> usize { + FAMILIES + .iter() + .position(|&(f, _)| f == name) + .unwrap_or(usize::MAX) +} + +/// The bytes a table's main trace takes: packed, as records, or eight a cell. +fn table_bytes(table: &Table) -> usize { + if let Some(records) = table.records_main() { + return records.bytes(); + } + table.narrow_main().map_or_else( + || table.main_table.width * table.main_table.height * std::mem::size_of::(), + |packed| packed.data().len(), + ) +} + +/// The rows a table of `n` ops is padded to, as most generators pad it: an +/// estimate only. +fn rows_of(n: usize) -> usize { + n.next_power_of_two().max(4) +} + +/// A chunked list's units ([`chunk_and_generate_skipping`]'s and +/// [`chunk_and_generate_segmented`]'s tables, in their order): `skip` slots +/// streamed ahead, then a job a chunk of `max` ops (`chunk(start, end)`), an +/// empty list making one empty table unless `optional`. +#[allow(clippy::too_many_arguments)] +fn chunked_units<'a>( + family: usize, + len: usize, + max: usize, + skip: usize, + tails: bool, + optional: bool, + chunk: impl Fn(usize, usize) -> Job<'a>, +) -> Result>, Error> { + let max = max.max(1); + let ranges: Vec<(usize, usize)> = (0..len.div_ceil(max)) + .map(|k| (k * max, ((k + 1) * max).min(len))) + .collect(); + let (placeholders, ranges) = if skip == 0 { + let ranges = if len == 0 && !optional { + vec![(0, 0)] + } else { + ranges + }; + (0, ranges) + } else if tails { + (skip, ranges) + } else { + if skip > ranges.len() { + return Err(Error::Prover(format!( + "{skip} chunks were streamed but the run has {} of this table", + ranges.len() + ))); + } + (skip, ranges[skip..].to_vec()) + }; + let mut units: Vec> = (0..placeholders) + .map(|_| Unit::Streamed { family }) + .collect(); + units.extend(ranges.into_iter().map(|(start, end)| Unit::Build { + family, + rows: rows_of(end - start), + job: chunk(start, end), + })); + Ok(units) +} + +/// A one-row-an-op family's units ([`cut_tables`]'s tables): none without +/// ops, one table when the whole is no taller than `rows`, else its cuts of +/// `rows` rows, as many as `padding` counts; or, without cuts (`rows` 0), the +/// whole table wide (`whole`), as the finish builds it to be split later. +#[allow(clippy::too_many_arguments)] +fn cut_units<'a, T: Clone + Send + Sync + 'a>( + family: usize, + ops: &std::sync::Arc>, + rows: usize, + padding: CutPadding, + pack: bool, + form: TraceForm, + generate: CutGenerator<'a, T>, + whole: impl FnOnce(std::sync::Arc>) -> Job<'a>, +) -> Result>, Error> { + let n = ops.len(); + if n == 0 { + return Ok(Vec::new()); + } + if rows == 0 { + return Ok(vec![Unit::Build { + family, + rows: rows_of(n), + job: whole(std::sync::Arc::clone(ops)), + }]); + } + if !rows.is_power_of_two() { + return Err(Error::Prover(format!( + "tables cut at {rows} rows: a power of two is needed" + ))); + } + let total = rows_of(n); + if total <= rows { + let ops = std::sync::Arc::clone(ops); + return Ok(vec![Unit::Build { + family, + rows: total, + job: Box::new(move || Ok(packed_if(pack, generate(&ops.whole(), total, form)))), + }]); + } + Ok((0..padding.tables(n, rows)) + .map(|k| { + let ops = std::sync::Arc::clone(ops); + let generate = std::sync::Arc::clone(&generate); + let (first, end) = ((k * rows).min(n), ((k + 1) * rows).min(n)); + Unit::Build { + family, + rows, + job: Box::new(move || { + Ok(packed_if( + pack, + generate(&ops.range(first, end), rows, form), + )) + }) as Job<'a>, + } + }) + .collect()) +} + +/// `job` if `n > 0`: one table, or none ([`generate_optional`]'s tables). +fn optional_unit<'a>(family: usize, n: usize, job: impl FnOnce() -> Job<'a>) -> Vec> { + if n == 0 { + return Vec::new(); + } + vec![Unit::Build { + family, + rows: rows_of(n), + job: job(), + }] +} + +impl<'a> FinishPlan<'a> { + /// Phase 5 as units: every table the header counted (and BITWISE, DECODE, + /// KECCAK_RC, REGISTER and HALT), in AIR order, each a job that builds it + /// as [`Self::emit_all`] does, or a streamed chunk's slot. A family's list + /// is held by its jobs alone and goes with the last of them. Refused unless + /// the units are the tables the header counted. + #[cfg(test)] + pub(super) fn units(self) -> Result>, Error> { + self.units_keeping(false).map(|(units, _, _)| units) + } + + /// [`Self::units`], and with `keep` the lists KECCAK_RND's and LT's units + /// read, kept for phase B to build those tables again ([`RestRegen`]); + /// with [`StreamSkip::bitwise_from_tables`], the count of phase 4's other + /// sources to run beside the units ([`Side`]). + #[allow(clippy::type_complexity)] + pub(super) fn units_keeping( + self, + keep: bool, + ) -> Result<(Vec>, Option, Option>), Error> { + use std::sync::Arc; + let Self { + memory_state, + register_init, + decode_trace, + decode_table, + max_rows, + l2g_memory_bookend, + skip, + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt, + shift_ops, + branch_ops, + mul_ops, + dvrm_ops, + copy_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + ecrec_ops, + is_wh_rows, + bitwise, + num_padding_rows, + halt_timestamp, + halt_next_pc, + register_final_state, + header, + cut_line: _, + } = self; + let pack = skip.pack; + let gpack = pack && skip.gpack; + let form = if gpack { + TraceForm::Narrow + } else { + TraceForm::Wide + }; + let tails = skip.tails; + let byte_checks = padding_byte_checks(shift_ops.len(), keccak_ops.len(), max_rows, &skip); + let p4_mode = P4Mode::of(&skip); + // BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables as each + // is built, the other sources counted beside the rest's builds. + let from_tables = (header.bitwise_from_tables + && matches!(bitwise, BitwiseCount::Beside(_))) + .then(|| Arc::new(FromTables::new())); + let mut units: Vec> = Vec::new(); + let mut regen = keep.then_some(RestRegen { + keccak_ops: None, + lt: None, + skip, + lt_rows: max_rows.lt, + pack, + form, + }); + let cpu_ops = Arc::new(cpu_ops); + // The lists phase 4 reads, shared by their units and, when it counts + // beside them ([`StreamSkip::p4_beside`]), BITWISE's. + let shared = P4Shared { + lt: Arc::new(lt), + shift_ops: Arc::new(shift_ops), + store_ops: Arc::new(store_ops), + memw_aligned_ops: Arc::new(memw_aligned_ops), + memw_register_rows: Arc::new(memw_register_rows), + branch_ops: Arc::new(branch_ops), + mul_ops: Arc::new(mul_ops), + dvrm_ops: Arc::new(dvrm_ops), + eq_ops: Arc::new(eq_ops), + bytewise_ops: Arc::new(bytewise_ops), + keccak_ops: Arc::new(keccak_ops), + ecdas_ops: Arc::new(ecdas_ops), + copy_ops: Arc::new(copy_ops), + blake3_ops: Arc::new(blake3_ops), + blake3_absorb_ops: Arc::new(blake3_absorb_ops), + ecsm_ops: Arc::new(ecsm_ops), + hint_ops: Arc::new(hint_ops), + ecrec_ops: Arc::new(ecrec_ops), + is_wh_rows: Arc::new(is_wh_rows), + }; + + // BITWISE, DECODE, KECCAK_RC, REGISTER, HALT. Counted beside the rest's + // builds, BITWISE's job counts its multiplicities first: the slide + // starts the tables after it meanwhile. + // With the tables counting their own lookups, the other sources are + // counted beside the rest's builds and sent to BITWISE's job, built + // after every table. + let (job, side) = match (bitwise, &from_tables) { + (BitwiseCount::Counted(histogram), _) => (BitwiseJob::Counted(histogram), None), + (BitwiseCount::Beside(pending), Some(tables)) => { + let (sent, side_count) = std::sync::mpsc::channel(); + let shared = shared.clone(); + let mode = P4Mode { + from_tables: true, + ..p4_mode + }; + let side: Side<'a> = Box::new(move || { + let histogram = pending.count( + shared.lists(max_rows, num_padding_rows, byte_checks, mode), + true, + ); + build_stamps::mark("p4 bitwise beside"); + let _ = sent.send(histogram); + }); + ( + BitwiseJob::FromTables(side_count, Arc::clone(tables)), + Some(side), + ) + } + (BitwiseCount::Beside(pending), None) => { + (BitwiseJob::Beside(pending, shared.clone()), None) + } + }; + units.push(Unit::Build { + family: family("BITWISE"), + rows: bitwise::NUM_ROWS, + job: Box::new(move || { + let from = matches!(job, BitwiseJob::FromTables(..)); + let histogram = match job { + BitwiseJob::Counted(histogram) => histogram, + BitwiseJob::Beside(pending, shared) => { + let histogram = pending.count( + shared.lists(max_rows, num_padding_rows, byte_checks, p4_mode), + true, + ); + build_stamps::mark("p4 bitwise beside"); + histogram + } + BitwiseJob::FromTables(side_count, tables) => { + let mut histogram = side_count.recv().map_err(|_| { + Error::Prover("phase 4's other sources were never counted".into()) + })?; + histogram.merge(&tables.take()?); + histogram + } + }; + note_bitwise(&histogram, from); + let mut table = bitwise::generate_bitwise_trace(); + histogram.fill_multiplicities(&mut table); + Ok(packed_if(pack, table)) + }), + }); + { + let cpu_ops = Arc::clone(&cpu_ops); + units.push(Unit::Build { + family: family("DECODE"), + rows: decode_trace.num_rows(), + job: Box::new(move || { + let mut decode = decode_trace; + let rows = cpu_ops.iter().map(|op| op.decode_row as usize); + let padding = std::iter::repeat_n(decode_table.padding_row(), num_padding_rows); + decode::count_rows(&mut decode, rows.chain(padding)); + Ok(packed_if(pack, decode)) + }), + }); + } + let num_keccak = shared.keccak_ops.len(); + units.push(Unit::Build { + family: family("KECCAK_RC"), + rows: 64, + job: Box::new(move || { + let mut keccak_rc = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut keccak_rc, num_keccak); + Ok(packed_if(pack, keccak_rc)) + }), + }); + units.push(Unit::Build { + family: family("REGISTER"), + rows: 64, + job: Box::new(move || { + Ok(packed_if( + pack, + register::generate_register_trace(®ister_final_state, register_init), + )) + }), + }); + units.push(Unit::Build { + family: family("HALT"), + rows: 4, + job: Box::new(move || { + Ok(packed_if( + pack, + halt::generate_halt_trace(halt_timestamp, halt_next_pc), + )) + }), + }); + + // COPY, KECCAK, KECCAK_RND, ECSM, ECDAS, HINT, BLAKE3. + let copy_ops = Arc::clone(&shared.copy_ops); + units.extend(cut_units( + family("COPY"), + ©_ops, + skip.copy_rows, + skip.cut_padding, + pack, + form, + Arc::new(copy::generate_copy_rows_as), + |ops| Box::new(move || Ok(copy::generate_copy_trace(&ops.whole()))), + )?); + drop(copy_ops); + let keccak_ops = Arc::clone(&shared.keccak_ops); + units.extend(cut_units( + family("KECCAK"), + &keccak_ops, + skip.keccak_rows, + skip.cut_padding, + pack, + form, + Arc::new(keccak::generate_keccak_rows_as), + |ops| Box::new(move || Ok(keccak::generate_keccak_trace(&ops.whole()))), + )?); + units.extend(keccak_rnd_units(&keccak_ops, &skip, pack, form)?); + // Tables held as their records are never dropped: nothing to keep. + if let Some(regen) = regen.as_mut().filter(|_| !(skip.card_fill && pack)) { + regen.keccak_ops = Some(Arc::clone(&keccak_ops)); + } + drop(keccak_ops); + let ecsm_ops = Arc::clone(&shared.ecsm_ops); + units.extend(cut_units( + family("ECSM"), + &ecsm_ops, + skip.ecsm_rows, + skip.cut_padding, + pack, + form, + Arc::new(ecsm::generate_ecsm_rows_as), + |ops| Box::new(move || Ok(ecsm::generate_ecsm_trace(&ops.whole()))), + )?); + drop(ecsm_ops); + let ecdas_ops = Arc::clone(&shared.ecdas_ops); + units.extend(cut_units( + family("ECDAS"), + &ecdas_ops, + skip.ecdas_rows, + skip.cut_padding, + pack, + form, + Arc::new(|rows: &[ecdas::CompactEcdasOp], num_rows, form| { + ecdas::generate_ecdas_rows_of(rows, num_rows, form, ecdas::widen) + }), + |ops| { + Box::new(move || { + let rows = ops.whole(); + let num_rows = rows_of(rows.len()); + Ok(ecdas::generate_ecdas_rows_of( + &rows, + num_rows, + TraceForm::Wide, + ecdas::widen, + )) + }) + }, + )?); + drop(ecdas_ops); + let hint_ops = Arc::clone(&shared.hint_ops); + units.extend(optional_unit(family("HINT"), hint_ops.len(), || { + Box::new(move || { + Ok(packed_if( + pack, + hint::generate_hint_trace_as(&hint_ops.whole(), form), + )) + }) + })); + let ecrec_ops = Arc::clone(&shared.ecrec_ops); + units.extend(optional_unit(family("ECREC"), ecrec_ops.len(), || { + Box::new(move || { + Ok(packed_if( + pack, + ecrec::generate_ecrec_trace_as(&ecrec_ops.whole(), form), + )) + }) + })); + // BLAKE3 is a table of the proof only when it has ops (`VmAirs`' + // `include_blake3` from the counts), whatever the build makes of it + // otherwise. Read off the ops, so a miscounted header is refused below. + let (blake3_ops, blake3_absorb_ops) = ( + Arc::clone(&shared.blake3_ops), + Arc::clone(&shared.blake3_absorb_ops), + ); + let num_blake3 = blake3_ops.len() + blake3_absorb_ops.len(); + if num_blake3 > 0 { + let rows = rows_of(num_blake3); + units.push(Unit::Build { + family: family("BLAKE3"), + rows, + job: Box::new(move || { + Ok(packed_if( + pack, + blake3::generate_blake3_trace( + &blake3_ops.whole(), + &blake3_absorb_ops.whole(), + ), + )) + }), + }); + } + let is_wh_rows = Arc::clone(&shared.is_wh_rows); + units.extend(optional_unit(family("IS_WH"), is_wh_rows.len(), || { + Box::new(move || Ok(packed_if(pack, is_wh::generate_is_wh_trace(&is_wh_rows)))) + })); + + // CPU … CPU32. + units.extend(chunked_units( + family("CPU"), + cpu_ops.len(), + max_rows.cpu, + skip.cpu, + tails, + false, + |start, end| { + let ops = Arc::clone(&cpu_ops); + Box::new(move || { + Ok(packed_if( + pack, + cpu::generate_cpu_trace_as(&ops[start..end], decode_table, form), + )) + }) + }, + )?); + drop(cpu_ops); + let lt = Arc::clone(&shared.lt); + units.extend(lt_units( + <, + max_rows.lt, + &skip, + pack, + form, + from_tables.clone(), + )?); + if let Some(regen) = regen.as_mut() { + regen.lt = Some(Arc::clone(<)); + } + drop(lt); + units.extend(segmented_units( + family("SHIFT"), + Arc::clone(&shared.shift_ops), + max_rows.shift, + skip.shift, + tails, + move |ops: &[ShiftOperation]| { + packed_if(pack, shift::generate_shift_trace_as(ops, form)) + }, + )?); + units.extend(vec_units( + family("MEMW"), + Arc::new(memw_ops), + max_rows.memw, + skip.memw, + tails, + true, + move |ops: &[MemwOperation]| packed_if(pack, memw::generate_memw_trace_as(ops, form)), + )?); + units.extend(vec_units( + family("MEMW_A"), + Arc::clone(&shared.memw_aligned_ops), + max_rows.memw_aligned, + skip.memw_aligned, + tails, + true, + move |ops: &[memw_aligned::AlignedRow]| { + packed_if( + pack, + memw_aligned::generate_memw_aligned_trace_as(ops, form), + ) + }, + )?); + units.extend(vec_units( + family("LOAD"), + Arc::new(load_ops), + max_rows.load, + skip.load, + tails, + true, + move |ops: &[LoadOperation]| packed_if(pack, load::generate_load_trace_as(ops, form)), + )?); + units.extend(segmented_units( + family("MUL"), + Arc::clone(&shared.mul_ops), + max_rows.mul, + 0, + false, + move |ops: &[(MulOperation, bool)]| { + packed_if(pack, mul::generate_mul_trace_as(ops, form)) + }, + )?); + units.extend(segmented_units( + family("DVRM"), + Arc::clone(&shared.dvrm_ops), + max_rows.dvrm, + 0, + false, + move |ops: &[(DvrmOperation, bool)]| { + packed_if(pack, dvrm::generate_dvrm_trace_as(ops, form)) + }, + )?); + { + let tables = from_tables.clone(); + units.extend(segmented_units_try( + family("BRANCH"), + Arc::clone(&shared.branch_ops), + max_rows.branch, + 0, + false, + move |ops: &[BranchOperation]| { + let rows = dedup_rows(ops); + if let Some(tables) = &tables { + let mut plane = bitwise::BitwisePlane::new(); + count_branch_table_lookups(&rows, &mut plane); + tables.add(&plane)?; + } + Ok(packed_if(pack, branch::branch_trace_of_rows(&rows, form))) + }, + )?); + } + // PAGE: one a config, from the memory state. + let page_configs = Arc::new(header.page_configs.clone()); + units.extend((0..page_configs.len()).map(|k| { + let configs = Arc::clone(&page_configs); + Unit::Build { + family: family("PAGE"), + rows: page::DEFAULT_PAGE_SIZE, + job: Box::new(move || { + Ok(packed_if( + pack, + page_trace(&configs[k], memory_state, l2g_memory_bookend), + )) + }) as Job<'a>, + } + })); + units.extend(vec_units( + family("MEMW_R"), + Arc::clone(&shared.memw_register_rows), + max_rows.memw_register, + skip.memw_register, + tails, + false, + move |ops: &[RegRow]| { + packed_if( + pack, + memw_register::generate_memw_register_trace_from_rows_as(ops, form), + ) + }, + )?); + units.extend(segmented_units( + family("EQ"), + Arc::clone(&shared.eq_ops), + max_rows.eq, + 0, + false, + move |ops: &[eq::EqOperation]| packed_if(pack, eq::generate_eq_trace_as(ops, form)), + )?); + { + let tables = from_tables.clone(); + units.extend(segmented_units_try( + family("BYTEWISE"), + Arc::clone(&shared.bytewise_ops), + max_rows.bytewise, + 0, + false, + move |ops: &[bytewise::BytewiseOperation]| { + let rows = dedup_rows(ops); + if let Some(tables) = &tables { + let mut plane = bitwise::BitwisePlane::new(); + count_bytewise_table_lookups(&rows, &mut plane); + tables.add(&plane)?; + } + Ok(packed_if( + pack, + bytewise::bytewise_trace_of_rows(&rows, form), + )) + }, + )?); + } + units.extend(segmented_units( + family("STORE"), + Arc::clone(&shared.store_ops), + max_rows.store, + skip.store, + tails, + move |ops: &[store::StoreOperation]| { + packed_if(pack, store::generate_store_trace_as(ops, form)) + }, + )?); + units.extend(segmented_units( + family("CPU32"), + Arc::new(cpu32_ops), + max_rows.cpu32, + 0, + false, + move |ops: &[cpu32::Cpu32Operation]| { + packed_if(pack, cpu32::generate_cpu32_trace_as(ops, form)) + }, + )?); + + // The units are the tables the header counted, or the build is refused. + check_units(&units, &header.table_counts)?; + if let Some(tables) = &from_tables { + let adding = [family("LT"), family("BRANCH"), family("BYTEWISE")]; + let built = units + .iter() + .filter( + |unit| matches!(unit, Unit::Build { family, .. } if adding.contains(family)), + ) + .count(); + tables + .tables + .store(built, std::sync::atomic::Ordering::Relaxed); + } + Ok((units, regen, side)) + } +} + +/// LT's units: a streamed chunk's slot for each chunk the windows handed out, +/// then the list's cuts of `rows` rows. +fn lt_units<'a>( + lt: &std::sync::Arc, + rows: usize, + skip: &StreamSkip, + pack: bool, + form: TraceForm, + from_tables: Option>, +) -> Result>, Error> { + chunked_units( + family("LT"), + lt.segments().len(), + rows, + skip.lt, + skip.tails, + true, + |start, end| { + let lt = std::sync::Arc::clone(lt); + let from_tables = from_tables.clone(); + Box::new(move || { + let rows = dedup_rows(<.segments().range(start, end)); + if let Some(tables) = &from_tables { + let mut plane = bitwise::BitwisePlane::new(); + count_lt_table_lookups(&rows, &mut plane); + tables.add(&plane)?; + } + Ok(packed_if(pack, lt::lt_trace_of_rows(&rows, form))) + }) + }, + ) +} + +/// The rest's families phase B builds again from lists kept for it (D-WHIR-NODISK +/// N3): their tables are dropped after their group's commit, as the streamed +/// chunks are. KECCAK_RND and LT are ≈ 82 % of the rest's packed bytes at 200 +/// Mgas, against lists a fraction of that; the other families stay on the host. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) enum RegenFamily { + KeccakRnd, + Lt, +} + +impl RegenFamily { + /// Both, in AIR order. + pub(crate) const ALL: [Self; 2] = [Self::KeccakRnd, Self::Lt]; + + /// Its AIRs' name before the index. + pub(crate) fn name(self) -> &'static str { + match self { + Self::KeccakRnd => "KECCAK_RND", + Self::Lt => "LT", + } + } + + /// The family and the index of a table's AIR name (`KECCAK_RND[3]`, `LT[0]`). + pub(crate) fn of_air(name: &str) -> Option<(Self, usize)> { + let (base, index) = name.strip_suffix(']')?.split_once('[')?; + let family = Self::ALL.into_iter().find(|f| f.name() == base)?; + Some((family, index.parse().ok()?)) + } +} + +/// A table [`RestRegen`] builds again: building it is calling this. +pub(crate) type RegenJob = Box Result + Send + 'static>; + +/// What phase B needs to build KECCAK_RND's and LT's tables again +/// ([`RegenFamily`]): their lists, kept past the finish, and the cut the +/// finish made them with. The jobs are the finish's own, so their tables are +/// the finish's. +pub(crate) struct RestRegen { + keccak_ops: Option>>, + lt: Option>, + skip: StreamSkip, + lt_rows: usize, + pack: bool, + form: TraceForm, +} + +impl RestRegen { + /// `family`'s tables in the family's AIR order: a streamed chunk's slot + /// (`None`) or the job that builds the table again. The family's list goes + /// with the jobs: once each has run or been dropped it is freed, and a + /// second take of the family is refused. + pub(crate) fn take(&mut self, family: RegenFamily) -> Result>, Error> { + let units: Vec> = match family { + // Its tables are held as their records and never dropped. + RegenFamily::KeccakRnd if self.skip.card_fill && self.pack => Vec::new(), + RegenFamily::KeccakRnd => { + let ops = self + .keccak_ops + .take() + .ok_or_else(|| Error::Prover("KECCAK_RND's ops were taken already".into()))?; + keccak_rnd_units(&ops, &self.skip, self.pack, self.form)? + } + RegenFamily::Lt => { + let lt = self + .lt + .take() + .ok_or_else(|| Error::Prover("LT's lists were taken already".into()))?; + lt_units(<, self.lt_rows, &self.skip, self.pack, self.form, None)? + } + }; + Ok(units + .into_iter() + .map(|unit| match unit { + Unit::Streamed { .. } => None, + Unit::Build { job, .. } => Some(job), + }) + .collect()) + } +} + +/// A family's units from a list held whole ([`chunk_and_generate_skipping`]). +#[allow(clippy::too_many_arguments)] +fn vec_units<'a, T: Send + Sync + 'a>( + family: usize, + ops: std::sync::Arc>, + max: usize, + skip: usize, + tails: bool, + optional: bool, + generate: impl Fn(&[T]) -> Table + Send + Sync + 'a, +) -> Result>, Error> { + let len = ops.len(); + let generate = std::sync::Arc::new(generate); + chunked_units(family, len, max, skip, tails, optional, |start, end| { + let ops = std::sync::Arc::clone(&ops); + let generate = std::sync::Arc::clone(&generate); + Box::new(move || Ok(generate(&ops[start..end]))) + }) +} + +/// A list a family's chunks are read from as segments: in blocks, or +/// delta-coded ([`delta::DeltaStream`]). +trait SegmentList: Send + Sync { + fn list_len(&self) -> usize; + fn list_segments(&self) -> Segmented<'_, T>; +} + +impl SegmentList for BlockVec { + fn list_len(&self) -> usize { + self.len() + } + + fn list_segments(&self) -> Segmented<'_, T> { + self.segments() + } +} + +impl SegmentList for delta::DeltaStream { + fn list_len(&self) -> usize { + self.len() + } + + fn list_segments(&self) -> Segmented<'_, C::Op> { + self.segments() + } +} + +/// A family's units from a list read as segments ([`chunk_and_generate_segmented`] +/// over them); an empty list makes no table. +fn segmented_units<'a, T: Clone + Send + Sync + 'a, S: SegmentList + 'a>( + family: usize, + ops: std::sync::Arc, + max: usize, + skip: usize, + tails: bool, + generate: impl Fn(&[T]) -> Table + Send + Sync + 'a, +) -> Result>, Error> { + segmented_units_try(family, ops, max, skip, tails, move |ops: &[T]| { + Ok(generate(ops)) + }) +} + +/// [`segmented_units`] whose tables' jobs can fail. +fn segmented_units_try<'a, T: Clone + Send + Sync + 'a, S: SegmentList + 'a>( + family: usize, + ops: std::sync::Arc, + max: usize, + skip: usize, + tails: bool, + generate: impl Fn(&[T]) -> Result + Send + Sync + 'a, +) -> Result>, Error> { + let len = ops.list_len(); + let generate = std::sync::Arc::new(generate); + chunked_units(family, len, max, skip, tails, true, |start, end| { + let ops = std::sync::Arc::clone(&ops); + let generate = std::sync::Arc::clone(&generate); + Box::new(move || generate(&ops.list_segments().range(start, end))) + }) +} + +/// KECCAK_RND's units ([`keccak_rnd_chunks`]' tables, as many as +/// [`StreamSkip::cut_padding`] counts, or the whole table wide without +/// chunks): a chunk's round ops are derived from the KECCAK ops that reach it +/// as the chunk is built. With [`StreamSkip::card_fill`] a cut table holds +/// those ops instead (`keccak_rnd::KeccakRndRecords`), for the card to fill. +fn keccak_rnd_units<'a>( + keccak_ops: &std::sync::Arc>, + skip: &StreamSkip, + pack: bool, + form: TraceForm, +) -> Result>, Error> { + let f = family("KECCAK_RND"); + let n = keccak_ops.len(); + let rounds = |ops: &[KeccakOperation]| -> Vec { + ops.iter() + .map(|op| KeccakRoundOperation { + timestamp: op.timestamp, + input: op.input, + output: op.output, + }) + .collect() + }; + let card_fill = skip.card_fill && pack; + #[cfg(test)] + let fault = skip.card_fill_fault; + #[cfg(not(test))] + let fault = 0; + // Rows `skip..skip + rows` of the table `ops` make: as records for the + // card, or built here. + let build = move |ops: &[KeccakOperation], skip: usize, rows: usize| -> Table { + let records = card_fill.then(|| { + keccak_rnd::KeccakRndRecords::new( + ops.iter().map(|op| multilinear::records::KeccakRndOp { + timestamp: op.timestamp, + input: op.input, + }), + skip + fault, + rows, + ) + .into_trace() + }); + records.flatten().unwrap_or_else(|| { + packed_if( + pack, + keccak_rnd::generate_keccak_rnd_rows_as(&rounds(ops), skip, rows, form), + ) + }) + }; + if n == 0 { + return Ok(Vec::new()); + } + let rows = skip.keccak_rnd_rows; + let streamed = skip.keccak_rnd; + let total = (n * 24).next_power_of_two().max(4); + if rows == 0 { + let ops = std::sync::Arc::clone(keccak_ops); + return Ok(vec![Unit::Build { + family: f, + rows: total, + job: Box::new(move || Ok(keccak_rnd::generate_keccak_rnd_trace(&rounds(&ops.whole())))), + }]); + } + if total <= rows { + if streamed > 0 { + return Err(Error::Prover(format!( + "{streamed} KECCAK_RND chunks were streamed but the table is one of {total} rows" + ))); + } + let ops = std::sync::Arc::clone(keccak_ops); + return Ok(vec![Unit::Build { + family: f, + rows: total, + job: Box::new(move || Ok(build(&ops.whole(), 0, total))), + }]); + } + let chunks = skip.cut_padding.tables(n * 24, rows); + if streamed > chunks { + return Err(Error::Prover(format!( + "{streamed} KECCAK_RND chunks were streamed but the run has {chunks}" + ))); + } + Ok((0..chunks) + .map(|c| { + if c < streamed { + return Unit::Streamed { family: f }; + } + let ops = std::sync::Arc::clone(keccak_ops); + let (first, end) = keccak_rnd_op_range(c, rows, n); + Unit::Build { + family: f, + rows, + job: Box::new(move || { + Ok(build(&ops.range(first, end), c * rows - first * 24, rows)) + }) as Job<'a>, + } + }) + .collect()) +} + +/// The units' tables, family by family, against the header's counts: the same, +/// or an error ([`FinishPlan::emit_all`]'s refusal). HALT and the four fixed +/// tables are one each. +fn check_units(units: &[Unit<'_>], counts: &crate::TableCounts) -> Result<(), Error> { + let mut made = [0usize; FAMILIES.len()]; + for unit in units { + let (Unit::Build { family, .. } | Unit::Streamed { family }) = unit; + if let Some(n) = made.get_mut(*family) { + *n += 1; + } + } + let of = |name: &str| made[family(name)]; + let units_counts = crate::TableCounts { + cpu: of("CPU"), + lt: of("LT"), + memw: of("MEMW"), + memw_aligned: of("MEMW_A"), + load: of("LOAD"), + mul: of("MUL"), + dvrm: of("DVRM"), + shift: of("SHIFT"), + branch: of("BRANCH"), + memw_register: of("MEMW_R"), + eq: of("EQ"), + bytewise: of("BYTEWISE"), + store: of("STORE"), + cpu32: of("CPU32"), + keccak: of("KECCAK"), + keccak_rnd: of("KECCAK_RND"), + ecsm: of("ECSM"), + ecdas: of("ECDAS"), + hint: of("HINT"), + copy: of("COPY"), + blake3: of("BLAKE3"), + is_wh: of("IS_WH"), + ecrec: of("ECREC"), + }; + let fixed = ["BITWISE", "DECODE", "KECCAK_RC", "REGISTER", "HALT"]; + if counts_of(&units_counts) != counts_of(counts) || fixed.iter().any(|f| of(f) != 1) { + return Err(Error::Prover(format!( + "the streamed finish would build {units_counts:?}, its plan counted {counts:?}" + ))); + } + Ok(()) +} + +impl<'a> FinishPlan<'a> { + /// Phase 5 streamed: the units ([`Self::units`]) built in waves and handed + /// to `send` in AIR order, from this thread. A wave is the units in order + /// up to half the gate's budget of estimated bytes, built in parallel; the + /// gate holds each table's bytes from before it is built until the packer + /// places it ([`gate::ByteGate`]), so at most one wave and the tables not + /// yet placed are in memory. A table is estimated at its family's measured + /// bytes a row once one of the family has been built, at eight bytes a + /// cell before. A job's error, or `send`'s, stops the gate and is returned. + #[cfg(test)] + pub(super) fn emit_streamed( + self, + gate: &gate::ByteGate, + send: &mut dyn FnMut(Emitted) -> Result<(), Error>, + ) -> Result<(), Error> { + emit_streamed_units(self.units(), gate, send, false, None, None) + } + + /// [`Self::emit_streamed`], and with `keep` what phase B needs to build + /// KECCAK_RND's and LT's tables again ([`RestRegen`]); with `slide` the + /// tables are built as they are admitted instead of in waves + /// ([`emit_units_sliding`]). + pub(super) fn emit_streamed_keeping( + self, + gate: &gate::ByteGate, + send: &mut dyn FnMut(Emitted) -> Result<(), Error>, + keep: bool, + slide: bool, + ) -> Result, Error> { + // BITWISE is the rest's first table: its position is 0. + let last = (slide && self.header.bitwise_last).then_some(0); + let (units, regen, side) = match self.units_keeping(keep) { + Ok((units, regen, side)) => (Ok(units), regen, side), + Err(e) => (Err(e), None, None), + }; + emit_streamed_units(units, gate, send, slide, last, side)?; + Ok(regen) + } +} + +/// [`FinishPlan::emit_streamed`] over `units`, in waves or sliding (`slide`; +/// a build without the `parallel` feature builds one table at a time either +/// way, so it always takes the waves), sliding with `last` handed on after +/// every other table and `side` run beside the builds ([`Side`]; `last` then +/// starts only once it is due): an error stops the gate, so it reaches every +/// waiter, and is returned. +fn emit_streamed_units<'a>( + units: Result>, Error>, + gate: &gate::ByteGate, + send: &mut dyn FnMut(Emitted) -> Result<(), Error>, + slide: bool, + last: Option, + side: Option>, +) -> Result<(), Error> { + #[cfg(feature = "parallel")] + let result = if slide { + emit_units_sliding(units, gate, send, last, side) + } else { + if let Some(side) = side { + side(); + } + emit_units(units, gate, send) + }; + #[cfg(not(feature = "parallel"))] + let result = { + let _ = (slide, last); + if let Some(side) = side { + side(); + } + emit_units(units, gate, send) + }; + if let Err(e) = &result { + gate.stop(&format!("the finish stopped: {e:?}")); + } + result +} + +/// A table's bytes before it is built: its family's measured bytes a row once +/// one of the family has been built, eight bytes a cell before. +struct Estimates { + per_row: [Option; FAMILIES.len()], +} + +impl Estimates { + fn new() -> Self { + Self { + per_row: [None; FAMILIES.len()], + } + } + + fn of(&self, family: usize, rows: usize) -> usize { + let wide = FAMILIES.get(family).map_or(1, |&(_, columns)| columns) * 8; + let a_row = self + .per_row + .get(family) + .copied() + .flatten() + .unwrap_or(wide as f64); + (a_row * rows as f64).ceil() as usize + } + + /// `table` of `family` was built at `bytes`. + fn learn(&mut self, family: usize, table: &Table, bytes: usize) { + if table.main_table.height > 0 + && let Some(slot) = self.per_row.get_mut(family) + { + *slot = Some(bytes as f64 / table.main_table.height as f64); + } + } +} + +/// [`emit_streamed_units`]' loop in waves. +fn emit_units( + units: Result>, Error>, + gate: &gate::ByteGate, + send: &mut dyn FnMut(Emitted) -> Result<(), Error>, +) -> Result<(), Error> { + let wave_bytes = (gate.budget() / 2).max(1); + let mut estimates = Estimates::new(); + let mut units = units?.into_iter().enumerate().peekable(); + let mut next_table = 0usize; + while units.peek().is_some() { + // The wave: units in order until one does not fit; its first table + // waits for its bytes (the lowest not yet placed always goes). + let mut wave: Vec> = Vec::new(); + let mut bytes = 0usize; + let mut tables = 0usize; + while let Some((_, unit)) = units.peek() { + if let Unit::Build { family, rows, .. } = unit { + let est = estimates.of(*family, *rows); + if tables == 0 { + gate.acquire(next_table, est)?; + } else if bytes.saturating_add(est) > wave_bytes + || !gate.try_acquire(next_table, est)? + { + break; + } + bytes = bytes.saturating_add(est); + tables += 1; + } + let Some((position, unit)) = units.next() else { + break; + }; + match unit { + Unit::Streamed { .. } => wave.push((position, None)), + Unit::Build { family, job, .. } => { + wave.push((position, Some((next_table, family, job)))); + next_table += 1; + } + } + } + // Built in parallel; no worker waits on the gate or on `send`. + let (slots, jobs): (Vec<_>, Vec<_>) = wave + .into_iter() + .map(|(position, build)| match build { + Some((table, family, job)) => ((position, Some((table, family))), Some(job)), + None => ((position, None), None), + }) + .unzip(); + let jobs: Vec> = jobs.into_iter().flatten().collect(); + #[cfg(feature = "parallel")] + let built: Vec> = jobs.into_par_iter().map(|job| job()).collect(); + #[cfg(not(feature = "parallel"))] + let built: Vec> = jobs.into_iter().map(|job| job()).collect(); + let mut built = built.into_iter(); + for (position, build) in slots { + let table = match build { + None => None, + Some((number, family)) => { + let table = built.next().ok_or_else(|| { + Error::Prover("the streamed finish lost a table of its wave".into()) + })??; + let bytes = table_bytes(&table); + gate.resize(number, bytes); + estimates.learn(family, &table, bytes); + Some(table) + } + }; + send(Emitted { position, table })?; + } + build_stamps::mark("p5 wave"); + } + build_stamps::mark("p5 end"); + Ok(()) +} + +/// A table built on the pool: its AIR position, its number among the rest's +/// tables, its family, the bytes it was estimated at, the table or its error, +/// and the seconds its job took. +#[cfg(feature = "parallel")] +type Built = (usize, usize, usize, usize, Result, f64); + +/// A family's tables built by [`emit_units_sliding`], for the build's marks: +/// how many, when the first started and the last was built (seconds since the +/// marks' start), their jobs' seconds summed and their bytes. +#[cfg(feature = "parallel")] +#[derive(Clone, Copy, Default)] +struct FamilyBuild { + tables: usize, + first_started: Option, + last_built: f64, + busy: f64, + bytes: usize, +} + +/// [`emit_streamed_units`]' loop without waves: each table starts building on +/// the pool as soon as the gate admits its estimated bytes, while earlier ones +/// still build, and is handed on in AIR order once it and every table before +/// it are built. A wave lasts as long as its slowest table; here a slow table +/// holds back only the hand-on of the tables after it, not their builds. The +/// gate, the estimates, the tables and their order are the waves'. When +/// nothing builds and every built table has been handed on, the packer holds +/// every earlier table, so the next one may wait for its room (the lowest +/// table not yet placed always goes). A job that panics ends the stream with +/// an error. While anything builds, a table starts only if the estimated bytes +/// building at once stay within the gate's build cap +/// ([`gate::ByteGate::with_build`]; by default the budget, which the gate +/// already implies), so a larger gate can hold more built tables for the +/// packer without building more of them at once. With `last`, that table is +/// handed on after every other ([`hand_on_order`]) and numbered at the gate in +/// that order; it still starts first when the gate admits it, and otherwise +/// starts once it is due, without holding back the tables after it. With +/// `side` (run on a thread of its own while the tables build), `last` starts +/// only once it is due: every other table is built by then. +#[cfg(feature = "parallel")] +fn emit_units_sliding<'a>( + units: Result>, Error>, + gate: &gate::ByteGate, + send: &mut dyn FnMut(Emitted) -> Result<(), Error>, + last: Option, + side: Option>, +) -> Result<(), Error> { + let units = units?; + let last_waits = side.is_some(); + std::thread::scope(|threads| { + if let Some(side) = side { + // A side that panics drops what it would have sent: its reader + // fails instead of waiting. + threads.spawn(move || { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(side)); + }); + } + slide_units(units, gate, send, last, last_waits) + }) +} + +/// [`emit_units_sliding`]'s loop. +#[cfg(feature = "parallel")] +fn slide_units( + units: Vec>, + gate: &gate::ByteGate, + send: &mut dyn FnMut(Emitted) -> Result<(), Error>, + last: Option, + last_waits: bool, +) -> Result<(), Error> { + let total = units.len(); + let mut estimates = Estimates::new(); + // Built, not yet handed on, by AIR position (`Some(None)`: a streamed + // chunk's slot). + let mut ready: Vec>> = (0..total).map(|_| None).collect(); + // The positions in the order they are handed on: AIR order, but `last` + // after every other ([`hand_on_order`]). + let order = hand_on_order(total, last); + // Each table's number at the gate: its place among the tables in that + // order, the order the packer places and releases them in. + let mut numbers = vec![0usize; total]; + let mut placed = 0usize; + for &position in &order { + if let Some(Unit::Build { .. }) = units.get(position) { + numbers[position] = placed; + placed += 1; + } + } + let (tx, rx) = std::sync::mpsc::channel::(); + let (mut sent, mut building) = (0usize, 0usize); + // The estimated bytes of the tables building now. + let mut building_bytes = 0usize; + let mut units: std::collections::VecDeque<(usize, Unit<'_>)> = + units.into_iter().enumerate().collect(); + // `last`'s table when the gate did not admit it as it came: the tables + // after it start meanwhile, and it is tried again before them, and goes + // once it is due. + let mut aside = None; + let mut families = [FamilyBuild::default(); FAMILIES.len()]; + rayon::in_place_scope_fifo(|scope| { + loop { + while let Some(&due) = order.get(sent) { + let Some(table) = ready.get_mut(due).and_then(Option::take) else { + break; + }; + send(Emitted { + position: due, + table, + })?; + sent += 1; + } + if sent == total { + break; + } + // Start every table the gate admits, in order. + let mut waiting = false; + if let Some(unit) = aside.take() { + units.push_front(unit); + } + while let Some((position, unit)) = units.front() { + // Started only once it is due ([`Side`]). + if last_waits && Some(*position) == last && order.get(sent) != Some(position) { + aside = units.pop_front(); + continue; + } + let mut est_now = 0usize; + if let Unit::Build { family, rows, .. } = unit { + let est = estimates.of(*family, *rows); + if building == 0 && order.get(sent) == Some(position) { + gate.acquire(numbers[*position], est)?; + } else if (building > 0 && building_bytes.saturating_add(est) > gate.build()) + || !gate.try_acquire(numbers[*position], est)? + { + waiting = true; + if Some(*position) == last { + aside = units.pop_front(); + continue; + } + break; + } + est_now = est; + } + let Some((position, unit)) = units.pop_front() else { + break; + }; + match unit { + Unit::Streamed { .. } => ready[position] = Some(None), + Unit::Build { family, job, .. } => { + if let Some(f) = families.get_mut(family) + && f.first_started.is_none() + { + f.first_started = build_stamps::since_start(); + } + let (tx, number) = (tx.clone(), numbers[position]); + scope.spawn_fifo(move |_| { + let t = std::time::Instant::now(); + let table = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job)) + .unwrap_or_else(|_| { + Err(Error::Prover(format!( + "the rest's table {number} panicked while building" + ))) + }); + let secs = t.elapsed().as_secs_f64(); + let _ = tx.send((position, number, family, est_now, table, secs)); + }); + building += 1; + building_bytes = building_bytes.saturating_add(est_now); + gate.note_building(building_bytes); + } + } + } + if order.get(sent).is_some_and(|&due| ready[due].is_some()) || building == 0 { + continue; + } + let built = if waiting { + match rx.recv_timeout(std::time::Duration::from_millis(5)) { + Ok(built) => built, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + return Err(Error::Prover("the rest's builders stopped".into())); + } + } + } else { + rx.recv() + .map_err(|_| Error::Prover("the rest's builders stopped".into()))? + }; + let (position, number, family, est, table, secs) = built; + building -= 1; + building_bytes = building_bytes.saturating_sub(est); + let table = table?; + let bytes = table_bytes(&table); + gate.resize(number, bytes); + estimates.learn(family, &table, bytes); + if let Some(f) = families.get_mut(family) { + f.tables += 1; + f.last_built = build_stamps::since_start().unwrap_or(0.0); + f.busy += secs; + f.bytes += bytes; + } + ready[position] = Some(Some(table)); + } + // Each family's span, its jobs' seconds summed (threads busy) and its + // bytes: a measurement. + for (f, &(name, _)) in families.iter().zip(&FAMILIES) { + if let Some(first) = f.first_started { + build_stamps::mark(&format!( + "p5 {name} {} built {first:.2}-{:.2} busy {:.2}s {:.2}GiB,", + f.tables, + f.last_built, + f.busy, + f.bytes as f64 / (1u64 << 30) as f64 + )); + } + } + build_stamps::mark("p5 end"); + Ok(()) + }) +} + +/// The order the sliding finish hands the rest's `total` positions on in: AIR +/// order, with `last` (BITWISE's, when the finish counts it beside the rest's +/// builds and hands it on last: [`StreamSkip::bitwise_last`]) after every other. +pub(crate) fn hand_on_order(total: usize, last: Option) -> Vec { + let last = last.filter(|&p| p < total); + (0..total) + .filter(|&p| Some(p) != last) + .chain(last) + .collect() +} + +#[cfg(test)] +#[path = "stream_tests.rs"] +mod stream_tests; diff --git a/prover/src/tables/trace_builder/finish_oracle.rs b/prover/src/tables/trace_builder/finish_oracle.rs new file mode 100644 index 000000000..3fd78a84e --- /dev/null +++ b/prover/src/tables/trace_builder/finish_oracle.rs @@ -0,0 +1,1070 @@ +//! ★ The table phase as it was before the finish plan ([`super::finish`]), +//! frozen as the oracle the plan is tested against: every table the plan emits +//! must be this build's, word for word. Test-only, kept for one release. +//! +//! A test turns it on for its own thread ([`with`]); [`super::build_traces`] +//! then runs this instead of the plan. It pads the cut tables as it always +//! did ([`CutPadding::WholeRun`]), whatever the build's settings say. + +#![allow(clippy::too_many_arguments)] + +use std::cell::Cell; + +use super::*; + +thread_local! { + static ON: Cell = const { Cell::new(false) }; + static RUNS: Cell = const { Cell::new(0) }; +} + +/// How many builds on this thread ran the oracle: a test checks its oracle +/// side was the oracle. +pub(crate) fn runs() -> usize { + RUNS.with(Cell::get) +} + +/// Whether this thread's builds run the oracle. +pub(super) fn on() -> bool { + ON.with(Cell::get) +} + +/// `f` with this thread's builds running the oracle instead of the plan. +pub(crate) fn with(f: impl FnOnce() -> T) -> T { + struct Off; + impl Drop for Off { + fn drop(&mut self) { + ON.with(|on| on.set(false)); + } + } + ON.with(|on| on.set(true)); + let _off = Off; + f() +} + +/// `build_traces` at 6cbf1c4ee, renamed: phases 3-5 in one function. +pub(super) fn build_traces_oracle( + ops: CollectedOps, + initial_image: Option<&I>, + memory_state: &MemoryState, + register_init: &[u32], + decode_trace: TraceTable, + decode_table: &DecodeTable, + mut register_state: RegisterState, + max_rows: &crate::tables::MaxRowsConfig, + private_input: &[u8], + is_final: bool, + l2g_memory_bookend: bool, + skip: &StreamSkip, + mut pre: Option, +) -> Result { + RUNS.with(|runs| runs.set(runs.get() + 1)); + let CollectedOps { + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + branch_ops, + mul_ops, + dvrm_ops, + copy_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + ecrec_ops, + } = ops; + + // ===================================================================== + // PHASE 3: MEMW → LT (timestamp ordering and overflow checks) + // ===================================================================== + // LT's list is these segments in this order, kept apart: concatenating them + // into one list reallocated multi-GiB copies at a block's finish. + let memw_lt = pre + .as_mut() + .map(|pre| std::mem::take(&mut pre.memw_lt)) + .unwrap_or_default(); + let lt_from_memw = collect_lt_from_memw(&memw_ops[skip.memw_lt_done.min(memw_ops.len())..]); + build_stamps::mark("p3a lt from memw"); + let memw_aligned_lt = pre + .as_mut() + .map(|pre| std::mem::take(&mut pre.memw_aligned_lt)) + .unwrap_or_default(); + let lt_from_memw_aligned = collect_lt_from_memw_aligned( + &memw_aligned_ops[skip.memw_aligned_lt_done.min(memw_aligned_ops.len())..], + ); + build_stamps::mark("p3b lt from memw_a"); + // HINT range-checks: selector < 3 and both address low limbs < 2^32 - 31 (matching + // the executor's HintUnknownSelector / HintAddressOverflow rejections). Three LT ops + // per hint call; the HINT table sends the matching ALU LT interactions. + let lt_from_hints: Vec = hint_ops + .iter() + .flat_map(|op| { + [ + LtOperation::new(op.hint_id, hint::HINT_SELECTOR_BOUND, false), + LtOperation::new(op.in_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), + LtOperation::new(op.out_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), + ] + }) + .collect(); + let lt_concat: Vec; + let lt_ops = if skip.concat_lt { + // The A arm: one list, each segment appended and then freed, as before. + let mut all = lt_ops.into_vec(); + memw_lt.expand_into(0, memw_lt.len(), &mut all); + drop(memw_lt); + all.extend(lt_from_memw); + memw_aligned_lt.expand_into(0, memw_aligned_lt.len(), &mut all); + drop(memw_aligned_lt); + all.extend(lt_from_memw_aligned); + all.extend(lt_from_hints); + lt_concat = all; + Segmented { + parts: vec![Part::Ops(<_concat[..])], + } + } else { + // The walk's LT ops and what DVRM implies are blocks of their own. + let mut parts: Vec> = lt_ops.parts().map(Part::Ops).collect(); + parts.extend([ + Part::Compact(&memw_lt), + Part::Ops(<_from_memw), + Part::Compact(&memw_aligned_lt), + Part::Ops(<_from_memw_aligned), + Part::Ops(<_from_hints), + ]); + Segmented { parts } + }; + + build_stamps::mark("p3 lt"); + + // ===================================================================== + // PHASE 4: All → Bitwise lookups + // ===================================================================== + #[cfg(feature = "instruments")] + let __sp = stark::instruments::span("p4_bitwise_collect"); + + // The committed bytes are read off COPY's `write` rows, in `dst` (global byte + // index) order; a zero-length `write`'s row moves none. + let public_output_bytes: Vec = { + let mut rows: Vec<©::CopyOperation> = copy_ops + .iter() + .filter(|op| op.functionality == copy::Functionality::Write && op.count != 0) + .collect(); + rows.sort_by_key(|op| op.dst); + rows.iter() + .flat_map(|op| op.value[..op.width as usize].iter().copied()) + .collect() + }; + + // CPU padding rows send ARE_BYTES with all-zero values. + // Add corresponding ops so the bitwise table multiplicities balance. + // Without the streamed chunks' ops (`skip.tails`), their full chunks pad too. + let padding = |len: usize| len.next_power_of_two().max(4) - len; + let num_padding_rows: usize = cpu_ops + .chunks(max_rows.cpu) + .map(|chunk| padding(chunk.len())) + .sum::() + + if skip.tails { + skip.cpu * padding(max_rows.cpu) + } else { + 0 + }; + + // The per-source bitwise collectors are all pure functions of their inputs, and the + // BITWISE multiplicities are order-independent (they ride a permutation-invariant bus), + // so every source can be collected in parallel and the per-worker histograms summed in + // any order. + // + // MUL/DVRM dedup their per-unique bit-gated lookups PER CHIP INSTANCE, so pass the same + // chunk size used to split them into instances so multiplicities match the per-instance + // sends. MEMW_R sends IS_HALFWORD[timestamp_0 - old_timestamp_lo - 1]. PAGE does a + // batched ARE_BYTES[init, fini] per row (skipped in continuation epochs, which the L2G + // table owns). COPY sends IsHalfword; KECCAK_RND sends XOR/AND/ARE_BYTES/HWSL; + // HINT sends ARE_BYTES for its 32 output cells. + // We never concatenate the lookups into one giant `Vec` (~140 M ops / + // ~560 MB at 10-tx whose only consumer is the multiplicity count). Each collector bumps + // the `BitwiseHistogram` it is handed: the heavy sources (MEMW_R one-per-row, PAGE + // one-per-byte, padding) count directly with no per-source Vec at all, and the small + // sources fold their transient `collect_*` Vec in and drop it. The histogram is a + // commutative monoid, so per-worker histograms tree-reduce to multiplicities that are + // independent of accumulation order. + type Collector<'a> = Box; + let mul_chunk = max_rows.mul; + let dvrm_chunk = max_rows.dvrm; + // The sources that are a sum over their ops are cut into slices of whole ops, so phase 4's + // buckets share them (a whole source was one bucket's long pole) and no slice's list of + // lookups grows with the run. MUL and DVRM deduplicate per instance, so each of their + // slices is one instance. The in-walk lookups and MEMW_R are split in the parallel path + // below; the rest stay one collector each. + let branch_segments = branch_ops.segments(); + let bytewise_segments = bytewise_ops.segments(); + // IS_WH's rows, shared by its BITWISE lookups and its table. + let is_wh_rows = is_wh::dedup_is_wh_rows(keccak_ops.parts().flat_map(keccak::is_wh_keys)); + let mut collectors: Vec = Vec::new(); + if p4_sliced() { + // LT's segments slice by slice; a compact one through a temporary of a + // slice's ops at a time. + let lt_slice = p4_slice_len(P4Source::Lt, mul_chunk, dvrm_chunk); + for part in <_ops.parts { + match *part { + Part::Ops(ops) => { + for slice in ops.chunks(lt_slice) { + collectors.push(Box::new(move |h| { + h.add_ops(&collect_bitwise_from_lt_ops(slice)) + })); + } + } + Part::Compact(compact) => { + for start in (0..compact.len()).step_by(lt_slice) { + collectors.push(Box::new(move |h| { + let mut slice = Vec::new(); + compact.expand_into(start, start + lt_slice, &mut slice); + h.add_ops(&collect_bitwise_from_lt_ops(&slice)) + })); + } + } + } + } + // MUL and DVRM deduplicate per instance: their slices are the + // instances of the concatenation (copied only where one straddles two + // blocks). The other sources are sums over their ops: block by block. + let mul_slice = p4_slice_len(P4Source::Mul, mul_chunk, dvrm_chunk); + for k in 0..mul_ops.len().div_ceil(mul_slice) { + let mul_ops = &mul_ops; + collectors.push(Box::new(move |h| { + let slice = mul_ops.range(k * mul_slice, (k + 1) * mul_slice); + h.add_ops(&collect_bitwise_from_mul(&slice, mul_chunk)) + })); + } + let dvrm_slice = p4_slice_len(P4Source::Dvrm, mul_chunk, dvrm_chunk); + for k in 0..dvrm_ops.len().div_ceil(dvrm_slice) { + let dvrm_ops = &dvrm_ops; + collectors.push(Box::new(move |h| { + let slice = dvrm_ops.range(k * dvrm_slice, (k + 1) * dvrm_slice); + h.add_ops(&collect_bitwise_from_dvrm(&slice, dvrm_chunk)) + })); + } + let branch_slice = p4_slice_len(P4Source::Branch, mul_chunk, dvrm_chunk); + for k in 0..branch_ops.len().div_ceil(branch_slice.max(1)) { + let branch_ops = &branch_ops; + collectors.push(Box::new(move |h| { + let slice = branch_ops.range(k * branch_slice, (k + 1) * branch_slice); + h.add_ops(&collect_bitwise_from_branch_ops(&slice)) + })); + } + let shift_slice = p4_slice_len(P4Source::Shift, mul_chunk, dvrm_chunk); + for slice in shift_ops.parts().flat_map(|part| part.chunks(shift_slice)) { + collectors.push(Box::new(move |h| { + h.add_ops(&shift::collect_bitwise_from_shift(slice)) + })); + } + let bytewise_slice = p4_slice_len(P4Source::Bytewise, mul_chunk, dvrm_chunk); + for k in 0..bytewise_ops.len().div_ceil(bytewise_slice.max(1)) { + let bytewise_ops = &bytewise_ops; + collectors.push(Box::new(move |h| { + for op in bytewise_ops.range(k * bytewise_slice, (k + 1) * bytewise_slice) { + h.add_ops(&op.collect_bitwise_ops()); + } + })); + } + let eq_slice = p4_slice_len(P4Source::Eq, mul_chunk, dvrm_chunk); + for k in 0..eq_ops.len().div_ceil(eq_slice.max(1)) { + let eq_ops = &eq_ops; + collectors.push(Box::new(move |h| { + for op in eq_ops.range(k * eq_slice, (k + 1) * eq_slice) { + h.add_ops(&op.collect_bitwise_ops()); + } + })); + } + let store_slice = p4_slice_len(P4Source::Store, mul_chunk, dvrm_chunk); + for slice in store_ops.parts().flat_map(|part| part.chunks(store_slice)) { + collectors.push(Box::new(move |h| { + for op in slice { + h.add_ops(&op.collect_bitwise_ops()); + } + })); + } + for slice in + memw_aligned_ops.chunks(p4_slice_len(P4Source::MemwAligned, mul_chunk, dvrm_chunk)) + { + collectors.push(Box::new(move |h| { + h.add_ops(&collect_bitwise_from_memw_aligned(slice)) + })); + } + let keccak_slice = p4_slice_len(P4Source::Keccak, mul_chunk, dvrm_chunk); + for slice in keccak_ops + .parts() + .flat_map(|part| part.chunks(keccak_slice)) + { + collectors.push(Box::new(move |h| { + h.add_ops(&collect_bitwise_from_keccak(slice)) + })); + } + let ecdas_slice = p4_slice_len(P4Source::Ecdas, mul_chunk, dvrm_chunk); + for slice in ecdas_ops.parts().flat_map(|part| part.chunks(ecdas_slice)) { + collectors.push(Box::new(move |h| { + h.add_ops(&collect_bitwise_from_compact_ecdas(slice)) + })); + } + } else { + // `LAMBDA_VM_P4_SLICED=0`: every source one collector, as before the slices (the A/B's + // control). The same multiplicities: the histogram is a commutative sum. + collectors.extend([ + Box::new(|h: &mut bitwise::BitwiseHistogram| { + for part in <_ops.parts { + match *part { + Part::Ops(ops) => h.add_ops(&collect_bitwise_from_lt_ops(ops)), + Part::Compact(compact) => { + for start in (0..compact.len()).step_by(1 << 20) { + let mut slice = Vec::new(); + compact.expand_into(start, start + (1 << 20), &mut slice); + h.add_ops(&collect_bitwise_from_lt_ops(&slice)); + } + } + } + } + }) as Collector, + // MUL and DVRM deduplicate per instance: instance by instance. + Box::new(|h| { + for k in 0..mul_ops.len().div_ceil(mul_chunk.max(1)) { + let slice = mul_ops.range(k * mul_chunk, (k + 1) * mul_chunk); + h.add_ops(&collect_bitwise_from_mul(&slice, mul_chunk)) + } + }), + Box::new(|h| { + for k in 0..dvrm_ops.len().div_ceil(dvrm_chunk.max(1)) { + let slice = dvrm_ops.range(k * dvrm_chunk, (k + 1) * dvrm_chunk); + h.add_ops(&collect_bitwise_from_dvrm(&slice, dvrm_chunk)) + } + }), + Box::new(|h| { + for k in 0..branch_ops.len().div_ceil(1 << 20) { + let slice = branch_ops.range(k << 20, (k + 1) << 20); + h.add_ops(&collect_bitwise_from_branch_ops(&slice)) + } + }), + Box::new(|h| { + for part in shift_ops.parts() { + h.add_ops(&shift::collect_bitwise_from_shift(part)) + } + }), + Box::new(|h| { + for k in 0..bytewise_ops.len().div_ceil(1 << 20) { + for op in bytewise_ops.range(k << 20, (k + 1) << 20) { + h.add_ops(&op.collect_bitwise_ops()); + } + } + }), + Box::new(|h| { + for k in 0..eq_ops.len().div_ceil(1 << 20) { + for op in eq_ops.range(k << 20, (k + 1) << 20) { + h.add_ops(&op.collect_bitwise_ops()); + } + } + }), + Box::new(|h| { + for op in store_ops.iter() { + h.add_ops(&op.collect_bitwise_ops()); + } + }), + Box::new(|h| h.add_ops(&collect_bitwise_from_memw_aligned(&memw_aligned_ops))), + Box::new(|h| { + for part in keccak_ops.parts() { + h.add_ops(&collect_bitwise_from_keccak(part)) + } + }), + Box::new(|h| { + for part in ecdas_ops.parts() { + h.add_ops(&collect_bitwise_from_compact_ecdas(part)) + } + }), + ]); + } + collectors.extend([ + // Small lists, one block each at a block's size: whole. + Box::new(|h: &mut bitwise::BitwiseHistogram| { + h.add_ops(&collect_bitwise_from_copy(©_ops.whole())) + }) as Collector, + Box::new(|h| { + if !strip_blake3_side_effects() { + h.add_ops(&collect_bitwise_from_blake3( + &blake3_ops.whole(), + &blake3_absorb_ops.whole(), + )); + } + }), + Box::new(|h| h.add_ops(&collect_bitwise_from_ecsm(&ecsm_ops.whole()))), + Box::new(|h| h.add_ops(&collect_bitwise_from_hint(&hint_ops.whole()))), + Box::new(|h| h.add_ops(&collect_bitwise_from_ecrec(&ecrec_ops.whole()))), + Box::new(|h| add_padding_byte_checks(h, num_padding_rows)), + ]); + // The per-row lookups (`types::MU_MAX`): the μ bound every row of a + // deduplicating table sends, padding rows included, and the byte checks SHIFT's + // and KECCAK_RND's padding rows send, over exactly the tables this build + // generates, a collector a table (KECCAK_RND padded as the whole run). + for (start, end) in list_chunks(lt_ops.len(), max_rows.lt) { + let lt_ops = <_ops; + collectors.push(Box::new(move |h| { + h.add_ops(<_row_lookups(lt_ops.range(start, end).iter())) + })); + } + for (start, end) in list_chunks(branch_segments.len(), max_rows.branch) { + let branch_segments = &branch_segments; + collectors.push(Box::new(move |h| { + h.add_ops(&branch_row_lookups(&branch_segments.range(start, end))) + })); + } + for (start, end) in list_chunks(bytewise_segments.len(), max_rows.bytewise) { + let bytewise_segments = &bytewise_segments; + collectors.push(Box::new(move |h| { + h.add_ops(&bytewise_row_lookups(&bytewise_segments.range(start, end))) + })); + } + let shift_padding: usize = list_chunks(shift_ops.len(), max_rows.shift) + .iter() + .map(|&(start, end)| padding_rows_of(end - start)) + .sum::() + + if skip.tails { + skip.shift * padding_rows_of(max_rows.shift) + } else { + 0 + }; + let keccak_rnd_padding = keccak_rnd::ARE_BYTES_PER_ROW + * keccak_rnd_padding_rows(keccak_ops.len(), skip.keccak_rnd_rows, CutPadding::WholeRun); + collectors.push(Box::new(move |h| { + h.bump_n( + BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, 0, 0), + (shift_padding + keccak_rnd_padding) as u64, + ) + })); + collectors.push(Box::new(|h| { + is_wh::collect_bitwise_from_is_wh(&is_wh_rows, h) + })); + if let Some(image) = initial_image + && !l2g_memory_bookend + { + collectors.push(Box::new(move |h| { + collect_bitwise_from_page(image, memory_state, l2g_memory_bookend, h) + })); + } + + let streamed_last_ecall = pre.as_ref().and_then(|pre| pre.last_ecall); + let (mut base, counted_iw, counted_reg) = match pre { + Some(pre) => (pre.histogram, pre.bitwise_ops, pre.memw_register_rows), + None => (bitwise::BitwiseHistogram::new(), 0, 0), + }; + // The in-walk lookups past the counted ones, block by block. + let mut skip_iw = counted_iw; + let uncounted_iw: Vec<&[BitwiseOperation]> = bitwise_ops + .parts() + .filter_map(|part| { + let from = skip_iw.min(part.len()); + skip_iw -= from; + (from < part.len()).then(|| &part[from..]) + }) + .collect(); + let uncounted_reg = &memw_register_rows[counted_reg..]; + + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + // Cap concurrent 80 MiB histograms at `cap` to bound peak memory. The two dominant + // sources — the in-walk lookups and MEMW_R (each tens of millions of items) — are + // split into ~`cap` row-range slices so they parallelize INTERNALLY instead of each + // pinning one core while the rest idle. Every unit (whole collectors + the heavy + // slices) is round-robined into exactly `cap` buckets, one histogram each, so the + // split heavy work is spread across buckets rather than piled into one. + // add_ops/bump/merge form a commutative monoid, so any partition yields + // byte-identical multiplicities (same as the serial fallback below). + let cap = rayon::current_num_threads().clamp(1, 8); + let mut units: Vec = Vec::with_capacity(collectors.len() + 2 * cap); + let iw_total: usize = uncounted_iw.iter().map(|part| part.len()).sum(); + let iw_chunk = iw_total.div_ceil(cap).max(1); + for slice in uncounted_iw.iter().flat_map(|part| part.chunks(iw_chunk)) { + units.push(Box::new(move |h| h.add_ops(slice))); + } + let reg_chunk = uncounted_reg.len().div_ceil(cap).max(1); + for slice in uncounted_reg.chunks(reg_chunk) { + units.push(Box::new(move |h| { + memw_register::collect_bitwise_from_memw_register(slice, h) + })); + } + units.extend(collectors); + + let mut buckets: Vec> = (0..cap).map(|_| Vec::new()).collect(); + for (i, unit) in units.into_iter().enumerate() { + buckets[i % cap].push(unit); + } + if let Some(reduced) = buckets + .par_iter() + .map(|bucket| { + let mut h = bitwise::BitwiseHistogram::new(); + for f in bucket { + f(&mut h); + } + h + }) + .reduce_with(|mut a, b| { + a.merge(&b); + a + }) + { + base.merge(&reduced); + } + } + #[cfg(not(feature = "parallel"))] + { + for part in &uncounted_iw { + base.add_ops(part); + } + memw_register::collect_bitwise_from_memw_register(uncounted_reg, &mut base); + for f in &collectors { + f(&mut base); + } + } + let bitwise_histogram = base; + // The in-walk lookups have been counted into the histogram; free them now. + drop(uncounted_iw); + drop(bitwise_ops); + #[cfg(feature = "instruments")] + drop(__sp); + + // ===================================================================== + // PHASE 5: Generate final traces (parallelized) + build_stamps::mark("p4 bitwise"); + // ===================================================================== + #[cfg(feature = "instruments")] + let __sp = stark::instruments::span("p5_generate_tables"); + + // A monolithic run or the final continuation epoch terminates on the program's + // halt ECALL. Intermediate continuation epochs do not halt, so fall back to the + // last cycle's timestamp and skip HALT-based PC finalization — the PC is carried + // to the next epoch via the register snapshot, and HALT is excluded from the + // proof (`include_halt = false`) so `halt_trace` is unused there. + let (halt_timestamp, halt_next_pc) = if is_final { + let (halt_timestamp, halt_next_pc) = cpu_ops + .iter() + .rev() + .find(|op| decode_table.entry(op.decode_row).fields.ecall) + .map(|op| (op.timestamp, op.next_pc)) + .or(streamed_last_ecall) + .ok_or(Error::MissingHaltEcall)?; + // Finalize the PC (x255) on the REGISTER table. The CPU padding rows carry + // pc=1 and chain the inline-PC `memory` tokens with a +4 timestamp cadence + // starting from the HALT chip's emit_pc at `halt_timestamp + 1`; the last + // padding write therefore lands at `halt_timestamp + 4*num_padding_rows + 1` + // (= `halt_timestamp + 1` when there is no padding). The REGISTER final token + // must match that last write to balance the memory argument. + register_state.write_pc(1, halt_timestamp + 4 * num_padding_rows as u64 + 1); + (halt_timestamp, halt_next_pc) + } else { + (cpu_ops.last().map(|op| op.timestamp).unwrap_or(0), 0) + }; + + let register_final_state = register_state.to_final_state_map(); + + // Each build below reads disjoint op lists and writes its own table, so + // they all run in one rayon scope. Disk-spill stays sequential: its + // generate→spill order keeps trace memory bounded. + let cpu_ops_ref = &cpu_ops; + let pack = skip.pack; + // A packing build writes each table packed as it generates it where the + // generator can (G-pack, `StreamSkip::gpack`); `packed_by` then finds it + // packed. + let gpack = pack && skip.gpack; + let form = if gpack { + TraceForm::Narrow + } else { + TraceForm::Wide + }; + let gen_cpus = || { + chunk_and_generate_skipping( + cpu_ops_ref, + max_rows.cpu, + skip.cpu, + skip.tails, + false, + packed_by(pack, |ops| { + cpu::generate_cpu_trace_as(ops, decode_table, form) + }), + ) + }; + let gen_memws = || { + chunk_and_generate_skipping( + &memw_ops, + max_rows.memw, + skip.memw, + skip.tails, + true, + packed_by(pack, |ops| memw::generate_memw_trace_as(ops, form)), + ) + }; + let gen_memw_aligneds = || { + chunk_and_generate_skipping( + &memw_aligned_ops, + max_rows.memw_aligned, + skip.memw_aligned, + skip.tails, + true, + packed_by(pack, |ops| { + memw_aligned::generate_memw_aligned_trace_as(ops, form) + }), + ) + }; + let gen_memw_registers = || { + // Direct-to-column fill from compact RegRows — the register fast path never + // materializes a `Vec`. + chunk_and_generate_skipping( + &memw_register_rows, + max_rows.memw_register, + skip.memw_register, + skip.tails, + false, + packed_by(pack, |ops| { + memw_register::generate_memw_register_trace_from_rows_as(ops, form) + }), + ) + }; + let gen_loads = || { + chunk_and_generate_skipping( + &load_ops, + max_rows.load, + skip.load, + skip.tails, + true, + packed_by(pack, |ops| load::generate_load_trace_as(ops, form)), + ) + }; + let gen_lts = || { + chunk_and_generate_segmented( + <_ops, + max_rows.lt, + skip.lt, + skip.tails, + true, + packed_by(pack, |ops| lt::generate_lt_trace_as(ops, form)), + ) + }; + let gen_shifts = || { + chunk_and_generate_segmented( + &shift_ops.segments(), + max_rows.shift, + skip.shift, + skip.tails, + true, + packed_by(pack, |ops| shift::generate_shift_trace_as(ops, form)), + ) + }; + let gen_muls = || { + chunk_and_generate_segmented( + &mul_ops.segments(), + max_rows.mul, + 0, + false, + true, + packed_by(pack, |ops| mul::generate_mul_trace_as(ops, form)), + ) + }; + let gen_dvrms = || { + chunk_and_generate_segmented( + &dvrm_ops.segments(), + max_rows.dvrm, + 0, + false, + true, + packed_by(pack, |ops| dvrm::generate_dvrm_trace_as(ops, form)), + ) + }; + let gen_branches = || { + chunk_and_generate_segmented( + &branch_ops.segments(), + max_rows.branch, + 0, + false, + true, + packed_by(pack, |ops| branch::generate_branch_trace_as(ops, form)), + ) + }; + // Auxiliary ALU / memory / CPU32 dispatch chips, each filtered out of the CPU + // ops above. + let gen_eqs = || { + chunk_and_generate_segmented::( + &eq_ops.segments(), + max_rows.eq, + 0, + false, + true, + packed_by(pack, |ops| eq::generate_eq_trace_as(ops, form)), + ) + }; + let gen_bytewises = || { + chunk_and_generate_segmented::( + &bytewise_ops.segments(), + max_rows.bytewise, + 0, + false, + true, + packed_by(pack, |ops| bytewise::generate_bytewise_trace_as(ops, form)), + ) + }; + let gen_stores = || { + chunk_and_generate_segmented( + &store_ops.segments(), + max_rows.store, + skip.store, + skip.tails, + true, + packed_by(pack, |ops| store::generate_store_trace_as(ops, form)), + ) + }; + let gen_cpu32s = || { + chunk_and_generate_segmented::( + &cpu32_ops.segments(), + max_rows.cpu32, + 0, + false, + true, + packed_by(pack, |ops| cpu32::generate_cpu32_trace_as(ops, form)), + ) + }; + let gen_bitwise = || { + let mut bitwise = bitwise::generate_bitwise_trace(); + // Fill the MU columns (11..=20) from the accumulated histogram. + bitwise_histogram.fill_multiplicities(&mut bitwise); + packed_if(pack, bitwise) + }; + // Each CPU operation looks up the DECODE table once; padding rows look up + // pc=1 (the CPU padding entry). When CPU is split, each chunk pads + // independently. + let gen_decode = move || { + let mut decode = decode_trace; + let rows = cpu_ops_ref.iter().map(|op| op.decode_row as usize); + let padding = std::iter::repeat_n(decode_table.padding_row(), num_padding_rows); + decode::count_rows(&mut decode, rows.chain(padding)); + packed_if(pack, decode) + }; + let gen_copies = || { + if skip.copy_rows > 0 { + return cut_tables( + ©_ops, + skip.copy_rows, + CutPadding::WholeRun, + pack, + form, + copy::rows_written_packed, + copy::generate_copy_rows_as, + ); + } + generate_optional(©_ops.whole(), copy::generate_copy_trace) + }; + let gen_keccaks = || { + if skip.keccak_rows > 0 { + return cut_tables( + &keccak_ops, + skip.keccak_rows, + CutPadding::WholeRun, + pack, + form, + keccak::rows_written_packed, + keccak::generate_keccak_rows_as, + ); + } + generate_optional(&keccak_ops.whole(), keccak::generate_keccak_trace) + }; + let gen_keccak_rnds = || { + let keccak_rnd_ops: Vec = keccak_ops + .iter() + .map(|op| KeccakRoundOperation { + timestamp: op.timestamp, + input: op.input, + output: op.output, + }) + .collect(); + if skip.keccak_rnd_rows == 0 { + // Whole, it may be cut after the build: it stays wide. + return generate_optional(&keccak_rnd_ops, keccak_rnd::generate_keccak_rnd_trace); + } + keccak_rnd_chunks( + &keccak_rnd_ops, + skip.keccak_rnd_rows, + skip.keccak_rnd, + CutPadding::WholeRun, + pack, + form, + ) + }; + let num_blake3_ops = blake3_ops.len() + blake3_absorb_ops.len(); + let gen_blake3 = || { + packed_if( + pack, + blake3::generate_blake3_trace(&blake3_ops.whole(), &blake3_absorb_ops.whole()), + ) + }; + let gen_keccak_rc = || { + let mut keccak_rc_trace = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut keccak_rc_trace, keccak_ops.len()); + packed_if(pack, keccak_rc_trace) + }; + let gen_pages = || match initial_image { + // Continuation epochs (l2g_memory_bookend) skip PAGE: the L2G table owns + // every touched cell's Memory init/fini, and every untouched PAGE row + // self-cancels (init==fini, ts=0), so PAGE contributes nothing here. + Some(image) if !l2g_memory_bookend => { + let (pages, configs) = + generate_page_tables(image, memory_state, private_input, l2g_memory_bookend); + ( + pages + .into_iter() + .map(|page| packed_if(pack, page)) + .collect(), + configs, + ) + } + _ => (Vec::new(), Vec::new()), + }; + let gen_register = || { + packed_if( + pack, + register::generate_register_trace(®ister_final_state, register_init), + ) + }; + let gen_halt = || { + packed_if( + pack, + halt::generate_halt_trace(halt_timestamp, halt_next_pc), + ) + }; + // ECSM accelerator traces. A program that does not use ECSM carries no ECSM + // and no ECDAS table at all — not a padded one. + let gen_ecsms = || { + if skip.ecsm_rows > 0 { + return cut_tables( + &ecsm_ops, + skip.ecsm_rows, + CutPadding::WholeRun, + pack, + form, + ecsm::rows_written_packed, + ecsm::generate_ecsm_rows_as, + ); + } + generate_optional(&ecsm_ops.whole(), ecsm::generate_ecsm_trace) + }; + let gen_ecdases = || { + if skip.ecdas_rows > 0 { + return cut_tables( + &ecdas_ops, + skip.ecdas_rows, + CutPadding::WholeRun, + pack, + form, + ecdas::rows_written_packed, + |rows, num_rows, form| { + ecdas::generate_ecdas_rows_of(rows, num_rows, form, ecdas::widen) + }, + ); + } + generate_optional(&ecdas_ops.whole(), |rows| { + let num_rows = rows.len().next_power_of_two().max(4); + ecdas::generate_ecdas_rows_of(rows, num_rows, TraceForm::Wide, ecdas::widen) + }) + }; + // HINT table. Absent entirely for programs that make no hint ecalls. + let gen_hints = || { + generate_optional( + &hint_ops.whole(), + packed_by(pack, |ops| hint::generate_hint_trace_as(ops, form)), + ) + }; + let gen_ecrecs = || { + generate_optional( + &ecrec_ops.whole(), + packed_by(pack, |ops| ecrec::generate_ecrec_trace_as(ops, form)), + ) + }; + + let (mut cpus_slot, mut memws_slot, mut memw_aligneds_slot, mut memw_registers_slot) = + (None, None, None, None); + let (mut loads_slot, mut lts_slot, mut shifts_slot, mut muls_slot) = (None, None, None, None); + let (mut dvrms_slot, mut branches_slot, mut bitwise_slot, mut decode_slot) = + (None, None, None, None); + let (mut copies_slot, mut keccaks_slot, mut keccak_rnds_slot, mut keccak_rc_slot) = + (None, None, None, None); + let mut blake3_slot = None; + let (mut pages_slot, mut register_slot, mut halt_slot) = (None, None, None); + let (mut eqs_slot, mut bytewises_slot, mut stores_slot, mut cpu32s_slot) = + (None, None, None, None); + let (mut ecsms_slot, mut ecdases_slot) = (None, None); + let mut hints_slot = None; + let mut ecrecs_slot = None; + + let sequential = cfg!(not(feature = "parallel")); + + if !sequential { + #[cfg(feature = "parallel")] + rayon::scope(|s| { + macro_rules! spawn_into { + ($slot:ident, $gen:ident) => {{ + let slot = &mut $slot; + s.spawn(move |_| { + *slot = Some($gen()); + build_stamps::mark(concat!("p5 ", stringify!($gen))); + }); + }}; + } + // Heaviest builds first so the scheduler overlaps them with the rest. + spawn_into!(memw_registers_slot, gen_memw_registers); + spawn_into!(cpus_slot, gen_cpus); + spawn_into!(memws_slot, gen_memws); + spawn_into!(lts_slot, gen_lts); + spawn_into!(decode_slot, gen_decode); + spawn_into!(branches_slot, gen_branches); + spawn_into!(bitwise_slot, gen_bitwise); + spawn_into!(muls_slot, gen_muls); + spawn_into!(memw_aligneds_slot, gen_memw_aligneds); + spawn_into!(loads_slot, gen_loads); + spawn_into!(shifts_slot, gen_shifts); + spawn_into!(dvrms_slot, gen_dvrms); + spawn_into!(pages_slot, gen_pages); + spawn_into!(keccaks_slot, gen_keccaks); + spawn_into!(keccak_rnds_slot, gen_keccak_rnds); + spawn_into!(keccak_rc_slot, gen_keccak_rc); + spawn_into!(blake3_slot, gen_blake3); + spawn_into!(copies_slot, gen_copies); + spawn_into!(register_slot, gen_register); + spawn_into!(halt_slot, gen_halt); + spawn_into!(eqs_slot, gen_eqs); + spawn_into!(bytewises_slot, gen_bytewises); + spawn_into!(stores_slot, gen_stores); + spawn_into!(cpu32s_slot, gen_cpu32s); + spawn_into!(ecsms_slot, gen_ecsms); + spawn_into!(ecdases_slot, gen_ecdases); + spawn_into!(hints_slot, gen_hints); + spawn_into!(ecrecs_slot, gen_ecrecs); + }); + } else { + cpus_slot = Some(gen_cpus()); + memws_slot = Some(gen_memws()); + memw_aligneds_slot = Some(gen_memw_aligneds()); + memw_registers_slot = Some(gen_memw_registers()); + loads_slot = Some(gen_loads()); + lts_slot = Some(gen_lts()); + shifts_slot = Some(gen_shifts()); + muls_slot = Some(gen_muls()); + dvrms_slot = Some(gen_dvrms()); + branches_slot = Some(gen_branches()); + bitwise_slot = Some(gen_bitwise()); + decode_slot = Some(gen_decode()); + copies_slot = Some(gen_copies()); + keccaks_slot = Some(gen_keccaks()); + keccak_rnds_slot = Some(gen_keccak_rnds()); + keccak_rc_slot = Some(gen_keccak_rc()); + blake3_slot = Some(gen_blake3()); + pages_slot = Some(gen_pages()); + register_slot = Some(gen_register()); + halt_slot = Some(gen_halt()); + eqs_slot = Some(gen_eqs()); + bytewises_slot = Some(gen_bytewises()); + stores_slot = Some(gen_stores()); + cpu32s_slot = Some(gen_cpu32s()); + ecsms_slot = Some(gen_ecsms()); + ecdases_slot = Some(gen_ecdases()); + hints_slot = Some(gen_hints()); + ecrecs_slot = Some(gen_ecrecs()); + } + + build_stamps::mark("p5 end"); + const PHASE5_RAN: &str = "phase 5 generation ran in one of the branches above"; + let cpus = cpus_slot.expect(PHASE5_RAN)?; + let memws = memws_slot.expect(PHASE5_RAN)?; + let memw_aligneds = memw_aligneds_slot.expect(PHASE5_RAN)?; + let memw_registers = memw_registers_slot.expect(PHASE5_RAN)?; + let loads = loads_slot.expect(PHASE5_RAN)?; + let lts = lts_slot.expect(PHASE5_RAN)?; + let shifts = shifts_slot.expect(PHASE5_RAN)?; + let muls = muls_slot.expect(PHASE5_RAN)?; + let dvrms = dvrms_slot.expect(PHASE5_RAN)?; + let branches = branches_slot.expect(PHASE5_RAN)?; + let eqs = eqs_slot.expect(PHASE5_RAN)?; + let bytewises = bytewises_slot.expect(PHASE5_RAN)?; + let stores = stores_slot.expect(PHASE5_RAN)?; + let cpu32s = cpu32s_slot.expect(PHASE5_RAN)?; + #[allow(unused_mut)] + let mut bitwise = bitwise_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let mut decode = decode_slot.expect(PHASE5_RAN); + let copies = copies_slot.expect(PHASE5_RAN)?; + let keccaks = keccaks_slot.expect(PHASE5_RAN)?; + let keccak_rnds = keccak_rnds_slot.expect(PHASE5_RAN)?; + let keccak_rc_trace = keccak_rc_slot.expect(PHASE5_RAN); + let blake3_trace = blake3_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let (mut pages, page_configs) = pages_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let mut register_trace = register_slot.expect(PHASE5_RAN); + #[allow(unused_mut)] + let mut halt_trace = halt_slot.expect(PHASE5_RAN); + let ecsms = ecsms_slot.expect(PHASE5_RAN)?; + let ecdases = ecdases_slot.expect(PHASE5_RAN)?; + let hints = hints_slot.expect(PHASE5_RAN)?; + let ecrecs = ecrecs_slot.expect(PHASE5_RAN)?; + let is_whs = generate_optional(&is_wh_rows, packed_by(pack, is_wh::generate_is_wh_trace))?; + + // Continuation callers derive the real cross-epoch boundary from this set and + // install its L2G trace after provenance is applied. Avoid building a + // throwaway genesis-only L2G trace here. + let touched_memory_cells = if l2g_memory_bookend { + touched_cells_from_memory_state(memory_state) + } else { + Vec::new() + }; + let local_to_global = local_to_global::generate_local_to_global_trace(&[]); + + #[cfg(feature = "instruments")] + drop(__sp); + Ok(Traces { + cpus, + bitwise, + lts, + shifts, + memws, + memw_aligneds, + loads, + decode, + muls, + dvrms, + pages, + page_configs, + register: register_trace, + public_output_bytes, + branches, + halt: halt_trace, + copies, + keccaks, + keccak_rnds, + blake3: blake3_trace, + num_blake3_ops, + keccak_rc: keccak_rc_trace, + ecsms, + ecdases, + hints, + ecrecs, + is_whs, + memw_registers, + local_to_global, + touched_memory_cells, + eqs, + bytewises, + stores, + cpu32s, + }) +} diff --git a/prover/src/tables/trace_builder/finish_tests.rs b/prover/src/tables/trace_builder/finish_tests.rs new file mode 100644 index 000000000..e7cc7444c --- /dev/null +++ b/prover/src/tables/trace_builder/finish_tests.rs @@ -0,0 +1,820 @@ +//! ★ The finish's plan ([`super::finish`]) builds the tables the frozen table +//! phase (`finish_oracle`) built, word for word, on every path that reaches +//! it: whole-run builds, non-final epochs with and without the L2G bookend, +//! and windowed builds (streamed ops dropped, KECCAK_RND chunked or cut at the +//! finish, KECCAK / ECSM / ECDAS cut, tables packed or written packed, the +//! derived LT ops raw or compact, LT concatenated or kept as segments). Its +//! header is the oracle's counts and pages before any table exists, and a +//! header that miscounts is refused. With the cut tables stopped at the last +//! real row ([`CutPadding::LastTable`]) the plan builds the oracle's tables +//! without the ones of padding rows alone, and every other table as it is. + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; + +use super::cut_padding_tests::is_today_without_its_padding_tables; +use super::finish::{FinishPlan, counts_of}; +use super::*; +use crate::tables::MaxRowsConfig; +use crate::test_utils::asm_elf_bytes; + +type Table = TraceTable; + +pub(super) fn run(name: &str) -> (Elf, Vec) { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + (program, logs) +} + +/// A table as (columns, rows, words row-major), held packed or wide. +fn shape_and_words(t: &Table) -> (usize, usize, Vec) { + match t.narrow_main() { + Some(packed) => { + let columns: Vec> = (0..packed.cols()).map(|c| packed.column(c)).collect(); + let words = (0..packed.rows()) + .flat_map(|r| columns.iter().map(move |column| column[r])) + .collect(); + (packed.cols(), packed.rows(), words) + } + None => { + let main = &t.main_table; + let words = (0..main.height) + .flat_map(|r| (0..main.width).map(move |c| *main.get(r, c).value())) + .collect(); + (main.width, main.height, words) + } + } +} + +/// Every table of a build, named, and what the statement reads off it. +fn contents(t: &Traces) -> Vec<(String, String)> { + let lists: [(&str, &[Table]); 21] = [ + ("CPU", &t.cpus), + ("MEMW_R", &t.memw_registers), + ("MEMW_A", &t.memw_aligneds), + ("MEMW", &t.memws), + ("LOAD", &t.loads), + ("STORE", &t.stores), + ("SHIFT", &t.shifts), + ("CPU32", &t.cpu32s), + ("COPY", &t.copies), + ("KECCAK", &t.keccaks), + ("KECCAK_RND", &t.keccak_rnds), + ("ECSM", &t.ecsms), + ("ECDAS", &t.ecdases), + ("HINT", &t.hints), + ("PAGE", &t.pages), + ("LT", &t.lts), + ("MUL", &t.muls), + ("DVRM", &t.dvrms), + ("BRANCH", &t.branches), + ("EQ", &t.eqs), + ("BYTEWISE", &t.bytewises), + ]; + let mut out = Vec::new(); + for (name, list) in lists { + for (i, table) in list.iter().enumerate() { + let (cols, rows, words) = shape_and_words(table); + out.push(( + format!("{name}[{i}]"), + format!("{cols}x{rows} {:x}", fold(&words)), + )); + } + out.push((format!("{name} count"), list.len().to_string())); + } + for (name, table) in [ + ("BITWISE", &t.bitwise), + ("DECODE", &t.decode), + ("REGISTER", &t.register), + ("HALT", &t.halt), + ("KECCAK_RC", &t.keccak_rc), + ("BLAKE3", &t.blake3), + ("L2G", &t.local_to_global), + ] { + let (cols, rows, words) = shape_and_words(table); + out.push(( + name.to_string(), + format!("{cols}x{rows} {:x}", fold(&words)), + )); + } + out.push(( + "public output".into(), + format!("{:?}", t.public_output_bytes), + )); + out.push(("blake3 ops".into(), t.num_blake3_ops.to_string())); + out.push(("counts".into(), format!("{:?}", t.table_counts()))); + out.push(( + "runtime pages".into(), + format!("{:?}", t.runtime_page_ranges()), + )); + out.push(( + "page configs".into(), + format!( + "{:?}", + t.page_configs + .iter() + .map(|c| (c.page_base, c.init_values.is_some(), c.is_private_input)) + .collect::>() + ), + )); + out.push(( + "touched cells".into(), + format!("{:?}", t.touched_memory_cells), + )); + out +} + +/// An order-sensitive digest of a table's words (a mismatch names the table). +fn fold(words: &[u64]) -> u64 { + words.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, &w| { + (h ^ w).wrapping_mul(0x0000_0100_0000_01b3).rotate_left(5) + }) +} + +/// `build` run as the oracle, which must have run. +fn oracle(build: impl FnOnce() -> T) -> T { + let before = finish_oracle::runs(); + let traces = finish_oracle::with(build); + assert!(finish_oracle::runs() > before, "the oracle did not run"); + traces +} + +fn same(what: &str, oracle: &Traces, plan: &Traces) { + let (a, b) = (contents(oracle), contents(plan)); + assert_eq!(a.len(), b.len(), "{what}: the builds hold different tables"); + for ((name, x), (_, y)) in a.iter().zip(&b) { + assert_eq!(x, y, "{what}: {name}"); + } +} + +/// A windowed build over windows of `window` cycles, `configure` applied, its +/// streamed chunks put back. +pub(super) fn windowed( + program: &Elf, + logs: &[Log], + max_rows: &MaxRowsConfig, + window: usize, + configure: &dyn Fn(WindowedTraceBuilder<'_>) -> WindowedTraceBuilder<'_>, +) -> Traces { + let mut builder = + configure(WindowedTraceBuilder::new(program, &[], max_rows).expect("the builder")); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend(builder.push(w).expect("a window")); + } + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + traces +} + +const PROGRAMS: [&str; 15] = [ + "sub", + "all_instructions_64", + "all_loadstore_32", + "misalign_sd", + "test_keccak", + "test_keccak_multi", + "test_keccak256", + "test_ecsm", + "test_ecsm_multi", + "test_ecsm_split", + "test_commit_split", + "test_blake3", + "test_blake3_absorb", + "test_dense_pages", + "test_ecrec", +]; + +/// Whole-run builds, at small chunks (every chunked table split) and at the +/// production sizes: the plan's tables are the oracle's. +#[test] +fn the_plan_builds_the_oracles_whole_run_tables() { + for max_rows in [ + MaxRowsConfig::small(), + MaxRowsConfig::uniform(4), + MaxRowsConfig::default(), + ] { + for name in PROGRAMS { + let (program, logs) = run(name); + let build = + || Traces::from_elf_and_logs(&program, &logs, &max_rows, &[]).expect("the build"); + let frozen = oracle(build); + same(&format!("{name} whole run"), &frozen, &build()); + } + } +} + +/// A run cut into two epochs, the first non-final, with and without the L2G +/// bookend (no PAGE, touched cells): the plan's tables are the oracle's, and +/// so is the final epoch's. +#[test] +fn the_plan_builds_the_oracles_epoch_tables() { + let max_rows = MaxRowsConfig::small(); + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm_multi", + ] { + let (program, logs) = run(name); + let artifacts = DecodeArtifacts::from_elf(&program).expect("the artifacts"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + let half = logs.len() / 2; + for l2g in [false, true] { + for (part, is_final) in [(&logs[..half], false), (&logs[..], true)] { + let build = || { + Traces::from_image_and_logs_with_decode( + &artifacts, + &image, + ®ister_init, + part, + &max_rows, + &[], + is_final, + l2g, + ) + .expect("the build") + }; + let frozen = oracle(build); + same( + &format!("{name} epoch final {is_final} l2g {l2g}"), + &frozen, + &build(), + ); + } + } + } +} + +/// Windowed builds in every configuration the block uses (and their A arms): +/// the plan's tables are the oracle's, packed or not. The configurations that +/// cut pad the whole run, as the oracle does whatever the settings say (cut at +/// the last real row: +/// [`a_build_cut_at_the_last_real_row_is_the_oracles_without_its_padding_tables`]). +#[test] +fn the_plan_builds_the_oracles_windowed_tables() { + type Configure = Box) -> WindowedTraceBuilder<'_>>; + let configs: Vec<(&str, Configure)> = vec![ + ("plain", Box::new(|b| b)), + ( + "dropped", + Box::new(|b| b.drop_streamed_ops().expect("before any window")), + ), + ( + "dropped, raw + concatenated LT", + Box::new(|b| { + b.drop_streamed_ops() + .expect("before any window") + .raw_memw_lt() + .expect("before any window") + .concat_lt() + }), + ), + ( + "dropped, KECCAK_RND at the finish, cuts, packed, written packed", + Box::new(|b| { + b.keccak_rnd_chunks_at_finish(32) + .expect("rows") + .cuts_at_finish(2, 4, 2, 2) + .expect("rows") + .cut_padding(CutPadding::WholeRun) + .pack_finished_tables() + .generate_packed() + .drop_streamed_ops() + .expect("before any window") + }), + ), + ( + "KECCAK_RND streamed, packed", + Box::new(|b| { + b.keccak_rnd_chunks(32) + .expect("rows") + .cut_padding(CutPadding::WholeRun) + .pack_finished_tables() + .drop_streamed_ops() + .expect("before any window") + }), + ), + ( + "MEMW-derived LT streamed", + Box::new(|b| { + b.stream_memw_lt() + .drop_streamed_ops() + .expect("before any window") + }), + ), + ]; + for max_rows in [MaxRowsConfig::small(), MaxRowsConfig::uniform(4)] { + for name in [ + "all_instructions_64", + "misalign_sd", + "test_keccak_multi", + "test_ecsm_multi", + "test_commit_split", + "test_blake3_absorb", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs { + let build = || windowed(&program, &logs, &max_rows, window, configure.as_ref()); + let frozen = oracle(build); + same( + &format!("{name} window {window} {config}"), + &frozen, + &build(), + ); + } + } + } + } +} + +/// The delta-coded lists ([`super::delta`]) marked every three ops instead of +/// every 4096, so nearly every chunk and every p4 slice starts past an index +/// mark: the windowed builds make the same tables word for word. +#[test] +fn the_tables_are_the_same_with_an_index_mark_every_three_ops() { + type Configure = Box) -> WindowedTraceBuilder<'_>>; + let configs: Vec<(&str, Configure)> = vec![ + ("plain", Box::new(|b| b)), + ( + "dropped, KECCAK_RND at the finish, cuts, packed, written packed", + Box::new(|b| { + b.keccak_rnd_chunks_at_finish(32) + .expect("rows") + .cuts_at_finish(2, 4, 2, 2) + .expect("rows") + .pack_finished_tables() + .generate_packed() + .drop_streamed_ops() + .expect("before any window") + }), + ), + ]; + for max_rows in [MaxRowsConfig::small(), MaxRowsConfig::uniform(4)] { + for name in [ + "all_instructions_64", + "misalign_sd", + "test_keccak_multi", + "test_ecsm_multi", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs { + let build = || windowed(&program, &logs, &max_rows, window, configure.as_ref()); + let dense = super::delta::with_index_every(3, build); + same( + &format!("{name} window {window} {config}, marks every 3"), + &build(), + &dense, + ); + } + } + } + } +} + +/// A plan over a collected run, as `build_from_collected` makes one. +fn plan_of<'a>( + ops: CollectedOps, + memory_state: &'a MemoryState, + register_state: RegisterState, + artifacts: &'a DecodeArtifacts, + image: &'a HashMap, + register_init: &'a [u32], + max_rows: &'a MaxRowsConfig, +) -> FinishPlan<'a> { + FinishPlan::new( + ops, + Some(image), + memory_state, + register_init, + artifacts.decode_trace.clone(), + &artifacts.decode, + register_state, + max_rows, + &[], + true, + false, + &StreamSkip::default(), + None, + ) + .expect("the plan") +} + +/// The header is the oracle's counts, pages, public output and BLAKE3 count +/// before any table is built; a header that miscounts any one family by one +/// is refused when the tables are built, not proved. +#[test] +fn the_header_is_the_tables_counts_and_a_wrong_one_is_refused() { + let max_rows = MaxRowsConfig::small(); + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm_multi", + "test_blake3", + ] { + let (program, logs) = run(name); + let artifacts = DecodeArtifacts::from_elf(&program).expect("the artifacts"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + let frozen = oracle(|| { + Traces::from_elf_and_logs(&program, &logs, &max_rows, &[]).expect("the build") + }); + let collect = || { + Traces::collect_epoch(&artifacts, &image, ®ister_init, &logs, true) + .expect("collected") + }; + let CollectedEpoch { + ops, + memory_state, + register_state, + } = collect(); + let plan = plan_of( + ops, + &memory_state, + register_state, + &artifacts, + &image, + ®ister_init, + &max_rows, + ); + let header = plan.header(); + assert_eq!( + counts_of(&header.table_counts), + counts_of(&frozen.table_counts()), + "{name}: counts" + ); + assert_eq!( + format!("{:?}", header.runtime_page_ranges()), + format!("{:?}", frozen.runtime_page_ranges()), + "{name}: runtime pages" + ); + assert_eq!( + header.page_configs.len(), + frozen.page_configs.len(), + "{name}: pages" + ); + assert_eq!( + header.public_output_bytes, frozen.public_output_bytes, + "{name}" + ); + assert_eq!(header.num_blake3_ops, frozen.num_blake3_ops, "{name}"); + same( + &format!("{name} planned"), + &frozen, + &plan.emit_all().expect("the tables"), + ); + // Each family miscounted by one, up or down (where it has a table). + for field in 0..22 { + for up in [true, false] { + let CollectedEpoch { + ops, + memory_state, + register_state, + } = collect(); + let plan = plan_of( + ops, + &memory_state, + register_state, + &artifacts, + &image, + ®ister_init, + &max_rows, + ); + let mut wrong = plan.header().table_counts.clone(); + let slot: &mut usize = match field { + 0 => &mut wrong.cpu, + 1 => &mut wrong.lt, + 2 => &mut wrong.memw, + 3 => &mut wrong.memw_aligned, + 4 => &mut wrong.load, + 5 => &mut wrong.mul, + 6 => &mut wrong.dvrm, + 7 => &mut wrong.shift, + 8 => &mut wrong.branch, + 9 => &mut wrong.memw_register, + 10 => &mut wrong.eq, + 11 => &mut wrong.bytewise, + 12 => &mut wrong.store, + 13 => &mut wrong.cpu32, + 14 => &mut wrong.keccak, + 15 => &mut wrong.keccak_rnd, + 16 => &mut wrong.ecsm, + 17 => &mut wrong.ecdas, + 18 => &mut wrong.hint, + 19 => &mut wrong.copy, + 20 => &mut wrong.blake3, + _ => &mut wrong.is_wh, + }; + if !up && *slot == 0 { + continue; + } + *slot = if up { *slot + 1 } else { *slot - 1 }; + assert!( + matches!(plan.with_counts(wrong).emit_all(), Err(Error::Prover(_))), + "{name}: count {field} off by one ({up}) was not refused" + ); + } + } + } +} + +/// The four families a block cuts at a fixed number of rows: each one's +/// tables in a build and its bus interactions. +fn cut_families(t: &Traces) -> [(&'static str, &[Table], Vec); 5] { + [ + ("COPY", &t.copies, copy::bus_interactions()), + ("KECCAK", &t.keccaks, keccak::bus_interactions()), + ("KECCAK_RND", &t.keccak_rnds, keccak_rnd::bus_interactions()), + ("ECSM", &t.ecsms, ecsm::bus_interactions()), + ("ECDAS", &t.ecdases, ecdas::bus_interactions()), + ] +} + +/// [`contents`] without the five cut families' tables and counts. +fn contents_but_the_cut_families(t: &Traces) -> Vec<(String, String)> { + let cut = ["COPY", "KECCAK", "KECCAK_RND", "ECSM", "ECDAS"]; + contents(t) + .into_iter() + .filter(|(name, _)| { + name != "counts" + && !cut.iter().any(|family| { + *name == format!("{family} count") || name.starts_with(&format!("{family}[")) + }) + }) + .collect() +} + +/// The block's windowed build with every family cut: KECCAK_RND at 32 rows, +/// KECCAK, ECSM, ECDAS and COPY at `cuts`, wide (`packed` false) or packed, written +/// packed and with the streamed ops dropped, the cut tables padded as +/// `padding`. +fn cut_build( + program: &Elf, + logs: &[Log], + window: usize, + cuts: (usize, usize, usize, usize), + packed: bool, + padding: CutPadding, +) -> Traces { + windowed(program, logs, &MaxRowsConfig::small(), window, &|b| { + let b = b + .keccak_rnd_chunks_at_finish(32) + .expect("rows") + .cuts_at_finish(cuts.0, cuts.1, cuts.2, cuts.3) + .expect("rows") + .cut_padding(padding); + if packed { + b.pack_finished_tables() + .generate_packed() + .drop_streamed_ops() + .expect("before any window") + } else { + b + } + }) +} + +/// ★ Cut at the last real row, a windowed build is the frozen build's — which +/// pads the whole run, whatever the settings say — without the cut families' +/// tables of padding rows alone: each family's tables are the frozen build's +/// first ones word for word, every table left out has all its bus +/// multiplicities at zero but its padding byte checks, the last one kept holds +/// a real row, and every other table of the build (KECCAK_RC's among them) is +/// the frozen build's. BITWISE's counts are too, but one: KECCAK_RND's θ/ρ byte +/// checks have multiplicity 1 (main's multiplicity fix), so each padding row +/// sends them as `ARE_BYTES[0, 0]`, and the frozen build's BITWISE answers the +/// left-out tables' rows as well — `ARE_BYTES_PER_ROW` a row, in that one cell. +#[test] +fn a_build_cut_at_the_last_real_row_is_the_oracles_without_its_padding_tables() { + let mut left_out = [0usize; 5]; + for name in [ + "all_instructions_64", + "test_keccak", + "test_keccak_multi", + "test_ecsm", + "test_ecsm_multi", + "test_ecsm_split", + "test_copy_ecalls", + ] { + let (program, logs) = run(name); + for cuts in [(1, 1, 1, 1), (2, 2, 16, 4), (2, 4, 2, 2)] { + for (window, packed) in [(7, true), (1000, true), (1000, false)] { + let what = format!("{name} cuts {cuts:?} window {window} packed {packed}"); + let build = + || cut_build(&program, &logs, window, cuts, packed, CutPadding::LastTable); + let frozen = oracle(build); + let last = build(); + let mut keccak_rnd_rows_left_out = 0usize; + for (f, ((family, kept, interactions), (_, today, _))) in cut_families(&last) + .into_iter() + .zip(cut_families(&frozen)) + .enumerate() + { + is_today_without_its_padding_tables(kept, today, &interactions) + .unwrap_or_else(|e| panic!("{what}: {family}: {e}")); + left_out[f] += today.len() - kept.len(); + if family == "KECCAK_RND" { + keccak_rnd_rows_left_out += today[kept.len()..] + .iter() + .map(|t| t.num_rows()) + .sum::(); + } + } + let (a, b) = ( + contents_but_the_cut_families(&frozen), + contents_but_the_cut_families(&last), + ); + assert_eq!(a.len(), b.len(), "{what}: the other tables"); + for ((table, x), (_, y)) in a.iter().zip(&b) { + if table != "BITWISE" { + assert_eq!(x, y, "{what}: {table}"); + } + } + let ((cols, rows, frozen_words), (_, _, words)) = ( + shape_and_words(&frozen.bitwise), + shape_and_words(&last.bitwise), + ); + assert_eq!( + (rows, frozen_words.len()), + (rows, words.len()), + "{what}: BITWISE" + ); + let cell = bitwise::row_index(0, 0, 0) * cols + bitwise::cols::MU_ARE_BYTES; + for (i, (x, y)) in frozen_words.iter().zip(&words).enumerate() { + if i == cell { + assert_eq!( + *x - *y, + (keccak_rnd::ARE_BYTES_PER_ROW * keccak_rnd_rows_left_out) as u64, + "{what}: BITWISE's ARE_BYTES[0, 0]" + ); + } else { + assert_eq!(x, y, "{what}: BITWISE word {i}"); + } + } + } + } + } + assert!( + left_out.iter().all(|&n| n > 0), + "each cut family left a table of padding rows out somewhere: {left_out:?}" + ); +} + +/// The cut families' tables of a build as (family, tables, rows of each). +fn cut_shapes(t: &Traces) -> Vec<(&'static str, usize, Vec)> { + cut_families(t) + .into_iter() + .map(|(family, tables, _)| { + ( + family, + tables.len(), + tables + .iter() + .map(|table| shape_and_words(table).1) + .collect(), + ) + }) + .collect() +} + +/// ★ The default stops at the last real row, and the knob at `run` builds +/// exactly the tables the block always built before, on two pinned programs: +/// the type's default and what an unset variable reads as keep the tables +/// with a real row; `run` is the whole run padded, table for table as listed +/// here and word for word the frozen build's. test_keccak_multi is 3 +/// permutations: 3 KECCAK rows (4 padded) and 72 KECCAK_RND rows (128 +/// padded), and its 200-byte `write` is 25 COPY rows (32 padded). +/// test_ecsm_multi is 3 scalar multiplications: 3 ECSM rows (4 padded) and 42 +/// ECDAS steps (64 padded), and its three 32-byte `write`s are 12 COPY rows +/// (16 padded). +#[test] +fn the_knob_at_run_builds_the_whole_runs_tables_on_pinned_programs() { + let unset = CutPadding::parse(None).expect("an unset variable"); + let run_word = CutPadding::parse(Some("run")).expect("the word run"); + let keccak_last = [ + ("COPY", 13, 2), + ("KECCAK", 2, 2), + ("KECCAK_RND", 3, 32), + ("ECSM", 0, 0), + ("ECDAS", 0, 0), + ]; + let ecsm_last = [ + ("COPY", 6, 2), + ("KECCAK", 0, 0), + ("KECCAK_RND", 0, 0), + ("ECSM", 2, 2), + ("ECDAS", 3, 16), + ]; + for (name, padding, want) in [ + ("test_keccak_multi", CutPadding::default(), keccak_last), + ("test_keccak_multi", unset, keccak_last), + ("test_ecsm_multi", CutPadding::default(), ecsm_last), + ("test_ecsm_multi", unset, ecsm_last), + ( + "test_keccak_multi", + run_word, + [ + ("COPY", 16, 2), + ("KECCAK", 2, 2), + ("KECCAK_RND", 4, 32), + ("ECSM", 0, 0), + ("ECDAS", 0, 0), + ], + ), + ( + "test_ecsm_multi", + run_word, + [ + ("COPY", 8, 2), + ("KECCAK", 0, 0), + ("KECCAK_RND", 0, 0), + ("ECSM", 2, 2), + ("ECDAS", 4, 16), + ], + ), + ] { + let (program, logs) = run(name); + let build = || cut_build(&program, &logs, 1000, (2, 2, 16, 2), true, padding); + let built = build(); + let want: Vec<(&str, usize, Vec)> = want + .into_iter() + .map(|(family, tables, rows)| (family, tables, vec![rows; tables])) + .collect(); + assert_eq!(cut_shapes(&built), want, "{name} {padding:?}"); + if padding == CutPadding::WholeRun { + same(&format!("{name} knob at run"), &oracle(build), &built); + } + } +} + +/// The readout's line names the rule and, per cut family, the tables built +/// and how many of them hold a real row. +#[test] +fn the_cut_line_names_the_rule_and_the_tables_with_a_real_row() { + let (program, logs) = run("test_keccak_multi"); + let artifacts = DecodeArtifacts::from_elf(&program).expect("the artifacts"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + let max_rows = MaxRowsConfig::small(); + let line = |skip: StreamSkip| { + let CollectedEpoch { + ops, + memory_state, + register_state, + } = Traces::collect_epoch(&artifacts, &image, ®ister_init, &logs, true) + .expect("collected"); + FinishPlan::new( + ops, + Some(&image), + &memory_state, + ®ister_init, + artifacts.decode_trace.clone(), + &artifacts.decode, + register_state, + &max_rows, + &[], + true, + false, + &skip, + None, + ) + .expect("the plan") + .cut_line() + .map(str::to_string) + }; + let cut = |cut_padding| StreamSkip { + keccak_rnd_rows: 32, + keccak_rows: 1, + ecsm_rows: 1, + ecdas_rows: 1, + cut_padding, + ..StreamSkip::default() + }; + assert_eq!( + line(cut(CutPadding::WholeRun)).as_deref(), + Some( + "BLOCK CUT PADDING: the whole run padded, then cut · tables (of them with a real \ + row): KECCAK 4 (3) · KECCAK_RND 4 (3) · ECSM 0 (0) · ECDAS 0 (0)" + ) + ); + assert_eq!( + line(cut(CutPadding::LastTable)).as_deref(), + Some( + "BLOCK CUT PADDING: cut, the last table padded (the default) · tables (of them \ + with a real row): KECCAK 3 (3) · KECCAK_RND 3 (3) · ECSM 0 (0) · ECDAS 0 (0)" + ) + ); + // A build that cuts nothing has no line. + assert_eq!(line(StreamSkip::default()), None); +} diff --git a/prover/src/tables/trace_builder/gate.rs b/prover/src/tables/trace_builder/gate.rs new file mode 100644 index 000000000..18d123030 --- /dev/null +++ b/prover/src/tables/trace_builder/gate.rs @@ -0,0 +1,322 @@ +//! The byte gate between the streamed finish and the packer: the bytes of the +//! rest's tables generated and not yet placed, held under a budget. +//! +//! The finish takes a table's bytes before it generates the table +//! ([`ByteGate::acquire`]); the packer gives them back as it places the table +//! ([`ByteGate::release`]), in the order the tables come. The lowest table not +//! yet released is always admitted, whatever its bytes: the packer is waiting +//! for exactly that table, so a budget smaller than one table cannot deadlock. +//! A stop ([`ByteGate::stop`]: a generator's error, or a packer that stopped) +//! reaches every waiter. + +use std::collections::BTreeMap; +use std::sync::{Condvar, Mutex}; + +use crate::Error; + +/// See the module docs. Tables are numbered from 0 in the order they are +/// placed. +pub(crate) struct ByteGate { + budget: usize, + /// The bytes the sliding finish may build at once ([`Self::with_build`]). + build: usize, + /// The most bytes it built at once. + building_most: std::sync::atomic::AtomicUsize, + state: Mutex, + changed: Condvar, +} + +#[derive(Default)] +struct GateState { + /// Each admitted table not yet released: its bytes. + held: BTreeMap, + /// Their sum. + bytes: usize, + /// Tables `0..released` are released. + released: usize, + /// The most bytes held at once. + most: usize, + /// The seconds acquirers waited for room, summed over them, and how many + /// acquisitions waited. + waited: f64, + waits: usize, + /// Why the gate stopped, once it has. + stopped: Option, +} + +impl GateState { + fn admits(&self, budget: usize, table: usize, bytes: usize) -> bool { + table == self.released || self.bytes.saturating_add(bytes) <= budget + } + + fn admit(&mut self, table: usize, bytes: usize) { + self.bytes += bytes; + if let Some(old) = self.held.insert(table, bytes) { + self.bytes -= old; + } + self.most = self.most.max(self.bytes); + } + + fn check(&self) -> Result<(), Error> { + match &self.stopped { + Some(why) => Err(Error::Prover(format!( + "the finish's byte gate stopped: {why}" + ))), + None => Ok(()), + } + } +} + +impl ByteGate { + pub(crate) fn new(budget: usize) -> Self { + Self { + budget, + build: budget, + building_most: std::sync::atomic::AtomicUsize::new(0), + state: Mutex::new(GateState::default()), + changed: Condvar::new(), + } + } + + fn state(&self) -> std::sync::MutexGuard<'_, GateState> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Waits until `table`'s `bytes` fit under the budget, or `table` is the + /// lowest not yet released, and holds them. An error once the gate stops. + pub(crate) fn acquire(&self, table: usize, bytes: usize) -> Result<(), Error> { + let mut state = self.state(); + let mut since: Option = None; + loop { + state.check()?; + if state.admits(self.budget, table, bytes) { + state.admit(table, bytes); + if let Some(since) = since { + state.waited += since.elapsed().as_secs_f64(); + state.waits += 1; + } + return Ok(()); + } + since.get_or_insert_with(std::time::Instant::now); + state = self.changed.wait(state).unwrap_or_else(|e| e.into_inner()); + } + } + + /// [`Self::acquire`] without waiting: whether `table` was admitted. + pub(crate) fn try_acquire(&self, table: usize, bytes: usize) -> Result { + let mut state = self.state(); + state.check()?; + let admitted = state.admits(self.budget, table, bytes); + if admitted { + state.admit(table, bytes); + } + Ok(admitted) + } + + /// `table` holds `bytes` now: its size once generated, in place of the + /// estimate it was admitted on. + pub(crate) fn resize(&self, table: usize, bytes: usize) { + let mut state = self.state(); + if state.held.contains_key(&table) { + state.admit(table, bytes); + } + drop(state); + self.changed.notify_all(); + } + + /// `table` is placed: its bytes go back. Tables are released in order. + pub(crate) fn release(&self, table: usize) { + let mut state = self.state(); + if let Some(bytes) = state.held.remove(&table) { + state.bytes -= bytes; + } + state.released = state.released.max(table + 1); + drop(state); + self.changed.notify_all(); + } + + /// Stops the gate: every waiter, now and later, gets an error naming `why`. + pub(crate) fn stop(&self, why: &str) { + let mut state = self.state(); + if state.stopped.is_none() { + state.stopped = Some(why.to_string()); + } + drop(state); + self.changed.notify_all(); + } + + /// The most bytes held at once. + pub(crate) fn most(&self) -> usize { + self.state().most + } + + /// The seconds acquirers waited for room, summed over them, and how many + /// acquisitions waited (a measurement: the finish's time not spent + /// building). + pub(crate) fn waited(&self) -> (f64, usize) { + let state = self.state(); + (state.waited, state.waits) + } + + pub(crate) fn budget(&self) -> usize { + self.budget + } + + /// The gate with a cap on the estimated bytes the sliding finish builds at + /// once, apart from the bytes it holds (built and not yet placed, or + /// building): `None` leaves the budget alone bounding them, as before, + /// which the budget already implies. + pub(crate) fn with_build(mut self, build: Option) -> Self { + if let Some(build) = build { + self.build = build; + } + self + } + + /// The cap on the bytes building at once ([`Self::with_build`]). + pub(crate) fn build(&self) -> usize { + self.build + } + + /// Notes `bytes` building at once. + pub(crate) fn note_building(&self, bytes: usize) { + self.building_most + .fetch_max(bytes, std::sync::atomic::Ordering::Relaxed); + } + + /// The most estimated bytes the sliding finish built at once. + pub(crate) fn building_most(&self) -> usize { + self.building_most + .load(std::sync::atomic::Ordering::Relaxed) + } +} + +/// Stops the gate when dropped, however its holder stops (a packer that +/// returned, or unwound). +pub(crate) struct StopGate<'g>(pub(crate) &'g ByteGate, pub(crate) &'static str); + +impl Drop for StopGate<'_> { + fn drop(&mut self) { + self.0.stop(self.1); + } +} + +#[cfg(test)] +mod tests { + use super::ByteGate; + use std::sync::Arc; + use std::time::Duration; + + /// The build cap is the budget unless one is given, and the most bytes + /// built at once only grows. + #[test] + fn the_build_cap_is_the_budget_unless_given() { + assert_eq!(ByteGate::new(10).build(), 10); + assert_eq!(ByteGate::new(10).with_build(None).build(), 10); + let gate = ByteGate::new(10).with_build(Some(4)); + assert_eq!((gate.budget(), gate.build()), (10, 4)); + gate.note_building(3); + gate.note_building(2); + assert_eq!(gate.building_most(), 3); + } + + /// Runs `f` on a thread of its own; `None` when it has not returned + /// within `secs` (the thread is left behind). + fn within(secs: u64, f: impl FnOnce() -> T + Send + 'static) -> Option { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(secs)).ok() + } + + /// A table over the budget is admitted when it is the lowest not yet + /// released, and the next waits until it is placed (within a timeout: a + /// gate that never admits fails the test instead of hanging the suite); + /// the gate counts that one wait and its time, not the acquisition that + /// did not wait. + #[test] + fn the_lowest_pending_table_is_admitted_over_the_budget() { + let done = within(20, || { + let gate = Arc::new(ByteGate::new(10)); + gate.acquire(0, 100).expect("the lowest pending table"); + assert_eq!(gate.waited(), (0.0, 0), "no wait yet"); + assert!(!gate.try_acquire(1, 1).expect("not stopped")); + let waiter = { + let gate = Arc::clone(&gate); + std::thread::spawn(move || gate.acquire(1, 100)) + }; + std::thread::sleep(Duration::from_millis(50)); + assert!(!waiter.is_finished(), "table 1 waits for table 0"); + gate.release(0); + waiter + .join() + .expect("joined") + .expect("admitted once table 0 is placed"); + (gate.most(), gate.waited()) + }); + let (most, (waited, waits)) = done.expect("the lowest pending table must be admitted"); + assert_eq!(most, 100); + assert_eq!(waits, 1, "one acquisition waited"); + assert!(waited >= 0.04, "it waited {waited} s for table 0's release"); + } + + /// Tables under the budget go in together; the held bytes follow the + /// resized tables. + #[test] + fn tables_under_the_budget_are_held_together_and_resized() { + let gate = ByteGate::new(100); + gate.acquire(0, 40).expect("0"); + assert!(gate.try_acquire(1, 40).expect("1")); + assert!(!gate.try_acquire(2, 40).expect("2 waits")); + gate.resize(1, 10); + assert!(gate.try_acquire(2, 40).expect("2 fits once 1 is resized")); + assert_eq!(gate.most(), 90); + } + + /// A stop reaches a waiter, and every later call. + #[test] + fn a_stop_reaches_every_waiter() { + let gate = Arc::new(ByteGate::new(10)); + gate.acquire(0, 10).expect("0"); + let waiters: Vec<_> = (1..4) + .map(|k| { + let gate = Arc::clone(&gate); + std::thread::spawn(move || gate.acquire(k, 10)) + }) + .collect(); + std::thread::sleep(Duration::from_millis(50)); + gate.stop("a generator failed"); + for waiter in waiters { + let err = waiter.join().expect("joined").expect_err("stopped"); + assert!(format!("{err:?}").contains("a generator failed"), "{err:?}"); + } + assert!(gate.try_acquire(9, 0).is_err()); + } + + /// One table's budget, every table larger than it: they all pass, one at + /// a time, as each is placed. + #[test] + fn a_one_byte_budget_completes() { + let done = within(20, || { + let gate = Arc::new(ByteGate::new(1)); + let placer = { + let gate = Arc::clone(&gate); + std::thread::spawn(move || { + for k in 0..50 { + while !gate.state().held.contains_key(&k) { + std::thread::yield_now(); + } + gate.release(k); + } + }) + }; + for k in 0..50 { + gate.acquire(k, 1000).expect("admitted"); + } + placer.join().expect("placer"); + gate.most() + }); + assert_eq!(done, Some(1000), "a one-byte budget must not deadlock"); + } +} diff --git a/prover/src/tables/trace_builder/keccak256_tests.rs b/prover/src/tables/trace_builder/keccak256_tests.rs new file mode 100644 index 000000000..5bbb89fdd --- /dev/null +++ b/prover/src/tables/trace_builder/keccak256_tests.rs @@ -0,0 +1,857 @@ +//! ★ Keccak-256 of a buffer (`keccak256` ecall): the KECCAK table's sponge mode +//! on `test_keccak256`, an assembly program that hashes eleven buffers (every +//! padding case, unaligned inputs, a digest written over its own input) and +//! commits the digests. +//! +//! - the committed digests are the reference Keccak-256 of each buffer; +//! - every KECCAK row satisfies the table's constraints, and every bus the +//! tables close among themselves balances, whole-run and windowed; +//! - a KECCAK trace forged one way at a time (each of the attacks the sponge +//! mode's constraints are there to stop) breaks a constraint or a bus. + +use executor::elf::Elf; +use tiny_keccak::{Hasher, Keccak}; + +use super::bus_net_tests::{named, unbalanced_with}; +use super::finish_tests::run; +use super::*; +use crate::tables::MaxRowsConfig; +use crate::tables::keccak::{KeccakConstraints, cols}; +use crate::tables::types::FE; +use crate::test_utils::{busless_air, validate_busless}; + +const PROGRAM: &str = "test_keccak256"; + +/// `(input offset, length)` of each call, in commit order (the program's table). +const CALLS: [(usize, usize); 12] = [ + (0, 0), + (1, 1), + (3, 135), + (0, 136), + (5, 137), + (2, 271), + (0, 272), + (7, 600), + (8, 20), + (6, 32), + (4, 140), + (9, 136), +]; + +fn reference(input: &[u8]) -> [u8; 32] { + let mut hasher = Keccak::v256(); + hasher.update(input); + let mut out = [0u8; 32]; + hasher.finalize(&mut out); + out +} + +fn buffer() -> Vec { + (0..1024).map(|i| ((13 * i + 5) % 256) as u8).collect() +} + +fn traces() -> (Elf, Traces) { + let (program, logs) = run(PROGRAM); + let traces = Traces::from_elf_and_logs(&program, &logs, &MaxRowsConfig::default(), &[]) + .expect("the build"); + (program, traces) +} + +/// The KECCAK rows of the trace that are sponge rows, `(table, row)`. +fn sponge_rows(traces: &Traces) -> Vec<(usize, usize)> { + let one = FE::one(); + let mut rows = Vec::new(); + for (k, t) in traces.keccaks.iter().enumerate() { + for row in 0..t.num_rows() { + if *t.get_main(row, cols::MU_S) == one { + rows.push((k, row)); + } + } + } + rows +} + +fn cell(traces: &Traces, (k, row): (usize, usize), col: usize) -> FE { + *traces.keccaks[k].get_main(row, col) +} + +fn set(traces: &mut Traces, (k, row): (usize, usize), col: usize, value: FE) { + traces.keccaks[k].main_table.set(row, col, value); +} + +/// Whether every KECCAK table satisfies the constraints and every inner bus +/// balances; `Err` names what broke. +fn check(program: &Elf, traces: &mut Traces) -> Result<(), String> { + let air = busless_air(cols::NUM_COLUMNS, KeccakConstraints); + for (k, t) in traces.keccaks.iter().enumerate() { + if !validate_busless(&air, t) { + return Err(format!("KECCAK[{k}] constraints")); + } + } + // KECCAK's interactions built here, on the test's thread, so a check a test + // leaves out (`keccak::leave_out`, thread-local) is left out of them too. + let left = unbalanced_with(program, traces, &|name| { + name.starts_with("KECCAK[") + .then(crate::tables::keccak::bus_interactions) + }); + if left.is_empty() { + Ok(()) + } else { + Err(named(&left)) + } +} + +/// The row of call `call`'s block `block` (calls in program order). +fn row_of(traces: &Traces, call: usize, block: usize) -> (usize, usize) { + let rows = sponge_rows(traces); + let firsts: Vec = rows + .iter() + .enumerate() + .filter(|(_, at)| cell(traces, **at, cols::FIRST) == FE::one()) + .map(|(i, _)| i) + .collect(); + let at = rows[firsts[call] + block]; + assert_eq!( + cell(traces, at, cols::TIMESTAMP_1), + FE::from(block as u64), + "rows of a call are in block order" + ); + at +} + +#[test] +fn the_committed_digests_are_the_reference_hashes() { + let (program, logs) = run(PROGRAM); + let traces = Traces::from_elf_and_logs(&program, &logs, &MaxRowsConfig::default(), &[]) + .expect("the build"); + let output = &traces.public_output_bytes; + let data = buffer(); + for (k, &(offset, len)) in CALLS.iter().enumerate() { + assert_eq!( + &output[32 * k..32 * k + 32], + &reference(&data[offset..offset + len]), + "call {k}: offset {offset}, length {len}" + ); + } + let calls = logs + .iter() + .filter(|l| l.src1_val == executor::vm::instruction::execution::KECCAK256_SYSCALL_NUMBER) + .count(); + assert!(calls >= CALLS.len()); +} + +#[test] +fn the_sponge_rows_satisfy_the_constraints_and_every_bus_balances() { + let (program, mut traces) = traces(); + let rows = sponge_rows(&traces); + let blocks: usize = CALLS.iter().map(|&(_, len)| len / 136 + 1).sum(); + assert_eq!(rows.len(), blocks, "one KECCAK row per block"); + check(&program, &mut traces).expect("the honest trace"); +} + +/// Applies `forge` to a fresh honest trace and requires the check to fail. +fn refused(what: &str, forge: impl FnOnce(&mut Traces)) -> String { + let (program, mut traces) = traces(); + forge(&mut traces); + match check(&program, &mut traces) { + Ok(()) => panic!("{what}: the forged trace passed"), + Err(why) => { + eprintln!("{what}: refused by {why}"); + why + } + } +} + +/// The `0x01` moved one byte later on a last row: the mask claims one more +/// input byte (a zero), and the count is raised to match. +#[test] +fn a_padding_byte_moved_is_refused() { + refused("0x01 one byte late", |t| { + let at = row_of(t, 4, 1); // 137 bytes: a 1-byte tail + // byte 1 of lane 0 joins the input as a zero. + set(t, at, cols::part(0, 1), FE::one()); + let count = cell(t, at, cols::COUNT); + set(t, at, cols::COUNT, count + FE::one()); + }); +} + +/// The 0x80 dropped: byte 135 of the last block is claimed input (FULL_16 on +/// the last row), which the last-row bound refuses. +#[test] +fn a_last_block_of_136_bytes_is_refused() { + refused("last block full", |t| { + let at = row_of(t, 3, 1); // 136 bytes: the padding-alone block + for lane in 0..cols::RATE_LANES { + set(t, at, cols::full(lane), FE::one()); + } + set(t, at, cols::COUNT, FE::from(136u64)); + }); +} + +/// A middle block skipped: its row turned into a padding row. +#[test] +fn a_skipped_block_is_refused() { + refused("middle block skipped", |t| { + let at = row_of(t, 7, 2); // 600 bytes: five blocks + for col in 0..cols::NUM_COLUMNS { + set(t, at, col, FE::zero()); + } + for lane in 0..25 { + set(t, at, cols::state_ptr(lane, 0), FE::from(8 * lane as u64)); + } + }); +} + +/// Two blocks of one call swapped: row 1 absorbs block 2's bytes and the +/// reverse. +#[test] +fn swapped_blocks_are_refused() { + refused("blocks 1 and 2 swapped", |t| { + let (a, b) = (row_of(t, 7, 1), row_of(t, 7, 2)); + for i in 0..cols::RATE { + let (x, y) = (cell(t, a, cols::msg(i)), cell(t, b, cols::msg(i))); + set(t, a, cols::msg(i), y); + set(t, b, cols::msg(i), x); + } + }); +} + +/// The digest written from the state before the last permutation: the last +/// row's first four output lanes replaced by the previous row's. +#[test] +fn a_digest_from_a_non_final_state_is_refused() { + refused("digest from the previous state", |t| { + let (prev, last) = (row_of(t, 6, 1), row_of(t, 6, 2)); + for i in 0..32 { + let v = cell(t, prev, cols::output_byte(i)); + set(t, last, cols::output_byte(i), v); + } + }); +} + +/// A block read one byte off: the row's address moved by one, its range +/// check kept consistent. +#[test] +fn a_block_read_from_the_wrong_address_is_refused() { + refused("block read one byte off", |t| { + let at = row_of(t, 6, 1); // 272 bytes, aligned + let src = cell(t, at, cols::SRC_LO); + set(t, at, cols::SRC_LO, src + FE::one()); + let end = cell(t, at, cols::SRC_END); + set(t, at, cols::SRC_END, end + FE::one()); + }); +} + +/// One input byte changed in the row (the hash of other bytes than memory's). +#[test] +fn a_message_byte_not_in_memory_is_refused() { + refused("a message byte changed", |t| { + let at = row_of(t, 2, 0); + let v = cell(t, at, cols::msg(17)); + set(t, at, cols::msg(17), v + FE::one()); + }); +} + +/// A byte past the input claimed nonzero (it is never read). +#[test] +fn a_nonzero_byte_past_the_input_is_refused() { + refused("byte past the input", |t| { + let at = row_of(t, 1, 0); // one input byte + set(t, at, cols::msg(40), FE::from(7u64)); + }); +} + +/// A call's first row restarting from a nonzero state. +#[test] +fn a_first_row_with_a_nonzero_state_is_refused() { + refused("nonzero starting state", |t| { + let at = row_of(t, 6, 0); + set(t, at, cols::input_byte(150), FE::from(1u64)); + }); +} + +/// ★ A row in both modes, or with the other mode's flags. +#[test] +fn a_row_mixing_the_two_modes_is_refused() { + let both = refused("permute and sponge at once", |t| { + let at = row_of(t, 0, 0); + set(t, at, cols::MU_P, FE::one()); + }); + assert!(both.contains("constraints"), "{both}"); + let flag = refused("FIRST on a permute row", |t| { + let at = row_of(t, 0, 0); + set(t, at, cols::MU_S, FE::zero()); + set(t, at, cols::MU_P, FE::one()); + }); + assert!(flag.contains("constraints"), "{flag}"); + let mu = refused("mu 2 over both modes", |t| { + let at = row_of(t, 0, 0); + set(t, at, cols::MU, FE::from(2u64)); + set(t, at, cols::MU_P, FE::one()); + }); + assert!(mu.contains("constraints"), "{mu}"); +} + +/// ★ Multiplicity columns that are not bits (a spread `+1, +1, −1` witness). +#[test] +fn non_bit_flags_are_refused() { + for (name, col) in [ + ("FIRST", cols::FIRST), + ("LAST", cols::LAST), + ("FULL", cols::full(3)), + ("PART", cols::part(0, 0)), + ] { + let why = refused(name, |t| { + let at = row_of(t, 6, 1); + set(t, at, col, -FE::one()); + }); + assert!(why.contains("constraints"), "{name}: {why}"); + } +} + +// ============================================================================= +// ★ Each sponge check is load-bearing: left out, the forgery it stops passes +// ============================================================================= +// +// A forgery here is consistent everywhere but at the one check: the walk builds +// it (`keccak::FORGE` rewrites one call's rows before the walk emits their memory +// traffic, and `keccak::rechain` recomputes their permutations), so the round +// chip, the BITWISE counts, the memory reads and the digest writes all follow it, +// and the committed digest is not the input's hash. With the check in, the +// forged trace is refused; with it left out (`keccak::leave_out`), the trace passes +// every KECCAK constraint and every bus the tables close among themselves. + +use crate::tables::keccak::{self, KeccakOperation, SpongeCheck, SpongeRow}; + +/// Whether `rows` are the call of `len` bytes at `offset` into the buffer. +fn is_call(rows: &[KeccakOperation], (offset, len): (usize, usize), data: &[u8]) -> bool { + rows.first() + .and_then(|r| r.sponge.as_ref()) + .is_some_and(|s| s.count == len as u64 && (len == 0 || s.msg[0] == data[offset])) +} + +fn sponge(row: &mut KeccakOperation) -> &mut SpongeRow { + row.sponge.as_mut().expect("a sponge row") +} + +/// The traces with the call `call` (`(offset, length)`) forged by `forge` +/// (which gets the call's rows and the input buffer's bytes). +fn forged_traces( + call: (usize, usize), + forge: impl Fn(&mut Vec, &[u8]) + 'static, +) -> (Elf, Traces) { + let ran = std::rc::Rc::new(std::cell::Cell::new(false)); + let seen = std::rc::Rc::clone(&ran); + keccak::FORGE.with(|f| { + *f.borrow_mut() = Some(Box::new(move |rows: &mut Vec| { + let data = buffer(); + if is_call(rows, call, &data) { + forge(rows, &data); + seen.set(true); + } + })); + }); + let built = traces(); + keccak::FORGE.with(|f| *f.borrow_mut() = None); + assert!(ran.get(), "the forger ran on the call {call:?}"); + built +} + +/// A forgery and the checks that stop it: `build` makes its traces (a forged +/// walk, and for some a post-build edit of the KECCAK cells) with whatever +/// checks are left out on this thread at the time; without `left_out`, the +/// traces pass. +struct Forgery { + name: &'static str, + left_out: &'static [SpongeCheck], + build: fn() -> (Elf, Traces), +} + +/// Every forgery a sponge check is load-bearing for, one per check. +const FORGERIES: [Forgery; 10] = [ + Forgery { + name: "a byte past the input set nonzero", + left_out: &[SpongeCheck::MsgZero], + build: past_the_input, + }, + Forgery { + name: "a hole in the read mask", + left_out: &[SpongeCheck::Prefix], + build: hole_in_the_mask, + }, + Forgery { + name: "the last block reads none of its byte", + left_out: &[SpongeCheck::CountOnLast], + build: last_block_reads_nothing, + }, + Forgery { + name: "a full last block", + left_out: &[SpongeCheck::LastNotFull], + build: full_last_block, + }, + Forgery { + name: "a middle block of 135 bytes", + left_out: &[SpongeCheck::FullUnlessLast], + build: short_middle_block, + }, + Forgery { + name: "a nonzero starting state", + left_out: &[SpongeCheck::FirstState], + build: nonzero_start, + }, + Forgery { + name: "the digest written 64 bytes past x12", + left_out: &[SpongeCheck::LinkOut], + build: digest_moved, + }, + Forgery { + name: "lane 0 not memory's bytes", + left_out: &[SpongeCheck::LaneRead], + build: lane_zero_not_memorys, + }, + Forgery { + name: "blocks 1 and 2 swapped, each block end at the other's address", + left_out: &[SpongeCheck::AddressHalves], + build: blocks_swapped_by_their_ends, + }, + Forgery { + name: "a row off the round chip by mu, a padding row answering for it", + left_out: &[SpongeCheck::ModeSum, SpongeCheck::TokensOnModes], + build: row_off_the_round_chip, + }, +]; + +/// ★ `forgery` is refused with every check in, passes with its checks left +/// out, and commits another digest than the reference. +fn load_bearing(forgery: &Forgery) { + let (_, honest) = traces(); + let (program, mut with) = (forgery.build)(); + assert_ne!( + with.public_output_bytes, honest.public_output_bytes, + "{}: the forgery commits another digest", + forgery.name + ); + let refused = check(&program, &mut with); + assert!( + refused.is_err(), + "{}: the forgery passed with every check in", + forgery.name + ); + keccak::leave_out(forgery.left_out); + let (program, mut without) = (forgery.build)(); + let passed = check(&program, &mut without); + keccak::leave_out(&[]); + eprintln!( + "{} ({:?}): refused by {} with the checks; without them: {passed:?}", + forgery.name, + forgery.left_out, + refused.unwrap_err() + ); + assert!( + passed.is_ok(), + "{}: still refused without {:?}: {passed:?}", + forgery.name, + forgery.left_out + ); +} + +/// Bytes past the input are 0: a byte past a 1-byte input, never read, set +/// nonzero, so the hash is of a block memory never held. +fn past_the_input() -> (Elf, Traces) { + forged_traces((1, 1), |rows, _| { + sponge(&mut rows[0]).msg[40] = 7; + keccak::rechain(rows, [0; 25]); + }) +} + +#[test] +fn bytes_past_the_input_are_load_bearing() { + load_bearing(&FORGERIES[0]); +} + +/// The mask is a prefix: a 20-byte input read with a hole at byte 17 and the +/// byte past its end read instead; the count still matches, the `0x01` lands in +/// the hole, and byte 18 drops by one. +fn hole_in_the_mask() -> (Elf, Traces) { + forged_traces((8, 20), |rows, data| { + let s = sponge(&mut rows[0]); + s.mask[17] = false; + s.msg[17] = 0; + s.mask[20] = true; + s.msg[20] = data[8 + 20]; + keccak::rechain(rows, [0; 25]); + }) +} + +#[test] +fn the_prefix_mask_is_load_bearing() { + load_bearing(&FORGERIES[1]); +} + +/// The last row reads exactly COUNT bytes: a 137-byte input whose last block +/// reads none of its one byte, so the hash is of the first 136 bytes. +fn last_block_reads_nothing() -> (Elf, Traces) { + forged_traces((5, 137), |rows, _| { + let s = sponge(&mut rows[1]); + s.mask = SpongeRow::prefix(0); + s.msg = [0; 136]; + keccak::rechain(rows, [0; 25]); + }) +} + +#[test] +fn the_count_on_the_last_row_is_load_bearing() { + load_bearing(&FORGERIES[2]); +} + +/// The last block holds at most 135 bytes: a 136-byte input as one full last +/// block (its padding folded into it), one permutation where Keccak takes two. +fn full_last_block() -> (Elf, Traces) { + forged_traces((9, 136), |rows, data| { + rows.truncate(1); + let s = sponge(&mut rows[0]); + assert!( + data[9 + 135] < 0x80, + "the block's last byte leaves room for 0x80" + ); + s.last = true; + keccak::rechain(rows, [0; 25]); + }) +} + +#[test] +fn the_last_block_bound_is_load_bearing() { + load_bearing(&FORGERIES[3]); +} + +/// A row that is not last reads a whole block: block 1 of a 600-byte input +/// reads 135 bytes, so a `0x01` lands in the middle of the input. +fn short_middle_block() -> (Elf, Traces) { + forged_traces((7, 600), |rows, _| { + let s = sponge(&mut rows[1]); + s.mask = SpongeRow::prefix(135); + s.msg[135] = 0; + keccak::rechain(rows, [0; 25]); + }) +} + +#[test] +fn whole_blocks_before_the_last_are_load_bearing() { + load_bearing(&FORGERIES[4]); +} + +/// The first row starts from the zero state: a 32-byte input hashed from a +/// state of the prover's choosing. +fn nonzero_start() -> (Elf, Traces) { + forged_traces((6, 32), |rows, _| { + let mut start = [0u64; 25]; + start[3] = 0x0123_4567_89ab_cdef; + start[20] = 1; + keccak::rechain(rows, start); + }) +} + +#[test] +fn the_zero_starting_state_is_load_bearing() { + load_bearing(&FORGERIES[5]); +} + +/// The digest address rides the link: the last row of a 271-byte input writes +/// its digest 64 bytes past where x12 named. +fn digest_moved() -> (Elf, Traces) { + forged_traces((2, 271), |rows, _| { + let s = sponge(&mut rows[1]); + s.out += 64; + }) +} + +#[test] +fn the_digest_address_on_the_link_is_load_bearing() { + load_bearing(&FORGERIES[6]); +} + +/// Lane 0's memory read: block 0 of a 272-byte input claims other bytes in +/// lane 0 than memory holds. +fn lane_zero_not_memorys() -> (Elf, Traces) { + forged_traces((0, 272), |rows, _| { + sponge(&mut rows[0]).msg[0] ^= 0x5a; + keccak::rechain(rows, [0; 25]); + }) +} + +#[test] +fn the_lane_read_is_load_bearing() { + load_bearing(&FORGERIES[7]); +} + +/// The sponge rows of the call whose first row has `COUNT = len` and reads +/// `data[offset]` first: `(table, row)` in block order. +fn call_rows(t: &Traces, (offset, len): (usize, usize), data: &[u8]) -> Vec<(usize, usize)> { + let rows = sponge_rows(t); + let start = rows + .iter() + .position(|&at| { + cell(t, at, cols::FIRST) == FE::one() + && cell(t, at, cols::COUNT) == FE::from(len as u64) + && (len == 0 || cell(t, at, cols::msg(0)) == FE::from(u64::from(data[offset]))) + }) + .expect("the call's first row"); + rows[start..start + len / 136 + 1].to_vec() +} + +/// A padding row of KECCAK table `k` (`μ = 0`). +fn padding_row(t: &Traces, k: usize) -> (usize, usize) { + let n = t.keccaks[k].num_rows(); + let row = (0..n) + .rev() + .find(|&r| *t.keccaks[k].get_main(r, cols::MU) == FE::zero()) + .expect("a padding row"); + (k, row) +} + +/// The block end `SRC_END = SRC_LO + 135` is also the next block's address +/// (the link sends `SRC_END + 1`), at any address, not only near 2^32: blocks +/// 1 and 2 of the 600-byte call swapped (row 1 reads block 2's bytes and row 2 +/// block 1's), each row's `SRC_END` set to the next row's address − 1. The +/// IsHalfword values are the honest ends permuted, so only the equation +/// `MU_S·(SRC_LO + 135 − SRC_END) = 0` refuses it. +fn blocks_swapped_by_their_ends() -> (Elf, Traces) { + let call = (7, 600); + let (program, mut t) = forged_traces(call, |rows, _| { + let (s1, s2) = (sponge(&mut rows[1]).src, sponge(&mut rows[2]).src); + let (m1, m2) = (sponge(&mut rows[1]).msg, sponge(&mut rows[2]).msg); + sponge(&mut rows[1]).src = s2; + sponge(&mut rows[1]).msg = m2; + sponge(&mut rows[2]).src = s1; + sponge(&mut rows[2]).msg = m1; + keccak::rechain(rows, [0; 25]); + }); + let r = call_rows(&t, call, &buffer()); + for j in 0..3 { + let end = cell(&t, r[j + 1], cols::SRC_LO).canonical_u64() - 1; + set(&mut t, r[j], cols::SRC_END, FE::from(end & 0xFFFF)); + set(&mut t, r[j], cols::SRC_END + 1, FE::from(end >> 16)); + } + (program, t) +} + +#[test] +fn the_block_end_defines_the_next_block_address() { + load_bearing(&FORGERIES[8]); +} + +/// A 32-byte call's only row writes a digest that is not the permutation of +/// its input, with `μ = 0` on that row (its mode bit `MU_S` still 1) and a +/// padding row of the same table carrying the real request and reply (`μ = 1`, +/// no mode) so the round chip's 24 rows still balance. +fn row_off_the_round_chip() -> (Elf, Traces) { + use executor::vm::instruction::execution::keccak_f1600; + let call = (6, 32); + let (program, mut t) = forged_traces(call, |rows, _| { + rows[0].output[0] ^= 0x0123_4567_89ab_cdef; + rows[0].output[3] ^= 1; + }); + let at = call_rows(&t, call, &buffer())[0]; + let mut lanes = [0u64; 25]; + for (l, lane) in lanes.iter_mut().enumerate() { + for b in 0..8 { + let v = cell(&t, at, cols::input_byte(8 * l + b)).canonical_u64(); + *lane |= v << (8 * b); + } + } + let input = lanes; + keccak_f1600(&mut lanes); + set(&mut t, at, cols::MU, FE::zero()); + let pad = padding_row(&t, at.0); + set(&mut t, pad, cols::MU, FE::one()); + let (ts0, ts1) = ( + cell(&t, at, cols::TIMESTAMP_0), + cell(&t, at, cols::TIMESTAMP_1), + ); + set(&mut t, pad, cols::TIMESTAMP_0, ts0); + set(&mut t, pad, cols::TIMESTAMP_1, ts1); + for l in 0..25 { + for b in 0..8 { + let byte = |w: u64| FE::from((w >> (8 * b)) & 0xFF); + set(&mut t, pad, cols::input_byte(8 * l + b), byte(input[l])); + set(&mut t, pad, cols::output_byte(8 * l + b), byte(lanes[l])); + } + } + (program, t) +} + +/// ★ A row's mode bit is what puts it on the round chip. The two Keccak-bus +/// tokens are weighted by `MU_P + MU_S`, so a sponge row with `μ = 0` still asks +/// the round chip and gets the real permutation back: the forged digest is +/// refused even with idx 56 (`μ = MU_P + MU_S`) left out. Only with the tokens +/// put back on `μ` as well does the forgery pass; with the tokens on `μ` and +/// idx 56 in, idx 56 refuses it. +#[test] +fn the_mode_bits_put_a_row_on_the_round_chip() { + load_bearing(&FORGERIES[9]); + for one in [SpongeCheck::ModeSum, SpongeCheck::TokensOnModes] { + keccak::leave_out(&[one]); + let (program, mut traces) = row_off_the_round_chip(); + let refused = check(&program, &mut traces); + keccak::leave_out(&[]); + eprintln!("{one:?} alone left out: {refused:?}"); + assert!( + refused.is_err(), + "{one:?} alone left out: the forgery passed" + ); + } +} + +// ============================================================================= +// ★ Box: the same forgeries through a real prove and verify +// ============================================================================= +// +// The checker above closes every bus the tables close among themselves, but +// not the Memory bus (the statement closes it), so on its own it shows that a +// forgery passes the chips, not that it passes the verifier. Here each forgery +// is proved and verified: refused with every check in, and accepted (Memory bus +// included) with its checks left out, so each mutation result above holds for +// the verifier. The prover's table workers are plain scoped threads, so the +// checks are left out process-wide (`keccak::LeftOutEverywhere`): the test must +// be the only one in its process, which `BOX_ALONE` makes its runner say. + +/// The environment word the box test's runner sets to say the test runs alone +/// (`--exact --test-threads=1`); without it the test refuses to run. +const BOX_ALONE: &str = "LAMBDA_VM_KECCAK256_BOX_ALONE"; + +/// Proves and verifies `build`'s traces with `left_out` left out on every +/// thread: `(accepted, the bus balance alone refused it)`. +/// +/// The AIRs are built fresh for each call. `test_utils` keeps one prototype of +/// each AIR a process builds, keyed by name and proof options, and serves its +/// clone after that: the first KECCAK AIR of the process would then carry its +/// bus interactions into every later prove, so a check that lives in an +/// interaction (the digest address on the link, lane 0's read, the round-chip +/// tokens on the mode bits) would stay in whatever is left out (box job l4d). +/// Each call takes its own query count (3 + a counter: a key no earlier call +/// used), and the KECCAK AIRs it proves with are checked to carry the +/// interactions the left-out checks make. +fn prove_without(left_out: &'static [SpongeCheck], build: fn() -> (Elf, Traces)) -> (bool, bool) { + use stark::proof::view::StarkProofView; + use stark::verifier::IsStarkVerifier; + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let _left = keccak::LeftOutEverywhere::new(left_out); + let (program, mut traces) = build(); + let mut options = stark::proof::options::ProofOptions::default_test_options(); + options.fri_number_of_queries += 1 + CALLS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let table_counts = traces.table_counts(); + let register_init = + crate::tables::register::register_init_from_entry_point(program.entry_point); + let register_commitment = + crate::test_utils::trace_register_commitment(&options, ®ister_init, &traces); + let airs = crate::VmAirs::new( + &program, + &options, + false, + &traces.page_configs, + &table_counts, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + register_commitment.preprocessed(), + ); + let shape = |interactions: &[stark::lookup::BusInteraction]| -> Vec { + interactions + .iter() + .map(|b| { + format!( + "{} {} {:?} {:?}", + b.bus_id, b.is_sender, b.multiplicity, b.values + ) + }) + .collect() + }; + let want = shape(&keccak::bus_interactions()); + let refs = airs.air_refs(); + let keccaks: Vec<_> = refs + .iter() + .filter(|a| a.name().starts_with("KECCAK[")) + .collect(); + assert!(!keccaks.is_empty(), "a KECCAK AIR"); + for air in &keccaks { + assert_eq!( + shape(air.bus_interactions()), + want, + "{}: the AIR proved with carries the interactions of {left_out:?}", + air.name() + ); + } + let pairs = airs.air_trace_pairs(&mut traces); + let proof = + crate::test_utils::multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) + .unwrap_or_else(|e| panic!("prover failed, which is not a verifier rejection: {e:?}")); + let views: Vec> = + proof.proofs.iter().map(StarkProofView::Owned).collect(); + let refs = airs.air_refs(); + let target = crate::compute_expected_commit_bus_balance_view( + &refs, + &views, + &traces.public_output_bytes, + 0, + &mut crate::hash_pin::block_transcript(&[]), + ) + .expect("fingerprint collision in test"); + let verify = |target: &_| { + crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + &views, + &mut crate::hash_pin::block_transcript(&[]), + target, + ) + }; + let accepted = verify(&target); + let mut sum = math::field::element::FieldElement::zero(); + for (air, view) in refs.iter().zip(views.iter()) { + if air.has_trace_interaction() + && let Some(contribution) = view.bus_table_contribution() + { + sum += contribution; + } + } + (accepted, !accepted && verify(&sum)) +} + +#[test] +#[ignore = "box: proves test_keccak256 about 25 times; run alone with LAMBDA_VM_KECCAK256_BOX_ALONE=1"] +fn box_each_forgery_is_refused_by_the_verifier_and_accepted_without_its_checks() { + assert_eq!( + std::env::var(BOX_ALONE).as_deref(), + Ok("1"), + "run this test alone (--exact --test-threads=1) with {BOX_ALONE}=1: it leaves KECCAK's \ + checks out for the whole process while it runs" + ); + let (_, honest) = traces(); + let (accepted, _) = prove_without(&[], traces); + assert!(accepted, "the honest trace proves and verifies"); + let mut wrong = Vec::new(); + for forgery in &FORGERIES { + let (_, forged) = (forgery.build)(); + assert_ne!(forged.public_output_bytes, honest.public_output_bytes); + let (with, bus_only) = prove_without(&[], forgery.build); + let (without, _) = prove_without(forgery.left_out, forgery.build); + eprintln!( + "KECCAK256-BOX {}: with every check accepted={with} (bus alone: {bus_only}); \ + without {:?} accepted={without}", + forgery.name, forgery.left_out + ); + if with || !without { + wrong.push(forgery.name); + } + } + // The row off the round chip with one of its two guards left out: the + // other refuses it. + for one in [&[SpongeCheck::ModeSum], &[SpongeCheck::TokensOnModes]] { + let (accepted, _) = prove_without(one, row_off_the_round_chip); + eprintln!("KECCAK256-BOX round chip, {one:?} alone left out: accepted={accepted}"); + if accepted { + wrong.push("a row off the round chip with one guard left out"); + } + } + assert!(wrong.is_empty(), "not as expected: {wrong:?}"); +} diff --git a/prover/src/tables/trace_builder/lean_walk_tests.rs b/prover/src/tables/trace_builder/lean_walk_tests.rs new file mode 100644 index 000000000..f83f6c400 --- /dev/null +++ b/prover/src/tables/trace_builder/lean_walk_tests.rs @@ -0,0 +1,446 @@ +//! The lean walk's parts against the walk they replace (D-EXEC E1). The whole +//! builds are compared in `tests::windowed_builder_tests`; these pin each part. + +use std::collections::HashMap; + +use executor::vm::execution::Executor; + +use super::*; +use crate::test_utils::asm_elf_bytes; + +const PAGE: u64 = page::DEFAULT_PAGE_SIZE as u64; + +/// A fixed xorshift stream. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } +} + +/// The decode table answers exactly the pcs the instruction map holds, each +/// at its DECODE row with the decode the DECODE trace makes: across runs with +/// gaps, misaligned pcs and a run that ends at the top of the address space. +#[test] +fn the_decode_table_holds_exactly_the_maps_pcs_at_their_decode_rows() { + // nop · addi x1, x0, 10 · add x3, x1, x2 · ecall + let words = [0x0000_0013u32, 0x00a0_0093, 0x0020_81b3, 0x0000_0073]; + let mut map = U64HashMap::default(); + for (first, n) in [(0x1000u64, 10usize), (0x2000, 5), (u64::MAX - 7, 2)] { + for k in 0..n { + let instruction = Instruction::parse(words[k % words.len()]).expect("a valid word"); + map.insert(first + 4 * k as u64, instruction); + } + } + let table = DecodeTable::from_instructions(&map); + let (_, pc_to_row) = decode::generate_decode_trace(&map); + for (&pc, &instruction) in map.iter() { + let row = table + .row(pc) + .unwrap_or_else(|| panic!("pc {pc:#x} has no row")); + assert_eq!(row as usize, pc_to_row[&pc], "pc {pc:#x}: the DECODE row"); + assert_eq!( + table.entry(row), + &decode::DecodeEntry::from_instruction(pc, instruction, 4), + "pc {pc:#x}" + ); + } + assert_eq!(table.padding_row(), pc_to_row[&cpu::CPU_PADDING_PC]); + for pc in [ + 0, + 0xffc, + 0x1002, + 0x1028, + 0x1ffc, + 0x2014, + u64::MAX - 11, + u64::MAX - 1, + u64::MAX, + ] { + assert!(table.row(pc).is_none(), "pc {pc:#x} is not in the map"); + } +} + +/// A plain and a lean memory state fed the same accesses. +struct Twin { + plain: MemoryState, + lean: MemoryState, +} + +impl Twin { + fn new() -> Self { + let mut image: HashMap = HashMap::new(); + for (k, addr) in [0, 7, PAGE - 1, PAGE, 3 * PAGE - 2, 12 * PAGE + 5, u64::MAX] + .into_iter() + .enumerate() + { + image.insert(addr, 0x10 + k as u8); + } + let plain = MemoryState::from_image(&image); + let mut lean = MemoryState::from_image(&image); + lean.lean = true; + Self { plain, lean } + } + + fn read(&mut self, addr: u64, count: usize) { + assert_eq!( + self.plain.read_bytes(addr, count), + self.lean.read_bytes(addr, count), + "read of {count} at {addr:#x}" + ); + } + + fn write(&mut self, addr: u64, value: u64, count: usize, ts: u64) { + self.plain.write_bytes(addr, value, count, ts); + self.lean.write_bytes(addr, value, count, ts); + } + + fn write_byte(&mut self, addr: u64, value: u8, ts: u64) { + self.plain.write_byte(addr, value, ts); + self.lean.write_byte(addr, value, ts); + } + + /// The same cells set, the same pages allocated. + fn same_cells(&self) { + let cells = |m: &MemoryState| m.cells.iter().collect::>(); + assert!(cells(&self.plain) == cells(&self.lean), "cells differ"); + assert_eq!( + self.plain.cells.page_bases().collect::>(), + self.lean.cells.page_bases().collect::>() + ); + } +} + +/// Lean memory accesses read and write what the plain ones do: an access +/// straddling a page, a first touch between two allocated pages (which moves +/// the upper page's index), accesses at the top of the address space and +/// wrapping past it, a store and a load of one byte with another page found in +/// between, single-byte writes among them, and a long random mix. +#[test] +fn lean_memory_accesses_are_the_plain_ones() { + let mut t = Twin::new(); + t.read(3 * PAGE - 3, 8); + t.write(3 * PAGE - 3, 0x0102_0304_0506_0708, 8, 10); + t.read(3 * PAGE - 3, 8); + t.read(3 * PAGE - 8, 8); + + t.write(10 * PAGE + 8, 0xaa, 1, 11); + t.write(12 * PAGE, 0xbb, 2, 12); + t.read(12 * PAGE, 8); + t.write(11 * PAGE + 100, 0xcc, 4, 13); + t.read(12 * PAGE, 8); + t.read(10 * PAGE + 8, 8); + t.read(11 * PAGE + 96, 8); + + t.write(u64::MAX - 7, u64::MAX, 8, 14); + t.read(u64::MAX - 7, 8); + t.write(u64::MAX - 3, 0x1122_3344_5566_7788, 8, 15); + t.read(u64::MAX - 3, 8); + t.read(0, 8); + + t.write(5 * PAGE + 1, 0x77, 1, 16); + t.read(9 * PAGE, 4); + t.read(5 * PAGE + 1, 1); + t.read(5 * PAGE, 8); + + t.write_byte(20 * PAGE - 1, 0x55, 17); + t.read(20 * PAGE - 4, 8); + t.write_byte(4 * PAGE + 3, 0x66, 18); + t.read(4 * PAGE, 8); + t.write(4 * PAGE, 0, 0, 19); + t.read(4 * PAGE, 8); + t.same_cells(); + + let spots = [ + 0, + PAGE - 4, + PAGE, + 2 * PAGE - 1, + 7 * PAGE + 12, + 30 * PAGE, + 31 * PAGE - 6, + 0x8000_0000, + u64::MAX - 5, + ]; + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for ts in 100..40_000u64 { + let r = rng.next(); + let addr = spots[(r % spots.len() as u64) as usize] + .wrapping_add((r >> 8) % 16) + .wrapping_sub(8); + let count = [0, 1, 2, 4, 8][((r >> 16) % 5) as usize]; + match (r >> 20) % 3 { + 0 => t.read(addr, count), + 1 => t.write(addr, r >> 24, count, ts), + _ => t.write_byte(addr, (r >> 24) as u8, ts), + } + } + t.same_cells(); +} + +/// `LAMBDA_VM_WALK_LEAN`'s values: every part, none, a list of parts; anything +/// else is refused. +#[test] +fn the_walk_lean_knob_names_its_parts() { + assert_eq!(WalkLean::parse("1"), Some(WalkLean::ALL)); + assert_eq!(WalkLean::parse(" 0 "), Some(WalkLean::NONE)); + assert_eq!( + WalkLean::parse("decode,memory,lookups,route"), + Some(WalkLean::ALL) + ); + // The decode is always read from the table: `decode` is accepted, alone + // it is none of the parts. + assert_eq!(WalkLean::parse("memory,lookups,route"), Some(WalkLean::ALL)); + assert_eq!(WalkLean::parse("decode"), Some(WalkLean::NONE)); + assert_eq!( + WalkLean::parse("memory, route"), + Some(WalkLean { + memory: true, + route: true, + ..WalkLean::NONE + }) + ); + for bad in ["", "yes", "2", "decode,", "decode;memory", "Decode"] { + assert_eq!(WalkLean::parse(bad), None, "{bad:?}"); + } +} + +/// BITWISE's multiplicities from `histogram`, row by row. +fn multiplicities(histogram: &bitwise::BitwiseHistogram) -> Vec> { + let mut trace = bitwise::generate_bitwise_trace(); + histogram.fill_multiplicities(&mut trace); + (0..trace.main_table.height) + .map(|r| { + trace + .main_table + .get_row(r) + .iter() + .map(|v| v.canonical()) + .collect() + }) + .collect() +} + +/// The lookups a lean walk leaves out, counted from its CPU and LOAD ops, are +/// the lookups the plain walk lists; and the LOAD ops' part carries weight: +/// without it the counts differ on programs with narrow loads. +#[test] +fn the_counted_lookups_are_the_listed_ones() { + let mut narrow = 0; + for name in ["lw_sw_offset_odd", "test_memw_split_ts", "test_keccak"] { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let artifacts = DecodeArtifacts::from_elf(&program).expect("the decode artifacts"); + let cpu_ops = collect_cpu_ops(&logs, &artifacts.decode).expect("the CPU ops"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + let walk = |lookups: bool| { + let mut memory_state = MemoryState::from_image(&image); + memory_state.lean = !lookups; + let mut register_state = RegisterState::from_init(®ister_init); + let mut out = WalkOutputs::for_walk(cpu_ops.len(), lookups); + collect_ops_from_cpu_into( + &cpu_ops, + &artifacts.decode, + &mut memory_state, + &mut register_state, + &mut out, + lookups, + ) + .expect("the walk"); + out + }; + let (listed, lean) = (walk(true), walk(false)); + assert!( + lean.bitwise_ops.is_empty(), + "{name}: the lean walk lists no lookup" + ); + assert!( + !listed.bitwise_ops.is_empty(), + "{name}: the plain walk lists them" + ); + + let mut from_list = bitwise::BitwiseHistogram::new(); + from_list.add_ops(&listed.bitwise_ops); + let mut counted = bitwise::BitwiseHistogram::new(); + cpu_ops + .iter() + .for_each(|op| artifacts.decode.op(op).count_bitwise_into(&mut counted)); + let cpu_only = multiplicities(&counted); + lean.load_ops + .iter() + .for_each(|op| op.count_bitwise_into(&mut counted)); + let expected = multiplicities(&from_list); + assert!( + multiplicities(&counted) == expected, + "{name}: counted ≠ listed" + ); + if lean.load_ops.iter().any(|op| op.width < 8) { + narrow += 1; + assert!( + cpu_only != expected, + "{name}: the LOAD lookups made no difference" + ); + } + } + assert!(narrow > 0, "no program with a narrow load"); +} + +/// The one-pass routing builds every segment the per-segment filters build. +#[test] +fn the_one_pass_routing_is_the_routing() { + for name in [ + "all_instructions_64", + "lw_sw_offset_odd", + "test_keccak_multi", + "mulh_max", + ] { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let artifacts = DecodeArtifacts::from_elf(&program).expect("the decode artifacts"); + let cpu_ops = collect_cpu_ops(&logs, &artifacts.decode).expect("the CPU ops"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + let mut memory_state = MemoryState::from_image(&image); + let mut register_state = RegisterState::from_init(®ister_init); + let mut walk = WalkOutputs::with_capacity(cpu_ops.len()); + collect_ops_from_cpu_into( + &cpu_ops, + &artifacts.decode, + &mut memory_state, + &mut register_state, + &mut walk, + true, + ) + .expect("the walk"); + let show = |s: RoutedSegments| { + let RoutedSegments { + branch_ops, + mul_filter, + dvrm_filter, + eq_ops, + bytewise_ops, + store_ops, + shift_cpu32, + mul_cpu32, + dvrm_cpu32, + bitwise_cpu32, + lt_dvrm_filter, + lt_dvrm_cpu32, + mul_dvrm_filter, + mul_dvrm_cpu32, + } = s; + format!( + "{branch_ops:?}{mul_filter:?}{dvrm_filter:?}{eq_ops:?}{bytewise_ops:?}{store_ops:?}\ + {shift_cpu32:?}{mul_cpu32:?}{dvrm_cpu32:?}{bitwise_cpu32:?}{lt_dvrm_filter:?}\ + {lt_dvrm_cpu32:?}{mul_dvrm_filter:?}{mul_dvrm_cpu32:?}" + ) + }; + assert_eq!( + show(route_ops(&cpu_ops, &artifacts.decode, &walk.cpu32_ops)), + show(route_ops_one_pass( + &cpu_ops, + &artifacts.decode, + &walk.cpu32_ops + )), + "{name}" + ); + } +} + +/// Routing a run window by window into the run's segments in place, counting +/// the CPU ops' lookups in the same pass, gives the segments the one-pass +/// routing's appended windows give, and the counts of the separate count. +#[test] +fn the_in_place_routing_is_the_appended_routing() { + for name in [ + "all_instructions_64", + "lw_sw_offset_odd", + "test_keccak_multi", + "mulh_max", + ] { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let artifacts = DecodeArtifacts::from_elf(&program).expect("the decode artifacts"); + let decode = &artifacts.decode; + let cpu_ops = collect_cpu_ops(&logs, decode).expect("the CPU ops"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + // Blocks of a few ops, so a window's DVRM ops straddle blocks. + let max_rows = super::super::MaxRowsConfig::uniform(4); + for window in [1, 37, cpu_ops.len()] { + let mut memory_state = MemoryState::from_image(&image); + let mut register_state = RegisterState::from_init(®ister_init); + let (mut appended, mut in_place) = + (RoutedBlocks::new(&max_rows), RoutedBlocks::new(&max_rows)); + let (mut counted, mut fused) = ( + bitwise::BitwiseHistogram::new(), + bitwise::BitwiseHistogram::new(), + ); + for ops in cpu_ops.chunks(window) { + let mut walk = WalkOutputs::with_capacity(ops.len()); + collect_ops_from_cpu_into( + ops, + decode, + &mut memory_state, + &mut register_state, + &mut walk, + true, + ) + .expect("the walk"); + appended.append(route_ops_one_pass(ops, decode, &walk.cpu32_ops)); + ops.iter() + .for_each(|op| decode.op(op).count_bitwise_into(&mut counted)); + route_ops_into(ops, decode, &walk.cpu32_ops, &mut in_place, &mut fused); + } + assert_eq!( + blocks_of(&appended), + blocks_of(&in_place), + "{name}, windows of {window}" + ); + assert!( + multiplicities(&counted) == multiplicities(&fused), + "{name}, windows of {window}: counts" + ); + } + } +} + +/// Every segment of `s`, in a printable form. +fn blocks_of(s: &RoutedBlocks) -> String { + format!( + "{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}", + s.branch_ops.range(0, s.branch_ops.len()), + s.mul_filter.to_vec(), + s.dvrm_filter.to_vec(), + s.eq_ops.range(0, s.eq_ops.len()), + s.bytewise_ops.range(0, s.bytewise_ops.len()), + s.store_ops.to_vec(), + s.shift_cpu32.to_vec(), + s.mul_cpu32.to_vec(), + s.dvrm_cpu32.to_vec(), + s.bitwise_cpu32.to_vec(), + s.lt_dvrm_filter.to_vec(), + s.lt_dvrm_cpu32.to_vec(), + s.mul_dvrm_filter.to_vec(), + s.mul_dvrm_cpu32.to_vec(), + ) +} diff --git a/prover/src/tables/trace_builder/stream_tests.rs b/prover/src/tables/trace_builder/stream_tests.rs new file mode 100644 index 000000000..174a2b996 --- /dev/null +++ b/prover/src/tables/trace_builder/stream_tests.rs @@ -0,0 +1,1670 @@ +//! ★ The streamed finish ([`super::FinishPlan::emit_streamed`]) hands on the +//! tables the monolithic finish ([`super::FinishPlan::emit_all`]) builds, word +//! for word, in the run's AIR order with every streamed chunk's slot in its +//! place, at any byte budget (one byte, 2 GiB, none). The byte gate's +//! lowest-pending rule keeps a one-byte budget from deadlocking; a job's error +//! and a stopped packer each stop the emitter and every waiter; a header that +//! miscounts is refused. The cut tables' padding rule +//! ([`CutPadding`]) is one of the configurations: the header, the stream and +//! the lists kept for phase B count the same tables under either rule. Built +//! as the gate admits each table instead of in waves +//! ([`WindowedTraceBuilder::slide_rest`]), the stream is the same, and a slow +//! table holds back only the hand-on of the tables after it. + +use std::sync::Arc; +use std::time::Duration; + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; + +use super::super::gate::ByteGate; +use super::super::*; +use super::{Emitted, FAMILIES, Job, Unit, emit_streamed_units, hand_on_order}; +use crate::tables::MaxRowsConfig; +use crate::test_utils::asm_elf_bytes; + +type Table = TraceTable; + +fn run(name: &str) -> (Elf, Vec) { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + (program, logs) +} + +/// `f` on a thread of its own; `None` when it has not returned within `secs` +/// (the thread is left behind: a hang fails the test instead of the suite). +fn within(secs: u64, f: impl FnOnce() -> T + Send + 'static) -> Option { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(secs)).ok() +} + +/// A table as (columns, rows, words row-major), held packed or wide. +fn shape_and_words(t: &Table) -> (usize, usize, Vec) { + match t.narrow_main() { + Some(packed) => { + let columns: Vec> = (0..packed.cols()).map(|c| packed.column(c)).collect(); + let words = (0..packed.rows()) + .flat_map(|r| columns.iter().map(move |column| column[r])) + .collect(); + (packed.cols(), packed.rows(), words) + } + None => { + let main = &t.main_table; + let words = (0..main.height) + .flat_map(|r| (0..main.width).map(move |c| *main.get(r, c).value())) + .collect(); + (main.width, main.height, words) + } + } +} + +/// A build's tables in the run's AIR order (`VmAirs::air_trace_pairs`, HALT +/// in, BLAKE3 as the counts include it), streamed chunks' slots as `None`. +fn air_order(t: &Traces) -> Vec> { + fn slot(table: &Table) -> Option<&Table> { + (table.main_table.width != 0).then_some(table) + } + let mut out = vec![ + slot(&t.bitwise), + slot(&t.decode), + slot(&t.keccak_rc), + slot(&t.register), + slot(&t.halt), + ]; + for list in [ + &t.copies, + &t.keccaks, + &t.keccak_rnds, + &t.ecsms, + &t.ecdases, + &t.hints, + ] { + out.extend(list.iter().map(slot)); + } + if t.table_counts().blake3 == 1 { + out.push(slot(&t.blake3)); + } + for list in [ + &t.is_whs, + &t.cpus, + &t.lts, + &t.shifts, + &t.memws, + &t.memw_aligneds, + &t.loads, + &t.muls, + &t.dvrms, + &t.branches, + &t.pages, + &t.memw_registers, + &t.eqs, + &t.bytewises, + &t.stores, + &t.cpu32s, + ] { + out.extend(list.iter().map(slot)); + } + out +} + +type Configure = Box) -> WindowedTraceBuilder<'_>>; + +/// The windowed configurations the block uses (as the plan's tests do). +fn configs() -> Vec<(&'static str, Configure)> { + vec![ + ("plain", Box::new(|b| b)), + ( + "dropped", + Box::new(|b| b.drop_streamed_ops().expect("before any window")), + ), + ( + "dropped, raw + concatenated LT", + Box::new(|b| { + b.drop_streamed_ops() + .expect("before any window") + .raw_memw_lt() + .expect("before any window") + .concat_lt() + }), + ), + ( + "dropped, KECCAK_RND at the finish, cuts, packed, written packed", + Box::new(|b| { + b.keccak_rnd_chunks_at_finish(32) + .expect("rows") + .cuts_at_finish(2, 4, 2, 2) + .expect("rows") + .pack_finished_tables() + .generate_packed() + .drop_streamed_ops() + .expect("before any window") + }), + ), + ( + "KECCAK_RND streamed, packed", + Box::new(|b| { + b.keccak_rnd_chunks(32) + .expect("rows") + .pack_finished_tables() + .drop_streamed_ops() + .expect("before any window") + }), + ), + ( + "MEMW-derived LT streamed", + Box::new(|b| { + b.stream_memw_lt() + .drop_streamed_ops() + .expect("before any window") + }), + ), + ( + "dropped, KECCAK_RND at the finish, cuts, packed, written packed, cut at the last \ + real row", + Box::new(|b| cut(b, CutPadding::LastTable)), + ), + ] +} + +/// The block's configuration with every family cut small (KECCAK_RND at 32 +/// rows, KECCAK and ECSM at one row, ECDAS at 16), the cut tables padded as +/// `padding`. +fn cut(b: WindowedTraceBuilder<'_>, padding: CutPadding) -> WindowedTraceBuilder<'_> { + b.keccak_rnd_chunks_at_finish(32) + .expect("rows") + .cuts_at_finish(1, 1, 16, 1) + .expect("rows") + .cut_padding(padding) + .pack_finished_tables() + .generate_packed() + .drop_streamed_ops() + .expect("before any window") +} + +/// ★ The header, the streamed tables and the lists kept for phase B count the +/// same KECCAK_RND tables under either padding rule: test_keccak_multi's 72 +/// rows at 32 rows a table are four tables with the whole run padded and three +/// cut at the last real row, and phase B's jobs build each of them again word +/// for word. KECCAK (3 rows at one row a table) is four tables against three. +#[test] +fn the_header_the_stream_and_the_rebuild_count_the_rules_tables() { + use super::RegenFamily; + let (program, logs) = run("test_keccak_multi"); + let max_rows = MaxRowsConfig::small(); + for (padding, tables) in [(CutPadding::WholeRun, 4usize), (CutPadding::LastTable, 3)] { + let what = format!("{padding:?}"); + let configure = move |b| cut(b, padding); + let (builder, last) = pushed(&program, &logs, &max_rows, 7, &configure); + let whole = builder.finish(last).expect("the monolithic finish"); + assert_eq!( + whole.keccak_rnds.len(), + tables, + "{what}: the finish's tables" + ); + assert_eq!(whole.keccaks.len(), tables, "{what}: KECCAK's tables"); + + let (builder, last) = pushed(&program, &logs, &max_rows, 7, &configure); + let (header, got, _) = streamed(builder, last, usize::MAX); + assert_eq!(header.table_counts.keccak_rnd, tables, "{what}: the header"); + assert_eq!(header.table_counts.keccak, tables, "{what}: the header"); + assert_eq!( + got.len(), + air_order(&whole).len(), + "{what}: the stream's tables" + ); + + let (builder, last) = pushed(&program, &logs, &max_rows, 7, &configure); + let (got, regen) = streamed_keeping(builder, last, true); + let jobs = regen + .expect("the lists kept") + .take(RegenFamily::KeccakRnd) + .expect("the jobs"); + assert_eq!(jobs.len(), tables, "{what}: phase B's jobs"); + let first = regen_ranges(&whole)[0].start; + for (j, job) in jobs.into_iter().enumerate() { + let again = job.expect("no chunk was streamed")().expect("the job builds again"); + let table = got[first + j].table.as_ref().expect("a table, not a slot"); + assert_eq!( + shape_and_words(&again), + shape_and_words(table), + "{what}: KECCAK_RND[{j}] built again" + ); + } + } +} + +/// A windowed builder over windows of `window` cycles, `configure` applied, +/// every window but the last pushed: the last window's logs with it. +fn pushed<'p>( + program: &'p Elf, + logs: &'p [Log], + max_rows: &MaxRowsConfig, + window: usize, + configure: &dyn Fn(WindowedTraceBuilder<'p>) -> WindowedTraceBuilder<'p>, +) -> (WindowedTraceBuilder<'p>, &'p [Log]) { + let mut builder = + configure(WindowedTraceBuilder::new(program, &[], max_rows).expect("the builder")); + let body = logs.len() - 1; + let cut = body - body % window; + for w in logs[..cut].chunks(window) { + builder.push(w).expect("a window"); + } + (builder, &logs[cut..]) +} + +/// What a streamed finish handed on, with the header, the gate placing each +/// table as it arrives (the packer's part). +fn streamed( + builder: WindowedTraceBuilder<'_>, + last: &[Log], + budget: usize, +) -> (RestHeader, Vec, usize) { + streamed_capped(builder, last, budget, None) +} + +/// [`streamed`] with a cap on the bytes building at once +/// ([`ByteGate::with_build`]). +fn streamed_capped( + builder: WindowedTraceBuilder<'_>, + last: &[Log], + budget: usize, + build: Option, +) -> (RestHeader, Vec, usize) { + let gate = ByteGate::new(budget).with_build(build); + let mut header = None; + let mut out = Vec::new(); + let mut placed = 0usize; + builder + .finish_streamed(last, &gate, |h| { + header = Some(h); + Ok(Box::new(|emitted: Emitted| { + if emitted.table.is_some() { + gate.release(placed); + placed += 1; + } + out.push(emitted); + Ok(()) + })) + }) + .expect("the streamed finish"); + let most = gate.most(); + (header.expect("the header came first"), out, most) +} + +/// [`streamed`] with the lists KECCAK_RND's and LT's tables are built from +/// kept (`keep`), and what was kept. +fn streamed_keeping( + builder: WindowedTraceBuilder<'_>, + last: &[Log], + keep: bool, +) -> (Vec, Option) { + let gate = ByteGate::new(usize::MAX); + let mut out = Vec::new(); + let mut placed = 0usize; + let regen = builder + .finish_streamed_keeping(last, &gate, keep, |_| { + Ok(Box::new(|emitted: Emitted| { + if emitted.table.is_some() { + gate.release(placed); + placed += 1; + } + out.push(emitted); + Ok(()) + })) + }) + .expect("the streamed finish"); + (out, regen) +} + +/// The AIR positions of a build's KECCAK_RND tables and of its LT tables +/// (`air_order`'s layout). +fn regen_ranges(t: &Traces) -> [std::ops::Range; 2] { + let kr = 5 + t.copies.len() + t.keccaks.len(); + let kr_end = kr + t.keccak_rnds.len(); + let blake3 = usize::from(t.table_counts().blake3 == 1); + let lt = kr_end + + t.ecsms.len() + + t.ecdases.len() + + t.hints.len() + + blake3 + + t.is_whs.len() + + t.cpus.len(); + [kr..kr_end, lt..lt + t.lts.len()] +} + +/// ★ N3: what phase B builds again — KECCAK_RND's and LT's tables, from the +/// lists the streamed finish kept ([`super::RestRegen`]) — is what the finish +/// handed on, word for word and packed byte for byte, for every table of the +/// two families, in every windowed configuration; a streamed chunk's slot +/// stays a slot; a family is taken once; nothing is kept unless asked. +#[test] +fn the_kept_lists_build_the_finishs_keccak_rnd_and_lt_tables_again() { + use super::RegenFamily; + let mut rebuilt = [0usize; 2]; + for max_rows in [MaxRowsConfig::small(), MaxRowsConfig::uniform(4)] { + for name in [ + "test_keccak_multi", + "all_instructions_64", + "test_ecsm_multi", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs() { + let what = format!("{name} window {window} {config}"); + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let whole = builder.finish(last).expect("the monolithic finish"); + let ranges = regen_ranges(&whole); + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let (got, regen) = streamed_keeping(builder, last, true); + let mut regen = regen.expect("the lists kept"); + for (f, (family, range)) in RegenFamily::ALL.into_iter().zip(ranges).enumerate() + { + let jobs = regen.take(family).expect("the family's jobs"); + assert_eq!(jobs.len(), range.len(), "{what}: {family:?}'s tables"); + for (j, job) in jobs.into_iter().enumerate() { + let at = range.start + j; + match (job, &got[at].table) { + (None, None) => {} + (Some(job), Some(table)) => { + let again = job().expect("the job builds again"); + assert_eq!( + shape_and_words(&again), + shape_and_words(table), + "{what}: {family:?}[{j}]" + ); + assert_eq!( + again + .narrow_main() + .map(|p| (p.widths().to_vec(), p.data().to_vec())), + table + .narrow_main() + .map(|p| (p.widths().to_vec(), p.data().to_vec())), + "{what}: {family:?}[{j}] packed" + ); + rebuilt[f] += 1; + } + (job, table) => panic!( + "{what}: {family:?}[{j}] a job {} against a table {}", + job.is_some(), + table.is_some() + ), + } + } + assert!( + regen.take(family).is_err(), + "{what}: {family:?} taken twice" + ); + } + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + assert!( + streamed_keeping(builder, last, false).1.is_none(), + "{what}: kept unasked" + ); + } + } + } + } + assert!( + rebuilt.iter().all(|&n| n > 0), + "both families rebuilt: {rebuilt:?}" + ); +} + +/// Every windowed configuration, chunked small and at four rows, windows of 7 +/// and 1000 cycles, budgets of one byte, 2 GiB and none: the streamed finish +/// hands on, in AIR order, the monolithic finish's tables word for word and +/// its streamed slots, under its header. +#[test] +fn the_stream_is_the_monolithic_finish_in_air_order() { + for max_rows in [MaxRowsConfig::small(), MaxRowsConfig::uniform(4)] { + for name in [ + "all_instructions_64", + "misalign_sd", + "test_keccak_multi", + "test_ecsm_multi", + "test_commit_split", + "test_blake3_absorb", + "test_dense_pages", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs() { + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let whole = builder.finish(last).expect("the monolithic finish"); + let want = air_order(&whole); + for budget in [1usize, 2 << 30, usize::MAX] { + let what = format!("{name} window {window} {config} budget {budget}"); + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let (header, got, _) = streamed(builder, last, budget); + assert_eq!( + format!("{:?}", header.table_counts), + format!("{:?}", whole.table_counts()), + "{what}: the header's counts" + ); + assert_eq!(got.len(), want.len(), "{what}: the tables in AIR order"); + for (k, (emitted, table)) in got.iter().zip(&want).enumerate() { + assert_eq!(emitted.position, k, "{what}: position {k}"); + match (&emitted.table, table) { + (None, None) => {} + (Some(a), Some(b)) => assert_eq!( + shape_and_words(a), + shape_and_words(b), + "{what}: table {k}" + ), + (a, b) => panic!( + "{what}: table {k} streamed {} but built {}", + a.is_some(), + b.is_some() + ), + } + } + } + } + } + } + } +} + +/// The streamed tables against the run's AIRs built from the header: one a +/// position, each table that AIR's width. +#[test] +fn the_streamed_tables_are_the_airs_in_order() { + let opts = stark::proof::options::ProofOptions::default_test_options(); + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm_multi", + "test_blake3", + ] { + let (program, logs) = run(name); + let configure: Configure = Box::new(|b| { + b.keccak_rnd_chunks_at_finish(32) + .expect("rows") + .cuts_at_finish(2, 4, 2, 2) + .expect("rows") + .pack_finished_tables() + .generate_packed() + .drop_streamed_ops() + .expect("before any window") + }); + let (builder, last) = pushed( + &program, + &logs, + &MaxRowsConfig::small(), + 7, + configure.as_ref(), + ); + let (header, got, _) = streamed(builder, last, 2 << 30); + let airs = crate::VmAirs::new( + &program, + &opts, + false, + &header.page_configs, + &header.table_counts, + &header.cpu_heights, + None, + true, + None, + crate::test_utils::halted_register_commitment( + &opts, + &program, + &header.public_output_bytes, + ) + .preprocessed(), + ); + let refs = airs.air_refs(); + assert_eq!(refs.len(), got.len(), "{name}: one table an AIR"); + for emitted in &got { + if let Some(table) = &emitted.table { + assert_eq!( + table.main_table.width, + refs[emitted.position].trace_layout().0, + "{name}: {} at {}", + refs[emitted.position].name(), + emitted.position + ); + } + } + } +} + +/// A one-byte budget, every table larger: the lowest table not yet placed +/// always goes, so the finish completes (a named timeout, not a hang, if it +/// did not). +#[test] +fn a_one_byte_budget_streams_the_finish() { + let done = within(120, || { + let (program, logs) = run("test_ecsm_multi"); + let configure: Configure = Box::new(|b| { + b.cuts_at_finish(2, 4, 2, 2) + .expect("rows") + .pack_finished_tables() + .drop_streamed_ops() + .expect("before any window") + }); + let (builder, last) = pushed( + &program, + &logs, + &MaxRowsConfig::small(), + 7, + configure.as_ref(), + ); + let (_, got, most) = streamed(builder, last, 1); + (got.len(), most) + }); + let (tables, most) = done.expect("a one-byte budget must not deadlock the finish"); + assert!(tables > 0); + assert!(most > 1, "every table went alone, over the budget"); +} + +/// Synthetic units: `n` tables of one family, each `bytes` estimated, the job +/// `fail` errs; the rest build a one-row table. +fn units<'a>(n: usize, fail: Option) -> Vec> { + (0..n) + .map(|k| Unit::Build { + family: 0, + rows: 1 << 20, + job: Box::new(move || { + if fail == Some(k) { + return Err(Error::Prover(format!("job {k} failed"))); + } + let width = FAMILIES[0].1; + Ok(TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(width), + width, + 1, + )) + }) as Job<'a>, + }) + .collect() +} + +/// A job's error ends the stream with that error and reaches a waiter on the +/// gate, within a timeout. +#[test] +fn a_jobs_error_reaches_every_waiter() { + for slide in [false, true] { + let done = within(30, move || { + let gate = Arc::new(ByteGate::new(1)); + let waiter = { + let gate = Arc::clone(&gate); + std::thread::spawn(move || gate.acquire(1000, 1 << 40)) + }; + let mut placed = 0usize; + let result = emit_streamed_units( + Ok(units(4, Some(2))), + &gate, + &mut |emitted: Emitted| { + if emitted.table.is_some() { + gate.release(placed); + placed += 1; + } + Ok(()) + }, + slide, + None, + None, + ); + (result, waiter.join().expect("the waiter")) + }); + let (result, waited) = done.expect("an error must not hang the stream"); + let err = result.expect_err("the stream fails"); + assert!( + format!("{err:?}").contains("job 2 failed"), + "slide {slide}: {err:?}" + ); + assert!(waited.is_err(), "slide {slide}: the waiter is stopped too"); + } +} + +/// A packer that stops (its gate stopped, nothing placed) stops the emitter +/// waiting for its next table's bytes; a sink that refuses stops it too. +#[test] +fn a_stopped_packer_stops_the_emitter() { + for slide in [false, true] { + let done = within(30, move || { + let gate = Arc::new(ByteGate::new(1)); + let stopper = { + let gate = Arc::clone(&gate); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(100)); + gate.stop("the packer stopped"); + }) + }; + let result = emit_streamed_units( + Ok(units(3, None)), + &gate, + &mut |_| Ok(()), + slide, + None, + None, + ); + stopper.join().expect("the stopper"); + result + }); + let err = done + .expect("a stopped packer must not hang the emitter") + .expect_err("stopped"); + assert!( + format!("{err:?}").contains("the packer stopped"), + "slide {slide}: {err:?}" + ); + let gate = ByteGate::new(usize::MAX); + let err = emit_streamed_units( + Ok(units(3, None)), + &gate, + &mut |_| Err(Error::Prover("the layout thread stopped".into())), + slide, + None, + None, + ) + .expect_err("the sink refused"); + assert!( + format!("{err:?}").contains("the layout thread stopped"), + "slide {slide}: {err:?}" + ); + } +} + +/// ★ With the deduplicating tables' per-row lookups counted on the pool +/// ([`StreamSkip::p4_rows`]), the finish builds the tables it builds without +/// it, word for word (BITWISE's multiplicities among them): the monolithic +/// finish, and the streamed one in waves and sliding, at budgets of one byte +/// and none. The programs reach LT, BRANCH and BYTEWISE over several chunks. +#[test] +fn the_row_lookups_counted_on_the_pool_build_the_same_tables() { + let max_rows = MaxRowsConfig::small(); + let words = |t: &Traces| -> Vec<_> { + air_order(t) + .into_iter() + .map(|t| t.map(shape_and_words)) + .collect() + }; + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_memw_split_ts", + "test_copy_ecalls", + "test_dense_pages", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + let what = format!("{name} window {window}"); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + b.drop_streamed_ops().expect("before any window") + }); + let want = words(&builder.finish(last).expect("the monolithic finish")); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + b.drop_streamed_ops() + .expect("before any window") + .p4_rows_on_pool() + }); + let got = words(&builder.finish(last).expect("the monolithic finish")); + assert!( + got == want, + "{what}: the monolithic finish, rows on the pool" + ); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + b.drop_streamed_ops() + .expect("before any window") + .p4_rows_by_parts() + }); + let got = words(&builder.finish(last).expect("the monolithic finish")); + assert!( + got == want, + "{what}: the monolithic finish, rows on the pool by parts" + ); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + b.drop_streamed_ops() + .expect("before any window") + .p4_rows_by_parts() + .p4_lookups_direct() + }); + let got = words(&builder.finish(last).expect("the monolithic finish")); + assert!( + got == want, + "{what}: the monolithic finish, LT's and KECCAK's lookups counted directly" + ); + for slide in [false, true] { + for budget in [1usize, usize::MAX] { + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + let b = b + .drop_streamed_ops() + .expect("before any window") + .p4_rows_on_pool(); + if slide { b.slide_rest() } else { b } + }); + let (_, got, _) = streamed(builder, last, budget); + let got: Vec<_> = got + .iter() + .map(|e| e.table.as_ref().map(shape_and_words)) + .collect(); + assert!( + got == want, + "{what}, slide {slide}, budget {budget}: the tables in AIR order" + ); + } + } + } + } +} + +/// Built as the gate admits each table: every configuration, chunked small, +/// windows of 7 and 1000 cycles, budgets of one byte, 2 GiB and none, and no +/// budget with at most one byte building at once (one table at a time), hands +/// on the monolithic finish's tables word for word in AIR order, as the waves +/// do. +#[test] +fn the_sliding_stream_is_the_monolithic_finish_in_air_order() { + let max_rows = MaxRowsConfig::small(); + for name in [ + "misalign_sd", + "test_keccak_multi", + "test_ecsm_multi", + "test_blake3_absorb", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs() { + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let want = air_order(&builder.finish(last).expect("the monolithic finish")) + .into_iter() + .map(|t| t.map(shape_and_words)) + .collect::>(); + for (budget, build) in [ + (1usize, None), + (2 << 30, None), + (usize::MAX, None), + (usize::MAX, Some(1)), + ] { + let what = + format!("{name} window {window} {config} budget {budget} build {build:?}"); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + configure(b).slide_rest() + }); + let (_, got, _) = streamed_capped(builder, last, budget, build); + let got: Vec<_> = got + .iter() + .enumerate() + .map(|(k, e)| { + assert_eq!(e.position, k, "{what}: position {k}"); + e.table.as_ref().map(shape_and_words) + }) + .collect(); + assert!(got == want, "{what}: the tables in AIR order"); + } + } + } + } +} + +/// ★ With BITWISE handed on last ([`WindowedTraceBuilder::rest_bitwise_last`]) +/// and the slide, the stream hands on the monolithic finish's tables word for +/// word, each at its AIR position, in AIR order but BITWISE (position 0), +/// which comes after every other, under a header that says so: every +/// configuration, windows of 7 and 1000 cycles, budgets of one byte, 2 GiB and +/// none, and one table building at once. In waves the order stays AIR order. +#[test] +fn with_bitwise_last_the_sliding_stream_hands_bitwise_on_last() { + let max_rows = MaxRowsConfig::small(); + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm_multi", + "test_blake3_absorb", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs() { + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let want = air_order(&builder.finish(last).expect("the monolithic finish")) + .into_iter() + .map(|t| t.map(shape_and_words)) + .collect::>(); + for (slide, budget, build) in [ + (true, 1usize, None), + (true, 2 << 30, None), + (true, usize::MAX, None), + (true, usize::MAX, Some(1)), + (false, usize::MAX, None), + ] { + let what = format!( + "{name} window {window} {config} slide {slide} budget {budget} build \ + {build:?}" + ); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + let b = configure(b).rest_bitwise_last(); + if slide { b.slide_rest() } else { b } + }); + let (header, got, _) = streamed_capped(builder, last, budget, build); + let reordered = slide && cfg!(feature = "parallel"); + assert_eq!(header.bitwise_last, reordered, "{what}: the header"); + let positions: Vec = got.iter().map(|e| e.position).collect(); + assert_eq!( + positions, + hand_on_order(want.len(), reordered.then_some(0)), + "{what}: the hand-on order" + ); + for e in &got { + assert!( + e.table.as_ref().map(shape_and_words) == want[e.position], + "{what}: table {}", + e.position + ); + } + } + } + } + } +} + +/// ★ With BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables +/// ([`WindowedTraceBuilder::bitwise_from_tables`]), the stream hands on the +/// monolithic finish's tables word for word (BITWISE's multiplicities counted +/// from the ops there), each at its AIR position, BITWISE last, under a +/// header that says so wherever the finish slides and keeps only the tails: +/// every configuration (with and without the tails), windows of 7 and 1000 +/// cycles, budgets of one byte, 2 GiB and none, and one table building at +/// once. Without the tails, and in waves, the order stays AIR order and phase +/// 4 counts as before. +#[test] +fn with_bitwise_from_tables_the_stream_is_the_monolithic_finish() { + let max_rows = MaxRowsConfig::small(); + let mut counted_from_tables = 0usize; + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm_multi", + "test_blake3_absorb", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs() { + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + // Every configuration but the plain one drops the streamed + // chunks' ops, keeping only the tails. + let tails = *config != "plain"; + let want = air_order(&builder.finish(last).expect("the monolithic finish")) + .into_iter() + .map(|t| t.map(shape_and_words)) + .collect::>(); + for (slide, budget, build) in [ + (true, 1usize, None), + (true, 2 << 30, None), + (true, usize::MAX, None), + (true, usize::MAX, Some(1)), + (false, usize::MAX, None), + ] { + let what = format!( + "{name} window {window} {config} slide {slide} budget {budget} build \ + {build:?}" + ); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + let b = configure(b).bitwise_from_tables(); + if slide { b.slide_rest() } else { b } + }); + let (header, got, _) = streamed_capped(builder, last, budget, build); + let reordered = slide && cfg!(feature = "parallel") && tails; + assert_eq!(header.bitwise_last, reordered, "{what}: BITWISE last"); + assert_eq!( + header.bitwise_from_tables, reordered, + "{what}: counted from the tables" + ); + counted_from_tables += usize::from(header.bitwise_from_tables); + let positions: Vec = got.iter().map(|e| e.position).collect(); + assert_eq!( + positions, + hand_on_order(want.len(), reordered.then_some(0)), + "{what}: the hand-on order" + ); + for e in &got { + assert!( + e.table.as_ref().map(shape_and_words) == want[e.position], + "{what}: table {}", + e.position + ); + } + } + } + } + } + assert!( + counted_from_tables > 0 || !cfg!(feature = "parallel"), + "no stream counted from the tables" + ); +} + +/// With BITWISE's lookups from the tables, phase B's LT jobs (from the lists +/// kept for it) build the finish's LT tables again word for word without +/// adding to a count BITWISE has taken: a job that did would fail. +#[test] +fn with_bitwise_from_tables_phase_bs_lt_jobs_build_the_tables_again() { + use super::RegenFamily; + let max_rows = MaxRowsConfig::small(); + let mut rebuilt = 0usize; + for name in ["all_instructions_64", "test_keccak_multi"] { + let (program, logs) = run(name); + for window in [7, 1000] { + let configure: Configure = Box::new(|b| { + b.drop_streamed_ops() + .expect("before any window") + .bitwise_from_tables() + .slide_rest() + }); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + b.drop_streamed_ops().expect("before any window") + }); + let lt = regen_ranges(&builder.finish(last).expect("the monolithic finish"))[1].clone(); + let (builder, last) = pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let (got, regen) = streamed_keeping(builder, last, true); + let jobs = regen + .expect("the lists kept") + .take(RegenFamily::Lt) + .expect("LT's jobs"); + assert_eq!(jobs.len(), lt.len(), "{name} window {window}: LT's tables"); + for (j, job) in jobs.into_iter().enumerate() { + let at = lt.start + j; + let table = got + .iter() + .find(|e| e.position == at) + .and_then(|e| e.table.as_ref()); + match (job, table) { + (None, None) => {} + (Some(job), Some(table)) => { + let again = job().expect("phase B's job builds the table again"); + assert_eq!( + shape_and_words(&again), + shape_and_words(table), + "{name} window {window}: LT[{j}]" + ); + rebuilt += 1; + } + (job, table) => panic!( + "{name} window {window}: LT[{j}] a job {} against a table {}", + job.is_some(), + table.is_some() + ), + } + } + } + } + assert!(rebuilt > 0, "no LT table built again"); +} + +/// Synthetic units that sleep: unit `slow` takes `long` ms, the others `short`; +/// each notes when it starts building. +#[cfg(feature = "parallel")] +fn sleeping_units<'a>( + n: usize, + slow: usize, + long: u64, + short: u64, + started: &'a std::sync::Mutex>, +) -> Vec> { + (0..n) + .map(|k| Unit::Build { + family: 0, + rows: 1, + job: Box::new(move || { + started + .lock() + .expect("the start log") + .push((k, std::time::Instant::now())); + std::thread::sleep(Duration::from_millis(if k == slow { long } else { short })); + let width = FAMILIES[0].1; + Ok(TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(width), + width, + 1, + )) + }) as Job<'a>, + }) + .collect() +} + +/// Synthetic units of one row each that sleep `ms`, counting how many build at +/// once (`now`) and the most (`most`). +#[cfg(feature = "parallel")] +fn counting_units<'a>( + n: usize, + ms: u64, + now: &'a std::sync::atomic::AtomicUsize, + most: &'a std::sync::atomic::AtomicUsize, +) -> Vec> { + use std::sync::atomic::Ordering::SeqCst; + (0..n) + .map(|_| Unit::Build { + family: 0, + rows: 1, + job: Box::new(move || { + most.fetch_max(now.fetch_add(1, SeqCst) + 1, SeqCst); + std::thread::sleep(Duration::from_millis(ms)); + now.fetch_sub(1, SeqCst); + let width = FAMILIES[0].1; + Ok(TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(width), + width, + 1, + )) + }) as Job<'a>, + }) + .collect() +} + +/// The build cap, sliding: under a gate that holds all `n` tables, a cap of two +/// tables' estimated bytes starts at most two at once (counted in the jobs +/// and in the gate's most), and the tables are still handed on in AIR order; +/// with no cap the gate alone admits every table at once. +#[test] +#[cfg(feature = "parallel")] +fn the_build_cap_bounds_the_tables_building_at_once() { + use std::sync::atomic::AtomicUsize; + let n = rayon::current_num_threads().min(6); + assert!(n >= 3, "a pool of at least three threads"); + let one = FAMILIES[0].1 * 8; + for build in [Some(2 * one), None] { + let (now, most) = (AtomicUsize::new(0), AtomicUsize::new(0)); + let gate = ByteGate::new(n * one).with_build(build); + let mut order = Vec::new(); + emit_streamed_units( + Ok(counting_units(n, 150, &now, &most)), + &gate, + &mut |e: Emitted| { + gate.release(order.len()); + order.push(e.position); + Ok(()) + }, + true, + None, + None, + ) + .expect("the stream"); + assert_eq!( + order, + (0..n).collect::>(), + "build {build:?}: in AIR order" + ); + match build { + Some(cap) => { + assert!( + most.into_inner() <= 2, + "build {build:?}: more than two built at once" + ); + assert!(gate.building_most() <= cap, "build {build:?}"); + assert_eq!(gate.building_most(), cap, "build {build:?}: two at once"); + } + None => assert_eq!(gate.building_most(), n * one, "no cap: all at once"), + } + } +} + +/// ★ Built as the gate admits each table, a slow first table does not hold +/// back the next tables' builds (they all start before it ends), only their +/// hand-on, which stays in order; built in waves of half the gate's budget, +/// the tables after the first wave wait for its slowest table. The budget is +/// the `n` tables' estimated bytes and the packer places each table as it +/// comes. +#[test] +#[cfg(feature = "parallel")] +fn a_slow_table_holds_back_only_the_hand_on() { + let n = rayon::current_num_threads().min(6); + assert!(n >= 2, "a pool of at least two threads"); + let one = FAMILIES[0].1 * 8; + for slide in [true, false] { + let started = std::sync::Mutex::new(Vec::new()); + let gate = ByteGate::new(n * one); + let t0 = std::time::Instant::now(); + let (mut order, mut first_sent) = (Vec::new(), None); + emit_streamed_units( + Ok(sleeping_units(n, 0, 600, 20, &started)), + &gate, + &mut |e: Emitted| { + first_sent.get_or_insert(t0.elapsed()); + gate.release(order.len()); + order.push(e.position); + Ok(()) + }, + slide, + None, + None, + ) + .expect("the stream"); + assert_eq!( + order, + (0..n).collect::>(), + "slide {slide}: in AIR order" + ); + let started = started.into_inner().expect("the start log"); + assert_eq!(started.len(), n, "slide {slide}"); + let latest = started + .iter() + .map(|(_, at)| at.duration_since(t0)) + .max() + .expect("starts"); + let first_sent = first_sent.expect("a table was handed on"); + assert!( + first_sent >= Duration::from_millis(600), + "slide {slide}: the slow table was handed on at {first_sent:?}" + ); + if slide { + assert!( + latest < Duration::from_millis(400), + "the last table started at {latest:?}, behind the slow one" + ); + } else { + assert!( + latest >= Duration::from_millis(600), + "in waves the last table started at {latest:?}, before the slow one ended" + ); + } + } +} + +/// ★ Handed on last (`last`: BITWISE's position, 0), a slow first table still +/// starts first, before any table is handed on, the tables after it are handed +/// on while it builds, and it is handed on after them; without `last` it holds +/// back every hand-on. +#[test] +#[cfg(feature = "parallel")] +fn a_table_handed_on_last_does_not_hold_back_the_rest() { + let n = rayon::current_num_threads().min(6); + assert!(n >= 2, "a pool of at least two threads"); + let one = FAMILIES[0].1 * 8; + for last in [Some(0), None] { + let started = std::sync::Mutex::new(Vec::new()); + let gate = ByteGate::new(n * one); + let t0 = std::time::Instant::now(); + let (mut order, mut sent_at) = (Vec::new(), Vec::new()); + emit_streamed_units( + Ok(sleeping_units(n, 0, 600, 20, &started)), + &gate, + &mut |e: Emitted| { + sent_at.push(t0.elapsed()); + gate.release(order.len()); + order.push(e.position); + Ok(()) + }, + true, + last, + None, + ) + .expect("the stream"); + assert_eq!(order, hand_on_order(n, last), "last {last:?}: the order"); + let started = started.into_inner().expect("the start log"); + let slow_started = started + .iter() + .find(|&&(k, _)| k == 0) + .map(|&(_, at)| at.duration_since(t0)) + .expect("the slow table started"); + assert!( + slow_started < sent_at[0], + "last {last:?}: the slow table started at {slow_started:?}, after the first \ + hand-on at {:?}", + sent_at[0] + ); + if last.is_some() { + assert!( + sent_at[0] < Duration::from_millis(400), + "the tables after the slow one waited for it: first handed on at {:?}", + sent_at[0] + ); + assert!( + sent_at[n - 1] >= Duration::from_millis(600), + "the slow table was handed on at {:?}, before it was built", + sent_at[n - 1] + ); + } else { + assert!( + sent_at[0] >= Duration::from_millis(600), + "in AIR order the first hand-on waits for the slow table: {:?}", + sent_at[0] + ); + } + } +} + +/// Handed on last under a one-byte budget, the table waits aside until it is +/// due while the others go one at a time, and the stream completes: the gate +/// numbers the tables in the order they are handed on, so the lowest one not +/// yet placed is always the one due (a named timeout, not a hang, if not). +#[test] +#[cfg(feature = "parallel")] +fn a_table_handed_on_last_streams_under_a_one_byte_budget() { + use std::sync::atomic::AtomicUsize; + let done = within(30, || { + let (now, most) = (AtomicUsize::new(0), AtomicUsize::new(0)); + let gate = ByteGate::new(1); + let mut order = Vec::new(); + let result = emit_streamed_units( + Ok(counting_units(5, 5, &now, &most)), + &gate, + &mut |e: Emitted| { + gate.release(order.len()); + order.push(e.position); + Ok(()) + }, + true, + Some(0), + None, + ); + result.map(|()| (order, most.into_inner())) + }); + let (order, most) = done + .expect("a table handed on last must not deadlock a one-byte budget") + .expect("the stream"); + assert_eq!(order, vec![1, 2, 3, 4, 0], "BITWISE last"); + assert_eq!(most, 1, "one table at a time"); +} + +/// A job that panics ends the sliding stream with an error naming its table, +/// within a timeout, instead of hanging it. +#[test] +#[cfg(feature = "parallel")] +fn a_panicking_job_ends_the_sliding_stream() { + let done = within(30, || { + let gate = ByteGate::new(usize::MAX); + let mut units = units(3, None); + units[1] = Unit::Build { + family: 0, + rows: 1, + job: Box::new(|| panic!("a generator bug")), + }; + emit_streamed_units(Ok(units), &gate, &mut |_| Ok(()), true, None, None) + }); + let err = done + .expect("a panicking job must not hang the stream") + .expect_err("the stream fails"); + assert!(format!("{err:?}").contains("table 1 panicked"), "{err:?}"); +} + +/// A header that miscounts any family by one is refused before a table is +/// handed on. +#[test] +fn a_miscounted_header_is_refused_by_the_stream() { + let (program, logs) = run("test_keccak_multi"); + let max_rows = MaxRowsConfig::small(); + let (builder, last) = pushed(&program, &logs, &max_rows, 1000, &|b| b); + let (header, _, _) = streamed(builder, last, usize::MAX); + let counts = header.table_counts; + let wrong: Vec = (0..22) + .map(|f| { + let mut c = counts.clone(); + let field = [ + &mut c.cpu, + &mut c.lt, + &mut c.memw, + &mut c.memw_aligned, + &mut c.load, + &mut c.mul, + &mut c.dvrm, + &mut c.shift, + &mut c.branch, + &mut c.memw_register, + &mut c.eq, + &mut c.bytewise, + &mut c.store, + &mut c.cpu32, + &mut c.keccak, + &mut c.keccak_rnd, + &mut c.ecsm, + &mut c.ecdas, + &mut c.hint, + &mut c.copy, + &mut c.blake3, + &mut c.is_wh, + ] + .into_iter() + .nth(f) + .expect("22 fields"); + *field += 1; + c + }) + .collect(); + let artifacts = DecodeArtifacts::from_elf(&program).expect("the artifacts"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + for (f, counts) in wrong.into_iter().enumerate() { + let CollectedEpoch { + ops, + memory_state, + register_state, + } = Traces::collect_epoch(&artifacts, &image, ®ister_init, &logs, true) + .expect("collected"); + let plan = super::FinishPlan::new( + ops, + Some(&image), + &memory_state, + ®ister_init, + artifacts.decode_trace.clone(), + &artifacts.decode, + register_state, + &max_rows, + &[], + true, + false, + &StreamSkip::default(), + None, + ) + .expect("the plan") + .with_counts(counts); + let gate = ByteGate::new(usize::MAX); + let mut handed = 0usize; + let result = plan.emit_streamed(&gate, &mut |_| { + handed += 1; + Ok(()) + }); + assert!( + result.is_err(), + "family {f} miscounted by one was not refused" + ); + assert_eq!( + handed, 0, + "family {f}: a table was handed on before the refusal" + ); + } +} + +/// A table's packed columns: widths and bytes. +fn packed_bytes(t: &Table) -> Option<(Vec, Vec)> { + t.narrow_main() + .map(|p| (p.widths().to_vec(), p.data().to_vec())) +} + +/// Where two tables' packed columns first differ, for a short failure. +fn first_difference( + a: &Option<(Vec, Vec)>, + b: &Option<(Vec, Vec)>, +) -> Option { + match (a, b) { + (Some((wa, da)), Some((wb, db))) => { + if let Some(c) = (0..wa.len().max(wb.len())).find(|&c| wa.get(c) != wb.get(c)) { + return Some(format!("column {c}'s width")); + } + (0..da.len().max(db.len())) + .find(|&i| da.get(i) != db.get(i)) + .map(|i| format!("byte {i} of {}", da.len())) + } + (None, None) => None, + _ => Some("one is packed, the other not".into()), + } +} + +/// ★ With the card filling KECCAK_RND ([`WindowedTraceBuilder::card_fill_keccak_rnd`]), +/// the streamed finish hands on each KECCAK_RND table as the keccak operations +/// its rows come from, and those records' host fill is the table the finish +/// builds without it, packed byte for byte: every cut (into an op at either +/// end, and padding rows alone) under both padding rules, windows of 7 and +/// 1000 cycles. Every other table is the same, the records are 208 bytes an +/// op of the table, and phase B is left nothing of KECCAK_RND to build again +/// (LT's jobs are the same). +#[test] +fn held_as_records_the_finishs_keccak_rnd_tables_are_its_tables() { + use super::RegenFamily; + let mut held = 0usize; + for name in [ + "test_keccak_multi", + "all_instructions_64", + "test_ecsm_multi", + ] { + let (program, logs) = run(name); + for padding in [CutPadding::LastTable, CutPadding::WholeRun] { + for window in [7, 1000] { + let what = format!("{name} {padding:?} window {window}"); + let max_rows = MaxRowsConfig::small(); + let host = move |b| cut(b, padding); + let card = move |b| cut(b, padding).card_fill_keccak_rnd(); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &host); + let whole = builder.finish(last).expect("the monolithic finish"); + let [kr, _] = regen_ranges(&whole); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &host); + let (want, want_regen) = streamed_keeping(builder, last, true); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &card); + let (got, got_regen) = streamed_keeping(builder, last, true); + assert_eq!(got.len(), want.len(), "{what}: the tables"); + for (k, (g, w)) in got.iter().zip(&want).enumerate() { + assert_eq!(g.position, w.position, "{what}: position {k}"); + let (Some(g), Some(w)) = (&g.table, &w.table) else { + assert!(g.table.is_none() && w.table.is_none(), "{what}: slot {k}"); + continue; + }; + match g.records_main() { + Some(records) => { + assert!(kr.contains(&k), "{what}: records at {k}, not KECCAK_RND's"); + let filled = records.host_fill().expect("the host fills it"); + let filled = Some((filled.widths().to_vec(), filled.data().to_vec())); + assert_eq!( + first_difference(&filled, &packed_bytes(w)), + None, + "{what}: KECCAK_RND at {k}, filled from its records" + ); + assert_eq!( + (g.main_table.width, g.main_table.height), + (w.main_table.width, w.main_table.height), + "{what}: KECCAK_RND at {k}'s shape" + ); + let ops = match records.fill() { + multilinear::records::RecordsFill::KeccakRnd { ops, skip } => { + assert!( + skip < 24 || ops.is_empty(), + "{what}: cut within its first op, if any" + ); + ops.len() + } + other => panic!("{what}: KECCAK_RND at {k} held as {other:?}"), + }; + assert_eq!(records.bytes(), 208 * ops, "{what}: 208 B an op"); + held += 1; + } + None => { + assert!(!kr.contains(&k), "{what}: KECCAK_RND at {k} built"); + assert_eq!( + first_difference(&packed_bytes(g), &packed_bytes(w)), + None, + "{what}: table {k}" + ); + assert!( + shape_and_words(g) == shape_and_words(w), + "{what}: table {k}'s words" + ); + } + } + } + let mut got_regen = got_regen.expect("the lists kept"); + let mut want_regen = want_regen.expect("the lists kept"); + assert!( + got_regen + .take(RegenFamily::KeccakRnd) + .expect("no jobs") + .is_empty(), + "{what}: KECCAK_RND left to phase B" + ); + let lt_jobs = |r: &mut super::RestRegen| -> Vec<_> { + r.take(RegenFamily::Lt) + .expect("LT's jobs") + .into_iter() + .map(|job| job.map(|job| packed_bytes(&job().expect("LT again")))) + .collect() + }; + assert!( + lt_jobs(&mut got_regen) == lt_jobs(&mut want_regen), + "{what}: LT's jobs" + ); + } + } + } + assert!(held > 0, "some KECCAK_RND table was held as records"); +} + +/// The records fault a block test plants ([`WindowedTraceBuilder::card_fill_records_fault`]) +/// moves the records themselves: every KECCAK_RND table with a real row is +/// then another table than the finish builds without it, however it is +/// filled. +#[test] +fn a_records_fault_moves_every_keccak_rnd_table_with_a_row() { + let (program, logs) = run("test_keccak_multi"); + let max_rows = MaxRowsConfig::small(); + let host = |b| cut(b, CutPadding::LastTable); + let faulty = |b| { + cut(b, CutPadding::LastTable) + .card_fill_keccak_rnd() + .card_fill_records_fault(1) + }; + let (builder, last) = pushed(&program, &logs, &max_rows, 7, &host); + let (want, _) = streamed_keeping(builder, last, false); + let (builder, last) = pushed(&program, &logs, &max_rows, 7, &faulty); + let (got, _) = streamed_keeping(builder, last, false); + let mut moved = 0usize; + for (g, w) in got.iter().zip(&want) { + let (Some(g), Some(w)) = (&g.table, &w.table) else { + continue; + }; + let Some(records) = g.records_main() else { + continue; + }; + let filled = records.host_fill().expect("the host fills it"); + let filled = Some((filled.widths().to_vec(), filled.data().to_vec())); + assert!( + first_difference(&filled, &packed_bytes(w)).is_some(), + "a KECCAK_RND table the fault left as it was" + ); + moved += 1; + } + assert!(moved > 0, "no KECCAK_RND table held as records"); +} + +/// ★ Counted beside the rest's builds ([`StreamSkip::p4_beside`]), BITWISE's +/// multiplicities are the same: the streamed finish, in waves and sliding, at +/// budgets of one byte and none, and the monolithic finish with the knob, hand +/// on the tables the monolithic finish builds without it, word for word. The +/// programs reach PAGE's lookups (test_dense_pages), KECCAK, ECSM and ECDAS, +/// COPY (test_copy_ecalls, and the committed bytes of test_commit_split) and +/// BLAKE3, and the last window's in-walk lookups. +#[test] +fn bitwise_counted_beside_the_rest_builds_the_same_tables() { + let max_rows = MaxRowsConfig::small(); + let configs: [(&str, Configure); 2] = [ + ( + "dropped", + Box::new(|b| b.drop_streamed_ops().expect("before any window")), + ), + ( + "cut at the last real row", + Box::new(|b| cut(b, CutPadding::LastTable)), + ), + ]; + let words = |t: &Traces| -> Vec<_> { + air_order(t) + .into_iter() + .map(|t| t.map(shape_and_words)) + .collect() + }; + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm_multi", + "test_blake3_absorb", + "test_commit_split", + "test_copy_ecalls", + "test_dense_pages", + ] { + let (program, logs) = run(name); + for window in [7, 1000] { + for (config, configure) in &configs { + let what = format!("{name} window {window} {config}"); + let (builder, last) = + pushed(&program, &logs, &max_rows, window, configure.as_ref()); + let want = words(&builder.finish(last).expect("the monolithic finish")); + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + configure(b).p4_beside() + }); + let whole = words(&builder.finish(last).expect("the monolithic finish")); + assert!( + whole == want, + "{what}: the monolithic finish, counted beside" + ); + for slide in [false, true] { + for budget in [1usize, usize::MAX] { + let (builder, last) = pushed(&program, &logs, &max_rows, window, &|b| { + let b = configure(b).p4_beside(); + if slide { b.slide_rest() } else { b } + }); + let (_, got, _) = streamed(builder, last, budget); + let got: Vec<_> = got + .iter() + .enumerate() + .map(|(k, e)| { + assert_eq!(e.position, k, "{what}: position {k}"); + e.table.as_ref().map(shape_and_words) + }) + .collect(); + assert!( + got == want, + "{what}, slide {slide}, budget {budget}: the tables in AIR order" + ); + } + } + } + } + } +} + +/// The knob leaves the count to BITWISE's own table: a plan made with it +/// holds what phase 4 reads instead of a histogram, and one made without it +/// holds the histogram. +#[test] +fn with_the_knob_the_plan_leaves_the_count_to_bitwise() { + let (program, logs) = run("test_keccak_multi"); + let max_rows = MaxRowsConfig::small(); + let artifacts = DecodeArtifacts::from_elf(&program).expect("the artifacts"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + for beside in [false, true] { + let CollectedEpoch { + ops, + memory_state, + register_state, + } = Traces::collect_epoch(&artifacts, &image, ®ister_init, &logs, true) + .expect("collected"); + let skip = StreamSkip { + p4_beside: beside, + ..StreamSkip::default() + }; + let plan = super::FinishPlan::new( + ops, + Some(&image), + &memory_state, + ®ister_init, + artifacts.decode_trace.clone(), + &artifacts.decode, + register_state, + &max_rows, + &[], + true, + false, + &skip, + None, + ) + .expect("the plan"); + assert_eq!( + matches!(plan.bitwise, super::BitwiseCount::Beside(_)), + beside, + "the knob {beside}" + ); + } +} diff --git a/prover/src/tables/trace_builder/windowed.rs b/prover/src/tables/trace_builder/windowed.rs new file mode 100644 index 000000000..95c88cb97 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed.rs @@ -0,0 +1,3853 @@ +//! A run's traces built window by window, with the big tables' full chunks +//! handed out as soon as their rows exist. +//! +//! A whole-run build ([`Traces::from_elf_and_logs`]) collects every op before it +//! generates any table, so nothing downstream can start until the last cycle is +//! collected. This builder takes the run's logs a window at a time, carries the +//! memory and register state across windows exactly as the whole-run walk +//! carries it across cycles, and emits every FULL chunk of the tables whose op +//! lists grow in execution order — CPU, MEMW_R, MEMW_A, MEMW, LOAD, LT, SHIFT, +//! STORE — the moment it is complete ([`StreamedChunk`]). [`finish`] then runs the +//! whole-run table phase over the same op lists, generating everything that was +//! not streamed. +//! +//! # ★ The same tables as a whole-run build +//! +//! A chunk is streamed only when it is the op slice `[k·max, (k+1)·max)` of its +//! table's FINAL op list, and that list is the whole-run one: +//! - the CPU ops carry the run's timestamps (`collect_cpu_ops_from`), and the walk +//! is a per-op fold over carried state, so its per-window outputs concatenate +//! to the whole-run outputs; +//! - the routing's lists are concatenations of segments (a filter over the CPU +//! ops, then CPU32's dispatch, then what DVRM implies), and each segment is +//! accumulated on its own ([`RoutedSegments`]) and concatenated in the +//! whole-run order at the end; +//! - a table is streamed only from its FIRST segment (for LT and SHIFT: the walk's +//! part, which the whole-run list starts with), so no later op can land in a +//! streamed chunk; +//! - the chunks are generated by the same functions the table phase uses. +//! +//! Everything that needs the whole run — BITWISE and DECODE multiplicities, the +//! CPU padding count, HALT finalization, PAGE and REGISTER from the final state, +//! the accelerators — is left to [`finish`], which is the whole-run table phase +//! with the streamed chunks' slots skipped ([`StreamSkip`]). So a windowed build is +//! a SCHEDULE, not a different trace: `windowed_builds_the_whole_run_tables` +//! checks every table against [`Traces::from_elf_and_logs`]. +//! +//! # Dropping the streamed ops +//! +//! By default the builder keeps every walked window until [`finish`], so the +//! run's op lists are all resident at the end of the run (≈ 300 B per cycle). +//! With [`WindowedTraceBuilder::drop_streamed_ops`] it keeps, per streamed +//! table, only the ops not yet in a handed-out chunk ([`Tail`]), and takes what +//! the table phase would have read from a chunk's ops as the chunk leaves: +//! BITWISE's lookups from LT, SHIFT, STORE, MEMW_A and MEMW_R, DECODE's lookups +//! and the last ECALL from CPU are counted; the phase-3 LT ops of MEMW and +//! MEMW_A are derived and kept, compact ([`super::CompactLt`]: a few bytes an +//! op), unless [`stream_memw_lt`] already put them in the windows' LT lists. The other walk lists are appended window by window; +//! the in-walk BITWISE lookups are counted and dropped. [`finish`] then runs +//! the table phase over the tails ([`StreamSkip::tails`]). The tables are the +//! same as without it. +//! +//! [`stream_memw_lt`]: WindowedTraceBuilder::stream_memw_lt +//! +//! Row order inside the tables that deduplicate through a `HashMap` (EQ, +//! BYTEWISE, BRANCH, DVRM, LT, MUL) is not reproducible between ANY two builds; +//! their chunks hold the same ops, so the same rows. +//! +//! [`finish`]: WindowedTraceBuilder::finish + +use std::collections::VecDeque; +use std::sync::Arc; + +use executor::elf::Elf; +use executor::vm::instruction::decoding::Instruction; +use executor::vm::logs::Log; +use executor::vm::memory::U64HashMap; +#[cfg(feature = "parallel")] +use rayon::prelude::*; +use stark::trace::TraceTable; + +use super::super::gpack::TraceForm; +use super::super::types::{GoldilocksExtension, GoldilocksField}; +use super::{ + BlockVec, CollectedOps, CutPadding, MemoryState, MemwBuckets, PreCounted, RegisterState, + RoutedBlocks, StreamSkip, Traces, WalkLean, WalkOutputs, WalkSizes, bitwise, block_len, + build_initial_image_paged, build_traces, collect_halt_ops, collect_ops_from_cpu_into, + route_ops, route_ops_one_pass, +}; +use crate::Error; +use crate::paged_mem::PagedMem; +use crate::tables::decode::DecodeTable; +use crate::tables::{ + cpu, decode, keccak_rnd, load, lt, memw, memw_aligned, memw_register, register, shift, store, +}; + +type Table = TraceTable; + +mod card_records; +use card_records::{ + CardDecode, CpuLogRecords, RegisterRowRecords, RowChunk, RowFamily, RowRecords, +}; + +mod handout_count; +pub use handout_count::CountThreadStamps; +use handout_count::{CountWork, HandoutCounter}; + +/// Whether the host has room to hold a chunk of so many bytes as records +/// ([`WindowedTraceBuilder::card_fill_streamed`]). +pub type RecordsRoom = Arc bool + Send + Sync>; + +/// The streamed families whose full chunks may be held as their rows, the +/// walk's own structs ([`WindowedTraceBuilder::card_fill_streamed_rows`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RowFamilies { + pub memw_a: bool, + pub load: bool, + pub store: bool, + pub memw: bool, + pub shift: bool, +} + +impl RowFamilies { + /// None of them. + pub const NONE: Self = Self { + memw_a: false, + load: false, + store: false, + memw: false, + shift: false, + }; + /// Every one. + pub const ALL: Self = Self { + memw_a: true, + load: true, + store: true, + memw: true, + shift: true, + }; + + /// Whether any is. + pub fn any(self) -> bool { + self != Self::NONE + } +} + +/// [`WindowedTraceBuilder::card_fill_streamed`]: the program's DECODE rows +/// as the card reads them when CPU chunks are held as logs, whether MEMW_R +/// chunks are held as rows, which row families are +/// ([`WindowedTraceBuilder::card_fill_streamed_rows`]), and the room rule. +struct StreamedCardFill { + cpu: Option>, + memw_r: bool, + rows: RowFamilies, + room: RecordsRoom, + /// A test's fault ([`WindowedTraceBuilder::card_fill_streamed_fault`]): + /// each CPU log held as records has its first source and written values + /// XORed with it, each row family's row its timestamp that much later. + #[cfg(test)] + fault: u64, +} + +impl StreamedCardFill { + /// Whether a chunk of `bytes` of records may be held: the room rule says + /// so, and said so for every chunk before it (once a chunk is refused, + /// every later one is filled on the host, so what is never dropped is the + /// prefix admitted with room). Notes the first refused chunk. + fn admits( + &self, + bytes: usize, + chunk: (StreamTable, usize), + stamps: &mut CardFillStamps, + ) -> bool { + if stamps.room_out.is_some() { + return false; + } + let room = (self.room)(bytes); + if !room { + stamps.room_out = Some(chunk); + } + room + } +} + +/// The tables a windowed build streams: those whose op lists grow in +/// execution order from their first element. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum StreamTable { + Cpu, + MemwRegister, + MemwAligned, + Memw, + Load, + Lt, + Shift, + Store, + /// Only when the builder cuts KECCAK_RND into chunks + /// ([`WindowedTraceBuilder::keccak_rnd_chunks`]). + KeccakRnd, +} + +impl StreamTable { + pub const ALL: [StreamTable; 9] = [ + StreamTable::Cpu, + StreamTable::MemwRegister, + StreamTable::MemwAligned, + StreamTable::Memw, + StreamTable::Load, + StreamTable::Lt, + StreamTable::Shift, + StreamTable::Store, + StreamTable::KeccakRnd, + ]; + + /// Its place in [`Self::ALL`]. + pub fn position(self) -> usize { + self as usize + } +} + +/// One full chunk of a streamed table: chunk `index` of that table, the same +/// table a whole-run build puts at `Traces::[index]`. +pub struct StreamedChunk { + pub table: StreamTable, + pub index: usize, + pub trace: Table, +} + +/// A run's traces, built window by window. See the module docs. +pub struct WindowedTraceBuilder<'a> { + /// Every instruction's decode by DECODE row: what the walk reads each + /// cycle's decode with (shared with the threads that generate CPU chunks). + decode: Arc, + /// DECODE's trace, until [`Self::drop_streamed_ops`] hands it to the kept + /// run, which counts the handed-out CPU ops' rows into it. + decode_trace: Option, + image: PagedMem, + register_init: Vec, + max_rows: crate::tables::MaxRowsConfig, + private_input: &'a [u8], + memory_state: MemoryState, + register_state: RegisterState, + /// Cycles collected so far: the next window's first cycle. + cycles: usize, + /// The walked windows, in run order, kept as they were walked: moved here, + /// never copied. `finish` concatenates them once, list by list. + windows: Vec, + /// The routing's segments, appended per window in blocks. + segments: RoutedBlocks, + emitted: StreamSkip, + /// The BITWISE lookups of the walked windows' in-walk lookups and MEMW_R + /// rows, counted as the windows arrive. + counted: PreCounted, + stamps: WindowStamps, + /// [`Self::stream_memw_lt`]. + stream_memw_lt: bool, + /// [`Self::drop_streamed_ops`]: the run so far, without the handed-out ops + /// (`windows` stays empty). + kept: Option, + /// The parts of the lean walk this build uses. + lean: WalkLean, + /// [`Self::keccak_rnd_chunks_at_finish`]: rows per KECCAK_RND table `finish` + /// builds, 0 for one table. + finish_keccak_rnd_rows: usize, + /// [`Self::cuts_at_finish`]: rows per KECCAK, ECSM, ECDAS and COPY table + /// `finish` builds, 0 for one table each. + finish_cut_rows: (usize, usize, usize, usize), + /// [`Self::pack_finished_tables`]. + pack_finished: bool, + /// [`Self::generate_packed`]. + gpack: bool, + /// How `finish` pads the tables it cuts ([`CutPadding`]): + /// `LAMBDA_VM_BLOCK_CUT_PADDING`'s rule. + cut_padding: CutPadding, + /// The cycles the walker collects and walks at a time ([`walk_batch`]). + walk_batch: usize, + /// The last walked window's list lengths, to size the next window's lists + /// ([`walk_sized`]). + walk_sizes: WalkSizes, + /// [`Self::setup_stamps`]. + setup: SetupStamps, + /// [`Self::slide_rest`]. + slide_rest: bool, + /// [`Self::card_fill_keccak_rnd`]. + card_fill: bool, + /// [`Self::card_fill_records_fault`]. + #[cfg(test)] + card_fill_fault: usize, + /// [`Self::card_fill_streamed`]. + card_fill_streamed: Option, + /// Where the hand-out's counting runs ([`Self::count_handout_on_thread`]). + count: CountMode, +} + +/// Where the hand-out counts what the table phase reads from each chunk it +/// hands out under [`WindowedTraceBuilder::drop_streamed_ops`] +/// ([`handout_count`]). +enum CountMode { + /// On the accumulator's thread, as the chunk leaves. + Inline, + /// On a thread of its own, started with the first chunk. + Thread, + /// The thread, counting. + Running(HandoutCounter), + /// The thread's counts added to the builder's: any later chunk is counted + /// on the accumulator's thread. + Joined, +} + +impl CountMode { + /// The counter, started when it is due; `None` while counting inline + /// (also when its thread did not start). + fn counter( + &mut self, + decode: &Arc, + stream_memw_lt: bool, + counted: &PreCounted, + ) -> Option<&HandoutCounter> { + if let Self::Thread = self { + *self = match HandoutCounter::spawn( + Arc::clone(decode), + stream_memw_lt, + counted.memw_lt.empty_like(), + counted.memw_aligned_lt.empty_like(), + ) { + Ok(counter) => Self::Running(counter), + Err(_) => Self::Inline, + }; + } + match self { + Self::Running(counter) => Some(counter), + _ => None, + } + } +} + +/// Where a builder's creation spent its time, seconds: the initial image and +/// the memory state seeded from it, and the decode table the walk reads. +/// DECODE's trace is generated on a thread of its own meanwhile. +#[derive(Clone, Copy, Debug, Default)] +pub struct SetupStamps { + pub image: f64, + pub decode: f64, +} + +/// DECODE's pristine trace (every multiplicity zero), generated on a thread of +/// its own from the builder's creation on: the walk reads only the decode +/// table, and the trace is first read where the first CPU chunk's DECODE rows +/// are counted, windows later. [`Self::table`] waits for the thread; a panic in +/// it is raised there, on the thread that reads the trace, as building the +/// trace in place would have raised it. +struct DecodeTrace { + thread: Option>, + table: Table, +} + +impl DecodeTrace { + fn spawn(instructions: U64HashMap) -> Result { + #[cfg(test)] + let delay = DECODE_TRACE_DELAY.with(std::cell::Cell::get); + let thread = std::thread::Builder::new() + .name("decode-trace".to_string()) + .spawn(move || { + #[cfg(test)] + std::thread::sleep(delay); + // As `DecodeArtifacts::from_elf`: `generate_decode_trace` lays + // the rows out in the order the AIR's commitment holds. + decode::generate_decode_trace(&instructions).0 + }) + .map_err(|e| Error::Prover(format!("DECODE's trace thread did not start: {e}")))?; + Ok(Self { + thread: Some(thread), + table: super::streamed_placeholder(), + }) + } + + /// The trace, once its thread has generated it. + fn table(&mut self) -> &mut Table { + if let Some(thread) = self.thread.take() { + self.table = thread + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)); + } + &mut self.table + } + + fn into_table(mut self) -> Table { + self.table(); + self.table + } + + /// The bytes the trace takes on the heap once its thread has handed it + /// over; none before. + fn heap_bytes(&self) -> usize { + if self.thread.is_some() { + return 0; + } + self.table.num_rows() * self.table.num_main_columns * std::mem::size_of::() + } +} + +#[cfg(test)] +thread_local! { + /// A test's delay before DECODE's trace thread generates the trace, read + /// where a builder is created ([`DecodeTrace::spawn`]). + pub(crate) static DECODE_TRACE_DELAY: std::cell::Cell = + const { std::cell::Cell::new(std::time::Duration::ZERO) }; +} + +/// What the table phase of a windowed build starts from +/// ([`WindowedTraceBuilder::finish`]): the run's lists, the state the walk +/// left, and what the windows streamed and counted. +struct Finishing<'a> { + ops: CollectedOps, + image: PagedMem, + memory_state: MemoryState, + register_init: Vec, + decode_trace: TraceTable, + decode: Arc, + register_state: RegisterState, + max_rows: crate::tables::MaxRowsConfig, + private_input: &'a [u8], + skip: StreamSkip, + pre: PreCounted, +} + +/// Where a windowed build spent its time, seconds summed over the windows: +/// the walk, the routing (with the window's bookkeeping), and handing the +/// streamed chunks out (`finish` is timed by its caller). +#[derive(Clone, Copy, Debug, Default)] +pub struct WindowStamps { + pub windows: usize, + pub walk: f64, + pub route: f64, + pub generate: f64, + /// `generate` table by table, under + /// [`WindowedTraceBuilder::drop_streamed_ops`]. + pub handout: HandoutStamps, + /// [`WindowedTraceBuilder::card_fill_streamed`]'s chunks. + pub card_fill: CardFillStamps, + /// The hand-out's counting thread, once joined + /// ([`WindowedTraceBuilder::join_handout_count`]); `None` when the + /// accumulator's thread counted. + pub count_thread: Option, +} + +/// The CPU, MEMW_R and row families' chunks handed out under +/// [`WindowedTraceBuilder::card_fill_streamed`]: per family, `[as records, +/// filled on the host]`; the records' bytes; the first chunk the room rule +/// sent to the host; and the CPU chunks whose logs were not the ops' (each +/// filled on the host). +#[derive(Clone, Copy, Debug, Default)] +pub struct CardFillStamps { + pub cpu: [usize; 2], + pub memw_r: [usize; 2], + pub memw_a: [usize; 2], + pub load: [usize; 2], + pub store: [usize; 2], + pub memw: [usize; 2], + pub shift: [usize; 2], + pub records_bytes: usize, + pub room_out: Option<(StreamTable, usize)>, + pub out_of_step: usize, +} + +/// The hand-out under [`WindowedTraceBuilder::drop_streamed_ops`], per streamed +/// table in [`StreamTable::ALL`] order, summed over the windows: the seconds +/// taking each chunk's ops out of the tail (`take`: a copy, unless a window's +/// list is the chunk and moves out whole) and counting what the table phase +/// reads from them (`count`), the chunks handed out and the ops `take` copied. +/// A measurement; the chunks are the same. +#[derive(Clone, Copy, Debug, Default)] +pub struct HandoutStamps { + pub take: [f64; 9], + pub count: [f64; 9], + pub chunks: [usize; 9], + pub copied: [usize; 9], +} + +impl HandoutStamps { + fn note(&mut self, table: StreamTable, take: f64, count: f64, copied: usize) { + let i = table.position(); + self.take[i] += take; + self.count[i] += count; + self.chunks[i] += 1; + self.copied[i] += copied; + } +} + +impl<'a> WindowedTraceBuilder<'a> { + /// A builder for a run of `elf` on `private_input`, chunked at `max_rows` + /// — the state [`Traces::from_elf_and_logs`] starts from. + pub fn new( + elf: &Elf, + private_input: &'a [u8], + max_rows: &crate::tables::MaxRowsConfig, + ) -> Result { + let t = std::time::Instant::now(); + // The initial image as pages, and the walk's memory state seeded from + // them page by page: the state the image as a map of bytes, seeded byte + // by byte, gives. + let image = build_initial_image_paged(elf, private_input); + let register_init = register::register_init_from_entry_point(elf.entry_point); + let mut memory_state = MemoryState::from_pages(&image); + let register_state = RegisterState::from_init(®ister_init); + let image_secs = t.elapsed().as_secs_f64(); + let t = std::time::Instant::now(); + let instructions = decode::instructions_from_elf(elf) + .map_err(|e| Error::Execution(format!("Failed to parse instructions: {e}")))?; + let decode = Arc::new(DecodeTable::from_instructions(&instructions)); + let decode_secs = t.elapsed().as_secs_f64(); + let decode_trace = DecodeTrace::spawn(instructions)?; + let lean = WalkLean::from_env()?; + let count = if handout_count::count_thread_from_env()? { + CountMode::Thread + } else { + CountMode::Inline + }; + memory_state.lean = lean.memory; + Ok(Self { + decode, + decode_trace: Some(decode_trace), + image, + register_init, + max_rows: max_rows.clone(), + private_input, + memory_state, + register_state, + cycles: 0, + windows: Vec::new(), + segments: RoutedBlocks::new(max_rows), + emitted: StreamSkip::default(), + counted: PreCounted { + histogram: bitwise::BitwiseHistogram::new(), + bitwise_ops: 0, + memw_register_rows: 0, + last_ecall: None, + memw_lt: super::CompactLt::default(), + memw_aligned_lt: super::CompactLt::default(), + }, + stamps: WindowStamps::default(), + stream_memw_lt: false, + kept: None, + lean, + finish_keccak_rnd_rows: 0, + finish_cut_rows: (0, 0, 0, 0), + pack_finished: false, + gpack: false, + cut_padding: CutPadding::from_env()?, + walk_batch: walk_batch(), + walk_sizes: WalkSizes::default(), + setup: SetupStamps { + image: image_secs, + decode: decode_secs, + }, + slide_rest: false, + card_fill: false, + #[cfg(test)] + card_fill_fault: 0, + card_fill_streamed: None, + count, + }) + } + + /// Count what the table phase reads from each handed-out chunk on a + /// thread of its own (`true`) or on the accumulator's (`false`), whatever + /// `LAMBDA_VM_HANDOUT_COUNT_THREAD` says ([`handout_count`]). The tables + /// are the same. Before the first chunk is handed out; later, or without + /// the `parallel` feature for `true`, it changes nothing. + pub fn count_handout_on_thread(mut self, on: bool) -> Self { + if cfg!(feature = "parallel") && matches!(self.count, CountMode::Inline | CountMode::Thread) + { + self.count = if on { + CountMode::Thread + } else { + CountMode::Inline + }; + } + self + } + + /// The hand-out's counting thread, if it ran: every chunk it was sent + /// counted, and its counts added to the builder's (the BITWISE histogram, + /// DECODE's lookups, the last ECALL, the MEMW-derived LT ops), its stamps + /// to [`Self::stamps`]. Chunks handed out after this are counted on the + /// accumulator's thread. [`Self::finish`] joins it too; the block calls it + /// as the windows end, so their end includes it. + pub(crate) fn join_handout_count(&mut self) -> Result<(), Error> { + let CountMode::Running(counter) = std::mem::replace(&mut self.count, CountMode::Joined) + else { + return Ok(()); + }; + let counts = counter.finish()?; + let t = std::time::Instant::now(); + let handout_count::Counts { + histogram, + decode_rows, + last_ecall, + memw_lt, + memw_aligned_lt, + mut stamps, + } = counts; + let kept = self.kept.as_mut().ok_or_else(|| { + Error::Prover( + "the hand-out's count thread ran for a build that keeps its windows".into(), + ) + })?; + decode::add_row_counts(kept.decode.table(), &decode_rows).map_err(|row| { + Error::Prover(format!( + "the hand-out's count thread counted DECODE row {row}, which DECODE's trace lacks" + )) + })?; + let counted = &mut self.counted; + if !counted.memw_lt.holds_none() || !counted.memw_aligned_lt.holds_none() { + return Err(Error::Prover( + "MEMW-derived LT ops were counted on the accumulator's thread beside the count \ + thread's" + .into(), + )); + } + counted.histogram.merge(&histogram); + drop(histogram); + counted.memw_lt = memw_lt; + counted.memw_aligned_lt = memw_aligned_lt; + if last_ecall.is_some() { + counted.last_ecall = last_ecall; + } + stamps.merge = t.elapsed().as_secs_f64(); + self.stamps.count_thread = Some(stamps); + Ok(()) + } + + /// Walk with these lean parts, whatever `LAMBDA_VM_WALK_LEAN` says (the + /// tables are the same). Before the first window only. + #[cfg(test)] + pub(crate) fn walk_lean(mut self, lean: WalkLean) -> Self { + assert_eq!(self.cycles, 0, "walk_lean after a window was walked"); + self.memory_state.lean = lean.memory; + self.lean = lean; + self + } + + /// `finish` pads the tables it cuts as `padding`, whatever + /// `LAMBDA_VM_BLOCK_CUT_PADDING` says. + #[cfg(test)] + pub(crate) fn cut_padding(mut self, padding: CutPadding) -> Self { + self.cut_padding = padding; + self + } + + /// Walk `n` cycles at a time (`0`: the whole window's CPU ops, then the + /// walk), whatever `LAMBDA_VM_WALK_BATCH` says (the lists are the same). + #[cfg(test)] + pub(crate) fn walk_batch(mut self, n: usize) -> Self { + self.walk_batch = n; + self + } + + /// Collects the next window of the run's logs (not the last one: that is + /// [`finish`](Self::finish)'s) and returns the full chunks it completed. + /// + /// A window may be any length; the chunks do not depend on where the + /// windows fall. + pub fn push(&mut self, logs: &[Log]) -> Result, Error> { + let jobs = self.push_jobs(logs)?; + #[cfg(feature = "parallel")] + let jobs = jobs.into_par_iter(); + #[cfg(not(feature = "parallel"))] + let jobs = jobs.into_iter(); + Ok(jobs.map(ChunkJob::generate).collect()) + } + + /// Like [`push`](Self::push), but hands back each completed chunk as a + /// [`ChunkJob`] — its ops, copied out — instead of generating it here, so + /// the caller can generate it on another thread while this one walks the + /// next window. The chunks are the same. + pub fn push_jobs(&mut self, logs: &[Log]) -> Result, Error> { + let keeps = self.keeps_logs(); + let (mut walker, mut accumulator) = self.split(); + let walked = if keeps { + walker.walk_owned(logs.to_vec())? + } else { + walker.walk(logs)? + }; + Ok(accumulator.absorb(walked)) + } + + /// Cut KECCAK_RND into tables of `rows` rows — the split a block proof makes + /// — and hand each full one out as its ops arrive (24 rows an op; the rows + /// of an op depend on that op alone). `rows` is a power of two, at least 32. + /// Without this call KECCAK_RND is one table, built by `finish`. + pub fn keccak_rnd_chunks(mut self, rows: usize) -> Result { + if !rows.is_power_of_two() || rows < 32 { + return Err(Error::Prover(format!( + "KECCAK_RND chunks of {rows} rows: a power of two of at least 32 is needed" + ))); + } + self.emitted.keccak_rnd_rows = rows; + Ok(self) + } + + /// `finish` builds KECCAK_RND as tables of `rows` rows — the tables the + /// block's split makes of the whole one, each built from the ops that reach + /// it — instead of one table, without handing any out during the windows + /// ([`Self::keccak_rnd_chunks`] does both). The whole table and its split + /// copy are never held: on a block, KECCAK_RND's 1,480 columns are the + /// largest table the finish builds. `rows` is a power of two, at least 32. + pub fn keccak_rnd_chunks_at_finish(mut self, rows: usize) -> Result { + if !rows.is_power_of_two() || rows < 32 { + return Err(Error::Prover(format!( + "KECCAK_RND chunks of {rows} rows: a power of two of at least 32 is needed" + ))); + } + self.finish_keccak_rnd_rows = rows; + Ok(self) + } + + /// `finish` builds KECCAK, ECSM, ECDAS and COPY as the tables of `keccak`, + /// `ecsm`, `ecdas` and `copy` rows the block's split makes of each whole + /// table — each cut + /// built from the ops that reach it, and packed as it is built with + /// [`Self::pack_finished_tables`] — instead of one wide table each that the + /// split then copies apart. The words are the same. Each a power of two. + /// Under [`CutPadding::LastTable`], here and for KECCAK_RND's tables, the + /// ones of padding rows alone are not built. + pub fn cuts_at_finish( + mut self, + keccak: usize, + ecsm: usize, + ecdas: usize, + copy: usize, + ) -> Result { + for (name, rows) in [ + ("KECCAK", keccak), + ("ECSM", ecsm), + ("ECDAS", ecdas), + ("COPY", copy), + ] { + if !rows.is_power_of_two() { + return Err(Error::Prover(format!( + "{name} cut at {rows} rows: a power of two is needed" + ))); + } + } + self.finish_cut_rows = (keccak, ecsm, ecdas, copy); + Ok(self) + } + + /// `finish` packs each table it builds at the bytes its columns need as soon + /// as the table is generated ([`StreamSkip::pack`]), so it never holds its + /// tables at eight bytes a cell; the words are the same. KECCAK, ECSM and + /// ECDAS stay wide (a block cuts them afterwards). The streamed chunks are + /// the caller's. + pub fn pack_finished_tables(mut self) -> Self { + self.pack_finished = true; + self + } + + /// With [`Self::pack_finished_tables`], each table `finish` packs is + /// written packed as it is generated where its generator can (G-pack, + /// [`StreamSkip::gpack`]), instead of built at 8 bytes a cell and packed. + /// The bytes are the same. + pub fn generate_packed(mut self) -> Self { + self.gpack = true; + self + } + + /// Derive each window's phase-3 LT ops (from its MEMW and MEMW_A ops) as the + /// window arrives, into the window's LT list, so LT's full chunks stream with + /// them instead of waiting for the run's end. + /// + /// ⚠ BLOCK PATH ONLY. The LT ops are the whole-run build's, but in another + /// order — each window's MEMW-derived ops follow its walk's — so a chunk + /// holds other ops than the whole-run build's chunk at that index, and LT + /// deduplicates per chunk. Every LT op is in exactly one chunk with its + /// whole-run multiplicity summed over the chunks, which is what LT's + /// constraints and the bus see; the whole-run order (and so every epoch + /// proof) is untouched. The chunks are a function of the run and the window + /// length. + pub fn stream_memw_lt(mut self) -> Self { + self.stream_memw_lt = true; + self + } + + /// The streamed finish builds each table as soon as its byte gate admits + /// it, while earlier ones still build, instead of in waves that each wait + /// for their slowest table ([`super::finish::FinishPlan::emit_streamed_keeping`]). + /// The tables and their order are the same. + pub fn slide_rest(mut self) -> Self { + self.slide_rest = true; + self + } + + /// With [`Self::pack_finished_tables`] and KECCAK_RND cut at the finish, + /// `finish` holds each KECCAK_RND table as the keccak operations its rows + /// come from, for the card to fill ([`StreamSkip::card_fill`]), instead + /// of building its columns. The words are the same. + pub fn card_fill_keccak_rnd(mut self) -> Self { + self.card_fill = true; + self + } + + /// A test's fault ([`StreamSkip::card_fill_fault`]): with + /// [`Self::card_fill_keccak_rnd`], each KECCAK_RND table's records start + /// `rows` rows late, the same wrong table however they are filled. + #[cfg(test)] + pub(crate) fn card_fill_records_fault(mut self, rows: usize) -> Self { + self.card_fill_fault = rows; + self + } + + /// With [`Self::drop_streamed_ops`], hand out each full chunk of CPU + /// (with `cpu`) and MEMW_R (with `memw_r`) as the records its rows come + /// from, for the card to fill its columns as its group uploads it: a CPU + /// chunk as the executor's logs of its cycles (each walked window then + /// keeps its logs, [`WalkedWindow::keep_logs`]), a MEMW_R chunk as its + /// register rows. Chunks are held so while `room` says the host has room + /// for their records' bytes, and from the first it refuses on are filled + /// here as without it. Before the first window only. The words are the + /// same. + pub fn card_fill_streamed( + mut self, + room: RecordsRoom, + cpu: bool, + memw_r: bool, + ) -> Result { + if self.cycles > 0 { + return Err(Error::Prover( + "card_fill_streamed after a window was walked".to_string(), + )); + } + let Some(kept) = self.kept.as_mut() else { + return Err(Error::Prover( + "card_fill_streamed: the card fills the chunks of a build that drops its streamed \ + ops" + .to_string(), + )); + }; + let cpu = if cpu { + let decode = CardDecode::new(Arc::clone(&self.decode)).ok_or_else(|| { + Error::Prover("card_fill_streamed: a program with no instruction".to_string()) + })?; + kept.logs = Some(Tail::new()); + Some(Arc::new(decode)) + } else { + None + }; + self.card_fill_streamed = Some(StreamedCardFill { + cpu, + memw_r, + rows: RowFamilies::NONE, + room, + #[cfg(test)] + fault: 0, + }); + Ok(self) + } + + /// With [`Self::card_fill_streamed`], also hand out each full chunk of the + /// families in `rows` as its rows, the walk's own structs (MEMW_A's + /// aligned rows, LOAD's, STORE's, MEMW's and SHIFT's ops), for the card to + /// fill its columns from, under the same room rule. Before the first + /// window only. The words are the same. + pub fn card_fill_streamed_rows(mut self, rows: RowFamilies) -> Result { + if self.cycles > 0 { + return Err(Error::Prover( + "card_fill_streamed_rows after a window was walked".to_string(), + )); + } + let Some(card) = self.card_fill_streamed.as_mut() else { + return Err(Error::Prover( + "card_fill_streamed_rows: card_fill_streamed first".to_string(), + )); + }; + card.rows = rows; + Ok(self) + } + + /// A test's fault: with [`Self::card_fill_streamed`], each CPU chunk held + /// as its logs has every log's first source value (`src1_val`) and + /// written value (`dst_val`) XORed with `by` (the CPU table writes its own + /// timestamps, so a late cycle would move nothing), and each chunk held as its rows + /// ([`Self::card_fill_streamed_rows`]) has each row's timestamp `by` later: + /// the same wrong table however it is filled. + #[cfg(test)] + pub(crate) fn card_fill_streamed_fault(mut self, by: u64) -> Self { + if let Some(card) = self.card_fill_streamed.as_mut() { + card.fault = by; + } + self + } + + /// Whether each walked window keeps its logs ([`Self::card_fill_streamed`] + /// with CPU). + pub fn keeps_logs(&self) -> bool { + self.card_fill_streamed + .as_ref() + .is_some_and(|card| card.cpu.is_some()) + } + + /// Keep the LT ops derived from the MEMW and MEMW_A ops this builder drops + /// at 24 bytes each instead of compact ([`super::CompactLt`]), the A arm of + /// the compact form. Before the first window only. The tables are the same. + pub fn raw_memw_lt(mut self) -> Result { + if self.cycles > 0 { + return Err(Error::Prover( + "raw_memw_lt after a window was walked".to_string(), + )); + } + self.counted.memw_lt = super::CompactLt::raw(); + self.counted.memw_aligned_lt = super::CompactLt::raw(); + Ok(self) + } + + /// `finish` concatenates LT's ops into one list before chunking them + /// ([`StreamSkip::concat_lt`]), as it did before keeping them as segments. + /// The tables are the same. + pub fn concat_lt(mut self) -> Self { + self.emitted.concat_lt = true; + self + } + + /// `finish` counts the deduplicating tables' per-row lookups on the whole + /// pool ([`StreamSkip::p4_rows`]). The tables are the same. + pub fn p4_rows_on_pool(mut self) -> Self { + self.emitted.p4_rows = true; + self + } + + /// [`Self::p4_rows_on_pool`], each chunk counted by parts + /// ([`StreamSkip::p4_by`]). The tables are the same. + pub fn p4_rows_by_parts(mut self) -> Self { + self.emitted.p4_rows = true; + self.emitted.p4_by = super::RowCount::Parts; + self + } + + /// `finish` counts LT's per-op lookups and KECCAK's straight into its + /// histograms ([`StreamSkip::p4_direct`]). The tables are the same. + pub fn p4_lookups_direct(mut self) -> Self { + self.emitted.p4_direct = true; + self + } + + /// The streamed finish counts the BITWISE lookups beside the rest's + /// builds, in BITWISE's own table's job ([`StreamSkip::p4_beside`]). The + /// tables are the same. + pub fn p4_beside(mut self) -> Self { + self.emitted.p4_beside = true; + self + } + + /// [`Self::p4_beside`], the rest handed on with BITWISE after every other + /// table when the finish slides ([`StreamSkip::bitwise_last`]). The tables + /// are the same; their order, and so the groups, are not. + pub fn rest_bitwise_last(mut self) -> Self { + self.emitted.p4_beside = true; + self.emitted.bitwise_last = true; + self + } + + /// BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables counted + /// from each table's rows as it is built, phase 4 beside the rest's builds + /// and BITWISE handed on last ([`StreamSkip::bitwise_from_tables`]), when + /// the finish slides and keeps only the tails; otherwise the finish counts + /// and orders as without it. + pub fn bitwise_from_tables(mut self) -> Self { + self.emitted.bitwise_from_tables = true; + self + } + + /// Drop each streamed chunk's ops once the chunk is handed out, instead of + /// keeping every walked window to the run's end (see the module docs): the + /// builder then holds less than a chunk of each streamed table's ops, plus + /// the other tables' lists and (without [`Self::stream_memw_lt`]) the + /// phase-3 LT ops of the MEMW ops it dropped. The tables are the same. + /// Before the first window only. + pub fn drop_streamed_ops(mut self) -> Result { + if self.cycles > 0 { + return Err(Error::Prover( + "drop_streamed_ops after a window was walked".to_string(), + )); + } + let decode = self.decode_trace.take().ok_or_else(|| { + Error::Prover("drop_streamed_ops: DECODE's trace was handed over already".to_string()) + })?; + self.kept = Some(Kept::new(decode, &self.max_rows)); + Ok(self) + } + + /// Per streamed table (CPU, MEMW_R, MEMW_A, MEMW, LOAD, LT, SHIFT, STORE), + /// the ops the builder holds that were not handed out in a chunk, and the + /// ops its held lists take room for (those plus a part's handed-out head): + /// under [`Self::drop_streamed_ops`] only; `None` otherwise. + pub fn held_streamed_ops(&self) -> Option<[(usize, usize); 8]> { + let kept = self.kept.as_ref()?; + let store = self.segments.store_ops.len(); + Some([ + kept.cpu.held(), + kept.register_rows.held(), + kept.aligned.held(), + kept.general.held(), + kept.load.held(), + kept.lt.held(), + kept.shift.held(), + (store, store), + ]) + } + + /// The bytes the initial memory image (its pages) takes on the heap. + pub fn image_bytes(&self) -> usize { + self.image.heap_bytes() + } + + /// Where the builder's creation spent its time. + pub fn setup_stamps(&self) -> SetupStamps { + self.setup + } + + /// The time the windows took so far. + pub fn stamps(&self) -> WindowStamps { + self.stamps + } + + /// How many chunks of each streamed table have been handed out. + pub fn streamed(&self) -> StreamSkip { + self.emitted + } + + /// What the builder holds on the heap, list by list (capacities): the + /// kept tails and lists, the kept windows, the routed segments, what was + /// counted ahead and the walk's memory state. A measurement + /// (`LAMBDA_VM_BLOCK_MEMLOG`); it changes nothing. + pub fn heap_parts(&self) -> Vec<(String, usize)> { + let mut parts = self.kept.as_ref().map_or_else(Vec::new, Kept::heap_parts); + parts.push(( + "windows".to_string(), + self.windows.iter().map(WalkedWindow::heap_bytes).sum(), + )); + parts.extend(self.segments.heap_parts()); + parts.extend(self.counted.heap_parts()); + parts.push(("memory state".to_string(), self.memory_state.heap_bytes())); + parts + } + + /// Each list the builder holds, its largest single allocation (a block, a + /// window's list, or the list): the most one append or one list's growth + /// can allocate at once. A measurement (`LAMBDA_VM_BLOCK_MEMLOG`). + pub fn largest_parts(&self) -> Vec<(String, usize)> { + let mut parts = self + .kept + .as_ref() + .map_or_else(Vec::new, Kept::largest_parts); + parts.push(( + "windows".to_string(), + self.windows + .iter() + .map(WalkedWindow::heap_bytes) + .max() + .unwrap_or(0), + )); + parts.extend(self.segments.largest_parts()); + parts.extend(self.counted.largest_parts()); + parts + } + + /// Collects the run's last window and builds every table the windows did + /// not stream. The streamed chunks' slots hold empty placeholders; + /// [`Traces::insert_streamed`] puts the chunks back. + pub fn finish(self, logs: &[Log]) -> Result { + let Finishing { + ops, + image, + memory_state, + register_init, + decode_trace, + decode, + register_state, + max_rows, + private_input, + skip, + pre, + } = self.finishing(logs)?; + build_traces( + ops, + Some(&image), + &memory_state, + ®ister_init, + decode_trace, + &decode, + register_state, + &max_rows, + private_input, + true, + false, + &skip, + Some(pre), + ) + } + + /// [`Self::finish`] streamed: the run's last window collected and the + /// table phase planned, `on_header` given the header (what a statement is + /// read from) before any table is built, then every table the windows did + /// not stream, and every streamed chunk's slot, handed to the sink it + /// returns in AIR order, under `gate` ([`super::finish::FinishPlan::emit_streamed`]). + #[cfg(test)] + pub(crate) fn finish_streamed<'s>( + self, + logs: &[Log], + gate: &super::gate::ByteGate, + on_header: impl FnOnce( + super::RestHeader, + ) -> Result< + Box Result<(), Error> + 's>, + Error, + >, + ) -> Result<(), Error> { + self.finish_streamed_keeping(logs, gate, false, on_header) + .map(|_| ()) + } + + /// [`Self::finish_streamed`], and with `keep` what phase B needs to build + /// KECCAK_RND's and LT's tables again ([`super::RestRegen`]). + pub(crate) fn finish_streamed_keeping<'s>( + self, + logs: &[Log], + gate: &super::gate::ByteGate, + keep: bool, + on_header: impl FnOnce( + super::RestHeader, + ) -> Result< + Box Result<(), Error> + 's>, + Error, + >, + ) -> Result, Error> { + let slide = self.slide_rest; + let Finishing { + ops, + image, + memory_state, + register_init, + decode_trace, + decode, + register_state, + max_rows, + private_input, + mut skip, + pre, + } = self.finishing(logs)?; + // BITWISE's lookups from the tables need BITWISE last (the slide, with + // the `parallel` feature) and only the streamed chunks' tails kept: + // their full chunks were counted as they were handed out. They count + // phase 4's other sources beside the rest's builds. + let from_tables = + skip.bitwise_from_tables && skip.tails && slide && cfg!(feature = "parallel"); + skip.p4_beside |= from_tables; + let mut plan = super::finish::FinishPlan::new( + ops, + Some(&image), + &memory_state, + ®ister_init, + decode_trace, + &decode, + register_state, + &max_rows, + private_input, + true, + false, + &skip, + Some(pre), + )?; + if let Some(line) = plan.cut_line() { + eprintln!("{line}"); + } + // Without the `parallel` feature the finish takes the waves, in AIR + // order. + if (skip.bitwise_last || from_tables) && slide && cfg!(feature = "parallel") { + plan.hand_bitwise_last(); + } + if from_tables { + plan.count_bitwise_from_tables(); + } + let mut send = match on_header(plan.header().clone()) { + Ok(send) => send, + Err(e) => { + gate.stop("the header found no taker"); + return Err(e); + } + }; + plan.emit_streamed_keeping(gate, &mut *send, keep, slide) + } + + /// The run's last window collected and walked, and the run's lists + /// assembled: what the table phase starts from. + fn finishing(mut self, logs: &[Log]) -> Result, Error> { + // Every handed-out chunk counted, wherever it was counted. + self.join_handout_count()?; + // The last window may halt: it is walked here, not by the `Walker`. + let decode = &self.decode; + let cpu_ops = super::collect_cpu_ops_from(logs, decode, self.cycles)?; + self.cycles += logs.len(); + let mut walk = WalkOutputs::with_capacity(cpu_ops.len()); + // The last window's in-walk lookups are listed and counted by the + // table phase, lean or not. + collect_ops_from_cpu_into( + &cpu_ops, + decode, + &mut self.memory_state, + &mut self.register_state, + &mut walk, + true, + )?; + let route = if self.lean.route { + route_ops_one_pass + } else { + route_ops + }; + self.segments + .append(route(&cpu_ops, &self.decode, &walk.cpu32_ops)); + let last = WalkedWindow { + cpu_ops, + walk, + logs: None, + }; + super::build_stamps::mark("p0 last window"); + let Self { + decode, + decode_trace, + image, + register_init, + max_rows, + private_input, + memory_state, + mut register_state, + mut windows, + segments, + emitted, + counted, + kept, + stream_memw_lt, + finish_keccak_rnd_rows, + finish_cut_rows, + pack_finished, + gpack, + cut_padding, + card_fill, + #[cfg(test)] + card_fill_fault, + .. + } = self; + // Chunks at the finish alone: none were handed out. + let emitted = if emitted.keccak_rnd_rows == 0 && finish_keccak_rnd_rows > 0 { + StreamSkip { + keccak_rnd_rows: finish_keccak_rnd_rows, + keccak_rnd: 0, + ..emitted + } + } else { + emitted + }; + let (keccak_rows, ecsm_rows, ecdas_rows, copy_rows) = finish_cut_rows; + let emitted = StreamSkip { + pack: pack_finished, + gpack: pack_finished && gpack, + card_fill: pack_finished && card_fill, + #[cfg(test)] + card_fill_fault, + keccak_rows, + ecsm_rows, + ecdas_rows, + copy_rows, + cut_padding, + ..emitted + }; + let (ops, decode_trace, skip, pre) = match kept { + None => { + windows.push(last); + let (cpu_ops, mut walk) = concatenate(windows); + super::build_stamps::mark("p0 concatenate"); + let rest = KeptRest::from_walk(&mut walk); + let ops = assemble(cpu_ops, walk, rest, segments, &mut register_state); + super::build_stamps::mark("p0 assemble"); + let decode_trace = decode_trace + .ok_or_else(|| { + Error::Prover( + "the finish: DECODE's trace is neither the builder's nor the kept run's" + .to_string(), + ) + })? + .into_table(); + (ops, decode_trace, emitted, counted) + } + Some(mut kept) => { + // The last window's ops, its in-walk lookups and its MEMW + // ops' LT ops are all still to count, as the tails are. + kept.append(last); + let (cpu_ops, walk, rest, decode_trace) = kept.into_run(); + super::build_stamps::mark("p0 concatenate"); + let ops = assemble(cpu_ops, walk, rest, segments, &mut register_state); + super::build_stamps::mark("p0 assemble"); + // With the MEMW-derived LT ops streamed, `memw_lt_done` counts + // the dropped ops too; without, it is zero and the dropped + // ops' LT ops are in `counted`. + let converted = |done: usize, chunks: usize, max: usize| { + if !stream_memw_lt { + return Ok(done); + } + done.checked_sub(chunks * max).ok_or_else(|| { + Error::Prover(format!( + "{chunks} MEMW chunks of {max} were dropped but only {done} ops' LT ops \ + were derived" + )) + }) + }; + let skip = StreamSkip { + tails: true, + memw_lt_done: converted(emitted.memw_lt_done, emitted.memw, max_rows.memw)?, + memw_aligned_lt_done: converted( + emitted.memw_aligned_lt_done, + emitted.memw_aligned, + max_rows.memw_aligned, + )?, + ..emitted + }; + let pre = PreCounted { + bitwise_ops: 0, + memw_register_rows: 0, + ..counted + }; + (ops, decode_trace, skip, pre) + } + }; + Ok(Finishing { + ops, + image, + memory_state, + register_init, + decode_trace, + decode, + register_state, + max_rows, + private_input, + skip, + pre, + }) + } + + /// Hands the builder the state a run walked from its entries holds at its + /// last window's start (`memory`, `registers`, that window's first cycle + /// `cycles`) and the walk's stamps, so [`Self::finish`] walks the last + /// window as though this builder had walked every window before it. + #[cfg(feature = "parallel")] + fn install_walked( + &mut self, + mut memory: MemoryState, + registers: RegisterState, + cycles: usize, + windows: usize, + walk_secs: f64, + ) { + memory.lean = self.lean.memory; + self.memory_state = memory; + self.register_state = registers; + self.cycles = cycles; + self.stamps.windows += windows; + self.stamps.walk += walk_secs; + } + + /// The bytes the walk's carried memory state takes on the heap. + #[cfg(feature = "parallel")] + pub(crate) fn state_bytes(&self) -> usize { + self.memory_state.heap_bytes() + } + + /// The accumulating half alone, with the run's initial image: what a run + /// walked from its entries ([`parallel::walk_windows`]) absorbs its windows + /// with and builds the finish's state from. + #[cfg(feature = "parallel")] + fn accumulator_and_image(&mut self) -> (Accumulator<'_>, &PagedMem) { + ( + Accumulator { + lean: self.lean, + stream_memw_lt: self.stream_memw_lt, + max_rows: &self.max_rows, + decode: &self.decode, + kept: self.kept.as_mut(), + windows: &mut self.windows, + segments: &mut self.segments, + emitted: &mut self.emitted, + counted: &mut self.counted, + route_secs: &mut self.stamps.route, + handout_secs: &mut self.stamps.generate, + handout: &mut self.stamps.handout, + card: self.card_fill_streamed.as_ref(), + card_stamps: &mut self.stamps.card_fill, + count: &mut self.count, + }, + &self.image, + ) + } + + /// The builder in its two halves, to run on two threads: the [`Walker`] + /// walks each window (the state it carries is the walk's), the + /// [`Accumulator`] keeps each walked window, routes it and hands out the + /// chunks it completed. Fed the windows in order, the two build exactly + /// what [`push_jobs`](Self::push_jobs) builds; afterwards + /// [`finish`](Self::finish) takes the last window as usual. + pub fn split(&mut self) -> (Walker<'_>, Accumulator<'_>) { + ( + Walker { + decode: &self.decode, + lookups: !self.lean.lookups, + keep_logs: self.keeps_logs(), + batch: self.walk_batch, + sizes: &mut self.walk_sizes, + memory_state: &mut self.memory_state, + register_state: &mut self.register_state, + cycles: &mut self.cycles, + windows: &mut self.stamps.windows, + walk_secs: &mut self.stamps.walk, + }, + Accumulator { + lean: self.lean, + stream_memw_lt: self.stream_memw_lt, + max_rows: &self.max_rows, + decode: &self.decode, + kept: self.kept.as_mut(), + windows: &mut self.windows, + segments: &mut self.segments, + emitted: &mut self.emitted, + counted: &mut self.counted, + route_secs: &mut self.stamps.route, + handout_secs: &mut self.stamps.generate, + handout: &mut self.stamps.handout, + card: self.card_fill_streamed.as_ref(), + card_stamps: &mut self.stamps.card_fill, + count: &mut self.count, + }, + ) + } +} + +/// One window, walked: its CPU ops and what the walk emitted for them, and, +/// for a build whose CPU chunks the card fills +/// ([`WindowedTraceBuilder::card_fill_streamed`]), the executor's logs it was +/// walked from. +pub struct WalkedWindow { + cpu_ops: Vec, + walk: WalkOutputs, + logs: Option>>, +} + +impl WalkedWindow { + /// The bytes its lists take on the heap (capacities), its logs included. + pub fn heap_bytes(&self) -> usize { + super::vec_heap_bytes(&self.cpu_ops) + + self.walk.heap_bytes() + + self.logs.as_deref().map_or(0, super::vec_heap_bytes) + } + + /// Keeps `logs`, the logs it was walked from, for the CPU chunks its + /// cycles fall in ([`WindowedTraceBuilder::card_fill_streamed`]). + pub fn keep_logs(&mut self, logs: Arc>) { + self.logs = Some(logs); + } + + /// Its cycles. + pub fn cycles(&self) -> usize { + self.cpu_ops.len() + } +} + +/// The walk half of a split builder ([`WindowedTraceBuilder::split`]). +pub struct Walker<'b> { + /// Each cycle's decode, by DECODE row. + decode: &'b DecodeTable, + /// List the in-walk lookups (`false` under [`WalkLean::lookups`]). + lookups: bool, + /// Keep each window's logs ([`WindowedTraceBuilder::keeps_logs`]). + keep_logs: bool, + /// The cycles collected and walked at a time (`0`: the whole window). + batch: usize, + /// The last window's list lengths ([`walk_sized`]). + sizes: &'b mut WalkSizes, + memory_state: &'b mut MemoryState, + register_state: &'b mut RegisterState, + cycles: &'b mut usize, + windows: &'b mut usize, + walk_secs: &'b mut f64, +} + +impl Walker<'_> { + /// Walks the next window (not the run's last) into fresh lists. + pub fn walk(&mut self, logs: &[Log]) -> Result { + let t = std::time::Instant::now(); + let walked = walk_window( + self.decode, + self.lookups, + self.batch, + self.sizes, + self.memory_state, + self.register_state, + *self.cycles, + logs, + )?; + *self.cycles += logs.len(); + *self.walk_secs += t.elapsed().as_secs_f64(); + *self.windows += 1; + Ok(walked) + } + + /// [`Self::walk`] of logs it may keep: a build whose CPU chunks the card + /// fills keeps them with the window ([`WindowedTraceBuilder::keeps_logs`]); + /// any other lets them go here. + pub fn walk_owned(&mut self, logs: Vec) -> Result { + let mut walked = self.walk(&logs)?; + if self.keep_logs { + walked.keep_logs(Arc::new(logs)); + } + Ok(walked) + } + + /// The bytes the walk's carried memory state takes on the heap. + pub fn state_bytes(&self) -> usize { + self.memory_state.heap_bytes() + } +} + +/// One window (not the run's last) walked from `memory` and `registers` into +/// fresh lists, its cycles numbered from `first`: [`Walker::walk`]'s body, and +/// what a window walked from its entries runs. +#[allow(clippy::too_many_arguments)] +fn walk_window( + decode: &DecodeTable, + lookups: bool, + batch: usize, + sizes: &mut WalkSizes, + memory: &mut MemoryState, + registers: &mut RegisterState, + first: usize, + logs: &[Log], +) -> Result { + // As a whole-run build: only the last cycle may halt. + if logs.iter().any(|log| log.next_pc == 0) { + return Err(Error::HaltInNonFinalEpoch); + } + let mut cpu_ops = Vec::with_capacity(logs.len()); + // Without `lookups`, the accumulator counts them (`absorb`). + let mut walk = if walk_sized() { + WalkOutputs::for_walk_sized(logs.len(), lookups, sizes) + } else { + WalkOutputs::for_walk(logs.len(), lookups) + }; + // The walk is a left fold over the CPU ops: walking each batch right + // after collecting it gives the window's lists, with the batch's ops + // still in cache instead of read back from the whole window's list. + let batch = match batch { + 0 => logs.len().max(1), + n => n, + }; + for part in logs.chunks(batch) { + let from = cpu_ops.len(); + super::collect_cpu_ops_into(part, decode, first + from, &mut cpu_ops)?; + collect_ops_from_cpu_into( + &cpu_ops[from..], + decode, + memory, + registers, + &mut walk, + lookups, + )?; + } + *sizes = WalkSizes::of(&walk, logs.len()); + Ok(WalkedWindow { + cpu_ops, + walk, + logs: None, + }) +} + +/// The accumulating half of a split builder ([`WindowedTraceBuilder::split`]). +pub struct Accumulator<'b> { + /// The lean parts the builder uses: with `lookups` the walker left each + /// window's in-walk lookups out, with `route` the routing is one pass. + lean: WalkLean, + stream_memw_lt: bool, + max_rows: &'b crate::tables::MaxRowsConfig, + /// The CPU ops' decode, by DECODE row (shared with the CPU chunks' jobs). + decode: &'b Arc, + kept: Option<&'b mut Kept>, + windows: &'b mut Vec, + segments: &'b mut RoutedBlocks, + emitted: &'b mut StreamSkip, + counted: &'b mut PreCounted, + route_secs: &'b mut f64, + handout_secs: &'b mut f64, + handout: &'b mut HandoutStamps, + /// [`WindowedTraceBuilder::card_fill_streamed`]. + card: Option<&'b StreamedCardFill>, + card_stamps: &'b mut CardFillStamps, + /// Where the hand-out's counting runs. + count: &'b mut CountMode, +} + +impl Accumulator<'_> { + /// The bytes the builder's run so far takes on the heap: the kept windows + /// or tails, the routed segments and what was counted ahead (lists at + /// their capacities). + pub fn held_bytes(&self) -> usize { + let windows: usize = self.windows.iter().map(WalkedWindow::heap_bytes).sum(); + self.kept.as_deref().map_or(0, Kept::heap_bytes) + + windows + + self.segments.heap_bytes() + + self.counted.heap_bytes() + } + + /// [`Self::absorb`], and the ops `window` added to each streamed table's + /// list, in [`StreamTable::ALL`] order without KECCAK_RND (the MEMW-derived + /// LT ops of [`WindowedTraceBuilder::stream_memw_lt`] aside). + pub fn absorb_counted(&mut self, window: WalkedWindow) -> (Vec, [usize; 8]) { + let walk = &window.walk; + let mut added = [ + window.cpu_ops.len(), + walk.memw.register_rows.len(), + walk.memw.aligned.len(), + walk.memw.general.len(), + walk.load_ops.len(), + walk.lt_ops.len(), + walk.shift_ops.len(), + 0, + ]; + let stores_before = self.segments.store_ops.len(); + let jobs = self.absorb(window); + // STORE's segment loses the ops of the chunks cut from it when the + // builder drops its streamed ops. + let cut: usize = if self.kept.is_some() { + jobs.iter() + .filter(|job| job.table == StreamTable::Store) + .map(ChunkJob::op_count) + .sum() + } else { + 0 + }; + added[7] = self.segments.store_ops.len() + cut - stores_before; + (jobs, added) + } + + /// Routes a walked window, keeps it, and hands out the chunks the run's + /// lists now complete. Windows must come in run order. + pub fn absorb(&mut self, mut window: WalkedWindow) -> Vec { + let t = std::time::Instant::now(); + if self.stream_memw_lt { + let walk = &mut window.walk; + walk.lt_ops + .extend(super::collect_lt_from_memw(&walk.memw.general)); + walk.lt_ops + .extend(super::collect_lt_from_memw_aligned(&walk.memw.aligned)); + self.emitted.memw_lt_done += walk.memw.general.len(); + self.emitted.memw_aligned_lt_done += walk.memw.aligned.len(); + } + // The table phase's dominant BITWISE source, counted now: from the CPU + // and LOAD ops when the walk left the lookups out (the list is then + // empty), from the list otherwise. The one-pass routing appends to the + // run's segments and counts the CPU ops' lookups as it goes. + if self.lean.route && self.lean.lookups && route_fused() { + super::route_ops_into( + &window.cpu_ops, + self.decode, + &window.walk.cpu32_ops, + self.segments, + &mut self.counted.histogram, + ); + } else { + let route = if self.lean.route { + route_ops_one_pass + } else { + route_ops + }; + self.segments + .append(route(&window.cpu_ops, self.decode, &window.walk.cpu32_ops)); + if self.lean.lookups { + let histogram = &mut self.counted.histogram; + window + .cpu_ops + .iter() + .for_each(|op| self.decode.op(op).count_bitwise_into(histogram)); + } + } + if self.lean.lookups { + let histogram = &mut self.counted.histogram; + window + .walk + .load_ops + .iter() + .for_each(|op| op.count_bitwise_into(histogram)); + } + self.counted.histogram.add_ops(&window.walk.bitwise_ops); + self.counted.bitwise_ops += window.walk.bitwise_ops.len(); + if let Some(kept) = self.kept.as_deref_mut() { + // Nothing else reads the in-walk lookups; MEMW_R's rows are + // counted as their chunks leave. + drop(std::mem::take(&mut window.walk.bitwise_ops)); + kept.append(window); + *self.route_secs += t.elapsed().as_secs_f64(); + let t = std::time::Instant::now(); + let counter = self + .count + .counter(self.decode, self.stream_memw_lt, self.counted); + let jobs = tail_jobs( + self.max_rows, + self.stream_memw_lt, + self.decode, + kept, + self.segments, + self.emitted, + self.counted, + self.handout, + self.card.map(|card| (card, &mut *self.card_stamps)), + counter, + ); + *self.handout_secs += t.elapsed().as_secs_f64(); + jobs + } else { + // And the second, MEMW_R. + super::memw_register::collect_bitwise_from_memw_register( + &window.walk.memw.register_rows, + &mut self.counted.histogram, + ); + self.counted.memw_register_rows += window.walk.memw.register_rows.len(); + // Every chunk of a run that keeps its windows is filled here. + window.logs = None; + self.windows.push(window); + *self.route_secs += t.elapsed().as_secs_f64(); + let t = std::time::Instant::now(); + let jobs = chunk_jobs( + self.max_rows, + self.decode, + self.windows, + self.segments, + self.emitted, + ); + *self.handout_secs += t.elapsed().as_secs_f64(); + jobs + } + } +} + +/// `range` of the run's list that `list` picks out of each window, gathered +/// into one vector (the windows' lists concatenate to the run's). +fn gather( + windows: &[WalkedWindow], + list: impl Fn(&WalkedWindow) -> &[T], + range: std::ops::Range, +) -> Vec { + let mut out = Vec::with_capacity(range.len()); + let mut at = 0usize; + for window in windows { + let part = list(window); + let (lo, hi) = (at, at + part.len()); + if hi > range.start && lo < range.end { + let from = range.start.max(lo) - lo; + let to = range.end.min(hi) - lo; + out.extend_from_slice(&part[from..to]); + } + at = hi; + if at >= range.end { + break; + } + } + out +} + +// The run's lists, as each window holds its part. +fn list_cpu(w: &WalkedWindow) -> &[super::CpuOperation] { + &w.cpu_ops +} +fn list_register_rows(w: &WalkedWindow) -> &[super::RegRow] { + &w.walk.memw.register_rows +} +fn list_aligned(w: &WalkedWindow) -> &[memw_aligned::AlignedRow] { + &w.walk.memw.aligned +} +fn list_general(w: &WalkedWindow) -> &[super::MemwOperation] { + &w.walk.memw.general +} +fn list_load(w: &WalkedWindow) -> &[super::LoadOperation] { + &w.walk.load_ops +} +fn list_lt(w: &WalkedWindow) -> &[super::LtOperation] { + &w.walk.lt_ops +} +fn list_shift(w: &WalkedWindow) -> &[super::ShiftOperation] { + &w.walk.shift_ops +} +fn list_keccak(w: &WalkedWindow) -> &[super::keccak::KeccakOperation] { + &w.walk.keccak_ops +} + +/// Every full chunk the run's lists hold that was not handed out yet, its ops +/// gathered out of the windows. +fn chunk_jobs( + m: &crate::tables::MaxRowsConfig, + decode: &Arc, + windows: &[WalkedWindow], + segments: &RoutedBlocks, + e: &mut StreamSkip, +) -> Vec { + let mut jobs = Vec::new(); + macro_rules! full_chunks { + ($table:expr, $variant:ident, $list:expr, $max:expr, $done:expr) => {{ + let list = $list; + let len: usize = windows.iter().map(|w| list(w).len()).sum(); + let max = $max; + while ($done + 1) * max <= len { + let index = $done; + jobs.push(ChunkJob { + table: $table, + index, + ops: ChunkOps::$variant(Segments::one(gather( + windows, + list, + index * max..(index + 1) * max, + ))), + }); + $done += 1; + } + }}; + } + { + let len: usize = windows.iter().map(|w| w.cpu_ops.len()).sum(); + while (e.cpu + 1) * m.cpu <= len { + let index = e.cpu; + jobs.push(ChunkJob { + table: StreamTable::Cpu, + index, + ops: ChunkOps::Cpu( + Segments::one(gather( + windows, + list_cpu, + index * m.cpu..(index + 1) * m.cpu, + )), + Arc::clone(decode), + ), + }); + e.cpu += 1; + } + } + full_chunks!( + StreamTable::MemwRegister, + MemwRegister, + list_register_rows, + m.memw_register, + e.memw_register + ); + full_chunks!( + StreamTable::MemwAligned, + MemwAligned, + list_aligned, + m.memw_aligned, + e.memw_aligned + ); + full_chunks!(StreamTable::Memw, Memw, list_general, m.memw, e.memw); + full_chunks!(StreamTable::Load, Load, list_load, m.load, e.load); + full_chunks!(StreamTable::Lt, Lt, list_lt, m.lt, e.lt); + full_chunks!(StreamTable::Shift, Shift, list_shift, m.shift, e.shift); + // KECCAK_RND, cut at `keccak_rnd_rows` rows: a chunk is out once the ops so + // far cover all of its rows (the run's padding rows only reach its last). + if e.keccak_rnd_rows > 0 { + let rows = e.keccak_rnd_rows; + let len: usize = windows.iter().map(|w| list_keccak(w).len()).sum(); + while (e.keccak_rnd + 1) * rows <= len * 24 { + let index = e.keccak_rnd; + let (first, end) = super::keccak_rnd_op_range(index, rows, len); + jobs.push(ChunkJob { + table: StreamTable::KeccakRnd, + index, + ops: ChunkOps::KeccakRnd { + ops: gather(windows, list_keccak, first..end), + skip: index * rows - first * 24, + rows, + }, + }); + e.keccak_rnd += 1; + } + } + // STORE's ops are a routing segment, already one list. + let store = &segments.store_ops; + while (e.store + 1) * m.store <= store.len() { + let index = e.store; + jobs.push(ChunkJob { + table: StreamTable::Store, + index, + ops: ChunkOps::Store( + store + .range(index * m.store, (index + 1) * m.store) + .into_owned(), + ), + }); + e.store += 1; + } + jobs +} + +/// Hands the whole pages inside `list`'s allocation back to the OS +/// (`madvise(MADV_DONTNEED)` on its page-aligned interior), right before the +/// list is freed: the bytes handed back, or the OS's errno. The range stays +/// mapped and reads as zeros if touched again, so the allocator's free and any +/// later reuse of it stay valid; the list's values are not read again. Only on +/// Linux, and only for values with nothing to drop (whose free reads nothing); +/// otherwise nothing is handed back. +#[cfg(target_os = "linux")] +fn return_list_pages(list: &mut Vec) -> Result { + if std::mem::needs_drop::() { + return Ok(0); + } + // SAFETY: sysconf has no preconditions. + let page = match unsafe { libc::sysconf(libc::_SC_PAGESIZE) } { + n if n > 0 => n as usize, + _ => 4096, + }; + let start = list.as_mut_ptr() as usize; + let end = start + list.capacity() * std::mem::size_of::(); + let (lo, hi) = (start.next_multiple_of(page), end / page * page); + if hi <= lo { + return Ok(0); + } + // SAFETY: [lo, hi) lies inside the list's own allocation, which the caller + // alone holds (`Arc::get_mut`) and frees next without reading it; `T` has + // nothing to drop, so freeing it reads nothing either. MADV_DONTNEED leaves the + // range mapped (zero-filled on the next touch), so the allocator's + // metadata, kept outside the allocation, and its free stay valid. + let rc = unsafe { libc::madvise(lo as *mut libc::c_void, hi - lo, libc::MADV_DONTNEED) }; + if rc == 0 { + Ok((hi - lo) as u64) + } else { + Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(-1)) + } +} + +/// Off Linux nothing is handed back. +#[cfg(not(target_os = "linux"))] +fn return_list_pages(_list: &mut Vec) -> Result { + Ok(0) +} + +/// A chunk's ops as the window parts they lie in, in run order: each part is +/// shared with the tail (and with the chunk on its other side when it straddles +/// a chunk boundary), so a chunk leaves the tail without a copy and its +/// generator fills the rows from the parts. +pub(crate) struct Segments { + parts: Vec<(Arc>, std::ops::Range)>, +} + +/// The same parts, shared (no op is copied). +impl Clone for Segments { + fn clone(&self) -> Self { + Self { + parts: self + .parts + .iter() + .map(|(list, range)| (Arc::clone(list), range.clone())) + .collect(), + } + } +} + +impl Segments { + /// One list, as one part. + fn one(ops: Vec) -> Self { + let len = ops.len(); + Self { + parts: vec![(Arc::new(ops), 0..len)], + } + } + + /// The ops of each part, in order. + fn slices(&self) -> Vec<&[T]> { + self.parts + .iter() + .map(|(part, range)| &part[range.clone()]) + .collect() + } + + /// The ops, in order. + fn iter(&self) -> impl DoubleEndedIterator { + self.parts + .iter() + .flat_map(|(part, range)| part[range.clone()].iter()) + } + + /// Its ops. + fn len(&self) -> usize { + self.parts.iter().map(|(_, range)| range.len()).sum() + } + + /// The bytes its ops take (the ranges, not the parts they lie in). + fn op_bytes(&self) -> usize { + self.len() * std::mem::size_of::() + } + + /// Hands back to the OS the pages of each part's list this is the last + /// holder of ([`return_list_pages`]), as the chunk is let go: nothing reads + /// them after, their holder drops them next. A list another chunk still + /// holds stays as it is. + pub(crate) fn return_pages(&mut self) -> multilinear::records::PagesReturned { + let mut got = multilinear::records::PagesReturned::default(); + for (list, _) in &mut self.parts { + let Some(list) = Arc::get_mut(list) else { + got.shared += 1; + continue; + }; + got.buffers += 1; + match return_list_pages(list) { + Ok(bytes) => got.bytes += bytes, + Err(errno) => { + got.failed += 1; + static SAID: std::sync::Once = std::sync::Once::new(); + SAID.call_once(|| { + eprintln!( + "BLOCK CARD FILL released: madvise refused a records list's pages \ + (errno {errno}); its pages stay with the allocator" + ); + }); + } + } + } + got + } + + /// The ops as one list: moved out of a sole whole part, copied otherwise. + fn into_one(self) -> Vec + where + T: Clone, + { + let mut parts = self.parts; + if parts.len() == 1 && parts[0].1 == (0..parts[0].0.len()) { + let (list, _) = parts.pop().expect("one part"); + return Arc::try_unwrap(list).unwrap_or_else(|shared| shared.to_vec()); + } + parts + .iter() + .flat_map(|(list, range)| list[range.clone()].iter().cloned()) + .collect() + } +} + +/// Whether the accumulator routes a window into the run's segments in place +/// and counts its CPU ops' lookups in the same pass (unset or `1`), or routes +/// it into segments of its own, appends them, then counts +/// (`LAMBDA_VM_ROUTE_FUSED=0`). Read once. The segments and counts are the same. +fn route_fused() -> bool { + static FUSED: std::sync::OnceLock = std::sync::OnceLock::new(); + *FUSED.get_or_init(|| !std::env::var("LAMBDA_VM_ROUTE_FUSED").is_ok_and(|v| v.trim() == "0")) +} + +/// Whether the walker sizes a window's streamed lists from the last window's +/// lengths (unset or `1`) or from fixed per-cycle guesses, growing them as it +/// goes (`LAMBDA_VM_WALK_SIZED=0`). Read once. The lists are the same. +fn walk_sized() -> bool { + static SIZED: std::sync::OnceLock = std::sync::OnceLock::new(); + *SIZED.get_or_init(|| !std::env::var("LAMBDA_VM_WALK_SIZED").is_ok_and(|v| v.trim() == "0")) +} + +/// The cycles the walker collects and walks at a time (`LAMBDA_VM_WALK_BATCH`, +/// default 2048; `0` collects the whole window's CPU ops, then walks them). +/// Read once. The lists are the same. +fn walk_batch() -> usize { + static BATCH: std::sync::OnceLock = std::sync::OnceLock::new(); + *BATCH.get_or_init(|| { + std::env::var("LAMBDA_VM_WALK_BATCH") + .ok() + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(2048) + }) +} + +/// Whether the hand-out copies each chunk's ops out of the window parts +/// (`LAMBDA_VM_HANDOUT_COPY=1`, the control) instead of handing out the parts +/// they lie in. Read once. The chunks are the same. +fn handout_copies() -> bool { + static COPY: std::sync::OnceLock = std::sync::OnceLock::new(); + *COPY.get_or_init(|| std::env::var("LAMBDA_VM_HANDOUT_COPY").is_ok_and(|v| v.trim() == "1")) +} + +/// A streamed table's ops not yet in a handed-out chunk +/// ([`WindowedTraceBuilder::drop_streamed_ops`]): the windows' lists, moved in +/// whole, of which the first `start` ops of the first have left. +struct Tail { + parts: VecDeque>>, + start: usize, + len: usize, +} + +impl Tail { + fn new() -> Self { + Self { + parts: VecDeque::new(), + start: 0, + len: 0, + } + } + + fn push(&mut self, part: Vec) { + self.push_shared(Arc::new(part)); + } + + /// [`Self::push`] of a part held elsewhere too. + fn push_shared(&mut self, part: Arc>) { + if !part.is_empty() { + self.len += part.len(); + self.parts.push_back(part); + } + } + + /// The first `n` ops (at most `len`), as the parts they lie in: a part the + /// chunk uses up leaves the tail, a part it uses the front of stays shared. + /// Under [`handout_copies`], copied into one list (a part that is exactly + /// the chunk moves out whole). + fn take(&mut self, n: usize) -> Segments { + self.take_counted(n).0 + } + + /// [`Self::take`], and how many of the ops it copied (none as parts, nor + /// when a part moved out whole). + fn take_counted(&mut self, n: usize) -> (Segments, usize) { + if handout_copies() { + let (ops, copied) = self.take_copy_counted(n); + return (Segments::one(ops), copied); + } + (self.take_segments(n), 0) + } + + /// The first `n` ops (at most `len`) as the parts they lie in, whatever + /// [`handout_copies`] says. + fn take_segments(&mut self, n: usize) -> Segments { + let mut want = n.min(self.len); + self.len -= want; + let mut parts = Vec::new(); + while want > 0 { + let Some(front) = self.parts.front() else { + break; + }; + let available = front.len() - self.start; + if available <= want { + let range = self.start..front.len(); + if let Some(part) = self.parts.pop_front() { + parts.push((part, range)); + } + self.start = 0; + want -= available; + } else { + parts.push((Arc::clone(front), self.start..self.start + want)); + self.start += want; + want = 0; + } + } + Segments { parts } + } + + /// The first `n` ops (at most `len`), copied into one list. + fn take_copy(&mut self, n: usize) -> Vec { + self.take_copy_counted(n).0 + } + + /// [`Self::take_copy`], and how many of the ops it copied (none when a + /// part that is exactly the chunk moved out whole). + fn take_copy_counted(&mut self, n: usize) -> (Vec, usize) { + let n = n.min(self.len); + self.len -= n; + if self.start == 0 + && self.parts.front().is_some_and(|part| part.len() == n) + && let Some(part) = self.parts.pop_front() + { + return match Arc::try_unwrap(part) { + Ok(part) => (part, 0), + Err(part) => (part.to_vec(), n), + }; + } + let mut out = Vec::with_capacity(n); + while out.len() < n { + let Some(front) = self.parts.front() else { + break; + }; + let want = n - out.len(); + if front.len() - self.start <= want { + out.extend_from_slice(&front[self.start..]); + self.parts.pop_front(); + self.start = 0; + } else { + out.extend_from_slice(&front[self.start..self.start + want]); + self.start += want; + } + } + let copied = out.len(); + (out, copied) + } + + /// (ops not handed out, ops the parts hold). + fn held(&self) -> (usize, usize) { + (self.len, self.len + self.start) + } + + /// The bytes its parts take on the heap. + fn heap_bytes(&self) -> usize { + self.parts + .iter() + .map(|part| super::vec_heap_bytes(part)) + .sum() + } + + /// The bytes its largest part (a window's list) takes on the heap. + fn largest_bytes(&self) -> usize { + self.parts + .iter() + .map(|part| super::vec_heap_bytes(part)) + .max() + .unwrap_or(0) + } + + /// The ops not handed out, as one list. + fn into_vec(mut self) -> Vec { + if self.start == 0 + && self.parts.len() == 1 + && let Some(part) = self.parts.pop_front() + { + return Arc::try_unwrap(part).unwrap_or_else(|part| part.to_vec()); + } + let mut out = Vec::with_capacity(self.len); + for (i, part) in self.parts.into_iter().enumerate() { + out.extend_from_slice(&part[if i == 0 { self.start } else { 0 }..]); + } + out + } +} + +/// The run so far under [`WindowedTraceBuilder::drop_streamed_ops`]. +struct Kept { + cpu: Tail, + /// The CPU tail's logs, in step with it, while + /// [`WindowedTraceBuilder::card_fill_streamed`] holds CPU chunks as logs; + /// `None` without, and from a window that came without its logs on. + logs: Option>, + register_rows: Tail, + aligned: Tail, + general: Tail, + load: Tail, + lt: Tail, + shift: Tail, + /// Every other walk list, the run so far, in blocks (the in-walk lookups + /// only hold what is left to count). + rest: KeptRest, + /// DECODE with the lookups of the CPU ops handed out counted. + decode: DecodeTrace, +} + +/// The walk lists a run keeps whole to its end, each appended window by +/// window into a [`BlockVec`] (blocks of its table's chunk, or 64 MiB). +pub(super) struct KeptRest { + bitwise_ops: BlockVec, + copy_ops: BlockVec, + keccak_ops: BlockVec, + blake3_ops: BlockVec, + blake3_absorb_ops: BlockVec, + cpu32_ops: BlockVec, + ecsm_ops: BlockVec, + /// Held compact (their carries in 16 bits), widened as each is read. + ecdas_ops: BlockVec, + hint_ops: BlockVec, + ecrec_ops: BlockVec, +} + +/// ECDAS rows a block of the kept ECDAS ops holds: the block's ECDAS cut +/// (`block_whir::BLOCK_ECDAS_ROWS_LOG2`), so a cut is a block; ≈ 91 MiB of +/// compact rows, one of the two lists whose blocks pass 64 MiB. +pub(super) const ECDAS_BLOCK_ROWS: usize = 1 << 17; + +/// COPY rows a block of the kept COPY ops holds: the block's COPY cut +/// (`block_whir::BLOCK_COPY_ROWS_LOG2`), so a cut is a block (48 MiB of rows). +pub(super) const COPY_BLOCK_ROWS: usize = 1 << 20; + +impl KeptRest { + fn new(m: &crate::tables::MaxRowsConfig) -> Self { + fn list(chunk: usize) -> BlockVec { + BlockVec::new(block_len::(chunk)) + } + Self { + bitwise_ops: list(usize::MAX), + copy_ops: BlockVec::new(COPY_BLOCK_ROWS), + keccak_ops: list(usize::MAX), + blake3_ops: list(usize::MAX), + blake3_absorb_ops: list(usize::MAX), + cpu32_ops: list(m.cpu32), + ecsm_ops: list(usize::MAX), + ecdas_ops: BlockVec::new(ECDAS_BLOCK_ROWS), + hint_ops: list(usize::MAX), + ecrec_ops: list(usize::MAX), + } + } + + /// A walk's lists as they are, one block each (a build that kept its + /// windows and concatenated them). + fn from_walk(walk: &mut WalkOutputs) -> Self { + Self { + bitwise_ops: BlockVec::from_vec(std::mem::take(&mut walk.bitwise_ops)), + copy_ops: BlockVec::from_vec(std::mem::take(&mut walk.copy_ops)), + keccak_ops: BlockVec::from_vec(std::mem::take(&mut walk.keccak_ops)), + blake3_ops: BlockVec::from_vec(std::mem::take(&mut walk.blake3_ops)), + blake3_absorb_ops: BlockVec::from_vec(std::mem::take(&mut walk.blake3_absorb_ops)), + cpu32_ops: BlockVec::from_vec(std::mem::take(&mut walk.cpu32_ops)), + ecsm_ops: BlockVec::from_vec(std::mem::take(&mut walk.ecsm_ops)), + ecdas_ops: BlockVec::from_vec( + walk.ecdas_ops + .iter() + .map(super::ecdas::CompactEcdasOp::from_op) + .collect(), + ), + hint_ops: BlockVec::from_vec(std::mem::take(&mut walk.hint_ops)), + ecrec_ops: BlockVec::from_vec(std::mem::take(&mut walk.ecrec_ops)), + } + } + + /// Each list's (name, heap bytes, largest single allocation). ECSM's + /// counts the steps' buffer each op's witness owns as well. + fn sizes(&self) -> [(&'static str, usize, usize); 10] { + fn of(name: &'static str, list: &BlockVec) -> (&'static str, usize, usize) { + (name, list.heap_bytes(), list.largest_bytes()) + } + let ecsm = { + let (name, bytes, largest) = of("ecsm", &self.ecsm_ops); + let (steps, most) = self + .ecsm_ops + .iter() + .map(super::ecsm::EcsmOperation::steps_heap_bytes) + .fold((0, 0), |(sum, most), b| (sum + b, most.max(b))); + (name, bytes + steps, largest.max(most)) + }; + let ecrec = { + let (name, bytes, largest) = of("ecrec", &self.ecrec_ops); + let (steps, most) = self + .ecrec_ops + .iter() + .map(super::ecrec::EcrecOperation::steps_heap_bytes) + .fold((0, 0), |(sum, most), b| (sum + b, most.max(b))); + (name, bytes + steps, largest.max(most)) + }; + [ + of("bitwise", &self.bitwise_ops), + of("copy", &self.copy_ops), + of("keccak", &self.keccak_ops), + of("blake3", &self.blake3_ops), + of("blake3_absorb", &self.blake3_absorb_ops), + of("cpu32", &self.cpu32_ops), + ecsm, + of("ecdas", &self.ecdas_ops), + of("hint", &self.hint_ops), + ecrec, + ] + } +} + +impl Kept { + fn new(decode: DecodeTrace, max_rows: &crate::tables::MaxRowsConfig) -> Self { + Self { + cpu: Tail::new(), + logs: None, + register_rows: Tail::new(), + aligned: Tail::new(), + general: Tail::new(), + load: Tail::new(), + lt: Tail::new(), + shift: Tail::new(), + rest: KeptRest::new(max_rows), + decode, + } + } + + /// The bytes the run so far takes on the heap: the tails, the other lists + /// and DECODE. + fn heap_bytes(&self) -> usize { + self.cpu.heap_bytes() + + self.logs.as_ref().map_or(0, Tail::heap_bytes) + + self.register_rows.heap_bytes() + + self.aligned.heap_bytes() + + self.general.heap_bytes() + + self.load.heap_bytes() + + self.lt.heap_bytes() + + self.shift.heap_bytes() + + self + .rest + .sizes() + .iter() + .map(|&(_, bytes, _)| bytes) + .sum::() + + self.decode.heap_bytes() + } + + /// [`Self::heap_bytes`] list by list, each named `kept {list}`. + fn heap_parts(&self) -> Vec<(String, usize)> { + let mut parts = vec![ + ("kept cpu".to_string(), self.cpu.heap_bytes()), + ( + "kept logs".to_string(), + self.logs.as_ref().map_or(0, Tail::heap_bytes), + ), + ("kept memw_r".to_string(), self.register_rows.heap_bytes()), + ("kept memw_a".to_string(), self.aligned.heap_bytes()), + ("kept memw".to_string(), self.general.heap_bytes()), + ("kept load".to_string(), self.load.heap_bytes()), + ("kept lt".to_string(), self.lt.heap_bytes()), + ("kept shift".to_string(), self.shift.heap_bytes()), + ("kept decode".to_string(), self.decode.heap_bytes()), + ]; + parts.extend( + self.rest + .sizes() + .iter() + .map(|&(name, bytes, _)| (format!("kept rest {name}"), bytes)), + ); + parts + } + + /// Each list's largest single allocation, named as in [`Self::heap_parts`]. + fn largest_parts(&self) -> Vec<(String, usize)> { + let mut parts = vec![ + ("kept cpu".to_string(), self.cpu.largest_bytes()), + ( + "kept logs".to_string(), + self.logs.as_ref().map_or(0, Tail::largest_bytes), + ), + ( + "kept memw_r".to_string(), + self.register_rows.largest_bytes(), + ), + ("kept memw_a".to_string(), self.aligned.largest_bytes()), + ("kept memw".to_string(), self.general.largest_bytes()), + ("kept load".to_string(), self.load.largest_bytes()), + ("kept lt".to_string(), self.lt.largest_bytes()), + ("kept shift".to_string(), self.shift.largest_bytes()), + ("kept decode".to_string(), self.decode.heap_bytes()), + ]; + parts.extend( + self.rest + .sizes() + .iter() + .map(|&(name, _, largest)| (format!("kept rest {name}"), largest)), + ); + parts + } + + /// A walked window's lists, appended in run order; its logs to the CPU + /// tail's, when they are kept. A window without them, or with other than + /// one log a CPU op, puts the logs out of step for good: every CPU chunk + /// from then on is filled on the host. + fn append(&mut self, window: WalkedWindow) { + let WalkedWindow { + cpu_ops, + walk, + logs, + } = window; + if let Some(tail) = self.logs.as_mut() { + match logs { + Some(logs) if logs.len() == cpu_ops.len() => tail.push_shared(logs), + _ => self.logs = None, + } + } + let WalkOutputs { + memw, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + copy_ops, + hint_ops, + ecrec_ops, + } = walk; + self.cpu.push(cpu_ops); + self.register_rows.push(memw.register_rows); + self.aligned.push(memw.aligned); + self.general.push(memw.general); + self.load.push(load_ops); + self.lt.push(lt_ops); + self.shift.push(shift_ops); + let rest = &mut self.rest; + rest.bitwise_ops.extend(bitwise_ops); + rest.copy_ops.extend(copy_ops); + rest.keccak_ops.extend(keccak_ops); + rest.blake3_ops.extend(blake3_ops); + rest.blake3_absorb_ops.extend(blake3_absorb_ops); + rest.cpu32_ops.extend(cpu32_ops); + rest.ecsm_ops.extend(ecsm_ops); + rest.ecdas_ops + .extend(ecdas_ops.iter().map(super::ecdas::CompactEcdasOp::from_op)); + rest.hint_ops.extend(hint_ops); + rest.ecrec_ops.extend(ecrec_ops); + } + + /// The run's lists with the handed-out ops left out (the streamed tables' + /// tails in a walk's lists, the others in blocks), and DECODE. + fn into_run(self) -> (Vec, WalkOutputs, KeptRest, Table) { + let Kept { + cpu, + logs: _, + register_rows, + aligned, + general, + load, + lt, + shift, + rest, + decode, + } = self; + let mut walk = WalkOutputs::with_capacity(0); + walk.memw = MemwBuckets { + register_rows: register_rows.into_vec(), + aligned: aligned.into_vec(), + general: general.into_vec(), + }; + walk.load_ops = load.into_vec(); + walk.lt_ops = lt.into_vec(); + walk.shift_ops = shift.into_vec(); + (cpu.into_vec(), walk, rest, decode.into_table()) + } +} + +/// Whether the CPU ops `ops` of the chunk from cycle `first_cycle` on are the +/// ones the walk made of `logs`, checked at each end of each of the logs' +/// window parts: the same count, each checked op's cycle the log's place and +/// its instruction the log's pc. The logs are kept beside the ops' tail, so a +/// window that came without its logs, or in the wrong place, shows here. +fn logs_in_step( + ops: &Segments, + logs: &Segments, + first_cycle: usize, + decode: &DecodeTable, +) -> bool { + if ops.len() != logs.len() { + return false; + } + let ops: Vec<&[super::CpuOperation]> = ops.slices(); + let op_at = |mut i: usize| { + for part in &ops { + if i < part.len() { + return part.get(i); + } + i -= part.len(); + } + None + }; + let mut at = 0usize; + for piece in logs.slices() { + let ends = [0, piece.len().saturating_sub(1)]; + for k in ends.into_iter().filter(|&k| k < piece.len()) { + let i = at + k; + let Some(op) = op_at(i) else { + return false; + }; + let cycle = (first_cycle + i) as u64; + if op.timestamp != 4 * cycle + 4 + || decode.entry(op.decode_row).pc != piece[k].current_pc + { + return false; + } + } + at += piece.len(); + } + true +} + +/// A full CPU chunk's ops as handed out: the records the card fills it from +/// (`logs`, its cycles' logs, in step with the ops) while the host has room, +/// its ops otherwise. A chunk held as records carries its spent ops for the +/// thread that lays it out to let go of. +fn cpu_chunk_ops( + ops: Segments, + logs: Option>, + index: usize, + rows: usize, + decode_table: &Arc, + card: Option<(&StreamedCardFill, &mut CardFillStamps)>, +) -> ChunkOps { + let Some((card, stamps)) = card else { + return ChunkOps::Cpu(ops, Arc::clone(decode_table)); + }; + let Some(decode) = card.cpu.as_ref() else { + return ChunkOps::Cpu(ops, Arc::clone(decode_table)); + }; + let first_cycle = index * rows; + let records = match logs { + Some(logs) if logs_in_step(&ops, &logs, first_cycle, decode_table) => card + .admits(logs.op_bytes(), (StreamTable::Cpu, index), stamps) + .then(|| { + let records = CpuLogRecords::new(logs, first_cycle as u64, Arc::clone(decode)); + #[cfg(test)] + let records = records.map(|r| match card.fault { + 0 => r, + by => r.with_register_values_xored(by), + }); + records + }) + .flatten(), + _ => { + stamps.out_of_step += 1; + None + } + }; + match records { + Some(records) => { + stamps.cpu[0] += 1; + stamps.records_bytes += multilinear::records::TableRecords::bytes(&records); + ChunkOps::CpuRecords { + records, + spent: ops, + } + } + None => { + stamps.cpu[1] += 1; + ChunkOps::Cpu(ops, Arc::clone(decode_table)) + } + } +} + +/// A full chunk of a [`RowFamily`] as handed out (`wanted`: the family is held +/// as records, [`WindowedTraceBuilder::card_fill_streamed_rows`]): as its rows +/// for the card to fill it from while the host has room, as `plain` makes it +/// otherwise. `counter` is the family's `[as records, on the host]`. +fn row_chunk_ops( + rows: Segments, + index: usize, + card: Option<(&StreamedCardFill, &mut CardFillStamps)>, + wanted: fn(&StreamedCardFill) -> bool, + counter: fn(&mut CardFillStamps) -> &mut [usize; 2], + plain: fn(Segments) -> ChunkOps, +) -> ChunkOps { + let Some((card, stamps)) = card.filter(|(card, _)| wanted(card)) else { + return plain(rows); + }; + let bytes = rows.op_bytes(); + if !card.admits(bytes, (T::TABLE, index), stamps) { + counter(stamps)[1] += 1; + return plain(rows); + } + match RowRecords::new(rows) { + Ok(records) => { + #[cfg(test)] + let records = match card.fault { + 0 => records, + cycles => records.later(cycles), + }; + counter(stamps)[0] += 1; + stamps.records_bytes += bytes; + ChunkOps::Rows(Box::new(records)) + } + Err(rows) => { + counter(stamps)[1] += 1; + plain(rows) + } + } +} + +/// [`ChunkJob::regenerate`] of a [`RowFamily`]'s chunk cut from the regeneration +/// walk: a whole table with the digest of the rows phase A would have held it +/// as; any other filled as `plain` makes it, with none. +fn row_regenerate( + rows: Segments, + form: TraceForm, + plain: fn(Segments) -> ChunkOps, + table: StreamTable, + index: usize, +) -> (Option<[u64; 2]>, Table) { + match RowRecords::new(rows) { + Ok(records) => (Box::new(records) as Box).regenerate(form, true), + Err(rows) => ( + None, + ChunkJob { + table, + index, + ops: plain(rows), + } + .generate_as(form) + .trace, + ), + } +} + +/// A full MEMW_R chunk's rows as handed out: as the records the card fills it +/// from while the host has room, to be filled on the host otherwise. +fn memw_register_chunk_ops( + rows: Segments, + index: usize, + card: Option<(&StreamedCardFill, &mut CardFillStamps)>, +) -> ChunkOps { + let Some((card, stamps)) = card.filter(|(card, _)| card.memw_r) else { + return ChunkOps::MemwRegister(rows); + }; + let bytes = rows.op_bytes(); + if !card.admits(bytes, (StreamTable::MemwRegister, index), stamps) { + stamps.memw_r[1] += 1; + return ChunkOps::MemwRegister(rows); + } + match RegisterRowRecords::new(rows) { + Ok(records) => { + stamps.memw_r[0] += 1; + stamps.records_bytes += bytes; + ChunkOps::MemwRegisterRecords(records) + } + Err(rows) => { + stamps.memw_r[1] += 1; + ChunkOps::MemwRegister(rows) + } + } +} + +/// [`chunk_jobs`] under [`WindowedTraceBuilder::drop_streamed_ops`]: every full +/// chunk taken out of the tails, and what the table phase reads from its ops +/// counted as it leaves (`counted`, and DECODE in `kept`), each timed into +/// `stamps`. With `card` ([`WindowedTraceBuilder::card_fill_streamed`]), CPU +/// and MEMW_R chunks go out as records while the host has room. With +/// `counter`, every count but STORE's is the counter's ([`handout_count`]): the +/// chunk's parts go to it in the order they leave. +#[allow(clippy::too_many_arguments)] +fn tail_jobs( + m: &crate::tables::MaxRowsConfig, + stream_memw_lt: bool, + decode_table: &Arc, + kept: &mut Kept, + segments: &mut RoutedBlocks, + e: &mut StreamSkip, + counted: &mut PreCounted, + stamps: &mut HandoutStamps, + mut card: Option<(&StreamedCardFill, &mut CardFillStamps)>, + counter: Option<&HandoutCounter>, +) -> Vec { + use std::time::Instant; + let mut jobs = Vec::new(); + macro_rules! take_chunks { + ($table:expr, $variant:ident, $tail:expr, $max:expr, $done:expr) => { + take_chunks!($table, $variant, $tail, $max, $done, |ops| {}) + }; + ($table:expr, $variant:ident, $tail:expr, $max:expr, $done:expr, |$ops:ident| $count:block) => { + take_chunks!($table, $tail, $max, $done, |$ops| $count, |ops, _| ChunkOps::$variant(ops)) + }; + // `$hold`: the chunk's ops as handed out, from its ops and index. + ($table:expr, $tail:expr, $max:expr, $done:expr, |$ops:ident| $count:block, $hold:expr) => {{ + let max = $max; + while $tail.len >= max { + let t = Instant::now(); + let ($ops, copied) = $tail.take_counted(max); + let taken = t.elapsed().as_secs_f64(); + let t = Instant::now(); + $count + let ops = $hold($ops, $done); + stamps.note($table, taken, t.elapsed().as_secs_f64(), copied); + jobs.push(ChunkJob { + table: $table, + index: $done, + ops, + }); + $done += 1; + } + }}; + } + while kept.cpu.len >= m.cpu { + let t = Instant::now(); + let (ops, copied) = kept.cpu.take_counted(m.cpu); + let taken = t.elapsed().as_secs_f64(); + let t = Instant::now(); + match counter { + Some(counter) => counter.send(CountWork::Cpu(ops.clone())), + None => { + decode::count_rows( + kept.decode.table(), + ops.iter().map(|op| op.decode_row as usize), + ); + if let Some(ecall) = handout_count::last_ecall(&ops, decode_table) { + counted.last_ecall = Some(ecall); + } + } + } + // The chunk's logs leave the logs' tail with its ops. + let logs = kept.logs.as_mut().map(|tail| tail.take_segments(m.cpu)); + let ops = cpu_chunk_ops( + ops, + logs, + e.cpu, + m.cpu, + decode_table, + card.as_mut().map(|(c, s)| (*c, &mut **s)), + ); + stamps.note(StreamTable::Cpu, taken, t.elapsed().as_secs_f64(), copied); + jobs.push(ChunkJob { + table: StreamTable::Cpu, + index: e.cpu, + ops, + }); + e.cpu += 1; + } + while kept.register_rows.len >= m.memw_register { + let t = Instant::now(); + let (rows, copied) = kept.register_rows.take_counted(m.memw_register); + let taken = t.elapsed().as_secs_f64(); + let t = Instant::now(); + match counter { + Some(counter) => counter.send(CountWork::MemwRegister(rows.clone())), + None => handout_count::count_memw_register(&rows, &mut counted.histogram), + } + let ops = memw_register_chunk_ops( + rows, + e.memw_register, + card.as_mut().map(|(c, s)| (*c, &mut **s)), + ); + stamps.note( + StreamTable::MemwRegister, + taken, + t.elapsed().as_secs_f64(), + copied, + ); + jobs.push(ChunkJob { + table: StreamTable::MemwRegister, + index: e.memw_register, + ops, + }); + e.memw_register += 1; + } + take_chunks!( + StreamTable::MemwAligned, + kept.aligned, + m.memw_aligned, + e.memw_aligned, + |ops| { + match counter { + Some(counter) => counter.send(CountWork::MemwAligned(ops.clone())), + None => handout_count::count_memw_aligned( + &ops, + &mut counted.histogram, + (!stream_memw_lt).then_some(&mut counted.memw_aligned_lt), + ), + } + }, + |rows, index| row_chunk_ops( + rows, + index, + card.as_mut().map(|(c, s)| (*c, &mut **s)), + |c| c.rows.memw_a, + |s| &mut s.memw_a, + ChunkOps::MemwAligned, + ) + ); + take_chunks!( + StreamTable::Memw, + kept.general, + m.memw, + e.memw, + |ops| { + match counter { + // Nothing to count when the MEMW-derived LT ops are streamed. + Some(_) if stream_memw_lt => {} + Some(counter) => counter.send(CountWork::Memw(ops.clone())), + None => handout_count::count_memw( + &ops, + (!stream_memw_lt).then_some(&mut counted.memw_lt), + ), + } + }, + |ops, index| row_chunk_ops( + ops, + index, + card.as_mut().map(|(c, s)| (*c, &mut **s)), + |c| c.rows.memw, + |s| &mut s.memw, + ChunkOps::Memw, + ) + ); + take_chunks!( + StreamTable::Load, + kept.load, + m.load, + e.load, + |ops| {}, + |ops, index| row_chunk_ops( + ops, + index, + card.as_mut().map(|(c, s)| (*c, &mut **s)), + |c| c.rows.load, + |s| &mut s.load, + ChunkOps::Load, + ) + ); + take_chunks!(StreamTable::Lt, Lt, kept.lt, m.lt, e.lt, |ops| { + match counter { + Some(counter) => counter.send(CountWork::Lt(ops.clone())), + None => handout_count::count_lt(&ops, &mut counted.histogram), + } + }); + take_chunks!( + StreamTable::Shift, + kept.shift, + m.shift, + e.shift, + |ops| { + match counter { + Some(counter) => counter.send(CountWork::Shift(ops.clone())), + None => handout_count::count_shift(&ops, &mut counted.histogram), + } + }, + |ops, index| row_chunk_ops( + ops, + index, + card.as_mut().map(|(c, s)| (*c, &mut **s)), + |c| c.rows.shift, + |s| &mut s.shift, + ChunkOps::Shift, + ) + ); + // KECCAK_RND's ops stay: KECCAK needs them all. + if e.keccak_rnd_rows > 0 { + let rows = e.keccak_rnd_rows; + let keccak = &kept.rest.keccak_ops; + while (e.keccak_rnd + 1) * rows <= keccak.len() * 24 { + let index = e.keccak_rnd; + let t = Instant::now(); + let (first, end) = super::keccak_rnd_op_range(index, rows, keccak.len()); + let ops = keccak.range(first, end).into_owned(); + stamps.note( + StreamTable::KeccakRnd, + t.elapsed().as_secs_f64(), + 0.0, + ops.len(), + ); + jobs.push(ChunkJob { + table: StreamTable::KeccakRnd, + index, + ops: ChunkOps::KeccakRnd { + ops, + skip: index * rows - first * 24, + rows, + }, + }); + e.keccak_rnd += 1; + } + } + // STORE's ops are a routing segment: its own tail, a chunk a block (its + // copies are not counted). + let store = &mut segments.store_ops; + while store.len() >= m.store { + let t = Instant::now(); + let ops: Vec = store.take_front(m.store); + let taken = t.elapsed().as_secs_f64(); + let t = Instant::now(); + for op in &ops { + counted.histogram.add_ops(&op.collect_bitwise_ops()); + } + let ops = match card.as_mut().filter(|(c, _)| c.rows.store) { + Some((c, s)) => row_chunk_ops( + Segments::one(ops), + e.store, + Some((*c, &mut **s)), + |c| c.rows.store, + |s| &mut s.store, + |ops| ChunkOps::Store(ops.into_one()), + ), + None => ChunkOps::Store(ops), + }; + stamps.note(StreamTable::Store, taken, t.elapsed().as_secs_f64(), 0); + jobs.push(ChunkJob { + table: StreamTable::Store, + index: e.store, + ops, + }); + e.store += 1; + } + jobs +} + +/// `rayon::join` under the `parallel` feature, the two calls in order without +/// it (the prover's guests take it without rayon). +fn join(a: impl FnOnce() -> A + Send, b: impl FnOnce() -> B + Send) -> (A, B) { + #[cfg(feature = "parallel")] + { + rayon::join(a, b) + } + #[cfg(not(feature = "parallel"))] + { + (a(), b()) + } +} + +/// The windows' lists concatenated into the run's, once: list by list in +/// parallel, each window's list freed as it is copied. +/// Whether [`concatenate`] copies each list's windows one after another on one +/// thread (`LAMBDA_VM_BUILDER_CONCAT=serial`), the control for the parallel copy +/// that is the default: on block 25368371 it read −0.52 s of base (FAST 417) and +/// replicated in its re-registered band at −0.21 s (FAST 419). Read once. The +/// two give the same lists. +#[cfg(feature = "parallel")] +fn serial_concat() -> bool { + static SERIAL: std::sync::OnceLock = std::sync::OnceLock::new(); + *SERIAL.get_or_init(|| { + std::env::var("LAMBDA_VM_BUILDER_CONCAT").is_ok_and(|v| v.trim() == "serial") + }) +} + +fn concatenate(windows: Vec) -> (Vec, WalkOutputs) { + /// The windows' lists one after another, in window order. Under `parallel` + /// each window's part is moved into its own disjoint slice of the output + /// on a thread of its own: the copy and the first touch of a list of + /// several GiB are spread over the pool instead of one thread. + fn cat(parts: Vec>) -> Vec { + let total = parts.iter().map(Vec::len).sum(); + let mut out = Vec::with_capacity(total); + #[cfg(feature = "parallel")] + if !serial_concat() && parts.len() > 1 { + let mut slots: Vec<&mut [std::mem::MaybeUninit]> = Vec::with_capacity(parts.len()); + let mut rest = &mut out.spare_capacity_mut()[..total]; + for part in &parts { + let (head, tail) = rest.split_at_mut(part.len()); + slots.push(head); + rest = tail; + } + parts + .into_par_iter() + .zip(slots.into_par_iter()) + .for_each(|(part, slot)| { + for (dst, src) in slot.iter_mut().zip(part) { + dst.write(src); + } + }); + // SAFETY: the slots tile `0..total` of the spare capacity, and each + // was written element for element by its part, whose length it has. + unsafe { out.set_len(total) }; + return out; + } + for part in parts { + out.extend(part); + } + out + } + let n = windows.len(); + let mut cpu = Vec::with_capacity(n); + let mut register_rows = Vec::with_capacity(n); + let mut aligned = Vec::with_capacity(n); + let mut general = Vec::with_capacity(n); + let mut load = Vec::with_capacity(n); + let mut lt = Vec::with_capacity(n); + let mut shift = Vec::with_capacity(n); + let mut bitwise = Vec::with_capacity(n); + let mut copy = Vec::with_capacity(n); + let mut keccak = Vec::with_capacity(n); + let mut blake3 = Vec::with_capacity(n); + let mut blake3_absorb = Vec::with_capacity(n); + let mut cpu32 = Vec::with_capacity(n); + let mut ecsm = Vec::with_capacity(n); + let mut ecdas = Vec::with_capacity(n); + let mut hint = Vec::with_capacity(n); + let mut ecrec = Vec::with_capacity(n); + for WalkedWindow { cpu_ops, walk, .. } in windows { + cpu.push(cpu_ops); + register_rows.push(walk.memw.register_rows); + aligned.push(walk.memw.aligned); + general.push(walk.memw.general); + load.push(walk.load_ops); + lt.push(walk.lt_ops); + shift.push(walk.shift_ops); + bitwise.push(walk.bitwise_ops); + copy.push(walk.copy_ops); + keccak.push(walk.keccak_ops); + blake3.push(walk.blake3_ops); + blake3_absorb.push(walk.blake3_absorb_ops); + cpu32.push(walk.cpu32_ops); + ecsm.push(walk.ecsm_ops); + ecdas.push(walk.ecdas_ops); + hint.push(walk.hint_ops); + ecrec.push(walk.ecrec_ops); + } + let ((cpu, (register_rows, aligned)), ((general, bitwise), (load, (lt, shift)))) = join( + || (cat(cpu), join(|| cat(register_rows), || cat(aligned))), + || { + join( + || join(|| cat(general), || cat(bitwise)), + || join(|| cat(load), || join(|| cat(lt), || cat(shift))), + ) + }, + ); + let walk = WalkOutputs { + memw: MemwBuckets { + register_rows, + aligned, + general, + }, + load_ops: load, + lt_ops: lt, + shift_ops: shift, + bitwise_ops: bitwise, + keccak_ops: cat(keccak), + blake3_ops: cat(blake3), + blake3_absorb_ops: cat(blake3_absorb), + cpu32_ops: cat(cpu32), + ecsm_ops: cat(ecsm), + ecdas_ops: cat(ecdas), + copy_ops: cat(copy), + hint_ops: cat(hint), + ecrec_ops: cat(ecrec), + }; + (cpu, walk) +} + +/// A completed chunk not generated yet: which chunk it is, and its ops. +/// [`generate`](Self::generate) builds the table a whole-run build puts in that +/// slot, with the same function. +pub struct ChunkJob { + pub table: StreamTable, + pub index: usize, + ops: ChunkOps, +} + +enum ChunkOps { + /// The ops and the decode they are read with. + Cpu(Segments, Arc), + /// The logs of its cycles, for the card to fill it from + /// ([`WindowedTraceBuilder::card_fill_streamed`]), and its ops, spent: let + /// go of where it is laid out. + CpuRecords { + records: CpuLogRecords, + spent: Segments, + }, + MemwRegister(Segments), + /// Its rows, for the card to fill it from. + MemwRegisterRecords(RegisterRowRecords), + /// A row family's chunk as its rows, for the card to fill it from + /// ([`WindowedTraceBuilder::card_fill_streamed_rows`]). + Rows(Box), + MemwAligned(Segments), + Memw(Segments), + Load(Segments), + Lt(Segments), + Shift(Segments), + Store(Vec), + KeccakRnd { + ops: Vec, + skip: usize, + rows: usize, + }, +} + +impl ChunkJob { + /// The bytes its ops take on the heap. + pub fn op_bytes(&self) -> usize { + use super::vec_heap_bytes as b; + match &self.ops { + ChunkOps::Cpu(ops, _) => ops.op_bytes(), + ChunkOps::CpuRecords { records, spent } => { + multilinear::records::TableRecords::bytes(records) + spent.op_bytes() + } + ChunkOps::MemwRegister(ops) => ops.op_bytes(), + ChunkOps::MemwRegisterRecords(records) => { + multilinear::records::TableRecords::bytes(records) + } + ChunkOps::Rows(records) => records.op_bytes(), + ChunkOps::MemwAligned(ops) => ops.op_bytes(), + ChunkOps::Memw(ops) => ops.op_bytes(), + ChunkOps::Load(ops) => ops.op_bytes(), + ChunkOps::Lt(ops) => ops.op_bytes(), + ChunkOps::Shift(ops) => ops.op_bytes(), + ChunkOps::Store(ops) => b(ops), + ChunkOps::KeccakRnd { ops, .. } => b(ops), + } + } + + /// Its ops: the table's rows before padding (KECCAK_RND's: the keccak ops + /// its rows come from). + pub fn op_count(&self) -> usize { + match &self.ops { + ChunkOps::Cpu(ops, _) => ops.len(), + ChunkOps::CpuRecords { spent, .. } => spent.len(), + ChunkOps::MemwRegister(ops) => ops.len(), + ChunkOps::MemwRegisterRecords(records) => { + multilinear::records::TableRecords::rows(records) + } + ChunkOps::Rows(records) => records.op_count(), + ChunkOps::MemwAligned(ops) => ops.len(), + ChunkOps::Memw(ops) => ops.len(), + ChunkOps::Load(ops) => ops.len(), + ChunkOps::Lt(ops) => ops.len(), + ChunkOps::Shift(ops) => ops.len(), + ChunkOps::Store(ops) => ops.len(), + ChunkOps::KeccakRnd { ops, .. } => ops.len(), + } + } + + pub fn generate(self) -> StreamedChunk { + self.generate_as(TraceForm::Wide) + } + + /// [`Self::generate`] in `form`: with [`TraceForm::Narrow`], the table + /// packed as it is written (G-pack, `tables::gpack`), the bytes + /// `TraceTable::pack_main_narrow` gives the 64-bit one. + pub fn generate_as(self, form: TraceForm) -> StreamedChunk { + let trace = match self.ops { + // Held as its records; its spent ops go here, off the walk's + // threads. + ChunkOps::CpuRecords { records, spent } => { + drop(spent); + records.into_trace() + } + ChunkOps::MemwRegisterRecords(records) => records.into_trace(), + ChunkOps::Rows(records) => records.into_trace(), + ChunkOps::Cpu(ops, decode) => { + cpu::generate_cpu_trace_segments_as(&ops.slices(), &decode, form) + } + ChunkOps::MemwRegister(ops) => { + memw_register::generate_memw_register_trace_from_rows_segments_as( + &ops.slices(), + form, + ) + } + ChunkOps::MemwAligned(ops) => { + memw_aligned::generate_memw_aligned_trace_segments_as(&ops.slices(), form) + } + ChunkOps::Memw(ops) => memw::generate_memw_trace_segments_as(&ops.slices(), form), + ChunkOps::Load(ops) => load::generate_load_trace_segments_as(&ops.slices(), form), + ChunkOps::Lt(ops) => lt::generate_lt_trace_segments_as(&ops.slices(), form), + ChunkOps::Shift(ops) => shift::generate_shift_trace_segments_as(&ops.slices(), form), + ChunkOps::Store(ops) => store::generate_store_trace_as(&ops, form), + ChunkOps::KeccakRnd { ops, skip, rows } => { + let ops: Vec = ops + .iter() + .map(|op| keccak_rnd::KeccakRoundOperation { + timestamp: op.timestamp, + input: op.input, + output: op.output, + }) + .collect(); + keccak_rnd::generate_keccak_rnd_rows_as(&ops, skip, rows, form) + } + }; + StreamedChunk { + table: self.table, + index: self.index, + trace, + } + } + + /// The chunk filled here, packed in `form` as its ops make it, and, with + /// `as_records` (it was dropped as its records), the digest of the records + /// phase A held it as (`multilinear::records::digest_of`): a CPU chunk cut + /// with its logs ([`RegenBuilder::keep_cpu_logs`]), any full MEMW_R chunk + /// or full chunk of a row family. `None` for any other, and for every chunk without + /// `as_records`: no chunk pays for a digest no slot checks. What a + /// regenerator deposits for a chunk dropped as its records, checked by + /// that digest. + pub fn regenerate( + self, + form: TraceForm, + as_records: bool, + ) -> (Option<[u64; 2]>, StreamedChunk) { + let (table, index) = (self.table, self.index); + let (digest, trace) = match self.ops { + ChunkOps::CpuRecords { records, spent } => { + let digest = as_records.then(|| multilinear::records::digest_of(&records)); + let trace = cpu::generate_cpu_trace_segments_as( + &spent.slices(), + records.decode_table(), + form, + ); + (digest, trace) + } + ChunkOps::MemwRegisterRecords(records) => { + let digest = as_records.then(|| multilinear::records::digest_of(&records)); + let rows = records.into_rows(); + let trace = memw_register::generate_memw_register_trace_from_rows_segments_as( + &rows.slices(), + form, + ); + (digest, trace) + } + ChunkOps::MemwRegister(rows) if as_records => match RegisterRowRecords::new(rows) { + Ok(records) => { + let digest = multilinear::records::digest_of(&records); + let rows = records.into_rows(); + let trace = memw_register::generate_memw_register_trace_from_rows_segments_as( + &rows.slices(), + form, + ); + (Some(digest), trace) + } + Err(rows) => ( + None, + ChunkJob { + table, + index, + ops: ChunkOps::MemwRegister(rows), + } + .generate_as(form) + .trace, + ), + }, + ChunkOps::Rows(records) => records.regenerate(form, as_records), + ChunkOps::MemwAligned(rows) if as_records => { + row_regenerate(rows, form, ChunkOps::MemwAligned, table, index) + } + ChunkOps::Load(ops) if as_records => { + row_regenerate(ops, form, ChunkOps::Load, table, index) + } + ChunkOps::Memw(ops) if as_records => { + row_regenerate(ops, form, ChunkOps::Memw, table, index) + } + ChunkOps::Shift(ops) if as_records => { + row_regenerate(ops, form, ChunkOps::Shift, table, index) + } + ChunkOps::Store(ops) if as_records => row_regenerate( + Segments::one(ops), + form, + |ops| ChunkOps::Store(ops.into_one()), + table, + index, + ), + ops => (None, ChunkJob { table, index, ops }.generate_as(form).trace), + }; + ( + digest, + StreamedChunk { + table, + index, + trace, + }, + ) + } + + /// Whether the card fills it from the records it holds + /// ([`WindowedTraceBuilder::card_fill_streamed`]). + pub fn is_records(&self) -> bool { + matches!( + self.ops, + ChunkOps::CpuRecords { .. } | ChunkOps::MemwRegisterRecords(_) | ChunkOps::Rows(_) + ) + } +} + +#[cfg(all(test, feature = "cuda", feature = "parallel"))] +impl ChunkJob { + /// A CPU chunk's ops, window part by window part, and the decode they are + /// read with: what [`Self::generate_as`] fills it from. + pub(crate) fn cpu_parts(&self) -> Option<(Vec<&[super::CpuOperation]>, &DecodeTable)> { + match &self.ops { + ChunkOps::Cpu(ops, decode) => Some((ops.slices(), decode)), + _ => None, + } + } + + /// A MEMW_R chunk's register rows, window part by window part. + pub(crate) fn register_rows(&self) -> Option> { + match &self.ops { + ChunkOps::MemwRegister(ops) => Some(ops.slices()), + _ => None, + } + } +} + +/// The walk's lists and the routing's segments as the whole-run +/// [`collect_all_ops`](super::collect_all_ops) returns them, HALT finalization +/// included (the run's end). +fn assemble( + cpu_ops: Vec, + walk: WalkOutputs, + rest: KeptRest, + segments: RoutedBlocks, + register_state: &mut RegisterState, +) -> CollectedOps { + // The streamed tables' lists (their tails); `rest` holds the others. + let WalkOutputs { + memw, + load_ops, + lt_ops, + shift_ops, + .. + } = walk; + let KeptRest { + bitwise_ops, + copy_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + ecrec_ops, + } = rest; + let MemwBuckets { + register_rows: mut memw_register_rows, + aligned: mut memw_aligned_ops, + general: mut memw_ops, + } = memw; + // HALT's register ops land at the end of their buckets, as in + // `collect_all_ops`. + let mut halt = MemwBuckets::with_register_capacity(0); + halt.extend_ops(collect_halt_ops(register_state)); + memw_register_rows.extend(halt.register_rows); + memw_aligned_ops.extend(halt.aligned); + memw_ops.extend(halt.general); + + let RoutedBlocks { + branch_ops, + mul_filter, + dvrm_filter, + eq_ops, + bytewise_ops, + store_ops, + shift_cpu32, + mul_cpu32, + dvrm_cpu32, + bitwise_cpu32, + lt_dvrm_filter, + lt_dvrm_cpu32, + mul_dvrm_filter, + mul_dvrm_cpu32, + } = segments; + // Each list is its segments' blocks, in the routing's order, never + // concatenated: the tables chunk them as their concatenation. + let shift_ops = BlockVec::from_vec(shift_ops).chain(shift_cpu32); + let bitwise_ops = bitwise_ops.chain(bitwise_cpu32); + let mul_ops = mul_filter + .chain(mul_cpu32) + .chain(mul_dvrm_filter) + .chain(mul_dvrm_cpu32); + let dvrm_ops = dvrm_filter.chain(dvrm_cpu32); + let lt_ops = BlockVec::from_vec(lt_ops) + .chain(lt_dvrm_filter) + .chain(lt_dvrm_cpu32); + + CollectedOps { + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + branch_ops, + mul_ops, + dvrm_ops, + copy_ops, + keccak_ops, + blake3_ops, + blake3_absorb_ops, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + ecrec_ops, + } +} + +/// A run's streamed chunks built again from its logs (D-REGEN §2.3, the +/// sequential regenerator's walker and slicer): the run is walked from the +/// start with the regeneration walk ([`super::collect_streamed_ops_from_cpu_into`]), +/// which carries the state the windowed build's walk carries but lists only the +/// streamed tables' ops, and each table's list is cut as the hand-out cuts it. +/// Chunk `k` of a table is ops `[k·max, (k+1)·max)` of its list, the slice +/// [`Accumulator::absorb`] hands out as that chunk, so its job generates the +/// same table. STORE's list is the routing's segment, filtered out of the CPU +/// ops as the routing filters it. Nothing it does reads or writes the windowed +/// build. +/// +/// Within a window it cuts in the hand-out's order ([`StreamTable::ALL`], then +/// by index), so fed phase A's windows it cuts the chunks in phase A's +/// hand-out order. KECCAK_RND is never cut: a build that streams it +/// ([`WindowedTraceBuilder::keccak_rnd_chunks`]) has chunks no regenerator +/// rebuilds, and neither has one that streams the MEMW-derived LT ops +/// ([`WindowedTraceBuilder::stream_memw_lt`]). +/// +/// Unlike the windowed build it takes the run's last window too (the chunks +/// that window completes were the finish's, which a caller rebuilding the +/// streamed chunks skips), and the windows may be any length. +pub struct RegenBuilder { + /// The ops' decode, by DECODE row (shared with the CPU chunks' jobs). + decode: Arc, + memory_state: MemoryState, + register_state: RegisterState, + /// Cycles walked so far: the next window's first cycle. + cycles: usize, + max_rows: crate::tables::MaxRowsConfig, + cpu: Tail, + register_rows: Tail, + aligned: Tail, + general: Tail, + load: Tail, + lt: Tail, + shift: Tail, + store: Tail, + /// [`Self::keep_cpu_logs`]: the CPU tail's logs, in step with it, and the + /// DECODE rows a CPU chunk's records read. + logs: Option<(Tail, Arc)>, + /// Chunks of each streamed table cut so far. + cut: StreamSkip, + /// A test's slicer bug: the first op of this table's list is dropped. + #[cfg(test)] + drop_one: Option, +} + +impl RegenBuilder { + /// A regenerator for a run of `elf` on `private_input`, chunked at + /// `max_rows`: the state [`WindowedTraceBuilder::new`] starts from. + pub fn new( + elf: &Elf, + private_input: &[u8], + max_rows: &crate::tables::MaxRowsConfig, + ) -> Result { + let mut memory_state = + MemoryState::from_pages(&build_initial_image_paged(elf, private_input)); + memory_state.lean = WalkLean::from_env()?.memory; + let register_init = register::register_init_from_entry_point(elf.entry_point); + let instructions = decode::instructions_from_elf(elf) + .map_err(|e| Error::Execution(format!("Failed to parse instructions: {e}")))?; + Ok(Self { + decode: Arc::new(DecodeTable::from_instructions(&instructions)), + memory_state, + register_state: RegisterState::from_init(®ister_init), + cycles: 0, + max_rows: max_rows.clone(), + cpu: Tail::new(), + register_rows: Tail::new(), + aligned: Tail::new(), + general: Tail::new(), + load: Tail::new(), + lt: Tail::new(), + shift: Tail::new(), + store: Tail::new(), + logs: None, + cut: StreamSkip::default(), + #[cfg(test)] + drop_one: None, + }) + } + + /// The slicer drops the first op of `table`'s list: a regeneration bug, for + /// the tests that check it is caught. + #[cfg(test)] + pub(crate) fn drop_one_op(mut self, table: StreamTable) -> Self { + self.drop_one = Some(table); + self + } + + /// Cycles walked so far. + pub fn cycles(&self) -> usize { + self.cycles + } + + /// Each CPU chunk is cut with the logs of its cycles too, as phase A held + /// a CPU chunk as its records ([`WindowedTraceBuilder::card_fill_streamed`]), + /// so the chunk's records can be digested again ([`ChunkJob::regenerate`]). + /// Before the first window only; windows then come by [`Self::push_owned`]. + pub fn keep_cpu_logs(mut self) -> Result { + if self.cycles > 0 { + return Err(Error::Prover( + "keep_cpu_logs after a window was walked".to_string(), + )); + } + let decode = CardDecode::new(Arc::clone(&self.decode)).ok_or_else(|| { + Error::Prover("keep_cpu_logs: a program with no instruction".to_string()) + })?; + self.logs = Some((Tail::new(), Arc::new(decode))); + Ok(self) + } + + /// The bytes the walk's carried memory state takes on the heap. + pub fn state_bytes(&self) -> usize { + self.memory_state.heap_bytes() + } + + /// [`Self::push`] of logs it may keep ([`Self::keep_cpu_logs`]). + pub fn push_owned(&mut self, logs: Vec) -> Result, Error> { + if self.logs.is_none() { + return self.push(&logs); + } + let logs = Arc::new(logs); + self.push_window(&logs, Some(Arc::clone(&logs))) + } + + /// Walks the run's next window (its last one included) and returns every + /// chunk of the streamed tables it completes, in the hand-out's order + /// ([`StreamTable::ALL`], then by index). A builder that keeps its CPU + /// logs takes its windows by [`Self::push_owned`]; one pushed here puts + /// them out of step, and its CPU chunks are cut without them. + pub fn push(&mut self, logs: &[Log]) -> Result, Error> { + self.push_window(logs, None) + } + + /// [`Self::push`]'s body: `kept`, the same logs shared, when the CPU tail's + /// logs are kept. + fn push_window( + &mut self, + logs: &[Log], + kept: Option>>, + ) -> Result, Error> { + #[cfg_attr(not(test), allow(unused_mut))] + let mut cpu_ops = super::collect_cpu_ops_from(logs, &self.decode, self.cycles)?; + self.cycles += logs.len(); + let mut walk = WalkOutputs::for_walk(cpu_ops.len(), false); + super::collect_streamed_ops_from_cpu_into( + &cpu_ops, + &self.decode, + &mut self.memory_state, + &mut self.register_state, + &mut walk, + )?; + // STORE's segment, as the routing filters it out of the CPU ops. + #[cfg_attr(not(test), allow(unused_mut))] + let mut stores: Vec = cpu_ops + .iter() + .map(|op| self.decode.op(op)) + .filter(|op| op.decode.fields.is_store()) + .map(super::route_store) + .collect(); + // Mutable for the tests' slicer bug (`drop_one_op`). + #[cfg_attr(not(test), allow(unused_mut))] + let MemwBuckets { + mut register_rows, + mut aligned, + mut general, + } = walk.memw; + #[cfg_attr(not(test), allow(unused_mut))] + let (mut load, mut lt, mut shift) = (walk.load_ops, walk.lt_ops, walk.shift_ops); + #[cfg(test)] + if let Some(table) = self.drop_one { + fn drop_first(list: &mut Vec) -> bool { + (!list.is_empty()).then(|| list.remove(0)).is_some() + } + let dropped = match table { + StreamTable::Cpu => drop_first(&mut cpu_ops), + StreamTable::MemwRegister => drop_first(&mut register_rows), + StreamTable::MemwAligned => drop_first(&mut aligned), + StreamTable::Memw => drop_first(&mut general), + StreamTable::Load => drop_first(&mut load), + StreamTable::Lt => drop_first(&mut lt), + StreamTable::Shift => drop_first(&mut shift), + StreamTable::Store => drop_first(&mut stores), + StreamTable::KeccakRnd => false, + }; + if dropped { + self.drop_one = None; + } + } + match (&mut self.logs, kept) { + (Some((tail, _)), Some(logs)) if logs.len() == cpu_ops.len() => tail.push_shared(logs), + // Out of step for good: the CPU chunks go without their logs. + (held, _) => *held = None, + } + self.cpu.push(cpu_ops); + self.register_rows.push(register_rows); + self.aligned.push(aligned); + self.general.push(general); + self.load.push(load); + self.lt.push(lt); + self.shift.push(shift); + self.store.push(stores); + Ok(self.cut_chunks()) + } + + /// Every full chunk the tails now hold, cut out of them in the hand-out's + /// order. + fn cut_chunks(&mut self) -> Vec { + let m = &self.max_rows; + let e = &mut self.cut; + let mut jobs = Vec::new(); + macro_rules! cut { + ($table:expr, $variant:ident, $tail:expr, $max:expr, $done:expr) => {{ + let max = $max; + while max > 0 && $tail.len >= max { + jobs.push(ChunkJob { + table: $table, + index: $done, + ops: ChunkOps::$variant($tail.take(max)), + }); + $done += 1; + } + }}; + } + while m.cpu > 0 && self.cpu.len >= m.cpu { + let ops = self.cpu.take(m.cpu); + // With the logs kept, the chunk carries the records phase A held + // it as, when they are in step with its ops. + let records = self.logs.as_mut().and_then(|(tail, decode)| { + let logs = tail.take_segments(m.cpu); + let first_cycle = e.cpu * m.cpu; + logs_in_step(&ops, &logs, first_cycle, &self.decode) + .then(|| CpuLogRecords::new(logs, first_cycle as u64, Arc::clone(decode))) + .flatten() + }); + let ops = match records { + Some(records) => ChunkOps::CpuRecords { + records, + spent: ops, + }, + None => ChunkOps::Cpu(ops, Arc::clone(&self.decode)), + }; + jobs.push(ChunkJob { + table: StreamTable::Cpu, + index: e.cpu, + ops, + }); + e.cpu += 1; + } + cut!( + StreamTable::MemwRegister, + MemwRegister, + self.register_rows, + m.memw_register, + e.memw_register + ); + cut!( + StreamTable::MemwAligned, + MemwAligned, + self.aligned, + m.memw_aligned, + e.memw_aligned + ); + cut!(StreamTable::Memw, Memw, self.general, m.memw, e.memw); + cut!(StreamTable::Load, Load, self.load, m.load, e.load); + cut!(StreamTable::Lt, Lt, self.lt, m.lt, e.lt); + cut!(StreamTable::Shift, Shift, self.shift, m.shift, e.shift); + // STORE's chunk holds one list: copied out (a part that is exactly the + // chunk moves out whole). + while m.store > 0 && self.store.len >= m.store { + jobs.push(ChunkJob { + table: StreamTable::Store, + index: e.store, + ops: ChunkOps::Store(self.store.take_copy(m.store)), + }); + e.store += 1; + } + jobs + } +} + +impl Traces { + /// Puts streamed chunks back into the placeholder slots a windowed build's + /// [`finish`](WindowedTraceBuilder::finish) left — which makes the traces + /// the whole-run build's. Refuses a chunk for a slot that is not a + /// placeholder. + pub fn insert_streamed(&mut self, chunks: Vec) -> Result<(), Error> { + for chunk in chunks { + let slots = match chunk.table { + StreamTable::Cpu => &mut self.cpus, + StreamTable::MemwRegister => &mut self.memw_registers, + StreamTable::MemwAligned => &mut self.memw_aligneds, + StreamTable::Memw => &mut self.memws, + StreamTable::Load => &mut self.loads, + StreamTable::Lt => &mut self.lts, + StreamTable::Shift => &mut self.shifts, + StreamTable::Store => &mut self.stores, + StreamTable::KeccakRnd => &mut self.keccak_rnds, + }; + let len = slots.len(); + let slot = slots.get_mut(chunk.index).ok_or_else(|| { + Error::Prover(format!( + "{:?} chunk {} has no slot ({len} tables)", + chunk.table, chunk.index, + )) + })?; + if slot.main_table.width != 0 { + return Err(Error::Prover(format!( + "{:?} chunk {} lands on a slot that is not a placeholder", + chunk.table, chunk.index + ))); + } + *slot = chunk.trace; + } + Ok(()) + } +} + +#[cfg(test)] +mod head_tests; + +#[cfg(test)] +mod card_records_tests; + +#[cfg(all(test, feature = "cuda"))] +mod card_records_card_tests; + +// The serial stage of a run walked a window per thread. +#[cfg(feature = "parallel")] +mod lean_pass; + +// A run's windows walked from their entries, on several threads. +#[cfg(feature = "parallel")] +pub(crate) mod parallel; + +// The head of a streamed phase A, stamped. +#[cfg(feature = "parallel")] +pub(crate) mod head_log; + +#[cfg(test)] +mod regen_tests; + +#[cfg(test)] +mod tail_tests; + +#[cfg(all(test, feature = "parallel"))] +mod block_tests { + /// The kept ECDAS ops' blocks are the block proof's ECDAS cut, so a cut is + /// one block. + #[test] + fn the_ecdas_block_is_the_blocks_ecdas_cut() { + assert_eq!( + super::ECDAS_BLOCK_ROWS, + 1usize << crate::block_whir::BLOCK_ECDAS_ROWS_LOG2 + ); + } + + /// The kept COPY ops' blocks are the block proof's COPY cut, so a cut is + /// one block. + #[test] + fn the_copy_block_is_the_blocks_copy_cut() { + assert_eq!( + super::COPY_BLOCK_ROWS, + 1usize << crate::block_whir::BLOCK_COPY_ROWS_LOG2 + ); + } + + /// The kept ECSM ops' size counts the steps' buffer each witness owns, so + /// the memory log names it. + #[test] + fn the_kept_ecsm_size_counts_the_witness_steps() { + let mut gx = [ + 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, + 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, + 0x16, 0xF8, 0x17, 0x98, + ]; + gx.reverse(); + let mut k = [0u8; 32]; + k[..8].copy_from_slice(&1_000_003u64.to_le_bytes()); + let witness = ::ecsm::compute_witness(&k, &gx).expect("a valid scalar and point"); + let mut rest = super::KeptRest::new(&crate::tables::MaxRowsConfig::default()); + let op = crate::tables::ecsm::EcsmOperation { + timestamp: 7, + addr_xg: 0x2000, + addr_k: 0x3000, + addr_xr: 0x1000, + witness, + }; + let kept = op.clone(); + let steps = kept.witness.steps.capacity() * std::mem::size_of::<::ecsm::EcdasStep>(); + assert!(steps > 0); + rest.ecsm_ops.push(kept); + let mut bare = op; + bare.witness.steps = Vec::new(); + rest.ecsm_ops.push(bare); + let (name, bytes, largest) = rest.sizes()[6]; + assert_eq!(name, "ecsm"); + assert_eq!(bytes, rest.ecsm_ops.heap_bytes() + steps); + assert_eq!(largest, rest.ecsm_ops.largest_bytes().max(steps)); + } +} diff --git a/prover/src/tables/trace_builder/windowed/card_records.rs b/prover/src/tables/trace_builder/windowed/card_records.rs new file mode 100644 index 000000000..2bc332e86 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/card_records.rs @@ -0,0 +1,493 @@ +//! A streamed CPU or MEMW_R chunk held as the records its rows come from +//! (`multilinear::records`), for the card to fill its columns as its group +//! uploads it: a CPU chunk as the executor's logs of its cycles, a MEMW_R +//! chunk as its register rows, a MEMW_A, LOAD, STORE, MEMW or SHIFT chunk as +//! its rows or ops ([`RowRecords`]), each in the window parts the chunk was cut from, shared +//! and never copied. A host reader gets the host fill of the same +//! rows, which writes the same words. + +use std::sync::Arc; + +use executor::vm::logs::Log; +use multilinear::narrow::NarrowColumns; +use multilinear::records::{DecodeRecords, PagesReturned, Records, RecordsFill, TableRecords}; + +use super::{Segments, StreamTable, Table}; +use crate::tables::decode::DecodeTable; +use crate::tables::gpack::TraceForm; +use crate::tables::load::LoadOperation; +use crate::tables::memw::MemwOperation; +use crate::tables::memw_aligned::AlignedRow; +use crate::tables::memw_register::RegRow; +use crate::tables::shift::ShiftOperation; +use crate::tables::store::StoreOperation; +use crate::tables::{cpu, load, memw, memw_aligned, memw_register, shift, store}; + +/// The program's DECODE table for the CPU chunks held as logs: the host's, +/// which a host reader fills with, and its rows as the card reads them. +pub(crate) struct CardDecode { + pub(crate) table: Arc, + pub(crate) records: DecodeRecords, +} + +impl CardDecode { + /// `table` and its rows as the card reads them; `None` for a program with + /// no instruction. + pub(crate) fn new(table: Arc) -> Option { + let records = table.card_records()?; + Some(Self { table, records }) + } +} + +impl std::fmt::Debug for CardDecode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "CardDecode({:?})", self.records) + } +} + +/// A trace held as `records`. A trace refuses records only over another field +/// than Goldilocks or for a table of no columns, neither of which a streamed +/// table is; were it refused, the trace is the host fill's columns, packed. +fn held(records: Records) -> Table { + Table::try_from_records_main(records, 1).unwrap_or_else(|records| { + let (rows, cols) = (records.rows(), records.cols()); + records + .host_fill() + .and_then(|packed| Table::try_from_narrow_main(packed, 1).ok()) + .unwrap_or_else(|| Table::new_main(vec![Default::default(); rows * cols], cols, 1)) + }) +} + +/// Whether `rows` rows are a whole streamed table: a power of two of at least +/// four, the height its host fill pads them to. +fn whole_table(rows: usize) -> bool { + rows.is_power_of_two() && rows >= 4 +} + +/// A full CPU chunk held as the logs of its cycles, from `first_cycle` on. +pub(crate) struct CpuLogRecords { + logs: Segments, + first_cycle: u64, + decode: Arc, +} + +impl CpuLogRecords { + /// The chunk the cycles of `logs` from `first_cycle` on make. Refused + /// (`None`) unless they are a whole table: a chunk of another height goes + /// to the host's fill, which pads it. + pub(crate) fn new( + logs: Segments, + first_cycle: u64, + decode: Arc, + ) -> Option { + whole_table(logs.len()).then_some(Self { + logs, + first_cycle, + decode, + }) + } + + /// The table held as these records. + pub(crate) fn into_trace(self) -> Table { + held(Records(Arc::new(self))) + } + + /// The host's DECODE table its CPU ops are read with. + pub(crate) fn decode_table(&self) -> &DecodeTable { + &self.decode.table + } + + /// A test's fault: the same chunk from a copy of its logs, each log's + /// first source value (`src1_val`, the `rv1` of nearly every row) and + /// written value (`dst_val`) XORed with `by`, in one piece: a CPU table + /// the run did not make, the same however it is filled. + #[cfg(test)] + pub(crate) fn with_register_values_xored(self, by: u64) -> Self { + let logs: Vec = self + .logs + .slices() + .iter() + .flat_map(|piece| piece.iter()) + .map(|log| Log { + src1_val: log.src1_val ^ by, + dst_val: log.dst_val ^ by, + ..log.clone() + }) + .collect(); + Self { + logs: Segments::one(logs), + ..self + } + } +} + +impl std::fmt::Debug for CpuLogRecords { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "CPU logs: {} cycles from {} in {} pieces", + self.logs.len(), + self.first_cycle, + self.logs.parts.len() + ) + } +} + +impl TableRecords for CpuLogRecords { + fn rows(&self) -> usize { + self.logs.len() + } + + fn cols(&self) -> usize { + cpu::cols::NUM_COLUMNS + } + + fn fill(&self) -> RecordsFill<'_> { + RecordsFill::Cpu { + logs: self.logs.slices().into_iter().map(cpu::card_logs).collect(), + first_cycle: self.first_cycle, + decode: &self.decode.records, + } + } + + fn bytes(&self) -> usize { + self.logs.op_bytes() + } + + /// TIMESTAMP, `4·(row + 1)`: the fill writes it from the row index, as the + /// CPU table's preprocessed column. + fn row_index_columns(&self) -> usize { + cpu::NUM_PRECOMPUTED_COLS + } + + /// The CPU ops the walk made of these logs, filled as the host fills a + /// streamed CPU chunk. `None` when a log's pc has no DECODE row (the walk + /// refused it already) or a cycle is past the clock's range. + fn host_fill(&self) -> Option { + let mut ops = Vec::with_capacity(self.logs.len()); + let mut first = usize::try_from(self.first_cycle).ok()?; + for piece in self.logs.slices() { + super::super::collect_cpu_ops_into(piece, &self.decode.table, first, &mut ops).ok()?; + first += piece.len(); + } + let mut trace = cpu::generate_cpu_trace_as(&ops, &self.decode.table, TraceForm::Narrow); + // Written packed, except in a build that holds every trace wide. + trace.pack_main_narrow(); + trace.take_narrow_main() + } + + fn return_pages(&mut self) -> PagesReturned { + self.logs.return_pages() + } +} + +/// A full MEMW_R chunk held as its register rows. +pub(crate) struct RegisterRowRecords { + rows: Segments, +} + +impl RegisterRowRecords { + /// The chunk `rows` make. Refused (the rows given back) unless they are a + /// whole table: a chunk of another height goes to the host's fill, which + /// pads it. + pub(crate) fn new(rows: Segments) -> Result> { + if whole_table(rows.len()) { + Ok(Self { rows }) + } else { + Err(rows) + } + } + + /// The table held as these records. + pub(crate) fn into_trace(self) -> Table { + held(Records(Arc::new(self))) + } + + /// The rows back. + pub(crate) fn into_rows(self) -> Segments { + self.rows + } +} + +impl std::fmt::Debug for RegisterRowRecords { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "MEMW_R rows: {} in {} pieces", + self.rows.len(), + self.rows.parts.len() + ) + } +} + +impl TableRecords for RegisterRowRecords { + fn rows(&self) -> usize { + self.rows.len() + } + + fn cols(&self) -> usize { + memw_register::cols::NUM_COLUMNS + } + + fn fill(&self) -> RecordsFill<'_> { + RecordsFill::MemwR { + rows: self + .rows + .slices() + .into_iter() + .map(memw_register::card_rows) + .collect(), + } + } + + fn bytes(&self) -> usize { + self.rows.op_bytes() + } + + fn host_fill(&self) -> Option { + let mut trace = memw_register::generate_memw_register_trace_from_rows_segments_as( + &self.rows.slices(), + TraceForm::Narrow, + ); + // Written packed, except in a build that holds every trace wide. + trace.pack_main_narrow(); + trace.take_narrow_main() + } + + fn return_pages(&mut self) -> PagesReturned { + self.rows.return_pages() + } +} + +/// A streamed family whose full chunks may be held as their rows, one record a +/// row, the walk's own structs read in place by the card (MEMW_A's aligned +/// rows, LOAD's, STORE's, MEMW's and SHIFT's ops): the records as the card +/// reads them, and the host fill of the same rows. +pub(crate) trait RowFamily: Sized + Send + Sync + 'static { + const TABLE: StreamTable; + const COLUMNS: usize; + /// The family's records as the card's fill reads them. + fn fill(pieces: Vec<&[Self]>) -> RecordsFill<'_>; + /// The host fill of the pieces' rows, in `form`: the table the chunk is + /// without records. + fn host_fill(pieces: &[&[Self]], form: TraceForm) -> Table; + /// A test's fault: the same row, its timestamp `cycles` later. + #[cfg(test)] + fn later(&self, cycles: u64) -> Self; +} + +impl RowFamily for AlignedRow { + const TABLE: StreamTable = StreamTable::MemwAligned; + const COLUMNS: usize = memw_aligned::cols::NUM_COLUMNS; + + fn fill(pieces: Vec<&[Self]>) -> RecordsFill<'_> { + RecordsFill::MemwA { + rows: pieces.into_iter().map(memw_aligned::card_rows).collect(), + } + } + + fn host_fill(pieces: &[&[Self]], form: TraceForm) -> Table { + memw_aligned::generate_memw_aligned_trace_segments_as(pieces, form) + } + + #[cfg(test)] + fn later(&self, cycles: u64) -> Self { + AlignedRow::later(*self, cycles) + } +} + +impl RowFamily for LoadOperation { + const TABLE: StreamTable = StreamTable::Load; + const COLUMNS: usize = load::cols::NUM_COLUMNS; + + fn fill(pieces: Vec<&[Self]>) -> RecordsFill<'_> { + RecordsFill::Load { + ops: pieces.into_iter().map(load::card_ops).collect(), + } + } + + fn host_fill(pieces: &[&[Self]], form: TraceForm) -> Table { + load::generate_load_trace_segments_as(pieces, form) + } + + #[cfg(test)] + fn later(&self, cycles: u64) -> Self { + Self { + timestamp: self.timestamp + cycles, + ..self.clone() + } + } +} + +impl RowFamily for StoreOperation { + const TABLE: StreamTable = StreamTable::Store; + const COLUMNS: usize = store::cols::NUM_COLUMNS; + + fn fill(pieces: Vec<&[Self]>) -> RecordsFill<'_> { + RecordsFill::Store { + ops: pieces.into_iter().map(store::card_ops).collect(), + } + } + + fn host_fill(pieces: &[&[Self]], form: TraceForm) -> Table { + store::generate_store_trace_segments_as(pieces, form) + } + + #[cfg(test)] + fn later(&self, cycles: u64) -> Self { + Self { + timestamp: self.timestamp + cycles, + ..self.clone() + } + } +} + +impl RowFamily for MemwOperation { + const TABLE: StreamTable = StreamTable::Memw; + const COLUMNS: usize = memw::cols::NUM_COLUMNS; + + fn fill(pieces: Vec<&[Self]>) -> RecordsFill<'_> { + RecordsFill::Memw { + ops: pieces.into_iter().map(memw::card_ops).collect(), + } + } + + fn host_fill(pieces: &[&[Self]], form: TraceForm) -> Table { + memw::generate_memw_trace_segments_as(pieces, form) + } + + #[cfg(test)] + fn later(&self, cycles: u64) -> Self { + Self { + timestamp: self.timestamp + cycles, + ..self.clone() + } + } +} + +impl RowFamily for ShiftOperation { + const TABLE: StreamTable = StreamTable::Shift; + const COLUMNS: usize = shift::cols::NUM_COLUMNS; + + fn fill(pieces: Vec<&[Self]>) -> RecordsFill<'_> { + RecordsFill::Shift { + ops: pieces.into_iter().map(shift::card_ops).collect(), + } + } + + fn host_fill(pieces: &[&[Self]], form: TraceForm) -> Table { + shift::generate_shift_trace_segments_as(pieces, form) + } + + /// SHIFT has no timestamp: the same op with its input's low half moved by + /// `cycles`, every column computed from that input, a row the CPU never + /// sent. + #[cfg(test)] + fn later(&self, cycles: u64) -> Self { + let mut op = self.clone(); + op.in_halves[0] = op.in_halves[0].wrapping_add(cycles as u16); + op + } +} + +/// A full chunk of a [`RowFamily`] held as its rows. +pub(crate) struct RowRecords { + rows: Segments, +} + +impl RowRecords { + /// The chunk `rows` make. Refused (the rows given back) unless they are a + /// whole table: a chunk of another height goes to the host's fill, which + /// pads it. + pub(crate) fn new(rows: Segments) -> Result> { + if whole_table(rows.len()) { + Ok(Self { rows }) + } else { + Err(rows) + } + } + + /// A test's fault: the same rows in one piece, each one's timestamp + /// `cycles` later. + #[cfg(test)] + pub(crate) fn later(self, cycles: u64) -> Self { + let pieces = self.rows.slices(); + let rows = pieces.iter().flat_map(|piece| piece.iter()); + Self { + rows: Segments::one(rows.map(|row| row.later(cycles)).collect()), + } + } +} + +impl std::fmt::Debug for RowRecords { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:?} rows: {} in {} pieces", + T::TABLE, + self.rows.len(), + self.rows.parts.len() + ) + } +} + +impl TableRecords for RowRecords { + fn rows(&self) -> usize { + self.rows.len() + } + + fn cols(&self) -> usize { + T::COLUMNS + } + + fn fill(&self) -> RecordsFill<'_> { + T::fill(self.rows.slices()) + } + + fn bytes(&self) -> usize { + self.rows.op_bytes() + } + + fn host_fill(&self) -> Option { + let mut trace = T::host_fill(&self.rows.slices(), TraceForm::Narrow); + // Written packed, except in a build that holds every trace wide. + trace.pack_main_narrow(); + trace.take_narrow_main() + } + + fn return_pages(&mut self) -> PagesReturned { + self.rows.return_pages() + } +} + +/// A chunk held as the rows of a [`RowFamily`], as the hand-out carries it: +/// what it weighs, the table it becomes, and the table a regenerator rebuilds +/// with the digest of the records it would have been held as. +pub(crate) trait RowChunk: Send + std::fmt::Debug { + fn op_bytes(&self) -> usize; + fn op_count(&self) -> usize; + /// The table held as these records. + fn into_trace(self: Box) -> Table; + /// With `digest`, the records' digest (`multilinear::records::digest_of`); + /// and the table's host fill in `form`. + fn regenerate(self: Box, form: TraceForm, digest: bool) -> (Option<[u64; 2]>, Table); +} + +impl RowChunk for RowRecords { + fn op_bytes(&self) -> usize { + self.rows.op_bytes() + } + + fn op_count(&self) -> usize { + self.rows.len() + } + + fn into_trace(self: Box) -> Table { + held(Records(Arc::new(*self))) + } + + fn regenerate(self: Box, form: TraceForm, digest: bool) -> (Option<[u64; 2]>, Table) { + let digest = digest.then(|| multilinear::records::digest_of(&*self)); + (digest, T::host_fill(&self.rows.slices(), form)) + } +} diff --git a/prover/src/tables/trace_builder/windowed/card_records_card_tests.rs b/prover/src/tables/trace_builder/windowed/card_records_card_tests.rs new file mode 100644 index 000000000..19d7a705d --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/card_records_card_tests.rs @@ -0,0 +1,559 @@ +//! ★ The card fills a streamed CPU or MEMW_R chunk, or a row family's +//! (MEMW_A, LOAD, STORE, MEMW, SHIFT), from its records ([`super::card_records`]) with the words the host fill writes, at +//! the places its group's store holds them: chunks whose logs or rows lie in +//! many window parts, alone and among a group's other tables, and, with +//! `BLOCK_WHIR_ELF` and `BLOCK_WHIR_INPUT`, a real block's chunks at the +//! production height. A CPU log at a pc with no DECODE row declines the group, +//! as the host fill refuses it; two CPU columns swapped are caught (with +//! `test-cuda-faults`). +//! +//! Card only: without a device every upload is declined, and these tests say +//! so instead of passing. + +use std::sync::Arc; + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::gpu::{TableColumns, upload_tables}; +use multilinear::mle::Mle; +use multilinear::narrow::NarrowColumns; +use multilinear::records::Records; + +use super::card_records::{CardDecode, CpuLogRecords}; +use super::{RecordsRoom, RowFamilies, Segments, StreamTable, WindowedTraceBuilder}; +use crate::tables::MaxRowsConfig; +use crate::tests::windowed_builder_tests::run; + +fn always() -> RecordsRoom { + Arc::new(|_| true) +} + +/// Every full CPU and MEMW_R chunk (with `rows`, every row family's too) +/// the run of `program` hands out as records, by windows of `window` cycles, +/// from the hand-out of window `from` on and through `windows` windows walked +/// (all of them: `None`): `(family, index, records)`. +fn records_chunks( + program: &Elf, + input: &[u8], + max_rows: &MaxRowsConfig, + window: usize, + (from, windows): (usize, Option), + rows: bool, +) -> Vec<(StreamTable, usize, Records)> { + let mut builder = WindowedTraceBuilder::new(program, input, max_rows) + .and_then(WindowedTraceBuilder::drop_streamed_ops) + .and_then(|b| b.card_fill_streamed(always(), true, true)) + .and_then(|b| { + if rows { + b.card_fill_streamed_rows(RowFamilies::ALL) + } else { + Ok(b) + } + }) + .expect("the builder"); + let mut executor = Executor::new(program, input.to_vec()).expect("the executor starts"); + let mut held = Vec::new(); + let (mut walker, mut accumulator) = builder.split(); + let mut walked = 0usize; + let mut ahead: Option> = None; + while let Some(logs) = executor.resume_with_limit(window).expect("the run") { + // The run's last window is the finish's. + if let Some(previous) = ahead.replace(logs.to_vec()) { + let w = walker.walk_owned(previous).expect("a window"); + for job in accumulator.absorb(w) { + let (table, index) = (job.table, job.index); + if walked < from || !job.is_records() { + continue; + } + let mut chunk = job.generate(); + if let Some(records) = chunk.trace.take_records_main() { + held.push((table, index, records)); + } + } + walked += 1; + } + if windows.is_some_and(|n| walked >= n) { + break; + } + } + held +} + +/// The group's store, column by column, as the card holds it. +fn uploaded(tables: &[TableColumns<'_, F>]) -> Vec> { + upload_tables(tables) + .expect("the card takes the group") + .download() + .expect("the store comes back") +} + +/// Mismatched words between two stores, and the first. +fn mismatches(got: &[Vec], want: &[Vec]) -> (usize, Option<(usize, usize)>) { + let mut count = 0; + let mut first = None; + assert_eq!(got.len(), want.len(), "the columns"); + for (c, (g, w)) in got.iter().zip(want).enumerate() { + assert_eq!(g.len(), w.len(), "column {c}'s rows"); + for (r, (a, b)) in g.iter().zip(w).enumerate() { + if a != b { + count += 1; + first.get_or_insert((c, r)); + } + } + } + (count, first) +} + +fn host_words(packed: &NarrowColumns) -> Vec> { + (0..packed.cols()).map(|c| packed.column(c)).collect() +} + +/// `records`' table filled on the card against its host fill, uploaded +/// packed and as the host's words. +fn check(what: &str, records: &Records) { + let packed = records.host_fill().expect("the host fills it"); + let fills = multilinear::records::fills().0; + let card = uploaded(&[TableColumns::Records(records)]); + assert_eq!( + multilinear::records::fills().0, + fills + 1, + "{what}: filled on the card" + ); + assert_eq!( + mismatches(&card, &uploaded(&[TableColumns::Narrow(&packed)])), + (0, None), + "{what}: against the widened host fill" + ); + assert_eq!( + mismatches(&card, &host_words(&packed)), + (0, None), + "{what}: against the host's words" + ); +} + +#[test] +fn the_card_fills_cpu_and_memw_r_chunks_from_their_records_as_the_host_does() { + let max_rows = MaxRowsConfig::small(); + let mut filled = [0usize; 2]; + for name in ["all_instructions_64", "test_keccak_multi", "test_ecsm"] { + let (program, _) = run(name); + for window in [1, 7, 33] { + for (table, index, records) in + records_chunks(&program, &[], &max_rows, window, (0, None), false) + { + check( + &format!("{name}, window {window}, {table:?}[{index}]"), + &records, + ); + filled[usize::from(table == StreamTable::MemwRegister)] += 1; + } + } + } + assert!(filled[0] > 0 && filled[1] > 0, "{filled:?}"); +} + +/// A row family's records: its place in [`ROW_FAMILIES`] and its pieces. +fn row_family(records: &Records) -> Option<(usize, usize)> { + use multilinear::records::RecordsFill as R; + match records.fill() { + R::MemwA { rows } => Some((0, rows.len())), + R::Load { ops } => Some((1, ops.len())), + R::Store { ops } => Some((2, ops.len())), + R::Memw { ops } => Some((3, ops.len())), + R::Shift { ops } => Some((4, ops.len())), + _ => None, + } +} + +/// The row families, in [`row_family`]'s order. +const ROW_FAMILIES: [&str; 5] = ["MEMW_A", "LOAD", "STORE", "MEMW", "SHIFT"]; + +/// Every row family's chunks of four rows, so that the small programs fill +/// several, their rows in one window part or many: MEMW_A's, LOAD's, +/// STORE's and SHIFT's (no small program makes four MEMW ops; the synthetic +/// MEMW chunk below is MEMW's). +#[test] +fn the_card_fills_row_family_chunks_from_their_rows_as_the_host_does() { + let max_rows = MaxRowsConfig { + memw_aligned: 1 << 2, + load: 1 << 2, + store: 1 << 2, + memw: 1 << 2, + shift: 1 << 2, + ..MaxRowsConfig::small() + }; + // Per family: chunks filled, and those in more than one piece. + let mut filled = [[0usize; 2]; 5]; + for name in [ + "all_instructions_64", + "test_keccak_multi", + "all_loadstore_32", + "test_blake3", + ] { + let (program, _) = run(name); + for window in [1, 7, 33] { + for (table, index, records) in + records_chunks(&program, &[], &max_rows, window, (0, None), true) + { + let Some((family, pieces)) = row_family(&records) else { + continue; + }; + check( + &format!("{name}, window {window}, {table:?}[{index}]"), + &records, + ); + filled[family][0] += 1; + filled[family][1] += usize::from(pieces > 1); + } + } + } + // No small program makes four MEMW ops (the synthetic MEMW chunk below + // covers MEMW), and a STORE chunk is always one piece (its hand-out cuts + // one list). + let [memw_a, load, store, _memw, shift] = filled; + assert!( + [memw_a, load, store, shift].iter().all(|f| f[0] > 0) + && [memw_a, load, shift].iter().all(|f| f[1] > 0), + "{ROW_FAMILIES:?}: {filled:?}" + ); +} + +/// A MEMW chunk the card fills from its ops as the host does (no small +/// program makes four MEMW ops): sixteen random ops, memory and register +/// accesses at every width, reads and writes, low words about to carry, old +/// timestamps up to 2^64 - 1, in three window parts. +#[test] +fn the_card_fills_a_memw_chunk_from_its_ops_as_the_host_does() { + use super::card_records::RowRecords; + use crate::tables::memw::MemwOperation; + let mut rng = 0x4D45_4D57_u64; + let mut next = move || { + rng = rng + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + rng ^ (rng >> 29) + }; + let ops: Vec = (0..16u32) + .map(|i| { + let is_register = i % 3 == 1; + let element = |w: u64| { + if is_register { + w as u32 + } else { + (w & 0xFF) as u32 + } + }; + let mut op = MemwOperation { + base_address: (next() & !0xFFFF_FFFF) | (0xFFFF_FFF8 + u64::from(i % 8)), + timestamp: next() >> 1, + old_timestamp: std::array::from_fn(|_| next() >> 2), + value: std::array::from_fn(|_| element(next())), + old: std::array::from_fn(|_| element(next())), + is_register, + width: 1 << (i % 4), + is_read: i % 2 == 0, + }; + if i == 5 { + op.old_timestamp[3] = u64::MAX; + } + op + }) + .collect(); + let parts = [0..5, 5..12, 12..16] + .into_iter() + .map(|range| (Arc::new(ops[range.clone()].to_vec()), 0..range.len())) + .collect(); + let Ok(records) = RowRecords::new(Segments { parts }) else { + panic!("16 ops are a whole table"); + }; + check("MEMW, 16 ops in 3 parts", &Records(Arc::new(records))); +} + +/// Among a wide table, a packed one and each other, the filled tables' +/// columns sit where their widened columns would, and the others are as +/// uploaded. +#[test] +fn the_card_fills_cpu_and_memw_r_in_place_among_a_groups_tables() { + let (program, _) = run("test_keccak_multi"); + let chunks = records_chunks(&program, &[], &MaxRowsConfig::small(), 7, (0, None), false); + let cpu = &chunks + .iter() + .find(|(t, ..)| *t == StreamTable::Cpu) + .expect("a CPU chunk") + .2; + let memw_r = &chunks + .iter() + .find(|(t, ..)| *t == StreamTable::MemwRegister) + .expect("a MEMW_R chunk") + .2; + let wide: Vec> = (0..4u64) + .map(|c| { + Mle::new( + (0..16u64) + .map(|r| FieldElement::from(c * 1000 + r)) + .collect(), + ) + .expect("a column") + }) + .collect(); + let (cpu_packed, memw_r_packed) = ( + cpu.host_fill().expect("the host fills it"), + memw_r.host_fill().expect("the host fills it"), + ); + let card = uploaded(&[ + TableColumns::Records(memw_r), + TableColumns::Wide(&wide), + TableColumns::Records(cpu), + TableColumns::Narrow(&memw_r_packed), + TableColumns::Records(memw_r), + ]); + let host = uploaded(&[ + TableColumns::Narrow(&memw_r_packed), + TableColumns::Wide(&wide), + TableColumns::Narrow(&cpu_packed), + TableColumns::Narrow(&memw_r_packed), + TableColumns::Narrow(&memw_r_packed), + ]); + assert_eq!(mismatches(&card, &host), (0, None)); +} + +/// A CPU chunk with one log at a pc the program has no instruction at: the +/// card declines its group (the kernel counts the log), as the host fill +/// refuses it; the same chunk untouched goes up. +#[test] +fn a_cpu_log_at_a_pc_with_no_decode_row_declines_the_group() { + let (program, logs) = run("all_instructions_64"); + let max_rows = MaxRowsConfig::small(); + let builder = WindowedTraceBuilder::new(&program, &[], &max_rows).expect("the builder"); + let decode = Arc::new(CardDecode::new(Arc::clone(&builder.decode)).expect("a program")); + let rows = max_rows.cpu; + let honest = CpuLogRecords::new(Segments::one(logs[..rows].to_vec()), 0, Arc::clone(&decode)) + .expect("a whole table"); + let honest = Records(Arc::new(honest)); + assert!(upload_tables::(&[TableColumns::Records(&honest)]).is_some()); + // A pc past every instruction, 4-aligned. + let absent = decode + .records + .entries() + .iter() + .map(|e| e.pc) + .max() + .expect("a row") + + 0x1000; + assert_eq!(decode.records.row(absent), None); + for at in [0, rows / 2, rows - 1] { + let mut tampered = logs[..rows].to_vec(); + tampered[at].current_pc = absent; + let records = CpuLogRecords::new(Segments::one(tampered), 0, Arc::clone(&decode)) + .expect("a whole table"); + let records = Records(Arc::new(records)); + assert!( + records.host_fill().is_none(), + "log {at}: the host refuses it" + ); + assert!( + upload_tables::(&[TableColumns::Records(&records)]).is_none(), + "log {at}: the card declines it" + ); + } +} + +/// Two CPU columns swapped by the fill are caught word by word: the +/// comparison above is not vacuous. +#[cfg(feature = "test-cuda-faults")] +#[test] +fn two_cpu_columns_swapped_are_caught() { + use std::sync::atomic::Ordering::Relaxed; + let (program, _) = run("test_keccak_multi"); + let chunks = records_chunks(&program, &[], &MaxRowsConfig::small(), 7, (0, None), false); + let (_, index, cpu) = chunks + .iter() + .find(|(t, ..)| *t == StreamTable::Cpu) + .expect("a CPU chunk"); + let packed = cpu.host_fill().expect("the host fills it"); + let columns = math_cuda::fill::CPU_COLUMNS; + // TIMESTAMP and PC_0. + math_cuda::fill::FAULT_CPU_SWAP.store(1 + columns, Relaxed); + let card = upload_tables::(&[TableColumns::Records(cpu)]).and_then(|s| s.download()); + math_cuda::fill::FAULT_CPU_SWAP.store(0, Relaxed); + let card = card.expect("the card takes it"); + let (count, first) = mismatches(&card, &host_words(&packed)); + assert!( + count > 0 && first.is_some_and(|(c, _)| c <= 1), + "CPU[{index}]: columns 0 and 1 swapped, and every word the host's" + ); +} + +/// The real block's CPU chunk 32 (windows 64 and 65, two pieces) and the +/// first MEMW_R chunk handed out from window 64, at the production height, +/// filled on the card as on the host; and chunk 32 with one log at an absent +/// pc declined. +#[test] +#[ignore = "card: a box test (BLOCK_WHIR_ELF, BLOCK_WHIR_INPUT)"] +fn the_card_fills_a_real_blocks_cpu_and_memw_r_chunks_as_the_host_does() { + let elf = + std::fs::read(std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF")).expect("the ELF"); + let input = + std::fs::read(std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT")).expect("input"); + let program = Elf::load(&elf).expect("the ELF loads"); + let options = crate::block_whir::BlockOptions::production(); + let window = 1usize << options.window_log2.expect("the block streams"); + let t = std::time::Instant::now(); + let chunks = records_chunks( + &program, + &input, + &options.max_rows, + window, + (64, Some(66)), + false, + ); + let cpu = chunks + .iter() + .find(|(t, i, _)| *t == StreamTable::Cpu && *i == 32) + .expect("CPU chunk 32"); + let memw_r = chunks + .iter() + .find(|(t, _, r)| { + *t == StreamTable::MemwRegister + && matches!(r.fill(), multilinear::records::RecordsFill::MemwR { rows } if rows.len() > 1) + }) + .or_else(|| chunks.iter().rev().find(|(t, ..)| *t == StreamTable::MemwRegister)) + .expect("a MEMW_R chunk"); + println!( + "CARD FILL REAL capture_secs={:.2} cpu_rows={} memw_r_index={} memw_r_rows={} held_chunks={}", + t.elapsed().as_secs_f64(), + cpu.2.rows(), + memw_r.1, + memw_r.2.rows(), + chunks.len() + ); + for (table, index, records) in [cpu, memw_r] { + let t = std::time::Instant::now(); + check(&format!("real {table:?}[{index}]"), records); + println!( + "CARD FILL REAL {table:?}[{index}] identity 0 words differ secs={:.2}", + t.elapsed().as_secs_f64() + ); + } + let multilinear::records::RecordsFill::Cpu { + logs, first_cycle, .. + } = cpu.2.fill() + else { + panic!("CPU chunk 32 holds logs"); + }; + let builder = + WindowedTraceBuilder::new(&program, &input, &options.max_rows).expect("the builder"); + let decode = Arc::new(CardDecode::new(Arc::clone(&builder.decode)).expect("a program")); + let absent = decode + .records + .entries() + .iter() + .map(|e| e.pc) + .max() + .expect("a row") + + 0x1000; + let mut tampered: Vec = logs + .iter() + .flat_map(|piece| piece.iter()) + .map(|l| Log { + current_pc: l.current_pc, + next_pc: l.next_pc, + src1_val: l.src1_val, + src2_val: l.src2_val, + dst_val: l.dst_val, + }) + .collect(); + let at = tampered.len() / 3; + tampered[at].current_pc = absent; + let records = Records(Arc::new( + CpuLogRecords::new(Segments::one(tampered), first_cycle, decode).expect("a whole table"), + )); + assert!( + upload_tables::(&[TableColumns::Records(&records)]).is_none(), + "an absent pc declined" + ); + println!("CARD FILL REAL absent pc at log {at}: declined by the card"); + println!("CARD FILL REAL VERDICT PASS"); +} + +/// The real block's first chunk of each row family held as its rows, at the +/// production height (each in the window parts of several windows), filled on +/// the card as on the host. +#[test] +#[ignore = "card: a box test (BLOCK_WHIR_ELF, BLOCK_WHIR_INPUT)"] +fn the_card_fills_a_real_blocks_row_family_chunks_as_the_host_does() { + let elf = + std::fs::read(std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF")).expect("the ELF"); + let input = + std::fs::read(std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT")).expect("input"); + let program = Elf::load(&elf).expect("the ELF loads"); + let options = crate::block_whir::BlockOptions::production(); + let window = 1usize << options.window_log2.expect("the block streams"); + let t = std::time::Instant::now(); + let mut builder = WindowedTraceBuilder::new(&program, &input, &options.max_rows) + .and_then(WindowedTraceBuilder::drop_streamed_ops) + .and_then(|b| b.card_fill_streamed(always(), false, false)) + .and_then(|b| b.card_fill_streamed_rows(RowFamilies::ALL)) + .expect("the builder"); + let mut executor = Executor::new(&program, input.clone()).expect("the executor starts"); + let (mut walker, mut accumulator) = builder.split(); + // The first chunk of each family, in [`ROW_FAMILIES`]' order. + let mut first: [Option<(StreamTable, usize, Records)>; 5] = Default::default(); + let mut walked = 0usize; + let mut ahead: Option> = None; + while let Some(logs) = executor.resume_with_limit(window).expect("the run") { + if let Some(previous) = ahead.replace(logs.to_vec()) { + let w = walker.walk_owned(previous).expect("a window"); + for job in accumulator.absorb(w) { + let (table, index) = (job.table, job.index); + let at = match table { + StreamTable::MemwAligned => 0, + StreamTable::Load => 1, + StreamTable::Store => 2, + StreamTable::Memw => 3, + StreamTable::Shift => 4, + _ => continue, + }; + if first[at].is_some() || !job.is_records() { + continue; + } + let mut chunk = job.generate(); + first[at] = chunk + .trace + .take_records_main() + .map(|records| (table, index, records)); + } + walked += 1; + } + if first.iter().all(Option::is_some) { + break; + } + } + let missing: Vec<&str> = ROW_FAMILIES + .iter() + .zip(&first) + .filter_map(|(name, chunk)| chunk.is_none().then_some(*name)) + .collect(); + assert!( + missing.is_empty(), + "{walked} windows walked: no {missing:?} chunk held as rows" + ); + for (table, index, records) in first.iter().flatten() { + let pieces = row_family(records).map_or(0, |(_, pieces)| pieces); + println!( + "CARD FILL REAL ROWS capture_secs={:.2} windows={walked} {table:?}[{index}] rows={} pieces={pieces}", + t.elapsed().as_secs_f64(), + records.rows() + ); + let t = std::time::Instant::now(); + check(&format!("real {table:?}[{index}]"), records); + println!( + "CARD FILL REAL ROWS {table:?}[{index}] identity 0 words differ secs={:.2}", + t.elapsed().as_secs_f64() + ); + } + println!("CARD FILL REAL ROWS VERDICT PASS"); +} diff --git a/prover/src/tables/trace_builder/windowed/card_records_tests.rs b/prover/src/tables/trace_builder/windowed/card_records_tests.rs new file mode 100644 index 000000000..15627a2c8 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/card_records_tests.rs @@ -0,0 +1,920 @@ +//! CPU and MEMW_R chunks, and the row families' (MEMW_A, LOAD, STORE, MEMW, +//! SHIFT), handed out as the +//! records their rows come from ([`WindowedTraceBuilder::card_fill_streamed`], +//! [`WindowedTraceBuilder::card_fill_streamed_rows`]): their host fill is the +//! table the build makes without it, whatever windows, walkers and room; the +//! logs out of step with the ops, and every chunk past the first the room rule +//! refuses, are filled on the host. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use executor::elf::Elf; +use executor::vm::logs::Log; +use math::field::element::FieldElement; + +use super::super::super::types::GoldilocksField; +use super::{ + CardFillStamps, ChunkJob, RecordsRoom, RowFamilies, StreamTable, StreamedChunk, Table, + WalkedWindow, WindowedTraceBuilder, +}; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::Traces; +use crate::tests::windowed_builder_tests::{run, same_traces, whole}; + +/// The chunk heights the row families' tests run at: chunks of four rows, so +/// that the small programs fill several. +fn rows_max() -> MaxRowsConfig { + MaxRowsConfig { + memw_aligned: 1 << 2, + load: 1 << 2, + store: 1 << 2, + memw: 1 << 2, + shift: 1 << 2, + ..MaxRowsConfig::small() + } +} + +/// Whether `table` is a row family's. +fn row_table(table: StreamTable) -> bool { + matches!( + table, + StreamTable::MemwAligned + | StreamTable::Load + | StreamTable::Store + | StreamTable::Memw + | StreamTable::Shift + ) +} + +/// Every family's `[as records, on the host]`: CPU, MEMW_R, then the row +/// families. +fn families(s: &CardFillStamps) -> [[usize; 2]; 7] { + [s.cpu, s.memw_r, s.memw_a, s.load, s.store, s.memw, s.shift] +} + +/// A room rule that admits every chunk. +fn always() -> RecordsRoom { + Arc::new(|_| true) +} + +/// A room rule that admits the first `n` chunks it is asked about, then none +/// for `gap` asks, then every one again: a rule that changes its mind. +fn for_first(n: usize, gap: usize) -> RecordsRoom { + let asked = AtomicUsize::new(0); + Arc::new(move |_| { + let k = asked.fetch_add(1, Ordering::Relaxed); + k < n || k >= n + gap + }) +} + +/// `chunk` with its records, if it holds them, replaced by their host fill at +/// eight bytes a cell; and whether it held them. +fn unheld(mut chunk: StreamedChunk) -> (StreamedChunk, bool) { + let Some(records) = chunk.trace.take_records_main() else { + return (chunk, false); + }; + let packed = records.host_fill().expect("the host fills its records"); + assert_eq!( + (packed.rows(), packed.cols()), + (records.rows(), records.cols()), + "{:?}[{}]: the host fill's shape", + chunk.table, + chunk.index + ); + let columns = packed + .widen::() + .expect("the packed columns widen"); + let (rows, cols) = (records.rows(), records.cols()); + let mut data: Vec> = Vec::with_capacity(rows * cols); + for r in 0..rows { + data.extend(columns.iter().map(|column| column.evals()[r])); + } + chunk.trace = Table::new_main(data, cols, 1); + (chunk, true) +} + +/// How a test hands the accumulator its windows. +#[derive(Clone, Copy, Debug)] +enum Walk { + /// The builder's own walker, keeping each window's logs. + Own, + /// The builder's own walker, window `k`'s logs swapped with window + /// `k + 1`'s (two windows of one length): logs out of step with the ops. + Swapped(usize), + /// On this many threads, from the windows' entries. + #[cfg(feature = "parallel")] + Threads(usize), +} + +/// The run of `program` by windows of `window` cycles, its CPU and MEMW_R +/// chunks (with `rows`, every row family's too) handed out as records +/// while `room` admits them and each records chunk then filled on the host, +/// with the chunks put back; how many chunks were records; and the builder's +/// stamps. +fn held( + program: &Elf, + logs: &[Log], + max_rows: &MaxRowsConfig, + window: usize, + walk: Walk, + room: RecordsRoom, + rows: bool, +) -> (Traces, usize, CardFillStamps) { + let mut builder = WindowedTraceBuilder::new(program, &[], max_rows) + .and_then(WindowedTraceBuilder::drop_streamed_ops) + .and_then(|b| b.card_fill_streamed(room, true, true)) + .and_then(|b| { + if rows { + b.card_fill_streamed_rows(RowFamilies::ALL) + } else { + Ok(b) + } + }) + .expect("the builder"); + assert!(builder.keeps_logs()); + let mut jobs: Vec = Vec::new(); + let last = match walk { + Walk::Own | Walk::Swapped(_) => { + let body = logs.len() - 1; + let cut = body - body % window; + let windows: Vec<&[Log]> = logs[..cut].chunks(window).collect(); + let (mut walker, mut accumulator) = builder.split(); + let mut walked: Vec = windows + .iter() + .map(|w| walker.walk(w).expect("a window")) + .collect(); + for (k, w) in walked.iter_mut().enumerate() { + let from = match walk { + Walk::Swapped(s) if k == s => s + 1, + Walk::Swapped(s) if k == s + 1 => s, + _ => k, + }; + w.keep_logs(Arc::new(windows[from].to_vec())); + } + for w in walked { + jobs.extend(accumulator.absorb(w)); + } + logs[cut..].to_vec() + } + #[cfg(feature = "parallel")] + Walk::Threads(walkers) => { + use executor::vm::execution::Executor; + let (tx, rx) = std::sync::mpsc::sync_channel(2); + let report = std::thread::scope(|scope| { + scope.spawn(move || { + let mut executor = Executor::new(program, Vec::new()).expect("the executor"); + let mut start = Some(executor.memory().snapshot()); + while let Some(logs) = executor + .resume_with_limit(window) + .expect("the run") + .map(<[Log]>::to_vec) + { + let next = Some(executor.memory().snapshot()); + if tx + .send((logs, std::mem::replace(&mut start, next))) + .is_err() + { + break; + } + } + }); + super::parallel::walk_windows(&mut builder, rx, walkers, |accumulator, w, _| { + jobs.extend(accumulator.absorb(w)); + Ok(()) + }) + }) + .expect("the walk"); + report.last + } + }; + let stamps = builder.stamps().card_fill; + let mut records = 0usize; + let chunks: Vec = jobs + .into_iter() + .map(|job| { + let was = job.is_records(); + let (chunk, held) = unheld(job.generate()); + assert_eq!( + held, was, + "{:?}[{}]: held as records", + chunk.table, chunk.index + ); + if held { + let row_family = row_table(chunk.table); + assert!( + matches!(chunk.table, StreamTable::Cpu | StreamTable::MemwRegister) + || (rows && row_family), + "{:?} held as records", + chunk.table + ); + records += 1; + } + chunk + }) + .collect(); + let mut traces = builder.finish(&last).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + (traces, records, stamps) +} + +/// ★ Every full CPU and MEMW_R chunk handed out as records, its logs or rows +/// in the window parts it was cut from (windows of 1, 7, 32 and 33 cycles, +/// walked by the builder or on 1 and 4 threads), fills on the host to the +/// table the whole-run build makes; every other table is the same, and no +/// row family's chunk is held unless asked. +#[test] +fn cpu_and_memw_r_chunks_held_as_records_are_the_runs_tables() { + chunks_held_as_records_are_the_runs_tables(false); +} + +/// ★ The same with every full chunk of the row families held as its rows too +/// ([`WindowedTraceBuilder::card_fill_streamed_rows`]; chunks of four rows, +/// and programs that load, store and shift more): 48 B an aligned row, 32 a +/// load or a store, 152 a MEMW op, 24 a shift. +#[test] +fn row_family_chunks_held_as_records_are_the_runs_tables() { + chunks_held_as_records_are_the_runs_tables(true); +} + +fn chunks_held_as_records_are_the_runs_tables(rows: bool) { + let (max_rows, more) = if rows { + (rows_max(), &["all_loadstore_32", "test_blake3"][..]) + } else { + (MaxRowsConfig::small(), &[][..]) + }; + let mut walks = vec![Walk::Own]; + #[cfg(feature = "parallel")] + walks.extend([Walk::Threads(1), Walk::Threads(4)]); + let mut held_most = [0usize; 7]; + let names = [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm", + "test_commit_split", + ]; + for &name in names.iter().chain(more) { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [1, 7, 32, 33] { + for &walk in &walks { + let what = format!("{name}, window {window}, {walk:?}"); + let (traces, records, stamps) = + held(&program, &logs, &max_rows, window, walk, always(), rows); + same_traces(&reference, &traces); + let families = families(&stamps); + assert!(families.iter().all(|f| f[1] == 0), "{what}: {stamps:?}"); + if !rows { + assert!(families[2..].iter().all(|f| f[0] == 0), "{what}"); + } + assert_eq!(stamps.out_of_step, 0, "{what}"); + assert_eq!(stamps.room_out, None, "{what}"); + assert_eq!( + records, + families.iter().map(|f| f[0]).sum::(), + "{what}" + ); + for (most, family) in held_most.iter_mut().zip(families) { + *most = (*most).max(family[0]); + } + assert_eq!( + stamps.records_bytes, + stamps.cpu[0] * max_rows.cpu * 40 + + stamps.memw_r[0] * max_rows.memw_register * 32 + + stamps.memw_a[0] * max_rows.memw_aligned * 48 + + stamps.load[0] * max_rows.load * 32 + + stamps.store[0] * max_rows.store * 32 + + stamps.memw[0] * max_rows.memw * 152 + + stamps.shift[0] * max_rows.shift * 24, + "{what}: 40 B a cycle, 32 a register row, 48 an aligned row, 32 a load or \ + store, 152 a MEMW op, 24 a shift" + ); + } + } + } + // No small program makes four MEMW ops: every other family fills several. + let asked: &[usize] = if rows { &[0, 1, 2, 3, 4, 6] } else { &[0, 1] }; + assert!(asked.iter().all(|&k| held_most[k] > 1), "{held_most:?}"); +} + +/// Each row family asked alone is the only one held as rows: the others are +/// filled on the host, uncounted, and the tables are the run's. +#[test] +fn a_row_family_asked_alone_is_the_only_one_held() { + let max_rows = rows_max(); + let (program, logs) = run("test_blake3"); + let reference = whole(&program, &logs, &max_rows); + let alone = |k: usize| { + let mut rows = RowFamilies::NONE; + *[ + &mut rows.memw_a, + &mut rows.load, + &mut rows.store, + &mut rows.memw, + &mut rows.shift, + ][k] = true; + rows + }; + for k in 0..5 { + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows) + .and_then(WindowedTraceBuilder::drop_streamed_ops) + .and_then(|b| b.card_fill_streamed(always(), false, false)) + .and_then(|b| b.card_fill_streamed_rows(alone(k))) + .expect("the builder"); + let body = logs.len() - 1; + let cut = body - body % 8; + let mut chunks = Vec::new(); + { + let (mut walker, mut accumulator) = builder.split(); + for w in logs[..cut].chunks(8) { + let walked = walker.walk_owned(w.to_vec()).expect("a window"); + chunks.extend( + accumulator + .absorb(walked) + .into_iter() + .map(|job| unheld(job.generate()).0), + ); + } + } + let stamps = builder.stamps().card_fill; + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + same_traces(&reference, &traces); + let rows = &families(&stamps)[2..]; + for (j, counts) in rows.iter().enumerate() { + if j == k { + // test_blake3 makes no four MEMW ops. + assert_eq!(counts[0] > 0, j != 3, "family {k}: {stamps:?}"); + } else { + assert_eq!( + *counts, + [0, 0], + "family {k}: family {j} counted: {stamps:?}" + ); + } + } + assert_eq!(stamps.cpu, [0, 0], "{stamps:?}"); + } +} + +/// ★ A window's logs swapped with the next window's (one length, so the +/// tails stay the same length) are out of step with the CPU ops: every CPU +/// chunk they fall in is filled on the host, counted, and the tables are the +/// run's. +#[test] +fn cpu_logs_out_of_step_with_the_ops_are_filled_on_the_host() { + let max_rows = MaxRowsConfig::small(); + let mut held_elsewhere = 0; + for name in ["all_instructions_64", "test_keccak_multi"] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for (window, swapped) in [(32, 0), (7, 1), (33, 0)] { + // Both windows before the last. + if (logs.len() - 1) / window < swapped + 2 { + continue; + } + let what = format!( + "{name}, window {window}, windows {swapped} and {} swapped", + swapped + 1 + ); + let (traces, _, stamps) = held( + &program, + &logs, + &max_rows, + window, + Walk::Swapped(swapped), + always(), + false, + ); + same_traces(&reference, &traces); + assert!(stamps.out_of_step > 0, "{what}: {stamps:?}"); + assert!(stamps.cpu[1] >= stamps.out_of_step, "{what}: {stamps:?}"); + held_elsewhere += stamps.cpu[0]; + } + } + assert!( + held_elsewhere > 0, + "the chunks the swap misses are still held" + ); +} + +/// ★ The room rule is read chunk by chunk, across the families held, and from +/// the first chunk it refuses on every chunk is filled on the host, whatever +/// it says after: what is held as records is the prefix it admitted. The line +/// names that chunk, and the tables are the run's. +#[test] +fn from_the_first_chunk_without_room_every_chunk_is_filled_on_the_host() { + let sum = |s: &CardFillStamps, k: usize| families(s).iter().map(|f| f[k]).sum::(); + for rows in [false, true] { + let (name, max_rows) = if rows { + ("test_blake3", rows_max()) + } else { + ("all_instructions_64", MaxRowsConfig::small()) + }; + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + let (_, _, all) = held(&program, &logs, &max_rows, 32, Walk::Own, always(), rows); + let chunks = sum(&all, 0); + assert!(chunks > 4, "{all:?}"); + if rows { + assert!( + families(&all)[2..].iter().filter(|f| f[0] > 0).count() >= 2, + "{all:?}" + ); + } + for admitted in [0, 1, 3, chunks - 1] { + let what = format!("rows {rows}, {admitted} admitted"); + let (traces, records, stamps) = held( + &program, + &logs, + &max_rows, + 32, + Walk::Own, + for_first(admitted, 1), + rows, + ); + same_traces(&reference, &traces); + assert_eq!(records, admitted, "{what}: {stamps:?}"); + assert_eq!(sum(&stamps, 0), admitted, "{what}"); + assert_eq!( + sum(&stamps, 1), + chunks - admitted, + "{what}: every later chunk on the host" + ); + assert!( + stamps.room_out.is_some(), + "{what}: the chunk room ran out at" + ); + } + // Never refused: no chunk named. + assert_eq!(all.room_out, None); + } +} + +/// A chunk of any height but a whole table's is not held as records: the +/// host's fill pads it. +#[test] +fn only_a_whole_table_is_held_as_records() { + use super::Segments; + use super::card_records::{CardDecode, CpuLogRecords, RegisterRowRecords, RowRecords}; + use crate::tables::load::LoadOperation; + use crate::tables::memw_aligned::AlignedRow; + let (program, logs) = run("all_instructions_64"); + let builder = + WindowedTraceBuilder::new(&program, &[], &MaxRowsConfig::small()).expect("the builder"); + let decode = Arc::new(CardDecode::new(Arc::clone(&builder.decode)).expect("a program")); + for n in [0usize, 1, 2, 3, 5, 31, 33] { + let some = Segments::one(logs[..n].to_vec()); + assert!( + CpuLogRecords::new(some, 0, Arc::clone(&decode)).is_none(), + "{n} logs" + ); + } + for n in [4usize, 32] { + let some = Segments::one(logs[..n].to_vec()); + assert!( + CpuLogRecords::new(some, 0, Arc::clone(&decode)).is_some(), + "{n} logs" + ); + } + let row = crate::tables::memw_register::RegRow::new(0, 9, 0, 0, 0, 0, 8, false); + for n in [0usize, 3, 7] { + assert!( + RegisterRowRecords::new(Segments::one(vec![row; n])).is_err(), + "{n} rows" + ); + } + assert!(RegisterRowRecords::new(Segments::one(vec![row; 8])).is_ok()); + let aligned = AlignedRow::default(); + let load = LoadOperation::new(0x100, 9, 4, true, [1; 8]); + for n in [0usize, 2, 3, 7, 33] { + assert!( + RowRecords::new(Segments::one(vec![aligned; n])).is_err(), + "{n} aligned rows" + ); + assert!( + RowRecords::new(Segments::one(vec![load.clone(); n])).is_err(), + "{n} loads" + ); + } + for n in [4usize, 32] { + assert!(RowRecords::new(Segments::one(vec![aligned; n])).is_ok()); + assert!(RowRecords::new(Segments::one(vec![load.clone(); n])).is_ok()); + } +} + +/// ★ A CPU chunk held as its logs is laid out for the CPU AIR the block lays a +/// streamed chunk out with: the records declare the preprocessed TIMESTAMP +/// column the fill writes from the row index, so the layout's check of it +/// passes. A MEMW_R chunk declares none, as its AIR has none. +#[test] +fn records_chunks_are_laid_out_for_their_tables_airs() { + use super::Segments; + use super::card_records::{CardDecode, CpuLogRecords, RegisterRowRecords}; + use crate::tables::cpu; + use crate::test_utils::{create_cpu_air, create_memw_register_air}; + use multilinear::records::{Records, TableRecords}; + use stark::proof::options::ProofOptions; + + let (program, logs) = run("all_instructions_64"); + let builder = + WindowedTraceBuilder::new(&program, &[], &MaxRowsConfig::small()).expect("the builder"); + let decode = Arc::new(CardDecode::new(Arc::clone(&builder.decode)).expect("a program")); + let opts = ProofOptions::default_test_options(); + let height = 32usize; + let cpu_air = create_cpu_air(&opts, 0).with_lazy_preprocessed_columns( + cpu::lazy_timestamp_commitment(&opts, height), + cpu::NUM_PRECOMPUTED_COLS, + Arc::new(move || vec![cpu::timestamp_column(height)]), + ); + type Air = dyn stark::traits::AIR< + Field = GoldilocksField, + FieldExtension = crate::tables::types::GoldilocksExtension, + PublicInputs = (), + >; + let lay_out = |records: Records, air: &Air| { + let cols = records.cols(); + let rows = records.rows(); + let mut trace = Table::try_from_records_main(records, 1).expect("a records trace"); + crate::block_whir::table_of_records(air, &mut trace, (cols, rows.trailing_zeros() as usize)) + .map(|_| ()) + }; + let cpu_records = CpuLogRecords::new(Segments::one(logs[..height].to_vec()), 0, decode) + .expect("a whole table"); + assert_eq!(cpu_records.row_index_columns(), cpu::NUM_PRECOMPUTED_COLS); + lay_out(Records(Arc::new(cpu_records)), &cpu_air).expect("CPU laid out"); + let row = crate::tables::memw_register::RegRow::new(0, 9, 0, 0, 0, 0, 8, false); + let Ok(memw_r) = RegisterRowRecords::new(Segments::one(vec![row; 8])) else { + panic!("8 rows are a whole table"); + }; + assert_eq!(memw_r.row_index_columns(), 0); + lay_out(Records(Arc::new(memw_r)), &create_memw_register_air(&opts)).expect("MEMW_R laid out"); +} + +/// ★ The test fault on a CPU chunk held as its logs (register values XORed, +/// [`WindowedTraceBuilder::card_fill_streamed_fault`]) makes a table the run +/// did not: its host fill differs from the honest chunk's, so a block test's +/// refusal of it is not vacuous. XORed with 0, the chunk is the honest one. +#[test] +fn the_cpu_records_fault_makes_a_table_the_run_did_not() { + use super::Segments; + use super::card_records::{CardDecode, CpuLogRecords}; + use multilinear::records::Records; + let (program, logs) = run("test_keccak_multi"); + let builder = + WindowedTraceBuilder::new(&program, &[], &MaxRowsConfig::small()).expect("the builder"); + let decode = Arc::new(CardDecode::new(Arc::clone(&builder.decode)).expect("a program")); + let rows = MaxRowsConfig::small().cpu; + let chunk = |by: Option| { + let records = CpuLogRecords::new( + Segments::one(logs[rows..2 * rows].to_vec()), + rows as u64, + Arc::clone(&decode), + ) + .expect("a whole table"); + let records = match by { + Some(by) => records.with_register_values_xored(by), + None => records, + }; + Records(Arc::new(records)) + .host_fill() + .expect("the host fills it") + }; + let honest = chunk(None); + let faulted = chunk(Some(1)); + assert_eq!(faulted.widths().len(), honest.widths().len()); + assert_ne!( + (faulted.widths(), faulted.data()), + (honest.widths(), honest.data()), + "the fault leaves the CPU table as it was" + ); + let none = chunk(Some(0)); + assert_eq!( + (none.widths(), none.data()), + (honest.widths(), honest.data()) + ); +} + +/// The card's DECODE rows find every pc's row as the host's table does, and +/// no row for a pc the program has none at. +#[test] +fn the_cards_decode_rows_find_the_hosts_rows() { + for name in ["all_instructions_64", "test_keccak_multi", "test_ecsm"] { + let (program, logs) = run(name); + let builder = + WindowedTraceBuilder::new(&program, &[], &MaxRowsConfig::small()).expect("the builder"); + let host = &builder.decode; + let card = host.card_records().expect("a program"); + let mut pcs: Vec = logs.iter().map(|log| log.current_pc).collect(); + pcs.extend(card.entries().iter().map(|e| e.pc)); + let probes: Vec = pcs + .iter() + .flat_map(|&pc| { + [ + pc, + pc.wrapping_add(2), + pc.wrapping_sub(2), + pc.wrapping_add(4), + ] + }) + .chain([0, u64::MAX]) + .collect(); + for pc in probes { + assert_eq!(card.row(pc), host.row(pc), "{name}: pc {pc:#x}"); + } + for (row, entry) in card.entries().iter().enumerate() { + let host_entry = host.entry(row as u32); + assert_eq!( + (entry.pc, entry.imm, entry.rd, entry.ecall != 0), + ( + host_entry.pc, + host_entry.imm, + host_entry.fields.rd, + host_entry.fields.ecall + ), + "{name}: row {row}" + ); + } + } +} + +/// ★ A chunk held as records parks and drops like any chunk: the regeneration +/// recorder gives it a recipe and a rank, its block index. With records kept +/// (`LAMBDA_VM_BLOCK_RECORDS_DROP=keep`) it has neither: the recorder keeps its +/// place in the hand-out, so every other chunk's rank is still its block +/// index, but names no chunk there, so no regenerator drops it or waits on its +/// fate. +#[test] +fn a_chunk_held_as_records_has_a_rank_unless_records_are_kept() { + use crate::block_whir::regen::Recorder; + let (program, logs) = run("test_keccak_multi"); + for (keep, rows) in [(false, false), (true, false), (false, true), (true, true)] { + let max_rows = if rows { + rows_max() + } else { + MaxRowsConfig::small() + }; + for (room, admitted) in [(always(), None), (for_first(3, 1000), Some(3))] { + let what = format!("keep {keep}, rows {rows}, admitted {admitted:?}"); + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows) + .and_then(WindowedTraceBuilder::drop_streamed_ops) + .and_then(|b| b.card_fill_streamed(room, true, true)) + .and_then(|b| { + if rows { + b.card_fill_streamed_rows(RowFamilies::ALL) + } else { + Ok(b) + } + }) + .expect("the builder"); + let recorder = if keep { + Recorder::new(false).keep_records() + } else { + Recorder::new(false) + }; + let mut handed: Vec<(StreamTable, usize, bool)> = Vec::new(); + { + let (mut walker, mut accumulator) = builder.split(); + let body = logs.len() - 1; + for w in logs[..body - body % 32].chunks(32) { + let walked = walker.walk_owned(w.to_vec()).expect("a window"); + let (jobs, added) = accumulator.absorb_counted(walked); + recorder.window(added, &jobs); + handed.extend(jobs.iter().map(|j| (j.table, j.index, j.is_records()))); + } + } + let held = handed.iter().filter(|(_, _, r)| *r).count(); + assert!(held > 0, "{what}: {held} of {}", handed.len()); + let row_held = handed + .iter() + .any(|&(table, _, records)| records && row_table(table)); + if !rows || admitted.is_none() { + assert_eq!(row_held, rows, "{what}"); + } + if let Some(admitted) = admitted { + assert!( + held == admitted && handed.len() > held, + "{what}: {held} of {}", + handed.len() + ); + } + for (t, &(table, index, records)) in handed.iter().enumerate() { + let want = (!(records && keep)).then_some((table, index)); + assert_eq!(recorder.chunk_at(t), want, "{what}: block index {t}"); + } + assert_eq!(recorder.chunk_at(handed.len()), None); + let (recipes, stray) = recorder.recipes(); + assert_eq!(stray, 0); + let ranked: Vec<(StreamTable, usize, usize)> = handed + .iter() + .enumerate() + .filter(|(_, (_, _, records))| !(*records && keep)) + .map(|(t, &(table, index, _))| (table, index, t)) + .collect(); + let recorded: Vec<(StreamTable, usize, usize)> = recipes + .iter() + .map(|r| (r.table, r.index, r.order)) + .collect(); + assert_eq!(recorded, ranked, "{what}: a recipe at its block index"); + } + } +} + +/// ★ A chunk dropped as its records comes back as them: walked again from the +/// same logs, with the CPU logs kept, every CPU, MEMW_R and row family's chunk +/// phase A held as records is rebuilt with records of the same digest, and the +/// packed columns made from them are the host fill of phase A's records. A +/// records slot takes those columns; with one log changed in the replay, the +/// digest moves and the slot refuses the deposit, as a row family's slot +/// refuses another chunk's columns. A chunk not dropped as records is rebuilt +/// with no digest. +#[test] +fn a_chunk_dropped_as_records_is_rebuilt_as_them() { + use std::collections::HashMap; + + use super::RegenBuilder; + use crate::tables::gpack::TraceForm; + use stark::regen::{RegenError, RegenWindow}; + + let max_rows = rows_max(); + // MEMW_A's, LOAD's, STORE's and SHIFT's chunks offered another's columns + // (no small program makes four MEMW ops). + let mut crossed = [0usize; 4]; + for name in ["all_instructions_64", "test_keccak_multi", "test_blake3"] { + let (program, logs) = run(name); + let window = 7; + let body = logs.len() - 1; + let cut = body - body % window; + // Phase A's records, by chunk. + let mut held: HashMap<(StreamTable, usize), multilinear::records::Records> = HashMap::new(); + { + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows) + .and_then(WindowedTraceBuilder::drop_streamed_ops) + .and_then(|b| b.card_fill_streamed(always(), true, true)) + .and_then(|b| b.card_fill_streamed_rows(RowFamilies::ALL)) + .expect("the builder"); + let (mut walker, mut accumulator) = builder.split(); + for w in logs[..cut].chunks(window) { + let walked = walker.walk_owned(w.to_vec()).expect("a window"); + for job in accumulator.absorb(walked) { + let (table, index) = (job.table, job.index); + if let Some(records) = job.generate().trace.take_records_main() { + held.insert((table, index), records); + } + } + } + } + assert!(held.len() > 2, "{name}: {} chunks held", held.len()); + // The replay, every window (the last one included), with one log + // changed or not. + let replay = |tamper: Option, as_records: bool| { + let mut regen = RegenBuilder::new(&program, &[], &max_rows) + .and_then(RegenBuilder::keep_cpu_logs) + .expect("the regenerator"); + let mut rebuilt = HashMap::new(); + let mut at = 0usize; + for w in logs.chunks(window) { + let mut w = w.to_vec(); + if let Some(k) = tamper.filter(|&k| (at..at + w.len()).contains(&k)) { + w[k - at].dst_val ^= 1; + } + at += w.len(); + for job in regen.push_owned(w).expect("a window") { + let (table, index) = (job.table, job.index); + let (digest, chunk) = job.regenerate(TraceForm::Narrow, as_records); + let mut trace = chunk.trace; + trace.pack_main_narrow(); + rebuilt.insert((table, index), (digest, trace.take_narrow_main())); + } + } + rebuilt + }; + let rebuilt = replay(None, true); + let (slots, _producer) = RegenWindow::new(1 << 30); + for (rank, (key, records)) in held.iter().enumerate() { + let what = format!("{name} {key:?}"); + let (digest, packed) = rebuilt.get(key).cloned().expect("rebuilt"); + let packed = packed.expect("packed"); + assert_eq!( + digest, + Some(records.digest()), + "{what}: the records' digest" + ); + let want = records + .host_fill() + .expect("the host fills phase A's records"); + assert_eq!( + (packed.widths(), packed.data()), + (want.widths(), want.data()), + "{what}: the packed columns" + ); + let slot = slots.slot_for_records(records, rank as u64); + slot.deposit_checked(packed, digest.expect("a digest")) + .unwrap_or_else(|e| panic!("{what}: {e:?}")); + } + // One log changed in the first CPU chunk held: its records' digest + // moves, and its slot refuses the deposit. + let (&(table, index), records) = held + .iter() + .filter(|((t, _), _)| *t == StreamTable::Cpu) + .min_by_key(|((_, i), _)| *i) + .expect("a CPU chunk held"); + let bent = replay(Some(index * max_rows.cpu + 1), true); + let (digest, packed) = bent.get(&(table, index)).cloned().expect("rebuilt"); + let digest = digest.expect("a digest"); + assert_ne!(digest, records.digest(), "{name}: one log changed"); + let slot = slots.slot_for_records(records, held.len() as u64); + assert_eq!( + slot.deposit_checked(packed.expect("packed"), digest), + Err(RegenError::Mismatch), + "{name}: the changed log's chunk" + ); + // Another chunk's columns, and their digest, offered to a row family's + // records slot. + for (family, crossed) in [ + StreamTable::MemwAligned, + StreamTable::Load, + StreamTable::Store, + StreamTable::Shift, + ] + .into_iter() + .zip(&mut crossed) + { + let mut keys: Vec<_> = held.keys().filter(|(t, _)| *t == family).collect(); + keys.sort_by_key(|(_, index)| *index); + let [own, other, ..] = keys[..] else { + continue; + }; + *crossed += 1; + let (digest, packed) = rebuilt.get(other).cloned().expect("rebuilt"); + let slot = slots.slot_for_records(&held[own], (held.len() + 1) as u64); + assert_eq!( + slot.deposit_checked(packed.expect("packed"), digest.expect("a digest")), + Err(RegenError::Mismatch), + "{name}: {other:?}'s columns for {own:?}" + ); + } + // Not dropped as records: the same columns, no digest. + let plain = replay(None, false); + for key in held.keys() { + let (digest, packed) = plain.get(key).cloned().expect("rebuilt"); + assert_eq!(digest, None, "{name} {key:?}"); + assert_eq!( + packed.map(|p| p.data().to_vec()), + rebuilt[key].1.as_ref().map(|p| p.data().to_vec()), + "{name} {key:?}: the columns" + ); + } + } + assert!(crossed.iter().all(|&n| n > 0), "{crossed:?}"); +} + +/// ★ A records chunk let go hands back the pages of the lists it is the last +/// holder of, and leaves a list another chunk still holds as it is (its values +/// intact); on Linux the whole pages inside each such list go back to the OS. +/// Through the shared handle: only its last holder hands anything back. +#[test] +fn a_records_chunk_hands_back_only_the_lists_it_alone_holds() { + use super::Segments; + use super::card_records::RegisterRowRecords; + use multilinear::records::{PagesReturned, Records}; + let row = crate::tables::memw_register::RegRow::new(0, 9, 0, 0, 0, 0, 8, false); + // 2^16 rows of 32 bytes: 2 MiB a list, whole pages inside it. + let n = 1usize << 16; + let shared = Arc::new(vec![row; n]); + let rows = |parts: Vec<(Arc>, std::ops::Range)>| { + let Ok(records) = RegisterRowRecords::new(Segments { parts }) else { + panic!("a whole table"); + }; + records + }; + let mut held = rows(vec![ + (Arc::new(vec![row; n]), 0..n), + (Arc::clone(&shared), 0..n), + ]); + let got = multilinear::records::TableRecords::return_pages(&mut held); + assert_eq!((got.buffers, got.shared, got.failed), (1, 1, 0), "{got:?}"); + if cfg!(target_os = "linux") { + assert!(got.bytes > 0 && got.bytes <= (n * 32) as u64, "{got:?}"); + } else { + assert_eq!(got.bytes, 0, "{got:?}"); + } + drop(held); + let want = format!("{row:?}"); + assert!( + shared.iter().all(|r| format!("{r:?}") == want), + "the shared list's values" + ); + + // Through the records' handle: a handle that is not the last returns + // nothing, the last one hands its lists back. + let records = Records(Arc::new(rows(vec![(Arc::new(vec![row; n]), 0..n)]))); + let other = records.clone(); + assert_eq!(records.return_pages(), PagesReturned::default()); + let got = other.return_pages(); + assert_eq!((got.buffers, got.shared, got.failed), (1, 0, 0), "{got:?}"); +} diff --git a/prover/src/tables/trace_builder/windowed/handout_count.rs b/prover/src/tables/trace_builder/windowed/handout_count.rs new file mode 100644 index 000000000..32838b52a --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/handout_count.rs @@ -0,0 +1,274 @@ +//! The hand-out's counting on a thread of its own +//! (`LAMBDA_VM_HANDOUT_COUNT_THREAD`, [`WindowedTraceBuilder::count_handout_on_thread`]). +//! +//! Under [`WindowedTraceBuilder::drop_streamed_ops`] each full chunk leaves the +//! run's tails with what the table phase would have read from its ops counted: +//! DECODE's lookups and the last ECALL from CPU, BITWISE's lookups from MEMW_R, +//! MEMW_A, LT and SHIFT, and the phase-3 LT ops of MEMW and MEMW_A. That +//! counting is the larger part of the accumulator's work, and the accumulator +//! absorbs the windows strictly in order, so it paces the windows. Every count +//! is either a sum (the BITWISE histogram, the DECODE rows) or a list appended +//! chunk by chunk in order (the derived LT ops, the last ECALL): a thread fed +//! the same chunks in the same order arrives at the same counts, and the +//! builder adds them to its own once the windows are walked +//! ([`HandoutCounter::finish`]). The accumulator then only takes the chunk out +//! of its tail and sends the counter the chunk's parts (shared, not copied). +//! +//! The functions below are the counts themselves, called on the accumulator's +//! thread (the knob at `0`) and on the counter's alike. +//! +//! [`WindowedTraceBuilder::drop_streamed_ops`]: super::WindowedTraceBuilder::drop_streamed_ops +//! [`WindowedTraceBuilder::count_handout_on_thread`]: super::WindowedTraceBuilder::count_handout_on_thread + +use std::sync::Arc; +use std::sync::mpsc::{Sender, channel}; +use std::time::Instant; + +use super::super::{ + CompactLt, CpuOperation, LtOperation, MemwOperation, RegRow, ShiftOperation, bitwise, + collect_bitwise_from_lt_ops, collect_bitwise_from_memw_aligned, collect_lt_from_memw, + collect_lt_from_memw_aligned, lt_row_lookups, +}; +use super::{Segments, StreamTable}; +use crate::Error; +use crate::tables::decode::DecodeTable; +use crate::tables::{memw_aligned, memw_register, shift}; + +/// `LAMBDA_VM_HANDOUT_COUNT_THREAD`: unset or `1` counts on a thread of its +/// own (with the `parallel` feature; without it, on the accumulator's), `0` +/// on the accumulator's thread (the rollback). Anything else is an error. The +/// tables are the same either way. +pub(super) fn count_thread_from_env() -> Result { + match std::env::var("LAMBDA_VM_HANDOUT_COUNT_THREAD") + .ok() + .as_deref() + .map(str::trim) + { + None | Some("") | Some("1") => Ok(cfg!(feature = "parallel")), + Some("0") => Ok(false), + Some(v) => Err(Error::Prover(format!( + "LAMBDA_VM_HANDOUT_COUNT_THREAD={v}: 1 (a thread of its own) or 0 (the accumulator's)" + ))), + } +} + +/// The (timestamp, next pc) of the last ECALL among `ops`. +pub(super) fn last_ecall(ops: &Segments, decode: &DecodeTable) -> Option<(u64, u64)> { + ops.iter() + .rev() + .find(|op| decode.entry(op.decode_row).fields.ecall) + .map(|op| (op.timestamp, op.next_pc)) +} + +/// A MEMW_R chunk's BITWISE lookups. +pub(super) fn count_memw_register( + rows: &Segments, + histogram: &mut bitwise::BitwiseHistogram, +) { + for part in rows.slices() { + memw_register::collect_bitwise_from_memw_register(part, histogram) + } +} + +/// A MEMW_A chunk's BITWISE lookups, and its phase-3 LT ops into `lt` (none +/// when the MEMW-derived LT ops are streamed). +pub(super) fn count_memw_aligned( + ops: &Segments, + histogram: &mut bitwise::BitwiseHistogram, + mut lt: Option<&mut CompactLt>, +) { + for rows in ops.slices() { + let lookups = collect_bitwise_from_memw_aligned(rows); + histogram.add_ops(&lookups); + if let Some(lt) = lt.as_deref_mut() { + let derived = collect_lt_from_memw_aligned(rows); + lt.extend(&derived); + } + } +} + +/// A MEMW chunk's phase-3 LT ops into `lt` (none when they are streamed). +pub(super) fn count_memw(ops: &Segments, lt: Option<&mut CompactLt>) { + if let Some(lt) = lt { + for part in ops.slices() { + lt.extend(&collect_lt_from_memw(part)); + } + } +} + +/// An LT chunk's BITWISE lookups: its ops' and its rows' μ bounds (the chunk +/// is one LT table, `types::MU_MAX`). +pub(super) fn count_lt(ops: &Segments, histogram: &mut bitwise::BitwiseHistogram) { + for part in ops.slices() { + histogram.add_ops(&collect_bitwise_from_lt_ops(part)) + } + histogram.add_ops(<_row_lookups( + ops.slices().into_iter().flat_map(|part| part.iter()), + )); +} + +/// A SHIFT chunk's BITWISE lookups. +pub(super) fn count_shift( + ops: &Segments, + histogram: &mut bitwise::BitwiseHistogram, +) { + for part in ops.slices() { + histogram.add_ops(&shift::collect_bitwise_from_shift(part)) + } +} + +/// A chunk handed to the counter: its ops as the parts it lies in, shared +/// with the chunk the layout generates. +pub(super) enum CountWork { + Cpu(Segments), + MemwRegister(Segments), + MemwAligned(Segments), + Memw(Segments), + Lt(Segments), + Shift(Segments), +} + +/// Where the counter's thread spent its time, seconds, and what the builder +/// waited for it. A measurement. +#[derive(Clone, Copy, Debug, Default)] +pub struct CountThreadStamps { + /// Counting, per streamed table in [`StreamTable::ALL`] order. + pub busy: [f64; 9], + /// The chunks counted, per streamed table. + pub chunks: [usize; 9], + /// Waiting for a chunk. + pub idle: f64, + /// The builder's wait, once the windows were walked, for the chunks the + /// thread had still to count. + pub drain: f64, + /// The builder's adding the thread's counts to its own. + pub merge: f64, +} + +/// What the counter counted: the builder adds it to its own +/// ([`super::WindowedTraceBuilder::join_handout_count`]). +pub(super) struct Counts { + pub(super) histogram: bitwise::BitwiseHistogram, + /// DECODE's lookups, by row. + pub(super) decode_rows: Vec, + pub(super) last_ecall: Option<(u64, u64)>, + pub(super) memw_lt: CompactLt, + pub(super) memw_aligned_lt: CompactLt, + pub(super) stamps: CountThreadStamps, +} + +impl Counts { + /// One chunk counted, as the accumulator's thread counts it; DECODE's + /// lookups into `decode_rows` instead of DECODE's trace. Its table. + fn count( + &mut self, + work: CountWork, + decode: &DecodeTable, + stream_memw_lt: bool, + ) -> StreamTable { + match work { + CountWork::Cpu(ops) => { + for op in ops.iter() { + let row = op.decode_row as usize; + if row >= self.decode_rows.len() { + self.decode_rows.resize(row + 1, 0); + } + self.decode_rows[row] += 1; + } + if let Some(ecall) = last_ecall(&ops, decode) { + self.last_ecall = Some(ecall); + } + StreamTable::Cpu + } + CountWork::MemwRegister(rows) => { + count_memw_register(&rows, &mut self.histogram); + StreamTable::MemwRegister + } + CountWork::MemwAligned(ops) => { + count_memw_aligned( + &ops, + &mut self.histogram, + (!stream_memw_lt).then_some(&mut self.memw_aligned_lt), + ); + StreamTable::MemwAligned + } + CountWork::Memw(ops) => { + count_memw(&ops, (!stream_memw_lt).then_some(&mut self.memw_lt)); + StreamTable::Memw + } + CountWork::Lt(ops) => { + count_lt(&ops, &mut self.histogram); + StreamTable::Lt + } + CountWork::Shift(ops) => { + count_shift(&ops, &mut self.histogram); + StreamTable::Shift + } + } + } +} + +/// The counter's thread and the channel that feeds it, in run order. +pub(super) struct HandoutCounter { + tx: Sender, + thread: std::thread::JoinHandle, +} + +impl HandoutCounter { + /// The thread, up and waiting for chunks. `memw_lt` and `memw_aligned_lt` + /// are the empty lists it derives LT ops into (kept as the builder keeps + /// its own). An error when the thread does not start. + pub(super) fn spawn( + decode: Arc, + stream_memw_lt: bool, + memw_lt: CompactLt, + memw_aligned_lt: CompactLt, + ) -> std::io::Result { + let (tx, rx) = channel::(); + let thread = std::thread::Builder::new() + .name("handout-count".to_string()) + .spawn(move || { + let mut counts = Counts { + histogram: bitwise::BitwiseHistogram::new(), + decode_rows: vec![0; decode.padding_row() + 1], + last_ecall: None, + memw_lt, + memw_aligned_lt, + stamps: CountThreadStamps::default(), + }; + loop { + let t = Instant::now(); + let Ok(work) = rx.recv() else { + break; + }; + counts.stamps.idle += t.elapsed().as_secs_f64(); + let t = Instant::now(); + let table = counts.count(work, &decode, stream_memw_lt); + let i = table.position(); + counts.stamps.busy[i] += t.elapsed().as_secs_f64(); + counts.stamps.chunks[i] += 1; + } + counts + })?; + Ok(Self { tx, thread }) + } + + /// Hands the thread a chunk to count. A thread that stopped (it panicked) + /// is reported by [`Self::finish`]. + pub(super) fn send(&self, work: CountWork) { + let _ = self.tx.send(work); + } + + /// No more chunks: what the thread counted, once it has counted every + /// chunk sent (`stamps.drain` the wait for it). + pub(super) fn finish(self) -> Result { + let t = Instant::now(); + let Self { tx, thread } = self; + drop(tx); + let mut counts = thread + .join() + .map_err(|_| Error::Prover("the hand-out's count thread panicked".to_string()))?; + counts.stamps.drain = t.elapsed().as_secs_f64(); + Ok(counts) + } +} diff --git a/prover/src/tables/trace_builder/windowed/head_log.rs b/prover/src/tables/trace_builder/windowed/head_log.rs new file mode 100644 index 000000000..c82cbeba6 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/head_log.rs @@ -0,0 +1,386 @@ +//! The head of a streamed phase A, stamped (`LAMBDA_VM_BLOCK_HEAD_LOG=1`): +//! for the run's first windows, when the pass took each from the executor, +//! passed it and handed it out, when a walker walked it (and how much of that +//! went on seeding its entries) and when it was absorbed; for the first streamed chunks, when each was handed out, laid +//! out and placed; for every streamed chunk, the layout's seconds by table; +//! and when the card's early setup ended (`LAMBDA_VM_BLOCK_CARD_EARLY`). +//! Seconds since the prove started, printed as phase A ends. A measurement: +//! the proof is the same, and with the knob off nothing is recorded. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; +use std::time::Instant; + +use crate::Error; + +pub(crate) const HEAD_LOG_ENV: &str = "LAMBDA_VM_BLOCK_HEAD_LOG"; + +/// The windows and chunks stamped one by one; past them a chunk counts only +/// in its table's sums. +const WINDOWS: usize = 48; +const CHUNKS: usize = 128; + +static ON: AtomicBool = AtomicBool::new(false); +static LOG: Mutex> = Mutex::new(None); + +/// [`HEAD_LOG_ENV`]: unset, empty or `0` is off, `1` on; anything else +/// refuses the prove. +pub(crate) fn knob() -> Result { + parse(std::env::var(HEAD_LOG_ENV).ok().as_deref()) +} + +fn parse(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("" | "0") => Ok(false), + Some("1") => Ok(true), + Some(v) => Err(Error::Prover(format!( + "{HEAD_LOG_ENV}={v}: 1 (phase A's head stamped) or 0" + ))), + } +} + +/// A window of the run, as the head saw it. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +struct Window { + entries: usize, + received: f64, + passed: f64, + handed: f64, + walk_from: f64, + seed: f64, + walked: f64, + absorbed: f64, +} + +/// A streamed chunk, as the head saw it. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +struct Chunk { + table: &'static str, + index: usize, + rows: usize, + handed: f64, + laid_from: f64, + laid: f64, + placed: f64, +} + +/// What a prove's head recorded. +#[derive(Debug)] +struct Log { + start: Instant, + windows: Vec, + chunks: Vec, + /// Every chunk's layout, by table: its chunks, their seconds and the + /// longest. + tables: Vec<(&'static str, usize, f64, f64)>, + card_ready: Option, +} + +impl Log { + fn new(start: Instant) -> Self { + Self { + start, + windows: Vec::new(), + chunks: Vec::new(), + tables: Vec::new(), + card_ready: None, + } + } + + fn at(&self, t: Instant) -> f64 { + t.saturating_duration_since(self.start).as_secs_f64() + } + + fn window(&mut self, k: usize) -> Option<&mut Window> { + if k >= WINDOWS { + return None; + } + if self.windows.len() <= k { + self.windows.resize(k + 1, Window::default()); + } + self.windows.get_mut(k) + } + + fn chunk(&mut self, seq: usize) -> Option<&mut Chunk> { + if seq >= CHUNKS { + return None; + } + if self.chunks.len() <= seq { + self.chunks.resize(seq + 1, Chunk::default()); + } + self.chunks.get_mut(seq) + } + + fn window_handed(&mut self, k: usize, entries: usize, received: f64, passed: f64, now: f64) { + if let Some(w) = self.window(k) { + w.entries = entries; + w.received = received; + w.passed = passed; + w.handed = now; + } + } + + fn window_walked(&mut self, k: usize, from: f64, seed: f64, now: f64) { + if let Some(w) = self.window(k) { + w.walk_from = from; + w.seed = seed; + w.walked = now; + } + } + + fn window_absorbed(&mut self, k: usize, now: f64) { + if let Some(w) = self.window(k) { + w.absorbed = now; + } + } + + fn chunk_handed( + &mut self, + seq: usize, + table: &'static str, + index: usize, + rows: usize, + now: f64, + ) { + if let Some(c) = self.chunk(seq) { + c.table = table; + c.index = index; + c.rows = rows; + c.handed = now; + } + } + + fn chunk_laid(&mut self, seq: usize, table: &'static str, from: f64, now: f64) { + let secs = (now - from).max(0.0); + match self.tables.iter_mut().find(|(name, ..)| *name == table) { + Some((_, n, sum, most)) => { + *n += 1; + *sum += secs; + *most = most.max(secs); + } + None => self.tables.push((table, 1, secs, secs)), + } + if let Some(c) = self.chunk(seq) { + c.laid_from = from; + c.laid = now; + } + } + + fn chunk_placed(&mut self, seq: usize, now: f64) { + if let Some(c) = self.chunk(seq) { + c.placed = now; + } + } + + /// The `BLOCK HEAD` lines: the windows, the chunks, the layout by table + /// and the card's setup. + fn lines(&self) -> Vec { + let windows: Vec = self + .windows + .iter() + .enumerate() + .map(|(k, w)| { + format!( + "{k} {} r{:.3} p{:.3} h{:.3} w{:.3}-{:.3} s{:.3} a{:.3}", + w.entries, + w.received, + w.passed, + w.handed, + w.walk_from, + w.walked, + w.seed, + w.absorbed + ) + }) + .collect(); + let chunks: Vec = self + .chunks + .iter() + .enumerate() + .map(|(seq, c)| { + format!( + "{seq} {}[{}] {} h{:.3} l{:.3}-{:.3} p{:.3}", + c.table, c.index, c.rows, c.handed, c.laid_from, c.laid, c.placed + ) + }) + .collect(); + let tables: Vec = self + .tables + .iter() + .map(|(name, n, sum, most)| { + format!("{name} {n} in {sum:.2} s, the longest {most:.3} s") + }) + .collect(); + vec![ + format!( + "BLOCK HEAD WINDOWS: k entries r p h w- s a (s since the prove started) · {}", + windows.join(" · ") + ), + format!( + "BLOCK HEAD CHUNKS: seq table[index] rows h l- p · {}", + chunks.join(" · ") + ), + format!( + "BLOCK HEAD TABLES: every streamed chunk's layout by table · {}", + tables.join(" · ") + ), + match self.card_ready { + Some(at) => format!("BLOCK HEAD CARD: set up at {at:.3} s on a thread of its own"), + None => "BLOCK HEAD CARD: set up at its first use".to_string(), + }, + ] + } +} + +/// Starts a prove's head: recording when `on`, nothing (and what an earlier +/// prove left cleared) when not. +pub(crate) fn begin(start: Instant, on: bool) { + if let Ok(mut log) = LOG.lock() { + *log = on.then(|| Log::new(start)); + } + ON.store(on, Relaxed); +} + +fn with(f: impl FnOnce(&mut Log, f64)) { + if !ON.load(Relaxed) { + return; + } + if let Ok(mut log) = LOG.lock() + && let Some(log) = log.as_mut() + { + let now = log.at(Instant::now()); + f(log, now); + } +} + +/// The pass took window `k` from the executor at `received`, passed it by +/// `passed` and handed it out now, with its entries. +pub(crate) fn window_handed(k: usize, entries: usize, received: Instant, passed: Instant) { + with(|log, now| { + let (received, passed) = (log.at(received), log.at(passed)); + log.window_handed(k, entries, received, passed, now) + }); +} + +/// A walker walked window `k` from `from`, `seed` seconds of it seeding. +pub(crate) fn window_walked(k: usize, from: Instant, seed: f64) { + with(|log, now| { + let from = log.at(from); + log.window_walked(k, from, seed, now) + }); +} + +pub(crate) fn window_absorbed(k: usize) { + with(|log, now| log.window_absorbed(k, now)); +} + +/// Streamed chunk `seq` (in hand-out order), chunk `index` of `table`, with +/// `rows` rows before padding, handed to the layout. +pub(crate) fn chunk_handed(seq: usize, table: &'static str, index: usize, rows: usize) { + with(|log, now| log.chunk_handed(seq, table, index, rows, now)); +} + +/// Streamed chunk `seq`, of `table`, laid out from `from` until now. +pub(crate) fn chunk_laid(seq: usize, table: &'static str, from: Instant) { + with(|log, now| { + let from = log.at(from); + log.chunk_laid(seq, table, from, now) + }); +} + +pub(crate) fn chunk_placed(seq: usize) { + with(|log, now| log.chunk_placed(seq, now)); +} + +/// The card's early setup ended. +pub(crate) fn card_ready() { + with(|log, now| log.card_ready = Some(now)); +} + +/// Stops recording and hands back the `BLOCK HEAD` lines; none when the head +/// was not recorded. +pub(crate) fn take_lines() -> Vec { + ON.store(false, Relaxed); + LOG.lock() + .ok() + .and_then(|mut log| log.take()) + .map(|log| log.lines()) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_knob_is_off_unless_its_variable_is_one() { + assert!(!parse(None).unwrap()); + assert!(!parse(Some("")).unwrap()); + assert!(!parse(Some(" 0 ")).unwrap()); + assert!(parse(Some("1")).unwrap()); + assert!(parse(Some(" 1")).unwrap()); + for v in ["2", "on", "true", "01"] { + assert!(parse(Some(v)).is_err(), "{v} must refuse the prove"); + } + } + + #[test] + fn the_head_lines_name_each_window_and_chunk_and_the_layout_by_table() { + let mut log = Log::new(Instant::now()); + log.window_handed(0, 1_234, 0.12, 0.14, 0.15); + log.window_walked(0, 0.16, 0.05, 0.40); + log.window_absorbed(0, 0.52); + log.window_handed(2, 99, 0.29, 0.30, 0.31); + log.chunk_handed(0, "CPU", 0, 2_097_152, 0.60); + log.chunk_laid(0, "CPU", 0.61, 0.62); + log.chunk_placed(0, 0.63); + log.chunk_handed(1, "MEMW", 0, 1_000, 0.70); + log.chunk_laid(1, "MEMW", 0.71, 1.21); + log.chunk_laid(1, "MEMW", 1.30, 1.50); + log.chunk_placed(1, 1.22); + log.card_ready = Some(0.33); + let lines = log.lines(); + assert_eq!(lines.len(), 4); + assert!( + lines[0].ends_with( + "· 0 1234 r0.120 p0.140 h0.150 w0.160-0.400 s0.050 a0.520 · 1 0 r0.000 p0.000 \ + h0.000 w0.000-0.000 s0.000 a0.000 · 2 99 r0.290 p0.300 h0.310 w0.000-0.000 \ + s0.000 a0.000" + ), + "{}", + lines[0] + ); + assert!( + lines[1].ends_with( + "· 0 CPU[0] 2097152 h0.600 l0.610-0.620 p0.630 · 1 MEMW[0] 1000 h0.700 \ + l1.300-1.500 p1.220" + ), + "{}", + lines[1] + ); + assert!( + lines[2].ends_with( + "· CPU 1 in 0.01 s, the longest 0.010 s · MEMW 2 in 0.70 s, the longest 0.500 s" + ), + "{}", + lines[2] + ); + assert_eq!( + lines[3], + "BLOCK HEAD CARD: set up at 0.330 s on a thread of its own" + ); + } + + #[test] + fn past_the_stamped_windows_and_chunks_only_the_tables_count() { + let mut log = Log::new(Instant::now()); + log.window_handed(WINDOWS, 5, 0.9, 0.95, 1.0); + log.window_walked(WINDOWS + 3, 1.0, 0.1, 2.0); + log.chunk_handed(CHUNKS, "LOAD", 7, 9, 1.0); + log.chunk_laid(CHUNKS + 1, "LOAD", 1.0, 1.25); + assert!(log.windows.is_empty()); + assert!(log.chunks.is_empty()); + assert_eq!(log.tables, vec![("LOAD", 1, 0.25, 0.25)]); + assert_eq!(log.lines()[3], "BLOCK HEAD CARD: set up at its first use"); + } +} diff --git a/prover/src/tables/trace_builder/windowed/head_tests.rs b/prover/src/tables/trace_builder/windowed/head_tests.rs new file mode 100644 index 000000000..4745a6a36 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/head_tests.rs @@ -0,0 +1,82 @@ +//! The builder's head: the initial image held as pages and the walk's memory +//! state seeded from them page by page are the image as a map of bytes and the +//! state seeded from it byte by byte. + +use executor::elf::Elf; + +use super::super::{MemoryState, build_initial_image, build_initial_image_paged}; +use super::WindowedTraceBuilder; +use crate::tables::MaxRowsConfig; +use crate::test_utils::asm_elf_bytes; + +const PROGRAMS: [&str; 3] = ["all_instructions_64", "test_keccak", "lw_sw_offset_odd"]; + +/// Private inputs of no bytes, a few, and enough to cross a page (the image's +/// pages and the PAGE tables' are 256 KiB). +fn inputs() -> Vec> { + let long = (0..300_000u32) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + vec![Vec::new(), vec![1, 0, 255, 7, 0], long] +} + +/// The two states' pages, cells and set cells are equal. +fn same_state(what: &str, a: &MemoryState, b: &MemoryState) { + let bases: Vec = a.cells.page_bases().collect(); + assert_eq!( + bases, + b.cells.page_bases().collect::>(), + "{what}: pages" + ); + for base in bases { + assert!( + a.cells.page_data(base) == b.cells.page_data(base), + "{what}: the cells of page {base:#x}" + ); + } + assert!(a.cells.iter().eq(b.cells.iter()), "{what}: the set cells"); +} + +/// ★ The image as pages holds the bytes the image as a map holds, and the +/// memory state seeded from its pages is the state seeded byte by byte from +/// the map: the same pages, the same cells, the same set cells. +#[test] +fn the_state_seeded_from_the_pages_is_the_state_seeded_from_the_map() { + for name in PROGRAMS { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + for input in inputs() { + let what = format!("{name}, {} input bytes", input.len()); + let map = build_initial_image(&program, &input); + let pages = build_initial_image_paged(&program, &input); + let mut bytes: Vec<(u64, u8)> = map.iter().map(|(&a, &b)| (a, b)).collect(); + bytes.sort_unstable(); + assert!( + bytes.iter().copied().eq(pages.iter()), + "{what}: the image's bytes" + ); + same_state( + &what, + &MemoryState::from_image(&map), + &MemoryState::from_pages(&pages), + ); + } + } +} + +/// The builder starts its walk from that state. +#[test] +fn the_builder_walks_from_the_state_seeded_from_the_map() { + let max_rows = MaxRowsConfig::small(); + for name in PROGRAMS { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + for input in inputs() { + let builder = + WindowedTraceBuilder::new(&program, &input, &max_rows).expect("the builder"); + same_state( + &format!("{name}, {} input bytes", input.len()), + &builder.memory_state, + &MemoryState::from_image(&build_initial_image(&program, &input)), + ); + } + } +} diff --git a/prover/src/tables/trace_builder/windowed/lean_pass.rs b/prover/src/tables/trace_builder/windowed/lean_pass.rs new file mode 100644 index 000000000..25a35e240 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/lean_pass.rs @@ -0,0 +1,738 @@ +//! The walk's timestamps alone, a window at a time: the serial stage that lets +//! a run's windows be walked on several threads. +//! +//! From one window to the next the walk carries each memory byte's value and +//! last-access timestamp and the 35 register cells. A window walked from the +//! bytes it touches as they stood before it, and the registers, is the window +//! the sequential walk walks. The values can come from the executor's memory +//! at the window's start ([`executor::vm::memory::MemorySnapshot`]); this pass +//! supplies the rest. Straight from each window's logs it keeps every byte's +//! last-access timestamp and the register cells, and records each byte's +//! address and timestamp at the byte's first access in the window (its +//! ENTRIES), whatever the access: a load's or a store's eight bytes, an +//! accelerator's state, ECSM's and HINT's inputs, a Keccak-256 call's input +//! and digest. +//! +//! It builds each cycle's [`CpuOperation`] as the walk does, so the register +//! values and addresses are the walk's, and keeps no memory value: no list is +//! built, and no accelerator is computed again. Its memory is one `u64` a byte, +//! the timestamp above the window that last touched the byte +//! ([`WINDOW_BITS`]), in 64 KiB pages found through directories as the +//! executor's are. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use executor::vm::instruction::execution::{BLAKE3_ABSORB_CV_OUT_DWORD, BLAKE3_BLOCK_BYTES}; +use executor::vm::logs::Log; +use executor::vm::memory::MemorySnapshot; +use rayon::prelude::*; + +use super::super::{MemoryState, RegisterState, clock}; +use crate::Error; +use crate::paged_mem::{PageMut, PagedMem}; +use crate::tables::cpu::{CpuOp, CpuOperation}; +use crate::tables::decode::DecodeTable; +use crate::tables::page::DEFAULT_PAGE_SIZE; + +/// The low bits of a cell: the window that last touched the byte. +const WINDOW_BITS: u32 = 24; +const WINDOW_MASK: u64 = (1 << WINDOW_BITS) - 1; +/// No window touched the byte (the most windows a pass takes is one less). +const UNTOUCHED: u64 = WINDOW_MASK; +/// A byte never touched: timestamp 0, no window. +const FILL: u64 = UNTOUCHED; +/// A timestamp must fit above the window. +const MAX_TIMESTAMP: u64 = (1 << (64 - WINDOW_BITS)) - 1; + +/// 64 KiB pages of bytes, a `u64` cell each. +const PAGE_BITS: u32 = 16; +const PAGE: usize = 1 << PAGE_BITS; +const PAGE_MASK: u64 = PAGE as u64 - 1; +/// The highest page number. +const LAST_PAGE: u64 = u64::MAX >> PAGE_BITS; +/// Pages each directory indexes: the low 8 GiB and the top 8 GiB, as the +/// executor's memory. +const DIRECTORY_PAGES: u64 = 1 << (33 - PAGE_BITS); +/// A directory slot with no page. +const NO_PAGE: u32 = u32::MAX; + +/// The most pcs a dense pc→row index spans (64 MiB of `u32`); a program +/// spread wider finds rows by [`DecodeTable::row`]. +const DENSE_ROWS: u64 = 1 << 24; + +/// The DECODE row of a pc: one indexed read over the program's pcs, or the +/// table's own search when the pcs do not fit one dense index. +enum Rows { + /// Row of `first + 4·i` at `rows[i]`, [`NO_PAGE`] where no instruction is. + Dense { + first: u64, + rows: Vec, + }, + Search, +} + +impl Rows { + fn of(decode: &DecodeTable) -> Self { + let n = decode.padding_row(); + if n == 0 { + return Self::Search; + } + let first = decode.entry(0).pc; + let last = decode.entry(n as u32 - 1).pc; + let span = (last - first) / 4 + 1; + let aligned = (0..n as u32).all(|r| (decode.entry(r).pc - first).is_multiple_of(4)); + if span > DENSE_ROWS || !aligned { + return Self::Search; + } + let mut rows = vec![NO_PAGE; span as usize]; + for r in 0..n as u32 { + rows[((decode.entry(r).pc - first) / 4) as usize] = r; + } + Self::Dense { first, rows } + } + + /// [`DecodeTable::row`]'s answer for `pc`. + #[inline] + fn row(&self, pc: u64, decode: &DecodeTable) -> Option { + match self { + Self::Dense { first, rows } => { + let offset = pc.wrapping_sub(*first); + if !offset.is_multiple_of(4) { + return None; + } + let row = *rows.get(usize::try_from(offset / 4).ok()?)?; + (row != NO_PAGE).then_some(row) + } + Self::Search => decode.row(pc), + } + } +} + +/// Every byte's cell: [`FILL`] until a window touches it. +struct Cells { + pages: Vec>, + low: Vec, + high: Vec, + other: BTreeMap, + /// `(page number, index in pages)` of the last page found; page numbers + /// never reach `u64::MAX`. + hot: (u64, u32), +} + +impl Cells { + fn new() -> Self { + Self { + pages: Vec::new(), + low: Vec::new(), + high: Vec::new(), + other: BTreeMap::new(), + hot: (u64::MAX, NO_PAGE), + } + } + + /// The index of page `number`, allocated on first touch. + #[inline] + fn page(&mut self, number: u64) -> usize { + if self.hot.0 == number { + return self.hot.1 as usize; + } + fn slot(directory: &mut Vec, index: usize) -> &mut u32 { + if directory.len() <= index { + directory.resize(index + 1, NO_PAGE); + } + &mut directory[index] + } + let slot = if number < DIRECTORY_PAGES { + slot(&mut self.low, number as usize) + } else if LAST_PAGE - number < DIRECTORY_PAGES { + slot(&mut self.high, (LAST_PAGE - number) as usize) + } else { + self.other.entry(number).or_insert(NO_PAGE) + }; + if *slot == NO_PAGE { + *slot = u32::try_from(self.pages.len()).expect("fewer than 2^32 pages"); + let page: Box<[u64]> = vec![FILL; PAGE].into_boxed_slice(); + self.pages + .push(page.try_into().expect("a page holds PAGE cells")); + } + self.hot = (number, *slot); + *slot as usize + } + + /// Every page touched, with its page number. + fn iter_pages(&self) -> impl Iterator + '_ { + let low = self.low.iter().enumerate().map(|(n, &i)| (n as u64, i)); + let high = self + .high + .iter() + .enumerate() + .map(|(n, &i)| (LAST_PAGE - n as u64, i)); + let other = self.other.iter().map(|(&n, &i)| (n, i)); + low.chain(high) + .chain(other) + .filter(|&(_, i)| i != NO_PAGE) + .map(|(n, i)| (n, &*self.pages[i as usize])) + } + + /// The cell of `address`, [`FILL`] for a page never touched. + #[cfg(test)] + fn get(&self, address: u64) -> u64 { + let number = address >> PAGE_BITS; + let slot = if number < DIRECTORY_PAGES { + self.low.get(number as usize).copied() + } else if LAST_PAGE - number < DIRECTORY_PAGES { + self.high.get((LAST_PAGE - number) as usize).copied() + } else { + self.other.get(&number).copied() + }; + match slot { + Some(index) if index != NO_PAGE => { + self.pages[index as usize][(address & PAGE_MASK) as usize] + } + _ => FILL, + } + } +} + +/// One byte accessed in window `window`: recorded at its first access in the +/// window, then, with `write`, stamped. +#[inline(always)] +fn access_cell( + cell: &mut u64, + address: u64, + window: u64, + write: Option, + entries: &mut Vec<(u64, u64)>, +) { + let mut c = *cell; + if c & WINDOW_MASK != window { + entries.push((address, c >> WINDOW_BITS)); + c = (c & !WINDOW_MASK) | window; + } + if let Some(ts) = write { + c = (ts << WINDOW_BITS) | window; + } + *cell = c; +} + +/// The 35 register cells, as the walk's [`RegisterState`]. +#[derive(Clone)] +struct Registers { + regs: [(u64, u64); 32], + index: (u32, u64), + pc: (u64, u64), +} + +impl Registers { + fn of(state: &RegisterState) -> Self { + Self { + regs: state.regs, + index: state.index_register, + pc: state.pc_register, + } + } + + fn state(&self) -> RegisterState { + RegisterState { + regs: self.regs, + index_register: self.index, + pc_register: self.pc, + } + } + + /// A register write as [`RegisterState::write`]: x0 is never written. The + /// decode rows are checked to name registers below 32 ([`LeanPass::new`]). + #[inline(always)] + fn write(&mut self, reg: u8, value: u64, timestamp: u64) { + if reg != 0 { + self.regs[usize::from(reg & 31)] = (value, timestamp); + } + } + + #[inline(always)] + fn value(&self, reg: u8) -> u64 { + self.regs[usize::from(reg & 31)].0 + } +} + +/// Where a pass's memory accesses go: the cells ([`Bytes`]), or a list +/// another thread applies to them (the split pass, a measurement's arm). +trait Accesses { + /// `count` bytes from `base` accessed; with `write`, each then stamped. + fn access(&mut self, base: u64, count: usize, write: Option); + + /// A load or a store: the eight bytes at `base` read, the first `count` + /// stamped `ts`, as the walk's load and store collectors access them. + fn load_or_store(&mut self, base: u64, count: usize, ts: u64); +} + +/// Every byte's cell, the window being passed and its entries. +struct Bytes { + cells: Cells, + /// The window being passed, or [`UNTOUCHED`] before the first. + window: u64, + entries: Vec<(u64, u64)>, +} + +impl Bytes { + fn new() -> Self { + Self { + cells: Cells::new(), + window: UNTOUCHED, + entries: Vec::new(), + } + } + + /// Starts window `window`, its entries cleared. Windows are numbered from + /// 0, below 2^24 − 1. + fn begin(&mut self, window: u32) -> Result<(), Error> { + if u64::from(window) >= UNTOUCHED { + return Err(Error::Prover(format!( + "the lean pass: window {window} is past the {UNTOUCHED} it can mark" + ))); + } + self.window = u64::from(window); + self.entries.clear(); + Ok(()) + } +} + +impl Accesses for Bytes { + #[inline] + fn access(&mut self, base: u64, count: usize, write: Option) { + let off = (base & PAGE_MASK) as usize; + if off + count <= PAGE { + let index = self.cells.page(base >> PAGE_BITS); + let page = &mut self.cells.pages[index]; + for (i, cell) in page[off..off + count].iter_mut().enumerate() { + access_cell(cell, base + i as u64, self.window, write, &mut self.entries); + } + } else { + for i in 0..count { + let address = base.wrapping_add(i as u64); + let index = self.cells.page(address >> PAGE_BITS); + let cell = &mut self.cells.pages[index][(address & PAGE_MASK) as usize]; + access_cell(cell, address, self.window, write, &mut self.entries); + } + } + } + + #[inline] + fn load_or_store(&mut self, base: u64, count: usize, ts: u64) { + let off = (base & PAGE_MASK) as usize; + if off + 8 > PAGE { + self.access(base, 8, None); + self.access(base, count, Some(ts)); + return; + } + let window = self.window; + let index = self.cells.page(base >> PAGE_BITS); + let cells: &mut [u64; 8] = (&mut self.cells.pages[index][off..off + 8]) + .try_into() + .expect("eight cells"); + if cells.iter().any(|&c| c & WINDOW_MASK != window) { + for (i, cell) in cells.iter_mut().enumerate() { + if *cell & WINDOW_MASK != window { + self.entries.push((base + i as u64, *cell >> WINDOW_BITS)); + *cell = (*cell & !WINDOW_MASK) | window; + } + } + } + let stamped = (ts << WINDOW_BITS) | window; + for cell in &mut cells[..count] { + *cell = stamped; + } + } +} + +/// An ECALL's register writes and accesses, as the walk makes them. +fn ecall(memory: &mut impl Accesses, regs: &mut Registers, op: CpuOp<'_>, ts: u64) { + if op.ecall_commit() { + let (buf, count) = (op.commit_buf_addr(), op.commit_count()); + regs.write(10, count, ts); + regs.write(11, buf, ts); + regs.write(12, count, ts); + regs.index = (regs.index.0.wrapping_add(count as u32), ts); + for k in 0..count { + memory.access(buf.wrapping_add(k), 1, Some(ts)); + } + } + // memcpy / memmove and memset through COPY (`collect_copy_register_ops`, + // `collect_copy_ops`): x10, x11 and x12 read back at T; a copy reads every + // source byte at T before it writes the destination at T + 1, a memset writes + // each row's destination at T and reads its source at T + 1, row by row. + if op.ecall_dma_memcpy() || op.ecall_dma_memset() { + let (dst, src, count) = (regs.value(10), regs.value(11), regs.value(12)); + for (reg, value) in [(10u8, dst), (11, src), (12, count)] { + regs.write(reg, value, ts); + } + if op.ecall_dma_memcpy() { + memory.access(src, count as usize, Some(ts)); + memory.access(dst, count as usize, Some(ts + 1)); + } else { + // The executor's own row schedule: which bytes a row holds decides + // which end at T and which at T + 1. + let mut offset = 0u64; + while offset < count { + let width = usize::from(executor::vm::instruction::execution::copy_row_width( + src, + dst, + offset, + count - offset, + false, + )); + memory.access(dst.wrapping_add(offset), width, Some(ts)); + memory.access(src.wrapping_add(offset), width, Some(ts + 1)); + offset += width as u64; + } + } + } + if op.ecall_keccak() { + let state = op.keccak_state_addr(); + regs.write(10, state, ts); + memory.access(state, 200, Some(ts)); + } + // Keccak-256 of a buffer (`collect_keccak256_ops`): x10, x11 and x12 read back + // at T, every input byte read at T, the 32-byte digest written at T + 1. + if op.ecall_keccak256() { + let (ptr, len, out) = (regs.value(10), regs.value(11), regs.value(12)); + for (reg, value) in [(10u8, ptr), (11, len), (12, out)] { + regs.write(reg, value, ts); + } + memory.access(ptr, len as usize, Some(ts)); + for i in 0..4u64 { + memory.access(out.wrapping_add(8 * i), 8, Some(ts + 1)); + } + } + if op.ecall_blake3() { + let state = op.blake3_state_addr(); + regs.write(10, state, ts); + memory.access(state, 176, Some(ts)); + } + if op.ecall_blake3_absorb() { + let (ctrl, msg) = (regs.value(10), regs.value(11)); + let (blocks, first_flags) = (regs.value(12), regs.value(13)); + for (reg, value) in [(10u8, ctrl), (11, msg), (12, blocks), (13, first_flags)] { + regs.write(reg, value, ts); + } + memory.access(ctrl, 32, Some(ts)); + for b in 0..blocks { + memory.access( + msg.wrapping_add(b.wrapping_mul(BLAKE3_BLOCK_BYTES)), + 64, + Some(ts), + ); + } + memory.access( + ctrl.wrapping_add(BLAKE3_ABSORB_CV_OUT_DWORD * 8), + 32, + Some(ts), + ); + } + if op.ecall_ecsm() { + let (xr, xg, k) = (regs.value(10), regs.value(11), regs.value(12)); + for i in 0..32u64 { + memory.access(xg.wrapping_add(i), 1, None); + memory.access(k.wrapping_add(i), 1, None); + } + for (reg, address, at) in [(11u8, xg, ts), (12, k, ts + 1), (10, xr, ts + 2)] { + let value = regs.value(reg); + regs.write(reg, value, at); + for i in 0..4u64 { + memory.access(address.wrapping_add(8 * i), 8, Some(at)); + } + } + } + if op.ecall_ecrec() { + // The signature (64 bytes) and the hash (32) are read, the key and the status + // (72 bytes) written. + let (out, sig, msg) = (regs.value(10), regs.value(11), regs.value(12)); + for i in 0..64u64 { + memory.access(sig.wrapping_add(i), 1, None); + } + for i in 0..32u64 { + memory.access(msg.wrapping_add(i), 1, None); + } + let parity = regs.value(13); + for (reg, address, at, dwords) in [ + (11u8, sig, ts, 8u64), + (12, msg, ts + 1, 4), + (10, out, ts + 2, 9), + ] { + let value = regs.value(reg); + regs.write(reg, value, at); + for i in 0..dwords { + memory.access(address.wrapping_add(8 * i), 8, Some(at)); + } + } + regs.write(13, parity, ts + 1); + } + if op.ecall_hint() { + let (id, input, out) = (regs.value(10), regs.value(11), regs.value(12)); + for (reg, value) in [(10u8, id), (11, input), (12, out)] { + regs.write(reg, value, ts); + } + memory.access(input, 32, None); + for i in 0..4u64 { + memory.access(out.wrapping_add(8 * i), 8, Some(ts)); + } + } +} + +/// What the pass reads each cycle with: the decode table and its pc index. +struct Decode { + table: Arc, + rows: Rows, +} + +impl Decode { + /// Refuses a program whose decode names a register the walk could not + /// hold (32 and up, but the pc's 255 read through rs1). + fn new(table: Arc) -> Result { + for row in 0..table.padding_row() as u32 { + let f = &table.entry(row).fields; + if (f.read_register1 && f.rs1 >= 32 && f.rs1 != 255) + || (f.read_register2 && f.rs2 >= 32) + || (f.write_register && f.rd >= 32) + { + return Err(Error::Prover(format!( + "the lean pass: the instruction at {:#x} names a register past x31", + table.entry(row).pc + ))); + } + } + Ok(Self { + rows: Rows::of(&table), + table, + }) + } +} + +/// One window's logs from cycle `first`: the register cells kept in `regs`, +/// and (with `MEMORY`) the memory's accesses sent to `memory`. +fn pass_window( + decode: &Decode, + memory: &mut M, + regs: &mut Registers, + logs: &[Log], + first: usize, +) -> Result<(), Error> { + let most = clock::window_max_timestamp(first, logs.len()); + clock::check_clock(most, "the lean pass")?; + if most > MAX_TIMESTAMP { + return Err(Error::Prover(format!( + "the lean pass: timestamp {most} does not fit its cells" + ))); + } + let table = &*decode.table; + for (i, log) in logs.iter().enumerate() { + let ts = ((first + i) as u64) * 4 + 4; + let row = decode + .rows + .row(log.current_pc, table) + .ok_or(Error::MissingInstruction(log.current_pc))?; + let entry = table.entry(row); + let op = CpuOperation::from_log(log, ts, entry, row); + let f = &entry.fields; + if MEMORY && f.memory { + memory.load_or_store(op.res, f.mem_bytes(), ts); + } + if f.read_register1 && f.rs1 != 0 { + if f.rs1 == 255 { + regs.pc = (op.rv1, ts); + } else { + regs.write(f.rs1, op.rv1, ts); + } + } + if f.read_register2 && f.rs2 != 0 { + regs.write(f.rs2, op.rv2, ts + 1); + } + if f.write_register && f.rd != 0 { + regs.write(f.rd, op.rvd, ts + 2); + } + regs.pc = (op.next_pc, ts + 1); + if MEMORY && f.ecall { + ecall(memory, regs, CpuOp::new(entry, &op), ts); + } + } + Ok(()) +} + +/// The walk's timestamps and registers, a window at a time, and each window's +/// entries (see the module's documentation). +pub(crate) struct LeanPass { + decode: Decode, + bytes: Bytes, + registers: Registers, +} + +impl LeanPass { + /// A pass from a run's start: no byte touched, and `registers` (the walk's + /// starting register cells). Refuses a program whose decode names a + /// register the walk could not hold. + pub(crate) fn new(decode: Arc, registers: &RegisterState) -> Result { + Ok(Self { + decode: Decode::new(decode)?, + bytes: Bytes::new(), + registers: Registers::of(registers), + }) + } + + /// Starts window `window` (its entries cleared). Windows are numbered + /// from 0, below 2^24 − 1. + pub(crate) fn begin(&mut self, window: u32) -> Result<(), Error> { + self.bytes.begin(window) + } + + /// One window's logs from cycle `first`, in the walk's order. + pub(crate) fn pass(&mut self, logs: &[Log], first: usize) -> Result<(), Error> { + pass_window::<_, true>( + &self.decode, + &mut self.bytes, + &mut self.registers, + logs, + first, + ) + } + + /// [`Self::pass`] with the memory's accesses left out (decode and registers + /// only): a measurement of where the pass's time goes. + #[cfg(test)] + pub(crate) fn pass_registers_only(&mut self, logs: &[Log], first: usize) -> Result<(), Error> { + pass_window::<_, false>( + &self.decode, + &mut self.bytes, + &mut self.registers, + logs, + first, + ) + } + + /// The window's entries: each byte's address and its timestamp before the + /// window, in the order the window first accessed them. + #[cfg(test)] + pub(crate) fn entries(&self) -> &[(u64, u64)] { + &self.bytes.entries + } + + /// The window's entries, moved out (the pass's list starts empty). + pub(crate) fn take_entries(&mut self) -> Vec<(u64, u64)> { + std::mem::take(&mut self.bytes.entries) + } + + /// The memory state the walk holds now, built from the pass's timestamps, + /// the executor's memory now (`now`) and the run's initial image: the + /// image's pages and cells (each byte's value, timestamp 0), then every + /// byte the pass stamped with its value now and its timestamp. The walk + /// sets a cell only by writing it (a load writes the bytes it read), so + /// these are its pages, its set cells and its values; a cell it never set + /// holds `(0, 0)` as the walk's do. The state's pages are filled at once. + pub(crate) fn walked_state( + &self, + now: &MemorySnapshot, + image: &PagedMem, + lean: bool, + ) -> MemoryState { + let mut memory = MemoryState::from_pages(image); + memory.lean = lean; + // The pass's pages holding a stamped byte, by address. A 64 KiB page + // of the pass sits in one page of the state. + let mut stamped: Vec<(u64, &[u64; PAGE])> = self + .bytes + .cells + .iter_pages() + .filter(|(_, page)| page.iter().any(|&c| c >> WINDOW_BITS > 0)) + .map(|(number, page)| (number << PAGE_BITS, page)) + .collect(); + stamped.sort_unstable_by_key(|&(base, _)| base); + for &(base, _) in &stamped { + memory + .cells + .page_index_or_insert(PagedMem::<(u8, u64)>::page_of(base).0); + } + let fill = |mut state: PageMut<'_, (u8, u64)>| { + let from = stamped.partition_point(|&(base, _)| base < state.base()); + let to = from + + stamped[from..] + .partition_point(|&(base, _)| base - state.base() < DEFAULT_PAGE_SIZE as u64); + if from == to { + return; + } + let mut values = vec![0u8; PAGE]; + for &(base, page) in &stamped[from..to] { + now.read(base, &mut values); + let at = (base - state.base()) as usize; + for (off, &c) in page.iter().enumerate() { + let ts = c >> WINDOW_BITS; + if ts > 0 { + state.set(at + off, (values[off], ts)); + } + } + } + }; + let pages: Vec<_> = memory.cells.pages_mut().collect(); + pages.into_par_iter().for_each(fill); + memory + } + + /// The run's side of the windows' boundary sums + /// ([`super::parallel::cell_hash`]): over every byte a window touched + /// (the pass's cells set by a window), its cell in `state` (the state the + /// windows hand to the finish) less its cell at the run's start (`image`'s + /// value, timestamp 0), hashed under `key` and summed. + pub(crate) fn boundaries(&self, state: &MemoryState, image: &PagedMem, key: u64) -> u64 { + let pages: Vec<(u64, &[u64; PAGE])> = self + .bytes + .cells + .iter_pages() + .map(|(number, page)| (number << PAGE_BITS, page)) + .collect(); + pages + .into_par_iter() + .map(|(base, page)| { + let mut sum = 0u64; + for (offset, &cell) in page.iter().enumerate() { + if cell == FILL { + continue; + } + let address = base + offset as u64; + let (value, ts) = state.cells.get(address); + sum = sum + .wrapping_add(super::parallel::cell_hash(key, address, ts, value)) + .wrapping_sub(super::parallel::cell_hash( + key, + address, + 0, + image.get(address), + )); + } + sum + }) + .reduce(|| 0, u64::wrapping_add) + } + + /// The register cells now, as the walk's. + pub(crate) fn registers(&self) -> RegisterState { + self.registers.state() + } + + /// The last-access timestamp of the byte at `address` (0 if never + /// stamped). + #[cfg(test)] + pub(crate) fn timestamp(&self, address: u64) -> u64 { + self.bytes.cells.get(address) >> WINDOW_BITS + } + + /// Whether rows are found through a dense pc index (else by the decode + /// table's search). + #[cfg(test)] + pub(crate) fn dense_rows(&self) -> bool { + matches!(self.decode.rows, Rows::Dense { .. }) + } +} + +#[cfg(test)] +pub(crate) mod split; +#[cfg(test)] +mod tests; diff --git a/prover/src/tables/trace_builder/windowed/lean_pass/split.rs b/prover/src/tables/trace_builder/windowed/lean_pass/split.rs new file mode 100644 index 000000000..49110a70a --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/lean_pass/split.rs @@ -0,0 +1,122 @@ +//! The lean pass split over two threads, a measurement's arm (test code): the +//! FRONT decodes each cycle, keeps the register cells and records the memory's +//! accesses in order; the BACK applies them to the cells and records each +//! window's entries. Together they are the pass, access for access; each +//! alone does part of its work, so a run whose pass binds can read how far a +//! split moves the floor. + +use std::sync::Arc; + +use executor::vm::logs::Log; + +use super::super::super::RegisterState; +use super::{Accesses, Bytes, Decode, Registers, pass_window}; +use crate::Error; +use crate::tables::decode::DecodeTable; + +/// One memory access the front recorded. +#[derive(Clone, Copy, Debug)] +pub(crate) enum Access { + /// [`Accesses::load_or_store`]. + LoadOrStore { base: u64, count: u8, ts: u64 }, + /// [`Accesses::access`]. + Bytes { + base: u64, + count: u32, + write: Option, + }, +} + +/// The front's sink: the accesses, in order. +struct Recorder<'a>(&'a mut Vec); + +impl Accesses for Recorder<'_> { + fn access(&mut self, base: u64, count: usize, write: Option) { + self.0.push(Access::Bytes { + base, + count: count as u32, + write, + }); + } + + fn load_or_store(&mut self, base: u64, count: usize, ts: u64) { + self.0.push(Access::LoadOrStore { + base, + count: count as u8, + ts, + }); + } +} + +/// The decode and the register cells, the accesses recorded. +pub(crate) struct Front { + decode: Decode, + registers: Registers, +} + +impl Front { + pub(crate) fn new(decode: Arc, registers: &RegisterState) -> Result { + Ok(Self { + decode: Decode::new(decode)?, + registers: Registers::of(registers), + }) + } + + /// One window's logs from cycle `first`, its accesses appended to `out`. + pub(crate) fn pass( + &mut self, + logs: &[Log], + first: usize, + out: &mut Vec, + ) -> Result<(), Error> { + pass_window::<_, true>( + &self.decode, + &mut Recorder(out), + &mut self.registers, + logs, + first, + ) + } + + /// The register cells now, as the walk's. + pub(crate) fn registers(&self) -> RegisterState { + self.registers.state() + } +} + +/// The cells, the front's accesses applied. +pub(crate) struct Back { + bytes: Bytes, +} + +impl Back { + pub(crate) fn new() -> Self { + Self { + bytes: Bytes::new(), + } + } + + /// Starts window `window`, its entries cleared. + pub(crate) fn begin(&mut self, window: u32) -> Result<(), Error> { + self.bytes.begin(window) + } + + /// The front's accesses, in order. + pub(crate) fn apply(&mut self, accesses: &[Access]) { + for &access in accesses { + match access { + Access::LoadOrStore { base, count, ts } => { + self.bytes.load_or_store(base, usize::from(count), ts) + } + Access::Bytes { base, count, write } => { + self.bytes.access(base, count as usize, write) + } + } + } + } + + /// The window's entries, as [`super::LeanPass::entries`]. + pub(crate) fn entries(&self) -> &[(u64, u64)] { + &self.bytes.entries + } +} diff --git a/prover/src/tables/trace_builder/windowed/lean_pass/tests.rs b/prover/src/tables/trace_builder/windowed/lean_pass/tests.rs new file mode 100644 index 000000000..dcff0158a --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/lean_pass/tests.rs @@ -0,0 +1,486 @@ +//! The lean pass against the walk: at every window's start its timestamps and +//! registers are the walk's, and each window walked from its entries (values +//! from the executor's snapshot at the window's start) is the window the +//! sequential walk walks. + +use std::sync::Arc; + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; +use executor::vm::memory::MemorySnapshot; + +use super::super::super::{MemoryState, RegisterState, WalkSizes}; +use super::super::{WalkedWindow, Walker, WindowedTraceBuilder}; +use super::split::{Back, Front}; +use super::{LeanPass, Rows, UNTOUCHED}; +use crate::tables::MaxRowsConfig; +use crate::tables::decode::DecodeTable; +use crate::test_utils::asm_elf_bytes; + +const PROGRAMS: [&str; 10] = [ + "all_instructions_64", + "lw_sw_offset_odd", + "test_memw_split_ts", + "test_keccak_multi", + "test_keccak256", + "test_blake3", + "test_blake3_absorb", + "test_commit_split", + "test_ecsm", + "test_ecrec", +]; + +/// Rust guests, built by `make compile-programs-rust`: longer runs over a heap +/// and a stack, KECCAK and ECSM from Rust, and HINT. +const RUST_PROGRAMS: [&str; 4] = ["hashmap", "keccak_precompile", "ecsm", "hint_min"]; + +fn programs() -> Vec<(String, Elf)> { + let rust = |name: &str| { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("the workspace root") + .join(format!("executor/program_artifacts/rust/{name}.elf")); + std::fs::read(&path).unwrap_or_else(|e| { + panic!("{} — run `make compile-programs-rust`: {e}", path.display()) + }) + }; + PROGRAMS + .iter() + .map(|name| (name.to_string(), asm_elf_bytes(name))) + .chain( + RUST_PROGRAMS + .iter() + .map(|name| (name.to_string(), rust(name))), + ) + .map(|(name, bytes)| (name, Elf::load(&bytes).expect("the ELF loads"))) + .collect() +} + +/// The run's windows of `window` cycles, each with the executor's memory at +/// its start. +fn windows(elf: &Elf, window: usize) -> Vec<(Vec, MemorySnapshot)> { + let mut executor = Executor::new(elf, Vec::new()).expect("the executor starts"); + let mut at_start = executor.memory().snapshot(); + let mut out = Vec::new(); + while let Some(logs) = executor + .resume_with_limit(window) + .expect("the run executes") + .map(<[Log]>::to_vec) + { + let next = executor.memory().snapshot(); + out.push((logs, std::mem::replace(&mut at_start, next))); + } + out +} + +/// 7 and 33 cycles for a short run, about a 40th and a 7th of a long one. +fn window_lengths(cycles: usize) -> [usize; 2] { + if cycles < 5_000 { + [7, 33] + } else { + [(cycles / 40) | 1, (cycles / 7) | 1] + } +} + +/// The lists two walks of one window differ in, by name. +fn differences(a: &WalkedWindow, b: &WalkedWindow) -> Vec<&'static str> { + fn same(x: &[T], y: &[T]) -> bool { + x.len() == y.len() && format!("{x:?}") == format!("{y:?}") + } + let (x, y) = (&a.walk, &b.walk); + [ + ("cpu", same(&a.cpu_ops, &b.cpu_ops)), + ("memw_r", same(&x.memw.register_rows, &y.memw.register_rows)), + ("memw_a", same(&x.memw.aligned, &y.memw.aligned)), + ("memw", same(&x.memw.general, &y.memw.general)), + ("load", same(&x.load_ops, &y.load_ops)), + ("lt", same(&x.lt_ops, &y.lt_ops)), + ("shift", same(&x.shift_ops, &y.shift_ops)), + ("bitwise", same(&x.bitwise_ops, &y.bitwise_ops)), + ("cpu32", same(&x.cpu32_ops, &y.cpu32_ops)), + ("copy", same(&x.copy_ops, &y.copy_ops)), + ("keccak", same(&x.keccak_ops, &y.keccak_ops)), + ("blake3", same(&x.blake3_ops, &y.blake3_ops)), + ( + "blake3_absorb", + same(&x.blake3_absorb_ops, &y.blake3_absorb_ops), + ), + ("ecsm", same(&x.ecsm_ops, &y.ecsm_ops)), + ("ecdas", same(&x.ecdas_ops, &y.ecdas_ops)), + ("hint", same(&x.hint_ops, &y.hint_ops)), + ("ecrec", same(&x.ecrec_ops, &y.ecrec_ops)), + ] + .into_iter() + .filter(|&(_, same)| !same) + .map(|(name, _)| name) + .collect() +} + +/// The register cells that differ. +fn register_differences(a: &RegisterState, b: &RegisterState) -> usize { + (0..32).filter(|&r| a.regs[r] != b.regs[r]).count() + + usize::from(a.index_register != b.index_register) + + usize::from(a.pc_register != b.pc_register) +} + +/// A copy of `r` (the walk's register state is not `Clone`). +fn copy(r: &RegisterState) -> RegisterState { + RegisterState { + regs: r.regs, + index_register: r.index_register, + pc_register: r.pc_register, + } +} + +/// A memory state holding only `entries`, each with its value in `snapshot`. +fn state_from_entries(entries: &[(u64, u64)], snapshot: &MemorySnapshot) -> MemoryState { + let mut state = MemoryState::new(); + state.lean = true; + for &(address, ts) in entries { + state.cells.set(address, (snapshot.load_byte(address), ts)); + } + state +} + +/// What a window is walked with. +struct Setup { + decode: Arc, + lookups: bool, + batch: usize, +} + +impl Setup { + /// One window walked by the production walker from `memory` and + /// `registers`, from cycle `first`. + fn walk( + &self, + memory: &mut MemoryState, + registers: &mut RegisterState, + first: usize, + logs: &[Log], + ) -> WalkedWindow { + let (mut sizes, mut cycles, mut windows, mut secs) = (WalkSizes::default(), first, 0, 0.0); + Walker { + decode: &self.decode, + lookups: self.lookups, + keep_logs: false, + batch: self.batch, + sizes: &mut sizes, + memory_state: memory, + register_state: registers, + cycles: &mut cycles, + windows: &mut windows, + walk_secs: &mut secs, + } + .walk(logs) + .expect("a window walks") + } +} + +/// The memory bytes whose old timestamp the window's MEMW and MEMW_A rows keep +/// (within each access's width), last accessed before the window (timestamps +/// below `first_ts`), sorted. +fn first_accessed(window: &WalkedWindow, first_ts: u64) -> Vec { + let mut out = Vec::new(); + for op in window.walk.memw.general.iter().filter(|op| !op.is_register) { + for i in 0..op.width as usize { + if op.old_timestamp[i] < first_ts { + out.push(op.base_address.wrapping_add(i as u64)); + } + } + } + for row in window + .walk + .memw + .aligned + .iter() + .filter(|row| !row.is_register()) + { + if row.old_timestamp() < first_ts { + out.extend((0..row.width() as u64).map(|i| row.base_address().wrapping_add(i))); + } + } + out.sort_unstable(); + out.dedup(); + out +} + +/// What one program run at one window length showed. +#[derive(Default)] +struct Seen { + windows: usize, + entries: usize, + /// KECCAK + BLAKE3, ECSM + absorb, HINT ops walked. + ecalls: [usize; 3], + /// Entries a memory row keeps the old timestamp of, stamped before the + /// window, each dropped once and its window walked again; and of them, the + /// walks that differed. + dropped: usize, + caught: usize, +} + +/// The pass and the walk side by side over `elf`'s run at `window` cycles a +/// window. At each window's start the pass's timestamps (of every byte the +/// walk holds and every entry) and registers must be the walk's, the entries +/// distinct, each entry's snapshot value the walk's; the window walked from +/// its entries must be the sequential walk's. With `drop`, up to three entries +/// a window that a memory row needs (see [`first_accessed`]) are each left out +/// once and the window walked again. +fn side_by_side(name: &str, elf: &Elf, window: usize, drop: bool) -> Seen { + let mut builder = WindowedTraceBuilder::new(elf, &[], &MaxRowsConfig::small()) + .expect("the builder") + .drop_streamed_ops() + .expect("before any window"); + let setup = Setup { + decode: Arc::clone(&builder.decode), + lookups: !builder.lean.lookups, + batch: builder.walk_batch, + }; + let mut pass = LeanPass::new(Arc::clone(&builder.decode), &builder.register_state) + .expect("the pass starts"); + let mut runs = windows(elf, window); + // The run's last window is the finish's. + runs.pop(); + let (mut walker, _accumulator) = builder.split(); + let mut seen = Seen::default(); + for (k, (logs, snapshot)) in runs.iter().enumerate() { + let first = *walker.cycles; + let regs = pass.registers(); + assert_eq!( + register_differences(®s, walker.register_state), + 0, + "{name}/{window}, window {k}: registers at its start" + ); + for (address, (_, ts)) in walker.memory_state.cells.iter() { + assert_eq!( + pass.timestamp(address), + ts, + "{name}/{window}, window {k}: the timestamp at {address:#x}" + ); + } + pass.begin(u32::try_from(k).expect("few windows")) + .expect("a window to mark"); + pass.pass(logs, first).expect("the pass"); + let entries = pass.entries().to_vec(); + let mut distinct: Vec = entries.iter().map(|&(a, _)| a).collect(); + distinct.sort_unstable(); + distinct.dedup(); + assert_eq!( + distinct.len(), + entries.len(), + "{name}/{window}, window {k}: entries repeat" + ); + for &(address, ts) in &entries { + assert_eq!( + walker.memory_state.read_byte(address), + (snapshot.load_byte(address), ts), + "{name}/{window}, window {k}: the entry at {address:#x}" + ); + } + let mut memory = state_from_entries(&entries, snapshot); + let mut registers = copy(®s); + let again = setup.walk(&mut memory, &mut registers, first, logs); + let walked = walker.walk(logs).expect("the sequential walk"); + assert_eq!( + differences(&again, &walked), + Vec::<&str>::new(), + "{name}/{window}, window {k}: walked from its entries" + ); + if drop { + // Bytes whose old timestamp a memory row of the window keeps: an + // entry for one stamped before (timestamp > 0) must be needed. + let kept = first_accessed(&walked, first as u64 * 4 + 4); + let full: Vec = (0..entries.len()) + .filter(|&i| entries[i].1 > 0 && kept.binary_search(&entries[i].0).is_ok()) + .collect(); + let picks = [full.first(), full.get(full.len() / 2), full.last()]; + let mut picked: Vec = picks.into_iter().flatten().copied().collect(); + picked.dedup(); + for i in picked { + let mut fewer = entries.clone(); + fewer.remove(i); + let mut memory = state_from_entries(&fewer, snapshot); + let mut registers = copy(®s); + // A wrong ECSM operand stops the walk (its witness refuses the + // point): seen as well. + let short = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + setup.walk(&mut memory, &mut registers, first, logs) + })); + seen.dropped += 1; + seen.caught += usize::from( + short.map_or(true, |short| !differences(&short, &again).is_empty()), + ); + } + } + seen.windows += 1; + seen.entries += entries.len(); + let w = &walked.walk; + seen.ecalls[0] += w.keccak_ops.len() + w.blake3_ops.len(); + seen.ecalls[1] += w.ecsm_ops.len() + w.blake3_absorb_ops.len(); + seen.ecalls[2] += w.hint_ops.len() + w.ecrec_ops.len(); + } + seen +} + +fn cycles_of(elf: &Elf) -> usize { + Executor::new(elf, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs + .len() +} + +/// ★ For twelve programs at two window lengths each, at every window's start +/// the pass holds the walk's timestamps and registers, its entries hold the +/// walk's timestamps and the snapshot holds the walk's values there, and the +/// window walked from them alone is the sequential walk's. +#[test] +fn the_pass_keeps_the_walks_timestamps_registers_and_entries() { + let mut total = Seen::default(); + for (name, elf) in programs() { + for window in window_lengths(cycles_of(&elf)) { + let seen = side_by_side(&name, &elf, window, false); + total.windows += seen.windows; + total.entries += seen.entries; + for (t, s) in total.ecalls.iter_mut().zip(seen.ecalls) { + *t += s; + } + } + } + assert!( + total.ecalls.iter().all(|&n| n > 0), + "the programs reach KECCAK/BLAKE3, ECSM/absorb and HINT: {:?}", + total.ecalls + ); + assert!(total.entries > 1000, "{} entries recorded", total.entries); + assert!(total.windows > 200, "{} windows passed", total.windows); +} + +/// ★ The entries are needed: an entry for a byte stamped before the window +/// whose old timestamp a memory row keeps, left out, walks the window +/// differently (or stops it), every time. (Not every entry is: a load's value +/// comes from its log, and a row keeps the old timestamps of its width only.) +#[test] +fn a_window_walked_without_one_of_its_entries_is_another_walk() { + let mut dropped = 0; + for (name, elf) in programs() { + let window = window_lengths(cycles_of(&elf))[0]; + let seen = side_by_side(&name, &elf, window, true); + assert_eq!( + seen.caught, seen.dropped, + "{name}: an entry left out went unseen" + ); + dropped += seen.dropped; + } + assert!(dropped > 50, "{dropped} entries left out"); +} + +/// The dense pc index finds every program's rows as the decode table does, +/// and nothing where it finds nothing. +#[test] +fn the_dense_pc_index_finds_the_decode_tables_rows() { + for (name, elf) in programs() { + let builder = + WindowedTraceBuilder::new(&elf, &[], &MaxRowsConfig::small()).expect("the builder"); + let decode = &builder.decode; + let rows = Rows::of(decode); + assert!(matches!(rows, Rows::Dense { .. }), "{name}: a dense index"); + let n = decode.padding_row() as u32; + let last = decode.entry(n - 1).pc; + for r in 0..n { + let pc = decode.entry(r).pc; + for probe in [pc, pc + 1, pc + 2, pc + 3, pc.wrapping_sub(4), last + 4, 0] { + assert_eq!( + rows.row(probe, decode), + decode.row(probe), + "{name}: pc {probe:#x}" + ); + } + } + } +} + +/// The pass with its memory left out (a measurement's arm) keeps the same +/// registers and records no entry; both find rows through the dense index. +#[test] +fn the_pass_without_its_memory_keeps_the_same_registers() { + for (name, elf) in programs() { + let window = window_lengths(cycles_of(&elf))[1]; + let builder = + WindowedTraceBuilder::new(&elf, &[], &MaxRowsConfig::small()).expect("the builder"); + let decode = Arc::clone(&builder.decode); + let mut full = LeanPass::new(Arc::clone(&decode), &builder.register_state).expect("a pass"); + let mut bare = LeanPass::new(decode, &builder.register_state).expect("a pass"); + assert!(full.dense_rows() && bare.dense_rows(), "{name}"); + let mut first = 0; + for (k, (logs, _)) in windows(&elf, window).iter().enumerate() { + let k = u32::try_from(k).expect("few windows"); + full.begin(k).expect("a window"); + bare.begin(k).expect("a window"); + full.pass(logs, first).expect("the pass"); + bare.pass_registers_only(logs, first).expect("the pass"); + first += logs.len(); + assert!(bare.entries().is_empty(), "{name}, window {k}"); + } + // ECALLs also write x10..x13 (and COMMIT the index); the bare pass + // leaves them out. + let (a, b) = (full.registers(), bare.registers()); + let general = (1..32).filter(|&r| !(10..=13).contains(&r)); + for r in general { + assert_eq!(a.regs[r], b.regs[r], "{name}: x{r}"); + } + assert_eq!(a.pc_register, b.pc_register, "{name}: pc"); + } +} + +/// The pass split over two threads (a measurement's arm) records the pass's +/// entries, in order, and keeps its registers, at every window. +#[test] +fn the_split_pass_is_the_pass() { + for (name, elf) in programs() { + for window in window_lengths(cycles_of(&elf)) { + let builder = + WindowedTraceBuilder::new(&elf, &[], &MaxRowsConfig::small()).expect("the builder"); + let decode = Arc::clone(&builder.decode); + let mut pass = + LeanPass::new(Arc::clone(&decode), &builder.register_state).expect("a pass"); + let mut front = Front::new(decode, &builder.register_state).expect("a front"); + let mut back = Back::new(); + let (mut first, mut accesses) = (0, Vec::new()); + for (k, (logs, _)) in windows(&elf, window).iter().enumerate() { + let k = u32::try_from(k).expect("few windows"); + pass.begin(k).expect("a window"); + back.begin(k).expect("a window"); + pass.pass(logs, first).expect("the pass"); + accesses.clear(); + front.pass(logs, first, &mut accesses).expect("the front"); + back.apply(&accesses); + first += logs.len(); + assert_eq!( + back.entries(), + pass.entries(), + "{name}/{window}, window {k}" + ); + assert_eq!( + register_differences(&front.registers(), &pass.registers()), + 0, + "{name}/{window}, window {k}" + ); + } + } + } +} + +/// A window the pass cannot mark is refused; the last it can is taken. +#[test] +fn the_pass_refuses_a_window_past_its_marks() { + let elf = Elf::load(&asm_elf_bytes(PROGRAMS[0])).expect("the ELF loads"); + let builder = + WindowedTraceBuilder::new(&elf, &[], &MaxRowsConfig::small()).expect("the builder"); + let mut pass = LeanPass::new(Arc::clone(&builder.decode), &builder.register_state) + .expect("the pass starts"); + assert!(pass.begin(UNTOUCHED as u32 - 1).is_ok()); + assert!(pass.begin(UNTOUCHED as u32).is_err()); +} diff --git a/prover/src/tables/trace_builder/windowed/parallel.rs b/prover/src/tables/trace_builder/windowed/parallel.rs new file mode 100644 index 000000000..8f7fb894e --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/parallel.rs @@ -0,0 +1,729 @@ +//! A run's windows walked from their entries, on several threads +//! ([`walk_windows`], `LAMBDA_VM_BLOCK_WALKERS`). +//! +//! The lean pass ([`super::lean_pass::LeanPass`]) gives each window, before it +//! is walked, its first cycle, its starting register cells and its entries +//! (each byte it touches, with the byte's timestamp before it); the executor's +//! snapshot at the window's start gives the entries' values. A +//! [`WindowWalker`] seeds a window's state from them and walks the window with +//! the walk's own code ([`super::walk_window`]), so the window is the one the +//! sequential walk walks. Three checks keep a mistake from becoming a wrong +//! trace: +//! - a byte the window reads that its entries do not hold reads [`ABSENT`], +//! and the window is refused; +//! - after the window, its registers must be the next window's starting +//! registers, and every byte it holds that the next window touches must hold +//! the next window's entry (the executor's value, the pass's timestamp); +//! - the run's last walked window must agree, byte for byte, with the state +//! handed to the finish ([`super::lean_pass::LeanPass::walked_state`]); +//! - the windows' boundaries must add up ([`cell_hash`]): each window's +//! seeded bytes A and the same bytes at its end B, hashed and summed, give +//! Σ (B − A) over the run equal to the sum, over every byte a window +//! touched, of its cell handed to the finish less its cell at the run's +//! start. That catches what the exit checks cannot see: an entry's +//! timestamp or value wrong on a byte the window then writes, whose end is +//! right. (An entry wrong on a byte the window only reads ends as it was +//! seeded and adds nothing to the sums: the exit checks see it where the +//! next window touches the byte, and the verifier refuses its rows where +//! none does.) An entry stamped 0 must carry the image's value. +//! +//! The threads: the executor (the caller's) sends each window with its +//! snapshot; one thread runs the pass and hands out each window as a job as it +//! arrives, but the one that halts, which is the run's last and the finish's; +//! `walkers` threads walk the jobs; the calling thread absorbs the walked +//! windows strictly in order. At most `walkers + 2` windows are handed out and +//! not yet absorbed. + +use std::collections::BTreeMap; +use std::hash::BuildHasher; +use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use executor::vm::logs::Log; +use executor::vm::memory::MemorySnapshot; + +use super::super::{ABSENT, MemoryState, NO_PAGE, RegisterState, WalkSizes}; +use super::head_log; +use super::lean_pass::LeanPass; +use super::{Accumulator, WalkedWindow, WindowedTraceBuilder, walk_window}; +use crate::Error; +use crate::paged_mem::PagedMem; +use crate::tables::decode::DecodeTable; +use crate::tables::page::DEFAULT_PAGE_SIZE; + +/// A window to walk from its entries. +struct WindowJob { + /// Its number in the run, from 0. + window: u32, + /// Its first cycle. + first: usize, + logs: Arc>, + /// The executor's memory at its start. + start: Arc, + /// The register cells at its start. + registers: RegisterState, + /// Each byte it touches and the byte's timestamp before it. + entries: Arc>, +} + +/// The walk's settings a window is walked with. +struct WalkSettings { + decode: Arc, + lookups: bool, + batch: usize, + /// The run's key for [`cell_hash`]. + key: u64, + /// Keep each window's logs with it ([`WindowedTraceBuilder::keeps_logs`]). + keep_logs: bool, +} + +/// One byte's cell hashed for the windows' boundary sums, under the run's +/// `key` so that no mistake with a structure cancels out in them; one-to-one +/// in `(timestamp, value)` for a given address. Timestamps stay below 2^40. +#[inline] +pub(super) fn cell_hash(key: u64, address: u64, timestamp: u64, value: u8) -> u64 { + // splitmix64's finalizer. + fn mix(mut x: u64) -> u64 { + x ^= x >> 30; + x = x.wrapping_mul(0xbf58_476d_1ce4_e5b9); + x ^= x >> 27; + x = x.wrapping_mul(0x94d0_49bb_1331_11eb); + x ^ (x >> 31) + } + mix(mix(address ^ key) ^ ((timestamp << 8) | u64::from(value))) +} + +/// The windows' boundary sums against the run's: refused unless they are the +/// same ([`cell_hash`]). +fn check_boundaries(windows: u64, run: u64) -> Result<(), Error> { + if windows != run { + return Err(Error::Prover(format!( + "the windows' boundaries do not add up: the walked windows give {windows:#018x}, the \ + bytes they touched {run:#018x}" + ))); + } + Ok(()) +} + +/// One walker: its window state, kept from window to window so its pages are +/// reused, and what it walked last. +struct WindowWalker { + settings: Arc, + sizes: WalkSizes, + /// The window's state: every byte it does not hold reads `(0, ABSENT)`. + memory: MemoryState, + /// The registers at the end of the window walked last. + registers: RegisterState, + /// The bytes the window walked last touched (reset to [`ABSENT`] before + /// the next is seeded), and the state pages it touched. + touched: Vec, + pages: Vec, + /// The seconds the window walked last took to seed. + seeding: f64, +} + +/// The state page holding `address`. +fn page_of(address: u64) -> u64 { + address & !(DEFAULT_PAGE_SIZE as u64 - 1) +} + +impl WindowWalker { + fn new(settings: Arc) -> Self { + let mut memory = MemoryState::new(); + memory.cells = PagedMem::new((0, ABSENT)); + memory.lean = true; + Self { + settings, + sizes: WalkSizes::default(), + memory, + registers: RegisterState::new(0), + touched: Vec::new(), + pages: Vec::new(), + seeding: 0.0, + } + } + + /// Walks `job` from its entries into fresh lists, with its boundary sum: + /// its bytes at the end, hashed and summed, less the same at its start + /// ([`cell_hash`]). Refuses the window if it read a byte its entries do + /// not hold, or if an entry stamped 0 does not carry `image`'s value. + fn walk( + &mut self, + job: &WindowJob, + image: &PagedMem, + ) -> Result<(WalkedWindow, u64), Error> { + let t = Instant::now(); + let seeded = self.seed(job, image)?; + self.seeding = t.elapsed().as_secs_f64(); + self.registers = job.registers.clone(); + let s = &*self.settings; + let mut walked = walk_window( + &s.decode, + s.lookups, + s.batch, + &mut self.sizes, + &mut self.memory, + &mut self.registers, + job.first, + &job.logs, + )?; + if s.keep_logs { + walked.keep_logs(Arc::clone(&job.logs)); + } + if self.memory.take_missing() { + return Err(Error::Prover(format!( + "window {}: the walk read a byte its entries do not hold", + job.window + ))); + } + let key = self.settings.key; + let ended = job.entries.iter().fold(0u64, |sum, &(address, _)| { + let (value, ts) = self.memory.cells.get(address); + sum.wrapping_add(cell_hash(key, address, ts, value)) + }); + Ok((walked, ended.wrapping_sub(seeded))) + } + + /// The state cleared of the last window's bytes, its pages kept if this + /// window or the last touched them, then seeded with `job`'s entries; the + /// seeded cells hashed and summed ([`cell_hash`]). Refuses an entry + /// stamped 0 (no window wrote the byte before) whose value is not + /// `image`'s. + fn seed(&mut self, job: &WindowJob, image: &PagedMem) -> Result { + for &address in &self.touched { + self.memory.write_byte(address, 0, ABSENT); + } + let mut pages: Vec = job.entries.iter().map(|&(a, _)| page_of(a)).collect(); + pages.sort_unstable(); + pages.dedup(); + let last = std::mem::replace(&mut self.pages, pages); + let now = &self.pages; + self.memory.cells.retain_pages(|base| { + now.binary_search(&base).is_ok() || last.binary_search(&base).is_ok() + }); + self.memory.hot = NO_PAGE; + self.touched.clear(); + let (key, mut seeded) = (self.settings.key, 0u64); + for &(address, ts) in job.entries.iter() { + let value = job.start.load_byte(address); + if ts == 0 && value != image.get(address) { + return Err(Error::Prover(format!( + "window {}: the byte at {address:#x} has timestamp 0 but holds {value}, not the \ + image's {}", + job.window, + image.get(address) + ))); + } + self.memory.write_byte(address, value, ts); + self.touched.push(address); + seeded = seeded.wrapping_add(cell_hash(key, address, ts, value)); + } + self.memory.take_missing(); + Ok(seeded) + } + + /// After walking window `job`: its registers must be `next_registers`, + /// and each byte of `next_entries` it holds must hold the entry's + /// timestamp and its value in `next_start`. + fn check_exit( + &self, + job: &WindowJob, + next_registers: &RegisterState, + next_entries: &[(u64, u64)], + next_start: &MemorySnapshot, + ) -> Result<(), Error> { + self.check_registers(job, next_registers)?; + for &(address, ts) in next_entries { + let held = self.memory.cells.get(address); + if held.1 != ABSENT && held != (next_start.load_byte(address), ts) { + return Err(Error::Prover(format!( + "window {}: it ends with {held:?} at {address:#x}, the next window starts from \ + ({}, {ts})", + job.window, + next_start.load_byte(address) + ))); + } + } + Ok(()) + } + + /// After walking the run's last walked window `job`: its registers must be + /// `registers`, and every byte it touched must hold what `state` holds. + fn check_exit_into( + &self, + job: &WindowJob, + registers: &RegisterState, + state: &MemoryState, + ) -> Result<(), Error> { + self.check_registers(job, registers)?; + for &address in &self.touched { + let (held, there) = (self.memory.cells.get(address), state.cells.get(address)); + if held != there { + return Err(Error::Prover(format!( + "window {}: it ends with {held:?} at {address:#x}, the finish starts from \ + {there:?}", + job.window + ))); + } + } + Ok(()) + } + + fn check_registers(&self, job: &WindowJob, next: &RegisterState) -> Result<(), Error> { + let (a, b) = (&self.registers, next); + if a.regs != b.regs + || a.index_register != b.index_register + || a.pc_register != b.pc_register + { + return Err(Error::Prover(format!( + "window {}: its registers at the end are not the next window's at its start", + job.window + ))); + } + Ok(()) + } +} + +/// A thread's error that only says another thread stopped first: never the +/// one [`walk_windows`] returns while there is another. +const WALKERS_STOPPED: &str = "the walkers stopped"; +const PASS_STOPPED: &str = "the pass stopped before the window after it"; + +fn is_knock_on(e: &Error) -> bool { + matches!(e, Error::Prover(m) if m == WALKERS_STOPPED || m.ends_with(PASS_STOPPED)) +} + +/// What follows a walked window, for its exit check. +enum After { + /// The next window's start. + Next { + registers: RegisterState, + entries: Arc>, + start: Arc, + }, + /// The state handed to the finish (the window is the run's last walked). + Finish { + registers: RegisterState, + state: Arc, + }, +} + +impl WindowWalker { + /// Walks `job`, then waits for what follows it and checks its exit + /// against that. The busy seconds leave the wait out. + fn walk_and_check( + &mut self, + job: &WindowJob, + after: &Receiver, + image: &PagedMem, + ) -> Result<(WalkedWindow, f64, u64), Error> { + let t = Instant::now(); + let (walked, boundary) = self.walk(job, image)?; + let mut busy = t.elapsed().as_secs_f64(); + head_log::window_walked(job.window as usize, t, self.seeding); + let after = after + .recv() + .map_err(|_| Error::Prover(format!("window {}: {PASS_STOPPED}", job.window)))?; + let t = Instant::now(); + match after { + After::Next { + registers, + entries, + start, + } => self.check_exit(job, ®isters, &entries, &start)?, + After::Finish { registers, state } => self.check_exit_into(job, ®isters, &state)?, + } + busy += t.elapsed().as_secs_f64(); + Ok((walked, busy, boundary)) + } + + /// The bytes its window state takes on the heap. + fn state_bytes(&self) -> usize { + self.memory.heap_bytes() + } +} + +/// A job, and where what follows it arrives. +struct Ticket { + job: WindowJob, + after: Receiver, +} + +/// What the pass thread ends with. +struct PassEnd { + last: Arc>, + state: Arc, + registers: RegisterState, + /// The last window's first cycle, and the windows handed out. + first: usize, + windows: usize, + busy: f64, + /// Of `busy`, the finish's state. + state_secs: f64, + entries: usize, + most_entries: usize, + /// The bytes the windows touched: their cells handed to the finish less + /// their cells at the run's start, hashed and summed ([`cell_hash`]). + boundaries: u64, +} + +/// What a run walked on several threads reports (the `BLOCK WALKERS` line). +pub struct ParallelWalk { + /// The run's last window, the finish's. + pub last: Vec, + pub walkers: usize, + pub windows: usize, + pub pass_secs: f64, + /// Of the pass's, the seconds building the finish's state. + pub finish_state_secs: f64, + /// The walkers' busy seconds, summed. + pub walk_secs: f64, + pub entries: usize, + pub most_entries: usize, + /// The most walked windows held, waiting for an earlier one. + pub held_most: usize, + /// The most bytes the walkers' window states took at once. + pub state_bytes: usize, +} + +impl ParallelWalk { + pub fn line(&self) -> String { + format!( + "BLOCK WALKERS: {} · pass busy {:.2}s · entries {} (most {} in one window) · walkers busy {:.2}s (summed) · reorder held at most {} · window states {:.2} GiB · {} windows walked from their entries · the finish's state built in {:.2}s", + self.walkers, + self.pass_secs, + self.entries, + self.most_entries, + self.walk_secs, + self.held_most, + self.state_bytes as f64 / (1u64 << 30) as f64, + self.windows, + self.finish_state_secs, + ) + } +} + +/// The run's windows, as the executor sends them (each with its memory at the +/// window's start), walked on `walkers` threads from the lean pass's entries +/// and absorbed strictly in window order by `absorb`, which is also given the +/// bytes the walkers' window states take then. The builder then holds the +/// state the walk holds before the run's last window, which is returned (the +/// finish walks it as usual). The builder must not have walked a window yet. +/// The first error anywhere stops every thread and is returned; a walker that +/// panics is an error. +pub fn walk_windows( + builder: &mut WindowedTraceBuilder<'_>, + windows: Receiver<(Vec, Option)>, + walkers: usize, + absorb: impl FnMut(&mut Accumulator<'_>, WalkedWindow, usize) -> Result<(), Error>, +) -> Result { + walk_numbered(builder, windows, walkers, |k| k, absorb) +} + +/// [`walk_windows`], each job numbered `number(k)` for the run's window `k`: +/// a test numbers them wrong to show the absorbing side refuses it. +fn walk_numbered( + builder: &mut WindowedTraceBuilder<'_>, + windows: Receiver<(Vec, Option)>, + walkers: usize, + number: fn(u32) -> u32, + mut absorb: impl FnMut(&mut Accumulator<'_>, WalkedWindow, usize) -> Result<(), Error>, +) -> Result { + assert!(walkers >= 1, "walk_windows needs a walker"); + debug_assert_eq!(builder.cycles, 0, "walk_windows starts the run"); + let settings = Arc::new(WalkSettings { + decode: Arc::clone(&builder.decode), + lookups: !builder.lean.lookups, + batch: builder.walk_batch, + key: std::collections::hash_map::RandomState::new().hash_one(0u64), + keep_logs: builder.keeps_logs(), + }); + let key = settings.key; + let start_registers = builder.register_state.clone(); + let lean = builder.lean.memory; + let (end, walk_secs, held_most, state_bytes) = { + let (mut accumulator, image) = builder.accumulator_and_image(); + let (job_tx, job_rx) = sync_channel::(walkers); + let job_rx = Mutex::new(job_rx); + std::thread::scope(|scope| -> Result<_, Error> { + #[allow(clippy::type_complexity)] + let (done_tx, done_rx) = + sync_channel::<(u32, usize, Result<(WalkedWindow, f64, usize, u64), Error>)>( + walkers + 2, + ); + let (permit_tx, permit_rx) = sync_channel::<()>(walkers + 2); + let decode = Arc::clone(&settings.decode); + let pass = scope.spawn(move || { + pass_thread( + decode, + &start_registers, + image, + lean, + key, + windows, + number, + job_tx, + permit_tx, + ) + }); + let mut threads = Vec::with_capacity(walkers); + for w in 0..walkers { + let (job_rx, done_tx, settings) = (&job_rx, done_tx.clone(), Arc::clone(&settings)); + threads.push(scope.spawn(move || { + let mut walker = WindowWalker::new(settings); + // After an error (its own or the absorbing side's), the + // walker takes the jobs still coming and drops them, so the + // pass never waits on a full queue nobody reads; the + // closed permits stop the pass. + let mut stopped = false; + loop { + let ticket = match job_rx.lock() { + Ok(rx) => rx.recv(), + Err(_) => break, + }; + let Ok(ticket) = ticket else { + break; + }; + if stopped { + continue; + } + let window = ticket.job.window; + let outcome = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + walker.walk_and_check(&ticket.job, &ticket.after, image) + })) + .unwrap_or_else(|_| { + Err(Error::Prover(format!( + "window {window}: its walker panicked" + ))) + }) + .map(|(walked, busy, boundary)| { + (walked, busy, walker.state_bytes(), boundary) + }); + let failed = outcome.is_err(); + if done_tx.send((window, w, outcome)).is_err() || failed { + stopped = true; + } + } + })); + } + drop(done_tx); + let (mut held, mut next, mut held_most) = (BTreeMap::new(), 0u32, 0usize); + // Each walker's window state as of its last window, and the most + // they took at once. + let (mut walk_secs, mut states, mut state_bytes) = (0.0, vec![0usize; walkers], 0); + // The walked windows' boundary sums, added as they come. + let mut boundaries = 0u64; + // The first error from a walker or from absorbing; an absorbing + // error is this thread's own and always wins. + let (mut failure, mut absorbing) = (None, false); + 'windows: for (window, w, outcome) in &done_rx { + let (walked, busy, bytes, boundary) = match outcome { + Ok(walked) => walked, + Err(e) => { + failure = Some(e); + break; + } + }; + walk_secs += busy; + boundaries = boundaries.wrapping_add(boundary); + states[w] = bytes; + state_bytes = state_bytes.max(states.iter().sum()); + if window < next || held.insert(window, walked).is_some() { + failure = Some(Error::Prover(format!("window {window} was walked twice"))); + break; + } + held_most = held_most.max(held.len()); + while let Some(walked) = held.remove(&next) { + if let Err(e) = absorb(&mut accumulator, walked, states.iter().sum()) { + (failure, absorbing) = (Some(e), true); + break 'windows; + } + head_log::window_absorbed(next as usize); + // One window fewer handed out and not absorbed. + let _ = permit_rx.recv(); + next += 1; + } + // On a correct run window `next` is handed out before every + // window held, so the permits leave room for at most + // `walkers + 1` held. With `walkers + 2` the pass waits for a + // permit only window `next` frees: it was never handed out. + if held.len() >= walkers + 2 { + failure = Some(Error::Prover(format!("window {next} was never walked"))); + break; + } + } + // On a failure the other threads see their channels close. + drop(done_rx); + drop(permit_rx); + for thread in threads { + if thread.join().is_err() { + failure.get_or_insert(Error::Prover("a walker thread panicked".into())); + } + } + let end = pass + .join() + .map_err(|_| Error::Prover("the lean pass panicked".into()))?; + // The cause, not its knock-on: this thread's own error first, then + // the pass's, then a walker's. + match (failure, end) { + (Some(e), _) if absorbing => Err(e), + (Some(e), Err(p)) => Err(if is_knock_on(&p) { e } else { p }), + (Some(e), Ok(_)) => Err(e), + (None, Err(p)) => Err(p), + (None, Ok(end)) if next as usize != end.windows || !held.is_empty() => { + Err(Error::Prover(format!( + "{} windows handed out, {next} absorbed in order", + end.windows + ))) + } + (None, Ok(end)) => { + check_boundaries(boundaries, end.boundaries)?; + Ok((end, walk_secs, held_most, state_bytes)) + } + } + })? + }; + let PassEnd { + last, + state, + registers, + first, + windows, + busy, + state_secs, + entries, + most_entries, + boundaries: _, + } = end; + let state = Arc::try_unwrap(state) + .map_err(|_| Error::Prover("the finish's state is still shared".into()))?; + builder.install_walked(state, registers, first, windows, walk_secs); + Ok(ParallelWalk { + last: Arc::try_unwrap(last).unwrap_or_else(|last| (*last).clone()), + walkers, + windows, + pass_secs: busy, + finish_state_secs: state_secs, + walk_secs, + entries, + most_entries, + held_most, + state_bytes, + }) +} + +/// The pass's thread: each window passed and handed out as a job (numbered +/// `number(k)`) as it arrives, a permit taken for each; what follows each job +/// sent once the next window is passed. The window that halts is the run's +/// last: it is not handed out, and the job before it is followed by the +/// finish's state. +#[allow(clippy::too_many_arguments)] +fn pass_thread( + decode: Arc, + start_registers: &RegisterState, + image: &PagedMem, + lean: bool, + key: u64, + windows: Receiver<(Vec, Option)>, + number: fn(u32) -> u32, + jobs: SyncSender, + permits: SyncSender<()>, +) -> Result { + fn stopped(_: T) -> Error { + Error::Prover(WALKERS_STOPPED.into()) + } + let mut pass = LeanPass::new(decode, start_registers)?; + let mut last: Option<(Arc>, Arc, RegisterState)> = None; + let mut waiting: Option> = None; + let (mut k, mut first, mut busy, mut entries, mut most) = (0u32, 0usize, 0.0, 0usize, 0usize); + let (mut sent, mut state_secs, mut boundaries) = (false, 0.0, 0u64); + for (logs, start) in windows { + sent = true; + let start = Arc::new(start.ok_or_else(|| { + Error::Prover("the executor sent a window without its memory".into()) + })?); + if last.is_some() { + return Err(Error::Prover( + "the executor sent a window after the one that halts".into(), + )); + } + let t = Instant::now(); + let registers = pass.registers(); + // The run's last window: the walk's state at its start is the + // finish's. + if logs.last().is_some_and(|log| log.next_pc == 0) { + let state = Arc::new(pass.walked_state(&start, image, lean)); + boundaries = pass.boundaries(&state, image, key); + state_secs = t.elapsed().as_secs_f64(); + busy += state_secs; + if let Some(after) = waiting.take() { + let _ = after.send(After::Finish { + registers: registers.clone(), + state: Arc::clone(&state), + }); + } + last = Some((Arc::new(logs), state, registers)); + continue; + } + pass.begin(k)?; + pass.pass(&logs, first)?; + let window_entries = Arc::new(pass.take_entries()); + let passed = Instant::now(); + busy += t.elapsed().as_secs_f64(); + entries += window_entries.len(); + most = most.max(window_entries.len()); + if let Some(after) = waiting.take() { + // A walker gone (an error elsewhere) does not stop the pass here: + // the permits or the jobs channel do. + let _ = after.send(After::Next { + registers: registers.clone(), + entries: Arc::clone(&window_entries), + start: Arc::clone(&start), + }); + } + let (after_tx, after_rx) = sync_channel(1); + permits.send(()).map_err(stopped)?; + let (cycles, handed) = (logs.len(), window_entries.len()); + jobs.send(Ticket { + job: WindowJob { + window: number(k), + first, + logs: Arc::new(logs), + start, + registers, + entries: window_entries, + }, + after: after_rx, + }) + .map_err(stopped)?; + head_log::window_handed(k as usize, handed, t, passed); + waiting = Some(after_tx); + first += cycles; + k += 1; + } + drop(jobs); + let Some((last, state, registers)) = last else { + return Err(if sent { + Error::Prover("the executor stopped before the run halted".into()) + } else { + Error::Execution("the run executed no cycle".to_string()) + }); + }; + Ok(PassEnd { + last, + state, + registers, + first, + windows: k as usize, + busy, + state_secs, + entries, + most_entries: most, + boundaries, + }) +} + +#[cfg(test)] +mod tests; diff --git a/prover/src/tables/trace_builder/windowed/parallel/tests.rs b/prover/src/tables/trace_builder/windowed/parallel/tests.rs new file mode 100644 index 000000000..8a844a007 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/parallel/tests.rs @@ -0,0 +1,795 @@ +//! A run walked window by window from the lean pass's entries and the +//! executor's snapshots, on one thread, against the whole-run build; and each +//! check refusing what it guards. + +use std::sync::Arc; + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; + +use super::super::lean_pass::LeanPass; +use super::super::{ChunkJob, WalkedWindow, WindowedTraceBuilder}; +use super::{WalkSettings, WindowJob, WindowWalker, walk_numbered, walk_windows}; +use crate::Error; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::Traces; +use crate::tests::windowed_builder_tests::{run, same_traces, whole}; + +/// A mistake put into the windows on purpose, to show a check catches it. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Tamper { + None, + /// Window `window` without its entry number `at`. + DropEntry { + window: u32, + at: usize, + }, + /// Window `window`'s entry number `at` one timestamp later. + LaterEntry { + window: u32, + at: usize, + }, + /// Window `window` with the next window's snapshot. + LateStart(u32), + /// Window `window` walked from one cycle later. + LaterFirst(u32), + /// Window `window` starting with x5 one larger. + Register(u32), + /// The run's first entry whose byte a window before the previous one + /// wrote, with a value other than the image's, stamped 0 instead (none if + /// no entry is such). The previous window does not touch the byte, so its + /// exit check cannot see it. + ZeroEntry, +} + +/// The run of `program` built by windows of `window` cycles walked from their +/// entries (the last window, the finish's, walked on the state built from the +/// pass), with its streamed chunks put back. With `dropped`, the builder drops +/// the ops it hands out, as the block's prove does. +fn walked_from_entries( + program: &Elf, + max_rows: &MaxRowsConfig, + window: usize, + dropped: bool, + tamper: Tamper, +) -> Result { + let mut builder = WindowedTraceBuilder::new(program, &[], max_rows)?; + if dropped { + builder = builder.drop_streamed_ops()?; + } + let settings = Arc::new(WalkSettings { + decode: Arc::clone(&builder.decode), + lookups: !builder.lean.lookups, + batch: builder.walk_batch, + key: 0x5eed_1234_abcd_0042, + keep_logs: builder.keeps_logs(), + }); + let mut pass = LeanPass::new(Arc::clone(&builder.decode), &builder.register_state)?; + let mut executor = + Executor::new(program, Vec::new()).map_err(|e| Error::Execution(format!("{e}")))?; + let mut start = Arc::new(executor.memory().snapshot()); + let mut windows = Vec::new(); + while let Some(logs) = executor + .resume_with_limit(window) + .map_err(|e| Error::Execution(format!("{e}")))? + { + let logs = Arc::new(logs.to_vec()); + let next = Arc::new(executor.memory().snapshot()); + windows.push((logs, std::mem::replace(&mut start, next))); + } + let (last_logs, last_start) = windows.pop().expect("a run has a window"); + let (mut jobs, mut first) = (Vec::new(), 0usize); + for (k, (logs, start)) in windows.into_iter().enumerate() { + let k = u32::try_from(k).expect("few windows"); + let registers = pass.registers(); + pass.begin(k)?; + pass.pass(&logs, first)?; + let cycles = logs.len(); + jobs.push(WindowJob { + window: k, + first, + logs, + start, + registers, + entries: Arc::new(pass.take_entries()), + }); + first += cycles; + } + let state = pass.walked_state(&last_start, &builder.image, builder.memory_state.lean); + let run = pass.boundaries(&state, &builder.image, settings.key); + let registers = pass.registers(); + tamper_with(&mut jobs, &last_start, &builder.image, tamper); + let mut walker = WindowWalker::new(settings); + let (mut chunks, mut boundaries) = (Vec::new(), 0u64); + { + let (mut accumulator, image) = builder.accumulator_and_image(); + for k in 0..jobs.len() { + let (walked, boundary) = walker.walk(&jobs[k], image)?; + boundaries = boundaries.wrapping_add(boundary); + match jobs.get(k + 1) { + Some(next) => { + walker.check_exit(&jobs[k], &next.registers, &next.entries, &next.start)? + } + None => walker.check_exit_into(&jobs[k], ®isters, &state)?, + } + chunks.extend( + accumulator + .absorb(walked) + .into_iter() + .map(ChunkJob::generate), + ); + } + } + super::check_boundaries(boundaries, run)?; + builder.install_walked(state, registers, first, jobs.len(), 0.0); + let mut traces = builder.finish(&last_logs)?; + traces.insert_streamed(chunks)?; + Ok(traces) +} + +fn tamper_with( + jobs: &mut [WindowJob], + last_start: &Arc, + image: &crate::paged_mem::PagedMem, + tamper: Tamper, +) { + let at = |w: u32| w as usize; + match tamper { + Tamper::None => {} + Tamper::DropEntry { window, at: i } => { + Arc::make_mut(&mut jobs[at(window)].entries).remove(i); + } + Tamper::LaterEntry { window, at: i } => { + Arc::make_mut(&mut jobs[at(window)].entries)[i].1 += 1 + } + Tamper::LateStart(window) => { + let next = jobs + .get(at(window) + 1) + .map_or_else(|| Arc::clone(last_start), |j| Arc::clone(&j.start)); + jobs[at(window)].start = next; + } + Tamper::LaterFirst(window) => jobs[at(window)].first += 1, + Tamper::Register(window) => jobs[at(window)].registers.regs[5].0 += 1, + Tamper::ZeroEntry => { + let found = jobs.iter().enumerate().skip(1).find_map(|(k, job)| { + let before = &jobs[k - 1].entries; + let i = job.entries.iter().position(|&(a, ts)| { + ts > 0 + && job.start.load_byte(a) != image.get(a) + && !before.iter().any(|&(b, _)| b == a) + })?; + Some((k, i)) + }); + if let Some((k, i)) = found { + Arc::make_mut(&mut jobs[k].entries)[i].1 = 0; + } + } + } +} + +const PROGRAMS: [&str; 14] = [ + "sub", + "all_instructions_64", + "lw_sw_offset_odd", + "test_memw_split_ts", + "test_keccak", + "test_keccak_multi", + "test_keccak256", + "test_blake3", + "test_blake3_absorb", + "test_commit_split", + "test_ecsm", + "test_copy_ecalls", + "test_copy_overlaps", + "test_ecrec", +]; + +/// ★ Walked from their entries, at every window length (one cycle, less than a +/// chunk, exactly a chunk, not a divisor of it, longer than the run), with the +/// handed-out ops kept or dropped, the windows build the whole-run tables. +#[test] +fn windows_walked_from_their_entries_build_the_whole_run_tables() { + let max_rows = MaxRowsConfig::small(); + for name in PROGRAMS { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [1, 7, 32, 33, 1000, logs.len()] { + for dropped in [false, true] { + let traces = + walked_from_entries(&program, &max_rows, window.max(1), dropped, Tamper::None) + .unwrap_or_else(|e| panic!("{name}, window {window}: {e:?}")); + same_traces(&reference, &traces); + } + } + } +} + +/// The memory bytes whose old timestamp the window's MEMW and MEMW_A rows keep +/// (within each access's width), sorted: where a moved entry timestamp reaches +/// the tables. +fn kept_timestamps(window: &WalkedWindow) -> Vec { + let mut out = Vec::new(); + for op in window.walk.memw.general.iter().filter(|op| !op.is_register) { + out.extend((0..u64::from(op.width)).map(|i| op.base_address.wrapping_add(i))); + } + for row in window + .walk + .memw + .aligned + .iter() + .filter(|row| !row.is_register()) + { + out.extend((0..row.width() as u64).map(|i| row.base_address().wrapping_add(i))); + } + out.sort_unstable(); + out.dedup(); + out +} + +/// A walked window's entries, whether it wrote a byte it touches, and the +/// bytes whose old timestamp its memory rows keep ([`entries_of`]). +type Picks = (Vec<(u64, u64)>, bool, Vec); + +/// What a tamper test picks from, per walked window: its entries; whether the +/// window wrote a byte it touches (so the next window's snapshot differs from +/// its own there); and the bytes whose old timestamp its memory rows keep. +fn entries_of(program: &Elf, window: usize) -> Vec { + let mut builder = + WindowedTraceBuilder::new(program, &[], &MaxRowsConfig::small()).expect("builder"); + let mut pass = + LeanPass::new(Arc::clone(&builder.decode), &builder.register_state).expect("pass"); + let mut executor = Executor::new(program, Vec::new()).expect("the executor"); + let mut starts = vec![executor.memory().snapshot()]; + let mut windows = Vec::new(); + while let Some(logs) = executor + .resume_with_limit(window) + .expect("the run") + .map(<[Log]>::to_vec) + { + windows.push(logs); + starts.push(executor.memory().snapshot()); + } + windows.pop(); + let mut first = 0; + let (mut walker, _) = builder.split(); + windows + .iter() + .enumerate() + .map(|(k, logs)| { + pass.begin(k as u32).expect("a window"); + pass.pass(logs, first).expect("the pass"); + first += logs.len(); + let entries = pass.take_entries(); + let wrote = entries + .iter() + .any(|&(a, _)| starts[k].load_byte(a) != starts[k + 1].load_byte(a)); + let kept = kept_timestamps(&walker.walk(logs).expect("the walk")); + (entries, wrote, kept) + }) + .collect() +} + +/// Built with `tamper`: refused (an error or a panic in the walk), or a build +/// whose tables differ from the whole run's. +fn caught(program: &Elf, reference: &Traces, window: usize, tamper: Tamper) -> &'static str { + let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + walked_from_entries(program, &MaxRowsConfig::small(), window, true, tamper) + })); + match built { + Err(_) => "stopped", + Ok(Err(_)) => "refused", + Ok(Ok(traces)) => { + let same = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + same_traces(reference, &traces) + })); + if same.is_err() { "differs" } else { "unseen" } + } + } +} + +/// ★ Each check catches what it guards. Over several programs and windows: +/// a window missing any one of its entries is refused, with no panic first +/// (the walk reads a byte it does not hold, and ECSM computes no witness from +/// such a byte); a window walked from a cycle off is refused (its +/// registers end off); an entry's timestamp moved where a memory row keeps +/// it (the window then writes the byte, so its end is right) is refused by +/// the windows' boundary sums; and the next window's snapshot (where the +/// window wrote a byte it touches) or a register off never builds the +/// whole-run tables. +#[test] +fn each_check_catches_what_it_guards() { + let window = 7; + let mut seen = std::collections::BTreeMap::<(&str, &str), usize>::new(); + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm", + "test_commit_split", + ] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &MaxRowsConfig::small()); + let entries = entries_of(&program, window); + let windows = entries.len() as u32; + assert!(windows > 2, "{name}: {windows} windows"); + for w in [0, windows / 2, windows - 1] { + let (entries, wrote, kept) = &entries[w as usize]; + let n = entries.len(); + for i in [0, n / 2, n.saturating_sub(1)] + .into_iter() + .filter(|&i| i < n) + { + let how = caught( + &program, + &reference, + window, + Tamper::DropEntry { window: w, at: i }, + ); + assert_eq!(how, "refused", "{name}: window {w} without entry {i}"); + *seen.entry(("drop", how)).or_default() += 1; + } + // A moved timestamp reaches the tables where a memory row keeps it. + let reached: Vec = (0..n) + .filter(|&i| kept.binary_search(&entries[i].0).is_ok()) + .collect(); + let picks = [ + reached.first(), + reached.get(reached.len() / 2), + reached.last(), + ]; + for &i in picks.into_iter().flatten() { + let how = caught( + &program, + &reference, + window, + Tamper::LaterEntry { window: w, at: i }, + ); + // The window's memory row keeps the moved timestamp and writes + // the byte, so the exit checks see nothing: the boundary sums + // refuse it. + assert_eq!( + how, "refused", + "{name}: window {w}, entry {i} a timestamp later" + ); + *seen.entry(("later entry", how)).or_default() += 1; + } + let how = caught(&program, &reference, window, Tamper::LaterFirst(w)); + assert!( + matches!(how, "refused" | "stopped"), + "{name}: window {w} a cycle later: {how}" + ); + *seen.entry(("later first", how)).or_default() += 1; + // The next window's snapshot differs from the window's own only + // where the window wrote. + if *wrote { + let how = caught(&program, &reference, window, Tamper::LateStart(w)); + assert_ne!(how, "unseen", "{name}: window {w} with the next snapshot"); + *seen.entry(("late start", how)).or_default() += 1; + } + let how = caught(&program, &reference, window, Tamper::Register(w)); + assert_ne!(how, "unseen", "{name}: window {w} with x5 off"); + *seen.entry(("register", how)).or_default() += 1; + } + } + println!("caught: {seen:?}"); + assert!(seen.values().sum::() > 60, "{seen:?}"); +} + +/// ★ An entry stamped 0 says no window wrote the byte before, so its value +/// must be the image's: one that holds another value is refused by that +/// check, before the boundary sums. +#[test] +fn an_entry_stamped_0_must_carry_the_images_value() { + let mut refused = 0; + for name in PROGRAMS { + let (program, _) = run(name); + // A program with no such byte builds untampered. + if let Err(e) = walked_from_entries( + &program, + &MaxRowsConfig::small(), + 7, + true, + Tamper::ZeroEntry, + ) { + assert!( + matches!(&e, Error::Prover(m) if m.contains("has timestamp 0 but holds")), + "{name}: {e:?}" + ); + refused += 1; + } + } + assert!( + refused >= 3, + "only {refused} programs had a byte to stamp 0" + ); +} + +/// The bytes each value-only reader reads in a walked window: HINT's input, +/// ECSM's operands, KECCAK's and BLAKE3's states, a Keccak-256 call's blocks, +/// the absorb's control block and message. (The bytes COPY moves, the committed output's among them, are +/// read and written through MEMW ops like any load's.) +fn value_reads(window: &WalkedWindow) -> Vec<(&'static str, u64, u64)> { + let w = &window.walk; + let mut out = Vec::new(); + out.extend(w.hint_ops.iter().map(|o| ("HINT input", o.in_addr, 32))); + for o in &w.ecsm_ops { + out.push(("ECSM xG", o.addr_xg, 32)); + out.push(("ECSM k", o.addr_k, 32)); + } + for o in &w.ecrec_ops { + out.push(("ECREC signature", o.addr_sig, 64)); + out.push(("ECREC hash", o.addr_msg, 32)); + } + for o in &w.keccak_ops { + match &o.sponge { + None => out.push(("KECCAK lanes", o.state_addr, 200)), + Some(s) => out.push(("KECCAK256 block", s.src, s.taken() as u64)), + } + } + out.extend( + w.blake3_ops + .iter() + .map(|o| ("BLAKE3 state", o.state_addr, 112)), + ); + for o in &w.blake3_absorb_ops { + out.push(("absorb control", o.ctrl_addr, 32)); + out.push(("absorb message", o.msg_addr, 64 * o.blocks.len() as u64)); + } + out +} + +/// A program's ELF and logs: an assembly test program, or the Rust guest +/// `hint_min` built by `make compile-programs-rust`. +fn program_of(name: &str) -> (Elf, Vec) { + if name != "hint_min" { + return run(name); + } + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("the workspace root") + .join("executor/program_artifacts/rust/hint_min.elf"); + let bytes = std::fs::read(&path) + .unwrap_or_else(|e| panic!("{} — run `make compile-programs-rust`: {e}", path.display())); + let program = Elf::load(&bytes).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor") + .run() + .expect("the program runs") + .logs; + (program, logs) +} + +/// ★ Every value-only reader goes through the reads that mark a byte the +/// window's state does not hold: for each kind, an entry inside that reader's +/// bytes, dropped, is refused before anything is computed from it (no panic, +/// no tables). +#[test] +fn a_dropped_entry_a_value_only_reader_reads_is_refused() { + let mut seen = std::collections::BTreeMap::<&str, usize>::new(); + for name in [ + "hint_min", + "test_ecsm", + "test_keccak_multi", + "test_blake3", + "test_blake3_absorb", + "test_commit_split", + "test_keccak256", + ] { + let (program, logs) = program_of(name); + let reference = whole(&program, &logs, &MaxRowsConfig::small()); + let window = if logs.len() < 5_000 { + 7 + } else { + (logs.len() / 40) | 1 + }; + let mut builder = + WindowedTraceBuilder::new(&program, &[], &MaxRowsConfig::small()).expect("builder"); + let mut pass = + LeanPass::new(Arc::clone(&builder.decode), &builder.register_state).expect("pass"); + let mut executor = Executor::new(&program, Vec::new()).expect("the executor"); + let mut windows = Vec::new(); + while let Some(logs) = executor + .resume_with_limit(window) + .expect("the run") + .map(<[Log]>::to_vec) + { + windows.push(logs); + } + windows.pop(); + let (mut walker, _) = builder.split(); + let mut first = 0; + for (k, logs) in windows.iter().enumerate() { + pass.begin(k as u32).expect("a window"); + pass.pass(logs, first).expect("the pass"); + first += logs.len(); + let entries = pass.take_entries(); + let walked = walker.walk(logs).expect("the walk"); + for (reader, from, len) in value_reads(&walked) { + let Some(at) = entries + .iter() + .position(|&(a, _)| a.wrapping_sub(from) < len) + else { + continue; + }; + if seen.get(reader).copied().unwrap_or(0) >= 3 { + continue; + } + let tamper = Tamper::DropEntry { + window: k as u32, + at, + }; + let how = caught(&program, &reference, window, tamper); + assert_eq!( + how, "refused", + "{name}: window {k}, a {reader} byte dropped" + ); + *seen.entry(reader).or_default() += 1; + } + } + } + println!("refused: {seen:?}"); + for reader in [ + "HINT input", + "ECSM xG", + "ECSM k", + "KECCAK lanes", + "KECCAK256 block", + "BLAKE3 state", + "absorb control", + "absorb message", + ] { + assert!( + seen.contains_key(reader), + "no {reader} byte dropped: {seen:?}" + ); + } +} + +/// ★ The state built from the pass for the finish is the walk's at the last +/// window's start: the same pages, the same set cells, each with the walk's +/// value and timestamp. +#[test] +fn the_state_built_for_the_finish_is_the_walks() { + for name in PROGRAMS { + let (program, logs) = run(name); + for window in [7, 33, logs.len()] { + let mut builder = + WindowedTraceBuilder::new(&program, &[], &MaxRowsConfig::small()).expect("builder"); + let mut pass = + LeanPass::new(Arc::clone(&builder.decode), &builder.register_state).expect("pass"); + let mut executor = Executor::new(&program, Vec::new()).expect("the executor"); + let mut start = executor.memory().snapshot(); + let mut windows = Vec::new(); + while let Some(logs) = executor + .resume_with_limit(window) + .expect("the run") + .map(<[Log]>::to_vec) + { + let next = executor.memory().snapshot(); + windows.push((logs, std::mem::replace(&mut start, next))); + } + let (_, last_start) = windows.pop().expect("a window"); + { + let (mut walker, _) = builder.split(); + let mut first = 0; + for (k, (logs, _)) in windows.iter().enumerate() { + pass.begin(k as u32).expect("a window"); + pass.pass(logs, first).expect("the pass"); + walker.walk(logs).expect("the walk"); + first += logs.len(); + } + } + let built = pass.walked_state(&last_start, &builder.image, builder.memory_state.lean); + let walked = &builder.memory_state; + assert!( + built.cells.page_bases().eq(walked.cells.page_bases()), + "{name}/{window}: pages" + ); + assert!( + built.cells.iter().eq(walked.cells.iter()), + "{name}/{window}: set cells, values and timestamps" + ); + let regs = pass.registers(); + let w = &builder.register_state; + assert!( + regs.regs == w.regs + && regs.index_register == w.index_register + && regs.pc_register == w.pc_register, + "{name}/{window}: registers" + ); + } + } +} + +/// The run of `program` built by windows of `window` cycles walked on +/// `walkers` threads ([`walk_windows`]), the executor on a thread of its own as +/// the block's prove runs it; with `cut`, the executor sends window `cut` +/// without its snapshot. +fn walked_on_threads( + program: &Elf, + max_rows: &MaxRowsConfig, + window: usize, + walkers: usize, + cut: Option, +) -> Result { + let mut builder = WindowedTraceBuilder::new(program, &[], max_rows)?.drop_streamed_ops()?; + let (tx, rx) = std::sync::mpsc::sync_channel(2); + let mut chunks = Vec::new(); + let report = std::thread::scope(|scope| { + scope.spawn(move || { + let mut executor = Executor::new(program, Vec::new()).expect("the executor"); + let mut start = Some(executor.memory().snapshot()); + let mut k = 0; + while let Some(logs) = executor + .resume_with_limit(window) + .expect("the run") + .map(<[Log]>::to_vec) + { + let next = Some(executor.memory().snapshot()); + let mut at = std::mem::replace(&mut start, next); + if cut == Some(k) { + at = None; + } + if tx.send((logs, at)).is_err() { + break; + } + k += 1; + } + }); + walk_windows(&mut builder, rx, walkers, |accumulator, walked, _| { + chunks.extend( + accumulator + .absorb(walked) + .into_iter() + .map(ChunkJob::generate), + ); + Ok(()) + }) + })?; + let mut traces = builder.finish(&report.last)?; + traces.insert_streamed(chunks)?; + Ok(traces) +} + +/// ★ Walked on 1, 2, 3 and 8 threads at window lengths 1, 7, 32 and 33, the +/// windows build the whole-run tables. +#[test] +fn windows_walked_on_threads_build_the_whole_run_tables() { + let max_rows = MaxRowsConfig::small(); + for name in [ + "all_instructions_64", + "test_keccak_multi", + "test_ecsm", + "test_commit_split", + "test_copy_ecalls", + "test_copy_overlaps", + ] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [1, 7, 32, 33] { + for walkers in [1, 2, 3, 8] { + let traces = walked_on_threads(&program, &max_rows, window, walkers, None) + .unwrap_or_else(|e| { + panic!("{name}, window {window}, {walkers} walkers: {e:?}") + }); + same_traces(&reference, &traces); + } + } + } +} + +/// A window the executor sends without its memory stops every thread and is +/// refused, wherever it falls; so is an error from the absorbing side. +#[test] +fn an_error_anywhere_stops_the_walk_and_is_returned() { + let max_rows = MaxRowsConfig::small(); + let (program, _) = run("test_keccak_multi"); + for cut in [0, 1, 5] { + for walkers in [1, 3] { + let built = walked_on_threads(&program, &max_rows, 7, walkers, Some(cut)); + assert!( + matches!(&built, Err(Error::Prover(m)) if m.contains("without its memory")), + "window {cut} cut, {walkers} walkers: {:?}", + built.err() + ); + } + } + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows).expect("the builder"); + let (tx, rx) = std::sync::mpsc::sync_channel(2); + let walked = std::thread::scope(|scope| { + scope.spawn(move || { + let mut executor = Executor::new(&program, Vec::new()).expect("the executor"); + let mut start = Some(executor.memory().snapshot()); + while let Some(logs) = executor + .resume_with_limit(7) + .expect("the run") + .map(<[Log]>::to_vec) + { + let next = Some(executor.memory().snapshot()); + if tx + .send((logs, std::mem::replace(&mut start, next))) + .is_err() + { + break; + } + } + }); + let mut absorbed = 0; + walk_windows(&mut builder, rx, 3, |_, _, _| { + absorbed += 1; + if absorbed == 4 { + return Err(Error::Prover("the absorbing side stopped".into())); + } + Ok(()) + }) + }); + assert!( + matches!(&walked, Err(Error::Prover(m)) if m.contains("absorbing side stopped")), + "{:?}", + walked.err() + ); +} + +/// The run of program `name` by windows of 7 cycles on `walkers` threads, +/// each job numbered `number(k)`, on a thread of its own: what it returns, or +/// `None` if it has not returned within 30 s. +fn numbered_on_a_thread( + name: &'static str, + walkers: usize, + number: fn(u32) -> u32, +) -> Option> { + let (out_tx, out_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let (program, _) = run(name); + let max_rows = MaxRowsConfig::small(); + let walked = WindowedTraceBuilder::new(&program, &[], &max_rows).and_then(|mut builder| { + let (tx, rx) = std::sync::mpsc::sync_channel(2); + std::thread::scope(|scope| { + let program = &program; + scope.spawn(move || { + let mut executor = Executor::new(program, Vec::new()).expect("the executor"); + let mut start = Some(executor.memory().snapshot()); + while let Some(logs) = executor + .resume_with_limit(7) + .expect("the run") + .map(<[Log]>::to_vec) + { + let next = Some(executor.memory().snapshot()); + if tx + .send((logs, std::mem::replace(&mut start, next))) + .is_err() + { + break; + } + } + }); + walk_numbered(&mut builder, rx, walkers, number, |_, _, _| Ok(())) + }) + .map(|_| ()) + }); + let _ = out_tx.send(walked); + }); + out_rx.recv_timeout(std::time::Duration::from_secs(30)).ok() +} + +/// A window number the pass never hands out is refused, not waited for +/// forever; one handed out twice is refused too. +#[test] +fn a_window_never_walked_or_walked_twice_is_refused() { + let name = "test_keccak_multi"; + // Enough windows to fill every permit with window 1 missing, at 4 walkers. + let (_, logs) = run(name); + assert!(logs.len() > 7 * 12, "{} cycles", logs.len()); + for walkers in [1, 2, 4] { + let skipped = numbered_on_a_thread(name, walkers, |k| k + (k > 0) as u32); + assert!( + matches!(&skipped, Some(Err(Error::Prover(m))) if m == "window 1 was never walked"), + "{walkers} walkers, window 1 skipped: {skipped:?}" + ); + let repeated = numbered_on_a_thread(name, walkers, |k| if k == 2 { 1 } else { k }); + assert!( + matches!(&repeated, Some(Err(Error::Prover(m))) if m == "window 1 was walked twice"), + "{walkers} walkers, window 1 repeated: {repeated:?}" + ); + } +} diff --git a/prover/src/tables/trace_builder/windowed/regen_tests.rs b/prover/src/tables/trace_builder/windowed/regen_tests.rs new file mode 100644 index 000000000..bb9bfa87b --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/regen_tests.rs @@ -0,0 +1,295 @@ +//! Regeneration's pieces of the windowed builder (D-REGEN, D-WHIR-NODISK §2.1): +//! the regeneration walk and the regenerator. + +use std::collections::BTreeMap; + +use executor::elf::Elf; +use executor::vm::execution::Executor; + +use super::{RegenBuilder, StreamTable, WindowedTraceBuilder}; +use crate::tables::MaxRowsConfig; +use crate::tables::gpack::TraceForm; +use crate::test_utils::asm_elf_bytes; + +/// The asm programs the regeneration tests run: every walk path (loads and +/// stores of every width, word instructions, COMMIT, KECCAK, BLAKE3 and its +/// absorb, ECSM). +const PROGRAMS: [&str; 9] = [ + "all_instructions_64", + "lw_sw_offset_odd", + "test_memw_split_ts", + "test_keccak_multi", + "test_keccak256", + "test_blake3", + "test_blake3_absorb", + "test_commit_split", + "test_ecsm", +]; + +fn run_logs(program: &Elf) -> Vec { + Executor::new(program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs +} + +/// The streamed lists of a walk, printed field by field. +fn streamed_lists(cpu_ops: &[super::super::CpuOperation], walk: &super::WalkOutputs) -> String { + let memw = &walk.memw; + format!( + "cpu {cpu_ops:?}\nmemw_r {:?}\nmemw_a {:?}\nmemw {:?}\nload {:?}\nlt {:?}\nshift {:?}", + memw.register_rows, memw.aligned, memw.general, walk.load_ops, walk.lt_ops, walk.shift_ops + ) +} + +/// ★ The regeneration walk is the walk on the streamed tables' lists: over +/// windows of 1, 7 and 33 cycles it emits the CPU, MEMW_R, MEMW_A, MEMW, LOAD, +/// LT and SHIFT ops the full walk emits (field by field), leaves every other +/// list empty, and ends in the same memory and register state. +#[test] +fn the_regeneration_walk_is_the_walk_on_its_streamed_lists() { + use super::super::{ + DecodeArtifacts, MemoryState, RegisterState, WalkOutputs, build_initial_image, + collect_cpu_ops_into, collect_ops_from_cpu_into, collect_streamed_ops_from_cpu_into, + register, + }; + let mut rest_seen = 0; + for name in PROGRAMS { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = run_logs(&program); + let artifacts = DecodeArtifacts::from_elf(&program).expect("the decode artifacts"); + let image = build_initial_image(&program, &[]); + let register_init = register::register_init_from_entry_point(program.entry_point); + for window in [1, 7, 33] { + let mut states = [ + ( + MemoryState::from_image(&image), + RegisterState::from_init(®ister_init), + ), + ( + MemoryState::from_image(&image), + RegisterState::from_init(®ister_init), + ), + ]; + for (memory, _) in states.iter_mut() { + memory.lean = true; + } + let mut cycles = 0; + for (k, part) in logs.chunks(window).enumerate() { + let mut cpu_ops = Vec::new(); + collect_cpu_ops_into(part, &artifacts.decode, cycles, &mut cpu_ops) + .expect("the CPU ops"); + cycles += part.len(); + let [(full_mem, full_reg), (lite_mem, lite_reg)] = &mut states; + let mut full = WalkOutputs::for_walk(cpu_ops.len(), false); + collect_ops_from_cpu_into( + &cpu_ops, + &artifacts.decode, + full_mem, + full_reg, + &mut full, + false, + ) + .expect("the walk"); + let mut lite = WalkOutputs::for_walk(cpu_ops.len(), false); + collect_streamed_ops_from_cpu_into( + &cpu_ops, + &artifacts.decode, + lite_mem, + lite_reg, + &mut lite, + ) + .expect("the walk"); + assert_eq!( + streamed_lists(&cpu_ops, &full), + streamed_lists(&cpu_ops, &lite), + "{name}, windows of {window}, window {k}" + ); + let rest = |w: &WalkOutputs| { + w.bitwise_ops.len() + + w.copy_ops.len() + + w.keccak_ops.len() + + w.blake3_ops.len() + + w.blake3_absorb_ops.len() + + w.cpu32_ops.len() + + w.ecsm_ops.len() + + w.ecdas_ops.len() + + w.hint_ops.len() + }; + assert_eq!( + rest(&lite), + 0, + "{name}: the regeneration walk listed a finish's op" + ); + rest_seen += rest(&full); + } + let [(full_mem, full_reg), (lite_mem, lite_reg)] = &states; + let cells = |m: &MemoryState| m.cells.iter().collect::>(); + assert!( + cells(full_mem) == cells(lite_mem), + "{name}, windows of {window}: memory" + ); + let registers = |r: &RegisterState| (r.regs, r.index_register, r.pc_register); + assert_eq!( + registers(full_reg), + registers(lite_reg), + "{name}, windows of {window}: registers" + ); + } + } + assert!(rest_seen > 0, "no program made an op only the finish reads"); +} + +/// A chunk's place in [`StreamTable::ALL`] and its index. +type Key = (usize, usize); + +fn slot(table: StreamTable) -> usize { + StreamTable::ALL + .iter() + .position(|&t| t == table) + .expect("a streamed table") +} + +/// The spill store's digest of a chunk generated in `form` and held packed. +fn packed_digest(job: super::ChunkJob, form: TraceForm) -> [u64; 2] { + let mut trace = job.generate_as(form).trace; + assert!(trace.pack_main_narrow(), "the chunk packs"); + stark::regen::digest_of(trace.narrow_main().expect("packed")) +} + +/// Each phase-A chunk's packed digest, in hand-out order: a windowed build +/// that drops its streamed ops, walked `window` cycles at a time (the run's +/// last window left to the finish), every handed-out chunk generated 64-bit +/// and packed. +fn handed_out_digests( + program: &Elf, + logs: &[executor::vm::logs::Log], + max_rows: &MaxRowsConfig, + window: usize, +) -> Vec<(Key, [u64; 2])> { + let mut builder = WindowedTraceBuilder::new(program, &[], max_rows) + .expect("the builder") + .drop_streamed_ops() + .expect("before the first window"); + let windows: Vec<&[executor::vm::logs::Log]> = logs.chunks(window).collect(); + let mut digests = Vec::new(); + for w in &windows[..windows.len() - 1] { + for job in builder.push_jobs(w).expect("a window") { + let key = (slot(job.table), job.index); + digests.push((key, packed_digest(job, TraceForm::Wide))); + } + } + digests +} + +/// Each chunk a regenerator cuts, walked `window` cycles at a time over the +/// whole run, in the order it cut them, generated packed (G-pack). +fn regenerated_digests( + builder: RegenBuilder, + logs: &[executor::vm::logs::Log], + window: usize, +) -> Vec<(Key, [u64; 2])> { + let mut builder = builder; + let mut digests = Vec::new(); + for w in logs.chunks(window) { + for job in builder.push(w).expect("a window") { + let key = (slot(job.table), job.index); + digests.push((key, packed_digest(job, TraceForm::Narrow))); + } + } + assert_eq!(builder.cycles(), logs.len()); + digests +} + +/// ★ The regenerator rebuilds every chunk phase A handed out, byte for byte, +/// and on phase A's windows in phase A's hand-out order: on every program, +/// phase A walked 5 cycles at a time and the regenerator 1, 5, 7 and 33 at a +/// time, each phase-A chunk's packed digest is the regenerated one; at 5 the +/// regenerator's chunks start with phase A's, in its order; the others are +/// the finish's (past phase A's). +#[test] +fn the_regenerator_rebuilds_the_handed_out_chunks() { + let max_rows = MaxRowsConfig::small(); + let mut handed = 0; + for name in PROGRAMS { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = run_logs(&program); + let phase_a = handed_out_digests(&program, &logs, &max_rows, 5); + handed += phase_a.len(); + let by_key: BTreeMap = phase_a.iter().copied().collect(); + assert_eq!( + by_key.len(), + phase_a.len(), + "{name}: a chunk handed out twice" + ); + let mut last = [None; 9]; + for &(t, i) in by_key.keys() { + last[t] = Some(last[t].map_or(i, |l: usize| l.max(i))); + } + for window in [1, 5, 7, 33] { + let regen = RegenBuilder::new(&program, &[], &max_rows).expect("the regenerator"); + let again = regenerated_digests(regen, &logs, window); + let again_by_key: BTreeMap = again.iter().copied().collect(); + for (key, digest) in &by_key { + assert_eq!( + again_by_key.get(key), + Some(digest), + "{name}, windows of {window}: chunk {key:?}" + ); + } + for &(t, i) in again_by_key.keys() { + assert!( + by_key.contains_key(&(t, i)) || last[t].is_none_or(|l| i > l), + "{name}, windows of {window}: chunk ({t}, {i}) is not phase A's and not past it" + ); + } + if window == 5 { + assert_eq!( + &again[..phase_a.len()], + &phase_a[..], + "{name}: on phase A's windows, phase A's hand-out order" + ); + } + } + } + assert!(handed > 0, "no program handed out a chunk"); +} + +/// ★ A slicer that drops one op is caught: for each streamed table phase A +/// handed out a chunk of, the regenerator dropping that table's first op +/// rebuilds a different chunk at the first index, and leaves the other tables' +/// chunks as they were. +#[test] +fn a_slicer_that_drops_an_op_is_caught() { + let max_rows = MaxRowsConfig::small(); + let mut caught = 0; + for name in ["all_instructions_64", "test_keccak_multi"] { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = run_logs(&program); + let phase_a: BTreeMap = handed_out_digests(&program, &logs, &max_rows, 7) + .into_iter() + .collect(); + for table in StreamTable::ALL { + let t = slot(table); + if !phase_a.contains_key(&(t, 0)) { + continue; + } + let regen = RegenBuilder::new(&program, &[], &max_rows) + .expect("the regenerator") + .drop_one_op(table); + let again: BTreeMap = + regenerated_digests(regen, &logs, 7).into_iter().collect(); + assert_ne!( + again.get(&(t, 0)), + phase_a.get(&(t, 0)), + "{name}: {table:?} chunk 0 with an op dropped" + ); + for (key, digest) in phase_a.iter().filter(|(k, _)| k.0 != t) { + assert_eq!(again.get(key), Some(digest), "{name}: {key:?}"); + } + caught += 1; + } + } + assert!(caught >= 4, "only {caught} tables checked"); +} diff --git a/prover/src/tables/trace_builder/windowed/tail_tests.rs b/prover/src/tables/trace_builder/windowed/tail_tests.rs new file mode 100644 index 000000000..90de31240 --- /dev/null +++ b/prover/src/tables/trace_builder/windowed/tail_tests.rs @@ -0,0 +1,75 @@ +//! The hand-out's two ways of taking a chunk out of a tail: as the window +//! parts it lies in ([`Tail::take_segments`], what [`Tail::take`] does unless +//! `LAMBDA_VM_HANDOUT_COPY=1`) and copied into one list ([`Tail::take_copy`]) +//! give the same ops, and so does what stays behind. + +use super::{Segments, Tail}; + +/// A fixed xorshift stream. +struct Rng(u64); + +impl Rng { + fn below(&mut self, n: u64) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x % n + } +} + +fn ops(segments: &Segments) -> Vec { + segments.iter().copied().collect() +} + +/// Parts of every length (empty ones too) pushed between takes of every size, +/// some a part exactly, some across many parts: the parts' chunks are the +/// copies, the reversed walk over them is the reversed copy, and the tails left +/// behind are equal. +#[test] +fn a_chunk_taken_as_parts_is_the_copied_chunk() { + for seed in 1..=40u64 { + let mut rng = Rng(seed.wrapping_mul(0x9e37_79b9_7f4a_7c15)); + let (mut shared, mut copied) = (Tail::new(), Tail::new()); + let mut next = 0u64; + for _ in 0..60 { + for _ in 0..rng.below(3) { + let len = [0, 1, 7, 8, 9, 64][rng.below(6) as usize] + rng.below(40); + let part: Vec = (next..next + len).collect(); + next += len; + shared.push(part.clone()); + copied.push(part); + } + let n = [1, 8, 33, 64, 200][rng.below(5) as usize]; + while shared.len >= n && rng.below(4) != 0 { + let parts = shared.take_segments(n); + let copy = copied.take_copy(n); + assert_eq!(ops(&parts), copy, "seed {seed}"); + let reversed: Vec = parts.iter().rev().copied().collect(); + assert!(reversed.iter().eq(copy.iter().rev()), "seed {seed}"); + assert_eq!(parts.slices().iter().map(|s| s.len()).sum::(), n); + assert_eq!(shared.held(), copied.held(), "seed {seed}"); + } + } + assert_eq!(shared.into_vec(), copied.into_vec(), "seed {seed}"); + } +} + +/// A part a chunk uses only the front of stays shared with the tail; once the +/// next chunk uses up the rest, the tail no longer holds it. +#[test] +fn a_straddling_part_is_shared_until_used_up() { + let mut tail = Tail::new(); + tail.push((0..10u64).collect()); + let first = tail.take_segments(4); + assert_eq!(ops(&first), vec![0, 1, 2, 3]); + assert_eq!(tail.parts.len(), 1, "the part stays in the tail"); + let second = tail.take_segments(6); + assert_eq!(ops(&second), (4..10).collect::>()); + assert!(tail.parts.is_empty(), "the part left with the last chunk"); + assert!(std::sync::Arc::ptr_eq( + &first.parts[0].0, + &second.parts[0].0 + )); +} diff --git a/prover/src/tables/trace_hash.rs b/prover/src/tables/trace_hash.rs new file mode 100644 index 000000000..1e77cdb16 --- /dev/null +++ b/prover/src/tables/trace_hash.rs @@ -0,0 +1,57 @@ +//! The hash state of the tables that deduplicate their operations through a +//! `HashMap` and lay their rows out in its iteration order: LT, BYTEWISE, +//! BRANCH, MUL and DVRM (EQ keeps one row an operation since main's +//! multiplicity fix). +//! +//! Every such map in the process hashes with one state, so a table's row order +//! is a function of its operation list: two builds of the same run in one +//! process lay out the same rows in the same order, which regenerating a +//! streamed chunk needs. The keys are random per process, as `HashMap::new()`'s +//! are, unless `LAMBDA_VM_FIXED_TRACE_HASH=1` fixes them (SipHash-1-3 with zero +//! keys): then the traces, and under `LAMBDA_VM_DETERMINISTIC_GRIND` the proofs, +//! are the same in every process. Fixed keys let a program that knows them +//! steer many operations into one bucket, so they are a measurement setting. + +use std::collections::hash_map::{DefaultHasher, RandomState}; +use std::hash::BuildHasher; +use std::sync::OnceLock; + +/// See the module docs: `None` is the fixed state. +#[derive(Clone, Debug)] +pub struct TraceHashState(Option); + +impl TraceHashState { + /// SipHash-1-3 with zero keys: the same hashes in every process. + pub fn fixed() -> Self { + Self(None) + } +} + +impl BuildHasher for TraceHashState { + type Hasher = DefaultHasher; + + fn build_hasher(&self) -> DefaultHasher { + match &self.0 { + Some(state) => state.build_hasher(), + None => DefaultHasher::new(), + } + } +} + +/// The process's state (see the module docs), read from +/// `LAMBDA_VM_FIXED_TRACE_HASH` once. +pub fn trace_hash_state() -> TraceHashState { + static STATE: OnceLock = OnceLock::new(); + STATE + .get_or_init(|| { + if std::env::var("LAMBDA_VM_FIXED_TRACE_HASH").is_ok_and(|v| v.trim() == "1") { + TraceHashState::fixed() + } else { + TraceHashState(Some(RandomState::new())) + } + }) + .clone() +} + +/// A deduplicating table's operation map, hashed under [`trace_hash_state`]. +pub type OpMap = std::collections::HashMap; diff --git a/prover/src/tables/types.rs b/prover/src/tables/types.rs index 755bc23a6..081cc71a4 100644 --- a/prover/src/tables/types.rs +++ b/prover/src/tables/types.rs @@ -353,13 +353,6 @@ pub enum BusId { /// and sends Bit[ts, idx_k] for the MSB (mult = μ). Bit = 30, - // ========================================================================= - // Copy accelerator - // ========================================================================= - // ID 29 is unused (32 is IS_WH, below). The copy primitive chains over a single - // [`BusId::CopyNext`] and carries the functionality selectors inside the tuple, - // rather than giving each of its three operations a bus of its own. - // ========================================================================= // Continuations // ========================================================================= @@ -368,13 +361,26 @@ pub enum BusId { GlobalMemory = 31, // ========================================================================= - // Address range checks + // LFM — the Lambda Field Machine (recursion machine; `lfm` module) // ========================================================================= - /// `IS_WH[Word, Half]`: range check that the top 48 bits of a `DWordWHH` - /// are a valid `(Word, Half)` pair. Callers send it through the `IS_WHH` - /// template; the IS_WH table receives it. ID 29 (the removed EC_SCALAR - /// `ServeK` bus) stays unused. - IsWh = 32, + /// LFM write-once memory: token `(addr, v0..v3)`. Writes send with the + /// preprocessed static read count; reads receive gated by is_real. + LfmMem = 32, + /// LFM 16-bit range lookup (the `LFM_RANGE` fixed table). + LfmRange = 33, + /// LFM public values: token `(index, v0..v3)`; closed by a + /// consumer-computed balance (the COMMIT-bus pattern). + LfmPublic = 34, + + // ========================================================================= + // BLAKE3 chained absorb + // ========================================================================= + /// BLAKE3 absorb self-referencing chain (row N → row N+1 of one absorb + /// group): `(timestamp, remaining, msg_base, ctrl_addr, cv[0..8])`. + /// + /// ID 35 rather than 29: 29 is a gap left by a removed bus and reusing it + /// would make a stale artifact decode as this one. + Blake3Absorb = 35, // ========================================================================= // Unified copy primitive @@ -385,9 +391,30 @@ pub enum BusId { /// previous one. The functionality selectors travel inside the tuple, so a chain /// cannot change operation half way through it — a guarantee that separating the /// operations by bus id would otherwise have to carry structurally. - CopyNext = 33, - // ID 34 is unused. COPY receives the `write` ECALL and reads its operand - // registers itself, so nothing defers the commit to a second chip. + /// + /// ID 36: 32–35 are the LFM and BLAKE3 absorb buses, and 20 and 29 are gaps + /// left by removed buses, which a stale artifact would decode as this one. + CopyNext = 36, + + // ========================================================================= + // Address range checks + // ========================================================================= + /// `IS_WH[Word, Half]`: range check that the top 48 bits of a `DWordWHH` + /// are a valid `(Word, Half)` pair. Callers send it through the `IS_WHH` + /// template; the IS_WH table receives it. Main numbers it 32, which the LFM + /// buses hold here; 37 is the next free ID (20 and 29 stay unused). + IsWh = 37, + + // ========================================================================= + // Keccak-256 of a buffer + // ========================================================================= + /// KECCAK's sponge-mode chain: block `j` of a `keccak256` call sends + /// `(T, j + 1, count − 136, src + 136, src_hi, out_lo, out_hi, state[200])` to + /// the row of block `j + 1`, which receives it as its own `(T, j, count, src, + /// …, previous state)`. + /// + /// ID 38: the next free one (20 and 29 stay unused). + KeccakSponge = 38, } impl BusId { @@ -408,6 +435,7 @@ impl BusId { BusId::Ecall => "Ecall", BusId::Commit => "Commit", BusId::CopyNext => "CopyNext", + BusId::KeccakSponge => "KeccakSponge", BusId::Keccak => "Keccak", BusId::KeccakRc => "KeccakRc", BusId::ByteAlu => "ByteAlu", @@ -417,6 +445,10 @@ impl BusId { BusId::Ecdas => "Ecdas", BusId::Bit => "Bit", BusId::GlobalMemory => "GlobalMemory", + BusId::LfmMem => "LfmMem", + BusId::LfmRange => "LfmRange", + BusId::LfmPublic => "LfmPublic", + BusId::Blake3Absorb => "Blake3Absorb", BusId::IsWh => "IsWh", } } @@ -449,8 +481,13 @@ impl TryFrom for BusId { 28 => Ok(BusId::Ecdas), 30 => Ok(BusId::Bit), 31 => Ok(BusId::GlobalMemory), - 32 => Ok(BusId::IsWh), - 33 => Ok(BusId::CopyNext), + 32 => Ok(BusId::LfmMem), + 33 => Ok(BusId::LfmRange), + 34 => Ok(BusId::LfmPublic), + 35 => Ok(BusId::Blake3Absorb), + 36 => Ok(BusId::CopyNext), + 38 => Ok(BusId::KeccakSponge), + 37 => Ok(BusId::IsWh), other => Err(other), } } @@ -531,8 +568,22 @@ pub const MU_MAX: u64 = u8::MAX as u64; /// several rows when its count exceeds [`MU_MAX`]. Shared by trace generation /// and the BITWISE collectors of LT, BYTEWISE and BRANCH so the per-row lookups /// they count cannot drift apart. +/// +/// The map hashes with the process's one state (`tables::trace_hash`), so the +/// rows come out in one order for one operation list: a streamed chunk built +/// again is the chunk it was. pub fn dedup_rows(operations: &[T]) -> Vec<(T, u64)> { - let mut op_map: std::collections::HashMap = std::collections::HashMap::new(); + dedup_rows_of(operations.iter()) +} + +/// [`dedup_rows`] over the operations an iterator yields, one after another (a +/// chunk handed out as the window parts it lies in). +pub fn dedup_rows_of<'a, T: std::hash::Hash + Eq + Clone + 'a>( + operations: impl Iterator, +) -> Vec<(T, u64)> { + use super::trace_hash::{OpMap, trace_hash_state}; + + let mut op_map: OpMap = OpMap::with_hasher(trace_hash_state()); for op in operations { *op_map.entry(op.clone()).or_insert(0) += 1; } diff --git a/prover/src/test_utils.rs b/prover/src/test_utils.rs index 7e6c493e6..9a0cc0724 100644 --- a/prover/src/test_utils.rs +++ b/prover/src/test_utils.rs @@ -26,11 +26,9 @@ use stark::domain::Domain; use stark::lookup::{ AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, NullBoundaryConstraintBuilder, }; -use stark::proof::options::ProofOptions; +use stark::proof::options::{ProofFormat, ProofOptions}; use stark::proof::stark::MultiProof; -use stark::prover::{IsStarkProver, Prover, ProvingError}; -#[cfg(feature = "disk-spill")] -use stark::storage_mode::StorageMode; +use stark::prover::{IsStarkProver, ProvingError}; use stark::trace::TraceTable; use stark::traits::AIR; @@ -39,6 +37,9 @@ use crate::tables::bitwise::{ BitwiseOperation, BitwiseOperationType, bus_interactions as bitwise_bus_interactions, cols as bitwise_cols, }; +use crate::tables::blake3::{ + Blake3Constraints, bus_interactions as blake3_bus_interactions, cols as blake3_cols, +}; use crate::tables::branch::{ BranchConstraints, bus_interactions as branch_bus_interactions, cols as branch_cols, }; @@ -51,13 +52,18 @@ use crate::tables::cpu::{ use crate::tables::cpu32::{ Cpu32Constraints, bus_interactions as cpu32_bus_interactions, cols as cpu32_cols, }; -use crate::tables::decode::{bus_interactions as decode_bus_interactions, cols as decode_cols}; +use crate::tables::decode::{ + DecodeTable, bus_interactions as decode_bus_interactions, cols as decode_cols, +}; use crate::tables::dvrm::{ DvrmConstraints, bus_interactions as dvrm_bus_interactions, cols as dvrm_cols, }; use crate::tables::ecdas::{ EcdasConstraints, bus_interactions as ecdas_bus_interactions, cols as ecdas_cols, }; +use crate::tables::ecrec::{ + EcrecConstraints, bus_interactions as ecrec_bus_interactions, cols as ecrec_cols, +}; use crate::tables::ecsm::{ EcsmConstraints, bus_interactions as ecsm_bus_interactions, cols as ecsm_cols, }; @@ -136,11 +142,14 @@ pub fn multi_prove_ram( where PI: Send + Sync + Clone, { - Prover::::multi_prove( + // ★ The block path's PIN, not `stark`'s default alias. A test helper that + // proves through the alias while the pin names another hash produces a green + // suite that means nothing: the prover's recomputed preprocessed root then + // disagrees with the AIR's and it fails as `PrecomputedCommitmentMismatch`. + crate::hash_pin::BlockProver::::multi_prove( air_trace_pairs, transcript, - #[cfg(feature = "disk-spill")] - StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) } @@ -225,26 +234,28 @@ pub fn is_halfword_sender_columns(interactions: &[BusInteraction]) -> Vec /// REGISTER commitment for a monolithic proof of `elf` that committed /// `public_output`: FINI pinned to the halted state, as the production prover and /// verifier build it. -pub fn halted_register_commitment( +#[cfg(test)] +pub(crate) fn halted_register_commitment( options: &ProofOptions, elf: &Elf, public_output: &[u8], -) -> stark::config::Commitment { +) -> crate::RegisterFiles { let committed_bytes = u32::try_from(public_output.len()).expect("public output fits in u32"); - crate::tables::register::halted_commitment(options, elf.entry_point, committed_bytes) + crate::RegisterFiles::halted(options, elf.entry_point, committed_bytes) } /// REGISTER commitment matching `traces`: INIT `init` and FINI as the REGISTER /// trace holds it. This is what a prover commits for its own trace (a non-final /// continuation epoch does it honestly; a forged run gets a root the halted /// verifier does not recompute). -pub fn trace_register_commitment( +#[cfg(test)] +pub(crate) fn trace_register_commitment( options: &ProofOptions, init: &[u32], traces: &crate::tables::trace_builder::Traces, -) -> stark::config::Commitment { +) -> crate::RegisterFiles { let fini = crate::tables::register::fini_from_trace(&traces.register); - crate::tables::register::compute_precomputed_commitment(options, init, &fini) + crate::RegisterFiles::new(options, init.to_vec(), fini) } // ============================================================================= @@ -293,12 +304,13 @@ pub fn collect_bitwise_ops_from_logs( logs: &[Log], instructions: &U64HashMap, ) -> Vec { + let decode = DecodeTable::from_instructions(instructions); logs.iter() .enumerate() .flat_map(|(i, log)| { - let instruction = *instructions.get(&log.current_pc).unwrap(); - let op = CpuOperation::from_log_and_instruction(log, (i as u64) * 4, instruction); - op.collect_bitwise_ops() + let row = decode.row(log.current_pc).unwrap(); + let op = CpuOperation::from_log(log, (i as u64) * 4, decode.entry(row), row); + decode.op(&op).collect_bitwise_ops() }) .collect() } @@ -436,9 +448,9 @@ pub fn collect_load_ops_from_logs( let loaded_value = log.dst_val; // Extract individual bytes from loaded value - let mut res_bytes = [0u64; 8]; + let mut res_bytes = [0u8; 8]; for (j, byte) in res_bytes.iter_mut().take(byte_count).enumerate() { - *byte = (loaded_value >> (j * 8)) & 0xFF; + *byte = (loaded_value >> (j * 8)) as u8; } // Sign/zero extend the upper bytes @@ -654,7 +666,12 @@ pub fn generate_minimal_bitwise_trace(ops: &[BitwiseOperation]) -> TraceTable AirProtoKey { + // ⛔ No `..`: every field of `ProofOptions` is part of the key. The format + // was once missing (it was added to `ProofOptions` after this key was + // written), and the first AIR built in a process then fixed the format of + // every later AIR of that name — a verifier asked for the default format + // verified a capped proof with capped AIRs and accepted it. + let ProofOptions { + blowup_factor, + fri_number_of_queries, + coset_offset, + grinding_factor, + fri_final_poly_log_degree, + format, + } = o; ( name.to_string(), - o.blowup_factor, - o.fri_number_of_queries, - o.coset_offset, - o.grinding_factor, - o.fri_final_poly_log_degree, + *blowup_factor, + *fri_number_of_queries, + *coset_offset, + *grinding_factor, + *fri_final_poly_log_degree, + *format, ) } @@ -1065,6 +1096,18 @@ pub fn create_keccak_air(proof_options: &ProofOptions) -> ConcreteVmAir ConcreteVmAir { + build_air( + blake3_cols::NUM_COLUMNS, + blake3_bus_interactions(), + proof_options, + 1, + Blake3Constraints, + "BLAKE3", + ) +} + /// Create KECCAK_RND AIR with pi constraints and bus interactions. pub fn create_keccak_rnd_air(proof_options: &ProofOptions) -> ConcreteVmAir { build_air( @@ -1112,3 +1155,142 @@ pub fn create_ecdas_air(proof_options: &ProofOptions) -> ConcreteVmAir ConcreteVmAir { + build_air( + ecrec_cols::NUM_COLUMNS, + ecrec_bus_interactions(), + proof_options, + 1, + EcrecConstraints, + "ECREC", + ) +} + +/// A production AIR behind the common trait object, paired with its label. +pub type LabeledAir = ( + &'static str, + Box>, +); + +/// The page base used wherever a test needs a concrete PAGE or GLOBAL_MEMORY +/// AIR. +/// +/// See [`production_airs`] on why a base has to be pinned at all. +pub const PAGE_TEST_BASE: u64 = 0x1000; + +/// The epoch label used wherever a test needs a concrete L2G AIR. A 1-based fini +/// epoch — `l2g_memory_air` debug-asserts `>= 1`. +pub const EPOCH_TEST_LABEL: u64 = 1; + +/// Every production table AIR, constructed and boxed behind the common trait +/// object. +/// +/// AIR-construction only — no ELF, no program execution — so this runs +/// anywhere. There is no ELF-free registry to iterate instead (`VmAirs::n` needs +/// a real ELF plus preprocessed-commitment builds, and omits zero-count tables), +/// so this list is hand-maintained — but exactly once. It previously existed as +/// three hand-copied per-suite lists, and all three shared the same blind spot: +/// the continuation-only tables at the end were in none of them. +/// +/// # This list includes the CONTINUATION tables +/// +/// The last three come from `crate::continuation` and appear in no monolithic +/// proof. They are not optional extras: a continuation proof is what the +/// recursion path actually verifies, so a per-table sweep that stops at the +/// monolithic 25 is complete for a proof shape we do not care about. +/// +/// # Four of these AIRs are PARAMETERIZED +/// +/// `PAGE` and `GLOBAL_MEMORY` fold a `page_base` into constant bus terms; the +/// two `L2G` tables fold an `epoch_label` into constant terms +/// (`BusValue::constant(epoch_label)` and `LinearTerm::Constant(epoch_label-1)` +/// respectively). Those constants reach the captured IR, so each of these four +/// has a DIFFERENT constraint program per parameter value — there is no single +/// "the PAGE program". This list pins one value each +/// ([`PAGE_TEST_BASE`], [`EPOCH_TEST_LABEL`]) so the per-table suites have +/// something concrete to check; anything reasoning about a whole continuation +/// proof must account for one program per distinct base and per distinct epoch. +/// `constraint_artifact_tests::parameterized_airs_vary_per_parameter_value` +/// characterizes exactly how they differ. +pub fn production_airs(proof_options: &ProofOptions) -> Vec { + vec![ + ("CPU", Box::new(create_cpu_air(proof_options, 0))), + ("BITWISE", Box::new(create_bitwise_air(proof_options))), + ("LT", Box::new(create_lt_air(proof_options))), + ("SHIFT", Box::new(create_shift_air(proof_options))), + ("EQ", Box::new(create_eq_air(proof_options))), + ("BYTEWISE", Box::new(create_bytewise_air(proof_options))), + ("STORE", Box::new(create_store_air(proof_options))), + ("CPU32", Box::new(create_cpu32_air(proof_options))), + ("MEMW", Box::new(create_memw_air(proof_options))), + ("MEMW_A", Box::new(create_memw_aligned_air(proof_options))), + ("MEMW_R", Box::new(create_memw_register_air(proof_options))), + ("LOAD", Box::new(create_load_air(proof_options))), + ("DECODE", Box::new(create_decode_air(proof_options))), + ("MUL", Box::new(create_mul_air(proof_options))), + ("DVRM", Box::new(create_dvrm_air(proof_options))), + ("BRANCH", Box::new(create_branch_air(proof_options))), + ("HALT", Box::new(create_halt_air(proof_options))), + ("COPY", Box::new(create_copy_air(proof_options))), + ( + "PAGE", + Box::new(create_page_air(proof_options, PAGE_TEST_BASE)), + ), + ("REGISTER", Box::new(create_register_air(proof_options, 0))), + ("KECCAK", Box::new(create_keccak_air(proof_options))), + ("KECCAK_RND", Box::new(create_keccak_rnd_air(proof_options))), + ("KECCAK_RC", Box::new(create_keccak_rc_air(proof_options))), + ("ECSM", Box::new(create_ecsm_air(proof_options))), + ("ECDAS", Box::new(create_ecdas_air(proof_options))), + ("HINT", Box::new(create_hint_air(proof_options))), + ("ECREC", Box::new(create_ecrec_air(proof_options))), + ("IS_WH", Box::new(create_is_wh_air(proof_options))), + // ---- continuation-only tables (no monolithic proof contains these) ---- + ( + "L2G_GLOBAL", + Box::new(crate::continuation::l2g_global_air( + proof_options, + EPOCH_TEST_LABEL, + )), + ), + ( + "L2G_MEMORY", + Box::new(crate::continuation::l2g_memory_air( + proof_options, + EPOCH_TEST_LABEL, + )), + ), + ( + "GLOBAL_MEMORY", + Box::new(create_global_memory_air(proof_options, PAGE_TEST_BASE)), + ), + ] +} + +/// Create the GLOBAL_MEMORY AIR for one page, as a continuation proof would. +/// +/// The preprocessed commitment is supplied rather than recomputed: it is a +/// blowup-dependent Merkle root over the page's genesis values, and it is NOT +/// part of the constraint artifact (the verifier gets it through the existing +/// static-commitment mechanism). Recomputing it here would cost an FFT plus a +/// Merkle build per call and change nothing any IR suite looks at. +pub fn create_global_memory_air( + proof_options: &ProofOptions, + page_base: u64, +) -> ConcreteVmAir { + let config = crate::tables::page::PageConfig::zero_init(page_base); + crate::continuation::global_memory_air(proof_options, &config, Some([0u8; 32])) +} + +/// The number of production table AIRs [`production_airs`] yields: 28 monolithic +/// plus 3 continuation-only. +/// +/// Every suite that iterates the list asserts against this. That assert is the +/// point: the list is hand-maintained, so without it a table added to +/// `test_utils` or to `continuation.rs` and forgotten here escapes every +/// per-table suite at once and silently — which is exactly what happened to the +/// three continuation tables. Bump it deliberately when a table is genuinely +/// added or removed. +pub const NUM_PRODUCTION_AIRS: usize = 31; diff --git a/prover/src/tests/argue_stage1_tests.rs b/prover/src/tests/argue_stage1_tests.rs new file mode 100644 index 000000000..ad91c89a4 --- /dev/null +++ b/prover/src/tests/argue_stage1_tests.rs @@ -0,0 +1,1199 @@ +//! D-ARGUE's stage 1 on the tables the argue really proves — the VM's +//! (Q1a, `thoughts/zf/gap2/fix2/D-ARGUE.md` §4.1): the base-field +//! precondition, parity of the host reference (`multilinear::fused`) with +//! today's rounds, its negative controls, a census of what each technique +//! works on, and the op report that sizes the stage. +//! +//! Today's side of every comparison is the code the prover runs: +//! `batch::prove` over the rules `multilinear_table::prove` builds — +//! `IrShape::program` for the constraint, `logup::claim_statements` for the +//! bus. The laptop runs the reference on random columns, with the grid's +//! corners computed: random columns break every AIR. The corner skip needs a +//! trace that satisfies its AIR, and getting one means proving a program, so +//! that half is `#[ignore]`d and the box runs it +//! ([`the_stage1_reference_on_real_traces`]). +//! +//! ```text +//! cargo test -p lambda-vm-prover --lib tests::argue_stage1_tests +//! cargo test -p lambda-vm-prover --lib tests::argue_stage1_tests::the_stage1 -- --ignored --nocapture +//! ``` + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::element::FieldElement; +use multilinear::batch::{self, Rule}; +use multilinear::claim_reduce; +use multilinear::constraint_argument::FactorKind; +use multilinear::eq::eq_mle; +use multilinear::fused::{ + self, Constraints, Counters, Mix, Options, Proved, Shape, Statement, Variant, +}; +use multilinear::logup::{self, Interaction}; +use multilinear::mle::Mle; +use multilinear::sumcheck::SumcheckProof; +use stark::constraint_ir::ir::Op as IrOp; +use stark::multilinear_air::{IrShape, Uniforms, beta_powers, live_nodes}; +use stark::multilinear_table::{TableLayout, weight_slots}; +use stark::traits::AIR; + +use super::lean_program_census::{VmAir, vm_airs}; +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type EE = FieldElement; +type FB = FieldElement; +type DynAir<'a> = &'a dyn AIR; + +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Self(0x9E37_79B9_7F4A_7C15 ^ seed.wrapping_mul(0x2545_F491_4F6C_DD1D) | 1) + } + + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn base(&mut self) -> FB { + FB::from(self.next()) + } + + fn ext(&mut self) -> EE { + EE::new([self.base(), self.base(), self.base()]) + } + + fn exts(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.ext()).collect() + } +} + +/// Every table the argue proves, by name. +fn every_air<'a>(vm: &'a [(VmAir, &'static str)]) -> Vec<(DynAir<'a>, String)> { + vm.iter() + .map(|(air, label)| (&**air as DynAir<'a>, label.to_string())) + .collect() +} + +/// A table's argue, structure only: what `multilinear_table::prove` builds +/// before its batch — the shape, the factor kinds, the bus at stand-in +/// challenges `z` and `alpha`, the batching powers of a stand-in `beta` — and +/// the constraint part brought down to the base field. +struct Table { + label: String, + shape: IrShape, + kinds: Vec, + num_columns: usize, + constraints: Constraints, + interactions: Vec>, + betas: Vec, +} + +impl Table { + fn from_air(air: DynAir<'_>, label: &str, num_vars: usize, rng: &mut Rng) -> Self { + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + air.trace_layout().0, + num_vars, + Uniforms::default(), + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let slots = layout.slot_of().to_vec(); + let interactions = stark::multilinear_logup::interactions( + air.bus_interactions(), + slots.len(), + &rng.ext(), + &rng.ext(), + |col| { + slots + .get(col) + .copied() + .ok_or(multilinear::Error::UnknownPolynomial { + index: col, + len: slots.len(), + }) + }, + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let beta = rng.ext(); + Self::new( + label, + layout.shape().clone(), + layout.kinds().to_vec(), + layout.num_columns(), + interactions, + &beta, + ) + } + + fn new( + label: &str, + shape: IrShape, + kinds: Vec, + num_columns: usize, + interactions: Vec>, + beta: &EE, + ) -> Self { + let constraints = Constraints::::from_extension( + &shape.steps_as_ops(), + shape.root_steps(), + shape.selector_of_root(), + shape.degree(), + ) + .unwrap_or_else(|e| panic!("{label}: the base-field precondition fails: {e:?}")); + let betas = beta_powers(beta, shape.num_roots()); + Self { + label: label.to_string(), + shape, + kinds, + num_columns, + constraints, + interactions, + betas, + } + } + + fn width(&self) -> usize { + self.kinds.len() + } + + /// The factors from committed columns: each view at its offset, and each + /// public selector's table — all base-field, in slot order. + fn factors_from(&self, columns: &[Mle]) -> Vec> { + let num_vars = columns[0].num_vars(); + let mut selectors = self.shape.public_selectors().iter(); + self.kinds + .iter() + .map(|kind| match kind { + FactorKind::Committed(source) => claim_reduce::materialize(columns, source) + .unwrap_or_else(|e| panic!("{}: {e:?}", self.label)), + FactorKind::Public => selectors + .next() + .expect("a selector per public factor") + .table::(num_vars) + .unwrap_or_else(|e| panic!("{}: {e:?}", self.label)), + }) + .collect() + } + + fn random_columns(&self, num_vars: usize, rng: &mut Rng) -> Vec> { + (0..self.num_columns) + .map(|_| Mle::new((0..1usize << num_vars).map(|_| rng.base()).collect()).unwrap()) + .collect() + } + + /// A session over `columns` at a random zerocheck point and GKR claim + /// point. + fn session(&self, columns: Vec>, rng: &mut Rng) -> Session<'_> { + let num_vars = columns[0].num_vars(); + let r = rng.exts(num_vars); + let claim_point = rng.exts(logup::input_layer_vars(self.interactions.len(), num_vars)); + Session { + table: self, + factors: self.factors_from(&columns), + columns, + r, + claim_point, + } + } +} + +/// One table's batch over one set of columns and points. +struct Session<'t> { + table: &'t Table, + columns: Vec>, + factors: Vec>, + r: Vec, + claim_point: Vec, +} + +impl Session<'_> { + fn num_vars(&self) -> usize { + self.factors[0].num_vars() + } + + fn statement(&self) -> Statement<'_, Gl, Ext3> { + Statement { + constraints: &self.table.constraints, + betas: &self.table.betas, + interactions: &self.table.interactions, + claim_point: &self.claim_point, + r: &self.r, + } + } + + fn transcript(&self) -> DefaultTranscript { + DefaultTranscript::::new(self.table.label.as_bytes()) + } + + /// Today's three rules over today's factor list, as + /// `multilinear_table::prove` builds them: the trace's factors lifted, + /// `eq(r, ·)`, `eq(ρ, ·)`. + fn todays(&self) -> (Vec>, Vec>) { + let table = self.table; + let (weight_r, weight_z) = weight_slots(table.width()); + let zerocheck = Rule::compiled( + table.shape.degree() + 1, + table + .shape + .program(&table.betas, weight_r) + .unwrap_or_else(|e| panic!("{}: {e:?}", table.label)), + ); + let bus = logup::claim_statements( + &table.interactions, + &self.claim_point, + self.num_vars(), + weight_z, + ) + .unwrap_or_else(|e| panic!("{}: {e:?}", table.label)); + let mut polys: Vec> = self + .factors + .iter() + .map(|f| { + Mle::new(f.evals().iter().map(|v| v.to_extension::()).collect()).unwrap() + }) + .collect(); + polys.push(eq_mle(&self.r).unwrap()); + polys.push(eq_mle(&bus.row_point).unwrap()); + (polys, vec![zerocheck, bus.numerator, bus.denominator]) + } + + /// Each rule's sum over the cube: the claims an honest batch carries. On + /// a trace that satisfies its AIR the first is zero. + fn true_claims(&self) -> [EE; 3] { + let (polys, rules) = self.todays(); + let mut scratch = Vec::new(); + core::array::from_fn(|k| { + (0..polys[0].len()).fold(EE::zero(), |acc, i| { + let values: Vec = polys.iter().map(|p| p.evals()[i]).collect(); + acc + rules[k].apply_in(&values, &mut scratch) + }) + }) + } + + fn today(&self, claims: &[EE; 3]) -> (SumcheckProof, Vec, DefaultTranscript) { + let (polys, rules) = self.todays(); + let mut t = self.transcript(); + let (proof, point) = batch::prove(polys, rules, claims, &mut t) + .unwrap_or_else(|e| panic!("{}: today's batch: {e:?}", self.table.label)); + (proof, point, t) + } + + fn fused( + &self, + claims: &[EE; 3], + options: Options, + ) -> ( + Result, multilinear::Error>, + DefaultTranscript, + Counters, + ) { + let mut t = self.transcript(); + let mut counters = Counters::default(); + let proved = fused::prove( + &self.statement(), + &self.factors, + claims, + options, + &mut t, + &mut counters, + ); + (proved, t, counters) + } + + /// The batching weights `fused::prove` draws after `claims`. + fn lambdas(&self, claims: &[EE; 3]) -> Vec { + let mut t = self.transcript(); + for claim in claims { + t.append_field_element(claim); + } + multilinear::challenge_powers(&t.sample_field_element(), 3) + } + + /// The first round `options` sends another message at than today's + /// rounds, or `None` — then with the point, the transcript after the + /// rounds, the next challenge, every factor's value at the point and the + /// count against the model all checked too. + fn parts_at(&self, claims: &[EE; 3], options: Options) -> Option { + let label = &self.table.label; + let n = self.num_vars(); + let (proof, point, mut t_today) = self.today(claims); + let (proved, mut t, counters) = self.fused(claims, options); + let (fused, fused_point, bound) = + proved.unwrap_or_else(|e| panic!("{label} n={n} {options:?}: {e:?}")); + if let Some(round) = proof + .rounds + .iter() + .zip(&fused.rounds) + .position(|(a, b)| a.evaluations != b.evaluations) + { + return Some(round); + } + assert_eq!(fused.rounds.len(), n, "{label} n={n} {options:?}"); + assert_eq!(fused_point, point, "{label} n={n} {options:?}: the point"); + assert_eq!( + t.state(), + t_today.state(), + "{label} n={n} {options:?}: the transcript" + ); + assert_eq!( + t.sample_field_element(), + t_today.sample_field_element(), + "{label} n={n} {options:?}: the next challenge" + ); + for (slot, (factor, value)) in self.factors.iter().zip(&bound).enumerate() { + assert_eq!( + *value, + factor.evaluate_in(&point).unwrap(), + "{label} n={n} {options:?}: factor {slot} at the point" + ); + } + let shape = Shape::of( + &self.statement(), + self.table.width(), + &self.lambdas(claims), + options.on_demand, + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + assert_eq!( + fused::model(&shape, options), + counters, + "{label} n={n} {options:?}: the model is not what the reference counted" + ); + None + } +} + +/// Every variant, corners computed: what random columns allow. +fn on_random_columns() -> Vec { + let mut all = Vec::new(); + for variant in [Variant::Today, Variant::Bus] { + for on_demand in [false, true] { + all.push(Options { + on_demand, + ..Options::new(variant) + }); + } + } + all.push(Options::new(Variant::Gruen)); + all.push(Options::new(Variant::Grid)); + all +} + +fn skipped() -> Options { + Options { + skip_corners: true, + ..Options::new(Variant::Grid) + } +} + +fn checked() -> Options { + Options { + check_corners: true, + ..skipped() + } +} + +/// The tables Q1 is sized on: the widest precompile and the VM's tallest +/// table. +const THE_Q1_TABLES: [&str; 2] = ["KECCAK_RND", "CPU"]; + +// ── the base-field precondition ────────────────────────────────────────────── + +/// What in a table's constraint part stops it running in the base field: a +/// live extension constant, a live uniform (a challenge, an alpha power, the +/// table offset), or a constant of the compiled DAG with an extension part. +fn not_base(air: DynAir<'_>) -> Vec { + let program = air.constraint_program(); + let roots = program.roots[..program.num_base].to_vec(); + let live = live_nodes(program, &roots); + let mut found = Vec::new(); + for (id, op) in program.nodes.iter().enumerate() { + if !live[id] { + continue; + } + match op { + IrOp::ConstExt(idx) => { + let value = &program.ext_consts[*idx as usize]; + let limbs = (*value).to_subfield_vec::(); + if limbs[1..].iter().any(|limb| *limb != FB::zero()) { + found.push(format!("node {id}: an extension constant")); + } + } + IrOp::RapChallenge { .. } => found.push(format!("node {id}: a RAP challenge")), + IrOp::AlphaPow { .. } => found.push(format!("node {id}: an alpha power")), + IrOp::TableOffset => found.push(format!("node {id}: the table offset")), + _ => {} + } + } + found +} + +/// ★ Every table's constraint part runs in the base field: no live extension +/// constant or uniform, and the compiled DAG comes down to the base field — +/// the precondition S1-5 rests on (D-ARGUE §2.6). An exception would list its +/// table here, and its roots would run in the extension. +#[test] +fn every_table_meets_the_base_field_precondition() { + let vm = vm_airs(); + let mut exceptions = Vec::new(); + for (air, label) in every_air(&vm) { + for reason in not_base(air) { + exceptions.push(format!("{label}: {reason}")); + } + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + air.trace_layout().0, + 4, + Uniforms::default(), + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let shape = layout.shape(); + if let Err(e) = Constraints::::from_extension( + &shape.steps_as_ops(), + shape.root_steps(), + shape.selector_of_root(), + shape.degree(), + ) { + exceptions.push(format!("{label}: {e}")); + } + } + assert!(exceptions.is_empty(), "not base-field: {exceptions:#?}"); +} + +// ── parity on random columns ───────────────────────────────────────────────── + +/// ★ (a) Every variant of the reference sends today's messages — rounds, +/// point, transcript, next challenge, every factor's value at the point — on +/// every VM table, over random columns with the corners +/// computed, at heights 1, 2, 3 and 5; and counts what the model says. +#[test] +fn stage1_sends_todays_messages_on_every_table() { + let vm = vm_airs(); + let airs = every_air(&vm); + assert_eq!(airs.len(), 26, "26 VM tables"); + for (at, (air, label)) in airs.iter().enumerate() { + for n in [1usize, 2, 3, 5] { + let mut rng = Rng::new((at * 100 + n) as u64); + let table = Table::from_air(*air, label, n, &mut rng); + let session = table.session(table.random_columns(n, &mut rng), &mut rng); + let claims = session.true_claims(); + for options in on_random_columns() { + assert_eq!( + session.parts_at(&claims, options), + None, + "{label} n={n}: {options:?}" + ); + } + } + } +} + +/// ★ (a) The same on the Q1 tables at 2^8 and 2^10 rows, and CPU at 2^12. +/// Heights up to 2^14 run on the box ([`stage1_sends_todays_messages_at_the_top_heights`]). +#[test] +fn stage1_sends_todays_messages_on_the_q1_tables() { + run_the_q1_tables(&[ + ("KECCAK_RND", 8), + ("KECCAK_RND", 10), + ("CPU", 8), + ("CPU", 10), + ("CPU", 12), + ]); +} + +/// The Q1 tables at 2^12 and 2^14 rows — minutes on a laptop core, so the box +/// runs it (`--release`). +#[test] +#[ignore = "the Q1 tables at 2^12 and 2^14 rows: a box run"] +fn stage1_sends_todays_messages_at_the_top_heights() { + run_the_q1_tables(&[("KECCAK_RND", 12), ("KECCAK_RND", 14), ("CPU", 14)]); +} + +fn run_the_q1_tables(heights: &[(&str, usize)]) { + let vm = vm_airs(); + let airs = every_air(&vm); + for &(name, n) in heights { + let (air, label) = airs + .iter() + .find(|(_, label)| label == name) + .unwrap_or_else(|| panic!("no table named {name}")); + let mut rng = Rng::new(0xA11CE + n as u64); + let table = Table::from_air(*air, label, n, &mut rng); + let session = table.session(table.random_columns(n, &mut rng), &mut rng); + let claims = session.true_claims(); + for options in on_random_columns() { + assert_eq!( + session.parts_at(&claims, options), + None, + "{label} n={n}: {options:?}" + ); + } + } +} + +// ── negative controls on random columns ────────────────────────────────────── + +/// ⛔ (c) One coefficient of the bus column wrong — its first, its last — and +/// every variant that reads the column parts from today at round 0, on each of +/// the Q1 tables. +#[test] +fn a_wrong_bus_coefficient_parts_the_messages_on_the_q1_tables() { + let vm = vm_airs(); + let airs = every_air(&vm); + for name in THE_Q1_TABLES { + let (air, label) = airs.iter().find(|(_, label)| label == name).unwrap(); + let mut rng = Rng::new(0xFA17); + let n = 6; + let table = Table::from_air(*air, label, n, &mut rng); + let session = table.session(table.random_columns(n, &mut rng), &mut rng); + let claims = session.true_claims(); + let terms = fused::BusColumn::new( + &table.interactions, + &session.claim_point[..session.claim_point.len() - n], + &EE::one(), + &EE::one(), + ) + .unwrap() + .terms() + .len(); + for fault in [0, terms - 1] { + for variant in [Variant::Bus, Variant::Gruen, Variant::Grid] { + let options = Options { + bus_fault: Some(fault), + ..Options::new(variant) + }; + assert_eq!( + session.parts_at(&claims, options), + Some(0), + "{label}: {options:?}" + ); + } + } + } +} + +/// ⛔ The corner skip on columns that break the AIR parts from today at round +/// 0 on each of the Q1 tables, and the corner check refuses at the first row. +#[test] +fn the_corner_skip_parts_on_random_columns_of_the_q1_tables() { + let vm = vm_airs(); + let airs = every_air(&vm); + for name in THE_Q1_TABLES { + let (air, label) = airs.iter().find(|(_, label)| label == name).unwrap(); + let mut rng = Rng::new(0xC0DE); + let n = 5; + let table = Table::from_air(*air, label, n, &mut rng); + let session = table.session(table.random_columns(n, &mut rng), &mut rng); + let claims = session.true_claims(); + assert_eq!(session.parts_at(&claims, skipped()), Some(0), "{label}"); + assert_eq!( + session.fused(&claims, checked()).0.unwrap_err(), + multilinear::Error::ConstraintViolated { row: 0 }, + "{label}" + ); + } +} + +// ── the real-trace half (box) ──────────────────────────────────────────────── + +/// ★ (b) + (c), on traces that satisfy their AIRs: KECCAK_RND and CPU captured +/// from a WHIR proof of `test_keccak` (`multilinear_table::argue_capture`). +/// +/// Each captured table, from a fresh transcript at its own GKR claim and +/// zerocheck point: today's claims `(0, p̃, q̃)` are the batch's true sums (the +/// trace satisfies its AIR and the GKR claim is the input layer's); every +/// variant sends today's messages, the corners skipped and then checked. Then +/// the controls: a cell the constraints read flipped — the corner check names +/// its row and the corner skip parts from today by round 1; and a wrong bus +/// coefficient, parting at round 0. +/// +/// A proving test, so not on the laptop: +/// +/// ```text +/// cargo test --release -p lambda-vm-prover --lib \ +/// tests::argue_stage1_tests::the_stage1_reference_on_real_traces -- --ignored --exact --nocapture +/// ``` +#[test] +#[ignore = "proves a program to capture real traces: a box run"] +fn the_stage1_reference_on_real_traces() { + use crate::tables::MaxRowsConfig; + use stark::multilinear_table::argue_capture::{self, Captured}; + use stark::proof::options::ProofOptions; + + let options = ProofOptions::default_test_options(); + let vm = vm_airs(); + let named: Vec<(DynAir<'_>, String)> = every_air(&vm) + .into_iter() + .filter(|(_, label)| THE_Q1_TABLES.contains(&label.as_str())) + .collect(); + let width = |air: DynAir<'_>| { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + air.trace_layout().0, + 4, + Uniforms::default(), + ) + .unwrap() + .num_columns() + }; + let widths: Vec = named.iter().map(|(air, _)| width(*air)).collect(); + argue_capture::arm(&widths); + let elf = crate::test_utils::asm_elf_bytes("test_keccak"); + crate::multilinear_prove::prove_with_options(&elf, &options, &MaxRowsConfig::default()) + .expect("test_keccak proves"); + argue_capture::arm(&[]); + + let mut seen: Vec = Vec::new(); + for captured in argue_capture::take() { + let captured = captured + .downcast::>() + .expect("a Goldilocks table"); + let num_vars = captured.columns[0].num_vars(); + // Which of the Q1 tables: the same layout, the same constraint DAG. + let dag = Mix::of(&captured.shape.steps_as_ops()); + let Some((_, label)) = named.iter().find(|(air, label)| { + let mut rng = Rng::new(1); + let candidate = Table::from_air(*air, label, num_vars, &mut rng); + candidate.kinds == captured.kinds + && candidate.num_columns == captured.columns.len() + && candidate.interactions.len() == captured.interactions.len() + && Mix::of(&candidate.shape.steps_as_ops()) == dag + }) else { + continue; + }; + let table = Table::new( + label, + captured.shape.clone(), + captured.kinds.clone(), + captured.columns.len(), + captured.interactions.clone(), + &captured.beta, + ); + let session = Session { + table: &table, + factors: table.factors_from(&captured.columns), + columns: captured.columns.clone(), + r: captured.r.clone(), + claim_point: captured.claim.point.clone(), + }; + let claims = [EE::zero(), captured.claim.p, captured.claim.q]; + assert_eq!( + session.true_claims(), + claims, + "{label} 2^{num_vars}: the captured claims are the batch's sums" + ); + let mut every = on_random_columns(); + every.push(skipped()); + every.push(checked()); + for options in every { + assert_eq!( + session.parts_at(&claims, options), + None, + "{label} 2^{num_vars}: {options:?}" + ); + } + + // ⛔ A flipped cell of a column the constraints read: the first that + // breaks the AIR, which the corner check refuses. + let mut flipped = None; + 'cells: for &slot in &table.constraints.reads() { + let FactorKind::Committed(source) = table.kinds[slot] else { + continue; + }; + for row in [0usize, 1, 5, (1 << num_vars) / 2 + 3] { + let mut columns = session.columns.clone(); + let mut values = columns[source.column].evals().to_vec(); + let row = row % values.len(); + values[row] += FB::one(); + columns[source.column] = Mle::new(values).unwrap(); + let broken = Session { + table: &table, + factors: table.factors_from(&columns), + columns, + r: session.r.clone(), + claim_point: session.claim_point.clone(), + }; + if let Err(multilinear::Error::ConstraintViolated { row: at }) = + broken.fused(&claims, checked()).0 + { + flipped = Some((broken, source.column, at)); + break 'cells; + } + } + } + let (broken, column, at) = + flipped.unwrap_or_else(|| panic!("{label}: no flipped cell broke the AIR")); + let mut claims_broken = broken.true_claims(); + claims_broken[0] = EE::zero(); + let parted = broken.parts_at(&claims_broken, skipped()); + assert!( + matches!(parted, Some(0) | Some(1)), + "{label}: a flipped cell of column {column} (the check refused row {at}) parted at {parted:?}" + ); + eprintln!( + "argue stage 1: {label} 2^{num_vars}: {} variants send today's messages; a flipped cell of \ + column {column} refused at row {at}, parted at round {}; a wrong bus coefficient parted at round 0", + on_random_columns().len() + 2, + parted.unwrap_or(usize::MAX) + ); + + // ⛔ A wrong bus coefficient. + let faulty = Options { + bus_fault: Some(0), + ..skipped() + }; + assert_eq!(session.parts_at(&claims, faulty), Some(0), "{label}"); + seen.push(label.clone()); + } + seen.sort(); + seen.dedup(); + assert_eq!( + seen, + ["CPU", "KECCAK_RND"], + "every one of the Q1 tables captured" + ); +} + +// ── Q1b: the card (`multilinear::gpu_fused`) ───────────────────────────────── + +/// ★ The blob the fused kernels walk is the constraint part: on every VM table, +/// the lowered program with its `ACC` steps, walked as the +/// grid kernel walks it, gives today's zerocheck rule (weight one) at random +/// rows. Host only — this is what makes the card's walk checkable without one. +#[test] +fn the_fused_blob_is_the_constraint_part_on_every_table() { + use multilinear::gpu_fused::{lower_fused, run_lowered_host}; + let vm = vm_airs(); + for (at, (air, label)) in every_air(&vm).into_iter().enumerate() { + let mut rng = Rng::new(0xB10B + at as u64); + let table = Table::from_air(air, &label, 4, &mut rng); + let lowered = + lower_fused(&table.constraints).unwrap_or_else(|| panic!("{label}: no lowering")); + let width = table.width(); + let rule = Rule::compiled( + table.constraints.degree() + 1, + table + .constraints + .zerocheck_program(&table.betas, width) + .unwrap(), + ); + for row in 0..3 { + let values: Vec = (0..width).map(|_| rng.base()).collect(); + let mut lifted: Vec = values.iter().map(|v| v.to_extension::()).collect(); + lifted.push(EE::one()); + assert_eq!( + run_lowered_host(&lowered, &table.betas, &values), + rule.apply(&lifted), + "{label}: row {row} ({} slots)", + lowered.num_slots + ); + } + } +} + +/// Card tests share process-global switches; this puts them back however a +/// test ends. +#[cfg(feature = "cuda")] +struct CardSwitches; + +#[cfg(feature = "cuda")] +impl CardSwitches { + fn set(faults: multilinear::gpu_fused::FusedFaults, xcheck: bool) -> Self { + use multilinear::gpu_fused::*; + // A mutation run (the box's) makes one check inert from outside, to + // show the test that relies on it fails without it. + let mut faults = faults; + match std::env::var("LAMBDA_VM_ARGUE_FUSED_MUTATE").as_deref() { + Ok("xcheck") => faults.xcheck_inert = true, + Ok("corners") => faults.corners_inert = true, + _ => {} + } + force_fused_faults(faults); + force_argue_fused(Some(true)); + force_argue_fused_xcheck(Some(xcheck)); + Self + } +} + +#[cfg(feature = "cuda")] +impl Drop for CardSwitches { + fn drop(&mut self) { + use multilinear::gpu_fused::*; + force_fused_faults(FusedFaults::default()); + force_argue_fused(None); + force_argue_fused_xcheck(None); + math_cuda::sumcheck::force_int_nodes(None); + } +} + +#[cfg(feature = "cuda")] +type OnCard = Result<(SumcheckProof, Vec, Vec), multilinear::Error>; + +#[cfg(feature = "cuda")] +impl Session<'_> { + /// The batch through the prover's entry (`batch::prove_resident_with`) + /// with the factors on the card — the fused rounds, when they are on. + fn on_the_card(&self, claims: &[EE; 3]) -> OnCard { + use multilinear::batch::Weight; + let table = self.table; + let n = self.num_vars(); + let public: Vec> = table + .shape + .public_selectors() + .iter() + .map(|s| s.table::(n).unwrap()) + .collect(); + let device = multilinear::gpu::upload_factors_from_columns( + &self.columns, + &table.kinds, + &public, + None, + ) + .unwrap_or_else(|| panic!("{}: the card takes the factors", table.label)); + let (polys, rules) = self.todays(); + let lifted: Vec> = polys[..table.width()].to_vec(); + let rho = self.claim_point[self.claim_point.len() - n..].to_vec(); + let mut t = self.transcript(); + batch::prove_resident_with( + vec![Weight::Eq(self.r.clone()), Weight::Eq(rho)], + Some(std::sync::Arc::new(device)), + || Ok(lifted.clone()), + rules, + claims, + Some(multilinear::gpu_fused::FusedInput { + constraints: &table.constraints, + betas: &table.betas, + interactions: &table.interactions, + claim_point: &self.claim_point, + r: &self.r, + columns: None, + }), + &mut t, + ) + } + + /// The card's proof against today's host rounds: messages, point, and + /// every trace factor's value at the point. + fn assert_card_is_today(&self, claims: &[EE; 3], card: OnCard, what: &str) { + let label = &self.table.label; + let n = self.num_vars(); + let (proof, point, bound) = card.unwrap_or_else(|e| panic!("{label} n={n} {what}: {e:?}")); + let (today, today_point, _) = self.today(claims); + for (round, (a, b)) in today.rounds.iter().zip(&proof.rounds).enumerate() { + assert_eq!( + a.evaluations, b.evaluations, + "{label} n={n} {what}: round {round}" + ); + } + assert_eq!( + proof.rounds.len(), + today.rounds.len(), + "{label} n={n} {what}" + ); + assert_eq!(point, today_point, "{label} n={n} {what}: the point"); + for (slot, factor) in self.factors.iter().enumerate() { + assert_eq!( + bound[slot], + factor.evaluate_in(&point).unwrap(), + "{label} n={n} {what}: factor {slot} at the point" + ); + } + } +} + +/// ★ Q1b device parity (box): the fused rounds on the card send today's +/// messages — the Q1 tables at 2^10 and 2^12, every table at 2^8 or the height +/// the card's upload takes it at — over random +/// columns, the corners summed (random columns break every AIR), with the +/// cross-check on: today's device rounds replay the fused challenges and must +/// agree. Each run must have taken the card (`fused_sessions`) and been +/// confirmed (`fused_xchecks`). +#[cfg(feature = "cuda")] +#[test] +#[ignore = "flips process-global argue switches: run alone, --ignored --test-threads=1"] +fn the_fused_rounds_on_the_card_send_todays_messages() { + use multilinear::gpu_fused::*; + let _switches = CardSwitches::set( + FusedFaults { + keep_corners: true, + ..FusedFaults::default() + }, + true, + ); + let vm = vm_airs(); + // Each table at 2^8, or as far above as the card's upload takes it: it + // declines a table under 4096 cells (`gpu::worth_the_device`), which the + // prover then argues on the host — EQ's 12 factors need 2^9, a two-factor + // chip 2^11. + let mut runs: Vec<(String, usize)> = every_air(&vm) + .iter() + .map(|(air, label)| { + let mut rng = Rng::new(0); + let width = Table::from_air(*air, label, 8, &mut rng).width(); + let n = (8..).find(|n| width << n >= 4096).expect("a height"); + (label.clone(), n) + }) + .collect(); + for label in THE_Q1_TABLES { + runs.push((label.to_string(), 10)); + runs.push((label.to_string(), 12)); + } + let airs = every_air(&vm); + for (at, (label, n)) in runs.iter().enumerate() { + let air = airs.iter().find(|(_, l)| l == label).unwrap().0; + let mut rng = Rng::new(0xCA4D + at as u64); + let table = Table::from_air(air, label, *n, &mut rng); + let session = table.session(table.random_columns(*n, &mut rng), &mut rng); + let claims = session.true_claims(); + let (sessions, checks) = (fused_sessions(), fused_xchecks()); + session.assert_card_is_today(&claims, session.on_the_card(&claims), "fused"); + assert_eq!( + fused_sessions(), + sessions + 1, + "{label} n={n}: the card ran the fused rounds" + ); + assert_eq!( + fused_xchecks(), + checks + 1, + "{label} n={n}: today's rounds confirmed them" + ); + } + eprintln!( + "argue Q1b: {} fused card runs sent today's messages", + runs.len() + ); +} + +/// ★ The production switches (fused on, the corners skipped, no +/// cross-check) on every table without roots: its constraint part is zero, +/// so its grid `{0,1}²` is all corners, the corner skip is exact on random +/// columns too, and no grid point is left to launch — the case FAST job +/// 273's B arm crashed on (a division by the empty row count). Against +/// today's host rounds; each run counted as a fused session. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "flips process-global argue switches: run alone, --ignored --test-threads=1"] +fn the_fused_rounds_skip_the_corners_of_every_table_without_roots() { + use multilinear::gpu_fused::*; + let _switches = CardSwitches::set(FusedFaults::default(), false); + let vm = vm_airs(); + let mut runs = 0; + for (at, (air, label)) in every_air(&vm).into_iter().enumerate() { + let mut rng = Rng::new(0x5C1 + at as u64); + let probe = Table::from_air(air, &label, 8, &mut rng); + if probe.shape.num_roots() != 0 { + continue; + } + let n = (8..) + .find(|n| probe.width() << n >= 4096) + .expect("a height"); + let table = Table::from_air(air, &label, n, &mut rng); + let session = table.session(table.random_columns(n, &mut rng), &mut rng); + let claims = session.true_claims(); + let sessions = fused_sessions(); + session.assert_card_is_today(&claims, session.on_the_card(&claims), "corners skipped"); + assert_eq!( + fused_sessions(), + sessions + 1, + "{label} n={n}: the card ran the fused rounds" + ); + runs += 1; + } + assert_eq!(runs, 12, "the twelve tables without roots"); + eprintln!( + "argue Q1b: {runs} tables without roots sent today's messages with the corners skipped" + ); +} + +/// ★ A proof made with the fused rounds verifies: `test_keccak` under the +/// production switches of the B arm (fused, integer nodes, no cross-check), +/// then with the cross-check, each proved and verified +/// (`multilinear_prove::verify_with_options`), each taking the card's fused +/// rounds on the tables tall enough (n ≥ 7) — among them tables without roots. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "flips process-global argue switches and proves a program: run alone, --ignored --test-threads=1"] +fn a_proof_made_with_the_fused_rounds_verifies() { + use crate::tables::MaxRowsConfig; + use multilinear::gpu_fused::*; + use stark::proof::options::ProofOptions; + let options = ProofOptions::default_test_options(); + let elf = crate::test_utils::asm_elf_bytes("test_keccak"); + for xcheck in [false, true] { + let _switches = CardSwitches::set(FusedFaults::default(), xcheck); + math_cuda::sumcheck::force_int_nodes(Some(true)); + let (sessions, checks) = (fused_sessions(), fused_xchecks()); + let proof = + crate::multilinear_prove::prove_with_options(&elf, &options, &MaxRowsConfig::default()) + .unwrap_or_else(|e| panic!("xcheck {xcheck}: test_keccak proves: {e:?}")); + let fused = fused_sessions() - sessions; + assert!(fused > 0, "xcheck {xcheck}: the fused rounds ran"); + if xcheck { + assert_eq!( + fused_xchecks() - checks, + fused, + "every fused table confirmed" + ); + } + assert!( + crate::multilinear_prove::verify_with_options(&proof, &elf, &options) + .unwrap_or_else(|e| panic!("xcheck {xcheck}: verify: {e:?}")), + "xcheck {xcheck}: the proof verifies" + ); + eprintln!( + "argue Q1b: test_keccak proved with {fused} fused tables (xcheck {xcheck}) and verified" + ); + } +} + +/// ★ S1-1 on today's kernel (`LAMBDA_VM_ARGUE_INT_NODES`): the device rounds +/// with integer nodes send today's messages (the Q1 tables at 2^10, fused off). +#[cfg(feature = "cuda")] +#[test] +#[ignore = "flips process-global argue switches: run alone, --ignored --test-threads=1"] +fn the_int_node_rounds_send_todays_messages() { + let _switches = CardSwitches::set(Default::default(), false); + multilinear::gpu_fused::force_argue_fused(Some(false)); + math_cuda::sumcheck::force_int_nodes(Some(true)); + let vm = vm_airs(); + for (at, (air, label)) in every_air(&vm) + .into_iter() + .filter(|(_, l)| THE_Q1_TABLES.contains(&l.as_str())) + .enumerate() + { + let mut rng = Rng::new(0x1A7 + at as u64); + let table = Table::from_air(air, &label, 10, &mut rng); + let session = table.session(table.random_columns(10, &mut rng), &mut rng); + let claims = session.true_claims(); + let before = multilinear::gpu::sumcheck_calls(); + session.assert_card_is_today(&claims, session.on_the_card(&claims), "int nodes"); + assert!( + multilinear::gpu::sumcheck_calls() > before, + "{label}: the card ran the rounds" + ); + } +} + +/// ⛔ A wrong bus coefficient on the card: the cross-check refuses the prove +/// (the Q1 tables at 2^10). Mutation `LAMBDA_VM_ARGUE_FUSED_MUTATE=xcheck` makes the +/// comparison inert, and this test must then FAIL. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "flips process-global argue switches: run alone, --ignored --test-threads=1"] +fn a_wrong_bus_coefficient_on_the_card_is_refused_by_the_cross_check() { + use multilinear::gpu_fused::*; + let _switches = CardSwitches::set( + FusedFaults { + bus: true, + keep_corners: true, + ..FusedFaults::default() + }, + true, + ); + let vm = vm_airs(); + for (at, (air, label)) in every_air(&vm) + .into_iter() + .filter(|(_, l)| THE_Q1_TABLES.contains(&l.as_str())) + .enumerate() + { + let mut rng = Rng::new(0xBAD + at as u64); + let table = Table::from_air(air, &label, 10, &mut rng); + let session = table.session(table.random_columns(10, &mut rng), &mut rng); + let claims = session.true_claims(); + match session.on_the_card(&claims) { + Err(multilinear::Error::DeviceFailed { + stage: "fused cross-check", + }) => {} + other => panic!( + "{label}: a wrong bus coefficient was not refused: {:?}", + other.map(|_| ()) + ), + } + } +} + +/// ⛔ The corner check on the card refuses a trace that breaks its AIR — +/// random columns, the corners checked rather than summed (the Q1 tables at +/// 2^10). Mutation `LAMBDA_VM_ARGUE_FUSED_MUTATE=corners` makes it inert, and +/// this test must then FAIL. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "flips process-global argue switches: run alone, --ignored --test-threads=1"] +fn the_corner_check_on_the_card_refuses_a_broken_trace() { + use multilinear::gpu_fused::*; + let _switches = CardSwitches::set(FusedFaults::default(), true); + let vm = vm_airs(); + for (at, (air, label)) in every_air(&vm) + .into_iter() + .filter(|(_, l)| THE_Q1_TABLES.contains(&l.as_str())) + .enumerate() + { + let mut rng = Rng::new(0xC0 + at as u64); + let table = Table::from_air(air, &label, 10, &mut rng); + let session = table.session(table.random_columns(10, &mut rng), &mut rng); + let claims = session.true_claims(); + match session.on_the_card(&claims) { + Err(multilinear::Error::ConstraintViolated { .. }) => {} + other => panic!( + "{label}: a broken trace was not refused: {:?}", + other.map(|_| ()) + ), + } + } +} + +/// The device tail's rule (`LAMBDA_VM_ZC_DEVICE_TAIL`: at least +/// [`multilinear::gpu_fused::DEVICE_TAIL_MIN_STEPS`] lowered steps) takes the +/// VM's long programs and none of the short ones: KECCAK_RND and ECDAS on, +/// CPU off. Prints every program it takes, so a box log shows which. +#[test] +fn the_device_tail_takes_the_long_vm_programs() { + use multilinear::gpu_fused::{DEVICE_TAIL_MIN_STEPS, lower_fused}; + let mut vm = vm_airs(); + let opts = + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + vm.push(( + Box::new(crate::test_utils::create_ecrec_air(&opts)), + "ECREC", + )); + let mut long = Vec::new(); + for (at, (air, label)) in vm.iter().enumerate() { + let mut rng = Rng::new(0x7A11_0000 + at as u64); + let table = Table::from_air(&**air, label, 4, &mut rng); + if table.constraints.roots().is_empty() { + continue; + } + let steps = lower_fused(&table.constraints) + .unwrap_or_else(|| panic!("{label}: lowers")) + .nodes + .len() + / 2; + if steps >= DEVICE_TAIL_MIN_STEPS { + long.push(format!("{label} {steps}")); + } + match *label { + "KECCAK_RND" | "ECDAS" => { + assert!(steps >= DEVICE_TAIL_MIN_STEPS, "{label}: {steps} steps") + } + "CPU" => assert!(steps < DEVICE_TAIL_MIN_STEPS, "{label}: {steps} steps"), + _ => {} + } + } + eprintln!( + "device tail takes {} programs: {}", + long.len(), + long.join(", ") + ); + assert!(long.len() >= 2, "{long:?}"); +} diff --git a/prover/src/tests/auto_storage_tests.rs b/prover/src/tests/auto_storage_tests.rs deleted file mode 100644 index efe844018..000000000 --- a/prover/src/tests/auto_storage_tests.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Tests for storage-mode auto-selection. - -use crate::auto_storage::{ - CUBIC_EXT_BYTES, GOLDILOCKS_BYTES, KECCAK_NODE_BYTES, peak_bytes, select_storage_mode, -}; -use crate::tables::cpu::bus_interactions as cpu_buses; -use crate::tables::cpu::cols::NUM_COLUMNS as CPU_COLS; -use crate::tables::trace_builder::TableLengths; -use stark::storage_mode::StorageMode; - -const GB: u64 = 1_000_000_000; -/// Larger than the table count, so every table lands in the top-k and the -/// per-table delta in `peak_bytes_per_table_increment_is_exact` is purely -/// additive. -const ALL_TABLES: usize = 1_000; - -fn empty_lengths() -> TableLengths { - TableLengths::default() -} - -/// Adding rows to a single chunked table must increase `peak_bytes` by -/// exactly the per-row contribution from the formula in the module doc. -/// Verifies the per-table breakdown is exact rather than averaged. -#[test] -fn peak_bytes_per_table_increment_is_exact() { - let blowup = 2u8; - let b = blowup as u64; - - let baseline = peak_bytes(&empty_lengths(), blowup, ALL_TABLES); - - let mut lengths = empty_lengths(); - lengths.cpu_padded_rows = 4; - let bumped = peak_bytes(&lengths, blowup, ALL_TABLES); - - let cpu_main = CPU_COLS as u64; - let cpu_aux = cpu_buses(0).len().div_ceil(2) as u64; - let per_row_persistent = cpu_main * GOLDILOCKS_BYTES * (1 + b) - + cpu_aux * CUBIC_EXT_BYTES * (1 + b) - + 2 * 2 * b * KECCAK_NODE_BYTES // precomputed + mult Merkle trees - + 2 * b * KECCAK_NODE_BYTES; // aux Merkle - let per_row_transient = b * CUBIC_EXT_BYTES // constraint_evaluations - + 2 * b * CUBIC_EXT_BYTES // composition LDE (2 parts, d=2) - + b * KECCAK_NODE_BYTES // composition Merkle (PairKeccak) - + b * CUBIC_EXT_BYTES // FRI evals (geometric ≈ 1) - + b * KECCAK_NODE_BYTES; // FRI Merkle (geometric ≈ 1) - let per_row_domain = (3 + 2 * b) * GOLDILOCKS_BYTES; - - // CPU adds 4 rows of persistent + transient (top-k by ALL_TABLES) + - // its 4-row Domain entry (a fresh unique key not previously present). - assert_eq!( - bumped - baseline, - 4 * (per_row_persistent + per_row_transient + per_row_domain) - ); -} - -/// Higher blowup_factor should produce a strictly larger estimate. -#[test] -fn peak_bytes_scales_with_blowup() { - let lengths = empty_lengths(); - let two = peak_bytes(&lengths, 2, ALL_TABLES); - let four = peak_bytes(&lengths, 4, ALL_TABLES); - let eight = peak_bytes(&lengths, 8, ALL_TABLES); - assert!(two < four); - assert!(four < eight); -} - -/// Lower table_parallelism caps the transient sum to fewer tables, so the -/// estimate must be monotone in `k`. -#[test] -fn peak_bytes_monotone_in_table_parallelism() { - let lengths = empty_lengths(); - let k1 = peak_bytes(&lengths, 2, 1); - let k4 = peak_bytes(&lengths, 2, 4); - let k_all = peak_bytes(&lengths, 2, ALL_TABLES); - assert!(k1 < k4); - assert!(k4 <= k_all); -} - -#[test] -fn select_ram_when_estimate_below_threshold() { - // 10 GB estimated, 32 GB available → threshold 28.8 GB → Ram. - let mode = select_storage_mode(10 * GB, Some(32 * GB)); - assert_eq!(mode, StorageMode::Ram); -} - -#[test] -fn select_disk_when_estimate_exceeds_threshold() { - // 30 GB estimated, 32 GB available → threshold 28.8 GB → Disk. - let mode = select_storage_mode(30 * GB, Some(32 * GB)); - assert_eq!(mode, StorageMode::Disk); -} - -#[test] -fn unknown_available_defaults_to_disk() { - let mode = select_storage_mode(peak_bytes(&empty_lengths(), 2, ALL_TABLES), None); - assert_eq!(mode, StorageMode::Disk); -} - -/// A shape with one PAGE table — everything the monolithic path proves today. -/// The top-k sum has saturated well before the table count, so the estimate is -/// insensitive to `k` in that range: this is why raising the *scheduler's* `k` -/// to `num_airs` does not move the storage decision on a normal workload. -#[test] -fn peak_bytes_is_k_saturated_on_single_page_shapes() { - let mut lengths = empty_lengths(); - lengths.cpu_padded_rows = 1 << 20; - lengths.memw_padded_rows = 1 << 20; - lengths.decode_rows = 1 << 16; - lengths.unique_page_count = 1; - - let bounded = peak_bytes(&lengths, 2, 12); - let unbounded = peak_bytes(&lengths, 2, ALL_TABLES); - assert!( - unbounded * 100 <= bounded * 101, - "estimate moved {bounded} -> {unbounded} on a one-page shape" - ); -} - -/// …and why `decide` must not simply be handed the scheduler's `k`. PAGE tables -/// are all the same size, so once there are many of them the top-k truncation -/// is doing real work: summing every table's transients inflates the estimate -/// by >20 % here, which spills proofs to disk that fit in RAM. -#[test] -fn unbounded_k_inflates_peak_bytes_on_many_page_shapes() { - let mut lengths = empty_lengths(); - lengths.cpu_padded_rows = 1 << 20; - lengths.memw_padded_rows = 1 << 20; - lengths.decode_rows = 1 << 16; - lengths.unique_page_count = 128; - - let bounded = peak_bytes(&lengths, 2, 21); - let unbounded = peak_bytes(&lengths, 2, ALL_TABLES); - assert!( - unbounded * 10 > bounded * 12, - "expected >20 % inflation, got {bounded} -> {unbounded}" - ); -} diff --git a/prover/src/tests/batched_argue_tests.rs b/prover/src/tests/batched_argue_tests.rs new file mode 100644 index 000000000..51f1a047f --- /dev/null +++ b/prover/src/tests/batched_argue_tests.rs @@ -0,0 +1,411 @@ +//! The batched argue (D-BATCH B-2, `stark::multilinear_table::batched`) on the +//! tables a proof really argues: every VM table a program lights up. +//! +//! Proving tests, so a box runs them: +//! +//! ```text +//! cargo test --release -p lambda-vm-prover --lib tests::batched_argue_tests -- --ignored --nocapture +//! ``` +//! +//! The per-table path's entry points are followed step for step +//! (`multilinear_prove::prove_with_options_and_inputs` / +//! `verify_with_options`), with the argue swapped for the batched one: the +//! statement, the commitment and the openings are today's. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use executor::elf::Elf; +use executor::vm::execution::Executor; +use math::field::element::FieldElement; +use multilinear::mle::Mle; +use multilinear::whir_chain::{ArgueFormat, ChainConfig}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::batched::{self, Where}; +use stark::multilinear_table::{ + self, ArgueShape, BatchedMultiProof, CommittedTable, CommittedTables, TableLayout, + TableStatement, argue_plan, +}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::statement; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::Traces; +use crate::test_utils::{E, F}; +use crate::{VmAirs, multilinear_prove}; + +type Tamper = fn(&mut BatchedMultiProof); + +/// One table's layout from its AIR and shape — both sides call this. +fn layout<'a>( + air: &'a dyn AIR, + (width, num_vars): (usize, usize), +) -> TableLayout<'a, F, E> { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + width, + num_vars, + Uniforms::default(), + ) + .expect("layout") +} + +fn batched(mut config: ChainConfig) -> ChainConfig { + config.format.argue = ArgueFormat::BATCHED; + config +} + +/// A program's every table proved under the batched argue and verified; the +/// tamper, when given, applied between. Returns the verdict and each table's +/// name. +fn vm_round_trip( + name: &str, + tamper: Option, +) -> (Result<(), multilinear::Error>, Vec) { + let (verdict, names, _) = vm_round_trip_elf( + name, + &crate::test_utils::asm_elf_bytes(name), + tamper, + &[Where::Device], + ); + (verdict, names) +} + +/// A Rust guest's ELF, built by `make compile-programs-rust`. +fn rust_elf_bytes(name: &str) -> Vec { + let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + std::fs::read(root.join(format!("executor/program_artifacts/rust/{name}.elf"))) + .unwrap_or_else(|e| panic!("{name}.elf — run `make compile-programs-rust`: {e}")) +} + +/// [`vm_round_trip`] over an ELF in hand. +/// +/// `ats` are where the prover runs, one proof each over the SAME committed +/// tables: [`Where::Device`] is what +/// `multi_prove_batched` does (the card where it takes the work), and +/// [`Where::Host`] the host reference. The proof's canonical bytes come back +/// too, so the two can be compared. +fn vm_round_trip_elf( + name: &str, + elf_bytes: &[u8], + tamper: Option, + ats: &[Where], +) -> (Result<(), multilinear::Error>, Vec, Vec>) { + let elf_bytes = elf_bytes.to_vec(); + let program = Elf::load(&elf_bytes).expect("ELF"); + let result = Executor::new(&program, Vec::new()) + .expect("executor") + .run() + .expect("runs"); + let mut traces = + Traces::from_elf_and_logs(&program, &result.logs, &MaxRowsConfig::default(), &[]) + .expect("traces"); + drop(result); + let table_counts = traces.table_counts(); + let runtime_page_ranges = traces.runtime_page_ranges(); + let num_private_input_pages = traces + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(); + let public_output = traces.public_output_bytes.clone(); + let options = ProofOptions::default_test_options(); + let airs = VmAirs::new( + &program, + &options, + false, + &traces.page_configs, + &table_counts, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + crate::test_utils::halted_register_commitment( + &options, + &program, + &traces.public_output_bytes, + ) + .preprocessed(), + ); + let mut pairs = airs.air_trace_pairs(&mut traces); + let shapes: Vec<(usize, usize)> = pairs + .iter() + .map(|(_, trace, _)| { + ( + trace.main_table.width, + trace.main_table.height.trailing_zeros() as usize, + ) + }) + .collect(); + let names: Vec = pairs + .iter() + .map(|(air, _, _)| air.name().to_string()) + .collect(); + let table_num_vars: Vec = shapes.iter().map(|&(_, n)| n as u8).collect(); + let config = batched(multilinear_prove::chain_config(&shapes)); + + let mut committed = Vec::with_capacity(pairs.len()); + for ((air, trace, _), &shape) in pairs.iter_mut().zip(&shapes) { + let mut columns = trace.columns_main(); + committed.push( + CommittedTable::from_layout(layout(*air, shape), |col| { + core::mem::take(&mut columns[col as usize]) + }) + .expect("table"), + ); + } + drop(pairs); + let refs = airs.air_refs(); + let plan = { + let layouts: Vec> = refs + .iter() + .zip(&shapes) + .map(|(air, &shape)| layout(*air, shape)) + .collect(); + let argue: Vec = layouts + .iter() + .map(|l| ArgueShape::of(&l.statement())) + .collect(); + argue_plan(&argue, 27) + }; + eprintln!( + "batched argue {name}: {} tables, bins {:?}, n_max {}, D_max {}", + names.len(), + plan.bins, + plan.num_vars, + plan.degree + ); + + let digest = statement::elf_digest(&elf_bytes); + let verdict = crate::with_whir_hash!(|H| { + type Tr = DefaultTranscript::Transcript>; + let absorb = |t: &mut Tr| { + multilinear_prove::absorb( + t, + &digest, + &public_output, + &table_counts, + num_private_input_pages, + &runtime_page_ranges, + &table_num_vars, + &config, + ) + }; + let committed = CommittedTables::<_, _, H>::commit(committed, &config).expect("commit"); + // ★ Every arm proves THESE committed tables: six trace builders lay rows + // out in HashMap order, so two trace builds are two different (valid) + // traces, and only one commitment makes two provers comparable. + let mut verdicts = Vec::with_capacity(ats.len()); + let mut all_bytes = Vec::with_capacity(ats.len()); + for &at in ats { + let mut transcript = Tr::new(&[]); + absorb(&mut transcript); + multilinear::gpu::reset_reserved_window(); + let mut proof = batched::prove_batched_on( + &committed, + &config, + &mut transcript, + None, + batched::ProverFaults::default(), + at, + ) + .expect("the batched prover runs"); + if at == Where::Device && multilinear::gpu::reserve_budget() > 0 { + eprintln!( + "batched argue {name}: device ledger peak {} MiB of a {} MiB budget", + multilinear::gpu::reserved_window_peak() >> 20, + multilinear::gpu::reserve_budget() >> 20 + ); + } + // serde writes each element canonically, so equal bytes are equal + // values. The roots and the argue only: the openings grind, and a + // parallel nonce search returns ANY valid nonce, so two honest proofs + // part there (memory: grinding-nonce-nondeterminism) — after the argue. + let bytes = serde_json::to_vec(&(&proof.roots, &proof.argue)).expect("canonical bytes"); + if let Some(tamper) = tamper { + tamper(&mut proof); + } + + // The verifier's side: layouts, preprocessed copies and stacks rebuilt + // from the AIRs and the shapes alone. + let layouts: Vec> = refs + .iter() + .zip(&shapes) + .map(|(air, &shape)| layout(*air, shape)) + .collect(); + let preprocessed: Vec>> = refs + .iter() + .map(|air| { + air.precomputed_columns() + .into_iter() + .map(|values| Mle::new(values).expect("column")) + .collect() + }) + .collect(); + let statements: Vec> = layouts + .iter() + .zip(&preprocessed) + .map(|(layout, cols)| layout.statement_with_preprocessed(cols)) + .collect(); + let sizes = [shapes.len()]; + let (group_layouts, domains) = + multilinear_prove::stacks(&shapes, &sizes, &config).expect("stacks"); + let mut transcript = Tr::new(&[]); + absorb(&mut transcript); + let mut probe = transcript.clone(); + multilinear_table::absorb_roots::(&mut probe, &proof.roots, &[]); + let z: FieldElement = probe.sample_field_element(); + let alpha: FieldElement = probe.sample_field_element(); + let owed = crate::compute_commit_bus_offset(&public_output, 0, &z, &alpha) + .expect("the commit offset"); + let verdict = multilinear_table::multi_verify_batched::<_, _, _, H>( + &proof, + &statements, + &group_layouts, + &domains, + &sizes, + &owed, + &config, + &mut transcript, + None, + ); + all_bytes.push(bytes); + verdicts.push(verdict); + } + (verdicts, all_bytes) + }); + let (verdicts, bytes) = verdict; + let verdict = verdicts + .into_iter() + .collect::, _>>() + .map(|_| ()); + (verdict, names, bytes) +} + +/// The VM's AIR labels, as `lean_program_census::vm_airs` names them: what a +/// table's name reduces to (`CPU[0]` → `CPU`, `PAGE:0x…` → `PAGE`). +fn label(name: &str) -> String { + name.split(['[', ':']).next().unwrap_or(name).to_string() +} + +/// ★ Every VM table, argued batched and verified: the monolithic proofs of +/// programs that light them up between them. The labels seen must cover every +/// AIR of `lean_program_census::vm_airs` but L2G, a continuation's bookend, +/// which no multilinear proof argues. +#[test] +#[ignore = "proves programs: a box run"] +fn every_vm_table_round_trips_batched() { + let mut seen: Vec = Vec::new(); + let mut note = |names: Vec| { + for n in names { + let l = label(&n); + if !seen.contains(&l) { + seen.push(l); + } + } + }; + for name in [ + "test_keccak", + "all_instructions_64", + "test_ecsm", + "test_lb_lh_8", + "misalign_lw", + "all_loadstore_32", + ] { + let (verdict, names) = vm_round_trip(name, None); + verdict.unwrap_or_else(|e| panic!("{name}: {e:?}")); + eprintln!("batched argue {name}: VERIFIED over {} tables", names.len()); + note(names); + } + let (verdict, names, _) = vm_round_trip_elf( + "hint_min", + &rust_elf_bytes("hint_min"), + None, + &[Where::Device], + ); + verdict.unwrap_or_else(|e| panic!("hint_min: {e:?}")); + eprintln!( + "batched argue hint_min: VERIFIED over {} tables", + names.len() + ); + note(names); + let want: Vec = crate::tests::lean_program_census::vm_airs() + .into_iter() + .map(|(_, l)| l.to_string()) + .filter(|l| l != "L2G") + .collect(); + let missing: Vec<&String> = want.iter().filter(|l| !seen.contains(l)).collect(); + eprintln!( + "batched argue: {} distinct VM labels {seen:?}; census wants {}, missing {missing:?}", + seen.len(), + want.len() + ); + assert!( + missing.is_empty(), + "VM tables never argued batched: {missing:?}" + ); +} + +/// The same program with one table's factor value, one bus output, or one +/// ladder half wrong: each refused. +#[test] +#[ignore = "proves a program: a box run"] +fn a_tampered_vm_proof_is_refused() { + let tampers: [(&str, Tamper); 3] = [ + ("factor value", |p| { + p.argue.factor_values[0][0] += FieldElement::::one() + }), + ("bus output", |p| { + let last = p.argue.bus_outputs.len() - 1; + p.argue.bus_outputs[last].0 += FieldElement::::one() + }), + ("ladder half", |p| { + p.argue.gkr[0].layers[3].halves[0].q_hi += FieldElement::::one() + }), + ]; + for (what, tamper) in tampers { + let (verdict, _) = vm_round_trip("test_keccak", Some(tamper)); + assert!(verdict.is_err(), "{what}: a tampered proof verified"); + eprintln!("batched argue test_keccak: {what} tamper refused: {verdict:?}"); + } +} + +/// ★ D-BATCH B-3 on real tables: the prover on the card proves the host +/// reference's canonical bytes — the roots and the whole argue — for every +/// table of `test_keccak`, `test_ecsm` and `all_instructions_64`, and both +/// verify. The openings are not compared: they grind under the production +/// config, and the parallel nonce search makes any two runs part there. +/// +/// On a cuda build the device path runs the ladder's layers and the fused +/// zerocheck on the card; on a host build it is the device path's fallbacks +/// against the reference. +#[test] +#[ignore = "proves programs: a box run"] +fn the_card_proves_the_host_references_bytes_on_real_tables() { + for name in ["test_keccak", "test_ecsm", "all_instructions_64"] { + let elf = crate::test_utils::asm_elf_bytes(name); + let fused = multilinear::gpu_fused::fused_sessions(); + let gruen = multilinear::gkr_gruen::gruen_layers(); + let (verdict, names, bytes) = + vm_round_trip_elf(name, &elf, None, &[Where::Host, Where::Device]); + let fused = multilinear::gpu_fused::fused_sessions() - fused; + let gruen = multilinear::gkr_gruen::gruen_layers() - gruen; + verdict.unwrap_or_else(|e| panic!("{name}: {e:?}")); + let (host_bytes, card_bytes) = (&bytes[0], &bytes[1]); + assert!( + card_bytes == host_bytes, + "{name}: the card's argue is not the host reference's bytes" + ); + eprintln!( + "batched argue B-3 {name}: {} tables, {fused} fused sessions and {gruen} Gruen ladder layers on \ + the card, {} argue bytes == the host's", + names.len(), + card_bytes.len() + ); + } +} diff --git a/prover/src/tests/bitwise_bus_tests.rs b/prover/src/tests/bitwise_bus_tests.rs index 1782bd0fc..93f83e607 100644 --- a/prover/src/tests/bitwise_bus_tests.rs +++ b/prover/src/tests/bitwise_bus_tests.rs @@ -4,12 +4,10 @@ //! - Completeness: Valid lookups to BITWISE are accepted //! - Soundness: Invalid lookups to BITWISE are rejected +use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; -use math::field::element::FieldElement; - use stark::lookup::{ AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, NullBoundaryConstraintBuilder, Packing, @@ -17,7 +15,7 @@ use stark::lookup::{ use stark::proof::options::ProofOptions; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField, alu_op}; use crate::test_utils::multi_prove_ram; @@ -195,15 +193,15 @@ fn prove_and_verify(sender_lookups: &[(u8, u8, u8)]) -> bool { ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } @@ -305,15 +303,15 @@ fn prove_and_verify_custom( ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/bitwise_tests.rs b/prover/src/tests/bitwise_tests.rs index c824764d3..e4178d047 100644 --- a/prover/src/tests/bitwise_tests.rs +++ b/prover/src/tests/bitwise_tests.rs @@ -4,7 +4,7 @@ use crate::tables::bitwise::{ NUM_PRECOMPUTED_COLS, NUM_ROWS, bus_interactions, cols, generate_bitwise_row, generate_bitwise_trace, is_preprocessed, preprocessed_commitment, row_index, }; -use crate::tables::types::{BusId, FE}; +use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField}; use crate::test_utils::multi_prove_ram; use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; @@ -415,16 +415,15 @@ fn test_preprocessed_commitment_is_nonzero() { #[cfg(test)] mod soundness_tests { use super::*; - use crypto::fiat_shamir::default_transcript::DefaultTranscript; use stark::lookup::{ AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, NullBoundaryConstraintBuilder, Packing, }; use stark::proof::options::ProofOptions; - use stark::prover::{IsStarkProver, Prover}; + use stark::prover::IsStarkProver; use stark::trace::TraceTable; use stark::traits::AIR; - use stark::verifier::{IsStarkVerifier, Verifier}; + use stark::verifier::IsStarkVerifier; use crate::tables::types::{GoldilocksExtension, GoldilocksField}; @@ -558,8 +557,14 @@ mod soundness_tests { let dummy_air = create_receiver_air(proof_options); // Use the prover's commitment computation (3 precomputed cols: X, Y, AND) - Prover::compute_precomputed_commitment_for_testing(trace, &dummy_air, 3) - .expect("Failed to compute commitment") + // — the PINNED prover, so the commitment the AIRs declare is built under + // the same hash `multi_prove_ram` recomputes it with. The workspace + // `Prover` alias is BLAKE3 whatever the pin says, and under an algebraic + // pin it makes the honest arm fail with PrecomputedCommitmentMismatch. + crate::hash_pin::BlockProver::compute_precomputed_commitment_for_testing( + trace, &dummy_air, 3, + ) + .expect("Failed to compute commitment") } fn create_sender_trace(x: u8, y: u8, claimed_result: u8) -> TraceTable { @@ -626,15 +631,15 @@ mod soundness_tests { ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - let result = Verifier::multi_verify( + let result = crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ); @@ -674,15 +679,15 @@ mod soundness_tests { ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - let result = Verifier::multi_verify( + let result = crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ); @@ -744,16 +749,16 @@ mod soundness_tests { ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); // Verifier uses DIFFERENT AIR with honest commitment let verifier_airs: Vec<&dyn AIR> = vec![&sender_air, &verifier_receiver_air]; - let result = Verifier::multi_verify( + let result = crate::hash_pin::BlockVerifier::multi_verify( &verifier_airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ); @@ -764,3 +769,110 @@ mod soundness_tests { ); } } + +// ========================================================================= +// The precomputed columns' closed-form multilinear extension +// ========================================================================= +// +// A recursive verifier on the multilinear path cannot afford +// `check_preprocessed`'s 2^20 fold per column, and BITWISE is eleven of the +// sixteen preprocessed columns a real epoch carries. `preprocessed_mle_at` +// replaces that fold with a closed form. These two tests are what makes the +// closed form trustworthy, and both of them can fail: +// +// * the cube test pins the VARIABLE CONVENTION against the row generator — +// it fails if any bit is read at the wrong position, which is the mistake +// a random-point test would also catch but would not localise; +// * the random-point test is the real gate: it compares the closed form +// against the host's own `Mle::evaluate_in` over the actual 2^20-row +// column, which is the function the verifier would otherwise run. + +/// A fixed-seed xorshift, so a failure names one reproducible point. +fn sample_point(seed: u64, num_vars: usize) -> Vec> { + let mut state = seed | 1; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + // Goldilocks is 2^64 - 2^32 + 1; the high bits are cheap to shed and + // nothing here needs uniformity, only unpredictability to the author. + FE::from(state >> 2) + }; + (0..num_vars) + .map(|_| FieldElement::::new([next(), next(), next()])) + .collect() +} + +/// The closed form agrees with the row generator on the hypercube's corners, +/// which is what pins which coordinate carries which index bit. +#[test] +fn the_bitwise_preprocessed_closed_form_agrees_with_the_row_generator_on_the_cube() { + let vars = crate::tables::bitwise::NUM_VARS; + // Asymmetric indices: a palindrome or a constant would survive a reversed + // variable order, so none of these is one. + for index in [ + 0usize, 1, 2, 0x0_0001, 0x0_ABCD, 0x1_2345, 0xF_FFFF, 0x8_0000, + ] { + // `Mle::evaluate_in` binds its first coordinate to the HIGH half, so + // coordinate i is index bit `vars - 1 - i`. + let point: Vec> = (0..vars) + .map(|i| { + let bit = (index >> (vars - 1 - i)) & 1; + FieldElement::::new([ + FE::from(bit as u64), + FE::zero(), + FE::zero(), + ]) + }) + .collect(); + let got = crate::tables::bitwise::preprocessed_mle_at(&point) + .expect("the point is NUM_VARS long"); + let want = generate_bitwise_row(index); + for col in 0..NUM_PRECOMPUTED_COLS { + let expected = FieldElement::::new([ + FE::from(want[col]), + FE::zero(), + FE::zero(), + ]); + assert_eq!( + got[col], expected, + "column {col} at row index {index:#07x}: the closed form and \ + generate_bitwise_row disagree on the cube" + ); + } + } +} + +/// ★ F3. The closed form equals the host's own multilinear fold over the real +/// 2^20-row columns, at points off the hypercube. +/// +/// This is the test the recursion's preprocessed-check design rests on: if it +/// passes, a verifier may evaluate these columns in O(20) operations instead of +/// O(2^20) and bind exactly what `check_preprocessed` binds today. +#[test] +fn the_bitwise_preprocessed_closed_form_matches_the_host_fold_at_random_points() { + let vars = crate::tables::bitwise::NUM_VARS; + let columns = crate::tables::bitwise::preprocessed_columns(); + assert_eq!(columns.len(), NUM_PRECOMPUTED_COLS); + assert_eq!(columns[0].len(), NUM_ROWS); + let mles: Vec> = columns + .into_iter() + .map(|values| multilinear::mle::Mle::new(values).expect("a power-of-two column")) + .collect(); + + for seed in [0x5eed_0001u64, 0x5eed_0002, 0x5eed_0003] { + let point = sample_point(seed, vars); + let got = crate::tables::bitwise::preprocessed_mle_at(&point) + .expect("the point is NUM_VARS long"); + for (col, mle) in mles.iter().enumerate() { + let want = mle + .evaluate_in::(&point) + .expect("the column has NUM_VARS variables"); + assert_eq!( + got[col], want, + "column {col} at seed {seed:#x}: the closed form and the host's \ + 2^20 fold disagree" + ); + } + } +} diff --git a/prover/src/tests/block_batched_tests.rs b/prover/src/tests/block_batched_tests.rs new file mode 100644 index 000000000..df3aed852 --- /dev/null +++ b/prover/src/tests/block_batched_tests.rs @@ -0,0 +1,716 @@ +//! The block argued under the batched format (`BlockFormat::argue` = +//! `ArgueFormat::Batched`): each group's tables in one argue on the group's +//! fork — a lockstep GKR per bin, one constraint sumcheck — beside the +//! per-table default. Round trips, the format read from the verifier's side +//! only, one refusal per way a group's argue can be wrong (each new check with +//! its mutation), the default's bytes, and the card against the host. + +use crate::block_whir::{ + self, BlockFormat, BlockOptions, BlockWhirProof, Deviations, prove_block_whir_with, + verify_block_whir_with, +}; +use crate::tables::MaxRowsConfig; +use crate::test_utils::{E, asm_elf_bytes}; +use crate::zf_format::ZfFormat; +use math::field::element::FieldElement; +use multilinear::whir_chain::{ArgueFormat, StackVars}; +use stark::multilinear_block::ArgueDeviation; +use stark::multilinear_table::batched::{ProverFaults, VerifierChecks, Where}; +use stark::proof::options::ProofOptions; + +/// The production format; one group holds a small program whole. +fn one_group(argue: ArgueFormat) -> BlockFormat { + BlockFormat { + zf: ZfFormat::DEFAULT, + group_polys: block_whir::BLOCK_GROUP_POLYS, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +/// A stack of 2^10 and two polynomials a group: a small program spans several +/// groups. +fn many_groups(argue: ArgueFormat) -> BlockFormat { + let mut zf = ZfFormat::DEFAULT; + zf.whir_stack = StackVars::new(10).expect("a valid stack"); + BlockFormat { + zf, + group_polys: 2, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +const BATCHED: ArgueFormat = ArgueFormat::BATCHED; +const PER_TABLE: ArgueFormat = ArgueFormat::PerTable; + +fn options(max_rows: MaxRowsConfig, keccak_rnd_rows_log2: usize) -> BlockOptions { + BlockOptions { + max_rows, + keccak_rnd_rows_log2, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: block_whir::BLOCK_COPY_ROWS_LOG2, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(3), + window_log2: None, + drop_streamed_ops: false, + layout_workers: 0, + layout_ahead: Some(2), + // Every table packed on a card, so a box run walks the narrow path at a + // test's size. + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + upload_ahead: true, + memlog: false, + // The rest laid out in waves of 1 MiB: several at a test's size. + finish_keccak_rnd_chunks: true, + finish_cuts: true, + finish_stream: true, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + pack_finished: true, + gpack: true, + regen: None, + phase_b_overlap: stark::multilinear_block::PhaseBOverlap::Off, + } +} + +fn prove_with( + elf: &[u8], + format: &BlockFormat, + options: &BlockOptions, + deviations: &Deviations, +) -> BlockWhirProof { + prove_block_whir_with( + elf, + &[], + &ProofOptions::default_test_options(), + format, + options, + deviations, + ) + .expect("prove") + .0 +} + +fn prove(elf: &[u8], format: &BlockFormat, options: &BlockOptions) -> BlockWhirProof { + prove_with(elf, format, options, &Deviations::default()) +} + +fn verify_with( + proof: &BlockWhirProof, + elf: &[u8], + format: &BlockFormat, + skip_prepared: bool, + checks: VerifierChecks, +) -> bool { + verify_block_whir_with( + proof, + elf, + &ProofOptions::default_test_options(), + format, + skip_prepared, + checks, + ) + .unwrap_or(false) +} + +fn verify(proof: &BlockWhirProof, elf: &[u8], format: &BlockFormat) -> bool { + verify_with(proof, elf, format, false, VerifierChecks::ALL) +} + +/// A proof argued with `faults` in group `group`. +fn faulted( + elf: &[u8], + format: &BlockFormat, + o: &BlockOptions, + group: usize, + faults: ProverFaults, +) -> BlockWhirProof { + prove_with( + elf, + format, + o, + &Deviations { + argue: ArgueDeviation { + group: Some(group), + faults, + at: Where::Device, + }, + ..Default::default() + }, + ) +} + +/// A group of `proof` and the position in it of a table shorter than the +/// group's tallest, with variables and with constraints — what a fault on the +/// padding or on `ξ`'s prefix needs: at no variables the prefix and the suffix +/// are both empty, and with no constraint `ξ` weighs nothing. +fn group_with_a_short_table( + proof: &BlockWhirProof, + elf: &[u8], + format: &BlockFormat, +) -> (usize, usize) { + let frame = block_whir::block_frame( + proof.statement(), + elf, + &ProofOptions::default_test_options(), + format, + ) + .expect("the statement's frame"); + let airs = frame.airs.air_refs(); + let constrained = |t: usize| { + let (width, num_vars) = frame.shapes[t]; + crate::multilinear_prove::layout_of(airs[t], width, num_vars) + .expect("a layout") + .statement() + .shape + .num_roots() + > 0 + }; + proof + .groups + .iter() + .enumerate() + .find_map(|(g, tables)| { + let height = |t: u32| proof.table_num_vars[t as usize]; + let tallest = tables.iter().map(|&t| height(t)).max()?; + tables + .iter() + .position(|&t| height(t) > 0 && height(t) < tallest && constrained(t as usize)) + .map(|position| (g, position)) + }) + .expect("a group with a short constrained table") +} + +/// ★ The block round-trips under the batched argue at one group and at many, +/// carrying one argue a group and no per-table argument. The format is the +/// verifier's: a batched proof under the per-table format is refused, and the +/// reverse. +#[test] +fn a_block_argued_batched_proves_and_verifies() { + let elf = asm_elf_bytes("sub"); + let proof = prove( + &elf, + &one_group(BATCHED), + &options(MaxRowsConfig::default(), 16), + ); + assert_eq!(proof.argues.len(), 1); + assert!(proof.proof.tables.is_empty()); + assert!(verify(&proof, &elf, &one_group(BATCHED))); + assert!(!verify(&proof, &elf, &one_group(PER_TABLE))); + + let elf = asm_elf_bytes("all_instructions_64"); + let o = options(MaxRowsConfig::small(), 16); + let proof = prove(&elf, &many_groups(BATCHED), &o); + assert!(proof.groups.len() >= 3, "{} groups", proof.groups.len()); + assert_eq!(proof.argues.len(), proof.groups.len()); + assert!(verify(&proof, &elf, &many_groups(BATCHED))); + assert!(!verify(&proof, &elf, &many_groups(PER_TABLE))); + // A cap the verifier does not hold bins the trees otherwise. + let other_cap = BlockFormat { + argue: ArgueFormat::batched(0), + ..many_groups(BATCHED) + }; + assert!(!verify(&proof, &elf, &other_cap)); + + let per_table = prove(&elf, &many_groups(PER_TABLE), &o); + assert!(per_table.argues.is_empty()); + assert!(verify(&per_table, &elf, &many_groups(PER_TABLE))); + assert!(!verify(&per_table, &elf, &many_groups(BATCHED))); +} + +/// KECCAK_RND cut into many chunks, batched: many trees a bin. No VM table +/// reads a shifted row, so no table keeps a claim reduction: each reads its +/// columns at the sumcheck's point directly. +#[test] +fn keccak_chunks_argue_batched() { + let elf = asm_elf_bytes("test_keccak"); + let format = many_groups(BATCHED); + let proof = prove(&elf, &format, &options(MaxRowsConfig::small(), 3)); + assert!(proof.table_counts.keccak_rnd >= 2); + let tables: usize = proof.argues.iter().map(|a| a.bus_outputs.len()).sum(); + assert_eq!(tables, proof.table_num_vars.len()); + assert!( + proof + .argues + .iter() + .flat_map(|a| &a.reduces) + .all(Option::is_none), + "a VM table kept a reduction" + ); + assert!(verify(&proof, &elf, &format)); +} + +/// The prepared openings settle under the batched argue as under the +/// per-table one: the dense fixture verifies, two groups' openings swapped are +/// refused, and with the openings unchecked (the mutation) the swap verifies. +#[test] +fn prepared_openings_settle_under_the_batched_argue() { + let elf = asm_elf_bytes("test_dense_pages"); + let o = options(MaxRowsConfig::default(), 16); + let proof = prove(&elf, &one_group(BATCHED), &o); + assert!(!proof.prepared.is_empty()); + assert!(verify(&proof, &elf, &one_group(BATCHED))); + + let format = many_groups(BATCHED); + let mut proof = prove(&elf, &format, &o); + assert!(proof.prepared.len() >= 2, "{} stacks", proof.prepared.len()); + assert!(verify(&proof, &elf, &format)); + proof.prepared.swap(0, 1); + assert!(!verify(&proof, &elf, &format)); + assert!(verify_with( + &proof, + &elf, + &format, + true, + VerifierChecks::ALL + )); +} + +/// ★ A group argued wrongly is refused: a cross-table weight off by `λ`, a +/// table claim absorbed off by one, `ξ`'s suffix for a short table, a short +/// table padded the `2^Δ` way, and the trees binned under another cap. +#[test] +fn a_batched_group_argued_wrongly_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(BATCHED); + let o = options(MaxRowsConfig::small(), 16); + let honest = prove(&elf, &format, &o); + assert!(verify(&honest, &elf, &format)); + let (g, short) = group_with_a_short_table(&honest, &elf, &format); + let faults = [ + ( + "weight", + ProverFaults { + wrong_weight: Some(0), + ..Default::default() + }, + ), + ( + "claim", + ProverFaults { + tamper_claim: Some(short), + ..Default::default() + }, + ), + ( + "suffix of xi", + ProverFaults { + suffix_xi: Some(short), + ..Default::default() + }, + ), + ( + "scaled padding", + ProverFaults { + scaled_padding: Some(short), + ..Default::default() + }, + ), + ( + "bins", + ProverFaults { + bin_log_cells: Some(0), + ..Default::default() + }, + ), + ]; + for (name, fault) in faults { + let proof = faulted(&elf, &format, &o, g, fault); + assert!(!verify(&proof, &elf, &format), "{name} in group {g}"); + } +} + +/// A batched argue tampered in the proof is refused: a bus output moved, one +/// missing, a factor value moved, two groups' argues swapped, and a reduction +/// where the format has none. The last is admitted with the reduction's shape +/// unchecked (the mutation). +#[test] +fn a_tampered_batched_argue_is_refused() { + let elf = asm_elf_bytes("test_keccak"); + let format = many_groups(BATCHED); + let honest = prove(&elf, &format, &options(MaxRowsConfig::small(), 3)); + assert!(verify(&honest, &elf, &format)); + let tampered = |tamper: &dyn Fn(&mut BlockWhirProof)| { + let mut proof = honest.clone(); + tamper(&mut proof); + proof + }; + let one = FieldElement::::one(); + assert!(!verify( + &tampered(&|p| p.argues[0].bus_outputs[0].0 += one), + &elf, + &format + )); + assert!(!verify( + &tampered(&|p| { + p.argues[0].bus_outputs.pop(); + }), + &elf, + &format + )); + assert!(!verify( + &tampered(&|p| p.argues[1].factor_values[0][0] += one), + &elf, + &format + )); + assert!(!verify(&tampered(&|p| p.argues.swap(0, 1)), &elf, &format)); + // A reduction on a table that reads no shifted row (no VM table does). + let extra = tampered(&|p| { + p.argues[0].reduces[0] = Some(multilinear::claim_reduce::ReduceProof { + sumcheck: multilinear::sumcheck::SumcheckProof { rounds: Vec::new() }, + column_values: Vec::new(), + }); + }); + assert!(!verify(&extra, &elf, &format)); + let inert = VerifierChecks { + reduce_shape: false, + ..VerifierChecks::ALL + }; + assert!(verify_with(&extra, &elf, &format, false, inert)); +} + +/// A batched group argued on another group's fork is refused: the index each +/// fork absorbs ties the argue to its group. +#[test] +fn a_batched_group_on_another_fork_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(BATCHED); + let swapped = prove_with( + &elf, + &format, + &options(MaxRowsConfig::small(), 16), + &Deviations { + fork_of: Some(|g| g ^ 1), + ..Default::default() + }, + ); + assert!(swapped.groups.len() >= 2); + assert!(!verify(&swapped, &elf, &format)); +} + +/// ★ The bus balance, summed over every group's batched outputs, binds the +/// block: a prover leaving one KECCAK_RND chunk out — every table it proves +/// consistent — is refused, and admitted with the balance unchecked (the +/// mutation): the block-wide sum is what refuses it. +#[test] +fn the_bus_balance_binds_the_batched_block() { + let elf = asm_elf_bytes("test_keccak"); + let format = many_groups(BATCHED); + let o = options(MaxRowsConfig::small(), 3); + let honest = prove(&elf, &format, &o); + let short = prove_with( + &elf, + &format, + &o, + &Deviations { + omit_first_keccak_rnd: true, + ..Default::default() + }, + ); + assert_eq!( + short.table_counts.keccak_rnd + 1, + honest.table_counts.keccak_rnd + ); + assert!(!verify(&short, &elf, &format)); + let inert = VerifierChecks { + balance: false, + ..VerifierChecks::ALL + }; + assert!(verify_with(&short, &elf, &format, false, inert)); +} + +/// A program's traces, built once, KECCAK_RND split at 2^`keccak_rnd_log2`. +fn fixed_traces( + name: &str, + max_rows: &MaxRowsConfig, + keccak_rnd_log2: usize, +) -> ( + executor::elf::Elf, + Vec, + crate::tables::trace_builder::Traces, +) { + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf = asm_elf_bytes(name); + let program = Elf::load(&elf).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let mut traces = + crate::tables::trace_builder::Traces::from_elf_and_logs(&program, &logs, max_rows, &[]) + .expect("the traces build"); + block_whir::split_keccak_rnd( + &mut traces, + keccak_rnd_log2, + crate::tables::trace_builder::CutPadding::default(), + ) + .expect("the split"); + (program, elf, traces) +} + +/// `traces` with every table that has no preprocessed column sorted row by +/// row: the same multiset of rows, in an order no run can change. A trace +/// build is not reproducible between processes (some tables come out of a +/// `HashMap` in its order), so a proof's bytes can only be compared across +/// builds over canonical rows. No VM table reads another row and every bus is a +/// multiset, so the sorted traces are still a block's, and they verify. +fn canonical_rows(program: &executor::elf::Elf, traces: &mut crate::tables::trace_builder::Traces) { + let counts = traces.table_counts(); + let airs = crate::VmAirs::new( + program, + &ProofOptions::default_test_options(), + false, + &traces.page_configs, + &counts, + &crate::cpu_heights_from_traces(traces), + None, + true, + None, + crate::test_utils::halted_register_commitment( + &ProofOptions::default_test_options(), + program, + &traces.public_output_bytes, + ) + .preprocessed(), + ); + for (air, trace, _) in airs.air_trace_pairs(traces) { + if air.num_precomputed_columns() > 0 { + continue; + } + let columns = trace.columns_main(); + let height = columns.first().map_or(0, Vec::len); + let mut rows: Vec> = (0..height) + .map(|r| columns.iter().map(|c| c[r].canonical()).collect()) + .collect(); + rows.sort_unstable(); + let sorted: Vec>> = (0..columns.len()) + .map(|c| rows.iter().map(|row| FieldElement::from(row[c])).collect()) + .collect(); + trace.main_table = stark::table::Table::from_columns(sorted); + } +} + +/// The proof of fixed traces, non-streamed, under `deviations`. +fn prove_fixed( + program: &executor::elf::Elf, + elf: &[u8], + traces: &mut crate::tables::trace_builder::Traces, + format: &BlockFormat, + o: &BlockOptions, + deviations: &Deviations, +) -> BlockWhirProof { + block_whir::prove_traces( + program, + elf, + traces, + &ProofOptions::default_test_options(), + format, + o, + deviations, + false, + &|_, _| {}, + &mut Default::default(), + ) + .expect("prove") +} + +/// A digest of what the default format's proof carries: its multilinear +/// proof, its prepared openings and its partition, each serialized. +fn default_digest(proof: &BlockWhirProof) -> String { + let mut bytes = Vec::new(); + for part in [ + rkyv::to_bytes::(&proof.proof) + .expect("serialize") + .to_vec(), + rkyv::to_bytes::(&proof.prepared) + .expect("serialize") + .to_vec(), + rkyv::to_bytes::(&proof.groups) + .expect("serialize") + .to_vec(), + ] { + bytes.extend_from_slice(&(part.len() as u64).to_le_bytes()); + bytes.extend_from_slice(&part); + } + crate::statement::elf_digest(&bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// ★ The per-table default keeps today's bytes: the default format's proof of +/// fixed traces over canonical rows ([`canonical_rows`]) — its multilinear +/// proof, its prepared openings and its partition — digests to what the block +/// prover before the batched format proves over the same rows, with a +/// deterministic grind. The batched argue's only mark on a default proof is +/// [`BlockWhirProof::argues`], empty. +/// +/// The digests depend on the ELF bytes (the statement absorbs the program's +/// digest), and the guests' bytes differ between toolchains, so the expected +/// digests are not constants here: `BLOCK_DEFAULT_DIGESTS` carries them +/// (`name=hex;name=hex;…`), proved on the same machine and ELFs by the same +/// body at the base sha (the box gate does both). A record: at noepoch/whir @ +/// 11e2de53d, on the laptop's ELFs (all_instructions_64 md5 9ded1481…, +/// test_dense_pages b753fbd8…, sub ceefc702…), the host proves 6e0bd41e…, +/// 3b7726b1…, ca028342…, the same in two runs, and this branch proves the same. +/// +/// Run alone, with the grind deterministic (it is read once per process): +/// +/// ```text +/// LAMBDA_VM_DETERMINISTIC_GRIND=1 BLOCK_DEFAULT_DIGESTS='all_instructions_64=…;test_dense_pages=…;sub=…' \ +/// cargo test --release -p lambda-vm-prover --lib -- \ +/// tests::block_batched_tests::the_default_format_keeps_todays_bytes --exact --ignored --nocapture +/// ``` +#[test] +#[ignore = "needs LAMBDA_VM_DETERMINISTIC_GRIND=1 and BLOCK_DEFAULT_DIGESTS, run alone"] +fn the_default_format_keeps_todays_bytes() { + assert!( + crypto::grinding::deterministic(), + "export LAMBDA_VM_DETERMINISTIC_GRIND=1" + ); + let given = std::env::var("BLOCK_DEFAULT_DIGESTS") + .expect("BLOCK_DEFAULT_DIGESTS: the base sha's digests, name=hex;…"); + let expected = |name: &str| -> String { + given + .split(';') + .filter_map(|pair| pair.trim().split_once('=')) + .find(|(n, _)| *n == name) + .map(|(_, hex)| hex.to_string()) + .unwrap_or_else(|| panic!("BLOCK_DEFAULT_DIGESTS has no digest for {name}")) + }; + let cases = [ + ( + "all_instructions_64", + many_groups(PER_TABLE), + MaxRowsConfig::small(), + ), + ( + "test_dense_pages", + one_group(PER_TABLE), + MaxRowsConfig::default(), + ), + ("sub", many_groups(PER_TABLE), MaxRowsConfig::small()), + ]; + for (name, format, max_rows) in cases { + let (program, elf, mut traces) = fixed_traces(name, &max_rows, 16); + canonical_rows(&program, &mut traces); + let o = options(max_rows, 16); + let proof = prove_fixed( + &program, + &elf, + &mut traces, + &format, + &o, + &Deviations::default(), + ); + assert!(verify(&proof, &elf, &format), "{name}"); + assert!(proof.argues.is_empty(), "{name}"); + let digest = default_digest(&proof); + println!("BLOCK DEFAULT DIGEST {name}: {digest}"); + assert_eq!( + digest, + expected(name), + "{name}: the default format's bytes moved" + ); + } +} + +/// ★ N-2's card gate (box only): every group's batched argue on the card +/// proves the host reference's bytes, over one commitment — the same fixed +/// traces proved twice, non-streamed, with a deterministic grind, so the whole +/// proof is compared — and both verify. On a device the fused sessions, the +/// device rounds and the ladder's Gruen layers are counted, so the comparison +/// is not of the host with itself. +/// +/// ```text +/// LAMBDA_VM_DETERMINISTIC_GRIND=1 cargo test --release -p lambda-vm-prover --features cuda --lib -- \ +/// tests::block_batched_tests::the_card_argues_the_hosts_bytes --exact --ignored --nocapture +/// ``` +#[test] +#[ignore = "box only: the card against the host"] +fn the_card_argues_the_hosts_bytes() { + assert!( + crypto::grinding::deterministic(), + "export LAMBDA_VM_DETERMINISTIC_GRIND=1" + ); + let device = multilinear::gpu::reserve_budget() > 0; + // Tall enough for the card (trees of 2^14 input cells and up, fused rounds + // past 2^7) and, under a stack of 2^18 and two polynomials a group, spread + // over several groups. + let mut zf = ZfFormat::DEFAULT; + zf.whir_stack = StackVars::new(18).expect("a valid stack"); + let format = BlockFormat { + zf, + group_polys: 2, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: BATCHED, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + }; + for name in ["all_instructions_64", "test_keccak", "test_ecsm"] { + let (program, elf, mut traces) = fixed_traces(name, &MaxRowsConfig::default(), 16); + let o = options(MaxRowsConfig::default(), 16); + let mut run = |at: Where| { + let fused = multilinear::gpu_fused::fused_sessions(); + let rounds = multilinear::gpu::sumcheck_rounds(); + let gruen = multilinear::gkr_gruen::gruen_layers(); + let proof = prove_fixed( + &program, + &elf, + &mut traces, + &format, + &o, + &Deviations { + argue: ArgueDeviation { + at, + ..Default::default() + }, + ..Default::default() + }, + ); + assert!(verify(&proof, &elf, &format), "{name} {at:?}"); + ( + rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec(), + proof.groups.len(), + multilinear::gpu_fused::fused_sessions() - fused, + multilinear::gpu::sumcheck_rounds() - rounds, + multilinear::gkr_gruen::gruen_layers() - gruen, + ) + }; + let host = run(Where::Host); + let card = run(Where::Device); + println!( + "BLOCK BATCHED CARD {name}: groups {} · device {device} · fused sessions {} · device rounds {} · Gruen ladder layers {} · {} bytes · {}", + card.1, + card.2, + card.3, + card.4, + card.0.len(), + if host.0 == card.0 { "EQUAL" } else { "DIFFER" } + ); + assert_eq!(host.2, 0, "{name}: the host reference ran no fused session"); + if device { + assert!(card.2 > 0, "{name}: the card ran fused sessions"); + assert!(card.3 > 0, "{name}: the card ran rounds"); + assert!(card.4 > 0, "{name}: the ladder ran Gruen's rounds"); + } + assert!(host.0 == card.0, "{name}: the card's bytes differ"); + } +} diff --git a/prover/src/tests/block_cut_padding_tests.rs b/prover/src/tests/block_cut_padding_tests.rs new file mode 100644 index 000000000..b17ce398a --- /dev/null +++ b/prover/src/tests/block_cut_padding_tests.rs @@ -0,0 +1,357 @@ +//! The block proved with its cut tables stopped at the last real row +//! (`CutPadding::LastTable`, the default; `LAMBDA_VM_BLOCK_CUT_PADDING=run` +//! pads the whole run): the verifier accepts the block without its tables of +//! padding rows alone, and refuses a block one table short of the real rows on +//! the bus balance, which is its only objection. The streamed prove counts the +//! tables of the rule the process runs under, and phase B builds every one it +//! dropped again. +//! +//! Box tier: each test proves several small blocks and takes minutes on a CPU +//! runner, so all three are `#[ignore]`d. Run them with +//! `--ignored --exact --test-threads=1 tests::block_cut_padding_tests::`, +//! the last one once with the variable unset and once at `run`. + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use multilinear::whir_chain::{ArgueFormat, StackVars}; +use stark::multilinear_table::batched::VerifierChecks; +use stark::proof::options::ProofOptions; + +use crate::block_whir::{ + self, BlockFormat, BlockOptions, BlockRefusal, BlockWhirProof, Deviations, + prove_block_whir_with, verify_block_whir, verify_block_whir_traced, +}; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::{CutPadding, Traces, WindowedTraceBuilder}; +use crate::test_utils::asm_elf_bytes; +use crate::zf_format::ZfFormat; + +/// A stack of 2^10 and two polynomials a group: a small program spans several +/// groups. +fn many_groups(argue: ArgueFormat) -> BlockFormat { + let mut zf = ZfFormat::DEFAULT; + zf.whir_stack = StackVars::new(10).expect("a valid stack"); + BlockFormat { + zf, + group_polys: 2, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +/// The production options at a test's size: chunked tables of `max_rows`, +/// KECCAK_RND cut at 2^5 rows, KECCAK and ECSM at one row, ECDAS at 2^4, so +/// each cut family of a small program has a table of padding rows alone with +/// the whole run padded. +fn options( + max_rows: MaxRowsConfig, + window_log2: Option, + layout_workers: usize, +) -> BlockOptions { + BlockOptions { + max_rows, + keccak_rnd_rows_log2: 5, + keccak_rows_log2: 0, + ecsm_rows_log2: 0, + ecdas_rows_log2: 4, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(3), + window_log2, + drop_streamed_ops: layout_workers > 0, + layout_workers, + layout_ahead: Some(2), + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + memlog: false, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + regen: None, + ..BlockOptions::production() + } +} + +/// `program` run, and its traces as the block's windowed build makes them at +/// [`options`]' cuts, the cut tables padded as `padding`, held wide. +fn traces(program: &str, padding: CutPadding) -> (Elf, Vec, Traces) { + let elf = asm_elf_bytes(program); + let program = Elf::load(&elf).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let o = options(MaxRowsConfig::default(), None, 0); + let mut builder = WindowedTraceBuilder::new(&program, &[], &o.max_rows) + .expect("the builder") + .keccak_rnd_chunks_at_finish(1 << o.keccak_rnd_rows_log2) + .expect("rows") + .cuts_at_finish( + 1 << o.keccak_rows_log2, + 1 << o.ecsm_rows_log2, + 1 << o.ecdas_rows_log2, + 1 << o.copy_rows_log2, + ) + .expect("rows") + .cut_padding(padding); + let window = 1000; + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend(builder.push(w).expect("a window")); + } + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + (program, elf, traces) +} + +/// The block over `traces`. +fn prove(program: &Elf, elf: &[u8], traces: &mut Traces, format: &BlockFormat) -> BlockWhirProof { + block_whir::prove_traces( + program, + elf, + traces, + &ProofOptions::default_test_options(), + format, + &options(MaxRowsConfig::default(), None, 0), + &Deviations::default(), + false, + &|_, _| {}, + &mut Default::default(), + ) + .expect("prove") +} + +/// The four cut families' counts in a statement: KECCAK, KECCAK_RND, ECSM, +/// ECDAS. +fn cut_counts(proof: &BlockWhirProof) -> [usize; 4] { + let c = &proof.table_counts; + [c.keccak, c.keccak_rnd, c.ecsm, c.ecdas] +} + +/// ★ The verifier, as it is, accepts a block without its tables of padding +/// rows alone. test_keccak_multi (3 permutations: 3 KECCAK rows, 72 KECCAK_RND +/// rows), per table and batched, and test_ecsm_multi (3 scalar +/// multiplications: 3 ECSM rows, 42 ECDAS steps), batched, each proved with +/// the whole run padded and cut at the last real row: both verify, and the +/// second states one table fewer of each cut family the program has. +#[test] +#[ignore = "proves small blocks, minutes on a CPU runner; box tier"] +fn a_block_without_its_padding_tables_proves_and_verifies() { + let opts = ProofOptions::default_test_options(); + for (program, argue, whole_run, last_table) in [ + ( + "test_keccak_multi", + ArgueFormat::PerTable, + [4, 4, 0, 0], + [3, 3, 0, 0], + ), + ( + "test_keccak_multi", + ArgueFormat::BATCHED, + [4, 4, 0, 0], + [3, 3, 0, 0], + ), + ( + "test_ecsm_multi", + ArgueFormat::BATCHED, + [0, 0, 4, 4], + [0, 0, 3, 3], + ), + ] { + let format = many_groups(argue); + let what = format!("{program} {argue:?}"); + let mut tables = Vec::new(); + for (padding, counts) in [ + (CutPadding::WholeRun, whole_run), + (CutPadding::LastTable, last_table), + ] { + let (elf_program, elf, mut traces) = traces(program, padding); + let proof = prove(&elf_program, &elf, &mut traces, &format); + assert_eq!(cut_counts(&proof), counts, "{what} {padding:?}: the counts"); + assert_eq!( + verify_block_whir(&proof, &elf, &opts, &format).map_err(|e| format!("{e:?}")), + Ok(true), + "{what} {padding:?}" + ); + tables.push(proof.table_num_vars.len()); + } + assert_eq!( + tables[0] - tables[1], + 2, + "{what}: two tables of padding rows alone left out" + ); + println!( + "BLOCK CUT PADDING ACCEPTED: {what}: {} tables with the whole run padded, {} cut at \ + the last real row, both verified", + tables[0], tables[1] + ); + } +} + +/// ★ One table short of the real rows is refused, and on the bus balance +/// alone: for each cut family (KECCAK_RND per table and batched, the other +/// three batched), the block cut at the last real row with that family's last +/// table — which holds real rows — left out of the traces, the counts and the +/// heights. The verifier's refusal is `BusImbalance`; with the balance check +/// off (the mutation) it accepts the same proof, so no other check objects to +/// the missing table. +#[test] +#[ignore = "proves small blocks, minutes on a CPU runner; box tier"] +fn a_block_one_table_short_of_the_real_rows_is_refused_on_the_bus() { + let opts = ProofOptions::default_test_options(); + type Family = fn( + &mut Traces, + ) -> &mut Vec< + stark::trace::TraceTable, + >; + let families: [(&str, &str, ArgueFormat, Family); 5] = [ + ( + "test_keccak_multi", + "KECCAK_RND", + ArgueFormat::PerTable, + |t| &mut t.keccak_rnds, + ), + ( + "test_keccak_multi", + "KECCAK_RND", + ArgueFormat::BATCHED, + |t| &mut t.keccak_rnds, + ), + ("test_keccak_multi", "KECCAK", ArgueFormat::BATCHED, |t| { + &mut t.keccaks + }), + ("test_ecsm_multi", "ECSM", ArgueFormat::BATCHED, |t| { + &mut t.ecsms + }), + ("test_ecsm_multi", "ECDAS", ArgueFormat::BATCHED, |t| { + &mut t.ecdases + }), + ]; + for (program, family, argue, tables) in families { + let format = many_groups(argue); + let what = format!("{program} {family} {argue:?}"); + let (elf_program, elf, mut traces) = traces(program, CutPadding::LastTable); + assert_eq!( + tables(&mut traces).len(), + 3, + "{what}: tables with a real row" + ); + tables(&mut traces).pop(); + let short = prove(&elf_program, &elf, &mut traces, &format); + let refusal = + verify_block_whir_traced(&short, &elf, &opts, &format, false, VerifierChecks::ALL) + .unwrap_or_else(|e| panic!("{what}: the statement did not frame: {e:?}")); + assert!( + matches!( + refusal, + Err(BlockRefusal::Proof(multilinear::Error::BusImbalance)) + ), + "{what}: refused for another reason, or accepted: {refusal:?}" + ); + let unbalanced = verify_block_whir_traced( + &short, + &elf, + &opts, + &format, + false, + VerifierChecks { + balance: false, + ..VerifierChecks::ALL + }, + ) + .unwrap_or_else(|e| panic!("{what}: the statement did not frame: {e:?}")); + assert!( + unbalanced.is_ok(), + "{what}: with the balance check off another check refused: {unbalanced:?}" + ); + println!( + "BLOCK CUT PADDING REFUSED: {what}: one table short of the real rows, refused as \ + BusImbalance; accepted with the balance check off" + ); + } +} + +/// ★ The streamed prove, as the block runs it, counts the tables of the rule +/// the process runs under (`LAMBDA_VM_BLOCK_CUT_PADDING`): test_keccak_multi's +/// KECCAK_RND is four tables of 2^5 rows with the whole run padded and three +/// cut at the last real row (KECCAK, ECSM and ECDAS at their production cuts: +/// one KECCAK table). Proved with the finish monolithic, with the finish +/// streamed, and with live regeneration dropping every table it can — which +/// then builds each dropped KECCAK_RND table again with no mismatch — and +/// verified each time. Run once per value of the variable. +#[test] +#[ignore = "proves small blocks, minutes on a CPU runner; box tier"] +fn a_streamed_block_counts_the_tables_of_the_rule_in_effect() { + let padding = CutPadding::from_env().expect("the variable reads"); + let tables = match padding { + CutPadding::WholeRun => 4, + CutPadding::LastTable => 3, + }; + let elf = asm_elf_bytes("test_keccak_multi"); + let opts = ProofOptions::default_test_options(); + let format = many_groups(ArgueFormat::BATCHED); + for (what, layout_workers, regen) in [ + ("the finish monolithic", 0, None), + ("the finish streamed", 3, None), + ( + "live regeneration", + 3, + Some(block_whir::regen::RegenMode::Always), + ), + ] { + let o = BlockOptions { + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + regen, + ..options(MaxRowsConfig::small(), Some(3), layout_workers) + }; + let (proof, stamps) = + prove_block_whir_with(&elf, &[], &opts, &format, &o, &Deviations::default()) + .unwrap_or_else(|e| panic!("{padding:?}, {what}: {e:?}")); + assert_eq!( + cut_counts(&proof), + [1, tables, 0, 0], + "{padding:?}, {what}: the counts" + ); + assert_eq!( + verify_block_whir(&proof, &elf, &opts, &format).map_err(|e| format!("{e:?}")), + Ok(true), + "{padding:?}, {what}" + ); + if regen.is_some() { + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert!(r.rest_dropped > 0, "no table of the rest dropped\n{lines}"); + assert_eq!(r.regenerated, r.dropped, "every table back\n{lines}"); + assert_eq!(r.rest_regenerated, r.rest_dropped, "{lines}"); + assert_eq!((r.mismatches, r.failed), (0, 0), "{lines}"); + assert_eq!(r.error, None, "{lines}"); + // KECCAK_RND's tables built again: every one phase A dropped, and + // none the rule does not build. + let again: usize = r + .lines + .iter() + .find_map(|line| line.strip_prefix("BLOCK REGEN rest: ")) + .and_then(|rest| rest.split("· KECCAK_RND ").nth(1)) + .and_then(|count| count.split(' ').next()) + .and_then(|n| n.parse().ok()) + .unwrap_or_else(|| panic!("no KECCAK_RND count in the rest's line\n{lines}")); + assert!( + (1..=tables).contains(&again), + "{again} KECCAK_RND tables built again of {tables}\n{lines}" + ); + println!( + "BLOCK CUT PADDING REBUILT: {padding:?}: {again} of {tables} KECCAK_RND tables \ + dropped and built again, {} of the rest in all, 0 mismatches", + r.rest_regenerated + ); + } + println!("BLOCK CUT PADDING STREAMED: {padding:?}, {what}: KECCAK_RND {tables}, verified"); + } +} diff --git a/prover/src/tests/block_onechain_tests.rs b/prover/src/tests/block_onechain_tests.rs new file mode 100644 index 000000000..8eedb79f5 --- /dev/null +++ b/prover/src/tests/block_onechain_tests.rs @@ -0,0 +1,770 @@ +//! D-ONECHAIN B1 at the block: one WHIR chain per run of a group's stacked +//! polynomials ([`GroupFold::Indexed`]) through the block's frame, prover and +//! verifier (REV-B2-JUDGE B1-1 (a), B1-5). The chain-level gates are +//! `multilinear`'s `whir_group_tests`, and on the card its +//! `whir_group_device`. +//! +//! On either build: a card build runs a grouped round 0 on the card where it +//! takes the stack, and on the host where it does not. + +use crate::block_whir::{ + self, BlockFormat, BlockOptions, BlockRefusal, BlockWhirProof, Deviations, + prove_block_whir_with, +}; +use crate::tables::MaxRowsConfig; +use crate::test_utils::asm_elf_bytes; +use crate::zf_format::ZfFormat; +use multilinear::stacked_eval::index_chains; +use multilinear::whir_chain::{ + ArgueFormat, GroupFold, MAX_INDEX_POLYS, MAX_INDEX_VARS, RoundOpenings, StackVars, + check_index_domain, index_vars, +}; +use stark::multilinear_table::batched::VerifierChecks; +use stark::proof::options::{BaseFormat, ProofOptions}; + +/// A stack of 2^10 and `group_polys` polynomials a group under `base`, so a +/// small program spans several groups, argued per `argue` and opened per +/// `group`. A card declines to open a stack this small (its sumcheck +/// threshold is 2^12), so on a card build a coset round 0 here encodes its +/// successor on the host ([`multilinear::gpu::coset_host_encodes`]). +fn format( + base: BaseFormat, + group_polys: usize, + group: GroupFold, + argue: ArgueFormat, +) -> BlockFormat { + format_at(base, group_polys, group, argue, 10) +} + +/// [`format`] at a stack of `2^stack_vars`. +fn format_at( + base: BaseFormat, + group_polys: usize, + group: GroupFold, + argue: ArgueFormat, + stack_vars: usize, +) -> BlockFormat { + let mut zf = ZfFormat::DEFAULT.with_base(base); + zf.whir_stack = StackVars::new(stack_vars).expect("a valid stack"); + BlockFormat { + zf, + group_polys, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue, + group, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +/// The production format, opened per `group`: one group holds a small program. +fn production(group: GroupFold) -> BlockFormat { + BlockFormat { + zf: ZfFormat::DEFAULT, + group_polys: block_whir::BLOCK_GROUP_POLYS, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: ArgueFormat::PerTable, + group, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +/// `block_whir_tests`' small-program options at `max_rows`: groups retire and +/// revive between the phases (3 binary levels dropped). +fn options(max_rows: MaxRowsConfig) -> BlockOptions { + BlockOptions { + max_rows, + keccak_rnd_rows_log2: 16, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: block_whir::BLOCK_COPY_ROWS_LOG2, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(3), + window_log2: None, + drop_streamed_ops: false, + layout_workers: 0, + layout_ahead: Some(2), + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + upload_ahead: true, + memlog: false, + finish_keccak_rnd_chunks: true, + finish_cuts: true, + finish_stream: true, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + pack_finished: true, + gpack: true, + regen: None, + phase_b_overlap: stark::multilinear_block::PhaseBOverlap::Off, + } +} + +fn prove(elf: &[u8], format: &BlockFormat, max_rows: MaxRowsConfig) -> BlockWhirProof { + prove_block_whir_with( + elf, + &[], + &ProofOptions::default_test_options(), + format, + &options(max_rows), + &Deviations::default(), + ) + .expect("prove") + .0 +} + +/// The width of every grouped chain's run in the proof, groups and prepared +/// stacks alike: its round-0 blocks a query. +fn grouped_widths(proof: &BlockWhirProof) -> Vec { + proof + .proof + .columns + .iter() + .chain(&proof.prepared) + .flat_map(|stack| &stack.polys) + .filter_map(|chain| match &chain.rounds.first()?.openings { + RoundOpenings::BaseGrouped(g) => g.current.first().map(Vec::len), + _ => None, + }) + .collect() +} + +/// Each chain's width in one stack's opening, in proof order: a grouped +/// chain's round-0 blocks a query, 1 for a per-polynomial chain. +fn chain_widths(stack: &multilinear::stacked_eval::StackedProof) -> Vec +where + F: math::field::traits::IsField, + E: math::field::traits::IsField, +{ + stack + .polys + .iter() + .map(|chain| match chain.rounds.first().map(|r| &r.openings) { + Some(RoundOpenings::BaseGrouped(g)) => g.current.first().map_or(0, Vec::len), + _ => 1, + }) + .collect() +} + +/// REV-B2-JUDGE B1-1 (a), REV-B2-A's frame probe under `Indexed`: a table +/// stated at 2^27 rows alone in its group (the frame admits it, exempt from +/// `group_polys`) stacks into one polynomial per column, and the verifier's +/// split cuts them into runs of four and one of the rest, each within +/// `MAX_INDEX_VARS` index variables and the two-adicity. The table is the +/// widest outside the chunked families and the CPU's: a CPU table stated past +/// `BLOCK_CPU_MAX_VARS` (2^21 rows, the block's chunk height) is refused by +/// the frame, which this checks first on the widest CPU table. +#[test] +fn a_wide_singleton_group_splits_into_runs_of_four() { + let elf = asm_elf_bytes("sub"); + let indexed = production(GroupFold::Indexed); + let proof = prove( + &elf, + &production(GroupFold::PerPoly), + MaxRowsConfig::default(), + ); + let opts = ProofOptions::default_test_options(); + let honest = block_whir::block_frame(proof.statement(), &elf, &opts, &indexed).expect("frame"); + let chunked: Vec> = + block_whir::chunked_table_ranges(&proof.table_counts) + .iter() + .map(|(_, r, _)| r.clone()) + .collect(); + let cpu_start = crate::cpu_proof_start(true, &proof.table_counts); + let cpus = cpu_start..cpu_start + proof.table_counts.cpu; + let widest = |keep: &dyn Fn(usize) -> bool| { + honest + .shapes + .iter() + .enumerate() + .filter(|&(i, _)| keep(i) && !chunked.iter().any(|r| r.contains(&i))) + .map(|(i, &(w, _))| (i, w)) + .max_by_key(|&(_, w)| w) + }; + // Table `t` stated at 2^27 rows, alone in its group. + let n = proof.table_num_vars.len(); + let singleton = |t: usize| { + let mut heights = proof.table_num_vars.clone(); + heights[t] = block_whir::BLOCK_MAX_TABLE_VARS as u8; + let groups: Vec> = vec![ + vec![t as u32], + (0..n as u32).filter(|&i| i as usize != t).collect(), + ]; + (heights, groups) + }; + let (cpu, _) = widest(&|i| cpus.contains(&i)).expect("a CPU table"); + let (heights, groups) = singleton(cpu); + let statement = block_whir::BlockStatement { + table_num_vars: &heights, + groups: &groups, + ..proof.statement() + }; + assert!( + matches!( + block_whir::block_frame(statement, &elf, &opts, &indexed), + Err(crate::Error::InvalidTableCounts(m)) if m.starts_with("CPU[") + ), + "a CPU table stated at 2^27 rows is refused by the frame" + ); + let (t, width) = widest(&|i| !cpus.contains(&i)) + .expect("a table outside the chunked families and the CPU's"); + assert!( + width > MAX_INDEX_POLYS, + "table {t} is {width} columns wide: one run" + ); + let (heights, groups) = singleton(t); + let statement = block_whir::BlockStatement { + table_num_vars: &heights, + groups: &groups, + ..proof.statement() + }; + let frame = block_whir::block_frame(statement, &elf, &opts, &indexed) + .expect("the verifier's frame admits the shape"); + assert_eq!(frame.config.format.group, GroupFold::Indexed); + let layout = &frame.stack_layouts[0]; + assert_eq!(layout.n_stack(), block_whir::BLOCK_MAX_TABLE_VARS); + let polys = layout.num_polys(); + println!( + "ONECHAIN B1-1a: table {t} ({}) width {width} stated at 2^{}: group 0 stacks into \ + {polys} polynomials", + honest.airs.air_refs()[t].name(), + block_whir::BLOCK_MAX_TABLE_VARS + ); + assert_eq!(polys, width, "one 2^27 polynomial per column"); + let runs: Vec = index_chains(polys, frame.config.format.group) + .iter() + .map(|r| r.len()) + .collect(); + let mut want = vec![MAX_INDEX_POLYS; polys / MAX_INDEX_POLYS]; + if !polys.is_multiple_of(MAX_INDEX_POLYS) { + want.push(polys % MAX_INDEX_POLYS); + } + assert_eq!(runs, want); + for &run in &runs { + assert!(index_vars(run) <= MAX_INDEX_VARS); + check_index_domain( + layout.n_stack(), + index_vars(run), + frame.config.log_blowup, + 32, + ) + .expect("the virtual domain exists"); + } +} + +/// REV-B2-JUDGE B1-5 at the block, and the grouped chain's round trip +/// through it: at up to two, three and four polynomials a group, under the +/// block's base (Poseidon1's 4-ary capped trees), an `Indexed` block proves and +/// verifies with grouped chains — each group's chains exactly the verifier's +/// split of its stack, runs of at most `MAX_INDEX_POLYS` whatever the group +/// budget is (the budget bounds a group of several tables; a table that alone +/// needs more is a group of its own, and its runs are of four) — runs of two, +/// three and four each occur, its phase-A roots are the per-polynomial +/// block's (B2-sep), and neither mode's proof verifies under the other's +/// verifier — refused by the STATEMENT: the two modes' statement bytes differ +/// in one bit, the fold word's bit 47, so the transcript parts at the first +/// absorb and the first group's argue refuses, before any opening's shape is +/// read (the shapes' own refusals, `ChainCount` and the per-polynomial count, +/// are `whir_group_tests`'). Each group's tables argued one by one. +#[test] +fn an_indexed_block_round_trips_and_refuses_the_other_mode() { + let seen = round_trips_and_refuses_the_other_mode(ArgueFormat::PerTable, 2..=MAX_INDEX_POLYS); + // Runs of two, three and four each round-tripped and each was refused + // across the modes: the three budgets pack the program into all of them. + assert_eq!( + seen.into_iter().collect::>(), + (2..=MAX_INDEX_POLYS).collect::>(), + "a grouped run of every width from two to {MAX_INDEX_POLYS}" + ); +} + +/// [`an_indexed_block_round_trips_and_refuses_the_other_mode`] under the +/// argue production runs, each group's tables argued together +/// (`ArgueFormat::BATCHED`), at the production budget of three polynomials a +/// group: the grouped opening takes its points and values from whichever +/// argue ran, and this is the pairing a default would ship. The same round +/// trip, the same equal roots, the statements one bit apart, and both +/// cross-mode refusals at the statement, never at an opening's shape — over +/// runs of two, three and four. +#[test] +fn an_indexed_block_round_trips_under_the_batched_argue() { + let seen = round_trips_and_refuses_the_other_mode( + ArgueFormat::BATCHED, + block_whir::BLOCK_GROUP_POLYS..=block_whir::BLOCK_GROUP_POLYS, + ); + // The production budget packs the program into runs of every width: the + // tables of a group, and two tables alone in theirs split into fours. + assert_eq!( + seen.into_iter().collect::>(), + (2..=MAX_INDEX_POLYS).collect::>(), + "a grouped run of every width under the batched argue" + ); +} + +/// ★ The base code's rate (`LAMBDA_VM_ZF_WHIR_BLOWUP`) under one chain a run +/// and the batched argue — the composed format, at the production budget of +/// three polynomials a group: a block at blowup 2 has grouped chains and +/// verifies; a verifier at blowup 4 refuses it, and a verifier at blowup 2 +/// refuses a block at blowup 4; and at blowup 2 a per-polynomial verifier +/// refuses the grouped block. +#[test] +fn an_indexed_block_at_blowup_2_is_refused_by_a_verifier_at_blowup_4() { + let elf = asm_elf_bytes("all_instructions_64"); + let opts = ProofOptions::default_test_options(); + let at = |blowup: u8, group: GroupFold| { + let mut f = format( + crate::hash_pin::BLOCK_BASE, + block_whir::BLOCK_GROUP_POLYS, + group, + ArgueFormat::BATCHED, + ); + f.zf.whir_blowup = blowup; + f + }; + let verify = |proof: &BlockWhirProof, format: &BlockFormat| { + block_whir::verify_block_whir(proof, &elf, &opts, format).unwrap_or(false) + }; + let half = prove(&elf, &at(2, GroupFold::Indexed), MaxRowsConfig::small()); + assert!( + grouped_widths(&half).iter().any(|&w| w >= 2), + "a grouped chain at blowup 2" + ); + assert!(verify(&half, &at(2, GroupFold::Indexed)), "the control"); + assert!( + !verify(&half, &at(4, GroupFold::Indexed)), + "a verifier at blowup 4 must refuse a block at blowup 2" + ); + assert!( + !verify(&half, &at(2, GroupFold::PerPoly)), + "a per-polynomial verifier must refuse the grouped block at blowup 2" + ); + let four = prove(&elf, &at(4, GroupFold::Indexed), MaxRowsConfig::small()); + assert!( + verify(&four, &at(4, GroupFold::Indexed)), + "the control at blowup 4" + ); + assert!( + !verify(&four, &at(2, GroupFold::Indexed)), + "a verifier at blowup 2 must refuse a block at blowup 4" + ); +} + +/// ★ The coset round-0 leaf order (`LAMBDA_VM_ZF_WHIR_LEAVES`) under one chain +/// a run and the batched argue, at the production budget of three polynomials +/// a group, at either blowup: a coset block has grouped chains and verifies +/// under its own format; a verifier of the natural order refuses it, and so +/// does a coset verifier at the other blowup; a coset verifier refuses the +/// natural block. +#[test] +fn an_indexed_coset_block_is_refused_by_a_natural_verifier() { + use multilinear::whir_chain::LeafOrder; + let elf = asm_elf_bytes("all_instructions_64"); + let opts = ProofOptions::default_test_options(); + let at = |blowup: u8, leaves: LeafOrder| { + let mut f = format( + crate::hash_pin::BLOCK_BASE, + block_whir::BLOCK_GROUP_POLYS, + GroupFold::Indexed, + ArgueFormat::BATCHED, + ); + f.zf.whir_blowup = blowup; + f.zf.whir_leaves = leaves; + f + }; + let verify = |proof: &BlockWhirProof, format: &BlockFormat| { + block_whir::verify_block_whir(proof, &elf, &opts, format).unwrap_or(false) + }; + for (blowup, other) in [(4, 2), (2, 4)] { + let encodes = multilinear::gpu::coset_host_encodes(); + let coset = prove(&elf, &at(blowup, LeafOrder::Coset), MaxRowsConfig::small()); + assert!( + grouped_widths(&coset).iter().any(|&w| w >= 2), + "a grouped chain under the coset order at blowup {blowup}" + ); + // On a card the 2^10 stack's openings fall to the host, and each coset + // round 0 encodes its successor there: counted, the prove goes on. + if cfg!(feature = "cuda") { + assert!( + multilinear::gpu::coset_host_encodes() > encodes, + "the host encodes counted at blowup {blowup}" + ); + } + assert!( + verify(&coset, &at(blowup, LeafOrder::Coset)), + "the control at blowup {blowup}" + ); + assert!( + !verify(&coset, &at(blowup, LeafOrder::Natural)), + "a natural verifier must refuse the coset block at blowup {blowup}" + ); + assert!( + !verify(&coset, &at(other, LeafOrder::Coset)), + "a coset verifier at blowup {other} must refuse the coset block at blowup {blowup}" + ); + } + let natural = prove(&elf, &at(4, LeafOrder::Natural), MaxRowsConfig::small()); + assert!( + verify(&natural, &at(4, LeafOrder::Natural)), + "the natural control" + ); + assert!( + !verify(&natural, &at(4, LeafOrder::Coset)), + "a coset verifier must refuse the natural block" + ); +} + +/// ★ The composed format at the block: one chain a run, the coset round-0 +/// order and blowup 2, under the batched argue at the production budget. A +/// block proved in it verifies, and is refused by the verifier of each format +/// one setting away — blowup 4, the natural order, one chain a polynomial — +/// and a block one setting away (the natural order) is refused by the composed +/// verifier. Each setting is a verifier constant; the proof carries none. +#[test] +fn a_composed_block_is_refused_one_setting_away() { + use multilinear::whir_chain::LeafOrder; + let elf = asm_elf_bytes("all_instructions_64"); + let opts = ProofOptions::default_test_options(); + let at = |group: GroupFold, leaves: LeafOrder, blowup: u8| { + let mut f = format( + crate::hash_pin::BLOCK_BASE, + block_whir::BLOCK_GROUP_POLYS, + group, + ArgueFormat::BATCHED, + ); + f.zf.whir_leaves = leaves; + f.zf.whir_blowup = blowup; + f + }; + let verify = |proof: &BlockWhirProof, format: &BlockFormat| { + block_whir::verify_block_whir(proof, &elf, &opts, format).unwrap_or(false) + }; + let composed = at(GroupFold::Indexed, LeafOrder::Coset, 2); + let encodes = multilinear::gpu::coset_host_encodes(); + let proof = prove(&elf, &composed, MaxRowsConfig::small()); + assert!( + grouped_widths(&proof).iter().any(|&w| w >= 2), + "a grouped chain in the composed block" + ); + // On a card the 2^10 stack's openings fall to the host: each coset round + // 0 encodes its successor there, counted, and the prove goes on. + if cfg!(feature = "cuda") { + assert!( + multilinear::gpu::coset_host_encodes() > encodes, + "the host encodes counted" + ); + } + assert!(verify(&proof, &composed), "the composed format's control"); + for (label, other) in [ + ("blowup 4", at(GroupFold::Indexed, LeafOrder::Coset, 4)), + ( + "the natural order", + at(GroupFold::Indexed, LeafOrder::Natural, 2), + ), + ( + "one chain a polynomial", + at(GroupFold::PerPoly, LeafOrder::Coset, 2), + ), + ] { + assert!( + !verify(&proof, &other), + "a verifier at {label} must refuse the composed block" + ); + } + let natural = at(GroupFold::Indexed, LeafOrder::Natural, 2); + let neighbour = prove(&elf, &natural, MaxRowsConfig::small()); + assert!(verify(&neighbour, &natural), "the natural order's control"); + assert!( + !verify(&neighbour, &composed), + "the composed verifier must refuse a block in the natural order" + ); +} + +/// The smallest stack a card opens, in variables: its sumcheck threshold +/// (`multilinear::gpu`'s 2^12). A smaller one's openings fall to the host. +const CARD_OPEN_VARS: usize = 12; + +/// Every stack the block opens, its groups' then its prepared ones, as the +/// verifier frames them: the stack's height in variables and its chains. +fn stack_chains(elf: &[u8], proof: &BlockWhirProof, format: &BlockFormat) -> Vec<(usize, usize)> { + let opts = ProofOptions::default_test_options(); + let frame = block_whir::block_frame(proof.statement(), elf, &opts, format).expect("frame"); + let columns = block_whir::prepared_tables(&frame.airs, &frame.page_configs, format) + .expect("prepared tables"); + let prepared = block_whir::group_stacks(columns, proof.statement().groups) + .expect("prepared stacks") + .iter() + .map(|stack| { + stark::multilinear_table::global_layout(&stack.shapes(), frame.config.format.stack) + .expect("prepared layout") + .n_stack() + }) + .collect::>(); + assert_eq!(frame.stack_layouts.len(), proof.proof.columns.len()); + assert_eq!(prepared.len(), proof.prepared.len()); + frame + .stack_layouts + .iter() + .map(|layout| layout.n_stack()) + .chain(prepared) + .zip( + proof + .proof + .columns + .iter() + .chain(&proof.prepared) + .map(|stack| stack.polys.len()), + ) + .collect() +} + +/// ★ The defaults' block at a stack cap the card opens (2^12, its sumcheck +/// threshold): one chain a run under the coset order, at blowup 4 and under the +/// composed format at blowup 2, proves and verifies, and on a card build no +/// chain of a stack of 2^12 or more encodes its round-0 successor on the host. +/// A cap is not a floor: a stack of fewer cells is lower (this program has one +/// at 2^10), the card declines its openings and each of its chains may encode +/// once on the host (the 2^10 tests above), so the count is bounded by those +/// chains, printed, and is 0 when there are none. +#[test] +fn a_coset_block_at_a_card_sized_stack_encodes_no_successor_on_the_host() { + use multilinear::whir_chain::LeafOrder; + let elf = asm_elf_bytes("all_instructions_64"); + let opts = ProofOptions::default_test_options(); + for blowup in [4, 2] { + let mut format = format_at( + crate::hash_pin::BLOCK_BASE, + block_whir::BLOCK_GROUP_POLYS, + GroupFold::Indexed, + ArgueFormat::BATCHED, + CARD_OPEN_VARS, + ); + format.zf.whir_blowup = blowup; + format.zf.whir_leaves = LeafOrder::Coset; + let encodes = multilinear::gpu::coset_host_encodes(); + let proof = prove(&elf, &format, MaxRowsConfig::small()); + let host = multilinear::gpu::coset_host_encodes() - encodes; + assert!( + block_whir::verify_block_whir(&proof, &elf, &opts, &format).unwrap_or(false), + "the block verifies at blowup {blowup}" + ); + let stacks = stack_chains(&elf, &proof, &format); + let below: usize = stacks + .iter() + .filter(|&&(vars, _)| vars < CARD_OPEN_VARS) + .map(|&(_, chains)| chains) + .sum(); + println!( + "ONECHAIN CARD-SIZED blowup {blowup}: stacks (vars, chains) {stacks:?}; {below} \ + chains below 2^{CARD_OPEN_VARS}; host encodes {host}" + ); + // Not vacuous: a grouped chain at a stack the card opens. + let grouped_at_size = proof + .proof + .columns + .iter() + .chain(&proof.prepared) + .zip(&stacks) + .any(|(stack, &(vars, _))| { + vars >= CARD_OPEN_VARS && chain_widths(stack).iter().any(|&w| w >= 2) + }); + assert!( + grouped_at_size, + "a grouped chain at a stack of 2^{CARD_OPEN_VARS} or more, blowup {blowup}" + ); + // A chain encodes its successor at most once, at round 0; the card + // opens every stack of 2^12 or more, so only the chains below may. + assert!( + host <= below as u64, + "no host encode at a stack the card opens, blowup {blowup}: {host} host encodes, \ + {below} chains below 2^{CARD_OPEN_VARS}" + ); + } +} + +/// The round trip and the cross-mode refusals at each budget of `budgets` +/// polynomials a group, under `argue`; returns the grouped run widths seen. +fn round_trips_and_refuses_the_other_mode( + argue: ArgueFormat, + budgets: std::ops::RangeInclusive, +) -> std::collections::BTreeSet { + let elf = asm_elf_bytes("all_instructions_64"); + let opts = ProofOptions::default_test_options(); + let traced = |proof: &BlockWhirProof, format: &BlockFormat| { + block_whir::verify_block_whir_traced(proof, &elf, &opts, format, false, VerifierChecks::ALL) + .expect("the statement frames under either mode") + }; + let base = crate::hash_pin::BLOCK_BASE; + let mut seen = std::collections::BTreeSet::new(); + for group_polys in budgets { + let indexed = format(base, group_polys, GroupFold::Indexed, argue); + let per_poly = format(base, group_polys, GroupFold::PerPoly, argue); + let a = prove(&elf, &indexed, MaxRowsConfig::small()); + let b = prove(&elf, &per_poly, MaxRowsConfig::small()); + let widths = grouped_widths(&a); + println!( + "ONECHAIN B1-5 {base:?} group_polys {group_polys}: {} groups, grouped runs \ + {widths:?}", + a.proof.columns.len() + ); + assert!( + widths.iter().any(|&w| w >= 2), + "{base:?} {group_polys}: no grouped chain" + ); + // Each group's chains are the verifier's split of its stack, exactly: + // runs of at most `MAX_INDEX_POLYS`, whatever `group_polys` is. The + // budget bounds a group of SEVERAL tables; a table that alone needs + // more is a group of its own (the frame's rule and the packing's). + let framed = block_whir::block_frame(a.statement(), &elf, &opts, &indexed).expect("frame"); + assert_eq!(a.proof.columns.len(), framed.stack_layouts.len()); + assert_eq!(b.proof.columns.len(), framed.stack_layouts.len()); + for (g, layout) in framed.stack_layouts.iter().enumerate() { + let polys = layout.num_polys(); + let got = chain_widths(&a.proof.columns[g]); + println!( + "ONECHAIN B1-5 {base:?} group_polys {group_polys} group {g}: {} tables, \ + {polys} polynomials, chains {got:?}", + framed.sizes[g] + ); + let split: Vec = index_chains(polys, GroupFold::Indexed) + .iter() + .map(|run| run.len()) + .collect(); + assert_eq!( + got, split, + "{base:?} {group_polys} group {g}: the proof's chains are the split's" + ); + assert!( + got.iter().all(|w| (1..=MAX_INDEX_POLYS).contains(w)), + "{base:?} {group_polys} group {g}: a run of at most {MAX_INDEX_POLYS}" + ); + assert!( + polys <= group_polys || framed.sizes[g] == 1, + "{base:?} {group_polys} group {g}: {polys} polynomials over {} tables — only a \ + table alone may need more than the group budget", + framed.sizes[g] + ); + assert_eq!( + chain_widths(&b.proof.columns[g]), + vec![1; polys], + "{base:?} {group_polys} group {g}: one chain a polynomial under PerPoly" + ); + } + for (k, stack) in a.prepared.iter().enumerate() { + let got = chain_widths(stack); + println!( + "ONECHAIN B1-5 {base:?} group_polys {group_polys} prepared stack {k}: chains \ + {got:?}" + ); + assert!(got.iter().all(|w| (1..=MAX_INDEX_POLYS).contains(w))); + } + seen.extend(widths.iter().copied()); + assert!(grouped_widths(&b).is_empty()); + println!( + "ONECHAIN B1-5 {base:?} group_polys {group_polys}: phase-A roots {} (Indexed), {} \ + (PerPoly), equal: {}", + a.proof.roots.len(), + b.proof.roots.len(), + a.proof.roots == b.proof.roots + ); + assert_same_roots(&elf, &a, &b, &indexed); + let (own_a, own_b) = (traced(&a, &indexed), traced(&b, &per_poly)); + println!( + "ONECHAIN B1-5 {base:?} group_polys {group_polys}: Indexed under Indexed {own_a:?}, \ + PerPoly under PerPoly {own_b:?}" + ); + assert!(own_a.is_ok(), "{base:?} {group_polys}"); + assert!(own_b.is_ok(), "{base:?} {group_polys}"); + + // The statement: one bit apart, the fold word's 47. + let frame = |format: &BlockFormat| { + block_whir::block_frame(a.statement(), &elf, &opts, format).expect("frame") + }; + let tag = block_whir::block_statement_tag(&base); + let digest = crate::statement::elf_digest(&elf); + let (sa, sb) = ( + block_whir::block_statement_bytes( + a.statement(), + &tag, + &digest, + &frame(&indexed).config, + ), + block_whir::block_statement_bytes( + a.statement(), + &tag, + &digest, + &frame(&per_poly).config, + ), + ); + assert_eq!(sa.len(), sb.len()); + let differ: Vec<(usize, u8)> = sa + .iter() + .zip(&sb) + .enumerate() + .filter(|(_, (x, y))| x != y) + .map(|(i, (x, y))| (i, x ^ y)) + .collect(); + println!( + "ONECHAIN B1-5 {base:?} group_polys {group_polys}: statements of {} bytes differ at \ + (byte, xor) {differ:?}", + sa.len() + ); + assert_eq!(differ.len(), 1, "{base:?} {group_polys}: {differ:?}"); + assert_eq!(differ[0].1, 0x80, "bit 47 is byte 5's top bit"); + + for (label, proof, format) in [ + ("Indexed under PerPoly", &a, &per_poly), + ("PerPoly under Indexed", &b, &indexed), + ] { + match traced(proof, format) { + Err(BlockRefusal::Proof(e)) => { + println!("ONECHAIN B1-5 {base:?} {group_polys} {label}: refused with {e:?}"); + assert!( + !matches!( + e, + multilinear::Error::QueryCountMismatch { .. } + | multilinear::Error::GroupedShape(_) + | multilinear::Error::ColumnOpeningRejected { .. } + ), + "{base:?} {group_polys} {label}: refused at an opening's shape or \ + chain, not by the statement: {e:?}" + ); + } + other => panic!("{base:?} {group_polys} {label}: {other:?}"), + } + } + } + println!("ONECHAIN B1-5 {argue:?} run widths seen: {seen:?}"); + seen +} + +/// The two modes' phase-A roots are one list (B2-sep). On a mismatch, names the +/// first differing root's group, polynomial and tables, then fails: a +/// difference is a design finding, not a test to adjust. +fn assert_same_roots(elf: &[u8], a: &BlockWhirProof, b: &BlockWhirProof, format: &BlockFormat) { + let Some(first) = (0..a.proof.roots.len().max(b.proof.roots.len())) + .find(|&i| a.proof.roots.get(i) != b.proof.roots.get(i)) + else { + return; + }; + let opts = ProofOptions::default_test_options(); + let frame = block_whir::block_frame(a.statement(), elf, &opts, format).expect("frame"); + let mut at = first; + let mut g = 0; + while g < frame.stack_layouts.len() && at >= frame.stack_layouts[g].num_polys() { + at -= frame.stack_layouts[g].num_polys(); + g += 1; + } + let tables: Vec = a.groups.get(g).map_or_else(Vec::new, |group| { + group + .iter() + .map(|&t| format!("{t} {}", frame.airs.air_refs()[t as usize].name())) + .collect() + }); + panic!( + "phase A's roots differ first at root {first} of {}: group {g}, polynomial {at} \ + (tables {tables:?})", + a.proof.roots.len() + ); +} diff --git a/prover/src/tests/block_whir_p1_tests.rs b/prover/src/tests/block_whir_p1_tests.rs new file mode 100644 index 000000000..e91a9ac04 --- /dev/null +++ b/prover/src/tests/block_whir_p1_tests.rs @@ -0,0 +1,923 @@ +//! D-WHIR-P1 S3, D-P3B P3: the block's base is its proof family's +//! ([`crate::hash_pin::BlockFamily`]): [`BaseFormat::P1_WHIR`], ZisK's +//! Poseidon1 (4-ary trees capped at height 2, its field sponge and W8 grind). +//! The format names it, the family pins it: any other base — RPX's included — +//! is refused; a P1 proof verifies under no other base; and the statement's tag +//! names the base's geometry. + +use crate::block_whir::{ + self, BlockFormat, BlockOptions, BlockWhirProof, Deviations, MAX_P1_CAP_HEIGHT, P1_TAG_OMITTED, + block_statement_tag, checked_base, prove_block_whir_with, verify_block_whir, +}; +use crate::statement::MULTILINEAR_BLOCK_TAG; +use crate::tables::MaxRowsConfig; +use crate::test_utils::asm_elf_bytes; +use crate::zf_format::ZfFormat; +use multilinear::whir_chain::{ArgueFormat, StackVars}; +use stark::config::CommitmentHash; +use stark::proof::options::{BaseFormat, CapPolicy, ProofOptions}; + +/// A stack of 2^10 and two polynomials a group, so a small program spans +/// several groups, under `base`. +pub(crate) fn many_groups(base: BaseFormat) -> BlockFormat { + let mut zf = ZfFormat::DEFAULT.with_base(base); + zf.whir_stack = StackVars::new(10).expect("a valid stack"); + BlockFormat { + zf, + group_polys: 2, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: ArgueFormat::PerTable, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +/// `block_whir_tests`' small-program options, `drop_levels` dropped from each +/// retired tree (binary levels; half as many 4-ary ones under P1). +pub(crate) fn options(drop_levels: usize) -> BlockOptions { + BlockOptions { + max_rows: MaxRowsConfig::small(), + keccak_rnd_rows_log2: 16, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: block_whir::BLOCK_COPY_ROWS_LOG2, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(drop_levels), + window_log2: None, + drop_streamed_ops: false, + layout_workers: 0, + layout_ahead: Some(2), + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + upload_ahead: true, + memlog: false, + finish_keccak_rnd_chunks: true, + finish_cuts: true, + finish_stream: true, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + pack_finished: true, + gpack: true, + regen: None, + phase_b_overlap: stark::multilinear_block::PhaseBOverlap::Off, + } +} + +fn prove(elf: &[u8], format: &BlockFormat, drop_levels: usize) -> BlockWhirProof { + prove_block_whir_with( + elf, + &[], + &ProofOptions::default_test_options(), + format, + &options(drop_levels), + &Deviations::default(), + ) + .expect("prove") + .0 +} + +fn verify(proof: &BlockWhirProof, elf: &[u8], format: &BlockFormat) -> bool { + verify_block_whir(proof, elf, &ProofOptions::default_test_options(), format).unwrap_or(false) +} + +/// The tag names the base's effective geometry under the family's hash: no +/// base, RPX's included, gets the untagged V1 tag. +#[test] +fn the_statement_tag_names_the_bases_geometry() { + let p1 = |cap| BaseFormat { + hash: CommitmentHash::Poseidon1, + arity4_cap: cap, + }; + let tag = |cap| String::from_utf8(block_statement_tag(&p1(cap))).expect("ascii"); + let v1 = String::from_utf8(MULTILINEAR_BLOCK_TAG.to_vec()).expect("ascii"); + assert_ne!(block_statement_tag(&BaseFormat::RPX), MULTILINEAR_BLOCK_TAG); + assert_eq!(tag(CapPolicy::Fixed(2)), format!("{v1}/P1W16/C2")); + assert_eq!( + block_statement_tag(&BaseFormat::P1_WHIR), + tag(CapPolicy::Fixed(2)).as_bytes() + ); + assert_eq!(tag(CapPolicy::Off), format!("{v1}/P1W16/C0")); + assert_eq!( + tag(CapPolicy::Fixed(0)), + tag(CapPolicy::Off), + "both uncapped" + ); + assert_eq!(tag(CapPolicy::Auto), format!("{v1}/P1W16/Cauto")); + assert_ne!(tag(CapPolicy::Fixed(1)), tag(CapPolicy::Fixed(2))); +} + +/// The base check: the block proof family's base and nothing else — not RPX, +/// not Poseidon1 at another cap, not another hash. +#[test] +fn checked_base_refuses_what_no_block_proves() { + let family = crate::hash_pin::BLOCK_BASE; + assert_eq!(family, BaseFormat::P1_WHIR); + assert!(checked_base(&family).is_ok()); + assert!( + checked_base(&BaseFormat::RPX).is_err(), + "RPX left the block path" + ); + for cap in [ + CapPolicy::Off, + CapPolicy::Auto, + CapPolicy::Fixed(1), + CapPolicy::Fixed(MAX_P1_CAP_HEIGHT as u8), + CapPolicy::Fixed(MAX_P1_CAP_HEIGHT as u8 + 1), + ] { + let base = BaseFormat { + arity4_cap: cap, + ..BaseFormat::P1_WHIR + }; + assert!(checked_base(&base).is_err(), "{cap}"); + } + for hash in [ + CommitmentHash::Keccak256, + CommitmentHash::Blake3, + CommitmentHash::Rpx256, + ] { + let base = BaseFormat { + hash, + ..BaseFormat::P1_WHIR + }; + assert!(checked_base(&base).is_err(), "{hash:?}"); + } +} + +/// ★ A block proves and verifies under the family's base; it verifies under no +/// other (RPX's, or Poseidon1 at another cap), and no other base proves: the +/// format names the base, the family decides it. Groups retire and revive +/// between the phases (drop 3). +#[test] +fn a_block_proves_under_its_family_base_and_no_other_verifies_it() { + let elf = asm_elf_bytes("all_instructions_64"); + let (rpx, p1) = ( + many_groups(BaseFormat::RPX), + many_groups(BaseFormat::P1_WHIR), + ); + let p1_proof = prove(&elf, &p1, 3); + assert!(p1_proof.proof.columns.len() > 1, "several groups"); + assert!( + verify(&p1_proof, &elf, &p1), + "a P1 proof under the P1 format" + ); + assert!( + !verify(&p1_proof, &elf, &rpx), + "a P1 proof under the RPX format" + ); + assert!( + prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &rpx, + &options(3), + &Deviations::default(), + ) + .is_err(), + "an RPX base is refused, not proved" + ); + // The cap is the verifier's constant too. + let other_cap = many_groups(BaseFormat { + arity4_cap: CapPolicy::Fixed(1), + ..BaseFormat::P1_WHIR + }); + assert!(!verify(&p1_proof, &elf, &other_cap), "C2 read as C1"); +} + +/// ★ The tag is in the statement: a P1 proof is refused by a verifier that +/// absorbs the untagged `…_V1` tag in its place, and a proof made and verified +/// with that tag in both places is accepted — the tag is the only thing the +/// mutation moves. +#[test] +fn the_p1_statement_tag_is_bound() { + let elf = asm_elf_bytes("sub"); + let p1 = many_groups(BaseFormat::P1_WHIR); + let honest = prove(&elf, &p1, 0); + assert!(verify(&honest, &elf, &p1)); + P1_TAG_OMITTED.with(|omitted| omitted.set(true)); + let refused = !verify(&honest, &elf, &p1); + let untagged = prove(&elf, &p1, 0); + let both_untagged = verify(&untagged, &elf, &p1); + P1_TAG_OMITTED.with(|omitted| omitted.set(false)); + assert!(refused, "a P1 proof verified under the untagged tag"); + assert!(both_untagged, "the mutation is the tag alone"); + assert!( + !verify(&untagged, &elf, &p1), + "an untagged proof under the tag" + ); +} + +/// ★ The statement the Poseidon1 leaf absorbs call by call +/// (`block_statement_calls`) is `absorb_block`'s own: the calls concatenate to +/// the statement's one run (`block_statement_bytes`, the block id's preimage), +/// and `P1Transcript` fed them lands where `absorb_block` leaves it. +#[test] +fn the_statement_calls_are_absorb_blocks_own() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + use crypto::fiat_shamir::p1_transcript::P1Transcript; + let elf = asm_elf_bytes("sub"); + let p1 = many_groups(BaseFormat::P1_WHIR); + let proof = prove(&elf, &p1, 0); + let opts = ProofOptions::default_test_options(); + let statement = proof.statement(); + let config = block_whir::block_frame(statement, &elf, &opts, &p1) + .expect("the frame") + .config; + let tag = block_statement_tag(&p1.zf.base); + let calls = block_whir::block_statement_calls(statement, &tag, &elf, &config) + .expect("byte strings only"); + let bytes = block_whir::block_statement_bytes( + statement, + &tag, + &crate::statement::elf_digest(&elf), + &config, + ); + assert!(calls.len() > 10, "{} calls", calls.len()); + assert_eq!(calls.concat(), bytes, "the calls are the run, cut"); + let mut direct = P1Transcript::new(); + block_whir::absorb_block( + &mut direct, + &tag, + &elf, + statement.public_output, + statement.table_counts, + statement.num_private_input_pages, + statement.runtime_page_ranges, + statement.table_num_vars, + &config, + statement.groups, + ); + let mut replay = P1Transcript::new(); + for call in &calls { + replay.append_bytes(call); + } + assert_eq!(replay.state(), direct.state(), "call for call"); + let mut merged = P1Transcript::new(); + merged.append_bytes(&bytes); + assert_ne!( + merged.state(), + direct.state(), + "one run is another stream under Poseidon1: the calls are load-bearing" + ); +} + +/// The WHIR configuration `base` dispatches to, by its type's name. +fn dispatched(base: BaseFormat) -> Result<&'static str, crate::Error> { + Ok(crate::with_block_hash!(base, |H| { + std::any::type_name::() + })) +} + +/// ★ The dispatch itself, whatever the process's knob says: the family's base +/// binds the family's WHIR configuration (`P1Whir`), and every other base — +/// RPX's, keccak's — is refused, not bound to a configuration of its own. +#[test] +fn the_base_alone_binds_the_block_hash() { + println!( + "the process's WHIR hash knob: {:?}", + std::env::var(crate::whir_hash_knob::ENV) + ); + assert_eq!( + dispatched(BaseFormat::P1_WHIR).expect("Poseidon1 dispatches"), + std::any::type_name::() + ); + assert!( + dispatched(BaseFormat::RPX).is_err(), + "an RPX base is refused" + ); + let keccak = BaseFormat { + hash: CommitmentHash::Keccak256, + ..BaseFormat::RPX + }; + assert!(dispatched(keccak).is_err(), "a keccak base is refused"); +} + +/// ★ No environment read decides the base: the block's prove, verify and +/// plan sites dispatch through `with_block_hash!` on the format alone, whose +/// body — the base check, then the family's configuration — reads no +/// environment and never consults the WHIR hash knob, and the plan reads no +/// knob either (D-WHIR-P1 S7: `LAMBDA_VM_WHIR_HASH` left the block path). +#[test] +fn no_environment_read_decides_the_base() { + let block = include_str!("../block_whir.rs"); + assert_eq!( + block.matches("crate::with_whir_hash!").count(), + 0, + "every block dispatch is on the format's base" + ); + assert_eq!( + block + .matches("crate::with_block_hash!(format.zf.base") + .count(), + 3 + ); + let check = &block[block.find("pub fn checked_base").expect("checked_base")..]; + let check = &check[..check.find("\n}\n").expect("its end")]; + let tag = &block[block.find("pub fn block_statement_tag").expect("the tag")..]; + let tag = &tag[..tag.find("\n}\n").expect("its end")]; + let knob = include_str!("../whir_hash_knob.rs"); + let arms = &knob[knob + .find("macro_rules! with_block_hash") + .expect("the macro")..]; + let arms = &arms[..arms.find("\n}\n").expect("its end")]; + assert!( + arms.contains("checked_base(&$base)?") + && arms.contains("type $h = $crate::hash_pin::BlockWhirHash;"), + "the macro checks the base and binds the family's configuration" + ); + for (name, source) in [ + ("checked_base", check), + ("the tag", tag), + ("the macro's arms", arms), + ] { + for read in [ + "std::env", + "env::var", + "selected()", + "whir_hash_knob", + "with_whir_hash", + ] { + assert!(!source.contains(read), "{name} names {read}"); + } + } + // The plan reads other knobs (`LFM_WHIR_SHARE_INVERSE`), never the hash's. + let plan = include_str!("../lfm/whir_block.rs"); + for read in ["whir_hash_knob", "with_whir_hash", "LAMBDA_VM_WHIR_HASH"] { + assert!(!plan.contains(read), "the block plan names {read}"); + } +} + +// ==================== the block path's source scan ==================== + +/// What a scan of the block path reads: production code as tokens, with every +/// `#[cfg(test)]` item and statement left out, comments dropped and the +/// contents of string and character literals blanked. A name in a comment, a +/// doc or a string is no site; a name in code is one, however it is spelled +/// across whitespace (REV-P3B14-A A3: a legacy prove left in code beside the +/// family's name in a comment passed the text scan this replaces). +mod scan { + /// `src` with comments dropped and string / char literal contents blanked + /// (a literal becomes `""` or `' '`; a lifetime stays). Newlines kept. + fn masked(src: &str) -> String { + let c: Vec = src.chars().collect(); + let mut out = String::with_capacity(src.len()); + let mut i = 0; + let ident = |ch: char| ch.is_ascii_alphanumeric() || ch == '_'; + while i < c.len() { + let next = c.get(i + 1).copied(); + if c[i] == '/' && next == Some('/') { + while i < c.len() && c[i] != '\n' { + i += 1; + } + } else if c[i] == '/' && next == Some('*') { + let mut depth = 0usize; + while i < c.len() { + if c[i] == '/' && c.get(i + 1) == Some(&'*') { + depth += 1; + i += 2; + } else if c[i] == '*' && c.get(i + 1) == Some(&'/') { + depth -= 1; + i += 2; + if depth == 0 { + break; + } + } else { + if c[i] == '\n' { + out.push('\n'); + } + i += 1; + } + } + } else if c[i] == 'r' + && (i == 0 || !ident(c[i - 1])) + && matches!(next, Some('"') | Some('#')) + && { + let mut j = i + 1; + while c.get(j) == Some(&'#') { + j += 1; + } + c.get(j) == Some(&'"') + } + { + let mut j = i + 1; + let mut hashes = 0; + while c[j] == '#' { + hashes += 1; + j += 1; + } + j += 1; + while j < c.len() + && !(c[j] == '"' && (0..hashes).all(|k| c.get(j + 1 + k) == Some(&'#'))) + { + if c[j] == '\n' { + out.push('\n'); + } + j += 1; + } + out.push_str("\"\""); + i = j + 1 + hashes; + } else if c[i] == '"' { + let mut j = i + 1; + while j < c.len() && c[j] != '"' { + if c[j] == '\\' { + j += 1; + } + if c.get(j) == Some(&'\n') { + out.push('\n'); + } + j += 1; + } + out.push_str("\"\""); + i = j + 1; + } else if c[i] == '\'' && next == Some('\\') { + let mut j = i + 2; + while j < c.len() && c[j] != '\'' { + j += 1; + } + out.push_str("' '"); + i = j + 1; + } else if c[i] == '\'' && c.get(i + 2) == Some(&'\'') { + out.push_str("' '"); + i += 3; + } else { + out.push(c[i]); + i += 1; + } + } + out + } + + /// `src` tokenised: identifiers and numbers as words, `::` as one token, + /// every other non-space character a token of its own. + pub(super) fn tokens(src: &str) -> Vec { + let c: Vec = src.chars().collect(); + let mut out = Vec::new(); + let mut i = 0; + while i < c.len() { + if c[i].is_whitespace() { + i += 1; + } else if c[i].is_ascii_alphanumeric() || c[i] == '_' { + let start = i; + while i < c.len() && (c[i].is_ascii_alphanumeric() || c[i] == '_') { + i += 1; + } + out.push(c[start..i].iter().collect()); + } else if c[i] == ':' && c.get(i + 1) == Some(&':') { + out.push("::".to_string()); + i += 2; + } else { + out.push(c[i].to_string()); + i += 1; + } + } + out + } + + /// The production tokens of a source: its masked tokens with every + /// `#[cfg(test)]`-gated item or statement removed — from the attribute to + /// the end of what it gates. An ITEM (`fn`, `struct`, `impl`, `mod`, + /// `const`, `use`, … after any visibility) ends at the first `;` at its own + /// depth or at the close of the block it opens; a comma never ends one, so a + /// generic `Result` in its signature is no end. Anything else (a + /// field, a variant, a match arm, a statement) ends at the first `;` or `,` + /// at its own depth or at the close of a block it opens — a comma inside a + /// generic type ends it early, which scans more code, never less. Either + /// way the close of the enclosing block ends it. No `<` is read as a + /// bracket: in an arm or a field it may be a comparison or a shift. + pub(super) fn production(src: &str) -> Vec { + let t = tokens(&masked(src)); + let gate: Vec = tokens("#[cfg(test)]"); + let mut out = Vec::with_capacity(t.len()); + let mut i = 0; + while i < t.len() { + if t[i..].starts_with(&gate) { + i += gate.len(); + // Further attributes on the same item. + while t.get(i).map(String::as_str) == Some("#") { + let mut depth = 0i64; + i += 1; + while i < t.len() { + match t[i].as_str() { + "[" => depth += 1, + "]" => { + depth -= 1; + if depth == 0 { + i += 1; + break; + } + } + _ => {} + } + i += 1; + } + } + // Past any visibility (`pub`, `pub(crate)`, …): an item keyword? + let mut k = i; + if t.get(k).map(String::as_str) == Some("pub") { + k += 1; + if t.get(k).map(String::as_str) == Some("(") { + while k < t.len() && t[k] != ")" { + k += 1; + } + k += 1; + } + } + let item = matches!( + t.get(k).map(String::as_str), + Some( + "fn" | "struct" + | "enum" + | "union" + | "impl" + | "trait" + | "mod" + | "type" + | "const" + | "static" + | "use" + | "extern" + | "macro_rules" + | "unsafe" + | "async" + ) + ); + let mut depth = 0i64; + while i < t.len() { + match t[i].as_str() { + "{" | "(" | "[" => depth += 1, + "}" | ")" | "]" => { + if depth == 0 { + // The enclosing block closes: the gated item ended. + break; + } + depth -= 1; + if depth == 0 && t[i] == "}" { + i += 1; + if matches!(t.get(i).map(String::as_str), Some(";") | Some(",")) { + i += 1; + } + break; + } + } + ";" if depth == 0 => { + i += 1; + break; + } + "," if depth == 0 && !item => { + i += 1; + break; + } + _ => {} + } + i += 1; + } + continue; + } + out.push(t[i].clone()); + i += 1; + } + out + } + + /// Occurrences of `pattern`'s tokens as a contiguous run in `code`. + pub(super) fn count(code: &[String], pattern: &str) -> usize { + let p = tokens(pattern); + code.windows(p.len()).filter(|w| *w == p.as_slice()).count() + } + + /// The legacy pin's names and entry points, as code: none may appear in a + /// production line of the block path. Whole identifiers (so + /// `harvest_child_under` is not `harvest_child`), and the legacy family's + /// `LfmFilled::prove` / `prove` method as a call (`.prove(`, `::prove(`). + pub(super) const LEGACY: &[&str] = &[ + "LegacyFamily", + "BlockStarkHash", + "BlockTranscript", + "block_transcript", + "BlockProver", + "BlockVerifier", + "BLOCK_HASHER", + "BLOCK_COMMITMENT_HASH", + "WrapHash::production", + "verify_against_artifacts", + "verify_against_chunked", + "harvest_child", + "harvest_child_verified", + "host_table_forked", + "lfm_prove", + "lfm_prove_with_hasher", + "prove_traces_with_hasher", + "build_artifacts_counted", + "build_artifacts_with_hasher", + "registry::build_artifacts", + "from_artifacts(", + "TableVerifyShape::derive(", + ".prove(", + "::prove(", + ]; + + /// What the block path takes from its family alone, as code: a binary + /// tree shape (the block's shapes are `TreeGeometry::of::()`), + /// and the Poseidon1 family or its configuration module by name (the + /// block names `BlockFamily`, so a flip of that alias moves every site). + /// None may appear in a production line of the block path. + pub(super) const FAMILY_ONLY: &[&str] = &["TreeGeometry::BINARY", "P1Family", "p1_commit"]; + + /// One file of the scan: its name, its production entry points (each must + /// survive the `#[cfg(test)]` skipper, so a misplaced gate cannot hide the + /// code after it), and the family-instantiated sites it must hold, with + /// their exact counts. + pub(super) struct File { + pub name: &'static str, + pub source: String, + pub anchors: &'static [&'static str], + pub sites: &'static [(&'static str, usize)], + /// Names this file must not use beyond [`LEGACY`]. + pub forbidden: &'static [&'static str], + } + + /// The scan: `Err` names the first violation. + pub(super) fn check(files: &[File]) -> Result<(), String> { + for f in files { + let code = production(&f.source); + if code.len() < 1000 { + return Err(format!("{}: {} production tokens read", f.name, code.len())); + } + for a in f.anchors { + if count(&code, a) == 0 { + return Err(format!("{}: the scan does not reach `{a}`", f.name)); + } + } + for name in LEGACY.iter().chain(FAMILY_ONLY).chain(f.forbidden) { + let n = count(&code, name); + if n > 0 { + return Err(format!( + "{} names `{name}` in production code ({n}×)", + f.name + )); + } + } + for (site, want) in f.sites { + let n = count(&code, site); + if n != *want { + return Err(format!( + "{}: `{site}` {n}× in production code, want {want}", + f.name + )); + } + } + } + Ok(()) + } + + /// The block path's production files, as built. + pub(super) fn block_path() -> Vec { + vec![ + File { + name: "block_whir.rs", + source: include_str!("../block_whir.rs").to_string(), + anchors: &[ + "pub fn checked_base(", + "pub fn block_statement_tag(", + "pub fn prove_block_whir(", + "pub fn verify_block_whir(", + "fn verify_block_whir_with(", + ], + sites: &[], + forbidden: &[], + }, + File { + name: "lfm/whir_block.rs", + source: include_str!("../lfm/whir_block.rs").to_string(), + anchors: &[ + "fn derive_capped(", + "pub fn node_program(", + "pub fn artifacts_of(", + "fn derived_of(", + "pub fn verify_block_tree(", + "pub(crate) fn verify_block_tree_as(", + "pub(crate) fn verifier_plan(", + "fn verify_top_of(", + ], + forbidden: &[], + sites: &[ + ("build_artifacts_counted_under::(", 1), + ("from_artifacts_under::(", 1), + ( + "verify_against_artifacts_under::(", + 1, + ), + ("node_program_under::(", 1), + ], + }, + File { + name: "lfm/whir_block_tree.rs", + source: include_str!("../lfm/whir_block_tree.rs").to_string(), + anchors: &[ + "fn build_nodes(", + "fn prove_tree_pipelined(", + "pub fn prove_whir_block_tree(", + "pub fn verify_whir_block_tree_proof(", + ], + forbidden: &[], + sites: &[ + ("prove_under::(", 2), + ("harvest_child_under::(", 2), + ], + }, + File { + name: "lfm/block_node.rs", + source: include_str!("../lfm/block_node.rs").to_string(), + anchors: &["pub fn emit_block_node(", "fn emit_node("], + sites: &[], + forbidden: &[], + }, + File { + name: "lfm/harvest.rs", + source: include_str!("../lfm/harvest.rs").to_string(), + anchors: &[ + "fn build_table_legs_with(", + "fn harvest_child_under<", + "fn try_child_arena_words(", + ], + sites: &[], + // The child's arenas are laid at its family's digest width, + // never the legacy pin's. + forbidden: &["words_per_root", "commitments_to_arena", "commitment_words"], + }, + ] + } +} + +/// ★ The block path names one proof family (D-P3B §2.1, review item 1): no +/// production line of the plan, the tree driver, the node or the block's +/// dispatch names the legacy family, the legacy pin's configuration, +/// transcript, prover, verifier or socket, or a legacy-family entry point +/// (`LegacyFamily`'s instances: `lfm_prove`, `LfmFilled::prove`, +/// `verify_against_artifacts`, `harvest_child`, …); every LFM site of the W3 +/// tree is instantiated at `BlockFamily`, at its exact count. Nor does one +/// name a binary tree shape, the Poseidon1 family or its configuration module +/// (`scan::FAMILY_ONLY`): the block takes its shapes and its hash from +/// `BlockFamily` alone. Read as code tokens ([`scan`]), so a comment or a +/// string neither hides a site nor fakes one. +#[test] +fn no_block_path_site_names_the_legacy_pin() { + scan::check(&scan::block_path()).unwrap_or_else(|e| panic!("{e}")); +} + +/// ★ The scan refuses the regressions it exists for, planted into the real +/// sources (REV-P3B14-A A3): the leaf prove put back to dcc4703e9's legacy +/// `filled.prove(artifacts, …)` with the family's name left in a comment; a +/// legacy name in a statement split over lines; a `#[cfg(test)]` that would +/// swallow a production function; a binary tree shape at the harvest's +/// family site, the Poseidon1 family named in the plan in place of +/// `BlockFamily`, and its configuration module named there; and the scan +/// reads past comments and strings without false positives. +#[test] +fn the_block_path_scan_refuses_planted_regressions() { + let planted = |file: &str, from: &str, to: &str| -> Result<(), String> { + let mut files = scan::block_path(); + let f = files + .iter_mut() + .find(|f| f.name == file) + .expect("a scanned file"); + assert_eq!(f.source.matches(from).count(), 1, "{file}: `{from}` once"); + f.source = f.source.replacen(from, to, 1); + scan::check(&files) + }; + assert_eq!(scan::check(&scan::block_path()), Ok(())); + // dcc4703e9's leaf prove, the family's call left in a comment. + let leaf = "filled.prove_under::(artifacts, &wrap, decide_lfm_residency());"; + let legacy_leaf = "filled.prove(artifacts, &wrap, decide_lfm_residency()); \ + // filled.prove_under::(artifacts, &wrap, decide_lfm_residency());"; + let e = + planted("lfm/whir_block_tree.rs", leaf, legacy_leaf).expect_err("the legacy leaf prove"); + assert!(e.contains("`.prove(`"), "{e}"); + // A legacy entry point spelled across lines and spaces. + let e = planted( + "lfm/whir_block.rs", + "pub fn artifacts_of(", + "fn planted() { super :: proof ::\n verify_against_artifacts (); }\npub fn artifacts_of(", + ) + .expect_err("a split legacy call"); + assert!(e.contains("`verify_against_artifacts`"), "{e}"); + // A test gate that would hide the verifier's entry point. + let e = planted( + "lfm/whir_block_tree.rs", + "pub fn verify_whir_block_tree_proof(", + "#[cfg(test)]\npub fn verify_whir_block_tree_proof(", + ) + .expect_err("a gated entry point"); + assert!(e.contains("does not reach"), "{e}"); + // A test gate that would hide the check of the top against the derived + // top program. + let e = planted( + "lfm/whir_block.rs", + "fn verify_top_of(", + "#[cfg(test)]\nfn verify_top_of(", + ) + .expect_err("a gated top check"); + assert!(e.contains("does not reach"), "{e}"); + // A binary shape where the harvest takes its family's. + let e = planted( + "lfm/harvest.rs", + "stark::config::TreeGeometry::of::(),", + "stark::config::TreeGeometry::BINARY,", + ) + .expect_err("a binary shape on the block path"); + assert!(e.contains("`TreeGeometry::BINARY`"), "{e}"); + // The Poseidon1 family by name where the plan names the block's. + let e = planted( + "lfm/whir_block.rs", + "from_artifacts_under::(", + "from_artifacts_under::(", + ) + .expect_err("the Poseidon1 family by name"); + assert!(e.contains("`P1Family`"), "{e}"); + // Its configuration module named in the plan. + let e = planted( + "lfm/whir_block.rs", + "pub fn artifacts_of(", + "fn planted() -> u8 { crate::lfm::p1_commit::P1_STARK_CAP }\npub fn artifacts_of(", + ) + .expect_err("the Poseidon1 configuration module by name"); + assert!(e.contains("`p1_commit`"), "{e}"); + // Comments and strings are no sites. + assert_eq!( + planted( + "lfm/block_node.rs", + "pub fn emit_block_node(", + "// LegacyFamily, harvest_child(, .prove(\nconst _PLANTED: &str = \"lfm_prove( BLOCK_HASHER\";\n\ + /* verify_against_artifacts( */\npub fn emit_block_node(", + ), + Ok(()) + ); + // A test-gated statement inside a production function stays out. + assert_eq!( + scan::count( + &scan::production("fn f() {\n #[cfg(test)]\n lfm_prove(a, b);\n g();\n}\n"), + "lfm_prove" + ), + 0 + ); + assert_eq!( + scan::count( + &scan::production("fn f() {\n lfm_prove(a, b);\n}\n"), + "lfm_prove" + ), + 1 + ); + // A gated function whose signature holds a generic comma is left out + // whole, its body included. + assert_eq!( + scan::count( + &scan::production( + "#[cfg(test)]\nfn t(a: u8) -> Result {\n lfm_prove(a);\n}\nfn g() {}\n" + ), + "lfm_prove" + ), + 0 + ); + // A gated struct-literal field or match arm holding a shift or a + // comparison ends at its comma: the fields and arms after it are scanned. + assert_eq!( + scan::count( + &scan::production( + "fn f() -> S {\n S {\n #[cfg(test)]\n a: 1 << 2,\n b: lfm_prove,\n }\n}\n" + ), + "lfm_prove" + ), + 1 + ); + assert_eq!( + scan::count( + &scan::production( + "fn f(k: K) -> bool {\n match k {\n #[cfg(test)]\n K::A => x < y,\n K::B => lfm_prove(),\n }\n}\n" + ), + "lfm_prove" + ), + 1 + ); + // A gated item whose signature carries a where-clause comma is left out + // whole, to its block's close. + assert_eq!( + scan::count( + &scan::production( + "#[cfg(test)]\npub(crate) fn t

(p: P) -> u8\nwhere\n P: A,\n u8: B

,\n{\n lfm_prove(p)\n}\nfn g() { lfm_prove(1) }\n" + ), + "lfm_prove" + ), + 1 + ); + // A gated field of a generic type ends at its comma. + assert_eq!( + scan::count( + &scan::production( + "struct S {\n #[cfg(test)]\n a: Result,\n b: lfm_prove,\n}\n" + ), + "lfm_prove" + ), + 1 + ); +} diff --git a/prover/src/tests/block_whir_pack_tests.rs b/prover/src/tests/block_whir_pack_tests.rs new file mode 100644 index 000000000..197e311b1 --- /dev/null +++ b/prover/src/tests/block_whir_pack_tests.rs @@ -0,0 +1,160 @@ +//! The GKR slot packing on the block's base (`GkrPack::Chunks`): a format +//! value, [`GkrPack::PRODUCTION`] (each extra tree priced at `2^22` cells) the +//! production constant, `Off` (one tree a table) selectable. +//! +//! Pinned here: its knob reads unset (production), `off` or a price and +//! nothing else; a packed block proves and verifies on the host; the unpacked +//! verifier refuses it and the packed verifier refuses the unpacked proof; a +//! chunk's output tampered is refused; and the public block verifier's +//! presets take the packing as a named constant, production unless the caller +//! names another. + +use crate::block_whir::{ + BlockFormat, BlockWhirProof, Deviations, prove_block_whir_with, verify_block_whir, +}; +use crate::lfm::whir_block_tree::pack_setting; +use crate::test_utils::asm_elf_bytes; +use crate::tests::block_whir_p1_tests::{many_groups, options}; +use math::field::element::FieldElement; +use multilinear::whir_chain::{ArgueFormat, GkrPack}; +use stark::proof::options::{BaseFormat, ProofOptions}; + +type E = crate::tables::types::GoldilocksExtension; + +const PACKED: ArgueFormat = ArgueFormat::Batched { + bin_log_cells: 26, + pack: GkrPack::Chunks { tree_log_cells: 8 }, +}; + +fn format(argue: ArgueFormat) -> BlockFormat { + BlockFormat { + argue, + ..many_groups(BaseFormat::P1_WHIR) + } +} + +fn prove(elf: &[u8], format: &BlockFormat) -> BlockWhirProof { + prove_block_whir_with( + elf, + &[], + &ProofOptions::default_test_options(), + format, + &options(0), + &Deviations::default(), + ) + .expect("prove") + .0 +} + +fn verify(proof: &BlockWhirProof, elf: &[u8], format: &BlockFormat) -> bool { + verify_block_whir(proof, elf, &ProofOptions::default_test_options(), format).unwrap_or(false) +} + +/// `BLOCK_WHIR_ARGUE_PACK`: unset is production's, `off` one tree a table, a +/// count to 40 a price, anything else refused. Production is τ = 22, and the +/// batched format at any cap is packed at it. +#[test] +fn the_pack_knob_reads_off_or_a_price() { + assert_eq!(pack_setting(None), Ok(GkrPack::PRODUCTION)); + assert_eq!(GkrPack::PRODUCTION, GkrPack::Chunks { tree_log_cells: 22 }); + assert_eq!(pack_setting(Some("off")), Ok(GkrPack::Off)); + assert_eq!( + pack_setting(Some(" 22 ")), + Ok(GkrPack::Chunks { tree_log_cells: 22 }) + ); + assert_eq!( + pack_setting(Some("0")), + Ok(GkrPack::Chunks { tree_log_cells: 0 }) + ); + for bad in ["41", "-1", "on", "", "2^22"] { + assert!(pack_setting(Some(bad)).is_err(), "{bad:?}"); + } + assert_eq!(ArgueFormat::BATCHED, ArgueFormat::batched(26)); + for cap in [0, 20, 26, 27] { + assert!(matches!( + ArgueFormat::batched(cap), + ArgueFormat::Batched { + pack: GkrPack::PRODUCTION, + .. + } + )); + } +} + +/// ★ The production formats pack their trees at [`GkrPack::PRODUCTION`], and +/// the block verifier's presets take the packing only as a named constant: +/// production by default, another only when the caller names it. +#[test] +fn the_verifiers_packing_is_a_named_constant() { + assert!(matches!( + BlockFormat::production().argue, + ArgueFormat::Batched { + pack: GkrPack::PRODUCTION, + .. + } + )); + assert!(matches!( + BlockFormat::pinned().argue, + ArgueFormat::Batched { + pack: GkrPack::PRODUCTION, + .. + } + )); + let pack = GkrPack::Off; + let named = BlockFormat::pinned().with_pack(pack); + assert_eq!( + named.argue, + ArgueFormat::Batched { + bin_log_cells: 26, + pack + } + ); + assert_ne!(named.argue, BlockFormat::pinned().argue); + assert_eq!( + BlockFormat::pinned().with_pack(GkrPack::PRODUCTION).argue, + BlockFormat::pinned().argue + ); + // A per-table format has no trees to pack. + let per_table = BlockFormat { + argue: ArgueFormat::PerTable, + ..BlockFormat::pinned() + }; + assert_eq!(per_table.with_pack(pack).argue, ArgueFormat::PerTable); +} + +/// ★ A packed block proves and verifies on the host with more bus outputs +/// than tables (KECCAK_RND and CPU are cut at τ = 8 on this small block); the +/// unpacked verifier refuses it, the packed verifier refuses the unpacked +/// proof, and a group's first output tampered is refused. +#[test] +fn a_packed_block_proves_and_verifies() { + let elf = asm_elf_bytes("test_keccak"); + let packed = format(PACKED); + let today = format(ArgueFormat::BATCHED).with_pack(GkrPack::Off); + let proof = prove(&elf, &packed); + let outputs: usize = proof.argues.iter().map(|a| a.bus_outputs.len()).sum(); + assert!( + outputs > proof.table_num_vars.len(), + "{outputs} bus outputs for {} tables: something is cut", + proof.table_num_vars.len() + ); + assert!(verify(&proof, &elf, &packed), "the packed verifier"); + assert!( + !verify(&proof, &elf, &today), + "the unpacked verifier refuses it" + ); + let todays = prove(&elf, &today); + assert!(verify(&todays, &elf, &today)); + assert!( + !verify(&todays, &elf, &packed), + "the packed verifier refuses the unpacked proof" + ); + for g in 0..proof.argues.len() { + let mut tampered = proof.clone(); + tampered.argues[g].bus_outputs[0].0 += FieldElement::::one(); + assert!( + !verify(&tampered, &elf, &packed), + "group {g}'s first output" + ); + } +} diff --git a/prover/src/tests/block_whir_tests.rs b/prover/src/tests/block_whir_tests.rs new file mode 100644 index 000000000..7df4abf72 --- /dev/null +++ b/prover/src/tests/block_whir_tests.rs @@ -0,0 +1,5832 @@ +//! The block proved in one multilinear proof (`crate::block_whir`): round +//! trips at one group and at many, KECCAK_RND split into chunks, the hold +//! between the phases moving no byte, and one refusal per check the block adds. + +use crate::block_whir::{ + self, BlockFormat, BlockOptions, BlockWhirProof, Deviations, prove_block_whir_with, + verify_block_whir, +}; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::{CutPadding, RowFamilies}; +use crate::test_utils::{E, asm_elf_bytes}; +use crate::zf_format::ZfFormat; +use math::field::element::FieldElement; +use multilinear::whir_chain::{ArgueFormat, StackVars}; +use stark::multilinear_block::block_groups; +use stark::multilinear_table::batched::VerifierChecks; +use stark::proof::options::ProofOptions; + +/// The production format; one group holds a small program whole. +fn one_group() -> BlockFormat { + BlockFormat { + zf: ZfFormat::DEFAULT, + group_polys: block_whir::BLOCK_GROUP_POLYS, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: ArgueFormat::PerTable, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +/// A stack of 2^10 and two polynomials a group: a small program spans several +/// groups, which is the block's shape at a size a test can prove. +fn many_groups() -> BlockFormat { + let mut zf = ZfFormat::DEFAULT; + zf.whir_stack = StackVars::new(10).expect("a valid stack"); + BlockFormat { + zf, + group_polys: 2, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: ArgueFormat::PerTable, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +fn options(max_rows: MaxRowsConfig, keccak_rnd_rows_log2: usize) -> BlockOptions { + BlockOptions { + max_rows, + keccak_rnd_rows_log2, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: block_whir::BLOCK_COPY_ROWS_LOG2, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(3), + window_log2: None, + drop_streamed_ops: false, + layout_workers: 0, + layout_ahead: Some(2), + // Every table packed on a card, so a box run walks the narrow path at a + // test's size. + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + upload_ahead: true, + memlog: false, + // The rest laid out in waves of 1 MiB: several at a test's size. + finish_keccak_rnd_chunks: true, + finish_cuts: true, + finish_stream: true, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + pack_finished: true, + gpack: true, + regen: None, + phase_b_overlap: stark::multilinear_block::PhaseBOverlap::Off, + } +} + +fn prove_with( + elf: &[u8], + format: &BlockFormat, + options: &BlockOptions, + deviations: &Deviations, +) -> BlockWhirProof { + prove_block_whir_with( + elf, + &[], + &ProofOptions::default_test_options(), + format, + options, + deviations, + ) + .expect("prove") + .0 +} + +fn prove(elf: &[u8], format: &BlockFormat, options: &BlockOptions) -> BlockWhirProof { + prove_with(elf, format, options, &Deviations::default()) +} + +fn verify(proof: &BlockWhirProof, elf: &[u8], format: &BlockFormat) -> bool { + verify_block_whir(proof, elf, &ProofOptions::default_test_options(), format).unwrap_or(false) +} + +fn groups_of(proof: &BlockWhirProof) -> usize { + proof.proof.columns.len() +} + +#[test] +fn a_program_proves_and_verifies_as_one_block() { + let elf = asm_elf_bytes("sub"); + let proof = prove(&elf, &one_group(), &options(MaxRowsConfig::default(), 16)); + assert_eq!(groups_of(&proof), 1); + assert!(verify(&proof, &elf, &one_group())); +} + +/// ★ `LAMBDA_VM_ZF_WHIR_GRIND_BITS` on the block: the grind bits, and the Q +/// they buy back, are the verifier's constants, never the proof's. A block +/// ground at 18 bits verifies under 18 and is refused by a verifier at 20; a +/// block ground at 20 is refused by a verifier at 18. +#[test] +fn a_block_ground_at_18_bits_is_refused_by_a_verifier_at_20() { + let at = |bits: u8| BlockFormat { + zf: ZfFormat { + whir_grind_bits: bits, + ..many_groups().zf + }, + ..many_groups() + }; + let (c18, c20) = ( + at(18).chain_config(&[(1, 27)]), + at(20).chain_config(&[(1, 27)]), + ); + assert_eq!( + ( + c18.num_queries, + c18.grind.query, + c20.num_queries, + c20.grind.query + ), + (229, 18, 225, 20) + ); + let elf = asm_elf_bytes("sub"); + let options = options(MaxRowsConfig::small(), 3); + let ground18 = prove(&elf, &at(18), &options); + assert!(verify(&ground18, &elf, &at(18)), "the control at 18 bits"); + assert!( + !verify(&ground18, &elf, &at(20)), + "a verifier at 20 bits must refuse a block ground at 18" + ); + let ground20 = prove(&elf, &at(20), &options); + assert!(verify(&ground20, &elf, &at(20)), "the control at 20 bits"); + assert!( + !verify(&ground20, &elf, &at(18)), + "a verifier at 18 bits must refuse a block ground at 20" + ); +} + +/// ★ `LAMBDA_VM_ZF_WHIR_BLOWUP` on the block: the base code's rate, and the Q +/// the rule charges for it, are the verifier's constants, never the proof's. A +/// block at blowup 2 verifies under blowup 2 and is refused by a verifier at +/// blowup 4; a block at blowup 4 is refused by a verifier at blowup 2. +#[test] +fn a_block_at_blowup_2_is_refused_by_a_verifier_at_blowup_4() { + let at = |blowup: u8| BlockFormat { + zf: ZfFormat { + whir_blowup: blowup, + ..many_groups().zf + }, + ..many_groups() + }; + let (c4, c2) = ( + at(4).chain_config(&[(1, 27)]), + at(2).chain_config(&[(1, 27)]), + ); + assert_eq!( + (c4.log_blowup, c4.num_queries, c2.log_blowup, c2.num_queries), + (2, 114, 1, 229) + ); + let elf = asm_elf_bytes("sub"); + let options = options(MaxRowsConfig::small(), 3); + let half = prove(&elf, &at(2), &options); + assert!(verify(&half, &elf, &at(2)), "the control at blowup 2"); + assert!( + !verify(&half, &elf, &at(4)), + "a verifier at blowup 4 must refuse a block at blowup 2" + ); + let today = prove(&elf, &at(4), &options); + assert!(verify(&today, &elf, &at(4)), "the control at blowup 4"); + assert!( + !verify(&today, &elf, &at(2)), + "a verifier at blowup 2 must refuse a block at blowup 4" + ); +} + +/// The bytes of a small block at each blowup, for a comparison across builds: +/// a card build and a host build of one sha, each run as its own process under +/// `LAMBDA_VM_FIXED_TRACE_HASH=1` and `LAMBDA_VM_DETERMINISTIC_GRIND=1`, must +/// print the same `RATE BLOCK DIGEST` lines (the card's commit, folds, +/// successors and openings prove the host's bytes at blowup 2 as at 4). Each +/// proof also verifies here. +#[test] +#[ignore = "prints proof digests for a comparison across builds; box tier"] +fn a_block_at_each_blowup_prints_its_proof_bytes() { + let elf = asm_elf_bytes("all_instructions_64"); + let options = options(MaxRowsConfig::small(), 16); + for blowup in crate::zf_format::WHIR_BLOWUPS { + let format = BlockFormat { + zf: ZfFormat { + whir_blowup: blowup, + ..many_groups().zf + }, + ..many_groups() + }; + let proof = prove(&elf, &format, &options); + assert!(verify(&proof, &elf, &format), "blowup {blowup}"); + let bytes = rkyv::to_bytes::(&proof).expect("serialize"); + println!( + "RATE BLOCK DIGEST blowup={blowup} groups={} bytes={} blake3={}", + groups_of(&proof), + bytes.len(), + &blake3::hash(&bytes).to_hex()[..32] + ); + } +} + +/// The block's shape: many chunks of every table, spread over several groups, +/// each argued and opened on its own fork. +#[test] +fn a_program_spread_over_many_groups_proves_and_verifies() { + let elf = asm_elf_bytes("all_instructions_64"); + let proof = prove(&elf, &many_groups(), &options(MaxRowsConfig::small(), 16)); + assert!(groups_of(&proof) >= 3, "{} groups", groups_of(&proof)); + assert!(verify(&proof, &elf, &many_groups())); + // The groups are the verifier's: the same proof under another grouping is + // refused. + assert!(!verify(&proof, &elf, &one_group())); +} + +/// KECCAK_RND cut into tables of 2^3 rows: several chunks of one AIR, whose +/// bus sums add up to the whole table's. +#[test] +fn keccak_rnd_split_into_chunks_proves_and_verifies() { + let elf = asm_elf_bytes("test_keccak"); + let proof = prove(&elf, &many_groups(), &options(MaxRowsConfig::small(), 3)); + assert!( + proof.table_counts.keccak_rnd >= 2, + "{} KECCAK_RND tables", + proof.table_counts.keccak_rnd + ); + assert!(verify(&proof, &elf, &many_groups())); +} + +/// ★ The hold between the phases moves no byte: the same traces proved with +/// every tree level kept and with most of them dropped (re-hashed from the +/// rebuilt codewords at each query) give the same proof, byte for byte — when +/// the grind is deterministic (`LAMBDA_VM_DETERMINISTIC_GRIND`, read once per +/// process, so run this test alone to see the bytes). Otherwise the nonces a +/// parallel grind finds differ between any two proves and only the verdicts +/// are compared; the byte identity of a revived opening is also pinned without +/// grinding by `stacked_eval`'s `a_retired_and_revived_stack_opens_to_the_same_bytes`. +#[test] +fn the_kept_tree_depth_moves_no_byte_of_the_proof() { + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let program = Elf::load(&elf).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let mut traces = crate::tables::trace_builder::Traces::from_elf_and_logs( + &program, + &logs, + &MaxRowsConfig::small(), + &[], + ) + .expect("the traces build"); + block_whir::split_keccak_rnd(&mut traces, 16, CutPadding::default()).expect("the split"); + let mut proved = |drop_levels: usize| { + let mut o = options(MaxRowsConfig::small(), 16); + o.drop_levels = multilinear::whir_commit::TreeDrop::uniform(drop_levels); + let proof = block_whir::prove_traces( + &program, + &elf, + &mut traces, + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + false, + &|_, _| {}, + &mut Default::default(), + ) + .expect("prove"); + assert!(verify(&proof, &elf, &format), "drop {drop_levels}"); + rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec() + }; + let whole = proved(0); + let (three, eight, all) = (proved(3), proved(8), proved(64)); + if crypto::grinding::deterministic() { + assert_eq!(whole, three); + assert_eq!(whole, eight); + assert_eq!(whole, all); + } +} + +/// ★ Narrow storage moves no byte of the proof: the same traces proved with +/// every group held wide between the phases and held narrow (packed on the +/// host here, into the bytes the card packs) give the same proof — byte for +/// byte when the grind is deterministic (`LAMBDA_VM_DETERMINISTIC_GRIND`, read +/// once per process, so run this test alone to see the bytes; otherwise only +/// the verdicts are compared) — under the per-table and the batched argue. +/// With no card, every reader of a narrow table widens it on the host. +#[test] +fn narrow_storage_moves_no_byte_of_the_proof() { + use executor::elf::Elf; + use executor::vm::execution::Executor; + use stark::multilinear_block::Narrowing; + + let elf = asm_elf_bytes("all_instructions_64"); + let program = Elf::load(&elf).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let mut traces = crate::tables::trace_builder::Traces::from_elf_and_logs( + &program, + &logs, + &MaxRowsConfig::small(), + &[], + ) + .expect("the traces build"); + block_whir::split_keccak_rnd(&mut traces, 16, CutPadding::default()).expect("the split"); + for argue in [ArgueFormat::PerTable, ArgueFormat::BATCHED] { + let format = BlockFormat { + argue, + ..many_groups() + }; + let mut proved = |narrow: Narrowing| { + let mut o = options(MaxRowsConfig::small(), 16); + o.narrow = narrow; + let mut stamps = Default::default(); + let proof = block_whir::prove_traces( + &program, + &elf, + &mut traces, + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + false, + &|_, _| {}, + &mut stamps, + ) + .expect("prove"); + assert!(verify(&proof, &elf, &format), "{narrow:?} {argue:?}"); + let bytes = rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec(); + (bytes, stamps) + }; + let (wide, wide_stamps) = proved(Narrowing::Wide); + let (narrow, narrow_stamps) = proved(Narrowing::Host); + let packed = |s: &block_whir::BlockStamps| -> usize { + s.groups.iter().map(|g| g.packed_tables).sum() + }; + assert_eq!(packed(&wide_stamps), 0); + assert_eq!( + packed(&narrow_stamps), + narrow_stamps.tables, + "every table packed" + ); + // A process-wide count: other tests add to it, never take from it. + assert!(narrow_stamps.host_widens.0 > 0, "no card: the host widens"); + if crypto::grinding::deterministic() { + assert_eq!(wide, narrow, "{argue:?}"); + } + } +} + +/// ★ Uploading ahead moves no byte of the proof: the same traces proved with +/// each group's columns put on the card beside the previous group's commit and +/// put there after it give the same proof — byte for byte when the grind is +/// deterministic (`LAMBDA_VM_DETERMINISTIC_GRIND`, read once per process, so +/// run this test alone to see the bytes; otherwise only the verdicts are +/// compared) — under the per-table and the batched argue. Each group's paid +/// upload is at most its upload, and without the overlap all of it. +#[test] +fn uploading_ahead_moves_no_byte_of_the_proof() { + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf = asm_elf_bytes("all_instructions_64"); + let program = Elf::load(&elf).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let mut traces = crate::tables::trace_builder::Traces::from_elf_and_logs( + &program, + &logs, + &MaxRowsConfig::small(), + &[], + ) + .expect("the traces build"); + block_whir::split_keccak_rnd(&mut traces, 16, CutPadding::default()).expect("the split"); + for argue in [ArgueFormat::PerTable, ArgueFormat::BATCHED] { + let format = BlockFormat { + argue, + ..many_groups() + }; + let mut proved = |upload_ahead: bool| { + let mut o = options(MaxRowsConfig::small(), 16); + o.upload_ahead = upload_ahead; + let mut stamps = block_whir::BlockStamps::default(); + let proof = block_whir::prove_traces( + &program, + &elf, + &mut traces, + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + false, + &|_, _| {}, + &mut stamps, + ) + .expect("prove"); + assert!( + verify(&proof, &elf, &format), + "ahead {upload_ahead} {argue:?}" + ); + assert!(stamps.groups.len() >= 3, "{} groups", stamps.groups.len()); + for g in &stamps.groups { + assert!( + g.upload_paid <= g.upload_a + 1e-9, + "paid {} of {}", + g.upload_paid, + g.upload_a + ); + if !upload_ahead { + assert_eq!( + g.upload_paid, g.upload_a, + "nothing hidden without the overlap" + ); + } + } + rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec() + }; + let after = proved(false); + let ahead = proved(true); + if crypto::grinding::deterministic() { + assert_eq!(after, ahead, "{argue:?}"); + } + } +} + +/// The streamed build uploading ahead: each group's columns go up beside the +/// previous group's commit as the groups arrive, and the proof verifies. +#[test] +fn a_streamed_block_uploading_ahead_proves_and_verifies() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + for upload_ahead in [false, true] { + let mut o = streamed(MaxRowsConfig::small(), 16, 3); + o.upload_ahead = upload_ahead; + let proof = prove(&elf, &format, &o); + assert!(groups_of(&proof) >= 3, "{} groups", groups_of(&proof)); + assert!(verify(&proof, &elf, &format), "ahead {upload_ahead}"); + } +} + +/// The streamed build held narrow: every table packed exactly once — by its +/// group as the group is committed during the collect, by the finish (the +/// rest's tables, held narrow from the start), or, under G-pack, by its +/// generator (each streamed chunk written packed and laid out narrow, so its +/// group's commit finds it packed) — with G-pack off and on, and the proof +/// verifies. +#[test] +fn a_streamed_block_held_narrow_proves_and_verifies() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + for gpack in [false, true] { + let mut o = streamed(MaxRowsConfig::small(), 16, 3); + o.narrow = stark::multilinear_block::Narrowing::Host; + o.gpack = gpack; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove"); + assert!(groups_of(&proof) >= 3, "{} groups", groups_of(&proof)); + assert!(o.pack_finished && stamps.layout.packed_rest.0 > 0); + // The chunks G-pack wrote packed: every streamed chunk, counted by this + // block (`BlockStamps::streamed`). G-pack's own counts, the BLOCK GPACK + // line's, are the process's and split the chunks and the finish's + // tables by whether a kind's width hint was there yet, so they bound + // the chunks from above and no more. A `debug-checks` build writes + // every trace wide (`gpack::Plan`). + let chunks = stamps.streamed.1; + assert!(chunks > 0, "the build streamed"); + let written = if gpack && !cfg!(feature = "debug-checks") { + assert!( + stamps.layout.gpack.0 && stamps.layout.gpack.1.iter().sum::() >= chunks, + "G-pack built every streamed chunk: {:?}", + stamps.layout.gpack + ); + chunks + } else { + 0 + }; + assert_eq!( + stamps.groups.iter().map(|g| g.packed_tables).sum::() + + stamps.layout.packed_rest.0 + + written, + stamps.tables, + "gpack {gpack}: every table packed exactly once" + ); + assert!(stamps.report().contains("BLOCK NARROW: Host")); + assert!(verify(&proof, &elf, &format), "gpack {gpack}"); + } +} + +/// The memory log (`BlockOptions::memlog`) moves no byte of the proof — the +/// bytes are compared when the grind is deterministic +/// (`LAMBDA_VM_DETERMINISTIC_GRIND`, read once per process), the verdicts +/// always — and its accounting closes: when the proof is done every term in +/// flight and the committed tables are back at zero, and what is left is the +/// prepared columns. +#[test] +fn a_memory_log_moves_no_byte_and_its_terms_close() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let proved = |memlog: bool, workers: usize| { + let mut o = streamed(MaxRowsConfig::small(), 16, 3); + o.memlog = memlog; + // Laid out on worker threads. + o.layout_workers = workers; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove"); + assert!(verify(&proof, &elf, &format), "memlog {memlog}"); + let bytes = rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec(); + (bytes, stamps) + }; + let (off, off_stamps) = proved(false, 0); + let (on, on_stamps) = proved(true, 0); + let (_, workers_stamps) = proved(true, 3); + assert!(off_stamps.mem_terms.is_empty()); + for stamps in [&on_stamps, &workers_stamps] { + terms_close(stamps); + } + if crypto::grinding::deterministic() { + assert_eq!(off, on, "the memory log moved a proof byte"); + } +} + +/// Every memory term in flight, and the committed tables as held, are zero once +/// the proof is done; the prepared columns are left. +fn terms_close(stamps: &block_whir::BlockStamps) { + let term = |name: &str| -> usize { + stamps + .mem_terms + .iter() + .find(|(n, _)| *n == name) + .unwrap_or_else(|| panic!("no term {name}")) + .1 + }; + for name in [ + "exec", + "logs", + "walk", + "walked", + "builder", + "image", + "jobs", + "laying", + "open", + "sent", + "rest", + "rest_laid", + "committing", + "ahead", + "pack_wide", + "pack_ready", + "tops", + // Phase B lets the tables go; a release that took more than was held + // would wrap past zero. + "held_narrow", + "held_wide", + ] { + assert_eq!( + term(name), + 0, + "{name} is still counted when the proof is done" + ); + } + assert!(term("prepared") > 0, "the prepared columns are held"); +} + +/// The rest's waves: consecutive, in order, each within the budget unless one +/// item alone is larger, and every item in exactly one. +#[test] +fn the_rest_is_cut_into_waves_within_the_budget() { + let sizes = [3usize, 4, 2, 9, 1, 1, 5, 0, 6]; + for budget in [1usize, 5, 7, 10, 100] { + let waves = block_whir::waves(sizes.to_vec(), budget, |&b| b); + assert_eq!( + waves.iter().flatten().copied().collect::>(), + sizes, + "budget {budget}: the order or an item changed" + ); + for wave in &waves { + let sum: usize = wave.iter().sum(); + assert!(!wave.is_empty()); + assert!( + sum <= budget || wave.len() == 1, + "budget {budget}: {wave:?}" + ); + } + // A wave closes only when the next item would take it past the budget. + for pair in waves.windows(2) { + let sum: usize = pair[0].iter().sum(); + assert!(sum + pair[1][0] > budget, "budget {budget}: {pair:?}"); + } + } + assert!(block_whir::waves(Vec::::new(), 4, |&b| b).is_empty()); +} + +/// The rest laid out in waves, KECCAK_RND built as its tables at the finish +/// (b1: BlockOptions::rest_layout_bytes, finish_keccak_rnd_chunks) and the +/// finish's tables packed as they are built, laid out narrow and committed from +/// the card (b2: BlockOptions::pack_finished) move no byte: the same tables, +/// partition and statement as with all of them off, inline and on the worker +/// layout, and the proof bytes equal under the deterministic grind. +#[test] +fn the_rest_in_waves_and_the_finish_packed_move_no_byte() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + for workers in [0usize, 3] { + // (b1, b2) + let proved = |b1: bool, b2: bool| { + // Rows of 2^5, the least a KECCAK_RND table takes: it spans several. + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = workers; + o.finish_keccak_rnd_chunks = b1; + o.rest_layout_bytes = b1.then_some(1 << 16); + o.pack_finished = b2; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove"); + assert!( + verify(&proof, &elf, &format), + "b1 {b1}, b2 {b2}, {workers} workers" + ); + (proof, stamps) + }; + let (before, before_stamps) = proved(false, false); + assert!( + before.table_counts.keccak_rnd > 1, + "KECCAK_RND is one table: nothing was split" + ); + assert_eq!(before_stamps.layout.packed_rest.0, 0); + for (b1, b2) in [(true, false), (false, true), (true, true)] { + let (after, stamps) = proved(b1, b2); + let what = format!("b1 {b1}, b2 {b2}, {workers} workers"); + assert_eq!(before.groups, after.groups, "{what}: the partition"); + assert_eq!(before.table_num_vars, after.table_num_vars, "{what}"); + assert_eq!( + format!("{:?}", before.table_counts), + format!("{:?}", after.table_counts), + "{what}" + ); + if b2 { + assert!( + stamps.layout.packed_rest.0 > 0, + "{what}: no table was laid out narrow" + ); + } + if crypto::grinding::deterministic() { + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + assert_eq!(bytes(&before), bytes(&after), "{what}: the proof"); + } + } + } +} + +/// ★ G-pack (BlockOptions::gpack: the streamed chunks written packed and laid +/// out narrow, the finish's tables written packed) moves no byte: the same +/// partition, tables and statement as without it, inline and on the worker +/// layout, every streamed kind past its first chunk written packed, and the +/// proof bytes equal under the deterministic grind. +#[test] +fn gpack_moves_no_byte() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + for workers in [0usize, 3] { + let proved = |gpack: bool| { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = workers; + o.gpack = gpack; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove"); + assert!( + verify(&proof, &elf, &format), + "gpack {gpack}, {workers} workers" + ); + (proof, stamps) + }; + let (before, before_stamps) = proved(false); + let (after, stamps) = proved(true); + let what = format!("{workers} workers"); + assert!(!before_stamps.layout.gpack.0, "{what}"); + let [direct, narrowed, again, _] = stamps.layout.gpack.1; + assert!( + stamps.layout.gpack.0 && direct + narrowed + again > 0, + "{what}: {:?}", + stamps.layout.gpack + ); + assert_eq!(before.groups, after.groups, "{what}: the partition"); + assert_eq!(before.table_num_vars, after.table_num_vars, "{what}"); + assert_eq!( + format!("{:?}", before.table_counts), + format!("{:?}", after.table_counts), + "{what}" + ); + if crypto::grinding::deterministic() { + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + assert_eq!(bytes(&before), bytes(&after), "{what}: the proof"); + } + } +} + +/// A table the finish packed is laid out narrow only if its preprocessed +/// columns are the program's, word for word: one word changed in a packed +/// table's first preprocessed column and the layout refuses it. +#[test] +fn a_packed_table_with_a_wrong_preprocessed_column_is_refused() { + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf = asm_elf_bytes("all_instructions_64"); + let program = Elf::load(&elf).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let mut traces = crate::tables::trace_builder::Traces::from_elf_and_logs( + &program, + &logs, + &MaxRowsConfig::small(), + &[], + ) + .expect("the traces build"); + let opts = ProofOptions::default_test_options(); + let counts = traces.table_counts(); + let airs = crate::VmAirs::new( + &program, + &opts, + false, + &traces.page_configs, + &counts, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + crate::test_utils::halted_register_commitment(&opts, &program, &traces.public_output_bytes) + .preprocessed(), + ); + let refs = airs.air_refs(); + let pairs = airs.air_trace_pairs(&mut traces); + let (air, trace) = pairs + .into_iter() + .zip(refs.iter().copied()) + .map(|((_, trace, _), air)| (air, trace)) + .find(|(air, trace)| !air.precomputed_columns().is_empty() && trace.main_table.width != 0) + .expect("a table with preprocessed columns"); + let shape = ( + trace.main_table.width, + trace.main_table.height.trailing_zeros() as usize, + ); + let mut honest = trace.clone(); + assert!(honest.pack_main_narrow(), "{}: packs", air.name()); + block_whir::table_of_narrow(air, &mut honest, shape).expect("the honest table lays out"); + let mut wrong = trace.clone(); + let word = *wrong.main_table.get(0, 0); + wrong + .main_table + .set(0, 0, word + FieldElement::::one()); + assert!(wrong.pack_main_narrow()); + let refused = block_whir::table_of_narrow(air, &mut wrong, shape) + .err() + .expect("a wrong preprocessed column is refused"); + assert!( + format!("{refused:?}").contains("preprocessed column 0"), + "{refused:?}" + ); +} + +/// A narrow table whose width map is wrong widens to other words than were +/// committed, and the prover refuses at the opening: the recomputed codeword +/// does not hash to the kept tree top. +#[test] +fn a_wrong_narrow_width_map_is_refused_by_the_kept_tree() { + let elf = asm_elf_bytes("all_instructions_64"); + let mut o = options(MaxRowsConfig::small(), 16); + o.narrow = stark::multilinear_block::Narrowing::Host; + let refused = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &many_groups(), + &o, + &Deviations { + narrow_width_map: true, + ..Default::default() + }, + ); + match refused { + Err(e) => assert!( + format!("{e:?}").contains("RecomputedCodewordMismatch"), + "refused for another reason: {e:?}" + ), + Ok(_) => panic!("a wrong width map proved"), + } +} + +/// The same on a card: the group's tables packed there after the commit and +/// widened there again for phase B, one width map broken in between. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "box: needs a card"] +fn a_wrong_narrow_width_map_is_refused_on_the_card() { + let elf = asm_elf_bytes("all_instructions_64"); + let o = options(MaxRowsConfig::small(), 16); + let (honest, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &many_groups(), + &o, + &Deviations::default(), + ) + .expect("prove"); + assert_eq!( + stamps.groups.iter().map(|g| g.packed_tables).sum::(), + stamps.tables, + "every table packed on the card" + ); + // The tables too small for the card's argue are read on the host. + println!("NARROW CARD: host widens {:?}", stamps.host_widens); + assert!(verify(&honest, &elf, &many_groups())); + let refused = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &many_groups(), + &o, + &Deviations { + narrow_width_map: true, + ..Default::default() + }, + ); + match refused { + Err(e) => assert!( + format!("{e:?}").contains("RecomputedCodewordMismatch"), + "refused for another reason: {e:?}" + ), + Ok(_) => panic!("a wrong width map proved"), + } +} + +#[test] +fn a_block_proof_survives_serialization() { + let elf = asm_elf_bytes("sub"); + let proof = prove(&elf, &many_groups(), &options(MaxRowsConfig::small(), 16)); + let bytes = rkyv::to_bytes::(&proof).expect("serialize"); + let back: BlockWhirProof = + rkyv::from_bytes::<_, rkyv::rancor::Error>(&bytes).expect("deserialize"); + assert!(verify(&back, &elf, &many_groups())); +} + +/// A group proved on another group's fork is refused: the index each fork +/// absorbs after `S_post` is what ties a group's messages to it. +#[test] +fn a_group_proved_on_another_fork_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let swapped = prove_with( + &elf, + &format, + &options(MaxRowsConfig::small(), 16), + &Deviations { + fork_of: Some(|g| g ^ 1), + ..Default::default() + }, + ); + assert!(groups_of(&swapped) >= 2); + assert!(!verify(&swapped, &elf, &format)); +} + +/// A prover that leaves one KECCAK_RND chunk out — its rows, its argument and +/// its count, so every table it does prove is consistent — is refused on the +/// bus balance, which is checked once over every table of the block. The +/// chunk left out holds real rounds (the first): the last one of a small +/// program can be padding alone, which the bus does not see. +#[test] +fn a_block_missing_an_instance_is_refused_on_the_bus() { + let elf = asm_elf_bytes("test_keccak"); + let format = many_groups(); + let o = options(MaxRowsConfig::small(), 3); + let honest = prove(&elf, &format, &o); + let short = prove_with( + &elf, + &format, + &o, + &Deviations { + omit_first_keccak_rnd: true, + ..Default::default() + }, + ); + assert_eq!( + short.table_counts.keccak_rnd + 1, + honest.table_counts.keccak_rnd + ); + assert!(!verify(&short, &elf, &format)); +} + +type Felt = FieldElement; + +/// The deviations alone: a block proved without the prover's own check of its +/// preprocessed columns verifies, so each refusal below is its tamper's. +#[test] +fn a_block_proved_without_the_provers_column_check_verifies() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let o = options(MaxRowsConfig::small(), 16); + let proof = prove_with( + &elf, + &format, + &o, + &Deviations { + unchecked_preprocessed: true, + ..Default::default() + }, + ); + assert!( + proof.table_counts.cpu >= 2, + "{} CPU tables", + proof.table_counts.cpu + ); + assert!(verify(&proof, &elf, &format)); +} + +/// ⛔ A CPU table's TIMESTAMP is the verifier's column (main's preprocessed +/// clock): a table with one timestamp moved, the clock a prover could +/// otherwise set row by row, is refused. +#[test] +fn a_block_with_a_moved_cpu_timestamp_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let o = options(MaxRowsConfig::small(), 16); + let moved = prove_with( + &elf, + &format, + &o, + &Deviations { + unchecked_preprocessed: true, + tamper_traces: Some(|traces| { + let ts = crate::tables::cpu::cols::TIMESTAMP; + let table = &mut traces.cpus[1].main_table; + let value = *table.get(1, ts); + table.set(1, ts, value + Felt::from(4u64)); + }), + ..Default::default() + }, + ); + assert!(!verify(&moved, &elf, &format)); +} + +/// ⛔ REGISTER's FINI is the halted file (main's final-register pin): a +/// block whose final register file is another one is refused. +#[test] +fn a_block_with_another_final_register_file_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let o = options(MaxRowsConfig::small(), 16); + let other = prove_with( + &elf, + &format, + &o, + &Deviations { + unchecked_preprocessed: true, + tamper_traces: Some(|traces| { + let fini = crate::tables::register::cols::FINI; + let table = &mut traces.register.main_table; + let value = *table.get(2, fini); + table.set(2, fini, value + Felt::one()); + }), + ..Default::default() + }, + ); + assert!(!verify(&other, &elf, &format)); +} + +/// ⛔ KECCAK's IS_WH messages need the table that receives them: a block +/// proved without it (its statement then counts none) is refused on the bus. +#[test] +fn a_block_without_its_is_wh_table_is_refused() { + let elf = asm_elf_bytes("test_keccak"); + let format = many_groups(); + let o = options(MaxRowsConfig::small(), 16); + let honest = prove(&elf, &format, &o); + assert_eq!(honest.table_counts.is_wh, 1); + assert!(verify(&honest, &elf, &format)); + let short = prove_with( + &elf, + &format, + &o, + &Deviations { + tamper_traces: Some(|traces| traces.is_whs.clear()), + ..Default::default() + }, + ); + assert_eq!(short.table_counts.is_wh, 0); + assert!(!verify(&short, &elf, &format)); +} + +/// ⛔ KECCAK's μ is a bit (main's multiplicity fix): a row at μ = −1 would +/// receive its IS_WH and IS_HALF messages instead of sending them, and could +/// cancel forged sends. A block with one is refused. +#[test] +fn a_block_with_a_negative_keccak_multiplicity_is_refused() { + let elf = asm_elf_bytes("test_keccak"); + let format = many_groups(); + let o = options(MaxRowsConfig::small(), 16); + let negative = prove_with( + &elf, + &format, + &o, + &Deviations { + tamper_traces: Some(|traces| { + let mu = crate::tables::keccak::cols::MU; + traces.keccaks[0].main_table.set(0, mu, -Felt::one()); + }), + ..Default::default() + }, + ); + assert!(!verify(&negative, &elf, &format)); +} + +fn streamed( + max_rows: MaxRowsConfig, + keccak_rnd_rows_log2: usize, + window_log2: usize, +) -> BlockOptions { + let mut o = options(max_rows, keccak_rnd_rows_log2); + o.window_log2 = Some(window_log2); + o +} + +/// ★ The build streamed into phase A: tables are packed into groups as their +/// chunks complete, the partition rides the statement, and the proof verifies. +#[test] +fn a_streamed_block_proves_and_verifies() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let proof = prove(&elf, &format, &streamed(MaxRowsConfig::small(), 16, 3)); + assert!(groups_of(&proof) >= 3, "{} groups", groups_of(&proof)); + // Streaming packs in arrival order, not table order. + let order: Vec = proof.groups.iter().flatten().copied().collect(); + assert!( + order.windows(2).any(|w| w[0] > w[1]), + "the groups are table order" + ); + assert!(verify(&proof, &elf, &format)); +} + +/// The stamps of one `BLOCK HEAD` item: the number after each of `prefixes`, +/// and both of each `-`. +fn head_stamps(item: &str, prefixes: &[char]) -> Vec { + let mut out = Vec::new(); + for word in item.split(' ') { + let Some(first) = word.chars().next() else { + continue; + }; + if !prefixes.contains(&first) { + continue; + } + for part in word[1..].split('-') { + if let Ok(v) = part.parse::() { + out.push(v); + } + } + } + out +} + +/// ★ Phase A's head stamped (`LAMBDA_VM_BLOCK_HEAD_LOG`) moves no byte: with +/// it on, the streamed prove hands back its four `BLOCK HEAD` lines — every +/// stamped window passed before it was handed out, walked, then absorbed in +/// window order; every stamped chunk handed out, laid out, then placed, one +/// item a chunk handed out; the layout by table counting every chunk; the +/// card's early setup, by default with the card's kernels — and the proof is +/// the one without it (its bytes too, under the deterministic grind), laid out +/// inline and on workers. Off, nothing is stamped. +#[test] +fn the_head_log_stamps_the_head_and_moves_no_byte() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + for workers in [0usize, 3] { + let proved = |head: bool| { + let mut o = streamed(MaxRowsConfig::small(), 16, 3); + o.layout_workers = workers; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + head_log: Some(head), + ..Deviations::default() + }, + ) + .expect("prove"); + assert!(verify(&proof, &elf, &format), "{workers} workers"); + (proof, stamps) + }; + let (off, off_stamps) = proved(false); + assert!(off_stamps.head.is_empty(), "{workers} workers: off"); + let (on, stamps) = proved(true); + let what = format!("{workers} workers"); + let head = &stamps.head; + assert_eq!(head.len(), 4, "{what}: {head:?}"); + assert!(head[0].starts_with("BLOCK HEAD WINDOWS: "), "{what}"); + assert!(head[1].starts_with("BLOCK HEAD CHUNKS: "), "{what}"); + assert!(head[2].starts_with("BLOCK HEAD TABLES: "), "{what}"); + // The card is set up as the prove starts by default + // (`LAMBDA_VM_BLOCK_CARD_EARLY`); a build without its kernels sets + // up nothing. + if cfg!(feature = "cuda") { + assert!( + head[3].starts_with("BLOCK HEAD CARD: set up at ") + && head[3].ends_with(" s on a thread of its own"), + "{what}: {}", + head[3] + ); + } else { + assert_eq!( + head[3], "BLOCK HEAD CARD: set up at its first use", + "{what}" + ); + } + let windows: Vec<&str> = head[0].split(" · ").skip(1).collect(); + assert!(!windows.is_empty(), "{what}: no window stamped"); + let mut absorbed = 0.0; + for w in &windows { + let s = head_stamps(w, &['r', 'p', 'w', 'a']); + let [received, passed, from, to, at] = s[..] else { + panic!("{what}: window {w}"); + }; + assert!( + received <= passed && from <= to && to <= at, + "{what}: window {w}" + ); + assert!(at >= absorbed, "{what}: window {w} absorbed out of order"); + absorbed = at; + } + let chunks: Vec<&str> = head[1].split(" · ").skip(1).collect(); + let handed = stamps.streamed.1; + assert!(handed > 0, "{what}: no chunk streamed"); + assert_eq!(chunks.len(), handed.min(128), "{what}: {head:?}"); + for c in &chunks { + let s = head_stamps(c, &['h', 'l', 'p']); + let [out, from, to, placed] = s[..] else { + panic!("{what}: chunk {c}"); + }; + assert!( + out <= from && from <= to && to <= placed, + "{what}: chunk {c}" + ); + } + let laid: usize = head[2] + .split(" · ") + .skip(1) + .map(|t| { + t.split(' ') + .nth(1) + .and_then(|n| n.parse::().ok()) + .expect("a table's chunks") + }) + .sum(); + assert_eq!(laid, handed, "{what}: the layout by table"); + assert_eq!(off.groups, on.groups, "{what}: the partition"); + assert_eq!(off.table_num_vars, on.table_num_vars, "{what}"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&off), bytes(&on), "{what}: the proof"); + } + } +} + +/// A host-only build (a box harness's measure of the host's pace) drains the +/// groups and tables the streamed prove commits, and commits none. +#[test] +fn a_host_only_build_drains_the_groups_the_prove_commits() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let options = streamed(MaxRowsConfig::small(), 16, 3); + let proof = prove(&elf, &format, &options); + let (stamps, drained) = block_whir::trace_block_host_only( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &options, + ) + .expect("the host-only build"); + let tables: usize = proof.groups.iter().map(Vec::len).sum(); + assert!( + drained.contains(&format!( + "· {} groups · {tables} tables ·", + groups_of(&proof) + )), + "{drained}" + ); + assert!(stamps.phase_a > 0.0 && stamps.build > 0.0); +} + +#[test] +fn a_streamed_block_with_keccak_chunks_proves_and_verifies() { + let elf = asm_elf_bytes("test_keccak"); + let format = many_groups(); + let proof = prove(&elf, &format, &streamed(MaxRowsConfig::small(), 3, 4)); + assert!(proof.table_counts.keccak_rnd >= 2); + assert!(verify(&proof, &elf, &format)); +} + +/// Keccak-256 calls as a block: KECCAK cut at 4 rows, so a call's rows straddle +/// tables and chain across them through the KeccakSponge bus; the proof +/// verifies. +#[test] +fn a_block_with_keccak256_calls_across_keccak_tables_proves_and_verifies() { + let elf = asm_elf_bytes("test_keccak256"); + let format = many_groups(); + let mut o = streamed(MaxRowsConfig::small(), 6, 4); + o.keccak_rows_log2 = 2; + let proof = prove(&elf, &format, &o); + assert!(proof.table_counts.keccak >= 4); + assert!(verify(&proof, &elf, &format)); +} + +/// The streamed build cuts ECDAS at the run's end like the whole-run build: +/// test_ecsm_multi's 42 ECDAS steps (64 rows padded) in 16-row tables, +/// windows of 2^4 cycles; at the default the fourth table, padding rows alone, +/// is not built. +#[test] +fn a_streamed_block_with_ecdas_chunks_proves_and_verifies() { + let elf = asm_elf_bytes("test_ecsm_multi"); + let format = many_groups(); + let mut o = streamed(MaxRowsConfig::small(), 16, 4); + o.ecdas_rows_log2 = 4; + let proof = prove(&elf, &format, &o); + assert_eq!(proof.table_counts.ecdas, 3); + assert!(verify(&proof, &elf, &format)); +} + +/// ★ With the builder dropping each streamed chunk's ops as it leaves: the same +/// groups and table counts as keeping them, and the proof verifies. +#[test] +fn a_streamed_block_with_dropped_ops_proves_and_verifies() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let keep = streamed(MaxRowsConfig::small(), 5, 3); + let kept = prove(&elf, &format, &keep); + let mut o = keep.clone(); + o.drop_streamed_ops = true; + let proof = prove(&elf, &format, &o); + assert!(groups_of(&proof) >= 3, "{} groups", groups_of(&proof)); + assert_eq!(proof.groups, kept.groups); + assert_eq!( + format!("{:?}", proof.table_counts), + format!("{:?}", kept.table_counts) + ); + assert!(verify(&proof, &elf, &format)); +} + +/// A streamed prove with the shadow regenerator: its proof, its stamps. +fn shadowed( + elf: &[u8], + format: &BlockFormat, + o: &BlockOptions, + deviations: &Deviations, +) -> (BlockWhirProof, block_whir::BlockStamps) { + let mut o = o.clone(); + o.regen = Some(block_whir::regen::RegenMode::Shadow); + prove_block_whir_with( + elf, + &[], + &ProofOptions::default_test_options(), + format, + &o, + deviations, + ) + .expect("prove") +} + +/// ★ The shadow regenerator (D-WHIR-NODISK N1) rebuilds every streamed chunk +/// from a fresh execution of the run and moves no byte: phase A records a +/// recipe per streamed chunk, each digested as it was committed; phase B +/// regenerates every one with no mismatch, failure or miss, and plans them in +/// group order and rest first; the proof has the partition of the prove +/// without the shadow, verifies, and its bytes are equal under the +/// deterministic grind. Inline and on three layout threads, the streamed ops +/// kept and dropped. +#[test] +fn the_shadow_regenerates_every_streamed_chunk_and_moves_no_byte() { + let format = many_groups(); + for name in ["test_keccak_multi"] { + let elf = asm_elf_bytes(name); + for (workers, drop_ops) in [(0, false), (3, true)] { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = workers; + o.drop_streamed_ops = drop_ops; + o.regen = Some(block_whir::regen::RegenMode::Off); + let (plain, plain_stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove"); + assert!(plain_stamps.regen.is_none(), "off records nothing"); + let (proof, stamps) = shadowed(&elf, &format, &o, &Deviations::default()); + let at = format!("{name}, {workers} layout threads, ops dropped {drop_ops}"); + let r = stamps.regen.as_ref().expect("the shadow's readout"); + let lines = r.lines.join("\n"); + let streamed = stamps.streamed.1; + assert!(streamed > 0, "{at}: nothing streamed"); + assert_eq!(r.recipes, streamed, "{at}: a recipe a chunk\n{lines}"); + assert_eq!(r.digested, streamed, "{at}: every recipe digested\n{lines}"); + assert_eq!(r.stray, 0, "{at}\n{lines}"); + assert_eq!(r.regenerated, streamed, "{at}: every chunk\n{lines}"); + assert_eq!( + (r.mismatches, r.failed, r.missing), + (0, 0, 0), + "{at}\n{lines}" + ); + assert_eq!(r.error, None, "{at}\n{lines}"); + assert!(r.would_wait_rest_first.is_some(), "{at}\n{lines}"); + assert!( + lines.contains("BLOCK REGEN plan (rest first)") + && lines.contains("BLOCK REGEN plan (group order)"), + "{lines}" + ); + assert_eq!(proof.groups, plain.groups, "{at}: the partition"); + assert!(verify(&proof, &elf, &format), "{at}"); + if crypto::grinding::deterministic() { + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + assert_eq!(bytes(&plain), bytes(&proof), "{at}: the proof"); + } + } + } +} + +/// ★ The shadow's check is load-bearing: a regenerator whose slicer drops the +/// first CPU op (or MEMW_R row) rebuilds a chunk that is not the one +/// committed, which the shadow reports as a mismatch — never a panic, and +/// never a proof over it: the proof still verifies, and the rest-first plan +/// has no would-wait to give. +#[test] +fn a_wrong_regeneration_is_reported_never_proved() { + use crate::tables::trace_builder::StreamTable; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let o = streamed(MaxRowsConfig::small(), 5, 3); + for table in [StreamTable::Cpu, StreamTable::MemwRegister] { + let (proof, stamps) = shadowed( + &elf, + &format, + &o, + &Deviations { + regen_drop_one_op: Some(table), + ..Deviations::default() + }, + ); + let r = stamps.regen.expect("the shadow's readout"); + let lines = r.lines.join("\n"); + assert!(r.mismatches > 0, "{table:?}: no mismatch\n{lines}"); + assert!( + lines.contains(&format!("BLOCK REGEN MISMATCH {table:?}[0]")), + "{lines}" + ); + assert_eq!(r.would_wait_rest_first, None, "{lines}"); + assert!(verify(&proof, &elf, &format), "{table:?}"); + } +} + +/// The shadow runs only where the regenerator rebuilds every streamed chunk: +/// with the chunks generated 64-bit, nothing is recorded and the prove is as +/// without it. +#[test] +fn the_shadow_stays_off_where_it_cannot_rebuild_the_chunks() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let mut wide = streamed(MaxRowsConfig::small(), 5, 3); + wide.gpack = false; + let (proof, stamps) = shadowed(&elf, &format, &wide, &Deviations::default()); + assert!(stamps.regen.is_none(), "64-bit: recorded"); + assert!(verify(&proof, &elf, &format), "64-bit"); +} + +/// ★ Phase B's order moves no byte (D-WHIR-NODISK §2.3): each group proves on +/// its own fork `S_post ‖ g`, so taking the groups reversed or odd ones first +/// proves the bytes of group order under the deterministic grind, with the +/// same partition, and verifies; an order that is not a permutation of the +/// groups is refused. So does each group's openings run beside the next +/// group's argue (phase B's overlap), in group order and reversed. +#[test] +fn phase_b_in_any_order_proves_the_same_bytes() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + use stark::multilinear_block::PhaseBOverlap::{Off, On, Pinned}; + let proved = |order: Option Vec>, + overlap: stark::multilinear_block::PhaseBOverlap| { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.phase_b_overlap = overlap; + prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + phase_b_order: order, + ..Deviations::default() + }, + ) + .map(|(proof, _)| proof) + }; + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let in_order = proved(None, Off).expect("prove"); + assert!( + in_order.groups.len() > 3, + "{} groups", + in_order.groups.len() + ); + let reversed: fn(usize) -> Vec = |n| (0..n).rev().collect(); + let odd_first: fn(usize) -> Vec = + |n| (1..n).step_by(2).chain((0..n).step_by(2)).collect(); + for (what, order, overlap) in [ + ("reversed", Some(reversed), Off), + ("odd first", Some(odd_first), Off), + ("in order, overlapped", None, On), + ("reversed, overlapped", Some(reversed), On), + ("in order, pinned", None, Pinned), + ("reversed, pinned", Some(reversed), Pinned), + ] { + let proof = proved(order, overlap).expect("prove"); + assert_eq!(proof.groups, in_order.groups, "{what}: the partition"); + assert!(verify(&proof, &elf, &format), "{what}"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&proof), bytes(&in_order), "{what}: the proof"); + } + } + let twice: fn(usize) -> Vec = |n| std::iter::repeat_n(0, n).collect(); + let short: fn(usize) -> Vec = |n| (1..n).collect(); + for (what, order) in [("a group twice", twice), ("a group left out", short)] { + for overlap in [Off, On, Pinned] { + assert!(proved(Some(order), overlap).is_err(), "{what}: proved"); + } + } +} + +/// ★ Phase B led by a leaf (`W3_LEAF_FIRST`, [`block_whir::PhaseBLead`]) on a +/// streamed block: with a leaf offered before phase B starts, phase B moves +/// that leaf's groups ahead at its first turn and takes that order (the +/// observer hears it, the stamps record it), with the overlap off and pinned; +/// the proof is the bytes of phase B's own order under the deterministic grind +/// (`LAMBDA_VM_DETERMINISTIC_GRIND=1`), and it verifies. Unled, phase B keeps +/// group order and records no lead. +#[test] +fn phase_b_led_by_a_leaf_proves_the_same_bytes() { + use stark::multilinear_block::PhaseBOverlap::{Off, Pinned}; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + for overlap in [Off, Pinned] { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.phase_b_overlap = overlap; + let proved = |lead: Option<&block_whir::PhaseBLead>| { + let heard = std::sync::Mutex::new(Vec::new()); + let (proof, stamps) = block_whir::prove_block_whir_observed_led( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &|_, _| {}, + &|opened| heard.lock().expect("heard").push(opened.group), + lead, + ) + .expect("prove"); + (proof, stamps, heard.into_inner().expect("heard")) + }; + let (plain, plain_stamps, plain_heard) = proved(None); + let n = plain.groups.len(); + assert!(n > 3, "{overlap:?}: {n} groups"); + let group_order: Vec = (0..n).collect(); + assert_eq!(plain_stamps.phase_b_order, group_order, "{overlap:?}"); + assert_eq!(plain_heard, group_order, "{overlap:?}"); + // One leaf of every third group from group 2: it leads from turn 0. + let leaf: Vec = (2..n).step_by(3).collect(); + let lead = block_whir::PhaseBLead::default(); + assert!(lead.offer(vec![leaf.clone()])); + let (led, led_stamps, led_heard) = proved(Some(&lead)); + let taken = lead.taken().expect("phase B read the offer"); + let expected: Vec = std::iter::once(0) + .chain(leaf.iter().copied()) + .chain((1..n).filter(|g| !leaf.contains(g))) + .collect(); + assert_eq!( + (taken.turn, taken.leaf, &taken.moved), + (0, Some(0), &leaf), + "{overlap:?}" + ); + assert_eq!(taken.order, expected, "{overlap:?}"); + assert_ne!( + taken.order, group_order, + "{overlap:?}: the lead moved groups" + ); + assert_eq!(led_stamps.phase_b_order, expected, "{overlap:?}"); + assert_eq!(led_heard, expected, "{overlap:?}: the order heard"); + assert_eq!(led.groups, plain.groups, "{overlap:?}: the partition"); + assert!(verify(&led, &elf, &format), "{overlap:?}"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&led), bytes(&plain), "{overlap:?}: the proof"); + } + } +} + +/// Phase B's overlap on a streamed block: an opening or an argue that fails +/// beside the other, or a panic on the opening thread, is the prove's error, +/// never a panic; a refusal keeps its transition's overlap and turns the rest +/// serial, with the proof's bytes unchanged, and the readout counts each. +#[test] +fn phase_b_overlap_failures_are_the_proves_error_and_a_refusal_turns_it_serial() { + use stark::multilinear_block::OverlapFaults; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.phase_b_overlap = stark::multilinear_block::PhaseBOverlap::On; + let run = |faults: OverlapFaults| { + prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + overlap_faults: faults, + ..Deviations::default() + }, + ) + }; + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + // First the overlap with no fault: every group but the last opened beside + // an argue, so the faults below meet the opening thread where they aim. + let (plain, stamps) = run(OverlapFaults::default()).expect("prove"); + let n = stamps.groups.len(); + assert!(n > 3, "{n} groups"); + assert_eq!( + stamps.groups.iter().filter(|g| g.overlapped).count(), + n - 1, + "every group but the last opened beside an argue\n{}", + stamps.report() + ); + // The panic at place 0: the opening thread's first openings, in the first + // transition, before any refusal could turn the rest serial. + for (what, faults, stage) in [ + ( + "an opening fails", + OverlapFaults { + open_fails: Some(1), + ..OverlapFaults::default() + }, + "a test's failed opening", + ), + ( + "an argue fails", + OverlapFaults { + argue_fails: Some(2), + ..OverlapFaults::default() + }, + "a test's failed argue", + ), + ( + "the opening thread panics", + OverlapFaults { + open_panics: Some(0), + ..OverlapFaults::default() + }, + "the previous group's openings", + ), + ] { + match run(faults) { + Err(e) => assert!(format!("{e:?}").contains(stage), "{what}: {e:?}"), + Ok(_) => panic!("{what}: proved"), + } + } + let (latched, stamps) = run(OverlapFaults { + refusal_at: Some(2), + ..OverlapFaults::default() + }) + .expect("prove"); + assert!(verify(&latched, &elf, &format)); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&latched), bytes(&plain), "a refusal: the proof"); + } + let report = stamps.report(); + let counted = format!( + "BLOCK PHASE-B OVERLAP: on · transitions overlapped 2 of {} (serial after a refusal {})", + n - 1, + n - 3 + ); + assert!(report.contains(&counted), "{counted}\n{report}"); +} + +/// `LAMBDA_VM_PHASE_B_OVERLAP`: unset `pinned` on a card build and off on a +/// host-only one, `1` on, `pinned` on with the openings' read-backs through +/// pinned staging; `0` or any other word off. +#[test] +fn the_phase_b_overlap_knob_reads_one_as_on() { + use crate::block_whir::{PHASE_B_OVERLAP_DEFAULT, phase_b_overlap_from}; + use stark::multilinear_block::PhaseBOverlap::{Off, On, Pinned}; + let default = if cfg!(feature = "cuda") { Pinned } else { Off }; + assert_eq!(PHASE_B_OVERLAP_DEFAULT, default); + assert_eq!(phase_b_overlap_from(None), default); + assert_eq!(phase_b_overlap_from(Some("0")), Off); + assert_eq!(phase_b_overlap_from(Some("1")), On); + assert_eq!(phase_b_overlap_from(Some(" 1 ")), On); + assert_eq!(phase_b_overlap_from(Some("pinned")), Pinned); + assert_eq!(phase_b_overlap_from(Some(" pinned ")), Pinned); + assert_eq!(phase_b_overlap_from(Some("on")), Off); + assert_eq!(phase_b_overlap_from(Some("Pinned")), Off); + assert_eq!(phase_b_overlap_from(Some("blocking")), Off); + assert_eq!(phase_b_overlap_from(Some("2")), Off); +} + +/// `LAMBDA_VM_BLOCK_SCHEDSTAT`: `1` on, anything else off. +#[test] +fn the_schedstat_knob_reads_one_as_on() { + use crate::block_whir::phase_b_sched_from; + assert!(!phase_b_sched_from(None)); + assert!(!phase_b_sched_from(Some("0"))); + assert!(phase_b_sched_from(Some("1"))); + assert!(phase_b_sched_from(Some(" 1 "))); + assert!(!phase_b_sched_from(Some("on"))); +} + +/// The driver thread's schedstat readout, from stamps alone: each turn's CPU +/// and run-queue delay in the tenth of phase B it began in, the first three +/// tenths summed; unreadable or all zeros is said as unavailable, never as 0; +/// with the knob off, no line. +#[test] +fn the_schedstat_readout_counts_turns_by_tenth_and_says_unavailable() { + use stark::multilinear_block::GroupStamps; + let turn = |start_b: f64, sched_b: Option<(u64, u64)>| GroupStamps { + start_b, + sched_b, + ..GroupStamps::default() + }; + let ms = 1_000_000u64; + let stamps = |groups: Vec, on: bool| { + block_whir::BlockStamps { + groups, + phase_b: 10.0, + phase_b_sched: on, + ..Default::default() + } + .report() + }; + let read = stamps( + vec![ + turn(0.5, Some((400 * ms, 300 * ms))), + turn(2.5, Some((500 * ms, 100 * ms))), + turn(9.9, Some((600 * ms, 20 * ms))), + ], + true, + ); + assert!( + read.contains( + "BLOCK PHASE-B SCHED: the driver thread over 3 group turns · CPU 1.500 s · run-queue \ + delay 0.420 s · first three tenths of phase B: CPU 0.900 s, run-queue delay 0.400 s · \ + by tenth (a turn in the tenth it began) CPU [0.40, 0.00, 0.50, 0.00, 0.00, 0.00, 0.00, \ + 0.00, 0.00, 0.60] · run-queue delay [0.30, 0.00, 0.10, 0.00, 0.00, 0.00, 0.00, 0.00, \ + 0.00, 0.02]\n" + ), + "{read}" + ); + let none = stamps(vec![turn(0.5, None), turn(2.5, None)], true); + assert!( + none.contains( + "BLOCK PHASE-B SCHED: unavailable (/proc/thread-self/schedstat did not read)\n" + ), + "{none}" + ); + let zeros = stamps(vec![turn(0.5, Some((0, 0)))], true); + assert!( + zeros.contains("BLOCK PHASE-B SCHED: unavailable (schedstat reads zeros"), + "{zeros}" + ); + let off = stamps(vec![turn(0.5, Some((400 * ms, 300 * ms)))], false); + assert!(!off.contains("BLOCK PHASE-B SCHED"), "{off}"); +} + +/// The overlap's readout, on and off, from stamps alone: the per-group `paid`, +/// the opening helper, the measured transitions on the phase-B ledger and the +/// overlap's own line; off, every line as it was and the overlap line `off`. +#[test] +fn the_phase_b_overlap_readout_names_its_transitions() { + use stark::multilinear_block::GroupStamps; + let gib = 1u64 << 30; + let groups = vec![ + GroupStamps { + argue: 0.5, + encode: 0.01, + open: 0.09, + open_paid: 0.02, + overlapped: true, + open_draws: (40, 7), + ..GroupStamps::default() + }, + GroupStamps { + argue: 0.4, + encode: 0.02, + open: 0.08, + open_paid: 0.0, + overlapped: true, + transition_reserved: 20 * gib, + open_draws: (30, 5), + ..GroupStamps::default() + }, + GroupStamps { + argue: 0.3, + encode: 0.01, + open: 0.05, + open_paid: 0.06, + transition_reserved: 21 * gib, + open_draws: (6, 0), + ..GroupStamps::default() + }, + ]; + let on = block_whir::BlockStamps { + groups: groups.clone(), + phase_b_overlap: true, + ..Default::default() + } + .report(); + assert!(on.contains("encode 0.010 open 0.090 paid 0.020\n"), "{on}"); + assert!( + on.contains("+ one upload helper + one opening helper\n"), + "{on}" + ); + assert!( + on.contains("open(g) beside argue(g+1) [20480, 21504] MiB measured ·"), + "{on}" + ); + assert!( + on.contains( + "BLOCK PHASE-B OVERLAP: on · transitions overlapped 2 of 2 (serial after a refusal 0) · \ + openings beside an argue 0.200 s, hidden 0.180 s, paid 0.020 s + alone 0.060 s · \ + argue 1.200 s · transition ledger peak max 21504 MiB" + ), + "{on}" + ); + assert!( + on.contains( + "openings' stream draws beside an argue background 12 / foreground 70 (the argue's \ + too), alone background 0 / foreground 6 · trims 0 · " + ), + "{on}" + ); + assert!( + on.contains( + "· trims 0 · the openings' read-backs pinned 0 / pageable 0 (0.0 MiB pinned)\n" + ), + "{on}" + ); + let pinned = block_whir::BlockStamps { + groups: groups.clone(), + phase_b_overlap: true, + phase_b_pinned: true, + background_readbacks: (5887, 0, 3 << 20), + ..Default::default() + } + .report(); + assert!( + pinned.contains("BLOCK PHASE-B OVERLAP: pinned · transitions overlapped 2 of 2"), + "{pinned}" + ); + assert!( + pinned.contains( + "· trims 0 · the openings' read-backs pinned 5887 / pageable 0 (3.0 MiB pinned)\n" + ), + "{pinned}" + ); + let off = block_whir::BlockStamps { + groups, + ..Default::default() + } + .report(); + assert!(off.contains("encode 0.010 open 0.090\n"), "{off}"); + assert!(off.contains("+ one upload helper\n"), "{off}"); + assert!(!off.contains(" measured"), "{off}"); + assert!( + off.contains("BLOCK PHASE-B OVERLAP: off · transitions overlapped 2 of 2"), + "{off}" + ); +} + +/// ★ On a card with a background pool (`LAMBDA_VM_STREAM_CLASSES=equal` or +/// `priority`), every stream the openings beside an argue draw is a +/// background one: as many as the same group's openings drew alone, all of +/// them foreground, with the overlap off. A device call they made from +/// another thread would draw a foreground stream and show here as fewer. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "box: needs a card and LAMBDA_VM_STREAM_CLASSES=equal"] +fn phase_b_overlap_openings_draw_background_streams() { + assert!( + math_cuda::stream_class::setting().split(), + "run with LAMBDA_VM_STREAM_CLASSES=equal or priority" + ); + let elf = asm_elf_bytes("test_keccak_multi"); + let mut format = many_groups(); + // Codewords of 2^16 and more: the card's, not the host's. + format.zf.whir_stack = StackVars::new(14).expect("a valid stack"); + let run = |overlap: bool| { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.phase_b_overlap = if overlap { + stark::multilinear_block::PhaseBOverlap::On + } else { + stark::multilinear_block::PhaseBOverlap::Off + }; + prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove") + .1 + }; + let (alone, beside) = (run(false), run(true)); + assert!(alone.groups.len() > 2, "{} groups", alone.groups.len()); + let drawn: u64 = alone.groups.iter().map(|g| g.open_draws.0).sum(); + println!( + "PHASE-B DRAWS: alone {:?} · beside {:?}", + alone + .groups + .iter() + .map(|g| g.open_draws) + .collect::>(), + beside + .groups + .iter() + .map(|g| g.open_draws) + .collect::>() + ); + assert!(drawn > 0, "no opening drew a stream: nothing to compare"); + let mut compared = 0; + for (g, (a, b)) in alone.groups.iter().zip(&beside.groups).enumerate() { + assert_eq!(a.open_draws.1, 0, "group {g}: a background draw alone"); + if b.overlapped { + assert_eq!( + b.open_draws.1, a.open_draws.0, + "group {g}: beside an argue, the openings' draws" + ); + compared += usize::from(a.open_draws.0 > 0); + } + } + assert!(compared > 0, "no overlapped group drew a stream"); +} + +/// ★ On a card, `pinned` sends every read-back of the openings' threads +/// through pinned staging, none the stream's own way, and the proof is the +/// overlap's own bytes and verifies. With the overlap on but not pinned, the +/// openings read back the stream's own way and none takes the pinned path. +#[cfg(feature = "cuda")] +#[test] +#[ignore = "box: needs a card"] +fn pinned_openings_read_back_through_pinned_staging() { + use stark::multilinear_block::PhaseBOverlap::{On, Pinned}; + let elf = asm_elf_bytes("test_keccak_multi"); + let mut format = many_groups(); + // Codewords of 2^16 and more: the card's, not the host's. + format.zf.whir_stack = StackVars::new(14).expect("a valid stack"); + let run = |overlap| { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.phase_b_overlap = overlap; + prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove") + }; + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let (on_proof, on) = run(On); + let (pinned_proof, pinned) = run(Pinned); + assert!( + pinned.groups.iter().any(|g| g.overlapped), + "no transition overlapped" + ); + let (on_pinned, on_pageable, _) = on.background_readbacks; + assert!( + on_pinned == 0 && on_pageable > 0, + "on: the openings' read-backs {:?}", + on.background_readbacks + ); + let (pinned_pinned, pinned_pageable, _) = pinned.background_readbacks; + assert!( + pinned_pinned > 0 && pinned_pageable == 0, + "pinned: the openings' read-backs {:?}", + pinned.background_readbacks + ); + let report = pinned.report(); + assert!( + report.contains("BLOCK PHASE-B OVERLAP: pinned ·"), + "{report}" + ); + assert!(verify(&pinned_proof, &elf, &format), "pinned"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&pinned_proof), bytes(&on_proof), "the proof"); + } +} + +/// A streamed prove under live regeneration `mode`, with `deviations`: its +/// proof or why it was refused, and its stamps. +fn regenerated( + elf: &[u8], + format: &BlockFormat, + mode: block_whir::regen::RegenMode, + deviations: Deviations, +) -> Result<(BlockWhirProof, block_whir::BlockStamps), crate::Error> { + regenerated_at(elf, format, mode, deviations, MaxRowsConfig::small()) +} + +/// [`regenerated`] with chunks of `max_rows`. +fn regenerated_at( + elf: &[u8], + format: &BlockFormat, + mode: block_whir::regen::RegenMode, + deviations: Deviations, + max_rows: MaxRowsConfig, +) -> Result<(BlockWhirProof, block_whir::BlockStamps), crate::Error> { + regenerated_with(elf, format, mode, deviations, max_rows, |_| {}) +} + +/// [`regenerated_at`] with the options changed by `tune` before the prove. +fn regenerated_with( + elf: &[u8], + format: &BlockFormat, + mode: block_whir::regen::RegenMode, + deviations: Deviations, + max_rows: MaxRowsConfig, + tune: impl FnOnce(&mut BlockOptions), +) -> Result<(BlockWhirProof, block_whir::BlockStamps), crate::Error> { + let mut o = streamed(max_rows, 5, 3); + o.layout_workers = 3; + o.drop_streamed_ops = true; + o.regen = Some(mode); + tune(&mut o); + // KECCAK_RND built on the host unless a test asks otherwise: its tables + // are then droppable, and these tests rebuild them with LT's. CPU and + // MEMW_R likewise (a card build's default fills them from their records): + // these tests drop and rebuild them as packed chunks, and the records + // tests ask for theirs. + let deviations = Deviations { + card_fill: deviations.card_fill.or(Some(false)), + card_fill_streamed: deviations.card_fill_streamed.or(Some([false, false])), + ..deviations + }; + prove_block_whir_with( + elf, + &[], + &ProofOptions::default_test_options(), + format, + &o, + &deviations, + ) +} + +/// How long a hang guard waits for a regenerating prove: ten times the plain +/// prove the test timed before it, and at least ten minutes. A hang never +/// returns, so any bound finds one; a bound scaled to the plain prove also +/// holds where every prove is slower (a host without a card, a costlier base +/// hash), where a fixed one fails a prove that completes: one-byte windows +/// ran past 600 s on the laptop under the RPX base, phase B never waiting. +fn hang_bound(plain: std::time::Duration) -> std::time::Duration { + std::time::Duration::from_secs(600).max(plain.saturating_mul(10)) +} + +/// The hang guard's bound: ten minutes under a fast plain prove, ten plain +/// proves past one minute. +#[test] +fn a_hang_bound_is_ten_plain_proves_and_at_least_ten_minutes() { + use std::time::Duration; + assert_eq!(hang_bound(Duration::ZERO), Duration::from_secs(600)); + assert_eq!( + hang_bound(Duration::from_secs(60)), + Duration::from_secs(600) + ); + assert_eq!( + hang_bound(Duration::from_secs(61)), + Duration::from_secs(610) + ); + assert_eq!( + hang_bound(Duration::from_secs(550)), + Duration::from_secs(5500) + ); + assert_eq!(hang_bound(Duration::MAX), Duration::MAX); +} + +/// ★ Live regeneration moves no byte (D-WHIR-NODISK N2): with `always` every +/// streamed chunk is dropped after its group's commit and rebuilt in phase B — +/// with each group's openings beside the next group's argue too (phase B's +/// overlap, and pinned) — and `auto` armed from the start (a target of 0) +/// drops them all too; the proof is the one regeneration off proves, with the +/// same partition, under the deterministic grind, and verifies. `auto` that +/// never arms (a target out of reach) drops nothing and runs no regenerator. +#[test] +fn live_regeneration_proves_the_same_bytes() { + use crate::block_whir::regen::RegenMode; + use stark::multilinear_block::PhaseBOverlap::{Off, On, Pinned}; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let (plain, plain_stamps) = + regenerated(&elf, &format, RegenMode::Off, Deviations::default()).expect("prove"); + assert!(plain_stamps.regen.is_none()); + assert!(plain_stamps.drops.is_none(), "{:?}", plain_stamps.drops); + let streamed = plain_stamps.streamed.1; + assert!(streamed > 0); + for (what, mode, target, overlap) in [ + ("always", RegenMode::Always, None, Off), + ("always, overlapped", RegenMode::Always, None, On), + ("always, pinned", RegenMode::Always, None, Pinned), + ("auto armed", RegenMode::Auto, Some(0), Off), + ] { + let (proof, stamps) = regenerated_with( + &elf, + &format, + mode, + Deviations { + memory_target: target, + ..Deviations::default() + }, + MaxRowsConfig::small(), + |o| o.phase_b_overlap = overlap, + ) + .unwrap_or_else(|e| panic!("{what}: {e:?}")); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + // Every streamed chunk, and KECCAK_RND's and LT's tables of the rest + // (N3, one class a group), dropped and back. + assert!(r.rest_dropped > 0, "{what}: no rest table dropped\n{lines}"); + assert_eq!( + r.dropped, + streamed + r.rest_dropped, + "{what}: every chunk dropped\n{lines}" + ); + assert_eq!( + r.regenerated, r.dropped, + "{what}: every table back\n{lines}" + ); + assert_eq!( + r.rest_regenerated, r.rest_dropped, + "{what}: the rest back\n{lines}" + ); + assert_eq!((r.mismatches, r.failed), (0, 0), "{what}\n{lines}"); + assert_eq!(r.error, None, "{what}\n{lines}"); + assert!(lines.contains("groups rebuilt ("), "{lines}"); + assert!(lines.contains("BLOCK REGEN rest: "), "{lines}"); + assert!( + lines.contains("BLOCK REGEN phase B order taken: "), + "{lines}" + ); + // The interleaved order is not group order: the bytes below hold only + // because each group proves on its own fork (`S_post ‖ g`). + assert_ne!( + r.phase_b_order, + (0..proof.groups.len()).collect::>(), + "{what}: group order" + ); + assert!( + stamps + .drops + .as_deref() + .is_some_and(|l| l.starts_with("live regeneration decides as `auto`")), + "{what}: {:?}", + stamps.drops + ); + assert_eq!(proof.groups, plain.groups, "{what}: the partition"); + assert!(verify(&proof, &elf, &format), "{what}"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&proof), bytes(&plain), "{what}: the proof"); + } + } + // The walk's end hands off (and arms) once the peak passes the pressure + // share of the host's target, so the target is out of reach and no + // host's memory decides it. + let (proof, stamps) = regenerated( + &elf, + &format, + RegenMode::Auto, + Deviations { + memory_target: Some(u64::MAX), + ..Deviations::default() + }, + ) + .expect("prove"); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert_eq!(r.dropped, 0, "{lines}"); + assert!( + lines.contains("never armed") && !lines.contains("BLOCK REGEN live:"), + "{lines}" + ); + assert!(lines.contains("0 of"), "{lines}"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&proof), bytes(&plain), "auto unarmed: the proof"); + } +} + +/// Without live regeneration nothing leaves the host, even where the `auto` +/// rule would want every table out (a target of 0): there is no rule to arm, +/// no drops line, and the proof is the same bytes. +#[test] +fn without_live_regeneration_every_table_is_held() { + use crate::block_whir::regen::RegenMode; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let (plain, _) = + regenerated(&elf, &format, RegenMode::Off, Deviations::default()).expect("prove"); + let (held, stamps) = regenerated( + &elf, + &format, + RegenMode::Off, + Deviations { + memory_target: Some(0), + ..Deviations::default() + }, + ) + .expect("prove"); + assert!(stamps.regen.is_none()); + assert!(stamps.drops.is_none(), "{:?}", stamps.drops); + assert!( + !stamps.report().contains("BLOCK DROPS"), + "{}", + stamps.report() + ); + assert_eq!(held.groups, plain.groups, "the partition"); + assert!(verify(&held, &elf, &format)); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&held), bytes(&plain), "the proof"); + } +} + +/// ★ The hand-off at the walk's end arms live regeneration and moves no byte: +/// with the target out of reach the `auto` rule never arms on its own, and the +/// hand-off (here asked for everything, whatever the forecast) arms it before +/// the finish: every chunk parked by then is dropped and rebuilt in phase B, +/// and the proof verifies with the same partition, its bytes equal to +/// regeneration off's under the deterministic grind. Without the deviation the +/// forecast stays under the line, and nothing drops. How many chunks phase A +/// has parked when the walk ends depends on scheduling (none, under load), so +/// the drops are required only when some were parked at arming. +#[test] +fn a_hand_off_at_the_walk_arms_live_regeneration_and_moves_no_byte() { + use crate::block_whir::regen::RegenMode; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let proved = |mode: RegenMode, hand_off_all: bool| { + let (proof, stamps) = regenerated( + &elf, + &format, + mode, + Deviations { + hand_off_all, + memory_target: Some(u64::MAX), + ..Deviations::default() + }, + ) + .unwrap_or_else(|e| panic!("{mode:?} hand-off {hand_off_all}: {e:?}")); + assert!( + verify(&proof, &elf, &format), + "{mode:?} hand-off {hand_off_all}" + ); + (proof, stamps) + }; + let (plain, _) = proved(RegenMode::Off, false); + let (unarmed, stamps) = proved(RegenMode::Auto, false); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert_eq!(r.dropped, 0, "{lines}"); + assert!(lines.contains("never armed"), "{lines}"); + let (handed, stamps) = proved(RegenMode::Auto, true); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert!( + lines.contains("armed at"), + "the hand-off armed nothing\n{lines}" + ); + if r.armed_kept > 0 { + assert!(r.dropped > 0, "the hand-off dropped nothing\n{lines}"); + } + assert_eq!(r.regenerated, r.dropped, "every table back\n{lines}"); + assert_eq!((r.mismatches, r.failed), (0, 0), "{lines}"); + for (what, proof) in [("unarmed", &unarmed), ("handed off", &handed)] { + assert_eq!(proof.groups, plain.groups, "{what}: the partition"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(proof), bytes(&plain), "{what}: the proof"); + } + } +} + +/// ★ The streamed chunks' regenerator starts before phase B, from each +/// chunk's fate as phase A decided it: its thread up where the streamed chunks +/// end and started once their fates are known (the default), or started at +/// the finish's end; the rest's starts with phase B. Every dropped table is +/// rebuilt and deposited, and the proof is regeneration off's bytes; started +/// with phase B instead (the knob off), the same. (A fate decided after the +/// regenerator asks is `stark::multilinear_block`'s +/// `a_regenerator_started_early_waits_for_each_tables_fate`.) +#[test] +fn regenerators_started_at_the_finishs_end_prove_the_same_bytes() { + use crate::block_whir::regen::{EarlySpawn, RegenMode}; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let (plain, _) = + regenerated(&elf, &format, RegenMode::Off, Deviations::default()).expect("prove"); + for early in [EarlySpawn::Fates, EarlySpawn::Finish, EarlySpawn::Off] { + let (proof, stamps) = regenerated( + &elf, + &format, + RegenMode::Always, + Deviations { + regen_early: Some(early), + ..Deviations::default() + }, + ) + .unwrap_or_else(|e| panic!("early {early:?}: {e:?}")); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert_eq!(r.head_start.is_some(), early != EarlySpawn::Off, "{lines}"); + assert!(r.rest_dropped > 0 && r.dropped > r.rest_dropped, "{lines}"); + assert_eq!(r.regenerated, r.dropped, "{lines}"); + assert_eq!((r.mismatches, r.failed), (0, 0), "{lines}"); + assert_eq!(proof.groups, plain.groups, "early {early:?}: the partition"); + assert!(verify(&proof, &elf, &format), "early {early:?}"); + if crypto::grinding::deterministic() { + assert!( + bytes(&proof) == bytes(&plain), + "early {early:?}: the proof differs" + ); + } + } +} + +/// ★ The regenerator's thread is up before the drops: with each streamed +/// chunk's drop held back, it asks for fates phase A has not decided yet, waits +/// for every one, and rebuilds exactly what was dropped; the proof is +/// regeneration off's bytes. A fate answered "kept" before it is decided would +/// leave ranks out, and phase A's end refuses that ("the early regenerator +/// took … streamed chunks, phase A dropped …"). +#[test] +fn a_regenerator_up_before_the_drops_proves_the_same_bytes() { + use crate::block_whir::regen::{EarlySpawn, RegenMode}; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let (plain, _) = + regenerated(&elf, &format, RegenMode::Off, Deviations::default()).expect("prove"); + let (proof, stamps) = regenerated( + &elf, + &format, + RegenMode::Always, + Deviations { + regen_early: Some(EarlySpawn::Fates), + streamed_drop_delay: Some(std::time::Duration::from_millis(100)), + ..Deviations::default() + }, + ) + .unwrap_or_else(|e| panic!("the drops held back: {e:?}")); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert!(r.head_start.is_some(), "{lines}"); + assert!(r.dropped > r.rest_dropped, "{lines}"); + assert_eq!(r.regenerated, r.dropped, "{lines}"); + assert_eq!((r.mismatches, r.failed), (0, 0), "{lines}"); + assert_eq!(proof.groups, plain.groups, "the partition"); + assert!(verify(&proof, &elf, &format), "the drops held back"); + if crypto::grinding::deterministic() { + assert!(bytes(&proof) == bytes(&plain), "the proof differs"); + } +} + +/// N4c's liveness: phase A ends in an error right after the finish, with the +/// streamed chunks' regenerator started and waiting on chunks not yet +/// decided. The prove returns the error (its window closed, its thread +/// joined) and never hangs. +#[test] +fn an_early_regenerator_never_hangs_a_failed_phase_a() { + use crate::block_whir::regen::{EarlySpawn, RegenMode}; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let out = regenerated( + &elf, + &format, + RegenMode::Always, + Deviations { + regen_early: Some(EarlySpawn::Fates), + streamed_drop_delay: Some(std::time::Duration::from_millis(400)), + abort_after_finish: true, + ..Deviations::default() + }, + ); + let _ = tx.send(out.map(|_| ()).map_err(|e| format!("{e:?}"))); + }); + let out = rx + .recv_timeout(std::time::Duration::from_secs(600)) + .expect("the failed prove returned"); + let why = out.expect_err("phase A failed"); + assert!( + why.contains("a test's abort right after the finish"), + "{why}" + ); +} + +/// ★ Live regeneration never proves over columns it did not rebuild, and never +/// hangs: a deposit one bit off is refused at its slot before any device work +/// ("not the ones dropped"); with the digest off, the kept tree top refuses it +/// in the opening; a walk that fails, a generator that dies outside its catch, +/// or a chunk the slicer never cuts leaves its table refused ("did not bring +/// its columns back"), the prove returning; and phase B told to take the +/// rebuilt groups out of rank order is refused before it starts. +#[test] +fn live_regeneration_refuses_what_it_did_not_rebuild() { + use crate::block_whir::regen::{LiveFaults, RegenMode}; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let reversed: fn(usize) -> Vec = |n| (0..n).rev().collect(); + type Order = Option Vec>; + let cases: [(&str, LiveFaults, Order, &str); 6] = [ + ( + "a bent deposit", + LiveFaults { + bend_rank: Some(1), + ..LiveFaults::default() + }, + None, + "not the ones dropped", + ), + ( + "a bent deposit, the digest off", + LiveFaults { + bend_rank: Some(1), + digest_off: true, + ..LiveFaults::default() + }, + None, + "RecomputedCodewordMismatch", + ), + ( + "a walk that fails", + LiveFaults { + walk_error_at: Some(0), + ..LiveFaults::default() + }, + None, + "did not bring its columns back", + ), + ( + "a generator that dies", + LiveFaults { + generator_panics_at_rank: Some(0), + ..LiveFaults::default() + }, + None, + "did not bring its columns back", + ), + ( + "a chunk never cut", + LiveFaults { + skip_rank: Some(0), + ..LiveFaults::default() + }, + None, + "did not bring its columns back", + ), + ( + "rebuilt groups out of order", + LiveFaults::default(), + Some(reversed), + "rebuilt groups in group order", + ), + ]; + for (what, faults, order, why) in cases { + let (tx, rx) = std::sync::mpsc::channel(); + let elf = elf.clone(); + std::thread::spawn(move || { + let out = regenerated( + &elf, + &format, + RegenMode::Always, + Deviations { + regen_faults: faults, + phase_b_order: order, + ..Deviations::default() + }, + ); + let _ = tx.send(out.map(|_| ()).map_err(|e| format!("{e:?}"))); + }); + let got = rx + .recv_timeout(std::time::Duration::from_secs(600)) + .unwrap_or_else(|_| panic!("{what}: the prove hung")); + match got { + Err(e) => assert!(e.contains(why), "{what}: refused for another reason: {e}"), + Ok(()) => panic!("{what}: proved"), + } + } +} + +/// ★ Windows of one byte (N3): each admits only its frontier, so a phase-B +/// order or a group that takes a class out of its rank order cannot complete. +/// With every streamed chunk and every KECCAK_RND and LT table dropped, the +/// interleaved order proves the same bytes, within a bound on the time +/// ([`hang_bound`]). +#[test] +fn one_byte_windows_take_each_class_in_its_order() { + use crate::block_whir::regen::RegenMode; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let started = std::time::Instant::now(); + let (plain, _) = + regenerated(&elf, &format, RegenMode::Off, Deviations::default()).expect("prove"); + let bound = hang_bound(started.elapsed()); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let out = regenerated( + &elf, + &format, + RegenMode::Always, + Deviations { + regen_windows: Some((1, 1)), + ..Deviations::default() + }, + ); + let _ = tx.send(out.map_err(|e| format!("{e:?}"))); + }); + let (proof, stamps) = rx + .recv_timeout(bound) + .expect("the prove did not hang") + .expect("prove"); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert!(r.rest_dropped > 0 && r.dropped > r.rest_dropped, "{lines}"); + assert_eq!(r.regenerated, r.dropped, "{lines}"); + assert_ne!( + r.phase_b_order, + (0..proof.groups.len()).collect::>(), + "the interleaved order is group order" + ); + assert_eq!(proof.groups, plain.groups); + if crypto::grinding::deterministic() { + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + assert!(bytes(&proof) == bytes(&plain), "the proof differs"); + } +} + +/// ★ One class a group (N3): with groups of 2^24-cell polynomials the block +/// is one group, holding every streamed chunk and every KECCAK_RND and LT table +/// of the rest. Under `always` it drops the streamed class only, the class of +/// its first droppable table, and holds the rest's tables in place (phase B +/// could not take one group in both classes' orders); the proof is the bytes +/// regeneration off proves, and verifies. A group that dropped both classes is +/// refused before phase B, never proved and never left to hang. +#[test] +fn a_mixed_boundary_group_drops_one_class_only() { + use crate::block_whir::regen::RegenMode; + let elf = asm_elf_bytes("test_keccak_multi"); + let mut format = many_groups(); + format.zf.whir_stack = StackVars::new(24).expect("a valid stack"); + let started = std::time::Instant::now(); + let (plain, plain_stamps) = + regenerated(&elf, &format, RegenMode::Off, Deviations::default()).expect("prove"); + let bound = hang_bound(started.elapsed()); + let streamed = plain_stamps.streamed.1; + let (tx, rx) = std::sync::mpsc::channel(); + { + let elf = elf.clone(); + std::thread::spawn(move || { + let out = regenerated(&elf, &format, RegenMode::Always, Deviations::default()); + let _ = tx.send(out.map_err(|e| format!("{e:?}"))); + }); + } + let (proof, stamps) = rx + .recv_timeout(bound) + .expect("the prove did not hang") + .expect("a mixed group is proved, one class dropped"); + let r = stamps.regen.expect("live regeneration's readout"); + let lines = r.lines.join("\n"); + assert_eq!(proof.groups.len(), 1, "one group: {:?}", proof.groups); + assert_eq!(r.dropped, streamed, "the streamed class dropped\n{lines}"); + assert_eq!(r.rest_dropped, 0, "no rest table dropped\n{lines}"); + assert!(r.held > 0, "the rest's tables held\n{lines}"); + assert_eq!(r.regenerated, r.dropped, "{lines}"); + assert!(lines.contains(&format!("{} held", r.held)), "{lines}"); + assert_eq!(proof.groups, plain.groups); + assert!(verify(&proof, &elf, &format)); + if crypto::grinding::deterministic() { + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + assert!(bytes(&proof) == bytes(&plain), "the proof differs"); + } +} + +/// ★ The rest's regenerator refuses what it did not build again (N3): a +/// deposit of other columns, a generator that dies, a table never built — each +/// at the rest's first dropped table — refuse the prove before any of that +/// table's device work, never prove over it, and never hang. +#[test] +fn the_rest_regenerator_refuses_what_it_did_not_rebuild() { + use crate::block_whir::regen::{LiveFaults, RegenMode}; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let cases: [(&str, LiveFaults, &str); 3] = [ + ( + "a bent rest deposit", + LiveFaults { + bend_rank: Some(0), + ..LiveFaults::default() + }, + "not the ones dropped", + ), + ( + "a rest generator that dies", + LiveFaults { + generator_panics_at_rank: Some(0), + ..LiveFaults::default() + }, + "did not bring its columns back", + ), + ( + "a rest table never built", + LiveFaults { + skip_rank: Some(0), + ..LiveFaults::default() + }, + "did not bring its columns back", + ), + ]; + for (what, faults, why) in cases { + let (tx, rx) = std::sync::mpsc::channel(); + let elf = elf.clone(); + std::thread::spawn(move || { + let out = regenerated( + &elf, + &format, + RegenMode::Always, + Deviations { + rest_faults: faults, + ..Deviations::default() + }, + ); + let _ = tx.send(out.map(|_| ()).map_err(|e| format!("{e:?}"))); + }); + let got = rx + .recv_timeout(std::time::Duration::from_secs(600)) + .unwrap_or_else(|_| panic!("{what}: the prove hung")); + match got { + Err(e) => assert!(e.contains(why), "{what}: refused for another reason: {e}"), + Ok(()) => panic!("{what}: proved"), + } + } +} + +/// The streamed chunks laid out on three threads are packed in the inline +/// order all the same: the same groups and table counts as laid out on one +/// thread, and the proof verifies. +#[test] +fn a_streamed_block_laid_out_on_three_threads_packs_the_same_groups() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let inline = prove(&elf, &format, &streamed(MaxRowsConfig::small(), 16, 3)); + let mut o = streamed(MaxRowsConfig::small(), 16, 3); + o.layout_workers = 3; + let proof = prove(&elf, &format, &o); + assert!(groups_of(&proof) >= 3, "{} groups", groups_of(&proof)); + assert_eq!(proof.groups, inline.groups); + assert_eq!( + format!("{:?}", proof.table_counts), + format!("{:?}", inline.table_counts) + ); + assert!(verify(&proof, &elf, &format)); +} + +/// ★ With BITWISE handed on last ([`Deviations::rest_bitwise_last`], as +/// `LAMBDA_VM_BLOCK_REST_BITWISE_LAST=1`), the streamed rest is packed in AIR +/// order but BITWISE (AIR index 0), which comes after every other table: the +/// partition moves, the tables and their counts do not, and the proof verifies +/// under the unchanged verifier. Without it BITWISE leads the rest, as today. +#[test] +fn bitwise_handed_on_last_moves_the_partition_and_verifies() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = 3; + assert!(o.rest_streams(), "the rest streams"); + let proved = |bitwise_last: bool| { + let proof = prove_with( + &elf, + &format, + &o, + &Deviations { + rest_bitwise_last: bitwise_last, + bitwise_from_tables: Some(false), + ..Deviations::default() + }, + ); + assert!(verify(&proof, &elf, &format), "bitwise_last {bitwise_last}"); + proof + }; + let air = proved(false); + let last = proved(true); + assert_eq!( + format!("{:?}", last.table_counts), + format!("{:?}", air.table_counts) + ); + let flat = |p: &BlockWhirProof| p.groups.iter().flatten().copied().collect::>(); + let (air_flat, last_flat) = (flat(&air), flat(&last)); + let sorted = |mut v: Vec| { + v.sort_unstable(); + v + }; + assert_eq!( + sorted(air_flat.clone()), + sorted(last_flat.clone()), + "the same tables" + ); + assert_eq!(last_flat.last(), Some(&0), "BITWISE packed last"); + assert_ne!( + air_flat.last(), + Some(&0), + "BITWISE packed last without the knob" + ); + assert_ne!(last.groups, air.groups, "the partition did not move"); +} + +/// ★ BITWISE's lookups from LT's, BRANCH's and BYTEWISE's tables +/// ([`Deviations::bitwise_from_tables`], the block's default), counted from +/// each table's rows as it is built, commit what BITWISE handed on last +/// commits with phase 4's count from the ops: the same partition and the same +/// group roots, and the proof verifies. The default is that; the rollback +/// (`LAMBDA_VM_BLOCK_BITWISE_FROM_TABLES=0`) packs BITWISE first, as before. +/// The finish keeps only the streamed chunks' tails, as the block does. +#[test] +fn bitwise_from_tables_commits_bitwise_lasts_tables_and_verifies() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = 3; + o.drop_streamed_ops = true; + assert!(o.rest_streams(), "the rest streams"); + let proved = |what: &str, deviations: Deviations| { + let proof = prove_with(&elf, &format, &o, &deviations); + assert!(verify(&proof, &elf, &format), "{what}"); + proof + }; + let from_ops = proved( + "from the ops", + Deviations { + rest_bitwise_last: true, + bitwise_from_tables: Some(false), + ..Deviations::default() + }, + ); + let from_tables = proved( + "from the tables", + Deviations { + bitwise_from_tables: Some(true), + ..Deviations::default() + }, + ); + assert_eq!(from_tables.groups, from_ops.groups, "the partition"); + assert_eq!( + from_tables.proof.roots, from_ops.proof.roots, + "the committed tables" + ); + let default = proved("the default", Deviations::default()); + assert_eq!( + default.groups, from_tables.groups, + "the default's partition" + ); + assert_eq!( + default.proof.roots, from_tables.proof.roots, + "the default's tables" + ); + let rollback = proved( + "the rollback", + Deviations { + bitwise_from_tables: Some(false), + ..Deviations::default() + }, + ); + let last = |p: &BlockWhirProof| p.groups.iter().flatten().last().copied(); + assert_eq!(last(&from_tables), Some(0), "BITWISE packed last"); + assert_ne!(last(&rollback), Some(0), "the rollback packs BITWISE first"); + assert_ne!( + rollback.groups, from_tables.groups, + "the rollback's partition" + ); +} + +/// The bounded layout (D-EXEC E3): with three workers and `layout_ahead = +/// Some(k)`, at most `k + 1` streamed chunks are laid out (or in the making) +/// and not yet packed at once, and the groups are the inline layout's. +#[test] +fn the_bounded_layout_holds_at_most_k_plus_one_chunks_unpacked() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let inline = prove(&elf, &format, &streamed(MaxRowsConfig::small(), 16, 3)); + for k in [0, 2] { + let mut o = streamed(MaxRowsConfig::small(), 16, 3); + o.layout_workers = 3; + o.layout_ahead = Some(k); + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations::default(), + ) + .expect("prove"); + let most = stamps.layout.ahead_most; + assert!( + (1..=k + 1).contains(&most), + "k {k}: {most} chunks unpacked at once" + ); + assert_eq!(proof.groups, inline.groups, "k {k}"); + assert!(verify(&proof, &elf, &format)); + } +} + +/// Closed groups waiting for phase A behind one another move no byte of the +/// proof: a no-disk block proved with no bytes for them to wait in, one byte, +/// every byte, and under the room rule packs the groups of the channel of one +/// and, under the deterministic grind, proves its bytes; with every byte, +/// groups do wait behind one another, and with none they never do. A block +/// without live regeneration keeps the channel of one whatever the knob says. +#[test] +fn a_deeper_group_queue_moves_no_byte_of_the_proof() { + use crate::block_whir::group_queue::Setting; + use crate::block_whir::regen::RegenMode; + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + // Phase A takes a group every ten milliseconds, so the packer is ahead of + // it in every arm. + let queued = |setting: Setting, mode: RegenMode| { + let (proof, stamps) = regenerated( + &elf, + &format, + mode, + Deviations { + group_queue: Some(setting), + group_take_pause: Some(std::time::Duration::from_millis(10)), + ..Deviations::default() + }, + ) + .unwrap_or_else(|e| panic!("{setting:?}, {mode:?}: {e:?}")); + let line = stamps.group_queue.expect("the queue's line"); + (proof, line) + }; + let (plain, line) = queued(Setting::None, RegenMode::Always); + assert!(groups_of(&plain) >= 3, "{} groups", groups_of(&plain)); + assert!(verify(&plain, &elf, &format)); + assert_eq!( + line, + "BLOCK GROUP QUEUE: none · one closed group waits (the default)" + ); + let same = |what: &str, proof: &BlockWhirProof| { + assert_eq!(proof.groups, plain.groups, "{what}: the partition"); + assert_eq!( + format!("{:?}", proof.table_counts), + format!("{:?}", plain.table_counts), + "{what}" + ); + assert!(verify(proof, &elf, &format), "{what}"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(proof), bytes(&plain), "{what}: the proof"); + } + }; + for (what, setting) in [ + ("no byte", Setting::Bytes(0)), + ("one byte", Setting::Bytes(1)), + ("every byte", Setting::Bytes(u64::MAX)), + ] { + let (proof, line) = queued(setting, RegenMode::Always); + same(what, &proof); + assert!(line.contains("fixed, the host not read"), "{what}: {line}"); + let waited = !line.contains("none waited behind another"); + assert_eq!( + waited, + setting == Setting::Bytes(u64::MAX), + "{what}: {line}" + ); + } + let (proof, line) = queued(Setting::Room, RegenMode::Always); + same("room", &proof); + assert!( + line.contains("GiB waiting while VmRSS + the group + 16.00 stays under the line"), + "{line}" + ); + let (proof, line) = queued(Setting::Bytes(u64::MAX), RegenMode::Off); + same("regeneration off", &proof); + assert!( + line.contains("the block proves without live regeneration: one closed group waits"), + "{line}" + ); +} + +/// The partition is part of the statement: a table named twice, two tables +/// swapped inside a group, or a table moved to another group is refused. +#[test] +fn a_tampered_partition_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let proof = prove(&elf, &format, &streamed(MaxRowsConfig::small(), 16, 3)); + assert!(verify(&proof, &elf, &format)); + + let mut twice = proof.clone(); + let first = twice.groups[0][0]; + twice.groups[1][0] = first; + assert!(!verify(&twice, &elf, &format)); + + let mut swapped = proof.clone(); + let g = swapped + .groups + .iter() + .position(|g| g.len() >= 2) + .expect("a group of two"); + swapped.groups[g].swap(0, 1); + assert!(!verify(&swapped, &elf, &format)); + + let mut moved = proof.clone(); + let table = moved.groups[0].pop().expect("a table"); + moved.groups[1].insert(0, table); + assert!(!verify(&moved, &elf, &format)); +} + +/// A table missing from the partition is refused. +#[test] +fn a_partition_missing_a_table_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let mut proof = prove(&elf, &format, &streamed(MaxRowsConfig::small(), 16, 3)); + let g = proof + .groups + .iter() + .position(|g| g.len() >= 2) + .expect("a group of two"); + proof.groups[g].pop(); + assert!(!verify(&proof, &elf, &format)); +} + +/// ★ A group over the verifier's polynomial budget is refused — by that check +/// alone: the same honest proof verifies under a budget that admits it. +#[test] +fn a_group_over_the_stack_budget_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let mut wide = many_groups(); + wide.group_polys = 4; + let proof = prove(&elf, &wide, &streamed(MaxRowsConfig::small(), 16, 3)); + assert!( + verify(&proof, &elf, &wide), + "the honest proof under its own budget" + ); + let narrow = BlockFormat { + group_polys: 2, + ..wide + }; + assert!(!verify(&proof, &elf, &narrow)); +} + +/// ★ More groups than the verifier's maximum is refused — by that check alone: +/// the same honest proof verifies under a maximum that admits it. +#[test] +fn a_partition_over_the_group_maximum_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let proof = prove(&elf, &format, &streamed(MaxRowsConfig::small(), 16, 3)); + let n = proof.groups.len(); + assert!(n >= 3 && verify(&proof, &elf, &format)); + let tight = BlockFormat { + max_groups: n - 1, + ..format + }; + assert!(!verify(&proof, &elf, &tight)); + let exact = BlockFormat { + max_groups: n, + ..format + }; + assert!(verify(&proof, &elf, &exact)); +} + +/// ★ The prover refuses a partition over the group maximum instead of +/// proving a block the verifier refuses: the whole-run path with the +/// verifier's own error, the streamed one as its groups close, saying how far +/// the block got (the group that went over, not a total). +#[test] +fn the_prover_refuses_a_partition_over_the_group_maximum() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + for (opts, streaming) in [ + (streamed(MaxRowsConfig::small(), 16, 3), true), + (options(MaxRowsConfig::small(), 16), false), + ] { + let proof = prove(&elf, &format, &opts); + let n = proof.groups.len(); + assert!(n >= 2, "{n} groups"); + let tight = BlockFormat { + max_groups: n - 1, + ..format + }; + let refused = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &tight, + &opts, + &Deviations::default(), + ); + let Err(crate::Error::InvalidTableCounts(prover)) = refused else { + panic!("the prover proved {n} groups over a maximum of {}", n - 1); + }; + let Err(crate::Error::InvalidTableCounts(verifier)) = + verify_block_whir(&proof, &elf, &ProofOptions::default_test_options(), &tight) + else { + panic!("the verifier did not refuse {n} groups"); + }; + if streaming { + assert_eq!( + prover, + format!( + "a block takes at most {} groups; this one reached {n} as its groups close", + n - 1 + ) + ); + } else { + assert_eq!(prover, verifier); + } + } +} + +/// ★ Block-wide LogUp at the group maximum, its worst case over the statements +/// the verifier accepts: the bus balances once a block, so its support counts +/// every group, and a group's most fractions are a table alone at +/// 2^[`block_whir::BLOCK_MAX_TABLE_VARS`] rows times the most bus interactions +/// any block AIR carries a row (BLAKE3's 1475). The error is +/// (len + 1)(|supp| − 1)/|F| with len the longest bus message (208: KECCAK's +/// sponge link between two rows of one Keccak-256 call, 207 values and the bus +/// id; 204 before it, the round chip's state messages) and |F| = p³, a +/// committed word decoding to one codeword: 136.77 bits at 1024 groups, 140.77 +/// at 64 (136.79 and 140.79 at len 204). The census reads both constants from the block's +/// AIRs, so a wider AIR or a larger maximum fails here before it moves the +/// figure. +#[test] +fn block_wide_logup_at_the_group_maximum_is_pinned() { + let elf = asm_elf_bytes("all_instructions_64"); + let program = executor::elf::Elf::load(&elf).expect("the ELF loads"); + let opts = ProofOptions::default_test_options(); + let counts = crate::TableCounts { + cpu: 1, + lt: 1, + memw: 1, + memw_aligned: 1, + load: 1, + mul: 1, + dvrm: 1, + shift: 1, + branch: 1, + memw_register: 1, + eq: 1, + bytewise: 1, + store: 1, + cpu32: 1, + keccak: 1, + keccak_rnd: 1, + ecsm: 1, + ecdas: 1, + hint: 1, + copy: 1, + blake3: 1, + is_wh: 1, + ecrec: 1, + }; + let pages = crate::tables::trace_builder::Traces::page_configs_from_elf_and_runtime( + &program, + &[], + 0, + 1 << 20, + ) + .expect("pages"); + let inputs = + crate::BlockAirInputs::new(&opts, &program, vec![1 << 10], &[]).expect("the AIR inputs"); + let airs = crate::block_airs(&program, &opts, &pages, &counts, &inputs); + let (fractions, len) = airs + .air_refs() + .iter() + .map(|air| { + let bus = air.bus_interactions(); + let len = bus.iter().map(|b| b.num_bus_elements()).max().unwrap_or(0); + (bus.len(), len) + }) + .fold((0, 0), |(f, l), (f2, l2)| (f.max(f2), l.max(l2))); + assert_eq!( + (fractions, len), + (1475, 208), + "the census: fractions a row, message length" + ); + let bits = |groups: usize| { + let support = + groups as f64 * 2f64.powi(block_whir::BLOCK_MAX_TABLE_VARS as i32) * fractions as f64; + let field_log2 = 3.0 * (0xFFFF_FFFF_0000_0001u64 as f64).log2(); + field_log2 - ((len as f64 + 1.0) * (support - 1.0)).log2() + }; + assert_eq!( + format!("{:.2}", bits(block_whir::BLOCK_MAX_GROUPS)), + "136.77" + ); + assert_eq!(format!("{:.2}", bits(64)), "140.77"); +} + +/// ★ The group maximum's edge on the prover's plan: tables that fill a group each, at the +/// production stack and polynomial budget, plan into 1024 groups, which are +/// taken, and into 1025, which are refused with the verifier's error. +#[test] +fn the_plan_takes_the_group_maximum_and_refuses_one_more() { + let stack = ZfFormat::DEFAULT.whir_stack; + // Three columns at 2^27 rows: three polynomials, a group to itself. + let full = (block_whir::BLOCK_GROUP_POLYS, stack.get()); + for groups in [ + block_whir::BLOCK_MAX_GROUPS, + block_whir::BLOCK_MAX_GROUPS + 1, + ] { + let sizes = block_groups(&vec![full; groups], stack, block_whir::BLOCK_GROUP_POLYS) + .expect("a plan"); + assert_eq!(sizes.len(), groups); + let planned = block_whir::check_group_count(sizes.len(), block_whir::BLOCK_MAX_GROUPS); + if groups > block_whir::BLOCK_MAX_GROUPS { + let Err(crate::Error::InvalidTableCounts(why)) = planned else { + panic!("the plan took {groups} groups"); + }; + assert_eq!(why, "1025 groups — a block takes at most 1024"); + } else { + assert!(planned.is_ok(), "the plan refused {groups} groups"); + } + } +} + +/// ★ The group maximum's edge at the verifier: a statement of 1025 groups is refused by the +/// group maximum, before its partition is read; at 1024 the maximum lets the +/// statement through, and the partition check refuses the forged groups. +#[test] +fn the_verifier_refuses_one_group_over_the_maximum() { + let elf = asm_elf_bytes("sub"); + let format = one_group(); + let mut proof = prove(&elf, &format, &options(MaxRowsConfig::default(), 16)); + assert!(verify(&proof, &elf, &format)); + for groups in [ + block_whir::BLOCK_MAX_GROUPS, + block_whir::BLOCK_MAX_GROUPS + 1, + ] { + proof.groups = vec![vec![0]; groups]; + let Err(crate::Error::InvalidTableCounts(why)) = + verify_block_whir(&proof, &elf, &ProofOptions::default_test_options(), &format) + else { + panic!("the verifier did not refuse {groups} groups"); + }; + if groups > block_whir::BLOCK_MAX_GROUPS { + assert_eq!(why, "1025 groups — a block takes at most 1024"); + } else { + assert_eq!(why, "table 0 is in two groups"); + } + } +} + +/// One table's claimed column value moved: its group's opening refuses it. +#[test] +fn a_tampered_column_claim_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let mut proof = prove(&elf, &format, &options(MaxRowsConfig::small(), 16)); + let last = proof.proof.tables.len() - 1; + proof.proof.tables[last].constraint.reduce.column_values[0] += FieldElement::::one(); + assert!(!verify(&proof, &elf, &format)); +} + +/// One table's bus output moved: its GKR no longer lands on its trace. +#[test] +fn a_tampered_bus_output_is_refused() { + let elf = asm_elf_bytes("sub"); + let format = many_groups(); + let mut proof = prove(&elf, &format, &options(MaxRowsConfig::small(), 16)); + proof.proof.tables[0].bus_output.0 += FieldElement::::one(); + assert!(!verify(&proof, &elf, &format)); +} + +#[test] +fn a_tampered_public_output_is_refused() { + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf, &one_group(), &options(MaxRowsConfig::default(), 16)); + proof.public_output.push(0xff); + assert!(!verify(&proof, &elf, &one_group())); +} + +#[test] +fn a_restated_table_height_is_refused() { + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf, &one_group(), &options(MaxRowsConfig::default(), 16)); + proof.table_num_vars[0] += 1; + assert!(!verify(&proof, &elf, &one_group())); +} + +/// ⛔ A CPU table's stated height is what its timestamp base and the final PC +/// timestamp are built from: restated (and so absorbed again by the verifier, +/// the statement re-bound), within every bound, a full CPU table halved or the +/// last one doubled, the proof is refused; past the block's 2^21 cap it is +/// refused before anything is built. +#[test] +fn a_restated_cpu_table_height_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let honest = prove(&elf, &format, &options(MaxRowsConfig::small(), 16)); + assert!(verify(&honest, &elf, &format)); + let start = crate::cpu_proof_start(true, &honest.table_counts); + let cpus = honest.table_counts.cpu; + assert!(cpus >= 2, "{cpus} CPU tables"); + let last = start + cpus - 1; + for (what, at, vars) in [ + ( + "a full table halved", + start, + honest.table_num_vars[start] - 1, + ), + ( + "the last table doubled", + last, + honest.table_num_vars[last] + 1, + ), + ("a table past the block's cap", start, 22), + ] { + let mut restated = honest.clone(); + restated.table_num_vars[at] = vars; + assert!(!verify(&restated, &elf, &format), "{what}: accepted"); + } +} + +/// A root the layouts do not claim is refused before anything is drawn. +#[test] +fn an_extra_root_is_refused() { + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf, &one_group(), &options(MaxRowsConfig::default(), 16)); + let root = proof.proof.roots[0]; + proof.proof.roots.push(root); + assert!(!verify(&proof, &elf, &one_group())); +} + +/// DECODE's columns are settled by a prepared opening: one per prepared table, +/// and a small program's only prepared table is DECODE. +#[test] +fn the_block_carries_decodes_prepared_opening() { + let elf = asm_elf_bytes("sub"); + let proof = prove(&elf, &one_group(), &options(MaxRowsConfig::default(), 16)); + assert_eq!(proof.prepared.len(), 1); + assert!(verify(&proof, &elf, &one_group())); +} + +/// ★ DECODE's index is found by NAME, and "exactly one" is the assertion. +/// +/// The opening binds the pinned columns to one table's reduced point. A prover +/// who could aim them at a different table's point would be settling them +/// against challenges they were never bound to, so the index is never inferred +/// from a position — and a table set with two DECODEs, or none, is a layout +/// nobody meant to build. +#[test] +fn decode_is_found_by_name_and_only_once() { + use crate::block_whir::decode_table_index; + use crate::test_utils::F; + + let opts = stark::proof::options::ProofOptions::default_test_options(); + let decode = crate::test_utils::create_decode_air(&opts); + let eq = crate::test_utils::create_eq_air(&opts); + + type Air<'a> = &'a dyn stark::traits::AIR; + + let one: Vec> = vec![&eq, &decode, &eq]; + assert_eq!(decode_table_index(&one).expect("one DECODE"), 1); + + let none: Vec> = vec![&eq, &eq]; + assert!( + decode_table_index(&none).is_err(), + "a table set with no DECODE was accepted" + ); + + let two: Vec> = vec![&decode, &eq, &decode]; + assert!( + decode_table_index(&two).is_err(), + "a table set with two DECODEs was accepted, so the index it returns is \ + a choice rather than a fact" + ); +} + +/// The prepared set is DECODE and every genesis page the cross-epoch rule +/// stacks — a page with thousands of nonzero genesis bytes — and never a zero +/// page or a private one, whose columns cost the guest nothing or are committed. +#[test] +fn the_prepared_set_is_decode_and_the_dense_pages() { + use crate::tables::page::PageConfig; + let elf = asm_elf_bytes("sub"); + let counts = prove(&elf, &one_group(), &options(MaxRowsConfig::default(), 16)).table_counts; + let program = executor::elf::Elf::load(&elf).expect("the ELF loads"); + let configs = vec![ + PageConfig::zero_init(0x1000_0000), + PageConfig { + page_base: 0x2000_0000, + init_values: Some(vec![7u8; 20_000]), + is_private_input: false, + }, + PageConfig { + page_base: 0x3000_0000, + init_values: Some(vec![7u8; 20_000]), + is_private_input: true, + }, + ]; + let opts = ProofOptions::default_test_options(); + let airs = crate::VmAirs::new( + &program, + &opts, + false, + &configs, + &counts, + &vec![4; counts.cpu], + None, + true, + None, + crate::test_utils::halted_register_commitment(&opts, &program, &[]).preprocessed(), + ); + let prepared = + block_whir::prepared_tables(&airs, &configs, &one_group()).expect("the prepared set"); + let refs = airs.air_refs(); + let decode = refs + .iter() + .position(|air| air.name() == "DECODE") + .expect("a DECODE"); + let first_page = refs.len() - airs.pages.len() - { + // The tables after the pages, in `air_refs` order. + airs.memw_registers.len() + + airs.eqs.len() + + airs.bytewises.len() + + airs.stores.len() + + airs.cpu32s.len() + }; + let tables: Vec = prepared.iter().map(|(t, _)| *t).collect(); + assert_eq!(tables, vec![decode, first_page + 1]); + assert_eq!( + prepared[1].1, + crate::tables::page::preprocessed_columns(&configs[1]) + ); +} + +/// A prepared opening altered in one value is refused. The openings sit last on +/// their group's fork, so nothing after them reads the tamper: only the +/// opening's own check can refuse it. +#[test] +fn a_tampered_prepared_opening_is_refused() { + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let mut proof = prove(&elf, &format, &options(MaxRowsConfig::small(), 16)); + assert!(verify(&proof, &elf, &format)); + proof.prepared[0].polys[0].final_value += FieldElement::::one(); + assert!(!verify(&proof, &elf, &format)); +} + +/// A proof that leaves a prepared opening out is refused: the verifier derives +/// the prepared set from the program, not from the proof. +#[test] +fn a_missing_prepared_opening_is_refused() { + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf, &one_group(), &options(MaxRowsConfig::default(), 16)); + proof.prepared.pop(); + assert!(!verify(&proof, &elf, &one_group())); +} + +/// A prover that commits DECODE's prepared columns for another program (one +/// entry differs) and opens that commitment is refused: the verifier absorbs +/// the root it derived itself, so the transcripts part at the roots block. +#[test] +fn a_prepared_opening_of_another_program_is_refused() { + let elf = asm_elf_bytes("sub"); + let other = prove_with( + &elf, + &one_group(), + &options(MaxRowsConfig::default(), 16), + &Deviations { + other_prepared: true, + ..Default::default() + }, + ); + assert!(!verify(&other, &elf, &one_group())); +} + +/// The statement as one byte run is the stream `absorb_block` appends: two +/// transcripts fed each way draw the same challenge after a root. +#[test] +fn the_statement_bytes_are_what_absorb_block_appends() { + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + let elf = asm_elf_bytes("all_instructions_64"); + let format = many_groups(); + let proof = prove(&elf, &format, &options(MaxRowsConfig::small(), 16)); + let opts = ProofOptions::default_test_options(); + let frame = block_whir::block_frame(proof.statement(), &elf, &opts, &format).expect("frame"); + let digest = crate::statement::elf_digest(&elf); + let tag = block_whir::block_statement_tag(&format.zf.base); + let bytes = block_whir::block_statement_bytes(proof.statement(), &tag, &digest, &frame.config); + let (a, b) = crate::with_whir_hash!(|H| { + type T = DefaultTranscript::Transcript>; + let mut host = T::new(&[]); + block_whir::absorb_block( + &mut host, + &tag, + &elf, + &proof.public_output, + &proof.table_counts, + proof.num_private_input_pages, + &proof.runtime_page_ranges, + &proof.table_num_vars, + &frame.config, + &proof.groups, + ); + let mut run = T::new(&[]); + run.append_bytes(&bytes); + for t in [&mut host, &mut run] { + t.append_bytes(&proof.proof.roots[0]); + } + let a: FieldElement = host.sample_field_element(); + let b: FieldElement = run.sample_field_element(); + (a, b) + }); + assert_eq!(a, b); + assert_eq!(bytes.len() % 8, 0, "the run ends on a felt boundary"); +} + +// ======== prepared openings on two dense pages (test_dense_pages) ======== + +/// The prepared tables of `proof`'s statement, in AIR order, as the verifier +/// derives them: `(AIR index, position in the proof's table order)`. +fn prepared_of(elf: &[u8], proof: &BlockWhirProof, format: &BlockFormat) -> Vec<(usize, usize)> { + let opts = ProofOptions::default_test_options(); + let frame = block_whir::block_frame(proof.statement(), elf, &opts, format).expect("frame"); + let order: Vec = proof.groups.iter().flatten().map(|&t| t as usize).collect(); + block_whir::prepared_tables(&frame.airs, &frame.page_configs, format) + .expect("the prepared set") + .into_iter() + .map(|(table, _)| { + let position = order.iter().position(|&t| t == table).expect("in a group"); + (table, position) + }) + .collect() +} + +fn verify_skipping_prepared(proof: &BlockWhirProof, elf: &[u8], format: &BlockFormat) -> bool { + block_whir::verify_block_whir_with( + proof, + elf, + &ProofOptions::default_test_options(), + format, + true, + VerifierChecks::ALL, + ) + .unwrap_or(false) +} + +fn dense_block_in(format: &BlockFormat, deviations: &Deviations) -> (Vec, BlockWhirProof) { + let elf = asm_elf_bytes("test_dense_pages"); + let proof = prove_with( + &elf, + format, + &options(MaxRowsConfig::default(), 16), + deviations, + ); + (elf, proof) +} + +fn dense_block(deviations: &Deviations) -> (Vec, BlockWhirProof) { + dense_block_in(&one_group(), deviations) +} + +/// The group holding AIR table `table` in `proof`'s partition. +fn group_of(proof: &BlockWhirProof, table: usize) -> usize { + proof + .groups + .iter() + .position(|g| g.contains(&(table as u32))) + .expect("in a group") +} + +/// The fixture is what the negatives below need: DECODE and at least two dense +/// genesis pages prepared, one stack a group that holds any, and an honest +/// proof of it verifies. +#[test] +fn the_dense_fixture_prepares_decode_and_two_pages() { + let (elf, proof) = dense_block(&Deviations::default()); + let prepared = prepared_of(&elf, &proof, &one_group()); + assert!(prepared.len() >= 3, "DECODE and two pages: {prepared:?}"); + let mut groups: Vec = prepared.iter().map(|&(t, _)| group_of(&proof, t)).collect(); + groups.dedup(); + assert_eq!(proof.prepared.len(), groups.len(), "one opening a group"); + assert!(verify(&proof, &elf, &one_group())); +} + +/// Two groups' prepared openings swapped in the proof: each opens against the +/// other group's derived roots, and is refused. With the openings left +/// unchecked (the mutation) the swap verifies: the openings are what refuse it. +#[test] +fn two_groups_prepared_openings_swapped_are_refused() { + let format = many_groups(); + let (elf, mut proof) = dense_block_in(&format, &Deviations::default()); + assert!(proof.prepared.len() >= 2, "{} stacks", proof.prepared.len()); + assert!(verify(&proof, &elf, &format)); + proof.prepared.swap(0, 1); + assert!(!verify(&proof, &elf, &format)); + assert!(verify_skipping_prepared(&proof, &elf, &format)); +} + +/// A page's block opened at another table's point (a table of its group and its +/// height) — a valid opening of its own stack, of the wrong claim — is refused; +/// the mutation admits it. +#[test] +fn a_page_opened_at_another_tables_point_is_refused() { + let (elf, honest) = dense_block(&Deviations::default()); + let prepared = prepared_of(&elf, &honest, &one_group()); + let page = prepared[1].0; + let height = honest.table_num_vars[page]; + let other = honest.groups[group_of(&honest, page)] + .iter() + .map(|&t| t as usize) + .find(|&t| t != page && honest.table_num_vars[t] == height) + .expect("the page shares its group with a table of its height"); + let (_, wrong) = dense_block(&Deviations { + prepared_points: vec![(page, other)], + ..Default::default() + }); + assert!(!verify(&wrong, &elf, &one_group())); + assert!(verify_skipping_prepared(&wrong, &elf, &one_group())); +} + +/// Stacks committed wrongly — two pages' blocks swapped in their stack, a page's +/// block holding the other page's columns, DECODE's block another program's — +/// each opened consistently: refused at the roots block, since the verifier +/// absorbs the stacks it derives from the program and the partition, so even +/// with the openings unchecked. (The leaf's form, whose mutation trusts the +/// prover's roots, is in `lfm::whir_block_tests`.) +#[test] +fn a_wrongly_committed_prepared_stack_is_refused() { + use crate::block_whir::StackTamper; + let (elf, honest) = dense_block(&Deviations::default()); + let prepared = prepared_of(&elf, &honest, &one_group()); + let (a, b) = (prepared[1].0, prepared[2].0); + let mut tampers = vec![Deviations { + prepared_stack: vec![StackTamper::ColumnsOf { table: a, from: b }], + ..Default::default() + }]; + if group_of(&honest, a) == group_of(&honest, b) { + tampers.push(Deviations { + prepared_stack: vec![StackTamper::Swap(a, b)], + ..Default::default() + }); + } + tampers.push(Deviations { + other_prepared: true, + ..Default::default() + }); + for deviations in &tampers { + let (_, wrong) = dense_block(deviations); + assert!(!verify(&wrong, &elf, &one_group())); + assert!(!verify_skipping_prepared(&wrong, &elf, &one_group())); + } +} + +/// With the openings off (`BlockFormat::prepared`, the measurement arm) the +/// block carries none and the host verifier evaluates the columns itself. +#[test] +fn a_block_without_prepared_openings_verifies_on_the_host() { + let format = BlockFormat { + prepared: false, + ..one_group() + }; + let elf = asm_elf_bytes("test_dense_pages"); + let proof = prove(&elf, &format, &options(MaxRowsConfig::default(), 16)); + assert!(proof.prepared.is_empty()); + assert!(verify(&proof, &elf, &format)); + assert!( + !verify(&proof, &elf, &one_group()), + "the formats are not interchangeable" + ); +} + +/// The KECCAK_RND count is bounded before the verifier builds an AIR off it, +/// and a table the block does not chunk (HINT) stays one table. +#[test] +fn an_inflated_keccak_rnd_count_is_refused() { + let mut counts = prove( + &asm_elf_bytes("sub"), + &one_group(), + &options(MaxRowsConfig::default(), 16), + ) + .table_counts; + counts.keccak_rnd = block_whir::BLOCK_MAX_KECCAK_RND + 1; + assert!(block_whir::validate_block_counts(&counts).is_err()); + counts.keccak_rnd = 3; + assert!(block_whir::validate_block_counts(&counts).is_ok()); + counts.hint = 2; + assert!(block_whir::validate_block_counts(&counts).is_err()); +} + +/// Every accelerator present, KECCAK, KECCAK_RND and ECSM in two tables each +/// and ECDAS in three: in proof order (`VmAirs::air_refs`) the five fixed +/// tables, COMMIT 5, KECCAK 6–7, KECCAK_RND 8–9, ECSM 10–11, ECDAS 12–14, +/// HINT 15, CPU 16–17, MEMW_R 18. +fn chunked_counts() -> crate::TableCounts { + crate::TableCounts { + cpu: 2, + lt: 0, + memw: 0, + memw_aligned: 0, + load: 0, + mul: 0, + dvrm: 0, + shift: 0, + branch: 0, + memw_register: 1, + eq: 0, + bytewise: 0, + store: 0, + cpu32: 0, + keccak: 2, + keccak_rnd: 2, + ecsm: 2, + ecdas: 3, + hint: 1, + copy: 2, + blake3: 0, + is_wh: 0, + ecrec: 0, + } +} + +/// The ECDAS count is bounded before the verifier builds an AIR off it, and +/// HINT (a table the block does not chunk) stays one table. +#[test] +fn an_inflated_ecdas_count_is_refused() { + let mut counts = chunked_counts(); + assert!(block_whir::validate_block_counts(&counts).is_ok()); + counts.ecdas = block_whir::BLOCK_MAX_ECDAS + 1; + assert!(block_whir::validate_block_counts(&counts).is_err()); + counts.ecdas = block_whir::BLOCK_MAX_ECDAS; + assert!(block_whir::validate_block_counts(&counts).is_ok()); + counts.hint = 2; + assert!(block_whir::validate_block_counts(&counts).is_err()); +} + +/// ★ COPY, KECCAK and ECSM are chunked in the block: any count up to +/// [`block_whir::BLOCK_MAX_COPY`] / [`block_whir::BLOCK_MAX_KECCAK`] / +/// [`block_whir::BLOCK_MAX_ECSM`] passes, one more is refused before the +/// verifier builds an AIR off it. Every other verifier keeps them to one table +/// ([`crate::TableCounts::validate`]). +/// +/// Mutations (laptop): the bound dropped and `KECCAK: MAX + 1` passes; the +/// one-table bound left on KECCAK or ECSM and `2` is refused. +#[test] +fn keccak_and_ecsm_counts_are_bounded_by_the_block() { + type Field = fn(&mut crate::TableCounts) -> &mut usize; + let tables: [(&str, usize, Field); 3] = [ + ("COPY", block_whir::BLOCK_MAX_COPY, |c| &mut c.copy), + ("KECCAK", block_whir::BLOCK_MAX_KECCAK, |c| &mut c.keccak), + ("ECSM", block_whir::BLOCK_MAX_ECSM, |c| &mut c.ecsm), + ]; + for (name, max, field) in tables { + let mut single = chunked_counts(); + (single.copy, single.keccak, single.keccak_rnd) = (1, 1, 1); + (single.ecsm, single.ecdas) = (1, 1); + assert!(single.validate().is_ok()); + *field(&mut single) = 2; + assert!( + single.validate().is_err(), + "every other verifier keeps {name} to one table" + ); + for (count, ok) in [(1, true), (2, true), (max, true), (max + 1, false)] { + let mut counts = chunked_counts(); + *field(&mut counts) = count; + let checked = block_whir::validate_block_counts(&counts); + assert_eq!( + checked.is_ok(), + ok, + "{name} count {count}: {checked:?} (the block takes at most {max})" + ); + } + } +} + +/// ★ The statement's heights are capped for the chunked tables, and only +/// theirs: a COPY table over [`block_whir::BLOCK_COPY_MAX_VARS`], a KECCAK +/// table over [`block_whir::BLOCK_KECCAK_MAX_VARS`], a KECCAK_RND table over +/// [`block_whir::BLOCK_KECCAK_RND_MAX_VARS`], an ECSM table over +/// [`block_whir::BLOCK_ECSM_MAX_VARS`] and an ECDAS table over +/// [`block_whir::BLOCK_ECDAS_MAX_VARS`] are refused; every other table far over +/// every cap passes. The controls next to the ranges (HALT before COPY, HINT +/// after ECDAS) are what an off-by-one in the ranges would cap instead; each +/// table at its own cap passing pins which cap each range carries. +#[test] +fn the_block_caps_the_chunked_tables_heights() { + let counts = chunked_counts(); + let n = counts.total().expect("fits") + crate::FIXED_TABLE_COUNT; + assert_eq!(n, 20); + let check = |vars: &[u8]| block_whir::check_chunked_heights(&counts, vars); + let honest = vec![10u8; n]; + assert!(check(&honest).is_ok()); + let copy_cap = block_whir::BLOCK_COPY_MAX_VARS as u8; + let (keccak_cap, keccak_rnd_cap, ecsm_cap, ecdas_cap) = ( + block_whir::BLOCK_KECCAK_MAX_VARS as u8, + block_whir::BLOCK_KECCAK_RND_MAX_VARS as u8, + block_whir::BLOCK_ECSM_MAX_VARS as u8, + block_whir::BLOCK_ECDAS_MAX_VARS as u8, + ); + let mut at_caps = honest.clone(); + at_caps[5..7].fill(copy_cap); + at_caps[7..9].fill(keccak_cap); + at_caps[9..11].fill(keccak_rnd_cap); + at_caps[11..13].fill(ecsm_cap); + at_caps[13..16].fill(ecdas_cap); + assert!(check(&at_caps).is_ok(), "a table at its cap passes"); + for (what, idx, vars) in [ + ("ECDAS[2] over its cap", 15, ecdas_cap + 1), + ("ECDAS[0] over its cap", 13, ecdas_cap + 1), + ("ECSM[1] over its cap", 12, ecsm_cap + 1), + ("ECSM[0] over its cap", 11, ecsm_cap + 1), + ("KECCAK_RND[1] over its cap", 10, keccak_rnd_cap + 1), + ("KECCAK_RND[0] over its cap", 9, keccak_rnd_cap + 1), + ("KECCAK[1] over its cap", 8, keccak_cap + 1), + ("KECCAK[0] over its cap", 7, keccak_cap + 1), + ("COPY[1] over its cap", 6, copy_cap + 1), + ("COPY[0] over its cap", 5, copy_cap + 1), + ] { + let mut stated = honest.clone(); + stated[idx] = vars; + let refused = check(&stated); + assert!(refused.is_err(), "the block must refuse {what}"); + println!("BLOCK WHIR refuses {what}: {:?}", refused.err()); + } + for (what, idx) in [("HALT", 4), ("HINT", 16), ("CPU[0]", 17), ("MEMW_R[0]", 19)] { + let mut stated = honest.clone(); + stated[idx] = 22; + assert!( + check(&stated).is_ok(), + "{what} is not a chunked table: its height is not capped" + ); + } +} + +/// ★ Every chunked table's cap is the tallest power of two at which its +/// columns fill at most one polynomial of the format's 2^27 stack: COPY +/// 41 × 2^21, KECCAK 511 × 2^18, KECCAK_RND 1,480 × 2^16, ECSM 667 × 2^17, +/// ECDAS 521 × 2^17. One doubling past a cap would stack the table into two +/// polynomials. +/// +/// Mutation (laptop): the KECCAK or the ECSM cap lifted, and this test fails +/// (the caps test reads the constants, so it cannot see a lifted cap). +#[test] +fn every_chunked_cap_is_the_tallest_table_one_stacked_polynomial_holds() { + use crate::tables::{copy, ecdas, ecsm, keccak, keccak_rnd}; + let stack = ZfFormat::DEFAULT.whir_stack.get(); + let widths = [ + ("COPY", copy::cols::NUM_COLUMNS), + ("KECCAK", keccak::cols::NUM_COLUMNS), + ("KECCAK_RND", keccak_rnd::cols::NUM_COLUMNS), + ("ECSM", ecsm::cols::NUM_COLUMNS), + ("ECDAS", ecdas::cols::NUM_COLUMNS), + ]; + for ((name, _, cap), (table, width)) in block_whir::chunked_table_ranges(&chunked_counts()) + .into_iter() + .zip(widths) + { + assert_eq!(name, table); + let cells = |vars: usize| width << vars; + assert!( + cells(cap) <= 1 << stack, + "{name}: {width} columns at 2^{cap} rows must fit one polynomial of 2^{stack}" + ); + assert!( + cells(cap + 1) > 1 << stack, + "{name}: 2^{cap} rows must be the tallest that fits one polynomial of 2^{stack}" + ); + println!( + "BLOCK WHIR CAP {name}: {width} columns × 2^{cap} = {} ≤ 2^{stack} < {width} × 2^{}", + cells(cap), + cap + 1 + ); + } +} + +/// ★ G2: no table may be stated taller than [`block_whir::BLOCK_MAX_TABLE_VARS`] +/// (2^27), so no chain is taller than the 2^27 stack; at the cap it passes. +#[test] +fn no_table_may_be_stated_over_the_table_cap() { + let cap = block_whir::BLOCK_MAX_TABLE_VARS as u8; + let mut stated = vec![10u8; 17]; + assert!(block_whir::check_table_heights(&stated).is_ok()); + stated[3] = cap; + assert!( + block_whir::check_table_heights(&stated).is_ok(), + "at the cap" + ); + stated[3] = cap + 1; + let refused = block_whir::check_table_heights(&stated); + assert!( + format!("{refused:?}").contains("table 3"), + "a table over the cap must be refused, got {refused:?}" + ); +} + +/// ★ G2 end to end: a statement giving table 0 (BITWISE) 2^28 rows is refused +/// at the frame with the cap's own error, before any AIR is built. +/// +/// Mutation (box script): `check_table_heights` returning `Ok(())` and the error +/// is no longer the cap's. +#[test] +fn a_table_stated_over_the_table_cap_is_refused() { + let elf = asm_elf_bytes("sub"); + let format = many_groups(); + let mut proof = prove(&elf, &format, &options(MaxRowsConfig::default(), 16)); + assert!(verify(&proof, &elf, &format)); + proof.table_num_vars[0] = (block_whir::BLOCK_MAX_TABLE_VARS + 1) as u8; + let refused = + block_whir::verify_block_whir(&proof, &elf, &ProofOptions::default_test_options(), &format); + let msg = format!("{refused:?}"); + assert!( + msg.contains("table 0 states 2^28 rows") && msg.contains("at most 2^27"), + "the frame must refuse a table over the table cap with the cap's error, got {msg}" + ); + println!("BLOCK WHIR TABLE OVER CAP: refused at the frame: {msg}"); +} + +/// The ranges the caps read are the chunked AIRs' positions in +/// [`crate::VmAirs::air_refs`] — the statement's table order — and nothing +/// next to them. +#[test] +fn chunked_table_ranges_name_the_chunked_airs() { + use executor::elf::Elf; + let elf = asm_elf_bytes("poc_rodata_commit"); + let program = Elf::load(&elf).expect("load the ELF"); + let page_configs = crate::tables::trace_builder::Traces::page_configs_from_elf(&program); + let counts = chunked_counts(); + let airs = crate::VmAirs::new( + &program, + &ProofOptions::default_test_options(), + false, + &page_configs, + &counts, + &vec![4; counts.cpu], + None, + true, + None, + crate::test_utils::halted_register_commitment( + &ProofOptions::default_test_options(), + &program, + &[], + ) + .preprocessed(), + ); + let names: Vec = airs + .air_refs() + .iter() + .map(|a| a.name().to_string()) + .collect(); + let ranges = block_whir::chunked_table_ranges(&counts); + assert_eq!( + ranges.each_ref().map(|(table, ..)| *table), + ["COPY", "KECCAK", "KECCAK_RND", "ECSM", "ECDAS"], + "the chunked tables in AIR order" + ); + for (table, range, _) in ranges { + assert!( + range.len() >= 2, + "{table}: every chunked table in two tables or more" + ); + for (k, idx) in range.clone().enumerate() { + assert_eq!(names[idx], format!("{table}[{k}]")); + } + let prefix = format!("{table}["); + assert!(!names[range.start - 1].starts_with(&prefix)); + assert!(!names[range.end].starts_with(&prefix)); + } +} + +/// ECDAS cut into tables of 2^4 rows: test_ecsm_multi's 42 double/add steps +/// (calls k = 1, 5, 0xABCDEF) over three tables at the default (the fourth, +/// padding rows alone, is not built), the 0xABCDEF call through all three; +/// their bus sums add up to the whole table's. +#[test] +fn ecdas_split_into_chunks_proves_and_verifies() { + let elf = asm_elf_bytes("test_ecsm_multi"); + let mut o = options(MaxRowsConfig::default(), 16); + o.ecdas_rows_log2 = 4; + let proof = prove(&elf, &many_groups(), &o); + assert_eq!(proof.table_counts.ecdas, 3, "42 rows in 16-row tables"); + assert!(verify(&proof, &elf, &many_groups())); + // The production cut leaves the same program one table. + let one = prove(&elf, &many_groups(), &options(MaxRowsConfig::default(), 16)); + assert_eq!(one.table_counts.ecdas, 1); + assert!(verify(&one, &elf, &many_groups())); +} + +/// `program`'s traces built once, KECCAK_RND at its production cut, then +/// `cut`. +fn split_traces( + program: &str, + cut: impl FnOnce(&mut crate::tables::trace_builder::Traces), +) -> ( + executor::elf::Elf, + Vec, + crate::tables::trace_builder::Traces, +) { + use executor::elf::Elf; + use executor::vm::execution::Executor; + let elf = asm_elf_bytes(program); + let program = Elf::load(&elf).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let mut traces = crate::tables::trace_builder::Traces::from_elf_and_logs( + &program, + &logs, + &MaxRowsConfig::default(), + &[], + ) + .expect("the traces build"); + block_whir::split_keccak_rnd(&mut traces, 16, CutPadding::default()).expect("the split"); + cut(&mut traces); + (program, elf, traces) +} + +/// test_ecsm_multi's traces with ECDAS cut at 16 rows, built once. +fn ecsm_multi_split() -> ( + executor::elf::Elf, + Vec, + crate::tables::trace_builder::Traces, +) { + split_traces("test_ecsm_multi", |traces| { + block_whir::split_ecdas(traces, 4, CutPadding::WholeRun); + assert_eq!(traces.ecdases.len(), 4); + }) +} + +/// Whether the traces `build` makes, after `tamper`, prove into a block the +/// verifier accepts (a prover refusal counts as not accepted). The traces are +/// built afresh for each prove. +fn traces_block_accepts( + build: impl Fn() -> ( + executor::elf::Elf, + Vec, + crate::tables::trace_builder::Traces, + ), + tamper: impl Fn(&mut crate::tables::trace_builder::Traces), +) -> bool { + let format = many_groups(); + let (program, elf, mut traces) = build(); + tamper(&mut traces); + match block_whir::prove_traces( + &program, + &elf, + &mut traces, + &ProofOptions::default_test_options(), + &format, + &options(MaxRowsConfig::default(), 16), + &Deviations::default(), + false, + &|_, _| {}, + &mut Default::default(), + ) { + Ok(proof) => verify(&proof, &elf, &format), + Err(e) => { + println!(" (the prover refused: {e:?})"); + false + } + } +} + +/// Whether test_ecsm_multi's split traces, after `tamper`, prove into a block +/// the verifier accepts. +fn split_block_accepts(tamper: impl Fn(&mut crate::tables::trace_builder::Traces)) -> bool { + traces_block_accepts(ecsm_multi_split, tamper) +} + +/// ★ A call split across ECDAS tables is held together by the Ecdas bus alone, +/// keyed by the call's timestamp and the step's `(round, op)`. +/// +/// - Control: two whole rows swapped between table 0 and table 1 (a step of +/// the 5·G call and a step of the 0xABCDEF call) is the same multiset of +/// rows, and verifies. +/// - Negative: the same two rows swap only their timestamps, so each claims +/// the other's call. No constraint and no range check reads the timestamp, +/// and both rows have `NEXT_OP = 0` (their `Bit` sends are off): only the +/// Ecdas tuples moved, and the block is refused. +/// +/// Mutation (box script): drop the timestamp from `ecsm::ecdas_tuple` and the +/// negative verifies — the call key is what refuses it. +#[test] +fn a_split_ecdas_call_is_keyed_on_the_bus() { + use crate::tables::ecdas::cols; + use crate::tables::types::FE; + let (a, b) = { + let (_, _, traces) = ecsm_multi_split(); + let next_op_off = + |t: usize, r: usize| *traces.ecdases[t].main_table.get(r, cols::NEXT_OP) == FE::zero(); + let ts = |t: usize, r: usize| { + ( + *traces.ecdases[t].main_table.get(r, cols::TIMESTAMP_0), + *traces.ecdases[t].main_table.get(r, cols::TIMESTAMP_1), + ) + }; + let a = (0..3).find(|&r| next_op_off(0, r)).expect("a 5·G step"); + let b = (0..16) + .find(|&r| next_op_off(1, r)) + .expect("a 0xABCDEF step"); + assert_ne!(ts(0, a), ts(1, b), "two different calls"); + assert_eq!(*traces.ecdases[0].main_table.get(a, cols::MU), FE::one()); + assert_eq!(*traces.ecdases[1].main_table.get(b, cols::MU), FE::one()); + (a, b) + }; + let swap = |cols_to_swap: Vec| { + move |t: &mut crate::tables::trace_builder::Traces| { + for &col in &cols_to_swap { + let x = *t.ecdases[0].main_table.get(a, col); + let y = *t.ecdases[1].main_table.get(b, col); + t.ecdases[0].main_table.set(a, col, y); + t.ecdases[1].main_table.set(b, col, x); + } + } + }; + assert!(split_block_accepts(|_| {}), "the honest split verifies"); + assert!( + split_block_accepts(swap((0..cols::NUM_COLUMNS).collect())), + "whole rows swapped across tables are the same steps: accepted" + ); + println!("BLOCK WHIR ECDAS SPLIT CONTROL: whole rows swapped across tables, accepted"); + assert!( + !split_block_accepts(swap(vec![cols::TIMESTAMP_0, cols::TIMESTAMP_1])), + "a continuation row reattached to another call must be refused" + ); + println!( + "BLOCK WHIR ECDAS SPLIT NEGATIVE: table 0 row {a} and table 1 row {b} swapped calls, refused" + ); +} + +/// test_ecrec's traces (two computed recoveries and a bail), built once. +fn ecrec_traces() -> ( + executor::elf::Elf, + Vec, + crate::tables::trace_builder::Traces, +) { + split_traces("test_ecrec", |_| {}) +} + +/// ★ An ECREC call's first chain, second chain and final addition are three +/// separate conversations on the `Ecdas` and `Bit` buses, keyed at its timestamp +/// `TS`, `TS + 1` and `TS + 2`; every ECALL timestamp is a multiple of 4, so no key +/// of one call is a key of another. +/// +/// - Control: the honest traces verify, and so do two whole ECDAS rows swapped +/// between table positions (the same multiset of rows). +/// - Negative (neighbouring call): a second-chain row of the first call and one of +/// the second call swap only their timestamps, each claiming the other call's +/// `TS + 1`. Refused. +/// - Negative (neighbouring chain): a first-chain row of the first call is given the +/// key `TS + 1` of that call's second chain. Refused. +#[test] +fn ecrec_chains_are_keyed_apart_on_the_bus() { + use crate::tables::ecdas::cols; + use crate::tables::types::FE; + let (a, b, c) = { + let (_, _, traces) = ecrec_traces(); + assert_eq!(traces.ecrecs.len(), 1); + assert_eq!(traces.ecdases.len(), 1, "one ECDAS table"); + let er = &traces.ecrecs[0].main_table; + let ts = |row: usize| *er.get(row, crate::tables::ecrec::cols::TIMESTAMP_0); + let (t1, t2) = (ts(0), ts(1)); + assert_ne!(t1, t2); + let ed = &traces.ecdases[0].main_table; + let find = |key: FE| { + (0..ed.height) + .find(|&r| { + *ed.get(r, cols::TIMESTAMP_0) == key + && *ed.get(r, cols::NEXT_OP) == FE::zero() + && *ed.get(r, cols::MU) == FE::one() + }) + .expect("a row with that key and no Bit send") + }; + (find(t1 + FE::one()), find(t2 + FE::one()), find(t1)) + }; + let swap = |x: usize, y: usize, cols_to_swap: Vec| { + move |t: &mut crate::tables::trace_builder::Traces| { + for &col in &cols_to_swap { + let vx = *t.ecdases[0].main_table.get(x, col); + let vy = *t.ecdases[0].main_table.get(y, col); + t.ecdases[0].main_table.set(x, col, vy); + t.ecdases[0].main_table.set(y, col, vx); + } + } + }; + assert!( + traces_block_accepts(ecrec_traces, |_| {}), + "the honest calls verify" + ); + assert!( + traces_block_accepts(ecrec_traces, swap(a, b, (0..cols::NUM_COLUMNS).collect())), + "whole rows swapped are the same rows: accepted" + ); + println!("BLOCK WHIR ECREC KEY CONTROL: honest and whole-row swap, accepted"); + assert!( + !traces_block_accepts( + ecrec_traces, + swap(a, b, vec![cols::TIMESTAMP_0, cols::TIMESTAMP_1]) + ), + "a second-chain row reattached to the neighbouring call must be refused" + ); + println!("BLOCK WHIR ECREC KEY NEGATIVE: second-chain rows {a} and {b} swapped calls, refused"); + assert!( + !traces_block_accepts(ecrec_traces, move |t| { + let key = *t.ecdases[0].main_table.get(c, cols::TIMESTAMP_0); + t.ecdases[0] + .main_table + .set(c, cols::TIMESTAMP_0, key + FE::one()); + }), + "a first-chain row given its call's second-chain key must be refused" + ); + println!( + "BLOCK WHIR ECREC KEY NEGATIVE: first-chain row {c} moved to the second chain's key, refused" + ); +} + +/// ★ The second chain's key `TS + 1` is what keeps the two chains' scalars apart. +/// +/// The forgery (`trace_builder::FORGE_ECREC_SWAPPED_CHAINS`) runs the first call's +/// chains with their scalars swapped — `u2·G` and `u1·R` — and writes the key +/// `u2·G + u1·R`; the ECREC row's scalar columns and relations stay honest and its +/// two `LEN` columns swap, so every constraint holds and every message but the +/// second chain's `Bit` receives balances. Those are keyed at `TS + 1` and refuse it. +/// +/// Mutation (box script): key the second chain at `TS` (the ECREC table's chain-2 +/// interactions and the builder's chain-2 timestamps) and the forgery verifies. +#[test] +fn ecrec_chains_swapped_between_keys_are_refused() { + use crate::tables::trace_builder::FORGE_ECREC_SWAPPED_CHAINS; + use std::sync::atomic::Ordering; + let honest = ecrec_traces().2.public_output_bytes; + assert!( + traces_block_accepts(ecrec_traces, |_| {}), + "the honest calls verify" + ); + println!("BLOCK WHIR ECREC SWAP CONTROL: honest, accepted"); + FORGE_ECREC_SWAPPED_CHAINS.store(true, Ordering::SeqCst); + let forged = ecrec_traces().2.public_output_bytes; + let accepted = traces_block_accepts(ecrec_traces, |_| {}); + FORGE_ECREC_SWAPPED_CHAINS.store(false, Ordering::SeqCst); + assert_ne!(forged[..64], honest[..64], "the forged first key differs"); + assert_eq!(forged[64..], honest[64..], "only the first call is forged"); + assert!(!accepted, "chains with swapped scalars must be refused"); + println!("BLOCK WHIR ECREC SWAP NEGATIVE: swapped chains, refused"); +} + +/// ★ COPY cut inside a call: test_copy_ecalls' 17 COPY rows (a 21-byte +/// memcpy's seven, a memset's four, a zero-length call, an overlapping +/// two-byte copy and a three-byte `write`) at one, two and four rows a table: +/// 17, 9 and 5 tables at the default (the tables of padding rows alone are not +/// built), and every proof verifies; the production cut leaves the guest one +/// table, and so does the streamed build at two rows a table. +#[test] +fn copy_split_into_chunks_proves_and_verifies() { + let format = many_groups(); + let elf = asm_elf_bytes("test_copy_ecalls"); + for (rows_log2, tables) in [(0, 17), (1, 9), (2, 5)] { + let mut o = options(MaxRowsConfig::default(), 16); + o.copy_rows_log2 = rows_log2; + let proof = prove(&elf, &format, &o); + assert_eq!( + proof.table_counts.copy, tables, + "17 rows in 2^{rows_log2}-row tables" + ); + assert!( + verify(&proof, &elf, &format), + "COPY: {tables} tables verify" + ); + println!("BLOCK WHIR COPY CHUNKED: {tables} tables of 2^{rows_log2} rows, accepted"); + } + let mut o = streamed(MaxRowsConfig::small(), 16, 4); + o.copy_rows_log2 = 1; + let proof = prove(&elf, &format, &o); + assert_eq!(proof.table_counts.copy, 9, "COPY streamed"); + assert!(verify(&proof, &elf, &format), "COPY streamed"); + let (_, _, traces) = split_traces("test_copy_ecalls", |traces| { + block_whir::split_copy( + traces, + block_whir::BLOCK_COPY_ROWS_LOG2, + CutPadding::default(), + ); + }); + assert_eq!(traces.copies.len(), 1, "COPY: production cut"); +} + +/// ★ A COPY cut may fall inside one call, whose rows reach each other only +/// through the CopyNext bus, keyed by the call's timestamp and its running +/// `(src, dst, count)`. At one row a table, the 21-byte memcpy's first two +/// rows are tables 0 and 1. +/// +/// - Control: the two rows swapped as whole rows across the two tables are +/// the same multiset of rows, and verify. +/// - Negative: the second row's source moved by eight, its successor alike so +/// the row's own `ADD` still holds: the first row's CopyNext send no longer +/// meets it, and the block is refused. +#[test] +fn split_copy_chains_are_keyed_on_the_bus() { + use crate::tables::copy::cols; + use crate::tables::types::FE; + let build = || { + split_traces("test_copy_ecalls", |traces| { + block_whir::split_copy(traces, 0, CutPadding::default()); + assert_eq!(traces.copies.len(), 17, "COPY: one row a table"); + }) + }; + { + let (_, _, traces) = build(); + let get = |t: usize, col: usize| *traces.copies[t].main_table.get(0, col); + assert_eq!(get(0, cols::FIRST), FE::one(), "table 0 starts the memcpy"); + assert_eq!(get(1, cols::FIRST), FE::zero(), "table 1 continues it"); + assert_eq!(get(0, cols::COUNT_0), FE::from(21u64)); + assert_eq!(get(1, cols::COUNT_0), FE::from(13u64)); + } + assert!( + traces_block_accepts(build, |_| {}), + "COPY: the honest split verifies" + ); + assert!( + traces_block_accepts(build, |traces| { + for col in 0..cols::NUM_COLUMNS { + let x = *traces.copies[0].main_table.get(0, col); + let y = *traces.copies[1].main_table.get(0, col); + traces.copies[0].main_table.set(0, col, y); + traces.copies[1].main_table.set(0, col, x); + } + }), + "COPY: rows of one call swapped whole across tables are the same rows: accepted" + ); + println!("BLOCK WHIR COPY SPLIT CONTROL: whole rows swapped across tables, accepted"); + assert!( + !traces_block_accepts(build, |traces| { + for col in [cols::SRC_0, cols::SRC_INCR_0] { + let v = *traces.copies[1].main_table.get(0, col); + traces.copies[1].main_table.set(0, col, v + FE::from(8u64)); + } + }), + "COPY: a continuation row's source moved across a cut must be refused" + ); + println!("BLOCK WHIR COPY SPLIT NEGATIVE: a moved continuation source, refused"); +} + +/// The two call tables the block cuts between calls, with their guest (three +/// calls each: one table of four rows, the last row padding). +#[derive(Clone, Copy)] +enum CallTable { + Keccak, + Ecsm, +} + +impl CallTable { + const ALL: [CallTable; 2] = [CallTable::Keccak, CallTable::Ecsm]; + + fn name(self) -> &'static str { + match self { + CallTable::Keccak => "KECCAK", + CallTable::Ecsm => "ECSM", + } + } + + fn guest(self) -> &'static str { + match self { + CallTable::Keccak => "test_keccak_multi", + CallTable::Ecsm => "test_ecsm_multi", + } + } + + /// `o` with this table cut at 2^`rows_log2` rows. + fn cut(self, mut o: BlockOptions, rows_log2: usize) -> BlockOptions { + match self { + CallTable::Keccak => o.keccak_rows_log2 = rows_log2, + CallTable::Ecsm => o.ecsm_rows_log2 = rows_log2, + } + o + } + + /// The whole run padded: the padding row keeps its own table. + fn split(self, traces: &mut crate::tables::trace_builder::Traces, rows_log2: usize) { + match self { + CallTable::Keccak => block_whir::split_keccak(traces, rows_log2, CutPadding::WholeRun), + CallTable::Ecsm => block_whir::split_ecsm(traces, rows_log2, CutPadding::WholeRun), + } + } + + fn count(self, counts: &crate::TableCounts) -> usize { + match self { + CallTable::Keccak => counts.keccak, + CallTable::Ecsm => counts.ecsm, + } + } + + fn tables( + self, + traces: &mut crate::tables::trace_builder::Traces, + ) -> &mut Vec> { + match self { + CallTable::Keccak => &mut traces.keccaks, + CallTable::Ecsm => &mut traces.ecsms, + } + } + + /// Width, `MU` and the two timestamp columns. + fn columns(self) -> (usize, usize, [usize; 2]) { + use crate::tables::{ecsm, keccak}; + match self { + CallTable::Keccak => ( + keccak::cols::NUM_COLUMNS, + keccak::cols::MU, + [keccak::cols::TIMESTAMP_0, keccak::cols::TIMESTAMP_1], + ), + CallTable::Ecsm => ( + ecsm::cols::NUM_COLUMNS, + ecsm::cols::MU, + [ecsm::cols::TIMESTAMP_0, ecsm::cols::TIMESTAMP_1], + ), + } + } + + /// The guest's traces with this table cut at one row a table: its three + /// calls and the padding row, four tables. + fn one_call_a_table( + self, + ) -> ( + executor::elf::Elf, + Vec, + crate::tables::trace_builder::Traces, + ) { + split_traces(self.guest(), |traces| { + self.split(traces, 0); + assert_eq!( + self.tables(traces).len(), + 4, + "{}: one row a table", + self.name() + ); + }) + } +} + +/// ★ KECCAK and ECSM cut between calls: test_keccak_multi's three +/// permutations and test_ecsm_multi's three scalar multiplications (one table +/// of four rows each, the last row padding) at one row a table (three tables +/// at the default: one call each, the padding row's table not built) and at +/// two rows a table (two); every proof verifies, the bus sums over the tables +/// adding up to the one table's. The production cuts leave each guest one +/// table. +#[test] +fn keccak_and_ecsm_split_into_chunks_prove_and_verify() { + let format = many_groups(); + for table in CallTable::ALL { + let (name, elf) = (table.name(), asm_elf_bytes(table.guest())); + for (rows_log2, tables) in [(0, 3), (1, 2)] { + let o = table.cut(options(MaxRowsConfig::default(), 16), rows_log2); + let proof = prove(&elf, &format, &o); + assert_eq!( + table.count(&proof.table_counts), + tables, + "{name}: 4 rows in 2^{rows_log2}-row tables" + ); + assert!( + verify(&proof, &elf, &format), + "{name}: {tables} tables verify" + ); + println!("BLOCK WHIR {name} CHUNKED: {tables} tables of 2^{rows_log2} rows, accepted"); + } + let (_, _, mut traces) = split_traces(table.guest(), |traces| { + block_whir::split_keccak( + traces, + block_whir::BLOCK_KECCAK_ROWS_LOG2, + CutPadding::default(), + ); + block_whir::split_ecsm( + traces, + block_whir::BLOCK_ECSM_ROWS_LOG2, + CutPadding::default(), + ); + }); + assert_eq!(table.tables(&mut traces).len(), 1, "{name}: production cut"); + } +} + +/// The streamed build cuts KECCAK and ECSM at the run's end like the whole-run +/// build: two rows a table, windows of 2^4 cycles. +#[test] +fn a_streamed_block_with_keccak_and_ecsm_chunks_proves_and_verifies() { + let format = many_groups(); + for table in CallTable::ALL { + let elf = asm_elf_bytes(table.guest()); + let o = table.cut(streamed(MaxRowsConfig::small(), 16, 4), 1); + let proof = prove(&elf, &format, &o); + assert_eq!(table.count(&proof.table_counts), 2, "{}", table.name()); + assert!(verify(&proof, &elf, &format), "{} streamed", table.name()); + } +} + +/// ★ A KECCAK or ECSM row is one whole call, so a cut falls between calls; a +/// call reaches its rounds or steps, its scalar bits and its memory only +/// through buses keyed by its timestamp. +/// +/// - Control: call 0 (table 0) and call 1 (table 1) swapped as whole rows is +/// the same multiset of rows, and verifies: the cut carries nothing a bus +/// does not. +/// - Negative: the same two rows swap only their timestamps, so each claims +/// the other's call (the Ecall receives still balance: both timestamps are +/// still received once); its memory, rounds or steps no longer meet it on +/// the buses, and the block is refused. +/// +/// No mutation of its own: the timestamp key is each table's bus design, +/// which the cut leaves as it is (one table of these rows is refused alike). +#[test] +fn split_keccak_and_ecsm_calls_are_keyed_on_the_bus() { + use crate::tables::types::FE; + for table in CallTable::ALL { + let name = table.name(); + let (width, mu, ts_cols) = table.columns(); + { + let (_, _, mut traces) = table.one_call_a_table(); + let tables = table.tables(&mut traces); + let ts = |t: usize| ts_cols.map(|c| *tables[t].main_table.get(0, c)); + assert_eq!(*tables[0].main_table.get(0, mu), FE::one()); + assert_eq!(*tables[1].main_table.get(0, mu), FE::one()); + assert_ne!(ts(0), ts(1), "{name}: two different calls"); + } + let swap = |cols: Vec| { + move |traces: &mut crate::tables::trace_builder::Traces| { + let tables = table.tables(traces); + for &col in &cols { + let x = *tables[0].main_table.get(0, col); + let y = *tables[1].main_table.get(0, col); + tables[0].main_table.set(0, col, y); + tables[1].main_table.set(0, col, x); + } + } + }; + let build = || table.one_call_a_table(); + assert!( + traces_block_accepts(build, |_| {}), + "{name}: the honest split verifies" + ); + assert!( + traces_block_accepts(build, swap((0..width).collect())), + "{name}: whole calls swapped across tables are the same calls: accepted" + ); + println!("BLOCK WHIR {name} SPLIT CONTROL: whole rows swapped across tables, accepted"); + assert!( + !traces_block_accepts(build, swap(ts_cols.to_vec())), + "{name}: a call reattached to another call's timestamp must be refused" + ); + println!( + "BLOCK WHIR {name} SPLIT NEGATIVE: table 0 and table 1 swapped timestamps, refused" + ); + } +} + +/// ★ The verifier refuses a statement that gives a chunked table (KECCAK, +/// KECCAK_RND, ECSM or ECDAS) a height over its cap, at the frame (before any +/// AIR is built), with the cap's own error; the honest proofs verify. +/// +/// Mutation (box script): `check_chunked_heights` returning `Ok(())`, or the +/// KECCAK and ECSM caps lifted, and the errors are no longer the cap's. +#[test] +fn a_chunked_table_stated_over_its_cap_is_refused() { + let format = many_groups(); + for (program, tables) in [ + ( + "test_ecsm_multi", + [ + ("ECDAS", block_whir::BLOCK_ECDAS_MAX_VARS), + ("ECSM", block_whir::BLOCK_ECSM_MAX_VARS), + ], + ), + ( + "test_keccak", + [ + ("KECCAK_RND", block_whir::BLOCK_KECCAK_RND_MAX_VARS), + ("KECCAK", block_whir::BLOCK_KECCAK_MAX_VARS), + ], + ), + ] { + let elf = asm_elf_bytes(program); + let proof = prove(&elf, &format, &options(MaxRowsConfig::default(), 16)); + assert!(verify(&proof, &elf, &format), "{program} verifies"); + for (name, cap) in tables { + let mut proof = proof.clone(); + let (_, range, _) = block_whir::chunked_table_ranges(&proof.table_counts) + .into_iter() + .find(|(n, ..)| *n == name) + .expect("the table's range"); + assert!(!range.is_empty(), "{program} has a {name} table"); + proof.table_num_vars[range.start] = (cap + 1) as u8; + let refused = block_whir::verify_block_whir( + &proof, + &elf, + &ProofOptions::default_test_options(), + &format, + ); + let msg = format!("{refused:?}"); + assert!( + msg.contains(&format!("{name}[0]")) && msg.contains(&format!("at most 2^{cap}")), + "the frame must refuse {name} over its cap with the cap's error, got {msg}" + ); + println!("BLOCK WHIR {name} OVER CAP: refused at the frame: {msg}"); + } + } +} + +/// ★ W1's readout on a real block (box only, `--ignored`): the block proved +/// in one proof at the production format, its phases stamped, then verified on +/// the host. Reads `BLOCK_WHIR_ELF` and `BLOCK_WHIR_INPUT`; prints the +/// `BLOCK …` lines of [`block_whir::BlockStamps::report`], the host peak and +/// `BLOCK VERIFY`. +#[test] +#[ignore = "a real block: box only"] +fn block_whir_on_a_real_block() { + let elf_path = std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF"); + let input_path = std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT"); + let elf = std::fs::read(&elf_path).expect("read the ELF"); + let input = std::fs::read(&input_path).expect("read the input"); + // `BLOCK_WHIR_PREPARED=0`: the openings off, for the arm that prices them. + let prepared = std::env::var("BLOCK_WHIR_PREPARED").map_or(true, |v| v.trim() != "0"); + // `BLOCK_WHIR_ARGUE=batched|per-table` (production: batched): each group's + // tables argued together, at the format's bin cap (`BLOCK_WHIR_ARGUE_CAP=k` + // for 2^k), or each on its own. + let argue = match std::env::var("BLOCK_WHIR_ARGUE").as_deref().map(str::trim) { + Ok("batched") => ArgueFormat::batched( + std::env::var("BLOCK_WHIR_ARGUE_CAP") + .ok() + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(match ArgueFormat::BATCHED { + ArgueFormat::Batched { bin_log_cells, .. } => bin_log_cells, + ArgueFormat::PerTable => unreachable!("the batched format"), + }), + ), + Ok("per-table") => ArgueFormat::PerTable, + Err(_) => BlockFormat::production().argue, + Ok(other) => panic!("BLOCK_WHIR_ARGUE={other}: per-table or batched"), + }; + // The block's base is its proof family's (D-P3B P3). `BLOCK_WHIR_BASE` + // was this harness's word for it: `p1`/`p1w` are still accepted and change + // nothing (an A/B against an older binary keeps equal environment bytes); + // any other value — `rpx` — is refused rather than ignored. + if let Ok(word) = std::env::var("BLOCK_WHIR_BASE") { + assert!( + matches!(word.trim(), "p1" | "p1w"), + "BLOCK_WHIR_BASE={word}: the block's base is its proof family's (p1)" + ); + } + let production = BlockFormat::production(); + let base = production.zf.base; + let format = BlockFormat { + prepared, + argue, + ..production + }; + println!( + "BLOCK BASE: {:?} · arity-4 cap {} · statement tag {}", + base.hash, + base.arity4_cap, + String::from_utf8_lossy(&block_whir::block_statement_tag(&base)), + ); + let mut options = BlockOptions::production(); + // `BLOCK_WHIR_LAYOUT_WORKERS=n` (production 6; 0 is the inline layout), as + // the tree's harness takes it; `BLOCK_WHIR_DROP_OPS=0`: the builder keeps + // the streamed chunks' ops (production drops them). + if let Some(n) = std::env::var("BLOCK_WHIR_LAYOUT_WORKERS") + .ok() + .and_then(|v| v.trim().parse().ok()) + { + options.layout_workers = n; + } + // `BLOCK_WHIR_LAYOUT_AHEAD=k` bounds the chunks unpacked at k + 1; + // `none` lifts the bound (the reverted version, BIG 390). + match std::env::var("BLOCK_WHIR_LAYOUT_AHEAD") + .as_deref() + .map(str::trim) + { + Ok("none") => options.layout_ahead = None, + Ok(k) => { + if let Ok(k) = k.parse() { + options.layout_ahead = Some(k); + } + } + Err(_) => {} + } + match std::env::var("BLOCK_WHIR_DROP_OPS") + .as_deref() + .map(str::trim) + { + Ok("0") => options.drop_streamed_ops = false, + Ok("1") => options.drop_streamed_ops = true, + _ => {} + } + // `BLOCK_WHIR_NARROW=wide|card|host` (production card): how phase A holds + // each group's columns once committed. + if let Some(narrow) = crate::block_whir::narrow_from_env() { + options.narrow = narrow; + } + // `BLOCK_WHIR_KECCAK_LOG2=k`, `BLOCK_WHIR_ECSM_LOG2=k`: force a KECCAK or + // ECSM split (production 2^18 / 2^17). + crate::block_whir::chunk_cuts_from_env(&mut options); + // `BLOCK_WHIR_UPLOAD_AHEAD=0|1` (production 1): phase A puts each group's + // columns on the card beside the previous group's commit. + if let Some(ahead) = crate::block_whir::upload_ahead_from_env() { + options.upload_ahead = ahead; + } + println!("BLOCK UPLOAD AHEAD: {}", options.upload_ahead); + // `BLOCK_WHIR_REST_LAYOUT=all|` (production 4096), + // `BLOCK_WHIR_KR_FINISH_CHUNKS=0|1` and `BLOCK_WHIR_PACK_FINISHED=0|1` + // (production 1): the rest's layout in waves, KECCAK_RND built as its + // tables, the finish's tables packed as they are built. + crate::block_whir::rest_layout_from_env(&mut options); + println!( + "BLOCK REST LAYOUT CONFIG: {} · KECCAK_RND built as its tables {} · finish packed {}", + crate::block_whir::rest_layout_label(&options, crate::block_whir::rest_slide()), + options.finish_keccak_rnd_chunks, + options.pack_finished, + ); + println!( + "BLOCK CONFIG: group_polys {} · stack {} · keccak_rnd 2^{} · drop {} · prepared {} · layout workers {} (ahead {:?}) · streamed ops dropped {} · argue {:?} · {}", + format.group_polys, + format.zf.whir_stack.get(), + options.keccak_rnd_rows_log2, + options.drop_levels, + if prepared { "on" } else { "off" }, + options.layout_workers, + options.layout_ahead, + if options.drop_streamed_ops { + "on" + } else { + "off" + }, + format.argue, + format.zf.banner(), + ); + // The options #1010's base proves its epochs under. + let opts = crate::lfm::proof::block_base_options(); + // The wall clock at the prove's start, to place a box's memory samples. + println!( + "W3 PROVE START: unix {:.3}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()) + ); + let wall = std::time::Instant::now(); + let (proof, stamps) = block_whir::prove_block_whir(&elf, &input, &opts, &format, &options) + .expect("the block proves"); + let prove_wall = wall.elapsed().as_secs_f64(); + print!("{}", stamps.report()); + // A device that declined any of it would make the readout a host number. + println!( + "BLOCK FALLBACKS: commit/encode to the host {} · device commit errors {} · openings on the host {}", + multilinear::gpu::host_fallbacks(), + multilinear::gpu::commit_errors(), + multilinear::gpu::open_host_fallbacks(), + ); + println!( + "BLOCK ARGUE DEVICE: GKR tree refusals {} · fused sessions {} · fused declines {}", + multilinear::gpu::gkr_tree_refusals(), + multilinear::gpu_fused::fused_sessions(), + multilinear::gpu_fused::fused_declines(), + ); + println!( + "BLOCK PROVE WALL: {prove_wall:.2}s · host peak {:.2} GiB", + host_peak_gib() + ); + let t = std::time::Instant::now(); + let ok = verify_block_whir(&proof, &elf, &opts, &format).expect("the verifier runs"); + println!( + "BLOCK VERIFY: {} in {:.2}s · proof {} tables, {} batched argues, {} groups, {} roots", + if ok { "ACCEPTED" } else { "REJECTED" }, + t.elapsed().as_secs_f64(), + proof.proof.tables.len(), + proof.argues.len(), + proof.proof.columns.len(), + proof.proof.roots.len(), + ); + // The statement's shape, beside the roots: two proves of one block give + // the same digest when they built the same tables and partition (row + // order inside the HashMap-ordered tables aside), whatever their proof + // bytes. + let shape = { + use std::hash::{Hash, Hasher}; + let mut h = std::hash::DefaultHasher::new(); + proof.table_num_vars.hash(&mut h); + proof.groups.hash(&mut h); + format!("{:?}", proof.table_counts).hash(&mut h); + format!("{:?}", proof.runtime_page_ranges).hash(&mut h); + proof.public_output.hash(&mut h); + proof.num_private_input_pages.hash(&mut h); + h.finish() + }; + println!("BLOCK SHAPE DIGEST: {shape:016x}"); + println!( + "BLOCK CHUNKED: {} (cuts KECCAK 2^{} · ECSM 2^{} · COPY 2^{})", + block_whir::chunked_census(&proof.table_counts, &proof.table_num_vars), + options.keccak_rows_log2, + options.ecsm_rows_log2, + options.copy_rows_log2, + ); + // The proof's bytes: two processes prove the same ones under + // LAMBDA_VM_FIXED_TRACE_HASH=1 and LAMBDA_VM_DETERMINISTIC_GRIND=1. + let bytes = rkyv::to_bytes::(&proof).expect("serialize"); + println!( + "BLOCK PROOF DIGEST: {} (blake3 of the {} proof bytes)", + &blake3::hash(&bytes).to_hex()[..32], + bytes.len() + ); + assert!(ok, "the block proof must verify"); +} + +/// The process's peak resident set (`VmHWM`), GiB; 0 where `/proc` is absent. +fn host_peak_gib() -> f64 { + std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status + .lines() + .find(|l| l.starts_with("VmHWM:")) + .and_then(|l| l.split_whitespace().nth(1)) + .and_then(|kb| kb.parse::().ok()) + }) + .map(|kb| kb / (1u64 << 20) as f64) + .unwrap_or(0.0) +} + +/// KECCAK_RND's split takes whole rows: each piece is rows `[k·r, (k+1)·r)` of +/// the table, column for column. +#[test] +fn a_split_table_is_its_rows_in_order() { + use crate::test_utils::{E, F}; + use math::field::element::FieldElement as FE; + let width = 5usize; + let height = 32usize; + let columns: Vec>> = (0..width) + .map(|c| { + (0..height) + .map(|r| FE::::from((r * 100 + c) as u64)) + .collect() + }) + .collect(); + let table = stark::trace::TraceTable::::from_columns_main(columns.clone(), 1); + let pieces = block_whir::split_rows(table, 8); + assert_eq!(pieces.len(), 4); + for (k, piece) in pieces.iter().enumerate() { + let got = piece.columns_main(); + for (c, column) in columns.iter().enumerate() { + assert_eq!( + got[c], + column[k * 8..(k + 1) * 8].to_vec(), + "piece {k} column {c}" + ); + } + } +} + +/// ★ KECCAK_RND filled from its records ([`Deviations::card_fill`], the +/// `LAMBDA_VM_BLOCK_CARD_FILL` knob) moves no byte: the streamed finish holds +/// every KECCAK_RND table as its keccak operations, the card fills each from +/// them at its phase-A and its phase-B upload (on a host without one, a host +/// reader fills it there; at this test's size the commit's own encode is the +/// host's, under the card's floor), and the proof verifies with the same partition, +/// tables and statement, its bytes equal under the deterministic grind. The +/// inline layout's finish is not streamed and holds none. +#[test] +fn keccak_rnd_filled_from_records_moves_no_byte() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + for workers in [0usize, 3] { + let proved = |card_fill: bool| { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = workers; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + card_fill: Some(card_fill), + ..Deviations::default() + }, + ) + .expect("prove"); + assert!( + verify(&proof, &elf, &format), + "card fill {card_fill}, {workers} workers" + ); + (proof, stamps) + }; + let (host, host_stamps) = proved(false); + let (card, stamps) = proved(true); + let what = format!("{workers} workers"); + assert_eq!(host_stamps.layout.records_rest.0, 0, "{what}: knob off"); + assert_eq!(host_stamps.records_fills, (0, 0), "{what}: knob off"); + let tables = card.table_counts.keccak_rnd; + assert!(tables > 1, "{what}: {tables} KECCAK_RND tables"); + let held = stamps.layout.records_rest.0; + if workers == 0 { + assert_eq!(held, 0, "{what}: the inline finish holds none"); + } else { + assert_eq!(held, tables, "{what}: every KECCAK_RND table held"); + assert_eq!(stamps.layout.records_rest.1 % 208, 0, "{what}: 208 B an op"); + let (on_card, on_host) = stamps.records_fills; + if cfg!(feature = "cuda") { + // Each table's group goes up at its commit and at its argument, + // filled on the card each time. At this size a commit is under + // the card's floor (a codeword of 2^12, `gpu::COMMIT_THRESHOLD`) + // and encodes on the host from the host's fill, once a table. + let held = held as u64; + assert!( + (held..=2 * held).contains(&on_card) && on_host <= held, + "{what}: {on_card} on the card, {on_host} on the host, not each on the \ + card at both its uploads" + ); + } else { + assert_eq!( + (on_card, on_host), + (0, held as u64), + "{what}: each filled on the host once" + ); + } + } + assert_eq!(host.groups, card.groups, "{what}: the partition"); + assert_eq!(host.table_num_vars, card.table_num_vars, "{what}"); + assert_eq!( + format!("{:?}", host.table_counts), + format!("{:?}", card.table_counts), + "{what}" + ); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&host), bytes(&card), "{what}: the proof"); + } + } +} + +/// ★ A KECCAK_RND table held as its records is never dropped: with live +/// regeneration dropping every table it can, the rest's regenerator builds +/// LT's tables again and no KECCAK_RND table (it builds both without the +/// records), and the proof is the one without regeneration. +#[test] +fn keccak_rnd_held_as_records_is_never_dropped() { + use crate::block_whir::regen::RegenMode; + + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let card = |card_fill: bool| Deviations { + card_fill: Some(card_fill), + ..Deviations::default() + }; + let (plain, _) = regenerated(&elf, &format, RegenMode::Off, card(true)).expect("prove"); + let rest_line = |stamps: &crate::block_whir::BlockStamps| { + let r = stamps.regen.as_ref().expect("live regeneration's readout"); + assert_eq!((r.mismatches, r.failed), (0, 0), "{:?}", r.lines); + assert_eq!(r.rest_regenerated, r.rest_dropped, "{:?}", r.lines); + assert!(r.rest_dropped > 0, "no rest table dropped: {:?}", r.lines); + r.lines + .iter() + .find(|l| l.starts_with("BLOCK REGEN rest: ")) + .cloned() + .expect("the rest's line") + }; + let (built, stamps) = + regenerated(&elf, &format, RegenMode::Always, card(false)).expect("prove"); + let line = rest_line(&stamps); + assert!( + !line.contains("KECCAK_RND 0 ·"), + "built, so dropped: {line}" + ); + assert!(verify(&built, &elf, &format)); + let (held, stamps) = regenerated(&elf, &format, RegenMode::Always, card(true)).expect("prove"); + assert!(stamps.layout.records_rest.0 > 0, "no table held as records"); + let line = rest_line(&stamps); + assert!( + line.contains("KECCAK_RND 0 · LT "), + "held, never dropped: {line}" + ); + assert!(!line.contains("LT 0 built"), "LT dropped and back: {line}"); + assert_eq!(held.groups, plain.groups, "the partition"); + assert!(verify(&held, &elf, &format)); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&held), bytes(&plain), "the proof"); + assert_eq!(bytes(&built), bytes(&plain), "the proof"); + } +} + +/// ★ A wrong KECCAK_RND table held as records is refused by the verifier: with +/// its records starting a row late ([`Deviations::card_fill_records_fault`]), +/// every fill of them (the card's at both uploads, or a host reader's) writes +/// the same wrong table, so the prover's own checks of what it committed pass +/// and it makes a proof, and the unchanged verifier refuses it. (A fill that +/// differs between the phases is the prover's to refuse first: its opening +/// re-hashes what phase A committed.) +#[test] +fn a_wrong_records_table_is_refused_by_the_verifier() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = 3; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + card_fill: Some(true), + card_fill_records_fault: 1, + ..Deviations::default() + }, + ) + .unwrap_or_else(|e| panic!("the prover refused ({e:?}): the verifier was never asked")); + assert!( + stamps.layout.records_rest.0 > 0, + "no KECCAK_RND table held as records" + ); + assert!( + !verify(&proof, &elf, &format), + "a proof over a wrong KECCAK_RND table verified" + ); +} + +/// Every streamed CPU and MEMW_R chunk held as records +/// ([`Deviations::card_fill_streamed`]), no row family's (whatever the +/// knob's default), the room rule admitting every one. +fn streamed_records(cpu: bool, memw_r: bool) -> Deviations { + Deviations { + card_fill_streamed: Some([cpu, memw_r]), + card_fill_rows: Some(RowFamilies::NONE), + records_room: Some(std::sync::Arc::new(|_| true)), + ..Deviations::default() + } +} + +/// ★ CPU and MEMW_R chunks filled from their records (the +/// `LAMBDA_VM_BLOCK_CARD_FILL` knob's `cpu` and `memw_r`) move no byte: every +/// full streamed chunk of those families is handed out as its logs or rows, +/// the card fills each from them at its phase-A and its phase-B upload (on a +/// host without one, a host reader fills it there), and the proof verifies +/// with the same partition, tables and statement, its bytes equal under the +/// deterministic grind; each family alone, both, and both beside KECCAK_RND's; +/// laid out inline and on workers; for programs whose chunks lie in several +/// windows. +#[test] +fn cpu_and_memw_r_filled_from_records_move_no_byte() { + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + for name in ["all_instructions_64", "test_keccak_multi"] { + let elf = asm_elf_bytes(name); + for workers in [0usize, 3] { + let proved = |deviations: Deviations| { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = workers; + o.drop_streamed_ops = true; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &deviations, + ) + .expect("prove"); + assert!(verify(&proof, &elf, &format), "{name}, {workers} workers"); + (proof, stamps) + }; + let (host, host_stamps) = proved(Deviations { + card_fill: Some(false), + card_fill_streamed: Some([false, false]), + ..Deviations::default() + }); + let off = host_stamps.windows.card_fill; + assert_eq!( + (off.cpu, off.memw_r, off.records_bytes), + ([0, 0], [0, 0], 0), + "knob off" + ); + assert_eq!(host_stamps.records_fills, (0, 0), "knob off"); + for (cpu, memw_r, keccak_rnd) in [ + (true, false, false), + (false, true, false), + (true, true, false), + (true, true, true), + ] { + let what = format!( + "{name}, {workers} workers, cpu {cpu} memw_r {memw_r} keccak_rnd {keccak_rnd}" + ); + // Regeneration is off here, and parked records need it: kept. + let (card, stamps) = proved(Deviations { + card_fill: Some(keccak_rnd), + records_keep: Some(true), + ..streamed_records(cpu, memw_r) + }); + let c = stamps.windows.card_fill; + assert_eq!(c.out_of_step, 0, "{what}: {c:?}"); + assert_eq!(c.room_out, None, "{what}: {c:?}"); + assert_eq!((c.cpu[1], c.memw_r[1]), (0, 0), "{what}: {c:?}"); + assert_eq!(c.cpu[0] > 0, cpu, "{what}: {c:?}"); + assert_eq!(c.memw_r[0] > 0, memw_r, "{what}: {c:?}"); + let held = (c.cpu[0] + c.memw_r[0] + stamps.layout.records_rest.0) as u64; + let (on_card, on_host) = stamps.records_fills; + if cfg!(feature = "cuda") { + // At its commit and at its argument on the card; at this + // size a commit under the card's floor encodes on the host + // from the host's fill, once a table. + assert!( + (held..=2 * held).contains(&on_card) && on_host <= held, + "{what}: {on_card} on the card, {on_host} on the host, of {held} held" + ); + } else { + assert_eq!( + (on_card, on_host), + (0, held), + "{what}: each on the host once" + ); + } + assert_eq!(host.groups, card.groups, "{what}: the partition"); + assert_eq!(host.table_num_vars, card.table_num_vars, "{what}"); + assert_eq!( + format!("{:?}", host.table_counts), + format!("{:?}", card.table_counts), + "{what}" + ); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&host), bytes(&card), "{what}: the proof"); + } + } + } + } +} + +/// ★ Counting the hand-out on a thread of its own (the +/// `LAMBDA_VM_HANDOUT_COUNT_THREAD` knob, [`Deviations::handout_count_thread`]) +/// moves no byte: the streamed block, its ops dropped as their chunks leave, +/// proved with each chunk's counts taken on the accumulator's thread and on a +/// thread of its own verifies with the same partition, tables and statement, +/// its bytes equal under the deterministic grind; with the CPU and MEMW_R +/// chunks held as records or filled on the host, laid out inline and on +/// workers. The thread ran only where asked, and counted chunks. +#[test] +fn counting_the_hand_out_on_its_own_thread_moves_no_byte() { + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + for name in ["all_instructions_64", "test_keccak_multi"] { + let elf = asm_elf_bytes(name); + for workers in [0usize, 3] { + for records in [false, true] { + let what = format!("{name}, {workers} workers, records {records}"); + let proved = |on_thread: bool| { + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = workers; + o.drop_streamed_ops = true; + let base = if records { + Deviations { + records_keep: Some(true), + ..streamed_records(true, true) + } + } else { + Deviations { + card_fill_streamed: Some([false, false]), + card_fill_rows: Some(RowFamilies::NONE), + ..Deviations::default() + } + }; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + handout_count_thread: Some(on_thread), + ..base + }, + ) + .expect("prove"); + assert!(verify(&proof, &elf, &format), "{what}, thread {on_thread}"); + match (on_thread, stamps.windows.count_thread) { + (false, None) => {} + (true, Some(c)) => assert!( + c.chunks.iter().sum::() > 0, + "{what}: the thread counted no chunk" + ), + (on, c) => panic!("{what}: thread {on}, stamps {c:?}"), + } + proof + }; + let inline = proved(false); + let thread = proved(true); + assert_eq!(inline.groups, thread.groups, "{what}: the partition"); + assert_eq!(inline.table_num_vars, thread.table_num_vars, "{what}"); + assert_eq!( + format!("{:?}", inline.table_counts), + format!("{:?}", thread.table_counts), + "{what}" + ); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&inline), bytes(&thread), "{what}: the proof"); + } + } + } + } +} + +/// ★ With records kept (`LAMBDA_VM_BLOCK_RECORDS_DROP=keep`), a streamed chunk +/// held as its records is never dropped and no regenerator waits on it: with +/// live regeneration dropping every table it can (the streamed chunks' +/// regenerator started as early as their fates are known), every dropped +/// chunk comes back with its digest, none is missing, fewer streamed chunks +/// drop than without the records, and the proof is the one without +/// regeneration. +#[test] +fn streamed_records_chunks_kept_are_never_dropped() { + use crate::block_whir::regen::RegenMode; + + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + // Records without regeneration: kept (parked records need live + // regeneration). + let (plain, _) = regenerated( + &elf, + &format, + RegenMode::Off, + Deviations { + records_keep: Some(true), + ..streamed_records(true, true) + }, + ) + .expect("prove"); + let readout = |stamps: &crate::block_whir::BlockStamps| { + let r = stamps.regen.clone().expect("live regeneration's readout"); + assert_eq!( + (r.mismatches, r.failed, r.missing), + (0, 0, 0), + "{:?}", + r.lines + ); + assert_eq!(r.regenerated, r.dropped, "{:?}", r.lines); + assert!(r.error.is_none(), "{:?}", r.lines); + r + }; + let (built, stamps) = + regenerated(&elf, &format, RegenMode::Always, Deviations::default()).expect("prove"); + let all = readout(&stamps); + assert!(verify(&built, &elf, &format)); + let (held, stamps) = regenerated( + &elf, + &format, + RegenMode::Always, + Deviations { + records_keep: Some(true), + ..streamed_records(true, true) + }, + ) + .expect("prove"); + let c = stamps.windows.card_fill; + assert!(c.cpu[0] > 0 && c.memw_r[0] > 0, "{c:?}"); + let some = readout(&stamps); + let streamed = |r: &crate::block_whir::regen::RegenStamps| r.dropped - r.rest_dropped; + assert!( + streamed(&some) < streamed(&all), + "records chunks dropped: {} of {} without records", + streamed(&some), + streamed(&all) + ); + assert_eq!( + some.recipes + c.cpu[0] + c.memw_r[0], + all.recipes, + "no recipe for a records chunk" + ); + assert_eq!(held.groups, plain.groups, "the partition"); + assert!(verify(&held, &elf, &format)); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&held), bytes(&plain), "the proof"); + assert_eq!(bytes(&built), bytes(&plain), "the proof"); + } +} + +/// ★ A wrong CPU table held as records is refused by the verifier: with each +/// log's register values changed in the records +/// ([`Deviations::card_fill_streamed_fault`]), every fill of them (the card's +/// at both uploads, or a host reader's) writes the same wrong table, so the +/// prover's own checks of what it committed pass and it makes a proof, and the +/// unchanged verifier refuses it. The fault is shown to bite first: against +/// the same prove without it, the partition is the same and the committed +/// group roots move (a fault the tables do not see would leave them equal, and +/// the refusal below would be vacuous). +#[test] +fn a_wrong_cpu_records_table_is_refused_by_the_verifier() { + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let mut o = streamed(MaxRowsConfig::small(), 5, 3); + o.layout_workers = 3; + o.drop_streamed_ops = true; + let proved = |fault: u64| { + prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + card_fill_streamed_fault: fault, + // Regeneration is off here, and parked records need it: kept. + records_keep: Some(true), + ..streamed_records(true, false) + }, + ) + .unwrap_or_else(|e| { + panic!("fault {fault}: the prover refused ({e:?}): the verifier was never asked") + }) + }; + let (honest, _) = proved(0); + assert!(verify(&honest, &elf, &format), "the honest proof"); + let (proof, stamps) = proved(1); + assert!( + stamps.windows.card_fill.cpu[0] > 0, + "no CPU chunk held as records" + ); + assert_eq!(proof.groups, honest.groups, "the partition"); + assert_ne!( + proof.proof.roots, honest.proof.roots, + "the fault left every committed table as it was: the refusal below would be vacuous" + ); + assert!( + !verify(&proof, &elf, &format), + "a proof over a wrong CPU table verified" + ); +} + +/// ★ Parked records need live regeneration: with a store and regeneration off, +/// or regeneration off and no store, not one streamed chunk is held as its +/// records, even with the room rule admitting every one: the streamed card +/// fill is never set up, so no chunk is even handed to it, and each is built +/// packed on the host, droppable as today. Records kept on purpose +/// (`LAMBDA_VM_BLOCK_RECORDS_DROP=keep`) are still held there: the A/B +/// control. The proofs are one. +#[test] +fn parked_records_need_live_regeneration() { + use crate::block_whir::regen::RegenMode; + + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let (held, stamps) = + regenerated(&elf, &format, RegenMode::Off, streamed_records(true, true)).expect("prove"); + let c = stamps.windows.card_fill; + assert_eq!( + (c.cpu, c.memw_r, c.records_bytes), + ([0, 0], [0, 0], 0), + "chunks handed to the streamed card fill with no live regeneration: {c:?}" + ); + assert!(verify(&held, &elf, &format)); + let (kept, stamps) = regenerated( + &elf, + &format, + RegenMode::Off, + Deviations { + records_keep: Some(true), + ..streamed_records(true, true) + }, + ) + .expect("prove"); + let c = stamps.windows.card_fill; + assert!(c.cpu[0] > 0 && c.memw_r[0] > 0, "kept: {c:?}"); + assert!(verify(&kept, &elf, &format)); + assert_eq!(held.groups, kept.groups, "the partition"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&held), bytes(&kept), "the proof"); + } +} + +/// ★ A streamed chunk held as its records parks and drops like any chunk (the +/// default, `LAMBDA_VM_BLOCK_RECORDS_DROP=park`): with live regeneration +/// dropping every table it can, as many streamed chunks drop as without the +/// records, those held as records dropped as them; each comes back as packed +/// columns checked by its rebuilt records' digest, none is missing or +/// mismatched, and the proof is the one without regeneration and the one +/// without the records. Without regeneration's drops, the parked records come +/// back as they were, and phase B lets go of every records table after its +/// group's openings; with no spill (no regeneration) none is let go. +#[test] +fn streamed_records_chunks_drop_as_records_and_come_back() { + use crate::block_whir::regen::RegenMode; + + let elf = asm_elf_bytes("test_keccak_multi"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let readout = |stamps: &crate::block_whir::BlockStamps| { + let r = stamps.regen.clone().expect("live regeneration's readout"); + assert_eq!( + (r.mismatches, r.failed, r.missing), + (0, 0, 0), + "{:?}", + r.lines + ); + assert_eq!(r.regenerated, r.dropped, "{:?}", r.lines); + assert!(r.error.is_none(), "{:?}", r.lines); + r + }; + let as_records = |r: &crate::block_whir::regen::RegenStamps| -> usize { + r.lines + .iter() + .find_map(|l| { + let at = l.find(" dropped as their records")?; + l[..at].rsplit(' ').next()?.parse().ok() + }) + .expect("the dropped line counts the records") + }; + let streamed = |r: &crate::block_whir::regen::RegenStamps| r.dropped - r.rest_dropped; + let released = |stamps: &crate::block_whir::BlockStamps| -> usize { + stamps.groups.iter().map(|g| g.records_released).sum() + }; + // Records without regeneration: kept (parked records need live + // regeneration). + let (plain, stamps) = regenerated( + &elf, + &format, + RegenMode::Off, + Deviations { + records_keep: Some(true), + ..streamed_records(true, true) + }, + ) + .expect("prove"); + assert!(stamps.windows.card_fill.records_bytes > 0); + assert_eq!(released(&stamps), 0, "no spill: no records let go"); + let (built, stamps) = + regenerated(&elf, &format, RegenMode::Always, Deviations::default()).expect("prove"); + let all = readout(&stamps); + assert_eq!(as_records(&all), 0, "{:?}", all.lines); + let (parked, stamps) = regenerated( + &elf, + &format, + RegenMode::Always, + streamed_records(true, true), + ) + .expect("prove"); + let c = stamps.windows.card_fill; + assert!(c.cpu[0] > 0 && c.memw_r[0] > 0, "{c:?}"); + let some = readout(&stamps); + assert_eq!( + streamed(&some), + streamed(&all), + "every streamed chunk drops" + ); + assert_eq!(some.recipes, all.recipes, "a recipe for every chunk"); + assert_eq!( + as_records(&some), + c.cpu[0] + c.memw_r[0], + "every records chunk dropped as its records: {:?}", + some.lines + ); + // Only the rest's records tables (KECCAK_RND's, where the card fills + // them) can be let go: every streamed one dropped before phase B. + assert!( + released(&stamps) <= stamps.layout.records_rest.1, + "a streamed records chunk let go in phase B, yet all were dropped" + ); + assert!(verify(&parked, &elf, &format)); + // Live regeneration that never arms: every records chunk parks and comes + // back as it was. `auto` arms once the host's bytes pass its target, and + // the walk's end hands off (and arms) once the peak passes its pressure + // share: a host short of room (a CI runner's) arms at once. The target is + // out of reach, so no host's memory decides it. + let (never, stamps) = regenerated( + &elf, + &format, + RegenMode::Auto, + Deviations { + memory_target: Some(u64::MAX), + ..streamed_records(true, true) + }, + ) + .expect("prove"); + let unarmed = readout(&stamps); + assert!( + unarmed.dropped == 0 + && unarmed + .lines + .iter() + .any(|l| l.starts_with("BLOCK REGEN dropped: Auto") && l.contains(" never armed ")), + "{:?}", + unarmed.lines + ); + assert!(stamps.windows.card_fill.cpu[0] > 0); + assert_eq!( + released(&stamps), + stamps.windows.card_fill.records_bytes + stamps.layout.records_rest.1, + "every records table let go after its group's openings" + ); + // Their lists' pages handed back on a thread of their own: some list let + // go by its last holder, none refused. (These windows are a few cycles, + // so a list holds no whole page: the bytes are a real block's to show.) + let mut pages = multilinear::records::PagesReturned::default(); + for g in &stamps.groups { + pages += g.records_pages; + } + assert!(pages.buffers > 0 && pages.failed == 0, "{pages:?}"); + assert!(verify(&never, &elf, &format)); + assert_eq!(parked.groups, plain.groups, "the partition"); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&parked), bytes(&plain), "the proof"); + assert_eq!(bytes(&built), bytes(&plain), "the proof"); + assert_eq!(bytes(&never), bytes(&plain), "the proof"); + } +} + +/// The chunk heights the row families' records tests run at: chunks of four +/// rows, so that the small programs fill several. +fn rows_max() -> MaxRowsConfig { + MaxRowsConfig { + memw_aligned: 1 << 2, + load: 1 << 2, + store: 1 << 2, + memw: 1 << 2, + shift: 1 << 2, + ..MaxRowsConfig::small() + } +} + +/// Every streamed chunk of the row families in `rows` held as its rows +/// ([`Deviations::card_fill_rows`]), no CPU or MEMW_R chunk, the room rule +/// admitting every one. +fn row_records(rows: RowFamilies) -> Deviations { + Deviations { + card_fill_rows: Some(rows), + ..streamed_records(false, false) + } +} + +/// The row family alone whose name ([`ROW_NAMES`]) is `name`. +fn row_family(name: &str) -> RowFamilies { + let mut rows = RowFamilies::NONE; + let flag = match name { + "memw_a" => &mut rows.memw_a, + "load" => &mut rows.load, + "store" => &mut rows.store, + "memw" => &mut rows.memw, + "shift" => &mut rows.shift, + _ => panic!("no row family {name}"), + }; + *flag = true; + rows +} + +/// The row families' names, in [`row_counts`]' order. +const ROW_NAMES: [&str; 5] = ["memw_a", "load", "store", "memw", "shift"]; + +/// Each row family's `[as records, on the host]`, in [`ROW_NAMES`]' order. +fn row_counts(c: &crate::tables::trace_builder::CardFillStamps) -> [[usize; 2]; 5] { + [c.memw_a, c.load, c.store, c.memw, c.shift] +} + +/// ★ The row families' chunks filled from their rows (the +/// `LAMBDA_VM_BLOCK_CARD_FILL` knob's `memw_a`, `load`, `store`, `memw` and +/// `shift`) move no byte: every full streamed chunk of those families is +/// handed out as its rows, the card fills each from them at its phase-A and +/// its phase-B upload (on a host without one, a host reader fills it there), +/// and the proof verifies with the same partition, tables and statement, its +/// bytes equal under the deterministic grind; the row families alone and +/// beside every other family; chunks of four rows, from a program that loads +/// and stores at every width (laid out inline and on workers) and one that +/// stores and shifts. (No small program makes four MEMW ops; a real block's +/// MEMW chunks are the card test's and the gate's.) +#[test] +fn row_families_filled_from_their_rows_move_no_byte() { + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + // Each program, its layouts, and the families it fills a chunk of. + for (name, layouts, fills) in [ + ( + "all_loadstore_32", + &[0usize, 3][..], + &["memw_a", "load", "store"][..], + ), + ( + "test_ecsm_multi", + &[3][..], + &["memw_a", "store", "shift"][..], + ), + ] { + let elf = asm_elf_bytes(name); + for &workers in layouts { + let proved = |deviations: Deviations| { + let mut o = streamed(rows_max(), 5, 3); + o.layout_workers = workers; + o.drop_streamed_ops = true; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &deviations, + ) + .expect("prove"); + assert!(verify(&proof, &elf, &format), "{name}, {workers} workers"); + (proof, stamps) + }; + let (host, host_stamps) = proved(Deviations { + card_fill: Some(false), + card_fill_streamed: Some([false, false]), + card_fill_rows: Some(RowFamilies::NONE), + ..Deviations::default() + }); + let off = host_stamps.windows.card_fill; + assert!( + row_counts(&off).iter().all(|f| *f == [0, 0]) && off.records_bytes == 0, + "knob off: {off:?}" + ); + assert_eq!(host_stamps.records_fills, (0, 0), "knob off"); + for rest in [false, true] { + let what = format!("{name}, {workers} workers, the rest {rest}"); + // Regeneration is off here, and parked records need it: kept. + let (card, stamps) = proved(Deviations { + card_fill: Some(rest), + card_fill_streamed: Some([rest, rest]), + records_keep: Some(true), + ..row_records(RowFamilies::ALL) + }); + let c = stamps.windows.card_fill; + assert_eq!(c.room_out, None, "{what}: {c:?}"); + for (family, counts) in ROW_NAMES.iter().zip(row_counts(&c)) { + assert_eq!(counts[1], 0, "{what}: {family} on the host: {c:?}"); + assert_eq!( + counts[0] > 0, + fills.contains(family), + "{what}: {family} held: {c:?}" + ); + } + let rows: usize = row_counts(&c).iter().map(|f| f[0]).sum(); + let held = (c.cpu[0] + c.memw_r[0] + rows + stamps.layout.records_rest.0) as u64; + let (on_card, on_host) = stamps.records_fills; + if cfg!(feature = "cuda") { + // At its commit and at its argument on the card; at this + // size a commit under the card's floor encodes on the host + // from the host's fill, once a table. + assert!( + (held..=2 * held).contains(&on_card) && on_host <= held, + "{what}: {on_card} on the card, {on_host} on the host, of {held} held" + ); + } else { + assert_eq!( + (on_card, on_host), + (0, held), + "{what}: each on the host once" + ); + } + assert_eq!(host.groups, card.groups, "{what}: the partition"); + assert_eq!(host.table_num_vars, card.table_num_vars, "{what}"); + assert_eq!( + format!("{:?}", host.table_counts), + format!("{:?}", card.table_counts), + "{what}" + ); + if crypto::grinding::deterministic() { + assert_eq!(bytes(&host), bytes(&card), "{what}: the proof"); + } + } + } + } +} + +/// ★ The row families' chunks held as their rows park and drop like any +/// chunk: with live regeneration dropping every table it can, as many +/// streamed chunks drop as without the records, those held as rows dropped +/// as them, each come back as packed columns checked by its rebuilt rows' +/// digest, none missing or mismatched. Kept +/// (`LAMBDA_VM_BLOCK_RECORDS_DROP=keep`), none drops and none has a recipe; +/// with no live regeneration and not kept, none is held. The proofs are the +/// one without regeneration. (MEMW_A, LOAD and STORE chunks: the rebuild is +/// the families' one path.) With no live regeneration and not kept, the +/// streamed card fill is never set up: no chunk is handed to it. +#[test] +fn row_records_chunks_drop_as_records_and_come_back() { + use crate::block_whir::regen::{RegenMode, RegenStamps}; + + let elf = asm_elf_bytes("all_loadstore_32"); + let format = many_groups(); + let bytes = |p: &BlockWhirProof| { + rkyv::to_bytes::(p) + .expect("serialize") + .to_vec() + }; + let readout = |stamps: &crate::block_whir::BlockStamps| { + let r = stamps.regen.clone().expect("live regeneration's readout"); + assert_eq!( + (r.mismatches, r.failed, r.missing), + (0, 0, 0), + "{:?}", + r.lines + ); + assert_eq!(r.regenerated, r.dropped, "{:?}", r.lines); + assert!(r.error.is_none(), "{:?}", r.lines); + r + }; + let as_records = |r: &RegenStamps| -> usize { + r.lines + .iter() + .find_map(|l| { + let at = l.find(" dropped as their records")?; + l[..at].rsplit(' ').next()?.parse().ok() + }) + .expect("the dropped line counts the records") + }; + let streamed = |r: &RegenStamps| r.dropped - r.rest_dropped; + let held_rows = |c: &crate::tables::trace_builder::CardFillStamps| -> [usize; 5] { + row_counts(c).map(|f| f[0]) + }; + let proved = |mode, deviations| { + regenerated_at(&elf, &format, mode, deviations, rows_max()).expect("prove") + }; + let kept = |deviations: Deviations| Deviations { + records_keep: Some(true), + ..deviations + }; + let all_rows = || row_records(RowFamilies::ALL); + let (plain, _) = proved(RegenMode::Off, kept(all_rows())); + let (built, stamps) = proved(RegenMode::Always, Deviations::default()); + let all = readout(&stamps); + assert_eq!(as_records(&all), 0, "{:?}", all.lines); + // Parked, the default. + let (parked, stamps) = proved(RegenMode::Always, all_rows()); + let c = stamps.windows.card_fill; + let rows = held_rows(&c); + assert!(rows[0] > 0 && rows[1] > 0 && rows[2] > 0, "{c:?}"); + let some = readout(&stamps); + assert_eq!( + streamed(&some), + streamed(&all), + "every streamed chunk drops" + ); + assert_eq!(some.recipes, all.recipes, "a recipe for every chunk"); + assert_eq!( + as_records(&some), + rows.iter().sum::(), + "every records chunk dropped as its records: {:?}", + some.lines + ); + assert!(verify(&parked, &elf, &format)); + // Kept: never dropped, no recipe. + let (held, stamps) = proved(RegenMode::Always, kept(all_rows())); + let c = stamps.windows.card_fill; + let rows = held_rows(&c); + assert!(rows[0] > 0 && rows[1] > 0 && rows[2] > 0, "kept: {c:?}"); + let none = readout(&stamps); + assert!( + streamed(&none) < streamed(&all), + "kept records chunks dropped: {} of {} without records", + streamed(&none), + streamed(&all) + ); + assert_eq!( + none.recipes + rows.iter().sum::(), + all.recipes, + "no recipe for a kept records chunk" + ); + assert!(verify(&held, &elf, &format)); + // No live regeneration, not kept: none held. + let (unheld, stamps) = proved(RegenMode::Off, all_rows()); + // The streamed card fill is never set up, so no chunk is even handed to + // it: every count stays zero. + let c = stamps.windows.card_fill; + assert!( + row_counts(&c).iter().all(|f| *f == [0, 0]) && c.records_bytes == 0, + "chunks handed to the streamed card fill with no live regeneration: {c:?}" + ); + assert!(verify(&unheld, &elf, &format)); + assert_eq!(parked.groups, plain.groups, "the partition"); + if crypto::grinding::deterministic() { + for (what, proof) in [ + ("parked", &parked), + ("kept", &held), + ("not held", &unheld), + ("no records", &built), + ] { + assert_eq!(bytes(proof), bytes(&plain), "{what}: the proof"); + } + } +} + +/// ★ A wrong row family's table held as rows is refused by the verifier: with +/// each row a cycle later (a SHIFT op's input moved by one; +/// [`Deviations::card_fill_streamed_fault`]), every fill of them (the card's +/// at both uploads, or a host reader's) writes the same wrong table, so the +/// prover makes a proof, and the unchanged verifier refuses it. Each family +/// alone that a small program fills. +#[test] +fn a_wrong_row_records_table_is_refused_by_the_verifier() { + let format = many_groups(); + for (name, family) in [ + ("all_loadstore_32", "memw_a"), + ("all_loadstore_32", "load"), + ("all_loadstore_32", "store"), + ("test_ecsm_multi", "shift"), + ] { + let elf = asm_elf_bytes(name); + let what = format!("{name}, {family}"); + let mut o = streamed(rows_max(), 5, 3); + o.layout_workers = 3; + o.drop_streamed_ops = true; + let (proof, stamps) = prove_block_whir_with( + &elf, + &[], + &ProofOptions::default_test_options(), + &format, + &o, + &Deviations { + card_fill_streamed_fault: 1, + // Regeneration is off here, and parked records need it: kept. + records_keep: Some(true), + ..row_records(row_family(family)) + }, + ) + .unwrap_or_else(|e| { + panic!("{what}: the prover refused ({e:?}): the verifier was never asked") + }); + let c = stamps.windows.card_fill; + let at = ROW_NAMES + .iter() + .position(|n| *n == family) + .expect("a family"); + assert!( + row_counts(&c)[at][0] > 0, + "{what}: no chunk held as rows: {c:?}" + ); + assert!( + !verify(&proof, &elf, &format), + "{what}: a proof over a wrong table verified" + ); + } +} diff --git a/prover/src/tests/branch_bus_tests.rs b/prover/src/tests/branch_bus_tests.rs index 2d1520232..799faeea3 100644 --- a/prover/src/tests/branch_bus_tests.rs +++ b/prover/src/tests/branch_bus_tests.rs @@ -6,12 +6,10 @@ //! - Padding: Auto-padding to power of 2 works correctly //! - Border cases: Edge values (0, MAX, signed boundaries) work +use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; -use math::field::element::FieldElement; - use stark::lookup::{ AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, LinearTerm, Multiplicity, NullBoundaryConstraintBuilder, Packing, @@ -19,7 +17,7 @@ use stark::lookup::{ use stark::proof::options::ProofOptions; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::branch::{BranchOperation, cols, generate_branch_trace}; use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField}; @@ -336,15 +334,15 @@ fn prove_and_verify(ops: &[BranchOperation]) -> bool { ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } @@ -426,15 +424,15 @@ fn prove_and_verify_custom(ops: &[BranchOperation], receiver_rows: &[CustomBranc ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/branch_carrier_poc.rs b/prover/src/tests/branch_carrier_poc.rs index 02d07ae9f..0f9fece44 100644 --- a/prover/src/tests/branch_carrier_poc.rs +++ b/prover/src/tests/branch_carrier_poc.rs @@ -16,7 +16,6 @@ use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use stark::lookup::{ @@ -26,7 +25,7 @@ use stark::lookup::{ use stark::proof::options::ProofOptions; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::branch::{self, BranchOperation, generate_branch_trace}; use crate::tables::lt::{self, LtOperation}; @@ -258,7 +257,7 @@ fn prove_and_verify( (&bytes_air, &mut bytes_trace, &()), (&balu_air, &mut balu_trace, &()), ]; - let Ok(proof) = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) else { + let Ok(proof) = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) else { panic!("prover failed: the harness must produce a proof for the verifier to judge"); }; let airs: Vec<&dyn AIR> = vec![ @@ -271,10 +270,10 @@ fn prove_and_verify( &bytes_air, &balu_air, ]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/branch_multiplicity_poc.rs b/prover/src/tests/branch_multiplicity_poc.rs index 39cfcd4a5..d5452e2b4 100644 --- a/prover/src/tests/branch_multiplicity_poc.rs +++ b/prover/src/tests/branch_multiplicity_poc.rs @@ -17,7 +17,6 @@ use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use stark::lookup::{ @@ -27,7 +26,7 @@ use stark::lookup::{ use stark::proof::options::ProofOptions; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::branch::{BranchOperation, bus_interactions, cols, generate_branch_trace}; use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField}; @@ -224,17 +223,17 @@ fn prove_and_verify(rows: &[Row], victim: Option) -> bool { for (a, t) in airs.iter().zip(traces.iter_mut()) { pairs.push((a, t, &())); } - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let mut air_refs: Vec<&dyn AIR> = vec![&branch_air]; for a in &airs { air_refs.push(a); } - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &air_refs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/commit_ood_host_tests.rs b/prover/src/tests/commit_ood_host_tests.rs new file mode 100644 index 000000000..ead5498fd --- /dev/null +++ b/prover/src/tests/commit_ood_host_tests.rs @@ -0,0 +1,83 @@ +//! The commit-time answers read on the host from tables held narrow +//! (`stacked_eval::ood_claims`): each column's value comes from its table's +//! one host widen, which the evaluation must not enter from inside its own +//! parallel map. + +use std::sync::mpsc; +use std::time::Duration; + +use multilinear::constraint_argument::{FactorKind, TraceData}; +use multilinear::narrow::{ColumnOf, NarrowColumns}; +use multilinear::stacking::StackedLayout; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// Columns a narrow table holds: enough that the evaluation's jobs for it +/// outnumber the workers. +const COLUMNS: usize = 64; +const NUM_VARS: usize = 10; +/// Fresh tables, each evaluated once. +const ROUNDS: usize = 16; + +fn narrow_table(seed: u64) -> TraceData { + let words: Vec> = (0..COLUMNS as u64) + .map(|c| { + (0..1u64 << NUM_VARS) + .map(|r| { + (r.wrapping_mul(0x9e37_79b9) ^ (c + seed).wrapping_mul(0x85eb_ca6b)) % 65_521 + }) + .collect() + }) + .collect(); + let raw: Vec<&[u64]> = words.iter().map(Vec::as_slice).collect(); + let packed = NarrowColumns::pack(&raw).expect("one height"); + let kinds = (0..COLUMNS).map(FactorKind::direct).collect(); + TraceData::new_narrow(packed, kinds, Vec::new()).expect("a narrow table") +} + +/// ★ Every column of a table held narrow answers on the host, each the value +/// of its widened column at its point, and the evaluation ends: the widen runs +/// once, and no job of the evaluation waits on it from inside the pool. Bounded +/// by a timeout, so a regression fails instead of hanging the run. +#[test] +fn a_narrow_tables_answers_end_and_match_its_widened_columns() { + let (done, finished) = mpsc::channel(); + std::thread::spawn(move || { + for round in 0..ROUNDS { + let table = narrow_table(round as u64); + let handles: Vec>> = (0..COLUMNS) + .map(|index| ColumnOf { + columns: &table, + index, + }) + .collect(); + let columns: Vec<&ColumnOf<'_, TraceData>> = handles.iter().collect(); + let layout = + StackedLayout::build(&[NUM_VARS; COLUMNS], NUM_VARS + 6).expect("the layout packs"); + let z0 = FEE::new([FE::from(7 + round as u64), FE::from(11), FE::from(13)]); + let (claims, counts) = + multilinear::stacked_eval::ood_claims::(&layout, &columns, None, &z0) + .expect("the answers"); + assert_eq!(counts.on_host, COLUMNS, "no card: every column on the host"); + let point = multilinear::whir_chain::ood_point(&z0, layout.n_stack()); + let widened = table.columns(); + for (c, value) in claims.columns.iter().enumerate() { + let at = multilinear::commit_ood::column_point(&layout, c, &point); + assert_eq!( + *value, + widened[c] + .evaluate_in(at) + .expect("a column takes its point"), + "round {round} column {c}" + ); + } + } + let _ = done.send(()); + }); + finished.recv_timeout(Duration::from_secs(60)).expect( + "the answers over narrow tables never ended: the evaluation waits on its own widen", + ); +} diff --git a/prover/src/tests/constraint_artifact_tests.rs b/prover/src/tests/constraint_artifact_tests.rs new file mode 100644 index 000000000..a1d6c5727 --- /dev/null +++ b/prover/src/tests/constraint_artifact_tests.rs @@ -0,0 +1,1348 @@ +//! Serialization bit-exactness: for every production table, the constraint +//! artifact SERIALIZED at build time and read back evaluates identically to the +//! compiled folders. +//! +//! `constraint_program_tests` already pins the in-memory capture against both +//! folders. This suite pins the extra hop that "constraints as data" adds — the +//! bytes: +//! +//! ```text +//! capture → artifact → to_bytes → from_bytes → lift → evaluate +//! ``` +//! +//! Every stage in that chain is a place a program can change meaning: a +//! truncated `u32` index, a constant that loses canonical form through raw +//! limbs, a metadata list that reorders, an op tag that decodes to a different +//! operation. None of it is visible to a test that only exercises the in-memory +//! program, which is why this suite runs the deserialized artifact — never the +//! captured object — against the folders. +//! +//! The folders are the oracle: they are the production prove/verify path, +//! independently pinned by the prove→verify suites and cross-version +//! verification. All three evaluation paths are checked against them: +//! `eval_program` (prover shape), `eval_program_verifier` (OOD shape, the +//! recursion path), and `eval_device_program` (the flat blob). + +use math::field::element::FieldElement; +use stark::constraint_ir::{ + ConstraintArtifact, eval_device_program, eval_program, eval_program_verifier, +}; +use stark::frame::Frame; +use stark::proof::options::GoldilocksCubicProofOptions; +use stark::table::TableView; +use stark::traits::{AIR, TransitionEvaluationContext}; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; +use crate::test_utils::*; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type Fp = FieldElement; +type Fp3 = FieldElement; + +const TRIALS: usize = 100; + +/// Deterministic SplitMix64 (no `rand` dependency). +struct SplitMix64(u64); +impl SplitMix64 { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn fp3(&mut self) -> Fp3 { + Fp3::new([ + Fp::from(self.next_u64()), + Fp::from(self.next_u64()), + Fp::from(self.next_u64()), + ]) + } +} + +/// Extension element → raw `[u64; 3]` limbs (the device representation). +fn enc(x: &Fp3) -> [u64; 3] { + let limbs = x.value(); + [*limbs[0].value(), *limbs[1].value(), *limbs[2].value()] +} + +/// One production AIR's serialized-artifact differential. Returns the artifact's +/// measured size for the size report. +fn check_air_artifact( + air: &dyn AIR, + label: &str, +) -> ArtifactSize { + let n = air.context().num_transition_constraints; + let num_base = air.num_base_transition_constraints(); + let (n_main, n_aux) = air.trace_layout(); + + let artifact = ConstraintArtifact::capture(air); + artifact + .validate_against(air) + .unwrap_or_else(|e| panic!("[{label}] freshly captured artifact rejected: {e}")); + + // The wire hop. Everything below runs the DESERIALIZED artifact, so a + // codec bug cannot hide behind the in-memory object. + let bytes = artifact + .to_bytes() + .unwrap_or_else(|e| panic!("[{label}] serialize failed: {e}")); + let artifact = ConstraintArtifact::from_bytes(&bytes) + .unwrap_or_else(|e| panic!("[{label}] deserialize failed: {e}")); + artifact + .validate_against(air) + .unwrap_or_else(|e| panic!("[{label}] deserialized artifact rejected: {e}")); + + let prog = artifact.program(); + let dev = artifact.device_program(); + + // Structural identity against the capture the AIR itself would produce. + let captured = air.constraint_program(); + assert_eq!( + prog.nodes, captured.nodes, + "[{label}] nodes changed on the wire" + ); + assert_eq!( + prog.dims, captured.dims, + "[{label}] dims changed on the wire" + ); + assert_eq!( + prog.roots, captured.roots, + "[{label}] roots changed on the wire" + ); + assert_eq!( + prog.num_base, captured.num_base, + "[{label}] num_base changed" + ); + assert_eq!( + prog.base_consts, captured.base_consts, + "[{label}] base constants changed on the wire" + ); + assert_eq!( + prog.ext_consts, captured.ext_consts, + "[{label}] ext constants changed on the wire" + ); + assert_eq!(prog.roots.len(), n, "[{label}] one root per constraint"); + + // Release-safe exact-once backstop, as in `constraint_program_tests`: root + // id 0 is the reserved base-zero sentinel and no production constraint is + // identically zero, so a root still at the sentinel means that constraint + // was never captured. + for (i, &root) in prog.roots.iter().enumerate() { + assert_ne!(root, 0, "[{label}] constraint {i} was never captured"); + } + + let mut rng = SplitMix64(0x5EED_1234 ^ label.len() as u64); + for trial in 0..TRIALS { + let mk_step = |rng: &mut SplitMix64| { + let main: Vec = (0..n_main).map(|_| Fp::from(rng.next_u64())).collect(); + let aux: Vec = (0..n_aux).map(|_| rng.fp3()).collect(); + TableView::new(vec![main], vec![aux]) + }; + let frame = Frame::::new(vec![mk_step(&mut rng), mk_step(&mut rng)]); + let challenges = vec![rng.fp3(), rng.fp3()]; // [z, alpha] + let alphas: Vec = (0..air.max_bus_elements() + 2).map(|_| rng.fp3()).collect(); + let offset = rng.fp3(); + + let ctx = TransitionEvaluationContext::new_prover( + frame.as_row_frame(), + &challenges, + &alphas, + &offset, + ); + + // --- oracle: the compiled prover folder --- + let mut f_base = vec![Fp::zero(); num_base]; + let mut f_ext = vec![Fp3::zero(); n]; + air.compute_transition_prover(&ctx, &mut f_base, &mut f_ext); + + // --- path 1: generic interpreter over the deserialized program --- + let mut i_base = vec![Fp::zero(); num_base]; + let mut i_ext = vec![Fp3::zero(); n]; + eval_program(&prog, &ctx, &mut i_base, &mut i_ext); + + for c in 0..num_base { + assert_eq!( + f_base[c], i_base[c], + "[{label}] prover folder vs serialized program, base constraint {c}, trial {trial}" + ); + } + for c in num_base..n { + assert_eq!( + f_ext[c], i_ext[c], + "[{label}] prover folder vs serialized program, ext constraint {c}, trial {trial}" + ); + } + + // --- path 2: flat device walk over the deserialized blob --- + let main_raw: Vec> = (0..2) + .map(|off| { + let step = frame.get_evaluation_step(off); + (0..n_main) + .map(|c| *step.get_main_evaluation_element(0, c).value()) + .collect() + }) + .collect(); + let aux_raw: Vec> = (0..2) + .map(|off| { + let step = frame.get_evaluation_step(off); + (0..n_aux) + .map(|c| enc(step.get_aux_evaluation_element(0, c))) + .collect() + }) + .collect(); + let rap_raw: Vec<[u64; 3]> = challenges.iter().map(enc).collect(); + let alpha_raw: Vec<[u64; 3]> = alphas.iter().map(enc).collect(); + + let mut d_base = vec![0u64; num_base]; + let mut d_ext = vec![[0u64; 3]; n]; + eval_device_program( + &dev, + &main_raw, + &aux_raw, + &rap_raw, + &alpha_raw, + enc(&offset), + &mut d_base, + &mut d_ext, + ); + for c in 0..num_base { + assert_eq!( + d_base[c], + *f_base[c].value(), + "[{label}] prover folder vs serialized device blob, base constraint {c}, trial {trial}" + ); + } + for c in num_base..n { + assert_eq!( + d_ext[c], + enc(&f_ext[c]), + "[{label}] prover folder vs serialized device blob, ext constraint {c}, trial {trial}" + ); + } + + // --- path 3: the verifier/OOD shape, i.e. the recursion path --- + let embed = |step: &TableView| -> TableView { + let main: Vec = (0..n_main) + .map(|c| step.get_main_evaluation_element(0, c).to_extension()) + .collect(); + let aux: Vec = (0..n_aux) + .map(|c| *step.get_aux_evaluation_element(0, c)) + .collect(); + TableView::new(vec![main], vec![aux]) + }; + let vframe: Frame = Frame::new(vec![ + embed(frame.get_evaluation_step(0)), + embed(frame.get_evaluation_step(1)), + ]); + let vctx = TransitionEvaluationContext::::new_verifier( + &vframe, + &challenges, + &alphas, + &offset, + ); + + let v_folder = air.compute_transition(&vctx); + let mut v_interp = vec![Fp3::zero(); n]; + eval_program_verifier(&prog, &vctx, &mut v_interp); + for c in 0..n { + assert_eq!( + v_folder[c], v_interp[c], + "[{label}] verifier folder vs serialized program, constraint {c}, trial {trial}" + ); + } + } + + ArtifactSize { + label: label.to_string(), + constraints: n, + nodes: artifact.nodes.len(), + base_consts: artifact.base_consts.len(), + ext_consts: artifact.ext_consts.len(), + bytes: bytes.len(), + } +} + +/// One AIR's measured artifact size. +struct ArtifactSize { + label: String, + constraints: usize, + nodes: usize, + base_consts: usize, + ext_consts: usize, + bytes: usize, +} + +/// Every production table's serialized artifact evaluates bit-identically to +/// the compiled folders, on the prover shape, the verifier/OOD shape, and the +/// flat device blob. +#[test] +fn all_table_artifacts_roundtrip_and_match_folders() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!( + airs.len(), + NUM_PRODUCTION_AIRS, + "the production AIR list changed size; every per-table suite's coverage moved with it" + ); + + let mut sizes: Vec = Vec::with_capacity(airs.len()); + for (label, air) in &airs { + sizes.push(check_air_artifact(&**air, label)); + } + + report_sizes(&sizes); +} + +/// Print the per-AIR and total artifact sizes — the recursion machine's +/// program-length budget — and hold a ceiling so runaway growth is noticed. +fn report_sizes(sizes: &[ArtifactSize]) { + let total_bytes: usize = sizes.iter().map(|s| s.bytes).sum(); + let total_nodes: usize = sizes.iter().map(|s| s.nodes).sum(); + let total_constraints: usize = sizes.iter().map(|s| s.constraints).sum(); + + println!("\nconstraint artifact sizes (blowup=2)"); + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + "table", "constr", "nodes", "bconst", "econst", "bytes" + ); + for s in sizes { + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + s.label, s.constraints, s.nodes, s.base_consts, s.ext_consts, s.bytes + ); + } + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + "TOTAL", total_constraints, total_nodes, "", "", total_bytes + ); + println!( + "total: {total_nodes} nodes, {total_bytes} bytes ({:.1} KiB) across {} tables\n", + total_bytes as f64 / 1024.0, + sizes.len() + ); + + // A loose ceiling: this is a budget signal, not a tight assertion. It exists + // so a change that multiplies the program size fails here instead of being + // discovered when the recursion machine will not fit. + const CEILING_BYTES: usize = 8 * 1024 * 1024; + assert!( + total_bytes < CEILING_BYTES, + "total artifact size {total_bytes} exceeds the {CEILING_BYTES}-byte budget ceiling" + ); +} + +/// Per-AIR instruction census for the recursion machine's constraint leg. +/// +/// The machine is straight-line and cannot interpret, so a serialized program is +/// UNROLLED: one machine instruction per arithmetic IR node. That makes the node +/// census a direct instruction-count estimate for the constraint-evaluation leg, +/// which is the last unmeasured piece of the epoch-verifier budget. +/// +/// The classification that matters, and why: +/// +/// - **Leaves are addresses, not instructions.** `Var` reads an OOD frame value +/// the DEEP/opening leg already placed in memory; `RapChallenge` / `AlphaPow` / +/// `TableOffset` are transcript-derived values computed once per proof. The +/// constraint leg pays nothing marginal for them. +/// - **Constants are `Const` instructions**, one per distinct pooled value. +/// - **Arithmetic nodes are ALU instructions**, and the base/ext split is the +/// expensive distinction: a base node is a `BaseAlu` over one Goldilocks +/// element, an extension node an `ExtAlu` over three. +/// - **A `Mul` with an extension result and exactly one base operand** is the +/// `MulBase` form — 3 base multiplies instead of 9. Counting these separately +/// is the difference between a real estimate and a pessimistic one. +/// +/// # The IR's own `dim` tags are the WRONG split for the machine +/// +/// `Dim` records what the PROVER computes: its frame is base-field, so a +/// trace-only subexpression stays in the base field. The machine runs the +/// VERIFIER's evaluation at the OOD point, where the frame holds only extension +/// elements — `eval_program_verifier` resolves every `Var` to `Value::Ext` +/// regardless of `main`. So a node is base at verify time only if its whole +/// subtree is constants. +/// +/// Both splits are reported because the difference is large and load-bearing, +/// and taking the declared one would badly understate the machine's extension +/// traffic. The verifier-side column is the one to budget against. +/// +/// A consequence worth naming: a base-at-verify-time node is a constant-only +/// subtree, so the emitter can FOLD it at build time into a pooled constant. It +/// costs zero instructions, which is why the instruction estimate below counts +/// only extension work plus the pool. +/// +/// Printed rather than asserted (beyond a loose ceiling): this is an instrument, +/// and pinning exact counts would turn every constraint edit into a test failure. +/// +/// # WHAT THIS INSTRUMENT CANNOT SEE +/// +/// It reads captured IR, one AIR at a time. It knows nothing about how a proof +/// is ASSEMBLED from sub-proofs — not that the split-table families are chunked, +/// not that `FIXED_TABLE_COUNT` forces a sub-proof for a table with zero rows, +/// not which tables a continuation epoch even contains. +/// +/// So any conclusion about workload sensitivity, epoch composition or sub-proof +/// count is outside what these numbers support, however inviting the per-table +/// breakdown makes it. This is not hypothetical: a previous reading of this +/// table concluded the constraint leg was "workload-shaped" because ECDAS, ECSM +/// and KECCAK_RND are 87% of it — and that is false, because those tables are +/// present in every proof whether the workload touches them or not. The leg is +/// workload-INDEPENDENT, and only `epoch_chunk_multiplier` and +/// `continuation_epoch_constraint_leg` can tell you so. +/// +/// Use those two for anything per-proof. Use this one for per-AIR facts only. +#[test] +fn constraint_op_census() { + use stark::constraint_ir::artifact::DIM_BASE; + use stark::constraint_ir::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, + }; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + println!("\nconstraint-leg instruction census (one instruction per arithmetic node)"); + println!(" prover-dim = what the IR declares; verify-dim = what the machine runs"); + println!( + "{:<14} {:>7} {:>7} {:>6} {:>8} {:>8} {:>7} {:>8} {:>8}", + "table", "nodes", "leaves", "const", "pv-base", "fold", "ext", "mulbase", "instr" + ); + + let (mut t_nodes, mut t_leaves, mut t_const) = (0usize, 0usize, 0usize); + let (mut t_pv_base, mut t_fold, mut t_ext, mut t_mulbase) = (0usize, 0usize, 0usize, 0usize); + let mut t_constraints = 0usize; + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let nodes = &artifact.nodes; + + // Verifier-side dim: base ONLY for constant-only subtrees, because the + // OOD frame is all-extension. + let mut v_base = vec![false; nodes.len()]; + + let (mut leaves, mut consts, mut pv_base, mut foldable, mut ext_alu, mut mulbase) = + (0, 0, 0, 0, 0, 0); + for (i, n) in nodes.iter().enumerate() { + match n.op { + OP_VAR | OP_RAP_CHALLENGE | OP_ALPHA_POW | OP_TABLE_OFFSET => { + leaves += 1; + v_base[i] = false; + } + OP_CONST_BASE => { + consts += 1; + v_base[i] = true; + } + OP_CONST_EXT => { + consts += 1; + v_base[i] = false; + } + OP_ADD | OP_SUB | OP_MUL | OP_NEG | OP_EMBED => { + if n.dim == DIM_BASE { + pv_base += 1; + } + let (ba, bb) = match n.op { + OP_NEG => (v_base[n.a as usize], true), + OP_EMBED => (false, false), + _ => (v_base[n.a as usize], v_base[n.b as usize]), + }; + // Mirrors `interp::binop`: base only when both operands are + // base values AND the declared dim is base. + v_base[i] = ba && bb && n.dim == DIM_BASE; + if v_base[i] { + // Constant-only subtree: the emitter folds it at build + // time, so it emits no instruction at all. + foldable += 1; + } else { + let is_mulbase = n.op == OP_MUL && (ba != bb); + if is_mulbase { + mulbase += 1; + } else { + ext_alu += 1; + } + } + } + other => panic!("[{label}] unclassified op tag {other}"), + } + } + + // Instructions the constraint leg actually emits: extension ALU work + // plus one Const per pooled constant. Leaves are addresses, and + // constant-only subtrees fold away at build time. + let instr = ext_alu + mulbase + consts; + println!( + "{:<14} {:>7} {:>7} {:>6} {:>8} {:>8} {:>7} {:>8} {:>8}", + label, + nodes.len(), + leaves, + consts, + pv_base, + foldable, + ext_alu, + mulbase, + instr + ); + + t_nodes += nodes.len(); + t_leaves += leaves; + t_const += consts; + t_pv_base += pv_base; + t_fold += foldable; + t_ext += ext_alu; + t_mulbase += mulbase; + t_constraints += artifact.roots.len(); + + // Every constant node must correspond to exactly one pooled table entry; + // if that ever stopped holding, the Const instruction count above would + // be wrong. + assert_eq!( + consts, + artifact.base_consts.len() + artifact.ext_consts.len(), + "[{label}] constant nodes and pooled constants disagree" + ); + } + + // --- two emitter properties the design depends on, measured --- + // + // 1. MulAdd fusability. `ExtAlu` carries MulAdd as a first-class op, but the + // IR has no MulAdd node — it emits Mul then Add. A peephole can fuse + // `Add(Mul(a,b), c)` into one instruction, but ONLY when the Mul feeds + // exactly one consumer; hash-consing means a shared Mul would have to be + // recomputed, turning a saving into a cost. + // 2. `Op::Embed` usage. It should be zero — the builder documents it as + // unreachable from the single-body capture path — which matters because + // Embed is the one op whose machine lowering depends on the word model. + let (mut t_fusable, mut t_embed, mut t_dead, mut t_maxfan) = (0usize, 0usize, 0usize, 0u32); + // Machine constants are interned PROGRAM-WIDE, keyed on the canonical + // 4-lane word — one `Const` row per distinct value however many AIRs and + // however many nodes use it. Summing the per-AIR pools therefore overcounts, + // and small structural constants (0, 1, byte/halfword shifts) recur across + // every table. + let mut distinct_words: std::collections::BTreeSet<[u64; 4]> = + std::collections::BTreeSet::new(); + for (_, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let nodes = &artifact.nodes; + + let mut uses = vec![0u32; nodes.len()]; + for n in nodes { + match n.op { + OP_ADD | OP_SUB | OP_MUL => { + uses[n.a as usize] += 1; + uses[n.b as usize] += 1; + } + OP_NEG | OP_EMBED => uses[n.a as usize] += 1, + _ => {} + } + } + // A root is a consumer too: fusing away a node that a constraint roots at + // would delete the value the quotient recombination needs. + for &r in &artifact.roots { + uses[r as usize] += 1; + } + + for &c in &artifact.base_consts { + distinct_words.insert([c, 0, 0, 0]); + } + for &e in &artifact.ext_consts { + distinct_words.insert([e[0], e[1], e[2], 0]); + } + + // A node nobody reads is a write with mult = 0 — wasted instructions, and + // the emitter must DCE it rather than emit a zero-multiplicity write. + t_dead += uses.iter().filter(|&&u| u == 0).count(); + t_maxfan = t_maxfan.max(uses.iter().copied().max().unwrap_or(0)); + + for n in nodes { + if n.op == OP_EMBED { + t_embed += 1; + } + if n.op == OP_ADD { + let a_fusable = nodes[n.a as usize].op == OP_MUL && uses[n.a as usize] == 1; + let b_fusable = nodes[n.b as usize].op == OP_MUL && uses[n.b as usize] == 1; + if a_fusable || b_fusable { + t_fusable += 1; + } + } + } + } + + let arith = t_fold + t_ext + t_mulbase; + // One row per instruction, and every constant is one interned row + // program-wide — so the pool is counted once, not once per AIR. + let pool = distinct_words.len(); + let instr = t_ext + t_mulbase + pool; + println!( + "{:<14} {:>7} {:>7} {:>6} {:>8} {:>8} {:>7} {:>8} {:>8}", + "TOTAL", t_nodes, t_leaves, t_const, t_pv_base, t_fold, t_ext, t_mulbase, instr + ); + let unfused = instr + t_constraints; + let fused = unfused - t_fusable; + println!( + "\n arithmetic nodes {arith}\n \ + base by the IR's own dim {t_pv_base} (prover-side; NOT the machine's split)\n \ + base at verify time {t_fold} (constant-only subtrees -> fold at build time)\n \ + extension ALU {t_ext}\n \ + of which MulBase-routed {t_mulbase} (ext x base: 1 XALU row, vs 4+ if lowered by hand)\n \ + per-AIR constant pools {t_const} (sum; NOT the machine's cost)\n \ + interned program-wide {pool} (one Const row per distinct 4-lane word)\n \ + = constraint-leg instr {instr}\n \ + + quotient recombination {t_constraints} beta-folds (one per constraint)\n \ + = upper bound {unfused}\n \ + MulAdd-fusable Add nodes {t_fusable} (Add over a single-use Mul; MulAdd costs the same as Mul, so ALWAYS fuse)\n \ + = ESTIMATE, fused {fused}\n\n \ + leaves (addresses, free) {t_leaves}\n \ + Op::Embed nodes {t_embed} (base->ext is free; would emit nothing)\n \ + dead nodes (mult = 0) {t_dead}\n \ + max fanout (max mult) {t_maxfan}\n" + ); + + // Loose ceiling: the design budget treats this leg as ~1% of the epoch + // program. An order-of-magnitude regression should fail here. + assert!( + instr < 200_000, + "constraint-leg instruction estimate {instr} has grown past the design budget" + ); +} + +/// The constraint leg's per-EPOCH multiplier, measured on real fixtures. +/// +/// `constraint_op_census` counts instructions per distinct AIR. An epoch does +/// not evaluate each AIR once: every SUB-PROOF carries its own trace and needs +/// its own constraint evaluation, and the split-table families are CHUNKED — +/// `chunks = ceil(rows / max_rows[table])`, with `max_rows` sized per table so +/// each chunk costs about the same memory (`tables/mod.rs::max_rows`). +/// +/// So the epoch cost is `Σ over sub-proofs instr(that sub-proof's AIR)`, and the +/// multiplier against the per-AIR total is what this measures. It is the number +/// `lfm-design.md` §5.2 is missing: its ≈69K line reads as per-epoch but is +/// per-distinct-AIR. +/// +/// Measured by building real traces, so the chunk counts are the prover's own +/// splitting rather than a reconstruction of it. +#[test] +fn epoch_chunk_multiplier() { + use crate::tables::MaxRowsConfig; + use crate::tables::trace_builder::Traces; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + // Per-AIR constraint-leg instruction counts, keyed by label. + let instr: std::collections::BTreeMap<&str, usize> = production_airs(&opts) + .iter() + .map(|(label, air)| (*label, leg_instructions(&**air))) + .collect(); + let get = |k: &str| *instr.get(k).unwrap_or_else(|| panic!("no AIR {k}")); + + // Fixtures spanning roughly an epoch's worth of execution. An intermediate + // continuation epoch runs exactly 2^epoch_size_log2 cycles, so a fixture's + // cycle count is the axis to read these against. + for name in [ + "fib_iterative_1M", + "fib_iterative_2M", + "array_multipass_20M", + ] { + let (elf, logs, _) = run_asm_elf(name); + let max_rows = MaxRowsConfig::default(); + let traces = Traces::from_elf_and_logs_minimal(&elf, &logs, &max_rows, &[]) + .expect("trace build succeeds"); + + // (label, chunk count) for every sub-proof the epoch would contain. + let chunked: Vec<(&str, usize)> = vec![ + ("CPU", traces.cpus.len()), + ("LT", traces.lts.len()), + ("SHIFT", traces.shifts.len()), + ("MEMW", traces.memws.len()), + ("MEMW_A", traces.memw_aligneds.len()), + ("LOAD", traces.loads.len()), + ("MUL", traces.muls.len()), + ("DVRM", traces.dvrms.len()), + ("BRANCH", traces.branches.len()), + ("MEMW_R", traces.memw_registers.len()), + ("EQ", traces.eqs.len()), + ("BYTEWISE", traces.bytewises.len()), + ("STORE", traces.stores.len()), + ("CPU32", traces.cpu32s.len()), + ]; + + let chunked_total: usize = chunked.iter().map(|(l, n)| get(l) * n).sum(); + let chunk_count: usize = chunked.iter().map(|(_, n)| *n).sum(); + + // Fixed (unchunked) tables present once per proof, plus one PAGE per + // touched page. A continuation epoch substitutes L2G/GLOBAL_MEMORY for + // PAGE (page_configs is empty there), so this monolithic shape is an + // upper bound on the page contribution. + let pages = traces.pages.len(); + let fixed = get("BITWISE") + + get("DECODE") + + get("REGISTER") + + get("COPY") + + get("HALT") + + get("KECCAK") + + get("KECCAK_RND") + + get("KECCAK_RC") + + get("ECSM") + + get("ECDAS"); + let page_total = get("PAGE") * pages; + let epoch_total = chunked_total + fixed + page_total; + + let per_air_total: usize = instr.values().sum(); + println!( + "\n{name}: {} cycles\n \ + chunked sub-proofs {chunk_count} (of 14 families) -> {chunked_total} instr\n \ + fixed tables -> {fixed} instr\n \ + {pages} pages x {} instr -> {page_total} instr\n \ + EPOCH TOTAL {epoch_total} instr vs per-distinct-AIR {per_air_total} \ + multiplier {:.2}x", + logs.len(), + get("PAGE"), + epoch_total as f64 / per_air_total as f64 + ); + for (l, n) in &chunked { + if *n > 1 { + println!(" {l:<10} {n} chunks x {} = {}", get(l), get(l) * n); + } + } + } +} + +/// The constraint leg for a real CONTINUATION EPOCH — the shape we actually +/// recurse. +/// +/// The monolithic measurement above is the wrong shape for the target: a +/// continuation epoch passes `page_configs = &[]`, so PAGE never appears, and it +/// carries an L2G_MEMORY sub-proof instead. Its composition is +/// +/// ```text +/// 14 split-table families (>= 1 chunk each) +/// + FIXED_TABLE_COUNT (5 final, 4 intermediate — HALT only on the last) +/// + up to 6 accelerators (COPY, KECCAK, KECCAK_RND, ECSM, ECDAS, HINT — +/// one each, and only when the workload reaches it) +/// + 1 L2G_MEMORY +/// ``` +/// +/// which gives **25 sub-proofs intermediate and 26 final AT MOST**, and 19 / 20 +/// for a workload that calls no accelerator. This test asserts that arithmetic +/// so the composition is pinned rather than inferred: if the epoch shape +/// changes, the count here stops matching and this fails. +/// +/// ★ The ceiling is what moved. MOVER: `892c7d1bc` (main's `c2ac5d546`, #977) +/// — arm (i), the TABLE SET; arm (ii) is the +49-byte epoch statement, pinned +/// in `machine_tests` and `blake3_chip_tests`. #977 took the six accelerators out of +/// `FIXED_TABLE_COUNT` (11 → 5), so the 25/26 that used to be THE composition +/// is now its upper bound — the same count, reached only by a workload that +/// touches every accelerator. `epoch_verify_tests` measures the other end on +/// the fibonacci fixture, which reaches none of the six and carries 16. +/// +/// The instruction total is a minimum in the other axis, since it assumes one +/// chunk per family — a larger epoch adds chunks of the CHEAP AIRs (see +/// `epoch_chunk_multiplier`). +#[test] +fn continuation_epoch_constraint_leg() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let instr: std::collections::BTreeMap<&str, usize> = production_airs(&opts) + .iter() + .map(|(label, air)| (*label, leg_instructions(&**air))) + .collect(); + let get = |k: &str| *instr.get(k).unwrap_or_else(|| panic!("no AIR {k}")); + + // The 14 chunked split-table families, at their minimum of one chunk each. + let families = [ + "CPU", "LT", "SHIFT", "MEMW", "MEMW_A", "LOAD", "MUL", "DVRM", "BRANCH", "MEMW_R", "EQ", + "BYTEWISE", "STORE", "CPU32", + ]; + // The `FIXED_TABLE_COUNT` always-on tables (`prover/src/lib.rs`): exactly one + // sub-proof each REGARDLESS of TableCounts, because a zero-row table still + // needs its proof or its constraints drop out of verification. HALT is the + // one an intermediate epoch omits. + let always_on = ["BITWISE", "DECODE", "HALT", "KECCAK_RC", "REGISTER"]; + // ★ THE SIX THAT LEFT, and they left the way BLAKE3 did. #977 — on this + // lineage, the merge `892c7d1bc` — took + // `FIXED_TABLE_COUNT` 11 → 5 and made these six `TableCounts` fields, so a + // run that never reaches one carries no sub-proof for it — the same EC + // campaign argument (PR #871) that moved BLAKE3 out first, applied to the + // rest of the accelerator group. + // + // They stay in the COST sum below because the sum is a ceiling: a workload + // that calls every accelerator pays all eleven legs, and that is the figure + // the recursion budget has to clear. What changed is that the figure is now + // a MAXIMUM rather than a floor — a workload reaching none of the six pays + // `always_on` plus the families alone. + // + // ⚠ BLAKE3 is deliberately in NEITHER list. It is `TableCounts::blake3`, a + // 0-or-1 count, and it is not in `TableCounts::total`'s accelerator group + // either, so the counts below are the blake3-free shape and a blake3-using + // epoch is one higher. + let accelerators = ["COPY", "KECCAK", "KECCAK_RND", "ECSM", "ECDAS", "HINT"]; + // The list is a census OF the constant, so it must not be able to drift from + // it silently — the failure this pin previously had. It is `always_on` and + // not the eleven that the constant counts, which is exactly the assertion + // that went red on the main-sync port and named its own cause. + assert_eq!( + always_on.len(), + crate::FIXED_TABLE_COUNT, + "the always-on list must name every FIXED_TABLE_COUNT table" + ); + let fixed_final: Vec<&str> = always_on + .iter() + .chain(accelerators.iter()) + .copied() + .collect(); + + let families_instr: usize = families.iter().map(|l| get(l)).sum(); + let fixed_final_instr: usize = fixed_final.iter().map(|l| get(l)).sum(); + let fixed_intermediate_instr = fixed_final_instr - get("HALT"); + let l2g = get("L2G_MEMORY"); + + let intermediate = families_instr + fixed_intermediate_instr + l2g; + let final_epoch = families_instr + fixed_final_instr + l2g; + + let n_intermediate = families.len() + fixed_final.len() - 1 + 1; + let n_final = families.len() + fixed_final.len() + 1; + assert_eq!( + (n_intermediate, n_final), + (25, 26), + "epoch sub-proof composition no longer reproduces the measured 25 intermediate / \ + 26 final CEILING (every accelerator reached; blake3-free, and a BLAKE3-using \ + epoch is one higher)" + ); + + println!( + "\ncontinuation epoch constraint leg (every accelerator reached, one chunk per family)\n \ + 14 split families {families_instr}\n \ + 4 fixed + 6 accel {fixed_intermediate_instr} (no HALT)\n \ + 1 L2G_MEMORY {l2g}\n \ + INTERMEDIATE epoch {intermediate} instr over {n_intermediate} sub-proofs\n \ + FINAL epoch (+HALT) {final_epoch} instr over {n_final} sub-proofs\n \ + fixed+accel share {:.0}% — the 4 fixed legs are workload-INDEPENDENT, \ + the 6 accelerator legs are not\n", + 100.0 * fixed_intermediate_instr as f64 / intermediate as f64 + ); + + // The global proof is one L2G_GLOBAL per epoch plus one GLOBAL_MEMORY per + // touched page — negligible at any plausible page count, which is what + // settles the page-base question as identity-only rather than size. + println!( + " global proof: {} instr/epoch (L2G_GLOBAL) + {} instr/page (GLOBAL_MEMORY)\n", + get("L2G_GLOBAL"), + get("GLOBAL_MEMORY") + ); +} + +/// The chunk counts of a REAL continuation epoch, measured first-hand. +/// +/// `epoch_chunk_multiplier` measures monolithic runs, which is the wrong shape: +/// a monolithic proof covers the whole execution, while an epoch covers exactly +/// `2^epoch_size_log2` cycles and carries a different table set. This drives the +/// actual continuation path — `Executor::resume_with_limit` for one epoch's +/// cycles, then `Traces::from_image_and_logs` — so the chunk counts are the +/// prover's own, for an epoch. +/// +/// Only epoch 0 is measured, and that is sufficient rather than a shortcut: +/// epoch 0's `register_init` comes from the entry point, so it needs no previous +/// epoch, and every INTERMEDIATE epoch runs exactly `epoch_size` cycles by +/// construction (`continuation.rs` errors otherwise). Later epochs differ only +/// in which instructions those cycles execute. +/// +/// Proving is deliberately not run: the register chaining that would require it +/// (`prev_fini` comes out of `prove_epoch`) has no bearing on table sizes. +#[test] +fn continuation_epoch_chunk_counts_measured() { + use crate::tables::MaxRowsConfig; + use crate::tables::register; + use crate::tables::trace_builder::{Traces, build_initial_image_paged}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let instr: std::collections::BTreeMap<&str, usize> = production_airs(&opts) + .iter() + .map(|(label, air)| (*label, leg_instructions(&**air))) + .collect(); + let get = |k: &str| *instr.get(k).unwrap_or_else(|| panic!("no AIR {k}")); + + // 2^20 cycles: past CPU's 2^19 chunk bound, so chunking is actually + // exercised, while staying cheap enough to build traces for. + const EPOCH_SIZE_LOG2: u32 = 20; + let epoch_size = 1usize << EPOCH_SIZE_LOG2; + + for name in ["fib_iterative_2M", "array_multipass_20M"] { + let elf_bytes = asm_elf_bytes(name); + let elf = Elf::load(&elf_bytes).expect("load elf"); + let mut executor = Executor::new(&elf, vec![]).expect("executor"); + let image = build_initial_image_paged(&elf, &[]); + let register_init = register::register_init_from_entry_point(elf.entry_point); + + let logs = executor + .resume_with_limit(epoch_size) + .expect("resume") + .expect("program runs at least one epoch") + .to_vec(); + let is_final = executor.pc() == 0; + assert!( + !is_final && logs.len() == epoch_size, + "[{name}] wanted a full intermediate epoch, got {} cycles (final={is_final})", + logs.len() + ); + + let traces = Traces::from_image_and_logs( + &elf, + &image, + ®ister_init, + &logs, + &MaxRowsConfig::default(), + &[], + is_final, + true, + ) + .expect("epoch trace build"); + + let chunked: Vec<(&str, usize)> = vec![ + ("CPU", traces.cpus.len()), + ("LT", traces.lts.len()), + ("SHIFT", traces.shifts.len()), + ("MEMW", traces.memws.len()), + ("MEMW_A", traces.memw_aligneds.len()), + ("LOAD", traces.loads.len()), + ("MUL", traces.muls.len()), + ("DVRM", traces.dvrms.len()), + ("BRANCH", traces.branches.len()), + ("MEMW_R", traces.memw_registers.len()), + ("EQ", traces.eqs.len()), + ("BYTEWISE", traces.bytewises.len()), + ("STORE", traces.stores.len()), + ("CPU32", traces.cpu32s.len()), + ]; + + // An epoch never builds PAGE — the continuation path passes + // `page_configs = &[]`. Pin that here rather than trusting the comment. + assert!( + traces.page_configs.is_empty(), + "[{name}] a continuation epoch must not build PAGE tables" + ); + + let families: usize = chunked.iter().map(|(l, n)| get(l) * n).sum(); + let n_chunks: usize = chunked.iter().map(|(_, n)| *n).sum(); + // Intermediate epoch: 9 fixed tables (no HALT) + 1 L2G_MEMORY. + let fixed = get("BITWISE") + + get("DECODE") + + get("COPY") + + get("KECCAK") + + get("KECCAK_RND") + + get("KECCAK_RC") + + get("REGISTER") + + get("ECSM") + + get("ECDAS"); + let total = families + fixed + get("L2G_MEMORY"); + + println!( + "\n{name}, epoch 0 @ 2^{EPOCH_SIZE_LOG2} cycles\n \ + {n_chunks} chunked sub-proofs -> {families} instr\n \ + 9 fixed + L2G_MEMORY -> {} instr\n \ + EPOCH TOTAL {total} instr over {} sub-proofs", + fixed + get("L2G_MEMORY"), + n_chunks + 10 + ); + for (l, n) in &chunked { + if *n > 1 { + println!(" {l:<10} {n} chunks x {} = {}", get(l), get(l) * n); + } + } + } +} + +/// Constraint-leg instructions for one AIR: extension ALU plus MulBase-routed +/// multiplies. Shared by `constraint_op_census` and `epoch_chunk_multiplier` so +/// the two cannot drift apart. +fn leg_instructions(air: &dyn AIR) -> usize { + use stark::constraint_ir::artifact::DIM_BASE; + use stark::constraint_ir::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, + }; + let artifact = ConstraintArtifact::capture(air); + let nodes = &artifact.nodes; + let mut v_base = vec![false; nodes.len()]; + let mut count = 0usize; + for (i, n) in nodes.iter().enumerate() { + match n.op { + OP_VAR | OP_RAP_CHALLENGE | OP_ALPHA_POW | OP_TABLE_OFFSET | OP_CONST_EXT => { + v_base[i] = false + } + OP_CONST_BASE => v_base[i] = true, + OP_ADD | OP_SUB | OP_MUL | OP_NEG | OP_EMBED => { + let (ba, bb) = match n.op { + OP_NEG => (v_base[n.a as usize], true), + OP_EMBED => (false, false), + _ => (v_base[n.a as usize], v_base[n.b as usize]), + }; + v_base[i] = ba && bb && n.dim == DIM_BASE; + if !v_base[i] { + count += 1; + } + } + other => panic!("unclassified op tag {other}"), + } + } + count +} + +/// The captured artifact does not depend on the proof options. +/// +/// This is the premise behind leaving `ProofOptions` OUT of the artifact — if it +/// failed, one artifact per table would not be enough and the whole scheme would +/// need an artifact per (table, blowup) pair. `AirContext` carries the options +/// alongside the shape scalars, so the independence is worth pinning rather than +/// assuming from a reading of the constructor. +#[test] +fn artifacts_are_invariant_across_proof_options() { + let opts2 = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let opts4 = GoldilocksCubicProofOptions::with_blowup(4).expect("blowup=4 valid"); + + let airs2 = production_airs(&opts2); + let airs4 = production_airs(&opts4); + + for ((label, a2), (_, a4)) in airs2.iter().zip(airs4.iter()) { + let art2 = ConstraintArtifact::capture(&**a2); + let art4 = ConstraintArtifact::capture(&**a4); + assert_eq!( + art2, art4, + "[{label}] the constraint artifact differs between blowup 2 and 4; it would have to \ + be stored per (table, blowup) pair" + ); + assert_eq!( + art2.to_bytes().expect("serialize"), + art4.to_bytes().expect("serialize"), + "[{label}] artifact bytes differ across blowup factors" + ); + } +} + +/// The captured artifact does not depend on the trace length either. +/// +/// Same failure mode as the proof-options axis, different variable: if anything +/// in a captured program folded a domain-size-dependent constant, artifacts +/// would multiply per epoch shape. +/// +/// The axis is structurally absent — no AIR constructor takes a trace length — +/// so the only route by which one could reach the artifact is +/// `composition_poly_degree_bound(n)`, the single trace-length-dependent method +/// on the trait, whose value the artifact stores divided through by `n`. That +/// division is only sound if the bound is exactly linear, so this sweeps a wide +/// range of `n` per table rather than trusting the two probe points +/// `ConstraintArtifact::capture` checks. A table whose bound had any constant +/// term or any non-linearity would be misrepresented by the stored multiplier, +/// and would show up here. +#[test] +fn artifacts_are_invariant_across_trace_length() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let k = artifact.shape.composition_degree_multiplier as usize; + + for log_n in 4usize..=24 { + let n = 1usize << log_n; + assert_eq!( + air.composition_poly_degree_bound(n), + k * n, + "[{label}] composition_poly_degree_bound is not k·n at n=2^{log_n}; the artifact \ + stores only the linear coefficient, so a trace-length-dependent AIR would need \ + an artifact per epoch shape" + ); + } + + // Nothing else on the artifact can vary with the trace length, but pin + // capture determinism so a future source of nondeterminism (map + // iteration order in the constant tables, say) is caught here. + let again = ConstraintArtifact::capture(&**air); + assert_eq!(artifact, again, "[{label}] capture is not deterministic"); + } +} + +/// The six PARAMETERIZED tables produce a different program per parameter +/// value. +/// +/// `PAGE` / `GLOBAL_MEMORY` fold a page base into constant bus terms; the two +/// `L2G` tables fold an epoch label; `CPU` folds its table's timestamp base and +/// `REGISTER` the final PC timestamp (main's preprocessed clock and final +/// register pin). This test does not assert that away — it +/// characterizes it, because it is a real property of the current constraints +/// and the recursion machine has to plan around it. +/// +/// # The variation is NOT confined to constant VALUES +/// +/// The obvious guess is that two parameter values give the same node array with +/// one constant swapped. That is what `PAGE` and `GLOBAL_MEMORY` do, and it is +/// wrong in general: the builder interns constants by value, so a parameter +/// whose value happens to already be in the constant table costs no new node, +/// while a fresh value appends one — which shifts every later node id and hence +/// the constraint ROOTS. `L2G_GLOBAL` at `epoch_label = 1` reuses the existing +/// `1` constant; at `epoch_label = 7` it appends. Same algebra, different node +/// count and different root ids. +/// +/// This matters for the machine-side fix: "swap one constant per page" would be +/// a cheap patch and it is not available. Promoting the parameter to a runtime +/// uniform is, because the ALGEBRA is invariant — which is what the shape and +/// metadata assertions below pin. +#[test] +fn parameterized_airs_vary_per_parameter_value() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + // (label, artifact at parameter A, artifact at parameter B) + let cases: Vec<(&str, ConstraintArtifact, ConstraintArtifact)> = vec![ + ( + "PAGE", + ConstraintArtifact::capture(&create_page_air(&opts, 0x1000)), + ConstraintArtifact::capture(&create_page_air(&opts, 0x9000)), + ), + ( + "GLOBAL_MEMORY", + ConstraintArtifact::capture(&create_global_memory_air(&opts, 0x1000)), + ConstraintArtifact::capture(&create_global_memory_air(&opts, 0x9000)), + ), + ( + "L2G_GLOBAL", + ConstraintArtifact::capture(&crate::continuation::l2g_global_air(&opts, 1)), + ConstraintArtifact::capture(&crate::continuation::l2g_global_air(&opts, 7)), + ), + ( + "L2G_MEMORY", + ConstraintArtifact::capture(&crate::continuation::l2g_memory_air(&opts, 1)), + ConstraintArtifact::capture(&crate::continuation::l2g_memory_air(&opts, 7)), + ), + ( + "CPU", + ConstraintArtifact::capture(&create_cpu_air(&opts, 0)), + ConstraintArtifact::capture(&create_cpu_air(&opts, 4 << 21)), + ), + ( + "REGISTER", + ConstraintArtifact::capture(&create_register_air(&opts, 17)), + ConstraintArtifact::capture(&create_register_air(&opts, (4 << 21) + 1)), + ), + ]; + + println!("\nparameterized tables: how two parameter values differ"); + for (label, a, b) in &cases { + assert_ne!( + a, b, + "[{label}] is documented as parameterized but two parameter values gave the same \ + artifact; either the parameter stopped reaching the IR or the test picked two \ + values that collide" + ); + + // Invariant: the ALGEBRA. Same widths, same constraint count, same + // zerofier shapes, same degree bound — only the embedded parameter + // moves. This is the property that makes the parameter promotable to a + // runtime uniform. + assert_eq!(a.shape, b.shape, "[{label}] shape must not vary"); + assert_eq!(a.meta, b.meta, "[{label}] metadata must not vary"); + assert_eq!(a.num_base, b.num_base, "[{label}] num_base must not vary"); + assert_eq!( + a.roots.len(), + b.roots.len(), + "[{label}] constraint count must not vary" + ); + + // Variable: node count and root ids — an artifact of the builder's + // hash-consing, not of the constraints. A parameter value already in the + // constant table costs no new ConstBase node while a fresh one appends, + // which shifts every later node id. + // + // MEASURED, and note the counts are not all +1: L2G_GLOBAL moves 1 node + // for 1 constant, L2G_MEMORY moves 2 for 1. The second node is some + // further CSE difference downstream of the reused constant (L2G_MEMORY + // at epoch_label = 1 contributes the constant 0, which IS node id 0, so + // expressions over it have more chance to coincide with existing ones) — + // that specific explanation is inferred, not verified, so the bound + // below is deliberately loose. What is being pinned is only that the + // delta stays local rather than the algebra changing shape. + // + // CPU's base enters at each timestamp offset it sends (0, +1, +2 and the + // previous PC's −3), so it may intern one constant per offset. + let constants = if *label == "CPU" { 4 } else { 1 }; + let node_delta = a.nodes.len().abs_diff(b.nodes.len()); + let const_delta = a.base_consts.len().abs_diff(b.base_consts.len()); + assert!( + node_delta <= 4 * constants && const_delta <= constants, + "[{label}] two parameter values changed the program by {node_delta} nodes and \ + {const_delta} constants — too much to be the parameter's own interned constant and \ + its enclosing ops; the variation is structural, not just parametric" + ); + let roots_moved = a.roots != b.roots; + + println!( + " {label:<14} nodes {:>3} vs {:>3} consts {:>2} vs {:>2} roots moved: {}", + a.nodes.len(), + b.nodes.len(), + a.base_consts.len(), + b.base_consts.len(), + roots_moved + ); + } + println!(); +} + +/// GLOBAL_MEMORY has a second, ENUMERABLE axis: private-input pages preprocess +/// OFFSET only, which changes the artifact's SHAPE rather than a constant. +/// +/// Worth separating from the parameter axis above because the two have very +/// different consequences. A page base is an arbitrary address, so its artifact +/// set is unbounded; `is_private_input` is a boolean, so GLOBAL_MEMORY simply has +/// two shape variants and both can be enumerated. This pins that the difference +/// is confined to the preprocessed-column fields and does not touch the program. +#[test] +fn global_memory_private_input_is_a_second_shape_not_a_second_program() { + use crate::tables::page::PageConfig; + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + let elf_page = PageConfig::zero_init(PAGE_TEST_BASE); + let mut private_page = PageConfig::zero_init(PAGE_TEST_BASE); + private_page.is_private_input = true; + + let elf = ConstraintArtifact::capture(&crate::continuation::global_memory_air( + &opts, + &elf_page, + Some([0u8; 32]), + )); + let private = ConstraintArtifact::capture(&crate::continuation::global_memory_air( + &opts, + &private_page, + Some([0u8; 32]), + )); + + // Same constraints, same metadata: the bus interactions depend only on the + // page base, which is equal here. + assert_eq!(elf.nodes, private.nodes, "the program must not vary"); + assert_eq!(elf.base_consts, private.base_consts); + assert_eq!(elf.roots, private.roots); + assert_eq!(elf.meta, private.meta); + + // The shape does vary, in exactly the preprocessed fields. Both variants are + // preprocessed; they differ in HOW MANY columns. An ELF page commits OFFSET + // and INIT, so the verifier recomputes the genesis values from the ELF. A + // private-input page commits OFFSET alone — INIT is the private input and + // stays a main-trace column, but OFFSET is the row's address and leaving it + // prover-chosen would let a genesis token name an arbitrary address. + assert!(elf.shape.is_preprocessed, "an ELF page is preprocessed"); + assert!( + private.shape.is_preprocessed, + "a private-input page still preprocesses OFFSET" + ); + assert_eq!( + elf.shape.num_precomputed_columns, + crate::tables::global_memory::NUM_PREPROCESSED_COLS as u32, + "an ELF page commits OFFSET and INIT" + ); + assert_eq!( + private.shape.num_precomputed_columns, + crate::tables::page::NUM_PREPROCESSED_COLS_PRIVATE as u32, + "a private-input page commits OFFSET only" + ); + + let mut normalized = private.shape.clone(); + normalized.num_precomputed_columns = elf.shape.num_precomputed_columns; + assert_eq!( + normalized, elf.shape, + "the two variants must differ ONLY in the preprocessed-column fields" + ); +} + +/// An artifact captured from one table must not validate against another. +/// +/// The suite above only ever shows the shape check ACCEPTING. Without this, a +/// `validate_against` that returned `Ok(())` unconditionally would pass +/// everything here. +#[test] +fn an_artifact_does_not_validate_against_a_different_table() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + + let mut checked = 0usize; + for (i, (label_i, air_i)) in airs.iter().enumerate() { + let artifact = ConstraintArtifact::capture(&**air_i); + for (j, (label_j, air_j)) in airs.iter().enumerate() { + if i == j { + continue; + } + if artifact.validate_against(&**air_j).is_ok() { + // Two tables can legitimately share every shape scalar (several + // are bus-only tables with identical layouts), so an accept is + // only a failure when the programs actually differ. + let prog_i = air_i.constraint_program(); + let prog_j = air_j.constraint_program(); + assert_eq!( + (&prog_i.nodes, &prog_i.roots), + (&prog_j.nodes, &prog_j.roots), + "[{label_i}] artifact validated against [{label_j}], whose constraint \ + program is different — the shape check cannot tell them apart" + ); + } else { + checked += 1; + } + } + } + assert!( + checked > 0, + "validate_against never rejected any cross-table pairing; the check is not live" + ); +} + +/// A pre-captured program can be supplied to a production AIR and is used +/// without capture — the scoped verify-path unban, on real tables. +#[test] +fn production_airs_accept_a_precaptured_program() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + // Build the artifact from one instance, install it into a fresh one. + let artifact = ConstraintArtifact::capture(&create_cpu_air(&opts, 0)); + let air = create_cpu_air(&opts, 0).with_precaptured(artifact.program()); + + let supplied = air + .precaptured_constraint_program() + .expect("the supplied program must be visible on the guest-safe accessor"); + assert!( + std::ptr::eq(air.constraint_program(), supplied), + "constraint_program() must return the supplied program, not a fresh capture" + ); + + // And it still evaluates like the folder. + let n = air.context().num_transition_constraints; + let num_base = air.num_base_transition_constraints(); + let (n_main, n_aux) = air.trace_layout(); + let mut rng = SplitMix64(0x00A1_1CE5); + for _ in 0..16 { + let mk_step = |rng: &mut SplitMix64| { + let main: Vec = (0..n_main).map(|_| Fp::from(rng.next_u64())).collect(); + let aux: Vec = (0..n_aux).map(|_| rng.fp3()).collect(); + TableView::new(vec![main], vec![aux]) + }; + let frame = Frame::::new(vec![mk_step(&mut rng), mk_step(&mut rng)]); + let challenges = vec![rng.fp3(), rng.fp3()]; + let alphas: Vec = (0..air.max_bus_elements() + 2).map(|_| rng.fp3()).collect(); + let offset = rng.fp3(); + let ctx = TransitionEvaluationContext::new_prover( + frame.as_row_frame(), + &challenges, + &alphas, + &offset, + ); + + let mut f_base = vec![Fp::zero(); num_base]; + let mut f_ext = vec![Fp3::zero(); n]; + air.compute_transition_prover(&ctx, &mut f_base, &mut f_ext); + + let mut i_base = vec![Fp::zero(); num_base]; + let mut i_ext = vec![Fp3::zero(); n]; + eval_program(supplied, &ctx, &mut i_base, &mut i_ext); + + assert_eq!(f_base, i_base); + for c in num_base..n { + assert_eq!(f_ext[c], i_ext[c]); + } + } +} diff --git a/prover/src/tests/constraint_program_device_tests.rs b/prover/src/tests/constraint_program_device_tests.rs index c41f02493..ab315fe62 100644 --- a/prover/src/tests/constraint_program_device_tests.rs +++ b/prover/src/tests/constraint_program_device_tests.rs @@ -155,32 +155,14 @@ fn check_air_device( #[test] fn all_table_programs_lower_and_match_folders() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!( + airs.len(), + NUM_PRODUCTION_AIRS, + "production AIR list changed size" + ); - check_air_device(&create_cpu_air(&opts, 0), "CPU"); - check_air_device(&create_copy_air(&opts), "copy"); - check_air_device(&create_bitwise_air(&opts), "BITWISE"); - check_air_device(&create_lt_air(&opts), "LT"); - check_air_device(&create_shift_air(&opts), "SHIFT"); - check_air_device(&create_eq_air(&opts), "EQ"); - check_air_device(&create_bytewise_air(&opts), "BYTEWISE"); - check_air_device(&create_store_air(&opts), "STORE"); - check_air_device(&create_cpu32_air(&opts), "CPU32"); - check_air_device(&create_memw_air(&opts), "MEMW"); - check_air_device(&create_memw_aligned_air(&opts), "MEMW_A"); - check_air_device(&create_memw_register_air(&opts), "MEMW_R"); - check_air_device(&create_load_air(&opts), "LOAD"); - check_air_device(&create_decode_air(&opts), "DECODE"); - check_air_device(&create_mul_air(&opts), "MUL"); - check_air_device(&create_dvrm_air(&opts), "DVRM"); - check_air_device(&create_branch_air(&opts), "BRANCH"); - check_air_device(&create_halt_air(&opts), "HALT"); - check_air_device(&create_page_air(&opts, 0x1000), "PAGE"); - check_air_device(&create_register_air(&opts, 0), "REGISTER"); - check_air_device(&create_keccak_air(&opts), "KECCAK"); - check_air_device(&create_keccak_rnd_air(&opts), "KECCAK_RND"); - check_air_device(&create_keccak_rc_air(&opts), "KECCAK_RC"); - check_air_device(&create_ecsm_air(&opts), "ECSM"); - check_air_device(&create_ecdas_air(&opts), "ECDAS"); - check_air_device(&create_hint_air(&opts), "HINT"); - check_air_device(&create_is_wh_air(&opts), "IS_WH"); + for (label, air) in &airs { + check_air_device(&**air, label); + } } diff --git a/prover/src/tests/constraint_program_tests.rs b/prover/src/tests/constraint_program_tests.rs index 2bd7bbe92..c04ed4e6d 100644 --- a/prover/src/tests/constraint_program_tests.rs +++ b/prover/src/tests/constraint_program_tests.rs @@ -153,32 +153,14 @@ fn check_air(air: &dyn AIR #[test] fn all_table_programs_match_folders() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); - - check_air(&create_cpu_air(&opts, 0), "CPU"); - check_air(&create_copy_air(&opts), "copy"); - check_air(&create_bitwise_air(&opts), "BITWISE"); - check_air(&create_lt_air(&opts), "LT"); - check_air(&create_shift_air(&opts), "SHIFT"); - check_air(&create_eq_air(&opts), "EQ"); - check_air(&create_bytewise_air(&opts), "BYTEWISE"); - check_air(&create_store_air(&opts), "STORE"); - check_air(&create_cpu32_air(&opts), "CPU32"); - check_air(&create_memw_air(&opts), "MEMW"); - check_air(&create_memw_aligned_air(&opts), "MEMW_A"); - check_air(&create_memw_register_air(&opts), "MEMW_R"); - check_air(&create_load_air(&opts), "LOAD"); - check_air(&create_decode_air(&opts), "DECODE"); - check_air(&create_mul_air(&opts), "MUL"); - check_air(&create_dvrm_air(&opts), "DVRM"); - check_air(&create_branch_air(&opts), "BRANCH"); - check_air(&create_halt_air(&opts), "HALT"); - check_air(&create_page_air(&opts, 0x1000), "PAGE"); - check_air(&create_register_air(&opts, 0), "REGISTER"); - check_air(&create_keccak_air(&opts), "KECCAK"); - check_air(&create_keccak_rnd_air(&opts), "KECCAK_RND"); - check_air(&create_keccak_rc_air(&opts), "KECCAK_RC"); - check_air(&create_ecsm_air(&opts), "ECSM"); - check_air(&create_ecdas_air(&opts), "ECDAS"); - check_air(&create_hint_air(&opts), "HINT"); - check_air(&create_is_wh_air(&opts), "IS_WH"); + let airs = production_airs(&opts); + assert_eq!( + airs.len(), + NUM_PRODUCTION_AIRS, + "production AIR list changed size" + ); + + for (label, air) in &airs { + check_air(&**air, label); + } } diff --git a/prover/src/tests/constraint_set_tests_a.rs b/prover/src/tests/constraint_set_tests_a.rs index 571ae2da3..38c778c04 100644 --- a/prover/src/tests/constraint_set_tests_a.rs +++ b/prover/src/tests/constraint_set_tests_a.rs @@ -230,6 +230,20 @@ mod ecsm { } } +// ============================================================================= +// ecrec.rs +// ============================================================================= + +mod ecrec { + use super::*; + use crate::tables::ecrec::{EcrecConstraints, cols}; + + #[test] + fn ecrec_constraint_set_folder_capture_agree() { + check_set("ecrec", &EcrecConstraints, cols::NUM_COLUMNS); + } +} + // ============================================================================= // ecdas.rs // ============================================================================= diff --git a/prover/src/tests/count_table_lengths_drift_tests.rs b/prover/src/tests/count_table_lengths_drift_tests.rs deleted file mode 100644 index f8c4630a2..000000000 --- a/prover/src/tests/count_table_lengths_drift_tests.rs +++ /dev/null @@ -1,173 +0,0 @@ -//! Asserts `count_table_lengths` matches `Traces::from_elf_and_logs` row counts. - -use crate::tables::MaxRowsConfig; -use crate::tables::trace_builder::{Traces, count_table_lengths}; -use crate::test_utils::run_asm_elf; -use executor::elf::Elf; -use executor::vm::execution::Executor; -use executor::vm::instruction::decoding::Instruction; -use executor::vm::logs::Log; - -fn assert_count_table_lengths_matches(elf: &Elf, logs: &[Log]) { - let max_rows = MaxRowsConfig::default(); - - let predicted = - count_table_lengths(elf, logs, &max_rows, &[]).expect("count_table_lengths succeeds"); - let traces = - Traces::from_elf_and_logs_minimal(elf, logs, &max_rows, &[]).expect("trace build succeeds"); - - let sum_heights = |tables: &[stark::trace::TraceTable<_, _>]| -> u64 { - tables.iter().map(|t| t.main_table.height as u64).sum() - }; - - // Exact-match tables: predicted row count equals built trace. - assert_eq!(predicted.cpu_padded_rows, sum_heights(&traces.cpus), "cpu"); - assert_eq!( - predicted.memw_padded_rows, - sum_heights(&traces.memws), - "memw" - ); - assert_eq!( - predicted.memw_aligned_padded_rows, - sum_heights(&traces.memw_aligneds), - "memw_aligned" - ); - assert_eq!( - predicted.memw_register_padded_rows, - sum_heights(&traces.memw_registers), - "memw_register" - ); - assert_eq!( - predicted.load_padded_rows, - sum_heights(&traces.loads), - "load" - ); - assert_eq!( - predicted.shift_padded_rows, - sum_heights(&traces.shifts), - "shift" - ); - assert_eq!( - predicted.copy_padded_rows, traces.copy.main_table.height as u64, - "copy" - ); - assert_eq!( - predicted.decode_rows, traces.decode.main_table.height as u64, - "decode" - ); - - // Upper-bound tables: predicted is `>=` actual (LT/MUL/DVRM/BRANCH dedup ops). - assert!( - predicted.lt_padded_rows >= sum_heights(&traces.lts), - "lt: predicted={} actual={}", - predicted.lt_padded_rows, - sum_heights(&traces.lts) - ); - assert!( - predicted.mul_padded_rows >= sum_heights(&traces.muls), - "mul: predicted={} actual={}", - predicted.mul_padded_rows, - sum_heights(&traces.muls) - ); - assert!( - predicted.dvrm_padded_rows >= sum_heights(&traces.dvrms), - "dvrm: predicted={} actual={}", - predicted.dvrm_padded_rows, - sum_heights(&traces.dvrms) - ); - assert!( - predicted.branch_padded_rows >= sum_heights(&traces.branches), - "branch: predicted={} actual={}", - predicted.branch_padded_rows, - sum_heights(&traces.branches) - ); - - // Auxiliary scalars. - assert_eq!(predicted.cycle_count, logs.len() as u64, "cycle_count"); - assert_eq!( - predicted.unique_page_count, - traces.pages.len() as u64, - "unique_page_count" - ); - - // Mirrors hardcoded `halt_rows = 1` in `auto_storage::table_specs`. - assert_eq!(traces.halt.main_table.height, 1, "halt_rows"); -} - -#[test] -fn count_table_lengths_matches_traces() { - let (elf, logs, _) = run_asm_elf("fib_iterative_372k"); - assert_count_table_lengths_matches(&elf, &logs); -} - -/// Runs one Rust DMA guest and asserts the sizing pass matches the built traces. -/// Sizing replays each ecall through `replay_copy_for_sizing`, which delegates to -/// the same `collect_copy_ops` trace generation uses; the fixtures cover a single -/// chunk plus the multi-chunk and overlapping cases of `dma_memcpy_cases`, and the -/// same schedule driven through memset. -fn assert_dma_fixture_counts(elf_name: &str, syscall_number: u64) { - let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("workspace root") - .to_path_buf(); - let elf_bytes = - std::fs::read(workspace_root.join(format!("executor/program_artifacts/rust/{elf_name}"))) - .unwrap_or_else(|_| panic!("{elf_name} not found — build its make target")); - let elf = Elf::load(&elf_bytes).expect("valid DMA guest ELF"); - let result = Executor::new(&elf, vec![]) - .expect("executor") - .run() - .expect("DMA guest execution"); - - assert!( - result.logs.iter().any(|log| { - log.src1_val == syscall_number - && matches!( - result.instructions.get(&log.current_pc), - Some(Instruction::EcallEbreak) - ) - }), - "fixture must contain a DMA ecall" - ); - assert_count_table_lengths_matches(&elf, &result.logs); -} - -#[test] -fn count_table_lengths_matches_nonempty_dma_trace() { - use executor::vm::instruction::execution::{ - DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER, - }; - - assert_dma_fixture_counts("dma_memcpy_min.elf", DMA_MEMCPY_SYSCALL_NUMBER); - assert_dma_fixture_counts("dma_memcpy_cases.elf", DMA_MEMCPY_SYSCALL_NUMBER); - assert_dma_fixture_counts("dma_memset_min.elf", DMA_MEMSET_SYSCALL_NUMBER); - assert_dma_fixture_counts("dma_memset_cases.elf", DMA_MEMSET_SYSCALL_NUMBER); -} - -/// The `hint` ecall routes three register reads (`a0`/`a1`/`a2`) and four output -/// writes through the memory argument, plus two LT range-checks (selector, in_addr). -/// `count_table_lengths` must replay all of that exactly, or `memw_register` (an -/// exact-match table) drifts. Uses a real hint guest so the counts are non-trivial. -#[test] -fn count_table_lengths_matches_nonempty_hint_trace() { - let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("workspace root") - .to_path_buf(); - let elf_bytes = - std::fs::read(workspace_root.join("executor/program_artifacts/rust/hint_min.elf")) - .expect("hint_min.elf not found — run `make compile-programs-rust`"); - let elf = Elf::load(&elf_bytes).expect("valid hint guest ELF"); - let result = Executor::new(&elf, vec![]) - .expect("executor") - .run() - .expect("hint guest execution"); - - assert!( - result.logs.iter().any(|log| { - log.src1_val == executor::vm::instruction::execution::HINT_SYSCALL_NUMBER - }), - "fixture must contain a hint ecall" - ); - assert_count_table_lengths_matches(&elf, &result.logs); -} diff --git a/prover/src/tests/cpu_tests.rs b/prover/src/tests/cpu_tests.rs index 85e19369d..50a4624a6 100644 --- a/prover/src/tests/cpu_tests.rs +++ b/prover/src/tests/cpu_tests.rs @@ -6,21 +6,32 @@ //! preprocessed timestamp layout (`timestamp_bases`, `validate_heights`). use crate::tables::cpu::{ - CPU_PADDING_PC, CpuOperation, MAX_CPU_ROWS, cols, final_pc_timestamp, generate_cpu_trace, - table_heights, timestamp_bases, validate_heights, + CPU_PADDING_PC, CpuOp, CpuOperation, MAX_CPU_ROWS, MAX_TABLE_ROWS, cols, final_pc_timestamp, + generate_cpu_trace, table_heights, timestamp_bases, validate_heights, }; +use crate::tables::decode::DecodeTable; use crate::tables::max_rows; use crate::tables::types::DecodeEntry; use executor::vm::{ instruction::decoding::{ArithOp, Comparison, Instruction, LoadStoreWidth}, logs::Log, + memory::U64HashMap, }; const PC: u64 = 0x1000; +/// A CPU op and the decode it is read with. +struct Held(DecodeEntry, CpuOperation); + +impl Held { + fn op(&self) -> CpuOp<'_> { + CpuOp::new(&self.0, &self.1) + } +} + /// Build a CpuOperation from an instruction + register values. -fn op_of(instr: Instruction, src1: u64, src2: u64, dst: u64, next_pc: u64) -> CpuOperation { +fn op_of(instr: Instruction, src1: u64, src2: u64, dst: u64, next_pc: u64) -> Held { let decode = DecodeEntry::from_instruction(PC, instr, 4); let log = Log { current_pc: PC, @@ -29,7 +40,8 @@ fn op_of(instr: Instruction, src1: u64, src2: u64, dst: u64, next_pc: u64) -> Cp src2_val: src2, dst_val: dst, }; - CpuOperation::from_log(&log, 4, decode) + let op = CpuOperation::from_log(&log, 4, &decode, 0); + Held(decode, op) } // ========================================================================= @@ -38,7 +50,7 @@ fn op_of(instr: Instruction, src1: u64, src2: u64, dst: u64, next_pc: u64) -> Cp #[test] fn test_from_log_add_reg_reg() { - let op = op_of( + let held = op_of( Instruction::Arith { dst: 3, src1: 1, @@ -50,18 +62,19 @@ fn test_from_log_add_reg_reg() { 30, PC + 4, ); + let op = held.op(); assert_eq!(op.rv1, 10); assert_eq!(op.rv2, 20); - assert_eq!(op.arg2, 20, "reg-reg: arg2 = rv2 (imm = 0)"); + assert_eq!(op.arg2(), 20, "reg-reg: arg2 = rv2 (imm = 0)"); assert_eq!(op.res, 30, "res = rv1 + arg2"); assert_eq!(op.rvd, 30, "rvd = res (not memory)"); assert_eq!(op.next_pc, PC + 4); - assert!(!op.branch_cond); + assert!(!op.branch_cond()); } #[test] fn test_from_log_addi() { - let op = op_of( + let held = op_of( Instruction::ArithImm { dst: 3, src: 1, @@ -73,14 +86,15 @@ fn test_from_log_addi() { 15, PC + 4, ); - assert_eq!(op.arg2, 5, "reg-imm: arg2 = imm (rv2 = 0)"); + let op = held.op(); + assert_eq!(op.arg2(), 5, "reg-imm: arg2 = imm (rv2 = 0)"); assert_eq!(op.res, 15); assert_eq!(op.rvd, 15); } #[test] fn test_from_log_sub() { - let op = op_of( + let held = op_of( Instruction::Arith { dst: 3, src1: 1, @@ -92,13 +106,14 @@ fn test_from_log_sub() { 10, PC + 4, ); + let op = held.op(); assert_eq!(op.res, 10, "res = rv1 - arg2"); assert_eq!(op.rvd, 10); } #[test] fn test_from_log_beq_taken() { - let op = op_of( + let held = op_of( Instruction::Branch { src1: 1, src2: 2, @@ -110,15 +125,16 @@ fn test_from_log_beq_taken() { 0, PC + 8, ); - assert!(op.branch_cond, "BEQ with equal operands is taken"); - assert_eq!(op.arg2, 5, "conditional branch: arg2 = rv2"); + let op = held.op(); + assert!(op.branch_cond(), "BEQ with equal operands is taken"); + assert_eq!(op.arg2(), 5, "conditional branch: arg2 = rv2"); assert_eq!(op.res, 1, "EQ result on the ALU bus is 1 when taken"); assert_eq!(op.next_pc, PC + 8, "taken branch uses the executor next_pc"); } #[test] fn test_from_log_beq_not_taken() { - let op = op_of( + let held = op_of( Instruction::Branch { src1: 1, src2: 2, @@ -130,7 +146,8 @@ fn test_from_log_beq_not_taken() { 0, PC + 4, ); - assert!(!op.branch_cond); + let op = held.op(); + assert!(!op.branch_cond()); assert_eq!(op.res, 0); assert_eq!( op.next_pc, @@ -141,7 +158,7 @@ fn test_from_log_beq_not_taken() { #[test] fn test_from_log_bne_taken() { - let op = op_of( + let held = op_of( Instruction::Branch { src1: 1, src2: 2, @@ -153,8 +170,9 @@ fn test_from_log_bne_taken() { 0, PC + 8, ); + let op = held.op(); assert!( - op.branch_cond, + op.branch_cond(), "BNE with differing operands is taken (invert)" ); assert_eq!(op.res, 1); @@ -162,7 +180,7 @@ fn test_from_log_bne_taken() { #[test] fn test_from_log_load() { - let op = op_of( + let held = op_of( Instruction::Load { dst: 3, offset: 4, @@ -174,13 +192,14 @@ fn test_from_log_load() { 0xDEAD, PC + 4, ); + let op = held.op(); assert_eq!(op.res, 0x104, "load address = rv1 + imm"); assert_eq!(op.rvd, 0xDEAD, "load rvd = the loaded value"); } #[test] fn test_from_log_store() { - let op = op_of( + let held = op_of( Instruction::Store { src: 2, offset: 8, @@ -192,6 +211,7 @@ fn test_from_log_store() { 0, PC + 4, ); + let op = held.op(); assert_eq!(op.res, 0x108, "store address = rv1 + imm"); assert_eq!(op.rv2, 0xAB, "store value comes from rs2"); assert_eq!(op.rvd, 0, "store writes nothing back to rd"); @@ -199,7 +219,7 @@ fn test_from_log_store() { #[test] fn test_from_log_word_carries_real_register_values() { - let op = op_of( + let held = op_of( Instruction::ArithW { dst: 3, src1: 1, @@ -211,6 +231,7 @@ fn test_from_log_word_carries_real_register_values() { 30, PC + 4, ); + let op = held.op(); assert!(op.decode.fields.word_instr); // The delegate CpuOperation carries the real values for CPU32/register ops. assert_eq!(op.rv1, 10); @@ -224,10 +245,15 @@ fn test_from_log_word_carries_real_register_values() { // generate_cpu_trace // ========================================================================= -fn ops4(instr: Instruction) -> Vec { - (0..4) +fn ops4(instr: Instruction) -> (Vec, DecodeTable) { + let mut instructions = U64HashMap::default(); + for i in 0..4 { + instructions.insert(PC + i * 4, instr); + } + let decode = DecodeTable::from_instructions(&instructions); + let ops = (0..4) .map(|i| { - let decode = DecodeEntry::from_instruction(PC + i * 4, instr, 4); + let row = decode.row(PC + i * 4).expect("an instruction"); let log = Log { current_pc: PC + i * 4, next_pc: PC + i * 4 + 4, @@ -235,20 +261,21 @@ fn ops4(instr: Instruction) -> Vec { src2_val: 20, dst_val: 30, }; - CpuOperation::from_log(&log, i * 4 + 4, decode) + CpuOperation::from_log(&log, i * 4 + 4, decode.entry(row), row) }) - .collect() + .collect(); + (ops, decode) } #[test] fn test_trace_width_and_real_row() { - let ops = ops4(Instruction::Arith { + let (ops, decode) = ops4(Instruction::Arith { dst: 3, src1: 1, src2: 2, op: ArithOp::Add, }); - let trace = generate_cpu_trace(&ops); + let trace = generate_cpu_trace(&ops, &decode); assert_eq!(trace.main_table.width, cols::NUM_COLUMNS); assert_eq!(cols::NUM_COLUMNS, 37); assert_eq!(trace.main_table.height, 4); @@ -261,16 +288,14 @@ fn test_trace_width_and_real_row() { #[test] fn test_trace_padding_row() { // One real op → padded to 4 rows; rows 1..4 are padding. - let ops = vec![ - ops4(Instruction::Arith { - dst: 3, - src1: 1, - src2: 2, - op: ArithOp::Add, - }) - .remove(0), - ]; - let trace = generate_cpu_trace(&ops); + let (mut ops, decode) = ops4(Instruction::Arith { + dst: 3, + src1: 1, + src2: 2, + op: ArithOp::Add, + }); + ops.truncate(1); + let trace = generate_cpu_trace(&ops, &decode); let pad = trace.main_table.get_row(1); assert_eq!( pad[cols::PC_0], @@ -288,13 +313,13 @@ fn test_trace_padding_row() { #[test] fn test_trace_word_row_columns_masked() { - let ops = ops4(Instruction::ArithW { + let (ops, decode) = ops4(Instruction::ArithW { dst: 3, src1: 1, src2: 2, op: ArithOp::Add, }); - let trace = generate_cpu_trace(&ops); + let trace = generate_cpu_trace(&ops, &decode); let row = trace.main_table.get_row(0); // Delegate row: word_instr set, but all operational columns masked to 0. assert_eq!(row[cols::WORD_INSTR], 1u64.into()); @@ -316,7 +341,7 @@ fn test_trace_word_row_columns_masked() { #[test] fn test_collect_bitwise_ops_shape() { use crate::tables::bitwise::BitwiseOperationType; - let op = op_of( + let held = op_of( Instruction::Arith { dst: 3, src1: 1, @@ -328,6 +353,7 @@ fn test_collect_bitwise_ops_shape() { 30, PC + 4, ); + let op = held.op(); let ops = op.collect_bitwise_ops(); assert_eq!(ops.len(), 7, "3 ARE_BYTES + 4 IS_HALF"); assert!( @@ -347,7 +373,7 @@ fn test_collect_bitwise_ops_shape() { #[test] fn test_collect_bitwise_ops_word_row_zeroed() { - let op = op_of( + let held = op_of( Instruction::ArithW { dst: 3, src1: 1, @@ -359,6 +385,7 @@ fn test_collect_bitwise_ops_word_row_zeroed() { 30, PC + 4, ); + let op = held.op(); let ops = op.collect_bitwise_ops(); // On a word delegate row the CPU zeroes rs1/rs2/rd/alu_flags/mem_flags/res, // but half_instruction_length stays (it is set unconditionally in the trace). @@ -376,7 +403,7 @@ fn test_collect_bitwise_ops_word_row_zeroed() { /// whatever timestamp the operations carry (a chunk's ops carry `base + 4·(row + 1)`). #[test] fn test_trace_timestamp_is_preprocessed_column() { - let mut ops = ops4(Instruction::Arith { + let (mut ops, decode) = ops4(Instruction::Arith { dst: 3, src1: 1, src2: 2, @@ -386,7 +413,7 @@ fn test_trace_timestamp_is_preprocessed_column() { for (i, op) in ops.iter_mut().enumerate() { op.timestamp = 1000 + 7 * i as u64; } - let trace = generate_cpu_trace(&ops); + let trace = generate_cpu_trace(&ops, &decode); assert_eq!(trace.main_table.height, 4); for row in 0..4 { assert_eq!( @@ -419,6 +446,9 @@ fn test_final_pc_timestamp_follows_last_row() { fn test_validate_heights_accepts_honest_layouts() { assert!(validate_heights(&[4]).is_ok()); assert!(validate_heights(&[max_rows::CPU, max_rows::CPU, 64]).is_ok()); + // The block's 2^21-row tables, and the tallest a prover may cut. + assert!(validate_heights(&[1 << 21, 1 << 21, 1 << 20]).is_ok()); + assert!(validate_heights(&[MAX_TABLE_ROWS, 4]).is_ok()); // Largest layout under the bound: (2^11 - 1) full tables + one of 2^18 rows. let mut heights = vec![max_rows::CPU; (1 << 11) - 1]; heights.push(max_rows::CPU / 2); @@ -427,7 +457,7 @@ fn test_validate_heights_accepts_honest_layouts() { #[test] fn test_validate_heights_rejects_bad_heights() { - for bad in [0, 1, 2, 3, 12, max_rows::CPU * 2] { + for bad in [0, 1, 2, 3, 12, MAX_TABLE_ROWS * 2] { assert!(validate_heights(&[bad]).is_err(), "height {bad}"); } assert!(validate_heights(&[max_rows::CPU, 6]).is_err()); @@ -457,3 +487,216 @@ fn test_table_heights_matches_chunking() { assert!(validate_heights(&table_heights(rows, max_rows::CPU)).is_err()); assert!(validate_heights(&table_heights(rows - (1 << 18), max_rows::CPU)).is_ok()); } + +/// Every bus message's values at one row whose columns hold distinct values. +fn messages_at_a_row( + interactions: &[stark::lookup::BusInteraction], +) -> Vec> { + let row = |c: usize| crate::tables::types::FE::from(c as u64 + 1); + interactions + .iter() + .map(|i| { + i.values + .iter() + .flat_map(|v| v.combine_from::(row)) + .collect() + }) + .collect() +} + +/// ⛔ Each CPU table's AIR sends its own timestamp base, cached or not. +/// +/// The prototype cache serves the bases an honest prover produces. A cache +/// keyed by less than the base would hand every table the first one's bus +/// values, and two tables' clocks would overlap. Each base is asked for twice +/// (the second answer comes from the cache when the base is cached) and must +/// send what the interactions built fresh at that base send, never base 0's. +#[test] +fn every_cpu_air_sends_its_own_timestamp_base() { + use crate::tables::cpu::{TIMESTAMP_STRIDE, bus_interactions}; + use stark::traits::AIR; + + let opts = stark::proof::options::ProofOptions::default_test_options(); + let zero = messages_at_a_row(&bus_interactions(0)); + let full = TIMESTAMP_STRIDE * max_rows::CPU as u64; + for base in [ + 0, + full, + 2 * full, + TIMESTAMP_STRIDE << 21, + (3 * TIMESTAMP_STRIDE) << 21, + 24, + ] { + let fresh = messages_at_a_row(&bus_interactions(base)); + for _ in 0..2 { + let air = crate::test_utils::create_cpu_air(&opts, base); + assert_eq!( + messages_at_a_row(air.bus_interactions()), + fresh, + "base {base}: the AIR sends another base's messages" + ); + } + assert_eq!(fresh != zero, base != 0, "base {base} moves no message"); + } +} + +/// ⛔ The CPU and REGISTER AIRs hand the verifier the columns it checks. +/// +/// The multilinear verifier checks a table's preprocessed openings against +/// `precomputed_columns()`, and an AIR given only a root (`with_preprocessed`) +/// returns none, so its TIMESTAMP or its FINI would be free main trace. Every +/// CPU table must return its own height's `4·(row + 1)` column, and REGISTER +/// its four columns over the halted file. +#[test] +fn the_cpu_and_register_airs_expose_the_columns_the_verifier_checks() { + use crate::tables::{cpu, register}; + + let opts = stark::proof::options::ProofOptions::default_test_options(); + let program = + executor::elf::Elf::load(&crate::test_utils::asm_elf_bytes("all_instructions_64")) + .expect("the ELF loads"); + let heights = vec![16, 8, 4]; + let counts = cpu_only_counts(heights.len()); + let output = [7u8; 5]; + let inputs = crate::BlockAirInputs::new(&opts, &program, heights.clone(), &output) + .expect("honest heights"); + let airs = crate::block_airs(&program, &opts, &[], &counts, &inputs); + let refs = airs.air_refs(); + let start = crate::cpu_proof_start(true, &counts); + for (k, &height) in heights.iter().enumerate() { + let air = refs[start + k]; + assert_eq!(air.name(), format!("CPU[{k}]")); + assert_eq!(air.num_precomputed_columns(), cpu::NUM_PRECOMPUTED_COLS); + assert_eq!( + air.precomputed_columns(), + vec![cpu::timestamp_column(height)], + "CPU[{k}]: the verifier would check another TIMESTAMP" + ); + } + let register_air = refs + .iter() + .find(|air| air.name() == "REGISTER") + .expect("REGISTER is in the block"); + let init = register::register_init_from_entry_point(program.entry_point); + let fini = register::halted_fini(output.len() as u32); + assert_eq!( + register_air.num_precomputed_columns(), + register::NUM_PREPROCESSED_COLS + ); + assert_eq!( + register_air.precomputed_columns(), + register::preprocessed_columns(&init, &fini), + "REGISTER: the verifier would check another FINI" + ); +} + +/// A block's counts with `cpu` CPU tables and no other chunked or optional one. +fn cpu_only_counts(cpu: usize) -> crate::TableCounts { + crate::TableCounts { + cpu, + lt: 0, + memw: 0, + memw_aligned: 0, + load: 0, + mul: 0, + dvrm: 0, + shift: 0, + branch: 0, + memw_register: 0, + eq: 0, + bytewise: 0, + store: 0, + cpu32: 0, + keccak: 0, + keccak_rnd: 0, + ecsm: 0, + ecdas: 0, + hint: 0, + copy: 0, + blake3: 0, + is_wh: 0, + ecrec: 0, + } +} + +/// ⛔ The block's REGISTER sends the final PC token at its own clock's end. +/// +/// REGISTER's name is a label; what binds the final PC is the constant its bus +/// interactions carry, `final_pc_timestamp(Σ CPU rows)`. Blocks of different CPU +/// heights must each send their own, as the interactions built fresh at that +/// timestamp do. A REGISTER built at another block's timestamp, or at 0, would +/// let the run's last PC token land where the CPU tables do not end. +#[test] +fn the_block_register_sends_the_final_pc_token_at_its_clocks_end() { + use crate::tables::register; + + let opts = stark::proof::options::ProofOptions::default_test_options(); + let program = + executor::elf::Elf::load(&crate::test_utils::asm_elf_bytes("all_instructions_64")) + .expect("the ELF loads"); + let stale = messages_at_a_row(®ister::bus_interactions(0)); + let mut seen = Vec::new(); + for heights in [vec![16, 8, 4], vec![1 << 21, 4]] { + let counts = cpu_only_counts(heights.len()); + let inputs = crate::BlockAirInputs::new(&opts, &program, heights.clone(), &[]) + .expect("honest heights"); + let airs = crate::block_airs(&program, &opts, &[], &counts, &inputs); + let refs = airs.air_refs(); + let register_air = refs + .iter() + .find(|air| air.name() == "REGISTER") + .expect("REGISTER is in the block"); + let rows: u64 = heights.iter().map(|&h| h as u64).sum(); + let sent = messages_at_a_row(register_air.bus_interactions()); + assert_eq!( + sent, + messages_at_a_row(®ister::bus_interactions(final_pc_timestamp(rows))), + "{heights:?}: REGISTER sends another final PC timestamp than its CPU tables end at" + ); + assert_ne!( + sent, stale, + "{heights:?}: REGISTER sends the final PC token at 0" + ); + seen.push(sent); + } + assert_ne!( + seen[0], seen[1], + "two blocks' REGISTERs send one final PC timestamp" + ); +} + +/// ⛔ The CPU tables' position is checked in every build, not only under +/// `debug_assert`: a table set read with counts that place the CPU tables +/// elsewhere (one more COPY table, one fewer CPU table, or HALT counted) is +/// refused with an error, and the counts it was built from pass. +#[test] +fn the_cpu_tables_are_refused_where_their_heights_are_not_read() { + let opts = stark::proof::options::ProofOptions::default_test_options(); + let program = + executor::elf::Elf::load(&crate::test_utils::asm_elf_bytes("all_instructions_64")) + .expect("the ELF loads"); + let counts = cpu_only_counts(2); + let inputs = + crate::BlockAirInputs::new(&opts, &program, vec![8, 4], &[]).expect("honest heights"); + let airs = crate::block_airs(&program, &opts, &[], &counts, &inputs); + assert!(airs.check_cpu_position(&counts).is_ok()); + let mut shifted = counts.clone(); + shifted.copy = 1; + let mut fewer = counts.clone(); + fewer.cpu = 1; + let mut wider = counts.clone(); + wider.blake3 = 1; + for (what, other) in [ + ("a COPY table", shifted), + ("one CPU table", fewer), + ("BLAKE3", wider), + ] { + assert!( + matches!( + airs.check_cpu_position(&other), + Err(crate::Error::InvalidTableCounts(_)) + ), + "counts with {what} read the CPU tables where they are not" + ); + } +} diff --git a/prover/src/tests/cpu_timestamp_tests.rs b/prover/src/tests/cpu_timestamp_tests.rs index b8c714790..e29b7a928 100644 --- a/prover/src/tests/cpu_timestamp_tests.rs +++ b/prover/src/tests/cpu_timestamp_tests.rs @@ -10,15 +10,14 @@ //! verifier reads the CPU heights from the right sub-proofs and rejects hostile //! ones cleanly. -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use executor::elf::Elf; -use stark::prover::{IsStarkProver, Prover}; +use stark::prover::IsStarkProver; use super::halt_pc_forgery_poc::{opts, run_logs}; use crate::statement::{StatementKind, absorb_statement}; use crate::tables::MaxRowsConfig; use crate::tables::trace_builder::Traces; -use crate::test_utils::{E, asm_elf_bytes}; +use crate::test_utils::asm_elf_bytes; use crate::{VmAirs, VmProof}; const GUEST: &str = "all_instructions_64"; @@ -30,15 +29,9 @@ fn craft_proof( ) -> Result { let options = opts(); let program = Elf::load(elf_bytes).expect("ELF load"); - let mut traces = Traces::from_elf_and_logs( - &program, - &run_logs(elf_bytes), - &MaxRowsConfig::small(), - &[], - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build"); + let mut traces = + Traces::from_elf_and_logs(&program, &run_logs(elf_bytes), &MaxRowsConfig::small(), &[]) + .expect("trace build"); tamper(&mut traces); let table_counts = traces.table_counts(); @@ -56,11 +49,12 @@ fn craft_proof( &options, &program, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let runtime_page_ranges = traces.runtime_page_ranges(); - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -71,11 +65,10 @@ fn craft_proof( &runtime_page_ranges, options.fri_final_poly_log_degree, ); - let proof = Prover::multi_prove( + let proof = crate::hash_pin::BlockProver::multi_prove( airs.air_trace_pairs(&mut traces), &mut transcript, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, )?; Ok(VmProof { @@ -150,7 +143,14 @@ fn hostile_cpu_heights_are_rejected_by_the_verifier() { proof.proof.proofs[cpu].trace_length = height; crate::verify_with_options(&proof, &elf, &opts(), None, None) }; - for height in [0, 2, 3, 6, crate::tables::max_rows::CPU * 2, usize::MAX] { + for height in [ + 0, + 2, + 3, + 6, + crate::tables::cpu::MAX_TABLE_ROWS * 2, + usize::MAX, + ] { assert!( matches!( with_height(height), @@ -165,44 +165,52 @@ fn hostile_cpu_heights_are_rejected_by_the_verifier() { } /// `cpu_proof_start` (which the verifier reads CPU heights with) matches the -/// `air_refs` layout for every HALT / accelerator / IS_WH combination. Checked in -/// release builds too, unlike the `debug_assert` in `air_refs`. +/// `air_refs` layout for every HALT / accelerator / IS_WH combination, with and +/// without COPY tables (several of them, as the block cuts COPY) and BLAKE3: +/// the tables between the fixed ones and CPU on this branch, which main's +/// formula does not count. Checked in release builds too, unlike the +/// `debug_assert` in `air_refs`. #[test] fn cpu_proof_start_matches_air_refs_layout() { let elf = asm_elf_bytes(GUEST); let program = Elf::load(&elf).unwrap(); let options = opts(); let honest = craft_proof(&elf, |_| {}).expect("honest proof"); + let register = crate::test_utils::halted_register_commitment(&options, &program, &[]); for include_halt in [false, true] { - for mask in 0u8..64 { - let mut counts = honest.table_counts.clone(); - let bit = |i: u8| usize::from(mask >> i & 1); - counts.keccak = bit(0); - counts.keccak_rnd = bit(1); - counts.ecsm = bit(2); - counts.ecdas = bit(3); - counts.hint = bit(4); - counts.is_wh = bit(5); - let airs = VmAirs::new( - &program, - &options, - false, - &[], - &counts, - &vec![4; counts.cpu], - None, - include_halt, - None, - [0u8; 32], - ); - let refs = airs.air_refs(); - let start = crate::cpu_proof_start(include_halt, &counts); - for k in 0..counts.cpu { - assert_eq!( - refs[start + k].name(), - format!("CPU[{k}]"), - "halt={include_halt} optional={mask:06b}" + for (copy, blake3) in [(0, 0), (3, 1)] { + for mask in 0u8..64 { + let mut counts = honest.table_counts.clone(); + let bit = |i: u8| usize::from(mask >> i & 1); + counts.keccak = bit(0); + counts.keccak_rnd = bit(1); + counts.ecsm = bit(2); + counts.ecdas = bit(3); + counts.hint = bit(4); + counts.is_wh = bit(5); + counts.copy = copy; + counts.blake3 = blake3; + let airs = VmAirs::new( + &program, + &options, + false, + &[], + &counts, + &vec![4; counts.cpu], + None, + include_halt, + None, + register.preprocessed(), ); + let refs = airs.air_refs(); + let start = crate::cpu_proof_start(include_halt, &counts); + for k in 0..counts.cpu { + assert_eq!( + refs[start + k].name(), + format!("CPU[{k}]"), + "halt={include_halt} copy={copy} blake3={blake3} optional={mask:06b}" + ); + } } } } diff --git a/prover/src/tests/decode_tests.rs b/prover/src/tests/decode_tests.rs index a761ac929..039c0b854 100644 --- a/prover/src/tests/decode_tests.rs +++ b/prover/src/tests/decode_tests.rs @@ -237,13 +237,46 @@ fn decode_commitment_zero_bytes_rejects() { } /// DECODE preprocessed commitment for the `sub` asm test ELF at blowup=2, -/// computed offline once. Mirrors how the recursion guest embeds the -/// commitment as a compile-time constant for its inner program. If the -/// AIR or FFT pipeline changes, this drifts and the test fails — -/// regenerate via the `print_decode_commitment_for_sub` helper below. +/// computed offline once UNDER THE BLOCK PIN (`hash_pin::BLOCK_STARK_HASH`, +/// RPX256 here), over rows in PC ORDER. Mirrors how the recursion guest embeds +/// the commitment as a compile-time constant for its inner program. +/// +/// ⚠ A fifth blessed constant, outside the four families +/// `compute_static_commitments` regenerates: it moves with the pin exactly as +/// they do, and `HASH-PINNING.md` lists it with them. If the pin, the AIR or the +/// FFT pipeline changes, this drifts and the test fails — regenerate via the +/// `print_decode_commitment_for_sub` helper below (`--ignored --nocapture`). +/// +/// ⚠⚠ **THREE earlier values exist and none of them is this one.** The two +/// inputs to this merge each carried their own, and the merged branch needs a +/// fourth because it combines both changes: +/// +/// | value | row order | commit hash | +/// |---|---|---| +/// | `e97168d6…` | `instructions.iter()` | stark's default aliases | +/// | `0a710a9c…` | **pc order** | stark's default aliases | +/// | `e6a99f70…` | `instructions.iter()` | **the block pin, RPX256** | +/// | `858a103e…` (this one) | **pc order** | **the block pin, RPX256** | +/// +/// Taking either side's constant would have failed, and for a reason the other +/// side could not see. Regenerated here rather than copied. +/// +/// ★ **Run the helper TWICE and diff its output against what you wrote.** The +/// first transcription of this value read `0x55` where the helper had printed +/// `85`; the second run is what caught it. A one-nibble error here is a +/// constant that is wrong in a way no reasoning finds — the test fails, the +/// value looks plausible, and the obvious conclusion is that the code drifted +/// rather than that the pin was mistyped. The second run costs nothing and is +/// the only check that covers the transcription step at all. +/// +/// The pc sort is why the row order half can no longer move for a reason nobody +/// chose: it was hashbrown's — a function of the hasher, the map's capacity and +/// the insertion sequence rather than of the ELF. Under the sort it is a +/// function of the ELF alone, which is what a constant pinned in a guest has to +/// be. See [`the_decode_trace_does_not_depend_on_the_map_that_carried_it`]. const SUB_DECODE_COMMITMENT_BLOWUP_2: [u8; 32] = [ - 0xe9, 0x71, 0x68, 0xd6, 0x2e, 0xb1, 0xf6, 0x56, 0x61, 0x9d, 0x04, 0x6e, 0x65, 0xed, 0x63, 0x4a, - 0x27, 0xa3, 0x4d, 0xcb, 0x6c, 0x02, 0x11, 0xd7, 0x65, 0xc9, 0xc9, 0xfd, 0x59, 0x34, 0x41, 0x5f, + 0x85, 0x8a, 0x10, 0x3e, 0xe4, 0xe2, 0xe7, 0x9b, 0x08, 0x0b, 0x67, 0xf1, 0xa6, 0x38, 0x5a, 0xfe, + 0x6d, 0xd9, 0x55, 0x7b, 0x5a, 0x0e, 0xac, 0xde, 0xa0, 0x9b, 0xd5, 0xe3, 0x99, 0xb6, 0xe7, 0x74, ]; #[test] @@ -279,3 +312,125 @@ fn print_decode_commitment_for_sub() { eprintln!("SUB_DECODE_COMMITMENT_BLOWUP_2 (sub.elf, blowup=2):"); eprintln!("{c:02x?}"); } + +// ========================================================================= +// Row order is a function of the ELF alone +// ========================================================================= + +/// A distinct instruction per pc, so a permuted trace cannot match a sorted one +/// by accident — every row differs from every other in PACKED_DECODE and IMM. +fn instr_for(pc: u64) -> Instruction { + Instruction::ArithImm { + dst: ((pc / 4) % 30) as u32 + 1, + src: ((pc / 4) % 7) as u32 + 1, + imm: (pc % 2048) as i32 - 1024, + op: ArithOp::Add, + } +} + +/// The same instruction set, reached through two independently-constructed +/// maps: ascending with no reserve, descending with a large one. +/// +/// Different insertion order and different capacity means a different hashbrown +/// bucket layout, hence a different `iter()` order — which is exactly the +/// variation a hashbrown version bump, an added `reserve`, or a change of hasher +/// would introduce, expressed as something a test can construct today. +fn two_maps_of(n: u64) -> (U64HashMap, U64HashMap) { + let pcs: Vec = (0..n).map(|i| 0x1000 + i * 4).collect(); + + let mut ascending: U64HashMap = U64HashMap::default(); + for &pc in &pcs { + ascending.insert(pc, instr_for(pc)); + } + + let mut descending: U64HashMap = U64HashMap::default(); + descending.reserve(1024); + for &pc in pcs.iter().rev() { + descending.insert(pc, instr_for(pc)); + } + + (ascending, descending) +} + +/// ★★★ THE PROPERTY: the DECODE trace is a function of the instruction SET, +/// not of the map that carried it. +/// +/// Sorting by pc is the mechanism; this is the thing that must be true, and it +/// is stated that way on purpose. A test asserting "the rows are sorted" would +/// pass on any total order and would not say why the order matters — whereas a +/// root pinned as a program constant needs exactly this: two parties holding +/// the same ELF, and nothing else in common, produce the same rows. +/// +/// ⚠ This is what fails without the sort. The two maps differ in insertion +/// order and capacity, so `instructions.iter()` walks them differently and the +/// traces come out permuted. Nothing in the system notices today, because +/// prover and verifier both build their map through `instructions_from_elf` and +/// so make the same arbitrary choice — they agree on a construction procedure +/// rather than on the ELF. A hashbrown bump breaks that agreement with no ELF +/// change and no failing test. +#[test] +fn the_decode_trace_does_not_depend_on_the_map_that_carried_it() { + let (ascending, descending) = two_maps_of(300); + + // The premise: the two maps really are walked differently. If hashbrown ever + // made iteration order insertion- and capacity-independent, this test would + // still pass below while testing nothing, so the premise is asserted. + let order_a: Vec = ascending.iter().map(|(&pc, _)| pc).collect(); + let order_b: Vec = descending.iter().map(|(&pc, _)| pc).collect(); + assert_ne!( + order_a, order_b, + "the two maps iterate identically, so this test cannot detect a \ + map-dependent trace — rebuild the maps so they differ" + ); + + let (trace_a, pc_to_row_a) = generate_decode_trace(&ascending); + let (trace_b, pc_to_row_b) = generate_decode_trace(&descending); + + assert_eq!(trace_a.num_rows(), trace_b.num_rows()); + for row in 0..trace_a.num_rows() { + assert_eq!( + trace_a.main_table.get_row(row), + trace_b.main_table.get_row(row), + "row {row} differs between two maps holding the same instructions" + ); + } + + // …and the index agrees too, or `update_multiplicities` would write the + // right counts to the wrong rows. + for (&pc, &row) in pc_to_row_a.iter() { + assert_eq!( + pc_to_row_b.get(&pc), + Some(&row), + "pc {pc:#x} maps to a different row in the two maps" + ); + } +} + +/// ★ THE MECHANISM, pinned separately so the reason stays visible. +/// +/// The property above holds for any canonical order; this says which one, so a +/// future change that keeps determinism but moves the rows has to come here and +/// re-baseline the pins rather than sliding past. +#[test] +fn decode_rows_are_in_ascending_pc_order() { + let (ascending, _) = two_maps_of(64); + let (trace, _) = generate_decode_trace(&ascending); + + // The instruction rows come first, then the CPU padding row, then zeroed + // padding to the next power of two — so only the first `n` are ordered. + let pcs: Vec = (0..64) + .map(|row| { + let lo = *trace.main_table.get(row, cols::PC_0).value(); + let hi = *trace.main_table.get(row, cols::PC_1).value(); + lo | (hi << 32) + }) + .collect(); + + let mut sorted = pcs.clone(); + sorted.sort_unstable(); + assert_eq!( + pcs, sorted, + "the instruction rows are not in ascending pc order" + ); + assert_eq!(pcs[0], 0x1000, "the first row is not the lowest pc"); +} diff --git a/prover/src/tests/disk_spill_tests.rs b/prover/src/tests/disk_spill_tests.rs deleted file mode 100644 index 93945bfff..000000000 --- a/prover/src/tests/disk_spill_tests.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! End-to-end tests forcing `StorageMode::Disk` via the `FORCE_DISK_SPILL` env var. -//! -//! Run with `FORCE_DISK_SPILL=1` set in the environment, e.g. -//! `FORCE_DISK_SPILL=1 cargo test --features disk-spill disk_spill`. Tests -//! fail fast if the var is unset to avoid silent loss of coverage. - -use crate::VmProof; -use crate::tables::MaxRowsConfig; -use crate::test_utils::asm_elf_bytes; -use stark::proof::options::GoldilocksCubicProofOptions; - -fn require_force_disk_spill() { - assert_eq!( - std::env::var("FORCE_DISK_SPILL").as_deref(), - Ok("1"), - "set FORCE_DISK_SPILL=1 before running disk-spill tests", - ); -} - -#[test] -fn test_disk_spill_prove_verify_and_roundtrip_small() { - require_force_disk_spill(); - let elf_bytes = asm_elf_bytes("sub"); - let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid"); - let proof = crate::prove_with_options(&elf_bytes, &opts, &MaxRowsConfig::default()) - .expect("prove failed"); - assert!( - crate::verify_with_options(&proof, &elf_bytes, &opts, None, None).expect("verify failed"), - "verification returned false" - ); - - let bytes = rkyv::to_bytes::(&proof).expect("serialize failed"); - let proof2: VmProof = - rkyv::from_bytes::(&bytes).expect("deserialize failed"); - assert!( - crate::verify_with_options(&proof2, &elf_bytes, &opts, None, None).expect("verify failed"), - "verification failed after serialization roundtrip" - ); -} - -#[test] -fn test_disk_spill_prove_verify_and_roundtrip_chunked() { - require_force_disk_spill(); - let elf_bytes = asm_elf_bytes("all_instructions_64"); - let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid"); - let proof = crate::prove_with_options(&elf_bytes, &opts, &MaxRowsConfig::small()) - .expect("prove failed"); - assert!( - crate::verify_with_options(&proof, &elf_bytes, &opts, None, None).expect("verify failed"), - "verification returned false" - ); - - let bytes = rkyv::to_bytes::(&proof).expect("serialize failed"); - let proof2: VmProof = - rkyv::from_bytes::(&bytes).expect("deserialize failed"); - assert!( - crate::verify_with_options(&proof2, &elf_bytes, &opts, None, None).expect("verify failed"), - "verification failed after serialization roundtrip (chunked)" - ); -} diff --git a/prover/src/tests/ecrec_tests.rs b/prover/src/tests/ecrec_tests.rs new file mode 100644 index 000000000..8a3a5b09a --- /dev/null +++ b/prover/src/tests/ecrec_tests.rs @@ -0,0 +1,649 @@ +//! Tests for the ECREC table: every constraint holds on generated traces (computed +//! rows, bailed rows, padding), and isolated negative checks, one per check the +//! table carries: the parity of R's root, the scalar relations, the bounds, the +//! `xA ≠ xB` witness, the zero key on a bail, the scalar bits off a computed row. + +use crate::tables::ecrec::{EcrecConstraints, EcrecOperation, NUM_CONSTRAINTS, cols}; +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; +use ecsm::{N_BYTES, P_BYTES, compute_ecrec_witness}; +use math::field::element::FieldElement; +use stark::constraints::builder::{ConstraintSet, ProverEvalFolder}; +use stark::frame::Frame; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +fn hex32(s: &str) -> [u8; 32] { + let mut out = [0u8; 32]; + for (i, b) in out.iter_mut().enumerate() { + *b = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).unwrap(); + } + out +} + +/// A signature `r ‖ s`, its hash, the parity of R's y, and its key's x and y, all +/// big-endian. +type Vector = ([u8; 64], [u8; 32], u8, [u8; 32], [u8; 32]); + +/// Two fixed signatures and their keys. +fn vectors() -> Vec { + let v = |r: &str, s: &str, z: &str, par: u8, qx: &str, qy: &str| { + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&hex32(r)); + sig[32..].copy_from_slice(&hex32(s)); + (sig, hex32(z), par, hex32(qx), hex32(qy)) + }; + vec![ + v( + "3284bae4a1b693dfbe8ac67400e4891f3b53681c5159a315561051f3d2c57139", + "97d4b2f2f728a54ed562a12237f6b11d32d626e7fcc24c3470b0729e095bd057", + "c0ffee0011223344556677889900aabbccddeeff00112233445566778899aabb", + 0, + "8733750fdf82edb525a2c9f9d201ad1b26e163b8093f3c8f6eb83377d157edb2", + "ce023840e38085bfcf2b27bd66be6a877f0ec281b073ca0f9ab31cfc0733b62a", + ), + v( + "7962d45b38e8bcf82fa8efa8432a01f20c9a53e24c7d3f11df197cb8e70926da", + "ecec6811812f04794c1f92be72948f95a6cda7874445a39db40aab31c835f2cd", + "0000000000000000000000000000000000000000000000000000000000000042", + 1, + "76d2fdf1302d1fa9556f4df94ec84cefba6d482e54f47c6c2a238c1baa560f0e", + "b754ac7e7a3e09c44184cb451a4f5fb557f32053eb015dffebb655b5cfd54d8a", + ), + ] +} + +fn op(sig: &[u8; 64], msg: &[u8; 32], parity: u8) -> EcrecOperation { + EcrecOperation { + timestamp: 444, + addr_out: 0x1000, + addr_sig: 0x2000, + addr_msg: 0x3000, + witness: compute_ecrec_witness(sig, msg, parity), + } +} + +fn eval(main: Vec) -> Vec { + let n = EcrecConstraints.meta().len(); + let frame = Frame::::new(vec![TableView::new( + vec![main], + vec![vec![]], + )]); + let no_e: Vec> = vec![]; + let offset_e = FieldElement::::zero(); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_e, &no_e, &offset_e); + let mut base = vec![FE::zero(); n]; + let mut ext = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base, &mut ext); + EcrecConstraints.eval(&mut folder); + base +} + +/// Row `row` of the table of `ops`, as values. +fn row_of(ops: &[EcrecOperation], row: usize) -> Vec { + let trace = crate::tables::ecrec::generate_ecrec_trace(ops); + (0..cols::NUM_COLUMNS) + .map(|c| *trace.main_table.get(row, c)) + .collect() +} + +fn failing(main: Vec) -> Vec { + let v = eval(main); + (0..v.len()).filter(|&i| v[i] != FE::zero()).collect() +} + +/// A computed row of the first vector. +fn honest() -> Vec { + let (sig, msg, par, ..) = vectors()[0]; + row_of(&[op(&sig, &msg, par)], 0) +} + +fn set_bytes(main: &mut [FE], col: usize, bytes: &[u8]) { + for (i, &b) in bytes.iter().enumerate() { + main[col + i] = FE::from(b as u64); + } +} + +#[test] +fn constraint_count() { + assert_eq!(EcrecConstraints.meta().len(), NUM_CONSTRAINTS); + assert_eq!(NUM_CONSTRAINTS, 981); +} + +/// Every constraint holds on a table of computed rows, bailed rows and padding, +/// and the computed rows' keys are the vectors'. +#[test] +fn constraints_hold_on_generated_trace() { + let mut ops = Vec::new(); + for (sig, msg, par, qx, qy) in vectors() { + let o = op(&sig, &msg, par); + assert!(o.witness.ok); + let (mut x, mut y) = (o.witness.x_q, o.witness.y_q); + x.reverse(); + y.reverse(); + assert_eq!((x, y), (qx, qy), "the witness's key is the vector's"); + ops.push(o); + // The other parity: a different R and key, still a computed row. + ops.push(op(&sig, &msg, par ^ 1)); + } + // Bailed rows: r = 0, s = N, z = 0. + let (sig, msg, par, ..) = vectors()[0]; + let mut zero_r = sig; + zero_r[..32].fill(0); + let mut n_s = sig; + let mut n_be = N_BYTES; + n_be.reverse(); + n_s[32..].copy_from_slice(&n_be); + for (s, m) in [(zero_r, msg), (n_s, msg), (sig, [0u8; 32])] { + let o = op(&s, &m, par); + assert!(!o.witness.ok); + ops.push(o); + } + let trace = crate::tables::ecrec::generate_ecrec_trace(&ops); + assert!( + trace.num_rows() > ops.len(), + "the table has padding rows too" + ); + for row in 0..trace.num_rows() { + let main: Vec = (0..cols::NUM_COLUMNS) + .map(|c| *trace.main_table.get(row, c)) + .collect(); + let bad = failing(main); + assert!(bad.is_empty(), "row {row}: constraints {bad:?}"); + } +} + +/// The other root of R: no `H` makes `yR[0] = 2·H + PAR` hold for the opposite +/// parity, so R's sign is pinned by the x13 value. +#[test] +fn the_other_root_of_r_is_refused() { + let base = honest(); + let par = base[cols::PAR]; + for h in 0..256u64 { + let mut main = base.clone(); + main[cols::H] = FE::from(h); + // Flip the parity bit the row claims; yR stays. + main[cols::PAR] = FE::one() - par; + assert!(failing(main).contains(&843), "H = {h}"); + } + // The honest row's parity constraint holds. + assert_eq!(eval(base)[843], FE::zero()); +} + +/// A wrong inverse, a wrong scalar bit of u1 or u2: the relation's carries no longer +/// close (some constraint of that relation fires). +#[test] +fn wrong_scalars_are_refused() { + let base = honest(); + let mut w = base.clone(); + w[cols::W] += FE::one(); + assert!(failing(w).iter().any(|&i| (517..582).contains(&i)), "w"); + for (bits, range) in [(cols::U1, 582..647), (cols::U2, 647..712)] { + let mut m = base.clone(); + m[bits + 9] = FE::one() - m[bits + 9]; + assert!( + failing(m).iter().any(|i| range.contains(i)), + "bits at {bits}" + ); + } +} + +/// `r ≥ N` on a computed row: the bound's overflow constraint fires. +#[test] +fn r_at_or_above_n_is_refused() { + let mut main = honest(); + set_bytes(&mut main, cols::R, &N_BYTES); + let bad = failing(main); + // RLtN is the first bound: indices 844..852. + assert!(bad.contains(&851), "{bad:?}"); +} + +/// `yR` and the key's coordinates must be below p: `p` itself is refused. +#[test] +fn coordinates_at_p_are_refused() { + for (col, overflow) in [ + (cols::YR, 851 + 32), + (cols::XQ, 851 + 56), + (cols::YQ, 851 + 64), + ] { + let mut main = honest(); + set_bytes(&mut main, col, &P_BYTES); + assert!(failing(main).contains(&overflow), "column {col}"); + } +} + +/// `xA = xB` on a computed row (the chord undefined): no DINV satisfies the +/// non-degeneracy constraint. +#[test] +fn equal_chain_x_is_refused() { + let mut main = honest(); + for i in 0..32 { + main[cols::XB + i] = main[cols::XA + i]; + } + for c in 0..8 { + main[cols::DINV + c] = FE::from(12345u64 + c as u64); + } + assert!(failing(main).contains(&916)); +} + +/// A bail must write a zero key: OK = 0 with a key byte set fires. +#[test] +fn a_bail_writes_a_zero_key() { + let (sig, msg, par, ..) = vectors()[0]; + let mut zero_r = sig; + zero_r[..32].fill(0); + let mut main = row_of(&[op(&zero_r, &msg, par)], 0); + assert!(failing(main.clone()).is_empty(), "the honest bail holds"); + assert_eq!(main[cols::OK], FE::zero()); + main[cols::XQ + 5] = FE::from(7u64); + assert!(failing(main).contains(&(917 + 5))); +} + +/// Scalar bits on a row that computed nothing would fire phantom `Bit` receives. +#[test] +fn scalar_bits_off_a_computed_row_are_refused() { + let (sig, msg, par, ..) = vectors()[0]; + let mut zero_r = sig; + zero_r[..32].fill(0); + let mut main = row_of(&[op(&zero_r, &msg, par)], 0); + main[cols::U2 + 3] = FE::one(); + assert!(failing(main).contains(&516)); + // And OK on a padding row (MU = 0). + let mut pad = vec![FE::zero(); cols::NUM_COLUMNS]; + pad[cols::OK] = FE::one(); + assert!(failing(pad).contains(&3)); +} + +/// The BITWISE lookups the builder counts for ECREC rows are exactly the ones the +/// table sends: every `AreBytes`, `IsHalfword` and `Zero` interaction evaluated on +/// every row (computed, bailed, padding), as a multiset. +#[test] +fn the_counted_lookups_are_the_tables_sends() { + use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; + use crate::tables::types::BusId; + use stark::lookup::Multiplicity; + use std::collections::HashMap; + + let mut ops: Vec = vectors() + .iter() + .flat_map(|(sig, msg, par, ..)| [op(sig, msg, *par), op(sig, msg, par ^ 1)]) + .collect(); + let (sig, msg, par, ..) = vectors()[1]; + let mut zero_r = sig; + zero_r[..32].fill(0); + ops.push(op(&zero_r, &msg, par)); + let trace = crate::tables::ecrec::generate_ecrec_trace(&ops); + let mut sent: HashMap = HashMap::new(); + for row in 0..trace.num_rows() { + let get = |c: usize| *trace.main_table.get(row, c); + for bi in crate::tables::ecrec::bus_interactions() { + let kind = match bi.bus_id { + id if id == BusId::AreBytes as u64 => 0, + id if id == BusId::IsHalfword as u64 => 1, + id if id == BusId::Zero as u64 => 2, + _ => continue, + }; + let Multiplicity::Column(mc) = bi.multiplicity else { + panic!("a range check gated by a column"); + }; + let m = get(mc).canonical(); + if m == 0 { + continue; + } + assert_eq!(m, 1); + let vals: Vec = bi + .values + .iter() + .flat_map(|v| v.combine_from(get)) + .map(|f| f.canonical()) + .collect(); + let lookup = match kind { + 0 => { + assert_eq!(vals[1], 0); + BitwiseOperation::byte_op(BitwiseOperationType::AreBytes, vals[0] as u8, 0) + } + 1 => { + assert!(vals[0] < 1 << 16, "a halfword: {}", vals[0]); + BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (vals[0] & 0xFF) as u8, + (vals[0] >> 8) as u8, + ) + } + _ => { + assert_eq!(vals[1], 0, "a nonzero check"); + BitwiseOperation::zero(vals[0] as u32) + } + }; + *sent.entry(lookup).or_default() += 1; + } + } + let mut counted: HashMap = HashMap::new(); + for l in crate::tables::trace_builder::collect_bitwise_from_ecrec(&ops) { + *counted.entry(l).or_default() += 1; + } + assert!(!sent.is_empty()); + assert_eq!(sent, counted); +} + +/// The swapped-chains forgery the block test refuses on the bus alone: its ECREC row +/// satisfies every ECREC constraint, and its chains' and addition's ECDAS rows every +/// ECDAS constraint — so the second chain's key is the only thing left to refuse it. +#[test] +fn the_swapped_chains_forgery_satisfies_every_constraint() { + use crate::tables::ecdas::{EcdasConstraints, EcdasOperation, generate_ecdas_trace}; + let (sig, msg, par, ..) = vectors()[0]; + let mut forged = op(&sig, &msg, par); + assert_eq!( + forged.witness.r, + crate::tables::trace_builder::FORGED_ECREC_R + ); + let honest_key = (forged.witness.x_q, forged.witness.y_q); + crate::tables::trace_builder::swap_ecrec_chains(&mut forged.witness); + assert_ne!((forged.witness.x_q, forged.witness.y_q), honest_key); + let row = row_of(std::slice::from_ref(&forged), 0); + assert!(failing(row).is_empty(), "the forged ECREC row holds"); + + let w = &forged.witness; + let steps: Vec = w + .chain1 + .iter() + .chain(&w.chain2) + .chain(w.add.as_ref()) + .map(|step| EcdasOperation { + timestamp: 444, + step: step.clone(), + }) + .collect(); + let trace = generate_ecdas_trace(&steps); + let n = EcdasConstraints.meta().len(); + for r in 0..trace.num_rows() { + let main: Vec = (0..crate::tables::ecdas::cols::NUM_COLUMNS) + .map(|c| *trace.main_table.get(r, c)) + .collect(); + let frame = Frame::::new(vec![TableView::new( + vec![main], + vec![vec![]], + )]); + let no_e: Vec> = vec![]; + let offset_e = FieldElement::::zero(); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_e, &no_e, &offset_e); + let mut base = vec![FE::zero(); n]; + let mut ext = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base, &mut ext); + EcdasConstraints.eval(&mut folder); + assert!(base.iter().all(|v| *v == FE::zero()), "ECDAS row {r}"); + } +} + +/// The table's generator constants are G. +#[test] +fn the_generator_constants_are_g() { + let (gx, gy) = ecsm::generator_le(); + let (sig, msg, par, ..) = vectors()[0]; + let w = op(&sig, &msg, par).witness; + let first = &w.chain1[0]; + assert_eq!( + (first.x_g, first.y_g), + (gx, gy), + "the first chain runs on G" + ); + assert_eq!( + crate::tables::ecrec::generator_le(), + (gx, gy), + "the table's seed and drain constants are G" + ); +} + +// ============================================================================= +// The `< N` bounds on s, u1 and u2 and the `< p` bounds on xA and xB: each is the +// only constraint refusing a row that is otherwise consistent (the relation's +// quotient and carries recomputed for the out-of-range value). Without u1 < N or +// u2 < N a prover takes the true scalar 2 (r = −z/2, or s = 2r), claims 2 + N, and +// the chain's last addition becomes G + G through ECDAS's chord branch, where λ +// is free; without s < N the signature (r, s + N), which Ethereum refuses, +// recovers the same key. +// ============================================================================= + +use crate::tables::ecrec::{Bound, CARRY_OFFSET_MODN}; + +/// Index of `bound`'s closing-carry constraint: the bounds are emitted in +/// `Bound::ALL` order from index 844, eight constraints each (seven carry bits, +/// then the carry out). +fn closing(bound: Bound) -> usize { + let k = Bound::ALL + .iter() + .position(|b| std::mem::discriminant(b) == std::mem::discriminant(&bound)) + .expect("a bound"); + 844 + 8 * k + 7 +} + +fn add_le(a: &[u8], b: &[u8], len: usize) -> Vec { + let mut out = vec![0u8; len]; + let mut carry = 0u16; + for (i, o) in out.iter_mut().enumerate() { + let s = *a.get(i).unwrap_or(&0) as u16 + *b.get(i).unwrap_or(&0) as u16 + carry; + *o = s as u8; + carry = s >> 8; + } + assert_eq!(carry, 0, "add overflow"); + out +} + +fn sub_le(a: &[u8], b: &[u8], len: usize) -> Vec { + let mut out = vec![0u8; len]; + let mut borrow = 0i16; + for (i, o) in out.iter_mut().enumerate() { + let d = *a.get(i).unwrap_or(&0) as i16 - *b.get(i).unwrap_or(&0) as i16 - borrow; + borrow = i16::from(d < 0); + *o = d.rem_euclid(256) as u8; + } + assert_eq!(borrow, 0, "sub underflow"); + out +} + +fn ge_le(a: &[u8], b: &[u8]) -> bool { + let n = a.len().max(b.len()); + for i in (0..n).rev() { + let (x, y) = (*a.get(i).unwrap_or(&0), *b.get(i).unwrap_or(&0)); + if x != y { + return x > y; + } + } + true +} + +/// Big-endian of a little-endian 32-byte value. +fn be(le: &[u8]) -> [u8; 32] { + std::array::from_fn(|i| le[31 - i]) +} + +/// `(2·x) mod N` for `x < N`, little-endian. +fn double_mod_n(x: &[u8; 32]) -> [u8; 32] { + let d = add_le(x, x, 33); + let r = if ge_le(&d, &N_BYTES) { + sub_le(&d, &N_BYTES, 33) + } else { + d + }; + r[..32].try_into().unwrap() +} + +/// `N − x` for `0 < x < N`. +fn neg_mod_n(x: &[u8; 32]) -> [u8; 32] { + sub_le(&N_BYTES, x, 32).try_into().unwrap() +} + +/// Carries of `Σ_i S_i 256^i = 0` with `S_i = Σ a_j b_{i−j} − Σ q_j m_{i−j} + sign·c_i`. +fn carries(a: &[u8], b: &[u8], q: &[u8], m: &[u8], c: &[u8], sign: i128) -> [i64; 64] { + let g = |v: &[u8], j: usize| *v.get(j).unwrap_or(&0) as i128; + let mut out = [0i64; 64]; + let mut prev = 0i128; + for (i, o) in out.iter_mut().enumerate() { + let mut s = 0i128; + for j in 0..=i { + s += g(a, j) * g(b, i - j) - g(q, j) * g(m, i - j); + } + s += sign * g(c, i); + let t = prev + s; + assert_eq!(t.rem_euclid(256), 0, "relation does not hold at limb {i}"); + prev = t / 256; + *o = prev as i64; + } + assert_eq!(out[63], 0, "relation does not close"); + out +} + +fn fe_i64(c: i64) -> FE { + if c >= 0 { + FE::from(c as u64) + } else { + FE::zero() - FE::from((-c) as u64) + } +} + +/// A computed row of `sig`, `msg`, `parity` and its witness. +fn computed(sig: &[u8; 64], msg: &[u8; 32], parity: u8) -> (Vec, ecsm::EcrecWitness) { + let o = op(sig, msg, parity); + assert!(o.witness.ok, "the honest witness computes the key"); + let w = o.witness.clone(); + (row_of(&[o], 0), w) +} + +fn set_bits(main: &mut [FE], col: usize, bytes: &[u8]) { + for b in 0..256 { + main[col + b] = FE::from(((bytes[b / 8] >> (b % 8)) & 1) as u64); + } +} + +fn set_carries(main: &mut [FE], col: usize, c: &[i64; 64]) { + for (i, &v) in c.iter().enumerate() { + if i < 63 { + assert!( + (0..1 << 16).contains(&(v + CARRY_OFFSET_MODN)), + "carry {i} = {v} outside the IsHalfword window: the bus would refuse first" + ); + } + main[col + i] = fe_i64(v); + } +} + +fn set_halfwords(main: &mut [FE], col: usize, bytes: &[u8]) { + for j in 0..16 { + main[col + j] = FE::from(bytes[2 * j] as u64 + ((bytes[2 * j + 1] as u64) << 8)); + } +} + +/// `r` and `s` of the first vector, big-endian. +fn r_s() -> ([u8; 32], [u8; 32]) { + let (sig, ..) = vectors()[0]; + (sig[..32].try_into().unwrap(), sig[32..].try_into().unwrap()) +} + +fn sig_of(r_be: &[u8; 32], s_be: &[u8; 32]) -> [u8; 64] { + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(r_be); + sig[32..].copy_from_slice(s_be); + sig +} + +fn two() -> [u8; 32] { + let mut two = [0u8; 32]; + two[0] = 2; + two +} + +const Z: &str = "c0ffee0011223344556677889900aabbccddeeff00112233445566778899aabb"; + +/// `u1' = u1 + N` with `u1 = 2` (`z = −2r mod N`), `qz' = qz + 1`: the u1 relation +/// still closes, every bit is a bit, every carry is in its window; only U1LtN's +/// closing carry refuses the row. +#[test] +fn u1_plus_n_is_refused_only_by_its_bound() { + let (r_be, s_be) = r_s(); + let z_le = neg_mod_n(&double_mod_n(&be(&r_be))); + let (mut main, w) = computed(&sig_of(&r_be, &s_be), &be(&z_le), 0); + assert_eq!(w.u1, two(), "u1 = 2"); + assert!(failing(main.clone()).is_empty(), "the honest row holds"); + + let u1p = add_le(&w.u1, &N_BYTES, 32); + let qzp = add_le(&w.qz, &[1], 32); + let cz = carries(&w.z, &w.w, &qzp, &N_BYTES, &u1p, 1); + set_bits(&mut main, cols::U1, &u1p); + set_bytes(&mut main, cols::QZ, &qzp); + set_carries(&mut main, cols::CZ, &cz); + // The best sub witness the prover has: (u1' − N) mod 2^256 = 2. + set_halfwords(&mut main, cols::U1_SUB_N, &two()); + main[cols::LEN1] = FE::from(255u64); + assert_eq!(failing(main), vec![closing(Bound::U1LtN)]); +} + +/// `u2' = u2 + N` with `u2 = 2` (`s = 2r mod N`), `qu' = qu − 1`: only U2LtN refuses. +#[test] +fn u2_plus_n_is_refused_only_by_its_bound() { + let (r_be, _) = r_s(); + let s_le = double_mod_n(&be(&r_be)); + let (mut main, w) = computed(&sig_of(&r_be, &be(&s_le)), &hex32(Z), 0); + assert_eq!(w.u2, two(), "u2 = 2"); + assert!(failing(main.clone()).is_empty(), "the honest row holds"); + + let u2p = add_le(&w.u2, &N_BYTES, 32); + let qup = sub_le(&w.qu, &[1], 32); + let cu = carries(&w.s, &w.w, &qup, &N_BYTES, &u2p, -1); + set_bits(&mut main, cols::U2, &u2p); + set_bytes(&mut main, cols::QU, &qup); + set_carries(&mut main, cols::CU, &cu); + set_halfwords(&mut main, cols::U2_SUB_N, &two()); + main[cols::LEN2] = FE::from(255u64); + assert_eq!(failing(main), vec![closing(Bound::U2LtN)]); +} + +/// `s' = s + N` with `s = 5` (Ethereum refuses `s ≥ N`; the same key would come +/// out: `u2` unchanged, `qu' = qu + w`): only SLtN refuses. +#[test] +fn s_plus_n_is_refused_only_by_its_bound() { + let (r_be, _) = r_s(); + let mut s_le = [0u8; 32]; + s_le[0] = 5; + let (mut main, w) = computed(&sig_of(&r_be, &be(&s_le)), &hex32(Z), 0); + assert!(failing(main.clone()).is_empty(), "the honest row holds"); + + let sp = add_le(&w.s, &N_BYTES, 32); + let qup = add_le(&w.qu, &w.w, 32); + let cu = carries(&sp, &w.w, &qup, &N_BYTES, &w.u2, -1); + set_bytes(&mut main, cols::S, &sp); + set_bytes(&mut main, cols::QU, &qup); + set_carries(&mut main, cols::CU, &cu); + set_halfwords(&mut main, cols::S_SUB_N, &s_le); + assert_eq!(failing(main), vec![closing(Bound::SLtN)]); +} + +/// `xA` or `xB` set to `p` (a non-canonical 0), the `xA ≠ xB` inverse re-solved: +/// only that coordinate's bound refuses. +#[test] +fn a_chain_x_at_p_is_refused_only_by_its_bound() { + let (sig, msg, par, ..) = vectors()[0]; + for (col, sub_col, bound) in [ + (cols::XA, cols::XA_SUB_P, Bound::XaLtP), + (cols::XB, cols::XB_SUB_P, Bound::XbLtP), + ] { + let (mut main, _) = computed(&sig, &msg, par); + set_bytes(&mut main, col, &P_BYTES); + set_halfwords(&mut main, sub_col, &[0u8; 32]); + let word = |m: &[FE], base: usize, c: usize| -> i64 { + (0..4) + .map(|k| (m[base + 4 * c + k].canonical() as i64) << (8 * k)) + .sum() + }; + for c in 0..8 { + main[cols::DINV + c] = FE::zero(); + } + let c = (0..8) + .find(|&c| word(&main, cols::XB, c) != word(&main, cols::XA, c)) + .expect("a nonzero word difference"); + let d = word(&main, cols::XB, c) - word(&main, cols::XA, c); + main[cols::DINV + c] = fe_i64(d).inv().unwrap(); + assert_eq!(failing(main), vec![closing(bound)], "{bound:?}"); + } +} diff --git a/prover/src/tests/gpack_tests.rs b/prover/src/tests/gpack_tests.rs new file mode 100644 index 000000000..2b1e714f8 --- /dev/null +++ b/prover/src/tests/gpack_tests.rs @@ -0,0 +1,461 @@ +//! G-pack (`tables::gpack`): a table written packed as it is generated is the +//! 64-bit table packed, byte for byte — on every path the writer takes, and for +//! every table of a windowed build in the WHIR block's configuration against +//! the same build without G-pack — and the comparison that says so catches a +//! column written at the wrong width or offset. + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; +use multilinear::narrow::NarrowColumns; +use stark::narrow::mutation::{self, Mutation}; +use stark::trace::TraceTable; + +use crate::tables::MaxRowsConfig; +use crate::tables::gpack::{TraceForm, WidthHint, generate_main, thread_counts}; +use crate::tables::trace_builder::{Traces, WindowedTraceBuilder}; +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField, VmTable}; +use crate::test_utils::asm_elf_bytes; + +type Table = TraceTable; + +fn run(name: &str) -> (Elf, Vec) { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + (program, logs) +} + +/// A windowed build of `logs` in the WHIR block's configuration — KECCAK_RND +/// built as its tables of `keccak_rnd_rows` rows at the finish (0: one table), +/// the streamed ops dropped, the finish's tables packed as they are generated — +/// over windows of `window` cycles, each streamed chunk packed. With G-pack, +/// each chunk is written packed by its job (on this thread, or on the pool +/// with `parallel`) and the finish writes its tables packed; without, each is +/// built at 8 bytes a cell and packed, as before. +#[allow(clippy::too_many_arguments)] +fn block_build( + program: &Elf, + input: &[u8], + logs: &[Log], + max_rows: &MaxRowsConfig, + window: usize, + keccak_rnd_rows: usize, + gpack: bool, + parallel: bool, +) -> Traces { + use rayon::prelude::*; + let mut builder = WindowedTraceBuilder::new(program, input, max_rows).expect("the builder"); + if keccak_rnd_rows > 0 { + builder = builder + .keccak_rnd_chunks_at_finish(keccak_rnd_rows) + .expect("before any window"); + } + builder = builder + .drop_streamed_ops() + .expect("before any window") + .pack_finished_tables(); + if gpack { + builder = builder.generate_packed(); + } + let form = if gpack { + TraceForm::Narrow + } else { + TraceForm::Wide + }; + let generate = |job: crate::tables::trace_builder::ChunkJob| { + let mut chunk = job.generate_as(form); + chunk.trace.pack_main_narrow(); + chunk + }; + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + let jobs = builder.push_jobs(w).expect("a window"); + if parallel { + chunks.par_extend(jobs.into_par_iter().map(generate)); + } else { + chunks.extend(jobs.into_iter().map(generate)); + } + } + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + traces +} + +/// Every table of `t`, named by kind and instance. +fn tables(t: &Traces) -> Vec<(String, &Table)> { + let lists: [(&str, &Vec

); 21] = [ + ("CPU", &t.cpus), + ("MEMW_R", &t.memw_registers), + ("MEMW_A", &t.memw_aligneds), + ("MEMW", &t.memws), + ("LOAD", &t.loads), + ("STORE", &t.stores), + ("SHIFT", &t.shifts), + ("CPU32", &t.cpu32s), + ("COPY", &t.copies), + ("KECCAK", &t.keccaks), + ("KECCAK_RND", &t.keccak_rnds), + ("ECSM", &t.ecsms), + ("ECDAS", &t.ecdases), + ("HINT", &t.hints), + ("PAGE", &t.pages), + ("LT", &t.lts), + ("MUL", &t.muls), + ("DVRM", &t.dvrms), + ("BRANCH", &t.branches), + ("EQ", &t.eqs), + ("BYTEWISE", &t.bytewises), + ]; + let mut out = Vec::new(); + for (name, list) in lists { + for (i, table) in list.iter().enumerate() { + out.push((format!("{name}[{i}]"), table)); + } + } + for (name, table) in [ + ("BITWISE", &t.bitwise), + ("DECODE", &t.decode), + ("REGISTER", &t.register), + ("HALT", &t.halt), + ("KECCAK_RC", &t.keccak_rc), + ("BLAKE3", &t.blake3), + ] { + out.push((name.to_string(), table)); + } + out +} + +/// A 64-bit table's raw words, row-major. +fn raw_words(t: &Table) -> Vec { + t.main_table + .row_major_data() + .iter() + .map(|v| *v.value()) + .collect() +} + +/// A table's raw words, row-major: widened when it is packed. +fn words(t: &Table) -> Vec { + match t.narrow_main() { + Some(packed) => { + let (rows, cols) = (packed.rows(), packed.cols()); + let mut words = vec![0u64; rows * cols]; + for c in 0..cols { + for (r, w) in packed.column(c).into_iter().enumerate() { + words[r * cols + c] = w; + } + } + words + } + None => raw_words(t), + } +} + +/// Each table's name and digest: its packed form's (rows, widths, bytes), or, +/// for a table not packed, its words'. +fn table_digests(t: &Traces) -> Vec<(String, String)> { + use rayon::prelude::*; + tables(t) + .into_par_iter() + .map(|(name, table)| { + let mut h = blake3::Hasher::new(); + let digest = match table.narrow_main() { + Some(packed) => { + h.update(&(packed.rows() as u64).to_le_bytes()); + h.update(packed.widths()); + h.update(packed.data()); + format!("packed {}", &h.finalize().to_hex()[..32]) + } + None => { + h.update(&(table.main_table.width as u64).to_le_bytes()); + for w in raw_words(table) { + h.update(&w.to_le_bytes()); + } + format!("words {}", &h.finalize().to_hex()[..32]) + } + }; + (name, digest) + }) + .collect() +} + +/// How a table differs from the reference's. +#[derive(Debug, PartialEq, Eq)] +enum Miss { + /// Another table count, width or height. + Shape, + /// Other packed bytes (rows, widths and every byte), or packed in one + /// build and not the other. + Bytes, + /// Other words. + Words, +} + +/// ★ The gate's comparison: every table of `got` has the shape, the words and +/// the packed bytes (or the unpacked form) of `reference`'s; and how many of +/// `got`'s tables are packed. +fn misses(reference: &Traces, got: &Traces) -> (Vec<(String, Miss)>, usize) { + let (want, have) = (tables(reference), tables(got)); + let mut out = Vec::new(); + if want.len() != have.len() { + out.push(("tables".to_string(), Miss::Shape)); + return (out, 0); + } + let mut packed = 0; + for ((name, a), (_, b)) in want.iter().zip(&have) { + let shape = |t: &Table| (t.main_table.width, t.main_table.height); + if shape(a) != shape(b) { + out.push((name.clone(), Miss::Shape)); + continue; + } + packed += usize::from(b.narrow_main().is_some()); + if a.narrow_main() != b.narrow_main() { + out.push((name.clone(), Miss::Bytes)); + } + if words(a) != words(b) { + out.push((name.clone(), Miss::Words)); + } + } + (out, packed) +} + +/// Columns topping out at `tops[c]` (in the row before the last), one written +/// as a field element above the modulus; column 3 a flag. +fn synthetic(t: &mut impl VmTable, rows: usize, tops: [u64; 2]) { + for r in 0..rows { + let top = r == rows - 2; + t.set_u64(r, 0, r as u64 % 200); + for (c, &m) in tops.iter().enumerate() { + let v = if top { + m + } else { + (r as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) % (m / 2 + 1) + }; + if v != 0 { + t.set_u64(r, 1 + c, v); + } + } + t.set_bool(r, 3, r % 3 == 0); + if r % 5 == 0 { + // Reduced mod p as `FE::from` reduces it: 2^32 − 2. + t.set_fe(r, 4, FE::from(u64::MAX)); + } + } +} + +/// `synthetic` at 8 bytes a cell, as a generator writes it today. +fn synthetic_wide(rows: usize, tops: [u64; 2]) -> Table { + let mut trace = TraceTable::new_main(crate::tables::types::zeroed_fe_vec(rows * 5), 5, 1); + synthetic(&mut trace.main_table, rows, tops); + trace +} + +/// ★ Every path `generate_main!` takes gives the 64-bit table packed +/// (`NarrowColumns::pack_row_major`): no widths yet (built wide, then packed, +/// the widths learned), the widths right (written packed as it is), a word +/// wider than its column (written again at the widths the trace needs), +/// columns narrower than learned (narrowed in place). `TraceForm::Wide` gives +/// the 64-bit table. +#[test] +fn every_path_of_the_writer_gives_the_wide_table_packed() { + static HINT: WidthHint = WidthHint::new(); + let rows = 1000; + let build = |form, tops| generate_main!(form, &HINT, rows, 5, |t| synthetic(t, rows, tops)); + // (tops, the path: as written, narrowed, written again, wide then packed). + let steps = [ + ([0x1234, 0xff], [0, 0, 0, 1]), + ([0x1234, 0xff], [1, 0, 0, 0]), + ([0x1_0000_0000, 0xff], [0, 0, 1, 0]), + ([0x12, 0x1_0000], [0, 0, 1, 0]), + ([0x12, 0x12], [0, 1, 0, 0]), + ]; + for (i, (tops, path)) in steps.into_iter().enumerate() { + let wide = synthetic_wide(rows, tops); + let before = thread_counts(); + let got = build(TraceForm::Narrow, tops); + let after = thread_counts(); + let taken: Vec = (0..4).map(|k| after[k] - before[k]).collect(); + assert_eq!(taken, path, "step {i}: the path"); + let want = NarrowColumns::pack_row_major(&words(&wide), 5); + assert_eq!( + got.narrow_main(), + want.as_ref(), + "step {i}: the packed bytes" + ); + assert_eq!(words(&got), words(&wide), "step {i}: the words"); + let plain = build(TraceForm::Wide, tops); + assert!(plain.narrow_main().is_none()); + assert_eq!( + plain.main_table, wide.main_table, + "step {i}: the 64-bit form" + ); + assert_eq!(thread_counts(), after, "step {i}: the 64-bit form"); + } +} + +/// ★ The laptop gate: every table of a windowed build in the WHIR block's +/// configuration under G-pack — the streamed chunks written packed by their +/// jobs, the finish's tables as it generates them — has the packed bytes and +/// the words of the same build without G-pack; on four programs, at chunk +/// sizes that stream every table and cut KECCAK_RND, twice (the second build +/// writes every kind at the widths the first learned). +#[test] +fn every_table_written_packed_is_the_wide_table_packed() { + for (name, max_rows, kr_rows) in [ + ("sub", MaxRowsConfig::small(), 0), + ("all_instructions_64", MaxRowsConfig::small(), 0), + ("all_instructions_64", MaxRowsConfig::uniform(4), 0), + ("test_keccak", MaxRowsConfig::small(), 64), + ("test_keccak_multi", MaxRowsConfig::small(), 64), + ] { + let (program, logs) = run(name); + let reference = block_build(&program, &[], &logs, &max_rows, 7, kr_rows, false, false); + for pass in 0..2 { + let before = thread_counts(); + let got = block_build(&program, &[], &logs, &max_rows, 33, kr_rows, true, false); + let after = thread_counts(); + let (bad, packed) = misses(&reference, &got); + assert!(bad.is_empty(), "{name}/{pass}: {bad:?}"); + assert!(packed > 0, "{name}/{pass}: nothing packed"); + if pass == 1 && logs.len() > 2 * max_rows.cpu { + assert!( + after[0] + after[1] + after[2] > before[0] + before[1] + before[2], + "{name}: no chunk was written packed" + ); + } + } + } +} + +/// ★ The gate is load-bearing: the streamed chunks written with one column a +/// width too wide (the same words, other bytes) or one column's bytes shifted +/// fail its comparison — the first on the bytes alone, the second on the +/// words. +#[test] +fn a_column_at_the_wrong_width_or_offset_fails_the_gate() { + let max_rows = MaxRowsConfig::small(); + let (program, logs) = run("all_instructions_64"); + let reference = block_build(&program, &[], &logs, &max_rows, 33, 0, false, false); + // Every streamed kind's widths learned, so its chunks are written packed. + let clean = block_build(&program, &[], &logs, &max_rows, 33, 0, true, false); + assert!(misses(&reference, &clean).0.is_empty()); + for m in [Mutation::WidenColumn(0), Mutation::ShiftColumn(0)] { + mutation::set(Some(m)); + let bent = block_build(&program, &[], &logs, &max_rows, 33, 0, true, false); + mutation::set(None); + let (bad, _) = misses(&reference, &bent); + let (bytes, words) = ( + bad.iter().any(|(_, k)| *k == Miss::Bytes), + bad.iter().any(|(_, k)| *k == Miss::Words), + ); + match m { + Mutation::WidenColumn(_) => assert!(bytes && !words, "{m:?}: {bad:?}"), + Mutation::ShiftColumn(_) => assert!(words, "{m:?}: {bad:?}"), + } + } +} + +/// The box test's builds and digests on small programs: the build with G-pack +/// and the build without give every table the same digest. +#[test] +fn the_real_block_comparison_holds_on_small_programs() { + for (name, max_rows, kr_rows) in [ + ("all_instructions_64", MaxRowsConfig::small(), 0), + ("test_keccak_multi", MaxRowsConfig::small(), 64), + ] { + let (program, logs) = run(name); + let want = table_digests(&block_build( + &program, + &[], + &logs, + &max_rows, + 33, + kr_rows, + false, + true, + )); + assert!(want.iter().any(|(_, d)| d.starts_with("packed"))); + for _ in 0..2 { + let got = table_digests(&block_build( + &program, + &[], + &logs, + &max_rows, + 33, + kr_rows, + true, + true, + )); + assert_eq!(got, want, "{name}"); + } + } +} + +/// ★ Box only (`--ignored`): on a real block (`BLOCK_WHIR_ELF`, +/// `BLOCK_WHIR_INPUT`) at the WHIR block's caps and windows, every table of the +/// windowed build under G-pack has the packed bytes of the build without it — +/// twice in one process, the second build writing every kind at the widths +/// the first learned. Prints `GPACK` lines. +#[test] +#[ignore = "a real block: box only (BLOCK_WHIR_ELF, BLOCK_WHIR_INPUT)"] +fn every_table_of_a_real_block_written_packed_is_the_wide_table_packed() { + use crate::tables::gpack::{counts, reset_counts}; + let elf = std::fs::read(std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF")).unwrap(); + let input = + std::fs::read(std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT")).unwrap(); + let program = Elf::load(&elf).expect("the ELF loads"); + let options = crate::block_whir::BlockOptions::production(); + let window = 1usize << options.window_log2.expect("the block streams"); + let kr_rows = 1usize << options.keccak_rnd_rows_log2; + let logs = Executor::new(&program, input.clone()) + .expect("the executor starts") + .run() + .expect("the block runs") + .logs; + let max_rows = &options.max_rows; + let t = std::time::Instant::now(); + let want = table_digests(&block_build( + &program, &input, &logs, max_rows, window, kr_rows, false, true, + )); + let packed = want.iter().filter(|(_, d)| d.starts_with("packed")).count(); + println!( + "GPACK off: {} tables ({packed} packed) in {:.2} s", + want.len(), + t.elapsed().as_secs_f64() + ); + let mut equal = 0; + for pass in 0..2 { + reset_counts(); + let t = std::time::Instant::now(); + let got = table_digests(&block_build( + &program, &input, &logs, max_rows, window, kr_rows, true, true, + )); + let secs = t.elapsed().as_secs_f64(); + let [direct, narrowed, again, wide] = counts(); + let differ: Vec<&String> = want + .iter() + .zip(&got) + .filter(|(a, b)| a != b) + .map(|(a, _)| &a.0) + .collect(); + println!( + "GPACK on, build {pass}: {} tables in {secs:.2} s · {direct} written packed, {narrowed} \ + narrowed after, {again} written again, {wide} built wide then packed · {} differ {differ:?}", + got.len(), + differ.len() + ); + if got.len() == want.len() && differ.is_empty() { + equal += 1; + } + } + println!("GPACK RESULT: {equal}/2 G-pack builds have every table's packed bytes"); + assert_eq!(equal, 2); +} diff --git a/prover/src/tests/grouped_run_claim_tests.rs b/prover/src/tests/grouped_run_claim_tests.rs new file mode 100644 index 000000000..e017efc73 --- /dev/null +++ b/prover/src/tests/grouped_run_claim_tests.rs @@ -0,0 +1,109 @@ +//! The column claims of one grouped run, from the prover crate (whose tests +//! CI runs): each column's batching power is indexed in the whole group, so a +//! claim cannot move between the run's polynomials with their sum kept. The +//! leaf replays the same batching (`lfm::whir_stacked`), and a leaf that +//! deviated would refuse honest proofs. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as F, +}; +use multilinear::{ + mle::Mle, + stacked_eval::{Claimed, StackedCommitment, prove, verify}, + stacking::{StackedLayout, borrow}, + whir_chain::{ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, LeafOrder, WhirFolds}, + whir_hash::{P1Whir, WhirHash}, +}; + +type FE = FieldElement; +type EE = FieldElement; + +fn column(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_add(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 11)) + .collect(), + ) + .unwrap() +} + +fn sponge() -> ::Sponge { + let mut t = P1Whir::sponge(); + t.append_bytes(b"a grouped run's claims"); + t +} + +/// ★ A run of three polynomials (one column each) under the composed format +/// (one chain, coset order, blowup 2), proved over the TRUE polynomials for +/// claims that move one unit of value from polynomial 0's column to +/// polynomial 1's. The claims' sum is unchanged; only the column batching's +/// distinct exponents (each column's power indexed in the whole group) can +/// tell. Refused; the honest claims verify. +#[test] +fn a_claim_traded_between_a_runs_polynomials_is_refused() { + let n = 8; + let cfg = ChainConfig { + log_blowup: 1, + log_folding: 4, + num_queries: 9, + grind: GrindBits::default(), + format: ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + group: GroupFold::Indexed, + leaves: LeafOrder::Coset, + ..ChainFormat::DEFAULT + }, + }; + let columns: Vec> = (0..3).map(|i| column(n, 5 + 31 * i)).collect(); + let points: Vec> = (0..3u64) + .map(|i| { + (0..n as u64) + .map(|j| EE::from(7 + 11 * i + 3 * j)) + .collect() + }) + .collect(); + let honest: Vec = columns + .iter() + .zip(&points) + .map(|(c, p)| c.evaluate_in(p).unwrap()) + .collect(); + let layout = StackedLayout::build(&[n; 3], n).unwrap(); + assert_eq!(layout.num_polys(), 3); + let stacked = + StackedCommitment::::commit(layout.clone(), &borrow(&columns), None, &cfg) + .unwrap(); + let roots = stacked.roots(); + let run = |values: &[EE]| { + let proof = prove::( + &stacked, + &borrow(&columns), + None, + &Claimed::PerColumn(&points), + values, + &cfg, + &mut sponge(), + ) + .unwrap(); + assert_eq!(proof.polys.len(), 1, "one chain over the run of three"); + verify::( + &proof, + &layout, + &roots, + &Claimed::PerColumn(&points), + values, + stacked.domain(), + &cfg, + &mut sponge(), + ) + }; + assert_eq!(run(&honest), Ok(()), "the honest claims"); + let mut traded = honest.clone(); + traded[0] += EE::one(); + traded[1] = &traded[1] - EE::one(); + assert!( + run(&traded).is_err(), + "a claim traded between slots 0 and 1" + ); +} diff --git a/prover/src/tests/halt_continuation_poc.rs b/prover/src/tests/halt_continuation_poc.rs index 50de6b654..c0d91a59f 100644 --- a/prover/src/tests/halt_continuation_poc.rs +++ b/prover/src/tests/halt_continuation_poc.rs @@ -84,8 +84,6 @@ fn craft_bundle( &[], is_final, true, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, )?; if is_final { apply_tamper(&mut traces, epoch_logs, tamper); diff --git a/prover/src/tests/halt_pc_forgery_poc.rs b/prover/src/tests/halt_pc_forgery_poc.rs index 15b4f603f..a0580619d 100644 --- a/prover/src/tests/halt_pc_forgery_poc.rs +++ b/prover/src/tests/halt_pc_forgery_poc.rs @@ -12,19 +12,18 @@ //! //! Production proof options, as in `page_offset_forgery_poc`. -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use executor::elf::Elf; use executor::vm::execution::Executor; use executor::vm::logs::Log; use stark::proof::options::ProofOptions; -use stark::prover::{IsStarkProver, Prover}; +use stark::prover::IsStarkProver; use crate::statement::{StatementKind, absorb_statement}; use crate::tables::halt::cols as halt_cols; use crate::tables::register::cols as reg_cols; use crate::tables::trace_builder::Traces; use crate::tables::types::{FE, VmTable}; -use crate::test_utils::{E, asm_elf_bytes}; +use crate::test_utils::asm_elf_bytes; use crate::{MaxRowsConfig, VmAirs, VmProof}; pub(crate) const GUEST: &str = "poc_halt_continue"; @@ -150,15 +149,8 @@ fn craft_proof( let options = opts(); let program = Elf::load(elf_bytes).expect("ELF load"); let max_rows = MaxRowsConfig::default(); - let mut traces = Traces::from_elf_and_logs( - &program, - logs, - &max_rows, - &[], - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build"); + let mut traces = + Traces::from_elf_and_logs(&program, logs, &max_rows, &[]).expect("trace build"); apply_tamper(&mut traces, logs, tamper); @@ -179,7 +171,8 @@ fn craft_proof( &options, &crate::tables::register::register_init_from_entry_point(program.entry_point), &traces, - ), + ) + .preprocessed(), ); let runtime_page_ranges = traces.runtime_page_ranges(); let num_private_input_pages = traces @@ -188,7 +181,7 @@ fn craft_proof( .filter(|c| c.is_private_input) .count(); - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -200,11 +193,10 @@ fn craft_proof( options.fri_final_poly_log_degree, ); - let proof = Prover::multi_prove( + let proof = crate::hash_pin::BlockProver::multi_prove( airs.air_trace_pairs(&mut traces), &mut transcript, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, )?; Ok(VmProof { diff --git a/prover/src/tests/hash_pin_enumeration.rs b/prover/src/tests/hash_pin_enumeration.rs new file mode 100644 index 000000000..cc25525fb --- /dev/null +++ b/prover/src/tests/hash_pin_enumeration.rs @@ -0,0 +1,366 @@ +//! ★ **THE ENUMERATION GATE** — the hash must be NAMED, never implied. +//! +//! `lfm/SOUNDNESS.md` §6.7 states the rule: every prove, verify, execute and +//! commitment-building call on the block path names [`crate::hash_pin`] rather +//! than a workspace default alias. Eleven sites were carrying an implied hash +//! and were closed by enumerating call sites BY HAND — and a hand enumeration +//! is worth exactly as long as nobody adds a twelfth. +//! +//! ⚠ **This is a blessed-set test, not a ban.** Three of the four symbols have +//! legitimate uses that are not on the block path: test helpers that build a +//! trace or a fixture under the toy permutation, host-side byte-transcript +//! differentials, and one assertion whose whole content is what the default IS. +//! Each blessed file carries its reason below. A NEW file appearing fails the +//! test and names itself, which forces the reachability question — *is this on +//! the block path?* — to be answered by a person, once, rather than assumed. +//! +//! ⚖ ASSESSMENT of what it does NOT catch: a call site can name a hash +//! explicitly and name the WRONG one, and a source scan cannot see that. The +//! instruments for that are the pin's own coherence tests +//! (`hash_pin::tests`) and the differentials in `algebraic_commit` / +//! `algebraic_transcript`. This gate closes the *silent default*, which is the +//! failure mode that produced all eleven. + +use std::collections::BTreeSet; +use std::path::Path; + +/// ★★ Calls that reach a prover or verifier through the WORKSPACE ALIAS. +/// +/// ⛔ `stark::prover::Prover` and `stark::verifier::Verifier` **are** +/// `GenericProver` / `GenericVerifier` AT `DefaultStarkHash`. They are the +/// SILENT spelling of the first symbol below, and this gate's first version +/// omitted them — so it searched for the class by the one name the class never +/// uses. Nineteen files kept calling the BLAKE3 alias against artifacts that +/// follow the pin, on branches whose entire purpose is that the two differ, and +/// the gate reported green. +/// +/// Matched as call forms rather than as bare identifiers, because `Prover` and +/// `Verifier` appear inside `IsStarkProver`, `BlockProver` and ordinary prose. +/// A line naming the pin is excluded by [`PIN_CALLS`] rather than by the pattern. +/// `compute_precomputed_commitment_for_testing` joined the list when a test +/// declared BLAKE3 preprocessed commitments through it while the pinned prover +/// recomputed them under RPX — the same alias, one more call form. +const ALIAS_CALLS: &[&str] = &[ + "Prover::multi_prove", + "Verifier::multi_verify", + "Prover::compute_precomputed_commitment_for_testing", +]; + +/// The pinned spellings, which contain [`ALIAS_CALLS`] as substrings. +const PIN_CALLS: &[&str] = &["BlockProver::", "BlockVerifier::"]; + +/// ★★ Items `prover` may name from `stark::config` — the hash-AGNOSTIC ones. +/// +/// **Everything else that module exports is a spelling of the workspace +/// DEFAULT or a concrete configuration of its own** (`P1StarkHashCapped`), so +/// this is an ALLOWLIST over a namespace rather than a list of forbidden +/// names, and that difference is the point. +/// +/// ⚠ The three misses this gate has had — `DefaultStarkHash` as a substring, +/// then the `Prover`/`Verifier` call forms, then the Merkle backend type +/// aliases — share one root: **the gate is lexical and the class is +/// type-level**, so a name list always lags one spelling behind the newest way +/// to denote the default. An allowlist cannot lag: a new alias added to +/// `stark::config` is flagged the first time `prover` names it, without anyone +/// remembering to extend this file. +/// +/// ⚖ Deletion would be stronger still — let the compiler refuse the spelling +/// rather than a test — and it was considered and is NOT available: +/// `BatchedMerkleTreeBackend` and `FriLayerMerkleTreeBackend` have twelve +/// legitimate consumers inside `crypto/stark` itself (`commitment.rs`, +/// `gpu_lde.rs`, the cuda tests), where the workspace default IS the correct +/// hash. `#[deprecated]` would fire on those under `-D warnings`, in the very +/// crate that must keep using them. +/// `DeviceTreeBackend` is allowed for the same reason: it is the marker a Merkle +/// backend implements to name its OWN hash as the device dispatch key, so it +/// carries no default — `algebraic_commit` implements it for the algebraic +/// backends, which is the opposite of reaching a default. +const CONFIG_ALLOWED: &[&str] = &[ + "Commitment", + "CommitmentHash", + "StarkHash", + "DeviceTreeBackend", + // ★ `GrindingDigest` is generic over the hash tag and selects nothing on + // its own — the same class as `StarkHash` above, and it joined this list + // rather than `BLESSED` for that reason: blessing the FILE would have + // excused every future mention in it, where allowing the hash-agnostic + // GENERIC keeps the gate on the concrete tags a caller pairs it with. + // Added when `80d746321` gave `lfm/algebraic_commit.rs` a host grinding + // test spelling `GrindingDigest`. + "GrindingDigest", + // `TreeGeometry` is a configuration's tree shape as a value: its arity and + // its 4-ary cap. It names no hash, backend or transcript, so it cannot + // choose what a tree is hashed with. `TreeGeometry::of::()` reads the + // configuration its caller names, and a concrete tag spelled from this + // namespace still flags. `TreeGeometry::BINARY` is a constant, not a + // generic: the shape of every configuration that is not 4-ary, the + // workspace default's among them, and allowing the name allows it at + // every site. So this gate does not ask what a binary shape is paired + // with; the consumer does. Both leg emitters assert the builder's arity + // against the shape's (`sub_proof::emit_query_from_bits`, + // `fri::emit_query_fri`), so a shape of the other arity is refused when + // the program is emitted, and the block path takes its shapes from its + // family (`TreeGeometry::of::()`, the type parameter its + // builder's wrap hash comes from). One binary shape does reach the block + // path: `FriShape::for_layout` defaults to it and the challenge replay's + // shape (`TableChallengeShape::derive`) keeps the default. That replay + // reads fold counts and index bits and no tree shape, and the leg shape + // replaces the default (`TableVerifyShape::derive_with`). A replay that + // starts reading a tree's shape must take the family's first. In the + // block's own files the name is refused in production code by the + // block-path scan (`block_whir_p1_tests`, its `FAMILY_ONLY` names). + "TreeGeometry", +]; + +/// Every item named from `stark::config` on this line, `use` lists included. +fn config_items(code: &str) -> Vec { + const PREFIX: &str = "stark::config::"; + let mut out = Vec::new(); + let mut rest = code; + while let Some(i) = rest.find(PREFIX) { + rest = &rest[i + PREFIX.len()..]; + if let Some(stripped) = rest.strip_prefix('{') { + let end = stripped.find('}').unwrap_or(stripped.len()); + for part in stripped[..end].split(',') { + let name = part.split_whitespace().next().unwrap_or(""); + if !name.is_empty() { + out.push(name.to_string()); + } + } + rest = &stripped[end.min(stripped.len())..]; + } else { + let end = rest + .find(|c: char| !c.is_alphanumeric() && c != '_') + .unwrap_or(rest.len()); + if end > 0 { + out.push(rest[..end].to_string()); + } + rest = &rest[end..]; + } + } + out +} + +/// The symbols that silently select a hash when nobody names one. +const IMPLIED_HASH_SYMBOLS: &[&str] = &[ + // The workspace's commitment configuration, and the `Prover` / `Verifier` + // aliases that are `GenericProver` / `GenericVerifier` AT it. + "DefaultStarkHash", + // The workspace's Fiat-Shamir transcript OBJECT — §6.7's axis 2, the one + // whose half-flip is SILENT. + "DefaultStarkTranscript", + // The `LFM_HASH` socket permutation — §6.7's axis 3. The default is `Test`, + // a one-round toy. + "HasherKind::default()", +]; + +/// Files allowed to mention an implied-hash symbol, each with its reason. +/// +/// ⚠⚠ **THE QUESTION THIS LIST ANSWERS IS "IS THIS DEFAULT PAIRED WITH A +/// NON-DEFAULT?", NOT "IS THIS REACHABLE FROM PRODUCTION?"** The first version +/// asked the second, and every entry's reasoning was *true* and one scope too +/// wide. `build_traces` really is test-only and production really does reach +/// `build_traces_with_hasher` — and twelve tests still built traces at +/// `HasherKind::Test` while proving against artifacts that followed the pin, +/// which is an out-of-bounds index inside `HashConstraints::eval` because the +/// socket chip's width is tenant-dependent. +/// +/// ★ **"Test-only" is not "safe" — it is only "production-safe."** Before +/// blessing an entry, name the CONSUMER the default is handed to and check what +/// tenant *it* follows. +/// +/// ⚖ And note the subtlest part: that mismatch did not pre-exist. Before +/// `build_artifacts` was pinned, `artifacts.hasher` was ALSO `Test`, so the pair +/// agreed **by both being wrong**. A correct fix to one half of a +/// wrong-but-consistent pair CREATES the failure — so a red test after such a +/// fix is evidence the fix worked, not that it broke something. +/// +/// Paths are relative to `prover/src`. Two files are excluded from the scan +/// rather than blessed: `hash_pin.rs`, because naming the default is what it is +/// FOR, and this file, because it has to spell the symbols it searches for. +/// +/// ⚠ Scope is `prover/src` only. `crypto/stark` DEFINES the aliases, so +/// scanning it would return the definitions and every doc line about them; the +/// pin is a `prover`-crate concept and the call sites that matter are here. +const BLESSED: &[(&str, &str)] = &[ + ( + "lfm/airs.rs", + "`lfm_chip_census` / `lfm_cell_counts` / `LfmAirs::new` default the \ + socket hasher. ✓ VERIFIED test-only: the census pair counts cells and \ + proves nothing, and `LfmAirs::new` has exactly one caller \ + (`wrap_tests::the_census_agrees_with_the_traces_the_prover_builds`, \ + over `programs::keccak_chain_program` — which pins keccak on its own \ + builder, so it emits no `Instr::Hash` and never consults the socket). \ + Production builds its AIR set through `LfmAirs::new_with_hasher`. \ + ⚠ AND the defaulted census pair must stay OFF the block path's \ + NON-IGNORED prove sites, which is a narrower claim than `test-only`: \ + `wrap_tests::the_fixture_epoch_wraps` proves the assembled PER-TABLE \ + epoch verifier against artifacts that follow the pin on every suite \ + run, so it names `lfm_cell_counts_with_hasher(.., BLOCK_HASHER)` and \ + `build_artifacts_with_hasher(.., BLOCK_HASHER)` rather than either \ + defaulting form. Pairing this default with a pinned non-default is the \ + failure the list exists for — the `LFM_HASH` chip's width is \ + tenant-dependent, so a defaulted census under an algebraic pin reports \ + the WRONG chip rather than a smaller number. Do not simplify that call \ + site back to the defaulting pair.", + ), + ( + "lfm/trace.rs", + "`build_traces` defaults the socket hasher. ✓ VERIFIED test-only \ + (`wrap_tests`, `blake3_chip_tests`, `machine_tests`); production \ + reaches `build_traces_with_hasher` through \ + `proof::lfm_prove_with_hasher`, which passes ONE hasher to the \ + executor, the AIR set and the trace builder.", + ), + ( + "lfm/fixture.rs", + "Fixture construction for the chip suites, under the toy permutation \ + by design.", + ), + ( + "lfm/machine_tests.rs", + "Host-side BYTE-transcript differentials: the oracle for the machine's \ + byte `TranscriptReplay` arm is deliberately the byte transcript.", + ), + ( + "tests/static_commitments_tests.rs", + "One mention, and it NAMES its hash: `KeccakStarkHash` is the third \ + configuration the static-root test needs, neither the legacy pin nor \ + Poseidon1, so its roots take the computed-once path. The roots stay \ + inside the `#[test]` body and are compared with each other; nothing \ + that follows the pin is handed them.", + ), + ( + "lfm/algebraic_commit.rs", + "One mention, and it is a CROSS-HASH CONTROL that names its hash rather \ + than defaulting to it — the opposite of what this gate is for. \ + `the_host_search_finds_a_valid_nonce_under_rpx` (added by `80d746321`) \ + grinds under `GrindingDigest`, then declares \ + `GrindingDigest` to prove a BLAKE3-ground nonce does \ + NOT satisfy the RPX predicate. Delete that second type and the test \ + becomes a tautology: `generate_nonce::` could hash anything \ + and every other assertion would still hold. \ + ✓ REACHABILITY, named as this list demands rather than left at \ + 'test-only': the CONSUMERS are `stark::grinding::generate_nonce::` \ + and `is_valid_nonce::`, both generic over the tag passed at the \ + call site, so no global is read and nothing is handed a defaulted \ + hash. The value never leaves the `#[test]` body — no artifact, no \ + commitment, no trace is built from it — so the paired-default failure \ + this list exists to catch (a default handed to a consumer that follows \ + the pin) has no subject here. `GrindingDigest` itself is hash-agnostic \ + and lives in CONFIG_ALLOWED; this entry covers only the concrete \ + `Blake3StarkHash`, which must keep flagging everywhere else.", + ), + ( + "lfm/p1_commit.rs", + "One mention: the re-export of `stark::config::P1StarkHashCapped`, the \ + Poseidon1 STARK configuration generic over its 4-ary cap, which this \ + module fixes at `P1_STARK_CAP` as `P1StarkHash`. It NAMES a hash that \ + is not the workspace default, so no default is handed to any consumer \ + and the question this list asks has no subject here. Reachability: \ + the prover and the verifier name the configuration at one place, \ + `hash_pin::P1Family::Stark`, and the block's own files (the five the \ + block-path scan reads) name `BlockFamily`, never this module. The \ + module's other production readers are `transcript_replay`'s \ + Poseidon1 arms, which the block's nodes run, selected by the \ + builder's wrap hash, which is the family's. The static tables' \ + generator (`bin/compute_static_commitments`) names the configuration \ + too, to recompute the blessed Poseidon1 roots; it proves and \ + verifies nothing. The file is blessed rather than the name \ + allowed, so a site spelling `stark::config::P1StarkHashCapped` at a \ + cap of its own still flags. A site that reaches the same type through \ + this module's re-export does not: this gate sees a configuration only \ + by its `stark::config` spelling. Blessing a file excuses every flagged \ + symbol in it, so this file stays a module of re-exports and constants: \ + nothing that proves, verifies or commits belongs here.", + ), +]; + +/// Every `.rs` under `dir`, relative to `root`. +fn rust_files(root: &Path, dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).expect("the crate source must be readable") { + let path = entry.expect("a readable dir entry").path(); + if path.is_dir() { + rust_files(root, &path, out); + } else if path.extension().is_some_and(|e| e == "rs") { + out.push( + path.strip_prefix(root) + .expect("a path under the root") + .to_path_buf(), + ); + } + } +} + +/// A line with its trailing `//` comment removed, or `None` if the whole line +/// is a comment. +/// +/// ⚠ Deliberately crude — a `//` inside a string literal would truncate the +/// line early. That direction is safe: it can only make the scan miss a +/// mention, and every mention this gate is about is real code. What it must +/// NOT do is count prose, because the modules that explain this rule discuss +/// the symbols by name in nearly every paragraph. +fn code_of(line: &str) -> Option<&str> { + let t = line.trim_start(); + if t.starts_with("//") || t.starts_with('*') { + return None; + } + Some(match line.find("//") { + Some(i) => &line[..i], + None => line, + }) +} + +#[test] +fn no_call_site_outside_the_pin_reaches_a_default_alias() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut files = Vec::new(); + rust_files(&root, &root, &mut files); + assert!( + files.len() > 50, + "the scan must actually see the crate, found {} files", + files.len() + ); + + let mut found: BTreeSet = BTreeSet::new(); + for rel in &files { + // `hash_pin.rs` names the defaults because naming them is what it is + // FOR; this file names them because it has to spell what it searches + // for. Both are excluded by identity rather than by a pattern, so a + // third file cannot join them by accident. + if rel == Path::new("hash_pin.rs") || rel == Path::new("tests/hash_pin_enumeration.rs") { + continue; + } + let text = std::fs::read_to_string(root.join(rel)).expect("a readable source file"); + for line in text.lines() { + let Some(code) = code_of(line) else { continue }; + let implied = IMPLIED_HASH_SYMBOLS.iter().any(|s| code.contains(s)) + || config_items(code) + .iter() + .any(|item| !CONFIG_ALLOWED.contains(&item.as_str())); + let aliased = ALIAS_CALLS.iter().any(|s| code.contains(s)) + && !PIN_CALLS.iter().any(|s| code.contains(s)); + if implied || aliased { + found.insert(rel.to_string_lossy().replace('\\', "/")); + } + } + } + + let blessed: BTreeSet = BLESSED.iter().map(|(f, _)| (*f).to_string()).collect(); + let unexpected: Vec<&String> = found.difference(&blessed).collect(); + assert!( + unexpected.is_empty(), + "★ a NEW site reaches an implied hash: {unexpected:?}\n\ + Every prove / verify / execute / commitment-building call on the block \ + path must name `crate::hash_pin` (SOUNDNESS.md §6.7). If this site is \ + genuinely off the block path, add it to BLESSED with the reason it is \ + — and check the reachability, because the last eleven all looked off \ + the path too." + ); + let stale: Vec<&String> = blessed.difference(&found).collect(); + assert!( + stale.is_empty(), + "a blessed file no longer mentions an implied hash — drop it from \ + BLESSED so the list stays an inventory rather than a wish: {stale:?}" + ); +} diff --git a/prover/src/tests/is_wh_tests.rs b/prover/src/tests/is_wh_tests.rs index 82701ca81..d59fca932 100644 --- a/prover/src/tests/is_wh_tests.rs +++ b/prover/src/tests/is_wh_tests.rs @@ -1,8 +1,7 @@ //! Tests for the IS_WH table and the IS_WHH template (spec #1003, #1015, #1017), //! and for the KECCAK `state_ptr` that is their first caller. -use crypto::fiat_shamir::default_transcript::DefaultTranscript; -use stark::prover::{IsStarkProver, Prover}; +use stark::prover::IsStarkProver; use crate::statement::{StatementKind, absorb_statement}; use crate::tables::bitwise::{ @@ -41,6 +40,7 @@ fn keccak_op(state_addr: u64) -> KeccakOperation { state_addr, input: [0x0102_0304_0506_0708; 25], output: [0; 25], + sponge: None, } } @@ -278,15 +278,8 @@ fn full_traces(name: &str) -> (Vec, Elf, Traces) { .run() .expect("run") .logs; - let traces = Traces::from_elf_and_logs( - &program, - &logs, - &MaxRowsConfig::default(), - &[], - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build"); + let traces = Traces::from_elf_and_logs(&program, &logs, &MaxRowsConfig::default(), &[]) + .expect("trace build"); (elf_bytes, program, traces) } @@ -321,7 +314,8 @@ fn prove_asm_with(name: &str, rewrite: impl FnOnce(&mut Traces)) -> bool { &options, &crate::tables::register::register_init_from_entry_point(program.entry_point), &traces, - ), + ) + .preprocessed(), ); let runtime_page_ranges = traces.runtime_page_ranges(); let num_private_input_pages = traces @@ -329,7 +323,7 @@ fn prove_asm_with(name: &str, rewrite: impl FnOnce(&mut Traces)) -> bool { .iter() .filter(|c| c.is_private_input) .count(); - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -340,11 +334,10 @@ fn prove_asm_with(name: &str, rewrite: impl FnOnce(&mut Traces)) -> bool { &runtime_page_ranges, options.fri_final_poly_log_degree, ); - let proof = Prover::multi_prove( + let proof = crate::hash_pin::BlockProver::multi_prove( airs.air_trace_pairs(&mut traces), &mut transcript, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .expect("prover must build the crafted proof; a prover refusal is not a verifier rejection"); let proof = VmProof { diff --git a/prover/src/tests/keccak_tests.rs b/prover/src/tests/keccak_tests.rs index 59eccdfbd..3d4a1f8d6 100644 --- a/prover/src/tests/keccak_tests.rs +++ b/prover/src/tests/keccak_tests.rs @@ -20,6 +20,7 @@ fn mu_is_bit_constrained() { state_addr: 0x1000, input: [0x0102030405060708u64; 25], output: [0u64; 25], + sponge: None, }; let honest = generate_keccak_trace(std::slice::from_ref(&op)); let air = busless_air(cols::NUM_COLUMNS, KeccakConstraints); diff --git a/prover/src/tests/lean_program_census.rs b/prover/src/tests/lean_program_census.rs new file mode 100644 index 000000000..55513ef03 --- /dev/null +++ b/prover/src/tests/lean_program_census.rs @@ -0,0 +1,319 @@ +//! Census (ignored by default): every VM table's zerocheck program as the WHIR +//! argue runs it, today's against the lean one — its steps, its roots, and the +//! live set that sizes a device round's per-thread slot file, with the thread +//! ceiling that live set leaves under the round kernel's slot budget. Lane +//! I-GFS, S0 of the narrow-round lever (`thoughts/zf/gap2/fix2/I-GFS.md` §15). +//! +//! It also checks, at a random point, that the two programs are one polynomial. +//! +//! `cargo test -p lambda-vm-prover --lib tests::lean_program_census -- --ignored --nocapture` + +use math::field::element::FieldElement; +use stark::multilinear_air::{Uniforms, beta_powers}; +use stark::multilinear_table::{TableLayout, weight_slots}; +use stark::proof::options::GoldilocksCubicProofOptions; +use stark::traits::AIR; + +use multilinear::program::Program; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; +use crate::test_utils::*; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type EE = FieldElement; + +fn point(n: usize, seed: u64) -> Vec { + (0..n as u64) + .map(|i| { + let x = i.wrapping_add(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15); + EE::new([ + FieldElement::from(x >> 7), + FieldElement::from(x >> 11), + FieldElement::from(x >> 3), + ]) + }) + .collect() +} + +// ── S1a: the zerocheck batches on demand (`LAMBDA_VM_ARGUE_LEAN_PROGRAM`) ──── + +pub(crate) type VmAir = Box>; + +/// Every VM table's AIR, named as the census names them. +pub(crate) fn vm_airs() -> Vec<(VmAir, &'static str)> { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + vec![ + (Box::new(create_cpu_air(&opts, 0)), "CPU"), + (Box::new(create_bitwise_air(&opts)), "BITWISE"), + (Box::new(create_lt_air(&opts)), "LT"), + (Box::new(create_shift_air(&opts)), "SHIFT"), + (Box::new(create_eq_air(&opts)), "EQ"), + (Box::new(create_bytewise_air(&opts)), "BYTEWISE"), + (Box::new(create_store_air(&opts)), "STORE"), + (Box::new(create_memw_air(&opts)), "MEMW"), + (Box::new(create_memw_aligned_air(&opts)), "MEMW_A"), + (Box::new(create_memw_register_air(&opts)), "MEMW_R"), + (Box::new(create_load_air(&opts)), "LOAD"), + (Box::new(create_decode_air(&opts)), "DECODE"), + (Box::new(create_mul_air(&opts)), "MUL"), + (Box::new(create_dvrm_air(&opts)), "DVRM"), + (Box::new(create_branch_air(&opts)), "BRANCH"), + (Box::new(create_halt_air(&opts)), "HALT"), + (Box::new(create_hint_air(&opts)), "HINT"), + (Box::new(create_copy_air(&opts)), "COPY"), + (Box::new(create_page_air(&opts, 0x1000)), "PAGE"), + (Box::new(create_register_air(&opts, 0)), "REGISTER"), + (Box::new(create_keccak_air(&opts)), "KECCAK"), + (Box::new(create_keccak_rnd_air(&opts)), "KECCAK_RND"), + (Box::new(create_keccak_rc_air(&opts)), "KECCAK_RC"), + (Box::new(create_ecsm_air(&opts)), "ECSM"), + (Box::new(create_ecdas_air(&opts)), "ECDAS"), + ( + Box::new(crate::continuation::l2g_memory_air( + &opts, + crate::tables::local_to_global::epoch_label(1), + )), + "L2G", + ), + ] +} + +/// A VM table's zerocheck batch as `multilinear_table::prove` builds it — the +/// constraint rule beside the bus's two, combined — with its round degree and +/// the factors it reads. +struct Batch { + program: Program, + degree: usize, + factors: usize, +} + +fn batch_of( + air: &dyn AIR, + label: &str, +) -> Batch { + const NUM_VARS: usize = 10; + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + air.trace_layout().0, + NUM_VARS, + Uniforms::default(), + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let shape = layout.shape(); + let betas = beta_powers(&EE::from(0x5eed_u64), shape.num_roots()); + let (weight_r, weight_z) = weight_slots(layout.kinds().len()); + let zerocheck = shape + .program(&betas, weight_r) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let slots = layout.slot_of().to_vec(); + let interactions = stark::multilinear_logup::interactions( + air.bus_interactions(), + slots.len(), + &EE::from(0x2a_u64), + &EE::from(0x3b_u64), + |col| { + slots + .get(col) + .copied() + .ok_or(multilinear::Error::UnknownPolynomial { + index: col, + len: slots.len(), + }) + }, + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let claim = point( + multilinear::logup::input_layer_vars(interactions.len(), NUM_VARS), + 7, + ); + let bus = multilinear::logup::claim_statements(&interactions, &claim, NUM_VARS, weight_z) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let program = multilinear::program::combine( + &[ + &zerocheck, + bus.numerator.program().expect("compiled"), + bus.denominator.program().expect("compiled"), + ], + &point(3, 11), + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + let degree = (shape.degree() + 1) + .max(bus.numerator.degree()) + .max(bus.denominator.degree()); + Batch { + program, + degree, + factors: weight_z + 1, + } +} + +/// ★ Host parity, every VM table: each zerocheck batch on demand is the same +/// polynomial as today's at random points, and holds no more values a thread. +/// The four big batches — and only they — are over the gate, and on demand +/// every one of them falls under it: a first round of at least 64 k threads. +/// The byte gate's EQ fixture is not big, so it keeps today's program. +#[test] +fn every_vm_batch_on_demand_is_the_same_program() { + let mut big = Vec::new(); + for (air, label) in vm_airs() { + let batch = batch_of(&*air, label); + assert!( + (1..=math_cuda::sumcheck::MAX_NODES).contains(&batch.degree), + "{label}: a round of degree {} is not one the card takes", + batch.degree + ); + let demand = batch.program.on_demand(); + let mut scratch = Vec::new(); + for seed in [1u64, 29, 311] { + let values = point(batch.factors, seed); + assert_eq!( + demand.eval(&values, &mut scratch), + batch.program.eval(&values, &mut scratch), + "{label}: on demand is another polynomial (seed {seed})" + ); + } + let today = multilinear::gpu::lower(&batch.program).expect("today's lowers"); + let lean = multilinear::gpu::lower(&demand).expect("on demand lowers"); + assert!( + lean.num_slots <= today.num_slots, + "{label}: on demand holds {} values a thread, today {}", + lean.num_slots, + today.num_slots + ); + if multilinear::gpu::is_big_batch(today.num_slots) { + assert!( + !multilinear::gpu::is_big_batch(lean.num_slots), + "{label}: on demand still holds {} values a thread", + lean.num_slots + ); + big.push(label); + } + } + assert_eq!( + big, + ["KECCAK", "KECCAK_RND", "ECSM", "ECDAS"], + "the big batches" + ); +} + +/// ★ Device parity, every big batch — the VM's four, found +/// by the gate: on the card, the program on demand — its slot file sized for +/// every node from the first round — gives today's round sums, round by round +/// from a 2^14 cube (whose first round is past KECCAK_RND's 8,096-thread +/// ceiling) down to the host crossover; and a shadow session walking today's +/// program over the lean session's own factors ([`SumcheckSession::shadow`], +/// what `LAMBDA_VM_ARGUE_XCHECK` runs) agrees with both. Needs a device, and +/// says so rather than passing without one. +/// +/// ```text +/// cargo test --release -p lambda-vm-prover --features cuda --lib \ +/// tests::lean_program_census::every_big_batch_rounds_the_same_on_demand_on_the_card -- --exact --nocapture +/// ``` +#[cfg(feature = "cuda")] +#[test] +fn every_big_batch_rounds_the_same_on_demand_on_the_card() { + use math_cuda::sumcheck::{DeviceFactors, thread_ceiling}; + + const CUBE: usize = 1 << 14; + const P: u64 = 0xFFFF_FFFF_0000_0001; + if math_cuda::device::backend().is_err() { + eprintln!("lean program: SKIPPED, no device"); + return; + } + let canonical = |sums: Vec| -> Vec { + sums.into_iter() + .map(|x| if x >= P { x - P } else { x }) + .collect() + }; + let mut state = 0x243F_6A88_85A3_08D3_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state % P + }; + let vm = vm_airs(); + let every: Vec<( + &dyn AIR, + String, + )> = vm + .iter() + .map(|(air, label)| (&**air, label.to_string())) + .collect(); + let mut big = Vec::new(); + for (air, label) in every { + let label = label.as_str(); + let batch = batch_of(air, label); + let today = multilinear::gpu::lower(&batch.program).expect("today's lowers"); + if !multilinear::gpu::is_big_batch(today.num_slots) { + continue; + } + big.push(label.to_string()); + let demand = multilinear::gpu::lower(&batch.program.on_demand()).expect("on demand lowers"); + let factors: Vec> = (0..batch.factors) + .map(|_| (0..CUBE * 3).map(|_| next()).collect()) + .collect(); + let raw: Vec<&[u64]> = factors.iter().map(Vec::as_slice).collect(); + let held_factors = DeviceFactors::upload(&raw).expect("today's factors go up"); + let lean_factors = DeviceFactors::upload(&raw).expect("the lean session's factors go up"); + let mut held = held_factors + .session_with( + &[], + &today.nodes, + &today.consts, + today.num_slots, + today.root_slot, + ) + .expect("today's session"); + let mut lean = lean_factors + .session_spread( + &[], + &demand.nodes, + &demand.consts, + demand.num_slots, + demand.root_slot, + batch.degree, + ) + .expect("the session on demand"); + let mut shadow = lean + .shadow( + &today.nodes, + &today.consts, + today.num_slots, + today.root_slot, + ) + .expect("the shadow"); + let t: Vec = (1..=batch.degree as u64) + .flat_map(|node| [node, 0, 0]) + .collect(); + let rounds = (CUBE / 32).trailing_zeros() as usize; + for round in 0..rounds { + let expected = canonical(held.round(&t).expect("today's round")); + let got = canonical(lean.round(&t).expect("the round on demand")); + shadow.follow(&lean); + let beside = canonical(shadow.round(&t).expect("the shadow's round")); + assert_eq!(got, expected, "{label}: round {round} on demand"); + assert_eq!(beside, expected, "{label}: the shadow's round {round}"); + let r = [next(), next(), next()]; + held.fold(&r).expect("today's fold"); + lean.fold(&r).expect("the fold on demand"); + } + eprintln!( + "lean program: {label}: {rounds} rounds of degree {} equal on demand · slots {} → {} · \ + thread ceiling {} → {}", + batch.degree, + today.num_slots, + demand.num_slots, + thread_ceiling(today.num_slots), + thread_ceiling(demand.num_slots), + ); + } + assert_eq!( + big, + ["KECCAK", "KECCAK_RND", "ECSM", "ECDAS"], + "the big batches" + ); +} diff --git a/prover/src/tests/local_to_global_bus_tests.rs b/prover/src/tests/local_to_global_bus_tests.rs index 8025596d6..7908ca2c2 100644 --- a/prover/src/tests/local_to_global_bus_tests.rs +++ b/prover/src/tests/local_to_global_bus_tests.rs @@ -5,12 +5,10 @@ //! program-end receiver (final value of each cell). The bus balances iff every //! epoch's `fini` matches the next epoch's `init` (the cross-epoch telescoping). +use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; -use math::field::element::FieldElement; - use stark::config::Commitment; use stark::lookup::{ AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, @@ -21,7 +19,7 @@ use stark::proof::stark::MultiProof; use stark::proof::view::MultiProofView; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; use crate::tables::local_to_global::{ @@ -340,12 +338,12 @@ fn prove_verify_memory(l2g_boundary: &[CellBoundary], memw_boundary: &[CellBound _, _, )> = vec![(&l2g, &mut l2g_trace, &()), (&memw, &mut memw_trace, &())]; - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&l2g, &memw]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } @@ -367,13 +365,13 @@ fn prove_verify_l2g_range_with_trace( (&l2g, l2g_trace, &()), (&receiver, &mut receiver_trace, &()), ]; - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&l2g, &receiver]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } @@ -407,7 +405,7 @@ fn l2g_root(boundary: &[CellBoundary]) -> Commitment { _, _, )> = vec![(&air, &mut trace, &())]; - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); proof.proofs[0].lde_trace_main_merkle_root } @@ -468,7 +466,7 @@ pub(crate) fn prove_global(boundaries: &[Vec]) -> MultiProof::new(&[])).unwrap() + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap() } pub(crate) fn prove_and_verify(boundaries: &[Vec]) -> bool { @@ -489,10 +487,10 @@ pub(crate) fn prove_and_verify(boundaries: &[Vec]) -> bool { airs.push(&genesis_anchor); airs.push(&program_end_anchor); - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } @@ -593,12 +591,12 @@ fn prove_verify_memory_with_trace( _, _, )> = vec![(&l2g, l2g_trace, &()), (&memw, &mut memw_trace, &())]; - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&l2g, &memw]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } @@ -656,7 +654,8 @@ fn prove_and_verify_global_with_traces( air_trace_pairs.push((&genesis_anchor, &mut genesis_trace, &())); air_trace_pairs.push((&program_end_anchor, &mut program_end_trace, &())); - let proof = multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let mut airs: Vec<&dyn AIR> = l2g_airs .iter() @@ -665,10 +664,10 @@ fn prove_and_verify_global_with_traces( airs.push(&genesis_anchor); airs.push(&program_end_anchor); - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/lt_bus_tests.rs b/prover/src/tests/lt_bus_tests.rs index e95a81285..7f21cd2db 100644 --- a/prover/src/tests/lt_bus_tests.rs +++ b/prover/src/tests/lt_bus_tests.rs @@ -6,12 +6,10 @@ //! - Padding: Auto-padding to power of 2 works correctly //! - Border cases: Edge values (0, MAX, signed boundaries) work +use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; -use math::field::element::FieldElement; - use stark::lookup::{ AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, NullBoundaryConstraintBuilder, Packing, @@ -19,7 +17,7 @@ use stark::lookup::{ use stark::proof::options::ProofOptions; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::lt::{LtOperation, cols, generate_lt_trace}; use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField}; @@ -289,15 +287,15 @@ fn prove_and_verify(ops: &[LtOperation]) -> bool { ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } @@ -373,15 +371,15 @@ fn prove_and_verify_custom(ops: &[LtOperation], receiver_rows: &[CustomLtRow]) - ]; let multi_proof = - multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![&sender_air, &receiver_air]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/lt_multiplicity_poc.rs b/prover/src/tests/lt_multiplicity_poc.rs index 52794d4da..6253c4552 100644 --- a/prover/src/tests/lt_multiplicity_poc.rs +++ b/prover/src/tests/lt_multiplicity_poc.rs @@ -25,7 +25,6 @@ use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use stark::lookup::{ @@ -35,7 +34,7 @@ use stark::lookup::{ use stark::proof::options::ProofOptions; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::lt::{LtOperation, bus_interactions, cols, generate_lt_trace}; use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField, alu_op}; @@ -241,7 +240,7 @@ fn prove_and_verify(lt_rows: &[Row], cpu: &[Vec], victim: &[u64]) -> bool { (&msb16_air, &mut msb16_trace, &()), (&arebytes_air, &mut arebytes_trace, &()), ]; - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![ &cpu_air, &victim_air, @@ -250,10 +249,10 @@ fn prove_and_verify(lt_rows: &[Row], cpu: &[Vec], victim: &[u64]) -> bool { &msb16_air, &arebytes_air, ]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/memw_aligned_tests.rs b/prover/src/tests/memw_aligned_tests.rs index 49ac8cc5c..d2b7fff96 100644 --- a/prover/src/tests/memw_aligned_tests.rs +++ b/prover/src/tests/memw_aligned_tests.rs @@ -6,14 +6,15 @@ use crate::tables::types::*; #[test] fn test_memw_aligned_trace_generation() { - let ops = vec![ + let ops = [ MemwOperation::new(true, 4, [42, 0, 0, 0, 0, 0, 0, 0], 100, 2, true) .with_old([42, 0, 0, 0, 0, 0, 0, 0], [50, 50, 0, 0, 0, 0, 0, 0]), MemwOperation::new(false, 0x1000, [1, 2, 3, 4, 0, 0, 0, 0], 200, 4, false) .with_old([0; 8], [100; 8]), ]; - let trace = generate_memw_aligned_trace(&ops); + let rows: Vec = ops.iter().map(AlignedRow::from_memw).collect(); + let trace = generate_memw_aligned_trace(&rows); assert_eq!(trace.num_cols(), cols::NUM_COLUMNS); assert!(trace.num_rows() >= 2); @@ -28,3 +29,66 @@ fn test_memw_aligned_trace_generation() { assert_eq!(*trace.get_main(1, cols::BASE_ADDRESS[1]), FE::from(0u64)); assert_eq!(*trace.get_main(1, cols::BASE_ADDRESS[2]), FE::from(0u64)); } + +/// An aligned row keeps every element of the op it packs: eight bytes of a +/// memory access at each width, the two 32-bit halves of a register access, +/// and the shared old timestamp; its MEMW_A row is the op's. +#[test] +fn an_aligned_row_keeps_its_ops_elements() { + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut ops = Vec::new(); + for width in [1u8, 2, 4, 8] { + for is_read in [false, true] { + let mut value = [0u32; 8]; + let mut old = [0u32; 8]; + for i in 0..8 { + value[i] = (next() & 0xFF) as u32; + old[i] = (next() & 0xFF) as u32; + } + let old_ts = next() >> 8; + let base = (next() >> 4) & !(width as u64 - 1); + ops.push( + MemwOperation::new(false, base, value, old_ts + 4, width, is_read) + .with_old(old, [old_ts; 8]), + ); + } + } + for is_read in [false, true] { + let value = [next() as u32, next() as u32, 0, 0, 0, 0, 0, 0]; + let old = [next() as u32, next() as u32, 0, 0, 0, 0, 0, 0]; + let old_ts = next() >> 8; + ops.push( + MemwOperation::new(true, 2 * 17, value, old_ts + (1 << 20), 2, is_read) + .with_old(old, [old_ts, old_ts, 0, 0, 0, 0, 0, 0]), + ); + } + for op in &ops { + let row = AlignedRow::from_memw(op); + assert_eq!(row.value(), op.value, "{op:?}"); + assert_eq!(row.old(), op.old, "{op:?}"); + assert_eq!(row.old_timestamp(), op.old_timestamp[0]); + assert_eq!( + ( + row.base_address(), + row.timestamp(), + row.width(), + row.is_read(), + row.is_register() + ), + ( + op.base_address, + op.timestamp, + op.width, + op.is_read, + op.is_register + ) + ); + assert_eq!(row.write_flags(), op.write_flags()); + } +} diff --git a/prover/src/tests/memw_register_tests.rs b/prover/src/tests/memw_register_tests.rs index 56b8a372c..db3b2dc71 100644 --- a/prover/src/tests/memw_register_tests.rs +++ b/prover/src/tests/memw_register_tests.rs @@ -81,3 +81,87 @@ fn test_memw_register_trace_generation_write_op() { assert_eq!(*trace.get_main(0, cols::MU_READ), FE::from(0u64)); assert_eq!(*trace.get_main(0, cols::MU_WRITE), FE::from(1u64)); } + +/// A register read's Memw message as the CPU sends it and as MEMW_R receives +/// it, at timestamp `ts`: the CPU's clock is one column, sent with a zero high +/// word ("CPU timestamps fit in 32 bits"); MEMW_R takes the timestamp as its +/// two 32-bit words. +fn register_read_messages(ts: u64) -> (Vec, Vec) { + use crate::tables::cpu; + let (reg, lo, hi) = (1u64, 42u32, 7u32); + + let cpu_read = cpu::bus_interactions(0) + .into_iter() + .find(|i| { + let read = i.columns_read(); + i.bus_id == BusId::Memw as u64 + && i.is_sender + && read.contains(&cpu::cols::READ_REGISTER1) + && read.contains(&cpu::cols::RS1) + }) + .expect("the CPU sends its rs1 read on Memw"); + let cpu_row = |c: usize| -> FE { + match c { + cpu::cols::TIMESTAMP => FE::from(ts), + cpu::cols::RS1 => FE::from(reg), + cpu::cols::RV1_0 => FE::from(lo as u64), + cpu::cols::RV1_1 => FE::from(hi as u64), + _ => FE::zero(), + } + }; + let sent: Vec = cpu_read + .values + .iter() + .flat_map(|v| v.combine_from::(cpu_row)) + .collect(); + + let ops = vec![ + MemwOperation::new(true, 2 * reg, [lo, hi, 0, 0, 0, 0, 0, 0], ts, 2, true).with_old( + [lo, hi, 0, 0, 0, 0, 0, 0], + [ts - 4, ts - 4, 0, 0, 0, 0, 0, 0], + ), + ]; + let trace = generate_memw_register_trace(&ops); + let memw_r_read = bus_interactions() + .into_iter() + .find(|i| { + i.bus_id == BusId::Memw as u64 + && !i.is_sender + && i.columns_read().contains(&cols::MU_READ) + }) + .expect("MEMW_R receives register reads on Memw"); + let received: Vec = memw_r_read + .values + .iter() + .flat_map(|v| v.combine_from::(|c| *trace.get_main(0, c))) + .collect(); + (sent, received) +} + +/// The block's clock is `4·cycle + 4` over the whole execution, so it passes +/// 2^32 at cycle 2^30 − 1. Below 2^32 the CPU's register-read message and +/// MEMW_R's are one; from 2^32 on they differ, and an honest block past 2^30 +/// cycles cannot balance the Memw bus (T200-R, 1.83 G cycles: BusImbalance). +#[test] +fn the_cpu_and_memw_r_register_read_messages_part_at_2_32() { + let below = (1u64 << 32) - 4; + let (sent, received) = register_read_messages(below); + assert_eq!(sent.len(), received.len()); + assert_eq!(sent, received, "one message below 2^32"); + + // Past 2^32 only the timestamp's two slots (after old 8, is_register, + // base 2, value 8) differ: the CPU's whole clock and a zero against + // MEMW_R's low and high words. + let past = (1u64 << 32) + 4; + let (sent, received) = register_read_messages(past); + assert_eq!(sent.len(), received.len()); + let differ: Vec = (0..sent.len()) + .filter(|&k| sent[k] != received[k]) + .collect(); + assert_eq!(differ, vec![19, 20]); + assert_eq!((sent[19], sent[20]), (FE::from(past), FE::zero())); + assert_eq!( + (received[19], received[20]), + (FE::from(past & 0xFFFF_FFFF), FE::from(past >> 32)) + ); +} diff --git a/prover/src/tests/mod.rs b/prover/src/tests/mod.rs index f2f40ad5c..71afb48f2 100644 --- a/prover/src/tests/mod.rs +++ b/prover/src/tests/mod.rs @@ -1,10 +1,28 @@ -#[cfg(all(test, feature = "disk-spill"))] -pub mod auto_storage_tests; +#[cfg(test)] +pub mod argue_stage1_tests; +#[cfg(test)] +pub mod batched_argue_tests; #[cfg(test)] pub mod bitwise_bus_tests; #[cfg(test)] pub mod bitwise_tests; #[cfg(test)] +#[cfg(feature = "parallel")] +pub mod block_batched_tests; +#[cfg(test)] +#[cfg(feature = "parallel")] +pub mod block_cut_padding_tests; +#[cfg(test)] +#[cfg(feature = "parallel")] +pub mod block_onechain_tests; +#[cfg(test)] +#[cfg(feature = "parallel")] +pub mod block_whir_p1_tests; +#[cfg(test)] +#[cfg(feature = "parallel")] +pub mod block_whir_pack_tests; +pub mod block_whir_tests; +#[cfg(test)] pub mod branch_bus_tests; #[cfg(test)] pub mod branch_carrier_poc; @@ -15,8 +33,12 @@ pub mod branch_multiplicity_poc; #[cfg(test)] pub mod bytewise_tests; #[cfg(test)] +pub mod commit_ood_host_tests; +#[cfg(test)] pub mod compute_commit_bus_offset_tests; #[cfg(test)] +pub mod constraint_artifact_tests; +#[cfg(test)] pub mod constraint_emit_tests; #[cfg(test)] pub mod constraint_program_device_tests; @@ -30,8 +52,6 @@ pub mod constraint_set_tests_b; pub mod constraints_tests; #[cfg(test)] pub mod copy_tests; -#[cfg(all(test, feature = "disk-spill"))] -pub mod count_table_lengths_drift_tests; #[cfg(test)] pub mod cpu32_tests; #[cfg(test)] @@ -42,19 +62,24 @@ pub mod cpu_timestamp_tests; pub mod decode_layout_tests; #[cfg(test)] pub mod decode_tests; -#[cfg(all(test, feature = "disk-spill"))] -pub mod disk_spill_tests; #[cfg(test)] pub mod dvrm_tests; #[cfg(test)] pub mod ecdas_tests; +pub mod ecrec_tests; #[cfg(test)] pub mod ecsm_tests; #[cfg(test)] pub mod eq_tests; #[cfg(test)] +pub mod gpack_tests; +#[cfg(test)] +pub mod grouped_run_claim_tests; +#[cfg(test)] pub mod halt_pc_forgery_poc; #[cfg(test)] +pub mod hash_pin_enumeration; +#[cfg(test)] pub mod hint_tests; #[cfg(test)] pub mod ir_stats_dump; @@ -64,6 +89,7 @@ pub mod is_wh_tests; pub mod keccak_rnd_tests; #[cfg(test)] pub mod keccak_tests; +pub mod lean_program_census; #[cfg(test)] pub mod load_tests; #[cfg(test)] @@ -84,11 +110,15 @@ pub mod memw_tests; pub mod mul_mu_carrier_poc; #[cfg(test)] pub mod mul_tests; +pub mod multilinear_prove_tests; #[cfg(test)] +pub mod multilinear_table_tests; pub mod multiplicity_forgery_poc; #[cfg(test)] pub mod ood_window_ir_tests; #[cfg(test)] +pub mod p1_stark_tests; +#[cfg(test)] pub mod page_layout_tests; #[cfg(test)] pub mod page_offset_forgery_poc; @@ -105,11 +135,20 @@ pub mod recursion_soundness_gap_poc; #[cfg(test)] pub mod register_tests; #[cfg(test)] +pub mod rpx_two_implementations; +#[cfg(test)] +#[cfg(feature = "shape-profile")] +pub mod shape_profile_tests; +pub mod transcript_counts; +pub mod windowed_builder_tests; + pub mod shift_tests; #[cfg(test)] pub mod skip_empty_tables_tests; #[cfg(test)] +pub mod statement_alignment_tests; +#[cfg(test)] pub mod statement_tests; #[cfg(test)] pub mod static_commitments_tests; @@ -121,3 +160,25 @@ pub mod templates_tests; pub mod trace_builder_tests; #[cfg(test)] pub mod trace_test_helpers; +#[cfg(test)] +pub mod verifier_pin_tests; +#[cfg(test)] +pub mod whir_hash_tests; +#[cfg(test)] +pub mod whir_identity_tests; +#[cfg(test)] +pub mod zf_air_cache_tests; +#[cfg(test)] +pub mod zf_lfm_bytes_tests; +#[cfg(all(test, feature = "cuda"))] +pub mod zf_p1_device_tests; +#[cfg(all(test, feature = "cuda"))] +pub mod zf_rpx_device_tests; +#[cfg(test)] +pub mod zf_rpx_golden_tests; +#[cfg(test)] +pub mod zf_rpx_vectors; +#[cfg(test)] +pub mod zf_vm_dp_tests; +#[cfg(test)] +pub mod zf_vm_one_row_tests; diff --git a/prover/src/tests/mul_mu_carrier_poc.rs b/prover/src/tests/mul_mu_carrier_poc.rs index 2ea0468a3..d719755bd 100644 --- a/prover/src/tests/mul_mu_carrier_poc.rs +++ b/prover/src/tests/mul_mu_carrier_poc.rs @@ -23,7 +23,6 @@ use std::collections::HashMap; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use stark::lookup::{ @@ -33,7 +32,7 @@ use stark::lookup::{ use stark::proof::options::ProofOptions; use stark::trace::TraceTable; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::tables::mul::{MulOperation, bus_interactions, cols, generate_mul_trace}; use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField}; @@ -228,7 +227,7 @@ fn prove_and_verify(rows: &[Vec], extra_half: Option) -> bool { (&arebytes_air, &mut arebytes_trace, &()), (&carrier_air, &mut carrier_trace, &()), ]; - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])).unwrap(); let airs: Vec<&dyn AIR> = vec![ &cpu_air, &mul_air, @@ -237,10 +236,10 @@ fn prove_and_verify(rows: &[Vec], extra_half: Option) -> bool { &arebytes_air, &carrier_air, ]; - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ) } diff --git a/prover/src/tests/multilinear_prove_tests.rs b/prover/src/tests/multilinear_prove_tests.rs new file mode 100644 index 000000000..4b2f2d81f --- /dev/null +++ b/prover/src/tests/multilinear_prove_tests.rs @@ -0,0 +1,217 @@ +//! The VM proved and verified end to end through the multilinear path. +//! +//! Unlike `multilinear_table_tests`, which drives the argument by hand, these +//! go through [`multilinear_prove::prove_with_options`] and +//! [`multilinear_prove::verify_with_options`]: ELF in, proof out, verdict back, +//! with the verifier holding nothing but the ELF and the proof. + +use crate::multilinear_prove::{self, MultilinearVmProof}; +use crate::test_utils::asm_elf_bytes; +use stark::proof::options::ProofOptions; + +use crate::tables::MaxRowsConfig; +use multilinear::whir_hash::KeccakWhir; + +fn prove(elf: &[u8]) -> MultilinearVmProof { + multilinear_prove::prove_with_options( + elf, + &ProofOptions::default_test_options(), + &MaxRowsConfig::default(), + ) + .expect("prove") +} + +fn verify(proof: &MultilinearVmProof, elf: &[u8]) -> bool { + multilinear_prove::verify_with_options(proof, elf, &ProofOptions::default_test_options()) + .expect("verify") +} + +/// The milestone this whole port is for: a program proved and verified by the +/// VM's own entry points, with WHIR underneath. +#[test] +fn a_program_proves_and_verifies() { + let elf = asm_elf_bytes("sub"); + let proof = prove(&elf); + assert!(verify(&proof, &elf)); +} + +/// The whole 64-bit instruction set, which lights up the tables a +/// two-instruction program never reaches. +#[test] +fn the_whole_instruction_set_proves_and_verifies() { + let elf = asm_elf_bytes("all_instructions_64"); + let proof = prove(&elf); + assert!(verify(&proof, &elf)); +} + +/// A proof is only a proof if it can leave the process. Round-trips through +/// rkyv, the format the univariate `VmProof` uses, and verifies the +/// **deserialized** one. +#[test] +fn a_proof_survives_serialization() { + let elf = asm_elf_bytes("sub"); + let proof = prove(&elf); + + let bytes = rkyv::to_bytes::(&proof).expect("rkyv round trip"); + let back: MultilinearVmProof = + rkyv::from_bytes::<_, rkyv::rancor::Error>(&bytes).expect("rkyv round trip"); + + assert!(verify(&back, &elf)); +} + +/// The proof is bound to the program: the same proof against a different ELF is +/// rejected, because the digest moves every challenge. +#[test] +fn a_proof_does_not_verify_against_another_program() { + let elf = asm_elf_bytes("sub"); + let proof = prove(&elf); + assert!(!verify(&proof, &asm_elf_bytes("all_instructions_64"))); +} + +/// And to its public output: the COMMIT bus's counterparty is the statement, so +/// restating the output leaves the balance off by exactly the difference. +#[test] +fn a_tampered_public_output_is_rejected() { + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf); + proof.public_output.push(0xff); + assert!(!verify(&proof, &elf)); +} + +/// A restated table height is rejected too: it is absorbed into the transcript +/// before anything is drawn. +#[test] +fn a_restated_table_height_is_rejected() { + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf); + proof.table_num_vars[0] += 1; + assert!(!verify(&proof, &elf)); +} + +/// The preprocessed tables are bound to the program, and a forged one is +/// rejected — the property the proof would otherwise be missing entirely. +/// +/// Forging goes through the AIR the *prover* builds: the honest verifier +/// rebuilds the real columns from the ELF, so a prover that committed anything +/// else opens to the wrong value and is caught. Driven at the table level, +/// where a single AIR can be swapped. +#[test] +fn a_forged_preprocessed_column_is_rejected() { + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::element::FieldElement; + use multilinear::mle::Mle; + use multilinear::whir_chain::{ChainConfig, GrindBits}; + use stark::multilinear_air::Uniforms; + use stark::multilinear_table::{self, CommittedTable, CommittedTables, TableLayout}; + use stark::traits::AIR; + + use crate::tables::keccak_rc; + use crate::test_utils::{E, F, create_keccak_rc_air}; + + let config = ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + }; + let air = create_keccak_rc_air(&ProofOptions::default_test_options()); + let width = air.trace_layout().0; + + // The honest preprocessed columns, padded out to the table's width with + // zeroed multiplicity columns — a trace whose preprocessed half is real. + let honest = keccak_rc::preprocessed_columns(); + let rows = honest[0].len(); + let mut columns = honest.clone(); + columns.resize(width, vec![FieldElement::::zero(); rows]); + + let num_vars = rows.trailing_zeros() as usize; + let layout = || { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + width, + num_vars, + Uniforms::default(), + ) + .unwrap() + }; + let prove = |columns: Vec>>| { + let table = + CommittedTable::from_layout(layout(), |col| columns[col as usize].clone()).unwrap(); + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config).unwrap(); + let mut transcript = DefaultTranscript::::new(b"forged"); + let proof = + multilinear_table::multi_prove(&committed, &config, &mut transcript, None).unwrap(); + ( + proof, + committed.groups()[0].layout().clone(), + committed.groups()[0].domain().clone(), + ) + }; + + // The verifier's own copy: always the real columns, never the prover's. + let expected: Vec> = honest + .iter() + .map(|c| Mle::new(c.clone()).unwrap()) + .collect(); + let verifier_layout = layout(); + let statement = verifier_layout.statement_with_preprocessed(&expected); + + let verify = |(proof, stacked, domain): ( + multilinear_table::MultiProof, + multilinear::stacking::StackedLayout, + multilinear::whir::Domain, + )| { + let mut transcript = DefaultTranscript::::new(b"forged"); + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( + &proof, + &[statement], + std::slice::from_ref(&stacked), + std::slice::from_ref(&domain), + &[1], + &FieldElement::::zero(), + &config, + &mut transcript, + None, + ) + .is_ok() + }; + + assert!(verify(prove(columns.clone())), "the honest table verifies"); + + let mut forged = columns; + forged[0][3] += FieldElement::::one(); + assert!( + !verify(prove(forged)), + "a forged preprocessed column must be rejected" + ); +} + +/// And the tables themselves are checked, not just the metadata around them: +/// moving one table's bus output breaks the balance the statement owes. +#[test] +fn a_tampered_table_proof_is_rejected() { + use math::field::element::FieldElement; + + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf); + proof.proof.tables[0].bus_output.0 += + FieldElement::::one(); + assert!(!verify(&proof, &elf)); +} + +/// A proof missing a table is rejected before any of it is checked: the table +/// count is a function of the statement, not of the proof. +#[test] +fn a_proof_missing_a_table_is_rejected() { + let elf = asm_elf_bytes("sub"); + let mut proof = prove(&elf); + proof.proof.tables.pop(); + proof.table_num_vars.pop(); + assert!( + multilinear_prove::verify_with_options(&proof, &elf, &ProofOptions::default_test_options()) + .is_err() + ); +} diff --git a/prover/src/tests/multilinear_table_tests.rs b/prover/src/tests/multilinear_table_tests.rs new file mode 100644 index 000000000..cc0ca9e65 --- /dev/null +++ b/prover/src/tests/multilinear_table_tests.rs @@ -0,0 +1,1186 @@ +//! The multilinear table argument on **real VM tables**. +//! +//! Everything else exercising it uses fixtures of four or five columns. These +//! are the tables the VM actually proves, at the smallest scale each has: +//! +//! | table | columns | own constraints | shape it covers | +//! |---|---:|---:|---| +//! | EQ | 12 | 4 | compound packing `DWordWL`, a linear bus value with coefficients and constants, a constant element, **deduplicated** multiplicity | +//! | LT | 17 | several | signed comparison, deduplicated multiplicity | +//! | SHIFT | 29 | several | **no** deduplication — one row per operation, `μ = 1` | +//! | BYTEWISE | 26 | none | soundness entirely on the bus, the shape BITWISE/PAGE/REGISTER have | +//! +//! None of their buses balances on its own — they send to BITWISE and receive +//! from the ALU, whose counterparties are other tables — so what is checked is +//! each table's own argument, with the output fraction left to the caller. + +use crypto::fiat_shamir::{default_transcript::DefaultTranscript, is_transcript::IsTranscript}; +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{ + self, CommittedTable, CommittedTables, TableLayout, TableStatement, +}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use stark::constraints::builder::ConstraintSet; + +use crate::tables::bytewise::{BytewiseOperation, generate_bytewise_trace}; +use crate::tables::eq::{EqOperation, generate_eq_trace}; +use crate::tables::lt::{LtOperation, generate_lt_trace}; +use crate::tables::shift::{ShiftOperation, generate_shift_trace}; +use crate::tables::trace_builder::Traces; +use crate::tables::{bytewise, eq, lt, shift}; +use crate::test_utils::{ + ConcreteVmAir, create_bytewise_air, create_eq_air, create_lt_air, create_shift_air, run_asm_elf, +}; +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; +use multilinear::whir_hash::KeccakWhir; + +type ExtE = FieldElement; + +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + } +} + +/// Four operations: equal and unequal, inverted and not. Distinct, so the +/// generator's deduplication leaves one row each. +fn eq_operations() -> Vec { + vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(1 << 40, 1 << 40, true), + EqOperation::new(0, u64::MAX, true), + ] +} + +/// Signed and unsigned, both orders. +fn lt_operations() -> Vec { + vec![ + LtOperation::new(3, 9, false), + LtOperation::new(9, 3, false), + LtOperation::new(u64::MAX, 1, true), + LtOperation::new(1, u64::MAX, true), + ] +} + +/// Left and right, arithmetic and logical, word and doubleword. +fn shift_operations() -> Vec { + vec![ + ShiftOperation::new(0x0123_4567_89ab_cdef, 4, false, false, false), + ShiftOperation::new(0x0123_4567_89ab_cdef, 4, true, false, false), + ShiftOperation::new(0xffff_ffff_ffff_0000, 8, true, true, false), + ShiftOperation::new(0x0000_0000_dead_beef, 3, false, false, true), + ] +} + +fn bytewise_operations() -> Vec { + vec![ + BytewiseOperation::new(0x0102_0304_0506_0708, 0x1112_1314_1516_1718, 0), + BytewiseOperation::new(0, u64::MAX, 0), + BytewiseOperation::new(7, 7, 1), + BytewiseOperation::new(u64::MAX, 1, 1), + ] +} + +/// Runs the whole argument over one real table and returns its bus output. +fn argue>( + air: &ConcreteVmAir, + num_main_columns: usize, + columns: Vec>>, +) -> Result<(ExtE, ExtE), multilinear::Error> { + let num_vars = columns[0].len().trailing_zeros() as usize; + let layout = || { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + num_main_columns, + num_vars, + Uniforms::default(), + ) + }; + // The trace goes in as it is: base-field. Only the challenges are not. + let table = CommittedTable::from_layout(layout()?, |col| columns[col as usize].clone())?; + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config())?; + + let mut prover = DefaultTranscript::::new(b"vm-table"); + let proof = multilinear_table::multi_prove(&committed, &config(), &mut prover, None)?; + + // One commitment for the whole trace, one opening, one pass over the rows. + assert_eq!(proof.roots.len(), 1); + assert_eq!(proof.columns.len(), 1); + assert_eq!(proof.columns[0].polys.len(), 1); + assert_eq!(proof.tables[0].constraint.sumcheck.rounds.len(), num_vars); + + // The verifier rebuilds the layout from the AIR alone — no trace — and the + // roots it absorbs come out of the proof. + let verifier_layout = layout()?; + let statement = verifier_layout.statement(); + + // A single table's bus need not balance on its own, so the share it owes + // is whatever it produced: what is under test here is the table, not the + // balance. + let owed = multilinear_table::contribution(&proof.tables[0].bus_output) + .ok_or(multilinear::Error::BusImbalance)?; + let mut verifier = DefaultTranscript::::new(b"vm-table"); + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( + &proof, + &[statement], + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &owed, + &config(), + &mut verifier, + None, + )?; + Ok(proof.tables[0].bus_output) +} + +fn argue_eq(columns: Vec>>) -> Result<(ExtE, ExtE), multilinear::Error> { + let options = ProofOptions::default_test_options(); + argue(&create_eq_air(&options), eq::cols::NUM_COLUMNS, columns) +} + +/// The milestone: tables the VM actually proves — their own constraints and +/// their buses — each argued in one sumcheck against one commitment for the +/// whole trace. +#[test] +fn the_eq_table_argues_end_to_end() { + let trace = generate_eq_trace(&eq_operations()); + assert_eq!(trace.columns_main().len(), eq::cols::NUM_COLUMNS); + argue_eq(trace.columns_main()).unwrap(); +} + +#[test] +fn the_lt_table_argues_end_to_end() { + let options = ProofOptions::default_test_options(); + let trace = generate_lt_trace(<_operations()); + argue( + &create_lt_air(&options), + lt::cols::NUM_COLUMNS, + trace.columns_main(), + ) + .unwrap(); +} + +/// SHIFT does not deduplicate: one row per operation with `μ = 1`, which is a +/// different multiplicity shape from EQ's and LT's. +#[test] +fn the_shift_table_argues_end_to_end() { + let options = ProofOptions::default_test_options(); + let trace = generate_shift_trace(&shift_operations()); + argue( + &create_shift_air(&options), + shift::cols::NUM_COLUMNS, + trace.columns_main(), + ) + .unwrap(); +} + +/// BYTEWISE has no constraints of its own: its soundness is entirely its bus, +/// which is the shape BITWISE, PAGE and REGISTER have. +#[test] +fn the_bytewise_table_argues_end_to_end() { + let options = ProofOptions::default_test_options(); + let trace = generate_bytewise_trace(&bytewise_operations()); + argue( + &create_bytewise_air(&options), + bytewise::cols::NUM_COLUMNS, + trace.columns_main(), + ) + .unwrap(); +} + +/// And a trace that breaks one of the table's own constraints is rejected. +#[test] +fn a_broken_eq_row_is_rejected() { + let trace = generate_eq_trace(&eq_operations()); + let mut columns = trace.columns_main(); + // `res = eq XOR invert` is one of the four; flipping the output breaks it. + columns[eq::cols::RES][0] += FieldElement::::one(); + + assert!(argue_eq(columns).is_err()); +} + +/// The multiplicity column is what the buses weigh by, so perturbing it moves +/// the table's share of the bus — the quantity a multi-table proof sums. EQ +/// does not deduplicate and constrains μ to a bit, so the perturbation that +/// still argues takes a row's μ from 1 to 0; a μ of 2 is refused. +#[test] +fn the_multiplicity_column_moves_the_bus_share() { + let trace = generate_eq_trace(&eq_operations()); + let honest = argue_eq(trace.columns_main()).unwrap(); + + let mut columns = trace.columns_main(); + assert_eq!(columns[eq::cols::MU][1], FieldElement::::one()); + columns[eq::cols::MU][1] = FieldElement::::zero(); + let tampered = argue_eq(columns).unwrap(); + + let share = |(p, q): (ExtE, ExtE)| p * q.inv().unwrap(); + assert_ne!(share(honest), share(tampered)); + + let mut columns = trace.columns_main(); + columns[eq::cols::MU][1] += FieldElement::::one(); + assert!(argue_eq(columns).is_err(), "a μ of 2 is not a bit"); +} + +// --------------------------------------------------------------- +// Every live table of a real run. +// --------------------------------------------------------------- + +/// Lays out one table through the `AIR` trait alone, which is all a verifier +/// has: the width and the height, never the values. +fn layout_dyn<'a>( + air: &'a dyn AIR, + num_main_columns: usize, + num_vars: usize, +) -> Result, multilinear::Error> { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + num_main_columns, + num_vars, + Uniforms::default(), + ) +} + +/// Runs a program and proves **every table it leaves in one multi-table +/// proof**, verified by `multi_verify` — which is what checks the bus balance +/// against what the statement owes. +/// +/// The COPY table sends the program's public output on a bus whose +/// counterparty is not another table but the statement, so the tables only sum +/// to zero for a program that outputs nothing. `compute_commit_bus_offset` is +/// the same quantity the univariate verifier demands. +/// +/// ★ Returns the argued tables BY NAME, in `air_trace_pairs` order, because a +/// count cannot say which. Since #977 the set is workload-dependent — an empty +/// table is elided rather than padded — so "how many" stopped being the +/// question and "which ones" became it: a count cannot tell a table that +/// vanished from one that gained a chunk while another vanished. +fn prove_and_verify_all_tables(elf: Elf, logs: &[Log]) -> Vec { + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, logs, &Default::default(), &[]).unwrap(); + let public_output = traces.public_output_bytes.clone(); + + let options = ProofOptions::default_test_options(); + let table_counts = traces.table_counts(); + let airs = crate::VmAirs::new( + &elf, + &options, + true, + &traces.page_configs, + &table_counts, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + crate::test_utils::halted_register_commitment(&options, &elf, &traces.public_output_bytes) + .preprocessed(), + ); + let pairs = airs.air_trace_pairs(&mut traces); + + // The shape of every table: width and height, which is all the verifier + // needs to rebuild the layouts. + let shapes: Vec<(usize, usize)> = pairs + .iter() + .map(|(air, trace, _)| { + let columns = trace.columns_main(); + assert!( + !columns.is_empty() && columns[0].len().is_power_of_two(), + "{}: {} columns of {} rows", + air.name(), + columns.len(), + columns.first().map_or(0, Vec::len) + ); + (columns.len(), columns[0].len().trailing_zeros() as usize) + }) + .collect(); + + // Captured before `tables` is moved into the commitment, and from `pairs` + // rather than from the AIR set, so it is the proof's own sub-proof order. + let names: Vec = pairs + .iter() + .map(|(air, _, _)| air.name().to_string()) + .collect(); + + let mut tables = Vec::with_capacity(pairs.len()); + for ((air, trace, _), &(width, num_vars)) in pairs.iter().zip(&shapes) { + let columns = trace.columns_main(); + let layout = + layout_dyn(*air, width, num_vars).unwrap_or_else(|e| panic!("{}: {e:?}", air.name())); + tables.push( + CommittedTable::from_layout(layout, |col| columns[col as usize].clone()) + .unwrap_or_else(|e| panic!("{}: {e:?}", air.name())), + ); + } + let count = tables.len(); + assert_eq!( + names.len(), + count, + "one name per argued table, or the census below describes another proof" + ); + // Every table's columns in one commitment: 55 of them still open once. + let committed = + CommittedTables::<_, _, KeccakWhir>::commit(tables, &config()).expect("commit every table"); + + let mut prover = DefaultTranscript::::new(b"vm-sweep"); + let proof = multilinear_table::multi_prove(&committed, &config(), &mut prover, None) + .expect("prove every table"); + + // Verifier side: the layouts are rebuilt from the AIRs and the shapes, with + // no trace and no committed table in reach. The roots come from the proof, + // and so does the stack — which the verifier rebuilds from the shapes too. + let verifier_layouts: Vec> = pairs + .iter() + .zip(&shapes) + .map(|((air, _, _), &(width, num_vars))| { + layout_dyn(*air, width, num_vars).expect("rebuild the layout") + }) + .collect(); + let statements: Vec> = verifier_layouts + .iter() + .map(TableLayout::statement) + .collect(); + let stacked = multilinear_table::global_layout(&shapes, config().format.stack) + .expect("rebuild the stack"); + let domain = + multilinear::whir::Domain::::new(stacked.n_stack() + config().log_blowup).unwrap(); + + // The verifier redraws the shared LogUp challenges, so the offset has to be + // computed against the same ones — which means replaying the transcript up + // to that point exactly as `multi_verify` will. + // The block is CALLED rather than re-spelled, so this promise stays true + // when the block changes — which is how the epoch path's replay broke. + let mut probe = DefaultTranscript::::new(b"vm-sweep"); + multilinear_table::absorb_roots::(&mut probe, &proof.roots, &[]); + let z: ExtE = probe.sample_field_element(); + let alpha: ExtE = probe.sample_field_element(); + // `start_index` is the carried x254: zero for a monolithic proof. + let expected = crate::compute_commit_bus_offset(&public_output, 0, &z, &alpha) + .expect("the commit fingerprints are invertible"); + + let mut verifier = DefaultTranscript::::new(b"vm-sweep"); + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( + &proof, + &statements, + std::slice::from_ref(&stacked), + std::slice::from_ref(&domain), + &[statements.len()], + &expected, + &config(), + &mut verifier, + None, + ) + .expect("the whole table set verifies"); + + names +} + +/// The `FIXED_TABLE_COUNT` always-on tables, in `air_trace_pairs` order — the +/// prefix EVERY argued set opens with, whatever the workload runs. +/// +/// One copy, shared by the cases below, because it is the half of each census +/// that is machine shape rather than program shape: a workload cannot add to it +/// or take from it. The counted tables that follow are the program's own. +const ALWAYS_ON_TABLES: [&str; 5] = ["BITWISE", "DECODE", "KECCAK_RC", "REGISTER", "HALT"]; + +/// The shared prefix of `argued`, as `&str` so it compares against the literal +/// censuses below. +fn always_on_prefix(argued: &[String]) -> Vec<&str> { + argued + .iter() + .take(ALWAYS_ON_TABLES.len()) + .map(String::as_str) + .collect() +} + +/// [`ALWAYS_ON_TABLES`] is a census OF `FIXED_TABLE_COUNT`, so it must not be +/// able to drift from it silently — the failure the sibling list in +/// `constraint_artifact_tests` actually had, where it named eleven while the +/// constant said five. +#[test] +fn the_always_on_prefix_is_the_constants_own() { + assert_eq!( + ALWAYS_ON_TABLES.len(), + crate::FIXED_TABLE_COUNT, + "the always-on prefix must name every FIXED_TABLE_COUNT table" + ); +} + +/// **The whole VM through the multilinear path**: every live table of a real +/// run proved in one multi-table proof and verified, buses included. +/// +/// The traces come from the executor, not from hand-written operations, so the +/// widths, the interaction counts, the packings and the multiplicity patterns +/// are whatever the VM actually produces. +/// +/// ★ THE SET BY NAME, NOT A COUNT, and the reason is #977. This asserted +/// `>= 20` until the main-sync merge `892c7d1bc` (main's `c2ac5d546`, #977) +/// took `FIXED_TABLE_COUNT` 11 -> 5 and made COMMIT, KECCAK, KECCAK_RND, ECSM, +/// ECDAS and HINT counted: an empty table is now ELIDED from the proof rather +/// than padded into it, so the argued set is workload-dependent and "the full +/// table set" stopped being a thing a bound could describe. It read 10 here. +/// +/// A count is the wrong instrument for that move twice over. It cannot tell a +/// table that VANISHED from one that gained a chunk while another vanished, and +/// it says nothing about ORDER — which `air_trace_pairs` calls the proof's +/// sub-proof layout in as many words, since `air_refs` must reproduce it. The +/// ordered census fails on any of the three and names which. +/// +/// MEASURED at `788f36a19`, box, CPU-only, no campaign env. `[0]` is a chunk +/// index and `PAGE:0x0` a page base, both as `AIR::name` renders them. +#[test] +fn every_live_table_is_proved_and_verified() { + let (elf, logs, _) = run_asm_elf("sub"); + let argued = prove_and_verify_all_tables(elf, &logs); + assert_eq!( + always_on_prefix(&argued), + ALWAYS_ON_TABLES, + "every argued set opens with the always-on tables; argued: {}", + argued.join(" ") + ); + assert_eq!( + argued, + [ + "BITWISE", + "DECODE", + "KECCAK_RC", + "REGISTER", + "HALT", + "CPU[0]", + "LT[0]", + "MEMW_A[0]", + "PAGE:0x0", + "MEMW_R[0]", + ], + "the argued table set moved — a table appeared, vanished, or the \ + sub-proof order changed" + ); +} + +/// The same over the whole 64-bit instruction set, which lights up tables a +/// two-instruction program never reaches: SHIFT, MUL, DVRM, BYTEWISE and CPU32, +/// five more than `sub`'s ten. +/// +/// ⚠ AND IT IS NOT THE FULL TABLE SET, which is what the old `>= 20` bound and +/// its "expected the full table set" message both claimed. Fifteen tables are +/// argued, and six counted families are absent: MEMW (the unaligned one; only +/// MEMW_A and MEMW_R appear), LOAD, STORE, BRANCH, EQ and COPY — plus every +/// accelerator. Since #977 an absent table means a table with NO ROWS, so this +/// says the `all_instructions_64` fixture executes no branch, no load, no store +/// and no `eq`, which is a narrower program than its name suggests. +/// +/// That gap is PINNED rather than fixed here: widening the fixture is a change +/// to `executor/programs/asm`, and this census is what would notice it. A +/// family arriving fails this test saying which, which is the outcome to want. +/// +/// MEASURED at `788f36a19`, box, CPU-only, no campaign env. +#[test] +fn the_whole_instruction_set_is_proved_and_verified() { + let (elf, logs, _) = run_asm_elf("all_instructions_64"); + let argued = prove_and_verify_all_tables(elf, &logs); + assert_eq!( + always_on_prefix(&argued), + ALWAYS_ON_TABLES, + "every argued set opens with the always-on tables; argued: {}", + argued.join(" ") + ); + assert_eq!( + argued, + [ + "BITWISE", + "DECODE", + "KECCAK_RC", + "REGISTER", + "HALT", + "CPU[0]", + "LT[0]", + "SHIFT[0]", + "MEMW_A[0]", + "MUL[0]", + "DVRM[0]", + "PAGE:0x0", + "MEMW_R[0]", + "BYTEWISE[0]", + "CPU32[0]", + ], + "the argued table set moved — a table appeared, vanished, or the \ + sub-proof order changed" + ); + // NON-VACUITY against the case above: this fixture must actually reach + // further than `sub` did, or the two censuses are one test written twice. + for wider in ["SHIFT[0]", "MUL[0]", "DVRM[0]", "BYTEWISE[0]", "CPU32[0]"] { + assert!( + argued.iter().any(|n| n == wider), + "{wider} is what this case adds over `sub`; argued: {}", + argued.join(" ") + ); + } +} + +/// Proves and verifies the guest at `executor/program_artifacts/rust/.elf`, +/// returning the argued tables by name. +fn prove_and_verify_rust_guest(name: &str) -> Vec { + let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .join("executor/program_artifacts/rust") + .join(format!("{name}.elf")); + let bytes = std::fs::read(&root).unwrap_or_else(|_| panic!("read {}", root.display())); + let elf = Elf::load(&bytes).unwrap_or_else(|e| panic!("load {name}.elf: {e:?}")); + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + prove_and_verify_all_tables(elf, &logs) +} + +/// Is `family`, or `family` plus a chunk index, among the argued tables? +/// +/// "Equal, or followed by `[`" — never a bare prefix, which would let +/// `KECCAK_RC` answer for `KECCAK`. +fn argues(argued: &[String], family: &str) -> bool { + argued.iter().any(|n| { + let s = n.as_str(); + s == family || (s.starts_with(family) && s[family.len()..].starts_with('[')) + }) +} + +/// And over a Rust program that calls the keccak PRECOMPILE — the +/// `keccak_permute` ecall — so KECCAK and KECCAK_RND are argued, **and** which +/// writes public output, so the statement's share of the bus is load-bearing. +/// +/// ⛔ THIS CASE PROVED THE WRONG GUEST UNTIL NOW, and the bound is what hid it. +/// It loaded `keccak.elf`, which is `executor/programs/rust/keccak` — a guest +/// whose `Cargo.toml` depends on `tiny-keccak` and whose `main` hashes in +/// SOFTWARE, issuing no syscall but `commit`. It has never touched a precompile. +/// The measured census at `413b3a3b7` carries COMMIT[0] and no keccak table at +/// all, which is what finally said so. +/// +/// ★ And the claim was false BEFORE #977 too — it was merely unfalsifiable. +/// KECCAK and KECCAK_RND were always-on then, so they appeared in the table set +/// of every workload, reached or not, and a doc sentence about which tables the +/// program "brings in" described MACHINE shape while reading like program +/// behaviour. `892c7d1bc` (main's `c2ac5d546`, #977) made them counted, which +/// turned a latent falsehood into a visible one. Another instance of the +/// pattern in `SOUNDNESS.md`: a claim no assertion defended. +/// +/// The accelerator guest it should have used is `keccak_precompile`, whose +/// `main` calls `lambda_vm_syscalls::keccak::keccak256` over five padding edge +/// cases. It is built by the Makefile's `RUST_PROGRAM_DIRS` wildcard like every +/// other guest, and before this commit **nothing in the prover proved it** — its +/// only reference in the tree is `executor/tests/rust.rs`, which runs it in the +/// executor and never proves it. So the suite had no precompile coverage on the +/// multilinear path at all. +/// +/// ⚠ The census here is by PRESENCE, not an ordered list: this guest has never +/// been proved on this path, so there is no measured set to pin, and inventing +/// one would repeat the mistake above. Its cost is likewise unmeasured — five +/// `keccak256` calls including a multi-block input. +/// +/// ✓ The ELF resolves on a CI prover shard exactly as `keccak.elf` does, and +/// this is written down so the next reader does not re-ask: the lineage's +/// prover-tests job runs `make compile-programs-asm`, `make compile-programs-rust` +/// and `make compile-recursion-elfs` before `cargo nextest run`, and +/// `compile-programs-rust` builds every directory under +/// `executor/programs/rust/` through the `RUST_PROGRAM_DIRS` wildcard — this +/// guest included, with no per-program list to extend. +#[test] +fn a_program_using_a_precompile_is_proved_and_verified() { + let argued = prove_and_verify_rust_guest("keccak_precompile"); + assert_eq!( + always_on_prefix(&argued), + ALWAYS_ON_TABLES, + "every argued set opens with the always-on tables; argued: {}", + argued.join(" ") + ); + // KECCAK_RC is deliberately NOT in this list: it is always-on and asserted + // in the prefix above, so matching it here would let a run that reaches no + // precompile satisfy a check named for one — which is exactly how the + // software guest passed as a precompile test for as long as it did. + for family in ["KECCAK", "KECCAK_RND", "COPY"] { + assert!( + argues(&argued, family), + "{family} must be argued — it is what this case exists for, and \ + since #977 it is a counted table that an unreached workload drops \ + silently; argued: {}", + argued.join(" ") + ); + } +} + +/// The SOFTWARE-hash guest, kept because it is the widest live table set in the +/// suite — and it is the one the precompile case above used to prove. +/// +/// `executor/programs/rust/keccak` hashes with `tiny-keccak` in guest code, so +/// it reaches no hash accelerator and argues the RV64 core broadly instead: 20 +/// tables, including MEMW, LOAD, STORE, BRANCH and EQ — the five families +/// `the_whole_instruction_set_is_proved_and_verified` does NOT reach despite its +/// name. A Rust guest doing ordinary work exercises more of the VM than the asm +/// fixture named for the instruction set, which is worth keeping a case for. +/// +/// It also carries public output and its copies (COPY[0]) and two PAGE tables, +/// one of them the stack page — the only case here that does either. +/// +/// MEASURED at `413b3a3b7`, box, CPU-only, no campaign env; re-measured when its +/// `memcpy`, `memmove` and `memset` moved to the copy accelerator (laptop, +/// CPU-only): COMMIT[0] became COPY[0], and the guest no longer reaches MUL. +#[test] +fn a_software_hash_guest_argues_the_widest_table_set() { + let argued = prove_and_verify_rust_guest("keccak"); + assert_eq!( + argued, + [ + "BITWISE", + "DECODE", + "KECCAK_RC", + "REGISTER", + "HALT", + "COPY[0]", + "CPU[0]", + "LT[0]", + "SHIFT[0]", + "MEMW[0]", + "MEMW_A[0]", + "LOAD[0]", + "BRANCH[0]", + "PAGE:0x0", + "PAGE:0xfffffffffffc0000", + "MEMW_R[0]", + "EQ[0]", + "BYTEWISE[0]", + "STORE[0]", + "CPU32[0]", + ], + "the argued table set moved — a table appeared, vanished, or the \ + sub-proof order changed. NOTE what is absent and must stay absent: \ + KECCAK, KECCAK_RND, ECSM, ECDAS and HINT. This guest hashes in \ + software, and an accelerator appearing here means it is proving \ + something else — which is precisely the confusion the precompile case \ + above lived in. A separate `!argues(..)` loop would restate the list \ + and could only fire after this assertion already had." + ); +} + +/// A proof is only a proof if it can leave the process. Round-trips a real +/// table's proof through both formats the univariate path uses, and verifies +/// the **deserialized** one — a format that loses a field would still verify if +/// we only checked the original. +#[test] +fn a_real_table_proof_survives_serialization() { + let options = ProofOptions::default_test_options(); + let air = create_eq_air(&options); + let columns = generate_eq_trace(&eq_operations()).columns_main(); + let num_vars = columns[0].len().trailing_zeros() as usize; + let layout = || { + TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + eq::cols::NUM_COLUMNS, + num_vars, + Uniforms::default(), + ) + .unwrap() + }; + + let table = CommittedTable::from_layout(layout(), |col| columns[col as usize].clone()).unwrap(); + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config()).unwrap(); + let mut prover = DefaultTranscript::::new(b"serialized"); + let proof = multilinear_table::multi_prove(&committed, &config(), &mut prover, None).unwrap(); + + let json = serde_json::to_vec(&proof).expect("serde round trip"); + let from_json: multilinear_table::MultiProof = + serde_json::from_slice(&json).expect("serde round trip"); + + let bytes = rkyv::to_bytes::(&proof).expect("rkyv round trip"); + let from_rkyv: multilinear_table::MultiProof = + rkyv::from_bytes::<_, rkyv::rancor::Error>(&bytes).expect("rkyv round trip"); + + // The layout the verifier rebuilds, with the trace out of scope. + let verifier_layout = layout(); + let statement = verifier_layout.statement(); + let owed = multilinear_table::contribution(&proof.tables[0].bus_output).unwrap(); + + for (label, round_tripped) in [("serde", &from_json), ("rkyv", &from_rkyv)] { + // The roots travel in the proof, so a format that dropped them would + // fail here rather than pass on the original's. + let mut verifier = DefaultTranscript::::new(b"serialized"); + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( + round_tripped, + &[statement], + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &owed, + &config(), + &mut verifier, + None, + ) + .unwrap_or_else(|e| panic!("{label}: {e:?}")); + } + + assert!(!bytes.is_empty()); +} + +/// Only the main columns are committed: the LogUp auxiliary columns the +/// univariate path needs are gone, replaced by the fraction tree. Checked on +/// every table, since the number of interactions — and so of aux columns — +/// differs. +#[test] +fn the_argument_commits_no_auxiliary_columns() { + let options = ProofOptions::default_test_options(); + + let eq_air = create_eq_air(&options); + let lt_air = create_lt_air(&options); + let shift_air = create_shift_air(&options); + let bytewise_air = create_bytewise_air(&options); + + let cases: [( + &dyn AIR, + usize, + ); 4] = [ + (&eq_air, eq::cols::NUM_COLUMNS), + (<_air, lt::cols::NUM_COLUMNS), + (&shift_air, shift::cols::NUM_COLUMNS), + (&bytewise_air, bytewise::cols::NUM_COLUMNS), + ]; + + for (air, num_main_columns) in cases { + let program = air.constraint_program(); + // The program carries the table's own constraints and then the LogUp + // ones, which this path drops. + assert!( + program.roots.len() > program.num_base, + "the fixture must have LogUp constraints to drop" + ); + assert!( + air.trace_layout().1 > 0, + "the univariate path commits auxiliary columns" + ); + assert_eq!(air.trace_layout().0, num_main_columns); + } +} + +/// And the committed count on a real table is exactly its main width, in one +/// stacked commitment. +#[test] +fn a_real_table_is_one_commitment_for_its_main_columns() { + let options = ProofOptions::default_test_options(); + let air = create_lt_air(&options); + let trace = generate_lt_trace(<_operations()); + let columns = trace.columns_main(); + let num_vars = columns[0].len().trailing_zeros() as usize; + // The trace goes in as it is: base-field. Only the challenges are not. + let table = CommittedTable::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + lt::cols::NUM_COLUMNS, + num_vars, + Uniforms::default(), + |col| columns[col as usize].clone(), + ) + .unwrap(); + + assert_eq!(table.num_committed_columns(), lt::cols::NUM_COLUMNS); + // And they all ride in one stacked polynomial, alone or alongside others. + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config()).unwrap(); + assert_eq!(committed.roots().len(), 1); +} + +// ========================================================================= +// W1-B step 2: the out-of-band preprocessed opening, prover side +// ========================================================================= + +/// Builds a table from the EQ trace plus a commitment over two of its columns, +/// standing in for DECODE's five ELF-derived ones at a scale a test can run. +fn eq_table_and_prepared_columns() -> (Vec>>, Vec>) { + let trace = eq::generate_eq_trace(&eq_operations()); + let columns = trace.columns_main(); + let prepared: Vec> = columns[..2] + .iter() + .map(|c| multilinear::mle::Mle::new(c.clone()).expect("mle")) + .collect(); + (columns, prepared) +} + +/// Proves the EQ table, optionally against a prepared commitment over its first +/// two columns, and hands back the proof. +/// +/// ⚠ The TRACE IS AN ARGUMENT, not regenerated here, and that is load-bearing: +/// `generate_eq_trace` orders its rows by `HashMap` iteration, so two calls +/// produce two different row orders, two different commitments and two +/// different roots. Comparing proofs from two independently generated traces +/// would compare that instead of the thing under test — which is exactly what +/// the first draft of this helper did, and what `a850dd29` fixed for DECODE. +fn prove_eq_with_prepared( + columns: &[Vec>], + prepared_columns: &[multilinear::mle::Mle], + prepared: bool, + table_index: usize, +) -> Result, multilinear::Error> { + let options = ProofOptions::default_test_options(); + let air = create_eq_air(&options); + let num_vars = columns[0].len().trailing_zeros() as usize; + + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + eq::cols::NUM_COLUMNS, + num_vars, + Uniforms::default(), + )?; + let table = CommittedTable::from_layout(layout, |col| columns[col as usize].clone())?; + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config())?; + + let refs: Vec<&multilinear::mle::Mle> = prepared_columns.iter().collect(); + let out_of_band = multilinear::stacked_eval::StackedCommitment::::commit( + multilinear_table::global_layout(&[(refs.len(), num_vars)], config().format.stack)?, + &refs, + None, + &config(), + )?; + + // The leading `refs.len()` preprocessed columns of `table_index` — the + // single-table case of a prepared commitment, which is what this fixture is. + let prepared_at = multilinear_table::leading_columns(table_index, refs.len()); + let mut prover = DefaultTranscript::::new(b"w1b-step2"); + multilinear_table::multi_prove( + &committed, + &config(), + &mut prover, + prepared.then(|| multilinear_table::Prepared { + commitment: &out_of_band, + columns: &refs, + at: &prepared_at, + }), + ) +} + +/// ★★★ THE PROVER-SIDE WIRING GUARD: the prepared root reaches the sponge. +/// +/// Nothing verifies the opening until step 3, so the question this answers is +/// the one that has gone wrong twice on this branch — **is the feature wired at +/// all?** The transcript defaulting to keccak while everything else moved, and +/// the grind dispatching keccak-only with a correct RPX kernel sitting unused, +/// were both "not wired", not "wired wrongly". +/// +/// The absorb lands before `z`, so supplying a prepared commitment must move +/// the whole challenge stream and therefore the proof. If the proofs are equal, +/// the root did not reach the sponge — whatever else the opening contains. +/// +/// ⚠ This is why a `preprocessed.is_some()` assertion would not do: the field +/// can be populated by an opening the transcript never saw. +/// +/// # ⛔ WHAT IT STILL DOES NOT CATCH, measured not assumed +/// +/// Moving the absorb to AFTER the first challenge leaves this test green. Run +/// as a mutation: the proof still differs from the no-prepared case, because +/// `alpha` and `beta` are drawn after the absorb wherever it sits, so the +/// inequality below is satisfied by a root that binds `z` to nothing. +/// +/// So the three placements split cleanly by who catches them: +/// +/// * **not absorbed at all** — this test; +/// * **aimed at the wrong table** — its sibling below; +/// * **absorbed after the first challenge** — NEITHER, today. +/// +/// The last one is the dangerous one and it is currently guarded only by the +/// specification: `the_derived_root_is_absorbed_after_the_carried_ones` proves +/// the orderings are distinguishable, but nothing yet asserts which one the +/// production code uses. A round-trip will not close it either — prover and +/// verifier share the position, so a consistently late absorb verifies. +/// +/// Step 3 closes it by comparing the verifier's roots block against an +/// independently built transcript, which is the same shape as every other check +/// on this branch that held: compare against a CONSTRUCTION, not against the +/// other side. +#[test] +fn a_prepared_commitment_moves_the_challenge_stream() { + // One trace, two proves. See `prove_eq_with_prepared`'s note. + let (columns, prepared_columns) = eq_table_and_prepared_columns(); + let without = + prove_eq_with_prepared(&columns, &prepared_columns, false, 0).expect("prove without"); + let with = prove_eq_with_prepared(&columns, &prepared_columns, true, 0).expect("prove with"); + + assert!( + without.preprocessed.is_none(), + "no prepared commitment was supplied, so nothing should be opened" + ); + assert!( + with.preprocessed.is_some(), + "a prepared commitment was supplied and produced no opening" + ); + + // The roots the proof CARRIES are the same: the derived root is not one of + // them, which is the other half of the design. + assert_eq!( + without.roots, with.roots, + "the prepared root leaked into `MultiProof::roots`; it is derived by the \ + verifier and must never be carried" + ); + + // …and the challenges moved, which is the absorb having happened. + let a = rkyv::to_bytes::(&without.tables).expect("serialize"); + let b = rkyv::to_bytes::(&with.tables).expect("serialize"); + assert_ne!( + a.as_ref(), + b.as_ref(), + "the table arguments are byte-identical with and without the prepared \ + commitment — its root never reached the transcript, so it binds nothing" + ); +} + +/// ★ The table index is checked, not trusted. +/// +/// The opening binds the pinned columns to ONE table's reduced point. An index +/// that names no table must be an error rather than silently opening at +/// whatever point happens to be first. +#[test] +fn a_prepared_commitment_aimed_at_no_table_is_rejected() { + let (columns, prepared_columns) = eq_table_and_prepared_columns(); + let err = prove_eq_with_prepared(&columns, &prepared_columns, true, 7) + .expect_err("an out-of-range table index must fail"); + assert!( + matches!(err, multilinear::Error::UnknownPolynomial { .. }), + "expected the index to be reported as unknown, got {err:?}" + ); +} + +/// ★★★ ONE PREPARED STACK, TWO TABLES, TWO POINTS. +/// +/// DECODE's prepared commitment covers ONE table's leading preprocessed columns, +/// so every column in it shares that table's single reduced point. The +/// cross-epoch genesis opening covers one GLOBAL_MEMORY table PER DENSE PAGE, +/// each with a reduced point of its own — which is why +/// [`multilinear_table::Prepared`] names a LIST of +/// [`multilinear_table::PreparedColumn`] rather than a table and a count. +/// +/// This is that shape at two tables: EQ and LT in one proof, a stack over EQ's +/// columns `{0, 1}` and LT's columns `{0, 1}`, settled at the two points those +/// two arguments reduced to. +/// +/// ⚠ BOTH OF EACH TABLE'S COLUMNS, because `check_preprocessed` skips a PREFIX. +/// A stack over column 1 alone would be inexpressible as a count, and +/// `prefix_at` refuses it rather than letting the count reinterpret it — which +/// [`a_prepared_stack_that_is_not_a_prefix_is_refused`] is the arm for. +/// +/// ⚠ EQ AND LT AND NOT TWO COPIES OF ONE TABLE, deliberately. The swap arm is a +/// no-op — and therefore a test that cannot fail — if the two tables' columns +/// can coincide. Two different tables with different traces cannot, and the +/// assertion below says so before anything is proven. +fn prove_and_verify_two_table_prepared( + prover_at: &[multilinear_table::PreparedColumn], + verifier_at: &[multilinear_table::PreparedColumn], +) -> Result<(), multilinear::Error> { + let options = ProofOptions::default_test_options(); + let eq_air = create_eq_air(&options); + let lt_air = create_lt_air(&options); + let eq_columns = generate_eq_trace(&eq_operations()).columns_main(); + let lt_columns = generate_lt_trace(<_operations()).columns_main(); + + let eq_vars = eq_columns[0].len().trailing_zeros() as usize; + let lt_vars = lt_columns[0].len().trailing_zeros() as usize; + + // The two preprocessed columns each table presents — the same shape a + // cross-epoch genesis page has, where they are OFFSET and INIT. + let preprocessed_of = |columns: &[Vec>]| -> Vec> { + columns[..2] + .iter() + .map(|c| multilinear::mle::Mle::new(c.clone()).expect("mle")) + .collect() + }; + let eq_preprocessed = preprocessed_of(&eq_columns); + let lt_preprocessed = preprocessed_of(<_columns); + + // ⚠ ANTI-VACUITY, asserted before the proof: if the two tables' stacked + // columns were equal the swap arm would be a no-op and could not fail. + assert_ne!( + eq_preprocessed[1].evals(), + lt_preprocessed[1].evals(), + "the two tables' stacked columns are identical, so swapping them proves nothing" + ); + + let eq_layout = || { + TableLayout::::new( + eq_air.constraint_program(), + eq_air.constraints_meta(), + eq_air.bus_interactions(), + eq::cols::NUM_COLUMNS, + eq_vars, + Uniforms::default(), + ) + }; + let lt_layout = || { + TableLayout::::new( + lt_air.constraint_program(), + lt_air.constraints_meta(), + lt_air.bus_interactions(), + lt::cols::NUM_COLUMNS, + lt_vars, + Uniforms::default(), + ) + }; + + let tables = vec![ + CommittedTable::from_layout(eq_layout()?, |col| eq_columns[col as usize].clone())?, + CommittedTable::from_layout(lt_layout()?, |col| lt_columns[col as usize].clone())?, + ]; + let committed = CommittedTables::<_, _, KeccakWhir>::commit(tables, &config())?; + + // The stack: both of table 0's preprocessed columns, then both of table 1's. + // The ORDER is the contract — run `k` of the commitment settles table `k`. + let stack_refs: Vec<&multilinear::mle::Mle> = vec![ + &eq_preprocessed[0], + &eq_preprocessed[1], + <_preprocessed[0], + <_preprocessed[1], + ]; + let out_of_band = multilinear::stacked_eval::StackedCommitment::::commit( + multilinear_table::global_layout(&[(2, eq_vars), (2, lt_vars)], config().format.stack)?, + &stack_refs, + None, + &config(), + )?; + + let mut prover = DefaultTranscript::::new(b"w1i-two-table"); + let proof = multilinear_table::multi_prove( + &committed, + &config(), + &mut prover, + Some(multilinear_table::Prepared { + commitment: &out_of_band, + columns: &stack_refs, + at: prover_at, + }), + )?; + + // The verifier rebuilds both layouts from the AIRs alone and carries each + // table's preprocessed columns, so the checks the opening replaces are + // checks that exist to be replaced. + let eq_verifier = eq_layout()?; + let lt_verifier = lt_layout()?; + let statements: Vec> = vec![ + eq_verifier.statement_with_preprocessed(&eq_preprocessed), + lt_verifier.statement_with_preprocessed(<_preprocessed), + ]; + + // Neither table's bus balances alone; what they owe together is what they + // produced. The balance is not what is under test here. + let mut owed = ExtE::zero(); + for table in &proof.tables { + owed += multilinear_table::contribution(&table.bus_output) + .ok_or(multilinear::Error::BusImbalance)?; + } + + let roots = out_of_band.roots(); + let mut verifier = DefaultTranscript::::new(b"w1i-two-table"); + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( + &proof, + &statements, + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &owed, + &config(), + &mut verifier, + Some(multilinear_table::PreparedCheck { + roots: &roots, + layout: out_of_band.layout(), + domain: out_of_band.domain(), + at: verifier_at, + }), + ) +} + +/// Both preprocessed columns of table 0, then both of table 1 — the stack's own +/// column order. +fn two_table_at() -> Vec { + let mut at = multilinear_table::leading_columns(0, 2); + at.extend(multilinear_table::leading_columns(1, 2)); + at +} + +/// ★ THE HONEST CONTROL, and it comes first: a stack spanning two tables, whose +/// columns are settled at two different reduced points, verifies. +/// +/// Without this the refusals below would be satisfied by a path that rejects +/// everything. +#[test] +fn a_prepared_stack_settles_two_tables_at_their_own_points() { + let at = two_table_at(); + prove_and_verify_two_table_prepared(&at, &at) + .expect("an honest two-table prepared opening must verify"); +} + +/// ★★ EACH TABLE'S COLUMNS ARE BOUND TO ITS OWN POINT, adversarially. +/// +/// The verifier is told the stack holds LT's columns first and EQ's second, +/// while the commitment holds them the other way round. Every root, every table +/// argument and every carried value is the honest one — only the DESTINATION of +/// each run moves. On the cross-epoch path this is one dense page's genesis +/// being settled against another page's claim, which is the forgery the opening +/// exists to stop, and it is why the stack's column order is a contract rather +/// than a convention. +#[test] +fn a_prepared_stack_with_two_tables_swapped_is_refused() { + let honest = two_table_at(); + let mut swapped = multilinear_table::leading_columns(1, 2); + swapped.extend(multilinear_table::leading_columns(0, 2)); + let err = prove_and_verify_two_table_prepared(&honest, &swapped) + .expect_err("a stack settled against the wrong tables' claims must be refused"); + println!("SWAPPED-TABLES REFUSAL: {err:?}"); +} + +/// ⛔ A STACK THAT IS NOT A PREFIX IS REFUSED RATHER THAN REINTERPRETED. +/// +/// `check_preprocessed` skips a table's FIRST `n` preprocessed columns, so a +/// commitment over `{1, 0}` — or over `{1}` alone — cannot be expressed by the +/// count that drives it. `PreparedColumn` CAN describe one, which is exactly why +/// `prefix_at` has to refuse it: reinterpreted as a count it would have the host +/// skip columns the opening never settled, with every value gate still green. +#[test] +fn a_prepared_stack_that_is_not_a_prefix_is_refused() { + let honest = two_table_at(); + // Table 0's two columns named in the wrong order. + let mut reversed = vec![honest[1], honest[0]]; + reversed.extend_from_slice(&honest[2..]); + let err = prove_and_verify_two_table_prepared(&honest, &reversed) + .expect_err("a non-prefix prepared commitment must be refused"); + println!("NON-PREFIX REFUSAL: {err:?}"); +} + +/// ⛔ AND SO IS A STACK WHOSE TABLES INTERLEAVE. +/// +/// The opening takes each table's claims as the contiguous run +/// `values[start..start + n]`. Entries interleaved with another table's would +/// settle this table's commitment against a slice that is not its own, so +/// contiguity is part of the contract and not a tidiness rule. +#[test] +fn a_prepared_stack_whose_tables_interleave_is_refused() { + let honest = two_table_at(); + let interleaved = vec![honest[0], honest[2], honest[1], honest[3]]; + let err = prove_and_verify_two_table_prepared(&honest, &interleaved) + .expect_err("an interleaved prepared commitment must be refused"); + println!("INTERLEAVED REFUSAL: {err:?}"); +} diff --git a/prover/src/tests/multiplicity_forgery_poc.rs b/prover/src/tests/multiplicity_forgery_poc.rs index e2a46447c..b7da15da9 100644 --- a/prover/src/tests/multiplicity_forgery_poc.rs +++ b/prover/src/tests/multiplicity_forgery_poc.rs @@ -39,10 +39,9 @@ //! //! Run under production proof options. -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use stark::lookup::BusInteraction; use stark::proof::options::ProofOptions; -use stark::prover::{IsStarkProver, Prover}; +use stark::prover::IsStarkProver; use stark::trace::TraceTable; use crate::statement::{StatementKind, absorb_statement}; @@ -50,7 +49,7 @@ use crate::tables::bitwise::{cols as bw_cols, row_index as bw_row_index}; use crate::tables::trace_builder::Traces; use crate::tables::types::{BusId, FE, GoldilocksExtension as Ext, GoldilocksField as Base}; use crate::tables::{dvrm, mul}; -use crate::test_utils::{E, asm_elf_bytes}; +use crate::test_utils::asm_elf_bytes; use crate::{MaxRowsConfig, VmAirs, VmProof}; use executor::elf::Elf; @@ -168,15 +167,8 @@ fn craft_proof(s: &Scenario, forge: Forge) -> Result Result Result::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -220,11 +213,10 @@ fn craft_proof(s: &Scenario, forge: Forge) -> Result`. +//! +//! The goldens at the end pin the host bytes of P1 proofs at the production +//! cap (`P1StarkHash`); `zf_p1_device_tests` proves the same cases on the card +//! and requires the same bytes. Regenerate them only for a deliberate format +//! change: `cargo test -p lambda-vm-prover --lib +//! tests::p1_stark_tests::print_p1_goldens -- --ignored --nocapture`. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::merkle_tree::traits::IsStreamingLeafBackend; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use sha2::{Digest, Sha256}; +use stark::config::StarkHash; +use stark::examples::read_only_memory_logup::{ + LogReadOnlyPublicInputs, LogReadOnlyRAP, read_only_logup_trace, +}; +use stark::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use stark::proof::options::{CapPolicy, FriMode, ProofFormat, ProofOptions}; +use stark::proof::stark::StarkProof; +use stark::prover::{GenericProver, IsStarkProver}; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::{GenericVerifier, IsStarkVerifier}; + +use crate::lfm::algebraic_commit::RpxStarkHash; +use crate::lfm::algebraic_transcript::AlgebraicTranscript; +use crate::lfm::hash::HasherKind; +use crate::lfm::p1_commit::{P1BatchBackend, P1StarkHash, P1StarkHashCapped, P1Transcript}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; +type LogupProof = StarkProof>; + +fn options(blowup: u8, queries: usize, grinding: u8, format: ProofFormat) -> ProofOptions { + ProofOptions { + blowup_factor: blowup, + fri_number_of_queries: queries, + coset_offset: 3, + grinding_factor: grinding, + fri_final_poly_log_degree: 1, + format, + } +} + +fn formats() -> [(&'static str, ProofFormat); 3] { + [ + ("legacy", ProofFormat::LEGACY), + ( + "dp", + ProofFormat { + fri_mode: FriMode::Dp, + ..ProofFormat::LEGACY + }, + ), + ( + "one-row", + ProofFormat { + one_row: stark::proof::options::OneRowMode::Auto, + ..ProofFormat::LEGACY + }, + ), + ] +} + +fn prove_addition( + rows: usize, + o: &ProofOptions, +) -> ( + SimpleAdditionAIR, + StarkProof>, +) { + let air = SimpleAdditionAIR::::new(o); + let pi = SimpleAdditionPublicInputs { + a: Felt::from(1u64), + b: Felt::from(2u64), + }; + let mut trace = simple_addition_trace::(rows); + let proof = GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof) +} + +fn verify_addition( + air: &SimpleAdditionAIR, + proof: &StarkProof>, +) -> bool { + GenericVerifier::::verify(proof, air, &mut DefaultTranscript::::new(&[])) +} + +fn logup_trace(rows: usize) -> (TraceTable, LogReadOnlyPublicInputs) { + let addr: Vec = (0..rows).map(|i| Felt::from((i % 5) as u64 + 1)).collect(); + let val: Vec = (0..rows) + .map(|i| Felt::from(((i % 5) as u64 + 1) * 10)) + .collect(); + let trace: TraceTable = read_only_logup_trace(addr, val); + let cols = trace.columns_main(); + let pi = LogReadOnlyPublicInputs { + a0: cols[0][0], + v0: cols[1][0], + a_sorted_0: cols[2][0], + v_sorted_0: cols[3][0], + m0: cols[4][0], + }; + (trace, pi) +} + +/// A logup proof under the configuration `H` on ZisK's transcript. +pub(crate) fn prove_logup_under(rows: usize, o: &ProofOptions) -> LogupProof { + let (mut trace, pi) = logup_trace(rows); + let air = LogReadOnlyRAP::::new(o); + GenericProver::::prove(&air, &mut trace, &pi, &mut P1Transcript::new()) + .expect("proving must succeed") +} + +/// The verifier of `H` on ZisK's transcript, with its own AIR at `o`. +pub(crate) fn verify_logup_under(o: &ProofOptions, proof: &LogupProof) -> bool { + GenericVerifier::::verify( + proof, + &LogReadOnlyRAP::::new(o), + &mut P1Transcript::new(), + ) +} + +/// [`prove_logup_under`] under the P1 configuration capped at 4-ary height +/// `c` (0: uncapped). +fn prove_logup_at(c: u8, rows: usize, o: &ProofOptions) -> LogupProof { + match c { + 0 => prove_logup_under::>(rows, o), + 1 => prove_logup_under::>(rows, o), + 2 => prove_logup_under::>(rows, o), + 3 => prove_logup_under::>(rows, o), + 4 => prove_logup_under::>(rows, o), + 5 => prove_logup_under::>(rows, o), + _ => panic!("no P1 configuration at cap {c} in these tests"), + } +} + +/// The verifier of the P1 configuration capped at 4-ary height `c`. +fn verify_logup_at(c: u8, o: &ProofOptions, proof: &LogupProof) -> bool { + match c { + 0 => verify_logup_under::>(o, proof), + 1 => verify_logup_under::>(o, proof), + 2 => verify_logup_under::>(o, proof), + 3 => verify_logup_under::>(o, proof), + 4 => verify_logup_under::>(o, proof), + 5 => verify_logup_under::>(o, proof), + _ => panic!("no P1 configuration at cap {c} in these tests"), + } +} + +/// Every Merkle path node a proof carries (trace, composition and FRI +/// openings): what a cap changes. +fn path_nodes(proof: &LogupProof) -> usize { + let trace: usize = proof + .deep_poly_openings + .iter() + .map(|o| { + o.main_trace_polys.proof.merkle_path.len() + + o.aux_trace_polys + .as_ref() + .map_or(0, |p| p.proof.merkle_path.len()) + + o.composition_poly.proof.merkle_path.len() + }) + .sum(); + let fri: usize = proof + .query_list + .iter() + .flat_map(|q| q.layers_auth_paths.iter().map(|p| p.merkle_path.len())) + .sum(); + trace + fri +} + +/// A logup proof at cap height `c` under every format, verified at `c` and +/// refused by the verifiers of the neighbouring heights `lo` and `hi`. +fn capped_round_trip(c: u8, lo: u8, hi: u8) { + for (name, format) in formats() { + for rows in [128usize, 256] { + let o = options(4, 30, 0, format); + let proof = prove_logup_at(c, rows, &o); + assert!(verify_logup_at(c, &o, &proof), "{name} rows {rows} c {c}"); + assert!( + !verify_logup_at(lo, &o, &proof), + "{name} rows {rows} c {c} by {lo}" + ); + // A height at or past every tree's levels clamps to them: the + // next height up is the same configuration there. + let trace_levels = (rows.trailing_zeros() as usize + 1).div_ceil(2); + if (c as usize) < trace_levels { + assert!( + !verify_logup_at(hi, &o, &proof), + "{name} rows {rows} c {c} by {hi}" + ); + } + } + } +} + +#[test] +fn p1_proofs_round_trip_at_every_tree_shape() { + // Rows 2^8 and 2^9 at blowup 4: LDE depths 10 and 11, even and odd, so + // both the full and the zero-padded top level are walked; uncapped and at + // the production cap, on the byte transcript and on ZisK's. + for (name, format) in formats() { + for rows in [256usize, 512] { + let o = options(4, 9, 4, format); + let (air, proof) = prove_addition::>(rows, &o); + assert!( + verify_addition::>(&air, &proof), + "{name} rows {rows}" + ); + let (air, proof) = prove_addition::(rows, &o); + assert!( + verify_addition::(&air, &proof), + "{name} rows {rows} capped" + ); + } + let o = options(4, 9, 4, format); + let proof = prove_logup_at(0, 128, &o); + assert!(verify_logup_at(0, &o, &proof), "{name} logup"); + let proof = prove_logup_under::(128, &o); + assert!( + verify_logup_under::(&o, &proof), + "{name} logup capped" + ); + } +} + +#[test] +fn the_formats_binary_cap_policy_does_not_reach_arity_4() { + // `cap = auto` caps a binary tree opened this often; an arity-4 + // configuration runs its own cap whatever the format says, so `auto` and + // `off` give the same proof under every P1 cap. + let auto = ProofFormat { + merkle_cap: CapPolicy::Auto, + ..ProofFormat::LEGACY + }; + // No grinding: the proofs must draw the same queries. + let (oa, ol) = ( + options(4, 30, 0, auto), + options(4, 30, 0, ProofFormat::LEGACY), + ); + let a = prove_logup_at(0, 128, &oa); + let b = prove_logup_at(0, 128, &ol); + assert!(verify_logup_at(0, &oa, &a)); + assert_eq!( + a.deep_poly_openings[0].main_trace_polys.proof.merkle_path, + b.deep_poly_openings[0].main_trace_polys.proof.merkle_path + ); + let a = prove_logup_at(2, 128, &oa); + let b = prove_logup_at(2, 128, &ol); + assert_eq!( + a.deep_poly_openings[1].main_trace_polys.proof.merkle_path, + b.deep_poly_openings[1].main_trace_polys.proof.merkle_path + ); + // The production format's own binary policy (`ZfFormat::DEFAULT`, cap + // auto) leaves a P1 proof as the uncapped-format one. + let prod = options(4, 30, 0, crate::zf_format::ZfFormat::DEFAULT.proof_format()); + let dp = options( + 4, + 30, + 0, + ProofFormat { + merkle_cap: CapPolicy::Off, + ..prod.format + }, + ); + let a = prove_logup_under::(128, &prod); + let b = prove_logup_under::(128, &dp); + assert!(verify_logup_under::(&dp, &a)); + assert_eq!( + rkyv::to_bytes::(&a).expect("rkyv")[..], + rkyv::to_bytes::(&b).expect("rkyv")[..] + ); +} + +#[test] +fn p1_caps_round_trip_at_every_height_and_bind_their_height() { + // 128 and 256 rows at blowup 4: trace trees of 256 and 512 row-pair leaves + // (binary depths 8 and 9, four and five 4-ary levels), FRI layers below. + capped_round_trip(1, 0, 2); + capped_round_trip(2, 1, 3); + capped_round_trip(3, 2, 4); + capped_round_trip(4, 3, 5); +} + +#[test] +fn a_cap_shortens_the_paths_by_its_height() { + // 256 rows: the trace trees are 9 binary levels deep, five 4-ary levels, + // the last a two-node group. Uncapped: 15 siblings an opening; at height + // 2: 9 kept, and query 0 carries the 2·4 = 8 cap nodes. + let o = options(4, 30, 0, ProofFormat::LEGACY); + let open = prove_logup_at(0, 256, &o); + let capped = prove_logup_at(2, 256, &o); + assert!(verify_logup_at(2, &o, &capped)); + let main = |p: &LogupProof, q: usize| { + p.deep_poly_openings[q] + .main_trace_polys + .proof + .merkle_path + .len() + }; + assert_eq!(main(&open, 0), 15); + assert_eq!(main(&open, 1), 15); + assert_eq!(main(&capped, 0), 9 + 8); + assert_eq!(main(&capped, 1), 9); + assert!(path_nodes(&capped) < path_nodes(&open)); + // The opened values and roots are the same: a cap moves path nodes only. + assert_eq!( + open.lde_trace_main_merkle_root, + capped.lde_trace_main_merkle_root + ); + assert_eq!(open.fri_layers_merkle_roots, capped.fri_layers_merkle_roots); +} + +#[test] +fn a_tampered_capped_p1_proof_is_rejected() { + let o = options(4, 30, 0, ProofFormat::LEGACY); + let proof = prove_logup_at(2, 256, &o); + assert!(verify_logup_at(2, &o, &proof)); + let ok = |p: &LogupProof| verify_logup_at(2, &o, p); + // A cap node on the owner path (query 0's main opening: 9 siblings, then + // the 8 cap nodes), of the main, aux and composition trees. + for at in [9usize, 12, 16] { + let mut bad = proof.clone(); + bad.deep_poly_openings[0].main_trace_polys.proof.merkle_path[at][0] ^= 1; + assert!(!ok(&bad), "main cap node {}", at - 9); + } + let mut bad = proof.clone(); + bad.deep_poly_openings[0] + .aux_trace_polys + .as_mut() + .unwrap() + .proof + .merkle_path[10][3] ^= 1; + assert!(!ok(&bad), "aux cap node"); + let mut bad = proof.clone(); + bad.deep_poly_openings[0].composition_poly.proof.merkle_path[11][7] ^= 1; + assert!(!ok(&bad), "composition cap node"); + // A kept sibling of another opening; a FRI owner path's cap node. + let mut bad = proof.clone(); + bad.deep_poly_openings[3].main_trace_polys.proof.merkle_path[4][1] ^= 1; + assert!(!ok(&bad), "kept sibling"); + let mut bad = proof.clone(); + let fri0 = &mut bad.query_list[0].layers_auth_paths[0].merkle_path; + let last = fri0.len() - 1; + fri0[last][2] ^= 1; + assert!(!ok(&bad), "FRI cap node"); + // The owner path one node short, another opening one node long, the cap + // moved to the second opening. + let mut bad = proof.clone(); + bad.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path + .pop(); + assert!(!ok(&bad), "owner short"); + let mut bad = proof.clone(); + bad.deep_poly_openings[1] + .main_trace_polys + .proof + .merkle_path + .push([0u8; 32]); + assert!(!ok(&bad), "opening long"); + let mut bad = proof.clone(); + let cap: Vec<_> = bad.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path + .split_off(9); + bad.deep_poly_openings[1] + .main_trace_polys + .proof + .merkle_path + .extend(cap); + assert!(!ok(&bad), "cap moved"); +} + +#[test] +fn a_tampered_p1_proof_is_rejected() { + let o = options(4, 9, 4, ProofFormat::LEGACY); + let proof = prove_logup_at(0, 128, &o); + let ok = |p: &LogupProof| verify_logup_at(0, &o, p); + assert!(ok(&proof)); + // 128 rows at blowup 4: 512 LDE rows in 256 row-pair leaves, four 4-ary + // levels of three siblings (a binary path would be 8 nodes). + assert_eq!( + proof.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path + .len(), + 12 + ); + // A trace opening's authentication path, one byte of one sibling. + let mut bad = proof.clone(); + bad.deep_poly_openings[0].main_trace_polys.proof.merkle_path[0][0] ^= 1; + assert!(!ok(&bad), "trace path sibling"); + // A FRI layer's path. + let mut bad = proof.clone(); + bad.query_list[0].layers_auth_paths[0].merkle_path[1][5] ^= 1; + assert!(!ok(&bad), "FRI path sibling"); + // A root. + let mut bad = proof.clone(); + bad.fri_layers_merkle_roots[0][31] ^= 1; + assert!(!ok(&bad), "FRI root"); + let mut bad = proof.clone(); + bad.lde_trace_main_merkle_root[0] ^= 1; + assert!(!ok(&bad), "main root"); + // The nonce. + let mut bad = proof.clone(); + bad.nonce = bad.nonce.map(|n| n ^ 1); + assert!(!ok(&bad), "nonce"); +} + +// ===================================================================== +// REV-P1-B's configuration-level tests (#1013 `rev_p1_b_tests`, the subset +// REV-P1-JUDGE §7 keeps that does not touch the base's statement). +// ===================================================================== + +fn prove_rpx(o: &ProofOptions) -> LogupProof { + let (mut trace, pi) = logup_trace(128); + let air = LogReadOnlyRAP::::new(o); + GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut AlgebraicTranscript::new(HasherKind::Rpx), + ) + .expect("RPX proves") +} + +fn verify_rpx(o: &ProofOptions, proof: &LogupProof) -> bool { + GenericVerifier::::verify( + proof, + &LogReadOnlyRAP::::new(o), + &mut AlgebraicTranscript::new(HasherKind::Rpx), + ) +} + +/// One extra zero on the symmetric main row (10 → 11 felts, still one block) +/// moves the P1 leaf hash, through the width tag in its first capacity; and +/// the verifier's AIR-derived width checks refuse the proof as before. Two +/// bindings, not one. +#[test] +fn a_zero_padded_opening_is_refused_by_the_p1_leaf_and_the_verifier() { + let o = options(4, 30, 0, ProofFormat::LEGACY); + let proof = prove_logup_at(0, 128, &o); + assert!(verify_logup_at(0, &o, &proof)); + + let main = &proof.deep_poly_openings[0].main_trace_polys; + let (ev, sym) = (main.evaluations.clone(), main.evaluations_sym.clone()); + assert_eq!(ev.len() + sym.len(), 10, "five main columns, a row pair"); + let mut sym0 = sym.clone(); + sym0.push(Felt::zero()); + type P1B = P1BatchBackend; + assert_ne!( + >::hash_data_from_slices(&ev, &sym), + >::hash_data_from_slices(&ev, &sym0), + "the P1 leaf hash sees the extra zero (the width tag)" + ); + + let mut bad = proof.clone(); + for q in &mut bad.deep_poly_openings { + q.main_trace_polys.evaluations_sym.push(Felt::zero()); + } + assert!(!verify_logup_at(0, &o, &bad), "the width checks refuse it"); +} + +/// The configurations do not cross-verify: a P1 proof under RPX and an RPX +/// proof under P1 are refused, and so is a P1 proof replayed on the RPX +/// transcript (the half flip), or under the P1 configuration at another cap. +#[test] +fn p1_and_rpx_proofs_do_not_cross_verify() { + let o = options(4, 30, 4, ProofFormat::LEGACY); + let p1 = prove_logup_under::(128, &o); + let rpx = prove_rpx(&o); + assert!(verify_logup_under::(&o, &p1)); + assert!(verify_rpx(&o, &rpx)); + assert!(!verify_rpx(&o, &p1), "P1 proof, RPX verifier"); + assert!( + !verify_logup_under::(&o, &rpx), + "RPX proof, P1 verifier" + ); + let half_flip = GenericVerifier::::verify( + &p1, + &LogReadOnlyRAP::::new(&o), + &mut AlgebraicTranscript::new(HasherKind::Rpx), + ); + assert!(!half_flip, "P1 commitments on the RPX transcript"); + assert!( + !verify_logup_under::>(&o, &p1), + "the production cap's proof under the uncapped configuration" + ); +} + +/// Every cap node is bound, including the ones no query lands on: those are +/// refused only by the cap-to-root check (`verify_cap_shaped`), which the +/// capped tamper test above does not reach (its flipped nodes are all hit by +/// some opening). 256 rows at blowup 4: 512 row-pair leaves, binary depth 9, +/// five 4-ary levels; at height 4 the owner path keeps 3 siblings and carries +/// the 2·4³ = 128 cap nodes, of which 30 queries reach at most 30. +#[test] +fn every_cap_node_is_bound_by_the_cap_root() { + let o = options(4, 30, 0, ProofFormat::LEGACY); + let proof = prove_logup_at(4, 256, &o); + assert!(verify_logup_at(4, &o, &proof)); + let owner = &proof.deep_poly_openings[0] + .main_trace_polys + .proof + .merkle_path; + assert_eq!(owner.len(), 3 + 128, "3 kept siblings and the 128-node cap"); + let refused = (3..owner.len()) + .filter(|&k| { + let mut bad = proof.clone(); + bad.deep_poly_openings[0].main_trace_polys.proof.merkle_path[k][31] ^= 1; + !verify_logup_at(4, &o, &bad) + }) + .count(); + assert_eq!(refused, 128, "every cap node of the main tree is bound"); +} + +// ===================================================================== +// Goldens: the host bytes of P1 proofs at the production cap. +// ===================================================================== + +fn hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// The production configuration's goldens: logup proofs on ZisK's transcript +/// at grinding 0 (reproducible bytes), at LDE `2^12` and `2^13` (trace trees +/// of odd and even binary depth, so the padded and the full top are both +/// committed and capped) under the legacy pair format and the production +/// format (`fri = dp`). Every case is at or above an LDE of 4096, so the card +/// commits it once `LAMBDA_VM_GPU_LDE_THRESHOLD` is lowered to 4096 or below. +pub(crate) fn p1_golden_cases() -> Vec<(String, ProofOptions, usize)> { + let prod = crate::zf_format::ZfFormat::DEFAULT.proof_format(); + let mut out = Vec::new(); + for (fname, format) in [("legacy", ProofFormat::LEGACY), ("prod", prod)] { + for rows in [1024usize, 2048] { + out.push(( + format!("logup/p1c{}/{fname}/rows{rows}/blowup4", P1_CAP), + options(4, 30, 0, format), + rows, + )); + } + } + out +} + +const P1_CAP: u8 = crate::lfm::p1_commit::P1_STARK_CAP; + +/// One line per golden case: the case's name and the SHA-256 of the proof's +/// rkyv bytes. +pub(crate) fn p1_goldens() -> Vec<(String, String)> { + p1_golden_cases() + .into_iter() + .map(|(name, o, rows)| { + let proof = prove_logup_under::(rows, &o); + assert!(verify_logup_under::(&o, &proof), "{name}"); + let bytes = rkyv::to_bytes::(&proof).expect("rkyv"); + (name, hex(&bytes)) + }) + .collect() +} + +/// The pins, generated by `print_p1_goldens` on the host. +pub(crate) const P1_GOLDENS: &[(&str, &str)] = &[ + ( + "logup/p1c2/legacy/rows1024/blowup4", + "d170f7af9bdf57279c94873aeeb67d5f32e3979eb4f4b9da600c2fc7f287a675", + ), + ( + "logup/p1c2/legacy/rows2048/blowup4", + "bc1593198a5d79275a08c74015e0e9bbd9482b4353bea26135f3fcf883d2f2c9", + ), + ( + "logup/p1c2/prod/rows1024/blowup4", + "06befcfb5f6ca85353b211dc324a6f1bf8b6b2b8c3fa6f7d4248784c045c85df", + ), + ( + "logup/p1c2/prod/rows2048/blowup4", + "1096736fcae6062250c6e12783f2471f315be8e84b001c97f9e36427c6cc7f0b", + ), +]; + +#[test] +fn p1_goldens_are_byte_identical() { + let got = p1_goldens(); + assert_eq!(got.len(), P1_GOLDENS.len(), "one pin per case"); + for ((name, digest), (pin_name, pin)) in got.iter().zip(P1_GOLDENS) { + assert_eq!(name, pin_name); + assert_eq!(digest, pin, "{name}: the P1 proof's bytes moved"); + } +} + +#[test] +#[ignore = "generator for P1_GOLDENS"] +fn print_p1_goldens() { + for (name, digest) in p1_goldens() { + println!("P1 GOLDEN (\"{name}\", \"{digest}\"),"); + } +} diff --git a/prover/src/tests/page_offset_forgery_poc.rs b/prover/src/tests/page_offset_forgery_poc.rs index 5da7d1955..5dae1f0ae 100644 --- a/prover/src/tests/page_offset_forgery_poc.rs +++ b/prover/src/tests/page_offset_forgery_poc.rs @@ -32,16 +32,20 @@ //! Run under **production** proof options, not `default_test_options()`, so none //! of this can be written off as an artefact of a low-query configuration. -use crypto::fiat_shamir::default_transcript::DefaultTranscript; +// The bare `crypto` DefaultTranscript's type default is keccak, which since the +// stage-6 flip is NOT the configuration's transcript. This harness must sample +// Fiat-Shamir from the hash production replays (`DefaultStarkHash`), or every +// honest proof it builds is rejected at challenge derivation. Same half-flip +// `config.rs` warns about; the warning applies to test harnesses too. use stark::proof::options::ProofOptions; -use stark::prover::{IsStarkProver, Prover}; +use stark::prover::IsStarkProver; use crate::statement::{StatementKind, absorb_statement}; use crate::tables::bitwise::{cols as bw_cols, row_index as bw_row_index}; use crate::tables::page::cols as page_cols; use crate::tables::trace_builder::Traces; use crate::tables::types::{FE, VmTable}; -use crate::test_utils::{E, asm_elf_bytes}; +use crate::test_utils::asm_elf_bytes; use crate::{MaxRowsConfig, VmAirs, VmProof}; use executor::elf::Elf; @@ -151,15 +155,8 @@ fn craft_proof( let result = executor.run().expect("run"); let max_rows = MaxRowsConfig::default(); - let mut traces = Traces::from_elf_and_logs( - &program, - &result.logs, - &max_rows, - private_inputs, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build"); + let mut traces = Traces::from_elf_and_logs(&program, &result.logs, &max_rows, private_inputs) + .expect("trace build"); match forge { Some(Tamper::RepointPrivateRow(f)) => apply_forge(&mut traces, &f), @@ -186,7 +183,8 @@ fn craft_proof( &options, &program, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let runtime_page_ranges = traces.runtime_page_ranges(); @@ -196,7 +194,7 @@ fn craft_proof( .filter(|c| c.is_private_input) .count(); - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -208,11 +206,10 @@ fn craft_proof( options.fri_final_poly_log_degree, ); - let proof = Prover::multi_prove( + let proof = crate::hash_pin::BlockProver::multi_prove( airs.air_trace_pairs(&mut traces), &mut transcript, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, )?; Ok(VmProof { @@ -642,15 +639,8 @@ fn craft_proof_with_duplicate_page( let result = executor.run().expect("run"); let max_rows = MaxRowsConfig::default(); - let mut traces = Traces::from_elf_and_logs( - &program, - &result.logs, - &max_rows, - &[], - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build"); + let mut traces = + Traces::from_elf_and_logs(&program, &result.logs, &max_rows, &[]).expect("trace build"); inject_duplicate_zero_page(&mut traces, dup_base, consume); @@ -702,10 +692,11 @@ fn craft_proof_with_duplicate_page( &options, &program, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -717,11 +708,10 @@ fn craft_proof_with_duplicate_page( options.fri_final_poly_log_degree, ); - let proof = Prover::multi_prove( + let proof = crate::hash_pin::BlockProver::multi_prove( airs.air_trace_pairs(&mut traces), &mut transcript, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) // The injected duplicate page writes only FINI, a main-trace column, and every // page's OFFSET/INIT stays honest — so the preprocessed check cannot fire and diff --git a/prover/src/tests/page_tests.rs b/prover/src/tests/page_tests.rs index 1a223644d..fa2d27c3d 100644 --- a/prover/src/tests/page_tests.rs +++ b/prover/src/tests/page_tests.rs @@ -258,11 +258,9 @@ fn page_commitments_empty_list_matches_none() { /// (`exclude_touched = true`) cases. Locks the PR's "exact same PAGE trace" /// claim directly, instead of relying only on full prove+verify integration. /// -/// Compares `main_table` (PAGE is a main-only table). `Table: PartialEq` holds -/// in both configs (derived without `disk-spill`, hand-impl'd with it), so no -/// feature gate is needed. The whole `TraceTable` still can't be compared because -/// its `aux_table: Table` param needs `E: PartialEq`, which -/// `Degree3GoldilocksExtensionField` lacks. +/// Compares `main_table` (PAGE is a main-only table). The whole `TraceTable` +/// can't be compared because its `aux_table: Table` param needs +/// `E: PartialEq`, which `Degree3GoldilocksExtensionField` lacks. #[test] fn generate_page_trace_dense_matches_sparse() { use crate::paged_mem::PagedMem; diff --git a/prover/src/tests/pc_double_read_poc.rs b/prover/src/tests/pc_double_read_poc.rs index 47844a389..d366cef14 100644 --- a/prover/src/tests/pc_double_read_poc.rs +++ b/prover/src/tests/pc_double_read_poc.rs @@ -21,13 +21,13 @@ //! //! Production proof options, as in `halt_pc_forgery_poc`. -use crypto::fiat_shamir::default_transcript::DefaultTranscript; use executor::elf::Elf; use executor::vm::logs::Log; use stark::config::Commitment; +use stark::leaf_layout::LeafLayout; use stark::proof::view::MultiProofView; -use stark::prover::{IsStarkProver, Prover}; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::prover::IsStarkProver; +use stark::verifier::IsStarkVerifier; use super::halt_pc_forgery_poc::{opts, run_logs}; use crate::statement::{StatementKind, absorb_statement}; @@ -35,7 +35,7 @@ use crate::tables::cpu::{self, CpuOperation, cols as cpu_cols}; use crate::tables::trace_builder::Traces; use crate::tables::types::VmTable; use crate::tables::{decode, register}; -use crate::test_utils::{E, asm_elf_bytes, create_cpu_air, create_register_air}; +use crate::test_utils::{asm_elf_bytes, create_cpu_air, create_register_air}; use crate::{MaxRowsConfig, VmAirs, VmProof}; const GUEST: &str = "poc_pc_double_read"; @@ -60,18 +60,27 @@ fn patched_elf(honest: &[u8]) -> Vec { patched } +/// The CPU operation of `log` at timestamp `ts`, read against the program's +/// decode table (as the trace builder's own ops are). +fn op_at(table: &decode::DecodeTable, log: &Log, ts: u64) -> CpuOperation { + let row = table + .row(log.current_pc) + .expect("an instruction of the program"); + CpuOperation::from_log(log, ts, table.entry(row), row) +} + +/// The program's decode table, as the trace builder reads it. +fn decode_table(program: &Elf) -> decode::DecodeTable { + let instructions = decode::instructions_from_elf(program).expect("decode"); + decode::DecodeTable::from_instructions(&instructions) +} + /// CPU operations with the honest timestamps `4 * (i + 1)`, in program order. fn honest_ops(program: &Elf, logs: &[Log]) -> Vec { - let instructions = decode::instructions_from_elf(program).expect("decode"); + let table = decode_table(program); logs.iter() .enumerate() - .map(|(i, log)| { - CpuOperation::from_log_and_instruction( - log, - 4 * (i as u64 + 1), - instructions[&log.current_pc], - ) - }) + .map(|(i, log)| op_at(&table, log, 4 * (i as u64 + 1))) .collect() } @@ -86,14 +95,14 @@ fn forged_ops(program: &Elf, logs: &[Log]) -> Vec { logs[row].src1_val = 0; logs[row].dst_val = 0; } - let instructions = decode::instructions_from_elf(program).expect("decode"); + let table = decode_table(program); let mut ops: Vec = logs .iter() .enumerate() .map(|(i, log)| { let honest_ts = 4 * (i as u64 + 1); let ts = if i == ROW_A { honest_ts } else { honest_ts - 3 }; - CpuOperation::from_log_and_instruction(log, ts, instructions[&log.current_pc]) + op_at(&table, log, ts) }) .collect(); ops.swap(ROW_A, ROW_B); @@ -172,7 +181,7 @@ fn craft(elf_bytes: &[u8], ops: Vec, tamper: Tamper) -> Crafted { final_pc_ts: cpu::final_pc_timestamp(height as u64), }; let forged = Pins { - timestamp_root: cpu::commit_timestamp_column(&options, &column), + timestamp_root: cpu::commit_timestamp_column(&options, &column, LeafLayout::RowPair), final_pc_ts: last_ts + 1, }; let crafted = Crafted { @@ -189,8 +198,9 @@ fn craft(elf_bytes: &[u8], ops: Vec, tamper: Tamper) -> Crafted { &opts(), ®ister::register_init_from_entry_point(crafted.program.entry_point), &crafted.traces, - ), - crafted.register_commitment(), + ) + .commitment(), + crafted.register_commitment().commitment(), "the run must end in the halted register file" ); crafted @@ -212,7 +222,7 @@ impl Crafted { None, true, None, - self.register_commitment(), + self.register_commitment().preprocessed(), ); airs.cpus[0] = Box::new( create_cpu_air(&options, 0) @@ -220,21 +230,25 @@ impl Crafted { .with_preprocessed(pins.timestamp_root, cpu::NUM_PRECOMPUTED_COLS), ); airs.register = Box::new( - create_register_air(&options, pins.final_pc_ts) - .with_preprocessed(self.register_commitment(), register::NUM_PREPROCESSED_COLS), + create_register_air(&options, pins.final_pc_ts).with_preprocessed( + self.register_commitment().commitment(), + register::NUM_PREPROCESSED_COLS, + ), ); airs } /// As the production verifier: REGISTER FINI pinned to the halted register file. - fn register_commitment(&self) -> Commitment { - let committed_bytes = u32::try_from(self.traces.public_output_bytes.len()) - .expect("public output fits in u32"); - register::halted_commitment(&opts(), self.program.entry_point, committed_bytes) + fn register_commitment(&self) -> crate::RegisterFiles { + crate::test_utils::halted_register_commitment( + &opts(), + &self.program, + &self.traces.public_output_bytes, + ) } - fn transcript(&self) -> DefaultTranscript { - let mut transcript = DefaultTranscript::::new(&[]); + fn transcript(&self) -> crate::hash_pin::BlockTranscript { + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -253,11 +267,10 @@ impl Crafted { fn prove(mut self, pins: Pins) -> (Self, Result) { let airs = self.airs(pins); let mut transcript = self.transcript(); - let proof = Prover::multi_prove( + let proof = crate::hash_pin::BlockProver::multi_prove( airs.air_trace_pairs(&mut self.traces), &mut transcript, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .map(|proof| VmProof { proof, @@ -284,7 +297,12 @@ impl Crafted { ) else { return false; }; - Verifier::multi_verify_views(&refs, view, &mut self.transcript(), &expected) + crate::hash_pin::BlockVerifier::multi_verify_views( + &refs, + view, + &mut self.transcript(), + &expected, + ) } } diff --git a/prover/src/tests/prove_elfs_tests.rs b/prover/src/tests/prove_elfs_tests.rs index 62baad6e9..4f56dd2cb 100644 --- a/prover/src/tests/prove_elfs_tests.rs +++ b/prover/src/tests/prove_elfs_tests.rs @@ -13,14 +13,28 @@ //! //! TODO: LT bus (needs LT table integration) -use crypto::fiat_shamir::default_transcript::DefaultTranscript; +// ★ The Fiat-Shamir transcript MUST be the one `DefaultStarkHash` names, not +// `DefaultTranscript`'s own type-parameter default. +// +// `crypto::…::DefaultTranscript` defaults to keccak; +// since BLAKE3 became `DefaultStarkHash`, that default is no longer the +// production hash. Proving under the bare type while +// `compute_expected_commit_bus_balance_view` replays under +// `DefaultStarkTranscript` derives the two sides' LogUp challenges from +// DIFFERENT hashes, so the expected COMMIT contribution is computed at the wrong +// challenge point and every program with non-empty public output fails to verify +// with "LogUp bus does not balance" — 19 tests in this file. `config.rs` warns +// about exactly this half-flip ("the type system cannot force this; naming the +// alias is what makes the production path follow DefaultStarkHash"); the warning +// applies to the test harness too. + use math::field::element::FieldElement; use stark::constraints::builder::EmptyConstraints; use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData}; use stark::proof::options::ProofOptions; use stark::proof::view::{MultiProofView, StarkProofView}; use stark::traits::AIR; -use stark::verifier::{IsStarkVerifier, Verifier}; +use stark::verifier::IsStarkVerifier; use crate::VmProof; use crate::tables::MaxRowsConfig; @@ -136,19 +150,19 @@ pub(crate) fn weigh_the_bus_with( None, true, None, - register_commitment, + register_commitment.preprocessed(), ); // Build air_trace_pairs for all tables let air_trace_pairs = airs.air_trace_pairs(traces); - let multi_proof = match multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])) - { - Ok(proof) => proof, - // Panic rather than return false: `false` is reserved for "the verifier - // rejected", so a negative test cannot pass because proving fell over. - Err(e) => panic!("prover failed, which is not a verifier rejection: {e:?}"), - }; + let multi_proof = + match multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])) { + Ok(proof) => proof, + // Panic rather than return false: `false` is reserved for "the verifier + // rejected", so a negative test cannot pass because proving fell over. + Err(e) => panic!("prover failed, which is not a verifier rejection: {e:?}"), + }; // Compute the verifier-side expected COMMIT bus balance from public output bytes let views: Vec> = multi_proof @@ -156,7 +170,7 @@ pub(crate) fn weigh_the_bus_with( .iter() .map(StarkProofView::Owned) .collect(); - let mut replay_transcript = DefaultTranscript::::new(&[]); + let mut replay_transcript = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = crate::compute_expected_commit_bus_balance_view( &airs.air_refs(), &views, @@ -168,10 +182,10 @@ pub(crate) fn weigh_the_bus_with( // Verify using centralized air_refs() which includes all tables let air_refs = airs.air_refs(); - let accepted = Verifier::multi_verify_views( + let accepted = crate::hash_pin::BlockVerifier::multi_verify_views( &air_refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ); @@ -189,10 +203,14 @@ pub(crate) fn weigh_the_bus_with( } let accepted_with_target_moved = recheck_with_moved_target - && Verifier::multi_verify_views( + && crate::hash_pin::BlockVerifier::multi_verify_views( &air_refs, &views, - &mut DefaultTranscript::::new(&[]), + // The SAME pinned transcript as the accepting arm above. A plain + // `DefaultTranscript` here would make the two arms differ by HASH as + // well as by target, so a rejection would no longer be evidence that + // moving the target is what the verifier caught. + &mut crate::hash_pin::block_transcript(&[]), &contribution_sum, ); @@ -232,12 +250,13 @@ fn prove_vm_minimal(elf_bytes: &[u8], private_inputs: &[u8], max_rows: &MaxRowsC &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let runtime_page_ranges = traces.runtime_page_ranges(); let proof = multi_prove_ram( airs.air_trace_pairs(&mut traces), - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), ) .expect("prove"); let num_private_input_pages = traces @@ -286,7 +305,8 @@ fn verify_vm_minimal(vm_proof: &VmProof, elf_bytes: &[u8]) -> bool { &proof_options, &elf, &vm_proof.public_output, - ), + ) + .preprocessed(), ); let air_refs = airs.air_refs(); let views: Vec> = vm_proof @@ -295,7 +315,7 @@ fn verify_vm_minimal(vm_proof: &VmProof, elf_bytes: &[u8]) -> bool { .iter() .map(StarkProofView::Owned) .collect(); - let mut replay_transcript = DefaultTranscript::::new(&[]); + let mut replay_transcript = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = crate::compute_expected_commit_bus_balance_view( &air_refs, &views, @@ -304,10 +324,10 @@ fn verify_vm_minimal(vm_proof: &VmProof, elf_bytes: &[u8]) -> bool { &mut replay_transcript, ) .expect("fingerprint collision in test"); - Verifier::multi_verify_views( + crate::hash_pin::BlockVerifier::multi_verify_views( &air_refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ) } @@ -358,15 +378,15 @@ fn test_cpu_only_no_bus() { _, )> = vec![(&cpu_air, &mut cpu_trace, &())]; - let multi_proof = multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])) + let multi_proof = multi_prove_ram(air_trace_pairs, &mut crate::hash_pin::block_transcript(&[])) .expect("Prover failed"); let airs: Vec<&dyn AIR> = vec![&cpu_air]; assert!( - Verifier::multi_verify( + crate::hash_pin::BlockVerifier::multi_verify( &airs, &multi_proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ), "CPU-only verification failed" @@ -1227,6 +1247,209 @@ fn test_prove_elfs_keccak_multi_call() { ); } +#[test] +fn test_prove_elfs_blake3() { + let _ = env_logger::builder().is_test(true).try_init(); + + let elf_bytes = crate::test_utils::asm_elf_bytes("test_blake3"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let executor = + executor::vm::execution::Executor::new(&elf, vec![]).expect("Failed to create executor"); + let result = executor.run().expect("Failed to run program"); + + // The guest seeds the 14 input dwords with k+1, compresses, copies out over + // m and compresses again. Cross-check the committed output against a direct + // replay of the executor's compression function. + use executor::vm::instruction::execution::blake3_compress_6round; + let words: [u32; 28] = core::array::from_fn(|i| { + let dw = (i / 2 + 1) as u64; + if i % 2 == 0 { + dw as u32 + } else { + (dw >> 32) as u32 + } + }); + let h: [u32; 8] = words[0..8].try_into().unwrap(); + let m: [u32; 16] = words[8..24].try_into().unwrap(); + let t = (words[24] as u64) | ((words[25] as u64) << 32); + let (block_len, flags) = (words[26], words[27]); + let out1 = blake3_compress_6round(&h, &m, t, block_len, flags); + let out2 = blake3_compress_6round(&h, &out1, t, block_len, flags); + let expected_bytes: Vec = out2.iter().flat_map(|w| w.to_le_bytes()).collect(); + + assert_eq!( + result.return_values.memory_values, expected_bytes, + "committed output must match two chained 6-round compressions" + ); + + // Must use from_elf_and_logs (stack RAM needs PAGE tables, like keccak). + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + assert_eq!( + traces.public_output_bytes, + result.return_values.memory_values + ); + + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "blake3 prove/verify failed" + ); +} + +/// ★ The chained-absorb mode, end to end through the REAL prover. +/// +/// The unit suite in `tables::blake3::absorb_tests` checks the mode's +/// constraints and its internal bus in isolation; this is the honest path that +/// exercises everything they cannot: the CPU's `Ecall` send meeting the group's +/// FIRST row, the MEMW ordering of a dozen accesses that all carry the ecall's +/// single timestamp, the BITWISE multiplicities the group's rows consume, and a +/// second group whose `cv_out` write lands on memory the first group wrote. +/// +/// The guest absorbs three blocks in one ecall and one more in a second, so the +/// trace holds a 4-row group and a 2-row group at different timestamps — the +/// shape that would break if the chain were keyed on anything but the timestamp. +#[test] +fn test_prove_elfs_blake3_absorb() { + let _ = env_logger::builder().is_test(true).try_init(); + + let elf_bytes = crate::test_utils::asm_elf_bytes("test_blake3_absorb"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let executor = + executor::vm::execution::Executor::new(&elf, vec![]).expect("Failed to create executor"); + let result = executor.run().expect("Failed to run program"); + + // Replay the guest's two absorbs against the ecall's pure form. The guest + // seeds cv_in with dwords 1..4 and the message with dwords 100..123, then + // absorbs blocks 0..3 under CHUNK_START and block 1 again under interior + // flags. + use executor::vm::instruction::execution::blake3_absorb_chain_6round; + let dword_words = |dwords: &[u64]| -> Vec { + dwords + .iter() + .flat_map(|d| [*d as u32, (*d >> 32) as u32]) + .collect() + }; + let cv_in: [u32; 8] = dword_words(&[1, 2, 3, 4]).try_into().unwrap(); + let msg: Vec = dword_words(&(100u64..124).collect::>()); + let block = |i: usize| -> [u32; 16] { msg[i * 16..(i + 1) * 16].try_into().unwrap() }; + + let after_first = blake3_absorb_chain_6round(&cv_in, &[block(0), block(1), block(2)], 1); + let after_second = blake3_absorb_chain_6round(&after_first, &[block(1)], 0); + let expected_bytes: Vec = after_second.iter().flat_map(|w| w.to_le_bytes()).collect(); + + assert_eq!( + result.return_values.memory_values, expected_bytes, + "committed chaining value must match two chained absorb ecalls" + ); + + // Stack RAM needs PAGE tables, like keccak and the single-compression mode. + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + assert_eq!( + traces.public_output_bytes, + result.return_values.memory_values + ); + + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "blake3 absorb prove/verify failed" + ); +} + +/// ★ F1 acceptance: a machine-level attempt at an UNALIGNED absorb must not +/// verify — and this records WHICH layer rejects it, because that turned out to +/// be load-bearing and undocumented. +/// +/// The executor refuses an unaligned x11 outright, so no program can reach this +/// state; the trace has to be forged. Rewriting `M_BASE`'s low halfword, and the +/// eight `MSG_PTR` low halfwords with it so the pointer arithmetic still holds, +/// is exactly the shape a prover would submit. +/// +/// TWO independent layers reject it now, and the order matters to anyone +/// changing either: +/// +/// 1. **The chip's alignment constraint** `FIRST·(M_BASE[0] − 8·Q) = 0`, added +/// for F1. This is what makes the rejection a statement about the ABI. +/// 2. **The MEMW argument**, which would have rejected this particular forgery +/// even before (1) existed — for an unrelated reason worth naming: the +/// chip's message reads would address `msg+1 …`, and the MEMW table holds no +/// access there, so the `Memw` bus has sends with no receiver. That is a +/// property of THIS forgery, not a general defence: a forger who also +/// rewrote the MEMW rows clears (2) and is caught only by (1). +/// +/// The test asserts the rejection and pins (2)'s premise — that MEMW really has +/// no row at the unaligned address — so the claim cannot rot silently. +#[test] +fn test_prove_elfs_blake3_absorb_unaligned_message_rejected() { + use crate::tables::blake3::MSG_DWORDS; + use crate::tables::blake3::cols as b3; + use crate::tables::memw::cols as memw_cols; + use crate::tables::memw_aligned::cols as memw_a_cols; + + let _ = env_logger::builder().is_test(true).try_init(); + + let elf_bytes = crate::test_utils::asm_elf_bytes("test_blake3_absorb"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let executor = + executor::vm::execution::Executor::new(&elf, vec![]).expect("Failed to create executor"); + let result = executor.run().expect("Failed to run program"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + + let one = FieldElement::::one(); + let zero = FieldElement::::zero(); + + // The first row of an absorb group, and its honest message base. + let first_row = (0..traces.blake3.num_rows()) + .find(|&r| *traces.blake3.main_table.get(r, b3::FIRST) == one) + .expect("the guest performs at least one absorb"); + let honest_base_lo = *traces.blake3.main_table.get(first_row, b3::M_BASE); + + // Premise of rejection path (2): the memory tables access the aligned base + // and nothing one byte past it. An 8-aligned dword read takes the MEMW_A + // fast path, so both tables have to be scanned, and both are chunked. + let memw_has_base = |addr: FieldElement| { + let in_memw = traces.memws.iter().any(|t| { + (0..t.num_rows()).any(|r| { + *t.main_table.get(r, memw_cols::IS_REGISTER) == zero + && *t.main_table.get(r, memw_cols::BASE_ADDRESS_0) == addr + }) + }); + let in_memw_a = traces.memw_aligneds.iter().any(|t| { + (0..t.num_rows()).any(|r| { + *t.main_table.get(r, memw_a_cols::IS_REGISTER) == zero + && *t.main_table.get(r, memw_a_cols::BASE_ADDRESS[0]) == addr + }) + }); + in_memw || in_memw_a + }; + assert!( + memw_has_base(honest_base_lo), + "the honest message base must be a real MEMW access" + ); + assert!( + !memw_has_base(honest_base_lo + one), + "nothing accesses the unaligned address — this is why MEMW also rejects" + ); + + // Forge: shift the message base and its dword pointers by one byte, keeping + // `msg_ptr[j] = M_BASE + 8j` internally consistent. + traces + .blake3 + .main_table + .set(first_row, b3::M_BASE, honest_base_lo + one); + for j in 0..MSG_DWORDS { + let c = b3::msg_ptr(j, 0); + let v = *traces.blake3.main_table.get(first_row, c); + traces.blake3.main_table.set(first_row, c, v + one); + } + + assert!( + !prove_and_verify_vm_minimal(&elf, &mut traces), + "an unaligned absorb message must not verify" + ); +} + #[test] fn test_prove_elfs_ecsm() { let _ = env_logger::builder().is_test(true).try_init(); @@ -1445,8 +1668,10 @@ fn test_prove_dma_memcpy_forged_value_rejected() { let (elf, mut traces) = dma_memcpy_fixture(); let forged_row = dma_row_matching(&traces, |_, end, _tail| !end); - let original = *traces.copy.main_table.get(forged_row, dma_cols::VALUE[0]); - traces.copy.main_table.set( + let original = *traces.copies[0] + .main_table + .get(forged_row, dma_cols::VALUE[0]); + traces.copies[0].main_table.set( forged_row, dma_cols::VALUE[0], original + FieldElement::::one(), @@ -1469,14 +1694,16 @@ fn test_prove_dma_memcpy_forged_intermediate_source_rejected() { // Shift both the current source and its locally-consistent successor. The // row's ADD remains valid, but the predecessor's CopyNext tuple and the // source-memory read no longer match. - let src_lo = *traces.copy.main_table.get(forged_row, dma_cols::SRC_0); - let src_incr_lo = *traces.copy.main_table.get(forged_row, dma_cols::SRC_INCR_0); - traces.copy.main_table.set( + let src_lo = *traces.copies[0].main_table.get(forged_row, dma_cols::SRC_0); + let src_incr_lo = *traces.copies[0] + .main_table + .get(forged_row, dma_cols::SRC_INCR_0); + traces.copies[0].main_table.set( forged_row, dma_cols::SRC_0, src_lo + FieldElement::from(8u64), ); - traces.copy.main_table.set( + traces.copies[0].main_table.set( forged_row, dma_cols::SRC_INCR_0, src_incr_lo + FieldElement::from(8u64), @@ -1495,8 +1722,7 @@ fn test_prove_dma_memcpy_forged_early_end_rejected() { let (elf, mut traces) = dma_memcpy_fixture(); let forged_row = dma_row_matching(&traces, |_first, end, _tail| !end); - traces - .copy + traces.copies[0] .main_table .set(forged_row, dma_cols::END, FieldElement::one()); @@ -1509,8 +1735,7 @@ fn test_prove_dma_memcpy_forged_wide_tail_rejected() { let (elf, mut traces) = dma_memcpy_fixture(); let forged_row = dma_row_matching(&traces, |_first, end, tail| !end && !tail); - traces - .copy + traces.copies[0] .main_table .set(forged_row, dma_cols::SINGLE, FieldElement::one()); @@ -1529,8 +1754,8 @@ fn test_prove_dma_memset_forged_intermediate_fill_rejected() { // Shift both lanes so the row stays internally consistent locally; only the chain // token and the MEMW write disagree. for column in [dma_set_cols::VALUE[0], dma_set_cols::VALUE[1]] { - let original = *traces.copy.main_table.get(forged_row, column); - traces.copy.main_table.set( + let original = *traces.copies[0].main_table.get(forged_row, column); + traces.copies[0].main_table.set( forged_row, column, original + FieldElement::::one(), @@ -1550,8 +1775,7 @@ fn test_prove_dma_memset_forged_early_end_rejected() { let (elf, mut traces) = dma_memset_fixture(); let forged_row = dma_set_row_matching(&traces, |_first, end, _tail| !end); - traces - .copy + traces.copies[0] .main_table .set(forged_row, dma_set_cols::END, FieldElement::one()); @@ -1568,8 +1792,7 @@ fn test_prove_dma_memset_forged_wide_tail_rejected() { let (elf, mut traces) = dma_memset_fixture(); let forged_row = dma_set_row_matching(&traces, |_first, end, tail| !end && !tail); - traces - .copy + traces.copies[0] .main_table .set(forged_row, dma_set_cols::SINGLE, FieldElement::one()); @@ -1587,8 +1810,7 @@ fn test_prove_dma_memset_forged_order_bit_rejected() { let (elf, mut traces) = dma_memset_fixture(); let forged_row = dma_set_row_matching(&traces, |_first, end, _tail| !end); - traces - .copy + traces.copies[0] .main_table .set(forged_row, mm_cols::IS_SET, FieldElement::zero()); @@ -1610,9 +1832,8 @@ fn test_prove_dma_memset_forged_intermediate_destination_rejected() { // The row's ADD stays valid; the predecessor's CopyNext tuple and the // memory write no longer match. for column in [dma_set_cols::DST_0, dma_set_cols::DST_INCR_0] { - let original = *traces.copy.main_table.get(forged_row, column); - traces - .copy + let original = *traces.copies[0].main_table.get(forged_row, column); + traces.copies[0] .main_table .set(forged_row, column, original + FieldElement::from(8u64)); } @@ -1646,9 +1867,9 @@ fn dma_set_row_matching(traces: &Traces, predicate: impl Fn(bool, bool, bool) -> use crate::tables::copy::cols as mm_cols; let one = FieldElement::::one(); - (0..traces.copy.num_rows()) + (0..traces.copies[0].num_rows()) .find(|&row| { - let get = |column| *traces.copy.main_table.get(row, column) == one; + let get = |column| *traces.copies[0].main_table.get(row, column) == one; get(mm_cols::MU) && get(mm_cols::IS_SET) && predicate(get(mm_cols::FIRST), get(mm_cols::END), get(mm_cols::SINGLE)) @@ -1678,9 +1899,9 @@ fn dma_row_matching(traces: &Traces, predicate: impl Fn(bool, bool, bool) -> boo use crate::tables::copy::cols as mm_cols; let one = FieldElement::::one(); - (0..traces.copy.num_rows()) + (0..traces.copies[0].num_rows()) .find(|&row| { - let get = |column| *traces.copy.main_table.get(row, column) == one; + let get = |column| *traces.copies[0].main_table.get(row, column) == one; // memcpy rows only: memset and `write` have their own fixtures above. let is_copy = !get(mm_cols::IS_SET) && !get(mm_cols::IS_WRITE); get(mm_cols::MU) @@ -1710,9 +1931,9 @@ fn commit_row(traces: &Traces) -> usize { use crate::tables::copy::cols as mm_cols; let one = FieldElement::::one(); - (0..traces.copy.num_rows()) + (0..traces.copies[0].num_rows()) .find(|&row| { - let get = |column| *traces.copy.main_table.get(row, column) == one; + let get = |column| *traces.copies[0].main_table.get(row, column) == one; get(mm_cols::MU) && get(mm_cols::IS_WRITE) && get(mm_cols::MU_COM) }) .expect("the guest must contain a real COPY write row") @@ -1730,8 +1951,10 @@ fn test_prove_commit_forged_value_rejected() { let (elf, mut traces) = commit_fixture(); let forged_row = commit_row(&traces); - let original = *traces.copy.main_table.get(forged_row, mm_cols::VALUE[0]); - traces.copy.main_table.set( + let original = *traces.copies[0] + .main_table + .get(forged_row, mm_cols::VALUE[0]); + traces.copies[0].main_table.set( forged_row, mm_cols::VALUE[0], original + FieldElement::::one(), @@ -1756,9 +1979,8 @@ fn test_prove_commit_forged_index_rejected() { let (elf, mut traces) = commit_fixture(); let forged_row = commit_row(&traces); for column in [mm_cols::DST_0, mm_cols::DST_INCR_0] { - let original = *traces.copy.main_table.get(forged_row, column); - traces - .copy + let original = *traces.copies[0].main_table.get(forged_row, column); + traces.copies[0] .main_table .set(forged_row, column, original + FieldElement::from(8u64)); } @@ -1786,14 +2008,13 @@ fn test_prove_commit_forged_index_high_word_rejected() { let (elf, mut traces) = commit_fixture(); let forged_row = commit_row(&traces); assert_eq!( - *traces - .copy + *traces.copies[0] .main_table .get(forged_row, mm_cols::DST_RES_0 + 2), FieldElement::::zero(), "the honest commit index must fit the low word" ); - traces.copy.main_table.set( + traces.copies[0].main_table.set( forged_row, mm_cols::DST_RES_0 + 2, FieldElement::::one(), @@ -1816,12 +2037,13 @@ fn test_prove_commit_forged_first_write_rejected() { let (elf, mut traces) = commit_fixture(); let forged_row = commit_row(&traces); assert_eq!( - *traces.copy.main_table.get(forged_row, mm_cols::FIRST_WRITE), + *traces.copies[0] + .main_table + .get(forged_row, mm_cols::FIRST_WRITE), FieldElement::::one(), "the fixture's first write row must carry first_write" ); - traces - .copy + traces.copies[0] .main_table .set(forged_row, mm_cols::FIRST_WRITE, FieldElement::zero()); @@ -1841,8 +2063,10 @@ fn test_prove_commit_forged_dst_res_rejected() { let (elf, mut traces) = commit_fixture(); let forged_row = commit_row(&traces); - let original = *traces.copy.main_table.get(forged_row, mm_cols::DST_RES_0); - traces.copy.main_table.set( + let original = *traces.copies[0] + .main_table + .get(forged_row, mm_cols::DST_RES_0); + traces.copies[0].main_table.set( forged_row, mm_cols::DST_RES_0, original + FieldElement::from(8u64), @@ -1866,8 +2090,7 @@ fn test_prove_commit_forged_mu_com_rejected() { let (elf, mut traces) = commit_fixture(); let forged_row = commit_row(&traces); for column in [mm_cols::MU_COM, mm_cols::MU_COM_WIDE] { - traces - .copy + traces.copies[0] .main_table .set(forged_row, column, FieldElement::zero()); } @@ -2206,6 +2429,129 @@ fn test_hint_range_checks_its_output_bytes() { ); } +/// The ECREC program's three calls: two signatures recover their keys (status 0), +/// one with `r = 0` bails (a zero key, status 1); the committed outputs are those, +/// and the run proves and verifies. +#[test] +fn test_prove_elfs_ecrec() { + let _ = env_logger::builder().is_test(true).try_init(); + let elf_bytes = crate::test_utils::asm_elf_bytes("test_ecrec"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let result = executor::vm::execution::Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run"); + let expected = "8733750fdf82edb525a2c9f9d201ad1b26e163b8093f3c8f6eb83377d157edb2ce023840e38085bfcf2b27bd66be6a877f0ec281b073ca0f9ab31cfc0733b62a000000000000000076d2fdf1302d1fa9556f4df94ec84cefba6d482e54f47c6c2a238c1baa560f0eb754ac7e7a3e09c44184cb451a4f5fb557f32053eb015dffebb655b5cfd54d8a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000"; + let got: String = result + .return_values + .memory_values + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + assert_eq!(got, expected, "committed keys and statuses"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + assert_eq!(traces.ecrecs.len(), 1); + assert!(traces.ecsms.is_empty(), "no ECSM call"); + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "ECREC prove/verify failed" + ); +} + +/// Soundness: one byte of a computed key forged on the ECREC row must not verify (the +/// key is bound by the final addition's ECDAS tuple, the MEMW writes and `< p`). +#[test] +fn test_prove_elfs_ecrec_forged_key_rejected() { + use crate::tables::ecrec::cols as ecrec_cols; + let _ = env_logger::builder().is_test(true).try_init(); + let elf_bytes = crate::test_utils::asm_elf_bytes("test_ecrec"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let result = executor::vm::execution::Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + let orig = *traces.ecrecs[0].main_table.get(0, ecrec_cols::XQ + 3); + traces.ecrecs[0].main_table.set( + 0, + ecrec_cols::XQ + 3, + orig + FieldElement::::one(), + ); + assert!( + !prove_and_verify_vm_minimal(&elf, &mut traces), + "Verifier must reject a forged ECREC key" + ); +} + +/// Soundness: a computed row cannot claim to have bailed. Flipping `OK` to 0 on the +/// first call's row leaves its key bytes nonzero (a bail writes zeros) and its chains' +/// messages without their ECREC counterparts. +#[test] +fn test_prove_elfs_ecrec_forged_bail_rejected() { + use crate::tables::ecrec::cols as ecrec_cols; + let _ = env_logger::builder().is_test(true).try_init(); + let elf_bytes = crate::test_utils::asm_elf_bytes("test_ecrec"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let result = executor::vm::execution::Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + traces.ecrecs[0] + .main_table + .set(0, ecrec_cols::OK, FieldElement::::zero()); + assert!( + !prove_and_verify_vm_minimal(&elf, &mut traces), + "Verifier must reject a computed ECREC row relabelled as a bail" + ); +} + +/// Soundness: one call cannot borrow another's result. The final-addition ECDAS +/// rows of the two computed calls (keyed at `T1 + 2` and `T2 + 2`) swap only their +/// timestamps, so each call's key is the other's sum; every constraint holds and +/// only the bus refuses it. +#[test] +fn test_prove_elfs_ecrec_final_additions_swapped_between_calls_rejected() { + use crate::tables::ecdas::cols as ec; + use crate::tables::ecrec::cols as ecrec_cols; + let _ = env_logger::builder().is_test(true).try_init(); + let elf_bytes = crate::test_utils::asm_elf_bytes("test_ecrec"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let result = executor::vm::execution::Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + let er = &traces.ecrecs[0].main_table; + let (t1, t2) = ( + *er.get(0, ecrec_cols::TIMESTAMP_0), + *er.get(1, ecrec_cols::TIMESTAMP_0), + ); + let two = FieldElement::::from(2u64); + let one = FieldElement::::one(); + let ed = &traces.ecdases[0].main_table; + let find = |key: FieldElement| { + let rows: Vec = (0..ed.height) + .filter(|&r| *ed.get(r, ec::TIMESTAMP_0) == key && *ed.get(r, ec::MU) == one) + .collect(); + assert_eq!(rows.len(), 1, "one final-addition row at T + 2"); + rows[0] + }; + let (a, b) = (find(t1 + two), find(t2 + two)); + let (va, vb) = (*ed.get(a, ec::TIMESTAMP_0), *ed.get(b, ec::TIMESTAMP_0)); + traces.ecdases[0].main_table.set(a, ec::TIMESTAMP_0, vb); + traces.ecdases[0].main_table.set(b, ec::TIMESTAMP_0, va); + let outcome = weigh_the_bus(&elf, &mut traces, true); + crate::tests::skip_empty_tables_tests::assert_only_the_bus_rejected( + &outcome, + "ECREC final additions swapped between calls", + ); +} + /// Soundness: the verifier REJECTS a forged ECSM result. /// /// A malicious prover must not be able to claim a wrong `k·G`. We tamper the result @@ -2367,11 +2713,12 @@ fn test_prove_elfs_test_commit_4_wrong_pages_rejected() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let proof = multi_prove_ram( prover_airs.air_trace_pairs(&mut traces), - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), ) .expect("Prover failed"); @@ -2392,12 +2739,13 @@ fn test_prove_elfs_test_commit_4_wrong_pages_rejected() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let verifier_air_refs = verifier_airs.air_refs(); let views: Vec> = proof.proofs.iter().map(StarkProofView::Owned).collect(); - let mut replay_transcript = DefaultTranscript::::new(&[]); + let mut replay_transcript = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = crate::compute_expected_commit_bus_balance_view( &verifier_air_refs, &views, @@ -2407,10 +2755,10 @@ fn test_prove_elfs_test_commit_4_wrong_pages_rejected() { ) .expect("fingerprint collision in test"); - let verified = Verifier::multi_verify_views( + let verified = crate::hash_pin::BlockVerifier::multi_verify_views( &verifier_air_refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ); assert!( @@ -2748,15 +3096,7 @@ fn test_debug_memory_tokens_sb_sh() { use std::collections::HashMap; let (elf, logs, _instructions) = run_asm_elf("test_sb_sh_8"); - let traces = Traces::from_elf_and_logs( - &elf, - &logs, - &Default::default(), - &[], - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .unwrap(); + let traces = Traces::from_elf_and_logs(&elf, &logs, &Default::default(), &[]).unwrap(); println!("DEBUG: test_sb_sh_8 Memory bus tokens (FULL)"); println!( @@ -3132,11 +3472,12 @@ fn test_deep_stack_runtime_pages_roundtrip() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let proof = multi_prove_ram( prover_airs.air_trace_pairs(&mut traces), - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), ) .expect("Prover failed"); // Verifier reconstructs from ELF + runtime_page_ranges hint @@ -3157,12 +3498,13 @@ fn test_deep_stack_runtime_pages_roundtrip() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let verifier_air_refs = verifier_airs.air_refs(); let views: Vec> = proof.proofs.iter().map(StarkProofView::Owned).collect(); - let mut replay_transcript = DefaultTranscript::::new(&[]); + let mut replay_transcript = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = crate::compute_expected_commit_bus_balance_view( &verifier_air_refs, &views, @@ -3172,10 +3514,10 @@ fn test_deep_stack_runtime_pages_roundtrip() { ) .expect("fingerprint collision in test"); - let verified = Verifier::multi_verify_views( + let verified = crate::hash_pin::BlockVerifier::multi_verify_views( &verifier_air_refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ); assert!( @@ -3217,11 +3559,12 @@ fn test_deep_stack_missing_pages_rejected() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let proof = multi_prove_ram( prover_airs.air_trace_pairs(&mut traces), - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), ) .expect("Prover failed"); // Verifier uses EMPTY runtime_page_ranges → missing stack/heap pages @@ -3241,12 +3584,13 @@ fn test_deep_stack_missing_pages_rejected() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let verifier_air_refs = verifier_airs.air_refs(); let views: Vec> = proof.proofs.iter().map(StarkProofView::Owned).collect(); - let mut replay_transcript = DefaultTranscript::::new(&[]); + let mut replay_transcript = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = crate::compute_expected_commit_bus_balance_view( &verifier_air_refs, &views, @@ -3256,10 +3600,10 @@ fn test_deep_stack_missing_pages_rejected() { ) .expect("fingerprint collision in test"); - let verified = Verifier::multi_verify_views( + let verified = crate::hash_pin::BlockVerifier::multi_verify_views( &verifier_air_refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ); assert!( @@ -3336,11 +3680,12 @@ fn test_heap_alloc_runtime_pages_roundtrip() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let proof = multi_prove_ram( prover_airs.air_trace_pairs(&mut traces), - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), ) .expect("Prover failed"); // Verifier reconstructs from ELF + runtime hint (ranges decoded to pages) @@ -3361,12 +3706,13 @@ fn test_heap_alloc_runtime_pages_roundtrip() { &proof_options, &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let verifier_air_refs = verifier_airs.air_refs(); let views: Vec> = proof.proofs.iter().map(StarkProofView::Owned).collect(); - let mut replay_transcript = DefaultTranscript::::new(&[]); + let mut replay_transcript = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = crate::compute_expected_commit_bus_balance_view( &verifier_air_refs, &views, @@ -3376,10 +3722,10 @@ fn test_heap_alloc_runtime_pages_roundtrip() { ) .expect("fingerprint collision in test"); - let verified = Verifier::multi_verify_views( + let verified = crate::hash_pin::BlockVerifier::multi_verify_views( &verifier_air_refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ); assert!( @@ -3451,7 +3797,10 @@ fn test_verify_rejects_zero_table_counts() { ecsm: 0, ecdas: 0, hint: 0, + copy: 0, + blake3: 0, is_wh: 0, + ecrec: 0, }, ..vm_proof }; @@ -3537,12 +3886,17 @@ fn test_crafted_zero_count_proof_must_not_verify() { bytewise: 0, store: 0, cpu32: 0, + // 0 is legal for all eight of these (each table is conditional), so this + // fixture stays a test of the REQUIRED tables' zero-rejection. keccak: 0, keccak_rnd: 0, ecsm: 0, ecdas: 0, hint: 0, + copy: 0, + blake3: 0, is_wh: 0, + ecrec: 0, }; let airs = VmAirs::new( &elf, @@ -3554,7 +3908,7 @@ fn test_crafted_zero_count_proof_must_not_verify() { None, true, None, - crate::test_utils::halted_register_commitment(&proof_options, &elf, &[]), + crate::test_utils::halted_register_commitment(&proof_options, &elf, &[]).preprocessed(), ); let verifier_air_refs = airs.air_refs(); @@ -3575,15 +3929,15 @@ fn test_crafted_zero_count_proof_must_not_verify() { (airs.decode.as_ref(), &mut decode_trace, &()), ]; - let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + let proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) .expect("Proof generation should succeed"); assert_eq!(proof.proofs.len(), 2); - let verified = Verifier::multi_verify( + let verified = crate::hash_pin::BlockVerifier::multi_verify( &verifier_air_refs, &proof, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &FieldElement::zero(), ); @@ -4026,8 +4380,6 @@ fn test_prove_first_epoch_without_halt() { &[], false, false, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ) .unwrap(); @@ -4043,12 +4395,13 @@ fn test_prove_first_epoch_without_halt() { None, false, None, - crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces), + crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces) + .preprocessed(), ); let multi_proof = multi_prove_ram( airs.air_trace_pairs(&mut traces), - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), ) .expect("first epoch failed to prove"); @@ -4057,7 +4410,7 @@ fn test_prove_first_epoch_without_halt() { .iter() .map(StarkProofView::Owned) .collect(); - let mut replay = DefaultTranscript::::new(&[]); + let mut replay = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = compute_expected_commit_bus_balance_view( &airs.air_refs(), &views, @@ -4068,10 +4421,10 @@ fn test_prove_first_epoch_without_halt() { .expect("fingerprint collision in test"); assert!( - Verifier::multi_verify_views( + crate::hash_pin::BlockVerifier::multi_verify_views( &airs.air_refs(), &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ), "first epoch (HALT excluded) failed to verify" @@ -4114,8 +4467,6 @@ fn test_prove_second_epoch_from_snapshot() { &[], false, false, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ) .unwrap(); @@ -4132,12 +4483,13 @@ fn test_prove_second_epoch_from_snapshot() { None, false, None, - crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces), + crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces) + .preprocessed(), ); let multi_proof = multi_prove_ram( airs.air_trace_pairs(&mut traces), - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), ) .expect("second epoch failed to prove"); @@ -4146,7 +4498,7 @@ fn test_prove_second_epoch_from_snapshot() { .iter() .map(StarkProofView::Owned) .collect(); - let mut replay = DefaultTranscript::::new(&[]); + let mut replay = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = compute_expected_commit_bus_balance_view( &airs.air_refs(), &views, @@ -4157,10 +4509,10 @@ fn test_prove_second_epoch_from_snapshot() { .expect("fingerprint collision in test"); assert!( - Verifier::multi_verify_views( + crate::hash_pin::BlockVerifier::multi_verify_views( &airs.air_refs(), &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ), "second epoch (register init from snapshot) failed to verify" @@ -4204,8 +4556,6 @@ fn test_epoch_proof_commits_l2g() { &[], false, false, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ) .unwrap(); @@ -4228,7 +4578,8 @@ fn test_epoch_proof_commits_l2g() { None, false, None, - crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces), + crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces) + .preprocessed(), ); // Inert L2G AIR: commits the trace columns, but no bus and no constraints. @@ -4251,7 +4602,7 @@ fn test_epoch_proof_commits_l2g() { let mut pairs = airs.air_trace_pairs(&mut traces); pairs.push((&inert_l2g_air, &mut l2g_trace, &())); - let multi_proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + let multi_proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) .expect("epoch proof with inert L2G failed to prove"); let mut refs = airs.air_refs(); @@ -4262,7 +4613,7 @@ fn test_epoch_proof_commits_l2g() { .iter() .map(StarkProofView::Owned) .collect(); - let mut replay = DefaultTranscript::::new(&[]); + let mut replay = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = compute_expected_commit_bus_balance_view( &refs, &views, @@ -4273,10 +4624,10 @@ fn test_epoch_proof_commits_l2g() { .expect("fingerprint collision in test"); assert!( - Verifier::multi_verify_views( + crate::hash_pin::BlockVerifier::multi_verify_views( &refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ), "epoch proof with inert L2G failed to verify" @@ -4370,8 +4721,6 @@ fn test_continuation_pipeline_end_to_end() { &[], is_final, false, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ) .unwrap(); @@ -4390,7 +4739,8 @@ fn test_continuation_pipeline_end_to_end() { &proof_options, ®ister_inits[i], &traces, - ), + ) + .preprocessed(), ); let mut l2g_trace = local_to_global::generate_local_to_global_trace(&boundaries[i]); @@ -4412,7 +4762,7 @@ fn test_continuation_pipeline_end_to_end() { let mut pairs = airs.air_trace_pairs(&mut traces); pairs.push((&inert_l2g_air, &mut l2g_trace, &())); - let multi_proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + let multi_proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) .expect("epoch proof failed to prove"); let mut refs = airs.air_refs(); @@ -4422,7 +4772,7 @@ fn test_continuation_pipeline_end_to_end() { .iter() .map(StarkProofView::Owned) .collect(); - let mut replay = DefaultTranscript::::new(&[]); + let mut replay = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = compute_expected_commit_bus_balance_view( &refs, &views, @@ -4432,10 +4782,10 @@ fn test_continuation_pipeline_end_to_end() { ) .expect("fingerprint collision in test"); assert!( - Verifier::multi_verify_views( + crate::hash_pin::BlockVerifier::multi_verify_views( &refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ), "epoch {i} failed to verify" @@ -4506,8 +4856,6 @@ fn test_epoch_memory_bus_with_l2g_bookend() { &[], false, true, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ) .unwrap(); let initial_memory: HashMap = image.iter().map(|(&a, &v)| (a, v as u64)).collect(); @@ -4527,7 +4875,8 @@ fn test_epoch_memory_bus_with_l2g_bookend() { None, false, None, - crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces), + crate::test_utils::trace_register_commitment(&proof_options, ®ister_init, &traces) + .preprocessed(), ); // L2G air on the epoch-local Memory bus (the bookend that replaces PAGE). @@ -4555,7 +4904,7 @@ fn test_epoch_memory_bus_with_l2g_bookend() { let mut pairs = airs.air_trace_pairs(&mut traces); pairs.push((&l2g_air, &mut l2g_trace, &())); - let multi_proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + let multi_proof = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])) .expect("epoch with L2G memory bookend failed to prove"); let mut refs = airs.air_refs(); @@ -4565,7 +4914,7 @@ fn test_epoch_memory_bus_with_l2g_bookend() { .iter() .map(StarkProofView::Owned) .collect(); - let mut replay = DefaultTranscript::::new(&[]); + let mut replay = crate::hash_pin::block_transcript(&[]); let expected_bus_balance = compute_expected_commit_bus_balance_view( &refs, &views, @@ -4576,12 +4925,606 @@ fn test_epoch_memory_bus_with_l2g_bookend() { .expect("fingerprint collision in test"); assert!( - Verifier::multi_verify_views( + crate::hash_pin::BlockVerifier::multi_verify_views( &refs, &views, - &mut DefaultTranscript::::new(&[]), + &mut crate::hash_pin::block_transcript(&[]), &expected_bus_balance, ), "epoch Memory bus must balance with L2G bookend + PAGE excluding touched cells" ); } + +// ============================================================================= +// ★ R1 — the conditional BLAKE3 table +// ============================================================================= +// +// BLAKE3 left `FIXED_TABLE_COUNT` and became `TableCounts::blake3`, a 0-or-1 +// count. That is a soundness claim, not a size optimisation: it asserts that +// omitting the table is safe BECAUSE a workload that used BLAKE3 leaves an +// Ecall-bus send with no receiver, so the omission cannot verify. These four +// tests are that claim — the two honest directions, the forgery, and the +// binding that stops prover and verifier disagreeing quietly. +// +// Why it matters at all: always-on, the table cost a real block's recursion +// wrap +31.2% of its trace cells and +44% of its peak memory while carrying +// 0.035% of the epoch's own cells (WRAP-GROWTH-BISECT.md). + +/// Build traces for an asm ELF, the same minimal path the suite's other +/// prove tests use. +fn traces_for_asm(name: &str) -> (Vec, Elf, Traces) { + let elf_bytes = crate::test_utils::asm_elf_bytes(name); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let executor = Executor::new(&elf, vec![]).expect("executor"); + let result = executor.run().expect("execution"); + let traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + (elf_bytes, elf, traces) +} + +/// (a) A workload that never touches BLAKE3 proves and verifies WITHOUT the +/// table — the recovery R1 exists for. +/// +/// The count is what the sub-proof arithmetic is checked against, so asserting +/// it here is asserting the table is genuinely absent rather than merely small. +#[test] +fn a_workload_without_blake3_proves_without_the_table() { + let (_, elf, mut traces) = traces_for_asm("sub"); + + assert_eq!( + traces.num_blake3_ops, 0, + "the `sub` program must not use BLAKE3, or this tests nothing" + ); + let counts = traces.table_counts(); + assert_eq!(counts.blake3, 0, "an unused BLAKE3 table is not carried"); + + let airs = VmAirs::new( + &elf, + &ProofOptions::default_test_options(), + true, + &traces.page_configs, + &counts, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + crate::test_utils::halted_register_commitment( + &ProofOptions::default_test_options(), + &elf, + &traces.public_output_bytes, + ) + .preprocessed(), + ); + assert!(!airs.include_blake3); + assert!( + !airs.air_refs().iter().any(|a| a.name() == "BLAKE3"), + "the AIR set must not contain a table the proof does not carry" + ); + + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "a keccak-free, blake3-free program must prove and verify with no BLAKE3 table" + ); +} + +/// (b) A workload that DOES use BLAKE3 proves and verifies WITH the table. +/// +/// The control for (a): the conditional must not have made the table +/// unreachable, which an "always omit" bug would pass (a) and fail here. +#[test] +fn a_workload_using_blake3_proves_with_the_table() { + let (_, elf, mut traces) = traces_for_asm("test_blake3"); + + assert!( + traces.num_blake3_ops > 0, + "the `test_blake3` program must use BLAKE3, or this tests nothing" + ); + let counts = traces.table_counts(); + assert_eq!(counts.blake3, 1, "a used BLAKE3 table is carried"); + + let airs = VmAirs::new( + &elf, + &ProofOptions::default_test_options(), + true, + &traces.page_configs, + &counts, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + crate::test_utils::halted_register_commitment( + &ProofOptions::default_test_options(), + &elf, + &traces.public_output_bytes, + ) + .preprocessed(), + ); + assert!(airs.include_blake3); + assert!( + airs.air_refs().iter().any(|a| a.name() == "BLAKE3"), + "the AIR set must contain the table the proof carries" + ); + + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "a blake3-using program must prove and verify with the BLAKE3 table" + ); +} + +/// ★ (c) THE FORGERY — stripped, and honest about how far the stripping got. +/// +/// The naive version of this test is INERT. A BLAKE3 syscall touches several +/// buses, and omitting the table unbalances all of them at once: Memory/MEMW +/// (its state reads and writes), AreBytes, IsHalfword and ByteAlu (its byte +/// checks and XORs), and only then Ecall. Those are numerically overwhelming, +/// so the naive proof is rejected long before the Ecall argument matters — it +/// would be rejected identically if the Ecall receiver had never been written. +/// +/// `STRIP_BLAKE3_SIDE_EFFECTS` builds the trace with BLAKE3's MEMW ops and its +/// BITWISE multiplicities left out, to narrow the rejection toward Ecall alone. +/// +/// ## ⚠ MEASURED: it is narrowed, NOT yet sole +/// +/// Under `--features debug-checks` this forgery leaves THREE buses imbalanced +/// beyond the by-design `Commit` residual: +/// +/// - **19 `Ecall`** — the one this argument is about. +/// - **16 `Memory`** — removing BLAKE3's MEMW ops breaks the memory chain for +/// the state region; `test_blake3` does not satisfy the "never re-reads the +/// digest region" precondition that would make the removal chain-consistent. +/// - **24 `ByteAlu`** — subtracting `collect_bitwise_from_blake3` did not zero +/// BLAKE3's whole ByteAlu contribution. +/// +/// `AreBytes` and `IsHalfword` DO balance, so the strip is doing real work: it +/// takes the imbalance from five buses to three. But this test does not yet +/// establish Ecall as the sole cause, and must not be cited as if it did. +/// Finishing it needs a workload whose digest region is never re-read, plus a +/// subtract path for the remaining ByteAlu multiplicities (there is none on +/// `bitwise::update_multiplicities` today). See RESUME-WRAPSLIM.md, F1. +/// +/// What it DOES establish, and what the inert version did not: the omission is +/// rejected with the two largest confounders removed, prover and verifier +/// agreeing on the false shape throughout — both build the AIR set from +/// `blake3: 0`, so the counts are self-consistent and the sub-proof cross-check +/// cannot separate them. +#[test] +fn a_blake3_workload_claiming_no_blake3_table_is_rejected() { + use crate::tables::trace_builder::STRIP_BLAKE3_SIDE_EFFECTS; + + let elf_bytes = crate::test_utils::asm_elf_bytes("test_blake3"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let executor = Executor::new(&elf, vec![]).expect("executor"); + let result = executor.run().expect("execution"); + + // ---- the control: the SAME workload, unstripped, is honest and verifies. + // Without this, a stripped trace that fails for some unrelated reason would + // read as the argument working. + { + let mut honest = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]) + .unwrap(); + assert_eq!(honest.table_counts().blake3, 1); + assert!( + prove_and_verify_vm_minimal(&elf, &mut honest), + "the control must verify, or the stripped arm below proves nothing" + ); + } + + // ---- the forgery: BLAKE3's side contributions stripped, table claimed absent. + STRIP_BLAKE3_SIDE_EFFECTS.with(|c| c.set(true)); + let stripped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]) + })); + STRIP_BLAKE3_SIDE_EFFECTS.with(|c| c.set(false)); + let mut traces = stripped + .expect("stripping must not panic the trace builder") + .expect("stripped traces build"); + + assert!( + traces.num_blake3_ops > 0, + "the workload really used BLAKE3 — only its SIDE effects were stripped, \ + so the CPU ecall and its Ecall send are still present" + ); + + let mut forged = traces.table_counts(); + assert_eq!(forged.blake3, 1, "honest count before the tamper"); + forged.blake3 = 0; + assert!( + forged.validate().is_ok(), + "a zero BLAKE3 count is well-formed; only the bus may reject it" + ); + + let proof_options = ProofOptions::default_test_options(); + let airs = VmAirs::new( + &elf, + &proof_options, + true, + &traces.page_configs, + &forged, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + crate::test_utils::halted_register_commitment( + &proof_options, + &elf, + &traces.public_output_bytes, + ) + .preprocessed(), + ); + assert!(!airs.include_blake3, "the forged shape must omit the table"); + + let pairs = airs.air_trace_pairs(&mut traces); + let proved = multi_prove_ram(pairs, &mut crate::hash_pin::block_transcript(&[])); + + let verified = match &proved { + Err(_) => false, + Ok(multi_proof) => { + let views: Vec> = multi_proof + .proofs + .iter() + .map(StarkProofView::Owned) + .collect(); + let mut replay = crate::hash_pin::block_transcript(&[]); + match crate::compute_expected_commit_bus_balance_view( + &airs.air_refs(), + &views, + &traces.public_output_bytes, + 0, + &mut replay, + ) { + None => false, + Some(expected) => crate::hash_pin::BlockVerifier::multi_verify_views( + &airs.air_refs(), + &views, + &mut crate::hash_pin::block_transcript(&[]), + &expected, + ), + } + } + }; + + assert!( + !verified, + "a proof that omits the BLAKE3 table for a workload that used BLAKE3 must \ + NOT verify. Ecall is among the buses left unmatched (see the doc above \ + for the two that are not yet stripped). If this fires, the soundness \ + argument on `TableCounts::blake3` is in question and the table must go \ + back to always-on until it is re-established." + ); + println!( + "stripped forgery rejected at: {}", + if proved.is_err() { + "prove (no consistent auxiliary trace)" + } else { + "verify (bus balance)" + } + ); +} + +/// (d) The count is bound into the transcript, so prover and verifier cannot +/// disagree about it silently. +/// +/// `include_blake3` is prover-asserted — unlike `include_halt`, the verifier +/// cannot derive it a priori. Binding is what turns a disagreement into a +/// rejection instead of two parties verifying different statements. +#[test] +fn the_blake3_count_is_bound_into_the_statement() { + use crate::statement::{StatementKind, absorb_statement}; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let elf_bytes = crate::test_utils::asm_elf_bytes("sub"); + let mut counts = crate::TableCounts { + cpu: 3, + lt: 1, + memw: 2, + memw_aligned: 1, + load: 1, + mul: 1, + dvrm: 1, + shift: 1, + branch: 2, + memw_register: 1, + eq: 1, + bytewise: 1, + store: 1, + cpu32: 1, + keccak: 1, + keccak_rnd: 1, + ecsm: 1, + ecdas: 1, + hint: 1, + copy: 1, + blake3: 1, + is_wh: 1, + ecrec: 0, + }; + + let challenge_for = |counts: &crate::TableCounts| { + let mut t = crate::hash_pin::block_transcript(&[]); + absorb_statement( + &mut t, + StatementKind::Monolithic, + &elf_bytes, + &[1, 2, 3], + counts, + 0, + &[], + 7, + ); + t.sample_field_element() + }; + + counts.blake3 = 1; + let with = challenge_for(&counts); + counts.blake3 = 0; + let without = challenge_for(&counts); + + assert_ne!( + with, without, + "the BLAKE3 count must change the transcript — unbound, a prover could \ + claim one shape and a verifier build another from the same bytes" + ); +} + +/// (e) A BLAKE3 count above one is rejected before any AIR is built. +/// +/// `TableCounts::blake3` is a 0-or-1 presence count, but `total()` adds it while +/// `VmAirs` derives `include_blake3` as `== 1`. Those two disagree for any value +/// outside {0, 1}: a count of 2 inflates the expected sub-proof total by two +/// while the AIR set gains none. `validate()`'s upper bound is what keeps that +/// disagreement unreachable, so the bound itself is pinned here rather than left +/// as an invariant nobody tests. +/// +/// It fails safe twice over — the arithmetic below shows the count check would +/// also reject — but a proof should never get that far on a malformed count. +#[test] +fn a_blake3_count_above_one_is_rejected() { + let mut counts = crate::TableCounts { + cpu: 1, + lt: 1, + memw: 1, + memw_aligned: 1, + load: 1, + mul: 1, + dvrm: 1, + shift: 1, + branch: 1, + memw_register: 1, + eq: 1, + bytewise: 1, + store: 1, + cpu32: 1, + keccak: 1, + keccak_rnd: 1, + ecsm: 1, + ecdas: 1, + hint: 1, + copy: 1, + blake3: 1, + is_wh: 1, + ecrec: 0, + }; + assert!(counts.validate().is_ok(), "1 is the honest maximum"); + counts.blake3 = 0; + assert!( + counts.validate().is_ok(), + "0 is legal — the table is conditional (see TableCounts::blake3)" + ); + + for bogus in [2usize, 3, usize::MAX] { + counts.blake3 = bogus; + let err = counts + .validate() + .expect_err("a BLAKE3 count above one must be rejected"); + assert!( + format!("{err}").contains("blake3"), + "the rejection must name the field, got {err}" + ); + } + + // The second line of defence, stated so the ordering is deliberate: even + // unvalidated, `total()` counts 2 where `air_refs` would build 0, so the + // sub-proof cross-check cannot pass either. + counts.blake3 = 2; + let with_two = counts.total().expect("the fixture's counts sum"); + counts.blake3 = 0; + assert_eq!( + with_two, + counts.total().expect("the fixture's counts sum") + 2, + "total() counts every claimed BLAKE3 table, so a count the AIR set does \ + not honour can never match the proof length" + ); +} + +/// (f) The six Ecall receivers carry pairwise-distinct syscall numbers. +/// +/// This is the premise the whole omission argument rests on: CPU sends the +/// syscall number as trace data, each receiver matches its own constant, and +/// BLAKE3's is matched by no other table — so removing BLAKE3 leaves its sends +/// unreceivable rather than absorbed by a neighbour. That is currently true by +/// inspection of six literals in five files, which is exactly the kind of fact +/// that stops being true silently. +/// +/// A collision here would not be a failed test so much as a soundness break: two +/// tables sharing a number could receive each other's sends, and a workload +/// using one could be proved with only the other present. +#[test] +fn the_ecall_receiver_syscall_numbers_are_pairwise_distinct() { + use executor::vm::instruction::execution::{ + BLAKE3_SYSCALL_NUMBER, DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER, + ECSM_SYSCALL_NUMBER, HINT_SYSCALL_NUMBER, KECCAK_SYSCALL_NUMBER, + }; + + // HALT and COPY's `write` spell their numbers inline in their receivers + // (`tables/halt.rs`, `tables/copy.rs`) rather than via a shared constant. + const HALT_SYSCALL_NUMBER: u64 = 93; + const WRITE_SYSCALL_NUMBER: u64 = 64; + + let receivers = [ + ("HALT", HALT_SYSCALL_NUMBER), + ("COPY write", WRITE_SYSCALL_NUMBER), + ("COPY memcpy", DMA_MEMCPY_SYSCALL_NUMBER), + ("COPY memset", DMA_MEMSET_SYSCALL_NUMBER), + ("KECCAK", KECCAK_SYSCALL_NUMBER), + ("BLAKE3", BLAKE3_SYSCALL_NUMBER), + ("ECSM", ECSM_SYSCALL_NUMBER), + ("HINT", HINT_SYSCALL_NUMBER), + ]; + + for (i, (name_a, a)) in receivers.iter().enumerate() { + for (name_b, b) in &receivers[i + 1..] { + assert_ne!( + a, b, + "{name_a} and {name_b} share an Ecall syscall number — either could \ + receive the other's sends, and omitting one table would stop being \ + detectable" + ); + } + } + + // The split the receivers actually compare on is (lo32, hi32), so equal + // halves would collide even with distinct u64s. + let halves: Vec<(u32, u32)> = receivers + .iter() + .map(|(_, n)| ((n & 0xFFFF_FFFF) as u32, (n >> 32) as u32)) + .collect(); + for (i, a) in halves.iter().enumerate() { + for b in &halves[i + 1..] { + assert_ne!(a, b, "two receivers agree on both 32-bit halves"); + } + } +} + +// ============================================================================= +// Keccak-256 of a buffer (the KECCAK table's sponge mode) +// ============================================================================= + +/// `test_keccak256`'s eleven calls, `(input offset, length)`, in commit order. +const KECCAK256_CALLS: [(usize, usize); 12] = [ + (0, 0), + (1, 1), + (3, 135), + (0, 136), + (5, 137), + (2, 271), + (0, 272), + (7, 600), + (8, 20), + (6, 32), + (4, 140), + (9, 136), +]; + +/// `test_keccak256` run, its digests checked against tiny-keccak, and its +/// minimal traces. +fn keccak256_fixture() -> (Elf, Traces) { + use tiny_keccak::{Hasher, Keccak}; + + let elf = Elf::load(&crate::test_utils::asm_elf_bytes("test_keccak256")).expect("ELF"); + let result = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("execution"); + let data: Vec = (0..1024).map(|i| ((13 * i + 5) % 256) as u8).collect(); + let output = &result.return_values.memory_values; + for (k, &(offset, len)) in KECCAK256_CALLS.iter().enumerate() { + let mut hasher = Keccak::v256(); + hasher.update(&data[offset..offset + len]); + let mut want = [0u8; 32]; + hasher.finalize(&mut want); + assert_eq!(&output[32 * k..32 * k + 32], &want, "call {k}"); + } + let traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + (elf, traces) +} + +/// The sponge rows of the KECCAK table, in order: `(row, first, last)`. +fn keccak256_rows(traces: &Traces) -> Vec<(usize, bool, bool)> { + use crate::tables::keccak::cols as kc; + let one = FieldElement::::one(); + let t = &traces.keccaks[0]; + (0..t.num_rows()) + .filter(|&r| *t.main_table.get(r, kc::MU_S) == one) + .map(|r| { + ( + r, + *t.main_table.get(r, kc::FIRST) == one, + *t.main_table.get(r, kc::LAST) == one, + ) + }) + .collect() +} + +#[test] +fn test_prove_elfs_keccak256() { + let (elf, mut traces) = keccak256_fixture(); + let blocks: usize = KECCAK256_CALLS.iter().map(|&(_, n)| n / 136 + 1).sum(); + assert_eq!(keccak256_rows(&traces).len(), blocks); + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "keccak256 prove/verify failed" + ); +} + +/// A message byte that is not what memory holds: the hash of other bytes. +#[test] +fn test_prove_keccak256_forged_message_byte_rejected() { + use crate::tables::keccak::cols as kc; + let (elf, mut traces) = keccak256_fixture(); + let (row, _, _) = keccak256_rows(&traces)[3]; + let t = &mut traces.keccaks[0].main_table; + let v = *t.get(row, kc::msg(17)); + t.set(row, kc::msg(17), v + FieldElement::one()); + assert!(!prove_and_verify_vm_minimal(&elf, &mut traces)); +} + +/// The digest written from a state other than the last permutation's output. +#[test] +fn test_prove_keccak256_forged_digest_rejected() { + use crate::tables::keccak::cols as kc; + let (elf, mut traces) = keccak256_fixture(); + let rows = keccak256_rows(&traces); + let at = rows + .iter() + .position(|&(_, first, last)| last && !first) + .expect("a call of two blocks or more"); + let (last, prev) = (rows[at].0, rows[at - 1].0); + let t = &mut traces.keccaks[0].main_table; + for i in 0..32 { + let v = *t.get(prev, kc::output_byte(i)); + t.set(last, kc::output_byte(i), v); + } + assert!(!prove_and_verify_vm_minimal(&elf, &mut traces)); +} + +/// The `0x01` moved one byte later on a last row whose block holds one input +/// byte, the count raised to match. +#[test] +fn test_prove_keccak256_forged_padding_rejected() { + use crate::tables::keccak::cols as kc; + let (elf, mut traces) = keccak256_fixture(); + let one = FieldElement::::one(); + let rows = keccak256_rows(&traces); + let t = &mut traces.keccaks[0].main_table; + let (row, _, _) = *rows + .iter() + .find(|&&(r, first, last)| last && !first && *t.get(r, kc::COUNT) == one) + .expect("a last block of one byte"); + t.set(row, kc::part(0, 1), one); + t.set(row, kc::COUNT, one + one); + assert!(!prove_and_verify_vm_minimal(&elf, &mut traces)); +} + +/// A row in both modes at once. +#[test] +fn test_prove_keccak256_forged_mixed_mode_rejected() { + use crate::tables::keccak::cols as kc; + let (elf, mut traces) = keccak256_fixture(); + let (row, _, _) = keccak256_rows(&traces)[0]; + traces.keccaks[0] + .main_table + .set(row, kc::MU_P, FieldElement::one()); + assert!(!prove_and_verify_vm_minimal(&elf, &mut traces)); +} diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index d1fd2009e..c244e7037 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -1183,6 +1183,13 @@ fn test_count_recursion_hashes() { // keccak_permute census (guest cost) measured for the standard proof; // `perms` here matches it when this test verifies that same proof (it is the // env-configurable dump, so a different workload legitimately differs). + // + // ⚠ These three numbers were measured BEFORE the per-table lineage merged: + // the statement encoding gained seven counts and the accelerator tables stop + // being always-on, so the proof this verifies is not the one they were taken + // from and a mismatch here is expected rather than a defect. The note below + // is a print, not a gate, which is why the merge does not redden this test — + // re-measure them on the merged tip before treating any of them as a pin. let census = match preset.name() { "min" => Some(3_595u64), "blowup2" => Some(448_737), diff --git a/prover/src/tests/recursion_soundness_gap_poc.rs b/prover/src/tests/recursion_soundness_gap_poc.rs index d2fb38e87..5b4f1e94c 100644 --- a/prover/src/tests/recursion_soundness_gap_poc.rs +++ b/prover/src/tests/recursion_soundness_gap_poc.rs @@ -16,13 +16,16 @@ use std::collections::HashSet; use std::path::PathBuf; -use crypto::fiat_shamir::default_transcript::DefaultTranscript; -use stark::prover::{IsStarkProver, Prover}; +// The bare `crypto` DefaultTranscript's type default is keccak, which since the +// stage-6 flip is NOT the configuration's transcript. This harness must sample +// Fiat-Shamir from the hash production replays (`DefaultStarkHash`), or every +// honest proof it builds is rejected at challenge derivation. Same half-flip +// `config.rs` warns about; the warning applies to test harnesses too. +use stark::prover::IsStarkProver; use crate::recursion::{MIN_PROOF_OPTIONS, precomputed_commitments}; use crate::statement::{StatementKind, absorb_statement, elf_digest}; use crate::tables::trace_builder::Traces; -use crate::test_utils::E; use crate::{MaxRowsConfig, VmAirs, VmProof}; use executor::elf::Elf; @@ -136,15 +139,8 @@ fn custom_prove_with_statement_elf( let result = executor.run().expect("run failed"); let max_rows = MaxRowsConfig::default(); - let mut traces = Traces::from_elf_and_logs( - &program, - &result.logs, - &max_rows, - &[], - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build failed"); + let mut traces = Traces::from_elf_and_logs(&program, &result.logs, &max_rows, &[]) + .expect("trace build failed"); let table_counts = traces.table_counts(); let airs = VmAirs::new( @@ -158,7 +154,8 @@ fn custom_prove_with_statement_elf( true, None, // As the production prover: FINI pinned to the halted register file. - crate::test_utils::halted_register_commitment(opts, &program, &traces.public_output_bytes), + crate::test_utils::halted_register_commitment(opts, &program, &traces.public_output_bytes) + .preprocessed(), ); let runtime_page_ranges = traces.runtime_page_ranges(); @@ -168,7 +165,7 @@ fn custom_prove_with_statement_elf( .filter(|c| c.is_private_input) .count(); - let mut transcript = DefaultTranscript::::new(&[]); + let mut transcript = crate::hash_pin::block_transcript(&[]); absorb_statement( &mut transcript, StatementKind::Monolithic, @@ -180,11 +177,10 @@ fn custom_prove_with_statement_elf( opts.fri_final_poly_log_degree, ); - let proof = Prover::multi_prove( + let proof = crate::hash_pin::BlockProver::multi_prove( airs.air_trace_pairs(&mut traces), &mut transcript, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .expect("multi_prove failed"); diff --git a/prover/src/tests/register_tests.rs b/prover/src/tests/register_tests.rs index b602bf348..92b24348b 100644 --- a/prover/src/tests/register_tests.rs +++ b/prover/src/tests/register_tests.rs @@ -132,6 +132,48 @@ fn test_precomputed_commitment_binds_fini() { assert_ne!(root_a, root_b); } +/// The columns the multilinear verifier checks are the ones the univariate +/// verifier's root commits, and the declared count is theirs. +/// +/// ⚠ TWO SEPARATE THINGS COULD DRIFT and each has its own assertion here: +/// the count constant against the columns function, and each column's +/// position in the order. The count is the one a caller writes down — +/// `VmAirs::new` declares `NUM_PREPROCESSED_COLS` beside a closure that must +/// produce exactly that many, and a mismatch there is a verifier checking a +/// prefix of what it thinks it checks. +#[test] +fn the_preprocessed_columns_are_what_the_commitment_holds() { + let init = register_init_from_entry_point(0x1000); + let mut fini = vec![0u32; NUM_REGISTER_ADDRESSES]; + fini[10] = 0x42; + fini[X254_INDEX] = 9; + + let columns = preprocessed_columns(&init, &fini); + assert_eq!( + columns.len(), + NUM_PREPROCESSED_COLS, + "the declared preprocessed count and the columns built for it disagree" + ); + + // INIT is `init`, in the register word-address order. + let init_col = &columns[cols::INIT]; + for (k, &word) in init.iter().enumerate() { + assert_eq!(init_col[k], FE::from(u64::from(word)), "INIT row {k}"); + } + + // FINI is last, and it is `fini` — read at two addresses that differ, so a + // column of zeros or a duplicated INIT fails. + let fini_col = &columns[cols::FINI]; + assert_eq!(fini_col[10], FE::from(0x42u64)); + assert_eq!(fini_col[X254_INDEX], FE::from(9u64)); + assert_eq!(fini_col[0], FE::from(0u64)); + assert_eq!( + fini_col.len(), + NUM_REGISTER_ADDRESSES.next_power_of_two(), + "the columns are padded to the table's height" + ); +} + /// `fini_from_final_state` must return exactly what `fini_from_trace` reads off /// the generated REGISTER trace, for both accessed and never-accessed /// registers — it is the continuation producer's trace-free replacement. diff --git a/prover/src/tests/rpx_two_implementations.rs b/prover/src/tests/rpx_two_implementations.rs new file mode 100644 index 000000000..573cce561 --- /dev/null +++ b/prover/src/tests/rpx_two_implementations.rs @@ -0,0 +1,179 @@ +//! ★ The two RPX implementations this workspace now carries, asserted equal. +//! +//! The merge that built this branch brought RPX in twice. The per-table branch +//! has it at `prover::lfm::algebraic_commit` (generic over `HasherKind`, shared +//! with the LFM's `LFM_HASH` socket and the algebraic Merkle backends); the +//! WHIR branch ported it to `crypto::hash::rpx` (monomorphic, so `crypto` can +//! offer an algebraic sponge without depending on `prover`). Both are real and +//! both stay: neither crate can use the other's. +//! +//! ⚠ **What that costs, and what this file buys back.** Two implementations of +//! one primitive, in one tree, both answering to the name `rpx256`, is a +//! configuration where "which RPX produced this root" has no answer in the +//! type system. Reading the two and observing that they are line-for-line the +//! same function is not a test — it is a claim about a moment, and the next +//! edit to either side is free to break it silently, because nothing links +//! them. Every consumer would keep compiling and every proof would keep +//! verifying; only a root built by one and checked by the other would differ, +//! and nothing in this workspace does that in a test. +//! +//! So the equality is asserted, at the lengths where the two could plausibly +//! disagree — the partial trailing group, the exact block boundary, the empty +//! input, the 40-byte grinding preimage — and anchored, where the shared vector +//! table has a row, to the digests the CUDA kernel is itself pinned to. That +//! last part matters most: `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h` +//! is one file consumed by the device KAT, so a host that reproduces it is +//! byte-compatible with the kernel as well as with its twin. +//! +//! The type names were made distinct in the same change (`RpxTranscriptHash` +//! became `AlgebraicRpxTranscriptHash` on the LFM side). The names stop a +//! reader confusing them; this file is what stops them drifting. + +use crate::lfm::algebraic_commit::{digest_to_commitment, sponge_leaf_bytes}; +use crate::lfm::hash::HasherKind; +use crypto::hash::rpx; + +/// Deterministic, non-degenerate bytes: no all-zero run that a broken padding +/// rule could accidentally agree on. +fn sample(len: usize) -> Vec { + (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect() +} + +/// The LFM side's byte digest, as `AlgebraicDigest` finalizes it. +fn lfm_digest(bytes: &[u8]) -> [u8; 32] { + digest_to_commitment(&sponge_leaf_bytes(HasherKind::Rpx, bytes)) +} + +/// The WHIR side's byte digest, as `Rpx256Digest` finalizes it. +fn whir_digest(bytes: &[u8]) -> [u8; 32] { + rpx::digest_to_commitment(&rpx::sponge_leaf_bytes(bytes)) +} + +/// The lengths that matter, and why each is here: +/// +/// - `0` — the empty input returns the capacity without permuting at all, the +/// one path that skips the absorb loop entirely. +/// - `1`, `7` — a partial trailing felt. The low-side zero extension is the +/// detail `sponge_leaf_bytes`' own doc calls easy to get backwards. +/// - `8` — exactly one felt, no padding. +/// - `9` — one felt plus one byte, the first input that spans a felt boundary. +/// - `16`, `17` — inside the first rate block and just past a felt boundary. +/// - `40` — five felts: the grinding preimage, `state ‖ nonce`, the one length +/// whose digest a device kernel also computes. +/// - `64` — exactly one rate block, so the padding flag `len mod 8` is zero and +/// no trailing block is spent. +/// - `65`, `200` — past one block, which is where a two-block absorb that +/// reset the capacity instead of overwriting the rate would diverge. +const LENGTHS: [usize; 11] = [0, 1, 7, 8, 9, 16, 17, 40, 64, 65, 200]; + +#[test] +fn the_two_rpx_implementations_agree_on_the_same_bytes() { + for len in LENGTHS { + let bytes = sample(len); + assert_eq!( + lfm_digest(&bytes), + whir_digest(&bytes), + "the LFM and WHIR RPX implementations disagree at {len} bytes — one of them has \ + drifted, and every root either produced is now ambiguous" + ); + } +} + +/// ★ FALSIFICATION. The test above compares two functions; if both were the +/// same stub it would pass. This pins them to a value neither produced. +/// +/// The vectors are the 8- and 16-felt rows of +/// `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h`, the table the CUDA +/// kernel's host KAT replays. Canonical felts serialized big-endian round-trip +/// through `felts_from_bytes`, so the byte path must reach the felt path's +/// answer — which makes this an anchor for the byte APIs above, not just for +/// the sponge underneath them. +#[test] +fn both_implementations_reproduce_the_shared_kernel_vectors() { + // Table 3, len = 8 felts. + let felts_8: [u64; 8] = [ + 3521541860211663897, + 5585621328801039182, + 3314063895810834828, + 6286715337571703139, + 9272399501810688383, + 17378448552699642502, + 9663403628134293866, + 8225575178453385283, + ]; + let digest_8: [u64; 4] = [ + 14052993739410942603, + 8384701950754250190, + 11473922331550289114, + 16644313465254305812, + ]; + + // Table 3, len = 16 felts — two full rate blocks. + let felts_16: [u64; 16] = [ + 9660685076555889599, + 4027567791223379602, + 11432600011703367870, + 6441517771629429252, + 8272264386868866348, + 16565648022353132158, + 16844837242675693755, + 12942506659476152817, + 11839051358503478840, + 1846358602548732379, + 118703897581348635, + 14480592082795401517, + 12015885875590073011, + 7433808365622677077, + 13247077855319202624, + 17837888200692576115, + ]; + let digest_16: [u64; 4] = [ + 18135965004560326100, + 1948492279228612931, + 17772968542724134453, + 12116464713281646840, + ]; + + for (felts, expected) in [(&felts_8[..], digest_8), (&felts_16[..], digest_16)] { + let bytes: Vec = felts.iter().flat_map(|f| f.to_be_bytes()).collect(); + let want: [u8; 32] = { + let mut out = [0u8; 32]; + for (i, w) in expected.iter().enumerate() { + out[i * 8..i * 8 + 8].copy_from_slice(&w.to_be_bytes()); + } + out + }; + assert_eq!( + lfm_digest(&bytes), + want, + "the LFM RPX no longer reproduces the vector the CUDA kernel is pinned to \ + ({} felts)", + felts.len() + ); + assert_eq!( + whir_digest(&bytes), + want, + "the WHIR RPX no longer reproduces the vector the CUDA kernel is pinned to \ + ({} felts)", + felts.len() + ); + } +} + +/// The sampler itself has to be able to tell two inputs apart, or the agreement +/// test above compares one digest with itself eleven times. +#[test] +fn the_sample_inputs_are_distinct() { + let digests: Vec<[u8; 32]> = LENGTHS.iter().map(|l| lfm_digest(&sample(*l))).collect(); + for (i, a) in digests.iter().enumerate() { + for (j, b) in digests.iter().enumerate().skip(i + 1) { + assert_ne!( + a, b, + "lengths {} and {} hash the same", + LENGTHS[i], LENGTHS[j] + ); + } + } +} diff --git a/prover/src/tests/shape_profile_tests.rs b/prover/src/tests/shape_profile_tests.rs new file mode 100644 index 000000000..804b2a8b5 --- /dev/null +++ b/prover/src/tests/shape_profile_tests.rs @@ -0,0 +1,145 @@ +//! Exercises the shape profile against the real VM AIRs. +//! +//! The unit tests in `shape_profile` cover the schema mapping with plain data; +//! this covers the other half — that `AirShapeInput::from_air` reads sane +//! shapes off all ~29 live tables, and that a captured segment stays +//! self-consistent. + +use crate::shape_profile::{AirShapeInput, BusIndexMap, SegmentProfile, build_segment}; +use crate::tables::trace_builder::Traces; +use crate::test_utils::run_asm_elf; + +/// Builds a real segment profile from a small program's tables. +fn profile_for(program: &str) -> SegmentProfile { + let (elf, logs, _instructions) = run_asm_elf(program); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &logs, &Default::default(), &[]).unwrap(); + + let proof_options = stark::proof::options::ProofOptions::default_test_options(); + let table_counts = traces.table_counts(); + let airs = crate::VmAirs::new( + &elf, + &proof_options, + true, + &traces.page_configs, + &table_counts, + &crate::cpu_heights_from_traces(&traces), + None, + true, + None, + crate::test_utils::halted_register_commitment( + &proof_options, + &elf, + &traces.public_output_bytes, + ) + .preprocessed(), + ); + + let pairs = airs.air_trace_pairs(&mut traces); + let mut buses = BusIndexMap::new(); + build_segment( + 0, + pairs + .iter() + .map(|(air, trace, _)| AirShapeInput::from_air(*air, trace.num_rows())), + &mut buses, + ) +} + +#[test] +fn real_airs_produce_a_self_consistent_segment() { + let profile = profile_for("sub"); + + assert!( + profile.airs.len() >= 10, + "expected the full table set, got {}", + profile.airs.len() + ); + + for air in &profile.airs { + assert_eq!( + air.buses.len(), + air.num_interactions, + "{}: bus count disagrees with num_interactions", + air.air_name + ); + assert_eq!( + air.interaction_message_lens.len(), + air.num_interactions, + "{}: message-len count disagrees with num_interactions", + air.air_name + ); + assert_eq!( + air.interaction_count_weights.len(), + air.num_interactions, + "{}: count-weight count disagrees with num_interactions", + air.air_name + ); + assert_eq!( + 1usize << air.log_height, + air.height, + "{}: log_height does not match height", + air.air_name + ); + assert!(air.width.common_main > 0, "{}: empty main", air.air_name); + assert!( + air.max_constraint_degree >= 1, + "{}: degree below 1", + air.air_name + ); + // Every message carries at least one field beyond the bus id. + for (i, len) in air.interaction_message_lens.iter().enumerate() { + assert!( + *len > 0, + "{}: interaction {i} has an empty message", + air.air_name + ); + } + } +} + +#[test] +fn air_ids_are_dense_and_ordered() { + let profile = profile_for("sub"); + let ids: Vec = profile.airs.iter().map(|a| a.air_id).collect(); + assert_eq!(ids, (0..profile.airs.len()).collect::>()); +} + +#[test] +fn global_degree_dominates_every_air() { + let profile = profile_for("sub"); + for air in &profile.airs { + assert!( + air.max_constraint_degree <= profile.global_max_constraint_degree, + "{} exceeds the global degree", + air.air_name + ); + } +} + +#[test] +fn bus_indices_are_dense_from_zero() { + let profile = profile_for("sub"); + let distinct: std::collections::HashSet = profile + .airs + .iter() + .flat_map(|a| a.buses.iter().copied()) + .collect(); + assert!(!distinct.is_empty(), "no buses captured"); + + // Indices are handed out 0, 1, 2, ... as bus ids are first seen, so the set + // must be exactly 0..distinct.len(). + let max = distinct.iter().copied().max().unwrap() as usize; + assert_eq!( + max, + distinct.len() - 1, + "bus indices are not dense from zero" + ); +} + +#[test] +fn interaction_totals_are_nonzero_across_the_segment() { + let profile = profile_for("sub"); + let total: usize = profile.airs.iter().map(|a| a.num_interactions).sum(); + assert!(total > 0, "captured no bus interactions at all"); +} diff --git a/prover/src/tests/skip_empty_tables_tests.rs b/prover/src/tests/skip_empty_tables_tests.rs index e7050ea84..186058336 100644 --- a/prover/src/tests/skip_empty_tables_tests.rs +++ b/prover/src/tests/skip_empty_tables_tests.rs @@ -100,7 +100,10 @@ fn every_table_participates_in_the_bus() { ecsm: 1, ecdas: 1, hint: 1, + copy: 1, + blake3: 1, is_wh: 1, + ecrec: 0, }; let airs = VmAirs::new( &elf, @@ -116,7 +119,8 @@ fn every_table_participates_in_the_bus() { &ProofOptions::default_test_options(), &elf, &traces.public_output_bytes, - ), + ) + .preprocessed(), ); let without_bus: Vec<&str> = airs @@ -162,8 +166,25 @@ fn droppable_air_names(counts: &TableCounts) -> Vec<&'static str> { ecsm, ecdas, hint, + copy, + blake3, is_wh, + ecrec, } = counts; + // ⛔ BLAKE3 IS DROPPABLE AND IS DELIBERATELY NOT IN THIS LIST — a + // COVERAGE GAP, not an exemption. It belongs here on exactly the + // argument the others do (0 or 1 by `TableCounts::blake3`, omitted when + // the run never reaches it), and it was added here first. The check + // below then failed by name: "BLAKE3 is no longer reached by any + // program here", with the other twenty chips all seen. No fixture in + // this file executes a BLAKE3 syscall, so its bus contribution is + // UNEXAMINED — listing it would only assert a table nothing builds. + // + // What closes this is a fixture program that calls the BLAKE3 syscall, + // not a change here. Until then the omission is stated rather than + // silent, and it is a property of the TEST SET only: the production + // elision reads `TableCounts`, never this list. + let _ = blake3; [ ("CPU", cpu), ("LT", lt), @@ -184,7 +205,9 @@ fn droppable_air_names(counts: &TableCounts) -> Vec<&'static str> { ("ECSM", ecsm), ("ECDAS", ecdas), ("HINT", hint), + ("COPY", copy), ("IS_WH", is_wh), + ("ECREC", ecrec), ] .into_iter() .map(|(name, _count)| name) @@ -242,6 +265,7 @@ fn no_present_table_contributes_zero_to_the_bus() { "test_keccak", "test_ecsm", "test_commit_4", + "test_ecrec", ]; let mut droppable: Vec<&'static str> = Vec::new(); @@ -355,7 +379,8 @@ fn dropping_a_used_table_through_the_real_verifier_is_rejected() { None, true, None, - crate::test_utils::halted_register_commitment(&opts, &elf, &vm_proof.public_output), + crate::test_utils::halted_register_commitment(&opts, &elf, &vm_proof.public_output) + .preprocessed(), ); let mul_at = airs .air_refs() @@ -399,11 +424,11 @@ fn dropping_a_used_table_through_the_real_verifier_is_rejected() { #[test] fn no_optional_table_uses_a_constant_multiplicity() { use crate::tables::{ - branch, bytewise, cpu32, dvrm, ecdas, ecsm, eq, hint, is_wh, keccak, keccak_rnd, load, lt, - memw, memw_aligned, mul, shift, store, + branch, bytewise, copy, cpu32, dvrm, ecdas, ecsm, eq, hint, is_wh, keccak, keccak_rnd, + load, lt, memw, memw_aligned, mul, shift, store, }; use stark::lookup::{BusValue, LinearTerm}; - let optional: [(&str, Vec); 18] = [ + let optional: [(&str, Vec); 19] = [ ("LT", lt::bus_interactions()), ("MEMW", memw::bus_interactions()), ("MEMW_A", memw_aligned::bus_interactions()), @@ -421,6 +446,7 @@ fn no_optional_table_uses_a_constant_multiplicity() { ("ECSM", ecsm::bus_interactions()), ("ECDAS", ecdas::bus_interactions()), ("HINT", hint::bus_interactions()), + ("COPY", copy::bus_interactions()), ("IS_WH", is_wh::bus_interactions()), ]; for (name, interactions) in optional { @@ -574,7 +600,7 @@ fn validate_still_requires_cpu_and_the_register_file() { } /// The counts ride in the proof, so they are the prover's to choose, and the -/// sub-proof cross-check constrains only their *sum*. The five accelerators and +/// sub-proof cross-check constrains only their *sum*. The six accelerators and /// IS_WH are not chunked — `generate_optional` emits one table or none — so any count /// above 1 describes a table set no prover can build, and the sum alone does not /// catch it: lower another count by the same amount and the total still matches. @@ -600,12 +626,13 @@ fn an_accelerator_count_above_one_is_rejected() { // Named setters rather than a match with a catch-all: a wrong field would // otherwise pass the assertion below while testing the wrong count. type SetCount = fn(&mut TableCounts); - let inflate: [(&str, SetCount); 6] = [ + let inflate: [(&str, SetCount); 7] = [ ("keccak", |c| c.keccak = 2), ("keccak_rnd", |c| c.keccak_rnd = 2), ("ecsm", |c| c.ecsm = 2), ("ecdas", |c| c.ecdas = 2), ("hint", |c| c.hint = 2), + ("copy", |c| c.copy = 2), ("is_wh", |c| c.is_wh = 2), ]; for (name, set_to_two) in inflate { @@ -617,8 +644,8 @@ fn an_accelerator_count_above_one_is_rejected() { ); } - // The chunked chips keep taking any count: this is a cap on the five - // accelerators and IS_WH, not a cap on chunking. + // The chunked chips keep taking any count: this is a cap on the six + // accelerators, BLAKE3 and IS_WH, not a cap on chunking. let mut many_mul = honest.clone(); many_mul.mul = 7; assert!(many_mul.validate().is_ok(), "chunked chips still chunk"); @@ -655,9 +682,9 @@ fn counts_that_wrap_have_no_total() { /// The accelerators are the ones a run most often never reaches, and each cost a /// four-row sub-proof regardless. A program with no keccak, no EC and no hint -/// ecall now carries none of the five, nor the IS_WH table KECCAK feeds. +/// ecall now carries none of the six, nor the IS_WH table KECCAK feeds. #[test] -fn a_run_without_accelerators_omits_all_five_and_is_wh() { +fn a_run_without_accelerators_omits_all_six_and_is_wh() { let (elf, logs, _instructions) = run_asm_elf("xori"); let mut traces = Traces::from_elf_and_logs_minimal(&elf, &logs, &Default::default(), &[]).unwrap(); @@ -669,6 +696,7 @@ fn a_run_without_accelerators_omits_all_five_and_is_wh() { ("ecsm", counts.ecsm), ("ecdas", counts.ecdas), ("hint", counts.hint), + ("copy", counts.copy), // Not an accelerator, but only KECCAK feeds it today. ("is_wh", counts.is_wh), ] { @@ -708,6 +736,25 @@ fn omitting_a_used_accelerator_fails_the_bus_balance() { assert_only_the_bus_rejected(&weigh_the_bus(&elf, &mut traces, true), "dropping HINT"); } +/// ECREC cannot be dropped either: the CPU's ecall send for `-12` has no other +/// receiver, and its chains' ECDAS messages lose their seeds and drains. +#[test] +fn omitting_the_ecrec_table_fails_the_bus_balance() { + let (elf, logs, _instructions) = run_asm_elf("test_ecrec"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &logs, &Default::default(), &[]).unwrap(); + assert!( + !traces.ecrecs.is_empty(), + "test_ecrec must call ECREC for this to be a forgery" + ); + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "the honest proof must verify first, or the negative below proves nothing" + ); + traces.ecrecs.clear(); + assert_only_the_bus_rejected(&weigh_the_bus(&elf, &mut traces, true), "dropping ECREC"); +} + /// The dispatch above is CPU-to-chip. KECCAK_RND is reached from KECCAK, not /// from the CPU, so dropping it exercises a chip-to-chip bus: the anchor whose /// terms go unmatched sits in another accelerator's contribution, not in the @@ -758,12 +805,54 @@ fn omitting_the_is_wh_table_fails_the_bus_balance() { assert_only_the_bus_rejected(&weigh_the_bus(&elf, &mut traces, true), "dropping IS_WH"); } -/// The committed output is the one bus target that is not zero: the verifier -/// computes it from the `public_output` the prover supplies -/// (`compute_expected_commit_bus_balance_view`). -/// -/// There is no COMMIT table to omit any more — the `write` syscall is served by -/// COPY, which is mandatory — so the forgery left is to keep the rows that send +/// COPY is the one optional table whose bus target is not zero: the verifier +/// computes it from the public output the prover supplies +/// (`compute_expected_commit_bus_balance_view`), and nothing requires +/// `copy > 0` when that output is non-empty. The COPY sub-proof's presence is a +/// number the prover picks, and the balance is the only thing standing behind it. +#[test] +fn omitting_the_copy_table_fails_the_bus_balance() { + let (elf, logs, _instructions) = run_asm_elf("test_commit_4"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &logs, &Default::default(), &[]).unwrap(); + assert!( + !traces.copies.is_empty() && !traces.public_output_bytes.is_empty(), + "test_commit_4 must commit output for this to be a forgery" + ); + + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "the honest proof must verify first, or the negative below proves nothing" + ); + + traces.copies.clear(); + assert_only_the_bus_rejected(&weigh_the_bus(&elf, &mut traces, true), "COPY"); +} + +/// The variant the test above does not cover: a prover who drops COPY can also +/// drop the output it committed, and that moves the verifier's *target* rather +/// than the sum — the balance is recomputed for an empty output. The CPU's +/// commit ecalls are still in the trace with no receiver, so the two sides +/// still have to disagree. +#[test] +fn omitting_the_copy_table_and_its_output_fails_the_bus_balance() { + let (elf, logs, _instructions) = run_asm_elf("test_commit_4"); + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &logs, &Default::default(), &[]).unwrap(); + let honest_target = weigh_the_bus(&elf, &mut traces, false).target; + + traces.copies.clear(); + traces.public_output_bytes.clear(); + let outcome = weigh_the_bus(&elf, &mut traces, true); + assert_ne!( + outcome.target, honest_target, + "clearing the public output must move the target, or this test is the \ + previous one over again" + ); + assert_only_the_bus_rejected(&outcome, "COPY together with its output"); +} + +/// The third way to forge the committed output: keep the COPY rows that send /// the committed bytes and drop the output they claim. That moves the verifier's /// target while the sum stays put, and the two sides have to disagree. #[test] @@ -772,7 +861,7 @@ fn dropping_the_committed_output_fails_the_bus_balance() { let mut traces = Traces::from_elf_and_logs_minimal(&elf, &logs, &Default::default(), &[]).unwrap(); assert!( - !traces.public_output_bytes.is_empty(), + !traces.copies.is_empty() && !traces.public_output_bytes.is_empty(), "test_commit_4 must commit output for this to be a forgery" ); diff --git a/prover/src/tests/statement_alignment_tests.rs b/prover/src/tests/statement_alignment_tests.rs new file mode 100644 index 000000000..779344259 --- /dev/null +++ b/prover/src/tests/statement_alignment_tests.rs @@ -0,0 +1,695 @@ +//! ★★ Every value a WHIR proof absorbs starts on a field element boundary. +//! +//! ```text +//! cargo test -p lambda-vm-prover --lib statement_alignment +//! ``` +//! +//! # The property, and what it is NOT +//! +//! The transcript hashes BYTES. The algebraic configuration's sponge re-slices +//! everything absorbed since its last squeeze into field elements every 8 bytes +//! (`crypto::hash::rpx::sponge_leaf_bytes`), so a value absorbed at an offset +//! that is not a multiple of 8 straddles two of them — which a field-machine +//! verifier replaying the transcript can only reproduce by decomposing bits. +//! +//! The property is therefore about OFFSETS, over every shape: *every absorb +//! after the statement lands at a window offset that is a multiple of 8*. It is +//! **not** "the statement's fixed prefix is a multiple of 8": the roots do not +//! follow the fixed prefix, they follow two variable-length fields, and a +//! statement padded to a round fixed prefix leaves them at +//! `(|public_output| + |table_num_vars|) mod 8` — 2 mod 8 at the shape this +//! system runs. That arithmetic is why the pad is computed from an accumulated +//! length rather than written as a constant, and this file is where the claim +//! can fail. +//! +//! # Two tests, because one of them cannot see the other's failure +//! +//! * [`the_statement_ends_on_a_field_element_boundary`] sweeps the shapes +//! against a recording transcript. It sees every shape and no protocol. +//! * [`every_absorb_of_a_real_prove_is_field_element_aligned`] drives a real +//! `multi_prove` and sees one shape and the whole protocol. It is the one that +//! can report a misalignment the statement padding does not fix. +//! +//! # Why the recorder is validated rather than trusted +//! +//! [`WindowRecorder`] has to know where a window ENDS, and only a squeeze ends +//! one — so it mirrors `DefaultTranscript`'s duplex output buffer instead of +//! guessing. A mirror that drifts would draw different challenges, so the +//! end-to-end test first proves the same fixture twice, once through the +//! production transcript and once through the recorder, and requires the two +//! proofs to serialise to the same bytes. The offsets it reports are then the +//! production stream's offsets, not a model's. + +use digest::Digest; +use math::field::element::FieldElement; +use math::field::traits::HasDefaultTranscript; +use math::traits::AsBytes; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::transcript_hash::{ + HasTranscriptHash, KeccakTranscriptHash, TranscriptHash, +}; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::statement::{FELT_BYTES, NUM_TABLE_KINDS, statement_padding}; +use crate::tables::eq::{EqConstraints, EqOperation, generate_eq_trace}; +use crate::test_utils::{ConcreteVmAir, E, F, create_eq_air}; +use crate::{RuntimePageRange, TableCounts}; + +/// Bytes one squeeze hands the transcript, and therefore the offset a fresh +/// window opens at. `DefaultTranscript::sample` finalize-resets the sponge and +/// re-absorbs its own 32-byte output, so a window never opens empty. +/// +/// It is private to `default_transcript`, so it is mirrored here rather than +/// imported — and a wrong value here does not silently weaken this file: the +/// duplex buffer would refill at the wrong time, the challenge stream would +/// diverge, and +/// [`every_absorb_of_a_real_prove_is_field_element_aligned`]'s byte comparison +/// against the production transcript would fail. +const SQUEEZE_LEN: usize = 32; + +// ------------------------------------------------------------------------- +// The recording transcript +// ------------------------------------------------------------------------- + +/// A transcript that records the window offset of every absorb. +/// +/// Absorption is DELEGATED — the inner `DefaultTranscript` is what hashes, so +/// this is the production sponge with a tape attached. Only the duplex output +/// buffer is mirrored, because that is the only part that tells the recorder +/// when a window ends. +struct WindowRecorder { + /// Whether `mark_statement_end` has reached this transcript. + /// + /// ⛔ THE ONE THING NO OTHER CHECK HERE CAN SEE. Every comparison in this + /// file reads zero when the mark is MISSING just as it does when the mark + /// is present and the pad is correct: a correct pad misaligns nothing after + /// a statement, so "counter 0" and "recorder 0" agree either way. Deleting + /// the call from `absorb_statement_padding` would pass the whole file. This + /// flag is what + /// [`the_statement_padding_tells_the_transcript_the_statement_ended`] + /// asserts, and it is not gated on `hash-metrics`, because the wiring must + /// be checkable in an ordinary build. + marked: bool, + inner: DefaultTranscript, + out_buf: [u8; SQUEEZE_LEN], + out_pos: usize, + /// Bytes absorbed since the sponge was last reset by a squeeze. + window: usize, + /// `(window offset, length)` of every absorb, in call order. + absorbs: Vec<(usize, usize)>, +} + +/// Of the absorbs recorded after the first `from` — which is how a caller says +/// "everything after the statement" — the ones that do not start on a field +/// element boundary. +fn misaligned(absorbs: &[(usize, usize)], from: usize) -> Vec<(usize, usize)> { + absorbs[from..] + .iter() + .copied() + .filter(|(offset, _)| !offset.is_multiple_of(FELT_BYTES)) + .collect() +} + +impl WindowRecorder { + fn new() -> Self { + Self { + marked: false, + inner: DefaultTranscript::::new(&[]), + out_buf: [0u8; SQUEEZE_LEN], + out_pos: SQUEEZE_LEN, + window: 0, + absorbs: Vec::new(), + } + } + + fn record(&mut self, len: usize) { + self.absorbs.push((self.window, len)); + self.window += len; + // Same invalidation the inner transcript performs: a challenge drawn + // after an absorb must depend on it. + self.out_pos = SQUEEZE_LEN; + } + + /// `DefaultTranscript::next_sample_u64`, mirrored. The squeeze itself is the + /// inner transcript's, so the bytes are production's; what is duplicated is + /// only the bookkeeping that says when one happens. + fn next_u64(&mut self) -> u64 { + if self.out_pos + 8 > SQUEEZE_LEN { + self.out_buf = self.inner.sample(); + self.out_pos = 0; + // A squeeze finalize-resets the sponge and re-absorbs its own + // output, so the new window opens holding those 32 bytes. + self.window = SQUEEZE_LEN; + } + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(&self.out_buf[self.out_pos..self.out_pos + 8]); + self.out_pos += 8; + u64::from_be_bytes(bytes) + } +} + +impl HasTranscriptHash for WindowRecorder { + type Hash = T; +} + +impl IsTranscript for WindowRecorder { + fn append_bytes(&mut self, new_bytes: &[u8]) { + self.record(new_bytes.len()); + self.inner.append_bytes(new_bytes); + } + + fn append_field_element(&mut self, element: &FieldElement) { + // Counted per `update`, the same unit the inner transcript counts, so a + // serialisation that streamed an element in pieces would show up here + // as several absorbs rather than one. + let mut chunks: Vec = Vec::new(); + element.stream_bytes(&mut |b| chunks.push(b.len())); + for len in chunks { + self.record(len); + } + self.inner.append_field_element(element); + } + + fn mark_statement_end(&mut self) { + // Delegated, like absorption: the inner transcript is the production + // one and its counter is the instrument being cross-checked. The + // recorder's own TAPE needs no flag — `misaligned(&rec, from)` takes + // the boundary as an argument, which is what makes it an INDEPENDENT + // second answer to the same question rather than a copy of the first. + // The flag records only that the call ARRIVED. + self.marked = true; + self.inner.mark_statement_end(); + } + + fn state(&self) -> [u8; 32] { + // Finalizes a clone: no reset, no re-absorb, so the window does not move. + self.inner.state() + } + + fn sample_field_element(&mut self) -> FieldElement { + E::sample_field_element_from(|| self.next_u64()) + } + + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + assert!(upper_bound > 0, "upper_bound must be greater than 0"); + let threshold = upper_bound.wrapping_neg() % upper_bound; + loop { + let candidate = self.next_u64(); + if candidate >= threshold { + return candidate % upper_bound; + } + } + } +} + +// ------------------------------------------------------------------------- +// The closed forms, written from the field lists rather than from the code +// ------------------------------------------------------------------------- + +const DIGEST: usize = 32; +const U64: usize = 8; +/// `log_blowup`, `log_folding`, `num_queries`. +const CONFIG: usize = 3 * U64; +/// `grind.folding`, `grind.ood`, `grind.query`, absorbed as one 3-byte value. +const GRIND_TRAILER: usize = 3; + +/// What a statement's fields add up to, what it therefore absorbs in total, and +/// how many times it calls the transcript. +#[derive(Clone, Copy, Debug)] +struct Expected { + /// The statement's own fields, before any padding. + body: usize, + /// `body` plus the pad, which is what the transcript should have taken. + total: usize, + /// Absorb calls, padding included. + calls: usize, +} + +impl Expected { + fn new(body: usize, calls: usize) -> Self { + Self { + body, + total: body + statement_padding(body), + calls, + } + } +} + +/// What the monolithic multilinear statement absorbs, field by field. +/// +/// Deliberately a SECOND derivation: the production function accumulates its +/// length beside its own absorbs, and this adds up the fields it is supposed to +/// have. The two can only be compared here, and a field added to one and not +/// the other fails here. Its ranges are sixteen bytes an entry, so they are +/// alignment-neutral; `table_num_vars` is not. +fn monolithic_expected( + public_output: usize, + runtime_page_ranges: usize, + table_num_vars: usize, +) -> Expected { + let body = crate::statement::MULTILINEAR_TAG.len() + + DIGEST + + U64 // |public_output| + + public_output + + NUM_TABLE_KINDS * U64 + + U64 // num_private_input_pages + + U64 // |runtime_page_ranges| + + runtime_page_ranges * 2 * U64 + + U64 // |table_num_vars| + + table_num_vars + + CONFIG + + GRIND_TRAILER; + let calls = + 1 + 1 + 1 + 1 + NUM_TABLE_KINDS + 1 + 1 + 2 * runtime_page_ranges + 1 + 1 + 3 + 1 + 1; + Expected::new(body, calls) +} + +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + } +} + +fn counts() -> TableCounts { + TableCounts { + cpu: 1, + lt: 2, + memw: 3, + memw_aligned: 4, + load: 5, + mul: 6, + dvrm: 7, + shift: 8, + branch: 9, + memw_register: 10, + eq: 11, + bytewise: 12, + store: 13, + cpu32: 14, + // These seven are 0-or-1 presence flags ("did the workload reach this + // chip"), not chunk counts. Only the NUMBER of counts moves a + // statement's length, so the values are free; 1 keeps the struct inside + // its own domain. + keccak: 1, + keccak_rnd: 1, + ecsm: 1, + ecdas: 1, + hint: 1, + copy: 1, + blake3: 1, + is_wh: 1, + ecrec: 0, + } +} + +/// The shapes the sweep runs, as `(|public_output|, |table_num_vars|)`. +/// +/// `(0, 34)` is what this system actually proves — and the shape a constant pad +/// would have left at 2 mod 8. The others are chosen so the residue of +/// `|public_output| + |table_num_vars|` differs: a sweep in which every shape +/// needed the same pad would pass with the pad hard-coded to that value. +const SHAPES: &[(usize, usize)] = &[(0, 34), (7, 34), (32, 34), (33, 35), (2, 1), (0, 1)]; + +// ------------------------------------------------------------------------- +// (1) The statement, over every shape +// ------------------------------------------------------------------------- + +#[test] +fn the_statement_ends_on_a_field_element_boundary() { + let mut residues_seen = std::collections::BTreeSet::new(); + + for &(po_len, tnv_len) in SHAPES { + let public_output = vec![0xABu8; po_len]; + let table_num_vars = vec![20u8; tnv_len]; + + let mut rec = WindowRecorder::::new(); + crate::multilinear_prove::absorb( + &mut rec, + &[7u8; 32], + &public_output, + &counts(), + 3, + &[], + &table_num_vars, + &config(), + ); + + let expected = monolithic_expected(po_len, 0, tnv_len); + residues_seen.insert(expected.body % FELT_BYTES); + + // ★ The property first, so a mutation reports the property. The two + // assertions under it are corroborating derivations, not the claim. + assert!( + rec.window.is_multiple_of(FELT_BYTES), + "statement at (po {po_len}, tnv {tnv_len}) ended at byte \ + {} = {} mod {FELT_BYTES}: whatever is absorbed next straddles two \ + field elements", + rec.window, + rec.window % FELT_BYTES, + ); + assert_eq!( + rec.window, expected.total, + "statement at (po {po_len}, tnv {tnv_len}) absorbed a different \ + number of BYTES than its field list implies", + ); + assert_eq!( + rec.absorbs.len(), + expected.calls, + "statement at (po {po_len}, tnv {tnv_len}) absorbed a different \ + number of times than its field list implies — a padding absorb that \ + is skipped when the pad is empty shows up here", + ); + + // And the thing that actually follows a statement: the roots. + let before = rec.absorbs.len(); + for _ in 0..4 { + rec.append_bytes(&[0u8; 32]); + } + assert_eq!( + misaligned(&rec.absorbs, before), + Vec::new(), + "a root absorbed after the statement at (po {po_len}, tnv \ + {tnv_len}) does not start on a field element boundary", + ); + } + + // ⚠ A sweep whose shapes all need the same pad would pass with the pad + // written as that constant. This says the sweep is not that sweep. + assert!( + residues_seen.len() > 1, + "every shape in the sweep has the same residue, so the sweep cannot \ + tell a computed pad from a constant one", + ); +} + +#[test] +fn the_monolithic_statement_ends_on_a_field_element_boundary() { + for &(po_len, ranges, tnv_len) in &[ + (0usize, 0usize, 34usize), + (5, 2, 34), + (33, 1, 35), + (4, 3, 7), + ] { + let public_output = vec![0x5Au8; po_len]; + let runtime_page_ranges: Vec = (0..ranges) + .map(|i| RuntimePageRange { + base: i as u64 * 4096, + count: 1, + }) + .collect(); + let table_num_vars = vec![22u8; tnv_len]; + + let mut rec = WindowRecorder::::new(); + crate::multilinear_prove::absorb( + &mut rec, + &[4u8; 32], + &public_output, + &counts(), + 2, + &runtime_page_ranges, + &table_num_vars, + &config(), + ); + + let expected = monolithic_expected(po_len, ranges, tnv_len); + assert!( + rec.window.is_multiple_of(FELT_BYTES), + "monolithic statement at (po {po_len}, ranges {ranges}, tnv \ + {tnv_len}) ended at byte {} = {} mod {FELT_BYTES}", + rec.window, + rec.window % FELT_BYTES, + ); + assert_eq!( + (rec.absorbs.len(), rec.window), + (expected.calls, expected.total), + "monolithic statement at (po {po_len}, ranges {ranges}, tnv {tnv_len})", + ); + } +} + +/// ★ The pad is a function of the WHOLE length, not of the fixed prefix. +/// +/// This is the arithmetic that killed the constant-pad design, written as an +/// assertion so it cannot be forgotten: at the shape this system runs, padding +/// the fixed prefix to a multiple of 8 leaves the roots at 2 mod 8. +#[test] +fn padding_only_the_fixed_prefix_would_not_align_the_roots() { + // The shape this system runs: no public output, 34 tables. + let (po_len, tnv_len) = (0usize, 34usize); + assert!( + monolithic_expected(po_len, 0, tnv_len) + .total + .is_multiple_of(FELT_BYTES) + ); + + // The fixed prefix is the body with every variable field empty, pinned as a + // count-free base plus the per-table-kind term: how many per-table counts a + // statement binds grows with the table set, while the rest of the prefix + // belongs to the encoding. + const COUNT_FREE_PREFIX: usize = 124; + let fixed = monolithic_expected(0, 0, 0).body; + assert_eq!( + fixed, + COUNT_FREE_PREFIX + NUM_TABLE_KINDS * U64, + "the statement's fixed prefix", + ); + let fixed_rounded = fixed + statement_padding(fixed); + + assert_eq!( + (fixed_rounded + po_len + tnv_len) % FELT_BYTES, + 2, + "a pad computed from the fixed prefix alone leaves the roots at 2 mod \ + {FELT_BYTES} at the shape we run: it would move every pinned constant \ + and align nothing", + ); +} + +// ------------------------------------------------------------------------- +// (2) A real prove, every absorb +// ------------------------------------------------------------------------- + +fn eq_table() -> ( + &'static ConcreteVmAir, + Vec>>, +) { + let ops = vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ]; + let columns: Vec>> = generate_eq_trace(&ops).columns_main(); + // Leaked so the layout borrows nothing from a temporary; this is a test + // binary and the leak is one AIR. + let air: &'static ConcreteVmAir = Box::leak(Box::new(create_eq_air( + &ProofOptions::default_test_options(), + ))); + (air, columns) +} + +fn committed( + air: &'static ConcreteVmAir, + columns: &[Vec>], + cfg: &ChainConfig, +) -> CommittedTables<'static, F, E, H> { + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + columns.len(), + columns[0].len().trailing_zeros() as usize, + Uniforms::default(), + ) + .expect("layout"); + let table = + CommittedTable::from_layout(layout, |col| columns[col as usize].clone()).expect("table"); + CommittedTables::<_, _, H>::commit(vec![table], cfg).expect("commit") +} + +/// The monolithic statement this fixture proves under, absorbed into `t`, and +/// how many absorbs it took. +fn seed_with_statement(t: &mut impl IsTranscript, table_num_vars: &[u8], po: &[u8]) -> usize { + crate::multilinear_prove::absorb( + t, + &[1u8; 32], + po, + &counts(), + 0, + &[], + table_num_vars, + &config(), + ); + monolithic_expected(po.len(), 0, table_num_vars.len()).calls +} + +/// ★★ THE MARK IS WIRED, checked where its absence is otherwise SILENT. +/// +/// Delete `mark_statement_end` from `absorb_statement_padding` and every other +/// test in this file still passes: the counter reads zero because it never +/// started counting, the recorder reads zero because the pad is correct, and +/// the two agree on a number that means nothing. The counting flag's own tests +/// in `crypto` drive transcripts by hand and never touch the prover's statement +/// path at all. +/// +/// So this asserts the CALL, not a count: the production statement absorb runs +/// against a recorder that notes whether the mark arrived. It needs no +/// `hash-metrics` and no process-global counter, so it runs in the ordinary +/// suite alongside everything else. +#[test] +fn the_statement_padding_tells_the_transcript_the_statement_ended() { + let mut rec = WindowRecorder::::new(); + assert!( + !rec.marked, + "a fresh transcript has not been told anything yet" + ); + let _ = seed_with_statement(&mut rec, &[1u8], &[]); + assert!( + rec.marked, + "the statement absorb finished without telling the transcript its statement \ + ended, so the alignment counter would count nothing for the rest of the \ + proof and read a zero that means only that it never started" + ); +} + +/// ⚠ The trace is generated ONCE and handed to both proves. +/// +/// `generate_eq_trace` emits its rows in `HashMap` iteration order, so two +/// calls produce two row orders, two commitments and two proofs — W1's finding +/// F2, and the reason the byte gate sorts its rows canonically. Regenerating it +/// per prove would make this comparison fail for a reason that has nothing to do +/// with the recorder, which is exactly what it did on the first run of this test. +fn prove_through_recorder( + air: &'static ConcreteVmAir, + columns: &[Vec>], + po: &[u8], +) -> (Vec, WindowRecorder) +where + WindowRecorder: IsTranscript, +{ + let cfg = config(); + let num_vars = columns[0].len().trailing_zeros() as u8; + let committed = committed::(air, columns, &cfg); + + let mut rec = WindowRecorder::::new(); + let statement_absorbs = seed_with_statement(&mut rec, &[num_vars], po); + assert_eq!(rec.absorbs.len(), statement_absorbs); + + let proof = multilinear_table::multi_prove(&committed, &cfg, &mut rec, None).expect("prove"); + let bytes = rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec(); + (bytes, rec) +} + +fn prove_through_production( + air: &'static ConcreteVmAir, + columns: &[Vec>], + po: &[u8], +) -> Vec { + let cfg = config(); + let num_vars = columns[0].len().trailing_zeros() as u8; + let committed = committed::(air, columns, &cfg); + + let mut t = DefaultTranscript::::new(&[]); + seed_with_statement(&mut t, &[num_vars], po); + + let proof = multilinear_table::multi_prove(&committed, &cfg, &mut t, None).expect("prove"); + rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec() +} + +/// ★★ The end-to-end claim, on the production stream. +/// +/// Run under both configurations. The offsets are a property of the CALL +/// SEQUENCE rather than of the sponge, so the two arms are expected to agree — +/// which is exactly why running both is worth its seconds: a disagreement would +/// mean one configuration absorbs something the other does not. +#[test] +fn every_absorb_of_a_real_prove_is_field_element_aligned() { + // Two public-output lengths, so the statement ends at a different length in + // each and the pad differs: at `[]` the pad is 3, at `[0, 0, 0]` it is 0 — + // the shape where the padding absorb is empty and must still be made. + let (air, columns) = eq_table(); + + for po in [vec![], vec![0u8, 0u8, 0u8]] { + for arm in ["keccak", "rpx"] { + let (recorded, rec, produced) = match arm { + "keccak" => { + let (b, r) = prove_through_recorder::(air, &columns, &po); + ( + b, + r.absorbs.clone(), + prove_through_production::(air, &columns, &po), + ) + } + _ => { + let (b, r) = prove_through_recorder::(air, &columns, &po); + ( + b, + r.absorbs.clone(), + prove_through_production::(air, &columns, &po), + ) + } + }; + + // The recorder is production's sponge with a tape attached, and this + // is what says so. A mirrored duplex buffer that drifted would draw + // different challenges and land here. + assert_eq!( + keccak_line(&recorded), + keccak_line(&produced), + "the {arm} arm's recorded prove is not the production prove \ + (po {} bytes): the recorder's mirror of the duplex buffer has \ + drifted, so its offsets describe some other transcript", + po.len(), + ); + + let statement_absorbs = monolithic_expected(po.len(), 0, 1).calls; + let bad = misaligned(&rec, statement_absorbs); + assert!( + bad.is_empty(), + "the {arm} arm absorbed {} of {} values at an offset that is not \ + a multiple of {FELT_BYTES} (po {} bytes). First few \ + (offset, len): {:?}", + bad.len(), + rec.len() - statement_absorbs, + po.len(), + &bad[..bad.len().min(8)], + ); + + // A read that can fail in the other direction: a prove that absorbed + // nothing after the statement would satisfy the emptiness above. + // "None misaligned" is a statement only beside the count it is out of. + assert!( + rec.len() > statement_absorbs + 8, + "the {arm} arm absorbed only {} values in total, so the \ + alignment assertion above is about almost nothing", + rec.len(), + ); + println!( + "ALIGNED {arm} po={} {} absorbs after the statement, 0 misaligned", + po.len(), + rec.len() - statement_absorbs, + ); + } + } +} + +fn keccak_line(bytes: &[u8]) -> String { + crypto::hash::platform_keccak::PlatformKeccak256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} diff --git a/prover/src/tests/statement_tests.rs b/prover/src/tests/statement_tests.rs index 830ecf7af..b14efc442 100644 --- a/prover/src/tests/statement_tests.rs +++ b/prover/src/tests/statement_tests.rs @@ -28,7 +28,10 @@ fn sample_counts() -> TableCounts { ecsm: 1, ecdas: 1, hint: 1, + copy: 1, + blake3: 1, is_wh: 1, + ecrec: 0, } } @@ -97,7 +100,10 @@ fn each_count_mut(counts: &mut TableCounts) -> Vec<(&'static str, &mut usize)> { ecsm, ecdas, hint, + copy, + blake3, is_wh, + ecrec, } = counts; vec![ ("cpu", cpu), @@ -119,7 +125,10 @@ fn each_count_mut(counts: &mut TableCounts) -> Vec<(&'static str, &mut usize)> { ("ecsm", ecsm), ("ecdas", ecdas), ("hint", hint), + ("copy", copy), + ("blake3", blake3), ("is_wh", is_wh), + ("ecrec", ecrec), ] } @@ -128,6 +137,21 @@ fn each_count_mut(counts: &mut TableCounts) -> Vec<(&'static str, &mut usize)> { /// destructured in `absorb_statement` and then left out of the array it /// absorbs compiles clean and changes nothing about the state, which is a /// prover-chosen number the verifier would no longer be bound to. +/// +/// ★ THE PROBE COUNT IS THE CONSTANT'S, NOT A LITERAL. It was `20` and read +/// 21, because the main-sync merge `892c7d1bc` (main's `c2ac5d546`, #977) took +/// `FIXED_TABLE_COUNT` 11 -> 5 and moved COMMIT into `TableCounts` alongside +/// the five accelerators already there — ✓ VERIFIED, `pub commit: usize` is +/// absent at `892c7d1bc^1` and present now. `each_count_mut` grew the field and +/// the literal did not. +/// +/// A bare literal here is the wrong shape twice over: it says nothing about +/// WHICH count is missing, and it is a second copy of a length +/// `statement::NUM_TABLE_KINDS` already holds — the same number +/// `table_count_values` returns as `[u64; NUM_TABLE_KINDS]` and the guest +/// absorbs as `statement_replay::NUM_TABLE_COUNTS`. Pinned against the +/// constant, a field added to `TableCounts` without a probe fails here, and one +/// added to the encoding without a field fails to compile. #[test] fn state_depends_on_every_table_count() { let baseline = state_after_absorb(b"elf", b"out", &sample_counts(), 1, &sample_ranges(), 7); @@ -136,7 +160,11 @@ fn state_depends_on_every_table_count() { .into_iter() .map(|(name, _)| name) .collect(); - assert_eq!(names.len(), 20, "every count must be probed"); + assert_eq!( + names.len(), + crate::statement::NUM_TABLE_KINDS, + "every count the statement encodes must be probed; probing {names:?}" + ); for name in names { let mut counts = sample_counts(); diff --git a/prover/src/tests/static_commitments_tests.rs b/prover/src/tests/static_commitments_tests.rs index 8441e7aae..db2a13671 100644 --- a/prover/src/tests/static_commitments_tests.rs +++ b/prover/src/tests/static_commitments_tests.rs @@ -22,7 +22,7 @@ use stark::proof::options::GoldilocksCubicProofOptions; -use crate::tables::{STATIC_BLOWUP_FACTORS, bitwise, cpu, keccak_rc, max_rows, page}; +use crate::tables::{STATIC_BLOWUP_FACTORS, bitwise, keccak_rc, page}; fn options_for(blowup: u8) -> stark::proof::options::ProofOptions { GoldilocksCubicProofOptions::with_blowup(blowup).expect("blowup must be a valid power of 2") @@ -297,60 +297,382 @@ fn bitwise_non_three_coset_recomputes_and_differs_from_static() { } } +// ========================================================================= +// One-row (S2) twins: a missing twin is a hard miss, never a recompute +// ========================================================================= +// +// Each static table ships a SECOND match table for the one-row leaf layout +// (`*_one_row`), generated by `compute_static_commitments --layout row` for +// `STATIC_BLOWUP_FACTORS_ONE_ROW`. The row-pair tests above are untouched; +// these pin the twins the same way, and pin that a missing twin is a hard +// miss (`None`, the prover's `PrecomputedCommitmentMissing`), never a +// recompute. + +use stark::leaf_layout::LeafLayout; + +use crate::tables::STATIC_BLOWUP_FACTORS_ONE_ROW; + #[test] -fn cpu_timestamp_static_matches_recompute_for_all_heights_and_blowups() { - for &blowup in STATIC_BLOWUP_FACTORS { +fn bitwise_one_row_static_matches_recompute() { + for &blowup in STATIC_BLOWUP_FACTORS_ONE_ROW { let options = options_for(blowup); - for log_height in cpu::MIN_LOG_HEIGHT..=max_rows::CPU.trailing_zeros() { - let height = 1 << log_height; - assert_eq!( - cpu::static_timestamp_commitment(blowup, log_height), - Some(cpu::compute_timestamp_commitment(&options, height)), - "CPU timestamp commitment drifted for blowup={blowup}, height=2^{log_height}; \ - see `static_timestamp_commitment` before regenerating", - ); - assert_eq!( - cpu::timestamp_commitment(&options, height), - cpu::compute_timestamp_commitment(&options, height), - ); - } + let recomputed = bitwise::compute_preprocessed_commitment_with(&options, LeafLayout::Row); + assert_eq!( + bitwise::static_commitment_one_row(blowup), + Some(recomputed), + "bitwise one-row commitment drifted for blowup={blowup}; regenerate via \ + `cargo run --bin compute_static_commitments --release -- --layout row`", + ); + assert_eq!( + bitwise::preprocessed_commitment_for(&options, LeafLayout::Row), + Some(recomputed) + ); + assert_ne!( + recomputed, + bitwise::preprocessed_commitment(&options), + "the two layouts commit different bytes" + ); } } #[test] -fn cpu_timestamp_has_no_static_value_outside_the_cpu_height_range() { - for &blowup in STATIC_BLOWUP_FACTORS { - assert_eq!(cpu::static_timestamp_commitment(blowup, 0), None); - assert_eq!(cpu::static_timestamp_commitment(blowup, 1), None); +fn keccak_rc_one_row_static_matches_recompute() { + for &blowup in STATIC_BLOWUP_FACTORS_ONE_ROW { + let options = options_for(blowup); + let recomputed = keccak_rc::compute_preprocessed_commitment_with(&options, LeafLayout::Row); + assert_eq!( + keccak_rc::static_commitment_one_row(blowup), + Some(recomputed), + "keccak_rc one-row commitment drifted for blowup={blowup}" + ); + assert_eq!( + keccak_rc::preprocessed_commitment_for(&options, LeafLayout::Row), + Some(recomputed) + ); + assert_ne!(recomputed, keccak_rc::preprocessed_commitment(&options)); + } +} + +#[test] +fn pages_one_row_static_match_recompute() { + let zero_page_config = page::PageConfig::zero_init(0); + for &blowup in STATIC_BLOWUP_FACTORS_ONE_ROW { + let options = options_for(blowup); + let zero = + page::compute_precomputed_commitment_with(&zero_page_config, &options, LeafLayout::Row); + assert_eq!( + page::static_zero_page_commitment_one_row(blowup), + Some(zero) + ); + assert_eq!( + page::zero_init_preprocessed_commitment_for(&options, LeafLayout::Row), + Some(zero) + ); + assert_ne!(zero, page::zero_init_preprocessed_commitment(&options)); + let private = page::compute_offset_only_commitment_with(&options, LeafLayout::Row); + assert_eq!( + page::static_private_page_commitment_one_row(blowup), + Some(private) + ); + assert_eq!( + page::private_page_preprocessed_commitment_for(&options, LeafLayout::Row), + Some(private) + ); + assert_ne!(private, zero, "OFFSET alone vs OFFSET+INIT"); + } +} + +/// Under one row, a blowup with no twin and a non-3 coset are +/// HARD MISSES — `None`, never the recompute the row-pair wrappers fall back +/// to (which would silently rebuild a 2^20-row BITWISE LDE and tree). The +/// row-pair layout keeps today's answers. +#[test] +fn a_missing_one_row_twin_is_a_hard_miss() { + for blowup in [2u8, 8, NON_STATIC_BLOWUP] { + assert!(!STATIC_BLOWUP_FACTORS_ONE_ROW.contains(&blowup)); + let options = options_for(blowup); + assert_eq!(bitwise::static_commitment_one_row(blowup), None); + assert_eq!( + bitwise::preprocessed_commitment_for(&options, LeafLayout::Row), + None + ); assert_eq!( - cpu::static_timestamp_commitment(blowup, max_rows::CPU.trailing_zeros() + 1), + keccak_rc::preprocessed_commitment_for(&options, LeafLayout::Row), + None + ); + assert_eq!( + page::zero_init_preprocessed_commitment_for(&options, LeafLayout::Row), + None + ); + assert_eq!( + page::private_page_preprocessed_commitment_for(&options, LeafLayout::Row), None ); } - assert_eq!(cpu::static_timestamp_commitment(NON_STATIC_BLOWUP, 4), None); + for &blowup in STATIC_BLOWUP_FACTORS_ONE_ROW { + let options = options_with_coset(blowup, NON_STANDARD_COSET); + assert_eq!( + bitwise::preprocessed_commitment_for(&options, LeafLayout::Row), + None + ); + assert_eq!( + keccak_rc::preprocessed_commitment_for(&options, LeafLayout::Row), + None + ); + assert_eq!( + page::zero_init_preprocessed_commitment_for(&options, LeafLayout::Row), + None + ); + assert_eq!( + page::private_page_preprocessed_commitment_for(&options, LeafLayout::Row), + None + ); + } + // Row pairs: unchanged (the static root at a shipped blowup). + let options = options_for(2); + assert_eq!( + keccak_rc::preprocessed_commitment_for(&options, LeafLayout::RowPair), + Some(keccak_rc::preprocessed_commitment(&options)) + ); } +/// The lazy commitment sources the AIRs are built with serve both layouts: +/// today's root for row pairs (unchanged) and the twin for one row. #[test] -fn cpu_timestamp_non_static_blowup_recomputes_via_fallback() { - let options = options_for(NON_STATIC_BLOWUP); +fn the_air_commitment_sources_serve_both_layouts() { + let options = options_for(4); + let k = keccak_rc::lazy_commitment(&options); assert_eq!( - cpu::timestamp_commitment(&options, 1 << 6), - cpu::compute_timestamp_commitment(&options, 1 << 6), + k.get_for(LeafLayout::RowPair), + Some(keccak_rc::preprocessed_commitment(&options)) + ); + assert_eq!( + k.get_for(LeafLayout::Row), + keccak_rc::static_commitment_one_row(4) + ); + let p = page::private_page_lazy_commitment(&options); + assert_eq!( + p.get_for(LeafLayout::Row), + page::static_private_page_commitment_one_row(4) + ); + // A data page: computed on demand, per layout. + let mut config = page::PageConfig::zero_init(0x1000); + config.init_values = Some((0..64u8).collect()); + let d = page::data_page_lazy_commitment(&config, &options, None); + assert_eq!( + d.get_for(LeafLayout::Row), + Some(page::compute_precomputed_commitment_with( + &config, + &options, + LeafLayout::Row + )) + ); + assert_eq!( + d.get_for(LeafLayout::RowPair), + Some(page::compute_precomputed_commitment(&config, &options)) + ); + // A SUPPLIED row-pair root is never handed out for the other layout. + let supplied = page::data_page_lazy_commitment(&config, &options, Some([7u8; 32])); + assert_eq!(supplied.get_for(LeafLayout::RowPair), Some([7u8; 32])); + assert_ne!(supplied.get_for(LeafLayout::Row), Some([7u8; 32])); + // REGISTER (program-dependent): both computed. + let init: Vec = (0..crate::tables::register::NUM_REGISTER_ADDRESSES as u32).collect(); + let fini: Vec = init.iter().map(|w| w ^ 0x5a5a).collect(); + let row_pair = crate::tables::register::compute_precomputed_commitment(&options, &init, &fini); + let r = crate::tables::register::lazy_commitment(&options, row_pair, &init, &fini); + assert_eq!(r.get_for(LeafLayout::RowPair), Some(row_pair)); + assert_eq!( + r.get_for(LeafLayout::Row), + Some( + crate::tables::register::compute_precomputed_commitment_with( + &options, + &init, + &fini, + LeafLayout::Row + ) + ) ); } +// ========================================================================= +// The Poseidon1 twins: the block family's roots at the key it asks for +// ========================================================================= +// +// `bitwise` and `keccak_rc` ship a THIRD match table, for a Poseidon1 +// configuration in row pairs (`static_commitment_p1`), generated by +// `compute_static_commitments --hash p1` for `STATIC_BLOWUP_FACTORS_P1`. The +// block's proof family asks for exactly that key, in its provers and in its +// verifiers; every other key under Poseidon1 is computed, as before. + +use crate::lfm::p1_commit::{P1StarkHash, P1StarkHashCapped}; +use crate::tables::STATIC_BLOWUP_FACTORS_P1; + #[test] -fn cpu_timestamp_non_three_coset_recomputes_and_differs_from_static() { - for &blowup in STATIC_BLOWUP_FACTORS { - let options = options_with_coset(blowup, NON_STANDARD_COSET); - let from_wrapper = cpu::timestamp_commitment(&options, 1 << 6); +fn bitwise_p1_static_matches_recompute() { + for &blowup in STATIC_BLOWUP_FACTORS_P1 { + let options = options_for(blowup); + let recomputed = bitwise::compute_preprocessed_commitment_under::( + &options, + LeafLayout::RowPair, + ); assert_eq!( - from_wrapper, - cpu::compute_timestamp_commitment(&options, 1 << 6) + bitwise::static_commitment_p1(blowup), + Some(recomputed), + "bitwise Poseidon1 commitment drifted for blowup={blowup}; regenerate via \ + `cargo run --bin compute_static_commitments --release -- --hash p1`", + ); + assert_eq!( + bitwise::preprocessed_root_under::(&options), + recomputed + ); + assert_eq!( + bitwise::preprocessed_commitment_under::(&options, LeafLayout::RowPair), + Some(recomputed) ); assert_ne!( - from_wrapper, - cpu::timestamp_commitment(&options_with_coset(blowup, STANDARD_COSET), 1 << 6), + recomputed, + bitwise::preprocessed_commitment(&options), + "Poseidon1's root is not the legacy pin's" + ); + } +} + +#[test] +fn keccak_rc_p1_static_matches_recompute() { + for &blowup in STATIC_BLOWUP_FACTORS_P1 { + let options = options_for(blowup); + let recomputed = keccak_rc::compute_preprocessed_commitment_under::( + &options, + LeafLayout::RowPair, + ); + assert_eq!( + keccak_rc::static_commitment_p1(blowup), + Some(recomputed), + "keccak_rc Poseidon1 commitment drifted for blowup={blowup}; regenerate via \ + `cargo run --bin compute_static_commitments --release -- --hash p1`", + ); + assert_eq!( + keccak_rc::preprocessed_root_under::(&options), + recomputed + ); + assert_ne!(recomputed, keccak_rc::preprocessed_commitment(&options)); + // A configuration's cap moves path nodes, never a root: the one + // constant is every cap's, computed and returned. + assert_eq!( + keccak_rc::compute_preprocessed_commitment_under::>( + &options, + LeafLayout::RowPair + ), + recomputed, + "the cap does not enter the root" + ); + assert_eq!( + keccak_rc::compute_preprocessed_commitment_under::>( + &options, + LeafLayout::RowPair + ), + recomputed, + "the cap does not enter the root" + ); + assert_eq!( + keccak_rc::preprocessed_root_under::>(&options), + recomputed ); } } + +/// ★ A blessed Poseidon1 root is returned at its own key alone: another +/// blowup, another coset and the one-row layout are computed, the legacy pin +/// keeps its own constants, and the two tables' roots are their own. Every +/// case on `keccak_rc` (32 rows), where a recompute is cheap; `bitwise` +/// dispatches in a function of its own, so one case off its key (the other +/// coset) is recomputed for it too. A configuration of neither family is +/// computed at Poseidon1's own key, in both functions. +#[test] +fn a_p1_static_is_taken_at_its_own_key_only() { + let blessed = keccak_rc::static_commitment_p1(4).expect("the block family's key"); + for blowup in [2u8, 8] { + assert!(!STATIC_BLOWUP_FACTORS_P1.contains(&blowup)); + assert_eq!(keccak_rc::static_commitment_p1(blowup), None); + assert_eq!(bitwise::static_commitment_p1(blowup), None); + let options = options_for(blowup); + let got = keccak_rc::preprocessed_root_under::(&options); + assert_eq!( + got, + keccak_rc::compute_preprocessed_commitment_under::( + &options, + LeafLayout::RowPair + ), + "blowup {blowup} has no Poseidon1 arm: computed" + ); + assert_ne!(got, blessed); + } + let coset7 = options_with_coset(4, NON_STANDARD_COSET); + let got = keccak_rc::preprocessed_root_under::(&coset7); + assert_eq!( + got, + keccak_rc::compute_preprocessed_commitment_under::( + &coset7, + LeafLayout::RowPair + ), + "coset {NON_STANDARD_COSET} takes the computation" + ); + assert_ne!(got, blessed); + let got = bitwise::preprocessed_root_under::(&coset7); + assert_eq!( + got, + bitwise::compute_preprocessed_commitment_under::(&coset7, LeafLayout::RowPair), + "bitwise: coset {NON_STANDARD_COSET} takes the computation" + ); + assert_ne!(Some(got), bitwise::static_commitment_p1(4)); + let options = options_for(4); + let one_row = + keccak_rc::preprocessed_commitment_under::(&options, LeafLayout::Row); + assert_eq!( + one_row, + Some(keccak_rc::compute_preprocessed_commitment_under::< + P1StarkHash, + >(&options, LeafLayout::Row)), + "one row is computed under its own layout" + ); + assert_ne!(one_row, Some(blessed)); + type Legacy = crate::hash_pin::BlockStarkHash; + assert_eq!( + keccak_rc::preprocessed_root_under::(&options), + keccak_rc::preprocessed_commitment(&options) + ); + assert_ne!( + keccak_rc::preprocessed_root_under::(&options), + blessed + ); + assert_eq!( + bitwise::preprocessed_root_under::(&options), + bitwise::preprocessed_commitment(&options) + ); + assert_ne!( + bitwise::static_commitment_p1(4), + keccak_rc::static_commitment_p1(4) + ); + // A third configuration: neither the legacy pin nor Poseidon1. Should the + // pin ever become this hash, name another here. + type Third = stark::config::KeccakStarkHash; + let third = ::COMMITMENT_HASH; + assert_ne!(third, crate::hash_pin::BLOCK_COMMITMENT_HASH); + assert_ne!(third, stark::config::CommitmentHash::Poseidon1); + let got = keccak_rc::preprocessed_root_under::(&options); + assert_eq!( + got, + keccak_rc::compute_preprocessed_commitment_under::(&options, LeafLayout::RowPair), + "a third configuration is computed at Poseidon1's key" + ); + assert_ne!(got, blessed); + assert_ne!(got, keccak_rc::preprocessed_commitment(&options)); + let got = bitwise::preprocessed_root_under::(&options); + assert_eq!( + got, + bitwise::compute_preprocessed_commitment_under::(&options, LeafLayout::RowPair), + "bitwise: a third configuration is computed at Poseidon1's key" + ); + assert_ne!(Some(got), bitwise::static_commitment_p1(4)); + assert_ne!(got, bitwise::preprocessed_commitment(&options)); +} diff --git a/prover/src/tests/trace_builder_tests.rs b/prover/src/tests/trace_builder_tests.rs index dfd4b0277..9bc6b693e 100644 --- a/prover/src/tests/trace_builder_tests.rs +++ b/prover/src/tests/trace_builder_tests.rs @@ -586,6 +586,7 @@ mod keccak_tests { state_addr: 0x1000, input, output, + sponge: None, }; let rop = KeccakRoundOperation { timestamp: 42, @@ -731,8 +732,11 @@ mod keccak_tests { fn test_keccak_bus_interaction_counts() { assert_eq!( keccak::bus_interactions().len(), - 84, - "KECCAK core: 1 ECALL + 1 MEMW read_addr + 25 MEMW lanes + 25 IS_WHH (25 IS_WH + 25 IS_HALF) + 1 BYTE_ALU alignment + 4 ARE_BYTES addr pairs + 1 Keccak send + 1 Keccak recv" + 370, + "KECCAK core, permute mode: 1 ECALL + 1 MEMW read_addr + 25 MEMW lanes + 25 IS_WHH \ + (25 IS_WH + 25 IS_HALF) + 1 BYTE_ALU alignment + 4 ARE_BYTES addr pairs + 1 Keccak \ + send + 1 Keccak recv (84); sponge mode: 1 ECALL + 3 register reads + 2 chain + 17 \ + lane reads + 119 byte reads + 136 XOR + 4 IS_HALF + 4 digest writes (286)" ); assert_eq!( keccak_rnd::bus_interactions().len(), @@ -753,8 +757,8 @@ mod keccak_tests { fn test_keccak_column_counts() { assert_eq!( core_cols::NUM_COLUMNS, - 486, - "KECCAK core columns (state_ptr DWordWHH)" + 907, + "KECCAK core columns (state_ptr DWordWHH; 486 + 421 for the sponge mode)" ); assert_eq!( rnd_cols::NUM_COLUMNS, @@ -769,8 +773,10 @@ mod keccak_tests { use stark::constraints::builder::ConstraintSet; assert_eq!( keccak::KeccakConstraints.meta().len(), - 52, - "KECCAK core: 25 ADD pairs + no-overflow + MuIsBit" + 791, + "KECCAK core: 25 ADD pairs + no-overflow + MuIsBit + 6 mode (58); sponge mode: \ + 136 mask bits + 119 FULL·PART + 135 prefix + 4 row rules + 136 MSG zero + \ + 201 first-row zero + 2 address halves (733)" ); assert_eq!( keccak_rnd::KeccakRndConstraints.meta().len(), @@ -846,15 +852,7 @@ fn test_from_image_and_logs_matches_from_elf_and_logs() { let logs = Executor::new(&program, vec![]).unwrap().run().unwrap().logs; let max_rows = MaxRowsConfig::default(); - let from_elf = Traces::from_elf_and_logs( - &program, - &logs, - &max_rows, - &[], - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .unwrap(); + let from_elf = Traces::from_elf_and_logs(&program, &logs, &max_rows, &[]).unwrap(); let image = build_initial_image(&program, &[]); let register_init = @@ -868,8 +866,6 @@ fn test_from_image_and_logs_matches_from_elf_and_logs() { &[], true, false, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ) .unwrap(); @@ -969,8 +965,6 @@ fn test_build_traces_for_all_epochs() { &[], i == last, false, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ) .unwrap_or_else(|e| panic!("epoch {i} (is_final={}) failed to build: {e:?}", i == last)); @@ -1024,8 +1018,6 @@ fn test_terminating_epoch_rejected_when_not_final() { &[], false, false, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, ); assert!( diff --git a/prover/src/tests/trace_test_helpers.rs b/prover/src/tests/trace_test_helpers.rs index 5544be69d..a8892483f 100644 --- a/prover/src/tests/trace_test_helpers.rs +++ b/prover/src/tests/trace_test_helpers.rs @@ -9,8 +9,6 @@ use executor::elf::Elf; use executor::vm::instruction::decoding::Instruction; use executor::vm::logs::Log; use executor::vm::memory::U64HashMap; -#[cfg(feature = "disk-spill")] -use stark::storage_mode::StorageMode; use stark::trace::TraceTable; use crate::Error; @@ -132,14 +130,7 @@ impl Traces { max_rows: &MaxRowsConfig, private_input: &[u8], ) -> Result { - let mut traces = Self::from_elf_and_logs( - elf, - logs, - max_rows, - private_input, - #[cfg(feature = "disk-spill")] - StorageMode::Ram, - )?; + let mut traces = Self::from_elf_and_logs(elf, logs, max_rows, private_input)?; traces.bitwise = trim_zero_rows(traces.bitwise); Ok(traces) } diff --git a/prover/src/tests/transcript_counts.rs b/prover/src/tests/transcript_counts.rs new file mode 100644 index 000000000..256c04d3c --- /dev/null +++ b/prover/src/tests/transcript_counts.rs @@ -0,0 +1,336 @@ +/// The Fiat-Shamir transcript's state machine, counted rather than run. +/// +/// Mirrors `DefaultTranscript` exactly: an absorb is one `Update::update` and +/// invalidates the duplex output buffer; a squeeze is one `finalize_reset` +/// PLUS the chain-advance `update` of the 32 reversed bytes, which is itself a +/// counted absorb; `state()` is a finalize on a clone that neither resets nor +/// re-absorbs. Four candidates come out of one squeeze, an extension element +/// takes three, a `sample_u64` takes one. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct TranscriptCounts { + pub absorb_calls: u64, + pub absorb_bytes: u64, + pub squeezes: u64, + pub state_finalizes: u64, +} + +impl TranscriptCounts { + /// Every finalize the transcript performs: one per squeeze, one per state. + pub fn finalizes(&self) -> u64 { + self.squeezes + self.state_finalizes + } + + /// `transcript_absorbs` as A2a counts it: the APPEND calls only. + /// + /// `absorb_calls` here includes the chaining re-absorb a squeeze performs, + /// which A2a attributes to squeezing rather than to an absorb anyone asked + /// for. Subtracting it is A2a's own non-vacuity identity read backwards: + /// `absorb_calls == transcript_absorbs + transcript_squeezes`. + pub fn transcript_absorbs(&self) -> u64 { + self.absorb_calls - self.squeezes + } + + /// `transcript_squeezes` as A2a counts it: one per `sample()`. + pub fn transcript_squeezes(&self) -> u64 { + self.squeezes + } +} + +struct Sim { + out_pos: usize, + c: TranscriptCounts, +} + +const SQUEEZE_LEN: usize = 32; + +impl Sim { + /// `DefaultTranscript::new(&[])` absorbs the empty seed, which is still one + /// `update` call of zero bytes. + fn new() -> Self { + let mut s = Self { + out_pos: SQUEEZE_LEN, + c: TranscriptCounts::default(), + }; + s.absorb(0); + s + } + fn absorb(&mut self, n: u64) { + self.out_pos = SQUEEZE_LEN; + self.c.absorb_calls += 1; + self.c.absorb_bytes += n; + } + fn squeeze(&mut self) { + self.c.squeezes += 1; + // `sample()` re-absorbs its own reversed output to advance the chain. + self.c.absorb_calls += 1; + self.c.absorb_bytes += 32; + self.out_pos = 0; + } + fn next_u64(&mut self) { + if self.out_pos + 8 > SQUEEZE_LEN { + self.squeeze(); + } + self.out_pos += 8; + } + /// A cubic-extension element: three base coordinates, each one candidate. + /// Exact unless a candidate lands at or above the modulus, which is one + /// extra draw with probability about 2^-32 per coordinate. + fn sample_ext(&mut self) { + for _ in 0..3 { + self.next_u64(); + } + } + fn sample_u64(&mut self) { + self.next_u64(); + } + fn state(&mut self) { + self.c.state_finalizes += 1; + } +} + +/// One table's transcript shape, read off the AIR and its layout. +#[derive(Clone, Copy, Debug)] +pub struct TableTranscriptShape { + /// Height in variables. + pub n: usize, + /// GKR input-layer variables, `n + ceil_log2(interactions)`. + pub m: usize, + /// The batched sumcheck's degree, `max(shape.degree() + 1, 2)`. + pub degree: usize, + /// Committed factors, whose values `claim_reduce` absorbs. + pub factors: usize, + /// Main columns, whose values `claim_reduce` absorbs at the end. + pub columns: usize, +} + +/// One commitment group: the stack it landed on and how many columns it holds. +#[derive(Clone, Copy, Debug)] +pub struct GroupTranscriptShape { + pub n_stack: usize, + pub num_polys: usize, + pub columns: usize, +} + +fn drive_table(s: &mut Sim, t: &TableTranscriptShape) { + // multilinear_table::verify + s.absorb(24); + s.absorb(24); // bus_output.0, .1 + // gkr::verify: m layers, layer i runs i degree-3 sumcheck rounds + for i in 0..t.m { + s.sample_ext(); // lambda + for _ in 0..i { + for _ in 0..3 { + s.absorb(24); + } + s.sample_ext(); + } + for _ in 0..4 { + s.absorb(24); // p_lo, p_hi, q_lo, q_hi + } + s.sample_ext(); // c + } + for _ in 0..t.n { + s.sample_ext(); // r + } + // constraint_argument::verify_core -> batch::verify + for _ in 0..3 { + s.absorb(24); // the three rule claims + } + s.sample_ext(); // the batching challenge + for _ in 0..t.n { + for _ in 0..t.degree { + s.absorb(24); + } + s.sample_ext(); + } + // claim_reduce::verify + for _ in 0..t.factors { + s.absorb(24); + } + s.sample_ext(); + for _ in 0..t.n { + for _ in 0..2 { + s.absorb(24); + } + s.sample_ext(); + } + for _ in 0..t.columns { + s.absorb(24); + } +} + +/// Today's uniform fold schedule, spelled independently of `ChainConfig`: `k` +/// per round, the remainder last. The LEGACY format's schedule; the closed +/// form below drives the config's own schedule, and +/// `the_uniform_schedule_is_the_legacy_configs` ties the two at the legacy +/// format. +fn schedule(num_vars: usize, k: usize) -> Vec { + let mut out = Vec::new(); + let mut left = num_vars; + while left > 0 { + let take = k.min(left); + out.push(take); + left -= take; + } + out +} + +/// One chain's transcript over the fold schedule `sch` — the config's own +/// (`ChainConfig::schedule`), so a non-uniform first fold (`whir_folds=first6`, +/// the production default) is priced as it is proved — and its grinds, of +/// which only those with bits touch the transcript (P2's query-only grind +/// spends one a round). +fn drive_chain(s: &mut Sim, sch: &[usize], queries: usize, grind: GrindBits) { + // `check_grind`: at zero bits it returns before its `state()` and before + // absorbing the nonce. + let spend = |s: &mut Sim, bits: u8| { + if bits > 0 { + s.state(); + s.absorb(8); + } + }; + let rounds = sch.len(); + for (r, &kr) in sch.iter().enumerate() { + spend(s, grind.folding); + for _ in 0..kr { + s.absorb(24); + s.absorb(24); // degree-2 round polynomial + s.sample_ext(); + } + if r + 1 < rounds { + s.absorb(32); // next_root + s.sample_ext(); // z0 + s.absorb(24); // y0 + spend(s, grind.ood); + s.sample_ext(); // gamma + spend(s, grind.query); + } else { + s.absorb(24); // final_value + spend(s, grind.query); + } + for _ in 0..queries { + s.sample_u64(); + } + } +} + +/// One whole proof's transcript: the statement, the roots, every table, every +/// commitment group's opening. +#[allow(clippy::too_many_arguments)] +pub fn transcript_counts( + statement_absorbs: &[u64], + tables: &[TableTranscriptShape], + groups: &[GroupTranscriptShape], + chain: &multilinear::whir_chain::ChainConfig, + queries: usize, + owed_probe: bool, +) -> TranscriptCounts { + let mut s = Sim::new(); + for &n in statement_absorbs { + s.absorb(n); + } + let roots: usize = groups.iter().map(|g| g.num_polys).sum(); + // `owed` runs the COMMIT bus's counterparty on a CLONE of the transcript: + // every root again, then z and alpha. The clone is thrown away, so it moves + // no challenge — but it is real hashing and a counter sees it. Only the + // epoch path does this; the cross-epoch proof has no commit bus to owe. + if owed_probe { + let mut probe = Sim { + out_pos: SQUEEZE_LEN, + c: TranscriptCounts::default(), + }; + for _ in 0..roots { + probe.absorb(32); + } + probe.sample_ext(); + probe.sample_ext(); + s.c.absorb_calls += probe.c.absorb_calls; + s.c.absorb_bytes += probe.c.absorb_bytes; + s.c.squeezes += probe.c.squeezes; + } + // multi_verify: every root, then z, alpha, beta. + for _ in 0..roots { + s.absorb(32); + } + for _ in 0..3 { + s.sample_ext(); + } + for t in tables { + drive_table(&mut s, t); + } + for g in groups { + for _ in 0..g.columns { + s.absorb(24); + } + s.sample_ext(); // the batching challenge + for _ in 0..g.num_polys { + drive_chain(&mut s, &chain.schedule(g.n_stack), queries, chain.grind); + } + } + s.c +} + +// ========================================================================= +// Driving it off the real AIRs +// ========================================================================= + +use multilinear::whir_chain::GrindBits; + +/// A grind with bits reads the transcript's state once (`check_grind`); one of +/// zero bits reads nothing. So a production chain reads it `R` times under the +/// default (`whir_grind=query`: the query grind alone) and `3R − 1` times under +/// the opt-out (`whir_grind=all`). +#[test] +fn a_chain_reads_the_state_once_a_spent_grind() { + let sch = [6, 4, 4, 4, 4, 3]; + let states = |grind| { + let mut s = Sim::new(); + drive_chain(&mut s, &sch, 112, grind); + s.c.state_finalizes + }; + assert_eq!(states(GrindBits::query_only(20)), 6); + assert_eq!(states(GrindBits::uniform(20)), 17); + assert_eq!(states(GrindBits::default()), 0); + for (whir_grind, want) in [ + (crate::zf_format::WhirGrind::Query, 6), + (crate::zf_format::WhirGrind::All, 17), + ] { + let config = crate::multilinear_prove::chain_config_under( + &crate::zf_format::ZfFormat { + whir_grind, + ..crate::zf_format::ZfFormat::DEFAULT + }, + &[(1, 25)], + ); + assert_eq!(config.schedule(25), sch); + assert_eq!(states(config.grind), want, "{whir_grind}"); + } +} + +/// The closed form drives `ChainConfig::schedule`; at the LEGACY format that +/// is exactly the uniform schedule spelled independently above, at every +/// height a chain reaches. Under the production default (first6) the two +/// differ, which is why the form no longer takes a fold width. +#[test] +fn the_uniform_schedule_is_the_legacy_configs() { + let legacy = crate::multilinear_prove::chain_config_under( + &crate::zf_format::ZfFormat::LEGACY, + &[(1, 25)], + ); + for n in 1..=32 { + assert_eq!( + schedule(n, legacy.log_folding), + legacy.schedule(n), + "n = {n}" + ); + } + let production = crate::multilinear_prove::chain_config_under( + &crate::zf_format::ZfFormat::DEFAULT, + &[(1, 25)], + ); + assert_ne!( + schedule(25, production.log_folding), + production.schedule(25), + "first6 is not the uniform walk" + ); +} diff --git a/prover/src/tests/verifier_pin_tests.rs b/prover/src/tests/verifier_pin_tests.rs new file mode 100644 index 000000000..d7bf8861b --- /dev/null +++ b/prover/src/tests/verifier_pin_tests.rs @@ -0,0 +1,426 @@ +//! What the block verifier holds as its own constants, from the prover crate +//! (whose tests CI runs): a WHIR chain's domain against the configured rate, +//! the query count every accepted format and statement derives, and BITWISE's +//! fixed height, which holds that count at the production one. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::mle::Mle; +use multilinear::whir::Domain; +use multilinear::whir_chain::{ + ArgueFormat, ChainConfig, ChainFormat, GrindBits, GroupFold, StackVars, commit, prove, verify, +}; +use multilinear::whir_hash::KeccakWhir; +use stark::proof::options::ProofOptions; + +use crate::block_whir::{self, BlockFormat, BlockOptions, Deviations, prove_block_whir_with}; +use crate::multilinear_prove::chain_config_under; +use crate::tables::MaxRowsConfig; +use crate::test_utils::asm_elf_bytes; +use crate::zf_format::ZfFormat; + +type FE = FieldElement; + +/// ★ A WHIR chain verified against a domain off its configuration's rate is +/// refused by name (`Error::DomainRateMismatch`), one size smaller or one +/// larger; against its own domain it verifies. +#[test] +fn a_chain_on_a_domain_off_its_rate_is_refused() { + let cfg = ChainConfig { + log_blowup: 1, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + format: ChainFormat::DEFAULT, + }; + let num_vars = 6; + let f = Mle::new( + (0..1u64 << num_vars) + .map(|i| FE::from(i.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 13)) + .collect(), + ) + .expect("a cube"); + let z: Vec = (0..num_vars).map(|i| FE::from(101 + i as u64)).collect(); + let y = f.evaluate(&z).expect("a point of the cube"); + let transcript = || DefaultTranscript::::new(b"a chain's domain"); + let (commitment, domain) = commit::(&f, &cfg, true).expect("commits"); + let proof = prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .expect("proves"); + let rate = num_vars + cfg.log_blowup; + let check = |domain: &Domain| { + verify::( + &proof, + &commitment.root(), + &z, + y, + domain, + &cfg, + &mut transcript(), + ) + }; + for log_size in [rate - 1, rate + 1] { + assert_eq!( + check(&Domain::::new(log_size).expect("a domain")), + Err(multilinear::Error::DomainRateMismatch { + expected: rate, + got: log_size + }), + "a domain of 2^{log_size} points" + ); + } + assert_eq!(check(&domain), Ok(()), "its own domain"); +} + +/// ★ The query count from the code's own rule, at the default format, by +/// tallest stack: 229 from 23 variables, 228 from 15 to 22, fewer below. A +/// one-column table of `n` variables has a stack of `n`, so `[(1, n)]` makes +/// the tallest stack exactly `n`; BITWISE (21 columns of 2^20 rows) stacks to +/// 25. +#[test] +fn the_query_count_by_tallest_stack_at_the_default() { + let q = |shapes: &[(usize, usize)]| chain_config_under(&ZfFormat::DEFAULT, shapes).num_queries; + assert_eq!(q(&[(21, 20)]), 229); + assert_eq!(q(&[(1, 23)]), 229); + assert_eq!(q(&[(1, 22)]), 228); + assert_eq!(q(&[(1, 15)]), 228); + assert_eq!(q(&[(1, 14)]), 227); + assert_eq!(q(&[(1, 7)]), 226); + assert_eq!(q(&[(1, 6)]), 224); + // BITWISE at 2^17: a stack of 22, the first height at 228. + assert_eq!(q(&[(21, 18)]), 229); + assert_eq!(q(&[(21, 17)]), 228); +} + +/// ★ Every WHIR format `ZfFormat::from_lookup` accepts that moves the query +/// count — blowup, grind bits, grind placement, fold schedule and stack — at +/// BITWISE's stack (25) and at 27: the count is the (blowup, grind bits) pair's +/// alone, 229 / 225 / 114 / 112. +#[test] +fn every_accepted_format_runs_its_query_count() { + for blowup in ["2", "4"] { + for bits in ["18", "20"] { + for grind in ["query", "all"] { + for folds in ["uniform4", "first5", "first6"] { + for stack in ["25", "26", "27"] { + let pairs = [ + (crate::zf_format::ENV_WHIR_BLOWUP, blowup), + (crate::zf_format::ENV_WHIR_GRIND_BITS, bits), + (crate::zf_format::ENV_WHIR_GRIND, grind), + (crate::zf_format::ENV_WHIR_FOLDS, folds), + (crate::zf_format::ENV_WHIR_STACK, stack), + ]; + let f = ZfFormat::from_lookup(|k| { + pairs + .iter() + .find(|(n, _)| *n == k) + .map(|(_, v)| v.to_string()) + }) + .expect("an accepted format"); + let at25 = chain_config_under(&f, &[(21, 20)]).num_queries; + let at27 = chain_config_under(&f, &[(21, 20), (64, 21)]).num_queries; + let want = match (blowup, bits) { + ("2", "18") => 229, + ("2", "20") => 225, + ("4", "18") => 114, + _ => 112, + }; + assert_eq!( + (at25, at27), + (want, want), + "blowup {blowup} bits {bits} grind {grind} folds {folds} stack {stack}" + ); + } + } + } + } + } +} + +fn small_format() -> BlockFormat { + let mut zf = ZfFormat::DEFAULT; + zf.whir_stack = StackVars::new(10).expect("a valid stack"); + BlockFormat { + zf, + group_polys: block_whir::BLOCK_GROUP_POLYS, + max_groups: block_whir::BLOCK_MAX_GROUPS, + prepared: true, + argue: ArgueFormat::PerTable, + group: GroupFold::Indexed, + leaf_scale: crate::lfm::whir_block::LeafScale::Fixed, + leaf_query: crate::lfm::whir_chain::LeafQuery::V1, + } +} + +fn small_options() -> BlockOptions { + BlockOptions { + max_rows: MaxRowsConfig::default(), + keccak_rnd_rows_log2: 16, + ecdas_rows_log2: block_whir::BLOCK_ECDAS_ROWS_LOG2, + keccak_rows_log2: block_whir::BLOCK_KECCAK_ROWS_LOG2, + ecsm_rows_log2: block_whir::BLOCK_ECSM_ROWS_LOG2, + copy_rows_log2: block_whir::BLOCK_COPY_ROWS_LOG2, + drop_levels: multilinear::whir_commit::TreeDrop::uniform(3), + window_log2: None, + drop_streamed_ops: false, + layout_workers: 0, + layout_ahead: Some(2), + narrow: stark::multilinear_block::Narrowing::Card { min_cells: 0 }, + upload_ahead: true, + memlog: false, + finish_keccak_rnd_chunks: true, + finish_cuts: true, + finish_stream: true, + rest_layout_bytes: Some(1 << 20), + rest_build_bytes: None, + pack_finished: true, + gpack: true, + regen: None, + phase_b_overlap: stark::multilinear_block::PhaseBOverlap::Off, + } +} + +/// ★ The frame holds BITWISE to its fixed 2^20 rows, so the query count stays +/// 229 by a rule of its own: the honest statement frames at 229; one with every +/// table at four rows, BITWISE included, or with BITWISE alone at 2^17 (which +/// would derive 226 and 228) is refused by name; and one with every other +/// table small but BITWISE at its height frames, still at 229. +#[test] +fn the_frame_refuses_a_bitwise_table_off_its_fixed_height() { + let elf = asm_elf_bytes("sub"); + let format = small_format(); + let opts = ProofOptions::default_test_options(); + let proof = prove_block_whir_with( + &elf, + &[], + &opts, + &format, + &small_options(), + &Deviations::default(), + ) + .expect("the block proves") + .0; + let honest = block_whir::block_frame(proof.statement(), &elf, &opts, &format).expect("frame"); + assert_eq!( + honest.airs.air_refs()[block_whir::BITWISE_TABLE].name(), + "BITWISE", + "BITWISE is the first table" + ); + assert_eq!(honest.shapes[block_whir::BITWISE_TABLE], (21, 20)); + assert_eq!(honest.config.num_queries, 229); + + let frame_at = |heights: &[u8]| { + block_whir::block_frame( + block_whir::BlockStatement { + table_num_vars: heights, + ..proof.statement() + }, + &elf, + &opts, + &format, + ) + }; + let refused_by_name = |heights: &[u8]| match frame_at(heights) { + Err(crate::Error::InvalidTableCounts(why)) => why.starts_with("BITWISE states"), + _ => false, + }; + assert!( + refused_by_name(&vec![2u8; proof.table_num_vars.len()]), + "every table at four rows" + ); + let small: Vec = proof.table_num_vars.iter().map(|&h| h.min(12)).collect(); + let mut bitwise_17 = small.clone(); + bitwise_17[block_whir::BITWISE_TABLE] = 17; + assert!(refused_by_name(&bitwise_17), "BITWISE alone at 2^17"); + let mut bitwise_20 = small; + bitwise_20[block_whir::BITWISE_TABLE] = 20; + let frame = frame_at(&bitwise_20).expect("BITWISE at its height frames"); + assert_eq!(frame.config.num_queries, 229, "BITWISE holds the count"); +} + +/// Format knobs a verifier's process could carry, each off the default: the +/// WHIR blowup, grind bits and folds, the leaves' query check and scale, and +/// the leaves' share form. +const OFF_DEFAULT: [(&str, &str); 7] = [ + (crate::zf_format::ENV_WHIR_BLOWUP, "4"), + (crate::zf_format::ENV_WHIR_GRIND_BITS, "20"), + (crate::zf_format::ENV_WHIR_FOLDS, "uniform4"), + (crate::lfm::whir_block::LEAF_QUERY_ENV, "v1"), + (crate::lfm::whir_block::LEAF_DOUBLE_FROM_ENV, "off"), + ("LFM_WHIR_SHARE_INVERSE", "0"), + ("BLOCK_WHIR_ARGUE_PACK", "off"), +]; + +/// ★ The block verifier reads no format from its environment. In a process of +/// its own whose environment sets every knob in [`OFF_DEFAULT`] +/// ([`the_pinned_presets_hold_under_an_environment_off_the_default`]), the +/// prover's format follows the environment while the verifier's presets stay +/// the pinned constants. So a verifier left with `LAMBDA_VM_ZF_WHIR_BLOWUP=4` +/// still checks a proof at the production format: it verifies a production +/// proof and refuses one made at blowup 4, whose statement binds the other +/// rate. +#[test] +fn the_block_verifier_reads_no_format_from_its_environment() { + let name = + "tests::verifier_pin_tests::the_pinned_presets_hold_under_an_environment_off_the_default"; + let out = std::process::Command::new(std::env::current_exe().expect("the test binary")) + .args([ + "--exact", + name, + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .envs(OFF_DEFAULT) + .output() + .expect("the test binary runs"); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + out.status.success() && stdout.contains("1 passed"), + "the child under the knobs:\n{stdout}\n{stderr}" + ); +} + +/// Run by [`the_block_verifier_reads_no_format_from_its_environment`] in a +/// process whose environment is [`OFF_DEFAULT`]; refuses to run anywhere else. +#[test] +#[ignore = "run in a process of its own by the_block_verifier_reads_no_format_from_its_environment"] +fn the_pinned_presets_hold_under_an_environment_off_the_default() { + use crate::lfm::whir_block::{ + LeafScale, pinned_verifier_presets, share_inverse, verifier_presets_at, + }; + use crate::lfm::whir_chain::LeafQuery; + use crate::zf_format::WhirBlowup; + use multilinear::whir_chain::LeafOrder; + assert!( + OFF_DEFAULT + .iter() + .all(|(k, v)| std::env::var(k).as_deref() == Ok(*v)), + "run by its parent, under the knobs" + ); + // The prover's format is the environment's. + let production = BlockFormat::production(); + assert_eq!(production.zf.whir_blowup, 4); + assert_eq!(production.zf.whir_grind_bits, 20); + assert_ne!(production.zf.whir_folds, ZfFormat::DEFAULT.whir_folds); + assert_eq!(production.leaf_query, LeafQuery::V1); + assert_eq!(production.leaf_scale, LeafScale::Fixed); + assert!(!share_inverse()); + let harness = crate::lfm::whir_block_tree::WhirTreeConfig::from_env().expect("the knobs read"); + assert_eq!( + harness.format.argue, + multilinear::whir_chain::ArgueFormat::Batched { + bin_log_cells: 26, + pack: multilinear::whir_chain::GkrPack::Off, + }, + "the harness's argue follows its packing knob" + ); + + // The verifier's is not. + let debug = |o: &ProofOptions| format!("{o:?}"); + let pinned = pinned_verifier_presets(GroupFold::Indexed); + assert_eq!(pinned.format.zf, ZfFormat::DEFAULT); + assert_eq!(pinned.format.leaf_query, LeafQuery::Lean); + assert_eq!(pinned.format.leaf_scale, LeafScale::DEFAULT); + assert_eq!(pinned.format.group, GroupFold::Indexed); + assert_eq!( + pinned.format.argue, + multilinear::whir_chain::ArgueFormat::BATCHED, + "the verifier's packing is its named constant, not the environment's" + ); + assert!(pinned.share_inverse); + assert_eq!( + debug(&pinned.base_options), + debug(&crate::lfm::proof::block_base_options_under( + &ZfFormat::DEFAULT + )) + ); + assert_eq!( + debug(&pinned.wrap_options), + debug(&crate::lfm::proof::aggregation_wrap_options_under( + &ZfFormat::DEFAULT + )) + ); + + // The named-choice form sets its five and pins the rest, folds and grind + // included. + let at = verifier_presets_at( + GroupFold::Indexed, + WhirBlowup::Four, + LeafOrder::Natural, + LeafScale::Fixed, + LeafQuery::V1, + multilinear::whir_chain::GkrPack::PRODUCTION, + ) + .expect("named choices"); + assert_eq!( + at.format.zf, + ZfFormat { + whir_blowup: 4, + whir_leaves: LeafOrder::Natural, + ..ZfFormat::DEFAULT + } + ); + assert_eq!( + (at.format.leaf_scale, at.format.leaf_query), + (LeafScale::Fixed, LeafQuery::V1) + ); + assert_eq!( + at.format.argue, + multilinear::whir_chain::ArgueFormat::BATCHED, + "the packing named PRODUCTION is the production argue" + ); + assert!(at.share_inverse); + + // The concrete case: a blowup-4 proof is not taken for a production block + // proof. Over one statement (the median mix's), the plan the verifier + // derives is the pinned format's, and its block id, which a top must claim + // (`top_claims`), is neither the one a blowup-4 prover's plan publishes + // (the pinned format at blowup 4) nor the one the environment's format + // would give; so the top of a tree proved at blowup 4 claims another + // block, and the verifier refuses it. + use crate::lfm::whir_block::{BLOCK_FAN_IN, WhirBlockPlan, verifier_plan}; + let elf: &[u8] = include_bytes!("../lfm/testdata/poc_rodata_commit.elf"); + let pinned_format = pinned.format; + let statement = crate::lfm::whir_p1_census_tests::median_statement( + elf, + &pinned.base_options, + &pinned_format, + ); + let verifier = verifier_plan(elf, statement.view(), &pinned).expect("the verifier's plan"); + let prover_at = |format: &BlockFormat| { + WhirBlockPlan::derive_with( + elf, + &crate::lfm::proof::block_base_options_under(&format.zf), + format, + statement.view(), + None, + BLOCK_FAN_IN, + None, + ) + .expect("the prover's plan") + }; + let blowup_4 = BlockFormat { + zf: ZfFormat { + whir_blowup: 4, + ..ZfFormat::DEFAULT + }, + ..pinned_format + }; + assert_ne!( + verifier.id(), + prover_at(&blowup_4).id(), + "a blowup-4 tree claims another block" + ); + assert_ne!( + verifier.id(), + prover_at(&production).id(), + "the environment's format claims another block" + ); + assert_eq!( + verifier.id(), + prover_at(&pinned_format).id(), + "a production prover's plan claims the verifier's block" + ); +} diff --git a/prover/src/tests/whir_hash_tests.rs b/prover/src/tests/whir_hash_tests.rs new file mode 100644 index 000000000..2481e62ea --- /dev/null +++ b/prover/src/tests/whir_hash_tests.rs @@ -0,0 +1,387 @@ +//! ★★ The hash seam, end to end, on a real VM table — the four propositions the +//! byte gate is made of, in the one place they can all be checked in process. +//! +//! This is the miniature of the A/B the box will run, and it is deliberately +//! shaped the same way: **one process, the trace built once, proved once per +//! hash.** Two processes could not be compared, because six of this VM's trace +//! generators lay their rows out in `HashMap` iteration order +//! (`prover/src/tables/eq.rs:128` and five siblings), so the same program gives +//! different traces run to run for reasons that have nothing to do with the +//! hash. +//! +//! | | proposition | why it can fail | +//! |---|---|---| +//! | **G1** | the keccak arm is the proof PR #988 produces | a threading mistake would move a byte | +//! | **G2** | the RPX arm proves and verifies, and its proof DIFFERS | a seam that silently kept hashing keccak would pass a verify and fail this | +//! | **G3** | prove under one hash, verify under the other ⇒ REJECTED | this is what makes G2 non-vacuous: prover and verifier agree on a wrong hash too | +//! | **B3** | the two arms serialize to the SAME LENGTH, to the byte | a digest-width or field change would move it; a hash change must not | +//! +//! G3 is the load-bearing one. "Prover and verifier agree" is worth nothing on +//! its own — they would agree on a hash that returned its input. What says the +//! hash is really in the proof is that a verifier told to expect the other one +//! rejects, and that the KATs in `crypto::hash::rpx::tests` pin the digest to +//! numbers this repository did not produce. + +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, MultiProof, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::tables::eq::{EqOperation, generate_eq_trace}; +use crate::test_utils::{ConcreteVmAir, create_eq_air}; + +type Proof = MultiProof; +type Columns = Vec>>; + +/// Grinding ON: the grind is one of the three hash consumers the seam names, so +/// an arm that left it on keccak would be a half-flip and this fixture has to +/// be able to see it. Four bits, so the search costs nothing. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::uniform(4), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + } +} + +fn fixture_columns() -> Columns { + let ops = vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ]; + generate_eq_trace(&ops).columns_main() +} + +fn air() -> ConcreteVmAir> { + create_eq_air(&ProofOptions::default_test_options()) +} + +fn layout(columns: &Columns) -> TableLayout<'static, Fp, Ext> { + // The AIR is rebuilt per call so the layout borrows nothing from a temporary. + let a = Box::leak(Box::new(air())); + TableLayout::::new( + a.constraint_program(), + a.constraints_meta(), + a.bus_interactions(), + columns.len(), + columns[0].len().trailing_zeros() as usize, + Uniforms::default(), + ) + .expect("layout") +} + +/// Prove the fixed trace under `H`. +fn prove(columns: &Columns) -> Proof { + let table = CommittedTable::from_layout(layout(columns), |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + let mut transcript = DefaultTranscript::::new(b"whir-hash-seam"); + multilinear_table::multi_prove(&committed, &config(), &mut transcript, None).expect("prove") +} + +/// Verify `proof` under `H`. Returns the verifier's verdict rather than +/// unwrapping, because half these calls are supposed to fail. +fn verify(proof: &Proof, columns: &Columns) -> Result<(), multilinear::Error> { + let table = CommittedTable::from_layout(layout(columns), |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + + let owed = multilinear_table::contribution(&proof.tables[0].bus_output) + .ok_or(multilinear::Error::BusImbalance)?; + let verifier_layout = layout(columns); + let statement = verifier_layout.statement(); + let mut transcript = DefaultTranscript::::new(b"whir-hash-seam"); + multilinear_table::multi_verify::<_, _, _, H>( + proof, + &[statement], + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &owed, + &config(), + &mut transcript, + None, + ) +} + +fn serialized(proof: &Proof) -> Vec { + rkyv::to_bytes::(proof) + .expect("serialize") + .to_vec() +} + +/// (G2, first half) The RPX arm proves and verifies. On its own this says very +/// little — see the module header — which is why it is one line of four. +#[test] +fn the_rpx_arm_proves_and_verifies() { + let columns = fixture_columns(); + let proof = prove::(&columns); + verify::(&proof, &columns).expect("an RPX proof must verify under RPX"); +} + +/// The keccak arm still does, unchanged. +#[test] +fn the_keccak_arm_proves_and_verifies() { + let columns = fixture_columns(); + let proof = prove::(&columns); + verify::(&proof, &columns).expect("a keccak proof must verify under keccak"); +} + +/// (G2, second half) ★ The two arms produce DIFFERENT proofs. +/// +/// Checked at the root, not at the whole blob: a root is the one field whose +/// difference can only come from the Merkle hash, so this distinguishes "the +/// hash moved" from "some challenge moved". +#[test] +fn the_two_hashes_produce_different_roots() { + let columns = fixture_columns(); + let keccak = prove::(&columns); + let rpx = prove::(&columns); + + assert_eq!(keccak.roots.len(), 1, "the fixture commits one group"); + assert_ne!( + keccak.roots, rpx.roots, + "a seam that kept hashing keccak under RpxWhir would land here" + ); +} + +/// (G3) ★★ **The arm that must FAIL.** A proof made under one hash must be +/// rejected by a verifier expecting the other, in both directions. +#[test] +fn a_proof_made_under_one_hash_is_rejected_under_the_other() { + let columns = fixture_columns(); + + let keccak = prove::(&columns); + assert!( + verify::(&keccak, &columns).is_err(), + "an RPX verifier accepted a keccak proof" + ); + + let rpx = prove::(&columns); + assert!( + verify::(&rpx, &columns).is_err(), + "a keccak verifier accepted an RPX proof" + ); +} + +/// (B3) ★ The serialized length is EQUAL to the byte. +/// +/// The sharper of the two byte statements: the digest is 32 bytes under either +/// hash and no proof struct gains a field, so the length is not allowed to move +/// even though every byte inside it does. A difference here is a defect in the +/// seam, not a property of the hash. +#[test] +fn the_two_hashes_serialize_to_the_same_length() { + let columns = fixture_columns(); + let keccak = serialized(&prove::(&columns)); + let rpx = serialized(&prove::(&columns)); + + assert_eq!( + keccak.len(), + rpx.len(), + "a hash swap must not be a proof-format change" + ); + assert_ne!(keccak, rpx, "…but the bytes themselves must differ"); +} + +/// ✓ The configurations name themselves, and differently — the string the +/// banner prints and the KATs are filed under. +#[test] +fn the_two_configurations_have_distinct_names() { + assert_eq!(KeccakWhir::NAME, "keccak256"); + assert_eq!(RpxWhir::NAME, "rpx256"); + assert_ne!(KeccakWhir::NAME, RpxWhir::NAME); +} + +/// ★ The GRIND follows the configuration, not a default. +/// +/// The grind is the seam's third consumer and the easiest to leave behind, +/// because it is reached through a free function rather than through a type. A +/// nonce valid under one configuration's digest is invalid under the other's +/// with overwhelming probability, so this is a direct read of which hash the +/// proof-of-work actually ran on. +#[test] +fn the_grind_follows_the_configuration() { + use crypto::grinding::{generate_nonce_smallest, is_valid_nonce}; + use multilinear::whir_hash::GrindingDigest; + + let seed = [7u8; 32]; + let factor = 12u8; + + let k = generate_nonce_smallest::>(&seed, factor).expect("nonce"); + let r = generate_nonce_smallest::>(&seed, factor).expect("nonce"); + + assert!(is_valid_nonce::>( + &seed, k, factor + )); + assert!(is_valid_nonce::>(&seed, r, factor)); + assert_ne!( + k, r, + "the same seed must not grind to the same nonce under two different hashes" + ); + assert!( + !is_valid_nonce::>(&seed, k, factor), + "keccak's nonce must not satisfy RPX's predicate" + ); + assert!( + !is_valid_nonce::>(&seed, r, factor), + "RPX's nonce must not satisfy keccak's predicate" + ); +} + +/// ★ The two transcript TYPES draw different challenges. +/// +/// ⚠ RENAMED. This was called `the_transcript_follows_the_configuration`, which +/// is a claim about the PROVER — and this body never mentions the prover. It +/// constructs both transcript types itself and compares them, so it was true +/// for the whole period in which no WHIR call site built an RPX transcript at +/// all, and it would have stayed true if none ever did. +/// +/// What it does check is worth keeping: that the two configurations are not +/// accidentally the same sponge. The claim its old name made is now checked two +/// ways — by the compiler, via the `HasTranscriptHash` bound on `multi_prove` +/// and `multi_verify`, which makes a mismatched transcript unspellable rather +/// than merely untested; and at runtime by +/// `prover/tests/whir_transcript_configuration.rs`, which runs a real prove and +/// reads the Fiat-Shamir counters afterwards. +#[test] +fn the_two_transcript_types_draw_different_challenges() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + use crypto::fiat_shamir::transcript_hash::TranscriptHash; + + type KeccakT = DefaultTranscript::Transcript>; + type RpxT = DefaultTranscript::Transcript>; + + let mut k = KeccakT::new(b"same-seed"); + let mut r = RpxT::new(b"same-seed"); + k.append_bytes(b"same-absorbed-bytes"); + r.append_bytes(b"same-absorbed-bytes"); + + assert_ne!(k.state(), r.state(), "two hashes, two sponge states"); + assert_ne!( + k.sample_field_element(), + r.sample_field_element(), + "two hashes, two challenge streams" + ); + assert_eq!(::Transcript::NAME, "keccak256"); + assert_eq!(::Transcript::NAME, "rpx256"); +} + +// =========================================================================== +// ZisK's Poseidon1 (D-WHIR-P1 S1): the same seam at a third hash, whose trees +// are 4-ary and whose sponge is a field sponge, through each configuration's +// own `WhirHash::sponge`. +// =========================================================================== + +/// The configuration's own sponge with the fixture's tag absorbed. +fn sponge() -> H::Sponge { + use crypto::fiat_shamir::is_transcript::IsTranscript; + let mut t = H::sponge(); + t.append_bytes(b"whir-hash-seam"); + t +} + +fn prove_sponged(columns: &Columns) -> Proof { + let table = CommittedTable::from_layout(layout(columns), |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + multilinear_table::multi_prove(&committed, &config(), &mut sponge::(), None).expect("prove") +} + +fn verify_sponged(proof: &Proof, columns: &Columns) -> Result<(), multilinear::Error> { + let table = CommittedTable::from_layout(layout(columns), |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + let owed = multilinear_table::contribution(&proof.tables[0].bus_output) + .ok_or(multilinear::Error::BusImbalance)?; + let verifier_layout = layout(columns); + let statement = verifier_layout.statement(); + multilinear_table::multi_verify::<_, _, _, H>( + proof, + &[statement], + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &owed, + &config(), + &mut sponge::(), + None, + ) +} + +/// The RPX and keccak sponges ARE the transcripts the block always built +/// (`DefaultTranscript::new(&[])`): equal states and challenges after the same +/// absorbs, so routing the block through `WhirHash::sponge` moves no byte. +#[test] +fn the_rpx_and_keccak_sponges_are_the_default_transcripts() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + fn same() { + let mut a = H::sponge(); + let mut b = DefaultTranscript::::new(&[]); + for t in [&mut a as &mut dyn IsTranscript, &mut b] { + t.append_bytes(b"statement"); + t.append_field_element(&FieldElement::::from(7u64)); + } + assert_eq!(a.state(), b.state(), "{}", H::NAME); + assert_eq!(a.sample_field_element(), b.sample_field_element()); + assert_eq!(a.sample_u64(1 << 20), b.sample_u64(1 << 20)); + } + same::(); + same::(); +} + +/// (G2 at P1) The Poseidon1 arm proves and verifies on the real table, and its +/// roots and bytes differ from RPX's under the same fixture. +#[test] +fn the_p1_arm_proves_and_verifies_and_differs_from_rpx() { + use multilinear::whir_hash::P1Whir; + let columns = fixture_columns(); + let p1 = prove_sponged::(&columns); + verify_sponged::(&p1, &columns).expect("a P1 proof must verify under P1"); + let rpx = prove_sponged::(&columns); + verify_sponged::(&rpx, &columns).expect("an RPX proof must verify under RPX"); + assert_ne!(p1.roots, rpx.roots, "the P1 seam must hash with P1"); + assert_ne!(serialized(&p1), serialized(&rpx)); +} + +/// (G3 at P1) ★★ Prove under one, verify under the other: rejected, both ways, +/// and against keccak too. +#[test] +fn a_p1_proof_and_an_rpx_proof_do_not_cross_verify() { + use multilinear::whir_hash::P1Whir; + let columns = fixture_columns(); + let p1 = prove_sponged::(&columns); + assert!(verify_sponged::(&p1, &columns).is_err()); + assert!(verify_sponged::(&p1, &columns).is_err()); + let rpx = prove_sponged::(&columns); + assert!(verify_sponged::(&rpx, &columns).is_err()); +} + +/// The P1 grind is its own: its nonce fails RPX's predicate and the reverse. +#[test] +fn the_p1_grind_follows_its_configuration() { + use crypto::grinding::{generate_nonce_smallest, is_valid_nonce}; + use multilinear::whir_hash::{GrindingDigest, P1Whir}; + let seed = [7u8; 32]; + let factor = 12u8; + let p = generate_nonce_smallest::>(&seed, factor).expect("nonce"); + let r = generate_nonce_smallest::>(&seed, factor).expect("nonce"); + assert!(is_valid_nonce::>(&seed, p, factor)); + assert_ne!(p, r); + assert!(!is_valid_nonce::>(&seed, p, factor)); + assert!(!is_valid_nonce::>(&seed, r, factor)); + assert_eq!(P1Whir::NAME, "poseidon1-w16"); +} diff --git a/prover/src/tests/whir_identity_tests.rs b/prover/src/tests/whir_identity_tests.rs new file mode 100644 index 000000000..ff3933134 --- /dev/null +++ b/prover/src/tests/whir_identity_tests.rs @@ -0,0 +1,448 @@ +//! ★ The byte gate for the WHIR hash seam, and the proof that it is a gate. +//! +//! `whir_identity::identity_line` is the instrument the coordinator diffs +//! across arms, so what it can and cannot see has to be pinned rather than +//! described. Five propositions, each constructed rather than observed: +//! +//! 1. the line is STABLE when the proof is — two independent runs of the same +//! prover on the same input give the same line; +//! 2. the line is SENSITIVE — moving one byte of one Merkle root moves it, and +//! so does moving one opened codeword value; +//! 3. the line is BLIND TO NONCES, and to nothing else — changing only a +//! grinding nonce leaves it alone, which is exactly the exclusion the +//! instrument claims; +//! 4. the SERIALIZED LENGTH is unaffected by the nonce normalisation, so a +//! length comparison between two arms measures the proof and not the +//! instrument; +//! 5. ⚠ **the normalisation does NOT make a GROUND proof reproducible** — the +//! thing this gate was first designed to do, which it cannot. +//! +//! Proposition 2 is what makes 1 and 3 worth anything: a digest that could not +//! change would satisfy 1 and 3 vacuously. +//! +//! # ⚠ The correction proposition 5 records +//! +//! The gate was pre-registered as "hash the proof with every grinding nonce +//! zeroed", on the reasoning that the nonce is the only nondeterministic field. +//! That reasoning is wrong, and the test below is what found it: the nonce is +//! **absorbed into the transcript** (`multilinear::whir_chain::grind`), so every +//! challenge drawn after the first grind depends on which valid nonce the +//! search returned. Two honest runs diverge in every root and every opening +//! from that point on, and the divergence is not in the nonce fields, so +//! zeroing them cannot remove it. +//! +//! What a byte gate needs instead is a search that returns the SAME valid nonce +//! — `crypto::grinding::generate_nonce_smallest`, reached in production by +//! setting `LAMBDA_VM_DETERMINISTIC_GRIND`. The nonce normalisation is kept +//! anyway, because it costs nothing and because it makes the line insensitive +//! to the one field that still legitimately varies between a CPU arm and a +//! device arm. +//! +//! # ⚠ A SECOND cause of irreproducibility, found the same way +//! +//! Even with no proof of work at all, two `generate_eq_trace` calls on the same +//! operations produce DIFFERENT TRACES: the generator deduplicates through a +//! `std::collections::HashMap` and lays the rows out in iteration order +//! (`prover/src/tables/eq.rs:128`), which `RandomState` randomises per map. +//! Five sibling generators do the same — BYTEWISE, BRANCH, DVRM, LT and MUL. +//! Row order is free to the argument (the bus is a multiset), so this is not a +//! soundness defect, but it means **a proof of the same program is not +//! byte-reproducible across processes for reasons that have nothing to do with +//! the hash**. +//! +//! So the fixture below builds its trace ONCE and proves it twice. What these +//! tests pin is the property the seam is responsible for — the WHIR prove path +//! is a function of its input — and not a property of the trace builders, which +//! is someone else's to fix. + +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::KeccakWhir; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, MultiProof, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::tables::eq::{EqOperation, generate_eq_trace}; +use crate::test_utils::{ConcreteVmAir, create_eq_air}; +use crate::whir_identity::{identity_line, serialized_len}; + +type Proof = MultiProof; + +/// Grinding ON, deliberately: the nonces are what the instrument normalises +/// away, so a fixture without them could not exercise propositions 3 and 5. +/// Four bits, so the search costs nothing. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::uniform(4), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + } +} + +/// The same posture with no proof of work — the deterministic control. +fn config_unground() -> ChainConfig { + ChainConfig { + grind: GrindBits::default(), + ..config() + } +} + +fn eq_operations() -> Vec { + vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ] +} + +/// The trace, built ONCE — see the header: the generator's row order follows a +/// `HashMap`, so building it per call would make every comparison below a test +/// of the trace builder instead of a test of the prover. +fn fixture_columns() -> Vec>> { + generate_eq_trace(&eq_operations()).columns_main() +} + +/// One real table, proved end to end — the smallest thing that has roots, +/// openings and nonces all at once. +fn prove_once(seed: &[u8], columns: &[Vec>]) -> Proof { + prove_with(seed, columns, &config()) +} + +fn prove_once_unground(seed: &[u8], columns: &[Vec>]) -> Proof { + prove_with(seed, columns, &config_unground()) +} + +fn prove_with(seed: &[u8], columns: &[Vec>], config: &ChainConfig) -> Proof { + let options = ProofOptions::default_test_options(); + let air: ConcreteVmAir<_> = create_eq_air(&options); + let num_main = columns.len(); + let num_vars = columns[0].len().trailing_zeros() as usize; + + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + num_main, + num_vars, + Uniforms::default(), + ) + .expect("layout"); + let table = CommittedTable::from_layout(layout, |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = + CommittedTables::<_, _, KeccakWhir>::commit(vec![table], config).expect("commit"); + + let mut transcript = DefaultTranscript::::new(seed); + multilinear_table::multi_prove(&committed, config, &mut transcript, None).expect("prove") +} + +/// The fixture is worth using only if it actually carries what the propositions +/// are about. +fn assert_fixture_is_not_degenerate(proof: &Proof) { + assert!(!proof.roots.is_empty(), "the fixture has no Merkle root"); + let rounds = proof + .columns + .iter() + .flat_map(|stacked| &stacked.polys) + .flat_map(|chain| &chain.rounds) + .count(); + assert!(rounds > 0, "the fixture has no WHIR rounds"); + let ground: u64 = proof + .columns + .iter() + .flat_map(|stacked| &stacked.polys) + .flat_map(|chain| &chain.rounds) + .map(|r| r.nonces.folding | r.nonces.ood | r.nonces.query) + .fold(0, |a, b| a | b); + assert!( + ground != 0, + "the fixture ground no nonce, so the normalisation cannot be exercised" + ); + let opened: usize = proof + .columns + .iter() + .flat_map(|stacked| &stacked.polys) + .map(|chain| chain.opened_elements()) + .sum(); + assert!(opened > 0, "the fixture opened no codeword value"); +} + +/// (1) The same input gives the same line, twice, when the proof itself is +/// deterministic. +#[test] +fn the_identity_line_is_stable_across_runs() { + let columns = fixture_columns(); + let a = prove_once_unground(b"whir-identity", &columns); + assert!(!a.roots.is_empty(), "the control fixture has no root"); + let b = prove_once_unground(b"whir-identity", &columns); + + assert_eq!( + identity_line(&a).unwrap(), + identity_line(&b).unwrap(), + "two runs of the same prover on the same input must give one line" + ); +} + +/// (5) ⚠ **The correction, pinned so nobody re-derives the wrong gate.** +/// +/// With grinding on, two honest runs differ in far more than their nonces, and +/// the normalised line differs too. This is the proposition that killed the +/// original byte-gate design; it is here so the next reader is told by a test +/// rather than by a comment. +/// +/// It compares the two proofs' NON-nonce content directly rather than asserting +/// two digests differ: the digests differing is the consequence, the roots +/// differing is the cause, and a test that asserted only the consequence would +/// pass for the wrong reason if the instrument broke. +/// +/// # ⛔ THE GUARD IS NARROWER THAN THE NONCE LIST, AND THAT IS THE FIX +/// +/// This test was intermittently RED — three green of four full-suite runs, and +/// eight of eight in isolation. Its early return guarded on ALL the nonces +/// while the assertion below sees only what `after_the_grind` captures: +/// `final_value` per chain, and `(next_root, ood_value)` per round. **A chain's +/// LAST round has both of those `None`**, so a run whose only differing nonce +/// is there passes the guard and then compares two equal lists. ? INFERRED from +/// the failing output, whose per-round tuples were identical and whose third +/// entry was `(None, None)` — not from instrumenting the search. +/// +/// So the guard now reads the nonces of the rounds the assertion CAN see, and +/// the probe below is what makes that narrowing observable rather than +/// asserted: a pair differing only in an invisible round must read as agreeing. +/// +/// ⚠ AND THE SENTENCE THAT USED TO BE HERE IS WITHDRAWN. It said a failure +/// "would fail LOUDLY and name the reason — which is the correct outcome, not a +/// flake". The observed failure named nothing and was a flake; a guard that +/// admits a state its assertion cannot satisfy is a check that cannot pass, not +/// a report about the prover. +#[test] +fn zeroing_the_nonces_does_not_make_a_ground_proof_reproducible() { + let columns = fixture_columns(); + let a = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&a); + let b = prove_once(b"whir-identity", &columns); + + let all_nonces = |p: &Proof| { + p.columns + .iter() + .flat_map(|s| &s.polys) + .flat_map(|c| &c.rounds) + .map(|r| r.nonces) + .collect::>() + }; + // The nonces the assertion below can SEE: a round with no successor + // contributes nothing to `after_the_grind`, so a nonce differing only there + // moves nothing this test compares. + let visible_nonces = |p: &Proof| { + p.columns + .iter() + .flat_map(|s| &s.polys) + .flat_map(|c| &c.rounds) + .filter(|r| r.next_root.is_some() || r.ood_value.is_some()) + .map(|r| r.nonces) + .collect::>() + }; + let after_the_grind = |p: &Proof| { + p.columns + .iter() + .flat_map(|s| &s.polys) + .map(|c| { + ( + c.final_value, + c.rounds + .iter() + .map(|r| (r.next_root, r.ood_value)) + .collect::>(), + ) + }) + .collect::>() + }; + + // ★ THE PROBE: a pair differing ONLY in rounds the assertion cannot see. + // Without it, narrowing the guard would be a claim; with it, a guard that + // still read every nonce fails here rather than once in four suite runs. + { + let mut probe = a.clone(); + let mut moved = 0usize; + for chain in probe.columns.iter_mut().flat_map(|s| s.polys.iter_mut()) { + for round in chain.rounds.iter_mut() { + if round.next_root.is_none() && round.ood_value.is_none() { + round.nonces.query ^= 1; + moved += 1; + } + } + } + assert!( + moved > 0, + "no round in this fixture is invisible to `after_the_grind`, so the state \ + that made this test intermittent cannot be exhibited here — say so rather \ + than leave the guard narrowed against nothing" + ); + assert_ne!( + all_nonces(&a), + all_nonces(&probe), + "the probe moved no nonce at all, so it proves nothing about the guard" + ); + assert_eq!( + visible_nonces(&a), + visible_nonces(&probe), + "the guard still reads a nonce the assertion cannot see — which is the \ + defect it was narrowed to remove" + ); + assert_eq!( + after_the_grind(&a), + after_the_grind(&probe), + "a nonce in a successorless round moved something the assertion compares, \ + so the narrowing rests on a false premise" + ); + } + + if visible_nonces(&a) == visible_nonces(&b) { + // The two searches agreed wherever this test can look — at four bits + // that is common. Nothing is being claimed about this run. + return; + } + + // The COMMITTED trace's root is drawn before any grind, so it does not + // move — naming that explicitly, because it is the thing that makes this + // failure mode easy to miss. What moves is everything the transcript + // produced after the first grind. + assert_eq!( + a.roots, b.roots, + "the trace commitment precedes the first grind and cannot depend on it" + ); + + assert_ne!( + after_the_grind(&a), + after_the_grind(&b), + "the nonce is absorbed, so a different nonce must move every challenge \ + drawn after it — the successor roots and the out-of-domain values" + ); + assert_ne!( + identity_line(&a).unwrap(), + identity_line(&b).unwrap(), + "and the normalised line moves with them: the nonce fields are not where \ + the divergence lives, so zeroing them cannot remove it" + ); +} + +/// (2a) One byte of one root moves the line. +#[test] +fn the_identity_line_moves_when_a_root_moves() { + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + let before = identity_line(&proof).unwrap(); + + let mut tampered = proof.clone(); + tampered.roots[0][0] ^= 1; + + assert_ne!( + before, + identity_line(&tampered).unwrap(), + "a changed Merkle root must change the line" + ); +} + +/// (2b) One opened codeword value moves the line. +/// +/// The roots are the obvious field; the openings are the bulk of the bytes and +/// the thing a leaf-hash defect would corrupt, so they are checked separately. +#[test] +fn the_identity_line_moves_when_an_opened_value_moves() { + use multilinear::whir_chain::RoundOpenings; + + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + let before = identity_line(&proof).unwrap(); + + let mut tampered = proof.clone(); + let round = &mut tampered.columns[0].polys[0].rounds[0]; + match &mut round.openings { + RoundOpenings::Base(p) => p.current[0].values[0] += FieldElement::::one(), + RoundOpenings::Extension(p) => { + p.current[0].values[0] += FieldElement::::one(); + } + RoundOpenings::BaseGrouped(_) => unreachable!("a per-polynomial chain"), + } + + assert_ne!( + before, + identity_line(&tampered).unwrap(), + "a changed opened codeword value must change the line" + ); +} + +/// (3) ★ The exclusion, stated exactly: a nonce and nothing but a nonce. +#[test] +fn the_identity_line_is_blind_to_grinding_nonces_and_to_nothing_else() { + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + let before = identity_line(&proof).unwrap(); + + // A different, arbitrary nonce in every slot of one round. + let mut renonced = proof.clone(); + { + let nonces = &mut renonced.columns[0].polys[0].rounds[0].nonces; + nonces.folding ^= 0xdead_beef; + nonces.ood ^= 0x0bad_f00d; + nonces.query ^= 0xfeed_face; + } + assert_ne!( + renonced.columns[0].polys[0].rounds[0].nonces, proof.columns[0].polys[0].rounds[0].nonces, + "the tamper must actually have changed the nonces" + ); + assert_eq!( + before, + identity_line(&renonced).unwrap(), + "the line excludes grinding nonces, by construction" + ); + + // …and the very next field along does move it, so the blindness is a + // targeted exclusion rather than a broken digest. + let mut moved = proof.clone(); + moved.columns[0].polys[0].final_value += FieldElement::::one(); + assert_ne!( + before, + identity_line(&moved).unwrap(), + "only the nonces are excluded" + ); +} + +/// (4) The nonce normalisation does not change how long the proof is, so a +/// length comparison across arms measures the proof. +#[test] +fn the_serialized_length_does_not_depend_on_the_nonces() { + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + + let mut renonced = proof.clone(); + for stacked in &mut renonced.columns { + for chain in &mut stacked.polys { + for round in &mut chain.rounds { + round.nonces.folding ^= u64::MAX; + round.nonces.ood ^= u64::MAX; + round.nonces.query ^= u64::MAX; + } + } + } + + assert_eq!( + serialized_len(&proof).unwrap(), + serialized_len(&renonced).unwrap(), + "a nonce is a fixed-width u64; the length cannot depend on its value" + ); +} diff --git a/prover/src/tests/windowed_builder_tests.rs b/prover/src/tests/windowed_builder_tests.rs new file mode 100644 index 000000000..e749aba69 --- /dev/null +++ b/prover/src/tests/windowed_builder_tests.rs @@ -0,0 +1,1511 @@ +//! The windowed trace builder builds the whole-run tables +//! (`tables::trace_builder::WindowedTraceBuilder`). + +use executor::elf::Elf; +use executor::vm::execution::Executor; +use executor::vm::logs::Log; +use math::field::element::FieldElement; +use stark::trace::TraceTable; + +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::{ + CutPadding, StreamTable, Traces, WalkLean, WindowedTraceBuilder, +}; +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; +use crate::test_utils::asm_elf_bytes; + +type Table = TraceTable; + +pub(crate) fn run(name: &str) -> (Elf, Vec) { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, Vec::new()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + (program, logs) +} + +pub(crate) fn whole(program: &Elf, logs: &[Log], max_rows: &MaxRowsConfig) -> Traces { + Traces::from_elf_and_logs(program, logs, max_rows, &[]).expect("the whole-run build") +} + +/// The windowed build over windows of `window` cycles (the last window is the +/// rest, at least one cycle), with its streamed chunks put back; and how many +/// chunks it streamed. +fn windowed( + program: &Elf, + logs: &[Log], + max_rows: &MaxRowsConfig, + window: usize, +) -> (Traces, usize) { + let mut builder = WindowedTraceBuilder::new(program, &[], max_rows).expect("the builder"); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend(builder.push(w).expect("a window")); + } + let streamed = chunks.len(); + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + (traces, streamed) +} + +/// [`windowed`] through `push_jobs`, each chunk generated by its job. +fn windowed_jobs(program: &Elf, logs: &[Log], max_rows: &MaxRowsConfig, window: usize) -> Traces { + let mut builder = WindowedTraceBuilder::new(program, &[], max_rows).expect("the builder"); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend( + builder + .push_jobs(w) + .expect("a window") + .into_iter() + .map(|job| job.generate()), + ); + } + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + traces +} + +/// [`windowed`] with the builder split: the walker on one thread, the +/// accumulator on another, the chunks generated by their jobs. +fn windowed_split(program: &Elf, logs: &[Log], max_rows: &MaxRowsConfig, window: usize) -> Traces { + let mut builder = WindowedTraceBuilder::new(program, &[], max_rows).expect("the builder"); + let body = logs.len() - 1; + let cut = body - body % window; + let chunks = { + let (mut walker, mut accumulator) = builder.split(); + std::thread::scope(|scope| { + let (tx, rx) = std::sync::mpsc::sync_channel(2); + let walking = scope.spawn(move || { + for w in logs[..cut].chunks(window) { + tx.send(walker.walk(w).expect("a window")) + .expect("the accumulator"); + } + }); + let mut chunks = Vec::new(); + for walked in rx { + chunks.extend( + accumulator + .absorb(walked) + .into_iter() + .map(|job| job.generate()), + ); + } + walking.join().expect("the walker"); + chunks + }) + }; + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + traces +} + +fn rows(t: &Table) -> Vec>> { + (0..t.main_table.height) + .map(|r| t.main_table.get_row(r).to_vec()) + .collect() +} + +/// Equal tables, row for row — also the six that lay their rows out in +/// `HashMap` order, since every such map in the process hashes with one state +/// (`tables::trace_hash`). +fn same(what: &str, a: &Table, b: &Table) { + assert_eq!( + (a.main_table.width, a.main_table.height), + (b.main_table.width, b.main_table.height), + "{what}: shape" + ); + assert!(rows(a) == rows(b), "{what}: rows differ"); +} + +fn same_list(what: &str, a: &[Table], b: &[Table]) { + assert_eq!(a.len(), b.len(), "{what}: table count"); + for (i, (x, y)) in a.iter().zip(b).enumerate() { + same(&format!("{what}[{i}]"), x, y); + } +} + +pub(crate) fn same_traces(a: &Traces, b: &Traces) { + same_list("CPU", &a.cpus, &b.cpus); + same_list("MEMW_R", &a.memw_registers, &b.memw_registers); + same_list("MEMW_A", &a.memw_aligneds, &b.memw_aligneds); + same_list("MEMW", &a.memws, &b.memws); + same_list("LOAD", &a.loads, &b.loads); + same_list("STORE", &a.stores, &b.stores); + same_list("SHIFT", &a.shifts, &b.shifts); + same_list("CPU32", &a.cpu32s, &b.cpu32s); + same_list("COPY", &a.copies, &b.copies); + same_list("KECCAK", &a.keccaks, &b.keccaks); + same_list("KECCAK_RND", &a.keccak_rnds, &b.keccak_rnds); + same_list("ECSM", &a.ecsms, &b.ecsms); + same_list("ECDAS", &a.ecdases, &b.ecdases); + same_list("HINT", &a.hints, &b.hints); + same_list("PAGE", &a.pages, &b.pages); + same_list("LT", &a.lts, &b.lts); + same_list("MUL", &a.muls, &b.muls); + same_list("DVRM", &a.dvrms, &b.dvrms); + same_list("BRANCH", &a.branches, &b.branches); + same_list("EQ", &a.eqs, &b.eqs); + same_list("BYTEWISE", &a.bytewises, &b.bytewises); + same("BITWISE", &a.bitwise, &b.bitwise); + same("DECODE", &a.decode, &b.decode); + same("REGISTER", &a.register, &b.register); + same("HALT", &a.halt, &b.halt); + same("KECCAK_RC", &a.keccak_rc, &b.keccak_rc); + same("BLAKE3", &a.blake3, &b.blake3); + assert_eq!(a.public_output_bytes, b.public_output_bytes); + assert_eq!(a.num_blake3_ops, b.num_blake3_ops); + assert_eq!(a.page_configs.len(), b.page_configs.len()); + assert_eq!( + format!("{:?}", a.table_counts()), + format!("{:?}", b.table_counts()) + ); +} + +/// ★ Two builds of one run in one process lay out every table alike, row for +/// row: also LT, EQ, BYTEWISE, BRANCH, MUL and DVRM, which deduplicate through a +/// `HashMap` and lay rows out in its order (each map took fresh random keys +/// before `tables::trace_hash`). Regenerating a streamed chunk needs this. +#[test] +fn every_build_in_a_process_lays_out_the_hash_ordered_tables_alike() { + use crate::tables::dvrm::{DvrmOperation, generate_dvrm_trace}; + use crate::tables::mul::{MulOperation, generate_mul_trace}; + let mut rows = [0usize; 4]; + for (name, max_rows) in [ + ("all_instructions_64", MaxRowsConfig::small()), + ("bench_32k", MaxRowsConfig::uniform(1 << 14)), + ] { + let (program, logs) = run(name); + let (a, b) = ( + whole(&program, &logs, &max_rows), + whole(&program, &logs, &max_rows), + ); + same_traces(&a, &b); + for (most, list) in rows + .iter_mut() + .zip([&a.lts, &a.eqs, &a.bytewises, &a.branches]) + { + *most = (*most).max(list.iter().map(|t| t.main_table.height).sum()); + } + } + // Enough distinct operations that two hash keys would order them apart. + assert!(rows.iter().all(|&n| n >= 8), "rows per table: {rows:?}"); + // The programs divide and multiply by few distinct operands: 199 of each. + let muls: Vec<(MulOperation, bool)> = (1..200u64) + .map(|i| { + let op = MulOperation { + lhs: i.wrapping_mul(0x9e37_79b9_7f4a_7c15), + lhs_signed: i % 2 == 0, + rhs: i * 7919, + rhs_signed: i % 3 == 0, + }; + (op, i % 5 == 0) + }) + .collect(); + same( + "MUL", + &generate_mul_trace(&muls), + &generate_mul_trace(&muls), + ); + let dvrms: Vec<(DvrmOperation, bool)> = (1..200u64) + .map(|i| { + let op = DvrmOperation { + n: i.wrapping_mul(0x9e37_79b9_7f4a_7c15), + d: i % 13 + 1, + signed: i % 2 == 0, + }; + (op, i % 3 == 0) + }) + .collect(); + same( + "DVRM", + &generate_dvrm_trace(&dvrms), + &generate_dvrm_trace(&dvrms), + ); +} + +/// The hash state itself: the process's state hashes alike in every clone, and +/// the fixed state (`LAMBDA_VM_FIXED_TRACE_HASH=1`) is SipHash-1-3 with zero +/// keys, the same in every process. +#[test] +fn the_trace_hash_state_is_one_per_process() { + use crate::tables::trace_hash::{TraceHashState, trace_hash_state}; + use std::hash::BuildHasher; + let key = (0x1234_5678_9abc_def0u64, true); + assert_eq!( + trace_hash_state().hash_one(key), + trace_hash_state().hash_one(key) + ); + assert_eq!( + TraceHashState::fixed().hash_one(key), + std::hash::BuildHasherDefault::::default() + .hash_one(key) + ); +} + +/// ★ The builder is a schedule: at every window length — one cycle, less than a +/// chunk, exactly a chunk, not a divisor of it, longer than the run — the +/// windowed build is the whole-run build, table for table, and it did stream. +#[test] +fn windowed_builds_the_whole_run_tables() { + let max_rows = MaxRowsConfig::small(); + for name in [ + "sub", + "all_instructions_64", + "test_keccak", + "test_keccak_multi", + ] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [1, 7, 32, 33, 1000, logs.len()] { + let (traces, streamed) = windowed(&program, &logs, &max_rows, window.max(1)); + same_traces(&reference, &traces); + if logs.len() > 2 * max_rows.cpu && window < logs.len() / 2 { + assert!(streamed > 0, "{name}, window {window}: nothing streamed"); + } + } + } +} + +/// The same through chunk jobs generated by the caller. +#[test] +fn windowed_jobs_build_the_whole_run_tables() { + let max_rows = MaxRowsConfig::small(); + for name in ["all_instructions_64", "test_keccak_multi"] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [7, 33] { + same_traces( + &reference, + &windowed_jobs(&program, &logs, &max_rows, window), + ); + } + } +} + +/// The same with the builder split across two threads. +#[test] +fn a_split_builder_builds_the_whole_run_tables() { + let max_rows = MaxRowsConfig::small(); + for name in ["all_instructions_64", "test_keccak_multi"] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [1, 7, 33] { + same_traces( + &reference, + &windowed_split(&program, &logs, &max_rows, window), + ); + } + } +} + +/// The walker collects and walks a window a batch of cycles at a time; any +/// batch (one cycle, a few, more than the window, or the whole window first) +/// builds the whole-run tables, ops kept or dropped. +#[test] +fn a_batched_walk_builds_the_whole_run_tables() { + let max_rows = MaxRowsConfig::small(); + for name in [ + "all_instructions_64", + "test_keccak_multi", + "lw_sw_offset_odd", + ] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [33, 1000] { + for batch in [1, 7, 64, 0] { + for drop in [false, true] { + let (traces, _) = + windowed_with(&program, &logs, &max_rows, window, true, |builder| { + let builder = builder.walk_batch(batch); + if drop { + builder + .drop_streamed_ops() + .expect("before the first window") + } else { + builder + } + }); + same_traces(&reference, &traces); + } + } + } + } +} + +/// And at the production chunk sizes, where a small program streams nothing +/// and the builder is the whole-run build over windows. +#[test] +fn windowed_builds_the_whole_run_tables_at_production_sizes() { + let max_rows = MaxRowsConfig::default(); + let (program, logs) = run("all_instructions_64"); + let reference = whole(&program, &logs, &max_rows); + let (traces, _) = windowed(&program, &logs, &max_rows, 64); + same_traces(&reference, &traces); +} + +/// The padding rule the windowed builder reads (`LAMBDA_VM_BLOCK_CUT_PADDING`). +fn padding() -> CutPadding { + CutPadding::from_env().expect("the padding rule") +} + +/// The whole-run KECCAK_RND table cut into tables of `rows` rows — rows +/// `[k·rows, (k+1)·rows)`, as the block proof splits it — as many as the +/// builder's padding rule counts ([`CutPadding::tables`]) of the table's real +/// rows (up to the last one whose μ is set). +fn split_keccak_rnd(t: &Traces, per: usize) -> Vec
{ + use crate::tables::keccak_rnd::cols::MU; + t.keccak_rnds + .iter() + .flat_map(|table| { + let width = table.main_table.width; + let all = rows(table); + if all.len() <= per { + return vec![rebuild(&all, width)]; + } + let keep = match padding() { + CutPadding::WholeRun => all.len() / per, + last => { + let real = all + .iter() + .rposition(|row| row[MU] != FieldElement::zero()) + .map_or(0, |row| row + 1); + last.tables(real, per) + } + }; + all.chunks(per) + .take(keep) + .map(|chunk| rebuild(chunk, width)) + .collect() + }) + .collect() +} + +fn rebuild(rows: &[Vec>], width: usize) -> Table { + TraceTable::new_main(rows.iter().flatten().copied().collect(), width, 1) +} + +/// The whole-run BITWISE without the lookups of the KECCAK_RND rows `split` +/// leaves out, and how many rows that is. A padding row sends KECCAK_RND's byte +/// checks as `ARE_BYTES[0, 0]` (`keccak_rnd::ARE_BYTES_PER_ROW` a row, main's +/// multiplicity fix), and the whole run counts those of its whole padded table. +fn split_bitwise(t: &Traces, split: &[Table]) -> (Table, usize) { + use crate::tables::{bitwise, keccak_rnd}; + let height = |tables: &[Table]| tables.iter().map(|t| t.main_table.height).sum::(); + let left_out = height(&t.keccak_rnds) - height(split); + let mut table = t.bitwise.clone(); + let (row, col) = (bitwise::row_index(0, 0, 0), bitwise::cols::MU_ARE_BYTES); + let count = table.main_table.get_row(row)[col]; + let lookups = FieldElement::from((left_out * keccak_rnd::ARE_BYTES_PER_ROW) as u64); + table.main_table.set(row, col, count - lookups); + (table, left_out) +} + +/// A slice of KECCAK_RND rows built from the ops that reach it is that slice of +/// the whole table, wherever the slice cuts an op's 24 rows. +#[test] +fn keccak_rnd_rows_are_slices_of_the_whole_table() { + use crate::tables::keccak_rnd::{ + KeccakRoundOperation, generate_keccak_rnd_rows, generate_keccak_rnd_trace, + }; + let ops: Vec = (0..11u64) + .map(|i| KeccakRoundOperation { + timestamp: 100 + 3 * i, + input: core::array::from_fn(|j| i.wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ j as u64), + output: [0; 25], + }) + .collect(); + let whole = rows(&generate_keccak_rnd_trace(&ops)); + for (skip, len) in [ + (0, 32), + (32, 32), + (50, 64), + (23, 1), + (100, 156), + (256, 0), + (200, 56), + ] { + let slice = rows(&generate_keccak_rnd_rows(&ops, skip, len)); + assert_eq!(slice.len(), len); + assert!( + slice[..] == whole[skip..skip + len], + "rows {skip}..{}", + skip + len + ); + } +} + +/// ★ With KECCAK_RND cut into chunks, the windowed build streams them and is the +/// whole-run build with KECCAK_RND split the same way, table for table, BITWISE +/// without the left-out rows' lookups ([`split_bitwise`]); at one-cycle windows +/// it did stream some. +#[test] +fn windowed_streams_keccak_rnd_as_the_split_whole_run_table() { + let max_rows = MaxRowsConfig::small(); + let mut left_out_any = false; + for name in ["test_keccak", "test_keccak_multi"] { + let (program, logs) = run(name); + let mut reference = whole(&program, &logs, &max_rows); + for per in [32usize, 64, 128] { + let split = split_keccak_rnd(&reference, per); + let (bitwise, left_out) = split_bitwise(&reference, &split); + left_out_any |= left_out > 0; + for window in [1, 7, 33] { + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows) + .expect("the builder") + .keccak_rnd_chunks(per) + .expect("a chunk size"); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend(builder.push(w).expect("a window")); + } + let streamed = chunks + .iter() + .filter(|c| c.table == StreamTable::KeccakRnd) + .count(); + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces.insert_streamed(chunks).expect("placeholders"); + same_list( + &format!("{name} KECCAK_RND/{per} window {window}"), + &split, + &traces.keccak_rnds, + ); + // One-cycle windows push every cycle but the last, so every + // chunk the ops fill before the run ends streams. + let full = split.iter().filter(|t| t.main_table.height == per).count(); + if window == 1 && full > 1 { + assert!(streamed > 0, "{name}, rows {per}: none streamed"); + } + // Every other table is the whole-run one. + let keccak = + std::mem::replace(&mut reference.keccak_rnds, traces.keccak_rnds.clone()); + let whole_bitwise = std::mem::replace(&mut reference.bitwise, bitwise.clone()); + same_traces(&reference, &traces); + reference.keccak_rnds = keccak; + reference.bitwise = whole_bitwise; + } + } + } + assert!( + left_out_any || padding() == CutPadding::WholeRun, + "no split left a KECCAK_RND table out" + ); +} + +/// With KECCAK, ECSM, ECDAS and COPY built as their cuts at the finish, the +/// windowed build (its streamed ops dropped) is the whole-run build with those +/// four split by the block's split under the build's padding rule, table for table, +/// at every cut size from one row to past the whole table; the other tables +/// are untouched. And a cut that is not a power of two is refused. +#[cfg(feature = "parallel")] +#[test] +fn windowed_builds_the_cut_tables_as_the_split_whole_run_tables() { + use crate::block_whir::{split_copy, split_ecdas, split_ecsm, split_keccak}; + let max_rows = MaxRowsConfig::small(); + let mut cut_any = [false; 4]; + for name in [ + "test_keccak_multi", + "test_ecsm", + "test_ecsm_multi", + "test_ecsm_split", + "test_copy_ecalls", + ] { + let (program, logs) = run(name); + for log2 in [0usize, 1, 2, 3, 6] { + let rows = 1usize << log2; + let mut want = whole(&program, &logs, &max_rows); + let counts = [ + want.keccaks.len(), + want.ecsms.len(), + want.ecdases.len(), + want.copies.len(), + ]; + split_keccak(&mut want, log2, padding()); + split_ecsm(&mut want, log2, padding()); + split_ecdas(&mut want, log2, padding()); + split_copy(&mut want, log2, padding()); + for (any, (before, after)) in cut_any.iter_mut().zip(counts.iter().zip([ + want.keccaks.len(), + want.ecsms.len(), + want.ecdases.len(), + want.copies.len(), + ])) { + *any |= after > *before; + } + for window in [7, 1000] { + let (cut, _) = windowed_with(&program, &logs, &max_rows, window, true, |b| { + b.drop_streamed_ops() + .expect("before any window") + .cuts_at_finish(rows, rows, rows, rows) + .expect("powers of two") + }); + same_traces(&want, &cut); + } + } + } + assert_eq!( + cut_any, [true; 4], + "every family was cut into several tables" + ); + for (keccak, ecsm, ecdas, copy) in [(6, 4, 4, 4), (4, 0, 4, 4), (4, 4, 12, 4), (4, 4, 4, 6)] { + assert!( + WindowedTraceBuilder::new(&run("test_keccak").0, &[], &max_rows) + .expect("the builder") + .cuts_at_finish(keccak, ecsm, ecdas, copy) + .is_err(), + "{keccak}/{ecsm}/{ecdas}/{copy} is refused" + ); + } +} + +/// With KECCAK_RND built as its tables at the finish alone, the windowed build +/// hands none out and is the whole-run build with KECCAK_RND split the same +/// way, table for table, BITWISE without the left-out rows' lookups +/// ([`split_bitwise`]) — the split's copy without the copy. +#[test] +fn windowed_builds_keccak_rnd_at_finish_as_the_split_whole_run_table() { + let max_rows = MaxRowsConfig::small(); + let mut left_out_any = false; + for name in ["test_keccak", "test_keccak_multi"] { + let (program, logs) = run(name); + let mut reference = whole(&program, &logs, &max_rows); + for per in [32usize, 64, 128] { + let split = split_keccak_rnd(&reference, per); + let (bitwise, left_out) = split_bitwise(&reference, &split); + left_out_any |= left_out > 0; + for window in [1, 33] { + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows) + .expect("the builder") + .keccak_rnd_chunks_at_finish(per) + .expect("a chunk size"); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend(builder.push(w).expect("a window")); + } + assert!( + chunks.iter().all(|c| c.table != StreamTable::KeccakRnd), + "{name}: a KECCAK_RND chunk was handed out" + ); + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces.insert_streamed(chunks).expect("placeholders"); + same_list( + &format!("{name} KECCAK_RND/{per} at the finish, window {window}"), + &split, + &traces.keccak_rnds, + ); + let keccak = + std::mem::replace(&mut reference.keccak_rnds, traces.keccak_rnds.clone()); + let whole_bitwise = std::mem::replace(&mut reference.bitwise, bitwise.clone()); + same_traces(&reference, &traces); + reference.keccak_rnds = keccak; + reference.bitwise = whole_bitwise; + } + } + } + assert!( + left_out_any || padding() == CutPadding::WholeRun, + "no split left a KECCAK_RND table out" + ); + assert!( + WindowedTraceBuilder::new(&run("test_keccak").0, &[], &max_rows) + .expect("the builder") + .keccak_rnd_chunks_at_finish(16) + .is_err(), + "16 rows is under the 32 a KECCAK_RND op's rows need" + ); +} + +/// Every LT op with its multiplicity summed over all of `lts`' chunks: what LT's +/// constraints and the bus see, whichever chunk holds an op. +fn lt_multiplicities(lts: &[Table]) -> std::collections::BTreeMap, u64> { + let mu = crate::tables::lt::cols::MU; + let mut out = std::collections::BTreeMap::new(); + for table in lts { + for row in rows(table) { + let m = row[mu].canonical(); + if m == 0 { + continue; + } + let key: Vec = row + .iter() + .enumerate() + .filter(|(c, _)| *c != mu) + .map(|(_, v)| v.canonical()) + .collect(); + *out.entry(key).or_insert(0) += m; + } + } + out +} + +/// ★ With each window's MEMW-derived LT ops streamed (the block path), every +/// table but LT is the whole-run build's, LT has as many chunks, and every LT +/// op carries its whole-run multiplicity summed over the chunks — though the +/// chunks hold other ops. Dropping one row of one chunk breaks that equality, +/// so it is what the test checks. And it streams more LT chunks than without. +#[test] +fn windowed_streams_memw_lt_with_the_whole_run_lt_multiplicities() { + let max_rows = MaxRowsConfig::small(); + let mut more_streamed = false; + for name in ["all_instructions_64", "test_keccak_multi"] { + let (program, logs) = run(name); + let mut reference = whole(&program, &logs, &max_rows); + let want = lt_multiplicities(&reference.lts); + for window in [1, 7, 33] { + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows) + .expect("the builder") + .stream_memw_lt(); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend(builder.push(w).expect("a window")); + } + let lt_streamed = chunks.iter().filter(|c| c.table == StreamTable::Lt).count(); + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces.insert_streamed(chunks).expect("placeholders"); + assert_eq!( + traces.lts.len(), + reference.lts.len(), + "{name}/{window}: LT chunks" + ); + assert!( + lt_multiplicities(&traces.lts) == want, + "{name}/{window}: LT multiplicities" + ); + let plain_lt = { + let mut builder = + WindowedTraceBuilder::new(&program, &[], &max_rows).expect("the builder"); + logs[..cut] + .chunks(window) + .flat_map(|w| builder.push(w).expect("a window")) + .filter(|c| c.table == StreamTable::Lt) + .count() + }; + assert!( + lt_streamed >= plain_lt, + "{name}/{window}: fewer LT chunks streamed" + ); + more_streamed |= lt_streamed > plain_lt; + // Every other table is the whole-run one. + let lts = std::mem::replace(&mut reference.lts, traces.lts.clone()); + same_traces(&reference, &traces); + reference.lts = lts; + + // The mutation: one row of one chunk dropped (its multiplicity zeroed). + let mut dropped = traces.lts.clone(); + let mu = crate::tables::lt::cols::MU; + let (k, row) = dropped + .iter() + .enumerate() + .find_map(|(k, t)| { + (0..t.main_table.height) + .find(|&r| t.main_table.get(r, mu).canonical() != 0) + .map(|r| (k, r)) + }) + .expect("a real LT row"); + dropped[k] + .main_table + .set(row, mu, FieldElement::::zero()); + assert!( + lt_multiplicities(&dropped) != want, + "{name}/{window}: a dropped row must change the multiplicities" + ); + } + } + assert!( + more_streamed, + "the MEMW-derived LT ops never streamed a chunk more" + ); +} + +/// [`WindowedTraceBuilder::held_streamed_ops`]'s report. +type Held = [(usize, usize); 8]; + +/// A windowed build over windows of `window` cycles with the builder `configure` +/// returns, its chunks generated by their jobs on the walker/accumulator split +/// (the block drivers' path) or through `push`; and, under +/// `drop_streamed_ops`, the most streamed ops it held after any window. +fn windowed_with( + program: &Elf, + logs: &[Log], + max_rows: &MaxRowsConfig, + window: usize, + split: bool, + configure: impl Fn(WindowedTraceBuilder<'_>) -> WindowedTraceBuilder<'_>, +) -> (Traces, Option) { + let (traces, held, _) = windowed_streaming(program, logs, max_rows, window, split, configure); + (traces, held) +} + +/// [`windowed_with`], and the tables it streamed chunks of. +fn windowed_streaming( + program: &Elf, + logs: &[Log], + max_rows: &MaxRowsConfig, + window: usize, + split: bool, + configure: impl Fn(WindowedTraceBuilder<'_>) -> WindowedTraceBuilder<'_>, +) -> (Traces, Option, std::collections::BTreeSet) { + let mut builder = + configure(WindowedTraceBuilder::new(program, &[], max_rows).expect("the builder")); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + let mut most: Option = None; + let mut note = |held: Option| { + if let Some(held) = held { + let most = most.get_or_insert([(0, 0); 8]); + for (m, h) in most.iter_mut().zip(held) { + *m = (m.0.max(h.0), m.1.max(h.1)); + } + } + }; + for w in logs[..cut].chunks(window) { + if split { + let (mut walker, mut accumulator) = builder.split(); + let walked = walker.walk(w).expect("a window"); + chunks.extend( + accumulator + .absorb(walked) + .into_iter() + .map(|job| job.generate()), + ); + } else { + chunks.extend(builder.push(w).expect("a window")); + } + note(builder.held_streamed_ops()); + } + let streamed = chunks.iter().map(|c| format!("{:?}", c.table)).collect(); + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + (traces, most, streamed) +} + +/// ★ Dropping each streamed chunk's ops as it leaves builds the whole-run +/// tables, table for table, at every window length, through `push` and through +/// the split (the derived LT ops kept compact and LT's ops kept as segments, +/// the default; or held at 24 bytes each and concatenated, the A arms); with +/// the MEMW-derived LT ops streamed as well it builds what that +/// builder builds, LT carrying the whole-run multiplicities. Also with CPU +/// chunks of 24 rows, whose dropped chunks each pad 8 rows, and with chunks of 4 +/// rows (one op for the rarer tables), so that every streamed table drops +/// chunks. +#[test] +fn dropping_the_streamed_ops_builds_the_same_tables() { + let small = MaxRowsConfig::small(); + let padded = MaxRowsConfig { + cpu: 24, + ..MaxRowsConfig::small() + }; + let tiny = MaxRowsConfig::uniform(4); + // The rarer tables (MEMW holds only misaligned and mixed-timestamp + // accesses and width-1 register writes) get a handful of ops per program: + // one op a chunk. + let one_op = MaxRowsConfig { + memw: 1, + load: 1, + shift: 1, + store: 1, + ..MaxRowsConfig::uniform(4) + }; + let mut dropped = std::collections::BTreeSet::new(); + for (name, max_rows) in [ + ("sub", &small), + ("all_instructions_64", &small), + ("test_keccak", &small), + ("test_keccak_multi", &small), + ("all_instructions_64", &padded), + ("test_keccak_multi", &padded), + ("all_instructions_64", &tiny), + ("test_keccak_multi", &tiny), + ("misalign_sd", &one_op), + ("test_commit_split", &one_op), + ("all_loadstore_32", &one_op), + ("all_instructions_64", &one_op), + ] { + let max_rows = max_rows.clone(); + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + let want = lt_multiplicities(&reference.lts); + for window in [1, 7, 32, 33, 1000, logs.len()] { + let window = window.max(1); + let (lt_streamed, _) = windowed_with(&program, &logs, &max_rows, window, false, |b| { + b.stream_memw_lt() + }); + for split in [false, true] { + let (dropping, held, streamed) = + windowed_streaming(&program, &logs, &max_rows, window, split, |b| { + b.drop_streamed_ops().expect("before any window") + }); + assert!( + held.is_some() || logs.len() <= window + 1, + "{name}/{window}" + ); + dropped.extend(streamed); + same_traces(&reference, &dropping); + if !split { + // The finish as it was: the derived LT ops held at 24 + // bytes each and LT's ops concatenated (the A arms); and + // the compact ops concatenated. The same tables. + let (raw, _) = windowed_with(&program, &logs, &max_rows, window, false, |b| { + b.drop_streamed_ops() + .expect("before any window") + .raw_memw_lt() + .expect("before any window") + .concat_lt() + }); + same_traces(&reference, &raw); + let (concat, _) = + windowed_with(&program, &logs, &max_rows, window, false, |b| { + b.drop_streamed_ops() + .expect("before any window") + .concat_lt() + }); + same_traces(&reference, &concat); + } + let (both, _) = windowed_with(&program, &logs, &max_rows, window, split, |b| { + b.stream_memw_lt() + .drop_streamed_ops() + .expect("before any window") + }); + same_traces(<_streamed, &both); + assert!( + lt_multiplicities(&both.lts) == want, + "{name}/{window}: LT multiplicities" + ); + } + } + } + for table in [ + "Cpu", + "MemwRegister", + "MemwAligned", + "Memw", + "Load", + "Lt", + "Shift", + "Store", + ] { + assert!(dropped.contains(table), "no {table} chunk was dropped"); + } +} + +/// ★ DECODE's trace is generated on a thread of its own while the walk starts, +/// and the first CPU chunks' DECODE rows are counted into it as they leave: with +/// that thread held back past the first hand-outs, the hand-out waits for the +/// trace and the build is the whole-run build, table for table, through the +/// split (the block's path) and through `push`. +#[test] +fn a_late_decode_trace_is_waited_for_before_its_rows_are_counted() { + use crate::tables::trace_builder::DECODE_TRACE_DELAY; + let max_rows = MaxRowsConfig::small(); + let (program, logs) = run("test_keccak_multi"); + let reference = whole(&program, &logs, &max_rows); + DECODE_TRACE_DELAY.with(|d| d.set(std::time::Duration::from_millis(300))); + // Counted on the accumulator's thread, or on a thread of its own (whose + // rows are added to the trace once the windows are walked). + for on_thread in [false, true] { + for split in [true, false] { + let t = std::time::Instant::now(); + let (late, _, streamed) = + windowed_streaming(&program, &logs, &max_rows, 7, split, |b| { + b.count_handout_on_thread(on_thread) + .drop_streamed_ops() + .expect("before any window") + }); + assert!(streamed.contains("Cpu"), "no CPU chunk was handed out"); + assert!(t.elapsed() >= std::time::Duration::from_millis(300)); + same_traces(&reference, &late); + } + } + DECODE_TRACE_DELAY.with(|d| d.set(std::time::Duration::ZERO)); +} + +/// ★ The hand-out's counts (DECODE's lookups and the last ECALL from CPU, +/// BITWISE's from MEMW_R, MEMW_A, LT and SHIFT, the MEMW-derived LT ops) taken +/// on a thread of its own build the tables they build on the accumulator's +/// thread, which are the whole-run tables: through `push` and through the +/// split, at several windows, with the derived LT ops compact or raw, or +/// streamed. The thread did count every table's chunks, and the build that +/// counts inline started none. +#[test] +fn the_hand_outs_count_on_a_thread_of_its_own_builds_the_same_tables() { + let one_op = MaxRowsConfig { + memw: 1, + load: 1, + shift: 1, + store: 1, + ..MaxRowsConfig::uniform(4) + }; + let mut counted = [0usize; 9]; + for (name, max_rows) in [ + ("all_instructions_64", MaxRowsConfig::small()), + ("test_keccak_multi", MaxRowsConfig::small()), + ("all_instructions_64", one_op.clone()), + ("misalign_sd", one_op.clone()), + ("all_loadstore_32", one_op), + ] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [1, 7, 33] { + // With the MEMW-derived LT ops streamed, LT's lists follow the + // windows: the build that keeps its ops at the same windows. + let lt_streamed_reference = + windowed_with(&program, &logs, &max_rows, window, false, |b| { + b.stream_memw_lt() + }) + .0; + for split in [false, true] { + type Configure = fn(WindowedTraceBuilder<'_>) -> WindowedTraceBuilder<'_>; + let lists: [(&str, Configure); 3] = [ + ("compact", |b| b), + ("raw", |b| b.raw_memw_lt().expect("before any window")), + ("streamed", |b| b.stream_memw_lt()), + ]; + for (what, lists) in lists { + let mut built = Vec::new(); + for on_thread in [false, true] { + let (traces, stamps) = + windowed_counted(&program, &logs, &max_rows, window, split, |b| { + lists(b.count_handout_on_thread(on_thread)) + .drop_streamed_ops() + .expect("before any window") + }); + let at = format!("{name}/{window}/{split}/{what}/thread {on_thread}"); + match (on_thread, stamps) { + (false, Some(_)) => panic!("{at}: a count thread ran"), + (true, Some(s)) => { + for (sum, n) in counted.iter_mut().zip(s.chunks) { + *sum += n; + } + } + (true, None) if logs.len() > window + 1 => { + panic!("{at}: no count thread ran") + } + _ => {} + } + if what == "streamed" { + same_traces(<_streamed_reference, &traces); + } else { + same_traces(&reference, &traces); + } + built.push(traces); + } + same_traces(&built[0], &built[1]); + } + } + } + } + for table in [ + StreamTable::Cpu, + StreamTable::MemwRegister, + StreamTable::MemwAligned, + StreamTable::Memw, + StreamTable::Lt, + StreamTable::Shift, + ] { + assert!( + counted[table.position()] > 0, + "the count thread counted no {table:?} chunk" + ); + } +} + +/// [`windowed_streaming`] with the hand-out's count thread joined before the +/// finish: the build, and the thread's stamps if one ran. +fn windowed_counted( + program: &Elf, + logs: &[Log], + max_rows: &MaxRowsConfig, + window: usize, + split: bool, + configure: impl Fn(WindowedTraceBuilder<'_>) -> WindowedTraceBuilder<'_>, +) -> ( + Traces, + Option, +) { + let mut builder = + configure(WindowedTraceBuilder::new(program, &[], max_rows).expect("the builder")); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + if split { + let (mut walker, mut accumulator) = builder.split(); + let walked = walker.walk(w).expect("a window"); + chunks.extend( + accumulator + .absorb(walked) + .into_iter() + .map(|job| job.generate()), + ); + } else { + chunks.extend(builder.push(w).expect("a window")); + } + } + builder.join_handout_count().expect("the count thread"); + let stamps = builder.stamps().count_thread; + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + (traces, stamps) +} + +/// ★ With a private input in the initial image (more than a page of it), the +/// windowed build is the whole-run build of the same run, table for table, its +/// streamed ops kept or dropped: the image held as pages gives the PAGE tables +/// and the walk's state the image as a map of bytes gives. +#[test] +fn a_windowed_build_with_a_private_input_is_the_whole_run_build() { + let max_rows = MaxRowsConfig::small(); + let input: Vec = (0..300_000u32) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + for name in ["all_instructions_64", "test_keccak_multi"] { + let program = Elf::load(&asm_elf_bytes(name)).expect("the ELF loads"); + let logs = Executor::new(&program, input.clone()) + .expect("the executor starts") + .run() + .expect("the program runs") + .logs; + let reference = Traces::from_elf_and_logs(&program, &logs, &max_rows, &input) + .expect("the whole-run build"); + for window in [7, 33, 1000] { + for drop in [false, true] { + let mut builder = + WindowedTraceBuilder::new(&program, &input, &max_rows).expect("the builder"); + if drop { + builder = builder.drop_streamed_ops().expect("before any window"); + } + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + for w in logs[..cut].chunks(window) { + chunks.extend(builder.push(w).expect("a window")); + } + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + same_traces(&reference, &traces); + } + } + } +} + +/// The same with KECCAK_RND cut into streamed chunks. +#[test] +fn dropping_the_streamed_ops_streams_keccak_rnd_the_same() { + let max_rows = MaxRowsConfig::small(); + for name in ["test_keccak", "test_keccak_multi"] { + let (program, logs) = run(name); + for per in [32usize, 128] { + for window in [1, 7, 33] { + let (keeping, _) = windowed_with(&program, &logs, &max_rows, window, false, |b| { + b.keccak_rnd_chunks(per).expect("rows") + }); + let (dropping, _) = windowed_with(&program, &logs, &max_rows, window, true, |b| { + b.keccak_rnd_chunks(per) + .expect("rows") + .drop_streamed_ops() + .expect("before any window") + }); + same_traces(&keeping, &dropping); + } + } + } +} + +/// ★ Dropping bounds what the builder holds: after every window each streamed +/// table holds less than a chunk of ops not handed out, and its lists take room +/// for at most a window's part more — however long the run. Keeping every +/// window would hold the run's ops, which here are many times that. +#[test] +fn dropping_the_streamed_ops_holds_less_than_a_chunk_per_table() { + let max_rows = MaxRowsConfig::small(); + let maxes = [ + max_rows.cpu, + max_rows.memw_register, + max_rows.memw_aligned, + max_rows.memw, + max_rows.load, + max_rows.lt, + max_rows.shift, + max_rows.store, + ]; + let mut long = false; + for name in ["bench_32k", "test_keccak_multi"] { + let (program, logs) = run(name); + for window in [1, 7, 33] { + let (_, held) = windowed_with(&program, &logs, &max_rows, window, true, |b| { + b.drop_streamed_ops().expect("before any window") + }); + let held = held.expect("dropping reports what it holds"); + // A window's part: one CPU op per cycle; at most 256 ops of another + // table per cycle (a keccak ECALL's memory accesses are the most a + // cycle makes: ≈ 170 MEMW_A ops on test_keccak_multi). + for (table, ((not_out, room), max)) in held.iter().zip(maxes).enumerate() { + let per_cycle = if table == 0 { 1 } else { 256 }; + assert!( + *not_out < max, + "{name}/{window}: table {table} holds {not_out} ops, a full chunk" + ); + assert!( + *room < max + per_cycle * window, + "{name}/{window}: table {table} takes room for {room} ops" + ); + } + // Keeping every window would hold the run's CPU ops. + long |= logs.len() > 100 * (max_rows.cpu + window); + } + // Without the option nothing is reported. + let (_, held) = windowed_with(&program, &logs, &max_rows, 7, true, |b| b); + assert!(held.is_none()); + } + assert!( + long, + "no run long enough that keeping it would fail the CPU bound" + ); +} + +/// The hand-out's stamps count every chunk it handed out, table by table, and +/// the ops it copied. Handed out as the window parts they lie in (the default) +/// no chunk is copied; under `LAMBDA_VM_HANDOUT_COPY=1` a CPU chunk whose +/// window lists are each exactly a chunk moves out whole (none copied) and one +/// cut out of windows of 7 cycles is copied op for op. The tables are the same +/// every way. +#[test] +fn the_handout_stamps_count_each_tables_chunks_and_copies() { + let max_rows = MaxRowsConfig::small(); + let (program, logs) = run("bench_32k"); + let reference = whole(&program, &logs, &max_rows); + let copies = std::env::var("LAMBDA_VM_HANDOUT_COPY").is_ok_and(|v| v.trim() == "1"); + for (window, copied_when_copying) in [(max_rows.cpu, 0), (7, max_rows.cpu)] { + let cpu_copied_per_chunk = if copies { copied_when_copying } else { 0 }; + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows) + .expect("the builder") + .drop_streamed_ops() + .expect("before any window"); + let body = logs.len() - 1; + let cut = body - body % window; + let mut chunks = Vec::new(); + let mut handed = [0usize; 9]; + for w in logs[..cut].chunks(window) { + let (mut walker, mut accumulator) = builder.split(); + let walked = walker.walk(w).expect("a window"); + for job in accumulator.absorb(walked) { + handed[job.table.position()] += 1; + chunks.push(job.generate()); + } + } + let stamps = builder.stamps().handout; + let streamed = builder.streamed(); + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces + .insert_streamed(chunks) + .expect("every chunk has a placeholder"); + same_traces(&reference, &traces); + assert_eq!(stamps.chunks, handed, "window {window}: chunks per table"); + assert_eq!(stamps.chunks[StreamTable::Cpu.position()], streamed.cpu); + assert!( + streamed.cpu > 10, + "window {window}: {} CPU chunks", + streamed.cpu + ); + assert_eq!( + stamps.copied[StreamTable::Cpu.position()], + streamed.cpu * cpu_copied_per_chunk, + "window {window}: CPU ops copied" + ); + let timed: f64 = stamps.take.iter().chain(&stamps.count).sum(); + assert!(timed > 0.0, "window {window}: nothing timed"); + } + for (i, table) in StreamTable::ALL.iter().enumerate() { + assert_eq!(table.position(), i); + } +} + +/// The option is set before the first window, or refused. +#[test] +fn dropping_the_streamed_ops_is_refused_after_a_window() { + let max_rows = MaxRowsConfig::small(); + let (program, logs) = run("all_instructions_64"); + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows).expect("the builder"); + builder.push(&logs[..7]).expect("a window"); + assert!(builder.drop_streamed_ops().is_err()); +} + +/// A non-final window may not halt, as a whole-run build refuses an epoch that +/// halts early. +#[test] +fn a_window_that_halts_before_the_end_is_refused() { + let max_rows = MaxRowsConfig::small(); + let (program, logs) = run("sub"); + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows).expect("the builder"); + assert!(builder.push(&logs).is_err()); +} + +/// A streamed chunk only goes back into a placeholder. +#[test] +fn a_chunk_is_only_put_back_into_its_placeholder() { + let max_rows = MaxRowsConfig::small(); + let (program, logs) = run("all_instructions_64"); + let mut builder = WindowedTraceBuilder::new(&program, &[], &max_rows).expect("the builder"); + let chunks = builder.push(&logs[..logs.len() - 1]).expect("a window"); + assert!( + chunks.iter().any(|c| c.table == StreamTable::Cpu), + "a CPU chunk streams" + ); + let mut traces = builder.finish(&logs[logs.len() - 1..]).expect("the rest"); + let again: Vec<_> = chunks + .iter() + .filter(|c| c.table == StreamTable::Cpu) + .map(|c| crate::tables::trace_builder::StreamedChunk { + table: c.table, + index: c.index, + trace: c.trace.clone(), + }) + .collect(); + traces.insert_streamed(chunks).expect("first time"); + assert!(traces.insert_streamed(again).is_err()); +} + +/// One digest per table: its width, its height and its rows in order (also +/// for the six that lay their rows out in `HashMap` order: one hash state per +/// process, `tables::trace_hash`). +fn table_digests(t: &Traces) -> Vec<(String, String)> { + use rayon::prelude::*; + let mut tables: Vec<(String, &Table)> = Vec::new(); + let lists: [(&str, &Vec
); 21] = [ + ("CPU", &t.cpus), + ("MEMW_R", &t.memw_registers), + ("MEMW_A", &t.memw_aligneds), + ("MEMW", &t.memws), + ("LOAD", &t.loads), + ("STORE", &t.stores), + ("SHIFT", &t.shifts), + ("CPU32", &t.cpu32s), + ("COPY", &t.copies), + ("KECCAK", &t.keccaks), + ("KECCAK_RND", &t.keccak_rnds), + ("ECSM", &t.ecsms), + ("ECDAS", &t.ecdases), + ("HINT", &t.hints), + ("PAGE", &t.pages), + ("LT", &t.lts), + ("MUL", &t.muls), + ("DVRM", &t.dvrms), + ("BRANCH", &t.branches), + ("EQ", &t.eqs), + ("BYTEWISE", &t.bytewises), + ]; + for (name, list) in lists { + for (i, table) in list.iter().enumerate() { + tables.push((format!("{name}[{i}]"), table)); + } + } + for (name, table) in [ + ("BITWISE", &t.bitwise), + ("DECODE", &t.decode), + ("REGISTER", &t.register), + ("HALT", &t.halt), + ("KECCAK_RC", &t.keccak_rc), + ("BLAKE3", &t.blake3), + ] { + tables.push((name.to_string(), table)); + } + tables + .into_par_iter() + .map(|(name, table)| { + // Streamed, never materialized: a block's traces are tens of GiB. + let row_bytes = |r: usize| -> Vec { + table + .main_table + .get_row(r) + .iter() + .flat_map(|v| v.canonical().to_le_bytes()) + .collect() + }; + let mut h = blake3::Hasher::new(); + h.update(&(table.main_table.width as u64).to_le_bytes()); + h.update(&(table.main_table.height as u64).to_le_bytes()); + for r in 0..table.main_table.height { + h.update(&row_bytes(r)); + } + (name, h.finalize().to_hex()[..16].to_string()) + }) + .collect() +} + +/// ★ Box only (`--ignored`): on a real block, the windowed build — split across +/// a walker and an accumulator thread, its chunks generated on a third, beside a +/// thread that keeps the host's memory busy — gives the whole-run build's +/// tables, twice. Reads `BLOCK_WHIR_ELF` and `BLOCK_WHIR_INPUT`; prints +/// `DETERMINISM` lines. +#[test] +#[ignore = "a real block: box only"] +fn the_windowed_build_of_a_real_block_is_the_whole_run_build_every_time() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let elf = std::fs::read(std::env::var("BLOCK_WHIR_ELF").expect("BLOCK_WHIR_ELF")).unwrap(); + let input = + std::fs::read(std::env::var("BLOCK_WHIR_INPUT").expect("BLOCK_WHIR_INPUT")).unwrap(); + let program = Elf::load(&elf).expect("the ELF loads"); + let max_rows = MaxRowsConfig::uniform(1 << 21); + let logs = Executor::new(&program, input.clone()) + .expect("the executor starts") + .run() + .expect("the block runs") + .logs; + let reference = { + let t = Traces::from_elf_and_logs(&program, &logs, &max_rows, &input) + .expect("the whole-run build"); + table_digests(&t) + }; + println!("DETERMINISM whole-run: {} tables", reference.len()); + let window = 1usize << 20; + let body = logs.len() - 1; + let cut = body - body % window; + let mut runs_equal = 0usize; + for run_index in 0..2 { + let stop = AtomicBool::new(false); + let digests = std::thread::scope(|scope| { + // The contention: a thread copying a GiB back and forth. + let busy = scope.spawn(|| { + let mut a = vec![1u8; 1 << 30]; + let mut b = vec![0u8; 1 << 30]; + while !stop.load(Ordering::Relaxed) { + b.copy_from_slice(&a); + a.copy_from_slice(&b); + } + }); + let mut builder = + WindowedTraceBuilder::new(&program, &input, &max_rows).expect("the builder"); + let chunks = { + let (mut walker, mut accumulator) = builder.split(); + std::thread::scope(|inner| { + let (wtx, wrx) = std::sync::mpsc::sync_channel(2); + let (jtx, jrx) = std::sync::mpsc::sync_channel(64); + let logs = &logs; + inner.spawn(move || { + for w in logs[..cut].chunks(window) { + wtx.send(walker.walk(w).expect("a window")) + .expect("accumulator"); + } + }); + let generator = inner.spawn(move || { + jrx.into_iter() + .map(crate::tables::trace_builder::ChunkJob::generate) + .collect::>() + }); + for walked in wrx { + for job in accumulator.absorb(walked) { + jtx.send(job).expect("generator"); + } + } + drop(jtx); + generator.join().expect("the generator") + }) + }; + let streamed = chunks.len(); + let mut traces = builder.finish(&logs[cut..]).expect("the last window"); + traces.insert_streamed(chunks).expect("placeholders"); + stop.store(true, Ordering::Relaxed); + busy.join().expect("the busy thread"); + println!("DETERMINISM windowed run {run_index}: {streamed} chunks streamed"); + table_digests(&traces) + }); + let differing: Vec<&String> = reference + .iter() + .zip(&digests) + .filter(|(a, b)| a != b) + .map(|(a, _)| &a.0) + .collect(); + println!( + "DETERMINISM windowed run {run_index}: {} tables, {} differ from the whole-run build {:?}", + digests.len(), + differing.len(), + differing + ); + if digests.len() == reference.len() && differing.is_empty() { + runs_equal += 1; + } + } + println!("DETERMINISM RESULT: {runs_equal}/2 windowed runs equal the whole-run build"); + assert_eq!(runs_equal, 2); +} + +/// ★ The lean walk (D-EXEC E1: each cycle's decode from a dense table, memory +/// accesses found by page, the in-walk BITWISE lookups counted from the CPU and +/// LOAD ops instead of listed, the routing in one pass) builds the whole-run +/// tables, and so does the walk it replaces: at every window length, with and +/// without dropping the streamed ops (the dropping builds through the split), +/// on programs with keccak calls, narrow and odd-offset loads and stores; and +/// so does each part alone. +#[test] +fn the_lean_walk_builds_the_whole_run_tables() { + let max_rows = MaxRowsConfig::small(); + let only = |f: fn(&mut WalkLean)| { + let mut lean = WalkLean::NONE; + f(&mut lean); + lean + }; + let parts = [ + only(|l| l.memory = true), + only(|l| l.lookups = true), + only(|l| l.route = true), + ]; + for name in [ + "sub", + "all_instructions_64", + "test_keccak", + "test_keccak_multi", + "lw_sw_offset_odd", + "test_memw_split_ts", + ] { + let (program, logs) = run(name); + let reference = whole(&program, &logs, &max_rows); + for window in [1, 7, 32, 33, 1000, logs.len()] { + let mut settings = vec![WalkLean::NONE, WalkLean::ALL]; + if window == 33 && name.starts_with("test_") { + settings.extend(parts); + } + for lean in settings { + for drop in [false, true] { + let (traces, _) = + windowed_with(&program, &logs, &max_rows, window, drop, |builder| { + let builder = builder.walk_lean(lean); + if drop { + builder + .drop_streamed_ops() + .expect("before the first window") + } else { + builder + } + }); + same_traces(&reference, &traces); + } + } + } + } +} diff --git a/prover/src/tests/zf_air_cache_tests.rs b/prover/src/tests/zf_air_cache_tests.rs new file mode 100644 index 000000000..20bd443d4 --- /dev/null +++ b/prover/src/tests/zf_air_cache_tests.rs @@ -0,0 +1,84 @@ +//! The AIR prototype cache (`test_utils::build_air`) must key the proof +//! FORMAT: an AIR built for one format and asked for under another is a +//! different verifier. +//! +//! A key of `(name, blowup, queries, coset, grinding, final degree)` alone — +//! every `ProofOptions` field except `format` — lets the first AIR built in a +//! process fix the format of every later AIR with the same name and +//! parameters, whatever format the caller asks for. Two test failures come +//! from exactly that: +//! +//! - `merkle_cap_vm` (`LAMBDA_VM_ZF_CAP=auto`): the capped prove caches capped +//! AIRs, so `verify_with_options(.., &default, ..)` verifies the capped proof +//! with those capped AIRs and accepts it. +//! - `zf_vm_dp_tests`: in a fresh process the dp prove caches dp AIRs and the +//! "default" verifier accepts the dp proof; in the lib suite an earlier test +//! has cached default AIRs, so the dp prove proves at `pair` and the +//! non-vacuity assertion fires. +//! +//! The options used here carry a query count no other test uses, so these +//! keys are this test's alone however the suite interleaves. + +use stark::proof::options::{CapPolicy, FriMode, ProofFormat, ProofOptions}; +use stark::traits::AIR; + +use crate::test_utils::{create_cpu_air, create_halt_air}; + +/// A query count no other test builds AIRs with. +const PRIVATE_QUERIES: usize = 47; + +fn base() -> ProofOptions { + ProofOptions { + fri_number_of_queries: PRIVATE_QUERIES, + ..ProofOptions::default_test_options() + } +} + +fn formats() -> Vec { + vec![ + ProofFormat { + merkle_cap: CapPolicy::Auto, + ..ProofFormat::DEFAULT + }, + ProofFormat { + fri_mode: FriMode::Dp, + ..ProofFormat::DEFAULT + }, + ProofFormat { + merkle_cap: CapPolicy::Fixed(2), + fri_mode: FriMode::Dp, + ..ProofFormat::DEFAULT + }, + ProofFormat::DEFAULT, + ] +} + +/// Every format asked for is the format handed back, in both build orders +/// (non-default first, then default; and the reverse through a second AIR), +/// and asking again (a cache hit) changes nothing. +#[test] +fn the_air_prototype_cache_keys_the_proof_format() { + let options = |format: ProofFormat| ProofOptions { format, ..base() }; + // HALT: non-default formats first, the default last. + for _round in 0..2 { + for format in formats() { + let air = create_halt_air(&options(format)); + assert_eq!( + air.options().format, + format, + "HALT built for {format:?} carries another format" + ); + } + } + // CPU (a constraint-bearing AIR): the default first, then the rest. + for _round in 0..2 { + for format in formats().into_iter().rev() { + let air = create_cpu_air(&options(format), 0); + assert_eq!( + air.options().format, + format, + "CPU built for {format:?} carries another format" + ); + } + } +} diff --git a/prover/src/tests/zf_lfm_bytes_tests.rs b/prover/src/tests/zf_lfm_bytes_tests.rs new file mode 100644 index 000000000..5b1191e96 --- /dev/null +++ b/prover/src/tests/zf_lfm_bytes_tests.rs @@ -0,0 +1,236 @@ +//! Device parity at the proof level, on an LFM machine proof. +//! +//! An RV64 VM proof is not a usable cross-build oracle: it is not a function +//! of the ELF and the format alone. Six base-table builders dedup through a std +//! `HashMap` (`RandomState`) and lay rows out in iteration order, so the main +//! roots — and with them the whole transcript — change from process to process +//! (the same build differs from itself). A cross-build `cmp` of such bytes +//! means nothing. +//! +//! This test proves an LFM machine program instead: its trace is a function of +//! the program and the arenas, the proof is made at grinding 0 (no host nonce +//! search), and each format is proved TWICE in the same process with the two +//! byte strings asserted equal (the in-run determinism control), so a +//! cross-build `cmp` of the written files means "the device proof is the CPU +//! proof". The program (`TrivialV0`) has public outputs, so the statement the +//! transcript absorbs — and the LogUp balance through `z`, `α` — depends on the +//! proof actually being the one the verifier replays. +//! +//! Files: `$ZF_S2_PROOF_DIR/{cpu,cuda}_{format}.rkyv` for four formats: +//! - `legacy`: every lever off; +//! - `one_row_1`: legacy + one row on every chip; +//! - `production`: the STARK levers of the prover's default format +//! ([`crate::zf_format::ZfFormat::DEFAULT`]: cap auto, fri dp, one row off), +//! asserted equal to what that default stamps so the two cannot drift; +//! - `all_levers`: every STARK lever on (cap auto, fri dp, one_row auto). +//! +//! Under cuda the `one_row_1` proof must build one-row trees on the device and +//! take the one-row device FRI commit (a silent host fallback would still give +//! equal bytes, so the counters are what make the comparison mean "device"); +//! run with `LAMBDA_VM_GPU_LDE_THRESHOLD` low enough that the LFM tables cross +//! it (the box line sets 1024). + +use stark::proof::options::{FriMode, OneRowMode, ProofFormat, ProofOptions}; + +use crate::lfm::proof::{lfm_prove, verify_against_artifacts}; +use crate::lfm::registry::{LfmProgramKind, build_artifacts}; +use crate::lfm::word::LfmWord; +use crate::tables::types::FE; + +/// The four formats compared across builds. +fn formats() -> [(&'static str, ProofFormat); 4] { + let legacy = ProofFormat { + merkle_cap: crypto::merkle_tree::cap::CapPolicy::Off, + fri_mode: FriMode::Pair, + one_row: OneRowMode::Off, + fri_schedule_override: None, + merge_equal_height_tables: false, + }; + let production = ProofFormat { + merkle_cap: crypto::merkle_tree::cap::CapPolicy::Auto, + fri_mode: FriMode::Dp, + one_row: OneRowMode::Off, + fri_schedule_override: None, + merge_equal_height_tables: true, + }; + assert_eq!( + production, + crate::zf_format::ZfFormat::DEFAULT.proof_format(), + "the `production` arm must be the STARK part of the prover's default format" + ); + [ + ("legacy", legacy), + ( + "one_row_1", + ProofFormat { + one_row: OneRowMode::On, + ..legacy + }, + ), + ("production", production), + ( + "all_levers", + ProofFormat { + one_row: OneRowMode::Auto, + // The LFM merge has no one-row layout: beside one-row openings + // the library builds none (`registry::merge_plan_for`), so it is + // spelled off here as the format parser would read it. + merge_equal_height_tables: false, + ..production + }, + ), + ] +} + +fn options(format: ProofFormat) -> ProofOptions { + // Blowup 4 (the blowup the one-row static twins ship for), 128-bit + // queries with NO grinding: the proof is then a function of the program, + // the arenas and the format. + let mut o = stark::proof::options::GoldilocksCubicProofOptions::with_params(4, 128, 0) + .expect("valid options"); + assert_eq!(o.grinding_factor, 0); + o.format = format; + o +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +#[test] +#[ignore = "box: set ZF_S2_PROOF_DIR, run twice in a CPU build and twice in a cuda build, then cmp the files"] +fn lfm_proof_bytes_for_the_device_comparison() { + let dir = std::env::var("ZF_S2_PROOF_DIR").expect("set ZF_S2_PROOF_DIR"); + std::fs::create_dir_all(&dir).expect("create ZF_S2_PROOF_DIR"); + let build = if cfg!(feature = "cuda") { + "cuda" + } else { + "cpu" + }; + let kind = LfmProgramKind::TrivialV0; + let program = kind.program(); + let arenas = arenas(); + for (name, format) in formats() { + let o = options(format); + let artifacts = build_artifacts(&program, &o); + match format.one_row { + OneRowMode::On => assert!(artifacts.one_row_roots.is_some(), "{name}: one-row roots"), + OneRowMode::Off => assert!( + artifacts.one_row_roots.is_none(), + "{name}: no one-row roots" + ), + _ => {} + } + #[cfg(feature = "cuda")] + let (trees0, fri0, all_fri0) = ( + stark::gpu_lde::gpu_one_row_trees(), + stark::gpu_lde::gpu_one_row_fri_calls(), + stark::gpu_lde::gpu_fri_calls(), + ); + let mut runs: Vec> = Vec::with_capacity(2); + let mut last = None; + for _ in 0..2 { + let proved = lfm_prove(&program, &artifacts, &arenas, &o) + .unwrap_or_else(|e| panic!("{name}: the LFM program must prove: {e:?}")); + assert!( + !proved.public_words.is_empty(), + "{name}: the program publishes words" + ); + runs.push( + rkyv::to_bytes::(&proved.proof) + .expect("rkyv") + .to_vec(), + ); + last = Some(proved); + } + #[cfg(feature = "cuda")] + let (trees, fri, all_fri) = ( + stark::gpu_lde::gpu_one_row_trees() - trees0, + stark::gpu_lde::gpu_one_row_fri_calls() - fri0, + stark::gpu_lde::gpu_fri_calls() - all_fri0, + ); + let proved = last.expect("proved twice"); + assert_eq!( + runs[0], runs[1], + "{name}: the same LFM proof, proved twice in one process, must be byte-identical \ + (otherwise a cross-build cmp is not an oracle)" + ); + let bytes = &runs[0]; + let path = std::path::Path::new(&dir).join(format!("{build}_{name}.rkyv")); + std::fs::write(&path, bytes).expect("write the proof bytes"); + // After the file is written, so a verify failure still leaves the + // bytes for the cross-build cmp. The balance depends on z, α through + // the published words, so this is the Phase-A replay too. + assert!( + verify_against_artifacts(&artifacts, &proved.proof, &proved.public_words, &o), + "{name}: an honest LFM proof must verify" + ); + + let tables: Vec = proved + .proof + .proofs + .iter() + .map(|p| { + let one_row = p.deep_poly_openings[0] + .main_trace_polys + .evaluations_sym + .is_empty(); + format!("{}{}", p.trace_length, if one_row { "r" } else { "p" }) + }) + .collect(); + let one_row_tables = tables.iter().filter(|t| t.ends_with('r')).count(); + println!( + "ZF LFMBYTES {build} {name}: {} bytes, twice equal, {one_row_tables} of {} tables one-row \ + [rows: {}] -> {}", + bytes.len(), + tables.len(), + tables.join(" "), + path.display() + ); + if format.one_row == OneRowMode::On { + assert_eq!(one_row_tables, tables.len(), "{name}: every chip one-row"); + } + if format.one_row == OneRowMode::Off { + assert_eq!(one_row_tables, 0, "{name}: no chip one-row"); + } + #[cfg(feature = "cuda")] + { + println!( + "ZF LFM DEVICE {name}: {all_fri} device FRI commits, {trees} one-row device trees, \ + {fri} one-row device FRI commits (two proofs)" + ); + // The two large chips (2^16 and 2^20 rows, LDE >= 2^18) are above + // the default device floor in every format. + assert!( + all_fri > 0, + "{name}: no FRI commit reached the device: the proof would be a host proof" + ); + if format.one_row == OneRowMode::On { + assert!( + trees > 0 && fri > 0, + "{name}: no one-row tree or FRI commit reached the device \ + ({trees} trees, {fri} FRI commits): the proof would be a host proof \ + (lower LAMBDA_VM_GPU_LDE_THRESHOLD)" + ); + } + if format.one_row == OneRowMode::Off { + assert_eq!( + trees + fri, + 0, + "{name}: no one-row device work without one row" + ); + } + println!( + "ZF LFM DEVMEM {name}: largest one-row tree {} B; device fallbacks {}; \ + reserved high water {} B", + stark::gpu_lde::gpu_one_row_tree_peak_bytes(), + math_cuda::device::device_fallbacks(), + math_cuda::device::reserved_high_water() + ); + } + } +} diff --git a/prover/src/tests/zf_p1_device_tests.rs b/prover/src/tests/zf_p1_device_tests.rs new file mode 100644 index 000000000..ebac08c01 --- /dev/null +++ b/prover/src/tests/zf_p1_device_tests.rs @@ -0,0 +1,132 @@ +//! The Poseidon1 STARK configuration (`P1StarkHash`: 4-ary trees, caps at +//! 4-ary height `P1_STARK_CAP`) on the device: the P1 twins of +//! `zf_rpx_device_tests`, and the P1 goldens proved on the card. +//! +//! Every test needs a GPU (`#[ignore]`d); the FRI parity cases also a lowered +//! `LAMBDA_VM_GPU_LDE_THRESHOLD`, and the goldens one at or below 4096 so +//! every case commits on the card: +//! +//! ```text +//! LAMBDA_VM_GPU_LDE_THRESHOLD=2 cargo test --release -p lambda-vm-prover --features cuda \ +//! --lib tests::zf_p1_device_tests::parity_ -- --ignored +//! cargo test --release -p lambda-vm-prover --features cuda \ +//! --lib tests::zf_p1_device_tests::trees_one_row_p1 -- --ignored --exact +//! LAMBDA_VM_GPU_LDE_THRESHOLD=1024 cargo test --release -p lambda-vm-prover --features cuda \ +//! --lib tests::zf_p1_device_tests::proved_p1_goldens_equal_the_host_bytes \ +//! -- --ignored --exact --test-threads=1 +//! ``` + +use stark::fri::device_parity::{ + Case, legacy_cases, production_cases, resident_cases, run_cases, sweep_cases, +}; + +use crate::lfm::p1_commit::P1StarkHash; + +fn check(cases: &[Case], resident: bool, seed: u64) { + if let Err(failures) = run_cases::("p1", cases, resident, seed) { + panic!("p1: {failures:#?}"); + } +} + +/// Every distinct DP schedule (the stark crate's pinned 29 shapes): 4-ary +/// group trees, their device paths against the host's. +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_every_dp_shape_p1() { + let cases = sweep_cases(); + assert_eq!(cases.len(), 29); + check(&cases, false, 0x5031_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_production_sizes_p1() { + check(&production_cases(), false, 0x5031_1000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_resident_layers_p1() { + check(&resident_cases(), true, 0x5031_2000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_legacy_encoding_p1() { + check(&legacy_cases(), false, 0x5031_3000); +} + +/// Main, preprocessed-split, aux and composition trees committed on the card +/// at both leaf layouts, and their device-gathered paths, against the host. +#[test] +#[ignore = "requires a GPU; run with --features cuda -- --ignored"] +fn trees_one_row_p1() { + if let Err(failures) = stark::s2_device_parity::run_tree_parity::("p1") { + panic!("p1: {failures:#?}"); + } +} + +/// The P1 goldens (`p1_stark_tests::P1_GOLDENS`, the host's bytes) proved on +/// the card: the 4-ary trace, aux, composition and FRI trees committed there, +/// every opening gathered off the resident trees and every cap read off them, +/// and the proof bytes equal to the host's. Each case must take the device +/// FRI commit, a device trace tree and a device composition tree, and gather +/// its openings and caps on the device; a host fallback fails the test. Run +/// alone: the counters are process-wide. +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD<=4096; run alone with --features cuda -- --ignored --exact --test-threads=1"] +fn proved_p1_goldens_equal_the_host_bytes() { + use super::p1_stark_tests::{P1_GOLDENS, p1_golden_cases, p1_goldens}; + use stark::gpu_lde::{ + gpu_cap_read_calls, gpu_comp_poly_tree_calls, gpu_fri_calls, gpu_merkle_tree_calls, + gpu_opening_gather_calls, + }; + let counts = || { + [ + gpu_fri_calls(), + gpu_merkle_tree_calls(), + gpu_comp_poly_tree_calls(), + gpu_opening_gather_calls(), + gpu_cap_read_calls(), + ] + }; + let cases = p1_golden_cases().len(); + let before = counts(); + let got = p1_goldens(); + let after = counts(); + let moved: Vec = after.iter().zip(before).map(|(a, b)| a - b).collect(); + println!( + "P1DEV goldens: {} cases; device FRI commits {}, trace trees {}, composition trees {}, \ + opening gathers {}, cap reads {}", + got.len(), + moved[0], + moved[1], + moved[2], + moved[3], + moved[4] + ); + for (what, &n) in [ + "FRI commits", + "trace trees", + "composition trees", + "opening gathers", + "cap reads", + ] + .iter() + .zip(&moved) + { + assert!( + n >= cases as u64, + "every P1 golden case must take the device {what} ({n} for {cases} cases; lower \ + LAMBDA_VM_GPU_LDE_THRESHOLD)" + ); + } + assert_eq!(got.len(), P1_GOLDENS.len(), "one pin per case"); + for ((name, digest), (pin_name, pin)) in got.iter().zip(P1_GOLDENS) { + assert_eq!(name, pin_name); + assert_eq!( + digest, pin, + "{name}: the card's P1 proof differs from the host's bytes" + ); + } +} diff --git a/prover/src/tests/zf_rpx_device_tests.rs b/prover/src/tests/zf_rpx_device_tests.rs new file mode 100644 index 000000000..10467e071 --- /dev/null +++ b/prover/src/tests/zf_rpx_device_tests.rs @@ -0,0 +1,176 @@ +//! S3 on the device under the production RPX pin: the RPX +//! twins of the stark crate's `tests::zf_fri_device_tests` (which cover Keccak +//! and Blake3; the stark crate cannot name `RpxStarkHash`). +//! +//! Every `#[ignore]`d test needs a GPU and a lowered +//! `LAMBDA_VM_GPU_LDE_THRESHOLD`, and fails when the device path does not run: +//! +//! ```text +//! LAMBDA_VM_GPU_LDE_THRESHOLD=2 cargo test --release -p lambda-vm-prover --features cuda \ +//! --lib tests::zf_rpx_device_tests::parity_ -- --ignored +//! LAMBDA_VM_GPU_LDE_THRESHOLD=1024 cargo test --release -p lambda-vm-prover --features cuda \ +//! --lib tests::zf_rpx_device_tests::proved_rpx_vectors_equal_the_cpu_bytes \ +//! -- --ignored --exact --test-threads=1 +//! ``` +//! +//! S2 on the device: `trees_one_row_rpx` (default threshold), +//! `fri_one_row_*` (threshold 2), `proved_rpx_one_row_vectors_equal_the_cpu_bytes` +//! (threshold 1024, alone), the RPX twins of `stark`'s `tests::zf_s2_device_tests`. + +use stark::fri::device_parity::{ + Case, legacy_cases, production_cases, resident_cases, run_cases, sweep_cases, +}; + +use crate::lfm::algebraic_commit::RpxStarkHash; + +fn check(cases: &[Case], resident: bool, seed: u64) { + if let Err(failures) = run_cases::("rpx", cases, resident, seed) { + panic!("rpx: {failures:#?}"); + } +} + +/// Every distinct DP schedule for B ≤ 23 plus the extra shapes (the list is +/// pinned by the stark crate's `dp_shapes_are_pinned`: 29 shapes). +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_every_dp_shape_rpx() { + let cases = sweep_cases(); + assert_eq!(cases.len(), 29); + check(&cases, false, 0x5a46_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_production_sizes_rpx() { + check(&production_cases(), false, 0x5a47_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_resident_layers_rpx() { + check(&resident_cases(), true, 0x5a48_0000); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn parity_legacy_encoding_rpx() { + check(&legacy_cases(), false, 0x5a49_0000); +} + +/// The RPX (d) vector proofs (`pair`, `dp`, `dp_3_1_3`, `cap_pair`, `cap_dp`; +/// LDE 4096) proved on +/// the device path are byte-identical to the checked-in CPU-proved files. The +/// device FRI counter must move once per proof. Run alone: the counter is +/// process-wide. +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD<=4096; run alone with --features cuda -- --ignored --exact --test-threads=1"] +fn proved_rpx_vectors_equal_the_cpu_bytes() { + use stark::fri::vectors::{check_or_write, proof_vectors}; + let before = stark::gpu_lde::gpu_fri_calls(); + let comp_before = stark::gpu_lde::gpu_composition_calls(); + let files = proof_vectors::("rpx"); + let device_commits = stark::gpu_lde::gpu_fri_calls() - before; + let compositions = stark::gpu_lde::gpu_composition_calls() - comp_before; + println!( + "FRIDEV rpx vector proofs: {} files, {device_commits} device FRI commits, {compositions} device compositions", + files.len() + ); + // Five (d) formats (pair, dp, dp_3_1_3 at Q = 3; cap_pair, cap_dp at + // Q = 20), two files and one FRI commit per proof. + assert_eq!(files.len(), 5 * 2); + assert_eq!( + device_commits, 5, + "every vector proof must take the device FRI commit (lower LAMBDA_VM_GPU_LDE_THRESHOLD)" + ); + // Every proof composes on the device (the AIR's constraint program); a + // host composition would not be counted here. + assert_eq!( + compositions, 5, + "every RPX vector proof must compose on the device ({compositions} device compositions)" + ); + let bad = check_or_write(&files, false); + assert!( + bad.is_empty(), + "device-proved RPX vectors differ from the checked-in CPU bytes: {bad:?}" + ); +} + +// --------------------------------------------------------------------------- +// S2 on the device under the RPX pin. +// --------------------------------------------------------------------------- + +/// One-row main / preprocessed split / aux (host and resident) / composition +/// trees and their device openings against the host (the RPX twin of +/// `stark`'s `trees_one_row_*`). +#[test] +#[ignore = "requires a GPU; run with --features cuda -- --ignored"] +fn trees_one_row_rpx() { + if let Err(failures) = stark::s2_device_parity::run_tree_parity::("rpx") { + panic!("rpx: {failures:#?}"); + } +} + +/// The one-row FRI commit (input tree from the codeword, then the group chain) +/// and query phases against the host CPU loop. +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn fri_one_row_rpx() { + check( + &stark::fri::device_parity::one_row_cases(), + false, + 0x5234_0000, + ); +} + +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD=2; run with --features cuda -- --ignored"] +fn fri_one_row_resident_rpx() { + check( + &stark::fri::device_parity::one_row_resident_cases(), + true, + 0x5235_0000, + ); +} + +/// The RPX (e) vector proofs (`one_row_pair`, `one_row_3_2_1_2`; LDE 4096) +/// proved on the device path are byte-identical to the checked-in CPU-proved +/// files. Each proof must take the one-row device FRI commit and build at +/// least its main, aux and composition trees one-row on the device. Run alone: +/// the counters are process-wide. +#[test] +#[ignore = "requires a GPU and LAMBDA_VM_GPU_LDE_THRESHOLD<=4096; run alone with --features cuda -- --ignored --exact --test-threads=1"] +fn proved_rpx_one_row_vectors_equal_the_cpu_bytes() { + use stark::fri::vectors::{check_or_write, one_row_proof_vectors}; + let fri_before = stark::gpu_lde::gpu_one_row_fri_calls(); + let trees_before = stark::gpu_lde::gpu_one_row_trees(); + let comp_before = stark::gpu_lde::gpu_composition_calls(); + let files = one_row_proof_vectors::("rpx"); + let fri_commits = stark::gpu_lde::gpu_one_row_fri_calls() - fri_before; + let trees = stark::gpu_lde::gpu_one_row_trees() - trees_before; + let compositions = stark::gpu_lde::gpu_composition_calls() - comp_before; + println!( + "S2DEV rpx vector proofs: {} files, {fri_commits} one-row device FRI commits, {trees} one-row device trees, {compositions} device compositions", + files.len() + ); + assert_eq!(files.len(), 2 * 2); + assert_eq!( + fri_commits, 2, + "every one-row vector proof must take the device FRI commit (lower LAMBDA_VM_GPU_LDE_THRESHOLD)" + ); + assert!( + trees >= 3 * 2, + "every one-row vector proof must build its main, aux and composition trees on the device \ + ({trees} one-row device trees for 2 proofs)" + ); + // Every proof composes on the device (the AIR's constraint program); a + // host composition would not be counted here. + assert_eq!( + compositions, 2, + "every one-row RPX vector proof must compose on the device ({compositions} device compositions)" + ); + let bad = check_or_write(&files, false); + assert!( + bad.is_empty(), + "device-proved one-row RPX vectors differ from the checked-in CPU bytes: {bad:?}" + ); +} diff --git a/prover/src/tests/zf_rpx_golden_tests.rs b/prover/src/tests/zf_rpx_golden_tests.rs new file mode 100644 index 000000000..e750ce706 --- /dev/null +++ b/prover/src/tests/zf_rpx_golden_tests.rs @@ -0,0 +1,445 @@ +//! Golden proofs under the production RPX pin, in TWO formats: +//! +//! - the LEGACY format (every ZF lever off; `ProofFormat::LEGACY`, the stark +//! crate's default): the RPX half of `stark::tests::zf_golden_tests` (which +//! covers Keccak and Blake3 and cannot name `RpxStarkHash`, a prover-crate +//! type). It keeps the legacy bytes pinned after the default flip, so +//! the rollback arm (every knob off) is still checked against bytes, not +//! against a round trip; +//! - the PRODUCTION default (`ZfFormat::DEFAULT.proof_format()`): +//! the bytes every production site stamps. Regenerate only for a +//! deliberate format change. +//! +//! Each case proves a small in-repo AIR at `grinding_factor = 0` (so the bytes +//! are reproducible) and pins the SHA-256 of the proof's rkyv bytes plus, so a +//! failure says where the drift is, the digests of its FRI layer roots, terminal +//! coefficients, FRI decommitments and trace/composition openings. The legacy +//! pins were generated at the (then default) legacy format before any S3 prover +//! code existed; regenerate either set only for a deliberate format change: +//! `cargo test -p lambda-vm-prover --lib tests::zf_rpx_golden_tests::print_goldens -- --ignored --nocapture`. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use sha2::{Digest, Sha256}; +use stark::examples::bus_permutation::{bus_permutation_air, bus_permutation_trace}; +use stark::examples::read_only_memory_logup::{ + LogReadOnlyPublicInputs, LogReadOnlyRAP, read_only_logup_trace, +}; +use stark::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use stark::proof::options::{ProofFormat, ProofOptions}; +use stark::proof::stark::StarkProof; +use stark::prover::{GenericProver, IsStarkProver}; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::{GenericVerifier, IsStarkVerifier}; + +use crate::lfm::algebraic_commit::RpxStarkHash; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type Felt = FieldElement; + +pub(crate) fn options(blowup: u8, k: u8, queries: usize, format: ProofFormat) -> ProofOptions { + ProofOptions { + blowup_factor: blowup, + fri_number_of_queries: queries, + coset_offset: 3, + grinding_factor: 0, + fri_final_poly_log_degree: k, + format, + } +} + +fn hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +macro_rules! fingerprint { + ($proof:expr) => {{ + let proof = $proof; + let rk = + |bytes: Result| hex(&bytes.expect("rkyv")); + format!( + "proof {} roots[{}] {} coeffs {} queries {} openings {}", + rk(rkyv::to_bytes::(proof)), + proof.fri_layers_merkle_roots.len(), + hex(&proof.fri_layers_merkle_roots.concat()), + rk(rkyv::to_bytes::( + &proof.fri_final_poly_coeffs + )), + rk(rkyv::to_bytes::(&proof.query_list)), + rk(rkyv::to_bytes::( + &proof.deep_poly_openings + )), + ) + }}; +} + +pub(crate) fn prove_simple_addition( + rows: usize, + o: &ProofOptions, +) -> ( + SimpleAdditionAIR, + StarkProof>, +) { + let air = SimpleAdditionAIR::::new(o); + let pi = SimpleAdditionPublicInputs { + a: Felt::from(1u64), + b: Felt::from(2u64), + }; + let mut trace = simple_addition_trace::(rows); + let proof = GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof) +} + +pub(crate) fn verify_simple_addition( + air: &SimpleAdditionAIR, + proof: &StarkProof>, +) -> bool { + GenericVerifier::::verify( + proof, + air, + &mut DefaultTranscript::::new(&[]), + ) +} + +pub(crate) fn prove_logup( + rows: usize, + o: &ProofOptions, +) -> ( + LogReadOnlyRAP, + StarkProof>, +) { + let addr: Vec = (0..rows).map(|i| Felt::from((i % 5) as u64 + 1)).collect(); + let val: Vec = (0..rows) + .map(|i| Felt::from(((i % 5) as u64 + 1) * 10)) + .collect(); + let mut trace: TraceTable = read_only_logup_trace(addr, val); + let cols = trace.columns_main(); + let pi = LogReadOnlyPublicInputs { + a0: cols[0][0], + v0: cols[1][0], + a_sorted_0: cols[2][0], + v_sorted_0: cols[3][0], + m0: cols[4][0], + }; + let air = LogReadOnlyRAP::::new(o); + let proof = GenericProver::::prove( + &air, + &mut trace, + &pi, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed"); + (air, proof) +} + +pub(crate) fn verify_logup( + air: &LogReadOnlyRAP, + proof: &StarkProof>, +) -> bool { + GenericVerifier::::verify( + proof, + air, + &mut DefaultTranscript::::new(&[]), + ) +} + +fn compute_goldens() -> Vec<(String, String)> { + compute_goldens_at(ProofFormat::LEGACY) +} + +/// The production default's univariate format, the one the goldens below pin. +fn production_format() -> ProofFormat { + crate::zf_format::ZfFormat::DEFAULT.proof_format() +} + +fn compute_goldens_at(d: ProofFormat) -> Vec<(String, String)> { + let mut out = Vec::new(); + for (rows, blowup) in [(16usize, 2u8), (64, 4)] { + let o = options(blowup, 2, 5, d); + let (air, proof) = prove_simple_addition(rows, &o); + assert!(verify_simple_addition(&air, &proof)); + out.push(( + format!("simple_addition/rpx/rows{rows}/blowup{blowup}"), + fingerprint!(&proof), + )); + } + for (rows, blowup) in [(32usize, 4u8), (128, 2)] { + let o = options(blowup, 1, 7, d); + let (air, proof) = prove_logup(rows, &o); + assert!(verify_logup(&air, &proof)); + out.push(( + format!("logup/rpx/rows{rows}/blowup{blowup}"), + fingerprint!(&proof), + )); + } + out +} + +/// PRODUCTION-default pins, generated by `print_goldens` at the default flip. +const PRODUCTION_GOLDENS: &[(&str, &str)] = &[ + ( + "simple_addition/rpx/rows16/blowup2", + "proof ee8ca7ebe2cd632fd40d3242450377f17f966c85d35ad69dc11d918a61a12fa9 roots[1] 19764f49df000e57080b4eada26d3d1d3b4d8a7356fe4fa0a779458ffcc0cc94 coeffs c71ca99567bf64cd75e4d2ca5a68533bd196fe44545180370ac90b29cd062b9b queries 4fb6a8d93089bd818a6a5a8b0026fe133491446029263b8d89cf3feec24f254a openings f5700d0bf2e5a02d28b973177bd7828d215bbabaa9c4c2a9c5ac59fb8475f293", + ), + ( + "simple_addition/rpx/rows64/blowup4", + "proof 961d5e394cf9913b261fd255b1b2ebd9d9304560a302cd25c6126a62c99728e4 roots[1] e2a8a7ce17b0c97ec91d741f84349494d7344943a84b0feb3a0fb93a43576846 coeffs bb0d9b495382e02c0b4ac8d0d3fca25bccacc363b6433ec1459c3ba4e26f81d0 queries a636ad70b084ad76ec0dcb2c9d904fe582d12e7379f0bb80541b81abf3febdc0 openings 090f06c93b8a9220d6f7d6dbb63302e708a513be939d85d12fd64ceda96da3a7", + ), + ( + "logup/rpx/rows32/blowup4", + "proof 67c9013a35ae703146e41c999cf082433031067576be62f0dd75b8adfac1fcbf roots[1] 26365ee78c3be44e7d96f1e77a2afcc1747e3736153693877bfc4f9dd6a88dff coeffs 7d98df3343a174fe6add0b9188e592bb5ad84b5797385185ff216f887bc5499d queries 57e99bd603476d6df18dedefa00dc727af256bd36ec91b0bc00773a77e6e739a openings dcb750dd32ae77ecc0b1928369db0bc1ee0ed3f1cad5bb0c4f5b7c258a3cf264", + ), + ( + "logup/rpx/rows128/blowup2", + "proof 978ebf4ab14b4f86c378642e53c3c9ab2cdae7732b2fdd09bd07ec90549ba8fb roots[2] 25c71fa19482dd430d9ce9b413bddf5f905f77a11ba5c9c38f32f64b9cb3738e coeffs 7d4fb0cbc585c68b5422bc19757d3d2151a7dde0f5edd29e312deaecb21428df queries 9281910f6451083a9a3ffc0c12ff31d952e3f718cd6a53022fe38c60d7b791b3 openings d0c9d124a01ace817bf58f0ecb4acb9c51b5ffecca32c04402af9de972856f82", + ), +]; + +/// LEGACY-format pins (every lever off), generated before any S3 prover code. +const GOLDENS: &[(&str, &str)] = &[ + ( + "simple_addition/rpx/rows16/blowup2", + "proof 76e4be044a53802e20b5a79c6fddea37893d4c7ba575d5b1b2d6007d6e741191 roots[1] 19764f49df000e57080b4eada26d3d1d3b4d8a7356fe4fa0a779458ffcc0cc94 coeffs c71ca99567bf64cd75e4d2ca5a68533bd196fe44545180370ac90b29cd062b9b queries 1db5f91a7ffd2fa4786315d948666c225859f74b79f97b0eee756531f12c4e29 openings a7254eb12c3eb00518026d8245f76c9c7283090720cf1aacf491af1edbb1a4df", + ), + ( + "simple_addition/rpx/rows64/blowup4", + "proof 54f04bbe46b0330480daf71af2fcafa1fc00dbf698fa538e9168d3c17ae253a9 roots[3] 136c65bbe688080ed90357acda8a3896f8fb8e93e6ab897590684c3d2e5e745d coeffs 51ee895a33735700296d7a776ff1ca28bf7892e7052c28e52a843ebecea5aa2e queries 45c4ec6e74ea93b1a8c98a912b1790936b6856f0884fcf92fcbb6d9be8872b2f openings dc4496a5b2413db451381eddc74765987fb5dd745f6adbcf6ace3814be57717f", + ), + ( + "logup/rpx/rows32/blowup4", + "proof 891ec879640f6a01244829495336a41dfb587214304e4420e2c47f619c3f4d03 roots[3] ec74768e299f79c15f92be8adaa27c38719ee5811fcbf07220163aa5a6d7bacc coeffs afb0be8e7a6b95223d78e5997d681828de2fcefe22704303fc5876b1e3a07fe2 queries cc4f54c61d2c1ad5bbf965c1eb622faec313ed0e8987babea88d9c2ac3cabe87 openings 720412c24f099a3897074effe7f9248cfbd0d370d7ca7cad49d7c794b2ffaf0a", + ), + ( + "logup/rpx/rows128/blowup2", + "proof 16cdff91c22119e7a5bd35be33d0a0e33c09413aba833c1ef4ba48b64bc03b0c roots[5] a928041f346311a1bd49ef81f791370075006cdce88efc45ed5c1608071e5d72 coeffs 709758625cbc3ce3eb8b0f6859198181e95484b5183965163762a3ac4a29852d queries b70a32fb34b697580ddfc50e9a7ac5b29271336301926b0bf62b75242d6c02bf openings 8832140f1efd004ebcd1b70f50a5680abb6d1ede4ca2120d3a826edf6fdef692", + ), +]; + +/// The LEGACY-format RPX goldens: the legacy bytes, unmoved by the +/// default flip. +#[test] +fn legacy_format_rpx_goldens_are_byte_identical() { + let got = compute_goldens(); + assert_eq!(got.len(), GOLDENS.len(), "one pin per case"); + for ((name, line), (pin_name, pin_line)) in got.iter().zip(GOLDENS) { + assert_eq!(name, pin_name); + assert_eq!( + line, pin_line, + "{name}: the legacy-format RPX proof moved (a field whose digest differs is where)" + ); + } +} + +/// The PRODUCTION-default RPX goldens (cap auto, `fri=dp`, and +/// whatever `ZfFormat::DEFAULT` stamps). A move here is a production format +/// change. +#[test] +fn production_format_rpx_goldens_are_byte_identical() { + let f = production_format(); + assert!( + !f.is_legacy(), + "the production default is not the legacy format" + ); + let got = compute_goldens_at(f); + assert_eq!(got.len(), PRODUCTION_GOLDENS.len(), "one pin per case"); + for ((name, line), (pin_name, pin_line)) in got.iter().zip(PRODUCTION_GOLDENS) { + assert_eq!(name, pin_name); + assert_eq!( + line, pin_line, + "{name}: the production-default RPX proof moved (a field whose digest differs is where)" + ); + } + // The two formats' proofs differ: the production pins are not the legacy + // ones under another name. + for ((_, a), (_, b)) in GOLDENS.iter().zip(PRODUCTION_GOLDENS) { + assert_ne!(a, b); + } +} + +#[test] +#[ignore = "generator for GOLDENS (legacy) and PRODUCTION_GOLDENS"] +fn print_goldens() { + for (name, line) in compute_goldens() { + println!("GOLDEN (\"{name}\", \"{line}\"),"); + } + println!("production format: {:?}", production_format()); + for (name, line) in compute_goldens_at(production_format()) { + println!("PRODUCTION GOLDEN (\"{name}\", \"{line}\"),"); + } +} + +// --------------------------------------------------------------------------- +// S3 (fri = dp) round trips under the production RPX pin: group leaves are +// hashed by the algebraic `Batched` sponge over 3·2^d felts, so the RPX leaf +// path of the group encoding is exercised here (the stark crate's S3 tests +// cover Keccak and Blake3). +// --------------------------------------------------------------------------- + +fn dp(schedule: Option<&[u8]>) -> ProofFormat { + ProofFormat { + fri_mode: stark::proof::options::FriMode::Dp, + fri_schedule_override: schedule + .map(|s| stark::proof::options::FriScheduleOverride::new(s).expect("fits")), + ..ProofFormat::LEGACY + } +} + +#[test] +fn rpx_dp_round_trips() { + // SimpleAddition 2^9 rows, blowup 4, k 1: the chain covers 10 → 3. + for sched in [None, Some(&[3u8, 1, 3][..]), Some(&[1, 6][..])] { + let o = options(4, 1, 9, dp(sched)); + let (air, proof) = prove_simple_addition(512, &o); + assert!(verify_simple_addition(&air, &proof), "{sched:?}"); + if let Some(s) = sched { + assert_eq!(proof.fri_layers_merkle_roots.len(), s.len()); + let values: usize = s.iter().map(|&d| 1usize << d).sum(); + assert_eq!(proof.query_list[0].layers_evaluations_sym.len(), values); + } + } + // LogReadOnlyRAP (ext3 + aux) 2^7 rows, blowup 4, k 1: 8 → 3. + for sched in [None, Some(&[2u8, 3][..])] { + let o = options(4, 1, 7, dp(sched)); + let (air, proof) = prove_logup(128, &o); + assert!(verify_logup(&air, &proof), "{sched:?}"); + // A tampered group value is rejected. + let mut bad = proof.clone(); + bad.query_list[0].layers_evaluations_sym[0] += FieldElement::::one(); + assert!(!verify_logup(&air, &bad)); + } +} + +/// Under RPX, the group path at an all-ones schedule commits +/// the same layer roots, terminal polynomial and paths as the legacy pair path +/// (the `Batched`/`Pair` two-element invariant, as a tested fact for the +/// algebraic backend). +#[test] +fn rpx_group_path_at_all_ones_equals_legacy() { + // LogReadOnlyRAP 2^7 rows, blowup 4, k 1: 5 committed binary layers. + let legacy = prove_logup(128, &options(4, 1, 7, ProofFormat::LEGACY)).1; + let group = prove_logup(128, &options(4, 1, 7, dp(Some(&[1, 1, 1, 1, 1])))).1; + assert_eq!(legacy.fri_layers_merkle_roots.len(), 5); + assert_eq!( + legacy.fri_layers_merkle_roots, + group.fri_layers_merkle_roots + ); + assert_eq!(legacy.fri_final_poly_coeffs, group.fri_final_poly_coeffs); + for (l, g) in legacy.query_list.iter().zip(&group.query_list) { + for j in 0..5 { + assert_eq!( + l.layers_auth_paths[j].merkle_path, + g.layers_auth_paths[j].merkle_path + ); + assert!( + g.layers_evaluations_sym[2 * j..2 * j + 2].contains(&l.layers_evaluations_sym[j]) + ); + } + } +} + +/// The production format sites at the PROCESS format (`ZfFormat::global()`): +/// a small ext3 STARK proved and host-verified under RPX with +/// `block_base_options()` (STARK base epochs) and `aggregation_wrap_options()` +/// (every LFM proof). Without a knob it proves at the production default +/// (`ZfFormat::DEFAULT`: cap auto, `fri=dp`, group layers); the knobs select +/// the arms — `LAMBDA_VM_ZF_FRI=pair` (legacy pair layers), +/// `LAMBDA_VM_ZF_ONE_ROW=1|auto` (both sites stamp the one-row mode; a table +/// resolved to one row opens no symmetric rows and commits the FRI input). +/// Every arm proves. +#[test] +fn production_sites_prove_at_the_process_format() { + use stark::proof::options::{FriMode, OneRowMode}; + let knob = |name: &str| { + std::env::var(name) + .ok() + .map(|v| v.trim().to_ascii_lowercase()) + }; + // An unset knob is the production default's value. + let default = crate::zf_format::ZfFormat::DEFAULT; + let want = match knob(crate::zf_format::ENV_FRI).as_deref() { + Some("dp") => FriMode::Dp, + Some("pair") => FriMode::Pair, + None => default.fri, + Some(other) => panic!("unexpected {}={other}", crate::zf_format::ENV_FRI), + }; + let want_one_row = match knob(crate::zf_format::ENV_ONE_ROW).as_deref() { + Some("1") => OneRowMode::On, + Some("auto") => OneRowMode::Auto, + Some("0") => OneRowMode::Off, + None => default.one_row, + Some(other) => panic!("unexpected {}={other}", crate::zf_format::ENV_ONE_ROW), + }; + assert_eq!(crate::zf_format::ZfFormat::global().fri, want); + assert_eq!(crate::zf_format::ZfFormat::global().one_row, want_one_row); + for (site, o) in [ + ( + "block_base_options", + crate::lfm::proof::block_base_options(), + ), + ( + "aggregation_wrap_options", + crate::lfm::proof::aggregation_wrap_options(), + ), + ] { + assert_eq!(o.format.fri_mode, want, "{site}"); + assert_eq!(o.format.one_row, want_one_row, "{site}"); + // 2^12 rows: LDE 2^14, so both terminals (T = 9, 10) leave committed + // layers. At that size a `cuda` build commits on the device, whose + // composition arm needs the AIR's constraint program — so an + // `AirWithBuses` table, as in production (`LogReadOnlyRAP` panicked in + // `constraint_program` there). + let air = bus_permutation_air(&o); + let mut trace = bus_permutation_trace(1 << 12); + let proof = GenericProver::::prove( + &air, + &mut trace, + &(), + &mut DefaultTranscript::::new(&[]), + ) + .unwrap_or_else(|e| panic!("{site}: proving must succeed: {e:?}")); + assert!( + GenericVerifier::::verify( + &proof, + &air, + &mut DefaultTranscript::::new(&[]), + ), + "{site}: must verify" + ); + let layers = proof.fri_layers_merkle_roots.len(); + assert!(layers > 0, "{site}: committed layers"); + let values = proof.query_list[0].layers_evaluations_sym.len(); + let one_row = stark::leaf_layout::table_leaf_layout(&air, 1 << 12).is_one_row(); + if want_one_row == OneRowMode::On { + assert!(one_row, "{site}: one_row = 1 puts every table on one row"); + } + let sym = &proof.deep_poly_openings[0].main_trace_polys.evaluations_sym; + assert_eq!( + sym.is_empty(), + one_row, + "{site}: symmetric rows iff row pairs" + ); + println!( + "ZF SITE {site}: fri={want} one_row={want_one_row} resolved_one_row={one_row} layers={layers} values={values}" + ); + if want == FriMode::Dp || one_row { + assert!(values > layers, "{site}: group encoding"); + } else { + assert_eq!(values, layers, "{site}: legacy encoding"); + } + } +} diff --git a/prover/src/tests/zf_rpx_vectors.rs b/prover/src/tests/zf_rpx_vectors.rs new file mode 100644 index 000000000..a5e3b3c2e --- /dev/null +++ b/prover/src/tests/zf_rpx_vectors.rs @@ -0,0 +1,40 @@ +//! The exported S3 and S2 vectors ((c), (d), (e) in the README) under the production RPX pin, +//! written next to the Keccak/Blake3 ones in +//! `crypto/stark/tests/vectors/zf_fri/` (the stark crate cannot name +//! `RpxStarkHash`). Regenerated in memory and required byte-equal to the +//! checked-in files; regenerate after a deliberate format change: +//! `cargo test -p lambda-vm-prover --lib tests::zf_rpx_vectors::write_vectors -- --ignored`. + +use stark::fri::vectors::{ + VectorFile, check_or_write, leaf_digests_json, one_row_leaf_digests_json, + one_row_proof_vectors, proof_vectors, +}; + +use crate::lfm::algebraic_commit::RpxStarkHash; + +fn all() -> Vec { + let mut v = vec![leaf_digests_json::("rpx")]; + v.extend(proof_vectors::("rpx")); + // (e) S2. + v.push(one_row_leaf_digests_json::("rpx")); + v.extend(one_row_proof_vectors::("rpx")); + v +} + +#[test] +fn rpx_vectors_are_current() { + let files = all(); + assert_eq!(files.len(), 1 + 5 * 2 + 1 + 2 * 2); + let bad = check_or_write(&files, false); + assert!( + bad.is_empty(), + "stale or missing vector files {bad:?}; regenerate with \ + `cargo test -p lambda-vm-prover --lib tests::zf_rpx_vectors::write_vectors -- --ignored`" + ); +} + +#[test] +#[ignore = "writes crypto/stark/tests/vectors/zf_fri"] +fn write_vectors() { + assert!(check_or_write(&all(), true).is_empty()); +} diff --git a/prover/src/tests/zf_vm_dp_tests.rs b/prover/src/tests/zf_vm_dp_tests.rs new file mode 100644 index 000000000..1feb57414 --- /dev/null +++ b/prover/src/tests/zf_vm_dp_tests.rs @@ -0,0 +1,99 @@ +//! S3 end to end on the production VM path: a real multi-table VM proof +//! (every table the program touches, the preprocessed ones included, under the +//! RPX block pin, host CPU FRI) proved and verified at `fri = dp`. +//! +//! Proves a full VM trace (the 2^20-row BITWISE table among them), so it runs +//! in the box lib suite, not on the laptop — like its default-format sibling +//! `skip_empty_tables_tests::dropping_a_used_table_through_the_real_verifier_is_rejected`. + +use stark::proof::options::{FriMode, ProofFormat, ProofOptions}; + +#[test] +fn a_vm_proof_round_trips_at_fri_dp() { + let elf_bytes = crate::test_utils::asm_elf_bytes("test_mul_8"); + let default = ProofOptions::default_test_options(); + let dp = ProofOptions { + format: ProofFormat { + fri_mode: FriMode::Dp, + ..ProofFormat::DEFAULT + }, + ..default.clone() + }; + let vm_proof = crate::prove_with_options(&elf_bytes, &dp, &Default::default()) + .expect("the fixture must prove at fri = dp"); + assert!( + crate::verify_with_options(&vm_proof, &elf_bytes, &dp, None, None) + .expect("honest verify must not error"), + "an honest dp VM proof must verify" + ); + // Non-vacuity: some table folds a committed layer by more than 2, i.e. + // carries more opened values per query than committed layers. + assert!( + vm_proof.proof.proofs.iter().any(|p| { + let layers = p.fri_layers_merkle_roots.len(); + layers > 0 && p.query_list[0].layers_evaluations_sym.len() > 2 * layers + }), + "no table used a group of more than two values" + ); + // The format is a verifier constant: the default verifier rejects it. + assert!( + !crate::verify_with_options(&vm_proof, &elf_bytes, &default, None, None).unwrap_or(false), + "a dp proof must not verify under the default format" + ); + // A tampered FRI group value is rejected. + let mut bad = vm_proof.clone(); + let table = bad + .proof + .proofs + .iter() + .position(|p| !p.fri_layers_merkle_roots.is_empty()) + .expect("a table with committed layers"); + bad.proof.proofs[table].query_list[0].layers_evaluations_sym[0] += + math::field::element::FieldElement::< + math::field::extensions_goldilocks::Degree3GoldilocksExtensionField, + >::one(); + assert!( + !crate::verify_with_options(&bad, &elf_bytes, &dp, None, None).unwrap_or(false), + "a tampered group value must be rejected" + ); +} + +/// The same VM proof on the device path (a cuda build, the default device +/// thresholds): every table whose LDE the device admits commits its FRI +/// layers with the device group loop, and the proof still verifies. The device +/// FRI counter must move — under `dp` every device FRI commit is a group +/// commit, so a host-only run fails here. Run with `--test-threads=1`: the +/// counter is process-wide. +#[cfg(feature = "cuda")] +#[test] +fn a_vm_proof_round_trips_at_fri_dp_on_the_device() { + let elf_bytes = crate::test_utils::asm_elf_bytes("test_mul_8"); + let dp = ProofOptions { + format: ProofFormat { + fri_mode: FriMode::Dp, + ..ProofFormat::DEFAULT + }, + ..ProofOptions::default_test_options() + }; + let before = stark::gpu_lde::gpu_fri_calls(); + let vm_proof = crate::prove_with_options(&elf_bytes, &dp, &Default::default()) + .expect("the fixture must prove at fri = dp on the device path"); + let device_commits = stark::gpu_lde::gpu_fri_calls() - before; + println!("FRIDEV VM dp proof: {device_commits} device FRI commits"); + assert!( + device_commits > 0, + "no table took the device FRI commit at fri = dp" + ); + assert!( + crate::verify_with_options(&vm_proof, &elf_bytes, &dp, None, None) + .expect("honest verify must not error"), + "a device-proved dp VM proof must verify" + ); + assert!( + vm_proof.proof.proofs.iter().any(|p| { + let layers = p.fri_layers_merkle_roots.len(); + layers > 0 && p.query_list[0].layers_evaluations_sym.len() > 2 * layers + }), + "no table used a group of more than two values" + ); +} diff --git a/prover/src/tests/zf_vm_one_row_tests.rs b/prover/src/tests/zf_vm_one_row_tests.rs new file mode 100644 index 000000000..459fd70b5 --- /dev/null +++ b/prover/src/tests/zf_vm_one_row_tests.rs @@ -0,0 +1,282 @@ +//! S2 end to end on the production paths (box lib suite: each proves a full +//! VM trace with the 2^20-row BITWISE table, or an LFM machine proof). +//! +//! - A real multi-table VM proof (RPX block pin, host CPU paths) at +//! `one_row = 1` and at `one_row = auto` with `fri = dp`, blowup 4 (the +//! blowup the one-row static twins ship for). +//! - The hard miss at the VM level: at blowup 2 there is no one-row twin, so +//! `one_row = 1` is a proving ERROR naming the missing root — never a silent +//! recompute, never a proof. +//! - An LFM machine proof (`TrivialV0`) at `one_row = 1`, blowup 4, verified +//! through `lfm_verify`, i.e. through the registry policy (built at run time, +//! `LFM_REGISTRY` not read). +//! - A regression: the Phase-A replay that recovers `z`, `α` for +//! the expected bus balances absorbs each preprocessed table's root AT ITS +//! LEAF LAYOUT — an LFM proof at the wrap's options under `one_row = auto` +//! (mixed layouts, one-row preprocessed chips, published words) and a VM +//! proof with public output at `one_row = 1`. + +use stark::proof::options::{FriMode, OneRowMode, ProofFormat, ProofOptions}; + +fn opts(blowup: u8, one_row: OneRowMode, fri_mode: FriMode) -> ProofOptions { + let mut o = ProofOptions::default_test_options(); + o.blowup_factor = blowup; + o.format = ProofFormat { + one_row, + fri_mode, + ..ProofFormat::DEFAULT + }; + o +} + +#[test] +fn a_vm_proof_round_trips_at_one_row() { + let elf_bytes = crate::test_utils::asm_elf_bytes("test_mul_8"); + let one_row = opts(4, OneRowMode::On, FriMode::Pair); + let vm_proof = crate::prove_with_options(&elf_bytes, &one_row, &Default::default()) + .expect("the fixture must prove at one_row = 1"); + assert!( + crate::verify_with_options(&vm_proof, &elf_bytes, &one_row, None, None) + .expect("honest verify must not error"), + "an honest one-row VM proof must verify" + ); + // Every table is one-row: no symmetric rows anywhere, and the input tree + // is FRI layer 0 wherever anything folds. + for p in &vm_proof.proof.proofs { + for o in &p.deep_poly_openings { + assert!(o.main_trace_polys.evaluations_sym.is_empty()); + assert!(o.composition_poly.evaluations_sym.is_empty()); + } + } + println!( + "ZF S2 VM one_row=1: {} tables, proof tables with a precomputed root: {}", + vm_proof.proof.proofs.len(), + vm_proof + .proof + .proofs + .iter() + .filter(|p| p.lde_trace_precomputed_merkle_root.is_some()) + .count() + ); + // The layout is a verifier constant: the row-pair verifier rejects it. + let default = opts(4, OneRowMode::Off, FriMode::Pair); + assert!( + !crate::verify_with_options(&vm_proof, &elf_bytes, &default, None, None).unwrap_or(false), + "a one-row proof must not verify under the default format" + ); + // A tampered input-group value is rejected. + let mut bad = vm_proof.clone(); + let table = bad + .proof + .proofs + .iter() + .position(|p| !p.fri_layers_merkle_roots.is_empty()) + .expect("a table with committed layers"); + bad.proof.proofs[table].query_list[0].layers_evaluations_sym[0] += + math::field::element::FieldElement::< + math::field::extensions_goldilocks::Degree3GoldilocksExtensionField, + >::one(); + assert!( + !crate::verify_with_options(&bad, &elf_bytes, &one_row, None, None).unwrap_or(false), + "a tampered input-group value must be rejected" + ); +} + +#[test] +fn a_vm_proof_round_trips_at_one_row_auto_with_dp() { + let elf_bytes = crate::test_utils::asm_elf_bytes("test_mul_8"); + let auto = opts(4, OneRowMode::Auto, FriMode::Dp); + let vm_proof = crate::prove_with_options(&elf_bytes, &auto, &Default::default()) + .expect("the fixture must prove at one_row = auto, fri = dp"); + assert!( + crate::verify_with_options(&vm_proof, &elf_bytes, &auto, None, None) + .expect("honest verify must not error"), + "an honest auto/dp VM proof must verify" + ); + let one_row_tables = vm_proof + .proof + .proofs + .iter() + .filter(|p| { + p.deep_poly_openings[0] + .composition_poly + .evaluations_sym + .is_empty() + }) + .count(); + println!( + "ZF S2 VM one_row=auto fri=dp: {one_row_tables} of {} tables one-row", + vm_proof.proof.proofs.len() + ); +} + +/// No one-row static twin at blowup 2 ⇒ a proving error naming +/// the missing root. +#[test] +fn a_missing_one_row_twin_is_a_vm_proving_error() { + let elf_bytes = crate::test_utils::asm_elf_bytes("test_mul_8"); + let one_row = opts(2, OneRowMode::On, FriMode::Pair); + let err = crate::prove_with_options(&elf_bytes, &one_row, &Default::default()) + .expect_err("no one-row twin at blowup 2: proving must fail"); + let msg = format!("{err:?}"); + assert!( + msg.contains("PrecomputedCommitmentMissing"), + "the error must name the missing one-row root: {msg}" + ); +} + +#[test] +fn an_lfm_proof_round_trips_at_one_row() { + use crate::lfm::proof::{lfm_prove, lfm_verify}; + use crate::lfm::registry::{LfmProgramKind, build_artifacts}; + use crate::tables::types::FE; + let mut o = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(4).unwrap(); + o.format.one_row = OneRowMode::On; + let program = LfmProgramKind::TrivialV0.program(); + let artifacts = build_artifacts(&program, &o); + assert!(artifacts.one_row_roots.is_some()); + let arenas: Vec> = vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ]; + let proved = lfm_prove(&program, &artifacts, &arenas, &o).expect("one-row LFM prove"); + for p in &proved.proof.proofs { + assert!( + p.deep_poly_openings[0] + .main_trace_polys + .evaluations_sym + .is_empty() + ); + } + assert!( + lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &o + ) + .expect("built at run time under one row"), + "an honest one-row LFM proof must verify" + ); +} + +/// ★ REGRESSION: one-row PREPROCESSED tables and the Phase-A +/// replay. The prover absorbs each preprocessed table's root OF ITS LEAF +/// LAYOUT before sampling the shared LogUp `z`, `α`; the verify paths recover +/// `z`, `α` with `crate::replay_transcript_phase_a_view`, which absorbed the +/// ROW-PAIR root unconditionally. For a one-row preprocessed table the replay +/// then diverges, and every expected balance that depends on `z`, `α` is +/// wrong: an honest proof is rejected. The balance depends on them only when +/// something is published — the LFM public words, the VM commit bus — which +/// is why a VM proof without public output (`test_mul_8`) verified anyway. +/// +/// At the wrap's options (blowup 4, terminal 2^8, 128-bit queries) under +/// `one_row = auto`, as the block tree proves its LFM wraps: layouts MIX +/// within one proof and at least one preprocessed chip goes one-row (asserted, +/// so this keeps exercising the bug). Verified through +/// `verify_against_artifacts` — the call the tree harness makes before +/// harvesting a child (`per_table_aggregator_tests::real_child_timed`) — and +/// through `lfm_verify`. +#[test] +fn an_lfm_proof_at_the_wrap_options_round_trips_at_one_row_auto() { + use crate::lfm::proof::{lfm_prove, lfm_verify, verify_against_artifacts}; + use crate::lfm::registry::{LfmProgramKind, build_artifacts}; + use crate::tables::types::FE; + use stark::leaf_layout::table_leaf_layout; + let mut o = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(4).unwrap(); + o.fri_final_poly_log_degree = 8; + o.format.one_row = OneRowMode::Auto; + let program = LfmProgramKind::TrivialV0.program(); + let artifacts = build_artifacts(&program, &o); + let arenas: Vec> = vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ]; + let proved = lfm_prove(&program, &artifacts, &arenas, &o).expect("auto LFM prove"); + assert!( + !proved.public_words.is_empty(), + "the balance must depend on z and alpha: the program publishes" + ); + + let mut airs = crate::lfm::airs::LfmAirs::new_chunked( + &artifacts.roots, + &artifacts.blake3_chunk_roots, + &o, + artifacts.keccak_rnd_chunks, + artifacts.hasher, + artifacts.chip_set, + ); + airs = airs.with_one_row_roots(artifacts.one_row_roots.as_ref().expect("built")); + let refs = airs.air_refs(); + let (mut prep_rows, mut rows, mut pairs) = (0usize, 0usize, 0usize); + for (air, p) in refs.iter().zip(&proved.proof.proofs) { + let layout = table_leaf_layout(*air, p.trace_length); + println!( + "ZF FIX-S2 layout {:<12} 2^{:<2} {layout:?}", + air.name(), + p.trace_length.trailing_zeros() + ); + if layout.is_one_row() { + rows += 1; + prep_rows += usize::from(air.is_preprocessed()); + } else { + pairs += 1; + } + } + println!( + "ZF FIX-S2 LFM one_row=auto: {rows} one-row, {pairs} row-pair, {prep_rows} one-row preprocessed" + ); + assert!( + prep_rows >= 1 && pairs >= 1, + "the fixture must mix layouts with a one-row preprocessed chip \ + ({prep_rows} one-row preprocessed, {pairs} row-pair)" + ); + + assert!( + verify_against_artifacts(&artifacts, &proved.proof, &proved.public_words, &o), + "an honest one-row-auto LFM proof must verify (the tree harness's call)" + ); + assert!( + lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &o + ) + .expect("built at run time under one row"), + "an honest one-row-auto LFM proof must verify through lfm_verify" + ); + // Still bound to the claimed words: one moved public word rejects. + let mut wrong = proved.public_words.clone(); + wrong[0].1[0] += FE::from(1u64); + assert!( + !verify_against_artifacts(&artifacts, &proved.proof, &wrong, &o), + "a moved public word must be rejected" + ); +} + +/// The VM half of the same regression: a VM proof WITH public output (the +/// commit bus's expected balance depends on the replayed `z`, `α`) at +/// `one_row = 1`, where every preprocessed VM table (BITWISE, DECODE, the +/// pages, REGISTER) is one-row. +#[test] +fn a_vm_proof_with_public_output_round_trips_at_one_row() { + let elf_bytes = crate::test_utils::asm_elf_bytes("test_commit_4"); + let one_row = opts(4, OneRowMode::On, FriMode::Pair); + let vm_proof = crate::prove_with_options(&elf_bytes, &one_row, &Default::default()) + .expect("test_commit_4 must prove at one_row = 1"); + assert_eq!(vm_proof.public_output, vec![0xAA, 0xBB, 0xCC, 0xDD]); + assert!( + crate::verify_with_options(&vm_proof, &elf_bytes, &one_row, None, None) + .expect("honest verify must not error"), + "an honest one-row VM proof with public output must verify" + ); + let mut wrong = vm_proof.clone(); + wrong.public_output[0] ^= 1; + assert!( + !crate::verify_with_options(&wrong, &elf_bytes, &one_row, None, None).unwrap_or(false), + "a moved public output byte must be rejected" + ); +} diff --git a/prover/src/whir_hash_knob.rs b/prover/src/whir_hash_knob.rs new file mode 100644 index 000000000..6d829e9fb --- /dev/null +++ b/prover/src/whir_hash_knob.rs @@ -0,0 +1,203 @@ +//! ★ `LAMBDA_VM_WHIR_HASH` — which hash the multilinear path commits, +//! transcripts and grinds with. +//! +//! ```text +//! LAMBDA_VM_WHIR_HASH=keccak (default, and what PR #988 produces) +//! LAMBDA_VM_WHIR_HASH=rpx the algebraic arm, for a proof headed into +//! another proof +//! ``` +//! +//! Read ONCE per process and cached, so a run cannot change hash halfway +//! through and produce a proof no single configuration describes. +//! +//! # Three decisions worth stating +//! +//! **An unknown value ABORTS, loudly.** `LAMBDA_VM_WHIR_HASH=rpx256` — a +//! plausible typo, since that is what the configuration calls itself — would +//! otherwise fall through to keccak and produce a perfectly valid proof under +//! the hash the operator was trying to move away from. A measurement taken that +//! way is worse than no measurement: it looks like the RPX arm and is not. The +//! cost of aborting is a failed run with a message naming the accepted values; +//! the cost of defaulting is a number nobody can tell is wrong. +//! +//! **The banner prints on EVERY setting, including the default.** A banner that +//! only appeared for RPX could not be distinguished from a banner that did not +//! appear because this code was never reached — which is exactly what a byte +//! gate comparing two arms needs to rule out. Its absence in a log is therefore +//! a fact about the run, not an ambiguity. +//! +//! **It is read here and nowhere else.** The seam it selects is a type +//! parameter, so every consumer gets the hash through [`with_whir_hash`] rather +//! than by asking the environment again. + +use std::sync::OnceLock; + +/// Which configuration this process proves and verifies under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Setting { + /// Keccak-256, the default. + Keccak, + /// RPX256 (XHash12). + Rpx, +} + +impl Setting { + /// The name the configuration itself reports — what the banner prints and + /// the KATs are filed under. + pub const fn name(self) -> &'static str { + match self { + Self::Keccak => "keccak256", + Self::Rpx => "rpx256", + } + } +} + +/// The environment variable that selects it. +pub const ENV: &str = "LAMBDA_VM_WHIR_HASH"; + +/// What `ENV` accepts, and what an error message lists. +const ACCEPTED: &[(&str, Setting)] = &[ + ("keccak", Setting::Keccak), + ("keccak256", Setting::Keccak), + ("rpx", Setting::Rpx), + ("rpx256", Setting::Rpx), +]; + +/// ★ The setting for this process, read once and cached. +/// +/// Prints the banner on the first call. Aborts on an unrecognised value — see +/// the module header for why that is better than defaulting. +pub fn selected() -> Setting { + static SETTING: OnceLock = OnceLock::new(); + *SETTING.get_or_init(|| { + let setting = match std::env::var(ENV) { + Err(_) => Setting::Keccak, + Ok(raw) => parse(raw.trim()).unwrap_or_else(|| { + let accepted: Vec<&str> = ACCEPTED.iter().map(|(name, _)| *name).collect(); + // eprintln then abort rather than a panic: this is a + // configuration error at startup, and the operator needs the + // accepted values, not a backtrace through the prover. + eprintln!( + "{ENV}={raw:?} is not a hash this path knows. Accepted: {}.", + accepted.join(", ") + ); + std::process::abort() + }), + }; + // Always, including the default — see the module header. + println!("★ WHIR HASH: {}", setting.name()); + setting + }) +} + +/// The accepted spellings, case-insensitively. +fn parse(raw: &str) -> Option { + let lowered = raw.to_ascii_lowercase(); + ACCEPTED + .iter() + .find(|(name, _)| *name == lowered) + .map(|(_, setting)| *setting) +} + +/// ★ Run `$body` with `$h` bound to the configuration [`selected`] names. +/// +/// The seam is a type parameter, so the dispatch has to happen where a type can +/// be named — one `match` per call site, each arm monomorphising the body at +/// its own hash. That is also why this is a macro rather than a function: a +/// function cannot return a type. +/// +/// ⚠ Both arms are always compiled, which doubles the monomorphisations of +/// everything below the call. That is deliberate: a feature gate would make the +/// RPX arm unreachable in a default build, and then the knob would be a control +/// that does nothing on exactly the binary most people run. +#[macro_export] +macro_rules! with_whir_hash { + (|$h:ident| $body:block) => { + match $crate::whir_hash_knob::selected() { + $crate::whir_hash_knob::Setting::Keccak => { + #[allow(non_camel_case_types)] + type $h = multilinear::whir_hash::KeccakWhir; + $body + } + $crate::whir_hash_knob::Setting::Rpx => { + #[allow(non_camel_case_types)] + type $h = multilinear::whir_hash::RpxWhir; + $body + } + } + }; +} + +/// ★ Run `$body` with `$h` bound to the block's WHIR base configuration +/// ([`crate::hash_pin::BlockWhirHash`], the block proof family's hash), after +/// [`crate::block_whir::checked_base`] has admitted the format's base — the +/// family's and nothing else. The block path reads no environment for its +/// hash, so [`selected`] (and its default, keccak) never reaches a block proof +/// (D-WHIR-P1 S7). Any other base returns its typed refusal from the enclosing +/// function, whose error type takes `crate::Error`. +#[macro_export] +macro_rules! with_block_hash { + ($base:expr, |$h:ident| $body:block) => {{ + $crate::block_whir::checked_base(&$base)?; + #[allow(non_camel_case_types)] + type $h = $crate::hash_pin::BlockWhirHash; + $body + }}; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every accepted spelling maps where it says, case-insensitively. + /// + /// Tested through [`parse`] rather than through [`selected`]: the latter + /// caches in a process-global `OnceLock` and aborts on a bad value, so it + /// can be exercised exactly once per process and never with a bad input. + /// What is testable is the decision it makes, which is this function. + #[test] + fn every_accepted_spelling_maps_to_its_configuration() { + assert_eq!(parse("keccak"), Some(Setting::Keccak)); + assert_eq!(parse("keccak256"), Some(Setting::Keccak)); + assert_eq!(parse("rpx"), Some(Setting::Rpx)); + assert_eq!(parse("rpx256"), Some(Setting::Rpx)); + assert_eq!(parse("RPX"), Some(Setting::Rpx)); + assert_eq!(parse("Keccak256"), Some(Setting::Keccak)); + } + + /// ★ And the near-misses do NOT. This is the list that would otherwise + /// default to keccak and report itself as the RPX arm. + #[test] + fn a_near_miss_is_not_silently_accepted() { + for raw in [ + "rpx-256", + "rpx_256", + "xhash12", + "algebraic", + "blake3", + "", + "kecak", + ] { + assert_eq!(parse(raw), None, "{raw:?} must not parse"); + } + } + + /// The two settings name themselves the way the configurations do, so a + /// banner and a KAT cannot disagree about which arm ran. + #[test] + fn the_names_match_the_configurations() { + use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; + assert_eq!(Setting::Keccak.name(), KeccakWhir::NAME); + assert_eq!(Setting::Rpx.name(), RpxWhir::NAME); + } + + /// ✓ The macro really does bind a different type per arm — checked through + /// the configuration's own name, so a macro that expanded both arms to + /// keccak would fail here rather than silently proving under one hash. + #[test] + fn the_macro_binds_the_configuration_the_setting_names() { + use multilinear::whir_hash::WhirHash; + let name = with_whir_hash!(|H| { H::NAME }); + assert_eq!(name, selected().name()); + } +} diff --git a/prover/src/whir_identity.rs b/prover/src/whir_identity.rs new file mode 100644 index 000000000..3231a705a --- /dev/null +++ b/prover/src/whir_identity.rs @@ -0,0 +1,108 @@ +//! ★ The IDENTITY line for a WHIR proof — the byte gate the hash seam is +//! measured against. +//! +//! # What it is for +//! +//! The control for every arm of the hash work is "with the parameter unset, the +//! proof is the one PR #988 produced". Hashing the serialized proof would be +//! the obvious way to say that, and it does not work: **grinding nonces are +//! nondeterministic.** `crypto::grinding::generate_nonce` searches with rayon's +//! `find_any` under the `parallel` feature and returns whichever valid nonce a +//! worker reached first; on the device arm the kernel returns the smallest in +//! the range it scanned. Neither is a contract — the verifier accepts any nonce +//! passing `is_valid_nonce` — so two honest runs of the same prover on the same +//! input produce different proof BYTES, and a raw digest of them reports a +//! difference that means nothing. +//! +//! So the digest below is taken over the proof with **every grinding nonce +//! zeroed**, and over nothing else that has been excluded. Everything a hash +//! swap actually moves — every Merkle root, every out-of-domain value, every +//! sumcheck coefficient, every opened codeword value and every authentication +//! path — is inside it. +//! +//! # What it therefore does NOT cover +//! +//! - **The nonces themselves.** A defect that produced a valid-but-wrong nonce +//! would not show here. It is not invisible: `check_grind` rejects an invalid +//! nonce at verify time, and that is the gate for this property. +//! - **Anything outside the `MultiProof`.** Table heights, page ranges, public +//! output and the epoch bookends live in the enclosing proof structs; the +//! caller hashes those separately if it wants them bound. +//! - **Serialized LENGTH is checked separately** by +//! [`serialized_len`], because that is the sharper of the two statements: a +//! hash swap must leave the length equal to the byte (32-byte digests either +//! way, no proof struct gains a field), while the digest is *expected* to +//! change under a different hash. + +use multilinear::whir_chain::{ChainProof, RoundNonces}; + +use crate::test_utils::{E, F}; + +/// The keccak the identity line itself is taken with. +/// +/// Deliberately fixed, and deliberately NOT the proof's own hash: this is a +/// measuring instrument, not part of the protocol. If it followed the +/// configuration then the keccak arm and the RPX arm would be hashed by +/// different functions, and "the digests differ" would no longer distinguish a +/// changed proof from a changed instrument. +type Line = crypto::hash::platform_keccak::PlatformKeccak256; + +/// A proof with every grinding nonce zeroed — the form the identity line is +/// taken over. +fn without_nonces(proof: &MultiProof) -> MultiProof { + let mut out = proof.clone(); + for stacked in &mut out.columns { + for chain in &mut stacked.polys { + zero_nonces(chain); + } + } + out +} + +fn zero_nonces(chain: &mut ChainProof) { + for round in &mut chain.rounds { + round.nonces = RoundNonces::default(); + } +} + +/// The `MultiProof` this VM's multilinear path produces. +pub type MultiProof = stark::multilinear_table::MultiProof; + +/// ★ The identity line: keccak-256 over the rkyv bytes of the proof with every +/// grinding nonce zeroed. +/// +/// Two runs of the same prover on the same input must produce the same line. +/// Two runs under different hash configurations must produce different ones — +/// see [`crate::tests::whir_identity_tests`], where both halves are asserted, +/// because a digest that could not change is not a gate. +pub fn identity_line(proof: &MultiProof) -> Result<[u8; 32], String> { + use digest::Digest; + + let normalised = without_nonces(proof); + let bytes = rkyv::to_bytes::(&normalised) + .map_err(|e| format!("the proof did not serialize: {e}"))?; + let digest = Line::digest(bytes.as_ref()); + Ok(digest.into()) +} + +/// The identity line as lowercase hex — what a box run prints and a coordinator +/// diffs. +pub fn identity_hex(proof: &MultiProof) -> Result { + Ok(identity_line(proof)? + .iter() + .map(|b| format!("{b:02x}")) + .collect()) +} + +/// The serialized length of the proof, in bytes. +/// +/// ★ Checked SEPARATELY from the digest and held to a stricter standard: the +/// digest is expected to move when the hash moves, the length is not. Every +/// [`multilinear::whir_hash::WhirHash`] has a 32-byte commitment and no proof +/// struct gains a field, so a length difference between two hash arms is a +/// defect in the seam rather than a property of the hash. +pub fn serialized_len(proof: &MultiProof) -> Result { + Ok(rkyv::to_bytes::(proof) + .map_err(|e| format!("the proof did not serialize: {e}"))? + .len()) +} diff --git a/prover/src/zf_format.rs b/prover/src/zf_format.rs new file mode 100644 index 000000000..3a9254322 --- /dev/null +++ b/prover/src/zf_format.rs @@ -0,0 +1,2144 @@ +//! ★ The proof FORMAT this process proves under — the ZF proof-format levers. +//! +//! ```text +//! LAMBDA_VM_ZF_CAP off | auto | 0..=16 Merkle cap, every univariate STARK tree (S1) +//! LAMBDA_VM_ZF_WHIR_CAP off | auto | 0..=16 Merkle cap, every WHIR chain tree (W1) +//! LAMBDA_VM_ZF_FRI pair | dp FRI fold schedule (S3) +//! LAMBDA_VM_ZF_ONE_ROW 0 | 1 | auto one-row trace openings (S2) +//! LAMBDA_VM_ZF_WHIR_FOLDS uniform4 | first5 | first6 WHIR first-round fold (W2) +//! LAMBDA_VM_ZF_WHIR_STACK 25 | 26 | 27 WHIR stack cap, in variables (S2) +//! LAMBDA_VM_ZF_WHIR_GRIND query | all WHIR proof of work: before the queries only, or all three (P2) +//! LAMBDA_VM_ZF_WHIR_GRIND_BITS 20 | 18 WHIR proof-of-work bits; Q buys back what the grind gives up +//! LAMBDA_VM_ZF_WHIR_BLOWUP 4 | 2 WHIR base code blowup (rate 1/4 or 1/2); Q follows from the rate +//! LAMBDA_VM_ZF_WHIR_LEAVES natural | coset WHIR round-0 leaf order (coset: Poseidon1 trees only) +//! LAMBDA_VM_ZF_LFM_MERGE 0 | 1 LFM chips of equal padded height proved as one AIR +//! ``` +//! +//! ★ Every unset knob is [`ZfFormat::DEFAULT`], the MEASURED configuration: +//! `cap=auto whir_cap=auto fri=dp one_row=0 whir_folds=first6 whir_stack=27 whir_grind=query +//! whir_grind_bits=18`, the WHIR base code at blowup 2, the block's round-0 trees in the coset order, +//! the LFM proofs' equal-height chips merged (`lfm_merge=1`, off beside one-row openings). +//! Each lever was measured net positive on block runs before it became the default. Every knob keeps +//! its OFF spelling (`cap=off`, `whir_cap=off`, `fri=pair`, `one_row=0`, `whir_folds=uniform4`, +//! `whir_stack=25`, `whir_grind=all`, `whir_grind_bits=20`, `whir_blowup=4`, +//! `whir_leaves=natural`, `lfm_merge=0`), so setting all eleven to off reproduces [`ZfFormat::LEGACY`] — the format +//! before any lever, byte for byte — for rollback and for A/B arms. The crypto crates' own defaults +//! (`stark::proof::options::ProofFormat::DEFAULT`, +//! `multilinear::whir_chain::ChainFormat::DEFAULT`) stay the legacy format: a +//! library value built without a format is the legacy one, and the production +//! format reaches the proofs only through the three sites below. +//! +//! # Where the format goes +//! +//! Parsed ONCE per process ([`ZfFormat::global`]) and read only where a +//! production format value is built — [`crate::lfm::proof::aggregation_wrap_options`] +//! (every LFM proof: wraps, nodes, the root), [`crate::multilinear_prove::chain_config`] +//! (the WHIR base proofs) and [`crate::lfm::proof::block_base_options`] (the +//! STARK block's base epochs). From there it travels inside the option types +//! the crypto crates already take — `stark::ProofOptions` and +//! `multilinear::ChainConfig` — which never read the environment themselves. +//! Host verification reads nothing global: it uses the options it is given. +//! Tests build those option values explicitly; none sets the environment. +//! +//! # Three rules, as in `whir_hash_knob` +//! +//! **An unknown value ABORTS.** A typo that fell back to the default would +//! produce a valid default-format proof labelled as the lever — a measurement +//! that looks like arm B and is arm A. +//! +//! **A lever this build does not implement ABORTS too.** The fields exist +//! before the levers do (so the option structs and this banner are stable +//! before the levers land), and a knob set on a build that only parses +//! it would print a non-default format and prove the default one. Each +//! `*_IMPLEMENTED` constant is flipped when its lever is real. +//! +//! **The banner prints on every setting, including the default**: +//! `ZF FORMAT: cap=auto whir_cap=auto fri=dp one_row=0 whir_folds=first6 whir_stack=27 whir_grind=query +//! whir_grind_bits=18`. Its absence in a log is then a fact about the run, not an ambiguity. The WHIR blowup +//! and the leaf order are the two words printed only off their defaults (` whir_blowup=4`, +//! ` whir_leaves=natural`). + +use std::sync::OnceLock; + +/// The round-0 leaf order ([`ZfFormat::whir_leaves`]), named here for the +/// callers that choose it (the block commands' `--leaves`). +pub use multilinear::whir_chain::LeafOrder; +use multilinear::whir_chain::{ + ChainConfig, ChainFormat, FirstFold, GrindBits, NonceLayout, StackVars, WhirFolds, +}; +use stark::proof::options::{ + BaseFormat, CapPolicy, FriMode, OneRowMode, ProofFormat, ProofOptions, +}; + +/// The knob names, in banner order. +pub const ENV_CAP: &str = "LAMBDA_VM_ZF_CAP"; +pub const ENV_WHIR_CAP: &str = "LAMBDA_VM_ZF_WHIR_CAP"; +pub const ENV_FRI: &str = "LAMBDA_VM_ZF_FRI"; +pub const ENV_ONE_ROW: &str = "LAMBDA_VM_ZF_ONE_ROW"; +pub const ENV_WHIR_FOLDS: &str = "LAMBDA_VM_ZF_WHIR_FOLDS"; +pub const ENV_WHIR_STACK: &str = "LAMBDA_VM_ZF_WHIR_STACK"; +pub const ENV_WHIR_GRIND: &str = "LAMBDA_VM_ZF_WHIR_GRIND"; +pub const ENV_WHIR_GRIND_BITS: &str = "LAMBDA_VM_ZF_WHIR_GRIND_BITS"; +pub const ENV_WHIR_BLOWUP: &str = "LAMBDA_VM_ZF_WHIR_BLOWUP"; +/// Where the block's WHIR round-0 trees put their leaves: `coset` (the +/// default: block `j` at `bitrev(j)`, so a prover evaluates its openings from +/// the polynomial instead of re-encoding it) or `natural`. A proof format: the +/// roots, the paths and the in-guest walks follow it. Poseidon1 trees only. +pub const ENV_WHIR_LEAVES: &str = "LAMBDA_VM_ZF_WHIR_LEAVES"; +/// Whether the recursion's LFM proofs put their equal-height chips into one +/// AIR (`1`, the default) or keep one AIR a chip (`0`, the rollback). A proof +/// format of the recursion alone: the program ids, the artifacts, the in-guest +/// verifiers and the proofs follow it; no base proof moves. Measured on a 7950X +/// (knob on against off, same binary): recursion −0.60 s median, −1.18 s large; +/// whole block −0.70 / −1.04 s; the top proof ×0.79–0.88. Unset beside one-row +/// openings (`LAMBDA_VM_ZF_ONE_ROW` on or auto) it is off, since a merged AIR +/// has no one-row layout; set to `1` there it is refused. +pub const ENV_LFM_MERGE: &str = "LAMBDA_VM_ZF_LFM_MERGE"; + +/// Every knob above, in banner order: what a verifier that pins its format +/// refuses to find in its environment (`verify-block`). +pub const ENV_KNOBS: [&str; 11] = [ + ENV_CAP, + ENV_WHIR_CAP, + ENV_FRI, + ENV_ONE_ROW, + ENV_WHIR_FOLDS, + ENV_WHIR_STACK, + ENV_WHIR_GRIND, + ENV_WHIR_GRIND_BITS, + ENV_WHIR_BLOWUP, + ENV_WHIR_LEAVES, + ENV_LFM_MERGE, +]; + +/// The uniform WHIR schedule's fold, as production configures it +/// (`multilinear_prove::chain_config`); the banner spells the default +/// `uniform4` after it. +pub const PRODUCTION_WHIR_LOG_FOLDING: usize = 4; + +/// The WHIR chains' `log2` inverse rate at the default +/// ([`ZfFormat::whir_blowup`], `multilinear_prove::chain_config_under`): blowup 2. +pub const PRODUCTION_WHIR_LOG_BLOWUP: usize = 1; + +/// The WHIR chains' `log2` inverse rate in the legacy format: blowup 4, the +/// widest the knob accepts and the default's rollback (`LAMBDA_VM_ZF_WHIR_BLOWUP=4`). +pub const LEGACY_WHIR_LOG_BLOWUP: usize = 2; + +/// The WHIR base code blowups `LAMBDA_VM_ZF_WHIR_BLOWUP` accepts: 4 (the legacy +/// format, the default's rollback) and 2 (the default). The query count follows from the rate by the code's own rule +/// (`multilinear::query_count::num_queries`): at the default fold schedule and +/// 18 query-grind bits, blowup 2 charges Q 229 at every stack of 23 variables or +/// more (six rounds from 23, seven from 27) and 228 from 15 to 22 (five), where +/// blowup 4 charges 114; a block's BITWISE holds its tallest stack at 25. Every +/// phase keeps the conjectured (Johnson-regime) 128 bits: the binding phase is +/// the unground fold at 27 variables, 130.393 → 130.344 per polynomial (128.393 → +/// 128.344 under `GroupFold::Indexed`); the query phases move 130.907 → 130.946. +/// The recursion's own proofs (`lfm::proof::aggregation_wrap_options`) do not +/// read it. Widening this list is a format decision, not a parser one. +pub const WHIR_BLOWUPS: [u8; 2] = [4, 2]; + +/// The WHIR base code's blowup as a caller names it — the block commands' +/// `--blowup 4|2`, the public block verifier's argument — so a caller chooses +/// among [`WHIR_BLOWUPS`] and nothing else. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WhirBlowup { + /// Blowup 4, rate 1/4: the legacy format, the default's rollback. + Four, + /// Blowup 2, rate 1/2: the default. + Two, +} + +impl WhirBlowup { + /// The blowup factor, as [`ZfFormat::whir_blowup`] holds it. + pub const fn factor(self) -> u8 { + match self { + Self::Four => WHIR_BLOWUPS[0], + Self::Two => WHIR_BLOWUPS[1], + } + } +} + +impl std::fmt::Display for WhirBlowup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.factor()) + } +} + +// A grouped chain is today's chain at `n + m` variables over +// `H_{n+m+lb}`, which exists only within the field's two-adicity: at the widest +// stack, the widest accepted blowup and the widest index, it must. +// The legacy blowup is the widest the knob accepts, the production one the other. +const _: () = assert!( + WHIR_BLOWUPS[0] == 1 << LEGACY_WHIR_LOG_BLOWUP + && WHIR_BLOWUPS[1] == 1 << PRODUCTION_WHIR_LOG_BLOWUP + && WHIR_BLOWUPS[1] < WHIR_BLOWUPS[0] +); +// `multilinear::stacked_eval` refuses any other config at run time. +const _: () = assert!( + multilinear::whir_chain::StackVars::WIDEST + + LEGACY_WHIR_LOG_BLOWUP + + multilinear::whir_chain::MAX_INDEX_VARS + <= ::TWO_ADICITY + as usize +); + +/// The WHIR chains' proof-of-work bits at the default, wherever [`WhirGrind`] +/// places them. The query count subtracts the query grind's bits from what the +/// queries must buy (`multilinear::query_count::num_queries`), so 18 bits run +/// Q 114 where 20 ran 112 (FAST 320: base −0.90 s at block 25368371). +pub const PRODUCTION_WHIR_GRIND_BITS: u8 = 18; + +/// The bits the WHIR chains ground before the default moved to 18: the legacy +/// format's, and `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20`'s (the opt-out). +pub const LEGACY_WHIR_GRIND_BITS: u8 = 20; + +/// The grind bits `LAMBDA_VM_ZF_WHIR_GRIND_BITS` accepts. 18 trades two grind +/// bits for two queries (Q 112 → 114 at every production height from 15 to 27 +/// variables), and every phase keeps at least the conjectured (Johnson-regime) +/// minimum it has at 20: the binding phase is the unground fold at 27 variables +/// (130.393 per polynomial; 128.393 under `GroupFold::Indexed`, D-ONECHAIN), +/// which neither the query grind nor Q reaches; the query phases move 130.926 → +/// 130.907. +/// Not below 18: each bit given up costs every chain the recursion verifies a +/// query (on the block, every block leaf's in-guest openings). +/// Widening this list is a format decision, not a parser one. +pub const WHIR_GRIND_BITS: [u8; 2] = [20, 18]; + +/// The process's format: [`ZfFormat::global`] (the environment's) or +/// [`ZfFormat::install`] (a caller's), whichever comes first. +static PROCESS_FORMAT: OnceLock = OnceLock::new(); + +/// One process's proof format. [`ZfFormat::default`] is [`ZfFormat::DEFAULT`], +/// the measured configuration; [`ZfFormat::LEGACY`] is every lever off. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ZfFormat { + /// S1: the cap on every univariate STARK tree. + pub cap: CapPolicy, + /// W1: the cap on every WHIR chain tree. + pub whir_cap: CapPolicy, + /// S3: the FRI fold schedule. + pub fri: FriMode, + /// S2: one-row trace openings. + pub one_row: OneRowMode, + /// W2: the WHIR per-round fold schedule. + pub whir_folds: WhirFolds, + /// S2: how wide a WHIR stacked polynomial may get. + pub whir_stack: StackVars, + /// P2: where the WHIR base chains grind. + pub whir_grind: WhirGrind, + /// How many bits each of those grinds spends. + pub whir_grind_bits: u8, + /// The WHIR base code's blowup, one of [`WHIR_BLOWUPS`]: every base chain's + /// codeword is this many times its polynomial. 2 at the default, 4 in the + /// legacy format. + pub whir_blowup: u8, + /// Where the WHIR round-0 trees put their leaves ([`ENV_WHIR_LEAVES`]). + pub whir_leaves: LeafOrder, + /// Whether the LFM proofs merge their equal-height chips into one AIR + /// ([`ENV_LFM_MERGE`]). On at the default, off in the legacy format. + pub lfm_merge: bool, + /// The WHIR block's base hash and its arity-4 cap ([`BaseFormat`]): the + /// block proof family's ([`crate::hash_pin::BlockFamily`]) at the + /// default, and the only base the block path admits + /// (`block_whir::checked_base`). No knob sets it. The LFM proofs never + /// read it: their configuration is the family's too. + pub base: BaseFormat, +} + +/// Where the WHIR base chains grind (P2), and so which nonces their rounds +/// carry. +/// +/// Of a round's three grinds only the query one raises the proven minimum as +/// placed. The folding grind sits before the round's first sumcheck message, +/// so the first folding challenge is redrawn by varying that message without +/// grinding again. The out-of-domain grind comes after the out-of-domain +/// point, so the only challenge it guards is the batching one, which has far +/// more bits than the target without any grind (`multilinear::whir_chain`'s +/// module header). So `query` drops the other two and loses no proven bits, +/// and the query count, which reads the query grind alone, does not move. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WhirGrind { + /// Before each round's query positions only ([`GrindBits::query_only`]), + /// one nonce a round ([`NonceLayout::Spent`]). The default. + Query, + /// Before all three redrawable challenges ([`GrindBits::uniform`]), three + /// nonces a round ([`NonceLayout::Three`]). The legacy format, byte for + /// byte, and the rollback. + All, +} + +impl WhirGrind { + /// The grind bits this placement puts in a chain config, `bits` a grind. + pub const fn bits(self, bits: u8) -> GrindBits { + match self { + Self::Query => GrindBits::query_only(bits), + Self::All => GrindBits::uniform(bits), + } + } + + /// The nonce layout that goes with it: a round carries the nonces it + /// spends, except the legacy format, which keeps its three. + pub const fn nonces(self) -> NonceLayout { + match self { + Self::Query => NonceLayout::Spent, + Self::All => NonceLayout::Three, + } + } +} + +impl std::fmt::Display for WhirGrind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Query => "query", + Self::All => "all", + }) + } +} + +impl std::str::FromStr for WhirGrind { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "query" => Ok(Self::Query), + "all" => Ok(Self::All), + _ => Err(()), + } + } +} + +/// The first-round WHIR fold of the default format (`whir_folds=first6`). +const DEFAULT_WHIR_FIRST_FOLD: FirstFold = match FirstFold::new(6) { + Some(k0) => k0, + None => panic!("6 is a legal first fold"), +}; + +/// The WHIR stack cap of the default format (`whir_stack=27`). +const DEFAULT_WHIR_STACK: StackVars = match StackVars::new(27) { + Some(n) => n, + None => panic!("27 is a legal stack cap"), +}; + +impl Default for ZfFormat { + fn default() -> Self { + Self::DEFAULT + } +} + +impl ZfFormat { + /// ★ The production format when no knob is set: the MEASURED + /// configuration. S1 `cap=auto` (STARK block −15.35 s), + /// S1+S3 `fri=dp` (−28.55 s), W1 `whir_cap=auto` and W2 `whir_folds=first6` + /// (WHIR block −9.10 s together), S2 `whir_stack=27` (WHIR block −13.80 s + /// against 25, 50 base chains instead of 145), each measured net positive + /// in an ABBA block run. S2 changes no STARK proof: only the WHIR layouts + /// read the stack. `one_row` stays off: in ABBA block runs it costs +3.2 s on + /// the WHIR pipeline (the prover-side cost of one-row LFM proofs) and saves + /// 8.0 s and 8 GiB of host memory on the STARK pipeline, so it is a knob + /// (`LAMBDA_VM_ZF_ONE_ROW=auto`), recommended for the STARK pipeline. + /// P2 `whir_grind=query` grinds the WHIR base chains before their queries + /// only. It costs no proven bits: of a round's three grinds only the query + /// one raises the proven minimum ([`WhirGrind`]), and the query counts, the + /// query grinds and the blowups are the legacy ones. It changes no STARK + /// proof. `LAMBDA_VM_ZF_WHIR_STACK=25` is the stack's rollback and + /// `LAMBDA_VM_ZF_WHIR_GRIND=all` the grind's. `whir_grind_bits=18` grinds + /// two bits fewer and buys them back with two queries (Q 112 → 114): the + /// conjectured (Johnson) minimum stays 130.393 bits per polynomial (128.393 + /// under `GroupFold::Indexed`), at the unground fold (WHIR block base −0.90 + /// s, FAST 320). `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20` is its rollback. + /// `whir_blowup` is 2: the base code at rate 1/2, Q 229 (228 below 23 + /// variables) at the same conjectured 128 bits, the base proof x1.78. It + /// halves the card's commit and costs recursion leaves; measured faster once + /// the host stopped binding the commit phase (KECCAK_RND filled on the card: + /// whole block -2.46 s median, -1.94 s large). `LAMBDA_VM_ZF_WHIR_BLOWUP=4` is + /// its rollback. `whir_leaves` is the coset order: the block's + /// round-0 trees put each query's coset together, so a revived group's + /// openings are evaluated from its stack instead of re-encoding it + /// (`LAMBDA_VM_ZF_WHIR_LEAVES=natural` is its rollback). `lfm_merge` proves + /// each LFM program's equal-height chips as one AIR (`LAMBDA_VM_ZF_LFM_MERGE=0` + /// is its rollback): fewer sub-proofs per recursion proof and fewer paths per + /// child at its parent, measured faster on a 7950X at both block sizes. It moves the block's + /// roots, paths and in-guest walks, not a bit count. + pub const DEFAULT: Self = Self { + cap: CapPolicy::Auto, + whir_cap: CapPolicy::Auto, + fri: FriMode::Dp, + one_row: OneRowMode::Off, + whir_folds: WhirFolds::First(DEFAULT_WHIR_FIRST_FOLD), + whir_stack: DEFAULT_WHIR_STACK, + whir_grind: WhirGrind::Query, + whir_grind_bits: PRODUCTION_WHIR_GRIND_BITS, + whir_blowup: 1 << PRODUCTION_WHIR_LOG_BLOWUP, + base: crate::hash_pin::BLOCK_BASE, + whir_leaves: LeafOrder::Coset, + lfm_merge: true, + }; + + /// The legacy format: every lever off. What all ten knobs at their + /// OFF spellings select, what the crypto crates' own defaults are, and the + /// only format the RV64 recursion guest verifies. + pub const LEGACY: Self = Self { + cap: CapPolicy::Off, + whir_cap: CapPolicy::Off, + fri: FriMode::Pair, + one_row: OneRowMode::Off, + whir_folds: WhirFolds::Uniform, + whir_stack: StackVars::LEGACY, + whir_grind: WhirGrind::All, + whir_grind_bits: LEGACY_WHIR_GRIND_BITS, + whir_blowup: 1 << LEGACY_WHIR_LOG_BLOWUP, + base: crate::hash_pin::BLOCK_BASE, + whir_leaves: LeafOrder::Natural, + lfm_merge: false, + }; + + /// This format with `base` as the WHIR block's base hash. The block path + /// admits [`crate::hash_pin::BLOCK_BASE`] alone (`block_whir::checked_base`), + /// so another base here is a refusal's input (the tests'), never a proof. + pub const fn with_base(self, base: BaseFormat) -> Self { + Self { base, ..self } + } + + /// True when every lever is off: the format proves exactly what the + /// prover proved before any lever existed. + pub fn is_legacy(&self) -> bool { + self.cap.is_off() + && self.whir_cap.is_off() + && self.fri == FriMode::Pair + && self.one_row == OneRowMode::Off + && self.whir_folds == WhirFolds::Uniform + && self.whir_stack == StackVars::LEGACY + && self.whir_grind == WhirGrind::All + && self.whir_grind_bits == LEGACY_WHIR_GRIND_BITS + && self.whir_blowup == 1 << LEGACY_WHIR_LOG_BLOWUP + && self.whir_leaves == LeafOrder::Natural + && self.base == crate::hash_pin::BLOCK_BASE + && !self.lfm_merge + } + + /// The grind bits the WHIR chains run at: [`WhirGrind`]'s placement with + /// `whir_grind_bits` bits in each placed grind. + pub const fn whir_grind_bits(&self) -> GrindBits { + self.whir_grind.bits(self.whir_grind_bits) + } + + /// The WHIR base code's `log2` inverse rate: what + /// [`crate::multilinear_prove::chain_config_under`] builds the chains at. + pub const fn whir_log_blowup(&self) -> usize { + self.whir_blowup.trailing_zeros() as usize + } + + /// Parse the ten knobs through `lookup` (the process environment in + /// production, a map in tests). An unset knob is [`Self::DEFAULT`]'s value; a set one + /// must be one of the accepted spellings (surrounding whitespace and case + /// are ignored, as for `LAMBDA_VM_WHIR_HASH`). + pub fn from_lookup(lookup: impl Fn(&str) -> Option) -> Result { + let mut format = Self::DEFAULT; + let get = |name: &str| lookup(name).map(|raw| raw.trim().to_ascii_lowercase()); + if let Some(v) = get(ENV_CAP) { + format.cap = parse_cap(ENV_CAP, &v)?; + } + if let Some(v) = get(ENV_WHIR_CAP) { + format.whir_cap = parse_cap(ENV_WHIR_CAP, &v)?; + } + if let Some(v) = get(ENV_FRI) { + format.fri = v + .parse() + .map_err(|()| format!("{ENV_FRI}={v:?}: expected `pair` or `dp`"))?; + } + if let Some(v) = get(ENV_ONE_ROW) { + format.one_row = v + .parse() + .map_err(|()| format!("{ENV_ONE_ROW}={v:?}: expected `0`, `1` or `auto`"))?; + } + if let Some(v) = get(ENV_WHIR_FOLDS) { + format.whir_folds = parse_whir_folds(&v)?; + } + if let Some(v) = get(ENV_WHIR_STACK) { + format.whir_stack = parse_whir_stack(&v)?; + } + if let Some(v) = get(ENV_WHIR_GRIND) { + format.whir_grind = v + .parse() + .map_err(|()| format!("{ENV_WHIR_GRIND}={v:?}: expected `query` or `all`"))?; + } + if let Some(v) = get(ENV_WHIR_GRIND_BITS) { + format.whir_grind_bits = parse_whir_grind_bits(&v)?; + } + if let Some(v) = get(ENV_WHIR_BLOWUP) { + format.whir_blowup = parse_whir_blowup(&v)?; + } + if let Some(v) = get(ENV_WHIR_LEAVES) { + format.whir_leaves = match v.as_str() { + "natural" => LeafOrder::Natural, + "coset" => LeafOrder::Coset, + _ => { + return Err(format!( + "{ENV_WHIR_LEAVES}={v:?}: expected `natural` or `coset`" + )); + } + }; + } + match get(ENV_LFM_MERGE) { + Some(v) => { + format.lfm_merge = match v.as_str() { + "0" => false, + "1" => true, + _ => return Err(format!("{ENV_LFM_MERGE}={v:?}: expected `0` or `1`")), + }; + } + // Unset beside one-row openings, the merge is off: a merged AIR has + // no one-row layout, and the one-row knob is the STARK pipeline's + // recommended setting, which must keep parsing on its own. + None if format.one_row != OneRowMode::Off => format.lfm_merge = false, + None => {} + } + format.check_leaves()?; + format.check_lfm_merge()?; + Ok(format) + } + + /// Refuses the LFM merge beside one-row openings: a merged AIR is committed + /// with row-pair leaves only (its artifacts build no one-row root), so the + /// pair is refused by name rather than proved under one of the two. + pub fn check_lfm_merge(&self) -> Result<(), String> { + if self.lfm_merge && self.one_row != OneRowMode::Off { + return Err(format!( + "{ENV_LFM_MERGE}=1 with {ENV_ONE_ROW}={}: the merged LFM tables have no one-row \ + layout", + self.one_row + )); + } + Ok(()) + } + + /// Refuses a round-0 leaf order the base's trees do not take: the coset + /// order is Poseidon1's alone (the block's base and recursion hash), so + /// under any other base it is refused by name before anything is built. + pub fn check_leaves(&self) -> Result<(), String> { + match (self.whir_leaves, self.base.hash) { + (LeafOrder::Natural, _) + | (LeafOrder::Coset, stark::config::CommitmentHash::Poseidon1) => Ok(()), + (LeafOrder::Coset, hash) => Err(format!( + "{ENV_WHIR_LEAVES}=coset: the coset leaf order is defined for Poseidon1 trees \ + only, and the base hash is {hash:?}" + )), + } + } + + /// [`from_lookup`](Self::from_lookup) over the process environment. A + /// variable that is set but not valid Unicode is an error, not "unset". + pub fn from_env() -> Result { + let non_unicode = std::cell::Cell::new(None); + let format = Self::from_lookup(|name| match std::env::var(name) { + Ok(v) => Some(v), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(_)) => { + non_unicode.set(Some(name.to_string())); + None + } + })?; + match non_unicode.into_inner() { + Some(name) => Err(format!("{name} is set but not valid Unicode")), + None => Ok(format), + } + } + + /// The knobs set to a non-default value whose lever this build does not + /// implement yet. Selecting one must fail: see the module header. + pub fn unimplemented_levers(&self) -> Vec<&'static str> { + let mut out = Vec::new(); + if !self.cap.is_off() && !stark::proof::options::MERKLE_CAP_IMPLEMENTED { + out.push(ENV_CAP); + } + if !self.whir_cap.is_off() && !multilinear::whir_chain::WHIR_CAP_IMPLEMENTED { + out.push(ENV_WHIR_CAP); + } + if self.fri != FriMode::Pair && !stark::proof::options::FRI_MODE_IMPLEMENTED { + out.push(ENV_FRI); + } + if self.one_row != OneRowMode::Off && !stark::proof::options::ONE_ROW_IMPLEMENTED { + out.push(ENV_ONE_ROW); + } + if self.whir_folds != WhirFolds::Uniform && !multilinear::whir_chain::WHIR_FOLDS_IMPLEMENTED + { + out.push(ENV_WHIR_FOLDS); + } + if self.whir_stack != StackVars::LEGACY && !multilinear::whir_chain::WHIR_STACK_IMPLEMENTED + { + out.push(ENV_WHIR_STACK); + } + out + } + + /// ★ The format for this process, read once and cached. + /// + /// Prints the banner on the first call. Aborts on an unrecognised value + /// or a lever this build does not implement — see the module header. + pub fn global() -> &'static Self { + PROCESS_FORMAT.get_or_init(|| { + let format = Self::from_env().unwrap_or_else(|e| { + // eprintln then abort rather than a panic: a configuration + // error at startup, and the operator needs the accepted + // values, not a backtrace through the prover. + eprintln!("ZF FORMAT: {e}"); + std::process::abort() + }); + if let Err(e) = format.check_implemented() { + eprintln!("{e}"); + std::process::abort() + } + format.announce(); + format + }) + } + + /// Makes `format` this process's format ([`Self::global`]) before + /// anything has read it, for a caller that is given a format field on its + /// command line rather than in the environment (`prove-block` / + /// `verify-block --blowup`). The banner then prints `format`, once, as + /// `global` would have printed the environment's: every reader of the + /// process format and the log agree with what the caller proves or + /// verifies under. Refused if the process format was already read under + /// another value (its banner is out), or names a lever this build lacks. + pub fn install(format: Self) -> Result<&'static Self, String> { + format.check_implemented()?; + format.check_lfm_merge()?; + let mut installed = false; + let got = PROCESS_FORMAT.get_or_init(|| { + installed = true; + format.announce(); + format + }); + if installed || *got == format { + Ok(got) + } else { + Err(format!( + "the process format was read before it could be set: {} (wanted {})", + got.banner(), + format.banner() + )) + } + } + + /// The process format's banner and schedule lines, on stdout. Always, + /// including the default — see the module header. + fn announce(&self) { + println!("{}", self.banner()); + println!("{}", self.whir_schedule_line()); + } + + /// `Err` naming the knobs set to a lever this build does not implement. + fn check_implemented(&self) -> Result<(), String> { + let missing = self.unimplemented_levers(); + if missing.is_empty() { + return Ok(()); + } + Err(format!( + "ZF FORMAT: {} set to a non-default value, but this build does not implement that \ + lever yet ({})", + missing.join(", "), + self.banner() + )) + } + + /// `ZF FORMAT: cap=… whir_cap=… fri=… one_row=… whir_folds=… whir_stack=… + /// whir_grind=… whir_grind_bits=…`, each value in the spelling its knob accepts. + pub fn banner(&self) -> String { + format!( + "ZF FORMAT: cap={} whir_cap={} fri={} one_row={} whir_folds={} whir_stack={} \ + whir_grind={} whir_grind_bits={}", + self.cap, + self.whir_cap, + self.fri, + self.one_row, + whir_folds_name(&self.whir_folds), + self.whir_stack.get(), + self.whir_grind, + self.whir_grind_bits + ) + &if self.whir_blowup == 1 << PRODUCTION_WHIR_LOG_BLOWUP { + String::new() + } else { + format!(" whir_blowup={}", self.whir_blowup) + } + &if self.base == crate::hash_pin::BLOCK_BASE { + String::new() + } else { + format!(" base={:?}/c{}", self.base.hash, self.base.arity4_cap) + } + match self.whir_leaves { + // Printed only off the default (the coset order). + LeafOrder::Coset => "", + LeafOrder::Natural => " whir_leaves=natural", + } + if self.lfm_merge { + "" + } else { + // Printed only off the default (on). + " lfm_merge=0" + } + } + + /// `ZF WHIR SCHEDULES: whir_folds=… n=20:[…] … n=:[…]` — the fold + /// schedule the WHIR base chains run at the production stack heights, up to + /// the format's stack cap, so a log states the rounds it proved and not only + /// the knob's name. Printed under the banner, on every setting. + pub fn whir_schedule_line(&self) -> String { + let stack = self.whir_stack.get(); + let config = crate::multilinear_prove::chain_config_under(self, &[(1, stack)]); + let schedules = (20..=stack) + .map(|n| format!("n={n}:{:?}", config.schedule(n)).replace(' ', "")) + .collect::>() + .join(" "); + format!( + "ZF WHIR SCHEDULES: whir_folds={} q={} {schedules}", + whir_folds_name(&self.whir_folds), + config.num_queries + ) + } + + /// The univariate part: what `stark::ProofOptions` carries. + pub fn proof_format(&self) -> ProofFormat { + ProofFormat { + merkle_cap: self.cap, + fri_mode: self.fri, + one_row: self.one_row, + // A test hook only; no knob sets it. + fri_schedule_override: None, + merge_equal_height_tables: self.lfm_merge, + } + } + + /// The WHIR part: what `multilinear::ChainConfig` carries. `whir_grind`'s + /// other half, its bits ([`Self::whir_grind_bits`]), is a security + /// parameter and goes in where the config is built + /// ([`crate::multilinear_prove::chain_config_under`]). + pub fn chain_format(&self) -> ChainFormat { + ChainFormat { + cap: self.whir_cap, + folds: self.whir_folds, + stack: self.whir_stack, + nonces: self.whir_grind.nonces(), + // No knob yet: the batched argue's proof type reaches no production + // caller until its wiring lands (D-BATCH B-4b), so a knob here could + // only make those callers refuse. + argue: multilinear::whir_chain::ArgueFormat::PerTable, + // Read at arity 4 only, so a binary chain (keccak, RPX) moves + // nothing. + arity4_cap: self.base.arity4_cap, + // The block's own constant (`BlockFormat::group`), set where the + // block builds its config: every other path opens per polynomial. + group: multilinear::whir_chain::GroupFold::PerPoly, + // The block's too (`BlockFormat::chain_config` stamps + // `whir_leaves`): the coset order is Poseidon1's alone, and the + // other chains this config serves (the W-LFM proofs over RPX, the + // epochs) keep the natural order. + leaves: LeafOrder::Natural, + } + } + + /// Stamp this format's univariate fields onto `options`. + pub fn apply_to_options(&self, options: &mut ProofOptions) { + options.format = self.proof_format(); + } + + /// `options` with this format's univariate fields. + pub fn options(&self, mut options: ProofOptions) -> ProofOptions { + self.apply_to_options(&mut options); + options + } + + /// Stamp this format's WHIR fields onto `config`. + pub fn apply_to_chain(&self, config: &mut ChainConfig) { + config.format = self.chain_format(); + } + + /// `config` with this format's WHIR fields. + pub fn chain(&self, mut config: ChainConfig) -> ChainConfig { + self.apply_to_chain(&mut config); + config + } +} + +fn parse_cap(name: &str, v: &str) -> Result { + v.parse().map_err(|e| format!("{name}={v:?}: {e}")) +} + +/// The first-round folds the knob accepts: the two arms that are built. +/// +/// ⚠ Not `first1..=first4`: a first fold narrower than the uniform one adds +/// rounds at some heights (Q would rise and the arms stop being comparable), +/// and `first4` IS `uniform4` under another statement word. Not `dp`: only +/// the first fold is a lever. Widening this list is a format decision, not a parser one. +pub const WHIR_FIRST_FOLDS: [usize; 2] = [5, 6]; + +/// `uniform4` | `first5` | `first6`. +fn parse_whir_folds(v: &str) -> Result { + if v == format!("uniform{PRODUCTION_WHIR_LOG_FOLDING}") { + return Ok(WhirFolds::Uniform); + } + WHIR_FIRST_FOLDS + .iter() + .find(|&&k| v == format!("first{k}")) + .and_then(|&k| FirstFold::new(k)) + .map(WhirFolds::First) + .ok_or_else(|| { + format!( + "{ENV_WHIR_FOLDS}={v:?}: expected `uniform{PRODUCTION_WHIR_LOG_FOLDING}`, {}", + WHIR_FIRST_FOLDS + .iter() + .map(|k| format!("`first{k}`")) + .collect::>() + .join(" or ") + ) + }) +} + +fn whir_folds_name(folds: &WhirFolds) -> String { + match folds { + WhirFolds::Uniform => format!("uniform{PRODUCTION_WHIR_LOG_FOLDING}"), + WhirFolds::First(k0) => format!("first{}", k0.get()), + } +} + +/// `20` | `18`. +fn parse_whir_grind_bits(v: &str) -> Result { + WHIR_GRIND_BITS + .iter() + .copied() + .find(|&b| v == b.to_string()) + .ok_or_else(|| { + format!( + "{ENV_WHIR_GRIND_BITS}={v:?}: expected {}", + WHIR_GRIND_BITS + .iter() + .map(|b| format!("`{b}`")) + .collect::>() + .join(" or ") + ) + }) +} + +/// `4` | `2`. +fn parse_whir_blowup(v: &str) -> Result { + WHIR_BLOWUPS + .iter() + .copied() + .find(|&b| v == b.to_string()) + .ok_or_else(|| { + format!( + "{ENV_WHIR_BLOWUP}={v:?}: expected {}", + WHIR_BLOWUPS + .iter() + .map(|b| format!("`{b}`")) + .collect::>() + .join(" or ") + ) + }) +} + +/// The stack caps the knob accepts: the three proved, verified and measured +/// to fit on the 32 GiB card (block 25368371, E2c). Not 28: its argument's +/// reservations would not fit the device ledger. Not below 25: nothing asks +/// for a narrower stack than today's. Widening this list is a format decision, +/// not a parser one. +pub const WHIR_STACKS: [usize; 3] = [25, 26, 27]; + +/// `25` | `26` | `27`. +fn parse_whir_stack(v: &str) -> Result { + WHIR_STACKS + .iter() + .find(|&&n| v == n.to_string()) + .and_then(|&n| StackVars::new(n)) + .ok_or_else(|| { + format!( + "{ENV_WHIR_STACK}={v:?}: expected {}", + WHIR_STACKS + .iter() + .map(|n| format!("`{n}`")) + .collect::>() + .join(", ") + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn parse(pairs: &[(&str, &str)]) -> Result { + let map: HashMap = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + ZfFormat::from_lookup(|k| map.get(k).cloned()) + } + + /// ★ With no knob set the process proves the MEASURED + /// configuration. + #[test] + fn nothing_set_is_the_measured_default() { + let f = parse(&[]).unwrap(); + assert_eq!(f, ZfFormat::DEFAULT); + assert_eq!(f, ZfFormat::default()); + assert_eq!( + f, + ZfFormat { + cap: CapPolicy::Auto, + whir_cap: CapPolicy::Auto, + fri: FriMode::Dp, + one_row: OneRowMode::Off, + whir_folds: WhirFolds::First(FirstFold::new(6).unwrap()), + whir_stack: StackVars::new(27).unwrap(), + whir_grind: WhirGrind::Query, + whir_grind_bits: 18, + whir_blowup: 2, + base: crate::hash_pin::BLOCK_BASE, + whir_leaves: LeafOrder::Coset, + lfm_merge: true, + } + ); + assert_eq!( + f.banner(), + "ZF FORMAT: cap=auto whir_cap=auto fri=dp one_row=0 whir_folds=first6 whir_stack=27 \ + whir_grind=query whir_grind_bits=18" + ); + assert!(!f.is_legacy()); + assert!(f.unimplemented_levers().is_empty()); + } + + /// The block's round-0 leaf order: the coset order unless set, `natural` + /// or `coset` in any case and spacing, nothing else accepted. The order + /// moves the block's chain config, its fold word and the banner (which + /// prints the knob only off the default); every other chain this format + /// serves stays natural (`chain_format`); and the coset order is refused + /// by name under a base whose trees are not Poseidon1. + #[test] + fn the_leaf_order_knob_selects_the_coset_order_on_poseidon1_alone() { + assert_eq!(parse(&[]).unwrap().whir_leaves, LeafOrder::Coset); + let coset = parse(&[(ENV_WHIR_LEAVES, " Coset ")]).unwrap(); + assert_eq!(coset, ZfFormat::DEFAULT); + let natural = parse(&[(ENV_WHIR_LEAVES, "natural")]).unwrap(); + assert_eq!(natural.whir_leaves, LeafOrder::Natural); + assert_eq!(coset.chain_format().leaves, LeafOrder::Natural); + assert_eq!(natural.chain_format(), coset.chain_format()); + assert!(!coset.is_legacy()); + assert_eq!( + natural.banner(), + format!("{} whir_leaves=natural", ZfFormat::DEFAULT.banner()) + ); + let block = |f: ZfFormat| { + crate::block_whir::BlockFormat { + zf: f, + ..crate::block_whir::BlockFormat::production() + } + .chain_config(&[(1, 27)]) + }; + assert_eq!(block(coset).format.leaves, LeafOrder::Coset); + assert_eq!(block(natural).format.leaves, LeafOrder::Natural); + assert_eq!( + block(coset).fold_word(), + block(natural).fold_word() + | multilinear::whir_chain::FOLD_WORD_TAG + | multilinear::whir_chain::FOLD_WORD_COSET + ); + for bad in ["", "bitrev", "reversed", "1", "on"] { + let err = parse(&[(ENV_WHIR_LEAVES, bad)]).unwrap_err(); + assert!(err.contains("`natural` or `coset`"), "{bad:?}: {err}"); + } + assert_eq!(coset.check_leaves(), Ok(())); + let err = coset.with_base(BaseFormat::RPX).check_leaves().unwrap_err(); + assert!( + err.contains("Poseidon1") && err.contains(ENV_WHIR_LEAVES), + "{err}" + ); + assert_eq!(natural.with_base(BaseFormat::RPX).check_leaves(), Ok(())); + } + + /// Every knob keeps its OFF spelling, and all of them at off are the + /// legacy format (every lever off): the rollback and A/B arm. + #[test] + fn the_off_spellings_parse_to_the_legacy_format() { + let f = parse(&[ + (ENV_CAP, "off"), + (ENV_WHIR_CAP, "0"), + (ENV_FRI, "pair"), + (ENV_ONE_ROW, "0"), + (ENV_WHIR_FOLDS, "uniform4"), + (ENV_WHIR_STACK, "25"), + (ENV_WHIR_GRIND, "all"), + (ENV_WHIR_GRIND_BITS, "20"), + (ENV_WHIR_BLOWUP, "4"), + (ENV_WHIR_LEAVES, "natural"), + (ENV_LFM_MERGE, "0"), + ]) + .unwrap(); + assert_eq!(f, ZfFormat::LEGACY); + assert!(f.is_legacy()); + assert_eq!( + f.banner(), + "ZF FORMAT: cap=off whir_cap=off fri=pair one_row=0 whir_folds=uniform4 whir_stack=25 \ + whir_grind=all whir_grind_bits=20 whir_blowup=4 whir_leaves=natural lfm_merge=0" + ); + assert!(f.unimplemented_levers().is_empty()); + assert!(f.proof_format().is_legacy()); + assert_eq!(f.proof_format(), ProofFormat::LEGACY); + // The chain format carries the block family's base cap, which only a + // 4-ary chain reads (inert under RPX and keccak: + // `whir_p1_tests::an_arity4_cap_moves_no_rpx_byte`). + assert_eq!(f.chain_format(), legacy_chain_format()); + } + + /// Today's chain format with the block family's base cap — the legacy + /// format's: every lever off. + fn legacy_chain_format() -> ChainFormat { + ChainFormat { + arity4_cap: crate::hash_pin::BLOCK_BASE.arity4_cap, + ..ChainFormat::DEFAULT + } + } + + /// One knob at its off spelling turns off that lever ONLY; the others keep + /// the default's value. + #[test] + fn one_off_knob_turns_off_one_lever() { + for (name, v, want) in [ + ( + ENV_CAP, + "off", + ZfFormat { + cap: CapPolicy::Off, + ..ZfFormat::DEFAULT + }, + ), + ( + ENV_WHIR_CAP, + "off", + ZfFormat { + whir_cap: CapPolicy::Off, + ..ZfFormat::DEFAULT + }, + ), + ( + ENV_FRI, + "pair", + ZfFormat { + fri: FriMode::Pair, + ..ZfFormat::DEFAULT + }, + ), + ( + ENV_WHIR_FOLDS, + "uniform4", + ZfFormat { + whir_folds: WhirFolds::Uniform, + ..ZfFormat::DEFAULT + }, + ), + ( + ENV_WHIR_STACK, + "25", + ZfFormat { + whir_stack: StackVars::LEGACY, + ..ZfFormat::DEFAULT + }, + ), + ( + ENV_WHIR_GRIND, + "all", + ZfFormat { + whir_grind: WhirGrind::All, + ..ZfFormat::DEFAULT + }, + ), + ( + ENV_WHIR_GRIND_BITS, + "20", + ZfFormat { + whir_grind_bits: LEGACY_WHIR_GRIND_BITS, + ..ZfFormat::DEFAULT + }, + ), + ] { + let f = parse(&[(name, v)]).unwrap(); + assert_eq!(f, want, "{name}={v}"); + assert!(!f.is_legacy(), "{name}={v}"); + } + } + + #[test] + fn every_accepted_spelling_parses() { + for (v, want) in [ + ("off", CapPolicy::Off), + ("auto", CapPolicy::Auto), + ("0", CapPolicy::Off), + ("3", CapPolicy::Fixed(3)), + ("16", CapPolicy::Fixed(16)), + (" AUTO ", CapPolicy::Auto), + ] { + assert_eq!(parse(&[(ENV_CAP, v)]).unwrap().cap, want, "{v:?}"); + assert_eq!(parse(&[(ENV_WHIR_CAP, v)]).unwrap().whir_cap, want, "{v:?}"); + } + assert_eq!(parse(&[(ENV_FRI, "dp")]).unwrap().fri, FriMode::Dp); + assert_eq!( + parse(&[(ENV_ONE_ROW, "1")]).unwrap().one_row, + OneRowMode::On + ); + assert_eq!( + parse(&[(ENV_ONE_ROW, "auto")]).unwrap().one_row, + OneRowMode::Auto + ); + for k in [5, 6] { + assert_eq!( + parse(&[(ENV_WHIR_FOLDS, &format!("first{k}"))]) + .unwrap() + .whir_folds, + WhirFolds::First(FirstFold::new(k).unwrap()) + ); + } + assert_eq!( + parse(&[(ENV_WHIR_FOLDS, " FIRST6 ")]).unwrap().whir_folds, + WhirFolds::First(FirstFold::new(6).unwrap()) + ); + for n in WHIR_STACKS { + assert_eq!( + parse(&[(ENV_WHIR_STACK, &n.to_string())]) + .unwrap() + .whir_stack, + StackVars::new(n).unwrap() + ); + } + assert_eq!( + parse(&[(ENV_WHIR_STACK, " 27 ")]).unwrap().whir_stack, + StackVars::new(27).unwrap() + ); + for (v, want) in [ + ("query", WhirGrind::Query), + ("all", WhirGrind::All), + (" ALL ", WhirGrind::All), + ] { + assert_eq!( + parse(&[(ENV_WHIR_GRIND, v)]).unwrap().whir_grind, + want, + "{v:?}" + ); + } + for (v, want) in [("20", 20u8), ("18", 18), (" 18 ", 18)] { + assert_eq!( + parse(&[(ENV_WHIR_GRIND_BITS, v)]).unwrap().whir_grind_bits, + want, + "{v:?}" + ); + } + for (v, want) in [("4", 4u8), ("2", 2), (" 2 ", 2)] { + assert_eq!( + parse(&[(ENV_WHIR_BLOWUP, v)]).unwrap().whir_blowup, + want, + "{v:?}" + ); + } + } + + #[test] + fn every_bad_spelling_is_refused_and_names_its_knob() { + for (name, v) in [ + (ENV_CAP, ""), + (ENV_CAP, "17"), + (ENV_CAP, "on"), + (ENV_CAP, "-1"), + (ENV_CAP, "3.0"), + (ENV_WHIR_CAP, "yes"), + (ENV_FRI, "binary"), + (ENV_FRI, ""), + (ENV_ONE_ROW, "2"), + (ENV_ONE_ROW, "on"), + (ENV_WHIR_FOLDS, "uniform"), + (ENV_WHIR_FOLDS, "uniform3"), + (ENV_WHIR_FOLDS, ""), + (ENV_WHIR_FOLDS, "4,4,4"), + (ENV_WHIR_FOLDS, "dp"), + (ENV_WHIR_FOLDS, "first"), + (ENV_WHIR_FOLDS, "first4"), + (ENV_WHIR_FOLDS, "first3"), + (ENV_WHIR_FOLDS, "first7"), + (ENV_WHIR_FOLDS, "first0"), + (ENV_WHIR_FOLDS, "first 6"), + (ENV_WHIR_FOLDS, "first06"), + (ENV_WHIR_FOLDS, "list:6,4"), + (ENV_WHIR_STACK, ""), + (ENV_WHIR_STACK, "24"), + (ENV_WHIR_STACK, "28"), + (ENV_WHIR_STACK, "0"), + (ENV_WHIR_STACK, "027"), + (ENV_WHIR_STACK, "27.0"), + (ENV_WHIR_STACK, "auto"), + (ENV_WHIR_STACK, "off"), + (ENV_WHIR_GRIND, ""), + (ENV_WHIR_GRIND, "uniform"), + (ENV_WHIR_GRIND, "0"), + (ENV_WHIR_GRIND, "1"), + (ENV_WHIR_GRIND, "off"), + (ENV_WHIR_GRIND, "query-only"), + (ENV_WHIR_GRIND, "20"), + (ENV_WHIR_GRIND_BITS, ""), + (ENV_WHIR_GRIND_BITS, "19"), + (ENV_WHIR_GRIND_BITS, "16"), + (ENV_WHIR_GRIND_BITS, "0"), + (ENV_WHIR_GRIND_BITS, "018"), + (ENV_WHIR_GRIND_BITS, "18.0"), + (ENV_WHIR_GRIND_BITS, "off"), + (ENV_WHIR_GRIND_BITS, "query"), + (ENV_WHIR_BLOWUP, ""), + (ENV_WHIR_BLOWUP, "1"), + (ENV_WHIR_BLOWUP, "8"), + (ENV_WHIR_BLOWUP, "0"), + (ENV_WHIR_BLOWUP, "02"), + (ENV_WHIR_BLOWUP, "2.0"), + (ENV_WHIR_BLOWUP, "half"), + (ENV_WHIR_BLOWUP, "off"), + ] { + let err = parse(&[(name, v)]).expect_err(&format!("{name}={v:?} must be refused")); + assert!(err.contains(name), "{err}"); + } + } + + #[test] + fn the_banner_round_trips_through_the_knobs() { + let f = ZfFormat { + cap: CapPolicy::Auto, + whir_cap: CapPolicy::Fixed(2), + fri: FriMode::Dp, + one_row: OneRowMode::Auto, + whir_folds: WhirFolds::First(FirstFold::new(6).unwrap()), + whir_stack: StackVars::new(26).unwrap(), + whir_grind: WhirGrind::All, + whir_grind_bits: 18, + whir_blowup: 4, + base: crate::hash_pin::BLOCK_BASE, + whir_leaves: LeafOrder::Natural, + lfm_merge: false, + }; + assert_eq!( + f.banner(), + "ZF FORMAT: cap=auto whir_cap=2 fri=dp one_row=auto whir_folds=first6 whir_stack=26 \ + whir_grind=all whir_grind_bits=18 whir_blowup=4 whir_leaves=natural lfm_merge=0" + ); + // Every banner value is a spelling its knob accepts, back to the same + // format. + let banner = f.banner(); + let fields: HashMap<&str, &str> = banner + .trim_start_matches("ZF FORMAT: ") + .split(' ') + .map(|kv| kv.split_once('=').unwrap()) + .collect(); + let back = parse(&[ + (ENV_CAP, fields["cap"]), + (ENV_WHIR_CAP, fields["whir_cap"]), + (ENV_FRI, fields["fri"]), + (ENV_ONE_ROW, fields["one_row"]), + (ENV_WHIR_FOLDS, fields["whir_folds"]), + (ENV_WHIR_STACK, fields["whir_stack"]), + (ENV_WHIR_GRIND, fields["whir_grind"]), + (ENV_WHIR_GRIND_BITS, fields["whir_grind_bits"]), + (ENV_WHIR_BLOWUP, fields["whir_blowup"]), + (ENV_WHIR_LEAVES, fields["whir_leaves"]), + (ENV_LFM_MERGE, fields["lfm_merge"]), + ]) + .unwrap(); + assert_eq!(back, f); + } + + #[test] + fn a_lever_this_build_lacks_is_reported() { + // Wave A implements none of the levers; the list names each knob set. + let f = parse(&[(ENV_CAP, "auto"), (ENV_FRI, "dp")]).unwrap(); + let missing = f.unimplemented_levers(); + // S3 is implemented on the host (stark::proof::options::FRI_MODE_IMPLEMENTED). + const { assert!(stark::proof::options::FRI_MODE_IMPLEMENTED) }; + assert!(!missing.contains(&ENV_FRI), "fri=dp is selectable"); + if !stark::proof::options::MERKLE_CAP_IMPLEMENTED { + assert!(missing.contains(&ENV_CAP)); + } + if !stark::proof::options::FRI_MODE_IMPLEMENTED { + assert!(missing.contains(&ENV_FRI)); + } + assert!( + !missing.contains(&ENV_ONE_ROW), + "an unset knob is never reported" + ); + } + + #[test] + fn the_one_row_knob_is_selectable() { + // S2 is implemented on the host CPU paths: `LAMBDA_VM_ZF_ONE_ROW` no + // longer aborts, and every spelling reaches the options unchanged. + const { assert!(stark::proof::options::ONE_ROW_IMPLEMENTED) }; + for (v, want) in [ + ("1", OneRowMode::On), + ("auto", OneRowMode::Auto), + ("0", OneRowMode::Off), + ] { + let f = parse(&[(ENV_ONE_ROW, v)]).unwrap(); + assert!(f.unimplemented_levers().is_empty(), "{v}"); + let base = crate::GoldilocksCubicProofOptions::with_blowup(4).unwrap(); + assert_eq!(f.options(base).format.one_row, want, "{v}"); + } + assert_eq!( + parse(&[(ENV_ONE_ROW, "auto"), (ENV_FRI, "dp")]) + .unwrap() + .banner(), + // Beside one-row openings the LFM merge is off (it has no one-row + // layout), and the banner says so. + "ZF FORMAT: cap=auto whir_cap=auto fri=dp one_row=auto whir_folds=first6 whir_stack=27 \ + whir_grind=query whir_grind_bits=18 lfm_merge=0" + ); + } + + #[test] + fn the_merkle_cap_knob_is_selectable() { + // The STARK cap is real on host and device, so `LAMBDA_VM_ZF_CAP` does + // not abort; every spelling reaches the options unchanged. + const { assert!(stark::proof::options::MERKLE_CAP_IMPLEMENTED) }; + for (v, want) in [ + ("auto", CapPolicy::Auto), + ("3", CapPolicy::Fixed(3)), + ("off", CapPolicy::Off), + ] { + let f = parse(&[(ENV_CAP, v)]).unwrap(); + assert!(!f.unimplemented_levers().contains(&ENV_CAP), "{v}"); + let base = crate::GoldilocksCubicProofOptions::with_blowup(4).unwrap(); + assert_eq!(f.options(base).format.merkle_cap, want, "{v}"); + } + } + + #[test] + fn the_whir_cap_is_implemented_and_selectable() { + const { assert!(multilinear::whir_chain::WHIR_CAP_IMPLEMENTED) }; + for v in ["auto", "3"] { + let f = parse(&[(ENV_WHIR_CAP, v)]).unwrap(); + assert!( + !f.unimplemented_levers().contains(&ENV_WHIR_CAP), + "LAMBDA_VM_ZF_WHIR_CAP={v} must be selectable" + ); + } + } + + #[test] + fn the_whir_fold_lever_is_selectable() { + const { assert!(multilinear::whir_chain::WHIR_FOLDS_IMPLEMENTED) }; + for v in ["first5", "first6"] { + let f = parse(&[(ENV_WHIR_FOLDS, v)]).unwrap(); + assert!(f.unimplemented_levers().is_empty(), "{v}"); + } + } + + /// S2: every accepted stack is selectable, reaches the chain config the + /// prover, the host verifier and the in-guest emitters all build their + /// layouts from, moves no other field, and 25 is the legacy value wherever + /// a default is spelled. + #[test] + fn the_whir_stack_lever_is_selectable_and_carried() { + const { assert!(multilinear::whir_chain::WHIR_STACK_IMPLEMENTED) }; + assert_eq!(ZfFormat::LEGACY.whir_stack, StackVars::LEGACY); + assert_eq!(ChainFormat::DEFAULT.stack, StackVars::LEGACY); + for n in WHIR_STACKS { + let f = parse(&[(ENV_WHIR_STACK, &n.to_string())]).unwrap(); + assert!(f.unimplemented_levers().is_empty(), "{n}"); + assert_eq!(f.chain_format().stack.get(), n); + let c = crate::multilinear_prove::chain_config_under(&f, &[(1, 25)]); + assert_eq!(c.format.stack.get(), n, "the config carries the stack"); + assert!( + f.banner() + .split(' ') + .any(|kv| kv == format!("whir_stack={n}")), + "{}", + f.banner() + ); + assert_eq!( + ZfFormat { + whir_stack: ZfFormat::DEFAULT.whir_stack, + ..f + }, + ZfFormat::DEFAULT, + "the knob moves the stack and nothing else" + ); + } + assert_eq!(WHIR_STACKS.iter().max(), Some(&StackVars::WIDEST)); + } + + /// ★ Q is charged the tallest STACKED polynomial, not only the widest + /// table: three narrow tables that each fit 14 variables stack to 16, a + /// four-round chain under first6 where each table alone runs three. + #[test] + fn the_query_count_charges_the_tallest_stacked_polynomial() { + use crate::multilinear_prove::chain_config_under; + let narrow = [(1, 14), (1, 14), (1, 14)]; + let c = chain_config_under(&ZfFormat::DEFAULT, &narrow); + assert_eq!( + stark::multilinear_table::stack_height(&narrow, ZfFormat::DEFAULT.whir_stack), + 16 + ); + assert_eq!( + (c.rounds(16), c.num_queries), + (4, 228), + "charged at 16 variables" + ); + // What the widest table alone would charge: three rounds, one query fewer. + let widest_alone = ChainConfig::with_security_folds( + PRODUCTION_WHIR_LOG_BLOWUP, + PRODUCTION_WHIR_LOG_FOLDING, + ZfFormat::DEFAULT.whir_folds, + 14, + 128, + ZfFormat::DEFAULT.whir_grind_bits(), + ); + assert_eq!(widest_alone.num_queries, 227); + // At the production shapes the charge moves nothing, at every accepted + // stack: the widest table stands at or above the cap, and Q is 229 at + // every production height. + for n in WHIR_STACKS { + let f = ZfFormat { + whir_stack: StackVars::new(n).unwrap(), + ..ZfFormat::DEFAULT + }; + for shapes in [ + &[(1usize, 25usize)][..], + &[(64, 21), (1480, 16), (8, 20)][..], + ] { + assert_eq!( + chain_config_under(&f, shapes).num_queries, + 229, + "stack {n}, shapes {shapes:?}" + ); + } + } + } + + #[test] + fn apply_stamps_only_the_format_fields() { + let base = crate::GoldilocksCubicProofOptions::with_blowup(4).unwrap(); + assert!(base.has_default_format()); + let f = ZfFormat { + cap: CapPolicy::Auto, + fri: FriMode::Dp, + one_row: OneRowMode::On, + ..ZfFormat::DEFAULT + }; + let o = f.options(base.clone()); + assert_eq!(o.format.merkle_cap, CapPolicy::Auto); + assert_eq!(o.format.fri_mode, FriMode::Dp); + assert_eq!(o.format.one_row, OneRowMode::On); + assert!(!o.has_default_format()); + assert_eq!(o.blowup_factor, base.blowup_factor); + assert_eq!(o.fri_number_of_queries, base.fri_number_of_queries); + assert_eq!(o.grinding_factor, base.grinding_factor); + assert_eq!(o.coset_offset, base.coset_offset); + assert_eq!(o.fri_final_poly_log_degree, base.fri_final_poly_log_degree); + // The legacy format leaves options untouched. + let d = ZfFormat::LEGACY.options(base.clone()); + assert!(d.has_default_format()); + assert!(d.has_legacy_format()); + // The production default stamps cap=auto and fri=dp, nothing else. + let p = ZfFormat::DEFAULT.options(base.clone()); + assert_eq!(p.format.merkle_cap, CapPolicy::Auto); + assert_eq!(p.format.fri_mode, FriMode::Dp); + assert_eq!(p.format.one_row, ZfFormat::DEFAULT.one_row); + assert_eq!(p.format.fri_schedule_override, None); + assert!(!p.has_legacy_format()); + assert_eq!( + ( + p.blowup_factor, + p.fri_number_of_queries, + p.grinding_factor, + p.coset_offset, + p.fri_final_poly_log_degree + ), + ( + base.blowup_factor, + base.fri_number_of_queries, + base.grinding_factor, + base.coset_offset, + base.fri_final_poly_log_degree + ), + "no security parameter moves with the format" + ); + + let chain = crate::multilinear_prove::chain_config_under(&ZfFormat::LEGACY, &[(8, 20)]); + let c = ZfFormat { + whir_cap: CapPolicy::Fixed(3), + whir_folds: WhirFolds::First(FirstFold::new(5).unwrap()), + ..ZfFormat::LEGACY + } + .chain(chain); + assert_eq!(c.format.cap, CapPolicy::Fixed(3)); + assert_eq!(c.format.folds, WhirFolds::First(FirstFold::new(5).unwrap())); + assert_eq!( + (c.log_blowup, c.log_folding, c.num_queries, c.grind), + ( + chain.log_blowup, + chain.log_folding, + chain.num_queries, + chain.grind + ) + ); + } + + #[test] + fn the_schedule_line_states_the_rounds() { + assert_eq!( + ZfFormat::LEGACY.whir_schedule_line(), + "ZF WHIR SCHEDULES: whir_folds=uniform4 q=112 n=20:[4,4,4,4,4] \ + n=21:[4,4,4,4,4,1] n=22:[4,4,4,4,4,2] n=23:[4,4,4,4,4,3] \ + n=24:[4,4,4,4,4,4] n=25:[4,4,4,4,4,4,1]" + ); + let first6 = ZfFormat { + whir_folds: WhirFolds::First(FirstFold::new(6).unwrap()), + ..ZfFormat::LEGACY + }; + // The production default runs the first6 schedules, to its stack, at + // its grind bits' Q. + assert_eq!( + ZfFormat::DEFAULT.whir_schedule_line(), + ZfFormat { + whir_stack: ZfFormat::DEFAULT.whir_stack, + whir_grind_bits: ZfFormat::DEFAULT.whir_grind_bits, + whir_blowup: ZfFormat::DEFAULT.whir_blowup, + ..first6 + } + .whir_schedule_line() + ); + assert_eq!( + first6.whir_schedule_line(), + "ZF WHIR SCHEDULES: whir_folds=first6 q=112 n=20:[6,4,4,4,2] \ + n=21:[6,4,4,4,3] n=22:[6,4,4,4,4] n=23:[6,4,4,4,4,1] \ + n=24:[6,4,4,4,4,2] n=25:[6,4,4,4,4,3]" + ); + // Under a wider stack the line runs to it, with Q charged there. + let wide = ZfFormat { + whir_stack: StackVars::new(27).unwrap(), + ..first6 + }; + assert_eq!( + wide.whir_schedule_line(), + "ZF WHIR SCHEDULES: whir_folds=first6 q=112 n=20:[6,4,4,4,2] \ + n=21:[6,4,4,4,3] n=22:[6,4,4,4,4] n=23:[6,4,4,4,4,1] \ + n=24:[6,4,4,4,4,2] n=25:[6,4,4,4,4,3] n=26:[6,4,4,4,4,4] \ + n=27:[6,4,4,4,4,4,1]" + ); + } + + /// The production WHIR config under each accepted knob value: the format + /// is carried, Q is charged the schedule's rounds, and at the block's + /// tallest stack (25) every fold arm keeps the legacy Q = 112 at 20 grind + /// bits, and the default's Q = 114 at its 18. + #[test] + fn the_production_chain_config_under_each_arm() { + use crate::multilinear_prove::chain_config_under; + let today = chain_config_under(&ZfFormat::LEGACY, &[(1, 25)]); + assert_eq!(today.format, legacy_chain_format()); + assert_eq!((today.rounds(25), today.num_queries), (7, 112)); + // The production config (no knob set) is the measured default's: + // cap auto, first6 — six rounds at 25; Q 114 at 18 grind bits. + let production = crate::multilinear_prove::chain_config(&[(1, 25)]); + assert_eq!( + production, + chain_config_under(&ZfFormat::DEFAULT, &[(1, 25)]) + ); + assert_eq!(production.format, ZfFormat::DEFAULT.chain_format()); + assert_eq!((production.rounds(25), production.num_queries), (6, 229)); + assert_eq!(production.schedule(25), vec![6, 4, 4, 4, 4, 3]); + assert_eq!( + production.log_folding, today.log_folding, + "the fold does not move with the format" + ); + assert_eq!( + (today.log_blowup, production.log_blowup), + (LEGACY_WHIR_LOG_BLOWUP, PRODUCTION_WHIR_LOG_BLOWUP), + "the legacy format's blowup 4, the default's 2" + ); + // P2: the default grinds before the queries alone, the legacy format + // before all three challenges; the default at 18 bits, the legacy at 20. + assert_eq!(production.grind, GrindBits::query_only(18)); + assert_eq!(today.grind, GrindBits::uniform(20)); + for (name, rounds25) in [("first5", 6), ("first6", 6)] { + let f = parse(&[(ENV_WHIR_FOLDS, name)]).unwrap(); + let c = chain_config_under(&f, &[(1, 25)]); + assert_eq!(c.format.folds, f.whir_folds); + assert_eq!((c.rounds(25), c.num_queries), (rounds25, 229), "{name}"); + assert_eq!( + (c.log_blowup, c.log_folding, c.grind), + ( + production.log_blowup, + today.log_folding, + f.whir_grind_bits() + ) + ); + assert_ne!(c.fold_word(), today.fold_word()); + } + } + + /// ★ `LAMBDA_VM_ZF_WHIR_BLOWUP`: blowup 2 is the default and the default + /// banner does not name it; blowup 4 is the legacy format's and the + /// default's rollback. Blowup 4 moves the chains' rate and the query count + /// the rule charges for it, and nothing else: not the WHIR format fields, + /// not the univariate options the recursion's proofs are built from. + #[test] + fn the_blowup_knob_moves_the_rate_and_the_rule_s_queries_only() { + use crate::multilinear_prove::chain_config_under; + assert_eq!(WHIR_BLOWUPS, [4, 2]); + assert_eq!(ZfFormat::DEFAULT.whir_blowup, 2); + assert_eq!(ZfFormat::LEGACY.whir_blowup, 4); + assert_eq!( + ZfFormat::DEFAULT.whir_log_blowup(), + PRODUCTION_WHIR_LOG_BLOWUP + ); + assert_eq!(ZfFormat::LEGACY.whir_log_blowup(), LEGACY_WHIR_LOG_BLOWUP); + assert_eq!(parse(&[(ENV_WHIR_BLOWUP, "2")]).unwrap(), ZfFormat::DEFAULT); + assert!(!ZfFormat::DEFAULT.banner().contains("whir_blowup")); + + let four = parse(&[(ENV_WHIR_BLOWUP, "4")]).unwrap(); + assert_eq!(four.whir_log_blowup(), 2); + assert!(four.unimplemented_levers().is_empty()); + assert!(!four.is_legacy()); + assert!( + four.banner().ends_with(" whir_grind_bits=18 whir_blowup=4"), + "{}", + four.banner() + ); + assert_eq!( + ZfFormat { + whir_blowup: 2, + ..four + }, + ZfFormat::DEFAULT, + "the knob moves the blowup and nothing else" + ); + assert_eq!(four.chain_format(), ZfFormat::DEFAULT.chain_format()); + assert_eq!(four.proof_format(), ZfFormat::DEFAULT.proof_format()); + let lfm = || stark::proof::options::GoldilocksCubicProofOptions::with_blowup(4).unwrap(); + assert_eq!( + format!("{:?}", four.options(lfm())), + format!("{:?}", ZfFormat::DEFAULT.options(lfm())), + "the recursion's proofs do not read the WHIR rate" + ); + assert!(four.whir_schedule_line().contains(" q=114 ")); + assert!(ZfFormat::DEFAULT.whir_schedule_line().contains(" q=229 ")); + + for n in WHIR_STACKS { + let stack = StackVars::new(n).unwrap(); + let wide = ZfFormat { + whir_stack: stack, + ..four + }; + let two = ZfFormat { + whir_stack: stack, + ..ZfFormat::DEFAULT + }; + for shapes in [ + &[(1usize, n)][..], + &[(1, 27)][..], + &[(64, 21), (1480, 16), (8, 20)][..], + ] { + let c4 = chain_config_under(&wide, shapes); + let c2 = chain_config_under(&two, shapes); + assert_eq!( + (c4.log_blowup, c4.num_queries, c2.log_blowup, c2.num_queries), + (2, 114, 1, 229), + "stack {n}, shapes {shapes:?}" + ); + assert_eq!( + ChainConfig { + log_blowup: c2.log_blowup, + num_queries: c2.num_queries, + ..c4 + }, + c2, + "stack {n}, shapes {shapes:?}: only the rate and Q move" + ); + } + } + // Every height a production chain reaches: Q from the rule at the worst + // round count up to that height, 229 from 23 variables (seven rounds at + // 27, six from 23) and 228 from 15 to 22 (five rounds), where blowup 4 + // charges 114 throughout. + for n in 15..=27 { + let c4 = chain_config_under(&four, &[(1, n)]); + let c2 = chain_config_under(&ZfFormat::DEFAULT, &[(1, n)]); + let worst = (1..=n).map(|m| c2.rounds(m)).max().unwrap(); + assert_eq!( + c2.num_queries, + multilinear::query_count::num_queries(1, worst, 128, 18), + "{n}" + ); + assert_eq!( + (c4.num_queries, c2.num_queries), + (114, if n >= 23 { 229 } else { 228 }), + "{n}" + ); + } + } + + /// `install` makes a caller's format the process format only before + /// anything has read it: once the environment's is out (and its banner + /// printed), another value is refused, naming both, and the same value is + /// accepted. A test process sets no knob, so its format is the default. + #[test] + fn a_format_is_installed_only_before_the_process_format_is_read() { + let process = *ZfFormat::global(); + assert_eq!(process, ZfFormat::DEFAULT); + assert_eq!(ZfFormat::install(process).copied(), Ok(process)); + let four = ZfFormat { + whir_blowup: 4, + ..process + }; + let refused = ZfFormat::install(four).expect_err("the process format is already out"); + assert!(refused.contains("read before it could be set"), "{refused}"); + assert!(refused.contains(" whir_blowup=4"), "{refused}"); + assert_eq!(*ZfFormat::global(), process, "the refusal changed nothing"); + } + + /// The named blowups are the accepted list, and print as their factors. + #[test] + fn the_named_blowups_are_the_accepted_ones() { + assert_eq!( + [WhirBlowup::Four.factor(), WhirBlowup::Two.factor()], + WHIR_BLOWUPS + ); + assert_eq!(WhirBlowup::Two.factor(), ZfFormat::DEFAULT.whir_blowup); + assert_eq!(WhirBlowup::Four.factor(), ZfFormat::LEGACY.whir_blowup); + assert_eq!( + (WhirBlowup::Four.to_string(), WhirBlowup::Two.to_string()), + ("4".to_string(), "2".to_string()) + ); + } + + #[test] + fn production_sites_build_the_default_format_when_nothing_is_set() { + // No test sets a ZF knob, so the process format is the default and the + // production constructors must stamp the MEASURED configuration. + assert_eq!(*ZfFormat::global(), ZfFormat::DEFAULT); + let want = ZfFormat::DEFAULT.proof_format(); + for (site, o) in [ + ( + "aggregation_wrap_options", + crate::lfm::proof::aggregation_wrap_options(), + ), + ( + "block_base_options", + crate::lfm::proof::block_base_options(), + ), + ] { + assert_eq!(o.format, want, "{site}"); + assert!(!o.has_legacy_format(), "{site}"); + } + // Security parameters are the legacy presets'. + let base = crate::lfm::proof::block_base_options(); + let preset = crate::recursion::Preset::Blowup4.options(); + assert_eq!( + ( + base.blowup_factor, + base.fri_number_of_queries, + base.grinding_factor, + base.coset_offset, + base.fri_final_poly_log_degree + ), + ( + preset.blowup_factor, + preset.fri_number_of_queries, + preset.grinding_factor, + preset.coset_offset, + preset.fri_final_poly_log_degree + ) + ); + let chain = crate::multilinear_prove::chain_config(&[(8, 20)]); + assert_eq!(chain.format, ZfFormat::DEFAULT.chain_format()); + assert_eq!(chain.log_folding, PRODUCTION_WHIR_LOG_FOLDING); + } + + /// ★ The RV64 guest verifier stays on + /// the LEGACY format after the default flip. Its presets NAME the legacy + /// format (not the process default), and both guest entries refuse every + /// other format — the production default included. + #[test] + fn the_recursion_guest_stays_on_the_legacy_format() { + for preset in crate::recursion::Preset::ALL { + let o = preset.options(); + assert_eq!(o.format, ProofFormat::LEGACY, "{}", preset.name()); + assert!(o.has_legacy_format(), "{}", preset.name()); + } + assert_eq!( + crate::recursion::MIN_PROOF_OPTIONS.format, + ProofFormat::LEGACY + ); + // The production default is NOT legacy, so the presets cannot have + // inherited their format from it. + assert!(!ZfFormat::DEFAULT.is_legacy()); + assert!(!ZfFormat::DEFAULT.proof_format().is_legacy()); + + let base = crate::recursion::Preset::Blowup4.options(); + let refused = |opts: &ProofOptions| { + for result in [ + crate::recursion::verify_and_attest_blob(&[], opts), + crate::recursion::verify_continuation_and_attest(&[], opts), + ] { + let err = result.expect_err("a non-legacy format must be refused"); + assert!(format!("{err:?}").contains("legacy-format"), "{err:?}"); + } + }; + for f in [ + ZfFormat::DEFAULT, + ZfFormat { + cap: CapPolicy::Auto, + ..ZfFormat::LEGACY + }, + ZfFormat { + fri: FriMode::Dp, + ..ZfFormat::LEGACY + }, + ZfFormat { + one_row: OneRowMode::On, + ..ZfFormat::LEGACY + }, + ZfFormat { + one_row: OneRowMode::Auto, + ..ZfFormat::LEGACY + }, + ] { + refused(&f.options(base.clone())); + } + // The production STARK base options are refused whenever the process + // format is not legacy (always, with no knob set). + if !ZfFormat::global().proof_format().is_legacy() { + refused(&crate::lfm::proof::block_base_options()); + } + // The legacy format gets past the guard (and fails on the empty blob). + for opts in [base.clone(), ZfFormat::LEGACY.options(base.clone())] { + for result in [ + crate::recursion::verify_and_attest_blob(&[], &opts), + crate::recursion::verify_continuation_and_attest(&[], &opts), + ] { + if let Err(err) = result { + assert!(!format!("{err:?}").contains("legacy-format"), "{err:?}"); + } + } + } + } + + #[test] + fn the_serialized_options_bytes_ignore_the_format_fields() { + // The format fields are skipped by serde and rkyv, so a + // serialized `ProofOptions` has the same bytes whatever the format, + // and deserializes to the default format. + let base = crate::GoldilocksCubicProofOptions::with_blowup(4).unwrap(); + let capped = ZfFormat { + cap: CapPolicy::Auto, + fri: FriMode::Dp, + one_row: OneRowMode::Auto, + ..ZfFormat::DEFAULT + } + .options(base.clone()); + let a = rkyv::to_bytes::(&base).unwrap(); + let b = rkyv::to_bytes::(&capped).unwrap(); + assert_eq!(a.as_slice(), b.as_slice()); + let back: ProofOptions = rkyv::from_bytes::(&b).unwrap(); + assert!(back.has_default_format()); + assert_eq!(back.fri_number_of_queries, base.fri_number_of_queries); + + let ja = serde_json::to_string(&base).unwrap(); + let jb = serde_json::to_string(&capped).unwrap(); + assert_eq!(ja, jb); + assert!(!ja.contains("merkle_cap") && !ja.contains("fri_mode") && !ja.contains("one_row")); + let back: ProofOptions = serde_json::from_str(&jb).unwrap(); + assert!(back.has_default_format()); + } + + /// P2: each spelling reaches the chain config the prover, the host verifier + /// and the in-guest emitters build from, as its bits and its nonce layout + /// together, at every accepted stack; Q does not move, and the knob moves + /// nothing else. + #[test] + fn the_whir_grind_lever_is_selectable_and_carried() { + for (v, bits, nonces) in [ + ("query", GrindBits::query_only(18), NonceLayout::Spent), + ("all", GrindBits::uniform(18), NonceLayout::Three), + ] { + let f = parse(&[(ENV_WHIR_GRIND, v)]).unwrap(); + assert!(f.unimplemented_levers().is_empty(), "{v}"); + assert!( + f.banner() + .ends_with(&format!(" whir_grind={v} whir_grind_bits=18")), + "{}", + f.banner() + ); + for n in WHIR_STACKS { + let at = ZfFormat { + whir_stack: StackVars::new(n).unwrap(), + ..f + }; + let c = crate::multilinear_prove::chain_config_under(&at, &[(1, n)]); + assert_eq!((c.grind, c.format.nonces), (bits, nonces), "{v} at {n}"); + assert_eq!( + c.num_queries, 229, + "{v} at {n}: Q reads the query grind alone" + ); + } + assert_eq!( + ZfFormat { + whir_grind: ZfFormat::DEFAULT.whir_grind, + ..f + }, + ZfFormat::DEFAULT, + "the knob moves the grind and nothing else" + ); + } + assert_eq!(ZfFormat::LEGACY.whir_grind, WhirGrind::All); + assert_eq!(ChainFormat::DEFAULT.nonces, NonceLayout::Three); + } + + /// ★ THE OPT-OUT IS TODAY'S CONFIG. `LAMBDA_VM_ZF_WHIR_GRIND=all` with + /// `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20` builds the production WHIR config + /// exactly as it was before P2, spelled out here as a literal rather than + /// derived through the code under test: blowup 2^2, fold 4 under first6, + /// Q 112, 20-bit grinds before all three challenges, the auto cap, stack 27, + /// three nonces a round, at the legacy blowup 4 (`LAMBDA_VM_ZF_WHIR_BLOWUP=4`). + /// The default differs from it in the grind (placed before the queries + /// alone, at 18 bits), its rate (blowup 2) and Q (229) and the nonce + /// layout, and in nothing else. + #[test] + fn the_grind_opt_out_is_the_production_config_before_p2() { + let before_p2 = ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 112, + grind: GrindBits { + folding: 20, + ood: 20, + query: 20, + }, + format: ChainFormat { + cap: CapPolicy::Auto, + folds: WhirFolds::First(FirstFold::new(6).unwrap()), + stack: StackVars::new(27).unwrap(), + nonces: NonceLayout::Three, + argue: multilinear::whir_chain::ArgueFormat::PerTable, + arity4_cap: crate::hash_pin::BLOCK_BASE.arity4_cap, + group: multilinear::whir_chain::GroupFold::PerPoly, + leaves: multilinear::whir_chain::LeafOrder::Natural, + }, + }; + let opt_out = parse(&[ + (ENV_WHIR_GRIND, "all"), + (ENV_WHIR_GRIND_BITS, "20"), + (ENV_WHIR_BLOWUP, "4"), + ]) + .unwrap(); + for shapes in [ + &[(1usize, 25usize)][..], + &[(1, 27)][..], + &[(64, 21), (1480, 16), (8, 20)][..], + ] { + assert_eq!( + crate::multilinear_prove::chain_config_under(&opt_out, shapes), + before_p2, + "{shapes:?}" + ); + assert_eq!( + crate::multilinear_prove::chain_config_under(&ZfFormat::DEFAULT, shapes), + ChainConfig { + log_blowup: PRODUCTION_WHIR_LOG_BLOWUP, + num_queries: 229, + grind: GrindBits { + folding: 0, + ood: 0, + query: 18, + }, + format: ChainFormat { + nonces: NonceLayout::Spent, + ..before_p2.format + }, + ..before_p2 + }, + "{shapes:?}" + ); + } + } + + /// The STARK pipeline has nothing to adopt: it grinds only before its + /// queries already. The lever reaches no univariate option, so no STARK + /// proof, wrap program or id moves with it. + #[test] + fn the_grind_lever_moves_no_stark_option() { + let all = ZfFormat { + whir_grind: WhirGrind::All, + ..ZfFormat::DEFAULT + }; + assert_eq!(all.proof_format(), ZfFormat::DEFAULT.proof_format()); + for preset in crate::recursion::Preset::ALL { + let o = preset.options(); + assert_eq!( + format!("{:?}", all.options(o.clone())), + format!("{:?}", ZfFormat::DEFAULT.options(o)), + "{}", + preset.name() + ); + } + } + + /// The default's 18 grind bits trade two grind bits for two queries against + /// `LAMBDA_VM_ZF_WHIR_GRIND_BITS=20` (the opt-out, the legacy bits) at + /// blowup 4, where the trade was sized, and move nothing else: Q 112 → 114 + /// at every production height (15 to 27 variables, every accepted stack, + /// the block's shapes), 111 → 113 below a fourth round, under either + /// placement. At the default's blowup 2 the same two bits buy four queries + /// at every production height. + #[test] + fn the_grind_bits_knob_trades_two_bits_for_two_queries() { + use crate::multilinear_prove::chain_config_under; + assert_eq!(PRODUCTION_WHIR_GRIND_BITS, 18); + assert_eq!(LEGACY_WHIR_GRIND_BITS, 20); + assert_eq!( + ZfFormat::DEFAULT.whir_grind_bits, + PRODUCTION_WHIR_GRIND_BITS + ); + assert_eq!(ZfFormat::LEGACY.whir_grind_bits, LEGACY_WHIR_GRIND_BITS); + assert_eq!( + parse(&[(ENV_WHIR_GRIND_BITS, "18")]).unwrap(), + ZfFormat::DEFAULT + ); + let four = ZfFormat { + whir_blowup: 4, + ..ZfFormat::DEFAULT + }; + assert!(four.whir_schedule_line().contains(" q=114 ")); + assert!(ZfFormat::DEFAULT.whir_schedule_line().contains(" q=229 ")); + + // The opt-out: 20 bits, Q 112 at blowup 4 (225 at the default's 2), + // nothing else moved. + let f = parse(&[(ENV_WHIR_GRIND_BITS, "20")]).unwrap(); + assert!(f.unimplemented_levers().is_empty()); + assert!(!f.is_legacy()); + assert!( + f.banner().ends_with(" whir_grind=query whir_grind_bits=20"), + "{}", + f.banner() + ); + assert_eq!( + ZfFormat { + whir_grind_bits: PRODUCTION_WHIR_GRIND_BITS, + ..f + }, + ZfFormat::DEFAULT, + "the knob moves the bits and nothing else" + ); + assert_eq!(f.chain_format(), ZfFormat::DEFAULT.chain_format()); + assert_eq!(f.proof_format(), ZfFormat::DEFAULT.proof_format()); + assert!( + ZfFormat { + whir_blowup: 4, + ..f + } + .whir_schedule_line() + .contains(" q=112 ") + ); + assert!(f.whir_schedule_line().contains(" q=225 ")); + + for (placement, bits18, bits20) in [ + ( + WhirGrind::Query, + GrindBits::query_only(18), + GrindBits::query_only(20), + ), + ( + WhirGrind::All, + GrindBits::uniform(18), + GrindBits::uniform(20), + ), + ] { + let at18 = ZfFormat { + whir_grind: placement, + ..four + }; + let at20 = ZfFormat { + whir_grind: placement, + whir_blowup: 4, + ..f + }; + for n in WHIR_STACKS { + let stack = StackVars::new(n).unwrap(); + for shapes in [ + &[(1usize, n)][..], + &[(1, 27)][..], + &[(64, 21), (1480, 16), (8, 20)][..], + ] { + let c18 = chain_config_under( + &ZfFormat { + whir_stack: stack, + ..at18 + }, + shapes, + ); + let c20 = chain_config_under( + &ZfFormat { + whir_stack: stack, + ..at20 + }, + shapes, + ); + assert_eq!( + (c18.num_queries, c18.grind, c20.num_queries, c20.grind), + (114, bits18, 112, bits20), + "{placement} at stack {n}, shapes {shapes:?}" + ); + assert_eq!( + ChainConfig { + num_queries: c20.num_queries, + grind: c20.grind, + ..c18 + }, + c20, + "{placement} at stack {n}, shapes {shapes:?}: only Q and the grind move" + ); + } + } + // Below a fourth round (14 variables and narrower) both sides buy + // one query less. + assert_eq!( + ( + chain_config_under(&at18, &[(1, 14)]).num_queries, + chain_config_under(&at20, &[(1, 14)]).num_queries + ), + (113, 111), + "{placement}" + ); + } + // Every height a production chain reaches, from the rule itself. + for n in 15..=27 { + let q = |bits| { + ChainConfig::with_security_folds( + 2, + PRODUCTION_WHIR_LOG_FOLDING, + ZfFormat::DEFAULT.whir_folds, + n, + 128, + GrindBits::query_only(bits), + ) + .num_queries + }; + assert_eq!((q(18), q(20)), (114, 112), "{n} variables"); + let at_default = |bits| { + ChainConfig::with_security_folds( + PRODUCTION_WHIR_LOG_BLOWUP, + PRODUCTION_WHIR_LOG_FOLDING, + ZfFormat::DEFAULT.whir_folds, + n, + 128, + GrindBits::query_only(bits), + ) + .num_queries + }; + assert_eq!( + at_default(18), + at_default(20) + 4, + "{n} variables at blowup 2" + ); + } + } +} diff --git a/prover/tests/calibration.rs b/prover/tests/calibration.rs deleted file mode 100644 index a87f13175..000000000 --- a/prover/tests/calibration.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! Asserts predicted `peak_bytes` does not underestimate jemalloc-measured -//! heap during a proof. Lives in its own integration-test binary so that -//! `#[global_allocator]` and `tikv_jemalloc_ctl::stats::allocated` reads are -//! isolated from the rest of the prover test suite. - -#![cfg(feature = "disk-spill")] - -use lambda_vm_prover::auto_storage::{SAFETY_FRACTION_DEN, SAFETY_FRACTION_NUM, peak_bytes}; -use lambda_vm_prover::prove_with_options_and_inputs; -use lambda_vm_prover::tables::MaxRowsConfig; -use lambda_vm_prover::tables::trace_builder::count_table_lengths; -use lambda_vm_prover::test_utils::{asm_elf_bytes, run_asm_elf}; -use stark::proof::options::GoldilocksCubicProofOptions; -use stark::prover::storage_estimate_parallelism; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::thread; -use std::time::Duration; -use tikv_jemalloc_ctl::{epoch, stats}; - -#[global_allocator] -static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - -// ...with the shipped binary's purge policy, so this binary is the production -// allocator *configuration* and not just the production allocator. The reason -// and the numbers are at `bin/cli/src/main.rs`. `prover/tests/jemalloc_conf.rs` -// asserts that this export pattern is read, but it does so against its own copy -// in its own process — nothing checks the copy below. -// -// It moves nothing this file asserts — `stats::allocated` is live bytes, which -// the decay timers do not touch; a resident-memory assertion added here later -// would read the wrong configuration without it. -const NEVER_PURGE: &[u8] = b"dirty_decay_ms:-1,muzzy_decay_ms:-1\0"; - -#[allow(non_upper_case_globals)] -#[unsafe(export_name = "_rjem_malloc_conf")] -pub static malloc_conf: Option<&'static core::ffi::c_char> = - Some(unsafe { &*(NEVER_PURGE.as_ptr() as *const core::ffi::c_char) }); - -fn allocated_bytes() -> usize { - epoch::advance().ok(); - stats::allocated::read().unwrap_or(0) -} - -#[test] -fn peak_bytes_does_not_underestimate_measured_heap() { - let (elf, logs, _) = run_asm_elf("fib_iterative_372k"); - let elf_bytes = asm_elf_bytes("fib_iterative_372k"); - - let max_rows = MaxRowsConfig::default(); - let lengths = - count_table_lengths(&elf, &logs, &max_rows, &[]).expect("count_table_lengths succeeds"); - - let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid"); - let predicted = - peak_bytes(&lengths, opts.blowup_factor, storage_estimate_parallelism()) as usize; - - drop(logs); - - let baseline = allocated_bytes(); - let peak = Arc::new(AtomicUsize::new(baseline)); - let stop = Arc::new(AtomicBool::new(false)); - - let sampler = { - let peak = Arc::clone(&peak); - let stop = Arc::clone(&stop); - thread::spawn(move || { - while !stop.load(Ordering::Relaxed) { - peak.fetch_max(allocated_bytes(), Ordering::Relaxed); - thread::sleep(Duration::from_millis(10)); - } - }) - }; - - let _proof = - prove_with_options_and_inputs(&elf_bytes, &[], &opts, &max_rows).expect("proof succeeds"); - - stop.store(true, Ordering::Relaxed); - sampler.join().expect("sampler joins"); - - let measured = peak.load(Ordering::Relaxed).saturating_sub(baseline); - - eprintln!( - "peak_bytes calibration: predicted={predicted} bytes, measured_heap={measured} bytes, ratio={:.2}", - predicted as f64 / measured as f64 - ); - - let safety_num = SAFETY_FRACTION_NUM as usize; - let safety_den = SAFETY_FRACTION_DEN as usize; - assert!( - predicted.saturating_mul(safety_den) >= measured.saturating_mul(safety_num), - "peak_bytes underestimates measured heap: predicted={predicted}, measured={measured}" - ); -} diff --git a/prover/tests/cuda_fallback_tests.rs b/prover/tests/cuda_fallback_tests.rs index 6fb776022..f99acf7d1 100644 --- a/prover/tests/cuda_fallback_tests.rs +++ b/prover/tests/cuda_fallback_tests.rs @@ -80,7 +80,10 @@ fn gpu_fri_fault_falls_back_to_cpu() { /// Batch-invert CPU fallback: when `compute_and_invert_denoms_ext3_dev` /// errors, `try_compute_and_invert_inv_denoms_dev` must return None so the /// caller (R3 OOD in `trace.rs` or R4 DEEP in `prover.rs`) builds inv_denoms -/// on CPU and the remaining GPU path keeps running. +/// on CPU and the remaining GPU path keeps running. The fully resident R4 DEEP +/// inverts inside its own kernel by default and never reaches that entry +/// point, so there the faults land on R3's inversions (R4's buffered path takes +/// them under `LAMBDA_VM_DEEP_INV_LEGACY=1`). /// /// The injection fires the Nth time the math-cuda entry point is reached, /// across all tables. We assert that the fault really fired, that the @@ -229,8 +232,8 @@ fn gpu_barycentric_fault_recovers_device_only_trace() { } /// R4 DEEP cliff recovery: with every math-cuda DEEP composition dispatch -/// failing (sticky — the fully-resident arm and both mixed arms must all -/// decline in the same prove), R4 falls back to the host DEEP loop, which +/// failing (sticky — the fully-resident arms, fused and buffered, and both +/// mixed arms must all decline in the same prove), R4 falls back to the host DEEP loop, which /// reads the host trace AND the host part evals — both empty under /// device-only. The recovery must download both from the resident handles /// instead of hard-aborting, and the proof must verify. diff --git a/prover/tests/cuda_path_integration.rs b/prover/tests/cuda_path_integration.rs index dd841d7b7..9c63298ed 100644 --- a/prover/tests/cuda_path_integration.rs +++ b/prover/tests/cuda_path_integration.rs @@ -14,9 +14,9 @@ use lambda_vm_prover::test_utils::asm_elf_bytes; use lambda_vm_prover::{prove, verify}; use stark::gpu_lde::{ gpu_bary_calls, gpu_batch_invert_calls, gpu_comp_poly_tree_calls, gpu_composition_calls, - gpu_deep_calls, gpu_device_only_calls, gpu_extend_halves_calls, gpu_fri_calls, gpu_grind_calls, - gpu_lde_calls, gpu_logup_calls, gpu_opening_gather_calls, gpu_parts_lde_calls, - reset_all_gpu_call_counters, + gpu_deep_calls, gpu_device_only_calls, gpu_extend_halves_calls, gpu_fri_calls, + gpu_fused_deep_calls, gpu_grind_calls, gpu_lde_calls, gpu_logup_calls, + gpu_opening_gather_calls, gpu_parts_lde_calls, reset_all_gpu_call_counters, }; /// The R2 GPU composition-poly path (fused `H = z·Σβᵢ·Cᵢ + boundary`) fires and @@ -101,25 +101,58 @@ fn gpu_path_fires_end_to_end() { // DEEP fires once per table that took the R1 GPU path. assert!(gpu_deep_calls() > 0, "R4 GPU DEEP composition did not fire"); + // The resident DEEP's default kernel inverts each row's denominators + // itself (`math_cuda::deep_inv`); `LAMBDA_VM_DEEP_INV_LEGACY=1` keeps the + // buffered kernel and its inverse buffer. A table that silently fell back + // to the buffered kernel would still verify, so this counter is what + // guards the fused dispatch, under either setting. + if math_cuda::deep_inv::rowwise_enabled() { + assert!( + gpu_fused_deep_calls() > 0, + "the fused R4 DEEP kernel did not fire at the default" + ); + } else { + assert_eq!( + gpu_fused_deep_calls(), + 0, + "the fused R4 DEEP kernel fired under LAMBDA_VM_DEEP_INV_LEGACY=1" + ); + } + // FRI commit fires once per table (commit_phase_from_evaluations). assert!(gpu_fri_calls() > 0, "R4 GPU FRI commit did not fire"); - // GPU batch-invert dispatch fires for the R3 OOD and R4 DEEP - // inv_denoms pipelines. A regression where either silently fell back - // to host inv_denoms would drop this to zero. + // GPU batch-invert dispatch fires for the R3 OOD inv_denoms, and for R4's + // on its buffered DEEP path (under `LAMBDA_VM_DEEP_INV_LEGACY=1`, or for a + // table the fused kernel declines). A regression where R3 silently fell + // back to host inv_denoms would drop this to zero. assert!( gpu_batch_invert_calls() > 0, - "GPU batch-invert dispatch did not fire on R3 + R4" + "GPU batch-invert dispatch did not fire on R3" ); // R4 proof-of-work grind: with_blowup(2) grinds at factor 20 (above the - // GPU min-factor gate), so the device search fires for every table and a - // valid nonce is served. A silent CPU fallback (or an invalid kernel result - // rejected by the host check) would drop this to zero. - assert!( - gpu_grind_calls() > 0, - "R4 GPU proof-of-work grind did not fire" - ); + // GPU min-factor gate). The device search implements the keccak digest + // only, and the grinding seed is the transcript's — so under the BLAKE3 + // default configuration the grind runs on host by design and the counter + // stays zero, while a keccak configuration must take the device path. + match stark::config::COMMITMENT_HASH { + stark::config::CommitmentHash::Keccak256 => assert!( + gpu_grind_calls() > 0, + "R4 GPU proof-of-work grind did not fire" + ), + // Every non-keccak digest takes another arm than the keccak kernel + // (grinding.rs routes on the concrete digest), so this counter must + // stay at zero for all of them. + stark::config::CommitmentHash::Blake3 + | stark::config::CommitmentHash::Rpx256 + | stark::config::CommitmentHash::Poseidon1 => assert_eq!( + gpu_grind_calls(), + 0, + "the device grind implements the keccak digest only; a nonzero \ + counter under a non-keccak digest means it ran on the wrong digest" + ), + } // Counters only prove the dispatches ran; this checks the GPU proof // actually satisfies the verifier. diff --git a/prover/tests/d0_king_gate.rs b/prover/tests/d0_king_gate.rs new file mode 100644 index 000000000..fa19fb8d3 --- /dev/null +++ b/prover/tests/d0_king_gate.rs @@ -0,0 +1,119 @@ +//! Cross-version proof oracle for the D0 commitment-hash migration. +//! +//! A prove/verify round trip inside one build cannot see a self-consistent +//! drift: a version that changes how it commits still accepts its own proofs. +//! This exchanges proof *bytes* across versions instead — generate at the ref +//! before a change, verify at the ref after — so a moved leaf layout, transcript +//! or wire format fails loudly. It is the LFM-side counterpart of +//! `scripts/cross_verify_vm.sh`, which does the same for RV64 ELF proofs in both +//! directions. +//! +//! `#[ignore]`d because it is an oracle, not a regression test: it needs two +//! builds, an out-of-tree byte store, and an operator deciding which two refs +//! are being compared. +//! +//! ```text +//! # at the OLD ref +//! KING_GATE=generate KING_GATE_DIR=/some/dir \ +//! cargo test --release -p lambda-vm-prover --test d0_king_gate -- --ignored --nocapture +//! # at the NEW ref, same directory +//! KING_GATE=verify KING_GATE_DIR=/some/dir \ +//! cargo test --release -p lambda-vm-prover --test d0_king_gate -- --ignored --nocapture +//! ``` +//! +//! This is the gate for D0 steps 3-7 (`thoughts/shared/block-compression/`): +//! the Blake3 leaf/pair backends, the B1 transcript, the `lfm_prove` wiring and +//! the registry rows all have to keep Test-hasher proofs verifying, and this is +//! what says they do. Steps that deliberately move the format re-generate the +//! bytes and say so. +//! +//! Two things worth keeping true of this file. It must compile *unchanged* +//! across the refs being compared — that is itself the API-stability half of the +//! test, and editing it to make it build defeats the purpose. And it must be +//! able to fail: flipping one byte of the stored archive has to make `verify` +//! reject. + +use lambda_vm_prover::lfm::programs::trivial_program; +use lambda_vm_prover::lfm::proof::LfmStarkProof; +use lambda_vm_prover::lfm::registry::{LfmProgramKind, build_artifacts}; +use lambda_vm_prover::lfm::word::LfmWord; +use lambda_vm_prover::lfm::{lfm_prove, lfm_verify}; +use lambda_vm_prover::tables::types::FE; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +fn dir() -> std::path::PathBuf { + std::path::PathBuf::from(std::env::var("KING_GATE_DIR").expect("KING_GATE_DIR")) +} + +#[test] +#[ignore = "cross-version oracle: needs KING_GATE=generate|verify and KING_GATE_DIR"] +fn lfm_trivial_v0_cross_version() { + let mode = std::env::var("KING_GATE").expect( + "set KING_GATE=generate (at the old ref) or KING_GATE=verify (at the new one), \ + plus KING_GATE_DIR pointing at a directory that outlives both builds", + ); + let opts = options(); + let proof_path = dir().join("lfm_trivial_v0.rkyv"); + let words_path = dir().join("lfm_trivial_v0.words.rkyv"); + + match mode.as_str() { + "generate" => { + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + assert!( + lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts + ) + .expect("registered"), + "the freshly built proof must verify where it was built" + ); + let bytes = rkyv::to_bytes::(&proved.proof).expect("archive"); + let words = + rkyv::to_bytes::(&proved.public_words).expect("archive words"); + std::fs::write(&proof_path, &bytes).expect("write proof"); + std::fs::write(&words_path, &words).expect("write words"); + eprintln!( + "GENERATED {} ({} bytes) + {} ({} bytes)", + proof_path.display(), + bytes.len(), + words_path.display(), + words.len() + ); + } + "verify" => { + let bytes = std::fs::read(&proof_path).expect("read proof"); + let words = std::fs::read(&words_path).expect("read words"); + let proof = rkyv::from_bytes::(&bytes) + .expect("the archive from the other ref must still deserialize"); + let public_words = rkyv::from_bytes::, rkyv::rancor::Error>(&words) + .expect("words deserialize"); + assert!( + lfm_verify(LfmProgramKind::TrivialV0, &proof, &public_words, &opts) + .expect("registered"), + "proof bytes from the other ref must verify under this build" + ); + eprintln!( + "VERIFIED {} ({} bytes) under this build", + proof_path.display(), + bytes.len() + ); + } + other => panic!("KING_GATE must be generate|verify, got {other:?}"), + } +} diff --git a/prover/tests/jemalloc_conf.rs b/prover/tests/jemalloc_conf.rs index b8661ce87..8f25a45bc 100644 --- a/prover/tests/jemalloc_conf.rs +++ b/prover/tests/jemalloc_conf.rs @@ -1,8 +1,8 @@ //! jemalloc's purge policy is compiled into the binary — checked by reading it //! back out of the allocator serving this process. //! -//! `bin/cli/src/main.rs` and `calibration.rs` each export -//! `_rjem_malloc_conf = "dirty_decay_ms:-1,muzzy_decay_ms:-1"` beside their +//! `bin/cli/src/main.rs` exports +//! `_rjem_malloc_conf = "dirty_decay_ms:-1,muzzy_decay_ms:-1"` beside its //! `#[global_allocator]`, so the never-purge policy travels in the binary //! rather than in a launcher's environment. Nothing about that export is //! checked by the compiler: a misspelled symbol, a wrong value type, or a @@ -11,14 +11,13 @@ //! //! This is its own test binary because the check needs a jemalloc process of //! its own: the prover's lib tests run under the platform allocator, where a -//! `mallctl` read would say nothing, and `calibration.rs` is behind -//! `disk-spill` and pays for a full proof. What it pins is the export pattern — +//! `mallctl` read would say nothing. What it pins is the export pattern — //! symbol, type, initializer, edition spelling — in the copy below, which is -//! byte-identical to the two production sites but not mechanically tied to -//! them: delete either of those and this still passes. It is a self-test of the -//! pattern, not a regression guard on the two sites that ship it. That the -//! shipped `cli` binary carries the symbol is a link-time property, read with -//! `nm` rather than asserted here. +//! byte-identical to the production site but not mechanically tied to it: +//! delete that one and this still passes. It is a self-test of the pattern, +//! not a regression guard on the site that ships it. That the shipped `cli` +//! binary carries the symbol is a link-time property, read with `nm` rather +//! than asserted here. use tikv_jemalloc_ctl::raw; diff --git a/prover/tests/merkle_cap_vm.rs b/prover/tests/merkle_cap_vm.rs new file mode 100644 index 000000000..b9cbee11e --- /dev/null +++ b/prover/tests/merkle_cap_vm.rs @@ -0,0 +1,121 @@ +//! A real VM proof under the Merkle cap policy the PROCESS FORMAT names +//! (`LAMBDA_VM_ZF_CAP`): every production table — the +//! preprocessed ones (precomputed + main trees), the LogUp aux trees, the +//! composition trees and every committed FRI layer — capped, proved and +//! verified through the public `prove_with_options_and_inputs` / +//! `verify_with_options` entry points. +//! +//! Knob-on only, hence `#[ignore]`: at the default format it would prove +//! nothing new, so it refuses to run unless `LAMBDA_VM_ZF_CAP` selects a cap. +//! +//! ```text +//! LAMBDA_VM_ZF_CAP=auto cargo test --release -p lambda-vm-prover --test merkle_cap_vm -- --ignored --nocapture +//! LAMBDA_VM_ZF_CAP=auto cargo test --release -p lambda-vm-prover --features cuda --test merkle_cap_vm -- --ignored --nocapture --test-threads=1 +//! ``` +//! +//! Under `cuda` the fixture is big enough that its tables commit on the device, +//! and the caps must come off the resident trees (`gpu_cap_read_calls`). + +use lambda_vm_prover::test_utils::asm_elf_bytes; +use lambda_vm_prover::zf_format::ZfFormat; +use lambda_vm_prover::{ + GoldilocksCubicProofOptions, MaxRowsConfig, prove_with_options_and_inputs, verify_with_options, +}; +use stark::merkle_caps::StarkCaps; + +/// CPU: a fixture that touches every instruction class (many tables). Device: +/// the fixture the cuda integration tests use, whose tables cross the GPU LDE +/// threshold. +#[cfg(not(feature = "cuda"))] +const FIXTURE: &str = "all_instructions_64"; +#[cfg(feature = "cuda")] +const FIXTURE: &str = "fib_iterative_1M"; + +#[test] +#[ignore = "knob-on: run with LAMBDA_VM_ZF_CAP=auto (or a height) and -- --ignored"] +fn a_vm_proof_round_trips_under_the_process_cap_policy() { + let format = ZfFormat::from_env().expect("a valid ZF format"); + assert!( + !format.cap.is_off(), + "LAMBDA_VM_ZF_CAP is unset or off: this test only means something with a cap" + ); + println!("{}", format.banner()); + let base = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup 2"); + let capped = format.options(base.clone()); + let mut default = base; + default.format = Default::default(); + assert!(default.has_default_format()); + + let elf = asm_elf_bytes(FIXTURE); + #[cfg(feature = "cuda")] + let before = stark::gpu_lde::gpu_cap_read_calls(); + let proof = prove_with_options_and_inputs(&elf, &[], &capped, &MaxRowsConfig::default()) + .expect("prove under the cap policy"); + #[cfg(feature = "cuda")] + { + let reads = stark::gpu_lde::gpu_cap_read_calls() - before; + println!("CAPVM device cap reads: {reads}"); + assert!( + reads > 0, + "no cap came off the device: {FIXTURE} proved on host trees" + ); + } + + assert!( + verify_with_options(&proof, &elf, &capped, None, None).expect("verify"), + "a capped VM proof must verify under its own policy" + ); + // Non-vacuity: the policy engaged. Every table's main-tree paths have the + // lengths the verifier's `StarkCaps` gives its shape, and at least one + // table is really capped — so the default verifier below meets paths that + // differ from the ones it expects, and its refusal means something. + let mut capped_tables = 0usize; + for (t, p) in proof.proof.proofs.iter().enumerate() { + let lde_log = (p.trace_length * usize::from(capped.blowup_factor)).trailing_zeros(); + let caps = StarkCaps::new( + format.cap, + capped.fri_number_of_queries, + lde_log as usize, + p.fri_layers_merkle_roots.len(), + ); + let path = |q: usize| { + p.deep_poly_openings[q] + .main_trace_polys + .proof + .merkle_path + .len() + }; + assert_eq!( + path(1), + caps.trace_depth - caps.trace, + "table {t}: a non-owner main path is not cut to the cap" + ); + if caps.trace > 0 { + assert_eq!( + path(0), + caps.trace_depth - caps.trace + (1 << caps.trace), + "table {t}: the owner path does not carry the cap" + ); + capped_tables += 1; + } + } + println!( + "CAPVM capped tables: {capped_tables} of {}", + proof.proof.proofs.len() + ); + assert!( + capped_tables > 0, + "no table was capped: the policy did not engage and the cross-format check below is vacuous" + ); + // The cap height is a verifier constant: the default verifier must refuse + // the capped proof (full-length paths expected), without panicking. This + // held only once the AIR prototype cache keyed the proof format: before, + // the prove above cached capped AIRs and `&default` got them back. + assert!( + !matches!( + verify_with_options(&proof, &elf, &default, None, None), + Ok(true) + ), + "a capped proof accepted by the default-format verifier" + ); +} diff --git a/prover/tests/onechain_release_stream.rs b/prover/tests/onechain_release_stream.rs new file mode 100644 index 000000000..f1d99857e --- /dev/null +++ b/prover/tests/onechain_release_stream.rs @@ -0,0 +1,155 @@ +//! The grouped WHIR leg (`GroupFold::Indexed`) a RELEASE build of the library +//! emits — no `cfg(test)`, so no emitter faults and no step marks — held to +//! the pins the library's own test build is held to +//! (`lfm::whir_onechain_tests::the_grouped_leg_stream_of_a_test_build`): the +//! program the unit tests gate is the program a release build emits. +//! +//! ```text +//! cargo test --release -p lambda-vm-prover --test onechain_release_stream -- --nocapture +//! ``` + +use lambda_vm_prover::lfm::builder::{Cell, Ext, LfmBuilder}; +use lambda_vm_prover::lfm::compiler::compile; +use lambda_vm_prover::lfm::edsl::WrapHash; +use lambda_vm_prover::lfm::whir_chain::{ChainShape, RoundStorage}; +use lambda_vm_prover::lfm::whir_stacked::{StackedChainWires, emit_stacked_verify_chains}; +use lambda_vm_prover::lfm::whir_transcript::WhirTranscript; +use lambda_vm_prover::tables::types::GoldilocksField; + +/// The grouped leg's instruction stream, emitted over hinted wires alone — it +/// is a function of the layout and the config, never of a proof — as +/// `(instructions, FNV-1a of their Debug text)`. ★ The SAME body sits in +/// `prover/src/lfm/whir_onechain_tests.rs`, built WITH `cfg(test)` (the +/// emitter faults and the step marks), and both builds are held to the same +/// pins: the test build emits exactly the release build's program. +fn grouped_leg_stream(p1: bool, heights: &[usize], n_stack: usize) -> (usize, u64) { + use multilinear::whir_chain::{ + CapPolicy, ChainConfig, ChainFormat, FirstFold, GrindBits, GroupFold, WhirFolds, + }; + let mut format = ChainFormat { + folds: WhirFolds::First(FirstFold::new(6).expect("first6")), + group: GroupFold::Indexed, + ..ChainFormat::DEFAULT + }; + if p1 { + format.arity4_cap = CapPolicy::Fixed(2); + } else { + format.cap = CapPolicy::Fixed(1); + } + let cfg = ChainConfig { + log_blowup: 2, + log_folding: 4, + num_queries: 3, + grind: GrindBits::uniform(8), + format, + }; + let layout = multilinear::stacking::StackedLayout::build(heights, n_stack).expect("packs"); + let shape = ChainShape::new_at(&cfg, n_stack, if p1 { 4 } else { 2 }); + let domain = multilinear::whir::Domain::::new(n_stack + cfg.log_blowup) + .expect("domain"); + let hash = if p1 { + WrapHash::Poseidon1 + } else { + WrapHash::production() + }; + let runs = shape.chains(layout.num_polys()); + let mut total: u32 = heights.iter().sum::() as u32 + heights.len() as u32; + for run in &runs { + total += run.len() as u32 + 1 + RoundStorage::words(&shape.run(run.len())); + } + let mut b = LfmBuilder::new().with_wrap_hash(hash); + let arena = b.declare_arena(total); + let mut transcript = WhirTranscript::for_builder(&mut b); + let mut at = 0u32; + let next = |b: &mut LfmBuilder, at: &mut u32| { + let cell = b.hint_word(arena, *at); + *at += 1; + cell + }; + let points: Vec> = heights + .iter() + .map(|&h| (0..h).map(|_| next(&mut b, &mut at).as_ext()).collect()) + .collect(); + let values: Vec = heights + .iter() + .map(|_| next(&mut b, &mut at).as_ext()) + .collect(); + let mut roots: Vec> = Vec::new(); + let mut finals: Vec = Vec::new(); + let mut storage: Vec = Vec::new(); + for run in &runs { + roots.push(run.clone().map(|_| next(&mut b, &mut at)).collect()); + finals.push(next(&mut b, &mut at).as_ext()); + let run_shape = shape.run(run.len()); + let base = at; + storage.push(RoundStorage::hint(&mut b, arena, base, &run_shape)); + at += RoundStorage::words(&run_shape); + } + let openings: Vec<_> = storage.iter().map(RoundStorage::openings).collect(); + let rounds: Vec> = storage + .iter() + .zip(&openings) + .map(|(chain, (current, next))| chain.wires(current, next)) + .collect(); + let chains: Vec> = (0..runs.len()) + .map(|i| StackedChainWires { + rounds: &rounds[i], + roots: &roots[i], + final_value: finals[i], + }) + .collect(); + let claimed_at: Vec<&[Ext]> = points.iter().map(Vec::as_slice).collect(); + let gamma = emit_stacked_verify_chains( + &mut b, + &mut transcript, + &layout, + &chains, + &claimed_at, + &values, + None, + &shape, + &domain, + ); + b.public(gamma.as_cell()); + let program = compile(b.finish()); + let text = format!("{:?}", program.instrs); + let digest = text.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, byte| { + (h ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3) + }); + (program.instrs.len(), digest) +} + +/// One pinned case: the base (`true` for Poseidon1), the columns' heights, the +/// stack's height, and the stream's `(instructions, FNV-1a)`. +type StreamPin = (bool, &'static [usize], usize, (usize, u64)); + +/// The cases both builds are held to, each with its stream's pin: RPX and +/// Poseidon1, five polynomials split [4, 1] and six split [4, 2] whose short +/// columns' selectors run past `k0`. A pin moves only with the emitted +/// program; move it in both files together. +const GROUPED_LEG_STREAMS: [StreamPin; 4] = [ + (false, &[8; 5], 8, (5241, 0x8d95_4e39_3cca_f387)), + (false, &[8, 3, 8, 2, 4, 8], 8, (5229, 0xac5d_a244_80ae_4271)), + (true, &[10; 5], 10, (28895, 0x52ec_ae97_ae80_6593)), + ( + true, + &[10, 3, 10, 2, 4, 10], + 10, + (28868, 0x02e6_e3fd_5c51_0e98), + ), +]; + +#[test] +fn the_grouped_leg_stream_of_a_release_build() { + for (p1, heights, n, pin) in GROUPED_LEG_STREAMS { + let stream = grouped_leg_stream(p1, heights, n); + println!( + "ONECHAIN STREAM p1={p1} heights={heights:?}: {} instructions, digest {:016x}", + stream.0, stream.1 + ); + assert_eq!( + stream, pin, + "p1={p1} heights={heights:?}: the pinned stream" + ); + } +} diff --git a/prover/tests/p1_grind_dispatch.rs b/prover/tests/p1_grind_dispatch.rs new file mode 100644 index 000000000..f9752657e --- /dev/null +++ b/prover/tests/p1_grind_dispatch.rs @@ -0,0 +1,56 @@ +//! ★ The Poseidon1 grind's dispatch gate: `generate_nonce_maybe_gpu` over +//! `P1GrindDigest` reaches the width-8 device kernel (`p1s_grind_w8`), and +//! only that one. The device search's own parity is +//! `crypto/math-cuda/tests/p1_device_parity.rs`; this file needs the prover's +//! `cuda` feature because the counters are `crypto`'s, behind `crypto/cuda`. +//! +//! Every arm returns a valid nonce whatever happens (the host search is +//! correct), so validity alone would pass with the device never touched. The +//! counters count only device searches whose nonce passed the host check, so +//! the deltas are the claim. One test, because the counters are process-global. +//! Needs a GPU: +//! +//! cargo test -p lambda-vm-prover --release --features cuda --test p1_grind_dispatch +#![cfg(feature = "cuda")] + +use crypto::fiat_shamir::p1_transcript::P1GrindDigest; +use crypto::grinding::{DeviceGrindKey, GrindDigest}; + +#[test] +fn the_p1_grind_dispatch_reaches_its_own_kernel() { + assert_eq!( + ::DEVICE_GRIND, + Some(DeviceGrindKey::Poseidon1), + "the P1 digest must declare the P1 device arm" + ); + let seed = [9u8; 32]; + let factor = 18u8; + let (p1, rpx, keccak) = ( + crypto::grinding::gpu_grind_calls_p1(), + crypto::grinding::gpu_grind_calls_rpx(), + crypto::grinding::gpu_grind_calls(), + ); + let nonce = crypto::grinding::generate_nonce_maybe_gpu::(&seed, factor) + .expect("a nonce"); + assert!( + crypto::grinding::is_valid_nonce::(&seed, nonce, factor), + "the dispatched nonce {nonce} fails is_valid_nonce" + ); + assert_eq!( + crypto::grinding::generate_nonce_smallest::(&seed, factor), + Some(nonce), + "the dispatched nonce is the smallest valid one" + ); + assert_eq!( + crypto::grinding::gpu_grind_calls_p1(), + p1 + 1, + "the P1 arm must have run on the device (LAMBDA_VM_NO_GPU_GRIND makes this fail, which \ + is the point)" + ); + assert_eq!(crypto::grinding::gpu_grind_calls_rpx(), rpx, "no RPX grind"); + assert_eq!( + crypto::grinding::gpu_grind_calls(), + keccak, + "no keccak grind" + ); +} diff --git a/prover/tests/rpx_device_parity.rs b/prover/tests/rpx_device_parity.rs new file mode 100644 index 000000000..441ff9bd5 --- /dev/null +++ b/prover/tests/rpx_device_parity.rs @@ -0,0 +1,580 @@ +//! The RPX device kernels must produce the host's bytes — the phase-2 gate for +//! lane K, the twin of `crypto/math-cuda/tests/blake3_fused_parity.rs` (Batch +//! backend through the fused LDE+commit pipelines) plus the FRI-layer tree +//! (Pair backend), the comp-poly tree and the bare permutation. +//! +//! Lives in the prover crate rather than `math-cuda` because the host side — +//! `RpxStarkHash`, `AlgebraicBatchBackend`, `AlgebraicPairBackend`, `Rpx256` — +//! lives here; `math-cuda` is a dev-dependency of this crate, not the reverse. +//! +//! Every comparison is byte for byte or lane for lane against the production +//! host path; a tamper arm proves the equalities are not vacuous. Needs a GPU: +//! +//! cargo test -p lambda-vm-prover --release --features cuda --test rpx_device_parity -- --nocapture +#![cfg(feature = "cuda")] + +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use lambda_vm_prover::lfm::algebraic_commit::{ + AlgebraicBatchBackend, AlgebraicPairBackend, RpxCommit, RpxStarkHash, +}; +use lambda_vm_prover::lfm::hash::LfmHasher; +use lambda_vm_prover::lfm::rpx::Rpx256; +use lambda_vm_prover::tables::types::FE; +use math::fft::two_half_fft::TwoHalfTwiddles; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::polynomial::Polynomial; +use stark::prover::{GenericProver, IsStarkProver}; + +/// The RPX prover, named: these tests compare against the CUDA RPX kernels, so +/// the CPU side must say RPX explicitly. +type Prover = GenericProver; + +type Ext3 = Degree3GoldilocksExtensionField; +type Fp3 = FieldElement; +type Fp = FieldElement; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// splitmix64 — deterministic inputs from a seed, with no `rand` dependency +/// (the prover crate carries none for tests). +struct SplitMix(u64); + +impl SplitMix { + fn new(seed: u64) -> Self { + Self(seed) + } + + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} +const COSET_OFFSET: u64 = 7; + +fn coset_weights(n: usize, g: u64) -> Vec { + let inv_n = Fp::from(n as u64).inv().unwrap(); + let g_fp = Fp::from_raw(g); + let mut w = Vec::with_capacity(n); + let mut cur = inv_n; + for _ in 0..n { + w.push(cur); + cur = &cur * &g_fp; + } + w +} + +fn coset_weights_u64(n: usize, g: u64) -> Vec { + coset_weights(n, g).iter().map(|w| *w.value()).collect() +} + +fn canonical(v: u64) -> u64 { + if v >= P { v - P } else { v } +} + +// =========================================================================== +// The bare permutation: device vs the host oracle, lane for lane. +// =========================================================================== + +/// Random states plus the two edge states the host-KAT names; raw (`≥ p`) +/// lanes included, since that is what an LDE buffer holds. +fn probe_states(seed: u64, n: usize) -> Vec<[u64; 12]> { + let mut rng = SplitMix::new(seed); + let mut states = vec![[0u64; 12], [P - 1; 12]]; + for k in 0..n { + let mut s = [0u64; 12]; + for (i, lane) in s.iter_mut().enumerate() { + // Every fifth lane is a raw twin `c + p` of a small canonical `c`. + *lane = if (k + i) % 5 == 0 { + rng.next() % 0xFFFF_FFFF + P + } else { + rng.next() % P + }; + } + states.push(s); + } + states +} + +#[test] +fn rpx_device_permutation_matches_the_host_oracle() { + let states = probe_states(0x0052_5058, 256); + let got = math_cuda::rpx::permute_probe(&states).expect("device permute probe"); + assert_eq!(got.len(), states.len()); + for (n, (input, out)) in states.iter().zip(got.iter()).enumerate() { + let want = Rpx256.permute(core::array::from_fn(|i| FE::from(input[i]))); + for (i, (o, w)) in out.iter().zip(want.iter()).enumerate() { + let w = canonical(*w.value()); + // RAW comparison: the device canonicalises its output, and that + // loop is part of what is pinned (R-952). + assert_eq!(*o, w, "state {n} lane {i}: device {o} vs host {w}"); + } + } +} + +// =========================================================================== +// The Batch backend through the fused row-major LDE + commit pipelines. +// =========================================================================== + +fn cpu_row_major_rpx_root( + columns: &[Vec], + blowup: usize, + weights: &[Fp], + inv_tw: &TwoHalfTwiddles, + fwd_tw: &TwoHalfTwiddles, +) -> [u8; 32] { + let n = columns[0].len(); + let num_cols = columns.len(); + let mut buf: Vec = vec![Fp::from(0u64); n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, &v) in col.iter().enumerate() { + buf[r * num_cols + c] = Fp::from_raw(v); + } + } + Polynomial::::coset_lde_full_expand_row_major::( + &mut buf, num_cols, blowup, weights, inv_tw, fwd_tw, + ) + .expect("CPU row-major LDE"); + let (_, root) = + Prover::::commit_rows_bit_reversed(&buf, num_cols) + .expect("CPU RPX commit"); + root +} + +/// Device fused row-major LDE + RPX leaves + Merkle, root only. +fn gpu_fused_rpx_root(columns: &[Vec], blowup: usize, weights_u64: &[u64]) -> [u8; 32] { + let n = columns[0].len(); + let num_cols = columns.len(); + let mut row_major = vec![0u64; n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, &v) in col.iter().enumerate() { + row_major[r * num_cols + c] = v; + } + } + let (handle, _lde) = math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep( + &row_major, + None, + math_cuda::DeviceHash::Rpx256, + n, + num_cols, + blowup, + weights_u64, + true, + ) + .expect("fused RPX GPU pipeline"); + handle.tree.as_ref().expect("resident merkle tree").root +} + +#[test] +fn rpx_fused_base_root_matches_cpu() { + for log_n in [4usize, 6, 8, 10] { + for blowup in [2usize, 4] { + for num_cols in [1usize, 3, 8, 9] { + let n = 1usize << log_n; + let log_lde = (n * blowup).trailing_zeros() as usize; + let mut rng = SplitMix::new((log_n * 1000 + blowup * 100 + num_cols) as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rng.next()).collect()) + .collect(); + let weights_u64 = coset_weights_u64(n, COSET_OFFSET); + let weights_fp = coset_weights(n, COSET_OFFSET); + let inv_tw = + TwoHalfTwiddles::::new(log_n, true).expect("inv twiddles"); + let fwd_tw = + TwoHalfTwiddles::::new(log_lde, false).expect("fwd twiddles"); + + let gpu_root = gpu_fused_rpx_root(&columns, blowup, &weights_u64); + let cpu_root = + cpu_row_major_rpx_root(&columns, blowup, &weights_fp, &inv_tw, &fwd_tw); + assert_eq!( + gpu_root, cpu_root, + "RPX fused root mismatch: log_n={log_n} blowup={blowup} num_cols={num_cols}" + ); + } + } + } +} + +fn rand_ext3(rng: &mut SplitMix) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.next()), + Fp::from_raw(rng.next()), + Fp::from_raw(rng.next()), + ]) +} + +fn cpu_ext3_row_major_rpx_root( + columns: &[Vec], + blowup: usize, + weights: &[Fp], + inv_tw: &TwoHalfTwiddles, + fwd_tw: &TwoHalfTwiddles, +) -> [u8; 32] { + let n = columns[0].len(); + let num_cols = columns.len(); + let mut buf: Vec = vec![Fp3::from(0u64); n * num_cols]; + for (c, col) in columns.iter().enumerate() { + for (r, v) in col.iter().enumerate() { + buf[r * num_cols + c] = *v; + } + } + Polynomial::::coset_lde_full_expand_row_major::( + &mut buf, num_cols, blowup, weights, inv_tw, fwd_tw, + ) + .expect("CPU ext3 row-major LDE"); + let (_, root) = Prover::::commit_rows_bit_reversed(&buf, num_cols) + .expect("CPU ext3 RPX commit"); + root +} + +#[test] +fn rpx_fused_ext3_root_matches_cpu() { + for log_n in [4usize, 6, 8] { + for blowup in [2usize, 4] { + for num_cols in [1usize, 3, 5] { + let n = 1usize << log_n; + let log_lde = (n * blowup).trailing_zeros() as usize; + let mut rng = SplitMix::new((log_n * 1000 + blowup * 100 + num_cols) as u64 + 4242); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + // Row-major ext3 = row-major base with 3 * num_cols lanes. + let mut row_major = vec![0u64; n * num_cols * 3]; + for (c, col) in columns.iter().enumerate() { + for (r, v) in col.iter().enumerate() { + for k in 0..3 { + row_major[(r * num_cols + c) * 3 + k] = *v.value()[k].value(); + } + } + } + let weights_u64 = coset_weights_u64(n, COSET_OFFSET); + let weights_fp = coset_weights(n, COSET_OFFSET); + let inv_tw = + TwoHalfTwiddles::::new(log_n, true).expect("inv twiddles"); + let fwd_tw = + TwoHalfTwiddles::::new(log_lde, false).expect("fwd twiddles"); + + let (handle, _lde) = + math_cuda::lde::coset_lde_ext3_row_major_with_merkle_tree_keep( + &row_major, + math_cuda::DeviceHash::Rpx256, + n, + num_cols, + blowup, + &weights_u64, + true, + ) + .expect("fused ext3 RPX GPU pipeline"); + let gpu_root = handle.tree.as_ref().expect("resident merkle tree").root; + let cpu_root = + cpu_ext3_row_major_rpx_root(&columns, blowup, &weights_fp, &inv_tw, &fwd_tw); + assert_eq!( + gpu_root, cpu_root, + "RPX fused ext3 root mismatch: log_n={log_n} blowup={blowup} num_cols={num_cols}" + ); + } + } + } +} + +// =========================================================================== +// The comp-poly tree from interleaved ext3 parts (the `gpu_lde` site), against +// the same row-pair leaf layout committed on the CPU. +// =========================================================================== + +#[test] +fn rpx_comp_poly_tree_root_matches_cpu() { + for (log_lde, m) in [(4usize, 1usize), (6, 2), (10, 3), (12, 4)] { + let lde_size = 1usize << log_lde; + let mut rng = SplitMix::new((log_lde * 10 + m) as u64 + 99); + let parts: Vec> = (0..m) + .map(|_| (0..lde_size).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + // Interleaved `[a0,a1,a2,b0,b1,b2,…]` per part for the device. + let parts_u64: Vec> = parts + .iter() + .map(|p| { + p.iter() + .flat_map(|e| e.value().iter().map(|c| *c.value())) + .collect() + }) + .collect(); + let raw_parts: Vec<&[u64]> = parts_u64.iter().map(|p| p.as_slice()).collect(); + let dev_tree = math_cuda::rpx::build_comp_poly_tree_from_evals_ext3_keep(&raw_parts) + .expect("device comp-poly tree"); + assert_eq!(dev_tree.leaves_len, lde_size / 2); + + // Row-major with `m` ext3 columns: row r = [part_0[r], …, part_{m-1}[r]]. + let mut buf: Vec = vec![Fp3::from(0u64); lde_size * m]; + for (c, p) in parts.iter().enumerate() { + for (r, v) in p.iter().enumerate() { + buf[r * m + c] = *v; + } + } + let (_, cpu_root) = Prover::::commit_rows_bit_reversed(&buf, m) + .expect("CPU comp-poly commit"); + assert_eq!( + dev_tree.root, cpu_root, + "RPX comp-poly root mismatch: log_lde={log_lde} m={m}" + ); + } +} + +// =========================================================================== +// The Pair backend: the FRI-layer tree, node for node. +// =========================================================================== + +fn fri_layer_parity(log_num_leaves: u32, seed: u64) { + let num_leaves = 1usize << log_num_leaves; + let mut rng = SplitMix::new(seed); + let evals: Vec = (0..num_leaves * 2).map(|_| rand_ext3(&mut rng)).collect(); + + let mut evals_u64 = Vec::with_capacity(evals.len() * 3); + for e in &evals { + for c in e.value().iter() { + evals_u64.push(*c.value()); + } + } + let leaves: Vec<[Fp3; 2]> = evals.chunks_exact(2).map(|c| [c[0], c[1]]).collect(); + let cpu_tree = MerkleTree::>::build(&leaves).unwrap(); + let cpu_nodes = cpu_tree.nodes(); + + let gpu_bytes = math_cuda::rpx::build_fri_layer_tree_from_evals_ext3(&evals_u64).unwrap(); + assert_eq!(cpu_nodes.len() * 32, gpu_bytes.len(), "node count"); + for (i, expected) in cpu_nodes.iter().enumerate() { + assert_eq!( + &gpu_bytes[i * 32..(i + 1) * 32], + &expected[..], + "node {i} mismatch at log_num_leaves={log_num_leaves}" + ); + } +} + +/// Small trees: every level fits the block, so the tail kernel builds the +/// whole tree in one launch. +#[test] +fn rpx_fri_layer_tree_small() { + for log in 1u32..=6 { + fri_layer_parity(log, 100 + log as u64); + } +} + +/// Deep enough that the per-level kernel runs first and hands over to the tail +/// partway up — the launch path a real commit takes. +#[test] +fn rpx_fri_layer_tree_medium() { + for log in [10u32, 12, 14] { + fri_layer_parity(log, 500 + log as u64); + } +} + +// =========================================================================== +// The column-range leaves (preprocessed tables), leaf for leaf against the +// Batch backend's `hash_data` over the same felts. +// =========================================================================== + +#[test] +fn rpx_row_major_range_leaves_match_cpu() { + let log_n = 6u32; + let n = 1usize << log_n; + let m = 7usize; + let mut rng = SplitMix::new(0xBEEF); + let data: Vec = (0..n * m).map(|_| rng.next()).collect(); + let reverse_index = |i: usize| -> usize { (i as u64).reverse_bits() as usize >> (64 - log_n) }; + + for (cs, ce) in [(0usize, m), (0, 3), (3, m), (2, 5)] { + let gpu = math_cuda::rpx::leaves_base_row_major_row_pair_range(&data, m, cs, ce, n) + .expect("device ranged leaves"); + assert_eq!(gpu.len(), (n / 2) * 32); + for leaf in 0..n / 2 { + let mut felts: Vec = Vec::with_capacity(2 * (ce - cs)); + for k in 0..2 { + let br = reverse_index(2 * leaf + k); + for c in cs..ce { + felts.push(Fp::from_raw(data[br * m + c])); + } + } + let want = + as IsMerkleTreeBackend>::hash_data( + &felts, + ); + assert_eq!( + &gpu[leaf * 32..(leaf + 1) * 32], + &want[..], + "ranged leaf {leaf} mismatch for columns [{cs}, {ce})" + ); + } + } +} + +// =========================================================================== +// Negative control: one corrupted input element must move the device root. +// =========================================================================== + +#[test] +fn rpx_fused_tamper_diverges() { + let n = 1usize << 6; + let num_cols = 3usize; + let mut rng = SplitMix::new(777); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rng.next()).collect()) + .collect(); + let weights_u64 = coset_weights_u64(n, COSET_OFFSET); + + let honest = gpu_fused_rpx_root(&columns, 2, &weights_u64); + let mut tampered = columns.clone(); + tampered[1][n / 2] ^= 1; + let forged = gpu_fused_rpx_root(&tampered, 2, &weights_u64); + assert_ne!( + honest, forged, + "a corrupted input element must move the root" + ); +} + +// =========================================================================== +// The proof-of-work grind. +// +// Unlike everything above, the claim here is not "the device reproduces the +// host's bytes" but "the device finds the nonces the host's PREDICATE accepts" +// — a search, whose answer is not unique. So what is asserted is validity +// under `is_valid_nonce`, never a proof byte and never a comparison with the +// CPU search's `find_any`, which does not agree with itself between runs. +// +// The kernel's arithmetic is pinned without a GPU by layer 8 of +// `crypto/math-cuda/tests/host_kat/rpx_host_kat.cpp`. What needs a GPU, and is +// here, is that a real parallel launch reduces to the same answer and that the +// production dispatch actually reaches it. +// =========================================================================== + +/// The digest the RPX configuration grinds over — its transcript's hash. +type RpxGrind = stark::config::GrindingDigest; + +/// A real launch returns a nonce the host predicate accepts, and it is the +/// smallest in the range. +/// +/// Minimality is not a contract — the verifier takes any valid nonce — but it +/// is a cheap probe of search completeness that plain validity cannot make: a +/// stride or bounds defect would still return a *valid* nonce, just not the +/// first one. It is deterministic despite the parallel grid because `atomicMin` +/// is an order-independent reduction, which is the property this test exists to +/// exercise and the host harness cannot. +/// +/// Factor 14 so the exhaustive host scan below the answer stays cheap. +#[test] +fn rpx_gpu_grind_returns_the_smallest_valid_nonce() { + let seed = [14u8; 32]; + let factor = 14u8; + let nonce = math_cuda::grinding::generate_nonce_rpx_gpu( + &stark::grinding::inner_hash_felts::(&seed, factor), + factor, + ) + .expect("GPU RPX grind (needs a GPU)"); + assert!( + stark::grinding::is_valid_nonce::(&seed, nonce, factor), + "GPU nonce {nonce} fails is_valid_nonce (factor {factor})" + ); + assert!( + (0..nonce).all(|n| !stark::grinding::is_valid_nonce::(&seed, n, factor)), + "GPU nonce {nonce} is not the smallest valid nonce (factor {factor})" + ); +} + +/// At the shipped factor the launch returns a valid nonce. Validity only — +/// scanning below it would be ~2^20 host sponges. +#[test] +fn rpx_gpu_grind_valid_at_the_production_factor() { + let seed = [20u8; 32]; + let factor = 20u8; + let nonce = math_cuda::grinding::generate_nonce_rpx_gpu( + &stark::grinding::inner_hash_felts::(&seed, factor), + factor, + ) + .expect("GPU RPX grind (needs a GPU)"); + assert!( + stark::grinding::is_valid_nonce::(&seed, nonce, factor), + "GPU nonce {nonce} fails is_valid_nonce (factor {factor})" + ); +} + +/// ★ THE DISPATCH GATE: `generate_nonce_maybe_gpu` reaches the device under an +/// RPX configuration, and does not under one with no grind kernel. +/// +/// Both arms return a valid nonce whatever happens — the CPU fallback is +/// correct — so validity alone would pass with the device never touched, which +/// is exactly the failure this feature can have. The RPX counter counts only +/// device searches whose nonce passed the host check, so the deltas are the +/// claim; the BLAKE3 arm is the control that makes the RPX one mean something. +/// +/// ⚠ **The control is a different DIGEST, not a different argument.** The arm +/// used to be a runtime `CommitmentHash` handed in beside the digest, so one +/// type could play both parts. It is now +/// [`GrindDigest::DEVICE_GRIND`](stark::grinding::GrindDigest), derived from +/// each configuration's own `COMMITMENT_HASH`, so the only way to exercise the +/// negative arm is to name a configuration that declares no kernel. That makes +/// this test reach the mechanism rather than a parameter nothing reads. +/// +/// One test rather than two because the counters are process-global. Nothing +/// else in this binary grinds. +#[test] +fn the_rpx_dispatch_reaches_the_device_and_blake3_does_not() { + use stark::grinding::{DeviceGrindKey, GrindDigest}; + + type Blake3Grind = stark::config::GrindingDigest; + + // The claim the arms rest on, asserted rather than assumed. + assert_eq!( + ::DEVICE_GRIND, + Some(DeviceGrindKey::Rpx256), + "the RPX configuration must declare the RPX device arm" + ); + assert_eq!( + ::DEVICE_GRIND, + None, + "BLAKE3 has no grind kernel, so it must declare none" + ); + + let seed = [7u8; 32]; + let factor = 20u8; + + let before = stark::gpu_lde::gpu_grind_calls_rpx(); + let nonce = + stark::grinding::generate_nonce_maybe_gpu::(&seed, factor).expect("a nonce"); + assert!( + stark::grinding::is_valid_nonce::(&seed, nonce, factor), + "the dispatched nonce {nonce} fails is_valid_nonce" + ); + assert_eq!( + stark::gpu_lde::gpu_grind_calls_rpx(), + before + 1, + "the RPX arm must have run on the device (set LAMBDA_VM_NO_GPU_GRIND and this fails, \ + which is the point)" + ); + + // CONTROL: a configuration whose digest declares no kernel must not pay a + // device search at all, on either counter. + let before_rpx = stark::gpu_lde::gpu_grind_calls_rpx(); + let before_keccak = stark::gpu_lde::gpu_grind_calls(); + let nonce = + stark::grinding::generate_nonce_maybe_gpu::(&seed, factor).expect("a nonce"); + assert!( + stark::grinding::is_valid_nonce::(&seed, nonce, factor), + "the host-search nonce {nonce} fails is_valid_nonce" + ); + assert_eq!( + stark::gpu_lde::gpu_grind_calls_rpx(), + before_rpx, + "a hash with no grind kernel must not reach the RPX kernel" + ); + assert_eq!( + stark::gpu_lde::gpu_grind_calls(), + before_keccak, + "a hash with no grind kernel must not reach the keccak kernel either" + ); +} diff --git a/prover/tests/rpx_device_paths.rs b/prover/tests/rpx_device_paths.rs new file mode 100644 index 000000000..6bbb39a08 --- /dev/null +++ b/prover/tests/rpx_device_paths.rs @@ -0,0 +1,332 @@ +//! The RPX device paths `math_cuda::rpx_paths` switches between produce the +//! same bytes, and the benches that measure what each one moves. +//! +//! - The Merkle tops: the half-warp permutation and the walk that uses it for +//! narrow levels and the tail (`math_cuda::rpx::TreeWalk::WARP`, the +//! default), against a thread per parent (`TreeWalk::ThreadPerParent`). +//! - The grind: the work-queue kernel on a card-filling grid (the default), +//! against the stride kernel. +//! - The permutation: the limb multiply and every variant the source +//! instantiates, against the 64-bit multiply and the host. +//! +//! Every parity test runs both paths in one process through explicit entry +//! points, so none of them depends on the `LAMBDA_VM_RPX_*` switches. The +//! comparisons are byte for byte (nodes, nonces) or raw word for word +//! (permutation states), path against path and, where it is cheap, against the +//! host oracle. +//! +//! ```text +//! cargo test -p lambda-vm-prover --release --features cuda --test rpx_device_paths -- --nocapture +//! cargo test -p lambda-vm-prover --release --features cuda --test rpx_device_paths -- --ignored --nocapture --test-threads=1 +//! ``` +#![cfg(feature = "cuda")] + +use crypto::merkle_tree::merkle::MerkleTree; +use lambda_vm_prover::lfm::algebraic_commit::{AlgebraicPairBackend, RpxCommit, RpxStarkHash}; +use lambda_vm_prover::lfm::hash::LfmHasher; +use lambda_vm_prover::lfm::rpx::Rpx256; +use lambda_vm_prover::tables::types::FE; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math_cuda::grinding::Knobs; +use math_cuda::rpx::TreeWalk; + +type Ext3 = Degree3GoldilocksExtensionField; +type Fp3 = FieldElement; +type RpxGrind = stark::config::GrindingDigest; + +const P: u64 = 0xFFFF_FFFF_0000_0001; + +struct SplitMix(u64); + +impl SplitMix { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +fn canonical(v: u64) -> u64 { + if v >= P { v - P } else { v } +} + +/// Random states plus the edges; every fifth lane a raw twin `c + p`. +fn probe_states(seed: u64, n: usize) -> Vec<[u64; 12]> { + let mut rng = SplitMix(seed); + let mut states = vec![[0u64; 12], [P - 1; 12], [u64::MAX; 12]]; + for k in 0..n { + states.push(core::array::from_fn(|i| { + if (k + i) % 5 == 0 { + rng.next() % 0xFFFF_FFFF + P + } else { + rng.next() % P + } + })); + } + states +} + +fn host_permute(s: &[u64; 12]) -> [u64; 12] { + let out = Rpx256.permute(core::array::from_fn(|i| FE::from(s[i]))); + core::array::from_fn(|i| canonical(*out[i].value())) +} + +// =========================================================================== +// The permutation: the limb multiply and the variants. +// =========================================================================== + +/// The device PTX of `mul_limb` / `sqr_limb` against 128-bit arithmetic: the +/// words are congruent to the product (they need not be canonical). +#[test] +fn limb_primitives_match_128_bit_arithmetic_on_the_device() { + let edges = [ + 0u64, + 1, + 2, + P - 1, + P, + P + 1, + 0xFFFF_FFFF, + 0x1_0000_0000, + 1 << 63, + u64::MAX, + u64::MAX - 1, + 0xFFFF_FFFF_0000_0000, + ]; + let mut a = Vec::new(); + let mut b = Vec::new(); + for &x in &edges { + for &y in &edges { + a.push(x); + b.push(y); + } + } + let mut rng = SplitMix(0x11B); + for _ in 0..(1 << 16) { + a.push(rng.next()); + b.push(rng.next()); + } + let got = math_cuda::rpx::limb_probe(&a, &b).expect("limb probe (needs a GPU)"); + for (i, ((&x, &y), g)) in a.iter().zip(&b).zip(&got).enumerate() { + let want_mul = ((x as u128 * y as u128) % P as u128) as u64; + let want_sqr = ((x as u128 * x as u128) % P as u128) as u64; + assert_eq!(canonical(g[0]), want_mul, "mul_limb at #{i}: {x} * {y}"); + assert_eq!(canonical(g[1]), want_sqr, "sqr_limb at #{i}: {x}^2"); + assert_eq!( + canonical(g[2]), + want_mul, + "goldilocks::mul at #{i} (the control)" + ); + } +} + +/// Every compiled variant reproduces the host oracle, raw, one permutation and +/// three chained. +#[test] +fn every_permutation_variant_matches_the_host_oracle() { + let states = probe_states(0x0052_5058, 509); + let variants = math_cuda::rpx::chain_probe_variants().expect("variants (needs a GPU)"); + assert_eq!(variants, vec![0, 1, 3, 4, 5, 7, 9], "the probe set"); + let once: Vec<[u64; 12]> = states.iter().map(host_permute).collect(); + let thrice: Vec<[u64; 12]> = states + .iter() + .map(|s| host_permute(&host_permute(&host_permute(s)))) + .collect(); + for &v in &variants { + let got1 = math_cuda::rpx::permute_chain_probe(v, &states, 1).expect("chain probe"); + let got3 = math_cuda::rpx::permute_chain_probe(v, &states, 3).expect("chain probe"); + for n in 0..states.len() { + assert_eq!(got1[n], once[n], "variant {v}, state {n}, one permutation"); + assert_eq!(got3[n], thrice[n], "variant {v}, state {n}, three chained"); + } + } + // The module's own permutation (this cubin's variant) through `rpx_permute_probe`. + let module = math_cuda::rpx::permute_probe(&states).expect("permute probe"); + assert_eq!(module, once, "rpx_permute_probe must match the oracle"); +} + +// =========================================================================== +// The Merkle tops: the half-warp permutation and the warp tree walk. +// =========================================================================== + +#[test] +fn the_half_warp_permutation_matches_the_host_oracle() { + // An odd count, so the last warp has a dead half. + let states = probe_states(0x5117, 1001); + let got = math_cuda::rpx::permute_warp_probe(&states).expect("warp probe (needs a GPU)"); + for (n, (s, g)) in states.iter().zip(&got).enumerate() { + assert_eq!(*g, host_permute(s), "state {n}"); + } +} + +fn random_leaves(num_leaves: usize, seed: u64) -> Vec { + let mut rng = SplitMix(seed); + (0..num_leaves * 32).map(|_| rng.next() as u8).collect() +} + +/// Every warp walk builds the same tree as a thread per parent, node for node, +/// from 2 to 2^18 leaves — which covers every regime: tail only, warp levels +/// into the tail, and wide per-parent levels above them. Arbitrary leaf bytes are fair here: +/// both walks decode big-endian words without reducing them. +#[test] +fn warp_walks_match_the_thread_per_parent_walk() { + let walks = [ + TreeWalk::WARP, + // Every narrow level through the warp level kernel, and a short tail. + TreeWalk::Warp { + level_max_pairs: u64::MAX, + tail_max_pairs: 2, + }, + // No warp level at all: per-parent levels straight into the warp tail. + TreeWalk::Warp { + level_max_pairs: 0, + tail_max_pairs: 64, + }, + ]; + for log in 1u32..=18 { + let leaves = random_leaves(1 << log, 0x7EE + log as u64); + let want = + math_cuda::rpx::build_merkle_tree_on_device_with(&leaves, TreeWalk::ThreadPerParent) + .expect("per-parent walk (needs a GPU)"); + for walk in walks { + let got = + math_cuda::rpx::build_merkle_tree_on_device_with(&leaves, walk).expect("warp walk"); + assert!( + got == want, + "{walk:?} differs from the per-parent walk at 2^{log} leaves" + ); + } + // A control that can fail: one flipped bit in the LAST leaf must move + // the warp walk's root (the last leaf reaches the root through the + // rightmost pair of every level, the tail's last lane included). + let mut tampered = leaves.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 1; + let t = math_cuda::rpx::build_merkle_tree_on_device_with(&tampered, TreeWalk::WARP) + .expect("warp walk"); + assert_ne!( + t[..32], + want[..32], + "a flipped leaf bit must move the root at 2^{log}" + ); + } +} + +/// The warp walk against the HOST tree (the Pair backend's FRI-layer tree), +/// node for node — the oracle the per-parent walk is pinned to elsewhere. +#[test] +fn the_warp_walk_matches_the_host_tree() { + for log in [1u32, 3, 6, 7, 8, 10, 13, 15] { + let num_leaves = 1usize << log; + let mut rng = SplitMix(0xABC + log as u64); + let evals: Vec = (0..num_leaves * 2) + .map(|_| { + Fp3::new([ + FieldElement::from(rng.next() % P), + FieldElement::from(rng.next() % P), + FieldElement::from(rng.next() % P), + ]) + }) + .collect(); + let leaves: Vec<[Fp3; 2]> = evals.chunks_exact(2).map(|c| [c[0], c[1]]).collect(); + let cpu_tree = + MerkleTree::>::build(&leaves).expect("CPU tree"); + let cpu_nodes = cpu_tree.nodes(); + let leaf_bytes: Vec = cpu_nodes[num_leaves - 1..] + .iter() + .flat_map(|n| n.iter().copied()) + .collect(); + let got = math_cuda::rpx::build_merkle_tree_on_device_with(&leaf_bytes, TreeWalk::WARP) + .expect("warp walk (needs a GPU)"); + assert_eq!(got.len(), cpu_nodes.len() * 32, "node count at 2^{log}"); + for (i, want) in cpu_nodes.iter().enumerate() { + assert_eq!( + &got[i * 32..(i + 1) * 32], + &want[..], + "node {i} at 2^{log} leaves" + ); + } + } +} + +// =========================================================================== +// The grind: the work-queue kernel. +// =========================================================================== + +fn seed_for(i: usize) -> [u8; 32] { + let mut seed = [0u8; 32]; + seed[..8].copy_from_slice(&(i as u64).to_le_bytes()); + seed[8] = 0xA5; + seed +} + +/// The queue returns the stride kernel's nonce for 256 seeds at the production +/// factor, at the card-filling grid, at the stride kernel's grid and a small one, +/// and at scan factor 1 (where some seeds miss their first block and relaunch). +#[test] +fn the_queue_grind_returns_the_stride_grinds_nonce() { + let factor = 20u8; + let fill = math_cuda::grinding::queue_grid().expect("queue grid (needs a GPU)"); + let scan1 = Knobs { + scan: 1, + grid: Knobs::DEFAULT.grid, + }; + let mut relaunched = 0usize; + for i in 0..256 { + let seed = seed_for(i); + let felts = stark::grinding::inner_hash_felts::(&seed, factor); + let want = math_cuda::grinding::generate_nonce_rpx_gpu_at(&felts, factor, Knobs::DEFAULT) + .expect("stride grind (needs a GPU)"); + for grid in [fill, 1024, 170] { + let got = math_cuda::grinding::generate_nonce_rpx_gpu_queue_at( + &felts, + factor, + Knobs::DEFAULT, + grid, + ) + .expect("queue grind"); + assert_eq!(got, want, "seed {i}, grid {grid}"); + } + let got = math_cuda::grinding::generate_nonce_rpx_gpu_queue_at(&felts, factor, scan1, fill) + .expect("queue grind at scan 1"); + assert_eq!(got, want, "seed {i}, scan 1"); + if want >= 1 << factor { + relaunched += 1; + } + } + assert!( + relaunched > 0, + "scan 1 must have exercised a relaunch on some seed" + ); + println!( + "queue parity: 256 seeds x 4 queue arms identical; {relaunched} seeds needed a relaunch at scan 1" + ); +} + +/// Small factors against the HOST's smallest valid nonce. +#[test] +fn the_queue_grind_returns_the_host_smallest_nonce() { + let fill = math_cuda::grinding::queue_grid().expect("queue grid (needs a GPU)"); + for factor in [12u8, 13, 14, 16] { + for i in 0..16 { + let seed = seed_for(1000 + i); + let felts = stark::grinding::inner_hash_felts::(&seed, factor); + let got = math_cuda::grinding::generate_nonce_rpx_gpu_queue_at( + &felts, + factor, + Knobs::DEFAULT, + fill, + ) + .expect("queue grind"); + let want = crypto::grinding::generate_nonce_smallest::(&seed, factor) + .expect("host smallest nonce"); + assert_eq!(got, want, "factor {factor}, seed {i}"); + assert!(stark::grinding::is_valid_nonce::( + &seed, got, factor + )); + } + } +} diff --git a/prover/tests/rpx_grind_device.rs b/prover/tests/rpx_grind_device.rs new file mode 100644 index 000000000..a10a36e45 --- /dev/null +++ b/prover/tests/rpx_grind_device.rs @@ -0,0 +1,205 @@ +//! ★★ The test that fails without the RPX device grind arm. +//! +//! Needs a GPU, like every other test in this file's family: +//! +//! ```text +//! cargo test -p lambda-vm-prover --release --features cuda --test rpx_grind_device -- --nocapture +//! ``` +//! +//! # What it is for +//! +//! `generate_nonce_maybe_gpu` dispatches the nonce search to a device kernel +//! chosen by the digest. While that dispatch read "is `D` platform keccak", an +//! RPX configuration silently took the HOST search — ~2^20 RPX permutations per +//! grind, thousands of grinds per block proof. A measured WHIR block arm came in +//! at 571 s against keccak's 39 s, ~510 s of it on this line, with a correct, +//! KAT-pinned `rpx_grind_search` sitting unused in the cubin. Nothing failed; +//! the proof was valid; only the clock said so. +//! +//! So the assertions here are about WHICH KERNEL RAN, not about whether a grind +//! succeeded — a grind succeeds either way, which is exactly why the regression +//! was invisible. +//! +//! # Two counters, not one +//! +//! A single "device grinds" counter is satisfied by the keccak kernel firing +//! under an RPX configuration, which is the precise failure the dispatch exists +//! to prevent. So the RPX arm must show `rpx > 0 && keccak == 0` and the keccak +//! arm `keccak > 0 && rpx == 0`; either counter alone would pass on a dispatch +//! that ignored its key. +#![cfg(feature = "cuda")] + +use crypto::grinding::{ + generate_nonce_maybe_gpu, gpu_grind_calls, gpu_grind_calls_rpx, is_valid_nonce, + reset_gpu_grind_calls, +}; +use multilinear::whir_hash::{GrindingDigest, KeccakWhir, RpxWhir}; + +type Rpx = GrindingDigest; +type Keccak = GrindingDigest; + +/// The production grind depth. Also comfortably above `GRIND_MIN_FACTOR`, below +/// which the dispatch keeps the search on the CPU on purpose. +const FACTOR: u8 = 20; + +fn seed_of(byte: u8) -> [u8; 32] { + [byte; 32] +} + +/// Every test here resets and then reads the PROCESS-GLOBAL grind counters, +/// and libtest runs tests on parallel threads by default. Unserialised, one +/// test's grind or reset lands between another's reset and read: the keccak +/// test's grind is counted under the RPX test, or a reset shrinks the counter +/// test's three grinds to one — failures of the harness, with the dispatch +/// correct. The Makefile runs its GPU counter suites with `--test-threads=1` +/// for the same reason; this lock makes the file correct under the default +/// too. Each test takes it for its whole body. +static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serial() -> std::sync::MutexGuard<'static, ()> { + // A test that failed while holding the lock poisons it. The next test + // resets the counters before reading them, so the poison carries nothing. + SERIAL + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// ★★ An RPX grind runs on the RPX kernel, and the nonce it returns is one the +/// host accepts. +/// +/// Remove the `Arm::Rpx256` branch from `crypto::grinding`'s dispatch and this +/// reads `rpx grinds 0` and fails — the search still finds a nonce, on the host, +/// which is the whole point. +#[test] +fn an_rpx_grind_runs_on_the_rpx_kernel() { + let _serial = serial(); + let seed = seed_of(0xA7); + reset_gpu_grind_calls(); + + let nonce = generate_nonce_maybe_gpu::(&seed, FACTOR).expect("a nonce exists"); + + println!( + "rpx grinds {} · keccak grinds {} · nonce {nonce}", + gpu_grind_calls_rpx(), + gpu_grind_calls() + ); + assert!( + gpu_grind_calls_rpx() > 0, + "the RPX grind did not reach the device: the dispatch has no RPX arm, \ + or the kernel returned a nonce the host rejected and it fell back" + ); + assert_eq!( + gpu_grind_calls(), + 0, + "the KECCAK kernel ran under an RPX configuration" + ); + assert!( + is_valid_nonce::(&seed, nonce, FACTOR), + "the device nonce must satisfy the host predicate" + ); +} + +/// The keccak arm still runs on keccak's kernel — so the change above moved a +/// dispatch rather than replacing one. +#[test] +fn a_keccak_grind_still_runs_on_the_keccak_kernel() { + let _serial = serial(); + let seed = seed_of(0x5C); + reset_gpu_grind_calls(); + + let nonce = generate_nonce_maybe_gpu::(&seed, FACTOR).expect("a nonce exists"); + + println!( + "keccak grinds {} · rpx grinds {} · nonce {nonce}", + gpu_grind_calls(), + gpu_grind_calls_rpx() + ); + assert!(gpu_grind_calls() > 0, "the keccak grind left the device"); + assert_eq!( + gpu_grind_calls_rpx(), + 0, + "the RPX kernel ran under a keccak configuration" + ); + assert!(is_valid_nonce::(&seed, nonce, FACTOR)); +} + +/// ★ The device reproduces the ORACLE table, nonce for nonce. +/// +/// The three rows of `RPX_GRIND_VECTORS` from +/// `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h` — the per-table branch's +/// host implementation, which this repository did not produce. The kernel +/// `atomicMin`s, so it returns the SMALLEST valid nonce, which is the column +/// recorded there. +/// +/// This is what makes the two counter tests more than launch counts: they say a +/// kernel ran, this says it computed the right hash. +#[test] +fn the_device_reproduces_the_oracle_grind_vectors() { + let _serial = serial(); + for (byte, factor, want) in [(90u8, 12u8, 1342u64), (17, 13, 300), (32, 14, 705)] { + let seed = seed_of(byte); + reset_gpu_grind_calls(); + let got = generate_nonce_maybe_gpu::(&seed, factor).expect("a nonce exists"); + assert!( + gpu_grind_calls_rpx() > 0, + "seed 0x{byte:02x}: the search did not reach the RPX kernel" + ); + assert_eq!( + got, want, + "seed 0x{byte:02x}, factor {factor}: the device must return the oracle's nonce" + ); + } +} + +/// ✓ The kill switch still works, and its effect is visible in the counter — +/// so a zero counter in a real run can be read as "no device grind" rather than +/// ambiguously. +/// +/// `LAMBDA_VM_NO_GPU_GRIND` is read once into a `OnceLock`, so this cannot be +/// toggled mid-process; it is asserted structurally instead, by requiring the +/// counter and the search to agree about whether a device was used. +#[test] +fn a_device_grind_is_exactly_what_the_counter_reports() { + let _serial = serial(); + let seed = seed_of(0x11); + reset_gpu_grind_calls(); + assert_eq!(gpu_grind_calls_rpx(), 0, "reset must zero the counter"); + + let n = 3; + for i in 0..n { + let mut s = seed; + s[0] = i as u8; + let nonce = generate_nonce_maybe_gpu::(&s, FACTOR).expect("a nonce exists"); + assert!(is_valid_nonce::(&s, nonce, FACTOR)); + } + assert_eq!( + gpu_grind_calls_rpx(), + n, + "the counter must be one per successful device grind, not a flag" + ); +} + +/// ★ The prove split's `n/airs on device` counts RPX device grinds. +/// +/// Each `PROVE SPLIT` line takes its device grinds as a delta of +/// `stark::prove_split::device_grinds_now`. That read the keccak counter alone, +/// so under RPX every line said `0/airs on device` while every table ground on +/// the card. This reads the same function around one RPX device grind; the +/// keccak-only count reads 0 here and fails. +#[test] +fn the_prove_split_counts_an_rpx_device_grind() { + let _serial = serial(); + let seed = seed_of(0x3D); + reset_gpu_grind_calls(); + let before = stark::prove_split::device_grinds_now(); + + let nonce = generate_nonce_maybe_gpu::(&seed, FACTOR).expect("a nonce exists"); + + assert!(is_valid_nonce::(&seed, nonce, FACTOR)); + assert_eq!(gpu_grind_calls_rpx(), 1, "the grind ran on the RPX kernel"); + assert_eq!( + stark::prove_split::device_grinds_now() - before, + 1, + "the prove split must count the RPX device grind" + ); +} diff --git a/prover/tests/rpx_host_kat_vectors.rs b/prover/tests/rpx_host_kat_vectors.rs new file mode 100644 index 000000000..2783d63f1 --- /dev/null +++ b/prover/tests/rpx_host_kat_vectors.rs @@ -0,0 +1,307 @@ +//! Generator for the Rust-oracle tables in +//! `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h`. +//! +//! The RPX device kernel (`crypto/math-cuda/kernels/rpx.cu`) is pinned to THIS +//! crate's `Rpx256` through the host known-answer harness `rpx_host_kat.cpp`. +//! miden publishes no RPX known-answer table (`lfm/rpx.rs`, "PROVENANCE"), so +//! the Rust host implementation IS the oracle, and this test prints the tables +//! the harness embeds, as C++ source: +//! +//! Table 2 — the bare permutation on eleven states: all-zero, all-(p−1), +//! `0..12`, alternating, two one-hot lanes, four random, and the +//! canonicalisation witness (see `permutation_inputs`); +//! Table 3 — the rate-8 OVERWRITE-duplex leaf (`algebraic_commit::sponge_leaf`) +//! at lengths 0, 1, 7, 8, 9, 16, 17 felts; +//! Table 4 — the parent `compress(l, r)` = one permutation of `[l ‖ r ‖ 0⁴]`; +//! Table 5 — the proof-of-work grind: the inner hash's four BIG-endian felts +//! and the SMALLEST valid nonce, at three grinding factors, taken +//! through the production predicate `stark::grinding::is_valid_nonce` +//! over `GrindingDigest` — plus, as the endianness +//! control, what the same kernel returns on the little-endian +//! reading of the same inner hash. +//! +//! Every printed value is CANONICAL (`< p`), and the harness compares the +//! kernel's output against it RAW — the kernel's final canonicalisation loop +//! is part of what these tables pin, so nothing may canonicalise on its +//! behalf. +//! +//! `#[ignore]`d because it prints rather than asserts. Run with +//! +//! cargo test -p lambda-vm-prover --test rpx_host_kat_vectors -- --ignored --nocapture +//! +//! and paste everything between the `>>> BEGIN` / `<<< END` lines over the +//! matching region of `rpx_kat_vectors.h`. The inputs are DERIVED HERE from +//! fixed seeds and printed alongside the outputs, so the header stays +//! self-contained data — the harness never regenerates anything. + +use lambda_vm_prover::lfm::algebraic_commit::{RpxStarkHash, digest_to_commitment, sponge_leaf}; +use lambda_vm_prover::lfm::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use lambda_vm_prover::lfm::rpx::Rpx256; +use lambda_vm_prover::tables::types::FE; +use stark::config::GrindingDigest; +use stark::grinding::{inner_hash_felts, is_valid_nonce}; + +/// The digest the RPX configuration grinds over — its transcript's hash. +type RpxGrind = GrindingDigest; + +/// The Goldilocks prime, for canonicalising raw values and for the `p − 1` +/// input. +const P: u64 = 0xFFFF_FFFF_0000_0001; + +/// The leaf lengths the phase-1 gate names: empty, a partial block, one felt +/// under a block, an exact block (no trailing permutation), one over, two exact +/// blocks, two blocks plus one. +const LEAF_LENGTHS: [usize; 7] = [0, 1, 7, 8, 9, 16, 17]; + +/// Widest leaf in Table 3 — the header's fixed-width `felts[]` array. +const LEAF_MAX_FELTS: usize = 17; + +/// splitmix64 — a fixed, documented PRNG so the inputs are reproducible from +/// the seed alone. +fn splitmix64(seed: &mut u64) -> u64 { + *seed = seed.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *seed; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A canonical random felt. +fn random_felt(seed: &mut u64) -> u64 { + splitmix64(seed) % P +} + +/// The canonical `u64` of a field element. `value()` is the raw storage, which +/// the field allows to sit in `[p, 2^64)`; one subtraction canonicalises it. +fn canonical(f: &FE) -> u64 { + let v = *f.value(); + if v >= P { v - P } else { v } +} + +fn fe_array(raw: &[u64; N]) -> [FE; N] { + core::array::from_fn(|i| FE::from(raw[i])) +} + +fn cpp_list(vals: &[u64]) -> String { + vals.iter() + .map(|v| format!("{v}ull")) + .collect::>() + .join(", ") +} + +/// The eleven permutation inputs, each with the name the harness prints on a +/// failure (and, for the witness, matches on). +fn permutation_inputs() -> Vec<(&'static str, [u64; HASH_STATE_FELTS])> { + let mut v: Vec<(&'static str, [u64; HASH_STATE_FELTS])> = vec![ + ("all-zero", [0; HASH_STATE_FELTS]), + ("all-(p-1)", [P - 1; HASH_STATE_FELTS]), + ("lanes 0..12", core::array::from_fn(|i| i as u64)), + ( + "alternating 0 / p-1", + core::array::from_fn(|i| if i % 2 == 0 { 0 } else { P - 1 }), + ), + ( + "one-hot lane 0", + core::array::from_fn(|i| u64::from(i == 0)), + ), + ( + "one-hot lane 11", + core::array::from_fn(|i| u64::from(i == HASH_STATE_FELTS - 1)), + ), + ]; + let names = ["random #1", "random #2", "random #3", "random #4"]; + for (k, name) in names.iter().enumerate() { + let mut seed = 0x5250_5800_0000_0000 + k as u64; // "RPX\0" + k + v.push((name, core::array::from_fn(|_| random_felt(&mut seed)))); + } + // ★ The canonicalisation witness (review finding on the phase-1 PR). The + // kernel canonicalises its output in a final loop, and a check that + // compares canonical values — or raw values that happen to be canonical, + // which is all but a 2^-32 slice per lane — cannot see whether that loop + // is there. This input is built so that it is not: its M-round MDS output + // lane 0 is `p − ARK1[6][0] + 1`, so the device's final `add` yields the + // raw twin `p + 1` where the field value is 1. Derived by inverting the + // permutation from that target + // (`crypto/math-cuda/tests/host_kat/rpx_canon_witness.py`); the harness + // matches this row BY NAME, replays the rounds to assert the twin is still + // produced, and compares `permute`'s output raw against the digits below. + v.push(( + "canonicalisation witness", + [ + 15055324559807314153, + 10242425218814686878, + 9326602342065331773, + 15451135068213333861, + 17942679252967467289, + 9284164080268346300, + 5090350781253234438, + 9328738269791029498, + 18385380985273671691, + 3238854716908013220, + 5495049682105235955, + 15773368383738726538, + ], + )); + v +} + +/// The grind rows: `(seed byte, grinding factor)`. The seed is the byte +/// repeated 32 times, as `transcript.state()` never is — which is the point, +/// since a vector wants to be reproducible from the header alone rather than +/// from a prover run. +/// +/// The factors are small so the harness can scan `[0, nonce]` one permutation +/// at a time in a fraction of a second, and large enough that a broken kernel +/// cannot pass by luck: at factor 12 a wrong hash lands under the limit with +/// probability 2⁻¹². +const GRIND_ROWS: [(u8, u8); 3] = [(0x5a, 12), (0x11, 13), (0x20, 14)]; + +/// The device predicate, spelled out over an explicit felt block. +/// +/// Used ONLY for the endianness control below. The pinned rows go through +/// `is_valid_nonce`, the production predicate, which takes a seed and so cannot +/// be pointed at a deliberately wrong reading of the inner hash. +fn grind_head(inner: &[u64; 4], nonce: u64) -> u64 { + let felts: [FE; 5] = core::array::from_fn(|i| FE::from(if i < 4 { inner[i] } else { nonce })); + let commitment = digest_to_commitment(&sponge_leaf(HasherKind::Rpx, &felts)); + u64::from_be_bytes(commitment[..8].try_into().unwrap()) +} + +#[test] +#[ignore = "prints the Rust-oracle tables for rpx_kat_vectors.h; run with --ignored --nocapture"] +fn print_rpx_host_kat_vectors() { + let mut out = String::new(); + out.push_str("// >>> BEGIN RUST-ORACLE TABLES — generated by\n"); + out.push_str( + "// cargo test -p lambda-vm-prover --test rpx_host_kat_vectors -- --ignored --nocapture\n", + ); + out.push_str( + "// (prover/tests/rpx_host_kat_vectors.rs). Paste verbatim; do not edit by hand.\n\n", + ); + + // ---- Table 2: the bare permutation ------------------------------------ + let inputs = permutation_inputs(); + out.push_str(&format!( + "inline constexpr int NUM_RPX_PERMUTATION_VECTORS = {};\n", + inputs.len() + )); + out.push_str( + "inline constexpr RpxPermutationVector RPX_PERMUTATION_VECTORS[NUM_RPX_PERMUTATION_VECTORS] = {\n", + ); + for (name, input) in &inputs { + let got = Rpx256.permute(fe_array(input)); + let got: Vec = got.iter().map(canonical).collect(); + out.push_str(&format!( + " {{\"{name}\",\n {{{}}},\n {{{}}}}},\n", + cpp_list(input), + cpp_list(&got) + )); + } + out.push_str("};\n\n"); + + // ---- Table 3: the leaf sponge ----------------------------------------- + out.push_str(&format!( + "inline constexpr int NUM_RPX_LEAF_VECTORS = {};\n", + LEAF_LENGTHS.len() + )); + out.push_str("inline constexpr RpxLeafVector RPX_LEAF_VECTORS[NUM_RPX_LEAF_VECTORS] = {\n"); + for &len in &LEAF_LENGTHS { + let mut seed = 0x1EAF_0000_0000_0000 + len as u64; + let raw: Vec = (0..len).map(|_| random_felt(&mut seed)).collect(); + let felts: Vec = raw.iter().map(|&r| FE::from(r)).collect(); + let digest = sponge_leaf(HasherKind::Rpx, &felts); + let digest: Vec = digest.iter().map(canonical).collect(); + // Fixed-width row: the tail beyond `len` is zero and never read. + let mut padded = raw.clone(); + padded.resize(LEAF_MAX_FELTS, 0); + out.push_str(&format!( + " {{{len}u,\n {{{}}},\n {{{}}}}},\n", + cpp_list(&padded), + cpp_list(&digest) + )); + } + out.push_str("};\n\n"); + + // ---- Table 4: the parent ------------------------------------------------ + let mut seed = 0x5041_5245_4E54_0000; // "PARENT" + let random_l: [u64; 4] = core::array::from_fn(|_| random_felt(&mut seed)); + let random_r: [u64; 4] = core::array::from_fn(|_| random_felt(&mut seed)); + let parents: [(&str, [u64; 4], [u64; 4]); 2] = [ + ( + "digits 0..8", + core::array::from_fn(|i| i as u64), + core::array::from_fn(|i| i as u64 + 4), + ), + ("random", random_l, random_r), + ]; + out.push_str(&format!( + "inline constexpr int NUM_RPX_PARENT_VECTORS = {};\n", + parents.len() + )); + out.push_str( + "inline constexpr RpxParentVector RPX_PARENT_VECTORS[NUM_RPX_PARENT_VECTORS] = {\n", + ); + for (name, l, r) in &parents { + let got = HasherKind::Rpx.compress(&fe_array(l), &fe_array(r)); + let got: Vec = got.iter().map(canonical).collect(); + out.push_str(&format!( + " {{\"{name}\",\n {{{}}},\n {{{}}},\n {{{}}}}},\n", + cpp_list(l), + cpp_list(r), + cpp_list(&got) + )); + } + out.push_str("};\n"); + // ---- Table 5: the proof-of-work grind --------------------------------- + out.push_str(&format!( + "\ninline constexpr int NUM_RPX_GRIND_VECTORS = {};\n", + GRIND_ROWS.len() + )); + out.push_str("inline constexpr RpxGrindVector RPX_GRIND_VECTORS[NUM_RPX_GRIND_VECTORS] = {\n"); + for &(seed_byte, factor) in &GRIND_ROWS { + let seed = [seed_byte; 32]; + let be = inner_hash_felts::(&seed, factor); + // The SMALLEST valid nonce, by the production predicate. `generate_nonce` + // would do, but its `find_any` returns an arbitrary one and the harness + // pins minimality as its completeness probe. + let nonce = (0u64..) + .find(|&n| is_valid_nonce::(&seed, n, factor)) + .expect("a valid nonce exists at these factors"); + // The endianness control: the same inner hash read the way keccak's + // lanes are. Its answer over the same range must differ, which is what + // makes the harness's BE row a claim about the reading and not just + // about the permutation. + let limit = 1u64 << (64 - factor); + // ★ CROSS-CHECK, not decoration: `is_valid_nonce` reaches the sponge + // through `AlgebraicDigest`'s byte buffer and `felts_from_bytes`, while + // `grind_head` builds the five felts directly — the way the kernel + // does. Agreeing on the whole scanned range is what says the device's + // felt block IS the host's message, and it is the one claim a device + // test on a box cannot make cheaply. + assert!( + (0..=nonce) + .all(|n| (grind_head(&be, n) < limit) + == is_valid_nonce::(&seed, n, factor)), + "the explicit felt block and the production predicate disagree \ + at seed {seed_byte:#x} factor {factor}" + ); + let le: [u64; 4] = core::array::from_fn(|i| be[i].swap_bytes()); + let le_nonce = (0..=nonce) + .find(|&n| grind_head(&le, n) < limit) + .map_or(u64::MAX, |n| n); + assert_ne!( + le_nonce, nonce, + "the endianness control is vacuous at seed {seed_byte:#x} factor {factor}" + ); + out.push_str(&format!( + " {{{seed_byte}u, {factor}u,\n {{{}}},\n {}ull, {}ull}},\n", + cpp_list(&be), + nonce, + le_nonce + )); + } + out.push_str("};\n"); + out.push_str("// <<< END RUST-ORACLE TABLES\n"); + + println!("{out}"); +} diff --git a/prover/tests/whir_transcript_configuration.rs b/prover/tests/whir_transcript_configuration.rs new file mode 100644 index 000000000..55a3e5d89 --- /dev/null +++ b/prover/tests/whir_transcript_configuration.rs @@ -0,0 +1,195 @@ +//! ★★★ A prove under `H` squeezes `H`'s sponge, and no other. +//! +//! ```text +//! cargo test -p lambda-vm-prover --features hash-metrics \ +//! --test whir_transcript_configuration +//! ``` +//! +//! # The claim a test finally makes +//! +//! `whir_hash_tests` has carried a test called +//! `the_transcript_follows_the_configuration` since the seam landed. It +//! constructed both transcript types itself and showed they draw different +//! challenges — true, and it never mentioned the prover. It was therefore true +//! throughout the period in which **no WHIR call site built an RPX transcript +//! at all**, and would have stayed true if none ever did. It is renamed to what +//! its body asserts; this file is the claim its old name made. +//! +//! Here the transcript is the production one: a real `multi_prove` through the +//! same entry point the prover uses, with the counters read afterwards. What is +//! observed is the prover's choice, not the test's. +//! +//! # Why every assertion is two-sided +//! +//! A single "squeezes" number cannot tell "the other hash ran" from "nothing +//! ran", and under the defect this exists to catch the RPX arm squeezed +//! thousands of times — all of them keccak. So each arm asserts both that its +//! own counters moved AND that the other's are zero: +//! +//! * keccak: every squeeze and absorb is keccak's; +//! * rpx: transcript work happened AND none of it was keccak. +//! +//! Without the second half the RPX assertion passed before the wiring landed. +//! Without the first, a prover that stopped squeezing would pass it too. +//! +//! # ⚠ Its own binary +//! +//! The counters are process-global and the prover's lib-test binary runs 600+ +//! tests in parallel, most of which hash. An early version of this test lived +//! there and read `215` squeezes of which `200` were keccak — the other 15 were +//! a neighbour's. A binary of its own, and a lock within it, are both needed. + +#![cfg(feature = "hash-metrics")] + +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; + +use crypto::hash_metrics; +use lambda_vm_prover::tables::eq::{EqOperation, generate_eq_trace}; +use lambda_vm_prover::test_utils::create_eq_air; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +/// Counters are global; tests here take turns. +static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serialise() -> std::sync::MutexGuard<'static, ()> { + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Grinding ON: the grind is one of the three hash consumers the seam names, +/// so a fixture that left it out could not see a half-flip. Four bits, so the +/// search costs nothing. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::uniform(4), + format: multilinear::whir_chain::ChainFormat::DEFAULT, + } +} + +/// One real prove under `H`, through the production entry point. +fn prove() { + let ops = vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ]; + let columns: Vec>> = generate_eq_trace(&ops).columns_main(); + + // Leaked so the layout borrows nothing from a temporary, as in the sibling + // fixture; this is a test binary that exits immediately after. + let air = Box::leak(Box::new(create_eq_air( + &ProofOptions::default_test_options(), + ))); + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + columns.len(), + columns[0].len().trailing_zeros() as usize, + Uniforms::default(), + ) + .expect("layout"); + + let table = CommittedTable::from_layout(layout, |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + + let mut transcript = crypto::fiat_shamir::default_transcript::DefaultTranscript::< + Ext, + H::Transcript, + >::new(b"whir-transcript-configuration"); + multilinear_table::multi_prove(&committed, &config(), &mut transcript, None).expect("prove"); +} + +#[test] +fn a_keccak_prove_squeezes_only_keccak() { + let _serialised = serialise(); + + hash_metrics::reset(); + prove::(); + let c = hash_metrics::snapshot(); + + assert!( + c.transcript_squeezes > 0 && c.transcript_absorbs > 0, + "the prove did no transcript work at all ({} absorbs, {} squeezes), so \ + the zeros below would mean nothing", + c.transcript_absorbs, + c.transcript_squeezes + ); + assert_eq!( + (c.transcript_absorbs_keccak, c.transcript_squeezes_keccak), + (c.transcript_absorbs, c.transcript_squeezes), + "a keccak prove squeezed something that was not keccak" + ); + assert_eq!( + (c.transcript_absorbs_rpx, c.transcript_squeezes_rpx), + (0, 0) + ); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +#[test] +fn an_rpx_prove_squeezes_only_rpx() { + let _serialised = serialise(); + + hash_metrics::reset(); + prove::(); + let c = hash_metrics::snapshot(); + + assert!( + c.transcript_squeezes > 0 && c.transcript_absorbs > 0, + "the prove did no transcript work at all, so the keccak zero below is \ + not evidence of anything" + ); + assert_eq!( + (c.transcript_absorbs_rpx, c.transcript_squeezes_rpx), + (c.transcript_absorbs, c.transcript_squeezes), + "an RPX prove did transcript work on some other sponge" + ); + assert_eq!( + (c.transcript_absorbs_keccak, c.transcript_squeezes_keccak), + (0, 0), + "the prover ran a KECCAK transcript under an RPX configuration — {} \ + absorbs and {} squeezes of it. This is the defect this file exists \ + for; it stood through four measured A/Bs.", + c.transcript_absorbs_keccak, + c.transcript_squeezes_keccak + ); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +/// ★ The two configurations do the same amount of transcript WORK. +/// +/// Only the bucket may differ. If a hash swap changed the absorb or squeeze +/// count, the per-arm bench line would report a protocol difference as a hash +/// difference, and a reader comparing arms would misattribute it. +#[test] +fn the_two_configurations_do_the_same_transcript_work() { + let _serialised = serialise(); + + hash_metrics::reset(); + prove::(); + let k = hash_metrics::snapshot(); + + hash_metrics::reset(); + prove::(); + let r = hash_metrics::snapshot(); + + assert_eq!( + (k.transcript_absorbs, k.transcript_squeezes), + (r.transcript_absorbs, r.transcript_squeezes), + "the two arms do different amounts of transcript work" + ); +} diff --git a/scripts/calibrate_threshold.sh b/scripts/calibrate_threshold.sh deleted file mode 100755 index 795eeb777..000000000 --- a/scripts/calibrate_threshold.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Calibrate the auto-disk-spill threshold: actual RSS / estimated_peak_bytes. -# -# Usage: calibrate_threshold.sh elf1.elf [elf2.elf ...] -# -# Builds CLI with jemalloc-stats, runs each ELF under `/usr/bin/time -v`, -# and prints predicted vs measured peak. Use the rss/pred ratio to adjust -# the safety margin in `auto_storage.rs`. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -OUT="/tmp/calibrate_threshold" - -mkdir -p "$OUT" -rm -f "$OUT"/*.txt - -echo "Building CLI with jemalloc-stats and disk-spill..." -cargo build --release -p cli --features jemalloc-stats,disk-spill --manifest-path "$ROOT_DIR/Cargo.toml" 2>&1 | tail -1 - -CLI="$ROOT_DIR/target/release/cli" - -printf "\n%-55s %10s %10s %10s %10s %10s\n" \ - "ELF" "pred(MB)" "heap(MB)" "rss(MB)" "rss/pred" "heap/pred" -printf '%.0s-' {1..110} -printf '\n' - -for elf in "$@"; do - name=$(basename "$elf") - RUST_LOG=info /usr/bin/time -v "$CLI" prove "$elf" -o "$OUT/proof.bin" \ - > "$OUT/out.txt" 2> "$OUT/err.txt" || { - echo "FAIL: $name" - tail -5 "$OUT/err.txt" - continue - } - - pred=$(grep -o 'estimated_peak_bytes: [0-9]*' "$OUT/err.txt" | awk '{print $2}') - heap_mb=$(grep -o 'Peak heap: [0-9]*' "$OUT/out.txt" | awk '{print $3}') - rss_kb=$(grep "Maximum resident set size" "$OUT/err.txt" | awk '{print $NF}') - - awk -v name="$name" -v p="$pred" -v h="$heap_mb" -v r="$rss_kb" 'BEGIN { - pred_mb = p / 1024 / 1024 - rss_mb = r / 1024 - printf "%-55s %10.0f %10.0f %10.0f %10.2f %10.2f\n", - name, pred_mb, h, rss_mb, rss_mb/pred_mb, h/pred_mb - }' - - rm -f "$OUT/proof.bin" -done - -echo "" -echo "Use the rss/pred ratio to adjust the safety margin in auto_storage.rs." diff --git a/scripts/ci_slow_tests.py b/scripts/ci_slow_tests.py new file mode 100644 index 000000000..afef3cdf0 --- /dev/null +++ b/scripts/ci_slow_tests.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""The filter GitHub CI's prover shards run under: every test of .config/ci-slow-tests.txt skipped, nothing else. + + python3 scripts/ci_slow_tests.py expr + prints the nextest filter expression (`-E`): not (package(=P) & (test(=N1) | test(=N2) | ...)), one group a + package, exact names only + python3 scripts/ci_slow_tests.py check + lists the archive's tests twice, unfiltered and under the expression, and fails (exit 1) unless the filter + skips exactly the listed tests the archive holds and every test binary keeps at least one test; prints the + counts and the listed names the archive no longer holds (renamed or removed tests: they run, nothing is lost) + +The list is read relative to the repository root (this file's parent directory's parent). +""" +import json +import os +import subprocess +import sys +from collections import defaultdict + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LIST = os.path.join(ROOT, ".config", "ci-slow-tests.txt") + + +def listed(): + """[(package, test name)] in file order, comments and blank lines dropped.""" + out = [] + with open(LIST) as f: + for n, line in enumerate(f, 1): + line = line.split("#", 1)[0].strip() + if not line: + continue + parts = line.split() + if len(parts) != 2: + sys.exit(f"{LIST}:{n}: want ` `, got {line!r}") + out.append((parts[0], parts[1])) + return out + + +def expr(): + by_pkg = defaultdict(list) + for pkg, name in listed(): + by_pkg[pkg].append(name) + if not by_pkg: + return "all()" + groups = [f"(package(={pkg}) & ({' | '.join(f'test(={n})' for n in names)}))" for pkg, names in by_pkg.items()] + return f"not ({' | '.join(groups)})" + + +def run_list(archive, filt): + cmd = ["cargo", "nextest", "list", "--archive-file", archive, "--message-format", "json"] + if filt: + cmd += ["-E", filt] + out = subprocess.run(cmd, check=True, capture_output=True, text=True, cwd=ROOT).stdout + doc = json.loads(out[out.index("{"):]) + tests = set() + for suite in doc["rust-suites"].values(): + binary = suite["binary-id"] + for name, case in suite["testcases"].items(): + match = case.get("filter-match", {}).get("status", "matches") + if match == "matches" and not case.get("ignored", False): + tests.add((binary, name)) + return tests + + +def check(archive): + full = run_list(archive, None) + kept = run_list(archive, expr()) + names = {name for _, name in listed()} + skipped = full - kept + held = {(b, n) for b, n in full if n in names} + stale = sorted(names - {n for _, n in full}) + bins = defaultdict(lambda: [0, 0]) + for b, _ in full: + bins[b][0] += 1 + for b, _ in kept: + bins[b][1] += 1 + emptied = sorted(b for b, (n_full, n_kept) in bins.items() if n_full and not n_kept) + print(f"ci-slow-tests: {len(full)} tests in the archive, {len(kept)} run on GitHub, {len(skipped)} skipped " + f"({len(names)} listed, {len(stale)} no longer in the archive)") + for n in stale: + print(f" listed, not in the archive (runs nowhere under this name; drop the line): {n}") + bad = [] + if skipped != held: + extra = sorted(skipped - held)[:10] + bad.append(f"the filter skips {len(skipped)} tests, the list names {len(held)} of them: extra {extra}") + if emptied: + bad.append(f"the filter leaves no test in {emptied}") + for b in bad: + print(f"ci-slow-tests: FAIL: {b}") + return 1 if bad else 0 + + +def main(): + if len(sys.argv) == 2 and sys.argv[1] == "expr": + print(expr()) + return 0 + if len(sys.argv) == 3 and sys.argv[1] == "check": + return check(sys.argv[2]) + print(__doc__) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/profiling/README.md b/scripts/profiling/README.md index 6f7b355c1..084304fd3 100644 --- a/scripts/profiling/README.md +++ b/scripts/profiling/README.md @@ -126,6 +126,7 @@ Useful prover knobs for A/B experiments (pre-existing, see plan §11): | var | effect | |---|---| | `LAMBDA_VM_NO_GPU_GRIND=1` | force the round-4 proof-of-work nonce search onto the CPU (presence-based, like `LAMBDA_VM_NO_GPU_LOGUP`). The production escape hatch if the device search ever misbehaves; also the way to A/B the grind on its own. Below grinding factor 12 the GPU path declines regardless, so wrap and recursion proves (factor 1) never use it | +| `LAMBDA_VM_LDE_LEGACY=1` | send every device LDE back to the legacy per-level pipeline (row-major commits: one butterfly level per launch plus the in-place transpose; batched entry points: the per-level batched kernels; the WHIR commit's encoding: `lift_spread` plus the tiled NTT and its cached twiddle table) instead of the column-major engine (`math_cuda::lde_cm`). Same values, roots and proofs; the escape hatch and the way to A/B the engine. The process prints `[gpu] LDE: …` once, naming the pipeline it ran | Residency and diagnostic knobs: diff --git a/scripts/set_ethrex_rev.sh b/scripts/set_ethrex_rev.sh index 5964420e7..4fb829a1d 100755 --- a/scripts/set_ethrex_rev.sh +++ b/scripts/set_ethrex_rev.sh @@ -13,6 +13,10 @@ # scripts/set_ethrex_rev.sh --guest-only <40-char-sha> re-pin only the guest graph # scripts/set_ethrex_rev.sh --show print the current pin(s) # +# A guest manifest may build its ethrex crates from a fork through a [patch] section; see +# fork_patches below. --show checks the fork's recorded base is the pin, and a re-pin +# refuses while the fork sits on another rev. +# # `--guest-only` moves the crates the guest ELF is built from and leaves the # host-side fixture tooling where it is. That split is what lets a benchmark vary # the guest across two revs while holding the fixture constant: the fixture is @@ -44,14 +48,49 @@ TOOLING_MANIFESTS=( MANIFESTS=("${GUEST_MANIFESTS[@]}" "${TOOLING_MANIFESTS[@]}") +# The pins: every lambdaclass/ethrex dependency's rev. A fork patch's revs (below) are not pins. current_revs() { - grep -ho 'rev = "[0-9a-f]\{40\}"' "${MANIFESTS[@]}" | sort -u | sed 's/rev = //;s/"//g' + grep -ho 'lambdaclass/ethrex\.git", rev = "[0-9a-f]\{40\}"' "${MANIFESTS[@]}" | sort -u | sed 's/.*rev = //;s/"//g' +} + +# A guest manifest may build ethrex from a fork: a `[patch."https://github.com/lambdaclass/ethrex.git"]` +# section whose entries name one git URL and one rev, with an `# ethrex-fork-base: ` line naming the +# pinned rev the fork's commits sit on. The fixtures and the native reference stay on the pin, so the fork +# must keep ProgramInput as it is at its base, and its base must be the pin. Prints, for each manifest +# carrying such a patch, " "; an incomplete patch prints " INVALID ". +fork_patches() { + local m section urls revs base + for m in "${GUEST_MANIFESTS[@]}"; do + section="$(awk '/^\[/ { p = ($0 == "[patch.\"https://github.com/lambdaclass/ethrex.git\"]") } p' "$m")" + [[ -n "$section" ]] || continue + urls="$(printf '%s\n' "$section" | grep -o 'git = "[^"]*"' | sort -u | sed 's/git = //;s/"//g')" + revs="$(printf '%s\n' "$section" | grep -o 'rev = "[0-9a-f]\{40\}"' | sort -u | sed 's/rev = //;s/"//g')" + base="$(sed -n 's/^# ethrex-fork-base: \([0-9a-f]\{40\}\)$/\1/p' "$m")" + if [[ "$(printf '%s\n' "$urls" | grep -c .)" != "1" || "$(printf '%s\n' "$revs" | grep -c .)" != "1" ]]; then + echo "$m INVALID the patch must name one git URL and one rev (urls: $(echo $urls), revs: $(echo $revs))" + elif [[ "$(printf '%s\n' "$base" | grep -c .)" != "1" ]]; then + echo "$m INVALID no single '# ethrex-fork-base: ' line" + else + echo "$m $urls $revs $base" + fi + done } if [[ "${1:-}" == "--show" || $# -eq 0 ]]; then # No `mapfile`: macOS ships bash 3.2, where it does not exist. revs="$(current_revs)" if [[ "$(printf '%s\n' "$revs" | wc -l | tr -d ' ')" == "1" ]]; then + forks="$(fork_patches)" + if [[ -n "$forks" ]]; then + # A fork patch is legitimate only on the pin: its base must be the pinned rev. + while read -r manifest url rev base; do + if [[ "$url" == "INVALID" || "$base" != "$revs" ]]; then + echo "FORK: $manifest: ${url} ${rev} ${base} (the fork's base must be the pin $revs)" >&2 + exit 1 + fi + echo "fork: $manifest builds ethrex from $url at $rev (base $base)" >&2 + done <<< "$forks" + fi echo "$revs" else # Non-zero on purpose: this is the skew the script exists to prevent, and it is @@ -79,6 +118,19 @@ if [[ ! "$NEW_REV" =~ ^[0-9a-f]{40}$ ]]; then exit 2 fi +# A fork patch sits on one ethrex rev: moving the pin under it would build the guest from the fork's old +# base while the fixtures move. Rebase the fork onto the new rev, update the patch's revs and its +# `ethrex-fork-base` line (or remove the patch) first. +while read -r manifest url rev base; do + [[ -n "${manifest:-}" ]] || continue + if [[ "$url" == "INVALID" || "$base" != "$NEW_REV" ]]; then + echo "error: $manifest builds ethrex from a fork ($url $rev, base $base)." >&2 + echo " Rebase the fork onto $NEW_REV and update its [patch] revs and ethrex-fork-base line," >&2 + echo " or remove the patch, then re-pin. Nothing was written." >&2 + exit 1 + fi +done <<< "$(fork_patches)" + # Only rewrite revs on ethrex deps: the same file may pin other git deps. PATTERN='(github\.com/lambdaclass/ethrex\.git", rev = ")[0-9a-f]{40}(")' diff --git a/spec/about_ecalls.typ b/spec/about_ecalls.typ index 9b37d5f21..7d146df80 100644 --- a/spec/about_ecalls.typ +++ b/spec/about_ecalls.typ @@ -31,4 +31,12 @@ Negative numbers (represented as 2s complement 64-bit numbers), are used for our / 64: `write` (@commit) / 93: `exit` (@halt) / -1: `SHA256` (@sha256) -/ -2: `KECCAK` (@keccak) \ No newline at end of file +/ -2: `KECCAK` (@keccak) +/ -5: `KECCAK256` (@keccak) — Keccak-256 of a buffer in guest memory, proven by + the `KECCAK` chip's sponge mode. +/ -3: `BLAKE3_6R` (@blake3) — the *6-round internal variant* of the BLAKE3 + compression function, not standard BLAKE3; see the named assumption on its + page. +/ -11: `ECSM` — secp256k1 scalar multiplication (implemented in + `executor/src/vm/instruction/execution.rs`; spec page pending). +/ -12: `ECREC` — ECDSA public-key recovery on secp256k1 (@ecrec). \ No newline at end of file diff --git a/spec/blake3.typ b/spec/blake3.typ new file mode 100644 index 000000000..42c26e89c --- /dev/null +++ b/spec/blake3.typ @@ -0,0 +1,174 @@ +#import "/book.typ": book-page, aside +#import "/src.typ": load_config, load_chip +#import "/chip.typ": ( + compute_nr_interactions, + render_chip_assumptions, + render_chip_variable_table, + total_nr_variables, + total_nr_instantiated_columns, + render_constraint_table, + render_chip_padding_table, +) + +#let config = load_config() +#let chip = load_chip("src/blake3.toml", config) + +#show: book-page(chip.name) +#let blake3 = raw(chip.name) + +The #blake3 chip applies the *6-round internal variant* of the BLAKE3 +compression function to a 176-byte memory region. It is an internal +Merkle / Fiat–Shamir hash accelerator: one compression digests a 64-byte +message block — exactly one 2-to-1 merge of two 32-byte chaining values — +and produces the full 16-word output (the truncated chaining value is +`out[0..8]`). + +⚠ *This is not standard BLAKE3.* The standard function applies 7 rounds; +this chip applies 6 (see @blake3-a6r). No external system will ever agree +on these digests. Standard library implementations (e.g. the official +`blake3` Rust crate) hardwire 7 rounds and cannot compute this function; +the host-side reference implementation lives in +`executor/src/vm/instruction/execution.rs` (`blake3_compress_6round`), +differentially pinned to the validated oracle in +`tooling/blake3-second-source/blake3-oracle/`. + += ECALL interface + +ECALL number `-3` (`0xFFFF_FFFF_FFFF_FFFD`). `A0` holds an 8-byte-aligned +pointer to a 176-byte state region of 22 consecutive little-endian dwords: + +#table( + columns: (auto, auto, auto), + [*dwords*], [*contents*], [*direction*], + [0..=3], [`h[0..8]` chaining value (2 u32 words per dword)], [read], + [4..=11], [`m[0..16]` message block], [read], + [12], [`t` counter (`t_lo` = low u32 → `v[12]`, `t_hi` = high u32 → `v[13]`)], [read], + [13], [`block_len` (low u32) | `flags` (high u32)], [read], + [14..=21], [`out[0..16]`], [written], +) + +Unaligned or overflowing state addresses are rejected by the executor. +The counter split order (`t_lo → v[12]`, `t_hi → v[13]`) is load-bearing +and was behaviourally verified against the official BLAKE3 crate through +two independent counter paths (oracle audit, 44/44). + += Chip structure + +== Columns and interactions + +#let nr_variables = total_nr_variables(chip) +#let nr_columns = total_nr_instantiated_columns(chip, config) +#let nr_interactions = compute_nr_interactions(chip) + +The formalized I/O-and-range surface below comprises #nr_variables +variables over #nr_columns columns and #nr_interactions interactions; the +full chip has 3,219 main columns and 1,397 interactions (the difference is +the mixing core's `BYTE_ALU[XOR]` lookups, whose SSA operand wiring is +normatively specified by the single-source Rust dataflow and the z3 gate — +see @blake3-scope). + +#render_chip_variable_table(chip, config) + +== Structure + +One row per compression call, fully unrolled: 6 rounds × 8 G-functions in +SSA form. The message schedule is a compile-time index permutation (the +`sched` array in `run_flow`, composed from `BLAKE3_MSG_PERMUTATION`), so +every round references the 16 original committed message words — there is +no state or message handoff between rows. I/O follows the KECCAK core idiom: an `ECALL` receiver binds +(timestamp, syscall number), a `MEMW` register read binds the `x10` +pointer, and 22 per-dword `MEMW` operations carry the reads and writes. + +Key constraint-design decisions (full rationale: +`thoughts/blake3/blake3-chip/DESIGN.md`, deltas in `IMPLEMENTATION.md`): + +- every eval constraint is gated by the multiplicity column $mu$, and the + maximum constraint degree *including* the $times mu$ factor is 3; +- 3-operand adds commit *two summed carry bits* with an explicit sum + identity (a ternary carry would be degree 4 after gating); +- 2-operand adds use an expression carry (no committed cell) with a + $mu$-gated booleanity; +- `rotr16`/`rotr8` are free byte relabels; `rotr12`/`rotr7` are inline + $mu$-gated Euclidean shift identities whose soundness rests on the + tight $[0, 2^16)$ bound of the `SLL` halfwords ($2^16$ is invertible + mod $p$); +- every add/shift output feeds a downstream `BYTE_ALU[XOR]` lookup, which + is its only byte range check; the message words, the previous + out-region content and the address bytes are never XOR-consumed and + carry explicit `ARE_BYTES` checks instead. + +The chip's wiring is single-sourced: the compression dataflow is written +once in `prover/src/tables/blake3.rs` (`run_flow`) and interpreted both as +column wiring (constraints + bus senders) and as the u32 witness (trace +fill + lookup multiplicities), so the two cannot diverge structurally. + +== Formalized constraints + +#render_constraint_table(chip, config, groups: "io") +#render_constraint_table(chip, config, groups: "addr") +#render_constraint_table(chip, config, groups: "range") +#render_constraint_table(chip, config, groups: "mu") + += Verification evidence + +The design was taken to a z3-gated model *before* the Rust implementation +(`thoughts/blake3/blake3-chip/z3_blake_verify.py`): the G quarter-round +and the init/feed-forward layout are UNSAT under free inputs, five +negative controls and two field-level bound-necessity controls are SAT, +and the concrete 6- and 7-round pipelines reproduce the oracle's pinned +vectors. Two independent transcription audits +(`thoughts/blake3/TRANSCRIPTION-AUDIT.md`, +`GATE-TRANSCRIPTION-AUDIT.md`) checked the gate against the oracle. The +Rust chip is additionally pinned by the 10 canonical 6-round vectors at +the syscall level and by an end-to-end prove+verify of chained +compressions. + += The 6-round assumption + +*A6R.* The BLAKE3 compression function restricted to 6 rounds is +collision-resistant and suitable as a 2-to-1 compression for Merkle +hashing and as a PRF for Fiat–Shamir, in the same sense the full 7-round +function is believed to be. (Precedent: KangarooTwelve's reduced-round +Keccak. Best public cryptanalysis of BLAKE3 reaches far fewer rounds; the +margin removed here is one round of seven.) + +*External review (2026-08).* The round-count choice was reviewed with +external symmetric-cryptography experts consulted by the project: removing +*one* round (7 → 6) was judged comfortable; removing *two* (7 → 5) was +explicitly not. Accordingly, 6 rounds is the endorsed floor. Variants +below 6 rounds are not formally ruled out, but they are not available on +the project's own authority: adopting one would require the external +experts to study the reduced-round margin specifically — a dedicated +cryptanalytic review, not an engineering or configuration decision. + +Any use of #blake3 as a Merkle or transcript hash *invokes this +assumption*. The z3 gate proves the chip computes 6-round BLAKE3 +correctly; it neither proves nor addresses whether 6 rounds are secure. + +*The assumption-free alternative.* The chip design is round-parameterised; +a 7-round instantiation (standard BLAKE3 compression, bit-compatible with +official parent-node merges) costs roughly 10–12% more per merge +end-to-end and requires no assumption beyond standard BLAKE3. + +*Ordering, reversed 2026-08-10.* The 7-round variant is the primary +target; the 6-round variant is the measured performance variant, kept +behind the round parameter and adopted only if that 10–12% is judged worth +signing A6R for. This reverses the ordering this section recorded before, +and the argument is the reference chain rather than cryptanalysis: at 7 +rounds the official crate is a direct external test vector for both the +primitive and its framing, and there is no assumption left to ratify or +defend at audit. It retracts nothing from the external review above — 6 +rounds remains the endorsed floor — and it is not a signature on A6R, +which falls due only if the default moves back to 6 +(`thoughts/shared/lfm-real-hash/A6R-signoff.md`). The chip specified on +this page is still the 6-round instantiation, so the above is recorded +intent and not a change that has landed here. If both are instantiated +they are distinct chips with distinct ECALL numbers. + += Cost + +Measured on the CPU bench box (32 cores, blowup 2, single-epoch +continuations): ≈5,473 compressions/s at ≥#raw("2^17") table rows, ≈7,194 +committed cell-equivalents per compression end-to-end (≈5,316 table-only) +— ≈12× the keccak-f permutation per 2-to-1 merge at equal wall time and +memory. Details and methodology: PR \#903. diff --git a/spec/book.typ b/spec/book.typ index 8bf8612af..151b80943 100644 --- a/spec/book.typ +++ b/spec/book.typ @@ -49,6 +49,8 @@ ("commit.typ", [`COMMIT` chip], ), ("sha256.typ", [`SHA256` accelerator], ), ("keccak.typ", [`KECCAK` accelerator], ), + ("blake3.typ", [`BLAKE3_6R` accelerator], ), + ("ecrec.typ", [`ECREC` accelerator], ), )) ) ) diff --git a/spec/ecrec.typ b/spec/ecrec.typ new file mode 100644 index 000000000..489b19435 --- /dev/null +++ b/spec/ecrec.typ @@ -0,0 +1,95 @@ +#import "/book.typ": book-page, todo + +#show: book-page("ecrec.typ") + +#let ecrec = raw("ECREC") +#let ecdas = raw("ECDAS") + +The ECDSA recovery accelerator returns the secp256k1 public key of a signature +in one call: given $(r, s)$, the message hash $z$ and the parity of $R$'s $y$, the +key $Q = r^(-1) (s R - z G) = u_1 G + u_2 R$ with $u_1 = -z r^(-1)$ and +$u_2 = s r^(-1) mod N$. It reuses the elliptic-curve machinery of the `ECSM` +call: the two scalar multiplications are two double-and-add chains of #ecdas +rows, and the final addition is one more #ecdas row. The #ecrec chip proves the +scalar arithmetic modulo $N$, the decompression of $R$, and binds the three +#ecdas conversations to the call. + +#todo[ + No `src/ecrec.toml` yet: the listings below are prose, and the code + (`prover/src/tables/ecrec.rs`) is the reference. +] + += The call + +ECALL number $-12$ (`u64::MAX - 11`). + +/ `x10`: address of the output, 72 bytes: $Q_x$ and $Q_y$ (32 bytes each, + big-endian), then a status doubleword (little-endian). +/ `x11`: address of the signature $r || s$, 64 bytes, each big-endian. +/ `x12`: address of the hash $z$, 32 bytes, big-endian. +/ `x13`: the parity of $R$'s $y$ (0 or 1), as a value. + +Status 0: the output holds the key. Status 1: the call *bailed*; the key bytes +are zero and the caller recovers the key another way. A bail proves nothing and +needs nothing proven: the caller's own path decides. The executor bails exactly +when $r$ or $s$ is not in $[1, N)$, when $r$ is not the $x$ of a curve point, +when $u_1 = 0$ ($z equiv 0 mod N$), or when $x(u_1 G) = x(u_2 R)$ (the chord is +undefined; this covers $Q$ at infinity). It stops with an error when `x13` is +above 1, when the signature and hash regions overlap, or when an address's low +32-bit limb plus its operand's last offset (71, 63, 31) reaches $2^32$. + += #ecrec + +One row per call, all-zero padding rows ($mu = 0$). $"OK"$ is a bit with +$"OK" <= mu$; $mu - "OK"$ is the status byte written. Every relation below is +multiplied by $"OK"$, and every range check is sent with multiplicity $"OK"$. + +== Memory + +$r$, $s$ (eight doubleword reads) and the `x11` register at $T$; `x12`, `x13` +(whose value must be $("PAR", 0)$) and $z$ (four reads) at $T + 1$; `x10` and +the nine output doubleword writes at $T + 2$. A big-endian memory byte $m$ of a +value is its little-endian limb $31 - m$, a column permutation. + +== Relations, under OK + +Byte-limb convolutions with 64 carries and a closing carry of zero, as `ECSM`'s: + +- $r w - 1 - q_w N = 0$: $w = r^(-1)$; +- $z w + u_1 - q_z N = 0$: $u_1 = -z w$ ($z$ as read, unreduced); +- $s w - u_2 - q_u N = 0$; +- $x_2 = r^2 mod p$ and $y_R^2 + p^2 - r x_2 - 7 - q_1 p = 0$: $R = (r, y_R)$ + is on the curve; +- $y_R[0] = 2 H + "PAR"$ with $H$ a byte: the requested root. + +$u_1$ and $u_2$ are 256 bit columns each (all zero unless $"OK"$). Bounds by +the overflow-carry chains: $r, s, u_1, u_2 < N$ and +$y_R, x_A, x_B, Q_x, Q_y < p$; $r, s, u_1, u_2 eq.not 0$ by `Zero` lookups. +$x_A eq.not x_B$: with $d_c$ the eight 32-bit word differences of +$x_B - x_A$ and witnesses $"DINV"_c$, $sum_c d_c "DINV"_c = "OK"$. On a bail, +$(mu - "OK") dot Q_i = 0$ for every key byte. + +== The three conversations + +- First chain $A = u_1 G$ at key $T$: seed $[0, T, G, G, ell_1 - 1, 0]$, + drain $[0, T, A, G, -1, 0]$, `Bit` receives $[T, i]$ with multiplicity + $u_(1,i)$, the top bit $[T, ell_1]$ self-sent. +- Second chain $B = u_2 R$ at key $T + 1$, the same shape on $R = (r, y_R)$. +- Final addition at key $T + 2$: send $[0, T + 2, A, B, 0, 1]$, receive + $[0, T + 2, Q, B, -1, 0]$: one #ecdas addition row. + += Soundness + +Under $"OK"$, $R$ is the curve point with $x = r$ and the requested parity; +$u_1, u_2$ are the reduced scalars of $(r, s, z)$; each chain is the +double-and-add of its scalar on its point, non-degenerate by `ECSM`'s argument +($0 < u < N$, prime order); the final addition is non-degenerate by +$x_A eq.not x_B$ on canonical $x_A, x_B$; the key is canonical. The keys +$T, T + 1, T + 2$ cannot meet another call's: every ECALL timestamp is a +multiple of 4, because the CPU's preprocessed timestamp column holds $4 (i + 1)$ +and each CPU table adds its base $4 dot S_k$, four times the rows of the tables +before it, which the verifier computes from the heights (the offsets 1 and 2 are +checked against the stride 4 when the prover is built), every #ecdas row +inherits its key from a seed, and only `ECSM`, #ecdas and #ecrec use the `Ecdas` +and `Bit` buses. Without the offset of the second chain, the two chains' `Bit` +messages would be one multiset and a prover could swap their scalars. diff --git a/spec/keccak.typ b/spec/keccak.typ index 11d900c73..84db88ac8 100644 --- a/spec/keccak.typ +++ b/spec/keccak.typ @@ -49,10 +49,69 @@ The state is then read into `input_state`, while the `output_state` is written b Lastly, the input state is pushed to the Keccak-round function, while the output after 24 rounds is taken off the bus: #render_constraint_table(chip, config, groups: "round") +The table has a second mode (the sponge mode below), and its rows are split by two mode bits: $mu_p$ marks a row of this permutation ECALL, $mu_s$ a row of a Keccak-256 call. +Every interaction above except the two round-chip ones is gated by $mu_p$. +The two round-chip interactions are the only ones both modes share, and they are weighted by $mu_p + mu_s$: a row of either mode asks the round chip because of its own mode bit, so its output is the permutation of its input. +The column $mu$ is the sum of the two bits (it marks the real rows for the table's cuts), so with all three bits a row is in at most one mode: +#render_constraint_table(chip, config, groups: "mode") + == Padding The #keccak table can be padded to the next power of two with the following value assignments: #render_chip_padding_table(chip, config) += Sponge mode: Keccak-256 of a buffer +#let sponge = load_chip("src/keccak_sponge.toml", config) + +ECALL number -5 (`KECCAK256`) hashes a buffer in guest memory: `A0 = x10` holds its address, `A1 = x11` its length in bytes and `A2 = x12` the address the 32-byte digest is written to. +Both addresses may have any alignment, and the digest may overlap the input: the input is read at the call's timestamp $T$ and the digest written at $T + 1$, so the hash is of the bytes as they were before the call. +The hash is Keccak-256 with the original Keccak padding `0x01 ... 0x80` over the 136-byte rate. +The call is refused unless every block's 136-byte window and the 32-byte digest stay below one $2^32$ boundary of the low address word: $(#`x10` mod 2^32) + 136 dot (floor(#`x11` / 136) + 1) <= 2^32$ and $(#`x12` mod 2^32) + 31 < 2^32$. + +The call is proven by the #keccak chip itself, in a second mode. +Two mode bits split the rows: `MU_P` marks a row of the permutation ECALL above, `MU_S` a row of a Keccak-256 call, with $mu = #`MU_P` + #`MU_S`$ and all three bits, so a row is in at most one mode. +Every interaction and constraint of the permutation mode is gated by `MU_P`, and every one of the sponge mode by `MU_S`, `first`, `last` or a read-mask bit that is zero off a sponge row. +The two round-chip interactions are weighted by `MU_P + MU_S`, so a sponge row's mode bit itself puts it on the round chip (on a sponge row that sum is `MU_S`, as written below). +A call of $n$ bytes takes $floor(n / 136) + 1$ rows, one per permutation: row $j$ reads block $j$ of the input, XORs the padded block into the rate of the state the previous row's permutation ended in, and sends the result to the round chip; the last row writes the first four lanes of its own output. + +== Columns +#let nr_variables = total_nr_variables(sponge) +#let nr_columns = total_nr_instantiated_columns(sponge, config) +#let nr_interactions = compute_nr_interactions(sponge) + +A sponge row uses #nr_variables variables, expressed using #nr_columns columns, and leverages #nr_interactions interaction(s): +#render_chip_variable_table(sponge, config) + +On a sponge row the chip's `timestamp` holds the key $(T, j)$: the call's timestamp and the block index. +The round chip links a request to its reply by that key alone, so it has to be unique per permutation: $j$ differs between the rows of one call, $T$ between calls, and a permutation row's key is $(T', 0)$ at its own ECALL's timestamp. +Every memory access of a sponge row uses $(T, 0)$ or $(T + 1, 0)$, never $j$. + +== Constraints +The first row receives the ECALL and reads its three operands. The length is read with a zero high word, so it is below $2^32$. The first row is block 0 and starts from the zero state: +#render_constraint_table(sponge, config, groups: "call") + +Consecutive rows of one call are linked by the `KECCAK_SPONGE` interaction, which carries the remaining byte count, the next block's address and the state; the receiving row takes the previous output's rate into `prev` and its capacity directly into the capacity lanes of `input_state`: +#render_constraint_table(sponge, config, groups: "chain") + +A lane wholly inside the input is read as one 8-byte access; the bytes of the lane the input ends in are read one at a time. Byte 7 of a lane is never read alone: +#render_constraint_table(sponge, config, groups: "read") + +The read mask $s_(L,b) = "full"_L + "part"_(L,b)$ ($b < 7$), $s_(L,7) = "full"_L$, is a prefix of the block. A row that is not last reads all 136 bytes; the last row reads exactly `count` bytes, at most 135. Since `count` starts at the length and falls by 136 per row, this pins the number of rows to $floor(n / 136) + 1$: the KECCAK rows of one proof are at most `SPONGE_MAX_ROWS` $= 2^40$ (the block format's caps are checked against it when the prover is built), so $n - 136 j$ lies in $(-2^48, 2^32)$ and equals a value in $[0, 135]$ in the field only when it does in the integers. (The argument needs only $136 dot N + 2^32 < p$, about $N < 2^56$ rows.) +Bytes past the input are never read and are zero: +#render_constraint_table(sponge, config, groups: "mask") + +The padded block is a linear form of the columns: $m'_(L,b) = "msg"_(L,b) + s_"prev" - s_(L,b)$, where $s_"prev"$ is the mask bit of the byte before ($1$ before byte 0), plus `0x80` on byte 135 of the last row. The term $s_"prev" - s$ is the `0x01` right after the input; on byte 135 of a last block of 135 input bytes the two meet as `0x81`. The permutation input is the previous rate XOR the padded block: +#render_constraint_table(sponge, config, groups: "absorb") + +The permutation itself is the round chip's, as in the permutation mode: +#render_constraint_table(sponge, config, groups: "round") + +The last row writes the digest, the first four lanes of its own output, at $T + 1$: +#render_constraint_table(sponge, config, groups: "digest") + +The block's last byte and the digest's last byte stay below $2^32$ in the low address word, so the offsets above add to the low word without a carry. +The equation on `src_end` (@keccak_sponge:c:src_end) is more than a range check: the link sends $"src_end" + 1$ as the next row's `src`, so it is also what places every block after the first right after the one before; without it a block could be read from any address. +#render_constraint_table(sponge, config, groups: "range") + = Round chip #let round_chip = load_chip("src/keccak_round.toml", config) #let keccak_rnd = raw(round_chip.name) diff --git a/spec/src/blake3.toml b/spec/src/blake3.toml new file mode 100644 index 000000000..94fcee9be --- /dev/null +++ b/spec/src/blake3.toml @@ -0,0 +1,357 @@ +# BLAKE3_6R — the 6-round internal-variant BLAKE3 compression accelerator. +# +# ⚠ NORMATIVE SOURCE NOTE. This spec documents the shipped chip +# (`prover/src/tables/blake3.rs`); the chip's wiring is single-sourced in Rust +# (`run_flow` interpreted as columns and as witness) and formally gated by the +# z3 model in `thoughts/blake3/blake3-chip/z3_blake_verify.py`. Where this file +# and those artifacts disagree, THEY are normative and this file has a bug. +# Cross-checked totals at spec-writing time: 3,219 main columns, 1,397 +# interactions, 814 constraints, max degree 3 (incl. the ×μ gating factor). +# +# ⚠ SCOPE. Everything below describes the SINGLE-COMPRESSION mode +# (`BLAKE3_SYSCALL_NUMBER`), and the totals above are that mode's. The table +# also carries a chained-absorb mode (`BLAKE3_ABSORB_SYSCALL_NUMBER`: one ecall +# folds a run of 64-byte blocks across `num_blocks + 1` rows linked by the +# `Blake3Absorb` bus), which takes the shipped chip to 3,268 main columns, 1,475 +# interactions and 851 constraints at the same max degree 3. That mode is NOT +# described here yet; the Rust is its normative source, and the totals pinned by +# `prover::tables::blake3::absorb_tests::the_tables_shape_is_pinned` are what to +# compare against until this file catches up. +# +# One row = one compression call: 6 rounds × 8 G-functions fully unrolled in +# SSA form. Message schedule = the literal per-round index table `SCHED` below +# (permute^r of the identity under MSG_PERMUTATION = +# [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8]); round r position i consumes +# original message word SCHED[r][i]. +# +# Security: this chip computes 6-round BLAKE3, NOT the standard 7-round +# function. Its use as a Merkle / Fiat–Shamir hash rests on the named +# assumption A6R (see blake3.typ). No external system will ever agree on +# these digests. + +name = "BLAKE3" + +# ------------------------------------------------------------------------- +# Inputs (read from memory at addr .. addr+112; see the ECALL ABI) +# ------------------------------------------------------------------------- + +[[variables.input]] +name = "timestamp" +type = "DWordWL" +desc = "timestamp at which the compression is performed" +pad = 0 + +[[variables.input]] +name = "addr" +type = "DWordBL" +desc = "8-aligned base address of the 176-byte state region (h|m|t|len,flags|out)" +pad = 0 + +[[variables.input]] +name = "h" +type = [["Byte", 4], 8] +desc = "input chaining value h[0..8], 8 little-endian u32 words" +pad = 0 + +[[variables.input]] +name = "m" +type = [["Byte", 4], 16] +desc = "message block m[0..16] = left_cv ‖ right_cv for a 2-to-1 merge" +pad = 0 + +[[variables.input]] +name = "t_lo" +type = ["Byte", 4] +desc = "low u32 of the 64-bit counter t → v[12]" +pad = 0 + +[[variables.input]] +name = "t_hi" +type = ["Byte", 4] +desc = "high u32 of the 64-bit counter t → v[13] (split order is load-bearing)" +pad = 0 + +[[variables.input]] +name = "block_len" +type = ["Byte", 4] +desc = "input byte count of this block → v[14]" +pad = 0 + +[[variables.input]] +name = "flags" +type = ["Byte", 4] +desc = "domain-separation flags → v[15]" +pad = 0 + +# ------------------------------------------------------------------------- +# Outputs (written to memory at addr+112 .. addr+176) +# ------------------------------------------------------------------------- + +[[variables.output]] +name = "out" +type = [["Byte", 4], 16] +desc = "full 16-word compression output; the truncated CV is out[0..8]" +pad = 0 + +# ------------------------------------------------------------------------- +# Auxiliary +# ------------------------------------------------------------------------- + +[[variables.auxiliary]] +name = "state_ptr" +type = ["DWordHL", 22] +desc = "per-dword pointers state_ptr[k] = addr + 8k over the 22-dword region" +pad = ["*", 8, ["arr", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]] + +[[variables.auxiliary]] +name = "old_out" +type = [["Byte", 8], 8] +desc = "previous memory content of the out region, dword o = bytes old_out[o]; the MEMW writes' old value" +pad = 0 + +# Per-G SSA cells, 48 instances (g = 8r + j, round r ∈ 0..6, call j ∈ 0..8). +# Each G block: A1(4B) c1,c2(2 bits) X1(4B) C1(4B) X2(4B) +# R1: SLL_lo(2B) SLLC_lo(2B) SLL_hi(2B) SLLC_hi(2B) Y1(4B) +# A2(4B) c3,c4(2 bits) X3(4B) C2(4B) X4(4B) +# R2: same shape as R1 +# = 56 byte cells + 4 carry bits = 60 cells; column base = 210 + 60g +# (see prover/src/tables/blake3.rs `cols` for the exact offsets). + +[[variables.auxiliary]] +name = "g_add3_out" +type = [[["Byte", 4], 2], 48] +desc = "A1, A2 per G: the two 3-operand add outputs (v[a] after each half)" +pad = 0 + +[[variables.auxiliary]] +name = "g_add3_carry" +type = [[["Bit", 2], 2], 48] +desc = "two summed carry bits per 3-operand add: carry = c1 + c2 ∈ {0,1,2}" +pad = 0 + +[[variables.auxiliary]] +name = "g_xor_out" +type = [[["Byte", 4], 4], 48] +desc = "X1..X4 per G: the four 32-bit XOR outputs (rotr16/rotr8 are free byte relabels of X1/X3)" +pad = 0 + +[[variables.auxiliary]] +name = "g_add2_out" +type = [[["Byte", 4], 2], 48] +desc = "C1, C2 per G: the two 2-operand add outputs (v[c]); carries are expressions, not cells" +pad = 0 + +[[variables.auxiliary]] +name = "g_rot" +type = [[["Byte", 12], 2], 48] +desc = "per rotation (rotr12 then rotr7): SLL_lo, SLLC_lo, SLL_hi, SLLC_hi (2 B each) and the output word Y (4 B)" +pad = 0 + +[[variables.multiplicity]] +name = "μ" +type = "Bit" +desc = "1 on real rows, 0 on padding; gates every constraint and interaction; pinned boolean by an ungated IS_BIT" +pad = 0 + +# ------------------------------------------------------------------------- +# Constants (documentation: no constraint below reads them, and the chip schema has no +# `constants` key, so they stay comments) +# ------------------------------------------------------------------------- + +# [[constants]] +# name = "IV" +# desc = "BLAKE3 IV[0..4] (SHA-256 IV words), inlined into round-0 arithmetic — not columns" +# value = ["arr", 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A] + +# [[constants]] +# name = "SCHED" +# desc = "per-round message-schedule index table: round r position i reads m[SCHED[r][i]]; SCHED[r] = permute^r(identity), MSG_PERMUTATION = [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8]; the trailing permute after round 5 is never consumed" +# value = ["arr", +# ["arr", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], +# ["arr", 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8], +# ["arr", 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1], +# ["arr", 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6], +# ["arr", 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4], +# ["arr", 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7], +# ] + +# [[constants]] +# name = "G_INDICES" +# desc = "the (a,b,c,d) working-state slots of the 8 G-calls per round: 4 column mixes then 4 diagonal mixes" +# value = ["arr", +# ["arr", 0, 4, 8, 12], ["arr", 1, 5, 9, 13], ["arr", 2, 6, 10, 14], ["arr", 3, 7, 11, 15], +# ["arr", 0, 5, 10, 15], ["arr", 1, 6, 11, 12], ["arr", 2, 7, 8, 13], ["arr", 3, 4, 9, 14], +# ] + +# ------------------------------------------------------------------------- +# Constraint groups. All eval constraints are μ-gated; padding rows are +# all-zero (except the keccak-idiom state_ptr pad) with μ = 0. +# ------------------------------------------------------------------------- + +# ------------------------------------------------------------------------- +# Constraint groups. +# +# ⚠ SCOPE. This file machine-formalizes the chip's I/O AND RANGE surface — +# the ECALL binding, the x10 register read, all 22 MEMW dword operations, +# the pointer arithmetic, and every explicit ARE_BYTES/IS_HALF/IS_BIT — i.e. +# exactly the surface the z3 gate does NOT model (DESIGN.md §1.1: "the gate +# cannot check this", §7 items 4/5/10). The unrolled 6-round MIXING CORE +# (96 add3 sum identities + carry booleanities, 96 add2 expression-carry +# booleanities, 96 inline rotation identity groups, 832 BYTE_ALU[XOR] +# lookups wired per the SSA dataflow) is NOT re-formalized here: its +# normative sources are the single-source Rust dataflow +# (prover/src/tables/blake3.rs `run_flow`, interpreted once as columns and +# once as witness) and the z3 gate that proves that dataflow equal to the +# reference function under free inputs. Totals for cross-checking: 814 +# constraints, 1,397 interactions, max degree 3 including ×μ. +# ------------------------------------------------------------------------- + +[[constraint_groups]] +name = "io" + +# ECALL receive: [timestamp, -3 as u64 = 2^64 - 3] +[[constraints.io]] +kind = "interaction" +tag = "ECALL" +input = ["timestamp", ["cast", ["-", ["^", 2, 64], 3], "DWordWL"]] +multiplicity = ["-", "μ"] + +# MEMW register read of x10 binding addr (keccak:c:read_addr idiom) +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [1, ["cast", ["*", 2, 10], "DWordWL"], "addr", "timestamp", 1, 0, 0] +output = "addr" +multiplicity = "μ" +ref = "blake3:c:read_addr" + +# The 22 dword MEMW operations at `timestamp`, per region. Input dwords +# (k < 14) are reads in the combined read+write encoding: old = value = the +# input words, timestamps advance. Output dwords (k >= 14) write `out` over +# `old_out`. + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", "k"], "DWordWL"], ["arr", ["idx", ["idx", "h", ["*", 2, "k"]], 0], ["idx", ["idx", "h", ["*", 2, "k"]], 1], ["idx", ["idx", "h", ["*", 2, "k"]], 2], ["idx", ["idx", "h", ["*", 2, "k"]], 3], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 0], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 1], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 2], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 3]], "timestamp", 0, 0, 1] +output = ["arr", ["idx", ["idx", "h", ["*", 2, "k"]], 0], ["idx", ["idx", "h", ["*", 2, "k"]], 1], ["idx", ["idx", "h", ["*", 2, "k"]], 2], ["idx", ["idx", "h", ["*", 2, "k"]], 3], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 0], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 1], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 2], ["idx", ["idx", "h", ["+", ["*", 2, "k"], 1]], 3]] +iters = [["k", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:load_h" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", ["+", "k", 4]], "DWordWL"], ["arr", ["idx", ["idx", "m", ["*", 2, "k"]], 0], ["idx", ["idx", "m", ["*", 2, "k"]], 1], ["idx", ["idx", "m", ["*", 2, "k"]], 2], ["idx", ["idx", "m", ["*", 2, "k"]], 3], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 0], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 1], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 2], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 3]], "timestamp", 0, 0, 1] +output = ["arr", ["idx", ["idx", "m", ["*", 2, "k"]], 0], ["idx", ["idx", "m", ["*", 2, "k"]], 1], ["idx", ["idx", "m", ["*", 2, "k"]], 2], ["idx", ["idx", "m", ["*", 2, "k"]], 3], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 0], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 1], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 2], ["idx", ["idx", "m", ["+", ["*", 2, "k"], 1]], 3]] +iters = [["k", 0, 7]] +multiplicity = "μ" +ref = "blake3:c:load_m" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", 12], "DWordWL"], ["arr", ["idx", "t_lo", 0], ["idx", "t_lo", 1], ["idx", "t_lo", 2], ["idx", "t_lo", 3], ["idx", "t_hi", 0], ["idx", "t_hi", 1], ["idx", "t_hi", 2], ["idx", "t_hi", 3]], "timestamp", 0, 0, 1] +output = ["arr", ["idx", "t_lo", 0], ["idx", "t_lo", 1], ["idx", "t_lo", 2], ["idx", "t_lo", 3], ["idx", "t_hi", 0], ["idx", "t_hi", 1], ["idx", "t_hi", 2], ["idx", "t_hi", 3]] +multiplicity = "μ" +ref = "blake3:c:load_t" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", 13], "DWordWL"], ["arr", ["idx", "block_len", 0], ["idx", "block_len", 1], ["idx", "block_len", 2], ["idx", "block_len", 3], ["idx", "flags", 0], ["idx", "flags", 1], ["idx", "flags", 2], ["idx", "flags", 3]], "timestamp", 0, 0, 1] +output = ["arr", ["idx", "block_len", 0], ["idx", "block_len", 1], ["idx", "block_len", 2], ["idx", "block_len", 3], ["idx", "flags", 0], ["idx", "flags", 1], ["idx", "flags", 2], ["idx", "flags", 3]] +multiplicity = "μ" +ref = "blake3:c:load_len_flags" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", ["+", "o", 14]], "DWordWL"], ["arr", ["idx", ["idx", "out", ["*", 2, "o"]], 0], ["idx", ["idx", "out", ["*", 2, "o"]], 1], ["idx", ["idx", "out", ["*", 2, "o"]], 2], ["idx", ["idx", "out", ["*", 2, "o"]], 3], ["idx", ["idx", "out", ["+", ["*", 2, "o"], 1]], 0], ["idx", ["idx", "out", ["+", ["*", 2, "o"], 1]], 1], ["idx", ["idx", "out", ["+", ["*", 2, "o"], 1]], 2], ["idx", ["idx", "out", ["+", ["*", 2, "o"], 1]], 3]], "timestamp", 0, 0, 1] +output = ["idx", "old_out", "o"] +iters = [["o", 0, 7]] +multiplicity = "μ" +ref = "blake3:c:store_out" + +[[constraint_groups]] +name = "addr" + +# state_ptr[k] = addr + 8k via the shared ADD template (μ-gated carry pair); +# the top dword k = 21 additionally forbids wraparound (addr + 168 < 2^64). +[[constraints.addr]] +kind = "template" +tag = "ADD" +input = [["cast", "addr", "DWordWL"], ["cast", ["*", 8, "k"], "DWordWL"]] +output = ["cast", ["idx", "state_ptr", "k"], "DWordWL"] +iters = [["k", 0, 21]] +ref = "blake3:c:state_ptr" + +# alignment: addr[0] & 7 = 0 +[[constraints.addr]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "AND"], ["idx", "addr", 0], 7] +output = 0 +multiplicity = "μ" +ref = "blake3:c:alignment" + +# addr byte range checks — without them the addr_lo/addr_hi linear +# combinations alias non-byte encodings (keccak:c:range_addr rationale). +[[constraints.addr]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", "addr", ["*", 2, "i"]], ["idx", "addr", ["+", ["*", 2, "i"], 1]]] +iters = [["i", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_addr" + +[[constraints.addr]] +kind = "interaction" +tag = "IS_HALF" +input = [["idx", ["cast", ["idx", "state_ptr", "k"], "DWordHL"], "hw"]] +iters = [["k", 0, 21], ["hw", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_state_ptr" + +[[constraint_groups]] +name = "range" + +# m is never XOR-consumed — explicit byte checks (DESIGN §4.7 / §7.5). +[[constraints.range]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", ["idx", "m", "i"], ["*", 2, "p"]], ["idx", ["idx", "m", "i"], ["+", ["*", 2, "p"], 1]]] +iters = [["i", 0, 15], ["p", 0, 1]] +multiplicity = "μ" +ref = "blake3:c:range_m" + +# old_out rides only the MEMW bus — same aliasing argument as the address. +[[constraints.range]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", ["idx", "old_out", "o"], ["*", 2, "p"]], ["idx", ["idx", "old_out", "o"], ["+", ["*", 2, "p"], 1]]] +iters = [["o", 0, 7], ["p", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_old_out" + +# The four shift halfwords of each inline rotation (SLL_lo, SLLC_lo, +# SLL_hi, SLLC_hi — bytes 0..8 of each rotation block). The tight SLL +# bounds are load-bearing for the rotation identities (DESIGN §4.2). +[[constraints.range]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", ["idx", ["idx", "g_rot", "g"], "half"], ["*", 2, "p"]], ["idx", ["idx", ["idx", "g_rot", "g"], "half"], ["+", ["*", 2, "p"], 1]]] +iters = [["g", 0, 47], ["half", 0, 1], ["p", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_rot" + +[[constraint_groups]] +name = "mu" + +# μ boolean, ungated — the ECALL receive anchors μ>0 rows to a CPU ecall +# whose ECALL flag is boolean; this constraint makes the argument local. +[[constraints.mu]] +kind = "template" +tag = "IS_BIT" +input = ["μ"] +ref = "blake3:c:range_mu" diff --git a/spec/src/keccak.toml b/spec/src/keccak.toml index 1c3bade44..9eb9c5f45 100644 --- a/spec/src/keccak.toml +++ b/spec/src/keccak.toml @@ -36,12 +36,33 @@ pad = ["*", 8, ["arr", ["arr", 20, 21, 22, 23, 24] ]] +[[variables.multiplicity]] +name = "μ_p" +type = "Bit" +desc = "permutation row (the table's `MU_P`)" +pad = 0 + +[[variables.multiplicity]] +name = "μ_s" +type = "Bit" +desc = "sponge row (the table's `MU_S`; its constraints are in keccak_sponge.toml)" +pad = 0 + [[variables.multiplicity]] name = "μ" type = "Bit" -desc = "" +desc = "real row of either mode (the table's `MU`, read by the block cuts)" pad = 0 +[[constraint_groups]] +name = "mode" + +[[constraints.mode]] +kind = "arith" +constraint = "`μ` is the sum of the mode bits, so a row is in at most one mode" +poly = ["-", "μ", ["+", "μ_p", "μ_s"]] +ref = "keccak:c:mode_sum" + [[constraint_groups]] name = "output" @@ -49,7 +70,7 @@ name = "output" kind = "interaction" tag = "ECALL" input = ["timestamp", ["cast", ["-", ["^", 2, 64], 2], "DWordWL"]] -multiplicity = ["-", "μ"] +multiplicity = ["-", "μ_p"] [[constraint_groups]] name = "mem" @@ -59,7 +80,7 @@ kind = "interaction" tag = "MEMW" input = [1, ["cast", ["*", 2, 10], "DWordWL"], "addr", "timestamp", 1, 0, 0] output = "addr" -multiplicity = "μ" +multiplicity = "μ_p" ref = "keccak:c:read_addr" [[constraints.mem]] @@ -75,7 +96,7 @@ kind = "interaction" tag = "IS_HALF" input = [["idx", ["idx", ["idx", "state_ptr", "x"], "y"], "z"]] iters = [["x", 0, 4], ["y", 0, 4], ["z", 0, 3]] -multiplicity = "μ" +multiplicity = "μ_p" [[constraints.mem]] kind = "interaction" @@ -83,7 +104,7 @@ tag = "MEMW" input = [0, ["cast", ["idx", ["idx", "state_ptr", "x"], "y"], "DWordWL"], ["idx", ["idx", "output_state", "x"], "y"], "timestamp", 0, 0, 1] output = ["idx", ["idx", "input_state", "x"], "y"] iters = [["x", 0, 4], ["y", 0, 4]] -multiplicity = "μ" +multiplicity = "μ_p" ref = "keccak:c:load_store_state" [[constraint_groups]] @@ -93,10 +114,10 @@ name = "round" kind = "interaction" tag = "KECCAK" input = ["timestamp", 0, "input_state"] -multiplicity = "μ" +multiplicity = ["+", "μ_p", "μ_s"] [[constraints.round]] kind = "interaction" tag = "KECCAK" input = ["timestamp", 24, "output_state"] -multiplicity = ["-", "μ"] +multiplicity = ["-", ["+", "μ_p", "μ_s"]] diff --git a/spec/src/keccak_sponge.toml b/spec/src/keccak_sponge.toml new file mode 100644 index 000000000..a38b14485 --- /dev/null +++ b/spec/src/keccak_sponge.toml @@ -0,0 +1,416 @@ +name = "KECCAK_SPONGE" + +# The KECCAK table's second mode: one row per permutation of a `keccak256` call +# (ECALL -5). The table holds both modes; this file lists the columns and the +# constraints of a sponge row (`μ_s = 1`). The mode bits and how they gate each +# mode are in keccak.typ. + +[[variables.input]] +name = "key" +type = "DWordWL" +desc = "`(T, j)`: the call's timestamp `T` and the block index `j` (the table's `timestamp` columns)" +pad = 0 + +[[variables.input]] +name = "input_state" +type = [[["Byte", 8], 5], 5] +desc = "state the permutation starts from: the previous output with the padded block XORed into its rate" +pad = 0 + +[[variables.output]] +name = "output_state" +type = [[["Byte", 8], 5], 5] +desc = "state after the permutation" +pad = 0 + +[[variables.auxiliary]] +name = "count" +type = "BaseField" +desc = "input bytes left before this row's block (`len − 136·j`)" +pad = 0 + +[[variables.auxiliary]] +name = "src" +type = "DWordWL" +desc = "address of this row's block (`ptr + 136·j`)" +pad = 0 + +[[variables.auxiliary]] +name = "out" +type = "DWordWL" +desc = "address the digest is written to" +pad = 0 + +[[variables.auxiliary]] +name = "src_end" +type = ["Half", 2] +desc = "the low word of `src` plus 135, as two halves" +pad = 0 + +[[variables.auxiliary]] +name = "out_end" +type = ["Half", 2] +desc = "the low word of `out` plus 31, as two halves" +pad = 0 + +[[variables.auxiliary]] +name = "prev" +type = [["Byte", 8], 17] +desc = "rate lanes of the previous permutation's output (zero on the first row)" +pad = 0 + +[[variables.auxiliary]] +name = "msg" +type = [["Byte", 8], 17] +desc = "the block's bytes as read; zero past the input" +pad = 0 + +[[variables.auxiliary]] +name = "full" +type = ["Bit", 17] +desc = "lane `L` of the block is input and read as one 8-byte access" +pad = 0 + +[[variables.auxiliary]] +name = "part" +type = [["Bit", 7], 17] +desc = "byte `b < 7` of lane `L` is input and read alone" +pad = 0 + +[[variables.auxiliary]] +name = "first" +type = "Bit" +desc = "first row of a call" +pad = 0 + +[[variables.auxiliary]] +name = "last" +type = "Bit" +desc = "last row of a call" +pad = 0 + +[[variables.multiplicity]] +name = "μ_s" +type = "Bit" +desc = "sponge row (the table's `MU_S`)" +pad = 0 + +# Constraints + +[[constraint_groups]] +name = "call" + +[[constraints.call]] +kind = "interaction" +tag = "ECALL" +input = [["arr", ["idx", "key", 0], 0], ["cast", ["-", ["^", 2, 64], 5], "DWordWL"]] +multiplicity = ["-", "first"] +ref = "keccak_sponge:c:ecall" + +[[constraints.call]] +kind = "interaction" +tag = "MEMW" +input = [1, ["cast", ["*", 2, 10], "DWordWL"], ["arr", ["idx", "src", 0], ["idx", "src", 1], 0, 0, 0, 0, 0, 0], ["arr", ["idx", "key", 0], 0], 1, 0, 0] +output = ["arr", ["idx", "src", 0], ["idx", "src", 1], 0, 0, 0, 0, 0, 0] +multiplicity = "first" +ref = "keccak_sponge:c:read_ptr" + +[[constraints.call]] +kind = "interaction" +tag = "MEMW" +input = [1, ["cast", ["*", 2, 11], "DWordWL"], ["arr", "count", 0, 0, 0, 0, 0, 0, 0], ["arr", ["idx", "key", 0], 0], 1, 0, 0] +output = ["arr", "count", 0, 0, 0, 0, 0, 0, 0] +multiplicity = "first" +ref = "keccak_sponge:c:read_len" + +[[constraints.call]] +kind = "interaction" +tag = "MEMW" +input = [1, ["cast", ["*", 2, 12], "DWordWL"], ["arr", ["idx", "out", 0], ["idx", "out", 1], 0, 0, 0, 0, 0, 0], ["arr", ["idx", "key", 0], 0], 1, 0, 0] +output = ["arr", ["idx", "out", 0], ["idx", "out", 1], 0, 0, 0, 0, 0, 0] +multiplicity = "first" +ref = "keccak_sponge:c:read_out" + +[[constraints.call]] +kind = "arith" +constraint = "the first row is block 0" +poly = ["*", "first", ["idx", "key", 1]] +ref = "keccak_sponge:c:first_block" + +[[constraints.call]] +kind = "arith" +constraint = "the first row starts from the zero state: `prev` is zero" +poly = ["*", "first", ["idx", ["idx", "prev", "L"], "b"]] +iters = [["L", 0, 16], ["b", 0, 7]] +ref = "keccak_sponge:c:first_prev" +[[constraints.call]] +kind = "arith" +constraint = "the first row starts from the zero state: capacity lanes of row 3" +poly = ["*", "first", ["idx", ["idx", ["idx", "input_state", "x"], 3], "b"]] +iters = [["x", 2, 4], ["b", 0, 7]] +ref = "keccak_sponge:c:first_capacity_3" + +[[constraints.call]] +kind = "arith" +constraint = "the first row starts from the zero state: capacity lanes of row 4" +poly = ["*", "first", ["idx", ["idx", ["idx", "input_state", "x"], 4], "b"]] +iters = [["x", 0, 4], ["b", 0, 7]] +ref = "keccak_sponge:c:first_capacity_4" + +[[constraints.call]] +kind = "arith" +constraint = "`first` and `last` only on sponge rows" +poly = ["*", ["+", "first", "last"], ["not", "μ_s"]] +ref = "keccak_sponge:c:flags_on_sponge_rows" + +[[constraint_groups]] +name = "chain" +[[constraints.chain]] +kind = "interaction" +tag = "KECCAK_SPONGE" +input = [["arr", ["idx", "key", 0], ["+", ["idx", "key", 1], 1]], ["-", "count", 136], ["arr", ["+", ["idx", "src_end", 0], ["*", ["^", 2, 16], ["idx", "src_end", 1]], 1], ["idx", "src", 1]], "out", ["arr", ["arr", ["arr", ["idx", ["idx", ["idx", "output_state", 0], 0], 0], ["idx", ["idx", ["idx", "output_state", 0], 0], 1], ["idx", ["idx", ["idx", "output_state", 0], 0], 2], ["idx", ["idx", ["idx", "output_state", 0], 0], 3], ["idx", ["idx", ["idx", "output_state", 0], 0], 4], ["idx", ["idx", ["idx", "output_state", 0], 0], 5], ["idx", ["idx", ["idx", "output_state", 0], 0], 6], ["idx", ["idx", ["idx", "output_state", 0], 0], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 0], 1], 0], ["idx", ["idx", ["idx", "output_state", 0], 1], 1], ["idx", ["idx", ["idx", "output_state", 0], 1], 2], ["idx", ["idx", ["idx", "output_state", 0], 1], 3], ["idx", ["idx", ["idx", "output_state", 0], 1], 4], ["idx", ["idx", ["idx", "output_state", 0], 1], 5], ["idx", ["idx", ["idx", "output_state", 0], 1], 6], ["idx", ["idx", ["idx", "output_state", 0], 1], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 0], 2], 0], ["idx", ["idx", ["idx", "output_state", 0], 2], 1], ["idx", ["idx", ["idx", "output_state", 0], 2], 2], ["idx", ["idx", ["idx", "output_state", 0], 2], 3], ["idx", ["idx", ["idx", "output_state", 0], 2], 4], ["idx", ["idx", ["idx", "output_state", 0], 2], 5], ["idx", ["idx", ["idx", "output_state", 0], 2], 6], ["idx", ["idx", ["idx", "output_state", 0], 2], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 0], 3], 0], ["idx", ["idx", ["idx", "output_state", 0], 3], 1], ["idx", ["idx", ["idx", "output_state", 0], 3], 2], ["idx", ["idx", ["idx", "output_state", 0], 3], 3], ["idx", ["idx", ["idx", "output_state", 0], 3], 4], ["idx", ["idx", ["idx", "output_state", 0], 3], 5], ["idx", ["idx", ["idx", "output_state", 0], 3], 6], ["idx", ["idx", ["idx", "output_state", 0], 3], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 0], 4], 0], ["idx", ["idx", ["idx", "output_state", 0], 4], 1], ["idx", ["idx", ["idx", "output_state", 0], 4], 2], ["idx", ["idx", ["idx", "output_state", 0], 4], 3], ["idx", ["idx", ["idx", "output_state", 0], 4], 4], ["idx", ["idx", ["idx", "output_state", 0], 4], 5], ["idx", ["idx", ["idx", "output_state", 0], 4], 6], ["idx", ["idx", ["idx", "output_state", 0], 4], 7]]], ["arr", ["arr", ["idx", ["idx", ["idx", "output_state", 1], 0], 0], ["idx", ["idx", ["idx", "output_state", 1], 0], 1], ["idx", ["idx", ["idx", "output_state", 1], 0], 2], ["idx", ["idx", ["idx", "output_state", 1], 0], 3], ["idx", ["idx", ["idx", "output_state", 1], 0], 4], ["idx", ["idx", ["idx", "output_state", 1], 0], 5], ["idx", ["idx", ["idx", "output_state", 1], 0], 6], ["idx", ["idx", ["idx", "output_state", 1], 0], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 1], 1], 0], ["idx", ["idx", ["idx", "output_state", 1], 1], 1], ["idx", ["idx", ["idx", "output_state", 1], 1], 2], ["idx", ["idx", ["idx", "output_state", 1], 1], 3], ["idx", ["idx", ["idx", "output_state", 1], 1], 4], ["idx", ["idx", ["idx", "output_state", 1], 1], 5], ["idx", ["idx", ["idx", "output_state", 1], 1], 6], ["idx", ["idx", ["idx", "output_state", 1], 1], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 1], 2], 0], ["idx", ["idx", ["idx", "output_state", 1], 2], 1], ["idx", ["idx", ["idx", "output_state", 1], 2], 2], ["idx", ["idx", ["idx", "output_state", 1], 2], 3], ["idx", ["idx", ["idx", "output_state", 1], 2], 4], ["idx", ["idx", ["idx", "output_state", 1], 2], 5], ["idx", ["idx", ["idx", "output_state", 1], 2], 6], ["idx", ["idx", ["idx", "output_state", 1], 2], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 1], 3], 0], ["idx", ["idx", ["idx", "output_state", 1], 3], 1], ["idx", ["idx", ["idx", "output_state", 1], 3], 2], ["idx", ["idx", ["idx", "output_state", 1], 3], 3], ["idx", ["idx", ["idx", "output_state", 1], 3], 4], ["idx", ["idx", ["idx", "output_state", 1], 3], 5], ["idx", ["idx", ["idx", "output_state", 1], 3], 6], ["idx", ["idx", ["idx", "output_state", 1], 3], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 1], 4], 0], ["idx", ["idx", ["idx", "output_state", 1], 4], 1], ["idx", ["idx", ["idx", "output_state", 1], 4], 2], ["idx", ["idx", ["idx", "output_state", 1], 4], 3], ["idx", ["idx", ["idx", "output_state", 1], 4], 4], ["idx", ["idx", ["idx", "output_state", 1], 4], 5], ["idx", ["idx", ["idx", "output_state", 1], 4], 6], ["idx", ["idx", ["idx", "output_state", 1], 4], 7]]], ["arr", ["arr", ["idx", ["idx", ["idx", "output_state", 2], 0], 0], ["idx", ["idx", ["idx", "output_state", 2], 0], 1], ["idx", ["idx", ["idx", "output_state", 2], 0], 2], ["idx", ["idx", ["idx", "output_state", 2], 0], 3], ["idx", ["idx", ["idx", "output_state", 2], 0], 4], ["idx", ["idx", ["idx", "output_state", 2], 0], 5], ["idx", ["idx", ["idx", "output_state", 2], 0], 6], ["idx", ["idx", ["idx", "output_state", 2], 0], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 2], 1], 0], ["idx", ["idx", ["idx", "output_state", 2], 1], 1], ["idx", ["idx", ["idx", "output_state", 2], 1], 2], ["idx", ["idx", ["idx", "output_state", 2], 1], 3], ["idx", ["idx", ["idx", "output_state", 2], 1], 4], ["idx", ["idx", ["idx", "output_state", 2], 1], 5], ["idx", ["idx", ["idx", "output_state", 2], 1], 6], ["idx", ["idx", ["idx", "output_state", 2], 1], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 2], 2], 0], ["idx", ["idx", ["idx", "output_state", 2], 2], 1], ["idx", ["idx", ["idx", "output_state", 2], 2], 2], ["idx", ["idx", ["idx", "output_state", 2], 2], 3], ["idx", ["idx", ["idx", "output_state", 2], 2], 4], ["idx", ["idx", ["idx", "output_state", 2], 2], 5], ["idx", ["idx", ["idx", "output_state", 2], 2], 6], ["idx", ["idx", ["idx", "output_state", 2], 2], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 2], 3], 0], ["idx", ["idx", ["idx", "output_state", 2], 3], 1], ["idx", ["idx", ["idx", "output_state", 2], 3], 2], ["idx", ["idx", ["idx", "output_state", 2], 3], 3], ["idx", ["idx", ["idx", "output_state", 2], 3], 4], ["idx", ["idx", ["idx", "output_state", 2], 3], 5], ["idx", ["idx", ["idx", "output_state", 2], 3], 6], ["idx", ["idx", ["idx", "output_state", 2], 3], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 2], 4], 0], ["idx", ["idx", ["idx", "output_state", 2], 4], 1], ["idx", ["idx", ["idx", "output_state", 2], 4], 2], ["idx", ["idx", ["idx", "output_state", 2], 4], 3], ["idx", ["idx", ["idx", "output_state", 2], 4], 4], ["idx", ["idx", ["idx", "output_state", 2], 4], 5], ["idx", ["idx", ["idx", "output_state", 2], 4], 6], ["idx", ["idx", ["idx", "output_state", 2], 4], 7]]], ["arr", ["arr", ["idx", ["idx", ["idx", "output_state", 3], 0], 0], ["idx", ["idx", ["idx", "output_state", 3], 0], 1], ["idx", ["idx", ["idx", "output_state", 3], 0], 2], ["idx", ["idx", ["idx", "output_state", 3], 0], 3], ["idx", ["idx", ["idx", "output_state", 3], 0], 4], ["idx", ["idx", ["idx", "output_state", 3], 0], 5], ["idx", ["idx", ["idx", "output_state", 3], 0], 6], ["idx", ["idx", ["idx", "output_state", 3], 0], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 3], 1], 0], ["idx", ["idx", ["idx", "output_state", 3], 1], 1], ["idx", ["idx", ["idx", "output_state", 3], 1], 2], ["idx", ["idx", ["idx", "output_state", 3], 1], 3], ["idx", ["idx", ["idx", "output_state", 3], 1], 4], ["idx", ["idx", ["idx", "output_state", 3], 1], 5], ["idx", ["idx", ["idx", "output_state", 3], 1], 6], ["idx", ["idx", ["idx", "output_state", 3], 1], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 3], 2], 0], ["idx", ["idx", ["idx", "output_state", 3], 2], 1], ["idx", ["idx", ["idx", "output_state", 3], 2], 2], ["idx", ["idx", ["idx", "output_state", 3], 2], 3], ["idx", ["idx", ["idx", "output_state", 3], 2], 4], ["idx", ["idx", ["idx", "output_state", 3], 2], 5], ["idx", ["idx", ["idx", "output_state", 3], 2], 6], ["idx", ["idx", ["idx", "output_state", 3], 2], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 3], 3], 0], ["idx", ["idx", ["idx", "output_state", 3], 3], 1], ["idx", ["idx", ["idx", "output_state", 3], 3], 2], ["idx", ["idx", ["idx", "output_state", 3], 3], 3], ["idx", ["idx", ["idx", "output_state", 3], 3], 4], ["idx", ["idx", ["idx", "output_state", 3], 3], 5], ["idx", ["idx", ["idx", "output_state", 3], 3], 6], ["idx", ["idx", ["idx", "output_state", 3], 3], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 3], 4], 0], ["idx", ["idx", ["idx", "output_state", 3], 4], 1], ["idx", ["idx", ["idx", "output_state", 3], 4], 2], ["idx", ["idx", ["idx", "output_state", 3], 4], 3], ["idx", ["idx", ["idx", "output_state", 3], 4], 4], ["idx", ["idx", ["idx", "output_state", 3], 4], 5], ["idx", ["idx", ["idx", "output_state", 3], 4], 6], ["idx", ["idx", ["idx", "output_state", 3], 4], 7]]], ["arr", ["arr", ["idx", ["idx", ["idx", "output_state", 4], 0], 0], ["idx", ["idx", ["idx", "output_state", 4], 0], 1], ["idx", ["idx", ["idx", "output_state", 4], 0], 2], ["idx", ["idx", ["idx", "output_state", 4], 0], 3], ["idx", ["idx", ["idx", "output_state", 4], 0], 4], ["idx", ["idx", ["idx", "output_state", 4], 0], 5], ["idx", ["idx", ["idx", "output_state", 4], 0], 6], ["idx", ["idx", ["idx", "output_state", 4], 0], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 4], 1], 0], ["idx", ["idx", ["idx", "output_state", 4], 1], 1], ["idx", ["idx", ["idx", "output_state", 4], 1], 2], ["idx", ["idx", ["idx", "output_state", 4], 1], 3], ["idx", ["idx", ["idx", "output_state", 4], 1], 4], ["idx", ["idx", ["idx", "output_state", 4], 1], 5], ["idx", ["idx", ["idx", "output_state", 4], 1], 6], ["idx", ["idx", ["idx", "output_state", 4], 1], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 4], 2], 0], ["idx", ["idx", ["idx", "output_state", 4], 2], 1], ["idx", ["idx", ["idx", "output_state", 4], 2], 2], ["idx", ["idx", ["idx", "output_state", 4], 2], 3], ["idx", ["idx", ["idx", "output_state", 4], 2], 4], ["idx", ["idx", ["idx", "output_state", 4], 2], 5], ["idx", ["idx", ["idx", "output_state", 4], 2], 6], ["idx", ["idx", ["idx", "output_state", 4], 2], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 4], 3], 0], ["idx", ["idx", ["idx", "output_state", 4], 3], 1], ["idx", ["idx", ["idx", "output_state", 4], 3], 2], ["idx", ["idx", ["idx", "output_state", 4], 3], 3], ["idx", ["idx", ["idx", "output_state", 4], 3], 4], ["idx", ["idx", ["idx", "output_state", 4], 3], 5], ["idx", ["idx", ["idx", "output_state", 4], 3], 6], ["idx", ["idx", ["idx", "output_state", 4], 3], 7]], ["arr", ["idx", ["idx", ["idx", "output_state", 4], 4], 0], ["idx", ["idx", ["idx", "output_state", 4], 4], 1], ["idx", ["idx", ["idx", "output_state", 4], 4], 2], ["idx", ["idx", ["idx", "output_state", 4], 4], 3], ["idx", ["idx", ["idx", "output_state", 4], 4], 4], ["idx", ["idx", ["idx", "output_state", 4], 4], 5], ["idx", ["idx", ["idx", "output_state", 4], 4], 6], ["idx", ["idx", ["idx", "output_state", 4], 4], 7]]]]] +multiplicity = ["-", "μ_s", "last"] +ref = "keccak_sponge:c:send_next" + +[[constraints.chain]] +kind = "interaction" +tag = "KECCAK_SPONGE" +input = ["key", "count", "src", "out", ["arr", ["arr", ["arr", ["idx", ["idx", "prev", 0], 0], ["idx", ["idx", "prev", 0], 1], ["idx", ["idx", "prev", 0], 2], ["idx", ["idx", "prev", 0], 3], ["idx", ["idx", "prev", 0], 4], ["idx", ["idx", "prev", 0], 5], ["idx", ["idx", "prev", 0], 6], ["idx", ["idx", "prev", 0], 7]], ["arr", ["idx", ["idx", "prev", 5], 0], ["idx", ["idx", "prev", 5], 1], ["idx", ["idx", "prev", 5], 2], ["idx", ["idx", "prev", 5], 3], ["idx", ["idx", "prev", 5], 4], ["idx", ["idx", "prev", 5], 5], ["idx", ["idx", "prev", 5], 6], ["idx", ["idx", "prev", 5], 7]], ["arr", ["idx", ["idx", "prev", 10], 0], ["idx", ["idx", "prev", 10], 1], ["idx", ["idx", "prev", 10], 2], ["idx", ["idx", "prev", 10], 3], ["idx", ["idx", "prev", 10], 4], ["idx", ["idx", "prev", 10], 5], ["idx", ["idx", "prev", 10], 6], ["idx", ["idx", "prev", 10], 7]], ["arr", ["idx", ["idx", "prev", 15], 0], ["idx", ["idx", "prev", 15], 1], ["idx", ["idx", "prev", 15], 2], ["idx", ["idx", "prev", 15], 3], ["idx", ["idx", "prev", 15], 4], ["idx", ["idx", "prev", 15], 5], ["idx", ["idx", "prev", 15], 6], ["idx", ["idx", "prev", 15], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 0], 4], 0], ["idx", ["idx", ["idx", "input_state", 0], 4], 1], ["idx", ["idx", ["idx", "input_state", 0], 4], 2], ["idx", ["idx", ["idx", "input_state", 0], 4], 3], ["idx", ["idx", ["idx", "input_state", 0], 4], 4], ["idx", ["idx", ["idx", "input_state", 0], 4], 5], ["idx", ["idx", ["idx", "input_state", 0], 4], 6], ["idx", ["idx", ["idx", "input_state", 0], 4], 7]]], ["arr", ["arr", ["idx", ["idx", "prev", 1], 0], ["idx", ["idx", "prev", 1], 1], ["idx", ["idx", "prev", 1], 2], ["idx", ["idx", "prev", 1], 3], ["idx", ["idx", "prev", 1], 4], ["idx", ["idx", "prev", 1], 5], ["idx", ["idx", "prev", 1], 6], ["idx", ["idx", "prev", 1], 7]], ["arr", ["idx", ["idx", "prev", 6], 0], ["idx", ["idx", "prev", 6], 1], ["idx", ["idx", "prev", 6], 2], ["idx", ["idx", "prev", 6], 3], ["idx", ["idx", "prev", 6], 4], ["idx", ["idx", "prev", 6], 5], ["idx", ["idx", "prev", 6], 6], ["idx", ["idx", "prev", 6], 7]], ["arr", ["idx", ["idx", "prev", 11], 0], ["idx", ["idx", "prev", 11], 1], ["idx", ["idx", "prev", 11], 2], ["idx", ["idx", "prev", 11], 3], ["idx", ["idx", "prev", 11], 4], ["idx", ["idx", "prev", 11], 5], ["idx", ["idx", "prev", 11], 6], ["idx", ["idx", "prev", 11], 7]], ["arr", ["idx", ["idx", "prev", 16], 0], ["idx", ["idx", "prev", 16], 1], ["idx", ["idx", "prev", 16], 2], ["idx", ["idx", "prev", 16], 3], ["idx", ["idx", "prev", 16], 4], ["idx", ["idx", "prev", 16], 5], ["idx", ["idx", "prev", 16], 6], ["idx", ["idx", "prev", 16], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 1], 4], 0], ["idx", ["idx", ["idx", "input_state", 1], 4], 1], ["idx", ["idx", ["idx", "input_state", 1], 4], 2], ["idx", ["idx", ["idx", "input_state", 1], 4], 3], ["idx", ["idx", ["idx", "input_state", 1], 4], 4], ["idx", ["idx", ["idx", "input_state", 1], 4], 5], ["idx", ["idx", ["idx", "input_state", 1], 4], 6], ["idx", ["idx", ["idx", "input_state", 1], 4], 7]]], ["arr", ["arr", ["idx", ["idx", "prev", 2], 0], ["idx", ["idx", "prev", 2], 1], ["idx", ["idx", "prev", 2], 2], ["idx", ["idx", "prev", 2], 3], ["idx", ["idx", "prev", 2], 4], ["idx", ["idx", "prev", 2], 5], ["idx", ["idx", "prev", 2], 6], ["idx", ["idx", "prev", 2], 7]], ["arr", ["idx", ["idx", "prev", 7], 0], ["idx", ["idx", "prev", 7], 1], ["idx", ["idx", "prev", 7], 2], ["idx", ["idx", "prev", 7], 3], ["idx", ["idx", "prev", 7], 4], ["idx", ["idx", "prev", 7], 5], ["idx", ["idx", "prev", 7], 6], ["idx", ["idx", "prev", 7], 7]], ["arr", ["idx", ["idx", "prev", 12], 0], ["idx", ["idx", "prev", 12], 1], ["idx", ["idx", "prev", 12], 2], ["idx", ["idx", "prev", 12], 3], ["idx", ["idx", "prev", 12], 4], ["idx", ["idx", "prev", 12], 5], ["idx", ["idx", "prev", 12], 6], ["idx", ["idx", "prev", 12], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 2], 3], 0], ["idx", ["idx", ["idx", "input_state", 2], 3], 1], ["idx", ["idx", ["idx", "input_state", 2], 3], 2], ["idx", ["idx", ["idx", "input_state", 2], 3], 3], ["idx", ["idx", ["idx", "input_state", 2], 3], 4], ["idx", ["idx", ["idx", "input_state", 2], 3], 5], ["idx", ["idx", ["idx", "input_state", 2], 3], 6], ["idx", ["idx", ["idx", "input_state", 2], 3], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 2], 4], 0], ["idx", ["idx", ["idx", "input_state", 2], 4], 1], ["idx", ["idx", ["idx", "input_state", 2], 4], 2], ["idx", ["idx", ["idx", "input_state", 2], 4], 3], ["idx", ["idx", ["idx", "input_state", 2], 4], 4], ["idx", ["idx", ["idx", "input_state", 2], 4], 5], ["idx", ["idx", ["idx", "input_state", 2], 4], 6], ["idx", ["idx", ["idx", "input_state", 2], 4], 7]]], ["arr", ["arr", ["idx", ["idx", "prev", 3], 0], ["idx", ["idx", "prev", 3], 1], ["idx", ["idx", "prev", 3], 2], ["idx", ["idx", "prev", 3], 3], ["idx", ["idx", "prev", 3], 4], ["idx", ["idx", "prev", 3], 5], ["idx", ["idx", "prev", 3], 6], ["idx", ["idx", "prev", 3], 7]], ["arr", ["idx", ["idx", "prev", 8], 0], ["idx", ["idx", "prev", 8], 1], ["idx", ["idx", "prev", 8], 2], ["idx", ["idx", "prev", 8], 3], ["idx", ["idx", "prev", 8], 4], ["idx", ["idx", "prev", 8], 5], ["idx", ["idx", "prev", 8], 6], ["idx", ["idx", "prev", 8], 7]], ["arr", ["idx", ["idx", "prev", 13], 0], ["idx", ["idx", "prev", 13], 1], ["idx", ["idx", "prev", 13], 2], ["idx", ["idx", "prev", 13], 3], ["idx", ["idx", "prev", 13], 4], ["idx", ["idx", "prev", 13], 5], ["idx", ["idx", "prev", 13], 6], ["idx", ["idx", "prev", 13], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 3], 3], 0], ["idx", ["idx", ["idx", "input_state", 3], 3], 1], ["idx", ["idx", ["idx", "input_state", 3], 3], 2], ["idx", ["idx", ["idx", "input_state", 3], 3], 3], ["idx", ["idx", ["idx", "input_state", 3], 3], 4], ["idx", ["idx", ["idx", "input_state", 3], 3], 5], ["idx", ["idx", ["idx", "input_state", 3], 3], 6], ["idx", ["idx", ["idx", "input_state", 3], 3], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 3], 4], 0], ["idx", ["idx", ["idx", "input_state", 3], 4], 1], ["idx", ["idx", ["idx", "input_state", 3], 4], 2], ["idx", ["idx", ["idx", "input_state", 3], 4], 3], ["idx", ["idx", ["idx", "input_state", 3], 4], 4], ["idx", ["idx", ["idx", "input_state", 3], 4], 5], ["idx", ["idx", ["idx", "input_state", 3], 4], 6], ["idx", ["idx", ["idx", "input_state", 3], 4], 7]]], ["arr", ["arr", ["idx", ["idx", "prev", 4], 0], ["idx", ["idx", "prev", 4], 1], ["idx", ["idx", "prev", 4], 2], ["idx", ["idx", "prev", 4], 3], ["idx", ["idx", "prev", 4], 4], ["idx", ["idx", "prev", 4], 5], ["idx", ["idx", "prev", 4], 6], ["idx", ["idx", "prev", 4], 7]], ["arr", ["idx", ["idx", "prev", 9], 0], ["idx", ["idx", "prev", 9], 1], ["idx", ["idx", "prev", 9], 2], ["idx", ["idx", "prev", 9], 3], ["idx", ["idx", "prev", 9], 4], ["idx", ["idx", "prev", 9], 5], ["idx", ["idx", "prev", 9], 6], ["idx", ["idx", "prev", 9], 7]], ["arr", ["idx", ["idx", "prev", 14], 0], ["idx", ["idx", "prev", 14], 1], ["idx", ["idx", "prev", 14], 2], ["idx", ["idx", "prev", 14], 3], ["idx", ["idx", "prev", 14], 4], ["idx", ["idx", "prev", 14], 5], ["idx", ["idx", "prev", 14], 6], ["idx", ["idx", "prev", 14], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 4], 3], 0], ["idx", ["idx", ["idx", "input_state", 4], 3], 1], ["idx", ["idx", ["idx", "input_state", 4], 3], 2], ["idx", ["idx", ["idx", "input_state", 4], 3], 3], ["idx", ["idx", ["idx", "input_state", 4], 3], 4], ["idx", ["idx", ["idx", "input_state", 4], 3], 5], ["idx", ["idx", ["idx", "input_state", 4], 3], 6], ["idx", ["idx", ["idx", "input_state", 4], 3], 7]], ["arr", ["idx", ["idx", ["idx", "input_state", 4], 4], 0], ["idx", ["idx", ["idx", "input_state", 4], 4], 1], ["idx", ["idx", ["idx", "input_state", 4], 4], 2], ["idx", ["idx", ["idx", "input_state", 4], 4], 3], ["idx", ["idx", ["idx", "input_state", 4], 4], 4], ["idx", ["idx", ["idx", "input_state", 4], 4], 5], ["idx", ["idx", ["idx", "input_state", 4], 4], 6], ["idx", ["idx", ["idx", "input_state", 4], 4], 7]]]]] +multiplicity = ["-", ["-", "μ_s", "first"]] +ref = "keccak_sponge:c:receive_previous" + +[[constraint_groups]] +name = "read" + +[[constraints.read]] +kind = "interaction" +tag = "MEMW" +input = [0, ["arr", ["+", ["idx", "src", 0], ["*", 8, "L"]], ["idx", "src", 1]], ["idx", "msg", "L"], ["arr", ["idx", "key", 0], 0], 0, 0, 1] +output = ["idx", "msg", "L"] +iter = ["L", 0, 16] +multiplicity = ["idx", "full", "L"] +ref = "keccak_sponge:c:read_lane" + +[[constraints.read]] +kind = "interaction" +tag = "MEMW" +input = [0, ["arr", ["+", ["idx", "src", 0], ["*", 8, "L"], "b"], ["idx", "src", 1]], ["arr", ["idx", ["idx", "msg", "L"], "b"], 0, 0, 0, 0, 0, 0, 0], ["arr", ["idx", "key", 0], 0], 0, 0, 0] +output = ["arr", ["idx", ["idx", "msg", "L"], "b"], 0, 0, 0, 0, 0, 0, 0] +iters = [["L", 0, 16], ["b", 0, 6]] +multiplicity = ["idx", ["idx", "part", "L"], "b"] +ref = "keccak_sponge:c:read_byte" + +[[constraint_groups]] +name = "mask" + +[[constraints.mask]] +kind = "template" +tag = "IS_BIT" +input = [["idx", "full", "L"]] +iter = ["L", 0, 16] +ref = "keccak_sponge:c:range_full" + +[[constraints.mask]] +kind = "template" +tag = "IS_BIT" +input = [["idx", ["idx", "part", "L"], "b"]] +iters = [["L", 0, 16], ["b", 0, 6]] +ref = "keccak_sponge:c:range_part" + +[[constraints.mask]] +kind = "arith" +constraint = "each byte is read at most once: `full` and `part` exclude each other" +poly = ["*", ["idx", "full", "L"], ["idx", ["idx", "part", "L"], "b"]] +iters = [["L", 0, 16], ["b", 0, 6]] +ref = "keccak_sponge:c:one_read" + +[[constraints.mask]] +kind = "arith" +constraint = "the mask is a prefix, within a lane" +poly = ["*", ["-", 1, ["+", ["idx", "full", "L"], ["idx", ["idx", "part", "L"], "b"]]], ["+", ["idx", "full", "L"], ["idx", ["idx", "part", "L"], ["+", "b", 1]]]] +iters = [["L", 0, 16], ["b", 0, 5]] +ref = "keccak_sponge:c:prefix_in_lane" + +[[constraints.mask]] +kind = "arith" +constraint = "the mask is a prefix, at a lane's last byte" +poly = ["*", ["-", 1, ["+", ["idx", "full", "L"], ["idx", ["idx", "part", "L"], 6]]], ["idx", "full", "L"]] +iter = ["L", 0, 16] +ref = "keccak_sponge:c:prefix_lane_end" + +[[constraints.mask]] +kind = "arith" +constraint = "the mask is a prefix, across lanes" +poly = ["*", ["not", ["idx", "full", "L"]], ["+", ["idx", "full", ["+", "L", 1]], ["idx", ["idx", "part", ["+", "L", 1]], 0]]] +iter = ["L", 0, 15] +ref = "keccak_sponge:c:prefix_across_lanes" + +[[constraints.mask]] +kind = "arith" +constraint = "no read off a sponge row" +poly = ["*", ["not", "μ_s"], ["+", ["idx", "full", 0], ["idx", ["idx", "part", 0], 0]]] +ref = "keccak_sponge:c:reads_on_sponge_rows" + +[[constraints.mask]] +kind = "arith" +constraint = "a row that is not last reads a whole block" +poly = ["*", ["-", "μ_s", "last"], ["not", ["idx", "full", 16]]] +ref = "keccak_sponge:c:full_unless_last" + +[[constraints.mask]] +kind = "arith" +constraint = "the last block holds at most 135 input bytes" +poly = ["*", "last", ["idx", "full", 16]] +ref = "keccak_sponge:c:last_not_full" + +[[constraints.mask]] +kind = "arith" +constraint = "the last row reads exactly `count` bytes" +poly = ["*", "last", ["-", "count", ["sum", ["=", "L", 0], 16, ["+", ["*", 8, ["idx", "full", "L"]], ["sum", ["=", "b", 0], 6, ["idx", ["idx", "part", "L"], "b"]]]]]] +ref = "keccak_sponge:c:count_on_last" + +[[constraints.mask]] +kind = "arith" +constraint = "a byte past the input is zero" +poly = ["*", ["-", 1, ["+", ["idx", "full", "L"], ["idx", ["idx", "part", "L"], "b"]]], ["idx", ["idx", "msg", "L"], "b"]] +iters = [["L", 0, 16], ["b", 0, 6]] +ref = "keccak_sponge:c:msg_zero_in_lane" + +[[constraints.mask]] +kind = "arith" +constraint = "a byte past the input is zero, at a lane's last byte" +poly = ["*", ["not", ["idx", "full", "L"]], ["idx", ["idx", "msg", "L"], 7]] +iter = ["L", 0, 16] +ref = "keccak_sponge:c:msg_zero_lane_end" + +[[constraint_groups]] +name = "absorb" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", ["+", "x", ["*", 5, "y"]]], "b"], ["-", ["+", ["idx", ["idx", "msg", ["+", "x", ["*", 5, "y"]]], "b"], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], ["-", "b", 1]]], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], "b"]]] +output = ["idx", ["idx", ["idx", "input_state", "x"], "y"], "b"] +iters = [["y", 0, 2], ["x", 0, 4], ["b", 1, 6]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_mid_0" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", 0], 0], ["-", ["+", ["idx", ["idx", "msg", 0], 0], 1], ["idx", "full", 0], ["idx", ["idx", "part", 0], 0]]] +output = ["idx", ["idx", ["idx", "input_state", 0], 0], 0] +iters = [["x", 0, 0]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_first_byte" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", ["+", "x", ["*", 5, "y"]]], 0], ["-", ["+", ["idx", ["idx", "msg", ["+", "x", ["*", 5, "y"]]], 0], ["idx", "full", ["-", ["+", "x", ["*", 5, "y"]], 1]]], ["idx", "full", ["+", "x", ["*", 5, "y"]]], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], 0]]] +output = ["idx", ["idx", ["idx", "input_state", "x"], "y"], 0] +iters = [["y", 0, 0], ["x", 1, 4]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_lane_start_0" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", ["+", "x", ["*", 5, "y"]]], 0], ["-", ["+", ["idx", ["idx", "msg", ["+", "x", ["*", 5, "y"]]], 0], ["idx", "full", ["-", ["+", "x", ["*", 5, "y"]], 1]]], ["idx", "full", ["+", "x", ["*", 5, "y"]]], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], 0]]] +output = ["idx", ["idx", ["idx", "input_state", "x"], "y"], 0] +iters = [["y", 1, 2], ["x", 0, 4]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_lane_start_12" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", ["+", "x", ["*", 5, "y"]]], 7], ["+", ["idx", ["idx", "msg", ["+", "x", ["*", 5, "y"]]], 7], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], 6]]] +output = ["idx", ["idx", ["idx", "input_state", "x"], "y"], 7] +iters = [["y", 0, 2], ["x", 0, 4]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_lane_end_012" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", ["+", "x", ["*", 5, "y"]]], "b"], ["-", ["+", ["idx", ["idx", "msg", ["+", "x", ["*", 5, "y"]]], "b"], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], ["-", "b", 1]]], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], "b"]]] +output = ["idx", ["idx", ["idx", "input_state", "x"], "y"], "b"] +iters = [["y", 3, 3], ["x", 0, 1], ["b", 1, 6]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_mid_3" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", ["+", "x", ["*", 5, "y"]]], 0], ["-", ["+", ["idx", ["idx", "msg", ["+", "x", ["*", 5, "y"]]], 0], ["idx", "full", ["-", ["+", "x", ["*", 5, "y"]], 1]]], ["idx", "full", ["+", "x", ["*", 5, "y"]]], ["idx", ["idx", "part", ["+", "x", ["*", 5, "y"]]], 0]]] +output = ["idx", ["idx", ["idx", "input_state", "x"], "y"], 0] +iters = [["y", 3, 3], ["x", 0, 1]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_lane_start_3" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", 15], 7], ["+", ["idx", ["idx", "msg", 15], 7], ["idx", ["idx", "part", 15], 6]]] +output = ["idx", ["idx", ["idx", "input_state", 0], 3], 7] +iters = [["y", 3, 3], ["x", 0, 0]] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_lane_end_15" +[[constraints.absorb]] +kind = "interaction" +tag = "BYTE_ALU" +input = [["opsel", "XOR"], ["idx", ["idx", "prev", 16], 7], ["+", ["idx", ["idx", "msg", 16], 7], ["idx", ["idx", "part", 16], 6], ["*", 128, "last"]]] +output = ["idx", ["idx", ["idx", "input_state", 1], 3], 7] +multiplicity = "μ_s" +ref = "keccak_sponge:c:absorb_last_byte" +[[constraint_groups]] +name = "round" + +[[constraints.round]] +kind = "interaction" +tag = "KECCAK" +input = ["key", 0, "input_state"] +multiplicity = "μ_s" +ref = "keccak_sponge:c:send_round" + +[[constraints.round]] +kind = "interaction" +tag = "KECCAK" +input = ["key", 24, "output_state"] +multiplicity = ["-", "μ_s"] +ref = "keccak_sponge:c:receive_round" + +[[constraint_groups]] +name = "digest" + +[[constraints.digest]] +kind = "interaction" +tag = "MEMW" +input = [0, ["arr", ["+", ["idx", "out", 0], ["*", 8, "x"]], ["idx", "out", 1]], ["idx", ["idx", "output_state", "x"], 0], ["arr", ["+", ["idx", "key", 0], 1], 0], 0, 0, 1] +iter = ["x", 0, 3] +multiplicity = "last" +ref = "keccak_sponge:c:write_digest" + +[[constraint_groups]] +name = "range" + +[[constraints.range]] +kind = "arith" +constraint = "`src_end` holds the low word of `src` plus 135" +poly = ["*", "μ_s", ["-", ["+", ["idx", "src", 0], 135], ["+", ["idx", "src_end", 0], ["*", ["^", 2, 16], ["idx", "src_end", 1]]]]] +ref = "keccak_sponge:c:src_end" + +[[constraints.range]] +kind = "interaction" +tag = "IS_HALF" +input = [["idx", "src_end", "i"]] +iter = ["i", 0, 1] +multiplicity = "μ_s" +ref = "keccak_sponge:c:range_src_end" + +[[constraints.range]] +kind = "arith" +constraint = "`out_end` holds the low word of `out` plus 31" +poly = ["*", "last", ["-", ["+", ["idx", "out", 0], 31], ["+", ["idx", "out_end", 0], ["*", ["^", 2, 16], ["idx", "out_end", 1]]]]] +ref = "keccak_sponge:c:out_end" + +[[constraints.range]] +kind = "interaction" +tag = "IS_HALF" +input = [["idx", "out_end", "i"]] +iter = ["i", 0, 1] +multiplicity = "last" +ref = "keccak_sponge:c:range_out_end" diff --git a/spec/src/signatures.toml b/spec/src/signatures.toml index 6ea23bb9e..a636a20d5 100644 --- a/spec/src/signatures.toml +++ b/spec/src/signatures.toml @@ -210,3 +210,10 @@ tag = "KECCAK_RC" kind = "interaction" input = ["BaseField"] output = ["Byte", 8] + +# Keccak-256 of a buffer: KECCAK's sponge-mode chain from one block to the next +# KECCAK_SPONGE[key, count, src, out, state] +[[signatures]] +tag = "KECCAK_SPONGE" +kind = "interaction" +input = ["DWordWL", "BaseField", "DWordWL", "DWordWL", [[["Byte", 8], 5], 5]] diff --git a/spec/tooling/chip.py b/spec/tooling/chip.py index 44cbbed83..071564639 100644 --- a/spec/tooling/chip.py +++ b/spec/tooling/chip.py @@ -786,7 +786,7 @@ class Assumption: def __init__(self, config: Config, data: dict): assert_no_unexpected( - data, set(self.__annotations__.keys()) | {"iter", "iters", "ref"} + data, set(type(self).__annotations__.keys()) | {"iter", "iters", "ref"} ) self.desc = data["desc"] self.iters = iters_of(data, config) @@ -801,7 +801,7 @@ class ArithConstraint: def __init__(self, config: Config, data: dict): assert_no_unexpected( - data, set(self.__annotations__.keys()) | {"kind", "ref", "iter", "iters"} + data, set(type(self).__annotations__.keys()) | {"kind", "ref", "iter", "iters"} ) assert data["kind"] == "arith" self.constraint = data["constraint"] diff --git a/syscalls/Cargo.toml b/syscalls/Cargo.toml index 6bcd8d1a8..c4097315e 100644 --- a/syscalls/Cargo.toml +++ b/syscalls/Cargo.toml @@ -25,6 +25,13 @@ critical-section = { version = "1.2", optional = true } # `src/allocator.rs` for the measurements behind that choice. Select dlmalloc when the # execution's cumulative allocation isn't bounded per block. dlmalloc-alloc = ["dep:dlmalloc", "dep:critical-section"] +# Software memory routines instead of the copy accelerator. By default the runtime's +# `memcpy`, `memmove` and `memset` hand every copy to the COPY accelerator (strong +# globals next to `_start`). With this feature the runtime defines none of the three +# stubs: `memcpy` is a software routine that moves 32 bytes an iteration, and `memmove` +# and `memset` are compiler_builtins'. See `src/entrypoint.rs` for what it does, the C +# contract it keeps, and the tests that sweep it. No guest in this tree turns it on. +fast-memcpy = [] [dev-dependencies] keccak = "0.1" diff --git a/syscalls/src/entrypoint.rs b/syscalls/src/entrypoint.rs index c08c542d1..84d99dff5 100644 --- a/syscalls/src/entrypoint.rs +++ b/syscalls/src/entrypoint.rs @@ -1,11 +1,9 @@ +#[cfg(not(feature = "fast-memcpy"))] use core::arch::global_asm; -use crate::{ - allocator::init_allocator, - syscalls::{ - DMA_MEMCPY_MAX_BYTES, DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER, sys_halt, - }, -}; +#[cfg(not(feature = "fast-memcpy"))] +use crate::syscalls::{DMA_MEMCPY_MAX_BYTES, DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER}; +use crate::{allocator::init_allocator, syscalls::sys_halt}; /// # Safety /// @@ -22,9 +20,134 @@ pub unsafe extern "C" fn _start() -> ! { } } +// The software memory routines, for guests built with the `fast-memcpy` feature: this +// `memcpy`, and compiler_builtins' `memmove` and `memset`. Without the feature the three +// are the copy accelerator's stubs further down, and this routine is not linked. +// +// compiler_builtins' routine copies a word in 5 instructions when source and destination +// share their alignment and in 10 when they do not (it reassembles every misaligned word +// with shifts), and the ethrex guest spends about a fifth of its instructions in it. This +// VM executes an unaligned doubleword load or store as one instruction, so this routine +// aligns the destination with one (possibly unaligned) doubleword copy, moves four +// doublewords in 11 instructions per iteration with plain loads and stores, and finishes a +// partial word with one overlapping doubleword copy instead of a byte loop. Its stores stay +// on the aligned path inside the 32-byte loop; the head, the tail and every load may be +// unaligned, and the prover's memory argument covers those accesses like any other. +// +// It is defined here, next to `_start`, so that it lands in the object the linker always +// takes, and it is a strong global, so it wins over compiler_builtins' weak `memcpy`. +// `memmove` and `memset` stay the library's under this feature; `memcmp` is the runtime's +// either way (below). +// +// Contract: the C one and no more. The two regions must not overlap, except that they may be +// the very same region (the one overlap the compiler emits, for an object assigned to +// itself). The library routine copies forward a byte or word at a time, so for a destination +// starting below the source it happened to give the right bytes; this routine does not (its +// first and last doublewords re-read source bytes the copy may already have overwritten). +// Rust never asks for that: `copy_from_slice` and `copy_nonoverlapping` cannot overlap in safe +// code, and `ptr::copy` / `copy_within` lower to `memmove`. A C dependency linked into a guest +// would be the first caller that might. +// +// Tests: `executor/programs/rust/mem_sweep` runs it in the VM at every length to 1100 and +// every source and destination offset mod 8 (and more, see that guest), and +// `memcpy_model.rs` checks a transliteration of it on the host for out-of-range accesses. +// Change the two together. +#[cfg(feature = "fast-memcpy")] +core::arch::global_asm!( + r#" + .section .text.memcpy,"ax",@progbits + .p2align 2 + .globl memcpy + .type memcpy,@function +memcpy: + # a0 = dst (returned unchanged), a1 = src, a2 = n; t0 walks dst. + mv t0, a0 + li t1, 32 + bltu a2, t1, 5f + # n >= 32: copy the first doubleword, then advance by 1..=8 so that t0 is aligned. + ld t2, 0(a1) + sd t2, 0(t0) + andi t3, t0, 7 + li t4, 8 + sub t3, t4, t3 + add t0, t0, t3 + add a1, a1, t3 + sub a2, a2, t3 + # 32 bytes an iteration while at least 32 are left; t5 = where the loop stops. + andi t5, a2, -32 + beqz t5, 2f + add t5, t0, t5 +1: + ld t2, 0(a1) + ld t3, 8(a1) + ld t4, 16(a1) + ld t6, 24(a1) + sd t2, 0(t0) + sd t3, 8(t0) + sd t4, 16(t0) + sd t6, 24(t0) + addi a1, a1, 32 + addi t0, t0, 32 + bltu t0, t5, 1b + andi a2, a2, 31 +2: + # Fewer than 32 left: whole doublewords. + li t1, 8 + bltu a2, t1, 4f +3: + ld t2, 0(a1) + sd t2, 0(t0) + addi a1, a1, 8 + addi t0, t0, 8 + addi a2, a2, -8 + bgeu a2, t1, 3b +4: + # 1..=7 left: the last doubleword of the copy, overlapping bytes already written. + beqz a2, 9f + add a1, a1, a2 + add t0, t0, a2 + ld t2, -8(a1) + sd t2, -8(t0) + ret +5: + # n < 32: doublewords if n >= 8, then the overlapping last one; else bytes. + li t1, 8 + bltu a2, t1, 7f + add t5, a1, a2 + add t6, t0, a2 +6: + ld t2, 0(a1) + sd t2, 0(t0) + addi a1, a1, 8 + addi t0, t0, 8 + addi a2, a2, -8 + bgeu a2, t1, 6b + beqz a2, 9f + ld t2, -8(t5) + sd t2, -8(t6) + ret +7: + beqz a2, 9f +8: + lbu t2, 0(a1) + sb t2, 0(t0) + addi a1, a1, 1 + addi t0, t0, 1 + addi a2, a2, -1 + bnez a2, 8b +9: + ret + .size memcpy, .-memcpy +"# +); + // --------------------------------------------------------------------------- // DMA memcpy symbol override // +// The default for every guest: `memcpy`, `memmove` and `memset` below hand the copy +// to the COPY accelerator. A guest that turns on `fast-memcpy` (above) keeps all three +// in software instead, and none of the three stubs is defined. +// // `memcpy` is defined next to `_start` on purpose, and not in `syscalls.rs`. // `compiler_builtins` defines `memcpy` weakly, and a linker extracts an archive // member only to satisfy an undefined symbol — a weak definition already @@ -59,6 +182,7 @@ pub unsafe extern "C" fn _start() -> ! { // the VM, which fetches one 4-byte instruction per pc, could not decode it. // --------------------------------------------------------------------------- +#[cfg(not(feature = "fast-memcpy"))] global_asm!( r#" .section .text.memcpy,"ax",@progbits @@ -107,6 +231,7 @@ memcpy: // Otherwise (disjoint, or dst below src) forward chunking is already safe. // --------------------------------------------------------------------------- +#[cfg(not(feature = "fast-memcpy"))] global_asm!( r#" .section .text.memmove,"ax",@progbits @@ -182,6 +307,7 @@ memmove: // seed, and below eight bytes there is nothing left to propagate. // --------------------------------------------------------------------------- +#[cfg(not(feature = "fast-memcpy"))] global_asm!( r#" .section .text.memset,"ax",@progbits @@ -235,3 +361,85 @@ memset: syscall = const DMA_MEMSET_SYSCALL_NUMBER, max_bytes = const DMA_MEMCPY_MAX_BYTES, ); + +// --------------------------------------------------------------------------- +// memcmp +// +// compiler_builtins' `memcmp` compares one byte per iteration, 7 instructions a byte, and the +// ethrex guest spends about 2 % of its instructions in it, almost all of it on equal bytes: +// at the median block 93 % of the calls differ at the first byte, and the calls that find +// their operands equal (32-, 20- and about 60-byte keys) account for 80 % of the bytes +// compared. This routine keeps the library's answer for the first byte (the same six +// instructions when it differs) and then compares a doubleword at a time, 6 instructions per +// 8 bytes. A length that is not a multiple of 8 ends with one doubleword overlapping bytes +// already found equal, so no byte loop runs for lengths of 8 or more. Every load stays inside +// `[a, a + n)` and `[b, b + n)`; the doubleword loads may be unaligned, which this VM executes +// as one instruction (the target spec's `+unaligned-scalar-mem`). +// +// Result: the difference of the first differing bytes as unsigned values, or 0, which is +// exactly what the library returns (the C contract only fixes its sign). In a differing +// doubleword the first differing byte is the lowest-order one (little endian), found a byte +// at a time from the bottom. +// +// Unlike the three stubs above it does not depend on `fast-memcpy`. It sits next to `_start` +// for the same reason they do: compiler_builtins defines `memcmp` weakly, so only a strong +// global in an object the linker always takes is sure to replace it. +// +// Tests: `executor/programs/rust/mem_sweep` runs it in the VM at every length to 300, every +// pair of offsets mod 8 and many differing positions, and `memcmp_model.rs` sweeps a +// transliteration of it on the host for out-of-range loads. Change the two together. +// --------------------------------------------------------------------------- + +core::arch::global_asm!( + r#" + .section .text.memcmp,"ax",@progbits + .p2align 2 + .globl memcmp + .type memcmp,@function +memcmp: + beqz a2, .Lmemcmp_equal + lbu a3, 0(a0) + lbu a4, 0(a1) + bne a3, a4, .Lmemcmp_byte_diff + li t0, 8 + bltu a2, t0, .Lmemcmp_bytes + // a5 / a6: where the last doubleword of each operand starts + add a5, a0, a2 + addi a5, a5, -8 + add a6, a1, a2 + addi a6, a6, -8 +.Lmemcmp_words: + ld t1, 0(a0) + ld t2, 0(a1) + bne t1, t2, .Lmemcmp_word_diff + addi a0, a0, 8 + addi a1, a1, 8 + bltu a0, a5, .Lmemcmp_words + ld t1, 0(a5) + ld t2, 0(a6) + bne t1, t2, .Lmemcmp_word_diff +.Lmemcmp_equal: + li a0, 0 + ret +.Lmemcmp_word_diff: + andi a3, t1, 0xff + andi a4, t2, 0xff + bne a3, a4, .Lmemcmp_byte_diff + srli t1, t1, 8 + srli t2, t2, 8 + j .Lmemcmp_word_diff +.Lmemcmp_bytes: + // 1 <= n < 8 and byte 0 equal: bytes 1 .. n one at a time + addi a0, a0, 1 + addi a1, a1, 1 + addi a2, a2, -1 + beqz a2, .Lmemcmp_equal + lbu a3, 0(a0) + lbu a4, 0(a1) + beq a3, a4, .Lmemcmp_bytes +.Lmemcmp_byte_diff: + sub a0, a3, a4 + ret + .size memcmp, .-memcmp +"# +); diff --git a/syscalls/src/lib.rs b/syscalls/src/lib.rs index 767f0ff71..f00c95082 100644 --- a/syscalls/src/lib.rs +++ b/syscalls/src/lib.rs @@ -7,5 +7,11 @@ pub mod ef_io; #[cfg(target_arch = "riscv64")] pub mod entrypoint; pub mod keccak; +// Host model of the runtime's `memcmp` in `entrypoint.rs`, swept natively. +#[cfg(test)] +mod memcmp_model; +// Host model of the `fast-memcpy` routine in `entrypoint.rs`, swept natively. +#[cfg(test)] +mod memcpy_model; pub mod random; pub mod syscalls; diff --git a/syscalls/src/memcmp_model.rs b/syscalls/src/memcmp_model.rs new file mode 100644 index 000000000..45d88aeb0 --- /dev/null +++ b/syscalls/src/memcmp_model.rs @@ -0,0 +1,191 @@ +//! Host-side model of the guest `memcmp` defined in `entrypoint.rs`. +//! +//! The routine is RISC-V assembly, so it only runs inside the VM; the sweep that +//! executes the real instructions is the guest `executor/programs/rust/mem_sweep` +//! (`executor/tests/rust.rs::test_mem_sweep`). This file transliterates the +//! assembly block for block over a flat byte array, so its index arithmetic (the +//! first byte, the doubleword loop, the overlapping last doubleword, the search +//! inside a differing doubleword and the short path) can be swept natively against +//! a byte-by-byte reference. Every load is checked against its operand's range, +//! which the in-VM sweep can only see through guard bytes. A change to the assembly +//! must be mirrored here. + +use std::ops::Range; + +struct Mem { + bytes: Vec, + a: Range, + b: Range, +} + +impl Mem { + fn in_range(&self, addr: usize, len: usize) { + let inside = |r: &Range| addr >= r.start && addr + len <= r.end; + assert!( + inside(&self.a) || inside(&self.b), + "load of {len} at {addr} outside both operands {:?} {:?}", + self.a, + self.b + ); + } + fn ld(&self, addr: usize) -> u64 { + self.in_range(addr, 8); + u64::from_le_bytes(self.bytes[addr..addr + 8].try_into().unwrap()) + } + fn lbu(&self, addr: usize) -> u64 { + self.in_range(addr, 1); + u64::from(self.bytes[addr]) + } +} + +/// `memcmp(a0 = a, a1 = b, a2 = n)`, one statement per instruction group of the +/// assembly; the comments name the assembly's labels. Returns `a0`. +fn memcmp_model(m: &Mem, mut a0: usize, mut a1: usize, mut a2: usize) -> i64 { + if a2 == 0 { + return 0; // .Lmemcmp_equal + } + let (mut a3, mut a4) = (m.lbu(a0), m.lbu(a1)); + if a3 != a4 { + return a3 as i64 - a4 as i64; // .Lmemcmp_byte_diff + } + let (mut t1, mut t2); + if a2 < 8 { + // .Lmemcmp_bytes + loop { + a0 += 1; + a1 += 1; + a2 -= 1; + if a2 == 0 { + return 0; + } + a3 = m.lbu(a0); + a4 = m.lbu(a1); + if a3 != a4 { + return a3 as i64 - a4 as i64; + } + } + } + let a5 = a0 + a2 - 8; + let a6 = a1 + a2 - 8; + 'diff: { + // .Lmemcmp_words + loop { + t1 = m.ld(a0); + t2 = m.ld(a1); + if t1 != t2 { + break 'diff; + } + a0 += 8; + a1 += 8; + if a0 >= a5 { + break; + } + } + t1 = m.ld(a5); + t2 = m.ld(a6); + if t1 != t2 { + break 'diff; + } + return 0; // .Lmemcmp_equal + } + // .Lmemcmp_word_diff + loop { + a3 = t1 & 0xff; + a4 = t2 & 0xff; + if a3 != a4 { + return a3 as i64 - a4 as i64; + } + t1 >>= 8; + t2 >>= 8; + } +} + +fn reference(x: &[u8], y: &[u8]) -> i64 { + x.iter() + .zip(y) + .find(|(p, q)| p != q) + .map_or(0, |(&p, &q)| i64::from(p) - i64::from(q)) +} + +fn pat(i: usize) -> u8 { + ((i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 +} + +/// Operands of `n` bytes at `64 + ao` (`a`) and `512 + bo` (`b`), holding the same +/// pattern. +fn operands(n: usize, ao: usize, bo: usize) -> Mem { + let (a, b) = (64 + ao, 512 + bo); + let mut bytes: Vec = (0..1024).map(pat).collect(); + for i in 0..n { + bytes[b + i] = bytes[a + i]; + } + Mem { + bytes, + a: a..a + n, + b: b..b + n, + } +} + +/// Apply each (position, xor) to `b`, check the model against the reference in both +/// argument orders, and undo the flips. +fn check(m: &mut Mem, flips: &[(usize, u8)]) { + let (a, b, n) = (m.a.start, m.b.start, m.a.len()); + for &(at, x) in flips { + m.bytes[b + at] ^= x; + } + let want = reference(&m.bytes[a..a + n], &m.bytes[b..b + n]); + let what = || format!("n {n} a%8 {} b%8 {} flips {flips:?}", a % 8, b % 8); + assert_eq!(memcmp_model(m, a, b, n), want, "{}", what()); + assert_eq!(memcmp_model(m, b, a, n), -want, "{}, swapped", what()); + for &(at, x) in flips { + m.bytes[b + at] ^= x; + } +} + +/// Every differing position up to 80 bytes (every remainder mod 8 of the length and of +/// the position, the overlapping last doubleword included); above that the positions +/// near the start, the middle and the end. +fn positions(n: usize) -> Vec { + if n <= 80 { + return (0..n).collect(); + } + let mut at: Vec = (0..10) + .chain(n / 2 - 8..n / 2 + 8) + .chain(n - 17..n) + .collect(); + at.dedup(); + at +} + +#[test] +fn every_length_alignment_and_differing_position() { + for n in 0..=300 { + for ao in 0..8 { + for bo in 0..8 { + let mut m = operands(n, ao, bo); + check(&mut m, &[]); + for at in positions(n) { + for x in [0x80, 0x01, 0xff] { + check(&mut m, &[(at, x)]); + } + // a later difference of the opposite sense must not decide + if at + 1 < n { + check(&mut m, &[(at, 0x01), (n - 1, 0x80)]); + } + } + } + } + } +} + +#[test] +fn same_operand() { + for n in 0..=300 { + for ao in 0..8 { + let mut m = operands(n, ao, ao); + m.b = m.a.clone(); + let a = m.a.start; + assert_eq!(memcmp_model(&m, a, a, n), 0, "n {n} a%8 {ao}"); + } + } +} diff --git a/syscalls/src/memcpy_model.rs b/syscalls/src/memcpy_model.rs new file mode 100644 index 000000000..b57a8a3e5 --- /dev/null +++ b/syscalls/src/memcpy_model.rs @@ -0,0 +1,225 @@ +//! Host-side model of the guest `memcpy` defined in `entrypoint.rs` (the +//! `fast-memcpy` feature). +//! +//! The routine is RISC-V assembly, so it only runs inside the VM; the sweep that +//! executes the real instructions is the guest `executor/programs/rust/mem_sweep` +//! (`executor/tests/rust.rs::test_mem_sweep`). This file transliterates the +//! assembly block for block over a flat byte array, so its index arithmetic (the +//! head, the 32-byte groups, the 8-byte steps, the overlapping last doubleword and +//! the short paths) can be swept natively. It checks three things the in-VM sweep +//! can only see through guard bytes: every load stays inside the source range, +//! every store stays inside the destination range, and the stores of the 32-byte +//! loop are doubleword-aligned. A change to the assembly must be mirrored here. + +use std::ops::Range; + +struct Mem { + bytes: Vec, + src: Range, + dst: Range, +} + +impl Mem { + fn ld(&self, addr: usize) -> u64 { + assert!( + addr >= self.src.start && addr + 8 <= self.src.end, + "load of 8 at {addr} outside the source {:?}", + self.src + ); + u64::from_le_bytes(self.bytes[addr..addr + 8].try_into().unwrap()) + } + fn sd(&mut self, addr: usize, value: u64) { + assert!( + addr >= self.dst.start && addr + 8 <= self.dst.end, + "store of 8 at {addr} outside the destination {:?}", + self.dst + ); + self.bytes[addr..addr + 8].copy_from_slice(&value.to_le_bytes()); + } + fn lbu(&self, addr: usize) -> u8 { + assert!( + self.src.contains(&addr), + "byte load at {addr} outside the source {:?}", + self.src + ); + self.bytes[addr] + } + fn sb(&mut self, addr: usize, value: u8) { + assert!( + self.dst.contains(&addr), + "byte store at {addr} outside the destination {:?}", + self.dst + ); + self.bytes[addr] = value; + } +} + +/// `memcpy(a0 = dst, a1 = src, a2 = n)`, one statement per instruction group of the +/// assembly; the numeric labels are the assembly's. +fn memcpy_model(m: &mut Mem, a0: usize, mut a1: usize, mut a2: usize) -> usize { + let mut t0 = a0; + if a2 >= 32 { + // head: one doubleword wherever the destination starts, then step to the + // next doubleword boundary of the destination (1..=8 bytes) + let t2 = m.ld(a1); + m.sd(t0, t2); + let t3 = 8 - (t0 & 7); + t0 += t3; + a1 += t3; + a2 -= t3; + let t5 = a2 & !31; + if t5 != 0 { + let end = t0 + t5; + // 1: four loads, then four stores + loop { + let w = [m.ld(a1), m.ld(a1 + 8), m.ld(a1 + 16), m.ld(a1 + 24)]; + assert_eq!(t0 % 8, 0, "the 32-byte loop stores to a misaligned address"); + for (i, w) in w.into_iter().enumerate() { + m.sd(t0 + 8 * i, w); + } + a1 += 32; + t0 += 32; + if t0 >= end { + break; + } + } + a2 &= 31; + } + // 2 / 3: whole doublewords left + if a2 >= 8 { + loop { + let t2 = m.ld(a1); + m.sd(t0, t2); + a1 += 8; + t0 += 8; + a2 -= 8; + if a2 < 8 { + break; + } + } + } + // 4: the last 1..=7 bytes as one doubleword ending at the end + if a2 != 0 { + a1 += a2; + t0 += a2; + let t2 = m.ld(a1 - 8); + m.sd(t0 - 8, t2); + } + return a0; + } + // 5: 8..=31 bytes + if a2 >= 8 { + let t5 = a1 + a2; + let t6 = t0 + a2; + // 6 + loop { + let t2 = m.ld(a1); + m.sd(t0, t2); + a1 += 8; + t0 += 8; + a2 -= 8; + if a2 < 8 { + break; + } + } + if a2 != 0 { + let t2 = m.ld(t5 - 8); + m.sd(t6 - 8, t2); + } + return a0; + } + // 7 / 8: 0..=7 bytes, one at a time + while a2 != 0 { + let t2 = m.lbu(a1); + m.sb(t0, t2); + a1 += 1; + t0 += 1; + a2 -= 1; + } + a0 +} + +fn pat(i: usize) -> u8 { + ((i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 +} + +/// Copy `n` bytes from address `src` to address `dst` in a patterned memory and +/// compare the whole memory with a byte-by-byte reference copy. +fn check(size: usize, dst: usize, src: usize, n: usize) { + let bytes: Vec = (0..size).map(pat).collect(); + let mut want = bytes.clone(); + for i in 0..n { + want[dst + i] = bytes[src + i]; + } + let mut m = Mem { + bytes, + src: src..src + n, + dst: dst..dst + n, + }; + let ret = memcpy_model(&mut m, dst, src, n); + assert_eq!(ret, dst, "return value, n {n} dst {dst} src {src}"); + if let Some(at) = (0..size).find(|&i| m.bytes[i] != want[i]) { + panic!( + "n {n} dst%16 {} src%16 {}: byte {} (destination + {}) is wrong", + dst % 16, + src % 16, + at, + at as isize - dst as isize + ); + } +} + +#[test] +fn every_length_and_alignment_mod_8() { + for n in 0..=1100 { + for sa in 0..8 { + for da in 0..8 { + check(4096, 2048 + da, 64 + sa, n); + } + } + } +} + +#[test] +fn every_alignment_mod_16_around_the_thresholds() { + for n in 0..=160 { + for sa in 0..16 { + for da in 0..16 { + check(1024, 512 + da, 64 + sa, n); + } + } + } +} + +#[test] +fn lengths_between_the_sweep_and_a_page() { + // Step 7 is prime to 32, so the lengths cover every remainder of the 32-byte + // loop and of the 8-byte steps. + for n in (1101..=4096).step_by(7) { + for (sa, da) in [(0, 0), (1, 0), (0, 1), (7, 4)] { + check(2 * n + 256, n + 128 + da, 64 + sa, n); + } + } +} + +#[test] +fn large_lengths() { + for n in [4095, 4096, 4097, 8191, 65536, 65543, 262149, 1 << 20] { + for (sa, da) in [(0, 0), (1, 0), (0, 1), (7, 4), (3, 3)] { + check(2 * n + 256, n + 128 + da, 64 + sa, n); + } + } +} + +#[test] +fn same_region_and_touching_regions() { + for n in 0..=300 { + for a in 0..8 { + // source == destination: the one overlap a compiler may emit + check(1024, 64 + a, 64 + a, n); + // destination right after the source, and right before it + check(1024, 64 + a + n, 64 + a, n); + check(1024, 64 + a, 64 + a + n, n); + } + } +} diff --git a/syscalls/src/syscalls.rs b/syscalls/src/syscalls.rs index 032861803..2d3d506fc 100644 --- a/syscalls/src/syscalls.rs +++ b/syscalls/src/syscalls.rs @@ -29,21 +29,40 @@ pub enum SyscallNumbers { #[cfg(target_arch = "riscv64")] const KECCAK_SYSCALL_NUMBER: usize = usize::MAX - 1; +/// Syscall number for Keccak-256 of a buffer (u64::MAX - 4). Must match +/// `executor::...::execution::KECCAK256_SYSCALL_NUMBER`. +#[cfg(target_arch = "riscv64")] +const KECCAK256_SYSCALL_NUMBER: usize = usize::MAX - 4; + +/// Syscall number for the BLAKE3 6-round compression accelerator (u64::MAX - 2). +#[cfg(target_arch = "riscv64")] +const BLAKE3_SYSCALL_NUMBER: usize = usize::MAX - 2; + +/// Syscall number for the BLAKE3 6-round chained-absorb accelerator +/// (u64::MAX - 3). +#[cfg(target_arch = "riscv64")] +const BLAKE3_ABSORB_SYSCALL_NUMBER: usize = usize::MAX - 3; + /// Syscall number for the ECSM secp256k1 scalar-multiply accelerator (-11 as usize). #[cfg(target_arch = "riscv64")] const ECSM_SYSCALL_NUMBER: usize = usize::MAX - 10; +/// Syscall number for the ECDSA public-key recovery accelerator (-12 as usize). Must match +/// the executor's `ECREC_SYSCALL_NUMBER`. +#[cfg(target_arch = "riscv64")] +const ECREC_SYSCALL_NUMBER: usize = usize::MAX - 11; + /// Copy-accelerator syscall number, serving `memcpy` and `memmove` (-30 as usize). /// Must match the executor. -#[cfg(target_arch = "riscv64")] +#[cfg(all(target_arch = "riscv64", not(feature = "fast-memcpy")))] pub(crate) const DMA_MEMCPY_SYSCALL_NUMBER: usize = usize::MAX - 29; /// Maximum bytes sent in one DMA ecall. Larger `memcpy` calls are split by the /// strong assembly stub so continuation table height remains bounded by cycles. -#[cfg(target_arch = "riscv64")] +#[cfg(all(target_arch = "riscv64", not(feature = "fast-memcpy")))] pub(crate) const DMA_MEMCPY_MAX_BYTES: usize = 256; /// `memset` syscall number (-31 as usize). Must match the executor. -#[cfg(target_arch = "riscv64")] +#[cfg(all(target_arch = "riscv64", not(feature = "fast-memcpy")))] pub(crate) const DMA_MEMSET_SYSCALL_NUMBER: usize = usize::MAX - 30; /// Syscall number for the non-constraining Hint ecall (-32 as usize). @@ -192,6 +211,109 @@ pub fn keccak_permute(_state: &mut [u64; 25]) { unimplemented!("syscalls are only implemented for riscv64 targets"); } +/// Whether [`keccak256`] accepts these addresses — a sufficient form of the +/// executor's `keccak256_addresses_ok`, cheap enough to run before every call: +/// every input block's 136-byte window, and the 32-byte output, stay below one +/// `2^32` boundary of the low address word. (`136·(⌊len/136⌋ + 1) ≤ len + 136`, +/// so this refuses at most a few calls the VM would take; a caller falls back to +/// the sponge over [`keccak_permute`] for those.) +#[inline(always)] +pub fn keccak256_fits(input: &[u8], out: &[u8; 32]) -> bool { + const TOP: u64 = 1 << 32; + let ptr_lo = (input.as_ptr() as u64) & (TOP - 1); + let out_lo = (out.as_ptr() as u64) & (TOP - 1); + // `len < 2^32` first, so the sum below cannot overflow. + (input.len() as u64) < TOP && ptr_lo + input.len() as u64 + 136 <= TOP && out_lo + 32 <= TOP +} + +#[cfg(target_arch = "riscv64")] +/// Keccak-256 of `input` into `out`, in one VM call (the KECCAK table's sponge +/// mode). The caller checks [`keccak256_fits`] first: the VM refuses operands +/// whose blocks cross a `2^32` address-word boundary. +pub fn keccak256(input: &[u8], out: &mut [u8; 32]) { + unsafe { + asm!( + "ecall", + in("a0") input.as_ptr(), + in("a1") input.len(), + in("a2") out.as_mut_ptr(), + in("a7") KECCAK256_SYSCALL_NUMBER, + options(nostack), + ) + } +} + +#[cfg(not(target_arch = "riscv64"))] +/// Keccak-256 of `input` into `out`, in one VM call. +pub fn keccak256(_input: &[u8], _out: &mut [u8; 32]) { + unimplemented!("syscalls are only implemented for riscv64 targets"); +} + +#[cfg(target_arch = "riscv64")] +/// BLAKE3 **6-round** compression via the accelerator (internal variant — NOT +/// standard 7-round BLAKE3; see `thoughts/blake3/blake3-chip/DESIGN.md`). +/// +/// `state` is the 176-byte region as 22 dwords: `h[8 words] | m[16 words] | +/// t | (block_len, flags) | out[16 words]`, all words little-endian, two per +/// dword. The accelerator reads dwords 0..14 and writes `out` to dwords 14..22. +/// Using `[u64; 22]` guarantees the 8-byte alignment the ecall requires. +pub fn blake3_compress_6round(state: &mut [u64; 22]) { + unsafe { + asm!( + "ecall", + in("a0") state.as_mut_ptr(), + in("a7") BLAKE3_SYSCALL_NUMBER, + ) + } +} + +#[cfg(not(target_arch = "riscv64"))] +/// BLAKE3 6-round compression via the accelerator (internal variant). +pub fn blake3_compress_6round(_state: &mut [u64; 22]) { + unimplemented!("syscalls are only implemented for riscv64 targets"); +} + +/// Dwords in the BLAKE3 absorb control region: `cv_in[8 words] | cv_out[8]`. +/// Mirrors `BLAKE3_ABSORB_CTRL_DWORDS` in `executor::vm::instruction::execution`. +pub const BLAKE3_ABSORB_CTRL_DWORDS: usize = 8; + +#[cfg(target_arch = "riscv64")] +/// Fold whole 64-byte blocks into a BLAKE3 **6-round** chaining value with a +/// single ecall (internal variant — see [`blake3_compress_6round`]). +/// +/// `ctrl` is the 64-byte control region as 8 dwords: dwords `0..4` carry the +/// incoming chaining value `cv_in` and the accelerator writes the outgoing one +/// into dwords `4..8`. `blocks` is the message, read in place — no copy, which +/// is the point of this call. `first_flags` is the flag word of the run's first +/// block; every later block carries none. +/// +/// The caller must satisfy what the executor checks, or the ecall faults: +/// `blocks.len()` is a nonzero multiple of 64 and at most `2^16` blocks, +/// `blocks` is 8-byte aligned, and `blocks` does not overlap `ctrl`. Using +/// `[u64; 8]` for `ctrl` guarantees its own alignment. +pub fn blake3_absorb(ctrl: &mut [u64; BLAKE3_ABSORB_CTRL_DWORDS], blocks: &[u8], first_flags: u32) { + unsafe { + asm!( + "ecall", + in("a0") ctrl.as_mut_ptr(), + in("a1") blocks.as_ptr(), + in("a2") blocks.len() / 64, + in("a3") first_flags as usize, + in("a7") BLAKE3_ABSORB_SYSCALL_NUMBER, + ) + } +} + +#[cfg(not(target_arch = "riscv64"))] +/// Fold whole 64-byte blocks into a BLAKE3 6-round chaining value in one ecall. +pub fn blake3_absorb( + _ctrl: &mut [u64; BLAKE3_ABSORB_CTRL_DWORDS], + _blocks: &[u8], + _first_flags: u32, +) { + unimplemented!("syscalls are only implemented for riscv64 targets"); +} + #[cfg(target_arch = "riscv64")] /// Compute `xR = (k·G)_x` on secp256k1 via the ECSM accelerator. All values are 32-byte /// little-endian. Requires `0 < k < N` and a canonical valid `xG` curve coordinate. @@ -214,6 +336,58 @@ pub fn ecsm_mul(_xr: &mut [u8; 32], _xg: &[u8; 32], _k: &[u8; 32]) { unimplemented!("syscalls are only implemented for riscv64 targets"); } +/// Whether [`ecrecover`] accepts these operands: the executor's own conditions +/// (its ECREC arm), so a caller that checks this first never stops the VM. Each +/// operand's bytes stay below one `2^32` boundary of its low address word (the +/// 72-byte output, the 64-byte signature, the 32-byte hash), and the signature and +/// the hash do not overlap. The parity is the caller's `recid & 1`, always in range. +#[inline(always)] +pub fn ecrecover_fits(out: &[u8; 72], sig: &[u8; 64], msg: &[u8; 32]) -> bool { + ecrecover_addresses_ok( + out.as_ptr() as u64, + sig.as_ptr() as u64, + msg.as_ptr() as u64, + ) +} + +/// [`ecrecover_fits`] on raw addresses. +#[inline(always)] +pub fn ecrecover_addresses_ok(out: u64, sig: u64, msg: u64) -> bool { + const TOP: u64 = 1 << 32; + let below_boundary = |addr: u64, last: u64| (addr & (TOP - 1)) + last < TOP; + below_boundary(out, 71) + && below_boundary(sig, 63) + && below_boundary(msg, 31) + && !(sig < msg.wrapping_add(32) && msg < sig.wrapping_add(64)) +} + +#[cfg(target_arch = "riscv64")] +/// Recover the secp256k1 public key of an ECDSA signature via the ECREC accelerator. +/// `sig` is `r ‖ s` and `msg` the message hash, each value 32-byte big-endian (as +/// Ethereum carries them); `parity` is the parity of `R`'s y (the recovery id's low +/// bit, 0 or 1). Writes `out = Q.x ‖ Q.y ‖ status`: the key's coordinates, 32-byte +/// big-endian each, and a little-endian status doubleword, 0 when the call computed the +/// key and 1 when it bailed (the key bytes are then zero and the caller must recover +/// the key another way). `sig` and `msg` must not overlap; `out` may alias either. +pub fn ecrecover(out: &mut [u8; 72], sig: &[u8; 64], msg: &[u8; 32], parity: u64) { + unsafe { + asm!( + "ecall", + in("a0") out.as_mut_ptr(), // x10 = address to write the key and status + in("a1") sig.as_ptr(), // x11 = address of r ‖ s + in("a2") msg.as_ptr(), // x12 = address of the message hash + in("a3") parity, // x13 = parity of R's y + in("a7") ECREC_SYSCALL_NUMBER, + ) + } +} + +#[cfg(not(target_arch = "riscv64"))] +/// Recover the secp256k1 public key of an ECDSA signature via the ECREC accelerator. +pub fn ecrecover(_out: &mut [u8; 72], _sig: &[u8; 64], _msg: &[u8; 32], _parity: u64) { + unimplemented!("syscalls are only implemented for riscv64 targets"); +} + /// Ask the host for a non-constraining hint (modular inverse/sqrt). /// `hint_id` selects the operation ([`HINT_FIELD_INV`]/[`HINT_SCALAR_INV`]/ /// [`HINT_FIELD_SQRT`]); `input`/`out` are 32-byte **big-endian** field/scalar @@ -272,3 +446,41 @@ pub unsafe extern "C" fn sys_argc() -> usize { pub unsafe extern "C" fn sys_argv(_buf: *mut u32, _buf_nwords: usize, _arg_idx: usize) -> usize { panic!("sys_argv is not supported: command-line arguments are not available in Lambda VM"); } + +#[cfg(test)] +mod ecrecover_fits_tests { + use super::ecrecover_addresses_ok as ok; + + #[test] + fn disjoint_operands_below_the_boundary_fit() { + assert!(ok(0x1000, 0x2000, 0x3000)); + // The hash right after the signature, and right before it. + assert!(ok(0x1000, 0x2000, 0x2040)); + assert!(ok(0x1000, 0x2000, 0x1fe0)); + // The output may alias either input. + assert!(ok(0x2000, 0x2000, 0x3000)); + // High address words are free. + assert!(ok(0xffff_ffff_0000_1000, 0x7_0000_2000, 0x7_0000_3000)); + } + + #[test] + fn overlapping_signature_and_hash_do_not_fit() { + assert!(!ok(0x1000, 0x2000, 0x2000)); + assert!(!ok(0x1000, 0x2000, 0x203f)); + assert!(!ok(0x1000, 0x2000, 0x1fe1)); + } + + #[test] + fn an_operand_across_a_low_word_boundary_does_not_fit() { + const TOP: u64 = 1 << 32; + // The last byte exactly at 2^32 − 1 fits; one further does not. + assert!(ok(TOP - 72, 0x2000, 0x3000)); + assert!(!ok(TOP - 71, 0x2000, 0x3000)); + assert!(ok(0x1000, TOP - 64, 0x3000)); + assert!(!ok(0x1000, TOP - 63, 0x3000)); + assert!(ok(0x1000, 0x2000, TOP - 32)); + assert!(!ok(0x1000, 0x2000, TOP - 31)); + // The same at a nonzero high word. + assert!(!ok(0x1000, 0x2000, 5 * TOP - 31)); + } +} diff --git a/tooling/blake3-second-source/blake3-oracle/.gitignore b/tooling/blake3-second-source/blake3-oracle/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/tooling/blake3-second-source/blake3-oracle/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tooling/blake3-second-source/blake3-oracle/blake3_ref.py b/tooling/blake3-second-source/blake3-oracle/blake3_ref.py new file mode 100644 index 000000000..ee2d48e85 --- /dev/null +++ b/tooling/blake3-second-source/blake3-oracle/blake3_ref.py @@ -0,0 +1,399 @@ +""" +BLAKE3 compression-function ORACLE (reference implementation). + +This is the TRUST ANCHOR for a future BLAKE3 accelerator chip in the Lambda VM +STARK prover. It is written directly from the BLAKE3 specification / reference +design, NOT copied from any implementation, and then validated externally in +`test_oracle.py` against: + - the official BLAKE3 team's `test_vectors.json`, + - the official `blake3` PyPI package (the reference Rust implementation), + - Plonky3's independent `blake3-air` compression implementation. + +Spec sources used while writing this file (all public): + - BLAKE3 paper / spec, section 2.1-2.2 (compression function, G, round). + - The reference message-permutation schedule and IV constants, which also + appear verbatim in the vendored Plonky3 `blake3-air/src/constants.rs` + (IV, MSG_PERMUTATION) — used here only as a cross-check of the constants, + the mixing logic is written from the spec's G-function definition. + +Everything operates on 32-bit unsigned words, little-endian, exactly as BLAKE3 +specifies. +""" + +# --------------------------------------------------------------------------- +# Constants (BLAKE3 spec, section 2.1) +# --------------------------------------------------------------------------- + +# Initialisation vector: the first 8 words of the SHA-256 IV (fractional parts +# of the square roots of the first 8 primes). Identical to SHA-256 / BLAKE2s. +IV = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +] + +# Message word permutation applied between successive rounds. After each round +# the 16 message words are permuted by this index map; round r therefore mixes +# the original message under permutation^r. (BLAKE3 spec / reference schedule.) +MSG_PERMUTATION = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + +# Domain-separation flags (BLAKE3 spec, table of flags). +CHUNK_START = 1 << 0 # 0x01 +CHUNK_END = 1 << 1 # 0x02 +PARENT = 1 << 2 # 0x04 +ROOT = 1 << 3 # 0x08 +KEYED_HASH = 1 << 4 # 0x10 +DERIVE_KEY_CONTEXT = 1 << 5 # 0x20 +DERIVE_KEY_MATERIAL = 1 << 6 # 0x40 + +# Structural sizes. +BLOCK_LEN = 64 # bytes per compression input block (16 words * 4 bytes) +CHUNK_LEN = 1024 # bytes per chunk (16 blocks) +KEY_LEN = 32 # bytes in a key / chaining value (8 words * 4 bytes) +OUT_LEN = 32 # default output length in bytes + +MASK32 = 0xFFFFFFFF + +# Standard round count for BLAKE3. Variant B is the same function with ROUNDS=6. +DEFAULT_ROUNDS = 7 + + +# --------------------------------------------------------------------------- +# 32-bit word primitives (BLAKE3 spec, section 2.1 "G function") +# --------------------------------------------------------------------------- + +def add32(a, b): + """Addition modulo 2^32 (wrapping).""" + return (a + b) & MASK32 + + +def rotr(x, n): + """Rotate the 32-bit word `x` RIGHT by `n` bits. + + BLAKE3's G uses rotation amounts 16, 12, 8, 7. Rotations by 16 and 8 are + byte-aligned (multiples of 8); 12 and 7 are not. The chip-contract reuse + map in ORACLE.md analyses each of these against the HWSL lookup table. + """ + x &= MASK32 + return ((x >> n) | (x << (32 - n))) & MASK32 + + +def g(state, a, b, c, d, mx, my): + """The BLAKE3 quarter-round mixing function G (spec section 2.1). + + Mixes two message words `mx`, `my` into four state words at indices + a, b, c, d of the 16-word working state. Two "half rounds" of the form + add / xor+rotate: + + v[a] = v[a] + v[b] + mx + v[d] = (v[d] ^ v[a]) >>> 16 + v[c] = v[c] + v[d] + v[b] = (v[b] ^ v[c]) >>> 12 + v[a] = v[a] + v[b] + my + v[d] = (v[d] ^ v[a]) >>> 8 + v[c] = v[c] + v[d] + v[b] = (v[b] ^ v[c]) >>> 7 + """ + state[a] = add32(add32(state[a], state[b]), mx) + state[d] = rotr(state[d] ^ state[a], 16) + state[c] = add32(state[c], state[d]) + state[b] = rotr(state[b] ^ state[c], 12) + state[a] = add32(add32(state[a], state[b]), my) + state[d] = rotr(state[d] ^ state[a], 8) + state[c] = add32(state[c], state[d]) + state[b] = rotr(state[b] ^ state[c], 7) + + +def round_fn(state, m): + """One BLAKE3 round: 4 column mixes then 4 diagonal mixes (spec 2.1). + + `m` is the (already-permuted for this round) 16-word message schedule. + The G calls consume message words m[0..16] in order. + """ + # Mix the columns. + g(state, 0, 4, 8, 12, m[0], m[1]) + g(state, 1, 5, 9, 13, m[2], m[3]) + g(state, 2, 6, 10, 14, m[4], m[5]) + g(state, 3, 7, 11, 15, m[6], m[7]) + # Mix the diagonals. + g(state, 0, 5, 10, 15, m[8], m[9]) + g(state, 1, 6, 11, 12, m[10], m[11]) + g(state, 2, 7, 8, 13, m[12], m[13]) + g(state, 3, 4, 9, 14, m[14], m[15]) + + +def permute(m): + """Apply MSG_PERMUTATION to a 16-word message list, returning a new list.""" + return [m[MSG_PERMUTATION[i]] for i in range(16)] + + +# --------------------------------------------------------------------------- +# The compression function `f` (BLAKE3 spec, section 2.2) +# --------------------------------------------------------------------------- + +def compress(chaining_value, block_words, counter, block_len, flags, + rounds=DEFAULT_ROUNDS): + """BLAKE3 compression function. + + Inputs: + chaining_value : list of 8 u32 words (h[0..8]) + block_words : list of 16 u32 words (m[0..16]) + counter : u64 block counter t + block_len : u32 number of input bytes in this block (0..64) + flags : u32 domain-separation flags + rounds : number of rounds (7 = standard, 6 = variant B) + + Returns a list of 16 u32 words: the full compression output. The truncated + 8-word chaining value used elsewhere in the tree is `output[0:8]`. + + The 16-word initial working state v is: + v[0..8] = chaining_value[0..8] + v[8..12] = IV[0..4] + v[12] = counter mod 2^32 (low 32 bits of t) + v[13] = counter >> 32 (high 32 bits of t) + v[14] = block_len + v[15] = flags + Then `rounds` rounds are applied, permuting the message schedule between + rounds. Finally the feed-forward XOR produces the 16-word output: + output[i] = v[i] ^ v[i+8] for i in 0..8 + output[i+8] = v[i+8] ^ chaining_value[i] for i in 0..8 + """ + assert len(chaining_value) == 8 + assert len(block_words) == 16 + assert 0 <= counter < (1 << 64) + + counter_low = counter & MASK32 + counter_high = (counter >> 32) & MASK32 + + state = [ + chaining_value[0], chaining_value[1], chaining_value[2], chaining_value[3], + chaining_value[4], chaining_value[5], chaining_value[6], chaining_value[7], + IV[0], IV[1], IV[2], IV[3], + counter_low & MASK32, counter_high & MASK32, block_len & MASK32, flags & MASK32, + ] + + # Local copy of the message schedule; permuted between rounds. + m = list(block_words) + for r in range(rounds): + round_fn(state, m) + # Permute between rounds. The permutation after the final round is + # never consumed, so applying it only for r < rounds-1 is equivalent; + # we permute between rounds to keep the loop structure obvious. + if r < rounds - 1: + m = permute(m) + + # Feed-forward XOR producing the full 16-word output. + output = [0] * 16 + for i in range(8): + output[i] = state[i] ^ state[i + 8] + output[i + 8] = state[i + 8] ^ chaining_value[i] + return output + + +def compress_cv(chaining_value, block_words, counter, block_len, flags, + rounds=DEFAULT_ROUNDS): + """The truncated 8-word chaining value: first 8 words of `compress`.""" + return compress(chaining_value, block_words, counter, block_len, flags, + rounds)[:8] + + +# =========================================================================== +# Variant B: 6-round BLAKE3 compression. +# +# This is EXACTLY `compress(..., rounds=6)`. It is a NONSTANDARD function with +# no external test vectors; ORACLE.md documents its canonical vectors. The only +# difference from the validated 7-round function is the loop bound `rounds`. +# =========================================================================== + +def compress_6round(chaining_value, block_words, counter, block_len, flags): + """6-round variant of the BLAKE3 compression function (variant B). + + Rounds 0..5 are applied with message permutations 0..5 (i.e. round r mixes + permute^r(block_words)), then the identical feed-forward XOR finalisation. + Everything else — IV, initial state layout, G function, feed-forward — is + bit-for-bit identical to the 7-round function. + """ + return compress(chaining_value, block_words, counter, block_len, flags, + rounds=6) + + +# =========================================================================== +# Full BLAKE3 tree hash, built ON TOP of `compress`. +# +# This exists ONLY so the compression function can be validated against the +# official whole-hash test vectors (which exercise `compress` under every flag +# combination and many counter values). The chip does NOT implement the tree; +# it implements `compress`. Written from the spec's tree/chunk structure. +# =========================================================================== + +def words_from_le_bytes(b): + """Convert a bytes object (len multiple of 4) into a list of u32 words.""" + assert len(b) % 4 == 0 + return [int.from_bytes(b[i:i + 4], "little") for i in range(0, len(b), 4)] + + +def le_bytes_from_words(words): + return b"".join((w & MASK32).to_bytes(4, "little") for w in words) + + +class _Output: + """A not-yet-finalised node (chunk or parent). Can emit a chaining value + or an extendable root output (spec section 2.3, XOF).""" + + def __init__(self, input_cv, block_words, counter, block_len, flags, rounds): + self.input_cv = input_cv + self.block_words = block_words + self.counter = counter + self.block_len = block_len + self.flags = flags + self.rounds = rounds + + def chaining_value(self): + return compress(self.input_cv, self.block_words, self.counter, + self.block_len, self.flags, self.rounds)[:8] + + def root_output_bytes(self, out_len): + out = bytearray() + counter = 0 + while len(out) < out_len: + words = compress(self.input_cv, self.block_words, counter, + self.block_len, self.flags | ROOT, self.rounds) + # The ROOT output uses ALL 16 output words (this is why compress + # returns 16 words rather than the truncated 8). + out += le_bytes_from_words(words) + counter += 1 + return bytes(out[:out_len]) + + +class _ChunkState: + def __init__(self, key_words, chunk_counter, flags, rounds): + self.cv = list(key_words) + self.chunk_counter = chunk_counter + self.block = b"" + self.blocks_compressed = 0 + self.flags = flags + self.rounds = rounds + + def _start_flag(self): + return CHUNK_START if self.blocks_compressed == 0 else 0 + + def update(self, data): + while data: + if len(self.block) == BLOCK_LEN: + block_words = words_from_le_bytes(self.block) + self.cv = compress(self.cv, block_words, self.chunk_counter, + BLOCK_LEN, self.flags | self._start_flag(), + self.rounds)[:8] + self.blocks_compressed += 1 + self.block = b"" + take = min(BLOCK_LEN - len(self.block), len(data)) + self.block += data[:take] + data = data[take:] + + def output(self): + block_words = words_from_le_bytes(self.block + b"\x00" * (BLOCK_LEN - len(self.block))) + return _Output(self.cv, block_words, self.chunk_counter, len(self.block), + self.flags | self._start_flag() | CHUNK_END, self.rounds) + + +def _parent_output(left_cv, right_cv, key_words, flags, rounds): + block_words = left_cv + right_cv # 16 words + return _Output(list(key_words), block_words, 0, BLOCK_LEN, flags | PARENT, rounds) + + +class Blake3Hasher: + """Minimal BLAKE3 tree hasher over the reference `compress`. + + Supports the three official modes (default hash, keyed hash, derive-key) + and extendable output, so it can be checked against `test_vectors.json`. + """ + + def __init__(self, key_words, flags, rounds=DEFAULT_ROUNDS): + self.key_words = list(key_words) + self.flags = flags + self.rounds = rounds + self.chunk_state = _ChunkState(self.key_words, 0, flags, rounds) + self.cv_stack = [] # list of 8-word chaining values + + @classmethod + def default(cls, rounds=DEFAULT_ROUNDS): + return cls(IV, 0, rounds) + + @classmethod + def keyed(cls, key32, rounds=DEFAULT_ROUNDS): + assert len(key32) == KEY_LEN + return cls(words_from_le_bytes(key32), KEYED_HASH, rounds) + + @classmethod + def derive_key(cls, context_string, rounds=DEFAULT_ROUNDS): + # Phase 1: hash the context string in DERIVE_KEY_CONTEXT mode to get a + # 32-byte context key; Phase 2: keyed-hash the material with that key + # under DERIVE_KEY_MATERIAL. + ctx_hasher = cls(IV, DERIVE_KEY_CONTEXT, rounds) + ctx_hasher.update(context_string.encode("utf-8") if isinstance(context_string, str) else context_string) + context_key = ctx_hasher.finalize(KEY_LEN) + return cls(words_from_le_bytes(context_key), DERIVE_KEY_MATERIAL, rounds) + + def _add_chunk_cv(self, new_cv, total_chunks): + # Merge the CV stack following the binary-tree structure. A completed + # subtree is merged whenever the total chunk count is even at that level. + while total_chunks & 1 == 0: + left = self.cv_stack.pop() + new_cv = _parent_output(left, new_cv, self.key_words, self.flags, + self.rounds).chaining_value() + total_chunks >>= 1 + self.cv_stack.append(new_cv) + + def update(self, data): + data = bytes(data) + while data: + if len(self.chunk_state.block) == BLOCK_LEN and \ + self.chunk_state.blocks_compressed == CHUNK_LEN // BLOCK_LEN - 1: + # current chunk is full: finalise it and start a new one. + chunk_cv = self.chunk_state.output().chaining_value() + total_chunks = self.chunk_state.chunk_counter + 1 + self._add_chunk_cv(chunk_cv, total_chunks) + self.chunk_state = _ChunkState(self.key_words, total_chunks, + self.flags, self.rounds) + # How many bytes still fit in the current chunk. + want = CHUNK_LEN - self._chunk_len() + take = min(want, len(data)) + self.chunk_state.update(data[:take]) + data = data[take:] + + def _chunk_len(self): + return self.chunk_state.blocks_compressed * BLOCK_LEN + len(self.chunk_state.block) + + def finalize(self, out_len=OUT_LEN): + # Walk the current chunk's output up the CV stack, XORing/parenting all + # the way to the root, and emit the root output. + output = self.chunk_state.output() + parent_nodes_remaining = len(self.cv_stack) + while parent_nodes_remaining > 0: + parent_nodes_remaining -= 1 + left = self.cv_stack[parent_nodes_remaining] + output = _parent_output(left, output.chaining_value(), + self.key_words, self.flags, self.rounds) + return output.root_output_bytes(out_len) + + +def blake3_hash(data, out_len=OUT_LEN, rounds=DEFAULT_ROUNDS): + h = Blake3Hasher.default(rounds) + h.update(data) + return h.finalize(out_len) + + +def blake3_keyed_hash(key32, data, out_len=OUT_LEN, rounds=DEFAULT_ROUNDS): + h = Blake3Hasher.keyed(key32, rounds) + h.update(data) + return h.finalize(out_len) + + +def blake3_derive_key(context_string, key_material, out_len=OUT_LEN, rounds=DEFAULT_ROUNDS): + h = Blake3Hasher.derive_key(context_string, rounds) + h.update(key_material) + return h.finalize(out_len) + + +if __name__ == "__main__": + # Tiny smoke test: empty-input default hash (compare to test_oracle.py). + print("blake3('') =", blake3_hash(b"").hex()) diff --git a/tooling/blake3-second-source/blake3-oracle/canonical_6round_vectors.json b/tooling/blake3-second-source/blake3-oracle/canonical_6round_vectors.json new file mode 100644 index 000000000..10ee8cbc1 --- /dev/null +++ b/tooling/blake3-second-source/blake3-oracle/canonical_6round_vectors.json @@ -0,0 +1,522 @@ +[ + { + "seed": 0, + "h": [ + 3626764237, + 1806341205, + 2195908194, + 2046968324, + 3900315155, + 2167613558, + 1210484339, + 3246154361 + ], + "m": [ + 3874773259, + 1332073689, + 3134603515, + 2937688618, + 432508404, + 1864753826, + 3921352636, + 2048741382, + 1118805955, + 60308648, + 3726325546, + 3738645480, + 2437440079, + 4155553746, + 1924014660, + 4006490763 + ], + "t": 13033757608824335107, + "block_len": 42, + "flags": 52, + "out": [ + 3470381567, + 3259559595, + 3171982207, + 2434484470, + 2453496512, + 3624177727, + 1500783166, + 2857307264, + 2908815487, + 3037433307, + 3879152609, + 869521091, + 1118447691, + 3315752744, + 2041348976, + 3076471061 + ] + }, + { + "seed": 1, + "h": [ + 3280387012, + 1095513148, + 1930549411, + 2798570523, + 3387541014, + 403123852, + 3589583794, + 1912923437 + ], + "m": [ + 4059906722, + 3871601465, + 131383004, + 2325348894, + 1001090105, + 92297589, + 2758633299, + 3693442237, + 2878940490, + 1302957853, + 3790218436, + 2170177477, + 148287319, + 3424825176, + 743061144, + 1609337231 + ], + "t": 14359731685826847253, + "block_len": 50, + "flags": 94, + "out": [ + 4071191273, + 2180888812, + 1086656188, + 1268894457, + 2666129712, + 1796871858, + 3910496071, + 2829038646, + 2734036659, + 310856722, + 813072437, + 3759806425, + 3202728316, + 3592162272, + 809631558, + 3753447717 + ] + }, + { + "seed": 2, + "h": [ + 242886303, + 364522461, + 3588440356, + 1323436024, + 2602510382, + 2606193617, + 4077622522, + 117874757 + ], + "m": [ + 1632151663, + 2258090960, + 2407373688, + 1014142328, + 102469680, + 1396478261, + 2191394736, + 3837860530, + 3422057796, + 3276568223, + 1519503515, + 4131333072, + 3238422834, + 2277860467, + 2104593779, + 3972123491 + ], + "t": 8504173462006699459, + "block_len": 58, + "flags": 124, + "out": [ + 1520873748, + 3386274828, + 2268646132, + 2891926386, + 2680601054, + 1060043663, + 2360846610, + 4103578245, + 1023812198, + 2132949004, + 2949933306, + 304921216, + 1147868525, + 2990135490, + 3286938319, + 2002726802 + ] + }, + { + "seed": 3, + "h": [ + 2337446730, + 2593816829, + 3596902313, + 1006443827, + 2045921456, + 646892613, + 2726705791, + 2247046192 + ], + "m": [ + 3183652505, + 275012945, + 2538753386, + 3717411168, + 3774472248, + 3956088670, + 4018314376, + 3774703581, + 418563100, + 583981819, + 931951836, + 1292897679, + 2512874164, + 2509342356, + 3883517040, + 3989790985 + ], + "t": 10579804601021778419, + "block_len": 36, + "flags": 31, + "out": [ + 4007210923, + 328045400, + 2438725180, + 326208257, + 3037127287, + 3191867341, + 897875462, + 3457968278, + 1392116149, + 1252158200, + 2970061409, + 743537389, + 2693293984, + 3933130730, + 1832113072, + 3693687142 + ] + }, + { + "seed": 4, + "h": [ + 1013818839, + 1701057193, + 665600858, + 285680177, + 3942586889, + 3286348376, + 2305023086, + 456053774 + ], + "m": [ + 3983477513, + 3464545456, + 3437897285, + 830799655, + 1330795424, + 3779789200, + 2602114036, + 2884935804, + 2173054921, + 763602979, + 2034044485, + 1289545638, + 3903568191, + 3789523705, + 2183442722, + 1777884721 + ], + "t": 5316417565031027709, + "block_len": 57, + "flags": 41, + "out": [ + 3389046179, + 2216925754, + 3888680557, + 866690006, + 165466574, + 2712732178, + 4102951254, + 2399377685, + 2315607722, + 4284158421, + 3072657499, + 773501543, + 1793536573, + 3003084712, + 1896007841, + 1969351479 + ] + }, + { + "seed": 5, + "h": [ + 2675342405, + 3185950873, + 4051686260, + 2787324501, + 3869338171, + 486215926, + 1059022248, + 2335435112 + ], + "m": [ + 2465058629, + 930847394, + 1200367645, + 3288765765, + 3423720279, + 2651938379, + 544169062, + 3742654890, + 4219466551, + 3746962816, + 1242556253, + 4129516530, + 879521323, + 2966284567, + 3838591282, + 1283288560 + ], + "t": 7653677975526109578, + "block_len": 18, + "flags": 67, + "out": [ + 346682991, + 270262248, + 2601144541, + 3997938779, + 2056340738, + 2008238187, + 1505739028, + 2712480509, + 3247758822, + 2303640909, + 2906048517, + 2417554421, + 375059928, + 1048950168, + 2028430931, + 3145353309 + ] + }, + { + "seed": 6, + "h": [ + 3530265750, + 1123655737, + 1940104, + 1602711601, + 3307725433, + 1171229348, + 3444200791, + 2929389929 + ], + "m": [ + 2945015643, + 3626164985, + 400010022, + 3437188107, + 3456510285, + 1250623880, + 4086115940, + 1547818437, + 3906320867, + 1552099921, + 2584484726, + 1307063374, + 2530408928, + 2255988210, + 2846451649, + 842776239 + ], + "t": 4108804320044427842, + "block_len": 26, + "flags": 3, + "out": [ + 849805168, + 3271909564, + 3519510472, + 4052162593, + 1913105236, + 2673574855, + 3059096669, + 2568909711, + 3012256441, + 251056470, + 2571889841, + 162028814, + 841094977, + 2913193055, + 1533365974, + 712663730 + ] + }, + { + "seed": 7, + "h": [ + 647892279, + 2795742288, + 2301595691, + 2179419893, + 161042648, + 1862494042, + 300026767, + 1823296038 + ], + "m": [ + 4070378921, + 1703729684, + 4192983756, + 3687093963, + 1243862422, + 776213899, + 2744112455, + 1599435267, + 884585951, + 1349251823, + 1946412080, + 1287489453, + 3411833895, + 1048386555, + 2467131055, + 2255701793 + ], + "t": 1350317716114554168, + "block_len": 53, + "flags": 42, + "out": [ + 2788339013, + 315507188, + 3524996285, + 1987664994, + 1810642625, + 3673881822, + 1405781943, + 2464695899, + 2067943261, + 3789991295, + 1966842759, + 3435464740, + 1773068141, + 3149656659, + 2026915971, + 4092802697 + ] + }, + { + "seed": 8, + "h": [ + 973694259, + 4133025703, + 542587089, + 3027165658, + 365867937, + 899355976, + 2756803948, + 1971964490 + ], + "m": [ + 1946188980, + 3567061697, + 384681428, + 1750902959, + 1109633622, + 270963824, + 1620083717, + 2838299811, + 1453582679, + 2969113350, + 3871375977, + 4063259978, + 832596604, + 2486621942, + 3783693026, + 3771309886 + ], + "t": 3645965004013668149, + "block_len": 19, + "flags": 91, + "out": [ + 2436573921, + 3354865794, + 1172422691, + 1864318850, + 548333301, + 3673300372, + 4072793263, + 3573011628, + 1151623047, + 4106489061, + 1631493012, + 147739614, + 1341160100, + 1164702434, + 543615615, + 1012557131 + ] + }, + { + "seed": 9, + "h": [ + 1603362544, + 595022250, + 27638352, + 2159432582, + 347096279, + 1627876803, + 3114132053, + 674984870 + ], + "m": [ + 1022254636, + 476516009, + 2535870938, + 1250600339, + 2895821580, + 901471249, + 1207677876, + 3476821989, + 3807057864, + 3776879099, + 2111885832, + 100859404, + 2563432515, + 2485498850, + 872106831, + 358645241 + ], + "t": 16927792517719413886, + "block_len": 64, + "flags": 21, + "out": [ + 4126052628, + 2238491576, + 700329201, + 1614539036, + 2494029070, + 687619623, + 3058576584, + 757884927, + 1778041274, + 211062928, + 3599623221, + 3465651495, + 3893106709, + 1833234406, + 2278011253, + 3515517844 + ] + } +] \ No newline at end of file diff --git a/tooling/blake3-second-source/blake3-oracle/official_test_vectors.json b/tooling/blake3-second-source/blake3-oracle/official_test_vectors.json new file mode 100644 index 000000000..77cd38adb --- /dev/null +++ b/tooling/blake3-second-source/blake3-oracle/official_test_vectors.json @@ -0,0 +1,334 @@ +{ + "key": "whats the Elvish word for friend", + "context_string": "BLAKE3 2019-12-27 16:29:52 test vectors context", + "cases": [ + { + "input_len": 0, + "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421cce14d", + "keyed_hash": "92b2b75604ed3c761f9d6f62392c8a9227ad0ea3f09573e783f1498a4ed60d26b18171a2f22a4b94822c701f107153dba24918c4bae4d2945c20ece13387627d3b73cbf97b797d5e59948c7ef788f54372df45e45e4293c7dc18c1d41144a9758be58960856be1eabbe22c2653190de560ca3b2ac4aa692a9210694254c371e851bc8f", + "derive_key": "2cc39783c223154fea8dfb7c1b1660f2ac2dcbd1c1de8277b0b0dd39b7e50d7d905630c8be290dfcf3e6842f13bddd573c098c3f17361f1f206b8cad9d088aa4a3f746752c6b0ce6a83b0da81d59649257cdf8eb3e9f7d4998e41021fac119deefb896224ac99f860011f73609e6e0e4540f93b273e56547dfd3aa1a035ba6689d89a0" + }, + { + "input_len": 1, + "hash": "2d3adedff11b61f14c886e35afa036736dcd87a74d27b5c1510225d0f592e213c3a6cb8bf623e20cdb535f8d1a5ffb86342d9c0b64aca3bce1d31f60adfa137b358ad4d79f97b47c3d5e79f179df87a3b9776ef8325f8329886ba42f07fb138bb502f4081cbcec3195c5871e6c23e2cc97d3c69a613eba131e5f1351f3f1da786545e5", + "keyed_hash": "6d7878dfff2f485635d39013278ae14f1454b8c0a3a2d34bc1ab38228a80c95b6568c0490609413006fbd428eb3fd14e7756d90f73a4725fad147f7bf70fd61c4e0cf7074885e92b0e3f125978b4154986d4fb202a3f331a3fb6cf349a3a70e49990f98fe4289761c8602c4e6ab1138d31d3b62218078b2f3ba9a88e1d08d0dd4cea11", + "derive_key": "b3e2e340a117a499c6cf2398a19ee0d29cca2bb7404c73063382693bf66cb06c5827b91bf889b6b97c5477f535361caefca0b5d8c4746441c57617111933158950670f9aa8a05d791daae10ac683cbef8faf897c84e6114a59d2173c3f417023a35d6983f2c7dfa57e7fc559ad751dbfb9ffab39c2ef8c4aafebc9ae973a64f0c76551" + }, + { + "input_len": 2, + "hash": "7b7015bb92cf0b318037702a6cdd81dee41224f734684c2c122cd6359cb1ee63d8386b22e2ddc05836b7c1bb693d92af006deb5ffbc4c70fb44d0195d0c6f252faac61659ef86523aa16517f87cb5f1340e723756ab65efb2f91964e14391de2a432263a6faf1d146937b35a33621c12d00be8223a7f1919cec0acd12097ff3ab00ab1", + "keyed_hash": "5392ddae0e0a69d5f40160462cbd9bd889375082ff224ac9c758802b7a6fd20a9ffbf7efd13e989a6c246f96d3a96b9d279f2c4e63fb0bdff633957acf50ee1a5f658be144bab0f6f16500dee4aa5967fc2c586d85a04caddec90fffb7633f46a60786024353b9e5cebe277fcd9514217fee2267dcda8f7b31697b7c54fab6a939bf8f", + "derive_key": "1f166565a7df0098ee65922d7fea425fb18b9943f19d6161e2d17939356168e6daa59cae19892b2d54f6fc9f475d26031fd1c22ae0a3e8ef7bdb23f452a15e0027629d2e867b1bb1e6ab21c71297377750826c404dfccc2406bd57a83775f89e0b075e59a7732326715ef912078e213944f490ad68037557518b79c0086de6d6f6cdd2" + }, + { + "input_len": 3, + "hash": "e1be4d7a8ab5560aa4199eea339849ba8e293d55ca0a81006726d184519e647f5b49b82f805a538c68915c1ae8035c900fd1d4b13902920fd05e1450822f36de9454b7e9996de4900c8e723512883f93f4345f8a58bfe64ee38d3ad71ab027765d25cdd0e448328a8e7a683b9a6af8b0af94fa09010d9186890b096a08471e4230a134", + "keyed_hash": "39e67b76b5a007d4921969779fe666da67b5213b096084ab674742f0d5ec62b9b9142d0fab08e1b161efdbb28d18afc64d8f72160c958e53a950cdecf91c1a1bbab1a9c0f01def762a77e2e8545d4dec241e98a89b6db2e9a5b070fc110caae2622690bd7b76c02ab60750a3ea75426a6bb8803c370ffe465f07fb57def95df772c39f", + "derive_key": "440aba35cb006b61fc17c0529255de438efc06a8c9ebf3f2ddac3b5a86705797f27e2e914574f4d87ec04c379e12789eccbfbc15892626042707802dbe4e97c3ff59dca80c1e54246b6d055154f7348a39b7d098b2b4824ebe90e104e763b2a447512132cede16243484a55a4e40a85790038bb0dcf762e8c053cabae41bbe22a5bff7" + }, + { + "input_len": 4, + "hash": "f30f5ab28fe047904037f77b6da4fea1e27241c5d132638d8bedce9d40494f328f603ba4564453e06cdcee6cbe728a4519bbe6f0d41e8a14b5b225174a566dbfa61b56afb1e452dc08c804f8c3143c9e2cc4a31bb738bf8c1917b55830c6e65797211701dc0b98daa1faeaa6ee9e56ab606ce03a1a881e8f14e87a4acf4646272cfd12", + "keyed_hash": "7671dde590c95d5ac9616651ff5aa0a27bee5913a348e053b8aa9108917fe070116c0acff3f0d1fa97ab38d813fd46506089118147d83393019b068a55d646251ecf81105f798d76a10ae413f3d925787d6216a7eb444e510fd56916f1d753a5544ecf0072134a146b2615b42f50c179f56b8fae0788008e3e27c67482349e249cb86a", + "derive_key": "f46085c8190d69022369ce1a18880e9b369c135eb93f3c63550d3e7630e91060fbd7d8f4258bec9da4e05044f88b91944f7cab317a2f0c18279629a3867fad0662c9ad4d42c6f27e5b124da17c8c4f3a94a025ba5d1b623686c6099d202a7317a82e3d95dae46a87de0555d727a5df55de44dab799a20dffe239594d6e99ed17950910" + }, + { + "input_len": 5, + "hash": "b40b44dfd97e7a84a996a91af8b85188c66c126940ba7aad2e7ae6b385402aa2ebcfdac6c5d32c31209e1f81a454751280db64942ce395104e1e4eaca62607de1c2ca748251754ea5bbe8c20150e7f47efd57012c63b3c6a6632dc1c7cd15f3e1c999904037d60fac2eb9397f2adbe458d7f264e64f1e73aa927b30988e2aed2f03620", + "keyed_hash": "73ac69eecf286894d8102018a6fc729f4b1f4247d3703f69bdc6a5fe3e0c84616ab199d1f2f3e53bffb17f0a2209fe8b4f7d4c7bae59c2bc7d01f1ff94c67588cc6b38fa6024886f2c078bfe09b5d9e6584cd6c521c3bb52f4de7687b37117a2dbbec0d59e92fa9a8cc3240d4432f91757aabcae03e87431dac003e7d73574bfdd8218", + "derive_key": "1f24eda69dbcb752847ec3ebb5dd42836d86e58500c7c98d906ecd82ed9ae47f6f48a3f67e4e43329c9a89b1ca526b9b35cbf7d25c1e353baffb590fd79be58ddb6c711f1a6b60e98620b851c688670412fcb0435657ba6b638d21f0f2a04f2f6b0bd8834837b10e438d5f4c7c2c71299cf7586ea9144ed09253d51f8f54dd6bff719d" + }, + { + "input_len": 6, + "hash": "06c4e8ffb6872fad96f9aaca5eee1553eb62aed0ad7198cef42e87f6a616c844611a30c4e4f37fe2fe23c0883cde5cf7059d88b657c7ed2087e3d210925ede716435d6d5d82597a1e52b9553919e804f5656278bd739880692c94bff2824d8e0b48cac1d24682699e4883389dc4f2faa2eb3b4db6e39debd5061ff3609916f3e07529a", + "keyed_hash": "82d3199d0013035682cc7f2a399d4c212544376a839aa863a0f4c91220ca7a6dc2ffb3aa05f2631f0fa9ac19b6e97eb7e6669e5ec254799350c8b8d189e8807800842a5383c4d907c932f34490aaf00064de8cdb157357bde37c1504d2960034930887603abc5ccb9f5247f79224baff6120a3c622a46d7b1bcaee02c5025460941256", + "derive_key": "be96b30b37919fe4379dfbe752ae77b4f7e2ab92f7ff27435f76f2f065f6a5f435ae01a1d14bd5a6b3b69d8cbd35f0b01ef2173ff6f9b640ca0bd4748efa398bf9a9c0acd6a66d9332fdc9b47ffe28ba7ab6090c26747b85f4fab22f936b71eb3f64613d8bd9dfabe9bb68da19de78321b481e5297df9e40ec8a3d662f3e1479c65de0" + }, + { + "input_len": 7, + "hash": "3f8770f387faad08faa9d8414e9f449ac68e6ff0417f673f602a646a891419fe66036ef6e6d1a8f54baa9fed1fc11c77cfb9cff65bae915045027046ebe0c01bf5a941f3bb0f73791d3fc0b84370f9f30af0cd5b0fc334dd61f70feb60dad785f070fef1f343ed933b49a5ca0d16a503f599a365a4296739248b28d1a20b0e2cc8975c", + "keyed_hash": "af0a7ec382aedc0cfd626e49e7628bc7a353a4cb108855541a5651bf64fbb28a7c5035ba0f48a9c73dabb2be0533d02e8fd5d0d5639a18b2803ba6bf527e1d145d5fd6406c437b79bcaad6c7bdf1cf4bd56a893c3eb9510335a7a798548c6753f74617bede88bef924ba4b334f8852476d90b26c5dc4c3668a2519266a562c6c8034a6", + "derive_key": "dc3b6485f9d94935329442916b0d059685ba815a1fa2a14107217453a7fc9f0e66266db2ea7c96843f9d8208e600a73f7f45b2f55b9e6d6a7ccf05daae63a3fdd10b25ac0bd2e224ce8291f88c05976d575df998477db86fb2cfbbf91725d62cb57acfeb3c2d973b89b503c2b60dde85a7802b69dc1ac2007d5623cbea8cbfb6b181f5" + }, + { + "input_len": 8, + "hash": "2351207d04fc16ade43ccab08600939c7c1fa70a5c0aaca76063d04c3228eaeb725d6d46ceed8f785ab9f2f9b06acfe398c6699c6129da084cb531177445a682894f9685eaf836999221d17c9a64a3a057000524cd2823986db378b074290a1a9b93a22e135ed2c14c7e20c6d045cd00b903400374126676ea78874d79f2dd7883cf5c", + "keyed_hash": "be2f5495c61cba1bb348a34948c004045e3bd4dae8f0fe82bf44d0da245a060048eb5e68ce6dea1eb0229e144f578b3aa7e9f4f85febd135df8525e6fe40c6f0340d13dd09b255ccd5112a94238f2be3c0b5b7ecde06580426a93e0708555a265305abf86d874e34b4995b788e37a823491f25127a502fe0704baa6bfdf04e76c13276", + "derive_key": "2b166978cef14d9d438046c720519d8b1cad707e199746f1562d0c87fbd32940f0e2545a96693a66654225ebbaac76d093bfa9cd8f525a53acb92a861a98c42e7d1c4ae82e68ab691d510012edd2a728f98cd4794ef757e94d6546961b4f280a51aac339cc95b64a92b83cc3f26d8af8dfb4c091c240acdb4d47728d23e7148720ef04" + }, + { + "input_len": 63, + "hash": "e9bc37a594daad83be9470df7f7b3798297c3d834ce80ba85d6e207627b7db7b1197012b1e7d9af4d7cb7bdd1f3bb49a90a9b5dec3ea2bbc6eaebce77f4e470cbf4687093b5352f04e4a4570fba233164e6acc36900e35d185886a827f7ea9bdc1e5c3ce88b095a200e62c10c043b3e9bc6cb9b6ac4dfa51794b02ace9f98779040755", + "keyed_hash": "bb1eb5d4afa793c1ebdd9fb08def6c36d10096986ae0cfe148cd101170ce37aea05a63d74a840aecd514f654f080e51ac50fd617d22610d91780fe6b07a26b0847abb38291058c97474ef6ddd190d30fc318185c09ca1589d2024f0a6f16d45f11678377483fa5c005b2a107cb9943e5da634e7046855eaa888663de55d6471371d55d", + "derive_key": "b6451e30b953c206e34644c6803724e9d2725e0893039cfc49584f991f451af3b89e8ff572d3da4f4022199b9563b9d70ebb616efff0763e9abec71b550f1371e233319c4c4e74da936ba8e5bbb29a598e007a0bbfa929c99738ca2cc098d59134d11ff300c39f82e2fce9f7f0fa266459503f64ab9913befc65fddc474f6dc1c67669" + }, + { + "input_len": 64, + "hash": "4eed7141ea4a5cd4b788606bd23f46e212af9cacebacdc7d1f4c6dc7f2511b98fc9cc56cb831ffe33ea8e7e1d1df09b26efd2767670066aa82d023b1dfe8ab1b2b7fbb5b97592d46ffe3e05a6a9b592e2949c74160e4674301bc3f97e04903f8c6cf95b863174c33228924cdef7ae47559b10b294acd660666c4538833582b43f82d74", + "keyed_hash": "ba8ced36f327700d213f120b1a207a3b8c04330528586f414d09f2f7d9ccb7e68244c26010afc3f762615bbac552a1ca909e67c83e2fd5478cf46b9e811efccc93f77a21b17a152ebaca1695733fdb086e23cd0eb48c41c034d52523fc21236e5d8c9255306e48d52ba40b4dac24256460d56573d1312319afcf3ed39d72d0bfc69acb", + "derive_key": "a5c4a7053fa86b64746d4bb688d06ad1f02a18fce9afd3e818fefaa7126bf73e9b9493a9befebe0bf0c9509fb3105cfa0e262cde141aa8e3f2c2f77890bb64a4cca96922a21ead111f6338ad5244f2c15c44cb595443ac2ac294231e31be4a4307d0a91e874d36fc9852aeb1265c09b6e0cda7c37ef686fbbcab97e8ff66718be048bb" + }, + { + "input_len": 65, + "hash": "de1e5fa0be70df6d2be8fffd0e99ceaa8eb6e8c93a63f2d8d1c30ecb6b263dee0e16e0a4749d6811dd1d6d1265c29729b1b75a9ac346cf93f0e1d7296dfcfd4313b3a227faaaaf7757cc95b4e87a49be3b8a270a12020233509b1c3632b3485eef309d0abc4a4a696c9decc6e90454b53b000f456a3f10079072baaf7a981653221f2c", + "keyed_hash": "c0a4edefa2d2accb9277c371ac12fcdbb52988a86edc54f0716e1591b4326e72d5e795f46a596b02d3d4bfb43abad1e5d19211152722ec1f20fef2cd413e3c22f2fc5da3d73041275be6ede3517b3b9f0fc67ade5956a672b8b75d96cb43294b9041497de92637ed3f2439225e683910cb3ae923374449ca788fb0f9bea92731bc26ad", + "derive_key": "51fd05c3c1cfbc8ed67d139ad76f5cf8236cd2acd26627a30c104dfd9d3ff8a82b02e8bd36d8498a75ad8c8e9b15eb386970283d6dd42c8ae7911cc592887fdbe26a0a5f0bf821cd92986c60b2502c9be3f98a9c133a7e8045ea867e0828c7252e739321f7c2d65daee4468eb4429efae469a42763f1f94977435d10dccae3e3dce88d" + }, + { + "input_len": 127, + "hash": "d81293fda863f008c09e92fc382a81f5a0b4a1251cba1634016a0f86a6bd640de3137d477156d1fde56b0cf36f8ef18b44b2d79897bece12227539ac9ae0a5119da47644d934d26e74dc316145dcb8bb69ac3f2e05c242dd6ee06484fcb0e956dc44355b452c5e2bbb5e2b66e99f5dd443d0cbcaaafd4beebaed24ae2f8bb672bcef78", + "keyed_hash": "c64200ae7dfaf35577ac5a9521c47863fb71514a3bcad18819218b818de85818ee7a317aaccc1458f78d6f65f3427ec97d9c0adb0d6dacd4471374b621b7b5f35cd54663c64dbe0b9e2d95632f84c611313ea5bd90b71ce97b3cf645776f3adc11e27d135cbadb9875c2bf8d3ae6b02f8a0206aba0c35bfe42574011931c9a255ce6dc", + "derive_key": "c91c090ceee3a3ac81902da31838012625bbcd73fcb92e7d7e56f78deba4f0c3feeb3974306966ccb3e3c69c337ef8a45660ad02526306fd685c88542ad00f759af6dd1adc2e50c2b8aac9f0c5221ff481565cf6455b772515a69463223202e5c371743e35210bbbbabd89651684107fd9fe493c937be16e39cfa7084a36207c99bea3" + }, + { + "input_len": 128, + "hash": "f17e570564b26578c33bb7f44643f539624b05df1a76c81f30acd548c44b45efa69faba091427f9c5c4caa873aa07828651f19c55bad85c47d1368b11c6fd99e47ecba5820a0325984d74fe3e4058494ca12e3f1d3293d0010a9722f7dee64f71246f75e9361f44cc8e214a100650db1313ff76a9f93ec6e84edb7add1cb4a95019b0c", + "keyed_hash": "b04fe15577457267ff3b6f3c947d93be581e7e3a4b018679125eaf86f6a628ecd86bbe0001f10bda47e6077b735016fca8119da11348d93ca302bbd125bde0db2b50edbe728a620bb9d3e6f706286aedea973425c0b9eedf8a38873544cf91badf49ad92a635a93f71ddfcee1eae536c25d1b270956be16588ef1cfef2f1d15f650bd5", + "derive_key": "81720f34452f58a0120a58b6b4608384b5c51d11f39ce97161a0c0e442ca022550e7cd651e312f0b4c6afb3c348ae5dd17d2b29fab3b894d9a0034c7b04fd9190cbd90043ff65d1657bbc05bfdecf2897dd894c7a1b54656d59a50b51190a9da44db426266ad6ce7c173a8c0bbe091b75e734b4dadb59b2861cd2518b4e7591e4b83c9" + }, + { + "input_len": 129, + "hash": "683aaae9f3c5ba37eaaf072aed0f9e30bac0865137bae68b1fde4ca2aebdcb12f96ffa7b36dd78ba321be7e842d364a62a42e3746681c8bace18a4a8a79649285c7127bf8febf125be9de39586d251f0d41da20980b70d35e3dac0eee59e468a894fa7e6a07129aaad09855f6ad4801512a116ba2b7841e6cfc99ad77594a8f2d181a7", + "keyed_hash": "d4a64dae6cdccbac1e5287f54f17c5f985105457c1a2ec1878ebd4b57e20d38f1c9db018541eec241b748f87725665b7b1ace3e0065b29c3bcb232c90e37897fa5aaee7e1e8a2ecfcd9b51463e42238cfdd7fee1aecb3267fa7f2128079176132a412cd8aaf0791276f6b98ff67359bd8652ef3a203976d5ff1cd41885573487bcd683", + "derive_key": "938d2d4435be30eafdbb2b7031f7857c98b04881227391dc40db3c7b21f41fc18d72d0f9c1de5760e1941aebf3100b51d64644cb459eb5d20258e233892805eb98b07570ef2a1787cd48e117c8d6a63a68fd8fc8e59e79dbe63129e88352865721c8d5f0cf183f85e0609860472b0d6087cefdd186d984b21542c1c780684ed6832d8d" + }, + { + "input_len": 1023, + "hash": "10108970eeda3eb932baac1428c7a2163b0e924c9a9e25b35bba72b28f70bd11a182d27a591b05592b15607500e1e8dd56bc6c7fc063715b7a1d737df5bad3339c56778957d870eb9717b57ea3d9fb68d1b55127bba6a906a4a24bbd5acb2d123a37b28f9e9a81bbaae360d58f85e5fc9d75f7c370a0cc09b6522d9c8d822f2f28f485", + "keyed_hash": "c951ecdf03288d0fcc96ee3413563d8a6d3589547f2c2fb36d9786470f1b9d6e890316d2e6d8b8c25b0a5b2180f94fb1a158ef508c3cde45e2966bd796a696d3e13efd86259d756387d9becf5c8bf1ce2192b87025152907b6d8cc33d17826d8b7b9bc97e38c3c85108ef09f013e01c229c20a83d9e8efac5b37470da28575fd755a10", + "derive_key": "74a16c1c3d44368a86e1ca6df64be6a2f64cce8f09220787450722d85725dea59c413264404661e9e4d955409dfe4ad3aa487871bcd454ed12abfe2c2b1eb7757588cf6cb18d2eccad49e018c0d0fec323bec82bf1644c6325717d13ea712e6840d3e6e730d35553f59eff5377a9c350bcc1556694b924b858f329c44ee64b884ef00d" + }, + { + "input_len": 1024, + "hash": "42214739f095a406f3fc83deb889744ac00df831c10daa55189b5d121c855af71cf8107265ecdaf8505b95d8fcec83a98a6a96ea5109d2c179c47a387ffbb404756f6eeae7883b446b70ebb144527c2075ab8ab204c0086bb22b7c93d465efc57f8d917f0b385c6df265e77003b85102967486ed57db5c5ca170ba441427ed9afa684e", + "keyed_hash": "75c46f6f3d9eb4f55ecaaee480db732e6c2105546f1e675003687c31719c7ba4a78bc838c72852d4f49c864acb7adafe2478e824afe51c8919d06168414c265f298a8094b1ad813a9b8614acabac321f24ce61c5a5346eb519520d38ecc43e89b5000236df0597243e4d2493fd626730e2ba17ac4d8824d09d1a4a8f57b8227778e2de", + "derive_key": "7356cd7720d5b66b6d0697eb3177d9f8d73a4a5c5e968896eb6a6896843027066c23b601d3ddfb391e90d5c8eccdef4ae2a264bce9e612ba15e2bc9d654af1481b2e75dbabe615974f1070bba84d56853265a34330b4766f8e75edd1f4a1650476c10802f22b64bd3919d246ba20a17558bc51c199efdec67e80a227251808d8ce5bad" + }, + { + "input_len": 1025, + "hash": "d00278ae47eb27b34faecf67b4fe263f82d5412916c1ffd97c8cb7fb814b8444f4c4a22b4b399155358a994e52bf255de60035742ec71bd08ac275a1b51cc6bfe332b0ef84b409108cda080e6269ed4b3e2c3f7d722aa4cdc98d16deb554e5627be8f955c98e1d5f9565a9194cad0c4285f93700062d9595adb992ae68ff12800ab67a", + "keyed_hash": "357dc55de0c7e382c900fd6e320acc04146be01db6a8ce7210b7189bd664ea69362396b77fdc0d2634a552970843722066c3c15902ae5097e00ff53f1e116f1cd5352720113a837ab2452cafbde4d54085d9cf5d21ca613071551b25d52e69d6c81123872b6f19cd3bc1333edf0c52b94de23ba772cf82636cff4542540a7738d5b930", + "derive_key": "effaa245f065fbf82ac186839a249707c3bddf6d3fdda22d1b95a3c970379bcb5d31013a167509e9066273ab6e2123bc835b408b067d88f96addb550d96b6852dad38e320b9d940f86db74d398c770f462118b35d2724efa13da97194491d96dd37c3c09cbef665953f2ee85ec83d88b88d11547a6f911c8217cca46defa2751e7f3ad" + }, + { + "input_len": 2048, + "hash": "e776b6028c7cd22a4d0ba182a8bf62205d2ef576467e838ed6f2529b85fba24a9a60bf80001410ec9eea6698cd537939fad4749edd484cb541aced55cd9bf54764d063f23f6f1e32e12958ba5cfeb1bf618ad094266d4fc3c968c2088f677454c288c67ba0dba337b9d91c7e1ba586dc9a5bc2d5e90c14f53a8863ac75655461cea8f9", + "keyed_hash": "879cf1fa2ea0e79126cb1063617a05b6ad9d0b696d0d757cf053439f60a99dd10173b961cd574288194b23ece278c330fbb8585485e74967f31352a8183aa782b2b22f26cdcadb61eed1a5bc144b8198fbb0c13abbf8e3192c145d0a5c21633b0ef86054f42809df823389ee40811a5910dcbd1018af31c3b43aa55201ed4edaac74fe", + "derive_key": "7b2945cb4fef70885cc5d78a87bf6f6207dd901ff239201351ffac04e1088a23e2c11a1ebffcea4d80447867b61badb1383d842d4e79645d48dd82ccba290769caa7af8eaa1bd78a2a5e6e94fbdab78d9c7b74e894879f6a515257ccf6f95056f4e25390f24f6b35ffbb74b766202569b1d797f2d4bd9d17524c720107f985f4ddc583" + }, + { + "input_len": 2049, + "hash": "5f4d72f40d7a5f82b15ca2b2e44b1de3c2ef86c426c95c1af0b687952256303096de31d71d74103403822a2e0bc1eb193e7aecc9643a76b7bbc0c9f9c52e8783aae98764ca468962b5c2ec92f0c74eb5448d519713e09413719431c802f948dd5d90425a4ecdadece9eb178d80f26efccae630734dff63340285adec2aed3b51073ad3", + "keyed_hash": "9f29700902f7c86e514ddc4df1e3049f258b2472b6dd5267f61bf13983b78dd5f9a88abfefdfa1e00b418971f2b39c64ca621e8eb37fceac57fd0c8fc8e117d43b81447be22d5d8186f8f5919ba6bcc6846bd7d50726c06d245672c2ad4f61702c646499ee1173daa061ffe15bf45a631e2946d616a4c345822f1151284712f76b2b0e", + "derive_key": "2ea477c5515cc3dd606512ee72bb3e0e758cfae7232826f35fb98ca1bcbdf27316d8e9e79081a80b046b60f6a263616f33ca464bd78d79fa18200d06c7fc9bffd808cc4755277a7d5e09da0f29ed150f6537ea9bed946227ff184cc66a72a5f8c1e4bd8b04e81cf40fe6dc4427ad5678311a61f4ffc39d195589bdbc670f63ae70f4b6" + }, + { + "input_len": 3072, + "hash": "b98cb0ff3623be03326b373de6b9095218513e64f1ee2edd2525c7ad1e5cffd29a3f6b0b978d6608335c09dc94ccf682f9951cdfc501bfe47b9c9189a6fc7b404d120258506341a6d802857322fbd20d3e5dae05b95c88793fa83db1cb08e7d8008d1599b6209d78336e24839724c191b2a52a80448306e0daa84a3fdb566661a37e11", + "keyed_hash": "044a0e7b172a312dc02a4c9a818c036ffa2776368d7f528268d2e6b5df19177022f302d0529e4174cc507c463671217975e81dab02b8fdeb0d7ccc7568dd22574c783a76be215441b32e91b9a904be8ea81f7a0afd14bad8ee7c8efc305ace5d3dd61b996febe8da4f56ca0919359a7533216e2999fc87ff7d8f176fbecb3d6f34278b", + "derive_key": "050df97f8c2ead654d9bb3ab8c9178edcd902a32f8495949feadcc1e0480c46b3604131bbd6e3ba573b6dd682fa0a63e5b165d39fc43a625d00207607a2bfeb65ff1d29292152e26b298868e3b87be95d6458f6f2ce6118437b632415abe6ad522874bcd79e4030a5e7bad2efa90a7a7c67e93f0a18fb28369d0a9329ab5c24134ccb0" + }, + { + "input_len": 3073, + "hash": "7124b49501012f81cc7f11ca069ec9226cecb8a2c850cfe644e327d22d3e1cd39a27ae3b79d68d89da9bf25bc27139ae65a324918a5f9b7828181e52cf373c84f35b639b7fccbb985b6f2fa56aea0c18f531203497b8bbd3a07ceb5926f1cab74d14bd66486d9a91eba99059a98bd1cd25876b2af5a76c3e9eed554ed72ea952b603bf", + "keyed_hash": "68dede9bef00ba89e43f31a6825f4cf433389fedae75c04ee9f0cf16a427c95a96d6da3fe985054d3478865be9a092250839a697bbda74e279e8a9e69f0025e4cfddd6cfb434b1cd9543aaf97c635d1b451a4386041e4bb100f5e45407cbbc24fa53ea2de3536ccb329e4eb9466ec37093a42cf62b82903c696a93a50b702c80f3c3c5", + "derive_key": "72613c9ec9ff7e40f8f5c173784c532ad852e827dba2bf85b2ab4b76f7079081576288e552647a9d86481c2cae75c2dd4e7c5195fb9ada1ef50e9c5098c249d743929191441301c69e1f48505a4305ec1778450ee48b8e69dc23a25960fe33070ea549119599760a8a2d28aeca06b8c5e9ba58bc19e11fe57b6ee98aa44b2a8e6b14a5" + }, + { + "input_len": 4096, + "hash": "015094013f57a5277b59d8475c0501042c0b642e531b0a1c8f58d2163229e9690289e9409ddb1b99768eafe1623da896faf7e1114bebeadc1be30829b6f8af707d85c298f4f0ff4d9438aef948335612ae921e76d411c3a9111df62d27eaf871959ae0062b5492a0feb98ef3ed4af277f5395172dbe5c311918ea0074ce0036454f620", + "keyed_hash": "befc660aea2f1718884cd8deb9902811d332f4fc4a38cf7c7300d597a081bfc0bbb64a36edb564e01e4b4aaf3b060092a6b838bea44afebd2deb8298fa562b7b597c757b9df4c911c3ca462e2ac89e9a787357aaf74c3b56d5c07bc93ce899568a3eb17d9250c20f6c5f6c1e792ec9a2dcb715398d5a6ec6d5c54f586a00403a1af1de", + "derive_key": "1e0d7f3db8c414c97c6307cbda6cd27ac3b030949da8e23be1a1a924ad2f25b9d78038f7b198596c6cc4a9ccf93223c08722d684f240ff6569075ed81591fd93f9fff1110b3a75bc67e426012e5588959cc5a4c192173a03c00731cf84544f65a2fb9378989f72e9694a6a394a8a30997c2e67f95a504e631cd2c5f55246024761b245" + }, + { + "input_len": 4097, + "hash": "9b4052b38f1c5fc8b1f9ff7ac7b27cd242487b3d890d15c96a1c25b8aa0fb99505f91b0b5600a11251652eacfa9497b31cd3c409ce2e45cfe6c0a016967316c426bd26f619eab5d70af9a418b845c608840390f361630bd497b1ab44019316357c61dbe091ce72fc16dc340ac3d6e009e050b3adac4b5b2c92e722cffdc46501531956", + "keyed_hash": "00df940cd36bb9fa7cbbc3556744e0dbc8191401afe70520ba292ee3ca80abbc606db4976cfdd266ae0abf667d9481831ff12e0caa268e7d3e57260c0824115a54ce595ccc897786d9dcbf495599cfd90157186a46ec800a6763f1c59e36197e9939e900809f7077c102f888caaf864b253bc41eea812656d46742e4ea42769f89b83f", + "derive_key": "aca51029626b55fda7117b42a7c211f8c6e9ba4fe5b7a8ca922f34299500ead8a897f66a400fed9198fd61dd2d58d382458e64e100128075fc54b860934e8de2e84170734b06e1d212a117100820dbc48292d148afa50567b8b84b1ec336ae10d40c8c975a624996e12de31abbe135d9d159375739c333798a80c64ae895e51e22f3ad" + }, + { + "input_len": 5120, + "hash": "9cadc15fed8b5d854562b26a9536d9707cadeda9b143978f319ab34230535833acc61c8fdc114a2010ce8038c853e121e1544985133fccdd0a2d507e8e615e611e9a0ba4f47915f49e53d721816a9198e8b30f12d20ec3689989175f1bf7a300eee0d9321fad8da232ece6efb8e9fd81b42ad161f6b9550a069e66b11b40487a5f5059", + "keyed_hash": "2c493e48e9b9bf31e0553a22b23503c0a3388f035cece68eb438d22fa1943e209b4dc9209cd80ce7c1f7c9a744658e7e288465717ae6e56d5463d4f80cdb2ef56495f6a4f5487f69749af0c34c2cdfa857f3056bf8d807336a14d7b89bf62bef2fb54f9af6a546f818dc1e98b9e07f8a5834da50fa28fb5874af91bf06020d1bf0120e", + "derive_key": "7a7acac8a02adcf3038d74cdd1d34527de8a0fcc0ee3399d1262397ce5817f6055d0cefd84d9d57fe792d65a278fd20384ac6c30fdb340092f1a74a92ace99c482b28f0fc0ef3b923e56ade20c6dba47e49227166251337d80a037e987ad3a7f728b5ab6dfafd6e2ab1bd583a95d9c895ba9c2422c24ea0f62961f0dca45cad47bfa0d" + }, + { + "input_len": 5121, + "hash": "628bd2cb2004694adaab7bbd778a25df25c47b9d4155a55f8fbd79f2fe154cff96adaab0613a6146cdaabe498c3a94e529d3fc1da2bd08edf54ed64d40dcd6777647eac51d8277d70219a9694334a68bc8f0f23e20b0ff70ada6f844542dfa32cd4204ca1846ef76d811cdb296f65e260227f477aa7aa008bac878f72257484f2b6c95", + "keyed_hash": "6ccf1c34753e7a044db80798ecd0782a8f76f33563accaddbfbb2e0ea4b2d0240d07e63f13667a8d1490e5e04f13eb617aea16a8c8a5aaed1ef6fbde1b0515e3c81050b361af6ead126032998290b563e3caddeaebfab592e155f2e161fb7cba939092133f23f9e65245e58ec23457b78a2e8a125588aad6e07d7f11a85b88d375b72d", + "derive_key": "b07f01e518e702f7ccb44a267e9e112d403a7b3f4883a47ffbed4b48339b3c341a0add0ac032ab5aaea1e4e5b004707ec5681ae0fcbe3796974c0b1cf31a194740c14519273eedaabec832e8a784b6e7cfc2c5952677e6c3f2c3914454082d7eb1ce1766ac7d75a4d3001fc89544dd46b5147382240d689bbbaefc359fb6ae30263165" + }, + { + "input_len": 6144, + "hash": "3e2e5b74e048f3add6d21faab3f83aa44d3b2278afb83b80b3c35164ebeca2054d742022da6fdda444ebc384b04a54c3ac5839b49da7d39f6d8a9db03deab32aade156c1c0311e9b3435cde0ddba0dce7b26a376cad121294b689193508dd63151603c6ddb866ad16c2ee41585d1633a2cea093bea714f4c5d6b903522045b20395c83", + "keyed_hash": "3d6b6d21281d0ade5b2b016ae4034c5dec10ca7e475f90f76eac7138e9bc8f1dc35754060091dc5caf3efabe0603c60f45e415bb3407db67e6beb3d11cf8e4f7907561f05dace0c15807f4b5f389c841eb114d81a82c02a00b57206b1d11fa6e803486b048a5ce87105a686dee041207e095323dfe172df73deb8c9532066d88f9da7e", + "derive_key": "2a95beae63ddce523762355cf4b9c1d8f131465780a391286a5d01abb5683a1597099e3c6488aab6c48f3c15dbe1942d21dbcdc12115d19a8b8465fb54e9053323a9178e4275647f1a9927f6439e52b7031a0b465c861a3fc531527f7758b2b888cf2f20582e9e2c593709c0a44f9c6e0f8b963994882ea4168827823eef1f64169fef" + }, + { + "input_len": 6145, + "hash": "f1323a8631446cc50536a9f705ee5cb619424d46887f3c376c695b70e0f0507f18a2cfdd73c6e39dd75ce7c1c6e3ef238fd54465f053b25d21044ccb2093beb015015532b108313b5829c3621ce324b8e14229091b7c93f32db2e4e63126a377d2a63a3597997d4f1cba59309cb4af240ba70cebff9a23d5e3ff0cdae2cfd54e070022", + "keyed_hash": "9ac301e9e39e45e3250a7e3b3df701aa0fb6889fbd80eeecf28dbc6300fbc539f3c184ca2f59780e27a576c1d1fb9772e99fd17881d02ac7dfd39675aca918453283ed8c3169085ef4a466b91c1649cc341dfdee60e32231fc34c9c4e0b9a2ba87ca8f372589c744c15fd6f985eec15e98136f25beeb4b13c4e43dc84abcc79cd4646c", + "derive_key": "379bcc61d0051dd489f686c13de00d5b14c505245103dc040d9e4dd1facab8e5114493d029bdbd295aaa744a59e31f35c7f52dba9c3642f773dd0b4262a9980a2aef811697e1305d37ba9d8b6d850ef07fe41108993180cf779aeece363704c76483458603bbeeb693cffbbe5588d1f3535dcad888893e53d977424bb707201569a8d2" + }, + { + "input_len": 7168, + "hash": "61da957ec2499a95d6b8023e2b0e604ec7f6b50e80a9678b89d2628e99ada77a5707c321c83361793b9af62a40f43b523df1c8633cecb4cd14d00bdc79c78fca5165b863893f6d38b02ff7236c5a9a8ad2dba87d24c547cab046c29fc5bc1ed142e1de4763613bb162a5a538e6ef05ed05199d751f9eb58d332791b8d73fb74e4fce95", + "keyed_hash": "b42835e40e9d4a7f42ad8cc04f85a963a76e18198377ed84adddeaecacc6f3fca2f01d5277d69bb681c70fa8d36094f73ec06e452c80d2ff2257ed82e7ba348400989a65ee8daa7094ae0933e3d2210ac6395c4af24f91c2b590ef87d7788d7066ea3eaebca4c08a4f14b9a27644f99084c3543711b64a070b94f2c9d1d8a90d035d52", + "derive_key": "11c37a112765370c94a51415d0d651190c288566e295d505defdad895dae223730d5a5175a38841693020669c7638f40b9bc1f9f39cf98bda7a5b54ae24218a800a2116b34665aa95d846d97ea988bfcb53dd9c055d588fa21ba78996776ea6c40bc428b53c62b5f3ccf200f647a5aae8067f0ea1976391fcc72af1945100e2a6dcb88" + }, + { + "input_len": 7169, + "hash": "a003fc7a51754a9b3c7fae0367ab3d782dccf28855a03d435f8cfe74605e781798a8b20534be1ca9eb2ae2df3fae2ea60e48c6fb0b850b1385b5de0fe460dbe9d9f9b0d8db4435da75c601156df9d047f4ede008732eb17adc05d96180f8a73548522840779e6062d643b79478a6e8dbce68927f36ebf676ffa7d72d5f68f050b119c8", + "keyed_hash": "ed9b1a922c046fdb3d423ae34e143b05ca1bf28b710432857bf738bcedbfa5113c9e28d72fcbfc020814ce3f5d4fc867f01c8f5b6caf305b3ea8a8ba2da3ab69fabcb438f19ff11f5378ad4484d75c478de425fb8e6ee809b54eec9bdb184315dc856617c09f5340451bf42fd3270a7b0b6566169f242e533777604c118a6358250f54", + "derive_key": "554b0a5efea9ef183f2f9b931b7497995d9eb26f5c5c6dad2b97d62fc5ac31d99b20652c016d88ba2a611bbd761668d5eda3e568e940faae24b0d9991c3bd25a65f770b89fdcadabcb3d1a9c1cb63e69721cacf1ae69fefdcef1e3ef41bc5312ccc17222199e47a26552c6adc460cf47a72319cb5039369d0060eaea59d6c65130f1dd" + }, + { + "input_len": 8192, + "hash": "aae792484c8efe4f19e2ca7d371d8c467ffb10748d8a5a1ae579948f718a2a635fe51a27db045a567c1ad51be5aa34c01c6651c4d9b5b5ac5d0fd58cf18dd61a47778566b797a8c67df7b1d60b97b19288d2d877bb2df417ace009dcb0241ca1257d62712b6a4043b4ff33f690d849da91ea3bf711ed583cb7b7a7da2839ba71309bbf", + "keyed_hash": "dc9637c8845a770b4cbf76b8daec0eebf7dc2eac11498517f08d44c8fc00d58a4834464159dcbc12a0ba0c6d6eb41bac0ed6585cabfe0aca36a375e6c5480c22afdc40785c170f5a6b8a1107dbee282318d00d915ac9ed1143ad40765ec120042ee121cd2baa36250c618adaf9e27260fda2f94dea8fb6f08c04f8f10c78292aa46102", + "derive_key": "ad01d7ae4ad059b0d33baa3c01319dcf8088094d0359e5fd45d6aeaa8b2d0c3d4c9e58958553513b67f84f8eac653aeeb02ae1d5672dcecf91cd9985a0e67f4501910ecba25555395427ccc7241d70dc21c190e2aadee875e5aae6bf1912837e53411dabf7a56cbf8e4fb780432b0d7fe6cec45024a0788cf5874616407757e9e6bef7" + }, + { + "input_len": 8193, + "hash": "bab6c09cb8ce8cf459261398d2e7aef35700bf488116ceb94a36d0f5f1b7bc3bb2282aa69be089359ea1154b9a9286c4a56af4de975a9aa4a5c497654914d279bea60bb6d2cf7225a2fa0ff5ef56bbe4b149f3ed15860f78b4e2ad04e158e375c1e0c0b551cd7dfc82f1b155c11b6b3ed51ec9edb30d133653bb5709d1dbd55f4e1ff6", + "keyed_hash": "954a2a75420c8d6547e3ba5b98d963e6fa6491addc8c023189cc519821b4a1f5f03228648fd983aef045c2fa8290934b0866b615f585149587dda2299039965328835a2b18f1d63b7e300fc76ff260b571839fe44876a4eae66cbac8c67694411ed7e09df51068a22c6e67d6d3dd2cca8ff12e3275384006c80f4db68023f24eebba57", + "derive_key": "af1e0346e389b17c23200270a64aa4e1ead98c61695d917de7d5b00491c9b0f12f20a01d6d622edf3de026a4db4e4526225debb93c1237934d71c7340bb5916158cbdafe9ac3225476b6ab57a12357db3abbad7a26c6e66290e44034fb08a20a8d0ec264f309994d2810c49cfba6989d7abb095897459f5425adb48aba07c5fb3c83c0" + }, + { + "input_len": 16384, + "hash": "f875d6646de28985646f34ee13be9a576fd515f76b5b0a26bb324735041ddde49d764c270176e53e97bdffa58d549073f2c660be0e81293767ed4e4929f9ad34bbb39a529334c57c4a381ffd2a6d4bfdbf1482651b172aa883cc13408fa67758a3e47503f93f87720a3177325f7823251b85275f64636a8f1d599c2e49722f42e93893", + "keyed_hash": "9e9fc4eb7cf081ea7c47d1807790ed211bfec56aa25bb7037784c13c4b707b0df9e601b101e4cf63a404dfe50f2e1865bb12edc8fca166579ce0c70dba5a5c0fc960ad6f3772183416a00bd29d4c6e651ea7620bb100c9449858bf14e1ddc9ecd35725581ca5b9160de04060045993d972571c3e8f71e9d0496bfa744656861b169d65", + "derive_key": "160e18b5878cd0df1c3af85eb25a0db5344d43a6fbd7a8ef4ed98d0714c3f7e160dc0b1f09caa35f2f417b9ef309dfe5ebd67f4c9507995a531374d099cf8ae317542e885ec6f589378864d3ea98716b3bbb65ef4ab5e0ab5bb298a501f19a41ec19af84a5e6b428ecd813b1a47ed91c9657c3fba11c406bc316768b58f6802c9e9b57" + }, + { + "input_len": 31744, + "hash": "62b6960e1a44bcc1eb1a611a8d6235b6b4b78f32e7abc4fb4c6cdcce94895c47860cc51f2b0c28a7b77304bd55fe73af663c02d3f52ea053ba43431ca5bab7bfea2f5e9d7121770d88f70ae9649ea713087d1914f7f312147e247f87eb2d4ffef0ac978bf7b6579d57d533355aa20b8b77b13fd09748728a5cc327a8ec470f4013226f", + "keyed_hash": "efa53b389ab67c593dba624d898d0f7353ab99e4ac9d42302ee64cbf9939a4193a7258db2d9cd32a7a3ecfce46144114b15c2fcb68a618a976bd74515d47be08b628be420b5e830fade7c080e351a076fbc38641ad80c736c8a18fe3c66ce12f95c61c2462a9770d60d0f77115bbcd3782b593016a4e728d4c06cee4505cb0c08a42ec", + "derive_key": "39772aef80e0ebe60596361e45b061e8f417429d529171b6764468c22928e28e9759adeb797a3fbf771b1bcea30150a020e317982bf0d6e7d14dd9f064bc11025c25f31e81bd78a921db0174f03dd481d30e93fd8e90f8b2fee209f849f2d2a52f31719a490fb0ba7aea1e09814ee912eba111a9fde9d5c274185f7bae8ba85d300a2b" + }, + { + "input_len": 102400, + "hash": "bc3e3d41a1146b069abffad3c0d44860cf664390afce4d9661f7902e7943e085e01c59dab908c04c3342b816941a26d69c2605ebee5ec5291cc55e15b76146e6745f0601156c3596cb75065a9c57f35585a52e1ac70f69131c23d611ce11ee4ab1ec2c009012d236648e77be9295dd0426f29b764d65de58eb7d01dd42248204f45f8e", + "keyed_hash": "1c35d1a5811083fd7119f5d5d1ba027b4d01c0c6c49fb6ff2cf75393ea5db4a7f9dbdd3e1d81dcbca3ba241bb18760f207710b751846faaeb9dff8262710999a59b2aa1aca298a032d94eacfadf1aa192418eb54808db23b56e34213266aa08499a16b354f018fc4967d05f8b9d2ad87a7278337be9693fc638a3bfdbe314574ee6fc4", + "derive_key": "4652cff7a3f385a6103b5c260fc1593e13c778dbe608efb092fe7ee69df6e9c6d83a3e041bc3a48df2879f4a0a3ed40e7c961c73eff740f3117a0504c2dff4786d44fb17f1549eb0ba585e40ec29bf7732f0b7e286ff8acddc4cb1e23b87ff5d824a986458dcc6a04ac83969b80637562953df51ed1a7e90a7926924d2763778be8560" + } + ], + "random": [ + {"seed": 1, "len": 0, "xof": 16, "key": "4e2873a644ab37671ab25c9962c09661f89625981a15d94c02b696cb27ab6110", "ctx": "lambda-vm oracle review ctx 0/16", "hash": "af1349b9f5f9a1a6a0404dea36dcc949", "keyed": "e2060fc733e3b0c2b258652b301a876b", "derive": "18592c7bb4d1ffeaa4c1a65c9033c1ed"}, + {"seed": 2, "len": 0, "xof": 32, "key": "51f2f6c1699b22240a722f58d8f4e1d847dab43424ab9c7f6cb007e12d4f5575", "ctx": "lambda-vm oracle review ctx 0/32", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", "keyed": "f53925740cc09cd9a37b1d00920479d275d847e0279c49a1b65001943158da28", "derive": "4624d0bf48875f0faa7f0a14e97b564152d1315d6f3777ff5c8bc4fa7df5b51e"}, + {"seed": 3, "len": 0, "xof": 64, "key": "772fa66e71671508c8633f754cdc205bc7ab06a0b8b0b056d1cb550195ec194b", "ctx": "lambda-vm oracle review ctx 0/64", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a", "keyed": "69d6a0893237ef913262d2dc8e2c71f8d6e8346c3745e7f622898791993de30b227fd4e9500b07b0e42b0f7978f7bc668cf0053f27d993bd3b85dd0567fe3549", "derive": "61aa6ed699d26c31cc82858c9df151de16f5459e752b62d1829cdf00dfb555a4c4240102469bc7e1076871b76c1276734dda1f5aa7147b7c7d8613f1cb763001"}, + {"seed": 4, "len": 0, "xof": 131, "key": "0876f38380a77686e29a0c55b4c16d1d07ae429f299d141e3245cc47f35b9d6f", "ctx": "lambda-vm oracle review ctx 0/131", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421cce14d", "keyed": "a5a703baaa710a23e4386586538ff90cffba019e51751a27d98a69e15c372232737faadb85d92412e653759d94aa5abd818b75a9b5108a252ccfdd8b71d92cb9e96b3f52777a355f36d69bc2c495975f889ca448bcf3e099b7940d7fe67038bf2004c04bcd164664285bd69852fbbca42ec60b44adcec2e4ddaea9cb1506320a577b9a", "derive": "a82dfed98f8522164b49badbafa145d39e734f9e868a3c0815b79df69affa65ba9ba052ac0d15b4021676f99dbfb8990a568b77c2c19cbfb5e6fefa1659c08f856b60f8311c676107d56120f9693ee995002dcc4b5e90c4bfdfe7ecbef71e0c40422c015ed6fe5e57432dc6b8c85ffa34b45e43bf94d075042041e47e963e235ea407e"}, + {"seed": 5, "len": 0, "xof": 200, "key": "9d0ff0dd6e45b03425a5b93d7f2f78b817bb8647ea6dfd057fc219ea00d1e2fe", "ctx": "lambda-vm oracle review ctx 0/200", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421cce14d30f8a03e49ee25d2ea3cd48a568957b378a65af65fc35fb3e9e12b81ca2d82cdee16c68908a6772f827564336933c89e6908b2f9c7d1811c0eb795cbd5898fe6f5e8af7633", "keyed": "58265ba18bfcdfa1863b0adc953d516e4a3cec2e2d09c4de65fff6d5ee7cb0b9bbf98267b3435d95521c20397e729b3c0094e9c3bfde552931c3451764b6574c7bd0d058b9cecd209a54df2da39a6160729b8e7df79395d5b48204055973b27f5f1ba83971fde2fdad3e6bbe2729eb3e76859edda23f577ba277546934bfe03e3351349cdcd05c71b04677852ab7cd0992c5faed20eb878fbfd374f5328a92bc8736dce061f787f0a9f435d8d9eab7e1ee5245a2e98a556917d2fa2b996b7ff267965f883cdc73de", "derive": "c49da0d52d5230577d675dd17125b7c9d12ee8ecd2dd6b5c7a3bb87e31249b0817a489137718c47e36f1f55fde71db1f47ea3a16c0e53d7e8f61778a199b664d88d598dc0464883110d1898a53f32c4e72728f774bdf5b949c984d645fc48cffd6fc8b867b36b8c568d91cadc0207aff6677ee79f0ca46b34aba8fbbb4d7caead47c39c50468d51c10a9e2d0a21f95360fdd067f234395e488b3a8110faa3c2d791c3bbb836c820a1f27bf8add1994ba93895fee5766cbd6e6524308800bd6763a18ce98b7716faf"}, + {"seed": 6, "len": 1, "xof": 16, "key": "317c727c3588b8406115f201e6afb5cd7abdc51490aaec9c29278c93d48ab42e", "ctx": "lambda-vm oracle review ctx 1/16", "hash": "f633b185e996eb1973f7b5e81da3add1", "keyed": "2cf20a61154aa7b99ec1e808d64504d3", "derive": "031fac35c559618b2d8df9ba8329cbb5"}, + {"seed": 7, "len": 1, "xof": 32, "key": "c515bba50da1bdc09d42110712b6094ae75f9ad50bc151f1efa27b7bf6a4483b", "ctx": "lambda-vm oracle review ctx 1/32", "hash": "ad821f67cec320b5cabc58516eb59f00ffc47e0a23db617879a2817b7560d081", "keyed": "5c20577f80e9d2525dbf76f370eae9b6f2e040990b06323106ceb3f4c59ca77f", "derive": "ab7326c844abcfd063239f0b96a23b1adec9eb82b0a0a5bff421bdc0567a8813"}, + {"seed": 8, "len": 1, "xof": 64, "key": "c54c0ceab664949da493637296c6e82ccb0cbcd3c0fab16029882933ca84927c", "ctx": "lambda-vm oracle review ctx 1/64", "hash": "ad821f67cec320b5cabc58516eb59f00ffc47e0a23db617879a2817b7560d08140f3fdb8e19afa84cf36653dab48871eee1ae3548a75bff74a0f7268bca72569", "keyed": "4c4897a7500db1cd436be752b3186744701743822d6e4926f7cf662b56749f49d2c4569660f7ca12bbfa1483e42df23f21c90d93fd563010e196bf2f19f01871", "derive": "e7696f2d701338d5725eea71e792aef3fe724ee2e257feedb59123334f8dfaa07d57460273286218154b7722ee7d6a6b2efe055ec942df34eabb325afe39bb42"}, + {"seed": 9, "len": 1, "xof": 131, "key": "eb899ae51ef9827c78b6d1b224ac020a475fcd4387a2dc0afb5ce436c256acbe", "ctx": "lambda-vm oracle review ctx 1/131", "hash": "2ca3a1b761c6412251d07b8a707fb85e306530bc393bf47664172bf5078e278177f5fa73496da85a1f43bcbab552dd7bd29627ef86ef7aa1af9499a15a1bf681c3a273bb00421538827f685d575e2f6040047674fd005d3c3a90e83b34a72f8c12b528c1c7ae262c79b5f9e51c66344eddd3b0f5a1387233bd10247c302f6de304d598", "keyed": "f80116f4d31674dd65a0cca63b2ad7ce5e07c25ab4936eab9532422760778ec7bd59719681aabdddb89e23855cb7c8b28536dd7776d22cfb840531734b1aba68aa70b2ef0e485e729618b12fdf9354081a1dcfeb29ed6e65daf5cc9e46e68ac025c7b4726ddaeebcd3aa8c3ef22190322d350729210c51e5bda090c9c2bc985aac4d04", "derive": "691c85c36007c7270f0608ede75a83818afae06b550aec1698ea81935db7657a35577fb9bd3121aab6c952a0367a3603d8e79072c07ed3796cb0a2e926677a06757d7690d3f66ca7ea5b805abba98c17646c6fe6f1894d4bd0f4519c5c7cf8a9cf2cf6672041432d20da197c01169972d47bd8e3f8bdb34e33d2fe89369964d5aee5ae"}, + {"seed": 10, "len": 1, "xof": 200, "key": "119a6021250e721aa23d6096a7e6f8e9e9de5cc085ec7540afbb588f84476632", "ctx": "lambda-vm oracle review ctx 1/200", "hash": "f17067d6d29eb1bb0208ce84d40ee47504c8656b67e7b663eb1d7182e090b0f4399fdbd2be2d9455eac4e40814d8e5b6ed8fd9090ba9c067a17c959a604ae5501bded22884e9617c71ea944374ce143bb040d8cf377b9a09046e9c769710e755940120d137f08a598f4eed9f69f3961ee13b1f25a661dabde0c6101b3cc7b492a72a9e83f4fb44adc30a86f8124b351dc18d9b7c0c711d3a49ca2b5eee3a341e45970d9981692df5b6b65103f9bc270c1733d30d815ca09b8cf134aef4c8bced5da99519932fdb51", "keyed": "2b3b662e366d5b27c2abfdac897acca87a74bfb3eaa3c44a23bec82ad6c923b80f799d1385fa77cb270689f7a9fde16d6f335b76dc4b16b5457377736ee78c26bd0cd9bbab04f8513bbc98709ab3bf5641d4f6de572607748ddde857df6532cafa4075aae6e0afd5940c62af20e71f78eb8aa4eaa25386cb2aa6a6e619fca07226715e4001230dce1ca4c74365273a2248fbb65decd69337fae6578474bb3a0d04e347f930b5ee431c64ed9b528143becf0e6da9c78a3afba6183a08d85877a6a64a79055738c96d", "derive": "566ebbc1e00fd71b6f35665b60075c8238bf87dd3bb4f58a0b7e94b6599a37a983f27b9274b35c9187160e010a70cd2b3562ddc6b779b1d3ff177114e8a22c827649fcbb26db2a6feddd5fc135c057c3c9e4ab63bb04d57bfd89c3ddec14b7a85cb3b12951ff8c5a0ab7bc983854787bad61ebba20e33f85f79cf8f49c0fada60b62c412e8cae0c2f75e374f24c5a41f274ce041a7379881517c69a72dd631a94ea72dd82ef1c3721004db70a1685330337a30da3678427d1aaaffe3a440f2f6b6b9a5fe912511b3"}, + {"seed": 11, "len": 2, "xof": 16, "key": "36d7a2702afdc652e3931fffedc5e1aaea8215f349f940ebd1eb941dd5e47318", "ctx": "lambda-vm oracle review ctx 2/16", "hash": "0274374489ae986e5475921ba458907b", "keyed": "54ff07b6a6acd7a1f6b0961b318bb08c", "derive": "09ad6852b188894de999199fb6fb74a2"}, + {"seed": 12, "len": 2, "xof": 32, "key": "a5d7f0a5d770f63da9dbd45bed6874726ed533458db65685690376ab47d6fa82", "ctx": "lambda-vm oracle review ctx 2/32", "hash": "ae2ed86f46c2a7fe7d0f242aa6aac5996059df9533838c17c786c04b9ca0f644", "keyed": "de176d34225fe68f469150df8f5ba8cd61bc4a03872962e312b8dda7e83669e2", "derive": "30dfa512613e64ea073c79b3bfde819c269dee858a1af35a9ebe53540098c6c4"}, + {"seed": 13, "len": 2, "xof": 64, "key": "39705c1c83976fb01c0ac14b1f5c5170b5ed292906fe65e483ea6410ac670871", "ctx": "lambda-vm oracle review ctx 2/64", "hash": "c91ff7a53e49408b407f10d8f9c8ed080b5de4811502f0d5f9c5096fb57df4f01b5b2fcda59f9150970abbd8e5bf0f30b5658b5d236b66a70e26a566f77a88c1", "keyed": "5a61e26a1347ee4e3712cdeef297b4a435a896cab11ca069195291f401412a26dbaaf4e03fd66a258bd656f094b4deb643342edf6eb2d237cc5e5c780acdea9a", "derive": "2e6195ec1ba4d7d8f198e6d5e55690bacb567b00bb3c0fee39103463f4353e7b85fd3d70172158b7053da07500906d6d5c3e74e661c46d364f096064bda855f3"}, + {"seed": 14, "len": 2, "xof": 131, "key": "f024acdc8f5356491b8fad84636e34b608fddaac3f4b5c0c67580d0ddd2766a8", "ctx": "lambda-vm oracle review ctx 2/131", "hash": "f997bcf0a91417c7a4a7eedf735d986be1b183d455057a5655fd77147c0db0396da06133facf9dad5523ccb852b43e531f415c4bf0aa170fa9407b1e6667d4e0e3bb59ab2ffabbedcf060af9bda300dcc6fcad1b223c6717b574a1b512777193a9ea7684ee725fa6a93210b782b128d2aecce569918501e61ccc664487174dbf6c390c", "keyed": "0a4090e3375db9ad9dcb59055a53cee9a40b48a7c6f85cb6de6e4b77d626e2540bf035a7e9363b04e0595d82107d0ad723686b76de610d2b32d7155e1874af4763a917b61a47067b0c25461f9a4896fcb1c660ead2410858b7a29aa4ccfdaf49ed28f505085121acf3d75cba353be74a598a8e81560f9e8e7174ed6c84a3f623ff3dbd", "derive": "0a8f4b6d80548fba6628b0f9342e27fee4be72a5498b6fecb46ac4a45b5e320cf94ddaf8c7a4131b6e5b46183cf08dd2ae5310f75dace83b9c5af6d7ce7041e01f40ed2afaf4a38e79409cb8203a5d75301d40a99a45f3f0289757874ef5df2a23ba941b3ba23bd70047c5148bdd45ef502e598a8c3ca0390756fd332cea76d1ccd0da"}, + {"seed": 15, "len": 2, "xof": 200, "key": "85bd87a61d3b05fcfd17c2258ad1a40b37396d6aa816e1127500d5d5928a0a90", "ctx": "lambda-vm oracle review ctx 2/200", "hash": "2c9c567961e877f29cf5469d28f8b1341295df42fb56e5e2c3c240bd63c57778f4bdcde180ae4be83337936fd51f01349e5356b15319852c5b50dffd1c099c34090283b352e8d350263103684047399418cc319d770105f23ca00634cc733f06e320db36d68954c6d44fc6b5853462ea1019a9941fb1041cfc00d590c1c4e487569ad44acac818b9b3ee7d406c736f41f70f72cee3b133b2241a3fb8627a0fe3fa92341a6b6fc5e1222aa688f49d936b6b2ec99cfd1fde1fe6611fdccdce74efaf54a9cabe220723", "keyed": "024283d0b8224be9032765f2cd1076dfc7f4cf6cb947a9c7367ffb4e082cdf137522dad8ec833a35c211797431e9de7d7ddd3fc35692b7cc943400095e978db894b9bc37e1ca7ed271d1edfc0befa95a64a39e71c34b46d4d557e3770f8f01b7c8fa7349ffec6e593e10af3d7bd713c8faef0b00272b347a7baa2a135f006c5f3efdca2ab53b73532f30eb377ee52f9833a556cf76a90e51da5c5e9b42666e09788d4308103f19d596fd961255801a65ce10c85735830e039830ffc36da06ef54eb17dbf3d99e3c2", "derive": "600c0981573e4afba64240a4246475ee49f8b73fd8c0f89a122077676c63e59ec6633c5ce0b851122fb91bbee915b6c929ec653b651986c0a5f9b75b421f7e2813c46f885bf203ff25e511cbaa904d96feb55c336ca224ba11b1bac173072dc69fe3e90fab5bf5b11819479652c5afe014a2e42406191ee4c456a03f7ff7c06ff77c4c9f9569592500ac9f1a5e1491080c163e76e9f4ecd849f6e2818e97db2cf67c7d616d801717002173f4f331da718840597c7b381d1e164a789169ac6c26ea34a470f8692bfa"}, + {"seed": 16, "len": 31, "xof": 16, "key": "a7f5e028fe27073413baf3c095354912dfd380d484e9cfeb548c1492fdc5bbf6", "ctx": "lambda-vm oracle review ctx 31/16", "hash": "e45042ca397670d5d2ef5eb5be0783fa", "keyed": "a5a9e20141df86780d1b27f11b0d1bd4", "derive": "a3bcef201e7ab8174a69e1a97ea66de5"}, + {"seed": 17, "len": 31, "xof": 32, "key": "cda155913dc188c34335e58f73091fe30cce6cf755a62f967c0837a51ddfb283", "ctx": "lambda-vm oracle review ctx 31/32", "hash": "b014f91e00ef5410cfef91b1bd8efd88d398ea2ceaa8a6522ce4157d6d57a5fb", "keyed": "61bad733c76597cf015c4bf9bf6bc766f06369fd79a6b42232946be96b84f719", "derive": "56ca83cbed45c5e4cf5925a22493de95b252ece7d578655be136a03985e2cc96"}, + {"seed": 18, "len": 31, "xof": 64, "key": "d0fca40ee0a6fbbb65f2e91e27643c0d4141d20b6aae333f64ba2fc4661360cd", "ctx": "lambda-vm oracle review ctx 31/64", "hash": "3a9668075cf2350c98e361bf3138f6de927f0c618974a38df8787c440a10fc6e6839bd4613297bf68e1f591f24087f4841a50e79309f96b17b132dd0e1ad627f", "keyed": "b6bde96e47e786800e0f501a77babd92291666f168d05a23cbe57cc7f6384d4e58ec2199f0bc02cb719632403663483a409986f48caef7f1692d5e32668b52e0", "derive": "01e6cfde455c562b5d41ba800be896c0bde376fa3803ce37bce821b92bbfc7e1a08b375bb84eeb5a2c3300dd667ee9212d441cd382b73b97fe0b2172aaf3499a"}, + {"seed": 19, "len": 31, "xof": 131, "key": "f6a765475e789f2922e039484a8f8b1d9da6328f67701ec6ae99c800b5f72b61", "ctx": "lambda-vm oracle review ctx 31/131", "hash": "c42704dfc6c9013843886334233782451a5f5fbb8fccbb8e1b6ce89f3c2c8d82674dc0555a7313e23254a13c195da6c8c36d5adff3ff1da19a739a78efbf324b9d32017bfeb744676230118e5f7714cf22b5ad0eb494bf3b60f67e788b11cb9cba815c3a79a60314ddc7c8be71d6febace72a99b1bb6524446fe6e24735335dfd4de41", "keyed": "ebc1ca503e3ff98fe60319c9464e35b592b1169515178d01a7a4a09b9cdf58d041cd589b7d91438b01cf16453b03917b9ff4f33a9027c6e0fda2d37f526942120908541d46f232981f2927e26bcd9a6a60382ad2f09ffc581331910c02176428c85439817e96262eba404a76705e60cb689c9676779b4945bd25a485f7332951a956a6", "derive": "651fe4214293835fc5aa7353337d3dbee3f9f8f674271c403d41e4f22168a427cb60094adaec81f5f811d54c7facf79d23b7c8090c364d6f25da9263d57b053422c7f6d4524aaeb1e064b5a13ecb76df59e74a28c3af83980717f9c50f4ac1969b57e6d397d3b4f0e36acb2221d3d82779866b9830dbfa0c98968519f5d8ee9f2e7d21"}, + {"seed": 20, "len": 31, "xof": 200, "key": "427d2be2a0a273bcbd83a71c6356ae4db1e6dae70d4ceb93f8d3ceb5b3c2b0f4", "ctx": "lambda-vm oracle review ctx 31/200", "hash": "f2c450b25acf5d07c0eca77f91565db7b1724e825cec605ffa4c3811108a19ce7ab9bedcad7e970e2bdb188f807fcc8d176d7af48528a57f7f7e97b4cf480db33418b8faa8e9e0ec76fa02adafb7751e5ebec1d9a088e7234b3976ca652b2a58fb50402ad4db627b4a3f087295ecb180d75a2ea2cbb99aca5ff1cb01ae0faed7a17692f03d13454834f14247437a9a00749b68b17c633ceacbbf1ed72edea9321d3e3edb41a76acf21dc406224524a1ede3bd7f535bbce38fdf3429798a341245eea76c3d9451fe2", "keyed": "0279a7640de3adf0825a3840dd78c65a80162306b133ee52bc207b2b7bf639959a6fc394a2e23e79ce0c7cc44569aad0d3d24b3c39c010d5839744a86fe1f47a1b55e4dcd2898d13aecc7e9223010097e5ccb05cbaeb2b258c8785fb1d2edbdbbb827c9c5e2592ad901028a5cb5cfda8215acd4f36c03088f8c0a976f410991cc499ea5da7587c671a17d200949e6e801a6f1e8ba9e9bc6e704313f9f577b5a649f94c75ab983708388e232cbaba93097fba3378046efec08279025967db8540dd0470d24c5eee3a", "derive": "03a483217460e212f6c9454f0d3a0e0b0ca69d4e0bc53ad07724d66565598b82cc6bbaf8456c3d9ff6eb092ab38ba6308c82717daefe30ee3a963a531e6006f550d72bdf9335fd81e98dc5ba2e694b34b4a5db8f13f015f9c0a0f7bca527e2e164635dd3503026b2060d6d1ee44f887d5c3debbab2189b285585801d85a5af8b0374e58bf839ef6eed1c6a912d57983ebd5292af99b15f1e4f08a5c8cce02b625e729f5a5a1130a1f78de0bc3da9860349f4825963906b01c49ed7fed54bee7ed8af418d109d6acd"}, + {"seed": 21, "len": 32, "xof": 16, "key": "d6a839c8611a7bd19c9791970aa61e9453a6ce5014f54c84af38377d711ce1d7", "ctx": "lambda-vm oracle review ctx 32/16", "hash": "1f8db1bfdd8dd100a84848f8824b888f", "keyed": "0bcb7116c7635074d6df75bf5d5c932f", "derive": "c3191d21509721b6885d7498e86005c7"}, + {"seed": 22, "len": 32, "xof": 32, "key": "6b8488c931ef0c80a7a15aa039a2a4da72e5fd97686f344c037c08c151a7ee29", "ctx": "lambda-vm oracle review ctx 32/32", "hash": "39eb5d3bf52d1c4855e9e7475cb795c9eda725f55f8e77cc5e97decb3a0bb2b7", "keyed": "cc5dfd72264e95db8a511bc890e2339bfe3c8330b368fbc5208b33f072a9e0a1", "derive": "659185a1756867f59590deba74d025854eb6b6d3d2a61ee8340ea9efea42cb09"}, + {"seed": 23, "len": 32, "xof": 64, "key": "ffae277d623842753e96b906f13377ec53f8218c863a7ffe52a5805ac20c9468", "ctx": "lambda-vm oracle review ctx 32/64", "hash": "ddf820aa0395b67040adf9de32bfa880bd7bd6a63776c5834309436e331523aa3bbab766ae85510377fce8259d0b5037bae8954b037f0faf9136d5858053f1af", "keyed": "108fb30d7a4ce089e673a206df2ba91600b992d5ba40f04fdbc0a42b99e6e1dfced81ddccb58738b92037a6903d2d669c9a8bb8880e45302f38824fc7aa3177d", "derive": "b14a08b72587ef6f8e95ad7c39c02ad3f635da206d80dae709fe6a215f0f1a5e6d897e6a0d6666a380bc4896dcc9eba9f711845f921c5a745eefcb16a88b9566"}, + {"seed": 24, "len": 32, "xof": 131, "key": "445637047e96984af1f65f8d683deeb28858305868dcd5da010644a281f75396", "ctx": "lambda-vm oracle review ctx 32/131", "hash": "04153ec8aae42ce4e967b24e4f0fcea47b0b5bc6ff27ce555d0e29ad1afb044681191753eb69bf3c246d04bb83f0c5fa6c2b169cc1f84188a72c52e28b86c953f9bcfc0428c49c4a9ef3ac2d6d9f372ba83f8fa4bbcf568b38c05d4e0baf9f8cc5a1b915b8f86f1eb8baf428b79bf4eb24eda5831a105e0cd7b9af8954d5342b4d259c", "keyed": "b78d7c2f73e8bb10ae2d6ba2af88034e071497f362978f2c6dfb41038c2ae7cc8466a2e599237145b1117907fc23c9f74b3f9d7cc81e8c75cf849d31c24d88e256feb79810ca0c90a07c066b95464a0ca3acf5848ad903883c4cbc31c033f6e71b0ea152433769feb322503af0700be9833e8b8101d4ab0bd7302880ff4df965e442c5", "derive": "33a9f9cb60dae21962da63ee1d6926ce0bf1292e6f03760396d5b292c295843b13ff19472fe964caddbe09559f9840d1609ad0a3be1ba5a4b6c0c4688585ecb1a399beef3d4bca758fd94f96e0aba1c86d815fb5ea72a7d03a29b7abc2459a77aaaf8db735dc80abade3181de8bd9ad2a0f4b7d155b49e8300666252337b4aa37c975f"}, + {"seed": 25, "len": 32, "xof": 200, "key": "6a021b8bc4caeed9bf9bdca81c38ab88786e351821c08d22314b361187bc7318", "ctx": "lambda-vm oracle review ctx 32/200", "hash": "e04c6f90e7c7c64dc4482711d6fc69dfd0311807c5de185abdc58bff4e5dc75c950ca3397b856cec6a773e31ea95470bf2f8d16072142c35104d90537e6644d191387ee0250d617def6ac698c20324cb1a7ad4ee0ca9a72a0be13478341383041ef3855c6871ca02f92c0a30ef98748200d1ff5c483f28068325d80880b37ec3050587fb66b3bd40cd84c701c163b0fd33a22a26c7fddf7833e7012f0d282613e263ffe5f93ba2997829ad0b6583e8973cd656e531ffc34418bb5c454ef1571a8d719ce4aa9dcbdc", "keyed": "3ee76706e52a25d0b11fda2008594cb7c0d108e0a30a799685e68a36fc4d576a872dae2c86182c06f744cb84088296f26397726455a9cd3076027851f94cb2db376daa3f2fb478f3df351487f0d55bb9b789f3481496894af5f7295e76c9144bd6db381d89aa984f0d0f1fee767c61899809979057129f8395ed90a6eaf53b910629dd38026013769f2a0761681449cbe68e03e7e87201bf02b68f1f89397bab16da8b38dbd50278618fdc4dae10946246fa6cca05264bd6f4333921e2fbeca60bba45240b2e248a", "derive": "f8ce88a2f6f80dacafd922474d734c038d7eb0e6bf14a82055d64f14343336c88bebf482471aa7d0b4119d6bbe762b4d9f6284ceb83acb215e575f14e0ec4331b1e5337f0b78f5aeb683830b8337df041c0270aea9d7afb7fc04875a0ec985e16a7b2a60473a3241a7aa404a713b86061d8f87d20135a29294ff9cf4137175064834410219d6d8cfa009f9f866d4560488eb5933925cd15a4461b25a9a2f002c02a39c2e3a7c0b0517ffca563c1398aafb02c7a786376321a38d09b8ae68491b5f2d1c43db26d2a9"}, + {"seed": 26, "len": 33, "xof": 16, "key": "90a4ae28a4be801aad16678441130895919436864549da4392404ce5abe7f28b", "ctx": "lambda-vm oracle review ctx 33/16", "hash": "ef56ebde8209e7f40467b2a34930f5d5", "keyed": "3cf2f08202f6d76e0ca02fdbf2cc3f19", "derive": "0fc74c6095da0ff7afbddb487f526fa5"}, + {"seed": 27, "len": 33, "xof": 32, "key": "b54f00030d216c7e1db895eecafc89352606995fd98cbbb88426abdea6c0e38f", "ctx": "lambda-vm oracle review ctx 33/32", "hash": "61472133b26ffd46ed8a0321fc278559d720a5ed6c43a55bd771bf08aacf247a", "keyed": "3df5ec7583584487ab47307fc118aef9bbcc697193ca78ce4d010a1537359e66", "derive": "346249306132497649f765a746576fec3c8640d5648f6c821e01c94d6635a0ed"}, + {"seed": 28, "len": 33, "xof": 64, "key": "dfdeb3bedd683ea5e7be357f352108909e305927738447cd1e3b12868f09c861", "ctx": "lambda-vm oracle review ctx 33/64", "hash": "9d2b24db2bc6fe675948d163f6d44d6575581aa479afe1c1e831044c1af8872a48a68959d7b6c82f8b8593a69af6270c73732fa96b5c53a83a5f3e73a0c7d239", "keyed": "350e2f86f87fae4ec59a64492eccd9da82d59a3e995b4aa5adcbbc66c1569fdde120c789a3a7a52f41fde3707ba3a06cb4a1d35af80ed299e9b3fccf538ea0a1", "derive": "568cbc5904963dafb7b00e79915b0b684b153b3a4a53d11067a7f01ef6b84676f75ca7844f50dd15ce9c60cac2e7652828df27ba9a671545b53ce885548f41e5"}, + {"seed": 29, "len": 33, "xof": 131, "key": "73092fc14a279f4086511f0fb033b375c241f8786cac69a76b9e7741e23ab604", "ctx": "lambda-vm oracle review ctx 33/131", "hash": "bb438da5ee6ee50a61e2c984008a4b816e327c95f6a47ac8c11d7ee537c0706eb77612e0f2dad149286b098f892043a62cd91c9fb3856556d7334e1a5de3ea36fd3df32cde9e7d68e3efcf38b4a0acecce095c698d464765526d39146e66a6f2382f72e0b2db791f43844de7975cd448d1183a9aa6c5346941a5059349e372dd7ea593", "keyed": "1adb706a90fcced156afccbacda27db9e57ff84b19b4f73cc72cdcfbe9221b01832e8ef79ef9829f9fd922eeb4584ae68d23d61d19516048748e3414a9d0c495fdf513523a881fad9f36703fc9d6eacd0e36aa3d79deaa64bc6ec476fc5364b70ccb188b1400f77ae4038cfe490cae20f58486f33a6c0e0adec5be45f037631ea24c08", "derive": "08bd3f3f80c8cc8d61c67f361948a703b3c648b00df9543c02e1d894a8006707a204e9605106f2ec362e4e5a230ae89267ff2708aa45d04be8f4e5be36b539c3138bc847f827671d46aa4321c5dfbeeab4ec376687bc91f9df331dae0a187ed31a88b733eb2176438d773a111158ae75561f32beda6cf7cd923dccd14350addc44f4f4"}, + {"seed": 30, "len": 33, "xof": 200, "key": "2a2c4ce3bb5f2815a142a9155a0092a92a965f8cb20c2100498799b76422b677", "ctx": "lambda-vm oracle review ctx 33/200", "hash": "dac08212ee797b2e713b25a596642be078022fbcd06859a4ceb0990e105f971581078d79719fce9ab8baecf7d44f7742c5e7faeacc67795455549120cf86d2599af82d19d76b4df7c7a9332eef0589a3ea71ff56f7d762adb57617854ed8340b59e5ce7716f61f7e16fef7db370a1ca355df79948ab0635bc33c8df93d72706344cae8a9d94cc17b35b869aa6bc28257e871f336ca931f68b5ca2594af9ad1b4518e3028422660db19d48a810cc0c837f1785e0d94091c847799c732a9b5a9effb41823c6f32030c", "keyed": "888a4c62b8c06353dbd9889296c692675f44f803a5465d1395109e0f8a07f2b2190490613894aa47ef30b3f545daaad007df969935d3bec5643320f3660ad959160382b47c1559f4cc5351ea29fcbbad62320644a1546343f7c1717418207b239abd8834f453ffa07b12c743f4d697dc8c8ef10eef74967c50a6d425afd387b1f13cc97f53774810547bfc3f8377a76a75fa7df2267e19d62dde7e5b84a6f64a6df693da3db93f2c20debb4763341e6e70a79cd290cfc5393336d4c18ae70f3cc11263eed615d07f", "derive": "4665c81a65f084fc050577c83edcc924438c5ae94efcf08833de53feb697ddbe43a6114cb825a7962455ac7834ea85add26679b75e80abe73b260e3db1f604cc570f918bf97eb9106d029033d92bcfd8bb800d160b9aeed608c94a098837cd09a8f04cb5c9a9820bb100740ae5f41023831a0fdfc40c7676b3112cb8865911211efb9f39c3f8abd15d6c255b694c94e151f922750acd1957b5ed83de0b8ddd28f685a14855174ceaca79094fbe6f76be6c61c9f7f28d582bee692c246f325cd16b7ec854f8583e79"}, + {"seed": 31, "len": 63, "xof": 16, "key": "bf567d39b3e6ef07817fe823e62a01cd76885b59b1ca71618f1145521868d4f0", "ctx": "lambda-vm oracle review ctx 63/16", "hash": "4262e019e820ef30e1e25fdad359c3ea", "keyed": "9f6b2487c3fd74c2a9d9be0e2581fe2d", "derive": "caa2126e0420650fee91fffaf7ede68e"}, + {"seed": 32, "len": 63, "xof": 32, "key": "fc4dd8ec37ce39f879d8e00a0c73114a45e91730dc70d90d18e5e654c0538431", "ctx": "lambda-vm oracle review ctx 63/32", "hash": "aaa39d28da9cb6b404bcb7d32a2920ba27738aef1c3483a6cae69684bff16889", "keyed": "7f999b8c7266cd4d41d379e431090084c2cb66cfc23b94b90de8aec7e08d04d0", "derive": "1d27ac79f3f877ffac23c9398d7c82514b41939ac6ed4b5eeb04c355a0a03802"}, + {"seed": 33, "len": 63, "xof": 64, "key": "228af733f9098e51b260f6f3273f85c2c90dfb84ef8e22d8a34b97c534a28a21", "ctx": "lambda-vm oracle review ctx 63/64", "hash": "d05537d45f4d3f83c2a99116e2cf7be2360a0b25ada3f5f5df23ea147ee2dbeb3839bdcdcfd838a0a8515e0648af60d51c422b7acbce9a9e6b88b1783b307fc0", "keyed": "7c5849505cf5500ff148af37164943ba45fd7d471f23e7b81d8cfdbb75ad5e560fe1e35b063ede7d7ee23ab2b82b3ac68495e05623b3ccb8b3497a5bd7cd11f3", "derive": "33d52e8cfa97f0fa1178a5517f8bfdf44b77ba41488613124d3061e601a1f09d39bab70e5a1fd7bf50d753f58d52773ec61a1969e103265b4a900a4fe9c8aa4e"}, + {"seed": 34, "len": 63, "xof": 131, "key": "2576e5cef6878d6ad1d871bd733ff3b19eb40c1ba2382108f942ca47a157dbf0", "ctx": "lambda-vm oracle review ctx 63/131", "hash": "c9e1b5beebbc4b4555f390e7671456ea4e99bd0c609cbc4879cd7993bbba429a1796e0c86ec72ab580ec6cb97624d1c4fbae8188d7e712b666e41f73258a6bff44ab4077ec5caca38a7433be48c486e99c165087a546594e84a04518d46d572f50fc452e62c5123889bd4082085cfa4dece385df7263294220fe2e284046529658fed2", "keyed": "61dc08eddf340491716507f5df1eb74b0864d8a736357041a057a556b5539a69ce82f9b91bace73c49b01d841edc68f1ab8dab66b8d0e87ab8366fba501f7a5c4ae280f7db9f459187b18cec8a753fb500de2b7f6e4a35f7475ee9b71d0f570407106db39e6831e609f03957fc1fcebc8799536498fbd1d713b2671474004cc5474594", "derive": "da1263ffa944e8b880d03b5e8e34ffc10a310abf855c028bd483542e51ef3fb27f50c9bc25a276bc993255b93c1132ba8fe8870e8a8326fab129ced339aa145930e1ff881082ca6b9abcf2dd11849b4fcc17e3d8e871b331f662b088f2d32682f96ee983e913867b78eadb514a5d57eb9a60574b6d5b5c4eb9ec42fee23de04b77b8f2"}, + {"seed": 35, "len": 63, "xof": 200, "key": "4bb395e425b26cc9923fcb8a4b7d3b4780c76f48520d1037b950aa020de21884", "ctx": "lambda-vm oracle review ctx 63/200", "hash": "7f7d40e208d339f811314b5eb5d1bac48af4124805f7a02c78240c0f967e67c29a41d9a649308da8c46487bea1bf6eb8e0f3a38eb7175e68cbd731bace4048adc87502e2708c7579a5d070c5c0c6fdf57164a8a9a8b7b1329882d0bc0208b5bda0717706e61f677ebf648fc890ecd8adc02df65793c0b4500c96695c832d473d97962410b3c4d4e9e4431af5990f22082d6aa4ec841067b90222972e63d8b387ac52e7bc31d5d7861cf4bde69a46ccc376550144c92fe0dc1b02d3355930a017bd1ad14b0690f0cc", "keyed": "1048cb6fbe2f95bd6d2dc387c01d9efb207f63a30adc9b34567e9b7c364380cdd90fddf1cfba6fb406bf7dfd24c4d84c17c562f20459a739218c839e31593d69b4676fba089efd4367b047c10b2f4543a24fd66a9913f4ef84cad9a299382ad63226d507a2f5fd97a3e54792d71bdb2b6ab407f6ccb2675b97e5f56773f81fd776014c88cf04d0a3d0adf2e715a8f6e94876da4dde49189cdad6bdd4a460df60869a03b5a2c27a2375b022293e7f847893874d6d1bc1d037fb812035a4394bc8a7dcd2fd67ee4afb", "derive": "ebb1db90953880b49ed3c1cc8af140b21342c04fef76b6ab6580f7ba7eaaf332d49f38d1ee789b89a5a179e977beb397ee57fc1735a98e5ed7dff73ade7df059c9f3212d358c1132a6db7f12348758c7c1190348f703193590a6495a313c9cd746cdf4a2b41228e4ef0e5e8562e7e31825a9565293a545bda5d536fb5060bef37025fa3f17998c237a01a22c430b6344358b3caf4e47d06c0c382be109b97ab283459bc6fc06c71309dc38f7bff007b78efa679eaf7d34ae4227f2348f408f6205409b2c3942fedb"}, + {"seed": 36, "len": 64, "xof": 16, "key": "dcd8315fa4d93ab7401b877d6faf3d91216ef5be067af7f88203136a6c930cd6", "ctx": "lambda-vm oracle review ctx 64/16", "hash": "d157ddbefa378438f4da2f56ba3999d7", "keyed": "7d9689c610ea2ee1adf08f5fbafda494", "derive": "5aab60d350237580fd2e72c7fc88e25a"}, + {"seed": 37, "len": 64, "xof": 32, "key": "70712f68d2f62b0ca8b89e939e02ea75ce4906752926f53e13e5123e565c2ab8", "ctx": "lambda-vm oracle review ctx 64/32", "hash": "2f9e3ee7adf6cdfe9c2eb4c5258a7a6407a7ca7c6097c31e9016b7a2d3041a6e", "keyed": "811d9602e043f9300dd08eef1b7d80d2636dce0beb1e3d3021375830073e779b", "derive": "779db5d851ab425e380f54d3e3b7dbd35e86163d5791b7a7e6596846d54fb981"}, + {"seed": 38, "len": 64, "xof": 64, "key": "0501a6428cd4abddb3ba9d7c7c7b7233b7de463f9b2bc76271f69c2587bae2f3", "ctx": "lambda-vm oracle review ctx 64/64", "hash": "927bc3149626165538fd524c3bbfc3b9dfc2569807604b16625f7f1a6284d1ef4c8f416a9ab9d7ba0a345a302a823d5b5cb1c8c6a5773d3d3ebfb894013b6800", "keyed": "dde8934e7634a0d90b5ee4875c1cdb9efbed89f6772920f6b79e355385ee69cd431d8038ecbc511073404d93af5e5a7e5458dc7c661f229d509bf012c4c04696", "derive": "7e5cbf68decda9fadec8f78802089a0b5174adff9e5dd810e048361ef1ff2584914c0e5c60d13e25751fde8a581879b4400b857cb9953e9210784cfea0919a63"}, + {"seed": 39, "len": 64, "xof": 131, "key": "999aef198891c1934233fe032a2b70c199be1eaf910d9a67882bab195bb665ce", "ctx": "lambda-vm oracle review ctx 64/131", "hash": "896f9dcadbeea0c1e49fe27f7d1174641cd6e5a9c281c0b953a4eda56e89b0fc4ba78af86bc387aaf7d8de8895fb03e759242a139971db04e2fd40b63ddcd4cffd009e5eaf22a8e39fb3c8d1bc12f05983f4a9a033e7b67fe505b89ab36af9b5ac95874685e6b6b27d78423187d00119aa2cdc3a273d1e1c08067be0452831de1e9608", "keyed": "0e51ec8678880a2633455762a587de426a4872976fd22d77d6e5911e97acd1a880178c3c23cf9d0bd160e0d892dba818d287da315646730c7da310809c43061c38e87f561025c098e716edff0196671d113fd61f7e85b5b6214be0221549619effe0aee19c32ed46c4a2fcb2a8642278ab8d06caa80ab65206cd974b3a2c31ba191859", "derive": "b2c4be9febf61f6cee8db15af3c3db73029ff205dbf16af4254b0a069c5a948ca7374ec85f27821bf9f6d144d1270c5d1e080c811844fb24c702d9f05e9804ff83d04e1decacd935d307cb5214cb1d2c26a177f48c6ae6f651ee87fa73977b70ae50c8035df33d208af5c2a0941c68a1c5461f5b8c39abee9b37a5ec8d15b61017e458"}, + {"seed": 40, "len": 64, "xof": 200, "key": "0fc85f0e2a4062e32fa0ff98397bb7d36232ca22dd0577918026b85492ca3a6f", "ctx": "lambda-vm oracle review ctx 64/200", "hash": "602a68f3084d758d683e9bb1e87f90cf0eb3f5fa9b1c72c67ae7ea73844265307fdd7bd060407314419dfeb50831db7271624dcd8a306666a73e2eb7dfc7e0d802dc0521fa05eb69203d791e90b1e4bea32887225cb5980e2631488f10f0d18a98f09a62030d4dee3f03f58a0f0151d3c533e731df86e4df2bafe9d395df1f7e43e1ecb51c6baf1115452d04c11cadd5d5fd467fdaba98bc04537ad265e45c8a52fea8831c8ebfdf37053576584270f4ceb12d4352e3a4ca386d0c94fae692f94bc264d956fdf6a0", "keyed": "ff0f40bef8b7b7229ae920abfca4a8c95648070075d8c960f1add315ecb9528df208bd76e4755128e09b8db80d5f3264ebfeda150910a1bb9597105681632a6eddf071300520c5260a2be70ea6840e2e9678ff236aab3548e151cb3d50cfe224d9d679c0c63ade61af9128d3ad2832e4370820d896486b3aa7a45edffb2a2a20b9f6d0390a3d0021c81be5a078261d05d3ff2c4b1fda0bfb20db84d70e3e1b3890861415a9f0be072e923e36a8f58e06003746e51cf12bc001ae7d7fd7afd24819c7aa4e7101fe28", "derive": "05dc3dedf0ff11444fc56c14533af0fea2487ec806b6fb375447d898f91d31df614676bac84d24a592f9e720be0f75b878894dd31d2cf91c55e98c029a4b966e3d8bd014d503040fefa66d032d47c4f6f0f49dc8ec3b417c75e2365bb1de95cc3ea68a933a15330ab3b9854d93715cf3761cd67769349c5e2e26f68510e01d7bd5145940aa2fe0fb6bdad0c4a0f3b2644387acaa6a09df2da5658686adb1530620d9fb3fa0f8cfa117993737d84c445c0aad5c3405efa569880c34f5f52a03c4edcba84e1307c607"}, + {"seed": 41, "len": 65, "xof": 16, "key": "3405ed72ee6e1e23b3056a704653ef50f87fe7b96d04fdb34006c9e4cfae0999", "ctx": "lambda-vm oracle review ctx 65/16", "hash": "dfb16d551fbe1c4aa129d94330963e3e", "keyed": "fab08e278a8ce8560dda049029890668", "derive": "b602c3e3951abc54f9cb839e0614a03b"}, + {"seed": 42, "len": 65, "xof": 32, "key": "5af3a92e08b2d165afa1ffaf4b4a718184038d58cc4f9ae4d82c26efd3f990c6", "ctx": "lambda-vm oracle review ctx 65/32", "hash": "4c7dfbf6b14dfd1b1b6cfb4d7cc5fddffffaed6a8b0d22fac9d1c2a23659d356", "keyed": "19edef3cb85a9dd2aa998b98d32f1894962d92f737a5d90b07ec6cf56c57631d", "derive": "f2900904342a7229e5a5291301b60d6e188d591338ce5a69393f0ef72ef608f8"}, + {"seed": 43, "len": 65, "xof": 64, "key": "8030ebe642164919b2ff6dd0813434a78d960adadd27bf31473939e8b852211b", "ctx": "lambda-vm oracle review ctx 65/64", "hash": "e1050d263b6e42425a3943383bb6decbc90f6d7efac5353c2d71bc7db0555783781da185db68be42db253d7db0e473adc4751ced78b31aade695fb6173743825", "keyed": "701bc44c78df866dac167e79d952882c02ed7360755cd86ba79900f49630eba449fea7cff119f08a77086143178fa25b866effd36b4ff7928bd9baa241b04bad", "derive": "f52acc57658d78b7b9b320788eb71fcc3af457c04e12c94068d7805529815ee65af87693a00b873b332f5e9b4084aedbd678245493d2a5d30a9361abfcd96ae3"}, + {"seed": 44, "len": 65, "xof": 131, "key": "ee528981c9463bb1736ed4617b4d514eb78d9e7589f2169a2e9c63de8ec916c0", "ctx": "lambda-vm oracle review ctx 65/131", "hash": "36077359619cc4e2676b8fa3026eb9c6794bdbc76d73209a22276f7ef6724faa9ca61c43e9851dc56752d630469146dc84791fbd82c8213854c08cc4fde6148b19486a004872a9b54f9677ef41f3efcf112d3dbf217e8684b6fb48e002b78292f01bae8e0bdd10411d5953dc17c9a2830d1570a9158e15f76d1b67e371078bd1adcf79", "keyed": "b76a2dde6eebd30fc4844e18e704a7d3a128ffa2b996aa89b8f8d8c57400c67c4bfdd7591266d43d359b9e89ba680b6ab6145bacd51d72c2869982fb6bf7c526b319c5b123dc6233f5664f54922ebddfd1d8d25841c3bf7a12ac4a044292ba069c318fa73d29ec98fb86a9e25b0f369a3b79c3742e12e4fff2f8672d38c8b37c145df2", "derive": "77c702e02e4ff332882554f0cd9310ee7be65596c76e1c77e50b7df84065547f71efce4cea00d8bed8bda6ba8d1740ded9ccc42fe1685c3c89f6624f0800295659d0fc690f4bfb0da9945790b9ca93a44bcd48286400ea9dfa3aa38ba0dc1dee2977a932056cd0ab02f96856a44919079c03359201fdccee8d31881923495e0f3fa163"}, + {"seed": 45, "len": 65, "xof": 200, "key": "83ebf4a74303c8137f44412c714708d00e639b7094a7810fedd9ab0c28165013", "ctx": "lambda-vm oracle review ctx 65/200", "hash": "a5e8ad40fb449a0ca7f50cdf35b8b5f192d84c646c48e1f3db70a8ad6f9de5743b26582ef3095962c2e70cde4205085562fa2003bbd4d6ff05d6e549bdf140cee596c179f3b4c83f06e1900fddac36630d16a89000010a2dffd35f70019fe3d35084e0171731907acab1ada9de117ac51a6bce94131931f4c1d6d2a16fadc000df322010c869c9e957f3d23798a08854abf3214792033766e0e6c01fe14d0061a397e264e4ba36e08e7262db6d20114816d1bd5c952e60b233bb83628da2161321d09ab7794445c0", "keyed": "ac1ccae4705f7f9247e0c6e92fbbaadb4c07dfd817b81a0c1ae6e6c3b8cf10a7281ff442a2991121b383029ad9fb0896965a77a4f8c0ba025eae95ed1acf98fcd59901212fae11e458be939b28ad7ab45eb0eb74e2c54b8fec25ed0c8dd2e4c5f785d98b07604e03e87f4936459f078d05c9ad9c01e1be625e61a982b6f01ff3398d47972624ec97cda2e421f73ecbc9f6c95feda44fd303c98065c1b848b1c623fcf9b2ddf7a94198894a9733731b5f9b33f93a5a502b39cb95d7a2c98eaf9de101f8b8d1e01706", "derive": "18b00d90ca81bd1c666c78c35bd781310aac74c7d982d6b30569bdfdfe157d2f2f41163cea7067b9d20572c946ac04344f8f680ce8b57f5c2f16e6a5e6b9579315283b0e661dbd1c873aa1061fc92c9e6c173540fad0a591d6886ef2953c9a6b5d2aef3eab4955afcc18e287b3adb4127b55febc507fa290746c8d5ccea4bec4ad278b3c06feea13725fb6e527d7e5956b65bb5743744117ae79d9a604055be0425cbd0007b127279a54aa09f5de423a72ef5b3d1421f5ffaeefb00a9adbdc338478b9d80086a385"}, + {"seed": 46, "len": 127, "xof": 16, "key": "3a7db0a1fefa0fe7771a95a8b9d8f21e7f7874fefd179b050e46904740340539", "ctx": "lambda-vm oracle review ctx 127/16", "hash": "4778a894166f5af5a67a7efde5af3adf", "keyed": "2dc7c28af3cfcee3cf578e4695d9b138", "derive": "f7912e7c9bd409ab95c01b5b7a068585"}, + {"seed": 47, "len": 127, "xof": 32, "key": "ce178a1a3b9de3ce2d077eef0e0e99be95106dfc3c0c7c89d99186078a9fd209", "ctx": "lambda-vm oracle review ctx 127/32", "hash": "cc328f5912ecc945f8cc29a43e6091d914392a54af859f09607d836b79a3e745", "keyed": "36a924463ef891bd94690c46fa412e395bf7bf812a9463645dd6135f08b116e8", "derive": "2b0da95bcadadbc0fb612055ed5370b016f354b620b1a5b7d108554bf1986cf4"}, + {"seed": 48, "len": 127, "xof": 64, "key": "7b57a2d7709314e0891179c6df8ed261124481c5a5619e735e48bf762831ab09", "ctx": "lambda-vm oracle review ctx 127/64", "hash": "acaab09bce3654c8398ebc7d7253b4b103d22133b8e0c880612b59ebc0575a1a786286c8b9278936372b029a79c963e10046142c1dbce65ec9048260cbcf6c5f", "keyed": "a095b8ac8eb13e496f2afa818bdd18a43bdda3959c2f2dd45f45322b6d0891a7c6ef9203ad21ebfb9da8463f13dd3f92f7bc461bfffe8fd9c12e03859e5cac1a", "derive": "a4712351b39281e204f97908eccc9326059fd4498f833ee9531bd083c567b1506dcc1170eed9ac0a3a88fc6fd4d8531bb756d0b2925ac515a68ef8a46da24bae"}, + {"seed": 49, "len": 127, "xof": 131, "key": "a103179229a58b5516244c74202acf3458ec9c505c675fcec68f583eb6d4610c", "ctx": "lambda-vm oracle review ctx 127/131", "hash": "997bab908b56bdf7ce69a0f1c6a4eeeecdc2d5516fd7fc18cf29a959fdc0e85000b8074e90d17c3fe63213476aa17a193b6f42e1a76db2ee05d5b511ca44d59fa84242d12aa698a21b63a7b96c899cbaec11c95214c0d55dfe5465c9f977c02ace60856fdcbbb35a7c591da0a1cdf5a9cc044c0d87f778acdb258fb21ca6617b51c29f", "keyed": "239e5db4ca6fe92c80f9680341715cbeafd6733cbf111248088215b72223553f47aa63c55e7a15bb8a9802984f53fbf43033e3c23b334fa16a06b59c18551ccca47b34c4f6fbf1c820942efd2d55cac5c3f9df826ebac493d70cb7dcbef7d4d2277a595461d7a8046c544ae779fc327efc6f3e56bc16ce190efbde419cc5b03fbe4d8f", "derive": "b86ba40d7ae3eccbe5a6d9e4533e65326a04355d58d803e97935b524b937358c9ba594a6053e38d043bc2e15396a77e4c1285000d8861244bafb12f66e740b9477c80f5d980da2da1cb40ee06fa3e2f639c8e7e37ce6e217161efaea2196d1d3d8d22f11716efa004cfa19595d44f78c084352c0b99062584a8d9e7eb29292f7cd7ebb"}, + {"seed": 50, "len": 127, "xof": 200, "key": "a480d2a7c82d1bcda32057f4f418bb56e9a6384fea983e669b9b9e75f065e6b8", "ctx": "lambda-vm oracle review ctx 127/200", "hash": "9cb06e7f440251995a8b91c1f92e5ff1047860df725c557372f26d4b14f80c4a7b5f294659a344c67b04affa0bf0a0872f64c7dfbf2c8336c88b09dbfab82250c389e98dc0c0a92ba3c850fdd7d14f2e0c1d3b1f8043d82aae6216a528dedafa75bbd034bf977dec016ca5b943b7fdb063fe1a4daaf7441b66ccd1180b87f3382e262ae1ea5e864424a123ff1c4886f98c8c219a26d2857a5724dedd58d483d3a7b11a436f15ff94e7d179516e6b900e0fc1978f53048e27b7d28ca5ccb6a0221abfa605fc3b1616", "keyed": "d7c39ed0632dc56e37158d34a01fd4983b4e3a80788e96887c7e4d98e27962689b3ed261b26b198b64919703ce60a32a620a67887fd956a03c9b07cb3d0a0bd1dcd42252b3ec4589573e70c5f7b28c78c9ba340b7984865a276ab331da59d0a7a5400593c4f1f55a9b3e6a0b3e7c49459254976ad5eb514171662d6eb7700a8332bc233252c9bb11f747e741df39ab0d3aba1202446e2bfda08f8ee69b77fc4d21a5ebe00096381bc15bb9508fc775e826d1d8fbdae063bb9780993e25f65c60c2dfb28bcf9c6f41", "derive": "e1124d9e220190870e3f796d4658bd050485df3008274f1449cd0d63b2ad056810a1f5b884141401aed964d6430d3ffd9d204d7e7b5fb0bf9c523cfccb1be6fe0910b9eadc21ca3dc8be146a96de93906937efef45a1b18c40f5a69524ecf651952f117fe8df66ce769e276736208172699b23d4db026d199c587ebd3b5cacd186ba696e968ab96ae32f4f5dbe52d15ecfd5fc6eb643b86058c1e1efd5df0576d1078126efd431f4e24474a762a013fae60bf874a1e75c29b7055a405b0c6dce05651fc7107c4e6f"}, + {"seed": 51, "len": 128, "xof": 16, "key": "ca2c9331b09bd8318cb472ef0ff2385f367fcf277359132fe18225b14e18293d", "ctx": "lambda-vm oracle review ctx 128/16", "hash": "7260b268b8f6338baff4c78c44faf52c", "keyed": "e5f4f67b8657119ba6aea3e641564caa", "derive": "9a9c46b5b7dd0b724aa8b0457b7ebc56"}, + {"seed": 52, "len": 128, "xof": 32, "key": "16dfa94a87ffe443e3dec0dc13f3bd6d19796c33de8e376ba772f551b9e59d8d", "ctx": "lambda-vm oracle review ctx 128/32", "hash": "4a63e95e25c4670d94ec1a1e01459041794e05b2e5a9bb1a7818935564290013", "keyed": "5973445a97d555a406ea562b257248f44a478e11d3dfca0b59c83c8123e60557", "derive": "6a43acd3d3798a352d137db9e0afdec0589c6d00b5b11d4dad5baaa4473f89f3"}, + {"seed": 53, "len": 128, "xof": 64, "key": "aa0ab7c74612cfe86fd6b0258a8253a5b40fb2c37e3280d3c99c9d641ad8046d", "ctx": "lambda-vm oracle review ctx 128/64", "hash": "94745003839a79dc8c227e95dfa24e6ed4de58c15b03360c8c16b4c3476c7a191a9234fa8180c5dc599f2664a94eb7d38253cbd97114942301d3c24ad0375f74", "keyed": "c01eab4bb9511b877e0a9b1e20c47d99a9ad9517297eaf140f5503a5becb4773b121527af1060e320e51ce5c60330f3f59096b08cf7794d828c3e12605d53f0f", "derive": "961479805b01dfe530e97214ea5ea55ff58b2b1a0b653ed7087e2ed83ab50ee39548af3dd3f50de83b067d2b97546bc41d956244e9e58a4ede6266dfad32950a"}, + {"seed": 54, "len": 128, "xof": 131, "key": "3f08fb1a9b5ee03886bce106e723704cf0c3c1258e196572e9bf0fa2d66f70ea", "ctx": "lambda-vm oracle review ctx 128/131", "hash": "1c018763f8d48fc81bbd5ab76e8e20b04159f50e19fda1406aedfcb712f9e018f9ddb4d9bffbbb29b38ef100700c68c8fd97cc14b510b77c746090620e4e1198db41b55e25ff6a08bba111c0287c61b5be00cac4803954d604ea293d16b983f53a3287ed43ed1f02b64debf46941a8273a3762cb81d54bca602dbd02dd525cbbef0ed4", "keyed": "3b46b8a4e038f114a6e4a0d5721cd4ae79cbf2dc28227ac353fa7f24f6c0248fc38f39da6fb574fb82e138c47f24824860136fe20023e754c9e603e20f2601b1ed7c29d52a9e555c681b663927e9f168fc423d077188224d49167f7c15631a317af5af52652ca91133285810e646a7efedde51452bcdbd889a296c821c77cfad74d625", "derive": "41618dc6fe81da9ba6d4a79f10381a6ac4a39959b1741082b18e9314fccf1e82f6a8ca667aeffb4bb6430fdab35f3e3df1ba578fee5debf55da9a1d347304feffe5139864ba1bf6a427adbecedc3f80b197f0d67ed92c2a8e986d52dbc0fc01aa7dc92f7e666e07e0cbb7ddcb45a7d8843d1ee417d97e3a409bce4304f36650050f572"}, + {"seed": 55, "len": 128, "xof": 200, "key": "d3329a66c41dfbc16d0b8e80fb27d046b6514604d87510ee726c05808310379f", "ctx": "lambda-vm oracle review ctx 128/200", "hash": "58bda839d3594b2568a91d58139ab02d39a296ffb3903c766ea08efb4c94d3f360c990b52efa7d0b3e5ea44fdc0c31563aeca3142f4cfb711f24739b2dd20383474d2c7999c39954ea4bd37860ab1903a35b42e826551e619111bae4d0a9b682676fe2ef9c12d1f55706ae1b070a59485045ade6ec1e7c335b1b4b98b5d7b73fb3f17c659996cfdc79054b02be7361b088bb2a27044041021949e37a147bb285142a218a1c6bae7c7a71a586822f1665be830c46e055ce443b3ded25793b8d151a1e0b0c3c031ae0", "keyed": "d3f6bdd9f8574460d01f25f2cc463b007cf28812493d05531ed305963789338306b85cb83986e0923fe12c1a30f79326255ba8406ded0c47cd6116f18e5d09f299fd7579f69223f91a546b535481a292948c832354dd4a823c2b458b5158f1c42a6fcfacc8ff5e35d047dd0a45daab8ea491e8e9622ce71ee4314213c0487c14dd7ef310f0461f67556243c5505dc7f73a5ae676b2ef017502048f2fb9ca4f84240bdccd955b6d2c9b50eb12dd5cac000d08b33d1874b910bd6bc808e25b75168476bb04ea76938d", "derive": "6450a7dec0869a1e3c8acfe39112f70f3d0b3e8c296f0c3096235b2917624fc98827f8cedff426ff6e189bd360d64d77fee2d293dc5f62b8b1a0ea90b73bfb40ffc5c139796e969bb589ac8afbfe8b8e501142b8e2d966d70e86c87de016ecf7815430225cfde36a706090c3e71439836ea38a481c6bca101ccf496de13b2ae4919b7d9d426c9d41560c598b624e45b366020a850c4e135f1c44e99d978f4d99d02a076c51080e6e8041552a9b9ea8ef6df548613e8d1115a40643c658a444698a75cb7a334481d6"}, + {"seed": 56, "len": 129, "xof": 16, "key": "8ed29fb36819743a8aafa7633b901089aed972c78f4e659dfa303fe3e7d5f763", "ctx": "lambda-vm oracle review ctx 129/16", "hash": "53933218cd3a7dc226dfda7657611642", "keyed": "f9806797d6ff511f653fc782afc27c5f", "derive": "bff8e1942bdc60787105dee8ecb3c163"}, + {"seed": 57, "len": 129, "xof": 32, "key": "b37d838c35afc413d4f4a0834775240e29b5cee2b4a4fd4cf076fafb3f075341", "ctx": "lambda-vm oracle review ctx 129/32", "hash": "788921dd78ce7a8af5f477677d8242ab2c1fec000dbea6e20356105594ef6977", "keyed": "6719b59a5e51d0a42ae683629e6d8b67c5c49a11cd69a78ddca2fba1af7767fe", "derive": "720c8d0efea80152a623dd2b67c8e707fbb93f2dfbdc6b2d39ddb1d649b61ec5"}, + {"seed": 58, "len": 129, "xof": 64, "key": "d9fd0cc1961360d4031cd4d5f1ec05cccb77565ef4b82eb4ae4d6f9a7883a731", "ctx": "lambda-vm oracle review ctx 129/64", "hash": "661f362fe1a4536f81abbb67e8c8505e9ecb0722aaa843a51f32613db21e4c16f165d3d237eecb99260e3ff88f8b43f3cacd0e323a9d0938e4ccbafb3212c4ff", "keyed": "b2b0f1978f87ad14a9d583d4cc5512192e15a3e9f425213a44895cd9ab3541023609832d8bd47a53d23d6a6ee091a860c8e5d3e8171c320c7d8374aa950b5a4d", "derive": "5dfa9b9f0ac8703982bd9f1dd20910601acb82b9d5a4cdcce3971b4fad21fcba9b5fe2ea7247dbc4f0ffb21f46daa8256ca0f04eed428f05f544ccd34b1754b6"}, + {"seed": 59, "len": 129, "xof": 131, "key": "ffa85eed76fc44128bcad9bcd7ae3713c2dc1206f3c61e0ce2774834da5e0617", "ctx": "lambda-vm oracle review ctx 129/131", "hash": "5a7a52e7bf5e79814bd861e2cb80e3af706036b544f8ed5e84a57b32e0acb6b641d009f9fa8d095ffdb0f00374c743c1c34edf3709f88cd33ced94e99887e3b1c709f0b835eb79c8726a36e1f032286d70b474843b87f6be8fb55584ef12b25adf776ecd544eef2794e72b77821ce05ba955d39be3ce18e791506e904051747883e6bd", "keyed": "98c57f6aa22ef67c09b40eb6c9f25133e3f439077d6f44669c5fc1562276affeb135ac5a030b438549803e05f07c384b1088cedd3849ae5dabcf2fce16fb7bac3065ba7b03de9b3c50c03508a29402463aa0876cff9604a71b8537856ddd8b49b0375e3e5582cd149293680909f40dafa59e7efa9ef0245ab3e810c1702e5edae2c261", "derive": "261ded465c98c65aa2623b43d3661f1dce5c48276fe0288ddfebd50967877716f60ec020080e6364f3c9577f2c0fc806b960eeb0ee61d96aa6e3340246e9a089f9bcc13d689454eb14a6994f5e6aeb2b5efd1d7b478557faf83a6b6da686b8a3f0aaae6639a66d948c3f76ca955b9fda44f09500e15cc75d98b999f2f72db52a63278f"}, + {"seed": 60, "len": 129, "xof": 200, "key": "285a6026fc81bdd9abc31cbbdea815f5ea45e1ec71d6617912542068e54634c1", "ctx": "lambda-vm oracle review ctx 129/200", "hash": "1121fbad05a0ed2ba8ff2498d45a9ea7e860384b021aeabdfed3c67f0eef7408ec6a15ec53723904e10c890e7323d1b509b3fdc0ba8fbaba2765eec1ab6db11f0feb360d1c92ee019b83a2f86aeec5554850ab924965ddd94d294cabe56b1db35efdada8d81e0b8832da33a47ec9ce135d96179ce3eeecfd71562187ceb9f5a550dd006d17f71fbd83826e1b1abbabb9719bd4a43ebfbf88572a9576d4e91fbb9da188de5a7cf2f2e2f3aae7ccb3eab767c49f82e127b172fafd76672179638768fb5fb06e946647", "keyed": "398d09d5b5cf7402631bc8eba3a4d4050e07a814376eec7edd52db6cee980fc104dcb8b8efc8a0132c2749f2565c78bc19db1163da064ea20a78c344c2219203fef359e4e5f9a40bbec9d401c14f041be0a034046c424d1c7754c25aa41f7ec749ec7f47e461a6c676b19fe4068f925a7d9792539d4e5acf92d16015ec05e2f4d889eddf55dfa5155098ccb903dc871e54fdbc09e83d546558db1ec6902c0cfda170f2fc2de829195766f65880fb8658d993301976f0e9f1799fbb5d27d38b344ca4d70b91b54b79", "derive": "1172b99ed6904a716737def55363442c70ee1ca85a08b6e62bcd0b1f15418b209babf8516c130a4732480acc4ca3d052c3d1644b71b7c6785d70083b6bbeb1945f4ff6179f9d80e177f44fd956e6ee4895807dfb70e7c8bbb8771bf3447f426c3e418af5bf5142d70090b1de34d2f1367aaf7f28ac844227bcf7633dc9c7f1c220164e9b70b00f1d26abb97c3e4f0746123788739224119e6071d4d7fa49892689c8e2463fb22ed28efc9207f7c64f2e768e172362e3382fa9ebeccab5f3d1a8abd7cb3c837c0a10"}, + {"seed": 61, "len": 512, "xof": 16, "key": "bc84ddc1f4c405a0d70a7a20b0a65b90abd9c1523a575929a31b51356a1406b1", "ctx": "lambda-vm oracle review ctx 512/16", "hash": "128659308e8a7103b89954328ff3dcc5", "keyed": "66ac5531db455adfae33d01f2424b2e3", "derive": "3714887158aff8b39e20460d1b1df7ce"}, + {"seed": 62, "len": 512, "xof": 32, "key": "74856534ef4035d35cf4cbb65aa0513428d5722c34950e422b79253741136300", "ctx": "lambda-vm oracle review ctx 512/32", "hash": "61e3770e80dffcb46dbd31d0f0cfe311cc746d2d051ce2fad4ce235b8249b6aa", "keyed": "9701c520da175568b55b68ebb9b9e38671c9354befc3751fdb9279cc6dde019d", "derive": "f81137c37343d28421733991ceba4f7a7a8445c58daf78a5f941c48935091771"}, + {"seed": 63, "len": 512, "xof": 64, "key": "08af96226b264d763b57413e3bf21605001c95eae5ea05b5c43da130647b664a", "ctx": "lambda-vm oracle review ctx 512/64", "hash": "ab098f1a5b5fa7e09c5681c5091a5c160ff00715c2824a4a2633401521ca46ab86239dbc65f0397dfda02711e457be739b40a4fd03bcf729503e57885a7bb1a1", "keyed": "b48711a18f51ffe38c41060057a47e0174cec29eaf16b10a0a6613cb23fb5b06e347e3ddbba4a2b6dde67587561452b2dd584c133bc5581062fe9f97b0ed98f3", "derive": "80ded338ae4f1d27505253e93dab1949bc51e8a2011ebfb24d99678057b6a18a8b0c7c12eebbc78bc7b343566b8fc7eb4be7705f7e975b35af72dffa911d343a"}, + {"seed": 64, "len": 512, "xof": 131, "key": "96f47b23ef2dd2dc947462660df783ddfbd7c45639e8d2107bdc5646657c5cf4", "ctx": "lambda-vm oracle review ctx 512/131", "hash": "c735b7a9e0afe293904be51a137cff72222202ca676435ae5b41efadcbc12653b85688a678e80d95f9109c10c689718a825311d8e99f5cc1f30bae70420a8e2ad9bb1b8fed710c789577500aed255249ba6ed21abd355de74dfee33d3648bcd439a79514b64416073c7db4b635b107a002ab71c9019de007996d51539ae9881759106f", "keyed": "f9d0d51d41f7a7916272ae75ab5330a4feeb92e927bb53e948ddf5a81d3cba6ebb757ccdb34719226cd4e425fad2f762cedacfdb57f65a63e9169cc5c35c46f02f37820e2d3087b3fd8a64bcd28d67df0ec9ce78a4d2caa02c0c80d43fb005a8b73e7f265c108c72a47baea9532698639dd9e12cc6275dd1b03c7bf687b3bbfd776f1f", "derive": "f5c3a9f44e179eaaeb3380f48f03fcc9a84ab03aa7db263b1580da1c6e3ebaf91daacf3cd305414ea5d90f2d368133acc4b579e6b84cade514e93df6ad20f9f39bfa10a796eda1cb142da2790e445294e3b12f3360d1f65bca819647c07234505d1fcbc63f56649047e4885bf1713f62c9a6b66f1e699d6f2324d0161507997e22c42e"}, + {"seed": 65, "len": 512, "xof": 200, "key": "bb310801144662e300b5f7e356d7d4fede0efe9a98aafcf4ff570ec3968023d8", "ctx": "lambda-vm oracle review ctx 512/200", "hash": "6fb76fa8d36be7d4f31b0bc7c952282740b91f430d12fdf645d2b5bd598da9c2fe8f616b84594763d2421979a7c0c31932952cbf669a5ca8e60ee3f3457c9e21041cedf59045c06e31625ce457205eabf5f7f6a9cddb34a5f79a7092c084915d509b7bc141d29630c9a520277a0bd5044cc3a2035b45464c928bc5e6361c9d2aa1be71be9f88406a2cb8a015df4ae3b0e44fbc709ed80d019598936df42e956251813351d4f7c420914363cab352f7e08e1747cbdfa7dee4c118daf1d6cc7269c34ee7ad2eed2b29", "keyed": "35a85673fd6202f1e550f2ee748e8d76d146134a5a2e7e503603d934fd884cddfb0b1b96046877118ce52cec5459f414606fb538cb75b3373e975566667a7088640cbd4b746faa3705236012ecc94b9a0de54783d679e87587ec9e7d9eeafc515ce532a88a68746fd615189d81f638d3ab2fa1d82a09c058d7a2fff2e91fffbc4a307f2836203f9897b52696448e041bc5628fd6063c43d480c4da2a8dcf16b2ef0cdbd039ebe96a08ed64f3470a026399a13d70990d6b569035521ef789471b904346448defaced", "derive": "06ac2355277dceb26a410851af772954e2e94f4ac8f2f43e55b06b0baa0ae50dfb33744efe8e6ea2aae1fe758c187707bc4975d381e67e173714267ae944d6504f8858e5c9f59a56db538ff002c7d0f305ad253951ace5e363ffc5652410e8ef74aa31cb92bdc3c370ab562dbd0e2094f1d687602ebcc0d3f9a911e40f1f917bb9937dc0d64c6f202a0f051ec1dc9bc9c96cae56fd5618ec9607ec7753a42b7e4e9f03ec73b4ac3264210c2873fd5caf468eb00dd8520e93f7b049dd2f05d99458d7a53126193407"}, + {"seed": 66, "len": 1000, "xof": 16, "key": "befa8b059d09f3a8dbc368577b4d33676a30b41578c029b9fdcf417ffe9a43a2", "ctx": "lambda-vm oracle review ctx 1000/16", "hash": "6f7175181e05840cf5bca9a1d59ddd39", "keyed": "31b3ca8354e46ac283cdfa7de0becca0", "derive": "3e44e34bf6578e1ff975f05cf77e3510"}, + {"seed": 67, "len": 1000, "xof": 32, "key": "e437cdb211ffa77321a331a3bd48759e26cd010375f1d68219457be4fd38b6f0", "ctx": "lambda-vm oracle review ctx 1000/32", "hash": "a7a6ab659e13f7f98a6c3500998ffc3a17416de7c9d5f3b3163cd123838e028e", "keyed": "ecb1d9e27fba7b612c9fb8a4f4715706b3b838a33c357279128fc49b9c454a40", "derive": "3b541d49dbaf94f167bca1d31223ba17009363e27fe1527b771392e0ec2e5f25"}, + {"seed": 68, "len": 1000, "xof": 64, "key": "75c4430c6cdcda06c095d3f03bdbda5b779e718476be00d654c73ac6330d70ea", "ctx": "lambda-vm oracle review ctx 1000/64", "hash": "89fa21877d363271797bdf327d4ba0f311139f593f4ffcaba00d13eabb9837203ae6f4ee806582664a8160a4a065346c6f923a6102e0f9606889040b16b6c36a", "keyed": "5b9f65641c5dc2438c736ea0207ea79f801385078ad64dd33afd2dcc8fbd101e90896b4c672eabefb1be15c3a2908e6d68a6b0d2789614aa75a1e771d3910800", "derive": "ef3972e6aa2c92126b3f8a982bfefb489d86031239d7d6da94c742d533016c602e5b053d60df3549775864d39350ec69c80b38178e0c297184cd014248c45fed"}, + {"seed": 69, "len": 1000, "xof": 131, "key": "0a5daf6694db1d9b7660c1b126fc81066d359c5b5142a13581a2f9a542d40a70", "ctx": "lambda-vm oracle review ctx 1000/131", "hash": "4629528981a3c2c0b64c229c8df8d050ff272caaf4a415032c625bb8e21538174b8d38c7e549836c968ed937cc98298f6deeec12ba4811da87545a4c831a965417181f7b5c59b5b60d1a2ab5c6c4e736949fa5e24c193a61025eb0bfa574fc360e0b237107b3922ea180fb81c5e694dbcbf10c5f541bcb085edcaa56119a3fc5cf3e5a", "keyed": "686683bd2c69e03888ff6a9249f7fd27a793f4573f129bb1aa2afce4d38d2e3118ef6ed035d090d68bc0cace642f168bbf4b1e14d382817cfeef8c51c87ae63c356094a40cbc2d3dd48a321ae5d988ea4c0594390e9277ab97358f76ab01790a841449d04340ec9e2bed929def2490f6d32fc831db678f01466f9692b7b435ce84255c", "derive": "7cedaf8af519c0efd6c7f7b7917b512cf33e3ee555262625041131f660ae4cbdfe96a4a81a3f14542197696cecd174efa1754894b8c30aac8b382b082f8252ea0060d11945ed3e3d6c94912dc4624ae3512551715b4e18d2d74ba41df6c633839f8b33c846561aa1ff08db9631f4ce0bfec8f7ebe94fe02282b72b31e32c373b4b3d59"}, + {"seed": 70, "len": 1000, "xof": 200, "key": "9eca31ee091f8bc6ff2a265069d32a2080cc6aadfae7b2f4473258faf88ea762", "ctx": "lambda-vm oracle review ctx 1000/200", "hash": "4c4c68cb630b73370f497003ed1fb20d23540096ef693cf499afb67ae5285bca71a17877c8c5fce6ed6bc9a78c42cbe77a978f3684ffb6179e619d3d9a1df86145546ade247622455b41af82eba6cd14fe268ed0d8a3e7d50675d796ac8464c42ad20edbee74e584441150b3b77ecb5cff422d1e684823f5b947aafc47c61400696302e36bbc05b367a09d229c153761fbae1604ccbc1e7d9cae9e7f0e938879405c2525459c5b98431c1eb70f8fffe3e909d5b5ea8027eac22fe790202255f1925d3274d7ddec2a", "keyed": "1abe83b666af57dc91ac342cbb0f1ba56490a53609cb4c38c3297f32cc31a919c9ed827bcbaadb32d861539f7920017aafe4e2613fa2c042bbe734da49d296e75bb5d0037f7daf91afc9ed8ec1eee2acf65fe9f12e6d2a43d0b809a6a67d77a28823189e03fd16ea2acfaece5c5b6439322d5d5456248826faef2feb18066953aacadd5391a37c9153fdd776f5dcfec1fdc831869a711a61ad54f677c10467df28362a1ea956a7b8c47fe50779e17700ed3065a9aa0a7b1b8e07c47156afc8d101b67316f199a427", "derive": "31453aadda0b9fc43f8f9e3163c11135dfd69c894e5924e9ae45c17e00466abba8c86a85f3b22a7f6ecdff2c2b8f599dc5dc8ab35b7436e70d922425b598ce2f17f6a10efbc0cbb7a1673b70d8417583a264647e1ccbdc8e219408986b448c7a14aed4e5191587a533eeb4e6fd8b493556ff5e1c28bf8994868674abeec5e1ac8c6eca2e04ada8f24d46db692784bb5e2dd336d5b24e28dca09d80a5c7888a2123cde8183f8ccaeae5225426e8932a1c726ca63e962d2236581c12abda25c6e214b45ea14bf32ea9"}, + {"seed": 71, "len": 1023, "xof": 16, "key": "33630b173362a69dbae2c07065866154f76f244e99f1315197f109eae4b74b56", "ctx": "lambda-vm oracle review ctx 1023/16", "hash": "21d7ad694e7edf37046214b4f872c6d3", "keyed": "9b73988949059085d68ecb8518945058", "derive": "816ad3aaaf4c9ff75502cc055e88f84a"}, + {"seed": 72, "len": 1023, "xof": 32, "key": "32552bfbc19d7b92b714b972065e0a182c3846351fdbf9f8da473a66d07bb089", "ctx": "lambda-vm oracle review ctx 1023/32", "hash": "7d4502d0e2d35b69bb2b6cd2f5ef61e38944736c200ed043d7b2ad52310e7be6", "keyed": "e63ba15cf67e4788f9aa504e42dbe78dfdf51de51aa5fd61a20bfea0c29cabe1", "derive": "6c90bdc9d0743dd9a18bcad1dff79d71c117b8c238fd54ca0af0b74648eefe0a"}, + {"seed": 73, "len": 1023, "xof": 64, "key": "58924bf7db31de95950c6df116b7c3d92d800d4d9194e144d568a96cda1f5404", "ctx": "lambda-vm oracle review ctx 1023/64", "hash": "4efb0d84bcec540b6313a0c6da16c19f6c548d825322430a54b3c1b15d60f17f3006c0199263f3183180a80c47752b2b890269524d0710254576dfa13fbd75f6", "keyed": "58ab643960e5e2fb588ae0192a44ee111dd1ad4ac3edbf46c4b56f69da3732cc08d70a89440a7cc3301f7aadb307a77378b569aadc756f45269eaa7250bd4e01", "derive": "b10359e0c1a9645b79fc3b3c49a4c0f5b8b4a33739e38572521b1908cc3c8b857349e672cc8d7f976b91ccb7bf37ed104cf6de35aa81700cc69123d311651dbd"}, + {"seed": 74, "len": 1023, "xof": 131, "key": "7ea3111b6bd050ee3b0f5804f3b26260e9be092dba7a3cedf86e31c2064c676c", "ctx": "lambda-vm oracle review ctx 1023/131", "hash": "392e9eb9433f4498cd9d9968ab58ba0f7a3a0e07e2187f0c5e35b655ed6e94a1318e6cbcc7fcd27f09a4d48eea14feb22add27d001c4ba2e331b5a8d37264fc034ad4fa8431284ebd1aae6f9b7cdf5e12862a46378b3bb2c94e8271e71979cdd4fe21eb3b582e83db6b50f712012cbcf904e021527d1835e06c99ccaef73a356fd72b1", "keyed": "94fdbd4d70e4266087dc015d7c404db05941db771b69b5958a8d563b2a59736c8df29b4f1462f39a868859446287e2a9f9193f616097f2799512d501078a6f2f16dc396e9a4aa7e42bf84b78d6a83fcbcecefad6cc7aa7e3d08638f4b69f4dc09ac3ff4087b95c90db8aee339dff663710f152c06626244c5ee58c75efcafd9b739f84", "derive": "30a5ba2a68b85a8a76ae8eb44b657782f4b845173d98169b9d79c3447b8e349489cc3141ea763b61d7e0cc825af7cfc9d3d3a78d86437b2e41531902bee1ddef2f5433032e5461840f6e89c691172eb486da91c1536ca59b4403baae9ec6eff03270b5cbc3b11d068b98a4614e3fe5e0354d10983df886cc00c4b98f1fb4eb2c7b9b9a"}, + {"seed": 75, "len": 1023, "xof": 200, "key": "a4dfc16ade4a493f07e112ebbd4a73ab4edfd7d7899761ccebfb8492a295b756", "ctx": "lambda-vm oracle review ctx 1023/200", "hash": "8478f370e33d3592d7991f12964bc4af23d387bb53aac03930f9ae194f4379196a814ec634008448bcde817ab14237fd0e6869649c3fdb74ec94ae5b1c0a434d0d5b3c60d38e53d07da811aebd7195f6c8bafa2a52460eb52ce7d64201c59dff2e764bc643f3579392f90ff9895fc5c573df0633f638b05936fd17fa84cde873e5b95a57e156073c2bc1ca48048f7078bb4520b81d7b1d4acf9558817e024c9a09872b3bf2260909b01ce4a21c594c91d579457749e0b0b801fd6c9d6b9e466a59d621f78d407ed8", "keyed": "17c56f3fe4fdc6d0e3df2cd517df23c1db7e100ec5aa8745b11007608c32c9a58a83573685159f9724b28e13330841964e626f9fcefd62a12314fb2a3f12ddc30d246beb5093926d199762853846e8e2cb050c44f6ca11cb4db8e6a69e0ee7e9c44b8a78c9ba43b6dde1a31ecaaad50d732eb22a7773ecae84098a5f8915562b5bf9ec1713e8e8a9d8d6daffe5e7b2daf3745a76ba1f2ef4f1d459f2e91103b89977e98e5192d10669d6351ed5779a9b44506fd5a707944179c2bb631c1dece54610f6981fe082ee", "derive": "bb14d0c8f0faefb4a6ce5cfacb240ea982419c862da6463a0feb08fec167edb8aabedd5abe2584b5b44508146e47399173927d08d0e05f2a442065c7d68d135d2bb4fa39c3eed3e5289074a695d4b7fadd8eb55f44aeb50195ff2424da1a9b0d34e7dcf73078d7a90414e1dbd603826596763990e46385fea060131d2597736ce4dbec61a6870d9e3b7332c5d9f5874fbede6b078646c6d0fb003a1f0435ad76aacaa47b804032622589fb474b8473e67794c42d99d5360c39ede03da8dcd427307bde1d74bfd73a"}, + {"seed": 76, "len": 1024, "xof": 16, "key": "122524e4b147ba890bcb4b36a1b39dc0588f4c21eb663bd20397e4b0cb2cde27", "ctx": "lambda-vm oracle review ctx 1024/16", "hash": "22c2bc21637d5a3a5c5772a2b41daf34", "keyed": "7914fffe0f698d80bc7ddab03e05c6fa", "derive": "076f9fc6b3a35312c5c79c687e9f1921"}, + {"seed": 77, "len": 1024, "xof": 32, "key": "a7be215c976dfd1555cb25917f28d872d4660e506ddac26586e1d1547ba4a5e0", "ctx": "lambda-vm oracle review ctx 1024/32", "hash": "9d05e80a80d43143baf493decaa5f031fb91f3ce6fd81bcc0270e9473ac94d6e", "keyed": "c3b7affcc71c6efe4fccb3278d0d41acf8085c81170a03470924311c8c6f6167", "derive": "52a053439dde9e4d68eac7ecf2e32e8a3baa17f6ec0e291fd0338718806a9c44"}, + {"seed": 78, "len": 1024, "xof": 64, "key": "5e72720477e1a7523601591e2adf103c8e5bc3c630a43315f36fb8a7cdf62e0f", "ctx": "lambda-vm oracle review ctx 1024/64", "hash": "d0abc07ee8290bc21e5fc11dcd7a2cbdd7ece80a5b916089bd023767d49e0a07e5ee1cc38e851dcf448494397f2374d5a0c370b6a70a2393a6ac06fd7ea6e145", "keyed": "f446907661aef0a2c702dc83f991902ae807f8c009fb16be9b93735456e4d41b0bb01a824d683c45854d11e2bce40bfe5cc82122fd9c465b06110fd966041fd7", "derive": "769c7cb81c0a7a115c8688f45902cb3573aec2266b8784f71443e657e777744a31a031cce9bf5d0252c5380b66dd9519b2c151632aa1a36c1e6790a09d97b094"}, + {"seed": 79, "len": 1024, "xof": 131, "key": "f20bbbcf5654a52a0d073e10a2c95a039f4c386ed0d8aae138eba6eb08cc443c", "ctx": "lambda-vm oracle review ctx 1024/131", "hash": "4e82ae97ec70084c61ae28f8f724ba3ceef17cfb6e2e89c4dd1f075a4d74f6743139e37de17ce3bdc7dfe370d212f46bb580cd659d445ce1a278781a2f83d4a06dac14aed49afa922506fdd0a59267cfc72ead8e7a7c0396428d9e941b6490908773bca2821de661219b6b51ebe33166d3eb293b95db50822e0b6f231f956bfbadd336", "keyed": "6cb89e82c21309efb425ed74278e6b7d8812d1f210b6ba85e7eecf2251a644d9670b8e05e721d60c7c9050ef68e722bc01cb59b1a3f91fe5395655e0805c1421ff3830033c234393af2a9abdae8b06827cfa9e01855e41f53fabe8cc47129cfa5a937530a337e207f86ab70523694c5071ffaf1aae614160b7feb18c271ac2d4c136a5", "derive": "0a9c479d7c63493957c01053e26c54b45ab2a302af25ce1ed7e820e82e4b787f897530604a257932d097a28518722fed68f570f6134c239c55f827141bef08930f41ef0c0e82b9acef19d4de5c661f101a09b8e8d80e47cdc189044b2151dea929234399774e12d44cf0c07390625cfd2cc17aa15eb24db2afad017904f3be4bf803de"}, + {"seed": 80, "len": 1024, "xof": 200, "key": "ff32ba83c1a39f53fb158d0e7b3adde55e6312af13b7143f67a106771f0ef5ec", "ctx": "lambda-vm oracle review ctx 1024/200", "hash": "4c7831baed8e7b961c5196ce58fe3b8954e90e77db26ca67cf7bdb66c4122a7efd33827be7372dbfe6c199374601eeb83c65ae54b3343b54d8fffc0b8e1a5af166e46ae44f5a140dd0d210f2e646099e91c4b4107ee5182028d32d66a534fa90dd9d45d4b54ee8fb482a9acdb0cf405711140e6944b619d4f185de6be35cc8e69157b876e7c6632c6dacce7ef3f591f1b45706a3d75744d6e6fcd8e193a52eaa273cba7afe60a52bb9c90772a3fb8e4004c51446cf31de17b7cac209e6033be62aef01cd86fe9e53", "keyed": "06fad5dca2d0c1ab104879bfcd86f10b9c2cd16a77cae797c40dd5584fdd65dd7003db899983836f9cbb8144c95d9dd5105fddeb5bd46559b6f89cd597e8a762de0a6814705813434c532380c9fdd6dd086c951a364e32a699f9dea809cade917f528f8ca390763079220b505be258c353f574ed30468a666055d60a7f8338137e57ca4d83b3e1dcbc86e30bb6a0ba69a9e33cee53bab754fc8c0ffcb4bd3592d9e1d6a274bb2d2da0e4bd8c32d34a3842fd5d3ce564d22ddb04bc35ff619f54250edce6c36201fe", "derive": "ceae53602dc8bda3ba5f70e37eac6c52e82380ed108241729c6def03ecb1c958b1dcfbc953368f85f1b95b96573bdc960bfcb1d23f86c7d4d05e05a79a9075707a833227e308173e06714ee3f6a24b2e620ff5daba96b8c63a3777ad6e50d3a6886465c194eb70406b261defad4dcc429813c4c46bae92b2d534561be169c5da0aa4e9b19ab67677cfa3262eab29ca8632484edb8f61bb6fcda32da33eb6bca12d5c5c595866a1e65695ad4f40a3a981e788b442fcafaa9f291e8ad4e7aa38119cff8cfddff5611c"}, + {"seed": 81, "len": 1025, "xof": 16, "key": "25de9eec69b24b07f16d278ec58ded50a93efa7241313709b81091cf0d732b97", "ctx": "lambda-vm oracle review ctx 1025/16", "hash": "eb7596ed1137e1cf9a7c4552bd016a96", "keyed": "36724602bbe0368b2123e429ed365a32", "derive": "b4f1243fc61d26dd1125b33a1c1b668f"}, + {"seed": 82, "len": 1025, "xof": 32, "key": "2838ed52fb997fd9e34ce40ab8e87831dead7c724b4254c8b59b6279a38b2c01", "ctx": "lambda-vm oracle review ctx 1025/32", "hash": "515d83974324f6b2fc3576abb7cb35d5a806079ac79b008183ab128bc687c3cf", "keyed": "17d066fd3a500a6a67ff4219bfb8607ea216f1c9d28373f287ed2c15632db630", "derive": "3485757d5c86ebf235ddbfa8d321602cb09e18dc428ba4170193c70b0f9c56cb"}, + {"seed": 83, "len": 1025, "xof": 64, "key": "4ee4408b9c7f1660100715784b4ce89905fc8f75be8ee48080dd392bfaf97a38", "ctx": "lambda-vm oracle review ctx 1025/64", "hash": "9ab339188c1e0f5503f7f464d22a46de9f6557bf26f6899f16cf6917ecd90c06823978130837cdd82a919b60a800eaaf284a6c910b8f1db578e5c2eb8885eb9f", "keyed": "c7cf6bddc2e55816047dbc1203a4d28ba617a5486b5a6771730edf6ce0d83863fbf07da183fead72816e6f986bd788136b341f97a1067139241acb9f11e736bc", "derive": "7e44d46c1f99a546c9352afd1bc02be2876b27664d1140c424e4cc244b16740dc62524e5a60330ab93d0faeb4fa3f2f6ae1a5e73b375b58fab7eaa9c723f4052"}, + {"seed": 84, "len": 1025, "xof": 131, "key": "9a751b6cc27fef7a0468d503107e27d8abd2f9c7779a39da42c6b0681f0c397d", "ctx": "lambda-vm oracle review ctx 1025/131", "hash": "67377651f2b0f5b916afb417f7fff4e3dece82aede4f89160be3a8956e41343e878a86f2352a5e7502583e1ad82173843897304d421f3e63849148580465a8533525c014f6421291cb028a1a84ac06e828e1d9ce378aaee0284e38e634f2cd9d74320302b4c1e0f165cc6cf90de17dfc9103afe60c8bb9ab81a062093a5d9df14ddd31", "keyed": "96c29d9fe588587498c961de7a23df82d564a8723ac5bc3e64d0bfe96b3fbb5a36bc482b4d9589b1421829b28d846e8df53729010627e3f855eda5e8fe3ca91dd83cc2b32d99e49fd511af21a0edf5eb56db0bed06584bf2c84a78ebfefcf12440af201ea7aef0c7f84068dabb3260ad3fa43eefd4b7015caa4bc55dbaec08916e9fb2", "derive": "ddecdfaf6e4b58a7e29d4508738406a4e5358e2492db8d5523d7ac7b4ec5b93f55d663fabe40000599cf1442159c864420d0ede939ec960b3c9182defd0d204542e9498a7bb769b828b181bcd75e36843db0480da80ee6b0fb698887c6c74c9991ea379564ca97b6e23ba329073fabd11dce75de6fc779f21670abaec2a74b7a8ba8e4"}, + {"seed": 85, "len": 1025, "xof": 200, "key": "2e9f9851066dc5a89e022a1b1fcb9dd3ea3ad37231117f01a9a30c393ea4d93d", "ctx": "lambda-vm oracle review ctx 1025/200", "hash": "561b9a7107a3a0ec41da537e6436d614de0928fd8c9f14e5e4db96469612c0461b5659031dc5b33f8f46b05c260b455cb4b400f1c16af79d18abb9c7ac134fca2d36a9024b31fe9ef4d3050628512ea0f945466b9c7d1ecb2a807d9f7e9c2611882747ed315e6e5236a11fd31ac672d36740ee41ac58db2330bc9f2efaffe5ee5ddf2253436013fe604435128ec434422b166e7b64a87c14eac7105e52e2c09c3fac5fdea6b3416ac44e29252c4dfcf4cb873488493a1aec0b9fa6ebe222744414c5982a6878d4da", "keyed": "8d7ad356afca0147cad9a7586babfb2e699d00d42c1dcf1342510b995beb3330ecb5d7f65a1170f325b0ab8a598fd3cc1d93e300a7124f032a5431255e931c56a0a90f67904d2996f01497d9eb3655dfeb7b55c44e3dcc3c41165d8c8608a51333a9474b6aa6c0c3622b5ee2261757381d12edd67939162ea1502ec17b94d9dea4ad33b42d8381591e1f91bd28f4f0b97f339c8d9e8ec07263c0e3bd12eba080fc4bde37413ecf9b7ba98ce52eb130185e219b826d8c747d132f58df61348f616fab68d72786be98", "derive": "0cc6758101c11c1e83655fe193ba15974da360aa10103233f08e46dd8e6dc9775ab0547d2ceaf644fff8b0601b04030793416915e97a0f0e5f0e4abf9a1d21e3d5b2ca252ec85be73fc26b22aa6f226eccddc2d7371498fa4ebe2e8f25f927fe57d6d4bcb7ae772146256a59d05e77a9e6e4710dd5f42605b0ca29122b9832b07e789c79806ec493a47313a088f17e30085f4aee3d90b7455a95f2024ebd00e096766724239a45c4a506ec3bc4ad44d1ea9a0e77921026537aa737350ae1a316c4df190b03e72b9a"}, + {"seed": 86, "len": 2048, "xof": 16, "key": "c37be63b7d15d6a397ff96ed27f207d7e29d9a22280c4c6e3147346f8b4c09a9", "ctx": "lambda-vm oracle review ctx 2048/16", "hash": "eccfe2bee6eca100dff3592bae156923", "keyed": "addf3e4da8e55eb85d9f0e38721f1f60", "derive": "be4ab18eeceb4e26cad91450546f2da5"}, + {"seed": 87, "len": 2048, "xof": 32, "key": "57a617ef49735dbededd8faa977a78443dcfa50bf196c96963bfb744972763fb", "ctx": "lambda-vm oracle review ctx 2048/32", "hash": "4749afc7faa42ccaf1222708f798c18a1e11146039f23da9f3b009486822f209", "keyed": "723b806ec4a871554612db8ce1c077f082793feb24af79da0da1a86d43d443db", "derive": "48e7b73f77f56d57dbda815fe5be0409bb668ddc0ef2c9774516ab27eca6b96b"}, + {"seed": 88, "len": 2048, "xof": 64, "key": "9c93f6150956b1b932501635387e36d88f1e9134fa430aa814181522c36d814b", "ctx": "lambda-vm oracle review ctx 2048/64", "hash": "e9e20cb49eab02a1b08dbccf5b09fd3b22cc43d84981865680eefefbae22584b3f53c0b3c9808b6d555e355e3b2fd4626dd5d00c4b65a781c1c7b5499969ac00", "keyed": "11ab1e09e164786a65ae13aecef2165c98fa89ef0567189b0dd8ab1ad6e78c4e2e5196272cdc58f02340a5f25dcad3bb2482c8e6b6e9b9c09fc181067046fbfd", "derive": "0ce71bf1f1bb53dc0c19f775da02b69d0bdd2d355f3c0d1a2fafb45551cafbefa6a15b3a1376a8435565449861c0341d37bc2c7cfc88958fb52b2555d7b707cc"}, + {"seed": 89, "len": 2048, "xof": 131, "key": "c23e6b9cb89f41a0c7d7e7bf22165dec740bac9f63895152802e43d3ac0553ed", "ctx": "lambda-vm oracle review ctx 2048/131", "hash": "dd91ebca42a55f4899cd0e25a0ec102c087d25c1b5c230ac1b5e8a074448720dfdb41736dffff0f5717f6c8e9a0dd6eda746cdc7f7634d4a69bbc82fad49f76ca38f7d58d8aa46728fd78231e9efc5d5249d0a412b2cf9b275c564d24c2fde7d5d968bfd3ea35d981e32d8f17ac66c4f0f29bc4dcd8082f324f1b1d6883ea2bfd5f6b6", "keyed": "457a806e76d666cd198569275734101273ea0c936e6b9380cf85ead6eead97a5f31a936c0bc6951aae3ca46178efe99f4d6ed98b83efb28ec5ec9d660e6b1048d8550b3001e29dfad2772f381b891577aa92e7f7d597411f537b3712bd5db8f7d1c5005699982fdba5bacd5ad66762f7d2b7fe2ec9a6dbc15a715fbf6e0f026100cdf0", "derive": "7819e47d406aeb41c0316d7db3f9d539ae763e768701334e313e938986369bb4297a90be5fbca72cb126074431248f0d570291c5906f71a01a479e66244d9cbfeddf966b83b53f578db9b5deb2568e62616142bef9edc4157c4e9e7fc91d3ad87f6c7340ab7dc493e3deaf6c4beee6278e100de4f90cb16e20c6ee62d5a7541ef9a5cd"}, + {"seed": 90, "len": 2048, "xof": 200, "key": "e8e0fe22f74b3bc9d2796d01e9a5461359ba22761ec3ae62fd250247808873e8", "ctx": "lambda-vm oracle review ctx 2048/200", "hash": "d1acdc026a5436d968dad88e6f04542b8d0df5846ee2a6ac387c44e9295e16df61c66897b623dd45a8033372bc77320274be1b5c38511ee24886b7d53e088f4e15ed3e140f5f99a829c4bfb70b70bdca42565e98bd38e785319eb5a4ee0bf5931a4bd6b8ee0b6e59eec5d9ea8bd792e426a1a56ea56e5545a8e8b65cad9ccf1d9678fd64c38f33879e17d1312359b2a4d314e8ef6ab2f1587e2ceb161bc4552df062416a58474ad3056defce3f2f6e7e7a175d6ecb012bceb91385930f8a47fa557640ce4276bb61", "keyed": "da61c6d28ac5f5f719ab444f8013ad9c56888df92dad2e939fc092438a0e83b17ac0824bea6c015330c7c3f2589a33e58bbb2ac5e14c5477c2b2024e698c47f52ede48d55563dadc46e783c06c24067aad1d8417cf7e2d8fec59444ea700cb6ded3dfb8379c73f39325c8a0ced75d62e995711542170784afba801ac92a14d95161fc109f220b6c120d4797155c3678ad0bbe529ce587250728b26fb64061a06a2ee67f8d1e3492b6a07d338d327ed709c42608bf2781279821cb1b21749203f6f6c9d04fff3a4ae", "derive": "2ed11775a07d4c1b517d354e5c89151b5336dab0588125e8f85990502ef6d85ef3309272cb31f4acb3a7fcf4942de8edc6f4329dbab4a1200d0facca24518edfe06f706e76a95b012e50a33e3f5ae3cdf41af17985317f19cccb7f753ea9bc57c30e0a7f1d0f7fd5234b5fcd2ed8a8ee1d60fd6b07af2b5ebfad1b5aef91c5f6be13d5f33981c9ef7f60a4ade34b9523a0d75bfb3a365e645a849caa98c12836dc0cc89fc2c4c322260a76c6ec499d39bb8ce8449eb3746a875356d5c27ac428513da2f9ae19424a"}, + {"seed": 91, "len": 4096, "xof": 16, "key": "0e8cbffd842407148829d0b971714b98068d3bc8afe1d04b5ebf2742245f5d1c", "ctx": "lambda-vm oracle review ctx 4096/16", "hash": "215da79fcb78be79dab730b78467a89d", "keyed": "17e84551f53d55feccf53cd8752232ab", "derive": "744b75aed4882da612ecabbeed555a24"}, + {"seed": 92, "len": 4096, "xof": 32, "key": "37d59cfec72fe64019e0139c588a3b2ba063e011e4b870cdb1ffe366582349af", "ctx": "lambda-vm oracle review ctx 4096/32", "hash": "2b2db219466f8183cd837cffb20505126a523114d0e2a69ddb425f9501933912", "keyed": "bd68a9f1b0d69d19bd1d0737004df16d3ff1ac827f80aef3804bb59f2d11961c", "derive": "1a485b0f2b1a6dcffa47e71159adceb8eec678d2576ee68d1dbde82c9a00d590"}, + {"seed": 93, "len": 4096, "xof": 64, "key": "cb00aa01b802dfc2eaa2256eed82a0208816629b4e905a9b3090c69e68274d03", "ctx": "lambda-vm oracle review ctx 4096/64", "hash": "0492d1ff2de887b4febea60dfa8ab27f9d322708de806629c2378282b3686518606927b089be53a118bde1205242973aa32eb3c32708cea83aecb0cb5ae0d08e", "keyed": "3939705bf049fecda37a7b3adfdfc277139ec2243353bccdafa11d7a841f662dd49379862bd294a7f3a04567bd9f9a5ba2785e7eff94fc16e5886eaf2439f4ab", "derive": "c71aa99c235917053a09e55bfda0ae9d5214f5faa6d54498f7ee690f6c6a2af0dae3578f9aef9ad1b4fb7af11656f97e95955af8d697b0570e41f18ef5c92e46"}, + {"seed": 94, "len": 4096, "xof": 131, "key": "8223c60b3ec3110cce7438b53f7223e9baa8c3abf2b43bffb5ce0df1b1118ca2", "ctx": "lambda-vm oracle review ctx 4096/131", "hash": "ad5d14700821d7ee7842e21815bde1f0f2c4374b57349b259b3d3de06337db474de14e8bf3d3315a5cec5cffd2a31c8a0a54150feda2b4f7dab19057a1e5d4ba13ddf1ad855968bf33cd77de4a46571f3b0c7869e5ec88ed34cbab8b4c18e2e3f64f736ca9cc06f145138de1ad586c2fbba5fd22cfd5d31c4fcbf8c3d71ab120ec552e", "keyed": "47765f1c959389a8a342f9050feb7d202d51f6c76e718dd9913289e00cac33d09e53dd6a9486f24fbe927dfa921c9e4060e5ec0587ab0eea545c998de6292fadc74ca5874629c9d2e0a85bd2d44dd0967bd6da74103f9b7062bbd3c0153b53f0342db20740e0e5065197a6c56362152649fd106662a2a3136f4bec163472633aedce57", "derive": "a966caccc348466a42226f6ed900ef7031c22eb751422ebc32c904e44dcbf5f5e3dd676a6cc16bbdfd6a87bd91d46c867bd850a864d2eb39be3f946a90b0ece4527838452b18d041b86b5e3b726905845f664afb0fed035d51dfdcb8e907b9a75ba216acd7f2182e34e5b47f65fc1977796f5a2132f4b6bcbc691bc9e7fae805c28287"}, + {"seed": 95, "len": 4096, "xof": 200, "key": "174e6662d2c05955c063523f5288eadb34328c4fa2b27165858897cae98b048d", "ctx": "lambda-vm oracle review ctx 4096/200", "hash": "05420df742c2a977945cd2090e7faf6d969e63a2c6238cc0770e40ac9a86669fe7ff09b94c0b2ceb41ae5ff5321864c1247f5bd42040ab4f02c49330807b307e30191bc6ae242805ff2512d417c847b12fb9d85555052ec43db978585716de1b6246b170d82cb92ac1fdef0b34818b4c06739152d25009f284eda20e493736fa55c688c9e6d6407c648dff4e72caad3b7f51aac4aec4c53b831b785006f0a1338212600b2ef65c3af11fcd729409bca76308a41da0639b6e23737ce70c86426fcc1beeed41559d7a", "keyed": "d9997ec1c3f46309869da8818b3e5f634a48adf8a2227dc3e2eeed9f9bd4b1842e442e1d444a2047e711de7c97bfd1b368f73c0157843e35ee4d46f4662d5eb259765ca7c4d50e57a7b93cbc7e2f18a085b083315782253a60049b5a1e5aafaa7f425ff81afc5f306ecd7306db3060d16f7df63e5dd81ea43fbee15f8c6d2198c1a001a576538bf22e489d1d812604c46e2dd6cfc405bb5241ac80b3dd5c0ef9e778daa460c3ad084bec4889c6a0009d200b765e131074781a0b247cde5700fc04d46a6fc1b019b6", "derive": "ec2762b33af35e89aa4c07c733054536582a31fb630bdd6fd8c25aed6ba6b36f1e66f577cc2458342d87c56474f9abeb6c81d2666fa1e9342d01abc12e35f1175395247bd1e654d74797bbd934424200de4f735307cc79fb0d640111559771911f0adb0ee081880b6a72c3281f6c2827453ee54c4cdd64fd419237353a1da8e6a08046c26b833407830a767febcc65e500e90195e13601285b4109d54c0d94eb55f15f3846b2b92dcc2e9ddc6d7f0232e50c151db38fd8495882764456e15893237c77345d5d95db"}, + {"seed": 96, "len": 4097, "xof": 16, "key": "69567c30361e6b67e55967d288477bf78c8e0af84fcda6abc9258567e858c90b", "ctx": "lambda-vm oracle review ctx 4097/16", "hash": "867aca04bbac9c1d3a07278138e9ad1f", "keyed": "38e03b39b3e55beb53bdd77ce350c832", "derive": "4753c2c7254782c32be47b7a410f4572"}, + {"seed": 97, "len": 4097, "xof": 32, "key": "8f930a77102fa62bb15ffc8cce6ff1314dcc31eb20f2d17050b09fc6602f520d", "ctx": "lambda-vm oracle review ctx 4097/32", "hash": "ed871b6a5ee95e25f03e981e5bec7758ec00523f4986852510db5a4162961c85", "keyed": "9d5efe1b892cbe1ae7ab8a04f0f56fd4882d0477a34a655b48218fae1c83165a", "derive": "3548c2cee59bbc4fdc4266169ddf3464dfdded393b0066bbe7a33490dd1107ad"}, + {"seed": 98, "len": 4097, "xof": 64, "key": "927ff7293c0dd8b741106fbb7479028a1fce6b6fefa6f6266296e989358c56e5", "ctx": "lambda-vm oracle review ctx 4097/64", "hash": "b6e61602cee007c2cc998ef402a2a50d02cc1b988dfbe7362139f06226f533e8f9556a1ec8b0ae749c7761da36cb5a97663eae8a13b13c7fd4ba39140ea12e1f", "keyed": "38390cedeb4636357acffbb91d85265c1d32f872c55e5e9e341e8f581df4e23e8823fa7cb5e239c857368bdd2c21984398d3662393910e2612f5c2ab1fe61780", "derive": "0f4b6becfee37bc9612d188bc1d80b5ff3f81396ada402e0daa5dc899e9e55c6cf2dc5cc759e77e0ffaa7eb26ec5785b1d589b9ddb14fab4963002670452194f"}, + {"seed": 99, "len": 4097, "xof": 131, "key": "b8bc393f6cd619e9b5748cc48c90ca3bdc39bf0ab4d2d43c502a939bdfbfbd56", "ctx": "lambda-vm oracle review ctx 4097/131", "hash": "c5e5938ad2e8304bf0d5bda578ddc94dfedc903f9d6cc4401b8b443ebb8cb6cf21ae9e8c453a5ea7d1a6e50c04995bbeb9bcedf458d0581d9c0ed2845d1c9bd9163da1e26478341d619b4aa441211ebad0344f44ad621d211e9b6295c9d74a121707802ec788a8f453c090cb6da6e95be787afa0b1c3efb8a82a0badb816919bf7be19", "keyed": "def95aa4c1ff157d04ff55cbd23310031603969f5b45decd791bb0ff5064643b6a31a69a16de293af9a7fe719c60cc5e3bba6c8bf4ebb04705da1ef8caba76635f84dffca60164493bb5beaec75224ec7a4b0818d4ad88ba3b8efdb70b52d10e66418757457e667b9e3239b3a5dbb8c6dd823dc93e0f3d0b7ee68736e010ecb271bafe", "derive": "68ae58a3f9156bf80acbc356843f6eff735adb807ef7fca62f034d02e85111f26b043baac74a36dda7531c8d24406dde16827c0e0dec91ec969d980e09ee2582b8e1b4f693efee008e9ff2f28fe8cab0b821b1a4dab4fb24813c09dccc3d195e3af888f9bb6e3487b53b3607913e9381e83e9c66784a7e1586b46e83681a9f8d2903e3"}, + {"seed": 100, "len": 4097, "xof": 200, "key": "4926ffd129f4d4da38a0464c3cd5ae104e540c96a195dd522573610ae621345a", "ctx": "lambda-vm oracle review ctx 4097/200", "hash": "b4b19f35b3eab71e25b86a0d12f234756d79531e32ea6b9ad70c9f275f949353ac8e6c18fbe4124bc16a572fdd3c1f89243eb10c5e0e86b4d344148513c3f802a8544d87bcc9aefa6a318602cce20fcc486fe7f265769cd4154a7b978f6e1d76946927f67af6967aa4906f74d09047fbdf203379d6f9eadff3612ac884b6839852a79716bc4bce6f7853325202b0fcf23573f41af4f8de558f9e62eb5268dc8712f7786194118f81694cdd73bc4d5153ae6c32fab2f35d5caf5f9c8d6ac0ee4ad0458f56c132efa4", "keyed": "ffdb15a8f97a8e8f27d1e6fa9f83b862bce80f164f894aa56a330e5366db864787fe52d1ec92af5ff062bb1f12564e8c004be44e59ae8734b131d609da13d6b915b1fb4c954afda7ad5ab0bcc086f203c03e05232d2be549ef0c73243418f249e31d68c2a022bd6a140402c444514213e8060df9d96d50ab7e10bc40e2dcae2506d4fa910e3639102967b8671489589b156068a28089dc5d278deaf7045ab64ff84c237245fd98f8951a5b17fe7de46a48eca62f6862cf986fa808b4ab8a3a0fbe3c65d49423574a", "derive": "c9bccf719c967863b07ccaaab3c44c0644af36192e6f7091680ffc5d2a36b4847082068e42de15377d55b4dd06ce66e60294168989306f12183933cb1e86289aadf85d7e09e3b6b71264be605b1d7774fdc4741c3b569362d2e158088430c1985a8eb41dde271c4d53eac15b4268794c2bfa46032d2f598e0d02f897b9767379b988466ecfbb7eecd9b9621c73ddc78e7017f63189aa06441594adf632bfd439ec66f98ab49d2591e0fdc906e7006ff057f149430f756ee06ab323506921f9552a475bf61d5ae158"}, + {"seed": 101, "len": 10000, "xof": 16, "key": "debf6bda8ae84c02678eb007feec1ae5037fe2fdd886c749bbd8ba07fa751ae1", "ctx": "lambda-vm oracle review ctx 10000/16", "hash": "298af4e1235a1a2cc4b1f7e6371e4296", "keyed": "985164b642278a61f7885477fcdd7d4a", "derive": "9edd4f3905cba501cf38174df53cfa57"}, + {"seed": 102, "len": 10000, "xof": 32, "key": "724ee3cb2af83fd3c7fe1479201facaef25de4d47ab374b04d51dc2a7dd2414c", "ctx": "lambda-vm oracle review ctx 10000/32", "hash": "8797cc68e0e8f52c95e0e1c805d3ed9b66ccee9b629f9c0ad014faaebd96877b", "keyed": "08bfaeb67847f9d8da30caf3daad2c4dd763013166dd954e33e001c5fcb38543", "derive": "195f9d3c9d30f5dec592cc0a81ed4d71c5bfe2b647ba8b2dbe48877e6b8ecc56"}, + {"seed": 103, "len": 10000, "xof": 64, "key": "07e8bda29e5416f49e9edd67d44e6a6eb3604796baa70a54c03d6db1b53a3246", "ctx": "lambda-vm oracle review ctx 10000/64", "hash": "691f34c88445969efaa89be227fd360393e3b9fd66075583c7214d3a22685894b5d5779035a9b3114de3e8c88c48d12f28165cf3b921f933e9758116f46b2cc6", "keyed": "8736e42e82ac72ad5296420d9b8ebdc1f7f16be14ee45f0f98c8bd608ad9dd878ed5a0794997ddf234b57a44eb56233bae99087f5de09bcab7b7256865fe77c6", "derive": "186867f6ee0a0a30cf90e931c687519c5c6015db1855bec7f687c0d14408f3f2f6170bb892a8104814730b5616eae4185abeb2184e4805c84304ff633871da89"}, + {"seed": 104, "len": 10000, "xof": 131, "key": "7cd1fd085fe8d82385a78adc0c2fb6967c37bceaf58c1ca36f75ccf94597297d", "ctx": "lambda-vm oracle review ctx 10000/131", "hash": "0a31dc0885cb9f01f608c472c927642bfc6da83faaeb3cbd08df693fa970e32fe63752f15ad7b44acc7aa84184f57f230256307e0e19d41bd7045b6e0acd2e0f1030d59fce33e6ec690eb1d61123b629f661af35f5b5289599a8c610e91954dfcfe2e10c51e989d2afac0f137db9cd9ed349bc9c268ecdb4de985890a780f9b4583491", "keyed": "789727759c1fa74d67fb10747ea2d6b63f880fa2c65384c7001edaa7a036c6a20e517f8722f41cba69eae3250ba124c6d5223382c7d7e715d12896091d4e06836afa08fb946a6e4095caed92fa1b32bf184170e17bd993664b243e25b4bad23b1fd502903ef89df669f42a33d3b8ecc0ae6e34c1e36d0fc30a205fca676b546bacdc8b", "derive": "6d23542e2401bdde33d6cfb6e1d5e711012e509a4d13075fc42b0f1f263c81807285351fe8fce9e755d45ebdaf1abec13e3031102472b775fde583672313104fbda0636b15485f925ce8a2a2bae14c4d7d9e4a595c1733c5e5017d9a54b67fa8238e43079b9ad609c0cc27c0bedf6bad841f8bf9f37cec048d6f99244df1e0c1e8bec2"}, + {"seed": 105, "len": 10000, "xof": 200, "key": "a20d1c6d3a8c8000132928d6d8de2d7a01b4352deb394230b46fe47a9d6d2319", "ctx": "lambda-vm oracle review ctx 10000/200", "hash": "de90c25b7c0cee636412502de2183d0f0ddac7238f059d540335550a92507c51181c3f525fe2072f17208a06e035a6b090c67184f792915181ff35b6986008ec6868104328fcc6f451da8478998ac6e310e4ce86f1a4356f839da5d21c44753a15486730c13028dd542d70ccbd220e32b4958e93845b6886937e7e1d2d720c3cb748876318bc9aecfcd7b7c2d66af44abe5757db8f0bc8eb697c1b6b4844d95569e295128700cfe2567def2ea7b13bcb53c315d020d9d9cf4abd77e4e242c16533fca443eb3aeeda", "keyed": "d5322c8131d3f7490f9b61b92ace6f127e372344af6acd440109be79ba3f3bbc0c9e7c68d251f1f6ba23b93d4f3c1e2bab8fbe4a6fabaabe17a27f12424117db0f4d24ca91ced9744601c9eea0083f0d0fa234af5ddd38bd316971f12570654c6871fb53edff0b2d2cc268a5898782b4e22c392d4b933e4f9120667caecb8c5f78bbc15ab143a97d13841dfebc3b7097569180a12b6d02350892625fab5bb723aa170c42da7f6213c005e76e82354ebcc2aa4074020000bb14a94035835748675a9ad1dac4d21abc", "derive": "49a119c6297ac0cfd9282cec209071d6b269b77b20cf1f32d700bb29d0174e65bd8149052f31efd0a6b1f9c902a6e8994ab821a84454877c412a7c8830219c8dd6dc0143accbe984687600e7e3b73f8c18929e98f434e8d151bf8ef849f3c48605d840a29065391ad342dcbd267e046d3558254f443eb539bf5878eb71721fed08922869cc7ab9d21218ca393dc300e970d0f7c0dcb15953639639c1ba29ffd6b411a297dd3288476d11c832cd326636d67ccacf258b48658877dc874594ee33163abf82196c02ce"}, + {"seed": 106, "len": 65536, "xof": 16, "key": "c7fcd83f19782a915cf9d246aec8c8ba5356c0236f5aab21745ca8706cd378e5", "ctx": "lambda-vm oracle review ctx 65536/16", "hash": "250e73c83ef8c2547bc66cbb62db7cf9", "keyed": "68a9254c8bca25432f39914236af4697", "derive": "c2610f8c15910689cc8b9071b3e618b6"}, + {"seed": 107, "len": 65536, "xof": 32, "key": "ed3888f753dca5e6740f4e89c20c13885e463030c76298c6c0cad9343166d9ea", "ctx": "lambda-vm oracle review ctx 65536/32", "hash": "d987ec830797a4b3c39f7b2da883106acde4dca8088a0c61c9ba75efb3db9f88", "keyed": "34b1c21894a95a245a74e0681ed5b38774e0b282adc477bdd1c58f3f5898f75b", "derive": "95b688c7d8a773cd6ed197bdfe9c3ddfcb3983c4b8a0aabaa36e1ffa6f21ad95"}, + {"seed": 108, "len": 65536, "xof": 64, "key": "5ca03baa8317d67cc39f60d104b35d1aef8869c74baccea4341bd825d246e351", "ctx": "lambda-vm oracle review ctx 65536/64", "hash": "32b03bba7802ca730ca05939ff1e1ffdec0a09af285130a4cdc148ef81d78eb98baa097a8ffad1e4b876f0bb59a18c7e2b45c8cb3813898e2403984f7a6c6071", "keyed": "3b0a32ca1166401b8d712f829dc16688095153d108912de3c8414e0a6750d6d0497d72fe25247c8d7e0e6e9c9023880cf0752310a7584748dc31891bfd9ef33c", "derive": "82899fe0ac33c8eaabc4b08dc0094cb450d9ccf0aebdda852eaab18adc4ad343d8d638e331496ef18fc23dfa48e3ce3cf77b3ad76403353d244b48e617b44da6"}, + {"seed": 109, "len": 65536, "xof": 131, "key": "f03938cf2f49ac7e99fdc4a957b6b3e99cc5288c4feacc5837526a08f35f5ad4", "ctx": "lambda-vm oracle review ctx 65536/131", "hash": "5ec02bd02ee4d382db8d10d0c4a2a014087558869944c60b97531eee05c2a5ee5fb8664fbf041bb64f09c5e997bd8a6db7dc18ecdf22285fe7a1ad368df8b0683e37c328cece89b0e90992bafc70f7987bb064338dc4cb025cf366aaee42c7e8fbcb9fe28a816b97c1766c7184bbdea668fe11db1f6aab31f189b28f9f4014a24b17b6", "keyed": "21c7d240682985db03161379efff7c36d04594c5a5b166b345a54a35f6fbb6c0c60a202e34a71ac97fb7f9888c86dfe66ef0781888245c332a45c7ff494385f6e7c950a1a561ab98dbb352ce4f9ea6fabb1684ac152ef00cb7d03f6811f7faf424e0dd4cb24de541a2a4762be4d2f76c22f752d18942a7c8961d0c99221da20a0b81a9", "derive": "f1c0de3ac81f9e3907a67c39b1a09949da0ff0561cfd84c6878cc7b969ee7cb89949bd0a4d217c7780f038cf7659a2e8e9f8472d9f414a3870e5adfecd48ecfe0187225bd5acffd898b25cbf7faeaa3afce321b2add22e62459e907f99843fd689ecc0e448c02b4a389db38c8fc52ca0f978e727077ec4dc58e5a6521ef07ad59a3c5a"}, + {"seed": 110, "len": 65536, "xof": 200, "key": "a7cbf3e166bbab1632153b7b7a9053d6b42f4d853b55e21867323ad5b55fcb0f", "ctx": "lambda-vm oracle review ctx 65536/200", "hash": "9d3608bf4225fd301cb08dbb27d22bdcd902d833ee5e8896e535d799c2ec10129c8aab47a6602262ad17b3589b66ee97047f29ab1606de811d00e3faa6bbfefc54afef9dbd61ca79675d3d583a819fae4e8adc0307da8e5e3202e2b25cc78af20e13d73a16656edb5ade79f77163d748813776916247f627452123b2aa46910a7fb7706460e8f72aa5bcb6f2685819960e6c85ce66ee560daf963c3afbe4a21e8fcf5750fcac93f62f13bd273d02082aa462f03497e97d0fca5a1b39c8a6fff4bd83aed05525c5e1", "keyed": "9b312955f025b12c33f0fb6d1a7724847b6bc5416bc8a7545b5b1c40e95047c479157113b57c0a4cc2d1acc5f16643f22285499f06f211c4ba208d9c2969e3a2bf21ccb3b1808ce309bb887eea150047a0c4e7ea981a57f7fbffbcf596ab616865fb3bdf428e77d1ec60e52a1f7f2f4796d13edfcc793ff7a5973793537793a902f315058d0c3f0a67968ff812657d46585c063165ff311c7ae19f218dde369cd5e89885ba08d2f4d5bb96c00ebc3e33dbda71fa8a873cbdd2c26b102aa49248fa4177f958fe5ce1", "derive": "8772e113081fe301b461adf0a1fbf105a199eee3d0c0f27836662a6ed7c62c01037ff56c18b298028babcb45aa1c7f13335f9fb8dc2f5931021be53ef21da4727b20927d3860eb17b3dd7ee51aeaae18aea989de6b897baa14547cb3e6c1bbec9bc63e66a078cac93815c4646e5690fb7ebbeabe010e44c9dd59eb2b5e6399b24c66f9d859390e170e94e45fe942d230788b9abce7392a3efe7d9652ca5e94cc63996cb47260dd9111864415368f206513609d744a807e7a2fd044b627d1ecc9b317bee514aaa7f4"}, + {"seed": 111, "len": 100000, "xof": 16, "key": "3b643c5a1b5e239a0799eaf8ff13931b82edb8fea42a47caa825b0244d4b7b00", "ctx": "lambda-vm oracle review ctx 100000/16", "hash": "dafc038f963a4fbb56a970d51433e9c7", "keyed": "ff1fe75b79c283887239712536a6f8b5", "derive": "4ca500d55e5ceec2fe5c02021403c4b5"}, + {"seed": 112, "len": 100000, "xof": 32, "key": "d394a71b890df7dfee174b95ed22722c143195fef201d138f2a8df894d16f144", "ctx": "lambda-vm oracle review ctx 100000/32", "hash": "58d738cba5b0b79e4d6f2035fb41acf271f1fed88b8e432a51958f43e827a212", "keyed": "6dd9792a242527b2d3013515dc5a7c75e36ddb6df6a6d41b373533f753c09454", "derive": "82f5c674b0ae22e41dd9424c64f66d4ff4ca3bc67ab20f0fcee1a97dfb6901f8"}, + {"seed": 113, "len": 100000, "xof": 64, "key": "f93f8bd6a30919f2178dd9860e65cc1da2d5a23e6dfda14cb14a2aa8e181732e", "ctx": "lambda-vm oracle review ctx 100000/64", "hash": "bf0c93ac003d8e92e511fe59d3309141fc1bc993c38d9a63737e4f508d68040f9422080ad44bb74dcb578d37bafb82ab021f758e98d347fe407576bce315f86d", "keyed": "90f3212d27f2a1919d209a19754424ff0b2704d7480d84ed882cb5fad926e7229d8d92a2b8daf1bc7b17e0333098265e7caf0eae401e0cbfcb898421a9245553", "derive": "7829eeda7bd48bb51b4b28147245a7bafb2df37e018029598098c99797cb174d2526858c341bf54a8d90eae6240572e0e590653c6486c5413bcbbdd390af9a25"}, + {"seed": 114, "len": 100000, "xof": 131, "key": "fcbd4502b4c333ae50c954a138f4b41913588e1d5e966ab5580f9d3c98d5bf28", "ctx": "lambda-vm oracle review ctx 100000/131", "hash": "4d1c5f62bf411a2b6f6711f4d9913c0669cc472b231099c41c7f9a45343ec421ac727a010b1e495a4311defea2baa88141a9dd60192a4d97a06f4818a527b4b65c322b4554b4a0c29fdeacd4fec1ba1afa2e0abec8b577c84f9417c376b2472272ac055878b0d42e5f9c4dd4939f27a4b46c0ef080c2e2abefe3fad3abb728c067cde5", "keyed": "bfa9973a8c02b8882e1450dd434a4b9d7072c877d8a2f94346fe06dbd5abd96d8fee4ed82aac7f3fec8addd2d7b197f030ea5322dc7d4217815a0a1f73987818ea935d09fa8f1513a8c18d87f9ba5f0f6a7fb57ab90c01c6aa8fa5aa59b304ae1240d0bcdd3e670786033ca4137ab6968a3491e2ebcddd638c633bbce0488b54eea92d", "derive": "a64ef45282602fb582847effffdddf218568cf916d38bab6cf711e8ec7569cb54d1ae624928ef8476942c2958c9d021e72ac012bd5c28cbbbfacf2239d968e783b310f23f6301eed08402d08078bda9da21a46c41c9728053e97b6d964f55aa3c8cfbb8698d6ef94dd2507ead85e454947d8bbea81159162c16c25134c0c92b519c0a7"}, + {"seed": 115, "len": 100000, "xof": 200, "key": "2268978ce6bb8db3b5133d660df23099096d6778ee1946a59d217165c34429a4", "ctx": "lambda-vm oracle review ctx 100000/200", "hash": "9d0a9882067f0f7e8adab6330d9e62498da7623daa31f39e984189b42830d444a4c7ad037fbe55db999626c6ccafb1737145a2ce28c63fa0380322bed696938395f8e18cf7c1ec1d032a81f044e4bd58a2f8b930b9f91fab466dcdbb97825ce61b696c1d9af7653380ca80c4ad3224338b2bb42d420c7d44d2d42e2beb9bd6e5d3f714df6dcd651c2ca655601c3ff2d50020df05b9f8c60617633f6f0648d4f57d1b98b9ff861dec6e0e3ee494729489d7b81f587fd0192fb7d93e25e28ca359967f6ce919f1ffa3", "keyed": "e6c5d4fb218f27ac862338bc0a6a272336d53ef38d415584fefe7989807ee222a683a3222031a9839dd5f02cd61687e9c6bd68792308dd86e43b3daf159defc21ca8f7663d4d8ce2c3c80539f9b9eae2656a4b299ca44d94381be56720599b799fef8e674e83f8114ac14f180deb55bdc0e3527fc2b69492540aa28dcf84f56ba9dcc89225122a0f7918b8892ee3afe40b24335ddb84a2f92796819602d8c88085cb77f45228cd7ed23d5b323c1bc7601fa85ceb1f9d57e541b834633ea7f5af70d52fcb3cc1fe4c", "derive": "1e6b77b3cfd3bb00000c04a2575b772c18720102877b8cc95379163219e67e03f3881cb7e78b893a11f19254cacf00302d68f8032999d7352f132341bb3a58eb002dfbc27dbd09694c82a8f34d006d1479508184954d36f58c821cfb7ae6965f4cce34037777213a433705a76390f33f6e5c80c4276dbdc1514a7e26437c5f490d5ec20b429016bb79d6eedf9b244694f6bd35cdc214fc27662b3e34bcc5ddee99c89fd4e4c443ad5f52958f64a4bc3514a4fb3b6bb4ec3e738836b2e03f0f096b98fdd9f91d1a43"} + ], + "known": {"empty": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", "abc": "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85"} +} diff --git a/tooling/blake3-second-source/reference-impl/.gitignore b/tooling/blake3-second-source/reference-impl/.gitignore new file mode 100644 index 000000000..28eddb5da --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/.gitignore @@ -0,0 +1,3 @@ +b3ref6 +b3ref7 +__pycache__/ diff --git a/tooling/blake3-second-source/reference-impl/PARAMETERISATION.diff b/tooling/blake3-second-source/reference-impl/PARAMETERISATION.diff new file mode 100644 index 000000000..b83cdb960 --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/PARAMETERISATION.diff @@ -0,0 +1,48 @@ +--- upstream/blake3_portable.c 2026-08-10 17:15:50 ++++ blake3_portable_paramrounds.c 2026-08-10 17:16:06 +@@ -1,6 +1,17 @@ + #include "blake3_impl.h" + #include + ++/* ---- LAMBDA VM PARAMETERISATION: round-count knob (added) ------------- ++ * 7 = standard BLAKE3 (bit-compatible with published vectors). ++ * 6 = the internal variant this repo's LFM chip prices (assumption A6R). ++ * --------------------------------------------------------------------- */ ++#ifndef BLAKE3_ROUNDS_PARAM ++#define BLAKE3_ROUNDS_PARAM 7 ++#endif ++#if BLAKE3_ROUNDS_PARAM > 7 || BLAKE3_ROUNDS_PARAM < 1 ++#error "BLAKE3_ROUNDS_PARAM must be in 1..7 (MSG_SCHEDULE has 7 rows)" ++#endif ++ + INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); + } +@@ -72,13 +83,20 @@ + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + +- round_fn(state, &block_words[0], 0); +- round_fn(state, &block_words[0], 1); +- round_fn(state, &block_words[0], 2); +- round_fn(state, &block_words[0], 3); +- round_fn(state, &block_words[0], 4); +- round_fn(state, &block_words[0], 5); +- round_fn(state, &block_words[0], 6); ++ /* ---- LAMBDA VM PARAMETERISATION (the ONLY edit to this file) ---------- ++ * Upstream unrolls exactly seven calls here: ++ * round_fn(state, &block_words[0], 0); ++ * ... (rounds 1..5) ... ++ * round_fn(state, &block_words[0], 6); ++ * They are replaced by a loop whose bound is BLAKE3_ROUNDS_PARAM, which ++ * defaults to 7. At the default the loop executes the identical seven calls ++ * in the identical order, so the parameterisation is inert by inspection -- ++ * and that is re-checked empirically against the official test vectors. ++ * MSG_SCHEDULE has exactly 7 rows, so the bound may not exceed 7. ++ * -------------------------------------------------------------------- */ ++ for (size_t r = 0; r < BLAKE3_ROUNDS_PARAM; r++) { ++ round_fn(state, &block_words[0], r); ++ } + } + + void blake3_compress_in_place_portable(uint32_t cv[8], diff --git a/tooling/blake3-second-source/reference-impl/blake3_portable_paramrounds.c b/tooling/blake3-second-source/reference-impl/blake3_portable_paramrounds.c new file mode 100644 index 000000000..32cff4421 --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/blake3_portable_paramrounds.c @@ -0,0 +1,178 @@ +#include "blake3_impl.h" +#include + +/* ---- LAMBDA VM PARAMETERISATION: round-count knob (added) ------------- + * 7 = standard BLAKE3 (bit-compatible with published vectors). + * 6 = the internal variant this repo's LFM chip prices (assumption A6R). + * --------------------------------------------------------------------- */ +#ifndef BLAKE3_ROUNDS_PARAM +#define BLAKE3_ROUNDS_PARAM 7 +#endif +#if BLAKE3_ROUNDS_PARAM > 7 || BLAKE3_ROUNDS_PARAM < 1 +#error "BLAKE3_ROUNDS_PARAM must be in 1..7 (MSG_SCHEDULE has 7 rows)" +#endif + +INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); +} + +INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, + uint32_t x, uint32_t y) { + state[a] = state[a] + state[b] + x; + state[d] = rotr32(state[d] ^ state[a], 16); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 12); + state[a] = state[a] + state[b] + y; + state[d] = rotr32(state[d] ^ state[a], 8); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 7); +} + +INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { + // Select the message schedule based on the round. + const uint8_t *schedule = MSG_SCHEDULE[round]; + + // Mix the columns. + g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); + g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); + g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); + g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); + + // Mix the rows. + g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); + g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); + g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); + g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); +} + +INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + uint32_t block_words[16]; + block_words[0] = load32(block + 4 * 0); + block_words[1] = load32(block + 4 * 1); + block_words[2] = load32(block + 4 * 2); + block_words[3] = load32(block + 4 * 3); + block_words[4] = load32(block + 4 * 4); + block_words[5] = load32(block + 4 * 5); + block_words[6] = load32(block + 4 * 6); + block_words[7] = load32(block + 4 * 7); + block_words[8] = load32(block + 4 * 8); + block_words[9] = load32(block + 4 * 9); + block_words[10] = load32(block + 4 * 10); + block_words[11] = load32(block + 4 * 11); + block_words[12] = load32(block + 4 * 12); + block_words[13] = load32(block + 4 * 13); + block_words[14] = load32(block + 4 * 14); + block_words[15] = load32(block + 4 * 15); + + state[0] = cv[0]; + state[1] = cv[1]; + state[2] = cv[2]; + state[3] = cv[3]; + state[4] = cv[4]; + state[5] = cv[5]; + state[6] = cv[6]; + state[7] = cv[7]; + state[8] = IV[0]; + state[9] = IV[1]; + state[10] = IV[2]; + state[11] = IV[3]; + state[12] = counter_low(counter); + state[13] = counter_high(counter); + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + + /* ---- LAMBDA VM PARAMETERISATION (the ONLY edit to this file) ---------- + * Upstream unrolls exactly seven calls here: + * round_fn(state, &block_words[0], 0); + * ... (rounds 1..5) ... + * round_fn(state, &block_words[0], 6); + * They are replaced by a loop whose bound is BLAKE3_ROUNDS_PARAM, which + * defaults to 7. At the default the loop executes the identical seven calls + * in the identical order, so the parameterisation is inert by inspection -- + * and that is re-checked empirically against the official test vectors. + * MSG_SCHEDULE has exactly 7 rows, so the bound may not exceed 7. + * -------------------------------------------------------------------- */ + for (size_t r = 0; r < BLAKE3_ROUNDS_PARAM; r++) { + round_fn(state, &block_words[0], r); + } +} + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + cv[0] = state[0] ^ state[8]; + cv[1] = state[1] ^ state[9]; + cv[2] = state[2] ^ state[10]; + cv[3] = state[3] ^ state[11]; + cv[4] = state[4] ^ state[12]; + cv[5] = state[5] ^ state[13]; + cv[6] = state[6] ^ state[14]; + cv[7] = state[7] ^ state[15]; +} + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + + store32(&out[0 * 4], state[0] ^ state[8]); + store32(&out[1 * 4], state[1] ^ state[9]); + store32(&out[2 * 4], state[2] ^ state[10]); + store32(&out[3 * 4], state[3] ^ state[11]); + store32(&out[4 * 4], state[4] ^ state[12]); + store32(&out[5 * 4], state[5] ^ state[13]); + store32(&out[6 * 4], state[6] ^ state[14]); + store32(&out[7 * 4], state[7] ^ state[15]); + store32(&out[8 * 4], state[8] ^ cv[0]); + store32(&out[9 * 4], state[9] ^ cv[1]); + store32(&out[10 * 4], state[10] ^ cv[2]); + store32(&out[11 * 4], state[11] ^ cv[3]); + store32(&out[12 * 4], state[12] ^ cv[4]); + store32(&out[13 * 4], state[13] ^ cv[5]); + store32(&out[14 * 4], state[14] ^ cv[6]); + store32(&out[15 * 4], state[15] ^ cv[7]); +} + +INLINE void hash_one_portable(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + store_cv_words(out, cv); +} + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs > 0) { + hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/tooling/blake3-second-source/reference-impl/build.sh b/tooling/blake3-second-source/reference-impl/build.sh new file mode 100755 index 000000000..d03229fb9 --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/build.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Build the two round-parameterised reference binaries. +# +# Only `blake3_portable_paramrounds.c` differs from upstream (see +# PARAMETERISATION.diff); blake3.c / blake3_dispatch.c / blake3_impl.h / +# blake3.h under upstream/ are verbatim BLAKE3 1.8.5. +# +# NEON is disabled and no x86 SIMD is available, so the dispatcher resolves +# every compression to the portable path -- i.e. to the parameterised file. +# That is what makes the round knob apply to the WHOLE tree hasher and not +# only to a directly-called compress. +# +# This is a ~1 second single-file C compile. It is not a cargo build. +set -e +cd "$(dirname "$0")" + +SRC="driver.c blake3_portable_paramrounds.c upstream/blake3.c upstream/blake3_dispatch.c" +COMMON="-O2 -Wall -Iupstream -DBLAKE3_USE_NEON=0 -DBLAKE3_NO_SSE2 -DBLAKE3_NO_SSE41 -DBLAKE3_NO_AVX2 -DBLAKE3_NO_AVX512" + +cc $COMMON -DBLAKE3_ROUNDS_PARAM=7 -o b3ref7 $SRC +cc $COMMON -DBLAKE3_ROUNDS_PARAM=6 -o b3ref6 $SRC + +echo "built: b3ref7 (standard BLAKE3) and b3ref6 (internal variant)" diff --git a/tooling/blake3-second-source/reference-impl/check.py b/tooling/blake3-second-source/reference-impl/check.py new file mode 100644 index 000000000..903c864da --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/check.py @@ -0,0 +1,296 @@ +""" +SECOND-SOURCE validation of the 6-round BLAKE3 vectors. + +Source 1 is `tooling/blake3-second-source/blake3-oracle/blake3_ref.py` -- an in-repo Python +oracle written from the spec, anchored on the official vectors at 7 rounds. +Source 2 is upstream BLAKE3's own portable C implementation (crate `blake3` +1.8.5, `c/blake3_portable.c`) with its round loop parameterised; see +PARAMETERISATION.diff for the entire edit. + +The two sources are independent in the ways that matter: + - different authors (the BLAKE3 team vs this repo) and different languages; + - different message-schedule CONSTRUCTION: the C indexes a precomputed + MSG_SCHEDULE[7][16] table, the Python/Rust iteratively apply a single + permutation between rounds. A bug in the iterative composition -- exactly + the kind of thing a single source cannot catch -- shows up here; + - the C drives the FULL tree hasher through the parameterised compression, + so its 7-round run is a direct external anchor rather than a borrowed one. + +Checks, in order: + [A] parameterised C at rounds=7 reproduces official_test_vectors.json + (35 cases x 3 modes) -- the parameterisation is inert. + [B] rounds=6 actually changes the function (negative control). + [C] MSG_SCHEDULE[r] == permute^r(identity) -- the two schedule + constructions denote the same thing. + [D] C at rounds=6 reproduces all ten CANONICAL_VECTORS byte for byte, + compared against BOTH canonical_6round_vectors.json AND the Rust + constants in prover/src/lfm/blake3.rs. + [E] randomised differential, C vs Python oracle, at rounds 7 and 6. + +Run: python3 check.py (after ./build.sh) +""" + +import json +import os +import random +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ORACLE_DIR = os.path.join(HERE, "..", "blake3-oracle") +REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..")) +# Where the canonical vectors live. This was `prover/src/lfm/blake3.rs` until +# P-a Stage 1 sank the primitive into `crypto`, so that the CUDA kernels, the +# commitment backends and the LFM chip could all be checked against ONE +# definition; `prover::lfm::blake3` is now a re-export and no longer holds the +# table. Checks [D] and [E] were silently dead between that move and 2026-08-15, +# because `parse_rust_vectors` failed with a bare `ValueError` from `str.index` +# rather than saying what had happened — see `read_rust_primitive`. +RUST_PRIMITIVE = os.path.join( + REPO_ROOT, "crypto", "crypto", "src", "hash", "blake3", "vectors.rs" +) + + +def read_rust_primitive(): + """The vectors file, or a diagnosis of where it went. + + A harness that dies on `ValueError: substring not found` when the code it + validates is refactored is a harness that gets deleted instead of fixed. If + this ever fires again, grep for `CANONICAL_VECTORS` and update the path + above — the parser itself is layout-independent and needs no change. + """ + if not os.path.exists(RUST_PRIMITIVE): + sys.exit( + f"check.py: {RUST_PRIMITIVE} does not exist.\n" + "The canonical vectors have moved again. Find them with\n" + " grep -rn 'pub const CANONICAL_VECTORS' --include='*.rs' .\n" + "and update RUST_PRIMITIVE at the top of this file." + ) + src = open(RUST_PRIMITIVE).read() + if "pub const CANONICAL_VECTORS" not in src: + sys.exit( + f"check.py: {RUST_PRIMITIVE} exists but no longer defines " + "CANONICAL_VECTORS.\nFind them with\n" + " grep -rn 'pub const CANONICAL_VECTORS' --include='*.rs' .\n" + "and update RUST_PRIMITIVE at the top of this file." + ) + return src + +sys.path.insert(0, ORACLE_DIR) +import blake3_ref as ref # noqa: E402 + +B3REF7 = os.path.join(HERE, "b3ref7") +B3REF6 = os.path.join(HERE, "b3ref6") + +FAILURES = [] + + +def check(name, cond, detail=""): + if cond: + print(f" PASS {name}") + else: + print(f" FAIL {name} {detail}") + FAILURES.append(name) + + +def pattern_input(n): + return bytes(i % 251 for i in range(n)) + + +def run(binary, *args, stdin=None): + r = subprocess.run([binary, *[str(a) for a in args]], input=stdin, + capture_output=True, text=True, check=True) + return r.stdout + + +# --------------------------------------------------------------------------- +# [A] the parameterisation is inert at rounds = 7 +# --------------------------------------------------------------------------- + +def check_official_vectors(): + data = json.load(open(os.path.join(ORACLE_DIR, "official_test_vectors.json"))) + key_hex = data["key"].encode("utf-8").hex() + context = data["context_string"] + cases = data["cases"] + + bad = [] + for c in cases: + n = c["input_len"] + out_len = len(c["hash"]) // 2 + got = run(B3REF7, "hash", n, out_len).strip() + if got != c["hash"]: + bad.append(("hash", n)) + got = run(B3REF7, "keyed", key_hex, n, out_len).strip() + if got != c["keyed_hash"]: + bad.append(("keyed", n)) + got = run(B3REF7, "derive", context, n, out_len).strip() + if got != c["derive_key"]: + bad.append(("derive_key", n)) + check(f"[A] parameterised C @ rounds=7 vs official vectors " + f"({len(cases)} cases x 3 modes)", not bad, str(bad[:5])) + + +# --------------------------------------------------------------------------- +# [B] rounds = 6 is genuinely a different function +# --------------------------------------------------------------------------- + +def check_six_differs(): + diffs = 0 + total = 0 + for n in (0, 1, 63, 64, 65, 1024, 1025, 4096): + total += 1 + if run(B3REF6, "hash", n, 32).strip() != run(B3REF7, "hash", n, 32).strip(): + diffs += 1 + check(f"[B] rounds=6 differs from rounds=7 on all {total} probe lengths", + diffs == total, f"only {diffs}/{total} differed") + + +# --------------------------------------------------------------------------- +# [C] the two message-schedule constructions denote the same thing +# --------------------------------------------------------------------------- + +def check_schedule_equivalence(): + """C uses a precomputed MSG_SCHEDULE table; Python composes one permutation + repeatedly. Confirm row r equals permute applied r times to the identity.""" + text = open(os.path.join(HERE, "upstream", "blake3_impl.h")).read() + blob = re.search(r"MSG_SCHEDULE\[7\]\[16\]\s*=\s*\{(.*?)\n\};", text, re.S).group(1) + rows = [[int(x) for x in re.findall(r"\d+", row)] + for row in blob.strip().split("\n") if "{" in row] + assert len(rows) == 7 and all(len(r) == 16 for r in rows), rows + + cur = list(range(16)) + ok = True + for r in range(7): + if rows[r] != cur: + ok = False + print(f" row {r}: table={rows[r]} composed={cur}") + cur = ref.permute(cur) + check("[C] MSG_SCHEDULE[r] == permute^r(identity) for r in 0..7", ok) + + check("[C] MSG_SCHEDULE[1] == the repo's BLAKE3_MSG_PERMUTATION", + rows[1] == ref.MSG_PERMUTATION, + f"{rows[1]} vs {ref.MSG_PERMUTATION}") + + +# --------------------------------------------------------------------------- +# [D] the ten canonical 6-round vectors, from the C, vs JSON and vs Rust +# --------------------------------------------------------------------------- + +def parse_rust_vectors(): + src = read_rust_primitive() + start = src.index("pub const CANONICAL_VECTORS") + blob = src[start:src.index("\n];", start)] + out = [] + for part in blob.split("Vector {")[1:]: + v = {} + for field in ("h", "m", "out"): + body = re.search(field + r":\s*\[(.*?)\]", part, re.S).group(1) + v[field] = [int(x, 0) for x in re.findall(r"0x[0-9A-Fa-f]+", body)] + v["t"] = int(re.search(r"\bt:\s*(0x[0-9A-Fa-f]+|\d+)", part).group(1), 0) + v["block_len"] = int(re.search(r"block_len:\s*(0x[0-9A-Fa-f]+|\d+)", part).group(1), 0) + v["flags"] = int(re.search(r"flags:\s*(0x[0-9A-Fa-f]+|\d+)", part).group(1), 0) + out.append(v) + return out + + +def encode_record(v): + words = [f"{w:08x}" for w in v["h"]] + [f"{w:08x}" for w in v["m"]] + return " ".join(words) + f" {v['t']:016x} {v['block_len']} {v['flags']}\n" + + +def check_canonical_vectors(): + js = json.load(open(os.path.join(ORACLE_DIR, "canonical_6round_vectors.json"))) + rust = parse_rust_vectors() + check("[D] Rust CANONICAL_VECTORS count == JSON count == 10", + len(rust) == len(js) == 10, f"{len(rust)} / {len(js)}") + + # The C driver takes block_len and flags as uint8_t; confirm lossless. + check("[D] all vector block_len/flags fit in u8 (driver is lossless here)", + all(v["block_len"] < 256 and v["flags"] < 256 for v in js)) + + # Inputs come from the JSON; OUTPUTS come from the C. + stdin = "".join(encode_record(v) for v in js) + lines = run(B3REF6, "compress", stdin=stdin).strip().split("\n") + check("[D] C emitted one output per vector", len(lines) == 10, str(len(lines))) + + c_out = [[int(line[8 * i:8 * i + 8], 16) for i in range(16)] for line in lines] + + bad_json = [i for i in range(10) if c_out[i] != js[i]["out"]] + check("[D] C @ rounds=6 == canonical_6round_vectors.json (all 10, 16 words)", + not bad_json, f"vectors {bad_json}") + + bad_rust = [i for i in range(10) if c_out[i] != rust[i]["out"]] + check("[D] C @ rounds=6 == Rust CANONICAL_VECTORS in " + "crypto/crypto/src/hash/blake3/vectors.rs", + not bad_rust, f"vectors {bad_rust}") + + # Inputs must match too, or the output agreement is about different things. + bad_in = [i for i in range(10) + if any(rust[i][f] != js[i][f] for f in ("h", "m", "t", "block_len", "flags"))] + check("[D] Rust vector INPUTS == JSON vector inputs", not bad_in, f"vectors {bad_in}") + + # Negative control: the same inputs at 7 rounds must NOT match. + lines7 = run(B3REF7, "compress", stdin=stdin).strip().split("\n") + c7 = [[int(line[8 * i:8 * i + 8], 16) for i in range(16)] for line in lines7] + check("[D] negative control: C @ rounds=7 matches none of the 10 vectors", + all(c7[i] != js[i]["out"] for i in range(10))) + + +# --------------------------------------------------------------------------- +# [E] randomised differential against the Python oracle +# --------------------------------------------------------------------------- + +def check_differential(n=5000): + rng = random.Random(0x5EC0D) + recs = [] + expect7 = [] + expect6 = [] + for _ in range(n): + v = { + "h": [rng.randrange(1 << 32) for _ in range(8)], + "m": [rng.randrange(1 << 32) for _ in range(16)], + "t": rng.randrange(1 << 64), + "block_len": rng.randrange(0, 65), + "flags": rng.randrange(0, 256), + } + recs.append(encode_record(v)) + expect7.append(ref.compress(v["h"], v["m"], v["t"], v["block_len"], + v["flags"], rounds=7)) + expect6.append(ref.compress(v["h"], v["m"], v["t"], v["block_len"], + v["flags"], rounds=6)) + + stdin = "".join(recs) + for binary, expect, label in ((B3REF7, expect7, 7), (B3REF6, expect6, 6)): + lines = run(binary, "compress", stdin=stdin).strip().split("\n") + got = [[int(ln[8 * i:8 * i + 8], 16) for i in range(16)] for ln in lines] + bad = [i for i in range(n) if got[i] != expect[i]] + check(f"[E] C vs Python oracle @ rounds={label} ({n} random compressions)", + len(got) == n and not bad, f"{len(bad)} mismatches, first={bad[:3]}") + + +def main(): + if not (os.path.exists(B3REF7) and os.path.exists(B3REF6)): + print("binaries missing -- run ./build.sh first") + return 2 + + print("=" * 74) + print("SECOND-SOURCE CHECK: upstream BLAKE3 C (round-parameterised)") + print("=" * 74) + check_official_vectors() + check_six_differs() + check_schedule_equivalence() + check_canonical_vectors() + check_differential() + print("=" * 74) + if FAILURES: + print(f"RESULT: {len(FAILURES)} FAILURE(S): {FAILURES}") + return 1 + print("RESULT: ALL GREEN -- two independent sources agree on the ten") + print(" 6-round vectors, and the 7-round anchor is external.") + print("=" * 74) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tooling/blake3-second-source/reference-impl/driver.c b/tooling/blake3-second-source/reference-impl/driver.c new file mode 100644 index 000000000..d6338508d --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/driver.c @@ -0,0 +1,188 @@ +/* Test driver for the round-parameterised upstream BLAKE3 C reference. + * + * This file is Lambda VM's; everything it links against in `upstream/` is + * upstream BLAKE3 (CC0 / Apache-2.0), unmodified except for the round-count + * loop in blake3_portable_paramrounds.c (see PARAMETERISATION.diff). + * + * Modes: + * hash default hashing mode + * hashhex default hashing mode, explicit msg + * keyed keyed_hash mode + * derive derive_key mode + * compress raw compression from stdin: + * one whitespace-separated record per line -- + * h[0..8] (8 hex u32) m[0..16] (16 hex u32) t (hex u64) + * block_len (dec) flags (dec) + * prints the 16-word output as 16 concatenated 8-hex-digit words. + * + * The `compress` mode calls blake3_compress_xof_portable directly, which is + * the 16-word (64-byte) output of the compression function `f` -- exactly the + * object CANONICAL_VECTORS pins. + * + * Input bytes for the hashing modes follow the official test-vector pattern: + * byte i is (i % 251). + */ +#include "upstream/blake3.h" +#include "upstream/blake3_impl.h" +#include +#include +#include + +static void fill_pattern(uint8_t *buf, size_t len) { + for (size_t i = 0; i < len; i++) { + buf[i] = (uint8_t)(i % 251); + } +} + +static void print_hex(const uint8_t *b, size_t n) { + for (size_t i = 0; i < n; i++) { + printf("%02x", b[i]); + } + printf("\n"); +} + +static int hex_to_bytes(const char *hex, uint8_t *out, size_t out_len) { + if (strlen(hex) != out_len * 2) { + return -1; + } + for (size_t i = 0; i < out_len; i++) { + unsigned v; + if (sscanf(hex + 2 * i, "%2x", &v) != 1) { + return -1; + } + out[i] = (uint8_t)v; + } + return 0; +} + +static int run_hasher(blake3_hasher *h, size_t input_len, size_t out_len) { + uint8_t *input = malloc(input_len ? input_len : 1); + uint8_t *out = malloc(out_len ? out_len : 1); + if (!input || !out) { + return 1; + } + fill_pattern(input, input_len); + blake3_hasher_update(h, input, input_len); + blake3_hasher_finalize(h, out, out_len); + print_hex(out, out_len); + free(input); + free(out); + return 0; +} + +static int mode_compress(void) { + uint32_t h[8], m[16]; + unsigned long long t; + unsigned block_len, flags; + char line[4096]; + + while (fgets(line, sizeof(line), stdin)) { + char *p = line; + int consumed; + int ok = 1; + + for (int i = 0; i < 8 && ok; i++) { + if (sscanf(p, " %x%n", &h[i], &consumed) != 1) { + ok = 0; + } + p += consumed; + } + for (int i = 0; i < 16 && ok; i++) { + if (sscanf(p, " %x%n", &m[i], &consumed) != 1) { + ok = 0; + } + p += consumed; + } + if (ok && sscanf(p, " %llx %u %u", &t, &block_len, &flags) != 3) { + ok = 0; + } + if (!ok) { + continue; /* blank or malformed line */ + } + + /* The compression function takes the message block as 64 little-endian + * bytes; serialise m[] the way BLAKE3 loads it (load32 is LE). */ + uint8_t block[BLAKE3_BLOCK_LEN]; + for (int i = 0; i < 16; i++) { + store32(&block[i * 4], m[i]); + } + + uint8_t out64[64]; + blake3_compress_xof_portable(h, block, (uint8_t)block_len, + (uint64_t)t, (uint8_t)flags, out64); + + for (int i = 0; i < 16; i++) { + printf("%08x", load32(&out64[i * 4])); + } + printf("\n"); + fflush(stdout); + } + return 0; +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s hash|keyed|derive|compress ...\n", argv[0]); + return 2; + } + + if (strcmp(argv[1], "compress") == 0) { + return mode_compress(); + } + + if (strcmp(argv[1], "hash") == 0 && argc == 4) { + blake3_hasher h; + blake3_hasher_init(&h); + return run_hasher(&h, strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10)); + } + + /* Whole-message hashing of an explicit byte string. This is what turns the + * socket spec's "compress(a,b) == BLAKE3(a || b || tag) truncated" claim into + * something executable against upstream code rather than against a compress + * call this repo assembled itself. */ + if (strcmp(argv[1], "hashhex") == 0 && argc == 4) { + size_t msg_len = strlen(argv[2]) / 2; + if (strlen(argv[2]) % 2 != 0) { + fprintf(stderr, "message hex must have even length\n"); + return 2; + } + uint8_t *msg = malloc(msg_len ? msg_len : 1); + if (!msg || hex_to_bytes(argv[2], msg, msg_len) != 0) { + fprintf(stderr, "bad message hex\n"); + return 2; + } + size_t out_len = strtoul(argv[3], NULL, 10); + uint8_t *out = malloc(out_len ? out_len : 1); + if (!out) { + return 1; + } + blake3_hasher h; + blake3_hasher_init(&h); + blake3_hasher_update(&h, msg, msg_len); + blake3_hasher_finalize(&h, out, out_len); + print_hex(out, out_len); + free(msg); + free(out); + return 0; + } + + if (strcmp(argv[1], "keyed") == 0 && argc == 5) { + uint8_t key[BLAKE3_KEY_LEN]; + if (hex_to_bytes(argv[2], key, BLAKE3_KEY_LEN) != 0) { + fprintf(stderr, "bad key hex (need %d bytes)\n", BLAKE3_KEY_LEN); + return 2; + } + blake3_hasher h; + blake3_hasher_init_keyed(&h, key); + return run_hasher(&h, strtoul(argv[3], NULL, 10), strtoul(argv[4], NULL, 10)); + } + + if (strcmp(argv[1], "derive") == 0 && argc == 5) { + blake3_hasher h; + blake3_hasher_init_derive_key(&h, argv[2]); + return run_hasher(&h, strtoul(argv[3], NULL, 10), strtoul(argv[4], NULL, 10)); + } + + fprintf(stderr, "bad arguments\n"); + return 2; +} diff --git a/tooling/blake3-second-source/reference-impl/upstream/LICENSE_CC0 b/tooling/blake3-second-source/reference-impl/upstream/LICENSE_CC0 new file mode 100644 index 000000000..0e259d42c --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/upstream/LICENSE_CC0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/tooling/blake3-second-source/reference-impl/upstream/blake3.c b/tooling/blake3-second-source/reference-impl/upstream/blake3.c new file mode 100644 index 000000000..00f91f444 --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/upstream/blake3.c @@ -0,0 +1,651 @@ +#include +#include +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" + +const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } + +INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + self->buf_len = 0; + self->blocks_compressed = 0; + self->flags = flags; +} + +INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], + uint64_t chunk_counter) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = chunk_counter; + self->blocks_compressed = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + self->buf_len = 0; +} + +INLINE size_t chunk_state_len(const blake3_chunk_state *self) { + return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + + ((size_t)self->buf_len); +} + +INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, + const uint8_t *input, size_t input_len) { + size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); + if (take > input_len) { + take = input_len; + } + uint8_t *dest = self->buf + ((size_t)self->buf_len); + memcpy(dest, input, take); + self->buf_len += (uint8_t)take; + return take; +} + +INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { + if (self->blocks_compressed == 0) { + return CHUNK_START; + } else { + return 0; + } +} + +typedef struct { + uint32_t input_cv[8]; + uint64_t counter; + uint8_t block[BLAKE3_BLOCK_LEN]; + uint8_t block_len; + uint8_t flags; +} output_t; + +INLINE output_t make_output(const uint32_t input_cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + output_t ret; + memcpy(ret.input_cv, input_cv, 32); + memcpy(ret.block, block, BLAKE3_BLOCK_LEN); + ret.block_len = block_len; + ret.counter = counter; + ret.flags = flags; + return ret; +} + +// Chaining values within a given chunk (specifically the compress_in_place +// interface) are represented as words. This avoids unnecessary bytes<->words +// conversion overhead in the portable implementation. However, the hash_many +// interface handles both user input and parent node blocks, so it accepts +// bytes. For that reason, chaining values in the CV stack are represented as +// bytes. +INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { + uint32_t cv_words[8]; + memcpy(cv_words, self->input_cv, 32); + blake3_compress_in_place(cv_words, self->block, self->block_len, + self->counter, self->flags); + store_cv_words(cv, cv_words); +} + +INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, + size_t out_len) { + if (out_len == 0) { + return; + } + uint64_t output_block_counter = seek / 64; + size_t offset_within_block = seek % 64; + uint8_t wide_buf[64]; + if(offset_within_block) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + const size_t available_bytes = 64 - offset_within_block; + const size_t bytes = out_len > available_bytes ? available_bytes : out_len; + memcpy(out, wide_buf + offset_within_block, bytes); + out += bytes; + out_len -= bytes; + output_block_counter += 1; + } + if(out_len / 64) { + blake3_xof_many(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, out, out_len / 64); + } + output_block_counter += out_len / 64; + out += out_len & -64; + out_len -= out_len & -64; + if(out_len) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + memcpy(out, wide_buf, out_len); + } +} + +INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, + size_t input_len) { + if (self->buf_len > 0) { + size_t take = chunk_state_fill_buf(self, input, input_len); + input += take; + input_len -= take; + if (input_len > 0) { + blake3_compress_in_place( + self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + self->buf_len = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + } + } + + while (input_len > BLAKE3_BLOCK_LEN) { + blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, + self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + input += BLAKE3_BLOCK_LEN; + input_len -= BLAKE3_BLOCK_LEN; + } + + chunk_state_fill_buf(self, input, input_len); +} + +INLINE output_t chunk_state_output(const blake3_chunk_state *self) { + uint8_t block_flags = + self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; + return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, + block_flags); +} + +INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], + const uint32_t key[8], uint8_t flags) { + return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); +} + +// Given some input larger than one chunk, return the number of bytes that +// should go in the left subtree. This is the largest power-of-2 number of +// chunks that leaves at least 1 byte for the right subtree. +INLINE size_t left_subtree_len(size_t input_len) { + // Subtract 1 to reserve at least one byte for the right side. input_len + // should always be greater than BLAKE3_CHUNK_LEN. + size_t full_chunks = (input_len - 1) / BLAKE3_CHUNK_LEN; + return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time +// on a single thread. Write out the chunk chaining values and return the +// number of chunks hashed. These chunks are never the root and never empty; +// those cases use a different codepath. +INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(0 < input_len); + assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); +#endif + + const uint8_t *chunks_array[MAX_SIMD_DEGREE]; + size_t input_position = 0; + size_t chunks_array_len = 0; + while (input_len - input_position >= BLAKE3_CHUNK_LEN) { + chunks_array[chunks_array_len] = &input[input_position]; + input_position += BLAKE3_CHUNK_LEN; + chunks_array_len += 1; + } + + blake3_hash_many(chunks_array, chunks_array_len, + BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, + true, flags, CHUNK_START, CHUNK_END, out); + + // Hash the remaining partial chunk, if there is one. Note that the empty + // chunk (meaning the empty message) is a different codepath. + if (input_len > input_position) { + uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, key, flags); + chunk_state.chunk_counter = counter; + chunk_state_update(&chunk_state, &input[input_position], + input_len - input_position); + output_t output = chunk_state_output(&chunk_state); + output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]); + return chunks_array_len + 1; + } else { + return chunks_array_len; + } +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time +// on a single thread. Write out the parent chaining values and return the +// number of parents hashed. (If there's an odd input chaining value left over, +// return it as an additional output.) These parents are never the root and +// never empty; those cases use a different codepath. +INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, + size_t num_chaining_values, + const uint32_t key[8], uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(2 <= num_chaining_values); + assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); +#endif + + const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; + size_t parents_array_len = 0; + while (num_chaining_values - (2 * parents_array_len) >= 2) { + parents_array[parents_array_len] = + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; + parents_array_len += 1; + } + + blake3_hash_many(parents_array, parents_array_len, 1, key, + 0, // Parents always use counter 0. + false, flags | PARENT, + 0, // Parents have no start flags. + 0, // Parents have no end flags. + out); + + // If there's an odd child left over, it becomes an output. + if (num_chaining_values > 2 * parents_array_len) { + memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], + BLAKE3_OUT_LEN); + return parents_array_len + 1; + } else { + return parents_array_len; + } +} + +// The wide helper function returns (writes out) an array of chaining values +// and returns the length of that array. The number of chaining values returned +// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, +// if the input is shorter than that many chunks. The reason for maintaining a +// wide array of chaining values going back up the tree, is to allow the +// implementation to hash as many parents in parallel as possible. +// +// As a special case when the SIMD degree is 1, this function will still return +// at least 2 outputs. This guarantees that this function doesn't perform the +// root compression. (If it did, it would use the wrong flags, and also we +// wouldn't be able to implement extendable output.) Note that this function is +// not used when the whole input is only 1 chunk long; that's a different +// codepath. +// +// Why not just have the caller split the input on the first update(), instead +// of implementing this special rule? Because we don't want to limit SIMD or +// multi-threading parallelism for that update(). +size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb) { + // Note that the single chunk case does *not* bump the SIMD degree up to 2 + // when it is 1. If this implementation adds multi-threading in the future, + // this gives us the option of multi-threading even the 2-chunk case, which + // can help performance on smaller platforms. + if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { + return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, + out); + } + + // With more than simd_degree chunks, we need to recurse. Start by dividing + // the input into left and right subtrees. (Note that this is only optimal + // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree + // of 3 or something, we'll need a more complicated strategy.) + size_t left_input_len = left_subtree_len(input_len); + size_t right_input_len = input_len - left_input_len; + const uint8_t *right_input = &input[left_input_len]; + uint64_t right_chunk_counter = + chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); + + // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to + // account for the special case of returning 2 outputs when the SIMD degree + // is 1. + uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t degree = blake3_simd_degree(); + if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { + // The special case: We always use a degree of at least two, to make + // sure there are two outputs. Except, as noted above, at the chunk + // level, where we allow degree=1. (Note that the 1-chunk-input case is + // a different codepath.) + degree = 2; + } + uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; + + // Recurse! + size_t left_n = SIZE_MAX; + size_t right_n = SIZE_MAX; + +#if defined(BLAKE3_USE_TBB) + blake3_compress_subtree_wide_join_tbb( + key, flags, use_tbb, + // left-hand side + input, left_input_len, chunk_counter, cv_array, &left_n, + // right-hand side + right_input, right_input_len, right_chunk_counter, right_cvs, &right_n); +#else + left_n = blake3_compress_subtree_wide( + input, left_input_len, key, chunk_counter, flags, cv_array, use_tbb); + right_n = blake3_compress_subtree_wide(right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs, + use_tbb); +#endif // BLAKE3_USE_TBB + + // The special case again. If simd_degree=1, then we'll have left_n=1 and + // right_n=1. Rather than compressing them into a single output, return + // them directly, to make sure we always have at least two outputs. + if (left_n == 1) { + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); + return 2; + } + + // Otherwise, do one layer of parent node compression. + size_t num_chaining_values = left_n + right_n; + return compress_parents_parallel(cv_array, num_chaining_values, key, flags, + out); +} + +// Hash a subtree with compress_subtree_wide(), and then condense the resulting +// list of chaining values down to a single parent node. Don't compress that +// last parent node, however. Instead, return its message bytes (the +// concatenated chaining values of its children). This is necessary when the +// first call to update() supplies a complete subtree, because the topmost +// parent node of that subtree could end up being the root. It's also necessary +// for extended output in the general case. +// +// As with compress_subtree_wide(), this function is not used on inputs of 1 +// chunk or less. That's a different codepath. +INLINE void +compress_subtree_to_parent_node(const uint8_t *input, size_t input_len, + const uint32_t key[8], uint64_t chunk_counter, + uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN], + bool use_tbb) { +#if defined(BLAKE3_TESTING) + assert(input_len > BLAKE3_CHUNK_LEN); +#endif + + uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, + chunk_counter, flags, cv_array, use_tbb); + assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); + // The following loop never executes when MAX_SIMD_DEGREE_OR_2 is 2, because + // as we just asserted, num_cvs will always be <=2 in that case. But GCC + // (particularly GCC 8.5) can't tell that it never executes, and if NDEBUG is + // set then it emits incorrect warnings here. We tried a few different + // hacks to silence these, but in the end our hacks just produced different + // warnings (see https://github.com/BLAKE3-team/BLAKE3/pull/380). Out of + // desperation, we ifdef out this entire loop when we know it's not needed. +#if MAX_SIMD_DEGREE_OR_2 > 2 + // If MAX_SIMD_DEGREE_OR_2 is greater than 2 and there's enough input, + // compress_subtree_wide() returns more than 2 chaining values. Condense + // them into 2 by forming parent nodes repeatedly. + uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; + while (num_cvs > 2) { + num_cvs = + compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); + memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); + } +#endif + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); +} + +INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->key, key, BLAKE3_KEY_LEN); + chunk_state_init(&self->chunk, key, flags); + self->cv_stack_len = 0; +} + +void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); } + +void blake3_hasher_init_keyed(blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]) { + uint32_t key_words[8]; + load_key_words(key, key_words); + hasher_init_base(self, key_words, KEYED_HASH); +} + +void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, + size_t context_len) { + blake3_hasher context_hasher; + hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT); + blake3_hasher_update(&context_hasher, context, context_len); + uint8_t context_key[BLAKE3_KEY_LEN]; + blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); + uint32_t context_key_words[8]; + load_key_words(context_key, context_key_words); + hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL); +} + +void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) { + blake3_hasher_init_derive_key_raw(self, context, strlen(context)); +} + +// As described in hasher_push_cv() below, we do "lazy merging", delaying +// merges until right before the next CV is about to be added. This is +// different from the reference implementation. Another difference is that we +// aren't always merging 1 chunk at a time. Instead, each CV might represent +// any power-of-two number of chunks, as long as the smaller-above-larger stack +// order is maintained. Instead of the "count the trailing 0-bits" algorithm +// described in the spec, we use a "count the total number of 1-bits" variant +// that doesn't require us to retain the subtree size of the CV on top of the +// stack. The principle is the same: each CV that should remain in the stack is +// represented by a 1-bit in the total number of chunks (or bytes) so far. +INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { + size_t post_merge_stack_len = (size_t)popcnt(total_len); + while (self->cv_stack_len > post_merge_stack_len) { + uint8_t *parent_node = + &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; + output_t output = parent_output(parent_node, self->key, self->chunk.flags); + output_chaining_value(&output, parent_node); + self->cv_stack_len -= 1; + } +} + +// In reference_impl.rs, we merge the new CV with existing CVs from the stack +// before pushing it. We can do that because we know more input is coming, so +// we know none of the merges are root. +// +// This setting is different. We want to feed as much input as possible to +// compress_subtree_wide(), without setting aside anything for the chunk_state. +// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once +// as a single subtree, if at all possible. +// +// This leads to two problems: +// 1) This 64 KiB input might be the only call that ever gets made to update. +// In this case, the root node of the 64 KiB subtree would be the root node +// of the whole tree, and it would need to be ROOT finalized. We can't +// compress it until we know. +// 2) This 64 KiB input might complete a larger tree, whose root node is +// similarly going to be the root of the whole tree. For example, maybe +// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the +// node at the root of the 256 KiB subtree until we know how to finalize it. +// +// The second problem is solved with "lazy merging". That is, when we're about +// to add a CV to the stack, we don't merge it with anything first, as the +// reference impl does. Instead we do merges using the *previous* CV that was +// added, which is sitting on top of the stack, and we put the new CV +// (unmerged) on top of the stack afterwards. This guarantees that we never +// merge the root node until finalize(). +// +// Solving the first problem requires an additional tool, +// compress_subtree_to_parent_node(). That function always returns the top +// *two* chaining values of the subtree it's compressing. We then do lazy +// merging with each of them separately, so that the second CV will always +// remain unmerged. (That also helps us support extendable output when we're +// hashing an input all-at-once.) +INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], + uint64_t chunk_counter) { + hasher_merge_cv_stack(self, chunk_counter); + memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, + BLAKE3_OUT_LEN); + self->cv_stack_len += 1; +} + +INLINE void blake3_hasher_update_base(blake3_hasher *self, const void *input, + size_t input_len, bool use_tbb) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_update(&hasher, v.data(), v.size()); + if (input_len == 0) { + return; + } + + const uint8_t *input_bytes = (const uint8_t *)input; + + // If we have some partial chunk bytes in the internal chunk_state, we need + // to finish that chunk first. + if (chunk_state_len(&self->chunk) > 0) { + size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); + if (take > input_len) { + take = input_len; + } + chunk_state_update(&self->chunk, input_bytes, take); + input_bytes += take; + input_len -= take; + // If we've filled the current chunk and there's more coming, finalize this + // chunk and proceed. In this case we know it's not the root. + if (input_len > 0) { + output_t output = chunk_state_output(&self->chunk); + uint8_t chunk_cv[32]; + output_chaining_value(&output, chunk_cv); + hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); + chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); + } else { + return; + } + } + + // Now the chunk_state is clear, and we have more input. If there's more than + // a single chunk (so, definitely not the root chunk), hash the largest whole + // subtree we can, with the full benefits of SIMD (and maybe in the future, + // multi-threading) parallelism. Two restrictions: + // - The subtree has to be a power-of-2 number of chunks. Only subtrees along + // the right edge can be incomplete, and we don't know where the right edge + // is going to be until we get to finalize(). + // - The subtree must evenly divide the total number of chunks up until this + // point (if total is not 0). If the current incomplete subtree is only + // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have + // to complete the current subtree first. + // Because we might need to break up the input to form powers of 2, or to + // evenly divide what we already have, this part runs in a loop. + while (input_len > BLAKE3_CHUNK_LEN) { + size_t subtree_len = round_down_to_power_of_2(input_len); + uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; + // Shrink the subtree_len until it evenly divides the count so far. We know + // that subtree_len itself is a power of 2, so we can use a bitmasking + // trick instead of an actual remainder operation. (Note that if the caller + // consistently passes power-of-2 inputs of the same size, as is hopefully + // typical, this loop condition will always fail, and subtree_len will + // always be the full length of the input.) + // + // An aside: We don't have to shrink subtree_len quite this much. For + // example, if count_so_far is 1, we could pass 2 chunks to + // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still + // get the right answer in the end, and we might get to use 2-way SIMD + // parallelism. The problem with this optimization, is that it gets us + // stuck always hashing 2 chunks. The total number of chunks will remain + // odd, and we'll never graduate to higher degrees of parallelism. See + // https://github.com/BLAKE3-team/BLAKE3/issues/69. + while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { + subtree_len /= 2; + } + // The shrunken subtree_len might now be 1 chunk long. If so, hash that one + // chunk by itself. Otherwise, compress the subtree into a pair of CVs. + uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; + if (subtree_len <= BLAKE3_CHUNK_LEN) { + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, self->key, self->chunk.flags); + chunk_state.chunk_counter = self->chunk.chunk_counter; + chunk_state_update(&chunk_state, input_bytes, subtree_len); + output_t output = chunk_state_output(&chunk_state); + uint8_t cv[BLAKE3_OUT_LEN]; + output_chaining_value(&output, cv); + hasher_push_cv(self, cv, chunk_state.chunk_counter); + } else { + // This is the high-performance happy path, though getting here depends + // on the caller giving us a long enough input. + uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; + compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, + self->chunk.chunk_counter, + self->chunk.flags, cv_pair, use_tbb); + hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); + hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], + self->chunk.chunk_counter + (subtree_chunks / 2)); + } + self->chunk.chunk_counter += subtree_chunks; + input_bytes += subtree_len; + input_len -= subtree_len; + } + + // If there's any remaining input less than a full chunk, add it to the chunk + // state. In that case, also do a final merge loop to make sure the subtree + // stack doesn't contain any unmerged pairs. The remaining input means we + // know these merges are non-root. This merge loop isn't strictly necessary + // here, because hasher_push_chunk_cv already does its own merge loop, but it + // simplifies blake3_hasher_finalize below. + if (input_len > 0) { + chunk_state_update(&self->chunk, input_bytes, input_len); + hasher_merge_cv_stack(self, self->chunk.chunk_counter); + } +} + +void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = false; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} + +#if defined(BLAKE3_USE_TBB) +void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = true; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} +#endif // BLAKE3_USE_TBB + +void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, + size_t out_len) { + blake3_hasher_finalize_seek(self, 0, out, out_len); +} + +void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_finalize(&hasher, v.data(), v.size()); + if (out_len == 0) { + return; + } + + // If the subtree stack is empty, then the current chunk is the root. + if (self->cv_stack_len == 0) { + output_t output = chunk_state_output(&self->chunk); + output_root_bytes(&output, seek, out, out_len); + return; + } + // If there are any bytes in the chunk state, finalize that chunk and do a + // roll-up merge between that chunk hash and every subtree in the stack. In + // this case, the extra merge loop at the end of blake3_hasher_update + // guarantees that none of the subtrees in the stack need to be merged with + // each other first. Otherwise, if there are no bytes in the chunk state, + // then the top of the stack is a chunk hash, and we start the merge from + // that. + output_t output; + size_t cvs_remaining; + if (chunk_state_len(&self->chunk) > 0) { + cvs_remaining = self->cv_stack_len; + output = chunk_state_output(&self->chunk); + } else { + // There are always at least 2 CVs in the stack in this case. + cvs_remaining = self->cv_stack_len - 2; + output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, + self->chunk.flags); + } + while (cvs_remaining > 0) { + cvs_remaining -= 1; + uint8_t parent_block[BLAKE3_BLOCK_LEN]; + memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); + output_chaining_value(&output, &parent_block[32]); + output = parent_output(parent_block, self->key, self->chunk.flags); + } + output_root_bytes(&output, seek, out, out_len); +} + +void blake3_hasher_reset(blake3_hasher *self) { + chunk_state_reset(&self->chunk, self->key, 0); + self->cv_stack_len = 0; +} diff --git a/tooling/blake3-second-source/reference-impl/upstream/blake3.h b/tooling/blake3-second-source/reference-impl/upstream/blake3.h new file mode 100644 index 000000000..423154ff7 --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/upstream/blake3.h @@ -0,0 +1,86 @@ +#ifndef BLAKE3_H +#define BLAKE3_H + +#include +#include + +#if !defined(BLAKE3_API) +# if defined(_WIN32) || defined(__CYGWIN__) +# if defined(BLAKE3_DLL) +# if defined(BLAKE3_DLL_EXPORTS) +# define BLAKE3_API __declspec(dllexport) +# else +# define BLAKE3_API __declspec(dllimport) +# endif +# define BLAKE3_PRIVATE +# else +# define BLAKE3_API +# define BLAKE3_PRIVATE +# endif +# elif __GNUC__ >= 4 +# define BLAKE3_API __attribute__((visibility("default"))) +# define BLAKE3_PRIVATE __attribute__((visibility("hidden"))) +# else +# define BLAKE3_API +# define BLAKE3_PRIVATE +# endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define BLAKE3_VERSION_STRING "1.8.5" +#define BLAKE3_KEY_LEN 32 +#define BLAKE3_OUT_LEN 32 +#define BLAKE3_BLOCK_LEN 64 +#define BLAKE3_CHUNK_LEN 1024 +#define BLAKE3_MAX_DEPTH 54 + +// This struct is a private implementation detail. It has to be here because +// it's part of the blake3_hasher structure defined below. +typedef struct { + uint32_t cv[8]; + uint64_t chunk_counter; + uint8_t buf[BLAKE3_BLOCK_LEN]; + uint8_t buf_len; + uint8_t blocks_compressed; + uint8_t flags; +} blake3_chunk_state; + +typedef struct { + uint32_t key[8]; + blake3_chunk_state chunk; + uint8_t cv_stack_len; + // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, + // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk + // requires a 4th entry, rather than merging everything down to 1, because we + // don't know whether more input is coming. This is different from how the + // reference implementation does things. + uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; +} blake3_hasher; + +BLAKE3_API const char *blake3_version(void); +BLAKE3_API void blake3_hasher_init(blake3_hasher *self); +BLAKE3_API void blake3_hasher_init_keyed(blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]); +BLAKE3_API void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); +BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, + size_t context_len); +BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len); +#if defined(BLAKE3_USE_TBB) +BLAKE3_API void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len); +#endif // BLAKE3_USE_TBB +BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, + size_t out_len); +BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len); +BLAKE3_API void blake3_hasher_reset(blake3_hasher *self); + +#ifdef __cplusplus +} +#endif + +#endif /* BLAKE3_H */ diff --git a/tooling/blake3-second-source/reference-impl/upstream/blake3_dispatch.c b/tooling/blake3-second-source/reference-impl/upstream/blake3_dispatch.c new file mode 100644 index 000000000..14dfbbe0c --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/upstream/blake3_dispatch.c @@ -0,0 +1,332 @@ +#include +#include +#include + +#include "blake3_impl.h" + +#if defined(_MSC_VER) +#include +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#elif defined(__GNUC__) +#include +#else +#undef IS_X86 /* Unimplemented! */ +#endif +#endif + +#if !defined(BLAKE3_ATOMICS) +#if defined(__has_include) +#if __has_include() && !defined(_MSC_VER) +#define BLAKE3_ATOMICS 1 +#else +#define BLAKE3_ATOMICS 0 +#endif /* __has_include() && !defined(_MSC_VER) */ +#else +#define BLAKE3_ATOMICS 0 +#endif /* defined(__has_include) */ +#endif /* BLAKE3_ATOMICS */ + +#if BLAKE3_ATOMICS +#define ATOMIC_INT _Atomic int +#define ATOMIC_LOAD(x) x +#define ATOMIC_STORE(x, y) x = y +#elif defined(_MSC_VER) +#define ATOMIC_INT LONG +#define ATOMIC_LOAD(x) InterlockedOr(&x, 0) +#define ATOMIC_STORE(x, y) InterlockedExchange(&x, y) +#else +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) x +#define ATOMIC_STORE(x, y) x = y +#endif + +#define MAYBE_UNUSED(x) (void)((x)) + +#if defined(IS_X86) +static uint64_t xgetbv(void) { +#if defined(_MSC_VER) + return _xgetbv(0); +#else + uint32_t eax = 0, edx = 0; + __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); + return ((uint64_t)edx << 32) | eax; +#endif +} + +static void cpuid(uint32_t out[4], uint32_t id) { +#if defined(_MSC_VER) + __cpuid((int *)out, id); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__("movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#endif +} + +static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { +#if defined(_MSC_VER) + __cpuidex((int *)out, id, sid); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__("movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#endif +} + + +enum cpu_feature { + SSE2 = 1 << 0, + SSSE3 = 1 << 1, + SSE41 = 1 << 2, + AVX = 1 << 3, + AVX2 = 1 << 4, + AVX512F = 1 << 5, + AVX512VL = 1 << 6, + /* ... */ + UNDEFINED = 1 << 30 +}; + +#if !defined(BLAKE3_TESTING) +static /* Allow the variable to be controlled manually for testing */ +#endif + ATOMIC_INT g_cpu_features = UNDEFINED; + +#if !defined(BLAKE3_TESTING) +static +#endif + enum cpu_feature + get_cpu_features(void) { + + /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ + enum cpu_feature features = ATOMIC_LOAD(g_cpu_features); + if (features != UNDEFINED) { + return features; + } else { +#if defined(IS_X86) + uint32_t regs[4] = {0}; + uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; + (void)edx; + features = 0; + cpuid(regs, 0); + const int max_id = *eax; + cpuid(regs, 1); +#if defined(__amd64__) || defined(_M_X64) + features |= SSE2; +#else + if (*edx & (1UL << 26)) + features |= SSE2; +#endif + if (*ecx & (1UL << 9)) + features |= SSSE3; + if (*ecx & (1UL << 19)) + features |= SSE41; + + if (*ecx & (1UL << 27)) { // OSXSAVE + const uint64_t mask = xgetbv(); + if ((mask & 6) == 6) { // SSE and AVX states + if (*ecx & (1UL << 28)) + features |= AVX; + if (max_id >= 7) { + cpuidex(regs, 7, 0); + if (*ebx & (1UL << 5)) + features |= AVX2; + if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm + if (*ebx & (1UL << 31)) + features |= AVX512VL; + if (*ebx & (1UL << 16)) + features |= AVX512F; + } + } + } + } + ATOMIC_STORE(g_cpu_features, features); + return features; +#else + /* How to detect NEON? */ + return 0; +#endif + } +} +#endif + +void blake3_compress_in_place(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_compress_in_place_sse2(cv, block, block_len, counter, flags); + return; + } +#endif +#endif + blake3_compress_in_place_portable(cv, block, block_len, counter, flags); +} + +void blake3_compress_xof(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out); + return; + } +#endif +#endif + blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); +} + + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks) { + if (outblocks == 0) { + // The current assembly implementation always outputs at least 1 block. + return; + } +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); + return; + } +#endif +#endif + for(size_t i = 0; i < outblocks; ++i) { + blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i); + } +} + +void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { + blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_AVX2) + if (features & AVX2) { + blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#endif + +#if BLAKE3_USE_NEON == 1 + blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, out); + return; +#endif + + blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); +} + +// The dynamically detected SIMD degree of the current platform. +size_t blake3_simd_degree(void) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { + return 16; + } +#endif +#if !defined(BLAKE3_NO_AVX2) + if (features & AVX2) { + return 8; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + return 4; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + return 4; + } +#endif +#endif +#if BLAKE3_USE_NEON == 1 + return 4; +#endif + return 1; +} diff --git a/tooling/blake3-second-source/reference-impl/upstream/blake3_impl.h b/tooling/blake3-second-source/reference-impl/upstream/blake3_impl.h new file mode 100644 index 000000000..88e71e41e --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/upstream/blake3_impl.h @@ -0,0 +1,333 @@ +#ifndef BLAKE3_IMPL_H +#define BLAKE3_IMPL_H + +#include +#include +#include +#include +#include + +#include "blake3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// internal flags +enum blake3_flags { + CHUNK_START = 1 << 0, + CHUNK_END = 1 << 1, + PARENT = 1 << 2, + ROOT = 1 << 3, + KEYED_HASH = 1 << 4, + DERIVE_KEY_CONTEXT = 1 << 5, + DERIVE_KEY_MATERIAL = 1 << 6, +}; + +// This C implementation tries to support recent versions of GCC, Clang, and +// MSVC. +#if defined(_MSC_VER) +#define INLINE static __forceinline +#else +#define INLINE static inline __attribute__((always_inline)) +#endif + +#ifdef __cplusplus +#define NOEXCEPT noexcept +#else +#define NOEXCEPT +#endif + +#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC) +#define IS_X86 +#define IS_X86_64 +#endif + +#if defined(__i386__) || defined(_M_IX86) +#define IS_X86 +#define IS_X86_32 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +#define IS_AARCH64 +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#endif +#endif + +#if !defined(BLAKE3_USE_NEON) + // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness + #if defined(IS_AARCH64) + #if defined(__ARM_BIG_ENDIAN) + #define BLAKE3_USE_NEON 0 + #else + #define BLAKE3_USE_NEON 1 + #endif + #else + #define BLAKE3_USE_NEON 0 + #endif +#endif + +#if defined(IS_X86) +#define MAX_SIMD_DEGREE 16 +#elif BLAKE3_USE_NEON == 1 +#define MAX_SIMD_DEGREE 4 +#else +#define MAX_SIMD_DEGREE 1 +#endif + +// There are some places where we want a static size that's equal to the +// MAX_SIMD_DEGREE, but also at least 2. +#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) + +static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, + 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, + 0x1F83D9ABUL, 0x5BE0CD19UL}; + +static const uint8_t MSG_SCHEDULE[7][16] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, + {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, + {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, + {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, + {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, + {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, +}; + +/* Find index of the highest set bit */ +/* x is assumed to be nonzero. */ +static unsigned int highest_one(uint64_t x) { +#if defined(__GNUC__) || defined(__clang__) + return 63 ^ (unsigned int)__builtin_clzll(x); +#elif defined(_MSC_VER) && defined(IS_X86_64) + unsigned long index; + _BitScanReverse64(&index, x); + return index; +#elif defined(_MSC_VER) && defined(IS_X86_32) + if(x >> 32) { + unsigned long index; + _BitScanReverse(&index, (unsigned long)(x >> 32)); + return 32 + index; + } else { + unsigned long index; + _BitScanReverse(&index, (unsigned long)x); + return index; + } +#else + unsigned int c = 0; + if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } + if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } + if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } + if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } + if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } + if(x & 0x0000000000000002ULL) { c += 1; } + return c; +#endif +} + +// Count the number of 1 bits. +INLINE unsigned int popcnt(uint64_t x) { +#if defined(__GNUC__) || defined(__clang__) + return (unsigned int)__builtin_popcountll(x); +#else + unsigned int count = 0; + while (x != 0) { + count += 1; + x &= x - 1; + } + return count; +#endif +} + +// Largest power of two less than or equal to x. As a special case, returns 1 +// when x is 0. +INLINE uint64_t round_down_to_power_of_2(uint64_t x) { + return 1ULL << highest_one(x | 1); +} + +INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } + +INLINE uint32_t counter_high(uint64_t counter) { + return (uint32_t)(counter >> 32); +} + +INLINE uint32_t load32(const void *src) { + const uint8_t *p = (const uint8_t *)src; + return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | + ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); +} + +INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], + uint32_t key_words[8]) { + key_words[0] = load32(&key[0 * 4]); + key_words[1] = load32(&key[1 * 4]); + key_words[2] = load32(&key[2 * 4]); + key_words[3] = load32(&key[3 * 4]); + key_words[4] = load32(&key[4 * 4]); + key_words[5] = load32(&key[5 * 4]); + key_words[6] = load32(&key[6 * 4]); + key_words[7] = load32(&key[7 * 4]); +} + +INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN], + uint32_t block_words[16]) { + for (size_t i = 0; i < 16; i++) { + block_words[i] = load32(&block[i * 4]); + } +} + +INLINE void store32(void *dst, uint32_t w) { + uint8_t *p = (uint8_t *)dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +} + +INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { + store32(&bytes_out[0 * 4], cv_words[0]); + store32(&bytes_out[1 * 4], cv_words[1]); + store32(&bytes_out[2 * 4], cv_words[2]); + store32(&bytes_out[3 * 4], cv_words[3]); + store32(&bytes_out[4 * 4], cv_words[4]); + store32(&bytes_out[5 * 4], cv_words[5]); + store32(&bytes_out[6 * 4], cv_words[6]); + store32(&bytes_out[7 * 4], cv_words[7]); +} + +void blake3_compress_in_place(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]); + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks); + +void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +size_t blake3_simd_degree(void); + +BLAKE3_PRIVATE size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb); + +#if defined(BLAKE3_USE_TBB) +BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb( + // shared params + const uint32_t key[8], uint8_t flags, bool use_tbb, + // left-hand side params + const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter, + uint8_t *l_cvs, size_t *l_n, + // right-hand side params + const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter, + uint8_t *r_cvs, size_t *r_n) NOEXCEPT; +#endif + +// Declarations for implementation-specific functions. +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +#if defined(IS_X86) +#if !defined(BLAKE3_NO_SSE2) +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_SSE41) +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_AVX2) +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_AVX512) +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +#if !defined(_WIN32) && !defined(__CYGWIN__) +void blake3_xof_many_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t* out, size_t outblocks); +#endif +#endif +#endif + +#if BLAKE3_USE_NEON == 1 +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* BLAKE3_IMPL_H */ diff --git a/tooling/blake3-second-source/reference-impl/upstream/blake3_portable.c b/tooling/blake3-second-source/reference-impl/upstream/blake3_portable.c new file mode 100644 index 000000000..062dd1b47 --- /dev/null +++ b/tooling/blake3-second-source/reference-impl/upstream/blake3_portable.c @@ -0,0 +1,160 @@ +#include "blake3_impl.h" +#include + +INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); +} + +INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, + uint32_t x, uint32_t y) { + state[a] = state[a] + state[b] + x; + state[d] = rotr32(state[d] ^ state[a], 16); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 12); + state[a] = state[a] + state[b] + y; + state[d] = rotr32(state[d] ^ state[a], 8); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 7); +} + +INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { + // Select the message schedule based on the round. + const uint8_t *schedule = MSG_SCHEDULE[round]; + + // Mix the columns. + g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); + g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); + g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); + g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); + + // Mix the rows. + g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); + g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); + g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); + g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); +} + +INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + uint32_t block_words[16]; + block_words[0] = load32(block + 4 * 0); + block_words[1] = load32(block + 4 * 1); + block_words[2] = load32(block + 4 * 2); + block_words[3] = load32(block + 4 * 3); + block_words[4] = load32(block + 4 * 4); + block_words[5] = load32(block + 4 * 5); + block_words[6] = load32(block + 4 * 6); + block_words[7] = load32(block + 4 * 7); + block_words[8] = load32(block + 4 * 8); + block_words[9] = load32(block + 4 * 9); + block_words[10] = load32(block + 4 * 10); + block_words[11] = load32(block + 4 * 11); + block_words[12] = load32(block + 4 * 12); + block_words[13] = load32(block + 4 * 13); + block_words[14] = load32(block + 4 * 14); + block_words[15] = load32(block + 4 * 15); + + state[0] = cv[0]; + state[1] = cv[1]; + state[2] = cv[2]; + state[3] = cv[3]; + state[4] = cv[4]; + state[5] = cv[5]; + state[6] = cv[6]; + state[7] = cv[7]; + state[8] = IV[0]; + state[9] = IV[1]; + state[10] = IV[2]; + state[11] = IV[3]; + state[12] = counter_low(counter); + state[13] = counter_high(counter); + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + + round_fn(state, &block_words[0], 0); + round_fn(state, &block_words[0], 1); + round_fn(state, &block_words[0], 2); + round_fn(state, &block_words[0], 3); + round_fn(state, &block_words[0], 4); + round_fn(state, &block_words[0], 5); + round_fn(state, &block_words[0], 6); +} + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + cv[0] = state[0] ^ state[8]; + cv[1] = state[1] ^ state[9]; + cv[2] = state[2] ^ state[10]; + cv[3] = state[3] ^ state[11]; + cv[4] = state[4] ^ state[12]; + cv[5] = state[5] ^ state[13]; + cv[6] = state[6] ^ state[14]; + cv[7] = state[7] ^ state[15]; +} + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + + store32(&out[0 * 4], state[0] ^ state[8]); + store32(&out[1 * 4], state[1] ^ state[9]); + store32(&out[2 * 4], state[2] ^ state[10]); + store32(&out[3 * 4], state[3] ^ state[11]); + store32(&out[4 * 4], state[4] ^ state[12]); + store32(&out[5 * 4], state[5] ^ state[13]); + store32(&out[6 * 4], state[6] ^ state[14]); + store32(&out[7 * 4], state[7] ^ state[15]); + store32(&out[8 * 4], state[8] ^ cv[0]); + store32(&out[9 * 4], state[9] ^ cv[1]); + store32(&out[10 * 4], state[10] ^ cv[2]); + store32(&out[11 * 4], state[11] ^ cv[3]); + store32(&out[12 * 4], state[12] ^ cv[4]); + store32(&out[13 * 4], state[13] ^ cv[5]); + store32(&out[14 * 4], state[14] ^ cv[6]); + store32(&out[15 * 4], state[15] ^ cv[7]); +} + +INLINE void hash_one_portable(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + store_cv_words(out, cv); +} + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs > 0) { + hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/tooling/ethrex-block-converter/README.md b/tooling/ethrex-block-converter/README.md index 1d14d6c9b..d1fd92441 100644 --- a/tooling/ethrex-block-converter/README.md +++ b/tooling/ethrex-block-converter/README.md @@ -21,7 +21,7 @@ which rev it was taken at; the synthetic column has not been re-measured since t | gas used | 420,000 | **2,428,684** | | transactions | 20 (all plain transfers) | 29 (real mix) | | serialized size | 32,766 B | 1,110,183 B | -| cycles | 8,734,622 | **30,498,818** | +| cycles | 8,734,622 | **20,296,269** | | keccak / ecsm calls | 411 / 80 | **10,659** / 116 | | keccaks per ECSM call | 5.1 | **92** | @@ -376,7 +376,13 @@ the same epoch 2^22: **45,074,552 cycles** (−11.24%), **142.37 s** CPU prove ( **936.7 MB** proof (−12.22%), peak RSS flat at ~48 GB. At the currently pinned `797df554` only the cycle count has been re-measured — -**30,498,818 cycles**, 10,659 keccak, 116 ecsm. Prove time, proof size and the GPU column +**20,296,269 cycles**, 10,659 keccak, 116 ecsm. Before that: 21,217,647 with the +ethrex crates unpatched (the guest builds them from a fork whose jump destinations are +a bitmap and whose memory hot path skips checks its callers made), 23,935,913 before main's guest-side changes were merged (the copy +accelerator's `memcpy`, `memmove` and `memset` stubs among them), and 30,498,818 before +the guest moved to fat LTO over one +codegen unit, the runtime's own `memcpy` and the leaner Keccak final block. Prove time, +proof size and the GPU column have no counterpart at this rev yet; re-run `/bench` rather than reading the rows below as current.